From 82522e2f611fe5f6dbab566110f57b3b373c878d Mon Sep 17 00:00:00 2001 From: "liang_chaoming@huawei.com" Date: Mon, 19 Oct 2020 20:22:23 +0800 Subject: [PATCH] =?UTF-8?q?[add]=E4=B8=8A=E4=BC=A0=E8=AE=AD=E7=BB=83benchm?= =?UTF-8?q?ark=20by=20z00560161?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...N V100R020C10 benchmark工具用户指南 01.pdf | Bin 0 -> 854207 bytes train/README.md | 50 + .../DeepMar/pytorch/README.md | 40 + .../DeepMar/pytorch/code/baseline/__init__.py | 0 .../pytorch/code/baseline/dataset/Dataset.py | 81 + .../pytorch/code/baseline/dataset/__init__.py | 0 .../code/baseline/dataset/add_transforms.py | 65 + .../pytorch/code/baseline/model/DeepMAR.py | 81 + .../pytorch/code/baseline/model/__init__.py | 0 .../pytorch/code/baseline/model/resnet.py | 217 + .../pytorch/code/baseline/utils/__init__.py | 0 .../pytorch/code/baseline/utils/evaluate.py | 105 + .../pytorch/code/baseline/utils/utils.py | 347 + .../pytorch/code/dataset/demo/.gitsave | 0 .../pytorch/code/dataset/demo/demo_image.png | Bin 0 -> 8645 bytes .../pytorch/code/dataset/pa100k/.gitsave | 0 .../pytorch/code/dataset/peta/.gitsave | 0 .../DeepMar/pytorch/code/dataset/rap/.gitsave | 0 .../pytorch/code/dataset/rap2/.gitsave | 0 .../DeepMar/pytorch/code/demo.py | 141 + .../pytorch/code/train_deepmar_resnet50.py | 483 + .../pytorch/code/train_deepmar_resnet50_8p.py | 587 + .../DeepMar/pytorch/code/transform_pa100k.py | 96 + .../DeepMar/pytorch/code/transform_peta.py | 84 + .../DeepMar/pytorch/code/transform_rap.py | 76 + .../DeepMar/pytorch/code/transform_rap2.py | 86 + .../DeepMar/pytorch/config/set_env_b023.sh | 31 + .../DeepMar/pytorch/scripts/run.sh | 62 + .../DeepMar/pytorch/scripts/train.sh | 173 + .../DenseNet121/__init__.py | 0 .../DenseNet121/pytorch/README.md | 25 + .../DenseNet121/pytorch/__init__.py | 0 .../pytorch/code/densenet121_1p_main.py | 515 + .../pytorch/code/densenet121_8p_main.py | 538 + .../pytorch/code/densenet_0_2_2.py | 225 + .../pytorch/code/densenet_0_5_0.py | 279 + .../DenseNet121/pytorch/code/eval.sh | 22 + .../tools/cpu_adapter_file/densenet.py.cpu | 275 + .../tools/cpu_adapter_file/densenet.py.npu | 275 + .../pytorch/code/tools/densenet_print.py | 300 + .../pytorch/code/tools/npu_config/1p.json | 32 + .../pytorch/code/tools/npu_config/2p.json | 44 + .../pytorch/code/tools/npu_config/4p.json | 65 + .../pytorch/code/tools/npu_config/8p.json | 109 + .../code/tools/profile/cpu/net_show_cpu.py | 40 + .../pytorch/code/tools/profile/cpu/run_cpu.sh | 20 + .../code/tools/profile/npu/net_show_npu.py | 51 + .../DenseNet121/pytorch/config/1p.json | 14 + .../DenseNet121/pytorch/config/__init__.py | 0 .../pytorch/config/hccl_sample.json | 9 + .../DenseNet121/pytorch/config/npu_set_env.sh | 52 + .../DenseNet121/pytorch/config/set_env.sh | 9 + .../pytorch/config/set_env_b020.sh | 21 + .../pytorch/config/set_env_b023.sh | 31 + .../DenseNet121/pytorch/scripts/eval.sh | 22 + .../DenseNet121/pytorch/scripts/run.sh | 62 + .../DenseNet121/pytorch/scripts/train.sh | 141 + .../DenseNet121/tensorflow/README.md | 46 + .../tensorflow/code/create_session.py | 22 + .../tensorflow/code/data_loader.py | 133 + .../DenseNet121/tensorflow/code/densenet.py | 158 + .../tensorflow/code/hyper_param.py | 44 + .../DenseNet121/tensorflow/code/layers.py | 25 + .../DenseNet121/tensorflow/code/logger.py | 92 + .../DenseNet121/tensorflow/code/model.py | 72 + .../tensorflow/code/preprocessing.py | 72 + .../DenseNet121/tensorflow/code/train.py | 140 + .../tensorflow/code/train_helper.py | 22 + .../DenseNet121/tensorflow/code/trainer.py | 128 + .../DenseNet121/tensorflow/config/1p.json | 23 + .../DenseNet121/tensorflow/config/8p.json | 51 + .../tensorflow/config/hccl_sample.json | 9 + .../tensorflow/config/npu_set_env.sh | 36 + .../DenseNet121/tensorflow/scripts/run.sh | 70 + .../DenseNet121/tensorflow/scripts/train.sh | 97 + .../EfficientNet/pytorch/README.md | 25 + .../EfficientNet/pytorch/code/LICENSE | 202 + .../EfficientNet/pytorch/code/README.md | 253 + .../EfficientNet/pytorch/code/README_NPU.md | 45 + .../code/efficientnet_pytorch/__init__.py | 12 + .../code/efficientnet_pytorch/auto_augment.py | 817 + .../code/efficientnet_pytorch/model.py | 432 + .../code/efficientnet_pytorch/npu_info.py | 7 + .../code/efficientnet_pytorch/rmsprop_tf.py | 122 + .../code/efficientnet_pytorch/utils.py | 624 + .../pytorch/code/examples/imagenet/README.md | 23 + .../code/examples/imagenet/data/README.md | 5 + .../pytorch/code/examples/imagenet/main.py | 531 + .../pytorch/code/examples/simple/check.ipynb | 177 + .../code/examples/simple/example.ipynb | 144 + .../code/examples/simple/labels_map.txt | 1 + .../EfficientNet/pytorch/code/hubconf.py | 43 + .../EfficientNet/pytorch/code/npu_1p.sh | 9 + .../EfficientNet/pytorch/code/setup.py | 123 + .../pytorch/code/tests/test_model.py | 124 + .../pytorch/config/set_env_b023.sh | 31 + .../EfficientNet/pytorch/scripts/run.sh | 62 + .../EfficientNet/pytorch/scripts/train.sh | 132 + .../MobileNet/__init__.py | 0 .../MobileNet/download_dataset.sh | 1 + .../MobileNet/pytorch/README.md | 25 + .../MobileNet/pytorch/code/1p/README.md | 1 + .../MobileNet/pytorch/code/1p/hook.py | 29 + .../MobileNet/pytorch/code/1p/main_apex.py | 498 + .../MobileNet/pytorch/code/1p/mobilenet.py | 179 + .../MobileNet/pytorch/code/1p/profiling.json | 18 + .../MobileNet/pytorch/code/8p/README.md | 1 + .../MobileNet/pytorch/code/8p/eval.sh | 22 + .../MobileNet/pytorch/code/8p/hook.py | 29 + .../MobileNet/pytorch/code/8p/main_apex.py | 556 + .../MobileNet/pytorch/code/8p/mobilenet.py | 179 + .../pytorch/code/8p/mobilenetv2_8p_main.py | 638 + .../code/8p/mobilenetv2_8p_main_anycard.py | 663 + .../code/8p/multi_epochs_dataloader.py | 31 + .../MobileNet/pytorch/code/8p/profiling.json | 18 + .../MobileNet/pytorch/code/8p/resume1p.sh | 19 + .../MobileNet/pytorch/code/8p/resume8p.sh | 27 + .../MobileNet/pytorch/code/8p/run1p.sh | 18 + .../MobileNet/pytorch/code/8p/run8p.sh | 26 + .../MobileNet/pytorch/code/8p/set_env_b020.sh | 17 + .../MobileNet/pytorch/code/8p/set_env_b023.sh | 18 + .../MobileNet/pytorch/config/npu_set_env.sh | 12 + .../MobileNet/pytorch/config/set_env_b023.sh | 31 + .../MobileNet/pytorch/scripts/run.sh | 60 + .../MobileNet/pytorch/scripts/train.sh | 146 + .../MobileNet/tensorflow/README.md | 47 + .../MobileNet/tensorflow/__init__.py | 0 .../MobileNet/tensorflow/code/README.md | 211 + .../MobileNet/tensorflow/code/__init__.py | 0 .../code/dataloader/data_provider.py | 240 + .../tensorflow/code/datasets/__init__.py | 1 + .../code/datasets/build_imagenet_data.py | 705 + .../tensorflow/code/datasets/cifar10.py | 100 + .../code/datasets/dataset_factory.py | 59 + .../tensorflow/code/datasets/dataset_utils.py | 240 + .../datasets/download_and_convert_cifar10.py | 198 + .../datasets/download_and_convert_flowers.py | 211 + .../datasets/download_and_convert_imagenet.sh | 103 + .../datasets/download_and_convert_mnist.py | 221 + .../download_and_convert_visualwakewords.py | 158 + ...ownload_and_convert_visualwakewords_lib.py | 286 + .../code/datasets/download_imagenet.sh | 99 + .../tensorflow/code/datasets/flowers.py | 99 + .../tensorflow/code/datasets/imagenet.py | 199 + ...imagenet_2012_validation_synset_labels.txt | 50000 ++++++++++++++++ .../datasets/imagenet_lsvrc_2015_synsets.txt | 1000 + .../code/datasets/imagenet_metadata.txt | 21842 +++++++ .../tensorflow/code/datasets/mnist.py | 99 + .../preprocess_imagenet_validation_data.py | 83 + .../code/datasets/process_bounding_boxes.py | 253 + .../code/datasets/visualwakewords.py | 129 + .../tensorflow/code/deployment/__init__.py | 1 + .../code/deployment/model_deploy.py | 677 + .../code/deployment/model_deploy_test.py | 574 + .../code/download_and_convert_data.py | 94 + .../MobileNet/tensorflow/code/env.py | 182 + .../tensorflow/code/estimator_impl.py | 133 + .../tensorflow/code/eval_image_classifier.py | 174 + .../code/eval_image_classifier_mobilenet.py | 181 + .../tensorflow/code/export_inference_graph.py | 164 + .../code/export_inference_graph_test.py | 44 + .../MobileNet/tensorflow/code/gpu_helper.py | 44 + .../MobileNet/tensorflow/code/logger.py | 86 + .../tensorflow/code/nets/__init__.py | 1 + .../MobileNet/tensorflow/code/nets/alexnet.py | 148 + .../tensorflow/code/nets/alexnet_test.py | 181 + .../tensorflow/code/nets/cifarnet.py | 123 + .../tensorflow/code/nets/cyclegan.py | 280 + .../tensorflow/code/nets/cyclegan_test.py | 110 + .../MobileNet/tensorflow/code/nets/dcgan.py | 205 + .../tensorflow/code/nets/dcgan_test.py | 121 + .../MobileNet/tensorflow/code/nets/i3d.py | 181 + .../tensorflow/code/nets/i3d_test.py | 149 + .../tensorflow/code/nets/i3d_utils.py | 289 + .../tensorflow/code/nets/inception.py | 37 + .../code/nets/inception_resnet_v2.py | 408 + .../code/nets/inception_resnet_v2_test.py | 338 + .../tensorflow/code/nets/inception_utils.py | 84 + .../tensorflow/code/nets/inception_v1.py | 347 + .../tensorflow/code/nets/inception_v1_test.py | 300 + .../tensorflow/code/nets/inception_v2.py | 596 + .../tensorflow/code/nets/inception_v2_test.py | 412 + .../tensorflow/code/nets/inception_v3.py | 585 + .../tensorflow/code/nets/inception_v3_test.py | 350 + .../tensorflow/code/nets/inception_v4.py | 347 + .../tensorflow/code/nets/inception_v4_test.py | 287 + .../MobileNet/tensorflow/code/nets/lenet.py | 98 + .../code/nets/mobilenet/.idea/encodings.xml | 4 + .../code/nets/mobilenet/.idea/misc.xml | 4 + .../code/nets/mobilenet/.idea/mobilenet.iml | 8 + .../code/nets/mobilenet/.idea/modules.xml | 8 + .../tensorflow/code/nets/mobilenet/README.md | 166 + .../code/nets/mobilenet/__init__.py | 0 .../code/nets/mobilenet/conv_blocks.py | 475 + .../nets/mobilenet/g3doc/edgetpu_latency.png | Bin 0 -> 48273 bytes .../nets/mobilenet/g3doc/latency_pixel1.png | Bin 0 -> 74345 bytes .../mobilenet/g3doc/madds_top1_accuracy.png | Bin 0 -> 81734 bytes .../code/nets/mobilenet/mobilenet.py | 501 + .../nets/mobilenet/mobilenet_example.ipynb | 445 + .../code/nets/mobilenet/mobilenet_v2.py | 249 + .../code/nets/mobilenet/mobilenet_v2_test.py | 219 + .../code/nets/mobilenet/mobilenet_v3.py | 405 + .../code/nets/mobilenet/mobilenet_v3_test.py | 82 + .../tensorflow/code/nets/mobilenet_v1.md | 136 + .../tensorflow/code/nets/mobilenet_v1.py | 482 + .../tensorflow/code/nets/mobilenet_v1_eval.py | 157 + .../tensorflow/code/nets/mobilenet_v1_test.py | 537 + .../code/nets/mobilenet_v1_train.py | 214 + .../tensorflow/code/nets/nasnet/README.md | 64 + .../tensorflow/code/nets/nasnet/__init__.py | 1 + .../tensorflow/code/nets/nasnet/nasnet.py | 554 + .../code/nets/nasnet/nasnet_test.py | 413 + .../code/nets/nasnet/nasnet_utils.py | 534 + .../code/nets/nasnet/nasnet_utils_test.py | 62 + .../tensorflow/code/nets/nasnet/pnasnet.py | 285 + .../code/nets/nasnet/pnasnet_test.py | 257 + .../tensorflow/code/nets/nets_factory.py | 172 + .../tensorflow/code/nets/nets_factory_test.py | 78 + .../tensorflow/code/nets/overfeat.py | 139 + .../tensorflow/code/nets/overfeat_test.py | 179 + .../MobileNet/tensorflow/code/nets/pix2pix.py | 297 + .../tensorflow/code/nets/pix2pix_test.py | 157 + .../code/nets/post_training_quantization.py | 181 + .../tensorflow/code/nets/resnet_utils.py | 278 + .../tensorflow/code/nets/resnet_v1.py | 406 + .../tensorflow/code/nets/resnet_v1_test.py | 630 + .../tensorflow/code/nets/resnet_v2.py | 340 + .../tensorflow/code/nets/resnet_v2_test.py | 474 + .../MobileNet/tensorflow/code/nets/s3dg.py | 603 + .../tensorflow/code/nets/s3dg_test.py | 150 + .../MobileNet/tensorflow/code/nets/vgg.py | 317 + .../tensorflow/code/nets/vgg_test.py | 584 + .../tensorflow/code/preprocessing/__init__.py | 1 + .../preprocessing/cifarnet_preprocessing.py | 150 + .../preprocessing/inception_preprocessing.py | 349 + .../code/preprocessing/lenet_preprocessing.py | 53 + .../preprocessing/preprocessing_factory.py | 98 + .../code/preprocessing/vgg_preprocessing.py | 383 + .../MobileNet/tensorflow/code/setup.py | 27 + .../MobileNet/tensorflow/code/train.py | 300 + .../MobileNet/tensorflow/config/__init__.py | 0 .../tensorflow/config/hccl_sample.json | 9 + .../tensorflow/config/npu_set_env.sh | 30 + .../MobileNet/tensorflow/scripts/eval.sh | 20 + .../MobileNet/tensorflow/scripts/run.sh | 66 + .../MobileNet/tensorflow/scripts/train.sh | 161 + .../MobileNet/verify_dataset.sh | 1 + .../ResNet101/tensorflow/README.md | 37 + .../tensorflow/code/official/LICENSE | 203 + .../tensorflow/code/official/README-GPU.md | 151 + .../tensorflow/code/official/README-TPU.md | 25 + .../tensorflow/code/official/README.md | 79 + .../tensorflow/code/official/__init__.py | 0 .../code/official/benchmark/__init__.py | 0 .../official/benchmark/benchmark_uploader.py | 157 + .../benchmark/benchmark_uploader_main.py | 66 + .../benchmark/benchmark_uploader_test.py | 123 + .../official/benchmark/benchmark_wrappers.py | 97 + .../code/official/benchmark/bert_benchmark.py | 354 + .../benchmark/bert_benchmark_utils.py | 126 + .../benchmark/bert_squad_benchmark.py | 654 + .../datastore/schema/benchmark_metric.json | 56 + .../datastore/schema/benchmark_run.json | 368 + .../schema/benchmark_run_status.json | 14 + .../official/benchmark/keras_benchmark.py | 98 + .../benchmark/keras_cifar_benchmark.py | 402 + .../benchmark/keras_imagenet_benchmark.py | 1685 + .../official/benchmark/models/__init__.py | 0 .../benchmark/models/resnet_cifar_main.py | 287 + .../benchmark/models/resnet_cifar_model.py | 262 + .../benchmark/models/resnet_cifar_test.py | 187 + .../benchmark/models/shakespeare/README.md | 31 + .../benchmark/models/shakespeare/__init__.py | 1 + .../models/shakespeare/shakespeare_main.py | 316 + .../benchmark/models/synthetic_util.py | 129 + .../official/benchmark/ncf_keras_benchmark.py | 457 + .../official/benchmark/perfzero_benchmark.py | 91 + .../resnet_ctl_imagenet_benchmark.py | 412 + .../official/benchmark/retinanet_benchmark.py | 294 + .../benchmark/shakespeare_benchmark.py | 359 + .../benchmark/tfhub_memory_usage_benchmark.py | 69 + .../benchmark/transformer_benchmark.py | 681 + .../official/benchmark/xlnet_benchmark.py | 216 + .../tensorflow/code/official/colab/bert.ipynb | 383 + .../code/official/modeling/__init__.py | 0 .../official/modeling/activations/__init__.py | 19 + .../official/modeling/activations/gelu.py | 40 + .../modeling/activations/gelu_test.py | 38 + .../official/modeling/activations/swish.py | 75 + .../modeling/activations/swish_test.py | 49 + .../official/modeling/hyperparams/__init__.py | 0 .../modeling/hyperparams/base_config.py | 323 + .../modeling/hyperparams/base_config_test.py | 299 + .../modeling/hyperparams/params_dict.py | 410 + .../modeling/hyperparams/params_dict_test.py | 322 + .../code/official/modeling/performance.py | 56 + .../code/official/modeling/tf_utils.py | 175 + .../official/modeling/training/__init__.py | 0 .../modeling/training/distributed_executor.py | 759 + .../tensorflow/code/official/nlp/README.md | 19 + .../tensorflow/code/official/nlp/__init__.py | 0 .../code/official/nlp/albert/README.md | 332 + .../code/official/nlp/albert/__init__.py | 0 .../code/official/nlp/albert/configs.py | 61 + .../nlp/albert/export_albert_tfhub.py | 88 + .../nlp/albert/export_albert_tfhub_test.py | 89 + .../official/nlp/albert/run_classifier.py | 69 + .../code/official/nlp/albert/run_squad.py | 139 + ...tf2_albert_encoder_checkpoint_converter.py | 132 + .../code/official/nlp/bert/README.md | 350 + .../code/official/nlp/bert/__init__.py | 1 + .../code/official/nlp/bert/bert_cloud_tpu.md | 110 + .../code/official/nlp/bert/bert_models.py | 349 + .../code/official/nlp/bert/common_flags.py | 118 + .../code/official/nlp/bert/configs.py | 105 + .../code/official/nlp/bert/export_tfhub.py | 86 + .../official/nlp/bert/export_tfhub_test.py | 87 + .../code/official/nlp/bert/input_pipeline.py | 231 + .../official/nlp/bert/model_saving_utils.py | 101 + .../official/nlp/bert/model_training_utils.py | 491 + .../nlp/bert/model_training_utils_test.py | 236 + .../code/official/nlp/bert/run_classifier.py | 432 + .../code/official/nlp/bert/run_pretraining.py | 187 + .../code/official/nlp/bert/run_squad.py | 149 + .../official/nlp/bert/run_squad_helper.py | 432 + .../official/nlp/bert/squad_evaluate_v1_1.py | 108 + .../official/nlp/bert/squad_evaluate_v2_0.py | 252 + .../nlp/bert/tf1_checkpoint_converter_lib.py | 195 + .../bert/tf2_encoder_checkpoint_converter.py | 108 + .../code/official/nlp/bert/tokenization.py | 545 + .../official/nlp/bert/tokenization_test.py | 160 + .../code/official/nlp/data/__init__.py | 0 .../official/nlp/data/classifier_data_lib.py | 676 + .../nlp/data/create_finetuning_data.py | 203 + .../nlp/data/create_pretraining_data.py | 486 + .../code/official/nlp/data/squad_lib.py | 880 + .../code/official/nlp/data/squad_lib_sp.py | 890 + .../code/official/nlp/modeling/README.md | 43 + .../code/official/nlp/modeling/__init__.py | 1 + .../official/nlp/modeling/layers/README.md | 29 + .../official/nlp/modeling/layers/__init__.py | 23 + .../official/nlp/modeling/layers/attention.py | 264 + .../nlp/modeling/layers/attention_test.py | 157 + .../nlp/modeling/layers/dense_einsum.py | 180 + .../nlp/modeling/layers/dense_einsum_test.py | 123 + .../nlp/modeling/layers/masked_softmax.py | 61 + .../modeling/layers/masked_softmax_test.py | 88 + .../modeling/layers/on_device_embedding.py | 92 + .../layers/on_device_embedding_test.py | 193 + .../nlp/modeling/layers/position_embedding.py | 120 + .../layers/position_embedding_test.py | 103 + .../modeling/layers/self_attention_mask.py | 63 + .../nlp/modeling/layers/transformer.py | 229 + .../modeling/layers/transformer_scaffold.py | 238 + .../layers/transformer_scaffold_test.py | 350 + .../nlp/modeling/layers/transformer_test.py | 192 + .../code/official/nlp/modeling/layers/util.py | 51 + .../official/nlp/modeling/losses/README.md | 9 + .../official/nlp/modeling/losses/__init__.py | 17 + ...eighted_sparse_categorical_crossentropy.py | 106 + ...ed_sparse_categorical_crossentropy_test.py | 381 + .../official/nlp/modeling/models/README.md | 17 + .../official/nlp/modeling/models/__init__.py | 18 + .../nlp/modeling/models/bert_classifier.py | 91 + .../modeling/models/bert_classifier_test.py | 105 + .../nlp/modeling/models/bert_pretrainer.py | 125 + .../modeling/models/bert_pretrainer_test.py | 111 + .../nlp/modeling/models/bert_span_labeler.py | 97 + .../modeling/models/bert_span_labeler_test.py | 124 + .../official/nlp/modeling/networks/README.md | 24 + .../nlp/modeling/networks/__init__.py | 21 + .../networks/albert_transformer_encoder.py | 191 + .../albert_transformer_encoder_test.py | 174 + .../nlp/modeling/networks/classification.py | 86 + .../modeling/networks/classification_test.py | 179 + .../nlp/modeling/networks/encoder_scaffold.py | 245 + .../networks/encoder_scaffold_test.py | 629 + .../nlp/modeling/networks/masked_lm.py | 186 + .../nlp/modeling/networks/masked_lm_test.py | 227 + .../nlp/modeling/networks/span_labeling.py | 95 + .../modeling/networks/span_labeling_test.py | 174 + .../modeling/networks/transformer_encoder.py | 191 + .../networks/transformer_encoder_test.py | 203 + .../code/official/nlp/nhnet/README.md | 167 + .../code/official/nlp/nhnet/__init__.py | 0 .../code/official/nlp/nhnet/configs.py | 107 + .../code/official/nlp/nhnet/configs_test.py | 121 + .../code/official/nlp/nhnet/decoder.py | 526 + .../code/official/nlp/nhnet/decoder_test.py | 182 + .../code/official/nlp/nhnet/evaluation.py | 185 + .../code/official/nlp/nhnet/input_pipeline.py | 254 + .../code/official/nlp/nhnet/models.py | 590 + .../nlp/nhnet/multi_channel_attention.py | 153 + .../nlp/nhnet/multi_channel_attention_test.py | 55 + .../code/official/nlp/nhnet/optimizer.py | 82 + .../official/nlp/nhnet/raw_data_process.py | 91 + .../official/nlp/nhnet/raw_data_processor.py | 228 + .../domain_0.com/url_000.html | 3 + .../domain_0.com/url_000.json | 5 + .../domain_1.com/url_001.html | 3 + .../domain_1.com/url_001.json | 5 + .../official/nlp/nhnet/testdata/stories.json | 29 + .../official/nlp/nhnet/testdata/vocab.txt | 23 + .../code/official/nlp/nhnet/trainer.py | 238 + .../code/official/nlp/nhnet/trainer_test.py | 100 + .../code/official/nlp/nhnet/utils.py | 91 + .../code/official/nlp/optimization.py | 227 + .../code/official/nlp/transformer/README.md | 218 + .../code/official/nlp/transformer/__init__.py | 0 .../nlp/transformer/attention_layer.py | 159 + .../official/nlp/transformer/beam_search.py | 138 + .../nlp/transformer/beam_search_v1.py | 675 + .../nlp/transformer/beam_search_v1_test.py | 101 + .../official/nlp/transformer/compute_bleu.py | 148 + .../nlp/transformer/compute_bleu_test.py | 64 + .../official/nlp/transformer/data_download.py | 439 + .../official/nlp/transformer/data_pipeline.py | 317 + .../nlp/transformer/embedding_layer.py | 103 + .../official/nlp/transformer/ffn_layer.py | 77 + .../code/official/nlp/transformer/metrics.py | 183 + .../code/official/nlp/transformer/misc.py | 296 + .../official/nlp/transformer/model_params.py | 96 + .../official/nlp/transformer/model_utils.py | 123 + .../nlp/transformer/model_utils_test.py | 62 + .../official/nlp/transformer/optimizer.py | 137 + .../official/nlp/transformer/transformer.py | 566 + .../transformer/transformer_layers_test.py | 97 + .../nlp/transformer/transformer_main.py | 497 + .../nlp/transformer/transformer_main_test.py | 190 + .../nlp/transformer/transformer_test.py | 68 + .../official/nlp/transformer/translate.py | 199 + .../nlp/transformer/utils/__init__.py | 0 .../official/nlp/transformer/utils/metrics.py | 490 + .../nlp/transformer/utils/tokenizer.py | 660 + .../nlp/transformer/utils/tokenizer_test.py | 204 + .../code/official/nlp/xlnet/README.md | 16 + .../code/official/nlp/xlnet/__init__.py | 1 + .../official/nlp/xlnet/classifier_utils.py | 162 + .../code/official/nlp/xlnet/common_flags.py | 146 + .../code/official/nlp/xlnet/data_utils.py | 828 + .../code/official/nlp/xlnet/optimization.py | 102 + .../xlnet/preprocess_classification_data.py | 457 + .../nlp/xlnet/preprocess_pretrain_data.py | 998 + .../nlp/xlnet/preprocess_squad_data.py | 110 + .../official/nlp/xlnet/preprocess_utils.py | 125 + .../code/official/nlp/xlnet/run_classifier.py | 196 + .../code/official/nlp/xlnet/run_pretrain.py | 156 + .../code/official/nlp/xlnet/run_squad.py | 304 + .../code/official/nlp/xlnet/squad_utils.py | 973 + .../code/official/nlp/xlnet/training_utils.py | 310 + .../code/official/nlp/xlnet/xlnet_config.py | 181 + .../code/official/nlp/xlnet/xlnet_modeling.py | 1290 + .../official/nlp/xlnet/xlnet_modeling_test.py | 52 + .../code/official/pip_package/setup.py | 89 + .../tensorflow/code/official/r1/README.md | 23 + .../tensorflow/code/official/r1/__init__.py | 0 .../code/official/r1/boosted_trees/README.md | 117 + .../official/r1/boosted_trees/__init__.py | 0 .../r1/boosted_trees/data_download.py | 97 + .../official/r1/boosted_trees/train_higgs.py | 297 + .../code/official/r1/mnist/README.md | 91 + .../code/official/r1/mnist/__init__.py | 0 .../code/official/r1/mnist/dataset.py | 117 + .../code/official/r1/mnist/mnist.py | 247 + .../code/official/r1/mnist/mnist_eager.py | 212 + .../official/r1/mnist/mnist_eager_test.py | 95 + .../code/official/r1/mnist/mnist_test.py | 147 + .../code/official/r1/mnist/mnist_tpu.py | 202 + .../tensorflow/code/official/r1/ncf/README.md | 7 + .../official/r1/ncf/ncf_estimator_main.py | 189 + .../code/official/r1/resnet/README.md | 156 + .../code/official/r1/resnet/__init__.py | 0 .../r1/resnet/cifar10_download_and_extract.py | 63 + .../code/official/r1/resnet/cifar10_main.py | 297 + .../code/official/r1/resnet/cifar10_test.py | 185 + .../official/r1/resnet/estimator_benchmark.py | 500 + .../code/official/r1/resnet/gpu_train.sh | 14 + .../code/official/r1/resnet/imagenet_main.py | 471 + .../r1/resnet/imagenet_preprocessing.py | 262 + .../code/official/r1/resnet/imagenet_test.py | 327 + .../code/official/r1/resnet/resnet_model.py | 552 + .../official/r1/resnet/resnet_run_loop.py | 989 + .../code/official/r1/resnet/run_imagenet.sh | 27 + .../r1/resnet/train_accuracy_rewrite.py | 164 + .../code/official/r1/transformer/README.md | 380 + .../code/official/r1/transformer/__init__.py | 0 .../r1/transformer/attention_layer.py | 148 + .../code/official/r1/transformer/dataset.py | 284 + .../r1/transformer/embedding_layer.py | 108 + .../code/official/r1/transformer/ffn_layer.py | 89 + .../code/official/r1/transformer/schedule.py | 130 + .../official/r1/transformer/schedule_test.py | 84 + .../official/r1/transformer/transformer.py | 417 + .../r1/transformer/transformer_main.py | 710 + .../code/official/r1/transformer/translate.py | 237 + .../code/official/r1/utils/__init__.py | 0 .../code/official/r1/utils/data/__init__.py | 0 .../code/official/r1/utils/data/file_io.py | 207 + .../official/r1/utils/data/file_io_test.py | 199 + .../code/official/r1/utils/export.py | 49 + .../code/official/r1/utils/export_test.py | 63 + .../tensorflow/code/official/r1/utils/tpu.py | 116 + .../code/official/r1/utils/tpu_test.py | 108 + .../code/official/r1/wide_deep/README.md | 102 + .../code/official/r1/wide_deep/__init__.py | 0 .../official/r1/wide_deep/census_dataset.py | 205 + .../code/official/r1/wide_deep/census_main.py | 116 + .../official/r1/wide_deep/census_test.csv | 30 + .../code/official/r1/wide_deep/census_test.py | 169 + .../r1/wide_deep/movielens_dataset.py | 165 + .../official/r1/wide_deep/movielens_main.py | 115 + .../official/r1/wide_deep/movielens_test.py | 120 + .../r1/wide_deep/wide_deep_run_loop.py | 133 + .../code/official/recommendation/README.md | 71 + .../code/official/recommendation/__init__.py | 0 .../code/official/recommendation/constants.py | 79 + .../recommendation/create_ncf_data.py | 117 + .../official/recommendation/data_pipeline.py | 959 + .../recommendation/data_preprocessing.py | 243 + .../code/official/recommendation/data_test.py | 358 + .../code/official/recommendation/movielens.py | 309 + .../official/recommendation/ncf_common.py | 331 + .../recommendation/ncf_input_pipeline.py | 184 + .../official/recommendation/ncf_keras_main.py | 562 + .../code/official/recommendation/ncf_test.py | 255 + .../official/recommendation/neumf_model.py | 457 + .../official/recommendation/popen_helper.py | 64 + .../official/recommendation/stat_utils.py | 92 + .../tensorflow/code/official/requirements.txt | 25 + .../code/official/staging/__init__.py | 0 .../official/staging/training/__init__.py | 14 + .../official/staging/training/controller.py | 337 + .../staging/training/controller_test.py | 308 + .../official/staging/training/grad_utils.py | 143 + .../official/staging/training/runnable.py | 79 + .../staging/training/standard_runnable.py | 181 + .../code/official/staging/training/utils.py | 342 + .../code/official/utils/__init__.py | 0 .../code/official/utils/flags/README.md | 97 + .../code/official/utils/flags/__init__.py | 0 .../code/official/utils/flags/_base.py | 173 + .../code/official/utils/flags/_benchmark.py | 109 + .../code/official/utils/flags/_conventions.py | 54 + .../code/official/utils/flags/_device.py | 85 + .../official/utils/flags/_distribution.py | 54 + .../code/official/utils/flags/_misc.py | 50 + .../code/official/utils/flags/_performance.py | 289 + .../code/official/utils/flags/core.py | 133 + .../code/official/utils/flags/flags_test.py | 162 + .../code/official/utils/flags/guidelines.md | 65 + .../code/official/utils/hyperparams_flags.py | 120 + .../code/official/utils/logs/__init__.py | 0 .../code/official/utils/logs/cloud_lib.py | 34 + .../official/utils/logs/cloud_lib_test.py | 48 + .../code/official/utils/logs/guidelines.md | 58 + .../code/official/utils/logs/hooks.py | 148 + .../code/official/utils/logs/hooks_bak.py | 130 + .../code/official/utils/logs/hooks_helper.py | 172 + .../official/utils/logs/hooks_helper_test.py | 73 + .../code/official/utils/logs/hooks_test.py | 159 + .../code/official/utils/logs/logger.py | 423 + .../code/official/utils/logs/logger_test.py | 366 + .../code/official/utils/logs/metric_hook.py | 97 + .../official/utils/logs/metric_hook_test.py | 217 + .../code/official/utils/logs/mlperf_helper.py | 194 + .../code/official/utils/misc/__init__.py | 0 .../official/utils/misc/callstack_sampler.py | 62 + .../official/utils/misc/distribution_utils.py | 205 + .../utils/misc/distribution_utils_test.py | 49 + .../code/official/utils/misc/keras_utils.py | 262 + .../code/official/utils/misc/model_helpers.py | 95 + .../official/utils/misc/model_helpers_test.py | 127 + .../code/official/utils/misc/tpu_lib.py | 34 + .../code/official/utils/testing/__init__.py | 0 .../official/utils/testing/integration.py | 71 + .../code/official/utils/testing/mock_lib.py | 36 + .../code/official/utils/testing/pylint.rcfile | 168 + .../utils/testing/scripts/builds_common.sh | 64 + .../utils/testing/scripts/ci_sanity.sh | 132 + .../utils/testing/scripts/presubmit.sh | 73 + .../code/official/vision/__init__.py | 0 .../code/official/vision/detection/README.md | 244 + .../official/vision/detection/__init__.py | 0 .../vision/detection/configs/__init__.py | 14 + .../vision/detection/configs/base_config.py | 120 + .../vision/detection/configs/factory.py | 33 + .../detection/configs/maskrcnn_config.py | 169 + .../detection/configs/retinanet_config.py | 171 + .../vision/detection/dataloader/__init__.py | 14 + .../vision/detection/dataloader/anchor.py | 292 + .../vision/detection/dataloader/factory.py | 102 + .../detection/dataloader/input_reader.py | 109 + .../detection/dataloader/maskrcnn_parser.py | 385 + .../vision/detection/dataloader/mode_keys.py | 33 + .../detection/dataloader/retinanet_parser.py | 429 + .../detection/dataloader/shapemask_parser.py | 478 + .../dataloader/tf_example_decoder.py | 156 + .../vision/detection/evaluation/__init__.py | 14 + .../detection/evaluation/coco_evaluator.py | 343 + .../vision/detection/evaluation/coco_utils.py | 374 + .../vision/detection/evaluation/factory.py | 35 + .../vision/detection/executor/__init__.py | 14 + .../detection/executor/detection_executor.py | 160 + .../code/official/vision/detection/main.py | 255 + .../vision/detection/modeling/__init__.py | 14 + .../modeling/architecture/__init__.py | 14 + .../modeling/architecture/factory.py | 136 + .../detection/modeling/architecture/fpn.py | 143 + .../detection/modeling/architecture/heads.py | 1108 + .../modeling/architecture/identity.py | 28 + .../detection/modeling/architecture/nn_ops.py | 86 + .../detection/modeling/architecture/resnet.py | 304 + .../vision/detection/modeling/base_model.py | 167 + .../detection/modeling/checkpoint_utils.py | 131 + .../vision/detection/modeling/factory.py | 31 + .../detection/modeling/learning_rates.py | 96 + .../vision/detection/modeling/losses.py | 490 + .../detection/modeling/maskrcnn_model.py | 342 + .../detection/modeling/retinanet_model.py | 170 + .../official/vision/detection/ops/__init__.py | 14 + .../code/official/vision/detection/ops/nms.py | 205 + .../vision/detection/ops/postprocess_ops.py | 412 + .../official/vision/detection/ops/roi_ops.py | 237 + .../vision/detection/ops/sampling_ops.py | 399 + .../detection/ops/spatial_transform_ops.py | 608 + .../vision/detection/utils/__init__.py | 14 + .../detection/utils/autoaugment_utils.py | 25 + .../vision/detection/utils/box_utils.py | 551 + .../vision/detection/utils/class_utils.py | 44 + .../detection/utils/dataloader_utils.py | 40 + .../vision/detection/utils/input_utils.py | 366 + .../vision/detection/utils/mask_utils.py | 192 + .../utils/object_detection/__init__.py | 14 + .../utils/object_detection/argmax_matcher.py | 201 + .../balanced_positive_negative_sampler.py | 274 + .../utils/object_detection/box_coder.py | 151 + .../utils/object_detection/box_list.py | 211 + .../utils/object_detection/box_list_ops.py | 1079 + .../object_detection/faster_rcnn_box_coder.py | 118 + .../utils/object_detection/matcher.py | 243 + .../object_detection/minibatch_sampler.py | 93 + .../detection/utils/object_detection/ops.py | 82 + .../utils/object_detection/preprocessor.py | 525 + .../region_similarity_calculator.py | 143 + .../utils/object_detection/shape_utils.py | 112 + .../utils/object_detection/target_assigner.py | 314 + .../object_detection/visualization_utils.py | 733 + .../vision/image_classification/README.md | 164 + .../vision/image_classification/__init__.py | 0 .../vision/image_classification/augment.py | 999 + .../image_classification/augment_test.py | 143 + .../vision/image_classification/callbacks.py | 258 + .../classifier_trainer.py | 458 + .../classifier_trainer_test.py | 386 + .../image_classification/configs/__init__.py | 14 + .../configs/base_configs.py | 239 + .../image_classification/configs/configs.py | 118 + .../imagenet/efficientnet-b0-gpu.yaml | 53 + .../imagenet/efficientnet-b0-tpu.yaml | 52 + .../imagenet/efficientnet-b1-gpu.yaml | 46 + .../imagenet/efficientnet-b1-tpu.yaml | 51 + .../configs/examples/resnet/imagenet/gpu.yaml | 52 + .../configs/examples/resnet/imagenet/tpu.yaml | 58 + .../image_classification/dataset_factory.py | 497 + .../efficientnet/__init__.py | 0 .../efficientnet/common_modules.py | 117 + .../efficientnet/efficientnet_config.py | 76 + .../efficientnet/efficientnet_model.py | 504 + .../image_classification/learning_rate.py | 120 + .../learning_rate_test.py | 89 + .../vision/image_classification/mnist_main.py | 171 + .../vision/image_classification/mnist_test.py | 90 + .../image_classification/optimizer_factory.py | 383 + .../optimizer_factory_test.py | 114 + .../image_classification/preprocessing.py | 391 + .../image_classification/resnet/README.md | 129 + .../image_classification/resnet/__init__.py | 0 .../resnet/cifar_preprocessing.py | 159 + .../image_classification/resnet/common.py | 398 + .../resnet/imagenet_preprocessing.py | 561 + .../resnet/resnet_config.py | 61 + .../resnet/resnet_ctl_imagenet_main.py | 192 + .../resnet/resnet_imagenet_main.py | 305 + .../resnet/resnet_model.py | 329 + .../resnet/resnet_runnable.py | 222 + .../resnet/tfhub_export.py | 67 + .../vision/image_classification/test_utils.py | 38 + .../tensorflow/config/npu_set_env.sh | 41 + .../ResNet101/tensorflow/scripts/run.sh | 69 + .../ResNet101/tensorflow/scripts/train.sh | 124 + .../image_classification/ResNet50/__init__.py | 0 .../ResNet50/mindspore/code/README.md | 480 + .../ResNet50/mindspore/code/eval.py | 89 + .../mindspore/code/mindspore_hub_conf.py | 25 + .../mindspore/code/src/CrossEntropySmooth.py | 38 + .../ResNet50/mindspore/code/src/config.py | 106 + .../ResNet50/mindspore/code/src/dataset.py | 263 + .../mindspore/code/src/lr_generator.py | 205 + .../ResNet50/mindspore/code/src/resnet.py | 393 + .../ResNet50/mindspore/code/train.py | 191 + .../ResNet50/mindspore/config/npu_set_env.sh | 18 + .../ResNet50/mindspore/scripts/eval.sh | 30 + .../ResNet50/mindspore/scripts/run.sh | 95 + .../ResNet50/mindspore/scripts/train.sh | 134 + .../ResNet50/pytorch/README.md | 26 + .../ResNet50/pytorch/__init__.py | 0 .../image_classification/__init__.py | 20 + .../image_classification/dataloaders.py | 369 + .../image_classification/logger.py | 310 + .../image_classification/mixup.py | 67 + .../image_classification/resnet.py | 370 + .../image_classification/smoothing.py | 91 + .../image_classification/smoothing_tocpu.py | 51 + .../image_classification/training.py | 534 + .../image_classification/utils.py | 106 + .../DistributedResnet50/main-apex-d76-npu.py | 1121 + .../pytorch/code/pytorch-resnet50-apex.py | 609 + .../ResNet50/pytorch/config/npu_set_env.sh | 31 + .../ResNet50/pytorch/scripts/run.sh | 79 + .../ResNet50/pytorch/scripts/train.sh | 138 + .../ResNet50/tensorflow/README.md | 38 + .../ResNet50/tensorflow/__init__.py | 0 .../ResNet50/tensorflow/code/Dockerfile | 36 + .../ResNet50/tensorflow/code/__init__.py | 0 .../tensorflow/code/bin/cloud_docker_init.sh | 47 + .../tensorflow/code/bin/hccl_sample.json | 13 + .../tensorflow/code/bin/main_sample.sh | 18 + .../tensorflow/code/bin/npu_set_env.sh | 28 + .../tensorflow/code/bin/train_sample.sh | 33 + .../ResNet50/tensorflow/code/official/LICENSE | 203 + .../tensorflow/code/official/README-TPU.md | 20 + .../tensorflow/code/official/README.md | 149 + .../tensorflow/code/official/__init__.py | 0 .../code/official/benchmark/__init__.py | 0 .../datastore/schema/benchmark_metric.json | 56 + .../datastore/schema/benchmark_run.json | 368 + .../schema/benchmark_run_status.json | 14 + .../official/benchmark/models/__init__.py | 0 .../benchmark/models/resnet_cifar_main.py | 285 + .../benchmark/models/resnet_cifar_model.py | 262 + .../benchmark/models/resnet_cifar_test.py | 187 + .../benchmark/resnet_tf_r1_benchmark.py | 259 + .../code/official/modeling/__init__.py | 0 .../official/modeling/activations/__init__.py | 19 + .../official/modeling/activations/gelu.py | 40 + .../modeling/activations/gelu_test.py | 38 + .../official/modeling/activations/swish.py | 75 + .../modeling/activations/swish_test.py | 49 + .../official/modeling/hyperparams/__init__.py | 0 .../modeling/hyperparams/base_config.py | 318 + .../modeling/hyperparams/base_config_test.py | 299 + .../modeling/hyperparams/params_dict.py | 410 + .../modeling/hyperparams/params_dict_test.py | 322 + .../official/modeling/model_training_utils.py | 491 + .../modeling/model_training_utils_test.py | 235 + .../code/official/modeling/performance.py | 56 + .../code/official/modeling/tf_utils.py | 175 + .../official/modeling/training/__init__.py | 0 .../modeling/training/distributed_executor.py | 735 + .../code/official/pip_package/setup.py | 88 + .../tensorflow/code/official/r1/README.md | 23 + .../tensorflow/code/official/r1/__init__.py | 0 .../code/official/r1/boosted_trees/README.md | 112 + .../official/r1/boosted_trees/__init__.py | 0 .../r1/boosted_trees/data_download.py | 97 + .../official/r1/boosted_trees/train_higgs.py | 297 + .../code/official/r1/resnet/README.md | 152 + .../code/official/r1/resnet/__init__.py | 0 .../official/r1/resnet/estimator_benchmark.py | 499 + .../code/official/r1/resnet/imagenet_main.py | 433 + .../r1/resnet/imagenet_preprocessing.py | 262 + .../code/official/r1/resnet/imagenet_test.py | 326 + .../code/official/r1/resnet/resnet_model.py | 552 + .../official/r1/resnet/resnet_run_loop.py | 979 + .../r1/resnet/resnet_run_loop_orgin.py | 901 + .../tensorflow/code/official/r1/resnet/slog | 581 + .../code/official/r1/utils/__init__.py | 0 .../code/official/r1/utils/data/__init__.py | 0 .../code/official/r1/utils/data/file_io.py | 207 + .../official/r1/utils/data/file_io_test.py | 199 + .../code/official/r1/utils/export.py | 49 + .../code/official/r1/utils/export_test.py | 63 + .../tensorflow/code/official/r1/utils/tpu.py | 116 + .../code/official/r1/utils/tpu_test.py | 108 + .../tensorflow/code/official/requirements.txt | 24 + .../code/official/utils/__init__.py | 0 .../code/official/utils/flags/README.md | 97 + .../code/official/utils/flags/__init__.py | 0 .../code/official/utils/flags/_base.py | 163 + .../code/official/utils/flags/_benchmark.py | 109 + .../code/official/utils/flags/_conventions.py | 54 + .../code/official/utils/flags/_device.py | 85 + .../official/utils/flags/_distribution.py | 54 + .../code/official/utils/flags/_misc.py | 50 + .../code/official/utils/flags/_performance.py | 289 + .../code/official/utils/flags/core.py | 133 + .../code/official/utils/flags/flags_test.py | 162 + .../code/official/utils/flags/guidelines.md | 65 + .../code/official/utils/hyperparams_flags.py | 119 + .../code/official/utils/logs/__init__.py | 0 .../code/official/utils/logs/cloud_lib.py | 34 + .../official/utils/logs/cloud_lib_test.py | 48 + .../code/official/utils/logs/guidelines.md | 58 + .../code/official/utils/logs/hooks.py | 146 + .../code/official/utils/logs/hooks_helper.py | 172 + .../official/utils/logs/hooks_helper_test.py | 73 + .../code/official/utils/logs/hooks_test.py | 158 + .../code/official/utils/logs/logger.py | 423 + .../code/official/utils/logs/logger_test.py | 365 + .../code/official/utils/logs/metric_hook.py | 97 + .../official/utils/logs/metric_hook_test.py | 217 + .../code/official/utils/logs/mlperf_helper.py | 192 + .../code/official/utils/misc/__init__.py | 0 .../official/utils/misc/callstack_sampler.py | 62 + .../official/utils/misc/distribution_utils.py | 338 + .../utils/misc/distribution_utils_test.py | 65 + .../code/official/utils/misc/keras_utils.py | 262 + .../code/official/utils/misc/model_helpers.py | 93 + .../official/utils/misc/model_helpers_test.py | 127 + .../code/official/utils/misc/tpu_lib.py | 34 + .../code/official/utils/testing/__init__.py | 0 .../utils/testing/benchmark_wrappers.py | 83 + .../official/utils/testing/integration.py | 71 + .../code/official/utils/testing/mock_lib.py | 36 + .../utils/testing/perfzero_benchmark.py | 90 + .../code/official/utils/testing/pylint.rcfile | 168 + .../utils/testing/scripts/builds_common.sh | 64 + .../utils/testing/scripts/ci_sanity.sh | 132 + .../utils/testing/scripts/presubmit.sh | 73 + .../ResNet50/tensorflow/config/npu_set_env.sh | 42 + .../config/resnet_config_16p_npu.py | 18 + .../tensorflow/config/resnet_config_1p_npu.py | 18 + .../tensorflow/config/resnet_config_2p_npu.py | 18 + .../tensorflow/config/resnet_config_4p_npu.py | 18 + .../tensorflow/config/resnet_config_8p_npu.py | 18 + .../ResNet50/tensorflow/scripts/run.sh | 90 + .../ResNet50/tensorflow/scripts/train.sh | 106 + .../ResNext50/__init__.py | 0 .../ResNext50/tensorflow/README.md | 46 + .../code/resnext50_train/configs/__init__.py | 0 .../code/resnext50_train/configs/pid | 1 + .../resnext50_train/configs/res50_32bs_1p.py | 115 + .../resnext50_train/configs/res50_32bs_8p.py | 115 + .../resnext50_train/data_loader/__init__.py | 0 .../data_loader/resnet50/data_loader.py | 236 + .../data_loader/resnet50/preprocessing.py | 152 + .../resnext50_train/hyper_param/__init__.py | 0 .../hyper_param/hyper_param.py | 50 + .../hyper_param/lr_schedule.py | 172 + .../code/resnext50_train/layers/__init__.py | 0 .../code/resnext50_train/layers/layers.py | 23 + .../code/resnext50_train/losses/__init__.py | 0 .../code/resnext50_train/losses/res50_loss.py | 36 + .../code/resnext50_train/mains/clean.sh | 7 + .../code/resnext50_train/mains/res50.py | 141 + .../code/resnext50_train/mains/train_res50.sh | 21 + .../resnext50_train/mains/train_res50_gpu.sh | 4 + .../code/resnext50_train/models/__init__.py | 0 .../models/resnet50/res50_helper.py | 24 + .../models/resnet50/res50_model.py | 222 + .../resnext50_train/models/resnet50/resnet.py | 545 + .../resnext50_train/optimizers/__init__.py | 0 .../resnext50_train/optimizers/optimizer.py | 228 + .../code/resnext50_train/trainers/2q | 172 + .../code/resnext50_train/trainers/__init__.py | 0 .../trainers/gpu_base_trainer.py | 253 + .../resnext50_train/trainers/train_helper.py | 39 + .../code/resnext50_train/utils/__init__.py | 0 .../resnext50_train/utils/create_session.py | 48 + .../code/resnext50_train/utils/logger.py | 103 + .../tensorflow/config/hccl_sample.json | 6 + .../tensorflow/config/main_sample.sh | 18 + .../tensorflow/config/npu_set_env.sh | 41 + .../tensorflow/config/train_sample.sh | 33 + .../ResNext50/tensorflow/scripts/run.sh | 99 + .../ResNext50/tensorflow/scripts/train.sh | 117 + .../Resnet50_HC/__init__.py | 0 .../Resnet50_HC/tensorflow/README.md | 40 + .../code/resnet50_train/configs/__init__.py | 0 .../code/resnet50_train/configs/pid | 1 + .../resnet50_train/configs/res50_256bs_16p.py | 111 + .../resnet50_train/configs/res50_256bs_1p.py | 109 + .../resnet50_train/configs/res50_256bs_2p.py | 109 + .../resnet50_train/configs/res50_256bs_4p.py | 109 + .../resnet50_train/configs/res50_256bs_8p.py | 111 + .../resnet50_train/data_loader/__init__.py | 0 .../data_loader/resnet50/data_loader.py | 237 + .../data_loader/resnet50/preprocessing.py | 160 + .../resnet50_train/hyper_param/__init__.py | 0 .../resnet50_train/hyper_param/hyper_param.py | 50 + .../resnet50_train/hyper_param/lr_schedule.py | 168 + .../hyper_param/lr_schedule_0907_back.py | 169 + .../code/resnet50_train/layers/__init__.py | 0 .../code/resnet50_train/layers/layers.py | 23 + .../code/resnet50_train/losses/__init__.py | 0 .../code/resnet50_train/losses/res50_loss.py | 36 + .../code/resnet50_train/mains/clean.sh | 7 + .../code/resnet50_train/mains/res50.py | 120 + .../code/resnet50_train/mains/train_res50.sh | 21 + .../resnet50_train/mains/train_res50_gpu.sh | 4 + .../code/resnet50_train/models/__init__.py | 0 .../models/resnet50/res50_helper.py | 24 + .../models/resnet50/res50_model.py | 222 + .../models/resnet50/res50_model.py_bak | 143 + .../resnet50_train/models/resnet50/resnet.py | 436 + .../resnet50_train/optimizers/__init__.py | 0 .../resnet50_train/optimizers/optimizer.py | 228 + .../results/res50_baseline/resnet50.log | 3 + .../code/resnet50_train/trainers/__init__.py | 0 .../trainers/gpu_base_trainer.py | 189 + .../resnet50_train/trainers/train_helper.py | 39 + .../code/resnet50_train/utils/__init__.py | 0 .../resnet50_train/utils/create_session.py | 43 + .../code/resnet50_train/utils/logger.py | 87 + .../tensorflow/config/npu_set_env.sh | 41 + .../Resnet50_HC/tensorflow/scripts/run.sh | 86 + .../Resnet50_HC/tensorflow/scripts/train.sh | 109 + .../ShuffleNet/__init__.py | 0 .../ShuffleNet/pytorch/README.md | 35 + .../ShuffleNet/pytorch/code/8p_main.py | 541 + .../ShuffleNet/pytorch/code/8p_main_med.py | 605 + .../ShuffleNet/pytorch/code/README.md | 60 + .../ShuffleNet/pytorch/code/main.py | 649 + .../pytorch/code/models/__init__.py | 1 + .../ShuffleNet/pytorch/code/models/_utils.py | 67 + .../pytorch/code/models/shufflenetv2.py | 208 + .../code/models/shufflenetv2_group_shuffle.py | 256 + .../code/models/shufflenetv2_wock_op_woct.py | 330 + .../models/shufflenetv2_wock_op_woct_8p.py | 328 + .../ShuffleNet/pytorch/code/models/utils.py | 4 + .../pytorch/code/multi_epochs_dataloader.py | 31 + .../ShuffleNet/pytorch/config/npu_set_env.sh | 31 + .../ShuffleNet/pytorch/scripts/run.sh | 72 + .../ShuffleNet/pytorch/scripts/train.sh | 143 + .../image_classification/vgg16/__init__.py | 0 .../vgg16/tensorflow/config/16p.json | 182 + .../vgg16/tensorflow/config/1p.json | 12 + .../vgg16/tensorflow/config/8p.json | 12 + .../tensorflow/config/cloud_docker_init.sh | 48 + .../vgg16/tensorflow/config/hccl_sample.json | 6 + .../vgg16/tensorflow/config/main_sample.sh | 18 + .../vgg16/tensorflow/config/npu_set_env.sh | 40 + .../vgg16/tensorflow/config/train_sample.sh | 33 + .../vgg16/tensorflow/scripts/run.sh | 60 + .../vgg16/tensorflow/scripts/train.sh | 140 + .../vgg16/tensorflow/vgg16/create_session.py | 22 + .../vgg16/tensorflow/vgg16/data_loader.py | 133 + .../vgg16/tensorflow/vgg16/hyper_param.py | 45 + .../vgg16/tensorflow/vgg16/layers.py | 22 + .../vgg16/tensorflow/vgg16/logger.py | 88 + .../vgg16/tensorflow/vgg16/model.py | 71 + .../vgg16/tensorflow/vgg16/preprocessing.py | 73 + .../vgg16/tensorflow/vgg16/train.py | 143 + .../vgg16/tensorflow/vgg16/train_helper.py | 21 + .../vgg16/tensorflow/vgg16/trainer.py | 142 + .../vgg16/tensorflow/vgg16/vgg.py | 63 + .../nlp/Bert-Base/tensorflow/README.md | 56 + .../tensorflow/code/pretrain/__init__.py | 15 + .../code/pretrain/bert_base_config.json | 13 + .../code/pretrain/bert_base_vocab.txt | 21128 +++++++ .../code/pretrain/create_pretraining_data.py | 442 + .../code/pretrain/extract_features.py | 419 + .../tensorflow/code/pretrain/fp16_utils.py | 35 + .../code/pretrain/fused_layer_norm.py | 141 + .../code/pretrain/gpu_environment.py | 36 + .../tensorflow/code/pretrain/modeling.py | 1031 + .../tensorflow/code/pretrain/optimization.py | 439 + .../code/pretrain/run_pretraining.py | 784 + .../tensorflow/code/pretrain/tf_metrics.py | 215 + .../tensorflow/code/pretrain/tokenization.py | 451 + .../tensorflow/code/pretrain/utils.py | 62 + .../nlp/Bert-Base/tensorflow/config/1p.json | 14 + .../nlp/Bert-Base/tensorflow/config/8p.json | 49 + .../config/bert_base_layer12_cn.json | 14 + .../config/bert_base_layer12_en.json | 14 + .../config/bert_base_layer6_cn.json | 14 + .../config/bert_base_layer6_en.json | 14 + .../tensorflow/config/bert_base_vocab.txt | 21128 +++++++ .../tensorflow/config/npu_set_env.sh | 39 + .../nlp/Bert-Base/tensorflow/scripts/run.sh | 67 + .../nlp/Bert-Base/tensorflow/scripts/train.sh | 157 + .../nlp/Bert-Large/tensorflow/README.md | 54 + .../tensorflow/code/bert-Nv/CONTRIBUTING.md | 31 + .../tensorflow/code/bert-Nv/Dockerfile | 31 + .../tensorflow/code/bert-Nv/LICENSE | 202 + .../Bert-Large/tensorflow/code/bert-Nv/NOTICE | 4 + .../tensorflow/code/bert-Nv/README.md | 1155 + .../tensorflow/code/bert-Nv/__init__.py | 15 + .../tensorflow/code/bert-Nv/biobert/README.md | 567 + .../code/bert-Nv/biobert/__init__.py | 0 .../code/bert-Nv/biobert/conlleval.py | 302 + .../code/bert-Nv/biobert/re_eval.py | 51 + .../biobert/scripts/biobert_data_download.sh | 19 + .../biobert_finetune_inference_benchmark.sh | 187 + .../biobert_finetune_train_benchmark.sh | 203 + .../biobert/scripts/ner_bc5cdr-chem.sh | 86 + .../biobert/scripts/ner_bc5cdr-disease.sh | 85 + .../bert-Nv/biobert/scripts/rel_chemprot.sh | 87 + .../bert-Nv/biobert/scripts/run_biobert.sub | 87 + .../run_biobert_finetuning_inference.sh | 122 + .../run_pretraining_pubmed_base_phase_1.sh | 87 + .../run_pretraining_pubmed_base_phase_2.sh | 85 + .../code/bert-Nv/configurations.yml | 206 + .../code/bert-Nv/data/BooksDownloader.py | 26 + .../bert-Nv/data/BookscorpusTextFormatting.py | 32 + .../code/bert-Nv/data/Downloader.py | 120 + .../code/bert-Nv/data/GLUEDownloader.py | 109 + .../data/GooglePretrainedWeightDownloader.py | 158 + .../data/NVIDIAPretrainedWeightDownloader.py | 27 + .../code/bert-Nv/data/PubMedDownloader.py | 93 + .../code/bert-Nv/data/PubMedTextFormatting.py | 44 + .../tensorflow/code/bert-Nv/data/README.md | 32 + .../code/bert-Nv/data/SquadDownloader.py | 54 + .../code/bert-Nv/data/TextSharding.py | 331 + .../code/bert-Nv/data/WikiDownloader.py | 58 + .../bert-Nv/data/WikicorpusTextFormatting.py | 46 + .../tensorflow/code/bert-Nv/data/__init__.py | 12 + .../tensorflow/code/bert-Nv/data/bertPrep.py | 387 + .../create_biobert_datasets_from_start.sh | 55 + .../data/create_datasets_from_start.sh | 46 + .../bert-Nv/data/images/bert_pipeline.png | Bin 0 -> 212516 bytes .../bert-Nv/data/images/images_nvlamb.png | Bin 0 -> 88164 bytes .../data/images/trtis_base_summary.png | Bin 0 -> 22331 bytes .../code/bert-Nv/data/images/trtis_bs_1.png | Bin 0 -> 18766 bytes .../code/bert-Nv/data/images/trtis_bs_8.png | Bin 0 -> 17355 bytes .../bert-Nv/data/images/trtis_dynamic.png | Bin 0 -> 20919 bytes .../code/bert-Nv/data/images/trtis_ec_1.png | Bin 0 -> 17665 bytes .../code/bert-Nv/data/images/trtis_ec_4.png | Bin 0 -> 17963 bytes .../data/images/trtis_large_summary.png | Bin 0 -> 23001 bytes .../code/bert-Nv/data/images/trtis_static.png | Bin 0 -> 18379 bytes .../uncased_L-24_H-1024_A-16/bert_config.json | 13 + .../code/bert-Nv/extract_features.py | 419 + .../tensorflow/code/bert-Nv/fp16_utils.py | 35 + .../code/bert-Nv/fused_layer_norm.py | 141 + .../code/bert-Nv/gpu_environment.py | 36 + .../tensorflow/code/bert-Nv/modeling.py | 1031 + .../tensorflow/code/bert-Nv/modeling_test.py | 277 + .../tensorflow/code/bert-Nv/multilingual.md | 305 + .../code/bert-Nv/notebooks/README.md | 173 + .../notebooks/bert_squad_tf_finetuning.ipynb | 624 + .../notebooks/bert_squad_tf_inference.ipynb | 577 + .../bert_squad_tf_inference_colab.ipynb | 765 + .../notebooks/biobert_ner_tf_inference.ipynb | 610 + .../code/bert-Nv/notebooks/input.json | 31 + .../tensorflow/code/bert-Nv/optimization.py | 467 + .../code/bert-Nv/optimization_test.py | 48 + ...ng_movie_reviews_with_bert_on_tf_hub.ipynb | 1231 + .../code/bert-Nv/results/models/.gitkeep | 0 .../code/bert-Nv/results/perf_client/.gitkeep | 0 .../tensorflow/code/bert-Nv/run.sub | 73 + .../tensorflow/code/bert-Nv/run_classifier.py | 706 + .../code/bert-Nv/run_classifier_with_tfhub.py | 314 + .../tensorflow/code/bert-Nv/run_ner.py | 871 + .../code/bert-Nv/run_pretraining.py | 818 + .../tensorflow/code/bert-Nv/run_re.py | 939 + .../tensorflow/code/bert-Nv/run_squad.py | 1158 + .../code/bert-Nv/scripts/data_download.sh | 19 + .../code/bert-Nv/scripts/docker/build.sh | 9 + .../code/bert-Nv/scripts/docker/launch.sh | 15 + .../scripts/finetune_inference_benchmark.sh | 94 + .../scripts/finetune_train_benchmark.sh | 118 + .../code/bert-Nv/scripts/run_glue.sh | 99 + .../bert-Nv/scripts/run_glue_inference.sh | 78 + .../bert-Nv/scripts/run_pretraining_adam.sh | 111 + .../bert-Nv/scripts/run_pretraining_lamb.sh | 60 + .../scripts/run_pretraining_lamb_phase1.sh | 111 + .../scripts/run_pretraining_lamb_phase2.sh | 115 + .../code/bert-Nv/scripts/run_squad.sh | 107 + .../bert-Nv/scripts/run_squad_inference.sh | 86 + .../tensorflow/code/bert-Nv/setenv.sh.geop | 15 + .../tensorflow/code/bert-Nv/tf_metrics.py | 215 + .../tensorflow/code/bert-Nv/tokenization.py | 451 + .../code/bert-Nv/tokenization_test.py | 133 + .../tensorflow/code/bert-Nv/trtis/README.md | 108 + .../bert-Nv/trtis/run_squad_trtis_client.py | 222 + .../bert-Nv/trtis/scripts/export_model.sh | 58 + .../bert-Nv/trtis/scripts/generate_figures.sh | 146 + .../bert-Nv/trtis/scripts/launch_server.sh | 24 + .../code/bert-Nv/trtis/scripts/run_client.sh | 51 + .../bert-Nv/trtis/scripts/run_perf_client.sh | 73 + .../code/bert-Nv/trtis/scripts/run_trtis.sh | 88 + .../trtis/scripts/wait_for_trtis_server.sh | 33 + .../code/bert-Nv/utils/create_glue_data.py | 512 + .../bert-Nv/utils/create_pretraining_data.py | 501 + .../code/bert-Nv/utils/create_squad_data.py | 561 + .../tensorflow/code/bert-Nv/utils/utils.py | 75 + .../nlp/Bert-Large/tensorflow/config/1p.json | 14 + .../nlp/Bert-Large/tensorflow/config/8p.json | 49 + .../config/bert_config_large_cn.json | 14 + .../config/bert_config_large_en.json | 14 + .../tensorflow/config/npu_set_env.sh | 39 + .../nlp/Bert-Large/tensorflow/scripts/run.sh | 68 + .../Bert-Large/tensorflow/scripts/train.sh | 152 + .../object_detection/SSD-Resnet34/__init__.py | 0 .../SSD-Resnet34/tensorflow/README.md | 89 + .../SSD-Resnet34/tensorflow/__init__.py | 0 .../SSD-Resnet34/tensorflow/code/__init__.py | 0 .../tensorflow/code/coco_metric.py | 281 + .../tensorflow/code/create_coco_tf_record.py | 369 + .../tensorflow/code/dataloader.py | 436 + .../SSD-Resnet34/tensorflow/code/exec_main.sh | 24 + .../code/object_detection/__init__.py | 14 + .../code/object_detection/argmax_matcher.py | 199 + .../code/object_detection/box_coder.py | 151 + .../code/object_detection/box_list.py | 207 + .../object_detection/faster_rcnn_box_coder.py | 118 + .../code/object_detection/matcher.py | 241 + .../code/object_detection/preprocessor.py | 442 + .../region_similarity_calculator.py | 135 + .../code/object_detection/shape_utils.py | 70 + .../code/object_detection/target_assigner.py | 310 + .../object_detection/tf_example_decoder.py | 210 + .../SSD-Resnet34/tensorflow/code/ssd_1p.sh | 44 + .../SSD-Resnet34/tensorflow/code/ssd_8p.sh | 41 + .../tensorflow/code/ssd_architecture.py | 484 + .../tensorflow/code/ssd_constants.py | 122 + .../SSD-Resnet34/tensorflow/code/ssd_main.py | 309 + .../SSD-Resnet34/tensorflow/code/ssd_model.py | 500 + .../SSD-Resnet34/tensorflow/config/README.md | 197 + .../tensorflow/config/hccl_sample.json | 9 + .../tensorflow/config/npu_set_env.sh | 36 + .../SSD-Resnet34/tensorflow/scripts/run.sh | 70 + .../SSD-Resnet34/tensorflow/scripts/train.sh | 104 + .../yolov3/tensorflow/README.md | 141 + .../yolov3/tensorflow/code/.gitignore | 13 + .../yolov3/tensorflow/code/README.md | 140 + .../yolov3/tensorflow/code/args_multi.py | 110 + .../yolov3/tensorflow/code/args_single.py | 105 + .../tensorflow/code/coco_minival_anns.py | 113 + .../tensorflow/code/coco_trainval_anns.py | 113 + .../yolov3/tensorflow/code/convert_weight.py | 38 + .../yolov3/tensorflow/code/data/5k.txt | 5000 ++ .../yolov3/tensorflow/code/data/coco.names | 80 + .../tensorflow/code/data/coco2014_minival.txt | 4954 ++ .../tensorflow/code/data/yolo_anchors.txt | 1 + .../yolov3/tensorflow/code/docs/backbone.png | Bin 0 -> 23402 bytes .../code/docs/yolo_v3_architecture.png | Bin 0 -> 85552 bytes .../yolov3/tensorflow/code/eval.py | 220 + .../yolov3/tensorflow/code/eval.sh | 61 + .../yolov3/tensorflow/code/eval_coco.py | 57 + .../yolov3/tensorflow/code/get_kmeans.py | 155 + .../code/hccl_config/.res50_baseline.py.swp | Bin 0 -> 16384 bytes .../tensorflow/code/hccl_config/1p.json | 32 + .../tensorflow/code/hccl_config/2p.json | 43 + .../tensorflow/code/hccl_config/4p.json | 65 + .../tensorflow/code/hccl_config/8p.json | 109 + .../code/misc/experiments_on_voc/args_voc.py | 88 + .../code/misc/experiments_on_voc/eval_voc.py | 140 + .../code/misc/experiments_on_voc/train.txt | 16551 +++++ .../code/misc/experiments_on_voc/val.txt | 4952 ++ .../code/misc/experiments_on_voc/voc.names | 20 + .../tensorflow/code/misc/parse_voc_xml.py | 96 + .../misc/remove_optimizers_params_in_ckpt.py | 32 + .../yolov3/tensorflow/code/model.py | 457 + .../yolov3/tensorflow/code/npu_train.sh | 58 + .../tensorflow/code/npu_train_1p_multi.sh | 1 + .../tensorflow/code/npu_train_1p_single.sh | 1 + .../tensorflow/code/npu_train_8p_multi.sh | 1 + .../tensorflow/code/npu_train_8p_single.sh | 1 + .../yolov3/tensorflow/code/npu_train_wort.sh | 50 + .../yolov3/tensorflow/code/run_yolov3.sh | 29 + .../yolov3/tensorflow/code/test_show.sh | 57 + .../tensorflow/code/test_single_image.py | 86 + .../yolov3/tensorflow/code/train.py | 287 + .../code/training/t1/D0/rank_table.json | 109 + .../code/training/t1/D0/run_yolov3.sh | 29 + .../code/training/t1/D1/rank_table.json | 109 + .../code/training/t1/D1/run_yolov3.sh | 29 + .../code/training/t1/D2/rank_table.json | 109 + .../code/training/t1/D2/run_yolov3.sh | 29 + .../code/training/t1/D3/rank_table.json | 109 + .../code/training/t1/D3/run_yolov3.sh | 29 + .../code/training/t1/D4/rank_table.json | 109 + .../code/training/t1/D4/run_yolov3.sh | 29 + .../code/training/t1/D5/rank_table.json | 109 + .../code/training/t1/D5/run_yolov3.sh | 29 + .../code/training/t1/D6/rank_table.json | 109 + .../code/training/t1/D6/run_yolov3.sh | 29 + .../code/training/t1/D7/rank_table.json | 109 + .../code/training/t1/D7/run_yolov3.sh | 29 + .../yolov3/tensorflow/code/utils/__init__.py | 0 .../yolov3/tensorflow/code/utils/data_aug.py | 450 + .../tensorflow/code/utils/data_utils.py | 294 + .../tensorflow/code/utils/eval_utils.py | 423 + .../tensorflow/code/utils/layer_utils.py | 89 + .../tensorflow/code/utils/misc_utils.py | 165 + .../yolov3/tensorflow/code/utils/nms_utils.py | 123 + .../tensorflow/code/utils/plot_utils.py | 35 + .../yolov3/tensorflow/code/video_test.py | 102 + .../yolov3/tensorflow/config/hccl_sample.json | 9 + .../yolov3/tensorflow/config/npu_set_env.sh | 29 + .../yolov3/tensorflow/scripts/eval.sh | 53 + .../yolov3/tensorflow/scripts/run.sh | 77 + .../yolov3/tensorflow/scripts/train.sh | 115 + .../Dockerfile/tf_ubuntu_arm64/Dockerfile | 135 + .../tf_ubuntu_arm64/install_ascend_pkgs.sh | 27 + .../Dockerfile/tf_ubuntu_arm64/postbuild.sh | 39 + .../Dockerfile/tf_ubuntu_arm64/prebuild.sh | 16 + .../utils/atlasboost/README.md | 166 + .../utils/benchmark_log/__init__.py | 0 .../utils/benchmark_log/basic_utils.py | 48 + .../utils/benchmark_log/hwlog.py | 276 + .../utils/download_dataset.sh | 47 + .../utils/shell/get_params_for_yaml.sh | 23 + .../utils/shell/hccl_sample.json | 9 + .../utils/shell/set_json.sh | 40 + .../utils/shell/start.sh | 139 + train/benchmark.sh | 117 + train/docker_ct_build.sh | 23 + train/result/README.md | 13 + train/yaml/Bert-Base.yaml | 46 + train/yaml/Bert-Large.yaml | 47 + train/yaml/DeepMar.yaml | 18 + train/yaml/DenseNet121.yaml | 52 + train/yaml/EfficientNet.yaml | 20 + train/yaml/MobileNet.yaml | 49 + train/yaml/ResNet101.yaml | 19 + train/yaml/ResNet50.yaml | 66 + train/yaml/Resnet50_HC.yaml | 23 + train/yaml/Resnext50.yaml | 28 + train/yaml/SSD-Resnet34.yaml | 25 + train/yaml/ShuffleNet.yaml | 18 + train/yaml/YoLoV3.yaml | 36 + train/yaml/cluster_info.yaml | 17 + train/yaml/vgg16.yaml | 34 + 1225 files changed, 345421 insertions(+) create mode 100644 train/CANN V100R020C10 benchmark工具用户指南 01.pdf create mode 100644 train/README.md create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/README.md create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/dataset/Dataset.py create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/dataset/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/dataset/add_transforms.py create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/model/DeepMAR.py create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/model/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/model/resnet.py create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/utils/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/utils/evaluate.py create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/utils/utils.py create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/dataset/demo/.gitsave create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/dataset/demo/demo_image.png create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/dataset/pa100k/.gitsave create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/dataset/peta/.gitsave create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/dataset/rap/.gitsave create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/dataset/rap2/.gitsave create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/demo.py create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/train_deepmar_resnet50.py create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/train_deepmar_resnet50_8p.py create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/transform_pa100k.py create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/transform_peta.py create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/transform_rap.py create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/transform_rap2.py create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/config/set_env_b023.sh create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/scripts/run.sh create mode 100644 train/atlas_benchmark-master/image_classification/DeepMar/pytorch/scripts/train.sh create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/README.md create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/densenet121_1p_main.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/densenet121_8p_main.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/densenet_0_2_2.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/densenet_0_5_0.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/eval.sh create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/cpu_adapter_file/densenet.py.cpu create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/cpu_adapter_file/densenet.py.npu create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/densenet_print.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/npu_config/1p.json create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/npu_config/2p.json create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/npu_config/4p.json create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/npu_config/8p.json create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/profile/cpu/net_show_cpu.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/profile/cpu/run_cpu.sh create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/profile/npu/net_show_npu.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/1p.json create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/hccl_sample.json create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/npu_set_env.sh create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/set_env.sh create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/set_env_b020.sh create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/set_env_b023.sh create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/scripts/eval.sh create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/scripts/run.sh create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/scripts/train.sh create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/README.md create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/create_session.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/data_loader.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/densenet.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/hyper_param.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/layers.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/logger.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/model.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/preprocessing.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/train.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/train_helper.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/trainer.py create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/config/1p.json create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/config/8p.json create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/config/hccl_sample.json create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/config/npu_set_env.sh create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/scripts/run.sh create mode 100644 train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/scripts/train.sh create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/README.md create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/LICENSE create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/README.md create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/README_NPU.md create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/auto_augment.py create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/model.py create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/npu_info.py create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/rmsprop_tf.py create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/utils.py create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/imagenet/README.md create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/imagenet/data/README.md create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/imagenet/main.py create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/simple/check.ipynb create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/simple/example.ipynb create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/simple/labels_map.txt create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/hubconf.py create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/npu_1p.sh create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/setup.py create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/tests/test_model.py create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/config/set_env_b023.sh create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/scripts/run.sh create mode 100644 train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/scripts/train.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/download_dataset.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/README.md create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/README.md create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/hook.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/main_apex.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/mobilenet.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/profiling.json create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/README.md create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/eval.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/hook.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/main_apex.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/mobilenet.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/mobilenetv2_8p_main.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/mobilenetv2_8p_main_anycard.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/multi_epochs_dataloader.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/profiling.json create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/resume1p.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/resume8p.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/run1p.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/run8p.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/set_env_b020.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/set_env_b023.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/config/npu_set_env.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/config/set_env_b023.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/scripts/run.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/pytorch/scripts/train.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/README.md create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/README.md create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/dataloader/data_provider.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/build_imagenet_data.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/cifar10.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/dataset_factory.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/dataset_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_cifar10.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_flowers.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_imagenet.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_mnist.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_visualwakewords.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_visualwakewords_lib.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_imagenet.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/flowers.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/imagenet.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/imagenet_2012_validation_synset_labels.txt create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/imagenet_lsvrc_2015_synsets.txt create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/imagenet_metadata.txt create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/mnist.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/preprocess_imagenet_validation_data.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/process_bounding_boxes.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/visualwakewords.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/deployment/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/deployment/model_deploy.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/deployment/model_deploy_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/download_and_convert_data.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/env.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/estimator_impl.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/eval_image_classifier.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/eval_image_classifier_mobilenet.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/export_inference_graph.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/export_inference_graph_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/gpu_helper.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/logger.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/alexnet.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/alexnet_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/cifarnet.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/cyclegan.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/cyclegan_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/dcgan.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/dcgan_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/i3d.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/i3d_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/i3d_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_resnet_v2.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_resnet_v2_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v1.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v1_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v2.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v2_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v3.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v3_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v4.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v4_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/lenet.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/.idea/encodings.xml create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/.idea/misc.xml create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/.idea/mobilenet.iml create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/.idea/modules.xml create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/README.md create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/conv_blocks.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/g3doc/edgetpu_latency.png create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/g3doc/latency_pixel1.png create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/g3doc/madds_top1_accuracy.png create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_example.ipynb create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_v2.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_v2_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_v3.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_v3_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1.md create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1_eval.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1_train.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/README.md create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/nasnet.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/nasnet_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/nasnet_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/nasnet_utils_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/pnasnet.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/pnasnet_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nets_factory.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nets_factory_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/overfeat.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/overfeat_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/pix2pix.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/pix2pix_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/post_training_quantization.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_v1.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_v1_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_v2.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_v2_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/s3dg.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/s3dg_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/vgg.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/vgg_test.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/cifarnet_preprocessing.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/inception_preprocessing.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/lenet_preprocessing.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/preprocessing_factory.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/vgg_preprocessing.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/setup.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/train.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/config/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/config/hccl_sample.json create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/config/npu_set_env.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/scripts/eval.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/scripts/run.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/scripts/train.sh create mode 100644 train/atlas_benchmark-master/image_classification/MobileNet/verify_dataset.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/LICENSE create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/README-GPU.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/README-TPU.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/benchmark_uploader.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/benchmark_uploader_main.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/benchmark_uploader_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/benchmark_wrappers.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/bert_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/bert_benchmark_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/bert_squad_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/datastore/schema/benchmark_metric.json create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/datastore/schema/benchmark_run.json create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/datastore/schema/benchmark_run_status.json create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/keras_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/keras_cifar_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/keras_imagenet_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/resnet_cifar_main.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/resnet_cifar_model.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/resnet_cifar_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/shakespeare/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/shakespeare/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/shakespeare/shakespeare_main.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/synthetic_util.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/ncf_keras_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/perfzero_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/resnet_ctl_imagenet_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/retinanet_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/shakespeare_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/tfhub_memory_usage_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/transformer_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/xlnet_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/colab/bert.ipynb create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/gelu.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/gelu_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/swish.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/swish_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/base_config.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/base_config_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/params_dict.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/params_dict_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/performance.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/tf_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/training/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/training/distributed_executor.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/configs.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/export_albert_tfhub.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/export_albert_tfhub_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/run_classifier.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/run_squad.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/tf2_albert_encoder_checkpoint_converter.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/bert_cloud_tpu.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/bert_models.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/common_flags.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/configs.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/export_tfhub.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/export_tfhub_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/input_pipeline.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/model_saving_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/model_training_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/model_training_utils_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/run_classifier.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/run_pretraining.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/run_squad.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/run_squad_helper.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/squad_evaluate_v1_1.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/squad_evaluate_v2_0.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/tf1_checkpoint_converter_lib.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/tf2_encoder_checkpoint_converter.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/tokenization.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/tokenization_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/classifier_data_lib.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/create_finetuning_data.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/create_pretraining_data.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/squad_lib.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/squad_lib_sp.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/attention.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/attention_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/dense_einsum.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/dense_einsum_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/masked_softmax.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/masked_softmax_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/on_device_embedding.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/on_device_embedding_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/position_embedding.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/position_embedding_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/self_attention_mask.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/transformer.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/transformer_scaffold.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/transformer_scaffold_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/transformer_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/util.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/losses/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/losses/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/losses/weighted_sparse_categorical_crossentropy.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/losses/weighted_sparse_categorical_crossentropy_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_classifier.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_classifier_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_pretrainer.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_pretrainer_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_span_labeler.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_span_labeler_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/albert_transformer_encoder.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/albert_transformer_encoder_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/classification.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/classification_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/encoder_scaffold.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/encoder_scaffold_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/masked_lm.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/masked_lm_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/span_labeling.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/span_labeling_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/transformer_encoder.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/transformer_encoder_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/configs.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/configs_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/decoder.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/decoder_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/evaluation.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/input_pipeline.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/models.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/multi_channel_attention.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/multi_channel_attention_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/optimizer.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/raw_data_process.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/raw_data_processor.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.html create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.json create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.html create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.json create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/stories.json create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/vocab.txt create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/trainer.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/trainer_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/optimization.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/attention_layer.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/beam_search.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/beam_search_v1.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/beam_search_v1_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/compute_bleu.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/compute_bleu_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/data_download.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/data_pipeline.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/embedding_layer.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/ffn_layer.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/metrics.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/misc.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/model_params.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/model_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/model_utils_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/optimizer.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer_layers_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer_main.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer_main_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/translate.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/utils/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/utils/metrics.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/utils/tokenizer.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/utils/tokenizer_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/classifier_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/common_flags.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/data_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/optimization.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/preprocess_classification_data.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/preprocess_pretrain_data.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/preprocess_squad_data.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/preprocess_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/run_classifier.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/run_pretrain.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/run_squad.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/squad_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/training_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/xlnet_config.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/xlnet_modeling.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/xlnet_modeling_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/pip_package/setup.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/boosted_trees/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/boosted_trees/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/boosted_trees/data_download.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/boosted_trees/train_higgs.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/dataset.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist_eager.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist_eager_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist_tpu.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/ncf/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/ncf/ncf_estimator_main.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/cifar10_download_and_extract.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/cifar10_main.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/cifar10_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/estimator_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/gpu_train.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/imagenet_main.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/imagenet_preprocessing.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/imagenet_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/resnet_model.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/resnet_run_loop.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/run_imagenet.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/train_accuracy_rewrite.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/attention_layer.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/dataset.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/embedding_layer.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/ffn_layer.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/schedule.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/schedule_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/transformer.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/transformer_main.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/translate.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/data/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/data/file_io.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/data/file_io_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/export.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/export_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/tpu.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/tpu_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/census_dataset.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/census_main.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/census_test.csv create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/census_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/movielens_dataset.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/movielens_main.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/movielens_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/wide_deep_run_loop.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/constants.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/create_ncf_data.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/data_pipeline.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/data_preprocessing.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/data_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/movielens.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/ncf_common.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/ncf_input_pipeline.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/ncf_keras_main.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/ncf_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/neumf_model.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/popen_helper.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/stat_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/requirements.txt create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/controller.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/controller_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/grad_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/runnable.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/standard_runnable.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_base.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_conventions.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_device.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_distribution.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_misc.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_performance.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/core.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/flags_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/guidelines.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/hyperparams_flags.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/cloud_lib.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/cloud_lib_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/guidelines.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks_bak.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks_helper.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks_helper_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/logger.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/logger_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/metric_hook.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/metric_hook_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/mlperf_helper.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/callstack_sampler.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/distribution_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/distribution_utils_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/keras_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/model_helpers.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/model_helpers_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/tpu_lib.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/integration.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/mock_lib.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/pylint.rcfile create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/scripts/builds_common.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/scripts/ci_sanity.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/scripts/presubmit.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/base_config.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/factory.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/maskrcnn_config.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/retinanet_config.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/anchor.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/factory.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/input_reader.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/maskrcnn_parser.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/mode_keys.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/retinanet_parser.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/shapemask_parser.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/tf_example_decoder.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/evaluation/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/evaluation/coco_evaluator.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/evaluation/coco_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/evaluation/factory.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/executor/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/executor/detection_executor.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/main.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/factory.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/fpn.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/heads.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/identity.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/nn_ops.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/resnet.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/base_model.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/checkpoint_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/factory.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/learning_rates.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/losses.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/maskrcnn_model.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/retinanet_model.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/nms.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/postprocess_ops.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/roi_ops.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/sampling_ops.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/spatial_transform_ops.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/autoaugment_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/box_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/class_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/dataloader_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/input_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/mask_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/argmax_matcher.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/balanced_positive_negative_sampler.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/box_coder.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/box_list.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/box_list_ops.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/faster_rcnn_box_coder.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/matcher.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/minibatch_sampler.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/ops.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/preprocessor.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/region_similarity_calculator.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/shape_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/target_assigner.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/visualization_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/augment.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/augment_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/callbacks.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/classifier_trainer.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/classifier_trainer_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/base_configs.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/configs.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-gpu.yaml create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-tpu.yaml create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-gpu.yaml create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-tpu.yaml create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/resnet/imagenet/gpu.yaml create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/resnet/imagenet/tpu.yaml create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/dataset_factory.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/efficientnet/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/efficientnet/common_modules.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/efficientnet/efficientnet_config.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/efficientnet/efficientnet_model.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/learning_rate.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/learning_rate_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/mnist_main.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/mnist_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/optimizer_factory.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/optimizer_factory_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/preprocessing.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/cifar_preprocessing.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/common.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/imagenet_preprocessing.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_config.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_imagenet_main.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_model.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_runnable.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/tfhub_export.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/test_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/config/npu_set_env.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/scripts/run.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/scripts/train.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/eval.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/mindspore_hub_conf.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/CrossEntropySmooth.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/config.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/dataset.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/lr_generator.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/resnet.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/train.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/mindspore/config/npu_set_env.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/mindspore/scripts/eval.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/mindspore/scripts/run.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/mindspore/scripts/train.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/pytorch/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/pytorch/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/dataloaders.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/logger.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/mixup.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/resnet.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/smoothing.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/smoothing_tocpu.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/training.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/main-apex-d76-npu.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/pytorch-resnet50-apex.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/pytorch/config/npu_set_env.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/pytorch/scripts/run.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/pytorch/scripts/train.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/Dockerfile create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/cloud_docker_init.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/hccl_sample.json create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/main_sample.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/npu_set_env.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/train_sample.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/LICENSE create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/README-TPU.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/datastore/schema/benchmark_metric.json create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/datastore/schema/benchmark_run.json create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/datastore/schema/benchmark_run_status.json create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/models/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/models/resnet_cifar_main.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/models/resnet_cifar_model.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/models/resnet_cifar_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/resnet_tf_r1_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/gelu.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/gelu_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/swish.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/swish_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/base_config.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/base_config_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/params_dict.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/params_dict_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/model_training_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/model_training_utils_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/performance.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/tf_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/training/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/training/distributed_executor.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/pip_package/setup.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/boosted_trees/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/boosted_trees/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/boosted_trees/data_download.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/boosted_trees/train_higgs.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/estimator_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/imagenet_main.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/imagenet_preprocessing.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/imagenet_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/resnet_model.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/resnet_run_loop.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/resnet_run_loop_orgin.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/slog create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/data/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/data/file_io.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/data/file_io_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/export.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/export_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/tpu.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/tpu_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/requirements.txt create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_base.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_conventions.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_device.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_distribution.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_misc.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_performance.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/core.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/flags_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/guidelines.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/hyperparams_flags.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/cloud_lib.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/cloud_lib_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/guidelines.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/hooks.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/hooks_helper.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/hooks_helper_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/hooks_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/logger.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/logger_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/metric_hook.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/metric_hook_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/mlperf_helper.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/callstack_sampler.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/distribution_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/distribution_utils_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/keras_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/model_helpers.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/model_helpers_test.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/tpu_lib.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/benchmark_wrappers.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/integration.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/mock_lib.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/perfzero_benchmark.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/pylint.rcfile create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/scripts/builds_common.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/scripts/ci_sanity.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/scripts/presubmit.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/npu_set_env.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_16p_npu.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_1p_npu.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_2p_npu.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_4p_npu.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_8p_npu.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/scripts/run.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/scripts/train.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/configs/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/configs/pid create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/configs/res50_32bs_1p.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/configs/res50_32bs_8p.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/data_loader/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/data_loader/resnet50/data_loader.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/data_loader/resnet50/preprocessing.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/hyper_param/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/hyper_param/hyper_param.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/hyper_param/lr_schedule.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/layers/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/layers/layers.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/losses/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/losses/res50_loss.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/mains/clean.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/mains/res50.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/mains/train_res50.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/mains/train_res50_gpu.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/models/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/models/resnet50/res50_helper.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/models/resnet50/res50_model.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/models/resnet50/resnet.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/optimizers/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/optimizers/optimizer.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/trainers/2q create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/trainers/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/trainers/gpu_base_trainer.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/trainers/train_helper.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/utils/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/utils/create_session.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/utils/logger.py create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/config/hccl_sample.json create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/config/main_sample.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/config/npu_set_env.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/config/train_sample.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/scripts/run.sh create mode 100644 train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/scripts/train.sh create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/README.md create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/pid create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_16p.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_1p.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_2p.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_4p.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_8p.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/data_loader/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/data_loader/resnet50/data_loader.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/data_loader/resnet50/preprocessing.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/hyper_param/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/hyper_param/hyper_param.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/hyper_param/lr_schedule.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/hyper_param/lr_schedule_0907_back.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/layers/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/layers/layers.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/losses/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/losses/res50_loss.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/mains/clean.sh create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/mains/res50.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/mains/train_res50.sh create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/mains/train_res50_gpu.sh create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/resnet50/res50_helper.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/resnet50/res50_model.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/resnet50/res50_model.py_bak create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/resnet50/resnet.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/optimizers/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/optimizers/optimizer.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/results/res50_baseline/resnet50.log create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/trainers/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/trainers/gpu_base_trainer.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/trainers/train_helper.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/utils/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/utils/create_session.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/utils/logger.py create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/config/npu_set_env.sh create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/scripts/run.sh create mode 100644 train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/scripts/train.sh create mode 100644 train/atlas_benchmark-master/image_classification/ShuffleNet/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/8p_main.py create mode 100644 train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/8p_main_med.py create mode 100644 train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/README.md create mode 100644 train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/main.py create mode 100644 train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/_utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/shufflenetv2.py create mode 100644 train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/shufflenetv2_group_shuffle.py create mode 100644 train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/shufflenetv2_wock_op_woct.py create mode 100644 train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/shufflenetv2_wock_op_woct_8p.py create mode 100644 train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/utils.py create mode 100644 train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/multi_epochs_dataloader.py create mode 100644 train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/config/npu_set_env.sh create mode 100644 train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/scripts/run.sh create mode 100644 train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/scripts/train.sh create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/__init__.py create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/16p.json create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/1p.json create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/8p.json create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/cloud_docker_init.sh create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/hccl_sample.json create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/main_sample.sh create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/npu_set_env.sh create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/train_sample.sh create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/scripts/run.sh create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/scripts/train.sh create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/create_session.py create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/data_loader.py create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/hyper_param.py create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/layers.py create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/logger.py create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/model.py create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/preprocessing.py create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/train.py create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/train_helper.py create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/trainer.py create mode 100644 train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/vgg.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/README.md create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/__init__.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/bert_base_config.json create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/bert_base_vocab.txt create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/create_pretraining_data.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/extract_features.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/fp16_utils.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/fused_layer_norm.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/gpu_environment.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/modeling.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/optimization.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/run_pretraining.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/tf_metrics.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/tokenization.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/utils.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/1p.json create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/8p.json create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_layer12_cn.json create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_layer12_en.json create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_layer6_cn.json create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_layer6_en.json create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_vocab.txt create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/npu_set_env.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/scripts/run.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/scripts/train.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/README.md create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/CONTRIBUTING.md create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/Dockerfile create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/LICENSE create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/NOTICE create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/README.md create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/__init__.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/README.md create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/__init__.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/conlleval.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/re_eval.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/biobert_data_download.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/biobert_finetune_inference_benchmark.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/biobert_finetune_train_benchmark.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/ner_bc5cdr-chem.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/ner_bc5cdr-disease.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/rel_chemprot.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/run_biobert.sub create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/run_biobert_finetuning_inference.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/run_pretraining_pubmed_base_phase_1.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/run_pretraining_pubmed_base_phase_2.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/configurations.yml create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/BooksDownloader.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/BookscorpusTextFormatting.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/Downloader.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/GLUEDownloader.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/GooglePretrainedWeightDownloader.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/NVIDIAPretrainedWeightDownloader.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/PubMedDownloader.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/PubMedTextFormatting.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/README.md create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/SquadDownloader.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/TextSharding.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/WikiDownloader.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/WikicorpusTextFormatting.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/__init__.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/bertPrep.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/create_biobert_datasets_from_start.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/create_datasets_from_start.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/images/bert_pipeline.png create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/images/images_nvlamb.png create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/images/trtis_base_summary.png create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/images/trtis_bs_1.png create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/images/trtis_bs_8.png create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/images/trtis_dynamic.png create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/images/trtis_ec_1.png create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/images/trtis_ec_4.png create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/images/trtis_large_summary.png create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/images/trtis_static.png create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/uncased_L-24_H-1024_A-16/bert_config.json create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/extract_features.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/fp16_utils.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/fused_layer_norm.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/gpu_environment.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/modeling.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/modeling_test.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/multilingual.md create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/notebooks/README.md create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/notebooks/bert_squad_tf_finetuning.ipynb create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/notebooks/bert_squad_tf_inference.ipynb create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/notebooks/bert_squad_tf_inference_colab.ipynb create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/notebooks/biobert_ner_tf_inference.ipynb create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/notebooks/input.json create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/optimization.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/optimization_test.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/predicting_movie_reviews_with_bert_on_tf_hub.ipynb create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/results/models/.gitkeep create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/results/perf_client/.gitkeep create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/run.sub create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/run_classifier.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/run_classifier_with_tfhub.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/run_ner.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/run_pretraining.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/run_re.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/run_squad.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/scripts/data_download.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/scripts/docker/build.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/scripts/docker/launch.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/scripts/finetune_inference_benchmark.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/scripts/finetune_train_benchmark.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/scripts/run_glue.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/scripts/run_glue_inference.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/scripts/run_pretraining_adam.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/scripts/run_pretraining_lamb.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/scripts/run_pretraining_lamb_phase1.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/scripts/run_pretraining_lamb_phase2.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/scripts/run_squad.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/scripts/run_squad_inference.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/setenv.sh.geop create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/tf_metrics.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/tokenization.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/tokenization_test.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/trtis/README.md create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/trtis/run_squad_trtis_client.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/trtis/scripts/export_model.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/trtis/scripts/generate_figures.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/trtis/scripts/launch_server.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/trtis/scripts/run_client.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/trtis/scripts/run_perf_client.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/trtis/scripts/run_trtis.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/trtis/scripts/wait_for_trtis_server.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/utils/create_glue_data.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/utils/create_pretraining_data.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/utils/create_squad_data.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/utils/utils.py create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/config/1p.json create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/config/8p.json create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/config/bert_config_large_cn.json create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/config/bert_config_large_en.json create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/config/npu_set_env.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/scripts/run.sh create mode 100644 train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/scripts/train.sh create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/__init__.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/README.md create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/__init__.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/__init__.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/coco_metric.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/create_coco_tf_record.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/dataloader.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/exec_main.sh create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/object_detection/__init__.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/object_detection/argmax_matcher.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/object_detection/box_coder.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/object_detection/box_list.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/object_detection/faster_rcnn_box_coder.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/object_detection/matcher.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/object_detection/preprocessor.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/object_detection/region_similarity_calculator.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/object_detection/shape_utils.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/object_detection/target_assigner.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/object_detection/tf_example_decoder.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/ssd_1p.sh create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/ssd_8p.sh create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/ssd_architecture.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/ssd_constants.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/ssd_main.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/code/ssd_model.py create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/config/README.md create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/config/hccl_sample.json create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/config/npu_set_env.sh create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/scripts/run.sh create mode 100644 train/atlas_benchmark-master/object_detection/SSD-Resnet34/tensorflow/scripts/train.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/README.md create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/.gitignore create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/README.md create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/args_multi.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/args_single.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/coco_minival_anns.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/coco_trainval_anns.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/convert_weight.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/data/5k.txt create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/data/coco.names create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/data/coco2014_minival.txt create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/data/yolo_anchors.txt create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/docs/backbone.png create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/docs/yolo_v3_architecture.png create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/eval.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/eval.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/eval_coco.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/get_kmeans.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/hccl_config/.res50_baseline.py.swp create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/hccl_config/1p.json create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/hccl_config/2p.json create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/hccl_config/4p.json create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/hccl_config/8p.json create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/misc/experiments_on_voc/args_voc.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/misc/experiments_on_voc/eval_voc.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/misc/experiments_on_voc/train.txt create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/misc/experiments_on_voc/val.txt create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/misc/experiments_on_voc/voc.names create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/misc/parse_voc_xml.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/misc/remove_optimizers_params_in_ckpt.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/model.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/npu_train.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/npu_train_1p_multi.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/npu_train_1p_single.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/npu_train_8p_multi.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/npu_train_8p_single.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/npu_train_wort.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/run_yolov3.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/test_show.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/test_single_image.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/train.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/training/t1/D0/rank_table.json create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/training/t1/D0/run_yolov3.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/training/t1/D1/rank_table.json create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/training/t1/D1/run_yolov3.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/training/t1/D2/rank_table.json create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/training/t1/D2/run_yolov3.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/training/t1/D3/rank_table.json create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/training/t1/D3/run_yolov3.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/training/t1/D4/rank_table.json create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/training/t1/D4/run_yolov3.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/training/t1/D5/rank_table.json create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/training/t1/D5/run_yolov3.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/training/t1/D6/rank_table.json create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/training/t1/D6/run_yolov3.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/training/t1/D7/rank_table.json create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/training/t1/D7/run_yolov3.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/utils/__init__.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/utils/data_aug.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/utils/data_utils.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/utils/eval_utils.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/utils/layer_utils.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/utils/misc_utils.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/utils/nms_utils.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/utils/plot_utils.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/code/video_test.py create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/config/hccl_sample.json create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/config/npu_set_env.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/scripts/eval.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/scripts/run.sh create mode 100644 train/atlas_benchmark-master/object_detection/yolov3/tensorflow/scripts/train.sh create mode 100644 train/atlas_benchmark-master/utils/Dockerfile/tf_ubuntu_arm64/Dockerfile create mode 100644 train/atlas_benchmark-master/utils/Dockerfile/tf_ubuntu_arm64/install_ascend_pkgs.sh create mode 100644 train/atlas_benchmark-master/utils/Dockerfile/tf_ubuntu_arm64/postbuild.sh create mode 100644 train/atlas_benchmark-master/utils/Dockerfile/tf_ubuntu_arm64/prebuild.sh create mode 100644 train/atlas_benchmark-master/utils/atlasboost/README.md create mode 100644 train/atlas_benchmark-master/utils/benchmark_log/__init__.py create mode 100644 train/atlas_benchmark-master/utils/benchmark_log/basic_utils.py create mode 100644 train/atlas_benchmark-master/utils/benchmark_log/hwlog.py create mode 100644 train/atlas_benchmark-master/utils/download_dataset.sh create mode 100644 train/atlas_benchmark-master/utils/shell/get_params_for_yaml.sh create mode 100644 train/atlas_benchmark-master/utils/shell/hccl_sample.json create mode 100644 train/atlas_benchmark-master/utils/shell/set_json.sh create mode 100644 train/atlas_benchmark-master/utils/shell/start.sh create mode 100644 train/benchmark.sh create mode 100644 train/docker_ct_build.sh create mode 100644 train/result/README.md create mode 100644 train/yaml/Bert-Base.yaml create mode 100644 train/yaml/Bert-Large.yaml create mode 100644 train/yaml/DeepMar.yaml create mode 100644 train/yaml/DenseNet121.yaml create mode 100644 train/yaml/EfficientNet.yaml create mode 100644 train/yaml/MobileNet.yaml create mode 100644 train/yaml/ResNet101.yaml create mode 100644 train/yaml/ResNet50.yaml create mode 100644 train/yaml/Resnet50_HC.yaml create mode 100644 train/yaml/Resnext50.yaml create mode 100644 train/yaml/SSD-Resnet34.yaml create mode 100644 train/yaml/ShuffleNet.yaml create mode 100644 train/yaml/YoLoV3.yaml create mode 100644 train/yaml/cluster_info.yaml create mode 100644 train/yaml/vgg16.yaml diff --git a/train/CANN V100R020C10 benchmark工具用户指南 01.pdf b/train/CANN V100R020C10 benchmark工具用户指南 01.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2fed7dede02177c73ceeb50e64385a18eec257c8 GIT binary patch literal 854207 zcmcG0`6HD3`~Eqnlip>SbCe}(CtKNNEMe#vvM&*`H6jeMWM?dum=Hn;p&|QL_OXWS zhLJ6Mim_!khOvEKjL!Ej`1)<0d7kIB-OF`d_dR%Ym2L}(2}@ky`TnD~pZ0>7*cH($ z&X)EU#3ZFfMWruTJ3T!9bo2D8oSevA9~Wzpdpef()>iHpMD#r@-H$J*IX` z_s|`A}2iaent9%2;BLRv#Y+#11sw*B1+btc2?GUDvDP`^tB$iIb0DF1=G;GBBE*S zWDABlnXBFfdHD-&?ylAk94~l}{+u9rnUVaz9VmSF+5XwlK^S?M2J48ekE{z#scQEO zTYhccmQ0vT&NH}vC+@Xi{+$wT^-F3Ddcn%UC7Kp4eUbvy=}>Mt`N)0{ak`qEAbu-kE~{XXXX5ECPe#@B6q#a$ zHU&uq*k2>)%H|4En-l4nQY4rtW8@SMSRuQM1^*kSZs1x-Jh_E`^{>+zhF%h3HW|uz z?^;Q@@LT2pUx;vg;ETz-bJhM{HTe(l+4o-t#RQ^J3DT#{I0!=6v;wyd)kqzsH#zrg^v;&AGtKV9NG zw#zRuVvy=Dt(L@KfIL%3mT3qh^?#wed(GAcS)5 zmP5WWO+-U)lWg(fd66N(hSP)7%6lqSmh^Bz@z?3UeC}~$qJQSUttuRxe})M+_>8|0 zlS%z+4u6-2W5P2N9C-uh`4Jj1*}u%~eNrpF<;zpoK^2{K1|gHH7bt@}W_dEvLesjDH&c5$vNBJgJ9pOouJv2uA8r+q6S-f&E=Ts-mA{_uv2 zm*GqovwIggi=2x8N4d8>YrtwgukyNpf0lRLm*;Z?4^_$+x2SVk&cX%ijjM-Am*A8c zzDpULXTuJX@SQ6zgQ%i!7|P`*Hg{{NLkvg2-Ob!y$=~DDPOm@8Gp8}vj2Gz&i?KXC z!&kamR`cKOVbEP`c)zcC2SVa>Ew%KLjf)u%&N8hO@M;;p^=i>QT?}pceMXj}d*L0H z$g>cm$CC`dCL=K*4s0w@Uf=A6z47Vm%aUc<@GYNVWdjlUey?YWSx#SKPsCavjiYK7 zGmPAh)Ho*YP-gfWuIM{kqB>MFm8;7i$wYH%n*5O%57gF$)F;fRCvf*ImjfAZZEghJ zh4J~Ap-(ruEMzn#zoJux>2yj@_<5Y4dDOciTVQv8y~{PP;i~PT&b#l-i_b8pTYC>{ z{lQbseNZKrH-!K6B0hwFKFC8OtS6e9a@XDKaokDeb0Q(9rWSr)xF&I}^2W~0udR9a z%<}76Q`Mpy|omJR{JWDNs_ii(%Q?Sbw6dAgv6838y)iIY^cNi1ZU;i?YCnh-?F)0RF zYX5DeJ%5&DJkjX7ywg7I=hGQo!On!=`qBH)^ni=kM}GS>%Dgf&fk7V>{P!x~ymx3} z!`Djxt@f}p0wdO7Fm%A@_lM_cS&0Z9Q_7Y!kbcd`;lL%U!Ahp{-NNsuv;*j3EZfCpPwkQTJcQ zf^W|-8`q%f5@L55C^nX4&VT)bfmM#2I`zGB=U)Rl^ux>VnA~g1$_=AzxllppOv1>A zoQN-k&cehzik0K@@~->i?$MOr8U5G#Z~fCZnrIR|iY1=C6IB<;y@B{O)6#yT-iW;9 zm66wmRhMp4W<0!*87^v3% z^)|*A?vm2Wx$@?}9%E6AE2T!-5__YYD09>0{NruP9}#)4PUou26_z~lr%yyg)aiv+ zS3_~aK_5&Wwq00JKOH>peeOt*Zcdh1*|_Z8(`~!(0_l0}Qk<`?<`SI&C7541Ki%C^ z?iP9{jl*Q!-$a;l=C7~DkMwy$*B2S#4qla~qm=#Q(H&4ZrFL~9_1@C$U%MH`9Qiwa z-kh}1jlvvE-9|H*V@IPkd#LYo=kzj^%YgbEWH6^5QZ#y*_05%EL*2Jzq~p#rT4GM> z%y@sua(cP_#_!bU7;~crs=}R7_A%io*%kIYh(G?b&cp>sMUpM&I@}Q=p7{7XU#~zXE76KGm#o*>BMlbc zJzGpLyrvnzm!FD}BD!3s#59A>I6V1fp_aH_f(u@cli_Z|C=q`tyK^`nePaECIkq%$ zN_H~wqto-5PW<@+LPPNI$@`QzmWM$rUN@nqvs%pkUSb>`9x#MK^X?0UpRD5ve<()x zb=dQuMBlsxqEvq1AG8aYh>0u5i@DW>hySA}*Ys5DK%({xC6?XF4^xZxz2>MY-ZZUs zVExbkhITQhD{=aw6>oV>ON+nhvC|$@>2`@l^KiPbjYZ1I9^ifT3c%O6uIig?YjitQ z_LVI$YWH98pV-WMo$3fahMkkK&#Qttt=4U)%Vpt1W{`JaG+fymU+uXS*A$?_m zfF@y~lY#DLb*{ah^$6U4)TJXart;gf=g#L=U&yGYZYR}+86MrGSlT-?k9mxq9empg zrQL+%Ie7Sa-!zaxyh0~MdGdc=a_f!gv>jd1D7jrsLK=B(CqByTEsn|d(Iz^9kh|u0 z?M-a*v)yksn-2p5xMXzp$*K<}11!9KMjd}7ByT5GhB4ONvKNv}qAYzzq5`d+xv(M| zJ9CX5F7OwpaI({=pL41&K}rX9upa6)XQffv?giakzbD$jm>NHFJub5@D?h#5+lk^EAM(1lb$!a7n$~ot#Ei%-zJK?(km1d^ z{@UER0Ncfp@qJGxZP=!10zpbdJCxh^@daO`y4seMo=!SEifS`oW3zdDe|Q`^aX&xn zk)#Gg;e@^(Wv`iWU*KamLn}H6g}q(1OJk&|fPSsY2_mkXReo@T=PvHiu7ynba;*NC ze01`HV?*idf158ZVsg3r7+}yAZt~~i>?9?TZG0ocW__cI>*l0jgKfUqYU28Ax`#I} z#e;RP6<*6i4?P$z+mtNm8@hKyr1iRAx6;$KYPXo75&bAuTc+;7E4$Kv@r&OLDSals zJM=s!O3rP6J&`FuK`J}#s7|B* z8+X#!n~3Ter{-9s6Pbf7(gXKUeu3dZAqo@3%yk{AYy}JFKCb4`wH-H7ne#tNyoR`Pa1fu#ND|skzdHVM9 zABCm42JE*;JzfevHmk(annU;PmZSLOM5ZGPyZbvq?y>V*6;rv*D1I)rve% z7QUSOvoZ}iytEmYjiVRgG{@tN_UB=;<#~#}?F&AP$Cp)|=V}Zq;F>iqa;xa|`N9Zxk`Am64tz zI<+S5EB71RIgMUaVXLyl^KL>@zf0g{@~@1%AH1#HE9qr3XWR30zZpCJvmKYcssAs1 z3hKwGIC?1UX8q%Ubhih2BGf9%)ZBe9_+#hwAhbMvljj1&-lSHS_1Wa6Gg1yBf}#K0 z;>86M+5dIEBO3V^vH!)#(mf3O;?w)Gt*)sgf)UMKxQ98X^+2a9?-jC7kr61(Ea*N)YJCz8W|z{;ja?woh5x*Mi>_PpIYFLE%gv%D*=qFp#(UREbR z4d*aDt563GEE$IjsHX37kP~I(rbFiXBz4k#2D$r8ls*X>^|hD?GYwKw*o*j8j6~t9 z8%V~x{gz+CR8&Y&a=y!+H8r^@{>0vRLYx9E5w9MDY(}WUM{~GY?mVW~I`|_ce5Ae% z$v%`ub~%wPSXdd8r9$~Jm+$-=+%YiAWLMx;d@#~N!;;FNwxtup-6z=4T3voett=-j z7HKRb(otO5aoPPe_vtn>nMj&L;{Gk=@(KL=)_GadsI3StEiP8+NLo&dJnArdo2SoP z0EPU*U2Q)gKr*_)a64ZicQubh_;fP5p^z_auiwZ5Mje?|U~(U-d$p%-8>_K7(iwwX za*Q3A;*CMl?{WZx5X#psBD3AlQ`WB?T}W>^VHmg0=ijb-*;Y^9(&d5bTR!W0y0ZMo zwFh$~KY1w(Gs^-dpSy2+#LzDIjv5r}6^m4s_(1L-l&lDAohlgLE0aD!vDd5zi-yfQ zil2%Ne7wtww(yep7GanNF z{&2?t+Hqi{k7mDn?E1HQ^6ZL3n-BmrpT{NXiEY)}9wB`n7@nDJimGSs-W_hJN=Nf* zr%PJg0rvb&XgA1KUMMs^|4^)~1g$+g?BI^mr^q%h?v<}dXdG%(8i9j(txi*Z*;G$B zh+vCF$44tUewyjjN^cy~r!Uluw?c+JC@c6y0vSxpSF9V(WW5oL!_y=ME)|FHom-hC zqvBe0Sw9$<@AM@If+UZLbeqX3GbL7v85|wky!?6?((51dy z{L~{76N#=qZxolVX1$a@-1W|JzISS8BXd^|t2Qu{-&`Z{fPP}XlT>TkrPsfou>ZP( zh%H0Lv3~A5!7r}`O0`5=0K~CT>3bg|8ZzoD^b(V%it=mq#?6@Xr-c@r~u|ZETG~haK=&1XMkA8xB zy6pDBC-RD5PP1iib)j>))G>W|a^ayUUDlv*96m(D^?Z{m#NTBvt%GOz+j1QOr(>)( zkkv(y^|$o=ey-QAS~PS;_OR?CLdwkeX$j+l0pXLOSp?Ls7TCfIOliuO9QYnk;d=Xv zCw9JSAZq;k+ngtN{xvNwud5?DV6Jl=Plc&A&>6LnG0S`BK^YzY|6a!#+z~r5t@bPf z_yd@RCI8v}68!D!QbCJX9A{Jf5fH+%e+vqk%WYpHkxcztEH3La?%1}s8v6c_1 z27M*?EUqB>A1Mb@X8f34wRk1R${}6;(3OSpd3eU*7yH+g)5X`SD>wUww8(tcm6Z{? zac%w3@<{5?GWTiGPs~TPf!hu+a4st>jq$VFzFF-5W%3><=Mkc#Y}HTSj#^xEEuyZwpSJ zh$`(CkitW|u|kH6Tqn0}XYLLm>c`+eVsavedJJI>PW)-g6>itHADze~cz4)937+K+ z=kd+to((_Q2jHFUxq_j>U*aR&es5$wMd~$v>KvC%WWbV*W%g;GCLdoCqVz^Jz zU0ES_@W+2((2Z-BNRxp-V|MbJ6GVDzU{HB*EBD9k4hUn4N#N^|4Ic_}1tO0_G=qY! zaqmAiPxF{fe&2n<4J^6qBKMukuQy&VVnEJsYd;fJf740kB}nlXPKGdrXy24x8Q?fgM+kAR5wGj4mg=yVn$zEhrRnJ&AR zVyA*ChC|R6yuIpg+*?lPwO%PX3fp6b)CN>6vD_!L{9THHE_c2$;d+SSr{P2Ht>F|S z0&pS$N&NGWYKHDW+SUKVIdVh5aNU^c<9G4!E!#6 z!{i3o4u;x=tlox$=8F?zVq0OeA8GZy#`HP)Wk!x%oGM{FlpPY{<@L{EhL2UC`7y7p zbHPsX0t69PPo$cgD_)ZSt3et%3?JTRtGKM6(0FocS56;<(-i*A^SY$q+HsVr$lIAn zkP9IYa--#)2L~o9cAJ&6jpfV6>}`#&QQU`04C+yTH(M}p*kfXfqZh}2{1ky!+W&Cb z)R&Gg3WHsElem4TWo{Fj16fm~Fev;VpYfM=ucnmjn-3Ggljct9+xS^-dsBlW4!_5K z-0~re%NPd~9ZEKPWgExIFCpJuszy4 zazV{`dBD0BO|}r>rYs1T-y;xV!a?ZmR+b@Q7^ zm46pHJQA`i7<*E@0D`zR?$I3cqK0b|`rjB%+g(bn9+gC@$xrL;t3^xf}Fl5iwigv(~sqX68{6zgw(!2SlcQ_Lv4Vd zb>MjKD?I3kWB;K`fc%aKcqfE#*ciU~bKat@duwaAGN_{kZ&CDf6yXFRZtJW>3ULa{ z$nTeSW8F^DDB)^Id`rUga&FSZ0SP80jB|xVsHID~hB<%Q;UL>QTY5?r8-T;ev zFJS~BLjHY22{F*W#i0Kno>V6Ddy!heHqxsNcorS{8#R*gJD1JwwUhAWb?k$3Ww5F| z8_o3ay!jgxg$ikqX^EsJ=^}h9;}*S0Q>5`SIjL5@JIU205s1cGS9`%J_%M;(n7A#X z^>T&@EiW^!KbtXDm~RizWloqn#nS}rE1CH{%c9DvTQiwi5|kX5U8PWQ;+`vh^=PVo z2`N^$9}Cp0W>apWOz@zQlcR=Y}9mDHY@`112xDvV!;)Hxht zNLg{PGFAHWMy92>f1BN=KCSyf>N4lnz&>iU;5`mhHpAi0^Ea9I-;-4;+l&SWV9*Jj z7~WHdDa2@5=2Z8|>j`0YuO})wu90o=nV)?Qmy56?oT>BYMg9iwa+&7u`^Gm7&RXLl zSFYT-8u?I0)pW;0kr$`?TI2Zw^W}E3s+d)wAciM4$8qu%J(;kTyHDWzeJ@h+|Yr%tKA1w2vH-A(b%W%4dLuGn228v_<(yGsoYUqQI z|1+*`LhtRurt{R`n6Y={I55c>FITXKE)ZH9mj{?EG3XS$c;zPZ%VyrzYFl|d>E-Qs zyF?}dN(3C~GZY(GaqEk53E$b|Vdl*6 zi>bU)H=^iB&r$Or6vn|a&xKSI@dM>*V0FnyXQC)RGB9wo9-Ls5k}u!Dqo=q^9&b(C zZ7N!Gym5FV4)3WOMDM$ldF+7Oi&MVWQ?HXoiw>eW$lrHVtw|MLzBfvg%1Q(h4H@A) zeFo)f5Um|~`=`UFmEl~~7@Ai zF!#;*TP@*mBHp0f6l_G$5lTmso>EwPQoE>O?Jaz?Kcq@iH|?f#0LIDQyC%_@p^og* zXn0OH>G?Q}?+y?2pz?ug4~jRygObxnbN?a>MI-qtI+p{{Vjfu z?+R|=CF;M{b~yGe4{Hoj$tI)Ri}sCLIN-7>kvGAS`9# zAaaTL`w7_)Dv<9I0uHGPDI{(RE(Kr1tjkyIjabOn?_o22d|LL0Huec!W3HQH#g`JN ze5_I4^1F*Qo1sQ&y|`@G&CxIWD>a*n-T{HVHJj&9_g{AB6{in%DTZ(Brroq&enIX% zj!4YMajtBowv(g!o=Z7iPYvp|$5kpm%^_3vK^GTW{0_~llYZGOR3!8?BU0y#l}qPG zVR9Ru@7Ognlc!U30v`^G2XFSxVjq!86NJuz^zjl?6aE?R2N9AAA)t`bAp!jV(g=;h zgLa*ZRS20D@^x|$FV2xKQy06bn}bsH^|6-`(uj4*{cOxwLY6x{F|qW`$1P*`RJN7y zExc4VUw`%HaZ(i9$}2ix7tihr!`uXna%LM&O)W*_=xM{FsG`$5E?^ks z63J}&%YH-SGpKAeL;}UCmfz9QLJMw-K?@TM-Q{bx*D`w3e#$?tI9tDW5W#L5wEL|m zyLoduuGF4JXaueyLMOxm=Yl~&ZV;v`yb(f}g9N@#W}n{WbT)h2`oHG$B{|0I|P`o^Ie!X`YQhsxq@5`(^35qBGu+mt;Q zX*dM`5sQ@2$)#&sCeH~Aye*;I@$}x5Tv+;YBFMc99xQxH7<77!Tr94f_uI;=BRO&S zlH=B21}^O_i8yxjp7VTk@X7TIt&S!HO!T4r{I>9`J7P9#uABH5T;A1U(exraqLyAs@^E71 zob<8Z`WRiWdY+bdC*MAG7vD4z1{!)el7ABZ^R^^BA0vsZ z@8|c^5Vg`0{vi$I?}YvR(N1t+dN8QXW8?<&=voV=!F1s}-qmtm!#d@2UWmd!v&2yy=106lyI|!QahRI3xsx^p;Of_OA)D8`Mc9EMlCsQ}QOh<1Ox|utD1J zrYKp%BSb_lq{Rdyga5L&M9%6qL6Fug@EGeepK zFN7bD)X||v@!ccM!zF5+U_Lf@pu`}vN#l2Xx>%WUnh=pBlj*tUqE`!S=TBp^#LT`in&7g0GO;!m=MzLqvfg5Ia@{$fotV$hEx zk`Y3K1HQRwh47V_hBvGRXHV9+(zoJwI81PfuBJpG-NuCN^>8?1-fzPhJ%o6P5r~~P zo2sj1|H>GOQDa2MAJ2XNFrm3$eM~`L9NcH6^bAM?$1NpAk%drDA=$EOo!A z6p*WBe#WlOzP6g*c<#)`3C}k^4ja3>O;3%AR=j`o{6MF|#6GyW&)mx?k!Y!JZ#NtE zzw!k{!GN9aj>g8_-Nn@gE5eq60BRm2oM0)GqkXMt$mCZyOEzKMbSbx!-&EvxIfP}zKdgP5-$C`fr;WrI5}FJ1G+ z;RR|SO1|^3Gi_AVB{rl)tSZffoD8$(d$hddmxQ5hw%a5HtBA0xNyAE>L{16DM*s028U78xdL+ zvVENM0|%C*ukmd*&agS!}sqPofNhg_dmHmm<~yW2aA=>TXv(h-O*O_skz|}qMR%% zq1=6Ct`B!!V;lbr)TD=3M$$@nR%Qi=PL&>%H091cm>Nj?q_w_!${Q~y1T$B^scXU5 z`NQ|9$I^B(mvuts=F&#?IRn=A1FM`={Ah4Eod6HJ!%duQ;Niz*WmL|Ysw@h+l6ZZPN#eAZEHoVj>8OgcQ2bkqN0n^}T7ciY$c(lI46tEvQu`ULcQXmcvANCKs9%0)ao1*K?8jaR> znN9dZB^To&1>eGa$y+^r(^_XHyx3&7aBvjADVvEj0=8x1R9_-TJFY^T26$H5&6~{f z{woYEgpDHqMmd@BQZMTfL7BZQzGeUtROkJiek#pqVx_j(_(x5H5Vg4^UGK;96>+x3 zb+&8w4vXuD-CV4cv%6_ItcPdr?XY2kFKVYL1NYIqy=_;0Zj)4rFqDpfZ6vRV-`x8} zxL^HyITRCaY{F(TLi(uCupDX;;N%nV7>I`qV7F4|q|R=JC#N0ej(P4QrhGDyGzYo( z{p{l94qTw2vO{e~R}^OHDhwK zIiN+fVP5*4SyndW`lmm@MZk6&$MJ41fLBf%$Zu|bbHxSQfUuRIgsb5EzM-SXo}-YW$~x7i_7yK_ zTN3aFsQKn*n4NhUv=s9J3BhU8~1qK|uPu@}?aifl|BoAR4>p#?FT5Jg@mN(4l&gMRMmh@u4e# z+Ew3Mp=rafGW^;9#%*R&*1a}~*>_cU7=S;)Rs5x?xXW@5PBvXu`bKvs!Uu~sO~gH- z#;_Tc`QLCOVY&mXWjGv=mXg$a8$f$nHTSYu>bi(4f06Yu@6qo^uC50)VrBQ}aXb0> z-wcdOrJ-+%Jqq5`pkyl3*kt5o#XSxSchF8nC;E*sW3`vBD|vscF>q4BTaFhX@S({_ z_KRFg=ORO_=E+a3rXAvVYabzVTQHR*&g-?-XJPJx%q;uMJ6P&Q9Dp15(^CI;cn=!7 z-JRC4d*sc?Du#RA4CIJ>63!O}{86-R6F<%7I z&rauD($W>@^xnnRQ=t0~Xh~-h{~%qqmONOOk+B5%c`&1)bF@_&)z*gJ5uY9UuI33z ztNXs!Hno=`r4CfvfR$vXxw(ej3=6i+S&rY->Mf3jy{wSY+({!$220(4_rt9$f%qIU zhf*P(J=%@v$yA$F9`td|ZpHQLbNMSf$6q@}h(?~A?IK4&s|M5$rKq72E=&g`28->K zRDL~eG^%qExZ!-rbSe-`f2Ji~I1+W08UGdq))c|rrz+|A@e*_0_~O!36=}DtslTtG z@UjlAc_rJ(d$qE=5Xx^K-gzcn4H+s#hcu|eJhq$M>R?87hZ}p9&Yjly{ns8@;{$2; z`YxyWEOoG31-djpRi(it;Qau&VX7~O(jB^1dghKDEp-sFy9xAa&@PcYCp#`Lqc&Sx z&!{xNNAH%%8GA8p`0*K#R|BNK4qkFZ0^#uCV<2X__a(;)dj1W_S=M;n4F(NBvqafba{6Y2n;`$$PSZwT{*|H*%Vm|He4kymnUy zpoN_;eA;O#4qv!U(ypEUDnsodKj^V^e>~sPwU0~n+1Tk*7%3O}atrv9o^7Yr`tEDl zVvZy%xxNUHAJK*?xye7yO9kYb9xlz30ntuUD&2L}>$uzz4(~&y${YBYomowIh@;_-vyZ#fF8cv zd(V#OjoEx1r=~5onH<`1sYmJuGJ_!24H3K9wnFab2v^4G9LYx#LDPk z?x4Yj+}GYtK7+Elfi$BrU%_0LPh8CQni(f+1vQW-q?&okL_?@HHKy*ftOgs96#J5e zK#k*)MtY*Z(84yJyXBvbGsLvSDOkY9dgIDc)UpmUyrFM5W8814{fn9Zqt%e7MFr)3 zAn|gA!_$O36MMedL|bEkN6r?vW1W3Bo??nzX#7FaeF$g}rXx+I(xoCI<=QB`G6G`M z4@Wchgt*xPn4BdV!kAJ+6*UXaz_bl*BDmkO|9zp<4YIQHkUNx4hghabUud3n8raiI z7~xE;AEmdl)BN`@ipU@IF;#rFJ?5G%wt5*NZ#gFeTtU3*775&ou&BMRz8u-aT5GDP z!QJ<${?(;qFHuxg1TvQZlXPmw=?7B=rnS{#tLw$__)-f9#vf42=e* zL>2)=3LB8Vnv%~dj67bP0mRJQZS{6XQws+0WXsfEtvd7tFmE85C~JTAUr~II)<$^q zOtPU+JWFa}abPJA z3Mu59NeQ4zP-U9JI}ojV3ZQgxDNW74-nJ(J53*(Q(MC$<`pUhR=KJ@!LqcMZk_bgD zCbZ$MKz`5qNaCCh-9-Obq9};!V=4(YTEmTe+FSKv9`@vYQz8Tg3=?2fGJSTY`PkUs zfZmg%cx#VJ7}1_-i2^KejTAwtD*yLr#<~$;d8#ykGu$bAODFUehY)x7+jY&1=BN&| zskS}Tln06O?sLc1F9|b774?TWRr+P-uag^U&_d4xu*^8SiOG(KQf^-w8h4eH5XGtbwHj^!zVDYLCx1M5xE=yG_yT>}g#0lSoZOOJxcH@#( z;5u|9c6nIb%6SivZT}ihb%>m>Lm5PmyisgLD8I`U{T_(_)s32bEvOPh~8MPIHntwX7;7RS7-P;2z+q@Xf zEhCT8>wGlGf66|qlXyZ5$KhSLcdJwNcDQ8J%FU|EtYWcJ#4c|rrg49`D-e6L{%F?mn=WPF#b;Xq6o}NqXpV(pS0+z@Hz}3AH<_O{8x3iBK@(6qi zV6$;hp!a-TWo#r3ki7A90jIPlv6F*5I^f+M>IoQHT)8y#*j*!w1+({Tv#yG{cYW36 zM+uF?i=BtT8?8hFWnz@}+8|+XC}jDsVjd7GV*hnG@IboxPbG_B!~peLZAcf&n|GzD z%f3sGBe?ar0@UqxFm)E_^}be%1F?L*&MPv^7lB6w4&3K*`&>XIK}oyDj7j*jgGjsoxOPjg zzteY~*wr60*M-wM+G$jUOl*zLF?0_VrBkTK-CQJ%<75kFHkGMTG^k>~1mAhz&X30b zh+(zooJn<9h~+Vs!p)hX&tyrAwtd>)@5;W-d3lk)T)J%DEWmFS=ncJZ*)&}xCu~*D z9TULxo5Yitu65tMShx?qX#o8qgJxvB3M)fO3VE#KBe?vh8zv-fG6FQR&t1swKK$*Y z3uHM_p}!(!Wr5RQJVr^Ds-i1F=&cgv2ar$r%e%khOK!{$BYYOHld1Z)A_JRUy^K3q zQfh#%7b8aO4a(j0YKPrvtg?TnDS4%`MDPyi(oA^y@;lvJ4)(fw9PD6!*Z({H z9sedXYgSwOxLJff4R(0}6IHj52AF#Wk=nCZ{t$#k); zGoTy7Dd0z^f8o01b#{0HZ>dg=6Iz`*`xQv)Q*UL6?Yg8qG3X5ixxP)p*2A3}8SLWy z;H-g;L(Tco!AY16Uc5NkQ2Z5VndP><1;RiFYTk;=AKO3ET1wuSpERkATk2!)D&YfgPg5&Xb2JQRh*0%wLRv}W%s=Ma_UY_Rg1rm0;1D{mH7T84qwgl*2 z03tah?ji_gd3^}L4aRLvEpFp>thKu+8+?`dm?r~1aICua?kBnc9D<{sIEEdop+>y1^^*9E51CRykm+>K@AL3V@Z?EyUe~? ztC-~<&ASaUH&E4{K#lbA_XQf+FfDuR9Nm^vnBdK1tNWvS)GE^ zNT6523(Q$H=ntLxRFxOf^wleKKgIdZIWOFN`yO4iynM}<4?vM)`WeUfUdXE&s85V& zUhJg>K-v7Z{E$_AJ61)HLUiI$V3(4~lm6#Heh0zE=`yRj*XH3jC8Mh25bf43#O}dW zeA{2^kO z-}d*ze{g}`jL(8}{vNKi^LF0Haf5sJhK*zt9WdWK2T2figPEQZY_wLVg))Mo0pky! z19=-HkOHB?fFgv?I-bi-JhE(DZnITtHE znQ5SmBJ@(6knY3yRut#UEj$h-?+hkKU#@i0`w@c-Yr!BI-1HZD?$Q*_*%(Uzg79&+ zqbd0+6I_F*&mHoTrS4BLXlcS_kWF{zU36A7v7mMsJ}J19zxEwZJfo72^`5U^=s0$# zz3%b6^AZne>a1o%{kX)Sa;)*aRi?94<1h(EfIUMZZ{{?W{`fY!halhcmZ3_kG^Ui3 zk#3jG*~q&{$U$5oYggyHn;wUYUr5#YVW9eX9tRW|`NO(1;O8m-1ENY3w@JSh*AEq| zWVZdwRr~X1GY_#?YZa*#nacs%(8+Z-jut6x>eD;o)*pFYw)l-fzY{f2Iwkv$L$3# z>4XKJTg?x`&XF{=3dp7;I*xEN$*ssku3)jPq-R}00B3_X%3-4UTNFC7K^}Hz$mdBA ztBQ6!%EEK>_lt=}QRADRBBO%mRNADz3JM{VPqPs7NGN8v)2GWT4IJo{?|w&#`n;vT zRgSZ}z{TsBXuuh!s<$mBB0n8TJYH?N9<8+G_v+#Dheyqox0-u$QPU}`zH~r~!Ez*} zfrbr|mQE{Q%du?8QEmT>&E(z-!LvA?0RV`0!^1!mV*E!8J;_CKCm-Z;F&I=q=jAqg zVv1UcHC?xMjrV+>8~Nurmq;N0I6O#!N}>qDXtV2jWCZtX2Mf8feul$XE(M06_&zlo;$-V}8RSy)PFU2LlouAjm8 zR|F`4AsHj%1=-Ra_iiJ9D)s>)ukipX35Zr}DjZNjDkjDa(qH?&fEoEBb04v!g5HK^ zG4~&;h~VK8Ij?^n(BOK@{n4Bp3lj?B=2b7s*B)YfbzVxVstf%+v9o9kj1(RJqQZ-w z*x-HK+=sNK{fSX7K1UWXl>!i(Eb>S*AT2&vlE{PwVHFUJ0nSQqn!49YrEX==?OTqU zBt@Ghq_ZE$i;n&&vf3$ZvR;Rc7bm7qWISWy$L(It0f$rDT4~3#9-3a9*x^eJw3usaD&1Twl+=*rzN2vxNv{;#;p-AIQFPm!MWurb zrRH53A6(Je6c3<;m*QsVQao=VIy@Y6kz}*{#O_-L3i0q^Yfnm`qrKn3Cm{8*Bm&vU z2H-&hPD`*a(jg#BnXQ)|L#{{+iH^N@Zelo_@0mh;7-rg7%3ZK(9pD8JVyP6oo8}80f{S^xy30^vz)&Rab-l#R*R!kl~GYH~;duhLMo5VA=Sy zR;%RlZRe#WP>0!sxeIPDzQu}<-v-%(le1_?&53#CJPBWD1&?N>5qe7-;O!P@o6GFG z+<6wG9dEiq-*?U{Kv1rRuDINIbY|-G*B`Fn-!K>n0Vc9=kX*tB_YX7;Y%ni-e*d`s zIq8Nfaw(pdvwiT$g zZ{cIl057d@C~@JQzvA%*_$x3&YaL)BxyVFW7%u26E?f06t)wZu$G2 zyVRuAd3Aa?p#IY|Z)GkmU_{tk z$%xA)(#bB5k==Uw7Zc~U0{|L4r_EksJbE*fJoV`~=9Wdh{EXjU?pnqG3p0S=LZdLd zp;+{h|6=|4CEmW7bGNeS;;PwOCk=7lPy1l{KfqX^R~8c-JlI<(Bf|_dIA4ek!aMsr z-v^-f9s!kN-06+%rkjWc(W8B43r7djTRK@Y>8d3}$=~ljknT^N4 zE%7~Wb{m1aSz6AoPPj-*1vkWrfF?v_i)q&N?K@Oaf36lh5*9pWH{8^TB!A;eGD3>H|G8c0ObS+!KP0|R$$hGlbDa2b?cUWwpT;Eyjef%!nyAx!adki*^hOar7OW<>81 znd-15l7Rc^$r(F|I*?^H0AATaCD*U@;0{v25df#A77?}AF{n~= ztrn0u&vyh4n+oc0;$497N>%#!eX-lN`&p7CNTzJ?pzAM_r+oq}x#{6SX@w^_;-{6Jv>F-NqZF%}oBVt#_FN*_3=vN(gr* zs{XL|2Wwa>-YqZyGfOf`iF)pOJn|k4v;g@+Cd6FahiAU-U~f#`I3~d0Hyf$9dtNVH zz~uIRPW%Wsr48uN|E`2~wEKjfjDm3UcQd?Qx4V?zUrgs$|p%oIf z1tJN@o0JL_y)<+7N|USnr%hdDQHv-fKiJ`8Zyn1+8*V?>!PWWC#livD-!{K3M<7^(85wd4u3I$M=6!ePvkG-S)PRkA)Ia zGJ=2#14ufgfCIu1QUW4fN)9dEV9+To&CuPQ207Fq-O@t~GK}=Q(er zSbObz-RoX!`^io3;#LDeUdbqPw7FDbNVFl(e&jbinNWy<^A?fSYv>Xdc?t7wt7qRn zR^w+d<(L`e^WHq^0ITN>8UN9mJ*ww2SYJ`Nhmn-wb>{(E0Fc-PAcTz)@wLL(GIvN- zjDfkEXscrA>-5wt2qSmYsriO`w!*XpHcmBM*hymmATl`LIxW>UK(>-4kk2AGZiKqd zX$GT)x1(RA9M_4q{NTzFpbABedU|hr>ksa%CJL8KVw^3XzRnx$vIG9?8#ww6#+KOZ z<}eGxDC9%n_QMzZzB@C(D__h{35fte4sOM9W!XImP36TAU_-@tjrsm8Sa$Yi3Y_gTNZMNSShG&43%E1xw+)wMim z=2jDPu;ezJfck1oX?hm;=vwm9B4h{C=)UVBmBVUDR0LCfQ{;`;a zamN5D?=GK!MTGJ&=@AxvL;twGVkuE+fIXipM=oQK>hC&+KOcT_?*jlYdIZLSK02y7vBqR@t8pw2U1Hp{KnEOWw7O6a40%H}RHu z4EgcS+jz&XOO~+Q$PHA56H|Rb%B0RR4&&k;@}+R&KJ_>Pc0Y(Q=ue{L&u4c-jNFiF z9paj;5JZLB4_=#~l+LD+JBt+zC+&it!%^-cd#xJj_D)|<6h}u_$4wN&)9H7isBw|gK+`<=rcyx{E+klB$v6yEY!GdW$V{Au&ZOf> zI~IaI6rfUH+m3&IRm9y<7%wXF@M_;gQ4F1(~?$;0HS={T#)DPzKhtx6Wg7#sL* zw`%sMJhVc+PdTvgrmH#Z9in0nP4gemsJ#c%y~=)N=4;UKdgB~M@alVY z5{C4LcvXu3v^Dwt*+m%49my$&wJwBc;bh{lfZhJ#hJ06n-cXHR6f|As+X3Gl*FHrq zbzpL;Z+U-;KejzIv23X?M+c)?9+jp~zO=PkXm@Y-zGQbye#xGl{2Gx4VOxGx|I&@3 zE0o(63n772h=NM)M&!lYq_kSiK5x`u8W_JbpukIBTFu77Rl*l)2@K zw+A5E9jIV?i$Dlz0i{CJF&-uxviL&Kf;DqqN6S~Z{K(F>o{w?S4Ege~BqXwlT^l8P zOzH~Tqu_&r(_&xprLmKqRQGcL@)dO8Bm93?_@745wt;ohPE*|3unkj=pSX$eokb`f zGyMRE?=Yh(dK5t~?Eg))wyjFnyPtv<5$Jukv{q75R*bysby4-fvZB5 z(8|k?z=|lOB<8x9l#i=_H%|DojIG2a{-@#233j&;&W1S5Owfa}!ia`oh?}^`t}I>_ z`Jlw#YcrwyyR)YHkqBqZPXm^fadY2^JD-T0TCaWV+e&X+!>QQ1v8=nc?WPUCVNIZk ziEYcDqsJIXfcgAwGXcp){3N~mf{YsM0#*Mm?~WMEv?|$7{|XI3`O8FmJf@i~_8i(h ztl|^$JKg;>3MLY!O^UcD-$9wvB+DjT9#nl>Rm%GUmwNSPif1?|G!ogsy5&lA5S zvPtn(E+sLT&(~Rx+fYMWo%cN&UispRgxT~RWaRu+3!wO9R|0+lq4W@k7 zC&2vau=9Hzo}}!|>TjH%;@8BIXDsG1o|*@Hg4FvAZ|S45tzX&Rwt8`RSQyDw*< z3L4B5Ox$^-UG%+8Y2G64Yf6#wa&H>aOaHM_Cpbj>8LYOH(DWb^?YvhZqX^SXRadV~ z*jvG@sIdmiyxXH>1szrNRaQP$y50rKyYIhgj+{Lw+ZUak!HZ6(ygC_#pDh%bjd-`L zoasJpJKQEdu919orZB*`-qL(~|LoBZ?*`I+?v7WAh{xWV~*_I#ZfHD-p9%n%I?4d zWQ-bur7I(t6EkbbLwzY)z>VprXpxW{*L6`H z&7(pCd&ENbQRs?N@Jr%-O4>^5;yg8FNyS!G1smKC5~Qi|@E6g|%Q3+kqFKy(SQU>M zsCULIoT;juXpxT5Nmq)uX4$7hs;l45hbAh|8MrGYu>c)tWk^30D)+YojT_28NJNEa z{;B!-Gsq`HnC`ju4OE>V(f*e4xVu#&UBZOzuH)BM_NEP2azyI5_Qiipw-il0+78-r zj>YFbxx%b7$(x9mg!3MY)XE+CMpidB``Pt;zTJn7YVlRB^EOyI*;AEAE7jI}y#=~N zgIDHPIR$r0EMm`E;Ai=6fw&d-(?auE&>UkYfF(}Cx7yr#-{`DT`tKX#zHf~HvWG4q zk>{0-*&2>hj}UH6(mTp$M~X1>)P8HaANvs&?XlK+>!pKlzFX)3xG8?h`rf=EX?ybN z_;s6E$v5;oPU*$KR7gN%_~I!qe@OG?ss`J34LIs7U3%wPcRQCk)LW%N{J4bg=+v>7 zd-U8f{(K~}#~dNE#}z4;-bkuEugJPAiq!RPF%I2dep7Dn;;Uw#V7dv%f?(nOSgAwX z?ukTawE&wTu8Px+`8+J?fF-l?NT9a2-=ECxpA~MEbRdN1(V?eT?reVe;f@s@qQ~EL z6}%M$Vf_Jdt9_i&^YgRS?G}H%=^C@KNP0Mh)wSzld6R9b%6f#x@Txi=)`SOEBI+k& z`pb3_Lu=!DL$wWYg}oKo_Dw-}tt@YK=2s_^|9UI4&AEZa9H(8j8Fd2P-UR#(kzv`>&1sv%t5#Fuurd6x0Oaq~WG0lt?rS|4k3 zrv0}S-@RqfjsXHn((WByx=cBibz73iI2c8gXP_b0aJ2NM0d_78nF<5$aBWR!2kb$6 zIdqLN=;5WyS?C2Tp8N%z(#U##l~}>Rt!r~jvK^Y5j<`+|WR_nU;YdF;w81n!FP>MY zw!ovpY@tfnqP3-^g1TDo3;O(Ug$mqHa2cM$YYT|1M<;Jj7bXsoOABn+%QC~xW@;5G zY{bpiKMI^SBu1?=4`>T0?f|wrkt%G;{Wr7xXZhbuk)Pji{uS`L4OFAHQzKarBnTo) zbFCI!{bxvy;4sw1$CD>w1Dz1r;|`1UZ3>wNW<{5mMQO!%G_Fyf{G&&?1g?|IZP)XW z^$h6?-ZFShN*(^W$V6m2PvfhSL~gwe^`>d|6V~tf%J&^4p?w{syZ55$-Heb;*6&li$`F7l; z@Q#w%Kf^u$y&K2KiP|u<8$o1j_u*9K2cgk84e8F7G7Lk1>60&NF9@d<)5XJ3dudqz zEIVK;ho5FCf(?}@HZKHOhQMx8S5hlWR-neM7IBzYU0E*wI7rP|YM1KJVHziyjvNQC ztc#XQ?PFYI_Np{w_L|gI$Mqy}KFwzBwP{W^5KkBV^22ULG(-BejK9W&XCy~B+YciQ zXA;<48{#=ZT+x+;(nyiKWq=%KD3NDZ{5*esc#3b=fMo}_PSYC93};FB&MMCRCO7?| zxx`*~!hmQ7Uqh(ng=E$G2`UctE4#G#asM*7AzzRgyCe7V>rx%9^k|d> z`V*K0l7p!Q^v=ljJ>PL8ItR@Njs&nw{AqFcdt{ye$h2dollCwl1(p~qwCa6kjytXx z?CPg@TP!lsBieR#uJilR6<_1Pd+1cl#9=&bERUCCE$xPV6Mx=J54x@&V;s?&rph47 zQI9`}^MtM(EsJ8uJPK6z6*-QY$L}w>Vap=1ZC$e*B+7g5Dfn;G3l3+#aNA&aiIfA=(F@jReRC353_f^ zRre(3P(I5e=$;7@39P==l%ho@`JdK}zX!1TdJ0mXxI~xmUGGkb4_R=!8*(nhH+ux3 z;?nrR^SjEDCgBoM5ZIn|_nM_B3yqs(Uo2gXD4TM1ZW|~`k_-#Jt6+5=Oj=qF*b)G_ z6aM#hT~ zjX~C0|Ngtaze}(XFM@O5yguKsnJ7{1PZuvyr)ocI5vP9}Q^}zrl}UWNUGxTfpxxb& z^9sX1e{VJ-v{ z=p75j2+>{p7=|e9s-9}s$GMq=Ci~_*r;UCj&PlL1OI$Af!JZPa&wrXbpAX=}#nld_ zSVE=&Mw{9lS>%>sZta%*vNNGEM=k`NUuLOmJ#=n(SD}ebgZxvu{tqXgjj>b3umu+6 z4sDXX`WIKt#XagzM32uE-6B3xt_k9^C0kPKZ4yjh*D^3swJvb>7cT$p&mvSq0`3RW zaRrvA{ii$Y`S;&2b^)i@t3#r&YfS@;Ke6SG1YxN2sVgm_yN(XcH1H)5l>}Ge*CSkN z{FYf&kEm4#)33TLjtGI-QWlML`RYrKhMYF59)*&A&4HVbgljf1M6#5e!@L@J{w~4b z_j*LSVr9fm=UQANqq!4}EhVKEnL>;I{|=+o=%L@1NsIq3om3n~yF`G@^y(4|>hdsx z!~iLnETK;cKuC=F$;#T$0DN!LVB9||aS-flfxpsS)fbz%>uZ&iJ!-T0O_tw|%}RvC z>;2mm(^n3wa22sT3UPg;^1H(Mg>sE_Ld+%Yxc7_+c4Wq4f4hJIqf(!8qfCD$Z~stE zRNvPW>OxZ!oW+qI#i18|~7*0=S#$!zyxYihG9jM!z5pv-dl;oC{G%6L*#y zf4VuGZ^MNP-#cBvK1h(jN1FKReaH$#w!{fT@ebGT`uTYp7Z6{$F63vR{#^WYXLuLC2H5HJmm z(rj|}pG>?Mj_RkiYA$*r)sQplhP3)3_EwmZ@p8is1AgPt{N`L196qGP^y zbs}`_lFVNEbM971g_dNDe5v=evY@euqk2tFY|IvFClVV&El#i7WrJQ=cu= zw7t#ucMm=>CS=i}-~Q>j^LtcEAB`UPZDo%l!revBcPDDv0&+u=u-A}E0FDR_8zHq& zPCT_NIxI>o_cXMEpxID>&4>XfHRvwIOUmR@O>V*DwaIv8S1l?ls;fk%hq#<0$JY0B zZW3foytx)hHS!Op$B}836}!|k0l71r9mD4W62|)AzsK;Dkp;EdftluYLtc0OW-hvo zGQ@5QB)&-GZ&d>M8q69o2O2&dCEp#e)P1ElXDJbQ>1J1z_JlMay4wgv)q+xA79VE* z!A&RC^+kewtmE)R<9nGE?c9&w(}pyK%ueT1QgHGuDeI^KnP_c3{03Ziu+dR=bws$v ze*fLa>Gs1N(F|JMhLCj$B{3cW8Q!b#|M6sZx4WDOrxmLFvq6UpkLoQ4dUf%@rS0lh zp;P3%ivk`CDc&2@u4^01G9ss)Y~4>wdPX7Wk^bdZa%ORJ^R&2z8Y7#jvAcdWET;Le z=-xE?m(qYiS{66s*nBRQq%T%1!aArg>y8{v#%<~oC-xCT9)-)M&(UeYjOilk zqI*VUny51@;#_5hw(RHblMcNxoe2rBOZ(Hg@%K(HvIz#D8Xkbo2nH+**Iq-Tt5)YP z7$Cy{@kZxrteA8%5XzkmjBIfy07WYI9Sx`2&ed;e-Z>2QoPG2#8RC{N>sX~ZD$*Xp zE*M9lN9ZHY@??d!$8lKe9qR=01@%ebsRvKJ_`-F?O?G=PjEl9Rkl#0%=uMOkSrec2 znob$lYNNOd?a&BRk0g1r@bglH0Q|)phIDr9?~Z-H$9UmD+S&lSwd0O7heg+x8Dtu= z*w(rdSS&90Jo$thQghC&?7qo~0%3?4fx=Bh|8QRL1kNg> zXD`8BX08oKfU3>MN@g(%`K73_SL|y#R6l7w`*xS_>fuOI8+q&vgw*`OXHP>WWowDO z`81SQSiG7mA?N2460+%^HW+xN)v}H4t7}&OVxmAz(J9gG|9d$BjPAYJ(y<;IndQODkKYogXx`WiRfCsRPClxb`+S+?)m8zrLAf^= zWolfoqHaW$Gfd^*TS%p0h*RPW^b)i*|7dD5te0pbP0>HSWzVN@0zAn3$iZ4hpAm#} zMXH2iKDc83vb})lDR6u<-LbaYF*|f>r`9y!fH*hT21625>3y`FNPKBVTgHLwJajF)dnaq9=R<0cLSCz&2UO$v&QTe&N-8dE*E@)|?7 zV!Ry1cHEZsh*fKG#?58D3{ed7h1Wf8cuS?LnYgoAYtQWggdudG{DH-60>b>((is~! zM}w#7e7_loHAC@L#sRRyzvTYYn;&fN8;wU102+d;na7Ws&>B!4`_a$t`I*?T?*&$f z&UvMU&erFUU6r-_U^De|Koa*>QTGkZL3wMg4U<@;6>7Y0)V~(2MMQ-V9s6lnBvnlE zUHZb8q`c~@M*>r-71dnQ91=o)c5q%diAu8pe$)?$?))h|gx#8<#Q#~h^+M!L8VqNF-5+K@yky_Yt2lbA zpL{&=?`+4&~H zt6n$n_SjvS-ioW2%dt{#FN~Ash{GFTNo5>bU0$gTmzzCw`)Nu(ce+HfN;mQNL`8?@ z;s#3Qv^jdW#G~QM|IkXml3o%{!}I;K=@Ef=^OagMbpAvoSj4IG5y>JW8@!oBOd9E^ zb)WrY^4qc+%*%@F;@IXlI@Z>hMnQd@9mdB+tQ$tsMRha0@cT^b?$D#%=0&%}x%asc z>S`U$5yLMNbvI3>R1MSId)rKuo>lwng$*LOLTNaA9`sukCM$DR=slNNrj_?>;uP9C$+#Dx z9=jXi>K{6e_$lm1#x%H7{o7s?)zO*H+ime6iwuYKToeSW? zYz3Ct9TdPJo9I^rGh|s`8D68zA`zv&nZ2Fx08_EdnqpKOrmYE-*gS!1ecZzq^%do+ zobPwphh>fF)ae=@(N=g%<1t{lZ9iXo=*M)>paP1GH`{@^64C$hndiT+lK_GVokj3a zA_5%a71;`!5$*O!iU@F0$2 zW*?Sa-tSPQiBHtF8*w%Z4??w;#RRy06I}^lLA@Y6!G0=!Anf(zt~}p+Yg}=IjRI`6 zqiB>kWE?PN6XhEpM-M};4vtHCr%AQk<_aVemN(ypD7VP|3_8En+pJpK7;*^{j%}2r zMc6+_!2a)blJk+z-}CvM6DpVCRe`;zGFy;kS$}f!*Z?U-c;(}V*_%9m&E><7f8-55 zKCRh$F!RbarsAdn%P3+TB5O%o#^my2KHJ&6cbXqfQM2AZYBs|eOFH`?wt`N;uN+)) zhW8C(R*kc`YyqfWd&o^!n0#Yyha0@0XYAmd47KRQ^C!UWB%g%|HNV#y~BleU+oYF|DDe>xc>oyCY%1f z0rp?71BcNU&QcB={eZu6vG}S8PGIXGr>|JOdd)@Kmn>CyqVxUWvD0~s`SsRILSuIj z%FQ-DC}=QUzKp3>UY3p8uaI5RmgH9Be@{n?0DXBHA%|Xh^KOZ6YnE6_eDtipc~T4` zM2nS@^9KrZOPx4V7mM9y6yC(D_e?U(G|hoZ_NLAE!TDSnG@G=Cs+%Ike(VDR^5bbjjcl=lFJi5*k(k+O{Epy^zS5nx*B#r1-d~f8l;!2qi5F=}X zTdnIlm&8|ddof8{vkGnB9Q?9y1FiMvT;rG)>FV8+FoC3h#nrEME0l3yb}aS!2gW;7Upp&Ye~RRQm$G*;0zmQutb3lXc}5Kp)K2@|Eub`6Xus0%WDBr8 z)ZMYewCMl=+78Nj%Ot>}*#OXEu~l0`wb+biCeN_$$yTJCZv4tWatQTfLnWoE<-29m zvRo^DU3uiZTzT4__4-QI5>&a^C$?woh`&5i>_FItyZusP&%~8a5f8~U`E;SnTD!5f&BW1lF5aaP73}r*vSjpy46UM zHSL+2n_?G<>dUGr!J?dHhq7q%y?V5lONLfV=ZatJD4vtn)0Dc1y!-vW|6x14|L{{z zHc59P$;k-3&2z2H0s!eqhQ&c)?z%OwFJ-1^DG{$aQZO|Xczc56ZD|J%n zr%Js)aY^!LzQF&{fhav{4MtsDMQt<{j&ibpgYU`v^6o4|Rc^hYr$r=kP9aFn!?OlI z@;D4gcS-A+&6_Fhj)Tb&YDAr8Gr;7Kjb=((#UB7`MXTlT!vHDS@8u2dMLSruOYt)W z@43AMVP(QlR#aXqX4lho#Uo3zA8Z#s8osp`d+&A8@TXu@Ayh+{DoC@!)KrfuSKltK zuTIu9Im?nTG*zMj&z(En!tiOu;D*m3yHEsYsxSxFTxfvSl$1{D7a)W6o)#M6GHL91 z`A5iqexYATyFe;Q2<^Uqt*YF*ws6i7GXi@bXl}|4})Nxx8+m{vtNbDBN zZ)_pj*RCt+Oum;5(e`|9GoBQzfvz7#7%~mC8;>^j+1Lf&Jt=6Kp*}Hrw`31})^I^_ za-M(J4g0SA7M1xwOv-=7TyMk??67G2m$914wNc*3q%`S82>{*-jd04Gdg-2q`g(0+ zr+`Z$h3=JEJi7T>%6snL)HUtVR*-Qwu5=(%XbfB-H}#H$@w&n}$9APDTvmaS9BghJjk&Jb2CL~BQ{!AMQ}<5v`k2l{;(CK3y>LrlEb z8WAU=7Nh)J6RlfZfg-!;d2b6`nE?x%y9miKta-mKgJVa$*w^R#)JZ|A$G5kuQHRi2VflbH(Z<@7!zJn4N&d-QMh5j7CrpbSUyZ4PDoTDG-;IvoSvi4S!7CqiaiLYMA|W_O%sKn zwGwDJ_>2%bLagE1@Q$t>gD!2n>_}deENrv{WEXkVy}eEhBYnQ_w5XhiDK@n5YV5Ua z%lIYe05@wOai`~VxreEGZ(F-pW|T_0A9JD_HAU1XHxv`Iy*mZ(HNCUYV2AUb=rfQr zD<2R4nb7&~-Lsp9guN~#Rt-WufIR0_RCKsQWnKR*j`lxxnI9)p{M|@L5@7dB22rwPA>i=(*o}^xN;z zU(GONQBL27o=y4RN^S%yywfmiyWaQZ7ZlZ+*mw(n40E`$Qcbm9SJNYdLrAS6YN72M z4c^T9`k^un(9gSdp;B2KcLQFH5v)?y`&aj#p3_qXoJovm&s%@%(p*UZ(Vjf$sY6iY zQIc`^wU4)7NoVU9j;n|gCWpm`J##C-7|F66uvd9lgMIecEbP~RWuv5P3;C?OCq zWJbzQa4>sCQcrSEBLZps8fx=@miuop$^Q?QJ5Wez1C-?&qq?}@;A0umgHE=Sj4Y#B zA461&bIFDkM*_=xksrPSzKmyI63L+SG9J{k_00HNL!e`t1po^mc4SkS_!yp~XAg|+ z7YNJ_D?pH0PUDwNdXD@_!PBNQYrMeIk8zs_Zv{Ug*6{2HqP13)s)=;ssF~`}<_q>2 zN#o7TcvUX9{6~3ovf%+Eob~?vp0e)u^pd#43tq5=s!LkzC7PGujrvP1=C)Vqgzg?( z8+B^l$4moS?3nMoB;?QEUD2B$#Dp(bqX?Q8u{6~MO!Yo$&Gh_BJo3BEsZXpW{R9X> zk~~I2{~=E7Y;HOHUdx8vtNRBh?K_f@60yB=j{${`N{*HQwX zgVe9k-*K+1l{sC~9+fD}a`=k+=`O9_Y4cq%QL0^O_oLCX<(DG7MT`k9GJ48Kl0^wr zh(JoX?e-*yZfY|FAYTr1bzDK^d+!s%lT2~G+V5#C+rRQbJKI#Xy#66G6)!P0zSE!1 z4nQfQTeICs=5)a)-g&xwnhiI~?rO&1h7}{T(%O-?`1f)-r*$Ot$y;zyH8P$Db3c=M z!i(C@^X8JO95W^2G^EhpiQ>r{8O7c>GRh zn&iWMn=94YC)K0X#SPm5)VRf2#I#R?e>{xnzKL*UTiOLtxoHgAMc*!8wyh20Y}2*f zCuLdU+9a9k1=xYy^I$v_G8&WX3EcP+N3Q!+m3^>~yaT~fA#G{4z0tMRw-_0PG_m;t z-Q5wXc90+x%KTBVp;0M#`Y?Mi<)u<(e|p8`XWu)V98N5M_K*lRP&k4Z%QvAZnODjCp$)w`#|hiWKY%q_f(AGlI#itB5!(fuMm(nf2DrRh2CMeZA5nPc(?tEguJlk*0F z&v;a}{Ok;xpoLGe!2kor8Dx05eSJT2VNjEy#(G@4V;_?diG|nM3{$iQ#|~9b)2ZjF zQI`bqQ^!bo-!)}k_|Y)2;w^>8$c*zfN^9^@{Y8yGX$1GB0$`-T5!E<(wG#Nacd8vT z=|JXbVzYcj(E0cdG3S(CNXj&zDd6>>IjU z0leJ~VPN2j{!*(tWtMJ;rNSLKDF|YyUz&3!-gn2A<2zOD*JI%eYZ(G005aw9or>hu zSo99ot)xu}l-$53U9Rl=dDBq-A&Zb%&EkeUlkq5)ZbIaYN9WjA@IGt7ua>{`n0L)D zK~A6jknCKDu?v6vqEad&5oF1jaq<Q~eb}J#&HATzVjGaA^N474){hYbN^}XCMD$V=>wkk$XQ{93{DjLIUfbF2n zFW@)~60yfedx9`z@vx@IuzL16yfU*pIxjAG$H)BYZIG=ksAQ;>Z(J*1gM^Lky9J3AR$Xy>4B|35;_Elq zUYR|Mke^6}KnKl@ECUu{L_Ix%-Zg7gzEs4yk||!|H-#!-?h324ioOTt=u)Lg_0ytg z>n1>GaSZAs3jBVtKMj$}{$);|Cisj~qult3I=oH*25}&18p`e72p$n#0k7H4j|(5J zZ;m~q$WCu-;K`XZpjE!?W_+rbCn`o(1Zs|Kb(U(5@CDd(uL7qo71%Zt*pFOBEw`5^ zk=kaHoFAxMzggF1MkZ~fA@TEE;QFigx<%zZgU5VqiIkB$qx?(9-P#rbt4zyB&J{*C zxc7YTWJ9UaJCWL-ft?oU+i8XE2}_4S4#xzef0creGr~P2n-W1ng;Ow)%}jC3$K6s4q{`X4qR8|mQrmP zvfT*z@`r{^4{ZX7!74X8LGr8>SWXP7F@jLeVbpA6D?4 zm;gzqc0t{M2{x%Ryr@z@t0!lh$BT-Z0+9!6xq6>v$#m4T$V%@}%6}@Tnh)5@>IBUbQ&&8b9!h zRJnd>KgaIfKkEWpu6-qqWLqk(+c8Wa_yXC+#rNFFslOBOHuQ$`w5V5|=r#;qmfxmw zZM8qFs8+Til$N|Yo~XZ^JLj`+7^?Mhoq*h2m1|Es$ga_Rnc0FF3J-`6es2~<5mgso z70P-`WWgdgYuwuiZhpz|05u?J)B}B}9PlztgVdrHqY2Dz!#ezded`x}e?A+(h!P3j zlDHzdU3^SFjp*{8znb$IVHRBy`Zo1eZ} z9wq*COv-NYQ=*R!b5R4J6Hwm}?d|i!3Ml8M3?2sNMge>3lq+CI(c4X8;d}2R6#$jN zCFq`GMIUSZ4Hn%=9NOMoBWke%$X@wSt-FdvnZOHU`a!8SPuPZM$tn>EmtVP_HUCTD zfs4xM4MLV`YW08J4>kGv*ss{ecs3aJcFU#j1t7o5x?G;dasl=(o9}c@@-%=&d4-U2 zsDbHct#$6G{bQE;e2W!8K0IbRzC`(wtS$0)Mc;3Ddh>FJLMlu&MPP|y(vQy7qu$4) z#6P7|L2@?s?7^VJO$0s)d!u`9W5qe_^CjURAhQFaX6|pCEshGO16u8e{qp1LdNTKB z$+HcylWpZBassYVsMbrtOJs4BdeuJV&HCv#J5?+oCIBZCuGSdgCLbth>+{oKA#Pir zZs$?q?&((@ZgEl9G#CN7=RfmeBtYzVqkYpd^;6`l0&77~fYH)pfoIMy6Ee+}oL`0x zgjKzpJmkl}Palw5=FD6Fk@4a#Jo;3XNN)N6#y93F$zlH+@XYETe%9;oMc)G z_|x>0@=UKQo2SXba@I5Xb=N1I{%KD99Xj-mr(wt1HS7JIxK7AKmCmoml3xOUvDr5t zJy`7Ci^{$56`VG1{J0C0w~H=ek+w0aDft(sf|M4~b(O^FEJ*?Gx3viasc>*RGEj{w zwEu{RXVoJ)X>!l{QUqK;`B?&CrC`(o)oe^=Ssin z1@{JTO8iVuh6y`UeqP2EEmu9UpQ59NC@ng0Y~8j>lOcPSzNk;8=(n3K;mVNSM3eH) z)6^eg*vP0AIUuH-SC%Sp-~2V}k0@23O8DDnd|~$0iw7-=bH4{zA)@Gsv%urd9n-7c zapVHn8l_3Ut)o!;5ZSCF_dsA*UH=k;cD(VRmYe~!T3Jmwg6)AY;qC0rXE#Blsm9Tk zx(9NvP+ciEc8A&Lk{vz@*Ax|AO#wbN!h=!TSavN7sGVZeGSDHL5|WL)Tagcj?oFF( z&YGJ{ucKg5SRa0t*cCn&JW#;hYw0#G5m-*cnn{ky69ow!7mw7@5A?Bh|6qvyjEM(Q ztJ65cuHV&~zYj^kJMwLmcxM=dSYh{9BN;kj*_rg6@}%F4o56Yj7{AP(pbEfM0H$RF zp1q@_zjIuqqUKI3=7jF>ixV@0#gAn%k|kLd8iXjTN@X}q&3P(IgvPJ1k%)Xag;f|&hw zr6ur@x#f4`dK4bAph0AKyX|@Q70Dcd5|AM=J>WY|Gxqqw0|crJ(x3i5?~gjHqQ^mz zAPqe!B1%-G#HW27HD5pbE7X1?rtV`Hii>@znd3}vWKXnabt{pGpsVu&fogm$v^ zf@wJssla$Qi{? zt(fUbD!qjm%;Nymy;N&1zgvT#^ytIM2D$cC8*AS9-CxOuSXCzj7C~YlZRc@(UFKf! z^+`#D_PD9+v5R%+)W`FT&*j~X{B2FZ8Cp`mvhyY`T5viChqqsrCg;9%yhO*I=u~g? zncqwUNcNtl4+S76{eBs+l-c*H@UP5AB2OaSpKK+LUhYW|Z;=281xVk3=rHuOQVoLD zIToBofL)rC=9)ZuYqBcv)q&VR)2#LHJ>u@jk6Lw>2hRap(zq4AryobOIdBgJOaeC6jMOvo~oGEl&LlBQT32$X5XJhLRnEroOwDa)7&JxV4`S04 z9OiwD9f>EJ_qlyWO*&LR^81@PAM-i|JVw3^^W@V(WZrxmM)c9v{FU7~q60?(kg1#I z`7B$P`ho=%+&#|-Q%H3nJSmm0OWz}y8|Ub30)8Ey$D>;Dq z^ONw>LzA1|?ph?#QQuL!Y%KqXm@{4+Xs=Ci2GdRzifcfcu4zPK9K}**{+X|Qm1B{0 z!eXRNK7AuCm*Xb7GqP@x;g?L0`IX)^o5Tn7($lAM_5SAddQ_d{|CPeLl;OT%@{sm= zo&WU}Pg0;(2IZZOsrxs>k2rh`F*ZQ)LKVyH{pq&FB<5P`n9$g?q^roD^+p?JQisgb zV7mu%$Krc@lP?vbOS-eemN{M}`zHuEpNd4fJcCt34FGSkm~o`x*?oN zTCiLlY@em07HC^=oaWyymk#9#L%Xgt7Qe2!6;rP}s%o)wMH1Tg;uP~WvM${sEzsLr zZu3!4{aLy4F0&4aYSpRu$NvA~MBknWLoCx^0!azR-=$)tg1O$1=;>fLX>eJl*F=}}&0O?%*I_6e>6xa1@+RGN$oGxor% z_Xxv-Hi=yy`2iQFTmHS=tYfW#en*t-q41Q{Ip@IZ+j{hT>_qszoxWg+|MP-nJDKRQ zimeA@V#?ZgqXD%tkoI^hjU~2+Qu#!FSG|=YjDh2T+-$XFJ~G#Kp7pN~=hD+2?vGiU zBINY^SIx-Sma<%1ig|BZV#T zn$3ozmR^B~=c22Vx)vcwwuiCTJv)wWSBEoL4 zxXL&@sp6<A*d?@^vxw(K>8Rp6 zG38y`-SI^dV+YoWWp9aKsC^r*<_#f}vhjj`zSfzV=&`ShKGAP4eEzctmoY&n;V>I9 zvaJ}3usgH}$qB^Q?zx0+y@a4uM%kihZTp8Xx7t@M2fyeyng-sd>o=GHru?8^9=zQA z;{o0PNsuf7EJW-|klEJ{zF?m%I1aHL>}-uX1jfcSL`z9AZnY3J>jzWy5sPKua}kgm z2iDk&qZ0}s^zj-uXdHN@rv0v#sUoNEx^^I%=d&i^fJfusP-cOwDgCDFJ7LFf zGs*TK@6!9Mx4dGycW>rf zEo?ulYUA~Jl3m&>FgFHLlTW1s1usrp4``O&SlMdfes%;rwh0->fOT%uj1e(+39E(n zFH+NcT=8t}`lmcc4u4ZM_sf%+24&L6pO$@bU9uOv<5q|OvVZVCE5g~M{siLoK_XSt zg7`3FNp&$S{T{tH?=s4C!8hx%7}@mBIuCCvwTp=jXc?jbB3+KJiE(HLFFlf&)dQ~4 z9zTCD&uCWb$~FyG06tnNv+kzP4AuK@0eT7tK4vwOoUJSjj`d8k^BlS-Anen>fw)p1 zyWL2vt}(K$F={cPW?^EpiLO3X;)wD<^qU2Kd(YY>Jqo#Chtkgl(YqFNc)SD`t2&dV z^mgVQJ?^+QRO>rkZuQKTga15EdcLD+9|ZHvdhg(s<{H0)+Wv;C@X8eK`0e|=sbdrO z-E#9>R_?lqlvM%$tNTD;WGs8Bu8JQ7>)7eeAy+S(?Zq}soIKP38vzKMrpVrSB9iTY zXvz7(qe<(im9;8*Ww^u3nR`At)!92h{3f3q#!^Xjwv`kNQWf3qBP8RNGqgHMc{XHg z-lay(vpIy0R(_$Q2>&6J!qHB)@MPxa_jJ_H(hFk6T@$m1z~VulFl?L;FaE%L?`ahSln*Wl#Wvx(JjMaOk>UUNN`t^EN4AT zieBMCp&1s@@AhuNV+8`JuXDa-k)ebaXfjCVIX$^!9d7>ydes2OfGn3rHlq}Mrp32s zcuYER#*>ukgDY?!0JrH@<4&;ab7W`G#9|-JUU8dC`JgF6T%$H&S?sLEt zh|J2o`yOCIuQdtBoDk9?1Xq?cBKNLQhFy(1Jc6SE`x+sEcSELfGnUOL+C^bpI1~5X zSrR4uhbJ%pG8ZVS1<~O3Nq$aowY*73-KtwWfw4$|Aw8R(lC`MDk>+29eYcl>8)K-c z|6j%!5qZRz5MjgIK8FdRf7omI4O)3xF*D~UNlNU0%T{Ku&edB|XIF7|`zXSp#Q(}k z>@Q`<_f!agk+>cGKqan1X02998(t=U^g#r$BfY@( z@O{@7bp^cGyFlb8ImtLzwy4S+?%H&l{1|%yA%4rEZAqpMUAIX`8cHGth$_q&38veFQNnWw1FN3ZpX-{&uWD7-8yT&mL$25|#78 z^f{ve9B${c@c;8j`!|87rry$ps#Ys-{uFFsr{DyC%B%RE<9RYxTuG(A0Ybb$}R+IS35FTYs|izhS4?!6>;f(1ssG zE_6&OSuL{eU8X~5FY}HOzaE;PAdvuohAmvQ069i0VV~A8OkYG&{x7rGWS-vu=x61~ zVi6XeO?|uqO~==q9u94H1cgu#6He^n@26NdR!TS{DAm>c0~N_fUzrS5jznJwyEhj_ zgcHqb=bhIOTQI{KCj+lgdds=JKY;2-GX2X3!@EEc;CDZ#rk^J|=J6sE##s9&>PN#+ zolO6!e}jdjZve1A>wxU_Jp{Nd6pq#N^&fVX#{v1U$rXDg$haG3Wpa&(1F(b6XzYv5 zlBbDYfg>9XbNL*^f%KA1Cn+aky>HNIv*W8i6|5z?wcWFQ26h3g%aXQ?Qz)=? z>0I!&D%E?3ps*}39S%=c#{$EEeNzshK))_AK&3KmSlTDu_y1@2lK>`G$vd{Yay`%oC?-vc3b%Y-)3{wY7}iO91?RBEgF=S5WNZ zRg-xl*BEL1CipU>_q*?Vn_(ehA`r?IH8g&CIoRFtVfQ_vj13|7^ET*%WMH}J{9`%y z3{GMLHK=InQ}tip`XnRbC3LLQ+!G;Xecq~N?BM5?98R6#H>;HCIsN_(eG`$xQy?SL zBrmy7{tX&9ithfAgUH76$r*Zd4ZbI3JcM{bsG~_JhiQ*gnw)cK6Ikwy#G4sl^oN=D z+9y=x!i`pQeCj*?X(F9IfrzLb5?=>);uPGL9h@Q{>pZ=2I-wazaS>ih{??_CsXroSbW^?jvb-kUY>)tC{@8Q{@X_T2#LUkb_D?17nq({R@X21L46+=oZBm?>P@zS z96ntpMs2@=utZgnVS6qCm}ZpyBlJ-ez=&jV5B1NuO~Dgd)+<4A2iD=V7hGEa;J@kG z`##!{4I#o6SxBbpK1Xy^9O@HCpAYE&*Q;`A##m&%Iwo@`dGw$!BgAqFp4Cr`#)rK@ zmsyp9;XYKl@-rCk_5a-s_rINJ9{B@)W$oc(zx0DuBITi|UvfQ%mN6Aag$azI`>=&A zWi?}wGYNJ0ENEq_!6m%=M%wp@)G4M`zuBt<@K#^H8ThWmBZ7eW4LEDfh;8?Svim>; z0EC`=twT61kz~Nv5XQ9*Trr^3ty8&p9st!R3$&meVE=+!h^bs$f6O8S%sv#r^1fVY z&w`E=QQ`D1$q>|-VK&BGl6lIY6;?uZ0{qkPoK8gsjG}}Jp~KKs;`w-y6Aaj&75ZP= zeHt1yBl0Fk#b@>g9n$TI<356-&Ca!}?lPVR1Q7cZL3d(OCrfsI0L~I{fqV-zCil$Z zLmE45XzJH=fb!rys99$Qu<<>@Vt~wyZ3k@m4}}-8J5P;H;+>yi#_C&!&#})jy+DFT z_`8<-Q&@MgiKtDjS`%qG>7mn>K8I~%wva;1@>I|iosoi+55m3mB5fvDou@FCdmOo7 zOeXU1xs0R_b`}^QOr9#w&-a>Lzxmt5gQoL+-E9G} zSD&cKUbeC*6FoddJv&bKO(6A=qo)h_HK|sn#+BTkjDoH>fgxm^-Kn)dad|Tg-CAjh zvmY7?Li?E~b-(fS0@16wL&xk)ReEw}8wN;S<@xnEy>HH(Aa9^%N@pjDu6zTq?p+)F zRhCa}b9)oX?ZV!(GjZ9I<;ez&LV}}b=wte-)AMa|QRW2UvE*A)gF6Avy)}b?&6kdcMeXnb81Dd;49`HuiLtkPw1_oSe zhbF6bRe>)I@c4y3dTLaEb_fHxHS5{ z9INzT>e&m>k&cvECpSVpey`#HoT00P)t~MzvF!qBjGchXVwMeaO0+0yuiTdX4r~bM zD@mf}3-6M^P3UZ2K1ZZP`BXEtN(*heri_c71{<(%^tpm}{0@WhLN?6MLx{bQ{q4hL z&j^sWC3o^!DiU5w=VW=xUlBcWyuA}7ZUr?dk%K;<8aFc=@oGdeyo<{2jOpF8cU}w4 zsa)dln@*x<=Sx5(z~z*_`K@MNnQIU!#CKN5>y3dJUXbzr6p=USiBrp1eCHf{Z!8~I z*wG)@Z2f99X_!?Wy zgl*`)-5yb?ZgD|(tQ2h7El(k&97h>c8YW0gJOI)rT$L@3dkBiH(*I%gN-fQ9fOU&> z!7kfh)LCh|v6K<0)37(MFSX4D*=(id*>^T^bZZ7c{1dGnt&D|i(3wI6CbFwfGVn?e z(frTfcIHR;?vuA+9SAr7P9(PO}Hl>l~m}27n%AOtdT2V~77=l!rdr{3?{gUUs6s5O!@D1G3v1{C4N9m+NAK z<$hp*iP~Rt_)&9folND^5q|Iu2MRy!C+-y{o3=7LWxkI zq-MXc+@!6rvmo}CtpeC9h~boH)NEda<$>fZI)7qd)gNyB@~BW8GKeJToEJPWh;JFo zcLL4oh@KHpHQ#{t?dD&CE7SBZ2QWuxWy@fI%g?rQMaAw-j? zde;JJ8Ucp3hp-LELrthGb)?q3zE&=RL3{xe+q1K$f(yR%-=4{p2aHXH(@)8KVrF@e zj@w#mDQu4;^TBRnNX!b+0>dAhfDRKpQ-q%wvUz|b^XyLhG?nzNC_w0MNqN(KWh`9O zJ8sxhk{3Ew|J+ue(iciU<>dK4gaZBy!v6urgKKA98Z|B>wEOtxN;)TEx`nmBAt+vj zih_Y9j<}uL1yMJ8;J{XSu_JI&>TkgL0CuiaCQ4zzZ3_%dccKw3PjoC(-h$$C*#ST5 zqLyC`C=ppBq3AQPyj3t{IU+jK-st6aXX>P5-1%6qbKvYe=0q0FCM+8%jWSRmaIU?8 zHF;LyVeoFx$0WM1>t1jBoz|K_3-n473LkBBW#NcLI&)$g$Ph|)O-B!9#6t=E5Z z9m#B9)eZIsk66Z$c6wcawe(07CgQ0RSwn+NzwbT`Bjz_|AN;w?b>w>X|GxFpH|Hww zRx%oC_)LGf7K#Q?8pL;)KAZ@n+3Tod!cg{ixQ-gicX|#ju_d1q#f_K9wUl_aA z6?meNflDp`B`Mh2M)J&EN6>I+c?9?eKzcEjrRM#_^+QuE>AdR0yt8lWo*?JG6hz8_ zK`BK9joo3;jZriZd!|QaXDf5R{xQC$`BHlJ8W48miZuF@pSmum;_t${W4HbZ6}0&= zk;SfZ6jf=vn#~(>Bds7W*_kFuUM1~?D-GUiLearzur)$TM%V8yC~jr;I%M@_BqNT4 zU%wcfDv>ts@gK4xqsZQ4&+7JQFtX=!5ecHL*_Rm7JmL&aa)Xc&4mhfc z2-bm?uL7u)1mzy8qzDHDDE@?0n;v#FO^Sw1-fH%Kk3QRbKx&^OzZ70ub1KPDq-6;;Sw`@L=mH@XGi+4a>xm3d^l5OO?a+ zyWj14ebVHJcKf_8lQZ+$8@QXjiVq_KGUnKN4N*J#bwjqpsdV2_{zJoCgiMyYM}Pa` zaN)$nM9%6`Vc}9|`%Whf7ko)m**Uk3sqJ-VMJQ5}{ zMZZWZ8U^XTcXYUJF7-9FtTVK>L233*uP}ubomSi6XUtNh4msQe1I7rkauu_zIjvm$ zb{DJtHS9rQ%x8^>hi~B(HCl$ZpvXq$(T_rTl8IG_kFTJZI!d42cD_#I%MnSu1G6HB zd*1{#N3_@Q+o2PBsEh$H25jgF9cN&q#ns-VP%*&57#1Wbl`C7mWN}GdY>Myv9U?vw zp6l7PZd7)6yXZg??9!z!|8%2P)wF}N3rrAfwph*<@n-_x@nYlT&ojUA37A`%6H&=Wj^jeQhd{S*+49iS(?!THpXSG6TrKjw9armTK5AB696!gi8BO{Q+dG!( zI)n^z5X)9?o`Mq^w#G7M`11=5S(Gr@5125e&hK~kmuk{ki9 zARV}o%WrJQ)!nf82OvzgtfLo)ZDUAu`?|b=)K)vBSRUd>OttTbwKzQ9Z?u=BQW>e;(>!+Yz722F84lTDMaacFDUGG4o( zxm`c=9Ng7>j(>i_wR!*5@nzufBb+|>nN)fPXQKwSl#)OG{Ih6e3o(66P@XPl^XXLr zQ$V^9HbWp`rVKf>-HCs@{+g*3_v5v;ZYE;b;y8SXSQ_u#*7!MM;RR59BT_tdwR~D! zQ4;gx2{*O{u8$+}MtLu-TB8xymWUzYk5t(;7nC1G1i89e7r*Y}XdYRik2Mw7TQ2z6 z87U_r?D(Ka@bnoFtZzW3^lTh<{_?yK6x;5BXN3t)JnWxWO&e^Fw;EF`o13)ko3dFq zBh&2pdGGzc?7>X=U$^!ngV*X;xdL0BxG43SxQe!xw3= z+_D|2lq>F|8=T#}fBW=0aE)xNi7dI@7V&pON^aGT9@GDZniRRc___?OQ8ai79L=1s zr)BJC(yGO&Q+ie$9%nWJ{I6X)k-_1b0~CCtj(-|0^pC;!zVv@6QF*O2yt%D`DC>g< z7nAn3G>|XDiI!U$>fCD08m84eo134G&l8CL)sN?6TK%4s`T^dAX}zrFe0{dH5MrxG zSbP10J&6$=tE-SAfbqgLmEXf6YNvxk?xH*tmE@X3 zHLV_S^GT>Ry(>- zE`b{mw2rjl3^eslEOBc$lO@y)hn?vf$ex+Y_Eo+q8CYRU)o~yq2?wXts!)siQPmbM zH=nNiPm{8*D<)I#NJuo4O*~ro`jSOdNQHi->=R#M_!~)rT7uNUe0~CE|Haw1#?%aX zJL~GryOtAhY?tG0NX%sq7CL%u{W)*z|Iotqd)4>>3NHHNhq#gQl612#6>$dZCUZm^eC-mGND&W_gj0qkE?_g*rU-??<~>8#J0iit&C}j^zg4Vk<;>ggdY+ z*!P})p#L+07OZW`q?kGnM{$2@>&MqjyhFM|mBXc;)_v;D8Xs7YG{-rz&q7CE?MrBJ zM8Anb%}p?xR>VN!Z`2(<4JOtZnw2<4$UP3HLPWzgy}G>*wc9{=EKIbl`jk?@m&nxZ z#@05)QW821k6$I$Z@dz}*j&_swS|xs5q;m1I^%m6rgy#v)}bkNnmmUqsL=(1y9g!J z=*rM9A!Mou_=v)+BSU;v{WYyh%^K(}l7Q9y%g9OTqE|;>kJ~DC77Md*&HGosNV_#Y z8Qsy2Mx95H?AAX0g`lj%uSDoObJWQJepSzIrgeqre+}V2pf2Z8Jc$Me`O=csg}yKO zpo|;*7Fyolwh{tg3*B2};`O%tM~O?5y^^@@5H4kA)dZ^k3M5rWK4wWZ7;A%IA7?`c zW63Y8ht%``PtF<$9`h3~7Zb#3F*NYV1QFe#^jPwE%&RfFPfl%tw^H{yKE;?_w4CRw zu6%1D>k~!hsqd@22=!b$%ENzwO+Hv0JFpefSdrjmKnI&zv2wpIP;kLAtY zL;T>AyYGzskiF>2wPv&sQ|G1Nc$GpzJ@b59>SM!<7GoZm5%E66lKp57XLBDmG4bS` zZ1vr+eAtN8^flq=o%yFAaD%PB7Ae=`SX3nx`J{#k%t@%L-#2AmZ=qqmWMa0}$mG@l zpWh})@DQK(R(w9?kO=SQF{}me5;R=Ql56kUH$imPIJ-R zw$AeK?{u04l3goj-M;_YoHxP4NUqBvUMIW_kC(wTtUgF zKbgImj>kO1a%$FD{xWI*r#t5uTcP8V8cQJRG)C&C=RI+!i2-+qY|DYSH1T5GA9t>%2cl_TL|DxkAusRQYj6LR~wGVXLxkdND*isY%m`*$*7U?4+Q^cTo{i6Cg#?f3 z52;Z~S&QPup&%QNPu%&<@hP=Bd-K4m999VKl+!F^>)$(BO`b;PM0S$IcCLlUJ9;3J z58rfO9rN%Caa`|_jNB+Fks(U`mV|Ir11r~OM+M@dA5G#V5yD9ctPdS*A4TrEzHfzmx9>Q zvX>yO9>Q&@B)&=IS52qUX(m+;&CXW|WA35?;PEnxJni@%beUO4A@;pkA)L`c;)(K( z#*sY4z&!59lUN}_vM6^(MUo9m10lQX!=3*;s&AKPYeEI`@UdG#D;JR&oZzs8tq}R! z>Pl1mon%xOI=7x4Ty@!FLtZcDoN_3~|!TU0nMGob-q1jK4u6JdSIgz39#~N|7UKMeIyGL7UWrox=W*fMXv{} zAL>RL-j8tY@{wUJ3S_{1!SxMxW2+}*Vu$MzE?VhBg7~kqsU2`-Wy9bOMBs-@Hf zY-E1tU=K`6matAT0^GBmZ0hXvf=OvitqEQW8q?uB4;`T$CdOV}y;`97=jsl71qH8| z$#pr)f}AxNdMK(wFav}dKo)rfvsEwPda#&iMm<^dff-1sijp4JzP)8VTisqO{#r|c z@M4}ZsnmWD^$#OD4c~Jk?ZP6lE^3}>9O*NzXcnJk7!ISW=d)&}RsJTCxB4F0hYXuo zL^if|x-Nub01eWNprw#9Z|yg+2>yDVd(U#$HhiL)3=aS`xLnlFdXI`a26N!4nj zeoGAe2Wi7yh=l^EUaNq}_11m)73&ILb;;+!sL$8OF1Ldjhom|pHodMD|*x2Czv(|@2)d?{V`tp8YGg5 z_g0zrPyU4_KQ1<;x@U+a%vWTA3+%&Ii=4WWxRfe14Ps)3$&k7nO{KFZc>)gr6KEo| zA7dAHmWlWhT695VB@xAykCrABR6s-b;C9 zj4>_q|2V4jRfnjivD|6mvnnftV^nhi1x)ykd9G|X-RP$r_N#N1{(C|hFvd82_XjVc4f zmd3;~vc05ORdl#a;Lb^0VcAlls3?eCubJlE#=8$!b6%nA$Cf%eo#(uJVQgh?y&xl6 za%Zbp-#`z`ML+QvJW0(b%Y2{O_MZ&{f^(eRT6^?J?ob%-oan>4{{Oy$do~7&FrUm{ zwMs9~6c99tgzc@ox_bP2Pd+TJfw(iEk%_>lu0p>H1RjIPuy(xJAc-TL1J|Ot^VpK zX_?En8YbS*JO+fuWpvuw_XUgBFNVAHCb-rdvKEmlt`WQIgN8Bh4fg zf)P&Y6HI#(KCdTwE)N)hdgv}nzSa@SUuNsL5QGO#4RQOMU6+BJPVZr1zw-ABT-pqd z#xSfbW0e$HYJB-;VVfZLBa(tJ1r;%~q@}LhkQuu^pSaWi4u>;yfyV5b27^+m&`;~WS4FgqmX886G@;;Cl|(H|3EDLcWeO$ixpvHKYUmT-x$)&UHP?!hMGaZpbA;v>y+T zX19L4Ao%xgmY2mpo8u-%Uk?9{XF*0Vg4qp2mwIEa{UKz=77_K`;0m z+-q~z?ceTKjUU<2Tq(tptI8j?wTO&ifxHvte%s6*)km*3@wzlk$4w^o#&e+XK~p+0 zpev8jss>gAyJnYCxQ{AI_7{tpU2)t_l^suEyd7HRtp5$K-h0&={78qPQ4tdJ0J4&i z?*_72O#+B@5$ae(Vg8ND8)CdcZHafW%i}8;v;NcX0~Tca`CI) zdrO;RAmwQ-lkc+-8eAqn1UN3{$M0JdB}J3EoLIR5^kBjk77g)+s4pCViQJ7o-=RzQ zxI4awO3S{#Shd~myWKF8pvJOR=>?I+($tI3u3Xu?AP^9Hp`dgxPrjy*)?U@8;*6UG zxojbZqt0MEt#+d76$(vLFZ-$lCqj}#_kFSwc-toPK;@>fMRKT2r?73u=0Dx7YS2$LctwTqBp;D~C@O!CauGUZz)dc0+gCi1fyxf_yl$W8 z3T3A}`k2yjWQ0zccVc~!xtkJdtyK$0*f<8&*C~1lm)J7VFKc^qjaS19j`m+Oz#su2 zi&mp4L%I2K=*w%DG4hgMH2=#YpQW%g1wRX28*`_W(hCF)GqHe+#5J-@*!phykP!=E zU_1jINmib`Sl?!Cn_=98>_wM-t^%Jbr~XcFLfA%Z0%{lUw|20YiqDL)5y}6=;Yg)M zNM0-t!Pq#`tXx6J{@`oR*H9pI&7T^#n8;HfZ(h|u)0io1PJ`GXdAN~8xzDR{-{0zG zaXdszDto(4>|nGN7U8)YPCCX4kl3j(xE4AZz?dY9KSLxZ82q`1wzvXcaub%`#E1HK zj?VbT>qlU2O50zI@Q*)5)g43*&Kt^0b}hAEk5}X+2+_oSe^OD_^>T3tS$J-1?D!oItBQ z!GtXt&Ab^St#r4URJ0=?8R!?+r5rJybYtR}(JbD=_K&GawK<~?mZC~l-=~}XIB%Fh^K9VFy185 zN30yb4vg6O{*E|RJ;l+LG)oW{;z%{%QoA8IU?7TvqQ)LXy?w{ftqLwC2yf#kjIM1` z0TheZ=zbYxp<*rt3j7*6?KtGb?7yXw-(9CX8t~7VF9~>2~keK3)6hH_F z`%``GrIWKTcdrsnr;SXbAb`kkeWX@~>DmQ|2i8g7Hw|Se-|mARM-k1QQ}|DhPJJ2H zU8+3qe&Y%Ho!3{ce<~>2gGP9q(;%&@2gO189e0Gw>>m zM?P3HO{@|)|2^epKQYyy+G+ED-zT8DD^z*>Re6WRwEQt<#dEz?YiH0B+?F%Gy?iQJ zN%W{V25@S_PxL=+KFHH>usNyPVA5+iODd-UiDnYoouqCoT<-VIs!BF*ek0r{D%_bv zc}8$(2BK$CzZ43V<-KrM*KshPAVmXQS4){=JNB_t8M+ru(J;FCOy0c5tHm_{m@hB+ z=MMnGzOiOqOK`Lk?@UcilZp3l8GX6->tx>+y+7_ufCa;9W>!J@Jy@t#6T4azjr+za zYA}qct}|-aUL<;Uc^Hua;&R;0axth?Wi_E?h#*?8$rK{oApEDT`W|Lv2uK{}tN41` z4OAjl#?f!1P}X&$H6&f%K4W$ zh#W^f`25s&So0bM}v=70k0q$Q@ZkRpz*^CymcNs`~Y>!@}R$ z^w04xUB{eV)(NUhj%m`<6|+Y;rsF(|;p z8amZkaP$H&oa_3Ky4D~t8VM+W4~Xto1PIVtu#F2U35081!HT%iSj8lQ>Fx4%u!o!JmYix{U zjq{^Q{5Yc)hQhH;`eQ!g|Co_nN+ZRE<${l68|UgkP!)cIwg_c-?D23=sr^ip4g1`C z9^pr!lVZb$KtTIJ#2D&7 z7H>s1-CuKunA}Xt*)B@@Ngovto+2f8A>uK^Nr)wVNtT%T`D-mxGsT)B%bl zg*I7Ruc57@JBL9zRtWb&7n2~cD;qW_)+>BVsXO$lTN+riZe(mtN9ldXoV@QFW=9h) zclwY~*VqcJkov8<#Q1N@H7|oB$tA}zQN9yHT#tWzQAATlG+I6HrR8MUCAEx!iw&c= zosLe1ZgBm@%{Ap5*3TQ&yOFX3g|EX<0pQ{>lv0?m+f~g@QFVYe0hbX;ej^+#@1pwd z z{y;X~agP(+tGWNhPZ(+$vdV|8oXv2=PmZc>MxIn6ycWt@Yo|16#Hw15v4t&i-CG#es0O9vjn_*kg(0`V6#W>|XC&>HC>S^hjxF>+Ry3~M!N5-KLX*C7vIVC2ToKM#VZPMCM#Qo58oiA<-%a|dp0u)mAVGXdemYrE84K)_tq zp23_OlvivE%4SR3iVrmZCFx^-k}g&h(0aN(8-BZ8-)W?<@I+?MxxNyxtkzduoTLF8 z^kHIA16=t|3|1QrE{IP6H`ZF)Qz=!gON0|=>p@$ih14 zL__j>_9qzy*&B*`fMo^D=?#0q;pWZRi z#sqN|tX&OEzGtunyD~q7*jEH!oT5tF4{j(;qW^b)O)HN#xEN>RxYy1fD=Zqt&x0(- zQ1R`$#7o5r>X@^Y)^7%T8$#2Va;X}k4XHTxuA?=f^>T?NzLDAQbFxPg5-T`h%jd&` zS2>O! z$?dLE`X&L@wKxY86ikAFWC_;%4GYo@m;EmH_Zvo+&;*`MZgknbqWn}V^IO`k2+HSY zI%`BB#94v@ zE<_X1%ihvR0s!}!JlH>+6sM5y7Ot2YOjxcFdX@k~M`O{bd; z4bUM(Y&QyoeV^LsZc`8Ufg0T>#?e!eNx*3f)YvYsj4aM;DruPYDp8z&xM6)9p@*1K zn{xqIl2Lguq>p^0y7;!kx!=zA(p`d2F)yNwH)Q#GAm%+@?v^mGeiN*-zFiD(!ZgLbd;(uZ2-Amniyra)qApelTT^sWr zLiumk&wW;2dqD6yYTf)c50=h!7u z8(^LtT}fTQPnck$8iSp`Vr=KV6>&>Y+B#zA@dmtG5a?tlwC0!@SG3$tqNmL&wbg8&Ue`VROkx4=3a-Khvvyv$Q2Kq)iSkxYQP4HBVw~=-@c;Y8rOVUN3L5 zu=#|uA&T18DgEl8-1d+%hX@?f8s+1sc`XZYodXyk_u!Dq!<8kluE&!}I`ZZhWhtR8k8xDI!3JF>NgL zjA35tm#8a@0%F&By(L$k%H^s>gv1{Z)TH7hdXaHSyPIVLERFRasS%WI*GC%7^gf&e zx2LV71F7YDOT0anD5Bq9b=HA9Eh;o1^y5^&?WB>! z`#p~=cT40DWtz^QY_gYlHIl>*)H69&+2I2KCq#!Ch6__=t+cpMpuY?XB zx-`J|+8-CVK?Np|XQ(gWgTICm?~{r0zYYGS3SL%^>Q9>c>}Jho_i-wO_7QowdJv$9BGy-%jUq}kpgWeH;|6knkbm@LqxdZ4f$0I1S zZytEyZ2DCLbx#NZVM@~Bm?j7{KV<)j`19ovdC!)tLxi^G(Kc+jun0Uk-B&TRQhZX8 zarLwM$=V7Fj^uU9UymP0N=3s=b!X%Y_jZH0$waLW`vQac%2!%7IId;Bn z1ao1>DIR`*1!|D_I?PW+*-SVf`DA=MMv)m&K!X3PDA71;+l^G*c|!iAvkqr*&3ASY z$pf{R#st1+mADw=+LehTw@~Z0&N;mWScthcTwy5J@Ve3F(O~J(c(fFUyVU!cY16V# zF7NlIE1RFcdl_lattsWF+-Ga-PP)h!iQ1?BOlDTjw+NsIp0{tF@rkgGR6~h4n+Vy^ z7iQ}8;hp&cDDlR^V>eJV$!m5b^|v65>oRr9fV$fqIH2?eeOx(xT_HvQq(#Nm(sWp94}on-2xt*QdL*_F&qe;n2rk|B zjkK}!3S#2zfbG2s2Gm@x^}Ai|!BZRGnXns4hxUuTNixX|I~Bddd~@4VXaA@uXdr`f z1j9`_1C1_PFxq06&$CQLupr3Rgx^<3Ck?7(M$p!DdA%{0d15@m0RGI(p!TQ6t zLo?E*Iw~1IQH^3~YZ7lcKv4Rg^?WB5nRtQs%W%q0G*Rk`0_q@;7(?~gD;EB$R(>{E z&7uA3I67?nvnFpM>V>Zx@NsbY&@jMbEmMlK@#>CoUC1vUZF>k*f!lWLgAd@v0p#^1 zdGYOCcLNW1ngSCjK%-s#o7FF}=~H?c*lFo&MTWDr9P4iUmpKY@yjR9tn`QY*uQi_@ zgf}6yyGX*6?wlVPbT9gnXWgt;>*r9dhfI54TyjEVT*rOa3bH_T+9Men2dkL+t2&SJi+;@bp+<|%oT)LGUHljeu z*(-XCc&PL#irn0o_kmwGtw;TTPrlDe2vE(d+y-2PBoET<);|wZxJv@`69c zSHfZfE7MFC*UazYbc=9~Mt&5IcenR`%t#+6O~>ds9idVV&bk1F4r zzIBVk7Qz9@fz=!fh*+@{ZjALFYfS+8sT`;$>3|$CeKuGnP{eqH7{9C^?yiBLJfrr% zklAe%6fc*B%WikorS7u*G6O`RbxP`K)xu@H`32tWJ%CsEF-3wbnJWGkQc3!}opDW0 zSLa~C%CGS|q+TA5eq@TM8$6I2&^`Rt8Dslk#mTVX7}tmU<{-kJB;BrFZ0q^6&Tg#S#vw(TrFlJTu38wwIZ^-=#P^X$Od z`j4R7uhj{rBm?u@-FrZm-B*YDn2Tf5b_)L1gy`%kkEGQ9MB5X%>YMOL#iRX=M+NGQ zKRVC<+6ZA8H7CeTbquM_&Yc9q=(c8n&pM{HpJ`5JX|~LdK0j&+JlT~QhB$=XVdrOI z!^MB0`@UQyzyt}zVYC?UP(k|;$D36mpp-&A!lnj7J_98W6Xt{}FE_2b3mqy-eEWKY zAx$xooYJG4h(Z`2oZpgVL;Cmd@eVx`Tl2AB%xVA+hXzLHt2C@@|C#TRoO7#=1Okx> z~ur0*5;ObO=20AoWsBS?d%56hwCzUKMhxis?0M3hVm-s@T#>l_?itte*DESi9QP5I!Dlpidkxb}8oig6 zv3QPy-DU&y(2wXec?saR1Tc*>ww_efkU2I{=UYE}Tt2Q(X>AD+__pmLu~<0ah-b8x z!KB)2tKcrs#vjSI)#H&}R-lIP)m8u&Pwh;Y&oVcDJzkk?LG+z7&Sy%h4z+1kjvtyo zZXa6Dx*T>3B^(`h{Gf$zt}xgfD1rk7ijqbRX;yuf`-7JtEqN zS1`+FJaR7sy$2T{g!B4rS@f;tvNW+=amYzo5)T+^*pVgHo0O?$_caR<- z!$7%xhd7%nSv-4B#bD;sm-2^56^;3z&G=J<>j^J|jo%khfi5S7CC~JJ9|5w1x@L z8BR}H2is6V%GL&P>bT1H~JUBW{W_8~?<^+q_fm`{s6?}?sJAxYiA}F7h z>vbIAQmHf2r54#&&9lX}Ub2C0c>Zg~Puc395NRoZ{*=*am_{=(5X%jBB~GWg8Qur9 zw#|gwx9KS2M+Xdn_}gW>HfLLT6Uam|QOBjruec<}b>`tEdZZTM={{kq-*b4lCh!r* z6*#n+M}FK*nc?mSbhGefbeWupu`NFcAQ_g5X!kGAXM{|Ywi-f6el{vaq zAUc_{Vg%3la`1Q|tc&oMiKJjKzt%eJm(apQ9pfruY?EQ^`l60+Zhl*1VzNy`@fX4* zm1t{7aP%Nuri#%xo9o$hk>v1Awf}($$#CT~SUgR^G)G_o!;7>o@u{SMe^~BlQkl@K zj#LYyKNCJI>op)A*N4l;(e1<%F?igd6QH?zb5N!bE5%l{J*5MTd zpRc1I$*!D^r*_RR${%f{(1Hefcr9a*kiBrh_gMZaCG%|14Z2bE{oT>JF6RuM+MHYjY=8qoa?zg!(q!A&2Oq zTYHyCe?L!yVbok4=(ZEpl7UTQd*wrJDMY44_Io!Q;-$swU@QG62M|y>qk0Kslqv0^ zX;a~z22kj;-YyUAK=Na`UbGRbVQS#q)oT9CI0tZaFQHj_=?<=^ z_qKAFFfum!MxLDfA21>rGLe1WP{47UdXy=@)%`FB_SsQ?c1uJt$!cSGN$Bx~iE9M; zhth;2!u@De7DBMwnfv&4y8NC`lJR9^fR=GU%C5Fl0zGo`0v;BN;tHl{JW~+9{F}`C z>C-7vrn67`JurvS>w7+-fdC)|C<7WiJ*Hy<1Vq%D9OeaIDfhkfdP8<&qq3vcfh2AAHzf9vwfF{n+6H%*>UO|5DzTcl$PutP%rf2j#?;vO0kO;ua|tZ8fXFe}p zTy&tx%8|p_z4$&G@2OVd2kqaB%z$?TS&0Ue<3<{2L56WXgO2pYABv`NxG@X%eMeKc z>LX2gcHH_C@g_1$LQ&_GF(<*P?i2B#+2h02-N>=cDFp;&K^ux76aDphaj4%H{`MbW zAC%F`Si~%r)e5ZTyEN@ zOcYVat&N#33`o)&q@^Dlh@O3~Zew?l_!5QaW3bOWY}n+~>~<#m<~2}{*48Gy!*3f> z6jm>*g2Yhf4b2$%3f||eaM>7auc$=$H`}H^{-ltsTs&G_z_qjQ%9!FUiidADH=Z=Nv&AWQ~!-u{o|sIHl)e&i)%R((|v`6JiKEq4#@m^C7&^ZM{-;ZkS$9HChlUh zIqP1WaZL~;$eD(cgv$ydv$MklWiF*>-g0^k+taU=us2Sq_!7rT<@QI&bVZ4Bg_}Dh zNz{MBeNw>@${|b%)HI2~-^wt6nv>6J@5q+nV_%^l5;PqD&v8u^xl-lBEhwtngX|JB zgQ%X?J5e0^3AYz}-+3q+9$R)?U zz4Xf1z44q7pYZl!$GcNwm1|MSuZ+;8Zv87UAYR z=bQ{oBd8vO`c!1rghQQ2L)`Q5Rg}i5pQ}C-vF;B$e=y9|ok}59@~{_OpiCwGJ5>^E z|3xt-EN%pZ`~PcQ1B;kX)$V0b3E9W-Z=E3*r9De2;Nf>a3mVIe@OY;8#>`CCvPJ7L zdUQYGDN~8WX-dc)^GH3A}#G*^W*E!vuoU)0{p zJUxECMFM)q%>rZ|%H>=4CA_?I-cw4=J~d-S>1Lx%#r|`DsgA%Uyc~BodJhtBw~uOU z-AnDDXg|aRx!{RzGkxX$E2A(G$qb=nAR3f&1&EhX@i5Jgn}HEt1AYjp+EL1<0VyX~ z5aTHuCeDHEW!|@q==%5@hp^x`xU`r}`{PKN9YFi~CGGqz?6N1KUwn|4LjXTI%6TT8 zad@}VXurfX>+s;CJp(^e^ryc;gHQXx0>(&m{}1`>{P*f0O9Q#~c>mK$ur*S9)#HuX z1IhWah(QPR`=PF@iQ@|33bL{Gr8v2gf)G7<23rfG5cuVsf02ydrJ(N4`36r}=kA8m zK@u-UKKHI0OO6bfVJ^GUn_cJ zkImwvVTIw}S^2&^v0k`436l11Bu5&J!F>oACHbX{{oC)bG6f_~2#5(hEJkmP@&$Mz z?%I5@zLIg5Dkq$Rfk+1>Ko!J#aU_G7RWPHK+09fwwSOSre8)EPxz4hs;`w&h%BSa< z6RI&u;qR!LfF_Wpma#Ph)(Ggn{`ceX`w-hB zsHarMhvccS^ttu>Ajj@o>S8b!9(imATx$Lr6)9O)ip;*!EqgZuZ~;lf=+EOfFb2Kp zRL7SK=?6W47L3rs8e_jzM7}I_-G(y6D1JWa!(}C zlvPF39642^Z=!zYo4-+97*;vW>4!I@?^9;V=}tA>O`nEMmK*;)p3@g_iJvw)+SQK8 z46}t(D3h!KLTO)O^%klDH|P0|eEvQRjCeeRoZeVh=+@jAN`s2HYVsbxRZM|dFe<$N z1F#s=wCior+V!Ic_uj;z=m4!4vDygaXZq@zI^-Cs!en%5_NkxUR<4QPpg?2 zM&kK^cU90@=`OqiHA-%V>peulf5i%!RJ4dagpKXafRTz0+}@c!AduRr}q+q>r;B2tz~qA5~P00*_sciB`XV_@Qb>?Rl$%WIq@hMSN=r z(IXUTev~C0l>TS#TN$K5b<7LbX+Am` zYHBH1i2;8A=0c`Qb34HY?q_obHcur{{;PG};~XeO&yt(QAaVY*2F=kK#l0M413-Bm zI2ebCmd-x~s0ac3G4cWk4M$|h9jeiMEnY51{NIH!x(Y^cn?O84zNXb7rZcH)-5CB(D<6x#s~eOX8w_EN3U%tA+kK2sjw&N)R- zs1q9Qr1mcOy}?q8`Y!?Tt6E^N1@R}siX!bT0 zVMdJ1W_L+1@SsHERtMl3t<;{{{*WJE2ebo-7JXpHL-K_|0zXDMl?bS!;p|l>%>RBw z@@zv$b4*i02$!<{L(sE7cwWJ*#sU=qOaNu_3uDQ>M4tv2#-VFZcqkOP0;dwq3Bb!; zJqhJmjRR2R`rbZHddzMKJqo)c8u(GyT5X-mirkfmmjb@rj-6fV@=(hc`1R^b;4on8 z?3F#+w*Haw8Kj-g?m{S!Fh+17GSdt;uJ?>?;^>!G+8Iv;bNS6@!&Ntz`*=a6NVp$&iExl_vP= zUMoGqU3F&EN5-YosyF`|JxYLic-Q=ozXA9@kzYCd^ULG;&;^PMb5Px3`l@tE*6MY; zRTUk1F0mco<$c1hSlC(Ve5@-~Awjevd2@uXN`9ORpjHcgu16 zm>KYkPhFv2;OIccMRX^O{7LL=tGFwF|HoWaQ zPF`agL}<*GAe21hG@yaOAIhJ7G1h`oudU*gTsWsoWrU(sRvc-dhL3gU@z5 z{v>c#?#-Jq^Vy#Ghi2`Q!0Czwgxa-~s;Km--C8TShjVJn%|J~M#ev6OFFMmJle73V z8Fr67a08U@pQ3l)MpfP(H~U6KN6O}1bmvE#>5cKoZ%cy#n4d28H*?=+$3!FcPjZSl zY_quWLUg%Zcs`et{It2=m(wz#hq$azPx%^_al+6L9!_=NRJ@ilJ zK<~F;Y7c}OvnCIFqKGbv(Xzk#_h=$d1d*7di|<$LOt90=Y3{Da1ZyelSHOEU5w96L zDaR7bDW219rn#*LCS9+!`pX3C|EF~1l>i^@`5)5JukV{CA#s+VHFk72S(uuhXuYW6 zS{xR1Z_A(wS!xhUQ^Ydt&hYivrn;vl4Rc*1n;);QphVvHQ8yQkS{>QMr<)-<#9a-7 zSqT$&&ON7q%0U?Ic>yfk4;{*=)f4QA;|QWPy8TEI=ou;gPf~PCmC^H70oBl0ym5^B z0|Lv<+?B1=r+nLEj$t4HV_3Bw^s^!1py;C8jFvnap_{3*40{jM9D1$&nDTBjw;-VmyAU$?fjVEt)G(+_7VdsQ5F#dP#MsFmS*@XzvAE+4edf9A` z3IdwSeGu7&jKK-Ph?u9oUd@?*)_0Decoz1)lx(j5$sBndL6D9Rl0cdWEAL{n`8*h* zAdta_{zxo~r|Izd;1Q1bQrS}TJ#f#Er53ihH9Gj)W*p(-bgUPr9?g_b?+?FAY<+Ox zQ!`-EuuO;`cBop)ow2;` z_>SUx|E=+6Ac%LWcmRS6fdS{ew`rln0c}w-YFX4Un>;*J@fV)rZ&_3UnEaWJEQMQ` zXM+<4(l=X=_PRLskqEB9jd(TxssLx|lDSC7X}ZnSB(G;z-4Y*I-tnH|pRb?h_t4}! zc0YeR*FY9N1eeaygVfnPdtQ?_@k)sOfM-2Vw16~#*O)m&Fs;*$k`81rZt=LCj|cr$ zL3!42)6Dn**OanL$$6*uJov{FQqf#~@$_*xeZyj#v9VSB$4H!__5f%lJk{dUe0s_P zcqz#nu2OWt#-fbyb1!l82v$#A;2qOng6%$log^{+L(IS%m?RYLve;D}Wzi%Ew(|&| zh_!hMO#N;+k71iI2Wq_*?$QQ(XR3>2C&L++eE7GsU;`#GP_S5H*o#=9QH^rI4iUhSAQuFp@mS5+t_6E7 zT?lVqVwkcJ-y%ce2#j#A`MQv;6dtBa?kN&t@op;1;Y6MZPal?ExLqa8YctFgJ1tR6^kUn6 z*%*SM3;gk$BEh627cVYjPV^Y-T`L>d0mDRGOXGIT*_T=XNCX4!)u04ncFGdgUepAv zsTKqo&9LZK#J>dkzn5=dKHbpGbdgc{9n9>GDf^DWCMwoZS?B@+l0mP(ly23}26tCg zNoi&EUlYsN{&D8oD0^brC|mhm1BsA8=>qf7Jpfm`R&N*MP0x{qFT85dboJz=Zru1}Gsv)7i50stFnKj>drl8M^Ej~lErLZ3m z43*^dn)eqiqPVAU@BG$<{0p9+W=2ni$4#7zZJuQM^MF07rq9t>l47rm@Sm!4qH4r@7(6I&SV7PXa2(=1t0k;8vWpomJ8R@HDF+q;=;XzU(Xn9gFUeHu}7G3*v#oxmPhj{S1Z= zY3W6FUh_nR;tg4dlCuDU1*c@Sg!$8rtUoZr9JPt~Tw=(e8{MSZWbz!UnBhKu)%iy@ ze-}Lyx;A^@p~!EPHQrYVkk-t42n-43GOzb@K&TVab+Hlv;0pGJ($zISjr zJ)Sw6tes=BU_AHQxTpC5)P+{|uuK4b5|6&KD819$Fq~_4&U9W1N&CO@nKLK8)e5Ra-kJ*eFUA=FRqR-Yje5XhdR^+ZVnB@E zh*zx@0GStT=V~4Q?YU>0v?p=G^>dEBZ~2P9!Ht><_ATXLSN*@!-|u^r1xoYGIpv~j@qi40V_cHc+0duu5$ib|(P}rX zHMg};RC`D2AxbA@%yrlIzP-fgebV7!r{9>5D>`MHtDce%+Gk@&y=isD?1C+EQEm|<$A&faUkF{ zfjL^}7+j6Bjw7aCN-MyvnR;&bg+lFD-n^*sXo7vJKkPLvI5f=N;{v5x!E*!DBZvD# zJzEUAHJ6_Z6Ls4dmkd9X>>;#z4zpBRrVR8q^KPp5ZjJY3ZtYc<(HEA$QZ6D)%4h|F?Z8KxgpGYI8!(*ZJqscuKFo z27sdI*pfm7@(!QVVt>T`x2!7lV=B5L(t;PRn3dX1!Q(!MP3Y=& zS&a|2>g5OyJGbt*d0-7>A(9hKK<~tiHL&{5FyOH+u46bTlAWxkz0AG`vemAI8a(*) z0Wv%}5gdap5JLg}|G5Ky%Y)T=+glD;!(54MB5LKJx(yeTvgf2@8k)~JVW7mn&v_WW z#k6EnBH6QxEZ7y8J~Dk`M6U{CJGtG|BJuR>m~766nJ%7;A?mR`?LZo|;}|(AHWl-q zjSPg1?W#ZC%xMtOdK(7#hgMQJjG*W-7S~X`0;`xmNK|m@;Ov}Uv&6!Yh!?wm48pY( z@CWZ})gdrGw-g-@WT>wO(tWhEN}tDtXa`|xvH*w|E)ljq6zD9~DTg4T36jL=$x8<^Y+T&Xqg;{@Z?zcj(`tpgb@HkdIGLN-=7bjQB1YTiB8U-e14 z(+?zSYRu=5`U3cOnxnfuYKG_DEqi@wbI7XW!M=T%dApx_URG$Wj`H3ExpR)1mbh5GyZSb}3cG4?~Xk&zLH1t5O+%Lawt2EuaovGvnu{I6MB z&#dNGO=#FRW^?qsw1tY9mEMASjT1z>A>sB;>|_zhZ(>+qQGL7Jr!yLpmLl-XK%3+`W7(et*)S8nF&_at^0kIy8B-iC#$VECI|zxh zx3rz+tKrLru{V{%n)Lo2WFmsN{^=J3+WX*c`VY00ml%r4JZl_$Zc*17AHUp>vKB6Q zS7VE4sM(g(zD(4EB&D1REjt*iI`OLTD58?!JU7GUhwF(3ddKY{odh#Dnccf%Yv&$@ z9coJZj$x~*eWZVn-5oZ1aFLqKzM~jd7N4?6C`bB8}w}ICkphHd)?Ycc**ay!Q z`h7)-0Gn&IPaRy*KqpFgcGHXtb>{eGz{^q=I4D>WcLr#;$G(pnNQTz=w0BprT%Eb! z^!VAZJLpFUDL4j1uPxJDhKo;tjM{WuF-XgN0INH9+TfTlq7sA33$oHa7sjqbLS-B^|cvCDA z)}zg;^QTq>1^=HspQt1*lqir7eVa7DUX6oQqR(<|h&TVMHU5^P1SB@|E-Sv=8b5cL zS&@%v#-@zdqZ9nBqxP^#I=$e8=XWhwBWXbNWj~=6q6u@xubA3_EEU%pyw~sOR++5t z=(LD6z$|S|2-PxO9Ir_vvkaMT*wh}zVz;XkvehrI%Wx)_3-Kir`-M{Z)+@rk;HV7u=Y`X%Hyv zR?1H)uBg@lgL5*+xlECB5p9T9sW99eW9kBvO5SmkMLC0ic1PSDNA=u}nVu1IawpDM zf)S};wwZj^Mn8{Lv9a~}^8o#({qiA{7XQXO{_>M;7o-dLT#{?;=z{qyH!sSXh>9JP zl_DTRi`7vR4Qt0tVYy}hhX#`xG~c3*K|^7FVJ}aE{d1Y}NgwYWPT11Jl}@O-8q0q# zJrC;GZOmIZrxCuUKn&ELypJCN|MI=Iqe)uEeELi5PCdvDgajW~zR3Wb4QfB+j&F1t z4PYg-Jj!w}ZNqxzNUQ^9^~M0$mJMOt(XkxI28%clue&flC8~WvD-hj7i-xlB}$ zbUacz(-cnP6JTSV1|6&XOhWlE&4XGLoKNx%sr!&5Lwkx1D9L=a58-SOW-k~cpA7q& zC7S(LLE=sKnXv~k$CoGJ)Ql!G@1jkoprEq*)4Qls0-4MPH5XD(S{mtHC}JGb%YOr?0dq*Tv}p7KWOU>x>Ra#PV-{v0O{z zM~=jF^Xu4sLU$+!Z%}+A_AfIm|Ffq9i6c2~3Q^dFX_LVf(*AvoD)+lm`A>GDq>(?= z=xtKJnTbphx`2U#EBdvQ*Og4%^43M&jcF>+E182oP{1i++EBRCE;h4O*GF(uypACO z-{Q8R7Rqp8^rjQZ z-#647w2mvt#=4my?o**nW5DNOXGx&|^(mz_C~`h_x>>v4q+oUn+Y%QrW?Ch>ZYv>z z=WRuaCz!P9JBUu6_BgJBJxu)NXVE%ouhSO?veQ_(CPu!gWk2V2Yfdqd^K6ShEza+Q z8mwIj!#{Vcf9pt^&DR@ZY@&r%UmCmHq`6?u`jgt0s(;6v3_+U{jxjVEfM*P!$j7O? z*1Sp?V~WU!;q40y8>H9o*3KtxeNF0fjxqO`|8$=RlQw5bMQ_AlaGT8KwoxGEew$Iq zlpZf8D#FL1aM~h(%X1`#cN$IrsahGG(mjZ{q-stKTeHh5h|PYFYLgRoi((+LjrLf` z$yMK%6jv$}QqXpxU6#3xY)+A!?>Z}mjX#0t9Qeh&WQJ{aPyF>PZm8s>rMxf=T_D#& z6SY#vs)NL_ME%?!^ePA8a+xxl6Qx_*VEnP@>;;zfRxCEbGwUu&5r%KRdMqioS>)7B zrJy4oLTCq_?1#&%(a@VPc(%GkRw?FvlHX7cG{>OI!~NDG4A*Zj8k^)};RMr~R4H;2OzlQL zEW@h=HzfpJK*dKdR;$Kf`|KIpt}iTR+Eg^USnlvQg#D`hu+lcPdS1Ha-9TCQLkD8Y zg;nBz?#(?~gUNdfcN`pWu@%THj@bGVy0g{X#ayy=QkX>*6O9)b@RMnT$!_V{SPIo26Qg~F;gesz4jptRFy94~ol4+{@ z;S2STw^Cs)9Wl^7%h;ikGnb*at|K0s<1@d;jNbJ}@(?e_uX`%v@YQ{95B3%$!pbl#n*w z{*9`$Zc|$=?aMkPmZ=e4`%JCyVHPu^*!#8DpZ^*>ApDIEzE9-?)XfWIjiQ^dBlpi)z)=oQmu)J${ zeMW8XD>eyQO;g7-Ytza;RB)2kCjpGFtr#=w*dt+zIGmFScAKV5_79hDXO)P*HsDCk z%lF}W+Wh4OVA$O2Mxikk1v$(Ln-brjYzIM&UE=7d58C z&jhrwcjvOVLoMB?(RLhtD%0i`oW%psDl?3l45h*Vw4v#s>;e44{8ZH`8|ifI{PZm; zk+Kcpm@irk&M_hHF4czyT?wgQXror1QR^K3CP+WSRne7!j#WA{d&X| z4A`&-cIt$fAE~%nA8R@xBo=oy1 zxgPVGbZA|iYvGDt>?X!uTqNfuNd-;Ht#Ve0{56ba=JhY@MxWGS3G@&P{$s%_>)FG3 zE_>iM8C*xNx8Ju3pas706M>)=oo9Plx;o8heeq40A5*P8(wTEIPs6CTk*4D0o}|e_A`|)e>}x+$h|u z_N4A5MA&{G8?I+;>2QmyOlFfRPQ5X3izDp7wk4D9xDG@-N7cM@g&FK|qmuh?I2w%B zpH>Q@pf5zqQgw>dd5BW;qxh{m#?AA~3>iOofQr1Jr9vEaj>n-gm zG_E{Z^!K0UiB{)C4UNGQ1r@=UI;`~Y(&52G4)T(ox;Cz}CJ-qLN(XMk94(t5+GbfV zB$4yF<9|I^5zsE1PR#!i7b!Wf36f-PIK8+?nQ1)*1ygNTF=pG62_J{NHB+T+(nMCd zo$TlBiSTFj^mGibdn-f9Cs)#R)}^4g@(QM70-0Prs*jn!D!*OLUq{ovCBU80?xQ?) zAx0^{9(s}?_A}7Ivxmm1M*u-sU7OgN!2zfzFt@S!6%#N@-r_N5Z+tDk0`2YHhX3|; zC4$Ywi=-^UZ2*7ePOrPG84lV1y8d2TP%~yaHWGJfbyZo_v>E0|z}r_7qP7clvs@s0 zz<*S-DZV$M*=lA?l7+|4n!z)_>^7`zA8I;(t`wl0QV+$vMzz^}x+5>zu^F8^W?M+0 zvqAgBXR(B@>KhoKb02+6{1xdIkuIFT2zl01)qiCzdpK691uF>C-ZE6GIdkR+Ck*eB zS9KJ>ePa_N$_No+C$S^fWPlTZ${$c1 z^`*P}VPRe%oF++GP6VmT@j^bBx^5D7*OH*yuRRS_oai*qJ6DQ$-6z1rT)QUFVPg9o2EBRpG(h z99zcdNZANin=N|>pZoX4sHKRW9C^$Yb}Ay8p+}iOt#xYr^ul$~%5T9@Hk_M0F&l%76AQ29oLB=p4n4nFV}a=yZ}VZz&9 z_n=`L$~DnA{TW)MNgl&QZxuXv{%T>zT)0EDfp=<0S3^~&#x_M%Zvj>uEhnEN7HDkr zP(Uzd8{ujXH7d*o{O@=B{>=vv-qltj$yL6>T^V{Eyt?JUXO5h}i~MBy-$oLau`KOP z(47qQkZ4AK9@L;MwVqlozrnR<%i5pZzzcGEq_X;htxz}`F~q8)l0MMprza$6=0&6l zl#td;xXQ)wz6;2dcMZ+6>Jv_Xu8&RjedieO@B}9j$ubqr(lpfF$_!gmG$QDy($Z&; zAkRA=jO~rnF*q&8-0<#HpfGlW5z|mC_lwR69O4XR%rDWHt-xTg#sD zuj)6%Ota`g;ck&x@i524C0UaSWr){c>~+|D&dfybY!iH6d_`JB`=C*Dy?YtsH8qjy zlYYYd;fohQ;`@3!`(H0d;XcH$j{TY6_Yw-5d020&GB5_GtTziadBKtC_uME&!fKld z_-ch=rI!yA+=a8o{!61%TDPAS<7KEO> zgE<<$xNH4?0Zq5OBX@CbF-1epo<&{fRr{4-=7S{8^Ff3WjbNb=XrBgiJj*gYZLsG zJR2BO;+TCb|6@CY%Y)x%u@c4&JgA<(_`)Rj`AA`B#mllD5OW8F{fgl<1CjS#O?5JO zxAj`YxUQaQRf>eRXV&-Qo|8z;K^1d3iAk`ND8zTv&ZgiE%Mn zDs!>4MYvIvx41v-YXJX3z;m2v77kO8YrveG5ZAc;%L5x`@9biOS9w~K81ZIPyW5Ci3!JdAEXu7n4J zLu3PE+F9JDE$wAgvi32}Ie8>3)+YR3_S3but=QEomXihVUL5nbI>;yg>%@Zz(n_hf!8iZpB;Yx5<#%BK!@JRy-gs<_YF+M@lg&WS!iccTmlLdduj=Wk?5yA0Hyd1>VxLAegf zm~S4X34B9|K%jB?NYIgG3VW2n6_dX59vDDrB(WV43g#SN;7QW zT2(8wDq(8ij+>l(GEy$foNB;iaK^X(e^?T41|gSY+U}$VwRPJWTuTw4M^b~-GE+xA% zSZ#X)<7k6>WuHDfRG*P-e&_2{m5YRhLWE3>f|;0CW6ayiKH$di^iCcPHP^&z$!nHX z#7yI>P5jc()n=Ey`FZQMd5jqpe>N^dvDx6!?m_a!;&;GHqXc_wGLVSoOk6i~?7KQo zSj|^*&$;hiBbY!*x#n4HiU4GE5=AZ!KPb3~3hF^^l*-#t^N`tg6k{@CpF*95g-~3i zmw0__5WDP95cV9bqrAv199 zVg;X$O5^eD5>W;d5Of(H5H3wW_K(`TPq&*yi*ULWzh` za1;5`U$K-i@@r+{f%VQ}r}`@E#mQ<5>EJ3C?vE`P&e4?b_3gfobvOwlN%lnP5zPKl zU?g%$WV;Bq@m7N^FySuHMzcz1E)JCgW#1{9>PdJ5-;TVJwO8(;h(WiLO3lTCX?+)) zn??oq&e9yI5gd;F+W<~V13T|?f%+GIx)Hax*#cqG7e*-LmkZUTZBQrxh zb-;D{L#vZyHVu5fus)-<4xfaQWPQaJf4*e7Mm3i%c`0y03N47*LyrF#FpqtzyQ%%i z+#!oMHg`Ljl2nmYxa=Nf3{!=BEbEvg-Gk+wg54B@G+=#wg_I|jJ7ce4c6?qVxcB?= zpjNth%xN}-;#T{aVk1KNaF`9mk#usBPB3N#=G;Y_ZE-VzW-RabDu)*q2jACvX4TuN zOMI^`GS|2=Kb`MUIlrXgL7YT7Nt74$#(DW_IQY!AaT}o$n9r)w?i>Gs`~A}w)%BtEwqd%{@SUDTFFPrGviUkn^lA9U;oEnAv@XD$ z0c6fiBwxHou7jAq`}HsXsdU~uUbZ2g%&A0oC?c$;@hklMy(hU3i1R<9?I>O9f_fi7 zY+u}y3#XR{(Ly)U=Mb&@z;)FZqt%EM^@40~tYd;BY8Ys8wB%oNwW2C~_J^$-CBjK` z)+@Y>W12BvwC<4GQih9D_DGk%bjY=em)%___!9rOi;Ww{wGSSI#|?4evHI<)-=hdj z1mCERm|pj9l8b{c&BL4rc5j)DR|KocLQXqyskclT|1BtW<0{cXYY40{`8`%tO}uw!YSfK=F&|-y?hTF@x;U4w{e7pcfX4|FaCCCy7g+TPrIC4F+6qqIeMWRd&6KHV&n|7HqL?>3A%yv(` z?xn?wyS36>LFnjarx^}L6b!Gy4&v77rSyp>71GjukL!0;TI3r*+#-N~li9=(pToDV z?Q-XXw+-U(lES{EXjh--myFqa-?Cpw1wx&RrOJGx0xreR`pKX}vhTfT9g9jqBNkDj z4hXirAc5LS^kvX`i(4XT+G~ieA$Jgnw5>V7w?wHaKL+{lPv2Qa>$R)S`HG8Y5UvCx zg_V(khqPX#hs4(FzW{2|vG#&jN6nG*d~Y@3NUHyP;0Yu($8?!;p`Ec}9?M=K{^t%)#m z6}Xo-z-E>vm$|&@!T`*s>Et!B{u_*&wEuJkL)I_slG zRY*5SU-#%RkCPO;V@=EFqn2milWE|fC{0n-bl~iZevI_Qu{G!hCG*B|mx4G-9;DZw zGaY+1!=1j?X;`1|I<+m|rxqxS2fiiJ^w|q!?#e^Y&(PyQ&a^RjVV<|b6nxs?)4_rA zJ+Qv~yZ6qR%YHR10fVmC2>KB)dMh0Bx>uE?#_Xz>(Qp4KYQ0ijKA6E64(?*DOtS{y zxgMHSXpx61_Pnc1an0D;A*ZFq!U92$amMPoL>8cJM8hSq(qJF%@C4zurw*4<5|_e! z6P48oCrKjAo#t`g*PD%jXs{po!#T`u^K;%HXkV^GOLx*45iy}1*LkhNs81l&GHBZ zuM->bJsHY%Ya*@2O#r7i`dQbd+aez3?$0D3BX+4Q{roJ@5*-3!pV7FdFN&f&F;2kC zti9ZWuZ{^hHClP%ShU9L#i+#)fg#7_c&uCjCHT;qtbLox(`z3)&Hi~d>+l{6^@1C)Au>Wxa?F!TFx88*9E{=_dSv15{MpIa9z6O_^}2`T8kV+~?E>!cJ~ zTgHU(+Z57=e&`)Bn-)doEMp`+3HEvY3#w*wt)F`$@~MNS+_!)Hm^5-ae>xxeWV$5q zCv@h+wP1KX)uh9L<-QKJlX_txY(<(GfEhGl>p40FrRYx-gk=B9P^QWv zKA_K5+RaiVyy_Bc9V-VGbynOpfed$@Pmp3=p?jB`&yU3Y3~2R1cUxOy5{(3hm{-pi z;{5tMsErFE04ec$Af?Ae(4k1~WhN|2J9HE*J*%4?A8J!(*pYqPaM=~qF5jzNp4Gp6 zB^W@oTbKHFQQwPdAK{}Mfk!Hkn3vu|=MgTpT{WtZ{6(%GMgKsN@qSCxPgXX9fN>LZ zcxtaUomcY4PM8w~G!hd-k7m`6HHy)S3IliUa5Sqo_JtEG_pUTQxhXo+AhcGe#{ln+ z7!OekM+7gdS~`$ZuI(@0sqtAWPwk*hEiYK$0Vw*w3FwHnz?r@N&*5-J-V*r+XIt;< za@jxf);gdU^7bIsS;s&ERo&~Q7GO_EO`Mv&R`op*@zh&M@njwl%DXn;U;G2_sg%wY zXx*s$;3GbO46^40!;CtUGv(YQ$5V5b;kuG>acD8%d95Ct96RY`VsAg`-Kbvd4&wYp2yvZD6LQn@z6W{I*7Rd{}i{ctvu z_#KlMGUeChBwf6+woY<%pz!nIwv5)6f`FREZDEpsu64pgc~8XQ<)h%t*7NRV`Wz5( z&?6DUFJtHJx#+?t4Uc;KnwLO118MH#-3zpS?xF}2*eANMR-?^Z>#LF%z0b@`Zhj0j z9r|fj3ivCzRZ`))dw~0}s2%qL>TG+bBwD9lQQJLeRwv6Qe%!m2CwrDvq9(!OnYv@6 zkNOmRsaD!*>vyl!QR{+(`V6e-M=t<#-}TrhUi0@Js^;hM`h)k`$J7-D5cs$*1(+&J|a`lllhx8G?9ij+(P|EHEsMhy9wZ!)c1W<}VykEp#P2cC)+kNa| zCpUQTz0X+A^Q6^eWHZcJpvxCiw{bPxllbCCrR10Pdl*S##s~L6n)?FXBD6}jk8-cR z1=CJc7?@`cobA~d^_{!mGk8=5LZmj7UOqEIdJmZtql3=+KOI*+-<(-^A^!y-&qu0c z2QQ{VzB~z{fU_Nj{VR{#ond?7FHZq1nTQYWFGn#2bGj@%0AbzY3Dv+VT$bGF%0_;_HUmT`nTzLKp35*$0AqgRFf+5>jakyN8oJZOmgEs z=8tAfa-fwZ=&9OUZNqo|u?@scTO5Wm&j0LNwvy)6^=#;v7Zb;fius8dgenEp_-3e< zc@xoK&fJya50S&K@#?m0eHUqnGI{fRNsG7p>WxFm^>wlJ-viFt zXtGA&vgZyHy9XbRjkI12luoHor|8t@R1yQy(!USP3CeMlW?7)_@(u2y0o(DNXvTS;ZN!%(hgJJrwI( z#hBwH`Y+tIR5k`*-xJzLOTcwcnN+GdNr_6zCftX_kWUNovpE{mdfV`6U z0ArOMt^11bJl%ac%b6UZqNLy5cM{$DZxFk0v~8dNkF_{&4~e$*p!(QSa|kbV-BYZnpO?9Ux;m6N!T6D>Lr^2-=m@hTJ@=QKM7dJTW-DD z{x$7S&&XkGJ@%F3D`xOuw-QDY+gq+RyvFyqgjJl%Dn4@FP1s^7-}n@q?>ZXCDfQ5~ zt!-}&k0sq#9}SFO-Hd;rrtH}PQfB_}YA7vr6QVX4nf1hhI0OA}m-hPeBl5l#&`rOB zM9|_e<=ap07mMLXNtUu_MWv8W^?5KXOp8f4!)K_9QqMVh(K@$+>P^B9C0x-`0k<5n z?Cx4tHG$oY2+8j}qwwZBClavTiQW^Nq5R%!id^94*WZD5QF|1h>IL8;)FhWZH z7$-&_1trYD%)*va{}j-XTf@9Qw;td;z(@Z-rmiv|>g;Q~|6LPgq)P^sE|FGXKp9%P zn-v6xM!H!=7(zg#q@AHgQt4)C>7k`n1ct7m>%HSEx?lOg@6NsFKIfe0Jm=h_*DR$J zic52yCE25_GaL!Oluybzp|H|y3ls>p*1ptl(#nuUwEJ=B06vg|ca!po$@fvjo+YeH**%%ef5evU_eD{vNNnNQ2ekwPORGh{ zkq)A#O-yrZJKDOwK+W9KNIvdmA9Rn-+kG4yd6>Uf%fV3Q(d(-6d*jjJ%?760|JA~f zTPs2eGG2xuJ|M~9wsT-*>Kvid#Q~Xz(+pitKwXtc#(0-KrZja-gWvhFiBuL;l)RQ) z>7In%c0amx(Ip)Y3xzt(C)Kp)-$uU?;&Sk(m&a&nz0bN9#ZmjIU(LT2vk}0N=?xMk zP8BpOM4?poOla=xJrGcyVH&BcC=?lrU#2B)#jc~?s47=W3SP9gwYE3$8rgjFtHmEH zN@se8T>9%WfW230} zcMaJ-+vmqy-5TMtGo=d&}mp*)$+{PZ`WqPmmmxRNN7O-Sj^k}3IA^X z%T*xl+N{r?)8z{8&&d&=^TEwT-Iq2^*(LsbC#wmOQwUl1ybcOg>qJ|Aziy1w6s$Cw zj8k1A)W*GJvO1vrNJ3JBKpUjaiF7f18LnV&#eRfuyHLj`X4>a8N;?esEQBo=0@NBG zxqpu$c@Pe%+2dyL8!W6>!{>`oBRu(>H_I#JCd-1l2#TVc8`08kmbcP)RDMRxiV#12 zYF;0o+#DkwN>a0aC7@;h? zA8YbAqAsWKfUp=_R6K0&=QBYi8JkO4@l~c^G2HU9<(pOot;*EKMUE-HKybcG%R7&s zl=_0ObLvYC!gO$vnjDZZ0}({Ov74%F7Yk+q@YL;KWuZA)y!o0>N-L2%dhhraqs?%i zp9H>>eEWA=`StOP)!ZdWIgTglmX-|_ zHoyJmGX%qvz*gnG48wuRd`lIY?XLPhCZGLg2g`QV5@4Jld5@gCw_RE=LN;Cdt+u2f zAE|QWNDluv9B*)w^I&{Q)-(b*I}aV$o@QnL4Quw$PDW*;8d1u^00EPskgj^A%|~c> z8wJy#*x@J)q`kqsp@o}S(M$`H!qWcs`p-Unt|b7-1hJ|`t2XF^I zBFmrLEG382r*qlvaB<_YIkrnER9UQ+)HallJEHn+j{SR=Q$u*K7+eN6L{r=PG%Ipo$QhJ99*5#R6@ z^PfVFU=Y7TY3Su(&UeW80Gl4x{yOk9GGMNoJ{X6AQa(Mvh@fdthn5n0cowtwl-mTo zmG1{0t}S#Yjkk3U8&FpJk;Fkc3mXX;%M#F4)X#l>du=p@_pT_bIL0j0+P4RvU7Oh9h zFv0Z?%4)VV%3s0 z_let+8;9DNxKC4QwO~1B3jThv*;sB9@?8Lzyn76-YBe69(p*QO!|Ggu7$tEUW7N97 zdb&n)s#!-vrG=WF?5zixKoBSkZ^1fr;5Y4_kukhgsBU8JcaT(`*faNn7~>gN0HK_d z4LuBcF@9hMsIy;f0@(Ylu+sJDnd6eE-g5TfuvkSj-B@XP7Nl^H8Hh;IHCnKoGZ|V^ za#1uFse>812Zg&@zsy^-uYx|=g@)EI??!Z@)?B_TwyO1?^d9GS+cqBas}5Tvhhg>S z5&Z^%*&CmlqOL#?Aw%U&6mCw%_JjaWi@861yW^pF*B951=U!lvh)*lh26Dnx2IV}I zEVRAz>NM2=ax(Gc7yT)y_>2q;5u!w8W?C$(w8AFS=t%+sEbX-r4(pk{ibhPkasbIQf3&l zB4l-hhErV=Ktk6t*NfB>2dVQi)jw3~Dbl7#UquWk;1$)SNcT;=!81s~j&X91xCSu; z7mdMV0qu#9M(MTequqB*MdG0*Zd2~9pBKo57PLH%)YQS7XH?vY{MwL9JSAINAHNp( zHd-tDESOQ-Ze@l;$QVj8X%ZcVsj4{FQL}4uKKFt>oE(t==*WMC*XW8zhSoMf?`=2JF={iIUioLxG1uM+x3H4+j~@JLoTLg+sP$Z`b$lR?hAoxJ#4+1Ip3)@V6^PXXxsd53c$g*I45&;I zkMh1O6zNZFH~qJ)5OGVAS!=_J*FPFm3ZpzUDeJ5}0lPY~=StK}!Vw%v{WQ0Ho{!kU zqWg)n2gTWJjZoH4k6l?EtTL*0{s6gd(^+!mO6 zq<_7!_U{sg@*`3}8!zk+gStX;nsm4iF8V-!6^~6yGhWvWnwv@e47X6gCmvD@_Z|WE zW@)D-F}Glv)>I-mtkWDLBt{$ zFx=Sm^>QTbB8rk@3SIzw=>C~W&qOsf{MYqp^}?X_QUs-BM0GS!qkAe z;s~|$&2W>fA0YTAQMFZ|7u#8^zyonomQ)0SC*KH!#7EzX` z=@_d$ClaeL@PZnj4B2X7gCahx7PKO=0j^Ke4`Odk`8y?HB0Q|xl%Bhboo7`C0nhZO zvdC@{AqkP;){>Cq5sy9Q{vc}fCV~mV9;enH^yIQ;*>DS%{;mZvW$jw^TqvJJ2xaUZ z*0%9F-co5pqy`ah{2(+GSfvJ(D^EL#maG`W*P05A*^o;c(MeNqEOVj zj#oR)ahV=5G+}yKVN=h-qsT2noVwq<0ZI-QYK2m+)9W|HZ?B^S>UBYftA-zh(tc$7 zHvy=V3qD%#nD9}nKiNe2m8Bs(bE&PmPhoiafYFi+&Baq`m3Q4l5-l6ozHf!hZpYwuUV+rL{U4 z&l*Z(b$7Rqq93k3^`V|C3gtna<~YLCmu46bUaMaDCuqzdXmGlRsN6ma>OEEuhRxbZ z8*+Q_yp@zB$ffDVuY6b(5+j4hnw9iEmx4?wqc`6c=|k2&$nHXK+qjCMl|F~@;gif~kEcO+v@!{1#oZ}y zXVI;`9~!AIj|P_laoy8wF{Z1rXxfMaG?<+&zt(?UWPM)4;HCq1qO3}=la;bZ-WaOd zl6`P_9rR}GzXcuzUVZ;k#kcepB5{}h$#Ud;@X7%VCub{mxwq$5jW_sGzk)45%DYvc zJ0BK-U&UJwiXTD)hV<@*b@!hDGVMM3_lKfTcY1!@0$`BilBUuS?YGyuS6aX=NV?W7 zyvLEPl1#|6BoR!(CdKwCrX1|cs6i7m_B&>mZhH(e4F{^7j z1=89{xAe01mkf5YpsDBIG_4Xx)~9SqaIK^AqVrm?I+~|FfTupg6)VIRIse=@H5QOW z+~H9~=TK4OPz-|0;>qi%Vgha?opPW4Ed_;NOtA8W-G|$5ESO_Juy7X{iV6uy9i{B0 zVC(M+W@gIZwK^%9bPZ4+AX~~nmOl?$*ChpAqrZQV7MzO;sVK+-%o6wYU@USEdnv$Z zblSwz3rxm+gA3kCrGoY2X~S!gbl&S%`V~}F!cZEJx~CIP`FtJ`AsEVpx<%m4+POJo zgQ0*gS-+>X2AT~ICx`PK!c?LA`rEehn9}1wT-xotscAOtyUL>a69F zUW@EKFa_o6=ZDkIiO|EUMwVZ~6wE*xKn5qW)fHb-n3f=26u^!Z)PtkY5m@Rd`FbOh zG(f~O>7oZ)0WGmYchHtU3=VPT+CVB+xVAPCI zNf2!xcB+bP4oujywSSkgWVrO6NqVY(@*r+T0c zX#m;*g&cs}R?8P|wf_?2t6g7RM;W@Y^hfql3+;a~7BKYm`I-eVMg?8L?O1k5OITC_n7W_lsK+C29jt-*LGY62#>%3!?{vj)BGZ8zZnlX7L@Zo{q;f_^W?!J&|l zY6EMWG8aHd%!AfX2z}K_0ur}idrg=}H-FCqRm%#5x#qRaN-v@eoXqFo`VhaZ$jxMX z&w*o7bUSFJ`TlWDTxy7{n^F<;-JPuL<=YHkKk1j`Y*ZnsdejsSPoEO*_Y&`$WuJ$? z&hHaY>`=a!2nDd=APB89_f?sOyX=y$dM&+|h|!ahi7@tfiFxpLx#B4heXkr)0&qe{ zGL&V0=hGVc2IOGltEe>IPL^Qz<2#A#w&&1y-vEa))Mit!Q>Arl6n`@Av%NM+4tiOk z`u2UAye2M6o!(%%Sq@46r0zaF;?b9Zz#Mz3GrCon?kr6_f@^>~ew0I2=oB2?pA7=b za8UXX^NH^MO<&#QX;BLe0|d_aGe`YtAo^Kl_O@NTZ0}=J!@&%a=nAG02F;?`wGy13N;>6_XEi#Iqtk{1gX zvlU;O0>`HyI7@3YIheu3!iqA6&+nEjGl*e+0+R^!l@ z#%J{xX#VW9IgNT7Ei1jk$}?RGKs_=^^VO@7o7=wnV97c4are##yJJNKY2Y%Q1!S;e zdv8Qet-Y@T6fUA8VYf}BSG_o1B8}gFQ)rHR&BFw`8pv3Ht8~v3DA#&MX6&3rn@Ez~ z_DwQryW!3{iYt2ihsFJ8>5r1b;d3C<)-^fUe)w-*`&Dw|zMHHkgP>WfiO1gmy1S+x zI43vnmk7|B?1QUJeqbZr;^01%|3+H(Pg>b*&7hBFkg}IYA*j?k`FTV7pT;5hViz=% zhBA8}=B3km9#`n(++BLsB;8OFLeIrGnd)N{hN_6*EjtskW4!I{tK0QbE+FTVj0<%^ zA{g`vZrxfZzkWo>=P7WWjR%?H!D%Ua>*7SoJ<{=jrbmxR>02tb6m2~Hg?1vlL19R_P zA>RTn%Uf(qv6~$F@ugxgnwFszTVh5mw948qF#qbor>8=9%pi-`tiVlj_2g*%ucW&S&yV8Q_+h%#HzC_E#O_xK|PfhhcZhAo=e@1MC;OJYo1zcx=ctG=E( z#wOhgP1>Da84ABDBLEb|LJ^fff1LeLha2VNjp0xRz`UZ*7Ox4STkvH8e*4~Vx&Pkv%rc4I!0wfp;gAs6Dy8}VgU5*RSHOJr*2+VMx61ft z#($pIJ5u}vTqdE2k-_s@#dkiz_2!zT?p6hv?0L9s@Dd7vXeUY$ zdlvcc^?`ovW}lm$-z9P2OCBYYA@*#+paD8Fpt^*&V>YD295=MdQu84Hgobm6a7uR+ ztD6!}-@D0>)tM>r^V@fYcXqw1R0ra#MHE9V=Y072O&W?&0iPN6JZ#VM#$eW_qer^+ zB>`rp0Q_K)A@aPbo%S%z1u5yC9d_x5zX<_mSDM2E#MXwc z)}vjfN;K#3ID&L)2!3%Y+&sbbn0%NfcaAwXL7{bG8}ut69OnMysm;w=RyiO=3*>Ys zg`;*|!7I{>sK60dr_@9nX=yDRr%aF=qq%xMmEPtC{p^N*&qtGI6lth_9s`2Gcmyt! zGaZnt1!3?MEHataQeLkf2H@gh360sqdbVT-jj1H_y}K_pv|v1{Lu{mCi)K~tByIpM|lxx`oa7Msjp~rnqBWgFPXd~nZ5>c|A`-f!$3lBWp zLD3OS1b1I;`TrtSLcC^+nzVF2#)D@sYY4H+^kznjVCQ-R7GZU9rKmY;_=P*@lV)p;ITd?g0ldFhYp?o?5|PCkLki5v-upsedwM1eC!L}t`h^zh;=$7f zzS^1s=_D?CoG5f*Tp39lmM9(uP^u-(GLnvz)YiE_F1{VSMEtB#hE1VVnXj}2j7hw9 z>s*v^p&*;u8x!IU@w1yCiaH54@wZ5w;r5vS_{v93GbSB{kOluYTixTbIW9El`&EL) z3tQ~DX@aC%#`uZwO@uQq*Umo?dsv<~2!TsTNBf_M3O)tsW%uItz zUAdqqH+%YhJTKx`bc(%$pN4liCK_)3F^5D_?Wq}S^&euaw|G|RYGK>JZe(ZG?R!Id zQ;I+N;J&$89hqSscul^&;R70=5{?N*YnK$>3*@mt-I6xaIN%I2^6;mChoNEv2-x0Z8PjW5&z;ap;*K8P{=C|!D8 zb;+nJZ6*?Rl_Qjluw9hXf-Fa2Z!kUS*)N6X2IP05WMQUSp9hu;(``5+(khsBlY0DC10H4g>>L^X~H zgh6JHYwslk5#S*CGR*BYBD^qC@g#Z#s4$*Bif=zx6NvR&L#9Ob4F(sKfa;*}v8r!5 zk^iiaL%a`A^g8yy3dVYzFT(p@p^)%VBkHL??V;k8jh8K%!%6puf%cp1z?k%Pd>_1)X%s2fgEqFtzlry-uXd#Y8 zop343!4uIScVOt@nv^HD7adihoOTC>}+G<_Aq#qFh!#C`8*R{E!!wP~aO=9x$6AO5?^&nTRhgRT1?;qh+LD}r-`?WT5UWL$@=NG=qpkX{tVfUtwEqGt#i4jh1IDGFc>?By z#E+-Q#D5B$?$JhCkF%d<^;>?Si0W1##|RU)`uK{em{21ew<7ROW>(;EtytXwe-@PJ z1}dHX&X8gAJDI<9C~{ak2oD&%D!`?gnj{QLCA!maWPO_MY{T-nNSB|UBD(rBw?~aZ zQZqM`2Q9uC;X`u*0xW_yB%1`@T3RzMC8zIuhAzco-j9 zR@E$7lo<85a=CK^_u>;5R1A|y;j^I15W`OJ%w@9?S(#{L8Z=4_PV0@BEiFZkJd4hr zQTv>{ko9xX|G2+Hjx?sx8e=a+LbLBeq8fRp;R-&V8XJhFkB3VzYPPvtXh0dlbiJVUok?h*bE$Oua%~Vf2F%?sZi5 z6J5}Ul5wz)ECC_R$AkL>n{j(`D-9-DQYepZ;%Qo)I*d2;CY6D#4o%!Xf2Qc+V80YZ zI|Ajsg8CnL%x18GX*`%(bJ?;g3`!SKwCL(SB)FmhwnauD&Kze1ibjAHAQ=1M0BvUq zo#tASk2#+Lgr{CGcmC9S5jFk_JotA014s-$^H%^!8kjJBt2@OF3PTgGrEMp<_b7py zQ$f(mA2bYRkhnZmg}|P;7f5rpVc`ONkKQApghU!mV(i2Wq=IUd(aZ+(1DK(c2U4X> z>p{N^vfv$vi;`MVhKG|6^UWX({_~PVQ3$rl8 zMOW=3!Sk#;*r*VIkQl>u?oMX!70YhgPP`{s!E8UEL%n!^Egx|axRh28+XY#h9>2h_ zW_Y*{VmTMFlM%aRq>EQOA()J7>+M$Tn8fyRqQ!8H7dkP@(wwokB@@ai58@!{IxxkH z5C@%xdPXra8?>7P7HG@l*NYw4Yoe7F4spORBgmzBl6-#LD({TH$%4KDXwD?6>)h`6ha5OEW;+?%RWlR zK>on!Kb|l0yM#Rg=3e^8>(tJK!SHP*4P2jN7jmDKOct$;agVyG?`{9vm21gO1=I&#e-i z<@w38GU(qK2Qc&=TFvqpjNt>6x(Jc*zVvXEYLkdQ4ppNdSU$h_!mGdaaUP-jG_3-p zX&q9;aXVOmbsv}iMD$8{dCcS5n~ToY1ytkr7N{Hi0$gY8n-fbXK4DS*sbwtTF_^K` zr7~uNy;&d~(h)x>O$CJ2`RKFp%o-oW&LD?<^2_(xNMQ{bTe_6|xw=$nk}8_MO}E-y zV@Px^29$Ir6~bp~?zUnBfn}(oM{a@7iwWSe7YEqjMLKrxSdPd0E~nLrc-^x~e|k?n ziiDRHyzA*GIKMjGa5lJak$G(8HV%K>irr-fUj66qMEcgWq8ik=Q{%-3Qj zMtBa|>qnyY*^i|(N%@)WD(cN1E2Isr-bU-@`Wn~qaf&8sPuA6wufNr?YK=&~^3- zqPR3AKECWtD^*H0kr1X6z160sPoUCr*Z7j1>F{rlN8oEmOh-1HX_AbdyR% zgtS}B(7sw~A_0#+MaQK1)%NO($JjxUw-TlhTJEaqPxtMDKrPkH1z%qf6@C?Icg5(DK_y3L(vrz%X^}lBj z-L&6UR-CDQj3>dZ#SOpP$v1YM4!VD8aR=}>9eyz`mWKkhIc{@+<1IUmKhFM ztwYk?0_0?wg!6z^otRIlFAXI$G3xqKlZ)mv0<$7Ox(sW!^rS>EH+c>)$m@05< zrJD(cdhxO`&ZJ0YjZvrI^(AQ!r`?HpmQcWQ1TC&EWV`Ky8*EH?+)Dft+vTz%kvEI< zK(vi}{UnvfAPg}4hAeli^Gqp!Jp);)0GcLR8LgY2hFdP!00w%Yf+64(vP6@2y}7Ff z=DNkD-{mtU(&6>YfAyrpK8r?lofRy`2*F``6O5SY7n@zT-H`6j?5nPmb5Mtzc<%3r zQoS58mH`K-9mQ5?R#f%vG2rUtM9?MW%zRpub$G0eiMt;x_Sc3?DL#%tFoc6S5kqh> zA?u9$;-K2!o__Xcf9S&5yF(GOh$8z=FuT9grLK$h!R4OKJ=W3bi3vEe?ayu`Q#}8i zAV{2FKrS`VJGwuyb0+4&W5G>ME>SJNOFJW+ayu05pK3&~6AaVa+rdTlSWnjV)dZ_ym~ zJK^lxoFz1 zn&g->W|scRZ;%@6H!i(#-R(edb(qE-d`*;xl(m^uyUt?=*$VREnvcH5;e5bKld~VM zP8#lmn+zb_iHVVUs1g*H@ScE~MkiQ~hN7-j=Oc9vBK5AFn%u@S_ zDqw!+kKfmb%;+eiD{`V-H(n2>n;yq25|_EZ8vur>mmPyx#n4%Hj=TndF{~U3^J72>GuS?hmx6UG)a8@h5y#Cuj4LuGWJXd@*$k=9&|BT6MBj z^HuF{pAw_xuRmxwdY3-8cIF(G*WOllz+4Bu&|UfrmB-Tk@F`)EeYjSVVSP66!(5#S zMR~0Kmjh?)x1XVtA14S>!XLBr13RsOZf&5XB77|TjX6i zh5TTG%lT@&U$gFFWr1E}`7GPpqkF?Mo0n$#6iHz1m(Hx#d+NvK2mN}^z8R&Nq)kwMrC=<&X!gDF`f9=iES#XPqQzlKtBw&eQx6~d3x`PfV|?R zQ2qb$UHR*Z;6FE{_yf7SVGJ4%Hx>seZU;$`4y=ROP+a!mSn6~s;?0fv^^UU|(Bz{$ z6j1hE@AEx>|1ao^u_(h&BI;8(nOz;oeQjxnz#Cz6S$K%o{Wbl z>ZcR`wgy~TI=j*HoXogvnMgW#3H?9|olk7Rc<*q){T5u?k~Q0a$RUvF$`lyK7|cx) zk$z*zy~;$AS6Ppp?%rc{6fJ0ZIG|5lnGQ2|FRK5`*oBmZgv@V}bQaD<xvNVewz2Y#h#9DwsdCv^ye6?JgoQ{FZBwxm}4-gLeAH6T@n;(Du+JF6U~g zbo;+~b%NK1A0D9-{Q_828Iw*rXV2Vv z1>T&*NS$XRV7sdRMRH1ovKnc(gh}k%)`9$-?@@lW=%c^hmjZ#+)k6#u7yl#)-l@Dq=-E_Fff!oP@~l` z4X$=jQxrs4w+{z>lC%%ZOSx*XUD+&DQ)|-A?`j$^p%oj~ezuI66;pSINSqt^g$vhM zj>!7CEpYz5zU)5Xtm_ry(zjcisI0nYcT9KE4#awqhhqM6`&db(se{q!!2p4+)pR?C zs91J5;~r1IL>)y33Q}jQS1wAXRVHmPPbNXZ<3jpUBPA2!H|2C^XZBA^9w8N1)g6hL zsq7e&ntjK@{cqS0wn zj9;N8hjO+?|M!g5iq+9$)W|ii7UcY9FeJT<=K&S#9UXkY^@I}!4tOGQ%FzT^>EPbj zS#yi)Ch}S_nka2Y&aX>EkJcL_%|-n9ss;NLzliALA8YMWS(Yij?^-i!Fb>%q=lR+e zlSvFyL%T7ut&}AXZ{aCpAvZ!4b}|@R?*ds46E7S z4=D3#n@x|5B8~{4+CNViE?i)beHDl4_QOm%)L2|Pn*(c|G|qZ-vi^X!I{QS34PN!O z_q48TlYYLDe`YmDb|GSoMK~X#d>bBRFfXgjPVXWD&i~V+rZUDi5`y;Oj)<`v%k{# zK1cOb?z~2whiVtpZYCb6sXxWlqkR!3YsXg%rh_+2Fc9$TT_6E8>u^W`PEs#d<-XnQ z)}+!^N_$WmO_(Riqg6_ODel>$ViKQkUGH1@ups5U`{%z(Bt1kJund8DVmv> z$x7ogX9R=bW}^Qd{I$#rEi8YsTw~{yeZaeQwqz{>3SxWRCZ zA)V2iLyUHO?v@{WO#LDrb^(#<_t4!Y&*#ZKG$ z8o_JtRXb*QH6;Y^qZ=)P*)yo?CQNfYG!?Z_fzl;sR9x#4T9d6g_KC?OiI(KY_kKn& zU}agiv_iw~!1vP4O=@`#t z#x^}^Q89NDm_a84^C6~L<`8>i-+dz>Wsr6H*n=)SGI*%wWT)5b18XBg3v%2UZ zBWD&FWohnCe{v>d2NK}Ie9e9~9?AB~>MV_uWX_j8xv@(!$+|$#nS0*W+$Tr3$m2m_ zi3DCPbSaK5Mq)y5M*==8A*k>%}x{zUR{}Q`e?W+F4@@t9z{O=Y$W71{}C*A}t zSQPRh{_RY|<3nuOBJ0v0k0USI&WdD*P%5v$GY%#idC#ni=N~W1h3~2q7_A$<+C&+X z?0opZ>OW}Sl?kzEI==4U$JBbWqDw}oYsYYB>?Y5U`-3Ozle)6ntR?q%z~#Me+a;_l zBr)-DH`O!CE|F_HG3Uw~r`POk@Ip(&Y?YQjEp<*g)&!cueujqtvQZoA z=sQ#ZX-1HW@1!sCw9S6_%eBF(=|HVdV}Op!a3SOQV@V}uIFm$d=%Nzy?97S9quhQp zImrx9YT!0%RNxKjG6BMIddTa0b5Q_%*2D8Eh$Tp3PyIboqC+Q}V^M@{%?SAM?H zdY!DmdoeQ>jmUB~vH)a^%YjH%2P>~ruj;)D}d13w|^3>m9tRWHG^Y`M}2MLL~`_sGz4HA9cb9<4t3`Cj41uOcUdckOmyDzX7uYLkDmB%Z~ z$|}JpD8#?{%12+5vy)!=u_xBXM)Ir59b%{AUBI@nWcQRxb*iyKCgS+y1&oNR8GXbJqL1RA2-4!zmUxBxcJMHT)3J}hyiho8v)%Iv%I5H^jC8RFP zMd`lpm}9uI#_>a$GkAiCQtn!zKwCZN&Ee5UdC!)dv){?Zx=B=do|k0VJVpRaecZPuwYlDEr*_uenhSAWyqeB}2I2dtc$?Q$6JPk9VU4 zIN<^99R0%&QbTMFow-%=ib&GV7T~l={~}4`1k^XDjXZ?X&28EzfdltL^bLiDbd}>) z0MHfV^qRq94W*I<;w?IZ$?Gdt^oiEng4Fp_Q8R@ zsftqElNU^Od_+_xF|)~^NfiBD$ogwPZE4Ux(DBR^gO5%OLN7hS8F&-|Ik{XC!V+^C1 zzSX~^G74)#^7rGZw!nUVbQ{utC0ft4hipjb{ue2AK1dq?-oN{*eZ&Ep`m!C^v+d28 zm8e%@uQ&JbFCh@v7V7l*c2%9J=;RYWUvKs*2!n*j>-S52)WKr!A&iOJ{@c!JTd47m zDU%Ki7x&8@yn1>yGS06ka#oPoMxuvtJSLc2v7$4m}%gJC{n70jVwOW zA<0Ibk98~BnV+Rm@6`DGYTip?A%Tx{)X#cuwZ^25gahkbt7P5$-~TLEzuxa>0(%BFQ&}_O zg)gFMpkHnDI!Ld1_d5yGrw=4y#Rx*f^9@@ZHmQ8dE23N8$_cMDhLU zk6-lr7Mm_5=jUIwXg($#CDstXI_fWfi2e7qC?q{&B_2Z-v59(0u>dD$*`fg9y^*tg zBQaTlorg1jY)`nq>Q}d(p1)bM?Ls9lwlArlMSRR+yOVY+_M8m|*%OUo@aguHhXQR< z1X*iDyIrw_^DGv5Dl96F?-IY;#wW&?Q6=HLX*%l%;*AF^hY!U>*PXt5E6hg6SEWEA zD1ahD_u1GjnawWn@V$`ukcfF_m*V7I-O2+y6F*!!K%htARJsfr_da$PMDyd-P3!Uw{YH z2oJL(wR_xjRVQI>2iV}8D0$k>@3Ipw7snafd(Tc@Exj_&6hO+{jTGty0J)ynZV^b) z%68d>eh$7}lLms8=Tu1OzFNBd1+XajMLK;J3-*}H<8xzKJ5n0xM4^A;crG3JeD6G( zm+bI-iHN6v+r#wB$z9lzuZ5g6K($%%C|Y*)NoTMsoVqwkxNrQz+j}b70n-L1UT&XGi_51H1yf9IzUmtAJLg&?xuQ2_-Ru$_ah zrxIEivepw>zF?$m@e@x&2QEQ9n1wp6B4>YL7_S~}=YiVYXlM1jPdu=k;CMaq&Q{^C zO|5|xzUO!x+njr>)0|xWrfC}mViTv1+d&CnQ!bBND2A3p3MmXnjj|A8?O}liyTHhe zjC0x@ha4-J@5Gla9?`d+Xr|D3L z&`OL3BtsCR@pppi~n{Iu|fNb& zC;cDPXLWnVE+?!=;Xx+=g(@DZ{r^g4hIxz}l53Rqu|9_~^Xs#WH#w;7EA2Rrp77Js zaZuFlt17Y9pMG4`U(}GDLz9diK0LX<2yYj)e0KHr#gt!Anl4)(BqHt#Xt0Ev8lMx5 zHl`d6>mTo9!{S+*KP#dRv3?eu>bv;`z2&w0YNx7yF#ZnYXqyI+YgQ=f@YQUzo>dPC zWUkmjlr4rlpYN`5Sf^Sqa(X36l^t%*oogcGwjM|!f&VwmlR z`EjLPqfaWX8wq0NUq=@EKULb!H;H6CP03;Cl*)$n@3MPE4;ZV3H>F8(M($VozPFFu zMkxWiisIc8iXzGD$QBhi9gfo2R}&s_^yfF!T%uW>_xi4%7cPLZ%MOlr3ZUoc{`RaU zhz`1x{MAs8;ht@vEamph%>w&DEv9&4kWbx{Ge1rCVC>}n6W|pwI7u!uKir^-)gKVD zDP8c^jrtwV9{qR!UF4OsZ)#*HMG6VQz)JC64DpFjj<9$;%_N<(ta@#zn;| zIQP}MA}`k2EbSU{_~M?NS`;6P5T(ZXcFJ5%&QBp0de044)|s4?y+(+6HG=uSa&h@^ z&WOXxbG*|#%Ioh?+xisnkhdb=TzhkHc6|5~QJDj!8*GiUV=?akYW^$XYHF<*E4EyJ zp!#$fzNdcY_rWGcCqkyEG16b*0-u)bL;yJZYc!YSQ5!CNb^^3n&yuiprV(C$v1L?J zf_b(i`)xK?&!ygO{I9BH!j7s#~&-Klkkw=WLT284AYg39@Zk z-Uf+S1L7AHzM@E7f;Tf+)_CI4Go>3z#-du~_?5!uz5gPPlh@Of%g~g%i=gHKX)bhk3483ZOzY^ISORYKpqm z&=74MKlc9w99uGNJdI6YYK7QVBNiSlfEf|yL4$| z(3`5DL_lyXcG*Rzc8|7uw!Yswdaen74D3@P{OQP26ZjFYe}us7V+O);$;*|ppit1e z+yOo4#C)@nzo<+%)vp^ez%7F4Z+it(7L^qI*iH<1(&PLrm=9w$pY^AW!TUMS%su`t zDr6qg?@gg-dvm8a!ruDCWSJ~WS`4WhRv@hvJX5E+z*Pr{2+)vOXZwW*Vb51A*ohbk z%pN<}HftxcTHGtI%X4J?K4Tmzg^XwYZQpxhSZ9h`;bLBgqrXiA7fD3lWZho8w+#ak z2_ZgNe%ng;1!RJ`^0bH5U;pZZ5R0TSC(qUM=f2sUwXaD@g!m1*DpD{iQmO|}bnPEu z)kY&|WNa&w=Qgl$N;&yZ9CjtG;mMlBv`B z??D5y|7+&&NxP71IhtstFYut@fCx!e3rxfM)B0YtU$@t3D>q>8{R_$rsYaMhf}4lL z_r5>1VH7M8QT9vh!r?~|=Z3wBL`xt@{S2v3BQG`X%}QWc>8(|h;hL=CJI85QS2e9hn|^Y5$j z>JaV$d|h}fsv=!1&6S&a6R*{*I>%{N9BACmUI2e;wqD^Xowou|zDKj=VT9M!o zFgsUl;vaiWBcd(mB?O1G)5f|eO6;a31j&XUxi3phFV>!RVx$a>nN9Vbju^{%UHD@o zM*fejs}75L>$=`+T?Ls7f=G^nbV*AM>JZXHODo+-*9;iINDB;I4mp5;bc0GsH%JRe zcQ<}#y!Wc_^L+msbAIQXz1LoA?R~x}x9&focuNY6bD4E`%=-Ic?sFm+yc`&g>IQaj z9eZ>;*kcNoo@=K%X%M}{)hMsisEc-5qwIi3TVOJtDxpao-*%oSX+;RUQo8&ID%^Xw zxSEh5;Pso1J+^m*vn0ap8g0J8wM)BK5=ElZQ^Aj?q-|Bzu-#Jw?E9E*Tru0K(w)1#A2u^d?r^7s{4; zG|0nvj0Ct?0Qseu`w2Zqki^7{4ge+l&r9+7h3;oW6sda3teD0Dg4TwkqkjWY*2lOx zJwLuYG&&EUC`*soDUrL))>iNIK7VP|ded?D<~r*mXshwuAevxzKDxsLB%Z<+mvN>R zBy()yk&?k|hLcnMm=riuqT+Y^J^a~;##h{9cBlbA>Nj&pnqOj)a)8LDb^xBPNi<_N zM~9`Gc~eQObX!Q0+hfo7cjumee5!yeLgwd5yn!$jjXF6{**Ftt^>P;jBHxoE-yqc7ysxZ z=ux8WgIXOiq4KwfrgI}*5Yz%BBd%V_q}4*ggu$DX(^xYc`t** zu`S4j2T9d!-<)?nbvVe+pRy{;N|ym92TEyK)3AL$%d`S8)Wh_y@rvO~tC7l$+bj|m zQEfii-eaoCbL;T3<7~qKK8{tbqwXMstW*Kt)E*)dPNAF=rBSrZc0w9WN%5e&*1cV%V#5wMfl}pqW6FaSP>N0byF)G z(+P!D{`LKU(di*xM5yq|nY_l2tQg9kcYMhPf! zY$?wJ7+50l6+<2E?`w@Dpk`W-*1Vdf${C1&##MduI6biV>*q2XpYwse3?n%mZSp$k z?$=sB{UKj2jHuOMtk!5<6N6h3ivCe09ojdlWk(K$a#C|F3kwq>uubL}wn^WNnWHPH z>YVuWAJbYluG-$Q`P1(*1t}V^t*v{3fgfM>Dw9LoKjx@#I`F5$0#NE~q};;n00 z5df=v6kuJd`}a7S#GR}E8J)K92&P>>S0Nt21W}Grmu7yTG-hsl6iR~>#f)uo)y=4V zND8GaZs6M*=Ns7A$9gXH*-OTGcQ&C_1~UHSB(ojvxjtzl6mOC`_gt%r#v(_DRn5PU zHbJI*kA9BlFG}@Vit7_?k4!D@mOrBKP@Qo*aiD;y!DRW4k5rk}er_J=O_ihWQyhud zqz%h@L2ZH)=|Cdf_VR)E=T@XR9-6TZTV_ThqxIVpP%s77o;CZ6uDZnwD*%iTNM`1r zy~u-r?d2hWOc*SA`G95qIUC)xfk@)vgN5}fr&=bOBU?|xcCb@YwT0pCcNXEQK^Lx- z=rP)il-9mnNA_l@f&kNc5&drxzLwp<(>8@=3nBD=F>el*m>Hm7BvYK5m*jpiOOrJh zl1^e-Q#`Ed4|3XD_VUwx?GR(;c#YJmHyHEHgKlfs-$}oMOu+p@(~3Kip4)e)7Bj!R zDcNj3%QJ9*^_z*ZbqIr~IqA|a3H~+?qQ1+)>alH>YfI1lkZ&5UGcsrUUD^^#*%2Ch zFA23K0oC(~4;8X$_j1GCf>L^UU>+^v*7lbDt^-c1cYI1q?-5AN+->1Yos`sr8yKe) zLdcKIoN)5-il`|*H>br`K+>}nF8YFiKvokV)F+^)GY(`2`Ep`#&uS@3+-1Nf)qM6j zluHl2Hp~1F${Nv<`E!@frJRSx89?Ja;K1v26B#HLUjqcS?R0zN$&DReqS0-W1rT3NOE=|84lDb$cpgJpKA6_Qh@f;P#ygT^t0%)qE%-eEzD_H2w)nWpMaaO~zs53-mKB}&wG$4ESTRh~ZCMv75>XZ; za~Dyi*Elp42K3szb{Dz#pXXo6rN>$HqV;<^JFA8Wo_uEtCQ`NbUl=Q%U-+YDH?RvP z#|#&O3U|$oDBV~;YZSlf%nLq^T_<-IjwvHEz1PB92ke&L`?WIZy6bz?D$-j79vG|3jQepUS;ql?0{_vv0OLJCxOv-avCdX9yODvJb9rxo_hM)OJ2nr zUr!IF5xqbJdu_Tnm7I9geDkih^>E&?gbMVC&le)Lp@Gz#un+z>Er%UlRq^v1;oL?B z42|$8Bbn8|R)7|db@=sxC{un>K1t0fK1b6Ekh4VJakL!|EXQ+S^y_br{5$XTbtq)q zqULp(XgJ-VJ9HLar_MzrpU0}_f8%*$L=nXI=1+>ySWeNzoL0}Ww%t1H7y+Z2PPDj< zS~%K=e)Djjb6@wJi{%9mJ9v(%BkDVgc|jcDc`P{fjDZv7tay#HL^rW+o2l|1AFv#0 zWgmVTm^{a=O}Kf_8be671TSJCQdoe)bEDljVNV9PsA*ewwM@1HT*KZ1;2B%4$DNGa z>(4wq?q03Uj-gYK^NfP=&E(Af8wX~eaq~tuM6Go$&!_tvn>PSpY&)YDnigW)fY&t0 zI*3MlD^nuP|A-MGMwts-MpOM<6UH|IW!_ZE7aEz(Osr6K+seolKwGNngg)Pns-d<~ zvo7-tA!w@v?_=;LqiK6fOZN-28F}kDzE49!Un=|E?59Qz!g%;5z(!S{sW#_MvgdGB z_>DgEtbRTd-`!&4hI<>55muXJ6CHb@eT0-o1E360{mqxlDbo0)`(^&G9kHX`KgLlCf-c1ErBpN%|dygg8loXl$_r~8$DBo z!4?9pKkCm{ z{7ANkOey@pWTc)Ot5ymeB`idMHF<*`kKqHTmMuhUs~&&}lNorKXAaM3Q>qCx$#cYp zFs20PoT8ShbLpAjn)ehx3TrMc*6&&QYRpVE{#qI{!Oy@pgt_mlTvv6K+C+>-y2#<8AFsiMyQ!#BT4@*i@uRd(Mkvy}N^v)rZRxupP4m$n3OA;j1@FI6CbLF1IYG` z_qhsY3hTMc&>wb@zwKIx`E&qZ;|^9$1rwrD4<;oI9eipBWaXnEfDu(0j*i?HIj7(k zgEJ)GTm(L1g3CXxC+QULivM9z`~V6SHk+^jB^(Q*x_^6?eTOP74$3ZQ7j8 zNN%9RwJJ9q1@ewm%7_iso`Q@kKLbyuaji-yi0ZC@VF`9}O0_xXKA|u9_45o<=Y8UC zo5oue)U#MWqgCn!QzhescCan7(nesv_WO6!Xwj<*dRK14uOH(}e}%OmfdO_`y%lh~ zKAJf-$$^@cp10Lp82i^{rUEgHl-8&Jx(3gk+j`Im<12EW+YkB*R|Sults6j)=4$>P zOv?@h;p>&F$^43iFpPQ%#K|SrE{s_dQ(pr6@`M2xQ>x5o4inns^3E>JzqqL|3y)>@ zK6W%~T-WVt)2-JYlhkb5=-y+?LYa4O+|q+;J3e`DpL6Vjn^INwLuagxu#V5gSc2Qu zZvVOdk~H5-By&l2%@GN3i%v@@fpXDDDRdfPc4#}^%wo(Oq3LfaFnrdu@_r~NGvvrr z9s}uc_S8?iwVg8A5!+1_9EKd4e*EnCHoS<*U;q=&U-6q_xkZG;m485IOx*CEuakCs ziw_))b$UY8Ct5RiK;Vcf8=V_pzPgKj$dzqhlemMGzd>U)o#fjGTkI9r$`k%44#s(# zO}{%3Yrvfp>Fvu{LXWbSquw}FKUy?@4_*(SHZ-k=1Co|8xx@VjJF9XrNyP8!8H`Wa z#zG%7o8rwx2kU9gPYFN-6oUXO+2*Ok^?**`ZM+h=w0^b6Y1j9-jTFNW?41TblBa{QNJj{u}WD!u|zBp@5fd`xK3bOaZliFDSYe zrP!`_qP-50t=9XjntoSna6od!KuY#FzbypO85VC(Y@NEQ93uoc8OiGQ`MhW2D4J-T zD;j6sr_<23Hi+1?t{muBG2Hz^%0VOcu@CL1&DCP&EkQxs>XR+=s}#PFoUX?R!?1dg zX_Q(}xtXY~I_XaiYTjYHtf;JX#JJQkDIwjFk+*Nm$>iX+5Va;SvHQrhU}OjY!X{%q zfRPAc*~*2xW)_kcIS7B*OWy#k@Sjypv{dfWLyy*%Ym#5X1d7$fT8==qxc68u;0u(P zE4zohFAYLt{eJGKUzfK&Z;+~Of6_k^YY*cKH@Isr?$(uidc#l+wyCgiaDZS01;4;i zc1<6nWfb#wYv;*1UQxanoU^6=bcqf)j6i?G`A7Ymiml|kbg=q)HosfJZ&p_LWS$mFn@HHD^Y-XN2^(wyH*{uA*RdnH{&Tw%Lm#y z$@nc@vy=KX16GVV&!AskuIrgdrbMiG!10RasHr0VXOjhwlCz92r8Uy>2o*-m8Pjqf@!slj(C#Z!*QRa7)CQP;}k7{T-VmsDG z&k6+c$Jx^&^8!O5z}dz#oB%u?nCWgB_(k_G!7{IIK`W|UppdTKFFK4LaeG-={dYgPSQGsw{ECo%rUz-I)G zol~23ZpDs~!T2a(^~nG@CBD19nWbK{ok_{~2Ed_Y9jQT zv6X%$dGjWn0h5_K*gE5x+Cw>|Sx?&6Jtrnbvkx!fOCK;5oQw0)u{uqC)<39mfO z!VCjic^dvIDux(1{wdF3PD&`@yi&CrjbQW4(ii8c#ho!*Sm3~B31XdvZ$?QM1_4sJ z)HIAAtD6Qa-Ydr!Ok@O;0x<(DAAW^FSwJ`ONkd*i)bk*E>nB3Xps=mvndBPWc8N(; zrttU*kJYt}Ev*^^{wp4ep-~yYne`MzMXS18vl_o-1D+ z$=Tq(z+tvqrRKtCR!Sr*z4)+t_lll2Fk6VSfH3icUap*eeRwVSzoh&icpT%8mt9-o zp7RU+J;1{@H4^%m0^})+t1oKPtKdZcfpk4;Vs3ee zQnQcV^%pOSNBl$?6^zm_%y({Hky}XIGzjm0X9;R4hGL5`sNEost!^|ea}Rfwrhn_> zcY&4@wX%oZ_BCRmW~;jTTQPBC*XDbV5M8SV1zK5WZt2n&xmg>L?G~vBinu{MSE0Lo zTT{68B>K>3#I%CN^7Kx~=q!ne+k!soinH;94u@CtapfZ_T7V>6P(N~`7MKiT*);-& zeelGhB(=h_i_oc&%?mFw5jFB;`G9<@HIhYmzF@m%oo@J<$kopX_avfUob4?(9$MO~ z(72a$bz(8rI)6bal~ZbylqB4RtE^7r$@`e77!_#&_eYe~WYSt8;gn_)w}XAPd!2}a zw7t-`7YLc!dVg}q^;r`Nn2xdwmOz4)grSr-RoaXlPc&XJH)64Q4wfnDx1dW! z*BS<*0_NcR{h6_|l9$7;0)>{WruD8ZUMH{OoG>5BjS5`R`TBMLLQGlcQQ+ID7v9AK z^v8)0Bo^Y9H^x<%ig2$gh#FW%5RDH@FI+l`~#Zt&LH{>Y4bIlS!o|3 z=$hj7au@rKB)8S-b{Y~11ZSDZ*CXwmhb!lqq*U7wufbcn=u(5Qe)QH&TFOZiLA!XP zY@ihJG~b3$QRC_4{_MRaxrT6jBr8`e0x9SL%n5L>XVGPLO+Z2~km}1*S}-&=??*R% z8{W*_V`HY_Sc!RuH~l-Re#)zm&(P(HO!;z13)AnMbL`Fuq=i zeZ@ODN~wlUrWeYlCGjltG!T{zsDVYS*4;xM7Qg+&ahQHpG$OwF0xj>D{tZnXR&D!* zrnDE*zM76#jKuD?(5aEJ0 z?gFNbg@jWXU6zYHaRrF=!1>$`xEb(+toHqg$b{$0&CKL%z1n zzzP~#)@m(V+-^8>Pa42b`GxvyrEM;v@wLFi)v&kedJ@n$+!iJuk{+Oc{U*G~1vth; z+wJ(j$^FR_-3NduV4)p{ew<1FrGvQ#i#XzBUrZ7}rWr>#gSC>QN0ol#hg(>l6Z`YO zQ{~dQ`|a%x8E(c+mwmufjq4(Bus$Tt9#KWqR7dGet@w0F$*3Ek(UfAK9T>vjiVj(x zpRE&rcT-Q{j+6zIl3vOV9%S09;PjBG3>Yg?Wf|jVN_2YL5VSBSU?1o2Q9%^xX8Slc zKhtCfd~j0UJ?ocCcrb08>$k6VdDup{0vIyyy^}e8F%6`C;$e2dyZv{xS2w0~Ze52` z#;;~{oj*EWrson3WTsI!9~vW5@hgmdpD(k~ZzMN>Bm7k?JR(4vbAAZ^cp&D#iIuPP zikMYZbNT|yD!My`#ATc5R~$!F$yv#Zz>m5=dgRoXX-r z_apYBHm`2Zj&JvE+_$ZVpK8?TYUln_psoe8Y>fR+f$BdC&+@&~mrr zMAs@Y^t&9Yhm4mE*<$_B0sL`;R)VVa@F6BA`Iml|0fz7aVyASNwYW!o%V3}Q<^Q5@?3#6S>eh!MH4J>If7?t8K&}>KH^qvQ^jTIcRk)2GT z*3%X!3S=ctq}5j$BXhnbl^k_vA6qB1fh9w~xH%gQ3e`fOCN->v%VGC{4HD1$nrD5I z`7itNX@cs4-b-&~8NsxP1HAbQ5~Ca*snkQ=#3cbR-`LZ}>W5 z*!lzs_nt)eDl<5BGKV2;|g4QUISm2 zZZqcS3Gs*aM!zN+)RBP-zl&8_OK%KN&R@#+4}`NB7P<(=UAFVo$P7+c0oVr>rNv!^1%QDz>%2a+bQ{C?nt}zA_t`tY z7DZX}^3=$JTU)Uf$EE|l`fEMswElp`(Ld9CGxB!<5;tDrrD$`SCeVbY%oOu|8tmxg ztUyU~nWxzFWiZNAa*;g7(cRtT{)~FE61ebZQU0Na>#xd&Ms7)Ekq8*kqRc;mS2HU` z9KzN)hhB8Hb)tbbkC496&n|c`#XpCGtQoBmnb|@f^#zY4ugsKDmhO^G%@sXFF&7^3b=x@Z@yIU1)@*=0h z9pjF50kNQM=DlIao!ND-*bVMSMEqMoZ~&)I`fJp!lH1EP;;Be;(<-OT`WsErk^ASJ|2cfHIPq;g8gk^yveH<73h6Q!5&C4wLGu)KdiZbJr2fFcx5>-F?CM zuFOcR^~N>7_NuuF_u9`OiP|(#yL7B3i_;4R|MlBGQm8zF@tys#f#nH9DN`Krs`#6h z%5}U7A<^)oW$kfTmH)^*lc4`3Do<$8hltj+r$FOLbOP8-XT;2a?rSchbksu4q&;_a}YVUfAm#6^ECO>YRtR;u_W9X4qefe~g z-&~GFKdgx7(PspsDx)-3-%}CP-+G=K8rslSlbShIJA+)$r#c_ zAO+@2O67r{fu#pdXyBmFd5uauowMRADfOVpFsWP%2b#Ez^3dkLucAMkA z4=d+5pYiO-+T0OZipxnBVf2Q!eh>R?pFR`gKqv=`Iz-6&UmeHkaM=+5=#g0_6%pVw zkahBKi)AG)iPdX#tJZ8F3IjHE3GW{j3iHY}1l2a11Y8pfVuE`#fTInp z00@Bk3??H%V2?|?Gqep<~(2d7dBCzr?Vk}$wopg^M@?j4`FTV^7Z3bJl9A(%P5 zwv-^GnJH?M=7-Mvwo>s%_n(EWJ9P7FGRpqGC>|y%C~GS{uSUTp^dgj1sc+u%ukfx3zcL^vpP&Gl zuSKEXwb_UTMj*#$7YDt{0Xfu-+ft18(RY6FPcC*_Q*5=fyoFlO*u8u(`0zojYwlzZ z`!nEai;Sx%_0pU=M120u*pd9_26pW7Cv>ao9?b#bq5sw-GpKi_w_8G!g1M(K=6?iT z>xd@Nb9!eFDEx9STForarz4}9Rr=c&02OsKy%N8>DHxfzJH!pm8M_`my3ze?M&{7E zGd@Gr@DSl8^Of6`(ADUOYV?xC4qir1xNLkT-MxXxE>&YZ4UTB*mVja8cBaRA5{8#* zSD`cN!j;4SCSV^8W4lt_uZJRJA4~IPpp}Owa>ynP-636o@Q4en)=pZjCmQLDfAR(A zNMF1Jt{Y*zb;}jcoF1+Z7@bnH0qstENYLO-)t%v8fs7JBdIhIG53ngBPX&2sD4uI` z6H5g-o4Mu8bTYoLVM-=o+Am^*0W0Acr>W4r!U>A)^ewEv7^C{9mJ?+OytMK4wcnL( zabz^5*ydYFwyL=#F;zVLj0 z1<#&tjj<&*;;q8baZCiB-PsqS$KDMn9Cpj3o+oD|i*9qjC6zNFvkE_NtOh+AZL5h= zluVdudXfqb+CP+U-N2juX$0dairXXxU?h7K?kjbYh!C+lBe)#I9q1~ZX@62!QR z)upYOx2vcY7^9o#PK{sHH0Bq@TuIhQC$S01k|F{HXRXg@#M^YSE*@a?0KM;NB`v@x zBoC0!A4M8*(Yv4x89)W!0|ZaL)mdSx(DfNOv~Ff@Um&w-_YxtbHyRV-3T}R`N6uYF za&j~LY4SQ{CsD4Po73dlQchrzXa4O--Jw7Kg84GEvP;K*&ehZY{KnT1P6(8;PEXT7 z9)+Vc0!^2Rjapx(9~TnX^)7(g^y^b=Q*zG!of7R*ZUfCmzoq&e7)HV54%Jc-%qY>! zUdlETaJ*LxC|Lb}%bi5v&y?3#m(A*C@Tej2`BG_D!2V0QM!hGiz&bCmYI&Kc9p-x2 znrD69)-SqA+p6i;G$>tU7eKR19vi&88(NydH9a^PNa~)@28K1&wEO zHs2<)J_!)+Yvy^4H_YQEVmtmN8+di2&Nz&~yTT^-9Rf-42 z5y+Us<@%7&r26=dd0?7aIJuGRMK;dAU+R$*fweANjBE3%=t?z~w-UPjYreuQDCsJE zzmG*(XyvZ$IgdGy6xa?^mI-r(9=#)*w{md7Q`AD^e1yx8FcSAVdeX-$NhJl;+q!ua zK@@sawwq8|fF^wP z#vv)X^)s3_sj5rh&4@-bIhrx(HOeVAP}&gMdK!K%nWyW*jE8!&JDGhm5yS*$@wF%~ zg_o}cuwT}dbA<*AvvY9O?`*%Fv#pCr^SqDabp!r!aOlx>fG*awzgK^BI9X&~giFt9 z1Q-uy8*6^8M)5bH2j%5&&e>Ohw^rE`&-S)Hr0(|~Bk2Cvt#DSY*%@~_?{_);2tkW4 zPQ|>p8-+>`qb5Cq=$vQle?gC4EWr3yFZsoHZEa%n!*4zdt)DBp_E%y0D9unBV13rX zT`3Ya>wKciwJi-}NXJ0*ruKDTC#0TDw8VDjr&Po%59zMuAZyi(uC`sd{%c>b^vnQ>FZ_)8t*}S_lJI(x(PWMV7kI3A5 zZJDxbcm$ROY0l>zkd|Q#-ydJXD91_Kmg;_PLeCkxF?V&Ay*r#v7*U+xO|uEs%LA(^ z6=72v_P9H*Y@oaWbE+z{1ctIDR`Ml${@fad*9~m69A=NK{n9D=RehWdsu8Hk#lAHxCoGO(){AQw{QCRlT28p#?QsYw ztF_w#Fp9P}?RsSPeV>EBU$p`il1|+xrh%7>ah`YJ>P7N-1DzfBkg=$oZi{1XvMaR} z!2S&T7^?RZT7RbZ1uZra%=pW+cAV>6*U^VQtMKr-vIrN?q9`KQYD{U+oOx{;u!C)f z+BeRbx8v--H8!#86 zLm~vEfdH*2jDnolZumP^juoJ*F7z;h zw0PU*vf52s~1YB$JKo^bOY6Eht-NjKQb(iT~dLL zq2H}Dbw}r#?>M_4uo4G@OQdHSul1=T9bAg|$1iG!RapukvjG{g`;wPtJjw==Gc~{e zNUxgNrX#QU^c?L3MzpSLyWC9DbIi(L4A{UPy!h2*JtC(J6d?Jesr4=E`T*@WWzUIJb;@Vd-emtDjm=DT~`2r&d*X zdfW~0kFWt?h=9L-o<4K+Ms|<=!ygt`)Mwz2K_YPn_5Wnu<178;6Zc#9hhAjTu7^<2 zYVY2Fnca9YqzmTTRA`F`)|4~uh+K$0Mx5Y&F?F#BH<+3qX~|~nvd9J&JZ&>|qvpHu z!7C>EG+-6v)3;7^0MjkTB88sD%o{Ht`1WGS#MU&aZ`E_OaY#M#)@0%HfR?z6RSzIQP{ zin#0y1^^$GcztH;Cpnv;-WP-@k2kI4fxh41n~wjDi$8bn+j%7J0;#hV!f`eX63p7; zqtkE*h}4kw8ztcpTSvuvTn+$jyA4s>f(`|?A|tBGb#SLZlDYyt5{P{=I#e4*p47Ng zBQHiol{|&bp}58o%t14|3~$k{>j$i2xJuAfwx058o@_VRMK*ZRJ~quoK2PF1dkrML ztE|=sxKH>7zGV(PZ<{S!0?kQ|S$D6H&T5<=r^JQkr80J*BAQ|7F$%4W0R4#VLo@N}G(<{)zc6vm@B*@xJkG8=A zbog3N>JU)b*0(@u1ZrvRQEj-{qPaHoXl{`^sW6=1AYpoJu{vobEI$aeg0BfAeiH#n z`8F_dY8z4>+U{#y*~L!n6^iXz^q}xU%j74L13Zn)gH-jg5^rtW=5!HO8Zfr*= zkn#O*Rd^62_v~Eq>D4N$wa5`6V1x8ukBi?}<_fy=Yh#qag}j)19i~PB0|maf1go-x zS%?^Ih-S(c!Nx;G_2b^=S%;&hs`H70f>z&Ko6#AMv?&0zY4K5s3gPjNb#cQ}#kaN_ z&F+}vR(SwtvrC4Yj}S;CX+-BW#7*DXhF|w=2Y!+KafiDSy=wG%kn|g+qYfc;kaSBq zlOBzcr)@(U-MLW&Cqpe{CcNM4a}+CnrrMfMz!KOIix8OIbt$y6E##`@GV_wZDE{A@ z9bg@I?)+d90zabH$a6Th+IPcXm@Do|>ov{rO&DoBBO60NpstW)>b3ab(d8wfkg={rdq(4E`QYh^}S!0sswPQ--hplXXEJ! zSc$HMG`tfCWbG;U%~IOdc|2`eq1Iz{La?K))3id1XNe0nL384to9|9K?t#_>gL-_u z>gcN9ur4Hl#MSLzf%T9f*UH!tvc<}EgujUnw{C1;dl6}{{=o+a-B9;yP>21U2~SZF z$W;6dClPc3#_D{+TXfy-j>n715M$go7=|cx3h(>%-j>wvOe-Q^y(;9Lv(|}o1l;Qe5tD}bvX@;u(<6ZI0N#!};U8Nf`zHy16XefrT!5)j25iqw zEIl>}0$00Ze9&{h2T@JZSPQ!OD&D69ougXcfsJx}AvXj{7MBhoV}!dObPFnlZuT3$ zJEY^#x#}o;h^TewP@@lkn1o#KJ9d(w2V0dhlEVcWu>dXKV_N+Wqq{T?G&6e`hVmP0 z9vSMiS`9Q9zeIsc-`*2HBsf9CD_Xq7fqcjoUd>m5cQRK`5dHp{ueR=Dhm!*_q*)M} zTA8EnfdbjR(ut~2zk49lE3Z&_GsIObDPvyWFz&&D9Q4K6QQBE|2mBYn`w99&J|pe% zWy{k-Y4Ft^T_(4zYTsu`S=9$QL5+SA&(2bS*qLgnCkYI(c2E6*-?~V&!)a))$cGRN zJGxjyT|HOB$Uy`Qw7#HgTiH9TYJbJ=zce>rFx^5^k{E&2RY4B_W{W!aRVD28`wmCVm&I=FeOjk_qXsfa`(Z?A5aXM2NPF{&BNl{2}Q0eA!N(Iq;{c zjRUu)?A3B`G`lkYmh|JnLZ~Rt0qv%@BpuJif>?4pZwq5rq-*)~$|z}F7Msj6JR)-0 zgIZuNl7!D0P_yAbu%dzUc zivQE0=;Y)KjvODf6RMrsKOW5<*=1 z4ZZ;<_?xnKguuNc%&!!#6$Lc>LOXo><+|M$jud>rbL1u(d3wE{Jz6D7#IMS*1>s_o zU@1S*C|H1xC=vRar7ym|H_hBW0^r_<2XmO|6JMvZ&wY3orYikUQAwt7EUn|XKLKb; zJOR6{uAE*mB!hPf{)~&y*<0@4`h+%>=RQKLLg>M4!?nhqPUU8i&`ZP5V5G-Qedw!{ zcjLITP|RJ!8LwZ0p4yYtT%A?Thg~6Z;-Z+$cWD_f!68vOxF`EzlU;xi$eKU;Sh#DH zn6BYHbR%FTbVmqA0M$kO0v4W%gO(TG>f(1tz@iG|rmT51PpFKBe5>vq;4FEV8~rET z($cY1Bo7d56*p-sL0zeC;A=q-21*p&d!?k>7`5dg!U9)i{=L)e9P9&HR;FOQ!;!0t z`T!W(k5Yx^N0O5XYxKy#5S}N?z-e{XCs~9J$G+-nibb@2kkg3a^`391LJa`X?#>U- zt%JG`uqrM+Yy=0$qR%B>aZB{zUuhHH(z2F2y^eQYWNB*>$IqI^EcmaNCUh+NMvtH! z0Bek4rRe-8HC2pWt;@RIgx2tW-X$j)^Rb0Y7Qohx7>$(8|B)kZ-QK{es5nWQIii3W zMy5I6)-rPJfkTp;bzNT_fKfspWdI05i!~%Z8!fPf zV%$jtI3yWZ#FC2_G0ZS0hSD&*@|qxT`-PAxPW5G0D894x?>qUv&UPU=OTp1t>+&s? z(E2C<*}%db_6G?{QDCCxlV!~VC$$9sXe<)R>{a~BmGI~&8qirl$TxMliqfwZ_9f3Br(V&tsVip&2!t@pQFUzTYD~$;r@>66 zYP{+ZDmf2;xV^+0%Co+ur=s`%!$Stwr`!UDw0R5S7Y^mFTKr)mZRd zCoHZjQB-ap&~UPeN`-jbu)#B;7~*7?fIT+PYCyUyL$bfUx^Ulg8x?sRy za<<%*yy&SJ>Rvy&alS;r%~R`3e%(KpG7)p7R2nUUtV~)z#bM-Ul$bFR_o} zwdmNbj%1R=Cq}$gJf8a1Qj&d$efZhG)>BotAJ)?&>+|29!u4iPOw7w`*h@)==pv@D z7C_Z|uEn+~hd+xVOI&VwZ!lPS5L_1JWOw%gb3Y6?$r{qbp&f7w{7yB03tH!r-WudJ zkPcj{DajKjhV5L|%pSo1q;C;ssg9le2MFJT?kha0T&9KY`qe6un}L7#Wjv{~ly+py z#p_#JTBXc9m@BBqyB23KOddx`aUf{%(3=GvvN9x+@sWg)6KLS*1rQooYH!pDz6{&K zA}g=j2_%DIOCXEE(!W}HR1=V8Np|VtPSi|FI|{G zg*0sEBtiqcd?bJGX&)(|Nx(Nm_zhAwz>308x2GA{Li{T8oO8V7Ttz^eo&JyENSRpN zjeq(uw%)UW0>!j~D#9;j>0!NIf$>7pk<-{?l`t@?ie93R@orjY*G4zf`WDM}keUaP zK+^$Yz#ayb1PC7#dYRtD+<2_@yQOrN+ zLU290rU_x5UkQZa;Cg71UHRqmI(JUVnC@@iy+vP$`t!4xd~D&7}V=EIgV(VX+yFRlCAeBuOXG z24ueoc6LWRX&!yRKp>bTPY2AAh^O^%_z$MfIr$9?#2(>Q^nt=PZdmcrd1U2>XonED z|MAd#fF2S^JdjgwSR@UEB)Tu!9RHCiA+YXlYG zf?!_n;Ek5RjP;RNLH7ld0kqPnoQfCPCvYnTqUUd88gKj*FdGhA7HMBJ@S|ocLtTaQ z!KkblN8*~X$J0~T1jx8$#Ro^_kZNO+bw}No=p$`HzfZM7yT}o#p05k%%sxkrve>TX>n8ZrWPX(w9H#Qlzb#}r0PFpu-Kw`h{ zOzKvKVKf8BN9w`>pe)vpoeT!D`4o$qRw1Lyys+xT)>Ebu+hWXHzvaJ0GY>pslNCVm(*2$|M)u(lg)L_~ZIdDmh>W$dX^}DD(*bM6%Xvxs z7*Ze+oxZttiUN#|fR`UQVMwNLAJDLbY^0WCYj;w`_1}C5=UM=?D+(%KZa)iiPkV2- z99j4NZ0d!5#V+|oskg`$*0F{zKy*x`BaB1=wL#p$XZ3KHP*9$;V#vg6pDg-9xPSvQ zk;y~tw$)nJf25i2|T0TWNe5a8khQ4`TO3{K2QhAV)|hXm?&8BmaFC zM$4OCw5*Zl6nvYi{$Oi;ohnwsWFZXV%+c}MllN~|L}SfR>-;@|BCFRMSO+%ZMyl&A z?4XOO1r+21(bjR6$kG=y^?Jnk92X=yOomnf^^|D*7^5{_q$<1}Y4zk$V$~GZSVR}- zqj$fQ10pzNE)Wm`0ECKmSsA_OZbD~=Y#%abYvtK}y9jPdRy{fu^q9~$5aGBY}xV=~a(>HQ55Mqo7TG~q=2 z8!XG~eeAU!L6O?Cp9b!8cP_yAxJZ|5YO?cUCnW>nbsl4n4!XU!YfmIO*Dg=PvUmKs zh)vEe!APTVDc5ab@sm?nAcWx47+u2ztNo9XvF#M{qxq!eLCWTAGO)x*=WEWiQ1iEq zAAKLLTK`_YD)RBWcPGjf>gwrP72PoGlzFf8db538VuK4iIhjt49A?0iA!}k-ge5$M zS!G3%J^ZnxmT#(MOE}HwytP*{m+AeN$$#9vNqFhQxtO8tO|O+1!-)Ex_RDaB{Nr=M zmsb|C>dTF^mZnCF5uMX9v>qM;ml?VH9}gBxISx(y?YS^f7&rLB^JMNH#_~0qHtxnp zkzCi=rdO^Z&ZmdZnv?m(c0Q&vC?=47Q_DVfJYJibn7&~E0t4D<| z!Jh>}YIW#|(3+T4@$ABtn#XJ8iH8EUBU9qaXw|qBq@e z)_pv5nZ&ED$Wf<}_UN4{0r+KnQ(WZSezHY52cIg9>Q6@g`B{%>#QLQ^iWHRKqr{nQC~>(RFgR`y#)y6TBx?L~tz6(Erj`mM}6u`dG3hpev5Z+oK|m zV7?db?43L2aW`uTJG#Y{DPSuqJblheSvOmzT0dy}XV{CG^|FqgFC)Jvo&}Kt_h_GM^9G#6VPLygI_~!b<@8A;hqBCN0~M(Ku^>; z3|8>^Bxsog;tbw9Ao*P3*e<`gx`p=d8Ma0Fo;VfB2)FDE&xJ{ykSbJ>X0t6g45!wV zL!%Lcs^HnNn^F~Vwx&%8izC}45+n1c^Og+YfCSw6ryhFqN4YO9-VvNvaT@CgsYUFt zb3KHUMT{?8VqMQJ9ev3&LfL=l^lo{0b}OH2<;<+-Pw_njlOc{z z>9VCBid;$r>_=$6zWg?2Rqw&d*dW){8!2~aGZna&iw9})eqbW^Hi`{TW05%8G_iwfJX9(@mNdMl%i4mV4t4#6Q7RO?; zIL=*JJ_S2UACkoJjonIn&T^Z{;mT@fe4KyJv0}m>L^m}QKXB$8FFua0Xj{6Ji#~Im z`g3zI=R0%4_pT!7=k47J1%qPhqMBeYcdtyiAmp^tt-x&P%d4r7=0r&)t5RfA-ThH` zG9T@?yY20?BQB`OFL0~86D>0M?3C+21Lf{N2M>P4Znj866BdqUJ_<1S=drFtoE?4} z7lBxMvFsr-yKXo{@$2bGM3S(Ex&$)IV94T?x1Sk4TPmzHX+yaej2U|@3|*^=6h9c6 z7hvI5qxR&&9V@h`pgE(C_nXewK7O)q8e*A_d?<8+Grg+>PyWP~?6q9L3d8R$Yu%-z-$6GP4%k;nV$ zj|>nQFou5Op;}UxNT=oZ18{|$XDoK|Ql*o1ttwrul_y5u#?}q zik1^k-&K7Z_C@u6zvj!B_*kf`wWxvisMwT1TGg)sF_0frP?b1xleMo2@$dVc_%lP3 z82GBpJ9(5fbPV1mY}>BOP`Hj@1uryH9&n_nDQmFPM>TvlaUKovRxKNr7Tbw$E!k9A z?|U)Ar5%Do+4dDpIE?2XWy(G+X?2r3 z4pl#bpdKt@_dWNz;lp&YBqlvPjNrtqI2;a<@7;0K8t1bnI?ZNj_UN|~ih7Th3OD~K zJ?$#wy4`KIYG&iUj=9O&b-FBI@M{0etqvSq_q$^sZ%5#-`Ekm6M8qaA>`}r0d)@cJ zY&0$9-qXKXEaDIJ6#tK@FAaop@89ovI@NK|6sL%c7E8$zjYNiWlP!!Q*|TNr*}{ya zMaB{phV18-WsD+w#&%i|lEK)wB8+7SW8ePYp`PFWjTc_bcl~VFb$z~fZX1;8z*oub z`L)zuub7}K+`1k>FuVNXv;1u-=fjMK8im_(sY^HO1K+P%4KL-Jj?CM6Z|f6BOCm;< z1V}n|+S77GC$1%ogOUKlpfp9ncY*psmp|{JVjDaOQ;arsa)_3BQRRO0)-w5PGNK{m zKx40=&$4vbN{7qb{k+TuW7p}Q@GVBiN~D*5S^4owSaPfAuTGqTVRXnw36abp^G zKD;=}jJMZ>($dIR~q?CJUA*#qStuLeB+rU&YQ+bh= ziN!Z2Pr*KncU|N51H#g1=CoN~*T44T%b)rci^U^GPC;6vB{{XJ6|n`hK@{93uIx*! zIk7vTKM-hIn7n`BN=fojhabqb_>T;5tB04V8eCP%CVB|Yjq6Vbd2G&@sv1+CzWSe^ z8JLB)&ENgIr2G|Jp&H9CTR}&hV*Ag!UL|fBCi5mbBMj@Q`He<>p7t#ihBbn?mCbNl z8?S8w$xFuEi32KQtcx6$Rj{uPjZ#cet>xAF^+fBRLO9YUeve!KwhPNKGe56dG2+qK zqj);|a$Xt5&*H_+LI+c%_Xt@ik#-b(6H7a^0B1ZS6 zxE=3rJP>qL;vU_)yltey|IM8XJWVB?ERUzXd2FykHjCm%|M=n{-ZJ>WM*(sYVX#8J z(Xl0&mru8v5Rx53;{D&3E0U3Jg4mVATv_r}QQyvR?bGr_%%E#$T$o_!EbTXlL`&4oA(+Svb!FVv? zrix+BiMpBliD9q|aJhGlB{g?}pc5EyH?_yIWSZu2md(`}Lan2IRqka42HPsv(`VA0 ze+WsKss=$zNR>+>@%w7jw}Oe&Tu z6L@i&x_#2>^*b{6Kn{Zcl?akMMf#7g8QzR)(rYPCA}Bd2J2hP}Fc)9ew2W+aqx#lN zj=_kK%eTvp8LHG;at_FONH8(dA(FzsUV8t79Zidqzpcznn%17LObnQaeEjCI+)NS! zZjNa4UVCw&%=s|qo;dE9xTeX#)`V>x#@zwrn%6&_qo)5T03S#=opMnT+#f?k$!KbZ zbdsNxQ7-NbyX;Grknygyzwb0UXm%r- z4zHbjoZrQyK4D@?LkrbV&jk-&4*}8B*liUg4cir73G31*52vrGGTE+3+?u&*>UVpW zY#SWCr@XZ0&$N^wZqEg3q4=%4cjGV#Vj+nW@OLcN5gU@?M#!%)xnFQV1vT9<>HgN0 zPTU*+V^wD zZeQa0zA!v$jRf)0DNRRPA6Pj^CtPG%KZy_+I6*?oA<&n>m*&z+Qc9PP{P{$r7F-CB z1&KfXA`V+s@j4a~LSAWtud>enj5xC69Z&Gm;hZmHh5sohG-@(aPBYkQ)Mj8UF5;Fb z8g*eDq=)@#C{LwxymfF=LSnfesRX#UZL`UipM!6tJDMTED6j7IQ;NJZ7*I+LHtTm@ z+aHH9W>E7DZMN_w(q5WgD{m$RjMa+#4)2WjK$b~0xJcM?3|hz!Mw+y5hOHuTL)Azv zuEml5%EdFFJn_F@PO^{%h~vJ~a(BNma9_)FLTAoS!AkQidodAS6$K$Wv0R7K@NPD*sUf*s}9) zFYY!!-m@emHGR{X-adAEspij7V{kMkMNf8z0bhQuqt+hSpZQ(Rq1Wz6i@B+yI-kXE zy>aIln)U&;G=%fBjOm9)g>dfQN^_9CmDDh>qUL0<7~2Bcv1_UMP8Sut3i@lhX5V_U z+gmn+3zxY)^(o?KZ4LrtKJSV`e~N< zEK(~*fbo@vCJvoy95?(|Nt3rO3C(oy36rvTxIVH8oUan^%pEJ+yKKbFK}eX)tHq@w zB@gy$$84@l(&3{Q3Dti`g0mKQ`7Uh<%$Kw7XzSLbzBTrnGQ?XKx9U~Ako4$T@;jDK z*A1%tZ+xyHI@a(-0l2P42*X+o4of*B{rmH_H|ix+V^W@ES6KFT0l(U8w^!5ou7d^e z8RJY$-JJ+}_mxmyzRDKP5cj-rkZ^3KkJ61~`57RgnDE)0x8I)LNPU?^h_fp|%3OFX z<~#v&fjs$dfxrZcvw4GJ0l1#A@w@hy@d)V{7;aK$*^k#}#SYmTc~l#e8>l3Ekco-L zo-YjraKs^K??Tx7^O7we!`|0-C8l~OZ<6nRl-nn0(tRA1vfzU;R?1Qql1o(4$OFcE zj+v@_#7WcQkQ9xC&Al79yqhV}e)GS+*Zm-p4v&rJ6HHhGYNjV?fhO3Vgf{4vCW_f$ zhdUEQIlNCZp!(9{!){2>=jdFbtuq@@z6)ZdVcs9s3RiJx zFR`%swVmJG!|&eEADjePcDJ8tky^I!XFMP_!#?#=YIu4}@(e80ddK;&a99XfZ$kW& zC!bc*ugw)^Tw1PQHw0A{NYR^E(;ibr5J><%VAn`54gW+*ZAla#Id+*BA&Xe8S&&^ox)_Oqd_u-6im8gnr+JYcrW?v(^7FsmY#3Z_rU$1j&C0 zGUS3oTaW6mbI2|fUh=t8fJvY!2l^3jBmky70?pJMxf*8{;t2!1%K&MyWT0w83Pg<7 z?gnt(SA{^J;$c|=sW*HMr(ra3Rnn`gCg)=@F~=D&7t5-$tVawuIws)COFOgvcOMhOs6-ANCkwSY z=Yl(l#O2RtyDnjx?6u^FQtP-!j`cy!UM|r~HU0XC`oF?T6x+PZ;}YjC zyKQ(aKS|SFrVdTJ6;{d{B;xmG13OEQJJ%jfTGja{&hlvAvlDUWY_wyB_j!>B;S1CB zq&ELs@0gksz4h_Bij!|}=V?d((pj@k!KmnXz#vIDgGL#@;3dxhgd(=1QR7w_Cp1bVWU_h0 zrU)7TI?$}lQ!LuTDKj`rzq!LQr>;CLGZJp|e^>F$=VWR#QG0k1V{v?E@y{)s4-@a2 zTjhiDPLh(mW^)ZsUvQX!h4gVGp6(kecD=}6mx#H{Cs)5)U2_|$i|4>Akh|u z>{Sk&t-s9w11TojN|Ph}g=Vf6GciutoANJG2g^h>G--9bVnvw*9OEOQp_r@DL#ruR zB?b}Fi(I^=cFJL*x8<*Re~{iHQFPetx{j+YQm1*UP4SCg$~LVyz6ouR9uBvCIUQEl zd!yVo;yBd7I#0iVo%KFYw_OqT>^icvqS~WhEQEDI2cpVXtifp_+16OOOm&Rk% zfLn`*LpFy0)qAn6VCw+n!n#iHT)V)<(u-=*a`z;0K_zH| z`$t=64m)|jW!DVI1%Ves$_fU7JLKNr=!9nKZy*%}pwnmr+2x|`Go0IMiI_#ArBnTr zv~zLPoE|l<$j~}!4w4K)?XO_0C>(>QS5KZ(?(g|WsMSwY!MP8wg0PDYy17xIMk{xL zPZb2WLC4bSFt=iB@=EYQB!-pGYRpZ~=xI2=eijZ2?}BCpsDfMChA-lH`J$FEhQJeQ zmEMvdUb~)djreV{$k+r;uj#L^c$QJ&=;Cyn57o%T7*^`HXcIqYgYbrf#SGB25~0RP z;gG`g+Zu%#u1H61gyGmswQ^|8!|kfXJfzrRZIYs=kNMXhHJ6>=m>kU~%=4VvwN2xM z9%v!4KJe{GWj~a;3f(VCSr%EihA4Ty_tI?2lH3P%O8^RN97;41RPq1@)h;*$&h)wVOAw0F5$HiLaf4H{R^#+rDP|m#9Mm*tPdj3n0OE zLKYPfEKL0J=S4x1`Jv)l1adNv&G3w{!Lgu#`SYJJM-4;7Gr%GMT*FyA3;WO(5>aK# zs&%v51&MXPmU92D8P^eOk}hI&_U6ty{#}=GiNN~Re_BqLHb~`Jz8E#&XKRnBTG1~i zYzZ*{;y78~K7rIrcX$fRNQY1sAmchP)DiXu$Ub0?xq3E1U^(Du)LC8=V z)sCZgl1Jz8^c!0;yGgA7N$|g98^%zdyM<=@Iyt`^&t*u5l{N7Qouv%P@qgO`6$XmX zD2A|sNF0DcN9uVSDb^=*KG!Y;HubDWs?Qj?R9L><*eY1oVN!=o>9Dl47lkm`yGXty z_iCZx>yXCCixD+`$b1IoDhL*fIT>^_v<_p4NYPB+(AnM1l~BuQ?4+ptdfjY*559_3 zzyNb^X+W-Vtp?u>8v;xyEWWLA^p=W@9HN5(=Nr>uIU79bl3Z-h zTsVsRvV}WCSPRd=sg4IF*oeLBPi$Q{!H3N%BSu5S&Bcw%SVgjE zfwaL4+-5c;*^*ykGm`t`33zK_0=|sfyHWO-A$zXODF3;7AGF%Xy>pm9bFO!vQn)|f z%3dKAiY{K96gYsR6POs?jO@>+=UN{NqXxMbrt1$v2s`hB@Ed+pF}_zo%3{cl_WGcT z5ExoIXIEEB9d4IXFR z8u|L(luJpI?~#f9Y=*jF_;0GX-g6mpWl#8&A-BM3*45-?KTDJwh!eDDjKOSpx)(=4 z?WGV$9KC7&={0-X2t}!|3ze&C4zc62wJd*Pf1m)g(HJ{Wp;hf)ORE4|x?Hv{rBqf0 z?vQ=bHGwHudC7$3ux9?o@j)5&+NAdj%wGG05Qkkt!SfsbuLJ^Qa{3i%8)KLOqyk?J zp;6i(`8_1Xc^qe)nAi4MV|$@D%TmkJ+s4*R$pG*#_0*n++Ub{HNuw+fqxsr93q7&t zQI%S8<>p#hSqE_7iCqU#@?I+56vEa^WaC+;0oKOS$HMkF`dwv>1Z|}UH+g|@GNiEW zwsuZst%K!rFh1>%r>b_7PI7`MF|Jp#0wwur;EJGPlnz&w1s^rN{h%l2H;;5f?)O z-=a^Ty~fK8FFV}=xTSp9ypQnC91d~3pMlx+yaVy3Rne-;w>TAsprGj-q~q2PG5cDy zN41HeO3DnZBnIzq>A9RH&mkRKTW3=$M72K25>B zI}mb~+?43BSKhk!YF(!NMHeMBdX>Kj*SVpQIFqvqqm*z>_dD{q@Fd3wtk^UEnU6v7 z*>?xGk7nasZOLO>reD5iYpj%cqzAjjUmvT0RfO~0Zf0N9={I$VxJQ};l8e6HX(W~sq@(oem#A(xpphw}c zyuIct9`+OY8OJU$qfwChmSQoZ!)u9J;*Q6b<)WGL>}aO7Gs5NeQh&ilPlcnA6P`X7 z6Lh(TdD)snr<9NyF$9=&avyZ=?l0=}f%c;^HKy0d=OE`dF?bRxf2M{`mp3#-RQ1J- z0D>$Tx%@f7aR8ZU&Bb}0EpoYjqI6yF{hDY9Znf+9$TG4q$e)kIw^4HGbd10SsewhO zvUN!f&)nG$8cUbi+>l_XQQky<#HRZOa_yrpi?CQjqxe`fd%LtjHlgklRahYjn(Ey2 zwOkCxau_aC)2D~z#5Zc%=$|lENIq^zXPN5D)0Fw^{lu<6oFnLXg6X!6Q%bZdqRZTq znLIgAx`>=*u`TA03d#t4^_Uvb)|m;-t^)@lHoK03f1`2dR$FxUvfzP8{K!!YO@1)5 zB6)*+LLxv&9JRBMoirb)RPPGSlEpyhYTKS zrdOOhuob{sx6!!wUEv^ z73<6bvc#nzhCl(~exyB~F`@#<@VT*w^6WrHD&_NMv;k4o*m>8p+Uz?K7FQa%-4~}& zPS9Ft9K6Q!HK&o4q_%Bm5N?<{zZf=ji4g;c#23Zmmo8R?)gH<~HsW-(7(}^G$ljn= z)C#nhLde&7TSO!fT?Df#r(3)IdY^6oLx<0Oy}t>5TR05)x?=1uf*3=J$Dh%p`Ka58 zX`*_#v=Gh3A0OJZ7u`mPmwyqGqc%QFIF>N|^IH-r94=CfQ(aDNi59g$*k$gPfWFFM ze91@ot2XIDE_p;2(+5(C@_y(vgczMJ3mBHwaT@A#arIvoD^QEIx!`;I$ZHCRP#nh3 zbuf(yaMe##V^a!xo=;5Lpo~yhx3dR(Y^w)PC7R+F!A}}S(;smQ%p~7jG(zb3ZarTZ zw6N(qsvjd=d|67S8cz%!4^l&_E4%O>Z1LL6vbl}_4^TjO8x74(_hpB}^BJ4_`n#Un zQ6b|yyFYzu+x76~$U~7Gu-j|oPZoLn;KE=ID64AFIb4$?NCk71%!$GrLP*1n<4J?- zcJ*)n`7>8@^`*Qc1IWo7Au`@% zdcJ#LyS#iU2-N<9j7zh@SI_YY08DPkdwF>&@>k+1?wo$z`$%YNXy@lMSvKpzx=M#&50>UT6inNPy&{7P}^;PNEzuHwr=>(N#_dxcR}y_zz* zDY;C=WDvH_gZk>{o4J`*Ne8|IlTP?^lzLpou}XJhQP0IT2CONn-!4r1AW5SM8>)EA zJq#D7tpvX6R05;6?~I!K9$YLQg3(Im4u+*3H2jc#YBa11GTu#z_Gr%OBQ|GBDM3A6 zx)X4G{S0LYvX3SO;jrO=0_j{SPc5|Jx$h=!8f&QNW8P3n_dnCT*7hvRe(PZ`=Nm{O z`BUkWiyI>3oU*{_dE+^nUR z$$2X*7sc}O&n=&-sI zUr@93>$dCv^a?8Y7Bxz_#evH?@(I!CYUT#)OJ#TXxQQjqtdRa{arbKVnNHaLjag)lIK7IIDo5tNi^B zVS{Rgyg3MXjk+H<#ZSgLg^;GViH>#9rV=h29KA#Z9(#WzWV&vV!ZHoPxm~up*oKV+uirfxONg8=1 z_DT+%6>7EB28UH#zY_S5^)k(wJtgo1n-;dx=*r>)bLYdElBMN>qlTc3W3mN$DVzYmoBPichv?duy1%{Z!U3~A;_ zK+H?JC%52vD^{*y1O#U}mD99SQD(~p&jA;kp1y4~j5_${=Kww<9CmXph6!V4{bwS( zAw$hr;x3SL=d_e)>nunOaU@jvcwW{h$D{#5V!;U!>GMc`%Sd)W*eP7;CDI!)V7f2| z?rPo(NruhAA--AURKfjmOw?tYfgE#LxgIXSAoku+0kFy4OrHLqyH)5mjm&Os2T?EUXibJ&ImxJqGl|`jpzWxL}}PR z&_m6bn*Xduk&zVG>#Eh%+;`KFNl~MK|lUx7z3e!#; zI45k7U|@0o#$m$SrrY}BBOLlY+u_~Ny6fL-a|8Od)Ed|6w;Pz&{S?61^dcJ*cLUbF zM{;GMWNZ5Dl3dnQ|AZF{uY4%Kx03R$icZ4QTSLq*#$lqQoQkDs{s_*#Sk;s>Fk7x& zVLW@Mg;UqOldkCS?fb_f!fTV+43ePCW99AN>zCC)Jl9;h8jJ>MT{8Qtc37Rj1!DJRqg5qGTIWOonEzLkK&JYp0SiwKeW#Ld7pAH(+*y+}%~ zLT=FTU-rZ*sElwwX(k}9EL6l$g?sa`mlo-XM0%Yv_+xuIt#FRh8ZN9vYafAHEZ3=B zXci;oFPw_P&`7BBf@RO=Hr0%ut3iKc&Tl;wsszn;WL zl7ky+l$KKL z-GB-8)`XUVkwb=T#B9-__qk)e#H)tI!l>_F2~n_BX=q#6MnjzsVYJb4S=G6mh9PYDKj3cTsJu_OT-~~FI7sexf zGw2T=Ez$>PHXWsqMg<$dE5ssyxGz;-rCmSxAHVnbuz8r#>G_z%@j~J924kp(~ z-d)Gf&|aCC`68gWkB$} z*^k44UpM56T$Ths9w)kWRNounZSUutZ(M#_;qt4PlZ1>aQ`L%?WBl8H_i{O|vkS*j z^EPq{eqQ?H#HHr2<VQV7*qy@f=4;X*qEuWA>2i|?zFEroZ84 zp-JA!#r>C@JEFi;$-$AH2x-kdY(I`mqQ5+TVi0D)S8*w$yb5k^o7HPo1k+N0HHh!} zq&>6RO-J<+OH}IZo5h5{p)Pc>4avL%0v(|p0qmWB2mI3qRlOA%Wg*jI&3V1wMD?AF zni-|V5(bj+(h{a4K&4_DZ`+KpjyUtC{y?&CQs~R$B7<-^90@f&X8tAUFhF^;@zPAC z7AYp8-?_WL4=mp$t47+8#NVJ%iKycOyt{+P6@0P_#k+ zsm6-k==<$5$XQyJs10qi5t_7T_FfUDdQvS(nL@3bvix2x*RI)sd;=JGSEmq~b|S2C zySRX+hg)JrdYbqA?U`cNo9o=&*pv0!1H~Q^1HL=>ZZT6( zO}&QjRj76T^Gm;f-xz9_-I9o@|JLMH7CJ*V7g&MX#&{D+!Hw-H^lO>>+gSY#t~Z8s zsqV4I6-HPjtoPs7aoAp>`4;gKF(61|B^tMCeyOc=Q)8->PG?fVCd|${KL^Sv@!hlD zljd9@y@NK8Z^y!fymPCVK4`HlG_I*ggYWTyr*(iHcT5J%B&dW| z4}+YS?53wh0r*Z@Dy{6M=kas{-2wui+u=xE$(pDO_5&V=RujpDi)QAw^R>)x{D;F} zfTeFO;J1qNbe_V@T{vKmrKdtz;w)^{rMDC^xdrI=`b2*p$+E&+4Hh#xU{MaZBl8&2 z&UEOopzu4}JSxN)4TSuBLi$z6L}N`<4u_ZwkPk}JR1gf0Z<7^qX;Zr=LC5<0|kyp(7Ir^tAAed#dQyxXtW--0TQ zs2X37=cAaa?oCt0i_2nOG?+dSS$~^eT#p%b@T$`VCskjN;*Ln!N_+?E*hIFexRK9Y z#nPxAAPT<1MYJKgvg-SJV(&MpZo50v+VyqFBMiTyci zi67!d$VkWMszu@BU?=&bCty>tMLhPAP@)wY_1Z|xWrBD&5F_>s&|(Z4P^mr3JJ-`X9`xFUzy;aku+ChD0TH}Vkf1=^?xlT@FCVKK z6A4?zJ!Fzzlgq+RHZMPrpnbU9Jb%L z)#J)OsAn|ZaO9J9>869A%MiGwn-b~n)}-rC;h@|6$8PO5mp#Y_^j40Leh^S}>1c2G zN|tSTluSmKZ+0Cpu=HHn(1i(;Ln;7nV_mp}GG_`wWbhPUlu)bPSXpQz(CC$gW_|Wu z&n^wt)!9g;zXw`d2D$tPzo<~<6KtiFNZ5Y-0GP{zDsb%nTo6ADRXact8lHImQ0v6`YH5y&6 z8y2rRLZtczT(Dr%!E+GpfLhs79Bcwge`gasT0L&>gN$dJdfi>6v$ukumCeT1y(1+0 zpOFsP3e<@}z24O??5Qb|3p0rTc8jBT*SnreVIyJoz)E%sg>fT$lrwt zB8wIPyd7&4-fyXg@O}vFHXbw3=x_+ST-6GFJS(LCArktsO#ONN|HgmJkf!;fv*%yd zU%r2>?k)I<2P`ezE-dlq{)%YQXfteOgr^6EY0|RCJ%Eh@xi7w6n0a;IV%4#=)XgPys}DyiJM ziACq{t-4;*iYM!-D?ldr5s+82ba>ZQKD?vNrIX3p^ih5Wygj-MsF0p4NQ>pAw3Z7g ztMO5v(Oy1_y>ktwC~YPHNID zoAi7lKt90N4qZ?RiqRRcPp>}V{# z^LbDU{XR#D;OoA?;SbfoCCpW4ZLeMF(#W24q|AdztDD|XeU`v~AFybY?CzDRXO{ii z2=A+f;3Ypj(gAJK2NN#g{!3bA1d$|FZ#=e6M1NkGKwJlPF4k$j{I;<6sj|}Ks4A8T^uy5Rru^)hL!ydH1kWQ8IE*$ zS`4;`g1>WgpPH;M0n^Wl1tEk3O*5aEgxQ8txXmjJqz%9xNVMD(ha2Bs+}-Q{o*)V( z;y5@fJUZoQKOa(|?Q;MQ0w4qc^{Vcl?NRSGDN&h*b&U9;dj-XGDv4JRIxi%IX+A*Q zIC*$3hNMr;Agl$Hw?U6Tl>_^N3D>N#=}5H4lHApy9N$fw!vYS zx1OR;CYx8NO~ctqo-u1n@Ag4d||&)??R!YnQ0qkNXAfw2=aH`%7fNyM8dCJ{}69yG|3GRHq{{i z039G_ozXzk7-<(FIur6G2yfB!BGT>x+RL=#FYa~Cc#xL>BJElTGEt^Um}a@m!O)Na z{Ix#xu7C=CRN)#unjUDDcl_4F?430* z`%yG78&5ajs=~Xw*QFksNqd)0c|tE>$6pV0Yq{+v6!I4uRkKK%D??ej6!$~<?Ig zsi65GPA#nZd`rT;kw+dW{Etm^fMO`*V(r&*{Mds77_Z`^UCl zhdfLKiIxqXUg$fkgr-S?oZV=P<~y$UU6svy%`@cQX#h8}#gK$75Y1o2v_AcK>!G8$ z+(dSFL2GYM8Z4*Y^xyzJHRcs&c-i9riE9s?2iM|aSnoS|gRy>NYJ(}H!U~vQ6#4!0 zvYmnUc>uVND6(PX-bFcUH_=`Nn#Vh<`jEnXK*INxNiBS0ntO|5sM_^sQJ%jH(EMr} zY8ouR83K-}Td46pHnM~Z;MR^9mCHA(^Q6Al34qZeHH(L@MJc{lN}qUn{ML*HnP7rP zk5tKeZ~*dOi_3d^Un@{zymr2u^6QAg8iXA22u|<`VmXg-%*n99suYLpM-|r;io)+Y z;YrU`m3ON511)aKf2SC<*JRx|$M!PY$lkm)ph)WVRFUZ5GW6* zddujJYZNvWiw{S81r3A%G>+o7wgfo1MNHIH_8U_01Oo;hw~m)Di$MA8f#5u+$=t6A zdXA+>-`PsVIWb0$)YvI{6*>N#o|xd`lM2gPwmYnS<`+qOg2W}FFN(u<7jW0aj1N7V zfT50K-eSH+-kpi*J;tps@buEgBb(etOP&750TC#qpGk0iV}_Ip8GOgDFpyj&e?ujn zgrnzqI>Lsk-rA2B_yHLz80cc58(BaVr$!8@SwQWW_Mw`cs8O=AZ zFDG4d(p#REA3)1e&@^_b2!LU73EX1eIln7Ec|}<{2(?ztN2?|mBx7CGU*r+VLmpl_ ztL&&T@5F7HkJEy*cK=wW1QlNK8i$5w<&ls! z$q7x{bb)n)zS{a$)EJAh(8%bb>` z!>Q{n-@19%rjER@YW`&b(XJc;b}j5mQ*ivV!LzG*(W(&Vto?^$-?paqPHB(j;^-4u z$?q(c-7^<^GsE-#>tIlJh;eL6cFXuO*=UF4tCyNNR*pVAi*gVOY2>a@i#;uQHYLm& z+YK5x+XZ=2@0)0mJku^5>vN6$1V9KKE}(NQ6>a?oIdM_)B<_qsbR^8i6FqrOZ+Yg3 zHMSo)+0&&i{<}PSaHoBV%HvZrVFs-9-iM1QoOBzcomCdvxh4EFI|$@^NlkG(MKbZO zo_Ab>n~}?5oo*;84+UD|5UE3Cv(rY)$0vY}sSR|e*xvIt>o5{ARjII!Aam{cE00F?9GRM0)Z_3@0RHv*>|pymaInv6l*qhmxuY+vdVXyikqkb- z*0m0{O^%Yy>WkvsIQH%1xKEy>ZxGAf)7PT+0krYL5Prl}qXWteivC$~&DC;NpvHeR z=H!THhG}jb$f1v8HKmV1AHmgr8|)~T(7O`cZlRL0p4>q4zv*tymY4+;oud-zGM392 z?-rdgC30!LAnuH9=`zOQD|3ce|BkRWya+mkf4AU#3qvyreO(H@&_~^5vLOx#wsi-q z1irP`WHRh;@l=*kz6G2N`#Qrnfm&q@*q}k#)O2hD%VrJ~aUP+DKNlCd`kN(=Oq&BFgEZ`L{l-5_;zLaLSt=T%C7LG=t?n{%#11}R;TAeI{ zYtC&~4^I*|nprpS{Lrr6+~Xc=R(zmZ0kg_OPQK@w{5F35p9P)kY_ikvHAH$Bv^P$b z`!0|MId1$AFjxFUeQGBWQfGRxYlg#!Bh>)Z=g7$lBTayJTh0BRPCvBm(gdD>sD3*o zF(KHyRWyx0tKhd4;2wd+zCH1LJdofT6=0^Q*9Ud|-K^C1w&}-X)InMcliIcl$e64f zD(rnx!1uplMU~OFTH&s2I=s-wIXu=rU30XDFP8eX=&?hP^w~=y6S34wXd3O+#3IH? zv-pFyCe43;2^G?j?0O);&ctN*bU^AVUAg&VX;qa*d<6tD;NL9I6a-(pd+fLWT`5L0 ziD;?MDYQXB+XV?#sW;^}=licC7`;57J!>wD8RY|KoX^TJ9M+95Aa1l8AKH06fx0RX zm)0^Dfd`Vx$geU366wb=4Q+9l7*G^zrl(}pe^_ApvUv|fBG31Uy0UdJsY1vLUytgu zgL&@Nq^ax&g=-jd86zcln28zB7=-eNss=8Z2xse+v!>Y(ZEmULSd$cA@2Z?1{Cjgi z(V((~RHgj+E3&slu6HL#A5Q<4vVv}X7X;9L6@;PNVOc*OeSSPWt}(NXF5Iet7{wG` zF;&q4>QXno!eEmrIJN3Z-YY?w|D3zTrUgRq8^w15Xs=1Q%}DL$Sr`=9S%tUu`eP13 zDR)-K;KC^%x3VZpMZ8}cN^6t0ME$qY45XT4YQIfl{~;Utc3IB#X?RIQ|1zGA0nPoV z<6AgyaiE8cie$FU%L!tdUiPDW04c>#?yPjan}bpuz8=A2?Tp}2`;wx&C9a6k6NTcn z$le5h4jr+uHEUB6Bq;O#%%xG03s|okt5<>jZV4%T*WfBKQ!aTn@*c@|P?c*mUQH#= z25T`?O`ndp0co=#8l@o>4+5CR?@CUgV~iTq=K&lV0PaRiDQ?VuL>+~KxxhNyZ^(XegR9Wp3W@_2QrMedYXvbA^s0dzcGOb6swM8N%|0`}mm34%D4ovg}i=N(T1yJ;g5;ytjw*UqU_OO2)JB+Z5|L89(wW<4Jg&N)nZ|( zu8kLGV8C@4L}SGOtavVsL%^}Sw8ZgjeUv$9pSH@?q<#AjrxvN+2Ky4J@MZ&-2cR^@ z``XKqA^;)|Qr!TGU9OVMhmL|e-t78K?6TZZ2dzm}(G1vreJL$vkej7+%1@TZ`?)=^ zlxFFKcY%mMl7Jtc8aoAy-{wdOcM6Rm8b&nwBYm+twWggDJoyd!&-eceeg-BD? zk8{yjroEv_RXoYO;wmTd<^-Jcx{X8r%8Iv5JWHnARi>;L1-s+04c}UGeM0`by%r(t z*-c5 zM?fk+hR^)6#KAXfRW$^CR04Wg$MgB5Dd7%!V1k(nIAAcj!K8tNg|%Gk|7CXn76UCH zp;{vy*3O-c^|#L=+-EG}Ib4v=W~p-8699jFGb7fyIlQ+d=@UrSL)xBzTx9}Q0*M9S zX6`u}H3KPG$ud+o4&=2H@KHfA6BVNL4E+6)CW@%69j-}(x%g&Z=vcINv7aOG^6|3a zv4Pux4o?ANS6aEHgj)xtZz50JbJg9QdZbxK2-$qDdip=0H;z@{51+EEr-HKHDPjyr zjx^In`IhugYSP5%nV*b zg@jDN?Xlpl^$|N0-Vz+ETN{LcK9ciPmZvc7+k2hCLE7na1TH>~{b;}!xU*7z;^Lhq zkNMgupkV%A>*1b9Viq6)Nmg~C!;oP+OYSe)QuTd^SK)ACze%;V@!;jQGiWdTsF-2D zfzI|P;-WA+YsYf3j?Xz!)GIoPU$|q=h~}cT`5^(V^!JkR#lX+D^DokS*8!U_5>%0b zMv05)>mWKhzZAz(Gmy=vn?iW`_JH>!mZ=*L_r3^ghiWC#b;X5hDscb``rVZ;_82L) z;LOE2K(D^JAs`*LRz^TNbr*@xx>+Nd#S$-gd5V9Rv!Lf2E>d3)myhEbhZm{MN3`WF z8FuDju5%7w^0@dO0Cd2IJ~umcw7f8H%;X&fje0+F>AC7hk0TvZuvUauad3zWxP2h{ zonP)-R0d>}7@WxBTPFM0=9+Rv<1h|^hXs!AA~aWi3IThsk0xqXR@7HQQuxoE}l zW?CG;=iu$839VNRZY^%|Vk}+k{YbBd?b6aRfMb9PYdp3j{@Bpy1)@6r^Gg@_V$f&X zHmmghyR>(NxTL1UgP?f7cVsA|%|84X$~nEbxA4*hhQ{|Utx0QZ?6T97fGKMn7BsKw z<{uZVV!>s<_ecovw$t4uo;T59xEW6Au#ju~0X)!p-GR8|0PPM45H6yNCqPusEe!=t zQ7C^Wvo)j96wrYcSyOGx@@W4HDDT37Nad-J$a$0BlJRN}Lq-8xjvR7Wgy&G44MoK)Bte_C^t zj6x#ZHgw>WHPde)I}4C}0r&D_F-{234!{#Gk+l?TVUbDGdyJeMyD``5M2h@hHvLT+ zIEO*)^q3U9E01!vP%AaFV`~zwAQv7@^TRjvl!0Dd*tS&_+x7uk)HCbBP_BT9Bs(@X zN9PgL7?DGoyL4B?L!7@J^$bUY{?9yLJ8XYg#qtG;#VVzTqT8XKN= z{`lz}P!Iuu4JDWlv;(%ZKQVp5C-wPXL8asDXT$eX_p;$h=EA7jvOyoI{2MBx|7Qlg zO=y&}6IV&#S4RI7&;WB9`YDF!2G2N4np1v!NwP)^+$im@u21VXDF+dI_$X-4s>ulK zFw?@4!LTRJYJ-<80@)(lAqizj4oLJLaV#DF^zeE3hjUtcfIPgFKWydj(|>A} zD$e1sw)^iN^$Nmt&!wXVS4qr~+|Bmx|66=C0BbMlbIDq&y-B&3G~IuCiHaFfk*cT8 zM?F%@-lnWDleD47u)(U!p#RSC z|8Va?_bJc~CG`0(ZqPG$Q_F~rbN!wtR!sd+#E?v8@Z=(OuL@g0&D%{iJH`2v6YX7Q z0FPW{`~9THnH#fkp_g!LgA<^As%|biwbW6U;8HLZK$bp{5EWN67{6z~K+I}ex}g4b zRuNO-pwoH)u>0CEM^b<$VwiytUS4@Y4i@`(E=C-O<#WITsEw?W;m%RAalP%f3TLO) z&!`ifm3CyXRS{ULS($?1EG}sEd~BF7;AMVidH=ozGB<`s`Yjy&jgvLFbNYI9)TtPs zxbfy2i=bIHiQ8?XQod8ZU%&0XLjLOKtwheG#<204rP6{Ls8|nGuT%LfXXadm|brZK+%Fc0jR^2S8Ih)@X}gShuWL$gPksY z76p!egRhUQzA#SM&Qr|&>y%9v!LclKtkW%XWyOeK0Xj-VB9MvK02_Hu^eM&mUoF!6 zQygEue^e;T`srBe&S8inZ7l9%NIBG>P9szYp8muo*8Gg;TFBzAxaZKGr6uY;lW6Rw zI0zb&f^#MuXoE7F?>RN0J4zpKG?4)@z+pwD4prrfIeEWd*Zx{v0%C8gpL*h`Ur{Ve zWe0-Kps?ozT%9EWr`R9J{7%&TYka#Mpl~2|RNYeHyme>KLBU(^_B}E3Opig!hM)~L z>i))`*;a-)faLoQ(14%av7^uTK-F&y6weddW2G)pOVk`8K1T~i4zEey&FXKoG@yX! zJta1z;x(60mTByg!6mC^zr7C3x;w^atX5AjsW_re58eIkK6}3nlMLvFb#EHyB z=E%Q(Lt~Fa#tZL z`*}ahyyUb}hN)Sd=m>3=$k#VA#VUn#RwUZ+_|p3V71&fk>FpXmGP9O650 z)@*v{i;($Nr$aZ?EW1s*RJv4l#4ED?VNwF0OHsmWQi&2o)QG~gZZVHsoZ~{8cNupO z{^~a`cj_E2iAjA#*^F-faT_=yEkyqOPKB#nW#!7PQ<$Uv1Go9s5|g{<262e!D;Y2^ z4=(^f8R`P=-Pun1puD0#cL*gfZ_OpW_NbN{levlhuHM!b;P-xJXWNBbgE2~GtLzTH zzfS^j!rd$K&B7((P{*X-m-ag)DE^AKhU#_hI4WW;h=tuol*gw8eFHY2#$SE&%reIq zihv|HR8F2iuDRHg5>US$EF;eN^%`5B44IMOS3@7ZdPj{JMolZ|A)~Q{Q8Jw-aVBkU z)U3kSw&~r!sthA@3%sASVM?`2A8LF^0olGWuf-F}UhhLD=YE|t{E51`oYwV>a~Rc- zB4J9le!{PDvQD}p=Aa=JC8$G{5II&ke){|Kft?EPC*z?Os<#jwb&!8;aZU~zti{ln z`V1F-KRq34q;o=(5JqTmR8w`K>6!P9vv1{|Fn{h?{-W<)8`b^83 zfKJqE%zBt%RMds}q$2@Cz~@0<&2Ku2`fK}f_6x6u`n2SYC{T}HXLNil?l6CtRTofr zBjp3dq49WQF|5HeY_X2b`*e`bmMIJt-+5t?7WV4Vw1wTb>{9Y-5IwGlXtsLzHByRA z57KFt8x2iuto*$BHT1QUp~eI};)O<}ZuL)<*^6Lvz0gr%>x-}mm&Ygon@g+yr{056Ij62?0$t?xobbaI z-MStfHgq<1gpwuBaPyd`lRwLSqd(90PEn}o^-yXLDgoj-bc<`7(!PL)Y?y+ds-?HW5cW(+eKz=0aGa-7s0OTw}q^ z2Gwy-Ja7+zScE<~m3TjOORNP~(yK##a)~iRvbY0X;A%Cs4B~e4XtORbDYs4y-!hms za=`;*`C$*EIe@ChUwhmtCyuqk4MSJs1FQtCSFQS=t%v4&Wi25*5+!{zX|7ZGRnwxc zg;>6nekZ?HKF@1xrmH`|XqLW{e=eiqO&Qv{6Z4|oqQF;F@kT%E@aWf_J~)H!Jp|nT zoYT#FXu`ZSVe4?_VbD;@P`8;d9+CEm2gi(d2{jjRg~);drImz3BE=O(s)+}~+PG8H zVWMT{Qbj*1#P{i%3E!*-V&Luipw9O1F^-b~a`~y`99`{rfZIX=CB&c-dbyg{UK;Vg zy=c_=Xvxb202hxhEJ$daCLv?b2h#FnS(w;OUgDEY#3%3KCT?UAJ4{YT(*b;kA;km) zO>}EQy^lBIXRKXST2^)RIS?dpRaynBctUzDDAh|v7E*~bAJ#FJ6zTD`sgXS~9s3U& zPldK&jpu@0;A|B}$tL?OA!fen;A@5pVb74}XJe}um7_rr$N1vV1L9$SOV?-_!(sdd z&r1xa7r>JZHeohC7Q4yc8TK~}&P@-Uw^YJzkm>n+!AGVGq$o3vFU+<2XX)2PzSZ(R zHng0SM$w{QUO%Co6a*ddhMu-^sGb(EjB^zrrhY=1x!&O_|0m9z(n3VNiQ5yc&u+z6z4Xv1Xh*S~1c;439ugm?b$K>ZRD68q$;wz_{ zsx7un0u5y06<6b`>E<5`6s`N4YhI5EcfKIn3r3_Xt9TVbXeUJieO;uEA`&ye5Hrpp zn)&-ZuTCUUNWi-jXQ50y_jI$S?ish24iY3m0F{#stw!Zc`ez>1*V&~+0gOQ?BhK7e zU(T!VFMt8CIISFeM5NUHg3m{XckYIDGT2cE6=CiS$*C{!n;_7aOJ!bdc{Td#qRRm( z2}b6C^m!fX8VPwZ9(48u+>dZ9`fKq)+K)Zd8GZo^1=E#)T1pdB`w-W| zjEs5_NTr^MjX1wJsKTg`3B6AL47K`_F+HgGoKY@Ezc5Ec=~I0p*4?)N>ndWaq%Jkv zq0*??O}87PEd;8B;V|%Ut&*+|yXzAC#=N+Y9`oS+&fuy9h?z|mCSMVl@5(AO4<~jj zYg`>4&RnaNs6|I%K83Lf%)5$Z3d28{ryOqa8V^9TNV>O4p6DDeltj!pp5Nh92}=i& z#y=^X#ig=G;)iTNX1w0g>U~;SJH~Fa4*R>G<`qv4`3dC- zL?k8RbM&t=SjiG-_Sk+r!9m1Ne+-W4^6ZG@!4^~|M6Yb4XU}CQ zAd_|D6oo)bj|V_+!HsveeW6oh zE`&pqPCT-lZXNs07P7vv*Wb<)(xE-YK>3Acg48SG2nZoYD4*$d6mFp&+pUk@1Ua`d zDLCyl4t9rhM0`}%lY~?dLI40i^UkM|7_37@87W-Q4CTrx^rx5g2VON|n$8}9HXx&) z2sg@BuYD9+7hj|dOhY+VBz6GB>8Kc5jbPROjA;s&q1I;rMz(AyX#=HOdLsLlGa=JP z`%J%;LHM(mTma#-yEA##ed(hHt8nMI;P7*zy+SvtBfL5k+w`8iI9D}F8HnfYljst? zd9=mquRdgH%`AO`wVd*+fe)vyt5Mt}m_X0ukCOA}f||T@EWTd+f7*l}7kVH8gDTee zUXkC!k8&rioO^}bc^Thg6N{Xg+LAqB?)Am>oR@ZZl$$_}g1T$V+#?f=ay1~*jZTLw z&+^qPu0_s|YrBRi*A8&SX~kC6d2~#qpTn2Z9f{s!I_#&vEcYFBzc}i{q_+ zS+n#hJeRwh*u#j+_L2yGpP`D*Ge9 z%Yh8?P;y;?v~c}lp?aBwN8I8;Xd*Iz(dgT_);>rEyprbsGRBo(rD9SGdbAKQ6DcDU zkF-h=KzViW?>r(c1K+*QWtJbT+yM~tcS~^tC@7WCiY)dphgc!LZ1CXVTEPsPUI(GR znDIQ+eL`0x2$}{+-|V5ih_7g&#gJAJXDz@-H=BLq7NJ!doJctrQJSU>r4FU16)mfM zJ{CrU%1W091grco@;@a2dwxSfK}m3_QtT0a9~^C3M1K|7c-@ojAIQDg*o=G~Zea`XjEL@iA z!hz`U2M{;{_usdwsLjy!X<9Zeq32-tuY}q`Bg2J;HUa1{@y+L-rPZnjw70rK z{TQ6Lw?^4OwD~-7T5QZ*9SL*cd>!>#ah*mH1_lwU!<;a`l^D&onW@?AWdGdHlaZ#A z==qSsl@M>la?5x!VLH?lNXn*#_R$7Q@77i%)Go)4EmY-d2FTv-AKhlPYmzd*>Yd%& zR(-@+EA`H|EAE}kcW*ABPh=A>=}@J&gxcrR5>aM9joC&bt$OW?9aXp%P`|05^EwL` z&dX(>1f#Hg<8O(a%UsiNNTwUJ_zY_uu-N=^OSW2G|NUtUcCw4}LG#LLj;1BKp?8L; zhU(61o{TgXxw862Kj0aYqHX0(trrliuT8TmC9K{dp$I(_(RY43%ztq$7R2<9QyR+L z>as@<#vY>N=Gb*C!SI}6cze!KSy_k7-V!feVc`*$O1L>ln#33%?e{8bu(s_8CE$kS zu8{cgi#wCgJ~nLL_C=40_jmUAxDYkI15qizm~ z+{f}nW7QV$Fu2PlL|P~%gutxWdHiD92*{jTkiq!f42C}eiu`RA^=9cOt=^2tUqp;s zhz18YVagS`llNuoOJarl&$#2U+sj!Z0k-|x{b+WBlCO%P^F z7|loX^v~kh%6ALahWu^|-2rH_48po{nPKj!UNRi+AmTgoI`YpNQmQoI|} z(7XIXA7SnDlibuqnx_x%3t)Lvrbq6@%+lvHQ_TE@?C0nYbnvfGs(RGpS)O0wRX}x1 zUCy7(-UddrNWitwSaZmZ$#$D`91dWegdSg7OG#$&tZ;Tm|zLXGQDE@h}HgeW#`h zS#N%6z{ICvd8NnQHQ-m3WteLcEwusd%YGr5c7Ze0MD+LUJ?-Tu!@#7uj|cKjEHb)> zCRg0lNgNsSLovaxTQ1hRep1m;r3` zlR!&zU0DwC3}8(UiG;2e;g+w9%bvP0N~Uu<2mvpq)d@GIac=UE_0>tVaY|pMthm)-$=aagi7FMqpk z%YZu{va*k;+qskx?NOqtpvaEEQHfc!p)^)G+4`Xb3rQVD5t=rd{@xl|R3|Xg^55&wrc;#NfpzP@gfavy%bSF^L>c|-K!4$S;)ZqOdz(9iU%;x;%nbn~jM5De~Q z`Q}^Sx@yKuQn+-%*~&d7i}&L+5206<`ry=|doVkR4>hKjZ1Yz|RTEO>&|r%;(@Y6JY^>#ATgzwiLGR(_9!$tT z_31>K9@TGQkB}8Iv}CBAbLBaA0?>Xux|O0PHtDF2d*1LN*-Gu0FWlOvyTN?@QNl1e**`1|dv3Qzn!Eb^ z3KrUCrAiK&;FyK$J*@Ic?@kEfdJAIa*Te2j@_C&o)x#bSnJba*J~(TuH!&M6)YPnU zP@IW?k;OT=WwVQ8ln7iCQl4#^aE`hLOPzGz3=`ftzfudGNZSxI0a?CF3!nIV7wWGd z$`=A1OF}7|hT5iWo9-+<(4N#N8Ba6_gmqzklZUy+Acdo?*z}E(O6P{4Nyn0kYZE%pEBm6UeAdfur~Z*$ z;}m10aGSkc`ErKHJ}FYc57X8vx&$FzfLpLY&-=ipD|@YS@GY=eX$k1eM92urs=zlR`aGE4wX8Y$qkI;Cd39!BQK(zS z`4)^1HuOOfjleuYO7fqby6eOI;4qh1xEPX)BFwI^#%gx0hg-qdq1pE5|LU^Fir*4x z6OCGkty9#*KIr|vS8-@N<7)4}r>f{-X%4T1N;Y}lpwU=erP+)b zwBnT4DPHSp@s^vfqT=q@qVw!>k%INB7AmAi6nMi1vOv!PYSSH`)ydE_qCvGw-MdEl zi%8)J0|N8rrOv0v_sT1}^MEuiJQa&6J>y{dK(>FA)IQ9o-c|EORV&Xg1sa@bc2v>y z`C1AUHc8u`lb!{FtqRR>tYPIsqmd6&24T5#lw#e8IefrUE7?NW++M(Jf|5foP9YA; zrNy8+@W@PMt2_(l9Jx(EpM>g&C0EvG_uIAVBgRv`dEl{@N?b%H#r*!cHK&jciqXXu zGR!pu$K#)(o*tqwPGIo@Gti*YOwDwptzishcHLOdQY#O-C-A9~2FuPY+C&P!Sxh{Y z(Dp2Ual5e-V@8#tk^SPJUNI;ymk3vnhdw}nI8YwEZr-TX+9rdhIrs>_|hgl zcW@g9biO+H-Uj8C9?ZPV2t`jO6EvBJNG-b+1f}r9_uYr)>5nnk#iGu_Rgbdf5gC)C z{5{_7D5-=?JHjZ8=KO$;0`8p42%SWtJ2Acb)U@6^?oZKd2v;b?^RZlewk7o{BC{`W zaGs9TbIc?f+|!}X`k(rh_{O{UBRo3pl%!=89jSJ*9F2dqNi`ymy0$0TUlwn+pLXWv z4|nIbj&5fh(_yn2(wo~c4c)V+(NTiJu#F>dy&jzyJ-N(!lpqh~(Ix9Z)0M&Rwyq)- zlfRrn2Mt@%YI3C8{~*70gP>uI+N9DKpLDU3E|D5XTjo1-v)!6+xu@_IgN5_xpwrx= zw|f}L`BmVMnsb}u2o5X~>v-pl_w9@#=oi5hyDMfkvBPjVxze|foUAcD7%=3lKn2ePN`=luzvqt7zQBjKYrx7@z%uYk=V{ZirW51h* zCO9@r+l@Y4C_o4<8XQte@_n|BpQr&B&#bF z8X<%m4Nza_-NH>^4`m!tVkt#1-~E(lPi7lLmp9LGTOu+q^+qx~P@I)hDACu%#-~`G z&@5@r^W4F?OTW>IQur+fPAA^Zo&2Hz)i>W!s;}g-j!!cgloajWlGPtI`diLBe6Wnd zI-YP*Ry^e=$A6_yV%s(u??pNRXXtpfENIUp#fHm*oP4&~>mh^;GvIU9B+aSvO4u-a7e9VzRC}E*$=w3m`6jkA=CxP-kD~Dnk)7Ni&(rK? z)FFzt7^BCm4Kwe-z#Q9Y1)eMB_?S8=X=%V*Zt3S3C1ra2Z8{O9erI&2D@x$j=B*2z z>H%a(B5a2b`}aTS6mwXJTfNafpgjL^7FQh-OUG?qjpBN>V3}Ov+#%9SA6yic*vj1- z26~~5;un$Q*-IA&3(6E!h>DxPVMu53$@Jju z1GPw~FeOR&Fl$tKUDb3Hmv_xIgistKnI9q(lDt{(c_)|HzvOyRhPQZ@I#nF%L-=bN z%io=ychV#&-RAT7FrYk*N6IUv5!Y>E`Oz{A)?xL=29VazTf&~myUy3mX~H?h`PV=U zn<4#n&MXEF%3+AEa zx1YSv9*2TYHR2K6?|D*Iy!ya;?R#z(ulz%mgLyMNkcf(Rubs1gr!*gAmSid#yfUAx zdD&OoZ!dRsF6JG*B+Hkgr*ExP3r_0$7Fq%~4?zda$G+?zKUvPT_;!@bqZV|G`yS+3 zMCR}4>2Gq@%hSV>+sGYPvWObfEj`0X)nSc3fs=fpdG5dL-O+i!WrR9RJ1d{Nnw167 zUVbHTtl(7}(nTvxTM|xz+oJip4)N34XCHh%Ks82iUzXh2x)QB;GbOB|SdtY~_{9e|?3!_4-1t6;ayvAss(`{9-Q&r$oqjbsJ91q) zy~vQe-5Ya!J>dH%tt?)gr{=LC@f4K#lEYnezHD+ZSFd^X`73$85hHb!f7~ z%f?m_ju+#4+}@o#3vZ+m_bgO=bxqle#v34I=9%{&%b_{vWtaEu43~Ewu^juDu-p3T zY-`3RH04{;UO5Wmh=qyuT0fS7U8HEBE>}>Z@kOT$;p(jh5s3SC>aa7Kig6xgw-3o! zYLSn}o{gHbm&aRI4V;RF7G~h=>n1~fZdF{}&S;dTH~l_;UA%fbrvK(+O5{g0{`?7& zY}pm}1ljkdGCMF1=!<+@{59Qp`f@uSAzXEU8rdVkh&3$toIK`})iz&#n)tAy6C>Bf z%E!reusmGFW1yWNzmM2{t?^Z7%j5?H~ja$GrJM&#Zr=9_hYv27jG=qJmc=EBU5FgDAu_{E#m4 z_d=ijg_nf_r_WOB#IpVaVBot5H3MZ<9*9ye9{!rJ7%)f60qcRqc)XN@Da#yv+F)lg zOqmvnEiJh5Lq6}Pn)nWkYsV~gEhaV!K@#uLN=-wh=^^^2zqT`Uk^|oAYsRi;Io?Pz z?Zv#WqiWCXUtbNVUT3MPkwr?lGGfc@cIP>|fgn!iFY#a7j|~c_*2s{?gBbV1OiGA* z?(C^`d2f|C>U~LTmF8eN*cht$ea%h2mUasoJWmD#JIvU8q!A-ozTB>~S{w^iS3RsD z>0B?nvcGiB#?sr{PS{3HP-bN{kEuztQa*}fF}Rd`hh5NX?n%J_z4||cf3us*c4s&a zggAXdSuP$&VbWObA<5g<-XbOCpVbj;5T~{w&2!l-9HqJ!hYYp2_#>*g21QUJWpys{ zH%;4Lm$1|lXUKO3*kIgwvkQrr5Gm7l#$z=Rte?Ss0-)|a(uBdJQ(yStJ{-3$+Kcf& z+ypMNa$&5za-n|9dg>+>Q*z6pDXnfi`$DOI=2fKdoNI14#PD|>;+iq7KP&`}*GW0cv=boPHiiB4KadjvRB)!+1qa+ffp7T5LWem0~P!SWCOu36o`X4a!} zFY=M|Jic`$=}I#06z=nd=FhWqS3p&zbLHY)eKMZJi0Sg`vgyZi343}a1aXVjdzbwd zs`g?ecX!wjb@E6a+f;%t`8t$OX3RH$X;>Kx^dlOsUvwaTL!%@X5jXH{}bNyqK+h7!zwlg3zoq2CFq#uvc0Os-p|QlkycoXU$*(P78+R zsHpHF#L2n=IBu&w(E1%`2Z?I)Pzx+bxvKvITt2M16 z53A|h2~5p4|J=3Wf3}v)7B^M933U<%E6yOuz>i)-5{%1;a;5mih6I`iB5~*##Tw!{ zrGmPzWM~BZ?&VFy)=;sI-bs1~1TaY|6%IWLpVUv_drEc}>#uO;IpVak#el_PpME%I z9DFre@eySWlc4kFU(`#hs}DDzo}PF?ly&SA$MUo3mywoaXms_r$c{d}DEhGa?Zt0x z3aRfVKLe3)iS`c7#=6>YPes9*A82-{b=h&(oDWrW_Q-DEcY5dtsPfOOiKK@V$oH?T zFVx+s@6ku@k5DKNTEC+>6?TXnM>OnQ-Vm)KJHf)--(ot**&>(t1QB{vT$kzO z6i3m`qg~cew*Fz{v`9hU<58ktLkXM1XyF*@HA!bu!(m^ns&EZ7%Z$ zA9*EhjoGNOO<$YKpzJEEy#Uy3R2KyigJ17EK=cSRsV+a%?f|L-Ne>oTzCwvE?&}Mo zPRk=7VtM?Q#@ZD83Rj1OA|n4IPoqk*q2KQbWDUJ20gq6q3N}wW^^$$4a7-@S#uYIL$Aj9lng;sip+%Zn)QpSkPyQGg4?ZwYr$yr45Gx>LJ*6{G2PacMNy#mqri(Wf z4~OgdHRlaZzY&lkZFYowGK&`McUlk!MQ?oO->dz@DpaLiYO0gEpMAY&SaGYmOk2f& zL~`Sa6N={-1G56l=$j^i@{y5b)cyIuzznotFyN+l(0ME=vl1?avuF1R<%$m83K1H? zbZQZKVoz>J4t}o3v~{Si8m6zbHsBlb0Y%MB!XEQso+vD6SgQG#z+^v}*L1%@<*Aqs zDI7$9qa<6HyJE%)(&w^>F;=1Mdet!s3vfLy#Pm;A2C4%W3yxY?f*}R#$eC)I*x!3_ zliUG7s{4r#;c{fWk~-mv;wU1!jxFMQ=)0&TuwzSeBAqAn&4-h+U7yvn185Q%M3;2n z7yN89lP&$M+qQMPRGIb?RCfF`8DMWo+t|G_CDMZ2zOne<8|4{{)cY98?pY&wE*QYf zJ~cn!l8tW){1?2+XRB zn`pm;n3cjW`dvGSFWQ_qsHn%h3x#_xu5T)i;#84k`*L~N1mAby6^ zQY3+XM2VFBgXXc#KHx6I+Ykqx~$7nLnu2gPJP-753E%Z;!LBRLNJthuW!a90Vigl5Ph zld*>51lL)yk?o8M^Eo87wu-K=ofS`u7b6r9Y4)`Y$CtSaua4!~dARtDbObt&;Cax; z?w`=bPf80-Tv79?mgqs}5}3hu1ZI^eS`pT>=Qo*-A@XB@YmLVKRkj-EG5tUi<=2vzH^2?rNexd7x{0YoAr*@(Q!HjigQ-{UOYz{;)r<%Qa81T?= ztONL;B&)d2V?%MH_D!){6J#lY$tUKM`h_N}$mnWigP{pi>J|OOY$pryv03-Y5n#QF zi4X1P>E*S6IU#n;T5Wq2#tHlac7Cbfap$I~{KBw?TK8e^c-deRUEGJ2HhWX&Yly=| ziAaRmEizoGu@0(Nx ze!MIt%z43Bu^!E|lyh#|nNWJkpi*eW;uD(nZ112d+p9(M3Mf!R9zf)nT zsaaoElsl7qP1UQsNW<1<)*V#TVvN&h#s2L;{+K~n93ppML`G%9!PP{Owht7;N_?x# z{HlwMn@i;^U9zcAPM%@ozH&h;T01T+=E21-^nbE%KOdCM?+E>~8&jw>#16|Mh0M$Z zG2NGmdJknw)1_j!%d~N`u6no_1#ocii~fl2oO6io^0|PXg(Z$!!*%-XFskGPsfvK} zM3S2@?%Q?%Yzs~S{zS%ZpREwusDt0o8_eR^(?SDgMwauh)8mlJ>EgjHCKshAgf{VC z{@E8}IX?KuBJi23Nz%5yChiGn(SOs~j^gwHLyq)C&aOz=d`gYY)bwpH(|Z_WZvO#g zHhI{+aL)-lm#t_;00zNowYs6%FJXZ>fD1JMs+9ykf4neA2#}vKT>Rj>d*#>nEsyx$ z58p@mavB9kf4!>?ZE3_LNK+y=+u(+^Ej(4=Dwjm$x@o*GPeYY7yJ!2&iW%EenLv17 zx-wG+XhiHD*#l-vQN`f0r=p4-3AmO6u+X^7v^q<3&4kjR^g_fbKmErG#f5L}Vn*&t z_NshmpNfx+*TqdQnVIrfdRF7`nn!Zd!E&snZ;wAegL`SrQ>wm)%+lvCO;UF15@f?Sce+;tPly-c7_>BbSSELT(kElI`4-# zRu9s65M!u#F7~8$PTm!3%IA;JI?@sUGKx9Y-!#L6!kOp^_x4<oUtEztcCr zdKZx|BC5)xMj~X_cd@^|U=eUT5otvG2<+Yj`GA8L1DM;AEsGV3h68&u#3fpu9ocx8 za@wltOsw_ooP4+^cg^Nf=4|cck}mR^fNzb)YWKa=mJOB2B`VDqLC;5>sGMmJI@{jq zXP<8M2QwR+&~k-DWIXGYq!}yPP1RJ3}t5f&WTgh3h4d{n|xgPa;^Zx1_}P` zI#kI3^SG`5=|*ujBey>zkTP?cE;lau@!qJBjs86;3c0NI<`r*{T;1b_RPUdK0uWa5 zzT`CybiCC29{m)(cLZjfSO-n@`kaxDqOq2#J?mK0^!R(Dg+0vhtI*hpx81vUa!>N9 ziprb558eJ4?lW1Dqdo;okK;Ryc7I|D?M1W8^WS8O2}TJaEa6IC;1bNuL-Fv*X^>he zRqPiKOJ4QcaZ&osZ>H@bO5`OQ=JShx*d#XHtl;wSQB|6l>g&Qb1j2IPEO&31TgBZW zK6LSidrYg0bfW%ZE%fqYq=G-uQWjh@Iz({52COB$SonR^3kw0~EUqixurx*g!g%h3 zGtE%UCO$!v?7yu2a*aq)pvR|cvEX{-@$b$dOtiRo-p}A>jJZZvzQu0+H_>9MXEoLF zj3dDiZG~oVm38&c*g;A2tt%_(Z1ho1Rn~k3 zl=!)g!>PV(ur>4>5w^7P?5hMZ%Tu>QO_~kwiICli`5orZvRmXE)3)LsdH#vLa2 zX+ek}R7bC}o?2Ma|G4mb9MDFDwFLAXpt{znVDWvs?%;LuJXzuNCd7VNW;z3*Y4;=jh<5R;JJBpW5<7DHq3zK{PK02?)Gt1kige(^Dgyz`$WxT zBXp_9%}5j$n9LB+IVnMg*N7R?iKT1r6aPkExH?G>zAvH^R*zgMO|ADh%cfTb)b3e3 z`&SR6Gar@97Lz31t~T)lX=~-XeEfSPj!=<1JG_{rGX0s?RflS_^y34s8>C#DJYl0< zKTeX6Js?Fo(wv$|a%#h^gdg(id z6ucqXZRuSMXSW@plpmaPz4LRV!8wkGsCYL!Ty5nNp#O+%l+7)EIa$#o&iJrzpK5L+ zrWHULMKU&xHDbORD`wW^fv|Ww0USCwjkRh1jv1;^zdiBI2@K^Cbe_HqPOgU+K0x{r z#bNMRLZ@}y19bB!7nU``hRyt(vSD8Le29}G?eU#>F%&35#O0JT1X@W!)QM)84DfGEv=Lx=N0v(6YdHkizqzjejKGR!(>n5y(B@}?w zmD|IeVU+`|RUfM9HE}3Tk{JOP9;))HU-h;vG?YbO8WhKZXrV`HZh`?^$`a{XBf^=LxBfDP_>zL@Gk(w^`M zNl7Z1D|Zd)5@P{KA}F|?B2-BJ#xvJEi%@oG9y zgHQOmy~9$dOUK>>7wCTKTmk4WUaW(CUzf@YW-1CRyZ4#gv0z>5_Eo7IVK*j)GM>d+ z`S*ro@L&`jZX#zb&aNwM@X$yuJ*PVY>%+ig2TXk&MCCA0gwb{+IAA}{?r;D*4((b% z<@kN3Ta9Hfwwf_1ywG1->3^6>v&*&kH-!cTOSe8engtKLYgq50Qaly{H)f`GHVs;W zupiYvq8OgG?LTUNzt|0rap_ux6X|@f!#Gt)(eGW?193pLcCJs=?$d&HU(Q{bI&cPw zIanjsVZ(Je?$=SG9Y#oUD`rSo)a+JHp~k{8gr}++6EV@U6o&c{l7h|sN-7?Dg#Gpz z9#g9SLjNhQNL_<;@>0FCEnb@`<@7Ey?*|^;dQGey@%Y&w;bKim);qqBvMyd`j33Pmq4G|C=qs86n z7xckJ3#hJFApM;ZRIvXTsi16I1kTJJQ4St-#4qZe4!);5b+g%Xct$=P8=~n-Jp<0o z;~(T#aIS8}a8SWmkN9tQ^+e?=Hv4A?bpr8p`39kDGlBk0U}4SUtW z@0e03B|Z8YcOttMVoVdpE5fwDFFf3N;`liL9X$ZT5|4p14pPdDi(j!<|fvxrJ!K++fsL<`ONp8z9f z8jyUXZ1OxhkAW0iUswT3E?_-dAzq|%K_E@_W*CDF6DHs~&oD}g)FQt203Lb8=`9#y zK+G-+EhBmS`f)jWDNZNv(`-1lzYz@!2_!}w5V&sk=;i~|d_C9>O6q$nL9{YnKHVsqlkhz&->h%s8FZ^JtXVzM;zk2 zC}NA-+52|f7j7ZZ{s`Cgnel|K^DvLw6LSYE(mL}aum>Sbgm~Fv(H4}S>ZBO+tK8u0 zxb7Mmb`-9OTD~>-Ptog8|MG1T!37pt2iL1e5At^$pzc5`Dx?ux;;_mZo(uRvFtZ^5 zg1v-Ofk#~y7TOXlHYLzOQ+878ZS}}pC_{DV@Pw{6Gk$Dw*P$MM4rhyFQ`g+c6w}Ta zQGbrQn|}gSR4@y;eTV+Lzr%^onI%;4KyH*^swRx#jiI;vK|J68t3wThdvC*fj1b-M z%Z%%D^bx~qajf~%D_UeV{>}XyAC1JMB!9Wq-w!k4qAoqXYI?R+!HW!};Epv5zdr(+ zeqGL)X499n8^$PbZ`hGwkc^upg`+y2)FC-{-G{VRH|@r#E!a5sIhM?MK+h2C%Kn%v zUEF&8{oSt}DAhT!js^F+GJOaUp|drY@pbe)Prx}2l24c-A6W4|Cwk6Ct3DRsjI#>y zm$f7fV%C~xIzC9mN{(M*Y)`TrtiP#4B|$UzAY8fO%@(d+qx6k2;yNT9y9l~`h>3+;1kNnE^E0M!s#*8QtxGDcejxuOuQY7a!(rDr;JZP}o=dOA zI@}PJp2uNLK|&8H!~mNvkra>WAl*T;S7x=H_AzM*>Nu3PJBo!aj-`^GhYo;EWsiG&r0WQ?OpC;vza~xlxk=8E1A-C zc3^U^hUT4jU7-8_|7Q7)0+c#Qt2`DnBOH%W#o{6LXo_pa2oY`?^XXe?H78^o2Y*gp zbRpQNDsB5G>yto7A3x681~>TSWEv~5J^^&5ORlcNZ-dT!=DOnz&m_NNs-Iy$-T99- zkA^0HnA_DmYm3VDQS+G!FKZ zMJ;(&!?j)m?|PqElJ6?}ExK(9sv>cbiw#HlT*rQE$LT2k^4O+C zQYzPo1LRp2K$&T=lHso=O;^S#rZ0Tby^L_ciSB+t%WuIX06^tu>FH5-V<|>l0aCv| zj!)q6F&+Gn0BlW1lf~d~kTYHr2o=N`AZB}bF4*&+y*-@(U&!}bHXvxN*dLjnZSLdo zc2LfczW-+z3JB=ny`f0)Ezw{oMlvgr_VS>kL?w~X8gwK#D2ry>l;f9jJPsxUh#el8 zneOKA?Ov&!&E6;Bu61r?vf)!xwU{c;@YA}da!w_0iS`nVVl=yE(x);=&P1!>vyo2T z9xu!3_xOo2-}g@Y z@^iO`%z@X*({ZL+O}V4*qI0%Zzhd0Sm`hVAmXFwdvDPcqYOc$unjx9e<04Yzu?c(#}P}bF6JW(;+_R-eJ zhi}|xvkX5*#D8HxdYk4_PH92G(^C#wqm($PMp`pOQeq4j)6@qj)!d^bf*Ri$dk}Uu z0cJtU`9OUAEzXU|K-c=^*SpMhs8y;@QWjTkNNeAhSa+01ksy4e-4FhNqAZ^6HSP2@IDQH*rOWz)gBM-?2o@`KcfD>Fe|dy*N?=+GixwK$i(I%G!2S zh``jwt*5XOebO>wA>Au-j%vy1LS()&q`+@U9lb=dQS)*gl={HM=&;xa@isVLPB zr0S%(Q{T9GZ3{zZniF|RD>p@wL&W~$iG%lS^m(Tuo(K0Dfq0@*oZ+uqO?O84 zJN}YaT&dqZJ7}UrxDESHHz)u`*o>#ht-0ueC#3o{MUOFj5M z{;KL+-P8*ZANad1y%kX9S$s=B`>#CSif2uz@$ZM#`LC|KMG|-gHt6InS4U%SfB^}$ zpFB?FES4eV@t3rulMX6_b$<0O&-n_-1IHgsPI8QFCVYSWy#++cnn&;olH8iFHF412 zc^5aDoDJ+#X~IP6u+htYXUKVH(NS07{XLG`>R*O6;H-Eu{Xft_$ql-WTrojuN~fIq zvAH_#KS4}zKG%5lG-G_GEhV_v=t+yvy%$lNI0i`P)l=feW-+pqq6@&;rTAvHRXQ){wwOxA2-oynVBo z-V3EurUvC=gt?!NdfXuAYTn11l#hqDSAc~0J?!zWL+1V`bfOI4c$c50!jqy$gR*Li z)Ny4&XLhb;tI1Y)sdk#CeZDWDi&OOcnzHmG%MbuRKf|6BpSc0vj7U_xwbt6~$2V!c z+;t|95`kTNV%GI4`fwL(Ihx0ve9V@aRT~2+7Z15Uesz~Lg@jUtsjrn_a0`rt`QJP) zm*1`iRU9Uz8oQE(CKp;u9IPwS)?$`3)cQklcf3q?2?YNo$mR_xr5JXB=BQlbg zLn)z9ASu4KP$^dAi(enq?l==@rPhS;;sd87X8f?`2<4gt18`W?VEbr@W=hPoj`_fx zxRf$aD(YpP5e@FSy#jW4x7zCYwv`{&pakuy&KCGT=H4?Zilzw|9Z*paM3*Qk2rfv@ zIm-f)bIzb-L4lPh>5guZMN?q->+q_?%cJeE2^>89bhX1(Dy7j2*v9-@1V_C1;Kj_}~{X3o50{7>$ zdOZe8p|T7h4oA(NQYP~fu2u&)zp3q1M>zDYBPNV@0FMuoJAt^{A=^A5{Tw_7w=^Kwf z`V{URdovPJe)je)9wR@JfzKGS-)zg7dWID&n$(4l4-TghUZH z^E$4iG0`Z8@>9s7w8l#?iR{06xY1R3@2MdH;M=b6D=Eb198m_ZUA`Fo8QcHdCQqQ+m-H+xfSsqEJhRGu zMf!fg1#BxpB`HdB;(4kEV||vvy0!WPSz|Y1hx-s78Q%XH(8F6s{{@F)Sfc@D)|*7! z$|4>;_3WqI_ED%Jd5ia?V>i$^!+2xRAode+@@u7kDvNBU~mKC`iewfEmZ?_JdZB5oa zOdDW=Hpi!O>ODIne{T+ZUHpt4Q!a%e%yAJSU$FAo*9bZd?FKrxW3*v@#iovETsRRf z>D(HnhS^eZAYECSYyeEH`QZ=<)wq3>mtd2E(;rE+pO{t$Hs2qR<+S(oH2%lB*~ueo z3H?=YG>AsSrl`#}y;*?vTSmllmnQH5auX&Yr<3lwX45|k@~;H>;2>mVCZPZCCt8-f zFFJl3&xEVt^aKDUY(Bs}{Eir~f(y4rY5{h-MfW=6pHezLw7=6K6JA@!=V#O{rlbo*7XfvgE&aKdbCeEWbueD2p6zAE%mRHTf|7a!fe-fs zOB0-d4-NKCBgw&zp$8y+aDtC}7~#kY_AG-3Twbd(-VNVnyR=(37r-|XaOelJ`tEq1 zNrfAIub<{jC$gg#gAZ!msd&J^LS1=^DDFI`4OWhIr!b_(g~82cAAe0`4@@vs80e*; ztbRv4?+?^hFuO@6lv6>!GE9BbLxDjd?Qlvttt)Fm{yBsIr@X?edv5Kq6*~nSDB@o|s+bZd#N{+ayaixSZ7*OJ zpr!bD&IWY2#NBKR*^hNG$&P`FxTc!Jw);G1Ixm!f!f$4pH?&7xC45RS^0)P5kUv$CLvwhqZ5@AGD^fQ}6p+Xt z=p{kIwUQmqfF*(!kJbmf%nBm2y&V2HYX%$TJX;(VX_C#xp|xe|tjXQXEb})ewP!#E zUe0OnmFomf?Go%Ouer#e_to{ZU0ap>{pid*N%k(7d}37rYuT6y^mlA=?A|U1&SUzJ@1Ad}nzb^EF@}u&CzRDT zVE}3qniNHRv>%8tOr_0OuYCdpwRDr)x6_w&s#a2I72w63Z&EIcWj87fJ24<7q_mp; z0#I6BPVYBA99}c+@#)H*Ma%N+U9B8Fh3A45mO+1n0ywidMx_WVnu^(SB4E_=N}EGE zC`B=}KUN5()mV!0mQdiY09yjSbb@Z3q9(a?71F?NMN%HH_B$p4mj_5Z47{zqn4-^T z2|5S03XsB%-7ANsuYuS1$0-=imx3&EISvIsRFoL*-buuGGy|Eu+0s zu&rlV(_h}!NRG{rOD4*)qlkfr+`JrsOe|0lrVOkOhD5;DtMw10lrU%k|MYoUoRIL1m>E<$~8Xp zgnUuMA#1Jc1!LVqeMw^rzP12i$#AIGG4NySwRqstfz+uzSUdywJXx#h2n&43I!=j% z;+YgMrdD1m-Z*#<&c%AR2|DpEZ+a{_PkRIGd7y57&>SW@g`gtH2cS9lJ8j2*?;YF6 zHlXMmPqMW+(1B=zy-Ys+b>M}$hs@@hNOs_t!%I%qTCl(eivjm5uZ>FtmS@F6z!$N{ z-PBIlthVz}M%YTBu1w2oKvGxXG1s%}SRfRdxE|1%?NkW|$H_MFT|>0B&CU+Acx#ZZ zTp&axZQ37VIk_zU#?6lTHKRkcR4@s&B}Jg$J}&e;VaD9~Mi6y1V7r@y(1kU6g1B7Q z?*GT@C^oGX^n_wOz<#THfo?fVI49S3eOg)Qm|0eD0IYe|Kk?b@Xo?-I z*Q%=ahr=(xi7rdOoGBB@r~Vncwzjv2`Ua$v6_(&NBxy(It}y+S*wYcO<;o|PVL^S_ zN4>hH)`5M#KIYN&4<``Z6R8>)u;^t;@tS1#=U9SZ0PrhR0qY3_khzWqqvL_!JPZXu~&~GknEHm zs6V5IqxW}`YWVvZuRk-hzOG0v^Y=3)fbwkM{%Ry;t!yKs-tA@lf4y#LHEqZx%Z)IVoT zrS;N-z0B?h%S1caou*$Woxv|(ppD&6_twR%^b|hbB^Vf6DwjhA=Po}tvTo^3dGz~P zgA7kZXmd9-uY$=qg7+7_c>u(2QY%3h?=RT91LQp@$yXJY8i#&$a?l&|H>qs3CxX$; zr}H9}*W6g3n$zAw$5~A9wtSup1K9HMs3G!-@92isdEzvXt%(9j#(7iRL5z+Dz)l;3 zyEB0aGb-GBVE^Sz>arTiR)x8rOiz8|Xa&0uZ|PC4lrUxkp6y;?0^|UQf(j9Ek6=XL z5|pD2zh1BY%&;(1`ZbZO{73sv(a9HVBSWJ+`7sNsf7ZwI%L6OYzGf`~Pn4!AIs(~0 z(LZ7SOhN+i@EKXS+Tnaw83yi}AiTb1%K?`HVcq?&|1{+$jh{qC-BEcV2KBVVz1lN{*^ zmo}1c9EQVdi!lhvh%W#DbSpqFdO*dDxx_Q^?9y4KMUudV_5x#MDMN%q8ytanBY9 z%pza}^;X7L7KE|cIRPzozX|wN#$k>4iq)G(js{+vldhfRlRNK$vbvnmlZ^e_7yWOh z{)%v#J(c!0r@;XyMyAg&tD?=NI7vYVJ{W#f-mgN<2mBRBfvk)K-@htweUKbA4C4rL z_)F6X&|V8ql}Rab2tr=oSnXn#*^o9U8u*uI@^j|p1Tu~-g&>%r(w!g&NAjpt>Mm)Ai+JrV@&6<7eP9P>=?$2uWVsep-98ps_@dcwYPt4@@%#p zD%$mW^1?ZR!05^nPywahfGcPyz7k1V_QVADkoHUgtlKJ!jL@f93*J~zcR#hU3)=e4 z%qskZ;F4+oT5}WS(rZ$-ja9w1Eqc^v{;cNK3jH506IRrh&Wm^Mz)`ewi#{Y3Dx@Hf z{7wd8)=TB`kibq*ABeY-PL4w#GB#`Oiy#I`WQZ8Ca-Ty z)B>aQ)oG<2s_KR@~FRtr;#fGZjJ9e#zTpGklO}UO%_wD)NR7IXAlcrUA|w zFZjsOANLww5>>E|&L#rRsBaAfU4zuCPe>itcZ@=#Z+SEvhkEh0t~_4{TiiOc1Gm#* z41#A%7df%Bp&?@(T7O}u`nzgQb>$rp&9BiVQHAk*h~FXq0X+MUDEqk4V!OcSPKzK+ zgd%xXe?rcRp#tK}Awq7hvt9E0&i4SolT_IUs>_Ea`S6lhLHNF^0ZrfB#b*(mz*X

7^|%+yM3cd6z+O33qpPc}eJcPivENM=Z< z5(xp{E5mh4@+>P+qQO{C!M=Tw?56S?W=WWp@*a*k6Fl)9JTxRik5c73_AYPFZUbeN zm}P4B4yrz?v^%nHWP9hw%=u*<3|}w{`fKA0oHMMn3ISeX)XIqlek_K)Yl6!{(-L82 zPnA2Xb>VtuXaD9?J{OD9%W%deTbsPj%qk|@0-&@M2G3;~MG)VVu!2Y&)H3o8MFNs? z`<(&`z5C>Tm5vV#?klT2)_m$>e!AfA7&m0^IRpzHk?pdKR-X?El zEa-OOUvl=6ux6zyttMq{g)_eMEyRd}rBya}mZMXbhYZ0hbK|F6u@-%;d$Q$+t1Li}%2Km3Db_)P~f2-({^xVjK>{uxX4 z?@{EOe@FGO{^olAVkiF1)b*cHwzb_TV}&FB79ygzKgYk<j^a2!1<_0$l1)W@Ik!W->#`o9FtMrM}HN)fE$@-6!6c7KPkA+T1doh{pd#bBYKs8D=(+iy6o78kvPz-x4<8-Rn%6vJodr1h0yV zsR*j$vYP8~VfU9HX0-i~!z`AA(w|DxFPKYH)xQ1k@!%-kL#^Oou*%#OZmF~!@X&QR z5p^utff@9WA7H9ze4WOBhqAvAm;28x$;JE^smQ|0#qzJU^zX`=e{zZcDE%)M%b)S+ zt#Dmu8pj=iUJuD&tk{lx9X(s@R)k@MC7ZSZgRTK-(|%^7T4wfC2?2RZdYQ4a}G7r<`Gb7*zR}Vh)vIDM6N|fj9NUmf;``QleMrE*(5Iqn`T8jT z>$4A3BRV3Dsc2`uLz%y;XAr52v7y{tTX!oWfN+~7!23;gSW%A-i9Fk)d-H!djRt2kR0g{g?%7_5-Pdf4DDW;B&BWr z;SyV{H&`I9&B>csmVih&Y^6i8Dolvn6C+pLuFj9gzQ*)bq*O-u@aYtd*T*mkb8lIo zj=e-37eR3g(6oD4DDcAzE5i(!!)m||HrCMD^+yu>1QO<8gpgI*y3g;!+k@L<(xH?_>Fj}E{t7eLl751*OeCaODR8tW{Bgn zY4^avfTsgZQ^7x^yo?G^{ZfAx_5S+gY!D69!+lUSEGb3f8MCW-NSf2n%DiV|P zpu$-j3ylbI6<3@A4JDk~T9Ks7ljTaq(S#m34RIppcHbLDL>vpS<_S&li=&{K!zgtx zb0hH=n89*A2m>O*+n1?bwwe+OE{GvPFl%;ni*TRpe;Vo}5!4l}P_d#G#AeltJsO4f zd-SKdwYTQg#V2nfihAaqx&O_x`Yg$kEvIz2f@PH!jM-%YM~y=5ZKq*C;06EGPyeE) z@C8kkaovgyj z%a?tpn?<+Z5d}U1!xfo%2ts8r@|kZX9GaAGxO{?GOi0xYyDKfio6KE|1#({I89Q*G zMpyt2`+!-+3qV4so=A|n>r0O=CAzmNGNfE4fF);k+U3hHQ79j?^a9mK=1JqnOefzm zC&o?XZo>X;8A%__?y8L5f%IUFEadPf!j2MonL@jSNwl8tHx>)IdbmIl>yZ|ziI!y# zMihoLUrW3A50MhNh~D^S4ECn1XzZ1E^%A>)YsXU#gtng@ez zsm&JanjbVAhI2zHg+DG#!#t14Z{jX?^VSDM*rUQO5fN62vxWK`cl;Y?&Al{2P8$1E zf?P;iqkAEVXsfULNc>+SpkE)sD23J@C`io=ALDulGY=>fK{kSU*yQ1jRz&dmKqLYE zkS~nW(R||wE%R9Q`zP!IW*ck@e5q{pGBmqyv|YRjjI>#k@I;5CEVX0ob?sa?(QThq z@7Z#-t$}XPk2?i!6DM3H-s@;}P@T`1t2G(5%+YJBX*l15tenF!H@cb!yK5U3CNkx1 zQ6&yjp{c{it4L_ecC|y#5pR-5t~&-l*PO_J5^g?RWT1 zx=2|4tc#w6Ie|`J=G~Pyvy<+6w3N|ZT*6F&JKa(1UtXAvbTF4&=~3~^vFij2e95B> zc=g=0l{*>9_5n^t{fOSZA)7}@8@w^H7Hc)@!6Ldodb+20u$Gl%(ZoE7)Y=XS$pV`?&{(cW}Y?W7kRW~)z? z&i^g&6Q}3KMd}=@?__#_jCp46gXROwAWA5sZ=b>~*d96=q)!ShBW`j7)ZP%4yB-a{ zGJV1@k=dk%Dqcb!%29eb^HEa^L3zbJH>7oi;>s-ww-s7BBAA>>{G1s;{-)0uwLCQgUEn_4;cwe7A0q57 z!)=0o?I>fPnpwKCNaJrO=NhY1g4mrBC`$YqNrcuKwnOeBHbonGS06ZXi|qOu|9G`L zGSnjr2u8e>@+d8b`6xS%a2?p3B$xEERm5#4*>ordmX;PkX$4+I>QSo+)n?xYT}-AzHx?d`tSI7#pmkDrrePn)G_I7wuW zFn5RjxZ1fq3;7Yi?EH~5^pur!#Y400q24t$HI{5*fUq-N9hBoh4EK2sVOtmXK#2C) zmywC>Usoiovxe`$bkZ`LuyR{bh#qtkGY`I5>D2}h!uy?X0u467aK24|{*~Gza73^k zSJjyT2(0bkRk{{DQ~1Yb#JdHWyLi8S=Q3%D^zo0pYZfUW}w`&7gcd z{3>WO-1$8=$`Y$S7K9IO)JhC2wQ6*wWdxAZK%hN(yh2S}poOWv!H^i)B6j#_M<%>9 z=J=>pf-m7LrJ@NHF7Hst=rZ-mcc?+C%n0<;fa%+jA+6#)MbG#G73F-PD#MAT9A1T6 z*(T+CZ@M8e)C^i{JpBmC2e?ntWzUNH!Q0+iX~P@J6i%JWAYe8w_LB&!)UZSrjd`qK zi1Z&Trz;ragVGgkX0J_UmE&d$G0s?toY|XPIpbIa{>c)dS5l-WnE2{wL$8>|JP%A| z4EVWsXc4!5gW%bjBkh9VN17hH#m>?wmiucMgg*&%mwPIcbn({1&8TCRy43ij_*vBq z3KU;tELG|P95cR;ElJ8mgdZ(zG0&AV7d&!ETe2>Mi!3c{ycERpsC5B11~$UUbON&v zywtnmw@jhlX;sw$u%Z}7CraotK=^2AYB{QMRdTEr8F$kz`tBCTx~|)<%L=bl>J;65 z2fWtDl~d~vDGQP%e7s3vl2l{mC=z&Io7yi4RIwj>?PdVbD`s5;Pa@m5n+>f*GRtQ%%lPL zxDK>IgW2~6Gep&IyO?dCC}O(H3zehdEz3g;-2n)LtLZ~v_%``a4b!#& z$UfftG}km}_3$cCm*t);yZ7761+^ZE4u!!ChvdW(JuNF&(2^{l-Q1_^;IU+y_znP) zk~wxLrRL3M{=sP$u|7!$raaZeN8J0y3mvuUl@9)KXk2xQDQ!ZRjieC#g#`sn@h%#) zav`Pc#oGpkIuE2YedQ9ZxZy+7FCAih)`2nEK(cmhoeU&d(p2E(H5Leb52IlBa{io+ zEmrL{U26#=T^%afArpCln|t-GrFHdDfDT_d#%Z=mzX08`FEfXyW>M=a$E-6?g9H}W z^QGm1SF#}*19JYu8f^uzEw)HHC zO_Fg0MtD^LZdLp3jm0}$<HpBy|G#oGe^WAllQjQZxf%99aPhx!GhEDn>TCY3O&k}?Um_7UcFw=> zG#AUih_C7D?{r} zJBArXtcIY()&V+3g87Uz<^k&zU4cOZX#RGTZ`oLs!vk}QfK)6>2X^bw+s9C#gBXR3 zyiCZ_8*OBHw*2J!z&u_jc9AwWLrM>L-9M9IqFc4&GW>x>#{yfi)#e1Tbn-g#W>XhCfoKv@wwi>wObjV45MU2(1M{Mz z{k**Lj%w~fXoTvtyJqj!-#lI)-u8mxeHT&RZtm|+9v)5{YO9Rh`SLx6)_Nfyq92=H zX1?Qfasu!ov?rngt$~N_$fWhY7zAa{6!R-mW6w}*JQ8!7nFL(AMJ@-+`VYP^w*#Kg#rxU9)4N*>N1N=S^90@AhZY}ou?7K zU)>WR>r&tS=M#X7#640<%^r<6?G5e zjQ2foo0-XPDu{yOM{gf|t;=e6PHd%whRpAF7M{#2OsvSO4~E|fv??WHy8+gu?-$QB z+H8ZV!k9S*!>^(-Qwsbig%WraCY=xOt^9?Lo?lY-ug0hQKds*Hz8#tSyp~-Fmr+p_ z>^Dh>sa)%Ro*1-E?hF%%^YQy)S$jb1X z6`oH4oB@B&d9Lb7C>MzOsSy?)J6x0CQxW8g`-_YJ!>k}7H8;|9SqS&oQg`uSX519D z%Y~`_EBm5(NB&OFo?AKU`gCM;%-ktI9#MnFSCmwBZ>K~{r6DUcLcPTS$Kr@Ab?^Ei z5JvA?HJnN?{ErjDy@c&byqWx%PIS6D`eRQYT+~qU!?9=I%)~l1<06u1V^rJ1jC*f? z(mi<7g5n@~(<2ZnPHJ0)ET#uzECD{fRAq8klnaUw5%p6sO*FzEOR;>GSA~A#X z?jRmQc(t;Za%}9x%egO4TpcBWe3M_W-eu0eWW>qT7ICR{|G4H5v!|rn_-T2x=Z>sU zT3*cAmP!tdaA?)GIyd)v4e8DDEp@1ujFfMMRCau{0D!ED|z z2GG;v!oTjXMPjiriM?bVRV_ICk}H;oU9)2zzO(xaUmY7VsKNRo|GpEa)P^Uz0ho=X zy+yH5*)4Vy3|&5!v?9J5rC9b(xtbp!n*(*PTpin5z>S9~+=G0mD@U&?GU;#Ub>Skn z#!A4oWZvFnMCCU!%El!EKH~L_K}7ZZ&Et(00+dNBvA6ISJVU6I2?eN5JH;Kl`G;*q z+YrBEQ(l;o4IPUs@BxdF2g`yO9a|`4JDqrNf6}xhHiDb1^i(}BZW719!ICjPg&wQl zR2e?zqwozsHA-swnvFs<8 zO9Sao2Ofjft5Tg_VlO!*fT6;kRmbgVnrETTVS&>ke0b`3VBfqnW-y zJWryY^gs@o;~Gss-{z(zwe@#XN+&6@8?{E4LT=q10L zdlE|B!u%99<8ZWa?((MuI-ZEeZ4%CDljtLB@CVkwO&`a~haKO`Lb$PBa=cP4pZ{Z` zh$qDgx!1Y;WuZp1iK zK>6w-Y|{FA0j#N|S~x&nyF4*S{QUrdn5O7Of3eb9X{i)?WkWT+94)$bb7ZU1P^ryD z1YlFdg=vKUKq8dMs{@vCc4(uqXrV@02Ho1AK7pb};PCvzyqH;Q31dDv=*ghboKZ^Z zdr_g(-N%*V^s!~Lj!B{C{7TMx(_AX0mSSj0ljFko(#pkOFW*zLJy`=i17wpIP}vKV zaGkI!`)HBG;+=Z#)xJD)^)NOolP(UNcR0)6HEzv;;US7!GoZv2rYG`XRxZ$%I6M%j zK07vR&-wdIHQe640~AK^w~5CbTA}?tYIh>&T@A9(m_91c9yip^7FmuS5MQMmwz@`N z@Mbsb?BDV1z2DUHAhpO_5TG_}jv=#>60Epq3CY65-jY`zHvv6p&}Wj-t=~2RKQhA6->MFIIV4$^TjQ_{0B@L$MZCMXMZ5$Z1-HZ&$%)3eC=?j0L{o#>9bvo6Pzh;bp<3q*bmZ(;nY5e7%nfs z+BVI*h793JBhA9wqsz?YhTuplz51iNd=KQ~;^a|U(SgZUFaMFepoFnKhouloL!I*3 zwq{+1@k&Ue5VxkKoXu{k+-K8zX6n6PxpQRM1e&L?$;#SC4nHjd&PJmDuEHsy!Q&m8 zu*P~iQZkbMsTfPPv1r|;+OllguK|pt2D_Wd(km}1Pf~F82*g9zxev*~T{CQh{DY-I zbby@9w^6yNG?g*$RkeD|%P`hUwNtKA3iQ2rQJ3;tviQ%aX=|XFKuUA*TroHGOB0ll z@lAR5ix*-tdsP>jV39vie*Gfxr{IHK!l1npm#XC&B}#|rEgrX9_ECh$(7O!#vR z-D4tl`|Nc1$D|76SPjdGR(5N0ae4HFqaJm5sGs_(g8(9<^&^y~BLm@VoUDZg+*t3H zscs2nA3X)U=8O2eTcL@8C>RpJ-c~c_K=tIex)3&9tDC!i69ameo!HG7 z%jY3{@v4VP9sQ}c9HVKeKDe#Y%xjNP(eD1!_vLVxej%2LK(a!a9kAuae-un5Z)9HL zg03NY)g5RreCQDqiY>a)J_&gi_WI+J3tXBDiT3@y=(?-DuQEgCdn)u7{T8QUYH|W= zPpgb>>Fang160%c39DR>fd_iT7~oMtB0x~=fnZivvC9C(-FvBywbU&Wpgk;O`D2-+uqibx7UNvUf< zQCdiu$^?}sqL(Ts)qYbx5u!-ni9MEFC`x6lAYK7C{fHPo-5nqhvo)X8}A4SflF4N3b>} z=9q80(!i1{Q{yD5RR>Mr45aY?=sT*yu}1alZE`P26shUP2^v>a2lh-X77H=#8;x`4 zNPZ7wgM98ggc&?R+2$cD4vSTxiLzmZibR$FQphT%tn${ z693gZUh-`kYrD)ooK~t##TA^LK5zV7h_YKG@x6go9BGo^zMk@}qJSPKwO1?^2}_t{ zPbmXh5zXtl2msZkYHzylbEEdbitlp8jV33OB`a%o0gAjd_9vH z4*v-m4yq-*r?1HHp(a{g&0Sc?@2{M zKbKlh1e~w*Wtt#EzFQyxjzo-|))BVkATz>DfnwBNOBM(nKXC5;l}lOLFC?^kZ{Ty3hbqodzc`I#Y0Kp+r`_w0IycQ_!6Sc5W#(sI9#VKQUT zswyq!D?q57RQIs|xRPL!ML>WvGX$o7p!T{7C4Gcrd1C*S!Zx-UG|u=btj=lleJV+epVY5Xu-NYHG0wM zj0ATWtw!(@R#ORTTaZlhlA822k`5cXX~_9IX>xKn-N2|b%&bEZZ|gi>W#?DaOE)T*l6S7(vrMF1Z^-fjtLrXmG{C3i8 zmweZlq|xn0C#Q6&V}uvgi%5uCT){Q)47r)CJMOkT{sUXZx5 zJA0Zaclwb@NHpiN>=cFe3++w<`6`J4YZ%o69!ia{L_>{IL{ zvg=v6*QIDPRFj{B2z15jVR2!~nvmrfUftjB3T$kRM#%!>F@HWXT&lh~cRWa~`J4iC zMh)Kp++rGGjt=?O_##0Lfr8at(cS9|$mCK>F-^IThUs!1FRt>=;!1bj&&%FeA8{w& zK%0em3My}o zH`E+}epHN09S%B>VMMwueTM|7>Ms{X1r)yqAF9c$k$IT@K1lK^_jO}(HExGC0+iS` zhOF2otu3tE_Zd3LV>WO%5IL$MeKV7y>%0H$5m^ajuc9OTISJkH@>&1*Gf&vT(cVbU z>c0r2zp_^U&l8*f4vhJmo%@@wV<7-A|DpeorxX4=Vdj6Ep6VY2#=mlQY=4~Pzv`&~ zY=5Q8Fw-;sedKKaN__nH;=g~NF7uC#{u5p#sly6`0XE13WX136^bRi+M&)PEYQHcm z{j8A>FF;5g%`R>ydVZq_B0Qf2%YlO#Z4W4c*+r3Lse#Na2v?p4i#Ok>T7V3s%rMN% zdXK0YK4}bWpodEa+GhDDMhLFBsU*&5us39qA56hFTfKQ`+Kg6O2&)#Htah9ojzfRQ zvwqSL(Cn+1?&~qAjF}jypQ=6iJ!ajxdHzvhDy`cdP^6LG7=C_;3jp!R)V2@ppN zbh5IJuBkeRu)`z0@`f&XwU3rAo7hT4FWBEr?ryiAp0;&*$f)&Im&m8(&%%0ber4>H zM$+JFrq#fZP(3cw^hC2}buMVltWt|uh(*+eTmXkL2n(w#*Ed|x#L>`ju^jcGX7+NF zcIG3Lm0gINvEm>lOSA%{muUln;VsQF5IM1Eenc_t@#e+U84gu~HNd+_T$JYVP+v5R z=B|761$=o#2PugM>TI1>xge=)gi|0aUwpWpr`|94HXD&nR;GH9-F z_?RO&{X!r_%}0D~zViMdjrzj{2=lAu@vhheCd;Jb%O;(b?Vu0a!54Ye_nR>; z(Q1JcQ#TZ8%KfJ?s*jfh#o1kxct-QrjC`((YxDf|>}!uiMs~^?HN1VMwWoEUD;;+5 zSN2zDaTRub^49|0QCxC9ffoVAqQkx8$8nc!D5zhibBWg90uVBWL(*@HUp zh4H!xQ9XFb4r%3z!7474(H6PLB5QL=d;K7(M5bH9crJV_bn!7=l;pj)6SV76NEL&- zN8fBxlg*rHC+((S5V@J+@`N1)Om~{cqpagCG_xmmJ0}w96&l4@JIdA8ztGYP=pQNl*$dGl6vtCPF(<2EUHy1Cxhxe~y{oYpOzVW24LIlq?5MaZWq!bZ`|1c=)aMGk5r2T!=Q6yEqml9?IUCp#e0p3lTzmJ&Gv_O9k8*tQDvY)-MG8&p#e|>Z0IR;3u z;p(W)U~V}5h6GPle!J*U(f$P$w-Ucf=D{fA^jQq{d{O$#!eYC+VsY3m5JLGi1{e{$ z)h;jsPSHNj*8~@JP9y?G@mv+K^p*Nz+PzD;C2_C}srKXo;|S`8MtOq6+%b8GZ97FT zm-toDod!66K~wQ$jv$-*Y5cStuj%xN|+d=au&IDg8ss9Nl3-=-{hP!Zp)~| z<60*KKys~;G1?sSsM-z|bCKKDOqs-G0Vz&0Nre}LCV`r>tEQ(Tb?M)4AVLhWzeAr} zJ-H}pZPqIA$3tX>k0sBuyF8u7;1E~ah>8Jyyosrg1`o(X@9^#{f7)#Cde`^$T&(Wg zK{DBAnks?X5MsLgA#fWxo=#m4?3R8s&g=csVtHFb#IJriW8qEGld%m|&a#bR1c2B= zwm4&wg9g8}jpSb{E^7R2`Gh8S$*na_c30Aezr}F+&p&%PuVCP@9)=R|T`>@B4z{_K z-@YJ-%rCXGU3R!M$!T%yP3fCz+?7n)*!GDiS+isYLq-%pmI;cwtSWkA?f{)*5=fAL zv1RL|aONeLA?w;Y;sL!Octvf4!mM|=8xSc#eMwUdWr)edcH5!|(YgJKQTi+6Il1dC z=pzN>#q4*~W2Dfb3L>&^+{M%@Oh;Cr)i;n~gjs`dZPe&{yeACm4sFEmKgK>oF*J$x8%h&O<>?erf zWi^JIDrpv{J0kN)GmnLQ;Sw)NEWsL741#yK)^xF`))87e8)5$XxTM0lNwf|n?`Z{o z_Plmu+&*;!0}K3zB=AToJD5W@b!eNa>spbYFtG!WsP?HsO5-hhivsj=u2tn7)2fLk z)~QZ$Ebb3ShoQlO9awuNzm2GX)tkZVoZ-k$4^fx8FPz~Do58*NSgPSiy(;~-4zZSr zE4T3$>xMKzx=yfUU*v9TJ+}=wb$+)k5q?>#uiG3xL5#t`(7;yBwNG*C8Esr{w7PCu zLtAU4d52%qPyf_5u4B?xO+;~9T0d0up8+}SBK*Mi=wE+drm|aH92$?>U}bx6ZmD6b zFj1L^+F)dRXLT?@pfh3{kK*o3{LHOcRjZ+8^yxb}SUL!nvedJFKQGg;dD44X_ zw9LlYZt8h(4{i!rZhmzFX9sb=jUReOa`$Tr{VXZMNqGQYB084-rDf323*fk{s1CYz zf%LKpE|WtuZWcE=QxA7pvTr{V*UrOpVc>@%=TFPPi#K;Y@66Kmb@cDCNg7>B_aZNM zG%M=LR2&C`a>z&s_muwDI2wWXs}Py8w$D5O9AeO@+50Dvs<%RS;;drMaXu3e%g{6< z1+kROXq8BsMSnD6nOXeBGGbk@>yKvP2Y>Zs0b~Q&l`t*!Y?b{AM=LPe7%s;~yrh|@ zlUr;aF#;C1#6!KsUeG8sPztF}_w6YE@LAZwH%?{4awH@vhkQme35OzTWJ(Odd1@Db zXZav=fqW27MIxl0!DW;|&3PaI073-J0{2M9vlyTU8WaZ7W?;MTmy-k=(!mVbhLg$f zm-Q+9x6fXP)_z#;+SOH|?bHOUG{}9xBE4x^Ud1FuQE3s(EGXiXR5RQS+bV-j1BA9} zO&U~16klE`-x|^KRUkZdGd@_(#VWgVA*vx)i9hVYK*^A>U@M6bMF#38zZg3C z66y{$eT4uD49ZzDR8rx>%gU@W$E$bo%5F>mDe}y^Q*9)Q=Z>`GC;==}ceg@Md(*H- z+(2J=|C^iYQJuk!+?*yQU#N{ITUjz@PhK>|fccO!k}(*&`Uykw;NQl^HJ^H~IbIypzJxJ?hGP(n5zxgb@C-`X;C%N? zTgqHDnl6E()qLGv=~Te7(2cw1=+RTnCCil1H!xwh) zvbbvv+|^*2k*t&%(`_ae-3-d#;nk|U8h~Pjv2>)> zOe|I&E{DGWqY_$JeOOS9rQRxY2U|UNL{Gxot9v^@UeSGkpeNwz{7>8nVEeyM zKl;y@n*X)Azkoln^(Rxq!}Avj^NFngNa=^*U*Mb1_t%JdkH z1Xd@s)IJs#g(b4ZK$9JLS3;!MKviT_%o3bQG2&kVkzhQz@>fEgaGfK#Cmkc$IQ4zi zRYAa(P4eIF)+nZC&J}^Zs*^%M`a%MXaVoy=vpA|)xUfEwn%GApnKEK-7%oIGmF#hzCg}A{Aa)~S;>`&w2Pbs}>v6W?EC0er$FKu7#M5 znKzoP%DEhoH_Y%Mq&N28z`y0QR?53dO04H*f3|~v`^?3+Y6IadpkAb!w_p_$gu_oA zwk(9YTtH*RCe{4myUXB|0I$XAJlToOs+BzFgTtyu#*?v5T`R5;oC;Z0zUcGv29YuJ z`frIE=D&1HjBG4__fRp=Gyb*wFKJB-^nZT*A6$Zs&$Qs(W~5IO+f}=7(zi}13~<8{ zbzPeT=GZ=cne;PYeva-*qYv-W$5>G{YmZiLCecE|;rL>GZk|O6oa+!ipgWge2}{c! z_!(cjI>T>3gK9e97)1%asy>UT@KZvnxh{@q`@5#PTn^E%Q!v28ySdIbH2#V!tsDa{h8H?bdv42aOK#ht4B(-89C!FZ8i!#}ILOFR6As5Xk)+t=WS& z7?9sKiF;;m^`wEbNuzGZ*l`$Q@MpPhF^2kGk{JwDY~#>kjE$Tc=s8v6v>!x=p)1%8 ze-}d>1(4}gP6qP)Ok>V%pKes4V1Y)pyo56)B)#kBcD;Ve-f6;T%}MIjDVvANb1zFG zMQLY*CW*I^L>_d z&G!-$%Vg>c#Vtdj$ASF`(PQZGbUNP$qb|N7>YKSG&nal=@R#fmJ)ju~%ri3r3gd)f zU~kNHqfx@xO|*f9ZwE!?N|H-5(+8Z(%oikztai+&%|Lxib#I)_MpfA4;t`yHU_GN? zatp_H*B}WjSwTxr13{TbBE{h*9;o!dKBTTb>i%XdWhMsPXQ}Oj4?%c0jc9|&q?KM=#~%jR6+-b=fHU8P}O&dk%?8@L%CF1LfpmtBR;I5mHgnP#?f^T7Om}&#?h^ z>YR2MRYwJCuhXU?q{UrnYfXyg?Z}YjItr4nM2gs=;hDU%qtqfD%A4U{%$Elr4+9fEf{2x0bSiknCPHnrcC5Yy4a#gy zly*XlUD9Ro4qk-Z+&wUpWew0i$h|hvQZZh7BpqvQo(f})>A{T<<}T$p-T;(}5(i4QQhLTGqa)MAo1#bkH}0I+55i83AAb@)JXozqOR3h725-tTKjm8UuGDak#R z|5Tn@%247VbDL-C-qIfs!!4Gzn*$Oh#}PYQBa)b=l!_JU^Cl;HeUmy)B=_nTIHC@n z@*|=RKlQ|fU&d5g^^RoBk|EI&Q{D@5F_94DdhqRf_B)#&M!x7ll4OG8diBR^n{DY* z4L@WM=d1mNz5{k!uT~aOlRP%76~Zbx-a}-Ikgmxv0T{Z=L=8hQIX!G32cTitSc}&2 zYvT}dzXRuUtBcI=6+hV4q#V|?Rz+BVVrfce!^>37gEw;J9_(}T0V^QbPF>(Yyw1hu z{r*m z*Gj)IUNaneoOGOmB)yQdUD2P>VLH@KXnVv9>P|v)V(9^*q@nE1yU%Y0vEI89N{u(T znJS>WfFo>qVVJG-O1&Wp)u^6`>L8GvJc~mRuDiakB1YItMm=*#m2;v@jk9 z3x)J^GQZyP=qQ8Q1kGa4NMfSJ+|n@a?g3?^k?+yidFaK8u`v&als<{rSQ|e}E<1aoX9~b^R7p5g9p1)? zikH#xuls4V6lnpp53wYCfDC?(AbVCs-A-|*>1K>`2^0zL)r?~6I})R+di7dHmLSqP zvjD9x;h(YQJ=W({$UjbOlTI#29TZty;l+w3_hXlQSz_Q?ykp=dIVYof+!sv3is{}h zC9i8h;fWA+)+^)NiQ>m_CHhn=CH7d+jpT;H`?N!{t9E_SWfXK|=Cq;Y(jOt!<>Q!8 z*UzXs>6nL;RH~QfaDtW{SBShvdcaHFi~YBg{690_j0e>aRAhyWUQ|_R!nQODBlPaz z=FUw+vhKyCr<7QvNo`tDwEUTB&gI%i)s767%3ddET~f;u{7e>fmku~zAt&1g|tUK@Ti%h+z*SSwFFZkw=qUop4VV!{8vuk(!ob0u*w*}HS%{r$~ z$Wrcvy=ISq^P+F1Qq)DLNDCjb&ESiIsq0gdLK3dle&nV7&=Km5{cRscv^@S6RH8&W z0{8wpm>AHWi=!Y`$7I$)SaVi|86rY5i>kiRS(?l$((wDcHM`};_2Tw=P+D1i%kDU> zJ5A^)x!(|uI!)pVInFkU^iNN&cKrdH(B#J*>@GFZk^AK;wOa2_`Q>|BgNT64M($i0^V~BNpnUKss_GM~K}Ru*f%DMs zjc7HrU=HOS=9jX*R-~6ZG!~(#_pA)`dajEA)s{Lxl5TMZHl(JPR(w+$y_cN}YbVZI z9xZMjTV=7g-sh|KZ85(3X?&-mDZRXcnxQ2?^O$+pQIFFoF9*3F)BT< zdZhpwfF(dDPaQ?qa2mh402;keSj&Bvf`QdCGe#4R62>fFrceKoVGzw} zoW7~6oVA-=4EHYgP<%rnrAC(oN8g+p%0U)tQp&_qV_n}0EID@j(_x_DD@VbmCIO=R zx6NGCqC3+@rUnuV)5QZ)%i4|-=xWbh7VYCKw6GbL@z#yDh4gbRktE&jz}07SA=4(J z#FK*L5U*zzVU-V8ijOM44~RbU>*KB}2430!iAq1G;s0`={GZx;{-Z=m|OfZ%_d zYT%#n;a_9}pU~mI>jplT{t^x_GXD!Y{kt3b-=R~6&-fK#GkXU|0)|h6&|g8l|J>?7 z045Xhe!p&tFx9zq3N z)fl7>4HI&HMG+LyOcVI&O(S_=DeNBIUgG>JP7UZuld}fBoxqHX_I{cfHMK_9oS<4-JF^UmrRF@2{To7-PotWY3mT1vbSZa3HvNO6K_AunT~3x`CvT-Mx6hKCr>)1an0o0S)ILul zY3T=wZq}jMQ7zdk{OlR7JwrubY|+ea475|db#x7CF409%7L~GRVM~^(Cqqg5w?g+* zJ6@?KiFFz6N}>3!0|G}@etBjnggbn=9oR({8w6&;stWoUf)E>j>_gB#jiF~qL4UR& z%C7c^W$%do;+0*Zqe$VMoed`gKq4kEAOct7kwNgI@c1JR6YI-0lPD9+LBj~*#Te8> zVuE%7>p=x_W%p!#gIx48t;b+eg92v=6>~k5-1C)}C{$-HJ;nwPwf7BeALR8H|K>1r zS00jiID>Xw!anykBZ)AQUVlNS;#PFF^1SxVCP_Jj)=V- z!Pn(ZVlkUtFPt;LD9(4whirxWm2prpXT^rWoY2s6BT?XMHghwgbHdoOnn`{soN6Mq7EWk2%%Gmex{u;W^(6M0gsa3+TIoBo=A)^}bO zA7Oj4JCH*uJ_x#Rhk9zZZabd#oDlnafze~3M|q;xZ{pW0)&nspp_Cj;rGT5@ow|Ox zJs4945$kou3Kh&;Vd3(HRE*pp%tzZm%p`akns5*y5V9Q6g#y)IUqXe*jlfTwF}G0f zg{9Q7k)a}jvm-mh^AGp=&1K3N^BldU8*#$6fC!$SIR~?WvBC4OX!|nAl}21CcyK)H z5o?qC-NOS_l9kCzwVxH`3c?)gxq#kN_rD0<$_~a36$X6+hTa)b%yz@%`#R7FKnUje zG0>yNuP-8?5sL_o6YHiTA4wtGHWd*$Q`E418Go;MPu)&20!OGOm`-(bfE*;hW}Qdc zl~yhIzPr`i)VW#MFs9BF%wk4N#j3dqREzweeoqpavF!?gvAbR%m`7rQa2zwGW-&G1 z^~J11mh~G%iUUiDr0wZp5Y+vOG5J+x-H%LWa#)U!wgG>l_ z)A{g;sY4rrGhcSxmBNlDV_9ky5MHN`nn7}oUp9amFwaaF5|9{uq)R3=?RCYl1Es+p z2s>(*O=s?;PUjxDR2CiyJZ}g+jcD%hC>GK@UCO283P0t=vS~lgxcuXPTd-Wtn04?& zbY>0XLqxn_cz47+8DqV`xk~9KXQUwI6kia^d5&hqB2v?slJz=n8b=85*}Z?H8xF4- zCK}Ru+go>}h&LvPzi;KrFQg$9rZ%x$zFr{AjNU2WYt?<$G5Ahjs<0WaRZ+5>8xor(tE7rBi&&`?bs0@ zXNjbV)DMYGMYI6763;9`c7xHoDI)n1;(>;Bz;GiZ?tvg0l{jTn%io>^M`Tc+3Pw1m z%fafXkrmI1OyJbgg9o(s>Q_l?#;F0h|1eCh^wiJ61Lvn33j4E8$F9$tv#_S=G`cJ& zy?d*#8v9w%yuUU9sb77*o4l?>rjMrxrV8Ium}ker(s#WOn^T;b*3iTRMd5exyh`Wc zu5U}_@C6h5Jt~TZCIkQB{VQG{xp}MWT~BE%DZb180gt5CP(i6mhtZ-&!TZOd%sSME zTdm=$h1^Se?S3C;-GmkSWGBxR+s(&0c9)IcZ<@6u8^7DPwrJQ^!L|^-Drj#32uJf8 zyZIhj#iu_BTd$Pj`< zcTp4=8{er!QInj-Bmqq>rMRQCw)Z%`fDy20jqG}g89A44@Lg5$Q6XsdH~jknsj6&} z{z%ByE_H-5_|I3a7G5x~@UJ3Ro&K zBYCLg4fp8{OtFeSmiRv5&${9bAw8|C^?9dScmLBC*45|Zl`@GNh!jI?3F*<)S2{a; z3Q3pW3g&Z$4O!zO;;FPPmRnHtiqKGp#?{KqV$^T_?`FdUi_-T z2&BE>$!fg7noH(@ZI?jQL+Iabze%Qi;WWNdVt)u^*aOD9bE5FW1fd)hgvP%A)!97w z_PSOu%pBQ~P=LR8wmI1>yI_9+C~x z9iGmeZw~)DiW5A)rwyb&LI}b8f~5_~X$iV;+r0xRtP4-?3>y`5&&G*HLi_d)_WhP0 zz~W|O1L+Ga5oKHtWezUCka9|g?Qx_423!^pB(AIu+~1cRW=8|2S?_K3+=w1_%qyRK z50tszGP8)&p@vLwSp8Iz<5x$(N=RPd1}ey^{ESDV+HC^2RZf8tzE>_+?pJMH`#AJ? zTgJh>U@%`r#(s&seedU;nNYEGfZ{t5-557ayc3cj9-j@oSTY!XufSer2@8|{J)x|S zl@-eL5FsQax3svVM>g_y!7P>RZa_FQ1to75Q^=4q*Z zzD}lj>94{Z=7sR|G&y1l^q=5z2UPWcia!cJt7|smUBB}?7;qGGo|m^cZA0AOunThx za0D+5jxO?0O~C~Nqc=w-kFR;OpT!l!?wJPx0~F*?>R6a*NO}ucL?4|El(QCR9pe_B zet$|o$RgY|_sD;nQSo-z!?~YhInGXTR{EVg!*R5MzlU4s;wFAdKQ0~M&XAU>L+;NQ zmbx!n!RNgbx%PM;mnW-+&Yd9rtya}+42{A>U?VHwsHbYjTKoTv^n>?&y{-ID>BkEZ zgGcxJBgnxZ9zzqa3yJjV^N*IArHLqBYwI(q>$A$wKgOfFCf1kxa?em53}QlRXV#TD zMHh_@Vuab&2?MDdB&5pOnktvDfBvEVAo{*@Fa}}}T&_JQVX@LIZmqZ~?->D5#9C%W zQvY-i+dEY*x++91s@9Gqp@xra+oZ%7nj5T2DHrTj9At2wOJi%IvXwp%YoSeYT2>9I ziWteO+F0Y}u!V3fR!zgS8?Ofq(a>G)Z7}{R1VIyBn%!HQedHFXGG0F2Y939(tQ=(x zQTOGYhvuRRf^VU9%)bcM*gqmgkO_ic&8&hqt!@%f`IV_;hv0L=w2?q36y=J}vsEPH zauV>Zi0!_h!8$u;ZUJ2&B!>o89eJb_f}2m;WJCChUQZV|ogm`Xo&?v{T)~H4W!})8Qtg}30zUb=okc1P++%Ak&Lf)(t zpi5%E>q3=&5!J{eN6dv z9yRcp_&I--``tF7ZZdA)>1*R4%_-sTil)~Ok9u4YCrZM0yc6GmdRnsjV<%DubG+x^677MC-zu(bT&zBgy++~{oDJ$gr2NspWBF*W4SyvX zxr3B*4W`$qfD_!2j7pSXou_JxW^VW=aHPrNY{O#ZVplA(v;#TpxAZngb;_2sp#kf~ z`_1#ff+JVaS14l@ohj9!+rTEme-EphzXH4qOkvi~{P|dJy|)~WrvM&T-}upxWL(g_ zVz8XmwKQ`;h_IjO=Fa*4_2Uj-r|hfUze90<7QOyIW{&*LMf{Vd_}|Aw{0VCRe@q16 z4~@aUIv5z}0e`90nb}zW>-8M)uXy;sdl;BMdxHMKL@+b6{0kHD=T`p#TwX}zw#NOV zA{V3uM1~+D{}BSC(i2!EENZ=5H034aHl9eSn+CR>S^eT<4Kw_wV7tt8p*IQcP+f6- z_98w4S!5U!B4ijIsCazjuUK&+`dNBVX%SHV`gN;sBaf^~hS8K;du@pHEj@L-L%}6f zV3AGu;r=Dq`+g;yMm_U+UjyyOR0LKC3uED5v0bBt)d=vodw`r^iP-W{2XbS|lk^bb z%K8X_B|?PmY{%b2SA#_weQ{lLz?H9tv#vm5&U#?*R(|GQ@x}WsWx^Hpf}lzbU!LWH zel7+%y1)z@yWYivSzH#6O~J-&;_GewX@3aXtT9{kcodJ(>FSt?$;QV=Iia%3-m=-k z&&tMs7qZboTW2L+k5KvHJTz`QMQZ1+$MpIbE}+E=rlQqRz)~JrSNV%}0wrZ1kE=W~ ziFSrdK^@H`OAmMALBQh%@seYeHk*vN=9ywoRtu19Q^T-1@?+VyB;F*|G=tl(+$A{_}e4%dF}p^XZ-6U`#(PZPj1kE5iU4WWMS@2>1Ffce ze<|5XKUs7z?g)cB7hUi~>2j}D9XRN+|0GPoCvvNaDn~pLVnGa{2X! zp4I1-YDmwS9*Em;!xURGwIvEK*;?>1eC^LWc1ccO17zagxDqKuTHsSXRg(rR%ku)aM0GB;Nny9T>d906$vJ z&-cT3`qy{Q*)hA6!i|NMF_7H7y@uLG#@}|{XM{CrZ^(Vvo zP2eX&X90dfG`98Xp&jVJ#_^bphWvO=ue9((4!3PUb6U^h8_N#ET26*rumD72d`$8h z75Ex^5NEv)X6FahYmntk2d|J_ZM)YfM3fstf}CY(a|Cr9!W@EqyD(&GDt-2FCHCO~ zt)jI)5*(XSh=>3-G&>O4-jmoOg0Sl$66O6{DsJ>FE?}P_>#vW?Vv@9?q$^9IKWaLO zZ?Yi!suqFCAc*&?e0Dy1I<|)nPOGE%O_Bj5;DdHPJ317z{Atp#`N8v&GRR7-hVq7W zvD5L~gM>P5(YR1#r{(e*`CMFfqp=E|2aX9z3nWqJh@^;GFdmnbpTlNuJj7~_V2Gkm zI6dZW`P6CnZgupJ>5guPp--K020;zB3$&D_GJ^=Poh+mhcV|4jQOA?Ikfz`!h1oFG z_`cec2Nr$L)|CRJr9$LgnSvDRI!Yq=Tp1Y>e+wc@s6J^f*f!gdHJ16jlrvPvAXpSl z!Az#}qG|ar{)$(_2FGwGUnTvZhL~MOpzngaJyx=ejZ4*M0J+q-I!x#>H&h`8>|C>4 zS>U(q+5=z)Dw#ZHEkmbvk2{AwvhZim`0VABiqL~`mZJuXX))faW^m^wi|G}XE9^=? zr}FhftjEuBlY0m2!Y_;a0P9g{$#*p-KIAnh#?gE8QaglTxiIrySh>)(zk7ML`-Z|r zEk$+ZPknK_RgKl!4t_OFqnXC*Sf<|9>gV`);JhV0jz`b-?%Pr$GJfvkLH>_fbM0Ud zpN;MJytnhh`ZA5f?o=wahbvW^p7q&vg(Y*-!bq4?+|$>#bd7ZE_~q1=mP41O&6^=H za0jM+ewHT5PnB`M@a{XBL9xFww_qe$$`n?UWb|8S&!@_miPF?uIOJ4;(zT4?u){fI z*c@eiA~QPh7HI9%?Tn^BpGnTdH!8QNsh#k88Kzx$y#uMR*^|B*SFI9<7AbUKmbe64-DOitk)ttt{;-6oDvft$_V^ z_5y>>@`!I#$m{tl0rY$iKfbzm?BT0!hO!)yKKYmX9>ElTfFA*0VLvD-#yqgD=0T!7 zE;F4HvMuJ?t$Oh7=q}Qkp2Ox~XH7Bbw@Nj?p(iut0_pFSpxCvKY?Hs4YK_jRTxiLp zwd0af5>Wx;qSId-_!?L(o9W_`k8TSgV}rDG_@+FU6)+#0AtcF*`l&3Be3dewyNxQPKcW8EO2 z(c8Jv8&-MdP0TV}oi@GQdlG7?m~XogSO`=<2H-P@w0}AU+DA5qO#hT6@TL@QE+E) zhIuov2i(FTKLefun_t`7L)%US^?{Op*DuG}jvstCJGV5i$^hqg9L_?1beFEfW2}G` zz*!LV*`a{@nHL3pRW;r@&rpn<{!%D*a#r-6@x(W}#4*S3_ei&bi4y0K5!!tyTv%Bz zl${?gqqUk)I8D6U{Slf?fBxo;u=Aa8+zMD4xw^g`mq&JaC0OQJ0g;U&O?PT3Tb?`X zd+Gnj-CIZ1wPfGI!4iTKG`M?khv4q+8a%i=1lQoM!Ciy9ySux)yS$y--B&_RbImzx)vBt&C+$xRJB+J`X?J65chob6nWoq=jJxjrC$s|>%NeZqOH(YCAywJCEnaMbuvU|F%zy{sar(ktWR3%ix9h$Qe zMEGg&?ZQVJl(j@eOAIRos>nx;+QmY<;p9Gl2uv`OE7zH{*uheb0M!+h#?tjo>MeH; z_#mF%hi{D|%d>jE?OvRkk%&$dCwasJh?}D{ffuJVHXtzbs)mCzrc>IgE&(mGvIJ>py>O9@M|3 zj((*Z#Nzugxa0A37BGVmPHdi~iK|ysc6?koC#lL0y+hXldyj?56)=dpqdBP!CBB)_ zU|Y;age@pc6|c^qDd6-tLbBX#r^<={nxg|Uq5qEQiY9`I)xouW>E!b$nf;c1uwu|JXj)CD^Am@ddiLhX`u3}$Ci zCmYi{w;yI&qIY^cP}_4UyaS7I`xRdxiq<8jGh;0cIL64`)+cv=VD*&L^F%z$!fa&8 zrPrrhcK$epSrSm?qTyIjvQYJ)BVmFX)>JsNP+y2bd358BpP)%E?MNoe*0>S8ey)Pu zeqEnjQFo$HIi$+E^gE)~77?b-e9a(@-i4Q0^;3@>ceD#!vS+gG6lU zew;HL$K(MJ;>85rY6bShc_@PvA2f|;xk^WWGy#12yWO9}N)&Ndn5oOWrmKEd zR{=%FIq*d&ad4AhiooyEwGecMbBi~z^ovU2wx&IQAc8D>(ZSp4BQpC0W|r%ToAm@K zOXt_Px60}!R=4ey8NC+=c$fQ~N5Fl&J1YeBX7Dv=y@hH!Bbto<`h#YG_+ z$JE_80>lso)#xZce)7H%Pfp%Y-Ie|&q!=35+#2X?clZ5~jCF6Y^jxZK3w$a{&!IfE zGZaTyYNw#mP%HZYs?|B8C|;u{_8QHFUG-d>%_aWnpg408M!@ql5JMS8ZhYG%pL^p| z*#hTpUdp&;u0@;BQbV}FY$UT0ZV!$nA)7hrl6Mo1mJ|kxxlF|#wyaE^irxNcP}k>= zfSod*p(rxgSS<4@$CEsQ^6cohx_nQmls_BoAa^KLq7)ic6YMqLwe=pw!HYj`AHr~( zYiDT$<_!zHtXAnnNO-)9p<@RxHZ0lA9?ae_Ro3)FNxs)^aD64{yq%b4H3DRvWAs8m zJCA_-c3rv&R;BW=R@3=G#^hW3K*-bc__Q&R(39yV{hk!iGSe-_4(xpyDyk@YQAC6dL(m42#~IT^1w!8K?*8`nK1a<~f{u86spAA`c*11TitD3+s}VU$;{)5=E`vR~tmR)=J-U9XHNE1Pb}#dah8C6%$8ioiv03M_ zndisDxgDGDO_EPGq`DW&@dxYw(UNRO_0{vsklvPAdQPrf>_*bbc;cw&a5!0`pZ`XJv%5GdHXMR&Q=CAqUO3r22eE0Ci-^9c=XJ` zP!cgPF*3Hp`*q|u;LDlW8Q9>_2$|{H83-8Y0lq&CpNXBVjDZcmrMZ=*g@J_~9*{o7 zZ)s*}BX6avXMjf|VBlb)XCNochesnXscUPBM@Pr{YY&7R-tSpP|9D)vUps96SzyL# z;W+hK9$ERO=hBkOR7X$?xjpqg-a2vH8ioLJtjPQKAo|XW9coB3OvFN2?PS$0$bk*2 zk}_XRWWRkG%NM9@Aovoa39?Nr#*+cg+ogy|0DjNaU1E1#+h~-ci>>3BHVUulXmNXW zwZu01dDrgt%K5lj`CDFIUTv+TlarH`Re4$2X%w;e3SQSh+tAbVGd~}nj=h$aR+%KM z&a1AjE^Fa%GK=Ty^KGrgvdi7sIvG9^pS1KS@S?c5_~zy&Aprp>7#OKCoVB$z7Z(>A zwZ@t>KS}D#%S$Y!a?9gMV{|5juP8JsRo}qC20U*rvY($HFE4Lk7Kt1j5eTop!v@c(_5HK+tJzqV-*b*PsC&9On`?j;Pve*`O z7#JC^qD9NSs?Kp=?|OlHO#Q1XQ&S46s{OiHxFS_0?l&g=QN+?|oa-{l;XoV9B%&fB z@XR^&dO|rIPx@AIP*AStjnb{rd);ry2?#n|Y{m|+fi__6THghhmQtg0$92$DRadVQ z$JyYH&(7|yt({$ta*LmSc05^Hs<(aj_U*#r;y!uLI})t9l2~Lfh3&2&B&;p#v$J)6 z5uOHz`T6;vcoi{BeSQ5tY+(-%F5fq7Dcqhorlw!N?hGYf-ks}(W#?dVdo1bDs)!=C zx3|N>!eZ&0l7V8e*${pHjQt&+>+X%}dH}A+-MBFB_t4O`Yi_4AAR2HnFhmoYkh54U zm8TH^UFPcI!rF)GC-JOjXJ^L)suCQ}{c=aB^@Ut3Sj1$x!T#~)m|ySyVmkmH_vO0G z)7;WBJSRFjS}Lz|Evr3xC2nGQ85;pXhgc9#bRAM%z`^0BY-sJ&^Ob{HtIleDdoX_d z?Ck8@W1Z{m$=cdl2pUaxGcO4mjn-DaTJUw0skXNEN;eA^uB@_hfWVJev?(r7)y}K^ zNqi8<_T}YN{@cR6!mrqKx-1H)x|00r)Xb*|2u@WKEo7y-^;x=?YChK2?$IAXE2As~TY zQE@s`Fr+v;kY&Y5Z!%XBOJAIq{pjdOLsQdS4Uv$L5a`nuy*4y?#4=xLuKO+*Pm+nl zYnxf=q4kLzadehDNas8?MB3}BD#js05i&R&&N|y&DT<9)9a{9J=H?Q|`3J|=$5gjc z!rcAC!^wR+pkF&XJ9AMY***ss|R8$faPfhvOs+`Wh zz*XO)|J3yp^6!bz;_`gKqG+2CfOjSb`Vf+rbZr!I)#LG#aI%DTWo6~Y<|gS5%?Dr{ z1SiM6Pn6SW^}v?D6?my&Bp~q4#FLNi>FMd~6VbOqdO8Df3Jwm20%nce(P}I1O8e@n zr(9ZJg)xjZ3mks*U^J<8W>(hb#s+%mV?BIUBUg*i_}#n?M!MyiGIOtI6O! zzP{ek!6#S(#V43ADk(WUG9r{!Tuk-p(?~U0y$OtLg-8ixzLtxgoLm%j#jKOwvcwJ}0x;=E{J~Rhdlo;(AiK7sSLMgDZUh z{{0G3!Iuu#OQPLP@Q1v_QGyAvCRtMt06VBOTA|ElW%c#*zTpUY*}xEIA|%tzAsCQD z&tq4NcWJrX2ywpL;YHrK+Z|2!yg5?6k6&nXEQ^jts)TU^65BEe@*(}6F`+SuOnzvp zsWsT{j^v2Pi`|qtfbyg->=+z8TxoWlDD;CyBjEGNl}h9M_~Ckhm({po45~^6`^;^ObIDFB4yuBwTCVogFg-K6B&sCcKAfQt8 zWfzK~{L&kSeSCauU|^6WCMqr6`}P2gHclHFn7>qVzEz90s~)aBp&#Yef5uZ}W@igr zzPY%(EPzbr8`=ei^NxVz0jJtZ5Ua45njm-O8@=2xs!m@{Ca_PEG+;lKDKi@yfYTQD zAro(o=6%qI=F)(0Hl`{gV|BGB`v6G+s?qE+-PuXV#->^}i_2~Y=_Qtj8mJpe-m$XM zQc@zBV(kY32M1TU%|K5dm7E;n3yk^t2|u<;xWTS{a>*>ga$g+8FR5&{J=4?IYwbRe z4H2p|D&=c!USI%0;E{4aocVt$&ki9f^oQ;r-_;;uTf!j6B}AFWp9z1Un9uz&9B0KV14B zA_r8&gLQOGzW{R)bxWKb`pe*+9eek2{N8#wg1S)IthfzNG8ms377#Z@5shtmAR{jz z;kuE%UNZ;U=Z3+B*Ds^rH9m&e@PI4CPCCn%Yk zQVLiD1MlL(W=;aQO;}i1TRT19iI~OS-de62N}w?}w?`$2vOp{~H*5;Fp1|RF;rem) zRqLwtcE$C=+i|S&3Q<&4lxjyxFcfU4|H8$u=POB4K_JkV14fVI72etap+GWd9djxG ze!mXJB;DEBnZalTpqWNSyZK6VjuhU(v`D_&9U=(P$@y_k0Mn8s+{R!s$2{ptsXXl& zP!I6;PdY;^QYwi8f1H?@29A`53<(SkRZys8UMKXmUYMUR7*wY$W?^ALI|f+i@bDI@ zI>4iDeBQw$yPnXm5#~c)x`We0T%veM=|*H?lq~=Ui^M7c-BCey zw6e5h)*&1yFE>6uo`e`vZbUxtA?|5fS^IdUS=}}u^M%8K>>3CtUcmsu2v&OPf%p@_ zRskIT*VMJ!m+OC-rhZfJY-mXGFg{ml5b_=u0Iv1zZTr)e&nD9ye$a>+yz->dX)y2K zw=V#kFzaa|pooRtd4A5V1_AZD(Z*OlP7N1k!%Qh<@=5oA8Ime~o!zS!UsnF{UyScfktfZIB z$#Q`sX~+o^%J zAX#khjc5IeE`ooCH$o-h9}@+Bb!5c_kgl$e zH8srEJ=62^&Q?}3pVEPF2LLV*REASIls%Y+2?PTbN7K1kN+9)TsHv&P48MLYd?J8G z;T22(JIbr8L;pT7Fc1?H1JGM(Y3Wp{0Fj>MWp@B%E%c_QrqDx@A_O>CS$`bw?UBI; zk1=p^YA7q$0xSjK)d>j+X=ylH5QvC~08L?IV~g--)ir=qRaI?$KDK;*dLlwG_TFrA zy)D(hfm{zYC%53mSMUY>7anC~WRNhA0AcI+81{caqz53PU0+{A28{W4^3fV@Xp!&< zBK+G$Umv_CPyIfUdpk-3LPgx~ub}W3oC1CnAu2$j_lexwuO#R@C0~qVe4HI>SKRo% zT`&jjc&9FYO{i8`{MI+-cfND1fToNh7~A=VK(LL767pFw8*N2JVlD{SkQ>|cjc#i<7k++zS=q=YBPXZ2@^S=m z3{Di?B;XT1Ix=GfdH{&1ACQpr^z^cTP~O=I`_?Bsd~|;@Pq;dGB{-842ou2={GiV2 zTB~9!>FMcZswa8b*+hhd{LQ%SV`F2Rz2T>;t)7oJ=6gj|Rab!Qd%c>JKBqQG{>ADn zivi~oPjxyz*x1t2(cZqKxY*3x+>dMo7$RC)aN_wTB|!h~9UjUL6+sOu7t2)vLq1fo zK5Agp1jxK}x;aV=2^l;rYYoHU;NYkN0v4dTSDbfKx3;$I4`;Bs+*B5RDb@n{Jb`Nm z*<SmYYoGhI7BDxVVamL4i^fbTPYwqh#D5CfE-vgR^&cQ4bsyYQAwn zn4r>XQc*Cow6FkhvWvW6Z_jubQcR@)+2FJL1F|XZXBYhiVYl;cG+=7_kae8{lk)Xw z{^tcvwd1K4DJiM`$2;SF9cHsR9zhvIs7Dr5W|0Y=#;PhifO?C^Q^y`@AA?3D8yOmA z;=&4$leh^83i<)NtD0SJvlXGys0DoAN~Ur;e^6zZq1JBEDE3A@qb47cCuAP%>&yQO z_bV=a0!Fr$w)Q9hfV9GhHefQW&{d~P^;rp{({FK6Q2fff{=h#-kEEm|orw(zL|}xJ z*VeKa)=pRlWAlXqmWYXo$yo0D-n@MXvl%8C#7&w10}bploGsOyJiOk(L;Ts_?8ANr zX4(3D6Z(eaY(+vIqgnYZNUg$ju1PJ|rQ4+*ZTHe;)&Rgv! zBqRg~fXK676QY6EXhF9?x%qO00|<@`!iYvADc1J(p8+Gon_7Pc@P$~d*4hA;9U;Hk zU=J-E!hd{lz-M^8u>p#bmzz64(<_5mQw*shXl+f0(I+G?Cx_q)_#Fp!>;m*BCnrCR zMjbG-eueodU!LXmc935rz*)xkm(pu!V4u^qva*tpkdT&!mjGlm0VgLX;51AikQNWE z55!X5AsfplP|{miesm%y-{^}Zl#-GnutECL^==(6{*(I7<|c#b4AmW6zSHf=GJw*t zLpbee72&8XMKXxY-gRMSB(!GtgTSQu3l)Twdvjm7u&{8vSZldhGyD|J<$7bLq!h=h zBf?`nXC=YPyq@-hp4D6S-=G)xmOd7_Jx}8Uf#(2mqMeH1e}TR_0QwUDhQ7!Sf1vLo z&3{B+C_W-tD}-FQCuMT{3(we#elXuNRjhv#^<{B@hX=$zL`w5mv&CzPu<7;XQ8)~1yvmjc zZ|5-(fVu`iRi$#ErNiNZ{Pa9+z9m9|frA?XkQC5qS$f1u-7kI|V|LOtsyXq&w3EU* zHH_Mbd&tZr!SY4e0dpUOOr<`2A_ryjHss5ga{@vTCW{3!BJxbf9T*tcs@vu8{7<7p zJ9Yq#@#6q)D~kAwd*z7SSW@-JW_vl9nCt@x5l^j2p<0}~?RmEzu%}&Xfqf*;uoShz zL`#cT*x7o;3P9RoL39eI( zwb<^45fsW&ZoE7{0>0ng>zWVz%hN7*a$uQIuheME%flh9ndEd`eSOd{;4-g+zMKVc zSeQ?5N#KcXvioi z1>vY4d*QjCXxSYOrU9K~;;n%Q_=gC7v>s;PnzRO53Lq&uuu8R>gUQ+1*gob4LS`2f z6v#u<-F~G2vqnRs0;1_(+S{9N4+Y| zF4q9Dk0cBnUbB;vlZy~04AQFj(hUq0z+?vejjn!>Bi!_`PJ|Po-`FppY7UN%a{yE2 z(rXCf>&>r)fNTJ5n*sk+1niwWphMKvAHzMLz>3aF4`<6d zIq-0zZ_*PISj`uz08+QRv$J8~?93L1mB@Mtd=_$DZWbhf8E_H5;ZjA{MQpOq7^xpu zZr;VPfrUJA05peS?S3cO5{J|2$I5%9>Dk$k@5BgEJ z!(uNb2LAT#M%%{5hExjcIKWgVCp%wBquztvH~>5d=*8PrPh1B`(-aN1uAUwtVd25S z6hwN}HaOky<^7`LpiR@>5YRWLtE0Hy668A3R7^B9QmVJt4i67EH#^5`0s*>^#%?dRa(Oh* z0BBg*^lo$j-GE3h&xxO|0&a$aYN{yc>*trs?U}}JjtHn4tKtis;f6NuXHNkpU8?O= z)aiOWz!Un`t;UoA_x}Alt9lnezn7PnlarIjSDKuE1`(XL())a5Ftif+)S*M`D{xdKq&D zWP$vPoaUG0g!dcMfT}x8HU9ElJOusY$99|Jg=+O0Gu}D*}ehCYzlB@SDxr9{0qjd+6`VRf@PKKt!QZ z>+VxHooxUgOWX6=US2+Wi*Uv5QgT(BN1Y556Ho~+J1njDM?Xiqo~3N=dB_10X1g=Q z%h)Z;gd3W6J@Iv)JDXCubW0{Y+{+6j|BbjanI8_-cw%()ZbV&KcgD3?CeESzad1v|(h>x+p6J3>-o zkMykW>E-o-N$*33iR;ShB-swak|Bsf^KaA-&ekmNc|V9Pk>Va47zpqRR*$E<@0cv; zVv#{xK(GL0G0>`z@bCa^sRyK?E$Mo|=b%G}0RaJV1Grduo<_$CjNV!Qfv<|sE11=D zRlwgaM*r)`3gE!mKdnXoQ3&rJIU)b|YteXrv+yizqwD;)Yt#}37Dm7tH?SK0>+$|= z@p-AjVnGJE$GQxLOcjYdF@nAtzHKI|eCJaV1&u7Zb1+X4jBG43ztG!khcJ2~FurUV zGB62|iuY^!Z&tsIPG1|~pRm22z8{nBx%s>s{rNO~{&UMgxT}zii^t=oSFM=M_5M(w z_!i6H2_&g4A(QU#xK7qjT5ksOI=rm_AqOF97%;-m(RRKeB2C_+%`C|L_mCtKu5yf_ z{*1?v#M*v*<(7*1LKMiE!HOB?c3bU*Ym~BQB~Y5gP?|*@DnT7{g=Yb#x{0S^$kZiJ z!tyAPKM9LEZY2CV?0R1=?&SUWK5t{4Gs}FV9nHa}D5BXwM9td=^KZG!66iJ9x>0BC zw3;+|TJVf>OTpU0&Kpy#rdin(_my(P zVm}wH91k*@5x`8@r~KDb{c-o7{@1~dGB;+B2`h!2@&)Nncb;Ux;RLWdVI8cA=KJ(! zz@-r(*;^S(R<$R#Wd7;be{b^duNBLC!D6qz&71N4<7XYOC(8Q* z1TZ^ANz}HBPuHv>F^TWrw-%l%(~~hjiLA+xUtkJ4`i96H*?nrVV(fz_Xy@-Sj&S6i zLx;kxfn&i&X5TOxNwxEx^mFp-7RBJRDAfU>8_gk`QasWf{VFXc91x1nQ^u@J*KhM|XqPlp@rZFcgZ~r`P+aNLS7(g>M#Idc*Vt zI>?3rp+5F~(spc({A27p6U>HQrJOFUxu{m;YK`!}MVWuqUZDqSs{R2pe_MNnj`i2g z|3xlTiE4k?lF*0dOG+nZ89RY3I7SS-5Gp;lNeM76blXbdw=lJ^w}P$QH1vu3m2ovh z>T($r(e(m=^(_0G{`uComGn6{E&yE~@>#mf-Mypv(nmTfJDwH&uw=IP8s9veV@U4h z@kX4b`D#_G&Im)}^}@Q9sKHCZ#vP+6(D!BgI;fa6ijDJ@ru7Swd9r|G^V8Dnod6$5 zdZKa0uGC%-*(Y?BOlYK3Sda8UZ^lyFAmg4nDTB~mXAKYT+t*|5mQ{fx1pym=sFhqd z^3s%eDp}df<9Grk)O?BO$40G(_}a9$kPrJohNt&b8qD^q`x7iwk9N{|q#YB-PJ%~o z;&J=BZRU~WAsH9iBnTy|Xg?b2fu*|tFzFf-!dT|x`rhELCla&dN8{jjQi^AQrbNsv z@e`Sp5VVsuLF8?#w-nl1*IRDGZ*gxNuI`x2ztJo*6w->)+!Szr4)3P=`T_e8G(oJK zB3UB6DCl6xfI7E>y$9!qH=UVV@U}mDztQvYdC=i-(J{Uhu7VO-N9910-~d(sM%#j> z-m%%c&3Btv=N9J7Ay$D%T!<+!@VuJN3+6v6UEdV?aBquK6jMZ*M11?OnS&QGPB7hq z_mdzV1wuE3*b#yNR(68-#h*+O%_eM>@cmiJZ4eupG5II7bi_-%~W{b&5;^KBLl?qMuRDWx( zLDtA?|LqO3@i6{FD!VSe&jgbSV&EW$zPn$>F25=m5Dn#B-p?Q4E_1w`7uT-s6Zti& z;5#f4uFRfqi-j_O4SPSbZVI`~Y0$V)uZ@e-#Ts7Zzjh<4$Jd-!|C5zsz4PnoMt=8V zdLw_^@EIIJ6my5@PMh|YcV27AUO4doWxQ}aHuL35Jl zUDzb2Z$>$X;4bt@SIxBY^8D}1MIr9- z=d+DtPD{gdL$-4$yMuxtB|P4Q{|FYn+wS|?9tGj~sQr4LT*NN?;H19Ax(m{Z%^Fmv zv5UY+yqAK1J(={zInOG$^b@K~%(S2RJ#}2Usnpc!OV3>1ku1W4?;La*BpOGttj_b2 zkS;gth`sbJ*#?^1@jY+X=t4SAWVR2!5YdSdFdow zxP0`3Bqwmo^#hK?nr|^|!3>mbET4tX99nz5g@a-Fh@y|e*$C}En-iSwREh*DDSJ3) z+D^*pCTQ;_Vp~c-UoBpAa9nTEfZ`Ho9!O>BIED^1oYd8m+%9-gW^qMd23HVqL|0A< z+rFw5a$Ww<_+uO}?y5~TwDl6Ix?L;FguD;6J3Q?I!5A_ITOsXk0&9LA4^0PDND?Tp z-}5cez@0-H^qU*UI07tPiA%&}zfQ@~u#$!yI{AE@s8L z)I3~m5L|>wn-yI;WYYO3{v}&nfi0}uN9)9~>S3kjKvLW!3JEyH1l%aJpZ-7bSny$B zP-$Yd9f=k-z=GC<_=-3C!&Do|CVRN`(=@_scJr|ptt|$`5WX$9z4I>$P?hBmQ)8~e zJT!5)U1EW#k!N$=j8q>;B7mOgXM60ZK7uu}jNm{~L6~nvZC|jc*(;fLGT8SDVUy&s znHFe8+Cdo~=sg%mS$A-6N^n(U*1}WP`th)dV*@Et?Kr2(vg%uo0zc~xES0;fi@+=i zW4CrhOkeqDHf$l2=}IZ42&}PxlVK1}skKL&dg&j=gsEPy_kiPDA&jB$z&^rwGb51w zegQtJsU|B^@w-aBvquY*kfLO4M8B78V2O9Lkz8+ zY#&s=zkl~)mUMEa7=;5(4xR&kNNA{&rAa-DE>?h_#cT-gHxu_x!F%yIb5|MLTrjD& z3;*oIs9I|&S~@KKf@%ASrN%deB~ItF4&y@j1K=&WF!P$gT2pl!GdwUMrPggyvC&}GrnvG}qLH0fG!@()F9 z?&;nB@LejXl79D2_S?j;WG|um*P0y&mgyBZtcjpm&WX|?_Qi?fMvlcJ&Jlts@r{lJ zhbo3DVBa{NXUzJ`{U_nuI;X-{{Os53%Z%4EPu9tBk-y_6{}&Yn{ypuTM(Cdz@&76R zo$nucy!@I_H2+jpoSaZJqI7t_TV za%KiRRz^B%R#r9^7Chih_-!n${*O2DnHd=9{~zxXwz0Ig`qy!hzkNd^PorR?Yhi1p zYhz%c=S;&d|3}X7uSev!w6Fv6ifz?_hyUM-4$!gwZ$$^_SXqFU{I)5a^^YQabgciW z=l}!5?>Y0df6kfz?XCU`o;V!K7KX|TBXk*A=Oev#-Xf9}#0r8KQr8|B`l$CRK-33L z7^zUdpxx@wctoL71MZy&Mde^&oI1aoQPTOb&IT4lR-K>sV4Ys(oC%B3;QO0REy8!T zoHSM*Z!kG;NET3Evm5=; zaG;h#eF;3Q7(#@m;p)Y&cuL{D$N!l7;Gqj5^caLyao@=ScldI40XmAizDP1aYMH)v ze;HqPwz<2Anf)#<`dQ1})5YEWp=hBZaA)=9T|GHeAB=341O1j&}5DG z=3B}QLy3%ex3Sdp!|}aUlt?`qkiC+zH~o6AvaPLez+DRe&U2ur|Ci@L&&J67hZow7IAuq z|7b_aN`cXtCefV|-Hklvk-LF*4}arDls9Ol3s%IzpW9*PLwltY!w=3r8Kngai7{UH zQD~w~2!3Yco*mDGDbS(QSfH`;yQ(d^&JJDE_o^Hv#`K`25 z#Ujck-D;U?R01k8>V8$1R3Cc!`#kd7k<^9?-QP5XWq^{ElsX?{s$HO#WbVSAm<8Sp zc*g3z8u-AAwHx=*NRW9fDtI&|63&G+MWdc!>(1$Umn$1_5=fR+J{Wfjln0Zde>Pt> zC9Qkj!a{6Lz0#JE!o%9vz}C9l(ALG zF%r73RYgey6)G(nYwUJjv6oSY*C8xksJ>A^-UsL#zc&6 z1RL$4w;MjF%6*^+N61SogXcOek;*%wX#I*9{Xt&l7(Tgq=7ZBLZ%x$<}aSAm@VEWlxuiW2Pi#ku3(Z@&>lWS+WbCo9Zq zVbQn{&z2j?W`GnNe3QS0jY=#k2O4^hcq*BypOy+EG@Z@-Xl8rO-?(;ZnAhjHAD?Hg zJEkUzzn%9DEOt%~0Sy5}o-t^VIq^wqOExy?Bjt_7MfS!# z=9?gVVVfr^{t;jDBhyWe1*;L~lTs4E{t$@JFUtZjQWMku~2zbFsZX!;_=on6fPMR$%!{%zyOqO z@K8tGfMJpG726j0PlFGVb(Lu#!ff8;UKg0pivqdG*fTtepyLSST%hjhwRSS0z58g#OkPTy9bM+SU{e$KauRHSbRccW{~b52?>$AyJj0k%9Y$Da3_*L8qz~hk3`k! zNMzRs${XO#`yFp&dziyL0%gYf=f}9ia=z%Q$u+ruw58J}!knxQF_SP!3sr6l8^{rJ z%Ld0VVJX^j5zCG#zHb@a30BzE5_NvDKXX$tq_1f3&1>e#BNeb3Pm(CaDPOZ}&sMmN zLFlODJSpFD$4}EFE8gAwia49$nA?`g*YKm&P8y1r=&l5k1HXkN$knM@y0hpWvT5W% z<{U``-*RIIb-4D;C#EKFx(^q&60zRFo$!tj;yH)Hy-kq>YZig|OF}|_qE5&EI zw$^&QbCx4L=sogMk$k;XQC-Pm7fShqu~`lOmcuqj%@r4E!j$Z57Y6z~=ibHDnxU#P zhwnV^G>)WY&e>&lGEAY(MMvFh$W?O}P)4vgX;#HE)Mqm2Y_uAAG$TZu|`+DM75mJ=bn-+%#6s1(Uf>->Y zc6Tp`+N7vQ5W3ko4cqM%aK-)e7bk_t=vWPd;oG=@I=hc5ES;GBGB?xFNFq!kCm&@l z4w}FNu`Zx+(|XMqL`;8TXT#?kXubDiyrXWnHki^7x4E;YmNj3@8(*3wd&NKOM;(b0(^w7G>nl7P+!Sens$Il#g3N-u2EXIJi z{gUGrmCvI6v|`&1m6;-)5dHLS;8LoC`*b{G%4sD^B}cYunsYaeCRfHMr+MG!C|dk- ziihtIL=&75nrdgxRdSniR%(xCOD;#k+`#h=r9S&qQC$`0%kM;AC|F z!Tsj)GY2=%?bprQu~!}VQggSDpNmv#qo3G|wLjObQ!T_%J>Z@8+j}@{DxgFoVbrXm zk>^RV5Kl6*IF_Z8lV+s%EAP$XIGZQz4A8oE(C*}en?{or)nB{@V?!1(=!Ksus4HQO9Zz*x+7CfXKCs5MZZH*`j%OHD~Kew_npzSLLex-aKF{#lIAFmUr$x44sE#wFfTM zYl>iZ*PhQR@f@Qum-hOKgG{|zZH0%N$=vzb_wXeR(oULixG5&DOVA;9#Scp3ksCQp z-A|3JgcZkpWB0}dr7F92{(gdVNIJbxNWucsec6;*Rq#_dg`*lyb;I#=p~Hcax#FX; zDWCIa$=(s;mY0GCV-Zc61t!mlv5Ty2+{5FGPRA8zxyMnLj^hn(PvckdqSvbHoBGJB zZI|oirB`FmQr}yDxwl{Dto(aRys_Krr0FK8KYcD9MXvo$TmIQn4Yx5SwCVG>xKS8{ zUy9}Gb!;+1A;)Gc9ZIg0H%akK2MIo!1#w%#?Uh&g*T8@P{dz?o_Em!&Joq=Rtz+3~ zq+MQ3ixGP>dq_Gr^kvMmaH0}r*U2X!ksqjPqc_?)2GqH(#U9Cl69g8;=eC|}eDPb( zL_N+Q{d$rx42QR;ziz!0o`wb|$f&!F0E;V{+~U~4$%LoMvfTgX^1+H(=vS?u)zUq*!6mZCuW8J`;vSW?) z<-_w5^-;GjD`7yFb*RUer)513Y*sr1>QLK@^}qmTUz;mU(O5bAtdbs+yTmx=i`=?q z&Es@dQN<2cIgGD1d>E!o=tcrsh1mLN(}J#~V_DMAPc9%&OAodr&R>oadg(8=8tXsD zdQH-I)tZ34`pM1yXQ;s&bB2q>U@ z(%=US3|F&1--yD>{AeP0OGV%2b*YAghXwY5D5*E_hXj}q^*dVh`8Oc8I67-8U|36T zJUn)FynN_zWPwVgB)(I!B;Licxy)o)5}>il`R`&VXM$);*I0M@#r0@O=rp|^Fx=#9 zjgXR-^wiBjm-Gc+udl$vY0hfnjU2vft)Cy?;@;gkHdc#so8+f^p2Clg!gKe=v0BV^ zua4KNrPvjmbieyvbAV&rRY%#H#gS^IM7M@r62Z}HPO`?|W*wlEwUx$d|X$-4+K?L};CgQnR^BgV_Rg_+O>93_@Uc+EWe zb@3XRkN_HK(R#}Ol248lD8eE3^AQ_14|Hn=@PRs$@>g7T2bJPGl@Tcp50R?Dx3;d8 z^Mf8lBR6uZ9BbV8Y>^y)gPZifmyn=mWd38hl%D=?4U*q({x5oWT1_Qc`AwEr_$mrq=Yr15G00uRS1Gm~I#r%Zl3W)pkttrxLVq zujyyKrgSd2DSEALy7yr^Jz>m!c!^h86H4k$sg@TuuHTB>VA!qdAD$mDk(R>f!QZ1L z+88o=VRlwwteI^d5KIE4EqZ z*cRkDx%+R@kGqpoVY{iZ!q+-kCT*~_gr5f8aec22yu)cdZXxswIX%Bit6M`qF?V`J zdp{~Xy7JoYu@b?W&pLQ~(bG}?)#`ovB+nwZkWcH+Tio=0o=QK~6|o&BO{is6VJh*w zN2@%JFbO>kg&0RD%b_$vDN^!LYw)^?nXV?f7#Amk>+h{}{iD`SDM>mTh&x(OjCfHW zgc$nJT76gU2m3)6qWPhLDZ+_QLW6NtsIgyxX~|2)b1wl8?}ZK>ZH((RAluSf2+P0jGr49yr! z?`5!K@Z8#Y>%Z`ZqPk6ag@3q!B=9~m-zYzE_MR=pBTGUl4}s4l?h|;CKm%6}y2qIM zV08w={FGVPP)@Sa+nM|5bl3P;S#P=Nu6w}{bpK6iN@>-EvKc28Vo2RQQes?KRxOzC z2h09P`_P|3U9NJMa>SB|SwpJxO1amD;0(5mA-K&k6O&c4_wR}^BV?5g>Y6p`;LeIu zc!k=RKipxVdQ_t#c-cmXj<8hh@njt{80{qxf@i~lmwrDMYS(VZg+B|X)Ju%Os ztB2G*L~({c@nB)t?#*)JnnB?XwHE105hF*DJfqC^HY)q~i%`PHOUT&=mYke*#IdA1 zNeSUdR9Tls2!hAZ@^x|ic8GE;iWR=a&9>~oIDW<|F{g?#^U{HJQS>4S0R&4e!UgPa z1L+xqVFD}Clph?pBm@S&S)w6 ztz77=&CYrWR_72J=xBOGY*|mh&33@*PR@`EC_?5T|0knzmBC8IFG4w*1ixkw(ycX) zpyS)=07t*AMyR2fR(JGdYH?R0jbNIAMRDdN8Mdalvo^`-bh8eIc~?-aBj`rhSCYbq z2ucAtX>*{2>)_-Sla6)?li^1^0I*pWiy)?5~! zqFJ7Wexy@YVt&0wm|1VVUa2}!67Ds`=L8yO)XkU^0Fu89$P|0sylml zsqjy%hoBDVFqC7g(N$UgRP{IsZn`~(_3d(t61MYA2IfD*cAXq#b9BMkcAdGtmfO)r zpTDFw(O7V&d&fYcogCw^*?~)9>3;PhR6ZC*`Vws+$J&+$QjsxfW#H+tqGtWYMfEf5 z*Lm`6*?Y4#EX1~0K_se-Wz??}B}T?eUo|EaGa?b|%AUfF1`9zczVWR^AyR8Og^)?%J{>qvCK<0ZJca(?@OXex+<2}t`iM2EPdZbr8PYE)YhThaquORU_AM;B zRUte}HT&Co$#e4b&!fga$oDa1XNvZyT- zpo@jvOtCKdC`l90Ec(mT75^bw2Z|PzhF<%esqz^peTspZQO}iE(dfwoa8d zmFR-Z9|_h8 z0D>%x(?5+20zC093R$Gur}q#pj~;?_^_)KhGyAQ=9iM^n$;hTj7pzcK;2rEtr5o{- z4;I-3Wz)CM=UIv0{3sa)qS1*avfm=R`=oy#QXaxG0mBaIhJi|0kkqiBiwDHjPhM{l zQfHxsl)kBcOKH~>Zk5=_l@xtETFURNHo2_da5giATNQ+#%kMXq2qjW|)Opwhev{{% zqT*WV{H;!na*c*zsj?yE1NS_i4@7ByDXPxr+_!~X#*}fm&tUai$bp0#j;z+%ul{?P zEWnEqKCjX5;3l7SVgN9Gzne`jhd-!a%MkHyHX+RC)ziaJe{u!?6u5@Qu;`;}V~ToB zSh}$aUPv8^P2mQt3!9#>y^y;1sc@;v1Ra{zYD~~#uhfDJH$u&-RU8F?!RL~pG-m}Afvm)=Y)VSfRT8$ zJ0##NXlA|_ZeFnZ3B4}jfxU&G^CV>I4noe@UbIr{*5uaEbHY#;KDzo)$>dZD6NIxc zQOoN+F^h#K|BS8mru9JnFz*AHyb2@zcQGK-|2%j8@3w6JOquD${OY9Plj3FJgcuJh+c3>m9FX>wyj9<`Tg`b(SV*QJdAv7_#%AwGzR{O?D>mw-i zOB4DD2}!})X-I**euRNiu`2nyN6ihiicv9KzV0H1AYZr<#=+;2;k@zQ=d~>+cFY|Tv&phyh}2R?8M&IwfTN>7Nq#+v7bYL9A7lMwCdiU8)EX9d zOCu*c=jpQrzL=|)ZmJ^{TINzmKN)y#)#^v?!8EmyMF< zO>#Qm8f}mUw<3>hD8P{S6gdvm1u9SvX8Sli?2Ia?Z*oKYO5Lh=WFv&Xidp}Io|Xu6&i}e?Xo^(Bx@F&bYBDn7-CVWfuz6BWnmSvth|!7Hk4$| zP@+VK73BFw7~zY};r6=on~8_TfK`kxk+_VoBJ>b}_2`Q9k-C#}@k1zO*L2>h$kSMA6HjGZ zTEb`UgL6~RHeSGrvW*=|bRT_B$`iv+T!l`n9HYL5qOOZ#vP{=8(3EgVF3anENgFZC zX4N81jdAYhGR&7o3HS|9WYnUMxWt;4%EKEv%`f^=I^f*&V2K{-dCQRtW?W$K{Ct*u zc|EK|FFh4?t@EQ=iSgQJ*p`@k0%KD#ZkUU_2owkbUytmGP>TRzZ|mSyr|W@?mwO~g z12C>ELJE)aTToTi9(0R72jP3m6yfR*h!l0$tH-9Et# z8zhmI<;EFsdCoW-r3`gKuyen&9!Z;2maQ%3;(j3+G65pGfZljR2#LD)2FHh z&yo>_O?C1D<<+>+ZqOTaQ?=JtaufnGg>u(=3Cn=~O0(?bvB<*lK!l3Z(Z1@?gz~G? zd}hI|Yspm4l&SkV#LbTLP~nhtdEp`Ax@8KJkW3R@4|YM7T|9cZk_03W|HZ8nsvC*Y zkJz@UOa^rAyP`$})Ux!%mz$o@NtHEXTT$i9{SIeJp~cP|)X+!AgO{(+KM%Ph!7JKR zP0v5*5;Try_PK5}@Flhv#D|`W9XA0M>pdd(v(&I|h432PiD%*`H?fqI0C>xNEYDRi zOzmP~&M~rQ1{JJeI1G}J_dZDaW+@_iI=7@;-9m+;ob(2fXS;SHC$%$a%vj%RWPO|m zD8J3IlI42)Af{p^7Ec%rE1(N7DXbzAJ&L~JTlf;beh;3?R%ykJhVMR-%<=c6G)`TI zj?l%Lpk2+<-lo*j-$mVazgJyp8l0VNbMztBBzipN zCVdb$AuSdi=RIE8hvu5ZVSU1CPVhKYn3T5Z)CS30sl=XpsZtM-=oRMMFAC6`Cr-?R zMYS6yuEEP8*e?4IBMkA_ZVVHzZLW|LuKLFvsv`-y-}7v3iffihWCDKEEeEWlOeV5Bvc4Q}GddUnSj%GikwwYORAgi*yT)-X z5o>6Nye(;5A-+etd)}!d4eS*&r5Z+t0T8Nt#A}D9rRmWZl-Bk1d~2I`Lt~t8fsP}d z1_~_y8DeVm8hPHwUn7zaEMuKYp85oS8k%kDhr@RH*pU5=qe7>{V6q{YTcs$>LEABz z*Ng)zjDA^LjJvLdDVQ*yv5QDf2Owq+(){+&m$zT9xAEMb0Ab(eH$;l?T`<8;t1OA( z6wH=Q;;ElxaQ(N=Pu?Nyu2PWGL8p81u^I;+q~a4!J~9`lEtfx%Q@}V92Vy;>!4zpx;liCuhkK_io@MYF{R zc5_Iq_ir(!(BX#2(x8`h+6Q%1jH{!)VGsrFXnrS;$=4%PABp5VCbu3WpLnE&f z-AAl~;EU*eN6MrD2-QZ0Ok2zpE>JoneH`#;WU&&8-qLT0js;=1C=~hb4+cWwnh<`` z(053|Es}AJ+$fof;Mg_RA>vr^C~dWomC>5sdUfQaUdWw4oh7Vpa&T&5`K;U`Vye;Z zY@0A&t96@l6*)(tMr?Gp@~nQ3_?SV?@hUfisM@64Mm2Ydhf**3{W**Ds!MD zz&PuX@E1p(_8`~~!`%oaP5*;tfd~6+y{4UIj9VF5Duo4Pi3wMjAEAcPHo#-=%(0ae zfF72CHG)jGHB2n;5E9Ad>f>-i5;aT=oRUuRiW$Crxoe6ZobqqhZr=XtQ8+kH#M^AM ze3a^owata=$eKH5=T2|QxM3Ag>78AeSq{3ct{Xd#A}30ZQVIHD=9o{?SxE_;8y;#8_;$rdWABibR*kw$HEYI9J;sRxkmkHZdZ|XTozUT>QH9@; zNxcl8-4`pBrzN(nJzz(#&)4frtqia1mxgVmFtyWh`&I|#nc$3Vh-wc@$2hN8d-xuB_UvO)U-Qo~41pg9XH zvT#^n2KVLkiiw{76ESOO`YSl;Yf|y&Nvl%^DXvyn$9>I3h)S(AZ5Mv(y6U8E%?PA` zx=O~5-G`&>tf#MZ-D8N{3QI;u`96;YaLv)PU3W#E-7N6}Uy8sC@t;TdBe zb^62N-PTmSdIa{qOB#@|hq&$eeZ4lK#mn=qwZd^<9?xbrWGqAM?#{DrQ8Cif(7-2v z;V$43bm@PLi1qDYAgu8sz;*DvN?tj&DAMi&*BvtEdGOiAZ)w2jWWvIPzb%=*OF0zH2ZO`k zvPN>-!LK+TvQqThYjlRE)U{rI$5t8|&|#k@J&hA!^|4ar#$$sJy5S$wSLX_T1uN^x zsK0WpuAD>IH#tfemkwc>vk}p~OntlX+M`M>w`dI~M?XwhK1{7bv}&?NuBIm&+tKd!*tOvnW4ds^j5A#Wf)svQ2}O?b+c;~rL{Q2k zJ9l8%i_1l88`?VpizCnhfFS4K_moSZWaiVbFqotSCfH$Au1B9ul~GceO;Xj1uE+jO zj({XT#n@wy3OlN#3~s>jhiuJ3L^l2S?smUYh<;UWMEI7=m#4_xXWlQLRX9^djMsC| z`{0M1mmKPYDkk82SQ|$c9uBQtgZWT~guKOC!MuuHpluQ;Bj#S0YTBj9^U91-@H6S} z%ibxb_WjBt#z9I++PE0TwnM=yqkHKww7dptGToD=RI3ZO<2>74w~(3KkuIe9Y?%S+ zdVr#pmZb1-^Ecm|=wOjD)W$;)fwH$KJGy>@Lg0sO)-1$Ves5r-ehR<66NoEm zz~JxlKbF4=lYbG#75;RkR``P<{$)k|N2!s3{_A{|-#hi?I@L{+UevnYq&|IM^C08#`&!D~O2Es~G=q`X_z<|2~R5(?9;#7inM6 z#>Upk@e8A1`?KQzhhE^Hm0$M1#MfWb|84YN_TYbmQE>iM`DJALviSaI`u*pn{)6

q1x^8nl|&kf59vr!YDlAjv8&*PIqgrgdo5B?}7ZkPQaxkqyQ#mklP!%oQ@AkPRlJ zv4Dy|9or^`$-C{_mS;fBd+j`kD@RXv*TB-_F5t1>zEMxh^!&a1i|`l2T_%_V8x4E} zMST)t3KLIobZ8=^mELHd)rw3;k=BvJLL#@}Vw!mnm)2>y9X)Ru<${yhvsJWWu+ZQg zp9j}%cVJT$k~S(>TScytw(5uG252>WmYrIwA(lOa{!;fsrqce`Y*~fFAf)&$@RNRLL+c zt$Dl{#+gPDdpFg43&Nsq3Kan#@Q-8MlWuv&a(@1b7qBFFE|P>urmkkX*4~;^X$)Q0 z1t_jI5R?1Yd;B1TM#(XSg8Es@oL-Opte2OTmnSh3;FG1S?1i+e*z`*N zhPmP}ip~Fokx|vt++mQ4G|dD)(}B+Y&LAu!*leDvr=w1OPC87-B}I2Cv894GFRk#kvzaP&#Adoqnqf92!hVH*^9U{cm8u_4nYQKVl%gsSK+z^8gV}XniI|CLq z3rgaJc-CO(6mxQ{3kAXPka97cz*&Z#-Y1+7TKUC0W`~26GxwE-_`wFTZX&0S^UeLt ztT+q6+3Tbj166Qd0*Ez;DpG7jMDS?S*SJsVM?}|!^rU+#H+z}F!e}|-O(Bk>#cRJS zY-TzmWejDv)><2UM`38Yd^Z;43xoS-B&ijROL?bA5Zb?6%^)_ECUk%^dkMM_HSy5{20i*=zK4 z6#IGsqBBEwHTq@|h>mKA@Ygv5djsa53d^1hWagLOZeGijZeZ9zXBBYVDhcS6jTKYt z278R>E@GSCSm7ADZHVvW*Q&p5EwVtV4GD|YOT13Y6j$(^#(O(o>oRpi`R&C{n)=s>pXk~MpoG{LX zBwa)F^V6Hh#2XeIXbZ7ulNn@0VR#=5w5!qQXxS|+{4_DjgL-(-MXNjcqnq65`qoE9 zFr_)0k_69z-%juAomG(m)t+}I zpkQ@RKc(^{g(R=c%P@pDsgLYowPG~gxDCu;i|rW{Oeb+#>Qhy~i5zTM1lMSN&>8&X z^(ta8Z8eLEwjid*9F6GrPK~Nj7o2QsvE(V+|GwAHYet~I-ZcW|k;zggiFSTKhFdb& z7XWU>Jl$|l6n_$cf&jaES%x09q$d-(w$sgqis*qW``Ov^IZ41Sy-HsxtOkyW+i8Pv zJF*68O)q7eE*r?vHELuhzpy3a!gGNVOX`>@Glg~wyTm*j)GPvNa7H=yh$QPsmgJ`A z@B{AVt<$L*c4W0b;ZR~8$D!OldwgRFUg8myC_aM?5=GUk9W}I`3e8liQ|^0*$*EMa z`8R((ZyQG_#|4gX`$mtj@8Z<^!}4o$w=SX>uo*I&&NE%{Hy03b zu3FPc>8Y8`c*~P$902ot4?*{>qwPJa4{_y_5_Ai6Y~?$~)jYv)sh3F__=lcwL8kl@ zK-taoW@xBmY)#l0Dfw|uxbp`;MuVim=%ce2q6$iwUL0=iq~BEZdT{l;Uyk}n+Xjw2 zqoaA09joLB8O`7Y0hb=0Nv={{$4g6U8k?9)yLoe+5vML_hy}ZLZ^vtldTkx1IJ|i{ zBk|Queyhw_o;^fO7Cs7p7GTdeKn}%P8|mj6Yd7>AXgxW< z*VyvZatAHX^X;qienC+*pqlD2kNQD)1Uc3n!`2=o4nq~Et(i9#rF1;#(W!F``;kTQ z5N2-yMdx~$st2YS;yK#@Tkyr!cHjZ2P>>XRX>mYMnl;UNjwh#AxrJp|*~;VW1W(`W zv2cdD(*A7!xoEQ2$!cMWWU|;Sk1X5~mWT3wfYE$t%gF^gD*XO7iJ#723jYaw!=m>7 zx99s`%~$^aiC+It!p|?o=l_KvEBl`gJpYAu%FOUrt%s3`_1}Y}zlQ%kNczX*{~$=} zQeO+-6NB-3R9|q+rH(&=8}L7GM!#Vj6X4Sm2GS?5y=CgjQ8rFFcW^abA%&DR?4X;A zi!M|uqYp>jb{&f`(;P_^QpnL!&C=iRe2|Q%2wT;#^~$bJXq0uuD7x`@Ex^`Uh+!@x zIN5*v9#lKe)|QB(Jx5v3aewn7!jlevvOirfI*vwQFG8~Xc(bPL{W_Bb+DUiCk$DPs8Z~s)CcKY$*6MGlIhM)a98MB45;{`blE%Lg~BR9s* zla3pQCZAm4tUVvOAN~h9F1CFm01o&Af_x+&`Pu5-)Ghg`weO=gPU)3|%4RM^JJf9? zC|7&^hR!?9{ASdeKl=wx7?_yq>N@Fu|N4I8t}t{$RM-c*dDHzpN`-YpGfK171ifaD z#Jc@#nSZ8%GA;pWesGVMG%WCaP!@zo!8cOky0F(<7BCIsm@w{=)<2#6V3|(gm(GUv3 zqGIS{p(NwP1FtswJi%nxE^rw^9&qco0GVjF#bAS*pr~CzJ0q)Cue;5uA!+bs1q|WI$C)*J47$ne2@a8X)>* z$QS}Y7|V@RF~O>fp~=wL6ETBGrA{p4TphNZ-2-?gq;U+kjdK|kFX&Ei=^3WslBSo1 zoDT2hooZbvY6pC%^I7>6{lJw}AZ++fY_SSqs)k+PCCt)`Gg{JJP2CJDI0~Xy6w67$ zcH;}ufLKZMj%D+gPg^#aG!bn=^nK((@4$mbZ>Al`@t&NLgU ztXbUy+kJggo$O<{T2KLT#B5ic`6|{N!m;XHWU!;Td8lIyAhKfnm2MYgea`^mFId+T zh@HH>_^gBc9laPDRgQzrTS5mm9cXkM-Gf1X7$x_L>pPbmaQ5!G{ZXDCOIuL+6Xk}$BztG%_=FW& zX_&zvJ0R&*aS~9NPKTh7qS2%vK}cx$=hmXY1agxIGz&v@x{S_MzpV={vcVvfDag!d z=4s?Ag>fpR$;>M~PYKZ^4Jgs8 zTGG;~F1AZa!g6cXjCKFSy&5bzH09{&HAMw&5Ua8C_W1O@%sEA6@oX*p+5O7(ovy^A z@l+tq=yTU4Ama%4R$J>Ff_8^E{$dA*_+T<%W?&5b{?VVY-Z)a$bEVcB)%aypS#)!N z(~8F2bo|2br%iDtFVfdX3*ouU^w!b1Y$nV5#GfpU8c+>;DdPTn2o%-8tz zV_V&eopLC0YwD<(2?c6GVGfz>DN2>Zc(8kv1YmH~%%?fIBs)AmfxUN6S|z%@KruDw zgoYSpCow;NnoT@i&fA_nvAsW2k@cA}cfGH!OH30!NtUiiQEuF+rN8ZmNXpsf_$Hvt zAFA8Hr{{2dGIYH|Uwi-nhh*59ws)#hcNN;`T3qf2Z<#a`acF|RD!B!6XFgwhw$jR< zx>C-k+jDP_BJRT>>T;&!W;ATQ5RdMk|XUnx)+A1di- z!qNqOq)SK2};i{`b{@53zcnv z-^!|d;U0MHBK@|M6;5hxC)g4va+`>~u@XxXO*DMK{@7i18c1k}7CzJ@c4=q}7^LVeG(o&CX>@#O zI?mEu0N18(g>px`S0i&ao=6Wu~jK8xhp?HDxNB`(%d9X2rIA-*es9KS{bICNH zn0<-L(qZF9Nu`&|w>$~R=33N}`E!CoTSm$_if8aOqaEsm_mwRj+Bs9mW_oFW9doKz zY+sqDL`tAY6e8GlaD!HXnSh8Md0T!LL=DQ)?YBFxTd)>*(`dwB5tG|No8wrzP%X=x zV_c*^&`_fDZ(pAbJFeJyefb|q4s5q$czt`03E%2?pEA1XK3vcz&g!J9^k?{VD#D1L z3bsv46D>bP9q$<+n)yQGoBP5-nUf6o)aU^&B=9G=_v!sRd~+klkbN-(!=WH~ zAUqID6G8Y$O&Q`Y94ILv-6}xEgP;J3X9bf#(ENCp%o$xyc0-Oud-7e2QX!MjmU01i z4fc_CO-sP5Oi#V!_ypa(IsLB``UN62kJKmfzf*u4;){pbr`=Zcc5|r>_16fmvO;3E!#I0)(z${0W}5L2v?cY9fO>*HgTMX$;mS z0g*e)4M**+4+Xkiq=dAp(<=zW?>8ubmRiKXv}PANtQ+bKVrs{;^vW~|mR4utbro&> zxM3^Z+-RKq^}$wuqIk|x#~R*aSA4O$x+PEy7{h_`3T83BbVR&0wi#KXP6 zIwkKiZ!vRxxSB!Pl+uWuu7H-2ykekMH;UP$aVQd9R<9y^x@>(UD}20^9RE(x<8S^& z`(s!pCnAwTVEU^`EBUuQ&~zD~Lhx%wc1Mh^sX_`9!oI>`>4YIESV+mAs}fy8W0MJ) zxsyei75+5sWR?U!mp@12e34LuoTFooL{}{N)PT_g*rg2Igj?57@3Jw-?%ChN&LA%L!a0heY~Z9E zg>jL?w!jEq;H#&gqmUyv9f!6m*l9Nl1vZt2S?h>QzJJoc^p?C?H93VN2B{jTQg3jM zf{X%!?f#07clMA@Q9()q#dN12I#d*a*E7->ga%z|Xb5OPGH}TjGw&Xy>f!xZkw%u~ zb-7`FHMa)!fPitw#BjQqmIA^S7!4d-6kkkdGk^xQY#g`sqBy&4=b+w*1O#H)qMZW# zP02X9Xm+Rx3m7!#FJ!$;;S$UwY=%Wfaa~+MU#RJ#><>y*!=nPAY0hEJLE`SaS!8N^pMKV| zmGGA*dX(`cNK4(a+n$s<1IR-!)ec>u3*~ySCV-Yx${@Fa*gjSjLDGgyc!mTi4R|3F zvn7Rmh;$auhGO8<}u!Dc60=n9;rX_Uj@QNz~~miu5Pi zHix0eoSqXATyq&*jrqotWtXL#EkeVToHDaHzD7t^>@B6@CZz3-HA_DGs6*p+5wUi_O$ z#*w6rL~V(7>VIqt2naV?ca)7MD4^jresMc zrAl(JbVh5N!_8PvY0@^A4%*y!2@-7KUgh=MY>i26RWV+CdPN5D+-u+MjWNby_U~&q zR6{#E+Iu!wzkm}PSHu4F1+Rro~Pd0jU)*p@(LCb5?Iz8RO&?DweQq_zMnEm zKbkjGDRo;nlAh1jFR=EW@K-OORuDJGeyOiqKn2unu7jE>z3E!JrmZaZX+qZ7xT21$ ze@*0WtJ@I0DoydrqXi*}2Xub$^FvgE1C}1oHg3~F5CaxA<^TgL5ai_yHVO}Nb3+G% zDyr%&Qae}jhY;4w9i&ZLQHstBK*HEsb&&=Lt4;lE@W{|A+K94Kc-b+=`Z#h?J1=B2 z53Ijx&gJQ0!_%w-tq&K&&ckeee&j;TsE?kDTzT4HzaoJ3z~|Pa3;tcY?ydWHt*sZp zWii?xsM5=(w7jc#T5o5(+)IWu1G%i^C)lS3T~m7R!qCMGUfM`GfzXw` zz`4jxi|`S_WmP87V_=ApHYII4k34G4N3~5dU;x8KkT| zn;sZ6Qxjts)E&P$r&d#_vP>AyoBq(K&kg`@Ru8%`FmkXMWPj zJrk^Sg#B+7S`)s6ybQN{lL37VbyLktMPu%nnQ#5kT6rO_8h)6Xa%qbLoBBKo9%Lml zUfvU+7PY>}t7hMQw)>mz{T2$W$$9<6=d^k_2o<9|8m*ozJ?U#UMyYY#ZnQ)*%R?E~ za08v%Aw$8>cr`mmb53I7f-;;rk-Qy4Yn(x0V3HcE!K-$?8B(qXK~pfWoQCBW?C}=1 z5x^50^Iwg>z{LilsVC=ovS$7WixN4AI%9V`2|K!V4{=PcAsOs&{fzLHd3V_=O1fSS zwS{zDW0J^8LTv0|H~?v%9D?Z7J0J@FN1|>sZeS9-cb|F43#o~0jMK%E>elBssHo=l zm*lQp?RnYXbNY*dx!(zUF>5Pix>W&?)I*%Si@@)wR9;@)H_w?(>ZSBrpLFON9_JtP z1**=EUb|aS1Tdb{<@3=&{MCZ(b^w^hKNLI7L$FpWgdYu;tUs9mPxbHUC1zUFQVrE- zSRV#kokrUXbYYlyk|hpeE)`;2=0iFdz-xr`9_w_zBLPO(&U_P<0)yDZxl(c%&F#+S zV}S1N_!yVI(T0DQ{!qbrt2uqu`8Jt8Tc!|Nc9nUG(G`!>jnn_+veu$B$OmeEq;t~M zTTK7e{F|xr1eoQN{`C%g5oS^i@x{qx2Ji*c$5lXR6t= z439R~Z&bZ$1$ipMgl#Rz%9JRwD?*aa5$enW%Yw3uI#nw36=g;yu(-}-ovN<{Bi1f(9~_xv3Y6`ny@1l&i{gMvJnaMU3a+;gL07o5v#8f`jdJLu~@USsOZpgeQE zque^ts2Hd4#p1fb#0Zt4akJFccsX3B!kJj#K_Ui+S&Ff*q-2bs!KO^oBtuEgZkFLuegFckx5=;visk^e#?}Z@Go^3HvVJse}ML#fBSL^ZH4_)ZzjO$u37IZ23)-aS~V&_z*l2L$ErQ6eXx1d_UF zyF{|h*EmwpaEM(BrMWa#+$>b9;T%3Ag@E#rn;F*t7XW4ez4Vfq2{)%b(rEyBByB0t zd=m%7cc=z-geBYAlCgvX32lA>&ty$+V5XVWZk5EJ_H#K=Fc_N_b8~UL)w&nkhSxv$%*a1z_FOUt%4BiM^rf<1^;bE zudzmigOsc2TPnE&nQobARF^}9{4D0sJzKD0Y(Rpz0p9l*HzQ0m@B9)y$Ip#Ek>IlD z=w!gokzP5WI`anCV} z*3LMEz0%^*?S7X?Ov}=wEI%pnS1Iu2U~@N?hNEGD1E4$(8TQ+3>fFMh892JL3JT=% zQrWOTK%`}99F3sBOUJjey&r|<)!4MtFhc2qJoUk7njJq9h9K7;@KZ20A~YTiO&Huz9m56K5{K{&uHPwOU!l2oD=+us zv<9WRTacBGoG;%FVIthRW+q~_Q(5gHwA9tLhLLd z8XbdaHGx<(#d1#0Bp$6>gyyQn1!9^1s!LviiF&Lwbe|C(K$y381nz_SHo=y8cr0>< z5L6f^%w6Wz&LC_WoOTcO$X ze3}t7KcOVmA5B?>$Yt*CS~sNAgf&|*=O)ZwkE2yDFX*bxh;UP6y`kIm8^n2m1geJu z0!QFByk@vl0QKT^UJ4uZ6HS`~k5*DeB!80iR0#EVFvTJN(pa(DS~^3T%2&g^PZ+OX zLY=h@XK8$25_567qV3G6Njz0HGX8xF!-8lwzg1^q*9z<6;h)cUVmIWxD(rOAN!a%1 zIPrJn&AR{Lm$vfaLXtAf@*?l-N__f8U@`EZ_->noWfw}SlquGuFi#9W{mO5L4rZ-Qw5 zOcR}(K#byPsw$?)rXDA0kz)$LL11}=p5kT($(Tmag3GSZtVr6i2{Z1Utcs>O*>)l_ z{h|fJE;w4F?$V@Q!=_-!tv}j7DWnVd{b~}ocVqpykK!Gv_6*(QFwGr9O%Z5$S;MWr z_M2d0j|KFsj^n(v$si&CYn3^CH@zS=w^53w;L6(I)j6cY$Jna$eQZo}LFu z!)-*uuQL!Gm$;Nhr}gmbpGni%rPNrZkA|8k<$TjwIRtARqN?Bb_~~`SGy&*?M$vl9 zFw%gpHWk|2EJ$vh`-W4~amG2lS4(fT0qjS1-9K22)#D2%2j#?v;5Omv$HZ1Tpy+ocTwOYb(yRg>kBVb43DD9@m zlfPvw`NRxCYG*q?!SF>VbY63^qNow?24iA$h0s_|6Sw4Yx-Fb>_)iDCm@JG%bsbpR zKO|j}aJSU;amP4%9$u;I;5lx2eFPq!9w%AU-Ca?`>aQ!G<{jFrOJ~yidJP!UCz}i< zbqmnW{oY5pt@s>s@TMO$%HUO@5!I*^nzBzSaqsUq!Rh-= zA)Y3yqTm(vxc;Tii96Hvw0K_UP<71SX&nC9)3168-P0sqq3MwdP8B8)phif8i-(X< zCH&=N^{S6D%v)e78dk~jaY%y@)D7I;$`p_+O$;$=JHjEyId46%4*#srW{uvzguA#o8bu1ArH$v7ep za*AvJIkg#1#o_>wx;`L-qg_LQ8$XEpU8+)&`n0?!+zL-fQp4ZarMJJCvHi^?8PsuFNrS#f}v~iNWL4pPEhxUf4S)WB( zh3F`&>{;n5nd0QRh!Td~FYTxGd#N)Ys7hcfEm}yV7wMEZHihn%nQs%a3*nb6jvA4~ zkP2&MCM|}hLy7W2P#v8ngOQv&ilu47BK=v2N#b^C;?>cFt_9icDZgK(l8YTuChmlj z+^+3pH?hnv+!OR9&^vZ)$Z9Ce*XH_vxjBN?bJCPzpdCfk3WQUv^#g67pp8b?3Os&& zK}s|8s}>+TM;IuzC$Htb84Nf(kN|a*vWIKnZyvT zvG`$nLYzQY94qod*J$+MA8F+#LggYjXmHwZGxSjiO6oGumWO2~hp!!Xf;cfCOY2yc=0Bt;zY$cSy?->@|@h6xoYpVG;sU8 zGo*xHuD7odW4M%S#&?CTMYT2631dZWwod^?(P(ItiuNwrnw3&RYi@QW#3T>%_P?>{?CSABxt2ieLvT}upkLv0U;uQ}I&E9A3pp$-VB zrZ7?&Ci>(MqU7J4KgCZjOr05J84)b^5-e}2_9%76S(h{`%7~NSxB6`N$sTRD{*290 zSt(bdm$FKUCaa#?ACNCFvq`UB4CMuZemk5DQYlE`I{F3$p5FD9_A3y=**z*Oq7jFv zA3-!sG_tUM;n_wmpO%WfuKC?0Dl8`D!*Dn~OxX`On<;#2Sgx23M%a{&O)#}i_!ukr zn#ER_x^2)nE6meK2LWc;4DMXPG)7pL)7j)B>4+5N{QV>n$%z*rMxqB zd+t^Jy#CT8LOedcrPH3`JI+etq<+*F^S4_u4kZ%ciga63E(xSVKZ?*`Yap{9 zr?n3fcU<4b{fV-Z`Siz;QIaRbaV-;(D%XW|^Z1p$ zP5;u!&5Fk)PDW3T&(3!?ubV{i@6kj8oeRHYM%#-9}3^O7RCQIVq=P&wI=Y0+6Sspy(x3wRrv z_@LIP*t=`i#j>Ahnm4%_=4%E9d7KP$G&P}oOW!W6RIBT)3>iC7h?A=xoQ%BOs?u%O z&<y7B(&^Hj#4T%`>V8OS?K=--uctif3OK|P?m_qA%IzpisyU;gJF69&XlSo9j(MO3@f~} z_g-Rdgwu?2x9@aCUJ8RAez;7caJjVIcM-Gmi2HgZ{vFQ|BzY0akEAi;m(dczC$$7H z3hS_Je37*}lKAceQQ~+nX|aN5#UXN#xKx#eSSS)(ig2(`OSoVZSO`Gz=}w;-(=)(K z#~RS-3=cX8gF-R;Q2gtQE3=d3w5RIxJM8&i?TlWQ)reXsGPNW)Ql^U*#63qw_ptnk#O|(D$2^+Tq1GsTXpB}_f)zY1f%YF)Urk2`W z7q6U+#>G`}3Y6X8MV%CVse@&4%dQfqpJ6HF_4b~$Dr*mNhAP@hXZH?}=YD4_O7uvHlR(<>XoLfY% zbj{JFRm4XT#{}-d`QbG1i2FJ2*xNoE)lp$IYArV@DgtVuKqTVV(EK~e5BZIbUggz1 z-qJ`cCt+tBMK1pHsVv2T8@NC^2d5+Nu*^P=BQ5t)yLO@FgwG^DwNflMa(T-t9d^%W zt{$5Od=%H9(7qNfS1cJl97dJykf6F8o}jMTAI{glTNo?XZ}51By=5)?07*jfulySi z!Te7`Q>L%1e`7dI%zx1r|0{;`XXXFk@jcV}Q}N|DXV5o~k|uB_OxxyCkX@EE;CuK} ziX|{rNxJCf(G4B(#>_-5XC)UuOHBVv+L>CxxK~l`LGx!UBjn4CxR*JJ`MY-JAMdx~ zJvhP%=kq4fDoVFDF$U&DAX@D5Aq^WXNyN zHqRp)tFY5;F%X%>;B+#um`H_4nuH&`?Xl%%(i3TA9W%n}a(AnRNb>@aZBuPh2a=AB zQy%$owav1bB5JHQGtDzXbDVZ#SdI&;w^HjWhkT8Oooz?JZgJe#g(ACB&Vy;VajeZ0 zHtv$^hDz%hU(Me)^aej(o~hWq3Vk{mhO-rJZ|FXE)YdhMY`LU5SRWo-4)S*7rF*}J za@z@aax?aeRM$cle7CFA89eyVExWjc#L|Oyf5Jw2WrF=cZvMoY91wXYzBst4an(0! z`(ws!ai1!pbGKp-9ZF5Q{2E2scW;yjRFn9ZEqFEBce$TgXrXQGLCME8B%J=cFtpv- zm9(}XZNhXkci>C1y5nSQcLK8%SQDd*7H#_`p!V3`%q+`*ACI;(GmKt(U5kzmva+;v zPAMyVsx%oYVf~Z|L>438A7NH6(Vh+SGr=7&fZb0qjQq27K5fHe`F_^-$oa_4>ZH?F zQ;+}b;{Nuu6fr#dxjE3{&Dj?V5!drthQY$deorwl-ELt1h-+#bMu`=nsVt$^sK0q{g73i*#Qi6b5X+9C4AM&9`DocHP82Xhxz_Fc zJXz7c38a}e1CZTD#Wvs(wBf8Z8=JvQG_ub{om061<8i)uw{7v_L(KU`;MWwkyH%>wfdw*+cKk0pz(_?To<9uz<&0c-+RTNo@RaH!0cEg<%`f2z0H%2Yd9_IpMSr>Q?DV_z4Vrv|ZQPdrrO61$MhA zI2~g}0)`(X$GtT#ul*jfKm}l{NWdzgOd#8;y|_g`IM{B!4R`{Npxh2-&vRq_Knk|N zyV~NKEKir3!bk_UUAq$KrsA+pPO@K@o8z7spIU+CHf8ld`#!LFq}?%}qyQb1O%&gi zAa&uDB0%Z}Nv#~tCPkPWXf;CNse5?Z|;6ryzsedo_6<)D^#a*O@WuwG_%o!*@3&*Uc5%>T~xdT?uc}i?YD8ighN~t7jGtS%H zcVo@NG>o`ZU~WH^*@lui?u8LU(IwM zV0)RoeB^SL{~3RMHto*h~_W8EtS6VCbUeUboD{ z9b%&OXwx+k*lrIr_)M=XS$o!2=Q>nXzSf3t?nSyd>|RNS9sxwkeu090LPFeXj^HJ$ z^XH2~29o63|0*|wVDh7TP!>9Pmkr**XrOKqJLJ8$KGJap!z3-4&yI(6EQbxRI<~b& zH4?E#biOPoGc>e%&M=}RA=l3?iLCcDego1|y1lEieqzTNmIO)xeHPr7PxA|G8e{p= z;Qj`oN8;Fo{$;hi^w5kG1=R4B#&BN>BlEn3yl0Vqsg!Md?itj9O4n`0db4vNEdjv0 zgT@`Vs2w5}kh;fnaQx&PIT|$2SlhDa2&_vh{bWhHhh_F_Rrc3${vb=nP)8i?Tw5hD z^lfQ#LWb4ddiD$E*EKf*)i%vIZ;3DY(V$+S?9qYPhbx0QrOKSdK_bHwv+gM>+N(Pi znb;JZ0yTD2e4ScLYQOGDdCgW?=~!RZkO=VL^=`u}G+9`$y0UyR+A#5kITEd@~nd92pc~Q?tbR~Qu7ct$m`lF(_l^({~fV91G~AT3bx8whFhGG z$U_ZN1g*esw>`cwC3Wov3B4=%+tN*(*$`tXVz@6~YH~Nua`5~1EEePaNotpviUx^} zqs-nMzETYp6Xwb z2h9k;Fwk!=$IPQm6Tre9#7c+*xVCjiWi&^|Khb!^Dw%x&bxuN*8Hfv*GMyML*Megp zgNf`f+C`Bj+v5r?t8v77FmS~NwNGgW}i2^}$MD`3Xju@Ev3)AE^ z_`&7T_)s3`?C;7>4d>kJ&JTp!#Tlim(g0=h)wwxI9ko)1i45E14ehBW({~RkqZ+-D zY=e;YUl!>o;2&W>JoN9mR!L6ohb^0@IVdgK)6LJTm99mGn6lgmeYduKIfGWCuRisB zZ>-es!e3_RNYg9F5&ZtK46@bE1$J^K4QI#mLZ^P&;1QUE$80E^aBKt4 z&hj7xNg1?y8L}CDEjr=Xh%n(Qe>b9d!}x*7Gy&e1`%Md;cj^W{@$g#x|)}tcZV_OO# zN8%{!5+SO*lPu=HI1htZ!iM@#YKa@>M0R7?i6DRRR0IX9-fE!DttMNwgTl>R5q(Kb z<%+#Wfe48r=Knq)YwxnGa##wY5p7CH%q$#7Mx!6^Ei5)H`PP8kmUy3VT8^6!vZBuz zhm0jKkhg+pYY4fC8)0qnJx>HkBoZj{90;2og3TSvb{Bo~(CXJ3(wa4BBsLg!J2;y? z-sU04=AhR1JdN)}oIp(`8_WVY&1g<#dXZcKEyBq}Q90H$qcI+GO_=+G?Zq0<4!hQ~ z6@l#6@W;yC*v(i@RC#2qY4;(doR#A2eR>~~9J-h3Ue{>$ z-+nmkzGgjDRrVlv&}m9s+|}?;y5mM>u^KMWLQX1GGiHlxFDT31ap1qY*M=I7-~#GO zAMhOUm*PBEgSF5v+G%}~Xw1f>d_=G_8Xs<^J zs&40L692q4lH6Wa2-?*rpkC+;QGExWtxnNFwx5PLr1~Sf8GWoGbkDS~>zkVJ=qE;Y z*qWx$P*@IXP&0`D&l2TUGNZ9UOHq;b+U85yzH6yW=VOGPiQh^ zZ1FVtBuouyIvA6YR-Ib}>!mxjbW`)H*?5wGtS-=pG$*wE>};|1?b57K*x@jIzre&l z5vTpI7aVekZ=PiH(i0Ar3iy8Z)5+Kd`a4@!N}s4vQnT6q@!iyzsAKaog7#a&DLk?G zzOiuX?CDUfImZx9S~*Lnxl~M{Y0}N)Id9X{Ed5x5L1llM`VU~6 zhC~T#TmtxjheUfw>F-CZqKPU1nISiF*6P}kHwES;k?i0=6ZhAJS;#f+Z%9#=d0Wb3 zVGm1*HyySJuO!|XalT9qy4g}i4AsWsTJCk?5L22FeBF^9YBT};>>9)w4h_g0I}b1% zN%feHv{opNv=$AAn(Ku#&ADSjCbo$)&7~D$2Yv%z)zP#u;?|L%U;spi|@VM$h=tt9Ese@ra$Eq4qa}~zb=w`LCL23v?+^9tz`59tj zP^Bm_pq?hh@|7tW>K=0_24e?b`x3noc71w6Lk5mvo~wEPB85^Qh)j;;{Vkpv9g-ai zfsfu;!NmlupYigSB6B!QypVs$Ct#uo7+|85Mfk7cL%+V(hr;B4Y5xWhTFy8G|1~k1 zK)0M26cNiGMsh?r+E+dqhGCN7hZ-Y4*=K-upD8UFsAxR~hG<+$jPG=6jjJyN6DtEu zco7CjlrX5Eu3(IBy#^G9Dar^j=$R#np=z!F(D(K+J_vbz#=&nGR5lRm;mCiqALXv*Exelz3A%TAIq1xVSH7H6-iYlQD}hT;N@K5Fqfacl zjKnWlguk_&q`a1LmYUn)C;c{9pKMT?Mq`{831}2E4rI?+iFxGja!=33&Y`l$l-I&m zU9pPAB!dX97(nxCcVE)lTycj!Hs0Xyn{1o^jSpjH{ws)$k@4%_xG3{~b8Y_+iie_8 zaME{lvo-pY0}qN$#mvyr6rY9RD-@lmk(r69BR+uE9xysB9gXbq>4Yuy9E}8x3~UUI z==jYX9b}E{1#GNrZLEJ2^#N$&1Z*sA>=kVF42w*X6SeEmK7itbulFQ949@eS_NkRQOK(5YpLPG^uH5jgo9a@d8=% z0>1(Tb#rw}D0hRZWvZD$4f+N`KQziK=vmnn^)Bc@bre2{wEUWynnocr$y=|*AeUs9 zagq7c%(c4K&W*tIj0{H3m#hP>j}&&YlXwaRRQ`BLV4&~aM!D1xNt**5T_08<3-%Kkm33SKASPH5C2gLf-6o638)DSC>0|Oj=BaMOAnI=v4)=2 zgyt+`GGcWquEhc>Dhq+oXoI_f#I7;qyliZ292}S{zT}EVyL*C@2YHyKAMT zl&v^cf>PpX8T1A}6cjX~lk9a*M?Is>XktW7!E?&XN8?q9r<%;o%`Ii(?ac;`I-QkT z^nBE8s-SuVe4h4_K8Twgk76aCyx;B?m@Vep-k(nfB5_`}g7EUYvwYs&u6Kvd&(AY6 zGjnnX9JY^Uih_cIY}VVFO(rre7Ao0Iz#D8BhBh~|k#V_PFSmMvT#NG#-d-L&079Yb z?PWGY@WIsv1_p(x`SJL7_Jpq96c@1eBCpxW7bhn#PX}3NSD|y?dRGPNh+36@+V2gY zueHjgv(fYL&<2`}r^m?9J9$4J0m2#ISiazz@5GnsB_t(b)j^k(2WPMsaWbDdhx?Yc zvd`78E|#rr)|<^pjw}bC5GD71SaW%v-y2CHInI}j#KOW7+P%NOr_pE( z-j9vxhGgROJ4m`-$9L(43w84#!07P}ZU*QTIp>s=n7PG1Z20J4#;_Dr=j9o!wKx?< zwzWDOj6a_4mDUZCR7r3di0!Mci6q(Fr;e)$rMesd$dCu*MQA1{ScNe3G}@yS`p;_aHENYNS&M z(4=^d{w_HsG$FR?4NEp%R8C}%Co+K#r&Vt{g=N#0GjjorMl0I9f3OOb=~9PAripXq zufHz6Z-*7!Cdwu^QfcLVJw)d2d9^K6f?pEeWMA1pdmN&S^;Q#V%6jrz@VtUZ;vT-pn2h z*-eiA?n>wn0xvHwB&8(NiF7*e3pzgRc04RBq>Kki+U{lxm0AommKGK#hliIC4ui*Y zD;-`!gQK#yuU7+jSF$1#id>4y^zz!*mEgr6ql3?#RrvAq(Ss>&vFeS zJ37<>hJBmHBCffyBP=W|2ia=1UXKW#SN`6H&qq7)&CA2Z$w(w|2m1yORht(V6AGNy z`_sY8^E2sI4m$$ZMZ8P*oH8@goBMe%gJ^<#fNL2MWQfF z2Y6}$^&hP8;H3HlG&D4tY&PDBpIi^hsyZksLw@kF69evX#}oa+{S_npt3T;OlYP{${BawctV4dd7S~y%b_BV5 z7-emr_W(U&KH!|b>j=ltENCrumP62yo_(izu(wi)2}X_1WQ6j1YXr!OyUWxotHZ@Lw= z;Z#PJ643MNF-)9&>)jB@OVpwYzMdW|SF6+Ze%b?gnlu?+``zGKUYB!CY_3@@{M{4M zZNcN%5_~G2i~EcGn1=j1-gLs}#R}YDxn&>iRBy0;_LM;!brkk|d;$*Quni3+YKaxl@u5-E#s1ZFRhkGg8)A{j!(mI2+_VMZ$w4FnY)GOi! zH~{k|IA9tEOx?eY5lE4WxB({On?V5r4hF30v$Hc%%t%pWIa0t(PRDX3$AhwzmX;Pn zNGPa*x&!q8$xC*WI-ovet9m?*RfeMxXj`1C2LdP)wN~x9*H#q^6BEEh00C6HnNw@b z7vu;QUJ=B=5daIcnL{kV6<-Yu#1(HC90Y<3_5JsoD>={t17KMKq6Z2D=t_7v5=$T; zI1-HQ9QK1|V!)?>C!?&i)ZD`2hL(OiM@~UajooUQi3=GhbV-10G$6LTP$nx9(4Fg7 zv7222z}d;%j@^>zS{>a2GwH92fEQ_un6#2oLLy)tkP^jRd#_0zBN{Lob}OjYACBRi z^F!RstOSGFiqUX3z0R-2>OrT|Z;ji==QeMz;AZVg7HBX&$e0pIeDC`DI+|s#pwZ0a zG#e(Xp;z@1w^YE`VB}E4I$UhT>j6*Dd8d;D+c{00{7ToyJsQVj0_wPt(W_Rca5~E` z%N_VzA~#cAitHGDn-rM=q|swJNxv&;k4csnivn&b|B+;mB=2Z^f985JM~Q2(%2R48 z=7A>nmyNDZydy`AwCQ`u3Z}-g9WmK~?!JRZd>i0!;NWtCOQn1&wpm19;}byapZmLj4m2%u-9 z$TeGE?j$ZD;Y3~d!HXm;J*mdG_T{r^5wCRdq6P-Qj$y9=W5VYQDCxBIy?lz#;t7CO z^<7F%1>lxCpbX@b-pUpL9JM9>U}uORA@Nr~W4m?04XC(d79Rr@?RW#+OnhkE)ELP_ zjjsBma#W|*#a5$(lN>oRVJ^qv58PGzLe_odhVvHA`1ttZrsGM`z2#+fkDc7o(%~h? zJJ{LiIzNye$5F?hZEdS7cz`D+_x^WJtq2Izreu~4Jp02CXgDv4g$teMtc&!WB3A5h zmeI>t2d#dpzKA*znC0V!>BKvBEk?>Jw6pQKlm!Z&6<|#MEG4zUa>ki9ZsMk3#DZin z_1^DyB~m3ME~uoUBD3hH7$@9Trij5@Bvrz`h3+;Xj)3&R_ZD*4Li7X}0vMwK_v)m@ zQU_52JJIN2s{#ro6b@w^L#-CjF-TJoEA0`jQynF0}1H$1K>X3S%AI=#ll?u9)Xv#u>`o{ zi$LFAVH@?fyBzMhFMoZ!hY{R70|b~ozZ?%w{Q z?xJU>|JU8yUzo~Y6qEnkyEpuQvsDwZ*K_-~ySl%(YrmWKU$$#&W|GJz=#ZBt?1`b@ z@vE`zljku}s*+_xtIJ1mv=Dt^9`PnpHzviOq&-~vHnhL8F zJrj0kPOHdjcF=j<2pOrujJ?IPdiL&p7(jKY3Z;EBx1W6#yhZbh9Jegx?X^xNkPrB@ z9$G+hwFLdi904AfX$j1MC*g>hBv``eDsCsJmXQbUeuux08@tdS%#;UT-WzEir z_iJ^GUm%knRay}-Xkm@(pzfruZ>1^EAD5f3xmFz-su$jvjFp!-=F-!^wVyp$(454! zIyl%k|$kjF-(JuHeG!x+UK8dFX=dhe@jT)C`}Q1GJI%}+lefHy^Sp9rp%iC!J1 zY)v{<#O+zQnmcF|@o8JNa#qDObTZB-D>PFhmj~8eS)7k#5LntrHEa{@swux6HEw)~ z{Tgr1LHB{LIZ1BpqT%+Ps|VP^v!bCj;*$m$Y?Xz2rs*-Zd)mBT=6LEF4l!4Qsz(|) zrCQojpA$}eMLXW|W6WLwA?8d*wQ@?;kw}fF6mO8ZM{bGZp@+uGw@e{Yo2(!5I+(aL zYB%i~yHYhQ6=QCgLRBn-^9)B712bH#L9jPOA8AzS)%Th{orPy1g>w%UK`T{F%8?|Y zRo?bRZWV?Rg((~tN^Xm_`%aHt6$UR+x2kzKP20FM+SV)$*w=o{o=Ihv5p9J2eULd2 z8A^4ugra6KdcN(cx2svEKCfKJ^QY=P6YF}g^Sb}#7>mHNp;6?m80HNBc&J?3U+cX1 zj5R0Y7q-wRWBSW@^$S0Q!s0Kj<$C*gNWhnUnZf?~{?C&?3;#DSOeHm#w;}j4s|x&=9|KX+K*OkOg|CLS zwZlJDU$c*OzWMF_A*&~~f4;nQ?zI4#WTt<)a;>_sxnYowSW0s%j`4 zQ5$Lgc>~j9@H)PnLdoJ>!d}O*&@NnQ4u?EN;Sa;DTJA&H~21P_aeU#S}9^c|Tj6%@*4#(A?% zqng)JBE~YmjXTUg&f&<`I`7QFr*`=b-GWa!vRtWIP+2r9_=0e*6LV;M8`vwWoz+j_ zI?==LbB>DbSCIN+G?q6Tj!AR0TB<3OqS{EEVlCIE9d8=6HgKtC0Gi6~8z3R3P)W^vd zRuSa6^j^*`(U+HtxFvGtF{Dx695;O5xp3!KlZQP$vIMM@C8|kp$4>XH)94&&8yn|U zep@(MSth-HUfz{j;3o7?kLE^q_FF{(>~n zle%IslPcffYN4NXaHro++cYjhnNCFF(4pz2!MI^w6-uN&KV|Pk%NU$7GFF%ktRTdOVb!0#4J@#p&V9ce1zpyvgT=4uIVHqJ|8u2=#T<| zkjyh(s)R?Qy~PjvT7=>ipR`KU$c5xJ@%VA2hif;Ns^3s)a(5dTz^2B~q*~{_{}2`( z{t+o}ei1|$37=F!p?K00Sx}l-CS&1Kx$r0%=hx6>7V)Y`i}wDu zd}9&udldhkss!>^mvoAi)iVG*6??_I0NC0~G$< z0)qW7YD$K$Y=29AF#n6f%DBi>#;M>h_@3YWZkJyjg(j0J_ja^5^oxPRVfsjC}mtBm`&j;qAoN8ZC%LW%o9rg42di*rS ze;Hey%G#^ohILiT%suWF(nSP>lamn`W3`~ggvJmmN2HC4B%T=l)KwY_@=y(|jnn^~i#OCi5D|il#(He#;t!q)u2NHq(Uv}LvSye6sxjE*A%Zq2 z`Yv~22T7~K$VD%LHmoT}hg9jD+Y90mQUM#iM&Lo@gec>wo2Sv7E#K+6KbiI9s~*w@ zf0Ph=7B&e@DMIxH!vzL!fC5~$arg2? z+=SCjabAie$NiM*f&XoA71p_bx@8(z3`5Or0t{LmUN1Mg6{*XJFLNqZk+%tuHoveu-@|UIfnW$H zP^+v3uGPk136GZLtld8@R$YXvBV$a)C2R~=Ys}yU$4!zO3f|s9p?sIzg+%7cU4fIk z*=eShG(zQ669xC%w^Rb;XM3I3vg!~htUmZ(ja|YHf=EZJS!V3;YV`i^N>mU7W@sWUhsr;y-iCwy(%U%G4HCS#_jjC!f z#T5vIeRk6yj^6f%)P&g2&rE4(by_zRj7L)aH=L^Nnqn8Uk`wO(S0mARTrpE*=?9&! z2_MMlHr%@H#TJ@Hd$TAi;97A|q3+AZ4WPuH|1S+hOT zAfzhTX5C0Ry|QLqjEM{e#TnewkVO?5aCgj*<3?RfCfI&$-owY|;qIbZ^S7^YeWRwz zGk^a!xe$%lqm>%`h4~w+hJ9L;GL^@vCs#5ui43y|n)*RDr}~bTTfyQ^j(LfS)oz8` z`uFxGDw3Kr=lWM}mQ)Wct#yEcALqqc3pQ*Sh*Jrn5f2vp#xZYm2} zs?W~cUH{=2OZ%#zRZwo96p1a~>z>R$uj<=DhH^dk)3fWH|7quVQD4}#9Edq;5stB^ z-x9E%*8mfY^Ilu1-hSQIn{68ttb4qsyW^MPrjbQYt;nT|j*}~oHeO=La=C~K(ZJHU zTqtE8a!xAtqbNuwYI?8S@-qkE4TKMBnyOQRCdJ6KRyICUMc>t#V0EBz6c>UoCKON2=&sW_FC;LZo~oUrc%4;m~vD(f{3Y3Jqt@Q+g4+i-Ssbx&fIT)ZqILQQIZ?CMi0Yn#YQX}xp5I63tVyC)Wz;` z6XDjG@FzZm7wfGlHU7jMne>Ue$HRZ#`uuN(A`9bR7$O53Q^Pu>#Ot5t-(TrNTbd?G_t2$wC zMjj8aDub@}7T!Od1%xQiJYGK&n@!1!W$_d!PV9_aED9MVkrFg?XD0bX*y}?1yqPKP zj4a4Oo%P$uAQuq20H9kj%)M!E_V-?mj_paClw@Q5B*C$PU>jlbGA39_m(>|1NtM*% z-ST+NQ6+Pv@x#&>0E+{=DnUdeJYDcF!L0o24C*}=A^b{9!~?PCG%`Aagzv=OZt{*Z zj1nNyQtcw+bqOk~2PPC1-5DI>>T@^I(u?Q>c-brHUhD#~e#)@#ORqY(&C}wlpCsI% z1N*9fogJEmOl#V(N%x4JYz>Bi09!?bhK3y5ZI$zz!e2%39 zsz!H8rdBtvd_-wsem9+3QX^~+kcsxBUP4e;1~DO~iSmIYL&cuF=F;v~U||9GOm2IO zy4o`_)-R3^M-=ZzheEm$w6q?)DEJ|{KIG9XQWm;+#Ma3ZHNHI}E`MzC-pe9*c_SOO z(l!DWT8XVp0>d+GY7>pUVzc15d1o{jtjA(gnVh2!HZwW<$wzn>!UWNkyqSMVLYa-q zTX_iIj2i=F0bO<4#sBU^1O~E6vz0mS=o0r8^vo~=T8JdKe(IE3{{+oxyEQ(Z=l zL4L85j@Q`Y2Ui6rX+W!0*dX#r*42xAgW>Ci2*!g;s)0j0RD^A{K;YGcc zdlbBBzWh92O7k7@33`@@zkXjks+@Oz#Djy$HuLia)i8j9f6}r%F#o}x0heccQ$-?f zNjU2C+jKcu^qCW=aGbtq<&aVo#WxW6!Sk&Ht~yf&AW6_Eo^&;H)H*%za1pmT+wnH%4)v+@PVu~ZZO`3evrSVk1d?pyd0 zO498K=}uCVUaTfD53619rxAw!rCk^Ds3PvrL91x_IJO0Bn$`aCbH<*t==DGH;&n90n#dc5^qCag9R3-7BKlkbqGMhjLwHxKNk z$0cniWztjR$fBHaHs$eiAqA!dGr?n}s`ISaY1=xZ?D(ZrkBU2>Jfsr76)D@OWcf3Y zPgM7A2b!m9f_tlcbDZDp4+EpS1leJz2UPoKPW8ks0~+WgIHAsL2?JGXpC!_Wn(6c+~X+p=R6UfUXQ zJ9e%!O{O=2A~C8qcI0m!cT)Gm7VW9KU^e=edP-L|SaLXa;0C(d-z__R%2ZjrceNR4 zSur}wymGY1d5NHO`1?kyM>Z*ol5vtV&Br}(s z7^STpGYZ(NKc&48*@~Rjl)Ofmdy`IyVPYCgv&?)`G^1%j?aR~7hUzFvkz`fjPOMz6 zf%n{AqQO7FbV5%14?#X!$+4wLnb*1S!Vw>z zov5QhZ=!Wjq8-0YoU`tX$}Lu$Enir`;zh}Wibw8Isf_2NZ%`yh_#%;|t@goy5D?KV zr}SR-sn71nYw@5%|Ksz;m5mSP$4EiNWHbsC3MxwZ7Qc(jO4S)X61yurB=H3H%?;VG zPy;e@b#KG<#Lv42`vM;S>9i*XsCp5(>TRlU8HY_@e14Rp?ge^XvI~+ducI#?(C^E8 zg;5e!(Vi_Fjm1ySoZZsF1{eI90_iY`gP*QZyx2TQqGp-|z5{znCPA*JwS>bJ()L&% zcklcfsLO~?#JQG=mMLNcDbdW(&12F_R+r(2q7nY)ugY$N+or8nuv?@weiBcZQqcYVE-B1X%&}iYN>bCcoBp? z$%KJ;7Fd$JC~SVW?R12a$QP=bj9Jd)8hpi{GS%QjPIIDCy?;igUUdK3`y3cvj2L0&&T(h41ND4Otb@TGV=X$dPyq^i0T7k1WA4r> zURuO5|89XfH!s43yGKXK&aPKuj5s#qeeh%lMzNGxcsuw}FyKkSg+Cr%wU#S{R^@iZ zOB8x`{iVx)PF#1fitsGKqqVM4NE$tOF{>#wD&}zIvq8_vk{SLL^tTrYmoYcR@8zRc zPEDnOq}pq27~`m|Z*RFfO+eTBBJ~_l3b@tIs)n`v`+$%Jzl)85{*vSn?WvSbel4nNz1`va1pf*{Kso(h>6 z0UFCY@0WyP%P0Y4OPManofp4qZzFt8kGv7&UGkRRo#Hzp7bWI3s_W*)g`ALuS2*E> zbOLD=IKiSVdH)A@XBk^bmThS>Gcz+YGcz-@)69@&w$sealxAjTW~bRsGc&V2?ygtW zHJa8_&-@r^=1=JA>gXu$ihDw_zu0Fjv5#3>dHxwX&dY@<#P&gB5r>agSD%;Yhu`lV zf=e=LVdp#C5WXsvZb}&`GvyMGjVa<2jt-^BZnc8y{Vk z`zhd&>O;=9E(VF6Nv;!SWOf!t%@mJl^_Eiw4c-9uY-OSWtsyF_GI&Y+TNohoEPV*< z)Gu2Mp4k~wjlw%ZT?=+%VUa)^?duT@mve>m~}cSZ95lc|w^6e@qOfZ3S;(dz!sg8t76n2q_*yvQ$` zo4@K}Hs(LOD*m0LG4o$R=ge$>v%L9Zs{f<}QsuP85rmlleS-G_TS4I2(x*1jjyOG1ug2)7Stu*|*P40l3#h4mYPIIL^s=D2@^AAu?vL&t?P(GKK+# z#O)VI8@iH9F$TfVRS$S}L7jl+#I>(ygxHCUzz`xQgcOlg2`$!35j==;q-a$HOyUvy z^mg|s^2tm!Ykeiow*844ZjpjL!E!v@%lF)hFQn|p{c~AmOD@-3YU1EmF<7&3C~}Sv zn@a1~{*@3Jpe!t=txPOoRh_OvE0JtFditw3+z**pwnP4{=dXhJ>iYQu&M?|ScGug} z)5F{Q1zqa{S~g_;+jod{>g)C1%dOE8br^1U5RbcxWP9ocjt(0!SaaL)Gqs@Ma?r*= zaN~QQlyA}v$>(gG$YhGNZmYM}Mtv#qvl72#JRsVN=#mb!Tk^hX@qqni87i&xD1v&8 zXI$+wFE)L{aL<9=E!$TsN>kL$v0^-D#MPw|6Hz!bq1_f4@gBGP{xeX9M0(O5O=(6g zF7wB`;Rq@7nLl}&6SNyoUx!JEYHhgjsAmLX(GN`~VZ9NHHLC)8aTe0w`Iy27)UAP! zE5^LO^&g;ot}jOaX4SI)1+G|FSpLOQ*jWC2z4rHN`H#W>!C&ReUM7Ns9qFrD-dmZm z=)|9a`8B__cE)yQcs;6zJuwk*xYON@=pkKNF;-TkW&AB8`yx^$LB1GfOYfqBw1hK3*Tggb~x6 z=A2zS2v|u2ME~u8KWq3Tbp7>lg&lvsdL@#^nyqu*%w&+wZ7q_*ft{$ zHAA*})f!DK*q7ZuB{QoYf+wYx+FzoV9*7;Wn~LN+WY(~Ll7b9{fTm%BJHM}YH4kiR z<(Tq|?ifkWi3)=u9vSjR4vcELqB#!h-O~>hjqcdWyY56Amg{ic`Vz0e5(K>UXr*#W zUfef3Ir+rwUWYAo-Y`BKo{6x;M#nR_Xe}x9+Q@p8VHAiHfwBaUpK8~az#syk{J+ZU zFXYNJxw8896)DQ#!H16EaFZ^k($9nQxLxvB&r?AQI9$Nw#9`bhz`oCN)wmnt7~$73 z=2Cd@RZ5Q@#{oG>#pDWMAW0otc;ZzlF0^yR_tVAnmFxENMw4;Si2<*h*_g}?K2J;; zO;bp8Dswlba2uel;|7^_S;S$1LANDY%d28h1o{&08r=1OuA6h%Zywi^Nz^t!JcO*& zTvd+C`I6#N-1=E(fLDUceGat}jOp0VVb5WM>B7N`l)Q^@Gy%guS$Mo&b210Qn~UxC zao7J02?T~X*myo(JT(ErXx~4TSZ3Ze7sb72Kq6Lj*~}WR>p4T4j^OqCbqxf? zq%Z)HfSD0x-#UA(hQAs+uz4Yo4@L$LjQA5%=7ZrK9s`bFkmwQXsyFbt${=t!;>dT) zZtwdt!O(KgBNz=4I$9i#a*lq;JaHvd#PeCjPFbubEC4Y`A)gtE{YaS z`Cb)ifL~s60gCX!cG>qubPa`lCEu*>bPb@`{AtgV0-aCG<$nhsk^IH=s zI_&tk;w1qr9m?2x{7A9E%R3Su~Ms3LyuP_py4w zG+px#n+siqANEiM;8o)GGWZv{YlQ7lBIkoe%6B(e0sk6^1C@PZbhg4e(%5$O3(bG; zr3ZXML}Lnh@#+FPKRm=ed&g^XaohL)!CQvi7=b~ow_A4hi!A)<(LZ?Sr8^0m1vJOO(@Y3_Je)ydFIehLL?yn?_b{HBwi;oUg>!jsWwM0@!kh1x}br3^# zrT3u$EL&vknRA<(OL(i5{Ehp|2EKrqowUe6B@9^=qHRY8-gP*2d7mXD$bV)9L*3>p z)e8NR%X&ftYd3B@JBhIM8Vxj}dW+T!4T^`)Ys>fvXfrJ{O|oOhXWm|3qDE7--OQ(0 z4NjmhxpWfW_k6P3Pz@ZZ2>|K5%axTQ`Fc{)hN~WN9CH2MVrvDr z;;HRwA-AKXn^#`DMZ(31hMQ%=$wkm((Fuk2BR5jwBi+J14fet)x2ifbX^g^#Go@b~ zw~E{BSrE zqjJrFh41d8fnJrAGet03KO(_UJ_)~fnb@k0eZyWje~t~uXy8t4jRUS+cSRVuzs#3F zVa`}VRtOLoxwr;>r+Ar6vEDJc5rU+lhd}qt>H-T9d>=eRL&1J0(o>11Bw?EETa4BW zYiKRgWv{d3O{iGm9-pa$6C#oYonh!5c|vLy4!5M@^znNIiZGk<#+FQ;(*XH2;(3UaE%^s}rJKbfrV=1WQ9BKhnrS~Y?{SMPxCtAplJMp0#9GQKFDK5n zqECLPJSE;o_?9P0LLe00r}dP9m!pcRD*@QyITq8Cgs17n$RdMiuV_l!itq+PF;3RM z3~G@eYy-e16%|{?Q<&COMrcXy=my;Jl1vsWn)r?c8F}Dvm@FR2rAbg3Ifn_AM&>;0 zxuw$Z6c@E?$=jR0t*k@T{*X(+}i8EC{y9Jm0wc`Ax!IoAeAz(S>; zX!z#fuorM%Dw*h3Abz-HpL{a0{c?NJG&X=%ld{Z^fe7C_)U@p?*q` zWaKut=q`-{%u4`&>m><}g-OS!3U)^o#M*enMKF3a9)|W5z@#61bW| zv$ODOjUi3T0t;t+l{AOF3;lT*Lw+$Ys)HQ1Yv&q0@7xhfM;QCN0+!Ap9=ufGPaqqU zG%&lsrn_|^5+*sH%b=zbdq;J(@01~N)H6-Vs&c2Aq^<1bnQHY?j{0hxAx&KB#L(Zg z$LEWsziWZo?2&z2OU;j?rd)Nl3m0n=SX)Gy?Ed0y^st*SHVJ*i?x_SR;K-$YwSdQ# zi(+&_+6T+R*y{#1i^a!h`NQIwl8FJP+og^_xV6PHz*ebtEW;ZT3r-0HGj6D=>)7N8 znuWCuoB=tc^e+RNtdDLqngn~lrfU0oT-7!|0wR;_4OYVBB9<{dk&$kq2r^m;_qDkw zpHW+{I^8q-CJiVi?-6mQ1d`pe{;opkX;fU^jPE% zhb>eXwG&xfl2tM<>f#IT<15-gOilhkd+woq)$zdOpW2MJuQv^PjXWmb+hBKXb+Qkp zHSn5vXfYmp_ALS(;5D=L`dec5kB4uZZ*T6jFn@SnS85jUfoxV)D)L{Q18?&EH!N$J{^VC{nV!Aedt_)M0ell#FrM z4Ue0QYln|~?P>Yvp__Whl!5s4Fw}0Cqj;PCIMdpIP8nMRTY2daVC6j92?P*qvHWGan#5BO1zn^nz^`Ii6|*+Ba_gS{`5FH2Lg^$nl}&QQmm#lQtn$7rK$Y)sJH%6 zR#eDvaLdqV%0$kmh2Q4 zyFZXk*?cNca1#m=lCMz*ssyLWR#@7*w46({*Uy6u9h*F)!0w^|#RhUI$P(WeWftO@ zzb!(rT5M5mV{V5+QZlM}YO5{fvU}oVdgz4g;EtYXTFm=oEZZo7w1n3fC#^DhR%)u0))DzVHu?py8#DxSvk^M0D31+7^geYNp~12iUJ^ugw?{ z@+&J0=~Db8MQQ@uQL3m*t&|u)1@lj1&2l5+AWL2378YvyeDJr>ro^=wGbWH z{T=fltDNS7_3!o$K7jn>u>;31!VL+dzoeB6=OThvy8j=df`piT72IrLJ;!6yc+VzI}#99%~F#hv2AfHf^ z*c##HtowPp=bqH?`S zR4a|E)fdZ#F(~L;tq{h~RdHycu0p$`zO0<-`NL1ZyMr_De}i$@m^uH;jOPDR|Iz=G zX!Q@O^>?(&{?C5#{|T+Kv-~OLKRkN=1g`8Xe*)KkhgKOGzJTfsYy)&4uI6T`tgjs7H8!WwEYE}=~SC<((Y z;w3YLKwk*QUqmdHiSwKfCB;s6%W{WbKB`&Uvtoz8W;3RKT$?t?@l@rz^{FQwKw?w2 zIq^|7$V(&wr@6Bb<{H>a3iXbmBYI5+!c@_rVFnM|GeJGfny7R!g@skyFmXgN!7->e zjxN`1iVcM<`24umqQ*>Q+5TaAjKt(huw#V8szCDt@dloufET5ozR)t(pQMzQNeh7! zv!p~TVV3Pz77S0B9a9ZRhZLAnRWGDzV4Rd#;I)=Ppdyr5uU-Tsb)iTz3NZs4v;uS_ znjwf7IC8C#fl@C8VjeXDYQCA{3>=wtx7HDf1(Qi$YbK0Fnk`d}7c((1DYG#W^rUH( z2?WMeexe_u6j-rYFA@5j1*J(Y=$U1@P&t-hSWgK7aP^9wi9u;oAWY)0Yl45W5Yec> zuC`No;^Ng{r&4MNy=}74iB5|))V{0XHZ<)?nF$74MqL~f5<4mIxZ`-ag;B1^wI%)| z2{7(u?Fa({(Fnj?!fSCX*`ck-fcLNzI=i#(bN|P6+fAtK`Rf>dvfs`)ehyz$jQ)_D z3c9V87cTzi+v(cV;U+%6nCKMl$iV&GW9W1a|C_I+zH(lZK5+`KCvC;GM6?axx%&yn z>Vej<(~o3o`tKA=tfsi*osTpvk9i%ITYtKJvk^F>DJ z8wvc&xaW9S9HnskKJp*Z8%rKJMLQY3Y-CQ$JuM3B@%$^ii!!`)$QR+wd+@G@&liV3 zB6XYhSDs|gVKlCmHJ^`;0Sq`QzTFIj5*u?|#nN^tT>9gy7EP1*gpPkPhh|ptddnE(U7KS8`d22G5~fBYU@M0GDO7@vz*rPF+!8Udowa+p|YIx zz5KRBw^CqMLqDV&Z0IG3jnbsBy->7dYPsvP4V8%+g&Ol!HzoM(8`0W-Vp)kt90;x7 zG_$4*u)1=V71@)SAmujSp08dFE`;@4OATfcJOhp~yCt!;} z(h0-Wz#02C@6~&7tSN_u0;ORkJq-)(#bjJJRso}g@tzm{R;>&ebJW?6n`SH(^-Ykf z6q0lX2Jtgv8xVGUNt{8@C1Jm?3E`AO1Ee=)M$H&yEHZq!sij79K{f;3kTdGCbR>m0 z*)9aTxujyCd{M8AWHLM&DkYdtzcrn$+N)M}%ODVwckINu+?f=CI?)l6b8o0;tl=q4*~oYE!XW{ij1UD}ZfsfeiF z5q1n#$#r4$jNRpnIAvl1pg$g{_V!UeLOy~mNLLR5L8|;fybWw6NARRr#q~6#v?GiE z5NDfYz;9JCn{luZF0RzfERCUDjeL%7XfSHzNB4kW+#D6Gk)@YvO!jX{9I=5A!UA{t zzE}fS&)8#vSKE&%2-_tFs|;^H!hQPn-to%fx*YEGIo|xVQYiQ_84|Uoq7Bx@P4CWG+B!ayY2_*w;QwRmmT2{*5zMiXpra5~wAXBioHRNrt z@rU?+_-}YB2DkgSMflO@^^6k&x3E zOXx1Vicr$ekrV5dDuLM&W_n57muE~4_j&VN8;^j`p}C2z=a9h)n&ETTw8JR*wgM|P z14;6eagy{a;4;G*4+lyNY|LUGF8f0~l_(12cf-T7id-J4_fU3IHbUNO81JD=y*zoE zH)mJN!jN)Z-=!s2Py0{;6067ssBOfp1+A>il@-S*v|&!)a9`Y&ngJffI0-+(TvJ)X|?b7WpIup)m9Me1 zpRkyn5v|jWCVL!4ZdaFY?31)FalSZj!NRoUBsOh3KPf;yfVa7^RNvW-pSTc z!0fXoQ>H+YNU1h2ird$Ub`987=KNr1c}h0|%o^ z_vS@---AhKPYceG7h%Zr8wZbnJpwR3i)f<%On=HvGyaLWz+tf#g&NyD6WfL^at>zm z@Pp-5mX8T4@Rnl(g=|l=s;Qs<1-#&=h5og${AI7-uh`@$9n1H0juC2&th{i8p6?I2 z)$#}N$Y_XB^lYwNY{5!8pu?QR$O0#RdY%R>DkEiHsP@;xQqDU z!0UO?>FDZYi;VZfHGX6R7QF*8lg4*xo#(X?bjbZcw)U;M?^JD_wz+5{*pz%yzB?rN77lx`FWkIN$k+`h2I`{QE7A3ohuhMOJK~* zQGC1x)x>I)<^za^5QU%~f*&3t)i8JXkV4ZT+J7nmi3QGwZ3r;(IPO@X5%GS1_<*>f zg25f#l%6`g=9y7#)XJ|$-qEUPOh}#v~*yHy@+OJMG86=$BsjplViBZ-DlH^kl zzYcLVk0_iD(0+6#6xkM91h=d<#1b`3NHTb6h>-_ z;YQ40t`>^eX$r?pey^Cp?CT9tXNpnZanA^7DT)x{rI?q$Kz4OVpAqfP8kz>*k~cq- zY)>^pP14~2a&MzE_dFEII-O0*f77r-m*-pU^|EA2$%;y#g`K(>!W}{@yV2ezwUA6aoC;^wyi_NEtFA1O%dxTxNA^o_;F;4Q0c+dV+Hw9_u%&B zVQ&3}$uYvfkn2~POIv6VXXjgJR2~30MU~{ev|(Q0wwlnOqg;jLz2~_nK;)7Gs6{1Q z#xg}`LJ_MJLHTQCOvi_XEvk+*2=nAa z(dvm(9j*Qn+@Q9i3&Ub?TL*cO_m7ZoBF#BR56Bxn`D@+uZQm)3AKiF5aJL1eZk4)M6U+BZ{0TN9(9 zy-c0yJ+s;e8RR*=+V_fhF}o@hWshIq#K2+&eKy({%-i>m6Ki(wgx;PY41q2r?n{e_ z+IEaGtqzl=ugepa3bwa6_l|M!iQ@Y~bE#dXNM~oqWq3zR|EOpS^~!R?6o^m|Z|KyM zkq2k*XJ6_{WSW)q6$nyqKV0|bCSju0!hEp|GQ&PEenGrjjmXrS5)Us_p76n>zKh)X*2p zSNPFiG*U86W;p)tF4Jo4)7m>dH|gA4qo%vRM|ER2D8~Lp#!vtf_7i+e7Aq0)=bE&i zSc&tuo<2^rn5D{XTa>Oc`=6NJnruyLEtR+}Cz}RD$9;&g4@12iGxY(eeWpt1`vj}Z zLXCO#of6zq-Pr7#h)By}-8?#IB#R%(&iW~$78IzLw-%bn1Lrl(oxqO$P@DWEVa4t3)k=XX20L$BE^ZNCc9^EP=j_G4(g(T7G-ZXpkS84{!{R0TS zURm*f8w-NBzp8XZnmmr7*owpDCqvoU+DodJE}MdpI*+@wMH(p_YtiJfsyCxf92lq_ zXn;cYt5W5KY~%iv+}>h7|8-eKgD+Q2ywm>DJKSvXnY#oM)aK4a1k zXq;Sfz94aeaZXZEaXU-|piM89CU{J+Lmr|HT3|Ie-+~fLq(HMLUVP+I_iVYnFEe)md71TqS=4 zAAh7zoqq#H$y$&~&-(Nn^vtTmA933s%uAu|v75IORh~ z6Eh>sZjAz{;r_ zWn&dsVxwBm(JMWB2Qk&via*^_Q6sovuHDy&r+?1aF6KC7PlhC6mBrHgg+*4#OErP} zx$>kC6JjOEfyh4F2l-(K6W6RZmMr=I_>uJe}nH3JWw*YD}x8 zYkYB(XDvXA1~0G7@vx8@QL@B)xnE+_0=S8<8ZE+kOxaHP1Ro)vuKqWWnVt1tO@aT% zw()qgKz~o?H`wLBdvFE>l{{Na@ z$;tR9Zsy?l8*Tn$s{epo?Wb~2k+8!A-GX|7X9a)P!)n)EZJ(0xPg@?E)9_Y9KQ+ zSVAB~Bove1oh4H&nH^oCkVU7ysR%pX^-^b47_5>8m-tFTFeQ#@KkA&%#LZ87{l&wa zC43!$fX2u1m$y^9ubrbG9}ln_Iqg8A9^S9F^Gnq^AF8<6NuigaBQbW!a@J%^zmc5W zc-ygpu}bSLo5_**6LX^Wqe*!a%xxHxN-_K%44Dseoq}g2);&`aoA()c% zH|-o`ySNRs$avt7ZZ2ohhOk6y!+iH8@il%NXbKXMQ&(PPe#UNZ zKc~J$tM?%J)Nq64+5N-$aUo znKQb%lvT}7AfxYFv6uE1wE_QIsrvbfAgKRLns1_=EvJ0tp3(7)Ia=0-Lv`@93vUu$ zan~=5*%T(G&(dwvexa1-`pg~|uMTCJ7xrVAHF5kR8bA8_6JYA7Nd4d306EzI@|@>n z`TJV^`@5k(Jp}(4{2wgF4IRnyJ+Xf@OCMQN$X-CBz&Bud-A)8@V&w1`_cQQrbv<3m zi|o;rCfh#Vtn-AEmlj5jXmn{XslH$GUShuKUgaoz*-vS&ecn{bZ(iwT^tx}0(ytRF zDS~y&>DRrvfU9G^cYao8=)HE^zkb0>N9Uy*C4-LUa^BPYvfTgTVz>zfIBr5%O zGlXD|l7Z|WFiKM+)-E(N%jPsU_G*}EbcCq5=LfWbWqE#^)5pnVAeh2^Q+BR>ZS{T6 zBP<^v08t3blqWV<;v`e=tVJcmknmm1Sb2l{rbHBwvP^E^{y&YSFF$ zb;KWCHdko=O{v`r;8r}y&goF@huwq|=_k$X_w2s*u34;+u0DtsPqs)S?n=fs9wl$x zN@!tEQxIGyKaS-^50N2n;FYiwshta&Oi6Z;5(mlh?hfnSXNxw)^DOK}6uwsub3sn; z_r3Q};x~MJ28-#1ndn1|vg;yqHTz7+zU9t>%C-Y0ndM*)=p!85du}EY)CL2<^k(;^ z=;uq9IYaM940VF-KFmK{XAwO0ZLVDsalg2HZXfon`*I(S!PN0V- z(BSudUQ4YE83#~ZRbFk{mC)ZD@;@zs^|4WxL136FWw3wU3>`pLX^P~bLb|~krmBMB z8tZusR}JpOf*OH}gG6MIW_RBKU99wYAS1XWopY8a;QG}ZMgqinFx!yM6(U~7o?s=( z3^T(3gut+Mp$E)uk&&%w199Z~OKoS|h@27?;c*C%lcP#(XlXZLEE#9 zb(J8(-RDNo%jhRpJQ~B21ZJ@77eFg}A&%kAG&>TB*#p80RD=PeC7$vX{ow+^8AFDi zfRTy~^R=V#c$$yh_I`^cS3~#~!t)4wrJ#qtZH&ImetTt$V~>wht3IZC z>_z@|h3F&~_1qhuNs6}Mki5&*?^rg42tM4w8K)crnN=RlLHiV8S#)ggIRf5TOA8X07 zrOZ8u6#|o%;A1buD8NStJ>ym^oile3i%&3InB`g4_R(yT}*OR}p8kS&8{sEPfn&v!eB z2Ee|eQ^p(+ij@2mJU>sh!H>jPgeHbN;GdL$GLCwx&i&Bp zz&8-1$@na#q^_%gB_{2Zw*6(HhM4*K>4jerFY zK7JW>X++&J4ciYrX}}us;!}8qUZK|X%>O6_u0Qim!9ypPR7PplT$#xZ*7Nhl@mo>~ z!#7E@B{Vy(UKp%&vM20}-ayT7^nMMwblT98q!uxDLOp2N{BZhA=rr*$h$U`4;t!yX zG#~JJj=`6I8a5X<`m$&?U)!mT9(IQ>L_u%?4gtcf1-*v2opW%8VC@7D(jC`@KMDA# zprcq3Q38|{9J(Xp<9*3BDx*8eDAzg4rN>&iHT^+qK`L2` zpbJ8>*8A$jvSJ)LhDk!nE4Yd^Zh=;rks1yY6H711{iqn$r=cMq#%Vq(-r40q)5}lmQ$>Y@R2pE{``CDc zBZW6&8|e%|p<)+|qdXf6b`H+p`%$CqK*clQM#@-`45)45xbQUG>gC_KkoaWis zZXazn4@iaQw8DE?Pv?@j9rEOMd=k+&y8@Jchgclw+34u&@6VOHM6#?teL`rFgoipz zN|y}x2*wuQCv@fJ;LSU5!=lfKJG05BG$nYO_xit0<(MK5PiUjVc( ztEkNlmo-_$&B9jQ1yV_~EI5=T((Al1D51$bQrc{Kmz@8ks<24JZN$nTw4zM*y>O+4 zoa3m9(`~_rd8{P|^y$aFV&^`)jQ&aMNrPBA_-eiOQC@g>5m9(ZK8d$IXh5spM#F4i zMNacJRcc(tf^vd$LUopkK65WuZ0avu4PKd^uW1W4bf6rs@TnohMr#-h_U!PzFQD^~ z)Yx*qS9~n0;;SQ1OIU|$+$_*me^qq8?^yyUJ#FzRPT*UB!&$AlY=8u+-*dj@1`PH%_Lo}=(zmlpJZ39GS94wv-qrFVE zj@;4j?PgKb;X4za?R_y^RK@?X~i zmV1&JoHOPFbi@n;Qjc{uD~Tgw&Y*T(={l{fo70px2$*DUznI`>W-LPtC!spPLZ{{# zj7LftL3QwjprF1vM>sQ;2>GEB4HOV~I8fsvwS2W03UY%s5G7D<-qfCk46o50_WdM| zV?df3W8Hx&P?X3CL<|~jt;-FQFHVdRQ&g5UtKbUcc65~W>T>?jYK!xMH6$Kbw})>% zb0-N2E&$6*r?k|Hi+yYeP^EZ5fo4urQYZtMo-d|Aiy~gw^ns-u$)>wWb2MVgcc`wu zJy0?TQI!drk&q55X(4+<_=udG7_p}M!KOo+sEhY4ox;(rn4#)Wj#6@Sde0v~Gm zRPO8pK3}Ny}s8MP99)pOu@Qk#GLwGC3@Y zW=kX%lvX76Lh3lGkS;A4GGcqJ66aI~6?0+DeLdf*&qhay^!5F3Pi(7pV0xBMH;|_) zJl~!@A98|+!w~=rQFcIa6q}aY<>yTYhXVT(CCf_frjrNLdXQvf#&wp8#X6z- zLforEAx@U2SOrz`y~+s5z&w3bkQ5xUOEfPvN-uCW&;3m6dvyG!1z+*I1SAAcpys*B zUoKSk{612AzYOy|18jU~rQHsw&sd(kw!>3YZ#WLwQL5ZCiVJajB>RIf2znzrE+$w> z)7&rc&ct7!7g_VQGTfb>i^3;E3tQMi$U(srhT4jbdw9YI2lYg0Mu*kfPq62zA`WjV zE!3ON8(R)}TlSVws-tXcdkL+z4qE1IKA}%5e;3>wbSSBw+iG_1>zd+Q1QXMhqN&K9 zqg+P~^2e>^>N|SxNCvA$d@8q##XUdy?HH@OCnD9n-HeicGD+PAQs}^EKKyvQsd|Hx zQl5%Kx7v91$M*8|lWI-Kk?qMCoTq0=cC(v5jcixPy#9$G##@Jor*fM7;7>jk+(*kW zw*66*QL(G-r9^ZapuR4?UfCovdob90^Y-Gs>Y-{=9e(L|660aTET^<%m2+P0aYpWH zVXjtsqI6vL*{M$V4iJ4881MeB>!R2jnHF38O3#xn=fN}DJpUtDYeNf=e8F#jhrj#u z6a1Gi#pQXOpqp=|vfa>!_^0t;NAL@-eLt%3F1WCj42;|9^3lma-!BSwIQ0Z6&Tx5RZO~@C{45?15<2<h&{hN22j(YOUrgsYH~lxv~}Hb>3ao+{ukO3tuV2#FKq z6aWC3K_Hk$UXol6_vSYX!R`P2jWaNZnskdzwF<$!Y@Ogvic>^a6|huYRcP_aVj=z- z8JucZz9EI!m_loBfFrz$5bye15Lw)AY^l%srTr`9=Szgx;|H{D_vc&iV@xwUgwt8U z%xC*j%JA+!w-!_uwbu{kf@~tao#9x=XzyT8#A5qt<4|z|=`4*MG7aa&lfG|~WAc5F zvL%;;%=YEkWe%HLq}fNI@08A9c)LDa(mDr2S531diG{1_4*fSZm2K4V2{h)GWy^~; zYgE+fk`)eu+(_EY;&GZ*rB&V+nzzc@+xbD;wOM6Hy}4IKZgiKSSA}qYWgqd^W#;(Ti-3Q*Z2vL%Ke%G&IOEa8!{@*C6%PR{(NA^#7zCtgF|Q^^ z0XQ)@IcxoUuzNhOFGDNHUkNSFaHlV5B7@Z`FR0KqO=_2keNYKVKDcMp85lwkI??%k z5bhx%PQ||5^n`*HzufeAtKdyNdA4*p#gL0uy?oy^LAa%GU3xIN+88w~K-^iuD2tHs zKZ$)_w1FrcuIxR~Hg%F9cAkGm69_W&#b^cA?K}rQZBuM}*x*w}B?2g&FzZo6n)iJ2 z`iwuws^1XhfRV;14MBhONFJ8zqu+3Eao$Gvo_*K>C!0z&jGQOm-#u{aMqzD~ffmZab^Ufpufx-&1B@JFdp2r+ z(@BO`*yfO7-;z3XmdBvYU}io-ayE2EX~wrRF~v`zVi6)qAw@LK&fwn01HaF?aeJu; zRz+a2nmO@l&1XjnJd#Q-lWXktaIRU@HSXurKK8k^)#0u^6&m1Y%3wU(er1+9jp52( znkE915J08FE+hWC8C(elaJYdV?c}!_Sn_jUj~7vmXS}A!AWMfQru{iETH%6uLhlQ5 zI=?lgm?R;_=VltXQh5n@w> z4YjS`aHG!&1ClM`0ybKW;x!Xcwx#)U!g;=ERwl(4Gpbu%*%!inwUsc*dC3rv8 zac(S6s1?z(*+!>x>8_{M(c8=E9^%?K$UH8jU_1{}$C)YwfFxD-td7InwQ&_Ak>hMg z4bx7%5K%eZqr!FOf%=NQvInfBu0++cf4SzZkSBrv3=O`sg`x~#%SBhdd%faBiU4mx z_bj|LTHWV^TE`pTl8A*y+|rh+hqM(Po#VlJInrb&m657(cau3D^@nuRxQkByjsd@2Xp^=Rp0os7BP~BvCq6Lc7pOo`?qOcX6%|f-dV}MdT9a zxXsr8An%=mEbZ1d-L!3IR@yczZQHhuO53(=+p4r}+cvw_*L$J=e?`aIyU%)`j~Eei zj(5H>pKIJ0QRe}(jj{#e99Jh&d6BM&SUeMX90c6OOLrNsq}ZwVjo*vMLKK0HGS9s@ z=?tqoiF75KS(`6+no@8!V>h!FMa=@$f=O3;BB3%po?+`(sh4U110hP7R8H5g5cG&N zpX3%Qu#*A&ymKB$n)}RS6dr9mo?lhF>(VS1g;(_7B($KGccztzEkNx*@h|5VbgBj@ z^_m5l%TY~GaiP3Gij#dCpd~wQ>#G#nB!>I>y@6kYRw#15H7u}z<&A9Kw8#|u=kg?VYH_cXhBgm8!fz(aTWCUV1& zIC8F>?iS+~JHi#V6qz195k&y~$g&_e(<2}Wf)sR6Byll@lX~|Op`}p$ME+bL{N1Eg z6Fip)zR4)w7^-iK=*EO1I3iCE8_7{vtcDyf60(N;s-O!69e|CC@hc#JTR-~ot2@T44mhF8|)7yYZTv}gAJvrh4mq>4U^VWj9>|| z)^oI96N(D&yocJA|NuSyZEjD51kRG5?{s?L|9q^0}L5u_Fr54FE zA-u_Bx=KyVBBenp9>Ll<*g1qDS&0E5#4bHY4z4=UDUrO$2zPgCsVxu&kVd;#i!cQb znt?@CV!3)>Nl#o>LyrU3ZB?`~V1y7NM;%H~#D1AZ>lq}G~a198(5|68Ao!>PEa zh=jzNFoWKY&8%D5iS5lLIEF7_-?EAe>z)i&$9$>ZaD-sb8$4$9bY~XvvoJa|2Qn1@ z_29xs{BC}(s)oyVro%sI@nfL6-JN)|%_t`oy4?LdrMg14igTDXLvYn9{Bx2tHO>xX z&3cWv#A3ti`MZAdlGj^!6>HP7+mCpoJfBju>l*BWPyI!nqCRi$oUPi5gn)KqEA$Sj}3ITcCpjlIOC9IF`&%@2Ho zg4l|aOM863Us!Y6jsmtsG1f}=sf&|HJ*H~PR~oOk#ogpV-h%Sw1E-u=pORT}D#kzhb$5?!N4pn~jAV-VQbjxp|>y=(na2goa zBIf>2IqC)1Z)eW@oU1igfI+a{N(~rF8%xwz)iWlt_GMJc)sI`*!xB{0<`z$KV+$o3 zoJ%yCJ;PsH2nA}$`Frm$E`Pe?4pPXqL_R22iU_y$qPrl}Sk!XlyToxm+y%`qc zC<^qXIcug#ztJ=B>P9^;*PQl!j}NsHh2xI49(px|-a9Mih84D?8v%H3HSbU z9fO-KUq^-7CS$4^?Oiz_4O$r9WxxP=|K-u#ROW52w>Z`qKy}!P3novbO>Nw}X2Fjg z1KYe5m78d;Y*MbdCwgv))WhFJK3MCSemTPZ%kP32SSpOf)SSkPx!Os9-CMIOqSS0O z8BJK1(7())PjIPHLk7hELww#2eoh4b%)&Tvx!>#{y>W>4HS?xGaeMvItUIf) zo$e&REY7($bAPkBu)82Hq>0A7WGN=IZ?pR{&=)F21F7iDkS446u1rKd1<8U7t@U2y zM~&24Z1g3;)0mN6L9`OT`)Nne2~}ckE7g+-<|hRm#?{O7aF%4z1NGuQagQ@?kTQ1Z zUY6weP~o1odck;=lhv+#a+c)6jyzY2#V=1S7uuxz(TfNVRFhKWR2NdPMz2gp#g1|6 zpxE^8L5FO~Dzetm zmCtq^8ZLrV1W)wS4JS{JR z%TRMtI(m8L61XoC7iV%DHIV44iaXq7tgB6o9vl}v#59a>($bi{V7-n z{#sn3x{L4ONx`4CXzu)j5z?IX1Zq`7u`Z$QLel#*a{VhSG*+VJqN|xSBe{hC7kYKf zxg$~&Nq8eeOZ>&yak5a#5nj9(^bAM2+(GMOWo+1}l~;~G*zCCjk14I~=56^Op={KC zH<`_XR6-!Q6mD_Yb3kX}e1vN1IliJ=Ig^imy^7|_FH#qajGM+{&gTF_Z>7!ei>bU% z_3T02eLyaOwXd(2yk{!f=60h#kza_Sb|USbA2NVR6UiSJ0X^{zJj1x#ZiLG4Itd3v<J!JBL?Ehxf4(ojeUmh}jxTeTDzz0SrvYEaG3Nn#B%M-`jAj#wAGCIqyIt z+%F|8^U}hE$ZVma;dfX(*c09)cGq0^(SG~mlaUB3j74iQc(cRm=&l++Dm=C+`TGou z(+L)}r^;?6WZJorYeew#xA6nnSU39iH$Da&!e4G*M<)kked~XYx@>=U1OLA_R{#H= z5dDP-{>b0{g%EMj|6M}E$o@|S4F~=IPVL6X^mnxz&8}+Cg=@c%h3bRRcuDuLBku{;Dg<6D-bk8RW5C2q|w^ z0;SO5fY^s(-6t1M1hAQMLW#0MTH@1>~lrH}c`Y`?-*DfYJKOtLvnvoy`RUYy?@JFLECJ|ne+BRh!ePt?O>F>rAm!t;>dcfy+??5k+t5NE0!Q9R z)mho=(X4*_6aQ~gC$6173SRc(s!N~Gg=1Z}nn}IRn|ySe*fH_|zGHm#*S3HLtD7;V zI!^B|*XxJg!W6ix*ew_trbVhz@aryy&tsp*o==2eBRhLnk1IYcq3gHfZOQ<4|FcU1 z67+VTuZ3?dT7y<=yXWl-}3$7Vvge&AB3%bz#o&K-r4N|2%>kEoQ$xpw>NC_ zrbCJTh96Ow_Y2rAMi)TnFOJ+4!!KS~zg>C5esX?I0&{&hKw9067WNqHvB7yj0lOlB zi)@V%RN{bQ=e+5@9q0VagsC}!_on~6=L-O^8+ZWp-lg3UkqHVQM^TCPD^IZpV-LG; zlxBvABl$6P+DX7P-hgQDVro#LR!cj zyL0PLZXT!P;C+?fR36ksNeZqnJUzLS(1$0VWN#pb0?u0@km!i{4QA4Q{JjvD!Y(HP zKjvq)UZ)u)x?Zscj_l6>KUd37v3xn^FV%k3MJ=WRJ$66T!82mfDMaK@%#t|{lFEyO zStjNj{XvQy{$!WNdMKrr1Gs%DqD|^`e4R_1wD`EN8C)9ZX4>>zono?sDyE|>oF|Jn zT*NjKy~8S3ln@Z~a)iELaG(2J_2OWHphI_!%#ArAkm1)t#U0X9;>NS}PB<+=MPN=a zVs!@tHvkXg=Ew}8eJnw0Ryxqf;Q)Lr_X~y4jLDc(aqhvqnwBy1H+;p6_{{$9d)tiu zebL;Q9d5!oE}c5kK(p`|flZ|LMG{%D7o)V%3%4JIKZ!yt5!>Mu^EgXNVgk-JBswI@ zCK5`OTghG&6kuyK6S}WJ;&y)aUXB_?R{x!i0<2H`+F$bhKhi)_4Rs58+evki1_{_vS4+0t-XBy;`L0*vs zp_%Lk7?~d5uCvZ5H2Ie0DhBaZ?Bq(Te-q)X&Mw3l?1?pi&j0CtEkxNq`K>_)T+ z#rDu-YRJB$RJ4iCgOpvIV>M@-^`8^k!ck1O{TBcnfgH>O5H<&hLql}sAusXH*C!9 z_Eu3i5}{K#5mr=cI^w2~jXUu}5ARueG;fNJ?FprwrfkQ<@4oGnGUysGAf{XFtKOGJ zx_w@F6D3;f_+g7%98+`e;I`%QbSj4}Zq#vT6E=v)*HLh=w+W@oA+d^k>z0gR6iJbe zaKUT=ItmMZB0)vPR}6hpEEDQ+nIg*fIz<5Eout|kV~WETQ*EVp96Gfl@N?tdJmkt= zUP|52OB5!c))TMoCDFq(IEI^Q#_@{vII<;im6R_@k|#ACfutnsKuS+b4B#YbItCQJ zkwwa|@w7qZR>m||u=SvRSKdGd!UP{#W3v#!w>8A2I>f0=7wNII7`@KCJVWV1Sv${W0{1r1A1zuZxUc!p8~{VjaNuR z6t`*Beb+A?TO(?c!I!p+FPifCbvHq0v0T;_i9o|)l7>rKmA=^<7Gm^9Rac(lYEHEG z)~LbCN11^}vcZ8?T0e{YXcQ9Y*KMyZK~!8HUl_m`-OBa_9qP?+hk!&8JE^Adl6tGrB4->LV%05HM(g;)K3bsyk)w;&v9sU3<>N zHY21`Nu6Gm_qw6R2Ule|+(4{uen@a8-WKQEhEa89Nl$qWu9tkhLI#!Ig5GIV5?fO+ zz5o|kfFoK=_J#>l+p|Y38eMqwMkvB+wtTZbCwd&_Pu#%I|FelB#N$fn-oc}03%<~4 zBVLBCRNR5#HihQ5NvOiR;6k!D%B|Z`T?4NR&G|f@uKGtdvx(Se28)S=yxKN|a!eq( z7p(mIYz$q^_F!4y-d1Gf^c@l!}C&1b38R*#~}mq;j*QApSd zeE!M#NOI+*39x4==hzjzyb@n@XGc&u_s#+i3C@J|XMV z;kX!|i%QuGc=>R%i3C?QFL5J$2zhUx(6zZ3@(%dzk|{4ip)`}2Gg^cq@%nO=X-+&7 zq!6pDu4ny5-u~`pm?rlZ`6`P;))G zrrn$!y+0Vzj9C=xOL!-!FO^ru0@_;)9VPm!Gb-oZXDBYAW8Wk}#ZB+2pr>A~58=_5 z99hjaEWOCrZ9(xtQnZFXzx3u0ke@VtT5lY%*#b<8O>xPoT(;lVW4caq_yRaVufMq+ z%BV9IP_B$VV{ZKLchMXB?5_E1`=I-T3CIsg5D~S1_=}=>n-~xtu~xkx8X$}vS1h`N zA!el}#g{=NU@Nb563CbqN0~y!*PVhKQ)8TY!ez|ZKfC{+mWqXf-w)bUUl|-{S!G!b zB99jho3c1Wj`2)iq4r~ACxrj>2PdJlhlZl(JPd11f{W4DP2C_Me(cR! zy8R+*IT|Qx6a&F7yN$RrfA5qF5Nv>HpDqW;vMEiN9P$OcZQ?9r=%ZJ*8g88@qbxsD zg4@cdB!`6J;IPX`X#yaYU5mlSsku1r+43-qrkzem*6zEWfSgq_nt~mIysmH3rp*i( zbU&OiQF*d2u^~KJmI46Y^auRi%dVxr!`yEz=GnX>JJfs`Q|+R``EADPTeI7#ssp#g zfyqVExxsj$fu7sU0IZv!1hq3BWlmwy`g*{CH$}&Y`*z1*$DUr%=TQ8>W}2?a!(JsN zcoz;l&JMY>g=$%&V%PhGj1%lTU_7${g{@tJI+~2F2SF6hq-nKQ{)wqZA7it-QQMWD z3^|onHKeO7iurTaiXt5yd+B&RV`zp+jRPaP`fQPj$Xt^CNscebEB9Rev`@lyx)Nwg zqI>=dVC%*Qyf&Es^<)LZ-s)33x@z6YYOpBGnL3h|M%Un@HB-XXn6hem?M4z|DODO+ zF=jv{HH*X1iJ76?-45e!f&n||{ycQZIjV?`tw3`U(LY5Yme$1{!WAnGDOi_aq}W$u zPb%0u!T6^HP;g+?C|1&kr`2<8bjq*wLmp>v*GNd(HU=B3#^E(Hgn;t9(J1M8vwCXCfnZbxqyVb}uRebwZGdiLBKS8lb(Tqm75y(^Iz!ju9~Na zR%&}vp8!k&U!wE?0jiSe&J6kEk^zp|`xR8`c*=0?t(klF^2O-j?-%S-GEu4z4k~O6 z_k|u7)^w?}(RiHKSzuMF&req{-#ISHrFA8wD}hI9Dp9&(QgzBL->TY+voxM8^RD}q zuJQH-q)$@P4p&jipfkMP8BRCIXSqORuVL<&1cXo0O8pAg0lotZ#D5kjzjYB#ZqvD|trAloZ2uyf{zT%?n z3OLa-O@=^KBqTU0KWZGJ(|i-5v{;Q6lHL=GV?C)8E7Up;7`a5<V~M-@bt!b1d>>R%dh87c%O;w|s~vG>yURj;q%&byw954b-Y+^>$NkX2cRM z_!c;*D}K;T#0h^~IvaPta52YE#c?n}xkQQQC=67p{E-YbyM;>W&?_4&&Yhx~434HG z`bI`Yd5;TZK%Yw#boJ(gxTw*_i(Cv+@4{4*G}Tkxt|vlA(Wyi2k7+`oGQ%`9A@| zf5AbFf5&T#O#kdL;b8n%)S%>UXG}mRYha=1WDP|pL%_iPcO1mP@~6_|A2{f*t^NZN z(UibtA;|v+2L(LmqK>@z^tO&cDUh&k`H81|nhJ$G>8GonPT8@;sKVGO0t2!kt4$f1 zWl;1EA)~Zf`1_U0+Qr600#fq);AbL5Nj(YFX}^W?yAKWOGn^nq;64Hkj!}n+AEJgk zHB9!s;MA?La}OA9&(*5&3>hCj#_jIbrHf^SfvVWvzjx}$`ueR%KgJB#k0~g9kFkNt zWYp9chDBM4F#WAy)z-iQ>VE+{e=YtG zzRtg?dG?2Ht7~>veNWC@*$7~MiYozZvtkm25R?$Z>A3-~pnrRpzimn@%tafSn#5{D zAdQ+Vzt1XfGJP=C0^WRQcUKzy6@Z5SCjc!FRw8E2ei+UJW%cKvJWhu_pvDmP6t;v+_vtEUdS>&v^TRK&ICEg zzJ=Q-=f@>LrnnwiG=6FK>HZ$eFoWrb$xB~die%??`8LonIlzuL?+!XZ*s@m7MJmtE z;H|Nq-AI9+2m(_lf#6NqB7kV8dwT!xeaxK+fXvY42Y;9zy5p2A?!aTFt$z2rj(H)X zyVkty41hT=A+GL-UPL0h2A^J;aNo|0FW%0tpBD}Vcsk#&+-Pmz4JTL;fj!^i>Vf0| z;yEe1k0qvdG3+LLwwS}78ld+^L-6tboqf-P6x~DIwtq4c*SEoM{{$|Vy=DJyh&;y=fp37=7@r;=QdzU1+68XAG^OMAPf5mCuI-LSU^h8jAnw+n~aNr^yYBtk8SPj|OEjwgAKYmm9G(y}jC;N&} z)ufG8;bkey6!gdb!H4nqrVC$?(PXew!3cHe{ZUBZgzuO&PB-j7n&v#RD%7Xaa^f(q z{=`F>F2Sg(8_s#{^+FA;MT(H4-Wa3Xg9PFKpw)Ssb1->`gVaxr1(Qj?cuc(d!GxK? zZ7P$C;spZ4xyNn759Da06h@a2;Xz&Cf*P5MlZG;9oJhUz0%p&=BO|ng8O_w6MdQVt z%A>#u^@p-aIWGw!k{bqel0a%1N&RRU7r8&DRgG?KB1+4kKqdZxIyw3BNkVMHK}p(A7N+%DQfr|nzZa&Psp z?%3}!=1(<)C;U?^7eUzTVPAoDCocg6!T7+e>rIC9_>zeP+y+XIr=hUK!_VK;ArysI z9@44v;_2WE(oUcpdv}r4E@cH@Tvv}Oh6TF##75r_JhZLX#a;l3HHs`Qh#qMuwz4ae zOOIu}dJaa}$SF=jeN*+TJ0_4FT?{kOI~}VVO!(4{JuLp@RW4rNHmpK;6l6vNyvd>; zWj3nTa;@`8h1=Rr9CGP3882QL=^hREI!I_CSjbmFeFE1gys|{utgpu@p|mZ{$c(g& zQnovGQGs@zZ3U#;oE{kINbZ=PL$ii7=2a_c>M{S31GPai>bO`+CY$gTuU+(d=K4Th ztfHC&^iGx2gI@ukPc?MHtbhvT3{^1+He{#qDI1n5GO7+bn#@z9sKW%(vhbIF8AWLx zM;DAZgC}$P+VQ{=O3?<&Ci-|#OK_AZYGzIhYaigtGlvLE?KxKZ4WVPMY{J&#Mf>h} zSQKjIleS7x00~yhvj_Odj6GRIPJaJb>M}A?dD3sYNup8mE0PaTtpt@4IiMVLFS}r& zoSr8?Vch;5;?6DFnL7cg*_Yq9f_9N0av(Y9UORKk!rvJCx4{|E#C4@;2uL{@-n{V> z17q#&CL6<&RaQ*Tzspdm%92d?y2_MgDvUZ(Bcod?e{~XLm1qXuoXmauP=X+aHO4^u zq5pXCCAmiNEd|q7qvy~KDCe$jv0KHsK@G`P@fCI{9`9B&3M(5*a^)^!fH+{#6n8Yx zobom%uaKWg-?xN-I(TWx7G`9CJ`4%ML_5cShn%Rd|x*xMuao(!78@vXE2~Bb}RZR@{{pSUaWrD*NfIEKay< zWKG6nvCr#_jLu+FkoQJU=$>wA_#8@?LH+UhdVZDdvy9nErrg2XM>{kq>?$gAU^7v9 z)_#CpBqf$!@Nx}<<;1E%6GmPVQA810|M6o9qq{CmTx673CGzG$m0UnLDxj>=?CK?` zYQcQ4g?SFYvXlz}d_{Pr1Wi8FDT6qX^=)^_9KwTN2NiE^m|8Z8Br6)SjPuiG0?7O^ zLpg#;fyWvO;Xo#`(t$QFX zF9GPtTGiuh&VJZj64h0>#3Aw!KVwq3?G}O6 zSv9ZfyeR38e$~4<>3%aQyLAJR#R>1l+`GkJ2jB}$Wnu&+S>HLX4A7kW(M4rv0NMHS z7^krf?wUY)J{hNvHBkSOTrSYGu&{n^XwWWZ!dYF2tu^3tRn_8;oM6zZA`Ye>~@Z$$P6RM6{!yLRntq9TvnUc!|y#Q6WhF!YhVXj1X zi(V7Gkf~)pxgH*$>GnYHk&7#zte)0!i6<5+YLSNd3r9)Wd9RLvDu6-Qz<0G@uQWi% zJKznduhIbRMou=Q+!jb`{`HUrbCqyK|DPt8t`bL+L?|&n2F|=B4jqL_M`mUyo5+-K zpUr)3*PZ1ke{)ucQLfzXJYs2c@Ik$Aahv4;;Z3NfPe=2uOaX;X7dmElmGCw1?ss$Rc8NOPl#S?+ z3=d#)id0IX6uaq|u%S(2v03DezHu%W+zAra!s=DZoTC*$?v|uY;ltmy0}|u$hGMRQ z0aqy2J2+@aWi)nBnj)TRJ)?2mSx$qIlchn8Cb6wpLogF6mODm_UW)I}4r&`C@_sI` zv}~WK$o6zI+3IW(G<;wAtI9M%WR;Un9cI;NaQsSyQXS|awA~ryxfZHVb6%db5n2|W zspG|acSnp-lqnarH$K4Utle4ai?x!%Yvu(UANubov|xlwXdByL1%Zq!&v8%WTP=w%1kVU~v`Yz=Rctf}O_^-t#71dL44j-M!bYdP*`MpAiIL!}a#u z{j^?-3oyUJA)KiE>+@WU@g+NkiyJYGn%G2DWrkI__>48$eQ-pbVu%&!m189P8>h%o zdli-I86Z|K^fcD(AM`nBs8*`Sh94ZCT0A4B#yf-fn#L99Fy-XiE>c)CjtVgAY@hO+ zyHh}FI%C|qo7{`1?qh21n> zu(QyYoxJzy2s*zPhLw8qkOIq{V)(LOeImq5VY-Dwlx^_uhB6JB^4Cz=Hdo>w^Ni$G z5nuK-rLxUY>MPH_+|Zpq32rw-cDJFs#~0N)W9B5+dlVVCRp?Y@J?P+|gP96?_tNBh z00DWD`MWsBAgTFir^2Do;6Go-8aa9{8p8*|R+bq#ak6xMku8i^v!nwXk`US4Wq6;jGe z#=CxX>$v>FZdp~Kn=^^jw;iiDu{h(woZmRi5$+B0R@LgeV-`Yc8Fgs+|SxQRH_WibbgCF=M z$c^xE%$KK7x@mhplhO9g=vfxarVxWcU^UCyJku;PKQf<5525Qhp`7&Sw(ds^My@%p z*s3c_r}n&l%`7855=dVDOb(k;uTTM+UszSi&r0CVAC$F=V%+XW&DY-PT$-*KEj1wT zK>E#D@T+sDFqqUxls!ObAqfRK$lQL48$JC;x=3gR8c?*0o`9sx+d2YIq05*Ci{)*; zO?)Tn8N%kvSq(i9dzn#{NylQxV#cL5lf;$U%-UPI*-5q8$&F^n@3)VpXf(PuS)ZJ^ zFQomH-Ekj4EORKKe>-#j(<<_R?y&mrK}~cuZ{kLp*|~m6c&gd2IQM6PEeHo+!4;aiH>k*@CT#XEs>8E&`%^SqIlL4&C4Q@j;YfLjS+P>8GIsd{o9^P_|ga|nSb{H{WEiinc-g$5i{ezgBE`+ z{trIirIvWj#t`D))F@VZsnuK97~H-%4lpmp2Y?(PfAb&_K>N&LI^m`@u`S#V}d}1`I7>(oaf{H3vF(+T_6z=3Cz5xPk#EF z6S@xKM>WBK@cIGlGT2z(H5+eLPxuq4_A?(>MFP zRqNPc8MNCqO?Ba3xl@mpThmw0=RMs>7zA*9AWUX2EEuyy5JcB+i4-+REV7}x`-udJ zy{M~u!hM!WIdw(@1QmQHkSUqmMBv>_j3!`&f>c@+kec2O0{=Ebbak01nMNWsp+nv9 zwB(pM5{d%u7tFY(P%>s;;#Bc^%DtGqvIIhL+gWj0N`@4X7(YzVylf9=87lI(0rhEO zXTTTS?HHsh8H8Bw3V%_^@jeq&RE7TZ#5;~?`NH|kN}<6dIwZPT&pFgN*LDE9tSiB{ ztF-)R=Pk-WBhyHlBljpyF2BbBaAqiqU;q`aC5$^b7r?pb8L=D&096*ic(5-IRCGR| z(pxo2J+S*CmO}h_g!p3MFIPdquL{eNwF@vYVPm!t^E~Z(M^F(sFt)(GAKLo}Z$m#2 z%8(8T`AJQCr>x}^3L=EiO9*0ju{ODQW?WKE4)T9t-bcz~@E7RJQGR$?nI5!73L;V& zuD8v=_@3CwdlpU{#(RmVcEXhly$*ab=j$|)RUm`W6;)f=uMuIZ68VdPQ3G?H0Jtyb zG3ORQE*uE8#zIb<^d#A-#1ifCo1LA+A6(sUUVq+g2*<=E?s$s~2iZu0a=8&X#pE3L zT76$_f7h9`GB*>aD=S1qfK@R06 zwKlAjcxD;Z<Yr{7wM=sng)_J|ca< z3i}!{E96bcFGhJPgQm@~Sa@k)HTIE=^OQS#z)N?6Q5f~PB$ZEQVl`j4*<>^PRTj|4 z+qLiY9iDWDE?G~EHZrysU#wS`xrLtND692EO(x>Y)Z-BsE$ zDJe@u7=pANM7@fDMa)`hH_|HpUVcAJ8K?~a8N{P-iD^y(8W%2}@Kue6fZFAKu(T>5 zE=QrYe)8O41gM!5dsTt6<`1eDfZS>>pffdbPzxwX8ws~{38O9erSTnXJLs7O|0g3F zu5mMltM{=mu9%Y>Do!{AR`Aj&y#p)*fu=T~YMN5=aq31tK2&0`lRAb~;Lc%pmtI{T7xwafWj%t1>EtY?y}Ar}(=J6q`b=FFN;TYT&p2#ta?3kLH##8YJQe5K}d z-BT;1eIU1W+Jw*i-RVcic?s^k>F%oSJkS|U{q9I{(lX6xAQPTiGL#^H|HcK=2!coL zldfFLO2A4ce?t;rgoz5qi}UQ}yVHa^tzEUPOKo3g{n=-L%^;gf12dN=3& z03h?*ITdI1BEfi8!IT8wHej`Orf9Y^|y_oa48~;f2|@MxDfjD zHL#L189KOKELIJLv`_#i39^>)BpabR77AJG6slp`@llJ~kJu&`))MizB4%?Ernqzr znqgCV2b6h6VCrr&J7#FFV>oMzQak(^T4bJWTTo+5(pLe6#ci>2g=G#B8{uDVsA&aj z0X#TT;NZud%1vPbc)`u^yqsryCjqq4U@8i<(>jypE~Ih}cKoTl*cdxZo3fAf=}BTvoTyNzGkm39)QY4_LG7Nt?@AS#rZe)Q z5bu*l@wbXP^IVruBe)+sz#?l~is#;NEhG*oN`Rnsm4#$$-FAnk-coB4?1z{7;L(S? z%~BdfhU3Qo2@_r`fg=YxJB8$#Z|i=nJH~kHIL25bV(AcsmyU0pn?KJahg3FD*HBj= z9LzBf!fVnwJSRVV8ijoVPkN8?$vt^nIQ*L+pt^``?3Wjp>(eC1z79p|?9!VOC#$ao>02*Dis|~vUp3UNdG3JSOZ*^ipfP?cr0zTMyt8YA~lA)gjAd);MjLehV zl|Y`_f3gEbI|lt@M-FnXdX{FFt~EM+)~d@)@&5BJl^zk_s0fd)_dwpzsD!C1Y$&aa zfai{}h=YSHT?jYE5S{;Zf8(b_qh(9dMq0**bXyMRVmZ5rdPq&C;lgw2K(()quNdX{ z^9|3UBF&fK_2)PJ(hmG8b)iAd1kp^GW7t*Bh5`2+cf))Hw8MG4c(XC^akZ;e#>9oe z-qG|Fz~s-d6Tt$cP;^Sp;nG;j3lOq{34-UH)35eXI5Veo8qg99J+Cl=LHpurF`L8| zNnw$E`AXyTHL!9mAXFWp)t&^(&PytD*_Xope?erqA$Rn1gO+agIG9@xjsO# z7wFbajmMXAn>&l~o!_ZX)h31=U5@CIg9kjD)id@k!>yJVWG7qc$Yo)E1{Gc}R z%mb#_;{uvAnZM6HMdw?>qlqEV)@(isK&Ux~Um<=4@UnSTR$(aU)MX7&SBSqY61zT@ zy1uQQn^g6@-@E%&zmzFr7EFtzI6Q0~B%=E9DY4hSxA)a8QRN$?eDs=~wf?d&0z8`J zD}gaiQr#egSNT=U^W-2_--#e*N3T06?I31W^T`Y9l(K3wOstz#FGV;+4NoT!9e@eB zKl>&2i_(3xcB6J|hFG3)s2v@b-vDNTX4BIOgNb8%snIV)1;A z>GE-m>)aKUyvGXl(*rA*nN=luyE52#(T>yiDb+>!ehzhyf>%U=7SUe3?_y^fFTR;! zxY`HDlhTK~^DRTa*CL$hN_TsS4^_u_*@y{=>6YGik zapQ8!sXYkVfVlv}K3Au@tt6QuB|RMSZJo(J!^vWulBBCh5jDvp)J{|p?lvs6kDtRY zdAz!~Q9=}CLX_nZIUzhZ&Svl=xo$qaK}5MkMD-KD>*>{YKpCSk|JXJo1d9Vr4chs8 zFz;1TR(I_DV-R+Vlk8TKN#E<`Ip^(r_pE2Z5Lb=MPy(MP!G%9GZ zV#Tv7I7IzezX@I|RtT-9;3bZw4=*w+eGxqGBrr9$B*;k*MpWj45Y@Zq5z(%Rnf&XrFWB%vG^8e@A|9_*9 z_`f+_h2yXJ_P=0&f2OOjGyn7Yp6#EKl>d$anEv6NVftUZGkHmN95qJjRyaF6}#f*%WR>r-@M1RMM3lIS{@g!AW1Qfk+4u)05CN2&9&4 z(N85Zb^;bGGPzMm3}$N}K;-P7zdF);dsoTqZdxS)`Bjq;yp=)4T~B|mmHHD*mukqf zdJE)Qd<6E--XepQ49PN!aR%paE9z+D_c{c1;SVc63Az=o)()68#3=FdH0XkZ2Rn-Umuedf+x3JP4Q-LZDoHEU5c4_Cb!}# zJvMA=BWsfa!}pbSg@oJK3mPO9J+ns+DF$S;*u zfV4&S(=|uORSR zHsE$9b8J9d0xmW^Hy}6q_h&5iQhrs(u>D?kpOm17h8i)Sjk54#&tpCM&2?~y-T{KZ zL-+TtV9xc(R(qc?s&Y+#ZWI%o?Qur_#??+aMt+z8ZZy8nRRC^aG-oe1XZsH|eGig@ zoj`=V@0XgdlA6#$-;j>ew(qyRw;8@~B}^fx;^%-Rh1IWD!S7G)kyQv1ZXA*ibzXFV z(ID@67IyS~av`mt)K`e_!NQ%5-Y|bt-I@1-v~-*D&i(sy zsbdKM9}i>qp$V!;nyauE&|P9*DG*tR-(4ktV=OHo@z_@2^ybLm*hMn{OZPFZd6gb# zIX-3V7PT~%uQvTui9a-P{`~#iXj}}qWeEO40|&n*x|2{_vNL#*fG*UB)}YT- z&WJ^r;&Fy+`iWG1nveHtJee-NDycT%)qEA zOa&o-(5IJ#8=m{5EmB*~T(5>celm5a+O+vs-9A6dO!xqN78PfWzXaOU+@ljMHW>GI zf~S)(q*27UKr^h%4T!)NGRy({UYRToaLYcCz6zaAg#3n;`boA>L+hv=S+HGiHw-!M zc>*jj!`Kzs&EySbtz@8Y@8M>navJ?y3Oo8Cau0EED*=SH z4s-!7&G{i#EwG2|$P0!X0Z}x7*;~n8zWLUK^M?xl6ACIcKe)u(j@c&j`h#Hf%MZfA?=0IuECgJn!~ylw5hPeWTZAGK0(;h7%V9}_yF6k zYD8&oUb#f3*hZ|8bwm(0n|kzLyCE-XNULun0+reF7P@rR1cOl9%pB!(Ms@~T08vX` zp|%osBfw^AZNs0T^(q0ToDZp!j4ZPkmnh2s86zgkJ{y4K>(E-o@*ES;euM%JPa7kA zzCQ?P78_(VA>Rqi0~D!M_7b}DhS#%_j@OscI+hZ1{?VQTv7yyip%3$F+I^Z>H{&&Y zF70*rucIA*HGnU(QaO}Y%8)?LcTJ;hz!+Ge?HW>)SwKwbY)o}Cy<9~AtuWLU=T-3P z`braNRKKn<5~d(=;0$M>hrqTTF@Y|#Fif!Qs&`I154rAjg^7O^@NnPy41Rk&hT7a= zDX*e17&paAO6g>Ek<4EF+T&SY)RNqG!ma$hbzaF_Z5;6Dj;AI$fQ56Y(T@_J)TTvM zu%kgE9C4Vf4?%ovCsp!A6BMBLKagN&Fb%@rXHC6;5O>oP?Ik z?j1dNcI?V&ALux*cYJKQB1S}{KY-k#H+gO3i<~7nWFtlNu%S+lNnst zfI!$e7JZ4{}*mu8I$GwNd=!o{%x|B_Df@=e`3?{@681fkMT~7Jpfp@~eQN=Mk2;Rk8@n$e)h9KcF z(y5?w$(zt#qSACAa{o565snpMS8YR$5%t5$<$|$;_V8^5)8AFxwXp6iDrKayU9sbV z2+$@%ST{%xjzWB(uEZS7ud=(3mzHHJn zbaEG=cBeo}G+J zF^!5Y#On&=0vcc_6WfOhjG-Y`z{VQ+lFAQn^(B_KeSe+JO_m7{%6wxdq*K#y_|l%8 z9lgsZ67%9Qvr;j%wvQ4JyQ^s(0zj$g;}rfR_@>UgM@pY7+~J$=4hXQq=AQy<$XseD z(pEPX_dVAVU$tfN1nWV$BIXN3L$bf>AaL}Tpr=~u0()z|;Tlxe?r>JE=)ro>gZ@x1 zU(R5K9fKV%Wlt%0>S^ObPBidaV4`wYkiQZFXQ5!#ayA;$Q;5c!@$EtWakL(=QB)|I z6+G^K*YE7u6My7lU4Wr%G9K0&NZ*nRNt|j#au5j43nc154<1}dTx=^rnJO!tX z;=$gHiH2E5Y3Bl#rnc2uZxBz+zyeXjr8FDoCGFG5OoW&2)C^W_9V|du?peST!o-VJ z**mz^Q}4~ZC`+u`%?JsCbkYsjcw6q28Xx7ph5ba&;~Y-c>R*WDKh<$GTYS9KTk6G% zovRJs12i<<=V`5+mJ!Dj2+v7r+=7N1g2&NiP|LjMKW3>NDykJxck^ck|0Lb+8i<^f zhRK!zme*=EEn?$~hVHZHJVrx-UDSu4UEK<^ZH>yrFEs2b=AmfXr!uQ#xhn+%{+os0sB@R>~>G-t5`VipO;zxrX74G!98+`SZaL4wKEpx;3YH0%8{m`LnShKvCP@1AX~^Acj#`J&^=k0db`}KV+Ohv z*RE@A5x=6fVD_Fjc;xWO#H7`XEb2YMRzkObICDgKt@w9C*L2 zn`JQ-Kq#~jdro3}T}e;JNVSYyxi{!q@Ql&Xu1eErtlXe-Q{a%m*7aTy-3k!!5MJ{))rs1`vjKY-NfiPa<&o{Ep2JtvxKdmrR-ttdEA z2T5#z>yZ|QbPuUsn%5|2GMEO@oqW6MRmNs+a0tB{#n=>Z3ZTrOfV15u<8uRMe9(dn z!rB!Xi~nX*Jd6A$X2t8a2h#2GD18DGX0?wQtsw7rUC#0cWc3l=NA-Ox1D`D4`%cyq zq&GA0`Jj*2;un_aSJM=(-Q~&C-7H(fcRXd8{yGW^zahlJ>{ibV3#p$a1!Lk(U>jnk zxVU6b#t2>bqm-GAK9fy)WRv3;M4FO-3G#&pz0NN0DNBJ_r9;1!!L8rJ?^KL=D3e8G zS4|mVtT8g}WN{!G=;%*CVIIW7AUmK6%>p>k`oq=}EiMQvk4oKpcmyz%*^B)tff_?= zzPV&0%XFF!y^PRi5Uf5Zypt4+9*GL`$qKH{wT{-#Gq8euX_fWQ^1n2fpqKOH5XaZb zs3idRalFTUviI$W?GG>|0z2vdeL7^|`2PY|^50w*f5%e(mSO%6&ZqnnS3)OXV`J;& zi2pr4|CK}e?`>U-^o;-XvoJIK9ijMM_|F*2UlEG`HL$|;w-0kVke?00x7|t7$ zxT3KcpoRZx>&o_}@)r3K5#{wm?6ZeWjs5OJ8(T~knlvG@JVee2!Qi(o{SOudsqVx+=VLXQ{c_??;(j-9>RV)!+L0Na)yxhp)szOhz zfg-H^;s~3VEE`JKIY(HOctP;SMerg9Fu%%SCuHfI^#XZCX0befEv%rxa#Q5j*^UCe z0i%5XwqzufhD+1@AKkc0xS<1Sul|_3hI(t^xM2R6r?01%Q969{aL>VELb@jp=grsm z=TG0=pBQ3zN!>Agx?9`2pCMFsSe{((-x`kYbWdmc;t{Z`8|ZMBgvQ=fo?{{xcj6zq zV*3zJdrw;~lo|6{8E@dME#m@}sX2dw&D}Ju;p@4!AcpiP8k<4XL(S{GO!hgFDr>Kb zg`7aB{o7eLSlr2{)FD ze+Ic}f4(j$U$%(XIC`19o8ZS6{)?cpsH#ba?SnTsvg_DGR}FH<%g*-sZchCbI;&<9 z0PT>NO<~Wu?@nc8vWBt8% z;O>$eX6!&NhhF72pz0|l&c1k!y<78lbFWW*t?ZD2SRDr93IjpJXy2S%BpNdb@In@z zsOc|j>k2NCT-D<9C5E_SIBRgvZ60`vNl5o@8JbgkV7)95${`$tD7Y}Q79zqTHSqkB zXA*-Su@t7Pmm5imetK0-i9vB6N7A3O0YR>L(EAAJ zxp=?)WyJGZQ%SHhz4j4qn*%(jF4qp@r=5A_rls@=xmIvf(56`4lON6DaouK1N)&Dk zJE=5=`ur5$(I$({SS?;M?fDy`d3x{0U(9EQ@A)k7Xezn6WZc9$Z40f@5~$H52Ha7R zhP|-}DL)@X4tUE$eT}ci5d7Xz3Qa@rZ!jL>08!i1;7%Y%0JBvcy`_a$^WDZ=&+x7B z(yDpwr*8MNh=a8yxOcb}-#P4pFUv+;Ba^4Qp?w&xIogs(xPCj}^aXPqWeD9?*CK6e z3%gWG(ibS-0DadzMZhAoEI3qr&k2;jxiK!wr;5QGl|}}k zll29XnkNZaFO7G)OiwH3Tc#H%%un7ed%Xuv!_j4wzS27*9o-sBnUdPMJ|9E2TPa_- z_h}tA4Je5ZVou^9YZ%4f$Cq{2=y4$RPPhJ%JQpm{0^v(y#L`Lxr}JkX7$YvYSk(dS z=ecj1a-3)4%b^wt^Hxf%6yZ>`5O0I=7DrF}LKvu8G^agKh++yj*kzI3BQQk@k|YF1YdWYYCkX(c237t;ksGrknKe%y2?sx?02rI;oQXlKRdv@TKiXdp}t-;?JF$^?L zedUWti1k5LzAI`Bx@t=^6$pv~axP6Zm&gEple@UDoIaE45rCG!dpxd-SrIJezqrOH zf4nuNO%g0I$*RnJ`Z?k|{rSy8*mt0>X~y`CX??};WG7)R=lsFR0pT=O`?oU_zipbe2th}|ha`>g#?D@($OeHx& zTDA}cS+GL-Cprc4*>5gF#Du_Een&hut5c*)%in&>UUrSlYXofUrjrX`0R%|uP*&&P>SZQ7$99fD= zEwVH^!EDW+Mg~)Sw0&x%WW!vyF!k_5=uLn{)H;l27O5mzEw-Ns58Br7bGmeQLK?)| zW$^-?j^Gk_-na!%_10=kj9pi$b&^JK-lS+ZoN7aogxoqm_VPbZ@?;aFeZR-2CgSH3 z(NfsY7S2zfNSryvpCor1Eil>qA~lU>2+y6aA11w(8hcZ6HF62)LVFn{KE^QArHs#q zBpFb@dYaWr6`AiX7~fsaXKq%GBqVKkH3bRlr4cx(uFNU|AV9vRH-10U8YDn6S*>sQ zg~g=qqWOx|Ruw6JgGwOv3wpWMM@FJZmKIZc9Uk2vTmn_rpPiH}99Y?5U{<$EeWf1R ziId_^I@)zvrrZk|d?Ybv69!xzT2MQJubNZ#Pg_~e*zsd!k#|#(w*o;-kvDPoe4)RE zcNfZ{!f$P0MO~?CTl}KtNtiC0&bH!ZbccrkPGza%8k$s}x4zFbcj z9SdYAVD?wsb5F(>pPAZhQYFE==pMvL9vmG(mpe(H)!4Lyg^3qnpwOGMKIf7ISQ0?{ zaw7@eTyqu7EXIU#>)!^)qK)XyZMq^Gi0)C!N;9ggIs`-jjH$H=Xs_QOJ#_?j+G0e<{FMuKV&>2;+#k#k4Hkf2%_BVWP!)q^Jc=7PJF7?~^NWN<3 z;+CW!%-KWm+w-VHoladOk3`NUbIeNxz*>v3DHOtwu69XQb2}3SOJ%fD?Juhi)MBU$ z3lg8pbmG5C0|m>l4rb?1-9s%9)LqBXDUJUOOczghxe&*k%+0vzO8h)#2#=yxVO$}g z=a%KS0Z&9YMujP^mXO~jX|ZJIs6)2g!eDs8N0_ocuWUxx70%gq%K#HdHt8bo4dAS* z(eNN1jrWf!tc`6f-bx=Lo>jw(j5vx98PX`hf^J1B&fuzYWWkfJ46X=tcML9lC8)Y< zoz=v< zT?)#4h43F6hN6s9MiCs2=nf|A?lsFDJr0jN=VT<%NPHnw9X$uQlzq(s-C1gr#FXxM z=9Jy6+@%n)3c1O=TK-|GOp0JhrLr^>yd4&9)ePP|DP)N_DJYXSf8Sdq;dQi%N8$%J z$zI+gVl(+jRMDz+CPgyOt=s4!N2wtd9gi8#=`S@mPzv@T`>->j<(j6*@bIpAB2XP1 zT!w3UaZ(Xp=>5akn`gX=*=@UM zgsUw`{D5$4c-Uu#Fo^P#`+d_1#4XJ#(%vY(SnK2GLvAQA_Gt9a%RkL?0c2Wch=82# z%PhpF9S88Qo96;ZqRg`*cf;F)aLI1N{O{2OE&|SbqrX0J#6UXtV;(-^b5s+(Pf%eR zcNh3k=RphzeoCEfYWS&HPZy@;;eYxqQ^eV-s$_WMOlHX#?N`be^?!esmrt8PVAoN` z4H_qw;H6l8ccdR}ll0SHTPO_hWSTK5eyDy2BY9;fQg#Po{V3#WawSwwEoo!3DUKPh zj}Lz147Mn1jI|V~l{~QRNyl71%8p0Yv~(ghtCU3f_foW;f$WNV^;oV<76$)O(sXi03C|GyF!W(5I1^SQ>Ru{?VNzLz1zPf0^-g2 z=ET$D^}fu8(i&GIk2-*b1X6Bo)bebpqIy z>_=lz0q`a2NHEPMUZ6x&a8;)Z5QlW>NTE4xA=uGcw(X%L>4w+EUY3)VtQ=$6lAOBa zpb+r^^QQ}cy&W>b6)5kZXEC$VIZarGh)3TXAJU|<#J(#jR#VtK{jd)Sy13w3M$=Fa zj$p2qXo3Tm){xlFq<`?#Cs7yPSc-IkVt46if}ys zJ@4uql0hT=$%5navCzBB#jIDggZ2~98L5}Rf3H32>GWSqL{orBBmNrMaFIcBSSoZR zdpFx{!(CiVoi=C^q4gwu z(O77+I`PPj*>8(^ChVej`p|khpK*;eJLCV!^2uwVyR}jJV;-{FhY*eBK2^#%WP4}! z*6^LCE1Y2)*M}WcJs{yGfO&=X^Uikv%l8$I2G~IIewg>6>e45Y5Ag_xIBTcPt3PdJ zr$Pvwl%Hv3kfh7H2YzK(D+P_kt`);^rCNLNjGI=H(z}fwvc_ahq7tHAyvG{#TKV;Tz>5YVP3ZgwMeG4S4xSV(A}``VYjB#`ymZ z`F(e#jw&_)`RO$PE9!KCr^)|%hkK9rjKKs39Jih{Fh(7YI7%IAZrBP~{qDCWyGhJ9 z)Hi$Q7wxgl69z3^we8J|bGHz%p>4t^H6N}`a|oZk;3F!XebW$&MMwlNuAJ^#I}9;p zoKGLfxYWI_U9=z^<<=fpR#}q^Tt(A5D%#H|t#t_P(UbrqmY;Ci`~Fu0>!WtLCUD*n zn$w*QV#@+FPsUUzCnX0@hF_p!I(In#?j*DP(;dXj!0~rl>U$UeNnQSLh?j4e@?VSp zgT1}P8IL|_`bNCWtnyQN@%at`STq{-u#o{Bz&{@hfcjejyzOnnMNr3!NPM7sx|+1_ zYcq?&#Vc2!{SGH7`zku-f%<~G8U6>l_kH1*t7F-OUdL?~icQ=&(Qo~Ql&ZPiE@|+q z`*S1fyQQ4E7nwEAl0P6EfR49?wMG+-sm7z4Tf>On_yH&}Bk9tzTQ+*1VI zC}wi0PUHq30-JP3$H#^pgIFXG07wHzN6shO`1-bsv7ApFlC0MMM-CWqu_w0Asy`f z&ox(LgC$LgQ`F)juIH)r*A%3aa3diM{E0cFCZ+lDrG`qaViRk{#pUS^{d_OCua4G64ml|5mNljU!(|^RJ0Esj?+>~vNG!yo&u}F+u!b* z!5UH^2LP)A<&S#`(VGx7)u9HL5(2ogANdvG0L6!V%6Q4L=tjRpD8Kp0ntDiYD>Fix z0H8&r2|o=u(sbq%$4EASN?)*VbK}rUI|ifUpSS0)<5vle9JP$-jyX1Pj`8{PGR8N! zIm%Em*g>YqUfytTWd>~s<}UI$mtDEQB}%TBr9b$bJ_RFv%&B9?Eu{tX*v>{cFm)uk z%0O4j>iS|Qu42@p%nZaGz56pLXZX&D{VS8|$ktb}19IjO~UBoBky}3-ugKz8wCN!UgO`1c8|~K7Os~@SBKth(6n(I$+`s z84UN#ym!s_#4-1?^bQd)aY_op+)IeCuWm0!v$h2^?;iIT6;)dG%UX78IQ+6BTq2m)zJ!7D(uh)sgh`sje?MM`KTlp9vm*H0T4&@ zquc_dKO&<>U&Qk?jAf=N#mHW}aQ!E23UPV(S$|dt4xi7|c-w0R%l2XhS*Rij$S%f+ zqWpwkY>@~G!itfuX+eLWyWKAX!J!IK3Cvz>!A(M#vln?LOe_oOmB*~CAbopzs9w|p zKnXjaj_$}-`64fJhRIu+F@vz3Q8&5ocueqLj?yun^E6Ej^$akv*LpR|grx_8{T1H2N2|VEz1a15O6q-1~Nb{3BtS)}XaQBdsL; zf^!n3j5-|%$vRqoCd+^#*OT_jnobRi$A_82@L}L*^|8BTWZij({@6H*(MJ@HO@Qw& zCswFI%Qk61&>;b%15pBT{i}bOI^DeqKhId&>MVH*q1&$8^2Sneo!Kc6nr2#ECgumQ zQIC}Wh)&Jmy!dyNYj1tvMto`*JBUr>hQ7!qS72MgwW0pks(QlXauFXF4< zW38?+UeF^D6hg`GQ!x~9!_GSowvUz!H9t2UbS80rLY2GIKj?QW(i<)p z2rA6Y3g*@dXGS)Yl%33I{cFnePNN`}(&15Q&au8Ddq8aB3E=bipQy)`h?zkJ4CNFl zv?W@I1b`BuD8s@F7EBE1aoSDfX{m>5DmGa6L$h`3Dk)4Ob1fa20{!BuX!f$fv%+hB z3-sSj*d;5pT(Jl&@=7BG2bogL6^1{i7;O7#M2K-x*dH>D2quq;g!Ts^@0BCSwZo>W zinUqI%8*C~#Bcd%aHXeEn)sG7ks81NCC^PQka)uFp(5u@eWn&nw_-a(Ipy*Z$*M=3 zajM0p@qP6qNy1Wxf;S`W_;5^!HTwSW$F^p z7nH+#7c`GqsHf83GO@EGZC#h_sA=R|iLmn9!MqQ{n8=(=TyDgxNETfO-$NRPY%9yB z*+>gqRZw1p2h_89YeoL0W7bY!F=V<8eD%lD_ACz1L)yTs}bwh~RC^@pHX zjUQbnnt3(4G-U+>fAYbXwUxHZUHoo`-qBp51AcG3kJEhaeq!qegeb*smp7dBzO(jD zcMe#v<20^X;cX|b+MlkUSSAI3WN1mID=d;hv|AQ<5L2K(FTm`=RA~p5Wj03%Brs{X zDXb^#5HoNfN->e_j3q)}G)y2IFUJ)2#pR^d7J2QPS|Sr!Q{KNVxO;qtW$-s$6*K1Q zentCpom%emBKs%^vG!A88?)`oZiXadB(}j%^~a#!VLA8A(0=y0wir)}tjrg2+z_7! z$xL?!c$RHKDlML5dhR}CrA<snr2f1bFH}?zZBZioqoO?ZIy85){B0 z-$4xO7SYk5!+u{A>JFQ?@mdO(JV&ZKS}fY&($anbIr|;iz8KFD*oxUFzt)m4}irap;I=jlo4uzTvk@$ga})u0jQYlj7(z+xETvo z9*&pT*pyx!PX8%>6nhk$H*2vY|LhHTuJ=t0>v*(b1L(HQF{MS(?-41dUHvV&h2-$= zRK@B@=NMu!vj3v>`(a8dC`s8Bd%OxH>ZY9Rq$;K}Zaq^)dQ`JcAT6u-;)@ldh9wy} z;R0+1dNoM9Hn~HoRvLsqJSQTfC95v4r>abQ=5dH87J zBzA&GD62V~^py!(Na=`i%y64f+rAM@3f7r6 z`jZDnQ>^z+--=!rJJeR(O>vt}39EMOBw}M^BfT?qYiWB$(T&Z_n))ojR8dP94BF$8 zVOtd5dQnG3+MICSKTrtI(K%1T-ynST$HpIY8}OI zFuZ3HRsM6r{ned~D@4bv+`R~-MB)#LPjlDJ@jP15m*2#;6M?CQGA^i5?nIf5O(N^e zHcfI>7FN5t<(IPgdpeKg!*2}_ORUCQ_qj_hB@+iQE62UuoV%am2ai+sOXK`oug8V7 z_;ega#lP5Ser+?(0+eX>POjaNHWR8w9MEi@|K5H;`GQ>?->ESu^I$C>dl-VyTmRrQ zlM*bc6VzkvGMb-KdrXM(aGyDVs~UGbtuNQCJyvW&QZh5ov6B5XE|zeaxV7MnBHrF% zYs&4mf~u+;MH0&5R@l2Hkd!IHuunCJY(hY4TC76xz48Vy0K&gkSEWJcBja~Q(sNI) zYbiXh^oRP_hp(YKrR4?X!SMCNz=e&A#!bhgBqX{S2hc4U89V%0kX)^cGPC8`l6HzyQj6po)DGA*P@)EO3qZ)`+rAFvcilyfG zmfDlvd)z@Bvg&I$olmGoEE!u{hMbKRS{8~cVn6<@^CTFD&cV@+_sVMaSNYi*$FO1UoYdliW|$IpnfJ9Wm39s|eeLr?+8!XO84zbJ^7eP6Z3Sr*%6-(wwp#hHN^t8@*a;0y?zZSo>-pHVNeYb`($@y?>YZGV2>j9x^SimWLcy4wEmTK~V=W3c}0S zOL}k$V)V^eHWeRPxyi+9Qgp02H+@bYMFef6$LFhMQUY|YJP_5>$b_0C@xt-fq^PC@ zzYDm*fAlWG?@hGAZxm9TExdrKgLq#O2;S){aVrkaCDu@2ICrUz&%RXVBHwk7*YHpw zY50w}_SMv<@8d;Tu*i>ld+f5)&$<1y{RQW@l`-<~WDMqi^+EkVO%DF=mbJfoxc-*t z{!gO@{X^jSUsO1jf11)5SpJR+ev3Z;w`^tlf2wfo-?i+2i*Wz-r2jxDnvCXx?c;|Q zeiPr&m)&vuQ1>h%M?wfCu$pUtHyGpTrDW(=h1Ss~JIG!>fX$x^5Qq9e!X z1Q)JM$ud11%)J$bjbbaO-|er{Os;qLn(+mXz)vOKO_sh+28i-~{6mV+$Yd%(x5 z5O)0jk&TxTA8ow*q zKtP}UKK9q0W~=JZ_PBaDhY#FKeb4TBi$R_FmIPal@_Y7C{xm6XJuU$+U670;U92;v z9f%L8R;Q9sNMs4Y7?sX^V$%velA=R!I1rQUX|9ozdZkz3!YR|K_svV9B z7w|iWMt=ciprAN|p$y_N(W+)1IfmVG*5)XgIbV|zEHg<7QYRSFXVMp|V)>0p7+ zW_%~Sx|fTKXo2!W2Hp5vV>OX@RrD}0y~54~S(3vBY_eb|(Nk83!02c4i5n)AYL3xd({(FV#Y0F>|!TYw@Cd)FP-(I7W7t*$!6~PZVpn{6S1- z{0$*hE6kADHZ;XLC5}Q84u8BrOrB){hf3Rx(N5>~YOGlPbo`{SnOjNC+cZw;;KT}7 zDFm`^T8Zu)#<+|(MME=BmS#gFl0e0hv+>KsThYou_SglqCYR&^cRR~%QeIq=t z_!TJIcqVptJAe)Rs0l=_q80_E!Fgh!<$hqN=YC?@U9CSvUj9>zBzVQo2|EjdBz)z< z)xdChQuUNRNE;wkfuh<30F4U3J8lC6#V@%#5wT6jsBx(d*0%Qb<>%*HExdv4yYvam zo_y}}ob_QS4yiiOlXIJZ&<54^U|27_=$5xKF7WI-=i-K~{MB^I5EsJLg7vdlyu*7OF| znfJ^eN#RGyB6sqXxM{?8cg1?C2SF>poTxBkAM_ZBx-eiwND!Q-^#IK>Tbq!UZFb!v z=~hYRc?VLqSvfSxHD&5SAH*S|!!Xm5&^X$dH+Ypziw0Ioi+D&g>YNp1{KO4{K(Xwl zb^=`92*}0cjY@afLbT_odI@_4fT3tfe1~B0pXp&4c(48Vu}GSpP$B3N5@~wcD_p4} zTsHQ0jPlYuTd)P927bz>cs{dZ`hqjN9T@#Ef`h+P$9)%M_nBsprO72mrI_5}7;69m zsIcYVfEJpfwTPl?R=F>vAy1Sxf4r*KPWhcyGZQ?20Cae2KK|pD#?H8m$|eOcOl@#O zk>qGx@zN*Wlyio2z{l3vtykwlulmQwM=ZFe->0VtzFvu6Zff2S6v~i(A{czc74z`Z z3uM|L1V(Tf9UH+V^FHbM&aGRK zIzxxI7@F3DxRnb}RYAb?@A&-ehyi61%~1;{M%0@Y!eB-==0`_2F#bxary^dbr;OEjS5;= zJmu~!l)P$B*cBh}_t%^?haqYlDUdWXnxGhOSTd0|(A?`DyqiR}ymc+!My4Al)xO6(|khDbL2M-!w|3z0I#joywN4Dey5hr6 zY7@8p{VLs1w4K-Pdh-=FYth61#BQ3;UXR?5oU)BpJLa$_L5v{7+eyRu4f z(%K7k98;X6o&%QXyft4Lc}e-V0_fsfZe7Ss<_`6_&XAPOX}#)MO6IM<8&O3GjxVB> zj>|7kGT#KJ-_whmDf2UGtm48aaSi^X+P7*!2Q(*ZjGQ_p&z-2YszdA#tW!v4&pUicdmGgOpIK$J28e(RPKKQEU zkcf53v7{A>6Iz5l(H7QQawT&+BNnC#&A?MagWdrU1N`fPOE8?z0}U>83X%H;G~_y1 zv|K_8bu}?+a!$)CcMP*BUJkv!3dPgg`8|`}b#StZO!Dl;$Evch2I*h&;TiN$7U+wo zKMU7)m6`h2=fs-l#ny2P&2l`mLhje%b8&82OCzwV!I~w^sIg2M>QSwN8EQ4Iu(sDu zB0Ou1=_Q(qKr>{ReCw7=Sp=6EgUHS8sYz6)DK!gBw+1VIb-E7bDrf$-;(+l@6aUQQ zLSpXdQh}+dv&P1w>QW2IG4%BQJ@yhfuaihBZr+?HLD$n#Dc=SZxeQHx4cNlGqyU>^ zyx{zMmv}ofjpJS=DLlnR0hGxXNAnC)bxJ`pU(tn7v#L5+_w4t62HOwH+K>@ z#WShG5RK{#ez2Vlix|tJlMeCx<}FOt^_>1nBib0F)HuV`EVaa(H%_gAS34EA0yPPa zg?QaBl&=+USuA?ln>L!*h(^)@Bk#K=SxI#gIA^g6Ydtoob}Q8-^z%8jh>xirsk?3j z`Ml-uDn1r8XR98{^i-zFcy3?lH zksqdyWe8j5>j9=OC>92`G+n*uEVyLCLv>{RtidxKsd*jzGac4WpGOfZzf=YcAF~O` zIk@_`y6MdPEvc*!9RT$d&W$p*5LBY|m&c+F11gj7t8B9&L4P_xffiMwVZE7P=4WW7 zHTbTMX*pMw(gi?QWk0~hI&~> zpRuF=7rjG&XSn_`BLAN^1^?IZ_B{pvE5XI_m+|3$PH=Jja{y-IVE@C#w)Y1G%?1gcnhJuqxhf}okQ_92-9s=<+THqaai@gbQa7(!FU7X)ZZFR@ak z9wVU}ya$YDm<}8e>$y-8F-VZ=5wO^(aoC*Y@Qb~wLYup638~V)sHI#88YTA-e!hDQ z3^Pa&kH^cZ9xtMJO56pu+oy-a)2^=%Z|4zJj|k2`AD;f)onBs^`nZ%KZRT@%GUrG_ z`9$*U=Xz>MdEWnlyx5zYDk)&l<>JCrH*v-@MB|~%9#`zdSw~*iemDmcBXG1_MoRsl z**t4;xL>*wBU~=3%USOo*KU-&*LJp)gYLD~ooa8*>^Ve|YF{#6NhwNSL7>on)aH`w z=_<4&NL#XIJoC7kyfjyII+6^hxr{W@GWZsliJ@qVi#F|^(<^a}s+FdiWI$CdbhdGM z-%(|+Y=2DjG$#9q9l=G>{Z9Qiw~OI#dkiB7EB)VRf`R^Dm+oI$!2ekMA7}xc8d5O_ zO$h(R)X7bv?1F#>ja{Gt>au3!hZT^OwZgvvz0rApxH1fgI$Sn!v;1PT zA75B~yi74hkix;(9Gapi@_JLjeL0V*wAk)ySfH@^a?AVe$W7^Oz5N-`&eycCLwcpB zQeIDamj51tVEg4kfa{Fet*&MmTF(s!p%I5^gIyJ#_PR_8qw3tEV6OKTgGR{I(S#O7 zw@jtbfe^(*i%EDMws61pS^w+Jygrp7>uxKMlx9euxljf_0Bq5Y#9E9|ATEd7GhF() z`NA`nTRjT-4|L#wca=FGT4e74XYi3f4VGm-8bQv2)*HX0%xtE+m?Jht{H}L> z+1A#`U>Epg0&eNkz}3(JPLYh-h0-~erz45f^5k7zua%z&quZ!Qj6vwSI5Itsv@eGQ z!v&x-W7mA3=P&7HhL0Ck?8reGs17^ZV^C5XOu|d zECMgxT)8wEwzTSu`qoSHyA6P3yB8W=5Q(kx}C#3ePj%k$ZDU+++jfo!Otuj zV=-~G#pm zlI!dcyXhdL2t=`#oI+3le9fythii%ztdzyN`hZXB6X!iO2N_M6m$FaE!0$+W@=f2s zVmwHLyDt zGKl8@L}6`)G(X)_4C$4NCZ{3B_=*uzw4ETf&^#q4+V@6;;bRUvovw(^9 znS-IKwGEtcI%Zr!lpm2^u>(D=AwPtWNhjlMjWTuEyF)zusB4Q&o=hpe`shD;Q{%Dy z*NcOB+=+2uYeEHZH9EV54z-JJnYbn*?UGnF`t-J>SSCMyCb)3~JH9z^u*;Mw zTLuZfL42C^1ltlQ+<`e|4&i%sR8Pw)HkxH*$JG|x-RMfehV7q0-^C^@al>W@mhc}M zcl=2$=+0*>zng@x1mzZ01Q7vW9g3Z~|Mk%9df@_wC&>}qy(~&LE9Qe4^U=z(ytqIDD`iYDKW(|)2Ct&^Vuwsp<#ArtZeEK!PGP-oGk zM#??e`8`0`gAzP$O?&xk5sf%lNzi?=VhO)E-gKhf@ZfSkS*+uabTSHi^643?S+G6C zb4=<`Oo-JYu_z*i%11h})GUEPeu%+^%_{GtxK{zVfpApdE^DOd9e7tfT<2Xo0*)i5 zZO}?DnXn6Y(|1_ZWGy8afiD9;lEw4)0m9Sg1&2}NhwuxVxZ?i`2Ccr~Kn^U=bYkQv zq(iMTRp7GOJ1H2=<2Wz;vrifr>U|S_uO1Nsd0j?BepJu=0rbd=eJOW_yH%*f#k&q0M$c_EPoH z*ym3AldMueWqoHdFh6W$(oS*^UJ^BcLRkfc9y1yz0N!c;K4GPN2QWdiv#h*i~vp%1t=clz%N zIp8dTiVIJ(Eavh{e0Ja7DWSP|9Cy6APiLxP|Di^=kn~a1-Khx{48k1gfD7YbwFX9# ze-wAgfJ*^qQsvGpVJ=Bj-4O;M0z(5M?pXFq_9uvw>vE72bp8bq=IMx2*LeHN&@%C; zUS?;TT1@&ohC{>i%*=)NCC%ybk!PL04zLI+>~|HGD0oI`jxBoLlC|~EthxOB)=KK6 z_{6f>?HRSjw)CdX$NCe+U&`7=D9nhyI>$v`CI%KV$vtrku5UuMX~1;$@rC|PnQ@OY zDlN(*!)y|IvOm0jq4wb`8kvm_@9C>%Xtk@~i<|7fox_j{$a0dkN76sJNVtxbd@4$$ zG~UucCN#C&U#=z&Ar&G((u9x%38o}e%>_~}CQ^S^GHaZ!R0OPuD7Vv){8aD#To{sh5Lax&09l;X`(SDfY)roAC;*fiP`gEaRf7W+n(GV7h` zyOLtYMs7i0hY1VjQ%bQ7Va*pwZg7ByS}AbKB(NdnUrI-n#QYrPxc1~-o9=TZ+=dqp z|~JD?&S zuIdt_afXJ8m%u$plQ%iUYfCwfH2P>VA|Hdm?TF4SW?q@fKC8=2#Pu9M zH60B#(f64fLCr(crEfJ#%w@%f?uM}!2CWw$_0`(5c#o670^-1kzX@FC7RYKQP;K5RK@CQ~eOg;Jo9ajJ{ zj|b6lL}fn+_mUZ?YB?dkh3rTF&={fYa%TDVB(Baf0Be+U#EP5RKDsVAw$hc_H%Gq7 z4WJa)KZ@mJpH@AgaJ51FYc#h`~w^PSHYQq`A=lc!u0QM6b9!14q5-shy0(| z2Q%Yex>5ev>OTNkO(|TqI07)jPJq9WHUG_)48Qz7GJAr}GgtuA*4AJ&3Z`?OacD!; z!Jd6ZN2;`l1O$3vSsQ25G>iCNy!f0K8ULyg(7I0ccEVbA-l!ard_H|9zd{W;y<~RZ zX%b5^OPTXGE5!q4(a#a(B_ZOdjVx3`)o0MkO<3tdYq^v1AUMT1{iQ+FAO-*!QU^O z8sv%ThwkWSalybS*TNG!mvBY_6xv`=G6 zfPZ}6ea3{NZVkNSx%K4e>g9eGKN?;MZ)9sHa)C^DB!O?GsIw|%RGq!Yj(M!Vw(i}`V8gE2#s+rJiSvKEMn{SOpHx=zV?H}12s8EBa z=Tp0;;)&Z`I=OIRAiMY0eVp&FUVlGxbSZ)`vT8W^wa2Vmv9>Zs*AL!&&Qrf)>{^(; z9>KEFvPgrOSx-m4_ao|l<5CiCoSnM)9CwWqeH1=y{&$06vs}Cuo7btU zi@KicY_)%NM77nKc%^Y<{edxcg5B{_JlOOzjh5%ozWrO)Fcn9PR*aGs@R@MvR^p9iuNSiZQ~B>E7%p7~>JlS)`hRMW(l36ymBOi}AT662dU>X~Jxr>-!EM^F4AP+OF&He(kgNE}$tkmB|Y zU}Ssc*Q$9rLmELwo!(EcAm)n9!S8CKt4vI{cazdfM7r5>{E|T9=0s)vdYXL~HiGtw zgJnrSL|22TFRB~khYWg9bH6BkKQFgW4fc*a;s9jXcTxJx#B1Q5NIS60z${Wx(MS}e zTu3XiO6FQML&|J~5(8Mg$i2ROK`3}%NKs_s|64?aS zR@d%Kq;PXI+<#F^QQ=AR_g)Y%kDFBk_U*D|FVwt734~eF+65#SbNzO$yh4Hx_B7yx z4EEdu=HQiQv~VBR@QY(#%KJS@Y=kHG4qj*Z6KZplm4Fs6>G)x^a}YaGD1Me_9OS<)`xHUBQ(_h>d+(mi@gZhCtW93(u2?#dwm6-M|-L!;sUuV-M#K$t$nS| zxqUEYa}4VGwrcG9NjlZWJ*-b_VkSji8b9scNP9M=D5bs-`cZTM7x5me6|s!G^JJDS ziHXcM*}nUC65>S8Q51vb{X#T24@r24RY_fNj)7-J;%L2$M<+}wh60w3VSjGSa|SgS zM$P3pY(puwx5iJK!xInRTeI5}oVbx?jbe9lIQMweMHd+x9sNuZzH>6{lXRc#%q4 zgi$uW+F6q?QyvKU%f!3P%(RE}f@N)r;BVotgz;IB5V_(zM_Nx;X+f2}yQMSv&)@W$ z8-%NOy|$|xNExrqXeHRasveIU>=`qWDi)0>sHzkpUn>@jj~^5Vj4^U6fNq_+h?qw3 znouB8MP>}uF?a8P^dC8iISk&c&VjUeD@^+oFbkyc;TuK&vhYc|N#6UE?iPq!;{M^( zPcZXYVii|hbeuv&_f@=zLdsxCROhjeLtzypM}G;vCQb80C|@sa!(1Qe=5C8aISoFVGz@wYO6g z=N*n4`WY-9VKtvF**GyRdJWrDn{U||hMvj~W%|Q{HOC>bOB^Aijg59CPG#Xp5#te_ zxT*%n>U~uiF_H=A`eK86SM4U2JLtlEb5y}{r|E^UbDKlU^rw}ta?Z*dD@!gY-hcuu zd+3lp0TCZdv`{U9|7Hpi^KQ5J(3zK)Q}@Z{CkecmEPuEBeaUmUtzynYpT}UoJ?Jl zXP0k@=!kOO5Kyny8>qYJ^PZ=^tyyR zpQ0{rAHzJN6_1`8@{?+u;tB6+*+u9|&0tNZ3wlDuXP@d&JQ!Ql8AB^AWsQX>grmCT-YKTTXLYpOFHVrPupO*TYYc|MyK;82K@LK;2h*9-J<5PK zbY~pJ$$bLdDqxj1L9PV1MyLbGBQ^}}$TgVT!s7JJ8?877GP>Ai=afitj+8lNrsohP zme{B!av9H8%hG;a(cyC6OWyipF-uS{i0jFk@qKtMo@Hia~ zcJaeE*fX3EPC4^%B?~J8`B$x?aas)6Gj6?1GzR7AYCVMQP82YP4^c^QLv3 z0cfS+syyXo0;_qW&aLl*?U8-{e(Fe&Wg=;H)Sj!B0>dGGE72XVU>Q|9rOW=IO}wkM z_T}5bF|0*EulAPD2KllPWHt|%ni`HDx7iY*n0XD<(Xa6+@O{SYbzsMv+BOm4r!cMc z6s;VutT$J@P`zK+*5AFF&eRsCwdOj}eYY(d=a{MA0kwT*l&|MVEXVqmvsLY!q3F+H zqiTcujJwzp>w#zjdB87Q_K1EPB9c@jE7_Zz*Maj;NYp;7c@(@fq>p86N4EISk-e*F z27#k^S(d$^FlwBj4i?dY9F!WL_Ch9D9REJEtFP7*N<;;!aj8C-Hk)cA!7K`5!!H;*z7H_AL06tNQly zT52j?IPeTs`F&rxHVL+8A|p+&u5u4x`3!*wpHN^wE1J^`-il_aobR@w(bOe|iP7?9 zZfi`^$goH{2&4ZPCa8MMOS+j;48dl@OPqM3g)n?a!`O$nS|xWXeU#WZl;Ygwl@30W z%Y6lR#!fEj3v*i{LiGpJZl)2a5wuyBsVVk_r?MPB_9Sja< z@V9s?zs?G@I;cLN(`kQZcb4lwP;Mt)umhXvpt8IGFPZFKXCvIIlXOrgEjDpjrFQQ5 zG4E;;SSK6UG*BBb7jt`26f#`aYJIdi$tE{BQX_|kHuxh`)F$2&N z84jAVUC1b=t)>Zal|e*n;t{t|t~X_f$_Pvy7i7{NH+WKT(`wz9w5ol`<^k?tk7vP! z>vJgAa+9L6Z9XptT(~0PrGw;$C_X?#a@&bE7YRex5*-gn0m*I1g`=aM2(IZW9EYG2 zGo3-=g$<;T_l^=LQ$p522BPVFe`SB>T4fT80#CdGSB^`NEN*_1q^*fN_h;CwY<%hK z3WJ~id*SmZHhu!;cwf=ttPn0^Axr(@_Tm_`` z>;=WnUc%$zG0~Pk@5|$Zu@P}MB=bDGsgZm-m4SIo`O&=A?*To}^=;H&!0 zx$0q~uV;hVRe9TCqrIgKO7%a>1q<=iV52MEcK%V^I_vhni|+NAuG{1NAD%f3{(MW`j5bV3Z6_i3NRhP==LfOy9xU zaunlByLY=XgOUx#Mv?pY&rlFLw}vyT9W~HA}l9yP-EPYMyWX5WrnUY|K;( z8l+*pUuDC0e7OL0tdDv$Rb`PZY3=&o0kFs~S-pEkSd}gT5Kwvp0L&om^REn6Cn=Nd z(PVOv4nPAQTg#ZF;bx{FrhrE7V8{q4Sd3ad*>hRGCH%NMf)~nCa8=ffKg9MU!{P^? zNNwycEjRTPO&FAz8>cFn#^Lh8_~PaL8rg?6VMM9}R5EdlzMq~JnoVN^QkM5s)*aDD zA+t%O7T}^v6nqW6&tSPwm(TrT`Ml`3;VVKyki?SNMIRc;lejvw?NLt3L;d-Kk%T>Q zyPvPf1>v~-1lr}z;rq?Db?Rc@_OUi1WQ?xJW zkA-CwoZkiO77lI0cBeAjTr^c~IfU;ZbE7=Bjr?YL1Xn4iOt+h2SDeQ{Ql-?|(?`-v zH=h0_?Lm!8puBOutqz~`No{8o!9Y>9Db3Q-MSvdJZqV|}V7#CtPOs7LJa;i&9|3bB z!umx-RvIIYHK;8g6Xf>-;#4drP}%YVSX|e=YrE4-YxF-Cz35MY&?RPaX!Wa)lIcNu z8*g_|GBy+q*T-K$4|{j{4j1+EL1M2GAbensoCAgQ#O@mW}z z@P+{M#z1muN3%=V0@$_z3XQ}misMrhiP#OWJ{UBY0!&|OzZ>Cq{-1})$N#d zI!A|0OF0CsxE}>%lvZuvi-!|;Q$@`2ks+PBLv?t}n}X{*=sN*xKY=$hrCzJhLIUR43JFl9<|~@++W>IW3_vVjL5jE$4%U?q2n$7$xIjh3sQwEB8V-|U zA%p^%4BEX_kHjFNz7pCdeQ-P2sdeX(@}@8GNB!?R%Al2yckUjOt~Xo=DTSc+aZpma z{57UrKF|%^N<59Z=7LD!kDIlQhBZ1j54jV6bq^EdqrS9!dH$a*=;{@<3%47>Ts953%|Hd-n8*RSLxW_z+fRIeSM z2N(H$+wCx5ZXx#g`4bJab>pp@R#JEsc$O7VteB_tYEC$$5TFwF^nrvcUxIhp$&^Km z(`dXq7gHxp;bGysG8}q@aRQgksUg6fg#e(-N&hrSYBKZ@3zYd@$p;ifceT!b#@5}t z!+JDR@6I($oXo*uVoV`0#1c-h#wuR9yK!B!F}>58Cqw=_2M{n_Kbl59bG4k=?o!Q) zZjld^jQPnzh-+$8vxOyXAlJGZScB~50p0_r-Q}~;)`XzVAuG2Y)J6X)8v`N(qn;Q3 zvLZ}%N1%PS1K9;x2DkI>0LinnvT`|%r>fFvG7a;?tc1X_)fk15QDmuvY5!C_din|5 zxb^W|zMZbCa#!U!Z z@kS%L@lQ{R&bx;=xL9rY^oA` z0E(A2RzDjOZb@|~0BxNhgfK&EX9tvODKpFsr#aZo zxKqv?(dg9@K&d-8vx#CaDGco*JH^P7UuVrpivqn_Izr6q>ps=*$3zWm3wG;-Wt=7jKVy7;nj_kPy2J_IEk z&KP^lIjBXrvh|OArEHsMnrd$%f8qE{11_pGa!rR>!Uq`&rHIK#k)P-Az z>8PLPJ&zN^pGetg#!%S&%a(aus{7lG3}M~7;H1%NkJ`cM!I2Sj!;vnPW4YAZhDk+w zu)ZnD_cd@SDOJJim9I9;K!qRoCbd*X;aZB!#`@xSYp$A8!@1gt`7#VnN!2l@O)w>6 zMMVZ@C{>;WF-V@e%`Ov)4OTO}14swhMRy*ZiqhPJZK6EbDjaLJ%o72;kquvbu&%j$*l~P%?%5PGrATRWmSCP~#4ls%G#D}b$+8;U!u57KygRJs#O^|SZ&$wmObP%y7atwb9iEN}pq?*2w_b~#{ z{!!&f$1|KJULseQxyvmhUiT!4m}c0b%GFzCQNyT)7A{@75Qzrl??@KnqmCICL%Oz> zyrlFvbH0fErQ2)K??=9aOpQ})s=Z$6l?EGHy)oiaiL41GU3QU+B^Vkq^GpuP13P^8 zgkb}BtNgiOBJM|ft@q?w`FDdJ7&!iHI{g3DLcm18$o`Lw75`UT2>u9F>p=Z>%KsU^ zW@P-+EWpI_@A^VU#=q7V{@=}!jQ;}cjDH2}e{A(1K(NUqju-?17?Hd9)QEx_Wq5(Kai*_Gs#dhO12$ui>97xoU)aEM<{!Icw5Uqxwfn_xqJ?6my zk~Q=*ji_*BokWLi!;<#CRY0*vszNiG(g@vXO#9um)N!g*DD^(U0_j!uWu8^vmxd*e zJ*$AoR;sx3DcP!|EmK^`;hd_tayysBj>~>&4wOij$l2Llzhe4Y@34M61J&yF_=M{9 z@bMDx{P33a?)L5O?CE)rMEgl_gpG@PoIbxqOnb~ah`OvNP%&$O@D7Q!H+iuy)p zPkR3|mKm8faT&36C<60$Ex!y!hR?+A?D5Vnn|5-%re-ZWVfuYt^jtX07mMCauXKB} zPM@haQ|!Gof3e;gEgAC0$!j~s6d6;`Zos+W@G=LoifQQP@e2J8rJ6Q$2X)N+xON%! z@(?|_Z_A?KL(36gLJ9F$2P~q31&i zgi1gR_ZSj48wM?k9=SAPSei^0=oB>6ATNYSb3s7MCDQClN@twA2tL~K|yY+ML z=G#urw;k;RM3f3DhmR-Bh zu_p&)&NT(Tg&eevl#`QZA>aEAdc8Y%#L}fbJy&)5K}}ILBHO4IWYiw3KUu!Jx!aVx z-c;SDH|(XcK04lE(50Rd;$4oCSHji$d^Sz1EgO_x^(1+gb{pijFbrMachVW^{RZ>2 z1b!@6H@uyGu1fgBUOrQAr6fmGS9B8}elVMr=o4OdV{bG`_!ai1D>Z_l8v3O~X-eQzpu)ef zGI=2-s->0R4Z`}Q;Jwz;#BG%@l3KetOnLWzxrJk&>`DD5V36!jM(5BeSjF43@`D~h zL^;@3{6AuTDzTrZ-Q)@w(u-0;s%j|&8pu)}lS4T&P9MZVCXLJ=6>*tgkm}RN6qU13 zNdf)NzxpNeovhoCyo97Lsi99KAeJZHb2O9<;G8oE&1*>YRPtDuem<%DOq@{tyaUg% z*1a$nITr;UlIs>##{F5Evk1q2iWYTtOk{A2Cp&b<#?&tAH6PG)--FNUz)mf=;`5M& zJM8MX+y#qCrZ!dmtH>;EEGcx}N9p!xpeDL-AQkavkl{LJY$j)@qBG4hJd(u-bM_*a z+wU>XrO+h6_i?YT?c$9fEGbT0aB#AkhE@`lk3st#ao&{>d~Xc7#4)i5BX;|G%rv(= z7?jCf*oxA>o_ru?(npWL+a(_vIIJ@8;^%;qv3}{!pj7>G0jZg8?r-ly^o>Z%!9Eb4 zwOk`1Pmnm4&RS>nq5piT&Lin4PRNt@<|rVmaV{ChJ*3(4l!AOUnX657t&mP&7fyc2 zQ&BNWWk?u(bF{|1P;5!SFe1V90*o}Gj<2!@J{)QdEmlydP?W@1u0;Zo+q~QFh7xuH zD?q8^K~NcY5C?Q&1r-(wn>SOdc8l0heDil~mdudpEW=VvpK>#*RL(goAFEY;8>EVCqa+SSojC)RTy-I*wrdh$Ge=sHL{KGJDl$pvwA|Ri+ZG_7gbk}AmyF0(x5riwqgM#CM5y4XYcWy;fb-t6w|a=V{C3K zBl~kG5|6^zPo$|& z#hG{Q(aiKUYAWpJ#07ErYtc`VYsO@e2C5-{0rpbNrI6}J*4Vgikw~~Y%OCI^> zslo-$=7-J%C*z96{hdh`O#zTJZNbaLKCP5gu?VW{inaAbl9oK%WDP4pb2ZU4O!&%A zHG5LoBa0(d!&%Ii;mnuZKg02u`_;wOzEky&S4wZ+cZcCLC2BT+duTHk9yJeLNLSH= zoYbMLw8^Wyq(HwGW~eYIItf^+hU8$os%xo=oXQ7FdZRI?j6yFtCyCy@WS$lgMhaLg zdl!g}`#&eC<{hlf`=@St)E?&{wPw9Pkl(;t__fT$gxcvnHq{z`5pAk%cR4WsD2@NX zp;^DS7Ga>}#GOnR^%W_wQAto*2}BC%?5(P@Dx@A&qDgz=&dMmYxbv&%IhH~~%Nww= z=wvJ@SOlYJKvmNxk2A`kaN#IH@2z018CFcRKK13?G6{)$-5Z^6{q9K$Igp|mMtHn* z3~R8IW|nJ(HHUGp`DGRY5ZyyG&s&2ObX09xXhBn7Vv4J~OE$cQ6#6_M!^xPCFp!a9 zI(fNrI{Kg?M@qjtu=;9lVBSEV6tRWX-~FCZDCNVQ)Rfa%ao6SkMF*pp7yWmC^NfGm zLI3A>sus2HOAz7*V&P%^fZQ-?chqt=GW?Waj|YEJE1(M0F7$n@DSnyx=Vt`nXF-#-yZaZ)Q9+c63*s#zg2L=UsrpAtF2xAk)Ls4J zy-9?|6nO4)<@DS0?9isSVP=~%d-7t(ga7))D5gZLw^U6Tzg8ftc*2oLnysTZ7rHEE zbi-&}d$N&HbZIx&1gS9KvcV!@jS^GukLTrzh(^pM@$UuL4Dz zS1&Js-O>ZZI!O^N)|fwM(8LHfZqd>V_DKkd)y2L_gsL-0aObZA;=9l6FRk?@Z{yW| zdk!=kpsr=i`O*$wzlI#_Ys&(l-WcZ7#MAE#s=_Y{61wBV9!7T>4FRDEQ{HY=!HO78& z2AZ%p_Rd%|S`$0xh=Yb{c`ibRW?g;yp%c;!Js*YRbSeJ|Xe(M2!85>UjSQ3(X97+3EBt5b!Tz9v(~eyGACn&qx=kvoX38goz}3No8d z56mHYDUvclPP+x2%tR9|U?ZlUJ|I_=xYUU$nS=@bAV|~ZrY4)k131TRS{hv6Z9?mf zQTn~TBPq*4${U&&_-O87T)%cd2;KFEqvHt5x1A^d?97aI{g_9JK04Kz=tN{Z?eYf* z680~51}4A*k?Lic;HW&YpX667ZGkiz`=uIXCP3K!@%PzWZ8;k1#h5eEeayfMDCO|^ zx%bWzVi7*U@VxjzH9k{t69sv?9NEv?$A;gh{N0Px@2RuuSDyRWP~OLA2$AvH4h@-4 zzz9PmvWiG%i%5u(3)z2{Dp~zp`4+CCs&94*Ca}tMv2IbZhBdlCj7GFCC$18%0GWg1 zc3??__n-|43x)u&3xuWf(zhHjTyX+vA!bW@qn_%xLViRJa#asjQR?5o$JoEG9jqy6 zM|Nfge$O8Xa)-I>VTRRJ*QWcUp*ObheJFYp02rNlOFunDOT7{fsxJTpbgZFM`IZAk zcvRkUwy$k%USyo&p@H9B{hfG(nTTv5=v$tP10w;dnjmHc{?e3m_-Rai;rJOLg%v^d zuVOajQT{U=cJD!q%6yP>&L;c7a+cd9zs8d$PkZ%p!U2#^1@Q+g>9$GX7&28asC48V z2?CsW2>}!54v-)LUqaOJMQ8rGTVRVX2m{HtA(RqvRM8sIA0_UZw1$EZ9t|e=3;F%7 z>;&X}CR{k(LNJ}*Dsl(`^*bGG)M_?{ZZNhKsfSW_&+KGKiIQkyR?_uEQuxSjAgOZ| zMhn`Fu;io?s0wm+oRF1p0HNvx3&9*Rj~ZOXP?PAxkf5%}x{Fmv<%Q`XVnr!2mV5oD z$Gukz9%v|0IP`L|``xfrl@{+$l*)j{Q31!v)dPM3&+ppjGFKgm-@Gj4w`Mibl6!hF zGVC(#FrV-EV3iP%P38Q;XBx+F5+<5}QkE=HFh9j?`o3y7BM~`=h>!_~)0yYbM}xk5 zd&(5{qcjJB0*4a1gc`gykG?Hmp!AnGBNSE(hxQ8s{kd^6Z$ey}0d(UXzP0J*fx=I{DkAn9A<)Jer*iIXfX{uF8vxWgZ8%1$f;Ko+}h z3mri5(p4b^OHK}nK2s@&vjN$M;3C;oRmg6V3n56M+Q&le2+brC3wgP{ z&^=63vY~a7UrGYX_ROAI*}U?S_vC3xJ%#CVqO;UBjI&1EYEH;-#V-YeN<=b6nM2qx z*i1aC%KS!1GSm+6zz~0Zq;Y;P|trns} z0f`g5Um+9f+TvFL&wXJV>Onw&C9L+wDO#6pv?co8vGY-<%x`*?CuvwHjPoE41;omT zb}Q#3JruU=!wWW#@beW#$W^}69WolB@Jb0;6C=P8G5{tXXc*Bg<+=*E#BdNHD`Rx->?1|#+zMIQ zQCwaorhYm+$u4A|W1xera5~{hV?bX7{Jl^8@(fapdVS zv1I0a+;}+5Sp|W?qK?G9h0f^8QTvQj#=V^znG@P?p%Cj+u!j!#aY{$mL4GW<8TQbB>Su2paW4LES^@f_<=1RXuj{=@rLiSmC}5j!==%7Ra~G80^j%A~uyfH;t58 zI8!ITQnn@O^8fGz-H}X3M`x>JTt3eedxnNo)$DMnIef3zfw(<{j#0EJBraCB(HP;34n;60K6EmGi-f4*{mim! z9ReFzNx0e&gQk;p?4wA&U6o*fgSw?M!U=`2saf!01v8SyUdQIv@Cwf{&L+2(0O!IN zh@!!Q^x~&613@>L!o|6A&KOsNnC5mXVn8K4nf&zDO54!{o_)LwO)Vt(pE1NREBnlm z=K(tM9$V>%H0)&E&>Ex@aTEz-Zz5Z6h(Eg@mhJ=NR>(hw2#t>EkI0OUF%m?I5R5{j z6v7=nk437EJ^Q-Fl%~KqjA29vMLSuXTKe|a-tCDXDG2l{P+RTx-#*uGQT69$8y4y*&#^!@#@AwbubmBakQ@w}B&)3RP zLjhf#gtX+jwLv<#pdad(3~mHT%ei&#vpF&zsFyV7VIBKSS(5Aaqq2w2gNjv`7Nx;3 zbB{^0Guo^Afgl%eG0TmvNE*yXctIcnB10yDwv2G_H^00pkpds?S8EkvuX3U1_o@-5 z4yHU+ixY^vWrxBSBUW1y>wd+r<*-Ec6Q#mYuZx>o(~YB-w!FWH!fU;t>Ck2ca5Gk~ox? zGJd|D=YZG2nD^&tfs>o*pp>NRMCogR68GN0W>Del#)3NA@&G!CSVH9BW7`kIZW4=h zI!f=@ey_lG14)>iD`mYg(~siK>e|fn_ZINzG&t7q2K6q;)y8-4>%t!yYtHS9xy7Lu z^>Ur4iCe<#ooX_rzZ%l-C~q8?{9}wemylID;G;|S{7}b!ZQOs#ls^ByCbeaKlqwyb zYLj+F+wwjjiD%>&pwFzS@xRg}m})ld)khNboKk+7gx?mblK52yY0jK<)JN^|=ml-l zwS+>mah~B2wkx(mPv=QC4W1lFU$UyPM>`V@6lKa3lRp;_kGmuv9Q5Qn1RcK3s=5bf ze-aRN582M+OI+V#PLB-ixN&kh4eMxs$AT2J)G?@FO3M3&WmFiCon3gt@^bHA=Xe9? zP4fzh=-w{1MNVsXj}CV~2i|pH*5AQ`Rf1j3+1tpP%A$tmlVrbxaP8o358p@l>y3>6 zm`+xjO|h~*#Pnh_EYifYzRhW&Az5* z=y)nJ^<7M(aq#2r`pyr)YZu?=Zy+k;UlOwa)3)n>5hee3x!3<@z!)btBy&9FC`5> zsc!7GEW~ukXib)zq0kZF6a&(U2Q(aP4+@VTK;sh8?C+%q2hXtHZRGEr)8|C}8=ZEr*MpBur%hLpzINGDT zXl{{Vyvjq}NW%W56gel{7tZx^nFwFZ)as>OZilm|v*|H7dG`7KFDjm6K)KRH?p#w4 zT$$-)B{VpIgAaC?K%I}z*U%pAwtRfpeSt(*s7U!B4Dpc$|#fp|Jr=O=*OuBkCZ6{3{KSR;r ze3oV)VHNjpZ&_`Fj)OJya&9ywn8r3)itM&CRgJV%&+uBXtTd~=hAgWLnl5dsRIT2? zRE-uaH;`4&Q13Us&|UUFb8G#6oHV_?5OMzWn(zA5Efap4iq_BejZ<8`WZAea)#F_0 z@&%Ppbl%b(V7IEdA9?V8pXvHU2D(RYg`)l5u<^nI|3FBA;Dpmr9#5IX4Sz*wza{2Y{CDnG9XHQRSf1iH|z$7;t9QuJ*;}p_dO= z_ET`W<}B;anQ!ys3T0SLU5$)7>3wgb5D8vH5EGNbgzs4|;22M-+lqPRY2eTPa9=~c zQ_6ZQg55*~?K5_~nGHXXLx1e`eM4UXNDeGIID(#b!|SWAg2CprwRgn!liHtO#PHXO z4um#BF`4f*+tO|AQa5O$(oO=(u_OrG>^=fkx8A3d(c5w4*V?%OH`NQ6c8`_w)-87f zcmn?@6NB*K4&Q;11y3))Ej3=Sk;9werQi8U5beg{gn9vM=E}KR3|IPg4gsmoi-&NPw8aFNZRJkN3WUkSnOduB=QV^xu?sm3}92bxWOBK|(v4en8(6w}FMAJ+LgXFJIij@w6YleC| zS_|Q^>Vch)f}Q3Bk1?m=y!FO0=K#E_TZXjkk5z8jsdq==vkd)g_Y@4X0D#OR9LbHh zV8xAwFKv3A1ZiKYv_p0?g9t?6L8`=rEhgXiP2Lk|821;J$Fr0Q}kXR-xTX=jnZmLwu1@(S+q*jG=lk9jm9a6EGvQ{ilF%Ann4Oce5$GLI5M zsXNKu9byvMqG0grfRs@TjhomZRd)cYVlj{qD4;STO48)-pa-59;5H=b<2)B|-gyMY zyWXv40KXHO9LH}>dRj+yXc>k~>r&9Ak+s65u zdOLNMYz%bhnpk9*>e2G0Wp5ps#UXT!(RaNThamFFnqNhO9EOM0kp=+dsSoRDp8Yyr zXdQc(C2yZ~fHMex8Bdb-q26)t95DAv4Y!N$#3dYXXz7JZlpljaXA>wpx#124%>o^k zmf~nu9~-3n9y>qVfduko*;bmj{DT0!7f?_J7~~^dxBp#H+y5>0sF}33b&ZGXwW_Pw zQ>?(r<-w&!)vl_^Q^+svCqB%aW}MLEm-i;qIaI@5W65kn8}x;6)7ur3wO@{~&NX;` zsFQB?xIfKAh=me@$xyy{SX7BUvMEYF6 z%JpbsFvP~D8Kjq`6C=`Rm#tW~_x0fR06}Nu)fJ^D59)WipGl?v4|nGnWZ9O!`Lu1@ zwkvJhwr$(CZQHhuN>yk%St^*~;vIywZve zxPo4A$YY{26jb`XMX|w2-7;+k&fC${AYB-{mDA!4+CkxgTb*oZV|| z(~`RDvF(U~6g?`Bz$HJLuiMbtxPhv+*-E@BJeYJc9|JFnG{6qFBIV`MXKp>}Ewa^P z2|Jt?_QiCLSYxwg-<{dKQ-Nuo=#1s0lDwm*Cop__I}@t&F7zAmntTG{0e*C5W`NInEP@a8L5K|5EokBw{~UvoMqqyDyFq&;O0qzx7^1r z0XPYskKJ}+Y~vKyWekiDQnC&U6E2);9dFIXsCd{o@=RVSj?^0V7ckIv_wEddQLjga zJ;a}xNuDxfJ9MA>#+q%U&i3bm(Z0OOTOG~LC7$hxNXl(8k3M$6noQTdB0`xa(Yr;X z$H}(M5Vn6Rzyg07&iqSKw07tBkv0XZwf@glz8IWJ= zS1nnTxjv1@4~rDPeSrL`@Q@|9K0I*<3Q1H)fv$DM3m#zb9JNjhQ#~1r7+Y=?m?vc> zs?cEy=Ha5uZ5OY?O}@#*sZfN3hYBhUAnHB1u@h4yPxC}6G<>({1G4Z9tlLIleW{Ih zTi$qGSiz&;Qu1yl5Jhgdm)a!%#)g4QULcWAbQ2_?E4x{h0xO$ksQi2j^P=(FAmSZ> zy2meB3%$V#{@Qn&y!W9EL1>KG4ZFZ{QWKj~t!Dmp#aGG;WN6FdvRN?0Hm~L%}OCfxG$U(2Nb$aZ2CV zl!CvjW|9jhdArE;4licJMZqrlfJjJh`nST)U>#Oj-Rop>I_a=E2^1<|ny46B0Eud$ z1aELdOA~C0^p4!5&akM*s3fCBEBYHRdP@N!dDI}Fm0a#K?O|FHdJh=udWoh-! zv89i)6PtKzq_&Y~TiQ^OO%+Y?UVGc$OJymvPQ(nH2h)$o@JDQ6&4ndLkCBxLbSuAM8XNtz$$_h&VGlJE6Oo>zi2!I2n^Wk-M(Oc6<=m-~-zt_HDVOy?J}V z^QS@Dsw8Sf5z8CqXs_P|Q=oKeRYLq9*4Jt%4j>J>HCWaMtOd?=fed7=Y2TD3pm%w9 zim+(04MV;Vuoq1trl8uBz%@9FpLEynimz;M2{+SjRa#}eRe}d7y zIXj|6Zdfnte?dPZ3P3Vo z%O~e#k(h5WZq)jYG8=BN!SW7=-3T>93&a8xB6|fEkLYO*xuNi7;MON3M!Rp9< zCjsFNp;>$02?{E8RT#i4>#e*M_rHhO|V2g~u z5*1DwLmPf`T=c|8h5J>O0@qN?tDwedNL&Jx5TJdaPN*j2AmCL23 zur15=-A6*qjr&ahX-vnBdryBRSm;tGAVQ^a08fgo^U65rTe7wDS|V&~iuKk@;Z#~{ zQ{RI37+!SO)k|#6!8PqM_iE-%r`hjI_D$#53HNI59~W2BYrN|7rZ6G9s=~Qe>ep># zRG-z#Q@1(ghjp$`*X}ImFT=M5@1N@ zcJg-s^@Qfu@RLUzMOdWCrYF0I1TSYTEtR>zZt#_4N@|{UELK9{np{;2sD2x^&J za5Z0nqvFN&bfMV1DnM_Io2oIza@fMl<5lIhO@Pg`><1n?K44W+#BHm&Xsq+_#mi1D z;2A_sI=M&oz$Q963!ghWqqL9RT7{_2EGLhpzD374I^RKPvb654dnzWy9-E*a8T)tDQCbXgJT)I~99lyyrGB)ca2yZMuDVT z0a*6;p)3&*AMb>^Put2bM|d!hILl(WsQ8;V=RK9P_ra}LUDqjHs`q91Dy~jnh3az+ z6=&+vK=~D;*V5nmEtLH{$nt_Q?aBZ`fimE^)fY!U|2*y%G~hr88vUsyhm(eNPtvYm zPSDqLAs8-?`-Ud8x-S^M*%rAbkD0!V!N(IYpG}opeaR7iQjEiMe0T5MJ~W-5S_3~^ zU%*QHpKxlHGuu}r7j|ov*YgiY{AevxhgeOH4vTyc0R^9ke2S&>#Jmwx@UK9FHNSKp zhvl?`&P9Kpv*=u8k7>gKEjLT*#2{SjoAS50sraA1=V)#y(k6D2vD&HASiDDK0p)!p zfT0bei$`QNkV#%wD{NYw3EygZE^nB{Mo|rz^iQhFM!?9-{HMPFy}YBHk&=nC7QMW%D7~_YyEDCn zje(ho&>zhV^fD&K76yWL?gU!Dp(FzVD<>-*6FVy#GXVz+BOM0^Cp$aA?~DQ=M?3rf z;~+t66BFbA<0vslI~V(ZsqFrEL9axw>}X)?WN+YTVr%3M0 z!X<;kDlG7XhQiV(@&=p@Wk9y`myQDv#L4oqs8!}Jq7RgWWj}%FwAHh2#qWv2506(I zA>D56y;#vQZ4~bv01>=ybj014*n`gOFLnmL8wbMcu-tY8khPB1PT@TBy8J@j7~NQO zyGWFfq@F@4dCDazvoS{#em$qtvtAo0Wl`(&lM^r`s$5QF|n4DAn} zFdg@A)i5tg+}|Ok^<#V6zJSJC2P^;P55vOnub4wN#(#SuviwKj!T;W8!~UnNke%@_ z#u)#c>c6<5D`_&e+YB(lPk^4_x%`MDe2@^0=a8M}@SZ{WT|0fuvvg5{ni%7oJWF#2 zFXEZ0HkfS6o%+nfQCdkyTb^Wty>Ta3!83?8H{N z%wwBWF1R!cjkLwc3%g_K6qFi^h$RPw6gA5j6pCu z#`#S&MMeD;lA=M3urMJ!~UbYz&+2TzSjszu`+ zTzZleUUdnGuIZS}Z)Cx=_M`R(Eeworu|`RX>a#?H0)6hs(rsax$YiUgl4o)y!N?7n zB(m!DB*AJ_69evR+?ebqs=+bSsg`6%hu_PS0)nN{G0llWZza76GL7{CqL%{45fP0M zjDTL#($GCVsrqi=7tiixa{IoXDg8csdn7cPPjOB3y880|;S@?d*R0w=zh()1-Af*K zPP>QacA~O^h;b{^fj6OHIb!?6 zPWG5NWpWePYP8mClq-?1(QDdNJA{2L#u`~|!tLll`g8nx1Ii}u1Jwm*oH_0=?lM+% z_JbR{jtdt4+2@5LiLd?*gBWq&5 z-D>~+=>4bR3kwqq<39%ck@d;K@UOT3ssUyBWAJ~mp0~9m>WC*|dQa-cn*d8Dz5}~c z@JmX7d1pBQ@?!9Qi~Gh<31+W%elsUFni!>McSbj7(0Qu~Wl9n>kP|)sT$dm=*q7u@ zGNvTSIePc__C5FZbO@z!!j#3Lk*=K}Zd2$y4xC@&>*GFd@80G8BS)@au6sz%0a1Fto?cL=4Rc`uqh@UB93?GzUMwh`WbA05^ zc*~gbh{yM1lUNh2ncJBd78lpWSycS}wGDm0V_Lgz#C4MMnQdglD`)Ws{(}-;oSz3_ z=R(>7La%nm`}Q&Yf!ev`9d=+ihQ|l$a4qNWh<#-?$d)|b0t?|dSMD7ds4;z$Kgl#*?tD75B z0W+T4qiiZ(A+^GvJq%{E!Y``&Z5n+`GpMCC<5bQ3AZkI?)jDUzkiVj7gWUTQGUACLjk25 zPs6FzgeSHrGcY4ucEe*orQ1*_mvZ)L<2!`T=wgpCgp=cl9vUh+Qb(^netQgvAq^(O zxJUrw&U}v6TTLP$E0;uC`l=Icv>PL5>E}jxDqH#+4Dp};SpZX?wTxQ#t4Sh&XZ#g1>+91 z)pm4o6l_*=MjL#XOA@$$1TY5pGXqJgy^FWqd%X84Prs_I~fjjY5Y* zG5dAewQnqKAG6Z632cHK`#AddMkxKemCA?AJ!%dQL6(sa~Uq~kmQ^~xWQSW;M$ zn0+YGYxGfTvj+g=!>r!&1?Y(r#)H_;Dn(u*2}7G9C9x;MLJ(@F%(|9P50_io>rB=z zKnWg94|5*U3L4bR+fH2wVm|ms$*CPzpJIW?I3t5TMboXT#+^3$LCAz-whgN@Dan4{ zt|uo;{n&EB7YTzIqttq?6d^m&GVBF3>TRLDz3@E!P&*w9uz^aMz76p7ew zSSciEtgTK+!g+gNVUuZmmn^9VAq%HBfu!t*9$HuG2OD?@VN=cQd1ayySS85>`CPAW z8Ij8Rw#u=%cvKj}9q9-S{4+H)R2S3~R~y3k-3b)+^$I;uY4_lR= zVeEnXTs|1Fi{J=vuCKSJ`@3=EY5SXWvP*X=Y>X#({>ne$a?svuVAruh5Z%4dXUuX| zWnnRQFrH5V3aIwGDV*6*(+M>tNtM*B$?XjV3s2agxGve><5!gx^KQ@tCP^X8S->LT zfbzeO+pXI)Di!X+Bk*q1ZlSntx@In+1bNIUQ6-Y+DvH~L$_H8=g|G1GG<`#lvy48M zB=z4}t$0$uC%XsiRiKLQ9bQX!TVZH2pPz9-wQZDZB-C`86RtJvj+EG3y z|2fNBtM`c@*9P&dCH7Mpk+XZ@+I}u_T=g~W^-jnjE*mrfEwN#u1M3$DDsMS`p)CcC zVQS6EW|u)|k1+b3q}G-lnw};^IDZ<@u4=S6iK9hT9{K?%K5B~n7X^EG+vCoy_XPX5 z8JmNOQDQk4+KjDTJ;}k<yCd z5|%AXE9fqy;f(OX7V`PCx}>E}!z;oh^pH{ZrgHH)*;eP7bY_lZD3UKb^Cv4Zh!wX39Yj15F+_w7;G48pL(k zz+f6#RVJn_le!*Z#~QAYujreh3Ml}8X*Ku%JnZ>-_oJVNI)xqkD_R5M5y$q~w_|~n zHbyzUNho=wHLQlyt6Zk>m>DbnDwt{@!!HcX+P+OD!?}oA(11veF2hCcSRB|_My~VG z$aC&49~amU#`7}fCAC;dhQi_y^M?-asj69ACV{w<#n7$l(cX&KN$<=a;RQr$*}?$& z*r7G-aNlZ6=k-;V)3-P55O3R>uNt&xF$qj%muw_uFDXT{IvRT8hvVUvL{@a0az>?t zgSXA{BRgF~8+1a{t<0nusd1&PwK(&V=}k+rZ`bS|DxQ2?oE$^0Mm4vZYNG5!!Z_~P zq!Qmj@n*#8#31`?L!$so4Dz?{unRRY2&2xmsgPF9^ssPQDCFm zc>#BrwXy~zWRFQs*(LSIyPec3UPv=;(f;IgQ-j?<%7`XaT10BfS?>Jlg3FAV3J{~E zXizTCO@;0U;4D9CPKHjhwB+P!4&Q+x-7xc*&y&jhi(+Jh{(Bf+9qeN!mMQfgX>g;aktVoeNlW!GJyy_E{b@3O)Bv%N@#Zp$Se&Lg`$c6W zDxzqGjHCNQ)YcdQGJq(14CJk^D((?L%oHD@4Bb!wkj1rgTnr`zBvP)!HnsVzO20bk zNMnK$kyM8z3_^hNLktBU(U1;Mh@q}}!7<@oqUyL3MS3pXFynRL2R-A{+6rUr6&#?z zisAL062}m6FgE#*2NT{L9+!Wo%_dX`hS=M%Dhhm?qr+jMT#TC>~LbVzoW4e$!+jZft7KZiN-DEkFS*LQPQ3$JzzM!ZP zW0*f$xio+Vr5-0CSp`)Bt3pMJ2J%iY46p|jK&$2F7z0AQvO^d+uYvsN;iA^pM1gp- zMnzr@OOYWfoYWSsed_@Dz3^*Wzl!N;5O2wW{GS@5^J;%m8UJ#cb9j& zmuqEkp66aJS@OQ zr3r|PkJ705;$j_NiO2PjDS`5VP`!{PVgDFHYvBIOPF5|ydkFDb_sU+!xMI_}F|^t&mbvJ=P-;in|0 zu@GyV53NA)VisPG7!7KP*+PPOp zQxe(F$<|HUc~Ij5B5iAK)Ul)?g?T=mM96bj6pl_4k~RpMV2t`38#gdgpxgLlLZqzUmMHaF8&g`yz9HxPtKOPqeksftlx zx&R;U$@7h2$(pY<9%RDFn37JLI)-b}Y-tWUL5Mp6nG(FbIzBa(gg11lZ@2?n(Du49 zSknS-NVT$yNifLzU36Z;$HgfjCvEk*boY2{b&j>j)k>Fx6{C-r*zF{{jB0b7e9#83 zh@w}%>O?*gz69LGz>m7kJDscbkDQJ>A9!xKm?;>Qb7ULBCCQB6M%7X-ZbYTRX?FNF zZKc!LEQW2<$l|C33Pm2Whr1Z1j5~`F5Fxw*-*CrNGr>8>=g6E#C6>Y_`4L z4W=nvqn*3ZVq`g(MV!=2-^rnl-2&ni9GvnRj*JRJY$c+puF+2K+zCRJ&F7qnweh6V zu4S>2Q^%B|xc2ZL8J&RBd!3!Y-rOB>a`T23KFk~lP^X}&;Hl!_pm~RcLGE)_=16uv zOz{Fo=`1JFHRwt__z>FgT-#n^nrZQ&kJfB29UrT`;I%IK z$v3~?CIsglvQOYH{W=k{+_0Dq2e1cOiC9b0I610Af>{5Ii~_l;w(^=?5&FcW@7(n1 zBRnvlgizPOZj}=^gDy%TrlE94JMRE_NqJiKx>Gvz#SAoRuit)05z494fUDuFpK$1t zpvGddX6yX%-J+{0rCgR;Bj!sm+O7|~Ndqg?-%k$^SL$mhbPrqCCP)FyrxR`;MJO69 z2z3yYqQQ?tC@=2T=+04RY4Oa1|P+y$Pj7vWW70$t$LP`WuRch zWQ8QUN)GO^5%+P;wR@JVTco<BD(Y^T(OrKK8mT7(@6*HqF?xGuLWW9(KUlj8}V9#M=U(G zjY6%86RqyIZxOP!5Ky~avDwLuJ2flT9Kp3aTY?9N@_r~aI&o;QxM;8|hMqD*jGryB zba3~}zK)#9mCt^FNG$jJf5$x-{%v&o|BS`|MBsn&@qe@Mzle$~P5=0&e}W~pf2fiF!I;<>{z6yT{?&>6cd(rGPZuNBzq%N){4v#k zL8FdTC1O<=VYY~WCj$3haW*WUqmt{75ZKNej(t#up>nqkr}0~Vx+P1dl%=^>G|}rm zM;)zybx6MX(KlEg&~UneQ!`@#y`M1?O*OX(r(JcKOg-9ip1NW;06}AP5S7IAAQdyr zPKL8t+Sn@KZ0{6OS*bBs)gH`Lx5~0My;513#;BBK>;=}dwhUVA%>3S9KU||#TPq%r zQq4fso3qtW4qp1>6hw$1XVITE_d}^KOoPGJnTECow{3#Ov`+{YlGwH4V0kPZ&3|9* zf{XF#5hQV}RfCIZ(n42OTCTh(5>#?XkZAiLOd|>{-_8=U3On z)$*V0X62f4)g-=G^RM^x&@4I=(jw+Z?P5h5Q`9Q5N@e$pI7;>YdhgEVt*+Zfirjf= z&*k;`Z0!AV_tD`_lqGO;d%W77n(E>UVE`c%|9_v`TDI7ih;o&D!F!i3oyaBj61L^lLT>aP^tjm)+|4lM&{a0n{EGsFJy zd{!_;!|ukjegI?sVJB-xjP_1kq)Utv(+n`x+6Zc^*=Mm5jA@f2ol{+NO|k6vFCR<$|F<%H{iVm>-&I6NuG> z{qj-#Wa5LT{|cq{3!a-|2UNyTY$cp2ZyvJxa!<5mRIFn5=I*?JSKH=#RlrW-maK3B zCzS+eKHvT6l9tSJOq7%AC~JmV+LS$qyzS(HcvSJiC_Ox&6IdpeFwW?CGj#p7dE=ZW z;TtWxAyBo@zPHI2_QMykjn4Vr!N2r%;0(?oHN(!aLV2zUNxce>|2_B)H!P1(lVJ1hPF`{M^}N-$Ksim!m)lhbVyb(S_H#M!^{wx68&i zF)@90iCE5MD>$(tTJ;;mJbeV_aB~SoQTihmi3~bp3CE1|(6z-QjcV9kGl(k*Sq99l z>Qal;W2_PQPr5wIb@|pY;^Y; z6koxWo_L?U-S4l=$&JS~l_e0-cCBw&E!_QSf!uSqEE9uE;)CcsA|ya}t=iJUl8PQnMY~P#&u+2lpPC|VRXrNUnBm_t{J?6b8#=Nq6i0OlZ-L~ae{Xw_ zYc8EbTl^I*6ep+g9;tylN%8C9v%Jhs50RV`T{VlJ6g0q}0n%q26(9j1elYdpP|BL< zX3K3L9CrIdH!QhMb3Ltdr!*@$W%C9!MON0i>xc+1--2FanXv2}#Kku+3}y8PWV>p* z3>P3CRjzoBGx{gp;vUuY!F&a*`=PSp3E+bRk~@XNRF8d3-I8m+0?q){JiGyk4-lYb zPmtWLCiJt5sz>#-ym3MCv@S5>B;21)laJr68BCqzCVZg8m_Vr9Vq8}Au|&?-t2Wal z@XP{+Tvy7Zh@J$-y%e|~3Zlp~hT+t`e3$OORHSH&1>`?Rl6JS^s+)HgdN-c#R(V8o zVVA@V$Uycw-l%Sb?3vLB=j*Aw+bwC#t@Oti=f&_YTla{+WXK z7&4L@;M3#=j;X*6-2e3)imU3R^!tumQVE+H?sv$%OOv!qQ zongZYq`bvWxsKJnS)8{wmspvR<1#WHjWorYVim0;CHBKanwUb&O-HSq+jqAtS8N%n zvtlZ%13Wc#sqbR!3af3aLCG>JjvBD;=+_e3_levz^L zGTp|YwO+dxYo%%XT_5bUb;OQYVjd}YGE>7u$ZhK4?^Bif>)o}~R7Le(oOi9+=rG_f zHgZ7;I8WM8KZ>#F&eVb9!>?Do>q@>TBOjO18Tm3NC{}_@VlseZ;+7?9Uia<*Z4=6N zohI({&l-3~GaGCr+L+kY zuZ24X!Wl((w6gS!$6WBwY5!+R!{|(KGIxI}B(4FJ^&JGU^aK%U!rt*c=^x~{&f>}p z>Re&1-mcv)kvq2XB_)_X{wiDfz?ZE6s*yOYnkX{0<)JJAV#QFj-Yi;ZOc;!1S*ys_ z2G9jyg+=Sq{OvwrZkdtUtZ7XQeKQ*(0lOiF{pMHjP#}4LTNJ>+c`>_XaMAeEdU1?Y zl_I1xwOtQ@UzD-$s0xEJ8 zS@x{U(+fl@i#d#SMLJpY{h=LfuCR~X(S8)&Vui$J`L_l9 zf$czX^IU)V#fS?ja=#UIBYdUHX!IyZR=N{wR{r$9MF3b7EkgRmg)+goYS$(HM=|bm z%wbmPNKt5L{TLj?3JWT>kvN~Nu9)uCyeUd8Lb-6N2-lW|mz^=?PFe}<$VrVcW;CFZ zMr&iSm4Zekpe_2U38BZ~f-Hi*eXVdu=P`n$3$NcVHlW|ODP`fqcYE;Kh4ii+0nS26 zLirdDguaUyGz8ksOEM^Sm&cpSyVBkB1ocHqn(SFx4*O@y(PA1pDz{BQ(P*5wBRRkP z(e`6oW0lgGIOt-)<~zXcbBwks39AVF!MlOb%mUjVqnh_B6QvJ@>XV~*jeiHrLom~Z zO$=dHV|aC`4#D5spZg9AACL3xidw1}*^yghPrQ`t#f{s)dfqC?<87A%iue zVH7UHW)97~AVT$q&ehm~gU1g5R#Gbkztu(cJ-!rY1m>LaTE~L7sb6-gQBP1$DYbar zY+)dH8u_wopy14#1JDm)$Kft@=YVNa0aMTI`^eb^Gbr%R3Kjzrwhb?lu$03_%nqlB zVw*l(e;{ROb~AVrfUwuh_e*SVOy>9Wl93P0l#T%ps``GUfo4qB;}aABT?aNMC7{d; zZ1xzH`OpRjnjkIq@2>-J3?SqzG>pi)yT5L^tSLLL*sfM!d)a>23)@||$fem>klCKE zmcG%3d#{X8Sb_sYo6DF@1q<81&)Z4M0}gY86%pT+3l`GRabPk!#K&8h8x`E3y7jIi z34{uO&U1YTYXrhS4fl;A>Lvhmmb{>{varXSAb87{(8EEu9+>M&^8i+MJDBW>vm672 z!sgp9HDKu5(FN7c`P(Gm-=%1BP?}6@3T5{0y@^`7fg(qTfaT^o01e~ZIRZe!lmz(H z7__8Rl{WLVINK`fp-NrI=uZWjIAv+st}s2y@|EV-%-QJEl4ioB)eT!3UhW%IY|-rc zR&5JFN|IxxhzaGaRFvO!+M1RM+q5sj!Z2l4yGpw|E;(|y8U6qQwrL{+Z|yIls%93Q zA|^#(Rp=%QG@@k^VT_>05LcjN0A0|d;HYHYSbKA%hu;QLOqL221$w4HN`jjR6)kvk zp8km_g8;f(8#1UPcs*baU-O~5!G`R}T${NQe*KHY2oqGtIwz=#wD1Yc%R1uZc9;RR zctBqOarsf6wFWf7PynepAe{2SV`5n>tT~*=853&#?YjNVJSw9eK#XEhgE@h!pF~A5 zm?+h;u%++rC>64-yz7eLoWckS3V543j zn9IQ-$VR`8hn^WKC-^m!Z3EW+jzB>_gS|Tu)OXYWSzb#3RGiAOjBo`VLam3GAgaE@ zBEAd7%_^*~p<1v;E_tmlkcdhWA=OquJa4npk*=Ckc%LA5S{MOc=*-*)7m~OqAR|T% zISL+&l|2hEU_l(R8=W95N{&H#8(ry7B9&h-JfOKam|9;5a)ri@fl3;~Pk~rxg;Wp> zSx_+SFmA^%5e!F8ElJR~hM1`JG%jyZ9F2Ml)=q50Z%-2`sPBr)P^OxLbE#z$mI}*) zK(~E?k5x>#?!v+4&9i%sh7X&X0I47FLm{ZI*b!I7vjHwr;@I!UrPd^HsfoJ#l(HUZ zVz*O2J6f)T7FFK!NXYiW?({CDQ+U)4pUX&p?d@7!))ePn^G1KaATAJGX^Vae1T@4l zi*4!cw2glZu_=wzBBa{c(uxut!AerU!nhICweH2UJRaycT48TmA1;mErCi!Z@3cpO-#^cjI<-EhEf1miaAC+-U9@sl` zo4YmT0Og%>#?18N;lx`tZ>mz($q1TH^ICmwBrZ}%muEUE-6rs;2`~~`1Eus`?+MUw({T-^~d)_Gw%^ghkNe-!4sS{2F9&%|@b`Eo@*=6p?T@rMo?^`b4SW!kF098Q((Hm{dzkP0 zyEelliuL<%k{m-HK(Xz5{SDv9!9lYPl==hO>DfpTVt(_kB59{8Z@(fccWqLYfk9c~ z>sY`1pOuJjsxFrRp2|`D+y9jd*(>Ea}&5JS2>KRyNYh?VW=> zOn7$D9>OBhMtv)Q;i<`p@RF%~Ys_1K@$I5V16Sh)dQyQ0sMa1Qx3wBB$S>j`-8Qg# z;Q;2|*Tkd;2SMlioQ10x;H|p*i}p(dPDZWVEQ1-J5pGGG+4YI>a2-e6SmDg{VtRNF zaA-ao zKLM6;rJQuwxrIk=zru5_#a zF6obj?cWX*{~w|JUqtKwgX#Mpbn&nBo%N4sfd7!Hv;OHj%gXt0u_x=F$pU{%-+wFA z|7rZe%KR5?^FOEhFSzB5s#L7MbB=llNaOkIL z0ixq?0}!?uf?>)uOf1NY5e%%69YmH%?y2YXH`9gb)-2F(ZFZyi`)sqXthOiBGHI=| z7>SEJm_huTJXXZb+stRBo4m*`P150+%^%Dbf$?#5br1Iuy}$qmY(AxF^m=^@b9y&7 zD96-Ydv7;mVQp>jUvNIZI@x%DXF4p6$V~0$nxGDaT-|hf8NG%Kn^i;Qq}041K7?)4 z!zD5@F}(4DE(S9OPy8rK!TaT>odF#A!2GDVOBII8mQGdS2l6Iu%^!Dj_7K#Uxi~^D zIPI5nf1mqr*FO#~IYG^lKB1)#Fy7#KsY72kwE(M)z~Ew!`S0px55S7q;o!1)%Et9$ z(w!v;=A|x)g0_Yq+3V%$+>#n=PKU-IF+C_SI|LgTm)@$E`*AUxH!zhU=SOl=u9ucP zxtF}6^t#yehm}ZS?tTEW1cBEEIv}XSmUU=v?0Q&myknTSIL8jtsPB;i@REzIG9hE2Tus zy$@tx)GZZf+efq1XH<}Wosz?Kq1WRgm2lI$vAoAyJmZma;cGzB(?6<O!!m3~v`kg|%2-KP4%L$L|=SGrAyr@p{~Y-wxULgghf?;Xe^pzFX)@f2&^ z{Ibuf?}e@}uaPQNBR_9^Rm^};u&3-qyMJ4N3XUsTPe=u{8S5{$ z@Vcos&0M@8vp~&p%F@myGEL*!NXS~B&~!bKNcFE+eqK(^ zWV=0UgxT5tHaq70XWfIGy_A@>`vv&S+BYX@aQM8S%ZM}Agg(}yo%Do<<{nu9K_-_; z^cA`#3IM!i^D%OZPfWj&RAW}sn+wLTbRwS}Ns~rM$tWZwY6cNpXmaCdy@g6UX&5B!WClplF(9(*8 z;L%jpcQwA{ibRUHny!+ns)NNQWF2+J5nwP`JkP&)c}xf-zanwdoQ?yUT-zKI3LFV! z6Z4zEH!_P`eq+44E@9$Y}G}Y=-v*ScB{aZ;F=&cVuNgjA!$a8G_3oQKgT_3Ck!xP=8{c* z1jDBTKqh)dT%(Nct+zCO88P4pG|Ki%*G;y9z>JellKhk?Gu$+zU@hBfuLXce?R4aw zNRs$?wi?+(RuckYeq{tUZw9Q>p9aFL!!LjJp1<_+Z6rkQxAOl5OX$iTzOhq=W_{c1RM9AYA{h9q# zprh1zE%PYvvLKdO^iqKU*BK<*zFgCK=w0}j%{lcV@nA1eu-i&L$ff9tXLPO^ve%D5TGh$7Ad#MlUZx=*;d>~ zg^c(gxCpHhD zu)SGq-gCwuds+Nk{T`7LN@4Sa6iyXD34(#rh{PEqD^aG~MZ(Gwk-e>ju< zv@I{M?#WEna@JLCS?abhcK{z=FoK%`>r_XETv1vvoM%(+@sHG{&uzj!S4FWh8c}Fe zL#69*c{p`R+&oH66*xxJBnp+J5Em6n`>q=XPv)0}9V!y@cq#ZEYSXTyR47cjOO$U^uA5e^cPlmr)z@*e zqrcQ}XH7%L`0T(2KoNyTqrMC+y-uGEUU>Kp_2GlVW&$sE`94=a;Jd!MLs|38e0z3$ z+J4n-kHy>$cpl2(suiCVwJpOLZ9PX5=*Q(&gl-CZN^s1;W~^1DrnTpz@1%}R+C}`1 zw>Wu*s)08T1o~bMV|)dsN-hAc-8P6uh>sI!S{D`Y*PLGb!I}aTsAjynuUw#4)5-Q4#e3=mNqtldx8aCquLj^LFa)S8V@NbMRY(>T!sTt# zm=;^4&@t`O;yuWfN%Iy$6S7+OP?9zwpk~+-3#(%am+35%n7Er5L0nC>l-Dz4!pod3 zjSnz4Ay>Lx^M~$nF^8kt*bQD{X$|*cL)g%)bY{0bp?`u-X=3WHT3WsXd;LCcn8F<5LdGcz+YGkb}dnVDrVGc&VfS2&H7joV#HtBez!!CSZ z#@-6L#j#(+4=_GuZ=J}X-NChQAnok-kalP#(>{S*D(91O6V2X1X18n~eEC76JKG-n z6>s;6qKp?3W0QOe@e0kg6_KS~Zh+ynJ2+R^>)>+SJyll-v~9SqwA+;c|oR({A9Zk>8p6_f*&DYscxc1-vAh|T=L$M>i5i$%}H&3VU#s_RC}rbSt1R#sPMXJ=Oz zFAtAc49+8+=u3}4a7G3;)YbjHtHr#wZ#E<%Uerl=RMhzJFa-@wWo4yRE`_nj!{yFG zg}QtWUzgbok-q+xpWi!!=Yw!uZkN+m`&}R}ZS-?qdPRYFY8FDp?|2y3|}9d~zkLPA1Jcuh2|2aQkCnhK(#qL{gwtT7-J z0ZpBekr8BMWVLkaZ^M!3Ar_XFzH?}7aoE1kS3^ZbMWY*$k&z7z4d!A+-@kumZoQ_5!^0A* zYDeZ58yz3{t|&V#-Q796e<&ZBS5#D-ou2A-d63>ujE;V0F!*+JIAy!hF6vv6*lSRG z>iE&t_97|@?z381D6om9l2=4c&+Bh$;9MId6qEv|o-LmzS`COg@mRcpRp8Y7v$gxH z-J!-p1rLw*oSYm`9xV+`pT%ON0@z+sf@;xnVinf&QOhbJZ^rI z)O2()$*5Em6qi?5bp(xgcz9k?)W(4&(e2((Y!m|}4<0YSoxOm6se}UDZw{)oTM%Fi z#G<7nCC#_Gy@z6OuU48Y)y2idHPB#zU$&7-MNx6*=*WCU5wnV_(d+TLWnvomHEQh* z3=C{+=+J??_(1+VM&Hxvc9q}Vt%nCslEYyipoaF|1U(3S7qo&nI|rlhaWqQ`UL^k( zw>U=>aJJsYOaW2R<#}Ix^h;G$b^rX_W+nCRc%g!&Qpqk)Pfzc$8kiJfG8w6< zVN<}5Gf^rp#u*Krl$iJ}8$D=oYwPXT(Hw9*ECyZwrPN2Qo#YbPjF9WVkBBg4NnsY! ztMv{J4k3v6HiBkZnVIOcS|WW?hQH>4=T$C|0}>|H)iIEfcNQs}RD;ySVbN*q?CtAl z8oRr@St_ST63CzKFA(J`Xy%1rKae|sbVg%N%?p`UXuz8l*mbDp#3iJdHJom8>$x{A0LT9UmQ$122(Jfn@e!=QpiR? zKtN9)>rza^F$Y>HY0)CggoTFQNe~}Z>esrXOr5+ydVYCHNJubuEGOl%+xBbnu9N~S z0rT_H{QUI8yyZBW!5(OVL+uTO3mHhr%iEQk>*woMC{tEm4ihgRCpQ@Y0Sg%TpW7^I zdw-L|{qXqMJLT!&aor@5II~35*3l8D>ZXo?ff30sm`g*df*vh^tQA4jh$?DEDI~2J zl$;Mm3;Y=|t%RCaq?NNrJx|i;Yr}*Q!SqpUyZpIutmyTJgOiiWi_RWA$@kn+^m5;~ z*RL9{`Qi1@oZQ^fW_2=w3e9LMv#5!*a>NRYaXC5ozPj=&5&hKXb>H|I6`P3|gC+jA zYia4^_`W_aR%xpS*XVUg$;qi5yi+X)-ePB`P8y#i6&IHl-O|Fs{@&hpLms20Ktr`u zT9hQeFpH+O<+NU_T^?X>baZqe5GY@y^F`zB?25}$(Z-1x^w_fHW66>-A+VpwH&7oT7jqS-VZ=U*%W_Caq->RT1#%Oz$}sb*zj;%Qg&h@ zDjr^S4u|Ba&+D_h^4_+SR_A`bi=cqEu5Q(B9g%e-r{sFSAXHUVMSlMGwKeVTE*?_S zfz8d=9|Mt8W&^9zDRh9Z8?h>PsB&mnUe+j@vNH36@D4@5?Wg`(m%E#g5wu8QTMpM5 z)a)#*t)a2p#cc(lPY}NZq5Z2 zA@JCXLS{1>n0Sq9X0}`1T`p%@s#$cJ^&;4KJU-9MOH1t3)MI8c6%NB;_?Kv1&7<)a z!aA)sG{k(0gb{W{q>Qw@)+Y#v0tH_yKx4^OI&Dg@XMkrQ!812E@9pg^Hd_7t`!_PN z*mgxmhPI?+cuWae0-T|tp_nj)y$hCH>XEtVBDrG_HhE*!hgILVr!9TomnQVp5dL}^ zQ4tXwUJn)1yLX!$baV*&8V{!pQ%*c*dBUN2FW2RjDWQat+e z!usVlFcT~nE929$`(pqZ7F{1Xj@YQA#2R9Habe+XrNI>NZ+5W;lCWohZzY#a5Bfw- zPOh-Cy`6ij3%G>=$h^EfCX&Ti^E$O#8(+zT2GgmC$Vjl-2wunU9~$2W_A;1F=TfUz z594lmUhuB(PVv931N~|rZw^sb7kA7}C(>C#qltTaUwXq~v{2tKy*%9;Zx{2+XQZW> zvm`>yQma)?>-m-Kq#|hzolM|aZFaUbHlh;O1B+s1bVS7U&CSiPUn&R3q>WX?JZ5Gx zxCl-IR!N`6=I7^ucR4gPbP_c|%K0NTOhVQgI%TucU#THCIC9YLlN>7v3HZjxMKu)_ zugl(##zQf*^z`J7&2;{xzl|87{kWW6U59F=D+gNfEQ2q)>~@5n?#>i%>o+I7h(dfg zDJh|2Y`*z{U8e~-irC(!)0e@2Qa?XTEo7dGX)5EUSZ-*W}!b1tU-Gi~WH;c$MGlp*HpNcLrKoT3MUeI>0Fp zxty&Ary8y`TWh!2Y}A9sT!B4f9O`<3C8nah8~`eCwNqK#B|PdrKCdbIK5bSj^(MN6 zI;7kEB`&JY&a7%X%BnGuf^oGD`%;;8d9>mDvW0+m``h2;6;S~G?m%E{iAa&{UW;V( z^AFf>|92Po&+#f1s3=^@%}}&|K1s=(imeo7e#)$5rcS2;KrOkr_Ui+&T|}EbWyBkD z0Wk7EPPL2;%G%reC5c*n&r{*#`1n$lB*+_nVR3OZ1#OprTrN`qPfFF{WU*?a!*#3) z@DxNbVZ5-$!?DqsC610Niz^YkS$CbhbyxX3aGh4B(TPo~rW!(|lq!wc7amLEYj4tPT zhUK-j6P$+P;=#$ubKv!usGS$fX7D{<_OYA0=M4`JAI+6yxW}7LWurX=hlEU~vr6yL z*bp}Q`1l-jj(v_6b?(V`2n+~lwC%Dn^MZfM6$}i*pC%JgOJ48tYynO~#N(l$r)Ost zCzJz&4g10(w(y0GPSplN8CELQUa4rRvbuWlpdIa=j_0ggwSr78vs4F`L`X=eaqmQW z{fYacvtWXz4Oo25=ZeeDH-03C9;)ABbhw-yuQr_ko=G1>$fleo@e_Dt@87?A-}9VZ z?)2>(X6!BE$WwE{rmhYQh~DYb>vdk<-3^vds?$W(0yhPXLKgOQc6Jtz*JJq15GHF?t9^m%#ip!Fd>B%@>xIoICA1mfOsa3|T5Px2AgY*NN+rnh z*A03DAY|V|((JR+Xg8m}+;64m3qp04Qn5$-K#FBGm3o}5z{0{-89q9nET-))c0ZkS zJIa95>UVeUPk)9B0f07-?<>wmgoGSe9xd_rm2U5j#)ejXRbP+CYmmOaeqHZom#64A z0`1IXm!zXdvRsu zal1EAtj}3F8@;+fI22*^wKRD8^?HmxCME_-v!I{=B%HzNcrcnoZRoh3E^%ZuKSxx1 zI$wt3hcPoxp5qk&5&u&`ttIA&@8y@W6U)!SZ8paDvm>AyB`K+h*w|Qrl0XvE1}&<{ zR0OD%Jv_Jt1!y#tDvH%0)q&Ya2|+NAqKx3cn1p=PLJav?n#agngcC-n6%yFgylcr+g9PU7D%KYuZxJAHX*4u6wI-f#HUP4)Gg2z`UI^2`zLBssp)|9HHxpc{)(zZ0 zoXnygV|K4H?RXJKS%`mKPB}^;6oZmU=LH{064pKqHd2 zm;tP3VzQxicmbwX9|CiCptz-LVx|DXMwviXP)sJ94sNzj^D(5WbRw4T_fOVbh-0RY zyy$jI0BaeHpD_TDQ-g6h{Zs%})9Dh-s?R#iG_?Q)-C?tjC+ZCKdA<^#=Nsf$);*v^ z8J(E8y}V>##ZgWsUPh7!Q_w^}K!9Q};KEUOkbKZnUC>y+MuPl0Dyd7Tp?Aw8F?39V#iW-1nl0Sr7mMq7sgo#E^Eht+vFhUVTFhWKq5U~>UX zgz2#{*tKtk_0N5@g=%|6dnX5m| z#<>0&Qor>(BHx1mTxieRt7jas2V*10I;;&zzM=#D6{Z%ac9^z+S!i;M4^~wNKoYZI zKK4QhH6q{~o~|@t?tS+(=$+R!U=G?hBCstk8A$2_8Da$X`t2lLo0)kp!=;!Wiz5_E z1g;6P2YIeEnkxf9Y}y@{!%jfX&1gMH54t5p$nD)cY%^-85YNH&n^5?SvLAY?P@$$c z4U7mf|5y$ZFoj!HR4j#r0(j;_@qH*(`4kigeYp327J;o<}rgJjf6}`5Bh^96J^Dl zS%Q=Led!bcB&qmpO#BC#{_8T>vj8Mag0``+u;{O7&QS68_I7b;zQb*x2#<+bl#eaq zkp9xwNyS>~I zj*pj}KmF$aAuB6ObV^%O^JsrRDJ8{p-aha6?Ch+mEhOo&8$co zYe8LqL`pn1GcHa+L*wVpP910r9jU|j%}2S5)}`NM))hd+6_u4TZne8ZvFuB8FaTSV zCabVcF0nQfJTMco)CLd{X@HVyk$#uQf&s&~NQBQUW&mSk;*G9v>2kj@xj)~ii;_~}+RysW_81{t#s2cElzzPaY zcoGDS^6n5I-YR80JE0{qX+I7Q)O2*Byn)3FkgDin9s$(|A0OXp_s0*1^L0*KdDvmq z$JU`}?3BdBLaM}(Tax_bi9hd38q-&|$46Jq!`z$_(g=f2=iz5TED|;jhh1)QvDhBs zr|HohIDr7yEN)l20Pxeuom2@9*xtMPrlzLE#b`72ie|AmoY&1*qr<~ywOW7f z-0@RZN2Qa92LQ|9;o#Q0y}9jln=H^uQK0pr<#52HIkY(#`=N=bDSn87w*b%;O@q(( zRfn8M!Ch;6qs`G2?!y3u{E>N83TOKCYH4B;!h3u^FQ_+805I&2rSN;+!5MUV+)8QQ z=1LT|V>23oUkVTqb{nDj`9@RLgbaH91cY6HfoHJWs#VqN{kfx)>F;@f3fa2^rXCa! zrVhM}NCPyI$nuYDM@L8RAmubTjseV`rz2DMd063kgRG z76YKRe*O-qF&$1vL|+$#+S=NHMK&yfnwnZP{LidkReM*3nc@!~hlPw>XS>A*`UuRR z{oLGK6mq#RB%{G7te;{?(+jDWj$_?=1$wSN^ zROApMKQX@LGSjp>9s<0{(y!$&b3B$uWdH7M%S2Z@}WPu&_YYu3}fN#>!%X2(|+l zai6+Za#9i&h=!GQL57B!ifT_w=WDVrDpek!#)1-{vyyDd0P&ea(QA0SA6-hdT`@c zmwm&eb&EXfjs9i`dFtbN88gM@7F+#z2>JX9W+Z7qNDB`KQ~5nzzMiXyn!$?HN1$jb z{qf^Rz*v;^0>qS|dq}!8HUVoC68i3W(M68D@Ouqlp@8_+(bg88=>%v{ah%yQf|e{k zYSz!6v(XQ|uYQPXX?;dtm9v_L%u@}Cr^>JOx;c+!pvHW=*U`AGmUe{-(|Uo=GX1*z)U&>pd6`V&}sz* zfB1-y$?J&@dpMP&-{aGj8l1{%nVj0nfUniMNT=JLdq$z1876o*ITA2a2OX8wXe$OauSq?syh_Ov;yWrILK1EzFE zMzI|I4oDGy1YcNyVgPHz(05=R1w#ul(HJZ*EiE+>>;QZoqw!!EBN+@Js?hwV&4sLRPqVmpV57-)OOMmM*7xl-ge`Xw-nwCM1|TG( zH2_FwZX$iw_VMY)x%l=HgUgMMfHZRe*xtSg=B;$>$FFsf!(dgJG6JYn1RvM@l#!~x zPEz62Mvr4QmZbt#{YT>me97NFLla!98$BHG3dgkF-P!-!m;c?N{I(1OT!wZ~GVr5l+L@kaoY`y+rkeUVmvW?iV!=T}h>5b#S$Ndbaz!|}wACmvFy3V^1#v%C92;&nZq z@_jR*aEP0yC!l}ka5@$PP3Uhou_T%xRMph<+8jUtiv$ifaJ$^@was4s_U+sD_BJ5q z)^U{{louBRY-Zemp;OXmMy8)zk@C1HDKqo*;v%+o2ObIbqqdrwScWpKx}xID*4wMA zr97x_=&QzP!06amU{4cX9S29j`_Ug-oo@CbB0)(9Fi4ow--YC}c>rsQt=|Jafd%2W zX11lJ zD{`oES5OOx*XOwvm{_Ewr0~&YC{e`($Ro&`Y{FICNmY;6S66=|HiE{B?k5{BFCKX6 zUnAr>;~DITuxV*&Os>ql*02!YXr8vQt2>+zWf=7Vv;lPiQ1Qg%JKUf(`;#P za%roT(bVkh>EU0zQ$6qv;NLKw#$QYpn$kL@u9B`Ly{iRnvu_+3`^Bvl^)gl!;JA zeEiPOpQ&kS;i`c7BSJwv01(w&plBPM2KKDuq8l3zPp(f0Fq_{$BIyBl;1{wE*f80j zSSkx99OlUPgyCpqC8gR6_AReI@pdI3@P__|{}Y&nmKGLfYk>LJXw(@2&pa?NFgu5U z!!dn^Xs9U=2@hz8T+YXh06D+P04Lq?*orT28JwSo^1v8F-SCvNC{iEEjU- z(b3RmCKHQ-vWM#tzemX}6Uiy_br+T`K zW>iE(M5Lpu8;i#)wX0l%ylnhe$nQ{{1q=*~kB{%_>I%Rxz(fOru9e86rGV%31pv#h zU%vpy2UrdNJ_uI{2#3kZ$@%;H+ufd@pL@}M0{v=|0Nxc}^p6aZGPcq;u=E8A&;OWB z@lPTY$G>fz|92L{pFM*9-%F+-_z#&A;tq!H|1rrz24rLUCrN{qqXBpbp0%LaA?GDK!!lbk^^p z75>n8JfAnjCv-IA{^Dy(@Oy-F=#a+;=g_s`@oEO2V$B4}v78@RS;Nx*e6( zN0`%}V~SPc9B<46@lgpn)_{J0P3=xTk}_)7GNA~^vPjco%syx%IY>)xOBq%iT+=#m zAPH+L<%e|MW1wfAY%EGc-FIU>iEtq3=d?D_=Xhd5Xw;MQP{5K@xN{_~sehTY&6f0Y zw|HE0TxWr`u+;V3{mId-!#cUK^!Z-RN5^ie86)fMOQLd^KOkt!q%s(Oe|ARqVZtb6 zw*3L?jJzRXY6Ee^0^i*KW?AE_Lt4F0Kbchbxdzn$h3nNrCk$h>#P_P8rTm@aNvROr zU*`iX=_zDTejS`aA@S3xNwn_cm|X-cPo`sb_v{p3uHgvm|( zbVMgagYW-q#4<7^*}`-Q*7!f8RLp)LqW^Q>|GM12ZopAb=Fe4*@V@=??Z2J*=bQh0 z8VcDsC(~b7@qG8s{Yd_Oxj!duF2cK3kzhU`hMo)#->WNrvBdfMdm}yVnxdBb zK~hd8QNOtPqs~!nX2ogA`>&mfq!*e}3RJg=4Rk12y{A_EjqXc1k4xu3s~MTf)JApNH!W^Sy@Uxp5N;HsVpo&x>vaOt@mT|rwZ_Y< z5TR_%@!6K!HHwdI4)?lfLL-|wD>nVq>!T2l29sCcOkxe6jEOg+neo{(k)o%6$6dez zS;(V%H&^}4j2SUDbF%ADOLcT}x^2OaFfav|1?5ZCM78ZKmRJ@LZULg^=WO~2ZT{!9 zg)`p`@uK;mf$)@~jj`gLV-X%iZj7%?F-|U=Us`SPxJ7YZyja!@Cv#Hdb;t{N+}1x| zwMM4ZUpX~oe`jFG0COqCg(=UkeV@Uaa6|`bfb`LI(PQ8S{XN`n!=joDJiXhs< z*c2yAe^pT%t^M|){{4E~ZU`234UU#8_E$|I zhl#~M4$Yh~v>rz;P_4#Z*k1nRsNDPjO@LHG)oUOHv{OaNzHftsow|s@) z!!+N4cAF>dMy@UWzOW3c8y1)lQb+x1zDxm*>ul zv4+*6wKPQ^H;nGmsr0hzm*|1UE@`>1MS@HQrqGjhl3!Jr21r?C?X0zoGlhd9AX0mf zN}t3?yJPL|C1P6``EG~h3ld?7h1bKb)b*MA7}7vIZ*J7~AF^lA25GV;dXqTg>D z3#lGa;_sg|wRC031D%I@?ZyS$)@}_}(n9HYuU(=aLxlt+8B8^2%eRo*33pGGI=Il@ zFyL4wB$r0$ZA>>Jdkc;0U0X??y`hD5ve5OE2L@@>psdeaUUN2gx8HOJeRm{Lv=Q#w zaAnVS+LvS&u51^JPBtTg>7`)vVn^#B&+{z=MF+Kj=Wne0i(p@+yLTGJZIBsaY<&s6 z>QT|(d-cbTnrXgUSKSS9r8LcDnlz~&J*Z^HI-xqOw@g>K&A+&}&a#)Wa88J0zO(L{ zs{gt})sA)-Uf~ekx55y3w@8PuiDbF-EE6)D9VI5xOtE2&ZtLnd86ZB`lg1z1!mM_E>T@cZjzPp65Cn zH*lD`I)!o|e;;EqRjU3)ZrR0D3wdiU0`khd&?{|pgEQ$O^%;t#DIhsz(k!OWCw5R* zoXpaAxd$aGLj&K0RUX*5JOxRdiExC>ksa~_hT;p|Z~@%MIoTL~gvVdt*F_{<_gQ5| zblGbvS(kb~oQc__lI%57Oi4mdsnk`?)#k+I%{3Iou}~e$tbRES?;9LZax1scdzpPE zj|3Di*A~X%zc@3|?lxIbSFmJk)s>5M=jHw+q|E&s56R9$Ac`cVlUgWGE*gF}qG0pN zRhhKRb(=uBecS>X=r0@%n?p-rD>=lhwqjhcU+?qpr^}rWp6jgJ`#`4j(%4BHGkE%i z&K$p{yq5ll#**r|GT|(*{=^OU5xr($n%kfVt|+g`=b2cL(Y-+-DBqNkW_~mDcZ4@j zQlS@-WT~HOg`X?Sc{vS4>~b1N%|w7VBzrjF!{ANfZTB5_Ea9EwL_Mn4vq#`9?u>oIG)G_b2{q!hmF_c5!^;|TL;qt}sAv$6?SMi^xT`c0IW^u| z;fsx6_>o|kj3k8M`#?DBBLm_&Y9kEI0rp;3UpS6rQ65it+LC9t%P*|_p}7H~c%zsm zxXBkwk5M<$ZPZ*-^~*P1jDb9_nEe_M7>pv#N8k)x@0tsjma~e~Z)uO5a}?Ot2vj=g zGlV+22!^SE4bSR53E|c*^vq_8X(;IgI9#nqoHwdyqi;FL;zux;)YDR}><9SM6Ekw~s_Sie32{-0E*2&Ptj5 zbotPQzYOB9hgV5~S37187U8^j(Uj?^T7PS7gLW`tG8ewb_gbY()S;Acrn^WCsYiH|^VPcAC9GQ09Opj|RCkSbDQn5=t);TBsViNSm$o6OvQZI8rEOh{1g%ARx?QUwt>Oi$dVoiDD zUw4px(cE;5D(JX&)3h!)L)C8_6acM;j9qS9A|6>_{vR+8E64vQyU+etq5J=-07U4&`mxdfuN-wA z9vFH_MuI?=>Q9On-jm|5hWy#`I@L+CS8inS<@W zuqGSRzhlk+y+(xjznZl%GyGeP$lpW#PoQWLcZ@D0RLE0&{YSO!9r$E1We`N!wSo(S zRLaZYq05}&p4zMT101F?{;U}}8Ju7?CjHz3R&f706zNgufW(9thxt@G_7^SUua$Vk zL`$J%JLiV&r~wcoGVx486m8-Zpnj-=@8I9do@1b>ID#cCT98Gf*PW8}Dv(obVoeP{ zHQUolq+3P>REpxO1(t^{NUD(4gR97yNm?nBHKVG?*b)5Tnb{W_2pTFTN$u(EnmJMb z%?hDBe}>)V^LCZ+`uy^c8^MFS{N4EOX&;FH_skf25y@US+Ql1x^9*mmTRMt4+?FW* zqKRqCsf%IV*oQJ%HH{#h8S9e7_tBM4qRTRTjdy=$OxdA2VW+b)${BYZQ&(+}Ngj2L z$@PY(GP1Nk;mgX52V$V!Zj^HS9gfcV zKwkb(vKygmqy}4IE@=G#zqbF%s?gNNUaaGu%XJP#ZATkNePK_T@C{5*mnUE^t@#^1kK#ZQw$n9_xJp378pvPe*IEHud0)yiC(vue{Fi13p%X!esS&!V)@PqT3e#0C;2EF z!Q*57$SP9E*HrV?Z=eZ@{b?pOWOYe@^q22zo$A@iQgV>m*&6JTPaz*TWfMt3B8yF- z;2H~J6aNqXPMWnAp0Wc;dUG`_W5*ei&e?M-kNw5DgkO6_L=_%D^;{IL+UbT*n@I-_ z^cXB8wL~c9S|ky2VlPZjXWRm>Ij^DAR}UwVxzu7T{R}HXW%5$C#8p_!FDID_U!Cen zR$?e(M2Kn3tR6ijl9N7INi8{Log?b{Tu@y}gOgrw&9U}ij4^kT=g~Rqc2DhAKV*?q zk7yn5Ad4^00IktE!#LCU8MmiW6ob9OQW- znwfv{nR7}N=`E#U@{CRbQP(zG%C1Gjg%z3I#C19}Xg}h~Ef|s0A|*2p7 zqrc^_h{*{S2dbAnk(D(|BYnR&u%!?6HcAN{Ll&4sN^Bv1^`fz*uOP7c%IoVhVuyEx zV)9MYuf2<_a`Cy!`^CH=TMVaNREE`LNWT$YNEFH;Udn)pBO@oHR0Fhr`jU5 zY`f5|Yl&OaZRB$pOA-`2*DGIsKWosWS?f8T*M(6CY>-7o?d3FcsKG7voE$8dzZi!S zvgbnwhtGgJ#_FIG5Q}SWt5IWQM&w@vm+ES4PSNFerftSZge#Eb7bcjlmG~rKA(LoY zpMhnw#Dso-bK`i=Unp>b%!7d1nt)3(yw{8v2>Szn?}xEjDh7c`t)-qBe-y^B*tSgU zZ1goAInLud^V{7A0>8Hp&g|~JN3r`5dJlB;cB^1Mm}TLg{C|%kvkN`Y$V!UDi_rw7 zm!5i}sNO7|e_9Bq{AI1&7)d#cias{trf1VdY^`b7)pBA{;=agvR_6FV=}UXAk*(ce zts)A9`KQTVK?kNQzX|Cydg+g^EUtk=XbN%uC`! z!W3=8YjT%&p&4>Yn7UtL6vC%D#dl_hraIK8RdFbWog*axtohs26|E z)udrY33nYfWnYk1#J{sKBLz){xmn(_y{A90Qh#>M_CujpQQ+B*ZFkgQl<-UuOt&?T zyBj8W*;YV3TA3b63v(bF{*d^Mx}q>^&+xQx8V9yvW=sfN*i{HT&XIyKn(I2%oYz)g zmDJIsuE~mhx#FEH1YuG{o12c;gNi=|?y-uoa7EIf2L3BH#1Gecj0hdmCTCAv8}x`T z!3n6l4GJ5{;W!SC0>gNqd^{amEc~;`LYJ)+?2M;jI!ZhWK_$4B$Ud`0qqvcL1~E+- zJSRmBDRS@O7m#6k-s`gY!Ee)|h7C{@nSsw+)smeWM4ZZI;U+9xF4=oh z-|vK$v+vk^uJ6hRV=aW@F$SY#d1@ykkVj5@>0Q8_o*oxyp4wF9o;DjMa(j-*iXIdqc{3RWtkx=KO)^m{^s7-9OvR%>%rZu=5Kp zZ0;PrtMZOy@OQq6khd|)H@?W2J+Zke74H-l$2${&%P{aXd+|-0a{+Bd_jicd$Bn@# zr7U3`+Fx9xpM@M!blS=WxLM0mrQo681 z^a}L7_{~V0lA?Rslewn@1OHX*y9jS^u$L6uPy>$V4NXaxeezsq*DzG5$f+gw(#)56 zo$hQ_Y%|~Fz5BGAvSPXBShSo;a zf^~j-LBfg*@=E-H-*_r#S#e>@%|c#;RuP)Dk;fniQq*2Q!3FzZO?G8=%}jO*E@JeB zRxNwXmozg+y0o+EbiX@RFhZJho`&}|`;6y)hGrBt*wm5#Ol;HU-hIt(F3~HlqSIQ| z2>m6d6ITnRFmCa)dc&PCb7GY4y(nplZpyOot`zD)25b+nKMoKjL#q%gQ2 z4LuNA6Y3!?2~>D6gmHvpTF~7Le(S7gA?ATrcoRb#TJU15s#aqXDY!8BeUV%Wvv0eT zs4pyH7urnN$)rb@+P|D1ABc?{~?bS$t)^qr)_%@RB3#RpP zH*fj}?QWUa4xgn?cpY{%Ikr4UEw|b+?#qt4?;ze8@)N!_jaAUN3Ud#|-_ z40&9*gtRZD?o=tC*h*HGI$L7L6f&L~kGb%ao3G@lx@>-Avx;QGHYqw*+bKLN1(+Zy zX{AN7=$Y%=vJ6c+D;Uq7EBxI1@^TuY>2An5)|?z`(&kv5KpeNc_Ssl|e}g5V4v!ha zrZR}Zbd#>SlmoZXF}%MF;!FspJLF@P57n*ov$P5 zLU=*%an{vG?l(%(To878O`#Au#f$s>w?1HLQ#QIb=S2st>vogw!x^8h7eHZl&V#U0ge!R!4FV9K)W&Q-q}$m7(E*uv?HRLZ++ru+p-b;lX#RUtT`v zt>JRGLbZJo%yKs)qck0(9S`4*eiUD>KQE1P_ND6gAM!pH)5eKG?0{Z6tD9#rLL`2* z{vJJsrddmX`wv+Xd`nfyXuI+INf7RFUHYeEyyyT?iW`W)Dt>~rm(9P|NmvH z^}o3nK;QqZSqvNdKcOKOj(>GlXJh|A?&!eu&yEhvod3?&{~GFlK;oLzxMR=+;l|@z z1LddJUe`Vm{Qwv7>W5eRvYvAthx9pIcJm>+LU-ZZ!cs!$E8ULb7sjb#be`K!mh#{F zS+NEz7oabd8^oejvJ(>y5@49zmBJl31>+lJr}r2sJ5XJf22uHdT>LQrVul2yVm21{{TDEetYz+pm?OQk_(5ke86j=JuWAuy@VhG>ZGaRidccaf}a z!_=CvzJ!URl01^PcJ0dtoL4g#yMAJ-Z^*G!iHGxLINQTGoA1O&F7w%SI6V>USeBj; z@qg~4c;^t?We%GHDTlY%FVDBXrH<@JYhE7jcQ=M&FJAGrZgOoove^A2@nk8qzF+XV zbyeYQh-)NsM+Z2e-jnhWc6}~h>E)Gw-(7X?Qq5Goia_UH{VkQ7T0aP!j}mI8N+hn9Uo0O9@e_2~_NMPNm9O#I_AMz%poSD$wIxYB4jiI4$ z-v=qp@*KduF_!Ikh%BkAc4)OM-MR^hx9hy(k%fz3w-#f8%C|8yrZ~Zl(5?6DLhKdN zB!tl7KiDce!#~}Jm>Gb58UESRfsOs&RiS@!0{%Yue=w0Yw522WhLM4OEqm^HEXeV; z6rV|w8f#nXu)P@gs=6R|8BcW%F1U&CmC99AH+Z?T&BvKkq({Z~F!Vc_92YKN&@O$+ zAE6d43N?VDZX+bMvhy1e@+r+iuqXEm`ea_t2;B+Z5luv~}y0M47Eci(Q7h$Fbab zCclKZIY~m1Dc-ZgNZ<(gO=1tcFnUnVO%P*C_&#rJ!lv{V&APJqRVIF+{ZXc`dzr8D zR1ZTpK_4^Q^vB!hd+?A-j!p8Q? z<_@i2(6&=TWq8M57B{MIEzvq;8|bcL)kk%(ymHU znA%iv6f}4num^e;?&Q<9nt$i0O~<;8a8Oh7-xdku1;-X{X?TjzBf_2Yxlt6>l-IPP znX0FIw0=ii*x9`ZcD<@bPKkNxs*^mAmyq>XZ{vL_nD zZ}qPlH)iTS)5+IR{NgZu*9kw|1LdzfasY*ppzTgc1%397o8QvKtwfL2X1KomMoOB6 zdJn4`ovGt{Q>c`!f@+G|dXmHz+R?r75f8#K5XEI-wjzEmn;q{!ZZh%w^})>-tS`SM zq*IaRz7X>+PgKOH#}O*U@-+I!nmu#=JvOfB3&lfiSKcHpveUs%muA0eEjt+dfuQ+F zv-r7WUcVG`eS>WQxaXTqlSx)zi-1{3sA2sez&Nxaf5M61({0BU`EqmU#ecxr&UJ+t zN{xEJ7Pt~3bMWh}5u1H+!yMk<2LF@_M-qGFedi5L^jciRi5I=Yh|Re1q2$V_ua~?VjO2@cXq9suhg)EjQ@7ADR*Qcj*MB_JL2a~C5MA+KU$)xC3 ztI&95_lT~#$%1*Yvc%D(%C6rMjK$*lDr!l*WBf0b61=`9@CnNLlet4! zgGd$=!G@ZxLvVrE28`gldjycQyDqst$$5N^29ICK{Wl{TA%BOBqg<_Ynb5A1oZK!C zPIS6N!Y;X}#hRVINBO8(<1UNQck$s3yLSML7#z>E%=@9k>Zb@pACwlo08jHGMnA*0 z(GM*QwRn>AM?LL{F;vZA_T@x28XC^}wqu!w<_35rcStUk>?YS~Z<*rwjza?)8urle zxCMj8GFj_i&{HmrzqG}qIw>NhnHbL0zi0S@!nSK>h_EZ%iDwn)Naern z5fL9pZd`So8#udpSI>zze~hWuM3^1=!zonely=R}2mevLT*&K+xMp+*^Oi5y(WNEv ziF3UjQ42@>t0GRXlI!M9>r%Q%NtITcj#^0_7pi;9?+bU>90zcPtWiF4oieqhPgaOl z$(3k_9+6!Ajfrn{Wh70ZK|?`j~;7#fF*-46q0)fz_MN{62|Pb4mi zKK)pp67bkl?{RxVr||#%bIo2>}wIk>CjuJh(LO61;J3q#?Mw zOLLp-oqzx5o^x-Vx^?TldiB<-f~J{kt+B?GZ;UbL_-52)8vaOD6kg}(`eF4)_lvbo z`%x4I5Z7g4YHa@|7tg+h?Ug3?+&erm5XrME0&bzbY0IarRaY!G>X_^VfTkG1kBgML zl&WwGn_}5^#?7V%wh;_lb0P}a+j*OrCe;~?ARwdEqab)$6^K8hi>XB*6<%88lGE|z zEB3|5tyewm8fOC^7ht)7v{_=Nqt-NHZWJ2R_$#RJc z7r0s%q0g*Mt;7z~c@418-*j4-iZG)8!cf1Dw+SOBGQNp-m=mIjd4R+FQOpd6Du7=< z+n5}b4W!!<+sjD4cDX-@;OmT96uZaS<3R&4w1`Qb+v0X4K!5eH{)3~(run3Lap$Z< zORxVt>Fac+G!rIj#~s|m))aT4hl?wY6$j9^g5LHOm-SHj%`84@+gH-OA-dwZy5c`N zsL(FA=BjemzhJQRJYS0bWfY+{p*BZu7etTdtuK-as8^=0)s0hGcR-h*RuSqd zL{p*(!zF znAzpAgLs6%CIdJ0(W=pMPR#df$XqDX>T-wgU!y=CQ1POi>Wj9eeBgEYFsGo@gVo`{ z0zG<;Ltz;plg!P>YxAT0g-JA+?2*M^w!I*F)*@Ys zr%~|HWvACVTKT2L1Wj1$N6Y@4XCFrmWmB-;;W#+@2GJ_6YiZ^jdMte31}cPrzvRHW zKQrZ?iH@dAMFRWU9DY>$ylvbp+T4NvfVX1==l{gF0}5;eR{XeHuNDebtyRyhsZJx!gc zZ?0!-w(`M$nZ!(QJv3sxuePVDO^Pq2&v-< zdT0{sN7XffYqC&hwh#d{%zsp{{vxk{U9`qU+kKhIKz-pjkfcGDmE>^uIZ7{B_Tv`1 z7MN8g_{}%2Ce!H6QkjMI=^aAv#?n!L3vjU#xO{A{_oH{0o(&H3o(LB!W(jeP!Um$> zU@P!TNTrwe!eXnJ^7Rl#P2+W+@}`L373(+4c+%~@9hisN?OQ*)k7vQ?v_DJPF-s~@ z1wz3O+J1%`jaWz9Co8b%=acr1OHu_iov!y*is=MAVcm*r-vP}zplgV#?oxWb&Pg8^ zRZ)Gh`>|h)MPce~ge}Drd9&HcV3kM_n`u*E)m9(Wo2M{ zY6G@>#Bn)nXL7;mbDb&!PflOYQZ%ouuTm+W1Vl#N;x1-ho*TW-*EIPrya#Tc|MR*0 z|LoVZO8v{}|NjB6<6kb;e{}kJ*#2w+z`^qm!5$CWKLmTvfA0M%~r^DXY@+ zJ2t()IO<>X{+rYP*RB2=zFK9hfPOjnB7hVBs@x@ z?oV-9QMMGMh!my9d<=9do#py&KlzZ`P_;#qkyL_5qkS@XHM2Y&Z)l9*X=h_w3$+$6 zuZY+^ihA{z4|lYdND*vyGP>=wGJ5U6?o)#M7Ug2;HYs%V6K<)Vll&eiF~x4mBe7d^ zFP(%zsi(d3qxy(MX-UNrZ?XF<0KQi=&aR(K)7N>i{`U+Ix9A_X}t! zAT@ZJtZSFRcDXfDh~o$|h(01H*80QueKV?B^sd=R@}6v&?JoXup0A|_PAQhmd6W7z zDjMU(o{L!l)$H>^`GhbKf$#B zJ-Z@1CkK)$@o#!XzTf*7{NG?#g>3KDrNq&PhOl)Sw0Z zV_{_#AL-)F7yNI?+tCt#oD>oa)BRvvPEM!2`9hD2axb*X;_6C#*uZ_n_Z{{<5z+m< zc_G8y$KQ%?r|pe)DYUJ++!JCQwVh{nFPmo0o6%7II`%)FBT46)ZxL5F^S4)nx2tz4 zQ2%z+cA)_SL;l0@pS}oLtAcjgJW$)dED^%OB|QB_t)$~I_NkzwGO>5fp)_HTrVFYw zb?d<~KxA6)6Q#OOe-MyrFfiFI@_I3?sIh3Gy>g(7adK;M54UqIuqdrMg|}cBO{?5r zFpT2-$Ek6_fp3AAs=+RJVqCzK+kCiqKB7hob8no}mm7TFhXnY*Shm$pKer0nwIvgH z&?EDP@=bjU#YdP92jpNb$_e(|CoO_;{?A`t#9El;#I{ z_feHWukqGpY;UAQHZK#~7-^ z_iPQ%94xT8`&hd%eykXJ8Zt4Tjcn!D8?XWH&d4si@IXf@h*M>15*xsSCn`qH$c>r> zH=g@|d#-rKVc?ZY>aDK^*&&QPz^PO`CjHE9ESdJAhPL7LaWTDRBABc2D8%TANCO@u z8hi^VQQBQJ;ZFSub2LYTI6VNGIk97U7ajW>E!BB2`ji?#%<3u(cP-XB7M&;i?T$Bd ztmRHqXuGVLo&+R1tl0C6_Kb%GO!a|yt`BsatCA)MPjQdpS1-^vp2jTfuQ|-R&N`U? zN`>^Ck&QNX6ipe+Vrh9-W_-v&w7as{IBRw3p+d@LFt>)xB}Y2Hy`CQ*SD>uEkh21>9n95>JxhU!O;t#}`+u3eyMoeUdfHWn_44=hnb-ySK^!k2o%w1Mw zVs0EZTCAJ4ne(YSReBW%4)$%ZJ!D`0K50Y(%IA2yAY|J}B)4~y^nC2BX7`x3NjlcM z6Yep{OKG;!dsuPFSh)6M8A2UU+uZr8--toZuL2RuZzOm5j>|v88GS-MIamJpRJNJk z{bOEr%?|m}%@&(Z;}$$~CJ-j5mTXXQDcuPvGW~oZF?l<5#iHxrYH(6~J5zV^Vj1Fe z?ZBZf6q(a>y?>-}#2EtX;oQh$*aff`KF$b$lPhFQ=BjmIY1vZk~pz zDjZ67d`v)=LqBLQHtk*?`yQrkGUcAKlQDQX7iC!(5aWC$TuNCPt?=Cby4(2^Slmsp zi(e`RA`E|(2;MRLdgT9l8gK|3>MH8AtLbcxfz{~TYH`F)m4=luBd=p&wvv0J)ZCl`3W%H$(yU2Pspa}(Rrnaw}lD#z>2>~<7=1Sw}< zLvVf4NKaVT30cwMkO`kw|2<2JMJWVs8s^Kig{xhXHM1Z3>Z$&I_4^3#VI{mzM`Gh> zS?L+$(z_++@UlYeslC=uvzwbqgNeG0244@pkeh5+JNQ^$W=z0;m_q(i>xkf&&Pb}; zeNT7jJbT*0C8giIW-LHQTgbZ{)Wizp&JhwR)6d!qd0)!D^KwglPDac58qr1d>msc6 z^k@gLI#ZsJWdRhAQI<)KgG&9HgfyPy5pZ5KYp1p%$j^k6K`mboX#6Pqr2&Hk(BK!A z_4Ap~VB=?r%*Q`rQKySR2lzUau}<02dNt;^ERIp%&NN(`q*v#{;DHmD?!=9~F9e!u zUUSHsQ2gX8yxdRjpvw1tTZ_8ol$T7t5q42xUZM8A8`ou14H+a|(o_9xHR35~ zeh?TH-P9vf)wpPE zI$y8(lw-uW<*1$8PNS>62v}A(!|TSmZcV%Jo`e zMN3b4Pp~MlIfEQ`{6W!ixs7$x{M;5ia3(O@t3qbJU})>Z9_!n*$cDZAUd+ORqE#P2 z8dzpfiFQays4;*mV)wjcueP%K6l*QGw=7d}m!1p2m9L7w^9h{Ds=qF@vZSwnbCR9x>`=BVr03(Z$1Gqd;iPOdEk3q+ zyNFdp!w`NYS3R>ZHzaY5HTE>p zCa?q1cG49`Fy1tjGS{w4x4rb8)z;Q3E&Ex8MIJ2sEMixf}kI{Sl-BX`^~R~N>v3Dui+@$Buqy}BvFKEt4w)gvIEwR z95yZ({IJD#Jq%LI5Q;BoH=c?4R9-%0o_7@LqsmTr4G22OX_Tqp+LQ?JUbOr4$bx=A zYT9S9<`wvM0xEUGzho&S)m8 z4|jtMt`A+0fBA`V!@T$w{J~7%rl1X(WqZM-?A^G%e^s1;)ZFH9jNg$^r`-!D)i|A+ zxvFKMN>!S&uVs#b-$KPD>3pMzpZXK>#RhJ2+b`z~S?G)s9 z1lKgP{i~NrHjGwQA3++O{W3t;>%6nB9kwklnyfRfoXVNRONyjxs6ew%>kx)Y?bN;W zdesr8pH43$0xle$^^xp2-vEHz8rY-xclw*~58|YJDYR&T_e(cSA?2iGmfZ|eQE!~1 z{m^wY1-<-9b{}5~YtTvSHqi-7@Vl4Ug#~e2se)6fKN=-pnN?9tdt}BQ$H?CBoYGpd z&Vv_y2MOoVy&^V!wO2~U)qNwaE`Gv2l=&#dSEdh-H)lGpAs3_O86<5Y%bxD?yZ}k@bmbz&6|2w)NI|_-$%O7(62dUvJtHk4LmAPR7LtL{%uYQ zt^7yH-gAMc0#GT*NXnFQpZI%GX({J;?IrubdnhOq%9rSc1_w8FvB7=ktq~{-i(5~i zkN!4@QBZb?Fq1I%QlI!RqOY(8uRu(T=nf*a7_kd>~*7CuCHbkc(VO;K; zqvW6e=o-60J6G;+HuzoNTwfm{_>n{^h(SzAqaMD80goU~8&0Ty7hTBlnLn@6mQ10P6uuv>;kwCh@Qd24oh~LxvpE$02uL9O?{^PazaQ zK+S0OlE9Jkl$O=tCy;~7t);H_XXn>P2U=MI%a>5zkDpgkdv~t0Y0+Y`q8`W<98M;J z6~9lJvB+KeF?zI9hRwF>*d@Rjawd&>j0&1*Exq(nXSisjc{Tq?xn;o(6l^ulq*Sc$ zhZTN%*~i?-eghOZ#Fl=wm^|pqYA~fI5-akxV$S^@-;b7rm`(kXfe_dhS9 z$pNSrC;kZPv#%HUDNbCSS7fmHX;3N(MjWbBt}e_5kgf(t+N5D-3v&qrs*SnjZs?=$ z+aAGiwgSA;z`w9@0Dm`>QU{`9y>q+v+6K(EO*i(Wlp; zod=Gecm_IurQJ4AE(T3>z*M$kG|Wk*#Ude2UO;gHc;;DV=ToZ#`fFN$gJiH^KoLSK z8I$3=jAp&kuo4yb-_J7chjzBk2ME6Z0!NF63zab&2|FQ1JvIl{AknjF(VpVVJcdlB z+*tmkX(t|;RwuHc?R8&|Oip9&!ak7^-IuVin{_)U_Yj$u&V=1w_N|hOdis-u<-*8< zI(PCPm|}K+Kc^)>2^<#d%7ErC#`mG%!;InIS~J$HVS<$H1dBgNosneRQNViiZtET` z=HFLMphm+M{>^K>`&e7@pB&Zy^hL-TEwqz|H}yd(CRlvN@RizbZ)p{8{!ceWY2!D~ zh$u>UwiD_;@hZO#NJA^cKt8O67=off-aC^(lpsR;m?G!=ro!Pxxj%zrra1>m{M5b;}!lQd};& zj7{jwnG^Tq8vDcI^?J{yPk)L}vrhAym6$Onf?J+?=->y}@|Fxj*a4BG#^)nDb@~Xl zyH?P`!2y_`*mS5TDd8j;{}6cUG<3=RnHo>gXvpF4IK)R&DZHoRU1G#n_~4?N$Jpg| z-BBECs(bBjkQ4n_wsR(bZIa>n5`y&W9Lt5jYydNy6cb)`lm!hIsPBY!j$<~URTx!V z80vNrasW0SB3SAmwP4xtQ_6BlBH)9&^G^TBCaNI`$~dyV6yt2nQK#LYlT&`9oV4)+ zVieCcW#!ZLkw0^hXY|Nd?y09ZP`r4sx^ij5NIIZnMsLt=xrzG0%gGQwAio!*&+M<= zI*C`Nmw^&>!!V0_hOr}yx*%bv(CM>Q2l}5c>MIUFu>y8oeL_D5#XoC2Th6SOhp0TP zwLU3ZtyiNgYwYqxvU)o4>1{vcZ{?Z~+nQ1Xw(Al-WLF6LcGix0_3}BI1>)6``l%}C z8i**d8RG2&_uTZv?;T$u1SMz$Kmw`{y{9A`2)2T&5c=MJcQ4nXmOXk5Cwzw2O|Js~mzeT~v}u4>=UuF2>tY{*Azx5OPPwdDWglCidHExj`{rndsH zbMXzV=Y&ww-Lwkgf*(3-)&|YIoSfe^vcIm5(sP{~nn;r`!M=N?nkNvdou)K56a#{T z+=Oi^_QpC7{9)Ld;Vk+lM=8O;IaT0$YKno!R^nQ<{fXd0;RhvDYyg6rafz$?6YE%2 z9Cbb$wUl$2^$iPjMMcnw*C&{5w;gmVYkX=RFRy)P;_VJ|EaO|2XzzIGH;%D`Yd+TG zwq14@*@H6uJ$o6ss>KtmhI|rTYp^%-ypu`i*W+Xh@!{2nm-zMxYE{T7BC6`CI@MYh zW<$z5aUGF?i#;Qf1Fuh638p&v8>|CgOUAdlApk%3uJm{6O*i%3T?t5rk+d@n8dawDT9Fd+XrI5F z^gQb38!M)Vs0ubuym4T@LP)gJv6H{HKZjr}EA3jW>zW7yTc~ zIg$Ev<*Tgw`o0kc^bCr$_sC~vG~3Mh_#hn)L;r)_ou?bNzT9I^FK_OIe0XYD?*?j4M&@AI_1IwEU;WPqZ?l>6(M{DU-b+h7m`t;iQUBES(d=_&GW&(liA zZo9%^%k~4Q5ZIj-cfY~94MNp-#B0!L65-tBN5@N(>~v2==}53IPRCDViOl`9{_#f@nDGD!WIM zz(Pp(QaCB+;=;)wIrQB!3G+l3ZkI8fC7Ysd*3jHc!47my)@V}uc&SP;8Pl(cXrips zBh!^yljl@By+-dueufty>$Ag{zkR{j5vD>|7Q+~B`8li$7Uf5ru@hR%F>wx}t!}AH zFA@QFQQf_aL${rdwp5nGkj?h7n71A}?r5Y5!1p~jE!K#2&;Tyf$7r8DTUZw3qJdxR z;qXClI8$w-rtoB5h4c97tMK=!O;(r+1l5iKLDRPR`R^GObNp^CDB>N#!G4)_hbdUj z-d7RWwW%=zs8s;$Z!bY5j3-2WT&$8Zbn37&h9^Yna3R&N(JyPCoWSBQZPh5V8=Zie z<8$)A`Gcf`irC^DF#tbC@MDID0<5}PNDb7MWt*|XOt_qNFBBAXwb?30Fz-zoaxKUOKq zyDL=aBTt)f6BRvPpx|t+-7muV5&~3qE5kJD513bnuL@0v&J5UYg^{I2xTNekd36$E0Ll%NpRVDY1GVzQs#}n)f%H+K9p?^h4|<6^fQ`EYSPy~AIhI>^ z%3u~vwL!JaFosl701}!~49tK_n>+%7rGrx385Ld{Iv^qrfOYNF2YbDMa`Rmg^RR>I zB(RQ;3AZuwywgb~J76=+`*2`_GswzYkxphrnREDE@as`_U^A2L^WoEoCvH+5pq#af z*k?b2p~3oT$e5*0ZH%v7SjG_C8%T3TMM7UIyv;G%qgS&u;4qo35fO`E`%>pSXi&5k zhFcK8dNrr=N!HzZX`bF=%6+iDKQ#m|kG`AE741DOPm%D1ul6QQ)mftQcKJj5f_cY# zLtVG}iPM6winQgEFl|%O7n2BiN(M2n`WQoueeA#KUcWC-T^EiTc9TuX?X63%V658h zU)J+A+uCxh2*`AfnyiMMC^W98@b3^?+}t(fmO3QhQaSzx-2T(=!C+y4VTZ|Yz}*J- z{tdYG3i7vifD3a{0GM*qUXU@57Ok=bup>ct013K3t#f{Z?kuD7AJ84PBR?9{_2Z|g zaC6NRN_BHlCX9Mnbuq7TvA&~*`CQQfSN?55q(dMb3mz8aeEs|LD-D_Xu0%|G3WCXk zAZRdr$yubzjGOcoU7LRabSvxw_ zM=5z2cxZT{@G%yJWsA+S?jbjq76!0i=^o)lZi<=K7A1mbU+;7c-UFI#SbslZ2h6MO zafn!C8Vnzs3iW>~yx07VksYw>oQdnYbBUw^7~y5|B$DuJBviM8d%4=DfnDwwQD)*w zluQ`K2_qdSt#CajKi_z%GAt5o<8@W!(zJ{?`Rnp|13y93^6l@2@3y^>yDq=>;unD#=l z{+rkhMJxD6Fg9Nz?lONqe+-O^6{r-LEVlWRGtYA1;9Yc~R2N9&L9R4^)@DG}dR!G6 zpG(;s!6*Bef8Vpav7}hpS@LtNs=J~87efWOZ5hw8Zx(VDqj<~6E8IthX<+cGMn;NM ztz$B#`oP~HjP@IZsk(P;oh8L#16oW8t(Z37B3TrZJ7sUKZOeoscmYT53@1Y?Imt~p zGl0k$W1Zf%nnpV$3R4)IIm9Y)GIaSfJ@Y6)n|kCgRI?j@LpwG}z5T!aJEpMS!}etFEd^VseW$1u&WELr!vRq zam?gvd^|T zN1sjd_RBYwH2b_}+7s1jfF<|WIdH}Rt35^1){ZIAtPU?%i1sAYjGg-y4e-0X z(rvFWssB2LrjL0%Nq!2$Lfio`iZ&WD+LR}Tu#i@0RA2oSFVk34{f+;kI`&($@Iq53 zwoV`Ebi3_c$%V~#zftJ!GfKX?` zGPOY7gmvblgP`!bBrW>NSEN`H^wn{z#RtAB*~h%%V;^=}mIQ`1K-PR>W(;l^XJ5RQ z<1R@9!#u}Yk`f#yZB`!r4tULSK@RNW{J-V(#O(#PISmP8fi4s?K1qxcReTe?p`t+a zIHwZgjLMGGT(Sh|6%9HjG!_^Xdm^w<^EDha&^SiX1FjvUn`bcRL#~9FZdcU&y#D}e zaPI>S0K`k1%o=HvxwhV{^iLomP=UT=1o7TEh|-nCq+IDAku^ z{A{}K41=y;V61(l@)eC{VkxQz$mDV1mq7(o$n0KA${jdc?$aB?sDSjw6R6Z~*lQJM zzbVU{nCkK_^4;Q@Y$ax~%RsWa;k@52Y)1{il(pk8R6XSBC9hgKiUtjem$|rHeB5mq zTFx2oA_d9${XC^O`fMcxi*)bh-qtp^OKF)0gjD~L%bu4y(kD*hH3XF zHyaou1~zxRmhzSq>9rH|8VuBK`Vn)zTPFCmd@Jhlfx>sSi{dy5mCDGeNqleRozUr{ zVkVV_S&~r}1>4Ue@})BeF7*j2&@hzbWCxg}VGfh=seKVL&ZwTxA*1mn9BNZb9$vg3B4-T&S5%3 zx_+)RzWM=3cKuX#-uiE_Cww*6qptBNggkE>iY_qk<=290$$lWW|sQ(7htw(jQ7L{3H z;-=nid$bvlL1ppZAj-QGg7NXAXCkd}=pPV0jXR_zy5@AW@O#LFG z2Io%^gOK-4PQ@Gt7X^GwEF_?n6S`@ZY&*+aLotVRMCG!|Tf93#wfgL~gCPDBr6k2OEJmpYc;tMh!|G9u2ZNywpCu zTI~KXX~~V3yGpbNFOF*Jv=lG;K7S{pl=0rUjdpE&$tFaVGR+bump)LB{us)-9#Q|h z%gRIXx64W&{l9QopI@lf%qw0P@eTwttf8n^DJ~m*!v3Dc+7-6maWk@ z+=ok_D3|)VmlPl*4$AxJ$UQH`^x44)9ZqbKsHB8go-dL5mLK ztoBQGFLo+Rew443JYyBxIQqw1CHTi%z5g=h)XboP=uWw__z)8Vl4uG#6|-ETk(r6G zM1Ks(FZ2p_rzj;x*9b4TKwxCgiL7b&+Bh!trwAT%^PZie!&--uxPmI}vc%(KNn2xY z8G0lvUEe@E8;p5|UrLQ&^F}IUG`g+`kS`yfl2bYh(RnU%Aaz%pxa4ajaHJ1Vbk?Hm@>&3IC$V;Xbi~rQH%_av5GjJ71&(s{W8kqm>O{Kle9h zzg_X|I4-`74T^qv-o50z9nb`LN|~5rPsbmGvCCtvIAE~bv}`WJbaTlo zIe2~)J%k0K%#Ec42?Li-))EKx#EcqJsC|49=OtP<9SI_^;ume;l{x* zZs`s00SV&i6JdbJyoi5eV=Cg3eT^N6D}f1ITKiRfn=DU~;T1mL+_VEDNcHH)Gvtc!1G%UTb^QP%&kZYk{kA7_OOhGPs-x z@(d4>Ud_{Hrx2Aw=)?g4HH!pXd-op=H{F>?zYj?COihnSt`m2sa$r!yZ9ewHZGGhA z`iR0@V+JzNw4qEnmdReLEQ1eR5|})GD4zuT)T?O14Ts-eM=9lfOa$NXj94eUmHU^h z?_fAZYl5lm&>RZk0Wyl~*-qU>yB8u~)*rJ%Omakgr*++&*FCa-#xX%2AFd#5SlrTHvjIwZle6(Tf#aDO+M$@5> zLnSW&ufsT7#6sk=2ERlrZ;pqiD~*5$JydGmKIXcl%kKKz?OG}{kctC9Cv1MGiD$XA zl2&~-zXv+n`}`lO)+OfV+KbRKlQK|!fO#=PR?y{XkC@9-TtyFE!B8vjdNmGb(PqHP zexSdhA5CM_4Y20ZPZxrohN%*2Z1z0GYI$YiCrW%fN}0%cPbYqv!=& zrJdsfDz)`vhuE{Uk{GdCz@Pa=idYy*+t?|DL3wpXy=E!S(nS}HDbf^?Gda_c$(^IzQ$ zg#tqwxO96jKKLN#zg%+qXZ1Ea*FS_{f35WV?=w*UcNU!fA2Cq#aQw~`|BnpRt?YLS zZ~KnOTLV4%{zrlTul|0=W85LZ`(Npo>(A2N*K8dBu>10`{rOr@@E@ivp5KN48|$_E z-^z!*=)r7SEBORTch2%e0W;Frsn606!s_Mq?nh^%RV*1(zYMpj%zV;S7rYtC@wu&$kcWiTPY zeX0JEZAs`vsqwbX0nb30mM_k`q!MfMCg$=AefUDbwM-&Q_UJ-0<^H45AS;=yE|U2N zv_rPJc}-CSiHgJc;B%#_@c!$3L^?b`Z7c?=LZ{dm38yZR@%{bfOX6^sAV*=w@YA<6qCfWBj+F?RMc8-R)83}s7QtiHKw&>esShjS z-dg?OGjDgDwq1yMi+*qJ_%UDdcwG{i-l1wj%|@}1Ul)y`!X<{_HxwPV7}Sk$5}ZV- zh1o8JQp@s*%Eu+qH)U7um_2lk<(~oZoqS2^Mh04Rtp&ce2hqhbuI1B{Dmk_XmaAJ>iPxNM)3a=aWRJtSQxd zysak7f7=|6s$%@^yp$&i27tU;8k8gua9kuBymikKUylVO^FVSzcEl`)`_y@~|Nf6O0isY9u2DLQ7 zLU@InUnrqRoL18T<*WJAys2fffbI^DZCX?!Tg*1$5Pd5yfME;`V7@*st5`7?J4a7m z6es_@0$;OuyFzrthQcyB;KD0=WUe5aLbDp}XqI2;Y?(LCZcX(5xCKC|6V8v3Z*Wc; zvfkK>u1G0k~%bv z{H4uJ6dr4B{K6Nk2lYR?xa3(AuD?_oMkit_g>D%NCoShG=IMMTeInmhZXV++nN!4j zKcy`c{sKGtiKN*5C|&WOzU5g8h9rJ@2eY@ZbWTw#Chl=hA~B^$KiwKC1Ks_oUvYJn zKS|}NThW&<9;}C%Vp$g@4{kZd>l*XAKEqI2#Kk*^-ire7seE+AeoTk{nV|xicvroa zq$e>wSAR%w==ErD>y0RBMNj>%SHgE{HHYhJ#<%Wz@O#Ad`{X(w{ZhrtMlV^O*=}uj z4bzT!qT3g-PLvMN2dW=Vkb&rVzE9nt;6*O$0}8-{@9Kj3xKoH?!SXyFlRsB{iKy#2 z6@S{-n;yji@8`4dZWStXZ(UHJhHD+Gog8Bror2*wa*ay&uPV@S zw2MDz3+zZbq}RH!mNFpRTcR+0dzGuCqj>9s-g>yWLYzfNy$kS(#vIbLKDslCMdO|E z&&D@*toKIco?^3JOiq0BzQUd^uKsPExwtx8nAqRh8F$9nYaTY+TUwT}`(1B>_DAj6YWanF5>>%byFSA2ffROT#5K)I&^@C(*^L z!5&@o!<%|08-P!ew3pIN@tikWC~r@qQ>lZlf~e3tIYDFRfV+?s^f*swS==P$fV!$q z7|y^OqWyg_ii+?FV+cvb@OnC7cQpHKiv}b_|9O|!nA|dNq&{!X%97&p5ATLQTPrhN@M_XqNS%+V(Ci1Ux0}DFL&^@FAzsI@nKb0Cxu|Iaj z5}MPiDWlXV{TQ+@dCcrV#6mRlMRnz2gYd&tKVEZeRx?4xS|%>4tSRUc9?PuTZhSaH zw+>5?0$=hZ+c2Z2{?bhCOZ=C1jVpMbsK?7%iH;rLs~j>j+uOV^etvub80yn*DJj>O zQIiT`nX?@ia>{)-+0}CJHPx(^%uZWeq!`pAId#tn{>5OO!YiR}rq$STt6U?jFJ*?J zB?Q&T)!+;R*5=6leR%nJ|CJ^9 z`Ze2KbN#=PCdmDpwd>#9n&0yN$27t1#;X|3j*~NNiv3Lg);?U-!28$<&v87@9Z5c= z5d+`7ICSh+c^oT9^4<|lA89Tn{RHC!PNFK+n|CPUD3Vg*II<0T38yuO4c+h z8^1N^a1na*;F{hetsPt8Hz~FO{9?B@aKm}$2DgBvL%U;%{>_~(-ND1$l2&{!&SYM% zU+3@+Gcox~%%mKVH2rGpxocV97;-)Fr5l>Di}qi)76EX6J5-QhRSCoK%a|3>Mq7#g z{PEi>?yc^bgh&dn^s3T(?UrMRW(y9AujM8?P*F&{{q@E9jAs;uDwU?{8nXvhz=^h7 zx)tr!7zY?J?nJapW>~+(cH%?Pc65`=+Td4ia&B;Q?$X(Cx924}ygE+gYF?;dC(y;` zYy3eam0BGlV@MttI>1=MAE<09Dj^VE#hp{~$oH5U{H9{{%O83WRN@72( zNJWPGy;#xwQ{*@$x9LWjGX*3x>VA;IA$@X{x6v-3+f!%(SQ*)_J-9ArKka8Um~DCt$?Lj-$A?^N0Z18WL-C{K42|Q^-Iy_ z@7_5ftN&(^a|DM&3H4I9T!$3p9^b-+>JjthB#rvduTCkWd5tG-4!QI5&-(P_=5c$? z>6B9W?Pj0(Z|$3yysx{-4W&3|lJvkN;aiV@3Y}reAybMKr5TE;V#o~|QT1Swwm#DanS?9X5}SHTxxP>a_sISFy4*@^8Hm1y1P-dH1iYyIU;EFCR6`xi+1AzH(qR z+0{l?Xga;svgp<{m-YH5)f|9Fl|k~AQMIe9>Kh`tvJ`$rVSI~)fR(~xUmXD|`N2y6 z{aNU3Nyw|}`1T*EQs>x;u>DR1SA#~tX<8HF8X(|(oPF~MgQeJ=pIRiv)?{Vxgu~C= zKt9Cs5*#(9#8KtUJm>cQBb88Nr`NvE+2}y;{09q@r>0Kp{vKznIu#nBq>`HnJ^50I_8kF&# zDI0uNjP)lbP9G{>56zf(NN~K7$4tJy`eh@fN)g%E1xxLB(Cdv>Jfb7a%S8g52?E;& zyV8=AI999L?pumnZTiZ$2cOuk-VV}y4&Ko@{@&V#l{uu~KDWoB^|-)QR&H0zu=;jR zom?z^-TdTWD0_Y;JO3m4y0$XN!PgyAW)K^5kAKv??6IEQFxt#loBFmXtbSJ=jbs1C${X$K zNa^le%x&%3pYD>)ZV_drs3|HYzmf*iwmF}(n%gPkyxPSM3259{IQZxTAzRBW*;9H( zIw!f3yW&~k&F?gzHbyRYHL|8{p}BRM#y?z9`|D>976~lk?Zw8Hh>cG+&$b$}m#Dg( ztu^-D6z0u2oiFNvJY|Nkf`H|keOnADSxm*M9Iy=fXFJS*x^k0d7Rq$$tid%0p@$j! zR|psGs(s5vpiqZ`#e1&;P+dAD$0xfZ=lvLZ_wjyA#yEa?bqTiULRvVYJ7ixJ48C=|R8 zy!Dk5BAv%u*C6_?@x~sg{}BWt*TMva4E_*0Nl->y?1AXtZBFDZ^s_cX(=gKD_ooKZ za3ZEN@9+1s${H`gI#DfR`C6uIt(Ub{#u-E=e&{1O@dZr~M=fq#YKITXHFFmX87jWt ze{J~2;-gk04!kO`^*u;+=7*O=|10kzC6j*5z(tuB5-vQFHx!o=WkfHcOqm#&#?#mm zXrnma97>TC5#m~#r;4N{3hSJNmC2ynKi+S;_&8;X#`YpgQJIx6)9OX!D8^Ut!-+z; z19ed#%w%Or>8uLmivh5`O$DnBFDcn>Auaoo^SoHCmyR|QM~TZevWQ=IR}yJcjZj& zcNK}o=SaDa9LPt!KmmPtIx@{ay}s^f(g(B=ij;rRiS4OEjmg?I4r8!cpcKmJyY@fZ zFCj+#+0&;-Wrk0xrknD>n1YhQBI*>*rxbQ_(gX#{3 zI#-N^)dC;@vEIr~bt(6AinYX0R1oMZ!fQ-a4A1{n<#TV4+!nt8z11da%|qFGn=@K^ zE&RjB823|3jjsNnKK>FNg2lldm}nrQ{x1W_a~39gE%5G``>1bUpp* zf6#OmUQu^%w5NrkVQA^@ZmB`KyQI4j5TtXEZt3on25F?b1qA5^>F)b|@BQ6>U>0jQ z=ZXE<`wX6y#c%sjUzu34;`t}v{HjUfTtjm{M8d<}KZUAv**|6}o=-L^sY#g)h2H8y z2jdZk>dW|yVzRDknlUFCMH69}h~vvXZH(#*a4hr7{TxU;JkVMWOOZaifHkO+p;N|y zxo?(JP#p_XaFma{X9oqn&CW#+v=i6d2}E8<+JdG;AM6hO(Mjj z1jaO9z*0yg^+@ISY_k)KLs$!s>fkJqWN*MP1fxMgg=cFWVTDSq+9UWhzWKU}hrW-& zDB_iQ3merfAL~C|BmNO*zmSN=6aP*MzWciH1TpWD8i!TAA>oR7M;=pzNU9LDxc;W^ zB$>oH`r$^=JbaGVO_gXsL8O_PIO|6#yR2#0xgI^d{8VfOS4Z;v!`$M&Uw$jPdwxB{ znQ7_iNUST6RWsaaXyGrfMG=M+$LOzlM}BB|LMM#2H2&b>o_jzTp=*u6-Pz824_WtSooX?6j7|_I=z*UpM(rUfA*_sp zD6407B#SM5#ezZV<8+Kj$KX@=nRB4lK{k?&2r1)J@s|xLj$o0jDt&%L6z$QKt0H6u zXQUlIFIow~5$&b2y18$HfU`0B?vT?ZRY1|6Jokn0jJ0V(CxV@4>cOVOKH{TFdbE`{4iit^YJ=$u@p$eT zvW7|~h9*Yso8F1*GNYjyB^K{R3GJmD6?gITlAN(ka9#SpxY8H}H1vu!(L%55^n6Ck zYoPBkGk~)N{rf{^4Z|dNrn!r)8Ilh}Vx(KXdd~4ilqwqUsvTV4%r=fwJG^sKb(2_T ztQ&|JyfTTldi%UzzjOx;i;iZzBBdRNWLo;H-km9Yn1o7Ioc8ylj zxwChs=q={_^wN$hQ0SnU_T~QpZ}( z{6eua@g2Z7ulo6pEf_QWQ+mu~f8EN=BrZn$Qtb<}<2gOgXr3xjjT^S#^9Yfb;m(>) zw%0ooIt5RQ)g1!Jm%`95&Ha_`VfV%XZ^)*6@T}`_2XJ1J6kea$6D83m$N0ylQgtxY z@y=lsh~avOjUqYHauNOE!Gr$y6D<|6PjKeOK6mQEgVJ$*ZjX>kLSm3jS}h}_!wS?X zv-}*7VqDh2UjYjfa(rES0@YQOM%&&37N_ZbgwnmOQs+H4jeiL>ia)Q;^*?k;ZTFJQ zs>gKYp_n=zLfar#w3ZEgL!|LE;T8svR2Vu8a#mqYpjV&>MG69;s73=_iFH*BHSuf< zA2t?y%@^vQH-|n1HakzopHc)6-~B6IHx}qxoR2cbPY)*s?Mlgt#ywkHe#$Y6b@o5B zM1fE0-SA@^tbj=>n5SfWpq(dZ{1#kS);o#`)(VEm2!lcUmU@i@eOP0}T)yr{GdSKE zT{R9%&ICl@LA^>oQdY^@kVegDNQ3#6L5#;o1g)(2bae(zbdJIh&FLERYHPJ5NycGj0t6c$%=ji+fq}ajw%jH%hI8h;>_T<)Ae`pg!gGo z4K{a~WNskD2fZK(c9h;}V%D_t#2$L_)uFXHo!paHR<0^)iTk!#Jx7V2C#nV7$4DB@rHDJL`er9Mc-B@wQ4b+cmX*1(rEKx zB;Guob^o)-%@YVXL#H>Lg;+O+XHi4VxU2|+jvKTGlh7Z0u+xgp`_YLfG}O#2YI4+0 zQWZ<&me{hM>#sOL0~`+gQBdb)cX_sJl6R)r1lCGZf=u@O1`%@-WwXuCUF#?%Hw2Dx zf{gedC|9}8`>35h_;7DFlevSBv_Esixk}kTyq?2YVR4%UY`eN);XOhSE4)IraAM}A zI~=vZPy@o3uV(mps^@6WMaCT8FwMXR14U|*R$Z!End135b(hdcR{s)>2VU#8p2VY; z^jO_qEt>Ty_qxf20V8M_OR&Q7bwg4oq8b}mz|8VIhd8r8S#ykYL&#kKB5gQ+Y9g2sJ zV%F^EUGCgQ2fO<*rQ3Vjg=pZoei=OjcBvI$JM2LVUC2^Ucar~OZY|4VImJ8D_+x^C zV@=pPmX9Yl-pDdr7JAn7I(UvVxXf;adiKiEuGYIJ(z8}KiuA3&G|`E?#{6reUW$Ie ztd)i;W!ltP?*r#AlDNiQ)Zd&H_qv)+_EQFmLcLqqj;g@gq{`APSv?_oG8o1!#Xf)Z zsqDJJOLn}n5KbK_&m-Bs!L zIv7~teH3!$b9Zf04l)bo)!XUdnhXIo(nraTb$^ue27mQ=U|2S4e-jdP;5}z&QcJ3p zh`4gTruvK%vN`>S)sdY_Ok$7y8-hjeo7|xEtHI#z%wLuT4;a3&a|1IO-XKU5SrN`J zZeAGM8CcmJ@*lxGX#`T3F$e$Fft@2T9mhOd@Qyosw{J2$Nuq+9+1XrX;_~x_0Rxx; z)tom0{RjMEI`1>17KPW9;IJy#NJEWegOzh`SE0tG1l1x8g-pr9tB!%o> z-t#+`kY;(|j6Q}e-nsRfAkij*FhOLXnwY(`Q9t#vasjpP0xZcr{^UKWGt?Q9j|^kC zFbN&iJ3?duwyLw1IAN)1K1s_JHqA#958Wm$<-aU`tp1}>a#Nkgvf_u)N8g zus(C~sWmBHsLbVr?1f_3=2XK?MV0r_XIs+^lSjLrLhdQ$QIrio-c*m$ zRD)NfFnyl>7#_h`pXDnLnaTpfQ>6M#btFdlg*q&EJ1I??V|}(|xA>{Q%WQRjVsq4l zc~OV9^=_u@5l?}YR(a(ve~f^Z$!eR(hlGw`S9no+=sMhbsza_Xvc0~J6HdM9 zm8K&!plMFC@{Y)!SYxbPEr5CH49rA?F1M;`1On?zyEH;~W_~FtXfaD&e*LrA7p$9!0{yInANu-U!g~!f)Y|n45lJUg)pTJgLicW;(ZoM*) zgIczN0_W(oh;UZ-6t;(ZS3V|!5p>Dtl^CrSq zSOoox^g>g3UG`VLfSEkVy!X@Bwogps{&BOlpEZO|0!%ss#p!-5tZ~&I4His)%q#t$ zuGdCXUf7!S10LG9d7?52M)G7*@vwaf(?~}s&%_{W$pjZ%mgWd`j1ua8nQ)MSVjD7* zek*Y}n4YYnYFfhrFSEkw3GU3rbj$b1?hSi$f54&WsAPc?Gg=m zrEmLozsbM;dNccL>kkKR^2ls&L$2S=&HAK1aq7RN4D{p0j@l=!M~KYhEx82Aebv~u zpZhoWsUP>3R>?+g3-vKz27g(uBtO^HlCZ}|K#TgN0f941zCiCEIKF4^QSVY^HZimj z_&nP#hgUxPDqLWNw;MAR9hHGl2&PITn?W_o7oAMgNgd13M0XC#V;wSh10)DQx-$6S z>K|K2-qcCu^xuEOXy)EQO78U3p!o^VQ8JCa)A^RrRXG=s?q188ADoB#-F3t=u9w?W z#q(P*3`{+hfm+KxvTv4*C7ZOQ%U4i0hjnf`O-dw1lGl;8{a_VSrEe5h#!*siW@BKX zIRj{=9Wz)IFI#x?6scU#0jIhK!BWcSm{5P?_@A6urv z%<#THJkph2l(pGNBA7E`iqh)l5EXh#_i(+yTJdt9#QluHC_uUnmi-8uf>()9R-Tr1 z%}Y3(p?4}6!Tis9e}V-W;Ffv=tO6u^tY9*0p1DjUv-eW_uvt? zWFIq=B8<`%#NxiLkbXSCXj2@|+-2z}<2kx4<_8UAaM#=Rq$MO@{8MdZsElBsm2Uu6 zbj41xdA|4>uf=~i@30F|1B!`8^vwBLy@O&?WB|1|km~$g-6910y^xuvsXV(F4=~op ztv-I#ICc(DE7rir$GCoBXUin;eyp$pU9k&{SH2VKszaz^iV@KId9X6Etzz# zKpxSh>>Rj?CO?syZD%;5lIslO6@v2Q~?g%%DiMLgXuJxprU-gDhg$*6q5%u`rq`- zMd+kL^zr|zgq=iC<+Qhs4(1f;2VkMTXaK}Z%MWU!8|vqSzWH1kQm0R4VS){-F6pc0y+&oMdVz;4bnL$yUb>^uTX0_^W zUd;Q8KgA-t>POb%?NoitJM|zKWgcCSqaj;HrFqrVyX8|&M>>NeqquBT#M40=rf=r7 znn-KOHqz+~dVYZ@+kIsT9B3r`yU~7b)xQuuE{$u;u&B?`u%gZbv!I&|g%#k06`9XP zroGPc5@sV9_;ExI{Jo<^YHV>3JPMn^4xupFRdn=;lI(Hht16~DSoNUyR47ao zPShO!t3|fMss7npZk+$Cz_Syw)OOn0-cR3SR3?U=gA9OUwggNYWiSn8S4rqMH@?)3 zNZ!7tca;+>$GBe8O!U#bcv;n0EhS1r91yU2&L z1oUgk%ZveTKeC7E^@AmtLON%=@*FN-rF{eQpm7T#V$c;REUaQDy0|oDLvHhKvtzP- zRtd>_<<`+1KZr-M5*Kw|JSg6&s>=wGC&McuCRXUmc(25mwDG<3iF$AN|61Y^B0_k5 zEjKWckO`S34+JSbm-p_~9bO+N>#YQE1Cq@N)#hKpzp0y32rQck@@qIP4L$io-DE^h z!vHuM-J3-?mJykF+T!|C;3A9oe5M{f8jv({X-(CkvW4B#usS!hT`T3A=PxO6O}hi* zZT`)7cmd*L-qkQ*w2Yx9JosotvhMEEIKO>M<>N6 zdVH?vre2rIXtc0)Zjbg|DWV0v%Rq*E1O9qf*S(reck89H-jl$5Pr)63LW z=YKGq;D{PI)wE3d4VTFGhhS>z1ljt%1J@Vo`%-3n^y{Cde(za23(M`88tFDKeawqD zn(e$gp~+{Vj*wQyA}zHOY_=l)Toar7Ru^CA8DTmVo~Fz77Itzan~Y~fS^RQ(3;_En{J zx=Lp91!dKTq&mZVf5{TPF;h(ydI+&$=fLrU?{o!OeVB zXOItid@BC~^JjUYq z{58nPrh~Y!t{JxA1+CH4M?egy+{SMw|JtOQE>6Ar-`yKOZ`e!J{QkVoI2!KU2Do$> zAj9DXBskH<35}87JPYm}tIS_*X?jpTp&MZQ$YjzXbj}|e7ZxdixhWEttw zio^dnk%u$N939xiM9xMb(>3wIe%)RjEH!hzoRJLZ_7Oj(r_0tRE|PZfQ- z5HUB;g8Nch2n{rQD&g#8YkSS!h}S{ZLe8biWYz`Q&Jq8em-zk(>F&U~oXQ+})8ozR z#d%qGid}*@QpZp7wa`7U2&t^#qhZU~!oZ`&&2&ifG!m!B%~rad)^z;IpMq`vlOmzf z9Fw{DmPVh$rHoK_S_E}O#f?7G@8sH0NmcIs3W5kHVD8Kem6gFJ?Q7m6N}%8+gMjM3#RDIhHh2_`Wl#oMm$S z!Lb$dSafM9q--I3TwE6$wU^cFDKnmx<#SW&rtDcTNsBj8*ks1R{S&R$Yf-q5D4R%J zuaf(XfTx7vx;+wtYp{aWDTT}QF<<^^xXt$M5OtvCun;-PhyOQ?Y8m451uPV@mW$k0 zdmiArN+;P2DgsDCMu(=%FK*Tz^In3;99V?lpWt>ninTEr+Z~N59;V|*px7y-u#^xI z4_-U=67~KqwNKuMN5;Jw-u}k+lH&~KkYS`V#aPcHD~8x!0bQtc6oDw9(>-V^Z_Xs_QG zuy7@D?Q{+1?rtv{(|1EaU+K?zRc+{^I&2xLJNj+iZ}4f91o?|wOc~*;HI-1nOo-tH zh3Lsc5Ii`wJc7=WiNq8vV|NE7Nk5Uxh(F&CsK(vNBbMvTaQ5Iw{0g~OAq%+JDit02;AWwaWksZ7Vm^?P9C*TlK zSC!6V5Wx;QRXWvi&x%8?7aIVEfJ@mzv<+o4k4YZ6Bx(j44^xb0gO_m9FrmF}II%mS z!DX0i=C4^El)_vMx?O7_+!VGIMS^|aIVglT!BL%t&>kSb`m-7RM@vX>C zmRo){>)OUE`;8A%DQa5C(TsL9=p$eQwyql_0oX7bkWDeB0Zz1%jjZ7LK+410RrmxK zNOmZ;OmBdJy03?_ORdOkd3=fSIXh(&nWjNJi=g7#OGFis;oX`-rD`Hiob(_Uc~CsC zlv^@^=V1-7Y{_`0WnnXZF@s?7uCDu>yP%kWM*c8K}Aqs z9@Gq?yWC4Px;09;l-7pL;`OwsxIO*z#lbk~kRRlZ4DB5urGKlEjTmNCOk#?6&ldr0Y>(AVp0+0+l{hRV|+3CeNsO(5Kn zO6TiRIX)4mNsK)V6v_ZUF><^MgD}9@sQ}ejNt~n!t{mgb57iSea0CEux`0iptO2#5 zX^(DmBBtiRI|ZDbUu!CKQ`t=p+s5pq$y(?>2PaY)&O9OWG^o*xolJBb*#A0HrZvVI zKVjqv@FvotsLN9q+swyC4@f}q#HYE0Lm1w;1&6E{3pB!*!O&U0)UZ-qjSdIw7%CsvRe+Ql;Y+4qS1a=*ehHsJ&ei!eemcd_zKZr{OE_qy|TV&oj z{IiK%b*M;tt|)(g;1AMfDqKWd!)0}pDnxX4oWsa(o5KV46C$|Grsc#e3bOFrr2?C` zxZ4;#Y{ig{!lcsKJ&L-PxW&<4_V%=oUC>MI7~@(w(<4?-J6Ws7<}jUe5usc-Zyj0j zg;^5}5nNeJ^UHEWw?Wh^!v=5Rs-zCD!AsDuFaa~I#sv*n{mK!j;^>TBDB)96;2qSP zG9lWu^{7OpSRd&}*$hHcFD@a;6)XSTW|1I@sq7Zvnz$zu#s4E=sNom0Y$#mpLCHmW z_bh{k2I-)sN;oJo(O|_k&ta1%`ya(rai);K+(S`gRzS|I=~tAI>e022(+q0cqs4FT z(<)aj;-MK7Stw}=tV4XULIM1GqJw99Bbbc)#8XrS=@Q$)FjJN0Y{;@-2dV02A>{^UX{``(O)VMFyV!Nm8x}RU62cuY;qOsk-?2IM`3>+^dsNu0@`?n7UbqarXt| zh;%aR#fd{u*Ejm`PaU~rjz76M9VO&Ra9X`_8IqT;eTGKFG*CKWhI;XLx zp7n@eA-8xP>OqhwbzO~^2bT}b61K6^ahmKj(PX^r|JS!WregeF(grQt*- zvOu+u(5wp7lcavJb&#wQ?2r!iB53%1mg$ZS&G;r^@2e2Z0tWRF4dsfpMn+_vtiT$V z**fIXPAJpqE;7+#Hknbjla(Pb+*zEBUp^S2(A6qxENJ1kc#{X%E z5;)ht2xULeqT5C*dG5T;dF%l4#q`RODk^{fK?v=&wTA;bGQwO>TYGQep)La1MX9iA zFH4mw>4BV~bq<#;&GJZcZL9sqK!kC<_UFR{5FB4xy%`4A;ZA3%;{oSul#?~1BlOGi z(M%R*OoYk4GN9vF;IKi_4Cp5HfNjx9`s^R>%3>l=t&9o0jEc<9;u(I3Hd%M?25``850~ln+@4_qVKkH5YDZc6>qVLKtzee6h zp0xNv-52u-G2mZT@g`dU9SNV%P^{5U?@CrGoRJO-(}x-wZN~8f0mr;B$RL*@M+zUc zVSQ!~{j?$0p{L4-^jt?MJJ07E>EE+5BOgMLxj5H_Uqqh%=#&MkuegAL@J{zvggyI` zIe(Ve{OMY!Hs54+UaSqpWlR{+_&7wYd`UG!}(H{0J0a-m;BR@Jj#rA`xD4P}5gd|#-@ncGUU87}Ao$02_ z4l>n5UM8jNBc!nJoO7eq2Y*x*ZlvNIAo*<7;U1UF(~yENE}a6L6k5u7`9tRR@+KSw zj-8kY?IZ0)0O}N={rc)r7AVL-`#y}$70lrwbc#csmel}83meb$FzZ)X^N*R;$_rK{ z@I7akU&vexgFWH300v!HGvRn5*4PelZ}3sRZ9cmugQj@}w?2+66B=DRZ5|v?ku^)N z`HV7_t#iG`tggB5!=U(GF~P%PRpetIV5GU{$IEX>YeLwF~9IHT}6s%_A?h!b2yji z1+>IR$0Ba{C;koDOyNE7?8RX&k3yaWbfW$#sFpQneKRkgU~e*$t#+(^@Mx%II9yH^$=Chj>0&jE;^~KIPeXhEdWhhw%_>gr~-X6Vb~SVvdS<>tp*;~)z;B5`WeJ?fzB0gtr_clVVMLQ zVvE_i{k!;+iY(at*TIlZ8CAp$x8X^nUJuJcj~tyTOE>2Qp-j&2c7}1En7Vl9?^L#F zvd6`uD)`*10(qofVT|k~Qk}lvC)_`myOgYmh7;J{^q$^)Wg{|R83+^2l-0;t=Xlpl zku=W2$(IE@%Hs3E+%|fLwMJyQ)PKqS{HvLabb2y++y&_6@J~Uk24kD+>4tbV^B+jEuqZikA_2g3 zhA2Y6E^m99n?#74;WXO2>sDJy9W7JrQWXMiNS_q8L&V&HBBKiTMs4oDmOm#ybfGqZf0RHB0e7@SCdNEZs z&BekE&}of(!7(mTFz(xm`*0QEDM+0(7x`iG2LH7y^(jL?hh+`WsY~rMF~b`0I3Zy zlT2v5oT4a-nj4XRH)jOalbcm-Z^oA)rd;%uxKF;vG5xS8WB?9xv#vL2p{aSnkd%I-hX+(6&=AFgxj(#r{`wx!)$4ivkBnq|- z7O!HUtx0=b?roBKU{uV3RR(XsX`fu&oEfq^vkffU3PD;?>bjKZ(HI3!qc3vYL9_va zWykX^Z{g~7u`x`4huH>T1Bl+13-Gd0PqIlyS6AkqM+V8L$qgz0C%d+-PXk#CfKMN1 z*@z|x%W;wjBea0~&Iie%kz3Ent_wk3I6*3?D7s7TT}@yR;+8g9U8@S1VH;rXLC|xb`$9NQ;8#cdgSajSF_%RXTeV1hp8WG>~*8r=boi_Aom%H}(W6tlq)wez-mE z&aOtLu&n2;Y?&($czG!VE&x|T$2DM(-C&L6O(ivGsCfv)c^y@pLn3}J(-&31rv~GG zy{PaG_2ldN%8-WGYX4|odUJ{XE_6SWyY(Qiu(D;^#3+A5A<)a*p(Bb|s)BJF74eH| z8VK($jkxxH9!q92nyl2dPAVeTXz7KCbe`68jQXq)Io6{~mDJV5rF^M@Nuoq(oGiwp`%?)Xo&ss0X}H(5vLZcizyqB5E5I^aarN-v9RnLRDp+oHHW0x{Nn2^++15tsY+&w&XbCayZ`w{^C3< zVPhM*b^y}JRlPm|0{48L4U)Tj*0&g;z7c*j-00sl1s?*M-#ZuMNMTCz0V1&stU)OQ zBi_2rX-&Y5l8|i8i{F(+c=Z3rB*>=KEYHEk;9shP1Z9N+IAQdx>OWkW89skVo7iGy zS$-+$-TDq1QAQ800dh5rK6WEFEI9hJ1V>z<+OiSJ34p&#KJ)~SEKgtWM3n8WaU;b@ z#uJnTMz#9S?HFAj%xAu}eSZ?yO-M)xjX=F~Da6dA7vljU0%rfjQyUQv)Grmy!KX$J zus5t^+yWN*aad81LW9f+sd3O&j!z#$bQLN#20HYyQ9zV5P@Dp0To_4mvivsTm}MZY zfc@wTO}1zYp~^Z!XA=`oGd81v_c$IJ6fF zB-MKvVyJx!Um{iMgETu-RU64Y5$j_SFZDlCmrE$1aK4|kc3Uz~AIgLO&C=WY5#yPz zoP>~8J;*uI(r=#>o`rz#G6ng`JXscD1F9ie_*Nl@U2k}?`(1!tPSnSo?Ioqelswgz zKa~Axy{>=sPMK)T;A4%%Br!I5U-jkmpZ}E-&2ZRfLGgck0E7FlssS0Xfyv5Yrayz9 zPh$I4@D-1?XBkY|S$`{&=l+^2hEeGFY}xn9?MQ%#`XpAXS{BH!FEZkmKNIUKqgE!P zk*6BhB~EvpKz^n)x?dW_6+)Q7^>gbp@(H<6>#}NDoPJF*GmMy{By$S;O?cgCa;frz zdlt&5_4!QebX_4{(%Ywxdperh5`J9)NAfOkH@PqWDq&j4bZyEL(B3oE*^N>ek<5TjDU zh&)eY@rQCox2-?9=?#7}-0hnN>Hm~b%)TR(Jpo+(4S*V>H_K%6hQJd|}?cT3H~kXt$_3)1@6p+wfr19N9hV zFb}*hdA@NQ{EM`{a~vW@VO@-!e(@sVv8FC zVLfBbXivwk8!v_AqY*hC^PAKX3?p#f;Qaa8cfj24?v5A@A~L>uc=z>Cxrd2BC6_@{ zAlMFB+jkq=z@b7M4tfc?$*4Q$MlD)BbdUVs8Sk1d&9S}yQE~qacuKk1da40iGOSvl zmZA&LQ1&OYR{%nN*%sQ^dUGRM_zh#sG(3t75n-Pv3)%W%d=Am1C*>k0?nB9yf0#(E z-`dg)S}tEBvgS|jA{=-k{XtwzrqkTEOT`tbSCUz|@%KfB&Rq%A$WFTdt}AwLUcp&f z%Be$tIj@1C3dl_)I0I2kA)imIMaslPIMN@{U{N#zomq~!C=fpBFxhl3!MWS68Py;N zallRjfvexXi#$^p3do*nV%gsQGb&I#Nt+Up_Wxoo!VP9sQYQ;8Pgj^5LXN{nYw^$K zk=I=Z>8oZ@z5I*>^z4Yiqdp!0yByTDC!8n=;}Q&JtRjOwQ%Zr`ck`?_^4g#4b;Dlp ze@K$hAV&U7)-^!#nEnso`oCqo!UoO(es{6CVDQLP%IbJ4DpE}+Pl{{!r#AQTFu^i0 z5$393$&F0#xf+800`%bNR;7NG18fO18~4-V{09U_)%s<1s)_lK-{+Bo^fcN-)wthm zZ9@k3Kpl>vzj8+%Ef~GiOEZYU)4$B!UR40+huHE&_&F%eYPgvrMe2VRCSVHLU!)Bg z)cRWY2zcVgOvX;BYK1nA+QQ%?IW}(;ixBx1sO(=Vol|;OI^%VNZ>~w7|9?XnNMmIF zHU&g@+X{@nMmI!(PrGvk%5`)Z0-gK^PPPHd?+4G*tTniUHb(W(zhYl3zquYn|5$Ua zx~W3fj@zU7*(&>sVM9@NmNXjyKNW~=&Hw29pMh@HGZ4@uDwt1zAnG$9PhmP37%=fp zp-X%RP=I4uO^f{#X)$1pTC7@I>*!p=Q1QsFH@9!AYd*D`1(q7NzY_tzx8_3go&beb zzN!V|?MZe}&f^*~iy3!O%aC*bxj^Nk6Wn9M#e`0cU8xN9{gj9-IG+S zg2bYQw|D2jS`g0Bd{jtl-3k{Y3Ch%WM zSpE2;w7pbBEA02T6ytN3qBSpD@%S>vf>~wzFS>56S8UCJG@kD z^^vAs(k+R5Ufkj4v902?GfA=H@e3R42S^O$ooV`-qI8HwSDd zl&mMrAM=~cU5mEn;ieZjM7S_GBQwfx>O}TC| zL!=qMtjum-5d4>2Y<659R4Ff3k}YF0#b~JUUDeyax0`k+)9oydY9Wi% zPXg?Z3}hcF2&-+nEbcyn3T=#%ti~%t_5o*LREf;H84)?0IUY!G>K2?4scM6XX!Y)0hB#Lh>G!%=9J7D)h#@Pi$H|w{G6d(sN`$0QdisKf zQ|V6J8GlR(FqspUe$*DbMMGIxB$-$~*-;t`dJWzp$PN5H5z~4wcvLYq4u`ZufKaW4 zqmvpnFrB~jp6YaWcuTEBsn3-C^qY9M6C$-;-gBe4E?wiJD(UeK^qs2;3#yN`*Gnw4 z(*Xk2zHq(Q${ox;MgcE7O}mvc_@#{^fU5P4YAXK@l*)l)OR%6BLFQ70K285kKx3_K zR7y}Jru#6lxTc(&KX$(7k{Ks`KJ`}l=YVg1aPc>{-m+r$?N88qA=e_O$9XP5+>;ub zK?AImL|S>~UqNY-T7$V6#+&MYYjg#Th-Tezww^QKS!15e;7vq{a$rxr;Ukx1^PVC%CbX+0jfBdj$Acv z!D>)SS?u-q{l5%==&8KyCI!4ck+hWuo_EAL>2qa^&N%UsqCL0|qT!pH=HXoT-wizt zMg^;n4k68>=0m2LuM(U*NYZ2qFH6h)c%}fgu_8f8%(f0OlB_f|h%!MVIY-VT$ZBfU z``~0uCho+3mia(n?3?bIYnN(QqYz4#OEOWf1{93usHPq#j zYr1jg;g(N0=GbETvJ*LVwU&Li)czobJlHr{3at>UbevGeMVy*=**nJbls=;6Durv) zs^eovwFndNAH=BJ8F#t$#Joe533u8x^-`Lk$T|CbZiXIlhQf5XvgXWTHC5y~=nOW! z(walcFG5;3zXRC}sA*^&HN{HuW(JEFtBTTwf=PI>fz@kw;@8|iutOH8@w+R8yI_J128>L6-v=z zIO-$mN3O30rg6RRfsINnKNV|i(%hxDAiM?Gs%T==$i%iJ>;rc(FPC~~{?gTWMlniY z8Z((ul841)7AT8|K5^QjxuiZoa8a4dS$1)}ce7RSHn0O>nTzG^J|={O>JzJ)+O#*4Ol^- zRV&WEu2gZqGAxv4S0scCdlR&4QA(Z|m&q*sIAz}JAXG3naX=P(TChhB@X-8TBJS%z zA>T?%D(t7_*1K8q2_EpAM&493q2c071pDAxPZo!jTvbD}8b8?U2Q4dFt1l_CE9Qlj z_1mz5K1K^Lx>N?YAAOPC16XdOyJoY$sp5bxQ~qvUPm6ziI5Ir}QQV56b#Na!(cj+m z*ea%)pvQtxjTH4rX)Z~;(WZIDMI7<1d~-~%=8v|Hc-gvJn)3uqH0DA<$%W`#Zf z;yY}A4(QfU0)gyTI&j_xG4H?6S)i86ysnPnp&^EA=-!q0$8T=Bj2v6fd&QXkXRjz4 zRtik2-=STqj$bZ3^vRyg3r}*?`9;2u>%gJphX6jNiaWS3_j;_5icpBwD}RrzzZ zd`P_#(%0z^q#(8+*YE}f83zX~umTB3QICb0FhX1vSCHXDHO6sA)Ei*TsPGm!2!q9Agwq3pYkBaAw{ z*y!kB=Kw)iC-ezQrF_=P^e|+IkG2PJ$saM|PQ z^R(=!O-FKOPOpbBOc;Y<%}gIEF*+2fODVTv9huJ+)!a7hf)$8P$tU(N$BL)HoxEsr z@OhXw=N=}C2Qe3OFk~I_s3^8O&f-{o%RQWRXrxa~Gl8-~J7AT{RM8Qtp}$LD5|%5>``|d)yK0D%f0@YT+|Nid520X|o2ur0DEMi&A`4fLZ1_${$Lz$A301NSB4W zl+5in0#0sfCkC(@c#e>wU)<~EV1DJykTa;Ggk96tIwQ8>4IjSzB4w7`(rj zgMfahxw09`_4)1j2l*hJgxO#@5_-($Qb+pmLp-)D=sqHA>dmH^LMmS*8iJ~##mNpN zvRcTv41?1C#A@x9`s%$B;&ZJ+8xg=6Bd>SyD>t2bxQ&sXFptrK3utqiu8jqej)oEL zBf%6qdx?>=>s9g2@<@x@DYbi}5~#VDtGW`j2fsJQr#1}&lrizzfSkQ`(_I2_^#<=}#Z=ueQV=a-R90~t{ej#2xH3IF5)Dq8q~E?pR>!iNf+A-s zNxatYIm74C46wKtavHN9EEzNdKdpv3`vR`iY!HTSDU8x>Kf2yZ`~TtTtmC4Jw!bYY z3?Mahr*wBSbazX4NeD>yAl=>Fos!bsAkrlrk|H4S9)I`V_ite4Gt4=ApS{-ic~%(w zMd*}9U)NKv|6dmsi!?Yx;1iOvmi%bV>$lJxc=AY$B^~?Xkc4NM!dGKKs1DMDjj)hs zM!`LJ@52r>9DiVgR3>~5yO`fw@;;LqXZVyN8`GVB3|%Q!z)KirU0$62wCOLCtL6K^ zY{F-|QZl9es5E&kStq=JMF$>BUTfNzA9iQROX>7e>1GAHB+^?bVxk_@@d_VoVxnHA z&*IyA#Y836thke7T-8e$Ylj%4hHVteO39LYnMbP^(&(;}w9V6hh%a4~IE1)cD1RYy zD8v659?^y^F7PdZPf<5)Ruowj=hQxw_;0vu$Osr|Zd%V)`C*9ZfT`eDGVHvv(eYyB zzHb`O%WXGGL&>y719v)3hgBvn%o+hgNwNz1w%t6A>!4(cc{J1ageCD!GY0;zIk=qh z^ebUWTyJ_yBL*awOmRN3okSCJ@73v9Hac%+1~KGA{aGW7G{oLazMEfepbP2^J>)E& z?()@qCzq3C3Pr-sHNMsG+QUa$&8Ko)E#~6Wkt3-+o;V-U+J*IoQP`7Iq<3YilA*0z@wQJ< zB9!Qf*fAnb4OE|e4=_L7`vgsrae7`vk_=#*+u}IJtIQ1e;8$s=9~uzecEe+dq|>tA zLg(w?SuCBs$FXRcC@IsztvYJxoBvpk%wQnWlMO3p z5z2NJ%mSJAU(rJTf>gu9hmkT`bJT*Otw43>q_n08h>*b2+CmDtq7q;qlJkxg4MZ0* zNEkx+u?+@wZ;mq<@~I<7JNu9DVxqZ?rr-a>$(O5c)uxZ*I=N?bNSHfRNFSzKVCbbl zz>qCa8kkQ`>zIWx1Z*;NVAIPi%w9ODUnTqfZJA|mE4LwB?ejCXX`u~&uTOBo4CsPO zuZh6&G!pN1)|N9^HKSuxRxBF@#uV{(xvgj?^4r$ZW8V#okc&bJ`h+L4U0E7|l%UqH zP%q8=XhDTk8YbI>44bh7g!v-XDH8T2T33ob-JChCU@Lq^n#tMlS9Ju4rJrlNAN_Wm zB!_NEtmH{EPmBdeA>2+EOxd_!!L8Lo)IL;fRV_i@+ifU9gW~>yO7ACV z6{v261Tr^>*%b)X%TN0r6osi+mmNp3r*b^uc`Rp?m}DQ@Cb?k!@; zGDoq=%|BVMUhBkf3;8t@#_I|w6U~1r;Ki0oEm6?3qRiSXBG66#wU8B2__;g79%qyI z8-+z}nQ~IQ3+hFNV9C)=0}*5y8v3~{>xZTAC-x7-jBcXZl=1ZLcn@&zfkjHDC^-53 z+VFV`^`grs%XWrajWOM$S=X_$Lg6K%@*V;4Rn!wHicSsrV=IXQL$vGl--NZcS%|2B zTE}lS54VY@KGs}mF)Jv@aOgxH?VJbZ8K-`|5Z`aPz)f~T8NedW=hQC0P;}0XpGLAr z9m!WC!#<2I%Y3YjVEJ78d=x0p8fxHQdO9m*R@yXqJngPcyS9L}cJ9y~p<13-_D_rl zpf=-QcE#Jae2thsJJy^6Y;8(j+5W|_$7QUm32R(mzF&OWY)^1Gtv`WQwY{8PkflVX`A>-r%6n8$`3m=he*Tv>3jf_d zW0#g^+nFBD@C2B*06g}7*2s|BuuQ+L9C(^aHjwrPbQ`)1XS)BZ9lF6G6B)jAeWsW_ zF%wCSRez`!PAML^)Ikvxr#jW|d0EqQR{o0flBP1AJ=3fUMFzN9DVe#-)F0noZo6tf zsdOGjE>y*@p8DhcayuH1{QfGbQN=Fv!=v`)XB5CnXr<2r9wfj+#o?KXXr#n>+3yHccygj5i^$;~n_rKmO2fgC?4awe6mERz4x zcNR4g=>X#1K%8zF*%xQ@?SUg1YNTjD!y%Gk`6UMCD5w`P8eSq8f)n^>%snfb`Q)Yb zd<_uk#YtdUHUhQE7T6BXcepK2V7GC+!1LjF08l8K1ilq@(@|5+)d_1}kDDDc4n2jd zimV6`&(4005mR{5fUCUMmoS9zA(U=fvS_EJ)3b60#8qB(@oa}}yU)b$c%@Z%bd~eq z=Umdo&~Q0MIa6Qz(Yhfa6Qd09#8_43@9NoKvVAJx9gd$y&J;2e5Q7!|TctQq zT{>)_5X8LWmIqQC(HcX$#ZK!v#HgJy>&z)C9RA26ikEZN%55Ubf^{ySqIg=W@;$v0 zocA2twd>?yc>nzN+vfmGBmstPi{aRhCnr_AQ-tOO8if3o^RiKph(U(_GOh6Rxz8x< zgjik)MS;8-uf$_iUu(Siv7;uZJgDTUVX4W(2*U2kTw;=! z&AKR)_?GJha1{-@FLKl^EJrvwzW`l&8*gtHi5p}E6I0t_)Xw=j^@c7iy*2QhW^@X8 zsA6li>;pYe#oj)DONdAaYVjrb?{$`}ej!LEY8vj#_|c5tF~D9@40If8e2K}z+ESmfxFFU+LVO;8p%EzIW3OaS6wJ18|5+CdU%mG&w7tpr&_QQ; z$;DN&CO5t)R`(qdI*HadJhJrW8n6eJkv_l%LwiSWoVxF9elfwv6th|?p9A4pd46!8 z8kFGeQU9Lv?(Rv-Jq{))rYT9%%h3jTC|U6DYV@duEf%X5?K`Q%EFf`dSkHKg+q+dX zt9}TPX`4waI8(@N2(tx3HT|>h6d%^RdpLYu`EQPshFge=X1nPS9c zy3ar9leB1D^8gXvG*KQ9sR6L!RMvl?ihSkSSY>JkgY{J7+R?nVXk~?AZi~i6jK^hP zh^?8VC6?~Jg$JaVeEEhM?KT(>MhWWbMY4|3h+s^z_M;gvWD)W__Q8mZ71g3ltt>-W z=`K&|PMEN_zHIqco#9ZhEBOh(Zgg6wu`j&WVTgnvh@>>3(xzu+;=lt^|4EyFdrMyi z(4A$SsFfXgP@hdh=m#uGEI7AVJy&OdWU^&)0-zQOLZYFTRy*DGW&kvFl`#nm2Awj| zf0E?_EN%A~@29FagH3vMOZ-C&Q9p#Kl8g>FJ4iRP9B(yMU!bid#>`4CAt7yqi;oy+0ae#(=2AuS%ODw z3tc^@Xlv64{9LZhFct|lnkz)_TRo-uXRuL?Q?;juCQi+@&&o+!o#}XS!X}CigN7;D zMDoVVr=#QvJ>tSdXh%LBd&E1W872do1x*TuYWyoAS^+f6>GtBW};d7V@%|%{EqoSgd(P2*8NvzOe zLBELC)wTkod9Z0*GYT_lOca^mOE#OO2gTMChAv8t92O*H&!q?1TD!=7VYDOlR@LTz z`>`h2RNGUb{1AznR%JD&(B~Hkm+hD`#&*oUgoneFJLqD|96o;YdICp9c;V<~?tiih z;LkOaAJOoi?$1U()}oGfr_xSmXL0-w4E3haVvbW$*c1SA0mmr+m}S_?KcL81LZing zJKyX2pI~LpTs}Fpl6PJn!2o*M-1%n%d2pp_vP5YZD+g@FOW6 z*oV(VK-#J&<-=1_&lkBZP3ODNpFVhT$aA&nR5+W{r$+7PQ+fqXi;6R`HOJ3mjWUbX zjW-ZpDqzuLnAo-KNkb`iF&a7W2`&bm86l$b1NrfZLIYC60_U@D>0|kV_^h4Ls_MqV zFO);zUbd2~wE7=kVu^R&TL@(v3#lssxeGv68n|rPG~(oI=fC?G2yTy)y*r}=zA?{@ zLVavevfi>!OMAb|4rzM2fak7$tF(-J5A<^E)?*<3v~ve2448X;uo!srW@m{v+q52F zv6>rA1LlGx6b5&43y~0!!6#uo)S#LVWd+heQQO?~PA~}Kh1b2j^|gy`RqamqnM=5P z`PCkHVo43j$5LJWWOD*Mq&zuxexzQuhdth}`C`-mbm-TGAZAN^)`n-UysR$ol9K0_ zGUEo;*Rkr*3B>8x>gLO ze_qjLYy6vuoVA8*ZZq%i+kMLUgJ@)o{(7%4TyMFC|sLx8-Qfwd1Fjt@z0yGhz= zcIP!-e8KoD-0`eUtkIGA9h#ulCQg*h)3~t{<%$QI%lH^*(5z`E@qr^6Pkg zy?t;0Q1|MQ8aCt`TMX@EwzOZ3Nv>M>%n;BtNog!~c71(7P-QCxlT_}Z)9ec!f>*yD zMr||*LLRmoUrrJ8DMI(m@`R?zA-vSA1)Koz@pVEUCHSO$&we|`=sno7^dqeZ#~tR3 zbduQ1hvvUiajQlR5vA8X(8LnwH8t3U+=gZX#~VP`yGVep?bBKw-Epi3V@~4)oT@XX zJIj-FzFkk3*$}yGT6K`E4{`%N$u8rHClc-98O)bOgc&}fja~*hCj1!|&wgI}01AV% z;Iecvm}Sc0)s1e+B)WmG)Li`CO~R#v&q@(VI4odUocadNpg-AHkJd+fb~DR`sZuGI z8g2w1eYKyD$O3SZ1%iA7_@3-9)%QQ6Lm_Jfof)5$MD3jksv%mch+oG?4>srlCVAF8 zLH#||u%9c;5&Vt3l$x@tnGun}_+SD{b{TR>PI*%ioo1XppyQ54+O^=AG=V0>B*xM0 zMjrIEv;*#me{Mhmuz;^L^EKSOYLlhjI8{=zHz(-#)?~WQkGsBhKoM>XhEg&BPjbLM z`U%Jru>ORRPy>&rUE}GEdY5FC3C$u5y;PO)8l-!J0=#~Y|D!Sy;%oqAvAz%)&>xir zG(hm*F(I=u@8-9~I&Xal<O zXuZl3?7Tg{GSgG|3^e>_{wyLcfKemEQRm5gQ{oA5D=s&Ob$>sXP|QbODwL z%FPhn$B%+wprPy3yRA3D;|~y<6|mqaSY5SY#?7Gb2Gm%+i$FM%3dAO62PZ9`AI>rE zgkUcEbg-rqpfxgXc=>y|2hi+z4^nw*6s2cRtw}t-k%YJ)-X}FkpL)NA_SLA2zcYD| zl#6P;a6gE%1{WR}C$QZZME_F=gL6=auVt5ihL;95SORO861n9Ed?m|OtwkoIKDMh! zJe`rNx3C~T^Di}lcG^y5KLDw&z{;aRPbg99eM-CDk&sR<_iCVKXhzo9wYX_#vo~nP zKdyy+;kYF%j0foAJj_Eb0C6)WgC&Lmph+Vw1E?@Fln@296RS}^T)^N<2=WDNm6-`M z!I? zg#FPEpc?K^1+oC!a1L!xm`BS%#En~|S>p!)7v8X`?4%U#6ePp%71t(wc`-k@VYRY% z(a;KF1y3)SV9SPToQo&Y3}lWXS!8s#7gxQXxAL+OtdBG*%J8b&Qmz!r%CZUq4dB*+ z9Kuq+(}4bHGuM^LEt+)n%b54jf>J=1zg~aOi%Vl~lrIKpdP={<(1~#3aCjh5 z&}m*v!*sC1bcV5VjZZ{-_BPi`WRCU2NQs~(bd;eh7i>&HAq|q@jHyFQ^oj--PqzWe z`if)-t?bq$mNO$bNk-&0fz=T=bcu<>Ov%iu|9sm*>oWRxd7u>&#XoAuO_~o3UThotxSZq7(RFU@XkOHhQTqzI!Y(dJK>t?jPJoZUYgk4 zaatYG5Kf$|V%>LDo&?rlvJtKv>#*9>)p+JuoJt$`V<%n)CZ|6>~uw(&Na~het1I&;7P;FeE5-WUlHlr9i zelg0~@-y1=TuFt{XjN*gLI4Xvj*hc7VI-f8U9RH?Zx`OOUq={B-g-@ z;vOBVI9TaZbCb$`!w6|d?Iyu1{&r+4Q2_XLzT0cUJR_h~@Bm^@cx*<$=A@ArWXOY$ zsr?@&41mqu%91!+<& ze0SP1#HszLt~mxbrt=@)j;!jB6Y%tt!g-Do;y<-MgpP$NYu`8A&~aruwG*ohxJTJ_ zZ}q0bO%C_cYf1Kxn9*^$fcg*1WbYx+Zmp0@^kvtJT*H8Fw|FbOZie#WX1ikIkKsX9 z9cW~zau!8zVm!g)QINGXSLVDD4c{AqfnuZR(meXq~068 zg>)c7x+;G(3llQI_N)Mr|;GBTVS(X-6 z0ncP@g9S$q&GL&3D+8+=>Z0qO?gAgCJv{Z8JO`5sCzcM#~CROH_c)n!s#!6_=b~+49x4 z__$Ie&}|q3^uO)AT5zdS*m`{NrPCYUNFndi(k+(}BnnddH2f=_c@O6=e#T0!fi{!I z#HeTa5JIipE=h$drmgvN#yvoX+6+s)jKtavr9$;h|57m zE0*#`p8;6LqT!4&I`Nqc{3>fwhc14V5u4dDJ7~QJ+?}zCP7bPbcVL$lOle$_ROU3?wIE;UO~kLl{>Zje1&cE)Fh{Lz zZKVjk7ASq=03#v5XQ=cx=B6HjLXu|@@d3h1A~d}Y4@iccb-y$tI03qu&-3v-3qVt< z_1-eS7p?}vk-^ve-hnI{AG){3gB?EuuTBKIwwaSX-{!pKiaz)^-NviZqNg5ZGqa~~ zqQR0tYbX=ir?qPN-K+YyLub-_f*|EE=jR#76LtJ`(i%^9&M^HiF%8O}>WJ(dxU_6; zk<#AN6EJjO1~zG*-G~onBOqzDkO*w~3&rG~6{9|MN@j#`x-c%XPpO5C4x-&pzq%B8 z%p%6_=oUNU#tDuZxB*xR?qR1ylq?{tnX3?5X+78Ad1eE2pJ?OV7|yUhRw#H-#tiP#|Gy zVgCR*fM9zVGHgD#kEJS(g1tNBj%eQ&u0ldh1O!9+w^`0i{og*%uxPrx7|mptyof36 zh6W<_SsfRf+1Qc@aA>KZxdjRQe($u0Q{D7rFx$)}@ep$MCYQPTh+IK)hPd9PlPYIU zfj-3Bl(LE%HJWV90vKBq@wz(3;#Ut_pYVG9Go<$>oV&=T+JDo=HhNLUJ4diX!_c!R z;(H=3h`j-<` z8s8-;Qba=pPkCe`0>bp|p$M5hie}Mgy7aY`uc6NBA){WdqvNY0m2H{0W)qh^lZ%|_%-`=)f4S=uK=_3vN^=90@aFuJh7pEOaDc>-52RA# z=O;`ZiP{x!dg4;()vRRn_b9Vr=@Gsg!)ZCZY<3iT^SyJmerELHCK(KTZt~PUIw4rr z68ngd*SD0oo}!4j$2^pY308uN0qTko1a$U-3zDN!QbhGhr!z zk6X(vcM0FXuGHMLJ37OAPBHIQG>+Oz;JjdkP+m3kO2&%?Xo2- zLvh+9naOihiuH_TP!2yFInycwuA1hX#N^z!MJ7beFC+@jOrJ!mg1_B2Kyc!=uSOws+hW)(Bt>QkTlkR*<)=)=@$fA? zA}Cvmd)0hd&rhdMFanM!=UKAkrmIDMYb#Sg)Zgohk@|OLUTIkf2YT7k>FMv70C?>| zJvGYO%#RH?vnCtTIjw*@?5aZc4OjZ|J`E}Er^%5OkV377WwnAI1#LCNu1YvUXy*R52DeWeR6r8lpV> zO5s&haYo$a@493&q7S;A2+D%i2i`5@{I+V`;9KxhtXJps1A>a5g+Wv9Nz1j^D7raY zJrS~hF0i6LiZ5_b$k3@W(0!<*fyW|Rje%Ks5<17#zFi#r)>8*?mPV3LW#H-4QvV{( zaJ?JeL!P+9Bx7v1x&QZXCHY>&3+q#s%oYbKn>|4$2*94L{_ySfum8<^tXKyOr( z4d6YXuM%o*{$pFF`OnGt;I%qbzXlW?lxrZVoEyI)Ert?2rJxCg8jmVPncvq?Wii7=i`<Bygk+BUxiAs3vPd7Axj2fRni-@Cm6{?$YQA02ez zu7IA%1lIAZw0~r$HHp2jPftk!D%)DUztW~4;n1BgFt*GnMat4-aG7HV$(vtSf2IE0 z0v-#)`(gGeyUhQPmQ%7X+P1;98P=wcl?UTR>TW1BKb@uLJZm{eKr@{pKOV)Q36X-K z5CucR-(NZu=D2S(sNdPa*1OA8UmqC0oxTwDD@dI6k_Lc)D27Dho zQNiPX6;k5+ z7jSA?iBxUGF_;ZqD!aUZIQjfS_K*GNabB10^r7`mk$;250A&YIV<}=B<4PBGuTjz2 zW+pU>OAM|mu4MX8M`Ke=e$e^XI=^#25F|qmlH^$VFTRU0KmnWD=i2=_$m|JG!;~FH z2dsMJBvD}LTdKDZ0``p$P5qc~oS8&o?GW{jww6y;L zIFS!}QPcv%sQOC){;q;E)byKQS1jdaIOqc$RJwo-pO7@*uf+h+f`aAy4VoA)D3?6o zu^`9i4!f>PduJy@DpKC%4O+E!-r>RFa2C`4`ILZX=O^Zg(mts>i$bHFzaj*y<0U`V z1cZX6-#$2HXYwa00RxJ-hUJC8bB0culL8T3q7=;s=RbpdlV!4P_5d|l<9>=%o>jd~ zAcLMDXQaZX$6)-bufN#j(Q3JrJO=HIPN;$z4x-hI!agu_zU@=|y-F2;fd0*0w`J8C z&^-y2M~fJ;dF4-`#>7Z@Xnvn2-g3a}NyPX2Z==r5Pgdt$0EZV#So9W!zz3Gf2fqoA zFCWnPb<$C3?(Kj&P6`}EPmMvmpt1PC8P3%13*P;X#+Uc1Fv~hieKuQT^*sM}ZtDZMXEl~dG~!lEgwk#i|kb^P}?#V zkt4WV`(xW;dAS-zN#4EkpE*lsvh_9udOaOkb-e!Lf_0?$f0nxQ%VWi0n z@*23eDxjqN=Qfn*C4>TU9%gW)T2uO@Jd=I<&g0%)8vEeCc%=~Ag<%AVAWpujg4oh~ z`OpMs;gk?*OwuGywBiCVf5)jstWq3gY83XnQGj>uh_V+j#4L5y<@OQ)AaG~;;}wqY z@6RysyUJO?&fi_TQUJ`0sHZRW(_!%EMKS&B4>mF$dqZAQy}5u~i|jK?6aG+Y9weS9 z8AshenglIrpO};EqD9SycI_RM)E=9ssVN`2$+;aPML?+Ay${&Mq&)EnOSbojybkX- zovWqD|G@3}6?joQB{%(Ckalk30udVi}xEBN`8VjL*O<@ebte9x@x z`A-M%Bi|S=2XMG|V4(w(7uf3qS5ySZNu>3EqE2$Ht`5BGw}5`KB3_;6^WD7 z>-8U{$rLo!yHW%TKS~A_!D-?fP=ib+{rIUsb_f8eL>mnC8bVn+h5?pMMoHA?9=j zeRK%=o=S)7Vu@rz6+$|!wlYrwh?utIXW9D5*q81PRI(X1NG*4&8FmFy2HwAbgVLKpZ|5VAl){QOl#lNG|;n3|@CDR3Ti4+Y72~>--$BYz< zZ>|gcJ8xK81EtQ18dw6@SR>_6Sm)>@ebtCJ0evfl%oq@ku>`E4eQgIMUst|;byQC8LU1wr`=-*u`4epd z*EIo(!5#4{mGQ&3{RN9<_{cobfLsp5nzAY5DQxPy1ew2mcH>o347~~&m@jwY(Jck= zUr&mh48OVjwIeEGmLm9`W9I>$zMAD)OuoP!8G2n$hLkVzSmt79|O?u$y(+ zF29$Kb2tNqwj+V*RP)7xU=dEjxcrB^H2H>@WqX}+NY?3L`6mGDJL!3QQ{KCu`2Y7? zzqB|cQadM!G8hfqvXfFG5;!F5ngqDLCi^zW_4;YwlMc8k{hkQgjL()tm%Xfc1`=oo zY_loF3Ur1~^RW$DsilKnQOIJ(smz`No#|bl;M_^ka$1W|THFncKV*!GJUs6St&a-l ztO+cr1?Hrcwk9!rt8-cje*L5QqavAqqQeVUs?~5v@k_SvshJfe)@KX$j?<&sB_|JS z3Kv0>%fXBdW^=Tg2iB>gDD2m$yI~*t)#`55hJ&Rv)nrGMJmkgZ|(xg z52y3&f?uvvDGL#v;_XG^t+?~IqcbI`OACT5@rrFg-$`TfKIqx!hUR5jnM*GnJh|{q z@&gkT z?iuXY;UtxOR-OFg4POL&NjnWR{?9O|_7Xo0TC7>7JodBskcX|XI~8I!UYYY!Rq4lTXo0f1kceXsk#8Zg!k@j^ zwj~h;j48e>{{>GC3TuVJUgOB*H6C3p3*wrQ)Qir?yG(!v&dF47)rw{wtfh;TUF(m< zL7GWS0<+iMJIV?o%-7V{Y>n3MJy0~RiP`kCC^Y8?FBp7VJka+RNQQTxT^ov&$5dln z6JzpHGTulGlZ0~c;+&OGb2QsQrQ|$#qpCzVz@@`qn=2c6Fb~;+AP8_+jE%+;%3x4Y za;4SMwNhkKNIcTp>(Jks19Z&|r;2UfYQ z9coTIeDfrMf~23LIz834Xooy%G~q7!`goN_7tis|Q;NEO z$3ajncRDFBW?Ko4qliI`bdB$mJ_PHlQ8b80dhQ0g+zwhU%D*y-C-0rov9~C-w{aZz zHp0(q@A=lVS)GM*;AeIJXeG%|P;v)^VZ;wROvw>ZzJfN9)Jxt19y?b1o53^9;}yxz zK8z~d_gJSFyqK~(1|4KG`*hKwIl;)O=TWP-ZM4@Yb=nWudS=50$qcct{gi`7l_9@I zkbe}eaM|`dr(3n}9FhE9kcDp7!&}IS>EyQ)a~ zE$lAibJv_eVhsg$C3zq3BTbAoL$beI23=J!(7@$tr6UXjmH!>qMOif=RL7Cm&Y9TV8KyDXKjjG#@b_q);jg!Gz)qx*vwF3eT-cQ+a200pqu#9bo35B<-)m_) zRC~ZOsaYS3o9$HMTJGsRs_q=S?uy|@brE2L%R}Qbb5}SGuPn~+6Eav_plbRT$*^$^ zNqJb0Z0)znrRin)ZsTe=(gfzgypP0T zuHo+EXL0kE=*MNQKom6Uk(l>Z@8;HgC#FxoL}wI7JWQ@bE9gRIj5A|1QeV(cx)UCt#Dy^h3m4Vqiqv0qR?Uh@D#k?ti-pUfa_iFQ zjJ6xaJO3@#_ytK;2Wc1!+kqYQM%EBc30qDy=@Of-n6e`M7H?V|F@|T-ztpC3YXyBv zPoGTowGAU)T=|A)CBJF2wv(pQIjSs}dV97&hA>gR=-zb}q`Iihm``Yf8i|D5yj{h- z3Nhs#u&9?TaM_31ZNnXobo!X}sU8t|(hF%<`H3orH9~7u)o~oGlm_b(|0=HGTOUe{ z%O_AEGrUYZ0#DXmQw$eg8$qMcRniH^!Z4=ToAR0qlLH+W8bs-&$HFU;ksr- zpw^n)MKtup{&pYgavb4$`&K2}_7t}6hSz(_p+GI$h>O~ep=?9i7p_ninkw1D`!Nbn z%E^lG+GBY-TPhs^^SD|<{`YSlTy^6`a*|WdR64)SnK!1_VEM52zey*_(mgrgzth;g%P`?v1VUOLm|{{Dja ziE1W5J>)U3`ISK;X}>H9RDxzhWb}NQzjJ_^?TAt$lPD~OXe8{;v2G;fO-)?1TB3kw z-)dz4l%vGXS!RDc_i>e{pq|BL-Nbv%c2q^=OM|)~D$IF;*gUHDVb;2z)Kq_Q`Dm}t zewc)r9{E$rm&zj%gk*V%td{H%XEUY+RY1m_F69G*eSDVj%SzkN&mkycF%#E2?fUP- zj_;i5HQk${MJcmmQ%TQSBYVoY=%qSeTu6oL%LG|up+mbqlPJk9Xc0yPCx9-`5rx|) zih60Pj;=CSJ;`2i3X-T)ZS35rN1BHOeGJU&7ORT=zzMrzzbwex^S#PTYB!vLYzrSu zV1$n{Fw9)2y0?x#-+%ff8zMqy*Sj&1DvORg{uJ5u~l+ z9JRam_w9jIfA?h{>|2i8g5C%HD`E zEMQM`bV`qb;4E#X5x44l7p}|o$3xYQmQCT9+tqn^fJY`%2GzGyO&(X#*Dc@m1>GQ^ z>mVoj%i@%T9X@~OtSx2H0i9iEP=eUaNIFI1VIakE8Yha(MEN_52klk_BgRJ1SGCO# z%B&VuYP&ib#_}+E=+MHp^Edr_O0$c2vdd=Up<(-3Lo0goSaqLxx~pN-r=WL&Tot{3Hzst{n!fkku`zY zLRW#-CWndd9epLr;Sro`0gS@8@63-0n-p`#kIRKV_IbE3eNpUT*7P_qX42~u)6Ku$ zHDn#+EvaPhX+gTPNqowWlqr}}-utuy@iTxQr*1U|pQF}gSk})bH}Mo)%IkGKwzT!5*IW$uMAnLxq2h8wv=C~Ca|Y?1oU{g7Pd zW<9&b;u#MWwNmBBvKmj$TJtvj%X(?L7( z4laq48sy5&$sSc0QBRUO9NDW{GVHj>2_N)evPK*E9!OlGe@`=EE%{7++`=8-?#HHH zs{Zq$XiW$zRmUojLS?$a6L??74DJE?owMIe=nTrKQ!Do_i<(SM+UBmFA&#lG4fP9- z);vNk4J#h=>gb|eize-*HM*ugC!tELL6BpF4#+2pLoFmDNiA@oQ~9hK`a7P^FVxzz zQoYh^&Z|%JGaH*iIbS}P+`iWugLQ~b;jwBmdGCz{*CM&H?+cekVU6unBO@Ek#t%$7 z&=mWn!ZrM%2J8Jb$I6@^iP6B?VB^sZ8aZ=5qfcy(ILh5b7Ui9avhnu5@e5aRAfkXa z;c*A`W_xK1^HPA5fIcIQPpp_wpJL; zoWLw&`c1Qld5eH7|FYS}dq6-0|?9?eY<+9jciFyMI351 zn`SgT^Be~J?_@z-)JfPM$F|_oPqMjc0UYkqvA~0p%vw{Cx6wI0Hd;)&Y?qy>{xO;F{reTheFb9Z6;P|E?^gs_ylDEV58VRc(AGMADY?+MIqOgI@hJvzmv9Tl>ChAP+h7HN`E-N%oiT zDGQTIps!SaxeO+sXvnYxXv#Hq4F`hwk+Qyd%9+o44&jyjc$iwH?o*n*Mv0rMRBK&*dD# z$=gMjRhEIV5hl#A-D&al*-3v)0}dPe1tlz_K>hcPd+Uxo1+@x;d3BfA6dl@D@7n0o zGBgW)i9~Ieq5?Me9(HUQ*%KulCU75fM28DQxk1K1(3zvmK9o--XfUB$RRhKl=5&#@ z9sPJWCEtD3nMbMSvCq@?)cENaNq-l^$wb&yD6BP&B%<&0QuOfEVbx`qXy%B?os{yO3NBG zDv!GVNo{CC#{L`Iv^N8GIOZ9uN#)dK%!_))MQhGWr=iTrv=6>XHx$u@KlE`iWt-7b zF74VS$Nex}60WsA zKoWFInMr=IsB3E@BA9N=O_z+oD@t%WzMR1Q{c}C|1^S{Wb5McyO#97^ez*rS+Tw6x z(vn=0B9Wj@dn%3eA@u9_ngVsZY_GKCcfGoy;!XHC46PQ~)C(xuJhgZB+Xe{J;f7gG zxj#;%la9b3RgsQHXk-I7S;tQW!Nk$Sxg5Vl6f)|0y0-$m9O%+I4&^Igdg2K@SV{wO zKO$H5wiCDUYJ_r6>&0nWudKE*!~;zcjPDeK`akm?-O*N77XCy|(qGklwi&5h%H`A>U)}2~oaZxo$e4dZ(8xhe!^av!YG8fX{zgAb#_bFJ_oGeW=oaiyT!OV-dQ%v1rI;ygFHi>IhqY;TH>$Tg zY=*s>Z+)+F%wU($z+5=nhi#3UO9%~#W0%2_Sn#5!5Wt1fDsSQ-6kh(~_(`-BGuHJ^ zK&(SL?Qcld!p=7-37KG0pdbm9LYYzIY$MG!^45Z2-Bq}1kA5)^dtS(2XIbC|b_y!G z{Y^H#Dq2Zs*^pRlQhBYlfU@?I+;4v5mb=(|_>RVHw07%(Kb38tLF-dxwsCskz+>mm z-gu&%abqEp*>-~}xcjMCKdmRT1pnG@c>O=3zA_-H zKns(OL3(Hjl^7c7?#@9{knUExdq7IML%NX$=|;L+TDn`Bz3zK&_aFa;x%bS8FV5-k z8yEtvZ8`JWNaUb=WG>VD{~sbsqnZTrFX)MiC8-5K@IybpyTe14fR%WRG8QiNA9{j$ zHE+rL7>9VFy7k@I5%ZC0zs2#xh!GkQ|KH;a1F&?@qbgpzMIWExTVDGq+ZM&%wGQo? z?r2>3)R#<=4$+y3Hzu?nU8N%aj`e6tMf|@<2K+%+Af2#Zr7yBU=>7lys~=)H;}vTF zY=ZY>Z^1J5f1&>Wd|Cd)p?S?Uf+ph@e{)-h0oP>YRZ9Lv_e09Y`DeGB^_Rcb%)gz^ zFJSSg=7O$%?d)YXDi82A?kn5TtA0MmM<+c2?Hr9$`SGE1YXu}*fT8T!Z@9?iMpsi~ zj&Kb6`oHwW5O5%mr=H`(9zY;)wEb!u%t&-m(xnxSy)@`3wHo=$Itc*Vr*=NgY@IFI zwbYk<&AfBr)exnwyY59Dfndar)bket17GdCw7$H1w{tekEZ*r|iAu_`nj^#YM~QLl z(<78unGIXFU3Oxr|BWDFAwkPqwY=}_5`>hf*kn0X%1~?PyL3vwFNRJ{BqH?dWa9KA z@B2fv0`I=-W=^2sb1^OYd{Wg%rWa!;a>z0e z_9}M~?5dNj1Y#3qw?xdv{HJPKO@ul67@f2MGJJdU#U{^q*nPelJ4!P z1E~W-1+#Gw*J@f3=`F6k;!Jk*BbYA%?FT zvQrIB_1BBnaXoMaB}Doa_lIsTg|8Hvbd40$fY9;YUveRr`0M$&*gyW0Z;`KfgRMutiHxF|<48 zo@Z=GuPedpTHY;LuU_o;@mNuRHk)k6F2-q?$!e-PUZgDfV_0M$M96{EZ(si7W!JCW z#1bx7Ftv6cM=5t6goi<9g?*eK%NAZa?XUatQ^|uH_ss5cGQQ03ikOB{xBc|ftT@Tu zdaL_Ge2z#qN=RIE5o;T|$wp6BMDjlP#bs`IN>Nzdk@lxdUxcDcPn_v<$hO|YRmV2)9lwGCia-jY{y4yT=+|e0VYL(i82nHf$>K&ZnRPY9gu9{V8&58QvfC~U08iPrZ!Q{Y%M^w~^zy@bq zL$A7VHp}8sY`xALQmB1`&6Igyl9Lrw=V~cr2&?y`a&kg8qgXy~WbLt2R6k!{bB=4@0OzQ5ty&T?1M=TyN@0#{n}J$IYRDt| zGrACx@Vt7iau5$mX3;zHw+Rl0AE#1T6t@~DnrDGFDOR3Q?o8QlLWWf?`Ji%W{#GwfOvs~)lAiz`rX_u?(fIsbXc{f zY74NAJg<+i0GnVCfRHiv9R$2l_eed9 zP+olT^#n8xSUaHESZ(G<4M*M*AvXT_Oo7;+byxF&DuIYaAezH=GjyI@;TS^jxj_Fb zGdSBfIKZ6kTZgwjruUzY$8UE#IZ*2uURA_MGXuC5AZsuovL>45L2qdty(w zBnW689=D_`^1E|JK%NfbS>MD@x!NZuS5k9Lx1~$qcf1HGA?jYO@#?W#Z#!BwE)Ow3 zlcYltZoNXDx|J4czzp!ZD$9JH{co~slQa~k66-)!@T>0SgVcDkPzX5!oe~@yq9YlW zUuXEsL{POy`0VS~#1esMFl*JUdT)Fvrs?%Bq2YflZ~slmLUST_EIJCDnwT({jS~Sh zTE71gi2=@gO1!F?kut{^)j%fUJXFFS*FI-aqF?bt75&ng?=gSO$SujGqvxio<74mO zxM~(kE#KBR77f263Z1RTb`^k)8$gOEW{z>V*oqG$K z{n#;5u}MIsRNU{7@}_P`4YM_^OQ+fS{kBUp=0$Kdfy+LvQaxq!_ zQY1!(=sls7ZAwIs2(4rH#pIy%C=o-Zs9hdrBu4`v>o`Q&IvKMLzDZ}zE*!}LsuW?v z_-~ILfcV%Y;9PirlLJg;DnRUC^hl7j4<4~5Yc*r{nz=$pQthVQ$y9URl~9On|Fp?=ZLgwL@&wU z%-aKJ&W3mAwBcU2BSZ61D!SKa^M2h7#aB`D*CERgSmp@;Ez;lX`dXNkm_Z@|f#CV! zTJ`DKdC+gBGhzR5gq!fWi>;>Xba-lKcz2SUBYtBB_^nMEd1SO^!cRI zJwaV*gEefZrvNByo^%T4<6>%ftBL(QZBiMb3E6OhIv<0-!k zO?RD&d?C4l6WSzrJSLJ|KGY&I%=^q5DW0TEq4S+yEzSD=KyxEo8~H1!Pn7DM^;fgJ z|4&tqp6O6S<($o-D0`QKNHYqPR#%#0M2le$_@NVHBCu{&Lnb3??Y(0?Rd`J*=*>7foUr}1@Z7! zc;zRlb{62-Fiv1JFxolUxHX>D_6*IZM5z9?uoXb15%`di{%7`jWSTH=Ah8f=Z#lC6b~~Mr_l2e@Jm}k0qc01KHN&}XJgvo$j+%5)}D@8Z=~~w z16}w+{b%u;lpPy9HUFFu6ovuUU9o$h-#63Jf0R4{zw(e-(-(yirW`F=x84lxe{+$L z{~9_Iq2#TG+{bfa{}D%k`tOy=@L&3$GGi;2?S6fEBF&H3zx>EL<+4xFtE+q;8l4o9 zS$v=8%S>NJ3eC|y!>Qbg+@0=rZ-W zESt4@zkEL`{DC>lzMe;+WU)Gk%O!}#Y&Rbecyv0Y)d53EV!&*6J^2IH;gWAeujW(o zctw-*b(?qpfz*RAw6d$&mqU7N+4{{Ifqb1Bj2oAvGiTzjTO&6%J%XHyt{D>^h?yzo z&fl!T!P*8ty+M}Ha=r-YNDeNC9F94hh)(QpYvT}Z%7EGzb zyDhR%5s~J`(==cz8Gl-607qx?&ynbp;*ndD;t}ZR)+3m`B~iO&Z~5jnjJ5&z&V$C+r_Dx<(0JvaLxd1SX!<`(MY&=2w?d+`ehDoxgz@3H8HHKf`2N zT!r{YOoEQsrLC#Ot6An1UQMJ)KqP|k_QD0^{pGm(m&IDk3>00NxY6(-tlud?b&P8A zNkY^5V^OPz>P==-BG=#NHyqKF%P35$b#fH?e9w+x5er(za%?Ss%y!frpkJ`NZ;1?7 zI;F*G0=j!%Nuuj&-(PX(_;e3Bll6ot-wyzOPD6!2#Co;Cq?5u|9d6_^&>P)DUSoI{ zu&vbnUf#>SJ;YV^N8fFK-1+`;hc$NVEt?HZqwc{H)j8Ip8hTy*_uT{f@*bb1JZ+E% zczMdWn+`2_@aqC&GvsCJL@g%lAcXo^zQZ^XZfahOR}=xcbB_X z{yJzn5_phoCb$D4N|&91zyU@>+{%eCZ@DEx>z-0cDVA6JI~TxRtzTbLV%MVQO%Y07 zfAH=bf%XiAsbBQpEPX=zDZ5%MW4w-$ z&szOetpQgN*M7T7iAGsN+5mT#%jus6N3=$WGtvm3Z{JY`>BM#v zKyWaG!Nw18@sTu~s!q`CG&x^swVx!Iwx8W#ikUGRzA*L;8165Ni4Ed`s;VYMIkXhQ zDnS?Zz{0_N)iRW2iOKhABWnGbp+Hg9bOsNS`OFulAn5Ssi|a|&KY8IRM1w2~d;E6j zyF24w?=9iWE*T=E#x~0Ol!z~7rzS#T+QO7^wv}LSb8XK`V(h2Pw{fw+d#e&T25Ze@ z%cdJueV&6Se}u=*E0S5Bpj(3|b;A}!#`WS4W@9ke*Mq~NB9pkoh-$MEP8aTQ)Dun% za4^1X=b^SRK-2~{2?BB(P&Q}EdMAT~8U%?8uYMeX(|HNc5L+ox6*;kQE3sPPE`5)j z7v8_ZTrs_aw7$G9zdJ+MwO#B9r2{4&MJLJyf=1WjSN{Ill}ZAOAEn;A489(I^jjYU z#>8HpnPg3#`RA#GF&KeRJvBL&I?t0)L^%$^qfZ`^;@!=D39cD1;g)CQh>+YPl6?Q3 z+VXiZu;?;#&W`sHzEbp*eycra!n*}?b+Bw&*obF!wqWFHvqIe!_Zd>9aKAMahP9y+^Od4vJ zYG1KDXy{M{{3RuA-{dDSyAWSR`eT*e-7H_MXj~$LcPO#!K9`R%VS_|F_H!nVIiazK zv0@+3g=_X;KD9lOlHELRYY2;c`R(u5u(t~BBpG(9#&*cC=H3zyDC5)UfNqlLg#mRc z3ih6mG6dYE22VVI0^c;{Ty?%_0H9`qZ^F@K5N5;$f5t*`L9RTHsqh-52!_#cB4@T28Ls0Nzo;Y%Nkt9&Iu4p+R|N~?fQegCdy9iI|S@S?=LuZQ0iz!PdFk7JqGom3T;k9FL>ncS=G2832osZ~|ySCd7Jc zC#@~M%7&MQ=sc9!IUfFg0!ejwPxXKtsD8M$F+2jx(AQqF)M#-m%RZnNwH`G0tg%Cq zt!;+}u>(hU^eX7JIf=6qD!qYpR8h)Ow_k!1lv&PNH~ktbGxcQ13iuFw4EgWX1d=Jg z2oXd^g{+6DkT9Ye7LtGH%`;Ak#wmv>AMgQS{?1)4kGegh#BwE7gtA%Z0e2P9un zBH6gSE-xhI6|omwIwCpq(JhWEvz|$iPI_NaUYrfT#IBv9Cc;(f_nPU7CBXFa z#})REWx0d7`Rb%;!J_>;%XQyN9h7&CA|PPH*Lf2IR|&+H+tpk%U2KzZqM2dN0O3Zq zD9JR3zIKWPO5D1Yu$rDjx_G?1#FHf*QFmc~z0C#n1)WFjo<+fkQ~yS(hA~-{avOmp zXh#VT;yjcx%hxz56wUgJ9bR0&(PR;QY+%jDvK^cN>SUlZ>_f`FOU?Z~vLZ1k0|;== zW)zT+?`N`F^dI4ufU+-&*lEUHlJ1{Gl;I=T}4g0;Up5IR2I02y1; zOtS?{>4PY0MbwW}=H7~I+`6!f+}t3`5YE}?eKrq>>)T#pWW;yPi&W-N5Nl@t)tCIHM!Eb)4ek=Zi zWk)Gf9=FMbTf5P5xK*4*!n*0&M>Fn+TrasEcghnXk*Ub;n(NPNOP}HY z4GX_eDec1&IymhLm_G%5g+ESYj@|F0SyBqmXjVYQi{t9PR~-|5dLFf02Y8H6Y#$I( zNpaKpP15zP!nqHzd$^~4KP0J9(Galxsu{#Uzy0{9Ruw{}1_7Vtk$T?jJN)Xcft8|c zXXLUW(3fQs&%VmAK;NQJp>oQFx}Own7O0WDGcv`P^^2_wr8q+LGsLPu*r@$-HrXOM zZ+moNZZLy!Zy5%4+SFI7&VYbv)@!zQrFcWl(lNS6{TQ`P)nOM3f5J1T%B4`HC? zk%IKfzwIczZ6#9~AuLT=7+azdi)bF9+G$NSBgN~NM{z8LF1Fddi~cdt-k2hq)1sv; zdqhZd{6Mv)1pmD{PGeuLt{1?3Q|9^OvN88S0X#ZVa!(s&6A!0{8xC~!SK17_k>8@2 z764xPxnW>zA9}i^cn{9Y0x9b=w0!pJfG(Krtrw791l}UEsZ&lUN+hFvKe{!D_Z4I=@W^W zRzQskE-M$<{Ci2=A5>5MgEUjvXC=(@e z545$PFhwx{z#8p`Wb{AtJ&9}RXV!3V>BCpzX%&07g$wgY(jg+}QASJ2*_Kpon4F%r zs3vYTH_^I@&Dq?+hGL`jYina%>^<+b!}Z}FO{dj1nd`+W-FFl34!Gn6z8K1P;JI0A zuk01-h$Cij^w%FclB<_``XdPu93Z+535=!q5Ae-T2XR@GWhA7^Dzf$D-FfmQuN1n` z@1aQ&EQP&7C`>jbC4F(Czd|iWye6 zuyqoVSR;%-{^kGVh>sb^!GZf{$7^Fsv$}B@99w!n%kohf0y;IT6jgmkkZK%@+LC;%V(n8lUWE$F7>!cV%4F%?l+OTNtj9d*2w-p6y>bH3KVt*)i)p%0+s+!fzu z=iQ_q9|3(~*dN4|G}hOJH9fGKA1NOz$c0l?TsEqot_O#rTs0Jf%uylaGwTMkocEPK z4==coKr_h0y{t*)+AZ$V`c)fEEn8(`xJ&|uQIIra>_JI|srJm{jedZhtF`T#z9MZ9 zd^yx`)5@AScS>T`YyRoa<6!OVkz-_BzRP67DKJ;F5}0+uL;I_kAaa%Rza(EX5k$VQ z_E&^U5|y4@iauNVtBJlJXUUpj z!MpwM&dB%Zz3YReY33z8ftaKIYL(~A%>Uw36GDo}j`pkbjU#X05@sJo;E^kE=AGAy z?2`sgsQ;p^RjdC38Rp!+I<;0IQue!O`r*@5hV*XeQT8a80z2{vV6qyLm+Wx$jyLkU z&3=z&zitWn^u|AuX+#$OfDGB~K;6UXP0O^8?fY@r{{m=x4?*IgiIsd_=Z<&HBF_H)uOWBwY;y(^LP?6aXr ztU)OuO^*Ub6@CA~IUtQiA&~5+|u#vL*-je?Es|ecJV{7&=`p^whwywKKs!?~{3m7EYP z^@WvShdlCEnO;U4ggw98z~9wd7cx%pb$mQLcZQ5+zudc%csnN#P6a&Yevrohd1u|| z9r#?Lg!&KuM)_B7KnWgO1R?mQE;?CE6q-f(gKfn*c(v=iCe(Iv8DMy0!X895CK3M zj*D-pPUUK28~E%U9?BFIIVqF4H!z96Dl;zr@F7F1SM}@?0BVjGB#-~YUqG{2Xx9jr z0KVB`-d@1D2KRHpBN!zqA)jnu<8ABgyqaiNDn=d@`4(9}t8{VmkvlJi8sOD)f`x}N zsecwo+KQ0#h7+OdU8OxVPZavSt;F$5>gC{(E~m?6Lcfw@P9*bVf5n#dS4wqI;RLLL^~a}@WJnJv2@Zq~8A3y`!%Og;Hn-HlEf7>Q@#RvFmZ^Kk~L8ZoRnWmRj< z#?srS>%~j{`ti%Sqgjs;>*f8@VpZRGCep6i$NI+^no*UlMIy_Y$Fti?j$=2`SxnikYceU!$A`;Qh_dGj3j%i$xyCj#Km)Is+|= zW5W0qbhhF>0H0dN({Db{_Xk~F=-?qrvNAoG|7v9=CG!*^zu^S~sUc5Lf+@*JNvkj0 zafA(#JA!k*0C=_floWv8YR7#b^iB5J`tGyzr^t>VWU7VCbP z10UBn#Cr6|3Yo3+eykRg+}8q@P*2#(;D<~{<|=M#ojt#i6XmhWJw0DUJc;;Xy7Nh% zPM!1$?hr0ji5uvF?j5oC_To#TNyzgYBMAG`SS&lR?#IK6^h;J-h*>W7|0GKX=ba0H zY@@WictT~?=CE0AT_*X-<7IV;gNUEub_xKK=QOnTLT*=542y}=>b>JJX+(xCqK~mE zNZCAQe;nuu+>kfa?ffs4T4HOIjE?gGDVi{c*$9wUZ%O#7RQ}5O+yXWha*VK&7~$2P z{mA`wKwvWky_inqQ}nvoXEiB#ZGc~S(t1#|0ZFW3A>g%Kju^ z*$cQ6OGjp9>g~s<kuT+j2w)zkK_U9Ixz?Rk|327>>2uSZA#2No!LgrFWl zlc)}np2xKp#2s#zH#0Uj(s0laKpY4&hChm!)lUgaMb;Sc`bB@?`o z$BV6{=r9S(zEw@H0Kt(t~Jxn zvweb4O{E+jKQ~vd?NctzFri>3D|u7I-=9s(o8jr(GVcRLa{yEL9Z)N5h{nH0zsrj` z^#)6tuH-xVW~H`3>D}R?!0Taar9jh$%oCW50$94n=|2L35@xveXE>{%4-k`DugBiO z6UKrP9+Bs6Jugz;*g$NP{12 zVVu7nbb@&vD)dRui{Bu(4Cr5bQ_+BxZ^tM57eJkUy4p;?QgF*Q%xk7kkPQHwIPXM2 zkW#vV^i#EPq$wGWu&dXcQr^%=pP^~iHu3Vq>)YDVlLV>FtH(;M5|p9e9qTz^;fTlk z3sAu#O!;4iTxXL@ofB-|oTu@$%rLe9{a?b*H!W~MEu}R#K-Sag0Fs;Y$*{wM8rw31 z3aSsS?2c99ZNCg@b~iVsQ8~dtsJKr+J`rb_!^c3a_79Do<{U6MV22Jz?yIpw=IV@* zlw^{$D;XtBVfGqQ`gj#$##F5UptE=YdSn!W2Xa-yD)&G5vvR*;ao!ls;&i+nsw;?{ z>ck}B2uq$4H_;1ppk;2YUc8o$j%2n>pYk;+cZ;Wewfnd3`_Hg_1zP|TY`=?2;610F zb9gtDVjs3)-|*$@!na^;_;{n|?FBrV9e-v*u0~?8b%23Uk{dg899*e+h!8So-8_VA>)wC=aTy>QTDEsw6{#(Z7VL z#QE;F7)Axn_QHnIjtD?puNoZj!cw5FtG$xKng-)cCWR?)-g|8fOXnDpRfI&`_;j-W z`Xn98P>uV-|9D}Yv@i1drGh1qa`76^hEu>N$;JF_S^xsTNNl99OBdp&F^0EPly>F- z`Ub7JOk<^u2Z^tJik1+#vTh+Vn(%*!(o3*!L2%_L8 zVC)0+0z1_jLX*Vz`(K5~RN^C@XJ@6L(X z1825{ka@W<-Yl=r@^LL>p>gH>UH5*!uc?P<&*xMoas`T_xq`W~l~Yob=Eq_wBo|tb ze-bU=`1y*E3{_Gg^TWNW9S(F7U~>yd_mH(oe|F2Fhi2Cqaecz`-z5NY2Yk#q7`V*W4|oevdj5Gh*SPEvGI5-DkIFZ$5WRoJ46O(bF%FT6hsG>{t45 zuw0G&H-1i=xgkMc_yFvADz}Zz7HM~az94j%Zy4g=I%%U+$1D8_?EcX}OaX3Z!|yk= zSYrNCi$n1PhNbKROZ)r8I+^$rahI*0n9Bz8pk4XEq_#J(VkP8o!H?!wcKctt!bR>f zk7NAd&A`91@_Sl8tc(n$fyL*ww45M~ylL?))kAHZYvlNYufm!}Jp(t_s9Av=GgiN*l$ zQAp&7s{!2S!cqb+A@(9sc!mk1w2XfJ>U-2-fSoZ2{#8`hT>7=dD(0Y)qFymWH_-Gk6*5jxs1Qoj*HCw<{5bBU@kxo$dNu>{H zVEF2Ul31wyt4l=D)O7;F!-;K;1Rjz!ubytQV~BeDN>o0zYBh&%G(LX&RcCy4btL3r z?e20BTx&`P_C@iRQmRS)W<1FK=3=#%KE<&`0X;9kN2r62eaqZm?T@Z=*EdIPWNpR$ z^0OwRH%tCYlbiKEljD8XR+Gli;^TVehYtWeH#y5^mjft`UW@#!NEQ2u24-kwZXaar zKCSQAW0V#$LDkQ#ZTDEDQUe5Fx2%R)$3drUAvhq|R3;&`*aG8P74jhzO*Q4DCw!<` z?fu1-t^B4p2gz5Ysp>LLWZdx>5~^No9c6hz|1sT57;C4kSCe@KEy!KSV zjw1iCa3>{}N3v0~iFgQA-vz|2)FIP*K+5b);XkwfZ~ z#_8;GM&D%k=^#0fNGAC6MlX$MlFR#c{p_*helRc)v4%nsHWu;rtH$? z-V~pH*F9$|n&c&e<&IK2?7WEhjBL%U*!0W*c3K zlgBxIN&JeIymsz{Lc^Yd5$Rg7(H|7Iqdm=}$X}scL-1HQB(1 z$Q6S;e2;95HccHyhVLh0wC|G=vA(kG9ey9Mx%(q$1ZKcrJW0f#F-JilSALhMmiUC` zV>+81$@B(Kmh$dvnd@8ZV$s!G^6%48EEex)aZ9xWKF?|LeX>J9FMA47An#Ef(Zv63 z(!e|eg-nN|to`)i(ieT_jcjr(*?=atTp(2+4L*30Pt z3P(Wu+}|w&z6f-pTMGQ@GyKTa)~n!YLsx|->xo%L;}FTT9{*kga0G7ajNmb$!bwj* zg0_B`kS+8|qRhQTe|BV=$FTnTy4B*K>KPdLl|wmYa{jSc7R{rcQP55{H=?LnMbld?msjQhF>4}gK zc)Oj<)n!cD?`E?}6p~<)pm<@>sN0=_{H&x71(oBu8GqjLM76gh{6Rs(m8F0KN2a-! zrg}aAcp&lQ{&sQ~DzM8joLnHh&+{E1?ij^+}W_uTTQgo|m_z_;Lg!N?VsU#$Lva^FIuX2xIhoQ1XHmQr_*>h=} zGI#!254)b@yq8j7WoWRR6gQ~>u_QyXqnz6*U*!{9wUnfHZG;HF{jhzceX?v62U1+| zKYc4@(Nq2iG4UEjxP8x^h3na<*p+jAD#kl~6?_$K0rtE_;HEH($N2|zxv_Yg_8(o> zcRL2C{tD`{n4T~ew+Wdx@v;z+Ib$%Fsc0ro`P)Gc6ddWVVZNQy`}EqLypr}O-9notFmG5BwnNaWb9_ts-Z1m;yqUvB{Rg*BGvf9B#(%BYqWKM1ilQXJXu@jTt@F95*9<3$R^?(vIaf_s7Y;oV!C|H#C9?M$%QzW7 zm1Aw4J$rKcb#s#_f;_xV^SeK_OZ~W(NcM7I;#3r6yr=wJSwb&D!R83`Z(PphaSL9JkrCx`(xA^Pno5?_JU>k%z2|?HamAdfre!{; zSux{qIX(oX56pd8in(-+(+Q-WcqiAgLvOg+k-6pE*E?t)ohW)A|Na5=**Jf5n*Fj00wYo+st;8A=CbQyZYeN5f3 z%kSOVf`s>KqM8i(L~Res30KF%djr>wsRr6#2ox)W$s%(}pe| zbC-Jy%{bQ2F@)ja%o!vcBw6c=Uf6rz@YlBm7)$TZU8FUOheak-N&HY`d@)VxXWh3BNXwbM}ru&p*}# zBUA2CgF$!-Tkz^vldkd#0=f#Ab||6Y*sNa)vJki-@)SZ1JS`-VOf5WY9ad4DmMsQYe>OQp6}pRM{oq*t+5?u6ve20#tD`*3*~ zOR0-_{T@+4IwA9U>ymss@0c;{!p>i#G^bPH8rRUOVL~!M5&;9IKi(TduasC(3A1mRJLr|V|A;U)q$N6-$cKP#XqZdME zm&|ndkH0scW%>lnEF(F9M>oy~y6YEz55r>093r}`hRtvPajh&Ndj|3-wk?Yvz%jYm zdy}n+Mw9M)*@aaTex>)tGpN-oNs$7qf-SsoLXOTH4{b0RKfrYX;t&SJem?^Y{l0iI z`dxxBWVlVddPH6>GxhQ`Uw!5muRWz?zXUv-y7Rg_$ATslA$s3E+5dy~TQh^Wbc-cy zT#5h^)Bxa$WM)D7m}6G=2T5Ld+YzX#dmSv)sxCr|RA$y?n*Kg3->Jv}@U zAcmTT76=?Edu(|h#Q=b<^`v_cQC_B|v?hr*vyC_uejmVQ0O|X8x@-fQ%N0Q2FMMGl z6N3&e?W8hq$c+W|;CcHTdg{5rTH48JR$Qx}$j0}legQI?Jp>WT?>?t~C_>pOH`zp_ z(g7J=C!$#2eSB{KmjnNg84mB{XsvG@<^JORNygt5M5zk_qBb16+os>a(bpqchTEQ~3{oFQ_7fg>9w58;& zxvXVi?9t6Q^Gf76A_q$Ru#qEA6jQptqQctF2kc2SDZi!dpHeen`n1i==6VwE1MbMA z+quXoX90_TqOZ&@ObY*U?DsdTc7I}D!C@*3A+?Ld07-wk>%-`MSm=VZovcQ= z)D*YRN>GsX<@^ue@2rbZAYA_MYXlDSLi<8Z>D6yY3bsC{(CJ)^!0@tDz7$0wI4_tj zl-L{qHdzK5v$6T=JrRyRJ-FRQbw#S%l%+^$2*G0Il)StJD|!(EZ~AT5N|hJU^CCi z$Z$)B^aB$kvQ4{IGBWdT&_k~_udR{QHEJ9jJ1V*^l$4(_uQ*YCPu#tL6Xcv~l~@%; zHxRuyhK6w6@zWJv*wUQdvUS+2I6D)Ua`o{R!^8RA>MQ(~FgN{F5x_6_TU{sj`~UJx z+;;)=S5~QFY1?axyG6sH;+O;j%`SVUv`elH%vW~?mBxMQTaC8VdD&e_(xWt0U-2mD zIU?}cT|2Z4B9>gWF-}(rWFV%aB!vx5%`Ca zdo%(;(K*m|<8$Ifn!7)>KSKVXHa7jafRqAj6yl7%NfSv>i4pU6Bm9-H*<*lq! zApt{4q#CrwL>gXQ!3Y^zb=smO(qbx((zsRxAc7msO0(p6c+EMTOA@#^n#UOCyl~>f zPp;Dtu&s1vjh9Uj71;WrP?8@GyYe67bH1y~;{T0QEmR zuR|nNyVs`*SyOO+B~z#pk`=r=sz8v5K298l;7OS*5TPY>lh7hhrK1I>8`kqNyi<_p z?vlg@?g9V99z)QtY>eDf$=jI|3NN;RMVLqX|Mfn*s2GzsZgEKzF*nWK3c!VJ_nGvq zwOLXb!Vwz|Op_xo5$vN8EOg6Y{!5n zmKk`5mOOu0=XenF2Yc*vS5i2I$h6q9JML(1?{XQ7{e8#q0J&;!H{%y$Ip#qzEJt!3 zi(70w8jbGw`bIus;r$rHbEkF|BHSYae$vUggGefFV%E3U3#biedT?=mpT;B%o#1sL ztq5oS;$kvDL4>oV01RX)N}sg~*%rx4YHg)pkpbQq&@?@|S42QnpGp?woIw$%_i}_A zMpJ&^BY}zg^as`-p)IiGcw!nlX>%?W5!*Y<>8HtXS0&Wcz^1Y==&;dX-*l_=29#23 zKkP4Ox{_kmp|oCmRn1UqLh4eT%Hx!WiuaCbV<68-xMU-#hD@*KNrWP$RzD>b;F>wh zkZ}r6eMz+_5hJN#l7aJ9!;-x766A1uUN0eo4fC7Z#jIXXE;>0Y(`A$cc~2Mj$nG3FOFZG&21J ztQmx#NWK8$w8Gcj>LvYRN>O75$6G~5`o3k8Xb+|bDo+GsfY5KEr`I>7c-wQF4MB^Y z)H5Hib!-QbSi1i#kDv#&a3#8zL7WcUBQCFUV62O#W zlVa%yEG}i%`zDRwKUBRq^{DzXX+lD$e-EMkd6g!TQNIsEF1-OZI@A>zTI7=$WuOu; z236=&f4NZO4Ak=hgBXFcP)SI{17icmV~D&PK?*`1I|xd-i-5>+0I47vftGkx`H1Mh zucIYGTeRrvnBFVzO>;{E3O3nf#S&sCT4AYId~R{+g?_sl(h{~&+JCI2Hi@i+wHeX)G-HIP?2e%oPI_Bf9tR9lSAkZrb0a7UB_7g8Iix(S! zHc#WrKsvBAzOBJ`&w1l^B(PjfS|b%V@~}!uGnuOH6?KlVrO^1+jPMoE=^tt!rKqO;QhA_-ssT|MoZI!fX2G&V|DQekSBLNP3& zCcpY4Trzy)LNFjH6TPqrTz1TkE`39i)y6p+z>>Teq(MdM4S-Ty3BI+!dV|<^OB!NE z5GV!h)`^Q;m{5))og-BgD!bdDf*Q+^rn5v*{WTaZB7qE6Xd+ZEMzzvbdhaPxaon0| zv1uboF6nw%CYvf31-UHp7x5EtICA7~>ukKSNMT@ukbm&7ziE(@*a1tGjJ_R?&S<&@ zT)qOIcl%oH3-P881FMy%rWVy&+8%mK$bjmtlTLujil0RTynvO@OJhDyzdJI2-){~9 zYUOQ(*%ck~mYN^?ojW5w`nB)=xyUe#Lkn!=P`@)3ZaFA_GF2Oy(?$sfbZ5Xqf&uSf zeY5c&&Y^%k@xT%K4_7<2`2KxXnAGkD7%m`A!0G@#iiux9`%y9Ed?!QI`0Yj6ne?(QzZ;qILGyMLg^=6qTG4Q~<+X6f+-YNR#8F?blM1-uMiy8jYf5-9gycAN)b z#bL=g2jhR-;rHO~jZzLD01yeNgQGsbNc7qUC3BO?Lzh3y6Z1~FN6R`>x&r`rb+2&oRHna$sr)ISiBX`7Tz{-{vgE}ENNtL+cd$gEU_iJ&eTDsf zZ18>+Wh=rS@n|K!p_egG65}j&HSL3?q2)GAMntCd>bT`o`=AaCf1ZSep0gB{w@}Hn z_U^>*i(E1@Vs>zBg@G*d`^$uzz)*{}GBlnGz=Nwbr>if@o9`X6rG%4<5%~mbn2bVN zZX|?-RKB$fBqM2)IzOq|QVIMsguFjJ7vBOK;5;k+5gEW9Wso_HLJ04@Y(lkCzrf|U zPty%2k@@RADc3Ha#B3m=0P=V1UCc^*AOSn%iXc`Q9SYk<-pB{rIPGy%+WAp3b0r= zNq-S&DcxoF;7D)#LL>=nfovXPfV?~;j7orhsd`qcy!_i6fTQpCC;)wEPALI)3^Lm$ye4Xck;?Byp8F8ovDl1db<07bLi=&Li+ct<48vSTn2%h(`159# zM2IZ=h4I-LBDLnlm^llU6CMzPYh@~CMxXCx4VOa~XSNG+JkpBGegSa3kbeRc;Fhfq zR##Wk{FsfwdSv=nk(8SCY+*r;IEAbw@kvtj2N3IG&M%B5vB^NotVL)!`bmMvPn1)F{f@>dUFZRwvoPoI`(i2X-i%7l(OC|y(zI#3f$q;gc^|7E4J%r>~ z4kZ&LjxQ1fy=w7Y0;U!3$hr;=+a{5FVOixvj&pGRxu~5WW zi1`)d_PBWM!{V=mb&CEfBg0wg&-k^~pBd{#2BdXi+Vh2K2nZ+fxc!}Rrdomc@>7u| z$6K>wef)>$4@Di(YL8%HG~|o`7?-}DJfxHOigf(|!H%Q|n+=Ei3UV@@`qQH==wp*CVV@`lmU35j0b?a|CMay1*WMHqe z-8K#D1B_<)EMp-F^{e{FAXB(t&qlCB>&HpOVEpeoeVPRFscEQs6|_Zo?}?jJQhOn2 z&IK4W`TfOQUQ9mX5Eyv~8>1WnTHZjc^Q2bdE*ab{_6E`^vDobrg1UJ+2M&7wuSu-( zJsWx7^k=@_%?{a%NX5VuOyN?wdUB%7@&D&Q_E@M2#vENP^6v%kznRyMyH3_$pQwx@ zs6mak21uj)z!HM(OgSq_is7W{SS#x~Cabu<9v*Q5g%q&>@l^4Y#=pdFBx!a9AN4@O*ZfWs{53*uUiXKcp3BGDXheP!=_tt8 z%3Z$t6Ti>wGJvf9w_5?bSR^YM=F#dq6QiItW^;cap%{V$JCUGD2p1&PJi;Z(WE={s zFu)xtqgGK@fP~;gZT(=m>qsb3iv1<$s9fjj7}GA#OzY|CY3kF^zL?k_g)|fE{rakU5zcR&(z#HNnRHiB}rvfQ-)*E z_KBg!1H%y5X_}&JCa!MHHyhobz$ktn{WlJR1?itf`j+p72@GNE%e_3yl#)}@V!2_M z9s?(U!agI|X1R|{y&QJQ(v>@A<_4a=Wnp!E8kU zAdC^QWwJe>44r@~U0US97k4JNsOIFyu}Wm-3=n13sK8r+Gk|gx4D0LIR9PxH%8Z0w zqT_rBhC`ZtRW3QI@}`?LSW9_mor{2CyEP|nM$6{^D+BErIu zFYW@4waC=Uu7D*t9&7i5fu<5E=QcBpVh~pwcR`|}g_EWAf^>RB&uLhXTx`lu5iz6F)!%FTdPR;VlIuXlJk z^JJ=OEuQ+;0D#TMMX;7`OS%gihOHpNk_IDqCV1qG6NCR9G(BjC4m;_i#?;Dzj(g?T zxe+o6j*6vUZkMIZFc{i~#Z}Fq#()sWQ4_n=!K1Qe5oiiGjtEGMB4B_*cz?AJNt4Vq)@rex1EuAi#FFS-L=P0=d>+K;(4r_N9p?%TLBJ^Ri%d%7 z*-*?nO=21Tw1%*^lHc*p7j&$lhGO_{P?-3*mi>@wYoPB*9|n0ApjY@hKqk6bKici7 z9h4Cz7#I!6ehVn{g&a2l-)t*xP9HmHQMoKtEG-Qy%uz(Qt)tVZsS=yW!9fz}u{BPo%3UIX?kSxD&AxMCjl-T6dYO)uR3?sx6-e3n%FHAvZ; zSdfg4rIq#hX)(yTTPk<|2iifCogmVemBZSgfJZ13GdznPq~KP}$MTCuTdVi}e@dzg z3*aT{)?kviOJY#BYaBI?I^wbpn1(xQpU!X!M+O1dL~CJ`!W*sY=}fubgA8=>Sf0>{ zP?`Yz1}-u(BP~PN)Y+3xgV9f_`Zkz}M^^GXRI-nx)~6%bk<`_^gqG=bgpZTQH1`~w zDa)IGXBVpKR4&D1;rbN=0AEY2ALXF90=p9toyU2cYHRsj@NJHh}w}uV&>*{ z?NW)o%){aw43RvPFfwL1T#tMwD}FWlkCZx))q{jR)S^D-8?iQ|zs+yY82zc+e)fX- zEb43)#>oEVufn{jD-7CaU-b4iqzUny{kTkjxkEW7jZo_;7QcP7YRovZ`GfI0f#7Ad zSbGYi84FK@dYP{qe!+$SpL(o7utNRR9u0$QuAm1DZ4rrTtV;h$y5$ zJsM>Etl5UIHBD?B<^?dtd<2^9IZGsi$4dy!lkbd9$r5DfAJHg(g~tr*hhO$Zndl5>v- zD>dP#iC)sDwQ#E78*cSRt!Q+1xMVWta$nalJ0HW5e=P++nFU0qU6F4)nJ-JOIHf8b z6}gJtKHbu0-%Mt3ZX&aeS!lV~%9n%rdR$j*a}gQsYt+(=(`>S=Q3s`5Me5Nijn}7c zT(bh0Q*%>W1N0w!RRK%wlYZRI8sE1X6X+oG=gzfnP)$_omlx1OvrA*&n@p?CAW(&s z_XX)=mwdnX|0;~Ybm?yD!#@#h9#i5kq;1Z;+8uhYv=^#_cxg?+`PbL3rDEU?wHW~e z(LZ@77<%n6gF=)Vf=j-j>X6!mVFX*;(Uq3|!)<-HgcoNg#7;`6rMn{=E7Rm_W_C}? zcR;P45cWQ@#VTu){#yW0=SKT_6z|;nM~?>wyQXLcjnO}fw7#%BGf%9LCdrZ$+#>k` zlKagLNK9lFc%z(Tqv%e$em?@{#D4~7(b1j&+iMBD+>&_<%y1AW(G;T_hC& z7EY~2{_)a z1|Q(H;6S8`YHydGyBCXoqrd161ASt6hseP^#hPe8_mbK#4=9oie<+pN zQUO$j&L~tVBi{TQ%myC9UA^=_`!x^;Y=qt`yALW1v#C6zmE)ZZIjx7hw43DbbA267Qj%$QX!Sw?+yqi0>0~l&``kX z_x`lj`9AG@yPJEL^hpxdF)c)sk?}-v4Pcv?C|sQD^X5`ze)_4gNUd(|zBR{?rS`j# zg)1-eU0uET)KOr{`U}t=Y5-tDGa0^>4~#i&OsN}9f2Lg-o&N&wn}vXDCp#a#bb_@y zr){r!PlhiPx*XxLX!NdBu0M>nSsVwpF1qB@V8((P_NUg|E9Pg`F~g{$5!Z+So~Oo1 ziRyD`ML&RF^ADd&F8DgV2&v&*><(<u6Jc11btw0o4N4&X|(R078#T7k8s9`gAV5QaT$RPBhXVs`8Y4GjZvO+8`?_o@{2Jb>SCda@NUzPn>`$j zIspC-5UYwPx^C~p5!_KHVOge^P7g)6UBijYO#Yl4=J&@>l?Vc*CsaK<1;7bMSKuNw zd>DZ1YkH@QT{hTFAIxb3dV#aBYd&h*++`0ois7C7zrO^PFCWDVn6gE7zOBAl5<%9~ zYtha2pZb`w#Y=xvkNdh<(q+J+eY)?6WSX4|9z|8x#R#^yiJAKJ-&Az1G6c+uW}NXr zSz8RmUmymI+jN~LZGYTV><>Y;BpJ&2l(YvW@ynb?048QbWjYt+*S>^;0#kK0B4WYAIJb=(+=7GE5^k^Sng532}BnLzL_@QApA zU(_bT8$D;rcypvobyK_pIJAZ>h!0r&3@e-nExkdCfoU(JWI}RQejS*P@+|3`CP&TJ zx7|(d@Ka6l6J0lZh)B!)0Mb5ImqT8#;O z!aZG!;W-sPw@)$DSD`UUQB{}wO*+4i?Q``YHmpEo6heC)RN!;D1zyYvtI1Zu16zhK^&B=REup|z~5?2X9&RoNd; zXbursz#=qw$y{r)(odb$%aUEQMAZov%VjSbl2S(X!76*W425Y3knzv_h?b&2XnMpf z^UQ-2>azx?Q=*i_HwmuK-iD7W@?V6Sb|Rapot6sx?XKo0j2ShOi6z5C2a;SmtLqCB z$nR1zTPuYE2weO!jg#ix-V|3NZzyOu`#OKcT@RM?rT9#q`-X7#*6%^EQHZQllMM{+ zQU1PXx`atxb3K#$M3;~>L%D+9jJzD>&`r(!G{EyUaB0b3^HO2H2Z(Qb=2IgMTU{If zLS`bz^A5f?cequiyZp|^g(a(Mg+c5MT&y7?2ntCSWV-7xN@tXX(BG|qI}c#&ttz=i zk5;@*nMhaEPy<;^o;(2zshCmIyf82wg)FNgQNtK4l z-w2lU1Bk!Ups0imbEx2PV5=h%Fr66MZDUe6vwz?Kd@+ZGtcq8&5w$8k_=O`z;Eb1x z4MtYnCdLZ>e!r$u@BEb9n%i1r9^yEILqI0FWi6cd0s8AfL}tjMR;tJ;#zS`D$8CoK z$>iSS3mu`*|E?%g7&ti+HU1F)VU3#6MInJ8T_1^bLIR1Eh zv!-<<@Vu0++p<)iC42T-;AWs+eSUW~w`S9i#VLH%`4dlH%d4(U(whT!sSMiS2BtmY zHR86D&{gBPblFStNpplc`YkQO(Iq}Co~sOX%~|T&>JgX~^2buwNGR0ZKTYa?iCNh$ znDFkjc@Sv%>g=n0g%2=PW#r(6+=z_Vc61s)(d+zHCg$tIS7P8q#ow^~)%RfDJ4R~o zQCJC6(|x(W>%X5Ht3-~|(fToMj`3`2>`1CMk zSn7jOWh}|~f^D`fuHnz9neSup4!1>mEPRd2)T9qS;)7?)xCi+COFUw(^s| znKS6M+<$s6+%cX3R-9l|5;$}4g@_6cqTfI7@+gc8@PEH=v|l-X&NwZbE@tYRxL-&U zl#>%T=?)uKx2*%&FTiMIUd-F0(bdNgX+J?NJt`h?^P=Kq;XU)T*f-anSBOYfb;*<2 zC~>)<22cMb2~n07R&nLnAq_#Tb3RyEl)4P587h+!LMs|C6NTk8_zRD%2?@z%K~0#CDDVc3A++jsKt8ZHLe^p|WxrR#!_zif?+@0y z(RhYH_hjkFYIJ?UvI+pv)Unzh4twm=_&SfY z5eFAd8wci2PUyJxA6Ixp4jI%W&4Q9d>@?Ir!c;*;y**Zw^02rs->In;4*&c>EJe_u zohkQ0pk5Dg)h{3QoX<}mkM!nv#_!OZp?bLQ*%N$~*D^DzCy+sVb5W(M+mIvRR6@25 ztry=;BU@OC@jE8E5oWgKJ7>`}2C7>njUzp)M#)(vi5ig!(~%X|i<-d*ueVpATH4bk z7Y!m!Y(09#NJ^d+zEqn`eN9T12`D+KC5!^vM1#Rbfe;A#8_c4(@L?-u}aMPUVEe)^Iwwo(WM(iy`6l^ z`P5ZR(b{1 zgv53N-z<0q##cnfU8E%~tX|fe#wje4Wp=AgZfqF$+Hwf#u(%r+KGyuSoBnw~g^H&3 z7MohV`<}Exar0wANla~U1C~F6=w-P%<@Xj@PtGQKW`Kpgrr@gY9lOqg^Q2GyK*=~C z?~Yu=5mU|V0s`Ck%|r11w8B8rOLH8O&KFg&0S@2nb*gIIFg^`$v9R+X(syS>c_OX-zRrS&kRV$UtoUT z;J>3x_kX-+MwK6I{A;R9`src*Ntk8LYg?WP$VWAU4^#8oOeWQwYWI?^I`DF7<~v8! zzi8Qh_Q2Au^DGqB`&h11keI?;?|5>TA*xgBO~dmu^*-J9 zgqo{Vc$w2_6+)=}`REJUenhpsxj3WmcInLXUX~sL3JYSeibd`uD+^B>#!2*$snpx> z*29wCY(Qxu3^q2)?R3pT*3}PMiQR8Mq{oLYnG%H_=k5)BPrcC|Zv-qimhqZ^$63uk zn@%tu!fBx+nX&u>(H;5kU~%~4KNug^tj0tM=CSxz{HNiAd*E9ni-nvEjTsQ(;-M>P zl-QCgvh?;Vh(tnq#^>Ph(k71kSn!B-9H}=I{c;|Tae&rPCXe5P z^EP4-9xK951b*T)&`cXO0{Qh@f8?eP%t0d;_6&rnC2OUJbLd8F#_6>45>AM$t=IoHX*9R^O?)wIW(d#XCv23~c2Q z91HBN^HHJa!v@v_HbCK!lh`zJr8sgo=^l-Fw5WI>e(sn&ZUvK52s>oG7mCSXql7cB$}oVwn&UVr+ z+>1H%vAv3UUx>5OfZG2VCMNm z>f2w;C{Y2b*fd3at-*qjT#(_^A%jddJKY1Tps_8vQm|DKfip`yqHUb6a<6z?QvO9C z9Z6I-oYN)_T@JWUX{7_s9M4#&hx1|vN@9?n(q zYVc%`|LMbZkmuMjMwjH}T4Y8OulWl*m164Jbv!C0C(_tmiBkFthP#;RNF{Y0{ukp~ zuE>Kf*K{Svx*2w*o@{tH`TE~h4GXh{ov8X9udy%32Q|W+=|&vzQ*@*ZgP30INQohQ z)7ErqtFeAw3g4H%EKsDu@O>)t_=!4TO`{`&;AD7Gi1U0FJwKWu;}u|ftxXzi#9a9E z7i@poQ!`#P`u$`JJR!k@Ll*yoxioIKY2Tc{K~sG_M27ba4D0J7O0XrZAvQ}mzD_yw zP#6cSd^U4qJ-kM^v0@9tzMU({G1J9~?;yDhly(>ChryM;A&x_W;XK_NulMwR{?sV? z!229yjF*LAXsXG22UkT*27?v$-gGX_MZ(<))Fbi!y<2<`{2}{nxB#n_gEpe=*C{DB zI})Aihej}Bm~#@8b6yUkTv3-8--py8_59hf(J=FyZ@jzVKXGcz=2k@)r*Hc=61skD z)6|T^pZ@sc6QC9}6Bs@I;uBy>w(up1VL#B3EOmm1@CeqfhT#@JwiISlsf+=vjmDb} z6+OKpnoDQ1K_JgsA_V1N4a$IbUJ8sr{5X}*?wwzj)CC)3gDDOC9jmXSqMPuZ!pW&q zqrbTA$}Eg>P%^4i8i&SX>T=9Vul!cmdQ&MZVbdryK5f<%p7fgnl(uiPU`?N%_Oc5W zl3$2pJt)00NowsANh>d9e~O^?b;d4{@?$eoz76jp-cP!X#!F|K!_E7rp5OL5kniY-MfSm)UzE+4YXCt9`%@;=!r$6Egy?y2< z{&_2=2wJZ{8Ab@PdiV&rEZt;89kYF?0Kz#EJ$C1B@A1O%8H#G)&aBe5;x;LCz?+cD zO-fG>qQoLdM@r#kCf%pr;3$I*=k-qS&D%YyqV)Rwt(H5sh4*)pRt+@L9mSM%u`oKm zJBkEE{EG=H)Tpn&qIMLErLCuwfmL;kD2kWdkxr=65lw&-_20w}?KBHL`P{c!1oUe^ z(7MTz;+we&ob5x)6dGx8;8*U>M#s6pYuu#K_z%Rq4Mbfu9!b;53s1yxY)0FwQT55U zhforiw_+cxq~6qgq9957*Ygn67_R^Be%~35MD>cp?5`~lzlMqEA+`I$A&)#fe0-j5 zH7c1)5VG4fVU>VqbLs1@OphoXl7>d2YQLKgnNm4sRq8MdZ13x+D)`N+K)Q|O`OJ)) zmv^q2moNb3)etj7q&yuYx|L3lsAb{_(u^YBBgI8pHZRJ|n>cF6 z+%P1t?Rj?+Um*N8WHzZd?vAFVTB2a-+WomsA>*dB{**Z!W6*Pnxmqy;pj(wJ7mZEF z8~9`+7@L9SdgditDaOJy4rvCWF@PPG4xnH{Y05}rX7l|BJftsOp|t08p?e_()H#zS zsV*yJ$(3b+@hTRs*TrH|QsFu{iq~-zE9(z#^I0X>z%#lmk8^HplEq6@MY5!Naua3! zgZNP-jQ^TRGq&b;dh%i}`T%GqP(azH34zzU$XkZ?``RHbH#(aEBqBJj1T`p0`8!{h z?k56&3utm+g$NbVaviysm*p0v33cx%wx6~@Vs1g|yqPUD#pzh{yKu( zNc&$wEo0xFKief6U@c%Y{(><-9ls*+bt?Ru@7HK?C8`szf*7Kame8) z7#VX?aWp#rCe_mS-$I;F7S1&G^1kI7`%KRPL@s7yugn-5*_eCv`D>IyT*p;uC5wdg zU#=NXYLz&wp|Q+ROw4#xcU3iMvC4>*K5edlgrS&&PROf^QCBM3z;wp=9VWq}wv70unCi;Q2r7O3BQqZ93`ysym&^HB>Bu zIvZ4_{+<~L{(Vs1|C8g_LA92tX&sju3l`D_n!7qRdxZ>HDF*fo+~F*5;@8Bg+aF;Q zm?;B95#AIIsar$}G#h#?DHbNMID1Zn)2`Rz@nQVupPbO+!#=7BLWQuA6Epq_eXm23>g8Jiy$FqgZzaXV^fJ z&T%lP-SV9ClM-&T<)AVdxBDR<+q1bsA4LK|K1MxF%bYarZ%+$bQ%!#>{^jV<5}Qb1 zQK}s7w~9O%`;zhRKX@xYFL2m}2aGx}i5d~}KG4n1r?b6GYZ`GM=foM zsxnlH4s{cI`nl)WxgK_6V}t0`E{xKOj4=lHX}G7qzPnQ`u4mVr(ZqGWli1`#6WqM8}>(yvsoa|TT*yTP%v^!&vY%=72CV|{-*O>_WRBD8#^~e`K%PnwnF&q0{)3l z(9=t{vq)|nvl=&1TWsEKy&`WHA5tL}SHQij%*iRVZen{_wwnJfNPo==4sC9d<**+{ zll;_#PjL>6{sJ>Qta0{DHBV1fCCfOX(pQpsp2@P@e_Gm?E}>|T zZ?Pf=`|LM9N~(P33dTcc;mIq-sIwImVh=e_w(?6*VJd5EJ!t=AGkZu!o-B?htUgm< z0UD>fruNN->qnrN zrde@_h<=z_$Lu+Z;g53mLo)lhfP9)s#;~Wr!irWPRiBcYm)8*y6#J(mc8x!=DhG2y>%36%gZ5OEexBc+e@{;=(E5hHsgM)CW> z7$eNq6sN4`Gx8Y1tW(3>q1l$jO~fi+r%Sqf6}cc|^lR1A22IQUX)-mgN5td|V%FBy z^T4h5{Md-mb32O-oMS*IVee$@tD#BgdJWTaWudPQaKOB3;{}W7?Uv1*X>U)#)qZ-HqGD5Pt2*{M%Xc5M;IGsa@jB zRnlB^u35YAlnx=2DvLyQwHOlNe-Q-MaA*EcPOP7P z;Fp!Nft?Y+JfU9S_`B>$Eh6z|EMoA4wP>A4Qm)%&<(JR$4I4SGPbGdr5wQhDDz zoc$BwaEjC7{Ca}jeh135; zRE4S~9Yrd>z!MCl#D&QyP?mbdk9E-xtq}I<-@NVY1v~9W(nZ4{W?5sZi^UHDm;uzr z#}zkR)ZK63>`*WM3;b#h&eqy2&%b?(B_sX&ya$%AWh7}VKnv^rtwk$yMTgEA=tbK? zYwFK?l!Uz%v+z54#17J%l-#`Sx12Yc>g+v_TiMEy-**K>T};SV12aiE?bD_(J;wK1 zu3F<(P4zIv_}DBh!M9bXXu`ZZpZ@0qS&38hz5BhL=)m9w6A6F2&EepTyB|P5VeqgY z9$dCnj-g|^0yH2VP6bD%RhnDlu;O1*+l!cp+=TfoXU2UPdhB-^0ZKhy9_G>hsq_Bw zlhfD_`athp3wl>^B27m?Y5PeVL-yxHAsIs0yV9qB?Z(B(J}>9YH8fbAF~FX&uU)*X z@kJI1eKoIbVxKA%<9_zHRATgLDKT#|sX^(S@cZ2kpgp;Gk|&X+Gn*WnYSw8lNb}>8>QDhu;BsF0dSXk@ z1A;_bGr?mo)i(hZr$2n5%+ZLY^*3?XbN9Oi{iY9V4P0w>F zJNKvv^~$8B`ZSByiszRMC6ii+io2iMOhgUY_(%ZuxL&j+RzdMj0yN zkAKC~5aWt2FbJ=8?PdT8jam*iOH3meLga0$y-!BS1FvycvQ3^0|LB&W$B;5p+X}5^ z$uQE%P{Kf!QQ^aDh@fP=mfNMI?YHuln29bF-$YQbO17r%)c?FWUMmc*QI5Q0mnJIy z)T7y^L)U!HeX(t-tfJE%X~;Xp#5w&QZNZfRe{k2Bv1Eap%jrduTegOL=6EE_ItE8m-2$n1oFAm)=4 zMBzYIju}{%o+qx@^|_)>$2l0yWTnEyWsSKs5u2wG@-$NwSVZrA&J}?Y-o}OO_x$2k*?gDeCGOM8rz|xBJjKzJSMGwBl zoO>)L6}mu1x>F9&-7Wls#i-s8Bl;+3ze4!_p~JDpJ&zBfsaDzY^X_l_an)Y&PqZ$q z>4_~Ow|Y9D$57sFLTSk;zQxHL0ou{>;funLIKJa4$b*kF+oiV%9zS`n2N*=U?XwQ? zkhp~dL<97{Vq^^#SpcEt#lRY*xc2)#{N)vtKcA$NwWIowf+>2D1en8M-?~jP>g(%G z4dU;6Z5Z-onfhWFcK#^J5*3ffFV^nWVopzKbPAOUB1Ew7OFheKe@LJ}{v}GX0PN-f zmwe0e?&HB6{6JpVzJ@gYpm>CT^tR~yjWKVIVp`Ndr-TOgbhCG-#eNqN`$rh>zeC?1 z`>7K(3Q@_u2Q!OBP}{iqPAzC1vV;!{ay9erQUtB2M{-3H;^W)h(K`xfxx0}zJy3z8 zo>pOpe^5-BxY4(r{hZ~e`} zKR+h}D%&FZ9F8XqpkMKDa=3qX7U9_!@|hl$kV;d`DyyzBOg>QRG-#?^&U1?sGnwIU zA#&}f#QfjFV}H?J%&f~**BId+rle|M=c5zBE?`fcy++!{>SNuT zE&VWxgiujGu9C%e{ZQS6%PfgVl>f4Rou9f2D3eJ1+7@s=o5Xb8tGl7s}DSuEqo5UATK)c zvzjCZ!|%-*e7;Lbe=zu>7hggceW`jhK5O(eFrfRE1_gv`{&_fkf<%2LCfF*egab0k z$xh~~mCk*iE7OjTZE1IXK7W^8cn4-CfOc>a!@YNliqFe&;T<{=ct{eoR(6Qp>eM$4 zj3Dk|c>c0#y#k|DI`|vBB}}Lb;`iNJ-ht9pFK@jmF+l@#B3nx|^XIYcOlO6?&es6} zu2iu~sRfe7dq?U_<7Qm8azCSbB7JgUU^Xx$icYT90;Q-&dMtbrmBO)D_0!vYU6U+r z13i8UxU6>#T;Tq1m#HUl@bHr1)z4!}1zNq^j7GhO>$4xD>BGRUjzj{3wA{vfJ?F6y z^qV$ANmwsZVbZzOSz(QhjKV6&TuNg;+peOfMC3?n_|~8|mU`r*1j2A1xG-A@4|a{=hj0hwy$ zrA`c`mo59fI1+~~%QO@(xOHpG84G99Kp>*Z(dvJjI8ZzBHx^YURIJT7iqFfM9Vm5K4nKS9ySwK z9t@$`dY5(VuRn)$VNRE)I0=8Y*q9Ar1xp%#n_BhEqHyd-%rgewhrh~nhjMhz zfKW<51*Y@1bYsevm2(`e{O$1WJmi=Uj@NWrGYJT%3a5(YS*CToy=!H5U~rBj>Y|YD zkdEKn)z~P%z)0pbky|Jy!|iWq)uyOymX>uOFO<(PF2LBTlNC3n=DytorZ0yI)^y)g z3{dGRa(JCl{HTSiES1)&3|}+bjAGOU7yZFH*(F4D zY{5xjo`p!y0|mJ6IdzbQXPV|(yQ6l%+Sbn}jlgbS=BaSK-%(cftaqq-O5~x}rl}?9 zLi(13mW1o1*@3>45FFIJ_u*qeFwC0a{IL`Bz$b+Co`ATY zC_7SFSQRCQe7zz{3>9OFC|P(h_@L)T$9c^U#AjFiri4y%bD z?2abN87AaTgccoCC^C7-|H|vQ#~l60f4vDW;V)YsjO2?D(~W~aX-t;@Qze1Eu_++` zeDq_&h>IFYe!!TPIuGn8mDMp5?F|Y1$j@w!i4|dX^KZTx2Ng@rN4#q^r z1G(?6k6`a(^ylX59Pp!dF$iKBDjNwtWU!kDB%25yOkw*m)p0|UO^ge@QQRJ%xR$&; zz3U9Qt7fX~53V!^J7rL~UBYjl%?njB-`etCzy6Hn_bu3=6)q_Qk=*~V*Ch+eC7VeV zeC(^KJC+>Ykc2@>lg6?T8EEXvL}#%4!lAHr<1wOBcMzz~P24$!)^&Ge{tOEe#64=i9Dlh&|4vO!9TR^3Mf3O-MtLp=p)QL2v}m{f)8Q z4^VQwH6Ck(I|~5?55o`yhq1zvL88-3mmtfV8S&lZgRfieJIH}v;;xMm*W^HS>&tuY zjqvl;LF=%}BCKJ&f|8zCFTYc`qis4HTEI*5k>9Z?RG=wcg72t1nrz{#Zz&#vk~CgyoCRf3kpIr0W{)73WPZ*C zzvCGJ$>>tKC1jM2Ct76*x((e0F>qh)@nS#|;!WA)+Cds>z9tG&Zl?0pvNw~y%GtQ7 z{zC-65}qB~K~pdv%Gf1;>@a5)o@xPWnidJC z^0{R`|5TrZ(S%3hGI2RjE*Xuvgem>XKO&`#5)38u*?CZ9l@P_f#q&pbC2%)?M9I07Aq z2~f38&X$_+QN)YEz&;`&huqYX->lS;8kph1Lzp#*=Q1J%Cr(xtr=9{wZhjzcOoff$kqZY>g~)e9Xn99GCsV9c%|s=e`lA!DD)kY0z=yL+~i7U zt3`p|WYa}xAO@%RlpK;vNAZPh+=(8-6d5LMZdL9o4~-c3xxS?RcSYrzdT)CE!P zCQn{Y*S0YK9fZKA9!vbIN9v-u0ExA&z?I)as^dUSvqmhr+Y8;WLhJW+508c?we0K| zG%H>#? z&lY}tE4^=8p+4WAzqw90rv-*?w2xT2E0sThw((}L`*l}dhZm4Ic-cr^p;fblr=727 zk6!nwGODjm$UDji)c)1{j&%_vWf(C;Yo;2dfW-uGodM6F#VSR^7srcz>aZvK-Pvnm zk%^~L^d@w}Syd!N|Uzbh>LU~0X|qMyvn%)2=e9Ov5}V$dGpHGsXX`c|Nh zgv3&D@DY)DOx3dOI8q>cL<-D8dSO|JSCamFPy8U$_>r!jqbVMVuQlTsBS6y9r8*xx z@{*>UcNnwb9D!I;&RZpS{4Toas9r0b`ASenA)g${<>SEkq?+QXC;b z>8rC;&_>d7Mg*619Za`^?pPz?AdffUt^6}zU`*+dtw`pO_~6`{X^?#GXUc)Lh!=3< z;2(@tmQ#GXCH$L`S$x-vR{b<_(|GM!^mOE|aBSYBxe`gcK#u6(h&QVR~(d>GU`F!xYxIj!+@P#%R#tkS>>9p#;-TV7ZnlPiI9F#dns zy>(Px$=4-}1&81g+}+*XB{&3k3GVLh4#6$ByK8WFm*DOe{Ckq_bob2rTWfk|eeWML ztd+%miu+V*-&0lh>|N(n>7kgy!&BeJjb)gw+Bck0J_4XQ;VJs4*hN|mcSawvBTQ;?)1tOPbXWIeg3_q zk^8ZiuFl4acLTv}uF9e2L2olwXVa=QPEn2~?KN&y`$K)=-ZvFX=C5#*w`(2`4rZdj{lr?4wCS%%Pn&##bjxc(QS>z3mmx4 znH$FBbND(V0Rn8P3ojECRW~3eXXRZI;4O?6T!i1^&+oQ`Mm-6OM=q`*>tSWZV|~eh zETKmQ(?%{hEYNMG&)imY1BmRpqr2{~B3P(}A_380yndd1K;mmB5v}@ z`*HIcz%JD7%j8>%J)Q{O>c03fJsw%ld1o#Fi(iiQjhVrO3E%D*118B&{uXH+KQwSV zjKp$Bkt5t440yQzR|*knA062$*C)3-*(lfGM3k#?_}zT~593LV{9cP>Tqp$~~er zk{3=#Zz0QkQ8gma)Vc|-ZT*om1n%ha`EuD12rYx*ZYB!!A3=bF)scj z`3~_jfPm<3BzRQel^2d3gq0HTkDu!TMP5@|7!6ntaHT0McAZ;wr9BC=0$r@;;fG zZ1~@wpHZ=XM!uecsSBnG^zoxhR$m*qT9HYJ#bfk?WX*;FWNx!}%UP1SfZc;n?8xg5 zk%nVMr=BqBu_aKHLB_&pB4y6I)EK9xxT;d3SL+h!TVvK_36VwEkv0$pm&?uPq=A%a zQCcbv2vW;B3WN=CCKIIVIwg`1JE2oQA>izQiere=52iv{a5}sudV7@G`@|`ffT<4U zXa#U0SrYT~=&F_CbJCc}?{#Mp`&c(u=o8?U$rSC6RF;isehKsYrU1Nlh-_`fh0iyo z6ElSM{I*urs!KgB6;u|qD(gTgbF0}oi(?LivbSh6kTSbsX` zzi|@8^&LSpHq!zT+zqfu4637S;^u1>P%Rz3(vnxZFQ{nNg_UR$0XjIev=Mus!w8yH zKv1O4o>0Chsvmdwh&0&(3N4eLrPY9a37$e!?WB<42%&?_wi{VJDn1x0cvdneyRKmN zVQ?QXR~to$NJ@-y$1LSVTQrbf-w^dpO%1l`_b>_(rngmpE8WTVEfWw1AyyesBW>%)66k6G#4qTy@!&o2dqqDSB z-jp?@#F++Fx5g-+qXOm{W}W2E$UHh}y=YKPNDb}UO5-4HOKW2#?Zf2Sv|qyD*)RvK zvOo1hDU?*sq!bD@8&RNRwR!&tRr|@ z4OxF&s=Q>cQ$GEYFWZ_7t519~72Q?^gpfK|F&4eJj~El9D;Z7T8>73u>+xFX>#9_L1sG_g=#u0mF1C?fC<(3o_{QmG&x1n7m9n(DY_ zo#|N}H-{!$qwKHxe$jMcA3y}V^?Zccqq)-_?G(VA)^9?l608>($Pw}U-NGnGSU|{t z<&11?n~H#zGl8{~E8{UmQtvNJWL&}8?Wmf#LZP`M`E8ldL1va#i zm7%?Zospj9&ns*F&(Mr)tQ`3C_`fb0nEz4A_Gc+z&rKy)8zVZkkNTgD3>={86dm;) zeiler>X{fp)2W&nI+)_KbFf3xi5r=jm^$DyGSWlSiJDnB7}?>|iCXA67zrC0SQ{GQ z(+OEySlcPu=otX6g^ird42;C=^jzr#%^d9IjO>K0Ep4o=jI12+0d+|mS(yOp{j+%r z(7e3Re>U7@`Lr7t4hRUSC;PS78lEj1aQT1pj|(gJ-KNBUunapr`(KvfVEBh=nEo;i z{$J1OkDC?#^Mn8A!z=$Z61CcbxOPm{)WJWZ?178{i(jmP-poY?YKUm)sKbvyR>k)E zlAbHha%=h#YaI52TU{@uN(QInph^}hRnMCU{FEJ@0kpu&qr{xG8pp=d*LW4qyMDLh z-Q!1+avkTfwF5WCA+SE4W}~_ZXtu}iX!b+*7(-yLIhiB&c&|N2k2<&V6t~r-d0O+E zw2s`vZ#u6h{cN{-#(kpgFH_@zn-&qixo)K&-)>Fv=T^`~+V75zVK|s-9GN&8cJjQG zt=|-8Un0mZ5lSmZx@NgB)ita7i^J#JWqEa1H?&P5dtdt>=H4@ve>Arhz|BqxE$}6G z!$y{GL5w9GQ*FLq-bFt*E(=ggPS(dA%F+{Zd%687WA)#F;n%opQ2vr8WnrePv2a{|0w^V|1}7Z^m-BWsgmNLmbs z7E*vD@keDdsQz2!7sd7$Lekc`uZIRUbHsrhx^AH<;2*3v>AP7tzk(J7MB^PZx41vM z2VJHHx+3sl8+)`*uoLa+`y!hw)EwzIu?c|2eNh*YULXweZ;z{w>Yvi{ddF}CtdC8Q zz)Qm4Uls%0}{mLeW|x@aS`IY8|o(uE$)wNO_hm_k5~DK zuxwi5dnVvBN7`j|n-8WI=Y)Z(=jhst_bvSMC6ZoCsa_NA@OclE$tcGUdOWG-X;fvl zBM(D{07*3}r8CuFE5%~9T8|j>;@K6AMuezVRiS)rr<~}14i)v_b#>$SN3NgxIa8=9 z-8cq_*$l32Phd%CgjQzbqlU6-mJub_h{!LY2|Wibfe}0Xa}|mLd*I2#O7wEX$m3i; zxd-K*p@{JHa;{;9%{I!RAUFB0BI7^GU#P7F8(JzAHLusc|LAXGMlwX}1w6V8{{ZUc zV?u!EYPXx-&@nlcoWF#$%=1`JdMxcEGRmAq^x5Nla6gw@j1;S`a!B^QI4vy!nXvi~ z42OvC5JD{)R|dmig762QdC+WPK0WqyHw=yP#EC>~`IlTQHALvwuIaJ}Pp(_R?e%+k z&H1I2^JH;7`@IAtrGXQJJvcZS)kbPUp6j)B@f?D~$I!*D;fH(Ui z_=mh4VBW`o9Z-mIY34&=r!R8Ui0`=aTq+gR-B%Q5A6_9jj7cZDqb+Qr_CB&^4-()6 znSJFrNCbl(iRpg8*v`>>C(+`Z0%12rY!6Z61V;A^VJ=5Fe@O)wu#o`L zjSug35lOm*=tQl|!<6sNa8ze{+3}qU5=|=ckpxGC&9qrx7 zEzF9;WP&@E`DV4R4)(1LBlQW^Rtq9Xt0%FW{4kNZbQoj3SSImO>|dMvXx9kET`{ih z3Q{DYSkRcd;A|WF&Kf1nQB}@KnpXA}B=(!eZNLWYh#$1M6pn4c=uiq5qL4tF%raxG z8l8~3M|k_unN?tBp0>Ue=0h41ycizb_X)N#m}}DLBVgvVh%DI^*{HQfIJhB-?Jmw8 z(|Ly#QDULWbC|f}Fz@(EbVS4V5GDqAJZQaK&RBs8mAcI87OC%Y-yh7A?xIzTb5KeW zv+Ub#oK+~kmYHLl(K(hA38Xw=B;bCOs)LfcDsis9h!Mv1nQ0g}zdh1jYUOoRM~A~N zk9g+cIQ6_RhFCj8Y0>yx()uBoF-EQaiS?uK=;FG9W~k_(lPXF+qt;R0Ji-rmw3Anz zR^YviHK3{?c}Tf$dLs+&^@6kZhWlI}8wT))c*<2~i`8AlN*&(U1DMgmy6c?VRf0-v zvm->H&+5Cf+N6vMm00dowoAYM6f5w9h-=AD=y$lsuxwB>`oykMbe0v(t%|cqBG~znTj6Kx^ zfbKfaLv`kPuE$h3xMt&S{B-z~*G-nKH2r;t@>HuDmF|EHK?5Zb%Bh$H{k`4v9L40y zd)DH@%%^ym5EXy7meos_Gp||r|xL42@A%V5=f>k*WLxO(1 z{Xlrr9h_}D1?$rJaGy2nt^1P9m233!cDwp$e9~DQ9eLgTnE7QJ$JoPX30#kGp+I8c zrLjPYIt_~j?!no=!LB{z+|HUqlBLp3muFx&=39p((D6DrH7u4A4mN7lRsT}o5XXI_ z#FI5g=|J6C2aAD5+4->IT?2JY5IzsrppP24o;=WkKmtUUg{QWGRrWNsfynI`NY1R# z_@onO4U_6pVSOD=O`vg_wyxsUsSiQ(C5287AMvpTOzqi?)%D(81@$5_U6Ezw8RY31 zWFYO*qYbZD_!Jk9_K|hi)$yj-?WK3Y1jNLD@Os!-8UJ!`{_s5Ie)*YldUiiu!=Hat zz^79%vbT1$GcdBp2l%I=)>aPO+yK{y0sp7hsz4`7kI(wY1tb1X9|};yg3rPH>k=Q} z4AIHiSsN%CIcU(y35(Jx8M!$8;mr#E@k>Yx`nS9LvswuT{GWd4uNr@T`)36*Muuj3 zg4Qnh8i4BQ@mV-nXc^g9SefwInHgx=**VzQ@Bx(w*;(8CFK-gGFfuayU*08VXYFY7 z+ZX@!fliT5$xhG8-sb0nF>s|5QvB1WRuB{X*+e10Qvp0NdySus|LHO+X#H&ZZ@-_N z;b+5t`u)sofBXIH^uO5vH#h#T$B$13aQqb=ETQRS@tJ=)_ICCT_)KgpfBWcvRP`@> z&w3TEC{zL1QPnMYUf;$S^Ofna5M+c)L^icpTkm0HL;~a(=>_fudu8|4OoFjv2S}+H zoif)VOBI4ksIPHK@Ldyopf(hk2!`sQw3t>vahi_&1X}M+p;LMt$$(a@>SVb!~)8i_L|fVz9B45q5S4SvA9D?M_4u+f2AZhmqm3 zpM>c`hmd8mxwf=wZ^@SVeNAyp!D#s>AtwYTLo)Td+BA~|CY!1t!6NH0z#^BZ5tt>P zzO7)jd*z-hI*Hu!%^W9JZXE1a-aDVX*}a?QcIMOO_40V2;=)+hbaqHcqk*2V5Nq5M zG`WeR?N(RTVA?8EF-^gFA((U`hfN-)Vm!inwX zN7z|pX6UKVt&@Y=&2;!jJ=!+9!&hqu4iB53HAqH6ou3IUmqR!h((V=Oq^F-3}LW zD!4?x`r^tm94|(e#kt=S!-?kJ`BV6ybbD!QOW}e`YOZAE;Gl~aVvd=&wX)QY)(h3T zl5mE=$=7Wujmo0=hjFqDhf)KRvc^ohg>YsaE1RU9^W(~~i|N4O@GCpw0M%PT~Qq{eupf?LprBPC~Y>ofG9W=@d^ zbSa&P`%nonBq*cqJpQ0-Bm-&x;%c?~n)umsf6#;{Wa9Y@2Xr!4{_tFCNosCOMkztZ@Jojs8DUTL=^P43$kCXg8=uoS5o6xI7ICTNt`;p z4u(wB9~rt4Na8Cf*;r5LWw?av+p4HCwb4nfXSINITgL#$w}A;7>R_t{7NLSot#=zF z-&EWfK4k9db8gE#p;{ZplbVx3yHQn>N=Z-np{x4!q7WNLN`2B0z+?W3nlxE~@p%>$ z^U6cDmb6g08F=pMGLVa9(dT=f*9-VOqR*y?HVL{CXi{w(cm!(B&E0so3i$3~gphAp{8ZHtJ;u zB)t;s9MyyYVb4!VeM~dFP#Md$=04Flmht#bOenD#FhX~yc3);lMse;cWeVdQIblBC zSr&XOd_wEYhIaR52@Usko4N|aZKG66Q}KXrQ0;c9aZm&|qwe(`lbdfkK;b$^!Hp4z z-cW|)QzPn5g?csK@AxL^J`6otLy;aMR981l%WBcMmIgDypv@ z+ePFlMndh!W{|0{V$lpRoBB3+QBLE66#=8h$c!A-O04I^Ni7D3fhjt&`;E3a@YBcc{LMl!%|}M39egGLS^$S1!5N;QsPN~vn5sz6?VbQBP%?` zXXEC8B~Yt5`RkE57j50NtY(;T6KR+f80PRo^^`{pHkT8^2${Q#0tI)H#4guxX+Y9) z4}I$j?z7m{h`fn<>Inm;l4b^M51u7H*jl#4r`*hs5;ugYAO5~f;X!3--n>`mcuRB% z1|x|Z9&qxo2Sbrx87m}Ha^2T2m<}P|f3RH-O%zf}54RD*bxmKpZfacgpjjVq?V@l) z;KHJd9@O=a=H>=Ow~|XvE9*xTJT4(;C@Msiam&7QCz<^qM)!*Qk+}$i;-Sf|kFYM7 zN1kjT(-g6phD4FaR{lu=lCtZvSH1HvaHx535LM%(HA$5215u4hYYB}GjsRo4U_FQJu zY^uihPH%c3{)2D+_H{8`bl8xpZSIy*nrLUlJG%zp0oYo^>R&u~c6;rRpf(n#2+L7@IA+Iz1vk=4hRntuhVBWte* z>w@N+d~2rnOgd~HHFxYnjqCiQ;>YYNs;!&c;j&^IrsOik7y_=oEJe}!yAM^@0mJlB zXs=wum!Ei~m_nGa5bi(IxF@Hz^yJ}M%ApVrVs?g>19 zjqvXIDB`qo+sYihQ!%`LMM3sHNC_+EHKOT=&XOwlUgy<6N_8;)*+DkjJ?g^h>{H@N zxZO-sFbpxwNAd2{qdYOV8q%GvD?Gae{fj%@uh6kR@p-Q8UgqcX)?-1P%~yM*Ry252 zbgMwAF;y1cWq0!HB8K>O3JD7?*L6T!v-|7I>#lO$Pdl>}MJ?MTW8G4@XI(AMD~$Yl zp7#~K==_w9Xw7`P#00*Tl~o3ZB5wks&AhzpcuBO1P`!m79udayQue8XESqn0cZD7m zH&J{#D1|rg`&tY+u=p}}Q-$aZ7~JjYM(Qh_B2uHBB4#sagKnD36WxhP?Ovb64VU_O zqKF*dk5vaZm@*f6zS}tEEQP2!Gjz7)FoxK*w2fHd@#0CCKF;)^vn+ zOE*z(7HGl@;0EgeGfCawi^EMZ)@ssR0DaTKS4?j4)TG+Z02#D-;dxWWah%W4rMVXS>l=j%1rn zjIme&rwZ?LN+YSVn4dk4U;5{U+VBAJFS=Xs^?V#yc3dKpTkidYb!6!Z5DW6QAJvT2 zjwxqo;x-(;>7dtP7DwSsPBW4TW+Tk!(>yJ&@*I2|Irn;VqUH_91{z$ZcjXBjB`_23 z>Y`g$4FgHJGK;HQA?Wg7D49&G=nuTIZ32%l!w-RgWhetmnX&!w5*Uigy16FGMtJtW zag*mLGJM7NW0oYl=9p<~B#RSmei29N?*}n9w1CACcfXFp9GxE2RyX<%6_=nAIfuKs zG7meRCnpY-xVTeld|isfg4>)I1b2)|TB3mtwG?wI&z!C?c)vXGAOoy4P48*>0+N5d zZnf~h#8#!v*5~-9{VH@~$;~2|V3tuH z$`h46)%-ZU7E|_KBw~9x32t;gP$U2Mt?m~2=+9W*PHX$b6)HhGByq=3#f#r6K zp2V4`*9j~K4abR@6a#W5{>Hw(6)d8KVodAMn3lB*X_9`9_Q{%-3p*|!f5-?iPYSSO zE+vKDmzcd14fYS9ty1~Qkg>)3B}}QL_rNJlW;t6E1BlQ)yG-mWI1XV*pl*ym3WCsC zEf0BpxrDb`m}ubm75-;H`ddD`^Psb$E?aV*`=wEy|GRg2HNmJ;TFg~ZA7k};MPE5= zB(YKSTY*CcUw4!Pp9{WWHhF{1PQX1c(-aZ4zOWQ%?bSp^?VjdMPpheauXINi#Hj*v z(05HRx!ETnK2v(g_OWoTQ*}B~^!~VkXNsi4UeL=Sxms7$3YluzKrf;*QQz2ReQ{ZH zBI6vu<2cB8617_HYkKpxih~G`AdYI%O!HpuqvducNy53}c)^SU3d`|U{j*&~Ox9`McKrO|NwJgw(QnwT)T|y(+)#-N2)f z=bqa=9`g_tiOOv}9W+ulYDoy-2X@EdmB&4TU^CoNwSg>!Iw78S5nn`nZnXkA&$lO8 zrlGeb&c_@aL2VD3f26>Lc1LGeJ;5D5Yg_Hy-=5V}$aR3)3A7Z!p`S@*Ym=b6Uf08f z8FSGp9}OFir0WW3T3gqwS;vuW7B5*ibXJcznNTTfskD6dwze!gpGyOq?Ah-ulg%yj zrTDP%93*xuFamLXYKPGBy_x1_CH1@ARV82DTlWzkjTV*_um_ipr_dAK=>*lD|1AzT zIc-n@d{6@egO{@&w&U1&saoQI`YFf{UeH{V5|wL1A8OD}El#_JOVL6lOUb2~HAE~J4)rzl`jpG1 z&Iiw`+B|XGU9?c$19Wsw5FI_M#nh9rLedBL+S{j&O&+J#kZ$f_+Pz-xGQ3`1e`Mnf zeSbbb+8=p(8u32-UW-8Q4M!yB=uETx%H+3-0m&q#h&RZ4M)>b0 zA=_SuVLXu)Lx=S@n40P67n=m1P!}>Aa4N0Vvx5`$gV>EciVC_-^JHwh)BADJO4w)A zXOu3`S3YdeIZ6jT)g6Ylb}ST|-2|#y7s14E+~{ew#UvGXF|)hw)Ih4Fq>@)Tapx`?P2KnHiD-*mjJgZA&}!Yu?G3OrvePkYm3i*yelcH7`j zulH+XWO?f+-n!Yc+X!;vRX%teKj@fj?&2p@eT3s9DJ}frdxvL2&=xsBovBW7Av!5| z6Yev$m6NsX-7wK<@hFE20Sc13ICM~kF$NnBkKgtkenl%-)X7y^KQ#b@_+>4+19*)i zl%`N@pCCk@!b3gx#F=_#(oC&A>)60nTusy|TIEh?_7Df?^H$jlFw2>NQ)GaOTsoxLamBX76XlfaKRc8D zbq?o?+^DTF@U?&+PcDKN?HZYZ0y=ExGkm?k4>5me)}y+H`mgtcBt6eQNbdy6vtx!G z3dCWyiP26JTHk44i9~h_hoC3k7K($Xccsj%<<+-d$HQDs*ARd8X%1v$L_HR5oxc=YH7r~|z}J8cWb zDd=SJlfk=)qNNbji;|gsu;}4<5!X=Fw?})bm3=yz4-m_h`_HWA3Uz#nojP)ZmF(p5 zMdD$!lby_z37cK@*%yfBj2cEnp+>wHS$^Zpfp+mw{6#LZ&$%oOK=q(fym8YiZ6+1) zgW?{@EKzMwv)C@nLZ1a9{=duLKd4 zz}@xT&hwOHokXIs?vd}E7H%31P)v?#gn@TL8m?{OYuX{F(T_KFg!%axL8cxRE{S*$ z-x*7OQ{AdnOrTLv78aHBGue?}%cm_DgpkFh_WOeA2KG52j)aSX#MVyon%L=mM9jL3 zpt~VQsCxP;usWxCbe};#+-M1eI6))3rGf)NVIJG%MoD!$9S&Aq$S8uw&cS9}^t=G# zr`-2NeH;_Y2R8eRI+rgyAu>43m!mb`zdd3baj}!gBF?7>e)y{U(KKC4kI4`>(zEfY zQ^$)8*{_S81T38l=o?8fFsM{uZ6lO;3Qa%UVyew)vUg+WivuL}%e-n8MfxQ|u#cBT zx?yQ8$RN+rwx6I4aWc!a_G}2Jn696UdN)BpNLEp(XY!K)3|0r3kI9V)D<|+Tk9FF7S6`q zkeA>d!3WOayodH>Qq))-(E{^sx#9Rk=Yh{BybCmfh#klSqV^c#*SgzDgO z8t(CUqL#L^nJ>jh_)jx9RlF?At1!U1d$~{PWd{M?h(+f>ti{2icrktqB;-e%w#F=% zD2YuzT>|oqVzY>K4q3CPmHS+j*X&y6t;Y_PjQxrhw6^?NE$7=jH5iLubC{(VjYvS$ zU0ZZyunsBXo}gA!@<%WEGPkw7c|*pwG2%M&+jbf6*;88rt3bniCdgBVJ+X z+B(Gi|=MVWztSu@f!8X+lkoTyBLAvhisgW zUgo=+6kp<5g~CEHJQV9h>#VpH@HP@YOk;pI=LRQW&+fifM({+PA`$BUaI-66!grn@ zH(ec1%kPG`_TwaRVVKLMo95oFcn9np1^s^I`*)zQ5y}28f_>)Ztt-pr4&=Pu5aBl7 zV|C@{)EH5}&;Tz!Zh?OGq%!sgyHrVH;WY$>63X+!&w>=Trd3g;jXQRuo(#U%9ZE;^ z-}jr;%_vpaFW^C~YIo9qOvW#qI*$dVd9to*j|6G88Urh{j&h`bsI!^rGJ}@{uVj`;)-~_fC`2q|EETQ zx*uOpu?{a;-Z49)*mS!`4Ad#3K9sch*r-DxW4|{#-&dJa_8taz$^4=Uo|SD#Tm77h zG>Q$n18L1WZ|K3V79vBLG2npcpfEBHjzy>gPS^GsmsFZIz&(t(!Uae1o~_3!%JuX;y#88pVv(&{D>-l-G0y;L+ZYJMJgRgv| z8fjtyUm6@a;i?rwI}cW%@3)U(s^%J(mrXWdD&wd_-UZ+w`)TDPH3{H4_$AXS% zQA)l6!W}x5B)m@tgUqq9*;S--jH7ytZ679!DS*f@v~{DPW&5dlJI%cfj^&J9SvzJA z_=&~c<^ZBQ2Ny&v)OO4KpabPz9WHH%0PeP3u6d8-fBTLi|#x(WjdUS zEzpm2MJ4GMVM2k|mpFardwsc#Z))>Su-P2bw)s{4^7?o2U~TceRdE=XrU7oZ;evO_ z6wFI8A~|J=)Iurdy!%LTWl59Mj{SlZuwS7U(uF@fp^_0?K7-^x_AOxVFROmHtO!G} zm40&4!~8sfU=Hqinoqq5#zE)WEoIR8;e%lYj*{YiwNrgA4gP`(!6f#g)5M0am6?Tb=9VZ?E066Org^+%RgyK|I+69kbUxg_%-{4t?*>W!P2e;WUah>U9aCq zBZ7vv=uxZKjcDoz8C>NO@1uV;AI6iX|A}Tx`tAGS`Q+jg|GJHox_kS#tmNKZh*0mC zAMQbG=u5Z$gWgJnm7VPzn2%9nm==-_^WqSd8kyEkPkv?@a#~N|_~+hPZ5ZHMNw)Xq zfijOMlohD!n)z(j^}szFp9=Xl2e%dM<5}4kn<{&$QWlB=8;*{6KU2AK`FGZWS!x6R zC3-y|IS7uY(Fv=9jsV5XNI&UaRFdjmB%9c3%@NpqmDNdO$W2qN-0#FxuQ)j@VjKD1 zjQsh7YcY#;c8+xt?Gi}cnqJnD(0_ALopxOKwXx)_t?*ez?1P7ar zEr=yJ;}^%*N7ol(qgu_u(RbpAIiRJ1k?x-f%2B4VOJvr8!5QI+iyW`@qaefuT%*75 zfui2Q%^4}&gD#6#;0fIQ$jwY{7Y=EsH9a+3n3}8J-dUxP7+D(xJ3esbYAkZ5V9@7; zNLd*FegY&pmp1=Js`Q*6521z!l0xt`iU_6!{iH6jtY)s5QRPXj)brH>9qJ=h%Ug3x zvAjM(qUCFNb)f;Ck5-Fu_)JoIu80 zd%5=M4>n(9i_>rkW1c{}c`dL5I>Xs->MU!5j4ic$NtagegAV#lI;L?rL#J3qQ^TE0 z*&3a-3XOb1mLEThnAQ@a7-XMf&^GEAI2a8ozHVi7Sia4AJ@Pp(GcNrnO2fkPC#Ct{ z7aINw+xRP@;~)CMU%?&!4+{;yc;EjdCb0hw2WDbn{F@>EQeA!$$Nvl3(S!59#P?TODO$u3&nEY0c>$?&&`tb7KW__TzDi{c|S^NL+dC>muH zGV6+y31P~DqwYSWEnX0QX+jbdpT)n(B#2Og?va=w@RJ53#B#A5(jPGUyu82%?4KhQ62){RxQ9EHOJ7DI7>= zp0bOMOyzFiIFa__O%zDmlS{$K$T60ur?;0=uG((K`TgPcrjz@RiS`LQ)R7b(O7#6R z@H72r&HCeB&l6IrO!A@h2 zf(1kcj%dLbF%{LxL4yfaur291fWwgyaL|f1R2n?RlNBZ{zi27Qe}#ik8@(*$%@6lK zn>oQg;0a=yNv8Iyx6V)Dfxf}cB)tB8kq{5`aC`egej1L7;N2tnL zK)M2VM!tD)e6S9Pya=#oQlh2B&s&JGS zAyG1|x9z{&ew34q##&80e7m-md82)GAQPym-+EbaMWoU&s|zVtlytq+)_J(T7Yx0h z5NrtR#Z%>A7h@B#-g?1vd2Ek?!`*^1PMj%|Y>0iM-+HlDm0*kSuSA>2*rMJgKnccw zT93KWcIuaM#oI`|2}{+kE@7D|+bS$gXp|kE5|87|5-T$pRi^w~%q1TEck5rKW9Hd~D`^8HAsvKTV z-%^}OOI#>Jdxe^#O=d}KkQ~lP18XDgWO#HvG1j4cc8U^FnJS#C)mXRRRj zXagL}6DAJ5y0Kv;eerP@e64?w2bXD>(HI)X`gf)g0Gl4*kfm@PyW02l1X)a*To_%Y zHFi~tYK#YLJPf*RcOx+^5)y1Xe>}68FWq=82wt)Jxa)VZINQ2|N7i}hORwl*D67Q# z7W7OSPQ3902?DCgW#$Ou1e}KDz6feE%zpU|ZnF zwXhX~Q!INV)0Q|`Coy`-`%7bn)rBPY1e?*33ZjsukR+1N6hrgA~-$GCV!$w8^5v-)RXf$}Ci-KH;L54$L?F9fQ5ic-rwWc=8XwEzo~S+r_zluk8a z)FOK=syOewi&M+)JQKSRt=L<>kb)P#u2EtbTaVp64P}=Tscc8~VdcY>C>A{o#i^5| z3M0{%&0hX;@x58!^J)d%!#d@LWr~#sFr(nz=ea0ZxySi*u2$+=3pVuN>~{!7$v~bU z^VGPu|eqO0y3F75`lFe$ipb@a}M4pa;S}mP6kz|7Q;?hE^)GdkXAc~iDORudp z9?9p`!3EMzQjHd2H(eV#F`N)6}}VMa4V9PSw9sp z^=Q!dCON4jd<|Ap6nDtqT(y(2sh8S&P=@j}l^@ROntPNJB=M2$t`#&(Qt;I(H~k!{ zQi{R5Z{l(BFwO1K!8=f?d6`f`@#!=sKWmH-&nQv?aQ<3S*5OgyP~tkU1-oz~FplCv z=@LF>qdcDColj{ndn^o>wh7jZyN*wuDj`Z8%tHKgyK4e_1~v?L8Bz%jX%tDQVJT@x zI_vf`R_M*QEiawEVwjzGev7TbIXF4agEM!^+ z_qK)Mb&>3%p@uq-PvlSQ^CfBe5K_l$721>;J;hR0@7u)c)0~-E44N04&2<>G!WSbf z-6x~u$)z{x66NM?uZ5EL22%E0CI(bp5_;6x+R3aEzJCsz*(R1B>FcyFa?;0Rf&v?>ylO47LA|Y7kQ;5x3l9fKjgjnEgnJ14-~!m$6kEZAl(|c zbb%0al+WOsKJ36Nyd}UCswqyl?f7!AiTT+OIMW!JR}=s}9)D&xspRG^9Kw$CluhIC z+*3-um=63FT}i<;ekHfCKWR3gv|ZqC)|3*|%FWGIPw^hu+RV7DtJFT;eT`)coZ|eJ zyrN#AOE+3~&Jbsg z|Nb5EOIDw!zg%fEtmb9%R!5@JRyz?GTzrHV;(7j@ecJ@NjnTDR&vm1(0J|^n?Ag!= zd-3WD!);Akt(ItjkxC_+C}JT`{rkOD^OXpQ+`ujM_0!dHp7WSEOaE)9cnTwQP9*{w zrouW;_=O}Vq_NXYJ!ou#HxtP$BZ zW+i_sxZWRWZQ7Cm{RXV`Da5Vz`x|FTgWnuvf*i&3Owg1Oh%4O?5fksQAIs9{Z5lL9 zfRSLBQu@|zblVXai`n`+vnE@+-D6mSG=C+Zy2~qLtEY32uIXpg9ivA{yIWgg5OR{T zmn*Xh7uDy&&+k5qB5p=Fq&QcUHqSF8MK4$2v!Lgrg4D0D6^=f$jOB(V@jeUSQ@e^E zASNfKmBY}bUTS|6&)39cB|=tXd@r$2UR3XQ6vI0@tJvMBHie)*L88ZCb?%10N(W*t zS%j9w#M}LKk+Ha$(Of0M3bVD0L!;=ZW-{@I7Csd@-nXO04`cUyG**OpJ`|=4&&Ysm ziEi|n3x`dO^ETBcHEH&Z=VRwB9krKW4!B*yyXfsXRr@!Q=O+AhBB`MFkW7=N5Ot`% zO)_?^6s{j>j9YCs9*EMK?Un8li`5L*G?Hhm9odfUd06bxry(jdge45ns+JLlP4iK! zQwF(!xQ5g0@DNKM**O;aRtrQ36U9;(E=<%ysGOybTm_FFkTGJcBG-Y9Wv+>@mF!{` z9))0HtCeH*@KnL|3x{o+r`?80n3Ip*v3}Y}YLzAU)`K%n-P%a{CDZ!DboKTRpK}CX8H&3 z{-dIQLDqoi6$N zg~W7aWnp(%BOzFWAcQo+Vsne*gW#-UaI0C!KZtfdueCf7EX-3cs)d;;icC(*kVZ&PxThUvBj-@^IXYh{wCvb zC}*B#TZlM2F|Kah>((U0X`H$gc{k}Xh+A&1oGjehlts%@UizxC>>*}UDe1^+Tf-DR zreGuz86k?U^R2dc3<|ma%1Vog1N4yR%^YUp{L^~t4d=v}44s9-{EM}sH_#a(w#|Pv z5EDJqZ&NTa{T@sVe@Dd7i8@*tIG9;m{R)f$BpeA_Tk4rv{mips01Q(FJu4FO|C!xH%gRE}@$)kqGY35&N{5!^cR4c?12f=@ie_$r?=i3fF8|zbKvB=q z#=;1n`L9H#Kb!V14F2?XzP83?;&iUgVlOJ@`bph1B)mE{E*l6GXoMQ}SE$ebxXl6+ zJ@fBw`47~;wH|;#`fpLwGjIUrr{C!Rft#Ldj2*L^7ffoOm2E%9{m@oX9= zjixoXV63Q!JPi%Crmz1XF1-r^s1g#k_&@b$=HDqz|3Lb?i~oDevj2(dPsD#B1tb;z zM*Odl{)$oi??L)|nEs8F@wb`(H;@9lFyLFXe7S#Qo)v)jUnBk5oPX1M{|{{X zJ8_eg)wMeKhd{sSo!1JkeG`xEiMM*6ck|4m4LPeT7d`n%}gBW3+*H-Jt5 zMEnO*zyu9wH!b~7#Qz%U&;R?s3F+@xx_==3-=C2HK$(9c{S)y|q(3qK#QLu>{_-&Y zdq(JQPZh96#P<8`3V8Yajq|^HKMc%&g`6_5Fth&Sb@)fd^#8h{bDA2J*o|;reEhH9 z+DHUqIdWa%p^5lYfh>Is;NrMh;3E1RSNvO?Vp`sxR-TKU4V&2V2z4?#W)+s4fIft*x-IDPto00N3u^6NY}3zYqI3=B}lL$WIU2) zkj>}Em!ul;vyKSxOT{C;lz}cJYRd8zuJ9h+OB~>(Qls?vjfT$ktD>e|wp9G80doMn z>pZH2%a?GlKV3iWY{ZiFWe&Dbc!Ex=6AZu40AD{&9whqu5r{!}p7id4c-zg5glnI9 zgwX{ETOX(SUDKf&hiAMtj@Q#)thrCLvf3lnkKT=F~XM|HmA6{}1b~J^8W=T8^ ziGe0H9L=FY)?8+^JfeV1ulFUtL#vjDR539%`M3O7ER&_I0l;<8Yl>iSh9^Ij`HY#x+k)|W&b4SP{OV9(XL zu1`yq$GdCuONJdOZ=9TM_dmVybh^Ro>9~DB!d9up>;Bzl*FshenFyBTi@7~A>ZK6P za-Gf)Bfq}JV-HW;{ps&rxNn+1@I?fsT@qfx&2m?14ImjEI$C%qz9#bYc?dV1_mt&_aj z>t_pqqw-;SzuXFE^p*8(n-`eClN`x4vU(@WVlD4uL&{EWwT*QhhE_Qpk45cPVf&I6 zjoLx^vJc7LQu!t;Hg94tj%V@kYJmH9^@G>tGV@#mIY`mipkDQZigFq20qkS-oG5*Fgr?ujR+a^d7^53|}?37Kt8k?yDd4_7_A*4zt}`C_u3*xlD#Sz?%9!g89wW z2BP`zpK(4H7}8O_Rz|w%ymHs^dhWJ4Qn-n6d`mR9@H_7?>i|^()`!tz=G5Qtd?0R!e%e zD$7+WxKQ&I@1D_kH5u2y)2zsCqQmm$hhIM9%sssowWjB3Zsot@dW)Z@R3UQsj$*EO zakcHuuBTZTGRc@(Vj}zs1%fK=bP=~Z0$b$-cUpVDIv5zae-B15%2KW?f!EM8FT8pj zYG@w!g=2y>iO~6>|L{ay3F?tju1Zd|ZJ_+{L}3Z8hooPF?V4I8-BGtgW7uPpk|Pyb zf`il*Kdtc`cZ=_)73EsCUy(h5lQt+G0`f04-g5n5{I)YwbDkuma?7|R%Do}xozA0b z+p4ZUp8SdzhZ01a(}T-&hBG4e{mV(F9)lG#Il~U`c-fK^z9ytplu&vk*Hyk^kzsk7 zKZ4iYt~@cBzx(1N3opCgJCa(56(>{AR@VWNZIk8FZOxANDGo?lu8GS}UpdaG9ftQg zyt9nj)~wtke;s$EtrsyaJ+jaBY(&3ugK~{PYNYmVPs-pH(s7VvGViq65$<6BvE+y= z=d)wyk%QyGrzJs>R|g2@C;MB_go4YA1cb9^`@XP-ysA!(peW<)$2Y}we<%8mZ<^mU zF^8e-xjKkxFlWWF_vy{=^U1hbFI02ByPT*3dfiwIhra3li6e^5ljB>m@&V z-BHea< z0y$A>+q##qa`{a&Gpv=<3zEw!Kcg(8^y>LMa!6Xf@8=K~8?cL7gBhpZkacCrYd$n# z##dIxPGp?VqfB zCNy~@z>0|JuR zBZV5X<hOnTkC7sc2`y1bn?#_3Z0cs&vXZ-NX~s=uD*s z2`8;H#KoVxjL+Zkb{|WrqJ-bZkIO zWY0^EtwaqqT&Q|%t17R@eUePm{hY`bI?HYYErK*R)^ ze!(m@cs*PRG^mJHq47L=L|H*d+GlL6s>-^^&L=>yz*harQ!mir4cK%I_IYhzj%Bar z0Io*!9Hef=7vyTASM z=Cb9zKs2vsUekW)%oxfUuVj}^HE+gwbAaJ)p%Pb+ewu5EM;mv4Jk5XH`1qzvNZ!y8 zc@ra5CaJox4K^i`gfAYeVOg)L$Q?b^5A-N*1g35><0B@F<8K#pr79Atx?m2sXk2tW zuF257qYAUuHsCY6QHcnmX6`I&eD1w9gLiz@%aI|uahzXOfcO&A8yEK-b9#fYMI2O5 z(7sf}>nHEN9TC$RY;PNbzK&$PGjm17DQ)KV-PQqqTKszW-M64jzE|qVtTIp9OyObR z=F@cap9D&5DCk6FZU8>0K5J5x&=Z|x_l>q&pi$gKy&z3VxcNQys+a^juV?5)Yw*B} zFh!8w$Ku3iqnytwXmHXz-EydTE;T(4C>**KBv8`*S;jF?TP3j!9@GdYnE7=m^m&#e zyP_&mP$U<9av9?6hRrRKr%%~#X42vs(CV-WMw490C|4|Jcl3*#s(h^^-Kti#zimK( zf%T+nSMlx~?q`<%Y`j3SXmTEZgZrkpJxFC6(9J%gU5=TnwswL+l45Q3?mmz#BOZ5hV}LhyQ=hFTj4Pw2ofW>- zhoH@^y#2gfMTE{{Fk?iQsC-ADr#Bw`A^e0>OCO5Xi{M*rsh@blXNF$onW1J9b-dW^ z(wb`5QUryxOwFYC`h9tz6ND;LFwF8Wa8SwE0%dwlF|wlPz)gp-OW({Iw(P#!^TOC- zdDP~v$SrAq7@VBXlc)yCZJ{v4+n8vrmRP^ z$J>Z47pov*ViEr0Y{no zxK3iZX%E6}u~xBhMGPElG>3QZA(qGWkO-)fbnh8S@S01UeU14Xkm(T~O(I1cwqDvo z$-Cg>C&DL9vY0wZ-3woyBp1L;lW{MDF|#wAAAC*V@&FZK84GWuTjuq4d#i$Nb5e63 zp?pfK1vEt&0z2lp$*>+_MczZ}!6Fy1_CfB_-6F=MyA~PPZb8#VN6jQTPCHnrDKs)S zI8}t%R%yFnhmT>NXNUmDXP-YCmWh!C+O!0LIXHePCx5oLE|!z5KtuKH21XXp&x2ec z*ZPObzE@Sla)=epjji*z$TJljH?hU68i*5(`u_KNsKLlm? zW*~1iH7Q18glg-Ee__!ddz!8{#@~ej_coCB?Ul#3)ptCj(11I zO!zbA(LHLz)&_sv*1j#}R_tIpcyK_v>F3Dcm;Fhg0lq6s8*kq!E1&i(sehjqAF zi3QKdkjIXlM{#hOK(o{L;K4Px=9@KdME{q%YD{CZER9;RAaQ`{DQG7Au~6HtyDr_O>As zXj$7ETT3n0&lxDVi&`mgB-~ZK2$k#kyJIM=X{zp4iWZT}G&FPzNo8L)Y?U#6F49+2 zWY>jH`2=xmBO0!%#hT^L9d70cUvJsuSgWahXR>*3_1o(Dc#~p?BCpff?r1PK6o)gl z^sG+y*Yp^^7^@i77?tXT7!__~;c#ITciMH$G^W`MD64)GZQ}0{Fr>s8&{8WV zKqp9`AAH50?LPf3lwv5#t{4ZmT352s9$Rd=dN$!3*&Slse)_}-6)B;B44+RpX18@W zv4}#GD+WAb%wLA4i*&;puJp>iU!ZgWB!7b;hc0QezBZ6p3|_s~qPIQ)c!ioMRmojW zk{dS{vGEoM86)#mkP+3bRc_~=y>R5?7p#+XICd|sfFIsdDO4q0CC?-!5MQ=*NY7&#qQ=to_m2BC%+6_l@=|mLM9twcm;gKm#mC)!n0(&r zlUwQ13SW;j)*S%D0|r~7J0xzQD6C%#fHSQ81OhVQS$ks4Wz;9FEhmx1P|fPK1+Kv6 zP~T!^l|d{eHC+}fIzq9qH?Ot%pvA_Wvd0gIWKs%5!bq50s7*HCYTEuxoLdid&09=G zrDpD1-CFqW>SaV!3XP(tmYsB?ZA+|Vnxr~9^J%5Wxx^dIQXTz8I`iqDyh^{5aXrR0 zI{76+EZKG1dC~>gUDb(fKPQDGZWRIi{D4(4moXPV3r8~yE+Au!X&`T#$G9O+*Rj2`5+yY_^Rjq!zI2o28I&* zjXx4Iv>B~N%5mBSCuLB1ph{|CZ)QdcoI^;@&ceui_MOI0+Wv7Q+v!$uV>@8@MRj!S zpw7g?3PTD*s-;K`oEiu@(U6f;gOq^}I9-vHL7tRB#8SZ$II^1*VDYp6?d0Bl$cc== zd#AG%QU+<@AjOl9oHnaZPD+G~%7M3_9c5qwcQTzjIl}QD)(dUI|F&Kj5DclDlsXs$ zhWVHEp7xW^4t+d3LGtH%@7w4*oU9kJAn5YX&;OYiPsSCeH)jLOKN&LqwtQwrW_l(- z+^iSm%?gd4zpoz*Ll0m$`Ah1m4rXMORtFm7{{ZE56>{gGKvn};`{~)4ryuzt`qQHy zPwo!@&Vf8_1OF;|7A6*;bqzG=FRmRr)cW(P*ZmvW}RyKKcCNL{RVe(i10mJEP{(=GOHs?(54{q~QVa%uZHT(*Rm4%I- zjhzFSO6weEB&}9=Qo==K>#S`zgzt2GXAo7sK1 z8InW%<)Zli8YE7I0$uwLcY){{;w}(LF`QQpU~eEcz|m$g znOJ2`SnnV3oi6PhKDOVZ4%By8G0?QwC1;ql<728F2WBZw}TvQHFC(?s~`SC^WKD{s$3)C1PmxppUL~`ehkr{^c ze}U~!aS6iQP&8f`8ad2>D&q8NjAj9oAi<$agRr=d0OHd)OoQrvoZ+{g6G>rch@jsaI z>59+egcMW|Q-%~&KTP>zsNnd8bHJx@<5$jMVg<5CmXk8=y!L;)t@J0dvo!0!uTV~x zaSjXAu^@iPagkz93s{IBGM$~U_KP2a!R++F3F1r_*M4#(sO4Tu%m5aIp8osb#^qQd_Q-swWm!-Gg4>R)Fx^(!ZG{1O^2DhtRXfaHr6bTRol?Q}rl z{qA6ZbN#PN%hToj1q3v9L%8+~6Hxe`H$V`OKm9a7i0l0{z`qgw*_Hkb3Gk}3F#X?F zFK27{&w!v!BPgH?BnV>sg)0DSJLgfr{!w6qJ1cORb6f!e+I)ZWC`kDNBMM^)V+&&^ zuV~15$`oK07NC{E$nF z{+%Hp)eDRufJYa`6i6~`{@0i;7BN3Co!Q`-V1ML~pwnmZok{`PSp)qzNEPU;{b(rt z=xPG+0D5P-Sk!=iJb2~z*4ha#oIr%|1CZb|0-ae#Pa8QYSwLN^58(gjY~pN9=au!l zH$fuoFAM?tar?y|kuU)fm;=~!f6@3(FZE}4f+7K!0w|S1AmA?mhmi$F2FB_%J(PmE z4gcm|{skRGZKuNhsWyPt&c!a!j}tadz(UCAr)4s6fa!sa*+AWP z-qTJm^`F6>atKhDd0H5%+sLVdnL$cMF!I13QCTUq6CFCZ{!5enY>EE^&fj7*=)z5F zOd!xt4+MiCr_}y}lZlZY%*+g&9&-WbfA+vLGbII1?msO)!7RYnpFJPjlg^yVNo@ER zWBrpCLW~uv&NKZ%QeU7Eqd@{4}pKSqFwt*3bF$2Cjp8P=F?jKCx6ySMF zhnO~$h<{6PKxg;l{W@I&fq(WWz!zs6tRSHJVPXT4g!7gTKKYNJQu;l?IT6FjUtmV_ zza!$)HJt+nSvu61!56s+{4)#q0}>|S*!o*jIr8U|p7z_A@Z{fTTGPtp5V=4}1jG85H2}d<5Z}Ut=lw7d~P+{pMGI ztRRk)I~9RjATOx!gu4DSK%hMOjcorfd<0q9uLUmDwV-_TC++=AyZ>OHK!yqoMA(1| z^8&bEdz(-`f*K2ikD%QBCr2dxHR=2R%}J-Xw4Jka=+GLryE7LFB$^z-2{$w5&ARhvr z-w3$HdD~_BJGVkQ*>VJ@qzY+-LI&dhz&EEW`U?(-p`FU@C!qrNb3UCtxtZ{kH-Hr3 zA}ru7dLSA5L&!M26x8m1H!MgS`?m`JH%gV&2L{T_0L7lRpMcjO@Sm9XwBM_R`Y)3B z1M@;P3YEm~%nLM^E=b}R=4FKhtDg$~Yfs?xN`Du`$sp-uhm1Z4pjb9WSs=qbMg1?R z{*d#XNA-Jhb9O)WZ^;ewFCKRh6%!MPp5ssI{c%(B#Uk@Wy%47}2Ou$>6e^4t`fjFx z_L#sYY2QB}JzeBEONL;9dL(2d`)jB?9gdwy9xz>2=nU~kF2KUhOb={81%RCQ$kR*x zU1JchglO$I!#xY1Cxe&^c>e|E34{CvCB#Zk#eVvcA8i8WUt;DT6bKBaL98e3k_#yR z6f+^$hvx9mIrl~XJ*{1!bHe`>*pp?S1AE?h|KNkafV~(Wm_V%b96)@ySdE=t8LCof z6oir=Btd{0)OoP~KgUoYU~ANQF)*Hwn!lIgkaqZQhW;xvT`+VeRv@isV+2Cd1rzvt zPRM#zoctKR{gUtky>nL9hk)k-r8$`81}Wi>o(m~ZtJ%Wn!3Y3Vn(=AXW(REiGXREk z#=txJz)$V~-`fFSPu>UmYrvnAs*V)sw^;)3>cemXuW8Qx0e(jv`L|D-1K-&Ke{^6B zA-@B?rtWY5vn8Y#^xGOpiT=eE|3IzKIRvCB32CDKs)y5!iV(s91b(X7pAmu;7|pYT zSpSf{{MZJ6kxWnY0F+iR4*&t3l)g$ZOn=phC#SkPI>4$xeZu&AF|BWZ^=6pJ(P$b=%B(lv(ev#@k?O2C=4Kt0}j~W0Fv_aM)~Jmq^E7m6JY>Vm>%6} zKJ**&{%j1LnkTSxhUuhp#G(XDitT|pu@#K-Z+*g__WLjLxR~04&dCGXQisapVt)9` z9#!x~d4M?B=vkP-e`x2P4)&nb^*cA6$OFh9VB}8yK^pki4v4%zC+7drX_Wx?F#}|y z4J= zq?Ew-Clh&n;H$;y=Sg9lPnSgsT)`gDg4O9Y?SS|6{>c$frFJe%oEQB?+xu}I!5V^Hj|0%?2sN>bx%w|VSb=AE1fDhRsO0s`jZc3dE70r(R_rtGJ)kY5`eY>XU0 zADxNyVm<7@E zx?q=o+CXxqwVy>ibj$I1{hc%zSq1-TF7T(lBIlA3i0}OZit)z@@qZj||E(@Mvu>!- zL41u7x(5Q1l0mKHS4<~8s-NAr|3i=JY~jCi%a3~v&UDBKS->xFK|eOI0>dyCmLK;Z zfPQSSJ_+JyPo4*ng4h9``FY~W*)RXYO=?SSjw%Ww+pCGyj*c&`RXx(MrYb?bgSkryS<(9Jb51RwN#wJ(S7D2(6+(nae!PcXASN`_9_%n6PK( z+h%@8Sfd9W*{76geB=l8`!xL+-4l*(IEhbC6T>*m$TRif30-Gro;4d2wT~Ueiutma zvtk?I7Z`P9V{{ihM$1xrt)q_aNJNZ@!HC&kNl$YJq@`JAPZ7$7-u>mx^Din2u-pdd z-Rnq+vxv=wKJNtiocu!HJeE;vi`S%3<%BJc@}8s;p?;IxSwjI{zEw~xK2gEe__jy+*m{VbqAC5|=Dp<$S(e|gL$#NOJL5f9LT|Byt zpEKi8^Z+mYxkrR}ZDWp3wWcx0mEA->MB$r6xZTl4_d{dH3M^po_3}5ZlxY=tIL+BQ z;dd+GG47Bl6De1i+@G(bAw#6GbeFflu|dgdUw*)GjP-HU#;$+BwXFSq4A-(~Xu8Yx zpct2X&HK=hwhvAm?nj@UpU_nzga_9&A2bK)W?r?~%HAVovrOu1l%GaTU=?2|a9Hm& z_7S)%3pXglUh6@A#T~Tu#7oEcYd}FQOU+oVhQRi9R6G~w1Q)iZkwY9fl9g^_=bX3i z8cJcg{Nma;?Nnw+d>sVyu`=${!l&abJ)a&P>IEM8ylYf9yg}b2G3f$_a)5BSMS{-C zi{tjV&QW#Cr#TOyD`*;4 zb>5DxG>)EzCl5 z_n!T;oxy~rUWt|I+$ol_04swY#JT<5rhW2;c0$c#yQAlpj+=R*xJvg6iyJJNKEtPN zf1(U8sg-ye6ujF|*vy6$tC*i4aPT#jT%!j40a!>T`w0q>{{W~MibyWeQPM7yk15V>~g?8G<~NF3FT4}_g;Z`=3Pm6Tnm+C*+aUt9!*&^eOZbn zW9qy*AvQvE{L{6d`MhYiu`k7W48k{$=l`Q$rO zmwnx-HhEe6EsYEIb`fmYjWA_>1ELIz6_Sz7kyRC;&5$2CcORNvOWF;9UIYVuOQs<5KnJb0XbJSa9^4Tyh3ea{8Si| z&UL=AsdXGvR$r#b5y}of>guvcsPsK{P@=8anf*h>UFwux^ z=kDi?7sHrsL9HV#bTuWU+j1{g((Lu~TsjDOjip>#lxchGk4(AX9NFG&v;5#r-Z|04t;sAsy-K&mXO`DhFEi`;%qy4Pwq7q#x< z-pYTm&o}P3h#xphNmQpAEcJri;N=HS@bTOWRGzy|b{0$0+a)3^HUTC`(n+-kWR zwB_XF;1fO;E<9mThMnymGV^i$k?_)imj?oJBR+>~E$s0(-*ETb(6agG%bBH#Sne*( z!S1Hj2ZK+WvmF^$8rh~|Q(beg>evo8T<5YP-41GP9Exzs<*!SFcu1}dTioLnJ(hg@ z$kGxK$+sn?%W}@R4IAGmu;TeVLYqs4Yvh#eghif$@-fz1%^o-I_k|ZWhlM``l{WF85%_lnK;p(pogO;q8$*mm&jMLMC z8_jkb&pLKhJ|;Mgy6)};wl9T+Vt&io6x0$7klhU&A&1Ku?BQY;WZ;Sj2z70f?b=kB zFVcryxtwC0z|CW9sIXW>n~_7pJcww#Z~uULzl|xX{HgT~Ze0CB_cwf{bft%C3}$Hr zPR^C#_$-bSW6Psl2~7T3L9((M+pr{Qc8$+1+LvW=KbNg-+9F#c_f=nI;#2R+Kn7t@B!eT00NN3v6-24i=)cCQ> zfehD{w}#L@>$>XRRC2t>o?Tlk6Z6u2mxwK-=n;(#5CK8!-TUw@Rt1Gl*06~R!@29i z{a7K@rMKPu3*JjpP44FJU|Or)TU}=4St zTy-mU=G2PY*Se^Ov&I}#Z97^(il1m{uBU z;t-3w&S+2UA(oH;;`;Q@IL-yHebyRanu=>RATu_lJqU( z?mKBQE(?-Euy)lu7AQzWiGp>xMeT*}F*k%g_&hj4x(j99GIfX1G|$>^PdjkcdISh3 z5hU6F@CodH((kSZ+9*AKwe9AuPZ-BHy~5xQF4MxB8lm^ zzJ(iROX)2OC*@vSn}~VK;;#jEsfSykYHzA%>`W9p@C$ldOIQc;;q|pRfJd8L&H3no zb2TEvH%wR+S?h_5Qj&x+T`j7;)zrx67pSoe)x6H{8*e>RepPC)v@r6dw6`aD$q6;p ztw8^4dmK}ohsQ=^fa6GtY7|*_0Y<~_vglVL?yv9ZSzky9O})C6l~3oumAE^%@bRGW z-Q>#4;P{u^{Pt?NVGmd`bY-dfjU%dMX6L57Nl^O{JliePacZi1EGpkrSyc^z+cH4K zf$73G*+{$nMr}>4gNxq|mpTt5+OpK5lxwqtuw{wXRJ@N^4ENKg4$SoBeH&Gv8E8i7PiHXp} zVbVUFLtcE9LF1CCzuvLYePds-!~8ht_fn8(%b;)Lge02I)VJ zR=R)N(Rt7Y|5%g2cf>;qoo+8;ZYg0oQDtUu=ODcINkTNs7X+}q0{_VQnkF~O~!oOL4@tb9{C?!YAi z|0yP;VJobF#n!U(ds|q9=I75%&j&WhB`T{U+;h4n845dzT6}Jn&>iC0FLTPp(c&2C z<*Upf(lU5WbTBpG#5-IXU4FgmTahE4b_GW@S*@U+hm7x|9$P2xn>f0u_b9bo_v)PD zmku$Qm&G)Vqb}d|l}#B!wKwirZl@-r)%)m=LH5eouVlOo(Pd;5Y%OsdDP~vSR@a7# zLT1C#NGDQLsK_+*{dPHgG0f2`5PW;A&-CHd&vqSc%Qoz>JDS|?ri_p2^-ts4ed6eNftyap7 zRsTn-*R|F+m38|hKMciwrd?|vLd2!~czD!$Z?C$BRe!lb0=4kj>?bj?J1Z&4NA=%~ zJ@0ZQBP>oY_&Ex+5@{ALuPO3%xD;B;<*G#bAoC*6cOUa7Tk8~NxGGPFs>gwC?fUBJ zhxgT=fp(qRXQ~O>r4AgjIDv;$ilodT21emBge84ua2-nRtnM;?v1n^ZWZbj!Wv^tr zw5vpAU=>M{bqkp^M&z z8TqI>6~(kWRO z3j?%`dlYmnDka^u)b3lMjAi=o=Yv`c-=kHBPP0TVeGOuLxZ-CMnKqP1-uXF2LL%2E zj9Bu)bq9FqjxM--$)RJCoMl6#W|DTAYH;n!&X=e1r49l279tfk;pWylFAHqz-I+(D zxQm`FxJlq#;5L0&qA#`>6k8~|ta>QLV=_RpTfs*c^X(Ac2mF5GdVv`H!ULT52o6J)T)iX4Nsd|;wUS5 zaDAVgkKzG`FkmIe*M3r2-+a{&>@4dVRu@do)?FC(iRf5%`~$x2?pnLPqsU7V9wUux zWxk<{V_Js6ym2O%1&@r6A zKDD9=j{&&xoyj=LB?vYP^+GdEfEFtIS9Ox6#_0NUGORMHHMyR~Qhgz6!A4OQB7gqPfmYEMYt$p4GE% zB3m0-Gy3+!zODn>Iu^d#9iFbw_$EEP7zBAeBF$Ik@n&j-B-p8Px3r_BNiYdApYZeO zTzUrd9PV4)#ORk5W{?` z1^Gd|>{GIXn@0zkm-^8gWO&9RWg{*XOoXYDU&pjcLaJ1V=TxS8=NrI?o82Dg%0KU> zlk0RjGU39e?xVFT#XhBOlyZpZ%k_fudiZr7V8YU+tS?_uQ>&6|)^)}yCAp`W z0(bhy4O>muGX^vR{3v zFqfMTzm%{;eCUsgC&#JCUQ~1W<5QvBHRN{w`UXWL@G3$#B}KW>-8_{qF%h!#-vhBx ztdMVzphhI-)QW0Q>sv>VD_zDjEU?iGaEs{JGROQRu)UJ5;2Wee*1bsTey`DjD-9oqR^<-gWsxx*`AYh2 zwA?|%a^Z5j)uud|pwUD0L9x;FNb8RKLq6D{0XuvsPjp{-@U8EU94bC@{EV>38th}r zaJA}i+PPlD>7nz@;0BF>RcyzQ(AWdmTkSK#{Ysr(v_hV~&+Jg?irb?*t=dG&^lpXr zEXY-*U|oLwtc+5%jD|S6Rygd|4Fh@^E zn~JjwUHv!N6O?LhbJf*ozZ_TSe{285aGB6xK7q^VKK)Bt{HLgw7O#hCt)77eD%;NLN ztZH`|wbee6PA;Q@Lu63EZeV4Sx!#jCX6k&zrHFT?4IpP`uc|ZVDG3@jLY%mXpQcAtoieK9&lK{ z6+=_vH}krIm$yDx&Njy;GSI$acmC#AypATU70fL z(36C?Wny;jf^TB>E7jIMy)Edl&Gu?!LtNW=MYj8#bKgmiC#zUzR|YzaadMW&HOkVC zAB{AW%4=i1y)<$-pIS5-pf96*J@9LMmCNUJUZn~%8VZ!Pka>f<>JAtQ`PF1Et31SZ z?DDdAzi5`Ne-!KNXc=(3V;Q|s`EsKs;96-qoUH<7w_hDgo3U7*_)E!qdkWd#Kkw-1 ztQgp;B&*;&dLFqWUbkjr?B>kpBDYXs%_VJ-?>kZ7)tirWlY}>6AkVC}nt44x*lYel z@gdl83rjLf2t5jpm8d3gqi#-%R#>h~RL-Y|kzB*Q+UPSRng6)#nxc6W19 zZV-H?G;zs&mtn~ME!5dnFDznMcA>yNEII@$C-coUyWRcKtKN0Pn!qEW-7!2jw@WB= zngcPzZI(DQ6WPhWRcXSGJYz*^HB|5O?J_}LrABs|{5kI&zSG}}GbtVqoji<+33 zsGX)yC7-hY(}e`VxL73mS5q?MIkG!MJK z=x7QB-uli%RhPyB8AQs%u4wh)i6y%6C3Cy-wb;BZdk3d!9fLH%qX1Wh+n1Qi8&e0J z#;e?u1Y2fE44YTTRXMK~&QpKK6mrrSesi@K;l>W_)OF7ky;q1>aUX76A)%c?uG$*C zhpX-hvweIux_v4b+la8BRMZJSBya!f_@xF3!WH2I;CXHc_1)_NP10A6$d=LTHM~)8 zRCtG(V0Bv@p>`7N9!uQ+BzTBR`swO-p0amO`$=On>kzRjWmih8JZ>AWDc|#avD@E? z&@qG6b%4DS;-rZ=3S%ldW*RQjeN0F7CeBMsD!1C4VpHmo?TZ6d|54$&gB35$o9I;X z3!~a>FJ{)3l14Qu{W!y~BJ?6;JQo;NM!-e);M+k>&SRhT9`l-f_PKbO#z(>puNy55 zei@lU5#il+<+K%rF2Rk!g3O>%YS_N2x|i-#bd!}&rH?NwR6kapATEn}4x2AP3=S5? zYt#>)W4m;WIUr;DL2-{8agTPj_Ctv$(vVD(^uXfE+hHLmImby$tKchxmw6xADJz5{ z^qTlaInno{`6|aBNlsK3P=qptw>%6TOGztyvg@U*9k!Nw3$dcO;ALY~qU24`ducT< zPj6d09+xzjVjd+ch~n^dWJ?ceok&X`}8YX@)LGSx)VtK%Uor*TkIpm`@X!;nt$7|;D)P)J>-UvFQlmR0a9X;N=truZ&T zgQXeCcV$U(k5;%$@o6E2+?gt^jgX}(LoOxS$m5JR8Cp+3Y3e?%6Zem}ZKMl%VP!44 zmhmQ)o{2kS*``M`sT@sWwsiJ?tAjg~{3w8_HK59o{7pf+;T56gHb1(equQ?Ruz3O9 zgsy!h_BQ(Q14p&8yl5?HH11<8`(xjijd#aBcm~^eQcwF|q4pQnz4N$2{2pTlGUM$X zjwW=|X`?QKpyW*IHxWQ?tV^;eLr=UC+3^AVPXAkfL|>3r3wZiYBwcsvV2ua@j`eHe zaDR4&R?_d-&Otn_Va-QQCGa}zi>7E8Xx-?0

BZ<93p=e^fB6gS}~M$x*(sOkh%t=r>a%cqDe@Hr6s z*OnGtF|y+&#Eq^EG+R>cqpc`z+#;&tw(n?iUS`5*)VRm{OhoDJEii6!I9faM?n2@ngan*N22*BBSa8azzGnzK5bhFnd==yyhy85R&c} z+N^QBxn%C=V!MC;sGI`Xz2mJGzt(GUmIKAHE)noTif&5%^-S&t1G6LorQ>gRTcTn1 zbCu>&a^6N6^t#`E?~i90WH814njEDGkND=U{Vi%a4L`-)5sF?NwB-C4%1gLhmkIK} z4C3+?6HV^0q$ZOWKH(U&CE;hOqZz2kzryfAv{Z!~$6e zK}+@>dXESS_skBGmr6Ysdi*Sx)tbvDKCK@MPTKQ0nCpZMi|$CqTrU>C%!anNKHO!A zz9T=UY;yOjgp*}zQq#JkP`KVZo#0VSJTBo^${1qGBOzBFtK5)DUkX}vrj3N*jncSj zl{M!W)!9rrF~s`OKCUyT0*z>b#>vZ?ElToMQ+F_$4%37!_;bNK&}-Fx4|1QZOP7-c zmtdQ^z9$w5!?htcAy3cZgm!U|3Ygz*K)!LOJ@`FO6F1rfS_1hMa%Le$(5ld??uOu( z_VW6nN)%mMY#q3*ybp}%x_1m3saYK_UFu&mW-n+uVxHgaz%%WcLjt4Ek3ADuN^#%> ze|Aa9*sm(Z*$c~$H!CrYQ*qt#S9-D%+Seyy^1{;W)|XiAuSPvQ0V!H63)`r4vL^a3 ziS{M-%U&PuWxwaJj@bP4R_cW%?SuV=6#SJB0fS`^`$pV#_XWF(USAc>&r9zoPz_2) z_dn4}nK>3eQ_ruK5MOGwS41<*;D5EK!`4hjVn5?&FZY`*<+(>`&g>Z5hu zBz5z%m)s3o-$t2pl8jufbZy@C`+|SH#Q4E+XX{aA z!huF$sn0Hj&-Gi^o>_`N@THvaoNau~+e;Mv`s?vh%EoRG1`5^M?b%1_L!kKoHO#OgSO1U2XZSIk?%MlZ%p>6(y6=5q`0YMRpAwxM9wG$V{jp& zuJ?Vo=3$7ZZaPg6**$$ls+BLawGqp*%5`J&hn>F>4Wn@UV z$2FcQrbvM!*w0*Sml**0hk20n99fQ-G*UlRT;yBJtcwQk2ZHbt@G+iGd&ud1{q;Z1Y>v10nBjAey5MGz!hhN2nu}KDJpPK z{0Gs;{&;dQ#S|$o1K2SH9tJ*|q4Q1de9{zg%x?+P9fdVscO$IQ0&(M32H7PHvcQd- zbuB5KsC$ht-SClQeoaxPP1j*t@W8Rr4WzFW;RW#Lkp#4<`gj_(*~NVA@Gil9ARP^- z=tc#TAq9scbLbJy%e)t$r*6#EzC2YgT5?G^LQIe0#yyXlEfiNJ!%?s5kzz%{JLTcl z8-Y=ST1a12e!726SCr=Ib=_i%1mWzr?0Be{sn>5u)>9zguGfpdEZ>5*k=)D}Lg+WI zhxd_mv|cj)YN-y&M?dE|4ayek5)|4Z8x>0N^p#&FbGt&)Xv_|q^7a8c{OtH;bzcdq8wGE3R&OYg9eXj1(YPV@k8ut? zDjYMdLaZEP9rCKRLS1fTTcx}EC}5Sy4K-}+%_{6&f0A*^m#Q9Kl&iD{aBmvro!~s* z+U{U#zLj`>6$#dG znmdc_2#I7UwawWhq|>9;+)FO)K0V+!5jBuYc-Jt>3Q?` zHogT}mOb`mL9Ye+dywtO-Qz6MpUvac#3=Qc+i?x^jb?Do)7Ly(PIH~-sYxMDg`X#L ziSO$q-=K0QG3`g4vv%=L&diw$*dX5naZ7b{%FI1*2~EzyofC2i)XY{cDme`HkUkc8 zld3!)w}Ib<<_eDcczarpr%oEDlRytnjB>Q6@PplVvcrTC3iJ(}I;oY;+7045nKx;p z-S8V^hoK%aBwfedBy+lZd561!vg7VHK)k&Si^S<)8Q@w~Ukl z=9ros92BA(ocq{Kz`$C^izb~hr|2zvD_J^wA!&2?1C#eJ@DRC#^=GgRV`mJX1y2&D zmt6bg|4~xc>%omAVZ4cltOpvjkIq zDAt>Z4v^t5==Qus1 z<{`d8gE_T(#hItxAo&UP2ie==>PeWk5?wU!vUYC^ym$~p)ixHd@(>(7)p})*iK1uT zlC}Y6AQ;w6kWib6Az4&g(kR#t?3JXutM^&+=^gaQl&hwY@3~ zQ?I_qt9R+zn1B8dRn_?_BKNK^YnZcSvL5c!8u}*dOyo^j*3U`8(^v!{S8ogODQRYR zQ;-yj@n}Zn6Fe1UqX}nWV0?Vs_GzLaT|J}Zo~D}iqMY#EIR0G(BrUE6{uwzxR(i#c z;gUID`m)6J9B#3e#C`clE9rNYCsPfqdDYWOo?Wd zkcY%Lhn8Ds7H@_!D>-DYu~IT>I%Fc2kV`pD!*d+LMRA0;Np~obX3mPexlK4r7B7fc zeu=C&GLz5i(sg+7OMR6-3aUrr_ZMqO{Wr~$xlxOjQ4QFA$J?(5l-PHf5!Sz#H6v^( zvNzMfZDfp6Zrd@w!AlpVY#I}n-w);}mz1W95^0<20auqPRlnT0;Yk-I(>8UjsGzjf zymz!UpoQS=96Y1PrC=GDjEwTu=AEmIyY`X{L_HFQ?(G}hY8~~Q9Xkv0gf10YmPv*- zbMb_nw*x}w(+s~a#PhghSBAE{C=JbD{C|wSb8sb%zwMogZQHh;$;7s8CllM6*w)0h zZQHi(>^N_p-+7PjJyqvc^&h+T?%w^a>eba<-Ji8q>IOHtE8;%Jz%v+=dtk*~-J_uN z6d}18VgkjVQH+5D=(;hJ!s%xSrk8+JZI5&A@(tCGO0U!o4^h48=Tj^KzI+tpq?p@z zD>S5{JZ%sH3sxLddvI81i(eSn1>scGqy2uz^S{F*dQT6uFK#;-cTl}yMMS_mK<`=V zU>oeHWGD+u-QD~T!|Y`f?pi)ugTsHgbG?AmugaH6#!1*qw7P4f{!*04o**Y)5Ztq) zr=_H|H^eRU$8$*Jej*lf&z?Ih)tYAOqxPU zQZ>esF%29WQO+hTqm@l3OS%_M5|g0LAWbsij#Fs(n;1RbPfIg8h<1!V#-gAp5?0WG zmBe5&hLO9M2Fa<68h*ESyl9ndM$%Az`7@ioakkOm^qjl&i#gd9buD!)w`PkQK16Ma zeNA(Hgfwp6*t`Uf-0~|gV9x7B@E9lr1Z#=8!ie*a0bhSFY$)SK+OGT?)X?6quZn;^ zudv_+rkJE0&Q!-++r+k~)!@RxiWH8JN}5p|ifWo(o}!;u*!K<&lXraqyrS|){(Try zyamxCLKlqqhH6Jd0Ac(&?zqqf*&`xn6a2YfVWBHVfUryiU}B-~y-r*p_x<+;*)k?x z(#s)>6zb3?ET0nVoJWGV*1ywUo1EK76})}biD{}3Ok>EfqdFq$QI1E91-JjX#vT#} zg1L2t4-*mTlDDm4#u}C3b`RMoxic3?jNB;Oe^TmoXlP*Qj;U}kxPz*Cg;42Fq&}yl zhB2+6tD(@4JbSDvj`^UfSePNO2p6_k%253i{cUOR0&6rK=&9H)xv^|@M%6m$?QWOl8cdJVo9TR`O4xXgcQfUnh znzFMS@g-MIq@v&!Rg(B32E0}n; zW+-sVk|QCF{08lhqjt2=@jma!RC~}ArHH*m(BB7QU#j(X$O=x40$K6F)2eR#KDh^% z&R5l^AE27fa7G!mFeZk2BdfgR_ES&sCLWfCS5gj+bA2(HqfY)9ok3_zSbUdh=?O}U z>b@%^qV96trxne?id4<|Jx!0X92h;hOfb+HWZPKl&zt4qu5+oH;!&oguKjF9?XQjA z)%P?C!`hbo9Yfq)(=bpnF-?O=;~fcQOt>vw!8i{Vv?ArqstUk*Qlz4LTG1}3SKf$_ z8%R&*TV%!^{LR>p^rT2+SFcAe=~Pj&RGX(c94?PXBk5d$xp?{@E6x5TEff|9_p3lK z(1YsGptGb&io^y&b6qg;l5oWZk!ZdgW7CCR9GkTcT7@vU`I6-lCmU>?-_chDa7`0v zox|pqB^9`g#adH#yEZd6c37>A-7)pkhb97AMO>}@-U=uVl^kxPP_dibh?*=h8&i^PF{~qe!9XkI>EdMj= zzpPLH`=~03lT{gvh`|7NXa+6kANVzR1(b?@geJ2DY~DZUIjIzd5Q!o>yl=U>nk@?m zy_?Luj@&K^xEG7<)s^pzx?Ih=OU4U}qI9FG?>y5aNZ@?LYbh*zZ?mNx-PmbU_uDnZ&; z??WQzF5~Y55d(uW#6>+ped4~z$5|jjLNuZV4l+$e%Fa{<&CVpp{3+^}??BoEf&>h( z2fav00!<`@^B+QCeX`ds+VwAoDp?r7qGYr;-4s)l;Yu&%N~i_JB<6ybN+k@Pi5z#Z zG=kznHl|&dhc|=WK;opf%|cRAaE9V?m)zk`mIviT1ttA$b@q5PK$BMw!0E?9T_d0n zqi|p^UNJYptg}An)s4Rq!7J^j9!5W=Ruh5gN5rY|D+lYNnjQxHkB_n$welTHDYd|28@UZY8 z;acgQB_*vH1<4`l?d0his_98mkETs!8k(Jzs_9hsH$2n#8-YgEI%><;uHN0(G1vE9 z@W;B-E;)`@_tMuekU(3EFrbhVxLcDq8AjSD>Fyyrx4*PZ1XqvT36#3=3Ed>)1~Um- zd|$x*ie%FxX!4-G^7*sO--)AmP|+40-E71TQH!i?C-h<@`(BCj09>=LJtD>2Rq*_#B`}N zHR}gRhR})}38b{X^af}J5lqoWA)hWNBPk0B$}6xqoNj9s3~SmrcQc6F)13%0PJU@p zYr+sL>>RIs@)7)_GpVziyxZ{K8^XX;d)MyY1)ngJS?ek=4Z*%uBzJH?>03^U5CsGw zFdnRV*}sK37S4hktJpGRYY(!lPui>KpU?f#J-S1Q?aNt3l{_gzAA12F=)p1cpnHzt zq^UohfOE-CB%;|dv61~e3~eA`;AYRbvEe#rQE_Imh-3u?MGe;0;tute#5G#pBdW_X z6?dh$?jdM>&p`PbFHf%K_3FSKWo8peg45Uda7-`GR%mP-zBV@6<@njGW|@~U{9@N{ z0-uYHFdN9H@KUMRTLNbhm!vU*U!OG5v5|32SqBC=NU+oqP9KWsFyD3$5C|^0Jcd`I z^rWd@OaUvM`+39J8F+ON^Hp9^`O$!rrZTMw=F{&cr@74}owka|a5A2F)%37}B!r+5 z7uG+$xkSknCW;TKG>;@!yd`@i~cYJ_NA?%XNjSx#mnx$E(>I&kF%v&x#(O zlW^yZCxjmP*PDf;jn))!3J*!Arz5<%plvrwzkb0iv(TbcoI zeL^;`@ol^)0ppPwLCF}7!N_?IkG@53ND7FiO$NcDt%TKn{_&GI*CWI!+oZuvdwVd z?n<;>P@3lRwoqv9vM|nu%}hv}t^C}G;p1FdM^sKt}X;XH%CLji>4lX9*N? zO%^O-+hGMM7TZLpLWkNr20}>?r++c&Pw*KGT5_sfE1eZ}ECvrB!yF@>brs!ZmRl@B zHqIi?Bsw7;2>5T?14C!*-0y#OAUh1RHCxKM0B4{rOgMPFjDlqDqs@^^cbUObv}2;4 z;;hC)ktA*WMX73uyy$s?qv1i|V>P)?F;SwY*L5+xK+54F8(hq#eZFi~I_94{9L-?3 z5=#AANO43_tJ16wxJVLSzAMi|VO<0WD(Saa)4yPA8nr}c-oN3|rR3JW6-?QL6q~>f z9nlr`_a$TX26M8}{uV5cZ;eq|j!7U(LJszbXT@!sofM@c!b|z{V`5(TR+P>iW0b%v zC#|N*_V+TL7JBswvxDYP0k`L1VWUu)k(Qb}*LJz2o=te#s?~cu(PBcu0O!XXlHBrC z_}TL0({&(2uH9q*Wh7eCV|9gFv0q`-+2By~>Ca@(d%{zZ>X4Q22WWDOz5`mq_T=c_ zUWl!&@~T>9qvfmWYAIdQR7IVRV{CtH%F|ohy97ysTZ$0q3NVbBG`y0JjLZ=r{)F^w z4cY-Ik3F|w3RMSY20k=8I~z`H-p|a;{1%iPSRD)7e;;VRgEHYyk(FP9JWAeetI4^}KFiw^;%+ienq@a$(v|^g)%LK< zA>;R{(`9=&Tb_fPJKthupTk^|mW3G05q-08#?Lx#0-UTRL}^eVAxhs<7D=9 zf9=i7UFF$752`6M;fn~Bb8|Gl4U5qhT7?;>h#7Su;$hlU$UH(;u{DGltVgr8xxp{~ zne@qCji*V#?zEjKqjfHOv;bRzvR0M0)b)3BB;dX{?7;CPg=3-c2b<&oYTn{OSIMzm zW4vaPvgmA}SXU<7`C1v8n%Puqm*M%D*|MEFhq;E}$b}Y)kd)Z=j-d^ zIKy<=c#_4kc$gtU9-d_Or2$6FLLms79hHSu#j~&Yl-I=n*gxp=`AHSM_58rO+;^U=n4tSb=t{~nzHbgLd+`* z+TjXTEQ;os!Aq&MuUO|3B$->EkKr{O#Q0LQk2_V4DI!<2@b9j$n%3D|ZlxXD{2p^s zY^|-95T;Yg;i1RRg(u~rH;cmO2j_*V03P=M@=LFGZq}Cg%=2cO(}- zVlBR6nLNkIBu59X*+23O!orHLi=KGCJb7WR$Kx3c&UcTqIRey|Zu^GiGk)v@oe~f6 zW+vp*W_B$e3>O^6>^wdI-gPcRV?}=}%3^vi=>@8}&QY<9v0QqwH)4xPjfHo2$+eLB z?ZVgd@@93jqNK6kVdqWf#8)ZI2N&OG*_NFO4A}knRp1IGzRAuAthc%9^9k0h!Q4P| z6**|rwbM&bB7t6QtF)w7iRQZTlX3#)VS1O1`YLSG^HrBh=+KDbV61ZapDK=@Vmd7n zrS$UZi+P`jXha087&F_>C1qubZUr8eAEAj1r)0CZ1s(JFP2;NwP4&w^7ToQ{_dD?6 zYw9gsP!Y<`o%(Y;GChSf)biWtRQkHY32OiL=;sr0QpTR{Cw5;RKuLex9#!V}zJIt( z-UXi*M!+nG?4VBcL#dOgNZ=GK#yheN8G^D8t1pwEraSpRtVs<_TRsIOCoTxP6rfI} zOKuk$-EQUY*)h<)f669Td~oiqrnfW$p9FTU>PeD@r;>}SVl{u(%h4>!O|ECNG2Y+x zm6{dNzdya_Uj)X!UIa@`>3*%WnhUy^F4o-c>!k_2#b^z)*e7V_{^=EN9q&9jILrwH zTNCQwXccUAq&cAL1?HG8<6BEz!?>y=+J3AI%`PMz>!j`#GfsH zDmR>Sz`V7Q)3kI~krB*3yF);V4sO}vDZSm+@4iQOw?rRZjXSW0Lef`0i;NwrckRyLw!M)|ZmNd)}h11G3-PAwuYi}}aKl_Z+0U$iQ`S3YkN1eSK z)5`C*2DBO;DA7jPjMl%<4sCmeF7Ix$c+F2el$Y9$?l{RYP>Jbs;{C;698tWaF}9^u zlYBwF>1A=9eZ^@hBKx|~;4T+*P<3J{tOt7~wxreZde$&GcS~?3+1S47j}*4vU1zem zqE;E#NhZxbVr%uWpb7P8AXh)ZrrEvTSm$K|*PSa8nfIK`f{`Xgk2I?8aWpFspKQun zSTKamh3-*I$;saODPK36n2v5GP0r6(sAL|fHFgbb-&>qBOg9WiZNKDxzcMJk5JOhj(NC^2>Nm6$}J~) z-dZ9lQ#l|XpDOK~D<3tQo)w-v3Hs5O!o$hR1Gda@rwLQnqa)MVLH-5a3$5A;U$wNb zU6nuKE%hp0Vl@)G?DQ~rA++7v2}l_&CO%zYFm8TwptH$~rSL*T zesqsJceSxWy%#tcHcc=f_TmAn(zQ;drkgyKD+s;(O80Ruj3$vF+D^QnGie?ZMi|AO z*-xX`jz_o#6skoiq zjl*qr5lIjHOn^JTUZNmvdgRsVw8b^>*XVxE4%rn1h-X0NC6u#h!ph8zoFJeDde5Dc zOgDG~!vBaillbIhC@<~l6Xitk4Pox3+i!_Lw$=Z!*qa_~+Vlg*hp0vLcgm0_q^~Ha zAE9iTFI0yyV)Z;0)-N=-EF%u51sT#cgbz4a;?FNcCr62|L1D*Qk7#TAGk)>FSp?uo zvbi_b%N&@>FX#h0=P%6kxy?+V4?V$+UjVj`A2s-fSAm9@Uvh$PmC$?KC1_1TQ-OX< z4g+ww0zxw%IUp)V5~NB~zCHLQXr;->09I=w{+iUk$@;|i7SD-0u)PeRUwNKM%fL{5 z=I>nhHZ4876)OqmBQq&eLqFmC1NX7-vapjS_-A1(2Xv_Aqw!9zQ0&kI`kh}P{?ZV4 zgiOU&8sn*h+{zmu3kJu+P$2-{Qj}?)?*PA{8-79W*{Z+D2|h!0G=O}e1kr$UVDrBq zaDp>s6T*k`r(258Rg$yiii{9Ov&lYP3iVbq=o z#x`JtfZ5!(2`57!So8w&=>z|RiZGjw1B(mqFMH1gz*t2vmgfIEFvM?u+0^)ifj!Sh z>H0{sV6I309F;@znZAlx3g?Hd!+GL_Fzdd`;WCqkHzWAyd*^y*#YNt5ek$!7?^~Td zk8}MqS*wX>uw}hS62CCBtBl;oNnS1&?U%6TOuy3Vg^e$S{ZBJOW??ZbADY>mb2lM9rxNW;K48LS^YwK6)ZH z>~PAM|A6X49QpfM^Ht@oPK~#W7ogDw`h9s3yycPtKAk>8<|7`#M z!LK_oi7@z~=c3(JjU zU%dX4>OTE%4%Q>LMHXPggIqV1gEyFi)hR-ySHM+^Z~?eEaZb&)*?#`Iep(wVn<;|z zfqT#V`WOxV@_!#nW63^NYt@?hJR z`sHk#-`_@LJNho0N#Sp}#%MeSE`LQ2eRy@?_jc-{+<0$7O3wrP{9=^85b}{C%qHxH z&TZ$Dx&~jB=+e_cj-0_QdiG8-9R77JN&bKwNTfNm<|KYgUHH-z>o3r6ixuQ>*N%0t zKW|~fbJ$jYptRm2-U5CtzcCrC0aT?5G^g5US1#Y}KZFD2TNps&=VzPuiM@;JETdT+ zaIVrPv*4c6bBe2+%b&G_f~NhMIpovl4fTq`D~u>M6W=pL6hIh1pI{I=lkKoA8eWMT zz_n2AKWr>rtL{hfyw(odAcrfnf4GoSIq@Z3%bv06e<~c>g?sk(z}%+}@RYqm8Pn?9 zW{2Rto}IdL`S|L}ebb8-lj^hoYTgx`X81x6dCGO7{yUx${^w;+IonSdZ`be;-VLQne!hm-@)t75IxX>OTgy-USufd-` z!SQx%Pi>y7V)oOWzb?sLnTf6`{if-EP1- zU<8Xj^QU$Ok6#!?+mHAea-R{u-3a*UFQ69UX+SsiPiHR0L%?+{LPw-G5mLfQ;W-4A zG7&|;a7{0$bgwdHFDqq;%nE@b5gaLVH6Fyy&l7~NA;CY4K$7;1Mf2raAwv4druyWj zy+1KQefQ*XL;4LEqkF8yLnC$u#Bzbjg}9;zaB+k44Ira?Yn9)4L-q+#hWGc)hqRxe z>+~s)4EQx`zDYG`*g#3z01IU_X%QM(ze!Fc(;j|Ic!6HX5m6j;BPiZ*mlp6x_3#9J zByN4ui8(xn)(der2xIdxxlKRDyS75(I?kQ&(J z(Yh6R_DDE05_NXZ#3*AfYgC-cp_~BK(Z@_8&mS#X*0!Aio zaJ`cbejzr5(j`iG{e=S+5=b7tO9LC~-JijE4Pbb#1y+TGI5}^C1Wrqm@w==6iS94| zjSSyMUu&oe?6`|I!^C49D4yA~$pEoh)xHT(vc}Jt zMz0CBM)C|-sU5V2(i$sUWp2u>)R(JIvPRRIT&iib#?wL&TLFK9T_3SrskcU1pJ=K% zUb(lXSs&8XZ?xu73F29eu*O)Qe5&cRCRm?_uL-A}TB*shb?m2k`Q9^wxreFAGAps%O7ka9`#9_%t*UU7bcdrS5n{+#SG6R06r zmAf!|f@$ydG3l<|UX^i)_~^B*7g)t}Nm*pUNKg;_38kYSK=>y>bOxGx>`rQj-)6O;h{*IPR)$VQAx1OoCA!^iSMTw|bZTM(Z^&!fU;;f!EjzUTV3Q=anmbbD z5jEnBD%msJa5s^T+C;@nDZnl_bQhYBujnRw7iKow21 znZnG~_wo`cg*Yo6yDe)6HwgS3q)@k0lJiV+>2v(Y zO@5Cb7EQn4w0MEjt2KD$XxTP@+VX2}nac{K-$xvI<#FK!YVl7oq>tMjcwn{aiUN3- z#}B$70Nlq`2M(#v?g^e*;|2!~n60>?RiRF#itn9I8_$rgN=8f=3Pj9ToZ1!1tD#3i)wV6nG1Gu`aHb~*DQn^@2=mz!f@PZ4 zv^FeA$LQPJcBR2lnPDB$4of1Wszf^`o1lfDt+Mr31`nDV1hs0XRK_FMg;)Gy7F11B zF-b_=$-i-3le#eR$;@ZIA&!D=M;> z2%l@Mn+vX+OZSSHv{=$;8B;PzIut3+t5Z>R&&#WmQP3!yXGlv*tfx{Ps|(i-$RDYy zBE?}f3BQw7?8>CklMFvbnvpk3Iectlw={i3d#UNS!~$zt?2bqncUeP0hkDit{-izZFnDwn?(nw;IRF2{zr(k|rGNZ6VQAKTzgKR_78-f`Gb~ zcca8h3*Fe|J*XNJx*X+A^kzmd$r%$mE#pm$f<|Z^-bP^~5C%x?tn@H7OY#2q#VdeA zqU?t1PKi|c&7`Z&xZ}F`&j7?$GL`#5X2i5M{iQ9>PSa8|Jyr9nCW5`Zze4dOWx&3! zX+cr+Y9KT-*OD-hXeeqkI(`C7@~Kgpx>+^VCOMurBBpV`BG+{*`P}+~Tk7K*QS6*a zH*1Phe)kga>B`CJh%=sY)Ld*EK+Ny*00;5S%6m)mx_~!}(+*DE1m80prZ}B|5A}9% zFO3Pnt4r!kvhDJA2wSVyfPU7l+SE9hL^w!!W_xlNr{#XNsy~)lp!B7BRk5a#uk9Zl zmMt5;qr*O|ruWSYH_?zDwa{$cs@3qiC`^vuaLAb4duU2l-Z;NK1g4QW;1M^mYs|39 z$%_{c-{lP3_u($?x1^_KP_g$33&)d1TZdlowl5o$K6?}7Ox`!|%I+Da!gYKlpJJd- zGa?zjawPAXmf7ySD!SHHyAL1jIUTsUpY_6>92WJFt*nQ}s@@?uW(VWe=GhY_CZ(9eoS+PtA{`2( z*D1D{F-@yxms8cHj()zVIx?BqZSUPNU#%ts#qc8>He6XypJO%-unv?IVn%&y(OqjS z^Fig88LPs*2pd}t z+waT?d)#4(ElHwldtExo$c_^ByS%lL2ze3)$*2xYHf$Lm?WPi5$FyLCy(Gb(`H5#W z3d+-!AG{FY(}H!!gj0^O1#?!Yn~rp>tKVBI z=*Bc*Pvwxe5J!1Ww*yp5j{mDla^_!!Y?rS$>Y&&+(RaVg1%HHBP@dZ!jA$8TCQF5d5g7L|MGi8Kr&z zDcM1a|Bu7{JC(=A#PMII`}c_1|32M6>;6AvbgCpuRj>U6z`g$iz{Rb|S`+7rxAKyr zdzUz=IV0hQ;+!JqA)>#(tsOsam5FnDab;soGwxe^Zr}|YAg#u{9Lf41eK5MzcW>j@ z-|t;8^ivifZ|)=ut8X@D4N$`@6(4)wF7K~w511x=!0vgc9n9Z*{NiEGRPr%w#d4tx@59*)6prdT1gfEs&?{5(=_~dB&{)4fVXo*{xbKhE2&7JTno4 zqsCKhE6IsdhOw4jj(=@V#V3S}nk64qqm3o92P`Q?Jcu-q^GmV^B$G%-*H&1<+!Mnj z6~!cV#liMZf99$v%8k5timWi-<{+$wsw{T*=gT8O>wJYNoX5LYZ#H_rv=MQa&; z1vU1e;}I>3YwRyzhnX9fNmW2r(Wwg0M-~|?CpP>pBrK)O?V>f$9CK6gnrQKya7^{t zph&&r-2O(_sxnX62Q{L%7xk!32wMIpsr{?^D5{Akb=_O2->2zUav%)fl$WdVy*!dTSlbrzCiZFM*ErD)8=e1|1or`Ri%h5!d z;|KCjpx?n3gg^Ti9;RLebeTfsr?~?Z+?#G&_2|}G1I`wKWFU^UCb;0X^}5jit`eTH z94d)*I@&6gbpBcMe+fR^m)+QENnhjn!K=9*UOy>7y4?YX=!O6DT*Vw;jaa+49Vn}- z@TF2Je0*%0ew7)VP#M%5S5wiFip#Il6ogcCrUf#-Tr-F{x@FUB$1BhY!*jl3P*bmF z*6%4Fiz26_(^|4~B5(cepnHra>)928h{D5kn;2#dYFQb)1UBeDa%d7Yp`_-JGDLm{ z=3)F>k4%2*=ic5OjctU1T@ctLYaxWGs0S8^TRItRw_8?5O)3JL{$l#}(esRu#&V$<9AbS9C^MKeLt znEGgD{sJE78{y2Aa@?AQ1NrA-NcBMqK=Z4@qOx{jK7zpgh!Ji?x5^pN{-dw*Q z?FP=#2uS{LGyTA5THo%>Uu{!uyBYk>%{tA1@zwj9L|BxOkX>c|6du?4_Gtz_Bc8mM z2XzcRO{cbh@Sh(mb&AwKidjsmls-^&pb@??MK^oE4~MHwMeHx29zr<=zX&{iawUA+2A84+XgI-?F^LR$wR zn%7O^9weK#+!B||HZ`oPXtX5GO)Rrfgm}GFCgCNvw83Beijv$dC%Nl1P$Vpte>X#g zJou)3J!9>?@zuqgFFN-r$&xNlf*R=UZQMG$JS~h{ij7pdIE$QPs-Sx{C*+01<7$JA z=|9^d+@{+ds=)0mBYxUhKOADTxXh|m-mR_H>eMuIq>zlr`87AWy2ydoW=#zLXpB`; zsbI20ViiB8>mcNMTEjQ5(J6 z-WWZET-e-v-5xGsmQX~mO(+}j?81>rggUi(FG&Ee!I zlR&Nd-Jmy*07Jmnwd>4k&ylnc;(5%W2|X!ro_u9~+g{NNPNfAp!{B6sr#v(=vCB9& zLamZgmff;xtF7)ru{7m_x}C?n&p2bD3m>ov4GhQS8;d#ZxyQ^)#eFwHZ1PbPAI_o#8Wri|@okQ>*88y+U6Uzqn*_eLVIqL1Hg( zT9ja@!Z4|&<|xMEcjP|mS!~_WT%GHT;y`OxiP_^J z0Z`RHk2;?X`W!FZMtGI|Moo4n&m=OMc7Y5eorjV=6T9DGIIPy2&t|8@NItoSdye_RnP)2FVv*@&i3y=% zu-dV4{1lz9-t_ryvbXqujn4E7Q^_z%x=+?iRQg<3l%eChOmq7{Eq3n1ypQ&K&$yoC zj2YyLKCJ1Beo48DeXN6zM`MH!g?xGlIH0VF>(tbh%gn4Z%%EB*1D2Kv-B=OxT#|Y3 z_aNWh7j7K1&OQg=p_VB&pZi5D>j|CQmfc@QYndc!``@al2P!;tm4<_w_}8i5=doiI z_m4W~X*$N}%I&T}H3AyhTn7#{8QM6~@3jKIt+t@xp5X4tW~O_FGgycf5RygwTRh4h zi7}P!U7SU?!0R>7w^eGX8INlbPPvgK-cj33oILFSs=2F)}Bt|(P~<30+`0Jg%o3k&=S{l+hfj8N}SJe(BX@`R+USg*;UJ$R;QnN z65ZAW2EEBu@jjyRrc2KvB7yA`04bW5P5>gi`WEuEn}7$a=arZ@cYX3_aZI(|6qhqG zo|iFCZnwi7@khgt%3h^i(PS7!)VZ*%i?B!C%#4mwy%<+psctEOsg{qvkO=oVyh=8! z-(eV^4YmsU<*uQIFsOjPrk~-VBbOxd+E#1yKaS!?(&HPuZ$_iCF>ZGjgCFD>d7K7c3SW9naa~Gw-)(6D(~--8x=DxAN2RvwAcKV zIx~N!sL(pY!KF4y*0OO}jYMR(Vb`4em#N_{$xB#E@rXVTd2Y%*INU)v4~ za7=;1dM1xn(_M{Vc4T4-F6Y@>2<_ky;xw3LrrLIP-VYI5Wu!yw)m|X>ie))KQhNIv+>cplmN;kV~vt z{bh}|)vCht<&s2xJy<@N(@Ah{6#2;i@Q}Rx<0RIFCDu(8G~w%IbSn3~wCIogvt=2Hq$x^WtTAd2KwT#TPhGhj zM>~P101Fvi-;1rU60#hjeB1`!1+q; z{#bG3efERC8d~`M@lr?Q+wH@6>ibcgYm>D{LquBNW=u=uOT_`#DP z$uIE8C@TCWMvI!ux2(_^D-&G2>|qZAZtmQP_2SW;%LH8=s54o}uyB_tp4_3}HW^3> znhTpVOia{xrbZ4M!Qos_Zds@+qbuEO|Mi}@r(EG;n!#N`VTuFa1iqWU7m55q_+b|Y znmMe*<^6vr3=-x;uG3j7sHr*tuk=ivfe@yLk#f`gADOP=E#PtCBf*E4nA@)<7PrUz^6#_^yw;sNyO3JyUDojC!xc!XS0C7ulWtJ#m0W=moi zk@qQGmM8?P6eN^)FRa&P(>K$BJ}0xvuX`P+nal+?p|BTH#Q_(^ibEwz$L5^R+0 z<7fiDFJ)#wR!k7sdyDkE_3edm{qcBd!s54@keU3&O=w=)-_KIFYEJ~x)QY^15k z)5tq<2G#RUk1_?GBQRasf?fNYt_>V5;+-Tf4E3^;QWS@vsU|3`8uUsX7OF~h%DRsK z{2D?3^04gY~H7gCpdxh#*B#y)QI>7{+>|$8>up84{Fr5|_rik{`yxpwV<6wa# z(mp+E)?Rw&&gIfgv2;jlTwFH>6gb_(4nhNRCV^(Yww&t|u_RVg$jW8eyUtR0wS{li z_$UT%L-*83pv`Qpi7(~hWw;-PM5;k?H^U;Fp(`!O?VBHullPnauLt?O1pntR3c2== ziL`#};29L#ux}{4%GIsh7A(zC5)N6iF zKz+57j+f2ytym2B$UXBP*YMYX%doqQJo*PWUu)ceC-F>vs2y*mDRa0`bS*pSuK#s5>7L-w=Ub z@$&s!gza}nr|yUPRg~h+Hj7b~WD|)d1EH-*+>Nhw6Gu6Wj;5W)c^2Qz3%vP9QRQE^ zC4FP#3dt@0{fclD9&nRuu>$2~Hv2+2#a&r6!^P+Wu63puFLr!fTDfqytRtsFD8f$x z-s~AcbaHG(G=kLqu||ERX9sfMsSReqPiVY)D5v&98mB08+gJjwPpk2A=+kcv9X+%*waIc4rxrM@26r%8ETO`D z8AG;L8T=pp%ojTZa$nQTTRyF)@#;Ew zF1GHy48w-kIe^2TNsgS4O7BL!;ph);XUVG-rdFk5A3ULW_pA@Jc+|O4yU^G!yi&!F z8X-k-^X=H!?`QMz+_6Chs4&e2N{3dNqvIvFrfrNys2=J@1u;kZE7;X)h5cZpw5c2k+@p3mei z$@%rw^JV#7{>AqjCViyZrK?58<=8~}SmBpX=1sXDEb;#FEWEiZ{nt59h%FC**EomV ztXJ2I#EsKzDrlU;p^r&WPNa`yw@$)M%4ApZH4>*~OJg6WKfo&`43`ky3zqm@B7i=S z)AZkei@JG>?g6*1C|i|R3Xc`ECP0 zRuH?)l1#4R_XV8l$O-I=ob0=+or*Y~0=^6mqoo~%^uRSF3r?@i2s4&YAA>Z`8j}JY z>~we0BV-}%IZg~olT+B2*aSuNDq$&)+St!H8z`z>lpFyjm!*w5z_y#av1w6Y#FEiH zxdn|qR{>r&d1sw1o_(vNwkBeVL7)l!iJUfaSj637pV&)L=qIgsTkzzCS|k23!Ju!& zSf9h9h7aqoIAY*rffntEk8}Fb$V;e}Nmw_zCKYRh%i6)}49$1WN!#81*J}^X39-th zn?l*uIJO~QhnsGAV){xUJjS7lhVWZr+t~6rW^aInhR}m=i{mo5Q$Man)l;+lYuz0kr5vjTDH*keGyf0KG3U>?c@b@dszz7!?kj(TKnx5F8{DJXJ1!gZaw!`)rdz!B&_)#20E3BmYYioq zO!WK#3=D)I-p&in(-~3@1T-y$>@RS!At}T5p7^zPKxIKRAYJc4n(%&ukesNLP z2y;Z6-E30Z`BfVCx*303w&B^Rcbe>TpSx6+9C10F1JRa@FUPr0`m7AX!tnqr+a5oZ z%)N>XyVXEln5L-{rl!x07RTyJIeC5;luZ_OW&>TPmgcxavl&81s5$r(MKqmcYlh!N zPiHA}dvMk%*<9>NmX2IUl>C#Apq&QK>+A1DA7v-Ts`{5XYHNR6w&|Loz~UiiHVK3n z!#v@w&X?oPxpP;QEnS`p*S@fYMv=&?>X>+FMlPpQ#`nd5%@WPq+WS|r;X+M1aOQPH z%sz)4njV)bo7U=R%yaYWo7X;)+>IKWzbKDYci8+8VePm>dufxli7@NP_6KA>c+(Tc*ES*?KL3%FV_s{3B zn@^!hiurm)HsCy1ldAShEn-c4qW^RUREdMh2T(@Cz`9dz{H|S}+3+7slY7X~qo3px zI{yUM08!%bn=Iz2N9E3#tZtRG3*NzR)cxV*Uqc{(y=YrMe7`|BGcb=0Tzjq9EA|9E zi+@iiC-weGbTZYCc!N93U0gLUjHpY7NTmmu=g$z>S2ep-H+y%Td51jzV2Y}&ZHEQV z=?)x4`J*8~&UkZs zmrhOc+nz3qAhKYenWJk$-XM-oEM)!#pFgsQ(V(3cyq{c|X;sE8e~dFhM+ z%jfjvAywoX&S_?n=2IssO zJiL#a{*0j+OT*;nhHP*>jrwIfzwVml!gS%12Chu&n&{llGN8JSfPFq!VoSj6q+XEC zCyk;p56U@eFC318O3)CR_iLQ`FzT#f|72s7*9Z7DEc9R_t6xR!H&iJ4R9Bfd{6_E- zA-`hm5PKLG2RA62HA>&&hU#C4oEgovb6LYOno@QFa+&a(fO@`>i2Udr9%Yg)w>A0X zbgV~ORpQ)uyG4+>il1rm6U~>8A_r=1-gwUekZ^v>%l^n;5E>J)c^buKRW=~cL8~xf zH^65P66MmwHfdWhkQaMyRk;Zr*+{48?K7E@o?*St_`$F5~~-(nrar}7s(oOW~!Ge6-ZOnDgh|q)`1Zg!cQ@ddl}<4CO}(O^yZNw z9U;UNo(Xt*AQLAFBMHNR=g*CkuoTiyoq>6k>m>4L6Mz+!Ita65I$tk<9&gJaJFb6?Kb18AAkSWr1{{i( zi6GE7l@4*+Vh42JM-tGjhXzBuo=&KQDeUC3z#Zq0MVLaUMjrYt08^j#6~+gJRRw;gExIgdO>P4x^gV`~Ga9J>(u_|_a2d#-*3yCXV zI8CF_3EBkJTotK16)~&sA;KgwjZn%F&n>DI#ByrBApyJE`y$EfmiQ%pIm?UgZ*a$5 zzUB3UvejH`R;TmBpV31JURrDWGARsh0&I{djea>Oh2cD>G8d%-AzHH;QzuyL5`W+h z|12g6wy{Fi4bbW3)}X?!DV>md_!mbt!Vqp5quiru#CJP~;4#ju${@weVR=9xYh9q8 zPxZ*%u!Ir5BU{1`_Q{Zgmu$fi7J;lBm&!?}wHlFhWcEv(Z40`Q>g!?`1nG{c^!fzt z{94oP%$cW9+`Vx=!x`TNEoMlix7L2p4%o+?=k|03_Rv<$zF8uDcjBEXdCaZ)EPDDWumXY-tS={U50BI#!eH34&r9}n zvrF=-D@kGHL|N?C`jP0+U8M(7)bPBFEJQL{Ix!$Eb2!(HDnVO8MAy`vWZrmqSe$Hl z;iHYz@hQ*v(Kvzj$ey%rft111I1q7XIiB&8gFM^k=ZjJY+1C7m;!n?FMveOCaEBe; zF}if!hTb)w4(eCURaH$J?ndsp;DzL_Hv2~7)la(5g2R$4MGU*sL&iDd)^V^D%_^j- zq)!lIG`j0{or2pD`3>ybxw*>Nb-iXQ9hFw^9hb?=?Dh=LIK|n|V#bHh+=`<^XUgHs zu}fbn{Grd#G4#d-l+2&~z7*d-n-(4Fj3_>LV@ zfdex_1|i7#%?qDOqsoJTKB@U#0AoxK=<0=Y|h}V`GfJ);CCPp7N7!%n9O6^q@k%rYYM6U z4%;_Nu&(zFT##166F+e0tBqN_f*z>P(G=LMM~1mW7WR#(X&4?8nDhd2yU*|_W3?DGcJw+uXq7(ZRCq;u+?Zh~b|kW1M9hc;duW+KbNZ-`V_KO|A$Aba9jRx0 z(_O2~Ku3H0DY9!$@L_$PjvyF&B-$a87Sxno2QPTlou6L#s=F3mK+8KCUP#Nk9$rvv zJECJSto!57(0`3G`bNaM9amS3?M_Au$aY^=3yiBF+Es673&PVr5O-K8fF8fM`38?W zV))B6N0I?}xWkLTFmo)yPUtg|Pn0QuI=lbQ4T!H-_J*lLJZ;FhBOqrV!ws!^H{}^k zXBfu~rF+O0pxF_;z5DbGvNgo@Otdwa12F9f?cS+=#`%cx-SPJF^W77AhauQCtPtfg zreT4_WE=t|WrUk&#Bu4pPpSk3#yCqI6UNlTBbEi&cI?Ef7Xe!c#SD5f zmm^Y7)WqNyb%aR$*8qm|p|<*epk@O#>PW*IYJZZttqKH518;S{8)em7WzhriUQ$lt+(fg~^H-2kar2EYq)b1{k~NoRE>JL#jG{b_dE@ zk4XfX%?wHhalB3)zP^Wv9@>MS&JF@0dHgmP+NIvhY^1^p)++E40FN7-Py_MU$E)(~ zZI|ckcLjB%iGXH-;fP(9=Y*shQ*{a|!@~+-lX(~y>T;v~6*pFYX`_S{Bi7)pgoj=H zcVLKI7HTkGGqCCYpK3swJ+Re1j#k(ehV!eiYA}{N%B(0ldi6bEGPSE#@EPMrwJ%mc znj`4{^vub%hZh`80hLD6^~zQ(p8l2kmb-+WzbXySS0$}L)<;-tClk%P`gDw|Yia(P z=C31{|0?CCz|KLJI&CYqas$s5ou=4JQ-~D-moPjNv6b};JeNeqh5ZX{mw?V;nL5P_ zX3re&9$q7Y+SpYrm*gzN<1fYx^_J*8L|Z3$!3gR{H4(^BgNR;3RTJh7&!kWM-qYoW z1)yyos}g{Hi}xO^s>it?eM|T5?=o&)alP<-1ZkhV`jX7rk0@_xn}*#rx~sMqu8$Ch zLkKH57vJ9^c?`ZLh}C}$Q-MUC0V9PBAU6Z8%q<}A#;od-eul21j);*${)cN^<7JyElRFM*OtoTXm4l{$56~v)iR>6i==2oKTNBq z8Z##s=)JOEU7EXy=+nwD&eYVBv2W$zT31l2ik@bj7ZfZSJaVuO_~OjDPHiL$GV>&M z;82txPg~9l+#aSF80hJX;WnL?%VFN48IqfC3(`gs-i;Dv9TJ&Z@jqoUlx!Lg)i&NZ z?J!J8#QEKsCFIzrcA2N~C-k~~fDT$d+s%ZnBiOlam!j*`-cH8w&Q;4DJP zwb$~-7@PB)->g;GdM47+TKiMaafhtr-Ho|8q+Ia)aToQe%Zk;MWNR1PBZ3v1N)nZ` zqGzw%WkSXsA5%LxV@FeXNVg^6%KL)VhfgXpE0R2;lxrh@ZX%xY4uyO8qd3O~bSOj? zjhN7m)tXpZpFOTETsCsF|HaHdl<$7jzv~Vqp?^Wl29dLkAEqgFGhL|VnriO@=;)3q zlr9dX3SY0kXo^<(cO8{ueCLF@mLV1G7!}I6u^6SR3o(dmp_+9trN|(Pf|8x0vYl9& z9B!$gbK%b<)1m|-7>BB{?gTuQAEg7B^#p%OY$CrC8+D;)_2=6J~>BD)));`=j zTnDauSHSLVi2bX6rY`0&%FyWv-mSflzLpQj@_5e!Bm3)@XddQeAL7sjN$DS1#=+ay zg8#bKb4YK;6Q!jCvOGNUz-q?<7t>^jpR(R_7_;;KgS{J));35;hnFv49IN**YUfti z?&U{|77~0b?mfpRB&{xJ%J@m@aol=dcOXK_^2jL;2OrU~;epc++^;7W?IayiiMK^J zq&YN^wq?R|Hts>h%abIhsGsx42%U?IXM@vcTsF#Bh z4r8xGNy|9ojJ>#;l`)gFy$D_7_LNGWbYQ49ADVUQr@YE9N%BcL6|uehbViXE2ZgQxDqX<{ zQxrKxMe6Z$i^|1`GvbC~M!VP5VmcjS1(aCU(vBJ`(wU zBc}!!u(O%?{0PBDBFdmP@r>I%&?CoYhfRsXrPmYL5<6%26UKGIPfQgXomq$kEb5wZ1%a z*_cq}muGhW^32&|LZx4xIsD5r4_%|M;0FzgI#wK~u9L+7zh}m^Bx2>~15a|~vIj<2 zbw-^hIpIAx)$tZ9hoQ{bcM~S!g>v$-+Qd0;ZHv;&=!j!>|GV8w@h$jchZ|3NYSON) zdbGhq^cQ40n)-v5^vDw9dbLR}tywmP?+o%^vlVI3t(O$)e|~2T*=M&7FN)d-XX6lB zW>ms1P|sZ#%Xw7`H4xBzX|0)U?lLS+^o8eSTHz z;$0HT7H2d~(Om2hCaBu-@Zjmr>NAM887w+{aX>e4!3>MU!zy4(N&XtIme@kkqSRDZ zt}sbaMXg}TDoJ~dqpg*FJ%(|B>b|ffnxgJ>Z%H*pJ44>>^hKJ-mF>+rY*Q3@lMQFd zyW28>(Z=gA_mLYNx(w@E&{EVQL{(R;k{whnXII*=38oqq#AGN+FUnNJFk%%Y27(-M z`|T^LIuJ zrvy2vlqlx#xQNb*m!v7cr+gI1b@ zX3|}QQjgQM=`a)??HDSF_R~!Zu4Q)xnV3l>b?~Z+Qr<1;5%TLjwWEs)Dl*l`8U>?t zRpj+%FqI+>zhzAWFiY4_WG7(te^vd&mtOs2rb?2OZ0SYalpVh@{xxT0((?g#*#1lA z4=GC=WG$6Tg!_{%f@dq?xFK8YNR;}M6@UlM29BerThrD9L<{Fibg7EgD$o|)Eh-#^ zt~bIQZBjFF7$`e6X0wKWB#IJ+%888KxLnQi6-^&(94m@F z<)7*;0TLsh5rTpvm~p!&f4 z&Fc+$4s`G-HvS2m!k91hZ!IVDfAMA3|6!Wh|96A%E1SvwchmnsZvLb2IWXGuufk{F z|F7@~MVnY}`p?3roSyJV9Xs&1t$8Sef4iIiQr0ZYjQV9PoLbHyRA;^!8;UEtq{R61FvM^+bHRdkPSN9zEGMTnwH-WjpQ8YoK z%+VV^RGFABxwQRRSu37OF#JwA8XSa<+SJPedE>Jb6F|eon@pT*b0(|@EY?$`F5$!hnU4Q$a??^-a^slku zEV7Q6V7tCg7*?0$$Nz2(a?t~Urmm#0`D58Mx_04?SYI&O*yF)ds2x?I6N&KTQX`?Z?2xN`38=v>@HO7DlnBtU zZDsK(a7o+f9Ahq4Wu*l&QwPG&ZyH%S$9|by51cFH`^~zXbHBl37gz@(3|I7yYkB=5 zMv;nDw$O!#f4$`uT=9<&ZpXLWYf)?r0mSJVp9X3KvQ}>vO(S!o2H+m%{_Lubk!PHK zOGv#)v%N^Y9<_i++FkyA({xQ=YR$sR)>d>x(e~=dWtPw#{JoiA5|XAfgPCTidFSW| z*uI%KJLVI`z5x0NxP(XB?O6BY-(FhTwT^eU`->Uo2M>kdFO{|vq^oUGQ-akHK**rxFYtlbV*(d~Yf7d5~IG}Wi449vt4&**^*#5L)< zy#jzE4iR8`LFQ=9FG)|CHsl!TJeL}ZSv0qJW@$HtgSd5)d0LEIg zxra`OB3+dsr=?6-d+QPGP4?4~Y?|2LBrUwBgl}~t-rDTKDr3h16_Qp-6`3#`@Tive zqhqZ--rEzpfPIvG1f#>>w5Lq2y|?avGO_-IT+{}!`)qwpAD8qH!-KhgeLb?Cc0RKs zTqCrU%^Vb*;}zx|o%~QF4*CTZu^f=m!AdB{=9bl}pX0nFt2vxTzo9y}O~B)@a1T*Q z1de)uA=`F4R=#4O%$^O~>TKDY|K#XV4+z`0tdl}LLrT`ao9Hh7!wMK49hHpx)zuL>O{ z@~x=l$b=_*p@8Qf`C-{do?Za{3u|nP3QBk#iO30c38P7rIt{QfkQodmPXyz@Sfgm# ze4j*r!>{%+mo;du*}1u8B*Q!3qHz<=%ukJah%4rmXDJnNaRPL5na`?E0d8$ZpOs#s zazHlSYh~Djcb)vO-R83!7^tdQ@pf+d39uRhvB!GzO4ib5(X(kjvG0V*3alh51Pa%d zP1*VsSJp{8Jj(_-XgdM4h+(j9>3HVF@FcEgG}_F&9K^WGR{dB%5k5&Kxu% zMUD^R${RHl|6we|Kv!Z>Qf3o8mqf~7S99zQ!;MJ%rL5s>FQxzln)4X0R?w*e9)A3z zI2m@~IPq_L_Ush!cJw_i6LXYG?bEKQ-*an(djjrpf@N^B`)<>W7l||Nip^m*iU~r= z)kwaOt(CHp&|Y-x!a5MSLbpB`@oAVsH^`*6zi$iDrz@XY^c&GNE#6mZI**>j5Vp^~ zj53_umDetmPBY&9PpS3`#|4WIsW{V#LN#Ne>$;q`l7`_gCr+=S4X>l9sb!oR6-goz zBn;Y3!ll9`=_q2~^du|!n4&c4``+Cc+&jo!O?(ZutS2WR6^$(9=UVDq5@mzDGcX*uuj-=Pa&woZthK5-}3YV-k$#d^Z@GXHctR|bxMFWAp71ytckj9romXA?D88DT2PNW-9pP_%z88SRYi~E0 z!IjwtQHx|Z+b$$5UL>#VwOt%!Yv&t|9~mVmdL&<;a&VCjvl9YBwgER)4tw)oN6Jv# zgeMhzClwh5z70l48BUj>lC>lz%B0ZgYZ0OVNmxFP!C5SDI!Jws7{1D6F>$pKe0=RP zUuEbQQhzwO_JcDv@^5|Z#P^Wb_;{-{#~&8sibN8D z;&ZvxCIOj52X2MT0^ezLWhbv;8m`s1-F*;W^+bd2-4EgtGwV(gha-esK0$o>5JP}2 ziy}=6Q~PGMx$rD7C_GKSU)s}TWi?Swhv~79jVIMWGSY^L=x_6^3Q4D@xdBjVCU_~lLXx|em% z;`DwTPXp9x>w6YyjAb$0bVp8_wx!9TDYe1Zw6~U zUaM0IDV`VbA#pCdviq$@+rn6NMwFsW_JxOxb85~)h|{Xi)cmIIGF__aD}=ry&760m z9BgAqJbVr>3#HtHfrCz-08SU)pxrrd@8m94U2H^TyPr7BT^;qQ$DB0=6C2pco^K;V zMEm0PI;eSM;mvgX4YWTb+#Pf67|ZN==Oyxx1grH@&JRWdt%gqns|3da6XzR2DCk7` zU>xv+Ffco-7kzHtet(d!&^RgF75-MjbJbn0c+_we#yV!B=v;eG8Zqdl`7XHq^F!JO zEpsIQhq|3O70A(Ma*jfDbUg89I&10NRjN@sZh&GfG}#`bd1-@;1`!9{3?9mgxa~ms z9GrwjV!XNW;R+fbe*1X~tajL!!^Rc*?-BYnS`^61}?)h50W5jScs%6SNc~ zxot;j*&iIrX3pFQyW`!e2Pv6lnH*28(o|P0l@ni|QhyxA!t!*hwVb@>?j&>gct3Sz zLkl0?@@0Qp3vMplc}rpS52Iqmz|s;tC|!cIw>Nn^?`JNT8Photw?3J>C@g@FHFH}F z)d+OCtbeZ?)`nT3hUHsIR;V``USSI|j0PhRKb{#p(~1SS#-s|{_5!#XQ=Dz2#gxos zGTEQ*;xjY<3=JY!Ybf3-yk??#wXTn8`Sh7E`JAar(CE;Lh4a!pqI9>R!>0(gl)<#HLQ#%As&Zo#8zq3uz5+ z;Wbg6a)-|_Iz!tRsixEPL~=e!Gs^-U0wz*6*Nc^n9eH(*L&Lgy(b1*zO+M&p1ux+; zTkQ0#AHMP%Sm^g~h{9=X)MZH+To;?QMq2g)%}~GV;0D1A>7;cZVoGbe+-7zFHLgEe z$d7c9*j6$v#IY8bi1@mWp}e>2hRe&6qr;*v7TX=2!Vi`Up#{0jW9j+Je{JKK5l+iK zPdM)|RUXA&CX(5R=Rm)03LP?N3!4Sy095vSHWLmZYxi-iAT$fQ6a%ROU8XOSSlT>gw5l`l{l;LRbfRB82A|RY_<{($d<$`$c0J9Xsr>c2{uR-EKYCd) z&yH!3|D46n63M5->1JxCld$}AM3LF?0mZvV`#O~^q{$?9IZJKd{z2=cf|d{3&caLo zTGb+GsG1uX8gZxpwA}}p?i;`K38evY@#sgB$ALiZ1f(f=-9d5%6GTwj>3^c0uODPk zrmNpfL4nLHWCkZN(M~1kJdJ!k*XDUEn$0PGuuM%9*%O?#;7L z9$}z!Kxl|TKkujsiQ+8W5n^Nd)t$TptV7o{yN;)=>xca#;f zwI;q_SdFm4&&M^86ps}e$WCb_bc9dn9R&Tu+9;sp&+YbiQY7$pz7M9L>%0`Uf#KX( zR@gl+QlxBu_UJ5t0h4hwJwJI!Zy#=R>+PSz zBTq-p2uQv=xd7&+iyj=Tv{LNsVu&nWB1+JLt zbPrk5jl>u4f4phcD7}Mw4PJ|1TLwwei*pFA=Z4Xcd1&24mr#6vMW2KT@Tr^r5<=S; zdbI$m#rByt38$2efZ#REJuDdBHRiIMAO=Q+rVZ~jJ867d;G4K%V?3kQIw=m(!3QN> zgeRmi$fBT_4qNiP)HoT;9~Ystec~E85Qag$aY7H-L5PCb-?P4oq~UR;3!g%FGsKmP zf4~i$%Av?|i>JU0yV4v@$=oYa^#%(^u`} z@){9oF6_4BhJ)*KF?fG$1T9v;ys%>J1u5*+PkIZ1Gx4*07q#O@{VuI0tm_dOW)ngO zRlvQ^%ao^q`)!dCEr0adw6|)8HckSw?^TFYKMVkL>JI;VJkcp*3R(|qJk&~ER^4`* zEBMi<=^2|EC1tj=HTx>mnLcg{wzD+5Od?R?7EjzBN8*I$D&+0Z$G&}?W}U|B$Df=4 z{XR#zAK9P?Ybq?ha>FeBSC=R~|5*P1W6_&QrsxkSoC_6~61(&1npm7wMk5Hid~f5FaUOWZ;PmZs9?Q1w+u(i3I!4KQv!D zh=o6#6t5dH!yiuRy=;wu|UmTUuD^z2icQfZ%an4RRM z?Z)uSit9H4G9S%{kDD+}&%TuQpDI?q>a4q^M}43=@mCr{b*AZK_C;PJw4cWI z<037?<%!UaSLB+X`Cb$l^5c?^Ib#REZwIu;srH()^JmRqrAfL=qpT=du-`tDKJ#uH zOz4tu*5s@oeRHLrD7i{-=U>>8T-vHWtr^~9lP*3ViuU-F7rLpScw0~@^{CA{K7K2K z{Xn~a|1jHjkulV?3cbw3 z*aER%0rD^ugB_Mx!RiWf6YqhH8X&&_UM#%CiVCTh10gU_jq+DsfotlAr|)@+gOpgo z^VFjzHk5r%zlzs!BDf!N6Ym9GC;&pqLwE;C*uh=s6HASvSbMVIu6X4&M=M_Ux#KkVl3Ur~`r75#Vuw6!j?C?TKUhX4T+@>|tYq z1#v^a#-o3s8?yoEXXub#yNUryW_^~Wp0D)bsTvHEKKcYdlT45_JJeNRmK-o2KSg35 z*!c`J(G2)DKhe^@ky-#8J@VxVKbefbJUvcvpH?b_qaC))Fcv$CX|Eai@S7d$RBtPN zNd7VBRG{jO4`%4yZqD;J+MeJRNQ`~th$^-nX*C2KJ1pL?aeFY{AS!z_-l*7}f@jRC zA(m(MszK*xP~Px!dl24WoKtik?3`ZH8xAjgwtaLjpys{EXXLJ^FVAz;^B3~$<9mTU z?eT8JLI9NiQXUrWsOo*3XVR`bt{X_F?^=#iD zbc8npSXuL7NSv7TMld_E|CuG^ok)3M@I~+WDiT`uYSig1MKqdxr%r6t*RP2KS>>pqx+qP$P~s&{ktUA2U#6TgrOStNmLW+*7kfpGHp2?i$t*g6v#dyZWpc*+zFOe`?^D`W&rzxu@{I5wH*aBTA?_ zkz7+`h2zZDZIrPxe}Cnf z%rl&Anz4%h2)sU~V?fbWHu6EG^-OiGKA}tB+i_UQbFeDA~giW)V(`AgVOdN+X1Qc6yF=$LDBYM z0I4&<`EJpDy9`yHR{ghP1qTW%%gOby;hYB zDx7%CiKTL%Q9m7pit4M@4APHMq?LS4j*exW0q*I?nTNd;%IDt>)=w@Uv# zegMp<4D_)&xdB{9W$o-N`t@<)GLOwr+_$omGtd3&@W|Xz+1rt$T`MVzby>ejDee8P zqvgocei5CV)eHu;6BxQ9WB?esghpu?X7NU_+Wz~zMgfgNAvpRX=O`826ldh@=?q#a z=QGACji_hkSZ*pxl5A}=G)WnbWQP*<4R(qv@pmQ`#nrSRUUdTs^=cYr%u?@BkKi+u zxd!7hUSro2#UXEn-><@oq}O>(Qa4d9g_Pb#^-kNrluZ-^M{Sxc#g>>ci&Njl*bpql zsI=D{jMH@8?|g6;;piRoMe*r}j{F$XmDH1TfD~6Ja(}{MCSX&LvM5MR{xaChl%_OY zJStTB#HJfwbQ6=3b{tq3LsC394pINrH-X}8GPN{8G*s~{a%dxdlXEiY9gp$kC&2fnxKF%P{(y zoA_#nV445d%MWxFUsbkuq1(LF*~gBY)G)-rb0FmC02%E8q0l~8YP8$R3r=Vs8wD9H z1&vpPD=aOWDAn<2Zls9&UFOfcX$kSW+@Dzpt`KX|UJk{yfgL&M>}Cl}nOly}1?HA%X{kn|p^6JVyFFp1+YvuK*%E|s30CsY?JAHWG>f=$R+F_b4 zW*hqlpF?ZgZHg~@Jo4CKkAt97jE|Ns``mWN1*=6HT$ZnyYP239+n#p0=Rth$Jb`;( z3jt)gzvqG8jw`;U3*1H)xp}?kf!S>>+g-~RWSQ&Nxa7eta%)#OzW2+!aFywgWu{-} z2_p}Z2OcCI_qTxKTa;FxaII?MY3|r9{5KwlmM)1*U*wdOfkR?@-f;XsEgi(2q>~D| zmZ?oZjPtL;WY)na|uU4r(LI_)*q0C@j>J#A=I%!c4VHf_h zKa`jfO;KQghm?Gp39j!^lUuCrp6dfF^7BR( zWe6l^Egt3dGM6VP)8@d%9x`y5qz>r==8VM@nUA~7i>@K1z38gOjf)gN5+e-D*7lD} zirVE%UscvK;W|vR`;J#9=NH^v+6*f!7LBS(ol8(EO{V0M1jg9N851L@g{thZ9F&vt zmW;5*xYa`#Ws~xhl(EKyMMAXppTg4-_U08i79Gd_#P|Kb=lC&c5LCFuwoToo zsAUk;B)9%$=F0eh+OmLctB?u);>NKB{wL?a($eN>L>Z=5+%1{iOVBEeZixN)5KtMH zHC)mcWZRmIGOuVA3w0?IA_CQm$+3wsWTq@Q)>#7^`ZCLzT*qFF(Wn}<^`Zx282 zUrtp{sf(`RyaJsM_Z~i6gT-VotJt`Ww^Rcs_K2!nB+{l;=VnY>>)4-c|}y;#{6FP4z9jU{OfmkqZ^# zM9k-cwH|^+@DE59fLN(NpD0{N;kPPHt)f|*y=c@Tq=mwwH7kWhPm4BJq3g{NfDN%H z+nkb*beU%?31##Weo@8P&EwVlb`jlbJ6%KCb@#N?{S{1`;?J2Z-~euujoJG!ix*Q0 z0as|b?lDFMma#>Aw_%v?2QlcnJJnvDqp(!P&G%(g>91A0PxvJ-M%`yLx&CNMBJx13f)`gE%<@ zaV5RfxSgK4-rXL<8G{}_xD9`~s0wRXYQ1FOZ$xR}>Hl`;{soDcS(*NanD|$!@V`58 zU%ZIp@3#NJ-YF-H$7Il>hWv9XuRon$AnJDptwL%K`FZU{FrWm1bj*V|k*%)Bm$cG| zpoHn}>6=!&_}7KZjgH!bP~DLF%@yYM+4Z+uK%g;(XBHkxeDqz-rSQ4kxvGnJz{89C z_QmDF?_j*x(P_dhwojv+whNfgiTG=u1|OH@79_yC+A^DGy^Uc9^{QF%&VgmO&$(?f z$?VbgR`sz(8Zr7?<+$0kC7;@Md%3X~e)jE%u}0EF+l}u_E^!6zJYemu-zR-mmiT+P z0@#=U#V-p15cza}>Ju<#Bx5vr((nfG-bizPFnM@sn0#OtY%-XQVy65ukV9d*FvWh! zXxw|SLjJINBtqe(9hX2W%@wSH)!&2%Uw=3b%=Jd*sey`VOCbcvR#Ggs%zsxAX*(!M zR3ZeaBXmbxIy?U<(e88WFxER}_U^7XCKa|4b`CpxeLX>a2dyP;{4@Rsumiz+ z>%Uzzeg!YnujE{1!wGSMZskCkF6wRsgr+2{@CzGL;6_3L3Dv7$ z`uO@q7*RxccDtk_&7Ebt`H|x(TXSc^)?#_|_@Z;^b?#I092n#qj1DyJQFWJ%&5hA@ zh;F|tZ)h{lm0Rz=bJz`54diuUJ3-g6fBQ$(2fSC-@H*YL&N=jvMsRr2lo5N8JJg~ceM{)mk&*sjjr3y4H?0owayg4L+H2cuDg%WtCq_4 z%MeC!f(O7uFoccg?e36iAqG&6J94+K?{nTTe)l^|&CSF5;JMhfM+@DJ_VX*#^BOof zoiQH-^a=Y%${yDi;HHSZz*#Xt?}A`{Sm%Pv?u1k=^!4z%&y~&>9~z*&f^a)D%zhlp zkQ(xuCTUf-z&ss_DERSoBpKcML<>Ja1SoFUmRoAW72!K<^uO07G8wYJ!fZ z9ugOP-I4j{AtR6+iung2w<)3JZ1b#2-3nc9kK@7Dyw~9Ms}5q<0U{r}<-vN#2T8ll zG|!t44~E}!L-XxZ*_$!P@~7N&?9+qwZXs_WrY*9WK(YYe{gj5&efH)n7E_t5*84e- zBFS@zsbuWv*Ir58mZOOxI^3SqoI%@D9HPPDoK7}A+Z+fQ zKhIccy?6Hp=R`8Y-HI2VVFPz@*mYmwuW{dD`<;tomo%f1J7+B#u-RkkApjdo_c!NL zxC6hKr=}3Ihzsw_cQXK{EivDF^x_^>AT|Zc9u-(7b-boRv5~v;i_tm>jp=;g&T_TyPNKbWSCBK@SToWx<`OEz`|j7&&@~cGc_J;cAx9#yKnDjzU)Bt2d;Y>uY_ly zYX5YP<<7J(c=v7kMarI|_45k==f3uN3T9BfE#(vcDp-Wmx9kD@sck&w%w~FbDcMdIe82}p< zm8MTWfABoO(2aMB|BX01pm*ULMQ&er&nYN!&(q}iC-f(Dx5ReO-P$?0k93lBmZ|Py z7!*Lok1wd(kD!O3-w|NRFpwe$X&Xb>r-+^63;GGZ-J|Py&JjqZ#dwPA1Wr6gsJSg* zLAZ$A0uxKt3%84~gZc3abif2P4bbN9$FKwCjpm=ltolW5OhmX=xT(_|I-aqk?h`g3 zu{*n0QFOGhq4goe98uk)nteZ>ele;sTU4(5VfpO(OaqxKZn`i@W2(fGj;a7V-S(3V zqu6D~c88P=V4t{oDK+E7X7y;b%-hvI0frJ=|tfm-kv^C6dcIk{$wlusmzHxU6A4At5;9R=z^Wx>GX zmTBcxc&A}SdnZzdUUm|2r}-0b>1Xb}xeI7ZJeIRR8<c9E^&+-xUFihU@Ex&y$_F)yB;W~EAaE5U8IY7 z42*`um%aGn@=-w}$f;M>+d)-5^Z@7OG#0X=@a*$xDRmAk3W+*WS{5UEVk#PDZN>g$ zZ3QNS!@d<#*nlzg;$dpYM{ z${XI|U<`Xe_D-yI4Jm}0#3Kj+@^Z&?v{yVKL8+#Ytb}vn3A|q+KZpvhlwixU9>_eGa2De6OZhS+Sxp` zL-T(CO+d20@O}trJ|Xx4yr#w&m?i_}5*!IVr_5y6&(|u*qUywM@EpZ36i+i1-gpiE zPBC*Da;1WD_piu50SZwSnKr*38=jIx0T=QY`nsyex;m0YIS=DU*NIR#yN*p z>R8Bhrkb-g80v=~qs01%O|(zOAsPcB&W1|RNTI8w{Qmw>t9O`doSkl@YUi+|C{LmK zIon+s^krRPb33^?XmCw?>do+G(x=JgzrqoJOUPsj?e&JsVrJzO9;BL~d0L1oqxw=5 zI*A$TiLQ&a$RtN_2Cho>e~`dA9Nu}n9T&F`9*5%}#K)<}VHuCd&`zDf!?_Liz#jaM zmLb7;{TVYpie)&6n)q78 zn6r4;_^ddEdaaHB&$V!z^q+%;@=j~lz;Vxj@f_spnj#4`D#Qh42iZSUpu{w2cQfs( z4g|MkJX6y3v&vbx8EFChUWv2We2U!66(`Ue$fGyt%ekWK7GLNGZsb)>ed#Re&n7f# zaA)!Pokue<9@XIhx3a}mf7~=YGH?>F30zJffKqX?Oc$vug^KY>+JJ^=;$M6^97CZ*#VkI z&*zH93kxt;GmNwo7=^rTfO zigZ;j|AUZz!DEelF`x06IB+HrRE z?vKFTsuR1aa|;|e9Ze)Fr5rFhB!7ujWjSyn`nE-{VSXp++EgvMGQ@`#QYL&QZqa8N z;>-VuBU~U`Tg>xlpI(p;rRqS8En5}j{(;upFu*vNm-O=_T0*@m|p5RtZAn=y%5Zj3-_FXZoZ&J++X2428<6qL{obk1m@z8zMa z{jfXUp6}sMWmQ9DET#3lcP~n>clqz&JMi1cHWla)-ppAz+%z-ONhZNGVj5m=IQYR( z%fARGE&G)TF2iRKW}xYk*78s3XHSAOb+7&{v6kkXs>UmK;~1Bb#4~st zhM=zVCN7#`5Pex31#tIA@bm3?*D+$$a;TXI@BAxTx*7tCITS*|lKn2CX*)yXLjNI3 zNVh^wXRFCgOy%q@=d{F4IsP*IH+V)V}JBi~&h0t|eALr#= z{#(@Gzk^pap7RWralmjvdzlu}5bt!g?k>b4wyTvSxbE}lh}{Izr0zI*M=Xz&zkPtK zm)~_Waw-#^sdf1>{2%af;isU~l9M8Bx+oR+GI~behj*iE@*Yy8_$wVz`M={>@`N6Z zyb?)itm>qt62wMQa#Xw} zM)xl`x-ST;k5>b_if!s<=&1h49%#Y^5FAG@LY(!x)z=s^IP72NZ;W30xc=maY3Lnz zrLX{#@R9+n5?|kpIu%w?N&^)>W$&>RLB?|^hSo1 zwpti~@;+t)daAdRc4RxZ!l&VbNXQoaavEqsqDSyL9NW8aJ!B#2p8{oWeBfD?O>!;FJG!hZcsz+RfMhkCXT&{ zD~H-WRG1g2K_gCyf@ubZhbUM$s%g?23OC98aMa>?Y8Vv>UZ~|%9&*i855VU^nt(^p z7D)*Lb#@25lcSgH%bKlJG_|&~1s2M(DVwg2vO%``O? zyGW;R;P=LDa;Ae)^>1&som;pjvvU?lacA}&dVcP<=iw<7Y4Jj>3U2c1zR<-q5>%xThJ;SJNZQ2!j9=yp=s(B;{EYNd1uUSGhmsfgBT zYysM=2HLDf?MhEz$TeDqH!~6p(jOe|^bayg<0txg&ijYZE%>r#%u6EHDILW7n|sSE zlMK~me;rNkS`A%H;$pji3L_TplA}?ZfVnTqD*r`mgdMTv=^RBv6Z1UMiX7TKNEZZf$XdwL4$g8Kewj%)Og}Pm zxKeg^0ntd&gZm5GWWu^3bQY?-jH#{XYXz8;Zm-d z?szBZZO7exsMkGiU)K_>t3K%ZUf|a9ZHc*&Ge8Q&>zeUMb9rtmoLocZU9K8V7+667 z0_A;M@JEWC$Q+KMTBdbNM@S?S5<*fLLJ`_59j@Re-upYvLFades^7zjQOW1ZIVD5+ z3kU6L42_0_daDs?*Ls03$+4LjZeq2nhB@*4ehI>sArH!K)l^i^bUz3mjnkYd>xr#A z+o&2`YXoF9$!Gc@%9aOJ)U}gjYYL^D(X4Ya-ea&t1f{n!DNcZRdAPQ}!Vu%xDkAh9 za1F9r_#s*9ixr7Y)z6^AcS8 zER#~(%H>CK2wKe=Oc6>=D`COS^(Wk*pA0OJR;#|&MS9t-V7qc#5cT~$v{jlDUxjYu!ctnyYT&M@rdx5yX;sidz4b) zlkob=*hmXV_h{X3ckM(EcpMrQ1-Im3Y-=nUH-y$*P2H-=&ok-JQMpn_OnexJ_AWRm zFtrgT=SYvYqgY6}^J^ATGUU=R5m_KWMDc|qC}_9ID9g)x)Kc*i@P>3q=^rDdX>9Wd znb5|xyX3qW(s&Fui!}8H8$sAD)cE2qrBVJMye=?% zyHzv_s%H3|CDdEO7G)Y~8fmdkx_4z@H6{Pp@qlvaE}B}mZXjO6IkK#9(C-KN8Z71uoPC3lVg*{re{9(8l?dO{AaB2n zwyNbqk(*Fa-w{=&U_dlkUS*Sg$0MICY$`*Vh=mrQKFcj=V;Bu5G73YFljID{N_gp* zE2rTm<9>KVR_}+^rrlaWqhLm%Id=hxB1=7(HXi3PO07=XCim_DRLfQ*^tJ*-EGQPp zg)5#P6*bwTFz5^fvSF+EN#m?zF78)IN?xBIT0Fwt4P5nIqz$KM+4h}o9sP(aLTsxd z$R?y^LwJ5tS~d}?dPcU^5iu;2?v@z48abjL`t;NbO2}h7*t3+{Xa%uh5kIY)O3R5Q zl}nGo(@t5jdb`PS3c!pd4fYf!(WXeMDI+sh+hsViB7U zOcHawsY*>Vd3_u?*3?m@T>NCsH@s)z&5CC$wpq@hGI>Rzt*cHc2OF`!KSBypw7P)S zAN@ba3t7r|%nM3{j%Yq87e`+0RyvLHJ^0COlq@A3+lV5QbgXrY$aUO`?|&F3-U;ul zD7r!Tn1uE0m<=t}v#9jUb9P z3Qr{IaX3?T;zGC&{(;GiHN(gt7H8bq2`5b@CV1*z1y4ABq*#{X`*7{NdUOf=8&yk@ zmiDTp{HNhrNzSjMOuMoX8pz1u&#o@0UaRfFFdIR9F^6`0FC3>4?^x)|du49-b~1_a*E$9zlq??(^b2h@UEo8Kgb1TM1NMB?ZeEvrxgJ zFh_wCDXWko{A@bfUdTBJBf=L&@qw6B!DAu^u`XOK5im2ucyqDBdTk|c{`RF(0 zOa-0LFG@-q<$?5)aS9ixc>yk5B;EhZZ9;OOS(1oBvFI1?$l3FIlp?a-&j%@GmqZ@+ zJE&Fv1jjjr8gS>9Skq)-QqGXkMyyF4k2Uo+*@?VfWUuaspEefz2T6cBjl%I#%L*Ac zf`|bQkuH_0>NV|@^T~bcBgNj!y2liw=I6?-|B5ErNZM|Czh!#pD}*qpkvBxp#9zXf zE%0pH(pxKt?iUkde=QHm2V^~;bD~nnBt>^Rxx#p!j5I4nQQL9qr@2%%^QP^1K{ST@ zvRK6!I^BXy&#Fc}inz|$$rp@Js za8e%!;&3DsK9uy(b#=ora;X{HZVB0@2vj&z!}}tf1Pz*@tC-enL*KTScgwZPM_y*? zxZoYtrf4AX*k8jxAr;!_sqTUO#AEyS?%clEMclF#IjLHf(YYdPJmjVgvp#ZPv#2a- zI(oPoMPvr;gERx`VO8I!HxyMxNMXx{u4tyDN~~h<2T8JZSt>_?X2FXYH^YFTEVHte z#|M#xRZK?%`<{WSFP;V0ku_a*$$~qstZ_E8?;CI6N@_dS%wb+Fd`cF4>3#)%zrxt+ z&%w_PZ60BjP~0kA_%WD(I|jl5^MAUI(t?a!%5cyX+DC11J z;?O?XFy#5&imaENPr$SB9IA)+%Vt`&>iM{ti>NGROrDmIiip5=uUDod10*X8Wsl;L zz43UZs07&bpa~2k8CNr{EL7g6ojn0pD^{o*xzlj1YUUF=uKLqlrafRiXrD&yq(ML%@CxLucYna!RL{9AO_u#|w@ z1>el=v|=~VjQ)^UHE`oh$?OphfCXFTQAR4$5SIZ5%a$a zU#l=!{YCgi*zh;WK2?MdgLrI)@wqTIw5gh6%BnpJ(zLvhp4ehpiBPM(pR}pOpSR$k z-h+BEgsWj+JgdnI6qzs1AsCkD(>uY;k?+}b2`R0??eN%jaNQEcE-Of^E?&Mkc>|oD zOBpkzV)vHUh9SAiE4!6y>7#!{Hdh@=);9%R3aC-~sAEyK@VA5~6~vQ>Ce;=qx?X2$ zBC8x4)&N{N+VM#GFb{JJya5M%4`MCLCZyT)DeAR`@4ObSQdML)X zkBd9NgrS!3b2qT_C9?Z!$DVDwm9^KFo4`UGDgeuDFELz2NG$U7$2RT_s7czgG zZG>)Dw}x<4Z6r?PN*>3Xj=s|Si6;Y+!hi0UN>g0v7QCEcOR_9gU9^vqil!^dx0G*z zWpQ41L)FCP48S5U=lV5k?TDl>wmxai@1!SlEvqwkFc-(Sn6BsOx|Oqi1W1Z3)tfS& z)d0Wj*sAgJXW|fjw!Lz(^xV|D;9XOas_4&l*v8bABh$jwO;Cmo+9OQ415r6zx zR3kLdR!oQt5!HOeR>zvKW~_yl-}9&ZIo}8sZh4OF%sauPIXT_z&$hdS0I#U|$h7C+ zr*Nk>Ry%ezD!vHGd>jQZhEG9?0{$3UKr$cAFhp&$4f%8Y_ze&8c+mY@ML6fJkpJ>) zf`Ds=yu7Z|Dj$9dKkwr0cnawwF`iaMB=~KC;B#mP=8)n$kml|6MvfUwVcQtrG1nO^ zX{N2}+hu93R4DCQoL^nmhg`dXM#i{eBgvp$$qZSIM4A;!te!;P8o)drOL+Wz$&cEK zI^m#`flOxKSoH!<)RlT20R3mtUC;^Mcixl3= zj$UQ$u?}m-0fQ?i4)sdgO3SOA9k#jOJYpU{s2-C}u5a(Q*UD>#$aRk@c?1e24Wte8 z4HO=ttV33fB|@QFz%!+s(FGb6{?OxRkczIL=Bt6lN)G)S_A*6xWd+~}8I)L(_fU!h zA<$__t_1<2A|K3S@FeIC!(Z}Kcy|?2dk;S5rbW%jPIx)oRiON@tVoVxmh$<gdEmr={kL`Tg^UCJ!B%?$(=qZ3|LV$D@|Re7oAaqORsAWCT%{mJl!fuw;8> zS+6^;Dsfq2GM5T=sA}86R${lHeH;~UC<tJ3r~bS`i)k9U*6{KY`xlYn#fa`IR zrr1YYLkl$nzXkUYTD>vED~5`PFrG6OxHW`0^;^Nct|@6MkArXmf5wl9N@FIfv&h06 zCd67_Vm3%lYn~4g>qfe47`NT@U&j-L8&l|NBNs%tDB{tE2bm;G36+@L36EB~;nZ}1 z>+Lzcs;3(h=z0f<#DQz*J;-va+(7*jFyC8JwK>OQVWOHwt`@x~#ILvtmu=*e5GoBu z;cZQE>}J_9bapVSNH9pwavm}ds~rP2AH|_zwSJ zdL8M9+=8I#Ft7vg2&|hAs%m3i=9=jWZHbO2-uAmV56AJzZAmPO!B-E1s_v_pMzqZm zUkXScY9fwJHFmR9b%Z(*XSQ1K=4utboFDk^{GOF&(cY>4bm`9|IPHq|fx%!bRNA9* zLQPnORgsSxganipOBBFkw?lO&+#1}d=qKXC@N|RXq%7_PC@`kUko%OXz8mRr;7zbi zLLcJ>+dus{NA7FRD2gFV;8o!iY^xhaehv=270NTv_fYJeT)-vEQF$KT1gWpV+#DD| zGzXMpU}!i0R>VwwkyRfL0N|**8Ulk)EOBkI!VP&NfATdnn>8F63{$T|St4^a$_Nh=T7r~+#ub$x zS*P*qood`}|BM-<=4!Xt?n0Q~uKss-P>Sjmj$S&FhKk!KSIpHJ#VM%Vo}RdS>VUT1 zGF-pETi-7t5>y-VN212LN@i+3Rl=CO^He6^Jjs0X=s)nyuk^){DjsrWjG8PKe)TT+ z5#H$av${Ilb|>FE(Q~|7Vm%v**dZMu8qHALn1#HdbhpXd7PIh~e?mBJ7kkNB8mHw%Cs}SC@CAt(ESF3(09sy&<-5 z(baSM9dNe<&jg?ts7(S?%f+H$2Bv~!jEYp%>zQV0#~LznWg88cq=9a=?y>=IU zvF>_{Rt7{{lMDPUOx*|^b&oC)okB$#W zT7TAB`}}g-n3#L7j}*-gA0a)(a~?ZWg3|Qx_66OvW3ON&w&w)om1VQ|%0?$j3QhDX z5v6CHC^66aQj}Dey&!V*#Ll`>wpJ<+wsfX!YM^We8hSUE@b-D~+Q*KMGF4);JyW%2 zzA)F**VEJoa?<_D0u*|9BZS$*+lOUcMy2mDC6Il~@pS-0SmJ-h^>;TYAS&ij>m$C- zg}HixchG*srAp5-{Ipp!t6nSW=IyfHklPNRAjE;4Z z4-yjm|69n8uFqA?Pb{Ab#z{7cm;VO7i!+}gn{L`sLkgQY%9dxkpLjnVW&_i~utj5a z7`Rrmhzv=|0ZCD%gEAmCh~XB>{=r#*MjTS{ypzNX4x+AB*i`CH_)bMqRo+;aB=|7= z4g^S*0Nzt43@6Juv|_=l&;rLkv=X=HR;bcb#45flVxEVIqxSY^cTDc@rsm;;ZFOpA zZ>H7Qmp^1o9OxXFSnsSG!$K4RE{4H&ijUE#hsded+afr^*U12^_|8VY?A7FYlRHxZ z$ZG+UP@h<6YA2;KeNtd338>I3aINraSt;ibD$N-lHM%OQh4=43F{_X(lf=;^TuY7( zRmY)KcTU2IN&2KkUi8`F5y6c`ssI;d)hu@{%;eKgABOBuhM9&hiJE4@j*Bq8KA|+* zlQWCOdcQnhw(?q6n=;5g)hc?SV@{ZjX?1S4wa_y+ZPnPchf6a|^lU1S5wp7RZS)ha z74aph8M##CWFzu|XK}2=cBhad|K z%C|N|xV144b*STi(jf(;^!Jd`+7?;F`Dx5&F)J`D#}yv(l$Tt?EJ9H>on~D^eNrta z`J!r!$EmW%bfintVcjv_CgDRk@>ild{5taLucCr`<;L+koCh71m%+2UX*sd327F;|m`lIBn)22|cdl%>yBnnxe1|g`EPM z19VOmsWO=B>qcwl8nD8|fZNRZ6#!71oC$v1D}<9?X0z2TPy9<%GDh!`J%h6-W)8n zyGvaW+u9nQ5VPr~Tk>xrLeU#8$Y3il7|#9-Rl?g64!j#ihPg{|L)Bt(a0tbC`|#KU z&%nvpbGf1^(S9^kqvJ)4*>1qKoK|@|LQ6-~l`bi$`Ez-dsyL{Zw~e=sxVaEm`t%C5 z4eX3M&{@UXN*mhf235+MmMgd!lK8HO?^3paA~vVsG%`C<{_xuNqfW|?47MG*{@Q9i zpQG*#braY3dg+kZn?|dbX6gK_5>fESu>3pKdK;V~mS$8=ot6BF=l>(JxEHH>DL^OY z1|*jn=|+>oJwUlGTmeRNQ}A(WYVw>s=c=XoXt!AF zy|DLIympG${+_7DE3X&O(=KR=Wu&nmQiKF#J0CgGFgG%~Whg1D#2gGLDYp-`4Am{( z0MugpV}H^2uGCa{`p7)`TUQIyJ~)x@qxPlQP>Ot;%yi%c}4k^er_dT)Y{f?Xv|OjUe$ z*T_rhov%<|TE+JRT??Yr%_Yuk;Muv*!{52+_$q!@zxQns52E?aQNKkoU(89H1+{}= zY-dCH4ziZbkv}?8T#qqHwq3m>Y~NLkQyu#b>H>lHNfMh$@7q%jOWXColyt@h)d*3a zD6N&sW{f7oa|Tu2$j>3}3SmcKim@lUhqe7xTUSiZ+B=N>qkQW>;8+`|vQQGHw)Hv^ z4%d>l+UG>utPI``R146GgjJ1c@_htdn*l2XdUYRq>ewtmfea>HmWd0tN)NLLMbQN)({t?W>~)yd8?Ll7|-Pb z5x-pr7B?fA(%oo}WiG3=sV4H1N~K)$g;))$svi-(tjMR%7gqBKDZV6WHcC(|)<^A! zflRvaRv%q2OEJ+gMtSq?6zSA>n7k6yAdB4e%Zm4Yt`Qx<{R;d+71-b%=+tDK_(PFf zS%)LE7R#MhN1C6_CwYTvNmn;3uD8mSHfl~C%O+9oOWFsjf+~CHsaQYlzdN^ujEZ zl6&UJX`Q^KOstTr-;?hG?D5^Z;ezW>^tDU@)1VN^rsit08Cw*8yHD05Ri~Q$EQ><3 zUlSPRBw9uEQwHth%H&G-WYgD9h4rh;`__*hHhZhBiCX1EaE;ozs&Q54SmT&6>hL1K zvf+1bf|o12(_uqJ_WhqIfEYo#^Qsn!E{YgeLQ`!@uK~K$x*AI&ezPVzeO){l>Jy5i z<97-m?Pc7y0;-E(CEEuNr&55sX&VzoEgrPsOoUdI7{%ouoDr=%b&qgvjfq$PYk0KIEDZkZab8`SU6 z3GXB4$`3uE&#K(Snz!d#XH7Ek_b3RQDmR`f(=I?jQdSuFCHOs5)n7wU`4$p$;e&lo%&Us_hBD7Mt3wC+EA8f< zT{797POY&~hq)Pd9+jMpLeknM+)_vV6{rN$E0vy!F8G05{t#DHj(FDNh&~4iMGtj@ z2M#5`L(~MTu#-XSglEWpx7dJt7}pD)NGsGd%BqSAw6reErmpCP?VkMFI~9W%|DS<_IvbQRV2Dfq-0b?1`r zy&I1eV5v@6h-(H4a!aI=1AO~V(Zf&mc&PiUevA(nd}u<>)+!B7B{o!52=o>Od3Pkp z@{FkzEw?AcrWA(|S65#q5^abJzY4#PHe<(I;jQGa>H=SX3)Msu2SPHu2b$}(!HUPW zv0aQid1&Hu@lD#3ii^{--ImF50$&m%=M+z@{I72Uf~Zo~E5kH1yQ}O61L{?(ZnB(k*Zv z?j(M=<)(l=kKK;GA3x9UJbj&hWMzG2uyV((=X~k5@>02&@xv8i#&py2WdWfLp2%{C zgWg7y^2REu!*=DtR`4@gq%t7RDH8ZWdLm{(hAl`4?yr(gz<(i)Cw0w^h;F#C9+RqL z=jGessAh!-!ezclB##aK5L`{BHRt;RFO|XsIpr0ihio4VZi8bfHRCHPDL=&gg^5JQ@83*OQLIF~s2n zNzXDF=F2wY6rQXB@z6o(EJbJ&JJcfO>rUWv)kEJI6;icY5!t39D88%1u9FL7_bmJ| z@P^--OyDIcmpN-Rb-!n@N<&x8iZ}E?NQZP#aUoKgT4nuif3lwBEqS;(#?TgQK&1_P zu(8+8U}yzLolw6(vTqfNsG_$s=>R$yd>>fs zX|!;(yA_+IYQcGVPL*bo3E!1BBe2>ykOY|*(HHc0t|JQ{7X4;B7M3QX1AHUXy`@%gOW|$$(@b^dkPJmvlnFZM%ChxdoDX7h zWabcy>o0{`v%f|qDTn$KeN>O7GLG$f^#)l#uJ4D#S>G$WhvbdTI^9R}mD(u@alPT# zMbGDoA~o7*#2~DOIhm3IS{S@Qgn>(*1FI#?4k<}&x_~xP1tWB{8Hs%fimo%vo%~41 z5LJv(2GnmG+R?+4{mE%%h(b5!LxMD#b9bCfLf44Oj_!Cvo^WJF6(}##tR-Ch1~gb! zFtV2v4Qo09h5I}&MmC*Awefy<3|4|5m^2Jeb2K^RhF5kJO+jB#4a^FhgNmxooJTYn zfY$=e3k~4veO2n^=Bgne%>Fen^*gR69-ZkhIZiaEMw^N;(P)FnPR{kJ2ZK?~19|^c z1(3y8*C`^kqHb4rtNK9SsiWh&t+iMx?AWzuO%dHdgRPBV={P4RnYzi=Xv4qL#C{%G zn!%6=y8!vaq~0`V*Hx-Mk5exr;$UJ^l3irW?MUKw5TgE>1orYp)N-&W4vi{Q;5qmZ z%M2D%OPb!j%`kC3`68}JV8E#Zlrkeh27ljfh%PX-4^K?Ol$X-ibMPgRp5fvRRSfEIb+cdP!f= zR>G@y>V9_~SdvfBHj3gKu5@ZW36k?v)HP9vUFv{!Ch#-K^&f@XE4|!oYdsR9tP?cL zM#SC82oxStd~qRZr2Mr)iTSKnq99}N?Fa;w?z)9 zNwG(>7DSHCt7SK~eA0TWo9@+MwEgL3cQo;MKyk&TtK?D@uEAAW6E5U6@&Vc2%?W}U zf%iVN?~-JOobN+~b}tTzcFDP_qu7;$_+p(<*HOGm zzuBJTqkR$o-{`mLMDzyZl)PD1ss+kMwsY+%bn(*&IgV_&#FbjJjdZ*Z?#Hp;FYfQ+ zO1Bw0F7>}ZU>JtE3znUNQ-~-&#C(;0%zQh~sM*Rm6Nn^dK$(y;hExv*B)V zMHaRStAXzOmb*tW3^|l@vb2OchN96Cx`hwn{eJibH~=>y+@;6e_?B*Zjt*)rr|O!E z(CZ=8L4hb63!pqVEE|}mJ9;K|%=(U?Qx!?Qk8r(FRd}&%Vi9#Nb+Z%rd>nn2_e1F}5A;xxRIWHGV#^ZVXge-^ z=E=0bfa-IW?dg}*3R&n9qs}zA;ybjx$i_~#M)O1tYKU0?;h zqJ2I@gVquogoZaYz0>dn)IB}UsdPPO7?)168qiLuQA!w(WaQX*s2`Hy?l0YJBRa9J zBqf`15ptu2h`f@}f1R6-N?<}UO&QhG9(X#3p507@x15c+3y0H0K_n^pxGBuLgs}3i&!G^ z=Dig!SPTYp_2!JW5cZb|S99|C(Z>&ZB(@nWTD}{Ym&4*vEBKPBkV? zdDXvww-U(>+Sf983H;uZ$ePQ-I}nzYdJHCE7iG@kzK^=JS>H81yF)c6OuN^iPS2U0 zn>DZL)?QDVme_YKiw<$ZbXCM?h|i6G4U$u^3qKZye;gtd>-9~kh#j2n+i4I0)2yy;7pSxcPAnIS6)wxXeflPxEENK+iu{%6@2-e`v%^`x z8K7N=H+N;+C?MV3Uv+ip%6CPgZJ5atA7`W0#`?|19&&I>*CN{ZCa{f0GN zTX~*E3594L!i9QbDuyEYY5G~kFcsx^yM%1KAaZ2i7BR!A(T+rr1I>+Q1kYrC*Jg9k zX4GM8`{L@b7g)%u54stt){3gciuxLO2K^QGA_$s=6LfDW}ji;D*eem zylk&O@o8^_d~J9X9FTR=i3twbZRB%Y>^A$&cFyZ5T01Q38r%m~C92hS8FBI}PL)g4 zFtH$xPf`Hj83IG5&X-|vI+slxE!< z+j;qihkKLzB{bptn#H^ezYIyEwT^1wf56*Kp8<)FO9k?u1%vVD-zHyMzauVI8=4X6 z+G$4%Eu&;2EoDvNh7(YXKWHG<T1kE)-@$p$D*w@Z$G54}hIpU;c*75U1yID_>p+RN@(L?5UY2!LH>E;pbARB`?lS6TkrsidCSr@T=(+s}y1?4G z3I$6dA92oW(LM5B2zNsGZbGxa(hw!zDyO}I-!q^b3SK`q-2aStuepW$koWW}xNp$g z*!5wQ2NKnu@CoF@*C9*14%%U1VQ0y=O$`}grj*p@e!k`Q8fH1Esymcw0ihpg}wBm^EeW@4#gHhiu&#Y;{;VH;GRB z*WsVxgT|ZbTGD*6N+_yGU3kbYZ$n@7W$==taC9;df0aecMS6=OL`oLel`V9$r*9Lw zR|fO)--s$Y1;}m+Dk-!|S|OHIlv3NwC!4!C539blVMckFJ8!@%h1lGJy#E}unCuo4 zK)U}4I>EHH{q^8A=VAUlj^|x^j-254Bh5Zf`+yGjc%TT6CaUel9uFGp6P+gI37C+j zUU7OtRmSShw_=d-OoC9CG)&TjQ}6*4mzTkUBr~>WB29x>Ri_KFraukkW@{J6QnjXcFxtS&svS1VLLBj5ZyKtbzzl6`z zpO8XBLz@-)zYc%6a5Y@L3%$-cks*-8N_|^oNv3F((!vB@J)DxA+GKt*3i^}Ds2h3< zRX2CU*ivbS!@6LF8V}J2+ucYE^9h){>S+|zSK#vbvGJmV575$Cu_n*qd-qy_%3a6M zS~H4mKs2bq^F@`;Syc_ohk(#OIfmP*5{>T*U_`Du)GsFNY|QLg*cVU zRdqZMmw~R$fYGqO7f!Lpn02SS&7_WOk&> zRKu^VlqzdvS~$d)eS|q>b7F;b4zoc_rl?qf$YnQ~3697Lzz`xq1vE%9n+PQ<} zJc9KN>HHGXk?c=9qhyh!v+M;*38aFln|5i7ZrS=*t5$^-8hP+t^j%_jJ!+}Hgfq5H z8)@}-bM3^`0erK+)G^&s&8s{gY3hOh#}tFe*=UKjHwwC0o_TT`Y^&BHM|V79GXjxW zrvkL-4>zs*Tkuh{&abdh8`7bT>U*>0-9Upq_&AR0A}ZF4FnP4wI-x4pwiC5=Oudc} zYvXBn2OJ`7;LtWvm0Qqb&w+Dn24)VGW~LgJ)>!2Fbl%t{A=IHjo}Z^!xJesR4R!V3 zqb`VWJ;{pI*DF=ctCtgmImLWit@&<6ue9P;A#Jou+E^957@=r@$#(n|i#-gLB!=cG z3cxwKXkuQEU_(`F2+bpS3Khot;Kle@F|+RbT6@8=~=n_7XDX<-BSu%458&uA8?%QQAm zvsmo*OIB)W(%gcY*Uh55MIJ|c$8@#LD#c+~XG2O!I9`d%PVEfO(q485^OxpiHAitf zJam~_iD;(-Dq}6mN+q}xznc6Zd=2el`b+>lCQ&$>(s2wu3~1`I=IXAjsqQ`SNV{p9 zjrlopM$lgkMSS`z^mmjtJBrs^mQqkOLo0>3O3wA&8v0kMspg$>(X0jujC!<(f9Y?t ze{OuopDFkaIEbD_i2C~HhK~i&emX$qGk}NAz?o+rf(L~jq?8nL$~TempV$T2p@Jtc z{+IqA!rn8=vg|4o+}C?=-hJM`7v4v_@bMxeG9$gutSs+TPUR>`AlF5yN@@uSc>xj# zNeGY-2szNu>Skt*8ew{%8yT9_vzQ;l2#{bFAbM60FwLyR?0xRDky*@o88#y`?mhRM zy}#|gkFg|D*X%9)Z))B0^6a_U`NjDQXI3R1?!Vk>T+C~VcF*~xxwC7hwE@3wzj;_0 z%CmIZBlf)0Ycw3}GKDd{_It?t$2!AWPckeOMC0C z&mxdKt>tKSo9mIU|X_%S9UgtxGB(#9pEeIDFxH_Irt^Cz7L_jd4X(ZB#TeNJ^1un zl3Wd~{Bzl|A*lts_-RlgIn>c8YL+6~HoT2SMOo{x{S17tdI*QVfH+oD1ko|U`$z*C z6RI5gX5an)HLuws6RpY`5xnOd~iNS*8dCK#E(n3e0Ra``!0r(B4xCI61t*l;NB^4)chs2s}?olY4M9D9{#heEOD0PJbz)2)l2uD zq&ZuBeXKWoOCBj?{n%p)lH}(-%L&NW%R1TMG{_|U~#dPsVneKPkZ>y(PHpLX4Bhhc-Rs3Dyoqkh~J#T!N zC1&pa^Z1)TkJw?8aoyvu@k+MhHbZ+!mAgXmyNK-($A0T8@RgH3qo>8^u3L#AyLGRe z*VOz9URi%)i>_=9rn>F^mCZG#Dy>3{wx3X|G)fvCe^bDT_-F8EG?)CH-+XyM&uFHj zE1GPYuA#|NQWQB}6T!|nnxvdF6<0NW1boxEtceaM$U?8eN_LtwGf2D1skV!G0nf4C zYSbFI<`sEZNn%KUG#Wp)?jD?y(R|834nJYh2%zOr+IS%`s4s%8|9K#l$WI2~`hv?& zcJ}S?G?A}L!qoiLK4aJst4AI&e(3*`&E2k|XC^MoY2DmAv$ZkVw(Iit+WM|n>G$aL z1YL$AF#nA!hU!A)RV2 zFDS~dLyk!_JnweTAAJ|+qyS-N;k7G?vM%|qA?D)V)6fmWXKanb(W!q68*-UR(H)GdNBN$a!=)w9z6K`E0=t80|0hJ*l27 zaC~q=;w9Y&#{-%S{1$r@FM#*{px1JAi>QwkONyDW$^#=>9ZfXQCs_8%*ZldN=}9rG zX7^@V6TKdgRM#r)j~dOzj20$eZAX#udZVbeGl?mBAWW%RtSA8cXXT}HB>tL@sze~w z^plSac9})N8#M5#TABo3-Z#RysA+y&>hNISoR#Gyo6O(hD67-q7C1qNW8sTuR3c&qf}|xlG1U3_cwiIjQK+uaDR*vUX7ff!ER(+8l4-^ z;aSH{M8%$6w3C;kEwk$z+p@Z&z76jpeL|9^5b&UCU>$CQD> zuJtf1uP(G@y)nP-Goa(+ITHjlbh2Ptu^>GVLK)YERVDuh@9qyFL66bFpWG(|vz};% zw7s&lzgL?m6?#?ZD3Hx9wVcgY+s%4YA7yB3kN5B&hclhE@e!=Jhv73m{3u#B!puHQ z@I^(sTW5Q_IS7L2@GwF{DHF6AOBZ5r#v4I4ZLPvM>gm0cmc<5Jfs?S*otNgfggxPa zn8J3neeGnvy11b;wy@o~8Qa}n?VdQ?tdt6+^;(iM@4y|Gc?8=2IoDYp}srGMMkak$Ay>IiPEAuj(FaI2;)E4C`KqY|Jw%>kt{c|jy5`uczj zDt*0G5;$QZ)Y?DlR~11mR=(sg-Uo-qx@nbUTjC=gf9ihd8@Rj3a0lq3UjqL}afx<& zOSOj|pQWQ|>z1!+j=-6Q#kxFFU6(RGw)1#4ufu)Be|C|(%1McES-8G`erHR$1edax zL^a~COjK%9wfk0<7MAW>x^`iAeY0yH*c4?lX zeIL9VHyY$MQk~;f@$JvBPOHX z8GM_5A^&{}?S~e{S-CvD>V47dN9fziQz9IYDb{Q7QYH&1i6+3tz$-J*G%yKtoCc9@zRnYQg= z(mwlKe_;oNBJ{7X!!&iwdy1nqB~qV3=PFLahu~d!cMro8X#RYfx`G17RhyDzV7T*} zE3oorSi#5gFne9rb<@xnP1!MdO}+mHNR}+fYJ|W6F&4fo32wZV+N3dz`gp+9;RAR; zRZZ0h2;)9NW8oeA^8x%##N4-STdHlN^9xn#`4k8IE*8^2Y?BYr46RX8GR44ad6`;J zYt^)>**4X>+6-+sF60uLz|0KWA;*Dk8pkqA&w@%(5%`)w&1vux!X%pYkTs{@jiK}1 z_|T}OIOLF`)4|pV=@USBdO=_F-LR?xCK}&3%_)iCSOJd1h*2iq9=_ zBa(@ujMF1^+NMigBHUR2pHR(hLU1uXZS^2cQ*_cKJ|nWiw+k zE1^~j?_ZI$AQOQkm6EKa@m!oQ*L8${y0&$Kc%*}aiIOPt+AT>Bwddd)a8Au6i(oDm z>jT?LS^kNh_Aq&$Mz^YP9d3)|`i0enjpZG+G{w*JYb)}$ytuT!sk};E{0?z?IzdN| zW0YT>8XN+6A5h|q+nGwDFrZ4?Y!VXZ*vn9;x7Q`%)WpPVUWh7{I^qX{5bx`yn~T(| zYJl+qm7q17D2J%ZQEkeCBJtoAeN)7}<9X*GaqLTYFm0OlMI^>1euwiNQ7^N&&)9Oq z&Cp->w?*CT1m4Eh>eTMBzV|Uy_%Ge_z-Q3CyP{OjL{#nl9DEfag!U!)Dj0$wueGva zkmoT*SJPy82ouJ}CU4g0ePF89aUN3TET}+t~x!!bODed&` zQ8aiY>DCMo!|#Ln5RYJHF$p7+#!wt-)u*DWUe7u82u|_%AePUFK?uR?cLY`BOB23J zGB{Fs0;b;ew{Tr6W}K?`VaAwEwX;9QD_f(U)Aqa>^2ce4kE`Cu!@T^;$-6W|R6O0yxN%LAH>YCLdKQk}BZ_Z6ctQ&U$IEyHQIWW0R%ZRq z!G%QA-9u2wuqkF}nh&XJrY)K}_oLxu)Z41FfPz!dS7(CdV;atT!KMRv!`` ztX1yUiwo=Lu8r>_N)*O}hT3>Ikw*;7cf94%8@}z9d)+6VEM=XL)L`}~DuSkKQ{#l` zVcZU@s=0#C-GS#Nxn9KBi&wUwc1u()G*%I>n3icG+?K0>FGwQ(2c`3pe4^aJvt<&| zit;$R*f(fubyietl3}Z!t52{VYlS8bvU3lC>%t>wyPteem9uRbF^US_eo4{dOig$xvW*nxfl_Gvm@sC)hZgVNfTwtL zy8LV5Df>ZPy34okaUPZ58bW3o5+cC84`Zj+#q%@%++BU1^M7DC((W2C(TXzewCay2 zTn&amM;ZrO?eVSxT>h4(=j~9IJH6?~yWt*=e;S?)Zs@{|z`q;Mr}U8V6g(?%U0%?i zk;TTWSxe}fQ zBWw6U3H@bO)I}~gUC-GLFF1b9P3*w$PyYP_oBW$hXEvc+v|R-#TbknRxR=uh_l@!HK$TslrhJ^+}R) zo}*s%a}-C@7#&Tm9&JNmLQoZ<*=Gy=Yc|>6WY3As_P5EiPl|kNLat`>)8a92K6C=Z z1sJNtFejKWxOZsUnL24GEeuUb<1uM`owVKU4d-0OX^&SnmB!5n@=>xNDK?OJubs!NjC~i z37+lU-^RV`b}hT83%mim#q?b}$9APMlxU}>R)kAue*u0AT!>mV^EtLy1|UdF@71`PPJ;)djIRS(r?(K)T}<^qW9N zBAT>u4PFP!Z-$Q|@S@A%Yq8Aw}pui|d_H*hz(=YWJ` z@4_ID4~XBi5kNX4QzapeYMCfR6*idb<&Ty@A(DJG3Nak6LUl{jd{ve$Q$a+o2rAhN zdRE>){}uclKC42Shn+7?&mt5M#HqRR95Gxl9)mm2j4N{c_%H(A>fLSf$fvu;xzoTW z_s1;Rs0t)LE7!k(I}5>UOw)PlqfFB(;#)7kR#XXUVL9f33#^FBR0c**Ir!K-kOHxc zh*joc0nc2a$GSn*sES*TahxsH=N9mGS*e?sl7;Hb9OcEc zyr@RgmD=p`;`DrNUVF7{kUX9Z8oNXM2R8f#Tm+sXz5WQlST5Hv01>Z=+IeXQo}fBR zEXyK|Tz>_R#OGm)5~p(kJ{{X|X?owah{v$J4Eq_y@K(>yfozMsXgeIcr$W+W$##3n zT1KyDaH88ZIEgEBHBQl#V!=vsLPM+Qa;Yw6Ogqb0GkH5dsw_4_Wi`ByUZQ0lehy=^ zpTZ~b3U2b(N^(pdRiH=mHTd)^@X9OjCMbRmPjMFS*#i3)=D;y&dQqk8#6a{h+(U2a zSuLfv_Eof6c8 zNm%$wemSq_WMx8%5aJ1$20Xy}HXj$o4E6Ke4^IUw=}UUPz*sUlej>>!TD{AQN~BH+ zuVoG1Ghmw-7|9~cvR?jlcAU<{I@OrE&e*9Chg~;uKZd>?y56>9Ml?^+NUTSPdaDDrFYj{YsIW^;NjX9i_0$xjlMacKjVi9U{) ze5gp_BZL=99%;}xje8b=crqo+=h6|Xu4B5E6`HQCJAI~^zeihDLk~!ONGGXsxji$t zFxA*?v?f|zd24oYHa^kZ*_)K^%sX_9c}N-b7PT|&F`=cwB!Nx2k==IeIx)2`%x0MI z&U28@*S6`e&ex_U;OFt4-U=T>a~2X;9-bevVGrTac`IlSh1uL9MYL#reaq%JRm@h= z0*C^5c?~mYSjBRFCC< z628F36S8m>Zq%BOj#(=%2ub#wUP>6{*`%(Pa-$W<+uc34rj;KlM=N(92)I1Zi z{4TvH%O1FGL^`~PcTD2eka-KG?d3sUEW=Rq9Qybj>SoL!a&Bg;h;xHUc>*TFN|LM1 zwkG0ABi9%(w~uv7kJ*voJ@xq%M{5ZSAgY$4pP7r~){)9dx+&pBz6p=j>>Q> ztGGMg1ly(NyIOA-mBpe_{e-Aa=Jm!*@iLxK-p}Lnp3IO*l=v%Xd8}Wa2F(*38>Xqb z;?YT%JSavGgo+mVjvw(NMm=uK@W-FAJVuq>?Rn~yGD^#I@dj?4g0Ox9>2XtAP=TNphZ%Y)YMNcP(KTvNcZncS zbqUKGHhK{#6nJ6kogT}j(#z}bjVDMQaviqc(F8%GmFv;}EBiad+8 zVy#mn!L^X3;z6{gZ*FU!jtda>$PS35nXz_v0x1Ea^jJ)l;*UDO!gZlbR_C7)ol zDFMvOFJU|^3Ou8v6CS{CBbuQEV=eSK&QS1YjB7m(*Eh4QH3plcbnqf%!%6C87e(1p zL_o*rL*@xeqfV?TypT2D4v#q53RD)DZ;}UdQdLB0J81VV5`$znuSm`M?ELKXWP4|E zV@q*LQ?*uYeS2wJI>z=BcSh-v*@j4sUuVdWK7KleG^6882DTv8E7aaTZ?1GiXF8Qr zlLc}Ja=w$uMGF_AOWZ{4Vf{OJQjju|@F@HIx77e6Y|r^Gsv4*+FIo55wxxMIx1g(< z)q(j;&2A=H&21_>1h&$waoN9b*)6;^e{xAnDcZ| ziA%O=DAZBRI?unyG%9nfIzSy2^OX`Gc+^utF!^3#OCEZmpl#`KFZtQOPAd%xa12;-0BjD3McawaS_|QJ^`2aD7sB zGWtt>0*e>hL^f{`lk%Hyc`Yqd>T$%9`cuN{16yngK$sV##oBDCAnC9mi*r-48A+Ot7sEaqa+7&neoiIGc%_atg93qs2oAudo_MuY6=c=02nw@_?y0rh97ZSx% zd{tmO{QC^kmj_+Or`VZ}jdf$&=&lKy!mcp6(yZf|-iq~Aag`=H_BJ{@eU_v_t93fX zvDO|w+3^QCdh7lEgP#6X0A)c4iJ6!$>Vd#Z2?_s|HmLHxkY8Xj3DkteHw?6hY7%qs zN!BfrzV=+l^Xz38UJs7VITn01M?pLhd75)DljS5;@N3rtMw%SJ+sZ^h$E2@cETh{y zk8dq}yHw38mF!SomGlG`m~zQw)-ShEHai=gRk4yQ)O26Cy>>9n!;d}mVKANcoAe|? z2ai89`0N2rl(?BV)b)E|<}qHmXY!`x22NnB5xme-a_6cN#oD~>V!-+}wGgq!SlPC1 z*%f)0mu>l8+15o{@fGpMDB>%XBuj^2BbJR|1_6$1`i6SF&;PYrVxhQ6O}E}cJx-!G zqK&^_w(N6GRl}og6rHCPM@L}fsp4>|_M5nSY&>nsS91ye`vHBBQCBt&qO*fH+tzG&2?B%xNO-^|`rk;)xd>wwbmUj87tbX%t_B*K> zseILU$A4=NKX>}Jv#9CSFwA~2Xi0*bOurL<)0=qpm2H22PnMRJm2E#aBMU}$dFmtZ zk$WXKx@uidXJh|@>2igvF^2KBL?*(7M6j|MCnKcKy*3TA{x(c z#*U{)0za=>67Se%uOsU3@fuxLZKtkVZC^7E>f7b@{EDo1PnfMmVSLJ}&+oIAZq*cO zXOuA>#6g>@(oRQ!=!jA?C!4zZ4~V}&k_()0{gjwl!r=Hie-kay6Tl}m>NwLiG+O29 z76(WC8QA2v!LRc^28%^XswYXjavved(oU(|CZJKnGnAN`>+6<7G@-!zg5relZ9wCJ9{Vrl=>Bi?C}~{Y>edSwlhYg@ zF>l*?NjEE6J)f!aRbt#OTShUhHdAt;&$KmK;vY`*9U(5f$(Ow%55I$vE%nn;lMO+{ z*CRmu6e8!>lPY}?j%Muol}BJ7jebfmS4^eq|1+AG4^Y=rZ=&xX*;;R~F3?7|Iyv3k z-`&_-x3n$I*imXu((q1nR%sSx+tfGJ+xFYPOa?eQ;52j!5y_*nXYn85Yam!KubRH8 zIWwZHiUBXF;xb-f>ay4P2{LaDZjJ;2SKu>nKOvPuI`fxVcmPjF{nn8Owxu{WT6M$3 zXiza_MiTC_VtO;jH0%oarCfLf(Zfp;kjW7s`_F$4e+l0~J5a&@2lraY34S4$#Fdi7 z2gN8G3H=%FVKK8$cQycc$1BPM8Hh9ikcB@a;SOR?#h}xWF$5>!wh*yq$VBLW4&zn6 zn#<%(JF82oYbkaV8=)y%svT;j!YAM@)nO?TE%+1g<6osHo}Oa~7Cz3)67Pq;Z)@oA z72A}J$X6UqbSVo?YQXwG|2MQH{}aZFh|wg2^qNIo#nm3kx-jLSZ{hMj4Sx-@Q+a<>fP{R)qF3LQTiqV!yU*pNJDceK^LxyXW^14 zd>Z2%LqRX8ND!H_;tC`Y!AOQ(njLll@q>#8#%1r7BotJUE7N{4t@Irpa@Mo?4Wgaw zI)*~>eS&Q2syocwrb+7C@`Pt*ICMlGhA-m3{v|{^&y@rLLsy081$n~bCf*Enc4S`* zdwG1ES=XVohVRsvNJ^^d`A4Z{Z?*2`i%q*MQf|CaaSG!23a8|i?J*v{H1^Wrhg3PD zz69-Acm@}Xt2!<(cxJ${jdzHT_ip%j2(BZMBrM4;G~iabFR>DIfpi3)gty|ixgNvG zAE8agt7=b%)q2x+iCxkRT24(o#v|5O4H(jDt_`n^1}D?dLIG zwH*eHNdaJ;F=+?jMfzMJodv-IA>w!P30_hIt`>;-Qcxvg0NZ3bu#dofK@?4`!a#fT zI~bb~3*Ah2GRMV=5lquIK8o>4EpkWh?cQE zjxO!+rWjY>o zuYgW{n&a1COV6BRL#vQyMR%jOK%^t9=>qPGnlTMa0yl#}8NhWCC&L>nVaQ6nIv%>& zLNwR!z;(fp1aK$?;t*p2#l{OcqPHhu2)l%#1ws`#OU*6cRho^A=r1|yl;Q6PBX{kHE%Y_9gPI8Jbbhk_Eo3qo}9_)$vj2!Sgovpd8 zsjc!>r#QDUwNc)%hL*KI#yhr$#;ZGQuy>4Ri&AJy@UwVK@#L0N;@c(>rIUDS5ApVH z!DFySB1jk(+p@{#kc`mrX&z*q<>+hh2Dm`9$1zN1F?hAchu)KmQ^S$@4n1goiy-lYt*+ zuWgw1jE)&BLva;d?6e|_8qb-OPgOKl4An^IHN$$Th#kg{D6K-K(}C4*IfD$WLrG_P zZQxgVe0#tH(*(;Pcf{DO{sHlqgAX-NUPf~ygGcwRl7AMD_5OD{{f-j0ZD}B zlz@2K%|%}FQWQHuj;WhH!Ndn|Qr7&Il`om4d?{~4HM9;Y!`SK6b)lP$ieAR5SMwqv z(_G?*wGba*MLE_Pcr2IAC{i;lpHq)nap>pa2Fp}Gh_49Ow}lq#z;Q@-P*bbRA_?JO zTsO;y`eRu>=4oyx|LzH5_Rf|M zZ}+VTO^n(_(@R{lF`VPK6|KYX;jtzLmrF!^81RDTd?bJ*Qg|sO{(&8qi$r``Rn+1N z^)t~cq>c#{(Nox0=c6m0K}$;l0AWJ!-lxFe2LxbA17M5<6Se>3VU4TEWG+-@d5~kWo{rV*^ObJ1f4|~g`6xfSK z{|F!IHX*=5Eh&Zm)`D0qXNiT=j_O`!eRFYB9jy~IdGh-yujmW!P|O*Qkk)GFWxKv!!&cRiUK2P^B|6TBJdRd zkhTDQ76kE!iK{LfMT^%#QW5#kU!S}Qy$Q;HU~{fj#?6ChIg~jsP=X4^Z{kp4VWgVX z?u|?KE8_iUoc@5z7+4;o)OE~>B7ZAoDRSU-i5CXZ2K)>sAZ)jtx?%^0QLwT}X8sI@ zF@`AM1C=QqyjsXx9_;VS2OHqEC1WioO|7YoZE=2y+2`-}u6c&_`JN?!f)hN-!q8|Q868g(7b}9 zVE`vficF8_2HH#BaPSG8TFm9%Hd@MPLF+6;mhmuk_V>@?@8P$*gi)s^c4szcghH6j z?6s%cTRYNi9qav+^a$5#n$BhS%P4UB`?$A%|F6B)M#;)hl`_$N!!;~R=)meKV%L== zkloDs!Uo1E#FmAowMt{ACk_OS^pclYF(t$(E^z}vmIU9<)RaPy#dX=Fv$z5Ne1U44 zf~?Fv2lv6txU&@uIwX&}rl{X=^AQUbvv`|gwsfMQY7 zV5SjRw?_|Pnp zY_bMDO%fvt$_X@mmu23U&&bF8@3@DF;7`y#P;+Ap7sUfb+ZA_V4=?5de388GX?)b= zoG;WhwMb>H)9l5%O6k1*9{xL;Mj##u0ZrcO;`D4Ht;jlvOLGfTwMJIHt=`a&FO|-C z&n|TxbOukdb5Khw4P5kMjJANC#fH; z7p?XiWtpILJa#XsT#wQt+#MPC>DgBm-o+s^F`coa@K}_Y%B- z52K91^p!bDwlWrcEI_2ExzhWnuauOb0?-3l*e(XZS%E`$m$4A_H>%rVut^;?2a=!~ zn(T4t98O*dbyHAm5>aK`6W@hzvuqyTU8NMdyb}e8nbPd*h!{YxYpvary&`n9fB~xG zC?%%P{)BlCJdOeN{mSE<$j$mM(>SeekZb1iN~lXzl*Vhhh@i7rpG3d^xA3>LXH15{ zzlm$ShnooZtKX1>y)9yX_jt9vzS-Q}9@T{&@OTfEfCfDN?R3DoS9BY@@IkHPK_E$z z*Fx7@B1hY?6LhS$!gHFdixfmrCE>c*;dy9jAqcu3Gl9Z&Wf^0*^N6r$)TLloZ}~t} zVu<#N#QWMhgj>9%k#dD5kzNhws^*o-BrHTkY{2R@q2l@q{vVV0ug>$7+A0;SBJ?7% z@8eGM7=;35hCRbF1Wggqwu!FIgE(&6de*E5VpG3d!n5(4458kSNATa{tK`I4&nU@H z2U#Z{%=qjPke1*_fs#Qp_gF#Mw2Lxivt~XXbc_BD2|2QE<26MnE*Vqe>awb%9}al|}k2Bh#~?C=h} z10fjkuZ1wlPdZJ3CdVoDF|P?*6VB8;L9@lGU0o5?`HEfsG9As+_(t9}UW#dr>5PgPK=IXCwOx?b{lm)R`ci}?D& z+eB3oiVGN?zX*?6rzQ0JW~1-N|2xxcDX|V5j36ebF#_yoX9G_f&(Ds?4UcqPj$E$N zzu^1e1GqWQ!sqcIoe-28?Wf=s2r@<->*jo1cpI_pS1X?7ut9zNl}a=AitDM?E?xP0 zLDLHJ)XX0Hl4;0Z!3=h2Sg*9A$HBjX*9fwCLAHxEvpq7wOdP!F6!RY9&|0^=z^!n? zlCU;cvnpCa!9Bc8th}r9<)UsXBfM>c;`wOCjj8aAv;b!Myi~`Clplka;eGIND6*}^ zS0n_^h@DK0WMn8W__IKPe0(gzEPk)yGJN4exs;}a z^fdogq;Eg|TE{lD`ivo=Eqvw62y&YE*5L$Q%r*7KuiB2G(>!66Giyy#H z9iyg|T4QOqd+yB9$x9bxRadXom(L%bK79B=d1OW>Gnj``#t3=a16{X1sX=cxWBBk# zQLfz@I&22L@MySh%Sz9cN@|QV5$wfyj&uYH&3Ou2D+?6ei5Kjg?BRvd&3&de%s$xI^f963LXlN&!~XqwAIpnS~@y~KJt%Gr{ZF+8qD zy@is>bAI@*;F*cVDA9A#?xafEqoM5zqM&$cjzh?iA$H;XPQ9czbN6O4wx~r0aYq01 zU$G30gC2hx?~&<_MNIESNZh{O%bnC-*dDm8L#;h_m+YgC&ZR#c{CjW@_%(urd^{6S zv4UYnsr)uWv2+_*WHZ%(*a{5W4ciE{!3hC2iB>B*)o}ux#PwVihQvu zCzZIWl`}psWeh!W*qwbJ{)x?~9iVBt2@f&#^(C$|ug~f&y(}K*oM$FsZ4_ktCS5V7PGOc5(pJP;jXk=bxxAYC}_>TEeO$hYt zf+XK^9)#BrD(9bshjN!Ft#MbPxOuJBTd7H0k~HoRr1Hy@Bz4t*mg{0-xt9xG*Fs>1q}=J-S8=2BmXLMKtg&J(tsR zc~AFb!SMV6tI>Cuiunf1ZL*Ht*=h8vc13q{Nt~1lekhOSHphE)`|)o-I_ZzE58Tib zsut=2ZgipE%L;Db>kxs#1!lGow@Vd{Q+1Z7kWKqY(Nx~W?>;OE?L|>kY_&&`x4~j# zpJhYoD_(GP;&1pW`~Vgd(^=tRGUi3KhF*o9H`zSmldmvtAF_Oi83NDXG)ssvGDG~S zi(?Z&%vmx{udyDG@jjmRv5!r7+A$;3T85v)$nc>G$=N{I$_PBK88oi{lB(NUV0_*6 zT|x2H=WW}i{OP{o1fr&im+yusRD2iiKEkN~3s9PeDUYUrEK?9nFTzgde;64@75aV6Vk-cS;}p-xG8RqvnH$=Ya#HIc`y-5PB&w_ zJu4s2K9241aV5|J<4JERj<@x+a2fW9;W8qy^f0`s%2dhY`A~qg5r*0_%a8WP*y;5} zPqlsf_f^5w9m5m&*9)SlII3ql+b@A)h&(Up5)Ut3=PFI8(Z04Y26Bk(%@7mrQ-~%S z_;v>0o>#4gQd4v>vNbanc}0*NMOOpcOBp%-iuJBV7^`eDCXRd1l0Htuc3o?=bLudk z70j>{-!^34F~$c`_706%)K0^Nku2*1WtY)9I}zt zk1cZ1cofZ?;v~*5;GOef;s&>T zNB8aD%K5fee8xe%7QchzWT8^W#S?Y#E6iSk{beXl0>U`pJj28YnzPL;_g=C=)hRi0 zpM*=YA$9OFDAsh=VJpwUq3XGsUW^Wjn!zvXnHQ9NUTPf{UDGa|_FY4dhiXFK0g4b) z7L?O4*PSLJnaWh5mBfCkBkz)qypau}{A9aWZq}>y&Fx)}U)P4cB=We{ z{oa{jg#K>e$|5Ny3Pyxm=~0B62faKKO`4yFU&f6=1&?ziOvPNaV4JxfdCb>|QUMLH zSa>tsTYy3N(>7r04~7&y78@yGGlAgGQ`+*2Xqpy9-Pp(fu!?U}l-Q`3gmTJg`huI<~1=0!KQ{q+G-2olYH6-xY`ie_3~~pcbetq5t}~ znucgd`?_bjp8Xu+O(WQUC%p4skF_IskW);{R1KmhIIPCx{|fbAhtec8$@sar^JM}L zj+SsQ=OA~nrz!UdNr=jFKFoVsCbm2bBBK6U(is= zVMA&b$*XNAE5NUKYse-5mo^K^ao|_5-qCg3gxqWDX8An!&>+ADyAur}#I5%zP;%IRf zGPtucxW6~*#R+u{ohD}~2+Bo1F|*&WEyLA&vmXD%YWs9r58)zJp*L1LTRc@;H&@oy zM>4qaXf}(EU8?A_*7F|nNUfmV=U1lHVsz!hji|#rdNS)b;5x?{vFpb9VoF+A<8m&d&ofSvBFbCo& z@*Td%@^Wp~(i8Er%v)5Ko0wG^MCG#2o#aJD%qS|r&Wt=i3y(0K(P?3ynr>zbcyReJ z3TGeVRbLiba1G`goT%_gHoJ5Mccmgw9sd=4-Fym4Y-$};L?(aFavylQv=II2ZM?H1 zBNF}d=D<&|iqQ^mZ8ygWW-c$l+=B?QxTu5i) zzlX?}II31DP7Z3M{5-KLHtWqc{dRGj^)>}O%D}pn; zJ2@3NH4N$S^I`@gJT6;$4sxNNz1L?e>BEX3*s^kzr!fITpf*W!m%_l9(hVEX(-ul| zZo6=uBERX~)@H3j^Y&*BP97+A1Y$OvIxXID>V5OGpEi+<;qq+X$FP zyQX_*jG4}$BM9N~z9aaRD^4c&(|uh5d|S*pHrXpL z@IyhBcqhoLAg*m|KFpEMJHi!~0^sB`g|wtjo6YX(F(Kx)d6>wAy~(P7!YH85W;j;S zMk1spz3$i>_I0n5iKy|8F(yY#C_sT9+AI!}dak1{$(kBQ_;@X|#Z$xwAP&NaRP z(~9_-SpY*oyuWvjEYxXD^FrTw+K5%rHFJ{iq5eHBE9eDfK?sv@!8i-=NnQVv zEfr50XI4K79}Og-vLa9Ays9Z{ZVP=%zhCWJ=$md}{GID&4sL8+?5s|Cjwh`zt!z%) z(|7rk$JIvDkS~=|V42OH987OkWuQcQ+gXzDv>pDs>Esj2XO{hph0ipEA7gS0HIs7T zf}p!v%o30f!4HFzaclBlv5Ffl9MNK%xMWF!2ou7L(J)Ge5;Uxmrs!?pCn-jppPg>X$0Nj1pVsqeqYSJB zM4y6};D2ZQ+td!`IC%Eac3@z_s|xNE&*5DkoyJ}4^>+9yv_1)+e7e6s)&$8^)ssx% zh3L4gZBdE@X{)gUD~EWyztyWr^ld6Bj%_2BM(pO)35hU$_L)Yju{F0aCm-MW%0N`L z*Cxn_xJUZQ#%W-b0mBU)acTmuU}%XKkf=6!6nq|{P4^_wbzIM@QDSbvPr?bJ&12ft z!5MhkGy+~U<@fq*%EpI9bRmLxL>=cn4}!LG593m0^%EZ4+mx>m4Ie*yNMxRW+*gR`q zJAd-V)ua4np79E_huvp=AoXbBq|PC*Rxq}U@Xmj+&whz(J5j8asqdK!?p4K3DzGk_PS}l z4OC;ffa^HXZC2aGdN#8)xi~pHxpiUj)abP>?4|Oo0^Umn%KyLva8G5g2y_FJT|GC`l8a!{^?t#wGwbs ztEXvq+A^&n*%FcGHtUd~)7)o|C(b)kP(nL0(`|3mt7cA({qRI+yY`_*O!C=`W*m6Y zzO_Hn-JBVt!(j^~k52b?`b)GnI^x^gZx3<8$+JSaj5hFyrSl(RiG1oQ*Te83DD)_@ zag*Spux1~0l@@NVqM*O4kJ7Dx>@$i`k6EyIE3-GE1=`zSo%8%a$5 zDT=c!Epor&2co>#I70XF5{2FDT#BkHvY;ZbnILGinTova!7sYZlDqhWNlhyd0ljT9 zfj=!UB-Z1CCy!FV5>19QbW@{Qe-8>_KQ>#YeMQTe<$HFWJd`A^GfSSXB6Ve)gmaW{ zb4nUwhV_#xOA~XA8k?Y5onP!wVTrh{)#JJ<{pd0tAm8JK+GG3&tx*O9ZGhiwi}np9 zj*53=FYtuI9TGhZ<4D!G@N#^XmJ3)^&;R1;Nnd)7d^H zo4JRWcFp&ht`?<5$&b|^>ofXyn4eFhQ*W`OXU(j}PTKdGeQLp4$|B&m2-hvF%}jMq zZEv!bIeD;uFx#G-UtL~Y-sx=Y>Bm=R9MG^3ws)eAi>w~|T4R=`zc8{!eeH>1lIV^s zq6shaOq_;D4$=iW3!lTc4Ypv^>39e~4f_7x-j5#DF@jYAMAVy%w3`Ti`8Zj=ngs$BM4g>B3kXu|ld(Y3+(q zEQp&WuF{2;;dqWNg_QBuO#vf0UCMAba3SYtG5aj&-$wX;h;Zvc>Wf)8oZe|S?>cqu zhQx?q@0+MKrmo&Rx~UvvGaO^p8ey9|7P1d#4u|{dIG0|g$4GZCv@|(V)JLFb#zq+F zmZbaT;)X1B62*!PLw}y>9Hc@IWQ^CrPr!vtTtiTy4JNb*NjLHDPT;x3H9h2nQ-0GbYYHC4Y2_sF2NN@dqNO|4SbFAa ze7ki<_-Y2|ELKvISp3^(Fedoln0Qi=u?KQR;1a_k`)(2!iQ@>Ki-+*NKx*Z0RPD0s z?ThjmnT9w8&dMg%Ti_bJ;{XnfzNXibYG~>iBAbwCCY4ed3~qPcbQfaElW=NL(gMBc zX`0=z(6Ks#;yB9J>0I_^A^smWrE;3__}oht>T=UQGk1FKaBgbj^upoHl-j5{-bL?< zXPv$Gl75RUl{U`vp>*yfnwSEEJ7duiOV6ezI?Sc6rF{gQ2PlhCcOzLo3;Ag@CFlGN z0fV6}&pe_s=qa7ZPq>={fw0u^SteOG49zr>v@P%@_{s#X$mAnrdT@o>d#2NUky3ND zmSF1O)zAQ8xNeameb8=Z2y^;WnL41U$60&@utHh|43$z&QX@}bb+t&{$YY$Z!{#Q1 z5W?g#@ecR!D4o_4u05%qu639Ff^8}gd#bv{Gh(MU7q>DnA;HegFMwi!p0NBx2Oo{CSezK6fI!o!CVDWG9bb)OdT<7EcA zH#sVW`fMFDv`kF%g3B2?9uUEx=*Y1NMVx_W7}Huo7bb4r#B&pA>>tsL?KOtMuHk1V zUmoV!M48mKc>Z*rvF)%3*xY0LbE8p(>ibC%Q0#GNeg}VhS^-_^&Qs;n%UFT2XBFoyP-9ujPkvZ4=fL`7mS#_&jreA4jc;9PKjs zkj>kYO1-sBpp@e_mvJ%MWHg6;IS!O@ozX)^+o1 zMA>(VgQ({r$Jb%NsdWsA4idKH7~+(zjRn(RhO~k}1JX2w@*FyJDu)v;BGmK-cJ2uF zAST5vlQ@(D5)|AqXklejC_(WD6!zhphlj|6wXe|UKR^tWemx`h?bn=YsOlwR@Z=DS z(3+gW9V18DYQMMo9~rVdfxho?c$WHE=1)v)H)_JogNp|z57?CBGmqW6=N6qD{kFOK zF}Bs?&TkmY9XI56pkz3fS4bIJufeRML?Tg6LfQ0HH!;nrOnz6C*OFyJdxp6}n!O=b z37~Yp;*z~Lbc^gg@6dWI>!v5Dx?nYJBR~YsFTp3^h-@I!zEHaG!%^%T>)MtFG{mP1 zJYjR64yqfd9-?*4h%M)4|Ku&xKo~j>!W@#G=KoW-u-CRXyHLFSse3oJ!Go$rHLvzun!Hf|As-DI$J^UNF~LQjz#M`Tc< zu#odCKjWy1pRuD6OxtD_{|eloE|)fUj=RKNMao6!DNE$gosJ=|JDlZ_v#Sh-Wsli1vigsJqH&VImBUceI-<#QB-$)`U4&n& zz1Jq<-)Gq-&9WD5(In>6_<&+=Vmn5gzcP3ge@Zt5Rp#sKQWSt53%tzJO5=Jp(e5?b zprH-^=Y3{F&<)TDKgx2PnT87*Wg`Th-*^nI|I=tsR>`KwEEcEQ`W6QGuYo7g+58ZB z&kGNoytTi#vi91w=hpHM=btDpWiyXH^VkoWhhb003$J1QkR z<`@PZO!f@>7H?;cZl#>7Y&Dfv$s{uc@}ZJphJLW!U+NpMs)~Vx=3p^w5>lBDvTkL8 zO$V$47vx-#igl_;9UfAlUIM&jA2eBN_&2aYeeHjScKqAG9*ZgVknQulyslUu>FLDV3sCeQ?5F2Do*u&hf#Htc7e zDT_(tTP&DkzQ`cwjHiNIn`TU>_(n5;WWYwI7ASV>D{o(h)7_H_cZ$2nt*}@Qs&huya3+MMQojj0_DvG`H^F$bJMbhnJ?TRE3w6$F{hrGarmkI96r8!69--;;YR z<7fH1rZHK88#A67Q7=tZ zKqKO2@k&j+-yG4KDLTs}NTx2s4r0t_dQx`p>t|ACWYLyc_Q?gtm!snHSE7&?M83Er zSVoOM1Fk5a%X1V7QG~p2MdbgMtM|Y?=X!%9tm6M&7P&3JbwY{Cf}$zHTyZLkP!Tu$ ziR+se)>aH^%b!;hb$Ygtlf`*G(L0OF8^&>?AYEb4=f6qO1Ul88qfdWW1m*r86D&j%o~onfErx6v+6 z;~iXOaWvz?zRq2~dU)de#nY$aX|HZkbHz>L2HX(O?47){e~x9huFWmn7}b`)&Bx1k zDBKJhJp07b@~aZR5TJONhS~6OKh1Vt>glNhoE6y?Qfa*`TOd&Y_=C_v0OO!DTP=lB zbq4p&$60!HrJtT=pYizL&4tm`q<9U(J`1cb^t0|$x}oV;6Uw@$&%-#9;@WWpwT%; zHc7EbHbXOrqyi<<6eUq26)nlKBwOjpmOYZ?AdMx*XOG80wnvud$)53y{bQE>#!qsZ zWy`kgwcfmW^M2^8+4r`1F}O=t}h7>M1E zd-z&9_jlia_3{(<-TT^RJE|W%zj%Ch+PHS%()A}_bK~)Q9v^Fjjhl_R{?vjD_Jr*^ zqYR3>eVw}%vE@<7>D$T}oq@cLTL1l7S3Ye2#GhP-Q>6Z5keG5D76Z+m0I6&S6ySK2 zOMC;-mdKpR{_`U4&Y}Yf5K01bNMWXf0_|jG>9WuR)aO!0gt>JQx^h+J%@j$ynh420G;?s1MI76M59w z8_A@mQR5W;+T#QWqbd@^Rc9(qp}!NN?)!U$XEcdF?ZZ0y3iNf_mNkbt)(zdaJ+qXB z7e^?26{F9kQ77Oa0_HLiAQJ^{A1$0ItG+DvX>8DCrkH{!22Hp{)oPs)tl!z9B)z_# zk5DN#HZv8NcIgP=$7O+CZ(6SD8iW{Yit5b&5Pi`cEYV>ySA#9O z2A_hD!28f=cn;qnfh16$UzSuT>IQnuCT>+p_FPm+m*MSjj$GPAJqdv4f4;nZsM>uADe^?ZWo4OP1(iDp%W^r!OA4di?15 zlb6qKn0LSz*o)`2KM~Oxj`FVTG+1U3d3#swEM75T9vrQVlQ_-h@VnwH(Oyi{UlFTF z+LX$LWE=A?L)k^o4V|=-TtPSUIe};PNxZZQ9%X{9e;$qRm@kSpK2$9lTSzLauvSl~ z`P{BOwd?N^S+{_&4~e>KqHM{u*g4^7eg z;DfZ5rHQl?Dc?W&-1j~C_#5B!%u{dsph2G7csF1CK>y$i7cO0R-&^1Rx<_C8=EvSJ z;AtLISjBM`l$ygi=-rHR=8Ehg*%0HaqTg=MXis<4iy6!f-_eTFFK1OZp_FJFs;K|M zT+{;jTzhg%c1k4&CFGLu9;8YTNs8x(61)*__XG6`$+6{=>|PNV&Z(};2T=(~B_|+l zvIPMTy%JN(L0Cqqmn)9V`3ImOa;t%GC$mRPWru+ZQ?O#%F6+IF2D$A(K#>O{>_eGi zKm0{}j`)iy%gDu?3IYBF>uo$#Tncguv7P|)W4%CI*};HcF18g-XY_lu(BwKdY}=F- zt{8?B869T_q|Ff!Z3Y-MPRt*Y8~c9IbPF&kn05#&=9>c^zz%h?<-^#O3zpML(!f(k z*z|^u<82Lpg6!nmr<>zP6@I+dPI72)GUPT|;Ny@RubjZ(Vc`wA##qUZa)DvFVRc@_ zZ=qPNkm9&GybhJ|x4~>&Am7&}85oN=hyI3l{=zd1mr zJBMDFzv)1}n7XkxeoCm zPeciqQBxRd%}bbZ6(U|vn)-i(1BwxoJQ*`^lvKn48gH_x3Mz3s#&0@m)lw@|E{BgK zd|B4Zg*cxl0;n`sTcBEwt&MLe2`RQS-GXobbO>&d$at8HPwAe+V6op242L>l&V-#Y zxY)6j)vTBh zw={-zb@3&(uu-m~FM4#p!dPU+;{GM}ZD+7+RA*=Ap5Ug>qydTu` z3a+7k5(O&_>x?ygwy9t%RoUXl4!m=ydf^k`}zQngj!Vaci$=S7W+a*sTU@xb`H z=$s-`*@|yzL8whgFL1|UKA&eRc4RpAMa4E0TX$9Yer8Or(K@4nZR0u{X%bcxuF8dc zea3nhuDD^DRNtXabwy_o@^(|1JCc0)f z2J1O9G7Z;P9YZy^M^#&jA^%RCbzuRUOn zV6y1c)pldyq59Hmst=wzbLwvQ&L}PWh9)!e=Dp5H`l@O|aCR>X_>6&H9m1dC`y-3@ zH2f1i=3FLSx> zN1 z$u$#?MXROS7J)(}HtvY!xML_UQrQxjmm#j%Uky6pnuLl0gc~h4l>MmD`;7rfY|bPC zr>^T7A2940#HMj^Te@f>WnoqgxK5=1#@(W zX{T2zYclKZ!_(XkKfjl4EYoK6?eAgODc1=yx+cSXpG~&mF0?g&AW-YbBa})8;WQ|W z*!EUnjC8ms7}V-CxgF5=ZFDc^Wd(H{rJ*L^3Ewk(=VIQooYG6cE{>CX)I9fq!b2{v z;K6(8USGJ5KCBCzVMV2+i@iD(01dF|9OGggm7Jk+`W}Kc3VjJYLAN#VzsO1ntMW13 zPXrG4x8VQA{his_U4QJh-OceGD4@=0cRAa^+%(}?)yz?%-Fy;$2;}E69Gp+d+mV^= z=}jUwfQOZBNj2mxNdeqUDUX3)U#7JnZ7mAPPnc7`sG^!aor@xB|Zi_eturs6^y^Il;Gvr-7OX zWwqI96DZu>-WB%&cvzDJ_Fk=shwNg4FBSVP&3kCv)d^|6brmko1QP`JTda`>)kQh| zFYr4elJo=?pJYf5b#7UT$4tPNv0dF{+HO8;2tm}vh%G_aAMj4M(tm~jN~igmGjQfU z>W#&%6TNa&Um!%=4#sfx%?g+5Qk!0(sdR#>W9f3M!NzKoI~#qKJtDu`U$V=oGuUn) zPZ!DVdGy6%kx!J`K|*C2vzSWoIU&~5a7-uO7^2Mdz5&0FLYaJeNQ(4wgXV^^8Pepf z^INcW8SgI!Wv&`V7#eh*W8BNX7aU2|(QbVsXS9r7jdyspI9{ACmiy&dvzh8+6P0dz zX0k+h4CzQ$H`~oU4fp|5fn6wiwTE9HHH@;3uS&GJz93ETh&$mpdmb*Jc6l14qZsbt z$5#lm8b7KAD-`kz_rP!0vYaMWeu6$s8efHbXOih6;r*^+$s% zniQO%bVap}QpGEoFP3M^i=3+QZM$phb#<&XH&a=d8g|VNu`@^2=-`UFbO^g?<~a<@#HL&F`F}CbE8>+rxy&tFv{3}ho(AG7m2`}<(Vk2v?PB?UADefi@0F7*M zcD8C5h1oOTfPVww=b>71;lv3r%_Oqcw_*DV%?0hSsQli}Y(;gmG@Pt(+D4nvL`8p$ zkhpem&~{O{_EcPU$v21N;S=yh_&-n@o)F^_(&DlrAFr>aR@G3)Swul$IEHKdinu-X z8hDMMO(}_biQpEHpq)<*t+=2N-$zqEc8+Rt!iF)lU*AUi^>t7L=s=2#Fi!72kP=CW zl+@sk#X>yTIo}nif4lE~B+wnyjCT`dgTvB-Uqz#I5f2Kn-Ccs;f{%!sjeO;s;I)jYV z7D-c2nv|3YWjblLgQ&w%UInh3+#mDD;;C-fGWOuR4HLGv8M>j$*#l3vGm~G(>$rzM z3EzR&(+s739e#nh6iSq%1;w&4nBIa9!pGoK;`VFQnKFUydQkiKlQqLHCH6~MFF*Mu z3fE-$Vahj^3%kb3Dzp+v9>93GjGOt#GGF;7MC~<F)%4rZt!+KTNeh zV$eJ&I5d^Db;CNWuQGXYLpKjCADEdcRdVyWys<~`jo^!9rYr*v9g)ZkI-aE?Iix-c zWsLH637!8xAq>I15vuGA?z21%3x*%)mKB+rHKnl`z6b|psPO7QMr-KkRVW;{z9it-`Fb=o!n#s$n_AzGlxbdCokz;x0^;{m@@q>R0R4z15T9x-7qX@6L#P zj9Q0h;04^(EATp0mI2i9U0z3TLk)@XaR$YHgJOsf-;|`T!ZUD1+`c0E!Rv5}+@65B z6-xRz;6z_TF|#0e-Un{mOk=j8Wt}!Ub=ufuKl~5j5$WSmi8o>M65jjVWs0WfKp=m8 zYk<$($F-qW4T0s;>35c=N=qfJ-mF*0Cd(@m#W{JpySgYU!gq+zhB(Ao;vzQ?lC0s9 zixE1Mmz8h8am62dkSP!LbE_z?%3_p+ZZ39}OX%}G0e=CvK)Qwx(VBb`ju4<5s?JZM z_B=9SLm_lzOIJhRp=i|9o!ClsCkKy-vQ9t*G0z_GZ9szuj(5UF-G-jO2SfhMtRuWH z(3$VfPQhjAF53UA%H2%9c7JUv5)2&K?`R6sWPFMzl5`#qR8Ms77=u2_ucM7F4>Zc=pr>GVb80JpDt|8T`r2{r zjCP^CQeIms9xPe{q4eZ(p>$w%@$}jAC;H?4^&uKc)-y?o*4@&hfJ)pO^zUTthR2Dr zcYILN86wQUKf_C48q{6N6e?4XD|EKNM3*Dq@{_X4>TYby20RE};58Ig`UQB8xJ64w z;C<36pygeLEX52DknX((GUsIf3y^A>PbIDU>!_MP4aZFCfWPLq0u!S_<(E+s zD|mkq3clMUX5a{pqMB|}C!?~1*q`yJp>Nw7n;e@R3mf5dIM2wlnQG4UgBk6%9>a|4 zaW`w08{zF4N|X!dbe5w8;k)1m2wmyjAT?4?kQJB?N9X%Rl(;Z4=L#A=q}z{&xDpEW zu~Q|~Fg19ox;sUTE}M@F@nRH4y{Xrr4w<6q1<^-E75=Y53$^KC)TUIuXm2*k1~-_| zU+$mmpF1{weEiI~Q8`@MGD{~`&Yqe$&33Yd`=)L2YDVU4hrpUySz44T1fq+Ogdxzq z*!6#`MShVH9{-!gm~hhzLGj$YB%?e&0!|Xv*q@s!8ru`p1#g3_Qy)nS?l1U}&4bG; zwLpbz0k(f8d@*CY5QaLeX(jAm%iTn9T4Q6!^ddzjpNL&SwNfteSUCu8}{@Pd{%7p$B$YpqNhb4GP?wY*V2Qts57 zhnkzsqt0?_PPGcPvC>w5ygR-&K6Bvk!R>WxFVEYs*HQ3(?yV_|0O#hixl6v+TiHu- z`+kD!4RIM!^Fo2gvf|J{`n>vSN@@wBpiq>}UPc-I0a5otr85V!FNz+o?1qw;_dP{V zE*Oc6Iz_IM#ZU8vxts9&)cZx}60CWc`;lNU14FfjXNBs1HlBf*^UOx^n*p*_lVzdC z%fF$sIJQ-us%C66kB}~Qba|Cx*sw}?CSsbmQZj9IUzhf14k}Z}dANVTA@USF*gEWN zR|(=_L)W&NH&JbVA{4AUq38fvs07WW*TQR88;;Uk5G^5t*QW@wCUE`=UNU(7e9jNm zef7A`XrFlYCIWaAU-416r*Xtkj#q@HiA62EzOC`?+7Cw}x`^Op&*5_?AA@0C?AEy% zODB|?op(?|p>;Pcsu$jd=yHc@;LtyD(9{x;_ye_L3X`SdG%QlzCeYdAm4}RyDmi}f zU6B~ois0BdHTO;*{}_Da%(%?B+&`HbsUq2SPN2MGojwFzliy4Ztj>)0$D4*#*Cso? zX|v&0)O}LBr!O)cq!q+b-&d;~*|+-$5~s*Hz&&V!REq#Am_|repr%#|s7k@mUn{@| z$W7v!ZK;kO)oIsgy=sDbIn!%@KUpZv8`@l@ zVXGdRqOe&WH~&)1v=QyB;RTLHhyW2-sjL>fFI+JzwPLYcuGlPJpSn!*V$rq9O3wRy zG$4Bz*WJj`97BBzywa_ier34^r+VYNINbkvko~|YE~*+R!HrJ4Y}G`kb+a^PSL$Yq z-FB9C1lqM&QoCc#d1e z-Z?loRfv))e4;!>Cr_jgG%?XgXPgl04hbAp;xJh0wB`K4Zlg7>?3t}*drGl8h#q^> zHA4&wvF3M(*uHbN(K#>aN5*oIA5!*sqLdIG-wV*F)Hcyo_*1kfndXoij!&5`+l?jI zek!S!-%Hr~?YbMZtbd=Ib;7#Ku3KJ+dtf?l+-EoE|KX4#r; z+k{aT5`s@^fBCSU=E{o-ISaS{Gh|YQ>S4N;gbb*UNs)^~Yryru;96?m98+nL3VYn50e&LRo~obx7Bl zA4h3etjdaJ`mz$4fg-nycHSWN^2*|Qh(o6J`l=tm1FL;W`g|aaegIV}5-EnEz7^il ztkG=8*u?8YX5?qtY*m(2ZsN+Zu|#J~QZihy3`-%+kkmXQO1k5XQl+H%Xg_Jv>dr`N z8EV>|Wf|T5o04qHG5Ih{NBGPd`h$Ix;$`J+LXXmJ>yhhFJEs*B2uj{9OR^#7^XbfW z$q03skbhso+m*KmLWu^@(?iECqXz9%DDz)g+L`M7cY>v@i!U)l8Ks6aB?UDt8f#6= zC|#weSIbOC?W)S-@?^h0-PT7kO0UwTGwf#_Hk38t*D&sX3T`xsxLP_wj&R(`siIai0YWtJqo{J7 zAv6>>Q}QH^lqyFB9?*8Nyftxnti@j4t5{xH8S;J)QP0TX9D=D1;8O-5d<1SDp{2Od zTpwH#8n|smUvEyJUH@aIcceN^j7Avsk~2-jlDJb6*dl4xmLEp5I~NSvI9nd7P$?Nd ziGEVSFoV|OJ#bH?vjQn0%cmyPhNfiH8qAO>Z5DV1SlOTw>HsZ@v=Z&%0_|d}F;$tX zI3>yJk9B9d3wkjxtG$VKwLXjfS9yMtcJX`d;J!hDp}YP4 zY1}PQ#*E<^$RP7HVT7XdjXdq7N00zT6pL^$_?*W_S0*UqCFWCc zN53ts=1@A(x{vX3u64D3ElMQTVRS*wdM+ROVpp@g)vy}d-RHRuJL%~D98@QN2EIf! zFjdu7$M1v3@jX8Q_jT7a<#+?$0sjEM3hxrPw{*%OM%WL?sv;XkSfgA6-h8NehLLM_ ze*j(n3p9sF&t~&?|DWK5of2Fg!ba4m{E((G)k>*SBZMAV*Y=_G>{I_EDcT`O%`PWB zAHk9~cTjqC&vP7tIyR|YQZCp_@JV7@sCq83gc;OcwSrO4`*kz#hGtHEMU^63K%4I% zIP|y^IG*z&yrWf-HCt=Xj8U-kSV;`Iaw_2XH;>^{Us2SFCeKfB*-$2mden9~c{-}_ zD#;&PL~)?*5wz=Z&QJzX#pcq=lDhB8?}Wd*ml7=9ZM_)|yzPUHH3i}n`C~G}=-(RP z5!(TcO{LE7O_oZtsy z=Tp`kt~e^=>O0|G6D1Xo@!0H7g-I^a%^=CQc|D0+tdq~#=9fYm<_xqTM8D{)f=*Pv zY_)>6XBYBxO4UKER}nl`&COa=@d{QlJ&R6YgAp!4xoEo-c@dI~z7J*nwkchHXZcy) zDbzj+X96d3J>t3`MR6Q&k@!q?xgQo025C>8n3?1lIU;|2$aVu}3;eQt%HudgZlgS& zh@!GbU1d0}*5Q%m2G(2Q<$BYxYK!?e;5?v=iz=>|>|+g2!UlR-c+K&8d_)b`!p(4Y z9v!BZekeJlM2Be>cWWcdsf=<;4Tie+h{2t%I+vXqkx2Q6Q$Y@N*W)1NQiBnh#*1)R zHw%fKl;WoLx8lqQr^!tm!LCDg6y&^f?HYRh_)(~#@QN;DrrWCO>DE`_d7={bT-S>o z{v$(|sU};I=h!d8E3GS(jcR{Epatym?;H}dU+3xi=NIWT9Njp+v3X?kq(C#N zoIbIBV(aj+6Wjduv$5MSl#Yf2^dx1^_+Dpfq=ZbZXX&!FgT^dZ+;utk*PgZKv1iB&XR1~u!1&?~_9D%o@KF0;YyL3jfbA_F#vZSiI#hC4SIoIX9 z-Y0l~upN7x^aLI<7ylbm4DV%hCF|T(q_Nq5eFJXL)ce^Pn0Wz|d=zGQwBHi_Bpljs zfNYm070X1sHGN=WkzzP8?Y1yCGq*Zni2)rFl)87f$ z6_S2@vkn1g1V_l+LK{{?^HmmvI!;aEUTF(VH>O!gAI(kGK< zkImTv;YJL027}M7p&a~oJO@M)N<0Jg&59Rz+@ndZHjSxsV)er7)bn9nWEnqr_corf zJD&DY%HG%Dl|6^Iw+MI|Vtn``5#Nxkt6pL|G_PRCs*-np+K;>-c+&Iz!1FvYcRpk# z`#88iGCbX5su}u@{XB-=72jyisaBdA`LvKTa(NIPgCya9jMlp;>QRTN*>p2Xyg)-S zUzlqYN`)|X1t!S8*lqV}MyX9lXLMF}(dc%AgUcvrsM<*02w5APf=nqGHu(1jI1UAFy40vMmx zCVCLOT<&+;hEZEmjH%QJaYyfKRyAhZP0p-}r>J_}-(3UB;jN%QLwKjR$Xl72>owcW zQ)kxKGRzq0rT8&6UF1-t87)zLTA5`s7Gwq65tK~Np zzLd^U^!~IMq$RVddX8mU_>CDUxUMUxts^)F8~Hv}@cIdj-*xC=w~!wXt>r_G?>Xj! zuICVd-LelDgmJ3Flzm)gEAX7a)(yjqHHRtp43$Pxb8OEt-wN*@t2=gUR`W%w(8sWi zAVK^T)n>t`7C@))gzx0x3Cr~?EA(7TGYoB1qn;bgJ5O$ouP#5i{JP}>PaSyr!H2G2 zx^gLKiXNa-#|-QBPsFvW=P%y%+Jh_43~MwwzMSqBA$Br&gQlQi^DghQl@a?RmUd^x zChG;`ub$%@;spLETt(Srl#lO7gqj7QYdXe`ssW0e@SFthq=6?TW{%FN%UyGH2@mch z=sDY!nZr_E*Ig#?EdQ;b85VP$3L0ojH>gnPN?L4f5(*P~fj=iYA!aa4@No&40f)qp za%9O$Y~rg)`Poj_P!!EKxUb29rZJDjUqIWos`@EY&4OL-!C5@Ul2n_h4o4)5R9WMW z<-QBvJ&_u2b0KI$dx4aj@Oi%}{F*Y#Kh@5zP(AvbbV*V!C|9<(*4ED)J8^JkxiiwKfOzx;VjeZf+!VL^-^#gqWjIJvr>#?b!+Q~4?vxAl1<4F|M#a(rw`{l?L)@eR9#vFV$Wx*= z{25pR{);eHkoGjB3uvU=fbwu5ju}@LH?-G-U+tfK}qy&NyKWKS| zS@jYst33^W5)~7}RCvL4ia{l#F&GdW4=`$`1(t2-9c2gHJuI$+;ZMxnV$5 zMxq+?&v=r4gTTNxldO|j8SPzu6-LPj{tV^wBHnJoKcTapz+w1eBsBPtCz9Gweo9nZ z{!U~HVq5@>&4_fIcv4UWwGz{oslR{pB<+yd-G&P%>er@1mZ3Q~EX;Dy<(|F2A&K@! zzdjC97zDB`$CAd%L8SaBe476zdcqOhq{z&)cqBEx2KNLNw_3U+I84kk%ZEg#tcre) z|3_f@GvNlhx*g zThewr0QRB2kF=X+mU*X#pYQgLJMiC7>(;7nlh!N_6q^08bRsq><8p2><1j?`XvWoW zkr`IRRX2<%UCp#!@MYTR>uK4ql@>gzaw~dZs&YajJW}N+APymVU{N<_u7GV5Gn((a zXslI>PaPQKbtz*pF8K0_n`vRpD_MrpZu+6!(cMF5Rt~N#ji8y{xr(xrIl2wfEFHb_ z;q1b*y5aLc(Z~xwD7x9!OA}n{(gaH~#rS-IBE}Zl%q`zyoMygCdIdLVT+zls-(dQ* zRpCC-R+Y6kC`zj%``#dpOMdK3(cqeuhwFw?r zWPGBRsufu)tC_F7Xp*JyLJ8{3Vi(NlJuOA>A?E|o9gKmJ`R0i*^ zEN2+@L+2xFcj$D7Vvu?B!}*SZ3-@|CV+8{^E1i|4bMiTbrHNB0Qq9kDOjVUFhAvZD z2npmMnO0e^%G91RPT#&8HNq_ZqB;ZCrZ?&7vKr_zmn8yoFY|5#V>I36`YTpqqna_^ z6)Kf7#(%zU1ZEUifmIAK-pvJsJP`rd7_~?V+B>N!3;u51u>GW>1sZc~)+0@$nHnwX zk~=nOONy;&wn8lP^7QI5fwQ3BnLBBE^do(zkMtX;I1iwvx<)o*X=S3^DL2a(Ha5>O z-}NsaKHtKq?{<{)kw%*CXmWkew3%QUTNyNAu#8Zi{*Nfsip0V{bc@9)O8?B{63+VUZhL)cUP|Yc8Ulr*0i+D=HO%K$zU8%VoWTVp@=D2e^aJ#4schMKK*{m)fy0LfQ zujEiQzXWm}8j@Uql02dNE)7zHUn)soFn?5*KB!0f_mdm7Ni7RR;i>4F>FSQ1%F@jU zE=-_fFM~U;+F%!yeGcg1E|6*KO z6xT@{tm~w8VS1J_CXI?(GYchqm@mm7h12eU?On_xP%3C}k$kk@lcghcavWOL=W!=m zN(HKNHx6+tK(7>~A2&a#yhjiDD>D2XrGCf4oW_0{elfg1a;?z!y#PNS4+W!4hT7I9 zU)Z7)zZAjxa$lATQK* z#|8P0JR-XoO#F^4AwSU~hi|GH^cF!%DpVe#xozs*Vzn>32K|&;xQiXylSY|F;iSgb z;a0uv_S;8^F);KJ>hg6WrWElz?)*9sc;pjypz69}+p;Edd-|ZN-9#6erbGQrCvEOz zdiV12>}2mUZt^`N@3{ea(=g1(ua;nreG+fBS4q8kK0l-JV`Y%Xd{35*D3#!rvE86n zum#i0yGCTF&jbPbQ6C=S?($li=Dku>%`>ZY_*#=-V`!>3_tFRBdaP)MTAB$9M6arz z#_-bih0KvI`UCXMC(s99#Z@{@yE0y>b!E@-CaW0FtNXOjs1n*yGI(8`UHvk+$d$}H z9}&J>`uB6y*1U|$V=d0-O!yGwOX<9PGSueUl?2-KWPC_ADJdyKEyw3Mps@qOs$|WlTg$D468R+=I8GU?S zQ6+RP4}M0`0>caa^MUV5GHJLWD(8O^?Ci7>xDLknpfjh}xxF`ul5Q&lqt&5F#vw?B z$W&+1Nw>3~2J{O4=UCuNUx2Gpsp@x2w&w(HQ|5ua(eYiwG)loX#uIN&9N#jS&SPD5 zSm>zC%R72#sUM33Wq^ltCseGI(s&de27osFp2ON=m(OgzW z=+p$?+MrK<=Soh)PlD74LV?N>#1g!O=j0+@#T0xjwon8OW^ckn@V|wT>!)cHIdN3^ z3;3N11wE=jS8;z9{%GL}Tsblab8pnzL`Ez1Uxyc$4+F=JFNU7$27_5!sv>2)!^hw} zuGf8}U%F$*wq?qtW$eRZFS4x|kv@{W&R{ZQ$U(Y1;+5^f42#&{_H7eiPr`Xk&pG@o z+%4w|-dG9aFRxe#oUL)&Gz>kAFXxGK6wstgBy+khDN6DQt~X@iugO$ncqw*KT^cB{ zap*=-qUt-$0RC{w#cUMW2||Fr+rcUGY|IyWqOoI5n`Bz4h<}gn=ZokeF~R8Le6#!!h8gMpvV6ZDuNe^q@GZ^?-Tt>yC z0ezNC_|yG_rF@jMc!v%4Dtr`h$N|G zUrB7sG_465oDZ!acqK7?8fI8W%_)0^{b{1a!E5J1;s&{&Cxeg>;CtOw8IyAp&}`~@ zfnb^Zu?Vt1hUovRp}Ke&Zv}GH|Rx-IVH!k%MR1>&HFI|R8fP>g}&oN zSw8PCcX)`Yw|^aN*Ezzz8?q2jO0&d5Jgv+dqb$TzGjoC5|6gPw9%0GeJ#Bj&1zJq0 zrrIu(6+0J6@LJs8lLN46s%Kw@U&Ke`*Y@#ebo+(aH%arVrp}F+DXxQY2@69fitfNi z{0&#&D7k`#I1p0qA&uC}b8X~f5-b{zGZZ7++ky8K48WCcq4Z1iQO zNRwBuqq0wgfx9;q@ioC#gM9+m>Opw;Pf&IKXRSY@DCvUvY=o~9XwCtG#?cKdj%0Z%(_OTl@@Ok&rQ30+u{FqDQq5+rj`0h)qy{)~#hp)#N41nkQ_a752`^c0==*G>faDQDv8>XR7#B zuFn4`Pf84b6xy68Mdp<>X1S)3OEfp9%0I6#mzkbRSb3I^lCKG@ykV!_9EL61sGsq) zlJ--&O247;7VCiAWA%x4Q!VC;!GMW99a1mmGV<$z0T!rAdj@^>u``Y?b(%x@$fG!| zVQgZfpGh?y&O7M+z8T+H)t^Qg_=+gwO~7Jg+mZ20%+cgfk^&vTP{k8%t{@dQOTL6h=j{Esi< zqsP>WtTDNePiG&(=qT&yaswA0!zgKHXR_lzqTf$H#XTtJtlY}d+T6n2`e9aCJ+QjC zdT6s^9@t)_extpe>N_MXd)w7VB)+?EGz+a1B{|-!a7pTVslj}7AXubl6xFL++R6&5 zZaW`|5vppKe3X2gNEVF;AVE2xm9eArmP*l4a;D4FOdrqlNfThaeq2^zVCrUZ-9wv2?oCfe!8iQoTbYiLuD$L{fNqrx6tM4sV2&r;>1KF zszAigDX8Ag^2A=U)I~eCZ^|l@ZAHM{f`yg0u|{VS8bNy6YC(VzSH{&tpw>@54< z#AHKcV{x1#8ZEI^Pgas)V~U_w0g1-H$sJXaJf!MfKppPX(|81aSfCx#;0Mie$PFuz z6wgH`aoO|i(0iC>A!_qd;E7(75Ejvq!v*IG@!H`+IElt_B_;#$_msz_Dn=V4Gpf=D zuu>ktVS+cm6#Aw|VXeR-pnmhC0^9XLf$NGf#`%k#u z=glOr(HeZp&Pl2jg@=8GaRHr4+vK5y%NRohG_)i|%_K(R#aX(Dqyh9~;cXH|W?d2wq%T{olq@PhLrWGttdr zH`fxi@=hC7Z4f1U5P(+p;HM0!3c3Rf-JzfI1iTkkO^Z;pqBheP@}I!-P#eI}LvKRy zv$2z3uO_}%&{a`+t>)q5@c!{}U0=M!LeV4}!7*kU+=vu~;^egtO;^mgg}&`@e(3ka z`#vLB#HaGg%=p1GODD7vOiZ6XxqM)Lc7Bm%+5g?%k73S=uj~|;(?Q$PD3ZgdO6u;S z1phV$JoXWSqo4DA5gbyAO2zF8jDYq)F;!61+SBk(-1B=;n6KhtplgkM>Q;ytwG3&Z z(L5(rXC}Ke(^V|y7mUDVP_x=DtWxpW(NPvu$q5S5sUL!m!8_ocFVLcF(p^?OD%e6u zo4zfcB_ywJl61(UNc&iE|47%p}he?yrY}rx40w^&aD#}TkE9(#1 zr&{>2K*$G#X7wH6u{KDalUjiBxl+vM@^0PqtK~3echb0fL+LInqNgX-;)Ls~D8%>S zjoOu-fbYN?MQi6#`-Y-R(%6a`1%l2bLY)}n<K0n6F9;dk*m2;GM=0L{%> zx!RiQ%#RzhhE=p0R!{HNCQI$HxoNX$&anOS+*!dL2{`?6-A36a)2I!R?1^lRz(kP_q>;v$dH31l58v<6>9~X|m z_|H&x?ler>k%#Ymi<))uiVENT8~7X47#R*1;r`$iamgXnAHw4D4KGY}Ign!$o!dyy z+3a>iuA#IPFxXGv^SHVv@$*Tz?XhKFJ#`;_%8-~DTR{r|M{ z>Lo%Lu<-Q@7#aj#uEY%<@z`LAoKNkd@0y`Gh-U}=zv;@psj4ErehfFnI{}dnIZ0o{ zJ1PE_sEYg+A@1B*7gfr2!nQg@IZ6$p@BZyzB6S37OG67a&7GDr?irTS==Eof$@a{c zJ)uuE-L4uE-bl(;_M~z~RMCJLX29_>e3Ks;li;{eDRjRVr2JH>pa&vbj3~U+)+zL0 z@5Nua7r*`(eodqnG4*~%%szhrQi_HG7ZsH%TNu2i(gpP4$&p{%nXp9@BangqK*S)6 zJJ`}s*nVo5 zs3Ja2P`5|m2z{+Q56_{T*zXm14Y@Ex7liY4ni^Kx9SL~AJVkR2dsWc`rp<>TZ@Twa zgjT9dkiF!${KVw+2*y1X#67dvJb_VTt#<<0E!bzctn^98Vyu8B|8j0C-QW$z!{+4N$2-bPg&q!)(zXU z^OQxQ$(v?JQMjQ(Fc$dSFkL@*1I-2FaX3EunhQeImOO9SNlE9jQgZDip9=KrnG}PZypM|D z>u^Cu1sZs+JI8%p4i)_FBK$JE4Xzl56)=mTgKg+qLO?o^B$q5mr&Q0OF$>r6O+%7> z)Aj5~msLZFi@0LNb6c>rBwRWb7ny+;>aw8_H*Ro?oMwSU_}6G;s8^4y1=X9Xng&MG^LdupNT| zx}y^51AIq6)J=8=dqT&e4ZngK{KF`GAKtl@zXjhS9E8hgyxukiQ-`Swn>v5~UPNk{ z{h(~i^kzQ^3PB}k1V)4}np!2RX0^;PHsf5qns1hU*LI68Re;RBdAdRO=uY~5xWG1s z(Xl`czkmw(pYdfbqB{M+p4*p(?s^KJ6m4{$1pJ?nzA*>64t}NkU>5QpL>00#s3Tx( zS+3G-50`pb4M8LHI%VFX8J}@BH6!=uy6jcBUG6X`II@t>psyLBg?S#=B?Ft_hDW(d z6hMKULE%l&xhBj|cB}k3{5g3(Um6Z5^f>qFPh{J95nkz)D2Qpz-#~BT?RXbJI3uiH zvA~V0i++J6%CsykctI{G2I3O)R@tf>W+8QJNvWdl%{T7Qag1I;_qNg!Ub{{F6|X;O z%^Z9?Ud0FSMN$9Hk8H*-V2me^RowAkZ8of+>hf3ccdpBtuBLS#z4z}YTLN#DG>M>_ zPT>}{aeK}XEI=;rRjqvJ8Ii7Xa(dMPqQ>tJ{O~QR`E+_Ky|vz98fH1zTyJja3CQWf zT(Nwpd#JU!dSIimd}wUQBe_h>_B%o8F5&wSR&Fq#HBx6Ak-rk0r|g7Rsd&m;uE{g= z)VcT=JfpD>!VjaKUPC%S#q10@3*1z%Qu4E0sbz~d>9 z_GASge;?qXgnO;T3r)S+#-}aw7@oB)l!FEpf%qV{A3bMOJsvr0!*fPwE<0;@(u#FD zYt?_rS>yYm7Ka<2f!WaEI}1(Jr%n8NU@(Y0ynT_-6%UrosV`0SxMofjzv2n5QTqIL zfv|{z(m*e?hWEpGujBn00#x)&C$N0mkrd)qQAC~P9|fKuc?SZURv-v2stYjDOHzNz z4;Pp`$4AcNe%wU9awp(KE@tH42alna*noHIMrcPWJ4O8Q6S$a)TnjzT^bNbGvw3{A zMG0~kP`*sfjNgE7Q0C=>Ol?f79$i&8pn)$_s^@IYj?Hde*SiOZmZ=8Jw!5S;h2;JX zuGsJ5naq&f(Eix&63x`DsFW92w)o<7yVqVlFgc;`$qjUO#>NNJ$IHV6-~cmrB||$F zcy09Ipzdgy)+}m1iu5ScLdvCPy-84M@fr%V)oyerU+-ES{se|w%{6m2-Zt2NqQ>=aVIFEq#5GBhJQl;q#&we z=;0E}HP?bD@H#y)aJG;pmSO3j+WFXx%f)haeylM&xZ>e`(D1X;9b%pDUyFfqCF-Tt z;J!3qrGJAMQu#ExM{Q%V4`0Q1Pod{F{>n`Z-X2E%a~;3lJVUmk zTtr1p9^8Y>RpfAH@dTlwbG$O@zWZI25%S;3PbIEIJauEz*w)y&qouQ@+4`meb$Sz12kn0mo?|-Thspe{(TS~8po@uvnH65UeC5wzLa~d7 zG-b1tbWsy)UElH@*D=%;)Q}tSQGBNd;Vfjacv71 zn6Si7=)OvjkW|Rlg{A!`Jjp$wv&q9h4_9!_XE&#=^(wWKSI%8`OYUj+%)-*v(%fP5 zRQ+<7*x`e~&L|Qa@cHrmp0dD5X zV=`ByMjSJC1d>UN0!`w=QB=p3vf*+i_B~Ytl$XoUWY&aONy=BLTG=W?5k7vvu$Ik1 z(&Yh=x+3fH5)CrddNOg77vKj=f}~3os8}Uo)2kIbio*U1HAj|$ zNy-;GN#v=_jB5@7MCk@+j7G99QG@l9@H%wI73DG5F62v9Nm1l9-#6^M&GNCy9aA%7 zO}YDoaN|#)mFFrPDO0z#tO?G>IWR*-B18wK7`Ytry-5P=S`IW5E%F5n|5UCkSADPy z#j?R)H}UXclpr(1w>4dp4a-tdmn%B;psPC-BySKocQ#wI^J7s1x9yOI9^PS(8`9Q> z+4RG_BF0|hx+B!7h$ko>4;}E46}c-A%TaR&eVEJ?pdPqfa}E;$cYol3R+*T%BV4Z zUXoigYM7ODR7G>SB@>@l2v$r!Wzi-EQe3_pfuZ^8>zElB`l-lusS_%e8j6R%M*i_j z@I-YXNm)MG=*5u?l1D$0B|j=jziSvk8i4>7^Qx@b+(luso`KilLZUFWrv+99NN-pB z6{XRx)wF8;p^|5tAvYOP224Z94`lxiRC9 zz;kq}b>F8BSGnrm2>rm$e<+Hb(0Lx60)?6jBpy;MPFg*|*w@8>X}k`eX_sl@ygj98 zQVWutp%MU>2jMRdNEMy;!22oyt=SGMtm%AwQtJ&7MEg@c1<;CU6iOK0eWs$jy+0MU*yuCV11lg?D!3!83wxA}--ATpG&=@sI zN=zGd7j_47lR7H zu{m%ZRkI9qsplxwc!1C>)0$r{RSRCJfcs>8_n(AyGH3`?ut(`2LuA^fI>`E@I|$=D zU_SW3fOj;aTq%DS{53qyjGWHBr{G7TDE10o2;1xW@p%Y88$%jtajHW6*44h=+8W~b zl8&NSfjUsA{{*8MPYNjaBs!K@=$gLlGriN+MzZQ7nRVXXNyD)LJ#4x=8nv70drn|n zQL7VXs@JajsD)*PeHdNrN4ZlNQ>jpL|;2 zVp37~5qMNl#Ej5QSo0`(dkQ136Wnlv0aeL^+N!O;25o8cs$dWCL@5#X=R+878!`nv z0+pLHRmi(#XeS4_(pO|~Y#=tG(Gs3sJHebjMl^zEP`iIswHT z3)$fd=i23Zc^Kb~z)W|C9e;y3VCWkSi}Gpa4-E)nRzx#Hcc<%br0+YIJ_w(Jll&NY zu@?1G8iUdA02Z``Y<^hWzp2Ct(>2AX<}24B>#O_iujYYiuJl!Adc+mk#!H$&FIEoe z(=9q>c{nc|Sp^OINkM^qFS?NV@n!t4gi19A%Rdw)2_T^6(Ec};1w#N^i6+}irg+|2qO#MEIu zZ?{`x$nM7Wq_y@iVP&dj?_9-I^jKptEiR#(Z)$=^CQc;ylB%`yAuebP_T%I%mb_Ti z@a@^}(pi-rABA()HP83l=%DU9j`Ma&L6aM>OBn)81ZV0ZfL1M{?g9Vgq-oS=^w@g= z9&k^QXIkrvKml^8r4E7yh~Je4mY2U}_?GS1lIn6Z&|y#m{qK0%+IY^+3y$jMX3f6P zXu0~OyHu}!P2eukXkmf^Cgi!MxHr2X|$5K;|!MvucvY)WP zd06KqGgjF-IFR@RZNOt=1@+FSwUbaV48MWlD=|l{5)ffp5oGf;hc8q0stzoQnFQ*Sscx1Eog|{X ztUzVV`C*=r4dxaOZ|VCd%Jy~&bq3i{3SIV(bB;pG?-?Wo6(jdJo5IWvxJ zzR82^ma#SzF8d-E=R<|S8c_k#$ZJ@5f})Hd70p>0M04#Q5iQC~2N%-9}irIAJ3 z$J?0|uJ*3osL>m)<3Gwu)rNOMQb4~)I@?dEDSqlO`xbm8h@&uAh5wTR@=AnDcofbD zuj6`RiX}V?ccF1hitQKb_v?P7FoR8TE7A;EK7g*dXUnR?--Gh0IDvLtk__&FYoh7q zN;0ERc5TA89|)k-)f>3hpQ4v3j%5p)Q&pBvhYKpJda)3Ke=DwW?nE}^%bHWvwKw7( zxR#7M!=yr%88GT!)dsP&Lghpus|N->=T}Kum!(xnIiMU0oKBmqfxP6qE9iL6Eeu(K zvKEJoD{j!wHmVItd%To-a)N+4(>9S#SqaZ+%y7*SzLWiGp1K5i_s}C^KOe`cb}~r+#8?df6%ezl7(d zpmkpm)|qOilbhq`Gd7Zo#706Sw)zE-FI~BI{amQJc@}r)H^(>6Y#y<8{PaP4(@3M+ zAfY^xaNbpmo!eDt7W|AbYev3F zFT|ekNML%wyay7+Wd{MBbwjmhra*(rg4uo1_FT*L zWad&!R9tP$cVl}bBb4C8#8Tnyz}UTQEKmI z4a+2@-=X>tDe+SB-$Jw!a2^U4*_Mvx(Tb5XG4DJ~4h(Pq5i3sAPYDY(q*QvE7bCIc%2*sUBMuK_5bo5 z9`-pcf|~=;gCk60WH^nHVYLpSM4;(iu{&i`?vc=YBg78|F}xPfAXrCFeo-rqYtnWE>WdG*AP2>LadpnRu?@}zizh}5_q(%m4{m+uOh)|MslM0d^dIE9~ z>NX}Z13?i;V%K@W3H1wysgvQd`P{;-b4cYMgwO?>7aP2%@e27j6k7_WY(up~M(;D< zx6kn(CO29s?sI7lHuNgoJW^8CkV+r5Ii?=Izs(Waob295Kvc1Ix*8YcG*!UGkeTQ6 z?&JSJ{%`V|piL0xjQ#VORWPjmR_~QwmQ^2SKiODk_t2SQ;~*G3?Vk1OvR1Nlvabog z0I1OB4CZd7XKyN+KbK>TR=S#xy<f&M<=M4Dh z^$aqi5)z6`hY;YSQED;U#$j5&U?;FID);%n$7l7P#eRs5W4VH6fWEbqv}mfRZPUD& zPjypQM;kVb$#RY`JuTSN zSA)GjN3QcJz@!`c;k49`*mr;|^32~jQOB^suYLDs-L+jSJayPFp~17?mbJE@v!p5B zEU)P}&GjLYP7*tfDwI~EG*3uS4y;L%ZoI;nVHlxLscL%=Gv#aqY(^K<4>}AaO(fN_ z#ALG+BkNa5F~g`bd3+fqrc$fz$7!lyk@$19$kFsLOunLO8;wE>|K~@1$+3Jv^r;n1 zFj1O}aQiUGP|H1p(J>()Cp}-JvWT>45|<_H9i0RX_aN37J1xs_CbbE$Hr=MqyIlO* zP89oIzaw~nwh?D?Lk9`BY+qXKyM?WP2ZfD~GG{x;e|ZqX-YnbN;6(3&6FDNn*+Yzi z%{}>2iSRD=I(eTl>R2+1RLYdZ3F8!NTPpL5;C7cC6gw}WlB_U8+6$8EcfBuXK5PIc zWDCyAG(v8f616o5Ht^0Fr*J3GNWR41yegLJyYG}Y-QZ!yHe2j^uV zt3JrODnM{Rc~8pUfkvE$7kL0Q&XeRtte=Gy_d&7Kavt)kHv*h#p!YE3jZ-scNsmD} zHbbW!lf#`mnK99^;hE(d$&2#$bQ?WHrl}kuZ{h`VjnX-8Q-z{!ijt$b0%9JDiW-pu z)&^c-nqt|Bt58Lc;3&HVOy8-@%a<2XJsGY`6=VVmyS6Hj_Um<~F6_qjozPdL7d zxH~`P5f`02gCZ-v1MlaNy~r-08nAcHD)B@`i+G|*Q{MC_3t~a^sOruziPol3_1Ts> zHN`ONj2UP+@e7{BBmukiA&L#@bb~+R^l&ib?y8Gw$7Wot$s@voaQ$cA*a2X{l7X+ymST0|DJzWsV z@+HnqHwh2yl0B1T^1E_nNg%Dlza#$+;TX09Npn*y-?MNRZFS(7iC}a^ZKbAybTs=z z7T57R{CnP)HKYQ{>Z*UmBcB@ggD;`}V&DVAnvcuJ$T3+w;#EE8*mzHFmbe+T`;Tm7 zJ7)Ln!A^zG@rGI>1;-WF)>-)Q4UnmUh&gRWwz->jIqEC zoia{@YpNH4=)wgiR8MrUV~@jY{I=rOig2cqMY)8$WDps=CaIDbBi(l`-xe|lP0WM% z?>X*K-3ejWs8y=7(oFxz)Rs^^sTwkU!T&-ynZu~=1!js`zuVzI@-t$PGN{|*IL)XS zyx#?DjDe@zRbd<;$Bbc_4n5!x+N$gn@l$h$yhqkR#e9hzA{$gb)wxC9BwhrDOEYSf z(z~F!MR2fUAGCSuhJ{!*Ft_D53Pxg3flSmv#dfFg2({LHjv|UFl(mws=CGAFj6H;t z@FFkV>w{W?&Yw*^uL@pb zAw1tYT_bCpl|byp;6|*``)6E|0W8>kr6*I=1F?Y}d5Zi4Zi*_L4GCuw*7+X&L(sh7 zJNpWi6E_HQRM)|=@`~kU)Vs5n;3>Z=FV_UpE!`wHQ96Fb<1C*(_7HpcKhLM-Sy|~8T}!ig^M{YkRG?%1Qobp_ z4}yav)UPT`iQJ&*Rnckks(ID*IN7Ze!#crXrz#ah-AwKvX6F$&i0QA$LDx}H8k&b} z*LO6{aGlUHvH7DiB}``ucXt29>5H3fc~4_-XUeVG_8(U*S$H8ynIZb>Q@RrUFqL_((a&*QlSXs;;YY9Fx${ zn5Hu~`I%MS?#=REyEgeDsXYZjVbiQ6k+~l|*cXTpZAK!jO9pRAQYuHmA$rzlR=G+$ z4TxGywL6K06}T2M!ti;Dt)(E%;M{x20Sf;<>QSm`;;T+|UW#F#$r&<*7|@>XU_|O; z@Iorj5BaUW2U_WIcxQXls4oc?gsaW@0S??)x5GG>9OF5R(GCp2m|4umM{6-$Y~SNp zW%4=Ktuad)POCqc(>c_}M=2d1}vL;JT&2lXW*k$l#6c?7SGeMHddPx9xKonNd zS6>wQ2()0Eng(k1!&EbwT{hr?#AcY}>Wul#FfO1l&6Z?EpNk2Z{<04-q6HFUm@+~q zh;bdH&R~+(@eLmZ__PhG-8MW~^2`L@p?FU4ouw*&5cK!1o`(^>X~o9RS+nxOBadP018(-k#)0^kJgaF?XbQMhcn;_wrv6`K>Y-G0Y9iv8ajcQm&7ud zoI-I1_q6D1NR4fpsRJ_DhFCJ1WUr|%$grK)f`G+t%_R_dPQ}QJAeoV%xH&WSF-mev zpYS-c1Ah0aB0)yAd@lg^wn1}UDhiIR&P>XvFJXs4Zx~)cKIGwt1Z6>mEi7so)N%|J zP7roKW8M?HpChsgw(tRRi#$f2BG2)O4dM24H-jf1Yd_E}Un@(>GYWfzJ#O3ypAw(F zalO;JTDeLerBBeOwPD|8pIl(rGr}E~l>uf$KP>ALduMRc?P1#K9$h`M@jQpRGCnN& zLnI~+*nA;IX~G1L2@c1PNp;Yq5Pfao;7>&Nl6QDlgILl~kVQ331Tg;yEf_kA0&*U> zITlgs2BjcD9X@>=bfjtt@bYvDUU4EO@x$7tQvMf$SpjqQ zF!rn=UlRUg!)lfTG1mcJ8$>+y>CLWTU9zrO2bMPXLwt0FJ#hLg=WMaB(yN!v%j(AV z^7i$Mmvw#A$20J)#&UdP;qQ*1VE~s&1DW_xv9oa8+v@_*CME%E533V8F78VHAHpjS_ zWaFG5jcix(})oE;wlPz+VcTjPzQc0D6wT6P+_=qA^re((>9w$|~ zAnYaQE68RVwXt(2At{1TdZd^4Y76sC92by?uAGIlbBRBPIjTpDAVMHjk;D)Y|0Vf0 zACxyYkucU5#XuO~;bv!kh*&TjbIuTD+jq_TW?FY;EdGJV#<1r?#Y(WX7FoALf&3hK zSY?6ZxQ3bYkUW_xV(2P{P951$Wm5w;!Llt&^K>|7O!8zEqUFLLkzcFuM5&>0bOZk@ z=Me4xDG4<}+^DOyfbazS)}=79pbROm>z# zyRbsBdv^2S{J!nI=F(JqrCP@t1=cI}%xujZm|0txTc6vQ+i#<+?Bq!uu6}1f%?Of( zWg$(nV+R(AA7SiyVCrdDIo3Zj(_1P|o2uig8{`2!Fz6F}rf-&fhkO}+lD9jzdg7x&mF^;4QbMC;tqdi8{JIQ9>M8*qAKZ*Og` z*}? zAXo^oFo*L%7vT*YvgNoItrW4z0G;eKWwIWq_ou_W1+RX^vq=SB!zLbtjDbgp3G#{NVe>M$MH|R+b z{6WKJg~!0_Gl}^lHsRoVqC{m;bi^rC8M;h0#bV&!q@)4{np)8h3D`@*38FTte&}3- ztN7~>IIZ|r^f2C!v9l7652Bf_wGisBK+{w%h8=WY;a|JYTl}VG8mj6_h_Hz3=MnyY zA%6hE!Sg2dqBt|x&J}WvW~H>ayy7soF|YsBQq^F(w~g zq{S)*L|gMugWT@ti288UNpYg)7K%s6Bd+&6`Iz#&Ac@^L!3i0T)D=jEDv_FNVZ@U| z0A~Cu@?p*j$u-OZ4)~h}T)2ef!n!cvfX8l%3{ShY+0Z(QlrK&emx^&3PscNAyXpG6 zk9H%PoyskDcXoB|SWd5f7hQ9vDKQXCdNsJj5kFw%iL#q3Y?Eto`Y0^zHH=@F7D@7f zrI*QdFlet4@&fwGG(D+cW5M?oNC8uaf{^32 zRGg^$f5^AMJ%KAGDNeCmtqZbUqa`Y0v052Tb+t!*<*YT1Zrn~Tahmp%#%=O{lIM+Q z3CYW*E(wwxcmk&=m?rkSKD4A9uu?74OkI>(j~z;eUkhKSm|e z2gJ|f!z>>%#xk-po+{$G?UfyI&A_wn+xHMunvgRbK{N~-aZBH?V8dOzY|MhQ=j9OV z%U~EQE5%wB|1?oDChOQ0>q?@416!gb7cU!=O^7jc2nQlRK-4RXYPxNvRJzT{5;jQT zGvpiwL}_W?hsbZl0(Q;7k}05i|i1@Qeh(EnHaGhBZ_kR>3KWgM;gDDLQq>nCoXc;eQv$BuE1)XjaC zV?FuI^5Sf6_R$B9-+APr$8Va0{K!ai3@~9pAr2MD2qkYYwLhj>`aDNNDEFbG-|JtR zA-qfm;4{jv^e@RBvLXwTfWFsx7!uiovTeL%Lr8C*inXmq=RsX9>_ERa56&ySJJbd)e4{xR4 zBfc$K9{D}2XZ&iHbl3Y|5XY(i-~Rlre(z z&l9_=KI4!-Aum>IZ-W|XR9){J_?51#8AdTZ3gU|RY7dfU$$x^cucOYc7A4m)%2mr1 zq^u`LOY-k!vw;C0TKC{P_MH4N=P+qNwDDjzHGBHO<3~mKn4J@cj-1LD^Ec>STtYh# zwfech{v=xa-Y%%IY3iA$?I;8L6jcyeE2Qda5~w+^CCJLZ zCohs+MrM=u+n)z%u3iU5SCMCvS&zK@hc4$dw zbsQI84SUkW3m!bwvPIJ(zs#*-hJt9GCCYY}(&`3%)^FN zqrEiUwPr0ko<|z5Jn;-_)((`BEDxbK@r=e)TPQ_Ty+KO*kfSvJasdsrkkMF3B5nPH z7C6;;vPmv%k*(Hc+q?QIRqM#`M7%0&Llb;;ArLWr0*6vkTv~*!Xg)7X9Q|>QJVIQD zaLs`^`R|e#RsyRqZBQEJ&SWvKc#db(l2SX@$f}xXrU%W5?H=!&YwaW%&aC+R5X%&* z+!4RU!ESyB^unzFgWL_@?vt%Wa4#g1Zelx|78?qLepu_BsLl%H3qfD=ml4vYf8$l? zKkuV`P4@ZkK-*&Ly5(prmnb@+u4ooR@U}tQf|Yf_+q|30Ci{>Xo6`4~dqs5tYbDe}=7MxNgf->=rt++;@|5^~ZdFwgD^F!?m0Z;MOIU{T( zyZw^TxJ=Fmu<}UKY)zJIUu0FW%1EoNG$xugeJWoqm(;f3k1zf!@7BkDmD$ZHKb#Lc zlM{8@wHK#nt7R3vgoR+iX%=czrM5M<;8hBhsY=_ti&Oocq{F?N0>2yNy2?UIUnd{4 z{7=Fv--ItemT9ke3JC|vNmxJ9lSXh4X`0_8Pxu7fOxS7AWgc8}G{`3`N7btndwf2K zh&P{uN5>?70S7kwtc@5sO@P+g0Gn_eqo+t(T%Mg?S)J@CDUrju=G3nC!d!1^sl7DQ z?C$C=7<~@FA(?2uCuOXp+v}rl-m|_D>DsXT^NFrV-WJ8W9imdzVk{S2AY!tSVgs-) z3lJlD{PokN68^`FM_b|w$z@;WMQ}RNu+g%ecf{$!!{iku^}c+Lli7M5BuWR@b6WyifW*0 zoQ!$1-_3~I8jw#1w&~kvJVI?3%m(`Hd&-#5tL<8zK<}`&$3(Pz(||W{Y{M~KtGe1n zDmC2keB|as&KOkDHEJj~vcg(cT9Y%j1aco4ZyDb6G>@Qebtt5zqG(QM?OscS0*)6zj6~V z{gpb${^n^H|Kf^fl$H%Ml*G5m9nH61gon)9O1W;fhoAPlJc?<;8`+yisgM&OcGFd1 zkcAyIASM0iHG_%ljfU#_KNpA_O7eVu3!DYlBZhC=u3Isa2y910_E4kX*sbK3Wkw}W zGfPi8j&9T@;R1!Zz*2qBbX{;}y@$x(kiWS~&5~_RB-as=MzzBsFZ6dob%IT?_G#;i zRG;P5@wJ}q?%l1M^d62B?m3LcG|O z^OANw0av*jLijg9nSt0-bwLq@?TK6DX>uKIKjU$=V+zvM0&&xKl?iXgx)ewpZ5=t}G97WNe3Aw6ZeWKr-S{BlVbS>ywW5*bamb zE`=es;)_r z&2%IvhY#shj>M5^>3BoBxlcO2Vfll(KlE9bIYYxGoJJ{6?>n%*dh+zi<2=`}wrg$g z_SVMM?p?jT>WJC%?vpt@n|*}q9s7I#Wp?;Bo?yhN$5vd74?rc+FHYWaFPh0wL< z%MSPp;HbIe(_V-%$gvxXzlM-;R)X>pHO2wpd!{c72$7wF$`>L;KCS3vfY?)Ob%!@O? zh(Lw{gR^gC^QR{eX=J~~_9{m3h9iW@)wt86A_tCB62hoCvM`P1Y?Vh=_OO);G zY9Ccf``9~!7OW(TRKcC;7aIm$#TcF+MYkHh&XVIAi%~qkpDcNikrr*utLMz_X#&!> zHDScDL3LT&pW0_M#_aU!G@1SiLGt%0e*31&r4R?n^6$C?d#Ml-6uEsQ%4AP5R4LRj zF~ST~8#wXYup?^uoQnISmPD2PKvHwVEEPRgqBl0$R=M*;J z<%)$p6;gYtD%7^gMu^dn;w!vCo>{6~7C72xW~Dmd%3j1g_r=p^&+nb)m>-RrX`i{U z+1-ixk)5xlWW)nFc)kM$pWmn=Ry4T4O+i$g)KSDhqc!Y_ zn~)~3Zv+C^AB6*9Y++NtSt;XDm0TjY)Wzf?;S~3B@iFo`{OKI|tk3fiK16qzkuo*e zt5{Ubn@vtn(Zs=7j)wgep}f4*#EI>R>wb|`jD}0rZJhQHWk;hDGu51Mq`#wmJVo?I zr@+qL;k`sJ!CStC)w(I+nb%%?E8opOmw*0c;}zpgW9#tV?X{kH=)k_s+zU^>5kIzm zWM$9A+ZWHj^z?^be|GX!(c;yEA%7CP3uCe0ZnV+|5lnjbPz;R~M`<5Wq~EE|Q_91{ zXUuD+Y|xCK^jQ}6kwo;s-lS2eM4=rK%Rhbx@=maVLCkLIVpZr)7yd3uPblx00{nxO)^m=A`^2f z#}vHQrKW6X9IY{HM4~3IM_Ebc2hMgAA^tNd2N!BlX-4|PcAH6O9& zWLYr`MIinuP(y80MMUSE&|R8!GEli8Fn?k~#O- zGu#_4^R9g3-5JTGci-@NuoeAohpe#7=j3>$LX$$>!|9w0e0ULOOa<~d`D^kb#ETlv zAqbp_;@99H|1f#j$l0E(Y!a7wdYOp@7Y7O@Aq77IPVG&2$gjbL^LTNuKD?)quk+hi zMOhLyK|ktC877m!A!3^8!V|v_F4)&i{%pLN@a7t53iU@#Vx|T$?nq#|;Zq5=<+!n- z|PaIgCr1? zYM|JvugOjnK8fKB1~l)H_uh4x{RnLBOODN)z?OdwF$c{sO63 z%nRf6E%BJ+Gw2Q&77Rz9=>hERJ_cJJ5y9ZxLaUbkoUIZ#cT2>`b1Y8BWm=$zRC4Wn z5p&7Ao{lEq_`W@fGp|z@&y{V}J_n!lmX->Fp;6mr^heAHe0HnYM`8MF@*O^z9fLmJ zg;v(kd^Zj&1-V=l?&6)vNWHmW4nF$Of!@I^dtQ)LIYoxOfDn~E2R}~0dwkl0&tHVY zm`Au)W-|%Ygsa$w;)1(H%dloCP2$y?tF83GYFPz^ezc1OEa~59VW{yGhPaOsx zfb>P?+SH(y%Z4}<-vxy@18dlTeXkaUN=X=nO2cNd58nFLXaOUn5K5WDR$q+WvyW8hq<{G(>*GFFSg>j*T1S@Q-8r(m+|e^u}9nhAF5fC zY=Xl>6Lq=?b6|pC=zD&l0gXQ;_i7DwwXKO83X`G~oNX zrYyjE6d7bq)?DVwhRZ^saSP&AocKPLfS(%D0I5a<;ss?<@>J&<{Ox(ubQOeAm6-I4 zvgvDvYzAz^Gxi5?)!J~~;?2N`46j<0Lf^F2Sk)^f<=)=Op{2g}EMr1R(+g%q=`*1I z4)Ifs47?NCL0m@}F(Z6Qiegprd@r`1qsrv0>|swArjKx59q+|obaed&i0ezXW9pVi ze?AF})a7|bl(XN3mz@NUc^mu}_|)A}rM)xb=(%of8{eg6U-XUck%4>a;VN%P49IMu>pae{kxO^G~41Sb{?1>|s%uGy6EmUyfx zai0pq#aJijt<5`blmW52=hg!4yVUPuaKm5qISREWaf2v z;BD~V7y_O-v3zC1_nHTS#E>;#>wD-K#_9$rPtYCn(}G4s!z#|r$fKxl&|VYwae9XY zKz+?Jb}d)#r%$evqm)I4#14@sgsY-Rx}GD7zKzrU2gwq$d6uwdf7;ejcP|s_Vjm8; zS#k=YhM=)X7TwVKAy~FA7fq3OnXT2Qz-k&u15^kW$?;hMcC?tpGxJ|^H1{NI#0|xw zVB#z|+MyvZ;V1N*A{tS^bLq$@_B+^%0oz^N%oS5DPy)}93ptF~K!i=*r2QPk4&;vh zP})5!9%LLYke>kI!X~5rV0|mxl)O1Ohaxi-T$y4-dSttzl&D|D$|>@mUF3o1$Km$R ze#OyUgkpG9R82*YEJZQGAI-alDCG<;$A8M@*x_?%)9_Y^I~*d)rIbXxR~$aOaxbnW z8&mtdDS4bAdqf|5f&2*B;zuvyEaJ~)dVikO1j&!ppY}Ys4D`#EfZ10Q?l3^_C9 zG4o+q_X{DSZc9uBX%lZl7S&JWR`J84TjZ5Q&jDKez-j*;{%$N-VWrx(;%C$T;4rLM4M`Sk_H@EQKq07 z5#`_-e3%je84;rW$^M9|`#Io-AuFO381;eu}_cP#o6$tOAljMtCP{A5= zz+6g+6%$fzUz3z|JlIDiNmC`mQ%>B5b8!WGylD{o1{i{`gb}FZKov~O(Ctx%?mN66 z9kIEpnssJXO2Q0P*&iVe~L#0MMDHx5QW%c%=JCb|7Vz16BM5}A`pa8f0C=Q zC;9k#m%Q2|y$hadKZy2NluT;rwoZQmEV3W>&;5Vm7{^?K}wm6$Va{5k$P2lU4V+aGg(q%CR?%1DnG37Iu?M zBP%?}We>-C`f(^5jtD0ss+qyy#kg}aej3h@lM0J4lf;gaYhF%^Lftg{=`-L3i*uDI z&on_%sp^Dx?7AXWR~UV8&W}<%uPeVl@SSn^UG{8v1_Q@8)xPKx$B_}3MZh+XA{PE_ zUpvwO3n7O^6cN}HNp(u!uo78JQy&068oDsF^GU|z9-n7%$^_va0qtbo9=J|WgL~l?8=0DGqOTHDT;UY*P zGDBuo5KjdnBtCEU4S5L@jTop(hU2-uyBFt8UDx=mPjV46ikt=}Ylg>Eb#|}ybB=1< zgjK#4A*GsbiHi93H~=vakWftv+!12;cX)Yej`Q_waAb<*T(A(-%TW?7QMB<8K<&b$ ze@|abYn5<*Y#wY3V{9~JKC%%z?IS#4a57GlFOvt9Ga&k3BVF1cEu{_tb4VU0Cvf4Y z1Q{=nONz&2MLrIHp&Rtz=hd75XC(XvClUd}$yuVonwP(aQ=|~Fq$|yFh*%uC#cX9E*C0_Od53ws9nV!`voEL1@9au(%WFV6|;!upDA73Rr&WnN>m;_O;)DCavni3Z7XY{DAM zVP+}lc5<(uF!Io|a{g6v+B2h48)Q}RG)u%OYwS=aQo${w*kej^9RrsY_=`mzB|m5N zcgC{?IM=!iW>oTZH*nT{-3zt-H%JN*5;%pc$bW2#4D5#HBa=H3%6o{+s5oE50!-Xe zjni=6zG!PM2G&p5287j?CMrKEDv4o)p_15IYVq^&r+tRG=^+B>?B-OMp?~^_{aK$b zck=131^sX!ZWW@WG}W!t+Td1fA6#D4cOv+;?%Q=deuO^Aq>PncuaW1`)dw49 zM&MuG0TDioz}s*`yo%EtC?4-UZ>pJfL1<{Sy`Bx$Nyf(t=<6u4+Qr~Yo zyh{Wj_^kd&Qvmn;Z#zuAh3*zdtT1Fj*Xa###%u8PG_x$K=Z#FW{4t-sDDa8XY-Sx! zS976OSznP*lj6+v{A_DX!utIqY*>}P@8oss-uPl%yS4`+HZWx;$Q3?oy#d<+R-#ci zi>+W2$=i zJ~6r0Q}52i4LU4WvJOj{Rm1KrnBH|>hiH+Bd}0;8f5y-ao?g%(Byc=Yn8x_JJoenz>6Pph3#!g3dOq|Uh`!$#vID?2b%MnBuV-hfPz6BYM-tYIvc zVy509$zJl5B>LJ^!4SkCaOXq`6pzP`3n3lMj0uRB28OM9QC3btxHNHCqT3yX7B*%1 zXn2Tx>H))kX#YWpp86@W$)t~A_jsWt!A?2oC`+R21dc-lO`r8dg^BSJ;&R)5mk+TR zh_zDG)OxBhufu35q?%izvgc^j4oG`~7|6tkb(%aeZIDcZ=XSqE7)YAAsbTF@6sa(( zn~tiEg9;-atS0&AaGBmG|F}03oyHvVQzFkn3rcP_HJV={UnHL*pWdt&x(OT!BROL% zkp(C6?E;)H%n1E4`OoAJc_wxdGeUl7yQ1YSditESqk59;AnF`=o>_jZ|5>sIR+Y|I z6Hykm+oZ9N7luN_g!xXrrzzn@Oc}qV2%#Xz0IjRbXwBrb!$ z{uG?|9}WwO*oZ_$ll=n)Qg&^-NabH5=r;AUS^tISBQc9y-Yi%N&hPG;TUb&!o<6v^ zpfKF|Wp_WWTiAo@|v3Ds!Ou2DTsVCxep?$ zSjXx?Gv|Z6?fX&k!{ig>lSiB=iRX5c^-`q8nr!8)%3^&V5jO_RVu;jWV76$;>G-oT1@&hoMU|2k(z>r4ug zSt!9lJfpn`SJ*|OhA`CXl$LM?2t*DX9#=yE0@s2FAWbz;g`tDh;g+kvf-lvclyR1w zc``M z0baDfGZ~jp!e_SYo0Y^1B1f1-#OC>S6R~GBKh|f&nK_%P%6+k#2fK6E{$wZyy?4P} zggn-n=T`V#xb7ZY_rsbGi)~R>)w9blONFou4=BAtJ_Nb~8QESz**@C8#(QewmK&>gB3%mBn%dY(AUu;v73pT5U2pCLaFtNu9Yi0mfC z4ls#`m34z^s|&$pD{fL{0=ZdLEAk(~PB6O$r$i>MM`^w$W1LI8k|d>sO&Px1sN6*S zy;ZV0OKq*Gf`T^`$p|4dNZtI?o>kIlVOdr!YS~U7T`Ze4@4?eDUX{tsc06IFcxBJj zjq|&D3riQzX}mAxw0_}i?UFgad}?{-xOR_D+lgsEaN3d`CU?-xJJQ|!q;SnWXqC{( zEJw|rL^283#C+^ zoYnk=&w=nFb;8e%e6bTGWEe=9p0iDkR*EuHu)p#I$-{9gPlzCKZi76LMBRY;sOJk7aJJDbh*(-%*#UKYs{D;;?i1UIe+F{dQPUQ+DHE%$P@4I>kV0PM=E$64Si(*xfb8m#g&Dn{aN{^ z;6lW_?G~aOr(AX=lbgjFY(w4)5W(&eT*(R0uS}FYD}`fwLesa&3L)WqBJ)9cPH0GL zyQ>1oN}C`B)XOT>`J~x0H8Ol)@j-H3ejY=w+AJ0^P14)l;gJ>g0*i&kX`FvKLgqr) zGDoB)MHTLtIrz2qau3963*~;qq*ef{gg6;TVJ};ojl+UgPzqbSw|DQ|*n3bqn#LLg z{PLdt2R0XuQ&l{)vxXFpps4s3_LBRc1hVmb|ES-FSMSkD4&kj6etgPcimFvtLL-5M zIXd~~RG3bJ|C+d_rlS?g`=NP?cxh4-Wbt+ISOPen1ipR&+=`EF3w98{hE<*p=jhTI zRucnsTCbCBun&sF<#jnGNBV7r3(Y6nW};m#LLCiCG_;C9{XsLJ|ITT$(R& ze#1ZIGB?@J1i}uswaQFosnVN^7NT7dE6>OCp+Ds>F0ahe3_GDWK+YdU>H`F*!I{NL zhD^Kn6B+Md9zrjr0@;2H!|{a+)@h2v?Mx0YOC%-ul*&J=?2U z7NSwJ9Iv!?bl2|LR%c$jcX|n}*8qWogFlbJn3567%y8_JvHr*pf9#v2u1Jw0!5Nmp z!M&zkARWiHc2$iite+tgEs%;L{XP5ij|86R;RD>VP5W(obwTzO2)|e$$`d3=I|CGn zzvEYY8|*(^@6tq6@hU5`^Rp!<;zyB)hTpWqwJBvd<&z8;l?F>jbhT_QrQPepV<$VI z3`Rz)x1(x6HBy3LSP1oHvgXnO@P!~+5Z&S!ULh5CVUGe+rH?i;%mTAKN5mwDkeAql zPAk(>Q8LZsg#_$bzN35r&X49Wv4)^D8Q=`Isu3Jmkr&ClBn8R@@%1n@e@Uo629HVY zT&|40y1wVjV(jI~$4lLI%~TqTlA_Ee!w!`H&i86>mMtx{;)-7>;Xn#_(L^+ZwiBH> zpxlcfKDJTGKsx&Jk?a7x=V{b-pId3XJ5?uTDhKFPE|FE2WgsCHLQRu$0-z;P^F0kA zbJX8jG>kQfbma{uV%x5@f-QeK4bK=3crvu;fS_F`UqD%ch{?ez+3A2&bbHtjEC_+L zOcrR^=Z>?Y&W0F@itVND&#H z@YBO_TG6ahAA`Yl1U}fArb6QTnB3Ei^W+WiiZ;mS$bNFf2wg3mmg#o&GI@sBF0)zT zOCEu+9^52F<<*87Z*%rYLlk6N>XBWN1n(qBO%miAxBWh99egh_8@4>q)D)wYAn{~X z8TBB37vA$J@&bBB+J+)M_}G<)FFgFs!IauYTUdz0o>I9i6>ksn1RO``qx4Wocw>{@=j&U z`i0Y^gjP-vmvZ^KI$Z+4RU4mA5!zlT>w*x46MJ~h3!Zh>A-awqNdg~12QP&u5gigs zvMJTD1z8kaK}?Bf_y;51u-$&2M0TnVz^T4MZlZO_v|y$I=|BUXIe7RW{U99(8`jEZ z^M>2F<80s6Rk(u`n6-AWqo4?vP=o`0hAi5w*k>#O=Sx=MxWiGEqB3kGC`sizom^pB zgsq>80?RX9$Mmfr>4I^5hwNi&fbJFDO+E(io450~j1>F>&GpqUdPMiYQNXP2bTBXx zil$?|ZG|jp@Jt8=Bs$@L(~ok#Y>0vvrwvvK{RW%JMV7?q*TCMm3hE${_{UWmR?TW) zCWd0Fo~w=~SpERCZx#t$Cl8ay$mZtWy5TWPID7rl!$CDTA6$@>tIC7Q{^NTOtR2|j z+uK{~dFN^miT*`;#ddXd{lv1TN@d5jx?2g4;lc4h zw0|jn8g#4AfyePwV>Y2jNMz`Uwn;@x0B_f_jwuV$HXJx`Xt;c;azDw@UM z0T%Z7;XpH7HCb1|DlBC94Djq~;}SRzGw4C!PxyLV6fMTULmKhU=6OtUvX7{yhH{>~N^ZddPh-y!Z#+6dp2d9VL%6FN%P{QV$r27+Jc=z;o>WC^2>&>; zP6PG6v{TJXtP)r~((6D#^ow#tnlXv-i>Q6;_lICK*U52m4c;fizg^s%%h$r9TU_tn z?mp3-TAg}q>Z!{YA9^5fhqjAIZKCegt>DQ=@2pKezIQ{t+XubZ9**zc7+hR`Xk`R7 z$JNcyOCR>HaX^*@S@r@O#wt0JjoEmx`kAp@*$*fj8G;8K}E!) z8hJZ(;ABIPMSE$elxFA}diI9y;+&vZ*0E?<^@HQ_0eCRpK<{|I$eq`2%u^R0X^9B@ z4-yQHnGDf1vkT6@`V4s`Myy?4k0eGymVexUm$R6ws^uwBYy0X?;$$p_(a)2Y3xss9 zlB++TIK+J%4qp=wAm%Xtl;>t5j*_+qbJ{xii-nw|G3WStWZa%Sa5WD640HJXTIDd)P(wdl=Q_w-Oy ziXlGvG(7Z1SrA2V55Pgm{e;gaLVfaZ1CPGW5r?A(PE~zF^&NJ&X6a^8WMsIy--pl% zRn#VCHK*E~Ct|HP*^!6XJL6bk?IFVV2xPs#_?5B6_rsmjmTiDT6sevW z)XR>jSdylxe()-6fT8ei>39k^VENE&nj9qWk*~$N8`>|TGgPgi$b_Dzhb6;6$}`dj z?1^fh2^GUs&SaZF2yB8#O1XanPs_nd^FbD+{3-8I>GIBDd()`o7H7q(zc4q-&yV53 z4kn$m;A3pWxxHhlgT?lr_q?b`VmGH4zHX_uqejL}Qj?8}NY3n44(tY9;>2-k7lQn! zbkE{#IdxW1JG4I9CwDmZ%@saeTi$cLI+m%O<4nx=K5)(HU|QI{-sC2^UCe=X;J7TeLij*~ytwzHG0?<~MZD$-4L3H|vu0 zSh*d0K3p4#y6p2l(L5^Cz@w<L?C4;q#oeU>#LO*d`JC4u zyTyLlU9fcQznvg=JXnOaC-ImxmU!>v7IuNVQfER8 zh~-#@C9kfuo9G{B?Lhsx$fUgkf~txj z_utkP-N+W}O|G59u8RlSQ*p45yidMnz?HRB*%t~2$#$J-g)AD|Z*WXP7h-#s3e#{n zbs10I?L21iDUo1MY$75WDv4X*u-grZ@^p)bNs8C0Cf8 zX5$g;-LB?XJK#N{y+gRqkV%P&LPXgu@)QK`7Bbjk+b~|LkdKocgrMez?l}FmUzx>% z$h9u4J+?g0vwy-Psz*f$9)ZrHmg6N9pQrV)GXw91Zv9_Hwt2q4ctO@U$zj3!sW{P- zhzunhk$;e*bw74f_#sw=nl+B3F;Tw{&%~I|!R|Jwqjm1+{6EX#n2x|rtG8S;j1BlT z>B{Cp;lrj3nv}0I>}>DHYka#j9oxRlYzQskC%pp1dy8HM%Dk z(cMg0S>2n)a>fJGyC=Q=0qY&+K-;6fU@qm4BFw;t!JnNcLN28b7_w$&OZF#W$#7|J zmu4dG+svBD852=zigIjs--00OGW^>z0vB85sjZq{B9D^|U)Pu+HUoT>vzbwW)55Q= zSnyLtt+-YU!J#18YT?I&i7>oA|Tn~4pyvY@Q zuCH__6ZZmnfPBFS-Ozf+B~^^|eL?mFu?A+b#s@}fUkItiBz~X~L>9F zD`~kf^CB`7uU&lV9@ghASjOg_s3qHrx;sDXxBW0}Z0uz_dWFUg@;%fwBQ?jT!T(2s ze?tTz3^nky8*rkrQ5UPV0{97{AnHsI}f8LEVHIz;T5z6)FX51_yC zM5!A^c{LFP*Yn>LBuQvYD3;BcjXmB9KaWHDF7;)}LO@EPyhzpxDvRcMo4jh%9}H!a zaUJ+*Q3OHBypN*)(?(Gcuv=at5|@bYkOQ1W7A*PcME0AGtyf*UoW>K1WSJ>WZVMg9BC_Te_~81)A~JpV zT9}b4PYJRlOx6^ds~{NjLhXHnPfX&dTiDTptuF52YFbRD6DROZ&$oUF;}KJL;?%QK z-^t4&IOv*eI`e(%SFwi3ff*He*aVN>DEjE<*+o@#O<5BKxA2_+X}gAIMaVXlho;sS z%C3%5sBj)il)|-sKI8^^5G$)S_0sjqg#(*Peb+UGRWDyDYw<?}O^6JH|%f z$6?#EzLZvbge>0f2ioHj$;p%YGKr`JY;Y(~D|9Aa)%K8WGKGUho}?Ix(X`C~Tc&I6X&+^XKz>ix$EJ&YfCJ++Us}bI-{0>@h@$yApOBymCJ`N51809@Vl}|WbCrfY{y5LZNzx1U z&*G~UNTK29bf>^MPX3(F*)_pzZt!kN>}#IfY-`5e?L)`wtiDk{U^!dPQD=5iKfLGI zW~op+=pNZ;?pJ4~)FIknZ!lRu+U^070+zCNIN3W@+*lqdd*(LO9htS#6{{Q0nvb5je#E<|=y zF^>e^-+OQ(E!l+YCE?!sHIY17B0?<)wakkZ@N{#9SafdmA>anM>7iRN3eFh9VI8h2 zBF!AZF4M4X#bEb0cW>^i8f#N+o|)cL*)VJC`rbiD{LUVbdpkfrV6#S>)VB&p`pwG- zcBqvbMvTDR=_Gb^%;(nOe}3Y=d-N}7$jsB=5#OSrEINtt6_*#PF)y@50>e#QC;$Jq z_a$(WRpq_suIDaQb#L9;tGcVZtM`4GrhBGmnCa>1o{iaoVSr&B7*JqDHW8H|DjFq- zh^PdKA|behL^S-=pb{m9pid1NHO3ehL}CmvglBj>pS2q&Y&n9To{HLn! zde-lJ-*>)c1zK575gU$l84ea65jM?D#Qp*PFIqWGwu|u~&Q>s26&Rh9Fs3#T-S8?& zS)H&3SM+vkQ%GJJxBJGs2F6B)M(ECa#44~jmIz%ra!ga4i*7_u*udu~KF!bcte+vZ zom<?+=fPBHCK~3 z0h6XXol31hyY)@z7BsHC(TXK~yLNZr)G^XRC+r$Z*;{}Ys{Y+4tCr&e=jYt(A`&MI z<7CVM7DF%iq4eJXf4R@6Vo9H7m#7d96sFlcD~qxuS<~hm%AKd^PTEpUGaKY`eaasb z4vZJ#*S*oVr-4@4ESZj`Y~PAc%BTA}x_XQ)XounZ^=^{4b@t{myzLJ^IckE?!azk- zeNl}DNG<0`W1RtV;G$+KK9(F^luEwD=O9Pwk}6CQzt>GwX8a^tjVB#`n*yYv%J<8IX5MfMfU zoN1$&>AFrVC8@c$OLBAE%w}C;eoFrLqxbyGmzD`vvfhvWbp~(~7ZaMiFA+Oq_4>&Z zoAUV7-5NeUZl89FG2S;Yv2$j2W_s<)DSdCLUD}?YP+JWER@s38S8bZ-;BGG6w{kf z2aPvPw(DC^C)w{xD$v$u2YNRNu+(8gkL-04Zo<9NFdb}4a;t&6HVlxMcrjR>wo6jW z$)0YSDiO*Kw7^^?CZF*BX7eJFNk^BIOx&NLuCEz{Z|le+N=#=ggBX@rrU6B~a1@Ph zDWpuV(C2y}=eops#tNNA2w`Y zU0X^+=#Pkv17HVW1X|Ghtk1)KmI&?kzo7JnW|H6x5iy3u>n%2KMqjyjjc+fyW}EL* zyBaaJQW@V*qR-bn0j17J&AFCbp;TxCrmeLpU+%4RR3`cd`-4d6K`rLf3$ymB)aHzC zVbHz0F1lCe|M&n&Wen0t8%|3~ES7;4w>XU658|8$LkIybM%xpX2g}%-(3Fl-nxW&^ z6gorEOwEbe%|Aq`*>YRALbT;1fM}^@Gy0lz9?Xs;@^-MCzTl;N%*r*4YFp;6PITI+DmIT{l@qLk2X+iM}JEN3v-svRvsTCgvBQQ&6c* zk~(5B;E7^cNop$%qCfEEpta~juKyGldN2 z+SJ_EE77Jalv=lmX?(E~mwIP25sjnf;ZoNdnfcqvuL;JHmJ zH|?_LV_Q=ti7ySoBDCkWr+Hya^m{VHp%bLz3{J;Nu(9u0aDjx zNm1MGRdlHJE)+`>qiCf9U-zJ_f~JX?2PT_=pCW5{1^v12uJz}r^7m5fK{Vqp0lSE? zSW>g%n5>rBcv;>p`+MXdGiSPC;*T|><_p$(T{=RQxP;E}bHr6W1s{XUTK(8q<3tuD z(XVAPo@8b;kw5@CW>i7du+|IG6iw1-jF-V0aTz+dbp}a>GFOIQm1#qpE$@_+vtF(4 zthn)_=e!SRBq^DSjo?@+hPR~ry{Qx$$RWwp^P~IG{uPCoTIxS&nMn7vjkeQi=E|`a zSu%<#HIvs%9-$DLzccrkpT}k=3`DXDRi<0E?3_Dk{knmv_Mw&`ebr=ZWoFIV*&W+9 zZg%3%ZeurgtAY1Muv zuWXYI$54HnT$uTBS&I>~=2(uz&}xo%LPxl}fX=0eo46lEG37^B7(FSO%Dx(8gt$4U zTX7SgZ^a!;*05sO4vE#3YIQ8#-jb9RKeGKSjFu9H1zgb*ct1L;rCC<6(bhxqNXf5Z zV=V-_h4OW%6iZZeFQa8VFRi6h!3ZJw%Kw(hY6rBaptWf@hnwOlQ^}Od?MBupsx9eC zUT@Z0iDwMaqS7eo-GZnmtSm)5VNQn^?DM^0UWJ|okf7Kf06M#G7R{2+GVukz7XHe< zSFmrbbD-T9dzCxn!E(Bo@9G&I z$+lLS5{qNhRC6(5CVNXYuShleVLL49r}!B>F86znRQOc%IrMGxOCZ!irjhsf135@7 zK3@#iL^JBaP|2mhzVo`)&D_~hD04{`n_VW!wW7vfD$@9^*yP=j%(}i@W+cvsl zuwRXL#hs2yGG$mpJ$)m1L9{5wfxl`+MOpK}C}BP-oTuS@BBSHAXbzj3VRjOWranLM zjE6Pi^hox!(=b|B`blufzpuhZ*O2NuK@Kd>*Tkz zTTb1g>auoYRo7#Zu*Y}Ipp`#BY^rh?>UI^9Dvn32;mu4|Mmv?ObuI6tbbSP^Bv>Ys466i&o65&a<9j%~(_3vWRD%*%FV6AG@GPU;r1Hqks0oAx zDWz(w&{|T#X6i;uhEWRE+Hn|PwsD&zVI^yMM7-!mL%K1JvcAYsREW=)t`aA2V$0&7 z*|=s}8VL`8lAWC$BtrzwHQBxCn4ubqoCJAV{!lUqMduP4VrO0KlXurw2LuL4Z75YL z4oQ-wwht&cZfbzZREhW(4}_c*#4)Azt@FCZ#-)ly_Xh-@i-cRX)t znwd&s>T?uC04>d19|0x8(n-=z;z7Gz*&Q#PY-gP0siT_)H^jE-@(#~Pq^&(9&gLok zU0eG$_YG{E9UAHEPAts4xL}W!%GQf}i3y&#aE>!%`W5JUX!?D?@Ra;CJ1Oedd?!@& z(pbh$kzVLUBiJDNOk#$s;DnA8)pm4Z=^rzY-#0EWUHkmM4J6vrmFP-BwLkA78oJo$ z)W|wj>37=(R3na6GUnsHwQ>@zk~LekO+^Rk2gXR!i2K%FD0Y4k6-n6R{bXLL^BF_; z`Vn()LTjg2uiou$zx?`3uet(t*=Xzwo9% zxcbV=US(Ksc-dhKgoamjgSp7mx=NEL$!b5$bmTb#~|-Sc`1V2~Du`(Ro(a5>d^2yCVQbp6o-c@EvT&(mhHA?q}c9QrBjnST24rtNz(9= zoC1lsh>QfWVkX9tWNwknIYjFjC-$FL!15O~{eZuO4)`K9F%ftD5W5{zpm&L{>VvUWR>u(1ltiwyqWdQt{h)^8GOsad4DbMAWON=!v@)wT40IXPa`42 z-0>9mRqmCSC0?D_x_{$7oI!_@hvJ7+=kmVR%D|e@6{EwW8`f`_JM;8STdvr0@ZvKs z-E`>{m!`-{F*zSsPqb6-$EdBwVV{?SsqqF)b+u?x&3E~N$v2`T)g|wdNqNbEK z7_A)$#bYrLM+~z4mA?)Id-zOi${N&@bcWFf5l|mRN?KT}$gluB01Il)Cv=i*K|{JO z2{?%SBN>}aO<9w>(4ZDm&PQkaf6&G)XwW4-x3M-5n0BFNqFynLm;y63jlSd};+wAc z`8CzNKPq%flQrPFNjQx78kbJgaUh4bWya&0o-kwnWHj7Lx1s{&BXl&aO?k*`B?bqw z?@RMPzMV)fvD+!;)wEn{&Si2+;77f1>8-XK6{D)b#&Y_eo&I@7l5*osQ$0>}RdER6S|f{`NQgfpCyF-a|OL+ft)4EoH&W)3K) z4vkC3y64BeAmz)_NYQ?ls3xgZQp!pt5lQQD(pg@Zjr0KEtk(Opv&XbIIN>%Sf8?wufqDECJ3oQ=k zgAU@!1)lcM>Vdw}BLRrBNNOR`ehIgQjiiLK=?X{?tDrK(i$-ow_{k8hWJU&#l=MG| z^Vw!)OdEA8QrvXnvGPxeo~_dEZ=`B|+z~0;)U+9W^G{QLV96A^zJnZuh9It2*WLSG z!eD5Iu8a++h1_R%8%k?ZFO-NRnDpiBzlBNH=W7b4h{a`2YVHH#Sj-ojfDK>J23^!_ zwb}x?Y67^fT1R3gHCB?eYZfRowh4(hNu6_0=T)#cZ$MHx?hyT=neZei_zmbyUN-JI zm&Vh{q}Tjol$vR7?o<#`OpOG&$(Bm9=EOQgG3X>6P%mW7;;W!fF^kTsp!grDTOYPUP@ z2#^gX)&bP6>zUY@=l}+By_fF!NKIBsR+5$@kq8O)Wn?74NG=$0q>|*2C<|-GjAD|6 zNJ;OIos-GkCF=toN_xJ5bGk#7eJbLitt8NrNrt#HhtlQpXQ7!kl*<)`93{E$ip?9* zH?j(F^u<=Qx#Y9~Pa?@(BvVwu#WlXiCCtz@9a~S{xNT-yEuOmJ#C3Cfwr|?f zxmxEmDO}_T#`%^o{*VfAtOmlM$)N zC2Lg2Wk^oP;(pr&xjowaoyB!TQq8W;u2s{k(reO_?fQwi6P3)$sdXI_nsB}7o@CO~ z8ylGdQ4VKgASrI z$$AO@7s@B0T{6%H73&g?DZu#KkDzN%-gTTr!pX$V_>(9#+uYU>3>zfn%Eis-i6{K{ zgKdb`_XQgLcO%+C=1;6-MW^5>m8UvD$%~a4Zx-gbZInDsmMoYyJO`j-c6s2WQl9ii)$vkM- zNezU#Cf!VGrX)501bG{pS~{xA_iVnr5j_!u`cGjD6^*!| z?yr~1AUC|57#gG{DUS_VF*HY#(y3N75I0<`0DWyo$-dzxMIOry zrsErZNn-`NOtW3hk|pzC%$77mDG^^c?7+w)hBw5s-i9T~=TganW^lANSwdYj^Le?zMJTukN*;2X7x%=F(1tdZT<;$IS@YcXOZj z-h5keq@73$kR`zokMP5=tj%)a^Z`ghnOx{8bA29LHtSPG{|Gf*m?rToQZych>!wfsNIaPu9!@ z!~0;ew?AtqPfyGde3&G(3l*Gy3*mPApsq-Xfzk?VXA=7La% zD@7?&k8|14!)?;-3!YzzJB8a}3O8H3K)P`m9Un<@c+f8>9FHC8BUm3p%5pV2P%5Z# zIZ>82oMCbl;l^Y3g549>12#S_Evi(Z`}&!#E{yg2I0z8+_BEOyYb5-H1`D}GKKQ;@=R&kqu&4&Ban5z#wbLPN+{((&`f-IY)Y z?}Av5^VhCTDGqnzVas5;-X#Ac>tDN#1t^V=+6sSy6<*QfvD5PZPx%K%Y&viTu#%nbkQCG|s(LMk?qtYa5-Ol5AaNe!aN_$m)6h z2v^W9OYL{oLk@Z~^-4nFpUa5d^Fm)HTmyXXo zF=1e>!ZVGNug?#(i;qao{e~e&*p35pML*;Ik5TH>DMYII-f7tD4!7=az0TdThgw-5 z#8?ZZS(Ttv5$nScY0!k&w_bXxzf(o}RRUvJEC|M=5HRSyWl4TM&QkVEZ>0})XvbnD9i`+}?I8FY7h73oM z$O2Xw>p&qco*!dRA2o_=Qak(}O@x3mHe1O4Y4KirMn}y^H z-D{?wjOOPA5IbbRm2h56cS6o*j{=fHP+!o`ONd0i7E&||^_Wh;v32)uKA&-Sx*$)` zm!Fss9IFX2wj|rD+#2OV{9^f7JQaezRvoIHa4ljpN@-AS2Cf5sW8r9>S~t3C_9kHu zM7ww2++1LS&a8;VkNU#5w@ixxOU53EEMdkD|8Eew)+j*m#1f~2ptU!IOsg?KFp;y# zcZK5zZA}NV`mIC$@tH{O{nO2;&>>~zoS9+F@>8?z#Tx$?&g0d&(61r*fx-`hB0_$? zb`QlE;=4GM>YZx)cFgm{C0vz~GUq3k8@hn4*IA_wis zH@Gql(|+vnQQv5p@uRq}BaWZh&V))mR*jVS{B5Szt$(GIR9`&(ku*}IwCbOA8?p9# ztWLrX2^)hG<<)qLsw_$IgeL&PrII1%Vmac`-o-CE1M*|@27LZ)^e1(0($C7G(!k6T zq79E&!XV6ZxVL2Wa#0Uo=4gDY9>HFXEa_pa49vL0Yvm_Th{fjRGCZa~ZVl${q^FbZ8ndiI2j8$MsgMD`Z!%v01!J#F0-`6|?GJWO zl7bkSb29HuwAdN5FZp}G=rsvvR+byJhb@xrCQ4H>^MHjo)PK^8ZoITJe~x3WrHmU5 zsH%s4Adq8Zpr^wR{88sT=)V^YkO;o&hNsbMn4z|9Ur!TvIMtziGUbVtDpkjFLH z@61_8mgzxfPK&EN%wG0*n99YN$HjqG!W{EV@|NtMpkM|^G))!nF~C^iI!mjx5K_Vu~E%{tqf30FwLi}%$eLWHfay{fMYha z#WyA6A`cJhLVjSm}SKu%|^}6m-ZlD+$k_$x(#Id zfMpj0lB2NZqB1i$?v>?+*chZVzW9bTip(6qsHdrgfzzUY2o!d4LO*Kfm*(W|d+kl@ ztC9C7JSl-*9n5Wp(ohd2ObZ%(a}p}dX`EKXkY@g|T1m$oJ*RLigK6Ml{uJK)-nibV zwLp=8E!Nx1#HYCXd+QdPE8u3p=4gj9*ZV*8maZ#(!X<^8w)Z$YZ;#cg%* zo1c)dRnl5=<);~0@r~uKkTG1`lVO{jj)_CXw26HsIf3eW)2IVP=iW z+gKOa)H{rf#lB7AtytCHX-H95yTQy%q4ekHT_h37Q_8-%TVr&1s~QGb$a+y6x76Q( z!1CXU_$kd@l5I+oVA$-Z8-AW=j&{B<)kA8?t`#qhNbXuN;2YdxxiF;F&88}eAv6~D zy21N}@AIt7G=iL#8-F%zCsc`-?u){kc#!9)ASJ!_OTw#zOS!CJQp=tkxB2Nnq@F7t zb9^5g`o|Hxppm}ZHsLvAur1bqioE(&G(~bM(i{b;vBwq*4#FU);wAx&1*6{CCod<09xW_J%qo-+k5k}v# zBDe~phi8`m8pJA5tDe8V@XsB0QftWzXG$~|@2l2hOOzr;3jRF+V3P53Qqf%{GYG=^ z`XnwT&Q2%dMe~|_TO>Dw+y9LIwUMwi-axE)z!Y*o9y~&W*nE|a_lWpIVxlxK@WvpP zT^wM?c#hg&G3&UXSP`jbBz6MMN%Z)M0WF`BkdZt2JyJ**Z9-%I1Yk`dURZL4w&j=( zSzkb#T6`RZv@TQHToO#zkq-{5C+@kEMf!s?AMCjka~OA}hHIYxH;iEZ8D?*$OtW0@ z^$LtinEc}_TXYCpC{K_8RT-NTx7i*awsOJx_frZD&sXo?pBh{1!{8?FR`fdSDhr`= z9e$a|MmwK0^9mY3MqFKZ-DFdIb$^jXy*V=YT2c-oHMJk3y3YM7*zAx+%I?WFG1b)) z(&8Q|fppxcha9}wih~&5wz+akTA6vJ9gB#FkTa9~4nMB?=iIuA1IN5PT?cVA zy0NoE_~b?}4eAk7sb)J0{-r2ct=%h4N0$mtvS5rN0rSf+Md<=T-oxMP=M{fH!X#(&5=taW*+&OSm*?$V@QXyP zG(uNzfG=(0=<&;jsCg)#%f!jf; z?#g4&O$Nn;&9R`$27=*Tv?TMGU7OilbUbmn3GkuJk z(H4N*JD{HayV0gki0Mzm>sQ3 zT33#MW7ejy$TWHk&~g%Zrwjgie1#lH95@cX2F4!9IJlTdvL%<>6t?IX*KI54N6ovW zc*Y?%E)wTRtw1J-!a$q}rJ$N{M}O=`GizCy|3H|3B-?Rd{Y%=aO-4m4WOHd$FSq1r zdgdw7Unp#|Bgmj5qff)zH_1QOIBs&*R_49^nD;g(JJ3J`v1&l>B}aR8mCP~EcjT^^xiXd(UwY3%86viCW_wr^^bMWLB`FZJWv^^^xBJ? zNH7r1OroDBv}%#vcvZgDk$$DUI%PmyWc?StGj%EhYN+YnGp69>wF#@HM1&~SI|tS~ zvQP%71e60#<;9v{LKnQxK`tv=4@v&9xF~YHrRybGg>iM$yDW0ur}N%Y|5Ne+bk@n{ z|9hfE?OK!(xACo*k*&Ctm(TF!B&*-e4sh}e;kU#>%nvVjgvHnMGu$BMxJ~0lH~&EO z#CKwj_4sNG((NUqWXY88!^}HgtcU)-O}*&EIdXBO=)v8Jcl`d>x!zULl7@Ag)VTQS zRD>%? z8{-z9zANzCmo}~xKQ+x)?|O=lFpFUY?(wx?So2WcE?*tIN19?}mN_!;t!u@b?W~@^+ z2#Ex_@IM#_Hq`Q-+K93j@%ZeW57?m&&{O(ME}PO{bcpm1*21!*Sxbn1g#*S^jmEf( z@O5O5pVp2Es@SJ3mMFWoD(qADUzRZRrqhy1X&5$QU5|DrUIiSGtQ0zK{YWhr(PQ(O z&xyF&>IxBtsrWZRH?}T&!-v(Z$C)FKZDUmd=X}v^H=(qSW3yQrd@$Pk_*loSRUcC` zM?8TFda5ZOD37<#X|dL1TxnZ;ihB*4Z$ zfz$oL?kWd4#!hc451#Bx%+^w#iN#Bl_yz(~fAYrwr5P{Y_rBr&8qWdQ9o2k}D{VRX zuFHj671NKcH>W_$l5|(um%o0vuWR{B1ovJ^Id{jNA#FLuvLvpKnPrAizYOAj#R2{( z?~R?ZEfu9JXVXRl^h#MxnTLEW$1m6(WLaJE$LGslP#)W{)VO1P;-*r#hyEbDUMVB2 zCJO>cE!QTo8$aCK}vJ7+KX|9^YxJiE{EBnEVWzHi!<>$NwZ(A-)Qvi6^xI z4o-)Ka)}XTs}(^m(Zdy^`g=~(rZwDcKE#*rZ(bOF)NL+zOJ3`oD4vXK)YP6m13}rY zOiv`LZOdHSXYu&{!`O|qTiw2-_0!I@lVfbQ%9;Ka_^TNv-K804UPbBCSlrBl77E*F zIpT-V{0QGuqOCyjF^mU> z?47XweSV@JmPF*HHd##3X@m@MpB1UR!`A`ndoAkVHP*mAY5+YU(Uuj$Jrm#48di*a zVr8E58=;{_q8*?)lHl836v!~8jFkDcoaobx&AvK(WCrF*uOe>U9^Q@fyn&-SzM=

b{YD$L`GEgl8X=6x_@cM2BZGzsw!}x4+ z3f+UTYAmgCO3Csi$1it$ec)7FyNkTW(ha|>q3;AcHsRTX#alRgF5lFf`EaZP(ab5D zP}#U1j#)TEA-e&hi;4fKR)l)k-OaqS8e<6<9T}Hq&5y+^G+uODFwmGXuztpJR53j< zBZwTrJ9W!Z`Ssk$HqfODMX{^*K+L5{60t|M;*FcJ#iXVYK_8GbW{+~`o3mtY?Gs*D zl|JY52kK?NTwzmTP1*8sBx#~k#>KZ|{N~+Pcj&~GxP(-{WCK(0l_zWB$i;7{g&(KV z*7~q@DNsmF5UH!_x8)CU(D$aeO?5vCgFj2niBJ>+!&DI>$1!X3`)Zj10+D%(@zp}6qNv*ti&eL<I;#L6 ze?bl=RVhCCo@^C6lcWy{4h_Y%Z{_ePRY_pa&KA{fSH;sBQuYkAU+*0|L7{6@*m0=j zdkuH3W3jU-M#R0QAYbjD_cEy(fcw-A+=q00!qEd?ZtD~YRZsIC)gNN%VU9zPU)osQ*9zIm=mge~j0L0*XscxlZQYw*6{ViN$fxZ6Q?}%<#>Vb* z6mCCN4*q1pHGGXQPe2+rX8(*mK*5K%~G+`2GPLTdepcmir4U*U zjIJl|gCNn~yEn6ei>F9>@XGnZ_brwBLhWgyikqRakq1Lb)KnDBxQE-QCNVxM;_M}~ zL0+b_CO6f_9>M>%0>=H1)E=t9#R_>*mm{ZOIECrEPa(@UJ0!`g*i%$-mZPKzcBDR% zAnIk@zTkIc1}(&e-+l#B=a)!RIWmYVbKCZ^ib}#?{RWEUG#iw6LZM+Cj#JI(NN@rFVOA@E6;fba`@K82g47Km8xepsKn zH5d0V`%HyFmfJ99sGXtRy~r`fev$T-(lI_=eL{W6V?;b=L?;kmwGO=P=o!kWJ8Llm z=@G3#b?d*8q6w`iviIH#UbNzCF^95H;3AIkVqnH8XRq+({{{T?bNXHFQg$D5VVc;L zPj?+L!@D_*R=Lm`m)(h;Pl41hut>P?h#W*@qqB?x}$cy}u7^Y>q7~buS+OozjY+2i?@$#F&WDH(3YhO2hD#4O1XFtgf4# zKN$g~GB~^M#7XPxM}XS#D#C9*C8w`!Sr;OSr{+;VJF2FD#qW)qpmePcvtl55NB8>1-OG0H&OD$jdaVMP1TOyUX`h zAVV{F<(v*d5Fvh!E9t}e#@`SVFMu<$Z2a2$tqVJD<}PYD0#3c-2j`%a2I>v#-U+X* z?jIvJ2Y69I`?xsmEa6|@2D^v%?|;F^p(5Y$4j^9(wu0vxP?+VSNiPd;!wI32QVW+# zE?3$jUmoqEt*x-=DlpV4azP25!dOllbJPOJzH(1j#bawBRiRxINOO%l!wCv{G{g@> zaM*+;_z*(qLum03Er@etynQ!!Ah?aje@t^x*U*#$TD>19~AOx(b3m`~4j!qVM`e z>)Js6+D^@go)nut_^1@wp{ow+1>N6Hm%J03v{V+eQrTOE_iHsqISJsB3Zp(22d3|> zjiP8@O(r~SVO*!aftrhlSVbiK`p4{fzHq;5`;qr&-f#N&BA)n?gM^kW&HGg@3i=bUAH=XMRsM4L^GxlY7H-LgCBl z1~ZEP%z~(z5=z&Jh*yIaT2tlngKIMEi@aW|SKX{aU@iMnllZ%1Y1(J$L5I{moA1o{x(| zYS3Sg!pbbSUDbz>sE<9Yi8O9zZYRNcZe;W4{DZOgI*Tq-SCFS^pZA2Y+JRT->87So z!Su?FBfCHlHaWi)`6EZAzj))JErdVac{j@zCgWgS#1rW}^s5|0_BFwbUU^DA>9OaKMkga~)!uMbi#&pQ6pusRA&(*6A#Oj9 z9I!n+;z{pBfc&7m`~GAbj-u7N+8oF zuLf?(a}G-|NyGu6_o#+Ozwyxs4?%SnQ*STn1zr69R%h6GHH}M(HF$o#+x&r!56*?>!X= ztqfhDLj%wwQ?oY^PS}W!^jCQmV=+(ayf(F7sSJ4;z=1&_nVj0)(lY;`j+n?<_#!-( z+NPRY`p;SAsw;w!S>JXBM%QtsC+_KFT)t^J+|B{JMzrYk?QpLoJgA3T{YXf>{Ge|8 zVk9YePgb-o$^G)c)Rp>-$n?tftTQ^+kDd=8KB#>2UNsL4vw_I&9MDfoz&Q+K!P_!r zn5oIo`<0c@1))yX6X|wUA|ky0JB!p@{8VPA&H*XNVKcTC8*vS>$}+jYXANjZu)nsgwKcx4*@M z#;cw`$Fb5*0TXBsY2%YrYQ3fzMAKHGAcYm8Y0cON^RQ(O>JMw_~6k*A?R)VZq*KjFn%tc z2D&ko7`8X3h~#A(zM-sU$=-IqgM5ns7F5`9SK zNU`#7>PHbQ`ij8q$E8kae9Q32^cQAQS^x1HK`bmb_T2AJrro7}Ga*GIH4B9MdtcIH za(ipJ=sQiTEK(H0zgZ6yPI#oKLM6O}p0ik>_NepL6_3cUbMl1-=$J9oP{IV{CKxmL z>_Ygr5B~8hYw~t=#{S{;?R&CIo8o4j?GA5idm$qMb;Fc~J9x=?jl;%?V~dqQ49zahWBGt!P^ z>-yo2w#m8Rkt7AbwOY=Z?muDwNN@zv|45*5N$8FU+`rsV=&32aGIxEgF7=Ln(Cgrm zZ)$C3u)9o_r06u_46xxm)ZzrFy8HcBJ1_hm6r|(Ca z8=V%rBgrC_+ZOyNAG{i2Ai3rN>$ikOEX>~6*>JyXKs(aKGo;rN_4q*d@^`=byl`u2 zX4)o0_WV*+C(U06#U*Edl&Ii<%x_u#l}GWlT47vCsNw&4L)wXx=7P)*1wW+2 zn)t=)Z*`IlZ%0L2k0)^J?8L8VLzZNwBBSm{*VkfsDeHeacYH*rGbQ`t$4J~@CUBK+ z>V#QE|JCKMtYD4l@QC>rd(8MJ<7dm;*40_S8dwYCo+k{dih#$TYd4Of!7EyCFkD0@ z+-<^ftzk9;HZKp>edk`zKmf#d%)of2&`jLz)S;QhKpp{PuXY%Lf$c=F;fH8r53o(# z-gaszVuvs5tz=Y$h=8Bp-gg7EynSboe_%;OYTV(P?RReQ?r@dy z7Tkgd>d@6;n}4lHm%`pwJ*KRDo77TAZE)`P`Z1T)rh%kOGp?RI-lPKl#~n@K{wR>~ zuy}8Mb$!>+(x?IW?`^y<;DkA0d)LRab%z&l*;JyP1V;h80J{xg^73Y7A}R>1bGjygQDz#JyiSPD!QL~`Dm5^c z+#Uxv_{9pjEzT75>|u>M)b-%J0v+uvb9g-?)_EGUg~D`Yi#8p3a~cug)4Ch87f(;> zb+S_)&%AdKQxV{7=RApzH7&WvxmCThN)OTLafNYR;iUIu8*1=m6u8vW7S0W(*hK>^%hdy|3d_)0mq+%_06+79t&Vvo$4u035J^ft6;krS4Oo%tMe9x@~9*3S^)?U;1 zChMw2pl6fLr#P@Hesl88p89b9C5gRwADlg=1R5b0%9 z%g!WH;Acn6*>}kONnt-T^2-p;(`lH&)_`NsHpPlLN2s5PuN!>`bNHUA(*>Z40+4&g zDNx%K-m|Pm8u0AQDg*=%Wc}REFWyUjwE@lR^ftuq)fn)!dmWF0;-X5M%Nl`=WKHBhEg=$E^g-Es$mpJ02H;H0E3wu25$t>vdHYc4HcctHg_UStb9C#wt{vGg)EhY?7qkcTu@%TUV4Gi2tN|Syb0b+MPgOTpf?$BQ*X{CP~G7LxOK9T2NEs?#S zj^uSR28Mrk;Pajd_)YotI1s@q;q1D{^0XMSfQhV&yN%9jqDd?`Sfd*wLCoVx!n;n0 zOYy8uS8h4HYV~CirCVA5pHWKiQ9D$pec&HAncrtK{fmm@G_B((_pc{|T zlSoL%5mkC^NM5?QlcSNFk(gL58pw;5hd9$!A>=)birN*q7a??Y5dU!oiR8^Jh;;FO zVZ@_PO7VIIwzvw^gfP&GsdJnMMA0O=^~VjfI*apAqkgkKXeQ9WXLj|yItVH&jT6=g z=wPjE+XL+m-l3A(ULgb8U*k|q9DhBVyl)_~-kAkzUmb+C-5ClkgYMb<376frugXDB8Zj!|Y?$sEs z<|r(1{(dZ$^%!E_x$f-kK|aM|@U7K>d@9u4tY=~eHrjbH9(VT~5FqS4^@@UeWbnB* z@ar`-?#>a_Y5OFNo2m7tI2tj)fD|@Y`*(B;SGXoo)G$4>@MCMy1J6z~Sek+6fO#6z zaDGI*kio;fSZ8|$tKQ9+X$XB!cu|F^iJt4@?9CT+hIi}Zhf^ocl%eNQ;?Bb@u#Yf` zQ!n5MI7BM&yNECc0T2r*9mnm5AnVvI+^c&D4^Z4fzFs6{9$fRsi)~S+I3d=AZhb!5 zX)U&7-TbvCk9kGF!`FJTIRNMo?!apiZsq7a3Ig-=J-fc{&fkmXrzi3!-j454AJE5T z*z<%Z^*X)2p)qA9_Pt&L;F7l2h5H2JK3^jPNS=gze6LGUrT4m{<4Hg4KILJblsV$! z&-K3+3Tff47;^?w{Ba%v&~o1ks1?@hV|gbiZU_|p3h{P;YUuC9GQ2$pF7&()>@$2X z#zVcZLdqpEzLr7G<;9AfhH$)FvW_wg-P`HF-E$>F4EXo}Gvv`4pr9|j;a24It#c&r zZsaA1k7F3#id(ui-=_L-(1-z8K)k(Y4tc!=6Lh;giTOSNfS}Bq*Pt55uOAj;o3N&4 zhtvbg%Y=(yOMstf>y(%6ZEuxgpv94o3pAAtfZclk*xB*HN&5J;bcNe3Jj3&0PTh+d zIQ(&DFF$}PjJ*wLh`veofp~N4bQW-@Ps*t#B7Gw?xpqSq{xl8@=pmNvX+@Cz1i!eQ zSONcVy_&rqpEP+}Y6hD|VH^&~?ZTV(qwB3%4+G^^A>O>T$-D>y;Fv{p-?n^G!#|gt z@2?5>T7GZAkr%!tit~XwAlsM&Z@DNd_GA|6!0*phdr{+1ksR@*ArC_UsLlRvXg-I< z(S!N`l&PO^-8A7o<5$#4E#xa5jp6gih>8AqqZ#oL6t7muZ4qg4VupdchZNE$GeGgV zDgf*xHq6$`MSiCzxsFcadK8%Qg7jL0bogT;0iT04QTA(^s^n6$Y4oLhJc+lXfcLR( zTa5YJW#K0-vmuf|?f*6H*WM)6Ip$iB67>{m7zNW3%)xNjSWN8S!TWK?$laaN_(#vz zKH??eQy(7VM-S*}|9JOP2w7pTv>12m)|&Di_zC-?Ew&TE+`-J%&Bff<{=Xzg6B`6} zHWn5#7P9{mIe0n#Tg%GJ`yZ`<02#A_xr3#f71@8<{y<=svbJ+GcOhezvNLuwmoPVV zG&3g?5<>XTr@dBF)C3(dMbZ5DwFmfk`w1{;>%Si|NV@P*6w9=6(S8S4Rmp^ru#Ajy z;Kd)6V2Xo?yiNH2ox&en%;C7dXWNpcp*z_R8vQ(5ql2IHb@T7aZ(D$p4?mP@*M!tk zXvVwx$6X&!W)1EhX6Wc!Nly{tx_&CzC(*G@Y6Lc#^GGT4q;{vM;C;i?8Ts;${ST{~$d${+U+SBByFH)e(4)=MPdjdA+JUkpvae{$JWgb0-ag7uuEXLbNS zYOR++vKBP8-IIrzV2m5&D$3rH=MP#QAn|ZbfW~|)_FmG$cGvN4xnlpG7~Yvve;X4{ zaD7O&5~AcFPoPw4%>}$2-d3#KSm0oC{wigiU#Nn4-~z{tssrlEFgDUiPz#{{MAWtv z(zc_9LpF^tDhHHgR$?an2`zvi`EyYsYp%}l=mT-S&R5esSYih{LNAiMlZYkh^VWU5 zuzoXYAM#a|7e7YIj_~)c0W|Ukgin(vpT!3hrlY3yCOZ9C;9a!2P%rCh-ljh;xoG{D zJ6ccQ#1*Nm$)2mdd`##*N2-&uit`zO|M?)8n*ISk&>)}tzxovGf4qmCjg{?xI2Gsr z>Qw*L`2WMH!ZLKD_1nGk zGR)u<2z0KAWs85vmeX1HWD8f4<_Ps{Zhl-GobAOwEv*5jl}QaQx4QiUesym7w+Fnu zxw(Ek?%eI2NivZ3bOv;H{j%)}cza$udml$tChgfIA=&)cpzG=Ce)D}D@Ce}h@bYr|`~*Ee``?*) z#=blZyszE80$#^oOO>D228;u)NY>-tH!eQzcUl%4*PZ{ZDU*_ZK7SlcE@Tx4(SK}& zDEEj^dWlVf;X>0GL%e;zBH1ky@}~~Wd|rXquLaMUeT#xxU_iK*p8*q{RFdX_w!`_G zhXG^knw15Osq#;$cYjyAAuwc-b|h4-SM)Eg3Q!8z_e;bR0^wLy#NC_Ri;k~7aCQPC zG?~mg=u`ojz^a^T!vrjAmeIxKwR~s48EQEcoI_nEz28G!1%8&S4`yu~=7dt_3<;dy zDr+M1pL6Z1b19h&lw!V$n+*hl42BAAo}JhLmPbkfGS_k%x-N~(8O+_~xI;(?(}EMl zg<3NSmfzUVWvr9dMgvSVs%-(d+3ov4O2Tu5^vcKxYx}?4a5&`fTtQMXTJ|AS$6uu& zaS*o)F%WDEkyoB4warB6mvrAZw?68Bs9EB42D5x_^s6RqK4NGR z!BB2i`nwC27@c|OQQMR=Fuzn9AC+=gOn|dTu^Big8m+W87K-`8|2W~u+PQ4>1r*eu z$a|lyb=r1m^5JlTnwh`}IP3!y5c5RQ5s;d}?yLXlgN_<|0c$!o zSrZm1FtlL=-#`9@IZGui(v+}ZagGlw066FL;V`-gjS*O;tn@0!Ou2M$CHiUe0xzM{ z`x62x1T0KBz!&i4dxcC9jez9Sn%0|j3jnncMRz(zK64O(fT*7p@Rtlh{E8cgNvw*c zO~e(Tg>^G#1o|YwaDiz7weS2 zR}ANKR=@@aaA>!MnS4E3(YweQ=w&9|6V1D95qARp$y8*5r!^1RULh9{WB;}0wqqja zY!&OrIfFGkp|vK>awzC^v);U%f`V&5)%r;-P`@dKqDgKeHIp;AKs&@|43IDS3UO@2 zVTQ6Y3k4D9+8mn5a@!gB_(( zEMXCnkSn4%GtgRk16OKrOtScuLbN5lRg1zD+eA|_2u5`-+|+wS7HBt*kZ?r!n+;|b zS(TcG%C!*v4+fH1_80S<$e&TTC@_a2j`*6RuYaO19EhQVTJW!Ux!hSiji1KI_+OC9xojCOaAs>^R2A6ShkeC7&IyEvRC%PXn zC^{{vSpm-K9Ll6HSsHuoA(>7{rz`0fg+CU3e0?4P3Tr zfkTX+{hy zO$dg#SMZdmzA*`ZFni#cmd^qvP6awa11|uHPjuD@SipXy7sPR4a8#Qj-}OR9+T`kt zmSC7R=2Iw2B?GDVOK#8FjU=dtW0N(@vvNm@t(V7Ml^Zocg+NUO^50zXUuh2dE*XMq zxMJl6v*oH9>cBvMIk09IQ6w-U`m{cYVRCBl60p6P@8H>-F%~Gmu{=2fB~$!+lPT8{ z(-|@n*jfd~NC0;AgedrLS(tTS7?#~#+eE~yla(^vZMrN4Wc}Z9wNOEM7L9#v5a6oA zBAoE@E;*7;7+-jD8%hN`p$v?N3Xn7{O+w)QrRRgsP}L`kY%o(;uOuwm154ikrIGbD zqvYas=i2SYC&)=gXjW&xDaY$APjPNE)QGPTnM?q>{S%_-%*aX$+uizIIMFSqr^QN* zX1#$QlT^#?-r5|$9)Xub8Q%Xy+o@Uuqdf6LVgRi=dsunO3#{Yw?YCmb2jBArDHI4U`B`N0R$_%5y>Yw6Vx?j$ z7`G*Y;!#Krvns{-K-law!p??hq(H?e zS^dB&1qQMY*@pG~;UH>Qe9k(ib{SrLCfcRmeA7u2&+yu5;-@Q~#GF(DM>!B*^SyK@ zj2LZRdvknB?PtDJ%(W(ogJfpZSPg%SU{XS9(W6(?YONkbp&1_Q&AsP@Ch3>+_n0y7 z+Z*~43VEuh8Q!dKX#5DS&-RtyUh;jwFJrzx^NQgH2UcXI*cEF|7%k!X;_-l3MIC#2 z7&UHFbQ}rMr`a`bv2|-?(La{ux?|2bDGo<-L7|nUS--Ky#lKsKuX@nJBX!Ya@`Uq$aN;K z+L5^yGlYx#%E?#!-l1i!GPD%1W^pe9-5#u z(f*8lh6q6+W$DpFUX8UQ?;hHl?=Lu?<(L>zRjZ5R=BtA+cMXP#3AA}HM^gV$Jl(0D z@Wd6T$k>YY++N4#rjwWC*`cDWkO$SiDe@WyQZ%`+JFAI>HQ1V)CzY(jhBp(P8+nvu zK}XwAVG>EkLws6M?OACaOSsh~p=+I(20h|P;q#`9Upr69g(I3FGqlq<(T~m^K5p*(+xA#@IyNF5Tcbu#5;^GMBn$@P!J?)lobzb4~ zgX2VKNOSFOh@Y2Gd*inZ4K9^y-j*kndqJSr=aY}Kr{klVY0ymcC-)}|Vs3h8swkx#y_I!<`db0jPq?3yMde~}sji;WvGK~&6J1-Gc^~P(ocGJ@qYU;M z*Qu|5TXjU-d`W}9!-Oj_@7fuKQ2HClvxpdEabF;HcLr9(>|iF zg`lt7p8U6;X)a%Z!lEdgzYPOU`uC2eK0&$h-G*f#h(Tc&aDjrWr!APP+_0w&+#K)6 zHp|Zo%cY%n^-C?Wf|E+|gE%$3Y_UJ&w3c7K3Cr>7SQ%2`K++_!C=2)v!Bu3lSG zC<%Q5;*=ftVDt7j_bU-o)%rRa`uy$neEi%?!rhMxbCyfp?GOuwVOpbS>HSyCLcC!- zrGc*Xs3i51dQ&1K2}5)$=66-iJS~q4<7t-6;)-0phey>C2!eUtzp`HXZ|KoyI8pid zgt`CvOiWLHS-pZj!PE34h}|+ClG?Qb_SPdx-Sge&bZcR0{?BT@M}^>w1?<8u@OjwG zw7bsIvOWDWeTCn3V50eb4aK6vd!@hoB4nch)G%z3dQ5_3w;~Mv$IqBa@V@GA3h6zpZ*Ev21=BsF0dhe zGhggM<$Q^&Kef1C_HbL3iYRO^V?Lv>`P|%edouETlhNxK5<_;8NQ3P{p7*JPp7047 zeKWM9@!_g5w7aYAyLB;8^@_gx%FfYOX-Ien5L#QB@kP1$Nluonid;n6@VA?;HqQ;4 zq>{{Ymx-fqZ&X<};gWqUb1!hKD?j)%*1c2da~I+q3w2|D6w3w_R3t~*9j(i71t5Yb zH$2MgS-YN~A0T2F5HId?PuETZ+oiLAmd?i&He~Aq3KvHyBJM#`n5Tnpkshv8`eju_ z-Mp%=`HDEO<0`Pd?P;3-a2+*_*=i1rz#KzHH;XpxL=Zpk_#U}T?FLsWPmCziMmM#a z=t*)(S}=X2bsE?^@+u?>o6#kkq;M9_)^y(CR5VVT83dt(OIK%x)lL&;wF?UQ8h6bY zqcrC!PT41?ikcaH&RJ6?&DdjqDk_vBp0c{Tvgan87cF8^sEIUWw^YoNtn(&49@@Xm zXs!N=&Q!euMlLj?FU{S2j|$kU-y!dcIi0m=D+^2B?ugSWq z#HKxCC}T5E-qEOT;Vx|w$G?lzwBxADxR%EV_39^cdB-02fZY?z{H2Jm_mSS~lnmg3 z@b3kjFGW}DS<}S%8tbC4N&{c!7HhL>U5*s+t-0>PcIMLN@5`@;f8O!l2KKKn_k)j- z?bhiQ9bGJ?hr>E^$!d)J?*IIb$v}(VK5MTIzyGSv6S~0=-d!Eezl_(Ayry`zT)rjB zaSL|egvf8v`!5Bw`cn~vwP}?(s}Ep3fBMF0sABeiLiSU*$F4o^h67{+)sGjI%S&Da zVhWkd6y8xQ(YL@`)nm&6>bPWT(|uX5RR4#w_YP_*iuOiDnn)86kQP7$M5Ol;iYP^z zih^`8Na(!;LbZS>z1M^$MLL9Dg0#>>?+`H3OGxPb@x8fk-hJ=hZ|2VT&zUo4uRUvK z?X}n5`>fyEzlqo2CE#_y1N*?cqpi2`7QwHN$?sjg_>oA>B0(a_udYe)SA6B))4qzQ4^Qz=SQ1@`wHa2 zCD-^`9rsnPO2<5|E^D4sr2$TQLUj_rdbY{%YL$368-B|yOQ&aT@YXSLGsB1bONPM( zM7y`@sh34VP`4H}7F~$yf8L6`ruiF`@>N-95l27D?6yc#@%0KtbVQs%QqO@`y0u3~ zg&}w4X-nS&(l!w-k+fxgUsD>u-fGuPnR25-2cbXSlz=YYb){Ppy5+-@>5eow$-sJZ zw!PMYa&|Xy>;pbbs~+?(TLhCHpl>BSPhE)Y zV#1<9XU)Q>TL6``#m^f|bny5)$Dnzh6aR>DJ5ns5Lwr}g+1+oOXgcIX2sF#sc^w#$ zPJ6X~OTLsK*2SOBaWq0E?ObSbRioO8Qn)9Q@zPy3%$2S^CvVW{gJ) zO1fvi)ac9M;kD`X;~=^@cOv*Hl+w+SXJ>Z1;mbXqL3aqiZsa+H2BUHB$MG7S($}m9 zRBS`4mkAwI?TXJhtzd2J&%~4%?ciaqWhbIhGd@=E0J0_3ygzOi0fJHeU{@Ll(;6aR zTA#V%h9SH)11&XIiCdNBYsmV(Z%4a77Y!Qvhyir%**z6ReY$8fM% z9!G-JvC2r_^QCvwtc3rPE$4mj&FnUo(i)0n32{4!t{4)StosJpZM#^1Z`oGw!^J`M zxB7bkGNs(6Vwcz-z12!-C;QV^=i?%FeX4U!D^H8rjrE1U-&Y%K3QqR#aJL`&;Z>8X zSxrUM3puZ61>|nd&Y;H3F0-GWg0?=lI=rpYBOaDf`~}~Eu_XD*wBp*&95pVOL@#vn zlouj>2jWk?Wd36QNLJsVEtX!F%#e?`h@?)@erQ~%U!4!RQgHRB(u6O5!M&ITe(w)- zoMIC#d4J>QzA{m{EhiQ{ulO&c)J80Swwg2>d>3;}=&XL~^z1LQ(+91TKbW5>W#0lI z84<}eQFA%X_vb1!gHjUUF_6#;CO6P$107dqEL5jGZ5l@LH!<#n;_n0(j{1$rj{@G9 zN>{yHvLidS3&`x(Q)YxzUerb1E!_*d6#NPPLbA4jyM9zMDY4frrq3tjZ7aT0Of669 z*?UG-$6qvlly>Gjp!cx+yxs2ju;P;Xyr2jAf!*<7M%D-93;te&<@di_8qR(QWT%&7 zjHRdWYbk@Nl2><>9Ei#z1`H(8Uk4cl(yxEi8}w{NR7(PN#;oyDEK^+>6OUy1pP7G5 zlF96UlO#>IZbz~E?fMQy`tt`G7EYh{-kN^~!&Z%Y{TU_vv@V+p)zVMFk7x%sb*5O0 zI7j8GgURgDX*%>7IdVd@%J9 z&Df-tw+7f(S4i8HGxavTAmp5dF=@~7IcfB7#??E;`J|uLOq)H3N0`3}{MqyAK2EqJ z{O_>)xocmGK%tNSSFY)g%{;{H8b3Hkf=Quqcf+o^|;T8_5! z*FPsp2RFO-aiZq;3&MZ4(pw18MuiW9O&&F`*j6XSxM=s}^5!jXJjLsOJrK(AaEQkw z6+E?=;>+dW=w~k|6SXmgNk@bOa+hSfMd9t`+N&+XzMT9x^SCH9F@2sRtYE)EV7d6T z@gr$a8oZN%A8^{y)vqo4reXG8Yj>B1L5s?p_Vjb>min)H!ZVLPDN3vqnfbRq4|{GK zmV#Flb0FTz`pJtJyNw?3Vihn(iHC&MJ#Srdii;J0)X)`2pwA9W!bj5&mWHkA_dwsM zD(Qc1&E}{QU#Wc4t%mxF`78eb#vhqnu@|viGz0BBu@yQ9SmYz~g*;)fN2N$az$I_v zhWcx08c%KQN!ZP*Jk4X?Z9X#->`69^S;&)n!dK-o&|OYvjh1d@ZdU!WvHdY@e(a%L zp^`Aqa#CdP=W;uGUv6GEq4*6DUq9;kQN?rRUQ`O*Y6)37+Kgb|zMd3!Oc4s_4X~6* zeINJQZ2!Y{Oh|-AZFWHZbVtg^=I7aKw)7?=N)ylJ^0Wx7JWzF^hYB|s%9wi=We;*e zx34??$e*b6`Q+tgyk81<9GU-KIf@ieykx4*VmY|~3-o)7MPF<*?CLpOBC)3aO1kT}_@ zg|rT`{Gaia&^E)}7W3ZAXpjV2+JAX+fuuW0-%9-%wh;f6Sm7UgP2z~hV9g?};vUs!?Bb|OIF_`x*c~6;DUtr85oWbg(`gmPH*=y@`z}9k@krkF zsODS0@CYiME!rQf=%%X8!%j=$ipmhckN?K@qyDP5V@hPAf4}Z|Z zT^^v9bKa#6IM#Am&eC(JnzfrIj@11qaTLrKWXsLB9!QGZ5(m1c4)w_&$wyWcxUlcQ zWIe0HL9m-45yq=e{(+x4J~!Jc`Ly522dzjA<9Cm&{@Pno^Z^@3QFSNT*Fe4a>qZ<% zb!rpKAvA_^PpkbFm`FcY9#wGdu8=-35?x4Lm8V@B<(thtUD?!)bl53;$j}|7l&^gd z+t|)-;3bS^mlk(>J4o|{uW^^|Bfpt<_aEoiV80f>f2D1YM--Z4+w52=y%mxM7?Wo9 z3;<24M{_k5%mZGHx&I zp3$#nJdu*zs zKY9C>dC1Jue27}t*qC3i^5&n1!8Vm8M)euOG9h6Z;Efmwl1kUB#pJS%?Swg+`2aVC z=g6(9h5WT!^wLL>Sso7zU9g)*CezWvb77_8M~+%S4_7FNnIdC$j6b3{9GrZ$RadP36@t_nJXv#)5gQ4urj zQqLfXuR}0BDP_VSzYMyNXdTj~&ewm)X8f$1tM+q0opZiBYIGv?^(_X&!)7F;U;!rB z%1Uuw@j52Q;7952c&6`kMFo(W#&RAGTAK7Ag{M(XM@a~f(j>r?U8pBwHdjN3!y#g8 zyW}}>3uzN0yu@O1Ds)66t?*2aV09W;{S9HGxj6ZDoHyMTlId<1f?RlF{xKW3Ck4(J zjN}=qIo5h_!`z44TK64F^|<_MUDvgi{SYHxUA^XaUmnL?C2KCiIOYStv`=^>bA;#oC%5@D&1?52uG*fKdEQ=h;`y<4);9jPTDpy zMO|gQRmRGemjRESRH9&GZ>MReULQ7b_w8;B!vY+B7w=^@j68#$6L40|9PNOZ@Uq1Y znbl#1`~eFAw+uQ_UhndE$CWh&=C%vaJUGEaApAq_AS%^r%l(FYTlWzL{NbB~R(M+w zU+lROuRV{;10{dUY5DDQhHPgRhur6~M#*;+YY=s5vP0{?p3^2*xEmPiGjqreMuKrs zQxYD->6@1ls(dlKpO?-BS@z?HRJv(^IuEA_C7UYlA-H7<=mSL{P?w*B@sO+q^(QY*Ca9sLzs{TZc5<4Ti z9w&c>v{7V$Gt;9|Ut%kNx($LSv+Km$zh(`suT20jxx31l9 zvt455wvdo%T`6{+JQFPc7?d!BzpqF?k!>6Hbh&C~B4SM|b-RRgF?Nw3{LRy(f->oX z^>btd)6hrOD)RMFull2qU(i8+0^ECB#_4d|A`z+?+K}$?fCH4+Qf{?*!wTy8KFzip zo4By@kpuL*Ptiu3AaNV%pxx)aW94cwUq{PK+Y6?F29-8MwQha(>xHQrp9-7D>nj&< zRQ(b4nM>O6yPLx8=d?8gN8hpF_@gHdYOcpm$1}-Fert$=Pfh8BIRn8|=EwfLPjPh7 zi+qej9B<@+3>RkTBK|0CaO_32PJQ-u`(f1gHd0T?8nFLq4lBz8?OuGLxXUXxfUt$TZW}G(!b^5dyc<7SBzvPugih+dUCi;? zoyyqG@&wZvwF;wzP17I=-#ut z+i2WBY1q5VPv@Ci`#kEP_jDY&QR$*Za^9bhZ7%jaYX4+v=#!zH<=y6`8Ww*?oVSiq zPaE^ct&O(hmtRfw>j7*Mu@{jq=!Lgi%ruGZS6 zy26qFn`}#i)QH9G&Z+x;%wgL3UU*C;$R$??nLA_rL?}Z$f(7%hsXt#}0F;!u{Nyjq z@KHI3`DM2?2hvEP{ow2AvH!jc;3bCiABOU2D=Iap&X9YDQhw!R1+!unDUAfB1gVK# zE4f?+ATLZ|1w>aQOE*Yu@+I`%rD=)_KA^oIXAmckRGz)CK=Z8o5C!cwBSTMxt>liE zT&{LTKzB}$g-+`NZt%+=K|WB=9U0l;;Idm?#|8?T`+wQgUT_Kxjlfg|dyxUrTRMJQcjYvbielQ=r2z z=B=n{?kf&)9|+DKyLuj){yrZrKOsZ3SJ{YRE)f!J;K&)OtZ<#-n)1_R5%qtYT$BBeS3`jB+9*U zSM)`5`BL(?PWKGGA7o9b`(|u!^R_dRJ7rCWjEXxV{N(PEvxHA9h?BCLj+uUl@bCV2 z|Kc`n@}m&j$kz8;%du3Mk&cUi99+27=OI~ZP#6;sxDeAmQ7ed9DH7{)r( zw@Jw0s<9E_Erl?VJC=?XBCgHrGg_hrOEpY>@sG?dT}BNxJ`y!PB7Rih(evM#S4{Fh z40(e(miT|+qT*uz)xct0Vlpxk(*HSm`hWNS9~gY6g(AU5!;A04!shCFE;*>bbh5;} zrtsyv-*4U(<=wD*@)jW~?ojl8P#J#Oog}2vGhOdw6~QdpUN44dD=}0vwNL*Hm_@Le z=2H*4af%2@z?^AUr=9B3E2E=ia2+O< zJ3kmidPL7(o7~q8SSvhejS{5EZ)2C)2+;SnDdt4@&;1b z7(9my`cLOuX&K+ z30yy_9wYZC#S{+{BCBI$4p@WG8qj2EjBDV2zeUZd|H0>`kIpB&LnTO;&uMrP@2Cj)5fOuc4hdj zx&vJlDH|RXg9WA-u*SVzIewGwEkh7#_t(W>!Kp|*dry}NFw7k&3CnF+!_NF!4p>7$ zu}4!!negq%2#F?{MaAhxUx@eCi~>%n)_DUKu(o!jzAc6H**8{;+fTV5W4Pl0PP+N@ju+^x`H_!&=w1=7dE^QlrT?jp2B`+Ue{{=5isNexz#s!`Sb>LYBU8B|TC{UaJ3o%peP$t( z$2oy-`365W)ho^3b6RL|I*wXU@Ug5#_!NnYiRfwNjKZBz#*L11pA_o0XRE;Hi#wmH z&)=H79TK)VLTU&w_indl)3!=A0@~i&rO+VHJT&SM11zliKv+?}V*4V2fZ$%E;g1*-- zet!1emh;eMThED;VM?BULBAE3)}6WQGs~Om8N6k#SN{O^C+;eWJo20~@MbdO$AkPg ziz}Cq_`3;H8J-`jyLNT&5eP>#>+<~ zb!L*e7N2#zdj(*)wi2onK+_Gn5L@Y{PKH>P?j|&*hANcd>SC@KRa}zSiu0nJnH4QB znYkl6aumx+O#k^6f2cDHN)I=hul^fF+LYkEb(l#-NOzF)Pnle!{?pl@CQ9!Z>?)u} zaHSsC);fFxy^xD=kooRq0!R)dm9!+%3Mx)&;Nzk;YXZrH)h>BxN3w0a1yUbxbecLe zm+eF#u9#~UA`mn*Uld9aMTfpcR;Ieg?p{7cP!hBFAwFFszMjIY;1y=QvDVaHX_(`7 zWTV-3)B5VLAuLLPG(hP~prx-FgVLltr#5tkyMFON?hhrY0_oJ5c?BKwjFr|yp8z(|gUqJzmCQsDgE zS(KZP{hUh;a(mouuoGDBrsrFscG2GternB&6b7-r+KI`I`>#Ln)bY#-; z4y;ET=;&MU_k4koyJ zc!5h!q&+dfzBS1`X$@3JzRWPhL1_Pey+W`OaZE_)J@S5`u{IgO2L|+aWQafMTSahu zPQQ&=!7J_83oDcM3;@^DV=twm1$ zhGm%I6d<2#9#PA! zQBE>TLWsJ1LRDw?k96#1{B0QXFdVdAOm-=25_|mPY?IWtqvs13?ol7u%v}D|Dhs54 z+dW8~@VGH5YIpWXPHT&|51UM)CoSbvF6v1iadW>H$SJ@1A||RRos_na3i{mlLMxGo zNvidWyl{G@XJ+4Ru$QB8xgGnvsf0^C-D|yHe;%Buks*98W+!$&FN(BzticNvLj5vB zeV|YJKlR0nuV5%581!s*p!1i*#|)WZ7m@|yRYvT#8nj-NNx(8CoF_dQ&^C;ANJ8d$o+RPz@(&T4p0>ncaJo5uC?iv*A zHq?e2mtu5Pq8ak3)1~v4YXzeUsjU9}BpsYP9G0<>imL1rF-;6Ie5>s3T&?F4w4@ZtWBSNlXK}>Tx_}nO3w;tl0%ya551nBmeY0W?z))(qGF&ec46U(Zy2tajW4CIhn~?ruc1{QYYs$!gs$e#sKLk!-HJF0b79!)*UZ( zgaLeWlhiK+G#?cBF#JY!8np5N&KleX21HCcioLGDke*|~+9E}I0ZyMODKiPG(j$s9 zw0Inth2PV;&S*{X{C=3{B0_(I;Q8v3`d>Mvewd*I`u15+^zv+uUB~g4GJL|OIJ;%# zF<-`6`qugDBpOL-kNby`h;BSI}g0Y;3lK$J*sU!bXeK@=5ZT2_2i~?b!c#}JmUc+v{f2j!w6+#B^}e5n zH0qvRdDH96V+0|Io1&J59%Eljy46f9QHa5vuv&bblwlO2Oxgheo^YAOb2=HMPnf?| z4pTq`xG$I$P3Jp#Gb4V6kNRbvm5#|WNe*QbS_Fo{RBv7W&N}BUGrx7Qv}I3rXgb%J z<--)&`9H^Dig!-c{E*s*OPu59$F_jQ0;lVaHco&cz`&MWq7Pm+RttSuP8r}`P+NNV zlG0I)f5HX6*+E;V@Ty)KTH4)UfKkYrE_U6z9i=gOS0i1ni?0h%I^}x@d-eqz#30o1 z+^&)6Bvy^O@YaLC+%vh>2JQak12t#rPsNQ>cJM)|Z9WKcvR3j{V?)E0$elonK6mvk zt2#o0c^anW?BcGji!x%4Y<7nPvg5aO*px*r(?;HXcZ$=jMb=Mb{oKgb^!?}NMF-Ee z*~KXV8!1lBF>Ej}Sc{xVa{JYWT=@a3eLCwOR=o5tBg!pP=g7iIZ`agO53RXH;TCVS zE)Z#mmHmCO^R;&8OIo*}6k1ze2|x~OFyD9Av-7K7INnR8TZ3cL`OxeyYl8amxiZ<{p(0o!BN5y7^Gq>wtAVpcosS`>qH#!RnJ6 z$lJMMt2o-)q~B3CRqq}tIB_hogVr`-)+qLm&*3jPjH=D_rDW*|YRr&59R)XmtuiL); zl9^PSHmVEeXG83Wu6{%$$FMBlFrPU~=#SQXv16HevdL$^LFPqv`NNnnQ5u<7p>j)j z;uwyMi={Hb^*J6}1&?uB2UoVX=A(VG&8n}>Or3DDVI}r**DXl61=<$sYWGOAbzr>( zODZ?8WiAh@7gp%5W14+SGbJ#bpCu|c48C*8uvW%$sFr+#66ZyHLRFtF!HLlx&5l_I zfM;^zw(&))JjZPtLfx|#W=+nD{@I@^AAXT*_=&2?_ANxybppwhHOh$wNHbcC-UR&< zQhV?v^cKSu+gn42cdZHqge`enY3*(^MIBP?p` z@ZJf5WYd+XPkL`%_GM``bJ0_IqQE!p?|~#uMRbEitbdbGpaedi?;%qay5jZw8`~1h~GZVMKH?*`LuS5Up71G3( zAAHBSn-<#+#oYsDK7|*VmWtK&0;j^m@>y1phhGN^2N*}HxR|&4dOInA8uGvPo@Spb z0R_3D%`g;sGHN^YD+Kn%@}kx^tS%J;vd*89Q!GDPq=Y_3@h02)P^|P$Ss45f{NP`{7GZZ4X+Db9sE{6!}?bw1C`N-UTOEW&%f-_C>ZBqKFD^m2KXjyN&NCZ0y+&H zM{E^LjnF_jPM?0T2$syi4<~F@(<ldJ41{*qGs&(nwr zxt=?u1FCE>-!daIH>c$(6~`l&De#AepHj&@lS>3SMw;s$Dy^LDk|q9nIb75z=Ec$3 zP|OYa&C%X*J30bKYY zm`IKUCC2Q`F$I zWnj~pQy$@|3NYGkph)tHG^Kwk%Y#MLj3-8;0vD0(G894sro!TBAU-;0D`}Hk#WXEB zV?_y`(N^SK%n>2#dKN6tE*S1ekXVGf>2U$am!pa@rJhIATT4($1lRkYjZySKq`n#F zRn!pfln_s1m(GQEj03<-o75Y_Z)V6~;ggRK`r?eWFl6=-d zj5lah7i3PQcqp4yJFH@X0?6029lu(W*~lUv3wZ7XR&nwrv6EF2<6B@m&gR~L-w1q6 z|8th{s>yzN=KJhN%4S=ts`lWso|H4XZeXyOJ4pboFKwAjjwQB;VV9v*T%ug#>|IQ< z73zFQ`S^xeZ}h<{s54hNv8{&MZ{`vV_4BaM$)~8daJ4jYo4k0IxQuA=w_1de@2kWt zh^f1)4=F{_g4EEafN+IH4 zb#IaP1yaLNQy-tig*~yqZZ?$rT2WscPT0Bi4;MPw!+OVH?5yq9;^2qCkV}Ox6|(vz zOB`lfE?WApVuU%@#wZfxzIYoCvzKK1|MjorHaDd{DCfR(q&5%Ia{siG`ZG!L*vl>= zM2~dv(g9My#Sy6Pd9Xo<-EGj|w~Q0&NU=2x<}CZQ8dLDY@sW76MQv8i7rM^G*D#Iy zfKzlZAquCfY1w`@DmGG*Uz5*1pW*qQx3tty66d4|H}yLxmmBw0-#?FAt&_!Zsv=9Z zm>hyRqqBw010u3$dGw3Non3Ssemc__sRp<_^aD#K;Z3|LJa*~u8FcvayOa4GLH$LM zlL>J8zMmeQ-1sW)_57DrD3L522#EF5S5vfJsw65>Z1&ak>X6QDRx*Z6>dNU;kJN8w zfmE@LE+r}phLhRF4w}kspqFY6$x4lH^U$3~KRr%~@JmGanjZ(!vKksK*6MzlKQ^ID z*=p(_do>5JRLw}7bw-k-%#(G_cN&Wlti;;13*tF;dON=!o)?5d`ZK_#Nbbchtj-H@ z6ZOXaA14ZP28l6YUt!6B*{RNJ@v6C^aGdPrj&qDLxg?UM#0nSe%|(juV>(C*CzN;Q z?!UJ{Y~_c#dcyLxRKCa3MmId*gY5+a$E zljds58BFMxsVx=ji6z!=!TP<_tu?FPjqs_u5A`$A z@YtgN`!ra%Sx@|{_t${m*@v=bV7uToNrXkT^f%jrafstBw@8PONea1E3WU`77iqw` zxuaUChEkCVP9j^5r=LN}oOzs%sK^c#70V>{WF~izoq_(C#Cs4+d*>*d5vrgk;dw9o`5ZqvvxzLb4Q8cxH5-4 zJe4LND^WCia73^-9W{RB#)pYoOvk2sjuxaM!l}2nJ}JG82MA1EK2XboQf@5!$sUO% zFFJ1$3pIR0S!5p~c1CldP6E51SGjm+#K$?rgI2USrzT%(B!vLqJUfC2ZL6WYLcdP( zy6USxJHP!hT3B+ULU(YgqFhMPJJa@f;#CGkDO`^W#~8WHGLtX$sykx)i+c{~EOXKa zjl>vOZq}Kg3Q8`kh4{Tqf(YB_;G;GTGK3}3{`*mOnF7S!QFIoy%L}Zdx^#ow44Sv? zoK@e~7v32R5!tkMFVq;oJlz=gm!6lK>E#;C(ACrSZwPhp_E^f4+aR3haSjyGCL@Bv zceo7FkVzbpe_9?=j7V{FFwaW;le9$JweW>C9~KQh0yM7XK{?}u^mPv>-YEClDt-5x zjIf0TQv;%#9Is)qWiFbHFrtkJw=$c*=?~>;)?3%tMWk-v?4>`we-&_m|H+V;yXl=J z&l|v{b`D&6NSU66PwW6t=)M{amK`VgMgq5Dpo`#9wXvgFLkS?M`@yr6@#U8U5?e(Y` z-dzx+4s8mxo?&0*Z=TA}QtGPm-l;^EhI77!9m=pnXYxz4Pi<6dk&hLYoA05DH>Es?Aqy?pN9l&0^dxY^w6G8+m#>c-gYXx@k?yR**p%V!2)F6m3P2dJ0CGO3s z8>ik&kNSjvcUFH1lrl?XQsx+_H&@}ejXN%vzk&6w^csK9S07E+gl7DCA2&Q(<1*N? z*Ta~&#(`Nxjq^BE2#bXU)G(GTD5S5%ZJ+ugJ=M>;$E%G>_qK_K6ukzp_PfSo&wZ_Q zYEiPsEFlqp_T7_%d|(hg@eB4m%2_G!Tp8jf z*4kDBsq8+&<{yY=OT`ndSA(t0KiqbJsJ&#HlCa+^Z);`k$hmY-mHx_OqW|ETvHyxX zi_M+W$07OD33mT{gEZ}u-sT%?)^S0H=w=`vS&2>3i&aqKw-^3?b+mk40L+#ORSg^R ze+Hd@g;C6bu&qbSk}r}W0KrbrNgV|Byl3aRj~m?_LnbTFCDkW8%TZbS<#yZb#At`$ zx&(xG{1|&)zLz(~U%bGVX!9Z^c@n@Tm?0Bkt7qc5y~Y|-3W$cw^=39`AY(?R|INYS z&?6}SBN^t-a|v{_{1@qd`E#=Fs!p}J&k*^ouXz8j!fX|B|ELzxm^f#O<7jI6_0Fw( z-!$|`Agjj@_^ClsF*{2-pJH%s39@j@bNpuWTJ2T@ng=K0xG#_ z*L)aoCBu1l<~$R+*}?||953X9T!NOf9pDGU^<`hdRHYNi^!7m#Z|l@%j|ZgHx_BJ{ z&=18ZcjevWZtvbBe;_TeA95&nC$%6T0oI5HZjm==Fm(sa3M3-8*V!(kE+fLj(ED^yc+^+1n~Mf*mBH?mbLLt#R>-s=ZH##FFc{`jXvyZk9QI}?z=ck&f$ zXy`sBV{ylK@*{(7lT^<};nWTT?zfbsz&23FZjkKUW3!SG4fOUN11tpW`i9NDWhz&K z6VHj4*Da+1_R9bfThpA-wxIlQ@;v-z^saAlAiO|F-cM{$JCZ_d^k1zF{P0Xz~4iR>8$%{ z`bB7=l(y(f>n8HFyH0&KjMdbkMW1I`%NdgLm{~{8#AeBFwsH6pFJ@5`pmPDNhF8Lk zRwo(8=?uwt<$xASwb5~Zn)9NJ;K#wx=Wj+wR=5=S@+hD5>Sd9!l;s4pQ6i${!%iQs zEzAH8HI267mEC8<6exGVxAsM9FA0A-eH41vsAqoV0BzNdLaBGY*lMmi_t+qL+3p9r zELCROY0)mkD1GbiKXm|ZX$Z|+%2Zy@$G#3Tcff9Nvdr9%wsp$gsgPiQeZ^LlXy2Qk zMVRhdnt3cV(01NA8aMzZvAuBcM@2>VKBW$k;g-Clf6LhXZt;JVg%fi{)GoVq9rvCfiayrVu zDjx6P851$ix%i$>&CFy|8tx{@!5(j}zaDLEZBwS>L4q@#p)Y zcFdiFXKX*kkKjN<2;>PIW9H}hfqNG%Gc(rB@LONR1L`N%o15q1XVsPuB%w2U5Dl5Ii5!&V~kXvvX)r zri+Wx5Q@C=n*D5)A!w`;waA|?#UZY>Qd!fl#kY@7|M73L4-{r$-tIi%6Qw%Ct)1Yo zZCnZ?`$01kb{yxu^QS!6lHdnX_s9_6RU#+a8PhgFwKV+Nl18ENhAhlZ`%dZOCdW=GjoRZv* zeV-(vCWV^5Ui#~KcPjx5JO{Q?$iW;|Q%?kO974i>)(BD;5@-t}OzJ~(Zzl7N1JL!% z$1dK*exQdDqIb|1U@}~NxBDv2T7{iwzRvpi#`AnFz8k?fMMK4Gg)Mxgd7^ZAAk66d z{{k1hXl%3{q!SHj`yt>`8Rw*!R`HxioVoi*K_44I?c?^_K>#h#tjT8w)f=vDK&A*9C%O~^TwxH|O zA2%$^Oy|hx&LeQ5Mo{P_%QZnCg|~pYb21@o>Uh8{#Wi0hSd&Kv7@G(dT5cdyi*6x?)x3?> z)nv6)yasrcOYLOdAh1L6n-cOSRb94QRk?*z}q_pzJdo*?xk{O-Y_L}QLC;J@G7eCNR zj&>aB$0;`s^y4GzBtQ=z1x8xR>_BnhsNEtDQ!^C;t$3Q$sI? zVSPuoHE4dGGb*)i3OO)r2y!8+>Fyg z6nLyMu=!{@BGOiwB9NJv_mt|GHj85?_}>j@6JR*XpT1ntDC$wzY-(LTEfY z?5MRG9dK@}tnvht@^cqF&%&w(W zh!+n!>S1&)(hl_LevqYCfcy88e1ldS%IsYD``Z|M%OL9aVLmc@|8fu3 z`KI%8*DGG*fmq;zaFiE*)1@AqpnkL1Kvrt@2dXg{=y%11*PkAdFJGlpYS0X5q0GA% zFdx^%a-)o362@IuxW<#AbZtd1?76e#tQR3rYZaJ|{fPm1I{Z1eiX=yW*X`Y%l0lh% zKGDc4e$dfq#!{}xng8&TyT2zKyp!lfR-++2r07vq1-cH{m?>C|cMCu>L_4-a4-NFM z2Q-;mZepx=oFD@{K3o{G!Ys|+$Qgr;Ud{8wf%x*Se`6EZe(xQV46j^% z;N92>P4Lz_cQmPr<)90wxUSYkeeXW}vDujC=}?7VadDG``QR1IO4!(5aYSoPJ{X25 zXGwRh*&z^M7nQ><`6;oHE*~Xptqb@u#~?l2Ueo)5@4u679wX_SFvKY}JXCMySZ)}5 z@m*St$39xNb$_vnXy)Qyr@O@CbQ^@(B??);Th0$()Cy@dOiSoyh8Al`$K*Umu zFVC#0!rn9`p-%TiA*Uvce0j#Ur|EQHXcPoOdAAWqTcn_uS-cO@Ll5pb&0hvctTy!E z$Y25fYUnFhj9Hd8yl;)!ih~ib7_LaX@-|~F-x~R4jg}E6H_!&s9*em!9b;8lw2B`a zh7BuPTrT9M9SLR$i4tO!1_O+bVWSqmDhnKIzyTCnwknQ z`7h|j0pNb%;(2;`W$0QucnQ+37&}LbOT?w+P9oNncpK;r)%N!s|A3B_bqSZl5=K}Y zgZIq9DE~A0iULoUa2Zpme*Y zk>GM_9J0(R+EwqeeEmWH%jf_N?OM@`NaCw}o1Gb%F5KCx?kgA8`!{PFhOju4U|sq2 zCMM`C!bg8?tss0Y&6v*7-&Zar62TbNtr4OmK<58N-g`zh^?mEYpdx|-3JB6cx-{w1 zr3eT@R6rC0i1c1UhtP{srArNnh)D092+~{V9YPO1A#@Uwo8Nz&|2^m4_k1{Gykopy z-u-2dHP;@i&OO)MYp?k{S`RTWI#s!*?6dQW>3d&An16teiZyy=w|`M73RHfV>`9qs z%P60XpEZp~jLMYhRpeBqJKu&ISfqd6 z_;b;(2R~n9+KnyeSU4c==cg!>Y4-g2!y;a^?= z>Om0L=}9U^SZ-gl1xAu z8?Q>1`$#=a*)u*dp)5+wH!)MNGZEy}z-wQ56&dRuWa{8Dlx+i7({Uj3Bo*?7_WY?~ z`+c-evTM&|0>_srIF-a-+bE$LSbX6$7=uZTUCA>QgWRf$!`Zugas}-teO)FtG&2%P zJswd%m^A-*Eyf+I8l5}J`<*MIKBlnIaV$F9Pej-p)SL`KK`_~`0Nq|B6Mf#EN{b5i z&`=%ZvQb=E&Qx2gXyAKNC|P#7m3Mq}AO}-e)t*JH&eH}`c@kSrB{|+8pT8UGK9Cm{ zSU}KAzJQ-1xnqPiQg0r%@U^^rkBLkb77*i9EMSynHn_~VxierTy|?HqxY~1wI9}JfoFJNvMmc8^B#$) zXl3Xh%&>t2Wc-mSs|-*)S!abflU9G_A`SD z4*wKO#7>Q##brf&3(Vs8%p=ufpHp9E6U0YJmi13TE6y_ zK*2yCkEHV5!aioc7n>2;j@BIKQFtK<&5L*>oicJNYIpfh!aV#D-K+Ryb=xIe1wp%uZ-P9Plm z4t#hrOJF-br-U={gzlHd6(OhH&YPZ=*#yN^ut^hkI4oK&U`B4~xn8DzYy1lJZJw=X zmk$eV?xE(&>GkOq!QMI;XHvp*@^`D>pyzgHKH8*ul?q&b^>dsYk4J2BFWW4%5g(P+ za>lPw`|cYF@X7A)CHb6s>qS%gx0J4bV64EYd+z3^Bkw&+VFCNP$oto}?86-9j~8Dg z4AroHcA#Tl5|0Vr*#ZV1pm_Kahw>JFz134^kt}?TTrNFpn=)~=^m+391XT^?0&YB+6?tIYGS_7x zV$xLtU0)+$mC7H_bz1x#xk@1TM_>kcZg{Ch;2@m0csqx9m^54H7#MKmNZ&xUl+z?e zg{%2!+@jd-!!vvb7(19FBTls=_*?FaVW76lE8lN~PmtnJ_*52&f6V1D#UMxH2OB*eX~bkRrc2Aakisf52+j?iF%e!S@A{VQ4tmdadvH%Myw^qN-U`Dw z#j*T;Us!^AySQg%_OTkEwyX!qqx*-|ZQ4R9!2?ow#r$Szrh|**oQG$BL!pgs zqVM#s-gxoVS^3r0N^3(2(SmKZg^n#-H@oRI1xmTsh;kv4(w5>Q!NqXKeRXuzAn1^F zty+4TJPd_M62$aL<2&*`NimO`jj()^Noh8>f80&0e_ zJHpo?e&`{%{GNA;P}8jdWR(fkVV&066r!xK);b#O+HB#n?%0ZsrGVt#?k(%4%;9Tz z>HJqUft8<=q3xu!H}!{zm5j`Bt6?H51tt^_2$PpmBip&4CO7V-hgQTp9Fh(1fX1V& zEfdWA_p7vwqyfK>$8+!a=M-i>9s|qGp^K;BnAX4ew+O{_7Ae7k*`S`cJ5_R@T0oHH z)Gg7(FYnV>pa^@#`!~t1-eQERO z*Y7r5uq=a-Tj4{0FC05>ImV(Eeq0m0Jr=!83_>D)tClV8G5~Hd6L-mHEBMkbFlXXX zVh=q`*fNV~#9ztun#D?38*TxK@m)QZa>J_O#tY=p8L=5Qj=LSRxjv4Hb2(oOgZ@^d zVvT*a5*%W_9p0Oj|Ct+LQ`;+jy6xuLx_;C|Dv?$CKq|kLhqc+7Jvr7c0kkzoFB5wC z3K~XgU3)g*US}M%lUO=S*f0oXNjBkJh)u60jXn=UNh{G*?;Z886NbS{TUS6*M3kxa zIcQlRx;P!!MY>&#C|@FGdIX>mi9G_aM!PRjC~xsa8S|_>iaLTNBc(~oAYa>wh>BW0 zucA&0@_}`aTF|8Wd!z<9KP->iVwDSy_VIj3FsR>8hvKqk@k`LfnsP`*m2PU!l1=lO zK|Ld(SUhZf3SPW+jl>G^;hce0ktRm;^NpA+1>Z$5j|!90HPT0)wM7D_%M&E>J->a7 z@HW#s%AegdIU@rn9Nz9F(PKSc`s_f^`gm=HV=~;dzZnM0@97fR%#tYV%3%t&=Gi6c zslv;pQ|z$0SJQMKv#GPTcwqx$lBc`~rm1jeeoNw%RQ|63$8|`_0!41%ANf|~hmxK) z4jEv}bt)g^grL&2Bfe*hs6c<5)T7*@Twf2_`1+Y~Eam0Rz5SdR2R;6sFG)U|zNTfp zfq7iAEp2T4DZW1~0^fB47wQz2>FYm0gx>h8;1ov{I}k!>)}zofg=cL(RwV^zH0;L^TmK>Ih z?239%x;OG!9DE^LRzd(ZZd|UryL(m06Evp!2Jo7X`bF_)3a-V?Xj)5_jkv; zVqWt(0BnKIU(|(N9F1rGENkOz8vzSbQaXEDalN%dAj+&u%GEbu1W$+QFsww9@_1i8ZoRVI zJFZ!yZi)5H{$#TxLf?d=Q40n8xa_HW-|n(v=49jv!2xno!Dm~B0dM3W=r^90Y@XP`A#dZXiSF42UsyC@ffIDuXyKU*}ycC}* zWrAo%mvJ^5Fv6evzIXa70$sn-WmI6FNw}-Xvw0;$hWQqbX>3H&wd$cSVv^D|xFNjU zBi@>g8XH#!Yxt5iMm$V2fGz21&-Lfgo$IxBz?u|Hk;miOAb%CZoq(h{U!v|cK_41( zm9=c(>fW;Ioy`o$!9`$!eOpbfE15P_^M1D9Ig=m-fNPDWiv3fBqfF*@RK!o7)((ZF zbJEQ=m*jKSYByc&57u23A~SkpOx$?h4P5xfYPh_A*@eju@M90Y?+seHuS^pb0zF&G z3VJu^Vgz$uHBt8meX3xC{*kU%o6%IPsFE;ALBy4n2nY=qbwOKEIOD=>N-Dl3un+L# z?%I+OJ*D?cX2dNx=hGQBLn9?&JL-E%RWk`%@^|#wy)|>%RLdy{&n}L)0V9mL+Kjp2 zJlFIwn0MMOn||!yjtm#|G5s&TH9tMI0_wd>&IRc-1-)AAb!@dNa+u#H<%&(@j zY^lpJ8<0GC>hyaMJbCs~wZ?7%&-T?1E6l&wupm|W>Ku7LJ5kD0YfYZN+%6#@2J^%h zc9MuplCrw`VpnqeCp0?^(W6j&$byb{LsmP}Drk)RO`c(Hqo3+$a?_U+8eV`Yn3gp4@Nph{aBYK9BJmR$x7y_$c*G6O9ZgYpg==Hw~bd{xdhL>8H@bmtUp=D!u2%^-CM zOGa%@0(p)SKQgFju?1~m>;v5EsVzXZ{OM(CGTnOMQg2r{MtX2vjXnD|Id`2D6xMqS z&Wmh)yQtB1P0XahgJcMtF&h(x&5Bi|j}1<4)&hCQ2EXfc!Wq9H9tY}f9}|NSLQ-v9 ze(%eJRM5v_7~RK5Obv9FCqH>q)(ySXXn0Q(%4L@cDfwLDERTO0fGgvZ!Wuf(+^T->K%MUF(I z!L4X)DRia;>IQo+p%NxBFKll5rdLTJuKQ=M4K7UgJ03-_l{QTijY z->XX#+nnhaiL3YD{a^7jze&Adxh0ow?$@F^>*{UNt1zP`RH{lArVBiJmfSOb5HVgJ zsBU{Dp4OoA(hSIYzRY9Jmh5#!ylRlpCx@(C2LpUmVCXLC1>Ae`I03h!h~7+$ijR$u zyZIBC4qI*#>dWTK!2sJHbg)%}&AR_k&t7iUiM8;Ly`FLJzH6nFJ!#EsxX>lmnIZWw zSqjDI7G&f*$NBiU{X^xDxL=YPr`*r6s6rNO-p0`bUnFS?JuZAWED__^DHqrJEX9Aq zG*U{0ttvp`%d*v95yUDuE#XndOdn6%-SX8jPu z=43QII@~|3StJK&ORI;hNmWQ0S*TH(?^1!kAT&VpA|ivyB-?Ok9^;<^ z5P=!SGG}Y|8TCx5V=lihQnlJW*~j9l?#gb+ES$Ma$s{!e{^@ZdY`UXAo0=~m(Lj4& zL2ak@xV-@swcUhBBE~Ec1F5wwjSbSpj*zvbFR8q~>)xg-yz)86pY}2RB%{|4^J<$D zT)`7KW9vk`T;bW&rf;`J{k~p31@sns3defRC}q}`N{#)u$MFs|LKHp<`WoF!*|tBeCgcX<64aUA!%V} zzq|4GZ8_D;4?#fm(c}KVO7*h2IZ{+?H1dM0hY!!A>Y%tE2gZ7jE4s(2EOy`%k3};4 zrA3gQjbH-7LfPWU*D1L+LO4;p@dfS`g&cpN=Cz3=pOC3e4!9*d!$U+;dFy)fa<)tRn?&whk+Jn zkX_36H%2gR;tyKUdhqM}%X9O+t!eP=!2N|EvD!}tfo1N2Fv310j&HK>Z~%T%7`Mq@uyHNTX$3@4{KbJI$EyGmja;?({dUxd zahfOlJha}Pf#yIiOCK<8h~K9^jBOqf@jR+3X`U9yi;sO(PdczJU`P^pArEF;@o~`f zZ`hTg3JaWQvTkE{-@C0pqGB>s5EGO(A8Il&?zO(yK7RiB%B|_b@AjcTtu>=Z;d?66 ztBDq}DT_6Jc;drVCEMPX$+fY&d)(^YX!!!A#66^;I(SqyY zBXeiFKx@d_*O92RZULxG6WwoU2)4`$0OlW(zq^fBh;)egNdd30$m}@2h0re_TxugD zojEsVxURBl^btM<{>=uNqvkhB;rZu~DSx;pJ>QTRIL-4ZB84qmPc7JHH^@$~DDHBl zRgvmGKiCAUF0k0Uvt$z(R5Z0JD~JhtwHd?xetp-sr-iFoIhO&L0+JF~{ARUZsxGTu zmp21x@L56wnL8yb>Ac(&Ry+rf&5P2TzT%A2VQ7zI#fA!)c>J1=pO$BuWS{VtQ#1AP zFHAE&UR6WjdA&C~e!q>i+h6(~>YDT}=K1~D3x+w5RXdI$YARg@=J*!B&ouloog1z+ zL&ok#Q-;WmOVAs{FXf933m%+r>{&5fnnZ8PButnrm0JuB%f=tRHktR!nE>YR=$r(; zoM8TizdZ|YA>QYqlYqs{v$~9*BlM~2f9)*(0?q1g1%H`>L(P_YMTQJ!n3iP2C1*sW zKt3!?zCi-Ul4{8Y15Ad704m&vM=th~C<+aP?rL2`4d*JV`Jq@vZK6cn%Si+sQ7x8Cpu5!5od zWgE7vRC<8bb)wlT;0=cbnk-*C`68LTOxmIqIRl|Q7n!I-c$y8o@uKWG)AYxOq2Px*~46%WXMZA zwC@;izwRQ~&VF=VwctMvVU=buXSADykxu+ndHT}S?!g* zX1$iK4i%g+tMka9XLL$&a0udtF*P}x0~}buFVN>Y`7`N-tWZaj4A%AXG79Ye#DMdM8hFk*-*|yGcuD`0YMxWKR3=Rx+OiX znpZ^1%=kTAe#q00SXH~>Y>d5|!bQF-r4!D82SCC3rsQ8LKI2}kJw*oTmP75Ye{Tc0 z@IRdug%5Vb-&!Y+K<3aXQf;u(TDi2wt&JtCZvod;YfRLTu$Oz#J318Y)}Dg~CqZP2 z4#yw*{ds~6PLPg)L$Xf%wIWWJkV+570=&Wej-gdMo+KYOBP3FLwu*KZz(rDm(7F%7 zF9)zTp)OiRK|-Dv^zVAB7SfK%e9y@i8UXL*Qbfqxx!ka|T^OZ)&_uZsw-qk7HcCWW z@m1ifCMBU6B>|_EL26u0(g=;zwT-OVB^yaCO}9c8X$oQ;SI1qMX#AEn0)F0slwD;_ z;2zbMdh1=dNotnXj@HSqS|$G3hO65Y7N=r^xQYul(ajYY373&x#sTG)15j0UGUy_3@A z4E1tpU4Z7A7%cE*Pp}J_BF77Cqlts)dWUJMZ|`r?SivA^zn*jLMJ4W|HZxnRMdPyg z$?d&X~UCy zen(MQ;zBv@SmVb&j~UyfhtkIx-so+_l=HTYB*pQ*Q>gcctkRb!0PfbRRo`?V6gJ*; zU-qn^#0meYrNt_%<)b0}{%)v=NoC1hFVBv0;n^vYmgfIA!fX6kxpPaMvCrG(Yv0u( z0676&zh5K#JWzil3jVb*&~z4+ZqMVq^-Z6m1p#9)`CFaMNKDZJuN`(ycq*)?o?W6R z3oY__Ym4}6Xvyq;(Hd7oi=R*1o9RaCx#KZL%N4(ddl8#gdT&YD9yVtNWD?@yu}a*B z$c2E-Q2ow2YJl~@E-s@H`th=V-pdo2>I!Mt>sQFv<2m;{DM?!9oQ9?aaE;MhI^wur zo;Ns}8uLT?Z8%PZ9GB$noH}9^0HFN9+*rD9lN4`l?ZKGSciQrG;V^CmRm^luS-8bQ*fu70Fli&G1p%qtBe6P> zdg<*O@PXEwpaVBsnk(+);MztJ>uWgP;sX4K-F5E>>mWV}JWKvD{xJ^9xr%$f2(PQR z;^lHYf}0yQaQsVSph7LpwlS0GcFAqqacKOWN(N>VPoi>4uetr!y8m3`!sc+!r3?W{ zdGI3zuY#+fne+oMkidaBfyTe5&ex`3ufV~oRf*n&`?AddJJN?A z7Bg(oQqL~goLi!XeQ<+6%1LX8*LE`Dg_+VSNH?orZBC)L2ESX3XI920o)}*832c|T+; zvTmKtB_qFH)+Q-Yk-lwPtOJ>X&BQW?x%H+}>U-;M7pZxE80p%(&MLjVcy45BtDg3D z$V#(qiTr|KljzPtQe)Z=!G`Frpmd7U7IcuYM?zt4Rg{EJ6BvEEeJQ5*%s*cjF5vda zPqD?*zjz*8w8F#xSEggBGi-!B_1^7z54j)9q}7fZPqZK~(T*fsndrX(;p9*k^k?KD(bjA!aiZDMm527ix zaq-bG9hC^qv69XHH~F2`M{w@;mW{*U3${dk|1sP)`9h~azXjKknTShH>f78@TEN|5 zIA3AohsG7Z;wXoMERcWGXF6PYsQan7=RWZ zSzHArV_g!1ChENg@m`m35j-q?e%21t{pJ!bypB(L{VB!?)KZG4+%z1-%N^hqo92c2 zS*SCe=`s(~jGBOa-eFyhrsX38$fWQ&5r9jIgmC@2OERN>akp~QGVg&+@^I0Y(>!dJ zzRqxlJCFn40|z3LgFrzUSm^B8OTB2rKD?LP`5^MmONW=PMsH<>xTTz?Hr$eh;JUu+ zx0{r>kV=LD$om0NwOOk5OWBE|=kvHGoxqBufL1lFO(QHz(!_%0y};a2)WdP3k$$|> z8pgf}3OE#*T5FiQdg@u#o93P_1mC2dF&S&Xx7i*QW6zyhgu|(X5`dg?Jd#p5<2W0% znNj?Eav^d#uekm93k{+hUV_ZC(nFtZm-+UycvwL*M)MClm9^#0gc7s|Ibs+3YKGqW zS<2LtiB=OB{3eDrepeM(6u3>NH8e1)T3l-}wbwQ6EbIZ{!(9h+4igiL@QmR9&o1Y# zy_+}w|8(U_cD?bKfqua^Y}fZ;yU)Bqw8Lsgf&L0kK?Oy)CyX!Jp~i|#IeyDaXgQzd zB}g2^Z{cz~XRnkAdYtQtC}>y$4ce@46aL)yBG#jjmnW+N1qu*Fp9t(e6c~7RmW9T2pldv1u#U@IN=(xl{(&;{kDNYF!z)iC({nC2ZC|?Ay{u?>VO${O*Csyyz-eZ0 zY0PR&xNVvKAnt7YG{iQ@qsf#lno`|x`!sRe)uXTtcLIWN7B731V6$631UkzLDB0)^ z!aR#M5c3Wjh~0HG=;TRqmBrZD_LH~`H4je@U3%dX6v2Fzw?pOQ#O+{fS@0|h3wCmueH(I%)9y#&hMf;XAMaNP z^ZsWt9zeVs1(R{O_(*Qv!5`l&O8o+pSGPFC{TFA#2cPQ^6>b;`mw)kY^gm$B^#97X z|5ag_J8`wK`y%Y_+fa1}*Jqm?EcTBnuswAHQ3#$4et#3_PPW%S@ zDcU`E#}9sQNg7FRdqk%Jid!3a1BT1nCuV2>GTu8!0bO{;%TLMO4i_%pF~9*hOBO+gO3v|ATw_-?jWx z>c2$)%GM$9nu{2Ktp0Z{`M<=k%s*K>_IaR4)4Ol>0v^ ziGQd5$zwMmNJMnaod^{Z^zL6sk^eN`*@m6ZNR$8nXu8xto%@HDT}(>izk(u4{pSF& z|3{YqpX~KKoCpQ8i_83bbR-B!ng2;J_pieK2dcKz_MyVEw~ps5HyEd?m_+rqSDq*? z5Jhs2sj`L1S!vGx{*&Jx2HVuW?5QgY*L#F>3f+|!HiztfKXW{@E$qNJ{e6w6Q{May(#(>zaoQ)$=`5r_LwE^J_URD@0Er~GKz z96yZ)_@5Ti#%?Z2&RPoMPG(j%gHSe&k+1Lip^g}nVndWBBvZb#Z8%8CJj8w|AM2;T zs4*=Fh&2mq>Y=@*f~%c*pH1@ZmB=*XkrQgh@}Xe316o<>^9)yV*N{r{PknhhTC_pC z$9^fWrYhU-*No#`slgYZVMR+U>c5wj74bsY`)nRiej;CqYFkXe90$3dObH4<_J>;LF}qe0$(I27jq^4S)pUj-cHu zgXc3yTm^pf9FJxTx=_MT1mVo^o22X2j)kSt*Qf^kb#IU_28O>7Q3}BB48rj(6jhc? zj`7ula5LQACj7cr>H1(j=P;Yhog0_1iHL?w;1C~Cpr*^Yz#zA3NDCwxgR?g|l7{-@ zSX~0CunKT2D-_yC0Jl94N0zi*Djc6n9X1<|S(%0Me{&5u=?D~$Tcp`bUUags0%mX0 zj4m4)Z|*#tqhy#*FA5g*(3Kh-eU@&S8$xO7Y?ySu8%^(N9`PD!`8`$bV7O=~c9_N^|1=qs57@w=-f-Y`2)saABz1 zl=}MNbf}@7)j1d9%bw^8#Q}i_jqcQkXFjMP7_opHf<1P{Eb;OW^Bizp92N5vRM+(7 zCjJ!7hWl%knmTftPEr`AIM`L%!u!@Lf9VvJ=2Tdmew$W574jh{^;Ba=jq7d9lD|(B zQu(e>C$mb~qec0f!mX7LZt1Jp(*0DsIsy>+6aO5gjaVM+JIk=>EtccxtlsXFJ08k? zLEIB}(_`a1Bt*B-^?q{?xDN2#P{yeu%1H=-$*@|l+^p~0z^rsbiA+x`LB3v&`$lAv zN8fW6YhjQLUpnTrgbumyg?@|@O|-Y3`{c5x)2>JQur>zQh~HRe@6LlvRB|&c-e-_L ziu46B);2#u_Wit1O{$<|e#ybap1koT60lqfQM&uktz+;lA=E=dK?W`z6UEo*;pp-# zJL2wh;G@|-lXpV~Bb@^#!D%Zo{A_VkR?nq;{4CE@`wqKH+8#9&_D>``d z{R-qgtCJeo$Lq%a2r*nno|v*I#Jh;Ma#Ld8JzyniM0q(I5_|>9Yq?4L+}|j8+0zs8 z%H?DA)4;jq2Yp&7v2$$vt!v!p>fo>_x_~lbz8)64M<+3)zca6XIaI1$=eM2xZ>A%8!N9?< zUsThQzjlzFYCgzXB(39Bj?kii@1n9oBcic*N|E>4_Rh-VaH5Ko0&lOIQCYVQ`e$d7 zgM%(lF4q?z?O214oPxUBL}Hr#t4);Qbov5|Uq}@H-uvWRep9Bv-fBSw^+j7rCIkiv zKT0iP#|VLnb{|jHgt8FiP3S-*L)Sr z1HMq^tNOL^5YT>O#hg?=6?KxmdK(fB7XR}pQs5M5Crif1`*^%CZ~E7F!zvqMP|w}q zQ*rC0ID6f{122e*$zOUsD!58+SRI~X7m-&hjNjCJ+!XTn#r=WzzQhkXa+%R=O1SIw zUWnsYN_KQ%1MAoSGQJ0b^tko%VXE3{&XUk_#%dpF!*a^ofKXRk+?J~1~opk~X1SF3j=PVyB=jU$NN zA7!gaF^|P!MvGazos182=W8}1v^u{t{~5SrhNg85hsNc7qi)-qRikXAmR8I4MO-8jd6@0*9IB8$m!tX(H*N5xW^B-jxZdw?Ug&gd4lOtqP@qf!WiLP5(E94CqpfcXqwemkN{EwDzpmxnu%4FN=G+%_5l+PT>%U#?hYJaq?+PWaPW6@Hc6 zVHr@K@O|wQKH$m6b#{8fYQ-*9u}iat^Si=L^}LS=jM=x-Y=sfMRsQ)T-G=Ce{l?%( zfDrGOVW$0n&&*`t44Z~&3mb>jnAg9q6|^VdH=gyfYy6f?xH?0OVta3kYRP|S{sI`1c$>Ts;_O{Tc)F1O| zT8x->2W~b%On#0oXScNqc2}=%I2BS!c=aU}|D`#yVWV%$I}%GSke2U%?~VuoYk-$s z7Xw&n&zGh)?lPoqKtf8VsgV!Zx_G6ubbICm$K#(?ecj#@uY6-G(#pj@jHwaHYF|*B z&eH-q2HchiyEot&-hQv73}G&}${AfFIE zg{!SR!`NgbUiV<_d0E_xW2z?B(e#V2So>!~1vGax0(#(g`piIg=` zsSo}7PVkl}^`53pHPReftqRk89*DvV469_PE&glPO8ca=dPRLF`_mk(K&#EiTSRRkTh!4v)Mp%BOxjd(O)qBXUi>?oo zezX{xUEyk6Wq*Gr#Ova=7c`wTG3FNSljq1-^mdH*T17{n;?4Hk9|uk10m>B*W?c&d zZwJtaI<+0cf-6L4#e96I{b<6Sz=8Jr+~mfQQUD_4-^zSlpBn*R{?g52Z|yUFreK+; z`Ge7PWRO!NZ71uFRJdS93DX%n`zVk#QuU-M{k0pFWMmTC`7kQUY)qFqqEdb|>od*E zF%#^#ciivY7}M8dnsaM0%5o5&YqG_dOwls&@((wk@$FK?YWMfT4J#109TmHsqJ)$yLu+1FCA9mjH1cl4DGxIsYMeEKWOuB|MU!QR!+qU89q^iU#i??u zE0DFw>VFz_r*?Jw)%AHtaAhR_?d`Z#!Tq(@WZ@vM$uiZ#*lpc*Y?~@~F0Q{NU?!hB z<`nj)f-?0HZX+5gqd(v&d_;zdGY(Ryobp2{Fxa;Tc+g#Smn|#bH@Yu&5A%&E9rOpX za&8dZ-x@{I%R&hCetuZEACTww?)hSpN9~8J?)8U-y-P}?DCR@XSEeB*=^AE>S%$=H z6a_7#bRfzL^}nDzM<_9tE$I3>LFpQ`iD&>QX|;%mTAS@2+VBE(XvZARYd-Jt+!3d! zk+2lA)Cu;zS=#UN%+Kc815wrxsp=69{2AyM$Lc9(n3`91_%#J@epL)*F_|Ts7CZ`V z3?Od#$|y{Y4jxNoWMm+bDHjs-v-$IE<=apGxI|?iquWKDXm` z6tCEqQKkKYbhmsV4>D^_9~6l4_MeY4b$s8!q8Q^DO+fyhh`#Tuc` zOjHz|?o@`zgk+v&I_|pD8oCj+ZDck_VkeVIhMLDeJZN!(gqTz-m?QxGk$(@zC0R&# z7D-2D?Uco5ouh`&07MnVxxyM_%)B0ttEX^3bhmy=C${MXFWs=Of1BVTwz}%3Fxua$ z^wE{~&8VW@ zchSJ-*75b(B#VU(&CD`Cmx-6G9hzBX7~o-lk3-x?-sZ+hWk)O)`pTjK?W9fCggdVN zX44P*W>#Nv_>IGAA(p0}%QsW5%{LxQKRTELKjdI~w-@cI(Nu2JA-~O{KVkaW!^6gg zf!Mn3`*V(D8yEXf!}Jm%daKVxtBw4P&JPW@1ZOkVpKPFRE%LcY>19W`F#GF8wGr;Y@#ZKP(I(uR`L6^7-}3Goa$u=OK9N{~~c? zeV4&yOIp>M8>Ngc|7*Y}d=-=vH@ zzzTz>5>?M0_z=Mp142mjDCe`o+mkVbd8z@iy&p{$%wi@bRlMt;B_;3!dr$Bet-D@c z{R;^j<>Ju#rdis@B==eCz~q@p3D5tQKhW{JJ>4(Lq4B0=d6jHdC)XMQ30bfrlOwNb zBLuaO$wdpu2Y1J4Qp&wZLg?PWAB&6q>NmQ}t{k!0zuuOfDX50Ig;Ersd-Hz8WIHL( zng0DP1@jFJIf?)*iv4d@;0m>cELqbO6>wcOGTfAIy37wsqmbl zHT(Kgem5EU0f%eFA(Q=!@;;&k2*+FudOJefy-AuuqI{#p%h*TgOh|am7GHG>Wxw))!HwiYLX7a?{m(_;KZOUV_ zM$@KfCwgbUUNEyfAPsvUVWN?kaK0*)S^j>ICYVXbP*&Sv#&L%~JK##Xf84?o!9hP| zR1b)4`aS)r7#4EIwO3sQxYw*VU}C|K2%#^`7MzCMUb3=DIF;X4l<^nbk>6M6=Y6!4 zeS4AH6WZmDrd&4I?)IQI_~%yhZTopXy@?-rV9_W6U5}*w$w{?L(}_QMTfvX<^AN)b zQz>4+e!PlE+Qq&7clqhV{WbxH`Sh^ycscSm#{u!m7!{GMlUw`xgrb<2OcLvCG=r2K z+Y8nfI%vaEY^iQ!K^}ZV-t$h$zb5jU;v*cjb6>WlEs*maIG2bPoj(3kqI;jDoa$rD z9nz0{lyThHmH24N(^uJLuf162Q@tuVrIbGtyH`yTmA}2MBvHWP8T_Nvol6t9B^u&N z8Mr~-$o!DEiFmFpBf$1H%`=g7PPTEzK$H)swgRbtZDRK~Ub$H(OCiL{`4lzP#aCNp zvYqL`Kd?Fxx*gHSodR3i4EbNF9-BF{%Mq6q+H2~^dK`XbMK`fJ2fU#Q-J&8A6`sf$ zY-Fy_4#RoMf1paKf{tvH=9ZFj}rYa0>ej}1=G4R2u-u0Hj?q@m zO~AtGojnmPUMZDIWwUmi>`>&#ZlC z$-8CGO#b%|(5NJmsAO0kyAnw!>jS&P6vz1Juzg}gXkYQsWjiC(>{Voh(my+cuHvwc z<&pr9jAjM;PBVRb(-meO$2nVgIK0WuyZ8WQbhfoSOdyl|ElnR}HsN}!g|4lMb%S0m zba}G1iTTI8!mXvW)+W{;xe7N-pDY8he`A=~9ufo_JaXYnlPyimmaPh*CId@AD)d{H z8zNKA8z~SbKjZx^zt=jucYMk`IaM)5c@2fmLr%zSNiOwE#eyMvA{GxSVj7VqHP0XGsDr9-75A5isOaf^F4x^{-1>baBR z=N3Y;Aca0{?^~_KGMwpG8Ey^SHkOlE+U@$a*wXWiB_r|?AlQf8t5}-zCZ;IZritML zs^7S&M2fTFi6Zf{=;1z^0h`q{Bk}6KE7OhV&wU&KiT+T!<){ooSQJ5FP|cbou4cMsgg|Lp-Z%lP0aeApj6G%4NZPX z*k9+t1t4WLJ3MMpzafzWawEIjNtl$7qbgkIR8Hy}LG@g^kRaLVLaI6oEcAWbB#J54#+T>p!!64m8>O5j)UYvqT1k z=a2BXCpXj%+3u#^(VWpG0UP=S2J7;2vhh=!i1B9bc&N<>^l$S?O=#Sq5oG`C{NPiR zWpa*WDxBtwwEbi3y6fK%!%Q)(LWgN3btU_`whO5Av5*!>lPBbLo(&L}aMaFP|*pl3C{!~C0=lYXzVJfBiW$iD@3&u@U7Q@7iygj}$-sej7a)_SnYY@ zM?G0#5o(l%hols(+}(3)Zysux3fs^*Lflc0bvvfidH$5xufMlN-A6HN$^Yt@#pon! z2;OtQ{M*GPC+Zj5(IGBjH~BVW=2@-BSZ6!xVQ-ZZ@ry2URdod+h;?50>u};1sl*z! z`Uj1|jveb2W&OiE9r`cttBxK}TChF7C6E~VO3eT+JHxoy)K4>%iJghB?Y8aLP;0SEl(Ts7>{~`3$M77eVbwcq+~ug3$iM_=nr_i_u}6HcrZy255ZwDEQX;sl#o1mUO-Mv=n|H-^WPIwK@&NPV$uhK3?W& z1%#yxgfLwGRr~SU+v?rBO0AS0l@K>q6V3kxDL~f0k}=Sxg|tr|5XW(23*_Ea=}1r( z9!QyFEHQ{%UO_X!rA9lgz4k0y7RY$(*W;4|O?m$0q+Yr@-E%H7=XZ!P`)d zuTUE!6t2RJuTG*azso-eHUisLW>*0lS9vA4Ib)#<>ABcmdf_hRlHG!_j;ZCTJW28> zs;kKPQ1*0d_puw%k7&xkrSQ&IGyB{oMBc!HPG(e2?XqnU1Q!?)Bxq+GM=9WFOQg|3 zFoT;7E|Den$P|E0kNw=M-Yu{oIQ4;=s%2hy+C#}k0iqliqtdaWxj*KFMWn&c)!*Wp>o)l76!t1)1tXi*Ttmf8*w_sRnTC%~=gdnR03 zMIH#buo+=^12sC2lL7C*B6oK1Q1VD}4a7YKblZ*J<%Pp)WSr9z+rSG zw+%{N`momVf*m}Pz!^9wxbnD#4lIVeL3y><-I1*DgUfh^5%yTh72`e&U8|rJj)fl^ z7bpklz!Hf}5mZ7*qa(6Ice z3gYi@R;(b=RmvX%9L}5)j>rYt(IWUf!ikzYT)WG;G!(FIP@2KZ16RMdWM9CGGTaS8 zd+ql~t*VSJZ>o@Pju!a!$m9ZD4u_Bf(k%NVJbvS9`x1K_1_ea2G~?js|7#nD&9n)!Qy^qBPYT&ol=Q%y`Q;Sfe< z^}PGmf^5hut0Z4;fYbJ6& z&1E=g%1k30zz5muj7tNtgx6`HJSMupNQ0L4JrSk^HZ;_YBwsTwRKXo{a9%TVJL;f4 z6&2;=-Wlbc`q&l4E`=A2Dt1_X@-p8F885;qUNMdv!i~mtg`zHbNMF+%;H)MDv~lFQ zC)oN>vlC1x9jTJSbd}==6u>RK{f?qypH<8H50vU5km1R6TiC7uXh?zzl_A>9wg}i>iW9-jjXtx>-|RZ zSmvNxtsi~RZ6$YR?l!`$O>RKG-^h9=dAEHLC2)n`k1T~6ca6f6IN_JH+@iT@tcXDf ztaP(3r9wxGB5Zs$X`PUet}LC_tU-JJZX>Egx_RfOXTQ8n=c$TFY2R28Ex|JDcG*7I zuPRc7#L7ao%l1LHfWm$iK3{My_xna5Ra4a+yY3nVv6q|@ol)%`Qdkd<+BDQy#a&|& zdX!4j->i^=GRSKnL5CoFB-B2WW|S?Oo@^ zx9tf$9eBjZ;x}&p1(A)gqcrKt=QmxzB#@|TtXjg{?!QQKH;}JE&`aht^8Skg$xdFF z(~5`AyW1}&8MNIu{|+JDK!k);XgT7bziU33Ap_xf*0|bI(*rQ1N8HUMka-MF3xe5E z#CUv=lSo=JQ%#8hz2E+Wk`SdOxY25vqx}cS|3SVesJSb9tV%mdC}|tS0;FOgZ7pSi0|8}rCJ>6T zAZ;t2!ktq;>yZ;n8=0yPW?aN4yQR+xlX=lT2Rdrr#K=KLRXGIvp($^TP{cL`6g0$L zYt&m&r%aBRJgVu6qtOW2PN<e`q^qK}2DHkt9dM;3uD>^M@D z8(IG0DrJsEi&doEe)3`Y-8Ft7B1D&yt4z% zg-IZxsH@B4M;%-P>Jvp4Z zzPNzYB!WL;ro+&1{kcPFZ8nF6WZUO(V+7)%FmJ;G(d?RyfaupfxAbdSe5*Y!U35UfV5u7CU|{G)!N>H169yJjk4F zI>EGSM&)QvaBR9=TZJ!x{j9m+=H)%eQB)-*&{~?cUm;r%5i5hvgbpv)r{%lDzsMAP zr?Gfv8lCoj-|Con3`v<+ln!H16iO1cqV$N)>aB1!fVBj?r(bv&IZP4eX-A>N=9fJd z+UrpCll(DwfT4nKxr;rb)l5~Z>ZnfC8|6~*W@97lN(!>-;ea#9TGX{Mt8!fpMQMG4 zIUwRjrqx)hvfL?yn^)$e2!c!noIf&3u1!e|U+S0J^;*3#!)y?lq#I_LT^qBH zbriSq4n-CUoDH8AWNTbkQfXzf(9-ka4AWk3m+3NNBUJP2eJS!{bIt9+=9OMT=tW2z za)il@mfZz&YT;mk3i6;$ZfzwW>Kl_~>e`4Z&kVqz;w7#$v4VC*pf`h_cT5M27}-U< z8(5{+)lgE03u9INkXa|CjCvuXE>~^YB)nv9Fm$2WZSOA%j2mIqh210D9M-lBgDUEn zJ7jlGD}SY}|3WR15o0t02?hgc&y^03emv{r45sL|usHZN?5Z&*05ZB-r(RJ(!r z%q^YjQe^uPtXllC7A1H%8eqxgE+D79g~1JZ^5FlybeamAktAku@ zcSnR62fo#*pHf;@5AOm5MLMcWP2Qd}TDP0!_cw%$?WoW^O4UVOp%A48Nhxd%(0Vts+WL*$!y9!rEBjecnAU zhT%%XxPaLsM6pE7?w2;{wc=UOcxA)VrXo*c#zhcaM4}J^v*dSY~UWea&s`oFN|rFH|%5$(A!I=dD|&P4PI`}v@A_f?pS&}^2Tg;CW3V3M!G=tLo>R=L0yG&te)x8Aw>Vu=Q{`3 zcdZrFoa(IwF_xymu@o20f;vi2KPS15l~Y}WGvRg$@P~Rdq_jOKx#St9ecE>H$k))? zV9m|Qs+!p8_LvIst?%la<&=stZdutyo0mt&biPT`vm>G5f*?yJ>I_=yG%p-h_0*aJ zeZZq~5s0huwz>I>+U2T(U7_VHcYh$Q2|kFvkmD|B4Sk% zt&<(=N)sHwC_KSiiH>;vusX#vkZ5aQax|Fhj9c4rM_uE1IDk8UQE{opZexLyW-Z+}O_tSv1GBQLNd4ozIQu#llcK0ir_CB=?mYTGDdre zH8^^HN1|GLGlvRA6>uEuQ+HL^Y?c!b$Wp2<8S&;)35_?@Y6)`1tGH>8nbv_!_v3PN zNZRG1nQgPahKmaEpg$Dic)=v%fUJ zk5=wkQKd+5?E*+kBr}YuY3RWS!A4s}B>@I7abb3}pz=ka3WlV(-x{>QX#ytAN-@fH z`5TEOJ=mHIAM`7pW^aHp7#_~JY8jhbyKI3rrk;ds6*XdEMD(FiB*cgia6|5LP7prC zGIQA?I<<95g@%NL8ws2ev{|NJMoW%}9Tdi&)^IoH2k$1uj8?tmL)f%&&&hm3-mG?X zDAKu8r=CElVkVS4A))0^b7$W0ZBz1r3WY?Mk>>Htmh9OS43UDH%SDhA9Zgj6BfIZ5 zZ51Ay2&2%LB~#`Ukz^P3gPK?ekAex984RM~Up8+>%S7-vXPQ5|C$2O(x+K6Ak-p-& zzY;bvKEbK0DHK?icC!eiut;Wzsl1s%4`|up~BtS`OZs&GclkH0!8ie5|uW<+l@zE9#WInP4fDUDT#I zQW2r0;$nmrC>GtOLH3OrPTcfSRvQ!su7IJoAU&uBW|O6p03>_*w9Oq~$r3DAmjf{s zAIlJMc?rD7-?4ZeoUvv5BSRIoWbNjAyM41>9wp+`g+6P%*QP1L{|Je40?j3*)J>a3 z!fC5#?V{f-KU3qOdazNF84a}Y2G>ZSz=Uc3PgsH6>>{-7g<7fg&Q z8%;8V*;-aRV&%wdiSxYhSP@tl5o>)ZVKbOqK=}U$wTQ+|OEt_Ohg-;4zzrUi8-e&e z^`9?kB1*);!gI(IE1S*XC#0x1@cvVrV(X?!gmQ!+gS*Lfo|F(wD9l|@&p}(~_OYrH z;?A;GnBKH#KxTD?ai8;aWnR!N>$ESW7#&SOKvOhKEzeJ z4QxG8jHLXo5J7SW-vEF4t`nR0Tk} zLDgt6*G;3G1g*_hqhU6ahY8UG@oE6(s7;1irj~iBQm7h^ydkTF=npC!ROZ5>&8DqX z^$QAQpW=pewHy&}enYI>rXhmRxS&i%+Dx9>%MyeEi5lZRZ&RT@AEx`j&?X_|HoLI-3WSA-D2VGKzo5(yEG{v)8)~T9%^K^sY`YbU2vGYmV-bpVs$ zhLNktd*ONQ^d^MO6thrnurW?U>;yN@9-GS1s0O}nNGK9K!;_PWD(klmf`+he1SYEL zdNaj>15dMQv+7C*_F$t*!@gdkx+EGKrx7r{5UBt z17i{2rMl7~1|i9fpjfNc6R^btLa?S>|Cn0lgV_MP592YYR2)NlU^n5+5?sGfza!5U zVbEO+nnpbNO#^Ot44S*&rnwI9>4;C8AWW>|QhCirkW{@gX}J(l^)#}YJf_Z;2>np4 zu5oe`m4ZUSxg~=Yy2Q!v_yntvx*k&mn4Klzj&PT}q3vNxiwf&H2s zh-QS??Aa!a6NZL~M}4I_AZwx(O6s%2TB*r!s%J=M14W$ST85F$hGGPR_-DGh*cPM< zJSx|}C|@6ywi!1#(*UC!%M7;+7$};=$Du%X=(PA|gk$u$htLwhDlFHh^2v{LjnqA*R^A zP^~`4*%Bn%OMBm?W(cSiR!tG>cR&67A*Po%JVg@2U^KrGn!5}cY<$;$=q?Z8OV0?iFrZN^ZV$!cd{Ou=HYsugWA zJ%wgsNCQMT!cucqjE2YnAGd=|M08JG0X-$2L zNZ0{ZMDn3Lx;YN=IB`j^P1d>0#(c(>FCi7a#(Es%p{R`8~ebEWbtHX2!Q8ALNlY9-oY zWi;JtyKND^xG<@R9QmAbjF67A`oOzM=*VH??dVc!jN2BOfE&E`U{NddioWL$@spHQ zxYJNg?kFP;zc`WulR$YOm^!bTnfA1_6+8@*jhUjEHo6$hR@Y+aON*gTn>Iml!ha!@ zTMGe|P1}wHbrazo3dYZcA!6W?O=OC{dxPh{5dlSQ6I%*)?|VC(+tCWHr3hg!w376# z`1!N&4#Ic%L{r0MBrTB!DCB6tZaUVkZ2tK((m5q+p`2RgyZ%R-ESPev5t?JDZAa=c zX>>G{A>l$x;OIAPL0P@rxD_*6M2Z5BZk0yf6?a95T7iq|OXF>AyRRdBSbNwI#kz&yw7VTSSR)z1cK$pSs; zwfgb6GD)z3Pz=vhz;N*!4d5O;Wdns2j8HMK)i!Gl(~__ z2@QpL_NcEG8M0FcwiM6}SezNK)MGE+pp4+PPga#$CQT^0C>WnPF9knc`tOVtg51kg za(+$#C&+B7@Uogi1PGh-i&Um_3xTwN>Q76f?TEU4W9e!LLNvZ!M*`e8TpPueU=e)h zZQo!uw@BbjZlju}oO{y<=*)bwjLw-8)960LoRAgYY*=9$DdG~w0n9;Kz%)$P-(S+w z0D_RzrG5l0gOGr*A*$O^R|GUO+h`WVziM;!bCF<3{%gVGV?Qi1PBD_CVg+UXUXcZ_BFaW<^@To5rtg8yotcG;9Vcrwe8wVWDjMQ! zaIC8l)Q@5$pm~EpCXWZ{Gw518-Yw$6G)yf#=@0DQ+)zNJ3=NNFdcR+Q_$;L!3 z7j1!#5b3@YX>9sJ6atj$BK~RooZU9b%h6m&b|h6qLt!YCr~mmSxjBJ~B&9)1`^I0 z%knJ!<9$_)mFuq;*73|rLYgw6&JEB={uXfWmY?BAqlp#ETuTy)g;G^zfiDk)90`Kgia@8tPRT@;%$_8(l66d@+p@{;_u71MW-I0Jbn&s;BlG z{4kYR1(i>0?*W&2bWtY$Vjtq0~* zz%y`Kx%4Ji7|)F4d7Di|-h5Yup;mc$Zd|0U()8f_dDO@*AVjSe#wgFhneUFNL76Y9 zY+Dkk6oLzp)6dNoP_C@Wc_<9E*@7vDjZnp>IM0SLK;C!H1@I`d4Vo`mAcVmb=OD9Zq z;84cAb|l7j2eZ2bO0$$P-3OI~Z_KHRWO0e_eI-!4uR%w~sBk+3#^s*Zr9oatpg?OF zBNF@4f=aTM=NHWc!R=AwE3J7p%@)MJ(ovNcac&39H~0r(SFoW@W)a0MYM_lAKxHD4 z5SVg=+%r;L&)*$utkAiO8rz)5oCE81&@G7WiOEkeh>BJ2Favis>mJd4<^V)zM zjC>Vu%)DT|=KRr6_;~=iG5`q%AxwEvc0?`$4L~d=Wi{KD$c*ciNHlL6pa2K^z*TxX zcDYn!5(iRL590xX&=N)z(7$W0o)!&_Bmhr5NiOIYj<~A&6tM`sDYCkU$eikmX-Aa@ zG?8764Bx7gu|Ty3+t@0~7=GMlX72N9D63QkkDC!A;|pQTZ|R@sK#49ya`?=G?%X)I zn4paf-B)aLJ0jeD5U*Mp%?+c-Y^uabwh4IS_re$rW{8N))Rblu{E%3~EjCA>GTTx@ zmR_|iqsZ@UsZgT^#q{4_LYUhMPAd8qFPb6!U!xwTLt>k0AdZ0n;iy6BU))0 zktWx~6chnN4Jn;4ux?DTFbGP1A<QZ6K$1|MA;-Ky{m zr@oS@RaU~aeC$ThIU0~xf({5&yVOkslK2U8s2M*83ndMj@vLRKo{T~kI^!7di{wBP zWoJgAsoDcxTJsUHh3=coKvCwFXQU@$901Ab5{G1k{IAf$ zG+Uw@xhl}aXh39ztIEUA4UltzXMiLwwzOd(FLgo5w+>^1_ay^B_e7dHy5boO+yGh3F_>t!08skg8FxK{Ucgh#3|GV zRGQ4=DKPdT1e~n*JRF4g7RL9gLwhn~3pA@p=z5E}X;AXO&s@!fQ!GLGb1gUm>_2`5 z3!+J*VJ9<_HkalD>eT3FCoO^K4(l0HED*gASsAg% zU?^sH!jmV4<@471i6(HCDB&Wz44ve*At2?YqaU5{>H~ommO8N%aq*re0;3}u(u8RV zqBI%Tf?!w(UXXri+bgy|siGv#Fb&Rj&iXrZrBI1rwk@={LW;;?`@2Qdi7Hm|{JA3Q zDyR_*O=WCk70C%Qk7wvrp6>{oLBy)s{N%>ssihFDPFn}J_$+rR`0LGgu8 z8i^`o+Wev;>vz+7BU zuydC>)vV@P=fDhq_vWWR=B?gu{n(nx)K|JPgA4_x|DU^1G|(+7hwdipT!q^%%576w zXy*w{gr>=x@}>n@{#c^HmMMEzR1+(72D;chpGIZ;eIQX&)61qwb}kgeV5lJ7?SgM| zR5&fCwRl8My7b>Qt%-++P#qWuKl{WSe{>KX@`Q9<5_5|<8cW=y+OF%3vu0MXBl-tT z%$yUAdP=!b;Z~Z8aTJm0j@PZ^_?^CUOjN>Hi8ANRgtd&j0sC3!45(~w`;VtP&@&7I zhDiiylPcp)p65|0DVqj&W#DNQpYo65KCJM;<^&EK#WB~m4rA) zqOxIMdUGv}8{QT9nSp8?MFK_@P0hR$oX#@Z7dt%R-k9gZBch+77o!{=>mT&NFWwKAh;5;+b0#Iy5YCN@=R-C^hpCc=`l8K9BvXK1>kf>o~ zsT(p15)>hGUfwM3j#Oe+KYZW|1ye8fr_}SxTK&(;Mj3L!{ zu((Xq(~=2}J18r5%uo_xAsLxTQ6L!}wZeZiPt^W9qB>|irL%pH1A6n!=8BA6niO|@ z4^OMXrFNGg`5DUzD4NO;3rQIw4Gl0F_$pK%0>$nGsZm=rcCz0ec*;PtglKGa zbsYqc$v)w4!MKZyn+7?A9m`G2j68Pp^$S~z06DVmVf+@2*5nJ#!L=U=Ygl`xHnnu6 zn+?dRmbIwY$ieVNr*KxVP=)8n8HW~T6YA;zx@-`FNu+d9b^QHpK&O~U7qtqR8^1%1 zLt{Gl@9{a*b->qj#=N*uYeQNxZC>Pk!3Ou=Q%Hz=)abA{Hz+Y+%jtuGe_NkviY)^j zn6iQP12UWo1&`X%NE;~)@PR|MBkH%pFvG$3T&=4p=Q3pLz=D@rFk`S3R4r;QK`Iu< zMz<`Pxu-oGDvr<+=USF@wj*2E8&!UeCVHru#`|IdSFk9|=ah#z2MPX}2n6Bn^ze7x zG!~ijlJVVYni;#t6a%i(rA70tZ(E?N2wJyo)O3}GjQD68`=J;cmz(jbn)6Fb$BnL_ z7GupZ^;Q`^YHz5^8Xf(6$Y!*oi)s~%Euq^O0GrF4xQcPc$`P8&lLpxtk0D}baWB$H z*g2c16ok(y^*wz)$+EgE->261=)0ciib@x>7BUz*|BZ+@_%vzFu!*q~%xic$${h^% z(YMrcKY^AY-ffVJgdBcIJxCl3n@boiRYS3pD$gcSPc4l!9FS<%8Z!>8Q65{kwXfZD zp1K(AtST#prZTnMYf*E8#vw`w&aBW>6ir+>IBV7-#!Jd_UBH3N{YqtcM3g}?5*}Y* z;6hUsjAg)CGs8s40#-J6fm{wP!^>B=3KGP@e5EE;;K*pt_hS^;*}z#4{f{`!m0(=D z!ucMhAw}gpp|kxCWx5S6A57kUm+fNrF>X~G52>_?Ub`-1va9ePatNK(;BQy@;bfSt zk=BBj$#-|}w+sP~*g%=+X|tf%hU6X^h~?(bkv}V++^?8QooX!t%QYxOX1+3r4XOq1 zM~?^w<`vX5vAz}eddu`g3e6ldMum*3HcLtl)2pUywQ|pQJxC2&oF?J(OxrcqoN-!6 z#8Rk+=mTKJ1eSB`+!obiMH8~|Eru}$3@(*5WbqjM+*bL~ z>vLde{J?)10SbmSomp$S&kFbrbr7^j)IcePSb%(POTgKO*6ha2UqGLP=BWKvx;|As zTZYe4K{XawJ&n`Kcl{L#Eki00Z`8SL*oF{`17j&z2nV;UFdHcJ87(pAHxN8&Xcnzj z3>sr3qU6#lXmr0q{4QIh@kS$T;H_kXOG%w5)#C$^p(<8G*6_|5f@R69y9)dk=ps=@ z1MQR`6Ql-1#(`xmptu~0EtT=;ii!@f$kA`F=TUh>O{`=!fgSvL$I7%aArw3)>I8v@dlw)T#rsF5bAhEbH;2?wE#j>*t-n>@yiq;p4ONxo`f1#zo8~&`tb{jo$hLLyLrSQ6 zZHy)(8P&$#Y5^t%W7|qv4A6F1@Ui-{soz>snp0bP4upE{a#7rY7+eh7 z)LPzck^SYPscSO?i!fzgztwC*I!mT|fIsSXEl$@00zkezY1B?;e^UR)O1x>-_~1`lbm&2lJ%jBJ}sAEy#vfTk_z|W{8Abg60l} zL9+2}Xo>3HQ=p0&fm+%C8M&uh1t-6mLe%`GnM{5T4Hcq=DcoV4XqiUMW_n2_s?=SR z?54-mToX%whvu?rqYDL*IruoYIj^K9=c>pwr)>*xXhwEAt=)LEyG;?f_A z5Yi!8nau5tLh}1Uqg1;9Wp)Ux2mgbt@sT6(CfX{5QKmvxIyd6y+1Rq;D}qgNb2JrB z?94G6)sF19VDOI(v8S%zHnEOD?PU!7W+S17X4V)uljnId%pH%11hfe?wDa3GMExc^ zk`-rrnT@%LDdd671`0%|7mm`?-t;Mlf&y3S)hQ@`cBo})3bXm=4gPV|hQeV58h$(9 z9vr*!%*5^H6{bcbYZVZOr1ujjF!R6H>|oYx8wxmCWXyyoOPcoyQoLy%L+Y$Ok3Xo~ zS9?j2R5xIO$=QHpsZ`pmDX26@ltMFOh_g#p0cy$P^XxC?OrIL>HGBA+dPMF7y0fFR z9Om&R7Rw;3CG2JG8JE$LkhQ|IuqHhM=6X@113tMeS(P0+OSIwn8#TBvz3~PNRVSeY z^{hHMQa4FcCKm`x0M%&T)=g7o-A`r$facjOZBC1Sc4xQ&K1oh7%$1QthUY)a zH+ny(AukShiSI4M2~pK+8mY1H0^Ftn^;2L%#4V523~X1GCDg>8WwX>B-9pUp1ImXW zBQloobNI){AHRJ1`KN>uCmA3y#Cvo(AOib|d`ZMSVku5sD~ z$>CD*)cMKrK{c|InT2NDC>q;Z;lxTt3bEUv)4<+{L@NjeXI&*XGX1n>H?rH4*sQ}+ zbRs?(Wy2{ln{UY7+!P0Pg&nQonQsPK$n)R<;ga%%;Vm0-^`XjqF^Yt@%ZP7Z{Wo>4i5Ro9*nAdUP za3NSAv(`75n6;VGKS&FOx{+t1w?fwpXIPtCh3)kqfjOb`WigQpdRd!E5=yTC?v~w# zT!kd$tVmhbra*rWwU1rEgDtJ_W_G#hDl%ymNQn+5J1UCrc3>t1Pb-n3uU;>?2qsuS_wZ=+!NhAQL zw6*S%$Wl~9(4B9S1w}wpQwK*WxG9p>LjNIS0;zGgK zK;=lcMY+{3tsu9;2!W-b3t?Pa5f2ObK#NAW&YAaSC?BW^R74#+*&OYynpCem*rn{z zjvC5HcoGteg^hD;?LHwZ8UwJP!>+rzi(rk&KTpk`+cm;n9vnwtl;H?#>dh(eQrnS% zXg4*uSAofm&B$fe=3HRAVp$s`YOy1MUH_m!{Zfi;&WdI?JiOCUJT5R1@O;bK3>{vn zdS@tP?Pd@D1)<)-YvRgXEAn{4CRV!p=dOF$^fI%k;<@inO`faq4cE}$VLg zl!xi9n0n5juvDV#7A_l3n}J*_7MET(4K{4#2-Se!uxmt^D_EVN;*Ia}-j_dp`210` zZ&k`?49GN{HP&WFDYxY`wj(=?HYS=$>13;mTFTe?K*S5|48#YZIZoNpi>geI-Ix1pgBDJMMkw%1= zO=IR$^0ZITl9;wbR>Z167FFNdkq|-~OLc0E0h8c2y+V{$t*s`_@Y^91%3WoYmQf#- z&3Gs&K(`yMcV&l{)0e+~{L`Q5l&DgUMn809+*(oVE;WhTZCm10W>hjAODp6II>u;e zu1@>4kp~h<^C$mbq zMKpKgkhUU$6%=rxWwbThMrMA5-9oKeb?FzulBv)ug?K+KbAl2CzW5M-RadYK3sqSy zdN;8ZOV2P@!GVN6B$r`NYK=-0l6uX#v`&guP_zgNb&6%z%IGv01}-D1n>Hn8-qs9m z8a%f^iVy|=b5F~5>?E+Y)tIyd01@%B4JdArFsrQCs#BQWT9MrrZaPH6VQH+$lE$YS zmp z=E;>JJpuI_Zo9M4?aEtXF1fs2q#T67Gjp~Dm2I~juHuROu^S;_Y>4mx(-L_s$+HC# zRDm!eO$NMOZ1Tt#-tfS-w@DctL+CGeDXV@8_9ek?NdDxzfBXB#zo2-H`nK%q&EA8M z=f0h6X7o~G4nR925NLQCK?Yvr<*;lV^WOpNqK1o;N4X79>Ir z%mb^LTj!#3VX$bit~sE_*vZeT5n}6VXDf^CXYadv3v9hWWD*8I&1EPhx83nWyW`emchsNZH>^jpcst_y4Vq>|Ig2fSp%VB(j0YRHproRn7-*NTSMQ3guLUy z=i$^jPJ)@OkqWKw-sD?~hH&{!HW`lRW z1=;ukQ(Z|cePoT=Nn0cf@R&S4qtRO}OJjddY)ln7tyEQdt#;c~GIFh|wR9v!M+=Q~ zz7=_c(y*_WjRBs(8Qx2pkZV*kCOA!9Ignyw3H+K3m7_%ft2tYjSQwi#E~MEC@g z<#n4S3iF+TxxlS3F>2n73P0jh)DsMxUO!xIEA(MVfoYB0c0^Q`2{f#$fE+iIMLjB( zGxja6uxtigsF*e7&}hn!3kp#%T_up$cmLzBzyI;4AHMwj_UXqT=wemjqw)h>PYXg& z=?uOS$2;A);ljg4ID$A0IwBT~n>!cq>V(rUjBCHt?SVx)yF_}t5QlGX9f*j7RT9pL zX#Wcm7Hemv%*ncUhFC$i6#pSu&XY8zR(pSEKM}NFR%+B zuXKf2!Oa3gczJ;&Ti_;ISDoU$&=oj0ICS&{`d`o$Qh$JN>L(aqUSL3J5$LS?6@7i% zp;g;nQUCgFhsQFj7xY=ZzQD{x;B$TDl}TywB`C3q8#gq(-W7^rh#=$pth{+;ee5bk z;f6vQPcY_}XBCh~ceB9GjJ(iQmPZ5byg!@fTl8E{Z#38BI+pel7G`Rt2;)URxBPlH zWg_$JUhSs9zkcAk7Z(+_i@FYj*0bGY*f#q^y9@)>lifHb_zv?DzSfNwao;2Oqs_*? zw#7a}y^_c6b1&WFw;%_#oRoj98-y!u?~Q$_8{tevvPqLAd2Nd$;;G?{+F#~t-6TOf z2Ao!(&1-vubjq6t9$q4e>aqv>;^MVI?Ti+kt$ zVhjrn_5S0-@0G(hhGmt;81)29=EWbcsF?R*Jba;ss3ha~_hFzkgcfp)lTN^(Myd}3 zsn-+Zq-yA7*@mh1AqWx_r0DY;B0T=yA$Jv05BL7fJKy^PCH1(UB|rXtC))Rdop%%l zM?J*kaH;zatlz;vj;O=5z66v$W5)X3&nQ8Usx3csA1F9QBOem5?)R6i;^E|UM#-C_ z?-TNm`83YMz4LiW2#X#2WgcI7I-dsQwipjFm-ir0QO=`~Rc8twk1@POSQcd&0I1yMs1w%K8z3nIGiytTzvm?=sL|1dM+alZ@vE>)gXMSyJvwSh|qrQeim9} zgmTv(m4#>I(*gq0d%4T7I62*lpz0q`w-RT zsA4i2q+M6ttgYo>;iwoL{SP0Qb_MyzG+)>#C3y6`kD7;=c{6a*H9iEm4W}Q~sShD` z!Q~@`{}I40vM$&1@ca8-m!m%3bM)f_of#2QFT|Tpqxl|Nexjd%JT1xgawz345;)UO zKqV&;dHGcC3UnuWp*1077y~((@?tkNG1d%*G|uPX3cuSCB10=PEAbB9r4whmVdSC* z$7BrmC%xDaTwsv`v{l;bC2F4Tr{V+0W7nO2@fzap2#@N?+zgqv1(WUTw=3DXk>-JxvKxN8YZjbX{xqC=#5xGVJBRYz&Uv&T! zqm+MAXw(17y%z(6xA0^J2K#EwSM6TF5bC6bbU6L5d$pAtLeY))nu0E66qG8 zyYIi|#qQq>>lhc>(}C5# z=EZ!oT1Ad&9eC~i*X_N~70EP|Ir3P2rS6&RfM!O%so=h9?~KJyOaxGL@BP=DehC%*7iIrnu3i13On2q;D8 zUz4!h(Q^gkL~e>1>gR$CHGIS)0okuPtZ=6Z^;{6s2^NglBj(>?V5%5sF;LE|)?a^A zB;rT1LnM75D_%|f4Mr`<8p_%t%Boc3e*IBOx;>;zpBxtFaNZSP?5AY?=-o ze*K~0v5z8QOs=Z;Is69qnJ$+|xSXOe(?Q>n`EAC90ELL)#L_|7XZ!09e4rx#P+H!q z8rrk=-)t^|+zkBof-tQ4kH3rDn*1%NgD7K2jKTi#cif#4zU6d$fXW?C>wo+md4U?g z>2#2m9@r|h-Kq9lP6vtbpt|w0JCUl)RKDqa5I+zE$&e{Y-}0!OK|%S5&`a!1;Do;A z%SwzxVwo%RD|;$XNJ_277yIF+x27G(BzxfpOn-Anm+NncCLF(!aTME z@<9RBVO`G?Wc%>m-6qRxq4N}9weC0NquQpKLXf*2XexCqe%SMnO6+oUun!uRcQz~8 z)RZg5+F-^YP%uz3rU6H9Z?Hp$R~OesO52lc}lBH!#6o{!x&x%ZznT_ZSQN2l*;rQ zO9kPfFxx$GJjx*y-Ti)Vc&KRo zSQHxu;dW22i>aXJP4YF9yWR=+d;g9QHKy5E2a`zB(T%G1KV9mHNR; z=O=H(+}{yK-E4!x zBwm`1>od&s^Pa-r+M;zUmoT9oWsX#60Sj19k~HX65^qTS_|n1@$-lo*yy|O*s(z29 zJ#ysJHOKuWPDL$)VYq0S)W;2XY-9L|B0E~boHiIG<6+>7I~#|CNqgN&n9$xy11#0$ zQpY(r0)Si+Pxhne+i~6(NzQrxP8&(*@g0sPb}R_;<$I7zpDVNZO6OJ(pwrH@-M8<_ z)KKVj-bsgGEbyv9YFww@LX#!-Ej|Du&s#8{m<;S%k~mwu>Lz@bTH8aVj#{sZp-4%%mOclaNpGoM^_z=V4x z#$042YG&0}na31+v6k?zQ(;WU$Z-y0=qyvjQ`bbxnw&$ufFX}s<;q}Jy8}WCtS(>T zLZr_~;O}D4$??LXIUE6~$1lhX5X>_JJY?=&;bt|1cv;r015An$kNNgFrNJzxb9BI~kaY2`_hb41txH zg8l`MVt?sNKgB^Dd)NL(?ehKSRJZOY_%d3|`_=xA$6K1tm1?PO*+2(I^#xv$13y`> z6AdG@js5Rga5=4$ky-*T$AWxjho#$$D3wIs^VCMDVLi4U%Re?};~e8*PC{quy2D}% z93g0_Ql?-xyeo2VV{Lw)>lGzB&bVZ_%p%k_saYr|7G3^kx6aesfJEgcqB7w!YY#Y6 z3+mb$`_Vq2(|KR`or$%eBI4WrYu3oB<;YazQZ>MyBy?1Qt}e9&GgiBgYGK9GWhKT| zZ(?`UCd8Fwl`TF`-5h>zz3a`#e(&G}d8ne@J8#*JoT@Moqf+AXi7Q2~oJ;T=V8Ca; zUaakW757`&%j&nXXk_?aq&i1Z)i^t~maCi$4X0}u{Ckife2IySJ7t}$q@3~|o+%yg z8Q~{rZ+MuY0rV~qw^gg}Z61c4kp!fapO>4rjya#$W|eNoS@UUXyl4_vmH`S7$PDHA zcA@AkT)~brgMu5yh7=rbD0|Tb;+N9mPn)#>@%_W0X{@)$aSCw}s#Hm5JezM!V{XHf zOv|n=E2VzacYmZu>d*`35>kE&uh{YM4mUx{)q?wYkt8*|vCswFFBeOOMD^El_1-Na z*+s0jeP6!QdbE}aSUE1(#D43SO|{m>_+mfHP5H_IbPV6nr!;*g$x!lPauw!i59Rh0}K05bsnbH00UNSy8=92LsNp<-0SQxf6r z#E7ndP@d4s1~jWHES)v-m8w83pH$_V=){jHOY`ae!B7+Wc?P%%-W7!COd00*J|*hX zmklul%p#=@s;!6Sso4jx#^oB=s4S*AI5&mj_aWp7Mm7ow0ciyn27>GpefkCks_)S^ z&Jl$x)r%Xfuy5lQ)-AAuJ1CZmaaw~c#@ zCer4|cPZ0^u6tlo$5c7g*2tK+Y)7jfI5+DMsH26w$GS!G?O1n#P<$d_Xb^TYOa*Xr zk;kVvP&4TvUvVUTcAS3fP#;J9X|Hj*lC@SAVBU7XsPbX8!L)R?7IA;hF-2DsXX7L& z>Pn2X#rF^C#e4^k!1TD=a-dA{=lEr z<&pCdHcs~TzVKkdU{#K^^U&zfxy@sbP^vF|R%^+2bja~x)S7y$x02kU8am8GJM-`%@>6KQdy88G`;@!Akia69XnpGITSdX{AtY3o zndl^kZ_-ma3h|0d(jtdAB-!*pKnav+LYVpLoyvuFnpC{w2nh?Vc={3gk^qy+#NpERyrWNd9DEHzv{W{jiFs_?iVP1RTG=BV z<)(~#vkr<%)RpgX>aTN(SVrIR4LGm?^dOQe-dHPoZ1n_M84|A{cWN{CMZ9CjTlH%r zUXG+BUAda@iZOJ1cx+3!W){%7<_~&}IQYTkp#!>sULN7ucgWGn!C2q=C;jy9VSP65 z`NhVEgOllxRMubApT0^V;fT^vuC%U-@VXpu<(gaRFIoz zLhC>0pc`-RZc|_@sTpxOP=}Sz&m7@%;{d zIP!9VgoC8SvwV}w0D6V~5}_Ix!A3M=>gT-@SN2yv#X@@h%ff8LQ-#sAcn!o{mOH|0 z$v9YODU3;01Om8-06xXKupderhh(;}?hD1ceDd4%Msdke`GrI!swAOtdDt)O*=(E= z2pmJceMio?XY zY6;$ztmo~C_GH;K15_!{-2uha`CPf7!gfVkK&-jh<%FGKv9hV`&K(sKZng$%{p9U? z=uKKQ8EPkL=bO%VK{retbf{2{Q>)Y4N*mn(3$5ZZ&oCALA@P)IQpR zWt0wGe4uAF8fNc(E;=CO>=b7IYJ&W9?fRK3%p6RAoFRW~)%NEZ!ovJ(iZF9OXUiXQ zG$LpXterBi%b z1j@s4BlYQAb6he}o#vWk0*OO1)#7zP@b>_!N0SIXFp_`78}nv&vrmK6nm zXDfARy@oD4tW>>XdNXJga?N72*RQHZVXjaVa>X0TE7q*!2NMj##-E20#mbAzrDdUw z^H-@psBnlH4bj6cjrpi@uYX`ER^Wy;P|zI>fn?6n8n0&yP}Sk>#(@ri@%6ZnNUE}_ ze5FG2&D2@pNx)}>184Izw5b8}8)Whj*%JvsEL{EjaK;EV7#AZO@F&FBLPfH_LAKXA_H57A%{a+#Z^D zU?)RPH+JZ*PwweZ%6{L?z}&60y`H$u}H+zIB$!;e#^l`B@;+h^vtD{soib2D^eBu5!d(jS?Y4c=vDQmQ7pVcZ+8G{-Z3{60b+ z{W;S}uKk4wn&#aYTMgbU=H^@Vjpi}8nEpuX+-PXxV{=S4+z1C4LT=lh>GRui)Uyk& zHpoShR~ZY1_KE!Q$3qZR;ZBxjdG+oj)L5DYfnzTb@{!=+(~|`NhMR?cTDIOI%k=uGI4T4eU^q zuX-Y|EOIxmy5qbxQe1)q_-HD3~ueN4(Yk}werG;)C&TtgWAmQ0B$$o<8< zIGHYlkk-T#%aK1~6`|LNlsi_A1vo?9%|FN9H7IJEGtblehPKa|<{T-79C>hwslP`j z;oB1R?5j3a+RH%dCNqq8vDBT1Vwi21GGh1VLxZvs;eis8x~OzA5|&!wcT0IP7ywKd zjpGukC}o|GOhVyW^@IBixK;_*0f;;@nuVWH-u5-Y0C(9+pQF#$9(od;nOIwP`H8N#yC(G#RnwGn-wU!Q zXk`6npq4!5?aGutc?n1SjYa*XiF>S8a*cIixtX7dH6k0m9351J#*n^JmsmIdD5EuA>eQyI6Ax-)53n-jU>fxQQbn#c*P{@qss~hDr%W(PBhO(pHRG zR&i-?!SQ?1Lf5h&RYJ8*L|eB3eF}!4@OI+RW7BJdF#ND~Jsw}X$w5qUv4ba*9fLkW z?BukZ977oh!P9p_yn{Ern>t92EhXG3@SRI|O1`nHimKjesPF;4#$OuX4U>{HQWzp? z4;5)9u*a)GxiGM~`65&gD)e2faAiMW5l8i{Kz4dp1SP(au72Whw&oPNQt`_qxvFG^ zL({xgD;|Maw7EQ~nZ(Fa|A4{Al`bY+#aKjmmvu<1&fvg_WgX%t4;BLc`Ox?!Xv9Gj zeZXg6uaICK`zy+2MTq*TWle78S(R0Zx{2seEJvIsi4$_9E&^CWy&;_j|HPC zgMD8b#)4-RaX)2KAwBW8Fd|KES;DPfFdCm6uYfCA)~JnNzRoX4YF5Jk#482md<;a{ zvx{YWz!`V*;k#R(ZS~>gUlzq?x|oMqjr5%_7ptR&1No|1oUj!)L*S)zxsYYn?nB1wo+Zu*yPz0(%i%NXYa;E z29&fhGIrBs`rVc`+0Shg+n*au&*e`MC@G@JXJEk0#l^(R%F4lQz-nY{z|75Itk1=y z|9pWClaUbc8V`(TTXalyO z3YE97hJz!?^x!7#KGDUQD-#MsC`^@jSjG`FzusK+0FemY-Dmg;#Ij87cr9Oh#`MKF z6FlFtEUk9jScHVxs^;)+o;V2z2nmkNk$kTjOq_B67G^VJXzcCaMDa*I1n{iOlbM^t zJ?rWitV@4vUh&~r!)bnB@nN9MX}ao8w|XjKDjRYT+{1L7$|C=LTm4nxF7145n}V&2i<%$eZD znZUN0&_^!6K(3%}u7G^5koBEDm1Y7OJB&|u1SQppBO${NcTb=TyVk5!+K25oH+3Iv zSLyu)`Ot0-(RIDQ=0`_V2;xz_FTeT>@5TI77f)+{;)$$~cV2Qch)%$a3eT2{|M%aS zSoqWv(`W$gXd1#d@Q7QzvwAG@Oqk57I~Y5w*6Q2xO|G~vv7kF^qaJ-@E~^v+Fz8eH z<=GIpD?jUIaJ~jgdnR04!C>*@NurD=^(@nh%14`-g|;_G2N=p#Ak6*%aJAnE}S^`R5>G82h&R(Ad1 z^8dgUJR!0AZilVao${_D%7E;=1hiPyu;`^}>%DFNBTXnx+^`Kt$fI{_8#Kv zRcI~d6bT>W484qM=R2vHhuS7>%Fn`!+^Z=j`KN;hrpc`Y-v(iRPro3h*!@i?RJI@q zt;x5PNM@t&aooaNNFQ1ML6}W+fjpjlMv8Epg)a{EolRu zbcu%4u%bn>VqIcd;Uq($2_5_Y*AkVEWB9bzT7zAcZ#2_GteMIA{q8CeL)c+E+i%}%~zd&*b z6nk*~qXRpJKfNIQLq{yn(VM71RIVGp-x{c4*|>8CA!Ja@9Nv6@N+TNG@cT;%q-rHh z(`X3$r5dxPn$WYFQ{?gd#`;D^VQuy(?qRk9Sz+QK{$z(+H&rca?5zTVr;a6%j_X(>=8c2BT$*FyMP%i-pj-HGP^Un#&wNO8g&b_&QMb0UBGb#7(MK=~b5EX=21DWxm2Sy1y?? zPBA@Z_th*Bv0HQFc3HVwuf$o+Kgb2SJ2D##hqKd6WvDy;4-{o%TFRDl*KB35IbsY; zveArXi2O$2Hvjn~*)(pN;e{BqtxUieI&M$=t$)@NsAehdbq(9EMJyk>!b}+Ba0leY z01{&8GGKu3=R#|`PcGiIa-*^0x!(ny`=B>p1+8E8a=B#(UUUDJu#Cab%fir)P~P*Z zypN!~m#4fR#;S+fst@P62s#+=p)G-ZY zeubp_w1ZZ{dIC|(Ml)p>6*^Ue z1ft@54g86eEBds{ejc|Qx+Y#k%JTRO?*xtR028PmeFiRse~~!!C^+!PraNQgDz^EB%#`?r*{OgoG?<560ck z1>65&C=?18M3#9v62B1>o?LhWcjwbT0nBWo!$-s1C34`0&EC?_#fk9pFp(lwqeWKZ zNmpYHRTI%yW0X}BSXSfgSCbG{qo-HnYgS{=kN(#770JU6XL{wmR_Oj%q`-n7p*^Ne z-|J@Vrd1a{RM!>RgY1l^JA%YgegW7qHNC;(n9SKfe#XCn%x1I6o-Xi2!uD%T!=Co?p|NGKOgx$jB z>dNQP(>E)&SI`306L{MO-yFU(Z2yLU6Plun5px=jO6KJ|_t2XHLt^Pvf6k2Jz62QI z-Ea@(BYk>&oYxK6=0C;uB0rhr_XplyR~Ym4=@MQ~_dPb)`&us)tWy(n?lw#I%kbEY z+kc2jclv|Wz){&?UlDC-t!261$YWFX`@pzppK{bJ?YdCNdo)h$x*KDQ@Zqj#eQo+= z7j)f$?RIA#DA%9rGj}mjl+E#2zxzo>(E84NLN5KElWh4WmfVWkPT$!g2l_I$e_g_6 z2d{jITZg@e8li*~Jo{idl!O;vDd8lVM~|T$zY`&%M)`$&4EwK2&mGYYD(qb^y`9?1 zbu5g}Zmq4%pS10(RkK}n;Am1#k=KW@!Qvyg&NBO*ctGb-Y2ph-*tW^Hf51)-%CQ)f zZ~c%AJ>(31f2~Ur;VOJOX>mh5j(NY;6tv6hL#*ydRXu8J@gCMt1P@DJKIknvjOf1r zoifkk5i=3iJ3TH3Z*W}x zRczeNdTIeCx0pRoN3$F$1cgudKbjg5Mxxuj12IGNza+x^CtxbL#3f`*)(7$!Q45?0 z+I1y7Gl-G|rd{Oyt+4iRqF9AoHGTh50LQIU*uQ-_M9n6h6f^g?b|TM>!h;=$?{8@p zHfg$>Gk*Q4(e+V&yQ7z8!80$p#TR?7?OxnpO&e)py?fYt7Y~>;<@kQYod45h#(xD= z?cevBV}DY~_zAnLs_1Ide=-m^`Z&E_*ve@!n1xU_5JIZ?US|>AZN*_FHycK0eR-}? zDC!c6C|{2sdY?{RuG0fyN~t<4;1a9+)Bm+piK6|;P(OrEKM(ZAf49m5FRc7ggCgMgGNz5nA z>Xm`+z`eRwkJ;oIoo!UM2j7Vyc-(t?#3?Q!JTH0t=)_d13NN9MdcKf`fYq7MQq zsTUCa1*lxq-XP{(s2E97yLbPFT}C#-sUu0QoBVFo01JL~qywk!ceO-@CLNA4$-`P4 zj=TXQjn4GDA8Z4I)9>kJ0iF6Lz+;Tn*%53o}*Wy1ybd@{~nuSbwuHR3)sNOa0MT z&SCOw^YQm2ULg>nP)v#z2#jFV&DmvrUFo{(@;^ne8MR`<+br`)c;AWi{ipj;p2g6q zU4eSp>s;qA|CU`Wr{8Rz=i3H|JHnms4V$617P6nDrSlhZiZ$xYdKm8~RjB&e)|z*QDwG&sUz*J`whhVl9id_49waVm&QOo&6Kl z+svKorsQUk7q@{QV~Jvp1Rl)O)CN{)rFs&ID%nHPxb|>X`XQsjsj{g249Loy@{4br zp?-zB$zerK=|u+Ty2(qmXqM{}^?Kv=48?Um;&qX(Zn{D1gsZK_#`i+Gv4@-zwBq;%Op97?bAG0C5nWNhJ;(;*B=VYK?^+0g2Y?reJ=Qxux>Uw3~K>hQ3jT{=g^x@6hdbFj)`1VzFKRbM0Q3U7R}Ba(Fx8@t~BYfOe3Y zK;Sg=(^2y_KQ5l_S+)|1%IP4eLm+w#^x^2G&)K;@_}sje!%iO#7SeYa3X|{iRQ@hV zFmd(R=MV@uZn}al>XNgz)=gR8UlMD#M?lf>frIgt>Rq;@61+sD-Cjdod3++x?%{=! z6NEVyR+4(yl7j?a(7B1HF>UmOgNy{HJR|m~!Du&6@ozS-{cX#3H3cts=HZsH8XoJ^ zo=mGAGW$MIk(vJ43xO5>zMs*hAqA^jNxj%kgfpK;1vLcku-l26 zF9^NZPbPS)`+RLK2JTh*r}JoM?S&tTnJvoFN(nept-zK+wvq|Nq3c!Dc^j*Q>*6vB z4sRtAwNB1Kxfor#(j)>Hz714TMJ;RM6$)o&sbubKlOMfyHBZ03@sND@VU*}qOdx0x zn{XYhMCHB=3x#^Ahx$bKwD(Z_c#YZ@iz71V8iYuGYe(DYZNu8tp?BEfYyR*>nTwStLhfSBckffR6fxqep}yE;QQ~kAc3J1(^3-U@h)gn{CB<1&dWp~L4-%`eKYBtOqwqO;c7tDVABKtki6JBWKU61X?EnA( literal 0 HcmV?d00001 diff --git a/train/README.md b/train/README.md new file mode 100644 index 0000000..8faa7a8 --- /dev/null +++ b/train/README.md @@ -0,0 +1,50 @@ +# 训练benchmark + +## 支持的产品 +Atlas 800 (Model 9000) + +## 操作系统 + +centos7.6 & ubuntu 18.04 + + +## 训练方法 + +1. 根据实际情况修改 ./yaml/ 目录下的对应的 yaml 文件，建议备份原文件，且保持 yaml 文件名与模型名称相同。 +2. 在当前目录（train）下，执行：`./benchmark.sh --help` 查看帮助信息。 +3. 根据 **帮助信息** 或本文件中的 **运行参数说明** 选择配置运行参数后，执行：`./benchmark.sh` + +## 示例 +- 示例1，docker 环境下启动 MobileNet 多卡（8p）训练：`./benchmark.sh -e MobileNet -hw 8p -y ./yaml/MobileNet.yaml -docker` +- 示例2，host 环境下启动 MobileNet 单卡（1p）训练，yaml 使用默认文件：`./benchmark.sh -e MobileNet` +- 示例3，host 环境下启动 ResNet50 集群（cluster）训练，yaml 使用默认文件：`./benchmark.sh -e ResNet50 -hw ct` +- 示例4，host 环境下启动 pytorch模型DeepMar单卡（1p）训练，yaml 使用默认文件：`./benchmark.sh -e DeepMar -hw 1p -f pytorch` +- 示例5，host 环境下启动 pytorch模型DeepMar多卡（8p）训练，yaml 使用默认文件：`./benchmark.sh -e DeepMar -hw 8p -f pytorch` +- 示例6，docker环境下启动 pytorch模型DeepMar多卡（8p）训练，yaml 使用默认文件：`./benchmark.sh -e DeepMar -hw 8p -f pytorch -docker` + +## 运行参数说明 + +| 参数 | 是否必填 | 参数说明 | 默认值 | +| --------------- | -------- | -------------------- |------------------------ | +| --execmodel, -e | 选填 | 需要执行的模型名称 | ResNet50 | +| --hardware, -hw | 选填 | 选择 1p, 2p, 4p, 8p, cluster/ct | 1p | +| --yamlpath, -y | 选填 | yaml 文件的路径 | ./yaml/{execmodel}.yaml | +| --framework, -f | 选填 | 模型训练框架 | tensorflow | +| -docker, -host | 选填 | 选择 docker 或 host | host | +| --help, -h | 选填 | 显示帮助信息 | NA | +| --list, -l | 选填 | 显示当前支持的模型与框架 | NA | + +## 查看日志 + +- 可在 train/result/ 目录下查看各个模型最后生成的含性能与精度数据的日志。 +- 中间结果ckpt或其他文件存放在 *device id* 下。 +- train_x.log 为模型训练过程日志，内容较为详细；以 hw 开头的日志为打点日志，仅记录数据。 + +## 注意事项 + +- yaml 文件中的值可以参考注释，根据实际情况自行修改。键不可随意修改，否则可能导致训练失败或训练结果偏离实际。 +- 集群（cluster）执行时，请保证各节点环境配置相同，且包括**配置文件、数据集、代码**绝对路径相同。 + +## Benchmark工具资料参考 + +https://support.huawei.com/enterprise/zh/ascend-computing/atlas-data-center-solution-pid-251167910/software/251732401?idAbsPath=fixnode01%7C23710424%7C251366513%7C22892968%7C251167910 diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/README.md b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/README.md new file mode 100644 index 0000000..80fd3b2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/README.md @@ -0,0 +1,40 @@ +# DeepMar_pytorch训练说明 + +### 1. 数据集处理 + +#### 1.1. 下载并准备数据集： +百度云盘https://pan.baidu.com/s/1q8nsydT7xkDjZJOxvPcoEw +passwd: 5vep +或者https://drive.google.com/open?id=1q4cux17K3zNBgIrDV4FtcHJPLzXNKfYG + +存放地址 +./dataset/peta/images/*.png +./dataset/peta/PETA.mat + +#### 1.2 运行以下命令，分割训练集、测试集（路径修改成自己存放数据集路径） +python script/dataset/transform_peta.py +生成 peta_dataset.pkl，peta_partition.pkl 文件 + +### 2. 模型训练参数配置 + +在train/yaml/DeepMar.yaml中修改相应配置， 配置项含义: + +``` +pytorch_config: + data_url: 数据集路径 + epoches: 跑多少个epoch + batch_size:1p 参数为256 2p 512 4p 1024 8p为2048 + seed: 49 + lr: 默认参数1p 0.01 2p 0.016 4p 0.016 8p 0.016 + docker_image: docker 镜像名称:版本号 +``` + +------ + + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/__init__.py b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/dataset/Dataset.py b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/dataset/Dataset.py new file mode 100644 index 0000000..dcfb0f1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/dataset/Dataset.py @@ -0,0 +1,81 @@ +import torch.utils.data as data +import os +from PIL import Image +import numpy as np +import pickle +import copy + +class AttDataset(data.Dataset): + """ + person attribute dataset interface + """ + def __init__( + self, + dataset, + partition, + split='train', + partition_idx=0, + transform=None, + target_transform=None, + **kwargs): + if os.path.exists( dataset ): + file = open(dataset, 'rb') + self.dataset = pickle.load(file) + else: + print (dataset + ' does not exist in dataset.') + raise ValueError + if os.path.exists( partition ): + part = open(partition, 'rb') + self.partition = pickle.load(part) + else: + print (partition + ' does not exist in dataset.') + raise ValueError + if split not in self.partition: + print (split + ' does not exist in dataset.') + raise ValueError + + if partition_idx > len(self.partition[split])-1: + print ('partition_idx is out of range in partition.') + raise ValueError + + self.transform = transform + self.target_transform = target_transform + + # create image, label based on the selected partition and dataset split + self.root_path = self.dataset['root'] + self.att_name = [self.dataset['att_name'][i] for i in self.dataset['selected_attribute']] + self.image = [] + self.label = [] + for idx in self.partition[split][partition_idx]: + self.image.append(self.dataset['image'][idx]) + label_tmp = np.array(self.dataset['att'][idx])[self.dataset['selected_attribute']].tolist() + self.label.append(label_tmp) + + def __getitem__(self, index): + """ + Args: + index (int): Index + Returns: + tuple: (image, target) where target is the index of the target class + """ + imgname, target = self.image[index], self.label[index] + # load image and labels + imgname = os.path.join(self.dataset['root'], imgname) + img = Image.open(imgname) + if self.transform is not None: + img = self.transform( img ) + + # default no transform + target = np.array(target).astype(np.float32) + target[target == 0] = -1 + target[target == 2] = 0 + if self.target_transform is not None: + target = self.transform( target ) + + return img, target + + # useless for personal batch sampler + def __len__(self): + return len(self.image) + + diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/dataset/__init__.py b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/dataset/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/dataset/add_transforms.py b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/dataset/add_transforms.py new file mode 100644 index 0000000..76aa64c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/dataset/add_transforms.py @@ -0,0 +1,65 @@ +import torch +import numpy as np +import numbers +__all__ = ["AddPad", "AddCrop"] + +class AddCrop(object): + def __init__(self, size): + self.size = size # two + assert len(self.size) == 2 + def __repr__(self): + return self.__class__.__name__ + '(size={0})'.format(self.size) + def __call__(self, img): + shape = img.shape # 3*H*W + h_high = shape[1] - self.size[0] + w_high = shape[2] - self.size[1] + h_start = np.random.randint(low=0, high=h_high) + w_start = np.random.randint(low=0, high=w_high) + return img[:, h_start: h_start+self.size[0], w_start: w_start+self.size[1]] + +class AddPad(object): + def __init__(self, padding, fill=0): + self.padding = padding + self.fill = fill + if isinstance(self.padding, numbers.Number): + self.pad_l = int(self.padding) + self.pad_r = int(self.padding) + self.pad_u = int(self.padding) + self.pad_d = int(self.padding) + elif isinstance(self.padding, (list, tuple)) and len(self.padding) == 4: + self.pad_l = int(self.padding[0]) + self.pad_r = int(self.padding[1]) + self.pad_u = int(self.padding[2]) + self.pad_d = int(self.padding[3]) + else: + print ("The type of padding is not right.") + raise ValueError + if self.pad_l <0 or self.pad_r < 0 or self.pad_u < 0 or self.pad_d < 0: + raise ValueError + if isinstance(self.fill, numbers.Number): + self.fill_value = [self.fill] + elif isinstance(self.fill, list): + self.fill_value = self.fill + + def __repr__(self): + return self.__class__.__name__ + '(padding={0})'.format(self.padding) + + def __call__(self, img): + """ + Args: + img: a 3-dimensional torch tensor with shape [R,G,B]*H*W + Returns: + img: a 3-dimensional padded tensor with shape [R,G,B]*H'*W' + """ + if not (self.pad_l or self.pad_r or self.pad_u or self.pad_d): + return img + shape = img.shape + img_ = torch.rand(shape[0], shape[1]+self.pad_u+self.pad_d, \ + shape[2]+self.pad_l+self.pad_r) + for i in range(shape[0]): + img_[i, 0:self.pad_u, :] = self.fill_value[i%len(self.fill_value)] + img_[i, -(self.pad_d+1):-1, :] = self.fill_value[i%len(self.fill_value)] + img_[i, :, 0:self.pad_l] = self.fill_value[i%len(self.fill_value)] + img_[i, :, -(self.pad_r+1):-1] = self.fill_value[i%len(self.fill_value)] + img_[i, self.pad_u:self.pad_u+shape[1], self.pad_l:self.pad_l+shape[2]] = img[i, :, :] + return img_ diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/model/DeepMAR.py b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/model/DeepMAR.py new file mode 100644 index 0000000..ce169aa --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/model/DeepMAR.py @@ -0,0 +1,81 @@ +import torch +import torch.nn as nn +import torch.nn.init as init +import torch.nn.functional as F +from torch.autograd import Variable +import numpy as np +from .resnet import resnet50 + + +class DeepMAR_ResNet50(nn.Module): + def __init__( + self, + **kwargs + ): + super(DeepMAR_ResNet50, self).__init__() + # init the necessary parameter for netwokr structure + if 'num_att' in kwargs: + self.num_att = kwargs['num_att'] + else: + self.num_att = 35 + if 'last_conv_stride' in kwargs: + self.last_conv_stride = kwargs['last_conv_stride'] + else: + self.last_conv_stride = 2 + if 'drop_pool5' in kwargs: + self.drop_pool5 = kwargs['drop_pool5'] + else: + self.drop_pool5 = True + if 'drop_pool5_rate' in kwargs: + self.drop_pool5_rate = kwargs['drop_pool5_rate'] + else: + self.drop_pool5_rate = 0.5 + if 'pretrained' in kwargs: + self.pretrained = kwargs['pretrained'] + else: + self.pretrained = True + + self.base = resnet50(pretrained=self.pretrained, last_conv_stride=self.last_conv_stride) + + self.classifier = nn.Linear(2048, self.num_att) + init.normal_(self.classifier.weight, std=0.001) + init.constant_(self.classifier.bias, 0) + + def forward(self, x): + + x = self.base(x) + x = F.avg_pool2d(x, x.shape[2:]) + # x = x.view(x.size(0), -1) + x = torch.flatten(x, 1) + if self.drop_pool5: + # x = x.to("cpu") + x = F.dropout(x, p=self.drop_pool5_rate, training=self.training) + # x = x.to("npu") + x = self.classifier(x) + return x + +class DeepMAR_ResNet50_ExtractFeature(object): + """ + A feature extraction function + """ + def __init__(self, model, **kwargs): + self.model = model + + def __call__(self, imgs): + old_train_eval_model = self.model.training + + # set the model to be eval + self.model.eval() + + # imgs should be Variable + if not isinstance(imgs, Variable): + print ('imgs should be type: Variable') + raise ValueError + # compute output + score = self.model(imgs) + score = score.data.cpu().numpy() + + # set the model to be training + self.model.train(old_train_eval_model) + + return score diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/model/__init__.py b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/model/resnet.py b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/model/resnet.py new file mode 100644 index 0000000..f85a707 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/model/resnet.py @@ -0,0 +1,217 @@ +import torch.nn as nn +import math +import torch.utils.model_zoo as model_zoo +import ssl + +ssl._create_default_https_context = ssl._create_unverified_context + + +__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', + 'resnet152'] + + +model_urls = { + 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', + 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', + 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', + 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', + 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', +} + + +def conv3x3(in_planes, out_planes, stride=1): + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(BasicBlock, self).__init__() + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = nn.BatchNorm2d(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = nn.BatchNorm2d(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, + padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * 4) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__(self, block, layers, last_conv_stride=2): + self.inplanes = 64 + super(ResNet, self).__init__() + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, + bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2) + self.layer4 = self._make_layer(block, 512, layers[3], stride=last_conv_stride) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d(self.inplanes, planes * block.expansion, + kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + return x + +def remove_fc(state_dict): + """ Remove the fc layer parameter from state_dict. """ + for key, value in list(state_dict.items()): + if key.startswith('fc.'): + del state_dict[key] + return state_dict + + +def resnet18(pretrained=False, **kwargs): + """Constructs a ResNet-18 model. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs) + if pretrained: + model.load_state_dict(remove_fc(model_zoo.load_url(model_urls['resnet18']))) + return model + + +def resnet34(pretrained=False, **kwargs): + """Constructs a ResNet-34 model. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs) + if pretrained: + model.load_state_dict(remove_fc(model_zoo.load_url(model_urls['resnet34']))) + return model + + +def resnet50(pretrained=False, **kwargs): + """Constructs a ResNet-50 model. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs) + if pretrained: + model.load_state_dict(remove_fc(model_zoo.load_url(model_urls['resnet50']))) + return model + + +def resnet101(pretrained=False, **kwargs): + """Constructs a ResNet-101 model. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(Bottleneck, [3, 4, 23, 3], **kwargs) + if pretrained: + model.load_state_dict(remove_fc(model_zoo.load_url(model_urls['resnet101']))) + return model + + +def resnet152(pretrained=False, **kwargs): + """Constructs a ResNet-152 model. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(Bottleneck, [3, 8, 36, 3], **kwargs) + if pretrained: + model.load_state_dict(remove_fc(model_zoo.load_url(model_urls['resnet152']))) + return model diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/utils/__init__.py b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/utils/evaluate.py b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/utils/evaluate.py new file mode 100644 index 0000000..c25ae82 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/utils/evaluate.py @@ -0,0 +1,105 @@ +import os +import torch +from torch.autograd import Variable +import numpy as np +import copy +import time +import sys + + +def extract_feat(feat_func, dataset, device_id, **kwargs): + """ + extract feature for images + """ + test_loader = torch.utils.data.DataLoader( + dataset=dataset, batch_size=32, + num_workers=32, pin_memory=True, + drop_last=True) + # extract feature for all the images of test/val identities + start_time = time.time() + total_eps = len(test_loader) + N = len(dataset.image) + start = 0 + with torch.no_grad(): + for ep, (imgs, labels) in enumerate(test_loader): + # imgs_var = Variable(imgs).cuda() + # imgs_var = Variable(imgs) + imgs_var = Variable(imgs).to(device_id) + feat_tmp = feat_func(imgs_var) + batch_size = feat_tmp.shape[0] + if ep == 0: + feat = np.zeros((N, int(feat_tmp.size/batch_size))) + feat[start:start+batch_size, :] = feat_tmp.reshape((batch_size, -1)) + start += batch_size + end_time = time.time() + print('{} batches done, total {:.2f}s'.format(total_eps, end_time-start_time)) + return feat + +# attribute recognition evaluation +def attribute_evaluate(feat_func, dataset, device_id, **kwargs): + print ("extracting features for attribute recognition") + pt_result = extract_feat(feat_func, dataset, device_id) + # obain the attributes from the attribute dictionary + print ("computing attribute recognition result") + N = pt_result.shape[0] + L = pt_result.shape[1] + gt_result = np.zeros(pt_result.shape) + # get the groundtruth attributes + for idx, label in enumerate(dataset.label): + gt_result[idx, :] = label + pt_result[pt_result>=0] = 1 + pt_result[pt_result<0] = 0 + return attribute_evaluate_lidw(gt_result, pt_result) + +def attribute_evaluate_lidw(gt_result, pt_result): + """ + Input: + gt_result, pt_result, N*L, with 0/1 + Output: + result + a dictionary, including label-based and instance-based evaluation + label-based: label_pos_acc, label_neg_acc, label_acc + instance-based: instance_acc, instance_precision, instance_recall, instance_F1 + """ + # obtain the label-based and instance-based accuracy + # compute the label-based accuracy + if gt_result.shape != pt_result.shape: + print ('Shape beteen groundtruth and predicted results are different') + # compute the label-based accuracy + result = {} + gt_pos = np.sum((gt_result == 1).astype(float), axis=0) + gt_neg = np.sum((gt_result == 0).astype(float), axis=0) + pt_pos = np.sum((gt_result == 1).astype(float) * (pt_result == 1).astype(float), axis=0) + pt_neg = np.sum((gt_result == 0).astype(float) * (pt_result == 0).astype(float), axis=0) + label_pos_acc = 1.0*pt_pos/gt_pos + label_neg_acc = 1.0*pt_neg/gt_neg + label_acc = (label_pos_acc + label_neg_acc)/2 + result['label_pos_acc'] = label_pos_acc + result['label_neg_acc'] = label_neg_acc + result['label_acc'] = label_acc + # compute the instance-based accuracy + # precision + gt_pos = np.sum((gt_result == 1).astype(float), axis=1) + pt_pos = np.sum((pt_result == 1).astype(float), axis=1) + floatersect_pos = np.sum((gt_result == 1).astype(float)*(pt_result == 1).astype(float), axis=1) + union_pos = np.sum(((gt_result == 1)+(pt_result == 1)).astype(float),axis=1) + # avoid empty label in predicted results + cnt_eff = float(gt_result.shape[0]) + for iter, key in enumerate(gt_pos): + if key == 0: + union_pos[iter] = 1 + pt_pos[iter] = 1 + gt_pos[iter] = 1 + cnt_eff = cnt_eff - 1 + continue + if pt_pos[iter] == 0: + pt_pos[iter] = 1 + instance_acc = np.sum(floatersect_pos/union_pos)/cnt_eff + instance_precision = np.sum(floatersect_pos/pt_pos)/cnt_eff + instance_recall = np.sum(floatersect_pos/gt_pos)/cnt_eff + floatance_F1 = 2*instance_precision*instance_recall/(instance_precision+instance_recall) + result['instance_acc'] = instance_acc + result['instance_precision'] = instance_precision + result['instance_recall'] = instance_recall + result['instance_F1'] = floatance_F1 + return result diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/utils/utils.py b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/utils/utils.py new file mode 100644 index 0000000..891dc7c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/baseline/utils/utils.py @@ -0,0 +1,347 @@ +import os +import pickle +import datetime +import time +# from contextlib import contextmanger +import torch +from torch.autograd import Variable +import random +import numpy as np +import torch.backends.cudnn as cudnn + +def time_str(fmt=None): + if fmt is None: + fmt = '%Y-%m-%d_%H:%M:%S' + return datetime.datetime.today().strftime(fmt) + +def str2bool(v): + return v.lower() in ("yes", "true", "1") + +def is_iterable(obj): + return hasattr(obj, '__len__') + +def to_scalar(vt): + """ + transform a 1-length pytorch Variable or Tensor to scalar + """ + if isinstance(vt, Variable): + return vt.data.cpu().numpy().flatten()[0] + if torch.is_tensor(vt): + return vt.cpu().numpy().flatten()[0] + raise TypeError('Input should be a variable or tensor') + +def set_seed(rand_seed): + np.random.seed( rand_seed ) + random.seed( rand_seed ) + torch.backends.cudnn.enabled = True + torch.manual_seed( rand_seed ) + torch.cuda.manual_seed( rand_seed ) + +def seed_everything(seed): + random.seed(seed) + os.environ['PYTHONHASHSEED'] = str(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + cudnn.deterministic = True + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + +def may_mkdir(fname): + if not os.path.exists(os.path.dirname(os.path.abspath(fname))): + os.makedirs(os.path.dirname(os.path.abspath(fname))) + +class AverageMeter(object): + """ + Computes and stores the average and current value + """ + def __init__(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = float(self.sum) / (self.count + 1e-10) + +class RunningAverageMeter(object): + """ + Computes and stores the running average and current value + """ + def __init__(self, hist=0.99): + self.val = None + self.avg = None + self.hist = hist + + def reset(self): + self.val = None + self.avg = None + + def update(self, val): + if self.avg is None: + self.avg = val + else: + self.avg = self.avg * self.hist + val * (1 - self.hist) + self.val = val + +class RecentAverageMeter(object): + """ + Stores and computes the average of recent values + """ + def __init__(self, hist_size=100): + self.hist_size = hist_size + self.fifo = [] + self.val = 0 + + def reset(self): + self.fifo = [] + self.val = 0 + + def update(self, val): + self.val = val + self.fifo.append(val) + if len(self.fifo) > self.hist_size: + del self.fifo[0] + @property + def avg(self): + assert len(self.fifo) > 0 + return float(sum(self.fifo)) / len(self.fifo) + +class ReDirectSTD(object): + """ + overwrites the sys.stdout or sys.stderr + Args: + fpath: file path + console: one of ['stdout', 'stderr'] + immediately_visiable: False + Usage example: + ReDirectSTD('stdout.txt', 'stdout', False) + ReDirectSTD('stderr.txt', 'stderr', False) + """ + def __init__(self, fpath=None, console='stdout', immediately_visiable=False): + import sys + import os + assert console in ['stdout', 'stderr'] + self.console = sys.stdout if console == "stdout" else sys.stderr + self.file = fpath + self.f = None + self.immediately_visiable = immediately_visiable + if fpath is not None: + # Remove existing log file + if os.path.exists(fpath): + os.remove(fpath) + if console == 'stdout': + sys.stdout = self + else: + sys.stderr = self + + def __del__(self): + self.close() + + def __enter__(self): + pass + + def __exit__(self, **args): + self.close() + + def write(self, msg): + self.console.write(msg) + if self.file is not None: + if not os.path.exists(os.path.dirname(os.path.abspath(self.file))): + os.mkdir(os.path.dirname(os.path.abspath(self.file))) + if self.immediately_visiable: + with open(self.file, 'a') as f: + f.write(msg) + else: + if self.f is None: + self.f = open(self.file, 'w') + self.f.write(msg) + + def flush(self): + self.console.flush() + if self.f is not None: + self.f.flush() + import os + os.fsync(self.f.fileno()) + + def close(self): + self.console.close() + if self.f is not None: + self.f.close() + +def find_index(seq, item): + for i, x in enumerate(seq): + if item == x: + return i + return -1 + +def set_devices(sys_device_ids): + """ + Args: + sys_device_ids: a tuple; which GPUs to use + e.g. sys_device_ids = (), only use cpu + sys_device_ids = (3,), use the 4-th gpu + sys_device_ids = (0, 1, 2, 3,), use the first 4 gpus + sys_device_ids = (0, 2, 4,), use the 1, 3 and 5 gpus + """ + import os + visiable_devices = '' + for i in sys_device_ids: + visiable_devices += '{}, '.format(i) + os.environ['CUDA_VISIBLE_DEVICES'] = visiable_devices + # Return wrappers + # Models and user defined Variables/Tensors would be transferred to + # the first device + device_id = 0 if len(sys_device_ids) > 0 else -1 + +def transfer_optims(optims, device_id=-1): + for optim in optims: + if isinstance(optim, torch.optim.Optimizer): + transfer_optim_state(optim.state, device_id=device_id) + +def transfer_optim_state(state, device_id=-1): + for key, val in state.items(): + if isinstance(val, dict): + transfer_optim_state(val, device_id=device_id) + elif isinstance(val, Variable): + raise RuntimeError("Oops, state[{}] is a Variable!".format(key)) + elif isinstance(val, torch.nn.Parameter): + raise RuntimeError("Oops, state[{}] is a Parameter!".format(key)) + else: + try: + if device_id == -1: + state[key] = val.cpu() + else: + #state[key] = val.cuda(device=device_id) + state[key] = val.npu(device=device_id) + except: + pass + + +def load_state_dict(model, src_state_dict): + """ + copy parameter from src_state_dict to model + Arguments: + model: A torch.nn.Module object + src_state_dict: a dict containing parameters and persistent buffers + """ + from torch.nn import Parameter + dest_state_dict = model.state_dict() + for name, param in src_state_dict.items(): + if name not in dest_state_dict: + continue + if isinstance(param, Parameter): + param = param.data + try: + dest_state_dict[name].copy_(param) + except Exception: + print("Warning: Error occurs when copying '{}'".format(name)) + + src_missing = set(dest_state_dict.keys()) - set(src_state_dict.keys()) + if len(src_missing) > 0: + print ("Keys not found in source state_dict: ") + for n in src_missing: + print('\t', n) + + dest_missint = set(src_state_dict.keys()) - set(dest_state_dict.keys()) + if len(dest_missint): + print ("Keys not found in destination state_dict: ") + for n in dest_missint: + print('\t', n) + +def load_ckpt(modules_optims, ckpt_file, load_to_cpu=True, verbose=True): + """ + load state_dict of module & optimizer from file + Args: + modules_optims: A two-element list which contains module and optimizer + ckpt_file: the check point file + load_to_cpu: Boolean, whether to transform tensors in model & optimizer to cpu type + """ + map_location = (lambda storage, loc: storage) if load_to_cpu else None + ckpt = torch.load(ckpt_file, map_location=map_location) + for m, sd in zip(modules_optims, ckpt['state_dicts']): + m.load_state_dict(sd) + if verbose: + print("Resume from ckpt {}, \nepoch: {}, scores: {}".format( + ckpt_file, ckpt['ep'], ckpt['scores'])) + return ckpt['ep'], ckpt['scores'] + +def save_ckpt(modules_optims, ep, scores, ckpt_file): + """ + save state_dict of modules/optimizers to file + Args: + modules_optims: a two-element list which contains a module and a optimizer + ep: the current epoch number + scores: the performance of current module + ckpt_file: the check point file path + Note: + torch.save() reserves device type and id of tensors to save. + So when loading ckpt, you have to inform torch.load() to load these tensors + to cpu or your desired gpu, if you change devices. + """ + state_dicts = [m.state_dict() for m in modules_optims] + ckpt = dict(state_dicts = state_dicts, + ep = ep, + scores = scores) + if not os.path.exists(os.path.dirname(os.path.abspath(ckpt_file))): + os.mkdir(os.path.dirname(os.path.abspath(ckpt_file))) + torch.save(ckpt, ckpt_file) + +def adjust_lr_staircase(param_groups, base_lrs, ep, decay_at_epochs, factor): + """ Multiplied by a factor at the beging of specified epochs. Different + params groups specify thier own base learning rates. + Args: + param_groups: a list of params + base_lrs: starting learning rate, len(base_lrs) = len(params_groups) + ep: current epoch, ep >= 1 + decay_at_epochs: a list or tuple; learning rates are multiplied by a factor + at the begining of these epochs + factor: a number in range (0, 1) + Example: + base_lrs = [0.1, 0.01] + decay_at_epochs = [51, 101] + factor = 0.1 + Note: + It is meant to be called at the begining of an epoch + """ + assert len(base_lrs) == len(param_groups), \ + 'You should specify base lr for each param group.' + assert ep >= 1, "Current epoch number should be >= 1" + + if ep not in decay_at_epochs: + return + + ind = find_index(decay_at_epochs, ep) + for i, (g, base_lr) in enumerate(zip(param_groups, base_lrs)): + g['lr'] = base_lr * factor ** (ind + 1) + print('=====> Param group {}: lr adjusted to {:.10f}' + .format(i, g['lr']).rstrip('0')) + +def adjust_lr(optimizer, ep, finetuned_params_lr): + """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" + # lr = args.lr * (0.1 ** (ep// 30)) + lr = finetuned_params_lr * (0.96 ** (ep // 8)) #decreasing the learning rate by 4% every 8 epoch + for param_group in optimizer.param_groups: + param_group['lr'] = lr + +def may_set_mode(maybe_modules, mode): + """ + maybe_modules, an object or a list of objects. + """ + assert mode in ['train', 'eval'] + if not is_iterable(maybe_modules): + maybe_modules = [maybe_modules] + for m in maybe_modules: + if isinstance(m, torch.nn.Module): + if mode == 'train': + m.train() + else: + m.eval() diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/dataset/demo/.gitsave b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/dataset/demo/.gitsave new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/dataset/demo/demo_image.png b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/dataset/demo/demo_image.png new file mode 100644 index 0000000000000000000000000000000000000000..d017bda085608c70900602193c0a5cb0ef146c2d GIT binary patch literal 8645 zcmV;$Av)fPP)k@wC;*_r?|-${-`|q~n5n8$tFiIpI5KB3 zHZVm6P(<(^nh+Tgih>73G@@!!resW|h?XoOswN=@^oUqQ06;PpoWY9aC|f(0006201=rP7^tcu0Dze}P(wuToeD6s;|OZgl!r-k2BxT> zifDu&Dz(&FMWohBOwM@*L={jgmIbi!?e7B;{JZZy1~UUQ%Tka3xcPh@V^E`Q(}rIr(w?Y7%(-}imrlR-);IDc|-5<+m( zl)Tr)Vo{0=hY^vdY1;4iwHh%fV?_f610ZsY;7C2O6-9$UL8}B8V;9=CRfW}ZK2C%8 zZa$xz=x(=bLcF}Zoc4zh0s|tVCw9jxaA-nfirf8GfftLFR@<&OCBw<8(OR_@0Af?m zn>7pp>#b3|-sz{k&VtZk6Uq!OBGtvRP@AcSYno;`W;#LT8?S}Ycfv|g{zmMc{S zROcKb=2{rZnHiwQwppE?oSd9AO(P;Af&isd&n`Oef>#8S0*;9o(7bnz_eR*YO&+Ji zcB>+R`Nu#0@h6{r(skW8-+Xg@eH~-$XZ_)DIHn_Hcl>A(sZw0iG)M*jAp{~=@3xKW z+88?rXBSNysBIeOqZ$I4G9j7}l6MB;*+cYY95(AsU^iPXe)#<#bX`9TL#;K0z|7wJ z)oL~F_O(`2i!mN^8~`}A3Ib3{DR~^naZIyDx;8$4g3msDzMF(Sp+i=1LdFDWVq^v) zDo}&-AdrT{9d}*V_w9b%1KDb|xVpL+hQWC{Ib9C>tw?bMNQP(t3V<%>?3|M+&FAy3 z?dS95`eC@gzYoFPZ*DI?{HP#KIY9ubC?*b}3w})HaM*hvF|?b9-R*V#=wtrLPk!QK z2$~-r?hey|8T(nA)09$r{P?PgalhHVeECwVJi5B-ySYO|6<93hL{3CXDMZAKF-8t< zGwjr{0s%35qW1-u#n6phQqHM5=N>)oKmGL6Cr_Rglo;cfM^~NJVgSe(f=2>GKm_Nw zs;O$J$vNkw8Z(RNe!nMjrC5lLnSiO*TG0TRiHS+UR8Y-Cq^j)>`%)DEFCJYiPL}(_ zp`uhRpo(U-3Nv?I--I~q_C(C=LWl$eh^4CY-XrKVO(K?4iLpiS29Rs1gEf6m%;Y>W znPIK96wOlSixoQ0&h3Un({vY?7fAHGZ(jK5kU_MRQdISrvlp|umP&{rgud^K%ZMl< zPI58TY-one96}%rh^VSri#Gsba4ZN2#OSMub98obe$Je$F3KSr@Onmv5d@! z4z*S@lk&dUd+!Oc_|i0uBZed0D5Z!T#RBANlfFbp-9dDly=W=2GaXlBgpj!Dir7ee$d06-~H ziV#uLG~#_>G9VxXLS!Nr9O9Id9iS)2J`}0D-EOMV^j&mbq*SYjNB{(=s-=`8uc|4( zr#+Dm+hG9R}TtnyQvy%x(83h20MV(e)rmkxhs0z5MDNpP5)&+lZ zasn#F6b*gg#<#>!v*d9Qv7GaiYbm7`5fO5p1Z+st?d|P$yA`pVbibS0W_EV^c-Uo| z=zRGo*gOr0Gz(#+;sKl8q53%M`Xy3GlX{9X-?!k)kR5q8E@LUF60jD%zhD3Mw_oKV z7gvv-fB2o%`2~eo6vAbl zb4r({)!*u7)uU)99P%98CvtO+&ze(6p-7aU2ybIP%^z^Dqp@Fm2l+qJlka z)|<^{v)N43Zv9=A6Cv&eK2q-EVGg-o1NwgJ{d;$!fJ~LU2<~WvYGOzk2nm?^Yi? z|9G)jOv9+EAAR^Cw6nYY(6wxy^0*81q**M+G`)KD#%5h$*LMv8s&gu$AO_ksQAA3b zV&l)xmWTcL>g6|}busJO_)MhaTp#Xk(>OQ)@G%_s__#^aVHn4;mI9!N=mXCeGbIGj zVc6$d_QT*kBce&k6GqQS&^N7fe80XUM+Bf~$ESf-6EPDJ(OSUFOc*UV=YsRz#afk+ zW78PIVHyvIgGeO<6YZMb*a?DXHw}Z0X*Ui&hH084Wi)UE(TBe6LK8|U4oIZD7e}S0 zlyaVq!m~(K)uUzufcHSEb!ywT?|T54rYWTa05QhVjG0^4W|gC=CZe`&#S{R<460PE z(=blM=z?qerj(M>I1IxP&_^ZKw(T@c$9gIvh*)cNDJ3u>qS(i_X-pxPqF_YqeVnSP znVOoJnZ+3A%O!Zef7mol3#6A@QcaxHV!W1&q+q*g{^CPX{dc~v#AqjnV$D<)EE%~i6h zKQ4)>5~9>f4x6T_wE{rbHNJ_6IF&q=oKh;K9KR}r;9BQ%E~sWzMWj}#B1SAINz@!G zq6wHIAgrpnmNb>V?_=9|?}b2%2%4Fg>0vyyZA7LJ0wOXJE0mHgi=|qoWTNQ65l0`~ zyoqtK%EiXRw46ukeW{iepotUVLA5Yu0tHAW!!%O#L=Dq)EW$eb1u%hYhOD`k71>5+InC>Hy!nDF9S! z4WS{zqcp84=Q3IGA`)UZsSyz)fM`uQ5jpS2aSS2Q60s{m#oQbP!ySXEpIp;q2$=yd_M2{{s@vI z*AF3h?~w^=B_T#60BG92>$GGh^4{0_K8c>5hR2T|0|TPP7zN+^iZz7BabWLV@Hts2 zMZ`FuCzf1F%C338-lddWN=k{nJK`Sky_i0#>iK-0rh&oQw%PCZufG1~-P`NS%ggiS z*^?(v?$&qL*Vnby3YJoGP17`S=EDLhI%0sDQ!UBZs)-)<+bNgcI{+ebK#mlUU1&QW zLhwvPPV0MqVY*!OZ(ctT(#6Ti`uhIGmtTy#;j@pvvzjk2FE2};-WDAX6FSx+&a-pG z`|Y-hUo0;7n`v|R(8V}EIh)3ev#QP`A6UVUMf>)1A`%mQxVIi z)lhEk?uca;+lSjX-+cMIMeBa};rIJ6J3GC&xw%c-(HS2$1MtD=+V{L4Aa}|+tvB`$?_x{ot#~rTwXa(wdS^4 zFlh|jIP?sr`R@L8tyOAf->zbRvOH;;X56J(bR0(!o6S26Za+7RV|>68v;hESsi)074QqHfkN(-Nu#q2K}{ZnxW4uU~5g&+b?~j}ANm zxM3IwgZG|@&`1@gDVI_a+4~Sid#!)_dgaevtDwN5c`6QPz# z^2CsGu2f2$1hDLyrDa{O*HZK-luAu17^dHR{^e$Kw^=`^q`nQyMb`vk64Jx8-|IASwo&<{J@v-JFq9ypYKTY#h+ymj5$*QF{xH;{Kn6f+sA|YgR76peV9hdZ zy?`LOrZr+C5HulG1R)eKM+Sji6JlT|A^_OMZr%$3l~SDs8Q6;dAaln18}&ABcnGDH$WMH2!;d*7>}xgUoL5sosml#+8*RRAIe z06-*S1|%^Fj9}7*wr#?!4O$bEnMq0lR@(EGdZ)e_Z9n8%q$*GsVs?y(f&gd=V1NeT z5Sd7)>Cm<6y$8a>ls3EFS;5o#hr|J6 zq^4v@6?R&RFe%7~rc=P3)`PKVj$WjCKyL;=CzhFZsz zYAz{_P>XY6zC2}zHKoHe3<;(z0Ei)|<}@9e5EqNt&GlR7BN(MpRV;+Yvm+nF>DdMQ zfb0z@R~c)rrshb&oO5twaRkS`M>JwM%BACeix`*#U;y3@BRcPVG(}bQ!Nn#Jib|?7 zp;bp1LSOJj4j;-6HTdoLItV*p?Rn^Q48Gw$a^YK0M zG__qMu$;4*&HCQ68^?XEwJ3Pc(KpV;qk&ArFs!$mV}nw_Qc5Yo(IPT45IvIUTI)Eb zoNCTF=X|Vw$7Z>LnHeG?;?WJRRcozn+g@B;G)>rSHq5N5#}`6GR5LSod3m|r4FJH5 zSglB{1)K*$XNWQM-bExrGa@hs0s;Y%!i+#b?;#*)cJ!l5DK%y1+>nqLen%Z1d73=2$&K&bRM*d92Xf4;CS!> zkNw3oO{j8qx_tWd>C>lATjzEU_usyJQERQKN|d8yMkLzpcDvm!=Ui)bNN8XROcb0m zGc-XHP_4DrT1zRlYAq^a$DKwzlH>QI2IDv$4hJ*q`~FB;YOTxV@@RE{+R^GGB2{&0 z*fqVBii)bJ3V?vb#!gIX&RD9N<~&U$A-MvYb7CeobjF|tT7d|WRAnluh+xa?{b9)C z_-@#3Q_)g&s^Zykj6V8lB1y@u`ej&kZMXE)zIb)L+m-XHi*|878D}5>+8=fy#(CGo z;B%>CPTq(0c7x1?82}8yImcZaeEZwqe0_O&d4Bom(c_Q$#q4m{O)zDv(9AIm`(e1- zZ&ONn7(ms@RHr;Sn#V|l-jw%;so-c~AxP{95UN#8qXFeK)sm~JC(@&X1i+pZt<);2 zkFTD@wy9MP;|N4U%8Fre+O_=>{CyG~r->CLXVB_$QerKoF#9&NOjJwJaqJbEwzDP} zs%R-vftZ_spfyeGT*)bwj6^OtX3i<8Y7@KNc2}+X7-q9sDVd!`zdSoTd$_;e-LF$h z4Y^~Yb^r)KW^`1b-bGVn@6l2>oEpwbw-EQ~8A*&FxH?#d=+zi7gC4*{U*EjQ%W$arq9nz$# zAp{~iHlobTvpX8chyyd ziig8tO9$!LcTL~;)61**$?34&rIb=ip4i10oL3VE?2w4vVa$$rO6j-1{q5D&<;lsi zX}S<2XpSMwx_+PY`%a-FLSi7-EzX*z10aXNhbDBrdfJzqM9GDIHak5zzr4I$tyai9 zP1A8O#}g3(1A;1{2G8tp-Y*>a*WbSR>bGC5*IOUs$;qjA@#xpKZR?yva&SzMi{;7o zVZFHe&i%XlmtVge$-i1(FS{mr_FaE|b$NMl(KI2(c+74t_?)vRc3^=C%m9@Q%nS|O zaYU%t;c&RUyJft(9cu%9QGsRyhvy02iM?|pSZN@6mdsQ8V!R&oWc+Vyrsu~h` zU?%U`wIOPi!)}iVrbvXMaIB%uxe!9h_I|X;L;;|UzHMV8;%2keYIS;bMMR8LY95Ax z09W&cLpD(&M?_4Xj?+E>#0)$i4;+F?!9Hq%%0e@V`k?Zm>{A798G6Cav1MY$v5|R z=8PCrMFoy+HzXrQW~LB=O3i76!~XjEdVkn9O>=p9xtcGV*e&LB04=37ZHO_JoKs3i zcl}78k3(N(YKBKna>V*^G;JKmX_}65C&u{u>X}j!5r=A})>+eANe`R#G?kMWFHX-c z&d#UkkZ89%><)V-@0>f1Ssj)1aeRfDP3_1#5YcfkgBYWNAyLlN4BMu!WhCdiruFQ! zT85IRhN8y@96t-p_uvIJ?Xh z-n_mUh8$dsK7t~F#u!A>&F!1F?_RNkdEc4pcDqM)vv$^ZU9A}%R*vu?KfY)`qp`Mc`DI6H5NhNb=`6`Tb^-@Rb(8;Tun%u5HHSFedll9 zy<)Q1(04w1*0;Xok<7X#BAMo5h9}F_K2J3p0<`l*H;Y>H-OZb;%k#~{?cMEL!1ALX zfAaj%6V~F!G{v?w%*Rven2)GqW@cQi=G$=_J++NX`EY&n=KAJaDdW|%$HV?Gj$<3U zu5WU#C5>t^wS^z;0IUdCl1W@eUFH0S|kBz*EfsBqVM~@ z>t=bF#ywf~4hV|Y0VMa)ExI^6m&Do+moyWU3FC-_|G!6TvlyV$C>SoY24ksd3p2Z zo4ebaLmxkW{vou^a{v^r)ig^@>s?wd9<64}r_Vlo_Uzea*i@L*#wqRCYTgftP_74jWfWVGptIcDsDP-L3QH=G}EC+M|g{Y&?2QRBV7p$weR~M)>iqK2_kZ;L#~(ir5kLRc-~Ow={TDA@e!H5-k3V?YdY7j| zWTjbj+wGRz0_L4=hIiN3JA+^U=3ifZ@X@oDTrg+}P}dLpcW+)k+}~WBEkFD2M;|LY-Ez^NEY2O(aje@^fBDP5d2)I3CqMt`AO6W7!^}b2{PO?)=P$qd z;B z|MD-#B7gs@FaFp6{ja|I@~%$n$CpdVJGR-UpMCuN+2g^JkY=kAC@gzk2@U>0x*n$Ia}t^|Ltd zLSSedbuqLqpb0A1THSEi5y=-{|7P{Ucfa>%|KZ8^egGw(UM!nFeErpLub!Wru1>!H z*(X2x;rFi2Rt+WeQx#>bB?&QBK?=O^=7hiEk$Xmw^{M2nNT4+v!s!^7wQ=5HSEUN4vN{A~5tfA=qQ z8J<46;Lx8{uv&9^VU z0?8-yCQrLO?z+~Wp3Iu)nGnHHRBA?0hele{`T1q>{{H6m-OFJ(Y_C3i_Q7YL{V2Gn z&mP^}{K?Im*Q@1nyM8be=jiY-uG*+}9HDh4rB<0V8Nwg_`1{Lw^W~Rc5~egfc;=?T ze`tzUt@b>y` z9mhA{zAXFgv&WC(8Pt*iD8%4>K*Ul6sshs&zy3FmpFTQU{dgD#0?|@tP1E=N{q0Rj zW6l{7`?m2-!_01Z{-{}%)AL6!-oE`8|JVQYFaP%4>KxK~oZj94?2mu)=-I{ZzWu7! z92A|O!h;`~&`2@wOc= 0: + print ('%s: %.2f'%(cfg.att_list[idx], score[0, idx])) + +# show the score in the image +img = img.resize(size=(256, 512), resample=Image.BILINEAR) +draw = ImageDraw.Draw(img) +positive_cnt = 0 +for idx in range(len(cfg.att_list)): + if score[0, idx] >= 0: + txt = '%s: %.2f'%(cfg.att_list[idx], score[0, idx]) + draw.text((10, 10 + 10*positive_cnt), txt, (255, 0, 0)) + positive_cnt += 1 +img.save('./dataset/demo/demo_image_result.png') diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/train_deepmar_resnet50.py b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/train_deepmar_resnet50.py new file mode 100644 index 0000000..52af411 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/train_deepmar_resnet50.py @@ -0,0 +1,483 @@ +import sys +import os +import numpy as np +import random +import math + +import torch +import torch.optim as optim +import torchvision.transforms as transforms +import torch.nn.functional as F +import torch.backends.cudnn as cudnn +from torch.autograd import Variable +from torch.nn.parallel import DataParallel +import pickle +import time +import argparse +import pdb +import sys +from torch.utils.tensorboard import SummaryWriter + +from baseline.dataset import add_transforms +from baseline.dataset.Dataset import AttDataset +from baseline.model.DeepMAR import DeepMAR_ResNet50 +from baseline.model.DeepMAR import DeepMAR_ResNet50_ExtractFeature +from baseline.utils.evaluate import attribute_evaluate +from baseline.utils.utils import str2bool +from baseline.utils.utils import transfer_optim_state +from baseline.utils.utils import time_str +from baseline.utils.utils import save_ckpt, load_ckpt +from baseline.utils.utils import load_state_dict +from baseline.utils.utils import ReDirectSTD +from baseline.utils.utils import adjust_lr_staircase +from baseline.utils.utils import adjust_lr +from baseline.utils.utils import set_devices +from baseline.utils.utils import AverageMeter +from baseline.utils.utils import to_scalar +from baseline.utils.utils import may_set_mode +from baseline.utils.utils import may_mkdir +from baseline.utils.utils import set_seed +from baseline.utils.utils import seed_everything + +# Apex +import numpy as np +from apex import amp +import torch.npu + +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter + +hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] +cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("pytorch") +config_info = get_model_parameter("pytorch_config") +initinal_data = {"base_lr": 0.1, "dataset": "imagenet", "optimizer": "SGD", "loss_scale": 1024} +hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) +hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) +hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) +hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) +hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) +hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) +hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) +hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) +hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) +hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + +CALCULATE_DEVICE = "npu:7" +PRINT_DEVICE = "cpu" + +class Config(object): + def __init__(self): + + parser = argparse.ArgumentParser() + parser.add_argument('--npu', type=int, default=0, help='NPU id to use.') + parser.add_argument('--set_seed', type=str2bool, default=False) + ## dataset parameter + parser.add_argument('--dataset', type=str, default='peta', + choices=['peta','rap', 'pa100k', 'rap2']) + parser.add_argument('--save_dir', type=str, default='/home/zhusiyi/dataset/peta/') + parser.add_argument('--split', type=str, default='trainval', + choices=['trainval', 'train']) + parser.add_argument('--test_split', type=str, default='test') + parser.add_argument('--partition_idx', type=int, default=0) + parser.add_argument('--resize', type=eval, default=(224, 224)) + parser.add_argument('--mirror', type=str2bool, default=True) + parser.add_argument('--batch_size', type=int, default=32) + parser.add_argument('--workers', type=int, default=2) + # model + parser.add_argument('--num_att', type=int, default=35) + parser.add_argument('--pretrained', type=str2bool, default=True) + parser.add_argument('--last_conv_stride', type=int, default=2, choices=[1,2]) + parser.add_argument('--drop_pool5', type=str2bool, default=True) + parser.add_argument('--drop_pool5_rate', type=float, default=0.5) + + parser.add_argument('--sgd_weight_decay', type=float, default=0.0005) + parser.add_argument('--sgd_momentum', type=float, default=0.9) + parser.add_argument('--new_params_lr', type=float, default=0.001) + parser.add_argument('--finetuned_params_lr', type=float, default=0.001) + parser.add_argument('--staircase_decay_at_epochs', type=eval, + default=(51, )) + parser.add_argument('--staircase_decay_multiple_factor', type=float, + default=0.1) + parser.add_argument('--total_epochs', type=int, default=150) + parser.add_argument('--weighted_entropy', type=str2bool, default=True) + # utils + parser.add_argument('--resume', type=str2bool, default=False) + parser.add_argument('--ckpt_file', type=str, default='') + parser.add_argument('--load_model_weight', type=str2bool, default=False) + parser.add_argument('--model_weight_file', type=str, default='') + parser.add_argument('--test_only', type=str2bool, default=False) + parser.add_argument('--exp_dir', type=str, default='') + parser.add_argument('--exp_subpath', type=str, default='deepmar_resnet50') + parser.add_argument('--log_to_file', type=str2bool, default=True) + parser.add_argument('--steps_per_log', type=int, default=20) + parser.add_argument('--epochs_per_val', type=int, default=10) + parser.add_argument('--epochs_per_save', type=int, default=50) + parser.add_argument('--run', type=int, default=1) + # apex + parser.add_argument('--amp', default=False, action='store_true', + help='use amp to train the model') + parser.add_argument('--loss_scale', default=-1., type=float, + help='loss scale using in amp, default -1 means dynamic') + parser.add_argument('--opt_level', default='O1', type=str, + help='opt level using in amp, default O1 means FP16') + args = parser.parse_args() + + # gpu ids + self.npu = args.npu + # random + self.set_seed = args.set_seed + if self.set_seed: + self.seed = 0 + else: + self.seed = None + # amp + self.amp = args.amp + self.loss_scale = args.loss_scale + self.opt_level = args.opt_level + # run time index + self.run = args.run + # Dataset # + datasets = dict() + #datasets['peta'] = '/home/zhusiyi/dataset/peta/peta_dataset.pkl' + #datasets['peta'] = os.path.join(os.path.abspath(os.path.dirname(__file__)),'dataset/peta/peta_dataset.pkl') + datasets['peta'] = args.save_dir + '/peta_dataset.pkl' + partitions = dict() + #partitions['peta'] = '/home/zhusiyi/dataset/peta/peta_partition.pkl' + #partitions['peta'] = os.path.join(os.path.abspath(os.path.dirname(__file__)),'dataset/peta/peta_partition.pkl') + partitions['peta'] = args.save_dir + '/peta_partition.pkl' + + self.dataset_name = args.dataset + if args.dataset not in datasets or args.dataset not in partitions: + print ("Please select the right dataset name.") + raise ValueError + else: + self.dataset = datasets[args.dataset] + self.partition = partitions[args.dataset] + self.partition_idx = args.partition_idx + self.split = args.split + self.test_split = args.test_split + self.resize = args.resize + self.mirror = args.mirror + self.mean = [0.485, 0.456, 0.406] + self.std = [0.229, 0.224, 0.225] + self.batch_size = args.batch_size + self.workers = args.workers + # optimization + self.sgd_momentum = args.sgd_momentum + self.sgd_weight_decay = args.sgd_weight_decay + self.new_params_lr = args.new_params_lr + self.finetuned_params_lr = args.finetuned_params_lr + self.staircase_decay_at_epochs = args.staircase_decay_at_epochs + self.staircase_decay_multiple_factor = args.staircase_decay_multiple_factor + self.total_epochs = args.total_epochs + self.weighted_entropy = args.weighted_entropy + + # utils + self.resume = args.resume + self.ckpt_file = args.ckpt_file + if self.resume: + if self.ckpt_file == '': + print ('Please input the ckpt_file if you want to resume training') + raise ValueError + self.load_model_weight = args.load_model_weight + self.model_weight_file = args.model_weight_file + if self.load_model_weight: + if self.model_weight_file == '': + print ('Please input the model_weight_file if you want to load model weight') + raise ValueError + self.test_only = args.test_only + self.exp_dir = args.exp_dir + self.exp_subpath = args.exp_subpath + self.log_to_file = args.log_to_file + self.steps_per_log = args.steps_per_log + self.epochs_per_val = args.epochs_per_val + self.epochs_per_save = args.epochs_per_save + self.run = args.run + + # for model + model_kwargs = dict() + model_kwargs['num_att'] = args.num_att + model_kwargs['last_conv_stride'] = args.last_conv_stride + model_kwargs['drop_pool5'] = args.drop_pool5 + model_kwargs['drop_pool5_rate'] = args.drop_pool5_rate + self.model_kwargs = model_kwargs + # for evaluation + self.test_kwargs = dict() + + if self.exp_dir == '': + self.exp_dir = os.path.join('exp', + '{}'.format(self.exp_subpath), + '{}'.format(self.dataset_name), + 'partition{}'.format(self.partition_idx), + 'run{}'.format(self.run)) + self.stdout_file = os.path.join(self.exp_dir, \ + 'log', 'stdout_{}.txt'.format(time_str())) + self.stderr_file = os.path.join(self.exp_dir, \ + 'log', 'stderr_{}.txt'.format(time_str())) + may_mkdir(self.stdout_file) + +### main function ### +cfg = Config() + +# log +if cfg.log_to_file: + ReDirectSTD(cfg.stdout_file, 'stdout', False) + ReDirectSTD(cfg.stderr_file, 'stderr', False) + +# dump the configuration to log. +import pprint +print('-' * 60) +print('cfg.__dict__') +pprint.pprint(cfg.__dict__) +print('-' * 60) + +# set the random seed +print(cfg.seed) +if cfg.set_seed: + set_seed(cfg.seed) + seed_everything(cfg.seed) + +# init the npu ids +CALCULATE_DEVICE = "npu:{}".format(cfg.npu) +torch.npu.set_device(CALCULATE_DEVICE) + +# dataset +normalize = transforms.Normalize(mean=cfg.mean, std=cfg.std) +transform = transforms.Compose([ + transforms.Resize(cfg.resize), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), # 3*H*W, [0, 1] + normalize,]) # normalize with mean/std +# by a subset of attributes +train_set = AttDataset( + dataset = cfg.dataset, + partition = cfg.partition, + split = cfg.split, + partition_idx= cfg.partition_idx, + transform = transform) + +num_att = len(train_set.dataset['selected_attribute']) +cfg.model_kwargs['num_att'] = num_att + +train_loader = torch.utils.data.DataLoader( + dataset = train_set, + batch_size = cfg.batch_size, + shuffle = True, + num_workers = cfg.workers, + pin_memory = True, + drop_last = True) + +test_transform = transforms.Compose([ + transforms.Resize(cfg.resize), + transforms.ToTensor(), + normalize,]) + +test_set = AttDataset( + dataset = cfg.dataset, + partition = cfg.partition, + split = cfg.test_split, + partition_idx = cfg.partition_idx, + transform = test_transform) + +### Att model ### +model = DeepMAR_ResNet50(**cfg.model_kwargs) + +# Optimizer +finetuned_params = [] +new_params = [] +for n, p in model.named_parameters(): + if n.find('classifier') >=0: + new_params.append(p) + else: + finetuned_params.append(p) +param_groups = [{'params': finetuned_params, 'lr': cfg.finetuned_params_lr}, + {'params': new_params, 'lr': cfg.new_params_lr}] + +optimizer = optim.SGD( + param_groups, + momentum = cfg.sgd_momentum, + weight_decay = cfg.sgd_weight_decay) + +model = model.to(CALCULATE_DEVICE) +# apex +if cfg.amp: + # Initialization + model, optimizer = amp.initialize(model, optimizer, opt_level=cfg.opt_level, loss_scale=cfg.loss_scale) + print("=> Using amp mode.") + + +# using the weighted cross entropy loss +if cfg.weighted_entropy: + rate = np.array(train_set.partition['weight_' + cfg.split][cfg.partition_idx]) + rate = rate[train_set.dataset['selected_attribute']].tolist() +else: + rate = None +# compute the weight of positive and negative +if rate is None: + weight_pos = [1 for i in range(num_att)] + weight_neg = [1 for i in range(num_att)] +else: + if len(rate) != num_att: + print ("the length of rate should be equal to %d" % (num_att)) + raise ValueError + weight_pos = [] + weight_neg = [] + for idx, v in enumerate(rate): + weight_pos.append(math.exp(1.0 - v)) + weight_neg.append(math.exp(v)) + +# bind the model and optimizer +modules_optims = [model, optimizer] + +# load model weight if necessary +if cfg.load_model_weight: + map_location = (lambda storage, loc:storage) + ckpt = torch.load(cfg.model_weight_file, map_location=map_location) + model.load_state_dict(ckpt['state_dicts'][0], strict=False) + # print(ckpt['state_dicts'][0]) + +### Resume or not ### +if cfg.resume: + # store the model, optimizer, epoch + start_epoch, scores = load_ckpt(modules_optims, cfg.ckpt_file) +else: + start_epoch = 0 + +model = torch.nn.DataParallel(model) +# model_w.cuda() +transfer_optim_state(state=optimizer.state, device_id=cfg.npu) + +# cudnn.benchmark = True +# for evaluation +feat_func_att = DeepMAR_ResNet50_ExtractFeature(model=model) + +def attribute_evaluate_subfunc(feat_func, test_set, device_id, **test_kwargs): + """ evaluate the attribute recognition precision """ + result = attribute_evaluate(feat_func, test_set, device_id, **test_kwargs) + print ('-' * 60) + print ('Evaluation on %s set:' % (cfg.test_split)) + print ('Label-based evaluation: \n mA: %.4f'%(np.mean(result['label_acc']))) + print ('Instance-based evaluation: \n Acc: %.4f, Prec: %.4f, Rec: %.4f, F1: %.4f' \ + %(result['instance_acc'], result['instance_precision'], result['instance_recall'], result['instance_F1'])) + print ('-' * 60) + hwlog.remark_print(key=hwlog.ACC, value="{:.4f}".format(result['instance_acc'])) + hwlog.remark_print(key=hwlog.PREC, value="{:.4f}".format(result['instance_precision'])) + hwlog.remark_print(key=hwlog.REC, value="{:.4f}".format(result['instance_recall'])) + hwlog.remark_print(key=hwlog.F1, value="{:.4f}".format(result['instance_recall'])) + return result['instance_acc'] + + +# print the model into log +# test only +if cfg.test_only: + print ('test with feat_func_att') + attribute_evaluate_subfunc(feat_func_att, test_set, **cfg.test_kwargs) + sys.exit(0) + + +# writer = SummaryWriter(os.path.join('runs/deepmar', str(cfg.npu))) + +# training +for epoch in range(start_epoch, cfg.total_epochs): + if cfg.seed is not None: + cfg.seed += 1 + seed_everything(cfg.seed) + # adjust the learning rate + adjust_lr_staircase( + optimizer.param_groups, + [cfg.finetuned_params_lr, cfg.new_params_lr], + epoch + 1, + cfg.staircase_decay_at_epochs, + cfg.staircase_decay_multiple_factor) + # adjust_lr(optimizer,epoch+1,cfg.finetuned_params_lr)n + + may_set_mode(modules_optims, 'train') + # recording loss + loss_meter = AverageMeter() + dataset_L = len(train_loader) # crop batch data + ep_st = time.time() + ep_st_mark=ep_st + # runing every batch data + for step, (imgs, targets) in enumerate(train_loader): + + step_st = time.time() + # measure data loading time + data_time = step_st-ep_st + + imgs_var = Variable(imgs) + targets_var = Variable(targets) + + # compute the weight + weights = torch.zeros(targets_var.shape) + for i in range(targets_var.shape[0]): + for j in range(targets_var.shape[1]): + if targets_var.data.cpu()[i, j] == -1: + weights[i, j] = weight_neg[j] + elif targets_var.data.cpu()[i, j] == 1: + weights[i, j] = weight_pos[j] + else: + weights[i, j] = 0 + + targets_var[targets_var == -1] = 0 + targets_var = targets_var.to(CALCULATE_DEVICE) + imgs_var = imgs_var.to(CALCULATE_DEVICE) + weights = weights.to(CALCULATE_DEVICE) + score = model(imgs_var) + + criterion = torch.nn.BCEWithLogitsLoss(weight=Variable(weights)).to(CALCULATE_DEVICE) + loss = criterion(score, targets_var) * num_att + optimizer.zero_grad() + if cfg.amp: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + + optimizer.step() + + + ############ + # step log # + ############ + loss_meter.update(to_scalar(loss)) + # one batch time using backward calculation + batch_time = time.time() - ep_st # include data load time + ep_st = time.time() + fps = cfg.batch_size / batch_time + # do not include data load time + + if (step + 1) % cfg.steps_per_log == 0 or (step + 1) % len(train_loader) == 0: + log = '{}, Step {}/{} in Ep {}, {:.2f}s, datatime:{:.6f}, batchtime:{:.6f}, FPS:{:.2f}, loss:{:.4f}'.format( \ + time_str(), step + 1, dataset_L, epoch + 1, time.time() - step_st, data_time, batch_time, fps, loss_meter.val) + + print(log) + + hwlog.remark_print(key=hwlog.FPS, value='{:.2f}'.format(fps)) + + ############## + # epoch log # + ############## + epoch_time = time.time() - ep_st_mark + log = 'Ep{}, {:.2f}s, loss {:.4f}'.format( + epoch+1, epoch_time, loss_meter.avg) + print(log) + + # writer.add_scalar('Train/Time', epoch_time, epoch+1) + # writer.add_scalar('Train/Loss', loss_meter.avg, epoch+1) + # writer.add_scalar('Train/LR', optimizer.param_groups[0]['lr'], epoch+1) + + + # model ckpt + if (epoch + 1) % cfg.epochs_per_save == 0 or epoch+1 == cfg.total_epochs: + ckpt_file = os.path.join(cfg.exp_dir, 'model', 'ckpt_epoch%d.pth'%(epoch+1)) + save_ckpt(modules_optims, epoch+1, 0, ckpt_file) + + ########################## + # test on validation set # + ########################## + if (epoch + 1) % cfg.epochs_per_val == 0 or epoch+1 == cfg.total_epochs: + print ('att test with feat_func_att') + res = attribute_evaluate_subfunc(feat_func_att, test_set, CALCULATE_DEVICE, **cfg.test_kwargs) + + # writer.add_scalar('Val/Acc', res, epoch) \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/train_deepmar_resnet50_8p.py b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/train_deepmar_resnet50_8p.py new file mode 100644 index 0000000..e8d461f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/train_deepmar_resnet50_8p.py @@ -0,0 +1,587 @@ +import os +import random +import math + +import torch +import torch.optim as optim +import torchvision.transforms as transforms +import torch.nn.functional as F +import torch.backends.cudnn as cudnn +from torch.autograd import Variable +from torch.nn.parallel import DataParallel +import pickle +import time +import argparse +import pdb +import sys +from torch.utils.tensorboard import SummaryWriter + +from baseline.dataset import add_transforms +from baseline.dataset.Dataset import AttDataset +from baseline.model.DeepMAR import DeepMAR_ResNet50 +from baseline.model.DeepMAR import DeepMAR_ResNet50_ExtractFeature +from baseline.utils.evaluate import attribute_evaluate +from baseline.utils.utils import str2bool +from baseline.utils.utils import transfer_optim_state +from baseline.utils.utils import time_str +from baseline.utils.utils import save_ckpt, load_ckpt +from baseline.utils.utils import load_state_dict +from baseline.utils.utils import ReDirectSTD +from baseline.utils.utils import adjust_lr_staircase +from baseline.utils.utils import adjust_lr +from baseline.utils.utils import set_devices +from baseline.utils.utils import AverageMeter +from baseline.utils.utils import to_scalar +from baseline.utils.utils import may_set_mode +from baseline.utils.utils import may_mkdir +from baseline.utils.utils import set_seed +from baseline.utils.utils import seed_everything +import torch.distributed as dist +import torch.multiprocessing as mp +import torch.utils.data.distributed + +# Apex +import numpy as np +from apex import amp +import torch.npu + +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter + + +class Config(object): + def __init__(self): + + parser = argparse.ArgumentParser() + parser.add_argument('--sys_device_ids', type=eval, default=6) + parser.add_argument('--npu', default=None, type=int, help='NPU id to use.') + parser.add_argument('--set_seed', type=str2bool, default=False) + ## dataset parameter + parser.add_argument('--dataset', type=str, default='peta', + choices=['peta','rap', 'pa100k', 'rap2']) + parser.add_argument('--save_dir', type=str, default='/home/zhusiyi/dataset/peta/') + parser.add_argument('--split', type=str, default='trainval', + choices=['trainval', 'train']) + parser.add_argument('--test_split', type=str, default='test') + parser.add_argument('--partition_idx', type=int, default=0) + parser.add_argument('--resize', type=eval, default=(224, 224)) + parser.add_argument('--mirror', type=str2bool, default=True) + parser.add_argument('--batch_size', type=int, default=32) + parser.add_argument('--workers', type=int, default=2) + # model + parser.add_argument('--num_att', type=int, default=35) + parser.add_argument('--pretrained', type=str2bool, default=True) + parser.add_argument('--last_conv_stride', type=int, default=2, choices=[1,2]) + parser.add_argument('--drop_pool5', type=str2bool, default=True) + parser.add_argument('--drop_pool5_rate', type=float, default=0.5) + + parser.add_argument('--sgd_weight_decay', type=float, default=0.0005) + parser.add_argument('--sgd_momentum', type=float, default=0.9) + parser.add_argument('--new_params_lr', type=float, default=0.001) + parser.add_argument('--finetuned_params_lr', type=float, default=0.001) + parser.add_argument('--staircase_decay_at_epochs', type=eval, + default=(51, )) + parser.add_argument('--staircase_decay_multiple_factor', type=float, + default=0.1) + parser.add_argument('--total_epochs', type=int, default=150) + parser.add_argument('--weighted_entropy', type=str2bool, default=True) + # utils + parser.add_argument('--resume', type=str2bool, default=False) + parser.add_argument('--ckpt_file', type=str, default='') + parser.add_argument('--load_model_weight', type=str2bool, default=False) + parser.add_argument('--model_weight_file', type=str, default='') + parser.add_argument('--test_only', type=str2bool, default=False) + parser.add_argument('--exp_dir', type=str, default='') + parser.add_argument('--exp_subpath', type=str, default='deepmar_resnet50') + parser.add_argument('--log_to_file', type=str2bool, default=True) + parser.add_argument('--steps_per_log', type=int, default=20) + parser.add_argument('--epochs_per_val', type=int, default=10) + parser.add_argument('--epochs_per_save', type=int, default=50) + parser.add_argument('--run', type=int, default=1) + # apex + parser.add_argument('--amp', default=False, action='store_true', + help='use amp to train the model') + parser.add_argument('--loss_scale', default=-1., type=float, + help='loss scale using in amp, default -1 means dynamic') + parser.add_argument('--opt_level', default='O1', type=str, + help='opt level using in amp, default O1 means FP16') + # distributed + parser.add_argument('--addr', default='90.90.176.152', type=str, + help='master addr') + parser.add_argument('--world_size', default=-1, type=int, + help='number of nodes for distributed training') + parser.add_argument('--rank', default=-1, type=int, + help='node rank for distributed training') + parser.add_argument('--dist_url', default='tcp://224.66.41.62:23456', type=str, + help='url used to set up distributed training') + parser.add_argument('--dist_backend', default='nccl', type=str, + help='distributed backend') + parser.add_argument('--multiprocessing_distributed', action='store_true', + help='Use multi-processing distributed training to launch ' + 'N processes per node, which has N NPUs. This is the ' + 'fastest way to use PyTorch for either single node or ' + 'multi node data parallel training') + parser.add_argument('--npus_per_node', default=None, type=int, + help='number of npus to use for distributed train on each node') + + args = parser.parse_args() + + # gpu ids + # self.sys_device_ids = args.sys_device_ids + self.npus_per_node=args.npus_per_node + self.npu = args.npu + # random + self.set_seed = args.set_seed + if self.set_seed: + self.seed = 0 + else: + self.seed = None + # amp + self.amp = args.amp + self.loss_scale = args.loss_scale + self.opt_level = args.opt_level + # run time index + self.run = args.run + # Dataset # + datasets = dict() + datasets['peta'] = args.save_dir + '/peta_dataset.pkl' + partitions = dict() + partitions['peta'] = args.save_dir + '/peta_partition.pkl' + + self.dataset_name = args.dataset + if args.dataset not in datasets or args.dataset not in partitions: + print ("Please select the right dataset name.") + raise ValueError + else: + self.dataset = datasets[args.dataset] + self.partition = partitions[args.dataset] + self.partition_idx = args.partition_idx + self.split = args.split + self.test_split = args.test_split + self.resize = args.resize + self.mirror = args.mirror + self.mean = [0.485, 0.456, 0.406] + self.std = [0.229, 0.224, 0.225] + self.batch_size = args.batch_size + self.workers = args.workers + # optimization + self.sgd_momentum = args.sgd_momentum + self.sgd_weight_decay = args.sgd_weight_decay + self.new_params_lr = args.new_params_lr + self.finetuned_params_lr = args.finetuned_params_lr + self.staircase_decay_at_epochs = args.staircase_decay_at_epochs + self.staircase_decay_multiple_factor = args.staircase_decay_multiple_factor + self.total_epochs = args.total_epochs + self.weighted_entropy = args.weighted_entropy + # distributed + self.addr = args.addr + self.world_size = args.world_size + self.rank = args.rank + self.dist_url = args.dist_url + self.dist_backend = args.dist_backend + self.multiprocessing_distributed = args.multiprocessing_distributed + # utils + self.resume = args.resume + self.ckpt_file = args.ckpt_file + if self.resume: + if self.ckpt_file == '': + print ('Please input the ckpt_file if you want to resume training') + raise ValueError + self.load_model_weight = args.load_model_weight + self.model_weight_file = args.model_weight_file + if self.load_model_weight: + if self.model_weight_file == '': + print ('Please input the model_weight_file if you want to load model weight') + raise ValueError + self.test_only = args.test_only + self.exp_dir = args.exp_dir + self.exp_subpath = args.exp_subpath + self.log_to_file = args.log_to_file + self.steps_per_log = args.steps_per_log + self.epochs_per_val = args.epochs_per_val + self.epochs_per_save = args.epochs_per_save + self.run = args.run + + # for model + model_kwargs = dict() + model_kwargs['num_att'] = args.num_att + model_kwargs['last_conv_stride'] = args.last_conv_stride + model_kwargs['drop_pool5'] = args.drop_pool5 + model_kwargs['drop_pool5_rate'] = args.drop_pool5_rate + self.model_kwargs = model_kwargs + # for evaluation + self.test_kwargs = dict() + + if self.exp_dir == '': + self.exp_dir = os.path.join('exp', + '{}'.format(self.exp_subpath), + '{}'.format(self.dataset_name), + 'partition{}'.format(self.partition_idx), + 'run{}'.format(self.run)) + self.stdout_file = os.path.join(self.exp_dir, \ + 'log', 'stdout_{}.txt'.format(time_str())) + self.stderr_file = os.path.join(self.exp_dir, \ + 'log', 'stderr_{}.txt'.format(time_str())) + may_mkdir(self.stdout_file) + +def main(): + ### main function ### + # pdb.set_trace() + cfg = Config() + + # log + if cfg.log_to_file: + ReDirectSTD(cfg.stdout_file, 'stdout', False) + ReDirectSTD(cfg.stderr_file, 'stderr', False) + + # dump the configuration to log. + import pprint + print('-' * 60) + print('cfg.__dict__') + pprint.pprint(cfg.__dict__) + print('-' * 60) + + os.environ['KERNEL_NAME_ID'] = str(0) + print("+++++++++++++++++++++++++++KERNEL_NAME_ID:",os.environ['KERNEL_NAME_ID']) + + # set the random seed + print(cfg.seed) + if cfg.set_seed: + set_seed(cfg.seed) + seed_everything(cfg.seed) + + os.environ['MASTER_ADDR'] = cfg.addr + os.environ['MASTER_PORT'] = '29501' + if cfg.dist_url == "env://" and cfg.world_size == -1: + cfg.world_size = int(os.environ["WORLD_SIZE"]) + + npus_per_node=cfg.npus_per_node + #npus_per_node = torch.npu.device_count() + if cfg.multiprocessing_distributed: + # Since we have ngpus_per_node processes per node, the total world_size needs to be adjusted accordingly + cfg.world_size = npus_per_node * cfg.world_size # world_size means nums of all devices or nums of processes + mp.spawn(main_worker, nprocs=npus_per_node, args=(npus_per_node, cfg)) + +def main_worker(npu, npus_per_node, cfg): + cfg.npu = npu + print("[npu id:", npu, "]", "+++++++++++++++++++++++++++ before set KERNEL_NAME_ID:", os.environ['KERNEL_NAME_ID']) + os.environ['KERNEL_NAME_ID'] = str(npu) + + print("[npu id:", npu, "]", "+++++++++++++++++++++++++++KERNEL_NAME_ID:", os.environ['KERNEL_NAME_ID']) + + if npu is not None: + print("[npu id:", npu, "]", "Use NPU: {} for training".format(npu)) + + if cfg.dist_url == "env://" and cfg.rank == -1: + cfg.rank = int(os.environ["RANK"]) + if cfg.multiprocessing_distributed: + # For multiprocessing distributed training, rank needs to be the + # global rank among all the processes + #cfg.rank = cfg.rank * npus_per_node + npu + cfg.rank = npu + print("rank:",cfg.rank) + dist.init_process_group(backend=cfg.dist_backend, #init_method=cfg.dist_url, + world_size=cfg.world_size, rank=cfg.rank) + + CALCULATE_DEVICE = 'npu:{}'.format(npu) + print(CALCULATE_DEVICE) + torch.npu.set_device(CALCULATE_DEVICE) + + # DistributedDataParallel, we need to divide the batch size + # ourselves based on the total number of NPUs we have + cfg.batch_size = int(cfg.batch_size / npus_per_node) + cfg.workers = int((cfg.workers + npus_per_node - 1) / npus_per_node) + print("batchsize:", cfg.batch_size) + print("workers", cfg.workers) + + # dataset + normalize = transforms.Normalize(mean=cfg.mean, std=cfg.std) + transform = transforms.Compose([ + transforms.Resize(cfg.resize), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), # 3*H*W, [0, 1] + normalize,]) # normalize with mean/std + # by a subset of attributes + train_set = AttDataset( + dataset = cfg.dataset, + partition = cfg.partition, + split = cfg.split, + partition_idx= cfg.partition_idx, + transform = transform) + + num_att = len(train_set.dataset['selected_attribute']) + cfg.model_kwargs['num_att'] = num_att + + distributed = cfg.world_size > 1 or cfg.multiprocessing_distributed + if distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_set) + else: + train_sampler = None + + train_loader = torch.utils.data.DataLoader( + dataset = train_set, + batch_size = cfg.batch_size, + shuffle=(train_sampler is None), + num_workers = cfg.workers, + pin_memory = True, + sampler=train_sampler, + drop_last = True) + + test_transform = transforms.Compose([ + transforms.Resize(cfg.resize), + transforms.ToTensor(), + normalize,]) + + test_set = AttDataset( + dataset = cfg.dataset, + partition = cfg.partition, + split = cfg.test_split, + partition_idx = cfg.partition_idx, + transform = test_transform) + + ### Att model ### + model = DeepMAR_ResNet50(**cfg.model_kwargs) + + # Optimizer + finetuned_params = [] + new_params = [] + for n, p in model.named_parameters(): + if n.find('classifier') >=0: + new_params.append(p) + else: + finetuned_params.append(p) + param_groups = [{'params': finetuned_params, 'lr': cfg.finetuned_params_lr}, + {'params': new_params, 'lr': cfg.new_params_lr}] + + optimizer = optim.SGD( + param_groups, + momentum = cfg.sgd_momentum, + weight_decay = cfg.sgd_weight_decay) + + # model = model.cuda() + model = model.to(CALCULATE_DEVICE) + # apex + if cfg.amp: + # Initialization + model, optimizer = amp.initialize(model, optimizer, opt_level=cfg.opt_level, loss_scale=cfg.loss_scale) + print("=> Using amp mode.") + + # Wrap the model after set_devices, data parallel + # model_w = torch.nn.DataParallel(model) + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[cfg.npu], broadcast_buffers=False) + + # using the weighted cross entropy loss + if cfg.weighted_entropy: + rate = np.array(train_set.partition['weight_' + cfg.split][cfg.partition_idx]) + rate = rate[train_set.dataset['selected_attribute']].tolist() + else: + rate = None + # compute the weight of positive and negative + if rate is None: + weight_pos = [1 for i in range(num_att)] + weight_neg = [1 for i in range(num_att)] + else: + if len(rate) != num_att: + print ("the length of rate should be equal to %d" % (num_att)) + raise ValueError + weight_pos = [] + weight_neg = [] + for idx, v in enumerate(rate): + weight_pos.append(math.exp(1.0 - v)) + weight_neg.append(math.exp(v)) + + # bind the model and optimizer + modules_optims = [model, optimizer] + + # load model weight if necessary + if cfg.load_model_weight: + map_location = (lambda storage, loc:storage) + ckpt = torch.load(cfg.model_weight_file, map_location=map_location) + model.load_state_dict(ckpt['state_dicts'][0], strict=False) + + ### Resume or not ### + if cfg.resume: + # store the model, optimizer, epoch + start_epoch, scores = load_ckpt(modules_optims, cfg.ckpt_file) + else: + start_epoch = 0 + + #model = torch.nn.DataParallel(model) + #transfer_optim_state(state=optimizer.state, device_id=npu) + + # cudnn.benchmark = True + # for evaluation + feat_func_att = DeepMAR_ResNet50_ExtractFeature(model=model) + + # print the model into log + # print (model) + # test only + if cfg.test_only: + print ('test with feat_func_att') + attribute_evaluate_subfunc(feat_func_att, test_set, **cfg.test_kwargs) + sys.exit(0) + + + # writer = SummaryWriter('runs/deepmar/6') + + # training + for epoch in range(start_epoch, cfg.total_epochs): + if cfg.seed is not None: + cfg.seed += 1 + seed_everything(cfg.seed) + + if distributed: + train_sampler.set_epoch(epoch) + # adjust the learning rate + adjust_lr_staircase( + optimizer.param_groups, + [cfg.finetuned_params_lr, cfg.new_params_lr], + epoch + 1, + cfg.staircase_decay_at_epochs, + cfg.staircase_decay_multiple_factor) + # adjust_lr(optimizer,epoch+1,cfg.finetuned_params_lr)n + + may_set_mode(modules_optims, 'train') + # recording loss + loss_meter = AverageMeter() + dataset_L = len(train_loader) # crop batch data + ep_st = time.time() + ep_st_mark=ep_st + # runing every batch data + for step, (imgs, targets) in enumerate(train_loader): + + step_st = time.time() + # measure data loading time + data_time = step_st-ep_st + + imgs_var = Variable(imgs) + targets_var = Variable(targets) + # if 'npu' in CALCULATE_DEVICE: + # targets = targets.to(torch.int32) + # imgs, targets = imgs.to(CALCULATE_DEVICE, non_blocking=True), targets.to(CALCULATE_DEVICE, non_blocking=True) + # compute the weight + weights = torch.zeros(targets_var.shape) + for i in range(targets_var.shape[0]): + for j in range(targets_var.shape[1]): + if targets_var.data.cpu()[i, j] == -1: + weights[i, j] = weight_neg[j] + elif targets_var.data.cpu()[i, j] == 1: + weights[i, j] = weight_pos[j] + else: + weights[i, j] = 0 + + # loss for the attribute classification, average over the batch size + targets_var[targets_var == -1] = 0 + targets_var = targets_var.to(CALCULATE_DEVICE) + imgs_var = imgs_var.to(CALCULATE_DEVICE) + weights = weights.to(CALCULATE_DEVICE) + score = model(imgs_var) + + criterion = torch.nn.BCEWithLogitsLoss(weight=Variable(weights)).to(CALCULATE_DEVICE) + loss = criterion(score, targets_var) * num_att + optimizer.zero_grad() + if cfg.amp: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + # for name, parms in model.named_parameters(): + # print('-->name:', name, ' -->grad_value_max:', torch.max(parms.grad), ' -->grad_value_min:', + # torch.min(parms.grad)) + optimizer.step() + + + ############ + # step log # + ############ + loss_meter.update(to_scalar(loss)) + # one batch time using backward calculation + batch_time = time.time() - ep_st # include data load time + ep_st = time.time() + fps = npus_per_node*cfg.batch_size / batch_time + # do not include data load time + + if (step + 1) % cfg.steps_per_log == 0 or (step + 1) % len(train_loader) == 0: + log = '{}, Step {}/{} in Ep {}, {:.2f}s, datatime:{:.6f}, batchtime:{:.6f}, FPS:{:.2f}, loss:{:.4f}'.format( \ + time_str(), step + 1, dataset_L, epoch + 1, time.time() - step_st, data_time, batch_time, fps, loss_meter.val) + print(log) + + hwlog.remark_print(key=hwlog.FPS, value='{:.2f}'.format(fps)) + + + ############## + # epoch log # + ############## + epoch_time = time.time() - ep_st_mark + log = 'Ep{}, {:.2f}s, loss {:.4f}'.format( + epoch+1, epoch_time, loss_meter.avg) + print(log) + + # writer.add_scalar('Train/Time', epoch_time, epoch+1) + # writer.add_scalar('Train/Loss', loss_meter.avg, epoch+1) + # # writer.add_scalar('Train/Acc', res['instance_acc'], epoch+1) + # writer.add_scalar('Train/LR', optimizer.param_groups[0]['lr'], epoch+1) + + + # model ckpt + if (epoch + 1) % cfg.epochs_per_save == 0 or epoch+1 == cfg.total_epochs: + ckpt_file = os.path.join(cfg.exp_dir, 'model', 'ckpt_epoch%d.pth'%(epoch+1)) + save_ckpt(modules_optims, epoch+1, 0, ckpt_file) + + ########################## + # test on validation set # + ########################## + if (epoch + 1) % cfg.epochs_per_val == 0 or epoch+1 == cfg.total_epochs: + print ('att test with feat_func_att') + res = attribute_evaluate_subfunc(feat_func_att, test_set, CALCULATE_DEVICE, cfg, **cfg.test_kwargs) + + # writer.add_scalar('Val/Acc', res, epoch) + # writer.close() + +def attribute_evaluate_subfunc(feat_func, test_set, device_id, cfg, **test_kwargs): + """ evaluate the attribute recognition precision """ + result = attribute_evaluate(feat_func, test_set, device_id, **test_kwargs) + print ('-' * 60) + print ('Evaluation on %s set:' % (cfg.test_split)) + print ('Label-based evaluation: \n mA: %.4f'%(np.mean(result['label_acc']))) + print ('Instance-based evaluation: \n Acc: %.4f, Prec: %.4f, Rec: %.4f, F1: %.4f' \ + %(result['instance_acc'], result['instance_precision'], result['instance_recall'], result['instance_F1'])) + print ('-' * 60) + hwlog.remark_print(key=hwlog.ACC, value="{:.4f}".format(result['instance_acc'])) + hwlog.remark_print(key=hwlog.PREC, value="{:.4f}".format(result['instance_precision'])) + hwlog.remark_print(key=hwlog.REC, value="{:.4f}".format(result['instance_recall'])) + hwlog.remark_print(key=hwlog.F1, value="{:.4f}".format(result['instance_recall'])) + return result['instance_acc'] + +# intermediate variable +inter_feature = {} +inter_gradient = {} +def make_hook(name, flag): + if flag == 'forward': + def hook(m, input, output): + inter_feature[name] = input + return hook + elif flag == 'backward': + def hook(m, input, output): + inter_gradient[name] = output + return hook + else: + assert False + +if __name__ == '__main__': + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("pytorch") + config_info = get_model_parameter("pytorch_config") + initinal_data = {"base_lr": 0.1, "dataset": "imagenet", "optimizer": "SGD", "loss_scale": 1024} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + main() \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/transform_pa100k.py b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/transform_pa100k.py new file mode 100644 index 0000000..e6411ee --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/transform_pa100k.py @@ -0,0 +1,96 @@ +import os +import numpy as np +import random +import cPickle as pickle +from scipy.io import loadmat + +np.random.seed(0) +random.seed(0) + +def make_dir(path): + if os.path.exists(path): + pass + else: + os.mkdir(path) + +def generate_data_description(save_dir): + """ + create a dataset description file, which consists of images, labels + """ + dataset = dict() + dataset['description'] = 'pa100k' + dataset['root'] = './dataset/pa100k/data/' + dataset['image'] = [] + dataset['att'] = [] + dataset['att_name'] = [] + dataset['selected_attribute'] = range(26) + # load ANNOTATION.MAT + data = loadmat(open('./dataset/pa100k/annotation.mat', 'r')) + for idx in range(26): + dataset['att_name'].append(data['attributes'][idx][0][0]) + + for idx in range(80000): + dataset['image'].append(data['train_images_name'][idx][0][0]) + dataset['att'].append(data['train_label'][idx, :].tolist()) + + for idx in range(10000): + dataset['image'].append(data['val_images_name'][idx][0][0]) + dataset['att'].append(data['val_label'][idx, :].tolist()) + + for idx in range(10000): + dataset['image'].append(data['test_images_name'][idx][0][0]) + dataset['att'].append(data['test_label'][idx, :].tolist()) + + with open(os.path.join(save_dir, 'pa100k_dataset.pkl'), 'w+') as f: + pickle.dump(dataset, f) + +def create_trainvaltest_split(traintest_split_file): + """ + create a dataset split file, which consists of index of the train/val/test splits + """ + partition = dict() + partition['trainval'] = [] + partition['train'] = [] + partition['val'] = [] + partition['test'] = [] + partition['weight_trainval'] = [] + partition['weight_train'] = [] + # load ANNOTATION.MAT + data = loadmat(open('./dataset/pa100k/annotation.mat', 'r')) + train = range(80000) + val = [i+80000 for i in range(10000)] + test = [i+90000 for i in range(10000)] + trainval = train + val + partition['train'].append(train) + partition['val'].append(val) + partition['trainval'].append(trainval) + partition['test'].append(test) + # weight + train_label = data['train_label'].astype('float32') + trainval_label = np.concatenate((data['train_label'], data['val_label']), axis=0).astype('float32') + weight_train = np.mean(train_label==1, axis=0).tolist() + weight_trainval = np.mean(trainval_label==1, axis=0).tolist() + + partition['weight_trainval'].append(weight_trainval) + partition['weight_train'].append(weight_train) + + with open(traintest_split_file, 'w+') as f: + pickle.dump(partition, f) + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser(description="pa100k dataset") + parser.add_argument( + '--save_dir', + type=str, + default='./dataset/pa100k/') + parser.add_argument( + '--traintest_split_file', + type=str, + default="./dataset/pa100k/pa100k_partition.pkl") + args = parser.parse_args() + save_dir = args.save_dir + traintest_split_file = args.traintest_split_file + + generate_data_description(save_dir) + create_trainvaltest_split(traintest_split_file) diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/transform_peta.py b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/transform_peta.py new file mode 100644 index 0000000..16150d7 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/transform_peta.py @@ -0,0 +1,84 @@ +import os +import numpy as np +import random +import pickle +from scipy.io import loadmat + +np.random.seed(0) +random.seed(0) + +def make_dir(path): + if os.path.exists(path): + pass + else: + os.mkdir(path) + +def generate_data_description(save_dir): + """ + create a dataset description file, which consists of images, labels + """ + dataset = dict() + dataset['description'] = 'peta' + dataset['root'] = save_dir + '/images/' + dataset['image'] = [] + dataset['att'] = [] + dataset['att_name'] = [] + dataset['selected_attribute'] = range(35) + # load PETA.MAT + data = loadmat(save_dir + '/PETA.mat') + for idx in range(105): + dataset['att_name'].append(data['peta'][0][0][1][idx,0][0]) + + for idx in range(19000): + dataset['image'].append('%05d.png'%(idx+1)) + dataset['att'].append(data['peta'][0][0][0][idx, 4:].tolist()) + with open(os.path.join(save_dir, 'peta_dataset.pkl'), 'wb') as f: + pickle.dump(dataset, f) + +def create_trainvaltest_split(traintest_split_file): + """ + create a dataset split file, which consists of index of the train/val/test splits + """ + partition = dict() + partition['trainval'] = [] + partition['train'] = [] + partition['val'] = [] + partition['test'] = [] + partition['weight_trainval'] = [] + partition['weight_train'] = [] + # load PETA.MAT + data = loadmat(save_dir + '/PETA.mat') + for idx in range(5): + train = (data['peta'][0][0][3][idx][0][0][0][0][:,0]-1).tolist() + val = (data['peta'][0][0][3][idx][0][0][0][1][:,0]-1).tolist() + test = (data['peta'][0][0][3][idx][0][0][0][2][:,0]-1).tolist() + trainval = train + val + partition['train'].append(train) + partition['val'].append(val) + partition['trainval'].append(trainval) + partition['test'].append(test) + # weight + weight_trainval = np.mean(data['peta'][0][0][0][trainval, 4:].astype('float32')==1, axis=0).tolist() + weight_train = np.mean(data['peta'][0][0][0][train, 4:].astype('float32')==1, axis=0).tolist() + partition['weight_trainval'].append(weight_trainval) + partition['weight_train'].append(weight_train) + with open(traintest_split_file, 'wb') as f: + pickle.dump(partition, f) + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser(description="peta dataset") + parser.add_argument( + '--save_dir', + type=str, + default='/home/zhusiyi/dataset/peta/') + parser.add_argument( + '--traintest_split_file', + type=str, + default="/home/zhusiyi/dataset/peta/peta_partition.pkl") + args = parser.parse_args() + save_dir = args.save_dir + traintest_split_file = args.traintest_split_file + + generate_data_description(save_dir) + create_trainvaltest_split(traintest_split_file) diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/transform_rap.py b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/transform_rap.py new file mode 100644 index 0000000..56803f1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/transform_rap.py @@ -0,0 +1,76 @@ +import os +import numpy as np +import random +import cPickle as pickle +from scipy.io import loadmat + +np.random.seed(0) +random.seed(0) + +def make_dir(path): + if os.path.exists(path): + pass + else: + os.mkdir(path) + +def generate_data_description(save_dir): + """ + create a dataset description file, which consists of images, labels + """ + dataset = dict() + dataset['description'] = 'rap' + dataset['root'] = './dataset/rap/RAP_dataset/' + dataset['image'] = [] + dataset['att'] = [] + dataset['att_name'] = [] + dataset['selected_attribute'] = range(51) + # load Rap_annotation.mat + data = loadmat(open('./dataset/rap/RAP_annotation/RAP_annotation.mat', 'r')) + for idx in range(51): + dataset['att_name'].append(data['RAP_annotation'][0][0][6][idx][0][0]) + + for idx in range(41585): + dataset['image'].append(data['RAP_annotation'][0][0][5][idx][0][0]) + dataset['att'].append(data['RAP_annotation'][0][0][1][idx, :].tolist()) + + with open(os.path.join(save_dir, 'rap_dataset.pkl'), 'w+') as f: + pickle.dump(dataset, f) + +def create_trainvaltest_split(traintest_split_file): + """ + create a dataset split file, which consists of index of the train/val/test splits + """ + partition = dict() + partition['trainval'] = [] + partition['test'] = [] + partition['weight_trainval'] = [] + # load RAP_annotation.mat + data = loadmat(open('./dataset/rap/RAP_annotation/RAP_annotation.mat', 'r')) + for idx in range(5): + trainval = (data['RAP_annotation'][0][0][0][idx][0][0][0][0][0,:]-1).tolist() + test = (data['RAP_annotation'][0][0][0][idx][0][0][0][1][0,:]-1).tolist() + partition['trainval'].append(trainval) + partition['test'].append(test) + # weight + weight_trainval = np.mean(data['RAP_annotation'][0][0][1][trainval, :].astype('float32')==1, axis=0).tolist() + partition['weight_trainval'].append(weight_trainval) + with open(traintest_split_file, 'w+') as f: + pickle.dump(partition, f) + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser(description="rap dataset") + parser.add_argument( + '--save_dir', + type=str, + default='./dataset/rap/') + parser.add_argument( + '--traintest_split_file', + type=str, + default="./dataset/rap/rap_partition.pkl") + args = parser.parse_args() + save_dir = args.save_dir + traintest_split_file = args.traintest_split_file + + generate_data_description(save_dir) + create_trainvaltest_split(traintest_split_file) diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/transform_rap2.py b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/transform_rap2.py new file mode 100644 index 0000000..ca1ffac --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/code/transform_rap2.py @@ -0,0 +1,86 @@ +import os +import numpy as np +import random +import cPickle as pickle +from scipy.io import loadmat + +np.random.seed(0) +random.seed(0) + +def make_dir(path): + if os.path.exists(path): + pass + else: + os.mkdir(path) + +def generate_data_description(save_dir): + """ + create a dataset description file, which consists of images, labels + """ + dataset = dict() + dataset['description'] = 'rap2' + dataset['root'] = './dataset/rap2/RAP_dataset/' + dataset['image'] = [] + dataset['att'] = [] + dataset['att_name'] = [] + # load RAP_annotation.mat + data = loadmat(open('./dataset/rap2/RAP_annotation/RAP_annotation.mat', 'r')) + dataset['selected_attribute'] = (data['RAP_annotation'][0][0][3][0,:]-1).tolist() + for idx in range(152): + dataset['att_name'].append(data['RAP_annotation'][0][0][2][idx][0][0]) + + for idx in range(84928): + dataset['image'].append(data['RAP_annotation'][0][0][0][idx][0][0]) + dataset['att'].append(data['RAP_annotation'][0][0][1][idx, :].tolist()) + + with open(os.path.join(save_dir, 'rap2_dataset.pkl'), 'w+') as f: + pickle.dump(dataset, f) + +def create_trainvaltest_split(traintest_split_file): + """ + create a dataset split file, which consists of index of the train/val/test splits + """ + partition = dict() + partition['train'] = [] + partition['val'] = [] + partition['trainval'] = [] + partition['test'] = [] + partition['weight_train'] = [] + partition['weight_trainval'] = [] + # load RAP_annotation.mat + data = loadmat(open('./dataset/rap2/RAP_annotation/RAP_annotation.mat', 'r')) + for idx in range(5): + train = (data['RAP_annotation'][0][0][4][0, idx][0][0][0][0,:]-1).tolist() + val = (data['RAP_annotation'][0][0][4][0, idx][0][0][1][0,:]-1).tolist() + test = (data['RAP_annotation'][0][0][4][0, idx][0][0][2][0,:]-1).tolist() + trainval = train + val + partition['trainval'].append(trainval) + partition['train'].append(train) + partition['val'].append(val) + partition['test'].append(test) + # weight + weight_train = np.mean(data['RAP_annotation'][0][0][1][train, :].astype('float32')==1, axis=0).tolist() + weight_trainval = np.mean(data['RAP_annotation'][0][0][1][trainval, :].astype('float32')==1, axis=0).tolist() + partition['weight_train'].append(weight_train) + partition['weight_trainval'].append(weight_trainval) + + with open(traintest_split_file, 'w+') as f: + pickle.dump(partition, f) + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser(description="rap2 dataset") + parser.add_argument( + '--save_dir', + type=str, + default='./dataset/rap2/') + parser.add_argument( + '--traintest_split_file', + type=str, + default="./dataset/rap2/rap2_partition.pkl") + args = parser.parse_args() + save_dir = args.save_dir + traintest_split_file = args.traintest_split_file + + generate_data_description(save_dir) + create_trainvaltest_split(traintest_split_file) diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/config/set_env_b023.sh b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/config/set_env_b023.sh new file mode 100644 index 0000000..7618849 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/config/set_env_b023.sh @@ -0,0 +1,31 @@ +############## toolkit situation ################ +#export LD_LIBRARY_PATH=/usr/local/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH +#export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/ascend-toolkit/latest/toolkit/tools/ide_daemon/bin/ +#export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ +#export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so +#export PYTHONPATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH + + +############## nnae situation ################ + + +if [ -d /usr/local/Ascend/nnae/latest ];then + export LD_LIBRARY_PATH=/usr/local/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/aarch64_64-linux-gnu:$LD_LIBRARY_PATH + export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/nnae/latest/toolkit/tools/ide_daemon/bin/ + export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp/ + export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so + export PYTHONPATH=/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH +else + export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/aarch64-linux-gnu:$LD_LIBRARY_PATH + export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/ascend-toolkit/latest/toolkit/tools/ide_daemon/bin/ + export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so + export PYTHONPATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH +fi + +# ln -s /usr/local/Ascend/ascend-toolkit/latest/toolkit/bin/adc /usr/local/bin/ + +export SLOG_PRINT_TO_STDOUT=0 +#su HwHiAiUser -c "adc --host 0.0.0.0:22118 --log \"SetLogLevel(0)[error]\" --device 0" + +export TASK_QUEUE_ENABLE=1 \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/scripts/run.sh b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/scripts/run.sh new file mode 100644 index 0000000..033cd74 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/scripts/run.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +rank_size=$1 +yamlPath=$2 +toolsPath=$3 + +currentDir=$(cd "$(dirname "$0")/.."; pwd) +model_name=$(cd $currentDir/..;basename `pwd`) +if [ -f /.dockerenv ];then + CLUSTER=$4 + MPIRUN_ALL_IP="$5" + export CLUSTER=${CLUSTER} +fi +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "pytorch_config") + +# 清除旧日志 +rm -rf /var/log/npu/slog/host-0/* +rm -rf ${currentDir}/result/*.log + +#mkdir train job path +currtime=`date +%Y%m%d%H%M%S` +mkdir -p ${currentDir%train*}/train/result/pt_deepmar/training_job_${currtime}/ +export train_job_dir=${currentDir%train*}/train/result/pt_deepmar/training_job_${currtime}/ +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir} &" +# device 列表, 若无指定 device 根据 rank_size 顺序选择 +eval device_group=\$device_group_${rank_size}p +if [ x"${device_group}" == x"" ] || [ ${rank_size} -ge 8 ];then + device_group="$(seq 0 "$(expr $rank_size - 1)")" +fi + +# get last device id in device_group, hw log in performance from the dir named last_device_id +device_group_str=`echo ${device_group} | sed 's/ //g'` +first_device_id=`echo ${device_group_str: 0:1}` + +if [ x"${CLUSTER}" == x"True" ];then + this_ip=$(hostname -I |awk '{print $1}') + ln -snf ${currentDir%train*}/train/result/pt_deepmar/training_job_${currtime}/0/hw_deepmar.log ${currentDir%train*}/train/result/pt_deepmar/training_job_${currtime}/ + for ip in $MPIRUN_ALL_IP;do + if [ x"$ip" != x"$this_ip" ];then + scp $yamlPath root@$ip:$yamlPath + scp ${jsonFilePath} root@$ip:${jsonFilePath} + fi + done + export PATH=$PATH:/usr/local/mpirun4.0/bin + mpirun -H ${mpirun_ip} \ + --bind-to none -map-by slot\ + --allow-run-as-root \ + --mca btl_tcp_if_exclude lo,docker0,endvnic,virbr0,vethf40501b,docker_gwbridge,br-f42ac38052b4\ + --prefix /usr/local/mpirun4.0/ \ + ${currentDir}/scripts/train.sh 0 $rank_size $yamlPath $currtime ${toolsPath} ${CLUSTER} +else + rank_id=0 + #for device_id in $device_group;do + ln -snf ${currentDir%train*}/train/result/pt_deepmar/training_job_${currtime}/${first_device_id}/hw_deepmar.log ${currentDir%train*}/train/result/pt_deepmar/training_job_${currtime}/ + ${currentDir}/scripts/train.sh 0 $rank_size $yamlPath $currtime ${toolsPath} $rank_id & + # let rank_id++ + # done +fi +wait + + diff --git a/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/scripts/train.sh b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/scripts/train.sh new file mode 100644 index 0000000..3090063 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DeepMar/pytorch/scripts/train.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash + +device_id=$1 +rank_size=$2 +yamlPath=$3 + +currentDir=$(cd "$(dirname "$0")/.."; pwd) +currtime=$4 +toolsPath=$5 +export YAML_PATH=$3 +mkdir -p ${currentDir%train*}/train/result/pt_deepmar/training_job_${currtime}/ +export train_job_dir=${currentDir%train*}/train/result/pt_deepmar/training_job_${currtime}/ + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "pytorch_config") + +export REMARK_LOG_FILE=hw_deepmar.log # 打点日志文件名称， 必须hw_后跟模型名称小写 +benchmark_log_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils +export PYTHONPATH=$PYTHONPATH:${benchmark_log_path} + + +#source ${currentDir}/config/npu_set_env.sh +source ${currentDir}/config/set_env_b023.sh +# user env +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 +export HCCL_RANK_TABLE_PATH=${currentDir}/config/${rank_size}p.json +export RANK_SIZE=${rank_size} +export SLOG_PRINT_TO_STDOUT=0 +export DEVICE_ID=${device_id} +DEVICE_INDEX=$(( DEVICE_ID + RANK_INDEX * 8 )) +export DEVICE_INDEX=${DEVICE_INDEX} + +cd ${train_job_dir} +curd_dir=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +export PYTHONPATH=$PYTHONPATH:${curd_dir} + +if [ x"$6" != x"True" ];then + rank_id=$6 + export RANK_ID=$6 +else + device_id_mo=$(python3.7 -c "import src.tensorflow.mpi_ops as atlasboost;atlasboost.init(); \ + device_id = atlasboost.local_rank();cluster_device_id = str(device_id); \ + atlasboost.set_device_id(device_id);print(atlasboost.rank())") + device_id_mo=`echo $device_id_mo` + rank_id=${device_id_mo##* } + export RANK_ID=${rank_id} + device=${device_id_mo##*deviceid = } + device_id=${device%% phyid=*} + export DEVICE_ID=${device_id} + hccljson=${train_job_dir}/*.json + cp ${hccljson} ${currentDir}/config/${rank_size}p.json +fi + +#mkdir exec path +mkdir -p ${train_job_dir}/${device_id} +cd ${train_job_dir}/${device_id} + +startTime=`date +%Y%m%d-%H:%M:%S` +startTime_s=`date +%s` + +# 数据集预处理 +python3.7 ${currentDir}/code/transform_peta.py \ + --save_dir=${data_url} \ + --traintest_split_file=${data_url}/peta_partition.pkl + +# 根据单卡/多卡区分调用参数 +if [ x"$6" == x"True" ];then + # 多卡多机 + export CLUSTER=True +fi + +if [ x"${mode}" == x"evaluate" ];then + pass + + +elif [ x"${rank_size}" == x"1" ];then + # 单卡 + python3.7 ${currentDir}/code/train_deepmar_resnet50.py \ + --dataset=peta \ + --save_dir=${data_url} \ + --workers=32 \ + --npu=${device} \ + --partition_idx=0 \ + --split=trainval \ + --test_split=test \ + --batch_size=${batch_size} \ + --resize="(224,224)" \ + --exp_subpath=deepmar_resnet50 \ + --new_params_lr=0.01 \ + --finetuned_params_lr=0.01 \ + --staircase_decay_at_epochs="(50,100)" \ + --total_epochs=${epoches} \ + --epochs_per_val=10 \ + --epochs_per_save=50 \ + --steps_per_log=10 \ + --drop_pool5=True \ + --drop_pool5_rate=0.5 \ + --run=1 \ + --resume=False \ + --ckpt_file= \ + --load_model_weight=False \ + --model_weight_file= \ + --amp \ + --opt_level O2 \ + --loss_scale 512 \ + --set_seed True \ + --pretrained True \ + --test_only=False > ${train_job_dir}/train_${rank_size}p.log 2>&1 + +elif [ ${rank_size} -le 8 ];then + # 单机多卡 + #source ${currentDir}/config/set_env_b023.sh + python3.7 ${currentDir}/code/train_deepmar_resnet50_8p.py \ + --addr=$(hostname -I |awk '{print $1}') \ + --save_dir=${data_url} \ + --dataset=peta \ + --workers=80 \ + --partition_idx=0 \ + --split=trainval \ + --test_split=test \ + --batch_size=${batch_size} \ + --resize="(224,224)" \ + --exp_subpath=deepmar_resnet50 \ + --new_params_lr=${lr} \ + --finetuned_params_lr=${lr} \ + --staircase_decay_at_epochs="(50,100)" \ + --total_epochs=${epoches} \ + --epochs_per_val=10 \ + --epochs_per_save=50 \ + --steps_per_log=10 \ + --drop_pool5=True \ + --drop_pool5_rate=0.5 \ + --run=1 \ + --resume=False \ + --ckpt_file= \ + --load_model_weight=False \ + --model_weight_file=ckpt_epoch101.pth\ + --amp \ + --opt_level O2 \ + --loss_scale 512.0 \ + --set_seed True \ + --pretrained True \ + --test_only=False \ + --dist_url 'tcp://127.0.0.1:50000' \ + --dist_backend 'hccl' \ + --multiprocessing_distributed \ + --world_size 1 \ + --npus_per_node=${rank_size} \ + --rank 0 > ${train_job_dir}/train_${rank_size}p.log 2>&1 + + +fi + +#taskset -c 0-20 python3.7 ${currentDir}/code/densenet121.py > ./train.log 2>&1 + +if [ $? -eq 0 ];then + echo ":::ABK 1.0.0 deepmar train success" + echo ":::ABK 1.0.0 deepmar train success" >> ${train_job_dir}/train_${rank_size}p.log + echo ":::ABK 1.0.0 deepmar train success" >> ./hw_deepmar.log +else + echo ":::ABK 1.0.0 deepmar train success" + echo ":::ABK 1.0.0 deepmar train failed" >> ${train_job_dir}/train_${rank_size}p.log + echo ":::ABK 1.0.0 deepmar train failed" >> ./hw_deepmar.log +fi + +endTime=`date +%Y%m%d-%H:%M:%S` +endTime_s=`date +%s` +sumTime=$[ $endTime_s - $startTime_s ] +hour=$(( $sumTime/3600 )) +min=$(( ($sumTime-${hour}*3600)/60 )) +sec=$(( $sumTime-${hour}*3600-${min}*60 )) +echo ":::ABK 1.0.0 deepmar train total time： ${hour}:${min}:${sec}" >> ${train_job_dir}/${device_id}/hw_deepmar.log diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/__init__.py b/train/atlas_benchmark-master/image_classification/DenseNet121/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/README.md b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/README.md new file mode 100644 index 0000000..7ba1672 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/README.md @@ -0,0 +1,25 @@ +# DenseNet121_pytorch训练说明 + +### 1. 模型训练参数配置 + +在train/yaml/DenseNet121.yaml中修改相应配置， 配置项含义: + +``` +pytorch_config: + data_url: 数据集路径 + epoches: 跑多少个epoch + batch_size: 1p 参数为256 2p 512 4p 1024 8p为2048 + lr: 默认参数1p 0.1 2p 0.2 4p 0.4 8p 0.8 + seed: 49 + docker_image: docker 镜像名称:版本号 +``` + +------ + + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/__init__.py b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/densenet121_1p_main.py b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/densenet121_1p_main.py new file mode 100644 index 0000000..5ef0e11 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/densenet121_1p_main.py @@ -0,0 +1,515 @@ +import argparse +import os +import random +import shutil +import time +import warnings +import sys + +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.distributed as dist +import torch.optim +import torch.multiprocessing as mp +import torch.utils.data +import torch.utils.data.distributed +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torchvision.models as models +import torch.npu + +from torch.utils.tensorboard import SummaryWriter +from densenet_0_2_2 import densenet121 + +import numpy as np +from apex import amp + + +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter + + +warnings.filterwarnings('ignore') + +model_names = sorted(name for name in models.__dict__ + if name.islower() and not name.startswith("__") + and callable(models.__dict__[name])) + +parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') +parser.add_argument('--data', metavar='DIR', default='/opt/npu/dataset/imagenet', + help='path to dataset') +parser.add_argument('-a', '--arch', metavar='ARCH', default='densenet121', + choices=model_names, + help='model architecture: ' + + ' | '.join(model_names) + + ' (default: resnet18)') +parser.add_argument('-j', '--workers', default=4, type=int, metavar='N', + help='number of data loading workers (default: 8)') +parser.add_argument('--epochs', default=90, type=int, metavar='N', + help='number of total epochs to run') +parser.add_argument('--start-epoch', default=0, type=int, metavar='N', + help='manual epoch number (useful on restarts)') +parser.add_argument('-b', '--batch-size', default=128, type=int, + metavar='N', + help='mini-batch size (default: 256), this is the total ' + 'batch size of all GPUs on the current node when ' + 'using Data Parallel or Distributed Data Parallel') +parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, + metavar='LR', help='initial learning rate', dest='lr') +parser.add_argument('--momentum', default=0.9, type=float, metavar='M', + help='momentum') +parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, + metavar='W', help='weight decay (default: 1e-4)', + dest='weight_decay') +parser.add_argument('-p', '--print-freq', default=1, type=int, + metavar='N', help='print frequency (default: 10)') +parser.add_argument('-ef', '--eval-freq', default=5, type=int, + metavar='N', help='evaluate frequency (default: 5)') +parser.add_argument('--resume', default='', type=str, metavar='PATH', + help='path to latest checkpoint (default: none)') +parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', + help='evaluate model on validation set') +parser.add_argument('--pretrained', dest='pretrained', action='store_true', + help='use pre-trained model') +parser.add_argument('--world-size', default=-1, type=int, + help='number of nodes for distributed training') +parser.add_argument('--rank', default=-1, type=int, + help='node rank for distributed training') +parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, + help='url used to set up distributed training') +parser.add_argument('--dist-backend', default='nccl', type=str, + help='distributed backend') +parser.add_argument('--seed', default=None, type=int, + help='seed for initializing training. ') +parser.add_argument('--gpu', default=None, type=int, + help='GPU id to use.') +parser.add_argument('--multiprocessing-distributed', action='store_true', + help='Use multi-processing distributed training to launch ' + 'N processes per node, which has N GPUs. This is the ' + 'fastest way to use PyTorch for either single node or ' + 'multi node data parallel training') +parser.add_argument('--npu', default=None, type=int, + help='NPU id to use.') + +# apex +parser.add_argument('--amp', default=False, action='store_true', + help='use amp to train the model') +parser.add_argument('--loss-scale', default=1024., type=float, + help='loss scale using in amp, default -1 means dynamic') +parser.add_argument('--opt-level', default='O2', type=str, + help='loss scale using in amp, default -1 means dynamic') + + + +def main(): + args = parser.parse_args() + print(args) + + if args.npu is None: + args.npu = 0 + global CALCULATE_DEVICE + global best_acc1 + + best_acc1 = 0 + CALCULATE_DEVICE = "npu:{}".format(args.npu) + torch.npu.set_device(CALCULATE_DEVICE) + + if args.seed is not None: + random.seed(seed) + os.environ['PYTHONHASHSEED'] = str(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + if args.gpu is not None: + warnings.warn('You have chosen a specific GPU. This will completely ' + 'disable data parallelism.') + + if args.dist_url == "env://" and args.world_size == -1: + args.world_size = int(os.environ["WORLD_SIZE"]) + + args.distributed = args.world_size > 1 or args.multiprocessing_distributed + + ngpus_per_node = torch.npu.device_count() + print('{} node found.'.format(ngpus_per_node)) + if args.multiprocessing_distributed: + # Since we have ngpus_per_node processes per node, the total world_size + # needs to be adjusted accordingly + args.world_size = ngpus_per_node * args.world_size + # Use torch.multiprocessing.spawn to launch distributed processes: the + # main_worker process function + mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) + else: + # Simply call main_worker function + main_worker(args.gpu, ngpus_per_node, args) + + +def main_worker(gpu, ngpus_per_node, args): + global best_acc1 + args.gpu = gpu + + if args.gpu is not None: + print("Use GPU: {} for training".format(args.gpu)) + + if args.distributed: + if args.dist_url == "env://" and args.rank == -1: + args.rank = int(os.environ["RANK"]) + if args.multiprocessing_distributed: + # For multiprocessing distributed training, rank needs to be the + # global rank among all the processes + args.rank = args.rank * ngpus_per_node + gpu + dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + # create model + if args.pretrained: + print("=> using pre-trained model '{}'".format(args.arch)) + model = models.__dict__[args.arch](pretrained=True) + else: + print("=> creating model '{}'".format(args.arch)) + # model = models.__dict__[args.arch]() + model = densenet121() + + if args.distributed: + # For multiprocessing distributed, DistributedDataParallel constructor + # should always set the single device scope, otherwise, + # DistributedDataParallel will use all available devices. + if args.gpu is not None: + torch.cuda.set_device(args.gpu) + model.cuda(args.gpu) + # When using a single GPU per process and per + # DistributedDataParallel, we need to divide the batch size + # ourselves based on the total number of GPUs we have + args.batch_size = int(args.batch_size / ngpus_per_node) + args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + else: + model.cuda() + # DistributedDataParallel will divide and allocate batch_size to all + # available GPUs if device_ids are not set + model = torch.nn.parallel.DistributedDataParallel(model) + elif args.gpu is not None: + torch.cuda.set_device(args.gpu) + model = model.cuda(args.gpu) + else: + # DataParallel will divide and allocate batch_size to all available GPUs + if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): + model.features = torch.nn.DataParallel(model.features) + model.cuda() + else: + model = model.to(CALCULATE_DEVICE) + #for item in model.npu_unsupport_list: + # print("npu_unsupport: ", item) + # item.cpu() + + # define loss function (criterion) and optimizer + criterion = nn.CrossEntropyLoss().to(CALCULATE_DEVICE) + + optimizer = torch.optim.SGD(model.parameters(), args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay) + + if args.amp: + model, optimizer = amp.initialize(model, optimizer, opt_level=args.opt_level, loss_scale=args.loss_scale) + + # optionally resume from a checkpoint + if args.resume: + if os.path.isfile(args.resume): + print("=> loading checkpoint '{}'".format(args.resume)) + checkpoint = torch.load(args.resume, map_location=CALCULATE_DEVICE) + args.start_epoch = checkpoint['epoch'] + best_acc1 = checkpoint['best_acc1'] + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + if args.amp: + amp.load_state_dict(checkpoint['amp']) + print("=> loaded checkpoint '{}' (epoch {})" + .format(args.resume, checkpoint['epoch'])) + else: + print("=> no checkpoint found at '{}'".format(args.resume)) + + cudnn.benchmark = True + + # Data loading code + traindir = os.path.join(args.data, 'train') + valdir = os.path.join(args.data, 'val') + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + + train_dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ])) + + if args.distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), + num_workers=args.workers, pin_memory=False, sampler=train_sampler, drop_last=True) + + val_loader = torch.utils.data.DataLoader( + datasets.ImageFolder(valdir, transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + normalize, + ])), + batch_size=args.batch_size, shuffle=True, + num_workers=args.workers, pin_memory=False, drop_last=True) + + if args.evaluate: + validate(val_loader, model, criterion, args) + return + + + writer = SummaryWriter(os.path.join('runs/densenet121')) + for epoch in range(args.start_epoch, args.epochs): + if args.distributed: + train_sampler.set_epoch(epoch) + adjust_learning_rate(optimizer, epoch, args) + + # train for one epoch + train(train_loader, model, criterion, optimizer, epoch, args, writer) + + if (epoch+1)%(args.eval_freq)==0 or epoch==args.epochs-1 : + # evaluate on validation set + acc1 = validate(val_loader, model, criterion, args, epoch, writer) + + # remember best acc@1 and save checkpoint + is_best = acc1 > best_acc1 + best_acc1 = max(acc1, best_acc1) + + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0 and epoch == args.epochs - 1): + if args.amp: + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer' : optimizer.state_dict(), + 'amp': amp.state_dict(), + }, is_best) + else: + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer' : optimizer.state_dict(), + }, is_best) + + writer.close() + +def train(train_loader, model, criterion, optimizer, epoch, args, writer): + batch_time = AverageMeter('Time', ':6.3f') + data_time = AverageMeter('Data', ':6.3f') + losses = AverageMeter('Loss', ':.4e') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(train_loader), + [batch_time, data_time, losses, top1, top5], + prefix="Epoch: [{}]".format(epoch)) + + # switch to train mode + model.train() + + end = time.time() + for i, (images, target) in enumerate(train_loader): + # measure data loading time + data_time.update(time.time() - end) + + target = target.to(torch.int32) + images, target = images.to(CALCULATE_DEVICE, non_blocking=False), target.to(CALCULATE_DEVICE, non_blocking=False) + + # compute output + output = model(images) + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # add tensorboard + writer.add_scalar('Train/Loss', losses.val, epoch * len(train_loader) + i) + writer.add_scalar('Train/Acc@1', top1.val, epoch * len(train_loader) + i) + writer.add_scalar('Train/Acc@5', top5.val, epoch * len(train_loader) + i) + writer.add_scalar('Train/LR', optimizer.param_groups[0]['lr'], epoch * len(train_loader) + i) + + # compute gradient and do SGD step + optimizer.zero_grad() + if args.amp: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + optimizer.step() + + # measure elapsed time + batch_time.update(time.time() - end) + writer.add_scalar('Train/Time', batch_time.val, epoch * len(train_loader) + i) + writer.add_scalar('Train/Time_Data', data_time.val, epoch * len(train_loader) + i) + end = time.time() + + if i % args.print_freq == 0: + progress.display(i) + + print(' * FPS@all {:.3f}'.format(args.batch_size/batch_time.avg)) + hwlog.remark_print(key=hwlog.FPS, value=' * FPS@all {:.3f}'.format(args.batch_size/batch_time.avg)) + +def validate(val_loader, model, criterion, args, epoch=0, writer=None): + batch_time = AverageMeter('Time', ':6.3f') + losses = AverageMeter('Loss', ':.4e') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(val_loader), + [batch_time, losses, top1, top5], + prefix='Test: ') + + # switch to evaluate mode + model.eval() + + with torch.no_grad(): + end = time.time() + for i, (images, target) in enumerate(val_loader): + target = target.to(torch.int32) + images, target = images.to(CALCULATE_DEVICE, non_blocking=False), target.to(CALCULATE_DEVICE, non_blocking=False) + + # compute output + output = model(images) + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + progress.display(i) + + # TODO: this should also be done with the ProgressMeter + print(' * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}' + .format(top1=top1, top5=top5)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value="{top1.avg:.3f}".format(top1=top1)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value="{top5.avg:.3f}".format(top5=top5)) + + if writer: # and args.gpu==0: + writer.add_scalar('Val/Time', batch_time.avg, epoch) + writer.add_scalar('Val/Loss', losses.avg, epoch) + writer.add_scalar('Val/Acc@1', top1.avg, epoch) + writer.add_scalar('Val/Acc@5', top5.avg, epoch) + + return top1.avg + + +def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): + torch.save(state, filename) + if is_best: + shutil.copyfile(filename, 'model_best_acc%.4f_epoch%d.pth.tar'%(state['best_acc1'], state['epoch'])) + + +class AverageMeter(object): + """Computes and stores the average and current value""" + def __init__(self, name, fmt=':f'): + self.name = name + self.fmt = fmt + self.reset() + self.start_count_index = 10 + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.count += n + if self.count>(self.start_count_index*n): + self.sum += val * n + self.avg = self.sum / (self.count-self.start_count_index*n) + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + +class ProgressMeter(object): + def __init__(self, num_batches, meters, prefix=""): + self.batch_fmtstr = self._get_batch_fmtstr(num_batches) + self.meters = meters + self.prefix = prefix + + def display(self, batch): + entries = [self.prefix + self.batch_fmtstr.format(batch)] + entries += [str(meter) for meter in self.meters] + print('\t'.join(entries)) + # 日志打点 + train_acc1 = str(entries).split("Acc@1")[1].strip().split(" ")[0] + train_acc5 = str(entries).split("Acc@5")[1].strip().split(" ")[0] + hwlog.remark_print(key=hwlog.TRAIN_ACCURACY_TOP1, value=train_acc1) + hwlog.remark_print(key=hwlog.TRAIN_ACCURACY_TOP5, value=train_acc5) + + def _get_batch_fmtstr(self, num_batches): + num_digits = len(str(num_batches // 1)) + fmt = '{:' + str(num_digits) + 'd}' + return '[' + fmt + '/' + fmt.format(num_batches) + ']' + + +def adjust_learning_rate(optimizer, epoch, args): + """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" + lr = args.lr * (0.1 ** (epoch // 30)) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + + +def accuracy(output, target, topk=(1,)): + """Computes the accuracy over the k top predictions for the specified values of k""" + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +if __name__ == '__main__': + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("pytorch") + config_info = get_model_parameter("pytorch_config") + initinal_data = {"base_lr": 0.1, "dataset": "imagenet", "optimizer": "SGD", "loss_scale": 1024} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + main() diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/densenet121_8p_main.py b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/densenet121_8p_main.py new file mode 100644 index 0000000..6d3b0eb --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/densenet121_8p_main.py @@ -0,0 +1,538 @@ +# -*- coding: utf-8 -*- + +import argparse +import os +import random +import shutil +import time +import warnings + +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.distributed as dist +import torch.optim +import torch.multiprocessing as mp +import torch.utils.data +import torch.utils.data.distributed +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torchvision.models as models +from densenet_0_2_2 import densenet121 + +from apex import amp + +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter + +BATCH_SIZE = 512 +OPTIMIZER_BATCH_SIZE=2048 +model_names = sorted(name for name in models.__dict__ + if name.islower() and not name.startswith("__") + and callable(models.__dict__[name])) + +parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') +parser.add_argument('--data', metavar='DIR', default='/opt/npu/dataset/imagenet', + help='path to dataset') +parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet50', + choices=model_names, + help='model architecture: ' + + ' | '.join(model_names) + + ' (default: resnet18)') +parser.add_argument('-j', '--workers', default=32, type=int, metavar='N', + help='number of data loading workers (default: 4)') +parser.add_argument('--epochs', default=90, type=int, metavar='N', + help='number of total epochs to run') +parser.add_argument('--start-epoch', default=0, type=int, metavar='N', + help='manual epoch number (useful on restarts)') +parser.add_argument('-b', '--batch-size', default=BATCH_SIZE, type=int, + metavar='N', + help='mini-batch size (default: 256), this is the total ' + 'batch size of all GPUs on the current node when ' + 'using Data Parallel or Distributed Data Parallel') +parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, + metavar='LR', help='initial learning rate', dest='lr') +parser.add_argument('--momentum', default=0.9, type=float, metavar='M', + help='momentum') +parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, + metavar='W', help='weight decay (default: 1e-4)', + dest='weight_decay') +parser.add_argument('--workspace',type=str,default='./',metavar='DIR', + help='path to directory where checkpoints will be stored') +parser.add_argument('-p', '--print-freq', default=10, type=int, + metavar='N', help='print frequency (default: 10)') +parser.add_argument('-ef', '--eval-freq', default=5, type=int, + metavar='N', help='evaluate frequency (default: 5)') +parser.add_argument('--resume', default='', type=str, metavar='PATH', + help='path to latest checkpoint (default: none)') +parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', + help='evaluate model on validation set') +parser.add_argument('--pretrained', dest='pretrained', action='store_true', + help='use pre-trained model') +parser.add_argument('--world-size', default=-1, type=int, + help='number of nodes for distributed training') +parser.add_argument('--rank', default=-1, type=int, + help='node rank for distributed training') +parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, + help='url used to set up distributed training') +parser.add_argument('--dist-backend', default='nccl', type=str, + help='distributed backend') +parser.add_argument('--seed', default=None, type=int, + help='seed for initializing training. ') +parser.add_argument('--gpu', default=None, type=int, + help='GPU id to use.') +parser.add_argument('--multiprocessing-distributed', action='store_true', + help='Use multi-processing distributed training to launch ' + 'N processes per node, which has N GPUs. This is the ' + 'fastest way to use PyTorch for either single node or ' + 'multi node data parallel training') +parser.add_argument('-bm', '--benchmark', default=0, type=int, + metavar='N', help='set benchmark status (default: 1,run benchmark)') +parser.add_argument('--device', default='npu', type=str, + help='npu or gpu') +parser.add_argument('--addr', default='10.136.181.115', type=str, + help='master addr') +parser.add_argument('--checkpoint-nameprefix', default='checkpoint', type=str, + help='checkpoint-nameprefix') +parser.add_argument('--checkpoint-freq', default=0, type=int, + metavar='N', help='checkpoint frequency (default: 0)' + '0: save only one file whitch per epoch;' + 'n: save diff file per n epoch' + '-1:no checkpoint,not support') +parser.add_argument('--device-list', default='0,1,2,3,4,5,6,7', type=str, help='device id list') +# apex +parser.add_argument('--amp', default=False, action='store_true', + help='use amp to train the model') +parser.add_argument('--loss-scale', default=1024., type=float, + help='loss scale using in amp, default -1 means dynamic') +parser.add_argument('--opt-level', default='O2', type=str, + help='loss scale using in amp, default -1 means dynamic') + +warnings.filterwarnings('ignore') +best_acc1 = 0 +def device_id_to_process_device_map(device_list): + devices = device_list.split(",") + devices = [int(x) for x in devices] + devices.sort() + + process_device_map = dict() + for process_id, device_id in enumerate(devices): + process_device_map[process_id] = device_id + + return process_device_map + +def main(): + args = parser.parse_args() + print("===============main()=================") + print(args) + print("===============main()=================") + + os.environ['KERNEL_NAME_ID'] = str(0) + print("+++++++++++++++++++++++++++KERNEL_NAME_ID:",os.environ['KERNEL_NAME_ID']) + + if args.seed is not None: + random.seed(args.seed) + torch.manual_seed(args.seed) + cudnn.deterministic = True + warnings.warn('You have chosen to seed training. ' + 'This will turn on the CUDNN deterministic setting, ' + 'which can slow down your training considerably! ' + 'You may see unexpected behavior when restarting ' + 'from checkpoints.') + + os.environ['MASTER_ADDR'] = args.addr # '10.136.181.51' + os.environ['MASTER_PORT'] = '29688' + + if args.gpu is not None: + warnings.warn('You have chosen a specific GPU. This will completely ' + 'disable data parallelism.') + + if args.dist_url == "env://" and args.world_size == -1: + args.world_size = int(os.environ["WORLD_SIZE"]) + + args.distributed = args.world_size > 1 or args.multiprocessing_distributed + + args.process_device_map = device_id_to_process_device_map(args.device_list) + + if args.device == 'npu': + ngpus_per_node = len(args.process_device_map) + else: + ngpus_per_node = torch.cuda.device_count() + if args.multiprocessing_distributed: + # Since we have ngpus_per_node processes per node, the total world_size + # needs to be adjusted accordingly + args.world_size = ngpus_per_node * args.world_size + # Use torch.multiprocessing.spawn to launch distributed processes: the + # main_worker process function + # The child process uses the environment variables of the parent process, + # we have to set KERNEL_NAME_ID for every proc + mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) + + else: + # Simply call main_worker function + main_worker(args.gpu, ngpus_per_node, args) + + +def main_worker(gpu, ngpus_per_node, args): + global best_acc1 + args.gpu = args.process_device_map[gpu] + print("[npu id:",args.gpu,"]","+++++++++++++++++++++++++++ before set KERNEL_NAME_ID:",os.environ['KERNEL_NAME_ID']) + os.environ['KERNEL_NAME_ID'] = str(gpu) + print("[npu id:",args.gpu,"]","+++++++++++++++++++++++++++KERNEL_NAME_ID:",os.environ['KERNEL_NAME_ID']) + + if args.gpu is not None: + print("[npu id:",args.gpu,"]","Use GPU: {} for training".format(args.gpu)) + + if args.distributed: + if args.dist_url == "env://" and args.rank == -1: + args.rank = int(os.environ["RANK"]) + if args.multiprocessing_distributed: + # For multiprocessing distributed training, rank needs to be the + # global rank among all the processes + args.rank = args.rank * ngpus_per_node + gpu + + if args.device == 'npu': + dist.init_process_group(backend=args.dist_backend, #init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + else: + dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + + + loc = 'npu:{}'.format(args.gpu) + torch.npu.set_device(loc) + + args.batch_size = int(args.batch_size / ngpus_per_node) + args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) + + print("[npu id:",args.gpu,"]","===============main_worker()=================") + print("[npu id:",args.gpu,"]",args) + print("[npu id:",args.gpu,"]","===============main_worker()=================") + + + # Data loading code + traindir = os.path.join(args.data, 'train') + valdir = os.path.join(args.data, 'val') + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + + train_dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ])) + + if args.distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), + num_workers=args.workers, pin_memory=False, sampler=train_sampler, drop_last=True) + + val_loader = torch.utils.data.DataLoader( + datasets.ImageFolder(valdir, transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + normalize, + ])), + batch_size=args.batch_size, shuffle=True, + num_workers=args.workers, pin_memory=False, drop_last=True) + + # create model + print("[npu id:",args.gpu,"]","=> creating model '{}'".format(args.arch)) + # model = models.__dict__[args.arch]() + model = densenet121() + model = model.to(loc) + + # define loss function (criterion) and optimizer + criterion = nn.CrossEntropyLoss().to(loc) + optimizer = torch.optim.SGD(model.parameters(), args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay) + + if args.amp: + model, optimizer = amp.initialize(model, optimizer, opt_level=args.opt_level, loss_scale=args.loss_scale) + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], broadcast_buffers=False) + + # optionally resume from a checkpoint + if args.resume: + if os.path.isfile(args.resume): + print("=> loading checkpoint '{}'".format(args.resume)) + checkpoint = torch.load(args.resume, map_location=loc) + args.start_epoch = checkpoint['epoch'] + best_acc1 = checkpoint['best_acc1'] + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + if args.amp: + amp.load_state_dict(checkpoint['amp']) + print("=> loaded checkpoint '{}' (epoch {})" + .format(args.resume, checkpoint['epoch'])) + else: + print("=> no checkpoint found at '{}'".format(args.resume)) + + cudnn.benchmark = True + + + if args.evaluate: + validate(val_loader, model, criterion, args) + return + + for epoch in range(args.start_epoch, args.epochs): + if args.distributed: + train_sampler.set_epoch(epoch) + adjust_learning_rate(optimizer, epoch, args) + + # train for one epoch + train(train_loader, model, criterion, optimizer, epoch, args,ngpus_per_node) + + if (epoch+1)%(args.eval_freq)==0 or epoch==args.epochs-1 : + # evaluate on validation set + acc1 = validate(val_loader, model, criterion, args,ngpus_per_node) + + # remember best acc@1 and save checkpoint + is_best = acc1 > best_acc1 + best_acc1 = max(acc1, best_acc1) + + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0 and epoch == args.epochs - 1): + if args.amp: + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer' : optimizer.state_dict(), + 'amp': amp.state_dict(), + }, is_best) + else: + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer' : optimizer.state_dict(), + }, is_best) + +def train(train_loader, model, criterion, optimizer, epoch, args,ngpus_per_node): + batch_time = AverageMeter('Time', ':6.3f') + data_time = AverageMeter('Data', ':6.3f') + losses = AverageMeter('Loss', ':.4e') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(train_loader), + [batch_time, data_time, losses, top1, top5], + prefix="Epoch: [{}]".format(epoch)) + + # switch to train mode + model.train() + end = time.time() + if args.benchmark == 1 : + optimizer.zero_grad() + for i, (images, target) in enumerate(train_loader): + # measure data loading time + data_time.update(time.time() - end) + + loc = 'npu:{}'.format(args.gpu) + target = target.to(torch.int32) + images, target = images.to(loc, non_blocking=False), target.to(loc, non_blocking=False) + + # compute output + output = model(images) + + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # compute gradient and do SGD step + if args.benchmark == 0 : + optimizer.zero_grad() + + if args.amp: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + + if args.benchmark == 0 : + optimizer.step() + elif args.benchmark == 1 : + BATCH_SIZE_multiplier = int(OPTIMIZER_BATCH_SIZE / args.batch_size) + BM_optimizer_step = ((i + 1) % BATCH_SIZE_multiplier) == 0 + if BM_optimizer_step: + for param_group in optimizer.param_groups: + for param in param_group['params']: + param.grad /= BATCH_SIZE_multiplier + optimizer.step() + optimizer.zero_grad() + + if i % args.print_freq == 0: + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + progress.display(i) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + print("[npu id:",args.gpu,"]",'* FPS@all {:.3f}'.format(ngpus_per_node*args.batch_size/batch_time.avg)) + hwlog.remark_print(key=hwlog.FPS, value=' * FPS@all {:.3f}'.format(ngpus_per_node*args.batch_size / batch_time.avg)) + +def validate(val_loader, model, criterion, args,ngpus_per_node): + batch_time = AverageMeter('Time', ':6.3f') + losses = AverageMeter('Loss', ':.4e') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(val_loader), + [batch_time, losses, top1, top5], + prefix='Test: ') + + # switch to evaluate mode + model.eval() + + with torch.no_grad(): + end = time.time() + for i, (images, target) in enumerate(val_loader): + + loc = 'npu:{}'.format(args.gpu) + target = target.to(torch.int32) + images, target = images.to(loc, non_blocking=False), target.to(loc, non_blocking=False) + + # compute output + output = model(images) + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + progress.display(i) + + # TODO: this should also be done with the ProgressMeter + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + print("[npu id:",args.gpu,"]",'[AVG-ACC] * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}' + .format(top1=top1, top5=top5)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value="{top1.avg:.3f}".format(top1=top1)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value="{top5.avg:.3f}".format(top5=top5)) + + return top1.avg + +def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): + torch.save(state, filename) + if is_best: + shutil.copyfile(filename, 'model_best_acc%.4f_epoch%d.pth.tar'%(state['best_acc1'], state['epoch'])) + +class AverageMeter(object): + """Computes and stores the average and current value""" + def __init__(self, name, fmt=':f'): + self.name = name + self.fmt = fmt + self.reset() + self.start_count_index = 10 + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.count += n + if self.count>(self.start_count_index*n): + self.sum += val * n + self.avg = self.sum / (self.count-self.start_count_index*n) + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + +class ProgressMeter(object): + def __init__(self, num_batches, meters, prefix=""): + self.batch_fmtstr = self._get_batch_fmtstr(num_batches) + self.meters = meters + self.prefix = prefix + + def display(self, batch): + entries = [self.prefix + self.batch_fmtstr.format(batch)] + entries += [str(meter) for meter in self.meters] + print("[npu id:",os.environ['KERNEL_NAME_ID'],"]",'\t'.join(entries)) + # 日志打点 + train_acc1 = str(entries).split("Acc@1")[1].strip().split(" ")[0] + train_acc5 = str(entries).split("Acc@5")[1].strip().split(" ")[0] + hwlog.remark_print(key=hwlog.TRAIN_ACCURACY_TOP1, value=train_acc1) + hwlog.remark_print(key=hwlog.TRAIN_ACCURACY_TOP5, value=train_acc5) + + def _get_batch_fmtstr(self, num_batches): + num_digits = len(str(num_batches // 1)) + fmt = '{:' + str(num_digits) + 'd}' + return '[' + fmt + '/' + fmt.format(num_batches) + ']' + + +def adjust_learning_rate(optimizer, epoch, args): + """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" + lr = args.lr * (0.1 ** (epoch // 30)) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + + +def accuracy(output, target, topk=(1,)): + """Computes the accuracy over the k top predictions for the specified values of k""" + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +if __name__ == '__main__': + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("pytorch") + config_info = get_model_parameter("pytorch_config") + initinal_data = {"base_lr": 0.1, "dataset": "imagenet", "optimizer": "SGD", "loss_scale": 1024} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + main() + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/densenet_0_2_2.py b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/densenet_0_2_2.py new file mode 100644 index 0000000..43f8e7c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/densenet_0_2_2.py @@ -0,0 +1,225 @@ +import re +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.model_zoo as model_zoo +from collections import OrderedDict + +__all__ = ['DenseNet', 'densenet121', 'densenet169', 'densenet201', 'densenet161'] + + +model_urls = { + 'densenet121': 'https://download.pytorch.org/models/densenet121-a639ec97.pth', + 'densenet169': 'https://download.pytorch.org/models/densenet169-b2777c0a.pth', + 'densenet201': 'https://download.pytorch.org/models/densenet201-c1103571.pth', + 'densenet161': 'https://download.pytorch.org/models/densenet161-8d451a50.pth', +} + + +class _DenseLayer(nn.Sequential): + def __init__(self, num_input_features, growth_rate, bn_size, drop_rate): + super(_DenseLayer, self).__init__() + self.add_module('norm1', nn.BatchNorm2d(num_input_features)), + self.add_module('relu1', nn.ReLU(inplace=True)), + self.add_module('conv1', nn.Conv2d(num_input_features, bn_size * + growth_rate, kernel_size=1, stride=1, bias=False)), + self.add_module('norm2', nn.BatchNorm2d(bn_size * growth_rate)), + self.add_module('relu2', nn.ReLU(inplace=True)), + self.add_module('conv2', nn.Conv2d(bn_size * growth_rate, growth_rate, + kernel_size=3, stride=1, padding=1, bias=False)), + self.drop_rate = drop_rate + + def forward(self, x): + new_features = super(_DenseLayer, self).forward(x) + if self.drop_rate > 0: + new_features = F.dropout(new_features, p=self.drop_rate, training=self.training) + return torch.cat([x, new_features], 1) + + +class _DenseBlock(nn.Sequential): + def __init__(self, num_layers, num_input_features, bn_size, growth_rate, drop_rate): + super(_DenseBlock, self).__init__() + for i in range(num_layers): + layer = _DenseLayer(num_input_features + i * growth_rate, growth_rate, bn_size, drop_rate) + self.add_module('denselayer%d' % (i + 1), layer) + + +class _Transition(nn.Sequential): + def __init__(self, num_input_features, num_output_features): + super(_Transition, self).__init__() + self.add_module('norm', nn.BatchNorm2d(num_input_features)) + self.add_module('relu', nn.ReLU(inplace=True)) + self.add_module('conv', nn.Conv2d(num_input_features, num_output_features, + kernel_size=1, stride=1, bias=False)) + self.add_module('pool', nn.AvgPool2d(kernel_size=2, stride=2)) + + +class DenseNet(nn.Module): + r"""Densenet-BC model class, based on + `"Densely Connected Convolutional Networks" `_ + + Args: + growth_rate (int) - how many filters to add each layer (`k` in paper) + block_config (list of 4 ints) - how many layers in each pooling block + num_init_features (int) - the number of filters to learn in the first convolution layer + bn_size (int) - multiplicative factor for number of bottle neck layers + (i.e. bn_size * k features in the bottleneck layer) + drop_rate (float) - dropout rate after each dense layer + num_classes (int) - number of classification classes + """ + + def __init__(self, growth_rate=32, block_config=(6, 12, 24, 16), + num_init_features=64, bn_size=4, drop_rate=0, num_classes=1000): + + super(DenseNet, self).__init__() + + # First convolution + self.features = nn.Sequential(OrderedDict([ + ('conv0', nn.Conv2d(3, num_init_features, kernel_size=7, stride=2, padding=3, bias=False)), + ('norm0', nn.BatchNorm2d(num_init_features)), + ('relu0', nn.ReLU(inplace=True)), + ('pool0', nn.MaxPool2d(kernel_size=3, stride=2, padding=1)), + ])) + + # Each denseblock + num_features = num_init_features + for i, num_layers in enumerate(block_config): + block = _DenseBlock(num_layers=num_layers, num_input_features=num_features, + bn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate) + self.features.add_module('denseblock%d' % (i + 1), block) + num_features = num_features + num_layers * growth_rate + if i != len(block_config) - 1: + trans = _Transition(num_input_features=num_features, num_output_features=num_features // 2) + self.features.add_module('transition%d' % (i + 1), trans) + num_features = num_features // 2 + + # Final batch norm + self.features.add_module('norm5', nn.BatchNorm2d(num_features)) + + # Linear layer + self.classifier = nn.Linear(num_features, num_classes) + + # Official init from torch repo. + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight) + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + nn.init.constant_(m.bias, 0) + + def forward(self, x): + features = self.features(x) + out = F.relu(features, inplace=True) + out = F.adaptive_avg_pool2d(out, (1, 1)).view(features.size(0), -1) + out = self.classifier(out) + return out + + +def densenet121(pretrained=False, **kwargs): + r"""Densenet-121 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = DenseNet(num_init_features=64, growth_rate=32, block_config=(6, 12, 24, 16), + **kwargs) + if pretrained: + # '.'s are no longer allowed in module names, but pervious _DenseLayer + # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. + # They are also in the checkpoints in model_urls. This pattern is used + # to find such keys. + pattern = re.compile( + r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') + state_dict = model_zoo.load_url(model_urls['densenet121']) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict) + return model + + +def densenet169(pretrained=False, **kwargs): + r"""Densenet-169 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = DenseNet(num_init_features=64, growth_rate=32, block_config=(6, 12, 32, 32), + **kwargs) + if pretrained: + # '.'s are no longer allowed in module names, but pervious _DenseLayer + # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. + # They are also in the checkpoints in model_urls. This pattern is used + # to find such keys. + pattern = re.compile( + r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') + state_dict = model_zoo.load_url(model_urls['densenet169']) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict) + return model + + +def densenet201(pretrained=False, **kwargs): + r"""Densenet-201 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = DenseNet(num_init_features=64, growth_rate=32, block_config=(6, 12, 48, 32), + **kwargs) + if pretrained: + # '.'s are no longer allowed in module names, but pervious _DenseLayer + # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. + # They are also in the checkpoints in model_urls. This pattern is used + # to find such keys. + pattern = re.compile( + r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') + state_dict = model_zoo.load_url(model_urls['densenet201']) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict) + return model + + +def densenet161(pretrained=False, **kwargs): + r"""Densenet-161 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = DenseNet(num_init_features=96, growth_rate=48, block_config=(6, 12, 36, 24), + **kwargs) + if pretrained: + # '.'s are no longer allowed in module names, but pervious _DenseLayer + # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. + # They are also in the checkpoints in model_urls. This pattern is used + # to find such keys. + pattern = re.compile( + r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') + state_dict = model_zoo.load_url(model_urls['densenet161']) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict) + return model diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/densenet_0_5_0.py b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/densenet_0_5_0.py new file mode 100644 index 0000000..df6c9cc --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/densenet_0_5_0.py @@ -0,0 +1,279 @@ +import re +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as cp +from collections import OrderedDict +#from .utils import load_state_dict_from_url +from torch import Tensor +from torch.jit.annotations import List + + +__all__ = ['DenseNet', 'densenet121', 'densenet169', 'densenet201', 'densenet161'] + +model_urls = { + 'densenet121': 'https://download.pytorch.org/models/densenet121-a639ec97.pth', + 'densenet169': 'https://download.pytorch.org/models/densenet169-b2777c0a.pth', + 'densenet201': 'https://download.pytorch.org/models/densenet201-c1103571.pth', + 'densenet161': 'https://download.pytorch.org/models/densenet161-8d451a50.pth', +} + + +class _DenseLayer(nn.Module): + def __init__(self, num_input_features, growth_rate, bn_size, drop_rate, memory_efficient=False): + super(_DenseLayer, self).__init__() + self.add_module('norm1', nn.BatchNorm2d(num_input_features)), + self.add_module('relu1', nn.ReLU(inplace=True)), + self.add_module('conv1', nn.Conv2d(num_input_features, bn_size * + growth_rate, kernel_size=1, stride=1, + bias=False)), + self.add_module('norm2', nn.BatchNorm2d(bn_size * growth_rate)), + self.add_module('relu2', nn.ReLU(inplace=True)), + self.add_module('conv2', nn.Conv2d(bn_size * growth_rate, growth_rate, + kernel_size=3, stride=1, padding=1, + bias=False)), + self.drop_rate = float(drop_rate) + self.memory_efficient = memory_efficient + + def bn_function(self, inputs): + # type: (List[Tensor]) -> Tensor + concated_features = torch.cat(inputs, 1) + bottleneck_output = self.conv1(self.relu1(self.norm1(concated_features))) # noqa: T484 + return bottleneck_output + + # todo: rewrite when torchscript supports any + def any_requires_grad(self, input): + # type: (List[Tensor]) -> bool + for tensor in input: + if tensor.requires_grad: + return True + return False + + @torch.jit.unused # noqa: T484 + def call_checkpoint_bottleneck(self, input): + # type: (List[Tensor]) -> Tensor + def closure(*inputs): + return self.bn_function(*inputs) + + return cp.checkpoint(closure, input) + + @torch.jit._overload_method # noqa: F811 + def forward(self, input): + # type: (List[Tensor]) -> (Tensor) + pass + + @torch.jit._overload_method # noqa: F811 + def forward(self, input): + # type: (Tensor) -> (Tensor) + pass + + # torchscript does not yet support *args, so we overload method + # allowing it to take either a List[Tensor] or single Tensor + def forward(self, input): # noqa: F811 + if isinstance(input, Tensor): + prev_features = [input] + else: + prev_features = input + + if self.memory_efficient and self.any_requires_grad(prev_features): + if torch.jit.is_scripting(): + raise Exception("Memory Efficient not supported in JIT") + + bottleneck_output = self.call_checkpoint_bottleneck(prev_features) + else: + bottleneck_output = self.bn_function(prev_features) + + new_features = self.conv2(self.relu2(self.norm2(bottleneck_output))) + if self.drop_rate > 0: + new_features = F.dropout(new_features, p=self.drop_rate, + training=self.training) + return new_features + + +class _DenseBlock(nn.ModuleDict): + _version = 2 + + def __init__(self, num_layers, num_input_features, bn_size, growth_rate, drop_rate, memory_efficient=False): + super(_DenseBlock, self).__init__() + for i in range(num_layers): + layer = _DenseLayer( + num_input_features + i * growth_rate, + growth_rate=growth_rate, + bn_size=bn_size, + drop_rate=drop_rate, + memory_efficient=memory_efficient, + ) + self.add_module('denselayer%d' % (i + 1), layer) + + def forward(self, init_features): + features = [init_features] + for name, layer in self.items(): + new_features = layer(features) + features.append(new_features) + return torch.cat(features, 1) + + +class _Transition(nn.Sequential): + def __init__(self, num_input_features, num_output_features): + super(_Transition, self).__init__() + self.add_module('norm', nn.BatchNorm2d(num_input_features)) + self.add_module('relu', nn.ReLU(inplace=True)) + self.add_module('conv', nn.Conv2d(num_input_features, num_output_features, + kernel_size=1, stride=1, bias=False)) + self.add_module('pool', nn.AvgPool2d(kernel_size=2, stride=2)) + + +class DenseNet(nn.Module): + r"""Densenet-BC model class, based on + `"Densely Connected Convolutional Networks" `_ + + Args: + growth_rate (int) - how many filters to add each layer (`k` in paper) + block_config (list of 4 ints) - how many layers in each pooling block + num_init_features (int) - the number of filters to learn in the first convolution layer + bn_size (int) - multiplicative factor for number of bottle neck layers + (i.e. bn_size * k features in the bottleneck layer) + drop_rate (float) - dropout rate after each dense layer + num_classes (int) - number of classification classes + memory_efficient (bool) - If True, uses checkpointing. Much more memory efficient, + but slower. Default: *False*. See `"paper" `_ + """ + + def __init__(self, growth_rate=32, block_config=(6, 12, 24, 16), + num_init_features=64, bn_size=4, drop_rate=0, num_classes=1000, memory_efficient=False): + + super(DenseNet, self).__init__() + + # First convolution + self.features = nn.Sequential(OrderedDict([ + ('conv0', nn.Conv2d(3, num_init_features, kernel_size=7, stride=2, + padding=3, bias=False)), + ('norm0', nn.BatchNorm2d(num_init_features)), + ('relu0', nn.ReLU(inplace=True)), + ('pool0', nn.MaxPool2d(kernel_size=3, stride=2, padding=1)), + ])) + + # Each denseblock + num_features = num_init_features + for i, num_layers in enumerate(block_config): + block = _DenseBlock( + num_layers=num_layers, + num_input_features=num_features, + bn_size=bn_size, + growth_rate=growth_rate, + drop_rate=drop_rate, + memory_efficient=memory_efficient + ) + self.features.add_module('denseblock%d' % (i + 1), block) + num_features = num_features + num_layers * growth_rate + if i != len(block_config) - 1: + trans = _Transition(num_input_features=num_features, + num_output_features=num_features // 2) + self.features.add_module('transition%d' % (i + 1), trans) + num_features = num_features // 2 + + # Final batch norm + self.features.add_module('norm5', nn.BatchNorm2d(num_features)) + + # Linear layer + self.classifier = nn.Linear(num_features, num_classes) + + # Official init from torch repo. + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight) + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + nn.init.constant_(m.bias, 0) + + def forward(self, x): + features = self.features(x) + out = F.relu(features, inplace=True) + out = F.adaptive_avg_pool2d(out, (1, 1)) + out = torch.flatten(out, 1) + out = self.classifier(out) + return out + + +def _load_state_dict(model, model_url, progress): + # '.'s are no longer allowed in module names, but previous _DenseLayer + # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. + # They are also in the checkpoints in model_urls. This pattern is used + # to find such keys. + pattern = re.compile( + r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') + + state_dict = load_state_dict_from_url(model_url, progress=progress) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict) + + +def _densenet(arch, growth_rate, block_config, num_init_features, pretrained, progress, + **kwargs): + model = DenseNet(growth_rate, block_config, num_init_features, **kwargs) + if pretrained: + _load_state_dict(model, model_urls[arch], progress) + return model + + +def densenet121(pretrained=False, progress=True, **kwargs): + r"""Densenet-121 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + memory_efficient (bool) - If True, uses checkpointing. Much more memory efficient, + but slower. Default: *False*. See `"paper" `_ + """ + return _densenet('densenet121', 32, (6, 12, 24, 16), 64, pretrained, progress, + **kwargs) + + +def densenet161(pretrained=False, progress=True, **kwargs): + r"""Densenet-161 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + memory_efficient (bool) - If True, uses checkpointing. Much more memory efficient, + but slower. Default: *False*. See `"paper" `_ + """ + return _densenet('densenet161', 48, (6, 12, 36, 24), 96, pretrained, progress, + **kwargs) + + +def densenet169(pretrained=False, progress=True, **kwargs): + r"""Densenet-169 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + memory_efficient (bool) - If True, uses checkpointing. Much more memory efficient, + but slower. Default: *False*. See `"paper" `_ + """ + return _densenet('densenet169', 32, (6, 12, 32, 32), 64, pretrained, progress, + **kwargs) + + +def densenet201(pretrained=False, progress=True, **kwargs): + r"""Densenet-201 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + memory_efficient (bool) - If True, uses checkpointing. Much more memory efficient, + but slower. Default: *False*. See `"paper" `_ + """ + return _densenet('densenet201', 32, (6, 12, 48, 32), 64, pretrained, progress, + **kwargs) diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/eval.sh b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/eval.sh new file mode 100644 index 0000000..16d5c5e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/eval.sh @@ -0,0 +1,22 @@ +export ASCEND_HOME=/usr/local/Ascend +export LD_LIBRARY_PATH=/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/ +export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/te:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/topi:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/hccl:/usr/local/Ascend/ascend-toolkit/latest/tfplugin/python/site-packages:$currentDir +export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin +export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + +export SLOG_PRINT_TO_STDOUT=0 +su HwHiAiUser -c "adc --host 0.0.0.0:22118 --log \"SetLogLevel(0)[error]\" --device 7" + +export TASK_QUEUE_ENABLE=0 +taskset -c 111-150 python3 densenet121_1p_main.py \ + --workers 40 \ + --arch densenet121 \ + --npu 7 \ + --lr 0.1 \ + --momentum 0.9 \ + --amp \ + --batch-size 256 \ + --epoch 90 \ + --evaluate \ + --resume checkpoint.pth.tar \ + --data /opt/npu/dataset/imagenet \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/cpu_adapter_file/densenet.py.cpu b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/cpu_adapter_file/densenet.py.cpu new file mode 100644 index 0000000..59b935d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/cpu_adapter_file/densenet.py.cpu @@ -0,0 +1,275 @@ +import re +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.model_zoo as model_zoo +from collections import OrderedDict + +__all__ = ['DenseNet', 'densenet121', 'densenet169', 'densenet201', 'densenet161'] + + +model_urls = { + 'densenet121': 'https://download.pytorch.org/models/densenet121-a639ec97.pth', + 'densenet169': 'https://download.pytorch.org/models/densenet169-b2777c0a.pth', + 'densenet201': 'https://download.pytorch.org/models/densenet201-c1103571.pth', + 'densenet161': 'https://download.pytorch.org/models/densenet161-8d451a50.pth', +} + + +class _DenseLayer(nn.Sequential): + def __init__(self, num_input_features, growth_rate, bn_size, drop_rate): + super(_DenseLayer, self).__init__() + self.add_module('norm1', nn.BatchNorm2d(num_input_features)), + self.add_module('relu1', nn.ReLU(inplace=True)), + self.add_module('conv1', nn.Conv2d(num_input_features, bn_size * + growth_rate, kernel_size=1, stride=1, bias=False)), + self.add_module('norm2', nn.BatchNorm2d(bn_size * growth_rate)), + self.add_module('relu2', nn.ReLU(inplace=True)), + self.add_module('conv2', nn.Conv2d(bn_size * growth_rate, growth_rate, + kernel_size=3, stride=1, padding=1, bias=False)), + self.drop_rate = drop_rate + + def forward(self, x): + new_features = super(_DenseLayer, self).forward(x) + if self.drop_rate > 0: + new_features = F.dropout(new_features, p=self.drop_rate, training=self.training) + return torch.cat([x, new_features], 1) + + +class _DenseBlock(nn.Sequential): + def __init__(self, num_layers, num_input_features, bn_size, growth_rate, drop_rate): + super(_DenseBlock, self).__init__() + for i in range(num_layers): + layer = _DenseLayer(num_input_features + i * growth_rate, growth_rate, bn_size, drop_rate) + self.add_module('denselayer%d' % (i + 1), layer) + + +class _Transition(nn.Sequential): + def __init__(self, num_input_features, num_output_features): + super(_Transition, self).__init__() + self.add_module('norm', nn.BatchNorm2d(num_input_features)) + self.add_module('relu', nn.ReLU(inplace=True)) + self.add_module('conv', nn.Conv2d(num_input_features, num_output_features, + kernel_size=1, stride=1, bias=False)) + #self.add_module('pool', nn.AvgPool2d(kernel_size=2, stride=2)) ######### xupeng add ########## + + +class DenseNet(nn.Module): + r"""Densenet-BC model class, based on + `"Densely Connected Convolutional Networks" `_ + + Args: + growth_rate (int) - how many filters to add each layer (`k` in paper) + block_config (list of 4 ints) - how many layers in each pooling block + num_init_features (int) - the number of filters to learn in the first convolution layer + bn_size (int) - multiplicative factor for number of bottle neck layers + (i.e. bn_size * k features in the bottleneck layer) + drop_rate (float) - dropout rate after each dense layer + num_classes (int) - number of classification classes + """ + + def __init__(self, growth_rate=32, block_config=(6, 12, 24, 16), + num_init_features=64, bn_size=4, drop_rate=0, num_classes=1000): + + super(DenseNet, self).__init__() + + self.avg_pool = nn.AvgPool2d(kernel_size=2, stride=2) + + ################ block 0 ################ + num_features = num_init_features + i=0 + num_layers=block_config[i] + block = _DenseBlock(num_layers=num_layers, num_input_features=num_features, bn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate) + num_features = num_features + num_layers * growth_rate + trans = _Transition(num_input_features=num_features, num_output_features=num_features // 2) + self.features0 = nn.Sequential(OrderedDict([ + ('conv0', nn.Conv2d(3, num_init_features, kernel_size=7, stride=2, padding=3, bias=False)), + ('norm0', nn.BatchNorm2d(num_init_features)), + ('relu0', nn.ReLU(inplace=True)), + ('pool0', nn.MaxPool2d(kernel_size=3, stride=2, padding=1)), + ('denseblock%d' % (i + 1), block), + ('transition%d' % (i + 1), trans) + ])) + + ################ block 1 ############## + num_features = num_features // 2 + i=1 + num_layers=block_config[i] + block = _DenseBlock(num_layers=num_layers, num_input_features=num_features, bn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate) + num_features = num_features + num_layers * growth_rate + trans = _Transition(num_input_features=num_features, num_output_features=num_features // 2) + self.features1 = nn.Sequential(OrderedDict([ + ('denseblock%d' % (i + 1), block), + ('transition%d' % (i + 1), trans), + ])) + + ################ block 2 ############## + num_features = num_features // 2 + i=2 + num_layers=block_config[i] + block = _DenseBlock(num_layers=num_layers, num_input_features=num_features, bn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate) + num_features = num_features + num_layers * growth_rate + trans = _Transition(num_input_features=num_features, num_output_features=num_features // 2) + self.features2 = nn.Sequential(OrderedDict([ + ('denseblock%d' % (i + 1), block), + ('transition%d' % (i + 1), trans), + ])) + + ################ block 3 ############## + num_features = num_features // 2 + i=3 + num_layers=block_config[i] + block = _DenseBlock(num_layers=num_layers, num_input_features=num_features, bn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate) + num_features = num_features + num_layers * growth_rate + self.features3 = nn.Sequential(OrderedDict([ + ('denseblock%d' % (i + 1), block), + ('norm5', nn.BatchNorm2d(num_features)), + ])) + + # Linear layer + self.classifier = nn.Linear(num_features, num_classes) + + # Official init from torch repo. + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight) + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + nn.init.constant_(m.bias, 0) + + def forward(self, x): + #CALCULATE_DEVICE = "npu:0" + #self.avg_pool = self.avg_pool.cpu() + #print("avg_pool move to cpu") + #print("tag0") + features0 = self.features0(x) + #features0 = features0.cpu() + avg_pool_0 = self.avg_pool(features0) + #avg_pool_0 = avg_pool_0.to(CALCULATE_DEVICE) + #print("tag1") + features1 = self.features1(avg_pool_0) + #features1 = features1.cpu() + avg_pool_1 = self.avg_pool(features1) + #avg_pool_1 = avg_pool_1.to(CALCULATE_DEVICE) + #print("tag2") + features2 = self.features2(avg_pool_1) + #features2 = features2.cpu() + avg_pool_2 = self.avg_pool(features2) + #avg_pool_2 = avg_pool_2.to(CALCULATE_DEVICE) + #print("tag3") + features3 = self.features3(avg_pool_2) + + out = F.relu(features3, inplace=True) + out = F.adaptive_avg_pool2d(out, (1, 1)).view(features3.size(0), -1) + out = self.classifier(out) + return out + + +def densenet121(pretrained=False, **kwargs): + r"""Densenet-121 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = DenseNet(num_init_features=64, growth_rate=32, block_config=(6, 12, 24, 16), + **kwargs) + if pretrained: + # '.'s are no longer allowed in module names, but pervious _DenseLayer + # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. + # They are also in the checkpoints in model_urls. This pattern is used + # to find such keys. + pattern = re.compile( + r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') + state_dict = model_zoo.load_url(model_urls['densenet121']) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict) + return model + + +def densenet169(pretrained=False, **kwargs): + r"""Densenet-169 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = DenseNet(num_init_features=64, growth_rate=32, block_config=(6, 12, 32, 32), + **kwargs) + if pretrained: + # '.'s are no longer allowed in module names, but pervious _DenseLayer + # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. + # They are also in the checkpoints in model_urls. This pattern is used + # to find such keys. + pattern = re.compile( + r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') + state_dict = model_zoo.load_url(model_urls['densenet169']) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict) + return model + + +def densenet201(pretrained=False, **kwargs): + r"""Densenet-201 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = DenseNet(num_init_features=64, growth_rate=32, block_config=(6, 12, 48, 32), + **kwargs) + if pretrained: + # '.'s are no longer allowed in module names, but pervious _DenseLayer + # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. + # They are also in the checkpoints in model_urls. This pattern is used + # to find such keys. + pattern = re.compile( + r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') + state_dict = model_zoo.load_url(model_urls['densenet201']) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict) + return model + + +def densenet161(pretrained=False, **kwargs): + r"""Densenet-161 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = DenseNet(num_init_features=96, growth_rate=48, block_config=(6, 12, 36, 24), + **kwargs) + if pretrained: + # '.'s are no longer allowed in module names, but pervious _DenseLayer + # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. + # They are also in the checkpoints in model_urls. This pattern is used + # to find such keys. + pattern = re.compile( + r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') + state_dict = model_zoo.load_url(model_urls['densenet161']) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict) + return model diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/cpu_adapter_file/densenet.py.npu b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/cpu_adapter_file/densenet.py.npu new file mode 100644 index 0000000..baa8f2c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/cpu_adapter_file/densenet.py.npu @@ -0,0 +1,275 @@ +import re +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.model_zoo as model_zoo +from collections import OrderedDict + +__all__ = ['DenseNet', 'densenet121', 'densenet169', 'densenet201', 'densenet161'] + + +model_urls = { + 'densenet121': 'https://download.pytorch.org/models/densenet121-a639ec97.pth', + 'densenet169': 'https://download.pytorch.org/models/densenet169-b2777c0a.pth', + 'densenet201': 'https://download.pytorch.org/models/densenet201-c1103571.pth', + 'densenet161': 'https://download.pytorch.org/models/densenet161-8d451a50.pth', +} + + +class _DenseLayer(nn.Sequential): + def __init__(self, num_input_features, growth_rate, bn_size, drop_rate): + super(_DenseLayer, self).__init__() + self.add_module('norm1', nn.BatchNorm2d(num_input_features)), + self.add_module('relu1', nn.ReLU(inplace=True)), + self.add_module('conv1', nn.Conv2d(num_input_features, bn_size * + growth_rate, kernel_size=1, stride=1, bias=False)), + self.add_module('norm2', nn.BatchNorm2d(bn_size * growth_rate)), + self.add_module('relu2', nn.ReLU(inplace=True)), + self.add_module('conv2', nn.Conv2d(bn_size * growth_rate, growth_rate, + kernel_size=3, stride=1, padding=1, bias=False)), + self.drop_rate = drop_rate + + def forward(self, x): + new_features = super(_DenseLayer, self).forward(x) + if self.drop_rate > 0: + new_features = F.dropout(new_features, p=self.drop_rate, training=self.training) + return torch.cat([x, new_features], 1) + + +class _DenseBlock(nn.Sequential): + def __init__(self, num_layers, num_input_features, bn_size, growth_rate, drop_rate): + super(_DenseBlock, self).__init__() + for i in range(num_layers): + layer = _DenseLayer(num_input_features + i * growth_rate, growth_rate, bn_size, drop_rate) + self.add_module('denselayer%d' % (i + 1), layer) + + +class _Transition(nn.Sequential): + def __init__(self, num_input_features, num_output_features): + super(_Transition, self).__init__() + self.add_module('norm', nn.BatchNorm2d(num_input_features)) + self.add_module('relu', nn.ReLU(inplace=True)) + self.add_module('conv', nn.Conv2d(num_input_features, num_output_features, + kernel_size=1, stride=1, bias=False)) + #self.add_module('pool', nn.AvgPool2d(kernel_size=2, stride=2)) ######### xupeng add ########## + + +class DenseNet(nn.Module): + r"""Densenet-BC model class, based on + `"Densely Connected Convolutional Networks" `_ + + Args: + growth_rate (int) - how many filters to add each layer (`k` in paper) + block_config (list of 4 ints) - how many layers in each pooling block + num_init_features (int) - the number of filters to learn in the first convolution layer + bn_size (int) - multiplicative factor for number of bottle neck layers + (i.e. bn_size * k features in the bottleneck layer) + drop_rate (float) - dropout rate after each dense layer + num_classes (int) - number of classification classes + """ + + def __init__(self, growth_rate=32, block_config=(6, 12, 24, 16), + num_init_features=64, bn_size=4, drop_rate=0, num_classes=1000): + + super(DenseNet, self).__init__() + + self.avg_pool = nn.AvgPool2d(kernel_size=2, stride=2) + + ################ block 0 ################ + num_features = num_init_features + i=0 + num_layers=block_config[i] + block = _DenseBlock(num_layers=num_layers, num_input_features=num_features, bn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate) + num_features = num_features + num_layers * growth_rate + trans = _Transition(num_input_features=num_features, num_output_features=num_features // 2) + self.features0 = nn.Sequential(OrderedDict([ + ('conv0', nn.Conv2d(3, num_init_features, kernel_size=7, stride=2, padding=3, bias=False)), + ('norm0', nn.BatchNorm2d(num_init_features)), + ('relu0', nn.ReLU(inplace=True)), + ('pool0', nn.MaxPool2d(kernel_size=3, stride=2, padding=1)), + ('denseblock%d' % (i + 1), block), + ('transition%d' % (i + 1), trans) + ])) + + ################ block 1 ############## + num_features = num_features // 2 + i=1 + num_layers=block_config[i] + block = _DenseBlock(num_layers=num_layers, num_input_features=num_features, bn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate) + num_features = num_features + num_layers * growth_rate + trans = _Transition(num_input_features=num_features, num_output_features=num_features // 2) + self.features1 = nn.Sequential(OrderedDict([ + ('denseblock%d' % (i + 1), block), + ('transition%d' % (i + 1), trans), + ])) + + ################ block 2 ############## + num_features = num_features // 2 + i=2 + num_layers=block_config[i] + block = _DenseBlock(num_layers=num_layers, num_input_features=num_features, bn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate) + num_features = num_features + num_layers * growth_rate + trans = _Transition(num_input_features=num_features, num_output_features=num_features // 2) + self.features2 = nn.Sequential(OrderedDict([ + ('denseblock%d' % (i + 1), block), + ('transition%d' % (i + 1), trans), + ])) + + ################ block 3 ############## + num_features = num_features // 2 + i=3 + num_layers=block_config[i] + block = _DenseBlock(num_layers=num_layers, num_input_features=num_features, bn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate) + num_features = num_features + num_layers * growth_rate + self.features3 = nn.Sequential(OrderedDict([ + ('denseblock%d' % (i + 1), block), + ('norm5', nn.BatchNorm2d(num_features)), + ])) + + # Linear layer + self.classifier = nn.Linear(num_features, num_classes) + + # Official init from torch repo. + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight) + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + nn.init.constant_(m.bias, 0) + + def forward(self, x): + CALCULATE_DEVICE = "npu:0" + self.avg_pool = self.avg_pool.cpu() + #print("avg_pool move to cpu") + #print("tag0") + features0 = self.features0(x) + features0 = features0.cpu() + avg_pool_0 = self.avg_pool(features0) + avg_pool_0 = avg_pool_0.to(CALCULATE_DEVICE) + #print("tag1") + features1 = self.features1(avg_pool_0) + features1 = features1.cpu() + avg_pool_1 = self.avg_pool(features1) + avg_pool_1 = avg_pool_1.to(CALCULATE_DEVICE) + #print("tag2") + features2 = self.features2(avg_pool_1) + features2 = features2.cpu() + avg_pool_2 = self.avg_pool(features2) + avg_pool_2 = avg_pool_2.to(CALCULATE_DEVICE) + #print("tag3") + features3 = self.features3(avg_pool_2) + + out = F.relu(features3, inplace=True) + out = F.adaptive_avg_pool2d(out, (1, 1)).view(features3.size(0), -1) + out = self.classifier(out) + return out + + +def densenet121(pretrained=False, **kwargs): + r"""Densenet-121 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = DenseNet(num_init_features=64, growth_rate=32, block_config=(6, 12, 24, 16), + **kwargs) + if pretrained: + # '.'s are no longer allowed in module names, but pervious _DenseLayer + # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. + # They are also in the checkpoints in model_urls. This pattern is used + # to find such keys. + pattern = re.compile( + r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') + state_dict = model_zoo.load_url(model_urls['densenet121']) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict) + return model + + +def densenet169(pretrained=False, **kwargs): + r"""Densenet-169 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = DenseNet(num_init_features=64, growth_rate=32, block_config=(6, 12, 32, 32), + **kwargs) + if pretrained: + # '.'s are no longer allowed in module names, but pervious _DenseLayer + # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. + # They are also in the checkpoints in model_urls. This pattern is used + # to find such keys. + pattern = re.compile( + r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') + state_dict = model_zoo.load_url(model_urls['densenet169']) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict) + return model + + +def densenet201(pretrained=False, **kwargs): + r"""Densenet-201 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = DenseNet(num_init_features=64, growth_rate=32, block_config=(6, 12, 48, 32), + **kwargs) + if pretrained: + # '.'s are no longer allowed in module names, but pervious _DenseLayer + # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. + # They are also in the checkpoints in model_urls. This pattern is used + # to find such keys. + pattern = re.compile( + r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') + state_dict = model_zoo.load_url(model_urls['densenet201']) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict) + return model + + +def densenet161(pretrained=False, **kwargs): + r"""Densenet-161 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = DenseNet(num_init_features=96, growth_rate=48, block_config=(6, 12, 36, 24), + **kwargs) + if pretrained: + # '.'s are no longer allowed in module names, but pervious _DenseLayer + # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. + # They are also in the checkpoints in model_urls. This pattern is used + # to find such keys. + pattern = re.compile( + r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') + state_dict = model_zoo.load_url(model_urls['densenet161']) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict) + return model diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/densenet_print.py b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/densenet_print.py new file mode 100644 index 0000000..3011376 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/densenet_print.py @@ -0,0 +1,300 @@ +import re +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as cp +from collections import OrderedDict +#from .utils import load_state_dict_from_url +from torch import Tensor +from torch.jit.annotations import List + + +__all__ = ['DenseNet', 'densenet121', 'densenet169', 'densenet201', 'densenet161'] + +model_urls = { + 'densenet121': 'https://download.pytorch.org/models/densenet121-a639ec97.pth', + 'densenet169': 'https://download.pytorch.org/models/densenet169-b2777c0a.pth', + 'densenet201': 'https://download.pytorch.org/models/densenet201-c1103571.pth', + 'densenet161': 'https://download.pytorch.org/models/densenet161-8d451a50.pth', +} + + +class _DenseLayer(nn.Module): + def __init__(self, num_input_features, growth_rate, bn_size, drop_rate, memory_efficient=False): + super(_DenseLayer, self).__init__() + self.add_module('norm1', nn.BatchNorm2d(num_input_features)), + self.add_module('relu1', nn.ReLU(inplace=True)), + self.add_module('conv1', nn.Conv2d(num_input_features, bn_size * + growth_rate, kernel_size=1, stride=1, + bias=False)), + self.add_module('norm2', nn.BatchNorm2d(bn_size * growth_rate)), + self.add_module('relu2', nn.ReLU(inplace=True)), + self.add_module('conv2', nn.Conv2d(bn_size * growth_rate, growth_rate, + kernel_size=3, stride=1, padding=1, + bias=False)), + self.drop_rate = float(drop_rate) + self.memory_efficient = memory_efficient + + def bn_function(self, inputs): + # type: (List[Tensor]) -> Tensor + concated_features = torch.cat(inputs, 1) + bottleneck_output = self.conv1(self.relu1(self.norm1(concated_features))) # noqa: T484 + return bottleneck_output + + # todo: rewrite when torchscript supports any + def any_requires_grad(self, input): + # type: (List[Tensor]) -> bool + for tensor in input: + if tensor.requires_grad: + return True + return False + + @torch.jit.unused # noqa: T484 + def call_checkpoint_bottleneck(self, input): + # type: (List[Tensor]) -> Tensor + def closure(*inputs): + return self.bn_function(*inputs) + + return cp.checkpoint(closure, input) + + @torch.jit._overload_method # noqa: F811 + def forward(self, input): + # type: (List[Tensor]) -> (Tensor) + pass + + @torch.jit._overload_method # noqa: F811 + def forward(self, input): + # type: (Tensor) -> (Tensor) + pass + + # torchscript does not yet support *args, so we overload method + # allowing it to take either a List[Tensor] or single Tensor + def forward(self, input): # noqa: F811 + if isinstance(input, Tensor): + prev_features = [input] + else: + prev_features = input + + if self.memory_efficient and self.any_requires_grad(prev_features): + if torch.jit.is_scripting(): + raise Exception("Memory Efficient not supported in JIT") + + bottleneck_output = self.call_checkpoint_bottleneck(prev_features) + else: + bottleneck_output = self.bn_function(prev_features) + + new_features = self.conv2(self.relu2(self.norm2(bottleneck_output))) + if self.drop_rate > 0: + new_features = F.dropout(new_features, p=self.drop_rate, + training=self.training) + return new_features + + +class _DenseBlock(nn.ModuleDict): + _version = 2 + + def __init__(self, num_layers, num_input_features, bn_size, growth_rate, drop_rate, memory_efficient=False): + super(_DenseBlock, self).__init__() + for i in range(num_layers): + layer = _DenseLayer( + num_input_features + i * growth_rate, + growth_rate=growth_rate, + bn_size=bn_size, + drop_rate=drop_rate, + memory_efficient=memory_efficient, + ) + self.add_module('denselayer%d' % (i + 1), layer) + + def forward(self, init_features): + features = [init_features] + for name, layer in self.items(): + new_features = layer(features) + features.append(new_features) + return torch.cat(features, 1) + + +class _Transition(nn.Sequential): + def __init__(self, num_input_features, num_output_features): + super(_Transition, self).__init__() + self.add_module('norm', nn.BatchNorm2d(num_input_features)) + self.add_module('relu', nn.ReLU(inplace=True)) + self.add_module('conv', nn.Conv2d(num_input_features, num_output_features, + kernel_size=1, stride=1, bias=False)) + self.add_module('pool', nn.AvgPool2d(kernel_size=2, stride=2)) + +class PrintLayer(nn.Module): + def __init__(self, name): + super(PrintLayer, self).__init__() + self.name = name + + def forward(self, x): + # Do your print / debug stuff here + print("{} mean data: {}".format(self.name, x.mean().item())) #print(x.shape) + return x + +class DenseNet(nn.Module): + r"""Densenet-BC model class, based on + `"Densely Connected Convolutional Networks" `_ + + Args: + growth_rate (int) - how many filters to add each layer (`k` in paper) + block_config (list of 4 ints) - how many layers in each pooling block + num_init_features (int) - the number of filters to learn in the first convolution layer + bn_size (int) - multiplicative factor for number of bottle neck layers + (i.e. bn_size * k features in the bottleneck layer) + drop_rate (float) - dropout rate after each dense layer + num_classes (int) - number of classification classes + memory_efficient (bool) - If True, uses checkpointing. Much more memory efficient, + but slower. Default: *False*. See `"paper" `_ + """ + + def __init__(self, growth_rate=32, block_config=(6, 12, 24, 16), + num_init_features=64, bn_size=4, drop_rate=0, num_classes=1000, memory_efficient=False): + + super(DenseNet, self).__init__() + + # First convolution + self.features = nn.Sequential(OrderedDict([ + ('conv0', nn.Conv2d(3, num_init_features, kernel_size=7, stride=2, + padding=3, bias=False)), + ('conv0_p', PrintLayer('conv0_p')), + ('norm0', nn.BatchNorm2d(num_init_features)), + ('norm0_p', PrintLayer('norm0_p')), + ('relu0', nn.ReLU(inplace=True)), + ('pool0', nn.MaxPool2d(kernel_size=3, stride=2, padding=1)), + ('pool0_p', PrintLayer('pool0_p')), + ])) + + + + # Each denseblock + num_features = num_init_features + for i, num_layers in enumerate(block_config): + block = _DenseBlock( + num_layers=num_layers, + num_input_features=num_features, + bn_size=bn_size, + growth_rate=growth_rate, + drop_rate=drop_rate, + memory_efficient=memory_efficient + ) + self.features.add_module('denseblock%d' % (i + 1), block) + self.features.add_module('denseblock%d_p' % (i + 1), PrintLayer('denseblock%d_p' % (i + 1))) + num_features = num_features + num_layers * growth_rate + if i != len(block_config) - 1: + trans = _Transition(num_input_features=num_features, + num_output_features=num_features // 2) + self.features.add_module('transition%d' % (i + 1), trans) + self.features.add_module('transition%d_p' % (i + 1), PrintLayer('transition%d_p' % (i + 1))) + num_features = num_features // 2 + + # Final batch norm + self.features.add_module('norm5', nn.BatchNorm2d(num_features)) + + # Linear layer + self.classifier = nn.Linear(num_features, num_classes) + + # Official init from torch repo. + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight) + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + nn.init.constant_(m.bias, 0) + + def forward(self, x): + features = self.features(x) + + # features_p = features.to('cpu')' + print('the features mean: {}'.format(features.mean().item())) + + out = F.relu(features, inplace=True) + out = F.adaptive_avg_pool2d(out, (1, 1)) + out = torch.flatten(out, 1) + print('the flatten mean: {}'.format(out.mean().item())) + out = self.classifier(out) + return out + + +def _load_state_dict(model, model_url, progress): + # '.'s are no longer allowed in module names, but previous _DenseLayer + # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. + # They are also in the checkpoints in model_urls. This pattern is used + # to find such keys. + pattern = re.compile( + r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') + + state_dict = load_state_dict_from_url(model_url, progress=progress) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict) + + +def _densenet(arch, growth_rate, block_config, num_init_features, pretrained, progress, + **kwargs): + model = DenseNet(growth_rate, block_config, num_init_features, **kwargs) + if pretrained: + _load_state_dict(model, model_urls[arch], progress) + return model + + +def densenet121(pretrained=False, progress=True, **kwargs): + r"""Densenet-121 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + memory_efficient (bool) - If True, uses checkpointing. Much more memory efficient, + but slower. Default: *False*. See `"paper" `_ + """ + return _densenet('densenet121', 32, (6, 12, 24, 16), 64, pretrained, progress, + **kwargs) + + +def densenet161(pretrained=False, progress=True, **kwargs): + r"""Densenet-161 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + memory_efficient (bool) - If True, uses checkpointing. Much more memory efficient, + but slower. Default: *False*. See `"paper" `_ + """ + return _densenet('densenet161', 48, (6, 12, 36, 24), 96, pretrained, progress, + **kwargs) + + +def densenet169(pretrained=False, progress=True, **kwargs): + r"""Densenet-169 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + memory_efficient (bool) - If True, uses checkpointing. Much more memory efficient, + but slower. Default: *False*. See `"paper" `_ + """ + return _densenet('densenet169', 32, (6, 12, 32, 32), 64, pretrained, progress, + **kwargs) + + +def densenet201(pretrained=False, progress=True, **kwargs): + r"""Densenet-201 model from + `"Densely Connected Convolutional Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + memory_efficient (bool) - If True, uses checkpointing. Much more memory efficient, + but slower. Default: *False*. See `"paper" `_ + """ + return _densenet('densenet201', 32, (6, 12, 48, 32), 64, pretrained, progress, + **kwargs) diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/npu_config/1p.json b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/npu_config/1p.json new file mode 100644 index 0000000..aef2e79 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/npu_config/1p.json @@ -0,0 +1,32 @@ +{ + "board_id": "0x0000", + "chip_info": "910", + "deploy_mode": "lab", + "group_count": "1", + "group_list": [ + { + "device_num": "1", + "server_num": "1", + "group_name": "", + "instance_count": "1", + "instance_list": [ + { + "devices": [ + { + "device_id": "0", + "device_ip": "192.168.100.101" + } + ], + "rank_id": "0", + "server_id": "10.246.246.76" + } + ] + } + ], + "para_plane_nic_location": "device", + "para_plane_nic_name": [ + "eth0" + ], + "para_plane_nic_num": "1", + "status": "completed" +} diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/npu_config/2p.json b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/npu_config/2p.json new file mode 100644 index 0000000..b9cb713 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/npu_config/2p.json @@ -0,0 +1,44 @@ +{ + "board_id": "0x0000", + "chip_info": "910", + "deploy_mode": "lab", + "group_count": "1", + "group_list": [ + { + "device_num": "2", + "server_num": "1", + "group_name": "", + "instance_count": "2", + "instance_list": [ + { + "devices": [ + { + "device_id": "0", + "device_ip": "192.168.100.101" + } + ], + "rank_id": "0", + "server_id": "10.246.246.76" + }, + { + "devices": [ + { + "device_id": "1", + "device_ip": "192.168.101.101" + } + ], + "rank_id": "1", + "server_id": "10.246.246.76" + }, + } + ] + } + ], + "para_plane_nic_location": "device", + "para_plane_nic_name": [ + "eth0", + "eth1" + ], + "para_plane_nic_num": "2", + "status": "completed" +} diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/npu_config/4p.json b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/npu_config/4p.json new file mode 100644 index 0000000..ed403c9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/npu_config/4p.json @@ -0,0 +1,65 @@ +{ + "board_id": "0x0000", + "chip_info": "910", + "deploy_mode": "lab", + "group_count": "1", + "group_list": [ + { + "device_num": "4", + "server_num": "1", + "group_name": "", + "instance_count": "4", + "instance_list": [ + { + "devices": [ + { + "device_id": "0", + "device_ip": "192.168.190.102" + } + ], + "rank_id": "0", + "server_id": "10.246.246.76" + }, + { + "devices": [ + { + "device_id": "1", + "device_ip": "192.168.191.102" + } + ], + "rank_id": "1", + "server_id": "10.246.246.76" + }, + { + "devices": [ + { + "device_id": "2", + "device_ip": "192.168.192.102" + } + ], + "rank_id": "2", + "server_id": "10.246.246.76" + }, + { + "devices": [ + { + "device_id": "3", + "device_ip": "192.168.193.102" + } + ], + "rank_id": "3", + "server_id": "10.246.246.76" + } + ] + } + ], + "para_plane_nic_location": "device", + "para_plane_nic_name": [ + "eth0", + "eth1", + "eth2", + "eth3" + ], + "para_plane_nic_num": "4", + "status": "completed" +} diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/npu_config/8p.json b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/npu_config/8p.json new file mode 100644 index 0000000..c506a29 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/npu_config/8p.json @@ -0,0 +1,109 @@ +{ + "board_id": "0x002f", + "chip_info": "910", + "deploy_mode": "lab", + "group_count": "1", + "group_list": [ + { + "device_num": "8", + "server_num": "1", + "group_name": "", + "instance_count": "8", + "instance_list": [ + { + "devices": [ + { + "device_id": "0", + "device_ip": "192.168.100.101" + } + ], + "rank_id": "0", + "server_id": "10.246.246.76" + }, + { + "devices": [ + { + "device_id": "1", + "device_ip": "192.168.101.101" + } + ], + "rank_id": "1", + "server_id": "10.246.246.76" + }, + { + "devices": [ + { + "device_id": "2", + "device_ip": "192.168.102.101" + } + ], + "rank_id": "2", + "server_id": "10.246.246.76" + }, + { + "devices": [ + { + "device_id": "3", + "device_ip": "192.168.103.101" + } + ], + "rank_id": "3", + "server_id": "10.246.246.76" + }, + { + "devices": [ + { + "device_id": "4", + "device_ip": "192.168.100.100" + } + ], + "rank_id": "4", + "server_id": "10.246.246.76" + }, + { + "devices": [ + { + "device_id": "5", + "device_ip": "192.168.101.100" + } + ], + "rank_id": "5", + "server_id": "10.246.246.76" + }, + { + "devices": [ + { + "device_id": "6", + "device_ip": "192.168.102.100" + } + ], + "rank_id": "6", + "server_id": "10.246.246.76" + }, + { + "devices": [ + { + "device_id": "7", + "device_ip": "192.168.103.100" + } + ], + "rank_id": "7", + "server_id": "10.246.246.76" + } + ] + } + ], + "para_plane_nic_location": "device", + "para_plane_nic_name": [ + "eth0", + "eth1", + "eth2", + "eth3", + "eth4", + "eth5", + "eth6", + "eth7" + ], + "para_plane_nic_num": "8", + "status": "completed" +} diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/profile/cpu/net_show_cpu.py b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/profile/cpu/net_show_cpu.py new file mode 100644 index 0000000..a0a18c3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/profile/cpu/net_show_cpu.py @@ -0,0 +1,40 @@ +import os + +import torch +import torch.nn as nn +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +import torchvision.models as models + +""" +alexnet | densenet121 | +densenet161 | densenet169 | densenet201 | +resnet101 | resnet152 | resnet18 | resnet34 | +resnet50 | squeezenet1_0 | squeezenet1_1 | vgg11 | +vgg11_bn | vgg13 | vgg13_bn | vgg16 | vgg16_bn | vgg19 | +mobilenet_v2 | shufflenet_v2_x0_5 | +vgg19_bn (default: resnet18) +""" +model_name='densenet121' +model = models.__dict__[model_name]() + +img = torch.rand(size=(1,3,224,224)) + +#print(model(img)) + +labels = torch.rand(size=(1,)) +criterion = nn.CrossEntropyLoss() +with torch.autograd.profiler.profile(record_shapes=True) as prof: + outputs = model(img) + loss = criterion(outputs, labels) + with torch.autograd.profiler.record_function("label-bp"): + loss.backward() + +#print(prof.key_averages().table()) +print(prof) +prof.export_chrome_trace(model_name + ".prof") + + +with SummaryWriter(os.path.join('runs',model_name)) as w: + w.add_graph(model, img) + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/profile/cpu/run_cpu.sh b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/profile/cpu/run_cpu.sh new file mode 100644 index 0000000..8efbefa --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/profile/cpu/run_cpu.sh @@ -0,0 +1,20 @@ +export LD_LIBRARY_PATH=/usr/local/lib/:/usr/lib/:/usr/local/Ascend/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH +export PATH=$PATH:/usr/local/Ascend/fwkacllib/ccec_compiler/bin +export ASCEND_OPP_PATH=/usr/local/Ascend/opp +export NEW_GE_FE_ID=1 +export GE_AICPU_FLAG=1 +export PYTHONPATH=/usr/local/Ascend/atc/python/site-packages/te.egg:/usr/local/Ascend/atc/python/site-packages/topi.egg:/usr/local/Ascend/atc/python/site-packages/auto_tune.egg:/usr/local/Ascend/atc/python/site-packages/schedule_search.egg:/usr/local +export CUSTOM_OP_LIB_PATH=/usr/local/Ascend/ops/framework/built-in/tensorflow +export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/fwkacllib/lib64/plugin/opskernel/libaicpu_plugin.so:/usr/local/Ascend/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so +export PLUGIN_LOAD_PATH=/usr/local/Ascend/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/fwkacllib/lib64/plugin/opskernel/libaicpu_plugin.so:/usr/local/Ascend/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so:/usr/local/Ascend/fwkacllib/lib64/plugin/opskernel/librts_engine.so + +#export DEVICE_ID=0 +#export SLOG_PRINT_TO_STDOUT=1 + +#su HwHiAiUser -c "adc --host 0.0.0.0:22118 --log \"SetLogLevel(0)[error]\" --device 0" + +#python3 pytorch-benchmark-resnet50.py +python3 net_show_cpu.py +#python3 pytorch-resnet50-profiling.py + + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/profile/npu/net_show_npu.py b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/profile/npu/net_show_npu.py new file mode 100644 index 0000000..3c0e8bd --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/code/tools/profile/npu/net_show_npu.py @@ -0,0 +1,51 @@ +import os + +import torch +import torch.nn as nn +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +import torchvision.models as models + +CALCULATE_DEVICE = "npu:0" +torch.npu.set_device(CALCULATE_DEVICE) + +""" +alexnet | densenet121 | +densenet161 | densenet169 | densenet201 | +resnet101 | resnet152 | resnet18 | resnet34 | +resnet50 | squeezenet1_0 | squeezenet1_1 | vgg11 | +vgg11_bn | vgg13 | vgg13_bn | vgg16 | vgg16_bn | vgg19 | +mobilenet_v2 | shufflenet_v2_x0_5 | +vgg19_bn (default: resnet18) +""" + +img = torch.rand(size=(1,3,224,224),dtype=torch.float32).to(CALCULATE_DEVICE, non_blocking=True) +print("img prepared") + +model_name='densenet121' +model = models.__dict__[model_name]().to(CALCULATE_DEVICE) +model.train() +print("model prepared") + +outputs = model(img) +print("cal done, results is {}".format(outputs)) + +labels=torch.rand(size=(1,)).to(torch.int32).to(CALCULATE_DEVICE, non_blocking=True) +criterion = nn.CrossEntropyLoss().to(CALCULATE_DEVICE) +with torch.autograd.profiler.profile(record_shapes=True,use_npu=True) as prof: + outputs = model(img) + print("output ok") + loss = criterion(outputs, labels) + print("loss ok") + with torch.autograd.profiler.record_function("label-bp"): + loss.backward() + +#print(prof.key_averages().table()) +print(prof) +prof.export_chrome_trace(model_name + ".prof") + + +# with SummaryWriter(os.path.join('runs',model_name)) as w: +# w.add_graph(model, img) +# print("tenorboard add graph ok") + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/1p.json b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/1p.json new file mode 100644 index 0000000..5669d75 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/1p.json @@ -0,0 +1,14 @@ +{ + "server_count": "1", + "server_list": [{ + "device": [ + { + "device_id": "0", + "device_ip": "192.168.10.103", + "rank_id": "0" + }], + "server_id": "127.0.0.1" + }], + "status": "completed", + "version": "1.0" +} diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/__init__.py b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/hccl_sample.json b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/hccl_sample.json new file mode 100644 index 0000000..96ec094 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/hccl_sample.json @@ -0,0 +1,9 @@ +{ + "server_count": "1", + "server_list": [{ + "device": [{devices}], + "server_id": "127.0.0.1" + }], + "status": "completed", + "version": "1.0" +} diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/npu_set_env.sh b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/npu_set_env.sh new file mode 100644 index 0000000..24ff126 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/npu_set_env.sh @@ -0,0 +1,52 @@ +# main env +export PYTHONPATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe +export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:$LD_LIBRARY_PATH +export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin/ +export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ +export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so +export PLUGIN_LOAD_PATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/librts_engine.so +export TASK_QUEUE_ENABLE=0 +export CUSTOM_OP_LIB_PATH=/usr/local/Ascend/ascend-toolkit/20.10.0.B022/arm64-linux_gcc7.3.0/opp/framework/built-in/tensorflow/ + + +export NEW_GE_FE_ID=1 +export GE_AICPU_FLAG=1 +export GEN_TO_SOURCE=1 + + + + +#export LD_LIBRARY_PATH=/usr/local/OpenBLAS/lib/:/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/aarch64-linux-gnu/ +#export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin +#export ASCEND_OPP_PATH=/usr/local/Ascend/opp + + +#export DDK_VERSION_FLAG=1.60.T17.B830 +#export NEW_GE_FE_ID=1 +#export GE_AICPU_FLAG=1 +#export SOC_VERSION=Ascend910 + +#export DUMP_GE_GRAPH=2 + + +#export DEVICE_ID=0 +#export DEVICE_INDEX=0 + +#export PRINT_MODEL=0 +#export ENABLE_DATA_PRE_PROC=1 +#export RANK_ID=0 +#export RANK_SIZE=1 +#export JOB_ID=10087 +#export FUSION_TENSOR_SIZE=1000000000 +#PYTHONPATH=/usr/local/Ascend/ascend-toolkit/latest/atc/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe + +#export PYTHONPATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe + + +#export CUSTOM_OP_LIB_PATH=/usr/local/Ascend/ascend-toolkit/20.10.0.B023/arm64-linux_gcc7.3.0/opp/framework/built-in/tensorflow/ +#export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so +#export PLUGIN_LOAD_PATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/librts_engine.so +#export WHICH_OP=GEOP +#export NEW_GE_FE_ID=1 +#export GE_AICPU_FLAG=1 + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/set_env.sh b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/set_env.sh new file mode 100644 index 0000000..6bd8f33 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/set_env.sh @@ -0,0 +1,9 @@ +export ASCEND_HOME=/usr/local/Ascend +export LD_LIBRARY_PATH=/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/ +export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/te:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/topi:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/hccl:/usr/local/Ascend/ascend-toolkit/latest/tfplugin/python/site-packages:$currentDir +export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin +export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + +export SLOG_PRINT_TO_STDOUT=0 + +export TASK_QUEUE_ENABLE=0 diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/set_env_b020.sh b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/set_env_b020.sh new file mode 100644 index 0000000..ed79fb6 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/set_env_b020.sh @@ -0,0 +1,21 @@ +############## toolkit situation ################ +#export ASCEND_HOME=/usr/local/Ascend +#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/ +#export PYTHONPATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/hccl +#export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin +#export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + +############## nnae situation ################ +export ASCEND_HOME=/usr/local/Ascend +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/:/usr/local/python3.7.5/lib/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/ +export PYTHONPATH=/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/schedule_search.egg:/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/hccl +export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin +export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp/ + +# pip3.7 install --upgrade /usr/local/Ascend/nnae/latest/fwkacllib/lib64/topi-0.4.0-py3-none-any.whl +# pip3.7 install --upgrade /usr/local/Ascend/nnae/latest/fwkacllib/lib64/te-0.4.0-py3-none-any.whl + +export SLOG_PRINT_TO_STDOUT=0 +#su HwHiAiUser -c "adc --host 0.0.0.0:22118 --log \"SetLogLevel(0)[error]\" --device 0" + +export TASK_QUEUE_ENABLE=0 diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/set_env_b023.sh b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/set_env_b023.sh new file mode 100644 index 0000000..7618849 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/config/set_env_b023.sh @@ -0,0 +1,31 @@ +############## toolkit situation ################ +#export LD_LIBRARY_PATH=/usr/local/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH +#export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/ascend-toolkit/latest/toolkit/tools/ide_daemon/bin/ +#export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ +#export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so +#export PYTHONPATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH + + +############## nnae situation ################ + + +if [ -d /usr/local/Ascend/nnae/latest ];then + export LD_LIBRARY_PATH=/usr/local/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/aarch64_64-linux-gnu:$LD_LIBRARY_PATH + export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/nnae/latest/toolkit/tools/ide_daemon/bin/ + export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp/ + export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so + export PYTHONPATH=/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH +else + export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/aarch64-linux-gnu:$LD_LIBRARY_PATH + export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/ascend-toolkit/latest/toolkit/tools/ide_daemon/bin/ + export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so + export PYTHONPATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH +fi + +# ln -s /usr/local/Ascend/ascend-toolkit/latest/toolkit/bin/adc /usr/local/bin/ + +export SLOG_PRINT_TO_STDOUT=0 +#su HwHiAiUser -c "adc --host 0.0.0.0:22118 --log \"SetLogLevel(0)[error]\" --device 0" + +export TASK_QUEUE_ENABLE=1 \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/scripts/eval.sh b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/scripts/eval.sh new file mode 100644 index 0000000..16d5c5e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/scripts/eval.sh @@ -0,0 +1,22 @@ +export ASCEND_HOME=/usr/local/Ascend +export LD_LIBRARY_PATH=/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/ +export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/te:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/topi:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/hccl:/usr/local/Ascend/ascend-toolkit/latest/tfplugin/python/site-packages:$currentDir +export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin +export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + +export SLOG_PRINT_TO_STDOUT=0 +su HwHiAiUser -c "adc --host 0.0.0.0:22118 --log \"SetLogLevel(0)[error]\" --device 7" + +export TASK_QUEUE_ENABLE=0 +taskset -c 111-150 python3 densenet121_1p_main.py \ + --workers 40 \ + --arch densenet121 \ + --npu 7 \ + --lr 0.1 \ + --momentum 0.9 \ + --amp \ + --batch-size 256 \ + --epoch 90 \ + --evaluate \ + --resume checkpoint.pth.tar \ + --data /opt/npu/dataset/imagenet \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/scripts/run.sh b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/scripts/run.sh new file mode 100644 index 0000000..dfeef3e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/scripts/run.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +rank_size=$1 +yamlPath=$2 +toolsPath=$3 + +currentDir=$(cd "$(dirname "$0")/.."; pwd) +model_name=$(cd $currentDir/..;basename `pwd`) +if [ -f /.dockerenv ];then + CLUSTER=$4 + MPIRUN_ALL_IP="$5" + export CLUSTER=${CLUSTER} +fi +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "pytorch_config") + +# 清除旧日志 +rm -rf /var/log/npu/slog/host-0/* +rm -rf ${currentDir}/result/*.log + +#mkdir train job path +currtime=`date +%Y%m%d%H%M%S` +mkdir -p ${currentDir%train*}/train/result/pt_densenet121/training_job_${currtime}/ +export train_job_dir=${currentDir%train*}/train/result/pt_densenet121/training_job_${currtime}/ +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir} &" +# device 列表, 若无指定 device 根据 rank_size 顺序选择 +eval device_group=\$device_group_${rank_size}p +if [ x"${device_group}" == x"" ] || [ ${rank_size} -ge 8 ];then + device_group="$(seq 0 "$(expr $rank_size - 1)")" +fi + +# get last device id in device_group, hw log in performance from the dir named last_device_id +device_group_str=`echo ${device_group} | sed 's/ //g'` +first_device_id=`echo ${device_group_str: 0:1}` + +if [ x"${CLUSTER}" == x"True" ];then + this_ip=$(hostname -I |awk '{print $1}') + ln -snf ${currentDir%train*}/train/result/pt_densenet121/training_job_${currtime}/0/hw_densenet121.log ${currentDir%train*}/train/result/pt_densenet121/training_job_${currtime}/ + for ip in $MPIRUN_ALL_IP;do + if [ x"$ip" != x"$this_ip" ];then + scp $yamlPath root@$ip:$yamlPath + scp ${jsonFilePath} root@$ip:${jsonFilePath} + fi + done + export PATH=$PATH:/usr/local/mpirun4.0/bin + mpirun -H ${mpirun_ip} \ + --bind-to none -map-by slot\ + --allow-run-as-root \ + --mca btl_tcp_if_exclude lo,docker0,endvnic,virbr0,vethf40501b,docker_gwbridge,br-f42ac38052b4\ + --prefix /usr/local/mpirun4.0/ \ + ${currentDir}/scripts/train.sh 0 $rank_size $yamlPath $currtime ${toolsPath} ${CLUSTER} +else + rank_id=0 + #for device_id in $device_group;do + ln -snf ${currentDir%train*}/train/result/pt_densenet121/training_job_${currtime}/${first_device_id}/hw_densenet121.log ${currentDir%train*}/train/result/pt_densenet121/training_job_${currtime}/ + ${currentDir}/scripts/train.sh 0 $rank_size $yamlPath $currtime ${toolsPath} $rank_id & + # let rank_id++ + # done +fi +wait + + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/scripts/train.sh b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/scripts/train.sh new file mode 100644 index 0000000..f79439f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/pytorch/scripts/train.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash + +device_id=$1 +rank_size=$2 +yamlPath=$3 + +currentDir=$(cd "$(dirname "$0")/.."; pwd) +currtime=$4 +toolsPath=$5 +export YAML_PATH=$3 +mkdir -p ${currentDir%train*}/train/result/pt_densenet121/training_job_${currtime}/ +export train_job_dir=${currentDir%train*}/train/result/pt_densenet121/training_job_${currtime}/ + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "pytorch_config") + +export REMARK_LOG_FILE=hw_densenet121.log # 打点日志文件名称， 必须hw_后跟模型名称小写 +benchmark_log_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils +export PYTHONPATH=$PYTHONPATH:${benchmark_log_path} + + +#source ${currentDir}/config/npu_set_env.sh +source ${currentDir}/config/set_env_b023.sh +# user env +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 +export HCCL_RANK_TABLE_PATH=${currentDir}/config/${rank_size}p.json +export RANK_SIZE=${rank_size} +export SLOG_PRINT_TO_STDOUT=0 +export DEVICE_ID=${device_id} +DEVICE_INDEX=$(( DEVICE_ID + RANK_INDEX * 8 )) +export DEVICE_INDEX=${DEVICE_INDEX} + +cd ${train_job_dir} +curd_dir=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +export PYTHONPATH=$PYTHONPATH:${curd_dir} + +if [ x"$6" != x"True" ];then + rank_id=$6 + export RANK_ID=$6 +else + device_id_mo=$(python3.7 -c "import src.tensorflow.mpi_ops as atlasboost;atlasboost.init(); \ + device_id = atlasboost.local_rank();cluster_device_id = str(device_id); \ + atlasboost.set_device_id(device_id);print(atlasboost.rank())") + device_id_mo=`echo $device_id_mo` + rank_id=${device_id_mo##* } + export RANK_ID=${rank_id} + device=${device_id_mo##*deviceid = } + device_id=${device%% phyid=*} + export DEVICE_ID=${device_id} + hccljson=${train_job_dir}/*.json + cp ${hccljson} ${currentDir}/config/${rank_size}p.json +fi + +#mkdir exec path +mkdir -p ${train_job_dir}/${device_id} +cd ${train_job_dir}/${device_id} + +startTime=`date +%Y%m%d-%H:%M:%S` +startTime_s=`date +%s` + +# 根据单卡/多卡区分调用参数 +if [ x"$6" == x"True" ];then + # 多卡多机 + export CLUSTER=True +fi + +if [ x"${mode}" == x"evaluate" ];then + taskset -c 111-150 python3.7 ${currentDir}/code/densenet121_1p_main.py \ + --workers 40 \ + --arch densenet121 \ + --npu 7 \ + --lr 0.1 \ + --momentum 0.9 \ + --amp \ + --batch-size 256 \ + --epoch 90 \ + --evaluate \ + --resume checkpoint.pth.tar \ + --data ${data_url} > ${train_job_dir}/train_${rank_size}p.log 2>&1 + + +elif [ x"${rank_size}" == x"1" ];then + # 单卡 + #source ${currentDir}/config/set_env_b023.sh + + taskset -c 1-40 python3.7 ${currentDir}/code/densenet121_1p_main.py \ + --workers 40 \ + --arch densenet121 \ + --npu ${device_single} \ + --lr 0.1 \ + --momentum 0.9 \ + --amp \ + --batch-size ${batch_size} \ + --epoch ${epoches} \ + --data ${data_url} > ${train_job_dir}/train_${rank_size}p.log 2>&1 + +elif [ ${rank_size} -le 8 ];then + # 单机多卡 + #source ${currentDir}/config/set_env_b023.sh + python3.7 ${currentDir}/code/densenet121_8p_main.py \ + --addr=$(hostname -I |awk '{print $1}') \ + --seed 49 \ + --workers 160 \ + --lr ${lr} \ + --print-freq 1 \ + --eval-freq 5\ + --arch densenet121 \ + --dist-url 'tcp://127.0.0.1:50000' \ + --dist-backend 'hccl' \ + --multiprocessing-distributed \ + --world-size 1 \ + --batch-size ${batch_size} \ + --epochs ${epoches} \ + --rank 0 \ + --amp \ + --benchmark 0 \ + --device-list ${device_group_multi} \ + --data ${data_url} > ${train_job_dir}/train_${rank_size}p.log 2>&1 + +fi + +#taskset -c 0-20 python3.7 ${currentDir}/code/densenet121.py > ./train.log 2>&1 + +if [ $? -eq 0 ];then + echo ":::ABK 1.0.0 densenet121 train success" + echo ":::ABK 1.0.0 densenet121 train success" >> ${train_job_dir}/train_${rank_size}p.log + echo ":::ABK 1.0.0 densenet121 train success" >> ./hw_densenet121.log +else + echo ":::ABK 1.0.0 densenet121 train failed" + echo ":::ABK 1.0.0 densenet121 train failed" >> ${train_job_dir}/train_${rank_size}p.log + echo ":::ABK 1.0.0 densenet121 train failed" >> ./hw_densenet121.log +fi + +endTime=`date +%Y%m%d-%H:%M:%S` +endTime_s=`date +%s` +sumTime=$[ $endTime_s - $startTime_s ] +hour=$(( $sumTime/3600 )) +min=$(( ($sumTime-${hour}*3600)/60 )) +sec=$(( $sumTime-${hour}*3600-${min}*60 )) +echo ":::ABK 1.0.0 densenet121 train total time： ${hour}:${min}:${sec}" >> ${train_job_dir}/${device_id}/hw_densenet121.log diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/README.md b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/README.md new file mode 100644 index 0000000..3b6b1d5 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/README.md @@ -0,0 +1,46 @@ +# DenseNet121_tensorflow训练说明 + +### 1. 模型训练参数配置 + +在train/yaml/DenseNet121.yaml中修改相应配置， 配置项含义: + +``` +tensorflow_config: + # 基本参数 + data_url: 数据集路径 + epoches: 跑多少个epoch + epochs_between_evals: 1 + batch_size: 32 + log_dir: ./ckpt + + # 1p参数 + mode_1p: train # train、evaluate、train_and_evaluate三种模式 + max_train_steps_1p: 100 + iterations_per_loop_1p: 10 + display_every: 10 + log_name_1p: densenet121_1p.log + + # 8p参数 + mode_8p: train_and_evaluate # train、evaluate、train_and_evaluate三种模式 + iterations_per_loop_8p: 5004 + lr: 0.1 + log_name_8p: densenet121_8p.log + + mpirun_ip: 仅多机执行需要配置: ip1:卡数量1,ip2:卡数量2 + docker_image:docker 镜像名称:版本号 + + # 指定 device id, 多个 id 使用空格分隔, 数量需与 rank_size 相同 + device_group_1p: 0 + device_group_2p: 0 1 + device_group_4p: 0 1 2 3 +``` + +------ + + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/create_session.py b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/create_session.py new file mode 100644 index 0000000..781356d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/create_session.py @@ -0,0 +1,22 @@ +import tensorflow as tf +import os,sys + + +class CreateSession(): + def __init__(self): + self.estimator_config = tf.ConfigProto( + inter_op_parallelism_threads=10, + intra_op_parallelism_threads=10, + allow_soft_placement=True) + + self.estimator_config.gpu_options.allow_growth = True + + self.set_env() + + def set_env(self): + gpu_thread_count = 2 + os.environ['TF_GPU_THREAD_MODE'] = 'gpu_private' + os.environ['TF_GPU_THREAD_COUNT'] = str(gpu_thread_count) + os.environ['TF_USE_CUDNN_BATCHNORM_SPATIAL_PERSISTENT'] = '1' + os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = '1' + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/data_loader.py b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/data_loader.py new file mode 100644 index 0000000..4c4a803 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/data_loader.py @@ -0,0 +1,133 @@ +import numpy as np +import preprocessing +import tensorflow as tf +from tensorflow.python.util import nest +import os,sys +import numpy as np + + +class DataLoader: + + def __init__(self, args): + self.args = args + + filename_pattern = os.path.join(args.data_dir, '%s-*') + filenames_train = sorted(tf.gfile.Glob(filename_pattern % 'train')) + self.num_training_samples = get_num_records(filenames_train) + self.args.num_training_samples = self.num_training_samples + + filename_pattern = os.path.join(args.data_dir, '%s-*') + filenames_val = sorted(tf.gfile.Glob(filename_pattern % 'validation')) + self.num_evaluating_samples = get_num_records(filenames_val) + self.args.num_evaluating_samples = self.num_evaluating_samples + + print( 'total num_training_sampels: %d' % self.num_training_samples ) + print( 'total num_evaluating_sampels: %d' % self.num_evaluating_samples ) + + self.training_samples_per_rank = self.num_training_samples + + def get_train_input_fn(self): + take_count = self.training_samples_per_rank + + return make_dataset(self.args, take_count, self.args.batch_size, training=True) + + def get_eval_input_fn(self): + take_count = self.num_evaluating_samples + + return make_dataset(self.args, take_count, self.args.batch_size, training=False) + + +def get_num_records(filenames): + def count_records(tf_record_filename): + count = 0 + for _ in tf.python_io.tf_record_iterator(tf_record_filename): + count += 1 + return count + + nfile = len(filenames) + return (count_records(filenames[0]) * (nfile - 1) + + count_records(filenames[-1])) + + +def _parse_example_proto(example_serialized): + feature_map = { + 'image/encoded': tf.FixedLenFeature([], dtype=tf.string, + default_value=''), + 'image/class/label': tf.FixedLenFeature([], dtype=tf.int64, default_value=-1), + 'image/class/text': tf.FixedLenFeature([], dtype=tf.string, + default_value=''), + } + sparse_float32 = tf.VarLenFeature(dtype=tf.float32) + # Sparse features in Example proto. + feature_map.update( + {k: sparse_float32 for k in ['image/object/bbox/xmin', + 'image/object/bbox/ymin', + 'image/object/bbox/xmax', + 'image/object/bbox/ymax']}) + + features = tf.parse_single_example(example_serialized, feature_map) + label = tf.cast(features['image/class/label'], dtype=tf.int32) + + xmin = tf.expand_dims(features['image/object/bbox/xmin'].values, 0) + ymin = tf.expand_dims(features['image/object/bbox/ymin'].values, 0) + xmax = tf.expand_dims(features['image/object/bbox/xmax'].values, 0) + ymax = tf.expand_dims(features['image/object/bbox/ymax'].values, 0) + + # Note that we impose an ordering of (y, x) just to make life difficult. + bbox = tf.concat([ymin, xmin, ymax, xmax], 0) + + # Force the variable number of bounding boxes into the shape + # [1, num_boxes, coords]. + bbox = tf.expand_dims(bbox, 0) + bbox = tf.transpose(bbox, [0, 2, 1]) + + return features['image/encoded'], label, bbox + + +# since the preprocessing is done here, we add args file +def parse_record(raw_record, is_training): + image_buffer, label, bbox = _parse_example_proto(raw_record) + + image = preprocessing.parse_and_preprocess_image_record(image_buffer, bbox, training=is_training) + + # label-1 for VGG16 + return image, label-1 + + +def make_dataset(args, take_count, batch_size, + training=False, shard=False): + + shuffle_buffer_size = 10000 + num_readers = 10 + + rank_size = int(os.getenv('RANK_SIZE')) + rank_id = int(os.getenv('DEVICE_INDEX')) + + if training: + filename_pattern = os.path.join(args.data_dir, '%s-*') + filenames = sorted(tf.gfile.Glob(filename_pattern % 'train')) + else: + filename_pattern = os.path.join(args.data_dir, '%s-*') + filenames = sorted(tf.gfile.Glob(filename_pattern % 'validation')) + + ds = tf.data.Dataset.from_tensor_slices(filenames) + + if not training: + ds = ds.take(take_count) + + if training: + ds = ds.shuffle(1000, seed=7*(1+rank_id)) + + ds = ds.interleave(tf.data.TFRecordDataset, cycle_length=num_readers, block_length=1) + counter = tf.data.Dataset.range(sys.maxsize) + ds = tf.data.Dataset.zip((ds, counter)) + + if training: + ds = ds.apply(tf.data.experimental.shuffle_and_repeat(shuffle_buffer_size, seed=5*(1+rank_id))) + + ds = ds.map(lambda image, counter: parse_record(image, training), num_parallel_calls=14) + + ds = ds.batch(batch_size, drop_remainder=True) + return ds + + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/densenet.py b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/densenet.py new file mode 100644 index 0000000..0a83ea4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/densenet.py @@ -0,0 +1,158 @@ +import tensorflow as tf +from tensorflow.contrib.layers import batch_norm, flatten +from tensorflow.contrib.framework import arg_scope +import numpy as np + +class_num = 1000 +nb_blocks = 4 +nb_blocks_layers = (6, 12, 24, 16) +bn_size = 4 +growth_rate = 32 +init_layers = 64 + + +''' +denseNet：121，169，201，264 +return _densenet('densenet121', 32, (6, 12, 24, 16), 64, pretrained, progress, + **kwargs) +return _densenet('densenet161', 48, (6, 12, 36, 24), 96, pretrained, progress, + **kwargs) +return _densenet('densenet169', 32, (6, 12, 32, 32), 64, pretrained, progress, + **kwargs) +return _densenet('densenet201', 32, (6, 12, 48, 32), 64, pretrained, progress, + **kwargs) +''' + + + +def conv_layer(input, filter, kernel, stride=1, layer_name="conv"): + with tf.name_scope(layer_name): + network = tf.layers.conv2d(inputs=input, filters=filter, kernel_size=kernel, strides=stride, padding='SAME', use_bias=False, kernel_initializer=tf.initializers.variance_scaling(scale=5.0, mode='fan_out')) # scale=5.0, mode='fan_out' + return network + +def Global_Average_Pooling(x, stride=1): + + width = np.shape(x)[1] + height = np.shape(x)[2] + pool_size = [width, height] + return tf.layers.average_pooling2d(inputs=x, pool_size=pool_size, strides=stride) # The stride value does not matter + #It is global average pooling without tflearn + + + #return global_avg_pool(x, name='Global_avg_pooling') + # But maybe you need to install h5py and curses or not + + +def Batch_Normalization(x, training, scope): + with arg_scope([batch_norm], + scope=scope, + updates_collections=None, + decay=0.9, + center=True, + scale=True, + zero_debias_moving_mean=True) : + training = tf.cast(training, tf.bool) + return tf.cond(training, + lambda : batch_norm(inputs=x, is_training=training, reuse=None), + lambda : batch_norm(inputs=x, is_training=training, reuse=True)) + +def Drop_out(x, rate, training) : + return tf.layers.dropout(inputs=x, rate=rate, training=training) + +def Relu(x): + return tf.nn.relu(x) + +def Average_pooling(x, pool_size=[2,2], stride=2, padding='VALID'): + return tf.layers.average_pooling2d(inputs=x, pool_size=pool_size, strides=stride, padding=padding) + + +def Max_Pooling(x, pool_size=[3,3], stride=2, padding='VALID'): + return tf.layers.max_pooling2d(inputs=x, pool_size=pool_size, strides=stride, padding=padding) + +def Concatenation(layers): + return tf.concat(layers, axis=3) + +def Linear(x): + return tf.layers.dense(inputs=x, units=class_num, name='linear') + + +def bottleneck_layer(x, is_training, scope): + # print(x) + with tf.name_scope(scope): + x = Batch_Normalization(x, training=is_training, scope=scope+'_batch1') + x = Relu(x) + x = conv_layer(x, filter= growth_rate*bn_size, kernel=[1,1], layer_name=scope+'_conv1') + #x = Drop_out(x, rate=dropout_rate, training=is_training) + #x = Drop_out(x, rate=dropout_rate, training=is_training) + + x = Batch_Normalization(x, training=is_training, scope=scope+'_batch2') + x = Relu(x) + x = conv_layer(x, filter= growth_rate, kernel=[3,3], layer_name=scope+'_conv2') + #x = Drop_out(x, rate=dropout_rate, training=self.training) + + # print(x) + + return x + +def transition_layer(x, is_training, scope): + with tf.name_scope(scope): + x = Batch_Normalization(x, training=is_training, scope=scope+'_batch1') + x = Relu(x) + # x = conv_layer(x, filter=self.filters, kernel=[1,1], layer_name=scope+'_conv1') + + # https://github.com/taki0112/Densenet-Tensorflow/issues/10 + + in_channel = int(x.shape[-1]) + x = conv_layer(x, filter=in_channel*0.5, kernel=[1,1], layer_name=scope+'_conv1') + #x = Drop_out(x, rate=dropout_rate, training=self.training) + x = Average_pooling(x, pool_size=[2,2], stride=2) + + return x + +def dense_block(input_x, nb_layers, is_training, layer_name): + with tf.name_scope(layer_name): + layers_concat = list() + layers_concat.append(input_x) + + x = bottleneck_layer(input_x, is_training, scope=layer_name + '_bottleN_' + str(0)) + + layers_concat.append(x) + + for i in range(nb_layers - 1): + x = Concatenation(layers_concat) + x = bottleneck_layer(x, is_training, scope=layer_name + '_bottleN_' + str(i + 1)) + layers_concat.append(x) + + x = Concatenation(layers_concat) + + return x + +def Dense_net(input_x, is_training): + x = conv_layer(input_x, filter=init_layers , kernel=[7,7], stride=2, layer_name='conv0') + x = Max_Pooling(x, pool_size=[3,3], stride=2) + + for i in range(nb_blocks-1) : + # 6 -> 12 -> 48 + x = dense_block(input_x=x, nb_layers=nb_blocks_layers[i], is_training=is_training, layer_name='dense_'+str(i)) + x = transition_layer(x, is_training, scope='trans_'+str(i)) + + """ + x = self.dense_block(input_x=x, nb_layers=6, layer_name='dense_1') + x = self.transition_layer(x, scope='trans_1') + x = self.dense_block(input_x=x, nb_layers=12, layer_name='dense_2') + x = self.transition_layer(x, scope='trans_2') + x = self.dense_block(input_x=x, nb_layers=48, layer_name='dense_3') + x = self.transition_layer(x, scope='trans_3') + """ + + x = dense_block(input_x=x, nb_layers=nb_blocks_layers[nb_blocks-1], is_training=is_training, layer_name='dense_final') + + # 100 Layer + x = Batch_Normalization(x, training=is_training, scope='linear_batch') + x = Relu(x) + x = Global_Average_Pooling(x) + x = flatten(x) + x = Linear(x) + + # x = tf.reshape(x, [-1, 10]) + return x diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/hyper_param.py b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/hyper_param.py new file mode 100644 index 0000000..5c014c4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/hyper_param.py @@ -0,0 +1,44 @@ +import tensorflow as tf +import math +import numpy as np + +def warmup_cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch, T_max, eta_min=0): + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch) + warmup_steps = int(warmup_epochs * steps_per_epoch) + + lr_each_step = [] + for i in range(total_steps): + last_epoch = i // steps_per_epoch + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + lr = eta_min + (base_lr - eta_min) * (1. + math.cos(math.pi*last_epoch / T_max)) / 2 + lr_each_step.append(lr) + + return np.array(lr_each_step).astype(np.float32) + + +class HyperParams: + def __init__(self, args): + self.args=args + nsteps_per_epoch = self.args.num_training_samples // self.args.global_batch_size + self.args.nsteps_per_epoch = nsteps_per_epoch + if self.args.max_epochs: + nstep = nsteps_per_epoch * self.args.max_epochs + else: + nstep = self.args.max_train_steps + self.args.nstep = nstep + + self.cos_lr = warmup_cosine_annealing_lr(self.args.lr, nsteps_per_epoch, 0, self.args.T_max, self.args.T_max, 0.0) + + def get_learning_rate(self): + global_step = tf.train.get_global_step() + + learning_rate = tf.gather(tf.convert_to_tensor(self.cos_lr), global_step) + + learning_rate = tf.identity(learning_rate, 'learning_rate') + + return learning_rate + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/layers.py b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/layers.py new file mode 100644 index 0000000..c144c97 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/layers.py @@ -0,0 +1,25 @@ +import tensorflow as tf +#from tensorflow.contrib.hccl.python.ops import hccl_ops +#from npu_bridge.hccl import hccl_ops +from benchmark_log import hwlog + +class Layers: + def get_accuracy(self, labels, predicted_classes, logits, args): + accuracy = tf.metrics.accuracy( + labels=labels, predictions=predicted_classes) + top5acc = tf.metrics.mean( + tf.cast(tf.nn.in_top_k(logits, labels, 5), tf.float32)) + if args.rank_size == 1: + newaccuracy = (accuracy[0], accuracy[1]) + newtop5acc = (top5acc[0], top5acc[1]) + else: + from npu_bridge.hccl import hccl_ops + newaccuracy = (hccl_ops.allreduce(accuracy[0],"sum")/args.rank_size, accuracy[1]) + newtop5acc = (hccl_ops.allreduce(top5acc[0],"sum")/args.rank_size, top5acc[1]) + metrics = {'val-top1acc': newaccuracy, 'val-top5acc': newtop5acc} + + return metrics + + + + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/logger.py b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/logger.py new file mode 100644 index 0000000..d0a1a69 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/logger.py @@ -0,0 +1,92 @@ +from __future__ import print_function +import tensorflow as tf +from benchmark_log import hwlog +import logging +import numpy as np +import time +import sys,os + +class LogSessionRunHook(tf.train.SessionRunHook): + def __init__(self, args, warmup_steps=5): + self.global_batch_size = args.global_batch_size + if args.iterations_per_loop is not None: + self.iterations_per_loop = args.iterations_per_loop + else: + self.iterations_per_loop = args.nsteps_per_epoch + self.warmup_steps = warmup_steps + self.iter_times = [] + self.num_records = args.num_training_samples + self.display_every = args.display_every + self.logger = get_logger(args.log_name, args.log_dir) + rank0log(self.logger, 'PY' + str(sys.version) + 'TF' + str(tf.__version__)) + + + + def after_create_session(self, session, coord): + rank0log(self.logger, 'Step Epoch Speed Loss FinLoss LR') + self.elapsed_secs = 0. + self.count = 0 + + def before_run(self, run_context): + self.t0 = time.time() + return tf.train.SessionRunArgs( + fetches=[tf.train.get_global_step(), 'loss:0', 'total_loss:0', 'learning_rate:0']) + + def after_run(self, run_context, run_values): + batch_time = time.time() - self.t0 + self.iter_times.append(batch_time) + self.elapsed_secs += batch_time + self.count += 1 + global_step, loss, total_loss, lr = run_values.results + if global_step == 1 or global_step % self.display_every == 0: + dt = self.elapsed_secs / self.count + img_per_sec = self.global_batch_size * self.iterations_per_loop / dt + epoch = global_step * self.global_batch_size / self.num_records + self.logger.info('step:%6i epoch:%5.1f FPS:%7.1f loss:%6.3f total_loss:%6.3f lr:%7.5f' % + (global_step, epoch, img_per_sec, loss, total_loss, lr)) + self.elapsed_secs = 0. + self.count = 0 + + # add by wx983399 + hwlog.remark_print(key=hwlog.GLOBAL_STEP, value=int(global_step)) + hwlog.remark_print(key=hwlog.CURRENT_EPOCH, value=epoch) + hwlog.remark_print(key=hwlog.FPS, value=img_per_sec) + + def get_average_speed(self): + avg_time = np.mean(self.iter_times[self.warmup_steps:]) + speed = self.global_batch_size / avg_time + return speed + + + +def rank0log(logger, *args, **kwargs): + if logger: + logger.info(''.join([str(x) for x in list(args)])) + else: + print(*args, **kwargs) + + +def get_logger(log_name, log_dir): + logger = logging.getLogger(log_name) + logger.setLevel(logging.INFO) # INFO, ERROR + # file handler which logs debug messages + if not os.path.isdir(log_dir): + try: + os.makedirs(log_dir) + except FileExistsError: + # if log_dir is common for multiple ranks like on nfs + pass + # console handler + ch = logging.StreamHandler() + ch.setLevel(logging.INFO) + # add formatter to the handlers + formatter = logging.Formatter('%(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) + fh = logging.FileHandler(os.path.join(log_dir, log_name)) + fh.setLevel(logging.DEBUG) + fh.setFormatter(formatter) + # add handlers to logger + logger.addHandler(fh) + return logger + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/model.py b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/model.py new file mode 100644 index 0000000..dae298c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/model.py @@ -0,0 +1,72 @@ +import tensorflow as tf +from densenet import Dense_net + + +class Model(object): + def __init__(self, args, data, hyper_param, layers, logger): + self.args = args + self.data = data + self.hyper_param = hyper_param + self.layers = layers + self.logger = logger + + def get_estimator_model_func(self, features, labels, mode, params=None): + labels = tf.reshape(labels, (-1,)) # Squash unnecessary unary dim #----------------not use when use onehot label + + inputs = features # TODO: Should be using feature columns? + is_training = (mode == tf.estimator.ModeKeys.TRAIN) + + inputs = tf.cast(inputs, self.args.dtype) + + top_layer = Dense_net(inputs, is_training) + + logits = top_layer + predicted_classes = tf.argmax(logits, axis=1, output_type=tf.int32) + logits = tf.cast(logits, tf.float32) + + labels_one_hot = tf.one_hot(labels, depth=1000) + loss = tf.losses.softmax_cross_entropy( + logits=logits, onehot_labels=labels_one_hot, label_smoothing=self.args.label_smoothing) + + + base_loss = tf.identity(loss, name='loss') # For access by logger (TODO: Better way to access it?) + + l2_loss = tf.add_n([tf.nn.l2_loss(tf.cast(v, tf.float32)) for v in tf.trainable_variables()]) + l2_loss = tf.multiply(l2_loss, self.args.weight_decay) + total_loss = base_loss + l2_loss + + total_loss = tf.identity(total_loss, name = 'total_loss') + + if mode == tf.estimator.ModeKeys.EVAL: + with tf.device(None): + metrics = self.layers.get_accuracy( labels, predicted_classes, logits, self.args) + + return tf.estimator.EstimatorSpec( + mode, loss=loss, eval_metric_ops=metrics) + + assert (mode == tf.estimator.ModeKeys.TRAIN) + + batch_size = tf.shape(inputs)[0] + + global_step = tf.train.get_global_step() + learning_rate = self.hyper_param.get_learning_rate() + + momentum = self.args.momentum + + opt = tf.train.MomentumOptimizer( + learning_rate, momentum, use_nesterov=self.args.use_nesterov) + + from npu_bridge.estimator.npu.npu_optimizer import NPUDistributedOptimizer + opt = NPUDistributedOptimizer(opt) + + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) or [] + + with tf.control_dependencies(update_ops): + gate_gradients = tf.train.Optimizer.GATE_NONE + grads_and_vars = opt.compute_gradients(total_loss, gate_gradients=gate_gradients) + train_op = opt.apply_gradients(grads_and_vars, global_step=global_step) + + train_op = tf.group(train_op) + + return tf.estimator.EstimatorSpec(mode, loss=total_loss, train_op=train_op) + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/preprocessing.py b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/preprocessing.py new file mode 100644 index 0000000..4ae0b90 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/preprocessing.py @@ -0,0 +1,72 @@ +import tensorflow as tf +from tensorflow.contrib.image.python.ops import distort_image_ops +import math +import random + +def decode_jpeg(imgdata, channels=3): + return tf.image.decode_jpeg(imgdata, channels=channels, + fancy_upscaling=False, + dct_method='INTEGER_FAST') + + +def random_horizontal_flip(image, prob): + if prob > random.random(): + image = tf.image.flip_left_right(image) + return image + + +def decode_crop_and_resize(record, bbox, size, scale, ratio): + with tf.name_scope('decode_crop_and_resize'): + height = 224 + width = 224 + crop_ratio = 0.8 + initial_shape = [int(round(height / crop_ratio)), + int(round(width / crop_ratio)), 3] + jpeg_shape = tf.image.extract_jpeg_shape( record ) + + bbox_begin, bbox_size, bbox = \ + tf.image.sample_distorted_bounding_box( + tf.image.extract_jpeg_shape(record), + bounding_boxes=bbox, + min_object_covered=0.1, + aspect_ratio_range=ratio, + area_range=scale, + max_attempts=10, + use_image_if_no_bounding_boxes=True) + + # Reassemble the bounding box in the format the crop op requires. + offset_y, offset_x, _ = tf.unstack(bbox_begin) + target_height, target_width, _ = tf.unstack(bbox_size) + crop_window = tf.stack([offset_y, offset_x, target_height, target_width]) + + image = tf.image.decode_and_crop_jpeg( record, crop_window, channels=3 ) + image = tf.image.resize_images( image, [height, width] ) + + return image + + +def parse_and_preprocess_image_record(record, bbox, training): + with tf.name_scope('preprocess'): + if training: + image = decode_crop_and_resize(record, bbox, 224, (0.08, 1.0), (0.75, 1.333)) + image = random_horizontal_flip(image, 0.5) + image = normalize(image) + else: + image = decode_jpeg(record, channels=3) + image = tf.image.resize_images(image, [256, 256]) + image = tf.image.central_crop(image, 224.0/256) + image = normalize(image) + + return image + + +def normalize(inputs): + imagenet_mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] + imagenet_std = [0.229 * 255, 0.224 * 255, 0.225 * 255] + imagenet_mean = tf.expand_dims(tf.expand_dims(imagenet_mean, 0), 0) + imagenet_std = tf.expand_dims(tf.expand_dims(imagenet_std, 0), 0) + inputs = inputs - imagenet_mean + inputs = inputs * (1.0 / imagenet_std) + + return inputs + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/train.py b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/train.py new file mode 100644 index 0000000..16b3eac --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/train.py @@ -0,0 +1,140 @@ +import tensorflow as tf +import numpy as np +import os +import sys +import ast +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '../'))) +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '../config'))) +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '../../../../utils'))) +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '../../../../utils/atlasboost'))) + +import data_loader as dl + +import model as ml +import hyper_param as hp +import layers as ly +import logger as lg +import trainer as tr +import create_session as cs +import argparse + +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter + + + +def parse_args(): + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + + parser.add_argument('--rank_size', default=1,type=int, + help="""number of NPUs to use.""") + + # mode and parameters related + parser.add_argument('--mode', default='train_and_evaluate', + help="""mode to run the program e.g. train, evaluate, and + train_and_evaluate""") + parser.add_argument('--max_train_steps', default=100,type=int, + help="""train steps for one NPU""") + parser.add_argument('--iterations_per_loop', default=10, type=int, + help="""the number of steps in devices for each iteration""") + parser.add_argument('--max_epochs', default=None, type=int, + help="""total epochs for training""") + parser.add_argument('--epochs_between_evals', default=5, type=int, + help="""the interval between train and evaluation , only meaningful + when the mode is train_and_evaluate""") + + # dataset + parser.add_argument('--data_dir', default='path/data', + help="""directory to data.""") + + # path for evaluation + parser.add_argument('--eval_dir', default='path/eval', + help="""directory to evaluate.""") + + parser.add_argument('--dtype', default=tf.float32, + help="""data type of inputs.""") + parser.add_argument('--use_nesterov', default=True, type=ast.literal_eval, + help=""" used in optimizer""") + parser.add_argument('--label_smoothing', default=0.1, type=float, + help="""label smoothing factor""") + parser.add_argument('--weight_decay', default=0.0001, + help="""weight decay""") + parser.add_argument('--batch_size', default=32, type=int, + help="""batch size for one NPU""") + + # learning rate and momentum + parser.add_argument('--lr', default=0.1, type=float, + help="""learning rate""") + parser.add_argument('--T_max', default=150, type=int, + help="""T_max for cosing_annealing learning rate""") + parser.add_argument('--momentum', default=0.9, type=float, + help="""momentum used in optimizer.""") + + # display frequency + parser.add_argument('--display_every', default=1, type=int, + help="""the frequency to display info""") + + # log file + parser.add_argument('--log_name', default='densenet121_training.log', + help="""name of log file""") + parser.add_argument('--log_dir', default='./model_1p', + help="""log directory""") + + args, unknown_args = parser.parse_known_args() + # ['--config_file', 'densenet_config_1p_npu'] + + print(args, unknown_args) + if len(unknown_args) > 0: + for bad_arg in unknown_args: + print("ERROR: Unknown command line arg: %s" % bad_arg) + raise ValueError("Invalid command line arg(s)") + + return args + + +def main(): + + args = parse_args() + args.global_batch_size = args.batch_size * args.rank_size + + session = cs.CreateSession() + data = dl.DataLoader(args) + hyper_param = hp.HyperParams(args) + layers = ly.Layers() + logger = lg.LogSessionRunHook(args) + model = ml.Model(args, data, hyper_param, layers, logger) + + trainer = tr.Trainer(session, args, data, model, logger) + + if args.mode == 'train': + trainer.train() + elif args.mode == 'evaluate': + trainer.evaluate() + elif args.mode == 'train_and_evaluate': + trainer.train_and_evaluate() + else: + raise ValueError("Invalid mode.") + + +if __name__ == '__main__': + + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("tensorflow") + config_info = get_model_parameter("tensorflow_config") + initinal_data = {"base_lr": 0.128, "dataset": "imagenet1024", "optimizer": "SGD", "loss_scale": 512, + "batchsize": 32} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + hwlog.remark_print(key=hwlog.INPUT_BATCH_SIZE, value=initinal_data.get("batchsize")) + main() + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/train_helper.py b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/train_helper.py new file mode 100644 index 0000000..2674c3e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/train_helper.py @@ -0,0 +1,22 @@ +import tensorflow as tf +from tensorflow.python.ops import data_flow_ops +import re +import os +from operator import itemgetter + + +def sort_and_load_ckpts(log_dir): + ckpts = [] + for f in os.listdir(log_dir): + m = re.match(r'model.ckpt-([0-9]+).index', f) + if m is None: + continue + fullpath = os.path.join(log_dir, f) + ckpts.append({'step': int(m.group(1)), + 'path': os.path.splitext(fullpath)[0], + 'mtime': os.stat(fullpath).st_mtime, + }) + ckpts.sort(key=itemgetter('step')) + return ckpts + + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/trainer.py b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/trainer.py new file mode 100644 index 0000000..c27a9ec --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/code/trainer.py @@ -0,0 +1,128 @@ +import tensorflow as tf +import math +import time +import os +import train_helper +from logger import rank0log +from benchmark_log import hwlog + +class Trainer(object): + def __init__(self, session, args, data, model, logger): + self.sess = session + self.args = args + self.data = data + self.model = model + self.logger = logger + self.print_logger = self.logger.logger + self.all_preds = [] + self.all_targets = [] + + self.classifier, self.training_hook = self.get_npu_classifier() + + def get_npu_classifier(self): + from npu_bridge.estimator.npu.npu_config import NPURunConfig + from npu_bridge.estimator.npu.npu_estimator import NPUEstimator + + run_config = NPURunConfig( + hcom_parallel=True, + precision_mode="allow_mix_precision", + enable_data_pre_proc=True, + save_checkpoints_steps=self.args.nsteps_per_epoch, + session_config=self.sess.estimator_config, + model_dir=self.args.log_dir, + iterations_per_loop=self.args.iterations_per_loop, + keep_checkpoint_max=5) + + classifier =NPUEstimator( + model_fn= self.model.get_estimator_model_func, + config= run_config + ) + + training_hooks = [] + training_hooks.append(self.logger) + + return classifier, training_hooks + + def train(self): + print ('training steps: %d' % self.args.nstep) + self.classifier.train( input_fn=lambda:self.data.get_train_input_fn(), + max_steps = self.args.nstep, + hooks = self.training_hook + ) + + def evaluate(self): + rank0log(self.print_logger, "Evaluating") + rank0log(self.print_logger, "Validation dataset size: {}".format(self.args.num_evaluating_samples)) + time.sleep(5) # a little extra margin... + try: + ckpts = train_helper.sort_and_load_ckpts(self.args.eval_dir) + print("=========ckpt==========") + print(ckpts) + print("=========ckpt==========") + for i, c in enumerate(ckpts): + eval_result = self.classifier.evaluate( + input_fn=lambda: self.data.get_eval_input_fn(), + checkpoint_path=c['path']) + c['epoch'] = math.ceil(c['step'] / (self.args.num_training_samples/ (self.args.batch_size))) + c['top1'] = eval_result['val-top1acc'] + c['top5'] = eval_result['val-top5acc'] + c['loss'] = eval_result['loss'] + + rank0log(self.print_logger, ' step epoch top1 top5 loss checkpoint_time(UTC)') + for i, c in enumerate(ckpts): + if 'top1' not in c: + continue + rank0log(self.print_logger,'{:5d} {:5.1f} {:5.3f} {:6.2f} {:6.2f} {time}' + .format(c['step'], + c['epoch'], + c['top1'] * 100, + c['top5'] * 100, + c['loss'], + time=time.strftime('%Y-%m-%d %H:%M:%S', + time.localtime(c['mtime'])))) + rank0log(self.print_logger, "Finished evaluation") + except KeyboardInterrupt: + self.print_logger.error("Keyboard interrupt") + + def train_and_evaluate(self): + epochs_between_evals = self.args.epochs_between_evals + + for i in range(self.args.max_epochs // epochs_between_evals): + + rank0log(self.print_logger, "Starting a training cycle") + + self.classifier.train(input_fn=lambda:self.data.get_train_input_fn(), + steps = self.args.nsteps_per_epoch*epochs_between_evals, + hooks = self.training_hook ) + + rank0log(self.print_logger, "Starting to evaluate") + rank0log(self.print_logger, "Validation dataset size: {}".format(self.args.num_evaluating_samples)) + time.sleep(5) # a little extra margin... + + ckpts = train_helper.sort_and_load_ckpts(self.args.log_dir) + c = ckpts[-1] + eval_result = self.classifier.evaluate( + input_fn=lambda: self.data.get_eval_input_fn(), + checkpoint_path=c['path']) + + # top1 top5 Log dotting + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value=float(eval_result.get("val-top1acc"))) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value=float(eval_result.get("val-top5acc"))) + + + c['epoch'] = math.ceil(c['step'] / (self.args.num_training_samples / (self.args.batch_size * self.args.rank_size))) + c['top1'] = eval_result['val-top1acc'] + c['top5'] = eval_result['val-top5acc'] + c['loss'] = eval_result['loss'] + + rank0log(self.print_logger, ' step epoch top1 top5 loss checkpoint_time(UTC)') + + rank0log(self.print_logger,'{:5d} {:5.1f} {:5.3f} {:6.2f} {:6.2f} {time}' + .format(c['step'], + c['epoch'], + c['top1'] * 100, + c['top5'] * 100, + c['loss'], + time=time.strftime('%Y-%m-%d %H:%M:%S', + time.localtime(c['mtime'])))) + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/config/1p.json b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/config/1p.json new file mode 100644 index 0000000..c1b3e8e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/config/1p.json @@ -0,0 +1,23 @@ +{ + "group_count": "1", + "group_list": [ + { + "group_name": "worker", + "device_count": "1", + "instance_count": "1", + "instance_list": [ + { + "devices": [ + { + "device_id": "7", + "device_ip": "192.168.193.103" + } + ], + "pod_name": "npu1p", + "server_id": "127.0.0.1" + } + ] + } + ], + "status": "completed" +} \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/config/8p.json b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/config/8p.json new file mode 100644 index 0000000..421197d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/config/8p.json @@ -0,0 +1,51 @@ +{ + "group_count": "1", + "group_list": [ + { + "group_name": "worker", + "device_count": "8", + "instance_count": "1", + "instance_list": [ + { + "devices": [ + { + "device_id": "0", + "device_ip": "192.168.190.102" + }, + { + "device_id": "1", + "device_ip": "192.168.191.102" + }, + { + "device_id": "2", + "device_ip": "192.168.192.102" + }, + { + "device_id": "3", + "device_ip": "192.168.193.102" + }, + { + "device_id": "4", + "device_ip": "192.168.190.103" + }, + { + "device_id": "5", + "device_ip": "192.168.191.103" + }, + { + "device_id": "6", + "device_ip": "192.168.192.103" + }, + { + "device_id": "7", + "device_ip": "192.168.193.103" + } + ], + "pod_name": "npu8p", + "server_id": "127.0.0.1" + } + ] + } + ], + "status": "completed" +} \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/config/hccl_sample.json b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/config/hccl_sample.json new file mode 100644 index 0000000..96ec094 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/config/hccl_sample.json @@ -0,0 +1,9 @@ +{ + "server_count": "1", + "server_list": [{ + "device": [{devices}], + "server_id": "127.0.0.1" + }], + "status": "completed", + "version": "1.0" +} diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/config/npu_set_env.sh b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/config/npu_set_env.sh new file mode 100644 index 0000000..092c803 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/config/npu_set_env.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +rm -rf /var/log/npu/slog/host-0/* +#安装toolkit +#export LD_LIBRARY_PATH=/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/ +#export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/ascend-toolkit/latest//fwkacllib/python/site-packages/te:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/topi:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/hccl:/usr/local/Ascend/ascend-toolkit/latest/tfplugin/python/site-packages +#export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin +#export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + +#安装nnae等 +#export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib/:/usr/local/Ascend/nnae/latest/x86_64-linux_gcc7.3.0/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/ +#export PYTHONPATH=/home/train/resnet50_tf/code:/usr/local/Ascend/nnae/latest/x86_64-linux_gcc7.3.0/opp/op_impl/built-in/ai_core/tbe/:/usr/local/Ascend/nnae/latest/x86_64-linux_gcc7.3.0/fwkacllib/python/site-packages/te/:/usr/local/Ascend/nnae/latest/x86_64-linux_gcc7.3.0/fwkacllib/python/site-packages/topi/:/usr/local/Ascend/nnae/latest/x86_64-linux_gcc7.3.0/fwkacllib/python/site-packages/hccl/:/usr/local/Ascend/tfplugin/latest/x86_64-linux_gcc7.3.0/tfplugin/python/site-packages/:/usr/local/Ascend/tfplugin/latest/x86_64-linux_gcc7.3.0/tfplugin/python/site-packages/npu_bridge:/code +#export PATH=$PATH:/usr/local/Ascend/nnae/latest/x86_64-linux_gcc7.3.0/fwkacllib/ccec_compiler/bin/ +#export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/x86_64-linux_gcc7.3.0/opp/ + + +if [ -d /usr/local/Ascend/nnae/latest ];then + + export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/Ascend/driver/tools/hccn_tool/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages + export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp +else + export LD_LIBRARY_PATH=/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/ascend-toolkit/latest//fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/tfplugin/python/site-packages:$projectDir + export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + +fi + +export DDK_VERSION_FLAG=1.60.T17.B830 +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 + +export SLOG_PRINT_TO_STDOUT=0 + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/scripts/run.sh b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/scripts/run.sh new file mode 100644 index 0000000..4f278c2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/scripts/run.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +rank_size=$1 +yamlPath=$2 +toolsPath=$3 + +currentDir=$(cd "$(dirname "$0")/.."; pwd) +model_name=$(cd $currentDir/..;basename `pwd`) +if [ -f /.dockerenv ];then + CLUSTER=$4 + MPIRUN_ALL_IP="$5" + export CLUSTER=${CLUSTER} +fi +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "tensorflow_config") + +if [ $? -eq 0 ] ; +then + echo "modify inner config file success" +else + echo "modify inner config file fail" + exit 1 +fi + +#mkdir train job path +currtime=`date +%Y%m%d%H%M%S` + +mkdir -p ${currentDir%train*}/train/result/tf_densenet121/training_job_${currtime}/ +train_job_dir=${currentDir%train*}/train/result/tf_densenet121/training_job_${currtime}/ +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir} &" +# device 列表, 若无指定 device 根据 rank_size 顺序选择 +eval device_group=\$device_group_${rank_size}p +if [ x"${device_group}" == x"" ] || [ ${rank_size} -ge 8 ];then + device_group="$(seq 0 "$(expr $rank_size - 1)")" +fi + +# get last device id in device_group, hw log in performance from the dir named last_device_id +device_group_str=`echo ${device_group} | sed 's/ //g'` +first_device_id=`echo ${device_group_str: 0:1}` + +if [ x"${CLUSTER}" == x"True" ];then + # ln hw log + ln -snf ${currentDir%train*}/train/result/tf_densenet121/training_job_${currtime}/0/hw_densenet121.log ${currentDir%train*}/train/result/tf_densenet121/training_job_${currtime}/ + this_ip=$(hostname -I |awk '{print $1}') + for ip in $MPIRUN_ALL_IP;do + if [ x"$ip" != x"$this_ip" ];then + scp $yamlPath root@$ip:$yamlPath + scp ${jsonFilePath} root@$ip:${jsonFilePath} + fi + done + export PATH=$PATH:/usr/local/mpirun4.0/bin + mpirun -H ${mpirun_ip} \ + --bind-to none -map-by slot\ + --allow-run-as-root \ + --mca btl_tcp_if_exclude lo,docker0,endvnic,virbr0,vethf40501b,docker_gwbridge,br-f42ac38052b4\ + --prefix /usr/local/mpirun4.0/ \ + ${currentDir}/scripts/train.sh 0 $rank_size $yamlPath $currtime ${toolsPath} ${CLUSTER} +else + # ln hw log + ln -snf ${currentDir%train*}/train/result/tf_densenet121/training_job_${currtime}/${first_device_id}/hw_densenet121.log ${currentDir%train*}/train/result/tf_densenet121/training_job_${currtime}/ + rank_id=0 + for device_id in $device_group;do + ${currentDir}/scripts/train.sh $device_id $rank_size $yamlPath $currtime ${toolsPath} $rank_id & + let rank_id++ + done +fi +wait + +#echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] all train exit " >> ${currentDir}/result/main.log + diff --git a/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/scripts/train.sh b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/scripts/train.sh new file mode 100644 index 0000000..a77062c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/DenseNet121/tensorflow/scripts/train.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +device_id=$1 +rank_size=$2 +yamlPath=$3 + +currentDir=$(cd "$(dirname "$0")/.."; pwd) +model_name="densenet121" +currtime=$4 +toolsPath=$5 + +export YAML_PATH=$3 + + +mkdir -p ${currentDir%train*}/train/result/tf_densenet121/training_job_${currtime}/ +export train_job_dir=${currentDir%train*}/train/result/tf_densenet121/training_job_${currtime}/ + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "tensorflow_config") + +export REMARK_LOG_FILE=hw_densenet121.log # 打点日志文件名称， 必须hw_后跟模型名称小写 +benchmark_log_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils +export PYTHONPATH=$PYTHONPATH:${benchmark_log_path} + +source ${currentDir}/config/npu_set_env.sh + +# user env +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 +export RANK_TABLE_FILE=${currentDir}/config/${rank_size}p.json +export RANK_SIZE=${rank_size} +export SLOG_PRINT_TO_STDOUT=0 +export DEVICE_ID=${device_id} +DEVICE_INDEX=$(( DEVICE_ID + RANK_INDEX * 8 )) +export DEVICE_INDEX=${DEVICE_INDEX} + +cd ${train_job_dir} +curd_dir=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +export PYTHONPATH=$PYTHONPATH:${curd_dir} + +if [ x"$6" != x"True" ];then + rank_id=$6 + export RANK_ID=$6 +else + device_id_mo=$(python3.7 -c "import src.tensorflow.mpi_ops as atlasboost;atlasboost.init(); \ + device_id = atlasboost.local_rank();cluster_device_id = str(device_id); \ + atlasboost.set_device_id(device_id);print(atlasboost.rank())") + device_id_mo=`echo $device_id_mo` + rank_id=${device_id_mo##* } + export RANK_ID=${rank_id} + device=${device_id_mo##*deviceid = } + device_id=${device%% phyid=*} + export DEVICE_ID=${device_id} + hccljson=${train_job_dir}/*.json + cp ${hccljson} ${currentDir}/config/${rank_size}p.json +fi + +#mkdir exec path +mkdir -p ${train_job_dir}/${device_id} +cd ${train_job_dir}/${device_id} + +startTime=`date +%Y%m%d-%H:%M:%S` +startTime_s=`date +%s` + +# 根据单卡/多卡区分调用参数 +if [ x"$6" == x"True" ];then + # 多卡多机 + export CLUSTER=True + python3.7 ${currentDir}/code/train.py --rank_size=${rank_size} --mode=${mode_8p} --max_epochs=${epoches} --iterations_per_loop=${iterations_per_loop_8p} --epochs_between_evals=${epochs_between_evals} --data_dir=${data_url} --lr=${lr} --log_dir=${log_dir} --log_name=${log_name_8p} > ${train_job_dir}/train_${device_id}.log 2>&1 +elif [ x"${rank_size}" == x"1" ];then + # 单卡 + python3.7 ${currentDir}/code/train.py --rank_size=${rank_size} --mode=${mode_1p} --max_train_steps=${max_train_steps_1p} --iterations_per_loop=${iterations_per_loop_1p} --data_dir=${data_url} --display_every=${display_every} --log_dir=${log_dir} --log_name=${log_name_1p} > ${train_job_dir}/train_${device_id}.log 2>&1 + +elif [ ${rank_size} -le 8 ];then + # 多卡单机 + + python3.7 ${currentDir}/code/train.py --rank_size=${rank_size} --mode=${mode_8p} --max_epochs=${epoches} --iterations_per_loop=${iterations_per_loop_8p} --epochs_between_evals=${epochs_between_evals} --data_dir=${data_url} --lr=${lr} --log_dir=${log_dir} --log_name=${log_name_8p} > ${train_job_dir}/train_${device_id}.log 2>&1 +fi + +if [ $? -eq 0 ];then + echo ":::ABK 1.0.0 densenet121 train success" + echo ":::ABK 1.0.0 densenet121 train success" >> ${train_job_dir}/train_${device_id}.log + echo ":::ABK 1.0.0 densenet121 train success" >> ./hw_densenet121.log +else + echo ":::ABK 1.0.0 densenet121 train failed" + echo ":::ABK 1.0.0 densenet121 train failed" >> ${train_job_dir}/train_${device_id}.log + echo ":::ABK 1.0.0 densenet121 train failed" >> ./hw_densenet121.log +fi + +endTime=`date +%Y%m%d-%H:%M:%S` +endTime_s=`date +%s` +sumTime=$[ $endTime_s - $startTime_s ] +hour=$(( $sumTime/3600 )) +min=$(( ($sumTime-${hour}*3600)/60 )) +sec=$(( $sumTime-${hour}*3600-${min}*60 )) +echo ":::ABK 1.0.0 densenet121 train total time： ${hour}:${min}:${sec}" +echo ":::ABK 1.0.0 densenet121 train total time： ${hour}:${min}:${sec}" >> ${train_job_dir}/${device_id}/hw_densenet121.log diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/README.md b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/README.md new file mode 100644 index 0000000..654c128 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/README.md @@ -0,0 +1,25 @@ +# EfficientNet_pytorch训练说明 + +### 1. 模型训练参数配置 + +在train/yaml/EfficientNet.yaml中修改相应配置， 配置项含义: + +``` +pytorch_config: + data_url: 数据集路径 + epoches: 跑多少个epoch + batch_size: 1p 参数为256 2p 512 4p 1024 8p为2048 + seed: 49 + lr: 默认参数1p 0.2 2p 0.4 4p 0.8 8p 1.6 + docker_image: docker 镜像名称:版本号 +``` + +------ + + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/LICENSE b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/README.md b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/README.md new file mode 100644 index 0000000..2501928 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/README.md @@ -0,0 +1,253 @@ +# EfficientNet PyTorch + +### Quickstart + +Install with `pip install efficientnet_pytorch` and load a pretrained EfficientNet with: +```python +from efficientnet_pytorch import EfficientNet +model = EfficientNet.from_pretrained('efficientnet-b0') +``` + +### Updates + +#### Update (May 14, 2020) + +This update adds comprehensive comments and documentation (thanks to @workingcoder). + +#### Update (January 23, 2020) + +This update adds a new category of pre-trained model based on adversarial training, called _advprop_. It is important to note that the preprocessing required for the advprop pretrained models is slightly different from normal ImageNet preprocessing. As a result, by default, advprop models are not used. To load a model with advprop, use: +``` +model = EfficientNet.from_pretrained("efficientnet-b0", advprop=True) +``` +There is also a new, large `efficientnet-b8` pretrained model that is only available in advprop form. When using these models, replace ImageNet preprocessing code as follows: +``` +if advprop: # for models using advprop pretrained weights + normalize = transforms.Lambda(lambda img: img * 2.0 - 1.0) +else: + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + +``` +This update also addresses multiple other issues ([#115](https://github.com/lukemelas/EfficientNet-PyTorch/issues/115), [#128](https://github.com/lukemelas/EfficientNet-PyTorch/issues/128)). + +#### Update (October 15, 2019) + +This update allows you to choose whether to use a memory-efficient Swish activation. The memory-efficient version is chosen by default, but it cannot be used when exporting using PyTorch JIT. For this purpose, we have also included a standard (export-friendly) swish activation function. To switch to the export-friendly version, simply call `model.set_swish(memory_efficient=False)` after loading your desired model. This update addresses issues [#88](https://github.com/lukemelas/EfficientNet-PyTorch/pull/88) and [#89](https://github.com/lukemelas/EfficientNet-PyTorch/pull/89). + +#### Update (October 12, 2019) + +This update makes the Swish activation function more memory-efficient. It also addresses pull requests [#72](https://github.com/lukemelas/EfficientNet-PyTorch/pull/72), [#73](https://github.com/lukemelas/EfficientNet-PyTorch/pull/73), [#85](https://github.com/lukemelas/EfficientNet-PyTorch/pull/85), and [#86](https://github.com/lukemelas/EfficientNet-PyTorch/pull/86). Thanks to the authors of all the pull requests! + +#### Update (July 31, 2019) + +_Upgrade the pip package with_ `pip install --upgrade efficientnet-pytorch` + +The B6 and B7 models are now available. Additionally, _all_ pretrained models have been updated to use AutoAugment preprocessing, which translates to better performance across the board. Usage is the same as before: +```python +from efficientnet_pytorch import EfficientNet +model = EfficientNet.from_pretrained('efficientnet-b7') +``` + +#### Update (June 29, 2019) + +This update adds easy model exporting ([#20](https://github.com/lukemelas/EfficientNet-PyTorch/issues/20)) and feature extraction ([#38](https://github.com/lukemelas/EfficientNet-PyTorch/issues/38)). + + * [Example: Export to ONNX](#example-export) + * [Example: Extract features](#example-feature-extraction) + * Also: fixed a CUDA/CPU bug ([#32](https://github.com/lukemelas/EfficientNet-PyTorch/issues/32)) + +It is also now incredibly simple to load a pretrained model with a new number of classes for transfer learning: +```python +model = EfficientNet.from_pretrained('efficientnet-b1', num_classes=23) +``` + + +#### Update (June 23, 2019) + +The B4 and B5 models are now available. Their usage is identical to the other models: +```python +from efficientnet_pytorch import EfficientNet +model = EfficientNet.from_pretrained('efficientnet-b4') +``` + +### Overview +This repository contains an op-for-op PyTorch reimplementation of [EfficientNet](https://arxiv.org/abs/1905.11946), along with pre-trained models and examples. + +The goal of this implementation is to be simple, highly extensible, and easy to integrate into your own projects. This implementation is a work in progress -- new features are currently being implemented. + +At the moment, you can easily: + * Load pretrained EfficientNet models + * Use EfficientNet models for classification or feature extraction + * Evaluate EfficientNet models on ImageNet or your own images + +_Upcoming features_: In the next few days, you will be able to: + * Train new models from scratch on ImageNet with a simple command + * Quickly finetune an EfficientNet on your own dataset + * Export EfficientNet models for production + +### Table of contents +1. [About EfficientNet](#about-efficientnet) +2. [About EfficientNet-PyTorch](#about-efficientnet-pytorch) +3. [Installation](#installation) +4. [Usage](#usage) + * [Load pretrained models](#loading-pretrained-models) + * [Example: Classify](#example-classification) + * [Example: Extract features](#example-feature-extraction) + * [Example: Export to ONNX](#example-export) +6. [Contributing](#contributing) + +### About EfficientNet + +If you're new to EfficientNets, here is an explanation straight from the official TensorFlow implementation: + +EfficientNets are a family of image classification models, which achieve state-of-the-art accuracy, yet being an order-of-magnitude smaller and faster than previous models. We develop EfficientNets based on AutoML and Compound Scaling. In particular, we first use [AutoML Mobile framework](https://ai.googleblog.com/2018/08/mnasnet-towards-automating-design-of.html) to develop a mobile-size baseline network, named as EfficientNet-B0; Then, we use the compound scaling method to scale up this baseline to obtain EfficientNet-B1 to B7. + + + + + + +
+ + + +
+ +EfficientNets achieve state-of-the-art accuracy on ImageNet with an order of magnitude better efficiency: + + +* In high-accuracy regime, our EfficientNet-B7 achieves state-of-the-art 84.4% top-1 / 97.1% top-5 accuracy on ImageNet with 66M parameters and 37B FLOPS, being 8.4x smaller and 6.1x faster on CPU inference than previous best [Gpipe](https://arxiv.org/abs/1811.06965). + +* In middle-accuracy regime, our EfficientNet-B1 is 7.6x smaller and 5.7x faster on CPU inference than [ResNet-152](https://arxiv.org/abs/1512.03385), with similar ImageNet accuracy. + +* Compared with the widely used [ResNet-50](https://arxiv.org/abs/1512.03385), our EfficientNet-B4 improves the top-1 accuracy from 76.3% of ResNet-50 to 82.6% (+6.3%), under similar FLOPS constraint. + +### About EfficientNet PyTorch + +EfficientNet PyTorch is a PyTorch re-implementation of EfficientNet. It is consistent with the [original TensorFlow implementation](https://github.com/tensorflow/tpu/tree/master/models/official/efficientnet), such that it is easy to load weights from a TensorFlow checkpoint. At the same time, we aim to make our PyTorch implementation as simple, flexible, and extensible as possible. + +If you have any feature requests or questions, feel free to leave them as GitHub issues! + +### Installation + +Install via pip: +```bash +pip install efficientnet_pytorch +``` + +Or install from source: +```bash +git clone https://github.com/lukemelas/EfficientNet-PyTorch +cd EfficientNet-Pytorch +pip install -e . +``` + +### Usage + +#### Loading pretrained models + +Load an EfficientNet: +```python +from efficientnet_pytorch import EfficientNet +model = EfficientNet.from_name('efficientnet-b0') +``` + +Load a pretrained EfficientNet: +```python +from efficientnet_pytorch import EfficientNet +model = EfficientNet.from_pretrained('efficientnet-b0') +``` + +Note that pretrained models have only been released for `N=0,1,2,3,4,5` at the current time, so `.from_pretrained` only supports `'efficientnet-b{N}'` for `N=0,1,2,3,4,5`. + +Details about the models are below: + +| *Name* |*# Params*|*Top-1 Acc.*|*Pretrained?*| +|:-----------------:|:--------:|:----------:|:-----------:| +| `efficientnet-b0` | 5.3M | 76.3 | ✓ | +| `efficientnet-b1` | 7.8M | 78.8 | ✓ | +| `efficientnet-b2` | 9.2M | 79.8 | ✓ | +| `efficientnet-b3` | 12M | 81.1 | ✓ | +| `efficientnet-b4` | 19M | 82.6 | ✓ | +| `efficientnet-b5` | 30M | 83.3 | ✓ | +| `efficientnet-b6` | 43M | 84.0 | ✓ | +| `efficientnet-b7` | 66M | 84.4 | ✓ | + + +#### Example: Classification + +Below is a simple, complete example. It may also be found as a jupyter notebook in `examples/simple` or as a [Colab Notebook](https://colab.research.google.com/drive/1Jw28xZ1NJq4Cja4jLe6tJ6_F5lCzElb4). + +We assume that in your current directory, there is a `img.jpg` file and a `labels_map.txt` file (ImageNet class names). These are both included in `examples/simple`. + +```python +import json +from PIL import Image +import torch +from torchvision import transforms + +from efficientnet_pytorch import EfficientNet +model = EfficientNet.from_pretrained('efficientnet-b0') + +# Preprocess image +tfms = transforms.Compose([transforms.Resize(224), transforms.ToTensor(), + transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),]) +img = tfms(Image.open('img.jpg')).unsqueeze(0) +print(img.shape) # torch.Size([1, 3, 224, 224]) + +# Load ImageNet class names +labels_map = json.load(open('labels_map.txt')) +labels_map = [labels_map[str(i)] for i in range(1000)] + +# Classify +model.eval() +with torch.no_grad(): + outputs = model(img) + +# Print predictions +print('-----') +for idx in torch.topk(outputs, k=5).indices.squeeze(0).tolist(): + prob = torch.softmax(outputs, dim=1)[0, idx].item() + print('{label:<75} ({p:.2f}%)'.format(label=labels_map[idx], p=prob*100)) +``` + +#### Example: Feature Extraction + +You can easily extract features with `model.extract_features`: +```python +from efficientnet_pytorch import EfficientNet +model = EfficientNet.from_pretrained('efficientnet-b0') + +# ... image preprocessing as in the classification example ... +print(img.shape) # torch.Size([1, 3, 224, 224]) + +features = model.extract_features(img) +print(features.shape) # torch.Size([1, 1280, 7, 7]) +``` + +#### Example: Export to ONNX + +Exporting to ONNX for deploying to production is now simple: +```python +import torch +from efficientnet_pytorch import EfficientNet + +model = EfficientNet.from_pretrained('efficientnet-b1') +dummy_input = torch.randn(10, 3, 240, 240) + +torch.onnx.export(model, dummy_input, "test-b1.onnx", verbose=True) +``` + +[Here](https://colab.research.google.com/drive/1rOAEXeXHaA8uo3aG2YcFDHItlRJMV0VP) is a Colab example. + + +#### ImageNet + +See `examples/imagenet` for details about evaluating on ImageNet. + +### Contributing + +If you find a bug, create a GitHub issue, or even better, submit a pull request. Similarly, if you have questions, simply post them as GitHub issues. + +I look forward to seeing what the community does with these models! diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/README_NPU.md b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/README_NPU.md new file mode 100644 index 0000000..8e0da51 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/README_NPU.md @@ -0,0 +1,45 @@ +# EfficientNet PyTorch + +## About EfficientNet + +If you're new to EfficientNets, here is an explanation straight from the official TensorFlow implementation: + +EfficientNets are a family of image classification models, which achieve state-of-the-art accuracy, yet being an order-of-magnitude smaller and faster than previous models. We develop EfficientNets based on AutoML and Compound Scaling. In particular, we first use [AutoML Mobile framework](https://ai.googleblog.com/2018/08/mnasnet-towards-automating-design-of.html) to develop a mobile-size baseline network, named as EfficientNet-B0; Then, we use the compound scaling method to scale up this baseline to obtain EfficientNet-B1 to B7. + +EfficientNets achieve state-of-the-art accuracy on ImageNet with an order of magnitude better efficiency: + + +* In high-accuracy regime, our EfficientNet-B7 achieves state-of-the-art 84.4% top-1 / 97.1% top-5 accuracy on ImageNet with 66M parameters and 37B FLOPS, being 8.4x smaller and 6.1x faster on CPU inference than previous best [Gpipe](https://arxiv.org/abs/1811.06965). + +* In middle-accuracy regime, our EfficientNet-B1 is 7.6x smaller and 5.7x faster on CPU inference than [ResNet-152](https://arxiv.org/abs/1512.03385), with similar ImageNet accuracy. + +* Compared with the widely used [ResNet-50](https://arxiv.org/abs/1512.03385), our EfficientNet-B4 improves the top-1 accuracy from 76.3% of ResNet-50 to 82.6% (+6.3%), under similar FLOPS constraint. + +## About EfficientNet PyTorch NPU + +The source codes are based on the open source https://github.com/lukemelas/EfficientNet-PyTorch with least modified codes as far as possible. + + +## Quick Start + +### Train on 1 NPU: + +(1) modify the last line in npu_1p.sh with the particular params: + +* fp32: taskset -c 0-64 python3.7 examples/imagenet/main.py --data=/data/imagenet --arch=efficientnet-b0 --batch-size=256 --lr=0.2 --epochs=200 --autoaug --npu=0 +* O1: taskset -c 0-64 python3.7 examples/imagenet/main.py --data=/data/imagenet --arch=efficientnet-b0 --batch-size=256 --lr=0.2 --epochs=200 --autoaug --npu=0 --amp --pm=O1 --loss_scale=1024 +* O2: taskset -c 0-64 python3.7 examples/imagenet/main.py --data=/data/imagenet --arch=efficientnet-b0 --batch-size=256 --lr=0.2 --epochs=200 --autoaug --npu=0 --amp --pm=O2 --loss_scale=128 + +(2) Execute run.sh，ALL the train log will be recorded in nohup.out. + +## Know issues: + +* Distribution train is NOT available. +* top1/top5 accuracy is lower than GPU about 2% in the same setting (dropout). +* O2 Performance is lower than GPU about 50 fps in the same setting (dropout, depthwiseconv2d). +* torch.rand is replaced with numpy implementation due to the lack of AICPU operator (aicpu). +* momentum has to be set to 0 due to logsoftmax precision(logsoftmax) + + + + diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/__init__.py b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/__init__.py new file mode 100644 index 0000000..182abec --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/__init__.py @@ -0,0 +1,12 @@ +__version__ = "0.7.0" +from .model import EfficientNet +from .utils import ( + GlobalParams, + BlockArgs, + BlockDecoder, + efficientnet, + get_model_params, +) +from .auto_augment import rand_augment_transform, augment_and_mix_transform, auto_augment_transform +from .rmsprop_tf import RMSpropTF + diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/auto_augment.py b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/auto_augment.py new file mode 100644 index 0000000..b6c15d9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/auto_augment.py @@ -0,0 +1,817 @@ +""" AutoAugment, RandAugment, and AugMix for PyTorch + +This code implements the searched ImageNet policies with various tweaks and improvements and +does not include any of the search code. + +AA and RA Implementation adapted from: + https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/autoaugment.py + +AugMix adapted from: + https://github.com/google-research/augmix + +Papers: + AutoAugment: Learning Augmentation Policies from Data - https://arxiv.org/abs/1805.09501 + Learning Data Augmentation Strategies for Object Detection - https://arxiv.org/abs/1906.11172 + RandAugment: Practical automated data augmentation... - https://arxiv.org/abs/1909.13719 + AugMix: A Simple Data Processing Method to Improve Robustness and Uncertainty - https://arxiv.org/abs/1912.02781 + +Hacked together by Ross Wightman +""" +import random +import math +import re +from PIL import Image, ImageOps, ImageEnhance, ImageChops +import PIL +import numpy as np + + +_PIL_VER = tuple([int(x) for x in PIL.__version__.split('.')[:2]]) + +_FILL = (128, 128, 128) + +# This signifies the max integer that the controller RNN could predict for the +# augmentation scheme. +_MAX_LEVEL = 10. + +_HPARAMS_DEFAULT = dict( + translate_const=250, + img_mean=_FILL, +) + +_RANDOM_INTERPOLATION = (Image.BILINEAR, Image.BICUBIC) + + +def _interpolation(kwargs): + interpolation = kwargs.pop('resample', Image.BILINEAR) + if isinstance(interpolation, (list, tuple)): + return random.choice(interpolation) + else: + return interpolation + + +def _check_args_tf(kwargs): + if 'fillcolor' in kwargs and _PIL_VER < (5, 0): + kwargs.pop('fillcolor') + kwargs['resample'] = _interpolation(kwargs) + + +def shear_x(img, factor, **kwargs): + _check_args_tf(kwargs) + return img.transform(img.size, Image.AFFINE, (1, factor, 0, 0, 1, 0), **kwargs) + + +def shear_y(img, factor, **kwargs): + _check_args_tf(kwargs) + return img.transform(img.size, Image.AFFINE, (1, 0, 0, factor, 1, 0), **kwargs) + + +def translate_x_rel(img, pct, **kwargs): + pixels = pct * img.size[0] + _check_args_tf(kwargs) + return img.transform(img.size, Image.AFFINE, (1, 0, pixels, 0, 1, 0), **kwargs) + + +def translate_y_rel(img, pct, **kwargs): + pixels = pct * img.size[1] + _check_args_tf(kwargs) + return img.transform(img.size, Image.AFFINE, (1, 0, 0, 0, 1, pixels), **kwargs) + + +def translate_x_abs(img, pixels, **kwargs): + _check_args_tf(kwargs) + return img.transform(img.size, Image.AFFINE, (1, 0, pixels, 0, 1, 0), **kwargs) + + +def translate_y_abs(img, pixels, **kwargs): + _check_args_tf(kwargs) + return img.transform(img.size, Image.AFFINE, (1, 0, 0, 0, 1, pixels), **kwargs) + + +def rotate(img, degrees, **kwargs): + _check_args_tf(kwargs) + if _PIL_VER >= (5, 2): + return img.rotate(degrees, **kwargs) + elif _PIL_VER >= (5, 0): + w, h = img.size + post_trans = (0, 0) + rotn_center = (w / 2.0, h / 2.0) + angle = -math.radians(degrees) + matrix = [ + round(math.cos(angle), 15), + round(math.sin(angle), 15), + 0.0, + round(-math.sin(angle), 15), + round(math.cos(angle), 15), + 0.0, + ] + + def transform(x, y, matrix): + (a, b, c, d, e, f) = matrix + return a * x + b * y + c, d * x + e * y + f + + matrix[2], matrix[5] = transform( + -rotn_center[0] - post_trans[0], -rotn_center[1] - post_trans[1], matrix + ) + matrix[2] += rotn_center[0] + matrix[5] += rotn_center[1] + return img.transform(img.size, Image.AFFINE, matrix, **kwargs) + else: + return img.rotate(degrees, resample=kwargs['resample']) + + +def auto_contrast(img, **__): + return ImageOps.autocontrast(img) + + +def invert(img, **__): + return ImageOps.invert(img) + + +def equalize(img, **__): + return ImageOps.equalize(img) + + +def solarize(img, thresh, **__): + return ImageOps.solarize(img, thresh) + + +def solarize_add(img, add, thresh=128, **__): + lut = [] + for i in range(256): + if i < thresh: + lut.append(min(255, i + add)) + else: + lut.append(i) + if img.mode in ("L", "RGB"): + if img.mode == "RGB" and len(lut) == 256: + lut = lut + lut + lut + return img.point(lut) + else: + return img + + +def posterize(img, bits_to_keep, **__): + if bits_to_keep >= 8: + return img + return ImageOps.posterize(img, bits_to_keep) + + +def contrast(img, factor, **__): + return ImageEnhance.Contrast(img).enhance(factor) + + +def color(img, factor, **__): + return ImageEnhance.Color(img).enhance(factor) + + +def brightness(img, factor, **__): + return ImageEnhance.Brightness(img).enhance(factor) + + +def sharpness(img, factor, **__): + return ImageEnhance.Sharpness(img).enhance(factor) + + +def _randomly_negate(v): + """With 50% prob, negate the value""" + return -v if random.random() > 0.5 else v + + +def _rotate_level_to_arg(level, _hparams): + # range [-30, 30] + level = (level / _MAX_LEVEL) * 30. + level = _randomly_negate(level) + return level, + + +def _enhance_level_to_arg(level, _hparams): + # range [0.1, 1.9] + return (level / _MAX_LEVEL) * 1.8 + 0.1, + + +def _enhance_increasing_level_to_arg(level, _hparams): + # the 'no change' level is 1.0, moving away from that towards 0. or 2.0 increases the enhancement blend + # range [0.1, 1.9] + level = (level / _MAX_LEVEL) * .9 + level = 1.0 + _randomly_negate(level) + return level, + + +def _shear_level_to_arg(level, _hparams): + # range [-0.3, 0.3] + level = (level / _MAX_LEVEL) * 0.3 + level = _randomly_negate(level) + return level, + + +def _translate_abs_level_to_arg(level, hparams): + translate_const = hparams['translate_const'] + level = (level / _MAX_LEVEL) * float(translate_const) + level = _randomly_negate(level) + return level, + + +def _translate_rel_level_to_arg(level, hparams): + # default range [-0.45, 0.45] + translate_pct = hparams.get('translate_pct', 0.45) + level = (level / _MAX_LEVEL) * translate_pct + level = _randomly_negate(level) + return level, + + +def _posterize_level_to_arg(level, _hparams): + # As per Tensorflow TPU EfficientNet impl + # range [0, 4], 'keep 0 up to 4 MSB of original image' + # intensity/severity of augmentation decreases with level + return int((level / _MAX_LEVEL) * 4), + + +def _posterize_increasing_level_to_arg(level, hparams): + # As per Tensorflow models research and UDA impl + # range [4, 0], 'keep 4 down to 0 MSB of original image', + # intensity/severity of augmentation increases with level + return 4 - _posterize_level_to_arg(level, hparams)[0], + + +def _posterize_original_level_to_arg(level, _hparams): + # As per original AutoAugment paper description + # range [4, 8], 'keep 4 up to 8 MSB of image' + # intensity/severity of augmentation decreases with level + return int((level / _MAX_LEVEL) * 4) + 4, + + +def _solarize_level_to_arg(level, _hparams): + # range [0, 256] + # intensity/severity of augmentation decreases with level + return int((level / _MAX_LEVEL) * 256), + + +def _solarize_increasing_level_to_arg(level, _hparams): + # range [0, 256] + # intensity/severity of augmentation increases with level + return 256 - _solarize_level_to_arg(level, _hparams)[0], + + +def _solarize_add_level_to_arg(level, _hparams): + # range [0, 110] + return int((level / _MAX_LEVEL) * 110), + + +LEVEL_TO_ARG = { + 'AutoContrast': None, + 'Equalize': None, + 'Invert': None, + 'Rotate': _rotate_level_to_arg, + # There are several variations of the posterize level scaling in various Tensorflow/Google repositories/papers + 'Posterize': _posterize_level_to_arg, + 'PosterizeIncreasing': _posterize_increasing_level_to_arg, + 'PosterizeOriginal': _posterize_original_level_to_arg, + 'Solarize': _solarize_level_to_arg, + 'SolarizeIncreasing': _solarize_increasing_level_to_arg, + 'SolarizeAdd': _solarize_add_level_to_arg, + 'Color': _enhance_level_to_arg, + 'ColorIncreasing': _enhance_increasing_level_to_arg, + 'Contrast': _enhance_level_to_arg, + 'ContrastIncreasing': _enhance_increasing_level_to_arg, + 'Brightness': _enhance_level_to_arg, + 'BrightnessIncreasing': _enhance_increasing_level_to_arg, + 'Sharpness': _enhance_level_to_arg, + 'SharpnessIncreasing': _enhance_increasing_level_to_arg, + 'ShearX': _shear_level_to_arg, + 'ShearY': _shear_level_to_arg, + 'TranslateX': _translate_abs_level_to_arg, + 'TranslateY': _translate_abs_level_to_arg, + 'TranslateXRel': _translate_rel_level_to_arg, + 'TranslateYRel': _translate_rel_level_to_arg, +} + + +NAME_TO_OP = { + 'AutoContrast': auto_contrast, + 'Equalize': equalize, + 'Invert': invert, + 'Rotate': rotate, + 'Posterize': posterize, + 'PosterizeIncreasing': posterize, + 'PosterizeOriginal': posterize, + 'Solarize': solarize, + 'SolarizeIncreasing': solarize, + 'SolarizeAdd': solarize_add, + 'Color': color, + 'ColorIncreasing': color, + 'Contrast': contrast, + 'ContrastIncreasing': contrast, + 'Brightness': brightness, + 'BrightnessIncreasing': brightness, + 'Sharpness': sharpness, + 'SharpnessIncreasing': sharpness, + 'ShearX': shear_x, + 'ShearY': shear_y, + 'TranslateX': translate_x_abs, + 'TranslateY': translate_y_abs, + 'TranslateXRel': translate_x_rel, + 'TranslateYRel': translate_y_rel, +} + + +class AugmentOp: + + def __init__(self, name, prob=0.5, magnitude=10, hparams=None): + hparams = hparams or _HPARAMS_DEFAULT + self.aug_fn = NAME_TO_OP[name] + self.level_fn = LEVEL_TO_ARG[name] + self.prob = prob + self.magnitude = magnitude + self.hparams = hparams.copy() + self.kwargs = dict( + fillcolor=hparams['img_mean'] if 'img_mean' in hparams else _FILL, + resample=hparams['interpolation'] if 'interpolation' in hparams else _RANDOM_INTERPOLATION, + ) + + # If magnitude_std is > 0, we introduce some randomness + # in the usually fixed policy and sample magnitude from a normal distribution + # with mean `magnitude` and std-dev of `magnitude_std`. + # NOTE This is my own hack, being tested, not in papers or reference impls. + self.magnitude_std = self.hparams.get('magnitude_std', 0) + + def __call__(self, img): + if self.prob < 1.0 and random.random() > self.prob: + return img + magnitude = self.magnitude + if self.magnitude_std and self.magnitude_std > 0: + magnitude = random.gauss(magnitude, self.magnitude_std) + magnitude = min(_MAX_LEVEL, max(0, magnitude)) # clip to valid range + level_args = self.level_fn(magnitude, self.hparams) if self.level_fn is not None else tuple() + return self.aug_fn(img, *level_args, **self.kwargs) + + +def auto_augment_policy_v0(hparams): + # ImageNet v0 policy from TPU EfficientNet impl, cannot find a paper reference. + policy = [ + [('Equalize', 0.8, 1), ('ShearY', 0.8, 4)], + [('Color', 0.4, 9), ('Equalize', 0.6, 3)], + [('Color', 0.4, 1), ('Rotate', 0.6, 8)], + [('Solarize', 0.8, 3), ('Equalize', 0.4, 7)], + [('Solarize', 0.4, 2), ('Solarize', 0.6, 2)], + [('Color', 0.2, 0), ('Equalize', 0.8, 8)], + [('Equalize', 0.4, 8), ('SolarizeAdd', 0.8, 3)], + [('ShearX', 0.2, 9), ('Rotate', 0.6, 8)], + [('Color', 0.6, 1), ('Equalize', 1.0, 2)], + [('Invert', 0.4, 9), ('Rotate', 0.6, 0)], + [('Equalize', 1.0, 9), ('ShearY', 0.6, 3)], + [('Color', 0.4, 7), ('Equalize', 0.6, 0)], + [('Posterize', 0.4, 6), ('AutoContrast', 0.4, 7)], + [('Solarize', 0.6, 8), ('Color', 0.6, 9)], + [('Solarize', 0.2, 4), ('Rotate', 0.8, 9)], + [('Rotate', 1.0, 7), ('TranslateYRel', 0.8, 9)], + [('ShearX', 0.0, 0), ('Solarize', 0.8, 4)], + [('ShearY', 0.8, 0), ('Color', 0.6, 4)], + [('Color', 1.0, 0), ('Rotate', 0.6, 2)], + [('Equalize', 0.8, 4), ('Equalize', 0.0, 8)], + [('Equalize', 1.0, 4), ('AutoContrast', 0.6, 2)], + [('ShearY', 0.4, 7), ('SolarizeAdd', 0.6, 7)], + [('Posterize', 0.8, 2), ('Solarize', 0.6, 10)], # This results in black image with Tpu posterize + [('Solarize', 0.6, 8), ('Equalize', 0.6, 1)], + [('Color', 0.8, 6), ('Rotate', 0.4, 5)], + ] + pc = [[AugmentOp(*a, hparams=hparams) for a in sp] for sp in policy] + return pc + + +def auto_augment_policy_v0r(hparams): + # ImageNet v0 policy from TPU EfficientNet impl, with variation of Posterize used + # in Google research implementation (number of bits discarded increases with magnitude) + policy = [ + [('Equalize', 0.8, 1), ('ShearY', 0.8, 4)], + [('Color', 0.4, 9), ('Equalize', 0.6, 3)], + [('Color', 0.4, 1), ('Rotate', 0.6, 8)], + [('Solarize', 0.8, 3), ('Equalize', 0.4, 7)], + [('Solarize', 0.4, 2), ('Solarize', 0.6, 2)], + [('Color', 0.2, 0), ('Equalize', 0.8, 8)], + [('Equalize', 0.4, 8), ('SolarizeAdd', 0.8, 3)], + [('ShearX', 0.2, 9), ('Rotate', 0.6, 8)], + [('Color', 0.6, 1), ('Equalize', 1.0, 2)], + [('Invert', 0.4, 9), ('Rotate', 0.6, 0)], + [('Equalize', 1.0, 9), ('ShearY', 0.6, 3)], + [('Color', 0.4, 7), ('Equalize', 0.6, 0)], + [('PosterizeIncreasing', 0.4, 6), ('AutoContrast', 0.4, 7)], + [('Solarize', 0.6, 8), ('Color', 0.6, 9)], + [('Solarize', 0.2, 4), ('Rotate', 0.8, 9)], + [('Rotate', 1.0, 7), ('TranslateYRel', 0.8, 9)], + [('ShearX', 0.0, 0), ('Solarize', 0.8, 4)], + [('ShearY', 0.8, 0), ('Color', 0.6, 4)], + [('Color', 1.0, 0), ('Rotate', 0.6, 2)], + [('Equalize', 0.8, 4), ('Equalize', 0.0, 8)], + [('Equalize', 1.0, 4), ('AutoContrast', 0.6, 2)], + [('ShearY', 0.4, 7), ('SolarizeAdd', 0.6, 7)], + [('PosterizeIncreasing', 0.8, 2), ('Solarize', 0.6, 10)], + [('Solarize', 0.6, 8), ('Equalize', 0.6, 1)], + [('Color', 0.8, 6), ('Rotate', 0.4, 5)], + ] + pc = [[AugmentOp(*a, hparams=hparams) for a in sp] for sp in policy] + return pc + + +def auto_augment_policy_original(hparams): + # ImageNet policy from https://arxiv.org/abs/1805.09501 + policy = [ + [('PosterizeOriginal', 0.4, 8), ('Rotate', 0.6, 9)], + [('Solarize', 0.6, 5), ('AutoContrast', 0.6, 5)], + [('Equalize', 0.8, 8), ('Equalize', 0.6, 3)], + [('PosterizeOriginal', 0.6, 7), ('PosterizeOriginal', 0.6, 6)], + [('Equalize', 0.4, 7), ('Solarize', 0.2, 4)], + [('Equalize', 0.4, 4), ('Rotate', 0.8, 8)], + [('Solarize', 0.6, 3), ('Equalize', 0.6, 7)], + [('PosterizeOriginal', 0.8, 5), ('Equalize', 1.0, 2)], + [('Rotate', 0.2, 3), ('Solarize', 0.6, 8)], + [('Equalize', 0.6, 8), ('PosterizeOriginal', 0.4, 6)], + [('Rotate', 0.8, 8), ('Color', 0.4, 0)], + [('Rotate', 0.4, 9), ('Equalize', 0.6, 2)], + [('Equalize', 0.0, 7), ('Equalize', 0.8, 8)], + [('Invert', 0.6, 4), ('Equalize', 1.0, 8)], + [('Color', 0.6, 4), ('Contrast', 1.0, 8)], + [('Rotate', 0.8, 8), ('Color', 1.0, 2)], + [('Color', 0.8, 8), ('Solarize', 0.8, 7)], + [('Sharpness', 0.4, 7), ('Invert', 0.6, 8)], + [('ShearX', 0.6, 5), ('Equalize', 1.0, 9)], + [('Color', 0.4, 0), ('Equalize', 0.6, 3)], + [('Equalize', 0.4, 7), ('Solarize', 0.2, 4)], + [('Solarize', 0.6, 5), ('AutoContrast', 0.6, 5)], + [('Invert', 0.6, 4), ('Equalize', 1.0, 8)], + [('Color', 0.6, 4), ('Contrast', 1.0, 8)], + [('Equalize', 0.8, 8), ('Equalize', 0.6, 3)], + ] + pc = [[AugmentOp(*a, hparams=hparams) for a in sp] for sp in policy] + return pc + + +def auto_augment_policy_originalr(hparams): + # ImageNet policy from https://arxiv.org/abs/1805.09501 with research posterize variation + policy = [ + [('PosterizeIncreasing', 0.4, 8), ('Rotate', 0.6, 9)], + [('Solarize', 0.6, 5), ('AutoContrast', 0.6, 5)], + [('Equalize', 0.8, 8), ('Equalize', 0.6, 3)], + [('PosterizeIncreasing', 0.6, 7), ('PosterizeIncreasing', 0.6, 6)], + [('Equalize', 0.4, 7), ('Solarize', 0.2, 4)], + [('Equalize', 0.4, 4), ('Rotate', 0.8, 8)], + [('Solarize', 0.6, 3), ('Equalize', 0.6, 7)], + [('PosterizeIncreasing', 0.8, 5), ('Equalize', 1.0, 2)], + [('Rotate', 0.2, 3), ('Solarize', 0.6, 8)], + [('Equalize', 0.6, 8), ('PosterizeIncreasing', 0.4, 6)], + [('Rotate', 0.8, 8), ('Color', 0.4, 0)], + [('Rotate', 0.4, 9), ('Equalize', 0.6, 2)], + [('Equalize', 0.0, 7), ('Equalize', 0.8, 8)], + [('Invert', 0.6, 4), ('Equalize', 1.0, 8)], + [('Color', 0.6, 4), ('Contrast', 1.0, 8)], + [('Rotate', 0.8, 8), ('Color', 1.0, 2)], + [('Color', 0.8, 8), ('Solarize', 0.8, 7)], + [('Sharpness', 0.4, 7), ('Invert', 0.6, 8)], + [('ShearX', 0.6, 5), ('Equalize', 1.0, 9)], + [('Color', 0.4, 0), ('Equalize', 0.6, 3)], + [('Equalize', 0.4, 7), ('Solarize', 0.2, 4)], + [('Solarize', 0.6, 5), ('AutoContrast', 0.6, 5)], + [('Invert', 0.6, 4), ('Equalize', 1.0, 8)], + [('Color', 0.6, 4), ('Contrast', 1.0, 8)], + [('Equalize', 0.8, 8), ('Equalize', 0.6, 3)], + ] + pc = [[AugmentOp(*a, hparams=hparams) for a in sp] for sp in policy] + return pc + + +def auto_augment_policy(name='v0', hparams=None): + hparams = hparams or _HPARAMS_DEFAULT + if name == 'original': + return auto_augment_policy_original(hparams) + elif name == 'originalr': + return auto_augment_policy_originalr(hparams) + elif name == 'v0': + return auto_augment_policy_v0(hparams) + elif name == 'v0r': + return auto_augment_policy_v0r(hparams) + else: + assert False, 'Unknown AA policy (%s)' % name + + +class AutoAugment: + + def __init__(self, policy): + self.policy = policy + + def __call__(self, img): + sub_policy = random.choice(self.policy) + for op in sub_policy: + img = op(img) + return img + + +def auto_augment_transform(config_str, hparams): + """ + Create a AutoAugment transform + + :param config_str: String defining configuration of auto augmentation. Consists of multiple sections separated by + dashes ('-'). The first section defines the AutoAugment policy (one of 'v0', 'v0r', 'original', 'originalr'). + The remaining sections, not order sepecific determine + 'mstd' - float std deviation of magnitude noise applied + Ex 'original-mstd0.5' results in AutoAugment with original policy, magnitude_std 0.5 + + :param hparams: Other hparams (kwargs) for the AutoAugmentation scheme + + :return: A PyTorch compatible Transform + """ + config = config_str.split('-') + policy_name = config[0] + config = config[1:] + for c in config: + cs = re.split(r'(\d.*)', c) + if len(cs) < 2: + continue + key, val = cs[:2] + if key == 'mstd': + # noise param injected via hparams for now + hparams.setdefault('magnitude_std', float(val)) + else: + assert False, 'Unknown AutoAugment config section' + aa_policy = auto_augment_policy(policy_name, hparams=hparams) + return AutoAugment(aa_policy) + + +_RAND_TRANSFORMS = [ + 'AutoContrast', + 'Equalize', + 'Invert', + 'Rotate', + 'Posterize', + 'Solarize', + 'SolarizeAdd', + 'Color', + 'Contrast', + 'Brightness', + 'Sharpness', + 'ShearX', + 'ShearY', + 'TranslateXRel', + 'TranslateYRel', + #'Cutout' # NOTE I've implement this as random erasing separately +] + + +_RAND_INCREASING_TRANSFORMS = [ + 'AutoContrast', + 'Equalize', + 'Invert', + 'Rotate', + 'PosterizeIncreasing', + 'SolarizeIncreasing', + 'SolarizeAdd', + 'ColorIncreasing', + 'ContrastIncreasing', + 'BrightnessIncreasing', + 'SharpnessIncreasing', + 'ShearX', + 'ShearY', + 'TranslateXRel', + 'TranslateYRel', + #'Cutout' # NOTE I've implement this as random erasing separately +] + + + +# These experimental weights are based loosely on the relative improvements mentioned in paper. +# They may not result in increased performance, but could likely be tuned to so. +_RAND_CHOICE_WEIGHTS_0 = { + 'Rotate': 0.3, + 'ShearX': 0.2, + 'ShearY': 0.2, + 'TranslateXRel': 0.1, + 'TranslateYRel': 0.1, + 'Color': .025, + 'Sharpness': 0.025, + 'AutoContrast': 0.025, + 'Solarize': .005, + 'SolarizeAdd': .005, + 'Contrast': .005, + 'Brightness': .005, + 'Equalize': .005, + 'Posterize': 0, + 'Invert': 0, +} + + +def _select_rand_weights(weight_idx=0, transforms=None): + transforms = transforms or _RAND_TRANSFORMS + assert weight_idx == 0 # only one set of weights currently + rand_weights = _RAND_CHOICE_WEIGHTS_0 + probs = [rand_weights[k] for k in transforms] + probs /= np.sum(probs) + return probs + + +def rand_augment_ops(magnitude=10, hparams=None, transforms=None): + hparams = hparams or _HPARAMS_DEFAULT + transforms = transforms or _RAND_TRANSFORMS + return [AugmentOp( + name, prob=0.5, magnitude=magnitude, hparams=hparams) for name in transforms] + + +class RandAugment: + def __init__(self, ops, num_layers=2, choice_weights=None): + self.ops = ops + self.num_layers = num_layers + self.choice_weights = choice_weights + + def __call__(self, img): + # no replacement when using weighted choice + ops = np.random.choice( + self.ops, self.num_layers, replace=self.choice_weights is None, p=self.choice_weights) + for op in ops: + img = op(img) + return img + + +def rand_augment_transform(config_str, hparams): + """ + Create a RandAugment transform + + :param config_str: String defining configuration of random augmentation. Consists of multiple sections separated by + dashes ('-'). The first section defines the specific variant of rand augment (currently only 'rand'). The remaining + sections, not order sepecific determine + 'm' - integer magnitude of rand augment + 'n' - integer num layers (number of transform ops selected per image) + 'w' - integer probabiliy weight index (index of a set of weights to influence choice of op) + 'mstd' - float std deviation of magnitude noise applied + 'inc' - integer (bool), use augmentations that increase in severity with magnitude (default: 0) + Ex 'rand-m9-n3-mstd0.5' results in RandAugment with magnitude 9, num_layers 3, magnitude_std 0.5 + 'rand-mstd1-w0' results in magnitude_std 1.0, weights 0, default magnitude of 10 and num_layers 2 + + :param hparams: Other hparams (kwargs) for the RandAugmentation scheme + + :return: A PyTorch compatible Transform + """ + magnitude = _MAX_LEVEL # default to _MAX_LEVEL for magnitude (currently 10) + num_layers = 2 # default to 2 ops per image + weight_idx = None # default to no probability weights for op choice + transforms = _RAND_TRANSFORMS + config = config_str.split('-') + assert config[0] == 'rand' + config = config[1:] + for c in config: + cs = re.split(r'(\d.*)', c) + if len(cs) < 2: + continue + key, val = cs[:2] + if key == 'mstd': + # noise param injected via hparams for now + hparams.setdefault('magnitude_std', float(val)) + elif key == 'inc': + if bool(val): + transforms = _RAND_INCREASING_TRANSFORMS + elif key == 'm': + magnitude = int(val) + elif key == 'n': + num_layers = int(val) + elif key == 'w': + weight_idx = int(val) + else: + assert False, 'Unknown RandAugment config section' + ra_ops = rand_augment_ops(magnitude=magnitude, hparams=hparams, transforms=transforms) + choice_weights = None if weight_idx is None else _select_rand_weights(weight_idx) + return RandAugment(ra_ops, num_layers, choice_weights=choice_weights) + + +_AUGMIX_TRANSFORMS = [ + 'AutoContrast', + 'ColorIncreasing', # not in paper + 'ContrastIncreasing', # not in paper + 'BrightnessIncreasing', # not in paper + 'SharpnessIncreasing', # not in paper + 'Equalize', + 'Rotate', + 'PosterizeIncreasing', + 'SolarizeIncreasing', + 'ShearX', + 'ShearY', + 'TranslateXRel', + 'TranslateYRel', +] + + +def augmix_ops(magnitude=10, hparams=None, transforms=None): + hparams = hparams or _HPARAMS_DEFAULT + transforms = transforms or _AUGMIX_TRANSFORMS + return [AugmentOp( + name, prob=1.0, magnitude=magnitude, hparams=hparams) for name in transforms] + + +class AugMixAugment: + """ AugMix Transform + Adapted and improved from impl here: https://github.com/google-research/augmix/blob/master/imagenet.py + From paper: 'AugMix: A Simple Data Processing Method to Improve Robustness and Uncertainty - + https://arxiv.org/abs/1912.02781 + """ + def __init__(self, ops, alpha=1., width=3, depth=-1, blended=False): + self.ops = ops + self.alpha = alpha + self.width = width + self.depth = depth + self.blended = blended # blended mode is faster but not well tested + + def _calc_blended_weights(self, ws, m): + ws = ws * m + cump = 1. + rws = [] + for w in ws[::-1]: + alpha = w / cump + cump *= (1 - alpha) + rws.append(alpha) + return np.array(rws[::-1], dtype=np.float32) + + def _apply_blended(self, img, mixing_weights, m): + # This is my first crack and implementing a slightly faster mixed augmentation. Instead + # of accumulating the mix for each chain in a Numpy array and then blending with original, + # it recomputes the blending coefficients and applies one PIL image blend per chain. + # TODO the results appear in the right ballpark but they differ by more than rounding. + img_orig = img.copy() + ws = self._calc_blended_weights(mixing_weights, m) + for w in ws: + depth = self.depth if self.depth > 0 else np.random.randint(1, 4) + ops = np.random.choice(self.ops, depth, replace=True) + img_aug = img_orig # no ops are in-place, deep copy not necessary + for op in ops: + img_aug = op(img_aug) + img = Image.blend(img, img_aug, w) + return img + + def _apply_basic(self, img, mixing_weights, m): + # This is a literal adaptation of the paper/official implementation without normalizations and + # PIL <-> Numpy conversions between every op. It is still quite CPU compute heavy compared to the + # typical augmentation transforms, could use a GPU / Kornia implementation. + img_shape = img.size[0], img.size[1], len(img.getbands()) + mixed = np.zeros(img_shape, dtype=np.float32) + for mw in mixing_weights: + depth = self.depth if self.depth > 0 else np.random.randint(1, 4) + ops = np.random.choice(self.ops, depth, replace=True) + img_aug = img # no ops are in-place, deep copy not necessary + for op in ops: + img_aug = op(img_aug) + mixed += mw * np.asarray(img_aug, dtype=np.float32) + np.clip(mixed, 0, 255., out=mixed) + mixed = Image.fromarray(mixed.astype(np.uint8)) + return Image.blend(img, mixed, m) + + def __call__(self, img): + mixing_weights = np.float32(np.random.dirichlet([self.alpha] * self.width)) + m = np.float32(np.random.beta(self.alpha, self.alpha)) + if self.blended: + mixed = self._apply_blended(img, mixing_weights, m) + else: + mixed = self._apply_basic(img, mixing_weights, m) + return mixed + + +def augment_and_mix_transform(config_str, hparams): + """ Create AugMix PyTorch transform + + :param config_str: String defining configuration of random augmentation. Consists of multiple sections separated by + dashes ('-'). The first section defines the specific variant of rand augment (currently only 'rand'). The remaining + sections, not order sepecific determine + 'm' - integer magnitude (severity) of augmentation mix (default: 3) + 'w' - integer width of augmentation chain (default: 3) + 'd' - integer depth of augmentation chain (-1 is random [1, 3], default: -1) + 'b' - integer (bool), blend each branch of chain into end result without a final blend, less CPU (default: 0) + 'mstd' - float std deviation of magnitude noise applied (default: 0) + Ex 'augmix-m5-w4-d2' results in AugMix with severity 5, chain width 4, chain depth 2 + + :param hparams: Other hparams (kwargs) for the Augmentation transforms + + :return: A PyTorch compatible Transform + """ + magnitude = 3 + width = 3 + depth = -1 + alpha = 1. + blended = False + config = config_str.split('-') + assert config[0] == 'augmix' + config = config[1:] + for c in config: + cs = re.split(r'(\d.*)', c) + if len(cs) < 2: + continue + key, val = cs[:2] + if key == 'mstd': + # noise param injected via hparams for now + hparams.setdefault('magnitude_std', float(val)) + elif key == 'm': + magnitude = int(val) + elif key == 'w': + width = int(val) + elif key == 'd': + depth = int(val) + elif key == 'a': + alpha = float(val) + elif key == 'b': + blended = bool(val) + else: + assert False, 'Unknown AugMix config section' + ops = augmix_ops(magnitude=magnitude, hparams=hparams) + return AugMixAugment(ops, alpha=alpha, width=width, depth=depth, blended=blended) \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/model.py b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/model.py new file mode 100644 index 0000000..ced7b30 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/model.py @@ -0,0 +1,432 @@ +"""model.py - Model and module class for EfficientNet. + They are built to mirror those in the official TensorFlow implementation. +""" + +# Author: lukemelas (github username) +# Github repo: https://github.com/lukemelas/EfficientNet-PyTorch +# With adjustments and added comments by workingcoder (github username). + +import torch +from torch import nn +from torch.nn import functional as F +from .utils import ( + round_filters, + round_repeats, + drop_connect, + get_same_padding_conv2d, + get_model_params, + efficientnet_params, + load_pretrained_weights, + Swish, + MemoryEfficientSwish, + calculate_output_image_size +) + +class MBConvBlock(nn.Module): + """Mobile Inverted Residual Bottleneck Block. + + Args: + block_args (namedtuple): BlockArgs, defined in utils.py. + global_params (namedtuple): GlobalParam, defined in utils.py. + image_size (tuple or list): [image_height, image_width]. + + References: + [1] https://arxiv.org/abs/1704.04861 (MobileNet v1) + [2] https://arxiv.org/abs/1801.04381 (MobileNet v2) + [3] https://arxiv.org/abs/1905.02244 (MobileNet v3) + """ + + def __init__(self, block_args, global_params, image_size=None): + super().__init__() + self._block_args = block_args + self._bn_mom = 1 - global_params.batch_norm_momentum # pytorch's difference from tensorflow + self._bn_eps = global_params.batch_norm_epsilon + self.has_se = (self._block_args.se_ratio is not None) and (0 < self._block_args.se_ratio <= 1) + self.id_skip = block_args.id_skip # whether to use skip connection and drop connect + + # Expansion phase (Inverted Bottleneck) + inp = self._block_args.input_filters # number of input channels + oup = self._block_args.input_filters * self._block_args.expand_ratio # number of output channels + if self._block_args.expand_ratio != 1: + Conv2d = get_same_padding_conv2d(image_size=image_size) + self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias=False) + self._bn0 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps) + # image_size = calculate_output_image_size(image_size, 1) <-- this wouldn't modify image_size + + # Depthwise convolution phase + k = self._block_args.kernel_size + s = self._block_args.stride + Conv2d = get_same_padding_conv2d(image_size=image_size) + self._depthwise_conv = Conv2d( + in_channels=oup, out_channels=oup, groups=oup, # groups makes it depthwise + kernel_size=k, stride=s, bias=False) + self._bn1 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps) + image_size = calculate_output_image_size(image_size, s) + + # Squeeze and Excitation layer, if desired + if self.has_se: + Conv2d = get_same_padding_conv2d(image_size=(1,1)) + num_squeezed_channels = max(1, int(self._block_args.input_filters * self._block_args.se_ratio)) + self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1) + self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1) + # self._se_relu = torch.nn.ReLU() + # self._se_sigmoid = torch.nn.Sigmoid() + + # Pointwise convolution phase + final_oup = self._block_args.output_filters + Conv2d = get_same_padding_conv2d(image_size=image_size) + self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False) + self._bn2 = nn.BatchNorm2d(num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps) + self._swish = MemoryEfficientSwish() + + def forward(self, inputs, drop_connect_rate=None): + """MBConvBlock's forward function. + + Args: + inputs (tensor): Input tensor. + drop_connect_rate (bool): Drop connect rate (float, between 0 and 1). + + Returns: + Output of this block after processing. + """ + + # Expansion and Depthwise Convolution + x = inputs + if self._block_args.expand_ratio != 1: + x = self._expand_conv(inputs) + x = self._bn0(x) + x = self._swish(x) + + x = self._depthwise_conv(x) + x = self._bn1(x) + x = self._swish(x) + + # Squeeze and Excitation + if self.has_se: + x_squeezed = F.adaptive_avg_pool2d(x, 1) + # x_squeezed = torch.mean(x, [2, 3], keepdim=True) + + x_squeezed = self._se_reduce(x_squeezed) + + x_squeezed = self._swish(x_squeezed) + + x_squeezed = self._se_expand(x_squeezed) + + # x_squeezed = self._se_sigmoid(x_squeezed) + # + # x = x_squeezed * x + + x = torch.sigmoid(x_squeezed) * x + + # x = torch.sigmoid(x_squeezed) + x + # x = torch.nn.functional.relu(x_squeezed) * x + # x = x_squeezed + x + + # Pointwise Convolution + x = self._project_conv(x) + x = self._bn2(x) + + # Skip connection and drop connect + input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters + if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters: + # The combination of skip connection and drop connect brings about stochastic depth. + if drop_connect_rate: + x = drop_connect(x, p=drop_connect_rate, training=self.training) + x = x + inputs # skip connection + return x + + def set_swish(self, memory_efficient=True): + """Sets swish function as memory efficient (for training) or standard (for export). + + Args: + memory_efficient (bool): Whether to use memory-efficient version of swish. + """ + self._swish = MemoryEfficientSwish() if memory_efficient else Swish() + + +class EfficientNet(nn.Module): + """EfficientNet model. + Most easily loaded with the .from_name or .from_pretrained methods. + + Args: + blocks_args (list[namedtuple]): A list of BlockArgs to construct blocks. + global_params (namedtuple): A set of GlobalParams shared between blocks. + + References: + [1] https://arxiv.org/abs/1905.11946 (EfficientNet) + + Example: + >>> import torch + >>> from efficientnet.model import EfficientNet + >>> inputs = torch.rand(1, 3, 224, 224) + >>> model = EfficientNet.from_pretrained('efficientnet-b0') + >>> model.eval() + >>> outputs = model(inputs) + """ + + def __init__(self, blocks_args=None, global_params=None): + super().__init__() + assert isinstance(blocks_args, list), 'blocks_args should be a list' + assert len(blocks_args) > 0, 'block args must be greater than 0' + self._global_params = global_params + self._blocks_args = blocks_args + + # Batch norm parameters + bn_mom = 1 - self._global_params.batch_norm_momentum + bn_eps = self._global_params.batch_norm_epsilon + + # Get stem static or dynamic convolution depending on image size + image_size = global_params.image_size + Conv2d = get_same_padding_conv2d(image_size=image_size) + + # Stem + in_channels = 3 # rgb + out_channels = round_filters(32, self._global_params) # number of output channels + self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False) + self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps) + image_size = calculate_output_image_size(image_size, 2) + + # Build blocks + self._blocks = nn.ModuleList([]) + for block_args in self._blocks_args: + + # Update block input and output filters based on depth multiplier. + block_args = block_args._replace( + input_filters=round_filters(block_args.input_filters, self._global_params), + output_filters=round_filters(block_args.output_filters, self._global_params), + num_repeat=round_repeats(block_args.num_repeat, self._global_params) + ) + + # The first block needs to take care of stride and filter size increase. + self._blocks.append(MBConvBlock(block_args, self._global_params, image_size=image_size)) + image_size = calculate_output_image_size(image_size, block_args.stride) + if block_args.num_repeat > 1: # modify block_args to keep same output size + block_args = block_args._replace(input_filters=block_args.output_filters, stride=1) + for _ in range(block_args.num_repeat - 1): + self._blocks.append(MBConvBlock(block_args, self._global_params, image_size=image_size)) + # image_size = calculate_output_image_size(image_size, block_args.stride) # stride = 1 + + # Head + in_channels = block_args.output_filters # output of final block + out_channels = round_filters(1280, self._global_params) + Conv2d = get_same_padding_conv2d(image_size=image_size) + self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False) + self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps) + + # Final linear layer + self._avg_pooling = nn.AdaptiveAvgPool2d(1) + self._dropout = nn.Dropout(self._global_params.dropout_rate) + self._fc = nn.Linear(out_channels, self._global_params.num_classes) + self._swish = MemoryEfficientSwish() + + def set_swish(self, memory_efficient=True): + """Sets swish function as memory efficient (for training) or standard (for export). + + Args: + memory_efficient (bool): Whether to use memory-efficient version of swish. + + """ + self._swish = MemoryEfficientSwish() if memory_efficient else Swish() + for block in self._blocks: + block.set_swish(memory_efficient) + + def extract_endpoints(self, inputs): + """Use convolution layer to extract features + from reduction levels i in [1, 2, 3, 4, 5]. + + Args: + inputs (tensor): Input tensor. + + Returns: + Dictionary of last intermediate features + with reduction levels i in [1, 2, 3, 4, 5]. + Example: + >>> import torch + >>> from efficientnet.model import EfficientNet + >>> inputs = torch.rand(1, 3, 224, 224) + >>> model = EfficientNet.from_pretrained('efficientnet-b0') + >>> endpoints = model.extract_features(inputs) + >>> print(endpoints['reduction_1'].shape) # torch.Size([1, 16, 112, 112]) + >>> print(endpoints['reduction_2'].shape) # torch.Size([1, 24, 56, 56]) + >>> print(endpoints['reduction_3'].shape) # torch.Size([1, 40, 28, 28]) + >>> print(endpoints['reduction_4'].shape) # torch.Size([1, 112, 14, 14]) + >>> print(endpoints['reduction_5'].shape) # torch.Size([1, 1280, 7, 7]) + """ + endpoints = dict() + + # Stem + x = self._swish(self._bn0(self._conv_stem(inputs))) + # x = self._swish(self._conv_stem(inputs)) + prev_x = x + + # Blocks + for idx, block in enumerate(self._blocks): + drop_connect_rate = self._global_params.drop_connect_rate + if drop_connect_rate: + drop_connect_rate *= float(idx) / len(self._blocks) # scale drop connect_rate + x = block(x, drop_connect_rate=drop_connect_rate) + if prev_x.size(2) > x.size(2): + endpoints[f'reduction_{len(endpoints)+1}'] = prev_x + prev_x = x + + # Head + x = self._swish(self._bn1(self._conv_head(x))) + # x = self._swish(self._conv_head(x)) + endpoints[f'reduction_{len(endpoints)+1}'] = x + + return endpoints + + def extract_features(self, inputs): + """use convolution layer to extract feature . + + Args: + inputs (tensor): Input tensor. + + Returns: + Output of the final convolution + layer in the efficientnet model. + """ + # Stem + x = self._swish(self._bn0(self._conv_stem(inputs))) + # x = self._swish(self._conv_stem(inputs)) + + # Blocks + for idx, block in enumerate(self._blocks): + drop_connect_rate = self._global_params.drop_connect_rate + if drop_connect_rate: + drop_connect_rate *= float(idx) / len(self._blocks) # scale drop connect_rate + x = block(x, drop_connect_rate=drop_connect_rate) + + # Head + x = self._swish(self._bn1(self._conv_head(x))) + # x = self._swish(self._conv_head(x)) + + return x + + def forward(self, inputs): + """EfficientNet's forward function. + Calls extract_features to extract features, applies final linear layer, and returns logits. + + Args: + inputs (tensor): Input tensor. + + Returns: + Output of this model after processing. + """ + bs = inputs.size(0) + + # Convolution layers + x = self.extract_features(inputs) + + # Pooling and final linear layer + x = self._avg_pooling(x) + # x = x.view(bs, -1) + x = torch.flatten(x, start_dim=1) + # x = self._dropout(x.to('cpu')) + # x = self._fc(x.to('npu:5')) + x = self._dropout(x) + x = self._fc(x) + + return x + + @classmethod + def from_name(cls, model_name, in_channels=3, **override_params): + """create an efficientnet model according to name. + + Args: + model_name (str): Name for efficientnet. + in_channels (int): Input data's channel number. + override_params (other key word params): + Params to override model's global_params. + Optional key: + 'width_coefficient', 'depth_coefficient', + 'image_size', 'dropout_rate', + 'num_classes', 'batch_norm_momentum', + 'batch_norm_epsilon', 'drop_connect_rate', + 'depth_divisor', 'min_depth' + + Returns: + An efficientnet model. + """ + cls._check_model_name_is_valid(model_name) + blocks_args, global_params = get_model_params(model_name, override_params) + model = cls(blocks_args, global_params) + model._change_in_channels(in_channels) + return model + + @classmethod + def from_pretrained(cls, model_name, weights_path=None, advprop=False, + in_channels=3, num_classes=1000, **override_params): + """create an efficientnet model according to name. + + Args: + model_name (str): Name for efficientnet. + weights_path (None or str): + str: path to pretrained weights file on the local disk. + None: use pretrained weights downloaded from the Internet. + advprop (bool): + Whether to load pretrained weights + trained with advprop (valid when weights_path is None). + in_channels (int): Input data's channel number. + num_classes (int): + Number of categories for classification. + It controls the output size for final linear layer. + override_params (other key word params): + Params to override model's global_params. + Optional key: + 'width_coefficient', 'depth_coefficient', + 'image_size', 'dropout_rate', + 'num_classes', 'batch_norm_momentum', + 'batch_norm_epsilon', 'drop_connect_rate', + 'depth_divisor', 'min_depth' + + Returns: + A pretrained efficientnet model. + """ + model = cls.from_name(model_name, num_classes = num_classes, **override_params) + load_pretrained_weights(model, model_name, weights_path=weights_path, load_fc=(num_classes == 1000), advprop=advprop) + model._change_in_channels(in_channels) + return model + + @classmethod + def get_image_size(cls, model_name): + """Get the input image size for a given efficientnet model. + + Args: + model_name (str): Name for efficientnet. + + Returns: + Input image size (resolution). + """ + cls._check_model_name_is_valid(model_name) + _, _, res, _ = efficientnet_params(model_name) + return res + + @classmethod + def _check_model_name_is_valid(cls, model_name): + """Validates model name. + + Args: + model_name (str): Name for efficientnet. + + Returns: + bool: Is a valid name or not. + """ + valid_models = ['efficientnet-b'+str(i) for i in range(9)] + + # Support the construction of 'efficientnet-l2' without pretrained weights + valid_models += ['efficientnet-l2'] + + if model_name not in valid_models: + raise ValueError('model_name should be one of: ' + ', '.join(valid_models)) + + def _change_in_channels(self, in_channels): + """Adjust model's first convolution layer to in_channels, if in_channels not equals 3. + + Args: + in_channels (int): Input data's channel number. + """ + if in_channels != 3: + Conv2d = get_same_padding_conv2d(image_size = self._global_params.image_size) + out_channels = round_filters(32, self._global_params) + self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False) diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/npu_info.py b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/npu_info.py new file mode 100644 index 0000000..e9e6775 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/npu_info.py @@ -0,0 +1,7 @@ +def set_value(value): + global _npu_id + _npu_id = value + print('set device id %s success'%_npu_id) + +def get_value(): + return _npu_id \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/rmsprop_tf.py b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/rmsprop_tf.py new file mode 100644 index 0000000..5557961 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/rmsprop_tf.py @@ -0,0 +1,122 @@ +import torch +from torch.optim import Optimizer + + +class RMSpropTF(Optimizer): + """Implements RMSprop algorithm (TensorFlow style epsilon) + + NOTE: This is a direct cut-and-paste of PyTorch RMSprop with eps applied before sqrt + to closer match Tensorflow for matching hyper-params. + + Proposed by G. Hinton in his + `course `_. + + The centered version first appears in `Generating Sequences + With Recurrent Neural Networks `_. + + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float, optional): learning rate (default: 1e-2) + momentum (float, optional): momentum factor (default: 0) + alpha (float, optional): smoothing (decay) constant (default: 0.9) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-10) + centered (bool, optional) : if ``True``, compute the centered RMSProp, + the gradient is normalized by an estimation of its variance + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + decoupled_decay (bool, optional): decoupled weight decay as per https://arxiv.org/abs/1711.05101 + lr_in_momentum (bool, optional): learning rate scaling is included in the momentum buffer + update as per defaults in Tensorflow + + """ + + def __init__(self, params, lr=1e-2, alpha=0.9, eps=1e-10, weight_decay=0, momentum=0., centered=False, + decoupled_decay=False, lr_in_momentum=True): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {}".format(eps)) + if not 0.0 <= momentum: + raise ValueError("Invalid momentum value: {}".format(momentum)) + if not 0.0 <= weight_decay: + raise ValueError("Invalid weight_decay value: {}".format(weight_decay)) + if not 0.0 <= alpha: + raise ValueError("Invalid alpha value: {}".format(alpha)) + + defaults = dict(lr=lr, momentum=momentum, alpha=alpha, eps=eps, centered=centered, weight_decay=weight_decay, + decoupled_decay=decoupled_decay, lr_in_momentum=lr_in_momentum) + super(RMSpropTF, self).__init__(params, defaults) + + def __setstate__(self, state): + super(RMSpropTF, self).__setstate__(state) + for group in self.param_groups: + group.setdefault('momentum', 0) + group.setdefault('centered', False) + + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data + if grad.is_sparse: + raise RuntimeError('RMSprop does not support sparse gradients') + state = self.state[p] + + # State initialization + if len(state) == 0: + state['step'] = 0 + state['square_avg'] = torch.ones_like(p.data) # PyTorch inits to zero + if group['momentum'] > 0: + state['momentum_buffer'] = torch.zeros_like(p.data) + if group['centered']: + state['grad_avg'] = torch.zeros_like(p.data) + + square_avg = state['square_avg'] + one_minus_alpha = 1. - group['alpha'] + + state['step'] += 1 + + if group['weight_decay'] != 0: + if 'decoupled_decay' in group and group['decoupled_decay']: + p.data.add_(-group['weight_decay'], p.data) + else: + grad = grad.add(group['weight_decay'], p.data) + + # Tensorflow order of ops for updating squared avg + square_avg.add_(one_minus_alpha, grad.pow(2) - square_avg) + # square_avg.mul_(alpha).addcmul_(1 - alpha, grad, grad) # PyTorch original + + if group['centered']: + grad_avg = state['grad_avg'] + grad_avg.add_(one_minus_alpha, grad - grad_avg) + # grad_avg.mul_(alpha).add_(1 - alpha, grad) # PyTorch original + avg = square_avg.addcmul(-1, grad_avg, grad_avg).add(group['eps']).sqrt_() # eps moved in sqrt + else: + avg = square_avg.add(group['eps']).sqrt_() # eps moved in sqrt + + if group['momentum'] > 0: + buf = state['momentum_buffer'] + # Tensorflow accumulates the LR scaling in the momentum buffer + if 'lr_in_momentum' in group and group['lr_in_momentum']: + buf.mul_(group['momentum']).addcdiv_(group['lr'], grad, avg) + p.data.add_(-buf) + else: + # PyTorch scales the param update by LR + buf.mul_(group['momentum']).addcdiv_(grad, avg) + p.data.add_(-group['lr'], buf) + else: + p.data.addcdiv_(-group['lr'], grad, avg) + + return loss \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/utils.py b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/utils.py new file mode 100644 index 0000000..d5285d7 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/efficientnet_pytorch/utils.py @@ -0,0 +1,624 @@ +"""utils.py - Helper functions for building the model and for loading model parameters. + These helper functions are built to mirror those in the official TensorFlow implementation. +""" + +# Author: lukemelas (github username) +# Github repo: https://github.com/lukemelas/EfficientNet-PyTorch +# With adjustments and added comments by workingcoder (github username). + +import re +import math +import collections +from functools import partial +import numpy as np +import torch +from torch import nn +from torch.nn import functional as F +from torch.utils import model_zoo +from . import npu_info + +################################################################################ +### Help functions for model architecture +################################################################################ + +# GlobalParams and BlockArgs: Two namedtuples +# Swish and MemoryEfficientSwish: Two implementations of the method +# round_filters and round_repeats: +# Functions to calculate params for scaling model width and depth ! ! ! +# get_width_and_height_from_size and calculate_output_image_size +# drop_connect: A structural design +# get_same_padding_conv2d: +# Conv2dDynamicSamePadding +# Conv2dStaticSamePadding +# get_same_padding_maxPool2d: +# MaxPool2dDynamicSamePadding +# MaxPool2dStaticSamePadding +# It's an additional function, not used in EfficientNet, +# but can be used in other model (such as EfficientDet). +# Identity: An implementation of identical mapping + +# Parameters for the entire model (stem, all blocks, and head) +GlobalParams = collections.namedtuple('GlobalParams', [ + 'width_coefficient', 'depth_coefficient', 'image_size', 'dropout_rate', + 'num_classes', 'batch_norm_momentum', 'batch_norm_epsilon', + 'drop_connect_rate', 'depth_divisor', 'min_depth']) + +# Parameters for an individual model block +BlockArgs = collections.namedtuple('BlockArgs', [ + 'num_repeat', 'kernel_size', 'stride', 'expand_ratio', + 'input_filters', 'output_filters', 'se_ratio', 'id_skip']) + +# Set GlobalParams and BlockArgs's defaults +GlobalParams.__new__.__defaults__ = (None,) * len(GlobalParams._fields) +BlockArgs.__new__.__defaults__ = (None,) * len(BlockArgs._fields) + + +# An ordinary implementation of Swish function +class Swish(nn.Module): + def forward(self, x): + return x * torch.sigmoid(x) + +# A memory-efficient implementation of Swish function +class SwishImplementation(torch.autograd.Function): + @staticmethod + def forward(ctx, i): + result = i * torch.sigmoid(i) + ctx.save_for_backward(i) + return result + + @staticmethod + def backward(ctx, grad_output): + i = ctx.saved_tensors[0] + sigmoid_i = torch.sigmoid(i) + return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i))) + +class MemoryEfficientSwish(nn.Module): + def forward(self, x): + return SwishImplementation.apply(x) + + +def round_filters(filters, global_params): + """Calculate and round number of filters based on width multiplier. + Use width_coefficient, depth_divisor and min_depth of global_params. + + Args: + filters (int): Filters number to be calculated. + global_params (namedtuple): Global params of the model. + + Returns: + new_filters: New filters number after calculating. + """ + multiplier = global_params.width_coefficient + if not multiplier: + return filters + # TODO: modify the params names. + # maybe the names (width_divisor,min_width) + # are more suitable than (depth_divisor,min_depth). + divisor = global_params.depth_divisor + min_depth = global_params.min_depth + filters *= multiplier + min_depth = min_depth or divisor # pay attention to this line when using min_depth + # follow the formula transferred from official TensorFlow implementation + new_filters = max(min_depth, int(filters + divisor / 2) // divisor * divisor) + if new_filters < 0.9 * filters: # prevent rounding by more than 10% + new_filters += divisor + return int(new_filters) + + +def round_repeats(repeats, global_params): + """Calculate module's repeat number of a block based on depth multiplier. + Use depth_coefficient of global_params. + + Args: + repeats (int): num_repeat to be calculated. + global_params (namedtuple): Global params of the model. + + Returns: + new repeat: New repeat number after calculating. + """ + multiplier = global_params.depth_coefficient + if not multiplier: + return repeats + # follow the formula transferred from official TensorFlow implementation + return int(math.ceil(multiplier * repeats)) + + +def drop_connect(inputs, p, training): + """Drop connect. + + Args: + input (tensor: BCWH): Input of this structure. + p (float: 0.0~1.0): Probability of drop connection. + training (bool): The running mode. + + Returns: + output: Output after drop connection. + """ + assert p >= 0 and p <= 1, 'p must be in range of [0,1]' + + if not training: + return inputs + + batch_size = inputs.shape[0] + keep_prob = 1 - p + + # generate binary_tensor mask according to probability (p for 0, 1-p for 1) + random_tensor = keep_prob + random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device) + binary_tensor = torch.floor(random_tensor) / keep_prob + + output = inputs * binary_tensor + return output + + +def get_width_and_height_from_size(x): + """Obtain height and width from x. + + Args: + x (int, tuple or list): Data size. + + Returns: + size: A tuple or list (H,W). + """ + if isinstance(x, int): + return x, x + if isinstance(x, list) or isinstance(x, tuple): + return x + else: + raise TypeError() + + +def calculate_output_image_size(input_image_size, stride): + """Calculates the output image size when using Conv2dSamePadding with a stride. + Necessary for static padding. Thanks to mannatsingh for pointing this out. + + Args: + input_image_size (int, tuple or list): Size of input image. + stride (int, tuple or list): Conv2d operation's stride. + + Returns: + output_image_size: A list [H,W]. + """ + if input_image_size is None: + return None + image_height, image_width = get_width_and_height_from_size(input_image_size) + stride = stride if isinstance(stride, int) else stride[0] + image_height = int(math.ceil(image_height / stride)) + image_width = int(math.ceil(image_width / stride)) + return [image_height, image_width] + + +# Note: +# The following 'SamePadding' functions make output size equal ceil(input size/stride). +# Only when stride equals 1, can the output size be the same as input size. +# Don't be confused by their function names ! ! ! + +def get_same_padding_conv2d(image_size=None): + """Chooses static padding if you have specified an image size, and dynamic padding otherwise. + Static padding is necessary for ONNX exporting of models. + + Args: + image_size (int or tuple): Size of the image. + + Returns: + Conv2dDynamicSamePadding or Conv2dStaticSamePadding. + """ + if image_size is None: + return Conv2dDynamicSamePadding + else: + return partial(Conv2dStaticSamePadding, image_size=image_size) + + +class Conv2dDynamicSamePadding(nn.Conv2d): + """2D Convolutions like TensorFlow, for a dynamic image size. + The padding is operated in forward function by calculating dynamically. + """ + + # Tips for 'SAME' mode padding. + # Given the following: + # i: width or height + # s: stride + # k: kernel size + # d: dilation + # p: padding + # Output after Conv2d: + # o = floor((i+p-((k-1)*d+1))/s+1) + # If o equals i, i = floor((i+p-((k-1)*d+1))/s+1), + # => p = (i-1)*s+((k-1)*d+1)-i + + def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True): + super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias) + self.stride = self.stride if len(self.stride) == 2 else [self.stride[0]] * 2 + + def forward(self, x): + ih, iw = x.size()[-2:] + kh, kw = self.weight.size()[-2:] + sh, sw = self.stride + oh, ow = math.ceil(ih / sh), math.ceil(iw / sw) # change the output size according to stride ! ! ! + pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0) + pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0) + if pad_h > 0 or pad_w > 0: + x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2]) + return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups) + + +class Conv2dStaticSamePadding(nn.Conv2d): + """2D Convolutions like TensorFlow's 'SAME' mode, with the given input image size. + The padding mudule is calculated in construction function, then used in forward. + """ + + # With the same calculation as Conv2dDynamicSamePadding + + def __init__(self, in_channels, out_channels, kernel_size, stride=1, image_size=None, **kwargs): + super().__init__(in_channels, out_channels, kernel_size, stride, **kwargs) + self.stride = self.stride if len(self.stride) == 2 else [self.stride[0]] * 2 + + # Calculate padding based on image size and save it + assert image_size is not None + ih, iw = (image_size, image_size) if isinstance(image_size, int) else image_size + kh, kw = self.weight.size()[-2:] + sh, sw = self.stride + oh, ow = math.ceil(ih / sh), math.ceil(iw / sw) + pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0) + pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0) + if pad_h > 0 or pad_w > 0: + self.static_padding = nn.ZeroPad2d((pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2)) + if kh % 2 != 0: + self.padding = (kh - 1) // 2 + else: + self.padding = kh // 2 + else: + self.static_padding = Identity() + + def forward(self, x): + x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups) + return x + + +def get_same_padding_maxPool2d(image_size=None): + """Chooses static padding if you have specified an image size, and dynamic padding otherwise. + Static padding is necessary for ONNX exporting of models. + + Args: + image_size (int or tuple): Size of the image. + + Returns: + MaxPool2dDynamicSamePadding or MaxPool2dStaticSamePadding. + """ + if image_size is None: + return MaxPool2dDynamicSamePadding + else: + return partial(MaxPool2dStaticSamePadding, image_size=image_size) + + +class MaxPool2dDynamicSamePadding(nn.MaxPool2d): + """2D MaxPooling like TensorFlow's 'SAME' mode, with a dynamic image size. + The padding is operated in forward function by calculating dynamically. + """ + + def __init__(self, kernel_size, stride, padding=0, dilation=1, return_indices=False, ceil_mode=False): + super().__init__(kernel_size, stride, padding, dilation, return_indices, ceil_mode) + self.stride = [self.stride] * 2 if isinstance(self.stride, int) else self.stride + self.kernel_size = [self.kernel_size] * 2 if isinstance(self.kernel_size, int) else self.kernel_size + self.dilation = [self.dilation] * 2 if isinstance(self.dilation, int) else self.dilation + + def forward(self, x): + ih, iw = x.size()[-2:] + kh, kw = self.kernel_size + sh, sw = self.stride + oh, ow = math.ceil(ih / sh), math.ceil(iw / sw) + pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0) + pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0) + if pad_h > 0 or pad_w > 0: + x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2]) + return F.max_pool2d(x, self.kernel_size, self.stride, self.padding, + self.dilation, self.ceil_mode, self.return_indices) + +class MaxPool2dStaticSamePadding(nn.MaxPool2d): + """2D MaxPooling like TensorFlow's 'SAME' mode, with the given input image size. + The padding mudule is calculated in construction function, then used in forward. + """ + + def __init__(self, kernel_size, stride, image_size=None, **kwargs): + super().__init__(kernel_size, stride, **kwargs) + self.stride = [self.stride] * 2 if isinstance(self.stride, int) else self.stride + self.kernel_size = [self.kernel_size] * 2 if isinstance(self.kernel_size, int) else self.kernel_size + self.dilation = [self.dilation] * 2 if isinstance(self.dilation, int) else self.dilation + + # Calculate padding based on image size and save it + assert image_size is not None + ih, iw = (image_size, image_size) if isinstance(image_size, int) else image_size + kh, kw = self.kernel_size + sh, sw = self.stride + oh, ow = math.ceil(ih / sh), math.ceil(iw / sw) + pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0) + pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0) + if pad_h > 0 or pad_w > 0: + self.static_padding = nn.ZeroPad2d((pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2)) + else: + self.static_padding = Identity() + + def forward(self, x): + x = self.static_padding(x) + x = F.max_pool2d(x, self.kernel_size, self.stride, self.padding, + self.dilation, self.ceil_mode, self.return_indices) + return x + +class Identity(nn.Module): + """Identity mapping. + Send input to output directly. + """ + + def __init__(self): + super(Identity, self).__init__() + + def forward(self, input): + return input + + +################################################################################ +### Helper functions for loading model params +################################################################################ + +# BlockDecoder: A Class for encoding and decoding BlockArgs +# efficientnet_params: A function to query compound coefficient +# get_model_params and efficientnet: +# Functions to get BlockArgs and GlobalParams for efficientnet +# url_map and url_map_advprop: Dicts of url_map for pretrained weights +# load_pretrained_weights: A function to load pretrained weights + +class BlockDecoder(object): + """Block Decoder for readability, + straight from the official TensorFlow repository. + """ + + @staticmethod + def _decode_block_string(block_string): + """Get a block through a string notation of arguments. + + Args: + block_string (str): A string notation of arguments. + Examples: 'r1_k3_s11_e1_i32_o16_se0.25_noskip'. + + Returns: + BlockArgs: The namedtuple defined at the top of this file. + """ + assert isinstance(block_string, str) + + ops = block_string.split('_') + options = {} + for op in ops: + splits = re.split(r'(\d.*)', op) + if len(splits) >= 2: + key, value = splits[:2] + options[key] = value + + # Check stride + assert (('s' in options and len(options['s']) == 1) or + (len(options['s']) == 2 and options['s'][0] == options['s'][1])) + + return BlockArgs( + num_repeat=int(options['r']), + kernel_size=int(options['k']), + stride=[int(options['s'][0])], + expand_ratio=int(options['e']), + input_filters=int(options['i']), + output_filters=int(options['o']), + se_ratio=float(options['se']) if 'se' in options else None, + id_skip=('noskip' not in block_string)) + + @staticmethod + def _encode_block_string(block): + """Encode a block to a string. + + Args: + block (namedtuple): A BlockArgs type argument. + + Returns: + block_string: A String form of BlockArgs. + """ + args = [ + 'r%d' % block.num_repeat, + 'k%d' % block.kernel_size, + 's%d%d' % (block.strides[0], block.strides[1]), + 'e%s' % block.expand_ratio, + 'i%d' % block.input_filters, + 'o%d' % block.output_filters + ] + if 0 < block.se_ratio <= 1: + args.append('se%s' % block.se_ratio) + if block.id_skip is False: + args.append('noskip') + return '_'.join(args) + + @staticmethod + def decode(string_list): + """Decode a list of string notations to specify blocks inside the network. + + Args: + string_list (list[str]): A list of strings, each string is a notation of block. + + Returns: + blocks_args: A list of BlockArgs namedtuples of block args. + """ + assert isinstance(string_list, list) + blocks_args = [] + for block_string in string_list: + blocks_args.append(BlockDecoder._decode_block_string(block_string)) + return blocks_args + + @staticmethod + def encode(blocks_args): + """Encode a list of BlockArgs to a list of strings. + + Args: + blocks_args (list[namedtuples]): A list of BlockArgs namedtuples of block args. + + Returns: + block_strings: A list of strings, each string is a notation of block. + """ + block_strings = [] + for block in blocks_args: + block_strings.append(BlockDecoder._encode_block_string(block)) + return block_strings + + +def efficientnet_params(model_name): + """Map EfficientNet model name to parameter coefficients. + + Args: + model_name (str): Model name to be queried. + + Returns: + params_dict[model_name]: A (width,depth,res,dropout) tuple. + """ + params_dict = { + # Coefficients: width,depth,res,dropout + 'efficientnet-b0': (1.0, 1.0, 224, 0.2), + 'efficientnet-b1': (1.0, 1.1, 240, 0.2), + 'efficientnet-b2': (1.1, 1.2, 260, 0.3), + 'efficientnet-b3': (1.2, 1.4, 300, 0.3), + 'efficientnet-b4': (1.4, 1.8, 380, 0.4), + 'efficientnet-b5': (1.6, 2.2, 456, 0.4), + 'efficientnet-b6': (1.8, 2.6, 528, 0.5), + 'efficientnet-b7': (2.0, 3.1, 600, 0.5), + 'efficientnet-b8': (2.2, 3.6, 672, 0.5), + 'efficientnet-l2': (4.3, 5.3, 800, 0.5), + } + return params_dict[model_name] + + +def efficientnet(width_coefficient=None, depth_coefficient=None, image_size=None, + dropout_rate=0.2, drop_connect_rate=0.2, num_classes=1000): + """Create BlockArgs and GlobalParams for efficientnet model. + + Args: + width_coefficient (float) + depth_coefficient (float) + image_size (int) + dropout_rate (float) + drop_connect_rate (float) + num_classes (int) + + Meaning as the name suggests. + + Returns: + blocks_args, global_params. + """ + + # Blocks args for the whole model(efficientnet-b0 by default) + # It will be modified in the construction of EfficientNet Class according to model + blocks_args = [ + 'r1_k3_s11_e1_i32_o16_se0.25', + 'r2_k3_s22_e6_i16_o24_se0.25', + 'r2_k5_s22_e6_i24_o40_se0.25', + 'r3_k3_s22_e6_i40_o80_se0.25', + 'r3_k5_s11_e6_i80_o112_se0.25', + 'r4_k5_s22_e6_i112_o192_se0.25', + 'r1_k3_s11_e6_i192_o320_se0.25', + ] + + blocks_args = BlockDecoder.decode(blocks_args) + + global_params = GlobalParams( + width_coefficient=width_coefficient, + depth_coefficient=depth_coefficient, + image_size=image_size, + dropout_rate=dropout_rate, + + num_classes=num_classes, + batch_norm_momentum=0.99, + batch_norm_epsilon=1e-3, + drop_connect_rate=drop_connect_rate, + depth_divisor=8, + min_depth=None, + ) + + return blocks_args, global_params + + +def get_model_params(model_name, override_params): + """Get the block args and global params for a given model name. + + Args: + model_name (str): Model's name. + override_params (dict): A dict to modify global_params. + + Returns: + blocks_args, global_params + """ + if model_name.startswith('efficientnet'): + w, d, s, p = efficientnet_params(model_name) + # note: all models have drop connect rate = 0.2 + blocks_args, global_params = efficientnet( + width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s) + else: + raise NotImplementedError('model name is not pre-defined: %s' % model_name) + if override_params: + # ValueError will be raised here if override_params has fields not included in global_params. + global_params = global_params._replace(**override_params) + return blocks_args, global_params + + +# train with Standard methods +# check more details in paper(EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks) +url_map = { + 'efficientnet-b0': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b0-355c32eb.pth', + 'efficientnet-b1': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b1-f1951068.pth', + 'efficientnet-b2': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b2-8bb594d6.pth', + 'efficientnet-b3': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b3-5fb5a3c3.pth', + 'efficientnet-b4': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b4-6ed6700e.pth', + 'efficientnet-b5': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b5-b6417697.pth', + 'efficientnet-b6': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b6-c76e70fd.pth', + 'efficientnet-b7': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b7-dcc49843.pth', +} + +# train with Adversarial Examples(AdvProp) +# check more details in paper(Adversarial Examples Improve Image Recognition) +url_map_advprop = { + 'efficientnet-b0': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b0-b64d5a18.pth', + 'efficientnet-b1': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b1-0f3ce85a.pth', + 'efficientnet-b2': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b2-6e9d97e5.pth', + 'efficientnet-b3': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b3-cdd7c0f4.pth', + 'efficientnet-b4': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b4-44fb3a87.pth', + 'efficientnet-b5': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b5-86493f6b.pth', + 'efficientnet-b6': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b6-ac80338e.pth', + 'efficientnet-b7': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b7-4652b6dd.pth', + 'efficientnet-b8': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b8-22a8fe65.pth', +} + +# TODO: add the petrained weights url map of 'efficientnet-l2' + + +def load_pretrained_weights(model, model_name, weights_path=None, load_fc=True, advprop=False): + """Loads pretrained weights from weights path or download using url. + + Args: + model (Module): The whole model of efficientnet. + model_name (str): Model name of efficientnet. + weights_path (None or str): + str: path to pretrained weights file on the local disk. + None: use pretrained weights downloaded from the Internet. + load_fc (bool): Whether to load pretrained weights for fc layer at the end of the model. + advprop (bool): Whether to load pretrained weights + trained with advprop (valid when weights_path is None). + """ + if isinstance(weights_path,str): + state_dict = torch.load(weights_path) + else: + # AutoAugment or Advprop (different preprocessing) + url_map_ = url_map_advprop if advprop else url_map + state_dict = model_zoo.load_url(url_map_[model_name]) + + if load_fc: + ret = model.load_state_dict(state_dict, strict=False) + assert not ret.missing_keys, f'Missing keys when loading pretrained weights: {ret.missing_keys}' + else: + state_dict.pop('_fc.weight') + state_dict.pop('_fc.bias') + ret = model.load_state_dict(state_dict, strict=False) + assert set(ret.missing_keys) == set( + ['_fc.weight', '_fc.bias']), f'Missing keys when loading pretrained weights: {ret.missing_keys}' + assert not ret.unexpected_keys, f'Missing keys when loading pretrained weights: {ret.unexpected_keys}' + + print('Loaded pretrained weights for {}'.format(model_name)) diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/imagenet/README.md b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/imagenet/README.md new file mode 100644 index 0000000..fcafce3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/imagenet/README.md @@ -0,0 +1,23 @@ +### Imagenet + +This is a preliminary directory for evaluating the model on ImageNet. It is adapted from the standard PyTorch Imagenet script. + +For now, only evaluation is supported, but I am currently building scripts to assist with training new models on Imagenet. + +The evaluation results are slightly different from the original TensorFlow repository, due to differences in data preprocessing. For example, with the current preprocessing, `efficientnet-b3` gives a top-1 accuracy of `80.8`, rather than `81.1` in the paper. I am working on porting the TensorFlow preprocessing into PyTorch to address this issue. + +To run on Imagenet, place your `train` and `val` directories in `data`. + +Example commands: +```bash +# Evaluate small EfficientNet on CPU +python main.py data -e -a 'efficientnet-b0' --pretrained +``` +```bash +# Evaluate medium EfficientNet on GPU +python main.py data -e -a 'efficientnet-b3' --pretrained --gpu 0 --batch-size 128 +``` +```bash +# Evaluate ResNet-50 for comparison +python main.py data -e -a 'resnet50' --pretrained --gpu 0 +``` diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/imagenet/data/README.md b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/imagenet/data/README.md new file mode 100644 index 0000000..310c6e0 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/imagenet/data/README.md @@ -0,0 +1,5 @@ +### ImageNet + +Download ImageNet and place it into `train` and `val` folders here. + +More details may be found with the official PyTorch ImageNet example [here](https://github.com/pytorch/examples/blob/master/imagenet). diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/imagenet/main.py b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/imagenet/main.py new file mode 100644 index 0000000..9a0bfad --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/imagenet/main.py @@ -0,0 +1,531 @@ +""" +Evaluate on ImageNet. Note that at the moment, training is not implemented (I am working on it). +that being said, evaluation is working. +""" + +import argparse +import os +import sys +import random +import shutil +import time +import warnings +import PIL +import numpy as np + +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.distributed as dist +import torch.optim +import torch.multiprocessing as mp +import torch.utils.data +import torch.utils.data.distributed +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torchvision.models as models + +from apex import amp + +sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)),'../../')) +from efficientnet_pytorch import EfficientNet +from efficientnet_pytorch import rand_augment_transform, augment_and_mix_transform, auto_augment_transform +from efficientnet_pytorch import RMSpropTF +from efficientnet_pytorch import npu_info + +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter + +parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') +parser.add_argument('--data', metavar='DIR', + help='path to dataset') +parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet18', + help='model architecture (default: resnet18)') +parser.add_argument('-j', '--workers', default=128, type=int, metavar='N', + help='number of data loading workers (default: 4)') +parser.add_argument('--epochs', default=90, type=int, metavar='N', + help='number of total epochs to run') +parser.add_argument('--start-epoch', default=0, type=int, metavar='N', + help='manual epoch number (useful on restarts)') +parser.add_argument('-b', '--batch-size', default=256, type=int, + metavar='N', + help='mini-batch size (default: 256), this is the total ' + 'batch size of all GPUs on the current node when ' + 'using Data Parallel or Distributed Data Parallel') +parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, + metavar='LR', help='initial learning rate', dest='lr') +parser.add_argument('--momentum', default=0.9, type=float, metavar='M', + help='momentum') +parser.add_argument('--wd', '--weight-decay', default=1e-5, type=float, + metavar='W', help='weight decay (default: 1e-4)', + dest='weight_decay') +parser.add_argument('-p', '--print-freq', default=10, type=int, + metavar='N', help='print frequency (default: 10)') +parser.add_argument('--resume', default='', type=str, metavar='PATH', + help='path to latest checkpoint (default: none)') +parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', + help='evaluate model on validation set') +parser.add_argument('--pretrained', dest='pretrained', action='store_true', + help='use pre-trained model') +parser.add_argument('--world-size', default=-1, type=int, + help='number of nodes for distributed training') +parser.add_argument('--rank', default=-1, type=int, + help='node rank for distributed training') +parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, + help='url used to set up distributed training') +parser.add_argument('--dist-backend', default='hccl', type=str, + help='distributed backend') +parser.add_argument('--seed', default=None, type=int, + help='seed for initializing training. ') +parser.add_argument('--npu', default=None, type=str, + help='npu id to use.') +parser.add_argument('--image_size', default=224, type=int, + help='image size') +parser.add_argument('--advprop', default=False, action='store_true', + help='use advprop or not') +parser.add_argument('--multiprocessing-distributed', action='store_true', + help='Use multi-processing distributed training to launch ' + 'N processes per node, which has N GPUs. This is the ' + 'fastest way to use PyTorch for either single node or ' + 'multi node data parallel training') +parser.add_argument('--autoaug', action='store_true', help='use auto augment') +parser.add_argument('--amp', action='store_true', help='use apex') +parser.add_argument('--pm', '--precision-mode', default='O1', type=str, + help='precision mode to use for mix precision, only support O1, O2') +parser.add_argument('--loss_scale', default=1024, type=int, help='loss_scale for amp') +parser.add_argument('--addr', default='127.0.0.1', type=str, + help='npu id to use.') +parser.add_argument('--nnpus_per_node', default=None, type=int, + help='number of npus to use for distributed train on each node') +parser.add_argument('--val_feq', default=10, type=int, + help='validation frequency') +parser.add_argument('--device_list', default='0,1,2,3,4,5,6,7', type=str, help='device id list') + +def device_id_to_process_device_map(device_list): + devices = device_list.split(",") + devices = [int(x) for x in devices] + devices.sort() + + process_device_map = dict() + for process_id, device_id in enumerate(devices): + process_device_map[process_id] = device_id + + return process_device_map + + +def main(): + args = parser.parse_args() + + if args.dist_url == "env://" and args.world_size == -1: + args.world_size = int(os.environ["WORLD_SIZE"]) + + args.distributed = args.world_size > 1 or args.multiprocessing_distributed + + args.process_device_map = device_id_to_process_device_map(args.device_list) + nnpus_per_node = len(args.process_device_map) + + + if args.multiprocessing_distributed: + # Since we have ngpus_per_node processes per node, the total world_size + # needs to be adjusted accordingly + args.world_size = nnpus_per_node * args.world_size + # Use torch.multiprocessing.spawn to launch distributed processes: the + # main_worker process function + os.environ['MASTER_ADDR'] = args.addr + os.environ['MASTER_PORT'] = '29688' + mp.spawn(main_worker, nprocs=nnpus_per_node, args=(nnpus_per_node, args)) + else: + # Simply call main_worker function + main_worker(args.npu, nnpus_per_node, args) + +def main_worker(npu, nnpus_per_node, args): + args.npu = npu + + if args.distributed: + args.npu = args.process_device_map[npu] + + if args.npu is not None: + print("Use npu: {} for training".format(args.npu)) + torch.npu.set_device('npu:' + str(args.npu)) + + if args.distributed: + if args.dist_url == "env://" and args.rank == -1: + args.rank = int(os.environ["RANK"]) + if args.multiprocessing_distributed: + # For multiprocessing distributed training, rank needs to be the + # global rank among all the processes + args.rank = args.rank * nnpus_per_node + int(npu) + + dist.init_process_group(backend=args.dist_backend, + world_size=args.world_size, rank=args.rank) + # create model + if 'efficientnet' in args.arch: # NEW + if args.pretrained: + model = EfficientNet.from_pretrained(args.arch, advprop=args.advprop) + print("=> using pre-trained model '{}'".format(args.arch)) + else: + print("=> creating model '{}'".format(args.arch)) + model = EfficientNet.from_name(args.arch) + + else: + if args.pretrained: + print("=> using pre-trained model '{}'".format(args.arch)) + model = models.__dict__[args.arch](pretrained=True) + else: + print("=> creating model '{}'".format(args.arch)) + model = models.__dict__[args.arch]() + + criterion = nn.CrossEntropyLoss().to('npu:' + str(args.npu)) + + optimizer = torch.optim.SGD(model.parameters(), args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay) + model = model.to('npu:' + str(args.npu)) + if args.amp: + print("=> use amp...") + if args.pm not in ['O1', 'O2']: + print('=>unsupported precision mode!') + exit() + opt_level = args.pm + model, optimizer = amp.initialize(model, optimizer, opt_level=opt_level, loss_scale=args.loss_scale) + + global total_batch_size + total_batch_size = args.batch_size + if args.distributed: + args.batch_size = int(args.batch_size / nnpus_per_node) + args.workers = int(args.workers / nnpus_per_node) + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.npu], broadcast_buffers=False) + + + + # optionally resume from a checkpoint + if args.resume: + if os.path.isfile(args.resume): + print("=> loading checkpoint '{}'".format(args.resume)) + checkpoint = torch.load(args.resume, map_location='npu:' + str(args.npu)) + args.start_epoch = checkpoint['epoch'] + if args.amp: + amp.load_state_dict(checkpoint['amp']) + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + print("=> loaded checkpoint '{}' (epoch {})" + .format(args.resume, checkpoint['epoch'])) + else: + print("=> no checkpoint found at '{}'".format(args.resume)) + + # Data loading code + traindir = os.path.join(args.data, 'train') + valdir = os.path.join(args.data, 'val') + if args.advprop: + normalize = transforms.Lambda(lambda img: img * 2.0 - 1.0) + else: + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + + if 'efficientnet' in args.arch: + image_size = EfficientNet.get_image_size(args.arch) + else: + image_size = args.image_size + + if args.autoaug: + print("=> use auto augment...") + train_dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(image_size), + auto_augment_wrapper(image_size), + transforms.ToTensor(), + normalize, + ])) + else: + train_dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(image_size), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ])) + + if args.distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), + num_workers=args.workers, pin_memory=True, sampler=train_sampler, drop_last=True) + + val_transforms = transforms.Compose([ + transforms.Resize(image_size, interpolation=PIL.Image.BICUBIC), + transforms.CenterCrop(image_size), + transforms.ToTensor(), + normalize, + ]) + print('npu:' + str(args.npu), ' optimizer params:', optimizer) + + val_loader = torch.utils.data.DataLoader( + datasets.ImageFolder(valdir, val_transforms), + batch_size=args.batch_size, shuffle=False, + num_workers=args.workers, pin_memory=True) + + if args.evaluate: + res = validate(val_loader, model, criterion, args) + with open('res.txt', 'w') as f: + print(res, file=f) + return + + for epoch in range(args.start_epoch, args.epochs): + if args.distributed: + train_sampler.set_epoch(epoch) + + # train for one epoch + train(train_loader, model, criterion, optimizer, epoch, args, nnpus_per_node) + + # evaluate on validation set + if epoch % args.val_feq == 0 or epoch == args.epochs - 1: + acc1 = validate(val_loader, model, criterion, args, epoch, nnpus_per_node) + + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % nnpus_per_node == 0): + if not args.amp: + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': model.state_dict(), + 'optimizer': optimizer.state_dict(), + }) + else: + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': model.state_dict(), + 'optimizer': optimizer.state_dict(), + 'amp': amp.state_dict(), + }) + + +def train(train_loader, model, criterion, optimizer, epoch, args, nnpus_per_node): + batch_time = AverageMeter('Time', ':6.3f') + data_time = AverageMeter('Data', ':6.3f') + losses = AverageMeter('Loss', ':6.4f') + lr = AverageMeter('LR', ':6.4f') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + fps_time = AverageMeter('FPS', ':6.1f') + progress = ProgressMeter(len(train_loader), fps_time, batch_time, data_time, losses, lr, top1, + top5, prefix="Epoch: [{}]".format(epoch)) + + # switch to train mode + model.train() + + end = time.time() + for i, (images, target) in enumerate(train_loader): + adjust_learning_rate_fraction_epoch(optimizer, epoch, i, len(train_loader), args) + + # measure data loading time + data_time.update(time.time() - end) + + optimizer.zero_grad() + + target = target.int() + images, target = images.to('npu:' + str(args.npu), non_blocking=True), target.to('npu:' + str(args.npu), non_blocking=True) + + # compute output + output = model(images) + + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + + losses.update(loss.item(), images.size(0)) + lr.update(optimizer.param_groups[0]['lr'], images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + # compute gradient and do SGD step + + if args.amp: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + optimizer.step() + + # measure elapsed time + fps_time.update(total_batch_size / (time.time() - end)) + batch_time.update(time.time() - end) + end = time.time() + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % nnpus_per_node == 0): + progress.print(i) + + # print(' * FPS@all {:.3f}'.format(nnpus_per_node*args.batch_size / batch_time.avg)) + hwlog.remark_print(key=hwlog.FPS, value=('{}'.format(fps_time))) + +def validate(val_loader, model, criterion, args, epoch, nnpus_per_node): + batch_time = AverageMeter('Time', ':6.3f') + losses = AverageMeter('Loss', ':.4e') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter(len(val_loader), batch_time, losses, top1, top5, + prefix='Test: ') + + # switch to evaluate mode + model.eval() + + with torch.no_grad(): + end = time.time() + for i, (images, target) in enumerate(val_loader): + + target = target.int() + images, target = images.to('npu:' + str(args.npu), non_blocking=True), target.to('npu:' + str(args.npu), non_blocking=True) + + # compute output + output = model(images) + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % nnpus_per_node == 0): + progress.print(i) + + # TODO: this should also be done with the ProgressMeter + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % nnpus_per_node == 0): + + print(' * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}' + .format(top1=top1, top5=top5)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value="{top1.avg:.3f}".format(top1=top1)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value="{top5.avg:.3f}".format(top5=top5)) + + + return top1.avg + + +def save_checkpoint(state, filename='checkpoint.pth'): + torch.save(state, filename) + + +class AverageMeter(object): + """Computes and stores the average and current value""" + def __init__(self, name, fmt=':f'): + self.name = name + self.fmt = fmt + self.reset() + self.skip = 0 + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + self.skip = 0 + + def update(self, val, n=1): + self.val = val + # the first 5 value are not accumulated in the average stats + self.skip += 1 + if self.skip < 5: + return + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + +class ProgressMeter(object): + def __init__(self, num_batches, *meters, prefix=""): + self.batch_fmtstr = self._get_batch_fmtstr(num_batches) + self.meters = meters + self.prefix = prefix + + def print(self, batch): + entries = [self.prefix + self.batch_fmtstr.format(batch)] + entries += [str(meter) for meter in self.meters] + print('\t'.join(entries)) + train_acc1 = str(entries).split("Acc@1")[1].strip().split(" ")[0] + train_acc5 = str(entries).split("Acc@5")[1].strip().split(" ")[0] + hwlog.remark_print(key=hwlog.TRAIN_ACCURACY_TOP1, value=train_acc1) + hwlog.remark_print(key=hwlog.TRAIN_ACCURACY_TOP5, value=train_acc5) + + def _get_batch_fmtstr(self, num_batches): + num_digits = len(str(num_batches // 1)) + fmt = '{:' + str(num_digits) + 'd}' + return '[' + fmt + '/' + fmt.format(num_batches) + ']' + + +def adjust_learning_rate(optimizer, epoch, args): + """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" + lr = args.lr * (0.1 ** (epoch // 30)) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + + +def accuracy(output, target, topk=(1,)): + """Computes the accuracy over the k top predictions for the specified values of k""" + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + +def auto_augment_wrapper(img_size, auto_augment='original-mstd0.5'): + IMAGENET_DEFAULT_MEAN = [0.485, 0.456, 0.406] + assert isinstance(auto_augment, str) + aa_params = dict( + translate_const=int(img_size * 0.45), + img_mean=tuple([min(255, round(255 * x)) for x in IMAGENET_DEFAULT_MEAN]), + ) + if auto_augment.startswith('rand'): + return rand_augment_transform(auto_augment, aa_params) + elif auto_augment.startswith('augmix'): + aa_params['translate_pct'] = 0.3 + return augment_and_mix_transform(auto_augment, aa_params) + else: + return auto_augment_transform(auto_augment, aa_params) + +def adjust_learning_rate_fraction_epoch(optimizer, epoch, step, steps_per_epoch, args): + """Sets the learning rate to the initial LR decayed by 0.97 every 3.0 epochs""" + + lr = args.lr * (0.97 ** ((step + epoch * steps_per_epoch) // int(steps_per_epoch * 5.0))) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + +if __name__ == '__main__': + cpu_info, npu_infos, framework_info, os_info, benchmark_version = get_environment_info("pytorch") + config_info = get_model_parameter("pytorch_config") + initinal_data = {"base_lr": 0.1, "dataset": "imagenet", "optimizer": "SGD", "loss_scale": 1024} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_infos) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + main() diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/simple/check.ipynb b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/simple/check.ipynb new file mode 100644 index 0000000..a147ef0 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/simple/check.ipynb @@ -0,0 +1,177 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## TensorFlow Consistency Check\n", + "\n", + "In this example, we demonstrate that our model gives the same output as the original TensorFlow implementation, when using the same image pre-processing. Note that this notebook requires TensorFlow in order to do the pre-processing. " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from PIL import Image\n", + "\n", + "import torch\n", + "import tensorflow as tf\n", + "\n", + "from efficientnet_pytorch import EfficientNet" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "model_name = 'efficientnet-b0'\n", + "image_size = EfficientNet.get_image_size(model_name) # 224" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAArgAAAJlCAIAAAC+AhaJAAEAAElEQVR4nHT9WY8sW5odiO15b5t9jjnOOffcIe+QebMqs7JYlVVsNptsSNCLIKEF6J/oTwgQoEe9S4AgoNESSUENkt3VLGZlsoac7nzumWMOn2y2PethR/iNvEXaQ8DD3dzcbJv7Xutb3/q+Df/P/6f/Y1EUm6asu5ZFwgD06PG7/+K/+d8cn74DPAUIDlcX//DLv1ovz/IEY+icQHGUZkVuPeyGASCkrVsul33fE0IAcNPJeDTKL87evH39SgihPdlsNqPxeDabHR0dLxaLq+vby8vrpmnenF84C6qqghDFaSKlXMz368327Oxstth3AK5XmzdnV0LEt7e3QsR5kjJCOaGcEaM0Ai7P85//+Z+cXZz/q3/1r/b29tqh18rMFvOLq8tyW3/4ycfFePTs2bPxdP7+++9vt1vC6NnV18bYtu44i7S21oA8LZqm2dvbqzbbpq0Q9nmaZFmSpJEQnFiMCL68vFxvN0cnJ6ePH5Vl+fLN6ydPnv7o40/evn17cXb++9/8djGfHx4eVtty4rhzLs/zOI7jNGKMNX0HADg42v+bv/mb6WL+L//lv8xHmffeObdarX75+3+4uLiICB/l4x99+PHl+dV//9//DwcHB4RRC0Fvhj/6059MDhbZtPAEAgTXy9uubRlj3377bd+0R0dH2/Wm3pZaa2etYJxzTiAihBhjyrLce//Jp59++u/+3b+LomQ+nX3++Rec8/Xt+mD/iHMOHKzrZhgGAJD3Xmtdl6brOkyJc2a13QAATk5ODg4OxqPpxx9+SDHbm8y6pv/rv/oPb1+/qaqKzMef/vBHl5eXRZZ89rvfz8aTvb29YpRfXV31fS+1TvNkb29vtrd4+frVL37xi+OjJ2kae+/Wm2VVraNYzOfToihubm6urm7KbSN4GseZdxBCwqhQ8poQMpqOEIJlvY0iHqWiqipljIfIGm8cpFzEUc4YBwCgZAAAOOes8cYYpYxW1lq3vLkVIqaUG+2UUt4DSimltFuvkiSJ4xhCb4xxzmEEEELG6JgLiPwwDN6aoigWi0We59ATQoj1DgDw9uzs/OpSRKzve4QQxrirG4rJ3nxhtSw3W875wUG62WyA81maCEqSWBRpRjE5OTl57733Xr9+e35xVRSjQZmLi6vzy4s4TrMkff36NbLeWfv45NF//V/9s8vLy9/97rNtWW7r6vD4qOr6LM9ne4s/+skff/v8y81mU9e1994YI2WPEIqi6Ob2CmMshEAIMMYWi0Wapm3bwsYqpYz13gOlTNP2UkpvwXw+F5yqtgfOeGv+1//yX/7yb/4jhBBCHMfxer1u23pvb288HkdRNJ2OvfcvXn57cXFBKbXWnpwcG2MQQhHGP/z0x998803b9cenjy9uVtc3twCAvYPDsixHebZYLA72987evJJSDsPQdY0Qwnv/5uwCQjiaTLXWIooJZ6v1tus6kaQHBwe//d3vwt2cF6ht2+Pj4yiKOGVZlimlpuPJaDRqmqZcb25vb733x8fHo9GobZqX1zd1XTvnFosFgoQxNhqNrq9vvPe//e1vx+NpFEUffPDBe++99//5f//rf/Nv/s0HP/qAMtE0jYNIG1d3PYCoLEspJULovXef/ulPfzIqsrPXry4vL4G3xfS4aRqttfceQpimOaW0ruu6bouiuLy8xIimaeqcG41GWuu+H7TWWmvvHaU0S+M0jaMoMmrQWjnnvDNd122362EYIIRphIo8PzjY04OczWbWKELIarnBGFMublfbuuny8QQi1g9SCKFVixB6/OjR+fl5GgnBueoHa21VbYui+PrbZ9bbKE2qquJCAACkNAAArXWe59PJfLvddl23WCwIYQAAIQSEuGkaay1CSCkFCEYIaa3btlNKEUKEEJRwjDGEGCFkjOm6bhgUpTRJEsKdcw4AQClljHmIwuW3/UAphRDGUZpkqVKqbVuMsdceYwwhlFJWVdW2LQCAc44QCscBAFhrrbWUUiFEnKfX19ec8zzP67bV2hBCejlAgKuqqtrGGAMhBgBACBHGnDHnnLUWAIAQghCGYxJCwl/GGADAe08I4ZwTdvdeIcSomKRpGi4QQrxara6vr5umBQBgjO+nHei9996HwzrnjDHWWu89xjhchdbaWgshRAh568JEFI4QnsQYSymNMRBCQgiEMAwaAEApRSkFAFgtvfeUIO+9kZJRzCnjnApGIITAeeet995jrLV2zkEIwxQRjrk7w92HQgghhAWGSqko4pwzaSShKMsySjHBmJd1M0gZx7HD8PbyhvGk6xqAoGlrp3Tdbi00ZbMtt52I0HRSSKuQV8b5bpBERABiCGwkKEKoqtrXr7dXF7gsN5vNhnM+nh8654zWQzfcXF03TbdarZq6W222ThuMqPdAayVXUg46T4pw5+q6HgZV1U1XNxDiKEqg80IIb13TVoyO9/bnVumuaReLRd02URQBABhjo2KcF8Vqs/a+Ojk5oZy9evUqz/Nw+6fTKaaPzs/PL8pLRqVzoNyU4MAfHh6O8xwBB5F11njvh2HwwBpjgPZt2749P2vb1gFAuZjv7/38538phIizNE1TCOGPfvSjruuklEmWwrrTRi43Le/4ITscz4s4F23XYQx/9rOfxlmqtfz22+s4jq216/X69evXt9c3o6zAkDjnTk9Pf/zjHzPGPvnRDwHB5zcXx09OJbBqkIiTtu8E533XSSlnsxkYT4QQGKIizc7evtXeU0oXi8WkGJVl+eybb16+fDlgzRC8ePPaGHOVjdbr9Wg0cUq+ffVtkY/ztOAYYs611l3X920LveAU56OiGI+OpWyHHiFUbus3L9+sb273ZnvL+UJL0zQ1QihNU2uH8ubsaDY2Su9Pi5hToPsIFxGGGjoL7e3lRVlt3l68zUfFH/3Rpwjy9XpV1VutpfO26xrnitlskucpQsjoc+8dAA5AqJSUUiYiOj46StP44vKsLluCaJqyOMpNVdVtu96UwyCjJJvPbVEUhHJvDaWcRzHG2GjX97JpmmFQSZJgTDEAHnqCkPceeueMFoJhDK3V4fdDMAQAWGsixo1VTjuCMBPce//27du+7z9672MHgRAiHxVJkuR5LiLmnAvTCifYGauNVMOAKcnznFKQ54k1SkQoYtTYYVvKWAgP9qt6XdVrDzSAVuneOkkoMMYABPM854TKfojT9NWb1998803TtgCA8XjMOefWee9Ho9H+Yu+bL37PEM7jxDnXdd2gnbISOR/TyDlDPOSEC84iyiLKALfr9bptW20cIXRQpu5apRSjQhntnPHAMYKNUW8uzy9urrMkoZgtFgvnnJQ9hDBMZwCAsiwZY0VRKKWurq60VlEULRaL09NTY8x2uwUQYYwZY0IISun5+Xld19V2Mx6P0zQlhGy322fPnsWxSNNUiDhMo4HdNl1LGD+7ONfKPn3/vTxPoyjqui6K8HvvPX779u1sNiOEbFbrJEmyLOv7PvwAkQdJkmw2mxcvXmRZJji/vr4GAGRZJoTouq5ZN9batm26rk/TNE3jrhuGYQAAUIYpw1prALG1FtG7K424IFMipfTWOecuLy/Xq9tyvbLWxhF31iIIGaWB+gPngfPQA6sNAnCUF1EUMSa6rqOECM4xJk3TDEOvtcYIYIzDJayXUilV17UcOqUGrTVCiDFGsEMI3d7eAuu6pt0/WDBCp5PJYm+v64btplZKaa2TNIEAQYwcxtbatm27ukHeYYSEEISQpqmyLDs9Pb2+vQbOp2lqnVNKIYTCmSulhmFQShljjDGj0WQYBoQQQnfMINz9pu8D2gXkCzAWOCIA3nsfcC4cVmtNBdkBM0YEkjscZYwxxsKPLqAUQogQ0nVdQLUApbt9GGMBX8NNCTtjjJVSzrndh4YNQqik0lobY8LOgV4YawnGAdfDJ+5oh1IqsBnn3A6zrbXQgh2QD8PQtm1ZltvtljER4J9zHkYjXLvxbkcUApbt/oZxDlcXnnTOwTBq3geiCR5sO/DeDeDunMOhvPeBO4WDP6Q+u3/NA0IA7j8rjO33PiU8sB4ZB5QBiADvEfAEAgwAIm/PLhGCxSQ7ODhs+vaLL74cBvXFl797/8OPCAaO+umsePL0dLV5dXl2aTAl11sAMabcOCetS7NCpBnU+u3ZRT4qIITe6kEPQ9fXTbVcydtlNQxDGiWcUmBBtS2rba217ppWKVNVy/Vmm2UFpbSpqjevXu0vDqaTudb25cWbrhuklKnzgjKllLfOaeOM9c4QiNI8LfL05OTk+vaGMTYMg4OAj3i4YYvF4smTJ3XbhKmq7/vVahUl8cHBQZqmUZQU2UgI8eyb503T9k2dCO69pQQbYMNtwAAyTITInAN5NiJMOADPLi8ARtPZjAuxXC7rru3l8OEHP3jz8tWb16/TNHWu7YauqirGGBGexMA7uFwurTs+ODpp29YYkyTJwcHB3/6nv//FL37xZnlOIIr3DpRSv//8s9lknuZZkiTamvFkVOjRdDq9WN1UVTWZTaw2Td2ORiNvbJ6kwzA4Y9PRuOu6vb29oeullF3d6EFeX10tl8skSW4vzoi30Ejb9ZUakANDtTVSYkRVW/fOMsYxxsB77q2DvvXeGeO1phDSOKGYtP1Q13VT1b+++e10NMYQ9W1HMU7jZD6Zer/ZXnw7j98fmmaRIOBNtTwvsV1fX22rUqRZ31bGqufPvjk8PhpNJ/sHY6XbQaKuH8pyBaDbP5gwjtIsX62T6Ja1zWCsxIgSioBHs3xxtDjhnA+d0YMzSjflkKRRxBJrvBaaIBwJzghwZpBKMuQIgBBTiimlCFoHREQhYhB7742xEADvjNbaaugRSmMe5gUEAKOYc4Y8UEpZpwlmYbKrqkorRSnlnJ+fn2utJ/NZkiQYoTQW2agwxgjBKKVWcOs0AhAh4I3lMTGujBKqJYDIUEbbpu+HXvDptrxt2s3bt+eECoTAzWq92W61lta4NE3TNM2yTA/y8PhouVyuNpvxeHp9fW2kVdYUxRhhDKy7ubwyg0QWMEg88g5TEKeMsSxLAnZiAqMQHUDqlXPSSmt6rZSx2Fqt7WC0hwBT0gw9RiCmHFIKHblZ3jZdSxjNkhxj3DRN13VFUczn881mc319Heb6ICc455qmCTN1nuc3y9VyucyLUZiGwnz98uVLCGHLqLUWYxzwoCxLY4wxLk1diOEQusMAxqgxRqohjkWaxmkmjFWU0uvr66qqwrSOEHr33XdPTk5ev3xVVdXZ2ZnqBwjh3bVjPB6NijRxzo2yfD6ZVoSWvow4t3F8fXnJCaMIY+C90W1VR4x/9MEPOq+8c9ZaRAH0AAMohMgZ6/s+zAk3V9fOG2+04BwA0Pedc45SGsI1bWQIMDCGO7zXWjdNAyEcj8fGaAA8o5QzEkURpVgNw1ZLrZUchq6tpewhAJQQIRhjbGi3UspyvWKEKqXyLHHGChHvLRbXN0sIobMWOh9xoZDp+/76+joQF2stRshIBawbuh4BGMfxu/k7bdtsq1IkcVmWXdfl+TgErMaYEHoGWGKMtW07DAPn0S4SpZSCod+BIiFkd+GUUufu4mbGmPcwIF/AOa2Msx4CBC3W2g7DgAgmmEEItNagl8MwSKkhxCH+DviNEOKc72AsPLkDvIDuRrlwbg+R2Fq7YxWBQwRYtdbuQupAKXY7h3g9XFd41w68ww7GGGdBiOm11nXdcs4ppYQQKWW40RjjHYSH89kxht3zO3gOp4rA3UsPw/rdPnfgbW0gJYGvhGfur8jvrj0cYSe93A3a/ePd+YT3Boa3G9vvPgsQAzEByEECsMeEQkwgwORg73BTrrarrbFKOyOHtm+7v/vlX//pT34yKSacc0BhksdJliCGGedo6I2RTklpndSWIswYgx6Uq6VRQ5DcrUXQO+g8sG673jLGEIBt3a3lxlmw3m60tovZXlrkFxeXfSMpJnlaAAettXXVnpyceAhfvXzbAcUIRwhZa6EHTdMQBDineujP3r4+PNz/4UcfM8b6vg/fIWl027Z12wAAQsiepumHH34Yp/lqtbq5uVlvN3+e/vjP/8nP/1f/bR5FUd/3/+F//g+//vWvq6qqSmSMcc4g4AAAGBLGRBynjMZCKR61xkElTVnfdr20DmCMr66uOCMEoiiK3v/wB1prpVTOLYsAi8B4PE2LZNustLJ132yqCDP8xRdfFfn4L//yn84m06os37x+TVIWCbGYzbM401oDBLMi996fXVwo6KVWEKPLy8vfffbbyXzaD0MU83/6T/+pc+7m6rouK+QBH43qqro4OzfGMEzGWR4JYa3t6gYAwFPXleuT+TyKIgiwlLLaVk6wJMm0NN45Bjwn1AMLMXIQKGA98gxB6Lx3ximtu0E2Q5EWq5sXyPq6rOpyc3RwGHEWJ0w4vF2V5cWL6XQ+XsyiJC3LOk6yN99+tb4+z9TEOT+aj7/+atm3lUhiCEGWZXuLaV3fyqFJs0hwZHQHgU4TNp/l3pm+6yARo9Ekz0cfHn+UpilA4KP3Pjyc7/3297958/ztaJzHaYQ8jJjACDFKCXDYWw88MtB5paz3SmJEMYKp4KngFayMA1pr7+zQ66FvvPeE0s6hJEkYY85ZbxX0mFCKCe9bG36QRum27rTWxYjEcby6XHVd570fj8eB9gUpHmPsnHEMUhRhjGXC+7bTziDXZxH3DhjdA8gwcZDj8SQfZFvfNjc3V9P5YpDddrvuug4R3Esptc7TlEeCUpqNR9erZZQlVHBpdN/3bddNJjNGKMekLauYxV3X9X0b1O8iyeeL2Ww2Wy6XZbkJEQ8AwAyqq5qyLB1GIo681H0/aOtEHFFKMaZd00aM+ghZ7+I0BQTn0zFnPOBuSG0IIay1VVU5ZyaTCYAu4PFisWCM8jvg7LfbLSEkTdMoihJlIYRlWfZ9Px6PCUaEkKZpbm9vR6PR3t5e3/fDMHjvtbWcc8bYaJSPJmOp7Xg8ur29HWS/LTcAeCGoEGJot3meB7U8SRLOedd119fXIZHhjDHGbDabtm2LoqCUHh4eVlVFCMIYRhG3Ns2yBGNYFAUAAEAfjvDFF5+V5ebjjz98c305DEopxQglmDkA0yimgmutCUQAeqUUcIZRDD0Yul47hRGlCEMArFLSGEuoloNgxGpNEEgizlnktGnKrdMKEuqdgdAjhDD0Rqlt31qjhRDOGc4IZxml+C6odb6RqnW1GqSnjjOmlILOq15tt9tys7XaUITDBrVp21YOg/e+3pbAeYaJc85q03VdiM32DvZjIVarlZEqZGQ45xhjY8xDMSAAatu2ASsDeAdQTJIkiOcIOYwxpdQYY7S7E97vsZnzu4DNO+cduBcJvHOm7/uuHaIkxkgjRLy3DsigtHt3FwoHxT4gepjbvxcB2/tNORNkgzuiprW1tu974BEhhAMOIXT3J8AY2x1/F9MH+AxqfBiBMAjhb5BSQrrBaBdEFMYYhHdqipRqGAZjTPj3PtD/TjbYyfs7OSFcQvgXY/Jf2v8hwwi0KZzVjiWEw4XzBN4+pBr+niE9JAq7exQ0mB052F0shNB57BD1mEBMMMCEYAwxdJ789Mc/ev321W9/9/e/+bvPALJJkogkLW+uv/ndbz/++If86Ah4GHE+n86G/lBEXGyZMhogYp2XxsbpKE0LA+Cjk8eX11dXF9dZllmrm6rO4uRo/2A+exykv4ury6vzKwhh38leaQzJe0/fG+eTtmpvb1aGytlkaow7P7tYzBanj5/8yU9/en5x9dkXX3VNzxiz1hIEIKTA+67r+q7B0O0v5l9//fXz588RQnEcu6YehmFblUmWNk1zfn7+g48+/PGPf3xxdfPq1Svv/XK5/Oqrr/70p3/y7pN30jR9/vx5P9RG93FEIbIUewyhtcQZ660Lf4kQADEpbVk3LBLFaJqPMkyodWa1WSdRfHpytNqsf/jxJ9vt+usvv/QJ5GmUjPIPPvjAe/j825fFePToyRNnAfTw6urq6urq8ePHcRwboz755KNWdm/evHn5/PnHH/3w3Xffnc/3Lq5uvvjqS+Odu74WMbfW3lxdf/XFl+PpWCTx3sFivVxtt9vr62tvLGOMEqKlcs69efmq7zqt1Afvvrc3m3d1U1XVbCyUUlkUzedzSnld1wygohgRzKqyHHoVRVHMY6WUUxpwy6KIUjqb7yXZuGpaowzywBkLCEQAC8bpKE8ienC4mE/GxSg9zZg+nL98+fJwPhqkgrpPGaLIvf/0VA3tYLV1bpyISSocxlki1lfnQI0YI5M0xoeLOBG5YF62EET7kyJjLKH04vwaI3I0nxwfH//g9J2Qv2QiHqXJ5cXZt988e3FzO9+beeS1loMaCEXWjGazSZamADjgAdTOGQWIJYRgghEiDEEMPUMEejt0AAEDERQcE+uzOMqybBiGvu+1HICzEELgrVJ66JW1NkuSMHveXt1ii5wxTdWU601VV93QjscFZdR766GjhGCMteylHgCy3jnGWJ7nwLlV2zRNgxFIkqwoxta4vlOMiSTOMaIQYko5wtT7HmPMo4gwhgQarJbO0EjcbtfKGhYJ5AEhJBbRdDxZLBbL6ytrbVU1AbMBAHJQXduHOQTCuwk0yNHr9VaMsiRJIKZd1zvniqJgjA2DCvEipXTo2ul4lCTx/v5+W9WbzYYQEsdxkkTDMKxWq6qqDg72IISYkGEYkiRJ0xRCACGUUi6Xy6ZpxuPxeDzGGOd5XhRF3/fz+bwoCuAsQqjruqZpiqLIsgwh1DSNlNJ6vwsECSFN36VpfHmpXr36drO51dYwxjzAIXK6vb09PDw8OT65uLhYrVZ9233++efj8fj44DDEXn3f39zcQACScdH3vTW8Kjd11TZNg4CXUo+L3Hs/DKruu9fltq5rAMB0PFnMZpuyrqqKICwYp5SnaQoAwABijIF3CGPKaBxxAICWvVKWMeCB9d5bp61FEELvnbUAQjcej05OHo3H44uLi9/97nfnF6tHp0+8sbLrPbDeRkkkKCOYMW8NJQhFEQYeYwygG4ahHwaEUNe0nHNG6aPjE4ZJFEVKmeuLy9vVRkpJCDFK902rrDVanxwe1XVttbmLKQFM44QgfH6xvby4YIwFIFSDRB4kcRLgUwgRYmJKaYC9HehyrnewGsT58HVyzt8RFAit8RhjrVV4CSEEwL0GbpxzACESUhjeQ+BDyE4QIhhjjAglzBIHAdql6neYGmJ0Y0xIoARIgxAqpXZJh5CD2L0rpFHiKIUQEs6MMcOgQkKBC/Gd0H8vGOxyH2EfQsgu2gYADMMQcmdCiM4OSqm+740xRTHeqRG7SF0p5f0dA3iYCNilBr5HAh7i9/fSCrsBfyhF7PZ/KFQAADDG3vndYRFCEAOIvvuIsNvDv7szQQ82CKFS/jtxwjnkPLIee09++/e/XOzN/+TTT+YjfnFzEUV8VMwQZr/9T79IEJ0VIzAei3Q0G8+r7bZptnWlIcBZkaV5lhGapFmSZ86C7XZ48fKsLmsCOac4jdLJuJhOp5hmSimCKEOMEQoAksAQby7enr+cvCSEYEgiIabj2cHBwfX19bkFl5fX77334V/+xX/1/PnLr778tizLdG9PDzJKEwS97FrvDIagLrdffv5Z3bbPnz+PoijP815J4EFVVTwSWZZZa4Mi+vz58/Pz8yzLRBxdnl3+23/771er1U9++kdy6NQwdH3ttBGzmQMOQQQgACj8xjD0yHpMMKNMUJ5keTaZTJig1vrxeJJlmZbKA3B1e/Ou7KIoqts2E1QIIUQyXxxKqZm4eufpuz/85JNvvv62rtskiW5uls6bpi0nk3GWpcZZ2Q/Lm9tv2bec8yjNRBwleaa0rtuGpyIAWJIksYj253vFaPTtt9++ePHCG3t8fAwBaOsmz/NPPvzo5uLy7O31Z7/93c3l1d58sb/YOzk6plT2fW+tXy7XBJK6rq11eZpNp/MrzLbrNQRYa22tL7J8MZsDIqwDIk488EBbhnAWZ8Ag412R5d5ajPFkPCqyZDzJsixJU5am6XZTJWl6ff282mLCeJKYJycn3prldtspeTgff/DuIwf8bLqou76pSgXtfJwfztK+b01b1tDi0SgbT9NR4XtFjOcsfnRyeHhw2De31tqurerSx3H8zunxdv3uF198gbzjTOCIK8OdNwxBaI3XiiAYvvIeAGBsUFOtByKOvPeI4DQWguMsFQihJE3n48l0OhVCVNtyvd52Xdd1XVN3nPMsy9LI1nXrnEuiOEyR68sNxaRv2+1mI5VarpcA+XycY4o8cGkaQ+NXy6WUfSIiQrE1EEFGcFRXXbmqxqNRwvO+MxhR7wjFqXe075yWCDg+KAsAwJTwSCRZmhV5ua0gwSTiclsOVuu2GSUZQbjIc90Pt5dX5+eXTdOUZamUIgQBAJwDZVmGWQyTuxmQUgoh9t7LfkiiOOI85oJhJxiHEAJt0ygWnHFEWqkizr02EWGrrscA1nXNOR+PC855CEP7vscYT2fjEIcNw9B1bRzHwYtwu1rPZjOl1M3NDSA8qBrHx8daay0HKaVSyntflmXTNJTwoOgigsKTVVVpq5I8K0YZZXC1vr25vUqzTAiBMR62lVIKQvj48ePFYvGb3/xmvV5/8N77r169IoQEzfzx48f7+/td152dnRV9K4RIothqU5WbqqooIVLKpmkEj4dh2K5XVdUAAIqiqKsSxQIDGMAGY8oJJRj3fU/uEu8QIsg5jWMBrBuAU1ZhCDAEHiAEICaIUQwBa5rGGEMxWd3eYAisVkPXd027LdfGGAQ9YyIWnAuGIXLOeOiJR95rqZR1Gjgv5TAMAwG+bdtIjNMonkwmXVsLzrMkl/3QVJUcBgBQVzfWQsYFJ5RSGovIORdzkcZJtS0BAMA5b+x6vaaUAgQEY72U42IUJbFy7mGAG/AjxOg7P0ogEBhjrbVydqeE7xAlfA0Cou8g+C43wZH3HkKMEIEQIwgYE95DAIB3wAFPcIhoqbU+hP6BCgSADIpCgNIdsO1kAAhhJKLwQeHVQCz8fYYeQODvXYQhUYIR2p1/QMcdHu9SGDvwDt6LwADatu07GYaFUlpV1T24IkopQihQK+DJw5EMxw/Du4v4w8fBezHA/+e28LkPOcfuveGZcKVBSMAYe+jvBZK7s/LA7a5xdz7foy8PUw/hSWOVtRo4AozxTkFARMQZw+QffvUf3//g3R99+uGf/8mPN9vjpmkgIMCzFy/PPvu7v+OQPn3vXaWG9XXVlXK7boeWUM5zPs9me5xFIo4458YqY19SnM2m8ZPT0zxNCPIYASnl737z+7Is9/f30zSdjKZGO+hQUYy8g5vVWmqTRFGe5CcnJ6cnj5CH600TaPJkNL1KbjjnURRhjFEYawQg8t55hECSRLPpuOu64KKSUlpr4ygJUlVRFHEcv3z5crvdVk2XJMlkMjHOnvfl29dvBKOnJ0fT6eTDD96Tff369WstB+ccAAiAkHzw4YuyvrySSillCGEAoM226lUPkd/fXzRd3zX1uCweP3o0DEM+Kvb39zMRbKKU0KwfaudpkhRpNo6ihFL++J0n2+12PC7G48I5c/b6TRqnn/7wR2/Pz7dl/fW3zxDj77z3/kcff7zebt6cnyVZutlsttttnmZKyvV6fXRyuFwujVTGGEpIFEVt3UyKkfe+KIoP3n1EKW2quuu6+XxulKaUHB6eXl1dXV3dOOv7ts2zkTYAQZIkGcYcWNe23TDUNGLFaEoIqapKtq11EHk/zos0gYJFZVXPxpO3Z68ZwVk6NkZhDDFyF8uS1wNPcmXAaLqXJYlzLo7jsq7HRS6NzGA6ydOnp8e3y2VEQDabrKBXuhcIEui1M0rL3pmU0w5ChBg0al5keT5JGemrzc3NldbWGOM97Lo4TrIPnj72TnkAoihCDGstpR4AcBAANfRMxAgABABAd2TcGuOcQ4IrqxBmWZqOxpnWM4BQFEXH+0fBWzfMZ1oZNcjz8/PXr9/mSXp4eIwQOntzvl6vkySJokhrvTxfMcLbvvHG5Xl+s7q5uLhohxZRIGU/HhejcQ6Q55w5ZLd1rawdFQNwZOh9tamzeMpp3jW261o5OGvI0EFrLPARxaTv2yQjnHMPgQM+zfPPv/zKWrttah4JTMi2XY+LEUIo5uLy/GK9Xr+5PN/lHYGF/aA8rAclCSGUYuDgMPTWmslkIqKI8ch765RO0ujk4HBQRikzDAOGKIki4CxGiGMSU351eQG9k203Libhhs7ncyFEmFO2220cx1EUhQnaGCOlLIpiMplcv317dXWVJIm1drVaSQvKqgYARELUdQ29q6qKM2qMCUG8MSZY6gQXCKGha7fbNaYozmIRCSGE8bZpGmP1tmyVUgVL76s50NXV1XK5DHry4eHhMAxa6yiK5vM5AGCz2Sgp0zQVQjBGEYLhXYyR5fLm/PwyJH0jkYzHBYRwPB5D6J2xjLGYi15pay2hnEAU6CbDhHFGCaIEIQ+s9/geJAAA3luE7mzzCJm+x1rrtms232wuLy8xxsPQxbHQUlFKRcyiKGKMeWel7JVSGHqEgNZ66FulFAChZkcZY6w2EMKDgwOKiffeSLU/3z+7uKjrWnaSRrFSWrsmQ5gQ0lQ1ISTmIs8yjHFd14wQKSVjzPZ2vV5nRYYhwhAW43EA/nD+QUXYodfOBRIwiTGGEJJShuoPhFDQkPy9LcAYY8xDA8Gd2zH8Fr13Ic3v4Z2X0BhnUABmjKm31oYJXCkVclg7ZrDD/h0XCSMD7osUAurf+SUDfAIQtA0LfGA54VS11vje9PAwQAcPfILfC+UJJYEZ13WtpAl4xDlfrTb38sadeeKOoHi/kwR2bAbcux8eIv1dFA/gjvf4+/fulJvA2HY3ZXdiD4gCuD8MvicKHmOMALTOBKLzPRFiRwv+MUsAAEBgMPTQO+gUUIoQVMR8lCTkow+evnnzYnv79tMff/L4yfHepKirflt2n3744dXN9qvPPu+b3ns76CGLi0Qkt5WOomTv+GR/fx8iBABw3tgWjEd7R4ddJOjTR6dFFmHky+36zctXSRxfX101TbNY7M/i2BinZrP5fLHdVi9fvep7KVhUt83N9e3+3kEURcfHx03TJCLZbrer1SaJ43tZDPR9zxlJRKSGvq278ePxX/zFX7w8uxxPJ9Pp9Pnz58GUEFLO2+3WWntxcdE0zQ8++iTP867rzi8vxuPpo0cn77zzThzH3vvHj0+LInnx4uhXv/yl885Z5y2y1gFvAEDew3pASqlBaQ+ANd64wUOfpSmlfD6flwRTSk9OTzdViQE0ziISU54YqZar8ubm5vXr8ywrlDRJkuzvL37wgx/8/re/C0FVnqTW2l/+4hfHp6dZltVNF0xY3nvGWCeHuq7zUVaWm7Issyxdrm6Xt7fDMGRZ9vjx48uLCwBAxEW52SqlppOJEIJOp5PRuGma6XQ6m0xfvXq12m4ODo8Y5UZbABBAJIoSQtjZxWUSpcdHJ2maXl3dvHj2bVN3GJWjIlGDBBDl2UQkBWXJtqzlYJrqghLirYuLZG8+hTAQwYEhcn51MxmNL26Ws8mYiOjt6zfGXBdF5pxrygpRIocmZtjIdqu68WixGBXWRmW1qbYNZWQURRBCAaHpur7bysHGcca8r5bL9WoTj9h6vWVURGnW1Ju2rTGmh3t7vVLO2V4OdVsbOzBBIxZhjCLO70IQAAAEhCDOqUewqispJeEsinhWpBgnAADMqJQ9AI7zKE3TvccLzsR8Pg+q0MHBEfQAATwajQAAl5eXX331TYZjyKnUQ5Ik+0eHq+3q7OpMqr4t2/W6aZoqit+dTMac0+1m8/btW+lot6+SOMaIO4viOB+P9owydbVFiBAcIcg9hFnKIpEAuInHPo5jpVRZ15O+/+bbZ5zzq6urw/2jPM8RhIeHhyFE6PveG+sApAGajDLGaGuwQRAhEUUi5gCAum2MMR4CyihhFDrkreOUzecLpcz5xdVgu5hxTlnfNdiBiHFOWbutCIbO2KZpJpPJ6enpYjEryxJjnGVZ1zU7IxuEMIoiCMF8Pt/f3++2WwCAEGI0Gm3KutxUIT0hpey6LolE0zQIgr7voygaj8fL27VSCmMsgMD3GyGkLMvYGmtNkiQQesJoWfZKKYvtfD4fj8dSyq+//tpae3h42Lbt8+fP0zQdn5zGcbzdbsuyBABMJpO9w4NhGIZhIITEcTQajbz3wVp4fn6e5/nTn7xHCFmtVowxQggSIqO87/tNVQOIhRDBs1aWJcOEcx5HHHir+k4rZa3Bd6kG552DAGAECMIA+SQSSRQLIW5vb5WUjDFGaZqmk+ncORPS/EoOQbSG3jZ9zxiBIIjMHgCAIcQQamOEEBHjp6enfdsRQoZhCJy1a9pBGh4lCEKrjdUGIRQcpghArXW1LcuynE0mRVEoNUAIy6aO1J1dXwhRVRUW0UN/3E57b9s2JCOCOyEQhaDlhHsUBGx/7wzQWgfzIiHEOQehDyiO8XdVFdZ4B8HdHaYUABCkBXhXueecM+HLsFPL4X194M48sbMKhspMd59cA3dWhvvI2BgppfFOCMGYCCxHKUXujY3ggaofptyH0v0u+N6ZG+I4JtgAAIIbJsuy+8JFsxPJEELW+N3B/QORf6e+7F4NFwX9H5zGw1d3pOF7UsRDyHf2Pmdx770Ip40hCuWRCOOH1Ofh0R7KJ7uNYWy9xwgAZyGwHKM8iuaTghyfZFdXhjLuPCyKAybSN29/27Q+KZIok1W/Pbt5NZ9Ps1FCOGOMRdl4NBoxxtrKOq+atjRGJUmU5sXP/vwvr2/OnRA9YZvNioskf/zOB0f445/+mXG2lxJRNhuPkixFiPjV6sPD/WdfPdtuNsvNplOq/uXfPH369M9++rM3b9788q//Ksuyb775StW3SqksSyj3jGEA/NDVCCERR/lo+ukf/8n18/+76RvS4hTaw1Gqus0oE5ii5fb2P/7NX3dyOD4+DYJkHKevXr2pVf3JTz5ZLBbS6bPXF3roP/nkE2ls8ezF7dV11w+T0eTmZsmYePzk6dXF1VANcZooqiAj0um94/1tW27a7dXmcjYtLBwWi3xasBfXL89fv3q8tz90dKhVVVXV9vdSStmqr7/45vZqub+/d3h4CBH58ONP/uZXv6ya+vT0tJhNXqzOPj97bpQeZaOnT55GlNQ3t5TyRZTl77zfDP1ifvjROx88e/Fs6OWjR4+sZ3mcPn3nB0eHj5WUcVQsph4hogb/T3768xffPj86OL69vUUedZXluNj2/d//6uv333//ZF8URfH8+fNxsV9thlevXj1+/Hg+3rfKq14ZZRHAGKJRlJe3q/VqZfqazmbG+76qUtAT9dYP8L2DZDqfzUaZNVBtlfRGcTo/er/v++Vyfb66IoR4a4Hzq7bO01gZvlmu33nCVzc3urNPnzwZz+e3t7dl00PAEEm7XlpPprPxat0lIrLaya6l3sOIYSVte3VdY865dUPVDyJOszhS2iKrdFdRxoAabNdAACBCccr2D/bigg/D0HWd9zDLsnxUIEiGYTDOr5ZrrXVB8wRlgYlHJELecc+wI6Zzt9dVlLgkm3/yR8UwDE1Vd22bpvmkGHV1Vd/c5gQr4w8Wcx4Jb+0kSz988gjqbjafbrfr08mk67ruensy2psmC9xjNdUHhdifzLRxk2I6SucHB08JH2srZ/vRm/Ozjz/+uCxL7/0kiry3nan20nRo2h+8/8FiMfvr/+V/+cnJidX64+l0Npudn13KUUEo7bvqzbkSUWIZTotFNwyD1h4xh4mCDiBGRNJD5CzEEMTZGCMAPG6qxltPGEnzTERsvVoG8FuMs2pbAqlnScI5GWpzsL//2W9+W/eDsQDpnhCQJHwYupevniGEptPpdDbalqt0Fc3nk4uLiyRJJpP96XS8tze/enM+KWZ7i8NRMWs7J6geqI2iLE1yq0Db1U3VO2Wnk33oHfJEECoItdZiD7y14yxHAF7dXGZFfnV15YwTgl5V7Ww2jePUGFcspp0ccNcsnDk4Oujr9gfvvvvs2bOjvT2tNWUEE/T67E3V1CcnJ1VVi7LinAOAb25WSZKMx6IqW4x4XV3n2WRvsTcMSuvWOYgQTZIMAH59eW0Hd7J/XNbVersBPrhcTT80xSjN89x7L6nQWqthAOSuOJAxFidZ3/eDkowJDyWEcJCKi6jruna7hQh1vZRnLymPOOcO+FCwBwDwwDpnrHFtU6Yx80Bq2Y1H+XJZYdTPpvPj44mxrbISEHp4+KiS3pE0mx7JzbY1gLIIOSedZkQkmGmtHQBS2s7ayWxKI7F/eKicNauV2mylshCQIk2Wl0vBBKBx1yoEGWcJ5xEAwNp+NJrsZAaEAKXYWj0M2nsbJbFzLhQKAgA4ExhjgHyUxH3fa608ZAgh4J2HDmJgjOp7eddxIWIAAO8ghDAVwlpHCIk489p6qYFS2lpnLOSAUkIRkVJ2XR+ECsqYc8574DywABjvvXPYOcq4ts4NMsuYEFFdt1prRrjRDkPCKYl4ghAa9EA8JhgT6wGAAITIHjhnvXPWO0QZsM4Cv9Mwwt1JYmGMwRAJxjEkgWMBAPq2C+5OCBHFxBNnjPHOQ3iX2nhowwzZiodAvkN95z1AEEEMALDeOaMDePdygAgywR8CPIQQ2oFQ4D0MCTIIQQB974OcgBFkEGLnvHcII2blAJ0VhGith2EgiGBK+76HEEOBESXAIw8IRCTwrdblSlZZljOkMTJFRuIU7R1NyA9/+MP33/vBYrE/mc6HQX/97OV6uxl6UzUtj6LFYn9vby+OYwcdISTLMsoTAHRZb733UcQJAUqrppWE4Lar4ziGEHDO5/O9m5sr58DR0ZH3/vr2ZrvdYkqKouCUOecmo7EQYlaMjTIXZ+dqGG6vb25vbw8Xx3mRTiYTxtgHH3zQ9/3rN68ghMFL1bZtmqaUUq211vL165dvzy+/efZtVuTzg/333v/QYfzy7ZtOKgeJcb5qu826xOTSOD8ejzmPpqNF18j3//zD5fJG8CjmQmsNIZ7NZkPbKaWarkEIpGnsvIoz0UvokaOM5NPCQjceF1HKAV7cXl8ZNUyK8eHh8XbTvHp5ZqUzFldVFUURpTTLkq7rmiiCEFZVyRh99s1XaZoeHu6PRvlyeYMxfHz66N3Hj5q6Xq+23timKm+uL7abVd/3xXg0nc9PnhwfnBzcVOdvlt8WhJ8+3VvM5sEuVOR529Zd19R1ncSZkSVwbjqdT2bTq6urN+dnyIPJZJLn6TAMRZE9enTijPkPf/1XZ+dv9uaL58+fd10ztE2WZVJKiMF4NJpOp9WgsslcZCPnjAIeYUQEA1rySGCMOYtG01GcJbe3y7OzC/VcHb3zjtTK36cng+ldDUORpc5PrHdSys12GyXxJ3uffPDBB2KUq9//vhq6NE2SJOn7LhQOcM4F40ZG0AOGCYEIYMqpeHN+ub+/n0SJc4BzniSJb3ulFGMMQBhFUVaccs6VVZSRKIqwd9A64ByGBHnglQEMC87TJNdSrdcbOQyUNABCjCl0nhWp9Y4BEAnuOeu9gxDiOKaETJN0bPX69nazXFFGj9594gW9fLmME0EZLEa5ECJJklFRUEqfPHkCgGv6DmOcpjklZDKZZFmRUL+32HcOaIuaunMA123f9oPg8Wy+ByAZT2ZXV1dcQIhwlKR1XUcxhxBW223f95NRMV4sptPpzc0N53zQbdd1Wpum6ZS+bds2Ge2DB+GIdbbve+/dZDIKIYK11juP/J1hSknT1J01fuj7pmlGWT4ej/mctXVNKAYwyMV3wqaUMs+Kvu+vrq6iKEKQMEaklMvlCkKwXC5Ho/FqtTLGZFkWJkQh+OMnJxhD67SxUuthvhgrbT3QlEHh2MH+nohYuV4zxrI0ZQQ753olOadBPWaMjcdjD+/mx+C0D/mFWERN08wnU+D9bDwJHt6u60L4e3NzE6SLsixDiBzajgEAlLpreqO1btt2u90GC8LDyFJKCSGE0IR9PILaGmNM0zTd0CdJEkKxkIYINXIBS0J0izEGwO8y6KE3g7NeKRVEdQAAQtBZF04pEAUbolvoAHAQIGOMUkArJYehadAwDI+P9oGHYeo7P78mLKKUvn37tmm6EMV+T0P20EJ818Ei3JGsyJMstd5BjAijiGBntHFWWwMtiikNlkBrbUjNBLPekydP+r4Pj7XWfd9LKZ1zFgKEiHOOIAwhxAg6a43W1hirLfQAOAshwhA5Y7umNU5b+z3nHbk3LUIIodZaKRPqb51zu45h4dJCUiMQhYfifCgx01pTrbqu45zf7UZpGIcsFaG8syzLYDkM32qLYVBNgm5x54FAd44KjOBOz7jDZWtDDYgxJuRT7kf7u8zFbvydcwR/RybAHzoHHw4C/C/bG9EfFknuvqIP5QT/h8eFEPoH/Rh2b3z49dgd+d6Z8Qdawu7VCEKkYczxOKYpFkd74/3ZeJxnZFvW0+kcEwYh0tYnSfLzn/8lofz585dpPto72B+NRoNSZVn2nXS+4gIyxuLIAwCE8FK6oddGK6tB27ZxItbr9XYdpWmyvF6ORqPtdk0I0XLQUhpjuraGwDVNl6bp0NRt089nsx998nESJV9++eWLFy++efZVcC1Np9N3333nxYsXgkcH+wfPX3w7m82iCIQpqSiyn/3sp/v7i6+fvbIAW0hev72Ots3jd9999Pjppm7aF6/athmkVav1tmxvb1aLxaJp2jhGf/e3v/3LP/unTdlFLJrOxt47COFqdTvoIc9TrTWAbNDNs+dfF0WhnGWIF9Pk+OSgrCupeozQ0dGJ7oYX3z4Hqe9aff329tXLyzzNotuSUxH8EIvFrFxvnFF9329WK0qQMQYBb5Ts27YpN7rvtsvb4/msjaNcCGttnhWEuqq+vV0t907+iGW+M6svn1+9OPvCwOrwyeTRe/OcTuu6FpxkacQIvrq6chYIxgEAQzss9vbSNEUEBwIRZ/F4Vgi2yPK4rNYQwqfvv0MRNsZ4aAfZXi0v19VKUDaZTGbz8XhcbFo4TiMhhFJD29R93wLGaZooCIB13vs0S6JYxL2gHA/KGOBVyG0nsWDMWospkVJaowCE0/1FMRn98U9/Ml9MEYJSys/fvnl2fna7vjngBxHLgWe2t8Y7TBmPRMQ4RZhhMilyBGAWJ9OT08lkYpy7vLiWRksptZFS9g4iYy1EPk5HSRqZRvd9X1XVdJxYpaEBhEJgbN923Pg4jp1RwGhoLTAGek8wRtAj7xzCSlvIQCYiysSgNMYoiRIA0DB0aZJ1Xffq179e3l5naayUct40TcUYQwg0VamU4pxZrTDOEKKxA8aY9Xq9Xm2FELPZLB9lyWhOCHE4qquWYQYABkgqpbjIrq5vZ7NZVXceQu/9er1lujnYf282m9XlhhEKve/7vqlqqw2lFAPoERZZVDWd7aUQInQSwxh75D0EzhmIUegbBT2wzmqlgDWAMQQ8xYQlcZikdjNgmF739vYwRNZaNUjnLABea9W2zdN3HvV9//r1W4TAMAxFUQT/gbV2s9keHh4yJsqyDGmIpmmU7vJRxjiezyeD0TfrFY+41FXVlJSTQVrj1TDoTnZc4LJee48QJQx4661se0xplmVRsnhz9rZvW6m1EIITapUOhWqrm9vFdIYQStN02d1SSruuc8bGcRyqVdu2lX0/mkwIwqIoCCLW2uDk38VznPPRaAQAEkKE6fIOZb0PfRG2223dtVmRB9oUVG4AQGiqASE0SltrofeY4l28GAC16zpr7xiDubPX+LAPxlQPOrBqF1xQ1nrvIfJaS0extVYpF/yCod+Rcy6OkhB7NE2T5nQ+n59fXAfke9jUL0z01Y6UDEMIZxutKyl75wzGUAjPmFbKQ2gJ8YyFoCtAZkgxMMaCNyIMl793wATy0WsFoUcIEBKwlQQ0Ch3qtLZSytBxBEKvtZZahUrCcHo7yX2XxZdStm3fNE2wHeyMhwHLjXc7wN45JYNRN4C38U5rHXysYZwDvWCMBWYjg8XsvtjSIhBuFri/65RSRO6qaQBGIaFwB9sIhbrNUGSxYzwIIYzJvTBwV14YEPpe+fhu+x5dgA8cA+A+y/CQJTwkCg/fvrsdd8f/w8N+//H9v7tjhpPEiASeFBowgAdyxZ0EogcHTEz9JBOzhB4txvNRmiWMbLeVlPr3n39Z5JM0zaM4e+fxO5TyR0/eM9YhhBAlsdYQ47Lc9n1vvYniKWWoqprNVVNVlZQy+A2tU8tlc35+7pzLsgwAkGX53/zN34RebGkcee+3y9Xl27Oqqv7rf/bfSCkrObTl1vJ4f7H34Yc/GI2Kv/3Vr7766gspJUIgTdOuGwBAbdtPJwuC8fXmOkmijz766E//9E/+4i///Msvv9Qezvb2sjx/c3b+5tuX20HuHR0jjJt+6KXiLAIIWm2apsV4ba1NeLFdVuW2PTw40krOJ1OEPaP4m2++GobBWp2P8/l8Xpb1l19+GTkujaGIjkbZbD5ebW/Pzs7jOH765J00TtMo2ay2n/3+a2ictVgbeHG9Pd5faGWVUlJq7WyapuOiEJxcnJ3v7y/GRVaXm7oqsyyLIl6W61gw6s3BfDqfz5MsrbvWI61h0tsaNHbVmlZ1PHGP3t13znz+1d9+8jhxzjnLESMAeuh9FEWcR9Za5zqEyGa75Zw/evL47OysG3q77g8PDwfV1221v7//85//2TAMn3/++Sc//Cik2YauhxBSQTHDDjqWjhHnNI7iYkSTBG5X1mohWDEZd123Xq6kUsgMLOFHp0fee0mE8a7teucAIjiKo/00jTjHGKZJFHHWds1oPqWRuL6++ru/+7v/+NXndV1DCBVCpZQEAeA8IzThnBASU84iQRGGmFJCUoiTmDkLtqtVXdfKOkp4kiRPnz69ur1xwA/DMMgOYQ8hpBRD6J1yVlqrDcVEDVLrZuCdtxo20GqVRCLP8zRNPAz+aqB7o7oBSidxNLhmtVpZ60ej0XhcXL15c3p6/Gi+94UQX52dtRHPsowTAr0TjHJCm7YyWqZxooxs6wZjbJy11sq6beo2y7KYR6PZYjCOAEBFOiYpxtRZQLiqqspb16y3idJJXjjgOKVa2QgjIQQGcL1cqX6orFaDvIDno9FIS0MImeQjJqK+e2Ot3dvbawYHnLfWQucxo1GScM4JwR5YCKE3TinljSYYc8EFY3GeBWBghDHKOaXO+UEN46IgCFtrZd8ZoyjDSRpJlYZekOiuNc3dxJcm+Wa7sta2bYcxUUr1vZzNsuVyLfWgrYoSYbyByAFommaLKTFOF6PRINu6LiH02vTaidvr6zSZOO8Jpcr6XkqkNY9EwqPgDIAQMsY450pL5xwntMO1Neb09JG3DgFICN6s1gQiTih0nmHSdN0wDBjC0NmM0rteSSEG9d7neX5yctL3vdZ216/3LglNyM5A0CuZZCkhBCC4q80zxvR9H94FAGCM8UQEZArH7/t+uVxut9Xjx4+NMUbf1awH6wAhDHmlzHeB6R0AeCel9A4754zx5N5/l+f5er1+8sfvHBwciSgJcbP3fjQabctaCAEwQfe1CXcQy5kLaoe6ayIEMPIIIoQwo4QzSLCDABLMIhElSegXFI5gjAlGDWNMaE0RviphC5eAabD3WwAAhh5DTwgDFEDnlNLaSTUM0PMkihhjljHfAefADqj8vdPQOYcxAQDsOkJ67wkhcSzuzCKht6McvPf0ztBw50UA960VH54YAMBa6xwIupHRdXBUBOAPLRfv4fWuWbu7Pw4jOOwDMNppQjtUBveVDgh9ZybABD8kCvdhOmCMBQKz2/NB5P+dUxI8kAe+F9b/YzvC7nHwjd49/4eEA/yhorB7jCCE91QGY4zRnQfFGLU7JnjQYdqCChM3Svg0j6cZHedRntA0wiRKkjhOm05qa6q6HZSvqoqwOIoTpZQFkFpGGc7znDCqlDK6Ixh2XXt9dXZ7e9s0XfhCnxw/ypL0+fPnEb+LcpIkqTb1i2+fpWkausRLqVa3Sykl57wvS4wx9f7V8xdffPHF6enpyclJnucffvjBmzevNtfrzz///Ozsom36UTF5/fr1ZDLBGB8dHr/77rucU6n69Xr1u9/9Np/NNlXdW7t3egqj6PXF2c1mm01G2lnrXJSI0KEiipIQdq+XK4TQr//hH/7Fv/hnz58/Pzt7NV9M8iL95//8n19dXX355efvvvvuH/3kj+u6Pb+86IaeiFQZva3KUH1Qb0vkgdPuaP9of7r3xWdfrldrQZlIUgBJ18vb21trbT+0TdNAZyHyk8lkMi4wBBh6b03MWA9BW64jOjs5WPR9jyHYO5h/9MmP4jQ5u77MpgWNWTd0ry/elPVaJPzo6MB4vVxe36yqb7/5fDqdequaitVVPww9J5GUsusGRBmLxPq2GY3Hj995NJ6O3r59mxfRzfJ2UHJcjII8a62dzWZ5nqdpiiEMXD50KVZKbfoqTVNptFXSeUUIccCvtyXGEBEGKVODdFIBAAAjCOI4zowxVg3We4sAFmxUjObz+cFijhHo2/of/uHv//3/8ld937Zd89VXXy2BT5KEUrrtOgvBbDZL0sgo6Qi2GHmKIUZW223XYQ+s1diwsiyrukUYMEyathqUFCJCCERRJGV/dXUlhJjtzYqiiCI+dINSylgTyqy7roMYSSnzPCeExAkREbFO9nIwxmCMkXdD0wwODtfrrutvr66HTnJOT09PL87PRNfv/ejjdybT8vAIE0QpJvOptTrLMh6xpq0owjzLtGHDMBBKI0QAAH0/KGkBQM4BiFjXa2Ok1ZZRwTkFDlhrtdTeO855vS0RQjc31/t7c0rQKM2bsnr2zTe//e2vt6v1Yj4VjEMAy80WQpxmxfHxsTK2bb64vrhkmHicdnKQUgLk0zRNkiiJYsqI1pJhop2lCEMGkygusoQQYjxuVe+9p5gIERMErLVyGMqyTKIYY0gIkaqPIl4U2WiUh2qdKIqSJPHeDsMQ2qvLQQseX15cU0aiKFZKLxZ7b9++hQhBhKjgn3/5xXK9pZwNSkYpY5HQzlLBizSD0Fcl0FoDjBAlAsecc61lL4dQDOy9HeUFwsA5F1biKMttCDGtMbLr9/b21us1dE5KtSqrLMvkMMRRBCEc2s5bBz0wSqF7TAoTZYB5glkIbJQKWQlljMGYhNS7lNZaizEmhDHGeim10h6CUKTQdV3wDEL4Xfr53vNvKKXBNF2WdQh5lbqL+wOk+dCDyNsgvAXBX0rpvLHWOgwhhMZoyhkhBHib57mh8OTkhFLedqEAe3jx4kUoL8AYY+eDo3ynpRvtjHFaW2sthKGFdsx5hBCi1Emp8d2iDIRSzHkU3PUBnEJhQpAWIIRhuAKChleHYUAEhtidEEQIxpgEET6kLZyzoZd5HItQvKC9C20MAogCAIx2gZGg++7Ru5EM/CDUPgTvpJQyKBlxHHddF7SNcEqBbHl0xxUC/CNEAveSgwyYiu5bL9yhrNU7Rc1DGPSYwI0ghGC32z3kBngNPMNaf5+DsPiBSXAH8P7esbi73f7elrjTDB4KCd8jCt/b/AMz48N/v0cL/uCE/9EGEQT+uzPcmVUBUDtxAvq7W0MIERwTFk3ypMiThANBIKMoYoxstxVAOEnzvpcXV1daWWXAYv/AGsfjJE4SSqn17q7wFANOBXDQaheL5NFJYo0Pa6JwyiCERpnJaHxycjIMQ9M0n337mVLD1VVVlps0zpqqXi+XWZq/88471XrlPey6Tg9919RVVXVdRymNIv6zn/10vd5+9eWzX//61/PZ3p/+6Z+9fXv+b//t/7i/v/j5X/zZk3dO//7vf/WrX/1S6+HV629X5dYCbxSKrAIUYkF4KozTiAHVDk4b2UiEUJJyC7z3/oP3nqxWq9/++m8//eEPumarBoQJ6PqGUlyMR3sHh0+evnt4cHpmz+J0tFqtolisVqvzi8ttWVVVgyGhmKlB7u3tHe4fJVH61edfhFV5HHIWWErp6elpmqaCUwBcuVlVVeWdmk/H1XYj+8Y707elUmqUxmIyar1VRvVKSj0QLyAl873F3tGhiPntv16fnZ9DiE3vrm5vGKEfP/noq8/epAmvvAWeaAUwJEKIYdCUsPl8fnR4TCnFGGZZko0ySKDqe+ARowJAfH5xdXl5WeT548ePb25unAOE0SQrUucZY5QJTChUOmVESnn2+qVzZrY36/r22fNvhBBZllngIRWEC2tt36m+b1IuMIQ8iTmPxsWIc87jKE4TT1DdNZvNetvWwGnpDKRkspjHIk6SpJfddrtlcXx8+ijP02pbMkJSwWMmCIBea2cssI4AgJzmnO/F8Xg8ldq8fn1+cXltrRVxwgQ3Rtd17ZxBaCYEI4TUSivrlFbaGmvtoBSAznijgcEYc8t62Q9ahRQy55TIXksjpbTKWm2QdcJZ36qbl8+BUcPm9urVc6y79x4fDEN/dXXFkxmAUBtpmqFra+MN6IFSA4sEwRgj4iFkFHDKgINDF+zQUA1SawMEEFRACIDzzmiKySTPzs/e9H375uUraFRoXrRZr5u6bsrKWxfa8kPny7LkMV/M5of7B9uySpJEMN73fTGdAgCQBw66KIryJBVRhDD0RmOMIWM4SSlB09E4zWIM0boe5KC7riMYW6sJRJRhq00SRRFnGFNCUd/3EHrOKQDOGleV9Wq5nkzHo9FIKbNarZbLpTEmjmOt9cnJCZxiSmkwD3XdcPLoyWK+/+btr6u6e+fd965vV7c3K4BQ0zTT8TgfTTD0m015c32dJInUCgDAkWCCBx9AQHSttbcOQdg0TTDhD8OAMbZKr+pmu950TVtkWde0XdOMikII8fTp0zRJVqvV3mw+nU4ZY0mS3K7W3nvO+S4Br1UTrPUhQRBUZQhRXddt267XbUCRxWKeJMmgVIiAu64LBgU1SCGEM7aua0WpAXcdPL33aZqERbPC0hUAAK2Mu28cRAjBmA7quyA1TPdSSmMVY6EzIFRac0ogAGGEH7/3hDG2XK4220oIARC9vr5O0iKE+855fN9FAOCQ7HDWeKOdtQ4h5Am0xg+9yvPcI08wgxACjyDAzlmjHUmIMSaMj9Y6JA6yLAumjZ0G4O/bdmHrg4cgTVNKeEj/OwcsMpSgIs+DxS/AG0aEMbYziNyhqbtLZOwOju6bJoUdwjF3Yg/GOIqiOI5DCiBEybtg2t73JgoQzjm9i5ixC09qbSGEoemn917L76ouKedRFAkhCKOhEMbdYye6b08ZGhXAuxoEsENldN//YMcVdvrQDs4fZhAesgrwjzwK/5gZwAedHB8yjIdUY8cSHqoX/1hU2B3/IV3YZYKstRDAHVeDBCHnEIIYeues8dYBDwAgTdfWTZvnI22c9147//XXX1/frI4fPc4xYTzSVvZ938kujM7hbOGs6VqNIJ9Op3Ec13Vd12Xbdk3T9H1vrQ2/0tVqtdls4jiGEHrn+q4ZusYZyxCiED778qurq6um74rRxFmLgF+tbt++fS0i9uNP//jw8Lip+7//+1/nmU7T/Od//qRpmuvry7Zt37x5MwzDH//k0ySNvvnmq2U5nJycQIyul9fLchMl0ZOnjzs53K5uYeWNU11XF0WxfzAP3DlLo8vL5uKbV//j/+9fSz0wRuNEQIL39vYenT55/Oidx0/fhZhVjVzMD5zFdQ+UQVU5ILTUUo1GI06F1U72anVzO5uM/vgnP/7Nb36zWt8KIQDymLLHjx9nWQaBowh+K/vrizPknXeqrStrlHfKaT0u0vEoF5xWXdv23XB1qQAQaWadY3HSKjuaTqwh88lRmsUpK8Bw6zFxHY5ijLDtulrwLI5TSpI0KYwpnfVt297c3GCCkyzuZb/Y3/PQ/z//H/+v999//+j4WCnV92oymUMILy9uMMZtJ+UQjIQCICS1c87EGPuhA0oJDAZnu7LsZccRsdpJbbU2SluAOcTcIWWAWa3XoaRexLFDcLVZ366W63JLENBaQuDiNH385HjommHo9w8PbqUJfXs4i7I8yYo8y3NEKHAWY4IoIwQhF3lrMUSEkO2b1wA4hChjTMTJYiGNtW3b8zgSkciybH9/EfDVWCVVL43WzvbaGHMvPBJsIWj6FhOivMIYa62lUh5Y5YhreoxI1zXVtsxE+uj0NE/SoWshhJShrIikaSD3zJOy69bdbXVBnXMAOMxwHMcIw2HouqHHGHvMFFBK6WFQQyc9RFXVNFUdRRGBSBkj+05zgQG0chCEeKu9tV5L3dZ7s3Eq6A/ee4dYVZWbSTF6fPqoqipGcSKi0K44SbI0TQNWnR4dcxY1TSPimDPGGHPQpXGCMdaDHGTvvYUeMIJ4HKdJVOQFZyQAsJR6GBSCXkpJIMiLVDAe7A6EIOCFcwZCaKzu+hZ5UZZl6IM0nTZKy+22AiCsAWjH43Ge50II59ztzUoOum4lpXyQRsTZCHFrPQBwWzYAAGvtYu+QED4MnRBx32vOAcTGGIMpEYLxSCCCEULO6M1mIyJGCGnL0iaplipUAAIA0jh5+/rN8eEhxUQPMhIiLI02m82ub2+A94vFYjaeeAQRgCGupZQHO0LTNH23bds2iiLvYQhetdaM8R0WDsPAOZ9Op9a7YRgAgs659XqNEBqGwSgdpteu6wwhHoPQrjjAWJqmRVGE0BMAMPQySBrhq0gIBUKE9vxxmgAAtDF93xurdnN6mLIJJs7qPM8//PDD59++2G6rvpdxTLMsa+oB4X7HNoKrwHsvjR6GIRGRIZThOxcnwcQbO7Rd6PTsjUUYE4gowk4bq3TQEQMSh0Uyw1JnFxcXoefgTgkPSgOhACEUx/Fd4nII7WdsqCgkBCVJ5D0MPb+TJAlFkgEpwykZ7YKKgPGdwh+gPUj3InTLJiQQO2l0kCh26QCEEAyUKNRTGhNEnVCiyXlE7jZ3ZxMxf9CpaXctcRxHSXInhIC7JR4C9O5IjHPOaPUAd/+gjfT3YDhs5n6tyB0hePjqDvL/s8AP/lFfqSCGfe+NO5LxkCuA7/Ve/Eduyh3h+M6vcF9x+pCuWUKs0oNWvZQCE4QwxNRhSASPlsvleDp78s47jx49hpj/+jef1VVDKbUO9H0PcUi9COeMtRZ42rXDxfnNarWaTFYHB3txHKfJ6Pr62c3NTVhKzhizv793dXXJOYM0H4/HkRAEYdX2fdtyKuIoYpTY8E11Rim13W7rttlutyKiJycncrDaKITQ2dnZ//w//dVHH3306Y9+/M2ziBL+2WefleXqv/s//G/3DxYAgNFkpK3ue5lkaZ5nFnpIEEc0SaIkiTDGjJEnjx//k5//LI1ipdSr3305nRZFkVxcvr29vZV62N/f3z/aBwA8fvIEYzr0crXc3tyuRJR5sNJaJUlmZz5ifNOtvYcIEQJRud68fP7s3Xff/eCDDy4vz1+9eiES0XXdzc3NdltWVaXVMClyLRXnfG825Qx9+83XbaPTfKSF6Ltms1kJRo5PTwalByUHKXtjAaFI6VVZszfnfafSZOyUgo4vxkeCEmtMnibQ26HrEpFmeYJRFImIsf75ty/KsmaCPn33ncePT7flcrE3PT4+juMUIbLZ3KVORlkexxEAgDNWVVVnBgAQhMFFpQAABUkuVjeUkv3JVFt1u1lzyj790Y/qfpBKvz6/uLpeRa3K8kI76BHNiiRN0yxJoyiRUt4sl9v1Oooi2Xfz6Xh/f8E5s97drlbWmdFoJGgcRZHUarxcKq211m3fOwAA8FIrrTUjCDgPnCMIB+hyzi2Xy/V6SxgPOcigdiqlrLVJksSJaJpmW667rts7OjXQK2sGOVBKWRJxTr33POIYw5D+VFaBDnZd1w2DMZhzXqruenOrRuaJeBJPUoedMUoDW6sGeuSoW6/WZ+vLzg92s4HQO+dG43w8GcVxXFVb3jSIYM65Na63KrS+xxAhgGXfJhEnlDbWtl0PrfMe1mXVN21Tl33TKD14az7+6Aec83efnDbb7e3yejKZIOCB91rL0Oe4LEvOI2PM+duz9WYrovjo6OjNmzdK6zuPHgBa67qum6bZlpvFbIoAJLEQgodmABA4p81mU7Ztb60llGFsKEZZWkzGBfR3EzFjBGMcxUFJdsGBRCmN41gIEepNRqPR5aV2zs1mszhOAWj6vn/58qUxhlLWtfLq+pmHGALy+tUZ5VH4Eu7v7//0pz97+/Zt27anJ0/KbYUhirM4LNId9OQQqCmji6IoRllYiWA6nVprrDZhrp7P59eXV48ePVKDXN3cpmmKIQKUjsfj169fl2UZok/tbF3X84MDYwwhLIAWIUQOOkCjMfddgO49Ct77oljc3t6GlT9DrRZh1Prv9sQYCyEMwpRSgrG9X4ZAKVVVVciIh94DQa4PDokA5xj70D8bIZQVeSAKXdcB6LSWzkGPYPgIzjAELs/z6XT6n371txhTznnbth6Stm0HaQjlEGKPsIiisLJir+QwDEYqf28/hBBa541U1lpgHWNMDxILQRGmCFuIgHVBRfD3jQTCyDDGdmtn+/uCpqDAZwmz1mIInLHaOi2V0dp7TwmRvTJOG6W11nVZYYxDP1PG7iLsO4+k3dUH3kXb905P7L2nlHjvAynZdVKy1urQF8GYgHAQftenCNwLHg+dAYHiUEqDGyToE+FmBdIWWieFodtZMcADZ99OOdhF7ejeCYgxDu1E/YNVIsP2sNPD9xINO8D+z2oJu8c7FWF3IQ+P/73D7rgCgg9Ywh8e86G0sHvvbhghhP6BlZJFUW+Vc85DQBmL4ljEEaOcuLydT4vWbC+6Nwuxt3+w/x5+/9vnbyGmPImdA9bqOI1igq3VGOPSLCeH4+NkMWuSels23a3gk+fPXvzd3/6DUur08aO6rl69Uc9fvwAYCSEEjKfTaZoXNzfL2ewwH4+urm5EkmAIPjnYx94hZ/48jryRn//+d19/8+vJ4tH/7f/yf/3gRz9ct73Cjs/yZxevr7bL4+niw6dPOQFfXm9OZpM5z29fvLVV1wNCMFTScw6dpdty/RaeA+Trujza2/viy9+dHB8hXWdUPz7IrbXrN/TXv33R9/3+wRHnOEmKw8PDyXjy4tWbs/OrOI6LoojT3Fm9vL3crK9H8/fKstSmvLm95JRc3a7Xm0obN51MiqIot+1XX35bbfp6O6jWTqfjySi5urjOk9hYqYeeMhyn0bYtiUIkiYBRCrN4Psb9MFi7New0GaMMtW1bVQ2jbDbfY4y3vWRxcmXZZ199/cM/+iMieDYvuqEfhmE2Svq+T0SWiPHbb9+GxCqldH1zEacJ8FLJinFYTEafffXlwcGByPj/9Nf/fjqaPn78WMR873Dv6Ojk4u1ZVVWRSL755pu+GKSUkYg5paPRiDktjWybrdcqHxWTdCTSZP/4eLldX94u+75Ps/jo0VGvpKyq0yeHxWw0KUZVVV1dXwxdr3QjTWu6gWIsnbpcL9M0h9s4mZ1oYzRC2+HiprbK2PF4Sn08GM8I7TtJiaAk8GjjnIYQeGC97hFwiGPXmtV2DQBiQngPlVfAoaFV0losBCSR9w6SzHgXQYKRI0nmosR65z3kNC5GIw/BoJV2zjuCqKCxQJYa20kJnSfrdXd1WzqAL5a3g9VWyWEYKOWtchJEQqTGzKGzMV/AyFFKnTabzarvzIfvvWMWsuu6vu+bptLYTLJs4+zQG8pxkiIibX211Eq1bWutVXQTigua9Zoi/Pz1i8VsDiEEg336zmNVyZvXZ8O6edF/s7+/gM56azABWktCYdtVTVN5j60ByEEEyVhkJYRd1xGKIITAGY6jdDZLI4YQevrk0fX1dS8HYfjF8oZSent7q5wVUdT3QDlTjCbT8SRLEgi9lurycpPEYjGbt3VXbjRC4MmjD+v6VmvY96asb8bTx8pAiBlC7u3Z8yh+39jBOrnYm719+1ZE0WeffXZy/Gi+t9h2jQZGIUtG5Ojx4farLw+PF+88Om3M5sWbL2XbJfzRyf5MSxXl6Y0evFUYEAKB1sp7TzAu8nw+nZdlSTHT0nAabZpNnufz6ej6+vrpo8dqkNbayXwmu14avVgszi8vHPDj8fjg4GA6njx//jyJ41lWcM4HZZbLZXDRxkKw+RxCKLXFGB8cHEgpN9tyW5aEkM36pqmrYjwq15vtuqSIURwhgh1zBAqRxHLoq205LrLTw4NtuYYAI+Occ7aXgPqYcc55XTZt34RClQA5ASOZgB2kCoM4EZAJa3WcpWM9DOc1sKrXxiDIKO3bluLsw49+dHhw+vXz57ebLSGEUt4NstdKGZMJ1Ko+EgmlxDrZShn6GBFCgA2BqZdBUQjmQWtVVRFGAQDbrkUIGS0Jp8a5ZpBxXkBEGI/yYlxWzTAoCHHfS4pJLKKQnTFSYYgYpKF4LY3SJEmapuu6IZQKQIgJddqaTlpjnANUW78sWwKJByB0IAAeEET9nfZApLlbG8J6F4yHEEJHKCTYIlsPNQAgH6UAoF726MGijsBBgigkSHsdU8A5t9Z2UlupelBBYyJCLByklBCALGLOOSkbSulslA1K7Hyy98yYhfLIuq4hwWGxwGEY4jiezWZ91+xMnRjTXX4hkAaEUGjzEFQN5xyAbpdA8Q8sC4GjeP+ddLFTU3Zgv2MJO5jf+TSDL5VzrgaJEcYIAGecU96b0EPau7CgtAPQQEghAoRCAKAwygLgzGA9jCl3CFnrMeVAO+O89wZixCkm2AOvgMG+kwlmxHoBGcMUQix4pJ0nqgcZiRgjRqOvvnz+7Nml85iymDE+KF3XZd/3Ug5ltdFyyLLs4HAOrGmaptmWwILDxwd5nNwul8++/TrLsv3Dg9vl9fOXL6SUT95557333iurTvdy78CnPBpnOfBAD1JCdPj4NE9iwSkCADi1vLpcV9Xlcrvcap4k+wdHpGq+efl2u906BSJMg65+vDf7+c9/7qAZTcZvLl71SvOMp2kKSbhPJji0274JN342W4xGo4O96cX5FQawbZub5e1sb//999//9NNPm6bZ1s1isRiPpv/kL/7yq6+++vzLr65ubg4wzUdFmmfKaK36tikZAo+ODjDG5WbrrFay//gHHxAMX7169ftf/8Nmszo5XLz33nsIocV4Wtd11beck1YN/bbG2BejDECQT0bZuCiyfH9/H2PcdR2EKEsjBMlEG621EBHnvK7bTdtdn59d3Vxvtqv15lYMCU/jLI0oRqpZt227Xq+rsiw3VZIkoXNtWNlv6GVd11VVWXDXKf2f/Oxn3tq26auqur68evn8xcHBEfLg6dOngkUffPDB4eFhU9feAc55URT1+dlkfzH2sBuGVg7BCtAPnXMujaOD+WJTV31dLddbpZQZjbuOrdfr7WptrBqPx6ePn+SjcVVVzjkaxYxzgGBZV8vVxgGfZVlrewDQMMhtVVLCESJ936/WK+B8xHgcCYqhU0oNnZKDMerjxVGRj+eL/eVyfX5+vt5WiOA4SpxzUZpgZZqutcalRb5YLIqiiDIRBTMzwcYYqRTECGLUy8FYa7yzxnsbuskCTAllofpfLBbzx49OFouFVnK12nDOvVcQoqZpCGHz+dw58OL1K4qglD1FOE3TIHVwRiaTSVVV6/Xy8vrKOAcJDLkYzrl12mozDIPWCnngCeYR5zRJk6it6iePT0O0N5tPilG2XC5/9atfAQAWe7NQqVtV1cXFFYQwjuMkThEiZVlrrdM0Z1R0XYe8jyIedF2tVTDYRy7CGJ+dnY1Go5ubm65riqJ48+ZNlmXAOs45gl4ppaWq61r2rdWmSDMAXSiQCylzQkgURTfXYZbERT7O0qKpO2eld/6nP/kZ57zc1hDgSCQEM+BRHKVpHBulp5MJZvTt1cXrl6+ttXmUVKvN31/dxEw4Y2bjCcGYEbo3na2aKkBp0LTCfBrm3LDuRjClhqATQhgaPwMAQr1VeBycaJzzyWQShLGQXw9G5pDzDtWzzjlEqPeec960fciFh6gxiOQ7aA9WfGststZDEEJeSgmCIERiAALGWOjEh8KCyJiEj0PobknSKE5CE4UAGFJKEhNKacQ4IcQaBSGMomg2m93eGGA88A4AADEVIkqSNM2zf/jbz1ercjQaIYwwooxxQkUUZQApCIlSxloPAUYUAgAxJh55DwBGkEDgvUcYW++scUoqjkAoOHTGAo0xIYzz0AbUORcsGuFUTViECd2NuXMOeRC+GyJmweEYFCAhBPAIYtT3MgTTu5DaA++cs8Da+7L/u+z4/apO/l7wh+C7fL9SCoJdQUGIgBEA4L67UUB3uIuJu87sxjy8Hd2vTOHvqyTgXedH0/e9Azh05kAoEDxnrZVa7Roi2fsVFowx/n5R6TurI0C7QFwri+7bLIIHEbnzfxD3f0/w3xVH+Ae9F3ex/kPxYKclfE9+wPerR4IHfaAf7vDwc8F9twQIQrfPu6yEf7Dzw9PbKRkIoSjmcRzhu8pbQIDkr66vN9sqSYvZ/lEv7fnVVdMNaV5YYKXskzR++uTk6PHhqMhGo8JL7ZxLIlEkad+0y9XN8tZjhv/3/93/brlcchaNRvkoz3pJtVJffPGFbzRjbHO7Ojo8yaIYYsoAoBA8/+bZfD6dzsYEoUF2t5vSEDY7Pq5X/fHJkydP34/W64M3Z2Xde+3GWQG9X61vU05+9OnHxgzamFdvztquG8+Zc4ZzmuWjQavleuWci6Ioz9Nyuzk+Pp5NR7Pp5Pnzb9XQee+fv37TdO3sYM9CND84HM9NWuRRlERRtC63by8vXr16XX/79Ww267rOAVuubnTbYAgZBsDqLCYI0GZ1a1VzeXX96tm3SstREi0mo8Vk5L3//JuvqqqKIj6e5MYo7/Rib1JMJ8+ffY0xaprGe/+kqR89esQj4ZzHaQEAoB4klHLOjVTter2pyzdnZ7erlXPGaKkQiGLKEOlsD+/X+BmPx97eNfeOomizLhHBCCOt9dnZWdnUhNI4jjnnk9HYaR8gpCzLtqofPXq0t7c3DGoERvv7+2oysdaFqSFlJOQF5xg774UQVERK6//4i19oazdlKYQYz6bU+9Vq06/WGtm+7z1wlEfaeh7R0WyOubi6upLWYQA5ZdqDTbP13rOIWefrervebKfTaZrm2rhIDwhBbY2FxjljAehlW603m/WqbdvCoKdPn84nc+BJ20vKY23MoHSUJBjTtt9cXl4ChD8o8sfvPEqSZLNZQggRwQCAduhtXUulWjlIo21I51kPADChDO5O6vOU0sVicXp6Op1OL87Ot9syyzLGGGN8GIbNZnN4dLK3N79eXmstMcZpklprGSPOmaYZ+r4Nyi3nLGY0TpMoipIsK0aZ6UxVVQiBWDAAAMVkPB7NJtPL8/O+refz6fmbtwcHBx9/+APv/S/++j+kWZzfb5zzZJmkSUopDXMCY6LIiXdl27YDUsYYHgmEOISwbVutrHfGO2O1EYy2bQsLwChumoYzxigVnNdlJ4QgXEAPEHCMIIKwg1aqHjhvjOn6ZrPZrFarMA2NRqOmacB9Pf0ddiIE7+tlAADvvPPOaDTabDbGGKvN6vY2ShLgPFQWGLu6vF7s78dxXLW91F3KI06obLq+bWPGMcZJcrc+S7DcY4zDai/h8c7PH9qrYIxCMNA0Tdu2SZIIykKyfLNeh5UIQgQmpby9vaVCJEniANrlgL21AIDQvjM0TQqJ6jBrCyE4H0I/hp32HoI5SmkUCYxi54x3xty1H8CBx8RxbD0IMn6IIMO0HkJGY4yU0gGYEXqXj3deSmmVZowVRbFc3UCDnHNKWwDMoEzby7Lurm82EHLGUkYjD4x1BHhotKckstZbbYx2mCLsEHQOOmigAwA44DVwHnjjvDFGaiWVtATt1n400HuCBIaJ98G70HXdMAwhXbJer8H9qkj4vjFRqFPl/O4adwJ76An4j2V/D+4sFw+Jwo4udF13V7SJcZDN7yHT7nD0e5I+gvieU/qQsNgNb/BXovtVoQNRCKmKcHfQfaPJNM+CgREA4JQy9+tr79Bz16FBa20CJH/n5fSh2iJ8EL5r4Pjdsg7gv2Ah/B5R2HEC73044R1ReHicoC64BytQw/uciLUW+n+0QOUD4Ifgu7cEohDyKkG78Pf8A+1uwf1mrTWq0zoO1NxarbQkhpHDvXcnI13WrfbOId7LUjuKBK56aazebG5ZiWeL8Qk/st6eX579+L2P+75f9UvCyN7e3tdff/3i+XMI8Hxv8d50Opstkjxr23a1Wh3uH+R5fvX1WdN3Z8/frK9Xt9fX4+mUcuadMdZu15umadquXm+3bdtaC0+evndD13E2Wq7L2+UWYz4umFEaAS8Yt8NwcXE2GmcfffJBr9Rg7DsffFAZuFzdZEWxf7Dolby6uRyG7uTR6f7+Qskhz/MiH2231atXb6Dzp6en+WTaDPLVm7P41/+wt7c3n+/N9vb3Dw6urq5EFO3t7Z2fn5+dnSklsyzLkqTd9PNpgSFSSllrDo8Okij+4osvfvfrv682W6XU0eF+zAVjuKs22+326+ffzmazxXhkobvdrIHT2svr5TVnZG8+LaYT79zB6fHp0yeU0r4blnVvjMEIxQApL7u2LdtOOX+9vNbGzOfzNI0pxZwh6HVfb6uyads2y7KTkxNvwfn5eWiDIwftIYiSeG9vbzQZW+CV1k3THEwXT58+nY6mWuvJaHR9fU0JF0KkaXp19eL8/LyXQxzHECJOWVVVs/l4uVymabq/v2+1vl4vGWNRFD158kQptei6JEmm02lVVQKivu+brm+bhnPunb7alJRthYjLpnaQ9NpRD+K8EIx0Q1tutlVVYW7Xq8319XUoo91syvF4fHR0dP72bGiV7junZFs329V6s1r3bXsTjfb3juLUWgD2Dw6zrLhZ3n7+2Zdv357HaYIxPTw+5lE0moy11pvNxmLovXdahWouaY0FHkAAMUIAOuPDUu0U47sZQRsAAICOMR6c8BDiJM7koJ0FeYaNMev1JcJ4NptNp2NjFQCAYbJcLq21WZ5U2/Lly5cnJ8eTyfjo5IhFLPTSEUmc53m72mrZWY0gxNYYa5V3WjCMMcizmGHy9vWLxXySpOLs7Kypt1mWiTiGGDMRH508ms73AjA7525vVxzT0ycno6o6e3vhnFscLNZdG2YGS6CGADmrurYtt07JPM/L1brIcuwAduCHP/jo+voae+eUVEqFHrQgTiiFgCDgvHNGazkMgzHaIwAhtNZE0STk8rfbLcYb50Cej5xzZ2dnhBClVFU1VdVkWWatN8at1+vQV1QZ44B/dHB0u1x2q+0syQ+fvIsgbKsaGkcoZYSWZYkSsatPC0QB3XcJDL6BcJsC7gIARqPRdrsFD6zywdBXlqXWOsB53/d92wX0CstdEiaEENPp1DkXsDMsJpemaXBgSKWHYbDWBkRs+64fVFjsIESofd9TStM0juPIOaOGzhlLCPEOBrDhnA9K930fyEc47eG+28+dpIywGnpHiOLMUa0HqbXCBDoH4ii1lBmjvLXeu7ZTV9e3AGLryGgyz/IpIcQDqbWGAHuHAcTGKK0MAAgBDB1y2lrlQIysczqUFjiHEHIIAowIZ4hgiBFBFHoMETLOSq3uxufeIxJk/Jubm2EYgPvOPUDv6++VuqtRNMaEfpHeQYCgte4hNCKEAAQQQheWwXhYKXBv2XtICOD9CguUMe/+ICYOtiTvvXX3Cy/dr2F958OwdscjQ2mMECKsg/XwlML3ZDKZBJi31mprQbBSIhqUqjuLjFLhq0jQdytfB7Oquy/mDI2gwP06TPC+h6Pz33kMd9f4MHbfXVf4UX/X/emeHIRtpzcEorOTCgghgb+Cu6H7TqL4A1HBP5AW7onCTlSw97cDobuVO3ZcwaPd8mAeQu986MnmyXo9ECaUIbeb7eCa1pjBIgOh0noYJCB8tpjn+aiqmo0eEIC/+dvfxFn67NkzpdTPf/7z0XS2/E//sNlsEKGCRW/Pr9q6iUR2uBeNshGC6NNPPw2OJEgwIQRiVDX1m5evD4+Pzt6+Xa3XTdcb7wilIomjKCnGex6Ll28ublbrXupERIhBoAyBAHHa1tVnn/3un/2Lfyqi6OTJo8XJ4f/w//13AHiEYd+3ddfufqgh1jHaRVHy+e9/V9fter394IMP9/cPCWHGmKurm+vr29nsSgixt7dntO67zlmbxGKSZ/uzaRRF2+UtH4339vZExAVlWus0TijDV5fn29U6inmeJd7Ydbs+OtyfTkZD3z55+s7h4eH+/sI6lSTRenO73txslrcnp0ciFsD5pmmMtVyIw8PDJElu28FBgAAw3vV1u7y+Wa9WWuuwoMbJ6WkSJ5jhWETGWcH4681bhNAwDJeXl9fX1+FHu16vrfFN1w5Kzhb7Qoj5fL4ty7quD/LpfDSBDi6XyxAlK2mCGBvsV9ba2WwGAGRCVFXVqlZK2SnZDL0eZFjRZzIee2BHRba3mHGKMcYRGy3GOef812/fnBs3SOmQJ4AwIhChCNIoFdZqhFmWFeMiqcvt1cVlW5d5wZmHHBMrlZXSqcHK3msp21o6562Dxnnr8zjJeOStOzp57CGuqxYiNsoSEUVJNxT/f7r+s0mybMsOxI4+V1/XoSN1ZlU9/YDu12hMA2hg5iuNf2HG+Ov4gUYBI402BAYEehrT6O6nSlfKyNAurxZH8sNx94x6PXRLK4vy8Lh+5dl7r732WqOhJaTve21VMkjTNMWE5EWhlFLkT58xwuj+HU20AdahzSVCsu8l0A5NNEZlmwJqSAh7/Pjper2uqkopgxlVSqzWiyD0kjRyAnkUwSgKMIYIoTiOptNJHMdxEkZxbBEs66ppqratjVF1vi7Kdde0Dnnu2hZqEQWsrrKmqhBCnKEw4t9/99X19fXnX7yQyjx++gghEiUxxKTp+iIvEUJxHDetkMpyr5RSUc4YY0mS1KpXSkEIAp8jaCGEQgij5XpVEQwBAIHPPeZHUeTzoGv6xA+rqsqX67LKfd8nCEjuOSEAY4wUvVZSKeVaD9SjWZY5Lp4L4ePxOEmSPM+n06mjvFVV9e233z579kwp5fu+EOLm5qasK0ppOhxGUbABEFtQ58VkMIQAcsqiIAjDMI2TPM+XdWGtZYy56+UEkuu6duWgu72rqtrDGI6wySkbjUZRFCmlZC+klIvFYjadDgaDPM+/++675Xwxm80ODg5Go5G1VhkDAPB9H0IolHbdOuYFLh2x1gKIHGTS1h2lVFe6rmtpLETYFa9ZllFKlYqR80SWn8bonQ6B+3mLVBvgClwLoMt7tks2Jg401lIigK21EACjbN/3o9EEAAOsVUr0fW+Mqpr+9n6pLeJ+ZBEGiBEGhQLc8/wwFm1nhBZtBwCCxkIGEQCcUusRp/bo9opSahF0KkAAbU0HMCX72tdVxvs4ASFUUroA5mZM9lF2j+1DCPe6TE3TAIswJUoZo60BFgC9r4m11kbvA/2PamWEkGuPAgCAhft9wNiaH2cSxlitNefcyVzCB6TX/V0BPykTA3cXOT4p2rs/g60APKXUmYRZazGlO26BbdvWWssJd5932QClFKNt3qOUslbBvUwCwtupVAD3dAQIHSf4ky7TnyQH8EErwe4ZErtzsifGwh+7ae+zBPCwswAgAC4ZgxDCPxFScL99kDfsEJr9B7ZnDO6hO3f2fJ9rQRAGAACILEKYEEQpJd+8fh8msTC2VZr6IaFAdLKsquPT48164VP05PyxFvBv/uN/zZb3J8eHRKIwDLu+55x/vLjLsux2vg684G//6z8+e/LEGkgQVsoGPDyYnBwdHc0Go8FggBDCGEopCUUIoTdv3y5Wy6uPlzcfryFlk+mM+V4v1LrJY39iEa9q0e9UUBgm3CcI2NDzoFV5mVkIV3mhjE3H49FoQCmGhH68vCjrhnECAMiyTMre9auSJMnzIgoTNxXWtb3oZV2W7jIs7+f5cv3dl18/e/ZMqv7V48cvzs/n83kUBRDCiLEf3s0pZx73P/vsM6v1mzevq3VFCKO+39VN4IXTw1m2WittDw6PX7z87HJT+r5PKQ4jj/7spx/evf7t7/6bVXI8HBGEN/nm9vZ2vVopKa0xz54+9ynCylqrdK+W9/dvf3i9Wi4xgP/yL/87z/OCKHr37h0ASBADAEzi0fn5ue/76/X69va27/svvvii67r3799ffPgAEExAen19vdqsLYKUsSAI7vFdEIZlXiznC8bYeDzWISjL8ssvvzQGAIyKuhpOxlrpgJDheKSVHA6j+Xx+e3v79OnTn7982XXdajkfDBKKCcJASZHndVEU2XpTFIUgvMmLqusboauup14wGE14GGHAjLXAQIpwEqXT4WgeR2WRy7o/mh1wQhGCDKDE87CR+fxO1AWygALiMT8dJqN4EEepx1jA4yzLqrqP41hIvd7cCa2evXgJIXz34cPd/N4N5rjbnXme7GpCKd5ThCBQSkkhGGOEUEAohJBzTgghAFmpsYAYY4qgErptWwzwMB0cHRyMRpOrqysAQBAGCM0sNMaoIAzc0O/zJ08fPXrU1JXWihDy/MVTt0Kt1otOCLPV/Ndt26qm1KpH2GJoMbKcIsoQJVCJ9ubmkmD8q1/+7M//7NeXl5fBip6dnfEgPTs7W6+z+/v7i4vL+7t5FCWTyUwpkCSDuq6//fZ7KfuDg4Mg8JqmQkCLrsYY+77PcKC1xsAO4qiwIFutnz59qoUJguBwOrt49160XRwFZZEp2TNCA8+nGDGKMWZKbaWBlBJ1XZdl6ZDVyWBMKZtMZmEYO8CWc14UlbXQGJCmQwjxfL6cTg+stZRyhIwwWmo9GI04585VYTab+b6f53nTNOPhKIxjKaVQUkMAAHBdfIchuxS273uXHHC+pV84bNnzvPl8vlwunTEjpbQsy6as6rqeTCaj0ajve9dBd6GIc35+fp5l2WK1cYrICCGpjSsNITQ7aV6t9CdxnjAMy7rSujAWQLAFDHzfd7baSimtZd/3ZVlWdZHEg73l8T687Rd9h3DoT2YB2uee1hIYDQxCAFqILTDWQsbYbipPu70CAEgFPM4Z571QCFOMsZQ9pRQhgDG2wGijtDIIWoKhU2itgEIWGIQlxtZaF/+VBUoLKbXBmBHKELEQaK0tsIyRh8U9pZQz5uKZo4tuuz9gO3PvjtTJEuxo84gQ0nWNNcCJiFtrjQHaGgCA1dbAbarxsHjdt3WstRCifTjUWmm1/aJdv8BIKaMoMnob3h6yT+I43rMKtvmN1o5VsC2RrXVH4ZIkR3xxJAzXR3A+XvucY38qdlxF4ZgrlFKtrZtegRACi3ZyEdtjcVdfG71PC/Y3A/oxgfEhALBvJex/6/Z5P3axRyPsrnfm9g1seSBgn3/86PXQHuIhxvBjaWeMP81JuvcRQpi4KZ5OCOZzBKFVWpB/+Oqr2eFBNBhCyvqu35RVp3SUxBeXt2W+8RCkFocMrW9XXdV2QU8Azdbzn//859OD2cWHqw8Xl1qhOBlZwHoB//j7358enf70s89Pj09E15VZ9fbt+zzPLz+8b5tKK/H06dP//q//Ogj8L159VpdNW7UG4SgaCG1qIVVvayCFhnVb9aq3QKu+N5RQxtyJS5KE++yHN68X+ebN5UWcJtyjdQO6vp3P5waAdDhUKi+Kou9bj/MwjCnlwKIoCtqqev/uQmpQZkWWZRjjwGNN182vbr7+wx+TJJlNxn/xF38xSKLr9+/dLVht1tyPMeObqirrxlr74eO16JpOSs8P5qurKB188ZNf3N5eX11eNr14MjtA4eDo6KhpK6X68Wggu/qbrzklpMhyBEGaJLEfKKWKTfbl7/+QrTZJEpZlDYyllNbrrK9KYmESxeenZ0EQFGX9/v2F54dhElNKB8PRwcGBkwmjhIwG8NmzZ47wdfnxmnnc9/26rlebtRcGk+l0S8kxJvSDQZICBH3fr+t2tVpVVTU5OFRKLZfL2eFB3wllNCEkDcPxYFhmeY/w47PzJ0+evH3zQ13XZZlrpRBChGDGmBf6iZWtqD98/4EwbpXuyiYvq7y99aP54fHJ2aNzgggFBEiDtB3F6fPzx1KKrs7DMLR97wXecJjezoXRKmbMDgYYQIZZFMSjeJjGaeRHhBANqJSq6zpljGpE0/WUszCIsrIYjIaUM2W0MaYTomoaXZZRHCAIEYTGPXXGWqlk2/V14/hlAFgoFMOUeAEbQgaptbYjVPYKQ6KUEkIZDSjhg3SkjaQeRQg0Xd2LLojDoijyPMMEHRzOsjWtixIjEATBcrkUoqvbppeSB1vMvGlq27cQWoIhhIBiSDwWBb7nselszCiOw+jf/Jt/nUaxz+jRbFrXdd52t/d3FxeXl5fXeVZQ4p+eTaezw7puo3hI6Ga+WDRNRxiN06SqqtV6XhU5pZQRrJRp2xYAEPoBxaRpOopJHHppOuza9v7ujhDSVXVfNwzBdDiO0sQR9MIwxBh7HnOY7Xw+70VLCPEDniaDoiiU1Jx5wMI8z6Mowojc3twJIV68eHFyfHpxcYEgNtYwyjvTxKNBEIbhIEEI2caGYejUuhxYLawuu2axWLiFyQV7FwlcBx1CGEWREzjaDxk6ho2raN3S5rjobkm11p6fn3PG3r9/zzn/5S9/OT+8NztHADf/5kIvQkhb4Frv2lonA7BHFKy1cRwz5gkl86LqldbG7mlATi+hqiqtJTDK8Rg4850KEMYYYrLNbEy/zUsw2YcNVxwDq60xWimjnX2XxQRBCPteEmIJAQAYABBjWxMKL2CY0TLLPI8RSnspcN9K6QNgKIWh77kzwyj1fR4EQVuWACKKiU+Y1hoBpJQ2UmILtLEIWqC06renwtEDlVIAGHcajTHY2TcbA4zdN18cf8VaS9i2RiduqoIQN3ABIUQYWQishS6Aoa2foQW7ih/uSIXAeZXBbWTaqxFrrQE0Lu7vC3GXFX0KkA9GEPcokXayFkp5nudysoef3Ed9a62QW26sy+FckqqU2mkvfhKDcihCU9fuRt2nI9sYvwvq+3Rk2xrQ8GG++CMM4MdTjg9zSr1zsgA7/IaQLTH24f0DHthRQuQ0G7aND7Pt8Owyhh8nChYACJHLFdy23G7vj3T/FUoJhBCEVqpeyh4AzxjTtjX52a9/+ebt2y/fvE6Gk0cvXkSDIWwbCxEiNAqTJl9fXFw+OTx49fQzoCQEBgJa1/Vnn/0UIPjb338VBHEyGF9dXb18+dnp0XFTttCC25vFh3eX89s7igkMUZUX69Ui8FjX1BcXF+/fv4/j2PeD5XxRlaUGuOuktph7wXg2aS0RSpZNCzGIohAowQmmGGst+66bTafD8eEPP7y5Xty+u770omA6nV5fX3dC9VIEUQIAaNsWQAiARwlhjDkDNAdgvn79OghiAABByEhVdG2e51KIJIk1Z9lydfXh/T2lX/7hDxDaR48e9X2fjk8mk9G7N2/fvr8AAKw3OWcEU960vR+E48nUC0KAiQVovcn/+OXXWdWcnpxEQXh5tQg4i4JwkKbZZqWl7JuWIhz6gaMLlXlRpeXq7qooCozxdDwjhJweHMZxcnb2aHFzlwxH88Xq229+GE8no8nMD4MkHS2WC3e/zmYzLU3TNEVRuP7rJs9ubm4I86YHs9PTU0LpYrHwBcAADsfTNE0hhEEULpfr+/n8/Px8cnAIAFhnmyiKrKmKuhJCJN6j0+OT8XCUF5ujw8PNen17e+t5XlUV2phOdEVRdF0znU5Pj0+G4/EvPv8zgPAqL4pWGESu7tfXizllHqceYyT0fKuN7gUl5GAy9TiDWt3cXOlezM5Op9Ph3e0lMurJ+VkSRghCCqnHfJ8ECCCggVKmUvbo6KSq6+VyqYwZjUYG2PV6jSiJ4wQRvFwui6pyy6VbxxljBKL9cuBRhiy4vLzEBpAIGmMMkhgiz/PiMJJGK2mgAZxaaFHf9G3bFkVZVZXve5xzoURd15t8HfR+lEZpmjRN3bbt9fV1U9Vx4IdBYIxZLueuxqLQuDKlapqiKGBX7p9w1QtCSODzuswpxX/+5//8YDrzPPZ//3/8Xw8PD3/605/e391QP2naqixzKSWl3ONe27Zv374HFmKCtip4WldV1fc991jfNXVVMMY8xruuy/OScx7Hicf5IBlm681Pf/rTwWD0D//wD1Ybofo6X+f5xvVxGSZNU7vJgaLIXDeBEMI4iaIojKLpdIokms/nrqNvrd1sNqenp2EYWmsXi8Xp6emjR49ce9h19xfXi2Q4YIRKpThj6WhorRVC5HkepQn1uJRytV5neT4YDTnnXd00TWOtdUWqC117BR7XwUU7fqJTNHKiDs4Cyqk7UEqvr6/Ho1FVVW7COVtvNpuNlPL6+rpt276X+zBj7dbkSagtBc/1CBwakaap74d124ThCnS9kE6wB9R17XkeAEZJQQhKogABKFW/p8QrpSgmrqvVbxWUP+nzuKrUQtRWpSsfjTFVWUIIPd8HALg4sW1SOFekLcAulBJtW+s0DlmIMUQYME7aqiYU+V7i+z7FxG7VCDAnFADAMSE7ejwUQlkZ+gE3WyknYKyyhhLie15R5Fprhokrr6WURmsXpVyCtW2QG+t2DxHQ931d1wCjhzMFhBAEsYU7SgGCwGitNdDAbn2Qt+oOZtd6sGjX0UAQ7nQRGHdTEdsuCUKIEOsuPQQI7QSe9/CPi/F2J5a1B+0d2cKlknumhTFm713p8AC7n1jZ6bIYY5yKA0JIOGtmQtBWT/qT+JLTuMT7GYRd+Henbo8i/Ela8LAN4V5qJxvlTiZ6QC180LX41LDYk2SB+aSa8L+LKLiX3X0CgE9ZgvsB/njYAeygFMda3bF/CKFIG0NmsPrn/8O/+Jv/9e++/OqHUvWTw7PJYFzXYr0podYjHAYeW95lOpUnx4f3N9fA4L/4i7+4u1u/ffv29nZuAZhMJkCDIluT89Ozx2f/+N/+3mOcUUJ81PUtEqO6Jb3kGOGD2dnzx2eXFx9/82f/zCL7+PykE+37y5u+VV484HHI06hdrYrVMkDAKMkVSdO4b2tErISWJAGM+eT8CEBzO7+2daOEuJUQk4ADFQaJHwaEkNFguM42Xdtyzv/h978/OztDUXJX1nwwIYwRwiil7XpddgVh1ASBRED5XuMF3mh0J+HybvndvObcX8A82xQHM6C+VR73v33zIUmS1kDOQ6sEBAARfPnx4n/OVxTa49l07AMfNDphb95+DaFNksTzKUSDk+Oz/+1//Vvf53Ecd00LjAXQHB7Njo8POaGlj+Mkads2y3MlpU99Tvn97d1yuQ7m8w8XlwGhIeWfP3s6m816qQ9++i826/VqtSDImx4NVqvV4n7JOeecnx4eNXWntX48Pno0Pl0sVuu3d2/lt9Pp9K/+u3/94sWLpmmElTxiyTDhATdGhaEfhxG04Pzs9I9/+DKJ4yLP5/f3vs9ns8kf//jb45OjtiuiJF5eznupjYZCKIwiPz5Kxk8498tieXx8jDfZEcZhEg/fvfPfqbIsTXP/8snPPc77tv3w9ffWwudPnx0dHM3vP0wHB+NfHQzHozBJN2tzfXuTjl6EQVyW5XKzuXj3g7X29OSEEep5nrKmqiohu+Vy7vucxX5dl51pmKZ902OMzw4OyiC4v1+Itg78kNGgKisNIUDIjWL7vm8hmN8tRoMx5Z4DhDshNoul53lUIiC1hwhgACEU+lwIcb+5bpse5MAYM5nMuBet3l5Upfhnv/pLhpYBD5u6e19+PDiY9tKsru9OTo9nBydCCGs1aBqPelrrD5cfsmzNAAEAzGYzrSWnkHPeZpYcJBSD2egxMvBv//M/5CtpRNbX33qet8jqKIqOTl9FaV4URVVVP7z/frVaHR4exkm4Xq+llIeHh61qv/r+m+Pj49PHpyxgTd2xgI0mw+PTI4SIzz0hxHK5no4HBNl8M8dQ+xxPRuPb+TVl4/v7+8Ewxhh3ff38+fP3799zSss8n00mTV3Pb+dJklBI7q5uZ8Pxsyfnq9WKYuB5/niYeJ73ww8/XF68Qwi9e/O91WI8Ht9cXXDOHz9+bOsOchuOotFwpKzJsqztOkopYlQqNRgM2rYFvTgYjpMwIoR8XK+rPGOMRb5HCJnP546GxhhTUjSiJ4RMR0NjTJ7ndZHzwP/ss89Xq1WZ5ePx2BnXEYRWWbZYLCmlQqqLj5frTdY0LSGEB/73b988f/4SQphl2XA4zIqKYmys5b6HnH8BI74XwBoVVZm1xTxfdlIMxgNQlLDufIQxRIHnO4IkIQhZZBBFHjaUa9FrBBACliDqUT/yIYStaLFxoLfck93cihx5SV3XQEGKqU8NAIAjbq01yFCyDXgEkoAGxpi6rM+OZtfX16GfMMPzZRZ5fhiGfVNzTgHgWuuu7zUFnhcYC8ta+LGzYyY+4EqppumE6jSQ2mKAIcIYEQwAxMZoY+qmgQpyTDHCVlsMkUe5Mbora89zg8pGNF1nFKU09HzOeNfLuuoJ9rgygR+JXuV5HiWpAxWUNkpJhBBlFBioGi2MJohYa50uEyQYQmggABjugzdAW5MFqaRBjFIKjKm7Dknl+74xpu5atmXNEAoJdCMJRkktbUUQ6j3PC9ORrZypmw6CUBrTCSE1AIQDAHoNTCeZgcOUQ6it1Vr3EGIAFSYWWKC1IhQIJa3VnkesNXWdd6LX2rStm8FhnEMIIYKEcdedAW7wUCnjioSqqhACFmoENQQG2J1Eo8V7RoGLzwAg94aFZoeawB02AAAAQiiECELOHGs74Wh2WgsOi3A5EGPM8zxrlBKS0i1SouUubaJESaW0RphSzgjGQhmhJLQAQmghAAhaBI21FgJAMVagLuvJy0eDQYoQIBiKrvPjmCyW93/93/+70XiMKfvq2x/uv1ql6UxZiyAZDAZaaiMgp3izWkOtgsB7dPKka9qsyG9ubu7v7621AJokSbRWWso4CZ49f/zNV18mcWyt5h4u67ppSy37aDw5OJxOp9PNelHU1cXFRVHWCoAXL174caotLrsuyzb3d3d936ZpysMkDH3GmBIdhND3/dPjo8ePH49Gg/u7m72cVr5aMcYQIVrruqwQwRDCQZJ+vLp08AshJFtvnDTsPtPCGFFKEYaMEWu5wz+FEDfXd34Yvnr1omm6sqxns1kcxzc3N3VdbxtgRvm+H/iDcZKUm5HHcMgJMmo8myZpCqxe3S4/fHhXVdXPfvrTOAoBMD7nR0dHQnST0VgIcX9/FychRbjYFGWZR4MhYywZTNLUGmWAsQQzhGkQhpuiNACdP3785NnTyeywaGoppUTo+voaQjscpsYY17i9uLhQSgVR5PGAEHJ0dHRychLHqRDi4/zjYDAIPV92fb7OXDP7YDq9u7u7v7m/vLzM1ysM4NHsoHveQAjfv3739VffRHEYJeHBwfRgdlzk5WqzzrPq6uYOAHR0eJoMYyl0WdYY89D3i6KoyyIZpG3bZlkGoR2Px5PJ5PjksMjKRVGsF0uC8PHhEdDm5Pjsw+XHNE3Hk1kj+8FovC7KL//4rXPqW9wvv/rm21E6GI0m0CcBZrqvv/vuO9G3s9kkieM8z1fLeZJEgACGie/7STJIksRjftM0jHJEkWNoGmNKQrbgJEa/+MUvlFJFUbimZtc00No4DD1I+77vus71xbeqhYQsl8uTkxPnKGitdaoJNzc3UezFcTwcDvu+xxA5/2IXhJRSnsf6vm+axtWgq9UKG3RwcMA9726z6bru7OwMI/T2/bvRYPjDm9dhGAKE/DjCnGlriqrshGWMuD6oq4Q8z3MmCJzzOI7Lsmya2gHpSZKMTqYnJydGAwghMFYIYQyQUvq+n6apR5kjGPq+72qR8Xic5/nR0dFsNpNSNk3jVHvn87mL02VZUkqPjo6CIOi6brVYhmFIONPArvMMQnhycnL2+NG7iw9lWZZNXTb181cvz6JHWZbdzu9//etfr1arsq4XiwXAyPGEXAXveR6CUEqpjdRa930vhAiCwJ1kN6MIIWzb1kks7zEGV59Np9ODg4Pb+b2rvQaDwWg0yvO8LMvSGGtt37R45xPh8AbXsQZgO9HnWPpt27qijcJtzdf3vRTKNTVcpMGYBkFAGK94I4SEmLovxRhba5zh4cMizKHffd87r4Q9E808mBvc1obaGKsYZZRiYylCKIoijLFbYZTSEFmIrDYSAEAZdi4MGKNtqmGgUsoY5z1BAABGA0KkUoYSZq0NWez2SmqllNHaGgOsgVIqgNGuusQGAGsNhBbuBCTc0Kk1RsrtmIYxxun8uw32ShoImq6Nosi1k/KsXK/XDjESQhBCINw6MO3LWvRgfg9CiHe2kBZ++tX+5KCdDOLD1MrdPw6id9sxO5cvhJDRdk9XNA9GGffsE4dOuf1x8M++4of73geAe9rEHmRytJhGt/sDcgil7/sYUceH3TFweynlTkoSPLzce+Rg33nZ/XZX8cNPRISH/YV/CkVsz4y7LLtEwe5OFyHEavMnCMEeingAMWwvhMc5QohSh6wYR35EW0aLcfoivu+CJtBaE4Q1ROqXv/jJcDj8T//5v/5///PfSlXNpkddJ4ZJuFgsqqz2fb/YbBiCj87OZ+PJfD7v2y6Jw2ePnyglCKez6aiqCu7Bn56/+sXPPsvze6OkMejJ+aMsQ103a+sy5BRCW/eVH3nj6WSdbTZlEUbx8fGhMDArSwQ1hhpj63ksTWPf9z1GIIQNMBiRxf088Hjg+Xm2Xi6XRVEopZAFEkBCMNSg7bpW9JSzJEl83x8M0iAIyrrq20aIPmIRALZpakg15wxC6MAFp+8bBEFelRiTdZ4dHh//5V/+5evXb9+9+/CrX/9CCZll6+VyzTk3Vjsn2TDwhFbKGqlsBywlgDIPUdaURdfW93d3q9Xy7PSEUhoFXhB4GOMkjp88eaaU2KyWw2QItppcsOrEgPlBkvjM10oZaaAFhLC6V9c3r4uqCuPYi2I/jjZlRf2AUnpzc1OW+Wq1SNL47OxsNBp9//33g8FgmKYIEoegrtdrZ9p2cHBACK3rRsrb1WoFIQyCqK5ra23dlMaYKIqs1QCYNI2bpvO8aLFYZVmBCJyMD+qq7zs7v8/axkgBlFJXV3fXV4vZ4dFwMJs8m8p+4xqZZVmuss3N9bU2BsLm0dk5tKDMs/n8brPceIxXRVZWeZym2aby/LiTqm3k6fnTIBr88Pr1//wf/tNnn30mu76quzRBq2X21e23GOMXLx5RQrrWvH/3Toru1auXAaN5vsEASm2UkMDY0A/gENSMAwAJp8aYre8c3PYLoQVJFN8v5thgionLllw3SiPmSM5KqTzPr69vHT0lDKPT01Pf99frzBjz6NEjsFN9qarKKK2UMj631lZFcfnhIk3TIssRhH3fl3mRpul4OGrrqq5bwlmYxGFTt12njaGet9lsjk5Prud3URT5Pg/TGAAgoa37tigFQshRHKzVEFrOeRQHEDkTXl+I3jX7EUJtW6/X2Pd9Rj2lVN91QggIMYbIiRFBCF30cqtJ13WYwaqqBoMBxthxv5fLJca4bVvO+Xw+b9v2/Pz84ODAtW+dUN1ms3Epqdugtfbg4MBNySqlsiyLomg6nUIIjw6mfd8v12vTWy8MMMbgAXbqFn1kwV48B+50e5wpjMuQXIwHu5k0RzRzsBmE0PEcxweH0+nUkQrdI6zFpxl6p6gzGo2CIBgMBq517ZoUrjvwEJTum7qpS4dCKymFEBibKOSck64VQshPYwsQGmOllG2L3HYY9RAklFIEkehV2/TWg0YDYBGwAEGCEYTA7PIGK7vOjd7tz4bLh4wxW8rCrjvucOysLDopCMCt6I3R0mjVd1q7I0UAgF5I0/WtkB73Mca22racldFdK/tOaGUgRBBiANButBJaAIyxxmifEifS7w4QEYIQdIfGFd0ZW2v4YNxOSrlcLuNBSghxNwamzGWumGwFmrTWQjulkm3SsI++Lmw7McRP2Pju9KoHgwz7b0QPdJTd1tyb1tquFcaYveIc3hlNuRvJ9RHgVo7MUQ6D/XgnRts7EO/GJR6Gc845YVT2yjlNbMciODfGCFH7Xqi1JgQzFriHghDEWJDn+T71QTvjKGuctNSnkc79MZr/PWPJP8kbHmZa+6cJwk+hHWMMLLHa7M2393/18KTtNgL2j4nTazdGWW0opb7vHw4CKVq969EobSkGCBFy+uhUyp4y9IuffzGZTI4OZt99+7oT6urqFpvWx5ZQGvgkZJODg1nI6dXF25ubG2UMpng8HASRX1R539ezg2E6CM5PZ57HPv/i2Zd/+EPbtc9ePf6zX/5rIcT8/rbYbH74/lshhIW2Fe3kYLIpiyhOuc/urq5X62I8nYxOD0POmqaJ49B1hozSCAIAAEEwW2/KIvMoS9P01fMXR7MDpdRSCillVVVS9qHH/Sj0A49Scngw5Z5HCPIDL+78MA4QQlXZ1UIpxV2GqJSgDBNCEQZCdIQYhAGlpOub5XKe5xtrdRR4aRrXde3W5b7vN8UGY1gUWV9XHCOPk0EUGgC0AY2U1pog8MuMbFaL5fwuOD8PPJ8gfHh48OT8EaV4eT8/Pjq6vPxIEB0kw3ndSgOlAlp3dVH3bUcpjfxIaqUAkFov16vlen14emKx1VYxxqbTadNUHz58ANB2XTcZDdM0dXaI7pGo67qua0r5cDjMmkwpVWSl7/vAQGNtkeVZlsVxHPqBd8K11l1bf3j/1hjQNA0hpK7bvs8sAN98/cP7d5eXl5dZkYfh8NXLA0LYarW6u7tfrzZffvn127fvj2fBaDIGAPRKirY7Pj5MkmS1zu7vb7USm83GKh2Fvs+4UmK9XFzf3i8WKwvROiuZx8+fPomjxPPCs+Ozk8OTm+s7owHGtK7b+Xw9Ho9F3T5/9GS5nP8v/+k/9E39L37z5wcvnl9dfey6LluvRd93deMR6nPPaiOENMZACxCA0Do5eoW6DiHUGGO1IZR1XUeUGg6HEMLr62vmYwBAFEXunabpuq6L4/jZ0xfPnz/nnPv+vYtbbdu2bZtEseyFBFJrDYzxOE/TNA7DzWo9n889TpfLZZ7nLhJoKU9PzwEAxoDp9CDPy3VenETx7Oj46ctXb77/wVjox0mvTdu2GBPEuLWtMQoAgzGklAJoILKUYaUkIRhCHschhNBN5fi+75zYKOEIob7t+r7HmAae3zSNtRBoY4xxtbtbT4XWdV27GO+Sg7quCSHD4dAxBJ0fkpTy7du3bdt6nnd/f79cLieTyfn5OSFkk2cOLk6HAz8Muq7Lipwu2PHx8fHRUV0WThUxCgMe+E3TZHnuntA4jn23mgNI0NZ0x1XSLglw8M9gMNBah2HoCOou1dNabzYbrfVwMnb5DSGk7/s8z4UQ/o7CBncCNQ6S9X0fABAEgStz3diCS0H26yZjDPed62e7GNZ1nVKNFBogXFetQxQc5YXybfjZztkTQim21jLGHJ+u73u4M5PcswWt/WRPrJFwsx3GYACM690rJRAC1mprNcYYIeCimnMiCMMwCP04DgEwQjoTJtO2LWMcAKCk6ftOKaOVYYwBigihCCGplMPJlDLWQs/z4E6BGEIM4NYQi3NurRZKSq0wxpwyyggllBFqLTNbw8St+qHW+ihNiqLouu6LL77wvdCloUK5afstNW+rJAEsBNjVo/sztq9r8QNHIrOznkIImZ3mhLs99lDBvvluHoo/WisFMLthB7LT5kIIufVwr7MJdyOde+7hwwobwK3Fhksl3cNCKWWcDwaJ7/tSSqWEEApCK2Xftn0YhsYqAKjve8ZoKbdTo3vexgNKwJ86O+xhEgihVWYPOTxMhtDOC8pxBdwHtNbGbk+OU7mAALqJDGC13mEkDzOzP/l2CCEAn9COh3gDpTQMw1cvH0nRAIwgRhaCvpeUUogRGaQjhzHe3t4DbX/1s8+Q0f/49789PxjEUfTq0fFkNuWcO7jp7du31apCUElRAcRHw6Mnz58IJTf5ejweYqhX6zlC4MWLp3/88h8vry66vrK2hVAFHp48O2UcVGW5Wq3my/v1ZrPaLKnvOfw/CnkaB2kad03di8YajSBhlBgMCIIYgul0ioBpmkYjnSTJZDRumqaqKlzly/WqKArf8+IkiaKo7tpNloVhiBHinEdBqJRy9KuGMgwRIYgyrJQSqkeIMraVdNVaB4EnRPfdd9/d3Ny0bf3999+P0wRDG4a+w2ndjdj3ve8xZQ2BiFDuBVGYJtFg0BuF53cUQ2PVcjn//ttvZN9hjH/6xRfj8ZAQoqWejmdHhycfP37MNzkhZDAYOAik77q+b6uq5pxjAAEwz188y7Lsw8VF05YWKMbwcrnyvfBXv/rV4eHst7/9h6vry2+++WY8HKRp6moyznxCCMXOwifAGDdKRBFL09gR2l2ZSCnN89zs+L3OSSiKgr7vMYrdo5Kmad9LY1CWVYR6Vdm2InPFuiv13r59vVgsXj6bPX369Oj4eDo9SJIEYsT9AGM8n8+ttR4js+mYEwohJgiv1+v5YhMEwXq5yosPx2enlNKLy4938+Wjs/M0Tu7gfci98WCIAPQpPz08noyGnPHA96MgLNabr3//R/vFZwfjSZZlsu2gthRhThlk2BpjtZVWMUwcOUvuuEgWAt/3rdYWgLZurLUe43GaxHFMIXGBKgxDrXXb9n3fJ0lydnY2GAzCMPS8QAjhbrbFYtGWmUPy1+t1lq+jIJxMJoezWZHn1hiEUByEFOHZeKK1ropsPBiVZVlscgBAlZec80GcTmazMIgwZZhS7ge4bnTTAoS5H6SpDcPQ8xkAgFAkJfU8vgtjCAAThmEYhtyjbuGm0N/T+qDnuTaKG6KBO+k3tSOoM0It0m4LDjsdj8cuNI5GIwCAmzWIoqgsy+vra7d4rVYr55KQJIkbQx8MBgAAF7BdIwZjnGUZhFB1rVJqOByOpxMDQV3XLiNxOK0UW7kqDBHY8b3dyWeMuSjrfnZyy27p3wVvtYcKnJrCYrF4//49pRSPRk4RwS2scDeXCCFcrVZCCLf/bgl2YQAh5GwLHPDu6lEhhNAuotuu6wDaqi4iQty3U0qhsy3YRTstnUQgBQC5kVHnsialIGQ7tevQYmOUlJoQsifN7YvOPSQO/4kegLU2SZIgDCFGWuteCCEEJrDtO4iIu7cNsMAYA6wB1k3FQwi1hZhsy3dsncK0w9CdwzKz1vZ9b7WyEDq8A2ggoDTA7MB5QBC2CBkLpNFSSaC3JmGuCzZIR+fn56vV6uLyyjlTS6XNTsYAEkwI1jvOHXxAndsXxH8Sw/b5gTsD7vZwaZa7rPtPfrqxrXNz3QbafQ29Xq+dYJfbFGPMGXu2de6+nRACdx2iPZzgM+bWDWflBSGMkygIAqMtQsj1lQBAlFIhOvcZz2dufbDWunxlBxj8CPm31roI7VIESrdZi9kpUe67FQ/Pz58c8h5ysDt/SPjgtf+VfSDkAP7/vBw7RO8Uu4HZnvbp9GAwGAFZI8IwwaIrNbBdL0nbqTdvPizm2XI+F10/TAdId6+enuweSzaMCUI2jQlCcn73/ocfvrXW1m375NnTOCII9MNBqDTnHun62mQyDMNf/foX7z+8dYP+r7//oxu2CaMjQqEXsCmZNE0zX6+80IMY9H3rcwJhCIEui1W+WeabjU0SxgaceVpjQhGBUGvFPA8BaK2tirJr2qqquq7Dw4ggGPqeF/jHx8d+GNzf35dlHgRe4HvW6nQQS9VTgoy1QeDFYUIoCoLAGNWJPo6jNE2VUkVdVWXtzMuzbOMHnBCyWNw3xWY4HMahX7W2KIokGRBCDARSq14ohBBAmHAGKYOUeX4QeNyo4Pz05OzsLIqi1WIxGAw+//xzguDdze18PndWK6EXdV2HEdXAaNF3CGohEbQ+xwRZCBTB4GQ06prC6L4uN02dD9OwLNaO8u0Ko8Fg4OwAjDFhGEopKeGEEI97jnzb9/1kNE3TOI7jvu+rol4t1hDaKIoQgEbpot64yswYQwkSove5n6ZhkkRRnMZxypi3XC6NBff3F7f3C4jscDgEwHZNRyk+OJzM7+6N0kKIg4ODeJBcXl73i4W19uTwKI5jbIwQihMqhRB9WxRFVzac4Jubm8vrK4YQQfDm8hJjnAQ+toYYlYTBNE2hBXkajZIQAYAAiPxgGCfvl4t//Pt/uL76eHx4MJvN2raF1hqpurqxGrR13Xc9DrkrIzxrZddjAAFGxpgoCN3DHMex1DrLsrZtfd/HCAMA3NrtuAWOoJ4kiVP77/st8uzuk7uri6qqojCsyrKtm9APEIBt02CMgTb5epNlmbW2LArG2HQ84ZgODo+rqprP51CZo/PDx+ePCCG319d92xKEjFLbbqsxjLEoDgjBO6AeYowAIAiBLTNaCwCcjiRTSjkNCSGEVpYQ4jEex3Fdt1ppz/OM+SSVs18cDQQu1jrZGZdDNE3jing3bOY+PB6PB4NB03XKmLpty7ruhECE5GUJMR6OxwAAp7WMCIEYu38u8AdRNB6PW9FzzpMkSdPUFRucMWstgcjjHtkKvPgIIedZvNeEJoTslBAjl8gaYzzPcym1CyFFUWw2m67r3DiG3gnsuJ13pSGEUAjhlm+0m6Fw1b87mw78IIy6qtHxKOM4RohIoTshKTHGWGWAW9YZYxDavu8t2AaYum4opRC6ctAihDl3ftb31gIIkVuxhZDuH0VIawUhMMaFQKiUhBBKKQCw7n0IASHYDVRKowFGnRRd2UjVF0VWVRUhCEIotQ2CwEDgPJkQJYiSusld9g8RohTHSWCttRCIXiECrDVGG0yQ5xEIIcZ2vW4whpSQ/TyhbFt3EX2PQc8DuwbQ9lbJm9FopLX+m7/5G4yoA88QQmmatm2rm3ZfMWNnMA0+RWKyE/ZxsRzuqQwP6uCHP5hdKrmPo+jHQwEIIWDp/sNuGsJt3PWY3FLg4mIQBFEUyb7eTxgCvE3I7E4vmWK8n4N1o7Pu2BFC3GMIQ4wIpTRN46KoKMXWase5AdAoJaXqnTOF3c4n2t2xwH1eiHeq5NuXwvvkBsIH7ZhPf7tNF1z2pnf0EWi11hqjP6FEfPor8ECXaf/an2FXDe63r412zJteCkSwNYgxjzEs+sYC1DQd6Vr7u99+NUxSYLVsmwW+CT3/6ZNHaRpjAuu6Zh7zfR9RIFR7ejq4/gARpUGI/s2/+c3J+fnbD+8HXnR0PG3bdq92sFpunj55/pMvfnF2/vQnz84RQuv1mlBQlOu2l0kyCBA6e3TKuY8RbZoGIUAxNEoYYzCBjGLOCCXIgXJWaciwg5QhhF3XLe7vHeWEcx5TOkxT3/fDMJzOZm6xC4IAIIQpIYQkUZznuVtHBmnqBj/C0DfGQAyTJJlMRn3fA4ziOF4t1/eLpegVAEhpDSxqlEjTFKGthBalmHIeBWFZlkVVdh0mEBirnYhsWzfWyOEoPT45fHR6Zq29v79vW8pI4vvJavX244fL8/Pzoqg44X7ij8fjy81NmW/Wc2GVhsAiABUAfV2GYdgQePnhjRY1NP399cXp+RnQndb68vJyvV57njcYDAghomuXy6VSClrr8ZpS6k/9MAwp5QghiIjv+0aauqjz9eb26rqqqiD0HDWh2GSMMa1VWZay6wAABa6n04PJZPbh4qNSilJOCMmLcjQaBXE0Go1Go2Geb+7mt9pIQgjVnrX67ub6+urqFMKiyHopJpNZEPpB6NelZ3WtleiaWgjVVjUQfb64W95f677zOQ4IwkrI3o7jyFqr+77NNl2ZHxwcPDo+OhyPXr/9/ujoyPe96XiiRD8cDUTXff/tt1VRcM5DP9DSrFcrKTSGkBLGGbbaQAz2EmkIAAts0zQEYamVMcbnXPZ9W9eU0iSNXbHiFHz3vUNHiKuqKs9LrfVsNptMJoPB4Hg6ur+/D4IAAACAOZjOMMZKy5PDozSKyzJfLhbGmDRNfd8fjZ7OPyyePHmyWK0YYo9PHz9/+eJ4enRxcbFYLYGySugyr6w2yCIjDSRuvEo5E8XdWim3QA7car/XdYkQQAhJ2XeVMMa0TV9V1Xg4mk6naTossryua60t8oDneWSnZYQQUtq4mS7XTMnz3KHiWZa9fPny4OCgKIqiKBhjw+EwSZJ0PHKlap7nmBJEcN02hNGTs1PXHVgul2VdDYfDZJCOp5MawaZpMKVa67quHUSRpqlLg7YQMdqOb2utGWZ7jHSPuzoxZrzTgbE7t4U4jhEhrmZt+9plCXtZJ+e05FKBNE5cfRIEAcK4rlsHM0AIuY+ttW3bEqlcsUGtoYQ5IiQkOIoizv31KpN1AyxCCNmdirYQAiEnVfQp/rkMw+ycEZwm9PX1tVuyHcK8Bzm07PyAuyZI27bGKoQBY7RpK0wg0EBpASBhjGECpZRBFAdB0LZN0zQIg31hvW+vUAoxotZCjAkAoKoyV7MyjzPGGGcIITfbCLB1HHAANGEIIdSLVikFId5yHN0wgBSOJ849igiGFhhrtTEQIYyxz7g70o8fP0qh3X1ydnamtW7bdt9z2acLCIH9O+7hcn+OgN2OPPyT18Po+LDi3/8v2aU1CCFrtlOO++aOu5GSJNl/o0ssHLC03zhCCO7kGRye7yAlR8TZ7gaEQnRS9oQQY4DDouIoCcPQpch93zdNtU9ShRDG7it7Ax70HRw09SfVv3skH6ZH+5D/MEvYf+BhC8Y9LejBCK7b1P6QMcAIIWQRQhYa+yeQg9sAxs6Am/Sgs9a6koNRT6oOQGigo9GwtmsIRsxoVNdtVeRAyYPxsK6Kd2+/H6bJdDpe5+soCvzjo9XibrlZGyvOzg+6rtsU+enZjDH41Vd/eCHbw5PTwSChzCuK6v37999///rR2fl4PHn69BmwzXqVvX33YTSaKCXCMAhDf96sEEKPHj0KguD66tYY41BHSinlvse4e9iUUl3d9H1LKbZGVVVVFEVdlg4tjON4NBrdLudufRGyU1rESYIIjqLAQGCAJRRh7HQ1NeeB58VlWWutuw51XbvONn3fStk3TROlyXQ6k0Jd3dwKIaXUxtjpdJrNF01bQ4AQZZRhz/P8MJpNpgCAIlsrKYq6KquiyrOLOG6b6vHp+NHpGUV4tVo5nxWP8Y3aDIdjDJG7L+9vbouiGI/H4/F4+mg2v7u7vr5sqpoSwglVQrairSvdd9VqcTs9PJiM083qHkP94eOFsokU4vDwcDIZffPt1443bq39yU9+ggDoWuE6tWgvpoFp23ZVVTqaNIQwy7L5ovM8bzhM67rWWnVdd3NzU9WF7/vcj56nz58/f/rV118DgCjh0+kEE9L3gom+77vr60tM4KPHZ57HyrJ89ZtfE0Leffiw2awY5642bZrK87zQDwDcknillNAaTKDUsq3aiPPTo8MXTx5zz2vK4rsfvpdtMxwO89VC9Z1REltTblbfZOuybYCxJ6dHT58+ffXqRRpHy+W8KLP53T2nDEIo2i7LMin1YDCIo0TvqhBjjJZb6RVrbSuE7/vNev1hsRgOh07kPwqCruuiKGKMuZrJrQVKKdcRr+t6sVg4sb+964zneUdHRwihuiy2g/4KAABGo9F0MppOJm457uoGGptE8cF05jE+G42H41GSJJ0Qy/kCQOBzzxpbFyXGGLiaBgBCiTuC3fK607IFmhLqeZ4QWyTAaQUqiBhjWm3u7u4oJk+fPo3jlG6tKKwzTHIVlYugVmnXrHXNuK7r3KPUNM1sNnvy5MmbN2+Wy+Ventkg6OYer66uHEXUtaWstY4hCABwAAzGeL1eH4/HEMK27/M8v7m9+fjxo9jF0SAIOGMAAE5o6AcUY6UUD7f0AvdoOCTMWhvHsUvUHHjmmmue51kE3W60Ve0W66qq3r97NxwOcbxdCj3PS9PUkU7WZY53nHm3miNCMMbD4bATUmvtcCPsJgiMub29pZT6vlwsFussD/woDCOXOEoptTUYw77vMdlGPld97hdxhzq4N92NBCF0056u8u7byjV3AADr9Vpr7YxXNpuNC5zuT1ybw+VMTdNsNus830RxsCupQZqmaZpGUWQM6IgQQmltlFJh5O2ir9sFqZRRxjBODQBCaCEbpRHAGljUNA2Ejp23tUcyO1UAtXNL2s/rb9vqxt7c3HieNxyPgEV5nq9Wq7ModltwTYd9vHdTAA+Dk3s2hRAM8Ydw+v5P9u+g3UwE2AlpmN3LWot39gfGmVrtHb92nFknAe4eW6fz4d7hFNofg/b74ltrbRwJGuNtmQ4gIWRPhGzb1rmOpWla13VVFXW9m6DZjbpASOyPcRG7s7vcf9H+5Lgd2EMaD7GB/UE9zBustWCHmcFtWrBd6rV1W9tRPXYhAOoftTD2O+YaNxhTxhjGEBiLMQIAMN/zwkCJUmurlDIWUM6IlCSBQYVjCuDRyUHfNdYa3+cW6KySXgTOnn6x2CxfX93HaczSCQPgeDJZr9cGWKTEN999Vy7v/sPbH/71X/8bqUw8SC1ATVUOUm+V3R4cHyzWV7PJ07//5itK+d2Hu7PT4y+++PyHH34Ik1hZI0SnlIDIjkejvpNt20OIIo94h2y5nHdNGccxJiBI/HSYdF3XtP3p9GyzydabPJlMDg+Pj46Omrdvrq6uOqnS0bDqbNasDYYQob7veyXSQWzqUgENkNGmW60zS8ByuXx8dnr++Kj4ar6cX43iwIo+JtxDxAitWg0BpoQrZYxl/nC0KuuuE0EYQx5nrQqH8Tzr6x6PZi/KLG/7VgtZNzppNYEhbnxTsXVWjUZsuShHo0FTqrYtp6Pm/cVHTNlgMlqv19+9/2FUTU5ePHoyPuSYxmG8XC5Xq5U2RmNbS2WFlFLGw2lVi8U8u76+fvbMjJPpy9OD5XLJjQRCnY8PHO3g5Fe/4YRXVTXw067rEj/2ML27vkUI9aj++PHjwcFBmIRhEtZdPT08CMPw7u7u9evXm03myqw4nTDuIUzDmB+djv/+t/8F0+72/u7g4OB2vnDj+x8/Xp2eno7H4+VyicDw5//yr37zm998++673/7297fZ/POTn82LTV6UWhuL+Xp5MRlXgzjpWxF7QQcqKZRqewtEEHCEbOgxpPoAe+eTyeX332+uPg4ZzW+ux2FY3t/JODocDd68ecOTadfqNJ7FaYQxZh5rBJbAiyUBAHYSAmMI8wk2Pvd8jwMfAcA3m02e58BCzogxxvMCyvlisSg3VdeJpVirHhweHvosNmArmAMtiIIwjROMcdc02ere8zxsVOzhkAXEitXNRykl9WJKaddLJ8AidAWhHQwGQsr7zdpa++WXX85m0zAMLQTzqo6o1rhFnvI9TDxddpuu6wAWShtoNQQQWCJEX1UVQiiOA2uxg6wRgpRSuzObAUKVTVc2klIKsJ/VUliapmMAVNd1g/gw8id93354exMEmZSSYW6RjeNkMBhIIbRSXddJJQiislcnL08vLi6U0MN0VFWVUTaJ0rpsfvePv3eGQIeHh5PR9Jtvvgk4O3316vrudpoOi6Lwff/zFy9ns9l6va6KMgiCIssHg8F6vf7973//6tWracSN7opsrayNg3Aynr159/Z+8TpNUycan0Th7GBCA4owAO3WFEdK6UgznHM3aeakfiCEcRy7ZTQIAillSK1VfUQwDQKYJHXT9VJ/9rNfCqG2A5bW+HEIOW9k37btx5urbcaMkCwFxsTVo0oZSqnvEZAmQgijhFXCo/jR0TFH+Obiwlp7djA2AGqrm7rElAFoter73gghfBIhzLTW1kDkxOkJGY1GVumP79/7vp+GYd+3om9lW3fWgJ0tshfgVohifT8cDoFqZ+Ppb37zZ10rmjy/vLzmnE/ScZZlou7TZCCUkKrooRgmHjQ+JYxz3ledsRYD3lZ930it7H4CEwBge+eIDY01GFvfZ9pIUTeIQWgtRZJhY61GSlJKWehpaay1WkvVK623mkgQAMpYW9a6l5z7RkNgsJTaKKm19MOYEGI0wBj6vi+lnt/dV1Wl1FZeEEGCITbSiL5Bfoh2OspSSgOBSxT8MNgX0MZumU/WWmQlAQgarURvgfUYgRC78lxr40ZQlAaMIc6557O27fM877puK+LZC2st55wgpIwRUkJrKUTM97EFUBs/Cq02UkoEIIbIAKiMhdoABBGAVmnZ95YQZ5wBIexk6yilLqVOkqTv+y+//HK1Wj169GgymX38+NGxmowx8/ky4tGWfojwNrRDYCF2RwoQtBBYCCwADnrAELpJCkfi+VRWQejyFUfndEentbbAWgiU0VJ01lofUm1MJ4WWPWaka2plGUFQI8AJr+u6kz3G1BgAEEQIdKKVQnHOge45QYzgMs8o99q265U8PD2H2txefjyaJAGjSPcB521Vxn5I3ASzVL1SijKGEfQ85gpxN3vddZ2xwPN8IHHXtcoAxoMwDHsB2kYgSMejA2swhkT3VllpJYDK9m1XbqpiXYZsM4iDqiqX97d1ue7aqq7rQZyU+UZ2vdYGIz4YjJhHbu8Wd7e3Z6ePMCW+l2AkRG/LolMGEhQqodertVGMkTiOKGdJW5uPH+6Hw2HVNkLq0/OzME6u53fz5QIitMo2lJK+Z4ZijEnbdqu27bvGIKw6ozXsWgUMlFKvdiZMdmWbqgx8GoRx08nlcplv9GAwgMZ2AABjEYRWmyzLNpvcahv6AaGI4MBQhRF0DVRpxWAyvL6+/sPXX3k++8u/+svpdHxx8QEAILoeQnswnh7PZuv5vKqqu4srVVar1WrvaRuGIfYxBciFB9l0TdN4mFKARN0aypummc/nVVV17XZC3WW4fd+vVquT4+Dly5c//PDD999/PxpNlstlUZVaa6xM3bWuM31weBiH0du6dqlJOhy4WfbJbDqdzbpys7xdre83DPKIx9WmbttWqp4xdjybNkWu++6LL774t//230ZR9P/5f/8///DNt03T1F23vL1HhKZxXOTl/c3t6fHZlrMmdaPrpqyAsZwQigOCmRNP/fbbbznnQoif/OQnRVFwzl+9eoUQevXy8+fPn4/H46qqoJdKrdM0rZry/v5+sVhUVZXnOaXU517gcwxtWzdd3QCw1YwzALjpQcY8IqXS1k0HpGkaBMH8fnl5c71er4XoCCFR5LnGuaMEHkynzlj55uqaYuKmJ6w2GG7n3XultVauSTJf3LkV0E26a62DIBiN0jD0ATC+73FOm3VZVZWT2W6ajlBKKU2SwWK19DxPCJXnOaY0iiKttcvbPM/zIs/10Y12sIhyq5sjBlprmcfjOK6qikuitfY43w7+EeyGJwfjEUXYjbc5Ud6dNQ6t6zLL1pTiMBxQSrWWCIHj40Nr9WazQgg4Sq8QdDhM+6ZdLpd3d3cY4+l0OhwOtTXr9fr7779/8uzZdDpVSn377bcWgq7rjo+PoyR+f3l1dXtzcHh0cn5mCFkWBeYl5UxK2beNNDJKQ8SIzz0LwZ5M4IppV/S4uAsA2MPFcCfXWJa5UyPVBjDfp5RyPxyOJ0qprVolJW0vLy8vtZYIQs/3HiC1VuvtyiuEchCRY6Lpnck1pZRz7lpLzjkaYkoIgZjUba+UMlLZ3VwlIaSrGlfVbcGJthOi7/seb+tLiBAycju7DwDQChJMpWiLonI1JWd+HKW+H7qU3VrrdgNhaK09Pjx0OhaEkL7bDu5DjNbr9XYfMEOUOOczAEDbbwnXeGcBxRCB0LZt68h9EGIpJSWulCRV1VhjgIUAQWC2oLy1tu07jLHUqhNbFWcHujDKXSdbGQ0htBa6eppQZoHzZQYWGAQRxAhjBB+A//sevMsOP1XzOyMJay3Z9VastRZsrzsAQGoNAKSUWgWM3kZTu9NVdMx/Y0xrgbuLnAKSI0bw3YsQkmVrpZTbmT1hQkrZK+naRq5j5VAutwWEkOd51kI3G1+VdVmWp6env/71P59Op69fv379+nXfS8dURbsX3CUKEEILsX6g1Omyh10L6RNmsAeDHY3mE4qwJyho7RqR+9TQPqAj7GEtdzNjALTWdNeSePjtO0jmRzgH3pm47u/VXXuLcs7JarOWWrVt23WdR4nou75vGSdh6Pdy3EmxWW6Y73Hq+X7Y86Berzzfnx0cVVUlFZxNT87PHyeDge/7iODValNuGgb50TBhzCtW5Zuv/1+PHj0axLyfJFKKu6sPAEBsFWNEqr4sGoxpEEQAEWVNK1WWZZgShJDnBVprgj1MEWfRbBrf3q261nDOtZJaYU2g6OWrX75AlMyXC855GEdgDqqyCePIuZwtF2ttJCEIGGMtiOK0E5YTTwpwf7fqWkUgsRoM04HsO6O0Vk0UstlsKJXGUAkhMAKAEIrcMwEoxRCArmnd2DfnXPaiV6IVUnRtEASPvjiZnR3P8/XV1zez8QRSGMZRGEdVmatecE45Rp7nBQTfrpavvxbBz391/eHq5ubGGDMYDJ4+fXpydMzP+O9+9zuOmU887KFxMhLjniOW+HESxVqqu5tbRr0XL14EQdD3/dnJaZ7nVVHWVdW1bV1V11dXRtk3b97UZfn06dNxnBKAuq4r6jZbrtarlc94mqZ+GHz22WevPvvMC4Pb29u3b98epaMiayj2J5NRXdeXlxdFVjBO677hnA7T5F/9q3/16tWr9Wb523/8u//yX/4LJMFwOEqnwySItbIRDySRmMN8uba9lFFV54XPmahqawwA4G5169SBXHZydHR0eHgcRVGSJEmSHh4eXl9fp8lwvV7P5/OiKK4WP1Dn4cRw0zQWWqdR+Ld/+zehHyRxyCmWvZBdTzASQgCGMMZAAwQQQghj6jzf0jTl3LMQUEqVFpt1bqQqNmstvSRJIIRtXVlroyA4nM3qMFR9DwAoyxJDgAjmlCBglegxwVVZlhRtstXN9UenAWB053D4vivSJBikYdM0g0EAIcybrm17Y0Bdt0VRMc7H4+loNMKUzGaHy+Xy9vYWYhJFsTH2/v4OAHBycjL2x9ACA7TjzTsJW6N0mRfL9UpKGQRBk9RhGE6CkSuhPN9H2MfOBdGaKIqiKHIsBN/3nQyfUsrzhnvtAZceYYwnk4lDv52TqjHm6uqqLMsoipzstBunRAi1bct9r2kat+y6qOYkrQAAdV3Ps3xTFo2SgDFLyKYsNlVBOdMQdEpu8g20IE6TMwixx6Bibdu6SODaEy7I7RWNXOx0y3pd11prYkQvHCqOMSHaYoAwAAAinBdl37Wj0QhKmeVrq3SapoA402ezjWeEALDlQ2hlAXYFrnKpiYvWYRh2vXCkCqE0YQwhpC1oOrG9EPsMwJi+7zmk7mSqnc4PIcQiZIxyLR5OsFM3EUIwP0nTtKqqPCubTtwvVl9+83UQRL2SGlijLVLSIggwkkYroJ0AVJIkVmkjldBKa80JoR51dTYkW6ljbY3ZyQZorYWUABiMEaFbrWVXp7opOIKZ1loIuWeBYIztA+E/17q01rqJWZdU9X0fBNsfrN31BSBGELsuPkJaa2mtdWLEDgPfBzCwUxFwAWkP/iO4jWR212Lf9hjAJz+Fsq7BLp+Q4pMjg8t7HnYQXKq9Bd7d9nd6Bm6rGGM3I+aGXV03zf1VEAQWgrIs3VwuY2w0nrhBFTc47RwsKaXT6cHh4eHJyUnXicVitVgsHhIOHnYZ3B4A57bwkIuwbalsqQz7fXO7ve9H7D+vd699foAevgix/2RWAiGEoVMJ+zRSAXZDKPsBzoeJghvxdbmCAcbs9KDIcrNM0zTCsQWaEdp2jSvX0jT1vCDgPsUcaCA6rW1fVRXS1PPCOJkslgWC3vnZ85/87GdlUeV5ThhazNfv3nxI0/TFq5cY49VqdX3z8Z/9818GgReHbLPZfLj8iBCqKugFIeceAKCXphUthJhSMhwPtMmlsJ7nc59oBQnTddvOF9fGgLLcDIfpdDooKhwG0XA49H3/+PjQQjBfLd9dfEiy7G5+X9YVjwJMGKG8rPKua6PQdwabqRcySjCGDNOqzAMeeEnqUTJMUik6RoBPUV22VbZgnu8xK3uh+x4TyjGxRlsAQp4eHR8NkvTDh49ZvukoM0ojABgjWmsL9P16vio3yCOTw4PpdLzOsiDwPZ/97revu7YOWLKeL4DRdV4iZUzXv/7u+6urq6ZpkiQB2izv501ZWWtF1zHGkAU+45EfTEdjIUQchNPR2A3gTUajn37xBSHk6uoq9HygzbPHT7777oe3P7xOo/j06Pju7n61WPzi1ReBH16/v7i8usrK4m5+//Kzz07Pz0bD4b/8i38BCZZa3d/fe2FggA3jiAHPQ/z88HwwTLu4wwZ5mHd9c3x8RCjinP7ln/+Z1vr/9n/5P1dV9ezRedOhw8Pjo+PTw8PDrKz6TuSrfDoY1XVLgKUAJWGQhGGzXuV5LvreAjAcj5wyoHvSOPfzPAcI3s/nRVl+++23fS+zLJvP50EQXN3nYRwbYw6PD8IwDONwOp36NQ+CAEHU972WAJitx4kxhiEe+EGhy9VqhYvK8zwNbN/3k8mEUsIYwxApIR3rVshudXE7m82SJAFWG6mKfLMJPNn1GFghhOpaYLXveZyRpi7X63WUhE1VxQFjCHoEcoY9hjECaehfX1+vN8uf/OQnozSyqifAtk0VRZHbMdGrvu+1AXVdh2EMISaEAAvrqumkwBgbA/K8DDiDxlKELbb7+a6maxljCAAI4WQy0VpbCAaDwWQyAT1wE2thGAK4Xei5UlVdD4ZD0LZFXUGC3XLg2tuDQRIEXlEURZFZa8/Pz09OjoQQWkeEoNls0nXdx49KKTOZjDCAs9lsOBm/f/8eQth13XA8cuqWEMKqqjjnZ2dnQRSuVitrbV6WxONhOiA+F0Yvs/zq9pZzPhyPe617baDRrRQKQIuxQajvW7cyumDm8i1XLzqWktP8GI/H6/U6yzKkJYQQIkI9jzGvarqu7bSxEMI9MAAA0coSwoIg6GS39yzYzU4CayBnvjFGSt33Uu5ciT3P930vCIKg3erudV2HtQUACKWdkIaQW2cH17GuqsoC3y2yjjtFCI7jGEGolPB9fzabhR7Psuzq6qqu66YXnIW+F3dcGQOk1B8+fOScV1XlDClcIuIY6UqprioxRB7jFBMSMfcrzjkiVAihjMGUYEqMMUJIKSVnRGstRNeLVilBKQmQByF05uBt28ZxOh6POfOXy+VyuSKUW2uNtcZagKC1W3FhwqhrAEklAYSUMWOMkNIzPibMx06LYmvA0TU9QgRihAhG5pMRkX0wubBlI8KtgoIr6x/G8i2v0Bjnz+XuWLNTTTDGIIS3gDx+qEe07bhvDSAAdHx2JcQexnDNJseicDmEwxvAzlsBY8wD39HVXUq318lwouNt01VlLXpJCBkMRg7AWCxWVdVcX1+LXnHmGw1Er4zYjrlC8ICnua3gkd3Nd+zDMwRbfSqwU31w6NSe6Ql/rJu0D/YQfDK7QghBiPXOcnOXORDOOdIC7Vwn9ptyVwTu8jP3eW00ACDLsqZppEystcYaaSSABmFK7ub3zONRECKAGaFOqjb0vThKEcBaQc78dDg4mMzWWbbpciMNZ4HoddX0ykBMvbaT7y8+3tzeUkrvFvO8bpDnrfIsiiI/jn7xz3/5+MWj+/v7VbG+W9wtssXBwaGX8GyzTtMhohBaLXRjLYBERTGPoxYiEoYRRvD2ZrXOLlab0hrkeVEY05/87OnR0dGbd2+Xy+X9ch3HcdMdUU4AAJfXV3S5lMpIo9tOKGUAJl4QYUp9z2vbpm27Dao54Y7I1rU1Y8TnNM+WG2iePHkynU2mo9G3331XlhUChhpju0YhSkPCKFbSdI0EqTk9PkySAQDg+uOllFKqPvKDJA7zPNfWYM4AgtODCQAmieP7xf397VXbNN99+3VXl0kYzOd3CMA4DKLHjxljXW8CHnjUc8PoXd31TQ8AcKpzWmhrbbbKrLWqV0aaoijcI9E0zdXVVdd1i8XC4z5jzAnprFYr95nlcul53ngwNMa0qn/66DHE6B9+/7thmp4cHY8m47ypirLcbDb3iznCOEkShrDc+tPTu9v74Wjw+ec/GQ6HX331R2vBs6cvtJF3d3Nj1PnZY+7Rqqo+vF9DiBGERtkkjI0H3nz/Ztn0fdeEnu9hDJTKRHd5eVmVOSXo3/0f/w9Pnz4/Oztrmma1WlVN/eHj5e9+9ztKaVO3DkDuOpGm6WKxcCLKxpiiKJhHjTGEka7rmqZ59OiRUVqKTvatlspI5YQBEgiTJAUGzO/m1tqjkyPP89xZGo/HdDDQWpVlsZzPjTFxHDd16XNvEMZpEGVZdnXxcT1fOFQz8gOGSRJGGGOojBUKSJ1v1hBCTgkjkZhMLTBSSmtMFPqib4sst1p1TS261ga+x/jTVy/rur67nd/d3cVxHMZJnpdFUQGImqTlnB8fnxZ1hTFFyE4mkzQKkyRBCEFpGSY+4xbBvu+d5+94OByPxwaAsizDMDw7O2vydr1eK6V6JbWR1lpkgVCyqopjYwiliGAAAOEsiCOMsTFqMBgEQeDcs9I0da5OzlrM5WduBNeR6nGCEUJGSbDjCY5GIwjh3d0d3UksCyFMZV2wtBAM0pFQxhoYBNFkNqOULteZF8UIwSiMEYZ+EFlrlbbGWvxApW4PTbv6HmNcVdXHjx9PT0+fPHniJCIWd2tKOCQEagukbntR1Y0wgHPOgxAAoI3t20YZi61tOsE9bgxwYsZSSudQZIwJgqgsy7quldJbfGJHX3Avl3O0bYukRgj1Uu0FGNweuvEQt+aiB/I4jlyCEQoCL0mSKIpk12ZZVpalUoqxQCrr+RwiamwnhWoWKyfi4uYJXTdEKqWNsQB4jDvpMAwR5Z7SGhrbdR2mpm5bA2yAMMLYGCO0UkbbzoEHvZDCAeG+7xOy5cPvm5UuC5FSSvWJSefUlXZBZTstuUcCXJ8eY7K/TMYYIVTbdl3XhUFsnRIlBm4bW1UJ/CPjIvNAnxE+mAbcNtqMsciAnVKk3fpI9dZaSqmDyiFGBLN9CKaUu0RQCCGlhHa7q2SnvEQpjYKQc75FzT3iLp9SyklTuARlP//ZS6F3ttTGmOurG6dfByGM4zQMwyAIOPeFEIvFQvRqvV47DIlzj/O+k/XD2h3sBJ0QQi5RePgrl+iYnZ6B6x4qpRxLdx/F9+jCj1CKB1MMxhhot+2SfRbijt1atf8u+2B6QkqJECaU7q+Cu6CbzcZpoCGEMMDWKPcnRGpVtxWE1miNLCAIj0ajwPOFELc3FUILA3SSDBj2MaiNBEIYKfVqtel72QtV1M23r3/4/vUPACFltAAyOhgKa37/+ts0TZ69fBFx9vr66vXr1xcX76u6hBb4aZROR63oDJJlXVVF7XtxFMVxSigNw0D4QZSmozxrPl4WBrTcg0pCpZuTo8fDUSRku97cv3v/pu/709Pj8c0JAMhAYK3tRE+Zx4zXNE0vNWkFRABC2gkFAAzCxA8CqCUEimBbl1kmukma1mXFEARGe5h6o0k2nVmpAUAUYQxgLwXUPkcEM9A0RVvVWqowDE+PjvP1pus6rRTGEGEsRAehnY6HBEEDMLBGC7FeLdaL5ffffXN0MKOUUoa1VhBjPwoJBNbaZDQo6qrv+2Q4cOJ0cRy7MLncrIu6stbWXetcEzWw7968Xc4XouuvN1e31zdlWXqeNxqMXrx4cXN9/ezpCwTgv//3/74oioODo5/95KeeH7Rtu7pd+WEwmUxCz0/jZJimBOGbi8tNnqVp+ujopKyqrmq01poF880CY7xaLX/2s5+dP3skgRJf6j98/U00SAnFF9c3AJiDg4N0NLq8+SqMUwTJx8vrb797fXx0+vLzzwAm2XKVpnEQRwcHBx5njCAte07J0fHh4bNnEKLvvvvuu+++Wy6Xp6enQqjvv//+r/7qr9JksFwufd/PssJVM8vlMh4cZVmWZVndVnmeCyUIIW1Xl2VJEAbWuLmsrm6qsmia5sjAYTLEABOIDAJREA0GA6ll3/d9017XddvWbVMZLeu6ruri6elp5HHOiLXWaFluypVSTdMczg4ORqPZbNY0zXq16toaQjhI47wqMcbFpqjqwk1A+Mw7OJgdHx7fXt3m683ibnH98frw8PDk6Ojo6IiB+Le//f3FxeV8Pn/+/LnnecvFqqqq0WTcdcL3g2fPntVtn2VZXde+FwaciK7vmlYIwX1vC8W3XVVVfduFYejI8FrKfLNhhFiDV5u1lNL3fYx2gnfAcs4BgpQzSmkneqSkMlpq1RYrIQTNcNNWSZLMDiZ+wK+uLp1KY9NW2sjj4+Oj44PNZnN9c4kMGQwGBgJjzOXlJWNsMpu6DoW2Vu28+7TWDtWY3y4msymlvCzrzTrTUoVBTJhntUGIhmEYBUEUxEqZpqqlUG1VueJSKeX8P92K6RJWp51weHg4mUxWqxUh5PDgWAPb98IY0/Zd3bSt6BFlFiBCKEZAKKV6aSEwANZtB6CRUmplAQBGA0ghpdwtsn0vmqZ1Ao4IEs45Z75SwmEJbi02xmgpXd4AIWSMWbDVBXdTkT7lDup3S60UQmtFCOm7Lo5Dl5Ct5vfX19eOCEXDYd32AJGiatbrHGFgjPGNCYIgCPxtcxpBraUFFjOckAghZJSyABCE9wv6rvdsIEIGgn4b9PWueAUuzLv6GCHgXLkxxlLK+/t70SvXyqm6xn56fQKuLcTKGAghwMj1QYRWBoKm71ye4U4pcNA5Iv02lYQGIg20NcDpBBInMOzEqbRWO14nYfRhdesivdaaEuCSOa21az1sFbABUEpLKS0EGFEHpztMCOw6DowxaLcKhmiXmmw5pJ7n+jLmgewjxK47iQkhrejdNreyklo3TSOlzMvaGOOapOPxmFK+xe0tUtL1emjfi74Xrnfmck1CCIB4nyhYiLZQP4L7VtouZut9IrWHB9BOlsr+eFoSQqiUdudt3yfawgNWu7bXHorbHukuM0A/TiyMMU78Y89gcDvW1m5iFlJKGQTWWDfrR8azMaLEQqC01kpNB6PxcODyMiGytu3zMhNCNU3nMvokiRCAy+XSPTmYYWX0wekh5qxum8VmVTV10ZWrYnPXrCustOnDH75r29oogRAIQ78QtVpeEx/7oQeptUAhIDEWYRjFsW80M8ZUVZVlhVb2+Ogc06DI2vl95vH45mq1WCxu7udKwigcPHv6edd1RdVUTQkJNMoYCDAlTScgRp3oPc8DENZ1SSkepannedjWnLM4CbTp14u72Cdx6E9Go8X9XLZdGIZIAwZp3fR925leA2i06Knnp1Ekhe7a+vb6Sin15u0P1zeXURQBYBBCjJMgCMbjcbZaf/P118ja1Wo1HCRAm9OzY0qQx6js2/F0kgzSssxXmyVCKIqifLHK8jIIgtnB0WAwcJS6rpfGwryotAGe51dVRSjXWvdCLbLbLMvcveiwRFcxI4QcJhEEgeMKPX369Isvvri7un589MQA23ad5/tRFNV1/Q//7e8RwU3TQAjn9d16vQYATA9mk+G4t7qosjgd9jL0Qw8i5AV8Mpsyjy/XG4TQ3d3dfD5//Pjxz3/+8+OTx2XRWoi6plmuszAdIEzTwWA0HndtbaxuZJek4Ww6rqrMaBWE4bc/fH99fXt1dfXx40eEEPX4dHKQDod+GHZdv86zzWYjhHJVWtU250+GddtGUZSXWdM0vew/fvwoVR9FwTAdhIHnEoWmrh3Qcn+3cKTFg4MjQsggTpIk8TwuhLib368XS6VUHIdh+NhVGI/Pzp0UZl4Uou0YYwThuqwYYx7nlBBrjGg7lxN4lDWYEUKW89XV1VUYhqPRIEmSk8MzSvAwHc4mh9k6L8vy17/8Z4fTkyRMl/eF40szxgCAWhmlFOd+38vVasWZ76b+nKWCe56rsiyrSikVBIHshZNUYpjAKCKEdE0LMXIqCNfX15SH6zwjEHmehzCxALiKxPe5K4O0NX3bkZ04sVtKmqZxxUfXdVmWffz40Vr79OnT0Wjk4FaM8WKxWK1WRgBjDGYUYzyfzxlj9/f3s9nsxYsXHz5+XK1WYRgmSUIYHQwGvu/P728JoroT88Wy78X97RwB/Oj4qO07a63H6XCYjoZDiolVFgGAMVZ7DRkIHbXIqTRKKdM0ffny5bNnzyCE8/lca312eiqEyPKyrFshZNd1QirGDUTaLXAEQ8w4MRpBYIxp6s480MRFiDgNx7puXBTxvdDJMu4rLbnTsnQLsTJGKaXt3gsAuT10pZ5se7dS71vLLuw1TaO1RAilaQp29D2M8WqVr9fL4+NjCCHGNAg8AADjzvoBu79FCGDM3O6NokgpxSgzwHqeBzEaj8eQYKkNQEhb4/m+GwVXZnsIrjCFyGIM3TiJ2xN3UO7hapteKYUQdp4gu9ePTBZ2s6/c4XyuJdR3WyVms3WT8ghhCIm+d4mCUTuLbfjJ19vuI5+RW6Vn9EA/YItm7JIbCLbNHWcc5WK5Mqbr+rqutTWMek6Qw4lAoK3WIeWcc8q2KMWOMbAfWNjvxp7GAXb+1w+rbbWzF3E/uEHWOI4Hg5ETaRVCILh1mSKEOshTSqeEBPZhHjyUZQQuzH9ydga7qVGjtVvM3aV3N4nv+07WZc972D8g7ibfpyD7xAI84DTsswe8nfOEWmvwYz0Gh9C446XGAABcq4JJ5horGGNg5B6ZIFrrsiw1l0Zp47JmysuqEN0W8JG9ur6+NQaMp9NH509kX1NKN3kGCeYenUwmxKfTw4PWyrKp//jtl/fffYs88ujFk07082pd1zU0NkmiOPIhtI0Sbz9+6Nvm8enJ8LPPnh0/6g67PMvqsmraHEPFOVqtN1XZZ5tGdPDo8JjxVLT3v/j5y/Fk2nUizy6VoBRHGDFGYy/yb+f3DmzveqUhosxDCAFgMabD4RhjeKe1FF3VdnmeH0w4Zd7h4ezo6KCviySOOKGDwQBaK7q+RyQOEzm0Rm+00HGQrptWScm5H4eRNHaxzObzeVHVVVFqrQPP7wAUWjRNUzX1YJAGLFzd3wMAqqpSXWut/ezVi89evnj/9l3f1tPZbDgZr/P17WIeB+HscHp8fFK3jbtrLQTaGqmVhQBipK0JY1cQK8qZqGsnsy+6LkmS6Wg6G09c/ogsWM0Xn798tVptPMb+3V//9eXl5dnxCbLg+PxsNBr93T/+w3/8j/8pigIhxKtXr1w3dzqdUkoX9/dd1z169Cjm/u3F5Ys//ylk4PPPP2+aZjyeQoa8KHz26mVZ1nmeTyaT4eSQvX0bxImyOAzTq+sFpTQdDkaT6cHhoRf4cZq8evWqa+vry48AQUxJOhgQzt6+/vD1119KyoVQaZo+efKkLEu3KIdh+OWXX/p+4NDd6XR6c3MTx/FyuXTVahAEWbEpy1JquV6vlRYvXz7nlFGCnEljXZQIAkrperVaLUbsiLuY57hOTpjS93mapkIIhAHzvDRNkyRp8wxCmOf5ar1om+b4+JhSqmSvZL9aLZqy2HsiQGCMlqKTXuIDA42yGCLRybXY5Jui7eq+FYNkKHuBAHrx7IWW+g+/+4MUUGt9enpKKXPRHQAwm83avnMJMUJoNh6l6TBNh0IIpDtjDATA1TRbwWyj3XKpgc3zPAxDNyWxXC4PkpFT70mSBCHQNE1ZVUWeTyaj+WrJCXULrh8GHuNVVWkOXaeWEGytyfOMc6a1YoxNJuMoCtu2VUo2TQ2ATdOkytqiKJq+m06naZoOBgN3FEmStG3rdA4AABaCOI5ns9nhZCqU+mb97WqxDOOEYUIJYZQ6OxJK0GQ0ngxHnHNrlDF8MBg5U0d3pO6/WusoipxQpmMN39/fX19fTyYTpwbtgOIdYG5cT6SqKmMMiwKPc9V3QggEAff4HrY1Bjh4lpBtyYUg2VMptbZaW8rAroMLXd4glLHWAoSlVBhjiLBrcgdBwDn/+O6DNnBHcfAIIRACSuloNGqaqm3bwWAwHo+11tfX13d3d/H4TCnj+2GSJIQsrbV5vtls+uEo7bpOayWlDALPgYuEEKCNi9BCSWUApkQIwbDnjhogCDBy1TPACCMkipIxxjljnBCCPI9DZI1Rbu6073vitJYQcqnAjpLo+uVo3wYywAopjDHK2F4Kd94AAH2nXO2OMYaYIIKttRYgu42yWmnlQhRBEBGst9ZQW4llB/U7dMFdO/3AFwNjLMXW3UMpBRB0bXuMcRqGGBOttdSKs22i0Pc9pXyfnBFCINtOSRDG3Le40O50egghAOhP9H5rjDFulqGTws1NuAyVMeYETI9OzrpOAAA451sWsIFB4JpTBgCDMY6ixDFkAQB9XW0zrQeiSXurzAdwwtYOyt05+0lIa20QBL7vOzalMZ+ojvbB5ILbkjsQF9etBg/4Cj8Sft7bTNoHREtKqTGf5C/dzebQtb1Em1LKaIkJ1FqTKJ3Kru/6pm+7wGOagKwrNtWKMYYD3HVdfDyO0zSME98PZcCUNstlPpocK6gXt1cJIf/iL/4SBfjj9cdG1jpgg5MDIbqyLnrZh0l4MDzaNVcAQBBAY6HCKMolu93Iw7Pp08fTfL1a3N2XRVHl+eX3wPMO1ov71TI/PTqeeKOmaZhonh2Pnzx7+uHy5oPPigpe36+zolrkEnGUFTnGGHFP6Vo3PaQMM4ABpBwt1jeMEW3atq9Ho2RwNH3yNI2jdJOXq6zxo4kXTRmyneiHg2gw9K+vb6IgtlZjzF48//n7dx9XfbbKq0aJHpRV3faS9IqMgokQ5Bez8/l8jqBNoxgi8OLpF5PJaMzRbX4rZDeJpj+8/mY2m95cXZ+fnrj5tE1eGUAJHxydvGKMU/9gNJs+ffnqzZs3ELOm1UpTIbXVxGrvJ5/9qm3bpqlOj84pJMN4qJQgFD05OdNaQ4wDHrup3Kot7xa3EsgoTYCGKR6EsxhCKKwIAZ/fXb18+Xw4TIuiMgY49k6WZZDAtm9602EPosDWOrvPPz4X0599djoYEnQwBZC0fSmsffTiGaV8tdokYTJM01evPnv9/Q/vX7+Zz+dK2hcvXjDmaaM263USx2Hozw5GXeMvl3ME4CovyeUdxuHTFz99//bdfXa7XK6Wd5uj2XkanIrM076PRFCUKzzA/6f/6X98f3HxX//uf1PA3s7XQZRWRR/ypKwKD/ss4RbomPubrDGdOD08UFoQlFRVfnX74ejoCEI75YPhKDK2BxbMpjOlRF3XuveB6k9nMwaAUqpuyjrPp6en52dnX5f93d1dmqaPH71s6+bk5AQhRFGQZVldScPQZHzcNM3t7a0UcDQcctrkm5XH6K9+8RNnqx0EYZmvbq6v4jh8/vj05GD82ecv62L15s2bxWKx2IjpdBqGYZjEvu83XX/y6Gnfy6IuAQDJcBiNDoXlVV5pjf1wAvuV5ws/EBAR98ALIUQvPc/jnudyAoSQFhr0xkf87uKCEELi6Ohwts5Wzaa5ub8JQr9Vfdu3Svaz8UT2HUMDoGTqcxQfNk3DGIviAYSwruvrm/sXLz/3PK+sWgih0uDmdg4ACMKk7eTLnz1drVZB3/dC/PSXv3CcOAvhdz/8oLXmnK9WK8754eFhU9UruDwepgShn372Ik1jQMimytu+WhdYZaatq0EcDVJebcyqrhmhh7MJZsbISrRl22uMCeM+pRxi3Pc9YYE0NvTY3f1tVWUvX55DCAFlQRBCxuVy2ZciHUSBlE2TtxQo2Wut86IvIHRYdBAEGkKllHborux7LVvZM8bqsgIQIArqpuCcc4+2bTtfXFMWElJFURQEQVU2GHHZFYSwvheB7wMLhZKUEx9jpBQQKhnETdOIrsOEWACUNZhSP46LovCTgTHmdrmmlBZFnfdKAnyYss+SZ1r3dVukw6SuW879MIw9z0PW9r0E1kBtdN9xj47SNF9lo9HI+b4CBF1HXGgFAJJ9r4yB2pniAKaBEjIZp1rLWjSuGCVdzxghhJTr0lhtLRJCCtE9CJxwj7hgDCGCECBnV6G1tQYCAq1BUhhrsed50CiCCSN8a8LZK4dROUF0SrAUvdGKM0oJtuaTx7ejCu5jlRMx23fcP6HoAigr4M5TlHnc5TRd0yBgOSVC9G1TSkJcMqGUcXCdC5NuT/q+n00mjkfstEfdZIQQgkeUhd5msymKQkpJKQvDUGjV97LrtvkQZ34cDJIkYYxRxA0GUsq+6a21DDNEEcXYyVA6+LZpGgQ0hqZvK/CJDfDAewkijLEx1uy4CBhRA6zLqNTO2NqJQjpMC+xkpvYwgzteQtg2tBtjDdTEKmWkVBhAKTRjntFISYMgEUJy7vUEme2Aj9UIaAgsBgjDXgrHQ2rqNo5DIUSWZVbJeDTK21ZawILQNoJ5mCCoe0HW6810OuEeXYnepSfKaNdlOD45OTk99cPID2NtTeX6ppxx3/dDz2AYNYkfBgADY8zNzc39ejGfz1erhTGmF60xxjLaK2dxayCEEFkIoTEKWl0VpRCibZosyyhhjx8/zrPs3es38/kFhHB+f9c1dRIPmqbrOiGNXqyWz16+evXZi16bH969E1oBYLRuGQ1n02FZV63ooihQ1vRN60dh23WiV8YYTjyPR0k0fPr0ue/7vt/VVbFerpu2AtJWecEoDMapEKIqmvl83oSi74w1zPf9JEkGBmhlXPFXlLXUNum6PXcGQTsejxnBm9UyCYJH56dpX158+EFrqYRJ46ipsu+++Wo0iKSUTVX4vm9USBE8mA4RIn1bvX73VinFPI/6npEGERglsZOILssyjIPj0yNKcVHkebEhEmMLnPJMs/PQU0bXTRPHcdu2FkELgRDCAuCkrPObFed8OBxaa8MwhhA2bZ/neRj5ZV1zzs/Pz+umqKpiOBz+6le/CsOw7WVzc6MNoH4IIJXaVqQZDEbQgros+rqGwD55dB6HQRSExpjHT86qqvrym68Pj46ePXsMgf77v/u7R48eBT7rm1YKHYRc9M39xfV8fvfV+3dHByevfvZoOjwmiA2T4Xg4aPtmvrwDReH7/qsXz/7mb/7mYDpmXjSdHKzuC6UFoWAwjAAAUgqENWU2CBkm4Ob2djIdSdX6PqcU+b6vGwWAghADoLquKopitVrVdXlwcHBxcbFeZwcHB5yFq2V+8eE6z2pMCaWYcyplX9fler1M03Q0GkjZJ0nCCKGUaEO5R41VWm+7vH3fI7SF/uq6vrm5oZzdL5YG2MPZgRS6yCs33hnHE0qxhQYiY4HpZNeLWipNGWyabrNZa60wJm3XUUrtQNfLuSsyOOdo54YshHCZvrW2LEuEkCNVAQAQwZTSKE3qur6/v7+4uBBCjCcjCGHX911bl4xXRckpC30/icIPHz5Ya6fT6R4bcOW4ECLPcydpbK3dyxxNJhMAQFmWTtjOobjGmCRJ3CRVEASOZ+6wyR+Wc4iRMlvgOvKDo6OjOE3c5FUUBNPJgUdw17R128yXa4BHvdIGIIQs2DePofUYcR7cbVvnRSWEStLhaDSiXooxruvaWqiU6aWAECJExHb2j7n7XwjRd1IKfTAbu/Kx6zq5gyuEEGjn0uuUB/eRqarbIAjcCZ9MJhCjjx8/lmW9beUC6OBfdxWk1A6vclPKzmOWMebmIxxtfj8fwTl35iwIuMsKXd3mmuhd1zHimhpOxcEA4BSjt3Q/iBEhhFCqtNbAMo9bCJQxmG4VFBDGBEJ3+eDOfIgQgjFFCLqCuJedOznGKFeJQr1XPASOzIGQQYg45j+C2FqotdLaYowIIVoaFyZc4H9Yze+73ftKFyHUdh3cDfvtm/EQQrmD0K212po9mXGfOsAH0gLWWu57ruT1dpiE22mp9H7S0qUgDoNsmsbhXtPpFIGtNXmapmWX53m+Xq/d/O2em+l856WUnHPfCx2O1fe9k+90SdWesgoAcCpMcCf8vOe1ELrN7+0/0aIG4FO5b60FECCEzD8xxwK78cWHWdT+nOx7NA/fhBAqKZVSRiullFMZxxA9VOJyPpRoa7K1veQYbYcs3A1QlvU4oAgRCDGEEGIKrHBfRObz+WQy8b0wGQwDj0ul26Y2Fv7mN38xHA57KZbrdSuVx33P8xjzurrykyAcJAYDv4kMsOtsI626uLjIm7KqKq2Usx1TxlhtpOi2iBN0HT4LIYRWC2Pvb24vwsCKkyQOFSKr+er6+rauW04ZJQz4UFuQFZW1Ngjjjx8/nj9+9Pjp86PjadXmVZUBKPu+Dzwax7FUXdvWgZ8IKbOugibwOBe9QghT7Blo0jAdxhNjzJvXv8vzsliXq8WSYY9BHAf+wXjUtSrf5FJo7emyrLoecu/m/cePxos1gJRQznxKVS+bMi/uyI0QQksFIUBA99CKvpZ9oGW32tz0oiIEKW1fvHyktcQYRVEoZLdcib6HbVsppaMoiuMgy7Ksrj0eIIaUUZ3sgUUYorrvLEZN12IMD+MgCP28ypquA9BQC6lHGWGd7IwFnFOkYdXq27trtuHJcBDHsdTK8bE557qUw+EQEgIA4Jz6fkh524t2va6iOAhDX0pBKAwiP01jY8zt/f3JCfP8sKvb+fJqkxVCGI/7STL47LMvZtOplJJAxBk7PJg+PT9Jhkmapl998/X/8p+uLOiz/PnZ+QlCaLVapIPw7fzm4v0HTmHXtJdX743qfvLP/uLpo6fjwQHUZHG/7rVRAE9nR3m51qYbDAYQ2jLf+GHMEETAGNB2faOUohRrLYsqq+vaGM050aa/u786OZ1xj4SRZ4GEiFXNZp1R3/fTNFU6hMg4YwtjTFFUCCHO/dXq5v5uiSClxPvu9TeOLDIcDpWQLoUP/WA0Gs1mMwRAVVUAgDRNN5uNy+5dsuuau1prjKG1VgpFCTs+Pn3+/PnFxcUf//jH77//Ph7Ev3zxmXvUCfF54AstuEf+f3T9Z49kWbYliB19rr6mXUV4qBSVWZklnu7X5JuZHgwa4JD/cQB+IPm5ARKNIYbT6OmeV0+XyKxUEeERrkybXX2P5odjZhlVPW1IJMLd3Ozqs/dee+21KCXG6bYr27qWqvZaLmmSRyHz3UZjlJQHH6PhMB+Ph23bAuAtanqMsXN+3SQsyD3z4ObmZrGa393dZVmmteac+zEQXyw2TUMQgmmSpqlv93garDEmjuOrq6t3794ppbyP6Kn36bW9q6ry/Ak/OD2bzfq+n06nXu3ARwVfCWmtu2JPCAnTDEJYlKUGzpua+vIlDqMwSZOAQwiN0kkcWmgAoYg5Z4XVxlqhlPFUGyllVRW73abve8qIUmq12pSNHo/HvlhM05Qpz63SytiD6o6DPkhACIMg8KCxZ0faYzMbAGDUj71ev9b7qs4PfPpcMEsHHsJhjEmjrbXY98u19qoSCJETqO7DRpZlXlL9w9CFDm5S0APmDngtRXDi1h1p5+4DrplWWlhrL8ezOI6LqrTW+uXcgCOSfCQBaGPsEb2v+95vGmPq5wmdc8Yc4GsvA2AOysrQOeicRYj4oGy0M04RwiCEWitnASReVdr62vfEBmBHWx9rra/XfV5rjlP+9scZvMN5OA1Q/FHA84nCKZ59OLUIPnRswsh+wDUBR6sqB7C10LfgtQYAY22UsarrvH05xdh7mhvnDGMEC9g1Td+2CCGMEEU4ZLynDGNqjNFUI4SiKAnD0HN9pHUelj+02NxRvPKYIjjnvFS/9aQNwg5X0MHTbeCOXg8/cjWchRhhjLX6Y33lP8oA/ihROOVbf/Q3pxP1YZPCZyfO2Q9kFLC/6NY4a5xFFkLsZ4I8+mImOeXc0zwhxk45gAgAgKTZYDAcZmkynY2jgJfVXms1OztLslRb87hY3d7exkn6/NXLqyfX4/H44fGOMDo5mzaiX5fboi6+e/1dK9pdubfQpHHIA0wpUULUdQ2hcxoq5yB0yAHkLILIUzrCKKjrer/ZX83ORSvul3eL+4e6bJ5ePZnNZuQowVa1HYQwSZJdVb65udHAYYwpgZNRytilMaa1zlgXh1QpBo3CCIaM9l3DeIghpIRHPOjbXgldl03ftPvtilI2GY9sr2VvvPzIYDA6m43mi2Q8O2M0kOpWmkYoaZxuO6WkwYQhRCilhFBrbd92GCMMAUZ2v1l6+8q23X/zu1///Hn+V3/5891+W5blk6tzIbrz83NrbcC41QZRB4HdbVeib/IsHY9yHMcQwqptatEZpYMgkEpUVXF5ecki7qxW1vRSGgRISBkm++XaeG4tIQQjTAnEyAPCdV0750LGEcFWm65tlZCgd4QQwpk2UkkjhBBKAwC8spkQyAHNOaUUL1fzt2/fPv/0GUAwDEOtLYGtU1rUvemV6qS+fm5EL7oOEGpVJ3tBCfru21+zMOi67vmLi7Ozi/X6QanGGDMYDJ5ff7J8uJN9o2SHoHVGTqZjcvXUYl4LBZTdFNV+tanr+unlBAAwGo2Ade/ev9FahpwySrHTxnZ1s5FSpmmKMQRQIWwYR4S6tiuMlZi46XRY13tj1H6/nUwzxoG1su/rfYGVMs5C67QfnYAQv3nz5v27W2tBHKech0oJKeVyOU+SpOuaMAw5pxeXZ/pW8oA6bYToKKWDQdY0ldYyzVJKsYfc9vt93/fZcBAmadM0ST4I46QR4l9+97u7h4dWyhwzaZQXMMGIQAL7vu77FiAYRbxpqHOWc6q11lIZ3Rjd+Oa0V+b2K2aapmmavn371vfsvZShj47OuYDz5XLZK7nf7yEGPgxsNpvJZOI5RkmSREHYt51f187Pz9u2bdt2sVicFv39fs8YGwwGfiHb7Xa73c5Pan399dceN4YQ+qmH4XDok4aiKJqmCYKg6zqfbCVJEoaB9b4YZV0UhQYAIGSNCYNASqmUKasaAIBYyGMcJ8lutzIAaeC0ccZYjGGIIaekrStCSN/3XScwJoTxqq7n87kDB3jAWtA0TSd60UspJQ04pZxS6gCQUmrtSQm90dJLajoHEMScBT572NRrv/pDDKy1Wku/yH7Yvz8Yi3ddkiS7soCHouwQIRA6eDCu12tvTDocDgeDQZIknrrhsw0AQJIk/ixVVXV2cQYd9DWoly3yPQLGmD0MXJgwYBD6XgBW5lAnCiUdQFXbAAAIo/uyhBBCjBxAPntACEGCKQt8CHEAGGutg9o4AA8uiMZZhAkB0BhjnBVKAocJAYQQCLA1whjnfIQDyAFgjAEAGWOAQ14ngFHq80J7JH6eSAb+vJ1mLvxbhNJTfP2QM4iOIkinmOeHDqA5HLj/Qntsrvd9f8obwAdeXEk68E/EAZB3zmeZThuvQbJerz033wNR4Djl61MBD+eAg8QIOiVDH5bspy16bM9DZRBCr7VgjPG6uj6tx+igZ3DCdSCEACBrLcY/Wjf50Q70wWDwh4jICUj4LxMF++P86o8fP+Wj9gPDMC9RgY5enUY7o51zzreXIITWAm8wK6UkhCVJ1nVCWxuGsQPQOogQ0tYxCAHC5Pzy4urqKWOk2O8QIUmcIYTyNH39w1sWBm3bRnEynk2TNO97+TBfXL24jtIkmoy7Yvuwedg/lOVqvq9LQhHljFKotWacyr4jCDrnAEc/zndCiDBAAAIABlkOjY3jWPZKVF3f9GEQX5yxIb26uroKo6goivl8fvtw37Ztp+VgNBZK+/E50TZa9IMkPDs7W9bNzc3NII3jOLh7fISIZGk0X676pqZBGDAaBwRZ2vdtsVlJKa+fXlxfPycw+N//49+9++HOaE0Io5RfXFwZY4piL5QyzgqteiXz4VDVCICu64Q1sBM9RThN0zzPnFHAmbPppG+rqtxDp7p6v7Uy+Gz06qPr3/2uxDjlnK7Xyy+++GK3K+7u7qqqmU7Pnjx5Upblw8ODL84GowkAYLfbAWCN0c45TCAhpKoqTKDVaLfbIgyUEkHAKMaUE2UkQC5OYuT9yqBL09havVwu15sGQJumKXQmiYIoikxvOedpPuSc73b79+/udsUeAPDixbO7xwet1dWTC4TAcr3Y7/fOWYjxervdbQutXRTEl+cXBSuaqj2fTHar5d2bN0KIwSBHCNVNCSFMhlyWXZqm/+a/+5tXH3+0WCxXm83t7eL92zdJGN3cvNmsl13TYgB/+OEHLdXm1z9kST4dXXAcAgWQA01Z1VkghPj5z/4ijsN//ud/BtZVRZkkgFKqdOuAihN+/exyOh1LKYXoCUVVVRmjBoOs6xovnt12dd3UaDgwxgBgug4AADCmlHIpRRCEGMO6rufzudb6yy9+9vLl87puz87Oqqry60UURcaY+XyupDTGLJdLaN16vc6yzFP6tVRZknLKGtL6GlTZA8UdE7Ivil/9wz8bYx4fH19+/Mn5E6GUsg5YB4x1AFpvw6itcgbSiAacBpymaS6E0EpI0ZbFNsAcYxzFIQ+YX5LartFGEYrbroEQZnnqHZu01t5h2fO6tdZREPrFse/78Xi8mi+Mlj7MV0WJIWzb1sHDoulXXu/W/bd/+7effvqpb1GVZeltnSeTSRzHUluMcZqmZVn2fe+p0dPp1HfNvTGVc26/33sEixAMnIXQIQJ94c54GIfJIB9p46qq2mzLsmqs0whAxlgUcy2VaIWSPXHQ06niMCp2O98fSbJUa015GCcDyiI/3y+Eapqma3vjLISQ87BuWwB6Z6EFP5ZTxhjOiJTaIwrQOgixX2ml1P7eoJj4IIqgwwxfXIR+pfY8eY/KeFgCY0wJcxD4qOBfxlkPa6dpSin1zDgvwqiU8m/5FqFfr4MgOCYKQkrpHPQ3mzEGOqO1xphhjAlBDuAgCN7f3c5ms7bvnHMCq77vAYIUga7rMCUYEGuAdke5Q2sw4845L/QLHIAQIOQggEIZqZVPSgCCvotiAcQAY0QpoRhZ56AQylqrpAYASSmNcRhLSjxRAPa9CIPgQHOz1msa+oLbH51HTQ7zgcdLcMIYnHMO/pg0nMpfPwbpQ52W8hQm/VuH11Hz+DQY6cNkGPEkiX0TrWkaZ70LVKiFpJQAYJUSWqpTO89YjSEaZHkURUVRIIigA7IXBFGMMYTIGquE9A0pRuixL2NPTUAPk5z6LO6oSOFJG+BHjsLRlglC4P6gZQDhwf35BCZ9mCV8mD/9117+Gz7MEk6vD3EXYwyCBEEAHNJaS6mUNRBCSjjGFABgjIMQKWWCgPjmdd9LTKgxTlnLEVRaOxhAiMlquVks18CZN29+wAhORgNntAcwSU2U1oQziEjXibrtnHPT52cBssApi12cZ2mb69JyJSwkhCBjlJLCasAwSeMQAQgt9VTVU6LknAPWIQCyPHfaffPV77umPRtP8jSTvazlel/QXoZKmTAJZ+dnq81GayOk5CFo277ruqoo1/NFEASzwShPE6NkGIV5kC6Wj9rqJCQhg70USRiHzKUJTWLyeF9V1SIMw88/++jjj3/SVfpv/+Ov9vt9EmZxnKRJvlkXfS8dwMAZHsZSrt+8fYsxgfzMOoARRYQyBxACFGGrNYbQWvPy+dOA4zfffXN3+87p/nx2Cazab1eb1eLq6qnqxX6z7xsRUP7mu7dSymE2+OjFx9W++uHbH/75H/5xNBr97K/+OooiRkhAcCP7UipCKEF4MX8IKEMYNlXvnKF+SNy68WzadR3GOElTY0wnD8O7npGA+h4B6Iy12i89gBOOMR4MsouLKz+sX7eVv+OTJOYBNUY5BybTUT5Idrvdvq8GgxEP48jCOIiBhU4YbCBUut7upJQA2LYslJFa6yBkGHLKmNaya6rvv/2ubhvGeJ6kqhfFbv94f9+2vXNuPJkFLHx7P69YP41DJJv7+V0SpB+9fMUxqLebvq2++OlnENj3794q2ZdliRChjHNOR6PBZDL5/POfXF5eaK2bpnHAfP/990KI8Xi83e69VaDWOuC6bfog8KRx5Bz06ijWOELwZDrKByl6AFrrLI+l6qRqEAZByIKQRXGAISqK4t2728eHu5cvX87nDyHjm82679s0jTmnwJmi3Akh+k5aa/M8Twc5RlRpK7q+aZpOqKZpskF+fnHVtu3jYsnDmAWRXzFZEHHeEsKEEF3TKqWiKMqyTHb9Gi7rtqscYMORLwUwhlr7ok30PQrDECHgnHXOQOggdBhDSnHb9UEQQIKDINBWeap2GAV5nhNCRN+u12tO2W63m47HAADPvfIovZTSmzX87ne/2+123pzGm0b6ajvLMoipMSbLsvl83vf9YDBo2/b8/Nx3oPu+99O5vtUVhmGWcgCAQzCKovFwtN5spRAUYSW0ErrrhLYdJtA65+Whnjy51FJpqazxKKju+xYajTHUWtrG9p1wEABipbJKOwytc85oSzANQ2iPyyUiB9VCbQ1C3vbJt4ecH3D3caXvez944qMaYyxg/BTtwjDkYeRTSc45o3w0Gmmt27YjjDLGMCLKaISQR9oxxoTS0WiU57m33vCtfU/18CKAQoi2beM49m63lFJgD90K34DwE6Faa4oPwcMYI2UvVe9n5CCELAwQQpTwoipb0Qut/GAUhNBCBxzwA1NaGkTJMf76OAQxJgghjBWQSCsF4KGjTwglhBJAMMZ+yBAhh5BRUjunnYNSaiUNIQQGhHMCjo4D7qiP5KOmr6R9sY4Q+nB+z2exp4/4G8P/w7cbDt1x8IE+4FGR4pRYHKLGUdPw1M7w5/+oKmm9OTt0gDHqnOUB8yafeZYrITebjbE6SWMrjJ9o8BgPZ0Ge54vFwh71FTxB0t8PCCHP4ACHhuOPDgj+qMMw9G6TP7Yk0Enz4A8YCfBH/o1/A/1RivBhlvAh9PLhP049BQAAgj9++SmxcEelZwihte4wCQIRQoepCmUccN7rkgBwQICkMpTwQT6qytvVZi216qTQWgcEG+Osg9A5stpufv2736Zx6De33++F6OqqaNvWWOucG06m05lBBGfZeDgcCiuLpq5k20thkQ2TsFUtEbjreoyJs8YYRRHEBDFCMUTOYWwRNsg55xMra61zpi4rqG2h9OPdvZWKIQwd2K7WUBnrBOOh0IbQwDiICLZaP8wfMaZtVWul0ihK46xr2sXjkoxi2bfWacZYlsRVU2Nkn1xNq6bxjl5piofDoey3XdednY1fPHs6HY0euxUwFkMwyPOr86vr62eb9SofjIYIIMqmZ1eY8F//5muIEUTEQ5qekmqM6bqu61tGoTUSOIMhAs6kMU+j8csX1xcXw6ZpPOHr9va+rpu/+9U/EUK+//5tkiRV2RkNx6NZHCd3d/fz+ZKlWZ7nWtns+cs0iKTUSghjAceEYIQBNABaB5CzFBMMEeSUcoIAJAQ5Zzin1loI3WCQBQHzaLPWuii01VLLvqu7ruvwwXzMMUaTJJFSvr15kw0GSou7u/eUwY8++ShN0+12jTEdTyfnswsPqvR1n/AwGtLlfMU5HQ0GlOK6bRAms+kwDMNdu/eT4lGSvr+7q6p6NjvPs0xLU2yLh9vFdr15/e2biEZxkHASPP/o6acfvWQ0btdb6kRMgNUCIFjvNwTY77/5vZYiDMM0Gzx9do0gRr3FGI9GI4xxWVZ1Xdd1ZYyxBkCAgyDa7+dCqBcvXiyX2yBI8nzodenruq7rNgxjhFRRFNrIqqrTND2/mFprm3a/2y8Hg8FoNCrLsq7r3W4Xh1EYhhcXF6PBEGP8+PgYMm6tXa/X0+nUP9Xb7RYehVyCIOBR6MdxHxdLHoIwJQ7CME7e3Lxre4EQaloZBAHCVMpOCCOU6ztdlg10QAiBMcUA+rBNEKaY+GjtWxs+gHlGt6/mTxisZ8k1TSMV8MVNHMdVU3oKgl+/fBuiKIqAca+3MRgMxtPJmzdvvBWyD2B5nj8+PuqjT4GfHfdkSa21Zyz6aiOOY49AIISur6/3+/18Pvd75ae5+r6n3DIaQAjzPMeY1027enicdALs97tdsVpvhNUsZEEUYowdAPP1BjhHnAsIAQQZa6uqqo2lmNRlpYyt295AhEnVCbmvynGWedTBOdc0XVGVXddZA+IsRQhRyq2Sh/rVQuAQJURrjaCL0ogi7Gt9hA5cRuegMU5KLYRCxyfdxwAhRFN3Xv7yzZubTgiMMQTIx3WvfGUtIAFljI3HY69Q6S9lURS+xPSFfhiGaZr6QKi1RuCQJTjnPCjtA4D36TiqXEhtJABgMBryKKy71jmHKWFhoJ011hqjiXOHaQ5PMwTOaK2kOqZOECGEMAYIOAgAwgBBgCCEBELonNHGOAAwoVpbrYVz0FqLEYEMeVFFCDFGfgs+1B1sOHz886iAP0C/5pxwBXBsLiCEeiH+MPodBYnBB+YFEByj/oHa+SOicATwvYHRqS7318jbeftkBULIOUfgYLpojt5afrMndWQMkUcRrDYIwNFweDabzR8fpdDOOqO1x5gRgMA6Tyg5dQFO9Ah/yTy65kEU+If6SKeC32dXSh9cUk+9LYww+CDzAB+wE/4oM/ivJQqn/OKEr2CMzR+oNR+SM99MMcZ4XqpzznNrAAAQYEYDpVtCSBhGfS9u35cU2SaNtHEwoNaHbADJbHZeVcV0NP6zP/0lo+jt29e7zWo0Gi0Wj5TSKM3Oz8+n06mXDLPWTqZnUsq6req28QSfvu+rqtJaIhQghDihjBMfPgGwQEPrflSgNNZoZZwxQoi2qoGxBOEgjRnh0EJCmNE1ZQBhW+12vbDaIqG0NbCuWiFlV9VWm5fPnr989uL25kYIsbp/aNvWtA2hNInDXjXWyYvLS7C0ccLquuYBurwYl8WoLosXzy+cM0W5m8/nu/1GiIPspdY2zTIArTGKhcF0dllW/WZbxnFW9elmsxFCNE3XtjWEMI5DRlHXdAiYotit581ut7k8m6VROJuMgLNVWQaMF7vy7v3dZDK7e3/fdeLd29vxeHTz5vbFswdC6MXZ5XK+KMvyh2+/S5KE8/DF0+cX5zNn4Xa7r4vSGNPXjVLCo4gIOs5pEATBIPUYl58DDsPw9PwcKNZN49lnfj2tym3btvuqvL29pZQbY5SWUomrq6tduWu73g9bV1VVlvu2bW1IdrtdU7Wy6UMShCQwSsleDLK0ruvdZmucVlpMzyZJkkglnlxdSaXW6/WLfHDswaE8GzqLnXaTyRQBpLWFAM+m530nRVN25Toc4lEaUMSt7PqunU6HH796MR7mv/ntPw2HQ21dlo/TNJ0/LoI08AvQYrFo23a/3/uKME1TQjCl1BqwLfZ/9qdTKTQhZL+rlLQAuP1+Twg5m13k2dA5FwTBu3fvtOlfvXr19OnTpmnu7u6ePr34/ocbSj2f2QUBHw4HGOPL84u2bW9v3ydpXJRsu9vUTYUxttacJK26/qDFFkVRNsjDOHl3e1e3TVnXmPLXb28AAC9fvoSQBEFMKXYOcMrCIAqCsG27YrcXoo+CwBiHEAlYmKZ5kiQe4TxGEXD6d9u2nljnce84jpVSm80mCLO+73f7nScl+B4BAGC/3/s1XVnl198sy8bjsXPOC3Z5yl7f915kYjqdelkLzvnZ2ZkPbLvdbr/f+xgQRdGLFy8+++yzzWaT57nXRXj37h0AII5jAIAvsouiSBNgjMnTPIqyd+9vpfAj+EgptSuLoippwNMsCyIOCX6cF5yyQRLTQUYZ4wiZ3kohmro6m50PRjmrmsfVuqq7IIwvL54UmyUAAEJsj4YIcZRyzou6CoKIUmSB01p7KxxKqdbKT3YQQkIe+EWMECKF8FFZS1VVlSejBUGgrfHJWVVVq+XmxauX0+l0s9mt3r7BGBt8mESIoigIAqWMgdbPNfgiezabMcYeHx9PshCEEM8yqapqu92meYohPpV9J7DaWmuMM0Y5Z13AOOdZkAyHQw+BrNdriNBsRryGAYDw5v0tQggD4CAimGBGfRGmlT6EIowRggAAo50D2s9Z0IMt1mEwxDlBQnrsGni7Ck96h85BZ6HHq40xB6MGCD/s65/KekLIdDr1CJM/HJ9vAQCatj0FUefcaRoToB9/6T4Ik/4uOgVjeGQPUM4AAFprZA6aIr7HwQLOGPPHFccxhgeJ7rauJ5MJpXS32ykh0zT1A+on+Q2/t4yxNE0ZY9YcSBXgSF30JTjiDBzJlQcs5Ph/n6D/2F84RuhTonAq8bU2nHNrD5oGB+oA+IPA/+Hrj1KE/8NEAfzhu9Tvs/HoEcYYQwcAAP6Jxph6NS17oDc655yzEOKD/JSHOfq+31SrYRz0MndHiU9rgbWaZHHktKKU9kI5wGcXT0fTi7IsewMhhPv9vmz63371dZ7nl5eXcRgqSsM0qYxuyh0mxALX9Q3FKGCxEJ2W0jmkJbYIaR1IKSHsCSEIAaVU11RKSIZJQFkAHKTEQXd++ZQRWu5Kpw1CJDs7u7h+rpSrWrNYz7tWchavt9u2lbe3j//j/+3/EsXs/HyqTW9yt1wu6kbFIE2SyDkXsADGgzRNF4u7s9n48WFhDJSd+O1vvs7SUXI+Voo/uXxBMH/nloPsbDYFRmOlEWVJVe8vL68e5vcOMu1Ip8Hg7HIyPgcPrTcT00YjyBgnlIbWGUrjMODv369FUw/ysXFJWVuMhxu3Ds7On/90Wm12t+/v+7JAovn47PzZf/Ovt9vt7u0b8q/+EmL0+atPFncPVsP9m/evfvazZ5fP3Hp7dfUkzbLf7NYaqWSQFoV+mBed6HkYIISMdoxEGtqXz57O5/OmLiGEnNDhcLhZrzHBIY+KogDWxmG422z2m82rV68wcY9393mep2kIodhtVp2Qr169Ms5ZKFkwcc69u32/Leqzs7M0m1re7XfLiIcY8VbZOE14yOZvt13dZ3E6nc2cNtvtFtpAd7CtNUpIHOckcN9+f//11988efLEOXb/sJxMJs+fPz9/cvXDD69//etff/vwFqb4+mcf2UrEcSqlPruYYkyUbhyzpajPnzzdld1f/MX/mbLoH/7h76qqqHeL1eNbuwuSJPIEuv1+jxBKklhrjRA0VvV1C5FL0uDb7756/uLJfD5H2gCoFosFxlhIsy/WQjYXFxdBEABg379/7wvxJ0+e/PSnP5VSRiHHOGrbttGyKApCSBCEX3/zzW5b9EI3vZldXEMSC43zOG9FAYhxmERZHqbO60FRgiiCBOpRyimQ0bPLuq7PU04pDW3/3W//7vr6+sWLF8R0fV9OB2lVbYAWgyzc71Uch7e376SUs9l5GEdCCKW6pmkIwpwx36HzzXIaZcoaYxyh3GFaCdkqbRnvZGehTdN4vV6ORiNnnRECc/54836QpH3fC4CNta8+/XSx21w8e7qZvy/KddeXGNPZ2dgB/e23v/eMmfF4SknY1I/OgoCnCCFGZRyb3W733Xc/XF9fT6YjxogD2T/986/CMNpsF0HIvFbSaDTJsqExQPX2bnP/5Ze/UNJ0fZvk6fR89Li874yquu7i8uk5pIvlSilEdCpbCWnQCAmRyFIkHbbaBEE0m0x3641ygHJORF8UOx4Gs9mgl8IoBDmzGjat6HtNsSeyuTiICSH7qvRhvus6D2hznrSdchCl+WA0GuHtVq3XSmsUBI7ibV36WG4ZXtfFrquHadI0TRQmAAAWUK/onGXJ2WSslFGiZxgHPMQWqLZ3zmnk0iTZbreMMYLxu5ubOI6dtQTjuqr8SJ5WqmtbKYSSsqt6jLEzwBnb9g09QH0qTWMIIYMBYyzOsiRJIIRtL9+8ffChi1Ky2pSMcz8oGwYJhBA5jAhBEFmhjTFOGwwURZRgBKHx6LRvwVAAKKXOEZ/QIOMCTCFkSmhKaRhGjHFffiillLKEEO+bPBkPAABN0/k2fxxwpRQELgy4B5y0NZxgaI2SQgtBIIAIUQSDgLMsbdrWY7Eey4EE+/jqOziHUAR+VP6RUHnBH0+9BEc/JOydRyiyurNKE0yyNCNDoo+ZhP9LaJ3/cZjkXrY5CeLD1LgFFNPeNBjjoiiSJPFNh6IojHZBEACACGE+jhLCOAs4kwr0fd8rKSDGAEGCnbPOWAuAbZpKGpmmqTRSaDEYDJq21cAZCCzy4dxZ78GhTdd1lHKKKXQAWIcIhA4Y/SNiccoXT7EfHR0xfGriEzIpe+9KChxSSgJAKcXGGGhdEATOWGct5yGltG9apQyh+NjNsVofTEmapgrjiFKitfJplpQCY2StcX2vsNN9B4FVSvCYK2AJpeTq6irPc2PVzbs3vm7gnAJgnzx5oo0qyv3t7W1d19fXTy4uzgaDrClL2ePdZrFZLiByXVsxgoNBVpdFwKhGUPZC9O0hhQ84JRQAoIWsur7YlX3TYog4ZVfnV5SQPMmeXz9bLzfv377PsuzFi1cvXjyFEBfFknN+Pp1tdqWUGlgXRVEch1arOB4653744Yftdo0J6ktRbcs4iF+8eAmhy7Lsq29+HwXxZl1cXT27f5x/9fvvESJnZ4IH4aefzspSXpzPzi6uk3iw3nyzWdfTyfknn2zjJNBWDUeDummc7iCyj4/3Wuu6Jh5WpZQCoHztzhhbLxctJVWxCSkZDlJKqTPi/v7+yVOe55nqxWa1FX1PWBDwEEL4+Rc/LYrq2++/r9smytIwTs6fPNUAXo2G8SDXCEhgWy2Z1QrYTvRPRs+ywYCGwWKxAAj6ES8WBIPRaLVZU0onkwmEsCzLm5ubKIrq/f758+effvppVVVv3rzx0xxBEADKR5NZGIa+EkqyfD6fL9frKIqevXj++Pj48HibZZGUcrNZXF9fh3kYxwmEuGt6zsKz2VmxK/u+z7IsCWOEkKNwOBxEURRFAee0B6pu9hCZ0Tj76OOXjLEkjYBLmqb53e++iuOYc/5Xf/XXr1+/vri4+Oyzz7RAEMK+k33fAwOcc5zzQZZThqUG3/7w9e++/qYoqjAMozCOo8SGPM8zzvl2u22aZjAYpGnqVyiEgX/ardV+xTk/Py/XWw85+gfMTy4ZYz766KMoirzM4n6/t9Y+PDwopRaLlTHGN5WzdIAQEmKxWq2UUpyFYRhyzofD3FpbVcV2u50Os8lwdHV11bZtXZRtWzdlYZVUSrVV3ZQVpXQyHOVJWtd1XZSEESHEfr8vywoA4JkBHqMOw1AptdvtPoQ0feMcOiCFllJySjHGCBLEkXbWWOdlTpTRQhnnIA+46SwC2MsqaK05D66urpqq8qqdfd9KKZbzR2PM/OERO4gRAw73nRC93G52QijvrZwkkTGuKIOmaZTuGWM8IKQNRK+chc4hrUCx3z083i0WizSNvdthFCaMBeRop+TteqWUXSsI49batm0Ho4nWuutayth0NoQY3D08FuUKQuhUzyimhCgtRI8xY8A6L0LgF8o8z2ezGaYszbPIGGwDrXXbdcYYr7RjnFV9H0URDXjqGQnAGWf9rUUZHgwz/yx4nrxfN5VSxiiP1sAj+VEpVRhnrSVY+oERCPFqtZrP587Bg9syxKe12znXK8MDQFmAMJZS9kI50PVCCakdQJhgB5CQmlBDKB8Mx15xQWqFKXEQAAiNtcZZbY1vYZydnTHGdrvdcrX0TUyPgkgpjbUAQj8XMx6P/Q54Oq3HRY0xQcROMPtJjcNau9vt/AGeAH9f/jLCjqwIeaqAPbDNOU/TdDAYAAAIKTwNxUvIwA/YfP7UVVVljhaRfgd8F2Y8HvtkxTnHGLPw4JGhjfZ3C8bY80N9JuGn/4+7AU5ohNei8OfBJw3+9+zoTGaMEUI4dxghSaL4YJxxtO3AR5soQhilB3sRKZW1tuu7i/MrQrqiqKQ8kEusM5ggITUAFiGAEHLoSGIwxs8Ter8MKaVWPMOLxgABAABJREFUVvTqZJYBIXQOnAieJ/zjxK44QMLGeM7vfw1ROOETp+wBY+xNA9yxxaO1tsZQHnw4Xgt+5FL8AfDgKZYIIQ9Mns6Mx5kO3ZkjXHSipBBKSbHf+ovk1Si7rhOyD4JgMpkEAQPWEILjMMjTBFiznD9mw3Ol7G673iwfIXRKCSGaJA61EgBY6ACjkGBmrYUWOi13u6Lv+6Ioyl3Z933IwulkMh4MP371acSDgPM0Tua3j2VZU8y0VM4AQtFoOMlSI8Xt/vVGGRCGnBmitf53/+7fOWDOL6baql/84md/8ie/+Md/+TWHDBMSQF637YsvPv7tb74xAjy5eP7ZT7/YF//bt9/+ejiaxql6d7+WBj99NgpjvSvUpuiUQXmSvb9/+H/8v/6ff/2v//L6+vL6xZNWdZhga/W7dzfL5XKYfyRERymJkrTrOt8+5DzO85xgoGX/7Nn1l19+nsSh7NuqKEyrkkk0f3wsN7uAMo+orXbbT/hPX316sSp2jtJ0NIqz9G+mo9V6/er6aVmWPnOM4rjv+13d3Nw/PH35Kk3TfDCqm845F8fxiTLz8PDgwd4sSSGEr2/eMsYiHmBCRuMxCwOAUT4ZPXv2DABgcViWpXMOYWit3hXFrijyPK+7KhFR19fWKsZx07Z1U1l3Np1eFrt9XbeDwehsPKuqarFYJElyNj3jfhhdmSCOOKeE0XE+Wdk9ACAfJ8DBuqvfvn2LGc6z4b4snQNF1WBMf/GLL4MoW61WX/3+++nFK8ZYmI4GOWqapi4rowFVdhDGr29uf/ubr9+9v4XApSkzxkEIrTYhD4b5AFjnjE2TFCNU1PXt7W0QMC9/BID1TlrsUIXDyWgcxzGEcDgcHmad6yYOo4Bx2QslpOyFozaO4yxJD5IYFlFMgHXAWgRgGid5nlOCtBJxFDnnKquHg8QpobpWNHXI2E8+euWFWXa73Xa7vZjOzsYTv2BVVSUgMhCxKLbaVEXZNm0YhoxQTRngEEKYpwPnHHQgieIoCI3SwDoAUBjGSsiuqZRSnHBnoYW2rhtECURIW9f2fdf3QitrLR0MlTQQQkJ510trvBPuwBlTFAWlGDqAAKyqYjweE4IW99uuU1kWMuZFcI3W2td8CCGELEJWiKYoPI0fhSGnlELkpBS73aaqCwDMbDY2VkUxJ2QAnKdEEUys0i0n1LtQ7nflYDT06rkIoX1R9H2LCCQF7fraul4bBSFMonA6Gg/ylAFACInDiCEoRUcIccYqoxFhUZK2fbcvK2ttEMZCCOxhgGNCgDFGlPgxubKuvLqD1ppSCoCN4zDLEudMWTZ1XSul/MqolPLLvXMQY8o5NMYwzgAATdfZthsOhw5BqXSYpL5OtcZ5YSWllBc0QhT73M4c1YL9Wo8Q8vtzmpP0wtteu8K/6yO3309PqEJHxuVutyuKAgDAKIYQak/6UwohBBCUWrVtCzDySYA8qv0wzuM4Oi0UShkItSeREMKMMVp75BmebID8H2utldI/IufAuw843+YDAHRd78/zKdR9qArgY88pyBljfHaOMQ7DyPd6fE4gjfbJ+o8YAIR+FOIU5E4yQf5r/Rd65oGfA/KX70QV9AmlByS0lH4/TywQ30fwFBwIIcbUJxn+khmjnWMIoaatgENd1wgh8jxljPhpZHA0bqaUOnToJmBjIMSU0uVmXRa1Mc634DEl7sjZt/bHBuJhEBf+aDHq709jDD6Oj/5RlnCCFtDRFMOTQk6Jgj0SJI0xWikYhP5T9hjdD4kRPjBJj72Mw7Xz/UTkDbSOiYuvr3yHAiFk7eGWoJSSfJAopXhAwjDEGL97964v2zxPy3KvFG/bOghGl1ezjz95NRhkZbU3K9R29dvvv72/vw04tU5J2XdxGEWBlMYZGwYB52Hf98vH+XK5LnZ7rbUxLmDB1ezJ+fR8OplkcdZWPQWs3u8e+/lms7u6uHz+/PlkMuk6QQgjGCthmrpumiqJ8+Ewe/7qo/F4/I//+Pe397cIofPJxasXH12cXT49K/JwdP9wW2+7qq03821XiTev766ffxIG6WA4G43PWRABxNtOvr+d/+e//c3z561TYDR9ko2W29W2ahrG0ZPry8snM4ccY4RQ2oo2SSNtXNc1Pk6PxsOyxEL4ZqREGBqjlRJZnnDOt9u16NrH+wfYcQbQ7e2d1ebFi5d1WbVNv9tsdlX52Zc/u5zPaRhoB4qqSfNsOIE/3L3fbrd++o6KjhFy9uQyGeZFWytr6qZW1pwo3NZazQDGZLvdFUVxeXk5GAxYENzc3FxcXKz2u6bvX7x4EWUZAEBamyTJ9cuPtttNWZbb3Vp2qulaoWSYBNWy+P3vv4ri4PrZZdM0GLmLyxkPSFt3Tdl0bX824YPBoGtk2/YYkKIowjCUfa+1DGWwWrfW2slkMv10mud5ng8fHxdCdfPl4+z87Cw6H9NJwMOvvvr9aDRp2n46O/vbX/3d/f3jsy/EdHL20cuXgzgVdVf7yWyp+u222KxWu722LuDEAqCsSZKkdYcJqzRN/RphrLbWTqfTLEuklNvtlnN6cXExnU6jKKL6IB7gnY0opV6ccfk4BwAQiOq6bqs6ZHyQZtdXT84mF8aY/X6/Wq2klGEQDLJsOh57P+WyLIuigJZbY5xWeRJP8jSKojQJ4zgeDoeXl0+ccx51QwhpZcqy3O12RVTkaSaEoIPYW3Z5yogXyANASSnD0AkhfJCQUvr5Qx4ySqlR2hjnZfn7XnjZRMICFnC/GBnjvE8bpdxa5+t52TRhGDqr27YLw7BpKgyRUgI4K/o+S+MkDucWIUiBw8BZBKE1wBjny1whOoShdaYXnVQdZSjLMh7ap88mnDMeoP1+KVSTDeM8T7uuOYUKv5b1fdt1XV0SQljT13Xd7MsdYeFkMun7tq5LxrGU/du332urgigkAWnbttx34zxD0EkhiKMwjSEEUsowSTvdtW1PudNar7e7x+XGWjvJp4wxyhnEqGkaIQThzAtTEkbDMKSCKmMo523f91JGAUGIGmPKsmyapmnq0xrqy2uMsdGOUsp5CCHkjHDO+34uREsIwZhyjsbjqVam67rdbrffl2VZekID5zyOR9ZBALFUUmmLMHUAEcqjGJ5UegAAxgJjgQNIKKmMhhhFSWyMcW3rqYiIYKtd1dT9nZBSNk3jIOCcGyUJIQGlnpOkjJZVtSsLIQRjjB0BAx+NMKP+ihwQCGP6PxRGNEfNA3BkGHwQloB/viCECB58qL35pG9Uo6NF9UlywJc3Pq6f+vQ+/nneBoQwdge9KZ829Up6TMKbVhwGLBH0yRal1JkfxRn98+vTKS9q6YO9R63w0R/B40NSyrZtjVReg6TY7T1jF0Hoz4M+eFwdYvkpIyEEExKt1+s8zyFyQchG4wEhpKp3WmsaMuQghJBQDPEBp3EAhGFMKS3rthQlQhhCpLXBlPqhD2utV+30mQrCB6NI8IFAAjhOQ5yuzn/ZgDhlQu6YFpzyiRPY8GO+5Q7tmz9AFPDBqOywCfgjK9N/yqM7nqR/eqKNMYQQZ5TnYIZhSD56+cLDsKvVChM4HCTjUTabTfx99nD/viy2SRzkafri2TOt9QZW1U4Wm227L4PxIOSMQpDwMGCMAGiUBto0fbnb7R7v7peL9XQ85mk4Ho6vr59fXj4JSFhVzX5bqNaWui33277vlZDeJSWKojiIu65/+/ZtXbVa6xfPnnPOm67fbZbns/FsNmWMAgCUkO/evncOvv7+NSGkrrrxeDR+On39/Q1G3Ch7+/YO4zBPBz//8hc37+6UNGfTyyzL7pf779/8xzwZjmcXv/zTv/j6q6+auhjmoQI2SpPletXUnUP4q69+H0apkkZ0AgAbBCxJIudM1zVKKeuMkr0UnVF927Zvb17PH+4ZwW1TrRzp23673V4/efrq40/Xi2VRFBbAZDhMR4N0OOqlaubz5XIZRPF2u/2Xr/65bdvxeMwIpZReX18/f/78yy9+fnNz0zTNvqik1GFIhTbGmCAIsnz45Omz169fGwv8qvTZ519QFiilQoSKulnv9s+fPxdabaoKMDbOh+PpeLF4XG9XhOGzi1lRbd++e3t5eT4YpaPhkDFSFPvJZDwc5Zzz5bx4enXt52uBhZeXl1XV3N7cHSZtCOSYs4BKA5XQ2grGWJ7nELqi2EHkrq+ffv75T54/+2i73Xed8ONFvneDEHry5LJq9sNxxmKMA6ScgBzko2yYZJvVYt8UUguAAWEUUiykRJyep3lVF4vHxyAInLUI4yRJB2n25s0b1Qsl5XgwzAcpcG6/3TJCfvnLX3oPJwCAn0zzXDA/S5llmbcg8uhlWZZ5PvZgJjpOrHlKDSEoSSIhOucMgIf/eEDOLyZJkoRh2NTd/f1t01RBEJVlcXPz3mtecRZMp9Pr62sIsZSyNp2XdLTaEoRF1++3u6pqrLVt3TVNE8ex7EW5rzzHEFPi1dMQQlEUE4T3ew/wDIRSQiiozKnRK4Qo6iaJYkwYxtBCNBqNdvvN3d3deDigmCgljNYEAtV3XVPv1+vJZOIHIqQUnqx+rLyBlBLhw/olhOy6LooiB9vJLMoHqZS90I4GiAeq7ZeYOqmEFppzPsyGYRjWtdGbar92fd/3agIA6tf9F7/4k3yY/cM//ToIWRiniOCB0xAjqeVmt9NaiEZI0TmdUIyCgKVRzCkhAPpmeVFVkQMQYetg3fXGGNU++hQtTGKAEWxbz3SzTW2tJYxhSrlzEKPtfieUPD8bOOek7LXWfd/5LIEQVFVNXdechZSQXvTGWEIohLBqah5EPIi0cQjTXgghZJyknej3ZbXabPf7vU8srINKWx+BfEXr+y/+N54P/+FLKeUne/27frDCKzIxxoqi8M0j70FwbEhLQhBhFCHkIADGOASttUjrNMsOnRdwUAHSzjqtuw54IiFCxN9FPtJ4Kckj0gwA8K2Tg9I2AMDLOB5CBUBKKQAOwRtC6APNKbqcOgInUOTDavgUxvwzeKTLQa21soZzPh6PhZL+BjbGQIB8+wBCqKXyOcfxSh20Lz2D27czfD/FZyTWHcZfDwMUzh309YX0KXiaJP7qHCWbqD/zPk1RSsVxTAiJonA0GoZhYIzJ88QYgzHQ2gDgvQKsUg4Y442UMKZhGGJMEUI+ygOAAEBGQ2cVOLSxDm6QAAAIsBfX8meMc04IOwIPh9eHiIL9QwuoDzMDz3vwf3dCO47Qy/+BgxT8gADhDkMnf5Aj9n1vnPWaHxD6efLDzWy0O7GnSbHbxHHct/UP331trX3x4kU+GlXFHkIYhuFokAVBkCVRHNLxMEWUpmhkhQpxSCBhkHLEsAMU0vXjljKMAazrerPZVVWFHH568eTnP/0cAhzHydnsajI5swp2lVK9jXiilIKW5OmgAsV+v3l8fMQA7mRhjNltC611HKd5knZS3N6u7m/f13WJMR2NRmVdF0Xz+LCYTc+//f4755xx9vzy/PrF86++/WY2m/1V+ldKm2KzzQajpxeXq8UGOMiCEDtQdkL0SurCWDKazv7t//hst1p8++1vhFYQo04I7YBT7v3tnLO0rlpGIULI95sopWma9qLzPWYETToeEIKE6Ebj4eXZLOB0fftDL2SaDaI041HIkiRwLlc6SfPNdu8V5SilVtlmX+6W609fvPK3zmq1qsrq9uad7HoC0WQyOQy3YMyDwC8Kg8GAZ2GWZfv9fjQaGWP2VXl9ff3s5YuvvvpqPJms1+v5eqUQCMMQQhgkcQ6F6NqyK+JBNBxk+/0evkEU0C+++OKTTz6qqur1D995H0WtFQAgDafGmKqqZa8e7h6tBc7Ys7OzrusgclIqAGxEg49/8vHFxcXZ2awAbRAE33333bvbO+DQ02fXo8mYMLzerpQySZYs149f/vxzxoJf/tmXX3/9tdRNyHWeIoL6rl91XVtXuto/yq5fru5aUSMGwyzABD4slmVV/PLLP/HYtR9Ic0cxnMlk8vBw17btxx9/fH4xO3gcjMdZlLZNI/reU9mdc0EQpEkSBsHZbHZ5cTEcDPysYMB5VZbA4rqut9tt2zQYYwRgXRZN00wmkzBgxX4n+nY8GgymwydXZ+Px+OHuXdWUnHMpZd/Jtm8YC8qy/Or3v9usd03TRFE8WU0Gg1GWZXEc96Zv69paa5XWULbWVfuibfskSfq27ZomT1OjVN+0EQ8YY20vdWiMtt7B1kjjJY2fPnk2Xy3ny0XX95hxgKCSRklTVA0E2DoYR4HSmgZh38ntZlWXxdOrS2tMHPAg5MBZ0bXlfkt4JlUHIYziYDAYcM7X69oYQCltmg4AK4VyFvSdKmANAQFUGKOF7IwRlOKrJ085J7v9qmlLaKwD0gKLaRol2EHS9ujqOi32VZTSgKdF2YzGqbFYKpEPBhDT4Xj07OULTNE//cs/3bx/b6yaTcdxwDmjMQvSMEAIOGsJIbvdrm17B6WxEBKaprlyyFpbb/dCK4fgRRQmSQIx8ot+GMcsCNI0bfsOIBjiaL/fR1E0Gg2rqmrbDsDDiBohLAzDpumsAZTSJEmBF8NysO+FQ7ZXUhrdin6xXvtFOU7zd7f3VVWVZe2cCyPvuwGl1s2+oJSORiOACYYIANBXtVIKUWb0QWqJMQYwUdaJXjinGGPUc/uVQoRYACwAiBDCmLW2l1JrnXBOKe37nlJCCNHOSqMBACEPKGOHYEwwhBB4UMRZvwxa4yBFCGKMnJfp9UCCNQ4CcBDlAw4cVZPRscQ89p4QhBACiDE2xp2KeIypb6NAd5DK9ifz1H04RaM/AtJ94+NoMGEdgn5YdH9/d+KLoKMUprU25MGpdeK/wecu3vDWYwy+5PWAEMT4lCIQQiA6eCyxOMYY+215eOM4V+yVrRE4KmEbYxAC5+ezs7OZlHJfbNuuFkIYq6zT1kGfTEuJDHBGOwgRJgxCDIBH+7TWvRQ6DEMpNYLm0A7QP4pIHiU9DiJIhPhi4NBQOGEA/sDdUW7hw1N66vgAcCB4fjhAcWoJfZhYnECIP4IrTi+PAymlAIKep+WcS5KEIOtHcvw3+ESQrNfL/Z5I2Wd5KqVknGKCqrpRSkHoGCPj8XAyGXHOu76xre0JF41SvVKdrou22O2bqggCVtdVHEWEkKZpmrImhFxeXp6fXV6ej+u67Vo5v32sdy1GvGslgXS72ltre9ETkkQ80GEMrSuK4vF+NR6PkzhGCGlj2rZWyhAEFsvHoijOz8+TJEEAZHF69+62q9teK2N1EIabYp+vF7v9BmAwGAwsgH0ntOyRSSKM26avmhYC/Laa//QnP704u1zN1/fz+89/8snls6vV+r7ru+2+mEwmSYzDKD+bPd1vO4TBCaWs6hIAQBmxjjVNZY1KovD6+okfV/3k448uz2bAmbZYSClfvHgBLdzXTSuFRZDHCSDYS0HEYQQhDBAVQgSIjsIsShKMMXMEzC4xxkLJcltcnF0O81E76YuiYGHkR/IgY0VRPT4u6rrmYYgQsgY0bf/s+ctNUVJKc+uqrn+cL4MgGIyGu7IaDOOqqnbVGlKLCGi6Ksnin3355fT8TCkDABqPzjCBeZ4LITabDafhzc33UujRaLRard6/vwuD+Pr6ummaYlu0bQuQQwS++uTl02dP4zgWNRK9+Ob3P7x5fcMYD4JwtdwY7bbbdRBEYUQhstqIUZpTBl6/+SabRKKJm+JOKbNZvHUWEtuu11tCSFVvGWdRGCR5whhlW8ZtoKXK02w4HOZ57h2lH3c7z2wqyp3WWgpBEA4YN8bEYTSfz29vbznnvoTyo/8Y47qusyzzYEBVVb6M2O12fvrLiyIghPyTZa3tRbvf79u2BsAyRvI89d3B94+3lNLxaJplWRJwQkgvRd13l9dPScDLsgYObsvi9nEeBMF0Op1Mhx7AMOZAJfOciTAMKbWerOcrOW+z5OUKDrNzkGinGWOT8cwLD7dNX1QlZpQF3PebldL7qhJa+XXKA7B1XWMIfvGzLxdKpGk6yLM0DJqmcc4gZH0jMopYnsenFcRTK4xxWgNCAoSUEKaue0TRXrYQWspglkdWhzQOo8CWRc8IpyQCAIgOVFD2vVECxbFFmGkrs5w6GPai3u87pYRSSnbCATSetkkWS6GMsVGYcEraril3mA7yDhrR1MBY5A6s707Ium3CNGNBxIWUyvggIbWSWgEJ27btpfSKBVIrZXRVVVLKbJCHcXR2fu6LWik9UE+MMRjDMAxns5noFUJ+JpABAAhmjeocNl3XG2OdA13XaWW9RFVVNSe74TCMCSEeD1AAOAsQxMpo70MNHAx4iCC2wFlrrHHAe7wC4IVkThA9hDAIAh+HvGC2x3HBceCZc26BsRA4e1zuj7VgK3onnTHGjwac+gIEeQEleLyghzj0R5XlCViA/wUDDiEEIcAYG6N92Q0A4Dw8hitACDkh/z7YeCjF/2iO/tH+a9mR0Od3yRjnGQMHPODI9UNHpeETL+E0rAiP1msnzMbfrh7GOO32KV/xIMQwHxhjvGWzR0NPtIZT9+R4Kg4hE2GQZnHXV+v1uu97Y5R1SmtonXbAAOeMdUoZACC2oCgKY5ynpHStsBYQwvq24eykR/kH1lZCCC8qehSkOvAqnDlkSB+2EjyacsIG/ihXUOpHHOL0/H748Q8TAuDMh3nbCXLw5yQIAuAQpgQh5E0+PTWh63r//YRw/z2EULxYzDHGz58/F0JgDAlB4/E4ioK+79++LSFylOG6Kcsy0lrXgLRtz1CQRmnEg7rRfSuM0qJXTlUeX8KIDrLBeDAbDydtVbV1W5WtVmC/rSkJnUXGOK9t0vf9frNnHAVB0HXi9evXH7/8KWehN6GRQgghgiA4O5tWbbPdbtfrNeXBZDL50z/5syzL9vvSIJfmg9FosN6tu2+abbG9ub0JgiDPhrPZOaNUdm3XNLvtzlpwcX71ZHhVNiVakSAMGKFFXRnRNV1blZuvv/n2p59h6+D5WRLwVMkmDDKCtb+6TdNACL3EhxCCEpRl2eXl5fLhVgiRprHSYrVYVk3NeXj9/MVisSjKUlmDGQ8hzvO8rdrz83OKyeLu8fbNzW6369r2m9/VWZYBADopPvnkk7OLc2csApAgNB6PLXDKGuec1KpqarGYKyPfv39PCKGcn52dJVlaFMVmvyOEzBeLPM+/+NmXt7e3u91OSjlfLa0tR6ORNv2vf/MvWlnoXBAE0+l4t9v9+//3/+fFixe//PkvNpud0XA6uVgvdv/0z//Q9/2TJ9fD4biuewhh27avX7/O81xbE8ZBGPIkjQjB2+32229/b+JECPH965vFakspnczOy6YeTSdnlxfGmKqtPv7Jq5vbN4/L+9/85jcW6uX9a2Q6qNum6d6/v7u8eJrFzxCUURjkg8Sjr4Sh0WTIw5/GcVjcF0kaeQGAoiiqqlos5ovFYr/fXz974mWGR6NRkiReeLguympfgDRVvUAIaUKhddA6honseoGw6oXqBY4TaF2x3WVPhmmUBJTVdd22rVKSIDTIkr7rJW8ZJnHIA0a6prm/v3t4eFjXu8vLyydpyKPw8fFxsVh2rRBCIYTevLlp6u7i4nIymeQjgBBKs8xrHyGEpFD+zjnGY30S8vLcaWPMfD4HjGqtrQXAHWhcAY+8QYMQhylwDAAEmBDIGNMYaaWkVB5w9ouvn2K4vDwvi40AhlIMw3C32yjB/uQvfvLmzZvVatV2zW6PMcbW6iwbaK2VMkY7jEkcpdZAYwyCVPVUaxhFQRQw0co33y8Q1kK2UrX5IAmCUGvZtW1BpNa669R6dctZXLeCUg5hsNks7x82ELrtdquMu3+Yz1fLyWy6K4ogiLIsBUK2dcMRGg9yRqhWAgAQBAdwxZSVc9B3aY0FUkplNKXUC5rtdrvVZkMImUwmy/XK0+UeHh6EkhY4a20cx9X2oSz3Shlfifq1lVKa58PFfOWZiV3XcxZCBq21RbXHGHtLTAhhVTaeWughOoyxF6vRR30qliaYEESwbFTbtWEYYkryPDfGIKU8+OzzBL/OW+sscMponwuGceQVlsq68oEwjCOPqzddCyFE8DDvhxAy1vpecq+kb0wIJX2ayzn3ti/OHurRUyBBH+gj2Q8ocgdKnZLH6G5OJ8cBp5TCmKRp6g3VvCdFXdchox4k8IXySZYAHpUHT9/vo5dnKnhWAQCgV9KvpZ4F7D/i8Qm/z8CB0+/V0WEZ/YHahPGtFh/deyl9EuDjsR+P9CNFHkL4MDcihHRdRyk9MfhOSUndlGkXR1EQBIHSUhtFGXHAGntQVkCQQOsgMNY6hLG3uYBHoy8f+I12kJ88LA4IjbXWGF+jA5/ZCCEoFcfwf0gsfGbwIYRw+vG0/wAAfGSw/lEa8WG68AdHjX68KOAPmJIHvQ0IsMeKvNyI1toooXTnmZj+LUII2T+WUGJlbdW1hCILwV6UiKL1chsl0Sef/ARREsfhfLmuuhZCuNpsHx/vy30BtAhgEOWZLvcIwGa/joajOExoxEMWIEjQXkvQKiyM0CmPDQVN06q+I4R5AfnVds8Yu7i6BMDePtyWZRmkwx7qmJGAJnd3D7vNPo5jbU21b9NwMP3k4vb2HWcojHDdrkaTcLF9d3l5SSm9ublZr9fT8TCLM90pivBskA8iPoro73//T2K3rpeL4XBYLPYmSAaDEQFoev6UhMHD/ePD4/3NzZtelBiSf/7Hf/nrf/V/Wi/3k3H2/bffIdEOLy7qunZGUxwJKcu2wRClg5wAI2Tf1GXAaZSGaciA1shJKbrnz572bZfG2evv3n388uPlcv38+ceD/CwK+/ni3hkxL+++n3/T9dVwmBPbNbK3Frx69SrJsNL1s+uzzX779vXXURxfXJ6tFqKuS23x2x++m81mVICfP796XKwowS/OZruqLstaNt3zp8+BQ4ThIAgGeRpwCqxZLpfbasV6ZZomVKDvZZQmGJL/7X/5X7tOTKZnWpqiqP7sz/58tV7//d//fVEU0rGPP//0o49fGqMABx998QwAe3v/PggC7YbGGGsdROib2++3v/7Ver2OhvGTJ08RkxZ0T6+f5X5SVBtO+LbcEgBXj3Nr9c3Nzds3r8/OzkAWjM+4IyWN1LOPMkrb+fY3VdNsCjSbnQUcVGW72e0os2madX3jWLct9vtqThCtqkrJajoZfPrJy76XcRxTFgxGQxomPCGVsl+9eWeKcnx2fnFxMZ/PH+7uswFbrFeerPDN99+9fPkyZHy9Xr5795YQcn19fXH25O3bt1EUv1vdYowRwkoJpUTTNMv54osvP396dbVer1UU5mn2XfktZTJP2W7z8O7td13br9f7qmziOJNCX8wu4AwBgFTnpJQQIgJEAAFHpBW9LwrLg2ZfMJ/Px9Oz8Xh8e38/HA57o+rVohYdFbIme1/A3dy8l1Ken1+Wbff4uGiaLggTymKttey8hXqYpMO7d+/PxtP9psUgFI0dZFNsiVX29t3jTz/54re//uevfvP1Zz/5VPe6hZ3u1Gw4rbbl25s39wA8e/lyNBw2TaO0DQLWtm3ZNJzzbJg2TSOl0FZ0slW26SQxRlGGsyzlQY5wQHCotanrvu8VJhoAK6XZF3Y8DrabjuDi2fVk/bg1vR6EsbSwrJvOyqbaAWyMtYTAXvbtZvfy5cs8zWgQ5ZPzcl9s12saBW9ubrXWWTqAGJVlefHkKkri//Sf/pNzDgLysFzUfYcx1tZwGmx2W+fcZrNZLpdRFKVxUuz2CKEfvvveKV3XMgp4FgzevvlhOBzmUeT6NhsMx0lQVJpYE3OKCOz6Zr/fa6Du7++fXb9IsnA4HJbV2yCKLQBV20BCAWNCKaUEJlRjpDHiNJBG7evm/OrSaPX4eG+Rsk5oo7RREEDGAoyw1c5agwHmPLMGWAcDypwzphPImTzguqlZGDRd28s+y3LjgNZ6MBiYXuveWqutARbAthYIIYBg0wsAAMacEgQhhAhjxDnjUAKjgXPOaOwsNVoDABAGnAd932stTzWrrymBdgghTuippieQeMcQ3zRxzjVN0zQNoZAnMQY2jVmWhU3b7asd8BO/yjgAHMIAIGONBQ5hCgDQxhDOAADKOSsExhhA5JztesEYcxxAgA7FtPVRFBj3o02lz56bpvEZc57nSZI457z0hac10LaTUmrZIYToMZeySlmjOaNt20DgB0BkEHBPK/aiy0pJdBAuM1L2URQ1defBvzwbbTYbSgjBdFeUYRhSSrWyWvcYQkIQBABRLEVnZS97EXCotdpv5wGlbeMVsqm1RirtjlqTp3lOa60QnbXak2E54sAaYA2AADoEAXRGaymgs8jTRhBUStujcHXTtAghZKFxiDMGCNeAYkYVJNpqiBEiUMsecESR6/uex4m1EACnHXIWQuAcMBYgCCzGUFuDkBNNvysLn2f8ZJTWjVVAQISEM42WgMCu68h8sfAnQkqpLSSEtF1ftdX19bXPQ+lRvXizWm82mzQfBIyT0UiKLgxDiuH57CzP849fvdpvd5vVFiEyHA6zdNA1bVVVrWnato2ieDAcZ2kghLIGAKA2m33X93QQJEk2HOY8ih8f79u2betadtKLopd1RQjNsny/33uFcyFU18v1ant2dnZxccE5f73YAQCyLJedkEJbaxkLKMUIEoJZ38s8HyJE2rafTs+s1cJYhNBms9nvas7jtukAAOPxuKxcWZZltb+9vX316pCNevqxT4o9BUxr7SBCwFbVPqS0mow5Ab0U2/mSEYyUZQH3Xai27a21nRQYYynldr/DEDDGwiju+ipNU+skIeTs+mq72XHOX370ymj39u07C8HV1dXf/upXz58/n51Nzs/P/+Ef3lNKKOV1Xb86f3Zx9SQfjlfr7ePj43K70wDsdjsHEec8jHjTtuW+kFJmWXJ2diarzeT8cgqhtGC5XEKH+r4vywpB0rfdaDA6m54xQud389u3txjjf/3nfxqETDUNJPBqOmtFhTH+N3/z31RNs9lthRA8jB1E6/VaSYdJEOfsbDp7+fwFI3wwGISMYwA3m02xK6WUlBFn7HQyebi79/Su/+F/+L9GUVTX9Zs3b+qqjQIMIQGGjMcTCDij4cXFWEkLAFbSOYtpMHSuC0I2yNK8q3hAMLLT6cQD9Ztt2XXb3YZagKp94ZTmEZ/Ops9ePkMIzOcPu2ILkEvT1BcuEEKHoG+yEEKyLCMMY4ruHm470Q6HQ8YoFA4RZ4GZzIbPXlwHAZdzKYoOYwwx+Mu/+FcQ4sViUVcN52Ge520jdrsdRtzaxhpoLbAGSKmiKBoMRhjjox6z8uWgb2kbYzAhvtfgZZv9Y1hti77v27ZXxniHns1223adtcDL4x2AUwKAx0uBHY0HSqlsOMQI7HYbjKCUcrOeP31yeTYdCaX8PjiItDV3d3e+tphMJspaf5MDeBBoM8AdIUdCKTXOlvXOWhtFQZrGAFqMEaFIyh5Cp4201mitEQaUUt+1SbIUQqi0EKKr6/rhYW4MHE/O4jT/6tvv+r4LMKzLspU9xDhn+dnFBcZ4vyuM1AjApqr7ph9kVimllTXGhAHHlCdxBmATBMFut/Pi1p6q5nFs717BOfeta4+peML8yRi6k4Jy5hEyjLEf0rPWAgSNsZ7klmXZarcYDodZlrV9t9lsAABZlkGENpvNCVf3CLCH3xEh1urTSQuCwHtY+N6E7JW1IAwiSjmEkEDiyAFq9q1xCKEXLfjLv/zL/X7/uJhvd7uiKIyDPAwAAP5wDkUhRhhjiBEA4NisdgABjDHBhCDfqD5UqOYosYwxxgSeeO/wqCiAMSSE6F6dbir4gZ6EX/l9xezXfwAAYyyNAs55FEUIE79XWuu67cCRN+ebAtjBE9rvjv5PWmt0nKI0RxEh+AHnDiEEIPZH7f/YQ9RSSm/L6XmIQRB4ZdLtduuFlk8HdcIz/Enu+94jbacJlLbtTtQ/d2Q+9n3rjcvbrvXSAEKI1WohpYSYHIzBHPLMAwghRsSCP5hWOCH/GP84Murf9Zs7tRI+6O/A03k4nXnwAbfg9JdHWMKcUIE/6i9Y56SUVv+BoZKXcvoRkDh+uf+sPzkWQE9T8LlXGIZpmhMKq1IjeuhhAQQNcERKGSUxpkRrjTGO45hoIrQwxlxcnE9mYynlbrcpd3uIHEJov90BYCml0ti2qjECzrk4DIf5gCCspbEWKKW8C+1+WwzORlo7o6ExUEkjek0Ii8IsDNKyriCEfad0AuMoyzNh9Mao7mHxIITyRxUnyfjsfFfV+7ppux4iLHq9XN7GcZqlA0YjI9dCCAIJY0Gx2yMEvC+O99jWCOaDUZrmzkGvh/r+7r1zbr/f73d1kgw4C5I0nk6nAMqqKoUQy+Xy+vqFXxcwxkIpqTXRWihllNJSAoy10cDY0Tj/+MXLNGTz23eru4csTZIoDIJAKGmc3ZeFNzSjjPuv2u02+9368mqWDQaTs1knm06K1VrMZrN0MCjqQklz8eTi8vKy6/vHx8f3799jiv7mb/5mMBh99dVvpRRJkjzCuQYQYgIQZAG/vLzc1/WbN2/I3X0QRUHIfDYDIWzrWmuNEWZhOhgMiroX0koplXZCmpDTJIyhha+/+WH9uHr79u36/rGqmsjaZ8+e7ffbqm0++cnH2SDtldS9VF0vG8HD4MWLjwMe3oS3lMYQ4iQHo+EYAxjxSEnHKXPW3t3eIoAppQHjVuk0jhkhnFBO6POrT5q2Xc8r2YEsnjx9+jSOkrquu04AgNJkCAE1qgt5HkVJURQXTz+Wso1CnMW8qZcGNdDJwQRDUJWrx9u791K4wWYW8gGh4SDNOQPaqvvHu8f1HHE8GAx4wB1yURImWcw5ZYw4hIU22oGqaopiyxi+v3/f9Y11Ko5DrTWE7qdffJokSZomq9WqqoooipIkOb+YMRput9vFfN22/XjECeYYU9HXnDMltX8em7rb7ws/PHl2diaECJNYSuX9lry74Gg0antprU3TNIoiRNRBoVkoAACy0EpJiPNaNBBCC6BfEAHGGCMGsV9W6rKMoqhuykEaA2BXq1WahF3XlGU5nz9eP7nwo2XWuThJwiDYlYVvoCZ5frCdNC4IAkqJtgACjBAyDlgAISbOSUoxADiMaBhxnyVYq4VoMEFexE9rTShi7Ij9QgwgjsJkMplMJqOHh0fVy9lsfHF1/f7utoxYlqW16LTowyjC0AkhC1dbpSGETdt3QvZS1E0znsz6vmece4enTvSbzWaz22ppDmw1FqRp7lV6pNQQYo9mexGFJMkopUqpcr+TWgOrhRDD4ZARWlUFBKCqKmW079HYtjPWYYR4FAZdkOc5xriqDgJZWuu6aUajkXZASulLCN+Yp5QqdyDiKaVIwDnnxiqvGuRDJkLar9TGGG00CanW2hntDILQAWcQdN403jk3HA6DMFwsllJbznnXdcZZY41/ogmCFjj4h51pDBEjNIhCPxfg4fpTTvNhUDl95Niqhwgh30v1x+KzRh/U/bedAs8peGNgfaKGCeWcE0J8MNZaA+dZCMYYA9GBfKC0PiUKJ5TixKX3KfKpeQEhdMD6uOVnSXySp7X2YjO+u3RxcfH8+XOt9e3t7XfffOuOQ4Y+7vrtaq29V5m/iN4qRUqpjs6fp4tyenHON9sVQsgXybvdhhCCKRMHm3jkz48xhmCKGfeX2B0ZhV4HkxxtM91xDPWUE/wRC+GUPJ1+eWJdnFiNp+sFjgqV6I9SBGuttQC4vu+dUe4DS62jOKRx7tiGsNZzfyCwjFFtnFESuoPOh++4YUoTnDb11hjjIHAOIkQQQiTNM4yxNNoBxwhmYZAF2Wg88Cdl8TjX+kDlqKpqv98bZQlBQRDUdQkRCBkF0LZ1s9/u2rohhERREvIQAhwEAWFt1yqMeBAkwGGlFHCE0QgRen5+XjV10zRagWLfIIqCIBoOYVdv1GBAKe2lJoxL65S2o+msV3a/3fEgojzc7N7f3T4mcQ6B2+2K3W5HCAHWCSHCMIyipO/bpmkZ41dXlx78+eTTbLfbmLb1zvEAACEEQq1WBkDnk2uEkJchgxBuNhvnXBSFgHPf0NJaa6WsMQgAp3SaJIPBYDaZPrs8SygpNsssTQZZslY7iJC1drVa1W0DIJxMJlEUQYyfPHlyfjGF0Pz6N//4m9/8Rqnu+YtraNR8syi75vLiKooSgOC+KLTWg9Hw9evXr1+//eUvf/nlFz9fLpdluZdSAozqtrEOtkImEJ4/eRpl2Q9v3nzzzTecc4ccJzSKAoQQozjLMkR5r3Xd903X1W1HKU2zrGvFxdl5HIZWm+1qXe+Lel+EjLMBGYQcKdGX5W69uKVoOB4Wbf3dd9/1QjhCXrz8KOJhlGRRmHAeWwN+95u/Z5Tv9/uqagjiWZTGYQS1VVpHPFBdb7Tu226Q5UWWn8/OCAiL9WozL5BmeTrEIGwqWRYdwTRNc87S5XLd1P352bPxaFqVXSNBGOQ0gI2uatEQDqKIhakLA1Y1iIe667uyADVsBtl0PBrFo7Asq7d3N9999x1j7MWr54yxtu60UpRzB4E2zteOEMKqbXbV5unTp9kPidmpXvUxDuMophRDjHfl/u37m4eHB+fM8zyzEAhlfvPVN23Tl2XlHIBFAQFWxmEatr1gDHIWUkqR0BYCiGkYx54s0kmBEPa2dZiQLMuCIBDKRFE0HA4hhJvd3hPHvGsAwrRpGmMcpbRuOoxxEETGAQAAcghihAFyvjDVkrPE2aBpC2AMwUDKHhN4cXEmhNjtdpiS2dmkaZogCMIoSvJ8tVo1TUMotRa0nUBCQUwdxFprRCjlwaH6hEBIBZEzxnj2BkIoigMAbFmWZ2dn2hzEbRjlGFNjjHMQOEoIzwfjwWA0mUzOz8/3u/L8bMopCgM6GuZplqmtinhAKW2ruu9tlqYEorput2RLMEaIVG0nu75pOgdhGIb5cER5IJXhLAzZQTZgkA8DHlZlraQGDopeIogJIRAgBLE1zmGAEcmHA9kLqXrK2Xg00ELWdamUUkYzxuIki9JUWie0ZpRZAJ89e5YkiXcrJoREUaSUaduWMSaN9bCic866Q0yyiPogWte1kt452gEIfACOAsdYQAnzRqBGGqd7ay10VkFgjILABpxiHO52u7qux9PJcDSqqhr0MgzDsiwRpto46yBwABjnB/LNEYh2yGlgKQXQIWCh74WfCIw+XCmltHFHFOEgVWStVUobY0Ia+NBy4vf56X8vDOUJAdbaozeE65vKORdFEQ9CTwE2R6XLg38QAJxzxkNP1mOUuiOh4Y8YiB/siTqhHcbqD3f+hHZ46MhH3/1+//j46BEFz1X0uYW/b32KE8ex36L/vZc5EUJ8YOp4CMOUUghDX093Xec7Mp4Ek6ZpLw/ZMKM/hnlwnFYwx6kNCCGCnktxuD8/hElOnzrFeP9Zv63TmTdHlYsTw8MejTfdia8A3CnPgEcmCgBOSgmdQdYQfFBh8r5UzjngLASH/xyAyDnkHKVUqdY3eoA1zmiCYBpHQqggpL1QbSectl7FByJCWBTWda2k5JwjBHa7XZrGWZ4cWakOAEAZRQgZKRqAdrstxlAGgVIiTZMoDI1Rzrk8zaADnIeDwYgT5iwkhJRlvS0KSjmjnQMYI5rksbVus92uVhvCqGecrtdrwlkYcs75ZDL56NUnUqvFaoMJL+vmzfvbs7PzJMvrtsMAEkIDnqTpEAGqtQZGt1UZBEEQBHEQxnHICCq6bi97adT51UVR7gghV1dXX33z1Xq99ifXp35aa6OtsVqILssDpVQYcX/3+7tWa80x9jxbXzRgCDHG1llCaVlV337/ndW9Fj2NgsFoSDEihMRxrIxeLJd120RRNJqM4zhWQj4uF4vFw2iQxHF8/ezZ7d3b27u7wSBwDiJNb+f3aZo9uXiy3W/m82VZlmma1XX97//9//xv/s1/+8nHP9kX26+//vrtzc1PPv/py5evGiE3+33XdZiyKIr8w2ytrlX5cN8CAD559fLllz9rtGzbdrPZvHnzZjlfPH369OryMs8yq83tu3fO2CeXl2Wx/eqrr8Ig+Ku/+qvLNP2nX/+LBzOqqjLI8SjkUYQYHU1nnPOmqquyWS4Wj/cPVdVUdVlV9XqxVMpEYXY2mVmppBC7bREQ2rY1gajY7p6cX8i2fXpx+e6H9w8PD7pTeTokiNy9fb/d75RSl0+uGYuAa7uu86Wql7n93e9+d3l5liSoLueiX46GLE7zMKKy7ygnw1EKARESia7XTkKkNGQX1xdBGnzz+ptNuduUuzROjNH7am+tK6oKOqeNSdMUQvjw8PDDDz/MZjM/8kApPT8/D8Owbdubm/dCdE3T9H0/Ho/HoxlCyNnH+4e5pwdjTIVsIITOQohxL1rrMIA0QJRQGkZJNsgHo+E4o3Vd99uDQGGapqPxeDY7V0o1nfDFnJfhE0JACGdnZw8PDz4pj5MsHwzablt3rXYAAr8WWWSQs1AarbVOIhYwMhmdPTw8KNkPBlmx22AMR4NJVZePi8VokI0nk6ZtOykgwedpPl+sirL2hEELkNa267pOSI/YI0yk0tYBjIhf9P1O+gUOIwqRQ4j5DouzCABsLZTCKaWlsAAGwDFKWNfqsmwgwFqazWYzHII0igmmDpOIheE4xpStNmutWweA0Gq92u735dX5RZakRru2l03XCaHKulHGGuvqrgUYX8zO/ZPIOfeSGJzz8/Pz0WjkV/lTXEmSJMsyQuF+v6/KPUUYYaysQQQjZyFGmBILgVDSWhvHcZYPpdTLxf1+v5dCn3yNCSGz2Wyz2Wht/AmhlGJyMD1iUaqUv47IF69ByELGfYUKLIQQa2Wk1BBCxthqvyaEUIyMs0J0wBmCE4zxeDy21iZJIpWqqqrpRBhHzjlljXbOnObgjQeQD2MOAACjjHA9MM5TWU+FpnP2BGg7a07HcrKU82rOCih3nNM7BTO/9PmuBz6+/DMiuwPx1htqe2Vin0N4rDsIgiiKGA/9UKKnOzjn/MXq+94TBby2rN/nUxYCADBWnyKrzyoQQmEY+r/3R7HZbLbbrf8xS9K2bT1UcEhbGQvDcDAYWGs9a/JDyOGD6tz460UppfQwXmiM8X3J5XK53W6dc3V7mKtEMWEBj+PYOUcw3Ral+WCsEQCAIMYYSw1OiMKPOcTxYp2iO/pAyPKPcghwVEc4ZVenDSGEwLHfcYITjDGefIqAJcCCo9Qm8hRYYAGwDhxdN6FD0EEItRCq782PItCHXE1bw8NEGi2UxhAa6yghAEBiIdBaG2sRxsbZXbFvmqrrU875dDoZDTJrrZcjjOOYUWqVxRhyzoXo4iSKw7BpKiUkzweDwWA4HBPCFvOlpxmPZ1MeJ9vtdrVekmIfRUkipRDqYb6QUgZRyDlXSgohWMiurq7yQfpkNqM8XK43eT64fPrs9nH5+u3Ncrsry3pT1NA6BjFn0Yvnn5zPpggAjVVV7hBCaRJiMqIUQ+ik7DabpRDNu3dJ1zcQIcrR/eP71Wp19elPfQaKEDLGxlE0mY7ruqybbd/31unhwBBC0jStK1GW5TCOrdYIAG9xBgnGlCIEpDMWw/vVYlduVouHkKGfffFFGLB9Vc4uzpuue1wu6qLBlAgliSCc86fX1+PxACJze/fWQgAwwgC/uXn4+OPrydn4/bs7oeRnn38BHZqv1r1Uzrmu67/77vuPP/6YUzYajH/62Rc/vPk2ybJ0kLuquV8uVjdvO6G2222SJFprQjh0oMXIGAOcWy2XPI7my1VZFvvt1iiZROH100vn3H63I8gJ0TmnV/ulAvLFk+sXHz+9//aurHtLCOOBgzZOwiRNWyUuLi7Gs7MwYOv1qqrq5WK12y53u+LTj5/c3z2sHxdGaQnFbrkOSbDfbK2ysheyF1EU9U17dX6Rxokz9v7dO+PsbDoejUZN1+w2a2dVHHGj+rLcEsIIQZQGj4vb7X4VRZFzDbC1FKAutn1fhjxrG2WUVqIDkOXDM4iaspbKtrUql8UCGx4PongQDadDGlLIUNlVohUAAgtM1dQEUQghDwJOAwSXN+9uZ2dvAcSYsNF4cnn1zDlXVu/rRgihCInCKNAGa4Ono+nlVf/9u/vttlTSeNw1imJGA+Bclg0QIs4CpZRz0JdTVdVcn11wzgFGRVF6RNcP5Hnd9P1+r5S6vb01Dvjuxs9+8XOp1f39o3E2y7Kr66fSWLXZ9EIQQqGFFvhq8iDWSRFp6mIyHoQBMRpEASuANcaWVdG29XpNhqPcIYwosQ4obcqy7KSQ2jqhMOWcc+OAcaBvGmMdRNg5VzZ1DGCSJIQHTnVBEIRhaK12zkVh6oDBqCqLzrfCrXUd7I+AKm6lssZOR2NCorqSXSMfHub7Xf2Tzz/TUjsNq6LuG5nmA07DAHcLWXRCEoQRJQhjykMeRqLrZ+cXaddVVaWkhgg3fecNUaMoadtWSu2VDzgPr66urq6u0jSdz+enjiEhJE3zyWS62j5WTS2ValVrre67jgUH+oKyRlSV3O2Ng09Ho+F4VBb1u3fvKKXOwjhN/Coxm51/+vTpv/zLv9Rd7+WDKKXWAQ+PO8yMUcaYvjcYQUIQJgwcZjKl7JVSpu+EN1gKWXjAxiGAzlpr/eAjOHhPGN+PP90bVVUhGh2qbV9HHtvtWZZ5flxbN0opJeShO2DNsQr/gHZwlDcF4KAf7JxzjjjnEEQnOAF9oLr4YfXsjiMDhBAj+0Po0sbj+T/qf5ODxUMQBAgfZiDruvaqTeg4Z+E3d5Jp8omCP3xrLcLQk5lOiYKXkPF5gCfhexDCayRYbU6dC58G+QfTHAWP/VnCR/8CjE9GMNoeNbMRAlLKuq4908WTErxUsW/HY4wZDVjAESLOOXMgpXglg6OaNUAIIWh+DOSnU+pxC/CBHKQ/CacLdEoRTqcdnuSZP2AeQAi9sP2HiYJPCg85BzzgEFo7DPDpqCGEGDoHHTx0npzR0jkHnQPmMLuBEVCyx2k+yEeMBRBCRDAACCBsnCXWWhYGHMIgDCmGlFJrtT9Nu/Vms1pQSvM0CcOwFWK5XD578jSKgjAMd/uNMdo5Y7XWUr6/uSGEMBZA2D88PGBMnz17lmWDXnYYw76XjIdd188XD0oaB9xgmHvbEAdY10ELAYAaY6iUWSzeL7e7KMtCiGjAEaHz1Xq/L4tdIboeaDPJh1k6CFmMEIpCmsRca804CsIYY9z2XRAiiAzC9rvXX19eXmKAV7vlYJIbqD/77LO3b9/5q9j3knN+cXFRFMHuu0UQBIQiz+vE3tTcGKM0dAAfRdEJIRZDra22BkZcYtCLZt2Uocb3+3VR7GIGkyQpi7aqqrJqAEbz+dyLkZEa7otNnPDz8/N/+2//rVStlP3//vf/a9u2QRz84k9/QUjg5cx/+sUXGJHlchmFIcb4hx9e/+yLLzjnT66uP/npx8Chsq4eFsuqqparzWK12RX7i4urIAjOZ2dePosgDKD9/tvvwpg3TZOm6eVs2os4CZlWXdd1V1dnT56eKaVWqwWN8UefvxyNRrWuWws/+vJLKaUFJklizOB2v1Pr5WA8whTVbWV02fTCmI4hEAX4/dubxWK5WiwQJCgl2+2W08Bqk2W5MzbkAQIQGEsgGuUDIcRklGOMWcQItgS74SBCWAstCbVJjNu+E0pTwte7R6Pd9fX1bMIx6IzQUcgCNsSO1HvdU/jy5aeU4qKq5/O5Xq5WRbmYz28e340vL4VVWZzEg+T6xfX59Pz+9kFKHfGg2BVt3w0yRjhz2sRpcnF1WdetMe7zz75crVZZliVxXtc1cFgrV1ddkiTG2PnjPSUBBKRrtQO47aRSCiCkDaDMMo6NQ0mSYESMMQAgDgDESEh9e3e3ffxhOp06BKvqIHwWRpFnDvruXl3X8/mch5GHuPyQUpqm+6r0JjRKKQcBJZQQaiFw2h1KAwgscGW5F4Jrcfb5Tz5pmkZrmWVJVRa73W48HiIEAITz+VwpZSFQ1tzePypp/NR+23dxnCZBABA01lkhlDVt2y+X68FAe7Ledt04ixCkQgilpbXIObfbNoNBBgCBAANntNKeGxRFEWmElGKQnw0Gmey7MMjybEQI220KygJOoSRuPEiiJMOYuDFDKVdKEURnk7MkjPI0Y5hZ7TDGWZoHPDTO0oA7CykL4iSzxnrqz3A49MW9L1hvbm78+LT3Do7juG3b9Xp993izXq9nkzHG2Fotu/5sNpFW9koaY7peFlWDKXPOtW17e3urteacuyPJy0cjv62yOTQuMcZdL7ypLCDc0wopxVma5HkKoNVCGmOapumaXgilpDbGYYwVOWjiGmMQcAgheJyXe/PmTdM0UivGeZqm2h7aGTQiGB566lYpDJGlADEWhnEURcA6YGFb1T6iW2ANPAzOAeA+DPbgWDv6xU1rbYyGEGJ2aEn4vbJHXwZ99AIA4Ed2pD1qByGEfE55KJcRtNb6KOuTJKU9FadTde3FDMCRQOCnW71+5anLcNoKwuQEs9sPqme/Yp+aF76PoJRihB44BBh7xUafolFK67oWQvhBCXsUbPiwyvdbhEdfhtVqZa0NgqCqKufcdDp1zlkAfcaglXUQ+BTfK7md4vdBocEnCkqdIr1PWP3Djg+WzRYeR1XR8XVKFPz1Qsdx1hMBAh5pj/YDQqI92DFY38nyYMNpAtNZQ7yUFnAOOAecv98AAAQBiBAAIODUOoi9wCTnDOO+7ynlcZYSRiGmjBKIASLYGkC0NTwMKCYYY87pgHMArFaibds8SQLG+rZ7eHhgjMVRMB1P0jgZj4d5nnNGimLfy84TgtI03e12Xd1YiHxRtdls6ro9v5zmg/Tzy6fnl1evf3j7q1/9nTZyOJrM53MeBqPRKMsSxtBmv7u/vyuK/dPpdLPbkiB0iL55/4/z1VZZJ43tpQQQa227qsmjxFpQljV0QCnhgNJGOBvwKCaEGKDSLBn1w8Ew+/033/zsZ1/QgEopv/zys91u99lnn+12BaVLCFXTHExivJ75aDTiAU3T1FsOWmvDMCQQIUz8mDJCCGDonJNGQ2eKpm57d342/vlf/tnZbIQR/N3//M2L6SgfDnpxGPJhjK22K4xxnuf5cNC05c3NzeP8ti7LFy+fPn169d//9/9d03SDfLjbFVq64WC0mm/m87nPZEPOi6L4D//hP/ziZz93zi0Wi8fNXZYP83z42WefffbTn373/etf/f0/Nl27Wq0++uij58+fU4zv72+zJI2jQPbi3dvvEEJPL86fPLnc7/dSyzdvXz88PIxGg060g8Hg8vLy6bPL1Wb99Xe/e//+PezGjBGt5dPrq+nF+d389v37m/PzGUCwaWrbOEIoBIBAQBkMDVktGn0QUacE4bqqNmTjjL04u2zbNo1DTzJHCF1eXKzXa6Fa05t+02pgBoPBxeUsbcOH+ZwzeH4xfVysHt8+MBZ0QkopqzojCG/7hhE6Ho/zfAqB9TfqelXxIKjarmxkK2QrxbrYFEXTAYMIfHL5NBvmT59dJ1FaVVUSxW3VejGi0WCIEGpF7xsN8+UOYvbqk0+DOGnb9n6+2O/326KEmCsDeZhSSrf7uqz71aa4vb1nNIij1EPQfS+FUIQoKWXX9nEccx76LlhTd2VZtm1LmHLOKaW9l8R0Og2jyJMkRqORL7astV3Xbbfbqqom05l29vLpEwsBcEhKqa2J4xhBgih1zmlkLXCnaX6rNEEcE/jnf/7nSomHh4d8kC0e53d376+vrzeblRDi7v6eUNT1HUJICxdFEee8E72oNedhGIaMB8DBsiwhxN7RwAEwGk+tg3GcxnHKORdCaeWMBghhBHES535F8+1hSul4PB6NRsqhm5ubMEgYDUUnR8PJeDQ9Oztbr9ej0QQTro0jPMKE91Jbayva/fDtd9vtNgrDkDDPnIrieJgPOedSKyGE1LYoCu/5OU6HWlsptXc2qqqmad7e3T34ktS7gIZhSCnf7Yq2fbydv+2aZjodx3EMnGm11lpXVeUHUA+QOzJCycVi8duvv8JWKaW6VjgIfKW7XC57IeI4Lurm1HU+Qcdd10HoCCFxHE4mk4uLsygOgLEeuIYOce6MtlJqpZQ1ltNASimVJAhijNDR4xQh1LbtbrdL0pQxlmWZMjoMQ2ktphggaNXB4NFHi77vOWUfRju/VyELD8NZ7jBl4JyzzngtUX10yzTGWKsRQhSSYw16CNs+3Pol8Y8inL/tfXHv4fQPMwxfPfvct2l7v7kgScDRKcqH9lNf/4/weX8+T5tzR+UAd+zoe+gCQpgkyXA4BACUZVkVpU8UTjoibdt6stpJ+fEEiiCEhJCn4P1BNW+dc9vtthethyuklBh7OS/lIQqllH/ufJP3FLAP9TrGEBzMGE/ffDpF/jfuA2boaa9O58EdFS9OaMqp3XDqm1hr4QcNi1MKBQ5DFofvds5ZaxzBp80dUhBgHUQeXbD+NgbAHXMd55xove0zgwBDCBFBCDmMsXWO7LbzkwDF5eXl2fnIGPP4WFioQYANQhIDRaB2Ogz4k+vrKEnny2VRGB5GMaOu3IfW5cNRHMerdcnD5Pr5C87T71+/mS/3Bjjx5lvOOUVqt7uXUv/Zn39qtHt3e7vZSQC0A4G0cDAZIga//v23i9WyF/Unn3zy9Prl7d3j7b/87mG5SdJhGMVWucFgsN4sZ9dTxNFv735zcT7dbtcDAtI0qep9EI6ns2y1WTFuJtO41yEE7tWzlwnPMAyhkdPwVb++/f79m49+8lGv7Jvv3zeNeLy7H6bpdDb85MUr6xQLqNJCavHnf/WX/9P/9H//6JPPuwJGaX55fuWgvb17r7VOU4ZDiBEAQkutG1B/8eSTIUwGcfoqfzGeDZQgfWeAw0kQno9GYl8Yq9vdkrpuNb9ZzW/nj3er9fz9w3cY42wc/fVf/7UC5pOffPr+3Z22Oh1lRV29vn09G82uXz7///0v/18esV/906+ur58sHh+Fsbt93bbfWgOSJN1ut+u7u74s27b/vutVVb969UoptawXw/EoCGMeplVV3T48xnF4+/5d11cvn12LYn23nishHp1jX/78808/Xy6a1W/f54yjkP7n//yf//RP//TJ5KzdFpt3iwHPIhCVj7vpbGad1v9/tv7jS5Y0yw/EPm3aXLuHfvF0qqos1VXdQAPoaQwHwOE5JGdFbrDiv4X1kIc7niEHQEO1QIuqyqrMysrMly+fDO3a3bR9mosvwjO6OL7IE/nCw93Cwtzuvb/7E1zlZVnVRRoEjGrTGa7n2cHoZDI4aFsluKY28TxfN5RI//e/eQmhefzkgSirkwdHB+PB1cX1L3/9D8pIgO3V9dvhuOfFjARy22z1VR2FiTRZvl6URTsa7l1evINe4JHAWtXUhRgkPvOVkJhIqTJErIFGarDZwMtzXmRxJz3hVi1yuSzehWEQ9/cEIqum7KWdyI9Gtr+332vzMmDmenm2unmDAFisixD/ZH1zOj+/yPJ6MBhaoT54/JwyP8uyMIkXi4UfRnEc5nmedjtekrbCLpdLYkmnlypl6lZASIQCTENEoNRWVny92ZZl6fv+Xi96cHr813/91xiCOAiapq7KYjgYTCbjMPRfvHipgR0Me27diyDJskI2elpNkyCh1GtKHpKwrmtttax14EfQ2LquvdCPvchWBhD2/OHjDx49yZdLa203jLAGDw4Pjvf3Op0E4w+m0+vY92azGbEQYai8QEoJIPa9pG1bo6EURvBKtJJixigbpF17dNLtdodpyjmvLRoNxkEUpmkXAJCkKUJocnh0eHh4M5vmeR54t/T4uNvt7+1VRXt0isMk8aMoHR4aY4qiQFFSTDOqsObcWtv1AqVabVSSJACRTqd3fn55fT0dDofdtOPUd6u2gLxsmqauKtdws5CuVqs07Y4O96fr5c1izhgDBM+XS6BNlmVpkji+AsZYS8WVapsmZmHiRUAhn0XGmKRDGwFZ0JsvFs6ooNejw+GwKcuyLNtikyRpHMe+F9ZtkyRJVVUO7B+Px7Plyun00jRdb7YOeI+AZYyVZVmVJSH0yZPnFOHvvvtuerMKPL/IawghI4Q3lSvP7aLwfd9nzG3Wu90uxsF6nRnjdbt7jLEyazEIOmFcVVWII0p8Y525kwtfINRjiMC8Llrdaq3LMrfWxkkYBAGyEBTCpwT7ngF2V78RDoUQkDKNbl19IMHWUmsthsBaAwyAEGAIjNGibYy61Uwaa5WwhJDQ97TWvKkZY1GcGHPbrxhguTBaaw0g506+3iqlEMKMeRAiCABGiGAMrEXwe8Y/hBAjZO92EO44A993pU4b43BcBKDVxgLge55ouaOeWG0Ws3kcx/uTPYyjb775BiMAAYEGpGkaMk9K2Y2TFRdBp4csVAZiEixXm/39/W4I27ZtWkEI8Xx6W0QRMMYAhIMwVtoCoVwf6QWAMqQVaJTEGENgqyqrqqptBRccIcs8orRVUltz64wptcIEewS71goR7LjtGCFggbVgF3lqjEXAWmHdKmQn63W7lV2rtGsybhEX4LqK247KPYdRhhASrVJKIog9RjEi0JqmaXDIIITAWqO1NYZQShC2xhqpwjBUSnGlESEOvQnDcFm/uV4aAXMSYIMACyIDIPEwOTk5dTpvR/uUUrkNDUIo8CM/YK73kVJWVXV9fc1oFIZBHKRVkZf5tiwy3tTDbm8xnZV5FgVBvt00dWm1AhAha7nii/XiZn4zGk72DvbDMNTGjPbGVzc3mJKa1xrY4wcn471JWVeb7TbyI0bDKIoPD48//LCIk2lZi6KsmqqOo3B/sheEtN9LEYC/+dVnlJLO83EQoSilFjV1u1YqhxggYpIYVWWVFcvLawoBE62iVC7XK+pNrNDDbv8CnnfiqJ90kjBKwujxk4+++up319PrtNsbdfvMT/vdHjYgSEmc0CACQimEOZAtBBAiTRkygnsMFtX8xcvfxn4wGvS1rSkdcyWFEAAgbUBVt9T3+mF/sjderWaM+ZR4p6en787eUkq0VUHqvX37Xqk3x/vz7TYHAI4H+0+ePl8tN2/fvkOUHh6dhGHIhXr37swYo4yu6zpJEmPkq7evAQDEI9PZLPD9pqnevXvjedTZFK5Wq8vLy8GwGwTBYDDodvvXN1c3b2dW6bpuVstlr9P1CL2Zzq3Bs8USE5qkqQD0008//fM///MkSV69emmAHQyGSRJfT2+wy6v1iPOto5Qul0utC8/zPBb2+32lQFMLSj0MiVJqtVzmeb6/Pw7DsCjK1Wo1Go2ePHv63dvvrqZXEFhM6KYos5t1q2TaTS6up/3hOOn1h5Pw9cu3eVkEQbSZLf70n/7zyOu8+Prlm+z16fEDpeTF5bvhqNPKerw3XKwWi8XyNr0JGGvhepU3TSVk27Z8fzzmbX20N/ng8VOGmWrFkitI4WCy11b1ZrX95JNPhuMRgFAoOZ1OL66v86yuatHv9+fzRZjEWqsgCHq9nssFHu9N9saDN2/ezedz3tQY4zjwGWNFUWBorZbZpiiKItsWlNJuEgOLmppjRK21SbfDOVdKL1crTEiadk8enhZFsd3k6/UaY2yAFULUTamUMlbFEbTWAKi1UcDCMIy1VmVVKmmM0XWNIQQfffqjw8NDg/Crd+dZtnELXafs7/Y6cRxLyRELBpODu4EPIYQcWb1tWydJV0ptNhtHMRNCjCeZw9ubpjl7/14ZzTkHEBpj6rqOO+ne3t42z5y57+4hhcjzXEvgkBul1K3gU8qqqtwy+3YyM8bJRNu2ndf5ZrV2dBbRcpACa+3tOsbzMEKMMWfgvVqtsiy7uDgLgiBJIkKQUsIYE4Z+4HlcNMvlHELorv8s2zqcI45jh1dvt1unAHcei1prx2x3I52jsp6cnBhjpZRV2VRNXde17/t13Sqti6JwzHl4l5foJtSqrtwm3m3TB92e+1ZbNxQTYwwwRhiDMY7jmBBSl5XzAHVnu9vtIoSqqsrzfJd15OIV3E1Y1JXneWkQaY/dCi4Eb1sDlMI2ppTGfmCt9bEHpJZCkDuXaIiRc7ZGCAEIlVI7wjyEEKDbPbcy1lqLgAUOpIYIQAic089dlqAxRigNIfSCsK0rKaULrIHYMfwR8RivKgCEvVP3EUIBAIgSfedZ7sZu5894+9a3yId18/1u1+Aup92CYAe8g9u4Mg7uOInWWgANRNoYm+Ub2uulSWiMSuORlNIYEQax54frLDNGEQq0kUqh3Vt8vzqh+O5KUHfLiFumiwbQcdqs1QBBB8MQQpCSCAGMoQXWJa072OP+7wLviPP3PyY7GMB1ALvNy27u3y01dl/ssAeMMYbunFv9j20VzD2HBoQQQhZZgBBCFgB0T5x5R53Ed5AYBv/IBBoomG3rIq+FML3hYDIaNnWZb7fk2dMPlstlWZZBEERxIKWsqsZaYIyt68a567tk7iD0AEAEqOAgEK3YLDe8rbUyCACKab7NmqpeL5fAmMVqxduSMR8C0EqOCCKEpv308PgAYrrabDFFf/zP/slkf3+9Xr99d+YFzI8CFnqhDreb6s2b93UlKQt8Ej58+Fgru8nzr7/+miA86PUptg9PTsOAff27L8fDPqQ2CsO+SS3QVb0RsgTSGqsQasMQ9Pt+GIKmLotqM52DvCyyYjPsjZ49/WjYHzw4OI3DBEPQ6yfjweBwb5Jt13WeZZvt0XFv2OlZoyQUXIFtgZqmquotwoDQIAw8q5WG9mBvpFseBARTs1jflO0Gk2fnZ5dvX78WQu3vHw4GI2AtNPby4ooy9MnHP2QBCyLvV5/9utNL27aZjA9OHzzmXHbS/v7hqeLq7Zuz3335YjQaMS/qdIdPnjwiCH/99dd1U47H47otjTHYYzTwe0oul8ui3Ho+jJNgb29c102nmxwfH89nS631drt1n3dgEQvC46PT5WKVl41V8MHpk9FgyDlvqna6XCmlw07CosAK4mBwLoUbqtJup21bznle1Zxzxkiapl7gKy5Wm+1isWhqjhFTSmFMg8AjhCGApZRVXWgtO90kSZKy2KxX2zAMs6pcbNbT5SKIgu6oY6FRACdpUjVtUavyYqq1TZO+0MBasykWB8Mj2Upu2yAIatvezK+t1ZgBA0VWrmLuTZfXq+2G6yaIAyHrAHWauiY0UMZe38yapoZW13Xd7/cTPzzaO4jjtNqWRdmwII06g/3J2Fi42mbKAESJboSFwFrb6XTfX1xcz6Zaa89j2+0WAEAICgNv2OuyZ48DhpfLJYKEMQYA6CYHvu83TdNWhVWSYhuHLI39R0+fdgeDwXh8eXmJymqz2TAvTC188+ZdbzCq67ooisFoTJjnduQ0iIq6EEIoq1zqj7JKA314dDgcjo0x69Wmbdu2Fa65x35Ew4SGIa65b2AY+QCALMtG470g8BClCDEvRlAIIUQtRK/bc9lmxhgXa1m1vGkaZYFHKGaeR2gMIHVqQKWjOOWcC6mJx4Rs19mqbNrJPprPl1prAwBU1iILAGgbYfQ2ihJ30yzLUikVhqGT3SOEiqLY8YjF3WNTZU1dY4SsMbPpdLNeu8rNOY+iyPf9TqdDKXWmyEEQRF5c13Wv19us1kIIx/6Jw7AsS960roq7dbXDsRmj7mbtdKd3JisKY+yWjA7zd0+YTCaUsuvra1fAXLvj+6HLBXVv5zCSPM/dClxZ7XkeABYaK5pmvV4pKTfrFUQg8JgI/LqulZKSC46xREhr1TS1ECIIAq1VVZUIIaVkGIZRFKZpQilxbYfnMc9jYJ2laRpGvlLKgRBaa60VhghbQICFEFtgkZM7CAMxuX/fvy20biVvza6QONcmYwxQt3R3V7F2sn73V3CrMQfFO+aH1pJLoYwhkAJrOecW4oAECCF159yAGUWUALegpNTchbI6lqKTBaI7IyYIodsauK2E1rcUhPuEStc2uRrctq1bQzi2BKNg0O8QBFerlR+Q4ahrlXry5MmLFy8Q0HHih1G03CwRVHHkAyuUIrvi7Uqsu1qcCnQHb+ze3RlnNU2jlMDUoQaIUgp56+oxBgBYbO1tmjPEyG1AwP/fY3eS7/MbnEkGunNcAPf4Cn/QK9w2KMD1H/oPvmtuI75uyaoYWez6BKuxhQRC7ag00EBgAEQIASd/gBAia4A2CDKKsDW0KmVT681iG7HADABBFBhDuFStkAZAwjypwHK1cWk6aRpXdauUQoQknU7TNEIaLvRmMy+ynCDQ1lW/l0aBJzS0Skd+UPtM8ma5XDZ1TRBkjAAAJPaOH57GcZJ2Ol4UVi1vpYCUPH7+9NMf/eTi8uJiOruaTTWYvXr/Vms9CY9Wq2qzea0NENp4fggJKYoiDMOmaRACg16HYoIsOto/8pmXFRpRhHBc1TmGxhrK27JpKsdpYoQiALVRbdvmZYYxbrZlDmmx3gaUJFFYZJmWvMzXScQ+fP48CL1f/uqzt999F/lJEke8VS3EdauqZtlWJReik8QBCZMwVoL7cfrJhz+MQ7/f7URh8Jvf/Ga12mw22WaTTafLMEr2D48xZRaQuim5EKN0ILS5vpmlnfjo+MEPfvCxNNoP6Hi0t1pt6loQbGaz9e+/fnl9Pf0hjcK0pwDeFC3GOIg7Xpj4YZLXWRj6RZk/ePDgn/8P//Szzz771T/UP/nZp8vlcm/v4M2rt1zUZVmUVd5Je4eHh9RjfsCG/YHHwoOjU4AYAmYwGPS7HSnl7OZmvV57ngeMraoKANDr7r19/+79+bvDw0Mv8P0wCMNQax0laRTGRlshVZZXvtCEECWt0ZBSZjQoy5Ix3+3OKWZNU3keTZIY3zp/qc1mK6WSSLVSAITLuoY5RR42hoRpf3FxITVqWiGENCAc9IZRFGutj3r7m8VmLpdx0k3TbpZtjDUHBwdFuUa+2VSLRhUGST8kH374fDZdZcLWjUg7keeHVZMboIDV62ydl1knjD/64ONxfxQP+92iQhZR6sUIbrfbqq4J85JeD9K6rMX51VV3MJBCU+L1eqEQ4v3b95eXF5vN5t/86z/76KOPHhwdImswcE71nrsVImAIAqNB7+GDY98LPc9L03QyHlBKR8MJF2o0GjFvGgRRUTXL5VZbzKUoi+rZs4MkToWSLReHhwedTiqEdJIwCHG323LOR8OJ4/z3ep3NZuP7YZqmWuvfffOtQWQ4HBIv6Iax51EppdBgtS1IeRswTynFzEOWW9UQ6hFKMWHQGAtQyx1jnydJ4nkeJgxoTaintGmLqmkaYbUExgCIgDXAcinqVVO3jRDC8e/d1hJCKIRomqYsa6ddrKrKDc3uuup0OpvNxhVmR21zfjhhN8UhIBPkEZplGbxdJN+mGQkhNpvNzXTKPE9K6fn+/nBydnYWReFqOccEMo94Pm2aijFyfHzY7XbzPEcIuG1OXddKcYwxY449SqIoYIxw3vR6nW63W9clAECIFiFUVcVoNBgOJ4vFAiHkfB6d1s4VS8d+N8a0bet5ntvWIy0JAkkUWGvXq8Wvf/UPnPPpdJqEEWMkCDyMgNa6MEoqzjlnjHk+7XQTl/Gx3iw9z6OMKi1aXpMGSSld0KXneRCGEIBOp8MYq6pKcY4Zc+WEUuwKkpTKGOPGR4aJK+2EELd6cCw/fGcLaO54/vAuGxASZrW21ioDHPkDACDvopuY1AkiAGIDkIWYML/b7WqtOZeYUq11K4QxABHMfE8KLY0GEGBHLDBGaR2F4Y6XsCtprhF0rAjHrNzRHTh3GY/fuwvYuwxDJ8V0vSbG2F0/Bso0CTqdpKkyo4XkNUJgbzK4uY4ur7TPsO8RaKXn44iwum2MhvfLsLvenCOTQzhcvXe+VQgDgsmOywkxghDfRwgghC5Cy+1i7F3oJbxzikR36Zp/8Lh/Nm4n+ztJyA5ZgXdWS/Yee9He0T93xMb7iAK6k5ZgZLFjyxqJEXXHY63F7pghgC6KAmoELYJQGwOAIQRh6CkBrIZn72+qbelhPOinjHqkqrnSAGFiLKirerMtqrryfT9FVGqdpJ3Hjx93OsnZ2dnZ2RmACGKwyda+R7tJMhyPMDI1NphR4jEviDjnEKMwjgFCGFMLQdxND4+Ptdbz9WpbNsz3LKGE0E1RbMoyq1quTZp2me+xIFwtNzk0nAtjrVI6r0qpFJdiu92ORiMlRa+Thkl8eXVT5Nte2lmtlxcbPRxogq1oZSdicRTXwqpGJX4ihOBNzbGJg/7hQRQnnSRJ8KqQUr96/ZKiIPDFdHbdSzsIG0LI3t7YIvPu/fuiaLJsgxAA0ATBUGtdFYWQNImSTpoSGJiW+sTrRFEUDNM46nZ6h4f719fr9++neVF7lKXOp9KYb75+IbmgjPjM41x+/c23RdFAjCb7R1LDvCivb6qvvnoNAEyTrtbTv/rLv53PFz//+R8TEqZpNFtmRS339sZ7J48QBnVdZ+d5q7mUvDvofvCzn/RHveGoc3R0dHVxrbW+uDi7ubnK8zzPqtGoIZDczKYPHjzgQr349nXTNBSjhw8f/ujHv5jNZtn0uqglogEkrOU1d+gWRcynXIrr6U1d19paoQyEsOHCD40ytmn4cr0FAHS7XWVAEnc6ac8YgMAtktaqRoBWabG/vxdFIQCgqioEiVLq/PyytZUb1Fbrddk2nUFfW5ttawj8pq6NxYcHRwDA1Wq73VSTycSnfjrqVm2rpCEEjfeGjSwqUayqRdzzq6aMO37Da0jIT372wyyrPvvqbJNty7qWqqnqDLNunPiE4Nl2fjG9WGyXRwcP/sU/+Rcf//wnIYu32+3rzz63EEXdbpd4LIjzvASQrleb1Xrr5p44ShvUQAjDMJJSbTabpmk6qWnbdrFYtG0bRQljTAgRRYmTYyVJ4hLo1+u1BRoAYCF6ePr0k08+uZ5N1+v11dU1oYwyLwgTCzCXSggx6I+ePvkwHSYdzrW6jQLSWnu+DwDyPG8+nw8RZB5DBD98fPrpp59mWTbLCwPAtizCMMQYF01trSUsyNaZqwcOfg9DD3sYG7ja5BDCsuYIoaJqlQaU+gFmEFOAaMOVEEIp7aq+UqqVDUIIWs11gwAMoqSqqvlyTSml1BBidsOQewAg3I3elXkAQNM0hJA8z+9yfonDGNzNlyvp+z42wAo17g0+/uEPMMaf/+6L9uzM1YmyLM/Pzx1FUQgxm81cXXGgvRsH881WCNFNO6enp+/fv0d3BoW+72+zAkBigaaUEUKC0COENC2ilBKKADQAulkNEYqc1fEtvRze/lLuRu95nh/FbnXCGAuj2C34GEXGhTm5fqLlbVX7lEkhmroWnDPGAs/3mWetFUI4zL/T6fi+v16vsyxzesKzs7O6rh0Q4jbxd1RECaF2Cn+lWueUBwDwvPC+du5+kbhtYvStgsYxPHbV7hbSv3N6pvg24WlXwl35ceIv55fsmg/XCyIMDATaWurqHLDGlas7/2+AoAFW3oVF7QwSzF3Ik72zN96VRvetW8T8Dt5wV++O+b+rvo5R57ZgUsqWl3EcYgCNVovZDUFmf7KHCfQDBowVovWkp5VkmFCPumC/uwJ/64K9K7fmThmBd1YEFkCws5qmhFJ8G93kNIrGGGdpATFGxgCEEJcC3guoxPeSo+93CfeXCztY5T4z1N5zXnJziHumEi7Ey5p7CgjXoEBrEb69BhA0CFiMMbQQIYAANAQBCyFCjsZpCNl5KgBgrFEQYEoQwYw3XHF1dXlZ59HpycFk0LeEkB988qlr7oRsN5sNAKiua4SAklpK6XmKUm8wmAjhmMa2TcJttj7Y2z+YjAmGZb6tGpTVxc1yzlsBAAi9AGPaCtkKTSk9mBywIN5ut4t1AUDR7Q0AQHm1ymrZyi8Wy2XF9YcPHk72DqaLbdl8ud5yZSQhRGhbciUURwjRwN8UebeTHpye7O0fvvjmq5cvXj5//vzi8mbVmJtpkcaBke3R/jD0Aqh9jwCfEFFvsaZRkBw+OFXalnXFmP/Jj0+vz6d///e/2h+ftKLBhFCfRkkQJmFRl0VV9gZ9QKq8KfKy4EpzwzGEQhpK/dFg0k0jK6WRRijZYv7t1y8pJZThp08fzxdrC/BktNc0TVPX1sKmlmcXN4FHKSaDQY8oI4V59PjptlgX89mbd2+lFOtVlud5vz/0/YUB+PxyWhTg6mYRxL2DgwPix+O9UZIkXhhorf2QHZ0+KIpikk6ibrSZXowm/Z/80Y+/++47TG2WbeMkABFMk14/7Wtt6rqFEFsLfT+2tgGWhGFoLF2uinfvr2+mV01ThaFflcV8Pue8DcPwYHRwenra7XbLug7jOIqSvb09ZTQ/Fx4LQEqoFzR8vlgtuTQQQt7UaZoiBJTQhFgArBCt1WYwGDw4Oa6qyrnaEQSttUqabLUSTWu14a1spaIkqIWc3qyjNA1oqrXmpazKhjGmgMrWhX/o9/vDESbbPOeyLZosK1etybWpD8dHsDCU4/dXZwR4FqhHj49eXW/9yG95JbUw2ABmcAD9gLK4Y6QWQv/u5ZcsDH724z968rhPTfTd2zfdbn/v6EEaRfPFuqwbQlkQRHXdcqnausGYKiV8L/zRpw+YR6azC6GsNJYFoRdGWdGIrPA8L4qStNtPkgQAYIx1v/itwJoQhFArqq+//VYp5fnhhx99kuc5F8oYcxglXGkDjAZQA2gsgwgBpJSxSiinvkEIBVHiprogSroGBFFUNc378/NONynLUhsJoDEGOv1eFMHxeOy4Am0rON9kWeFkXQGjUkovL9wUBSEMggAhtNwsdiXcFYmmaTDGlEFKkeC8qipKaRSESpvNNhsOh5wLpe5S77S5FdBT7HYKbl50WjWMcZZlzuu3LMvNZoMxPjk5GY/Hb9++hczTQpZFkcTxh8+eSyn/7u/+zlprpLIEa62dG1UURdRjlQGDwUBIzhgFAHied3Cwn4ThF1980bR1EPqdbmqMmV7fQAiHw2FZEXcYuwLgKnGWZa5ZkVK6xbnLG+TtYqfvh3e0DKa17/vul3LUfWOBK3v9fteZPadxGEVR0zQQWUbxdrsty7wuq263iwKfIEApS+OQa1VVlZScEMQYSZIIAKCU6Pe7u6oJgFFKVJUyxhgt4ySkDCMMCHUMeS2EQAhYC11FRwhhTF1D4yKUjDEGWHmXE+EuIaDvod/6NpTB6O/jBoyFEBGMMWUsCGOpjJRSKkMZQphqY4qyNqYpq0op4/IwscueQIhz3gqpjSaIaGu0sQAAiKBrd3YtF7wLlR4MBu6A3ZXmulLXcrnzD3duUXcthfvb3UaK311jVitooJIyjpOmrLJN/uThI4/6WhqtFG8ERq1SmhACDQTaYA/vuhCEsCuxjhRyv8DfVfHbVGjP86wliGDnEOiytrWy1ipjDQTmXqN8izTchw3APT+r3T7F/S3cigfubCruAAP3avehIHd+XMdgLbx/im5/BFh4j9XhHgQCiiBCCGhye+QYQuQsNKCCECMAgQFWQwswxgjasszbtl4vlgxZCCEjmFeK7B0euZlvu117QbR/eGyM4aJZr9dN0zRNdXF1rYzFGA9GE0LIxfVXtlJx1496AW/aqJMoq8ssny4WEGDGfIOVMXq9yRAiw/FYKrTaVlICyuKsKNS6lFrN5ot4WVzeLOuWbzZZmLxbbevZYls1Ku50oRAAGlXXpZJSyjD0vTSy1vpp3Bn2s6YyEEHK5usN10ZalFW11ko2pU9BJ2CybUMfl3lmjY5i3/cpMHK1Xl5eX0dR9HR/UstGKIUIlMAM9oaXZ+d0hSaHo4HuCitZ5PPlajOrsjqnJKjazGNE6NpHhHnWY1AZjaA2Um43xZs3s8GgJ7WcL86EVsvl8ujBabXZZts88HzWCQHCUdKBxqbdXhh52zwrivLq6ma9yTCig70BsF7TyKxofvvFK0qw1iBOyHyxTjrrw6NT4lM/TLOivp4vlFJRFB0/OFmv1wSD5Xr169/++uBwItvm1esXVV7M58vIT0I/icIo9rvbbeZ5AeWqKOu8qAghzPcBJJdX8yxvOp2EkLDTDXq9DheNhqCuawjBcr0u6xpTihCaTCbK2MVqaTR4/PR5mnYJIdRjV5c3X375ZVVVUkqlDMbUFSFCiDO8whRRhpUS1moX11uXFWNsOBz3Bv58uUCEXE9nb88veCN5w/O6MRrv7R2Usi22Ncb4pz/8aRiGnHNoUL7JicfCMDSNmp3frIpZZ+gn/UACDolqiyor271R8Jvf/fpnP/15t5sOBv11phtVAKtrnqms6oAkSSOLLERg3az/6ld//dV33/7iZ398eHjM/KCoyuubG63MxfXNm9fvMEC81YHvU+LFSWcymRhjiiLr9XqS89HePmYeC+KHT56zIH779u12k7nhNUw6QRy7Aa5voBMyZJsFQqhtRFEUL7977fv+aDR59vz53v7hxcUVoujZs2c3N7OiKDCmztjGGK2Ui9nVnCunRoNwCwBar7dKmX6/3zT817/+zZs3b7iS7o0U5xhjzoUxhjeNu/2VRV2WpbW384q1Njw6lFwYITWAXGlrrbLAWrvNclfFlVJunK1bHgSBklIDK6RoBJdGu5nJ3T25EFhr3/et0s4NiTGGAN5ut6vVyvlXQgjruo6iKAiC7XbrRsbNZuPynKIo2h9POOckik+OjgkhV5eX2hgMYBLFXArFFUKIQCiEsBAE1jw5OTk9Pf3iiy+gdTO0H0WB1RITCIDxPAqhdRFWSilXjOtaGKMIQY7/6EDxpqmMUWHoV1UFgJGSU4qzbFPkt8w7x2Nw90ZjLeecCenkf03TuGBAAMBqsXBz4VQIz/MIwm6l7TPP3XmjIGCEbJpGKZWmqUtwcH2kE/W5dcZkMtmhF+CeN2IlK2WEtp6FFhGEAdFaA2WFksYYCLEjFQIIzZ1Tshu1HQvB9TrCZTFbs6uRuzHUQqKdRhFCCyHAGGAMCTEQYsakMVwpJCWlVFkrODeq1lpbC6SUFkGAETTQXRJS6lt7D2O0ujVpxgjvhuldDQZ3/s0Y4zAMwzCsqqooCheQtusndtXOfaZcZ+NcDdzXEEJCAgix4Obk8MTHtG3qXnfQlE1btRBgaJHVliJqLdbaEuy7LdIdBnbrLWHbW3sDY/Su+gIACETGKvfu1moDrLMdgxD6vm804FxprYHVACCXK+38K9253a0Ddn0nuKcLdd/FCNk7pqfTO7hztTOx0HfOlfrOyNm1NPieHdadvOVenIS9BRt2rQ8hmmoIIb77xRGAEFkLADR3tFCCoQXSak4J3m6XRjeL6TSbDOqyIG/evKGUums3TdMHD46cVwlEVghxeXk5n0+rqup0OmEYQQgBBcoIBRXyYC9Kjw4OtdBvXr/7+sW3GEMFIBe6rMVynYdRfBSk26wSQhCPaQuzrDS2CKKYYH+x2vQhoYRlRfl3f/fLMIqKosDYmy7WEFk/DCFjLAyhJCxiGEOCcKvleru5PL8IPb8zGGbbLfODYRpI3mJky6zKc7vNmFVV5Hd7vcj3mRd4Btj54uLt+fttlj/qPvlv//2vF9NFq0Rn0N+fHD5/8vzf/3//P8bqr1589c/+2Z/u7e9vy+pmOW9bAxAM0hggwgi1pjSykaoUyhjNfY8GDK43xXpzc/JgZACqxLauy9Vm+u7dO95KBDEht1f2aDTpd9Lrm0uMO5zz5WbBmH9ycgKAWWfb4WA8X67TtNM2sq7FNss7nd7Jg4cnJ6dpt//mzbu64VJKz6MGAqFMmFhrzOXshvNqf2+0XM3aOieUYIa1lp1u2lZqOp0e7lHG2GSyD6g/n8/TNNXabDc5hJIxf+/gaLVaNUIyRgAmSadDPJJlm022SdI4juNut9vr9ZJO+tvffnF9fT2a7B89OEEIhXE0Ge8z6l9fX9d1U9fNg+PTwWCw2a5kywEwGDPGGCVovV4W20xKORqNAEBXF5dxnB4eHoZ+MOj1H5w+evy48cPP86LCns/8YDpfGmBVIwa9HqXUalNkJUIobzJ3G/OTSGODKaIeMdAGUTBbzoRsNquNEGA8Hn733YvDw8Mf/ejPMIXv3r9U55UtG0wRZrAzSCgl5+cXBLDBqK8F+O71O0wZYnSyv//27fvFYjEa7XV7vdGo9P3ISmuMCbwQAJAkHaWEtbbX6dR1bSTKy2Y6X3S7XUQopr6BBQCgP9yzEM+XayllFCWEENdI9eOQKy6lHO9N/DBCCL1///7i8vrZs2fr9frg+GQ0mrx+/XY2nydJAhHKs9pFSlprjVFN07j7/vXV1PO865tLz/P+9E//VGtzfX0DAMTIYEaM0dvNylpLCHPiwNs+o63bplFK+b4f+BHGeLPZFEXhoHI39Lua5Dg9RVG4AunqDYRQqMo5L/lBwCgljPnGdHrd2yHpbmxyd21r7d7eXlEUWZa5f3G6dlcYqqpyDgGDwcDzvLIsX7165RN6fn7e6/cPJntX05u/+m9/mXTSXq/nhcFsMW/qygHpO5Ld3t7e0dHRP/zDP8Rh5IrKarV6//atUipKUt/3nRXgeDxu29aBOu7dwzB0qLUTebmQ38FgMJ/PIYSOdrDdbm9uFr1ez92dgyAwxkjZEkqFEAGAbr+DMUbaMMbCMLSqGg6HGOPlctlUZRiGYehTgtzCxWfxeDxECC0WM9eseFFojKrruqqqpqkhDMMwZYxst+sgCMIw9Dx2By1QpWjZbJXRrWjqtuJCMsYQwYhgd0hKaQuANgbjW2KaO2Y3L7r4JWOMNoZSqqzZeRiA28ggsEP73f/uZlwHZbmJf7cU0FoTjD3PU8rlZmsHEyqunUWxOwZjDFcSAGAgQBbsVh47IoIxZj6fuyVdt9t1rFWnMm3b5n59vb90cDQCuDM4ckzDukUVNMo+ODq22sym18iii/OrthEUUcY8QhglnrIAWMCYhzDW3yc5uebAaKFcrpXT0cA7l2WICIK3hlTGWGfmaK2FEDHGeCvdM4HVEGIAkFtRAXDn2nlP33ifHeIerlcgBO/AjPuEht0awvWLDhxydR5CCAByiId7Ze1SIO76hN1Sw1rr9gsEY3ULeNjdX9NYd3oNsvfCKq2lDEWRV9XFejn97tW3+/t9ihA5eni82WyyOocMYR81qpWV4pz7PgMEPv3w2enTU+eN6vqacrEdJD1iwP5wb29/bK0mhABqB7/uKwmU0lJwi9Rg1E2SjrGy4fVsugqibpZJrdNOb7zeZFxRC7yLi6xtWwwYBKSYW9VGXMOaQmRslVWiLbRqGbGWG0sgwERrNZ8tzy9nSoGHj5904z0pteBn/U4ItfDDLrVy2y5GvU44jB4/eej7PoBQatNw7Q/3t1mBMaW4Za3yUPb7V1/MVmeDMX3waLCaLzZr+dtff350/LjY2HwD8kJ2B/tFLhvhhQExrZ/EMOkwCzYwqI8fHXsMqXdiH4TAEz//2T+rK/3Fb1+EUeCn8fgofvfunfFNoTNVVD8ffwqoiIbk4uY7GkLWWIYxBKDXGyCtDvefrK7X0+lyr3/y/J98opT68usvJ8Pen/zikzLfAH7FGxiHSS8cbTZ5HMSklNOr6wdHBx7d400VQr+SEgqgKrwXjUzR5qvV4ckxxO3ewWQ06VVFtFllv/n1Zy2vhWg5b549f/LhR/uPHnavr76a3WwD9kGvN2iLBkg87uz3Bqf7TUAI2t8/ev7scSfs/6//6/8bK3X97p3FZDAeNW1bcSkxaSAyQchgHZC41G2zXaIo6XlhiANrbFtW59PLbr8zkyJJkrTjjyadslm+Wy6AgZXRSdzdH+/3Uz3ojweDkYHg25cvi3HZGXa90OeyzctsnRd7g0dWGw9j0UpoZaBxrUAISZp0Gt28en2Jib+R4OXFshONL66r7oNv/+mff6z+Jv/85ReLXD4YnRRV8dXr6aPTIz/qNUXJAPQYezAZ6217/uXbB8cTv0dpiKSqHz08poSEUbotK2VxJYSxNscw8JPYC6tWQRgY27x7f9m0Kk5626x6++5svd4SQozFk/F+EESEGohI04qyapVSUZwuNhtjTKWAA7pZlHLOr2bzLMv8OLmZThfLZVEUvu/XdY0hV1ICY5SGjVBFrZoWWMustcu8ADh+8PBxGHTfv3n/+ruXvu+vW+ncfIVUCKEwJJrrosi01tT3IGYoQFhrg3FtrJbcAimUMG3tK+H7PoKgbRvetFmWAWsxxj5l2yy31kIL2ob7HtVcCVlDCFEYJn7YH432x+O3b9/6FHmeN59dDwaDThpMp9Nf/PzHhw8fX91c52XhBT6EEAIEEfY8L4xSiDZppx8EkbGV54fbrG6aRiMiBHj91cu9TTEejyuTN1k13h8Fvp9qZSmp66oTR0EczedTrniN0e/evqGdzs16PfIGsshfvHkT+EwYHXc7f/Gf/+MnH31MCLm5vi7L/MHRcbGViYc557zYWD90woEkSQe9IULIKh1FUZ7nnPO6rjGmFS9JTSmliMCqqaOwEycRhDDPMgS9NEogggqJMGHO6zdgDBojpfQIOXh4ut1mcRzXda2VtdZqa96+v4QQYhoQQspaWGu7cTLuDzabzWw2swbyqm04Xy6XcRxXXq21RQgFUehKTlO0KtIMASOsVQYzhBEWFjRlgTFmBFurtBRK1G4klRK5Hui28ABEMMHIQgiNCzc1hgICLTRaSWEplW5HbbSBADCEIABWKS4luKtVTSuAC9xCpBUyihKIVbFea63TtBNHHudcSYOBaZpGtyaKEkI9rXXohdpwN3/vHALsnceiY6pyzg8ODlwP597OFS3XE7ja6UAX1xfunKDcj0CCqroeTYbTxbQqCmMtN0pZs9yso26qjSmqUgMttYYAS6MYwBAZYC3CFiIIjIUIIIhvQTIufT90vWbTcEKYQDpkVFmbJOliMet1uq6PJ4QRiAkhRENtgXL6UoDBXXx2EAS3XcidHdZ9/sFuO+A2Dt9TUI1x0mXXojmrAntPS0kIuiU23OtCdh2V6/ixxxhFBCIIIaS+hgQACImHLNJaA4ghZVIIY41BRGnrxPwWkNWm6HYioYzUBjP68x/+nPPm5uZ6MugTp5+eTCbWWggt59xRjq293T4KycuyvLm5cRBiGMVB6ENM8rKIq6TbTUkU+bX44ac/nt4s1uutNRgNqFYWAGQhYChMYtPtjw4O4ryoVtuMqyXnzTpfKSWgsRh70FBjMCKMeV7RlAZoaDSDUkNphWilIghyAKIo2SzmBFlCqIfx4ydPpdTXN+KjD58/e3SaBES2xXY1laJOk3A86G23W8LY0cFhI1XdtIv5attshaLQwr3hXlPkVurPf/WZT9n+3t7xwWHLNQRCijKOWBD4zMNlucYIBkECgdW61sJPuqzfiU+P9gfD3rDbybbN5dX8t7/6ewADguEvfvqjxjSDYU+K5vLycjzsP3v2dDwYnr1/RzGCEEqhx4NhGicWaIqJzzzG4kePTluuLs5vMKYHh8f7B5PxeHx2dma1gBj5gQcRghgQQoRSlAVB0ukN9iLfm82nZdMqBcIw/D/9z//Tt99++/b9exp2Hj1+Qj2/rKvrm+V8Mfvqm6+32WY8HkYBsxXg2kyXiziO426HBqGwerZaQog7g2GSJEiztNd7/+7td999p7X+8IPnf/SLP3nx7dcPnzwtiqKoqu9evqwbnq832BqK4Hdvz6tGta1QFglpt3mNLEQIKQ3STr/bHwBkAcZFUdK8whTlRZPGSafbT6NuU8t6u4XrtdDG9/1+v//k2dPBeFDyer1djSZjjKFocVPXQGqtRL7dLhYLDniPpU3D5/M1wT5CjBJwdbUxA+rTRfoWdbt9SkjTNMiCxWwpFJdS593CY5hg1lRN3Ta6sQ0UaEkxlpP+6PDoSBt7fn6+WWZhEEPrHPcQIVhKWW6zYrURZQWMpSmJomAyGR0fH+d5HgSB51WHh8dSysVy1jYCIdTr9QhxA59ar9dVVTmSoxuqhBBFUbhbiduyu/l7sVg0TQMArFshlDYWKmMbLhoujTGMsawsRoPesw8+WC7m59c3CsDlemX9pKqbppVOw+2s27S2CBEjjVKqahpjjLsNWWtFpW6dNBGWAGqtRcullGmSuBkIQbRbhWqty6p1Axwjt/HTAFNKMCa0bVsLFUDEACSV9oJwOJ7EcUgIKYqiqqrhcHh8fMBb0TQN542LZt1u14Swtm3LskySpNfrbzYba60WUutb+9u2bbMsU0pQStO04/t+XZRlVg6Hw/dv3nLOp9c3UkoKEcXY97zIDylAURAUiFZl2el0hr0+sqAq6s1mE4YhY6yu2+V6W1WV5/lhGC0Wi07ac17C2+1WCOWm2/28gAArpRhjGN+mMDgtpcsRCMMwCHxjlLtxWygb3jojhKblLnWlKAop9d2YfhtS5ebpzUoEQeAIiZ1Oz0LoTCeti3lUyrH/mqYihCmluJRSGQsQoR5EhHkBAABhQdmtgNAYo7RQSittpTJBEFmgpeKEEGMhNLfLbwAAocgDFNxFMXk+pQwLxeEdb+C27KBbJwBwJxHEhLE7tYU1t/JLpwp2VhCU0unNfKct3OHhUkpjlbrLl9rlHUAIHRXGWYmv12t452zRtvgPSukO59B3/s34nnLSWkt9r2kao6SWsj8cHB8fz+fzrMg9zyOUWQullFxIRJi5pxRwOMcucsLJOHfvaG+rsfE83zEn6rrspp0d6dKdVYSQtcoYC4EBAFprMbg7ewDs6vf9Ef8PSju820fgu7gNN+474qC5M5aA3wsozD0A6PuHe9U/ICiAO2Ps+7QGe5stuTuQ75+mtRaSY4yjKEpi/+joiAANISyKgqC7jHCMsdbS7W/cSsZ1cLEXj8djl7v46NGjz3/3l/tpHHQ6XtIxlOZcttn066+++bvPfosQ6fUGg/5ESrVcrLW2QewDDY1sN8vs6EESx2S13QZRreDqg6PUWq2UEo3NM94WFiMQel7YC3lbG2MJ8pWEVSGbVmAAPS+IAnpxeUMA5aKdX553wni1Wl3dvI1p2Atitj9myLcKL6ab2eX05Vf84uri4Pjoj37+81ap2Wy23W4b3qajRw+PTged9IvPP2MI5tt1ur8/7KaLxXKzWOdVvc0qn6qjk4ed/uhg3n17NvdoY0VtbVNmUpSKgSECx6NOup7ODsYDIM0244Hf+eJ338xvLv/8//g/zWZX6+Us2yw8DK4vz9+9mYxHwyovHj98ItqaIOyMz0Rb9zvdOBl/S15uN+uW19vttqhKxsjJg/3xeLxe3Wit8zwXjaLIsxCWVf76/QYjVLSqbMR0sWUUd5NOGEXvL64hCRALPQolYPv7D/oGKG1L+Rb6pOsNT589aduaX8l1mf/uxYvFYoExxohqrX0/fPLkWd/3i7b1ka8hoUHkJXKZl9+8erPYZIQGQRhXdb3dbjebDUAQQax5Nb++JogmjcKQWOQtN2Wei8FgcHRwmJ2/Q14oDSirKk5jQLzeZP/Ro1P4Iizy/PJmSdC6zKum5mVZ3cwXxlrf96VWrWy3eVbUZZTESZL4ASkLLtom9IM0TQdyoHAaBtHV5fTN6/fpaMBYyCiazkwvNMtFNs48j9DD/YNO3DG6LLMaIAARWK/yw4Nxp+OXADWGKymE5OtyCQ3v9/tRmvCiKcq8rsu2bberdd62hPnU8602VZaXeWa5JAjPLqfGmDzPO51Or9c5PT113LT5fOk+L4z6bdsiJBBCg0HfMfjcnRHcKcJd6PPe3p6r31mWIYQ2m03btl6Stq3gSgKIAYAGGkwRgVgpCZA9fXz68PGjr198fT2fDXv9bLWYdA/cShhCDCySQltrrYEWQG2MlBpogABCFkEDgQUOXMUYW20kF04iD8GtXbEL79vRrCCEkHoAY2tMzTVXZSNMmBWEEIOoUJUy1kDMleFNYwHWAEspKcWMkc2mdQsUTG7pWlEUVFVT13W/70spyjIfjQaB7+cEIgwwgRRjn7Ga15vVuixL5nsIAc55hXFVVRiT09NT6oVX5xdtXYuWbyAaD0cPjo6H/QHDxBp1c3E1vZrWWWWt5nWjPa/b7UdRVBRFni+2eQ4A9DwfQii4ckNk03AhlNsuB0EwGo3Wqy3n3K2chRDWArdEz/PcAr23twchuCWACxHHrG3bsqiKvNTKFEUBIc6ywhgDwK0ehGDGGAAWWQMhANpC1zv2h0NCiBvAKKXWWheA6U6+e31Ebr2AHD7f6902Ny9evHDVxSkMd+sDJ8DZcRHMXe6ze8FdxXXvcn9f8H3pBdC1R8B8LwpwjaNSCkHgWhkXK+qAgR0Rz/7jIqa1BvBWMXhfTeB+8Fawd6cJvG++5I5z993dqznWoavr7qcgRoEfVE3T1qXV5vj48Pj4+JtvvnG+TL6UABFtjdSaYesconaX965dcIf3BwfvnuZWHs6L5eToeL1aQAAwRIxgjhEGEBgNjDUWQgsBAM5T5P5jtyC4V9Hhri0jmIA74+rdokHfpWP/wU8BALRUAACXNn2/J4AQIgR3hEprLbgXLuXoLDuOiLkTnjhW5q4juT3/1lirrbVRFEUeieOYAE3c5WiMSdM0jkN3oNbaPN+6ZVUYBWmaOn5Tt9t9/tHHrkMv6mZblHXdaq0vb+ZlowgBuGwbfr1eb7IsOzg4enL4hHFfCH129o4yiH1rYTkYE5TJpNdSChmlRhNRhdDGo/7xaHjgIVaWuRSCYSJEu1mvy6IGxq63RbYtFkAT5vkE99Kw3M7WsynF9NW3r15/+7IbB/t7IwJ0vpm3vITQTqcLgLyLy9k6z65uFpiyThB7SfePfvpz2ZS//81vCQSPnzw73B8ncdhPH3Q78cXldVluPWbHo/SjT54i8vHbd5dXV5dXl0I0RrRZXpfHe92jyWQ+u/ntr35JiacM7nZGP/7h87rcLpfr64vX8/kyDMOPP3hclc1sevNf/8t/+R//hz8/2N/78NnzxXz6+rtXX37+hZBtmRej0eD//H/5vyVhgIA9PjzaPzi6ns3LKnv//n3TZucXby7P3vu+77Po0aPHw86wzOoalr7PWq7zfJvX7aPT4wcPTo1Vm3X26NEjS3yuNMJet3tAvUBKOc/Lk0cPfd/f2x9fXp5bTMqmmr1eAgCePn0+GIzyrEyS9Ec//fnR0clsNpteZH6aPu6knk+tVu/O3s/zLOmkL968qptyuV60vB4M+oPBABOx3V4b5NMo6Hf71A/P3p2viyo0PRQFilDGaG1ALU3qBZhQQD0v7rKwUy2z+fSa161ROomiOEqN1S4p23nhedQnKUGU8IaXdbHeLOqsenBy8uDBg9PHJ7WuLhZnV5fzzdoEHdjrRVoB3wMQkDyr57Mbq1Xkh6PB+OZ6LVpgIej2PMWBbAwJESSU+FpqRT2SJnG2Ks+vr0aD0X5nsn+4Z6Ru67LItwCTti6dqK8tK1E3QGoMIAqBBbZuyt/89tdJ3Pnmm2+KonCKfGOU5wV+yuIk5JwXedk0jbuzSymLohBCOL96h5p6nucirR324Hbqa2F2O1RjjDbWOdVgDPf2xj/+yU+mi9nV9KYWsrWWW1TWzg1G3XKkgYEQMuwwXm2MgQBgjN062RhDQ6KBxQBCAIzWUghn7Mo51+J7vfvulp2kfbex5opDrQ2EXClkre/7FjELLcTUAKQA1oq/P7+YzyFv6/3JnkvryTZrjGnge4SQ2WzO2yYK/TDwWgADz2eEct5ACzpJPOz30jjivBGi3W631PfGwxEAYDqfGWUZ8UiIkyhhXjDFmBFPYyO5skr7zA+YTzBaLXLFVVtWouU+ZaPR6OnTpw3fSimXi9VmvVXW9Pv9KEqUMnEcE8K0NgihKEwIIRjRumoJIU3T5HnOmIcQaRsVxziKotVys91ujVVBEFBKyjLnnBtjmNet64ZLYYAFCDM/iJK4aYWbuBxnjhBCKCWMYkzCII7jWC6XnHN+O4GbbrcbhqGWMsfQammtdUIMDK2fdCnzuVCc8ywvq7rFGHPOMWEuIBlC6H+/dEdaCXeXd5p5rZXWEACDMbZWS8nRbX6x6xEhpgSj7wd0c+9h7wZZB4M7m4EkDhzE4uKbnSkfxphg5vQgu+H4VoqJsLuQdlwTVw4dtCaESNN0OBxCCG9ubtbrdbfb2a3q7xdU91OOQLPrgbTWCCPqsapolVK8abmSQRiWVRWEoTFGaIUtdHgeYbdxXOaeiaHTjOwm/vvV/RYAsNookdUCQQus9n1fttxzsdQWAKMt0NbCOyOjPwx0sHfS0D/AD3a/l+t43EmWUtZ17Viut3jVPS0lvEc+AADZf3zM8B7L4e5pOxWoe0cM79Yfu5eF90iU7mul3NjQCoF8n6VJGAQeNIK8ePGCUur4OFpL5192S2C+TegibsNkjMmy7P37SwghY2w0GmXb4vLyEkJU13Un7QMAGPMcmRljPBj0Op3kqHOapBHGAFHdyC1GOgpwzU3bLhGmmIUhDmDox2F0ejw8OT4JTVxVlRYSY2y1raqqbQQA6PLi5usX3xIMu71hnPSeffDR67dvZ4ejxx/88PXrV1/89rPzq2tg7fHh3tHx0yCkbdv48UUYdaeL/GaxaIVNoxggZDSglJ69mWZZBhT/+IOng8Gg20l+9uOftFJcXt383S9/9fnvvrq+OYcYdPqDh6enx0cT89NPmnp7+f7tu3ffag1fvny9Px6PhpNut7deb19++41S4vLy6vT0EY3Jo9NPrbbz+fLm8uLx6ePlfHV9frE/GC2ni1fffvebX3/213/9l1WRQWg//Oj5v/nXzcPT0+dPn0ZJ9/kHn8zXq9dvX718883F5Wsha4DBcDj0/bDT6w1H427fsBTf3FytthurOQB2na2jlecs8Jbr1XQxO33wNE56VcM3N6ssy3//5puszLsEXE5v3l2c14IjyoQp4zjujyeD4bgRV9ICCUAj1XKbl7w6ODjAGHLOCWEPP3qeTroXl++3oopSbxLtldWWEahx5XfA5KSzrhHwVdD3kuEhDOD0aqaRvlxcHT488TzPC3xp9GgyvLi8nK2z7eeff/f6ndaahl2IW143YdI7PjnqxMnN1XXTNC401vd9gMO6rlfLtWZSa6uU5q2CgEwm49a0L9+/Wa8KRhEELIo6UprJZMAwK9smW9dt20LgP3/09Le/fuFjmhfSJrQp5LWZez4KAkowNFgTRuNxQixe5et3F+/3h3sP9h/MLmf5NmME11w2QraCa2195sV+YIWSXNSmcD56Usqj44Ms3wg+opT+0R/9tCiquq4nk9HTp8+FEF999dXZ2Znz7THGuLsAAMB9mtwnFiE0n88BAHmeu9a+MdyVFmBvc3oYJYHvQQifPn368PTBv/t3/+7q6opQtlyttbEVb9q2cYF+lFLfY3Ecp1EohFBaGKVdvQEAQIggJNpaq60TYyqtRNNaazGAoedzAMWtSlvvlqluxm3b1mgZBIHv+5RSaExd1y6a1o13CKGqUK9fv/axbppmPB5HRwfb7RYA5DEyHo+l0C+/ecEYG0/2MMbQ2EGvA61WsmEEDHo9ZxbpwlqVUglN9kZjPwp3dSXLy6vLmUZgneUOitdGrbPcvHn37t27hycP4jD60U9/spzN66qCEIZhjDHmrXAdmBCCBaHz8K+rNumkAADG/Ci0wDpbQIsQcAsad9/cDb5pmh4dHRljKMOMMd/3tJauYjUNr6pGa4sxBQBgRD0WQAgxpne1BmhtINQASIyNtcZYWFaNO8/KqqKoILQYYwyt53ndtIMwxBg3TdMq5aT/rqwCAJqmcf20uwk7JBjc5fo4edt9eOB+Tb2FKBDyfT9JEmeTQCgFiPzjImQdYOBACYyxBehOlWfdkeyWWcZY51Bp6S0M4JADfee4jCAkhFBKwzB0AIajSRpjHC9k56EJAHDqU3jPoQjdGQnsXJt2xc8tyDCm2hpEMAt8KSXnnCvpbL+3Rc45x8RaBKnHnEMUvNs+uKMFd17RuwP+g+oLIcQItE01Ggw5b4a9bpZlWmvOpTHKAoMsgNYgiCECCGGp9G598H23cXfM/7uP+12avnu47grdi4yy99Yxrguw9w53t7aAd/RGDNzX9v6V4J7uTp3LfHGXsb1TYVRV1Ulj3/fDMOj1OknoGSvbqiLv3r1zBu9VVdV1WZZlr9c7PT0NAm/XygEA+v2++7zdXM+CIDg8PDw5fqj2TRh0IIRS6uvr66Ojg5OTE4jsxcXFbHbDGDVGC5HHEfvko2etai5v3rRiK7lg2EfYYmiA42ebBlRiuUGexxN1YIwBGkAIoUUYoJBRa+GzJw8BAJ98Qo4fPBTadPqD7958zdv16GDQ8GI627NWYo95UXTw4PTk8Oj1uzeFsHlVbi83rTZ+3IU0baRgDF1dX799/wYxutzMX755FUbe/sHkaj49OT794S8eDCb7Xtz967/527/5+18aY8aj42dPnjx5+iiOBoMhhxBZyWc3+ds3l6HHDvZPjo8enl9eX1y+V1b/6Kcf7R8f+X74ze+/ffd2o3gLLTBSLWerv/yvf1Xlxdn7t8vZbD1fEwIePTz56NlHl+fnTVn7jH747Nnjp4/zzzNGMYYWUTYad8LQ73a7WVYtVgsvCI8OT+M+efntl+cX705PDgjTb159s5y9p5RS6i2X6yxrev9zn/nBX//t386mK0zYi/NvMIZxHAshNptVHMedTidJksePn2JMpzez2Wwx6E+kMPP56uXLV9tNDhmK41AoLpVQVpVtfra83mbzp49PHj48HJFenq2aKsPUHHVGA+m3ddOgZSfqTU7TdBA0tSjzxusQqRSjEEGiEGiMWed5NasNYgBYRFDAgn5/dHpy8uT0QRh44/G4LitjDCEMQugFYS/tRX707cXLNOlhTXmr3727qFoedILpzWI2q9Ihc/Iw3/f6/S7SlBKi1c1iNg39/t54H1uMod9UcnZRagC8CAwnoDc86vfT9XadZdlsM//h4x+W20woVbcVQmC5XCxuloPhHq8bYwECkDIS+L6PqeKiQWBv8mg2m2mt2rZJkvgXv/iFY7zzVvZ6sq7rOE6DwAtD/+joAEKrJAAAOIK3+5C7QcpBmi7Q1t00726CBGNMCYbGGqgQ1B4AHjS9Xu+Dhw/L7eb3X/xOa5P2vCwrgiAQWhFGEEHWWo+RbrczGvQ7nU5d5lJwKaWRQkppjXJmf6t1YZRCCEFrkbGSOzG6UZQ6kZs7SAMBxghgxNtauTZNKaC1VcpZDVpr0zg0jpputU+gomS7Xo0HcdvWbVu7fshaC6FVSmGCPJ92O70kicuyRAj6vqe1VrcLadvWZVmW6/UaYqqlAsa2bcuYL1tZ1vVoNIrDqCpq0onSTs9Yu16tpFTa1EVVaqmkMj/9yY9+/JOfvv7u1fXVZdu227KYff4F0I3v+xjTbrcPye12BiGb5yUAqNsNwjAuiirPyySBvV5PWhlFUdu2cZxYC62pXT2O49j3fYisI5c4q0dCyPnVuQEAEYYhVgZIo53uFGNsNNDGOLddpAxCzqnXoLzgnI8Gg25vYJSezWaL2ZQgFASexyhLIgKBtVY0tZZiUbZcabebcMsRrbUGcL1eE6kIF64mueaSUtqNPUqdPYCEEFJKGCMQWq2lE4haa32fpWkqpcyyTCht79wGHdJu7K3oQN9N3tZ+v+p2V697poOW0zS11kqhXY9l74IibxEXeMtGdAXMcQJ28IDWuixLp4CQUoZh6FoZcGc8sPvBnRTiPgCwey9KCMIAAIAJKYqCBT6iBBR5yzkB0ACL7kosvEuOsPc2Aver7K5RcP8SBp5WIvBYkkRW6V6343tsu82augZGAWMRsAQB53ZogbmPydl76RXonsXC/cq9wwLBHVPBdWAOVMD3vBrvYxL2Tv14V+C/39FYa401EFgNLABAWHXHpbCuDVFKSWPc3xFh6nSnUkoIsVJqMEiGg56WjTs8hBCwiFJM3OrLrVfrunZhuHEcIwScoEhIrrV2puuz2aybpJQyqFFbCkq9cX8SRZEQCmrw6OTxeDKaTq/X8+VmufIIUbx9O/sqK8r+YEB9jxKP4GC7yajH8jL3uDEa+B5GEEpgy2I6wwLhxPd9aFHd1FJqipk2lnMZRslg2JdKxZ3gy9///rMvfvUPv/yrm/lchJHkomgz5GGDzCrLwcV12crFfLOtZFYIQLEXJhbHjfGJFwslv3v3qmjq0WRIsL2Zz062x6tt+dU33z1+snny9DkN0tPHH7x4fX52vS7q6vK3L168eL83GfW6MYV22E8/+fjj549P82LdNiXEdrQ/efj40duzt3VeAWw6abjdbn0PPjg+PJocfvv1q2+/eTW9nMlWQgsYQUVWJFF08uDgj37y4+Fw+PLFN0mnG0XRw4cPoQVfffXVcrPwfb+ot61ARZVt86zMKyk0Y8Gzpx8gW2OokOEBhQwjz9OeZ61t8nxTFjmExOhKNvmrF18bi0bD/SQMjDFVXtR1jRFkhCAA4iDmdTNvpkVer9dboPDl2aXvhZtFZohseBkmzIuD19++/eL3v1FGZOXm5Xdfv1u+/gn/8MnTE7+LvIh51EuSqMz5bNoC3SiCrIe6STxG/Wxb11W5WGUoZ3UjMKLr7cYLIml0mo7LvCjrliLb6XQm+3uD8chKgft9n3lSSim1UsZIwzkvtlXbqMhHwFIl4fVyscnK8fFoucirEiQDGMcxxrjXS+qm8LGfJOkya37z688m4wceGwTUD8IuHNNtuW4VQBAYA4y1fhx0SLfg+bJcWQI6g34xW19cXBwODkTLp9OpFBphRsOQUCKl3FY1hghZYI06P988e/bsk08+GY0mVVWt1+tvvnnx9ddfTyaT4WAMAJjP55eX574furGgbVpnOeXilxwVwN1NjDGbzSaKIsfodm72slXcCA4tAtYoabQshTC8PhyNAkb/+r/+NysVIdRIY5T1WTDbrPr9fqfbcbFyCJmqzps2N0pBayiGCFuoJJe10RgCSSFWCFNK4zBSSjVNU7oYiLpWSlkICCGIkl3hIcDEcQjiUIlWCGE0l9IYJaMoslppZTzPQ1ZbAztxoDXrdbsQAEpI2zRu1cJbPru5HgwGH3/4UZqmm81GcO4mEN/3uZTWGGCtFNwtepKEuvKwXm4AQEKIYlt0k27gBVKWkOBOt1O2TdFygmHoe9YYyvz3VxcWwU63vy0ygLEGwCKIPa9crR24miZdgJG11o2hiICmaSjxMKNa67blYRiFYdTI2q2EoijCmFqDXUvnXkQb6QZxFy2ttUaIAIDcCkAp5SwrKKXAQoCMMcbZIjnyI0KIMNpwQRAhjEmpALDO0vFu6IdAK4UQRgAj4AcsJpQSpqRWUmtlpFDur9bvDe7Pgg5PopRCqNxiy7llh2HoQrAQQu6N8jzfsRNuWavwe0vjHSzPORd3ADiA+JbCAoCzxzDGFEXRtq2DkN12zBhzR9Qvm6YRQoRhSKi3oya4I3evY+5StbTWLr7BfV6sNa7h0P/Y9Xm3BHHAg/uaECKsaYUgoS+FjKIojuPFaulkC8rs+KTABTbaO5RoJ6kAd+QJN7vfg1VuNZlxHJd51u/3KUbGakLIZDJx9FsAAHT4DQAAGAsg0NDcM0zcLXHAvfUBupN9ur5ESeU0Hbu33jUQu1K9+1kIob1byoC7RgRCCMD3KMjtH9EaZCGEUFu9U5O6X1xrvVux2Ht6CvcnSDvxYDAo823blMYqazXBkEURmUwmYRienJxMJpP1evn111+v1+urq6vDw33HftRGGWOSJFFKzWazXtpN07TfHw66PQCQqOX8Zv7mzburq6syKx+cHgOgB71eJwn9gCkhinL9zYtvxqO94WhfAwsBy7c8ShBlMcESuxNhJG/KpWzLanP08KehHxljiqKqqgbalguV5zmhnh+Fr969n62X//4//cXVzXXNuReib15+5fs+BjhMoiCMa8GXr199++o1RlRZK4zFFnGryvUNpGQ4HkG8li2PA3/Q6/cH3fdvvmNBsFyvtkXxuy+/+ua7d0GYbItmucw1wHHSf3jy4N37N21rPK/Dq+LFt++hIb1O7wc/+MH11dvZ7Prw5OSf/9mfpb/vfPPNV9pqaJXVKkmixXQ1HI3+cvo3dV1Lro00P/zw49PT09989g9NXaVRTCn98ssvFYD/+uMfHB6cDAaD86srF7Th6CCYms2mlFIChJVS62ydlQWul1FE44g2TaZNk6b+3ri3XM21rAMfGovrKj/af/Dk0clochwGkX/pbTabm5sbjxGnowPGMkbXy83x8cnh3vF0ulwuVmfvzh+ePn725KnfpfuHBwaabVlkxXaxWQFqs3o7Ph6tqvWvv/7NVi4ODvoEaUZhamLADfZEFKYEoWLTGohYGMXA2z86pP6NBWS52q63Odeqn8QGglffnnkUR4GPCNHKci6kVhSjqqqsMVEUBV5oDCiK8uLs8vPPP0e9ANpMVLLX6VuL60rUFW+aNgpBEATD4ZAQFCfRt1+dJT7DY4QxOD8/pySZjLqPHz0d7z1IO73r2fXL17/ntiqb/O3bK4BVZ5im3cQS8+bd26PRwXy5tLX62Sc/7g+6DKM8z7d5MZhMev0hQpD6rNftHh8cDnr9rN1GUXR9fZ1l2ddfvzg5OVmvV+v12sHvvJXz+dyR5AEAQigI6Gazcas6h9u7W5IrPE43YYyJ43i5XHqel6CQ80YrQTEihACDgdFG6aaqy+3mL//zf4qDkCvVVDWGmCIKkO0PeyfHhwCApirbusnzbbZd9ztpGPhxkkZByHlT5EQpQSkyigEAPM/rJKkyumma28mPMYSQAZYQAjDa3XEwtp0k6qaJKxJaa4IRQqiuay055zyJAkqJlqKTdKIoMlZ0u2kYhptNZq1ljBkNqqrinH/66Y8ZY7/85S+VEhhTznmn02k0hxAS4gVBEAY+ACBNU6VuRWJxGB3sHzmGZtvybL29np4/fPgwq8uGt4NeP0hj0XIC0clkeP7+7Ndf/JZiRBGez2ZxHD969Air2mgguDLGMM9Zmxgp2+F4lGflZrMJkxhC5Aqnq2du3CSE+H7AWy2lKsvy0cMnRVHUTQkhbNtWSu6qbKsllwJiZKyRyhhrCaWeH7RtayEwABpg3YwOLTbWes5hEMGqql69ecMIBlbHcex5HgKgqqqmLBCAYeRTTKIgrAkGAHApjDEAwTCOoiROOmme58roXTVCCGlrDLA+w5QRK4zSEhqICfJ85nqyMAwxQWVVNG1dN5UQYpttIKIW4n8EgNvbmdvVY2P+0eod3lk1cM6rqmqa1q08et2Bq8Ft20qpmzutDQDe7sTu0AhXutzqzdlRuI0G5zwI/J2K0j3T3iMr7DZi+O4h1C3pr67rOI69MFiv1/PlYjQaGWNcxLVRVmuDACCE1HXjKqIDDBx9wYkv4D1u4G6+9yjRWg+6vZY3BGEhxP54ooUEACAAMYQYQWOABQAYa4DZLa3uwx67ir7jEOwwDCXkDhrZkRnBndAD3nu45xvlNgjfV3pjzF3bcK9XsMZCZK2FCLqmHyGi72koXDeJMHFne9e+5Hm+N57EceQxnKapR5DglVaC5HnuWj9njXm4tx8HoRBCNq0/GFZZ7nkMYZwvVgghH2Ib7Ac09Fk/L2VWFsvNkhDy0R/9IN5LrTazzbzf6Z6ennqUVXmxXW9Y1Ds4frjeLLfn31ogyyaztqkbUK3z0biPSZRXnHOLiB+EHtPs83dfJUkSeaESWrbcKgQ0AAius/zt77/aFvUvv3zx4s153O1pP/TTTlvAohI+kkmCoFh9dHq4hkW23mBMNUDKC6xP1m2ltWV+fD3bYqCBgb3TcSfc+/mnHz+dPLo6f/Py/KYocz8Mk3730ZPnEDbzzbmwRa/bLxtTtwhif7W1EMWSqlezdfL2NerDm5uX08Vb/vn1k6en/+xffdQ7NBIvpiJQzIAEjx/3vv3qJU1wOopDmmiuBuMBJaCu8mE32R/18tVNsbnuDh+ul6s/+ad/+tW3Ly6vbjbrWS1kq1Qgw1ZwSgcEB8aorBBjbabLaRLri8urVbamgep1PSSb681XSjcoUMNxVyvz3bu/B6z96McPykIengxfXbwHCu4N95bLpdEo8Lqc68gfDjpJnQvT8geHT/7kZz/v9bzeIOz1klffnSO+ff3uzWe/+5J14mcfPP/tV19ClgZRpLNu25ibdyy7bkzLfUx7HTI6QowxDxMWkYqvt+q61OevX78+OT7tjIdJPOgeUNkMAv+hFODy4po869Rl5WFNAcCmUFXerjabqhVCYEzrtuB6K4zeltVltW4SGtugldAwepGvIAGA4Kt375XXZd6eByYXXzcffPDg5oUgbao0HaWn2iM30/nF5aUfdf783/zxu7OrpAf+6NEHH326H8Xsb//2PyHYfPnl7J/8cfPDp4ef/Xomm7OpmFqDa2EW+fLTP/njv/iv/wAlRDYB3N/vH3kBvZpdGCAKtTn/7s2rzy+22frf/tt/W2/Kjx5/mHaT54+e+pidXV5QzL56+eLJs+dGAwjR2dmF7/umzpRW6+lC1S3zPYzxeH9PK4UxLpvSQHP86MT+d3B9fY0IXpWbNIqEVpEftE0FKFEGtC3/6U9/enR09J/+6q80wlxy5vlGKAtNVucfP/k4jmNsiOd5NMBQIeUr2qWex8aTkVGaeBELUkB8znlb1VEIMbLWNlxixli/H2Isy6rRWlvK7PfoKNbAWimTgAZIhxQCABgk/f54NBohhG5ubvI8d6J2AAAC9HYu5HWapvl67RH04OnDuioyVR4/2T97+252Hjx//mG7XWQ3Vz/+8U+Xy6Uucmas7/t12+R54RFcc3H17n3S7Syurzrd7vn710oZRqEUNSGk14208GUpGGTD/lBKmRcVhqhWDcAA++zb714c7u8/PD7Z2xshADpJiHu9N2/e+Mxbztb7bMIABhBpo2EtTNtaCGwQxGkMCRJWX6/nStc4ACH0pG15VXMtEEEWkP3DzuvX5WiYMOYtFgurjZQySTpFVnb8BCFU1iVUejwatWUNtbEaKKmlEFIrayAhBBOEEGorwxsb9oL1qiQUNWWxtz8GiALsIUp5o2g02G4yL+1JY5jPDruKc24tc6EC3ZgxRrNt4QGtlXXrD4ARAMgjNPSDXpQ0TcMgQ5ZopZuSi3YZxbHRdrvdGmu7vZ7WepOti6qyGDCCa95KKQeDvsMhOr2uUgphbLk22lprrIXGAIJ9z/PCELuodEIoxkTdZRBoIwklfsCEpAgDY7HWAGGjjRFSciGcnBJA6NSilNK6riFCFgCldRCGzPO2WYbx7XZ/ZzHkzCs55253thNJug4pohQA4GHqER8D4nvx2fl0van8oOP5KVZKttIYQzAmxiAAENS+hz1KrIUAQAiwVVBo43kBZhREqK5KikngMUyg4FWd5/00pRBCSqy1QMnNet3tdrMstxYagJQGWkOLISQEQeJ7juUjbzcO0FpgrDUIoTtvJYsRoARBCJQSxAPb7eqoewSAtVKGESvKjZPUWuDkyIAgZAAwGmil3d/CGogwthZqbRBEhBIAEMYUIWSNgRa5RCqtrcLIIKIAsloraxGlDkcCAFBKHZuwrgoITBQyRkC93RrRpmGwLPM6z1maIAuiKCFBEJydna1Wq/V6bYwKPf/g4CCKIiV527bX19edTjoeDm/huzQNIsAYIwRoLRG0aRRTnwae/6d/8qcYQi2l4Lyt6iJvtLReEDdb1U/3O9FQQyVEXfNC6QZgm+WrbqfTNPX0agYh3D+YUBPUm+bddA7BHBpAMeuEaRwmQNu6EvPlti5tmWulcDceeV68zouQdeJxUGTb9fS6G4cnRwdPPvhY84c319fG2nVe3WzybZ5nrdHUN0ZZC25mZ2kcv3/f7A+COGFHx5Nvv/kMGWUBb1oZaEY9Qz3DqFYyu7papRGIezaJYdHOmrYKfcq82Fi+Wc+zLFtMV1aItmgfPnxsZVDnvOpIKYwWgteWkrjXHYoa6sauV/lstiAIIUSUsVyp4+OHDx8/Sbr7dV3/5tefnV9d5XVjlFZCQAiX80UQhRST9XLVNM1kb9RU7bdffQNJYbU+PX0UhmC7uarrTbcXjfZGAADKwm++fmWMFEIJoa6vr1ebQrR6s8nC0D99/DiKotV6W5RZp9vvpP3pdC6lIpjFaYqpvZku3r17k2fN2eXV+6sLYcHj0wcpRr3L67KpgYbAYqBgXckmb0VV+pjK1kS9oeelECQei0cj3DaFkI3h9PW3ZwjdxEHqsRhZr5MOO53BZDjYH3eN0gELDJdVXmotr6+vV4u11tZYIJTmWmmINAK81YyG2gBsgVK6rppWcUiwNsZoQKmX5+Vms8m2RZaXYRA8efJkMJrQyEhhlbRxELc1/+2vf9twYaz+l//yX/zxL/7s0YO97fa6m/yFENuzt5e9lNjEo9i/erfqJ72//Ku/+Lf/1//7/v745myxt7/f743iOC7rcrPOpBYs8NtGUoYfPnyotT4/P+/1Oo+fPhoM+6cPH7z47uVsdsM80jTN/t5h2/IHD46bhm9FG/iBhRZ5tGqbqqrCJB6NRkprZBHQQLVcCdlUtUOSa1h/D8YiGARBv9+llJ6fnzsWlQHwVgaNoMMGmqbJ8k3btkpwYwzCgCK8WMzLKg/9gJB9hz93Op26KFc3U3WXoezGPldslFLulnY3VzkFFnDcN611VVVu6CnL0gHduwnJYaduf1FkG2U0I/TWJcL3h3Tc6/Xm8TLLsvl8Pp1O/TBwhMfZbMGiSBkDCcUAtpwrbSxCddP6QUSY5wWRB5CpSgCAMCYvyqwVXuC7AcxaCy1ABFBIV6uVERxDuFmvkyBMw+BHP/z00aNHX332K865aGUQxdQL5vN5lVdprztdzBshAUYSIeZ7rVJKaykUZQgCSjAAxlpzm8AkhJzN5knSMcZ4XpAknWyz9bzAwfhFUbikUAsBQkhKucOHEcEEAmBv50gNLDQGIeBOlzaWMhyGAQCg20udY6Yrgdvtutvtx3EMaZt00iRJgIUum8Nt55jvGQOUUlIrAADBzFjbcp7n1hjDvNuRXWuttMaEVFWltMSEOLJFFEWEMWOM4treyeqk2ypJ3rYCIWSM0hpCqK0bcCFysL+TasB7W39He3cXkgubJYTsQkQdtOC27xhjNzS7/3UXkrt+HPljt8T5AxWl+8V38MmOLRgEsVt8VFXVNI0zHnWXN7wnBDDGVFWllPIDBgCwELpGwRpHSnB6K+Z5nlTcOTAieztku3d3G4eiKNpW7NgD7gxbe8cksN+flh2ocH/RsINtdo0OtCbwfAwRY55oeVEUCCEnrLXWamD/EYDw/WbEHfY/Ai3s/84/GRd3uYMx7B1hwn187iMr7jdFCDVNK9qGcy6F0lojiAmjpK7r5XJ5Z7VroA+done73QZB0O12CcFOyuW4slmWWauaumiF0Nb6YRD5AcNEC818348Cj0orUVnwqhRKqfW8DgIvjGLCIEqtBbJusqrKWBoq3vqA/OSDX3T7Pcem7HuoLoyUUgtpNRItzrioqjrPaqlMHA02mWibst+ZsDDarpuQJv/0X/757Ob6v/zFv2+kzhqVcfXg8NTrDM7Pz02rFEICQupTRIiQjQZw2PN9D12evfwuRT/54cNJP2155nuE67LY5ttmwU2xyjIpVx99fOz5/uuzBrSShgJRQ32SRkG3x8IYN20OjcTWtGUzvZj1k6Gq9OX59G9//aVWwCoIDLKC1KXMs6bcVN2kqw2EgAZhooxerfP9PdEbDn/9y1/dTOdeGOztH3hemCSd2XxV1lWYpE1e86rOsqyuQSeKiYWqlZtq1U0jYElZ1Nm21cpmoM03pdZ6ONqPgv77s5s46m7W2dnZWVmLH37yZ8+ffSgU/+7Nd2XVDCd7UWfAwuD85mq52NZldTO7/tVvDQRtHLH+IB30R1988cXVbD45Pq6LViEEFDICWoOJ8SEE2DJoFYGIEIpQ5/rGrNZFFIvnz9JOetjWM2S9B8c/GPZ7Z+/ez65nl5tZXbSdtP/8+YcnRw9WedbWNTCQAMIwgX7Ehao4f/Xda2WBVEAoAwnGngcJNcZAZbxAa2AbLuqmNdhaBJW2QRgDAHQLrm6mUmrF7PHRg9Fw8uyjB5PB+WpTfPTBR0Ja0fIyy7NC/Mf/7T9++skHP/npx9k21aL83Re/nN6cHR/vzzbXccCSDohTXK0qY5uHjw+SIIyDPka0KIrlapXnNSK0qY2SbpxSf/u3fzOfT09PT39h9c3NzTfffKO1Wi6XYRjn+fbw8JDztt/v39zcFKL1kS8RwEZLpfKimC8WaaeTxDFCyGMMGGiE1rUAGDPGGtsQgjjnjjPoed7BwYEQ4vXr10VVQgghgMYYjCmmhBAym8201tpId9OBEAJlhLGj0Qii7zMAtZFxHN9cX8fMc3curlRZlvCOWQ0hNOY+QHpLm8KUBVEshKiallJaNe1yvYEQ9no9t5C21rqVitSmbtq2EQVq0g4WUs2XK6sVIURpvbe3hzHmUh4eHi6Xq6op9w72IUaLkrdVRTwPMbYuKgUgC8Kq4Rpq2/K4z4IgEO6GK0TdiKLOtVXo1hVASykxYQjYqsgwRJHnGalE21jP63aSm+tLo0HgR449oA3AhKV9f7K/d3V1hQgFCLZcCK2VAa3gnHPPc85UUInbcqi1bkX95vVZt9PJtgX3JARYSu37YZ6X2pqyrgCCg2iIMa6quhUCACClNLepilADANwCQgOjFIBAKCEUD6k3GA3H+3tGiiAIsjxnHinyClOMCI7jsG3byEPOmMjo21t8EAS9Hthut+7O7GiwOKBOYWsERQgRepuIba3lglsApJRSCSBvxRoI3xIUjLEu/NBaC4BxZBprrVIGAJcDAAAwCAFCXBxz7jZQuwIM7gIhnQTDCYARujWxZr7n9jiEUYQx0EpqJaXU1lBKme8RQrQ1DW+lVoTdcuvsvdLofgvXeewofrvC5jgTnU7n6dOnR0dH3W73l7/85Ww2c7ty123s9gthGHLRWGuBRQB8zw10dddx8hAGbdv4PiMUOacAx8EcDod1Xbd1k2XFeDIJw7AVBt3TMlhrtVaE3jIizT37pt3W5g/aCADAjtccBCGE0KVdKHXr57hbKLgr6T634K5XsLvn3D8Sx4yw1gCAd7/m7r/ozivFtXS7ps1aqw3YbvOmqWTTFnVDKA4ZhRAT56CZpmmv1+t0EopwURR1XQOrDw8PR6PRfD5zvWHTNGdnZwCYMAyVNVXZYIzjMAwow5BWecFrHvhRGIbd/ph6kUMmm7y21q5Wq7rOKUNhxJq2XK1mZZX7Pk3iuDNOhnHv+vp6cTEFAFxPsyiK0rjDGC6qRgkhuFTaHBwe7R8eCymXm3U/6Hb7ndkVZEBFSfIwfDSe7CveXi9W5NtXcXeQdvrb9s3Neruta2Mh86kw2gJ4fHT06OTRZrNBJttur88vXvXTD5Oe3xTF4ycnb9+/KZsSwvbRo71f/NMf9Yej716/upqtr6/e5lkzGe11uwkwIi/WecY6aZ95ejQMJuMhMDb0tAm1Fpts2wJt20aJVvkkDr243xt6JLJCKWuI73W6faM582OLWdGIV6/frtfruNNJ0h6SZjIc+UFImCe0mk6n1tpBr18URewHyECKcJJ0wjCwhvDWMJJGnb6xfHZz/f798vmHwZ/9i3/5+9//P61BZZm1jRiNRj/4+Ecff/zhN99+9d///u/8JPjBj39SC/Xim1e1UpAhQNB0tdhu5r4HP/zg4ZPBiDJfK4gxk1y/+e6dAqApBUWBEhYaD0NEkU8w8pCmCFnLptdVU208n0z6T0bd/SbLsm2OIGsQDHDvwX5qx6DIcmhgCH1RNKtZOR6PT49P+t1BW7eXZ5dffPX1N9+8CPzEQmwh0sZqC0HFASIAQWR0IBXEWCmlrDHKKmOkFgDBMAwhhBAOFotFXbeE0dFk3EnR8TFGZI4h4U0FDYjjVKn1dl2fn13+7CefduLBR88/lZy3VZttOURAyOrwaGy5/uGnzy4u354+PHh49PDVi7P5bNa2UhuDIIaAaAXDoPvkCb64uCjLfDAYbLPN9fW1UuLm5ioIgrIsKcXFJru6uqqr1t0xLbZcc6kEUtCL/VSnRV2cn58/ffJkOBwOe32feEAaaCFBBBlojKHUcyOXI8x7nnd2drZcLgmjnudxpY21BGPGGMbEJS0RijzPYwRrrVtey5YLIbSR0ILr6xIAECfhaDRSShlCIYRRFFGtXdQkIQTjWz3k7R7hVgp/O0tBeCubjONYa+1sBMMwdGI2F53gMOG6rokXCm2cm2Tb1sZqSrHVZn9///DwsKnqP/rjX/wv/8v/Y7vdQoifPn16mU1rIYdpByEk9Y0FKIrTRq6F1IhZRHAQhUJrY4wCQMFboxitldZKKg6gQdgwQvvdntUq8oOQUoYggrbKi7/9m//e7wyHw72maa6mN1zp8WSSpB0AgKGEEGatFUpKaQGCSpmqaqqSJ0lCCOFcW2sppcbYtm0uLq486jvTSSGkNUBw6XtBI1tHU5dGN4LXzmAYI3mbLXjLZoMQQowwxhBjDIGU3AKNCOwNer7Ptm3dZOuizBhjDa+jKPZ9HxF89u5tUrAwDHt1TQg1t243MfX8rMgpYcZa6LKkCQYQCiU97JyaqFvwE0Ka1jRN4/u+sVpp7UZ/hEErBOe8E3cIpgDcum4wj8Rx6HbVAFpjjLLCQqQVgBAbe/sXd/SFXaNwX4u/K2y3akZ0y+50vZ2+C5tw1kk7eeeOeeCG3duVyr0IJQc83B9/3RdB4DlvzTiOkyR5/Pgxxvjg4MDFljoDUCfgVEo5GY611mhtrYEQodvkCOIwIUopgMb3Pd9nABqnqrXWFkXhXpAQImXpPgt1q1yVhRZbhPRtlNT3NfgPire9l5e94wTEd/wMoA0GEGjDIta25a4FsdCJIYGDJu6/4B/0Cn/QLhhj7oInzf2uZdey2DtRxu6UGmMspFwoayGApKwaaDTsdgDEJEmSp0+fOunjZDJKwsglvh8e7LnXYoxFQQAhfPv27WefffbkyRO3QdFSaqFk3UomNDR+FFoDtZScS4QQwqTb6w9HZNgLmqY5O3+7eHGxup4TbI0VdVn+8NNPfvqjTzGGy8UioOSDJ48Txmaz2WbTJFHY7caM0KbhlRUeRAFiUYelHQ+gNltfMSqTBLf1/PJ98Z//4j+cnjzQWgdRyBuUV3ye5SyKLPNKzmvBEcHQQCHaKIn/+GcfPTgKXr58MUh+9PXvv9hu53WzP9kbf7OY9oe9/vBnFlkv8I21B/v7o8k426wDSAJYEEgoTHwSIiggtIRC37NGS+qJvcMQGOsl0o/Iw2f9yvSkVPPZ8mJ9Pd9uGcEUe1Ahj1CIIfUpN2q5WCqjqe+Bc6sBYGEYxEkjlag5IGQ02XPKZj+IfOYFoVfXteKt1nq7zowvkzAhxJMCrVc1b2gYeRR3k6jtdw+fPf2hVv+vr79++ez5B3Ec/6t/9X84mTwottnV1dV4PN4/3U/6cb3aCiiEVVlTLDbzKs89jyadSCj18vWbn/3ojzq9IaSBQXi93CLmecgnzKvqFkIMLVASQAi0tEVbaCkR6rQNEo1qcpQE+0cTM7/cfvW7L43m42H3w6fPTx8c+8zzPD9gnlLq4bMOIaTIyt998dX5+eV2nUGAPv7hT25uZsZCDYBWVmojtQUQYISgVa1srQIWAM/zSEARxhaC2XJetfVqtTo82kcZUlwRRk9OTnh73ukNhYTzxebd+4u8qsMwtABxaT7//HcffvT88cODtDP88Y/+WAj9d3/z13uP+tcX6/6RpzWGEM8X0x9/9Cdnr26qOl+uprzVabfn9qlCiP39/aPhkzSNgyAIQv8//If/7d27Nw8fPhwOh5s821Girq4uoiiRUh4fH0nU1nXLm9YYQynu9XplWeZ5vlgshsPheDQxxihpKPE85pu7WcrNQFpJY0ye5/P5XGvNkIcwEfqWoGyMQchGUSSllJILITB0YcHKGrNcLjGBRunlcs4YO8T7URQlSZItV9baOI7jON5utw7NBsD5B2h9pxYjhFgEAQANl+ttXhSFUhogYrSV2gIALMQQQ+oFABFrrVCmbXnTil4caK051w6fJpQhBIUS787eI4TSNP3w448++OD5559/sdquPmQfW6ut1ZRihy1bYBkjlOJWC8actA8iDKVSQnAIAaXEQqu1llporbWWwDAMbTroGS77aeJhhCEKKGnLYrOYy8bEnXSxWm+zPE27aW+ICH7z9n1Wc8assdCNdJ7nIUgwolobjAmlTAmltIR3PHkpZVmWjx492m63UmZB6BdFkaapgLo3HHieV9d1lmUIkziOAQAGAOdIILUyBiCEGMEYYz8ItNZNUyFCMKNB6FVNfTO7lqLdbDaT8f5g2IvCZL3eLlYLLvn67MYNx/3+IAxDV+puszqp5zB/h9vvUG7neyGUcoaMxhghpfM2uEMRDKHEkdKSNCKMFQVqea2NZChw1kx3Ln7KWKUUFFwKIYxR92dWeM9+YGfc55YFO3/Joq593zcQACVvvRc95hNSVZULLEKUEEqNMRBYAKyDKwAADtg2dy4CUv7/2PrPJ0nW9T4Qe236zMryVe3HmzNnjr8WBHDhCWI3tNqVQhG7EdJ+kBQKfdJ/o5C+ahkSueAuyQV3QRDLC4C87tzjz/jpaV/eZKXP1+pDdveZC7BiYqKnp6o6u6oyn+f5PT/Dr/kK9c+67j/qBPDxePz555+PRqNaZFTThHu93nA4jOO4ziqilHJRCSGqkguhtFYIX37aaxzl2kuKEMR4KYQQUFuWtU6zi4uL9XrdbXeUUlEUEUKvmZWXtR8ijH8jkOm6M/iH36lLNcbYtS2MsRKSM6akpJQSiKqq8jyv1mkAXZsvX+avXcISEAIItf7O2entI4Fv9RNvNwrqN2Ud4Er1cL2g0VpjTBGxDMOqEMrzklfMcRxVR1LUIvvNZpMkSegHtZj15OREa33jxo39/X2T0sPDw+fPnydJMhsvqpwbhlFkuZQ6j8uZvdBaN1sdvxF0+j2DoIKzNCsIIa7ph532XsO/ee/mwc39Fy+eTifnWbIxDBqGoW27VZFLoSmiFjYbfsu1g/e//xOCLi07kiSJ4/jKXpsgDALf3d0ZNpphv9Nses5qFa3mI9+hZZEBbWmgmZYX4ykxDa4kscyeYyKsANByGSO+6Qak3bK0yGwT5Gl0evpme9jVWiNqnJ5MtreHjVaLGLQsyyIRG1pohsq4GLT6oe9BpCxMh8NOI9wdbDm+LwlNZ4uiYgxhkKRpoxHu3Ni3AldwNRlNTBMevjguMglRaXuu59jNjhu2fY1kFK+pSTplWZblKo0xMbx2k7q2YjLJizjaHB4fWaZjWRYm0CS00fCDRigE0wp0eh2LUilAmetoVSyma4RBniZBw3ftTp5K1wlfPH/9h3/0j1++eDMcDhXjRZWHzeC9Dx+VqpotJ5bvP3z/PmNgudy8eX00PT+3DTTsNikG8WZpWEGj0daAZhUzEACAMi4rXlJETHo1Q2gJkAZQMcksLmxo5Hl69PzNw5t37j14Fwv8+skraiLXbJi0BZRXFDpLKiGyLE6OLj6HEKZpzjm3XceygpJVXConaJSsUhXXRAOkKYWUmKZt8SrlQgghkEGCht8ddNv9rhv4X3/95Zvjo/lqvndj2O020zRthG6j5S2WZtjsOl5r/Ktfz1Zrw7GJZTsNOV6sv/z22f4vf9Xt/xPLhM324PbtR5999m2ZFXEEVlZ1Z//B8Zvx/vCW47gvX76M4xgAZdmGZZGSszhJgsJ1XMMwSLPZDJuNMAx2d3d93zVNmucp5xWAqqpKwyRFkdh2hxr44Tv3LQ+9enVYEpzneR4nlFKKMSQwiqLZdL69tVNyUQiGCEWESg201nXpIoQYlCilptNpVVXX2+v6b8YYQFApALSQUtbSJAUBQgjhesLDmMAazIQQ1vyGTqeTraOyLBFCrusCANI0ZYzlRdVoNIRQ1xACxrgWExZlzmbzujKt1lFZlmXFbNvO8qI+MS3LJoSUZamKUmnABcCECAmkkpZpGJRowDGl8/n85etX77zzbp7nf/QnfzxfLF3XzYrUtSgrkOAlY4yzglJKkCYY5tmm0wkFr5KYl1WZZVmRZa5pZLIUgl1ByhBjDJGGECTRxrOt3Z2tphfMRxee7czHUwR1WbI4mxwevQlbnUa7Y7neMlqP5wvDtgSATAguBFJKAawUgJAYhiYEIwQhArVGl/OK88owDC6Kbq8ZNr3xGKdxEm1mShtaa8d1DcOIk2QTx74fAIxqA0EFtAIaKaSABggigolBFZD1n6DhhWEjbLWKMsMYJlUFIKQG3r9xAwBwfHbKCra7u3t4knElN2kiAfBYBTDCGK/XG8651FAIIYFmgquqhBBq9N2kWJex+laD+RjXhpy1RQGpLX2EEKZjGiapGEQIWJZhO2ZREq1NoRWEUEkIuOQcXEIOlFybEFxXo5rfUPcNWuu6QblUExBommbdOkitamQFIAgxklpxKYSSUKHaSlxKqfISvmUXeI1YXG3HFADg2qTBNM04jmtYwnEchNDJycnXX3/NOa/1/FtbW+12u4blashhvphyziHAVcWlVNfwvhAijisIoWlRw6B1cGtRFJ7V6PV6SIMoiuI4dm2HMVYul6ZpXQMVNUMTYQQJBFBdox1vD/rwN2/XdXoxnTUaDdswWVFmWWZQWrethBCltVJAKAk1kFrXgZ/4SocC3uoHroGca7gCQY0AqJeJ6h9kZ7wNbLzdygAAuJBCSGoYrBLL1caiuN1sFEVJDMOYzWZ1Tzqbzdphs9FoRFH0s5/9rO6jPc+lGB8fH49GI4RQvCmrUmOIhBBaymgZ15ENftjoDwaU4kYz8DxHQ8WkkJox7RhOu9PrDA/u3X3w+Oz0ZDI6jVZL37Zn8+zi9EwJ6RjtWbJeLJa9Xi9eiUbDDbwmpdQyMtfNXNdttVqbdbRard55+P7jdz+yqGFZVpXwFy9eHDy45/t+Eq3jOOaVyIF6+fqVApKLyvOcVtOvWK5ELjkti0xXawIdJctoFWGkMYSc88l00fA7o/FyE5eNRvT48eP7dx+UZXlydPLNFy91lRmE+I5JKXY93PCoY2mDIEpJ0OgqZaU5Nk23VMi0W5ZlW86CUmpZPQUyBKrFbGUSp+G1bMu3XEJM0B20DYPevn3n3u07URSJzz7PyyJsd5hQioJCctf3HT+oHTPjOHMs8s7Dh82yIAi02+293YP1clmWueeFW8ODJI7iOKrKJEOSYFdwGPgtjSCCpNlsjkYjH2FEYa/X8aT92befLfPNex99PGj1ZvPV0O1XVRatJqwqsIE911DaZZXkHBY5LyteFSyrkqwobddtNBqWbdbr84oJhJXjUtNCYsER0FKrxXj82S9/FVh0q7v98fs/AloqpVynhVBjOp2+evHi/Px8vV47tk8IYVI0Go12ZwthnC/mSVE6jsOBQkCbAFjQgBjVtBjOIGOs4kxqDanEBvADu9MPt3cHi9W03XFb7cbBjd3DV6/DdqMSJdcgY9J1veVmM10tNMKlEpbn375PlObPX70uODdsV0Jqes3+9n60PL994531bC2YmaVqPFq98k/6/cHJmwtCsZKyYpkCmhpA6SrN1ivFV+tllqfdbvsnP/kd07a4qFzPTvLsEmvVtK7BtYf/vYObT7/4ipelQWhcFZILw7Bsy9IAvjk+Miy73W5zDRTFAmMFgAFo7fwIIbRtqyiyyWRaM8allBromv/FpRZCQIhr12FqYNM0ob7cK0MAiqKQigvbgRD6vp9lGQDg4cOH8WJZCzivNWnXNKvrMfH60gYA0AAkSVJfalerVZqmrut2u936cpFlWX1NrJmMZVlKjjqdDpBCSA5tm0tRFrlhENt10zQ/OjqyLOv9x+/1er31en1xceGG4SaS0XKaZZkoE89qUawdAzVce2fYs10zimJZldlmvUmTMAx1dikkQxgQRDGGSklWVbzIqZa2YW4N+zLLdoaD1y9edsOWQEGcpkoCw7SF1Ks4idKsktK1rKqqmJBMSASUkiWEGEGCiVBaCKlqNF5KoZSkBrRsImSVZpudnZ2KpfFm4fuOENUmLsDVS6cg4EoWRVFWlVCyLopIYyS+47hVVQWAAgB4vm85pmUbQpa25zJehc2G0prxEgJsWUaSJLZnt9ttcZlJUYirkOUoimsLh3oIllJqDWsvNZlm9TIeVhW68mKqSynCEClVVVX92UB1AjWvDNuAENaulJZlmCaFENaTNwCAEAQgBgABDS3LVFdZiNfg+dtAen2rCYk1w8akXg1y1OvwGuGo1x/wqmu59leAEGIprxmO+spIqibGXqUVfld6EUKuG9T7BUppu92GENYmB4vFgjG2XC7rRUbdfBdFIeSliwnGGEIEwXfOj5xXNQ/Ptq0k2WitayuRra2tbqv94uWzOI7jOBZCaADqA9ZXfglAKagUggQidL000VfQf42y1PD+NR5T/+7JJu53e7du3boYj+fzOTENIYRnOzVrW0ONgBZKXeMK9W+tAdTokntYv5K/gW1cNQoAaADU9Ut9fZrXPxpehYFdAxIQQiEU41LrapOk0ToOG64CiHFJut3umzdv6h5wPB5v9QfD4XA8HteSByHEaHQx6PXq14gxFu5sS6GFEhhADLGSqqpYWZZlWaZpzCXjQHS3BgppqUXORLN5oxKkjJhlmZ3BTc/tdNrDaLmosrQqysn02aunz8925gQb89nMNE9tv3Pr7p0HDwLHMdbLMs1LgzQ9p5vGcr06t83Wuw/f0VonceRaTS1opx36vn9rf3c0nizWERdiMpk0Gn6r3fA9x7LNaD1WsnQMbEDDMRGGuuG7WZI8ePBgf2dvd2f/my+fbg9382ziOlQrslzE89lXR0dHmJBBZw/ena1WUcMX29v9oOFVooyTBTVYxWEQuBAFAISW3VOSQtjV2iZ07HtOq+lZNjIIHJ3NMDQCv21iZzmP4niFEBhs97e2hqZjp+ORREAi4IWNvGK6rLI8t/2A2lbQaqZxEqfA8yi1zIvxKI2Td9995+XL1/E6smyjFTbDsG0Q03Ua7WavqopuZ7C9vUepOdjams1mOzs7y+VSGd755Mxwye7dHUThOlrMlhOrzM9GY9cOmaoA1n7D294Z+LaBoJzN16tVtFyuC8bTLF9vIibUjVu+UopgSAnO8ipJ1hpIwyAYY5sYCOHQbxmG8fr5syqL33//fcdyms3m4eHh4eGZba+Wy+X5aAmAtbd/L4viRqNRVGWa5YevjzEljWbz3v0HpxfnNgSGaVKT1GdaTbEB2uCiyvI8ztI4jipelixfxt042XDNhjvDTqd18+bBdHLuB3ZZpYbjMiFNoDdptk5SJoWS3PU9Jqr1YnNxEf1v/svRQ/8OpJbfCB+++/70pPXg3oN/8f/781cvzi1o/+qXX7IU/F/+2//rz//Dr6RUWZaBIm60mq12QClcrsaZsmpewvb2cHt7eHp+xkX1zjvvLNc/Y4whRLjgjuOmaVw7393c7m2idZ4VW1tbnu0AjSBBCCHLdl8fHSFiHCgNEESYKggQJSax05TX0149VK3X6+3tbQCA4FIDTSjBBKqK1xNEbacDoKqqSnIG6pBAQk3TTNJadIoxxqPR6Pz8fG97h1Jag881bc1xnEajgYmRZdnf33dehQxlWVY3bavVKs/zGs7tdrsAgLOzs9paOI7j1WpVlqVJeLPZhBAoBTDGQkrGuW2b9+/fj6JoNplMp9PJbNob9ur5xMRQ82q9WiZxKgUzDWRRUpn0wf3b7737iAn+RryRqqqKPIui0HXBpX+twhiTOpGVcyFVt9ms0jTerHnZpxQfHBysF8u9nd3RnPWHeJOkCsHTi/O4KpXWJauSvKh58kgjg1AAkEmJaVkKFFrXO3JkGEQqbUBCoCWE4Ly6uDjb3R1iDIsyaYTuer2OoqhevSsIam16lmV5UdS/PsRXUQVacykB5wDoGjbGBFVVFcWbZBPlRQYJ7na7x8fHz1+8CINms9lcraL5fK6ANizT8zyDmlpriFCW5+tNZBiG1qwu2IhgwzQbYdjtds9fvqqrHab0svRibJgm55xCUs9+UkqpCKYUAECxSQjBGFYVZaysq3iSJMvlsqg4AMQ0LEQourIcyNL0kq7/FmRdf8ZqHmLdU9aliFLKJKsZl5zzellQuyn4vl9vDQgh9fq/vj+6MoG+nsjrX4cQstlsahP0t8MjTNO8tDCpqtVqVRTFarUKw7BmD9TMA9M0a+JdmqZcVAAAoBEAqAbxwRUZE0Jo23a73a7TQbPcrkkerVbLtezFcpYkSZakACDLsjgXl45kXHFR112kITSuDDmuO29xFXZ1XZKvewUpZafdvnXr1ve+973PPvv8+PhYKVUxRohRwz8KKFHf/woOQP8AUbhuPr7r1QC87OOAhm9Zev/GAHAV9HCtH6nBAmpYGGOtleCKK2matuv6lJrk/juPgkZzPp+OJxdCsOPzIw2Y1urOndvpJj57c7ocR8P+tmU5RIaObWVF2mm3EQBZshFMACWJFm3XzPMcFlBnpdhUGx0VZWW49nC4XcrCtz2r4QINAFcQYaVxmpbLycJ1rJ/89u90m81vPv9yE604561GICA7O2Mffnj7+cuneZ4PtvoCRIcn3xSsKnQcp/yf/cWXZVm2m80SpcARm9k5ZKEuYkMWTQtOxusGAT5SIUW+3/n1558qzaRkUlVB4AkGiSSQIV2gZFFubP58c7pZKgszCoNoUcSr4rd++Fu+Z7iWfv365dnxWXvQvXv/7iaJkLFWqOCqDJpkFU33w93jN6dbrYFP7Gw23d3aluXY0E6BtJClkCxsur1+U0swPovu3n6v4fb++f/3X8o8r3JaFdXh4eHZ+dHrw+dpQu7ev4cVjuZRb2s7DOInXz1zHJcgo6qE74DAD/O0YpVy7EBKWG4ypODh89dsb6gV9337T//s987PjhkrISwOX3wjis3R8wVi+SeffHJyeKh3ITbYN0++RZbx+O7HF2fJVz973QibUspxOg0cZ7vZeXj3DoUgXqyyeZap11hxzyRQMq5FN3Adx0GixIXiUChM+2HoQjKfz7vNbrfb9R9Yq9VKMm5SQ00zXm48C97dv8PKajPG8yxXae5A9uCgq7RcLBbYRwXMrNAymy0AgOXYSqmf/fIXiGBCCCZGwSXnKedcKgAA4LACANSf+oqrTbacLxfkNQVAOa6VK9buBHv72/ce3LoYn9y8swMKWnE2vxi7lpluopJXfrN1dHHmurbGBCL+13/5N9udbRe5qKR3tu6ATDtG8F/8Z//V//w//cXxizeyKkenx9Fy6hh4U+QGwZgaeVYhbPihn+eAENHtds/PT588eUIp5hVTSr/78PH5yQXn8sWLV0hC3/BBmfmGf/rquN9xH7//7s9//vOCFb1+e7lYIQQw0KIsdnt9JITOCw8TI/QQQm8OjyUcb+1sNxpmnpez2UhruLWzW1QcY4oo4VLxSnBZbxaJFNowdZqmSinbtiFFWZZKRRzTjDarwWBwdPim1+tFUbS/d+PJt8/u3LxzePFiPB97LbtddiAERVpgTXu9QGKVyyqOksFWXwLJdCG4bLVCClxbcsOgCigmmB84YdMXsuKivHFzb7GcAgg3yWaVRFGW2LbthWGlJVLAa/gKIcZ12O4EgUeodffuPd/xWq1Wt9XfrJKf/OQP4ziGpmktYlegKBOW4TKJX7853dnb7fW3s6y49+D+eDwtisqxrFltIayI5TlKcFZmeV4Cwfqd8M7N/abvGwjyvDx8+aTf7n7+618mm5izYpWuyopv7TefPH2+s7uvMVysNs2wIZXUgld51mw2GS9FWTRbg/V6RaW0Gg2L4EpVGvHAM9vtsBEEUbQqiurOze3p6BRKdP/WgxcvXnXDgdgLzs7OViLyAh8jg9Vpx0hjigtWKVFr+aCWUjCuJTZNl3NGMRZCNEJ3PhtLyS2LYGxOV5NVsu72Bu98/Lio+Nlq8eTk8FZ/2/PDXmc4mk7TNN3e2TMd128ZBEGMseBVutloJVqBv9UNt7a6s6M3CIFkExNCXNMCACjDK8vSIjbSCCGtMK5EpRmyDM9xHK8VEGQzxnhVAuXzyooWwqbtbLOAkCKEpFKQSJNSghAU30Ua1l3CNYfAMIw6PL12Gq0xjCRJoGlKrhDAJsVS8rRIlFImpZJzRKkUlZJMCkEw0EqxKtXAwFo5nksMzFgJoUYIVEVCXNekulCl4zhKgc0mQRqUMSekJ4SwbTuO4263e35+3u1266IYxzGo5R6c27Zdx1s4TiiEYEJyJTWo6aUUIaQdw3BNRfGrs+ODvb1NllDTaDSDfLUc9Dqji8kmySzHL5iGECPLoSZg2bxgSkICiQZSQSUt2yJSAgQlQqUUXCgAMUQGUEJIaFETagUVMzGyKc2yNFosguFwb2/vt377H01mM68RCCECwzgbjUMaCiEARJZhVJyzPJdSE0IkwoBQrZSWAgBlEo0xxIBRpLHmtYMLQAAiAoCSGiJkVgpjASiBQAGpeJ12wYqi5o1KoTExFBCFgIBSAxNkmFryoNVSiEqIoiTzgwaxLW97Z7/b7VuWk8TZZDxfL1ZRtDYIDcPQdX1eCaEYxk7Y9Gzb5iAnVAItMZGEQNcOfM+rfaABxr3B1s7Nm3FevPzq6/KcSYw/vHsLY8irCgLFijKON4vlcjKbLeczk+LQ9xph+I9+77ddy14t50VRzKIojuOvv/5yNJ0YhoEInE6nTPAsy5rN5oeffAwA+Pbbb8/OzkrG9m/cOBudLVYriBG2DN82y6qKomi2mAZNTyUiSTaWTXr9zmVUHq/ORxeffvZrzjQE5Pz8fDjYoYbx9OnTfr/farV2t4eWZRNqhM3O9nbleUF3u+F5XsVTDQXnBRcFobbtkDyLPd/UQpZsY5iIGDxZrggJWFUli816vS6K6uTNRRJXvEDLvQlpWRamJjXMgD5/+s34nN++fdBttbf6nU4YKilsgxoI2pRUeSY4r7JMlIxVQHFmUgIts8jy2eii8h2EUKfTAqDOe3XiON7b2/vss0+DwGs0Gvt7By9fvnzx4sWdO3cePHjw5mK8s3fw/AX/t//zv9zev0khqpLVP/3X/0O71XUdhxXlzRv7+M6tk8NjwZltOstlHgRBv9+vqmo2nwshfN9HlHzxxReuF3iet9XvbQ8Ht2/eCMMQY6y0eHjv/mI2X0xn3WZruzeoY0HevD6cLxYQwkYYXvq6awURtuSli3vJxDULGmM8Go/rRqGG1KSUGiAIIbLA1YJNScmFEpRiAhFXsqoqJWTtDbe9vV1V1XKxTldx2Gp6nut5Xg05EoTbzVZZ5kDLXi/4xS9/9vid+9bHH4WBR4g3n8ywgu89eu/urbsvvnruur5lOb/4xS+///0f/vN//udho1VW3LBsJXSZFCVn6yq/efNmt9uHEI5Gozp+5uLiYjgc5nlJqfkf/8PP6jQNANRyuSxy5rkN3wvTJCtyxjlvNTuGYSgFlJJlVUTRKkk2zSai1MZEm6ajtS7LsjZo0/rS149zLpSS+pJZfY0xFkVhmjZCQAihpLRtR2uVpmmddt3pdIQQnMvpdFoTGG/evDkdz5aL9bA/GPSG5ycX9aWfMdbv94sq32w2pmNqqLzQZ0wAVAfcQaWU7ZgYoixLFnOdZ0kjDCGEWZFuNomUynEc1/Nqkh0mCAJUcVZVlWVQ13XX63VV5kIr07RLVlFquq57fn6Obbu+lF8z1Wu+57Nnz4bDYcXZbDbjnDcajXrfAbXK0wRDYBoGhlIj1et0H967f/PGQRqtn3397f7OLtRgdH6RJvFsMQ96fS6SvCzr3Tm13fUmWUUR46J2/SvL0nYs13Zs246iyLTdkom8LKEWlm20Wq1bNw+2tgaiYnmW5EkOJd/uDdvtdhEvnz17drEStmWWrNxEbL5eQYTvP3xQchbFm3pcpphwLqu8qHOnlASW5XiObVuO4IBpTggyTavT6eRHR+tVivAqz0qtIUbUc/w6dLSqqmiTgKv8J6UUl5cGGI7jQKAMy9EQVVVVFWm9CcIG7ff71DAB0rZnZ1mmAAQa1PwtxnhWlgrjfDa7DqGuh8v6bK2XX9eYAbjyDKg7gBqUumYycs5ns1m95KqXblVV1ckAdQBUvcWQUlzxGy4nY6W+s+Ko1xyyVGVZrlYrjGFVlJc+2UDV3EYhRA0q1JQLSmiNkYCrWKmat1gfeaPRqE0jDMOoAQbP8xhXhBCICdZKK6gh0BDVHAuMcZSlUrCpaUohqrIoy3wQhjUaQSktipJzTghECNmWTchaCMG4vNZqMsYQghB+t3TQECr9XXokQgi8BfUjhHq93p07d5bL5dnZWf2L1ApJ/ZsSBlgrNOB/QhL51pd///Y2HwL8pt4BX6VIyCtmdE1evjTJuHo98zyfTqeCVeT8bAYAsG0zbPZu33oAtCjKjFXi9atXlmW02804jqFWtkdM6vi+f+POA9u0tJTRas3K3DJM1/UNw7Bs52I6m6ynQ3Tw8L13tG0dn5wavn1+ftpsNj3XZawESgcNf2t7sF7N/u4/PLt1Y//Gzb3+sEsJ8n0/iQZVVZmO+8033yBKalcQBOF4Oo2iyLKs7e1tivBoNErTdDgcttvtNE03Mqs4bwbB3t6ebblxHI/H48VsMluMe73O9t6g3+/dun1js1mfnBxJrUzbWa43luVYpnl6ftZsde/evyOlrlW8W7s7y/XGKoyKQ4xty5ZSyoozSikxqGEQJgVjTAN5cXG2NejZluHYhu+4rkNXsxIqqgq1WcTzyXy5XE4v1kBD12obCB7s7jR9b7NavfvokUVQ4JnbO/2L0cmrF+e9doNC2mr43WZjb3srbPgaoMlkslqtlsulaRDfJJLYDkG2Y7W7nfl8qhRar9cY6TzPf/aznw37/bJk5+ejfqf/0Uef/OAHP3rz5o2U8OmTl9iGBHLb0KvZaL1eBGFo2t69g92iqP4f//f/2/np6ae//NWzL7/J0jRw3cOTk+H2bc6qlDMAQLsVNpvNWvj3k9/5R67rGoaxf+NGu91er9fn5+dPn3zd7na2+j3Oq3W0bAXhJl5//vk8iTYIwNVqBSGsmAAIGobRarV6w63Rcl1fNbKiStP0+pN6fbIBcEkFQphijDfZulYJAgTr/FxIEITItdyizDCmVVFiiPb3b7x48SxNU99366pJKW01GyVnrMwLVnHOe+2O4Czn+he/+MUf//7vzabjwPO//70f/r/+n//v9WLzu7/zB48fvPf5rz6/eHPy4OGjVtj+F3/+ryzLLuMUAcxyluJMKVVlSVlUdciqaZrvvPPg7t27f/M3f/P8+fMbN2794R/+4Xy2IMQQQi3mq6Io/vk/G1uWZVu+bflSSk44AGC1iuoLk2EYrmf5geO4JsYQE9VsNjUEZVnWjuxKKam+01m9TVkCV179QRAghOI4poR0ui3DMIo0Wy4XmyhxHYex0jTNNE0Nw8KIQo1cx1dSzmaz/Z39RsNP42w2GfeHg36///z506xIh/4OxtjExmoTQYg927FMahi0GQZJsok3mypLEUIQ3iiKbLPeTJcrAKBUQGOCQaV1ZNmmsk3HMkzLEkpWFWdSiESYhrFYr6J1nCTJgwcPDGqdjUYQwp2dna2dnc8///z4+Pj+/ft5WdRQ7Xg8TtM0iqKaol8UxaDTXi4XgldQal4UvMq0EqZpvnj6rCrzV69eGZQCqYqyaHe70+l8Npv5QRi2OxUTWuvNZhPHcRRFGsBut1tHbIRmIwiCmksPBQAAOK7VaXcankcJWC5WrMhv7e/dffTO0eHhbDKp0lX3YHur469C+/mbo5UEfsM1sOlZpgB6dHEKEEaEYEwwRHUTVhSF1ppSqqU2sGUabsVlvIyyLKEG8X3fMD1quI2wxQr9zVdPkzidzWZhGLJkzbRkzK1rIUZAA2ybRrvbMymBELKqUJybBqm4XEWbW7d3EUKj0aioOKWQi3IdR4ZpaQzzolIKYEIUJVJJZRim76+mF7W6tS6o4K0Y4ustwHV9qgtefbfalLPeMlwrR8IwrANN6odXVQWulvRKKSkvjRCuC9jfo1sCACzLqqqq/rGmadY1XrAyyzLXteueBkJYVWVRFKXKQ9OtqaONRiMMw+tjzrLMdd16eVc3GYwx13WBljWMgJSsVG2XpCRCZcUtamw2EUZgPp0GnluWJVCSdNrPnz+PNymEsF6C+H6j1emVFXOcFSGECUUpRQjLS3AFvKVJVAoAqS9TMQEAFCMEvhNPUkoHg0G32/3iiy+eP3+epqllWUVZ1lJYXasi38qUghDqK34ieKtRkPIyvOO6b6gfe32heKvbQODqCevDgEBcv/gQQqAl5xwoIYXQEhVFsRQVKzKCqZUkSVaUUKtub6vXbSvJXce3LKcqc8bKalEqpRBud/rh1tbW7bs9kxoEojRpxusIKI0QLiueFFGr3VCYlizfZGsvsPdu7w2G20qoQbcDIXxy8ma9mN+6eYCBzso4L1Ni4rDbgBokyYZtFnESa63T+XQVLZVSRZZDqAlBQGsEYbvV4hV7/fLw9PysEtzzG0XJNln+5ux8s9ncvnXr4PadVr+9tb892B2+fvm8qop2u+k1fYShUIpJ4QaN3f2biIqg0bRt17Ls6WyVV8XNO7dtz/3lz365ipabNBlNLhoNXymZ5zlEIEl5WQEpMaWOZfiB51RVgTFG0tbcwoZNobmexlXM4lVGobmZRZtFBCplAtpp+rbV6Da3tvt9AuTO9sAm0d1bNy0D2RZtNoLx6BQCvjPo+Y0gilOkxfagl2dpxdnBzlBrHW3WySaWkksuKIaNRmO6XiOENpsNJnDQ7ziOlaTrxWLheZ7r+kmS1du+yXjh+1WWpnm1sg0a+u7jB3dWcVwyvt1pPrh1YzKZBQbe7nbQB+9fnF78+ptvIISOaUt5UtOOTNNstVo2pVrxTbTc29vb2dlBCGTJ+hdPvx6NRlLKNEmChl8WeeC5O1tbrGTL+UILmec5UBoAVFVsk4wRQu1eN1CgVqPVDGTHkxDCrMhrXKHOxRFCCH6ZcIMwvdqfAQAAVBporTTgTGIsms1mxYqG17ANGwAYeL5g0nP8XreZJFnJWRj4g15/Npsl65UEGmNsEbKMVgYBgpWMl1orgEAcp8enJ2fHZ9Px7L/8z/+3f/KP//Rf/vn/uEqSra29e++8s5gtAUBKSCE1ywrbdksNF4tVHEdBw+e82t3ddRwHY7xarfr9/mazdlx7PltwLoVkoROMJ4tGo8HYpc2cUsI0IcY4z1OlsO+73W47L2IhSg2w51vNZjPJ0qIotP7uYlpHDQEptaw5zLCerjQAWkMhlNaSMeZYtucGrVZLtVme57YNZtOJZVlaXwrWnz9/jhG1Tce27eM3b0xiOra1WiyllEWebu+87/uuYdHNeq0htCyn3+lHaWE5rmubge9aBGklTAL2d/cmkwnUvCwKKaVl2QDikgkAaVGVUbxxXbvX7bTbTS2NyWScpqnrWK7rQkTSJC/L8uL0wrbdra2t12fHZVlub2+3Wi14RcM2DKPT6fi+P5lNCSF1xk9dim7cPBiZdLValHmiWGUb1KIkS5OLi7OG59++fXu5XGZx4jjO7u5+nGZff/bsJ7/3B+99+NFguP3rz744OTlZbxKlgZDf2fjU5LskSQEA3eFwE0Ul42lWKaUUK5WsbIothO7eunnv1k2RJ5PxBQJ3Agd+76OH01gKoXYP9j//+qsyy7b2hovl0m+1DNPSGkqhGGNVXiilHMv1fR9oSinlUmZZNltGURQBoExztUnybrfruc3lfHF6dJ5lGUGo3WhVWAEAGr6LMZYKcM4RppZlFUUBgO1YZqPRsCzLdxxKKQTq0e3HhJCXL19PZ7Og2ZrNl3K5QibN0nwVb0rGDdOSUudV6SpgeB7CtJZhYGIQanLOueBS6qJkNZGwvmGMEUaYwCLLrikI9TtSj541gHcdcQkASNN0s9lQ171GI6T8blt/vbCvz/or1wFF6WV8GqVYMF7fTfKqdv+ry6BpmoZBAQAYQEJpzfEMgqDRaNTNd62GqJGPuj+rwcU6uQpCqCFUQkrOGecAYYSQZRmEEIyQYxpxtIFaScFqjPbi4iJLCzfw69Jety9FWdU/i8tLHwINQO1WAN/SEVylPIJL7R64dPKunyoIgq2trcVi8fOf/3y9Xn9nIIFx/fV1lvRb8gR1/cRv/aDfMHf6h3DCdzxHiPQV7VEppYUAGl1LSzDGBAHJK8EqraS2KNRSKSClJAhTw7CEYFEUxdFKa3jzxv4nn/xgMNhaLWZlkezs7ChRGQYpiuJ8fIKMCEPkOA6SusgykxoIkXgTb5JkuL0vmfzm68+PTo+6gy1qO9PRmShkHm1Mk87Ozl6/frmajRuBNxmN9vd2tJZnZydCcM5KzvnR0eF6vS5L4dkOxSTP8zzPJeO8yImGWsjx2Uhu6Z2dnayonr548fTZM8MwiONixp1G02s2Dc+zLVsjaDf8zSheJ2m7FWqtFSKcw4qp1TphOkeGlRUlMexGq7FK1mcXZ91ud7jfj6PNKp4yxoKmhTA0bGhZthYBY5yxSgloYNMyG0QxyzIGTTq6OM+Ykol+/uzEdQzbMCwIeK5kqRpe6FoOpWYz6Az6uwd7u0lc9rttA5vtTtMwSbMZGCZarOar5UKKvCwgUDzJkotzfHb6WgOUZVlNyfE9gxAbKCUEMyhQSgVBsNmsKaKc882m8jwPY2JZVq87WC2jL7/8crOJlVJ7ewdxtAk75maTpGm6WqyIQR/de7B34wAhcvfmzePDF77X+OSD99uNMFovT96c2Lb9/NmT3d3de/fu9Xq9OuQaY2xT9O23X88vTmotsmGQYbthWdZ6vT45Pwt9b9Dvm4bx8umzZBP3O7042lBq9vt9BfRysdYIIkwXq/XR2XnBq0ajYZomhJeZcjUvSWrFGBOX4ngJIcREKaUoolJLrSWCiCkluZCCcVa2WiFQ0He9VquluMqTLIvTMAjSNC1YaVnOzs7W/u7O02dPWK6aba+qytV8QTCeTMTu723/x//4H3784x97jiW5fvjeu2cnp1988/Xe/v4/+ZN/8tFvfX89W82S1Y9+57f+u//unzLFCKCGSWoPOCn1ZpMAABHEFxdjxv6j69o1RSCKop/+9KfNZiiESJOsLMuiyG7dusU5XywWdQuf53lN8242m0VR1JeYLMvm83mz2awv+hWvEyY1hlDKul+44tXXeOYlVVsLKQFUSZLUzwl1NZ8vOee2ae7s7PX73b/6y3/rOt5iOddaVxX/8suv//M/+8On374gkPBKCM6bQeNF8qzZbEpe2qaRp7ECKM+zW7fvQo3m00WhdJ7ntmlIEVKoo+Wi12198N7j56axWK3G52cCYGI5EBMNFcJEKVRU3LZtqZSU9cu1qV8913Ul0LbtbG/vjsfTpy9eeo1wd3f3q6++Wi6XRVUxxg4ODgzDcAwXYzyfzyeTSe0F53kexBgh1Gl4qmoiWc2qrOFaN28e3L1zG2lV5KlFyQ9+8IO//fd/u0mT5XpDTdsyDNd1wzAMw5ALhb/8uvbPgRBBCGu/qfqKWZZllmWUUs5lXlRVmUMIJTd5lbEiw1AO+/2jo5Pd7YGCIM/ToszHs7Hv2r/74+8/fPdRXpST0fH4IiVA/PAHnzTC9vNXrxmXDAiliG1atul4nhf4YRRtqqpiWZqVhZCKGiaTImdiMl1qhaPVIkvSOzcO7t++QxHutJtChJPJpCryMs8RMYCUCgCtYZ32KaUEwLEdgg1DQyiY2OSV6xrIdKwgtJyAWIXlBWGnS+xCEaMsKmJajHO1SRQEBassw5BSSgAqITRCnHMupVIKGwYAQNSSNq3JVYG/mpO/0zvU/USNGdSKwRrZrkmRoW3Lq1BppcQVy49c0/2+m3SvtirXps61yQeEsNY71FYW9R3qh3i2o+vmhnN9KXGMy7JUStWinkvTCEJqEiWEEGkApJJa10CdQQjEFGPcabUYY8J2CEHzWUYQ7LTCg929Vqt1ydjNsrKs0qzYbJLZYoUJrYXHEMJL8F5rABC8DHSAV70QqkM66l+KEGIYBEENlKaUmo67s7Pz7bffvnjxwnVdBQEhxLSsOMv1lVBZfwcAfAcoaq2vHKF/I3Oynqr+3iZCX9pRIAABhFC/FYGttIZAXqNHhBCENNQKQWCY1LFMiyKMIMWQLFdTy7KC0F+vl98+fXp4eAjAbz9650HgNwkhrm1KUUrJknTz6vmzaLNaRzBPM9MwkAZVmvueQ7ExG0+mi2UcJVKB4/Mx1/rg9j3Pb8yWC5Zr17U//PDDna0BUaKscl4UBGjXNLJN9IYVaRqnaZxl2fnFKWPs8Xsf7e/uKS6KPBcFD3zPIrQWAhGEBZNZVpRCFHm5WK0IIbGJuKIC0lVaRumF0kxU1cV0djGaClZ+8skn/W6v2+0iaOWlOjqe+C3DdoLD8WGS5a7rpnlyfHFouui9jx5OJiPHwm2rGbYczjkuMSE6W0shtUEdrSGCNoX2eLoqiqLVbC4mqUFRw3UM3QjtpkVJYPY2cCWEhgALoQjRGkgAVZxE8aaSQGIDTedzz3cgwXGWQIIdx3j28htCzWana9nOaHq6jud+EDKZb1IJAHBsuxu0CEFZIiqWtlotjKHr2ZQSSqllGa1WmGfJ1tb21tb2y03S7Q7ef/+jne090zR/+fNfCVQtVwXn0rHDRjPodfqBGyCEgiBwHcuxvSjarNazn/zkd2bvLF69evXjH/12zYSXUq6j5Wx0Eceb6WxcleVZtFgul+12896dO1vdru/7DgXQ9m4c7G0NhmfHZ5yVUGvbsTbriBDieC5nksm5kjDJiqqKN0mswGWQASJGFEVM8Jp+DxAkhACIkbgk/BNKEUIEUaB0jSYAqRVXTDKI9GYdAwQBQEDB0flFyaqLi4s8K4ila7/6RhA8fvedyWRS66p5IUzP0EoMegAjNJ1Oj44Ofd/3/PDZ4bP1atUO23/5d3/lNNwbOwfrLJ4l89uPblPPSKucI247VlHleZIMutt5nhOKMKatZlsp8OLFK8ex6oE4TTcfffSRZTmLxaLmWPV6rfF4nGUby7IoJYSgWh0XRbFSKgwxIYYUME1K02Cm4VZVVevZOJdaSkKIAppxyRhjQgil6y3M9axAsFFHxRBsCCFGo9F0OrUN84MP39va2tnbO6AUX1xcWJYFARJCGph6tsuqqiQkidbb28Mw8FeLueXYSvB+tzdfrsqS2bab5/nh62NGsYHR9lZ/iFpaMaClQRBQ/NaNPaWEkrxgDCugiSwrAYiBheKcY0orITZxiok2bPvG3p6S3DDtaL3JaLG/d8Oy3VcvX9qW84Mff284HAIAXr16NZ1O66Hw1vZtpdTR0VG9G261Wr1ebxVF4/G41epAzihUUDLPMm/t728P+uPR+dnZ2WI2f/TokUZQaT2ZTYuqfPz4vUePHkkpP//88xrNajQahuVEcdLp9uqVcO3fhyGq/7lYLJJkcxmcyAVnZVWUkpUFg/Mo7W9h6gQMGReLZam1LNnt/XB70Pr6myd/9o//+J1HZ18/fTrotLkGVZ6VXAgJIUCmaSoFBFdJkqzWcyllxQSTAhNqOraJoBACaiC1qqoKKOC73na/59hWt9U2bZXG0fHpeZYXze7AcRwFIeMSUwMAIISouGCMpVnBGEuTzXS18H2/TjqQRFSSGG7DtAOpSRMZAADbtqXQ601UliUhBLKyRq2vQ5jqIlfLMq9T0a/tk+sdELiSHtTNSt0H1Ah/rY+rUcN6oXNJUZJSqUs7B3nlAI3Qd37MCCEItYXtuvMoiuzSBNowag5BTZn0PK/f75+dnc5ms5TEje5QSpnnue/7dcdQ1+faQKlG12sswbZtrTUCUGkNlEYAWoYJMcKIAowYY3macs7LnCkhHcva39/f2909fPV8vV4DjZIkqSomFcjzLM1L07IvdRzi2gkbQYjfnuYvv6PBNSfgEkGRXHFhWaZrO67rPn36dLPZbG1tG4JzzgGE9VuDEbpGFK57grdXCVc02d+wtXgLx7hsHb5bgyB8fR/4m1qJa5klAgpjQgl2bTPwHKiV4ExyRtzAAQAYBnV9F0L45vgYIXR2dr41GNgWAS4tqwxo1Wn3zcc2F1UUX5SFlBICrYRQSkAlZLpOFufjpu3fvf+QIvy//OVff/PZFwc3bjiOt3/wkCrtINwNAiREXmSLxWx6draOlgqoMAySdDMeX2RZZhjk4b27XsPTUM2WM9d0XNf2PK+E2DadsixbHR9APJ0uBAaO5/teY7WJSmR7QdttdCCxBS8QsgY7/b0bN7787LPPPvtssYrTpJrNI6gRxi7neJPkQdi2nGmRpwhjqRiXJC83xBz0txp5lrR6NkSVEDlXaZpUeUIpMQI3VBKZFkJQj8bnx29O7ty6HXp+GkVVnO1u7d462F/MJw2/VcJhkiSUmsvlMk3TaB2v4wLBU8Gw7/bareHR+XFRFJPJuCiTTqd1++buxcUFolSoSmOrKgvTJsgAP/rtHyiliqIgEHmeU5VFJQoJiEgZY+z+/ftaK0rQ9vYQABWt1k+fPNcKnp6eF3mVZ2WWFb7fvHnz9pPXh2kW39jff+fRA0LQaHL25MkzyzK2trYqwZ/PnkNEVtFSIeg2/EcfvJvO89VyvlzMAABVVSIAKUHdMMzyxPd7w24rzZJXz765ODn88MMPb+3vmalohkEax0m6cRwni5PValW7eGkF1/FmvYmVBElWYEo8z4vTTT0WEMOqqkpDoJSquKg4k1fxvugqUrY+FbXWSnGgNNQAQngF3QHPceNo8/r167IspeSnp6eHr9588MNH9eBOCLl58+Yf/xFsuN6LFy8meuR7/ni2HAyDNE0++uj9V69eKSA50qlItAEitpm9WX4wO33w/jv5UVmk+Q3j1rsfv/v1F98k67hSBrawbVi27TLGtNJxHO/v77qes9lEhKA7d+6Mx2Ol1GQySpK43W67rtvv9/uDoZDlt0++nM3HjuNaltUMmhDC09PzIAhd1yfYRJBapougmcRlxZXl2IZhSFkBKQkhAEGpmJQKIQQvLWvqtXE92CFCUH19EJfbXqa4GI+mQohWq1VfQ2vrQM9x1utNq9V58/rVsNfP83xne/j4vUeff/45Y8zznP/Tf/t//PyLL//yr/79t19/E7ZaW4PBrCwtk/b7w8FgoFjuUNT03fls0m63h/3BnTt3pqtNVkkOqIQCE4MVZV5UjAmQFoZBfdemlDY7bc92Vsv5JonTOGu3uwBAANHZxfmDzb2bN2+Ox+PaJXcymQyHw9qIwjTNwWAwm81qGecmSdI0zeM1u5o0LMtst0Lfc86E2NvZdRxnOp1algUJtj13vYlXUfTJBx+u1puXh288v7Fery3LCludIGwGjXCxWOirlMJOp0MpXq1WrWYIgJrP56PRCGPsO65j2XbgFgIwbYyXSaExR3Selh//1k+Wy3kZbb789a+iePPh977/0ScfuoF7Pp6lBQNaScYZ15Qa1DCB1mVZpnFKCaCUQAxkpTQEmGLDtCCElmkGnmdiVOUJ1OLs9KgV+MNWEAaha1uubSJMGr5rmqZGiBpguY7qk8W2bWpatushlJdlmXBUbJjW2HXtCpja9CwXK2TGaSSlQghwyCGEFsGAYCkFgJgahta6KAql2HUmk+M4mEiICETsMo1Ja6CBYZDr8xRC+HYmcp1JUS8uayUkAEC/5T5Uk3BrEOJtGgT8LnkZOJZz/V81c4JSqiVvNpvNZtOgdGtra2dnh7FquVzqq6yH+pCuj6EurvWx1UuQml5QgwpaSKWBhhADCDSod7XrKmeMGQTHm00tkmyFzdls9tlnn3meN+hvRUlMCHEtRykA0GVs5jWmgmpfRogVqyD8bvS/LsOX7EWgJYZSCC1lrRDJ83w+nxNCanyRMcY4v+qiLpmP18+jlCJXDA905Zeg1H9i73D9kOtG4XJhAS6xhBoEusQqwHeu28SoOxSFaztIJRFQGBsEYTWbzZJ0YxrmrTu3MMZxHH/66ae2aX380fs7Ozu+4yjNwoa/Xi+jaD1bwJIpxzAt0zAgbYcNrMHCnbbDFitKz7I/fvTe9Hz0+s1Ry/Vc3xt2O5zzyfnZyeHr9Wa9s7uVZcnXX32htYJIC96DUBuE2u3m7t7eD3/4w4vFKk/S4+PjdqMVuj7Lqvl0bhjW7u6ubZjt/qDd7882q+Pzz04vzgGC3YPbYRg2wnbF1Hq1AVB4ntfr9frDHcP4FiNjFcWnJyPGBNQQY9zoGo3Q397eWa9XRZkIIYQsp4uxfl72B504jtpdp8hyznkliyiNPLOLkYa4ZBUXyretQOmirJLBsHPn4ObP/vZvn7960e/8yKAoiSPWbnzvBz8ImgEhwLBpkmzyvJRSRukqDAbreGU5vtTq7v17tmcenbxptJvUovs3972g8ezFSy6ZF7hhp82EMCzqOA7GHQRgUeTjyfnJ2XGRp5IHCKHvf//7Fxfnz55+ixBYLBZQq3pPYRp2rzsYDreVAlEU5Xnx6PFHb9680YCuVimAcrWKp9OpBmo4HDYajS+/+mpv7+DG7ZsV4xqokhVAy5oeLwS/ODtPsk3Dczvd1gC12+227zrdbhsCVUcMFEXmOI3NZnN6dCy56vf7q/kijZPbt+6UJSuKojYBlFqneWZbztZ2k0tW2yEDdGk1mGVZFCde4OO6lTVoPeIwLjnn6BLZk/UUQjHFGBOKGl7Da3gvXjyLomi+nNUGNScnJ7fe2bFtF0IohLQoeXjvPoGo1+t8+umn4/EFAuDpN/G775q3b99Osvjw8PDboxduw6NWBQHmIvniydc7ezvUNUeT8TrffPT9T2az6Ww+QTkIg0YQBNE4siwbYzidjYMgIBRPp9Mg8LTWk8nEdd3RaDQYbN27d+/o6CRJkvc/fG8d7ewf7MznS4SQknq1Xsxny3v3HoRhK2w0i6JKkgxCSohRFqxgG0xJ7Y6gtSYEAwQxlrZNEOdISK117ZpcZ8RcmzYyVkEA6vw9pNXp6emLFy8++fjDNE09LwAAvHnzWnJu27ZlmFEUvfvuu2m8SZLk1q0bvu86juc4zo2bNzWA1HT/6t/9r3Wybac3UKIUQkymU1kkjoGkoGfHJ8eHb9q9vm1btlVt8lgCJJUqmQg8L4qivCyEwJ5vC0E3cbpYLK2hkaQ5hJgxtlgsbNseDodJkozH49u3b89ms6qqut1u7bxbLwgwxp1e9+joqF6EdzqdwWCgpARaOpaN2h3Pd4f9waDXn88mglc723svXryQUldVtb29PZ8t6slMSlkUhWHaeZ4bjtfr9RzPH0+m9V62nlNv3LhRlv3Xr19vNgmltNNuV25wuTBWKi2L0/OJYVm//OzXtomTdG243h/92bvRp7+qpss8T+/euXdy9KY73P693/ntf/Gv/sK0naKSOEritKTUsExbawg1LGSxd7BLDLOs2CLaxFkOCbVdxzDMdjMkECDNQ9+xDbqcjIvNcnfYN11CCOn1ekwobFhSMIWIaTnNZlMIITXE1KxLe5oXURSBYJimhdZaYUvqqip5UVRpUYbNluBcVJWQDCiNIbKIwTVPy6ymBxVFUVVVDRjUr5u+chOHV5aL9a3G8+vvX39dlxkhBCEkCALDMGopRG9np+4bEEJKoZoxUP/Eq5L3nQWQUrreB9VPWMMASqkaTkjTdDadUkobjUYduxD6AYe4Nga9Jj3Ux++6bh3uUBtaSynrc4rnXGtdezJyzYWSeVEVRYEJrCkXQohWo1EzuD/7/NM0Tff29g4ODo7PTuM4oYZRszI1gDXiUhtXY0zqYoy1BlfO09fV+vrvupUhGEKqgyBot9u1iMP3/aIo/LCBMWacB0Ejy7Kao3ANAFxCC995WwGEEATq7ef/h7fvtgyXlkqX37l2qkAI1SNHPWQY1AIACC7KUlsEYgQNgm3bJo6HnYwSSoVU0iDBYBgn+fnhEeaVeTY1e6Pbdw4M4p3FsWkF2w92Taedp79iZVFJ0u/v2haZjy/uPP4IWE9t2+ZEWyZ47+PHnWEbU+L7/rBhNDvDX3/+2SrexHn281//dPdg/52P3z09P1+uV9q2EMY7/WGn1fr+979vGMb6IrOp0bF6Igc0aGLqKiqCbi8TJB2v50m5u7+FtXaQfOdgeHR0ZCkWjU7XJgwPdkCeLmaTdDx6/ZWVl4Wq5Gg04UJJDebrxLBsADjDdBXFv//7f/jk6ZeHr54Tw9QARStWFWvJCQRgdJZ1mq3FfBrHXGuH88bOzt679z5Yr6Ojo6PhoPHo3o9PDzfzeeba0e1HH1A/iBSYlOzVYpVSmiDW7Q7CMPj26RE1umyzLAr13/zX/2fPbWYpv7gY+b7n+ZYTwIPbnaoq281dhMByNaG2scmWLavlNzyC/TAMF/PYoPZoMjs+ebNazR+/92i5nLteCyHwzetv3xy9PD09MabI913Gq513tvYf767E0g+8jYzLimVR9vTps3RtIgQMg/hHh3kROxbd2e0lSXw+HlGK7965pbVeTEYY01evXnW7/X7Q5VIYWCTRCiN+786Nbr/f63dOT0+JQTMhSFn2BwMCwOHhYVEUu8PWcrkUeYEpNSm8desGooaU2nCNKFnnZWYYxELIRy4AYDa+WCYpQkjKypGg1WoxzuNNorgQjBNCtGS8LDSso8wJIVAqBJUmEHieL4FO07QscylAVSpKpOd2hBAGYsk6gdB9+fzihz9EDdPH2ImS6M38/O6923s7+62wvb+79z/+y39hW7PtoTYgqbLSxvaTL540dwajeESUSQ3MaXVxcvw//Pl/79p2mZbRYvq9j7+/dbBzcnGOMWEQRrziZlnpEjDt9fx1Ho2ejBzT3kS5bQXxpjg/mw62hmo8w6b76vUbqZTTevH9j3/w5mRUsqcQQoJgWZYPHv7w8ePHk9k8TrJnx08SmQIL54SnMBOigiXt7W+Xk9lyNrENT0plW37FheCqqngQtqRWaVYQy2Y89kyAEGCMacUQxgiIIkvjOO602hDyp8++IYQ0Az9NU89BrVZjcXF8e3/4H/82kyJttp2//uu/+tM//dM7d/cWi8VideLNsOTrP/79j//Nv/z/UEoRIvl8ub29rYr1q9Gb3rDX2TpQppmkeRCEn79+cxZVCjhhy91EGaxSXqSiDYmpMQVCcmLQIGwdn55+++SZFHodrRAAjucaFuUYNDthUkTnq2V1hLTnWO0msSyDgtb+1qvR6XI+C/3gbHSmFZOswJL3fHdjW0Tb+1s7v/78U9ezDQEnJxer2Xwxm1vUOj09JcTIi8yyvSQrLTdYrNOT8YVlWb/7+7/7r//NX2Qs1SZYrC4aOszzFQAlrwQF4mDQ//DRo3gdXbw+EoaV5JkQlRvY9WoyyzO7YcTl4q9/+uw/+9M/ipbLKt74CP7P//S/54I5Er1+fRgtUoCx4rD1TrfjhX/113+zd+cOakCMccmUhJzaduAGrnRNP4QQWthqQ9MyciWBQS3TsG3gKqmajR1CkJQyMEIAwDezJEJeQTrCdO3A9DwPY8w5r0qhFSCQWpRYmFY5K9KyLEsEDYQyyaNOpxNvxt2d3TKO8/Wi1+spXqZxDBAyLLNglZAaW4a2qI8JAKAs86riaZoTwlzX9bzANO2aqG+aRg0e1NN/kqwhQrX7pL7KE6rrWcVYlucQQsuyoBBlVWFCOIcY2xgjCCS1COccAOVaXlmWnuNMxqN2u62VJIQApQnFgnHHcdbrtWEYjDHHcaDWd+/eRwjEURJv0tfl6yxOiqJAGgIA283GdDqHWguhvvn2WaPVWa83iNoKUtdzGWNFUQWe222GDoEQg4KoJM0cz81LpgVHGJdF4joO59wkMI83oefZlEzPzk5ePtdKGcgwkPEHP/m9v/7r//VUnZ2dXQAuhZDNsJWqolAlEgpogABEAEEIqWkCBIWQQiuIIECEiUtk1LZtg9TRnVgpgTEOw3C1XAKtWVnZplVmuUmoECLfJJZtKa01lwRhSACXQitNIAIAAKWV4BBjAABABEKpAeIaYgAkxLDWZyIiLlWahYGRhKgUEiBuQoQwwBJgLaBkFCDL0ABgLCWEyoYKSJVn2aDbidbLzt7eYjoyqDvs9ohSghqYGgRxjYCOk4gLsL09zPNytVp9+eWXVZHdPNilBrm4GH/99bdt2261WuOL8/V6WeRJK3CDwBdKvPf+h0kcMSbyPG+2W47nR1G0WC6/98FBu9tdJht/tcxZ1ex12r0uk+Lo6OjuzVumaU7Hk5s7e/du3uZZ8eKbJ4fPDg8ODt774PFoNEEEe03/3UFvHUVJXnDJLhbjF6eH680qitbEdpvDba21H7jUMvOiqtvGoiguzs+FlKZpF0UhFTBtr9vpNFstx/MXk9n2zrDT6d28eXd8cXZ2fuzYuNnweCXzTDi2aVse5+riYpJs4iAImj6KNouj41dSaAiVVAwT1es3yzJbR4tut90IPUKQbZs3buynWfzkyRNCsBDs9eHLDz5479atWwgaEEJKzcViPJ/PPc91XReittJunmeDfrcoMqn9XhGWF8nF6HQ6nbpOCCB1nVaWFnESNZshxsA06c7O1qvXp1G0qqqKi9J1XQAVAMig5sX5aLo3M017NJot5iuEMCFGnpdVpbTWGhhtI2yEw+1h75137knFnj39NklYEARVVc1mCwBAu91+/Pjx+avTRjPc2tqCEEZRtFwuhVKMMct0NAQS6LIsZ7PZfD7P0txxnPVmVVR5q9NeLBZh2Pze9753ej568uTZfLbkUiilAUBMijrUCkKoAZYKCKkrJnLGhRBcaQVRVlzNBJem9FhpqLU2TdtxHCk555xJUV+wGOdKqdryvU4r4JwTQuI41lpyzuvJxjRNxlhVMYTQ+++/73p2VRVJkhCCKDGAhoyx3d3dJNkcvTnTGvguMQyzKDOgRLpJ33DRabUbYWN7Z3j05kRy6bp+nKbbwy2T0GQTSyld37MMK62KX376K4ig5ft5xSaL5eloDDDKssywHULIycnZcrkWghmY7O7u/uiHPyaEfPrZl89fvIDY1BAJybP5Ks9z0zSLXHCmPDdshpXkQikGkYZaBZ4tpaiKVGNEDWhY2KwQoZgQYttu7SeIMYZaN5vNPM/r0CCMcbvXv3379mAw8DwvL6qw1dzfv/H11986jjUaTb7++uvd3e1nz55t7WzXw32el81WyJlQSlnYnI4nk8lk/8Ze6Afbg+F0MddKeJ7TbrcrIctCZFmuIUAYE0prT+jabN+2bWoa3W4XQ11xFgSBZ9tRFDHGkiSp8c8iSwfvPPQbwd/8zd9oXgaeO59OyjzNko3mzKak12n3Op1uM9ysI86K2XzFRXn/wd0sS4Rgk8nk+OxYSvmT3//d8dPn8/mSUIoQkbLSWmCMv/z1Z7//h38YNhrDXr8sGStKAMBoNNIKep7num5RVH4rjPNsulpwoC4uLhqNRrvZFEppJTzXBVKs1stbB/u3d7fDoDE6PtFCmtSajsZlkb1z87YmOM4yiEjJhFQKYYooEaJWElpCMQlATdoVUq/Xa4wxRhQA6LougsQ0bcOw6g4AAEAIJoQAcPnxnkwml0y+BjsAAQAASURBVMo6StFVgILWup5E33Y+qO1x8yohhIxHozAMl8vlcDgUjNUCSIwxvkoThjXmLES9a6gxfHAVY13Hm9V9ALqM65RXEBdRStXM/+uptFYuvD2k1qNzrTKglBqUKowwxkoJVvIkSTCGSsgwDIPAy5KUYFLwTCrY7QxraoLneYQgBIAbBLdu3UJAj0aj1WpV5UXNWBRCGIaRbGJWFphQQpCSosyLmnAthGAQCMmv95j1MquerYuiSOMMEuJ6nmVZddArxhASRQmyHRNqhZE2MFGKr9frL7/8siwLKWVZllIBx3Hrd8RxHABQUVZaS6VgzcGsTUdqpB9eOcxev1MIIAR1nVpJKa3XOtdz/29QDQDQGmhwbbn4HWzwn8QP4Fu36yd5+59v3w1ojdCVPSW8XABd60QulzgmqVMntNYkjTdVXrBKaEAglHme5Hnl+c0g8DZRdH5+QSByLLsVNmaj5bNnz+7fHN65dasoslEaLRYLCMTO9nA5n5Ss8sMm59VyueRc9reGnzx4GIZhvMmxbXthkwHgAeA2QkLIeDppB61eo7PZbHzD/eTRB7dv33767ZPjF28E4D/9u79xXRdT2h/swJQMAlsY4NWL14gaSZoWvIg2GwV0o9VIEVxfnN69e9e27dVqtVrOm81wb6/f6w1evXi5s7f75s2bvKw4U93BcHtrCxNy/Pq4KuXF6cX+7sHs9juj83G2KRoeVRA0g75hkjSqCOXtsG8Sd7lcFunJYDDgInddF2EHE7G71+/1/9iyzTzNhGBpvsRYS1X1B52zXx9TAoLQZ7xoNJvUtBEh7VaPS/3sxYvxaBFFsTo6fvH6BSZyb3+ogXj67IuizLQWmCjbwat1hTFuhB5CwPOs9XotpXAc6+R0nj9b37ixHyerwzev8jxttVqEYFaJmr+zWsWHhyfDwbZWGwAwpWar2UGQtMPd5XKZJBvXtQGUVVWt1+u8SJeL9WI56/V6AADGWN3Cf/HFZ4uz1d7+vu81irzK0qJAVVZU6/VmMBi0Om3bdrMsW62nm80mz6uqkvPZcZ7nt27dWa/XAKOw1VquN6ZtSC2CwMOGmaX5ZrNhrIIQAwQVJhBCLpTkFS0LDYCCAGCETeMafsQUQ4NohGqa0rXCWyNYay5KVtX70XpzcU3DXi6XtQc7Y4xSbBhGfbXFGB0fn4aNVk2/z/MUIeJ5gUHt87Mj33P2dnpCsDAIqqJMNzEQ3Avcw9fnm83mz/70z37/D39vPJpOx7OTk5Nok52ML6AGnuM4lh3nuQCg1evGSSaESNK1T6jperPFfDAYlFEihY436XqTzOarg/2dBw8e2AY9Pj4eTWevD4/SNHcCUymVFZXUACAcJ2VZyE571W23Pc+PVuvavoZS2m21NOCLaA0BtmzXNhAzke/6NUIOIaz99RBCjhdUVTXYGtq2naVJVVWG3X/n8XuDwaBajqlhvfv48c7ulpT8l7/6xZMnT773vY+//vrr9XIluRCMf/v1V+vlCkJ45/bdZ4eHYRjeu3Xwkz/4fQAABCperxqt9mwyzfIUQogIlkpxKRTQhOC8yjSCcZYihEpW1ZnUBa+q6XQ46FmuQ7NsvlyslyuMMedV6JgG1oNO0yJwsd4EgceV2tkaths+r1hZFK3AMxDIs/irLz87PT42zIYf2D/40SfffPON1lZ30F2u19TAlJiGYSGEGOOMCc5rhx+9FQYGwlWaP7x33zTtz7/6su6J3xydtDpd03U0JhLD4/PTk5OTXLB7d+8yzqXiiHNCUKPhdwLfMcjk/OyP/5v/+g9+73d//NHH0/Hk3q2b/9O/+tdFVn7x5On5+Xmz3W22OobjEcsxHNf1g3W0wdQAAFJKKUTUtAAiUkouqrp0EUIxJhhdMrWv9W81+RxCrZSqmNgkMQDAsixiUEQwvExaRkVVaq01BFIrJTUXnAnOpUiKJAiCPM8D1xMKOJa1u7t7dPiGlRUimFIqGK/yQkGg6i2+hhhjBXSdWQUhrJ9TsuqSq0+IEIIJrrWmNTtQK4AgggjXfEbOueD1OaiAFlJIrYSqd2SQM4YRUkoKISiGFGOBgJQyDMKKlQAAVlZllXf8jlSXdAettWUZgpW26RdF4VhmlsQHBwe8Kia+O8mS8XhU5HkYhq1miDCWvMrKAgiOCTEN6jl2zciRUgKl6+IdxzGG2rIsi1qUFvUes057NCgmGAoEoFIYAc+xfNfhrNKiIhR5nocQePbsCca0qirbtiEi2DCrkl2+uZRXDAqhIFSwzneQ17YQl+sGwzDUVbiDAIoSVNdmQkiR55cUEKXqJU5dzpVSCgKgEYB/v8D/vW7g77UCf28T8fZ/XfYNACKE4BX7EiF03ShcdVSqplHWGg1MoFKC+I4PAdlESckz2zJ6ndZ8GVGMqqo0TQMqXebF6HzM85Iz7bthnTa2u7ubxCvGK8uyGOcFF5Ojk+3tbcMkyLBbTbfV6zMJT0ZTDeEijSfRqm7BZKpWq/nJ0UmzEeab1MLG44/f2RvuJKtNtFy3g2ZjpzOaTr559qTT73EC9WL8+YtvhNLEMoFWFVHQsKBmVVkmUsVCeoHfbLeoYczybLmKOJdAQgCAY/smsatSUGpyxlqNsBk0vvzmay3kYjqbTUb/h//9/+7Rw8fjs/Ojo8N4nbdbzSxleVJYJm23W/3ebjPIi7TClO7uDZst13EcwVVVVaYNBluD+XzeCM3VKlsup4ZhQC22trYajYZpoBs3bmigEEJFUYxGk7RfLRdpvUfwvEBrPR6PLRt1e608jxsB1bBOqyNWYhgGMSgxDMN13fqE2T/YzrIky6M054NhQ+mK8SwvEqcyHewgDIDGWgHLdMaj2f7e7U6n1+tu2bZNiOG6PlQ0jiHCihrYMCzDMNI0v7i4+PbbJxDCXndg27bvuPVQcnh4KFNtmGYQBHEc10pfBZEQYrFYOZ6PIDk7HS/Xq7pyr/OESZUW5SraZEU5fvIsL8qq4kVR9IaDg4MDQoznL14sVguIkWWZWutKEIQQgJxxkTOOMdYIQwJd79LxhiuJiAEx1QAIJQkAWmvLshuNRiV4lmVVBTAlhBAphWEYlNL6wgoA2Gw28/l8MBiAK0KvFJfM56IoGo0GZxxoRIkNNLFt1zTt8cXIts0kiZuNxve+90m72frqiy+ePXnqena741CMKaWWa/X6nVarhSlJSxEtV4wJ03V8z18v1kmWbtKs0Wl9/dW3UZwIjRrNZpJVAZPNbu/gxm3Ttra2djzP+/4nn9zY3/vmm29+9otfTyYT03bCVo9xkWW5UAATQ2NomW6WZfPZptPsNcO25CLP0jxPKfHaLd+xCUK65IwY0DSgY1LH9auq4kIqIbM8SZIEIeQ5tuV40/ky2awxxjdv0jwr8jw/PT2loji4dXtnZ6/f6756/eLg5o0g8AAAf/zHf/z68GUURQcHe1WZAyXjhBV51vZ9yzR8398d9k3bWm9izsrNajmaziqmpEYAUg0RMQwTYtO0sEQYgc1mAwmuu7Qyz7I0Rgj5vuvlLpMiSZIkS+v1dFN7ZyfHrVb48MG9r7/+Ks+zvb299x49WK1WaZyMz06hknG0LpJ4fHa6ibL+lm/ZREp+cXFmO1an39nf38eYTKeLwG8Y1B5PJ1k2t23HcZyyLH3XW87mugsPdvdMy5nMZ2maSqAVBELJrMhLJowiz0bnp+MLAIBRVev1mlDkeY4ULF6tmi3/zs29j/+r/+KTjz4IXb/yw+nFpCx4WQmp4Nl0WSkkEdGYlkwtokRI6AbNxWqtMcB1uLxhWY6tNSyqCmGglFJSSykBgEoCrWs7/0tK4OWqG2PTNCHSEiFCiOu6dTpiva2vvRcvSW3oMtO1nggt01RSBq5X5oXf7c5ms7u37yzni/F4bJqmNCXjvG4aFMZKyrxkNesQXLn6KKXqJ6/n4PqQ6mIGAADgMhRKa13TAGtFw9vYQy2RuIQupARSaiG15ABQwyAEeYQgADUrq7oOebbTDhuKs6IoBCubDR8hdHx8bBCshEjT9PMvfu27ttba81xK8CZax3EchmGrFd6+ecvA6Pj0DEONIcBQ51m6jladdldKWY/jQogoiqSUDd+1LMfApJA68H0NYJEXJWdaayClhMqkpNHw251WlkZlHpdFIpVsBo0oikzTTNK402lhaqyimAsmlWZc1vZxGF86PWkp37ZC0kpBjK7Lf125McYIXFqh1PyJ6/tflvYroeNvsCJ/8z7wLbeGv9c0XAdYaK2v/RxrMqXWumZBQqAJITWFFFzlRkgp64ahfseF4PVJKiQnRcGBxFLqNM2Bxs1GKITKsyLZLMOw5fkehiiKVgZGrWaz024jsIEQ9vvd0UUoWNVstdZpDBAKO123EWZZFucFdV1ETWxbgev96otPJ5NJkiS7W9th0OBSZEWutS7LcrNaP3jw4PF77y2j9cuXL8uyPLhz6/OXT2/cf5By0e50EMGGa79+8qTRav/2D394MZ6ejcYSaCKoZCU0cdAJLUTiLC6rvCwqxw8YF68Oj6qi2NvZjaLYpNbewb7W+pOPP+x2u7/4xc9czw589/T09OzsrNfp3Lv7EEP81RdfSA4W03mn1frhj77fbTams3FVVWGjJYFGCH7zzddFmTmOM5/PlZCO4ziO8967j/OinC9mhJCLs7MkSYMgmM8n5bPnEOrFcgohXC2jLOd59goC0zS8Gwe37ty9RQxSVrFUqmT8VmcoJCvLfLmcZ1mhFBBCRVHs+7FSsNttO77z4uXTO3cPgoZlO8bFRHb7nuNiw6AGRZblGYbFmazZNDU41usOtra2jo6OPvvsUyVgVVWmaQ4G/cFgEAZeVVXL5boq5Ucff/CjH/1ICJGnybXnySJeJUkynU6TJCuKwnV9wzIJNgihs+mSM7labtZRTAhxHA9Bs8hLjKzpbMW5Wq7i6eyz3nDLcZxG0+n1h4ZhHh2fAAAtx3IcJ89zUYoavWSMoaKowTcAQB3YWlUVV/Kax8s5D8NWTUoaDAYlZ0dHR0VRmLZVnxWEkBpmJwTV6YVnZyfvv/+YEFQvIJRSCJKsSKqKVVW9ocCEGBhT32vevfPwf/npv2k2HV7ySK5Xi9W7999p/6S1Pdw+PjoFeprmZZwmaVYUJTMMy/G9n/zkJ0qp06Pjl89fnZyduZYTBCEAYBOnRcX2dg9KzuaLdV6Uecnfe+8hlyqZLnf39nvdj1ut5pvjo7OLUdhqY2oIBZI0KyoGMSUYMSkl0xRaeVaen1+EzaDXbiEEKcWsTJFnIag67ZAQNJ0v0jTVhFIIOBd5XuR5JqWsqqqquEGxUDpdR+vlgnM+GAyqqnr6/Fl9BW8Y+MXLVxfnZw8e3PvX/+Yv0k307uNHJ2enf/j7f0AoKvOi19159913R2fnZ2dn44sTzwuINlq+u5nPbde5uBhRCD3XtqghJGeVEFpLoE3bsmxCLdMgPsKAmoZtWpbjSM6FQalpOJatEczLwjRNx/OEUkWWaS3LspzP52mafvjhh9Pp5Pj0ZGtrSwiRpmldbMqyRAA2m83hcNjrd7yw2e931tGCGijP809/9evdg/1OpzeeTBREAABKTMt0bMfp9/uGYVTr1Wq1slzvxetXZ6Ox63l5FFVV1Wy1AMYlZ/WUVVPeKKWbKuNASS5FFGnFtwadjz58/5OPPnz3nYd/9zd/++c/+2d5ns8m80ePHr85vYjWMTQMO3RLBd6cjxZp+fWr16vlehVtLMejhqUglEJjjBHAlWBlngMM6gkcQmwYJsGw1tTVTp3wirpbZ/kYhiExopQatoUoEVrV1DmEkNAKaFCPhhBCpZVGEGBkQANIhRCimLCyzONE7t/Y2dpezhcIQCUkvBpJCUQK41yIS4UipfSq3iilGOcQY6l17YfKa1mBUtTA5IrGWFWVUhJRQi2zPioopdBKSwkwQrVXoAQQaUwgwSbFSEqptaSERlEUhkGnFXa73TxPLdNcIaSkaPheGDa01vU6nyCgBEs38dnpiW3bCELLMsLANwh2bQsoXaSRlpwAWZWZlFJyZhCkNQVaAa0uBRcSqNoAVkOCMQJYVBmlVDCxWq0AAARCrbVBoWVYge912g2L6GS9iOO0qqSUkvHarElYtgkRtiyjDmfnohKCIwwRwhBhCAHCWF1GsaDLdlADjC+BIAgBpYRSAqTinNdg4dt4Ut2uqUtrxd+wS/qu+bhqIdRbWa/g7dUD+I0H6toGHl5SLCGCdbMCYb2AQkpdSiqEENSiCmohWZ6nZZlrLSEkSinyxaffNpvNRrM96ARcSjMvkiQbjUamSS0Du65JCbGo4TpGt91ot9sAhBgh07Ecz23wZhA2o9XStO3Dw0PDMi3H6QwGrU43LauT8Uxr/dnX3zx//hxqXTGxt7OLIfSCcAcbi9k8q9g6SddJGsUbYJqmYVQYvXhz1u5144Ld6g2oRf2GdzoZ3bxzY2t3mPN8vp5KAIsKEqoNigzT5DnfJLESkiJqYlMKaVne7ta+SY04jrXCWZI7nm1Rw7XsVrM5mUzanT2p+uenx2kctcLm+48/mI4nx28Od7aG+/sHW/2BEExyhQDUUnUHvYODgzhaxUlUVcV8PjWpMRz2McZpGo9Gk81mM+gPu80ewVavu3V0enx6cd4ftKlpE0LsihPD8rCbpXyTZLPlaqfaKxg7Pb/QWG1v94tcaKCzlMWbUivabg0s0zMNL45Tx264rquhWizH/WHY64fPnn3rN9AObnMuy5KVBcdEASAqllesuHFwqyiy6XRMKb1xc38dLT/99S+77V6j0WiErmUZGEPB1WQ8++rLJwjqQX/HdRoIg06zhTGez6dJkuA+hYhAiGuQEQAUhs1WqzWZzE7PLggh29v7fqN1cT5K4tx13SjKgyCI1hshBDU9gLiScBNnfsDjTUEMwQUwTNe2HIyIFEWZV5YFBZOs5AhUylB1o1CP/pcbO6GRRoQQqKAQIssy06S1XKo+86+I8UpK2W63i6KwLCvPc4zxaDR6ezaqJzCt4Hy2lELXa2lEsZLQtrwffP/Hf/fLv8bQ6HWaUbT65c8+bTU6P/rRj6zH7nCwv1rE0Tq23Ubgt6SIkiSTCgrBGl5jd3e3Ktj4YlSlRZnllWHnadEK27u7e6PRJE3mg8GOYzqKgTeTYyHE/v7+6dnF8+fPT09PWVE0Go1Gs5NlmdIo7Aw1gGcXY5bHjWZLMthsNjUQZZlnOcFItTuNslgzXpyfHg/6W77vT8ez2WTuuZUCIE94TdHQWiMNHMfxPM/3/bPTY9v1Oq7b73bOz8/Pzmc3b2zfuHGjZRtMKMaFBmg+W55POTWeS1nZlnHn1m2KyXQ0llIe7O/ev3fn+fPni/G03+/fuXOz0wxLyYskogSFQSMKEqaSdZIlaVkwKTWmpmkhDKBWXFBKNQQaAmqaZZnbjtNsNjEEleCDbg9TZFnWaDRSWlDToqa1juJ1FM/XGwWgAvDVm5PNekUQ1thgsgj9ABmm6Xq+6wIslWbzxfjdd9+ZzVb//qd/02i179x+ECfFsxcv0jT1Ah9CXKsndnZ2tGu/OT72uh3HdcfzmdBKSNnqdSsm0qKozfUAABhAirAW0u90AghWy7nW/IPHH/7oBx/fu3Wz1+n89O9++tN//9PDw8Nuu68xma+jjMmU8Tyv+v2+1jit8nS+nK4ixoRUwHQQwBgDLHTFuSREagG0ALZrYYwFVlpDQghGuJ4wr6uFlEIpBaFGCGkgFdA1ks+lgBBKXSv7QJ1LCQBQWmulpJRSSQW0FhJjnCXpztbWfDoLPP/09HRveyf0gzhLiywHCEKtJRcMMaUUlwJJDDECCGJ6KUTSWl3/XC2B1EpDACAECGoIAIIQIQwJBbo+jLqfAwhqBQCCQEOAoKqLGpNaKgh1zXtI4kgpSUm70wofPXpYVZXn2oJXaZpYlmma3VpgWVa5ZRDByqqqCMGddivLMkox1Mqx7MDzgRIIoeV82g7Mpu+UrXA0nk4mE6WRF4SDQWs8nderE0opohghZFsmIQQj6vt+WZaKy6IoJBd+4FFComhFfRtoxaqMFTnQwiCQYmhZRlUVjuMEDS8vC4xxxbjv+/W7XDCGsUIQIEQ0BForCC49ITAEUkkpZN3PXTYKGtROl0qLazihHniuJZEY4zpUWv+9Yq90DdhcpztcchquQIW3GwX4ViqEUgoCqIGSEiqF9ZVuokYULr8G8FpiijCQklesFEJgAiklCAEyGW0Ws2z/Jr5x40ZgEssqkiTBJyfb2wPLsnzXc22bYmJZBsYAYwAhqZNnCSGmbTEpqGlEm9Xd+/dWUfSrz7/kUty8fVcqcHJ2KqU0GxbTsszyo7NTxliv03UsC1uG0/Abio+Ws8+ffWs5tmGZF+NJlmWdrYMkSTKmciYark0Ny7DN0/OT8FUjiiKtK4yISbRnEdsARZHbXoAQyZM0TdOIJUWcdVrtTrsPAUAIVxV7/epNf9D+8vMvmu0WBroqUlZmNw92ORO1I+TNGzf+0Y9+bBn00cMHN/a3OefjiwulhOe5q+VsNptXFRP80i9svV4P+713331nuVzPZovlcu06Xhh2PvnwEyXB7u7ecjM7PNQff/S9oOFGUXR4+KYsROC3CXY500qB5XK5juZxHDPWtizHc8OyzDHirtPC1COYmqaNkZXlpVJgNpspxBfLiQSWApuL8ZtOt4swIcRhlVrMN1laJOlmE2WNoNXuNLVWjmMrxYsilZL1+k3Xsra2er1en1BUlqVt2K7rep43n83yvHzy5Fmv39rdGu7s7NTeTT//68+m89l8Po/jVAghhSqZLEvGKpHnpet4BFOCTYRInrEsXTl2U0OqQBnFWRAEEIjDN+e2be/v3XlzdFYUxXy+EgqlORMiz/NSFBxgA0kAhRZMQA0lF1prqCAAACuopNZMaiwRJBjiOI6zLKul7cQ0EEKO49RhcQjBirO9vb3j42PbdhFCjUYjiqI67b6+4mRZVu8Ib926lWWZadqUmhhjKXVZltvbu//4j/703/27fzdPFqZJi5xVpRCVenN4MpstFssNAvirL59+9um3q1WUpjkhhEg+HAwGveH+9tb+cPvi9PzibDSfzizDJYicHJ5IDaCCnulqAV+/fN3tdaIkgqfnr1+9QAj4rocwOjo99dyg0Qw73b7puElaKAUgJs2w3Ww0m60QKEYxEjwzMOy0wmbDefn8xeuXL1hZvfPoPVGJxXStWogQY1JsLMsiCHPBtNbENOoP6mC4fXpydHFxUU8MBAMp5fn5ee/B3fv3H7733ntff/OFFzRuWTnE+OnT57ZleI579/at403EymqziQJ35w9/7/ezdTSbzZIoun37Vthum6b56y++nI5H6/U6zcqyzPOSZRVXGlMppQJFUZVlHgSBKArLMn3fT/LMtUwIYcUrAHXYbtm2qbWm8zkmBjFdSKzxfLXJvl6tN37YOB1NlWCV1EmWhZ4rAbb9xibOok1qOa7m+WrN4yS7eeM2hHhra4tgA0JsUKsoqs0mMSxba5Vl2Xq9DoIgzZLxcr5TFR/91g+PJuM3x0eQ0CjeQICzOCGNhoYk28SUmg3XU0qlrPQ8F1nGVm/449/5cb/T/PmvfjE6O4VA2Z59885tzhSxYVyWitBWf6ucj72whTE23cAwDCZUWZZaA6G1S0xCCOeyYqwOF6+9gyilGAGtYW2MI6XQGtY2RFdwMaq1D5xz0zbraa/GV8Bv2uNcF4wr8RuEGmipCEK15xJC6OL0zDGtOhknyXNqGlprcWWgVJNP0ZVeH77l8HPdZ1+iBRASQrTmdXo1QqjW5dYR0vWRCCGu2Xn1P6nCZZkjAJEHIdJVVfqOuz3sd7vt+3fv/fuf/vV6BQnBZZbbtgEhzNM4DLagNi3Lms/nlGCNoG3bGILA8w1CBSsxxgCYeZ4vl0vjzsHB/t5gMLBtu3r5Kk6KMk9LVkENMaaXeg2ICCEKwDzPiSK9bldrvVgseVXV5nJxHEGoLcuCUC8WM8FygyIAlO1QCi1Kze3t7bJkCAFKcbTZaFRyXmkgKSVKKS6U1hJAjBBSXKnv0jU1AN8FZ7y9fYDfubNrhBBGCCGMBMcYI6000grommwAtH675CulgNIAXL7vAACof0OK+TaicIVAAKkUBKqWp14vLxBCCNWh2BARWHtr1pzo+vAwgSa2anIrgYo++fbZ0ZvT2bur3qDr+E5Nb2m324Qiz3Fdy5RC8CrbxNCgsGLpYNhbLBYV50qpKIoMg4xH08lsRqmJKVEIL1fRYrXMivLGjRvvfe99CPHp6SnPy8l0jjEtLLssS9/3d/YOzkcXF/O5Ajot8hcvXwZhwzEHlm3fuv2QC5CX7MHW8H52/1//xb9CSGGD1qaeeVGaBPu2qXk1WSyEEJvVRkvUcAIJdBynFxfjD957//bt25v16he/VDt721VVnZ4cua4bBE6SrO/duxN4jThOkk1sWcZ7773Xabcg1IwVRZ5MZ2POy+GgRymWCiZJMpvNDYO6TZ8xtl5viqKqWcoH+zcGfTabrqJ1XJbcNJ2HDx/GcXTz5s3+oDudTpbL1fPx67LQDx9sW6Z3cTGeTuemBXd3dyFAX3311aPbj+M4yrIEIkVMixCKoEmp6ftmmqbr9bI7DE2T5nnKhPA8i/FUa+iaju/bWmshVkWhINTdbrvVChEi+we7Sqlos5Cq+uCD91w72Nvbc2y3LBln2vO8ZrP13nvvb9bR+++/Px5fpEl+dHSEMDRN+sEHH5jS//zLL2azGcbUMIzhcGh7HgSYsZISI8uyb755Em2Sen+52SSlKtI0pZSmadHtDDFF5xeTu3ce2I5/fHx8cXHBJKtXqlVVKaUwdKAGGCIEINEQaQCV1krVKBgEQAoguRCYE4QRQgBrCGEttTBsq1bY13sKQjDn3HEcKWXNVPACv1Y2Y4x93693k5TSIAgNwxiNJmXJPM9rNpuGYZRlGYatmzfuVuVfCiGbYXsTjaPVBiFiGM5yuV4tI6VgsimHw+0H9x/NZovpdGoRpIR89u2TzTq6e+vu40ePb+3fOj+bLKYLLdSbs9Pbt+9yJous8BuB1npvd//k+LTKi7Kq2q1WbYNvWywvi64xUAqcn11kRamUosRkgjdb4b17dxAQ69X84mwNFXAcK2z4jdA/PJz6/oZiA2OqBACaEGzm+dI0TUxJVVVFmWutr51rTcvpmuZwOFwul8v1qqp4HMfVrZtHJ8cP799bLJa7u7vHR6/W682dW/uDwWA8Hr/z4P6dO3cwRC+ePvv1rz/9kz/5k63BMEvSyWSipFzN55JVVVUEre6vv/pGQoIQdhwHUikkRJhggype1WKwa/dczjly7KIspWCmQYMgqAhOksSwTNs2q0roouQAlFy0esNWOzw8PNze3tZMxElm2y7ExAtCprSA0PUa7VarLMuy4gCqMAw/+eSTrOBZlm02G9O0Wq2WZVmMlfVSX2vptZvNXud8Mv7m22+/+uZrTOn7H75zfHxcFEWdG4QAz4uy2Wi1ghBjzEMPQ5BlSc7Lw5PjX3/6819/+gubEKj1h+9/RCg9H51vbe1xCRTCiKDe1rYXNtfrdZImpmkLIQQTAIBm2A7DkBCDMZFlWS6y2jpsuVwihGoOIwBQK1hLEK6pCVdhSKSqqrJSdWrz29D0d2Pib5rr1Ytw0zSzLOu0WpPReG9nN8/zLE3Pzs4ePnw4nU6TPMMQQUKv/YKuq8J18wGu2BL6LbHD5WYdIYhI3QrUjULNVIjj2Pf97yrZ1f5CCOFbDq8YE1wqbhDDtezt7e3Hjx+XeWZZ1vHxsW1bt2/clGallFoulwSbW1tb9Qp1Mh512i0AgBJcG7TZbEIt0zjabDZCCMZKQsi//bf/9sMPP7xz7/67777T7Q0mi+Xp2ejk7KLRbBkmJYQKIYQU9UWjqgrlqN3d3Yqx+XzBObdtCyE0n893t4fNZgiB2kQLXmSB55gW9l0vKmQQeIPB4Msvv4yiqNPplWU5mc8NagF0CR7UrRpCECHKJVNKXfkw/Ub9vm7p3l4TXLYUGNe24pc7C6hqBOd65/BWt6CBvmwUfqMX+Ae3tx9RP+Sqobz8JF0dPIAQa1hrUpTQef1eYwwNw7AoMS1KKSbLcnPn8cOf//zn8Wd5s9l8cPcepXg9WRn30cHejmng1XLq2RRQvVodWTRtuO7h0888z9sZtlYrkJfFeDx+9uY1Maht2zdu3vbDxnQ67fQ7727veJ5XVgBTr9keGH0aRdEqL9qDrXu7OxY11utlt9+bTyePHz3SWnf+/2z9V49kWZ4niB199TVt5tpDZWRWypJd1VXdPdM709gZjliC5GBflnzhVyD22xAgAYJskAPs7GBJDsjpaVXVpbIqdWboCNdu2uzKow8fjrtX7gztoZCI9MowN7v3nv//J4PoxbOncQDK+Vw7yUA4OTzuhHZ3FI/79PTN16PJbrc3qitpFUqCblOBxVSi0louR2EvSiMhOEMkZGQ6PfnsC/nDH/7w/Y/e2T0YvXz50hgDYXe5nB/vP2ia5psvvjk+PFytVs7ocr2kGKYhdc4sti0F5O37756enBzvvjVF06yDvvj1L2W5Fc6tL087lCFpXnzx9c7OXoTIo3v3Pv74dz/9yQ/bVhjI834Ew3hy+NbVqsZxR8NwvH/ver66ODtdbjvHh/fm86cvXrzMk16/P3z88PFu93iw0399/urp0yeehk/zTp7HSZgDbeKM7R8fnZ4/BwRBjBabmbXy4e7BarVqqwXHAcJwd7+bd5Pr6wULbcOXBAaPHzxczVdivX60t/f2wYEOZdtwgDAmoCiqi8uzPM8fPNyfXoOXrz/N8zQMwySMyrJdr6qz08X+g+PuZHh6etrJe9ra58+fd/LeeLxzcnZxuZjPNxsAsQJguyn9o5YlUXeQNm0Lqd1Uy8lk53t/9H0A4fnl5XS5aKQwxnHeaGshhFKqvXH/+vr6Jz/5o5NXr6ezS8ZYEFASUkyR1EpqbZHFNEARg1GAWeAAUNLUTWtREXOBgaOEBBD3k/TZsyf3Hz4Ypum404HKOKVDRIpt7SyL07QoKmN0lmXT2VW3l1X1+uh49/z8XEggVYAFnExGs9llng3u33v8248/7WSyk6dSNG2zvTx78c1nv98ZdAed0RdffOM643ZVu0q7ykIE804edRJYw9dfv37z5avDw+MffO/7jw/v/fq3Hx/vTupimYTBeHfn2bNn//Qv/iLuRqOdzvn5JY0ZDLAmkHPZ3ZnEcdzUHFrZG/RoVdV1qWVbrWfboverX0+Rc8AI0TYBgbMofHDvaLta74zP63oLoOgM4sMHw/V6UW5tnKSbzSpN006nBzGq6xaTFiK2XC6TJEqSLBsMTq4uSMxcCPJub13MPv283tkbXl5fNk2V90YkjIpaRuno7Ozk6+cn//pf/quy3LK89+rLJ3/197/Ms2C2WLzzzjvRoNs0zTefPr+cXpMwGXR7z1+8Udo5GHCh06RLAyLrKghwr38wv5geHR0xQE6evD7cO+Cci1qFYdDrjKuyBQBZgEeTPcYYQk1ZVHnOwiS9vpqdnZ31+32t7HpdYkzns2Wexsv1artdjybDTbm+9/DhB/c+Wv2Hv5pezx8+fPv+cfb1V89+9fe/YkGUsiyiCSKkPxjHaSq1KMoyJuHR0duL1fLvf/7bLB8GQVCV7Ycf/eiXv/ylw+zJyzdRFB3eOyaDbpAkaZrmnbjcbC+RXczm//e//H9874P3v/v+D148+cZZ++bl6bDff3h0aI0zkg8TUpZ1wzUPgjyMUxYmSYIQWq23ZVlSDOqmdM5ZJ7M8woRJKauqjnHoz5tiW8ZpJoWO06wVSkljLADYEkIMVo5ilqbDbrY/6my326qqtFIYUWuMc04ZraWOoogQopTWVvvV2VoLMEGIcKmHO7sF57zlgNKC8/l60xuNoyxvm6ZpmpA4ay1wgLGQ0uBW4WQxBAhCiolTEkJIgMNOO6WgMSGNO0lU13VV1ISQkEVWGtMKamFCgghRpZTTLgAYIQQBdBBRAp2DnU4HQAuMxdAe39s/Pj50Vr54+SzN4of37j979uLkzSXGmCAKTWSd+eKzz9977z2KCQJItAJjpLRRXG3Xa9FyilkcRkKIo919pVTZbk6vry0hewcHvX5mgOFt3dQFgJZYSSCkAXGOSK0tAITFhWvCQWLrzbRY9AY9AMDl5eWD43uj0YgSJHgdBmkUUICtNJDQII6Y4K7YNmman19cnZ+fUkYoRQj7akcPFSCjnTYOIwchYowAALSzEDrKoLZaae0AwRgjdAPZeBclQliI2jllrIWIYEYdJkpZEkVWKWCtgxY5BwBkACinrVWQBs5YP8AhhCB0wELnjLWWYEZJ4M0jhFBjjBDSGgMppgjB2+nNOQSchQgTipzTXhwTRAmEWAgNLM6TGCOYxklTVt3xcDQYEgwJAGBnZ2d/f3+xWOj5/NH9B5Rij+r8+Cc/ULJdr5EQAhN0sH+ktNhuC62Ntc4XOhsH4jR/9913n798dXp2UdXt8YOHGGMH0Ww2Oz09XRd1URQIoULrothMhqNBvzcajSJG37x+PhmNHz66PxoN0jTdrBcf//ZXsw2nIU3SMAjjvBOnnWwP7fzoRz/6u5//Pee8qWpK4zwPGY3aRiqlFtOZXyNWq+VyudzZ2Xnr8cOdnZ08z50zb968cc5FUaSUMMbkeb4pV0KJVrTL7UYobowpqjJL4kYKimHe6dR1jSl59M7bBqJayNmrGca05TbLsjCCRS0wxl8/f9rfnVSq/fmv/35TbL+XfLe/twMAwKHdVFNta+tibeo4IQ8eHkDAe91ESXVxccYYeeedtzGky8X65cvne3t79x7tHh0cQGjXmyWluN/t0CBYr5fGuNVmPZ3Bi6sXs8VZnEOIYBzHV1fTqqritIOMbFo12Tkcj3NrUbEuX7165QxoqgI6FBA6GY8RAtfX17xVWToYDfcCls6vl5988sl8fnV4sFuUK8ZIFEWj/iiKMghh0zTXV4skSZS2s9Vqs9lcXE3rVrXKF6vD44ePHtx/9NVX3/wP/8O/Kyv1zjv39/Z2RqOJsXaz2WCM8zyP0jSKoi+//NJfghgDaxm6DWIris1kd3xwcFDX5XqzCEMGkYuiyHkGFEHsgLvti3IQEUq91kprXdeaQNDvdfI8c1Z1u12vb/BYqF9ujPPB74m7EWBLznlVVd1ut6oq788OgqBtWy+EfvTo0T//5//1YjmdXV/FCdtui1ev3gCHjLEY27ZtfdwChHAymUipIW9vSvwoiZJYcrVarb765ututx/H8ZPnz/JOt5tn0+l0NB7/7Gc/+/2Xn3c6napq5vO511J4sxbnXEkTRVGSJEmSdDod7zt/8uSJcy4gJI1oQAnLUoiJBWgwGIx39qSUytxItBhj1aYNIoYTZLS+vrpwFiZJAh24urhkjDhjlZSvXrwstmtKEEHYan12dpHn6fX1dHo9M1bfP76ndffs5DWC5L33PuCt+B//3b//8KMPfvzjP67L5vDw8NNPfjlfLt9+++08z/M89/EVnPN33313vamqmnNp06w7Ge9uy8ba0oPPXllpjBJC+Mg/hKAx2H8XHmHySLvUWmjFlaTWsjCg2ljnuBRlU/d7nSSKsbMAwuF45Iydzq6jKNrd2f/e974XsQRCfHV5NZ1ON5t1twf7ozFAcFuWxpgwYo7bsiw3dUEp5YJ7dZiUcrPZIMLW63VZN4yxvb29nZ2dJEmllOfn5/DSMMY2m01TV5NB/yc/+UkSsPV8tl6ufKIwwcxCG4ahUsYYB7AeDfp5nkttjDFlWSpeKcmVZBBYB5BVimAcMRIGlBKkW4GsacuNMwZZoySvSyelHI4njeBWC0whQwQ4hSCOgzhJEiFE0zRt2xrNjTFed4gxuYMZbg8eH653A/7fUQDer1+WZb/fz/N8s16729pMzvnd//2Wor7ZdD1icYdb3DEREHpPIHXO+d4+AAAhxOOF8Nap6wl4IUSvEwFAIHCYuCgMwjAWQsxms9VqNZ1el2VprcbERVGQRDFj4atXL87Pz322sXXOWkspUdAQCKfTqXPOH5MAAGVN2sm39aqum+l8bpxzFlZtU9W1tRZhbG87KQhhwS3OH6WhRzd9FYUxpthum6aJokjwhhAyHO4FFF9enZZlCbrdbbX1jzVfAcUYo0HY6/Vms4V1SBmnlFUaQIAJpZQwJf/wkd6BB3/Y77/1r+wfYpX/gDp4GMb+IYnhJvECAOycs9CZbxVUAmDBDctwE5QJ4B/inCGEGCPg/oBF3SEN4FuxGd9mIgghAaGMMQhuFDN3YBIxzi7XK2ttmqbW2ovrq16e5d3OdL7grSzKrXPQAWylSnY70+mVarmUClSN1lpqk8TZcDIejifGouls8fr0zDgUxpFn1IRWQqjRaNTN8vl8utUGQQCMlW1z9vpys17uTEZaiN/97mME3f379//7//7/8B/+7u+qqpwvZ9q1cRIEIXaVzbL0+Ph4uymFkISmDDOnjbU2DILhcJDnuf/9d3Z2/ujHP3zvvfecc5988juf5b67O+n1ev5ms9bmo06cJ9YYFGDRmqosZut5Pnj0aO9tp818MX1zcVaW5dH+watXr16+fLG7s7O7O5FmFXcGR0dHo8N7ELnPPvv0yZuXw1G/app8pzOvZ4t2qrUqimLQ7StT16375LNX6+Wq0+lA5yAyEFkuakzgxcm5US7Pu9tiPV9M337/HoQuS0PBAwh0y8vVanV5MW24btt2NO5FUTSZTCDmQjqjNHa45dYBgbBtGmEt6PUG9+7BF+pNVZVa6mq7IRjHQZilYafT8RdrWZaCn1elOHl1JoQYj/uLxQJhu7+/2zTNx68+Vspp5bKs8/g7HwJCDIRCKA1wkOTSgMVqu1iuwyS//+BxbzBI8ot7Dx8JIUaTie9FBAiWVQkcquvaQkAIFUIAYMMwvHVJaS+Tccrs7IwJQ0kahgnr9Tt1XYdJWDcNwIBihizQwDkELHDQaqUQAMDDYlK0AcVhOB5P+pQATNxqvWWM3N6QmHOpsd5sNuPx8NvArOdQ67r2IS1BEEihtdZhGK5Wq6Ojo06n8+bV1b37OwiSTz75VNQCIcJobIxLkkRK1bZtvzuQUsq60tYgAMMgTvojZwxv2rJpV9s3w+GQBeF4dwcA4AD43g9+CBD++uuvy7L078SrKIIgapqmrmsptDHG5+fc2duyLBNNo5QqrW4RqKqqruurq6vxeNztD+I4JiyklBpjsiwzxmx4AwCw2kFrozDKkohgRhF0wPCmgi5cLaaU4jAIOlma5+nFs1fROOt2+3neff369SpbP3781uOHj8uyXiwWWkitxGAw6ve7YRjVdTMejwljvV7PN1nM53PnHOfNd7//g6vr2bPnr42Tx8cHg8G4ePpSSO4PCXgT+mv8xKCUattG6xhjjDDw/hcfDoEDginxykccMGYsQBABHKUJDlh30Pfnaz7sM0K5FkoaANC9e/fTKGsaXpX1aNzjnC/XG351FkZxmMTOmYuLCy6EAxYA6IcqrY22Vim12Wy41Lu7u2A688b35XIphBJCnJ+f7+7vdDBO804YBG3bfP3114cHe0f3HvCm9R++708PgsBBDDGmdR1SpJUsNpu2bY0xeZoc7O2WZQkQFkJY1UKLYRRkUdxJojcvnkcMhgRlSdTrdjp5ksQZImS+XCYUpnl/d393OOpTSgEGGGPfjOXnKmNu2G5KqSfgEUKEuD8cKrdJDHeyBnNX2iRVmqa+KslbJwAAd7oH/zPOOUTw3bV6d954igFC6O8jPyh4AdBdzImvKvBKPf8fVEq1bWvVJooCQjFFkDEmuLq+ntXVJorCtm2N0QhBIbiUQrRNt9sd70xOT99UVeUpQkppEkc+Xun8/Nx3QmIMwzA0xoRhaCxsmta6jdEOYKKUUspARCwEzhghJYAYYIQRdRAYayGEl5eXm83GP5p8c1tICSGktVZJYYxhSRTHcV37VqrQOcc591ORtRYgXPveSAB9jjJCALibQerb3NDdl3KnJvn2YPdt2QGE6D/7Af/Ec84B6wD4g7DROQOAAzfJzdA551OcMcYQ3TonAQDAQQgIIc5ahL7FYdyaNoE19n+e7+T/Ad+WV90xIzfDVlUXFxemFZxSiixarlZKKWfhcr35/WefI6CTJGIU1U1p3pwUm+3Zm5dN07Ag6Ha73W4/6/YQprxqWRjSIAy1swCuVpuyqX1B+KN7j3d2dsIgIAhXRV2V9ZMnT87PT188f2qMQgi+9957H33/o9/99te//u2vjo+PHz4+3G63mOkgYUkeleX27Ox0uVwZ7bbbUiigJCQ4QjiMozQ/OFzABcbYWDXZGe3t7b399tuEoBcvXlxcXCAEwjAkhKRZzAKy2a4Wi4WNwWQy6WSpUmq+nF3OrwwwNAqXqxUl5IvPPn/9+jUwdjTeGe3thFkCJEQo0o5whcK8/+jgSFu1FrxpShBHxUJcXV5/8fzTOA4hMoSQ/f4IArzZdi4uLtbLzf37D5Mw2a63zuI87SRJNJvNri+XH374XsQiY+D8+qrltRAcYMAwXi1n52dXJ6eXvFWEsDAiDx4eIpxvquuq2m6L7bA/CFjeNBJCTVm8WtdSnTIW0iAsipIgPJsvtBTQgdG4Lw0/OTmJoww4BlzDW+OtAcPhcDHXccIePHhwenr69Rdfl2WrpE3TfO/eo/K03W63AOE0TVkQbDbbuuFV1eztxZuqXBUFIuyf/MVfKGOvr6+DhYvjUBmbZZlPU19tirIsfDO9T9r3Xi9/CYZJbJF69fpZWW7DiHa7Hc5rC7RUrUPYQugghgBZX1LirObcaIcQQsCbjmAQ4jSNIdBZHrWioRRjAv3PKKUgBuv12t1qyAkhcRz7RDa/YzVNE4aRD6oDALRtHcfxzs5O+7j57kffN1q+fPmyrWqtAIyx1bDT6fFG1nUppdxu18V6EwVhkiSDXhglMSOBioXWer3e1E27s78XJ9nF1eWjt9/5/g9/+Nd//3e+DdLv4v4Z6h86YRhq1QghiqK483k755IkljePFQgxtc4UdSukWqyLKIqO7j3odDp1XSspev1Bt9v93dcvAQBhGPZ6Y0qCtm0pgUeH94uimM6uopA1tUmTxBjFKBwOelcIrdfrzz/70hg3mUxWy83nn3/xvQ8/7HW7T58+H/a6EIAvv/zye9/73qA/urw6Hw76LAzbtv3Lv/xLzvmLV6/6w3FRFGEUvPveO7/5+HfrbfPW2++sVouyXFurrQV3p4V/xNxVC1JKfd6Xd7JUVaW1RkxY6yhjXtOgraE0yPMuDYPNaiGdGfVGwGpuFGaUpnGadoqiFFwBU5VlvVovV6vF+cWbJM0JAUFIKEWtFOtia5z1HYZBEEDeSqmsn2C0adv2Ljxgu91uq/LRo/yuaWI4mjjnltfXXz17zuvmR9//3uMH97u9QRgn1tqm4UqpgEUIEcZCoJXVqqnK7XrjR8/xoHtwdG+xWAAAWi6KwteEdkaTnSzL9gfdLMuU0kIZSgOpTBTF2rjDg4kxRltjnF3NrxverFar9XpJwvSuY4lg6C9say0A5qbi0loPnvlDwtx2Mt2g0w54HyPFxNcH+LnNYwOEEIscvPVkOuc08OH/3mTvS5sQQTfTw53j/2628Aq4u9X51mUJfTQqxjgM4zRNoihgBGd5lKZJy2sppdbq+vqaiwYTiAkEDkEEojjs9UebzQpT0u33NsW2LisHAcQoJOFmswqiCDsXRfFwOGzbVigJIdbaNlwHkcXUWgcxYWEYQkKU1tYCabSRADjl1UtcNAsHgEN+1NBadzqdfidvmgYhJKW8vLy0epimqVKq3G435crfuU1b+Wg7iAljTAjl7QwYAwA99WCk1gEj7laLcHPSf1uLAP+gG/32wWyMcQA5CDFGhBBEqBe3+igI/yFrrY3WANGbeQLau7gGCB0hHjlwEN6MFd6V6gxGCEBrnf0DeuGcIxj7L8gPiHcvpRQwGsGbb9P/IUKItELEaRrE0ezqmhAy6PW1MbPlIkui84vL+/eOhVCbTSV4c3Z6Ya2FEDuApDRCWePAYrm6/urJi5cvpdHrddnt9br94Xa7VQamWa/bG0zGBwTT9aaoamEs2m5Wy+U6DBkXNbB6td3mve73f/JHypl//+//3V//X/8vg/3xYNDr9XoHRwdJms8W89lsaQ0KgqST953DBIebdb0p5oyFYRC3ZeOVq4wxreWrVy8wxrPZ9d7ezng8NlYZY9I0jaJosZhPp9cNaliADg728jxP0xQRMp9Ov/766z/72Z90Op26bRDBddt+8fVXo/5IKTXpD8uyRoidnl87SncP9nFAdg4fWKC04WHW0QgAgt9+/HCxvCaEyIb7DNFOp3f/+NG7776/mq0/vvrddrPFIJgc7x0f329r3jRNFif37z9eLtdNU0nFsyzud7ohhdba9XIFAXNQXZ1PIxawGEqtAtKJAycle/To7fOLi/Pzy/3DURJnm00VRTAIEm2XvW43DEOnlORCKHV5eam1btu2qUtnSZr0er0e53y5XJ6eniJsCUFv3ryZz+f9/hjGJM+73rc2nU4RpgDsOgvbtgEAHR4ertfbsm7H43Gn08WIrjZrraV/ykgp2rbVWgKA2rY2RjHGEIYQASONUgogyBiBkGIK15vFRVPFcUwYVIZLLSAHBhhjjHXQIgQQgdAZba2GTmNgDSLAV6k5oJRuW15xUVNKMUZNW4Hb0DFrHKN0u93Cb7mDfAqpt0sopdbrNQBwPNpxzlVVNRj1OecPHjzI0ywM44uzVdsoSiJnKyWt0y4IoqbiSimlRBiyLYIaQkSoMm652hCMMSLAuazbKco6DKNtWYwmuz/+45+SMHrx6k1AKeecMZJl2V3Abb/fJ4RAgJ1z/l21LffFNpQgSpC11C9q1lpntUX4erHudXQr9SSM8k6XUgqBS+Jo0O8ghJwDFKOAId5aKVqtRJpEZHcnjFjbbEXbWKeBNfPp9ePHj7Ms46IRos2yrKlLrdV777138uZVliXW2mdPv5lOr/b2dnZ2xwDaOIRBFDMWrFaroig4BwBYY9V8Pr1//55SQgiglHjx4lUrFAuCzbpxziEElFKUYg9xSyn39nbTNPa5mf7I8fyLUOtut5emqV+Cq6pCTUMIo5S2QiwWC2e0MyrP0zRJ5vPZW8PJ61dnEEIMUVEUq9XCAUsoTLPwrXfeDsL4+avXRbnqdLoQo+VyHbn0blHzHJYVVgmhtc7zPM2ypmlq3voTDgBQtyIMY6Pd1XTWSjAYT/Jur274wfE9aF1VlL5DiFLNGIPgJh4YIXSn5qvrerFYrFarIAggwh6W9xRMGIbO6jSJTk7P25ZHSRaGcV1XV1fXy/UKINQ0TVEVYRJOJpM4okYnq5L7qCWMsXE3FmIpJcbUx5YDcNP17J/sflDw1z9CKEpTf2x3845HGjqdjnNuvV4rpYIgcMCAW1jeWmvADQ0RBfQ/W4U9Gwhu46jdbdAvhNBLjG8wDGP85xwEgTGmqgpCACYAWNQ0zlrV8rqqqn6/J4SgJIgj6IuIgIPWOKGUQ7ARXGillFquV9pqCMBgMKAB8zAAi8LuoG9Xq9Vq5SDl0lpeI0IZY5hQjDGLoiAMpdRcSeOsc04bKbUQSkjFrTZJnGVZxjBjjPmlvq7rTp52Oh2lhNa6LNuqqjjn1gIPjXjFtBcopmmKseBCy1YoZa3DEGCMMYLQWuMbce9ohTuJIsbYC7Tv2ByEUJIkQRBwqYwvqkL2bqf6//tyztx2RflHnLlRJqIbsABChxB0wHiTJETo7ln4bZDgVkwJ7/7ZXzbGKuccRMhn2VlrpZRKUZJ3O0VVCs4tBFEUSa2YYXmnZ7SMomR3b//q8myxWCMEmlZZp8fdPIwzf5UUVV1ezl6fvHlzdspoKKQGmEiphbQsiLK0E7Do/HpZ1/V0Ot1u18AZ7YixkkISZb3FYrqq269fvop+0xlMJv/mv/vfSYh//dtfLZbb42M3Gu9VpVjMys2qFUJkaT+NB1GUKKXP8bVSpirr9WLZ7w6MUVpLhIAQrVLC9443bVVWweXl5Xq9stY8ePDAWss553NOCDnY29/dGR9998Pjw/2/+U9/+8UXX3yRd9569Mga0O8OOlm33Gy11sY4Xlda8TQKi/n8xYtXm6rpj0dvv/8dbZWx4tHjXhAAYNWjh8fz+TWCDtVNXddJkhZFpYUrt2q5aIqNOth/2M07xbYVXI/Hu7u7uxGjm80mYIxShqFua6GUGQyGvbwHjHMIAos2y2qebnv9LM06R3v3pOLbqv3uRz9l9MnzF1Mtg3vvvesvOK01gsHjhw8OD3YZxUaK1XLx+vXLSRQgSEXqpHBSOM55WZZNUzRNMxx1x+Nx0zTQwqOjB9bAyWQ37WabFU0ipqQBTgUsDBgJgohRiKBx2mrJRVs5CzeL2ezyvNuN4iQkDHPebDYba20QRzuDnfl8rq2xliotIQJREg0GfYzIbHnZtq1zpsMSQgIDVdqNAQAMUi6VFNJqhBhEEGlnjTEMpt7s65zWRlkjq2pbVuvL87MoihxQ8/nMOWOts8ZBiCmlRVH4G0BrbYx2twlohBAf9Q0h2t3Z9w84KSXn/NGjRzvjyfPnL05OTsui3p1Mut0hcEio1lmgpBFCrFarTqfDhYEQ0jAwzm43JYIwixPnIKUaYmQRxoT883/xvzi+f/+Xv/4NCyOrmqIoMG59Aa7fAofDYdM0HgoeDAbW2jhOvJQnYNhaGyhFaAAhbKXUDhoLst4AInQ9WwyHQ0qpT49o63LU60ZRtFisivWi3xt0krgsy6uz0ygKJpMJoTiPo6urq24v3RkO37x5lUzYvfvfEUKcvHkpVZumUbeXD4adJ0/K3b3R7OqaBbjbyy6vzkaD4Q9+8F0EdNnU3W7XODudXn/2xReboliv11988cVovPPBB+9pB4aj3pMnklKitXDOCSEoxW3bQhgSQpS+idKKokAIAaCNoghjuN1uAQCdMB73BrvDcZzlUVkvpovr66uqqPuDAQKg2Gwvzk4RsPfvHaVpenFx/pQmQoiDg4OAkrZtGSNvv/1wd3f09PmL9WaBMG3bUmvZtBVlYRiym8YB4BgTyhh/ytatODg4iNMsCEOfQjGfz4UQUkpizHK9XqzWGNFeLwYAbDYbkHc+eu/d7WothArD0B8egkvCaFNWVdNqexO93DTtaluVjVgsFlGS+rW1bVtxPSenF4yxDx7fE0peXl4iStO8m2Rpsa3qtlHGEAgBggDf1gNihCmh1HoUWghhzU0vMwDAi+QRQl6s4A9vxhilxDc3+rMtYjelDH4UY4xFYQgh9HSGtRZT/J/hzLfn/Y0M3lrrzE0/5B2GYW8dgP7s9MwIuIlgv0HO/KyQJVGapmkaRwElBEPkELYIDnxPI0bYWsi5woiKVhBc1UohjKWUTdNYCJQ1np5TWiOEhJJhGPqHmNZ6vlqGYbdpZV3XXOggCqMoitMUEay1VdYYZ40x9q4+EUGrtNGac66U6nf6YRheX11dnE57vZ4x0aDfy7KEN9XzF99Mp9M4DIMg96csxjiOY+ecBTDP8/39znK1vZrO1+tCa40RRIRgQgSXAH2LaLhdV9C3Xnf8AoTQPwSkNloZAwBExhjjEZ5b6sHeCQggsErfALSeuHDOAXhXJ62dQxgTCKxzxptvMULAmf9sagF/yPX6Q6anLw41VgGjMYLAGo83cM4DRkiaxrPZDGPc7/d7vd78elo1TSfL1qvVtqyfP39ptGi4VEp081RKcXJ+cbMlKMUYs8AhRPb3Dpfr1agzGQxGQqrlchVEodS6rOvpdGmMms8XQrbdbs6SQBtOooiLelk0xZMXwjpH6XsfvLuzs9Od7Ny//53r6+vVqlmvWwc209l2uaq3203IyjhOTOYAQJ0sHw9HWtv5fK4kuB2iHaU4DEOMcd2UWuvVajWfz/w4X5ZbCN39+8ez7fTq9OzzTz5pq+qdx+/s7Oz0OllbVi+ePw9IkMdJWZa9tCtrURU1tA6GaNTPlptyNOhqRKwBxqLzszkOWJJEQiqtyHa5XM+f7O/tJEkEEQwoC1l+XdVff/EUoVfIIeCivd0HzthPPv7d9XRxuL/XyXta8tlsdv/wPoHIIGOsKLe11cCXDmtpKUsEt+VGAIAhzOyQiVauVvLNq3lZWWfi8/PVwUGzf3RorW6bDQAs7w7f/+C7GMEoJC+fPQUAfv38ay54ng2TSXc2XW+WhceXMMa9Xq/f719eXkZRhBCazxZBEGEMneIhgcCCXpZ0u93tar6YnWNMGQ2Vc69ffBOG8dHRvSgk1oi2BXEc+/tQSi6EMMClaeo3Ff+lpGnaHfQnkwmltJYrqRrGQhaSYX+QZVme59Pp9PXJidRKa20gCkBACAEOWOAQQBBAhJy1WkoOnQDQEgIXy3kURd1e30dNG62ttZ7v94OCd3MZAxC6saRTSn3/HudcSulzspVuKaW7u7t1ml9dXUOApdQQUIIt5wI46NxNnfz19HJ3dzdOU865A9BBBADAhCHCpJRt1eR5DjEihIRx+vzlq6+ffIMpydJeXdda38RI+zo7rXWWZQAA3ykHIazrxhOc1XbBpfbnASQkQCSEkIVRyCgEtqya+XKNoet2uwhYhFCvE2dZBoxlGPW6HSnVatFaazF0bV1Zqx89uCd5FcfR8eF+U21G437eScKwF0c/zfIkieLr68vPPv8dY7g/6CYR+fCDd8KIbTYbTBxlEANGlbTWjsdjSkkr+KeffzGdz968efXpZ7///g++iwj7/ItvMAFxGm23Zb/f55zfZNVDGMcxFwAAUFVVFAVSyjBiw+EQIeAf1qOM9HuDTpp3u4NRd9hUbV1VVhtkXRbGW6Fk02IMRcujIOzmnaYWy+V8NBpheHM6pllsnWYBevr0G6lMbziK4mCxWiNMJ5PJzZ7kc4Vvl2CqbdM0rZAIY2stpbSu6yAIjo6ONk3dcM45D5OYt/XF5ZVs+eOHD5S227Ju25bQIIqRKkouBQOwbITf8oMojqJIaGut0wDTKNUW1FxDCB1iDoiyFbqsj8pBmnfnm1WW5tP54vxqVlT1ZrMJolhKhQgeDEeUEa51WwpPAcBbjRuCntXx3RDkhjwm1I8O6KZT4CaOzBPqfiPknJes9FSgVsrXo/iPxeI/hP1hjBG8cef7Y8yPIL6G+G77vL2zbkgHdNu65G41leDWbIkxRthwsYVIEpw4gDyyYrRumqapecSipuFRFPd7uaMUo7AsiyzLhBBcCK8ehQRDr5yAoOFtHMdVXZ+enfV6PUIphNhaoJSRykCkAVKQcmpp1TYWOH/eOucMuPkFBRdRFGllN5tNnuSelPSizrqugdMQujQOPbVhlLq4uLgRP2ZxnCSUUsKCo6MjYxwXmrEtxtjYm0wkY40xBoJbRSiEEEB3aze9+wy/rWT8A0Nxu+jfsTw3MMC35AUQQgeMb3Jyt8pIAD1mYKxFDhiIMAI+AdMzEH/4fu9gDGtv4pzsrYjVv0OMMYFIcetHEGutNMoPlIRz3uv1Op3OYrGCEAZxpLg8PT0FAHz11Ve///g377//LoJgOp3meQ4AfPjwIef87Px8s9kgRDAlFgBCWFU1k739o8Pj1WZ9fn4hhKjKxrl6szVpmgZhoq0jNMIYNkromistB5O95Wp6cnGVPnn2i9/+djjsQwgpiRlN+r1Jrztu22YxL+pKSOEwtGVZbzZF0zRxlN6/fz9N06apFk2ZprE3zft7Q8i2qqrj4+MgYJ4UGo0GHkfa3d0dHfa/+err5WLR1DWv6p/97E+6eWc8HK1X6/n1dNgfXGwu8yjr5Z26bqIwylM23hmuNpv+oBukvXXFjQL/8Itfj3b37t07atr63uGB0ezzz57Vhdqul3x9qbU9PDwWXHl6uJP34nG0WbWb9fL8/Oro6PD4+N6bNy8oxvfu3evlg7LcQgejMBNcXV2+urqaYUQtBMjB9bZoGo6Qyzvxer3ebFaff/Py17/6dP/wEKHw/Ozqr8U/HB8fctGu18tis0qT5P7xseT13u6OUq7T7e/t7f3DL3795erZeLSPURjH8cHBQZqG280yTlhVVfP5PCBBp9M5O71cLpdpwoDTUciSOHz86N7u7v7l2fnTr7/p9PqVKcuyni9WcZxkSZTn+bjfM8gyRgCC/X4/SZK6roU2Nzo1B706GgDQ6/UGgwHGeNyMmrammDhnBqP+wwcPjo+PP/3si5OzM+ectspBijGkYYABhBJBCRGECDltnNaaEhcEtNvLjVFKoU6ns1xswzDUivvHkw9GhbfabAgdRM5ve74mLo7jqqqLogiCIMsy7/jfbrfXl9fbTdm2YrlcZ3EmueRcRixgLAzDMAiC1XIZxzGm3DaN1AoxBAmmlAIEhRBpmjsIpJTL9fb/8x//v6vN2loLEEIa3E0wAIAwDJ0r3S2G7C2mfsPz1EMQBBBTf54J46SUAGGIVZZlaRxNxWVZlkq0jLE8zxljkIVpmkdR5CxECH311TfrdTEYdMbjUdvWLa//8Z//SVVtV+tFmsb7+/tZlkjZZFnEAvTq1YvhoB9FkWybpi2GqHv/wXEny54+fWqt6Q86//DLv79/+EhZpZSazmeE4O985zsWgqfPnpd1++mnn/7Lf/XfFFX7y1/+QmswmuwAAKJgWBSFUr6hOEjTFBPoSSIPzsdJOBwOMYaLxaJpmoRBYG1T1Z1Ob2eyK4XWQrZSRXGstNBK7YwnhCKKcEjZg3vHelHP54BgmmVZr5dB5IbD4XR6PRj0G863ZeX/3m43F1JXVaUBbNt2vd1sNlsaBHEcd7vdrAO11kJp51y329XOTqfz6+vrMAyllGEYGucW87kR/GBnEoXhdD47PT29ODvnTZNnXYSwEKKuW8Y0IhRDSKM0z/M4jquqarlkQdRyWbWtELXvGIzzbmCMUurN6VmUpNaBNO9oY+arFQvCx+98p6iq5XqltA5iZp0tioIQ0u/3HZR3gg/gkMfJpJRhGN8uiDdzsH/QV1Xp/8GvTAELlFJeGtLr9QAAbdNUVeUpOedcY9W3cWmMoD/17wYCa5GX2t8Mr14DT4gn+D1+cMe7/5dnUpYHUkqEbRhRSqk2CkCHEOx2+whuCWRFUTe1TGLTVK1WEMZ8MOhxzv19Ya314/Ww1w+CQCmVJImUfLPZ7OzsjEaj7VpjjIMwTJKEhj6cCgihtLUO+YxJezclIIRaa7MsaxtR13VZlhjjJEl2d3cRQk3TXF2eXV1F3/vog3feeSeO4xfPnsFbOxXnCGHMOYe4Lctyuy3Xm7KqKmstxgRBYgHwsxP69pzg/hCtePcC3zI+tK30E7MDN20RGNs7LMdaCxyw1jr/aRv9raHhf0ZJfFspAgH0MwAAANzqS8C3EAUIIcIY3cbX+jHU80oIA/8X3QhZjPIzJUGGEGcuTy7X6zU7JKrhSqkwoFkSKcHzXqdcLSRveVWcPXt6cHCQh6QTZSHaC4CeL5dOS4qwbOp+xqhpQ6Q7ASJOKK7q7fVquVEQLxeSBmEYRxVfsjBIsrAVXEmnpej1xwElr5+9ChiBXEEI9waT4+99JHm7nV0hDLLAJZO+tbm11monpVy0YjudvSrbTqdntI1IeHRw/8mTJ+OdSX/QaQTHCGV90ij76vQlhG406L589VTwllIMDBMY7R8cMcJWq9WvfvM73pr33/3gpz/7R7/6h19Op3NCSJ7HOwfDTppBCKqqClx7cLA3mIxZ1vnq5aun56+evT4J0yxRycvTZ9Aaq8pelnz4zuNBJ1fd7JSw4bB/cnKymM3vHR0RQtbLhdLxdrPhTR2lLIzo3t74m2++eH56SgghoRFAdEf9+WKltV0WzcefPesNeghSDDiJQ6VAvzs8u7iSroqiTrWVETPPvno1GAwORsfr1frJ8rmWYltsIDQf//zTw/HhZDL667/6t69fv6zL7U///MfHx/cPdwx1GFqXxqmQ8sWTpz//zRf/9b/4R3/853+S9PNu3oFSYthsl6uXL7621l5dTfvDgbMfFuVcmcrAZlvKvf1DgDmJu5QEV/PXNDr68EdvQQjPz89Xyy0NwiBMALSm4lpZTBhFyADnnKUBsbDdVlOEkAWAheFoNIIQOkylgwrSKO9LhyTAEAeMUsYYAQZCSCnecGGk4G01GuVZls2vz5Ww3aSLHGY4OD+9iOK03+9uywsDVBDRMOm8fnE6u1qPxj2nIUEBYVgbaaypVSVafrh/IISUklNMEIBNCdK4E+f9T3/z9S/+7tfr9SYOotVqrZQa9kdBEBjgOr0uAKA76F/NpgTiTp5YawFwcZ4ghLUDYbcTJNnu7u7O3j4hrNhWQdzRWkOICTUs73POrZZ+nUq7HcowpMgKWbbcLiVjDCARpwhjbFSgqkppAaBGAEDTOu0cDJAmvKr6nVDp1kJrEWR5hzEG27obkXgwbriYzWZZgh4/mqR5lufBeCcri/rVyevBeHR0/96rN+dZJ1dNuZkbrFUYkP3xEADgOOdVebR/gKwRLWeD4XA4qstSK4sgiZ3o9LuvT89UUZ3O5wcPHkmJW0GvFytA6W8+e/rRRx/9+E//0bNnzwaTISuDbtjF0JQ14FJlvZ5F6Gy22tkZW4wWZbMpClQU8Le/oQyX2y1CSBZtJ+/GcdyKF4yRvclgPu0sFqvxoPvrX/32vQ/eV8qcXpxri4ej47Y1qmn8kay1DoKoKrevNi8Ho+GoPzk/m+7vHJEwni9W3XSIKF1vCsJwHEcNr5WSFrokSSgJgFIsTvlqY6wd9Ear1coIe3B8iB0Bmm8W11nMZk7n3Y5F+Gq53t/d/fjLb6zS3U4nyDtSyoqL5WbNCK0lN8YkSTIcjw6ODj/+7e/jOIYOPn70+OLiqtxsnYOmMYDpPIodimez5cmrC2TJ1dkVpgRYG8RJWaxXy2XTtEEcUQQBZIN8QBgFFhjtNuuV74XyB6cnd24RfubPaEKgc1YpTgFqiwpCCJ2jAGmpCMajwdAY40kWAAALAo88W+dCFkgptQMEYeec1do5xwhBADhjIUQMEwOMNcZ58AA4CIHgbdM0zhoIAIIgyVKjlT9pjFabpjbG+EF8eXm9s7Pzw+//8P333xdCfPLJZ998803Tihq2yCFtZLfbMcaU1YJSigPpcHB+uYAQEuiMMZiEUkoojVBWG1wW5dffPOt0OlnWXy4KSum6mNGIBdByJS1EBFN1d+YRwgIaUGKc1kY6Y6GzxDnR1AggBOxqPdOGA4BYEpVl2bayVlA19s11YVlP4wHJDnXYSANbxChN55U0xiRJ9tWz07IsnQWUsjBCRVEhpBhjDmlCIWVMSqm0ITRwDnKpKaUBCSmAVtlWNEkYTsaDYb/X76VX12shoZQAUoec005DZAi1xmrr9M2YY4zVADrCEOXq5uxHwNbtZjzsFdt1GEfQWBYiijAw1mJKWAy8iNVizpXRLmQEI6q081pF41zokLXAwSCKIgAZFyDGjGrFK6606Hc6YRhvt0sYBMI6UhRbQki325Gc12XZtFWWpFFA1+slo8QYV5YcAdjp9wa9fhCGm/VWaLVer5frTcM5YyyMkywK4zY5PD7Kull/1IcMffrZJ2dnbzDGOOh6V6j2ueEIOgellHcCHMowolHACMTYKFUUxf7B3t7eTp6nELl+vycVb9v26dffpGk62dk72D++vLzebArrXBhHB7sHO7uT8/NzIURV1w7hwaDXH/Z8jiyELggC3mrBZdPoCrhZVToHBr2+F1crZZ4+ffry5assSXGSMBbOrqfPn70c9ruMMc7b9x8dZFk2Pz8/ffn6s2+eXC9WYRD0ul3JBW9qUTfVctHv5I/uHabpbrHVnW7XB//1ej0MUdM0mJIgjpqrK+fseGeijMGUvPfB+1VT14IPx6OdnT2h1M7B4XpVzBbr4c5ACtMb9q2DpycXEFIhtUFgUxRCqbzTKcoSANC07QcffDQcjV6+fFlW2zzP8jyt2vav/uZv9/fGeZ5+8OF3i3JDKDo6OgLKLK/m8+v5cr6s6vZidv3RR4/eeecd74DYnYybzfrevXtPqnKzKcqy9Pahs9NTTEhdVoNu3xiDICSIZiElLEAIUYy3i81g0smyhHO+XK4228oa0OkNx+OdV29eWwNoRAklLa+lauu6slaTMBoOhwcHB3XdaK032/Lk5OSrr74K47SPaZ5bL8tRSmltAQAIUgOUMZa3WgkFEQnChLIsiDJlnOTcAZqkEYQYQYIxUUZra6qm7ruOBc5oDaFxwBFGrTUQI0QwBcz36joIMEHaKC7apq39w9c455Vu/X7fo7hWaaUUhDgM46ZqhRDAoTxPRpNJlmXWACGUt8xpIa2yCNg4DD0xSahL47gsy+Vyfkvxurque71eEASU0k6nExBa1WVd13dKcs+m+b3QSyvu9FB325sXtyNoIUZCaWvtcDiM07SuawcBxpgwSikNGCGMYYwhdFWxFcWCMRaG4XB8GIbhbDbbbreiqQ8PDx0wTdMsFgtKqbZmuVweHx/3Bj1E8Lsfff/z5/92vi3NxWWY5mGc7B8enZ5frJfrqqo/+uCjqqiMMtv19vXVyWg0SuPw4vLs7Ozk4VuPO1l6cnJyeLhvGQ4ohchVVSVFq7Xqdbu8FQTXxpgcd/wS7JVx8/n88ePHTVVzKR7dfzCdTueLWRAEhARxlOb90Wx2/fLkhDf1wcHefpQlWZ51ehYADwmkWQwccp0syMOdnZ2dySSO0/l8Xtc1pTROM8EVhA7c8hcsIFrrsiyttVEUBYytVitob9QtZVkO+wNt7B0mfCeCSZKEEBKwyGPXXgOBILy8vLTW5HnucSPGGCFEShnHsef7vw01CyGyLAMQ0TAghGjjvLiMEKIt0Fp7vv9OSeAp87v13SdD38wQin9bbXC36Ptf8w76NrcsOMLwjmuw1sI7+gDe/AmC316Gb1L9/BvwONkdAuH//M486T+fo/v3mqZ5/upllCaT8e5wPO7P5+JqKrQyxtzGsSKIICTYQdS2rc8MxgBShge0p5RqqtoHNmCMjVFt2xJCAsrCMEySpG6FMYbSwH+ShNE0TZumsU5LaZWSykhjfQIEBgBIKYui6na7xpjT09PRaKKUWq1WSZx5V+Tz589nszkAoCxLb5XyvOFtNAX03ZUIYvStPk+Mcdu2UjkAldbGWIOwde6Gr2nbFgSUYBgFQcDI3s7kvffeCyieXq+CgELojLPaWWidMU4KraQxBhgDnbHAIAeA75UiRmMCMb5ti4YQ+BhIQoyDQisECXEQY6yUaVsBCIijEFoqRau1TaMwoAg4o3jrnLPWQKudYwha57Qx0iFACHG3ydOe/cFeCNM2FWPMAeOsjChiBKyWs16epWnqrC7LkhDU7/fzbtdBeDFdVFVVlmUrhXPYIRYl6WDUx5QmabxYL4aT4Qffe/9qefHFky+GwwENQgsgItRB6CywFrg7ISjBjJE4jjF2FCNrVFW3om6Hy1HN6+kCW6tFWzPGrDVH9x54s2mx3S6W67Is799/+P7777fKdDqdJInKuirKGjPa73fzPB8O+71uHkVBN8+UbHhTA+iAsYWWFxcXddlIqbWyVVkroQRXCLRJGAguN5uibVsEQK/XwZgYB7hQJ2/OPvnqm1dnF2HaiZMYaksJqrngbQ0UCZAb9D788IMPivXqV7/7rOEtpmQymXDO/a0IMXr4zuOq3O6OJ/P5HBA82t3pDgdhkiy31b1HWTFdTHZ3LKDLzZYF0XI7+1d/+idtI9dlXZV8ulpEYbzcbgBAWYq4Ev3esCgKQLAQelPVmFDlHI1jKZvz65kG7lGSdOLENvV0Oh31BqLi5+fnq/lKcimktNb+9Kc/PTzcn81mSoqQUVmVWZ4QQg739p8/f44ccNqdnZ1pZauyiljAheJla4QMwpg4GJAQWzS7vKrbhZSqbTglYH9vFEWJ0ma1ng4Hed3yrNOJ43ixmnPOAXLOgdVq9aMf/einP/3Z2cX5l198/fXXXxsAnzx5ksQZYUEQBJQwKTWXyjjfuguNBkojLqwUBpKIRT2AAxrkm81qvd4qS2kIACTOIaOh1K1PdNndG1lrm6ahFGLiVyznD+A7X7g/uZUSTcOFECyglHVkK9q2dRb69UsKDazPeDFSyjTrUCaEEFLr5XyxWW2VUoKr0WhUl9W1uwbAw7TIn/QGKJ/vtF6vJRdhGAIAlJBWGwghRRgD6NE/Y4xRN+qEO4URISQIAq/PhbfmeP/09zsis6Ysa6m1d4R7yRUicL1eIwSSJLLW7oyHlNLxsN80zTaAnprx0U8+FYcEjDGmtKhFfT2bDvsDzvl6vX7r7cfzusYYH48mUae3fv4KlvV+ZxTG6fT0jCK6mq9+/fNf/bN/9s9++sM/Qgh9HXz96/NfrWZXo/HO9z98/+p6JtsqzTIrW6CVBYZiGIZBFDKjeCt5UWwyDHw9x57Z999Lsd164P3o6N7l9BpBGIeB1Uo1DTJmvq2EEEGWKdFy6aSBq23z+0+/+NGPfhQlufPsLMJSqdevT3hbF5s1QRBCHAZ0NBiWdVW3XLSNg4hSCgDivLXWhmGojeScO2IIIUme9/v9tqqB/UMk9h157AdKxhiwLgxi/7H7JGOInFLKaM0YG/SHeZoul8uW1xAZAG3La+8F8IoZ/0qSxNsugkBjSiGEHjT2JzEWCt2mKd9p4jxxcHcl3IwUWvtksLsp4dtEwLcVi/bbL63vsPE7aNpa6+6UifgmjviWTb/5HDxN5m7LJvzVDm5bqj2iwBiruKhbLucL/PTZ6eXVdrtdbraAYGOt8Up9CJxXPBhjtbz9bAl0zquI6rrcKgGABdACYAEAYRiEYeCA5aL1ZSLW2iCIrLUFr7Q1YRgSgiChlBKMYStQUXIhWghhTAOvWvO9MOv1emdnT0pZVVUSZ97Bu9lslsuV14R2hxnDASEEWAesi4KQENY0DQKw2+kkSbJYLIpN2daN1cYobQxQ1iprtHHIGGV8baWJoogShKDzXGeSJPePD09ev2l5BYEhFBrpwxWgNcBYTUiIIEDQGme11dA5B7A1IGDeZQmcMxhDgIFDwDh9eHxfCCE4N8ZgCAlCFiKKML7du6y1BGFjXKs5Rg5B6IDRhhtrEKAOQGulUhLD8G6y9F6eNE2BNQQC2zSVMyHQwkGDEKDIiHrTOZgwijlXAFhtXNMKA7Za66uL67qunXNhHFBGlQNVq4JWqaZ5dXYyX84/+Oj9w3vHR8fH470dhADGFBGLEQUQOagdABYCgBEJGEI0zbMkizAE0OmmFUK03aSz2izPLxs/MhflhhFKCDk4OEijGEBgIGJRyLRSVtW81gYoLcKIaRtJrSCm3qcuBAQWMBbGcUzShI4nQUCdMYVWk/E+cM5ZuN0WTdn4eP+ry2lAaJYnbcOVUsW2Ag51uhkXajqdn5ycbVabgIZZkhYNn0/fHB4ct2VJIUyDABmHnBv1e4NO+vFnX52cvAbWTSaTIAg6vW5dlcvlcnc82W7tbLlYrJaffPl5tS2kVpM82xbV9XQxWy46gxFXerHe1I3QxhkHFLAOwZq3FgJAURpmxhhpGsJCErCD46P9o+OnT5/GnU6WJcvlcrZaGWcGg16jzZdPXzx5+Wq9WU464Dq+LpbbzWwVEEoxDcN4POhxzl+/PhGKG63SOOnFcb+TP378zu6BMwCy6JxSWhY3oicIMMVQSgkdilgAHIpokCVZXdYEA8iwEtAZ0+9EO3t7LZfnl1dBlNiVCkIcJyyoA3/3AgDGewfdfn++XDx98vy3v/9dXbfdbjdg0aYow1BBgCCiFjjgEHAIAiy4ktoABx0gDlASBJBEiy0XGha1Xm0bGirrEARUK1BVDQiMtmY6nzn4HmFUFRpTEgWhsVYoCSHElFDCEMYOAGV0QJm22rtmOp2MsbCtmvV6HUdpGIbAoTAkaZR6ATDGGEIShpFPAijK2lkbhqHfYIzyp/jNDsEYi1jAjfTPU0YoTUgcRgAASnzNouOcW2MAAFIKawxCCMKb6iD8LRLRN/fcsYngNlLNWptQom/PsE2xLcstQmg0GgVBEMUBY2w2m/G2bhvHGFOSI0q8/WxdbH0DBYCQUvr81UvOmygI/JsPopAGrD8czNdbGqDr+frP/qt/sqkFRISwMIzi9XqdZR3R8q02bVntTSbO2uC9D9R2+/Tp0+n5m4f3/3QyGnz+5ZdNuQFGCV475whFWRyMhoPd0eD8/PTy8jLsZtZZ46yUcrlc+rjAnckEY7yYT5E1YRStFtcI6IDCOCQn0/rs5NQAd7i3PxwMeVVK7S4uLliQBJTFcQwx6OVZIzgEOmCk2JZXZ+dVXRtjaBhJZaQULgiTJAMAcKnXm6U2zhi9LUvOOQ1hXdfsNmXBD2oIobZtCUReCOwX6CiKMET6ttQAOheG4XA4rKqqrZs4jtMsiaNosbDO2TRNunkHIUewkx6LDgLGWBRFcRy38ia/kkLgZw6/pn+b2/a16f7M9sbIOxWhV7f4xMyExeBbdse7/8IfWPNv9xFDqG9Vbx6y8gSE1hq6O8HdjTLR3cYE+evwTrLgh1ePbznnttutdz8Nh8MkSTSQcZYmSWIAvJrNV6uVcy7Pu5xzYIw1zgGgtRZKGuMQEgw7hIDWURyGAFgh2rquW14TmjnnOG8hhFmWZVm23WyWy3m3N8AYRXHAKDPGRZFRWvv7xc80CCGEASEIwoBSOumPlFL7+4ej0agsSw8IxXEcxzG8TZQKgsADP9ZaIYSz0Bjj6R5vDVBKlWXZ6XQHgwFjbDZdXFxceKDOAAgtMBYY46S2xhilDIQqyzJjlNEKBpRRMhkNhoP+V599Vm4Xoi0hpkZrZxwIAADAWWwNhIhQBCDUCFMIgDHKKg2s0kYboxxyxnIAE+1Eq/XBvf3VarWcLUyjnDPQIYxAQLBzWraat3UU0CiKoJWi5WkcIOi9EdZZA6BBQFljJZCMsDvdQxzHWRb1+/26LEjbltYoRiIBreQVwaC3O+h953EcB3VdS95A4JTS55cXxkEIYVk0SqkoihKWOuQ25WaxKa6Wq+l8WhQbGtLBzt6r03ON2Gj34OLiAlijlbXUIQi0ccA6bK11EGIQBmGURmEcOGuAdcwQGhAah8tiU1XV8cFhmuaboryer6ADeWcQhKkzVmqbdHOWRM6Bi6vLOE7DJEYIRFEAcRcgpJT2GhOCIETOSEEwSuKQMSKadiNVr9c72DsIw3g2XZyfnmrpJhObRGlTVozSo6MH1mpC8HZbUhoIbZbrbVFUlAQZQ8gCXtWy5MvpdV1VvU5Kkni9mH/y8W/3J8M8iZuyujw9V0phiPr9fhQEDBMj1cnJCaN4sd0WRbHZrDabzR//8R/v7O0hZ8uitgYVRVUUldFuuSjH4+FiseBSBSxq25YQttlsBoOhMYYhCjCaz+d//uf/5MGDB6/fnO7vHTrnWBApLaq2QiTQFm02a6VEURSyUL2swyAZDXe6WW6UxYTtHR++eX1uEXzw1n0lbbGtI0TPiqu6ElHaffjoUZZ35/P5crkyxjAaWmdpwBxEGJA0ybU2COE0TvZ39xbb1w8fPEjSfDabFVVjFA8YGQ27RVmLtvWsPOdSa9s2EkI42E2fPHvxN3/38/l8XpZ1nCRSGS5VnCQIYc9SGQctBNpZrYGUyhiHWUCD0DjHQiosen1+XQkrLNYAKwcsIJDcRMUlQYgxXi6XhBCMI2stxjiKIn9JuG9lIHpUEEDrnyOU4iAIwjDQQnoENY5SSqnWliLqnPN5cIzeTAlaWSkEBBhFJGTBdrvFEAaEQQiVlFIIZK3F2GuCCMIhCzzsCazFN5FqwN3F5BmDEaKENrfRqnfLn7ftecP6HR6YpqkXlCUB9f6xbrcrhHj2rC7LMss9n02tRU1TzWdXXgVmrd167yiEQgjtbKffg9b50lspZafTqarq2YsXRwf7o9Ho6uoKQrhZrV+/Ovnzf/oX3//+9z/5/Web1WJ+dTUZDI0xWqu3v/OOaZvzN2/evH65v7//pz/5wcOj3b/5278/ff5k9+hokKfrYkugWc2uCCFRHIooQM7u7u5QAtqmaprWO18YY6enp0KITif76U9/ooT827/9W9E2SvB+v3//6PD6+npydGgDFge41xsMB92Qkel2O+h0jw/vvXl1kiQRAKAoiv6gSyhdL5ZRFO2NR03bOq3WRSm2G6ENC6JOliZZ1gppy3K53iitAUCtFACAOOvUdY0g9FgLZIE/pDnkSRj5Q8J3kiGEKKHW+MNABZRGUbS3t3d+fm614bwpig2vq6JcEQp290bHh0dN01ycnVdVo5QKwxAR7HNCAQDb7VZIFQF3hyf5MGy/qfuT/u7qBTcuksiL9oUQ/kISQnSTmzDWuzSkO9Lqjob49stY821nP2PU/46St7c/fENheF4MoT/AWn5QuJtZza371A9YngiIeynnXGgTI5j3uhDTsiwbzjEmzjoLjXNQOyDNTayhArKqi06nk0S7jDGlhHMmyzJrTRgGlFIuGm2kVLyuy7ZtldEAEQAAFw1GNE1jbZwQoq4b7SzGMAipcRoAl+d5r985GB9IKR8+fOvy8tLjCkVRMMa8H8qPAoSQMIw8KKhNSxgjhDBCrLVpHCNICETrxZIi0ss7SZIM+/3VYuGHRamAhVZpa325M0TeA9HWNW9LAuGgmz+8f/zud97pZimCBmilFScAQOcAgMA6BADEmHOOKcEYAwQCihEGnDuueCdBUjplLKKIheFop4MCRSmWupa6VpZbJ7QxziotlZHaOnX//n2jOsVqDZwKQ8YQiALW8hpY32VtgTXOQmM1AgixGwOnR4z8TOaMJmHAthu13iyBFqN+NwrozqiXRmG32726upqvC6HMpqzX17OGCxpEmIYYUcRCi4hSalvy7XZrodPWKAVZmpycLxfrX1CK6xZyQViMACaYUEQw8dJTQomFLW9REvtpXWiOkQ3iMO91KCGrzbIoS0BomudRkrZCE0KCMLEOXc+uLy/O4zhOkshiZ5DlvCmKLeetttZaQGlUVRXnwWAwIAiGQWStU1ZrZZVqV/PFinNK2XZbLhfbq6vperFlNLx/75FR+uriXGvd6+RcNLJtiqJoW77eFISwlgsIIbTQGdtL84PJ/nq9BAELMGUIyqZ59vVXfxOQNI2vr6v5+SWldJPMgVB5nodhMOr2r6+vD4+P2rZdLpfr9ZL00Hcev922rWjlYrNWSpXrerFYioYbAbp5b9QfaW31PduUjZTy5aIpyEYKlYQ4z3Op3HgwXM6W15fX3iaktY7zFNMAQWosjMKM0bgs2s16M+xM3nn87sF4FwO8XW8AxL3h8Gqx2dYVxbEGoKlViYVsGysBb9uDBw8mh4dPnz4reFvMV0XTQoDTKHXOGQg0cK3gs/kSItLpdNazldo5CHssTzPnnFcJiLYxxmACy6oBiHV7Q2PBtqyF1C9evGiaZrlcsyDI8jwM4+VyXdX1gAVKGe1abQFCCCCMCAUAUGaRARgTFgYWWBYHXJvl2VmtFWKMxYkDSGoBgHXIGaWExkKri+sroRREblMWgNg4ixElUGG/ZFkIvD8IIWiMxhgTiqzTbVsrpYqiKIrCaJcleRynQjTrxawoSm8o3dk72pTVYjrzFXYEYdHyxWzezXNIKKaAYWylc0o6CJANedP6XMI4DjGEPuqHYqSUwgQyTLw02mfp+NP6bv+7g3y9fs0/6z25MBqNvCvVSMk5D0I62d3BGLeiOT15rbUOQ5bGSZxEdn/HOdc0jTOCYOzPjKZpfAt8GidpmjrjumkKgO32e9bas7Oz/qB3lCanF+cxDs5fvz47vwwwefjW26Iq21YoUccBFcJaBpGVz59/OR4ML05ezK9O/9v/5b/sPrrXFJuL6SxENiDW8DqiuGokZAqEVPBquZgNOulo0PvO22+9eXXSNI3WWgjF2CKO416nm0RxMhruTsaff/5501QfffDuwcHB6xdP04h97/De4+MJsI4gymterWfUqNFgnIZBxAIuRF2U2sg0y7TWANooYBgiMHC7Ozt5p9dKMV2uuFB5mrAwgBiXdWWdM8AZo/I839nZ8Z9/GIZW6Tvvu+ca7tT+8LZ5j7HQk00+oKbX6y2XyzSNy7LcbFYYOqVEGIZBQOM4DAL68vkL/1UGQeAgCMOw0+nUvD07OxNCYkZ9yrKfTiCE2gJ/KqPbWH5w61H0nJS9rSr29eJ3RMONI9TctAb7OKn/UqNg8Q3X4AEzAKj/fe/8dcbcOOhuORf8benDHbvhnOOce7K12+36CWa5XFa8shakadpJOzSkIWMtxEpJAjEwABgAgcMAU0QBAgCAgPrmaxCGjFIshIUIRFEohNjdnYSMXF1daa2bpgLADga9TVESAqwFUkpCbBhgY1VZbYUQiJBON+t0Mm1NUWy5aIoCllHpGb3ZbLbZbPr9flEUSZKA2yCpOI45523beoVQwKCHpT3QEkWRc7CN47ZpNqv1xdn5zs5OEsW7k52qqqSUQhmHrXHAJzL6K4fSgFLCW58tQR8+uD/q954/e3JxfppnyXxOrLMEYQeBscoYi4ljIUEIOaAB0CQMCEFCt9KuMEwIkQ4aTLACmoYm7dI4jjfNgusKYkkZQE47rRCwiLo0i/7kT76vuPrbv/6bpl6n0YgworWGzodPY+wcsNBoZ62F+GYcVEo11q7XawD0YNh3zhFCiNWyEnUnSQ73d41oeFVAxXdGg4BRgh1XRmsltZLaQmIJIcCCRsh2oVopylI00llrs043SqFz4M2bmXY2juMwDK1LCKYAIsaYN1J6B4iFVltLCCIMa6ebpqIExXGUZnHIsrpt61ZwJeu2hYh0uv08zabzJZe6aThlQZKlSolys1VaD7KeVNwYw4XQxgCM7h61t8iY8UlZbcvn8+XFap0meRp1OJfbbVmWFbBOKZslqTWuLCpgXVluMXSbdYEg6XaDIPQmbEwpSdIOxjjPumkYOGeA02nM8N6O0VxruVo262nJAIoIM62o3RYbR/rdLE5E3tFcxiyUUdJWpRby5NXr09PTfj4MgmBbldPpbLXc8Ir3OmE3yVTLCQ0iyh4e39tsNqKVjDHGGNAiz7sqVREJP//dp6vZ3CqbZdnr09fD8chBZ6QCDTBWQQykNHmQ59lgONjN0oFoRK8bGms3qzaOekLDpjZx1MnSQRzFB5PDeluct8vR/l4QBDgKoyx/8uTJ61cnSppS8oAEzhqpdSPV8zevZuvl/fv3CWSvXrx++uS5NHpnb/foXr/hgnO5WW+BQ1pZBOmD+48dIF9986Su1i1vkiQJwpZzXhRVf4AoY3ud7vV84ZxjNDTxzcOX0YAQgrpGCGGBQxQDg1kYGOhmy4XWikUB5qzm9Wqz1FIBqAHUymhtzGw+F0pC6NbbjYM6TqMsyzAlTmsLnPRCKoccAM4ZRqhztm3b1WoFIW6rtq5rgpnnMpVSWhslTauFs3C9adu25U0DAIjDIIoigjC89btjAAnGAWNW6SAIkihet23EAu9QhxAmSQKss1Zvt9sooCREAABnLAAOQgicY4z9AQcm5C5lrygKD0hYaz2xba1VSlVViTEWmp1dXISMKaUoYxjDnZ2dw4P9OA4f3DuYz+cXF2dv3rwxxkCWCCHmy4WS3G9Uo9GIEWyMCSPWkTLr5JPdnU63u7Ozs1wuwbrOMGXafP6rXx9Odj94+/GTp8+G777zzTffpFk4vH8AEZhereODncmk980333z2+4+jKHr74b13331nud7wr7k1KmZ4NNyp6jpOojyJjRSr5XyX7Q77fcHVq1evvLp5PrejwTAKwtOzk36n6yO8A0ac1REl3Swt1qsgaA/G46bkq9XScavKzaJqZFU5B0WUEEYZY1GYDAaDJEkgwVpyYE1ESW8wPL5/Txirvvzy+avX63UAMCIUB0EAMVLGcc57g763ODLGBoNBQKgS0hkbBUEcRhELCCEIwCiKQsqUUkZpA8mNmADjzWaTpulms8IIhRHTWmFMkiRGCK1WSwiMvekDk/6bFUoCAMIw9D2QQEj/jPbFPx63UMZ57d4dnuQPfq/4uxMf+B9QtxkJfkrw6bxeZ+fHjv9yUHDghsPmnGOMGUE+JgfcpgsDZ7x3zt5kRWt8m/57p/vxTIQP8rsbWfyRUy+aTqeTxklAGa95VVTAuizJtdYIGOsQhBAi5whFAAIAwhBFUeAv77ZtheRCCGcsY6TX6zFChGjTNLHWCswZY9CbIzDudHLGAiVNu92UZRkEAWU4y7Jer8clr5uiaVpr9aXB3W7Xx9H693+XGdU2DSEkyzLG2Gaz9eORNlrrxn+SHh2M4zSO4729PUpYFEVFUdR1G8cxpXSz2WClMMbOQgMsgNg565y21mIMMcZZGschOzw8xAj86pe/ePPmzWRnDwKrhcZBgADWSmutWcQQgQAYDRSlsDsK8jwRLU3mRq02QQAZAMLyphZFNZXWZCxIOwxiBoGStTCtxBBGWZjF2XivMx4lF+dXWlcAOmOF4RpYCADAADPMlHXOQaudAxBiAm5TOOu6Pjs7m89J3sn63Q5ZLGYQuk6nM+zmURScXJ1q0QyHw6oulqvNcluWrawaWTTSQAQwtq7lXDRN00qhtXYA+GLTxXKTdXLOpdS63+9jHDSNYizwhBYAN/CUM0AoZazCGGNGCSFSSM65xiAICMa45hUimFLaNM3MrhbrDXLIGEcIkVIr4zALCCFluZVSRnFAKaaUBiH1J4S/E7xghCBsjHFGA+skkWVZLhYLZYCUerstt9vter2RXFDMrLVBECRJVpZl0zRFUfQ6HYwpAIhQSghFiCCHkjjtdbp13a6Wy8GgFwS0LjcsQP3eYRSSJIlW6yUaRtbdLBwQIi0Vr2oQmmG39+rVi8Fo6IwhELVV/fvffjydTn/8o58e7B2IVjRlVWw2GOCDnYM8y8ptkWWduqw6WV5ui16WcyGyOPngne9uiqosS6v0cr4Y9oZlXTdVTRFVQgPkGGNGqrKpO91sZ7y7k+EsHXCuz7aXohGjwdgC9/z56/HBXpqCuhKdrOscOnlzUXfbb774/PGffcTiKIrjB289Ojy+n/X6Upn1eusMCMPQahtmiTK2auq64d3eQCwuIYEQwv5w0Ov08qwzX75+/eLlqqijrLMtmiDq5nnXASyl3ayrZX15eHjonMOYhnHU6/W2223NW3+3Q4wQItZa4CBGgBCSd6KqAlIrCB0AFlNioa15ixAISQCQK4uCYoghsNZACBhjzt3E4EN4c2/f7VW3zzWntb51pXMcx56DLKstgkQL6xxo23Y+nxdFBSFmmHpB4mKx4vKmVooR4hwUQjliA0p50wJjZcujMDRKSikRsFqGGNwU2BTFxjmXRDEAoK5L34nppwRjDCY3K2mA0N2W5plsjzx7b5tXXbVtyzmnlLZtq60eDYYY47OzE601ApYxkibR9fVVEgcIDY8P941RTVW+0kZKLoWVUgZBEMXx3SPSWqSlGk+GLAo7aeYfkZvtdrFcdls17nQuMDl7/fzF11/+8I9+/Nknv7t3/N52s9Za/+ynP6aUfvXVFxhZSpHW7vTVy9Fk/Od//uf90ejp85c0iMI4mi1Xe0eHL1+/IoSMRyPnjB9x/BnpyeztZnNy8rrYbAF04+Hgm8uLq8vLIKCD7vji9KzfzQ+P9mfXV5vVZtz7U6D47OKkLflyetXtDoJev9sfWQctBBUhXiBiIVBKZUmKMa7bpiy3L168EFIuFgut9cuXL8Mkzno9raWvRLDAhWFYlmVVVWmShGGohXTGekldEsUEIoQQxSQIAoqwL7Pd1txrSoyWs9kMQjidTge9PsK+MQHHcSwVX60XxXZtjCEkJMT6wUI1qrl9+e/iBgbwtgOMIIQWoJtb4zbYwB/DXqngrwTvi9FaF0UB1B9c+36A8NdVXdd33LP7L15eGIsQkoz4i82PFxBCa8BdHQCEUCkJAGA3PpqbgD8vXYzj2BjjS7E9hhHHcT2vJRdNVVNMva9ESkkw6/f7VhtgDQDAAWch8YGGnFfeCFpVFcI3udG8qSFMPRroE0S01qvlfLUWRVFYCHrdwXg8TtNstdystxufY2ahresaY6iM0lonSdTr9XRjkyRZrVYQwl6vhxDKsghCKISoqgoh5FOFMCY3Salh3DS8bVs/LXmSYjgcGmN63T7G+M2bN8vlMo7jKIrCMAw0pZQCKIHW7qbAVkEH1rIRvOllCaV0bzLOsmy5XIaUEoSVUpyrCBOAsTHaGIUQKNsSAAuIidN0POkd39sFwKw3w+2b6yBiEKFtU5xenXcHHUTg7sHusNdfL7GVoi03nHMK0bCbj0e9wSg+O3/1+adfbLfLg90DAkEj2jztKK4gxAg6hHyGIwQOQ4ijKCIEOWAKztfrNYSmqqrxcECaqgDWLVYrUVbVqkjCYG/nEaN0NptvG1PUlhuEo6zXYRYhB4GVPIlImGdCRQAgCGjb8qKse72eMppSHIYMAW0NQNAKLq6VzvM8YdgZA6yNwiCJEiV5kGa6EuWsQgjIGloIWmiVEnm3b6QELri6WCk1N8ZgTFqhoyhargtCMGNsuqqNxUlvYlACcTAZ72plGQ2XyyXUPMDatOu4GwKr21JB6LIsQ9g4oMKESW5ns+vryyulXJpkUZg0Db+8mAFHMA2ENnHAMGXCyLyXSyU7+cQY3TYqjsM4whi1lNbWCgQQw/H1dtbNc65NYPPVtpDcCmQooYQShgnQyunWtAYgG1C41+/ylkPndCOrdS2E7HUHH7z9QRCFJ69O5peLTp4/OJ4QQo4PjmazmY4SCOx6M6/rcjTpZ1n2/vvvE6l0e6II7Xe6URSpsqU0sg5k3UxKCR2EKoSWRBZRkQyHg6MJHvT689mCN/VwONyK1cXFRd1utUidEARn8+liNl0JpX7z1UvnnPv8aUyT733vB5Nh9+rioizaYtsuFwUCWPLNuDdWFKoNfjR+z1kL1mR0b+/ly5fvvvvuT//0z5I0f/LsRZD39x48xrPlclsARrJBvuXr+XK9baesYybZSBqBKZFaUBJuNiVADCOaJUQZDQFCECNCnLMWOmX1crnodDrWWKM4sqobh3y9GacpAKCpm5yGhoSCqyzLIA6lbm1T+TT7+Ww9GPbuHT+8uDhTHK4XFYSublqtt3EcEwSNMcBIGpPW6i2vURhKRxgKFutlnvUoCVYrPhp10ji9vr5uyioMQ4IBQw4ayxAmCPKmJYREQQ6hM8Bo5zOoOSEEMiCxKXjZljWFgIvGSgkJvLw6CYLgvQ/eT5Lki8+/XG02O5O9kIVVVUsBqMUkkFEUCSHqukYCwRp6HfhsMRuPx/mgu23KTV0EaUSBWW5XSZ7My9X+/j5D8SDLlstlkqXj/f2mrABNAI3+6ue/6Wa5I+nk8K0vv/xy2y729/fnyxVscLfX21bNqt7EQail6nbN4/t7Voo+680uLwq6wFyuZMEYe/DW/evlvK6qpqrvHxwFGP03//xfnJye8UbuPzqaDhZX0+tHjz/8+Pdf/+r54qFNj59dHDVmXdQI6Ht7w14Eyu31D986nM5noFlzwXuDQb1ZxpNxWV0nKQyoPjyY/NN//Gcf//aTF89evXl2miTJW28/kso0Skikf/G7X+TdDMR2drXt9IbDcbCp6zevT7/30w93d/Y51w8fPPr88y94K3YmXcbCarls23Zvb28D0cn5BWOsEUJvuVTuyctLgCAlUVWZVbMghECnnbO7gxHStq2FVaYtGpLhtmw3m20nzZI4a+q6l3d405aqHg9HkICybpU2hIJhP1+tVkeHu5vN2ml+vL97g9UTUjR82ypMGcJMa6C1SwNqCOLG8VoEUd448MXzV5xzi4OsF2itWy496O0MkEpaBL0exc++PgXc1yJ7hMkf6l7iaq0lEW3btmkbY0wYhlmQamiU0jQkPraZEIIhustUyIOQ89pKGVOAkFO8sgrf5KQ5hxB2EGCMfIYCQshiFiWJaGsHbBRFjOI0ctfX14NufzgcPH/1sj8crLcrRPB4Z2c6ne7ujRhjmAELNKbQOCUMZ3FQy0paLY3ylAcOCHTIWQt1F5D47GK+uztBwFV1df/ewWIx25blV19/MR4Pv/+DD/f2dn/5Dz/f3RtWVfH0dP7BBx90Op20H/zjP/3p1dXV5b99c3gwCsOwrlunbSeMjXG2VgEJ5EpLUSysAQDwsiIIDbpdxljTcqco2um1jaAEjIbdIMRBSPI878X02bMXm6KOgkAA2zSVtmo0PsIYP/7OOx9//PE3T7/hnKciTWRirT3e2edClE3dCse11M44AjgQUcgQoIvtcl+M/vbv/vpf/7N/9r//7/63v/qHX3z5xRcJRAhhpOFmUzgSpkm+XVQogA4756RD7WLzJlpVf/zTH/b692Zn09evX1MWPwjv5a97m22VpT0jUJ5MTl9cL+ZFSAPaRaJpWAoO3hru5uH19aybEALldrPoHj/EnAhncEC51VprjAkLmDHKWI0IhUb3805bFqPhkPMqzfLJaGisIgjATq8bM4oRyvN8f2d3b2dS1/XVbO73sCAIIGXcWi2ltsBqASHWWkttIKQQOI9KIYSw78z2dBe6Me0MhkMIISYoCP0ahoKARiFBCHBgtdZKibquKbpJ8fONovCmGAN5KY3gCkGitHDOYYwIQZRhL/c1jEEI0zT1CJhWgGAWR2mWdvzPK6W8zd0YhxARvMCQVFUlGkX2cDbM6rKZz6eMYMaY3ywhhG3bOufqqpjNZt1ux5eRUErH4/F4MjzcP3j69JvFfE4pvXfvXhylwIL5fF7WNQ1oGNA4jBCwgisjDaKQRVQ7LY0EAARRFCdJEATGARaGhAUAIKXMw8dv/eiHPyaE/N3P//4v//IvAUKPH78Tx/Fqtbq4uPjOd77zT/7inx4fH/+P/7e/XCwWDx+/fTWbel3VcrOlYYgI09Ygn/qOAAlu+NTpoknSYZT1rcPd/kQ0tXUIODyfzie7O0672exSadvp93ZHw+12+/U3XwJoV6tVnudNw58+++bNm9dlWXWyLrC44fVi6SRXcR718l4UhDqcv//+hwCRf/jFrwDCo929w3sPpILfPH+dJFmvP7IWfvXlN69Pz2bzeZZ1MHGIYGchcAjC21wwZcqytABSSgmR5KbPxLfRU7/iQAg9BurhQT/vUxL4J6bfxqSU2bDPRaO1fvb8yYfR+1K0zpqTk1dJGkVRAABwzkrVcmuNURGISJRqI4OAluXWGMWV6fRyBJAQwlqw2WzauhVCOATvmOlv47d+qYIO3KRAAnCLEt/Av1fXZ1WdJUmitWrLOozje/eO+91OkqWj8VBdXG22K+AQpUGW5UEQpHkeRZF35PtL0bOnBwcHHlFI09QfD4yxo6OjmnPoQFU0EAIlTRwmQRBRRAeDAQ1I0zRBEFxenb/11lvH9w4//ez3eZ6vlmvnXKeThyyCCZJS13U96HT7vR5GaL3eUAiPj48n4+HVxeX0evXw4UNC2L17RxAChMCPf/Kjb54+j6Lggw/e3z86jOLk2bNnp6en3/ve9/71v/7Xf/ebT7lSv/7dx0+fRZLXWRTtTgZH9x7MZrP1ehmHEZcqiZKmqt/9zvsXV5fdNMEYE8yiIEmTKGBE8KZpmjBiSvK93cmTJ19utsuje/tOBVKKw6N7603R6XT29g9ZEG/WhVKmFfzy+qoV3DrQCm4BxD54X0qNSa/XW6/XCKGQMevUcDg8PT/rdSMMIEIAYwwd9CBT27ZVXQdB4ACo67pqar8Z+2dLVVVKKavNYrWEDnDOjTHdXu4T0P1Xprjwaj7OuUMIImKBU8Y6AC1EEEKN9V0SA759eS29F9P5B+Cdm9HHJ3jY32/2/i15TsFvsR4qa9u2aRpmg7tUgzvDJIRwu936N4wxxhDd4Q3+r/M8F7ytekK3bU93Fzy6qawEEBMWEMmhtdYojYkFhFBMjFb9fn+83U4X0zRN0zxjjGVZRgwIWMQIgw4AiPK8m2WdMIq32y20wDloLbDWWAsgIl6oG0VRr9ehlAJrCEGeeQlD1u/3q6qaTqdSiiAIrNXdbnc87CUR+8kf/aAs6//3f/ifRsPxO2+/9fOf/7zXG1BENdB1XRmtRdsQBDHGQghCSJ7nBwcHfuPXWvv+CCiF/xzCMPTQZpqmaeA6vZQwaByo68poUJfF5dX5ZDIRTStbYa2DEDmHlLJaa6SEA4ZSbEGAMFY3ymwLjE7jxMjm6urqSwi/+8H7b92//7M//bMnT79O0+h6cWpRk2T9iovVusmHfWWl1C2EKu8M63p9dt48Whw8fHRELTm/vDDa5N3u7q5erp5st+vxeO/k5M16vdZa0iiNk8gkYZzFANjxzsRYUFbtwcGBkqAsS0rpeDS5Or+C34qLNkpbZ5zvpUIAE4QQYIwRgqWUDhgymezGIWsIJRgyFmzKQmi12WwsBI2UTStQCAHQNedcKIcgAsY5ZYyV2kJoEfQ9wpBzDrE3zDiEIbituRTSM0kQIeCcKauSiypPsySNnfUx5jiOY4pwFCUQwqpuhFC+/uQGvVc3N4k1QBullEuSKAqTMGTWAq+z2NnZAQBMp1NPdiqltTbWQGeRNVBJa7RoGymFyZMcY+zCJKIGWHt9dbFcLqUQ22KdpnGexAC6NIuKosAYcs6jJGFRaIBTt/IWJaSU8k/+5E8+/u1vBVeT3d2yrC+nV9fX18vlcnw4aCtbtTAOwziK0ryTxHEaxYvFCjHcy/vD4TjrbdZlJS6virb+3SefffTRRwai9ab4/aefAgAoZY8eP66qZrlcLtcrJU0URQcHB8P+4Pz07OmL5ydnZ9//8Y9/8ctfbzYbYYCUMs6yum1uZLrWhIxFYRgEASS44vhqWQeMWY0Pjh9fX5wDeN3yVVVuHtx/VBRFsVwLIfI4Ge/slctVt5vXdfnZZ5+kaTqZ7GKMtbOc84ODnJEgTzvEIYzbOEzyPMYQni8qErDVasaCyADQSleUTSNVrzsMo0Q6s9mWXGohbZp287yz2SywgxBihJBzwDcoOucgxAR7+D3AhPriE+8V9EcyvbGY2yAIkjjy8asEI6+v9q3WENgoZJtN1bbt55/9rtdNPV07X1wZo/r9XpJGjBFtAgAthDBwuBENhDhPcgcdJrAs2v3JkLcaI0Ax00IXVckI9R02ADh6CwLfVbN8a1DwadPo7gHktNnfm1hr8zyyIGQ1mEwmB/tDY1slURSiXj9VyhgNMCMAqqriSdwNCI16fadNWZZ+EGmaJg7CWhsMIMPEAKW4kJhELOjlQyllU7dRyNbLLaXYOUfR4mc/+2Op+GKxyPP09ZsXea8zHPb/6m/+Y5jmz549ozSoqmp+NYcQZ0naSVJndLUtrs4vZleXWRIG49FqtVquF3sHRw7ioqoODw8///Kr/+f/63/6X/9v/s29+wcXF2dRkmKGGQuttdeXV9Od/QcPHgwml2/evHry4km3k0WUtHk+HI+y3oBQ1rTcaD3p9FerlZYiD+Nfv3oTdfDPfvanx8f3i7IhmAx63d3JcLsp+t2MIPf9jz5MInJy+vLe8b5S8nK72f/gw/lyJbXhrWi5hJhML68uL67ffTdEBLOAGeMcBCxivp6YUro3GFZVNV+tZovVtmh6o1EURcYYA5yvViKYWAvgrdHAb/BN0/igJMyoc44yxoXAACKCvf2PMeZN6h7bl1ImSdI6sFwub5ISwpAFCCCktTbWQUI9MODHDv+/d1bYW7IM3slX/XPcv1U/N3iDw82lfqvB8rfPXU4X5/zusjTGIHBjs/QO+5u53AE/glBKBed3OgZwm53gp6I71eRNaISXSULMMKGYIAQwgQihgLIkja+vr9955529/Z2L64vRaDQajTBFnTTZGe7Fcay1mc1m6/XaWWAtLDZly4W1wFoLDNDWagUwBoQQBZTSAmNclluMIACgaZo0i3hTZllW19vNZl1VW0ZvnE3f/ei9qqom4/5bDx+8efX89avncRB28+Rgb2y1mU7nSjZRGIYhgsBAALrdrnMuiqLxZKKUury8VEp1uj2/ZnjnyF1kGcbYwjbOgu4g08o2TSWlhk4v5ldRwDbrZVmWWlsHqbYIamA0grJFiFBKIUbYIGoMQIggwNsaQQgQ1do8ef7sP/7VX7t/ZD768IN/89/+r/6P/6f/c7ZZZt2BRcFOlmEWvDo9gcTkUdAbDf7xn//01etvZvOzV69eTHZ6D47fSfLs5M1Zq1RRNJvNGkE66A3H4zGGriq60CitBLUuShkO8X/6T/+pLOrZbBmFGQvZ9eUMQaoVsMaCm3YPB31KNPDDK7wT6gYhDcNQKeWAIZ00a9sWQnhweBwGdDGdKa2zXrcsK+tcqyQGwCHTCqGNYSz0xc1GO+MABM4h680wQogoiZMkYgEFAGinAbAQOs75aDTq9TpKtC0vOa+lAghahA0EgGDCAtJBPXJbRGbryhogpdTaQoi8h9UYx7n0uhWInHfk81bWTRkYvVqt9/f3b2o8kQHACKFms4X/bZ2FRmtjTLFt6kpkWdbJOjqyUsper0cpJRidnp5uixWGttPJAIBZljVNE0VBHMfD8ahp67Zt0yh88ODBW4/un528ubq4DBi5vr6ez+evT948ffZiNpuTgEEIr1eXRVFA6w72d78zfnt3skMxccaUZ1UcpyQMpDVcG4dxI1WxWMfh+YPH79StlNoJZSilk92dnpTr7cY556sFnz198dv/H1f/9WPZlt4JYsuv7Y+NEz7S5828tizLsuib7JkeUaORoO6XbgECJAz0ogcBep8n/QkyDxLQGAGj0Uz3DHum2d3FYpEsslhVl9ffzLxpw0ccf872y+thRUSV5uAiEHkjMnLHPmuv9X2/72d+9au6rgeDwWq1YgFvpbwYXxZlxaKUR2EQhovV2lcJymhrtScoKSXjrD9btr0u4zSMkkHdHlMWH+zfe3D/zt3btz775JNlvKgAdEIwhDiELInLsri4uNza3n333feTrF9X4uz0AmPqtDFOY8IhBco2yjLEOQ1CSqmQZrjVw4Q1rfzy6QsHICBUO9RKXbYCQhiFsQVOKgshdA4C55cjBg5jCCElUYQQwX7zcshrtY1vYvxHSikGV6ZDN0v5agQIIUIoSZI0iRizaRxbrRfT2bMvv9w92EUYBAEDENMAaStkXVUtkLI1xkRRMNraY4xFUbR3a+/Vq9dNfQkwSNKI4SBJuutlvpgvr3KbihpCCJ3xhHOModU3dHTTNA0A1ssKfl1AWIexZpxoIyF0nSxOs6jXy7a3t794+kyqhmCXxJkDSCkjpdSynVwKislgMOikmf/5Qoi1kJPLcRAEFJNaG2BdwLgzdnI53tyJIcSyVVEQLxfzKGBGcmDsy5evjVHjyUW3m6WdLOt2vvaNr//RH/+jTz9/trW1k8SZ0+a8upStYB3y9sNHs8mlEW1V5JQSiklVFev1knHKw2C5Wllrv/71r1ei+dnPfva3f/fXnV6vKCpj3fHxYbfbDwPeybJ8tVZC7t7eE6odX54n3V5IidRqVdfdshn0ho/fDQ9fvX737Xc+//RjHSeiKkVRrIuqXC2j+4+GBxt1IxnBnPM7d297k2ZK0KO3Hgx7SRJxhAHQ4nIyret2Y2NjPp9rZe/evT8cbQnltHXKOAu01lYZp63VWrMwCILA60vzPJ/P52UlkD/XhbQQeMsizJm35kEI8SjGjENCMaYOIGehAwgzziFqW4kxwRDWrbDaRFHCOV8sFr2sI4QwGO7v78m0vbi4yPPcpzV6AB9jjDD0hYLV1lcJfubtp/tekuD5NJ6Lja6N/z1DEF4Jep0X+nvTAg8Y3BAIfDsutfKPzJVoAmt/BMaeLwmAbx9vFBAed/QwsL8w/2TdsBevKuBfWyloCCljhFEaUIYg4AFLVDSWommqx48fr1YLoZU2EiBqjJGNyOI0CPkSYaNdyCNKaV5Ws9kCAIAwhRBB64zRwFgMYKuldRo5W1ZFHAZpFlurgyBIkigMeZIkVZVTShVB1uokyYr1Yrlcvnrx7Du/9b3vfedbf/93f5/n69/50ffjMHn58mVZrMI+GQ23Is58El6nt+HJqvA6KsxzmMB1DpYQYjqdQowwpgihBq0QhZubI4RQVeeylXEcrleVaMq6zEXbAAAQIsZCbAllRGtJKYQQIOgQgAAiggmlOKCkKvIgCA4Odo9fv/7Ln/71arUiAf/Rb3//w08/PhtfzpZTzMIH7z5454Ov5XXxiw9/DqgLI/T222/tHfTrZnlxeXR0dDjavLW1t7sqq9PDs7bVvV4HQ6q0uDw/nYwvlvM5oaCXpUkcLMvl+tkyEs4A5yAcbW/10iFnMXTYWVysCm+bAZHDECAIAUAUE4yxA9abcsVBnCQRRI5zSrS2VjsPARkLDASEMRZwW5aIcR5GABEDAJHIt3hK+lLVWYAgNN4SASEURVEYBWEYEoq1lj6SCkK0MRrdurXPOJlPxqPRxsGtbW2k1cY5g4BXmRtnoZCmqSXGmLNQCm2Ma5oWOA+IYedu1DsAOGSMraqmbevlcmmK3D8Yp6enq1XOOackBA4VeYWQD467au/attUa1Osq4XFdVUqL995+/O6773788ccvv3oipQRGAqAZYwiB5XKpVCKlbqRYLBbGWR6FaZwM+wMl2jSOJ5PLpmnKujo+PXn55nUr5cbmJg+CxXqilE7iKO4kSS/lSWSVFlJVQkJqLqaL9fpoNl+t8loaVAoTxAkLIx7Fcdbp9AfQAQBxVbcQ4CxLs7TrEbzxeLzOl/1BN4rjh2899nz1RikPElZN3e/3HYLWWmgdItg5J6RsRSNd4SDeibKA48ls8frNYTeNvv319w/2dter5fn5eVVVSsiLs/M0TWXb9rYTJRQASEq9mOdC2bbRjdCyqefzeRTwzY2Rs9pqaYAZDvs8CEajUVFWSqlWKL8cMKGL5TrtkSRLaRi1UgCIpJTrsggIse5qbQAHAXAQIUpZ20oECXDIE6yNVTe6L79DEUJ8zSuEyJ31egQHgZ/OeliLEuq06KY95HDAwqdPnyVpxCM+GAyatkQISVm3ShCKjFHOOenUqpgnSbYulo/ffnB0dFKVjZBNOuhgx4KAiTAMggZTapX1dqzIOV8ocE6BvbJ6dMbVdQ2hg5yD34jPIRDJtu5maVEU2qiN0Wg+mxAM3378OI1C2bT5ag17mFJuteaUJcPYq7Ywxt452NP9kiQpisJvZ55H5qnpxpiL07MsS502SRjljHlPaAjhF188QQhMp+Moi/b29r58+qw/3Mi6g8OjE855FCUBZYzw6cUEGqeaNqIcQ+SsRcBqLZ3DWqu9/W1KeN0IpYUy8oMP3quq9avDF51lZ3//FmfhclWcnpaD/mbEg9VisZwtR1t9LfakaIIgQAAuF+vxdAEB4WF099Y9Jc3O3i7F+PLyHAF4sL93MT05enMc8OS73//tMAxXeXF0cnrnzp0k6xASvH5zVFe5knWWBN/77newBb989qIoijjpQsQg1saCOMlu3wkZC1qpIMCMQ+egtRZTknWz2qFnXz5ZLpfOuYODA+vw6eWlB5YdcNYaKSWGgJDAWxhxiLWxSmpEKGNca22tYzxwxmJMHIRSG9FKY4yQCmGyWq0uLi6apqEEZVkWDjeOjo68A7qyVggBsaU8oJRaiIwx0FzpG2+kB/Bau1iWpVeK+kRy/0b7/v6mFLDXaZB+uuGnCTcYMsaYgCsswTnnDZ2MMU3T+CneFQhxhRpYpRTB6Cr+57oa8EVJkiQ3Yt2bqYe1FkDkuCEYhYxRQrTWwGgA7GAwkG29t7eD0Hd+/ou/e334Jkri9Xo9hpd3794djUZt25b5CmUZIUS2DQLOQYSgIwQDZ4S4+vkkAFK2DBOPKDPGIDBVVezs7GAMe/3uYjZGGOhGb21t3b1356//+ifDfq+tmy8+/URryyhBEAYEZ0nUlFWdr+nGZhSERuluEidJ0hjkTS19ULWvGIQQURRJo03gfGUJEIxj6kctYcx4FCIH0jhpcRswqkLa68QBowQCSjFA1FiACGZx3KrcOQON00Zr5etQiCChmBsWcIqyrBtn2fF8/vTFy0+/fNLt0T/6x3+yKMr/5r/777MguRifgafwB7/7w7c/+OeNKl+9fipkVdf5cKNvTTOZjj9/+mUSxXGaBHGIqAlZqFrb1nUj5Go1b0V5b//Wd77zre2dzcvLs1evXt3ONq12rXg+nc2Wi3K9KLKshxwEAHiZCXQAAgetgxBgByC64roq3RKSxXFMKc6yhEwuJ/1+zzk6mUwAhBA5QOh8PF2uV8oBGkYQIGUsxqaVQpaNcVZr4xz0jrzOWYQAwjgMQ4SgkG1VS6WEdppQTCmlFBtjJpNZvlw+fOvuW48eSNmslkuttVF6OpmtVqUDwBirlGGUYAwRIhgTSimjQZp2KOXu2qvcoxQIX2mFkyQzph2Px73e4NWrN6vVatDfCIJIa5skmVJGtLJphDckoYRTEkUMzadTIcTW1ohC8Pyrpy+eP4FAc4YgMtrIbpxZa1slQVMTHiiji6pijOV5/uMf//jpk88RsO+9957n30IItbNBGmMbagREWwOKR6Phwd7+7ta2w2Q8mSmloMPLorIoIBgUtZGGEJ4ECQwFCKIsjDub2/urvC6qpm3bTKogCClny+Xy1atXnW76p3/6p3/4+38wHo8PDw8BANt7u5fj6Whz0yJUtVqDVmkDMHLWAgAwo5hiACFCOEShVGq1mFxGdNDpnB4fStGyQYYx/vFf/Iezk+OnX345GAy63a6UsmkaB+FoYzvgKWWJVPbLL55fTuZvjo6Xi3Wn06sbFced/nAIrJ2MLy6mk1W+jGIeJuHuwc7m9s50Nv/wo0/Kuto5uLUVbkpjhW6MA4ThME6VUmVTAWet8csGQWCdA8AhhIDRjlDnnDPKKus13NC6K7PbKwkWhP4EbZvas7g9iuP7ISGEVgACiyGLgpTR8NXLNxsbJ5yzg7t7mEIHQKu01gYzRBhHCARRVFRrAGxVVQ8fPL51a/fi/PLybOacK+pSCCVqra0BUhrjLATWGobRtTaa+v0dIWSMb0d+LWv0vjQEok40yOKszPPRxub3vvfd8/Pz14evPv30U9FIb8YHjAUIaG0RckEQxkFY5YUW0ru/bW9v72xubW2MLi4uiqJo2zbigbW2qirRtsgBY1trWMBpJ4uA29RW9fv9yWTS1K3DoJGmHK+SuPvq9fHF5QJjPF+sAs6X83xzuEEcjMOoEycnrw+3t0YMYStlvl6HER1u9JSVYRJSEoH5jCDmEBxuDnnET04Pd3a/s7u7ORhsfPzR5xfn51qaXqf79OmLT5pP04ORg6DKC+QQ57wRcrEsirKGhMZJlvUGy6K8/+itqq0YxvfferjIFw8fvL052rkczy8n03VVVUK+Ob3Y3N9/+PBBW1dffPFFwAkwGYKU8+hytpzP58qhW7duhZiejcfW2sFgA3MGCHYAYUKcc1o4Y4xytm3bPM/djTUhgFrrbtaDAFsItJb+mNSa2OupvDFGKIkx9o7BxlljDLCOcg6sFcawMPAYtT9967p2zrWtZIxtb2/funVrsVhQSkXTtK2E2CBCEbbKGiEER+w33Y59KXxjz1DXtT/j/fcgdPVQ34AQN00w59w7JV85hV8XB5gS+Gv3JBbHsX80fK1wNfvQ+uapAc56IaXX8nimAvoN6Y27FkD6m2OBBdZd1xlCC8lQhKCDwM6ns+V8uru3HX4WNk0TxSGEYLmcCrEZhjyJA2dV05YYo6pYdbqJUto5BxFwDiFotNHOOoxDb+6cprE1SikRhcF0Ou5m0WqlN0b9KA4QAkobB+x6vd7b2XfGIOvOjs/evHljret3e1oaBDCyrt/pD7p9VbeirDujrSiIX3z1wt+HwDqvy2iaRirtbyPB2E+RILgykwiyhBLSCima1hijZIudTePg3p19wiKIDIIWEQAcBBgQhpEBADpjtdMGWgMAgto6qAtRxSGXUpyfX9aN6I9GQRJfzuZHp6eP3n7367/1rVenp2eX41a163L91VdPf/sPfvjW/t3dg43p7OyTTz4LOLp1e/f+vQdPT84W65UWmocBhBJBlKRBlkTz6Wy0MTA2SZJwXS7cpWhFHWds//Ytp13dyMNXJ/PFQisXSBkHgV8A1hjkLKAIeJ2s8Uveer0YhI5xAgCI45gsZvN+v9/v91nAAbAGGETwui4XeREEAWFMims0zAGhDcQIAAQhQJgAgPS1oay11ihrpdFaWieBb6igK8tysZwtl3PZVsYqKZu6KZVqCCEIXOG0QRAaZCGQYRgLU3oqAEY0CKI07WCMpVABD431HB9nrdZGBkGQpqmrV9YCn0DaNrJphLXAOYgg1dq2bVuVTdtKhFCa0iBg/U6yWDlGaK/TXS4WL149Pz4+imLOOaeUM053d3dXeZllGcZ0OBymadqIlhACnX3z5s304gwjAKzb3B55XzAeBFvb2+uqao1ilDMOuoN+2ukaAC/H02JdYojTqFNWCoAmy8LuYCvO3GpdKrMyjmHO11XFeDDcGHW6WVPVJycnRVVxSsu6whgv5quvvvqKIOwzcAG+MlkL4mgTb82XBWKMBWFdt0LK67ZDl0ohBxinFOL1enmia701irnd3hyEAfnq+ZMvv/xcCbm5vb23t7e5vUUpNcYJIYq8WS1L4LAz4Px8usrrJO4DGFoLkrjX648GG9ucQkzJYj6u63p/1Ot048dvv/v2228fnZx+8dWTStaU4YAHcpUbYy2A2hhqqEMOIAcMck4jhCnhEGKtjXHQU1IwJgjh610JIQSAJZD8WiPOCfYyMCmuGin3G2HtGGNKkJO2zsVgNKAYY8xPjy60U/PV8oOvv0sIoYQba5W0UrfOGaWttpVSarlcAYfSTsI4VVbNl7OAppeLiZWW09A5SBAOw7BtJcbgN1kIV53WlYDt6jQC1875AAAhzGy2qCt5sJ/e2jsYDjaMtvmyXKyWRrk4yCgNjXZOWwNsUzYQQm8V5zOEbrhmXirpPdv9VzHGZVkO+6lzlmJAkNvcGq7Xa8bY2dmZMcYAaC0IoqQolBLozavzbr/3+PE70/FkNp0apYt1ub+9/aPv/+Cnf/HjOwe3pGiMajcGvW437fe7y9UUQvfFl19eji+6Wfr47be63SwIeduq09Pjhw8fDof9MOL+/frggw/G59Pz88u1KUeb26YRErdxmCRxRlmwXC6Pz87jJNkY9qVR/bamYUAw6m8O4qT//gffiJLssy+ffvjxZ8u8dSSshFaW0KjDghDz6J13HychXyyLspZ5URZltf7qeRjFe3sHQmmMMUDw8PhoMV/dAOlKqfW6uBhfZr2tq+mDMcvlsm6UTxJyFgKMlCJ1XQNr/JkNAECUEYQo5W3bSm0opka75SqHAERBaIBB0PT6QyVlXddKyCQIjDFpmoq2BgAEQZBlGbp24PaWNn7BeHYq5eSGe+tPaP84X1W6WvumyMsTPB2BEOKJijfk2aqq/Pf4ccPNagQAeDs/v3j8rQC/YfTp/1Fgr8wD3LUrM6XU+2PeABXep/9a3A48TQFjjLxjtWwFcAQiAG0Y8iSJ6rrEGD59+vRb8bfiKIzDAGMcR9Hi7BJYNeynQUhW6xnnfDAYpFlY13VRFI1QEDpAXMCRYYxzjhnVGlqrOU+Xi4Ix/OD+HaWa4+PDjeFgZ3dza2trvV5ujjas1efnp07ai4sLUStKaV3WjAVJmA26A+QQRbzf6UdB3NYNApRiNr6YnJ2dYYzTNKU88LCcb0G98BhB4jEbwmjTNGVZatiJorBSjZI1IZRA4qzt9zo7W1urPC/yeVWv0z7DmEPiDPBZGxYAhwnCCPlz00ilWoHiUAgxnQulRKeTQkLWVXU5WYxnP8t6/Vt37j4/PB5ubX/7O7/ViPq//C//y69/84Pbd3YppUmUTqant+8cbG/vLrQ9PTrR1oZxAB1kmA2yPkYo4kypQSuqui1evHgOkOmPepubm9PpdLlYT+azdZEjxLIsFq0gUNzMoZyzBEMMfx1bDaGPHLPXO1jjgCX9ft/7kW1sbizz5bPnX41n4+l8xlnIAo4QaWqplEGQBAGCEEmnnXMQIEwpAOiG8KK1hhhgghkLEA4QxYSgG05vHEcY6rouv3r+dLGYQWAxxpxE63UhWs2yCGMMIUYIBzSAEGltrBG+c2ob4Q00hBDGaoSAtdo6naZpv9+jlFtrO51Olnal8Lk4VAo1n88BQNYArT1vyBljnQNSyr2dXSGa1XJprLh9cBCG7OzsDGKMEADQdTqpUCaKIuegNz3M8xwgdO/2nfu3bw27WV0Ve3t7t27tb+3uxFmntzE8m06fv35dF7KbZQhjbdx0Pp+7ZVPVWqg4TCiJHUTLdaUM6fdj51ArDHC00xtKqauqcQjXonVL17R1nucbGwOrddu2/W5vsVj8wz/8w+Ry/ODBg/Pz842NDcpYp9cVs0WvF7XaAUK2dnYxYXmeCyEQQm1TLRYLqzQhRNU5J4AzTIDN1/Otjbsh5+OLs/v373r5tRAKIBinWVnXwGinzXK5UtJJZWezlTIgCFOthFKqEWa9KseTxXCQbm/v7Oxsibbe3+/t7+/fvnunEXUt6n6/G6VR21YEuNFouHfrQDvw1cuXy8UaYsQDbGqHECbEO9ci4IyzDgCAEL7ey6xSCiGEELDWxknomWXGGEiJt6KTovXE7xtQ1I8DkjhUtZnPZ920b4DZGm2tr4LjV8bp4Wa/081YgIOIhyGhAev1Oodvvqx0dX5+0dTqg3e/aax1zs3n8/t3h5eXU6dcEMQYYcY5QdSYHMMrWpnvKa+mm9b6uuY3mz/nHICII+gc4DxcLFZ/+7OfB0HgHCSEikYyGiBERCOrugEAURJ6Po1oW0oIo9RZO5tOgXNt2zrnOp0OZ6wsyzRJ9vb2yrJ88eJFFLHVaqW1FLLqDrpCCILRdDrd2t2TQocR7/eH08mMh+FiuRLSfe0bD+u6Fo3o9fqXJ+ds/9b77773s5/8ZH9v78VXXwoh9g+2h8NeI5ujo6Pt7c3ZbDafLZ4/fya1YMEfPXr08Nvffv+zzz77m7/5q/39fUIIw1jUzdZo1Ov1ZrPFnb2DrZ3t1WKpjCUId7NunCatUq3UeVlB5Dp3b705PulkSZnnNApu372Xl+3Z5XxVVo2yk8Vyvlht7e4ui+LZ8xedNFEOvv3+1wMEXr38CmH+ox/97mq1+uijj4qiaq8Jeqcn5+p6DOeFf9baPF+Nxxf9UoVJWpZlEMd9ytG6CtN0XeRp0iGUIgS01kZJdP3yfgBexaC1Dhg3wJXLpVZqc3MTOmCMSZJEUVrXtVJKQuC02djYsEb5UsDnae3u7nLOAcQQe2E9glBfb8S/Nuf2taZ3tvAHtj+V/e8SRVEQRb+uR631swNvou9RhOvt3oFri6QrFp61/rz3p2AcRR54gBA6cMWBcM4h6HwRkKZpp9PxI1ofHnFTW3gkw1885xR610gHAKFhwDqdThxGAadhFM3n89Vq6fvG+WSqne100/6gu72zKYS4vDxDiPT7PQjB5Vg3DYKtAohgAoOQeRGHQVQI0dYKISBli3H24MEDQtA//Orn1vastZ1OWpbr999/v23r8/Pz559/FYdxVTVNswhZSCk/Pj5ta/HwwVt13WppZKuk1GnaCYLo2bPnvun1FGkppR9AhFHs4Rxnr0ZC1FkIq7Is87YeDnrAmoCgQTfzDfGw1w0jPp62i+Usz2XUVYwHDlvjtIUWOQQhJIhAgoCxRjtjTBLHFBOEcBgElFKhTdmsBmX15//uP7z33nvfvXX3wcNHz1+/KerqyZMnUov5ev7Tn/70zvH+t779wbvvvvvkqZ3PVr/4+3+go24jBYJXMR+QQCnlbDoNGF2tF1WVQ2IpJzTgURQRhk1rj46OLi4m1bq5vX9/a7Q7uZiGLJKNRACa6z4fQgLhVdQ4QgiTG0GEattWKUXm9fJueJ9nwTRfTKfz8XQ+m636/U2tdVMKZfxwCxtjlJEWWuQghtgYo4VEHlOzVgphjIqiECBcNaUQbRCy3qAbRqwqW9FYZ4k1dDGrlRZNU20M+xDhr5697nQ6e3sHRV5ppaOY5fkcABPHcbfb3dyKeBCdn19O18ugE2utKWJIA9kKqxXFCBoji0pJu7GxgSwb9kaccIcgIWRt1wCh2XRBKQuCpJatVhK3oqjHZRjfi3urXLw+PL51e/fxO3tcOTVZFMUaQ7S/O7CGNHkr1s2t/dvtovpo9cnDB4+Pj96cnJ1ipy8uwf7e1pvT1zwh9+/fr9uKsqD/6IGRYlT2iqpyWoaOcqMBQLPVspf18sUyxLxcrThhFmNdzAihMazD0CVJqHW5OHv58NFbP3n+BSHszr0HlJPzywtr7cP7D756+mW/19FSVMVqOb/cHHbuPf4AIbIuSkIosLCfZPcO7u7u7mXd/mQymcymnU56fnmBgOv1ekVRCFjfufNWHMdJlF5eTsaTMgzccPSgaZpinZe1jsOwrcFMrgEAbd2WpFnXolzn1hijq7aqbLPssLASlVZVxoeXx8fI7A8ebY/HY0yjk3lug5Wi4ziO4+7gT/7J/+zLp0++ePKkPxz88Ic/HI5GH374UZ2v2moNMQJGG9RhnCKEpLXOaUgRMlap2gAJIMYEMg4huso1MABYIQjGnSjyQ7X5dJZlmedwQQgB5mXTUkqTThcAAAgxyFjMP/nyKwjdg0cPWnMUWguxm03XX355+ODR1mI5/Ud//Pv3Ht7jETs8fP2rn79ZrVZxnJTLVbX6NAx6UVAqUVS12BhtAYMAQAQyCJAFMMkGShUaobWQCFiHCYLIOYgIc0YhRCBC2upatUoJRHAQBEAZ72vbztrFerWxsdHpdKSUACBPROecY2gBAEbntXKiBTvbu23bSqG2t3Z87GSapt1utyjWSZLUdYmxu3fvVlmWdZ0Xy0snqijJYoaQaRk2L776MkljxoiDQCh5Nr50ENZ1bQlZt/WnH38lW5mlw8uLWRjGxtk///P/sddPP/74V0nIAgYIdL0knj4/STAhyp4U5bgoFaCfvj7vfn64vbX1znf+8bwOf/XxJz/4/Xb71uNffvTV1jCtnehuZffYrYPtpL/Bp1uBhSBOQaNU1rVlKZWp1znsdm9/+NHPfa/92z/8nZ/99V9PXp3Opovv/fCHFJAQU2zAre29um5so3Qlwu7gg0fvIQMWi/V8suhkSRZHqm2+8cH7mGKMXBSyVghMwLLI79y+1xsMl8v1dL4qi5qnarI6WcybLdph2VarNEIIIL2arvv9vjNO5Y00miAaxwnlDFECIPR5fNYASnmn08MQWWUJopiRYl1ShBGAy/kKWMsJZ5hB6FbV4uxy/sH7780XBXAnL18fJUmqteEshEBWTWuMi+KEE9w6cEO7uRr2uat0ym63e1MQ+DPAc8z7/T4Oo8VkKqoaYiTb1hgj29YmCaWUEAIQBAhihP3OHg9Gr18873Y6PIzrMldaIAf63YQgLIQKApYkSdUI5wDm3DlIQO0gAAg2ojXLhdZaahXGkX++lFJOA4AgxEhbo5ra6chajRCxBk7Wi8FgcDaePHp4f+/WbWd0K5qN4XAxn/bCCMXJ7t621ujTjz7/z/93/wep5Z//+Z/v7OxU9bpuciFrpVtAYBRGjIdKGWMMgtgh55zrdrtVvt7b3HZWnbx8eXt38wlB3SziAVrlLe+yj1589PjddxLUad84igPOs3ICpqXIQGAMHC8XX/yb/3Zvb2tnY1jXSxKHNGXni2kNAIZwd28PADCfTnwwWFnV66oyFihthTKttUEUBmFsMW4BOLt41dl4f3xxKuqCIbTR6fz2D74HrEMObI02h53eql8HmEYsWBUFQ9gq4BkgELogCIIodc4JIaxzhRAGgiCNZ7MqCFgWd6Ikzs/1iyfPf/C9H37j3ffXi/nRyfFP//ov337/nX/+T//ZF88++zf/9s+OT179L//X/4t7j9758tlnhZIbQPQodhYigtmgUzdyXq9REl5MZ0KKwWhDNWVZrKBFs/q8w7hU5vd+/4fVqjh9c5byZJAMhgidvD6igBLCaqC0tQAQS7EDoARmFMYAOWsQBCQIImdxW8mAxCQIAs75cr2+vJx4Hg0m1OuCICbQKgCAAU5bYA24ojFa69x1VLQXMQJDCGKMIeqIIcYQjK9KkiRJ1+t1UeYMIx2ZfF06YNKk0+/3m0omSRLwaKFygkmn07EWzGdjznmWdXqDISJ0Op1TRBjl0AIHHXQQMGghAs4oaWvbbPWGUsrFYuF1zJDgKIpGo5HUqm2kEFIbSQiyxjlnoigGGB4eHa2KFUBQGP3s5YtW1HEni9O0aRpp9fl0fDmfLvI1PD8r8/zg0e7Z2dnVWItzAuHF5SVn5MmTJ1k3q5oCqqbH+1HMtBUAcgyTOE6jKHIOUIhevXwDDKx4OOh1nAG7O1u9Tn+1yikmUsrTo6PN3VHTNF7lDIBcrVZlWWqtNzc3nXM8CueLxc725t6tg29+81sA2Ebi+XwuWlXlRZ6X3tY6YAEAyCiVxGEcRbtbWwQjYxTFaPf+dr8/3NwY1XVb162UcrlYn5+fO2PW67Voasbpzs7WW/fvMcbyYnV8eOicK9f5bDwJebS3vR2GkRAKOm9EgZRo8zyfLxfL1RpRohbLxbx48/o0jKIwjC1wq2UNQfDm9SnB/xAl2eHx0WpZAUQpCgDSjtLf9KNFPmrWuTC88sF1zmF89T3WWiubgDGjVNM0AIBulmIIVCu0kJzTiLPRoE8IoQiu12shxGJeWGut05yzfDVXWgjRYuKiOBkOw9PjS6HBcrnc2to2RjPCZ9Plem1FvW4roxWMghhCmGVd6BkHGAIHHDTQOeucd+oEzhmttDHOWAwBQwhCIqTknFLKKSNOQU+WvJKoIeSnv57EDq4Tou11wI8Hjf2+b630+IHnWBljbjz7wHX0cF3X3o8ZIaSVtQ5aa+tW4KYtyrqoGn8ICaWlMgARRAgEGBOMEGqaxigNgYPWMsacc8poHoaz2WyF3Afvv727uzscjV6+ebm1taWUOj8bLxaLTpZ0u73peDqbzIb93jvvvIOgOzl6s//97969e/DZp58+fHAvTvj2zlvYCGttJ8vm65W1ut/NHMaj0XCxWFhrX71+eXZ2hgAaDAZVse52sy+n406va4xe5/PlejGfzxxCaZoul3OM4eZo0DTl0dHR9PL8yRefpUnUvX1rOp0WVU4pBcDmVamUIpQrpSbLpUZUSQMRwSzkUdzpD+ftlbufFOImcdHvb/7MBtcKw9/kGPpuHkKohNTSm74aCIAC0FPAgLW+z04ijhEMA97rpMN+z2pZrJdNXTV11e31OefG0FaqtqkhRiFnN5OpG+WO/+j7Nnftr+DnDj670scbtm0Lrs08PGbg5wgOAv/N/ler1qsoDDil2BkMol6WxlHojBVCKCk4wUmahDzwWndjLLS/VjSYa59/j0bcwAk3F2OtVVJ5Ldug1wcArFYrY9ThIUnTtJMmCKHlcgkASJKk3+9vbm0s1/WPfvd3/q//9/9bmiaDwWA6n29ubdRN0+/3DXCuqK71HdQ5hxFZ10VdlqtFG4U8iiJK4GQ+m80vwyjKyzLPc855kG58/uVnx8fHW7vbUkoNDLTUWZCmnX7WLYtCicrbO3pgxl0DOVrrwWCQZZkyzjgIMXLaWGuNs0pbSmkQJZ7k4RHKTpoS7s5OzqGxwJGIhXlezmfrqijDIOVBjEgQJykmzFhnLXIWBwEFDhpjIMQ+INRvAlVVBQHz+qm2bZUShHYopSSh6bBbqjrppd/94Xd/FPzuu99498tnX2bdzttvv62d+u//h3/1r//1v/6Tf/xHSRQDYNMw4sRobbWFzjkGMQ0Zo0ESxdPxBQCOBny3u5eloWwbFkbvP3o7jbOzw5PDF0eGmOOTE92Yja3N8+PLVmnnHOXcWrte5ZSx/nCglKLsKrfWPyDW6jzPyQdf/+bjd967vLxsmqMoiTe3trWxUkqICfb+Yg4CrZ27yQgx/rlCV8MYCCFEEFlr4JUwAmICCbmqFTyb5lqxcyXTzfPaGGcN6HaGhJD1ep2EcZIk1gKGLMBktVycnp5bgITUGGMMMI8zSmkUhkkUp3EYRWHIA0JIPjmXUt67e39nZwdCV9SVUqJt21Ve9Afdqmw4DwDE4/G0aWrOuYCoWC+qpk67aavNej53wA4GvW63m6/WVutVWUCCN3Y2kyhpdNvrdl+8eBEy2un1vvG1D+7e3v/Vhz8/PTl6/M7jOItmX0yKujiwt5TIKbVbW5sR2cQQBUFUVdUg6VyeXPS7g8eP3v7008/39+788R/+MYTo3/7bf1eVzZ1be2kYztbL9Xo9nU4RQhjTMAz7/SHGeDqdEsIev/1uVeb7O9vAuadfPX/z5s3Dt947OjrBGJd5cXE2RpiKWmZxenh47D0om7xMO+lGrzufz4M04Yw4Y/f39yeTWafTGV9OptPpfD5PksQZ07a1XilM0cHtfcJZq+TqYrK1tRVBRh2+s7v/wQdfQ5gul8t5Z7WqKhYEcjxZl8Xx6dlsMSeEEAwmk9LLXpK0QyltlbQ2aKrq2Vcn2riiKJRxYRgoS5RyhCMEoLMOOoAgooR6oBLjqxMUWAcAggBaY7VSfhdHCHnlC0KIc0oI2toaGascMAgDALXSDSYuQMTalnNe1q1U8uLyOIxoFJMo5oNB7+DOuz/5yV+ZGkwvp2EYTyaXL1+8yZe2KoBVoK5K2YBu1ziH0yR1xiCMIcTQeVsY44xzTlOMAYAIUaOgc8pZq4w2VkHoEIQIAgAABo4iTDiJosiWDYTwxhvffx6GoXMuSRK/leR53jSNfzacc20rvLzes+SiOPSobxiGcRwbY5bL+atXr66YEAgFQYhpYC1Q0ihlnIOcBX7u5pzDntiMIEWYc66RF9pZiK4sAaBRESfD4bAqVt7UNs/z09PTLEtaJUOeUFzHYba9ubeazy/H58N+/5vf+vp3v/2tyeUYAvutb37t049+Uaxn7779YG9v79/8638lpVRKyrZmATUmPDk+rpo66/bTOJpOp1arVup+pyNEOxpthAnPeonUzWw5l6a1QEnt+sHAIVC1tTJyMp81deFUSwPaG/Q0wOuyWSyKpJNwzoUBjdDE4kYIfToez0uAsBSmbmXTiNbCKyGcNt56KAxDYKw3GwAAeNtgrbWDgEBAKXXOiqaxWiOEMETieoRPKXXWAu9I480JrLXGEIw4Rs6ouixMGpVFThF468H9Fy9eaNE4a0QrWqEQkYQQ4yAP2M0Aws+JPVfA2yYaY/BvBDt5kop3TaCU+rhTXxO0UvoS07hrQbwvFMp1QFCVL4wUCIIk5N/82gdBwH75979oy0LLhoAUUNjWsq1bKXUQ/v9FU/q93V7FOhgP1Ptf319nEAaEoDzP9/f3IQKr1co5t1wui6Kg+3tplq7Xa6XUer2mlELk9u48GI1Gn3zyyT988vH29mZv0F8sl9Za7wgHfHyJgz4OmmAXch5FEQB2a2ur2+9FAV0u52VTIU5HW5u7+3vTxfT27f3+sPfkyRPoHGNMtdaX1J00Ggw2CMbrhYlCBiEEDmJM60ZVVdXUZrlcdjqdomoBAEobAJGDgPOQOCdFTgjjQQAhbKUQTYshgg7oxpyeHG9uDCnGd+8+DBiNks7LF28ux4udvYOibDDi1mArIXAYQaqNhNcTed9Fe5JHGIZaS6UED1gcx9bqsixfvXolXfX49haKybScf/7yaZIk73/zg+3bO//jn/8P+/vbG6PB9ubWoNP9ra9/88HdO+cXp5evXkBtgIEIOOwAwQgjSgjqDHpRwKIoMFpgaJSWlHDGo8l4nN1JO51OkiQM89I1tWiFkN2NQV216/VaaM2ikDFGKL3Rzd7UnYyxphLj8Zg4CIqqvLgcjyfTDQcYY42Q1tqAIgsQdBBboK1z0DiIAMKMeaIWgh5AhH77c+DaroFzxjgOQxpFIed8PLnMsgwhUFc1JjAIorou37w+DIIgCgMf/lFVFTBWKalU29Q556Fqm9l0DBGO057V+ujwMIoighm7thLiBBOMIYQbaWCt9u5anU4at81kcjmZTOq6Yoywfqfb6WNKq6paLpdtwAxBOCBJkFJGHIZxnCmllkW5KistpLN2uS4owsPBII7iVDZJGGRx1Da1EhJCONwcvf3e+1rL88vL7sbg9r3bWZZ1OulPf/qXr1++aKrtH37jm00tCMTHl1MhRECw02prNMIffK3T6cVJ6AzY2dqs67rf7zGK10W+LNfz6SwMAkK5Z9kgSg5u39Fap1lS1+3lZBHFQTfN9m/fKcvaGnewv6ekXa3KJO1Iqeq6xr4GNGY+nQCrgpAtp5M0jY8W61534CvEK3K1Na0UoARJkvQ3hm3bSK0n85kBDlKShSmDNOHhwzv333373TRKp/M5xez+/YeNFAYiA9DlbLpcr4qyJoQkUWothoQ4F7aNW+eVUMpai0lSlrJpGusQQlHTQmuVMSZ2wl5TvgkhlGMv1FbSaK2t1tA6hKCzTgsphcDItrDGGHfSrG5KjEAU8k6WNE1VN3XdVEIIzmlTE84559Hu3iiOw/WaQ+TW6yUPqdRCNC3j/d/50Q+qcv3hR59Ya5FDGNDx+RRaSJCjCFltq7JFsKKUZlEHeF8OQm5qXAeNs9ZATDFhBEMHrBSqaZVspdT9Tka84ZizGIE4Dj1HeHk58f72fs+NosiHt/py4aZWmE6nbdtKKSEMb0Klfczglb+ItXEc+XGyDxj0vS/EPIwZodQ6bAH0iuIsyzDjEGBlLCIYAqSsQRBihFgUKaV02xKMsMN5VdZVvtHLAkaCOGpEWzW1tso6l2Tp/v7+xpOJKFuKKLSwzCvZqpOjo36W3r17u9tJDl8+393dPtjffvP62f17+wGFW5tDa+26KNJO0un3LHDz1Xy+nJWnOcK3i3IdMMIIogzPZxOl1NsfPN7f3xNOLPNZnMbD7Y3ZfGmxk87EGChgx9PJ1OrN0XBze2tjc3PcOOtwGCf3Hz6+ffv2Kl8+f/FiPJtHLCiadla2wEFlbFMrZSxCqNPpYIyNubIiiOMYOeBViNgn53lFldHEecQY1kpBb4lonZEKYxyFEULIXnMDMbjyPkISa9l6gWW+WmyPBtbq4XD47ntv101Zlc0qXwuhECYYUmutViqKuzdcV18T+EpFXXdRN1WCpwucnJxkWebRMnideHfTrflvvjndAQDQaMqDfFFao61W0/EFBO9tDbe3RsPFbFIXeRvFYRgHjCgNLYLeyPkG4btZb/4I919F1ynV1lqhBAC0bds0Ta+NDpFoqlu3bsVxfO/ePYzxcNj/xc//vq5rTCBi9Pj87PF77/ZGg9PTY2WNdpZH4a/HLsqJpjXGIYIpYd1Bd2dnp2mqwaC3zle1QI1o0m5GCLQYVqJ9+tWzVlbvf+29x48fr/PVatFoUbWtrMo2oK2U2llICGEUCVlJKREizjnP4kCQVHXrQH6FxxhLMKOEYwiCQLdSyLb1DhwEYSWQEKItBYfBap47o+PvZL/17W/NJmOI+cnJGWaxkBYgKrUFRlmDrAKtbjHy2k4Cr1/gymqo8ZTVKIo4p1EcdDqdg7c/eO+994YbG93N/jvk/fliKp3p9Xr/7J/9s+GwnxfLt+7fm84ufvF3v+CMnJ+fbmadVohGKgcg4Mw4IKQSUn3x2edSyqyTGCkBtNPxpbWWEhQ0Av4+3Bxubm5uHh+eDkcbbazGZxdNXRHKSMilUBjjII4ghFIIzjnCV3Wnxy/zlTo5OSGff/bFxfnleDyeLxeEcY/LxWlHKOOc09Zp64wxyhhtnXUgiThwyDlngXPO63AAgogyHIaccuKAhtBxTlnAMIFSlUJCTGCc8DSNkySJ47BY5x7cQwgpJRgjhIKiXI4nlx0GjSxCDnY3+3WjlJUEg143BRABAL3+db1eA2OdMxCAdcz8kyNku7e30xv2jDFNW02nY8YYY4FJU05YnATdXtLtJrOyJJQGnPmiI4qidZ5PppdtU3HOI86lEBGj3U7aSEQoIADEEZ9NL6Vsf/zjH//yV3+/u79blA1ETkhjDXr18o3S4vz8IkvSR/cfiFoZZfrdIafBfLq6e+fh4eHher2+d+8epXyxWCwX63WRTyaTdZGPRluUYqu09xkUQqyLvBbSaPts9TyKkrt3bx/cvrNYzjtZBxOUZdmbV0ckCHmcSOtqqbixeVmWdZ0liXMuCHhdl0o3cRyu10uEnLUSIVQUxXg8Ho/H0+nUWhuGYVmW2hqIOywMAHLz1ZKFQZjEabczX63qorxz5w7G9MOPPjo6OtndO3jr3YQHkUUo6WS8rqVWSZIgSupSU0oZZsDhulVFWUqpLQTWOSGUg5gxZhyoq0oZ7Y25bujWlFKEsN+n8rz02zTn3BO7rhQcppVSZlmWpJGda62ltZoQtFjOrdVBQG7d2u10UwhhEDBKab/b6/W7zhlK8dHpG2v1q9cvLicXq+Xl9ubgBz/4zosXX1ntppcLQjiCnOEAcpXFmVLKWm2VVUa1bcsYgwQ7BAEA1l0nUyPoIAAYQIIpBBZxgiGXxBnNOIUOIGAhcBgTHvFOlvR7HWKcc269Xvt+0X/8zR3kN0d4CCGlvJHEFVn9JhfKO8v6AKEoiryfz3q9ruuWUmodVNpqZaUx1jrOQwcxQppARAixDhhjgNHQGkeItRZiFAZBQDB01skWY1zU1eNHX2OMRXFMCUrTtCgKhFBdFVoJ2aLzo6PlYtrL0qoGVZnni/mt2wdSVCdvXn/rG1/78vMvzo6OIsbu37tTNfXrn70CGHW7Hcrpt77xwcbGYDqfcc5Oq9Ja2+t2O1nUtMXxm8P33vsg7XXni1VeV1lnkGRp0UhMWNO229vbUpnVumjqUimlRsO8qHLDxpeXjPNBZ/Duu++WVVXV7XSxjLKOcmvjBOFhBAkNRSsVIQwDrLWGAHhEh1LqguBGlWDh1TFrjXH+yLTO8yIxgH4SgQD0b4e9Hk84CK0DACNIcLFsqiJ3adoIlaQZhHBMxov5imDmYAMAoowTFmDKAACYehNxdyOMdNee9/+TU99dmy16cwUfs4Qp8XwXa627RqcQuOI0XA2nkOfqkCCO8tXy9evXH3/8sc8ylVJOJhPj4GCwARGmlDoLlbU3BfHNkvMo/Q3y4RehX6Wykv788EYL/uKtlsvlkhG8s7s9GAxGo6EPiOr1etP57PDwcLaY/8Ef/P7G5vCzzz7r9Lrz+VxKTRjt9XpaubaRShmMcRCECEBKqRLtar1+c/iaUpRm8Whr8K1vf/1v//ZvgojVTfOzv/vbN4ev7t69e/v2rS/0KymlllBKuVwuGaKyFQjosJ9BiLW2lELGOEIEY9Pt9huptIHWAWNAKwSixltaMcaKoqiaGkIYRVESRUEQOGN6UbKRdT///AsIwfjisiiKw+OTtNPdUDrr9qJVJaRp8to5YC2o6xpyeDMb8qMu/556VmmSJEpLY1wQsEePHr3zzjvDO/H+/r4SWklz7949QkixXh8cHIimfvrls9VqkaXR+fHkk08+vHP31rvvvrvdixeLhV0utbVREhEW1K1Yr/LRsH9xdj67GEshtjZGAY7SLBZCXBx9eXF2vtHf2Nnf+6u/+hm5zfZ3bo9Gmz/5i78OaAAh9GbbrZIIoSzLKKXaXMUv+5VW1/VisSBV04qLSyFEFCXGGKEMIiyO44vxxP+GyliptNZXowe/gCCExl5Z5kEIAUYBDQlBlBIAEYQGYaiNFNIMhkldNwCATtIhFDZtaQ2IkyiOQwTgfDFWSqQZz5IYQKF0ubE1Oh9PwijZ2d29uJy9ePkGED7Y2JpOlpgwhAGA0M+2/SQbI+eHjHmez2aMBhQiRwiRqq2bEiOKMYR4NBj00jQOw1Acn6zXa+dMkiTA2MV8tV6vq6JumjagQRREWkhrrTFKSxAELImpEpVs6+3tzeV69eE/fHo+mXqFEKYhJ3i5mmdJfLBzbzIZ/+rnn+6NtuI02d65fefuW0na/+Y3vv3pF19GSTdKuhjTxWp5dHb2/PWbk5OT4XAYdjo+tA0iEEXR6mK8WCyqRtatkEpVdbtcrXZ3d8/Pzvb3d/d39yzAL46OOAuFc09fvTo6O1sWhZSSIjpbzI0xlBKCYdnmlBIppXZq1MuSOH765MmTp1+9evVmMV92Oj0eBnGa5Hlu1nbENwLKlDVV2wAMDEaLMm9FWzt7Ppu8Ojyar9ajvb3nL16FSQIwWuWFc45QTsMIQtgUcwCwsko3WmmrtbbQAoCUUgA5xsMgCFopLLDGaEIwdABYZ43HDwwCyPsPEowthBjjgHNGmdYaQ0QQxjQQQlz5jGqFESAYKdkoWTNGsjR+cP/2aHNDyjbLsjiOd7aGYRiEaYgxvDMeCdG+9fhgMpn8xV/8hRD1rf1divDpyflPfvKXvd7g5PAcAosRIBhCgK3BXp8ppXYIM4QRchBCbZyxyAEIIMAUOeikFsYCAgFnLEgiTshqvgDWOGMRQhA61IIqLxhBBDN0bSDvzwOfUHqjhfOmgT67IQzD8Xh9Y7Pjo4T93x0Oh5zTw8PDtm2jqOvDR6qqCsIkDMOmaZS1xiHgkAOIUi+yd9ZahzGwwBkLIWKEojAwxrAw6Hc7FAICnBa1gxZjPNraLIvlarUC0JZ15ZxZLJcYmiggw14i2tJZFUe8mwZ72zsIQif13Vt3f/63P3v44HtW2idPvrhz687OztZ0Oi2LNcQIE3f79sFv/+hH2prnL18fHh6ul8s3b06UEhg4iuDFxdnG5i6PMmmMkDqva6kB4UEQJlmnv7t/u6obZTFEbDrPO1m/sfJ8fnF2fGKM62QJZThKI61EEASirpRsMSZZlkRxp21lXlZa62pVSyk5oYT4ztJcG114Vr9X3CALrs5mK5SXNGJPLAfQA+n+ht+M891VKi8Jk2SdF5WQ55P5ZLF2RhdlM5488RU5IowHoQWo1YpgGkShEtq/uf5c91jRjXLHnyvwN4ycvYzCg0z42tXAOWcBQAgxxpTRHpC4OukpketCasMY4GHUNE1ZCUjZYGNzsLG9rlqIiLSWYYYJghpwetU++ku6GZB5zoTHXXxF4mkKyCAPFadp6nMTlFIYgpOTk+n4Mk6iR48eZVkCAGiaJgiCi9UyrytxfvbTv/2bt996+P0f/ODLJ58vl8sgiBhjlHBGIUbUWu/CHtZtXVVVnufaKkRI1Im393a29nctgjQK9u/cTrvp3/7NX75+/TrLsk4nU0oZ4xjjSYythov5UivZ7yZbW1utqMqyVNI4B5fL5XpVN60UFgkh/Fi8bVvOeRzH1hkAQF23dVVxzmlK4jDinCsh5bqNYh4yfOfebQDAs2fPXr15ub29nXQySBFAzkKrjSGEEHIlZkHwuqC8To/0Rj4A+INJIATX6/XR0ZEQAj+VH3zwAUKoaUSv0/3888/DMOQo/OjjD/+7//a/iZMwjgOp2vW6eP7kL1ez8g/+8FuXs9nl5SXEaJuxUafHOUcAEoybdaEqMR0vYONWi+XovfcGvf4YPJlMJs65vb19B+EnX3yuLXr/nff+4I//6NmTr47eHIbOsTBwjXHIeY8WL7tFCHnPt7Isy7Ikw+GwrmtECQS4qBqlFAt4UdVCCIAQhNdZIIgg6GNGc4wxQgTCK3kuIQSSq74QY4wJghBDBKQWSgnMsTaVc7BpQdMWxhhnoVF6OBymSVTXubEyiikPIMJmYzNrmqWolv1udOf21mDYddCenF1enB0GUcYZJoQZ4+paCq2oIxC5IEowhlnWreu6qqrj42MeUKWEzy+mlKZZ3O93kyRr23Y+n3eSpFyvtVJ1XjZCNk2jjbPK6kZZ7TAkVulWmqooGuC6nTSOeNvUnJMw4nF6kHWHyujZbB6E0eHhOOQ8CljI+yGnk4v16dGRk5BHOcRBFCYIha2G773/jdOzc4fIi9eHx6enl5eTeVHEvQGJki+/esmVSdM46/WNhReXk7ZttbZKqSzrGuDyPD8+Oy/XBeOzbm/w7PmL8+ksSZLz2Wy2WlqCllURsjDudTppss5XVVUwzpdFLpXY2hha4Cil/X7/888/f/PmzdnZWVU2lAc8DKIkEUqFIQ/jKAw5xrASbWtE5Wwy2kghbJ3JZzOaxQfDQdjpFEUhCtsIpYzGmLKAI4KrRnSHHS+ObdrWARgk3FrbtC1mAFoHkQKYMo7ihFODMcbMV5nXXbXfGQEAPkzB/09jjDdetNZSijwIL0QjhNjY2Njd3T08fF3XpTGUcWyd0rqt67LTjYOQGNuu8wrTTCkpVWWdund3f39v8/Wb53W1Hgw29/b2Fp89e/rFUwjo5VmdxqHQChjtnIUIUXK1Uzvr/0MAIwAcghhgiBAi2FprnbXKaWchxRAhSggZjYayruq6Nloaa53VVYkgsARxjxv7jDRrbVEU3jDHW0f7Zs473sRxHAQC49AX5VEUAQCEEL6GQAgYY/zAwt+fpmkcCi1AQhljXBQ6BxFChDFGMFPGSSkxRBZajFAYBGmSgIjLtvXCIisFNFrUcL2YrBbzZ8+eSVmLtonjsCiKd955HATB3Tt7Z2fw/r3bZVmWZUogCgO2vTU6Pzm5OD/f3hoBC8fnM4YDp6Go1fNnT1ZFzhhBBFOCGCNFuQ7CcDDsVXXx4OG9s7OzYr2+HF/s7+06a87OL4MkoyxykBiHCcU8hACxOO3Haf/s5EnWHXbi6OWzp4QlASNgMu0kcV2Xp4dv/s6q3miICOml8fmkgtZYZ6xSzkoIDbCqrmsplVKKJGkQRv5AJRB5ipn11l9eHoagn0EYKb12zvMY4LUU1jnn3RJvCgWIEWM0iFIDSRxGdZl/9fJNGHCNeKurRlmpLXMWcQQQFloLa6x05jqLwY/ePHv3ppu/+RK8NhgOgsAXju7a/uhq0ODcFSlYOyGEB+oIIdY4KQUC0NgmicPh5rZyQGqQ9Dvb+weQcgcQQgQiYoADpJXN0j+DV1MVjP3PucG63LUDhD/22ralFPvEhMVigRCqqiIKWJIkFCOEUBzH/X5/e3v71atXEMLJbMoCThn71YcfNk31ox/9qNvv7986WC7Wzrm2bY1yRjtPUdJaO2PX63XTNMbpjc1RnEUbW5ssCv/DT3/SzdKkkzFO3n///enkMsuyN69eM8YBqAihLInbWsqqQQgPBhv37z9Uqj06fjOdzI2Fq2V+enrBw1RBWpYlpZRQJFqJMcUYQwuN1VEUBUHAGPHPnTFKa4mdK1bLJAru3b1NGCrr9dbWSDtd1mtHUNmU2sBWlAEACDNv9AnoVQwH/o0XpbRtr9y0wjBaLufPnz///PPP0wh99cnTJEn+8A//CMfw0198Sin9+OcfCy1evzzpdFJC0Mao1+9sTi7nz58e3n1rcz6fL5YLHkTdtjVOAwedc7JpOlHCN6nKG6jB+jI3dzQjuCjsdDoty7Jt2/fee+/f/Jt/98tf/aqqm3/yH/0pxtRBsJgu/IgTY9w0TRSFXooVhqHvYfzWRDa2NpeLNQDA7zuE0STJrLXGOQ++IYQQpgReGexbyPyWB5Fnh1JKKWbYR6rzgDJGMAEQulZChACA9WizTymXjbQGdLs9Y8zF2eV8Me717na6iRBN3ehWVIyhfj9LNCmKZVmtL86P+xvb3/vut3bPx//w0aevXp8lSZbEHQCQUtIYQxFGCPq5uwd1y2qtnd7c2uh2syAKEYJpmt6+fXtreztJsvPz85cvX3JG0iQRUuWr1WyxopQHYeiMHQ4GvU63m2V1sTJSWK2kEiCJW1FbI+IoOnr9hobJ+1/7hgO421saYy7PL0JGs3TQVmp+ucCAf+Pr3yPUFUXx4uUhYyyJ07IRv//7fzhbrI5OLi4nY8aCshUvXx8OhxujnZ3VapUylnU6w80tqTRCSAiljJVKTRfzgEdKGimq3qAfJmm32//Zz/5uLUQtVVkUECBIyXw8iyLV6ffWdblYLZfL+fbWSGuJCbz36GE3S6nSBwcHH3/8sR+Ka3XVNPinpdvrdbtdh5y12lqrhbYM7+/eYYydHJ1O18tO1u12u1Vd7+7tC63UdA4NCaKIBdw4UJb1YNgBAFRVJbVACPGQNqJtRa20lkZjSR1yURRlveyKM1U1fvP1FJOAX1nQ13XjuYrQAYSM0RpBSCg1sgFGCyXrpsQY7m1v37tz6+mXnyIHOKVxwHudNI3jMl+pthV1XWOtjRwMM2MFtKYoV91eShne3h7VTbnN9t//4N2ybCFgi/l6ays1CmutrDXOWEgAxti6q0EAxgRhCiEkmBhkMaIYY+NKDDFAABqCrq3BrdLffP/91WI2nVzmea6U8MAjIWQ+nS8WC611FEU3jDAfIJRl2fb2Nsa4KIqiKPzWnCQJY1e+EYwxKWVVVZTS+XxOKZZSDofDXq93eXnpeenrdWEtWBelMSaMoyu7KoB7vQwAUDY1Qsg3rN5oKNeilTJLEkqpNjqKIgLdcmabpjk5O+t14uVyKUQzXy2jNLmcTnZ3RvPZ5XDYHwx6WorT4xNrDScEQeicy1fFwcHtfJlzzt959H7E04vxC4bxj377h5Tx/uZGEIaffvqxA6htW8LZw4cPj98cnp+fGyV3dnbu378/XomyahIUMBZEcQpxAHArlBpPZpub6+lsube9tb+9/eTzJ6t1SQmKOdu8f4+HXFkjtahWqyCNeRhvbgwJWa2rVish6lpZIEQjmsoY7LttjzBbayHBjLH1em2M0c5CDzAQbICz1hqt0bUbEiEkjCJ/dvokT0KIg8BY45yjmGKMHUC7+7fv3bn98uXzslXK2E7WYTyAhM9ms7oVWLswZsyhuhVl3WAl/el7NTi4Rin8J+A6lslfQNM0w+HQFxNaa2yIuc5icL8h0/Cog68bpEPOgTiJjVKYMATdy1eH1oG7t+/M58tWSgix1MKPVtpWInvlAO1/8tXv6JxPA7LW+iqhLEu/03LCOecbGxueZ5MkiTEqSaI4jkPOsizr9/u9Xm9zc/P169dVVR2dnty7d+/4+Pju3TsnZ6f/8l/+y//8P//fv/Xgwb//dz9um6Yqq7aWEFxFiyEnpFFVUWqratWkJluv19NZkNeryWwax+HZ2VkSBQ8fPxptDDBEZ1pXTWMNUEpR6N0tbRwF/X5/Y2MDALNer6uyAZBQOncWJklSa9Q0DcYYI+KAdBAgRCA0cRzHcez9+41U0BkllFFqc3Pj5PTIGFWu1zhA/WHvnfff+9U/fKiBNkYB5KAFWkuJWmysMc7SK6qHL+4ZC/xT7/NLOedCXpnFDYcDzjkW1fnJ+QyR3X+6rYz75MNPHj58tC5Wq9Vqd3OvKNdNK87VJEnLr33wzcFgYCGiQZh2e0EQhHEEAKjbplgvZ5Mp0JYC0o3TLOquxwsg9MXRaVMBb+whxIu3Hj/6q7/627PTsqh/GfDkO9/+7h/+4R/+6u9/9erlSz+jrMuq2+1ACMMgUEpoI7yjRpqmpKolYbyqqrKsWBD68bAQAjpHMfJPF8XWWiilFLVI0qAsakphp5PFUZKmHcaYECIJ2zjmUlUEmfVqiom9dWvv4nKd9hIAgHVm0E21cSykYdAt2+Vy2TrsqqZtypbAMAm6wIB2aQt7tpLV3qiHI/f89ScER5ubd773rd9SzcdCYOIia0FdrHjAEQ5Xed516tatWxfHL6WoKTAP7t3WTqch1RZ1O/sQwr297TAML8ZnQjZBQPa37xH36uTsTIm2myWXk8nG5vDuvdudXvbs2bOiKUd7O9Px5cfPvrp/56ACmiR0uLcxnc4PDg5u3b7f6/WXq3wyb9qmZRzWzTpN9imCg34nz/NulnYGvGmay/F0Op1LrZyDf/f3v9TKTcbz4XD7/Pz89YtXDFMpxeuXL3u9nkOJtrTT2Vzka0PYdLnojzatNknaWSyWQqrFfJX1R++88+03h0eEdlS+zqXBJEYIadnGcQdDMBtPsiwp82LQ6a2Xq04Sbw22todbCKF79/b+/ue/bBsdx2m/Z6QwZV6EQZzG6bosvnr6+vXLo+FwyMMgCIJ+v58is7qYJEmSUIo2N7TWs/llmnYqWRRFYWzbKFGvlmmaUkoJak7PhG+UG2GNUahS1lrgsBICYYwBAspaoTGlEEJnHQ5ia60RQhuNkDPOYuesM0LWwFjCKacBhEaLRjYNADaIWC+JZ9O5Ee3te/d6cbYaL5njnaDfVi3qJ4uZaEWhTDhZtqtqEScyiqKh1Max1lDGh5fj+t6Dt771W3+8XpXrEm5sPgjj4/PzS+0oj7qTk2kYphhho4SoDNAiShNoMQZI1I1nADDOCSHGKCkawIjRkkIYEAqBM1JpqAMWLJaXO9tb48kZZYyyYLFY8QC+fHOxzseMEONcI2rGMmC0VTAJQqgtUG58Ps663ShOgyiL4zhI4rKcYIy73a536vXefM4CrczR4XEQBEaD+SyXwq1XdZqmlLerfA0d6KSZkxpqmwURUBIaHVLCaEoYraUoqtJhq5Bu6zlHEsHmcnK0uTFcVaswYrcf3uHswXo5/+rw1bDf394/GI42Xrw8vHfvDkP4P/tP/vHLly85D9MkO1baj6tbIVb5cv/Wzv6dna+eP/38xZf/6X/6p1EUHTfho0ePIEYAuCiKMMbZdjKdTrNsO4rTphGjrf5/8V/8X8Ju1gBjGO5kg+WiSDqjt955fzJfCocEbEAQZGn6qy8/GQ6yWTvb4v0aris7v721f5vfef+DD+qmXKxWyujTs4uqrSFxo41tjAhEq7rVVVEJaY20IQkVNAhRadpWEUxxq2uH+XB7UKuKuSumqnMOARNgjBlrEQiCwBijdYsQ6nbiTqfjm3ivQRCisd74yAKnbW+YWq2+evZZyLiBVgtVAjefTLXWjFJKCETOygYCR5ESSkJ7pX/xoMWNkZHv12/UEDd2jdABLVXIg7ZtVSO6yZV5YhyETdNoYyJMWdqtUAUACHhAwtRPh0kcG6WqqkaQHx1eiNoqpcoq55Rxzv2hZZXgUeas00YaAyABwF1FWmNEEKMQMIkMhNiqQEiopKa0TjdHd3b3Xnz1bNDtKKsgMohCaRoOASGgqVfIift3dqfnW6Ja8w4q5IrGcJ3Pd4ZblycXf/Yv/9v/7b/43/wn3//Dv/npX+0Nu69evTg+efnue+85UOdFXluCgNzfGQnRbA47rWoWp6/SNP6D737rzZsXH//9Xw2Hg69944Pf+u63ptPpl199Bl24f7A7uVhwTmeTOQLQQXd6fv7pl5+t1su6Lp2DRVG2TobduKiL0sU06jpnGq1pGFMeamsIQUHIrXZJxBAEw4Mt0VSfffYZ55yn1BG3u7dFGSYAYm1Pn79iCh4MtoVyg6x/Np4BTlpgITaOoABxC5GFCBIMCTbAaKcttJhhIfQyL6IogRhHCSkrJ6QJSBB0Nw8OdoNeRo2K+zFL2eKskBrkK2NMOBhuCVn2s95iupBt072/9/jrjz/8xS8Pbm2LujJtIPN8dnq2Hs86aa/VJi9biORg785KgVUhAATLRX56ev71D7aePvkyDvjD+2Gxrj/+1S/bvPwn//GfvvXgkVEgX+aXl5N79+6FAWnrti5bBCy0ADnc6wyllGS1WiGEPOf2hr/gM0J85evLVXOdr7VeFVprQtjNt3U6HUJQUa66vaRpeVWv7t+//+jRg6wTffX8iTQtpdQCqLW+uJys19O0IwEAm8MNv+MwxgJCrbLj8Xi1WOarajDYxHb/8rj95LMza939+7Tb2fjmB98N4+6Hv/poXaw6GaMMXo5f376971rdtm3AeRjxolgXRaGUUEqEcaRKFcRRWZbL5fLFq9dpmn77299ez5swCHqdznQ6lbodDAa3D25tbm/l5brX6wSMl2VurGorMJtPHj1+uL29bwzc2MwxYZwHxgKtrVLm1p3bzlirTZLFaRx1s6ypyrZtV6sZxnhjY2Mw2JDCKGXKskSI3Lt3TyjpBRpN09CAxXGcJMn4bLzDGWGYc04I0lo1VdlUVdM0GNMo4GjQQw5MJpeU0r29vTDt5Hm+Wq2KYg2soRTHcZiEAQQuyxIEXVm2D+7e2dndnlyOoyj65dnl6enpbDYHEHojBABQ0zRVVQFM/J54fn6pjblz586jt95mpH3+/Pnx8alDMMuyKEqksK1Q23HqHESUoVYKIQAg1mEHiLW/js01Nzm210MEv4S01gAhP+M00lhrEUIeS/cdtp+2enaID3lzV+b8CDpIMB0Oh9ubmztb24vF4uzsrKoqjyoRgrw0uWka5ngcxwQTCHiRNwCAgMaAu7quVasYY85ZJaVSQspWypZSHoZh2+n48TOhFCsJAJJaaWt0bjCjhJCmRaAoOOfdbnc47BeiMRgCqwxwFEJAsLaqrKrT8fjs4twovbW9U5Z11TYQY+M0IySKIuyBOYyVVMC5AASEEK2lyKW2tqyrtql8R+gdJ71YDiGUJIkfT3g5O2PM2/L4ubXnnCMEnLHWauuQsz61FQJnMIHWWGt1GDDKOq0Qi+mEcWwIWS/z9959GwO3v7uXrxZJlsWcdeLo/u1bWZTEYYAxbKr69PgszWILgXGwbtu8rAFG/Y2N7mBggKsawcO4kYoFQavkYp0ra/Z274Zh2jQVoQgi04hcSiHUKi8bxl0QBtq0ccyAAW2jqlJGYS/J0uFwpKaLuh3ntSjLkvKw1+tZay8uJ5OLUytlXbdZt7cx2n7v7sHd+/fG43ErzeV4vF7nAJHOIGtbqbUFgFz7FCgIEYJXWhsIod+swLWJxWg08jaLXpBilPpNfeAN01BK6UWqvqTwb5C9zi/1Cw9D5K0OjNJKSg/bAgAcghBCY63UqpHCY9EEEvQb3sk3ns3gOu7LXWcreE6i5y70ej0vjvXBub5kaZrGP2g3XAFKqfd6BJ6WeZUZgQgh/X6/003rur44O8/zPIoCzqkfKzjgn1wDoVNaKg08O8G3wko1EBKCmdZmna/6m5mUKk0zY8zZ2Zk0MgjZcrnsdrPxeDxfjM/O+a39rZ3dze3tjc8++2xzuIkxlmULrAvDcGtrS1TNj//yJ4/u3//BD78/mVx++NEv3nr8sG7yw8PDW7cPOARJOohi+nu//4MnT75wWJ89Ofv61z+YTuacx2VZX15MX786euft9zmLEOSDweDiYuyzNBFCDFMP/7x+/bqsCudMFCXOAc55FFlnoZYIACel1qrV0BJsGbHW4vVaY+h8fHbTCKUcY0EYRsZZxhjlFHreCqbGgkYKZKgwTlnjIECQAIgQIuDae8DXeX5eczNF9XMHj2/5PRBjPOxmFxc1xwRoY43uZtnF6Rmwttvtto0GCJ5eXlT1+vji9Lvfe/+3vv+9D35456snT6FDb14dWm1O35xENOCEZkmHE0qDEFtCSbAYLxsh8nWJIFASLObFfJY3tcaYMxo4jZFDp6en/9//5r9+/NbjP/mTf/Tq+as/+7M/K8uV1syXqv5+enStLEuilGCMUYqdY845QpBSBgAbhpxzDgCQEniLvGvPDeL56mVZjsfj2Wy2Xm9FcVjXZV4ESter1TwM73Y6vTCibz18W8haSjmZz+oqr4o6L0oIkA+M0VJpqynCGAflOr84Pz07u4zQvZ3RAXHbDrYH27au69VMTS9ODvbvhLwLrA4oUrpN02gyraVa9OhAShlGYRQF1mqKCecUEkgQlk4KIV6+eFFW1cXFeHt7+/d+5/embPXhh7+sqspamyTRYGNkra6qYtDrt3XVimY2mQJoMQZWm93tnbox1nGEA6Xdal2BQuR5XlT1F0+e7G5vj4aDoi6CkBigg5gt1rPd3W0hFIDQGHhxPj49PXWQjkZbi+W8qhtCyJ07d1brJUDIWHV2fiLXTdLJVqvFuqyappKikS03Si5W807WwZjGcdK29Yvnz9Kko7U2RkVRENBh00ZNU4m2Ltf5ajHd2drsdtKyzAMKHr71YHd76+TkBCHQtsJahxCCCIVhkCSJtRYgGPCIcCaEmM+Xs9nMWBeG0e3bd7766qPluqiqqtvtDgej/nBj2V065/YO7uR5nuflbDFv2nlVC4KdFE5Jo6G11ip9xbu21mpjIcQQIG9yZxG4NpKhFlpjDIIwCAKMoZTSakMwZoxZpX1+wg3dGkIghHKuRA70djZ3dnZE25RFfutg//DNS6FkGsXQgSovZtNpnCRJFMMgMhJfnM4Rwbu723EcQkcwprKRZZ4TQpVsnVVKtwCaEFASEIddq1WjpVQSAGAktNZGUYScM84ao/zBoLXW1ijZOueANc5ZQjBn1GmsRbvM14vZ/Jvf+Ma3v//dxWJxfnbZNE1rhAkQxthqgyEKObOEAgDSOBSt8FQMraW2AACgjSwrGfFEXL/8KP3qJDPG7yw3gYF+dE0Zp5gYYAhC0P0akW6ahgWcMGacRZTEnIVKVVVV5KvJZGy0vn/vXhIGRholNI4xI9wpHfFAaz2dTne2tjtpevj69dNnXw4HoziOR6MtxsBwuJEkyWq1opSdnZ09ffrUWBXHISUMQtjrDWotEeTGlAgba0wrVsY2lNbj6QkLTCcdXV4cG2Om09n58eT8eNofBRbRi8vJxXTeNA0hrJNmlAVayCSKlMDGmLwoMWHGouPzi+0sc+Docnw+Xy7LslbSsRBjRM/Oxk2rW20AoggSBIE1zpgr6xdjDLmmB0optVRhGPrb5X4jtsOrCeR1ZooxJs9zXyiYK8IaueEb+mO4LEtGKIRQUCZbIdrWnxZlWRrgMMZhFMVxTDgzxpR15TUFfoX7asBfg590+MrAH/+eK+C/yhjLssyPAzwN1iel3dAL/B+dc1K1/liCAjrngDF+rzdWbW9vU0qrIl8sZhCGvi0sqxwAT4CwAHCskZ9lZFnGKK/rtixLo6V11FipdGNMPJ8vm0as17lzMEmS3b3t8/MThIHSjdLt4eHLKECP3noIkZ1MzybrqJt1AsqsBYvFgkGqlPriyRdlvvzB9767f2cXQG2dWq5m2jRluere2hmNRt/73ve01oPhdz/78ot33nlnNput8nW/3y/ypmkaHlw4S+Oot7d7Fzr8xRdPQpaWZekHTBhjKaVSTGvtYwsIpWGopHBGu07gVTBSSW2Ntg5bRyCAWjtEkLYGWZcXpRRCageFlqsFC3DWSRGBtRBUChKEQZRN5nOlbVVLrYyPlQXOIUj8KcsY88DPjYLJm4v/phv3VRVoHFSmE8YMYuhQzMOTo9Nub2M2mwFIBxsbtak7G33KwMMP3n3ra+8d7PY+/eiTOwe3L87PGSSr9TzKQqOMqBuDzNaoQzLmNEDWybq1SnMaOAMWs+b8bJ6vG2sgCXkUIdUK6NDx8SEC7tGjt95+58Fi+VvPnj1DCEFCbvAzjHGYJj7KgHokym+FN42gd46z1jZN4/VC7lqq6zcsaxWlOE6ibi9NkmRndyilaFoyn09OTs7++q//BkJw//7dR48fnp6eXp5OZ4tF2yiCKIQYIdDWAvjCl3EJUV2VWkpO0aifUqynk2NIwHCjU1Xo5OQsX9daitOTY4xAJ47Pzqcuwff2b5XVerg/hBBGQeCAo5ikaZpmcdWUEGOKSVlWT46fIYTCIPYdWC9LT4+OGyn63e69Bw8wpePppKqLs9PjsiyGwz6hCAO4OQqDgOerJWWP8up8Nl8TxjElzlqpLOUMEdxKOZlNXzx/0ut2bu/vff1r79+6u0+ha5pZVbfGuHW+nM/nhAX9ft8Ys1jOlVKYIAihcVopu14vgYbj+eWzF88a0bZ1CZ0FzhAAPCPdKtlL0zCI1uu1JLQsy6KRjDEIndUKAhsHHMWBM9oaxViMgHvr4b2Dvd0oDg72dijFxTynlADgpBKEwiAIrIVCyUbI7eH+zu5od3f7zp39qmnDMHj9+tXz58+llL1eL4wTRCjnPMk6zsEwjIW0TBiCA62AkJoxZN3V7NZaq+xVG+TJfjcMrOt5P77ZDX2hTQiB0BljrDGEYo/H/ubw1ZPSB91BXdfL1Zwz4u5Z5xxy9uG92y9ebEkpkzisqmK2WK6WudGOsaXVDmNcNyWlOI06nMeMhtDiN68PDw8Pd3ZqhFCSRr1uYg2gBI42N31OYy2chRZCjAAAEHiWgNYaIcwY01I1dXl2Crb2thmhiGBnIUSIEIooVQhVRVFLVQs5XyystWHEyyoPIw90FYvVCgNIUBcB6LQRQjjgEED+vA8C6oABEJZlqcVVEqafdHp0wRcNvinxU0OfPAQAcAQxToBBAaMIIWe1tQ4jUFcFpThJIoCR0hoA18mS0cZgOg1ns1kQBePxmO1sf/X0GceorZsfff97k0s5Hk+rMgdGA+sG3d5wMHp5+ObFq5f7+7fu3LvfybpCKABAK0QUx62Qn3/5BGO4t7ezsbmljQ3C8NXTs53dDatdqxuhGmVyyiymyujcqFKb9Oz8iGAkW1OXLbQ0L9tWTg6PzlZ5ESZpb7hhEG5lo0SdJkk07Dvd7mxthpQOhqPVcvnxR5+ygK9WqzCJkzSN4xRTJoReLtdNqzVAQZTxgDNH2laLtkEM+IWHr0mI3tTLOUfQlSzQGOOu3a6MZwNcCwF86KLfG/1W6YGfm4rhRrNQVVWxztumsdYaqSCESkpjTKfbHY1GFgLn3LrIb1Rj4Prlt1ZwbYPoP/cbsn+glFI+taEoCu/c6pwbDoc+vvKG9uiv/1oxcZU45bRxwEAIX7+2m5sbe3t7XigPoK2b0lsL+6IHIQT8sXqNbEEIEQKUQq3butFaa4RV27ZhHCEEmqbe3t6MO+He3m4QYufUzlZvb2/n/Ox4Ort8cP9WnNAHD++cfn4kEE/TFFqwWq06UUID2lTV4elR93n2v/rP/ue/+4e/82//7Z8N+71//i/+Wd2Ux+vJaCu1oNna3iKE7d3a+6u/+puf/OQnRNpeF4vWWoOLvJXChWF0//5bXrIYb3T8IcUYCzgjCKZp6oA1Rnk6nfeMgsAGCTfGUOowNlrrkPEw5IxQhBDF0OcT+TYlCENjjDWiN+gORhtCqEVeyPXaUe4oWRS1UqZu2kYY612DDbDI+n+JXJ+1xhg/Pbx6O5zzhaB/swghphFZEPXTDrEAcy6bFkPkRSUAc6Fk0dS7G5sffPPdtz94L+yk/+Of//jk+Gx/by+O0k6c3No5eOv+W+dHJ/+v/8f/Mw7TXtLX0hgNAsa7SRoQWkrunGtqu5hVeS6a2nLmMGTWioDRjY2N5Wr+//mv/9//0R//yR//ye/1+vHkcqple0VDhLBt2ygMOOfEGOUcdc4Zo5SSfnVC6Bi7eoQwhmHI/e27UesiJBGCURR0u0mSRJTBIOCMYR7gKIraVp6djtu2Bg7v7Ry0lSrypi5aTBnlGENmoTJaUkoZJtaYssyFaDrdeGPQMXaKgqYx0mmjLJtOlnledbsbWUrnk8X21g5jjOzunZy+2d0bjfY2Br2+1hpTVJY5pzQKwyAI5vMpYbQsy8ViMZ9MoyjCiNZlNZtMB9lwNBoSwrSzjNFWyaZpIggvp2NjzNZoiAFcLWej0cb3v/vdx4/f6gz64omer9fdTj+grBFNrQSkTLT57PRYNvXZ6dnWsGjbOkqjjcGAAZPnubaWEp4kyWDQ184Zo5zTXtzPOVUqlkb6dcMDVrfVdDHudHp37twqiqpt5Eqo7dGw3xtAiPcOdijhlGBMibGyEpISQAhNYp5GO4zT9XIxvjwvy4IRKNvm4NZ7y/n47KTe3dterlZVvbCuDSOY0KDX73Q6KUJIKPn69WtMW+NQIxpEQJoxZcqXr54kWdztdwgh4/F0Xa6X+VoKJbVqRCta5SAyziJMAFC+jjbGWWuccxZ6B0OvNkfAIWM0AI5SSAjDmBpjlJLdNEEIuSt5mDXGuGu81+9xV/a0ntgI4WCw0c00xUQJMZ1NVNssl3MHDCMIIgqAbapKSxXxCGOWr3JVqzgO27aNkyDP6yiqgpAX6/Ls5PzNq0PRyN6gzwnd6G/4LTvJhlJKxgmhSLTqur0zTdMY4AigzjmgtfbIrAXYGE4ZY8RevQClhMZZlnV7vV5Z1z//xS87aUIJqvIijcLNfrcuszSJgAEB40oIq2yWZWmaEsLysiCcQYQgwb4IyNclvHZo8cm/NyJkd+3M430UKKVKKeMsoAxRGAQBBMBZa5UmCEuMpRBG6SzuCK3yolBKgQxUQhoLHIZKG6ENYbQ/7N/a3Tm4e4cw0rYtIShLUy3aZVHcunfHEfTRRx9BCCFAWlspJaW81xucnZ0Nh8OmaQbD7mw2S7P4ww8/bNt2Ml5sjXoh56tyJao1IgITIlvRzTpBEFilV/NFGsVlUDjnBoNBbRyGcDmfFVWNCSrmoGrqoqwPDg5CbDm0IcUBIXduHzx69Gh8cT5/8/rmrPXThDAMeRRtb2/P5utlUWmtOQCUUsaYklo7fVMfeLzXOWeNEUIwQn1j5znb/raL5spJ0ztVeMdMv+/fEA/htcsyIcQZQzCBEHLOG0JuEGbGmGdHeiqZdldql4AFNz/EX5gvF24qaf/We96Pv2AvuyjL0le0vtrO89zPi/1VXZ30AEDoKCUYe7swBKyPxNOLxez4+DgMuZCNr3vatgEAREkIALoZuIDrSCrv7EIIGQy7bRtUVSWloZzFUXD/wa2sEwOot7aHADshqyyL1vkq66RFsV4sFlHArLW9Xu/dd989znHTNEA7iCCGiHAWMqpNyyn+5Yd//53vfu23f+d7f/t3P97d2/jd3/v+Ol+t//Ynn3/x0fHJmw/e/8Yf/sGfKOsePHjw4YcfrdZFGCZRlK7XK6WcMY7zcHO0LZo2DENfB+jr648Cfl1g1QgRykJfUWGMGYGQUoJCioHnC3PKPOgCIDTAxkHEGItBnGRJnueAy163n3XSqhWZ0lKZZVnXQjbKCKlbZY0DECDoEPCpyr8hYfWe3D4k4X/invLrYtHYkPG6qk5PT7v9zmq57PV6SqnVek153GrZ6/UQJUrrv/zpTy3UYdRmWXZ6Nk54uMrLftZhPOQ8HE9FPzNSyul41tQSYxoE3FprakMptYY1tZECSGFFq8OAIUjzvIyigGD85s2bH//4349Gw9/7vd/7N//9/7BsayFEGIZpmnrhg1KKmOvksZurxxh7SqNfqTdKD28AQin3PvZ+aTZtdXF5VpZF01TbO5txHEOIup3+3t6e0kIIcXp0jintJAMpjHNOWWOVBdZxHg66PUpIvlyUtaAEbG9uDAaD2erp1lba63U6aeYcfPrkxcsXJwFGSQhdN4wCmsbpNz5497/6rw7Hp5dv3bsrGmGMYYAA5waDQRyEGF6BvW1d13XdSTOA4HI+N0q9fv06fhT94z/5Rw6g//CTv/zyi8/KqrEQDIbDuijTLEYIRCGfzwwC4P7dO7vbO2VTWqCDOAyzCGNsWgcxipK4lUIsDMLsO9/93sMH9yLOkjiknMdEpGkqlFotcwhdv9+XxmitpdRBEPgxZJJEiBLnzPb2Zt2Kuq63NrceP36HYXL05vjli1fYWcKDrdEwDGPr3HRyuVwtgUOtFM4aY2AQsO3N0YMH9wa9/snx4WfQKNFJYk73tt5+6+Hp2XGRr5To56sVC3QQBftxb7Q92NoadbsdyrCDoG2/HUTJZDJ99fKoqVUQJowFlPIk66xWq8vLy8nism1bZaTWVrSqrCsESafTYyyABFsIalErZdC1Rym8VnZZa+E1zHsDsZqrmyD9sjFXBBd301Fdb8RXNC6/dRKC5pNlliWj4eZiOamLUopKioYgqGQjpIIQxglNo9gCjAgVSmmpQBhFQZilHaedkYZ3gqLMQ55YC1arUkqbryoHoNZOCCXlzFor69ZJfcUAdxBBuLG1RSkH1pVluVyupZRBwNMolqulZU0URYwxDKGzhgZRp9PppHGWJtPLi/ls4pTWyoWURgHTUsVxnMUJgtBpU6xLxtju9k6SpBCh6XRat6JVkmHkKA4DRkkErxMH/NbvrRQ8zHBTUflyyjkHnQGOQAgpwdA6g7EBgBAyDHtlUzdtHZuYECyVGF9MtdbzWtRK5E21sb11MZuNdnbTJNraPxjP5tLaqNfrJPHGsH9xelZXRdzpjLa2oiRBhK6K8mIyHV9O/fhsZ+9g79bBmzevut3O4dHrME7LMg+iBKJJGFEeoKrBUmBnqWpBsZRRksoGTFfTy8uZlFY25uTotTXQ0bibJoMsTkKaxFHVVETLLkejbkQJNKIGolnPLuI0KVfzfLXQRnbSLuNIGTOZngup+xv90WiQpCkm561WrZRKtYQGvr2ZLa/SBNC11SZCyCCNMfaIgr/b7lp86JcxY6zX6/kBq9baZ5b6xt13Sjd/0RpDEPb2cbIVbdNACK1zdV0DjBBCeZ5PZtNWSY8/c849DcVep0X4B+TG+hD+xuvmubDWemDJ10aekXBDZbhGEawxphWNvzaEAEKU4itahgsDKduqqtq2AcBRSuraMsaMVRhjZ4FHxb00zitlOKf9frfbyyjFUoq8WNV1PWBppxuMJ8dlNb9z506URqenx01bR1EghHj69OlyMacYtq1M4u75+aWqW2QdRcg6IKQoy5x0u2kn2d3b+cu/+Pd/94uf/Yt//k//T//n/+NyMRtPj6fjSdMWJydH8/nyxfPX9+6+1e1vfPvb3/nbn/3y4vKXRjuPR8YuqutyY2OQdZJut7u1tSVqQylVQjRNA4FjBFMa34Ax1iHf0FsDiEIYY4JIwAJrjX+4IHQAWIicscLh0GHFGOsOe5AaS7hFZl0XQZgc3L8rlT27mJar0hFuFDBOA0QhdMghjDAlV6PVGyLCFdqqlCcVeacKXyn6R3i5XBJOLi4uvvoqDONAKrUxGM6WeZokNEj2et2v/9bXqnad18svX72gHP/W77z34NHjTz76ECjplJ5O55cXk3y25CHAjFPO11U5uZimSScIorquResIploZpSyEGCFijLMGWQviKKnq0ig5HIwmk9m/+lf/3Z/+6Z++++67x4evz8/Pvd9almUQuMViQQhFlGHGmAPGgWsKLgB5XnjwxG9PN4iCP06iKGCMUYbTNLFWF4XRRkIIGSO3b9/mnPf7fcbY0dFRkTdbW5007a1WudQCOWeM09oMe/3hcEgJgtZwggPG+v3esD8Yz1mv1+l20iSJAYDI8oBk+UoaSTaGach5HJNOxt97+95sPpbNSuSwaRrOKWIoieKqrpBC3awjlPSBbKNRRAipWxHH8XqxXExndx/cL8uSE8wpq1GTpFmaJKdnx3udHYLwgwf39ne3Dl+/+uijjz76+MPbj+8HCXu0+SAIoulsgZoqIBGlfJWXQup8vex2+5PpIomi9brq9XqWVbu7u1iQly9fnp9NsqwbREldN0HAEMGz2Ww2myVJcuvurf392/1+/+jsZDab/f/I+pNfy7I0uxPb/T79uf19nb1nvZube7iHR1uREUmySBECS6oipYlEQAPpHxEEaFjQRANBBQIaSiqiIJVEJqtYTCYzM5gZnUd47+bWP7PX3L45/dm9BsfMMgp6IwfcYM179+zz7fWt9Vvj4ejBg/sMk7ZuNssVQ7iuW58SRtB3T55dXc8wo8pYay1lgVEy2213myVwKvz+92/dOuun0XI5DwPv1umNwbBXFvuD6UgpRQkKxh6ltD+ID45G/UHIOWAccs9brUpE68GIef6ZEHqx2M3nV0Koj8Y/rZusKPcQacKQNq3Wlge8KPYQEkhwCJx1CkKAKXYQOEAgVMYYB6Gz2nQlRbibADrfFmxbibRFCGFEO1G9O9cQAowxYJ1z9q0D6O255hylNAi8ei+N2gch10IqIQkhnFPusel0fHk9k7Lt0SGFuKhaB1HkRxBojCEmiBJUFbkeJMPeUDTNeDSZDA8QQsYCLSFjHkS21RX3iTRSC1HledsKht/26yhDiWOEIO5bTwgHOKE+xoxiKSVAyOM+ZhwA4FEvDROjFMM0jVIGwHpx1VTVnds3EbCLxQwAYLVhzGOYiKYNPL8D+Hc3IShaazUElhHPZ5zESZdDq6qqaZqOEdnNBPZtiVR3EjnnOOfaCGd1Z2aEzgFnKEFdSSDjRGot2tpBoKWQTb3LMzKc9KJACymtq8pMW73YLrMq4wRTBEPOtVVCK4dhPBxWSgGEmOcRyuumqes2L4ssKyyABweT0Wh0dT2fLeab7dILeBAEddtw7upqX5a6qQUEzCpQVGK1kGxX5AFcbzJRmzRO5cApXYVhPBiMp9MpxxAA4Pv+YjlDKEzTNE58IYRQIqBQN9W+qa45u7q6SnziB+Mw7FVts9mttWm1llVdZHlZN7mQjZDGQcSBw5j4PkUZevPi/P+7wXddve9cAu92yd3qwfM8Smld1/BtKUO3Y36XjO/eARjjzmvSvQmklJ3fkFJKOMMYK60dBAY4AACm5F0MEr2lJry7XOK3RGf7FseEMRZCdL9h517s/sR3NdCd9vDOAtlp2m92KFZBCAniGGPnDEK0GyPeYjwoQsA5I6XyPA8TYgxUSrYt6B49znmSJEHoYQwm0+FoNMiy3avXL0HZ5MVKmwphI2Q5TQZJGjqg79+/u1oslvNFEIS7bY4ROz47/Hf/7i/mF7PJZMJ8rq3ClBFCjFUQoefnzx0ys8Xl85fPzm4ef/PN7//7f/OHbL/tnd1+//33/+ZvPgt8/l/+l/+n//3/4f/o8eDnP//Ti8vrsiyrqiKEeB77/Is/NG15//7dfr9/enr6zZePOeeyNVaZtm1V4L+7lf2PBy+rpHOd6dB2HwqIEIAIUAgRwsYoQh3A2guDk7NJmND5al41bdu0gLDeKCAa+VHbGwK5XENhLdQAWgQABA4BiGF35iEA/g5j1Y103ZD3Tjd995GrpbhxOD69dRb10t/9/rfU40VVEkIAgmVRCCW//Ozzps032TKI6Y3j27uyCno97EcvXr64eXQSDwcvnz6v9nkyTKCFLPC6lJMpck91Zq83tXOd15VRz1oghHIGhElMKb++uhgMAupHX335yBr0v/hn//OzszPO+Zdffrnf7yeTyfHR4WAwIJ1lppMxO0mq+7d13+Vu/n23n0MIGQ0ppVEUUIa7j3vTiKLMfN/vlIaDgwMhxH6/Pzo66feHlDJKmVWmqhqIAaUEQyiUSpJk0O9zgpE1YNAfDfqjwTDtxe4Rs1Y/f3J9/uq3TVn1er2D6fHxZFhVrZEAQiya6uWLRx9//J6xZ4+++wq4QVVVVWW9yBNCXF5eUoqPbhz5vt/1v4VxOplMBoORHwZffP7l+asXvu/v8ixN0wcfPMzLijLmhcEnn3ycpvHTZ4/TKLx982y1mG+2KyXkXuV3796/dTBuWrnerWezBcIUIWKtjaLEGEcIe/nioqnL/Wbb7/d7Xvbzn/9pFKdXV1fz2dLzgqTH2lZcXV4Nx6O2bfM8h8jleb5arRywx6cnADiPewDYtm2rvLDa9NMeAoVzTgm5Xiyrujqb3mqlMMYJaeM4VkpsVuvf/OY315cX/UEQ0M3QAAEAAElEQVTPaTUZDwl2URxcX16uN8u7d+9+89WXWuuf//0H1mo/oAirolxvdjWlKE4TiGGWb40Bo8lR4MeYoqLcImyePH30+tVF3TbcZ9QAIdpWyIHvCaeFFLjsqps082gUJYSQ66tNd9jZP4KRIdilvN7s6tq2hRh7nud5npSye4S6Z4ZSCqyTSrw1vvydStwd1oe3Jq9fn+e7rCizJA7CiAJrmrK4f/9+WZZF2VBMWmV3u53nRUk8QEgK0drWAmC1loNhz/d9zrnP/CTpMeYZ7ZylcRwDB7Os+OC9e1mWnZ+fy6Zty0oqwxBmhDZFaaWCQRj74TBO3+3dDqaD/X5vHUjSHiasbhsCMHKAMV8JbZXmjC3nC63kzb/3pwgBpdRisZhfzxgmg94QQmi1WSxmRVEMBoNGtB30ESGECcQEtm3bxRwopV3VTfet6MD78I98+13CDUgjpbTO2beLT4RQJ7xz32ulzIq8rCutddent4WOADc8mGRl3u+lVVWHgX89XyJgMQTDNLlezFXbPHjv3r1794qmPjs+Pjw8FK2CEA4GA8796+vrMAw77GNZlsaqfm8YRcloNLDWNm32+uJF0zSYoCiOldJFUW/mpUOo38N1rQf9ye3b95qbom3lZDKZDO/6vo+s0Vr6vi/KHUKon4QYA6mFVY1PkUIII5rEoRwNBmlwdnY6HPWrptFWXs8WdVMsn64Wy21ZtWUrIOYAQgAh5z5m+N2rwr3VA4wxRmlCCEB/14fUvXQhhG8act9SaOu67ngVURS9U+Y7WFb3QfUYE03bMeyKouhKEBil5q3pAWMcx7FDcLPZ7PPszsmdd3Yc+Ef4hO4c78YO87azEWNcl9U7D1n3k33n/unC+oyxLojRad0BDSBySinVWmstgahDpGS7vbG61093u13b1s5NuvhVmHBKCecepUYKihB513kIAMjz/XotGEenZwcOhMbo9fJiOBqdnh0cF+NXr86ZT+I4nE7HURQtZjOEkMeiy9evm0aMhpOL17N+rxfHMYIIU+/gaBr34tVmeXF5Ph71/uk//ad/709/SqDOst3Tp48ZJ++//54K0qpqfI8UhZjPXv/5n//FP/9fH/34xz9+9vzl119/Wdd1msaez/7jf/zlYnk5nqTT0Y2jo6NvvnwcRRGwWAsFnO141e+ifJRSbpFWEHfqpwEAAKedQ6CrQUYIWgsIhYhiP2SEovGk9+D9e03T/O1vG4CwVHkrxHa/lwpIbZJ+b7HaGtdpqAA6AC2wwELrGPe6h5dS+k7S6D5Cf7xxeLO2BOBwNPjw44/+3t//OcT2X//bP+OcX83mk+kxsA5j7DF+eXHhBwQYe3p0fDActx4YTKb90fBf/at/3U/Ss7MPnz1+1rSScNYUjUWQBT5hVBujjQmCQCtACJFSGqM9jxNClBZG6TgIF4vVycnJeDRV0oRJiDG7upr/7d/+7d/7xZ/cv3//8vJys9kghHq9HoSQhJToukr9oNVGlHWa9ttWcu5pa4MggMhVTemcDYLA8xiEMG+3zpje0PP9INsXzrkk6RnjCCEIOIJwkWerxVUQeF99/ipOIpemj3752WQySWKY53vPj7NsNxqPCHae77+6vGiFPrl5i43GLu1fFMV1BS4vruazq81iezAZCMs//cPn/TR+/8F7RydT3/fXm83/98/+bDw5+Of//J9vtbt49Kq0dZIkOPC+PX8+Xy7KsoRJhDEG1gZJOh6N0ygWVTl/fQ5EHfUGDttXr194PsnyXZqmAGIt2tBjy6tLz4EHp6d/8x//ehqF3/vge88eP07S4VlviBsJqnb5/EI0refHeVMRQrFlH7738Y3j4/l8/tUXX1aFrYpNkcCrpbnX60fJCVw1hEdBFK3Xm16vJ5oWaqBqbThUuXu5vGwO7D6Tw+G0KIqLy9Vms/EHgx8cHG63293X30T9eLlcBgmLkzGxra3ztmlQfLxcb0aDOAipxyl08ur83Gf+MBk/uPsxI/28mm/2xcu//neUgTD1k16MMOj3U9/3ijKzwFFGjAaNbCilQlS12I2nvZPTBOKjJEm++265Wru8LD3fg4BJYbnnW4PqSt04uyWEamolpQIIJkm/LCuHHCTQGvvGucMwoW8KqABwXSNfN3o6KSDBnh90HmwIIXQIWIgQ9LgPnYNB52lEUrWdfQ9jiog7Ojmcza9BS5f7XWyiIIx++bsv/+RP/iQZHjdqZR2J/KAfaSmUKltInBSGUqoay/yoyPQffvc1YdT30oBHz1++dM75AS+LHWNsNI7PTk9fvHgRheHZ8Uk/Top83wFGnLH9JM3znDM96MdN0ziH4jjtBQEFSmkbBcxBqJRtm/12pw6Oj2iAk+Fhke0O793Ki+z5Zv7RRx/1y10pa1pkQsoaqjiOAaartqCqWdU5QigOwv54oJTebrcYY6hVnq8hwJz7cRQ1tVBSGQWUsD4P2rZtmyaOQoIB97AxjUdgMupvNxuPB865qmp6g34jVCU1pmSdV5iHdaterxZhktqAcdt4nleV+Xqx9CihlPb7g+m9+01dX19czje5xzlB3pNXq+VWMMa+/PJJr9eLUq9tqqwsnTWDg4kFVkO1y7Znt0/rsuz1esC6+fViv83uH53Mz1eMc8IIBiTqDddbUzY87Y/PL3ZaubT33uXrpqmls3B+segPi122Pzo6Go0mr169qhVI0zhvtHMK89AZWMlKKR3G3qvLq7Ztdzabf1v+/Oc/r5TYGZtZ5/Fou8mxF7VZYwHxmWe0bnWJgcMITAaDoij2TSOlopQyj/rcAwC9iTk4TEl3oBsIIKMMwQooqK1RRvu+H8YRIhhTYoFDBFPIIIQQIwe7Ix8FlmMAMISmNT4JcMqUdIAC51DbtgcHB7tsf319ba2tq2YQDaSUnQwAEHwnMBgHNYAaQMa9KAxevnzpe8En3/tgPp+XefFu5dTtTboFyjvoQuc67PQMQojSpMhLSnkvTJUSUkojtZTK8yKGvaODo6+++CqOU9Wq8XBY1zVn6NbJ2X6fC6w//v7DXZ599dVXGOPhsP/42bdJHFBotwvSbg98in/20UN59/3JZLJarXv+kN7w9ot9hcv33nsvJhF3wTg9tNYeHeCvv3720Uc/idLR41dPhqMxD/x+Pz04nI7GPa2q9P79KAp+8dNf5Ms82+3u3bl77+RnL1++LNaOMytL2fdGRT4fHIS//vzXn/zskwfvvffBxzdfvf76T376sXa6qMrffHrB0uDP//ZXv/iRd3x6uz/6MvQjCOFus3/94nXoh7tVLSsU0FFAA6hgQDCJQVEUmntvMkQewggooJwxHGGAYN20WklnwGg0KPYNcuzixZNsv9ust0EUc4pnr15Q4kdxL8+3pt4bUWvZWgcxxpBijVANRWw86nGjbVOXlNIupMo5L+vCOJ2m6Wq1nE6n2jqtpdPGD/kPf/z+rbPDX/3qNyJrRSan8QS0DjlrRO08oow8GB0tXhWKpcYfN+sFVj5QPoX+i9ez6dHx3Y8++Pz3f1jO5v1BCsNAQWQhQxAzmmLACWuVUcwnrWyEai3SmEGKCGIWcXA5O++nief5m90iHQY3bhxJ03779Lv7d+/dvndnMB5iDPMyO75xRAhFHg/G42HHcUR/hP1CCDlgGGOcM9/nnufFSXgaHOX7zFoghJRScu4nceR5bLVaMUa11svlcrNe37hxDAAoikJb6BwCxlkDAy+Oo7QqG9FIp11ZFIvrxT4vTk9vJn64uJ5dz2Zff/XlyxcvoLWhR5zVu82mbVs2HiKEuvxFnmUYwDIvnj5+Ytq2LQsfY2RNGoVy0/TjREsVcu/Ro0cdALWY5gfTMUE4z/O6KA9Ob242m9VyvdntlTXAIQdBEARNVW+2qxtHh8M0ffHiPA6D+Xw+nR48/OFHWlvRyg7pihBpmiYIku12Nx5NPc97/fr1fD73fX84HM7n87a1V1dXx8fH7733XprG1to8zzv7dBjG4TiU0nDmU8rLvFmtVslJihEkGGmtsiyDDoxGo/FgCN5/HwBEKfV938jOxGTaVoZh6JzpbpOUYs45p954MD44OJjP503T7Pf70Wik1/W9+zdv37vp+yoIgiDwEUL9weDAm3JOLQQIId/3F8v1crl8+vRlVTZhGN1778OifHZ5Nd9ss7cWJ9g2QoqGMY4xJRhaI6VWWuvNZlOWJSLeOwOUfWtL7D48f6y/vVvKWqcBdAihDi0AoQMAIgD6g37btlYbyrDnJ0KItqp3601DG0xQtzqNIo9zbg3oTOCBH/X7GiHU/WgYpwihVtQdJUJr2So5m80ODibc9/r9Xl21nX5GMFNKVaqSSn3++093ux1w7vatM8bYerm6urqo8uLu3bu+5929c/b0u8dJGByMhqvVar9elZRbgCBGUEiHUS2Ecy4idLlYffHFNef0/ffu3blz7/Li1XK++kx9nniYENLr9aw2ntfhE4xo2toYzws8ylqEq6qSUslWAADeNEYSihDS2gDYJeiMMaqulVRt1x7peV4n6XmU+L7PKE2SxFmYZRnzfFK3eVlQwCGEDhjZtFVVQYI59zsbdhyEWkhOPaWEUkoI+d133+V5LuoGApskyc3TM0jwLs9iiqMwHI/H2W6/Wa3XywVjrJfG2PlpFB8dHGa7PQAuidM49DnnSRpq6/rDgcWgFcJaizBBlBVFsd3v6kogQEWrAMA+9RFBiOC6rufzubbWQTiZTEaTMYRwsV7BP0rBdFZwhNC23m02e6P/RmttAGTUcw44C7MsL4rKOIgxQ7Bz81GMaFVVAIAgCDzPYky6KzghLAiCsiybWoC32fcOXbrZlm/Wr3+kH3TEl3ciQRcg7BYExgohhNTq3VKDEEIgsM4BBJV5w8TsFIswDB1GgGCHEcQYWfvGIOkcC33nHKeYAhRQnkZRP46dUhfPHhtDncMAWOc6teMNFsJaq7U0RiGElFIAWEJQgEmZSaFrDJWUQgjBKOYeDkPf93kYBj/4wcf77dbzmDEmTWOjhTZKaEEIYRTHge95XpZlo36Pc44AxgjvtvsvPv8GI6CUCHicZXnTNE3TJEkymUy6ndF8Pp9Op9ba6+tr/IZY3I5Go48+xsfHx1fzK88frjfz7X72n/wnPzFaDgaD3W796Otv890eOvCTn/zI9/litjRW1cqURWM0cBYVRfnoyZPhcHh4eDgcDm+fna5329U3XxICFvPrqrxVZHmSJOPxuKnaTnEJw7BpmuODw06Qe2P2RC7gAWNslhdvziUMgHXGKGshdNjzmJS2q6Lt1i4IgY6LKqV0VdXUqqoqCGQrjRCGMUapIkQb+3f2LACAtcBaYIxTyljbPbmGUmiUxZAYZRHCQkhCqFKaMd7v9yGEq+3md7/73X6/F0JFiVe3ohJtnCaIM5EJIcTd23cmw4lo2zRKdWt95g16Q9no1WwzGvank0MKSMiCKIop4e/EMAFb6vE/dlB2Gw8HURepBW/f+J7neZz3er00DtbrNafs8PDw+PgYACvb1vM8QilNe/Hx8fF6ve2OpDexWgggcvqtrxhCB6CN4/jOg1uzq/lyuWzbjXXaGOWA8XwWRjyJYkxQlu0wxoPBoK7ZbrdbzHeU0rKQm00GrMEQAwvjoPf9D7//3ZPHT775rqirgAd//Ze//O7Rk4Ojw3y99SkZDYZKtAjAk+MjMxmWecYpB9a1dVMVZT9JhZSvnj8bj6efPHhYNvV6ve6HYZVlse9XeS7KWjdCtK2sGo5oyIPjw8Oje8ee533z/Gn3zl4st8zznIN5WTSNIAgb7Tj3ldJV2YwGw8V81U/To6OT5XKV+mGcqNMb89Ummy2WvV5/u8k6NniRZYSQo5uHAfdms9nV5Xe73X673d44/oAQ8uTJd7vdzhkrpRwOvSRM2lZZ88bY3DTN9eUFAo5SjiFq63q9Xi8Wi/FgeHl53ZmqtLZXs/lqtZJSWvAG35HnLQQGIxZF0aA3PDs5i6L0N7/5jZSykfXH339v6IZ37969ffemamZRFEnZ1nUVxUEQBJh2LyFLacRooyXdF2W2L+G60ebpYr7hLLx79wEEZLnaLeZr5yDhnpK2aRqlbdO0dV13snPVNIx1nvE3fPg/NoWBP6pGfBcnwxAB2OlvFkKAAEAQQgg4Zc5YgxXDBCEEjNUEGeOquuScO2cow5RyhLCUQin1m1//rm1bISSEuGuXwZi2olmul0op8iaDahFCURT1Bv3z8/Msy7W2GEPQ4e4dRgif3rjR7/UghP00YYyN0nQySNu2Xc7mpq3u3Dj+osxkVUAtl9eXs9lsbzCmpCwrbUHa70Vhcnrr5t0HH6RR/Nnnv18vF22jB73h4eRktZiVWeVDLwrCKIg79VgJmWWFErKtagKJsI4gXCMsperW284ZQjihyGiltYFvkbqDwaCuawBtFIUIgVbURakAsMeH0/4gHQ5vD3rDLMueP3tZVY1RgmJCMYmjwECEoDNat1XNMBG1sIGlPmeE1nWd7zOrnWxa4CAljMUEY+xzz0HgIGCM1UXZ1sIZSymliCqlgXUgiheLVRSE/V5vazb7zZZTOhoMgHWPHj+pquoOoYST1X7LPN4qDSBcrdf7PDcGYGe1sf00vX3r7nQ6pQxAjLf7XVbkxhgHQZ7nrXpTtM049zyvq0wUTSuEMBDWlf722ydSqP5oyLnf1Lqu5Xh0aA2W2gZB1Mm61gKldJJGWtnO7WGMAcB1FRtvJlcjIYTAoXeZiM5/0B213R72nRfy3V4ZvS0occ5BDEF3ZL4Vk6EDxuE4iaWktiNxYoAQpIAQhjVwDlrjnAXGAKONxgYCADhlVmmHkHWOI5d6LCTIBR4jyGMk9Lkxpm07BomDEFjoEAIIWOgMAsBjBCFGKW2MCRtCiNfr9YySTdMQijklo9GorHLZlpNRj2J36/bN/X6/Wa2vd7OiyhspgiBaLueEkGGaEOCsMZEXEIjSJHZGXV0ufd9v6jpNDfdD51wjlIMN5ZwQUlRV1TSnN28KLRbrhUc9zHAt6pt3bmZtvtleI6zCiChrAXCjcVpX1euLlxfnr5qiko383ae/vvGf/zPO+Xa3vnfve5Qn4OtHCDKhXH65+LN//W+ddv/Ff/ZPgiA4ODhAGABnxsNgnxWr1Uxr7XPe7/XmV981RaOFiuO4G9femfEtBEK0QL0BuzHGMIEQQqOkEABAizEuy9IPPEoSCN1yOVdKTCajPM+bpunGd2e1MUa0smokpT58G9FyoDvx3m4WAIEQA9BZ8brMqoEQt432fa6kwYA1Vdvr9YBtfeL99Kc/PTg43G72n3/5tbIOU+YFYdkKhAgirG2lVnazWk+OD2dX1xjjk7PJd1893sy3HAfL1ezbr57cvnUj9P0yqxXW+21WFJVSikBKCMIQgT/6ejvNOGutUpYQDBHqFprdzbNLf+R5brXpIpr9frqXsmkaghDwfX8yHSEErOvSjxZj0j1HSikLbNvWQjakInEc1nXbwWU766wDuq4LQgiARqoGaihlq7Vs2zbLiu12v1iWSZIgB1VrRduoWnKPpmcx0Ob6/LWHMfHC88ePr2eLXSZvnxzfenjv0aNHUIlPPvqAIJxtNw7AYX8EAPAor5UxrTocTZxz2MFJr/f9j9+7vL762khZFhFjDoKQeZvFEgIAHRz0Bz/6wY9/9KMfcUK/+ebRZ3/43dV2QSk/PT3t98vR9CAMw+Vqtdls8ixLkp7RLvCju3fvPXz/QdvWq/niqy+/tRY8/PAwCNF4PFUGv76YYYzTNJVSSqn7/T5jLPD8QS8ZjvqUSISAUnqz2VVVkedlWdYe49YC56CUSrSqbSUAxPdDKSVnhGAYBl7aiwfD3ma1unz1usqLLCv6/f79ew8ODg6urmZPnz598uTJd99demYDgG3rklFg4yAIgvF43Ov1Fov148eP1+tNf5T+5KcfPTh5EMfxq1evPFJK2QIAIMGIsLyshVCYUCk1cLvz11ePHz/P9qW1wA+ipsFBGMbJYECZ0TDLhNSgaTWC0PND6yCESFtrHAQQS22723x3tr5NZ735sm/pde90hW5c4Jy+mScsNMZYZzGEBJO82DNMGCbOmaZptFKcsTAI8qwkBCvtjHzj4BNCOueU0lVVta3sGm8NM1LopmnSftIpLt0wJKUsigogrJQBAEGIlNLdYpIxxqj3q1/9SmvNCE3TOI3iXpo+fO/ByfHh559/XldlL4lHvbTK9pnWxOrj8XB2tY6DkPrOSG0Byqr61etL3w/ff//9k+PTNE7asljMlpHveczfLjaDmPq+7/shxcQ51zQNtJBA5FGGIXnjOzPGaUMRDrhnoi74p9tGOOc48zEl1to4jjB0HsOj8aBpGqu0Fi0hJAj8IAiSJIEY5Hm+2a6KvIKYcM6Bs5xQ7Swj1KekravcGYdx2zScsaYRXaaDU+qMEaKpyrLX6x1MJlKK5WpTN2LQ63f1u8vl2vO88XiMEFKy5ZQ45xjzOOcQ4rZ9M2fk2Z6yAAqDGHcYV7UspVRa10JCynqDIWeeMXC1WO+K/Xw5s8Ad3phgj0GClTUAAoegdtY5RxnzPC+MoiRJuut7XddFUQhDhgNvv9+LtqgrWeSC0ibt9Xa7fZYVxnV2MgIhdG/rbTGBfsApw8YY4JBzVoi2quqOuhaGMUKorrWUwtTK9/m7Vs8u6uWc63px3lkKuqm3O3abvJbWKGcdcBZ0RkTgFIwxYox1RYjvvO4IIYShAQ6aNyRTp411FkFYVpUzFicR0oRD4BGkm8rJdjoZRlHEOa/rWslGSanVm/YmQjDnzPN4Z93vvnZVcXw0TpIk7cVKqbZuutOYMyAFtE7UVcEZPj05vnP79Pe///1qv3QOxpw5585fvuCch0F0OBrt93sEsRSKDzwDaVW2UegBoCEiVd1C5ChnYRxhyqq6rqpiMBiUdaWMHo5HbdsUVZGX+cHRwW8/+5vr66s79+8JVd66c/PevTvWSgjNo0dfbZars5OzKAhmV/PZbNb9oL97/BJRUlbKOOphXzn4+vXl61eXV5ezMAyHg0FHIQXAUgKkEE8fPzZCKiGt0gRji0wURUYZAxyEkAd+3EuV0VVTVlUJAKjdm+8ehE5LJVULIdQEG6tDFEAIlBJZ1kDkrq4v6rqUUkKAtNbAAYyxMbJtZBiSztlqre3cgu+uQxhjjIgBXXuMttYiByCFqhUB96yyHqF1XeMEIQN9Hnzw4CGl9OnTp+v1GmHsR5Fxjnk+DVErRdU2/XTQNPXyara4Bg8fPhSV/v1v/lBVhRFQ1WBZrjni0+GwKZWBti5a2QoEoO+xwOcIoVbad4rCW2OEdRABALtwr7OmOwwBAGVZKoGiKOKUzWaz/X7/4MF9q7UxhnRGnziOAbQQOoi6QgtqpeimD+dMVRdN0xCCdrvdt988Em1bVaXWmhCktRaygsgbDlPRts7ZJI0wdB1lYjqd7veuKCoCURwnvsfasvAIDhj77He/LTabD+7da5rm22+/HYbeOI5EvpveOxUnBx7j/dD3/TANA9G06/Vyt975B54RUjRNP+3101RrnfoBdiakZNJLn758GUTBar2NfS8rq2KfH984+/jjj89u3l6t908fP/nssy/Oz88HB6O6XkdhjzGPEtY2QkqDEAUOEszzvO71Rrdu3WkbYQzYbHZ/8ze/CcNQKOsAurpcWIicA6JVUZQIIaqqCn0updxvN/ut53lekiS9Xs/3vcV8KWTLmBfH1hlLCNNa523eNE1VCYK9fq9njRsP+xg6UVeiDuIgTJLo+fPnxhgA0GK+YvSxdmA0mQ7GEy+M5qu1gdT3uRIl56xT+Lua+UePHjdN05lmyrL84U+/5wfk8tHrNLa7LO/U77KYn7++WCwWGNH+cOJ5Qbavlsv9YrGBgEwmNAosQmZ2vSxriTGpW52mQz+wShrPCyj3GPMcxAAgZXQnLHcBYvf2C74l1b/Ld4G3mds3GhdjADhjjJTGdhBGgiiGomkhpwgyAAAEFgArpRTCaSOsQ+8eS2OM0dBa5ywAjvg+juO4S68hBPwgYdx/F2BzCGqtt9ttXhZhGPq+bxwUQrSNBgAo6VphlNBKqUo3681OtLWRKgy8fhIP0uTB/fvD/uB/+U//2Xq5evHi2Tdff/3pZ8vxwyln3DmHEaWUWgvqqnrx4sWzx09+9MNP7t+5DYGlECZRWBUjn3GMakqI1aYWEjqHEUqSxO9AvELXZQkhZIQgBzvfUJTaPM9X5aYuK0p56Ee+7wEHtVQQuMD3R4PhbHZFGeMMRVGUDvrbbLfbbYWQu812s9m2bRv4UTdUEc93AHoEj3u92XLVFGU0mrS1wLAyyvo+n46mBwcHCKHPPv+9UToMY0xp7PteEFJKDYS9JLUAFGUNIPTjZExJXZStqEdJwjxfGmusxdyjHtfOFk2DIbAAttJShIjnI4KhkT2Ak77TysZx4vHo4DDLdpm1YJNv6/OqVdILgzdreEoJpYzzzk/dmam7d2pnr1ZNSykVQkmprYPOWQDAeDxumtZaYJxLkh58g01kDoDdbtNVCadpijFWSlVV1dQtpYRzhjFJ06RbgXXzLqW4EzC6hs93zsE/TkZ0dNvu0yhkY4yBCNDOC2kdxpgTmm03HuPb9Srb7z3OMeceD0KPV1p3EBFkDbTGGu2cRRBBYChBaegxQpPQG/Vjq2urm1s3jnzfRwgVBSXA1BR1Xsssywjn2GGnWqmFfPsM0l4Y+TwMqdFCipYyTBBtqny/32qpkiiuqwJCeHXx6sGDBwfjiYGkCyKVWX59fa3qlgJEuOeUVroty3rXIdJr1QS6aXXqYJ6X1urJZHJ64yZEbrVaEUJu37795MmTtm1u37716NGj+eL6enZprT27eXR0PIYE7narO+j0+PhwNpvh0Asjf36lZrMZAaxtxPnL13/yJ7+YjA9+/8XLOI0gZhhCgDBj3mpVvHhx/qu/+XUvCgnmCDJRqd2mHU1iaODf/NUvN4sVRaSf9jjh+23WVG3VVs45SDDn3A8DLCWmxAmotbbOOOes1Qgh6zSEEGNIGfEwV0rUZeF5XpJGHcUyDAOttbNAma7EDnVT4x9NCW9v6m+VVICABdZY3U0J0BrrnLWaYAiBZYgQQg0R2NmA0cmg7xx88fz8V7/6jVQGYMy4v91nzAuss41QGNHBYLDZOIIQQO7B3Xta2W8ffb3Pc4gAg55zuN6rAovp6AYBbpCOenGa+TuCiHPGSGUt/uO9g7X2XSgdQgggJJRGUUQIMl23APbjOPYYV0qt1+vNZugxlqYpcc42TY0x7FIxEDoHLCYISKeUpAxjh+qmBMB6nt+2jdwYCEDbts4ZhEHTlk4bTNzB4a0iyztQiajFycmJtSBJkn7/8ae//+1+vw/4xKMsSJPTG0cH0/Hzp49EnUHd3jo5yDeL8/Nzq9tiVT37Bj54/wPG2Ha54Nw/OT21XvDtV19HYRhQvy4LKzV2Lg2j0OdWiOuLl57vHx2OZ8sZ9/1rJZJev1F6MBoNRkNl3edffvX46fOrq2vfC+++/8F2t9rv8/V6c+/+A+rxF8/P58slhDBNhlK1PqO9dLBdb/7qL/8iDP1iX5Ak2rbZL//6V0JphOmNs5u93oAxr67r7hApiiLLMtk2TVUJ0Q7SKIl7Uuoy2zPG4jjGGBdZnqYpJUxplSQ9jJtucldSF1nWUsY5h+Ph4XQihPj97z9br9enpzd3WfH1o++en1+cnp7evHU77fV//JOfvny9iuOQU3Dj5LCuCq31bDbDAAshRqMRhJumab7++utPfvzBzVvvA/whMPWXX39VFLu8EHVd73Z7iD3uhxDSzbbI9pVzOAgT4IhU7nq+EBcqzwuEiMeDVhhtAKMeZSTLCqldf+BBRDBhDoKiKJxzHvPfBb3ePTlvZ/Y3Y+y7RJC11iiFEIIOAOuMVcgBYDCEjjMCAdBaEkICz5cIr4rlarVCGHPOjbFvekoRVcgZo5pGUEp9n3cbHKUE5yyKon2eVVXFme95HiQYQugcVNJsmh0AwFj3Zu8AoTFWSNHWWgjprOac+2FPojrLi9VyczgdHxwcXc/ngecBhD/44Hu93qDI/7vSunK3kUJBTHxKDicHUZpACL/96uuXz576FB0fHUZRSCg6OTm6e/f2t1/+xlpb123dNAih0A+4xygh0DoCVVcxgCFy0IVBOB6ONOy6ghSAFmOI0Js1uXOOS4YQCEO/aSvrqHOubZu6Ll+8eNFWbfeb+z4vs3zfrL0g0tYFEDqIenHc6/U45+v1WhhXFIVVRmsV9PqDweD09ERKyZkf9L04TYwDo8Hwxo1jKeWzZ88Wm63nedKa1mrOOggBMBARz2udqYpWAuAnkUZoX5S1kNvVWmuNGPPDwCIAIVbastB//epyvlwa7frpIAgij3JGqRBivduGYRgG/pvzFzipFWMsjKOOhPPmRHaOEhL4/sAP6rruBGHnuukBQuSybJ9lWwsBRK7zEyCEjLWHh8fgTbO5AqBDybFucn07xb4JjzlnOPe6T+m7jEmHyunOxj+OIHYvjA7E9SZvYgzqOhgB1IRQTCLPyzD2CQk9j0AUYOJB5BDVThoIAWUcIoMJBBZjPB72MQQHBwcIAoRQmsaybR3QYRwHQcA513os5cm7qaWbnAAATdOUZdntVpRShZFSmMIZY4xqRVcPEfpB56Yqsqyrf/zii6/W6+1ut3MsKfPCWg2s9WiAmSMAaiEJhEJK5GxTlYSwzmaBEGLcHw6HTVsJpRerZVVVCIEPP/zwgw/fv7y+DOPwvffvv7o43+03s8V127Y3Tg8/+eST3/7+01ev1IuXzz2fS6GKPO+E1WxTedRnhH/36Pn3PvzRvbsPn19s86oMw8DIRmrtoDUWvDq/DAi/f/v2o68ev74419L24qQXj7JdzSkXTVM0Mo7TTgTqwsOr1Wq329VNgwjSxpR1rY2xznXtNgBYQgjBnnMGY8QpbZoKY8g593ymlKqqknN2MJ5EUZTt8y5jos2bWkj3NuWIEOqEK2PeBI4cMMYKqRtjBIQQdJg0BZIwAEATjAl2oc+cVb00unXjGABwfXF9fT1P0560jnJP28ynZL/bW+uiKJCt0kpFYc85gzU0BhfbZrfOAHIIA4xpuW8oZNb3OEVGWecgBhgAC4B1QHdgjv/xXQ508UYAHHSO+V7XmSmFUErhNKrr2ig9Ho87+ETetgcHB6RpK7iHUgqtJSHIGNNFh7tsSd9PAXK7fRMEfpIkUrVAou68sNZ2Qo2WIrZeUeyN1QHzOWeibqSUWlvG2PHJ4aPvQtmWEIG6KnqJf3w0SSPei4JhGjlVf/T+7Vsn4//3/+e/XS6XBwfTbL8ttqu03zuYTva77N/9238LAe7Sm0VRNGWRhJGHKUPwcDy9ePVyW2THN04BsFEcMN8/OD6M0r6C0GG6yfKL+Yr5QZj27yW9vKjWee6MjeNESROGYdUICLHnBXmeK5FD6G6dnmVZXlVNU4swDMfjaRCkWusXr147gKIkqesWo64VBhsjnXMIWEIIDUMMAcZICLHdbpVSSjTj8ZAx1p04YRB1e9A07XMWQoh838/zsq1qlpA4CEb9wXA89bxgOBxu3Ha1WiHCAADrzWa13vz6N58aYwil/eERQmg0Gv3iF7+4vrpYzq7X63Uv7g0GAwhhmqZX88u2bV++fPnx9x8eHR3lu32Ri/W6MJoADG7ffXD3zn1E8PX17IvPv3r67EVRVATzNO13SgAyYDAcp2nfWXxxcZlnOaKMUd8YV1YNZWWe560UhJDuqbP6rSHrbTbSvoHB/R2F6Z3u6pwryqxTSh0w0DrrjLXaGsMYcc4B65yxgDhCMSeUEyqd6k5zhABGlBAGoQAOxlHCOFVKbdY7pRvGqNYqL/YOvEmZC8GqtmnbFmMKESKEUUohJgCAN5WMDrZSgJg6VEvVAowhpRRSz2GA6GqTXc1XH330fYYRZ8SjjPvBJz9cPT2/3Frn9wdhlACMhqMB9fjl5eV01M/3m0fffpVtFowT6OzJ0eG9e/c8z3POQYg7/ybGuKqa/WYLIYQQA+sgAEoprQyjdNDvl21FMWKY4CBk1IPWQGt8RqMoEkIYoyI/QA4E3DNWQ+fKsuzcspxQCCGlBADbXWopBFEQtFKOhsPDo6MwDC8uLr58el4VpTMWQqiE0lorZfK8pJT6XoggybKMUjoej51zbSu3u81gMLAElaLBGIe+Z62tm4oFnAGrROsIYjzKqnqbZxjjOOlRSj0/UkZjSvZ5dnV9HSbx9PBAW5uXtQWuqEqBpe/7EOHZajm0Q5/zzl3lMc4455z30x6E0BnbgQiBc5zzIAicFFK2jJE0jaXRnudprRkjjCPPJ4iS4bD3zjxogQuDpGNVtW3btm9ancIwLMtSG+ks7Go18jwDAHQSQte51XkaOj3jTaCgg/ZYi94Cc5xz0AFtjBTCGWuMsdoQiBilZ2dnp0fH2LgqSqIoUq3oTJHTKOx8iOTNFGswAoyx0WhgnRmNBm3bQujCMFytFxyxOPSj6F0DMupmICnler3uYm+ydQhYjxFOQ2stMkq1IgqjOE67xUddNsbYOE5DL1wtN/1+32q32+yX81VZltCbNE1jtIxDPkgSz2cYYo9S6IxRpJdExkGIYNILrbWQQITQ2dlZnufX19dlWc/nM2vt7du3r66utNZh6BNC/IBDNFBK7HYbLzCEkLOzM8bYr379a4/7s9liMZsnycBoGAbJ4eQYWFTty6++fPSjH/z48OjJt3/1rYLOQgcpYIyfnp54BOdZmWdFWTaf/u4L2drx4BAAbETzve9/z2P8228eAQOaqh0Nxt05NpvNqqryg6BDkkktnDNCK2DeQKs8z+OMQAiUsko0AFjOAxJCjPHr169Wq+XB4eT4cNrv9/e7rBsWhZDWWsaYEG9srZ2ojxDqEs4IIYiUdc5YAaAmlGIItQQO6CAKst2eEyRF63OuRNWfjm7dPGkbOZvNnHM88FM/FMZODw8wZUXTiEI6CMuyhBZapYssf/b0qVGk3JXQItm2jFNIcd1UAVf79S4JeJ7XSmhjDAYIQ+AwNMKgt4zwt05ziBDqtmQYwu5G1z0f72iwnXe4u6KUQtR1TboKsq6+AgCglKqqohs8pGoh7AEIhBBB4DNOilI0sk3iEEKAMWYcMcYQcEEQVHXhMZ9x8tYPX6/XWwjwaHJDKTEajcbj4dWrTCsBndlu10a3P/zBRz5jd26d+r7/xWefLq6udqtFGPfu371zeHxCKP/662/XyxVC5OjwJPAjZ6w1YNgfcQIxRKNeevUK1k1V1+Xl7Fpr6ZHkzp07yPNrYy6u5xChII4o8w1wZVXv86Ku65Sz6eSwLMuXL18t19s4joeD0X6XKS3D0D86Onn58uVyMe86Cdu2tWUZhjGlNIpTRMhms6nqVho7HA63261zpp/GnHPZNp2FExjdtu1qtULAJkmCUCVa6XHfOVfXbV3UjAUIIWMsAMhaa6wyVrVtu9/vvTCA0PncS9N0X5Ra66IqlTQI4d2+0BocH083m43WctiP7t+/73G6WS6UUmEYYswghHfv3u0N017fa5pmvV5rJ30Wto2RwlSNFKLxePj69cX561fbzf7i4qqua8aYVjYrC0RYHMdpb1DXddtKjFgcp17QC/yQELbZ7ouiAgCVdVNVRRiGACDP8/Jd3i3nuhHbvI2hvwMAwLe82+6/y3IbBEHo+cgBCKG1zr6pVACcMoih0rKuFUU4SZIgCDb5FmPcNtK5t2YcByGEVVU1DSqroqqKo+PJw4fvU4YXi8W3j54YY7pCyG79wTlnnAuhIHznWSPdMoRYQ73YCyKjlGybpi7butRSYogRZi/OLwCkV7O5bOv57CrLsrYqD4fDKtubsspaUbdtwHgvPYk87+hgijHs99NeP6mqYrVcXFxfGKMSn3HOA8/3uQchrOt6t948f/48iqIoiD3P6/RwowyEMAiCsq26v3O3DZfdY4/xO2EmCL3BoHfz5s3OqNQAkaapMaYuytVqxSmllJ6cnJZliSAJ/cAY43PvYDoVQqi2/erZq3dMnrquyY54jC/XKwDQO+zgdrt9cX6epmmUJHVd8DBAlLStdsA5hKUUm2yflUXke9ooYB0hKNvu6roejvrTYJj2e9JoZWTAmZBqX+Q08Db7nRcE08NjhMhyttyst1VTU8yCIKCUCqWapgk8rzMHNE1T5QUhhKC3RMW/W2YZSvF0Oj44OCibmnOulBpPJ0dHB7PZTGgVBIFSqizLsipbKcqi8X2/Y2kIIYRQzjljVGeJ7/b63ZqmU2c7DawDzXVFpl3EpqMWdh/pDl3QxeXTpNc0jWhaYwy0DkLocR5wDzrQT1IzEirt9ZK0rmuPMsZYfzzu/jjqM0ppF1VgHgXQSdnGcbzLtgBY5lEAISKwMwFD6KzVxrjuvlQURTfEdEpymsbdxpNz/tXz87Isp9PpjRs3FovFHz79/cXlpVKaQJymPUIII2y322FElLNJ3M8VJcRqqUWjG9oCo1HAkiAAjhsuB8P+dp+3sjUm3Oz3Qgg/4NIY7WyYxO+//95gPPjtb3/7299/Kv62kUpMp2Me+HXbDoY96nGA0Ww2+8Mf/vDwex9CCKWUh4fHjx49VsoUeVUUpfNQEIQYMCDxH/7wxUcf/uD7P/z+X/zHv4TOWWi1tkrrwWCgpKIRn06Of/iDH//3/+bf7neFp61DcHpwEIcR5xy6zqCner2etWC32wshOusiIQRBgAhUVkmjrXtD4UQYUEoBcN0eIQx9KWVTl5yztm0xgdvttiiKDsnQCAkBzbLSWcx93LatsdAY4xzolAUCUfeWfavoA0wA45hABKyBzhCC2rZmFBqpPEZEW3k+HQzT169fP3v2rGkaaW1vOLKtPhwMpdZK2zwrrbWE0DgMrVZ1Wea7fLOqjXGD3rBpKkwJxlAJyYgHmPF9P/IDz/MoIdYqKVtrrXP8XR7nj7+CIADAkTctvugdlDOO4+Pj4+4iYa3t9XpdSQoxbesT0jaVtWa7W6f9MeFMGO0cQIhevL4CAFDIdoutLMXx4VErd0LUHiPMo8AqjKSX4F6PYQydE9vt8vbZ3f1+j1A9u14IoWSFsmVWYzjtDwghJzduWAKRjz78yff6aYig/ebll7Kt+wfeKgf3puYf/ex/cuvo5tXl1bfffptn5Z3JdDab/clHD6MoyrJs/PBB0zSDwaDf7//Vr38HAChN+z/89V+cnp1BCr2AE4qs1bdOz85fXW43e21cVjWD4bhqRa8/9KKIOMFCbxB7H3788Jtvv95ut9PDG3F45/r68tatW/v95X6///SL33qep2F7cHBw/73jb797cvPG6Od//x94fvTLv/nbJ8/OnWm22yUAmjHeyThFUfm+3xsMcdMc37rzWS077v3NmzdHw8F6vQbOIsiSiELjVtdXg17/4ul3nudx6olaNah5fX5hNIQQT8aH+/1zhgLMvbK0tSzDKBqP4rqVxrKsuYh75E9+/o+aqrl+dV2s60k8VpVsdD3t93fblXX6erkZHn/P8cF2s8ln5ygKN1eXl9vdZHywr/TVV88WiwUwIAgGQTDSWishrQVAm7astMKU0rpsymrNuZem/aYpX1+sZrOFx31MGUEwiWKMqe+HzgEv9DDGEAGttdTCWIMABAAoJXxGkbNSSGgNjcLA40ohK32MmHHIGKOMAxZZB62DhDAHEeO0A3z5geecW6/X/V406I+cw/PrTVW2QYDCwG9azTg0tu4xgDBUsjmYHPTSyesXKyuQNa7YNsR5URiFLILQUULX1/PBYKDatpUCxfHV6qrrGi6VMMZ4QZikyWB8AACoynK1WhnRooD/n//F/+WHH390MhkdHk6aah+P0g/e//Ds5tE3j74Nk/j45MQh2AoRBoDAVrbyyaNXTdUMBoNbZzd93y+3RcFUr9eHDgx6w+OD44uXl+WuSb2erLQBFkBnsXXa+IHn+16eZ+fPFqJFvtfHGBKCQGSlkavdjHOurQmC4NXV5e377wdxlGdl27a77Ro5pKSNo8lquanq9vT0pDUtYFrqar3feZ7//NmXWtbvP/hoc73102l5sQrSUKuWeLxW7bOXjzGGhFpCwcXlyzhOwyB++vTlndt3x+OxD3Fd1+vZMkpibUxuyigKsryWohn2Bx8+fLC4vM52WwJottqH2L8UiwrIj08/pphcXV3t13mzb7+7/o55PE4S/24gdbkr9rPtPOn31ovXpsjee++94XgcBR7E1EFY5FVZZ4gQZTRmOEkiwom2AmPo+74gqIZGELTIM5976/XWSOM0PDk6/vj9jz//4ovPf/2ZsgYTRimtqirw84ODg34UpGmfMbbP8uVy3daNz704TBhjYRBJrZSyu91OKQuxZR4LbLDLdpODSW/QW2/XyijmmDKqY5AHLuA+D8OQUtrDPqN0NZ/vd5tbt87u3DzL99vzp8+bun71+JuD6VE6mSiloLGY0DRNQ2RIyDorPiKQch8zCpBrpfSSBBE4GI8gQhaCKYH7LLOGWObRMNZaLxaLui6N0nVdG6uSKAYAjKajOI4JQpTSPM+zRTmbzdaXu/XlrmkaVTmk2Wa25nh1enriLH7x8qIois02l1IOBoNNtZmMBiQAZb4TEI8GaRhwDZUyLaHIGtWPozwHthEDL1jldbF9Mkp+fufmSRDGaW/w3vs3Hzy4+//4r/9lWZYE4dn1usyqyWi0ud7J5upnP/vZPn+5vV5csHi+yIhIX39Xxugu5uLRN48oDSsnfDI7OZ7MlueMosXmiT+9eeveg08/+0OSJBGlrahBo9M4wRino8GmLKcnN66ur4ur2Ycffhg4XC72ydERFBY6PekN1vPZe+8/KPMtG/WvZrO6yjmjlDNgrBVal7KxxWQy2eV7LTHrxXmeKyM9z8MeoYhKK+qmOj6cail831/NZ1fXG+cwcLRupNRAaS2BBJRDC6xsAUSqFYQQAkCdZclk6iqrnEm8cFs0xkg/9Mum7veSo+nEo3hxPe/1em0rb5zcHPSnzpKnj5+fv7y6uroeTKbb5d46oBrthYFuxY3DI6WEsZoQRD3/6MbRq1fPpVFJkvhhlGf7ptb7bXlyPEFCyryK0r5pFUceI7yqZNKLN5s1AB4mBFlslDVGEgg4w4xDhJ1SgjIPe0xqVbfKAGwAzspmbMDR8VG+3++rbL1dSSEmhyNCOCOcdTiRqmmMWwupgzgpy5JzboGrq8IqiTHmnNZtNQziqi7yfI/68Wjc88lQG2GE9KPo1q07F6+vLy9eeZwNhpOyqOMYUWyTiBGK+70oTW6fnR5gZPbb7Y2TaRjGgc+MMbv1brcv7txmH37wUZqmf/7nf/7ixQtjTJqmo9FoMHy/108YY1m+k6qJk+Dw8EBKKaXo9Xrvv/dz7n2eZYUXxft9fnyjvy8qxEgcp8tV5vuBgdQ5KIXOspwQElKLoO3FSRTwG0eHFLmQs6v14uT4MPK4sYoRxAgOGAt9H1hbVc1sthiMrOf5/X4fAty2bRBGDCHEmGgl6yUEYYJwW5fA2pOjwxunJ/v99utH3+bb/enpiXOGELTeLA/GEwQgcNb3OWOkrUxdl4fB0X6fSSGAhfPZzOMBgcj3vN1mmfZYGifOgLYVu13WtiZJwg8/ef/h+x8+fPhQ1qIzpqZx0qEzvTDYzq8rUYe9aDabPX/+/P333//3f/6vyrIUrcKIOgekUNZaxrxsu8OYItTlEzEhb0yIm83G98N32wQhRNM0HQcQAAChwwQhSAkhEKMOegshRG/DQtZarXRHsHDOCSGklF2+qIOqurd9EG98DAh1v3NZlhhDa32M4NtdmiMEMe55nocQiyIhhDbGMEbTNM2LXZz0EAJBEI1Hh69evfo6f7Lf733f76h2zrmmqay1lDNK6cnJSSvqXj+RUkIIge+PRyNrNYWgLPJst5H1ME1ThBAn5OHduy+ePbl765Zuq6tX57vZ5fc//CCNIynldMopHc4XOO6xP/n5h4DQJ8+eAlIfHB8dHBxBCL/+6tsvv/zy82+W9+/fv3v3HuO4ruvVYl3kuVPaD/wf/PBjYNxut8t2e4SQF3haawutlO1ydQ2AgdBiZD3O4jQiFBdFsct3z56fY4yPjm8MBgNj1NPvHl3NFwihIltKqQM/vn//xnAwvp5ddoIEhxhCv6Ni19Wu+0FQSp3VSegjaAOfe56X7zfaSM5pF0fUWldVhRHFGIum3u23u9WyQ+dK64xRAIAJGEdRVFg9GvSm4wmQshf6geeHFMdhlBvhnKuqqnMYxEE4SHuc8yzLuly/tqbLqXe3fKNCABlwhFICMakbUVVV3dQWAogBAbSsDNWEUGgdsLXab2XnVRFNLcq6LIqmrHabLUFYCBH4/sOHD/M8v17Mu5REWxaT4ajX6/l+kGVZU9WEIEppEveUNdaCDtTYlYlba4ssV0JyykAQIgARgIHny1YYpQnCiHFrLbBOtgJDpIkiWE/HE89neiWFrAiFjFHlJGGoaer54qppK8Z9hIjnceqx1rZOqM5NhighRlHNMKNVU1NNIUZaS+0sAEBIqVu1z/dKKehcXddlWXJO0zgBAHSdVd3f+erq6uriwvf9KIooc7fvnAAAjJGjce+9B3eyLHv06JHWGjiDIE2S5E0Ezlql1G6x5U5NJ+PjW2ejXgqc3u82bVV6nGqte73k7Oy0rttXry+bppkeDQ+Og96w9+jRo2fPX1xezz76/g8++N5H//gf/6P1ep1l2WqxzPc7Z5VoWkIQgPbw6OTqcj5frl9fzrf5fliWs9V8NlstNmuEodNKmbaWhWzLKPSevz7/8a2H77333sX1VVEUlLFubXTz5s082z158kS2ghAynU4vXr/ebDYmSW+dnmljGOfGmFoJhFBelZCSSrRSKweBhQBAiDBmnPsQyKrSRiVpRCmdzWZCNtPDCSGk3+9ZpyGwDQGmldrIopTn5y+E85qmaYVGmHYy8NsQ+LvoN8IYo3dVXoYYLYVWSilrEQA8DLzI93/20x/FYfAv/qv/6vrV8zCMsR3+7EffH8T+1WIutCI+Bxi1QihjqHUaOmNMr9fzOB0M+lZLitFyuSzCcL5Z3jo7G41Gy9kcAXjz7CgIgny3D8MQvGV+d2qKUApjAuHbXDrGmHDkLEJOax2FodayqiprQeQHjLHNZnN1dbWczwBAh4eHjPHVcrPorzinRVEQ5lHiEWVklEaU01JUwCFCEKXYOaOUaGWDAEyiqN/v+b6XEkwJkcwbR72z6TGlWJqW+Z7nee/dvru6Xl2vl9PxZDSZbFbr8XgaMW8yiQhBD+7fYRwFPtvtV1rxMsuPDiZ3bt8PuH/56mq13D98/5OPvveDeqsW6wXzGcaYerQ37P3kJz+5c+fON998U9TFYDwYjAcGqiDxbt49LYoCQDoYTtebgih7eXV9eHKrruV+ufV4kGdlglnAQ4BwEsaY0dFokvKWEEIp2S5nyInIp4NelG+92OeirRilSRhOR8M0TQdpr23bzS5rGlEW1WazC8K01xv00j6wEGMMAZailK0IPR9SmzVtmRfByeHdWzePDqe+x549fz4ZD5umwQhYq5USbdMQhCF0+2zrrDk5Ofn4ex9/9tlnxhhobFtUEQ+HvXS93PTjKImiOEQM4e0uU14bhWg0Gq2Xq/woe/ny5ezi+tWrV17gj6cTJeRsNiMeDcKwN+oLK6+urv67P/s3f/Uf/nKxvnoHhG8bnYMaY+z7YUWaLvcLAcIEvwt9hWHcbaCkUkrpbid3cHAwGAwo5RBhCGFX3aSNkVIqLZTqYEEWAIIQMAgaYxgjVmsHDCYQQFtVhXOGUooQ6bSv7qeA37TXAEYpAIBSQjDEGA8GvSgKRqPBHz77tMgrCKlo/o5PzD0MoAFQtm3NOf/pT3/65RffzOevMH5TctZ9dJ0jvu8fHk2Pjo4AAF988Rl0llE0Go3CMBwMBk+ePMG1QUkgpZ70k36/N58vd+siPIQHvV692wQ+nw77SRxC6GbzK6PUr/928T/9z/7Jydk/yevG99Wrq1eYip//gx9keZmOfYjQ8DiJZ37TNOlhevuDmx7A6/WWYpTvi322cWEPOLDdrtOkX+QZQg4TWDe10MJYsd3ZurZBEAQhZwwDp9u2yfLVernkHEMI16tZVeYI0zwvIITDwbghZDgcQ0Dqum5qkWWZ57HhcCJVjTEghBRF2TTNbrfbbrcQQlXnPkP79SJJYmhVK+o4jg8OJnEcdxyhsiylaqllm/Vyt10rC3e7TBvlRbGUqqkK5rF+vw+BDcNQy1Y1DcfoaDJMOQ6C4Kos6rIySksHYBcB4BwA0BnNgLFSycD3DyZT5nHnXINC4GjVaEYJxDbLsu1+p5QAGHKfcQ8BqJwzShtonAKgKozv+1EQJqOxaNpity/zIgzM/Hq2nC+CILh5+xYAN4bDYavkbre7ePWSUNRP0jRNnTaUUoJpVVVSGyS0UgYTRB3llDGCCYLxoHMJwN1ul+f5YDCIoqjrWeiawN6lb7oVtZK1kBWlcDwZTKeTXi8JfWrNe+vFMsuyvMwa0TLfY5TXqmltYyBQSmEMgyAghpjKAOg6JwRCoLOINU2jtXxDm4bIGMPIG2+N7/ud7UlK2Y1iHSdqOBwOBoNer3fsQBAEUkrRqjAMCSGejyn7YD6fr5ZrpRznPnCGUex7KQLwZx/dds4o2YDG8kHMKNUUOwJ9TmjkF+X+0XeF5/vEcxRiC/XD7z2Ik1BbMTkc0cC7e/e2cyYvdsvVQkopZF2LGkBtlPYRs9BWQmHuWYeF1LusmK2X+6potYoHqTHGaimBrZWeTA/SJNrk1aefflrXdXdz4J7XvYbH43GR71+8eCFb0UvTDz/80Pc8xthoMESc1kqEvSQvCqGUc+78+rIUjTVAOEMJBRRbDLVzAGNKvJE/2O02XhgMBr3NbillOxoNB4PeyxcvhGisUciBKODv3f/w9PQ08L0/+/e/pZQq7TAhHELrEEBYG/DWo/B3jeHd4KC01kZ3niQILHDG5wRB41QzSseffPjgVej1k/RgMjqdpMa0r2ZXrdVeFCJGaimU0tQBqdVsNhNNG3gMGYsRGA/7B4PR8XgcRt73Pnh4cnJ6fXn1/Onztm0JREcHh13IgmLWNE1dtX7Aq6qx1nKK3trMO0IIMlYqpdu29X0fY6yUqaqqS5Irpba7/JtvHydJL0kCAGDS7yFnq6Yk1MMQul2+SXrh9GC8ywprHfMwIbEQwmMoTQIMQRxGvTRmjJ0mCZqMMUZJL+4PexA5RHFvkJZ1bVvdFhUyLgo87FwYeAfT4UEvLoqQUnx0NLJGrNfLpip7SVwUxfx6cXJ0A2KOoFcWMg7dZ599893Xz7Jsd+PGjbIsRVUutxth9DbflW2VVTnmRDpx/uzlw4cPb79365e//OV/+KunnudpC6palE2LMU16/cvZ8uDomBBijeE+dRAxxgBEgzRJPaq1Nkq/fvXC8zxtJHKGYLBeLeqiPDg4iOP4cDLtpWmaptfX1xbzO7fvWgBfvniFkH/z5i2C+YtXrxljUsrQ80XdjHp9UVcR9z1MV7NrZ9TD+/eW89l6vRRNtVosKKWnJ0cdsW06ngBrv/nq6yAIHj58QABcXs/qRnieBwllhCsh1qsFwQQjxClTPt/vDQQGWivr+mf/8Kdp2t9ttvP5HGEcp2kj2otXrwEAL85fvvfwQRhHz7/5ElN0fHy42WzCoNehkJRSVdUqYZIkCcNo0IdKKSGE1ho4CBxyFhr7ZtXXtm3Xzai1TtN+GIZ3795FCFngjH7ji2mEqOvaGFNVVWHfgGgYY85Yay0hpEs/dpd70bSbzcY51+8PrUXOuY5ST/C7xTOVUgjZKOAopQ6oIPTCiLdtO58tm0YFXjocTrqCnG7B3IpKyrYoqs56RilFCFtgCCMdA1GKpigl32LPYxiCqsgBsL7v379z8/79+3du357PP/nv/uy/bZrk8vKy2a08oE2VES0Tjhf7cp1lt87OJtNxEgVVla83C0rpi5ebzWo5nE4gsM+fP3/66rVFqBDtbLW2Dkpty7LKmgojuq/Ll9fXdycT62QY+U1dl7vicDK9eXJWZkeff/45ANrzAmPEfr/VzgDgpGr7vcnx8dFwOFRKZEXWCjMcpL7PpJSDwWixWud5yTFO4xhCqGR7fHz8ox/9pMjr3//+i0ffPm5FPRoNkiQZDE8wBoyx5XK121ZVKR4/frxZZ9Vm6XNeGNFPpr042mwgAq5D0zvnwjBsqhoB5zEErXHWIB6pjlLLKAGm2Yrdbj/op5PRGEO0326laDzfh0ZTiLBzSRTLVjhtHIDOOQQhZ4xz3uv1pNEIoe164wU+9T2lFAYQ4bCsdS0qP9AY432etW3t+bw3TDinvWESx34r6uVmJWTLOSeYFHkW+/7d27eN1LOLS2f1sNcPfb7dbo1UyLjRdHhychRF0XK9+n/+32ezq+tn6eOzWzeFaDDEPmedWcEaZYzBGgLrtFFt2zrn+oMeJQQhJIWo67qp6yiKOGNVWSIIMULaOWctRijw/TAMPau0loTi27dvnp4cEQoo9h5+773rfrTbZbvdrm5E28p9lVeybWQDw8AY4/s+JRggVNalFAJjzAnpEmzWWo9xwHj3lEFOCSFxGHa+HAAsdKBTEHu9XtM0m80GAOD7vnPu+fPnLGR37txhDBfFnipIWehx3O9F+x12VjHqhZxiZyOPe16w2+02q8V0Oj2Y9g8m03t3bydJXBRZnueNqP2Av76+On99njeIcuYAYL733vsPeoP0w4++5/vhdp9fXF399j/8+6+++mYwGARBwDDpHjHDlLOmrMvr60vKPMK5AsARJIxGBPuxX5a1dQb7HFBsEOpNp2kcXl9ff/7NnzPG6rpGb1lAEMLtdtuNUNcAkmPUi8Lj42MAQBLFRVNrranvIdkqqbTW+9nMABeGMSQYe4x4HECoW2WBAwBAo7pSUAhdFEVnZ6cnJ0dKdfyMlhKOgeOcD4eDw8MDZ20YhmEYIswo87RF2gBtUSu01uJdYtYY48AbaIeU0gHjKPajAGiBEGQYa9mcv3giyt3/9n/zv4LAVfu8yHOgqzLL8rKwGHLODQTK6A7cVdd1P+1ZpWutnmy3AWez8/MwDDECo9NxmkSH49HD+/dk1RR53tbN4M7tuiiVEpxza5y2jmDWti0h2L6tVtdaW2eNsxBZ8KZ4j2CMrQVtK+q67axdwNLFYvUf/vKvDo8OjHEeD6Sq+/0+8X2ulLq8vAiCsN+POacAIAgwAKCuYMA9Rmie7ZBVaRiM+r0IAABA27abxXy3XTrn/NCXzQGipC5qIxXQqs6zNIhuHx+FnA76UVvtGYYecYCSti4wtIN+ihDKsuzy8jLyo17an4ynVdX86le/3a1BksD5YqO1Pjo+yIryD59/MRj04jgeTydCyYvL6+1uff76VRAEQsl9XqrNjvsedG56cAAQPDs7XW83vV5vPBnmZU2xs9BZZ8siXy2pClrOuce5dZoyjAwry7IoirquyzwfDAbGOM8LECTAISk09ehgOC2K6uXLC6nA3XvvUcqNVFE6KHQ2HU/ybDdMerqubxweplFcb6+B1giA0PeMVuvFQjTNjRsneZ4HXuQxdnR8CIx98uSx73sAw/l8XhY1YdRam+92/d7eAtAxKiaT0enNsyKv/Cf+brcL/CiKIinE5euLtmmscWdnZweTQ2MMxJgxVpTZ6elpGAcX1xfb3Xq/23mcG8SttVprJbUxEAMMAWc0AiEqy7KqGiEUApAQ0l3FW2UJIUmcQoze3Ve4HzBGIITaGuc6wVZrLZRqMaYYI8/jlL4p3tVKdS5xp3FnMtday7aVUkgpPa/zcjIAAMaIYgKABdBijBAGCEEEgQNKyqZpSoTQeDx2FjpXdqWOSimlpHVSqoZQMBwOX7w4/6u/+qvLi5nHk7puyrLsyAph5MuWN01lrG7rQik1HCRlmROoerE/HaUHk/77791W5QpC+OTJk5cvzzEkHNndLhPFbhAF33vvzs1bp0+fPnm5nZ+cHP2n//gfQuSwLDBNWoFagefLUkgIKf3DZ4/2RSm1tcAZAJtaaK2LSi9W+ddQR1F08/SW53klKLoIPoSwad6Y7BpRh2Hohb7v+3mxr/JsDu16OSvqQkoZRcFgNByNBpRyPwjqukYO9fvDomraVjRNwxjr9XoeD7u9SecErOt6tZ4RAkejUdsKz/PaxiyXy8V8Q5w4Ho1Hqffww4/iQS/PtvP1Wr887xY0HSU9CvxeklBKA99fVBIi14VaulBAnmdVVd08OjKiFUIO+qNRL9VKrtdbAKwOgmy7Y5jEUQQBwBgnSRJF0fHJSV6VQsnXr1+3bYsZdc55jBdOWO2g09Qw44QQwgHLfHJ4NEHIjifD0Wiw2W0226XV2k+iClhotGjruiyBtkBr3E2vSlKAlGyvLl/nxX7QH+nxaLtcU4rn8/n5udeZHI1xKu0D56AzSqm6qpoaS6Wy7a7I9iYI9vt9V8rV/fouc8E5T9M0DEPnXOf+7rI8hBAPYaeNdaZ7ke93G6tNL4lOTm8cHB5rrYu6Wa422+0eYZKm6VoI5IDnecwPILDE+AAThlGeZdZahnESx/1+P/QDAIC12hLknMMQ5nlelqUxilPWucwGg0FZlmWZK6UQhFrrsixj3hsMx+vl6rPPPuv3B//g7/390A8ePXqsWmG19AgOfB74LPb9KIqcEv344Ec/+tG9e/eiIIzj2Pe5lFKq9vLqinrk4MbRj372o6qpn798Jo2+995da+18seLcX212L87P/+V/8/9qGtEfDOKkBx1QVgWef3RyAp27uHj1+tXFarWy1rbCSGO78CHMQataaWQrW9/3ISUAY4S5cmiTvYnwdGNQ3TQIA+fckydPOCMAgPV6XeVF5PFemnaNWZbh7g3XtV1gRjGjoecZBwCCGGPCqHNOW9MlZrPN7uTk6PXV5Wy2+OGPf/Dhhx+WdfnFF19AANq2ZhRD64rdvq2bb799tFmt/cENKaVzAGOMCMUWYousQ0KoNyiCbhMBYQeuRlg6hwCwnHMLLaV40I9lW81ms2y3+sf/8O/fOrv55JtHTV1K2f7yl3/VjRfGGGON1hoj0gl7oR80QsZpvBNtmqbL6yulVJHve5NokMTOaATgnbPTwWB0fn5e7DPnnDPdRotgRBChBDNKqLPgXeJMSomB4x7h3IvjOMt2+/3e90OCcffBVkpZZznn1/N1nufcI0+fP0fQfvLJJ4RhZLTbLObe8Q2OEQl83w9Fq5y1uqp0XedVsVuuz05PPrhz6/79+1/++j8WRXF9fZ2Vhed5zONJkmhre71eEEXT0bitatu2x4fjyWRSFIVTYrWYR1E0HPTC0Pe55wdpFAVdWBxC6JBr2joIAmPMcDh4+OB2h2GP4/Dg4ODps8ez+SqKU8qC++8dXM8ut9vtaDSazZf7/f7o6OinPzv6wx/+YCGAAMRJ+Pr1edLvMUZfv37BGaqKXRj6YRxZiLRGWlXKAgqBgYB7nueHnHOtNfcCAIkD2A8TCyBhftlKTKR1CCICIaGUU2XLsn59fiGUthZIKWWr0jAGYeSMDT3/9Og4CILMlcjp3WpOgEkCXtaV75E48qwRR8cTYwxjyPOC73/y4f27946Ojh49evTehw8nk8l+nz/67jsHAfe9ycEYIXLn3u3b9+4++vbxZrcqiqo3HvQmvTD0d7vMKQ0w1Fq/OH/ZcUazLLNW7/P9J598/M/+8//i+Yunv/39b6fT6XZXm7eVtZRQSjgExGigFRCtqSsppWCUQoi7fafvhV3BrgUOIRRFURRFhLOmrRBC1rnOG2+t1UY4oLR2lGJ/kHYvciFEkeVV5ZRSzhpjTNvW1lrrNMIAIgfcmzBCd85SSglBCAOEAWOEc8oo1loZq8pq3/3NkySh1NcStm0rhKCUcA/3+31jm27PvV6vu3/gZrMhHpZSllXugHFGIwwDzsLQLwsZh3yzKmkQGi04gdvVHBj5vY/vxWF0fDw6nAzqqq3K8vLiervd3rt77/jGCeFEmtY46cX+j37yQyklEQ5TYoBztcsrLQ3pDwZsV+msbqUw2hHOPM9TSuV51pQlropbt+6cHt+KgnhPsvlslW+q+fV1kVdpkihlIEZHRydhEkopAYDJYZjn+X6/t1aPBoPTW6dnZ2fD4fhvfvW3xXx1+er64Oh4NJqslt+VZR2FCaX0+fPnTS27cOw+2+73+8Vicf7qWVXlaZoihCHgHo8ppaPRKInNaDJGBJ8ej6V149Fgl+27JgmEECM09L0oCAOfG6kIBB6FoUcAAEY1vu/306goCmgd5UzUjRBtOPAI9apWQ+JhjIumbOpaCqE9z0ilpHTOMUwoIUkYSaOttUYYp42Bxg+CMKKUYUIwIaiqs1Y10rQABm1bWqdZhjgnSkgCaehFSdjP0T4ZTygmr1++sEojAJMolHVdGjsYDAAARqrl1Ww5X/ivg+VyCYGNoygMAoxAo3We5865NOkL0YqmqovSAGeMM1r6Huv3kslwFIZhVVXQOqdNlRej/mCQ9nzGGWNCiNIBYCwwVrWi0KZs6jiOjZa75fra8zgjWgmtddfB6HuhBXC72+3ynDKvErIFiDGmrKmqyhgDrKMMA8CG0wPZ1M45L4qo7xkMVSuklK1qIYQU4w6WAID1GGeMPXr0qJ/2AADdTU+2befomp6c7bblo++evzq/Lgsxn6+Hw2HohQRh0bT5bssJDjnr9Xqnp6e3z06mZyNK6Wa7ffz0CUIIAbjf7/M8z4p9EAeDUf/49GR8OPzok4+kFpzz2Xz93aMnmFFlDOWhks7zo+n0CEFS17UUupeGN2/fIwgv5qvXFzMKnNR2nxVBFPnEk62w2viM+36YZVl3DwEOzZdL6MBmvccYN01DgOvSp57nh1HUFXZEUSSaViu93zdxFHXHkWgUQqip6rIsjXZJkkxHYz8KX7++7LJXzth3qwGEUL8/NAYiyD54eP/jj36YZdlX33y7XG7GwxHBHsGYEISIhyAVrYEQb7fbLMscwA4giDmABCDWnV2E2A5AjxAiGFNKoyhqhVZKWaMggcYojL3ReNzUweXrF01D/6//4v92986d+dX12Y3TX/ziF3/9q99h3pdSSKEsgFoZSKG1oBN667IiFGljHIKAYkQJYnQ6HP78pz+9eH35+sXzsqzff/AgieIvv/xyvV5zzjtWmNa6KmuIsHGgqx3DGFv7pkXsHZela4T2vKApq6IohBBBEDQCE8KMKbOyChRrhXFW7rYF0W3jM6qkiX1vh7Gy2qPE1MIa49q2Ksq2LBLf++GHH354/z2CsKUQcIwDxiznvh8EQRhHQRAIIfr9vse4URoHkGM8HQ6PJpOvPv/MKssIacpKi5YRcvPGKWPk/NULBCwnWLXNq8tXrW5aJc5un944+pBz/urVq14vqVstJFRG52UTVi3lnlAOIAYQ4z7qIcK8gFHSGyYQk+12ez2/FNpMDifL1eVml02m/afPH0tVJihgFPmjBCNKfQQQyooSYyyMiYOgWG0BYfkuI4gRHhgLqIfzooFIOOxhxDzPC+NeT2upjBAKQjgejPM8F03bVNWg15tdXQSev9/u6qyICC6z/cXFq7ptAo9B6DR0u+06y3NjVJ7nCMLhcKiFZB7d7TZSq/5wMJxOylYAiiElkGDi8Z/85Cc//PFPRtPJ1WIunZJOCdUuVvMTdOi06VKCQojNPivLklLaHw3zfP/0xfO6Lo+PD4fDoWrFxfkrGh87CyCEBHOCEQBYSlvXLbTQWgcAxJhy5nPOu6VDF/Kx1lZ11ekBjLFWSYwRhARA4IC1zjjgMEacs7YxhJDOqNjN18rzOuZJh1vvzIPd/30TTIJvPBOdYuF5jDJUV4XWglDgHLROa+2khB3Pqvv1VjvnAOc8CHzGUa8X7/Ziu936vu97sVaubWyapnE/Qgj4nkcpFo3J8qzY75aLK4phEPgIuhsnB6ItfU49hhEwUcylbIUsla4h1GdnxzdvnmZZ9vr16/NXz4TRXuzznvd6cXFjfyZE0251VRfzxaLRUlhFPF62ZV7vLRTWtcpIbHkQhgHHZdkURXt3coNSvt3mba21ctyjnheMRweb9XfWOANBHIaD/mRf7OfzheezOIyttpyypJf2+33PY23VLvX6s0+/iKJECDUZHkxHh9/ox3lWO0uHNn358mWeVc5hpVTXrt627fHx8XKJfN+HEIVBP44Ga5yZCIaEA4wQZbpt1ru97/MgCBChyuiusZMF/nDUT3x/06ydaiOfD9JASinrkmMYhb7RknOOEAnDuNCukSCvlVSABj1GaIDNrktbSdU0TVWUnYiitQ6SOIgjrVQlWuNs1TZpmhI+Yj4nBElVN7KGyPQHyY0bhwgDI12+29dFCQBgmAc8hBYFPp9OJwThKq+EtqN+L/Y9DAmE8PBgKoWywDkIyrKsq1q2YrNc9fv9MPS7lgRg3Xa7LorCGtCK1liFEbXQwrc3wtAP+mlPNC3FxGO8yPLdZhuGoVHaQGSUhg4wQhmhWqq6rFwtrEFaibrMPY95nGmptJYAgG7jiQkVyjRCUo0QaQ5OjobDYRiGSimlBKesy6cZq7orHWNUA9AK0YpWa11lGSEkCgLO+XA45JzGYWStTXtxHEZSSoyhMcYoFcex1voPnz5aLBab7UoJsNuWv//0y+89fDidHi3nK6eNkm2/l4wGibWWADPoRcjnl7PZ5esrBOGgP6yqarndF0WWJEkUJ3HahwhDiG/fOgtCb7ffb+fl1Wy5Xm+DKD45PUvSwfV88eTJq4ODA621Vq4s2t0255Q1Qitle2FMoBJUHowOHMJCmZKUhBCMaZnlqhWWe42pz1/k3f1h0Ot5nrcv8qZpuhuLc67f73ucBkHgEtuLk3K/C4PAWhuHkTQ6CII8zz1MJbCcseFgACAMPZ8CpKVSUkIInTYIQopw3Juu1+vJ5PBnP/sF5eSzz355NbtK02G2rxAGhTYUQ5/QXFdKKY/xIA3qupbKaq1lqx3AlIcO4HeO7C4x29154jjO88IaKbWGkCijpVaQUGMdYj4Lg9998dWz86s4jn/4s7+Hw56AjDrYKmOdhYgg4yzQDjrnXGcgqNoGM3J++dpqtS8LSvH3PniYxsn/8O03V5evV8tt6AfHxzd6aVqWZb/fx4gy5jkHi6JI01QphSntliMQvqkr63Tfuq57vbTf72ttr6vaOdcBTBHBTdNAjAjihPOHDz80Rl1eLUldFNPpQcjQaNDfrje50FBbI1ohlFXap2R8eHT/7p1/8Kd/6jP+7//8z+f1nFJKfY9Zo6zNy9I4RznvFj/b9brIssDj84urQRI/ePDg7MZNYOHxyaGUbZZvouHo+9//PmVkNB48f/48jHwhRBj6vn/85MkzxkmW1wi1ry9mWV62bU0IRxg8e/7KWHBxdTkY9LjnX88W77///tnNG59++unXn33aG/Qn49F+v724fBXGsbGyKPLpwfjmrTuPHn/rgBayBhqGQQSgrlsLAMiyrJekrdLKusv5It/tX7++HAwG4+kxQogx3mrrGqmNff369cHBUZjEddXmZYUQ8r2g27sjhNq27SU3Hn315fHR4cX5K+v0/+w//QnnfHZ5ZYDzfX8wGhFGy6b2fL+ua611V9C+bddCiKaql9sNAIDuvF22x4T0hwNEiRf4v/ndbyFj9+7fL+siSuIgCjEn14tZvtoKoaqm5swLw7A7hTtXOaX0xo0bbVV+9913P/7JD3/0gx9+9913wWDYofIRQtYaKbRzjXMujUJGvW7lH/oBY7SLd/u+PxgMGGNFVVprO6Vnn+17vR5CCCDYDQHWageBA2+qCjoqXLcJ63L/Xb9hF0DowuWdsaALQWACCUGEYEUQ4wgAlqaxlJRxzChmFgWB1+/3CcXnLxfOQqUcITBJ4jRNGaNKN2VZdplyzwsAAPv9vq70yclJ2WbD4fD2rVtHRwdStM+fP3357Gm23/qc3r51evrD79+9d/vrL744P395+9bZoJ+Gh1NXVdYcIweKogq8QApdlvliNe8NB0EQDA9G+yp//ejbsq1aKf7sX/+r4WRcliXiuD8caNNaa4eT1CHRHwQQAS2FlNIowbnEEIhWlUWz3ezQgPh+yIhfVyLLivHo8PDggFIcxhHn9Ktvv3n27Nnt27eXbEkpPT4+OT4+klo9f/784uKiqdt+fzidHBZ5s9tlnC9Fa3rJYDQazedzhIgxzhgLADg8PDw7OzO2vbw6F6I5PDw8PDw6mJ5SEjz69lmRNz6UddMGHgfWFvke8aCuS22hMrosSzB04+Fg0Ov34kgrkYRRruvQQ9CAfbnH0IRhjGGHhMP96SSOU2wBQEha0bZStkXSp845LaRgQkvVAQnquj4+Pp4cHAynk5OTk812K51pNxtCiKqtdj1KQNvWraijKLh1+/Tj739Qlrmom9VqtVyuIYRJ0kOQZvuqK/VJ42TUHzRVu19vpJS9JL2+uG6r+noxZ5QPxyNrbZHtynzftiaO4ySMKMVRNPQ8bz5bbrdbjwfOWkopxljXuimLXd5IKY12d+/etdb2+33f91++fDmfzzuSdEcK747UjumklBqmPWOMUibwI8ZYU5dKtlEUNU2TF1krFUAEICi1xZhsd/ur2fXNmzcnk0lHouzyJlIrY1SHbQgjv4O4YIypz4+jY601hrDz/SRJ1E971loHTC9Jl8tlx6geHx0RQp4+ffrtN08ppQfTG1qq2ez6+mp5MD7glFPKKaXD4fAHn3xMCPn897//+psvtNZtz9utN0KoyXAitcuzjHn+D95/OJ2OKcWnZ8eUo2+/+/Yv//Kvj48Pj04OAWJp2p8vN7Pr+auL69V6G/jRD3/44/1+DwDI99nV1eJXf/s73+eb9W46HQ+S/mwxL7Ly/r3w5Oxm2bRZVlwtFuPxFAJrteGUcc6rogyC4Pj4OI4YQujRk8fz+TwIgqat1pvN4eHh4cEEAEAQvnnz5vKabjeb/X5PED44OhwMBvP5XApR1W2nWuZlEfoBxUQoqYWEEFqlu66v9XrLuX/3zn1K+JdffHV9NY/iFEEspY7jmAckjcJeFO822+127Xn+wfExxjgvaqlM3TZKK+MwwqyjeXZ95R0hprtTAWMQeAMycsAKIaqm3ux3DiINUDoYawtGh8cPvvfJi1evIAuJg8gB6CDCb59iCDFEWZZFUdQ0+ujk9PXr8+F0vN1uh4P+4XT6+9/95r/5r/9l3TSex3/zm88PDi5u3bp1fHQ0nkz6/X6v1/N9P8+V53lN4zrXrXWIUkIZIhg559q29TjvNBtjTBAE0+m0LMuyLJXheZ73+gmlWIrmxtnNNIq//OJzMuwdEuhVZb1d7BlkQOfjfq/a7frjgc+R7/svXz4/vnfSIPnvf/kXj88fTw5OAKXCCWFMEASEYh4E4SBZLpeRSR1lN++/n6b9z759/vh88b+b3NbcP753L47D3X67XVxP0ygXggPrqHf3vQ/2+3y5zgf96ePHT6tC77f7/nT7zTffcM6ZJXlbcM4n/Ykq3DrPPY/tL653+81w2L8D7L6qvSTi4YEFFKHo/YeflIWSbdvm5d0bNxBC/z+m/vxXsizPD8POfvcl9rcvuWdlVXV1V3f19KwaikNyJEoEDesHGoJo2LIBA/yTCAOGbNgwJWokETSHM8OZ6WlO9Vp7ZeWe+dZ4scfd79n9w80qzvvp5UNkIHDj3nO+57PupsF/8Ye/+/U3TyGGACEAGylVXS8Y9QZh7DHkWlvMV6nrQo8eTqIkdQJW+F5YlqtxwoDFQiDkIKnqd9/76F/9q38VhJHneeeXL6MoCgIvCGnoe1998/nOXi9JnSLLkiQiodWYc1ivNhmAFDG63GYHB0ebTeZQ5FNFAJue35wcHnz+y08vLs8q7RVFcfv27d6gn6R7N4t892B/sSrmy8Vy+6fOn/+VlNJqgBFpy8Zn3nrd8Kb1fGc42fF9t24qxVuMoe+FZZEfHIym0+brLz8+3I/393qzG+dy+qkQKu0Nhju7yoCq1tqaWmte8DAKI3ewWi7bOh+5Axb6GhkIYQdRWGshgkIIDazregAiAFHdNNfX10m/t91mHXHriZoQpIHRWhkLAUSQYC0FdR1pDNecejhwXSlaYKt+z6nXsiq3DoqJ51gtPS91fX+zWfUHydVs/js/+VFdb4tsGwXuzihdLudAG6Ekb6W1UChbtaZqEReqLOtWCAs8l6ZPXj0PggDFeNEsetgNKclXV6MYxpFz7zj43r0fFvnadd1bt+54bvjNk1eQEKGJG+5kJW2aXBmlI+foBw9dTK8urp9+/bg0VX+vv7+/r5Rp22YyHNH3v/f151/5vo/cnctpxZgfMn+70sxBfogZU3hAIRYIGyVBUejNSmhpEIJFXTZKIIcahDnX08tr2Zo07ffTwebs3PM83/Oqqtps2tHkiLlJA+jh6b0PPvhgvV7/6qc/LcsyHu6rzQYRssjy8cG+BeDVxUsWQoSQpdwUgeu7EEkj+ajX3263n//6s34/ZTiIo54HozffnBfz8v79+yHTtdluNisNbF4VtZLCWozhwd7h89cXUpI43FUCxUEfAHB19TIMYBiT1Dut0urjj3/BG2VNW5Vqd3+/rOTNbLnNiij0CYJ1lm/XK6sVY+zskjMWC24Knse+f7B37DkuhvbhwYnreRevLj988O7ZdPrq8uLWnftFWVPSUwYmvp+maV3MPWaPx0lg+Wo1W6+3UoIk6GWFmF5z5jPXGYUYtSWBQgJjgdGhP2AJhhA+eKfHRdtKUVT51fV53VYWQovUeBQc7O+Mhr2ybhSAR/sHo+Hk7OyqyMuqacf9cd20i9klRqHPEIV+rcTNanFwsOf77mazScdRU5aQSOJgAxqIkOdThjEwCkDsOU7XpBqG4Wg0opR2EC7GeHIYdTbR1WqVZRkEXGvdFKVw2OeffQEA8AOvW9OttVLynZ0dzjlE1mithMYYd0mhFnlFUQyHfWOthTjLCsfxmqpsmsZz/MdfP0MAvPvofS1p5CX3bn1wdCcDALS8QYTuHh9qIy+WUyd1BZI09a+2SxC6QRSfrVev50ul1N34ADA/nqTz5eLNcqag/d73PwQRli745ae/+dWTL/f29m6uZ6vVKtuC5UJfz65+/stfvX6z3tmJq0buH55CTL969sJznOnV9aCX7O3tvXzxbGc09Fy2Xcykai+ml0mv12Lr9pNxfHyxWHzz8oUxxqEkdBnk9fHhzptq7UE8DlC92SJKIyUFxT945+6r89dTUweO2SyvwjDWCL188cRjnlYWWSoaNbtZWYOBJZi4ge9gRK+mS6VUy6XFThoOWq4QIcxLlVKUhpboO7fuikb82Z/9pTHG9+PtOouSsJeOl8s5sDa+lUx2jvb3Tl69ePHq1SvWy2ezTdNw3w99J+BWqlZxyTEm1EKEoCGoLnOH4iSJEIFGWKBw7Pmi5ZEbjwa9alNHftJUJZB20BteXFz8vd/7Q9nyJ18/1kIqkpaqshb4DsUuxQhqoFFrTCOkEYTi+c0scFwrDNLwR9//cWbQ//A//H90MK5FmdeNDSKd7L1Y1+XZ2enpbrgTDvbS/Wy3erxt6iryw6auKKIMQ4Is1sYaRYnjeQGldLPM1rNNFMZpMgy99FJeb5aFdeF4b0/wdjq9OTk6/OaLr377J7/17PHXZG9v/8mTJ5PJpDsXep53dXlhrDZW+76rlIxDP4mibLOZT2/aqtay4YYDLRE01nKHhYzCusoYslWxpRge7O2FYXx1fpZvVy+efcOtCX3fdX1GPYydxXzzi5//Oi82Uspbx0fj8TiKos1m/fXXX+Z5eXx8vL+7e3F2JqWcTaeMMQnAdr3u9Xqb7Ur6vu+7w37PddzlYtEF4wwGve12u1qtRuPB/fv3CUKDweD5i6fr9TJKQmttmsYWGkQJdVjLOUAVBLRpqjyrwoDHYdq27XK1Frz1A1db2DV515UAFgthKivG4x3HoZPJhFAmpQwCz3UZtKAVjRGtltw6uK5rCMHezq7RiHgsDNKm1hYiCHGWFXX9YjLeybN1XTdxGHnMMwZwLrNtUUnQNLIqFcVqm9eN4L7TZzBiiLskdZmPgWh1iyByXIfGlPMzgCB1KGaUuQ6m2ILAcxhlmDEcBR5hGEJoCa6leOd779WffH5+fj6bzTBlmLKibKW2AGFGnW8jSCEAsIMEpFBSlK7rdsmAECNjjLKmwy0AANACjDFFGANolZZSRmloDMDdGUfbLpEGY7yYzylG0GqhpG1qhxIGQFWWO73BwrQB02lAldHZZq5V5DFMEDjc3w2DgBIo2oYrPZsvLy+vlVWO43hu0PnRhFBKGi4VpbThQgitpYmjfq+XdAGjq22mrGHERrF/sP/w3Uf3x8NkvVqsVovADcq6KfPs+vxCNjpbbbwgSvbh/v5+4PnFltcQVet6cbV6/NXTo6OTfrw7ny2fPXvt+esgjlTjzLaV0gIT6HoMIZRlW2P5GMX9gY+wVrqWkkvBJUfAUodhgl0vGiplhAI3y41oeNUqRhxMWFaWvBEE5WEYEgQAIdhxIKFV1fz615+8eXPueR5C5OTkFiGkKKq3mlAhOedF0ZW0ESGUMXS73Vpr8zxvqjJNU8ehr1+/Pjk+/IM/+D1j1MsXL7755nGWZfv7u1LyssoRIWEchWkICLPEgSCPfaepNQagbcrlYmblVsnMdwe+553ePtFaZ5vVV4+ftEJSjLUUeZ5fXV3t7+76vlfkW6P0IE3CKNZaDyeJaLlqmyQMT/b3e1GY+H6/1/v666/Xefbq7NwyUnB+dXWJmIMILpViFIa0n/pJb7ITusgldLVYYoiCIJB5XZZl08im1QpA6jBlUCsqrQk0FluDEWobY7V6eP8BJjD2nbIsz8/fTGfNzs7O6P338rIZDodRlKjUWguiMD466D168OhXv/zk8dNns9m0aSXnHBKICIGdHg2hThjYOSSB1l18MiEEWNhpfrXWGCMIIXsb+Mi78bqu67qurbUHBwedpuHtV9M0HdIG/060ecfKdUmRs9ms3+9DZK+vr40xURRBaIuicKCvtBBftUpyY/TD+3cBANAa3/fPz8+VaOM49TyvK5oqiqwpM85525ZZvg08GgQexeF2NUvTOPBYHHqXF+fnUp29fiE57/cHk8kkc4rVdoMxHgwGO4f7rusWWf78+fOiKOM4Xs4XVVU1ZfV0+/Srr76a7O3GYXK4DxAi20357MmzXq9/fHx8+/btR/cefPH5p5vl6ujwpJ/G89kUGNsu13t7ew8evMMcbzZbbLclAODo6CjbbLfbbRKFeV0sFgvXdYPAC4LAgbBq6r3DvbAX/vBHH/bHvb/86V93X8pqtfKY5wy8uhZtq7S227yKEdxut92B3mGk8/gBALSWGEBtJIRI8nY02ONSAIAePHiglJxeXRdFHqWJ53lCCEqcPM8xpp7LtLY3N3PGSBBF7733vnJwnpXGZEIIIY01GCEMIVDKQggBQIy5ShlCSJIkTdOUzdbzvLxYYQI/+uhHjBHeVGVZUhpt1uvlMh8Oh0EcVVXz7NmLPC/cKOhoC4xx09TKaERRBxIzxlqlJpNJtll5nrderz/66KMwDPOyuLqacQkgYlYrXleYWIyAVbIbOgkCo9GIYrZdbQhGHXJgre344s4tn2WZEMIaABEmzMGI+kE0nuzOs6yuytGgd7o/fu/ROzuT8XY5w1aR8c7k8vKyM9T7fnfVYByHGFpj8GKxCDxfcrGYzYGBh/tHDjPWKuxhhziM0fGo1wWnu5HbtmWa9u/cPiKEBQGd3awff/3p3sk7oUc9N47CZDTMZrOr168uzi/e+D7LN9vRaEQI4qLxA8cPnNt3jpCxUCvN22K7PTo6ogxHPpsM0zigXfAkIURrrVvJ/Oho9+Bg7/DJkyeb5UoIEQSB3/ni3ob8NEHgHR3vCSUsgtQhy/VqW0GtFWaEUZcQwoWwAPV6g6qiUZz6fqC1Lus635ZaAc4lx6A/Gq5WK4yh1irP827V8DzPamWkQAAapbNy7VDiu6yfjLbbbL3cFnmNmSNkxWsBfXZ1NZUtH48n+/v7i5tZW3OHuv10wBeQEggNlVyX2+pmtoichAIv8Qax3/M8rwR1Xdm6aptGuC7qj0dlWVqgNQK1EkYrCC2yuKma3XhSCqEATOLkZrkyxnz44Yf/+B8ffvzxxxeX14EfMs/HpBUKAIiklF2zSefzFEJSagkheVZ0/m1E3uYuK6W4kkII3/e7+H1oQdfqBKRmQ69tW6sQJRQq1UoBoHEY2RmP6jyz0mLHodBEnuOliU6FaQqaUtcjDuZcC4Bp6AAvcDhvd3fGRVForVqufR8j4vlh4vuF67oIkboSeV7WTSmF1tr6XoAsoogaaSl2ECBWa8n1zu5+GPkQaKns5dVMG4GBZdRJo/TrL7969epN28gk8DGQL558leflhx/d9YSVQci5EFytl5vVdCUqZSUqVs1iXixvKse1SUPqVretRsRYrRFBjufQ1s2yarttIIZStUoLrZVSUAhqLUHQpdSTgAICDcCttMpgQBwFyKbiSoi6bAhEQwt6acp8ShkzmDZFXZZV29qO9KlrzRgrSwmh6myrdV1XVSWEYIwJAQlS/X7/vfff5Zz/8uOfKyWiKIzjuKqqyWQkpdzd3cnzbL1e7u5OEEI745FQEjOEMADYQAZ3BolWB29eXyGIXOzsj3qH+0PRZq5rNa+v3rwIohhK0RRboXQShYHnWwuyLCvrWiiV5xWGMEygB5i0/M3Vme96kef2dkbRoDedXv/y168oxGmaXt9MF4vF6OCgn/SmN4ttXrhe0FR1f3d8MBqNh5HWDUWCQqC1dQlxE6dpeFuXWgGplGxAGAeaUUwZc7CWSrZCNAoZDbW5Pn896KeTOLl7eNDzXKLkII4Px5M8EAihkFA3DvKyzldLZNHR6a3333tXSj1brBarddcErYzU1vmur7JteVmWQgj3rebLdRiDACllrNIdLQ0AEAVvGt4R1p7ndQnmWuubm3mapp39TAjVtl1mHdNad9O5MaYLzu/swXEcO47Tpbl3idGU4qqqZtk6Dv0syyC0DsN+GCIEMCRpL3781ddxHE5Gwyj0i6Koivzi7A3SXFSbfho7WLkubeu8FzrDfvrVV1+MR4Of/OQnWZYZTH7yox/c3Ny8ePHq6io4OrnVHw2TXiqV8uLwiy++enN+7rruYDCklNZFGXr+8HZPCCGa9vmr15S5kRtui3LUG663m93R+Pd/+3du3749m948/fLLNEn2d3YgMGAEMQLrcnt8fHpycuvi8vLrrx8rZfKizPPcc1ylJaWU+L6UPInD0WgUxcFWVIBY1TQAqvV6/sH33vvq6y82WcGYWxc1jiiyRBtAiUtDv5Mwl2WJKemlA9/31+t1zVtCaM9xgN0ACIMgOjs7E22KEUr6yaMHD16/ft3NakIIyRXGmBLSIuI7juu6QpnFck0wDMMwCSMDreu6jDV1JbTWBFOMqLBtV1P3dviD0FqYJL3Ly3Opyp7nc8nvPXznv/6nf7zdrF+8ePHs2TMjjOt5pGp2d/c9L8iKcrZYW9jF0rx1T8Bvi6a6xq+maYC1XfVSWZY7Ozvvv//+z37xl3meIwwC5mkNsywDVk8Gg5t5Dqz2PVpC23UmpHEPQsTzwiGYIgytVcBAALUF2tpWCCmVUoorXVQcIaKUkVI6GBTZNjkYP7x354fffx8ofv76he9Q0jbi3fff++KLL3zfCyP/+nK1uzexShdFAaxJ42hnPJa8KTfZ4c5er9fLi2spBcbYAE0IOTnai6Lo+voaQJhlNgpch0LOa0IAofBmdmVwX0vQ7w9Ho8FkvBdHaVVuB4PR69cvF/P1xcWl4zDXoScnR4yxyWQwu+BJ4DsYxYGzvzPAGN++fVpVFa+U1lpzaARu2xZqk4ZRGAWbzSoIvGH/9uHhoWj5dr3Jsm3TNOPxGEIIkUnS0EKlgTFAz1eV0m3gJ4cHd3bGp0Uu3rw4k7IYDMdpGidp6AdRXdeEMMf3rEGQiFZWnPNnz54s1yuMcZblw+FQCm4opRgpabXVqm2U4L0ocAkDGr55fXl5NoMYBSHdbtZtq/o9H0LspoOH9x/0kqjJy7LI27bN1pkxBAOteG6ZdbA1vJBt0QuCZ998VWS9pN9jjue6jDrE8zwviFiiuvWLUgwAkIIDYAzC2HEBodooN4yZH1RVBSAUxkZB4HmBMWC1WhHWYMoMQFy0GNOugs8ozTk3SjiOE/qBAVZqBQRH6u3s2VU+BrHX1Q4FjhtHUed8C8NQANMqqbX1CQYIGqOs1ZQyn7rVek6A3R+OXIKgVhgCg+gPvv89x3PrRry5vFplmvkeJqDcrm/fu//Oo3fPLi/qtoGQKI2qWrTcYowBQEqaLnsfGEspZQwjhNI4pJQCABCwxELXD9Mg2hZrAA1G9qrN8/W8qfYizz3am+zvHvziP/7i9ctXSZKenuz0egNjYRwF7Wr6qth0TbJV2cwWawPgyfGBVuKLr7+6vLqR2hLqzWdr4ji7uwd3xgdnby7aViCIPdcXQkih1sumrDKIDCGIYAdaRCAGgALjNlq61LEYWwMRdTyKrLJa6FYaDbsWKscgbKwRLW+kcixw3SAIoi7B+vJy6vt+JwXvjFhKGQgJY5gxhjELQ69uqs1ms7+/+8EH7y8WMyHEYNhL42g6nSIMb98+3dkZZ1mGCbx77zYQ5Xa7rtqKS1EWBaprzIJhEgT3bjW1rKri7vHh++8/EHyzWt+sV7O62GJoCbYuRUopZIGxKgj8k5MTgDBCaOwHCCENwDrPAQDpqOe7nmib8+vL7Xa1vJnOpzeR6+8fH/aNQczrjUbM8WaLTdNwn/kGiDQMYt/xHaaVch0W+o7S7TbPXTcgaG2tdRyXa1vxRvAKIy+kPvNdK1UjubAKWE2hffLVl4N+b2c8vHfn7jAOj3d2AADFci4MwRhjL/AJy+T28vXZi+evzs7Obt99uLsz7o+G+PnLm+VCamGMsVZ3ooqmwVLybhvojrPf+oYYxkYBAcBbZW4rhDJGKZWXJZey86FZa1+8epUkieu6WZY1nAOEIMYWwr+ztdi/UzFlq6pqmgaTt1R3lmUAGEqpMrrhUlsLgDna3e/UwZ7LPIdZLfePDwe9IUEGWdPwartZJB4+GJ/euXPriy8/advaQc6j+7fHk+HLZ1+2dSF5/dUXnyZRnMQJgpP1aiG1JoS0gl9cXECEYsEhhP1+/x/9o380GY6n0+nL5y9Q51TSJuz17t1yRqNRWZZXV1f37t2bzudBEOyPRuV6/fjTT+u8+IPf+R2t1MXFRb8/NMa8ub588uTZer29mS2m84XneUIqzrlRnQST3zo6FLwpy3ww6E2n14IXzHVW10uM8d/87K+/98F7D+7d/7M/+7PReO/k6IgSD0HihT4lgdYmiiIusu12C1oxGk6SJKnrGgNIIIzCsMpzrqTvsarMm7ocjUY/eP8913EgAAwTlzkIojAMmXSEbDHG2gDOJbQIMGYVWK3z1TIjAamqBgBECGEWUeppZTiXEFMIodbGaqOVUUq5rrtYrCAxGgjmkr//9//wwx9+cHV11YpmOr3627/9+aA/dBwnL4v5Ynn2+rwoKmv/Uz8OZYxS6jGqga6qajAYTKfXURxvNpv93Um22f7Tf/pPPc/75JNPzs4X1oIksoP+KPRY7DMlKyNbUecOBkaJ9WaZRoMoiU9OTuYX5105pJRSG4lI1ytmKGWEulpbKXXdSKVaY4yFOPWIICgO3NClSeAW2xpq8f333iFv3rz50Y9+1B1Bev0oTkIjpdKiyDeB6733zjt3bt97+s1TYO2o1yeESeYQiDDGymrKsOf4ruu6riuV8X1fGfPq7Hy7yeuWe4GvtG3btizLxWLVhUBHURoFUZr2hZBtVV1dny0WU4KQ6zFCSFVVcZTcuX263W6C0FNKAGAODkZ/+7fPLy8vGXWTpMeYK7iy2l4qYO1l3hSU0tFggBAyStdlRSk9OjryfbeVVZ5vExqHsWeQrduGy9z12WRvuH+4E4ep0YXjBzavs6Ia9CIAScsl5xJT0h8MGXPbRtjt3Fh1cXVeVU0cp4QiYK1seS6ly5iSsi3zWqvd0ej9dx4dH+6/fnWxmq8opYQyl3l5VouWT69uXJcFQZBv19iqwbDf76Xb9fLm+nw2PddaLxaFlj2MqdGrKrtkeAhtXmW1lJuo1/fjJAzDOI2TXg84NkrijuGDECotAQCMYGstwoByx3MDKSVhjCD87OnLk/1d1/H39vbrhkNM094IErJab1erTWeVcX2PMSY5/06720Gv9u/WRhsLjJVcKKWaqrZCIW0RsEBqRCA0FmhljDFKA62t0cBIzmUaR4nHdvr9fhwga9bLxc319Kd/+e+CMOZKbbKKuP5gd99x3TgKZrPZ3v5BnpdcCMFNbmrON8vl0hglpAYGdv4oShjGFFi0XK55W9WVbNsWIgsAiOMwjmOHgtAlUrTLm6l3sPvg7r3bp3cYsp99+mXb8tOTO0mSQAhv5jPX8cMwvHvat9bmZbXZ5spUiKrRePfho/uv3lzml9uS52lvFEZe1fJBv3fvwR1vSFphrq9mXFqhEMIBxhATCyA3RmuDEGLWUgugtUQbhxJiIWqFlq2EAHuOT5lDsOFcOi5hjGitt1kmFccQOQ6lYcyFEJsNhFAZAxCyEGJKZVkyxhzGusNoRxt12VMAmJcvn282qyQMgiBIe/HDd+5WRfn4m6/btmaExkmYF9v1ZvkHf/AHBPoaGVBAU5eyzHhZOo7E1L9369bV5Ww1u754/SpwIUKi5UVZloS5SptO1sqVLcvSEjKe7KZxooztFKwQQq11VZRlWaqU4uFQK7XeLlYIhq53+97dYX+APCfFA41xUZaAcwuRtRBjmvTcRjTT2Q3Ck+Eg7ichgKpe122rAFRGQ5d52PG50HnR1GUJMVpu1k1Vu4S6lMRxLOpKlGXX+XR5foEMSHtx4HoY46apMEa+73u+A6EFAABoizIrXjWrbUYdz4/ToiiqqsIY+wGGyK7XawgAhF2COPE8j32bV6YoRRBrbbTWWisAlDEGU+aSjhUSZd18V5BmISrrJi+rtm0Bwo7na62rpqUUftvL87bkDEKIEFyv1wAAP3DjOO5Eyl0AeZz08mxrgBVtO97duZnPLNBJvIsx7vd7cRhiaDarpRACWOUw1O9F77//7t7ezmef/Zy31WQy2t+bOC597913/uqv/ur5sydFvtVarzbrIAh+8MMPF1neNM18uXj+4sX+/n4jeBRFYRgmUeQ5LkGYYiy5mt/MNsvVcDhMepNRv+cQXOXZwWQcuU5VFaur85ubm8X1OdBy2Etd1xWidRwHAPTonffOz8+vr27KunGYGwZRKzilVHLR6/XiOP693/u9bLt88fzZeDLarpZBGjdNAwl2PO/m5ub169c//tFHX3z2ObDg9umtsmivrueDgQswaupmOB5LVXQV24PBoGtHE0IoLZhDLNB1nq2BJcCcHOydnJw8un/3Zz/9j/ObGbDWYw4kOO33uJIvX71qhYAQKmwwot0Wp7XgTSPbom2FNbAzbFlrtbZKKYJI1/jdcokQQhBTwrQynuOt12tK6eHx0XqTUeYeH53WVfs3P/s4z/PRcFcogzF+8fI1JiSMkrfiboy7e8wACwnsclY6sXwnHt/d3f3JT37y8ccff/PsqeOBJIyzbTW9vqCEuKRPED7cHTnMJpEfhp7vupRha63n+13p2tuMBYgopRhDbY2xkDiOR11rIRdSSg0AQpBQtRoMBlEUSSmrstBaDwaD7733Drmc3uzPbh4+fHh9fTEY9N55eO/Nq2dStFW27veig71dB6P1coENEK3kVjStNhBZaZSSiKurqwVb5XleIEowcepCvD5/WRSlNoC6SVd2xxgRoq2qCkIfQodgHPjxjz/6vfn0uiyrq6upgtLzfAgo55oCFURB1RRewKbTBWVkuZ43vAhjv67brFi7bggs0Qa2fNM0jQK8rppnT548e/YsDpI0Se7cuXVyfAthk5WrRmQAKMpA2RZ5sRAyG0z2mEdfnb2aX3/e1tBzUkqdPC9Lx1FKCcGrqirK2ncBhLhtW8JwK5qyLBFChCDXddu2NlL5aQ9aA42WLS+z7M7R0aOHD0aDweOvvnEY2x1PVtst5w2hqJdEZVM/f/F6bzIeDZK2yeMowMBOxv3f/slHR8sB51wJ7VLH98O2XLhUUtjeOhwt1tus2VikNdaAGt86AMcYEpfRLskYIaSVstZgjJuqyPNSCOFQ0kGpvu+/unh9/ebM8zxMqVKmKnKloQF4OptrrV3Hd5gXBAFCkeJCyNYa2xk63qbHQkgIwRB1iWNaSiGEaFurNVCat1w2bXLQdzBCBgMlrVRAK6iVNMqj5Pjk1Io2W69CBx7u7lCkZZNnMCibsmw4cVzH91abTSXN4fGd86vrb755ut7mZd1IKcMwtG/bmQGEqDN3UkIoYd1acHw4Wa4W63WhRDUY9Oq6NEpbDUaRc3Kyb43+zXpm2poAsJ4vbm6uf/63HyulgiiWGuRFLrUZDgNIHY1IFEUC0my+WhRl0Yrt9XRWtC/eXMyXSyksh2RTN1LoTVGu8m26G283hTEEExdCBGHXiqnjGAvRKi2kBFpDJS0EyALQ8hohDI1VygCjjbIEtsBYrSUlBFnb8koJDiFMe3Gvn7jEJYw2TVPVddM2EEIPQT8MWvG2LKPzSXc7k5Syyde3bt0ihCyX8yp3wsgfhYPBYGCN7lrjr+Y3q7VbVUXTNO+//+6j9x5t63q5zRohhFRK68RjadILPDeJApeyzSZ7+vS5sdzzMSbAo6htJWW+H6V5I7OyKFuZpIOR50VRVNf15eWlljJJYsvYosy+PL+8d/vO0cFhGodlXkgjDbbLbJM3VRBEFuPWGJ85XhTWUuksYzSmFrYAtMbMN9l6u0FQi6Zu27ZslNGkFw2khcAUhqumqoMkXa+yuZj1o+T44CCN0wqTtuaj3T2XYCkEZg6XVhvru4xBRFzH832FbFluSl6FaYQ9Txp7PZtzYQDCZxeXbV3GaV8Z3bZcQ9uVZRCCMMbGuh5jXWoyYwxBBID5NpJPa60VVx0X2ZnQulG7w5CllHVddz0pHcHHOWeMfjclfKsQAtbazkZhrekUJ12ECYRQGsOlCALP853bt29//LO/WS7n42EaxeHp6XHbNMCofJtJLhRXyNrJaEgw5G3z4ME9SnFZ5fP5fDDqa2Ncz0vS9IQ5TdMoqYej0f7+obOYQ0zGOxNCKYRwsVj0lHFd9z/8+V84hK1WK9WKOIhF3QS+f3p4VNZtlW1n08vrNy95sXYd5rouVDUF4p37t2Y3F59+8vGPfvyTg/3JbL4cDod3Hz5wXX+1Wi2Wq22RV1XV2akCz3eZE8fh6ekxsAejYS+KoqsLxvzg17/+tdFgs84c4vz5n/75v/gX/+L3f+f3/+2//Xfr5Wq2WF5c3gitCXXaVmR1prUCGAEAui6Mb02wAFqQRGGx3cxn00E/fXD/9mAw0Lx++fypUmpnPBJKv744N8BGaeL7gQHg7fOlQdPwt17ZONnki84LAyERUkOgrMXAoi4giVInL6okSTwv0Nq6ruv5/mpTeh59+er6+moxmYySJDk+uX3r1p3zs0vquJPRZDje2eQZdb0Ak+n1kjDs+76QsqoqC4EbuACA9XodBH622YyHI631+++/L6X80z/90zdn5wiBOA7rsomC8Phw/9bpQeCR+fxM6QZBrWTDHAIAOL88460iLccEGmu1kYwxQGl3es+rminruhBhog0yCAELLYQUojiOu/Swm9lc8cZ32XS2INba+Xx5dLAXBHcZhae3Dge94NNPfg2RGfRTq+WTb75eLZa7473p5RUjjqLA8VwhRNMogO06u5JSAgRHO5Mw8suGr7eNMsh1fUodx3WresOopRnExBIyJoS4jh9FEaW4yss4Gt6+9SCOfMZIVRVScW6l6zI/8pnH4n4Ux+Gbi1f3H92DAH3xxVfz2dpA5Dqx0ApZgKkTBp7nebxu+/1+5McIdtoWsH8wSpKobCLHR8wB9SbLixVlwPdZXZfLRbVYVaKlLpYAECFVXbcIuwSQTmRkLWi5zPMc+IRzbqwChjRNY5SuqioOwjQOqzxjFMdRmK8WBMGd4QBC2E1FAAKHEIARhuDo9GQwGFxeXfR6yaCXrpbz5Tx/+fL5eDS4d+/e7/z2bymlNssNMHAyHAfEsxaHUfrxLz+BditbCZDADUcO97hWxubr5dv8LUKg6dqcNUaoy2OpizJNYylkXddAA2Tger1O014Qxkqp9Xq93uQQ07ysrIHS012QImMMWdO2bdtU1Gfd23bIKrTAWKO1diiFEFJMfNc73D+oinK73TZNY1pBDLDGKKG0UhhCC4FRgrjU99xNud1slpFHF2uoRLt/tE88VdUNKivKfMTcelMpC/0oPj7xqoZTx82nC4QQJRIjRBCVCloDtTFaSwEhIdwYo3ibbebbzcICFfhkPPLPL26aeuM63Gp3dW13d3ffvX/b853tavnsm8eLxYIQVlZ8On+Tl3UQxu9//4NH3/t+FEXZ8nUL0LoErY1QAKXMl5ttO7uWEljaDyM/DCKjAQTGWrxaNS+v3rhOOBxOsEOBNRBgKXTL6zByEQQYYYKh0YAgZC3CiGqNgLUQIsKwVVZrDQCkhAyShDmEESylbKoCAJuG/iCJjAFBkDZNM51OyypTSrmSQpR4PqMUA6Cl6jScWirJBR8O+1VVdHg1JhAhtN1uf/HLnx/uH9y7d2+7XS/TWGt5dSUdxzFGa0yWWXE2nbZtLbv+BUQocbabDFjU6w2IQ+LEb9qcUAugSke71lpEaC70umxV3hZ1tsmyiZQuYw4lVR6Ilg+iwEYuRcq5sxdF0cHeflVVj9df5nnruq7RQBV5wBUhlAbe3vEJDePPHz/J2tZRbhwEOIoloVfXV5v5Teg6vSSGwIJa+0EQxcFqU6hGQQOhAtZga6EUtizbLCs94iFEwyiBEGFGnSBMh0Mh2uVN3gLo+56hQGAjtMjrshQtclmSxIQyQOjjp8/WmywvCkopJlDUugt/TJJkMBgAYMqyBND04jiKoqqqgLW8FV1R9bdbPKgbbiwEAEhlECLM8ZjjKKV4w6VUXCghOgMzRpg6ro+x6XiH77p/rbVK6U4MRBnu5HV5ngNgwjBcrVZCSERIPw0tAFVVzW4u0zj44fe/NxgMPv3VL0+OjqPQXzf1fHENDKQOQxgrbd5//4Pxzuj1m+dFUXhhhCjbOTiElLU8u5zOj46Odg6OgzgeY1gUldZ6NBq9fPkyy7Ik6U0mkzdv3myW69Vi6VAWukHkB1EQ7ownru/N5zcuQcNedH3+ynXZeDQMGDg8OuRS+g48e/k0ScIo7V1PrzBF08V6sVhqrTnnRVEhhJQyCAGIwHa79h3yxZefpXHc7yVnZ6+vLs6i/k5e1HUlpOC+456/ufjisy9vnZ6OR4PXL1/czBdSm/V6Eac9SPFqPfMoC8OwE1+3bQug6aRUge/6ntOWBcPo9PQUAbNZzD1KQj/YbrcOpZgSLWRZlmGcOI6DKYnCpBMO53kOlSGYMT/AeK2V1dpQCq2FQkhrFQDQaGCMiqJE65XneYSQzWYDALLWwzjyveg//MV/HI+Hg2Fv2Eu32UZIq5TBiKS93nq9Xq1WWuu6bimlAP4nVzmmpGs2933f932CUBAEw35669atp0+fXlxccA4AABrYg4O93/7oxw8f3MVWXV29/OLzyyTy6jw/e/P66uoi8NMsb9tGHvSSLpkKQAMJdrvit7ZpmqZpRZZXFmJrsQWw67vaCQ0idrUtyJBwqeu6BQA8fvKUBH5kjFkul/fv35lOz6bT6bvv3Ps3/9uf3Fxf3bt1p6qKi4sLl7Jekn72yeeMuuOTE4I9iayxREm92myzbOOFQdLfMRYrixBzYyckjgsA8IModKTvedaq7XYthCCYucxL05QQ0lTt8fHJu+++63v0/PzNk6dfF3lzfDoJQ78/TChDuwejNI3/7M/+/X/9W/9YcDVbzJtW7O3uJ8mgrqSSgBCapk7TNNvVhhDSNE1T10K0hGCIZG8YdRAigJbzRuk2SQOpRMvl7u7k/XePl/PmV7/46vr8gjFG6CSGYRQlhBApFYQQIyqlrEzb+ZuV1GWZY0iklFEYWmvruo48dzAYbBZzl7EgCOqqslpeXV1EUTTZ3aOO8/L1m53dye/+7u9y3my368X8Zrkw1sqb6UVT54f7k08/PaPEKbMiYH4/HEZB6jrBYLhzdbDWliG61phKg9erqubTslS+q76LNcQQdWsWxWg+vQEArBZLJSTGeDqdBkHgMGYtaJrGAlTXdVFUSpswTsMwZNT9rjAQY8wYdV3XGtPwGkIIje142W4hs1pvt1tGKGPMc9zA843SxTaTLUeVhRBqpQTnxhiKEMRYSMAYA8C4Lrt9+/bhwc5meVPW5Z07t3//7/8ugPjyevr67PJ6tirbKVdmsVxd3yxni+VkvLvdbuM40VJLI9u2bUXbgXJCCIyQ5zm+42JiAZeTneGdW4eHR7untw6//OJTKfne3s7Ns8Xr168JnPxXf/wPTk5O3rx58+ybJ9vVerXJAEKt0lXdsiAmjpfX7dVskS+Xq802y3M3jKgflm2T1QhTDzCSbxfbvModq4QGAA7SXhqnOwmzFlpDsm1VVxJjDICtK4Exgggw5vq+TwhVEihlgUX7QVgURVU1QkgNrOeyQTqcjEZxHDuUYGiLIl8uZlVVWKPapmBO0N20lOJeL+nKMpQSlGKMYZfSo5SEEBKCAKBpmhZFcXR0NBj2nnz9+Obm+uDgoN/vX1ycPXjwoGmaIPC0Zm3bpmmilHr28uzZq7OLqxvXpQ7FUpnValUWPNvWUtjlcj0cDl3fsQDVTV035aJ+HgRBrzco60Yqw1zXIOT7/ma1ohj10nhvZ2KUYAQHnnt6uO87pIt74qKZ3Vx3BR9cStd1y7qS2vhB4oR+qDX1XYMxCYLKqFVRYEpqIYtWGGMYdV2HGqP7bpgmg822FkJBgClmnEtKPBxg1fCr65u2bMf9XuhHvKmyqkYIJT3TSrPJCwmhlyQCSowsRgh5FEoMFESMur4/GI/A8+dCCOYQBAnQRkvpMkYp9X0/DEMhWmstALYrqoAQCs6buhVCCM4hxAAgrfXbJIb/pDaAQohOgNZll3U50ACALgECQWm/1aB3P1prY1T3JienRz/4wQ/KsvzpT386nV75vu+HgWf9qiiEEIvFgivZ9SAsl8so8K6urh49fGc8GhmpLs8vHMcpiuJ3f/d3lZHX0yul1Psf/GCzWflhsLO7N1utGiHS4WidV1KbvKotwi9fvHj16k1RFAeHh5zz0Wjk+77jOP0khcrqOInDcDweU4QpJsDYxXx6fnYmefvOg3vICght4FEM9cnx/jdPnjx8cNePYmVsVRWct4vlTd2gLjO0bnjNRRAEymilxMH+3rNnTzDGn332WT9NP/rR93/5q59fX1x60U4URRjT8d6EN1XoB7/424//6I/+6KOPPvp3f/pnSsvBYAiwQch4fjBfLnp+Gsex67rQ2G5Q6Ch5hJDibZIkg2Hv1snxYjZv2+b4+Oidd975i7/4i2fPng3H4+FwmPR7zPevb6auF2CMIaGIEsZchBBhDgBAK6uUghA5zLNACl4rpY0BiBBroeM4WmtCWLeHcs61IpQEYTj41S8/++CD91+/PlNKXF2chWGY9vq7u7vD4Xi+XAAEKaJCbJN4vM03TdOgLuw8jlrZ5nnued5isRj2+y9fvjw+/P3NZvPkyZPFYpGm7ocf/uiP/vM/Ws8W6/nip3/5V3m2EM2W8yo93OkPUqWU5IbExPf9OHKI1Z10FmLEGHNdF3AOAAIQS2WEqJRBEBFCnE51waXWlgMADvb3eoOh4zj9JJzfTEmUhAihME45V5T4CHhXF+tH7/xQNmhv/1a2zsOo99Pf/PVgMOwNYoe6FoKyri6urzbb7WBnHPXHAtGqrud5A9xIaFRUEmJ9794JQujly5eby0WXLSpgK1p+dXV1fHqyzmaYMAihNHpVVZPdnYP779F0/PTp00VZQJ9aSDALRsPh2dnrvdMf3mxQljWGDVu4vlxvrOdjF2sq3IDcfnA3z/Ph3s5ms5ndzAVSkle6UTakV69WnNcIm3QbRdEJo+PNZuN6o2EaIEh5a6WUSeS1gwhBhyIHAhZ4Q96CKDS+785mU0IRsxgHgTIGutQarDTwUDBb1UGys3f7nVfPnkSBJf20xKpxwOV8Oc1XDeBQY5TPZ7OZ53mUqaKYNU2DENxsl0LzWrSHp7cGgwH2wmwum6bZbJpGrP/i199c31yf3Dr9J//knzz8gx9e/Nmf7w7culXL5ZoqLYpWu2hZ2C7T8K1Klr611hRcS6k19edFizFGQZoJIcpCtRJj9ejdfRdiRGaS17ObfHfnyPVioxDG1PM8SqDWHEDpR0RtCATQ9V0IIedcAeV6zBLiMua4tK5LKQtlioavSjH3EtJLBkVRAIgxJXVdSykRgBQiAsD15fnOqPfsm696EZpMYmjKJMUORvPF8usvPoeItmWxWa/3Dk4uzy9cLxr2R2VZHR2ddk7dJEwJdmtx1VXyOo5LEKWEIUiRtaFHrGmN1IHrqLq6f/tYypLz9ta9k8VmxnWzczQJU//Zy2dXNxcAkCiKtlmJDB32egcHt630Hn/+erPJXLs8ObmFlKrbtm5EvalCGhLqCm0C5of9OE2ioiiAVnHiYai23DiOI7WoeGWxxczhnHPTeJZVeY0Q6ilCqe3qshhjnFshhFWAUZd6KAiC/d2dvf3d0PO1VMYobQRcI8Zc1/OVhlUjpptrIxWElmtbF4VSilHsUDbsp4HnM5cpgnzX830fIUAwPtodQwhnFxeB6/STI0qpaiWG9PL8uiv7VpI/fPABISTPxM125tHh0X5ACAHASMURQr7v1nxqCd856rsuqfjWWAUA8MOEWoQ4f/7pr/w4OtqJnz6/cZh++fI3SRol/e8Jw5eLhcvoo4cPQz+4ubmpb7ZK6kHPKSuupEHIq2rAWHIzzyyAQgizKObztec5k0FvsVhsy7yp6rasmrwiEFrjCEmkcXRlEaJpukMwKbaV5AooSTEkSmitPeZYz10v5/ly2RTj8aCfRNFwOMQWqFYkXjCM+4ubG0cBHfLB8clysdjmVV1riHG4G29X69ly7noQUWmEgtQx0Kcs5Fx5FpTb6qvVV8Nhv21LjFEUB01bQKQBssooA2zRCAhIrzeQhmNmGiGklBZCofVive4UJF1QLiQEYAysRZQaCDnn1LUIQ4wxRBgQaCGAAFCAmItl1VAGdnYHQgS9vr+YG4ylajW0wHKuaradZZNoXM0zntnFVfHZ1eMvvnx9fHL1w49+1Ns7Tlf5xfVVAA31k3K70gBnVeNF4f7h0W9+86vFYq5EUxUmTdM/+J0fCyHiOH7x4gVvxOuXbxhjTdqO+7sIEduC6Zubq6urIAjee+97WbahPq3rMvR97elqbTd5U2Sr5XYtpYBAxsMgHseVKPaOJuk4GU/2P//ym5evzqxRxTZrVWyt1Zqn8cBzQmMUQkC0Nc/K7z94b7GYPbj76Oj4YDFf1qVtOfR8UmwbhPBiUXQJ7tJ6f/E3n/7xH//x6e1ZXn7e1mBvb4wsqhb50EstskpxDGEn1gtDv3OLIAzGw9HBwUHTVNvt9unzZ1dXV4iQk9unq3wOIXRKp6rbg/39KOohicoqi1gU+r7BSuIWY4yQEarByO+lO0oprYwUSmttLXBd10ColFqtFnt7e3lW3r9///LyEljW8owQstkuwsh99vyJUtIYBa1er5eHh4eDYX8wjF+8fOZ5dr1exSl2fKSymnkEABlF7kc/vAsh/OwzpaV6sViyPjvdS3/3x9//67/+66dPnk9G6b0Hh3/4h3/v2dNnn3/2eDqd1XU92ZtwwKDXr5FTWlYoCANv3XIImOKtwRZjiCnQVjiQYD/ibVO1lZAAIJeyABtjjIJIEawIxX60GwSe0dXzs9cPHt1qdfHJV88kVySOgl6v9+TJY0bw7//e7+zsjl88fTqfzzGGFxcXDmWTyeTk5AQA0DTNcrne9wOhtVFcK17mGUIg8CjGQRQ4bZ3nWUmg2d8Z7e4M16ttVeTXZ28Gg4HWKgzD/mjY66eU0izLAACO50KAG2CVUtv1pmsSioOYQFaUxXad5duiaRrJ1fnrq+n0Sgi5v3MYxYHneVmW1SVHlu7u7kdRIoUaDcdx1FssFsvlfDabu647HKaMkcXyRkruunuMeRAWFgKpVFvXV5eLm+kCWLa7u4uxUxZV3ZTzxc1ms7ZWEwIgsgghoA2AFgGgtNRaaQUAQBBi0XLNqOu6SkloYRT2gIWuExgNKXGNhrxVrhOMRkNj4NNnL9fr9cOHD/cPjhn1Xr16tc3qshLbrM5bpyzzPM8xga2Qdc1Xq/V0Oj1ynTRO6raJIkoguJ7N+72kbUoJvu1eUgphzBizFgohLIJdrjMEQEsttepaUpLAK6scQP3jH394crr3y1/+cj5bd+VyAGBKPEIAoYwyjDAjBFHkdctc95RSTAhmGmul1CDsxXEoRY0xskBjDCnFzAF78TgvisvL66LMMMYEQ6tklmXvPbp7/86txc35wdH+znhweryPMfx//r/+34y5mDrSaEKY5wZFUTHmfvjhj2bL1dNnzzuOVgrtUo8xVtccY4whsQZaoI0WhhgMAbTKAsmFkQJyaYxWRdHmxXazmOdVjRD5xS9/6bnudDqdjEb93pAQVtctQPTlq/NyPbs5e+34gVLm5NHRT37yk//fn/359XSWVa0TRI3ki8XKcb26qpSUTZ2LpqWUYgSNMcZnnUCEENKVaXVQR+dNANpYa7sMSgghQZgxhBDSQlprESJdb9NmvZU+l1xYa9um6YIsu2uutDJSdUfMzp5qjAEARVGUJL04Dq02QBvHpY7jQAssBFEUMcZ6Q9mFBnbv0x1nO6cf57yz7GOMs9bUda2Ncl2300JDaB2X7uxNlFLWGoSBECLPt0VRSCkpRr1eb39/b7Q3thD0++knn316dplTXOyOJ3fu3HrDKLRgf7wjRDufXi+vrqI47W0yLq3j0rrly9W8FWY4mmBMiUuVUgAhoZWyBrsMY9il1dZ1TRG2CEqlFuvN3mQceL4B6OL66s3Feefiqcom9SOErBBCSwUBVqrdbFcUQd91bm5uBkkchyMhWoSA7/ubzebOyfHt0zsMT2c3X7eNFFwuZnNtTSt5nRdAm06brI3VimgFq0pCZLVWhALOG8pQV4d4dXXFudTKdpqDKIw7J5vqosX/TpvAd8YHrXWnJum6Ud6+BmAIELTAWmA0gBBgCC3A89mSYFjk9Yvnr6wxSmgA0GqZUSf1XQdD4HneYNAbJPF8Nr28vFjMZ45DPa9LQ7FJkuzv759fXfb7/fniBiHQdcuXZfn14y+/+OLT9XodRn4URa9fvy7y6v79+03TfPXV17v7px988MGzZ8+Wy2WvNwAApGlfax2GYZJEURQAqBkjk8kxJhBCe3l5fnNzU5dbP3AYha5DBDfz2cpoCDHZ3z+Mwn5Vyc+/eBJFyd7+Ydu2RVEQgkfDYa+XxHFsjNqsFhijsiiSJHn33Xf7/f6LZ8/ruvb9sENfOiqNMdbpPYuioJT+vb/39168eNEFYL948WJ/f19/W/btum4XXaWUMuptORLn/Pz8fDq9qqqqKssoiuI47p6RPM+n02mWl1GUTHZs27Zd2EAH/3SPT1eB1lTi7fdr33pirYWEEGUNoag7s0FkO02i41DL8VtSCsIuDgEhhyDEeWOt3W63jLHVcqMNQARbCBupwrivlJCKG4DCtPe9dx+d3Dr+6U//Ok7juq4fPnzox07cC0/uHFRV894H72GG//X/8ifbTQkhsgZGaej5TBvTXbEOo8IWWGiUUohShKDj0IZLhFBRVMYYhIgxwloNLLAWWgutthZDa21TFRTpwMP9Xkgw3CzmV2dvoEUEE1Bka96UCoE3b14EPo3iwPfZ3v7Oer1kxAGp6Q36NW8tgo5DCZCtqJPIH4/7cS/tj0eI0NV6XbVN07RaN0Hg9WIPQy1FYWTl+RQTK2QtDSUE7e/vxnGcZdvFYhEEoeM4FoK6KAuWua4bOp6ByBhQ1+LmZgHhvPtqX728MEZRSjEiZSG2m7qqKt/3h4P9tpEI0tFodHISHB+XV1dXz549aVsxu5lbqzGGdcVd1+EtwBhIDkpV+b5FkEqtiqocpDvpMC2L2vWosWaTbYoiwwTWnGj79uo5DoOEWoOUhtZCBClChItGaydJkmKzRohZiz7/7Jv5zXVRcUw9CGHdKmMQgO7NbHtzc+O67nhSU0rLxmSlbATEGvBVaTEREmmDkcUOC1zHF628vp4OBgPPYePh4PDk+Pr6+vJP/lfN8WJ6LW2Y9FLGmBaylS2nwlrbtryoSmutNdC8vUtxtzCdrV6Hode2dZwE/cE9bdqL86v5bMOcAADIKHJc4nqEEIgJwRjKllv7dskjhDjM8zwHY5xt15RS1yWrJl+tF01TxEmQprExajyJlK622awsa98PpIIIAKxBEEfMdRzP7Y+GvX5vZzK4vDonzE96vcnO/tdPXxhrtAVNVe8fjA9PTpkfnp1fFfnKcRwMLYQwjmNrsDEQoM5tjI0BSloNAUbQalSWYjbfFBU1mpfldrNdN5WBiDXSPH78LPBc13UfPXww7PcZddumocyNHRaG0WqT3bp1C2O6rfPJZDLo9bJBXtTXDIGSt1a2ElhkFFBAQSOVwAh2MR6VVQRh3/W6LQFCKIUgGBNCPOa8XWikcpnTDRPGKmuhlFqIFiOktW5rfn19Pez126aBEFCEhRAYgQ5HAdSp67JrInYdh/T70BhKyHA4HA8Hg16PEAItoJQiBLTWlqA4jn3f7xbHTlin1FuF3XczZYeNW2vdbd1RaX7gEUKk5NZaQhCAp9baTqlXlsVsNpvNZkWZMQkJIYNeL3DcKI4/+uAHqm0GyfXV1SVWKna8STqQUkKj66Kst3kpBGz5q/OLqmqalkulsjJHkApZKw46fSrB1AIrobYYaGkIJFbroigIwg4mSgq5bf0gcAO/bpur65vNNseMUkoNBJ7nMcakaBGAezvjIst5U/mBJ4RYLeceRRANm6bSWmICssXaygMK0CDuU4A8wgigy+VlEPkEosDzVaSRaavaIIh8zyXYV3SLMSKkU5pSABWEtqqKzWbDueylAyGEUtp13aJo2rZVQnUDwbf+hbeEXScK/q4RoBsjMMYIEGCMNsBAQJCFEAAEIAL9dASAmc0Wf/7v/wNCAEKYxH1t1GqTW+1DAFyXMca0bTsEMQi8ouMjlrOiKPzAPTzaf30+3pkMmqr0A7cq87reJkmEIfjtn/zkf/6f/6ciU6PBUHIxn82iMNyst/ObWZyO7969e3lxlWWZ5wXd56yqihBECBKSKyWEqB88vNO2zYsXLzbZNSLcDzEAsuWcEh8AY6za5hlCZDzZD+N0srfPXE9pS5jDtyUAllLaNd1IySmlxpidnZ2mrvv9PmNu27ar1cbzAqWUtbQ7onQFMd31tNZ+8sknv/Vbv3V6eooQ6mwF3QvWxdoYgwDoWADOGwRgNwp06XBdg0Y3iyul+v3+6enp1dWVEMYCxBjzfX9vb284HHp+2Ai+3W7LsszzfL1F1lrfjYwxRndqMAMhhBABaDDCjgMRQtZqa/U2WwrZMsYgwQh0AhYLIEQEMUIxhsoqKfXNbL7N8uVizrkkmGmthYRxf7jZrDhvZqvNm7OLD3/0g72jg//2n/+32+36f/yf/r+37x0xHz18/85v93706tWr/rAXRH7dVkkvJoRJoQjDEAMhOEKoMzsopTADBGFpldEAAMuo04qGUme9XjPqd2kfxmoIEQAWfNvUBywkwFjZKoS0gDeXZ5dvXm1WN+PBhAAjpJCugzFB69X85joYj8e375xeX1wiYIVQrRSnp8ebTeZ6Xjwev//Bu2/evBFa7e7v7R4ejCe7WVl89bh5/eZ5J+LVErx+83x2c1nXtVT1cDIghBioleZcVAeHO7s7e01TZdu14K3gLSXM8VyMAMGYMaYNqqqKQOIxr65rZAmldMtzhEjdis1mk2WZtZYx4uz52LIn37zUWidJ0uv1EIKdIOvw8Pj165frVUYZieNBmsbQurzlBEe1UVUt4yhIk0EU5Y7nAgjLsnQcD1irtYQYWWCrquoC1wBW2KMMYYMQIZhgxphLCMvXmRQCaJPnOUOQ1+o3v/xkOp1CghHCBDtcccH5elVss/rmZj4YjD797HFZ152op+YQYyhE6/na9UJjoZScMjoYjbVqF7N5eVzEUdDvxxevXzx79kw0ebo/hjujTQ4PJ5Ok3yuKIs9KgCCljjb2zZs30liplJTaQNCJRh3H6fdo2ovC0I+iYDzpQ6QGg96vf/VZXUlrAUTSWMG5blqpNbdAQ+NqZTDG1kKlDIGEENK54LqHDSLbNIU2Ioj8IKSvXr2EiJ9dnM3n58zxrXW0Br7rXd1MX756I5rm4OT08Ogkz5fPXr34za9++eEPfzwYjetWgGdvNLRVzRH1uNIvX7xSxgquEEKhHzSwAQC4rhv5SdelhiCm1IEQWqCBNsz1RAvyUrx6dQ2gAkADaOq68v3QWKBa6VC5u7N/7/ZJLwqhNnVVyloX6/neOL57967vhXt7e03D/x//+n/9N//m3xBKHt6/17btar1lyLz/6N7VdGY1t1Y7xIXWVFVlrR0MBth3uvMi/rblXSmFARRNCyE0ShulMcaO41CMtdaO5wkhtJBSYgiQMaaqqqqqgDZt03TjRSdHxRhLqTAi3dLmuW6/30/T1KWMEOK5Thfk7lCG3p5bgdYa+9T3/U4z/53P6rskzW7B7dRz3aZFnaqTTXXneK2V1rrrboDQdqfhpqnCKE17w7Is6+VKKqG1vrm6UUrtT/Y++uCH+nvqT/7kT8pVni/Xk95Ia5ktt/ObGcWM+ZEBcLXeaq2jKG7bTVMWca/PCFZto4x1qAuI4VzVbVNLrgRQAABrMYC6gwqUUlIstmvqMEjopswhI34Ydvtx0zSUUoRIFPiH+7v5NptNrzzPa+oKQqu0EKIhFHs+cxgmZH+z3D55/HQ03BsPJ5ttYS0s48QPPYtA3VZAm7aWkgBC3TjqeW5SGuM4lBDkeg7wCGU4CsNXr15I0SqhAt+HFoiWKynzbWaMdRyGEEQIdo5iQnA3nH2brQQQggBAjFHXg8YwBQBYawB4i1liCCCEUijPdYwxnDeMEM9jQos8y3u9pMjzqijbuvz0s98wiAeD/j/6B3+/yPKf/eynTVvVdc1Fw0UzHA77gx6hCBNrjGIOllJhDO/dv7u7O/nss0/yPN/f36eUzufLpmkWi0W/399sNkVRhGHYVbN6nmeMapoqTWOt9Xq9rKrCAh3HkbVqtZ45fnV3J42iKN9mN7PrMCSndybD0VgpIxW4np/fsw+//+G7ZVP+7OOfVyKP4khK1g0H6/Wyi1A0Shhj8jyHEP7bf/tvEUKr5WZv9+D169fMxcZqqYSulNKyS4klhHzx5eeTnfF/8V/+8Zdffvni5fP3v/f+zc3NaDwWVnSjcGcI4pwThD3Pq6oKAej7fpqmbduul0utdVmWnPOOj3MchzLXGNN1tZdlWTe8bOqiKLqVB1OktS6LqnvGu9xaY6y12lpLGOoyZgDUGMNOgYdxBADoOsbe4hBaSwgNwJQ4FuKibsqy3mYlgMgPYq21EiyK+3XTIIK22fL1xfl0Po3jgGs12Rv+/n/2u7/7h7/dNhXXZRCwrFo+f8W3m2K9tYzlceSmad8L/CJftbxVRtlOUmZt59pQEkjJgTIARIyxMAyn0xkMWNsKY4B9OyIQTAhhkDKEMXQI9CjCQKu2WkyvtKjTMNgbD4mRfDhIZ7MZhOTh/fcpw9Pr89PT07ZK+/30qy++/uzxZ48evstbaRG4de/ujz76vrJivlxFSTgaDbzIuZxdvTl/fT2/6nYmpbZnl2dAG9/3uxUfY4wxDSO3LDNjZBC6+7sTaO319fX0eiZA6zks8nxCsBK8lbgsakqdvd2DruiIUhqFSVmWGFHPDT03DMOwg2RvpitrZFEU1lqEQCd5dRwqBB/0J4Qi523SGlXSGu0QRBhSi8WizLXWVimU5ZXRuBVSGcuIAwBgnm+VrptGCNU0wveRaHVT5g1vAUBhnKYpZpjs7U+UkKJuBv30ZP/w3p1bnZBwuc0EV0oCTEkYu4S60mgvSABml9N5WTf9ft8L47zkQquiVVm1PDzcBwgXdRMjd9AfSVFzzi/Oz99998G9W7f/ly+/mF1cfO/hw1u3bllrv3x6FXiOqsoq2zRlRahjtNTK7u5MlDatFJxLLhQAABIEkU2i/ssXz548eTzeSX47+jEmVhshZE0os6ZbvCAAVkrJOdda+i6TSiiFtLZCKKusEKpt6yQOMcaOS4PAYw6GGDsOMZZvN1d5Nn/x6rysmt0khshKrqznHx2eCgV+8atP3nl4+/XFFaMg3+ZFIzZF6yfm6cvXjTQIM2WQx5zFfP2z7cdJ3NtsMgi7olWJAec1D8NQCNG2LWPAZQ4mUCmgrDEaAISl0otNXtclIcj3HKWAAUI0LYI2CaMgiKIoYYzWZa611lqevXl17949q3WahNl2NZvNp9Ppxx9//N/9d/9HhxKjeJmt9w+Pfvj+u5Ph4Kuvv1muNoM09IPo/PJ6ud5Ya3dvHSspq7JsMdZaKy6UUgAA0fJuh8AQuq4LreWct217NDyp69pIgzF1GaWUipYjSDCmGCmCaUdGGGCoz3yPAUa6Aoj9/f179+6NRqPO3uK67lshNGEYom4OMMbQkHVHzG6B6MYFakzXJNsZ7TqAtJsVeqn5dktDnamvO/JKKb7TrjLm+n7c748559lsboxxXLpcLpMwGKZDLTSl9KPv/9jznCZvQic0GmyWWbYuGHZ7qeN53qg/2N3dG/YHv/n1r68vzl0CB2mQmgAyBjFZZ9tZti6b2nEcih3OOYSQ+T6ApikrpTRxqLB2WeSt1UXLnSBEzGmLEiGEMWWMYYQGg/7u7q7nuLwpXUru3DpyGTVCQGAwhMtlUVfFcDh0gvD167M4Gh0cHK3WX1dl3XnPwiTq/IptWysFqWMBtNrwfhoSggEwWb4mBKXxeDzov35hy2wLAEEWUISRhcgih1DGHD/yurVICNEVCXbYr1LKWg2A6eCBTqAKIehaVKw10L4dFBACGCIhdZdn5XQGFoigBQThuq4BAEHgYQCbpqG+n6bxYDBoqjJJI4gO9vZ2KKWLxRxCUNd1mxVxeE8aeXywTyhSSlIEf/Hxf2yqgmKYhBGyoNgWyEIlBAZYKvPkm6ee56VpWhRVr9eDELoucxxHG5XnRVUXjkMog0HohKGLoD44CCeT8XbtBLEMgvDWvfGgPymrVlv6t//648nh7qPv/eD3/vPf+eTxp8vl7GRvslqtsqykmBBCHJcaY8q81loDY6uqOjs7Y4wd7h/cvXt/u81Xm2W38XfJ1t8GuNGyzL/88vNHjx7u7k7quhSiXa0W4/Gw1+s1TUMxfssUNFhJ1TVPFkVR1/V4PDTGrNfr7yqODw8Pb25uCHGDMBZCdZ3gGOOWy46u7bA9XrRVVTkk+I5FMsBC2D13SmlAiWOMIoyGYbRYLCAyhMKOKkUQoc74CqG2FhrjOA5hDoJEGmkhZo4XhqExpskFQoAx1uvHrocs0HVdDUdJmqZx6N29d/rXP/2Lbx5/jRD6Z//sn/3v/5t/+tVXz16d/81/83/4wyQe7Yz3OOdnZ2e//PmVNrIbaKy1Xd5DJ+0yCgAAjAFp0u/1BhC+6lqCIXAtQMAihBEhmFJEKcIYAiOCIGHM+g5A0I7HQwrQaNAn2/WN1U0/jXd2x8zBCMAw8oRsCMXD4fDy8tJa3YoGU0oYdX13vl4BgpJBf7K/N9gZ51U5Xc4Xm3WYJhBiA0DHvmCHuVEQRLFsVgijNA1PTw+zrMiL7Xa9FILfu3fHajO/mRVFwYMQGwCVaYtq28g8z5MkSdNUSbdthTAGd4XfhHmO53necDgWQpyfn19fTgEAnHNCECZIKVHkFXOIEO1gMNjd3XNd9+bm5mq9ZIwRzDjna91eX88JYXEcK4nyPF8ucqX0oNeHLtNaR0EIGKhbiSnBCqZhxKWsiu1mXQIEXSd0KQkCb7mYa8GN1ruT4bvvPUAW3MyuFrN5MtzZZNu2rtK03xsMXNdveMscl7ke18axyAnCbFvknFNKmR+ulhuLEaZYaqEBjeIAWKYE10I2ZVMV5eHOgWmU7/u8aD766KP33v/R5fX18+fPOSex33f9QBlTt3y53liIKIbQpZRioaQUqhZtvq2Oj0/6g+ji/OpfPv6XUjWu69Z1xSiwxkBAO9cDsAghAiEUonN5UADwt7XrBGPaCapbni/m10JGrg+DMO3109294Wq1dT08HidhGNaVrBuOiXd5MX/00E9743ceff/45F4cOZ99+ut1Vk8mcJOVT5+9sgBLbZjjeW5YVPV6sRLSCiFcx2vbtmkainBd1xhBo5UUHEGgNAOQSCm1llWrEQaQMGWsMhAjapAjjCVCaW0sAHnZvjm/aKraJVArodoGQng5nR8c37qazQxCVdWsVqsoihCEYeDN5/PNcgG0GPdjClTgYCOafLscjUZ7uxMDQVGVTdNYa9uq7i5aJ0ew+m3PrNXadrI1pblpjTFWm6IoN5tNlReEYgTC7hl2HIdSR2LRJS1yLjEEcRxHURT1k9FwEIbh7dt3Tk5OfN+XUkqhB4MBxpgQ9h3l0e3rzKXf/f4dhPAd9P3W4/otqGCtxextEtx3r+/+7nhBB0UopYi1no/CyCgtBunIcRyM4H6WY4QgtMASCMBv//j3iyKbz+e/OfuybVvYQU8sSEMnjeOTo8OT4+PIDy5evUpcjyGIjQr9IB2OLCZVVfKqBBakaVo3tAKVRbab/gHETVMhhDTGtRK6tFzrMI5Cz3ccJ47jwPUcx6UEua5rDcQYhmFIEdzZGUdhkK9X2WatjeSiUkrESTA5OJ0vNnXTDIa70uh1kVkEltuNQraVbaNabY2GUFtd1XVhqh8+uNOhVuvNvChK16Oe7/b6aZZljPllWbqumyTYcZzRaOy6roG6g3O6bJzurhBCdAfTblD77joDACTnAL6te0AAAAMBBhYC33FaXreSE4QR1NZgiGyShi+vLga9vkODtq5837t16wRD9M03XzuMvPPOO5w34/FosZhfXFxw3gBglsub4+N9yjDCgPO2qoogdPI8L4qibdvpdHp0dNTrrWezeVPzIAgAIMvl0vM8zwvKsu4IrDiO66Zq21obnmXrtBdfXZ25njMcDZrQdzxlbUWoTlLPcdyiKLbbpm4lRM7TZxcQ/22rycNH7+/uHeWV2GxWbVtba7WxommNdT3PS5LE8zyjNIRw0B9qrRlztDaO436nD+iozy69inNOKb25ufn5z38+mUyOj4/Pz8878c1sudBa+6773UPRbYSU4aqqIITHx4cdCWitzbKsKIogCJqmybJlrz+sqoYyTyijtdYGtFIopQjDruu6wPE8L1tX3agNIQSoMxAYIZSUEkKojQ7dYGdnXJa5MYYxogvdMVAQvwX+CMIIIQuRkppLZS2AEFsLpbZaG0aA0Rwjq2QjeMub7PFXX/Ami0IvjFxj1M/+6q/Ksrx797ZLCYTg11/86uz6zf/l//R/66UTCMnnn3w6nV5t8qyf+lKrTirRkY/YCmgBQh0Qi8fjSRQmGFFjAIIEIWoNBhaCtzoZBCy0FiDCAICCS0YwF2q4M/Rdp98fkaLMMAHvvf/OcNB79fJ5kkSnp6dZtsm3+XDY/+GPPjw6OiGEiVZ++cXXq9XKMutH8e6gP5zs1ko9f/3m7OrSCcJkNBJCcC66b9p3PcfxCCHWaTFEu3vjBw/vf/XVV/ObS9k2UmrXeR9jGAehaAS0FlgDLTJKC1kDqDhvyhJJKaRspdSdOgxj3Ta8u+ml1PP5PM9zz42MsY7jBL4HoKYUIwRaQsqy3m5z11FVKdvG8EYIUeZ5ORdN2wrfww4DAHlVsy22GUKo3xu3jdputyqFDmO8NZ4bIAC0JFZaoCiCThB4o8F4d7IXxV4v9pUQvG1/9IPv//jDH3z5+Re8LVtehlZRijm3Na+btjLAtlwqo7ereV5UrVByAWaL+Waz7Q8HBti4FzIH19oCoNq22WxWBEOHktFopJX6qz/7y8DzEcCb+ZZRerhz1GK+Wc8wkBRqgK1WldLaGht4TBqrjaUAGUhdy6TUUsqygE0tz4qr1ebqwx+++3/95/89hPCnf/2zn3/8GyUVsKppRPcUdcgzJebtI4odCHHkxR0mwzkvy8zCZru5ESqLYnc8SRLrjsYDACCwpGlB3RgpLELMaOx5SdNqY/Annz5erTYHeyMpzemt+0k6LIqiboTjBfPF0vFCLpXnh0Jaoy1GhDGmtcEAftupozGGjkMdhyJstOEGasyABcoAqIzU1mBKXM/HhCndtq0A2hgtr69vNqtF6DoIACtFtl2HYXhxcYVc33MDz3vueG5RVDVXR0dH5+fni5upFtwhWLXNF5//5ma22CznvK63m8VmM9ZCYogMArLl3ZrrOA7FREppjHEohRhLpaw2wFjR8i5CJ45jznmWZcU2832fQNJBo47j9Pt9BKCUEgHjuq7nuJPJ7nA43Dvewxj3er2dnd0oirTWShqEUJz2up0GQdKtiR20SfDbHejvbkgQQmssBLCbAkx3hO2YQaC/Ax6+HSCg1tpxnG4R/E5N9vb3EPu+X+R5zwkZJm1dhm5ijEqTYZZlCD7fZlUDTBzFvV4PUdLK0vd8oOzl6zMr5WY2CyhllDoQAiGBEF7sDpI4jsKm5Q4milDtup3AKogCNwo22+12uy15k7AYEGowZI4bxLHrev1+P1+v67rGCCghm6qA1iAICUGXl5eUQAcjjKHVYGdnMuinp6eni1I7QXC9WEhIhLEGgiTp5XUeJqGjXY1A2QpRagmUMUJroJWIoujgYA9B/eLF8+16paUY9nsOo5Rg3rRJnPgOwAg7HrMWKssRBoRgY4nWsEMXtIGEIiwhJpBYBCHs8E4AgLUSAohRx/UAhCACBkJY1RnBME2iIPARQk1V5Hle1/VkMsEQ5dl2Odv00yRJksB3CUSMYt93r6/Luq6ns+vz83Nj1J379zxoMQKB5zV1MZ/PrdVx5I0G/ZOjw80mU5IncTzo9euyHg9HjLHz+arf73MuMcZxHDdN47oMIlCWOecNc3DdlI6LP//is/393TSNMUp5rW7KddvIplaiVU1ZlqXICrFYl3kGPvvsTV570qT3Hvw4L+nZ08eEED/wlZB5nnfIfxzHGGPetHme+75fVdV8Pu/utzD0OyUjpdR13U5kkOd5r5cQQl68eNbvp//wH/7Rl19+uVgsKMVZlmGM0bc3cxchGgRBVRedYTWKIt/3yzzvMoLLsjw+Pt7Z2VmvnwEAoiiizFtvc9/3AcRMyaqq6raq69oADQBwHL9j5Ywx3aDQQQ5cKWuxMcpx6Wg8uLg845wrLTAhECELgP4uPhXBjghomsZWljEGEVXallUruQgTNwmdyO+v1nNkOKPo5vIMmiZJol4abzar8zdnd+/cuX/nflu1P/vZz37xq1+cHJ+2iv/s479ezLPNai2UePTo/vX0vGmqLuidEAIU0EBj5ECoHMcBAPR6vQ6q0RoQQoFF0EJjuqBJbblRykBkpMvW29xYHic7jke8KPaZG0QJcSh9+OBe4LvX19ecN3WNz85eWwsjP+jkLZ7nTKeLMIhHk7ExJu0PD44Ofd/flPmzr1/+7OO/nc5n490dgDAgBGgNIEKEQsosAEJKCozUBiEQhr7W+ubmZr3aQos8x7fKBqEneegySgghCFNCQt9zGdVa87aGEFJCtFJWmyQKHccrYLFYVKu61lprJXzXo8zJ83y5WGcZZIyMxsPBIEmSZDqdTq9njuOlaT+J+1lWTKeztlHaIteJIMR1JbXWSlplYMBca1BZVdPreVOLMIw55zambSsv1luMsTaSEDeO0i4o1PPpoJdk29XF2UrJpmmr84vXs9k157yqC9/3AQjzqry+vsaEaAsAIpttZiCixEEEur43JLTX6y3Wq9DzNRBNW5RtVZTb+UJRaIb9wajXe/+dR1dvLhlzD3Zjz3EVV23Jc7tSonI9miZBw9uqqY3Fged6gVtUTV5WLZcGQIDeYtFhGAtZQ4jyDJydXZy9uRiO+qent968nraNFtxCQBEiEFqMMWWYYKWUIsTFiFmDO1gbApRnpTZtlNAwCigleb49P3+zzRYMWc5b13W1Met1ro0Ng57nhsoBs/naaPmb3/wmiZwodN59dHdvb+dmPru+nlYNZ160yYvReHI9XQwGI4JpB+5hjLVSHZYhhYDYModEKHAcijHmkhMEGWPdEyul4FJCCDFhECGp1HYzJxBJ0QreOgSHgWelKLIWE8C22fU1wF88DuM4z/NBf7Rcr7RU/V7vP/yHP0/CiGCkrFktFxdXl4jQ0aCf9gfY8XnddJVjjud3LnnOOYaIeqTbnhFCdV23bQuMfUslKD0ajdyhEw36VVUZqaIoCoPQGMMoSZLk6PDEZU4XTW+sSuPk9PR0MpmM9kZdf3cQhFrrpml83w+isNvOgUWEEISwgcB2PYRGg7cDRIcxGqO1tba7RBC9VSd0HxJj3KgWo66KhnSURLdAK9WJhRmAuFvXtNYAcgUQdjyl8yiIAse1yrqhI3nDiBOF6WA4uvfoUV4UrWyttRaCptkEgedRtphdFdsscv07J8eU0nTQX642bV0FUTjqD3ZH44vpNN9sDekhALW1UkrC3F4SWgDW281bpZvn1lUhlBRStm1Di6LL1cEI1IQCI5IoHAwHke8wgrLtKo5j36PTy0vKcJIkQrRZ3hBGr66vs5K3nFuEWeAFSTw52BOKG4JWmzyvCqs1ogYR8uLFs5OTkw9/+D3Xo01TPn/+XCkVRdFwOISAQAjDMFQMFkVjtCzL2onwd/xOt0Z3BER3iu2Ko/4u5BM4DkKw87MgBDCEABgAzGiYGqOl5HVdWi07GRbG3vVmDYxllB4d7d6+fbvX60nRGqPaVm42q6+//urBgwfb7fbm5iZJkt/7z35vtxdXVZWmKUQ2igKlxPPnz1erled5Ozt7nPPXr1+fnZ2naQ9Y9PXXX+dl884775yfX3Z0/tXVFSHIWE0IgdCNkyDLVxjDq6sr33f393fLrdM2sqlF2yjBYRRSL06TiGX58uLsIvTHi3Xx5JvpZHL1D//4v3wSTMPwvAMIEYC+7wVBEATBW6EigOv12lorhIpD6jDv8KB3OX2rZHybn81Yt99ba33f75JA/8E/+AeMsT/90z9dr9f7+/sYYwxhRyRRipMk6ff70xvZ7c1CiDRNJ5MJ5/zm5ub58+cnx6eO41RVFcW81wsJdYQQWZa5XoAZ/Q6Q88PI9/3NsuqEk1rLblDoDE3fcn8aIeh5LgBWSqG1Yk4AADDadozkt/pH2I/6QihlrYcpQqQLZuZSjKg/GfXTNL64oINeGIUuZTCOo0E/vnXr5PHjx8f7h7dPbt06PmWYvnn5+v479//P//y/R4j9zd/87ZuXF6Ef5VVZl1utVNM0HeSJMbYGvc0DLZqO7XIcJ8u2TdMiyCCkECKocZfBC4DRFkCkrTWrpqYEpL3g4Oi4l/ppHLRVra0l9753LHGdtWvkQC/2qra+fjWDBp+cnK63zauXb7KsODo6GfVBFMWj0ah/ZzeKojfnF0+eviiL2k8GQ+gOBzuz2aytbdsaiqEXOFgxwdu6rvsp2N8/JMT95vGLqlBxsrNcbAihv/7k6cN7Dw+PH5X1N16cbCue57kxRoq2OxUpaYSSXckhACDf5MZkXElrLYKEuW6a9BljbgjS9M7V1cV2uxFCcLkGyN1kGWG6LGoLZMuJ1g7CJkl9qWpVGYyhtVpUjeAqZA6LIOdCtLytm64oljrMD/11vl6tVi5kCAFtJIbAdd0651DBiEU74z7PijRMlvP5J7/+NcLg7oNby/kCJGFeVdRlDvKWyzVjbq/XE1wRjOq6gcQgP9jt9Yui4nnRd32pWbZUjjM0crPZrNI0rSSHFS5B+GS6Tk7vtlpyCC+yDaFo+fnHAiz3dvbf+Z0PF/PV118/EdpS7DQS8FoYASInwbpmhEIIN5tNEgR5u1V1YSH44fcezlfL//u//B/feff91WY9m806iI8gYUxXiEcYoUHIugkaIgORaTivagAASHqxsT5C0HUTBC1m43JLZEtd5uR5WdV1UdV5iVwnoF7QKun7ociUx1h/sCN5S0n/iy+nf/WXXyXDvU5pvNpKjAMInDTuaanapiaEOIw2ba21Bghsq4IxxusCIkgdYoA1WmH8VszPWwkAQZA4xFFKbdeF4zj9dFS1XAHQ2qbkGGootA9x0PotgtZxnOEDmim52AoIfcFhODlZXM+sZJb0LPTSOM3z/PxiG4Y7lNKWOnfv3vU87+XLl1qJnVFvu91CFMRhLJiw1mptCMQWISmU5DIJk7dWBUI45wghxpzT8R6ouB6Mer1e27Z5vh0MBkdHBwcHB+NxIoRgjHqe5zhO11UxGI2/890hjMI4BABoIxBCmAAILQC6AwQIhgAAi2inM5BavhUtAmOsgcAAC2AnwMYQQmggNMAQwroJsrPtdUteB5gDAAA0CANjFRcKY+x6TCtgBO/3E2ttrTgIaKmEpoYFoRYtUmg4jAbWcs47A5gUR0VRcM77O/e9qJ5Nr3o2VKL1k5QJaqGlTuBjHnjMZ0gbTqlTNQ3B2GWMWeth2oviJIyMVLwRFBCHBYKbLKt816lb0QqqoR96tJJNENP94+EgdQdpSJGNQg6tbupmna/bRiLgRZH60y8/L4qqbfT8q2cIOkeHJ7S1ByePMLGxj68ulkZBXvJeHKlGuC6hMBENePnkxf0HJ7/90XsXb75wYBnHwU4/xShEyFccbPK84TIMQxJgbIxuWwuhR4jFVAghmkY0TVOUknNskOsE3ezYXW0HIMllUzYIQUopIZ0mVcKunKmuw9D3/bgoCgghc0gyGA+Hw/39fUpxU5fXiwsEbLZd37p1ApSBROXVKon90TDtJ8Hjzz599+4DpIESijGGkcuN+d/+zb9fZ9t79+8/ePSO6zvXs2tnEJJeYEDj7ITHzl5p22Q/EbLZiDnpy0rPLNAEgSgKGlDunIYOxdZaGqido3RT7z2/+ooQVmod9vvWDUqkt/m6QUJTgzCJQfDg/v3F7Pzlk68maXSBQNNUaZpabbKs+PDD70spy6zcbjcIQoShVGI46h3u77/77rvz+byVO12U8nQ6Lcu23/ebRhZF4zjBzc2SMfab33xeVXxnZ2ex2OR5fnx4jCnqDQdSy/V6PRqPx+PxarW6uJgDiJgXXk9XT5+9mYzGhJCDwzvV8nIUhbcPjr5yvzyaHJaV7PcnvXd2Z6t1WVdu4M4W1xAT32dc8cgJj45vvw2HtoA6DiFEGqvrRiqT9IJmI7wg5FLFvbSVwgIg2oox5lAGCDHGQggZcRzmtK3orKGcc60VJtBiiR2zyK4PzGRbcgUax7WOiybjIcYwiiKhZG8wfP/DHzp+cLnaDI5P7v3gR+vs5vWrV3fv3v2dn3zwv/snfywa8fOf//LZk6eKyzhOV8uyc/k6rgONrZp1SCmEUEo5GIweP/kGuS6XlhGvqITRJvCDuq4B0J7DqroAQGe4CVkwuXV7Xm2wj69fzQkwn37xKRmkkyzLsk2RhIlD3EYKZInvebxp87xMomBnPGaMMYr7vdj3GK9qqI1qOLLWaI0sQADUVRGFIcGYUIQBdF2XuRRjaBEc9+I0SKwBxTbTUgZu0Pq8LhvVyun15d7upJeGNzdXUoqOOiLU9f1QKVWUy6IoMKYIISWN1roRXEqFEPLcoJs0q6oqmrqu6+Vy6fte5xfvJJBFUXQzqTFmu91KqSHEcRxr1HYUjtYaIgkt7A7NCKEgCHzf97zA9b1OvRIEXlO1FBADQCva1Xq92m6EsV4YCQO5hqttMV+sfO9aa1lVNTdkfn5pre33+74fjkaYEOYwz5r6O7aYMRIEASGkg/saYbXWjJHRqBcnbq/XE4I7Dj0/P6+bcrtdU0r3D/YopXXdnp/d1Gp2dbFOkl6+LS4vryCgvR4jiGrFCXNcyqy1nue4jmOBRgj6yO/1etoaiNBgMFBGbzYrpXQURd3noYRhjKGxHaduLUSIAYCkUMYAApG1UCnFOU/imDHCeWuBCcPY910IYds0q9WqrBpCGIGormupDcGsqVediBphpa2p2qatKy6FUioMw47rVUp1m0qnhf5uAO9qLL47RnSxjEopABBCBiEE7FsVntZvj8vdl4gR2dnZ0Vp3+tYORe9C+Mu67ISxGCJMkOd5QRCEYWgGilIqpewOfwgh13WDIBgOh12z9u7uLoRwuVx29QoXy21H8DuEQmMBBh0NGXi+EGK72QwGg+Pj416vxzDxPG9nMmYOXSwW1trBoLe3t5MkyWDQG41GQehprV3XdV2nu1fDMGTUgd/a8eG3P2/hxG9rhIx+WzfQyey/ox6+pSSs/bZt6O9qFLoT7Xcu/+9Yc/ttDMB3/+z+77ciR/XdJ0EYUEQRQtZi13UpxcYYQgiAtsMkAACCa9/3pRAUYcFb3/cHg4GWAkBjCSrrWki9zYu6bq2FCLK6WBtrXN9jyFrNeZXXVSGaGlmjeCswooRAAKRSdQsIIb3RTllkZZNXxer/z9h/9tqWZdmB2JzLbX/8tc/HMxEvbPpkZmWX6aYaajaJLlJsQdAfoAT9GkHQPxCg/iB0gwSKFOiKzC5WZaUL755/15vjt11eH9a5N14WKUEbgcCLE+eds/c6e68555hjjlH0ot5oNJr0E0HOTg9Xq7Iqm/l0cX4+zbPBRVk9Pz779NMvlDTgxWy2FDTJs36v10vTPC+itq2dgzhOJ5NJlvbPz+blbLp/+9He/u7TF08fv3d3Z2drPB6v68qXLefcWOvBhvW11pZ11XXdejq/e/fu3t6tPM9Xq1VZnq9WpZTSOUCkiFYppaQJIjFxHC9WqzRNkzyVUi7WCwAYDHqD0VApmfd7O/s7SHyaprcjjohZlt2+8/D169cnJydnZyer5XwymUyGg3K95o+Swe64WlWMMc7Y7dtvjYaT9aqqW50VQ2vw8Py41+9HUZznOVC4nJ6mh3FvUBijgOrlvJWqHU9y7b3WWqkuoi5OImdwvdZlVROMnHOEIicsiRMhRJH3vSNp0kuSfLFYWIucMM6i6XReriUSHidR26jH7z64sb/9m9/87suvfn/z5m3GWFEUq9Uq4iKKIin17u7uwctfV1U1Ho2SJAn9BSm7w8ODy8vLNM9Go1G/31dKnZycnJ+fN01TFEUURavVKtBrnj179qMf/ejevXtffPHF4eHBW2+9BQTQ4Wg0ioSoq4pRGglBKQ1bwdHRUdu2+7t7o9Gov7/ftu3bb78dx//u5OQkinPnTBTn1mnvbegJSiMvLy+lkXmRKTuPomg46gOA9T5sREFOLWw729vbhOLDhw+Xy2VZVZRu9rHQ8ONcBPuPoii891K2Unahxgh8IPDm2bNnd27fjDjvj4aDfl7V62GvP7+cPvv2yZfffG2Mi5OkPxzVq+rJt09en736V//y3wohdra2Hz9+b9jrDwbDn//8v5qeT2WnAYgxjhIRRwk6j0Bsp7TWW9vD8IwPBoNOOQJJJ53DDfci7KWEEM5ZLOjO1hZnbD5dMAdtve6q8vjwgHWVlaVdNPVSNFEUOeeoo6rVla/A2HtvvbW1tVWXVZ7n4/G4LMuuLKn1uYiHSb6cLYiHIkkpZWXTBsqlB5C6U7pTSkopVZKRoecsUp3xUnemXs9npyfnhLBytfjFz3/81t0bhwdPLi8v2d7uYj4b7dxP8gwRl+tKmRVaBQBN01VV1e/3R6MhImmaZl02QogkSapWV1Xlve/1etbatqvruh6NRsaYSHDnXNM0stPWesZEcLC8btCGWYkoSiil2ijwmCRJUWQeITR7er2esga801JrZ6UzpZTT5SqdzcG6dasNxuu6O5tNtVRtW3ddB9xTypUyaQpxnPrNc6g2rDGwAEAphplDa/Vg3AsSfiy6QQgJT0jAA6I4IbTmIk6Sfhj4MWZdLmE5u/D+MsxzT0Y9AonWlqAQPGYRg6aSqrNOVc0a0Vdt0+/3kZKqqtM8FzSaz+dFfxAc6pz1YZYvtM873dZVR4qIUug6jYhRWoT6uC617LS3TmvtvCWAHIiImO7kcjavWzne2uacV01rjEtSwjnPsqzX66lWgPecM5+kkYhv3Lq1v7//9OnT0Dnrui6EJcbYteD5NfnIWtvrF3Vdd50yxlEaIh714AHA2o20ESWMEuasl1JKq51z1njwYQIbEQljrN8fcs6jiAduUUgNw/YdQmNgrXddRwgJ/dG6rler1WQyCRColBIAgoET51xw5pyrq8YYwwh99OiRlNIqvb+39/jRo5s3bwrOOefBELyq1ovFYm9v586dO3Es4jjO85wxFuZ0gkzKJo/ZXOQG27wO8CGKBy2EcMLX8f4/P/wbzp/Xn/DmbX9NZXgzUYArYuObh7XWeROI3AGfoBQBAqmFUrrJ6hjlzjkk4FMSx7EzljEGzgz6w7ZrwFmt9XZ5c7lczpezzrh+b2QsGGMYdt4TIYCi9bI2BKg1mWBOoZWdJMDzHjJqOmOtjQDO59O2rpzpjPMkSkmUL6ruxfxydn6ipXIOag0iH/a29zqpv3x5cH6xQkeSmDlLtce2M22rOmWgceVqOV8uqqZUSguuvbdCsLatj04Ok5h+/eQb9G2vV3z99Vf/+M//959+/E3dSCQUkYRhua6u6q61yl5czgk9GI1GlFIkDAkLSIkHAmgpAOM+dknI9nhGm6ZZzFaIPk5ixknZNZcvp+PxOFLdzs1d7/329mR3d3c6nUZRtF7V337z7OXL51VVtU1DUAzyESXJwevzfi+3hkwm48loDOBHg8F0Op3PytFwJ06jy9l8d//GcjXN+z0L2hPTtAskXdZLKIWyrjrZxHHcdrUF44mJOM+yDDEjpOu6tdKdt6n3pJGd99jLRL2Wr54fzeaLNOqTQdRJxXkyv5y/fnEQZ/lyUWWJUKq7sTfe3enfubsrYj9bHB0eHo/HY2s9CuqcOT4+/uijj0LUTJIk2DiNRqNeXmAYF0JcLBZh+iDwW8Nqh0KiqiqlVFVVi8Xi5z//+bNnz5RSUqvj42Pn3GQyCQw2QshsNrtSYUIAsl5XiBdSG1WeS+3+5M/+fjHo//Z3n4k4f3l4jIQVRR8oyfKEM5YlqZYFtqg7Va1neZ6HB1k7HfZzxkmcpFLKy+nF02dPmqb56U9/yjkveplqDQA6Z5wDBAoAgfsVpq5C6cI4YYwE3lV/ONKdHA6HFH0aR4h4dnia3IsHRX6weL2+WD148GA02drfv/nw3qO90W5Ff/6Xf/mXv//9J1ZdXpz/NQXkPCpX64dvPcyyvFcMx6PtqmyrskPvr5tft27dUkrVdR1FkTJSdYoxZgE3pYXz1xVakaW74x2v3PnJmWsVI66raqstm180WTao7PTsdJ4n6f7+bn/Sn16et3U3GPTQw2o5N8bkRSrbar2cKUqJ8yJKOEFVt6ZteZpSRgVBZFRQQghBAK0VehCMm1arWia91HmPjrRNg9YXaVYUBSCORwNKSb+XHR2+KqsFoOm00c6nadIfDqqmrqrGGOMRwmWEMlQp2Va1FYJ4Dwha636/773v9/vGqrDtxnFMCeu6DoD0+33GRFnWZVl2xjHGjNFh9gYpF4JFUVTWjXVWaVnXpFNyvV5zzofDoQZT17UzSmRxPuh7Sg5PT8+m04uLi15exLHQwDzLoqgX97eMMYy11njKuFRhwscbY7SxcZpY7whSB9Z6p61qulpKqYkCQCaZc44hq+q1McY7pJQSwnq9QZ4VStrlYgoABKMoHs1m86ZpOOej/tZ4vMs5Xy3mSVpwQeKId0ksVSO7VkRkb29HGntxcUE8DaiXiKHrujgNMzwUiANPvENPvLNgraeMEcoJMqM7AMSUUMIomL2dXaU69DYRkbNKt03pdFEUqtMUWRpRwSKlDSccKCOEOQfG+raRTVtrbSmlQDDLitAvfPHiRQiKXddtbHnfqGtDth7y3BChEZESLrgI+4UxTmtrrdUqMCo8pdQ5sNZWsqaUbARwgHqPhCDn0Xg4yvIkyzKKpOuauq5lJ9umAxuQCUsI6aQMvvIeYLFcLlcr6xwgeu/Lsmzbtu06wSgiplEUpq26pkZrKCU7W+NhfxDUym/t3wigUZqmpxfncRwjesbI7u72zZv71wYB4RmmhBNk4RyMsVyg94B4Hb83QV0p82bIR6TXje/rtODNdXsTRXizRx5a6X/nxf9irhDoC0g8Inq/UZt5881h0ZwLtkbOe0s8ESLyHr33nHMCPkoyYwwS75zbc7ZpmsVisX/74q37b89ms7IsTw+ercp127Z1W8tSybj2QFDLndFwua6sMl4pS2hZrb33eZ6rznddE0c0imNj8fnrw/nF2fnZUV2uwPk0zSnhlAve2aZRs6r1ThjruaOAsfV0PlvVdXt2cXr71i4SI4303jayJIzFmZhMtj/83juUuPGkV1XTo+NXsqul7qqmjFLRzStESgh1QChnVipGRW/Qs8aen11eDREAIhEiapqmaRqlZBzHRVEwxrquq+uackUoifNYSrlu1oyxXq833hmPRqOtrcn3f/TjUG+cT2e/+c1v4zi+tX37xYsXy2VZFIW3pKl1VUpG6auXx01dDorigw8+un3rxmx2yZm4cePG2dG8afVwsjXZ2s7z/OzypCiSsnJxJgS3QFrOiYigkZXWSyTMQxXFIvLEew3oCIkZ95yjN4wx5h3U62o5q8q8bWozvVyvyyXnvN8byqbb3eo/e/L69evjNE17gxHjJInZ43fuj7e2J1vDqqp+//vfewdt0wkeUcqiCMNzlOd5kAR1znAe3759czIar9drRL9cr4Jur1JKCEFpCLqm18u7bliW5Xq93Nvb+4//8S//2T/7Zzdu7HmE8/NzY8ztm7fG46Fsu9ViPpsuQNtaV+ghjpLJeMs4WzftdPYsoTrJjiil/91//w8PDk9XZdO0pTa+kw2PIuMk4YwxOhqNeMnLZcN4aq02RnVd5xDiWBRFFiVCyk4I8e7w3V4vb9taqa5pqvlyMR5sOee8B+8doSQ8yNbq9Vpaa43RQdMWkSF6RFxM50kSe49CRASIN3YyHN/Y3mvbFi1BCxzFarpGczY9ny/mK5hkVakEz1fLOkngrTt3Hz589Jf/7t9/8/XTnZ2du7fugycI1FrrjXUWGFpr7dbW1nw+X61WWX/gnOs6TUik7Gac8hqJdM5lccqpWC1my9kqonzcL7a3d999/Jg5T9Os37ZaKxfFcZwWu7v7QsTGqF6RKWO0Nf1evlqtLi4usizrugbR07Zt2ooSzJLIIenqRjAGzhtjwFrvHANf9Io0TW+lwhiznK+M85xSR9nu7v7t26LoDQBclhbSdFvbu+LZs7Jux+Phsmnh/LLX61kPRETO14zzwWBAd3erqjo6OtJaMkK4oNrI6axO+4n3Ps8nl5cXOzs7URQFGq1zTnCKaAihg8GA86hpOqWU1Npa5pxzzkDQPKFACBkMegEwNFZ3stVGxUmUJEmqEu8dIUgpFXHUyq696ELB5AhxhGmnpLYpj/KiEEJwJsPcPwAQROMVUJLGiXPOgkdEKjgyJJwQTsBA1ZSUcpDAOW9lJ+UshDdjjPdIkCLh0/ni9PQ0y4ooilZlqw0kcZGmuYhT65FY7wBX5RrBFr3MgSMERMR2dm/89Kc/GY8n//Jf/su6beI0uZzPbOsIIev1GpEKHnEeeeu0tg5cSIHjOI6iJJSbWhulFAE0WhGSxIIZbZ2RBH3CmZbd+XqZpP3t8bZHaqwr28Z7ZCSMXDLvQWrlLERpEkexUl2UZhcXF0mSXFxcDAYDzjfDmaF6ppSGZnkAGDeFNTKCjBIOzFJKEWmQfGjq9koi1xEC1npjZNd1KAiljHPBGDfGYFC2wc28AKMCwXmPV/Q+4sAYa5jgwQiYEmSRcAjz1dIhaGcv57PAYFLWKGsSLrTW6D2nlCUJA4zjeDweP7hz786dOzs7O3mWhesCgCSOh6OCUDeZTLa2tt56660Ay4URgxDUA68tsAsZja9l6f5OFA8fGB7pN7sS3puQGYSfzF8JK4WEAN+Ybrj+xusXr9OCvwMwhE+7+rowSEmvPyH8WNenHXj+3nvnCCL1DhGoJ+CAACFEMCEcUqK1jilLin4xHN+4c88515TVcrmcXrw4Ozu7vLxczFfrukJE53G9Xh8cn/hGOme9EIyLiFHO+aDf0xDPTcsRqfeXFxfr5bRczBG91aaqqkhIHifW+LKxaZIPxrvD0q9WpfOUEE4JV9bNzs4up8dn56/fun9jd3dr8Pi+0a4oBl7jaDT52c9/nCT84Ojp7s67Un7/8PDln/ziT05PpkmeWnuOKIw3QGic5so4HgnX+jzioehs23ZdLkMby6FHRsCCNJJ0JIq5I44ISLIkLHKURpyPR6PB7u7uaDTK83yxWDx98fyTTz45ODgIWeY/+Af/4MZw99WrV94fpmnqLazX1eHhSRrHjLGz04v0rTjLCs6itm3rcp0kyWA0vpheDrdHk53tumsvpxdpFrMILqcn+ze2sjSnREndWLuirGWM55Ht9SLwdLmo66ohjqvOoneCJ4IKZHxh6/WidJpwkrS1ZEx0tWQo59P1L/7o/tnp/OJssV6vOWHT84vBYHT//n1jnJZqvVycn548fvze8+fPOeda2V4/995//vnn1Wp9eXk5Hg2Gw+HOztbe3k4kxMnp0Xxx4YHkeRrG2o+ODrz3WZbduXP7zp1blOLdu7et1VmWvH798osvPnvvvccns7Mnz5/s7e399Kc/jqNoOVsKQtuyGQ4Gy+WSUKaVopQbQBFTad18OqXs5N/+u7/82R/9V1Tw+XKWpjkXsbVGNbpt2/5wIARvlXTOMcKDrZdSnXOO8DDlZLUNUqdm98b+2dmZUuro6GA47Kd5Updd6OKG0TEuKCHonA3K0NZp55yUxjnjwQFAL+W727veWBDeWq+aVvD49PQ8SZI4KbZ3blAWr5drQptutj49PVUXlHO+t7v/1ZdfIhJKxWSyrZSZjEZa28PDw4ODA6XUcNi3WodyKEqTJEleH74yxoT7ymrVdtYYo1FfP8teeWNMrxhIZafzlXHYdcrkMNne/9M//gWru/ZiNp3NZ1mWeYpHZ+dZkRPGKCXAoiRJt7e3Y8F+/etfv3z58t69e+PdIRBU1ogounf/bqfs9HJ+0U0FZdYrI5XVxoNN42SnN9y/sbs6OTg+Pq6bzhiX5sVqVTqP3uPNW7ezXnE2XSRZunvjbn/r5dnZSatwuS6X6zKezyPG27ZV1hRpMRqPEVFraYxqmibiPE3jNI0BICnSoHcbishQ552fn0sp9/duhsy06zqtbWgsOW+tA2uCzowLLV3nTZbkgfRxPciLiJ1sGWF5msci0lp3TdvWTehh7+3thSGfFXGdact2LW1HKZ30M9gg3si5CGLkWZbNZrPQB3LeOm8pI3ESIYFOK0qplCqKqLWubYLEKXDOV6syKI61bVvXNaVUaxka5wHnXK/X5+dNnAjvPXprrdZWMYZZ3o8TNpmMd/a2CVBrzWq1tM6XZck4j5O0rBqlVCRiIYRR1hgHSCglnPtONqxmhBDnDaB1XgOyKKblajYZj3pZ0bWQxLzXy5fL+asXlxAPBr0+ofz04mK1WnkgKWGyNXGabKqrOC6KnFGUS+0Jdm13fn4eoMUgIBiy2gAVvBm63qiJCSL13mltpdRt03Vdp7VGpNaGrMIDkBDsvAPvMVBbgvkCIYQywnlkjFuv11abruuMMZQh59wY0FoHfmW4ASilSqm2bYuiiOP4ugXgvaeURjGtjHZGU4QkjntpMplM7t6+896772xvb+dpRikNBBpnjOCMTsaEkHtv3YnjeHtncs29uFJJQgAS5NbDlQbn0v+8TRA6CVcNBXtd1l+TEq4TiOuWAbzhPnCdKxhj8I3j7/ASrhuW1z+E8xqBELLRb6GUIpI38g8alCGcgwDhdFYGEXHrfGCJAyXoAJkwiN6AAy5EEgmRpYMiH91/cHc2mzVV3XWdajsA8M5VVfWf/vqvZ7NZ1TaE0bprp7OZMto1LVADshEkihiHtmtKZaWcTCbGGNPZOMrSJC2rplrX4GkURVlWNLWyFjwQypmIotTFXLi9m7vvvf/o1u29W7duEkTOxOvXx3XV/uo3f71YTr3ttnd6WcZ7RYoMRRIxxjolEZl1HpDzCLU1mcgBwVrbdHWCCWFIOVWmU8YAcXFGqYjqulpVTWKjfr9f9PtOt8aYKI5u3Ljx4MGDnZ0dRGzb9l/8i39xeXkZENOYR4QQ3anDVweRQ0K991qpriiKJE7BOa3NfD4PHKCmaRbLGaWEAGmaNe2l59PzyXzw4UfvrddzIFbEJIpY261FtNUfxB5U3VRaVeiNYEwzHScInuJcGdVSzBljEY+aSjMqE8EIcGcJBS5oDACqMQDESlgu6unFHCybjHeKfHR0dLS1vfvBBx80pby8vBxvbXGWrldtlqquU0mSybapq9aD/fLLr95//Hi1XgghtrcneZ6v1gtwfja7bJqK0CgoN4d6IM/zGzduvP/++1EUzefzd999d71eHx4ejkajf/Nv/s2f//mfK2aSLM7z1Dj96tVpW9UJi4s0oZTncRbladOpVsvluorTBAgjXGjnf//JZ3/yZ//Nh9/7aL5Ycs7jNKvKRiuJSONIOA/lcr23t3fnzp15tT4+Pl4sFs45ZCSk4FEigmXrT3/64+Vy2e/3P/vsM2vtdD4TLArjDgSBMRIczBEwFKWMMO+tMRYQGYuiiP/ow/du37zVtq2zxhu9XlWz+eXJ4RFjrJUagNTKURHd3b+5tbN7b73+3Ve/Q8Qoisbjcdu2zrm2bXd2doiHOI6zPLFON22V5QlB8N7GcTyZTOI4Xi6Xob1LKWeMwdWYNGOMUhd2Cc551h+U63XbqTjNOu3m63JZVZ4yNl3PpdfT9ZwIggbLssyKHACWy/n21tbdu3dXdfv7z57/1f/6q9PT0yfPD372Zz98++23e71eVGSURsvFerFYJEkkKCcuihDROm9sxNkoy3aLYf+uL3o9aezlxQwJl8pp46u2++Vf/21W9IeT/Xv339IQi7hPWLmubFm33vt1VaMH723E+IYKVzchRoqIcULjOA46fZ7TMEXW6/WMMePx+OnTp6H1Za1VSjdN1xRNFCUBKTVyo8fZydY7YJEgBBB9cD5USmllPUIcC0So6zLP+kopb6xxYLV2zoFggpGzk4Oq7vV6OWNsOCyct0F+QIV6wnvw3jrXSam1zotCaW2spQDKSKoIAFhvPLqm09Z2y+VyOBwi0lVVR1EURnq0003bnU8vpJSEk7yftW073OohIo8peo/EGqutRUTY3t3yRouIOme2d0aT8VBrdXh4eH521nb1xcWFMtpYRxkLIflacifEEkoYAHgL6zp4YjFrjPcoVc2oTzgnDAWD/b3x9tbDIouUkudnUczd4VHnPVrrZKdlIx2hXBjnYTFfEc6kDo4ZDgCWizkhpJ/2yrIMPH+lVOBehHAV1NTDMpIrJUGzKZidMcYaZzcWRyZEQO8xABDuSmm/bhsbhI8IcQ4oYhRFcRwLzpRSZdkYJf1mggiNdtqZVnaUswTBhwl3glIr653UinRtSlAIYbQCgoPR0NXKW5um6a0bN/pFz3s/HA5v7t+4c+t2URQEwFobC0EpNUoTQjw1aZrs7e0ytpmJ8mC7zhhj2laGkEwpDzmTtR6IfzOQXydPbwIAbxIUrv/8dxKFa+zhmpBBrqwfNlDNmzbi3l+rPl9/8ianAfMdJrF5jSBSRPQeAwhkjTfaAQBBD0AQCRAEgp4gEOIJAkAURQDEE0tAW+ekcuCQswQ5yXusPyAMiZIdA88pM6r7wQffW61W89ViPp++ePXyy6+/en30uuvaNKM+5kUSUUIIAcJZB5gA7Yym2iGzoL033mnVYd11XSel9Y4ygUYZo4wlUcLG4+zHP/no7cd3CcjL2eF6vU7i7PjkfL0si2xyfHz41v0brw9eXV4cPX730dfffPvf/W//B8qYsRaJ00YbZ7mHdVXGSZoJsVqvlFKDQa/f72dZ5HwnpWyaOkxLIbF5IYbDwWDQF0L86MP/KuBDy+Xy6bdP/tf/+Mvlctl13fb2NkUyn86EEOPxOHS1D18fnL5+0e/3b9+5sVpWRVbEcd7VrerkxcU5IsZx1DSVs42ISJIy7Ewjy0auv/jqs+EkY9wnKY8Ttn9ju+kuBSdKtoz7OGJKkaZRXesIU1q2RndN03gPSRJryUpvm6rSHeoc0aGggrOIEq6Umk4Xo9FouVwvp+v/8G9/CZStlussy+Ko+OM/+q8fPnz788++ssa//+6PmrW5sXP32yfPQ+C31gcKs/f+vffe6w+KcrV2zs0X0+Vq3ssz7y2ltFNtJ5u2q6Vqh6P+/v5+f1A0bXV2ftK0FaBL0shYxTmv6+rps2/f+8n3Hj1+VK3Wv/nNb85PTntJsbu164xt604b2+MjqS1qWreNRaKMdtav1tXF5ezJs6ePHz/+5JNP5ouFb6HtmjTJpdFt21LCxsPRw/uPdnd38fLk/OLMOkMphaDTkMU7Ozt//ud/3h8UezduHB0dTbZGv//4t3EixuOhYEEvTjrrQ2/FWvTeWGcQkVKklHNOCcXBoDcajXppT3WSeZyvgnqsXcxXynmlbGdM1utN66bPY835cP9GNtm6uT6v61prfe/evadPn+Z5+vr1iyyPj14fPHjwYG9vuz/IV+u50i1Dok0X5YPxeGytnc/nAUuOksIYG4YnQ4HtXee9j6JoOOxbJHUnLZJ8MFwtprP1+tuXL3/5q79hHqE/HCijPSWAmGQZjcTFxUW5LnuDofHw+Rdf/cVf/KvT09PBYHBwcs5+/7vtvd3xzg5oK6WezmeLxYIhWy2X3jqOhANRRsuuXl/OpjyqzBIRgdCTs9PlqrqcrcbjbSSslXaxPjs8OYvSXt2269pEad8YI9UqiL3IrkPvWMG6Vp5351bLpqq9M8PhcH9nN45FuV7PZjPHwiC43Nvb7bpuPB5/8803QR48iqKmaZVSgXRalrW1Vi7nzrng47BpElPKGFuVlda6bVujXZTEeZ6HrDZLci2Xsu2stYJxAEAPdV0SCk21Qm/SNOaCUkbjOEqSxCsMtb6UnTEm1L6BrkjZZusP4Ed4nZJIq65rjS0wTlhgtIV9hBDCOOlko5S6cePG/Qf3yrJsVLtYLOaLTggRx3ww6FNKu7au6xLRd9JJVfcHGaH07Pj8k09+vzWZ3Lp16+LycjpfFEUBuMH2A3/CWu8cgEdKqbPgvY4iATQ4mnilOi0b1bYyEmkSnZweDIfJvbc+6mXZl1980nbl7vbo5aujtm4AN7QABwgAQggPgIwCgNa6lYqAd95Hcdy2bXDrCKdRFEXgD3rvlVKBtRQ4OIHbGP5trTXaXnffA1bGGFMqjBtZazylBDxhVBDcECGNMQ5QCEcIUVLXTd00DXpIkujaVtH4oA8trmI2ZVfui0EaISTagViws7Nz9vx1LPju9tY7jx5OJhOtVBzH29vbRZb0i8x733UdAe+tAXCIaK1mjIQstmlqShnnXMlWa911QUaNOQdaa+/AGkcYuW4x4H/GLXiDZbhBArRR1xjANXXAOZfneegLhLUNPZ1rJCDwD95MFLTWb+YZ4Ru99x50yEM2GMYV8pEkyUYp1pMrBIgQQoWICWNIiUfwSD0jeDVnwSgDTkQM4AG0AUQgFAhQnhNKwXlwS7SGRhEVaeRk3hvcuHMHjHz74f2HD946Pj4s6+rs/Hy1XCqlFrM5aJMkCRrmtRoV/aaqu06Vcqm07g2GRVG0lC87YExkWdE2XdM0nNE4SXf3Jx9/8tv56mg2Pzk7Oeo69f3vfX9nZ/fOvX1G+ju7Q6XrmzfGX31tVqvV8dHpz3+2yPIkiiJCY1TolPPeB9Ph5XJOCIzHw+GwD+jqZaOURHS9fqq19J70+1v33rpz8+ZNQqCqqlcvXx4cHLx8+TJIBPZ6PUZ5EsUHr16Px2OXpEqps5Nz59z+/n4sEs66P/6TXzgLf/M3vymXNYDz3nnv9vb26moZrMbrpqxqncYciY/yFCj85rd/o211+/Y+oVYI/ujRg+X6tKrWy9X5ZKu3t79NmQ9MndEwd861req6Dl2CQJUyq8Xae9p1KuaGIIuimHOBSLQ2+3s3e73e0dFRHKeLxer+/Qfz2aqu5a39u+PxTrlufvOrj3f3946PLp49fZ1nA8TDyWQSppCKojeZjIIthfPmy9l0sVgYq5IkGg36jNNu1QKyQK8OzHpjzPHx8fHxcaCoHx8fl2WZpunR0dFHH33029/+9k//0f/m7cdvf/XpF0evXjvniqJAcNaYiAul6rZtm6Zz4CnhIo6cgoQNV6tVVTWfffbZH//xnyZpCosFIaRt693d3cV8dXR01O8N3n33/f39/fl8eXD4+vT0tG3byWRCOV8sZ2HS9dPPPs7zHD/9VKr24cOHz58/F0I47995dLssS4B1KPdDb9R/J9CJzjnnLSMiyGY7bWYXl+PxeHY5DeYXZxfnk53dnd3d2WKRFL3y4LBx9nQ2G83nyujVeqm1Lsty2B8oJXu94ttvvwVv41hQitpIKVvO6XDY90Yvl9PQhV+tVvP5PM2TrpNRUlBKCXEhUei6TsnOOSci0e/3W6lqqYDxtMgXq7ns5NHJ8d/8xjFvyXy6unv7Pjr/6sVLpdQFuSSEIvBV3f3l3/ztl99+c7Sa0UE2j1BqN/3mWbH/ZTK+sb292zRLwvPx9i3TSiW9rFupJVDCYwKeNq66XJ4iK4bj0eHx6bevTs/OZ7fuvcVH261UJkre+/B7r89nNDn75qsvBOM3dveePvvWY2w9LasV8bi/dwMR54s5ARdHPIpTD/bxu+/fvLG3XM57vR6hdKXqJIoQ8dmTJ5Px+NnXT8f5EKTb3d0ryzoTPR8L8NFqJYXotaoEQvv9AaW0aRrnXJwmhNKma5XWy+WSc+6od+iMN1bZXq9HkXDK0jQnBLz3SksgGMdR27a8SEWcAWEsikPHXSnlhGpUS1NWFH2lVOoya610and/Z7Va1XUtg7y4MaF6bqVtuy7JsqppOqXy/kgptSxnvUG/btssKxarJVD2zctXo739H//4x+jMN998c3JykmVZL8uDyAZjrKpqIUTMRcQHB69mL56dg/OIyeWa3B3t3br/w0p9HUVRYBqP+ruMMcEiTpl22jgjjSOEsIT0mh54IEi0103dlqtVLdhw2GdEpGlP2+jZy5kzp69fz7qODId9TE6iNG5bWasGGWZRHDEOQESUUMpFHMXjrRByBr2Bt3h6eaGU4pRGbdPv9613gRxgjBGMM0KVUm3dMI+MEOvAWIeEAhJtTX80PD4+jaJIW6us5YRgzBMx9N4bgM5o6TWiVloKFFmW5WlkjTeAdScBwBGWpAVjIqjiOKcBKbN478a2EFwrlbEiTWNvrLcaFBRxlqUp9URPl9S5rEj9at1omabpvUcP9u/cmkwmw+Ew1OKYRmvdOec8egAHAMgQwLaaGMfC5uIcc8YZY4z2VVUzxhgTrXFgHCISSjoHwgSkx8IfcgwDynIVy0FrExJNZa9aFTYwFRwhhBLS1g2lVFBGCHEUPKGcc0Jpa22AAa7xg2s8xlyhN9dQEyKiloGLYJwPxBEeJUJQb4FQirDJP1gUbRoTSXbVNyJICCACIASTOg9hFMWBB04BwglYYJEFAApkOAEAEzKR6xlmxNHg7o8f/exDKaWU88uDuq6b9apcrZfT6Xw2s7LjlExnF4Slx8eHs8VcGTm97GbTE4+wXhNr7Wp6rpR0zuzsbv3o+z/98U9+cHR08Mmnv12vYdS/m+0n1douZq/29tth3+7sbP3ub3998+be7bsf/Opv/2pVm1cnp7cf3MW//lXXLZSieTpaLtpBOgBJJv3k7OysVKuIKi7Ezrg/uH/HE0zTmEdCCCGNXiwWv/v0s+l0Wte1MJHTjtGiV6TeY1MCIZYiCpLpxoFB4bGfZ5SALufncvXBR+/1s+SH3//B8vLsP/yHXw56fSXXzpjbN7edzS4vD1489R9++D6luJhPKaWf/fbTTz75uG4r7pPVZZkPop29gRAxJXyyt/X5F5+O9jOWYZ4lfZ9PRP/2Xtt1ZcVsSvhqZaeXry/O1LqEqoT33v1gNl8V/Z6gguV57V1pjde0W7TSiZ2bb7149erF8QWNs7IsH2yNs37+69/+JuqJSq6/ffnlx1/+pm3bPEs++fjj/f1tQshyNUWvd3a2hRAfvv/B86dP2q4mgBTJcrk8Oz0fj8dGbmxKRsVwPB5//PHH77333unFuZaGIjs9PusaabW7sX9rejnnLPryl7/+8MMPp9++nBovZdsuL248eHBi1jROPXRK19Ypqdzu9m6ntAWM8572ohhsHRyeEcr+x3/6T//5//I/zxfTdx7dWy+WwzwhYMb97Mb2aHZ2UlfN+YtDvW5Ruaaq4yJJipwwXLbr0sjy9WtEmorsz/70Hzn772Peb6vm9HyNiN7H3nvvkFJOgBprtDGcEOcQCVKKEeG2s4uL+cVqKYSYjHZu3nrr66+/XqzKrLdTN/6rr1522gyGdmu0V9ft+cEpSBdF0f1b7yyXC9seN7UdDnafvDgwwMAjTYqDk7NHjx6LKJkMJ7LqOKGpSPb3hs43hMCgP0ESEUp2JrfOzqY7O+PPPvusyHNtuizLtOzyrN/v7V6evGLGbvWHKUud9MRRwfOudqzX6w0Gg9FoRDyUZVmWpUdouvZiNl22datl1dTOOXQu7CNtA7/61d9WVf39D7+/Nd6WWimlrDVd11ln205iHO3f2C+KvKnquikBlGglUjqZbBtPhIi11sFhr20luC7Yl1EkgV5Qaw8AWRzv7e31inw+nzvnCIHFcrmzvUWBv3r16sWLF3W1LopCSjlbl9uT0f7+/v7+LYrEWouUIpLz83MRJcPhgDBmrFyXgbHs8jzf2dkpimK5XJ6fn9d1vdmR0TNORMS09aGXIaV0zpnYa62BABVMGaOkBe+opcWo1+v38zxHhlEUsUggQXTEtt8VvkKIwWAQQP5gCXhNJQuHc87b75hohBDBGCLGcdzWjUdwzvXywiNkWSalPDw8/G//mz8bDocvX748OjqaLuZOGyEEF2LAedifGW7CyabiJBCucWtrK0yyXXPQwpxx2IuFEAFfSRPRNI1ROo7jra2tPM+91ZzTote7OD87Pj785JNPopiDdXmeEkLqujbGtEorLUPjRltHCcuywoEH5b23nqAxKpT+VVU55zSluF6HK7Xeuc04PhIgAQ0inFHAKIqEEIHZdL2kAT+nlFNKCWPe/YFygOAizCARAM45Y94Z770viqKrm7Ztg5UAgU0zXlDinFXKG629t0oR3cm6rpMo5pwzzr1zjewCrcE7LPL+9vb2vbv393ZvhP4FZ1Ge52EUiiBx3l21A640XGEzFxB6XmFSIyA63uvQ9USk1loA69kmSF9H6/AHreSbxIJroKVtJCGE0Kt8AkkQd3YOEJ2lngB4hxYd2I2rjXMu3JPXsMQ1tPAHKULIE5x3sJFVsNYi5SzgDUwE9AUZC+/cJB9vAiEbEMIDXClHAniE0LLdKDy9MfDy5qMRoFF3RToFAM55kiRbWwOtO6u0laqpyma9csZySry35xenL168eHnw8vzyfLFeVVXVqbY3SUNzqutabeRg0IsEr8uq38t3trbzNCXURRHXRs1ms/l08YP3f3L37u3zk4PT06O2mvbSNIv4IE8ZGqtKq7xgqZFL1ZZxVEScVnXdH2TGGCHY2++8s7W1BUCo4J9/9uVitby8nK5WK6nt1XISLnzocCH4jSU8QQ9IOONCIAVrnEMw1lhnwONnn31y69aNwWAgtaKUOo9xkkzLGdLoxt62MUpqfXI26/Xyda1Xq+nHX3zcqMoRdXx+FBdk/+5jEfMXr18Vw+LdDx6wyGnTKGPuvXVrsj2ez2fOz7uuaTvlISqKFD3Kbi27mhIarJUYIYZSAOeMb+tGlud5njfN2lrNCCjdaWXTNL19+65S5vJydnkxF0IsFyV4jmAZM1HE4zjdaDMzsl6vv/zyyw/ef5dz7n1qrBoOx3fu3IqiqGmaRER1XXdd571njL37wfv//T/8h1LKX/7yl6vVqm3bwWAghAgstDRNf/e73733zuN33nnn6y+/2tvbM8Ycn56OJturdYmUMMYAVNt1Fqhx3gN0XccYr6tVmsbPnj197913xpPhqxdPOMHZ7HJ3d//dt98ejLbatj0+OQcAj8g5D+l9VzeeAWHMI9i6cwhNWY3vjqp6vb09qcumaptYJd6j1YYQwrjgnGullO4QkRPkIhKCUYoEfds2Xdfack0pvZieM8aiJOkDuJWv20Y7C+CCa30YmXn58qXWVtB3rbVF0ZNGFUXhveOcU+R1XTdNc3l5uV6vvTYRZ2mv1+/3hRCTycQ5NxqNAEWrXIgOZ2enxujQjpdSemuFEMvlXBBPKdFd1bV80M8SETVtbY1kGwkwIYwxZ5cXy+VSalVVVSulIyC1rlUHBI13qLX1rtePlqvy408+08p//8PvD3p9493lbIqU9IphkiRxJLb3bsVxvFw/my4q5GZdd53SIknz3sB5sMpyFu1u7wWGoHMuUHgQfd7rdU1XlmUc9weDgVTdfD5v2zaKeVnW49EIOb+4nK3Xa/DWOvDen50u1qt20N+6sX+vaarzs5P9ne3JeDuM8aR5z4Wpe7RRGk3i/rLi1trVahWoQJSzYPtrvfPep2nqr3Y9KXmWp1JqGtMkSXgcSSk9c957kcTDrUlRZHGabNq9FDV4jZ4gI+ilapu6Q0RGqZSybds0ipUyIR8AT8CTIJsTrM4DVd0QtNag90gAnEeA2WwaupWj0Wg5n1ut/d//swePHu7t7X3++edPnjzpuo4hCdCW3Uh4OE4oIzQEwul0Np/Pw0leg/mwmbhTQScnwNobXi7woG2QJAmPRGA1MkaapvEISZb2ej0RsbIsZ8vFslyzmIdnaTAYOA9to8MW3ykZ+PDGO8ZIGA2SUmZZ9p3NPGKYZUXEMDLjwXPBOWXee4tABRdC9Hq9uq4D/n+d3Gwsp71334kPeQDglDpj4LpJQah2Wmut2qbrmrqujNIhSQrAYCqEC6NghBLCnHPaaedc07U8EqHH4bwXUdzr9SZb4/1bd4uiuHnz5mAwCFRTQkgcx2/C+FeQo3HOacqMMVLpsNpamYAuMCaMCZRjwTwQsumVxIK/GaqvqZRBiuP6W66pCQ48gL8ylEdPACgFSo3WgIw4IMQjgHNovSMOgDj/h3MNIXGEPxyOuL4Wj9QBggfn0SMlhBHKCeXIGFKKjCHZeE8QSq8gBICrsU7wfhMNPQlZA3hARH+dEvg/yA/e/DO80WohV4f3lrGUc4TUZcMJWAvOAwUw6sb9R+//8KdKqUY2VVWtyrKTTVt26/X69cGrZ8+ezefTLEv6eQHWHR0evX7xcrGcx7G4ffvmeDLUneq6VnCapfE7jx4521T1Ko7o+fn5xfnRgwcPABoAXxTJerVQepEXPkmFdfbBg7cYY5Tw0Sidzs6fPn1mNDSdWq2q1bIEoGnaF0Jobdu2Xas1ABIfukuUEAIEw7C9Z4Qgs0AtAes9MsrTWNaLT7/4/OzyosgH+zdufPvtUwBS9AevXh9Zh4i4XCwOT6Z5njdNtVqtVs06SRJKhQHbaq2cn63WX3/zrO2Wg/GoqszLV6/4a1rVhgucTi+Iq5SyRhOKQAmREpFEUQx5nrdtm+X9QP/inNd1zQjxzmrVpklU9HPnzOXlzCE8vvuYc/7lV18/e/oSkWprD48vF6smpEFREqd5NppM8jSRUh4cvvr888+fP38+X0wHvSIMt4+3tvO8N59PHz/+4Je//OW3336rtV7XFaX05OTk66+/Dkyj1Wq1Wq0QcTgc1nVdVdXyYnF8cjSZTB4+fGiMKYpiejnbu7F/G2nTmShOD0/P3cGRMt4ZiUhbabI8Mc5qrb/84os//sXfe3T/rd//+m+auszS+ObN/ffff59H6cvXB03TcBFLKYEAY8w66xxSoBTRaysE7/UGJ63enmxJKe/fv//86Yu23YxigcM4jnu9PmdRXVVBGueaNkQp8c50Xad0J5yuqsY/A0KIMlYI4cAyxigllDNrLYAbj8dlVR0cHC0WCwYWEdMsdgBRFCmtKUOrVdgY1+sVXA15CcGiiGdZMplMTk/PjXHGdk1nlPGr1UIqI4QwRgG4rmvyLJ5MBtPptE+dBddWNekXD27fGg6HB4evqtWScSaQ0HVZVVW1rKpayk7JWikDvq4bZQwTPIkiZY2xlhIGgHHKOqW/+uZbSvkH776nrZmvljf294fjbbCOAEpHjl8ffvXt8/V6HedFwNqt8dahMyaKoJf3er0BAFhrsziZTCZNUyVZyiNhZyunDaW0qqrZ9LJpmjTPkihu6k4q44EAY73hkBBiAKq6jOLi/GJ6crrYmuy1jZrP1vfu3n94/57W8nx6XpalUl2cJsUg6w36hBB1NA13mDEmFKzOWefsuiqTJB6NhixiQfMnyPuvW0kICe/U1uRt6r2ngqVZyuMIONHGGCWJIQCgne0xAYB13axWa621YNwYU9c1HY+99yQ4iTnvPTjnrbUEHAFnvPHOO2lkx533umsBgEVCyZYQ0rbtYNibL6az+eW///f//s6dO3meS62QEgBQ1mitOdkok/grynooAT0SKXUcx0LE3nshCAC0bVtVVSgBhQjEfgx8BeU05SyNIsG4Up0ymnGSpmnbtlGaCEoc+Kqum6YJ86XOEu8NAIkSkcpUypX1BihYqT3x4IECZSLKoiyzmVKq1+uFcfMr+fc0JDTee2W0vaLUhZwgpjEA9Pt9xljoULgrZCu8x1qj1Ya6GCKK0Q48IYjg0BmLFMA6sG61XlptwDvKiGCMMaa11EpbA8pbjJxIEgCntbTW8igKCmO7u7tFloVJgclktL29/dajx6vVikexto6LKM1zY4zU5ppDsNFV0TokBO0VweKab+EceI+Bp+m9d06BJ4RsugmbtsUVD/GarKCMvg6cb4ZVFieISCBwTfNjCAABAABJREFUZ51z4LQlYeQBgASCpkfvLQCgQ+O+wwwopUgZYewagsI3hiYCNEKZCN9L0HtCuBA8jlmUAFJA6pARJNYDAliH6AFho/eAQBwCIoaUwV3LQhD0mzZEeNt/OVEI7nzXd3LAXQghWgfsDSll4LwD9OjBO62RsSge9mLwvatPs8ZQ04H3ZVmenZ8ul3OpwghRtVwsdrb2KbKyWpWrpsh6Mc84if/Tf/jliyff9nrF7mSnSsTFhXZKp4JPBv2379/jPHr44J2Tk7MXz1/neS9JMhLhZJwIETdN9/r465Pji7ZRRTHkxqBXWtZae+I9xyIinMWpsh14vAob3w2dKmWsdwDehwUjGCfxYDh89OP3IpF8/PGnb7314O7D+y9eHa1W5Xgrn6+qo7PLTfZfNvr0UmsJAAaiZS0Zo3Eszqb1r3/3ldLN4dHLNOP9r0+tY02btYtyvXxKKAB42XZpAllWEPRKrmULztIozimJm3pttSnX1YO37htjpheXaZx4C03TvP324zt37x4eHsVxnKa9d95596uvvn7y9PnZxfz+/QfgyWJZ1o3inCNgkfeTOIuiiHIBWoEn89XKE9zZ3tu/sTudXqxX1eHhYZZlPIqPjo+fPX9+cXm5s7Ozs7v74P79H/zgB1mW/fKXv1RtF0DBYLluua6s++ijj2azmbX2/Q8/+P3vPmY81gAnZ9PRZKvVxjOnjdPaSq2aqrHe8SQ11sZp7sEeHx9W1fqdtx+8/fCtp0+f3n/rwa2b+51sLufz84vpcr2Ser6u1ojIOImiNMljEXNrbStlkcS9PCvz1Bo9m13GiciKtJUpoRimHuI4Hgz6URRTAkp13ntvjdbSOSUiRgCcN845FIQ7ZrySbagtYmk0F5FwEQA29UZ4irPo4mLKOT86OmnbNoo4Mvruu++EJ2W1WBRFEcSpil6mmwa8B3CIhDFOCHny5FlZdcaC8VQkyjnHOEnSwigJ6DyY4bC/u7c9X1yqpgQAMHJ71H/8zsM4jsvllHrLKBdCxABgPIgocUhsSzkhTiunlfeeRjFjTHWt944QKrVOkoQCqZvm1eHBcDS+dePmux990DWy7lotDSFkuixfPH12cHBMKc2Btm0LDiMRA2XoPQLlhGptrLWMbSR3vPedVowRKWWwK6ybqlNyNBoNh0Ot9dAarQ1aH0UJ51zpbjqdtlU9Gt9pG9l0crZYr5frsqq7Tl3OZwDu8nK2rlbrqmRxBMRTSpFRIWIAEjgKTdOoSgWRUQc2y7LhqMBg7sIJEss4bPXHV6oUSIHzZMN/NNZqqzrVNrKTUgKQUH1GdFNQdl3XdZ2LYs55CIfXzDL7xgFX/QIPnoToACA4L6uqKIqdyRaLRNM0o8FwPp+vy/Wvf/ObL778UghhrZVd55zjnAu20TwmANa5ULw65whi1htGIg5PLCGYZZn3XnaKIBU8iuM4WJ4g4kZqNIquz6ppKiDonXHgHXjvvTRaOyso6fX7Ae5udOucZ5SKmDmP2hpnSZxmSDtKGWEUCVJKojQKOW8UJUmWhc3RWusRjDFS6zRNmZRaa2U0WuOtC+cTAI8gzLyuq2ssOuQTWmspdViETfVvNaM0NFm8sd4DIjJKjAbOKacE0HHOoygCK5KIg7fh9w2/mvc+jL9+/4c/6PV64/E4juOwmL3eYDQa7e7vhzPQ1hLGPKK21khJNiYxLtBUlVKBeFFrc/160CYCAERqPdow9EjQeEc8GO+N9yGSI4bi3BOyaV44j1dJwneERED0CB68A7DOO+Occ0h8kFG34KkH67xF571HDwBeG3vdeEIPxjrnzSZpeANLsJsGmsMgHY10s+yUI4uAUCDUIgMgzoHzAOARDSJ6C1dYCCWEIA0jkmj9FfXBodskBB4AGFyvyR9gCddJ0t+BFgRjgOA9OAveow+9HotRXljrm9Ah8j5QUBnhVb2M4zgrBneS9A7cI4Q0bb1czm/eutu0VV1X33zz1fHxMaAjtBNCOFnXlU5iVxR9KSVnab83GfS3er3hRx/9II7j99794PDweHt7N45jzsTZ8mi1uryzde/OW/c+++SL2dQWeYEIzrp+nyMUXacRnHet9Z33aJQGfz2V+h11NMsS9BvLwc1NK3iaZw8ePbp37/7nX3395NmzB48e33/n0ddffztdLkSWaOsIInDqCVBGMKZaa2Yz03WeEkv4bC1n6wtnVVn5JBu1XX7z1h4X2/PFdLVadK2klFqYWUwN5OBRGaOcocip4M26Iox6cFHE93f3Tk7Ophez0Wik2q5put3d3Z2tHQB67+5Dj+TsYvbr3/yubSWltO2U936YZePJdtu26DEax3GWauu79dooxSOBiGVZCSFaqY1xJydH55cXeZ7HcXz//v1WdjwSq3L9V3/1V0+ePLm4uJjP55PJ5OL0rCiKO7duc87X63XExa3d/STB5Xw2nU4fPnh078HD5y9eNdJ6333z/LfK2OFg3HWqVTKKIhs7rbUGrNtuPOipdmVk+9UXn/70h9/7+c9+nCb83t37QPhsOV/Xqq7rddPOpguPHWMsyrLReDAc9hFxVa5U11mp2nKN3r589tQax0QE6IQQSEFr7bwLekrO2SuqTShsNGUeiWAUnXPeWw2mNy52tveklHXdtp0ybSNolCRR16mqbYx34/GSEk6RRDwyHhizSmkjldbW+oAQE8owioRzJo6F6WpnLBIf6PNtK2ezGSAHZFGciijiLCrrinPaNpJzkmXJ1vaIUtCm1eUqjuPxaPDO2w8fPHiwXiysUkYqdnp66r1HRpu2Nei1d+uuqZoaCFHOamdRK2V04HAKIeI4bTsVwt5sufziqy+jJP7h93/w7Mnzw4Pjum7TONPKLKqOJQWlvLOG8JhznkYpRWa1o4SF/gIiFr18uVxShlXbeOLjNKnqstfrEYpSylhE/X7fOTedThljxvmIMmWs1MYY00nL4my1WqVFbq09OzvzYOI009Z89fW3dV1W9TpJU8IEpVFdlqVoA01GSimlJoQkSeI7r7V23hAKSLzfhA0LKIxRSvkootZopZRHQq8Oa51WSnmvtA10QgAQSRxF0drYwEptmq5tW9XpJEkYYzFHwhhBorTS2hpjnQPnALylBDjjlFLG+aDfo2EmmkCWxh5j55xgeZZEbDKKIs4isapKdD7P8zTLnHPeOaREG8OBEcYIMuY2wSbkmMEk2jmXpqm1NmgYBEbC9ZBkYBpnWcYprZfLYKMcbAy7rrEuWCFQxsloMNzd3c6ybDa/PD4+RtdRwrgQjAkE6gC9xyhK2i4O+V+rZAD/AcBardSmilLWOOe88kor4yyllHJOtA6te0AAgtqapmnW6/Xt27fn8/m6rsiVW2PoHYQKOERNxhgQhM5FjAsevKOAIgHiHTJGkAvmnLNKgjWMCB4JEVEtVVXVHmySRr1eL47j0Xhrd3f38ePHVPBIJEFsW2vNBHeErteVtb4oUkKI1qYs6yAcqbW+rsUDtBOQD3VV/wcAyW+GHy1jAikjxAeqpt8MDVCDgcXiN65Ob4wzAICH6+FJ8AQQwbtNiHVAHHUbkQNC7KZq98ya62iEiP5q0a5/+nCwK6rBm90HAFCbd7kAO2lP0DjrTUQ4Ae/B+SufUkRABG/NplymngIlAS25co4IHQdEDJQFfIOgcP2feCXr9HeQhoCQMcuQIBIABEKDhBmlQC2AJQCEIUUw1mjbNZUxhnq3qlZ1Xdd16b3nglGKzrmst8WTYrRFst7We10DAGdnZ21X18upEIIQQIadWrFolPf80en67n2S9/bPz8+//Ob16emptX68vbU12blcLaYXx/u7ZHfnzsVueXG2ns8XL18cjgYTRJqlseBcNVKpxjkHQEZ5L6zCdesBKSGE1HVJCAVnkVAkYJz11nnrnr98ZR05PjnR2s3m86opl+XSORAiIoxFJPLe85Tt7O6KiF1eXholOGcAsF4tVqsmSSJOo1bJVjmP8c7OHe+5EIOtLa1UV9d11LuhtbZaW2ujzPPIW6WNUtrISCT9frE13knjuCkrVcsKK6UMY4xjfHxwqq2/d//2weHhv/7X/3o+n/cGIyD06OiAiWi8uz0a9s5OVUTTPM+jKFJStq201jIRgzNam7Kqy7LMiv5k2wRqeWg3RGmSFjnxsFwuF4vFx598UpVlIiJKKadstVpRQESMGPfOyVaPtyaL1frbZ0///B//j09fHNSNnGzv1K+Pu04x0QauQ5Ik6HwHXoLvtKac6dJQCr/77a8/fHz/g/cfF0nUSv31N8+VI4PB9qpWlHMUzFviiY+iaDzo94pe3VReG+K807qrag7keHoRZ3kUJ73+kEfMOh38xI1VVbW21s+ns8ViEUURAUcIMNwI0RqjlFLIDRE94F51urOyka00qhcxSiJpLKJvmu7k5CSO0lBaK6mzNNdGWbDWWqTIGNvb22McsjjO85wS6KqSMFoUOaU0SNFHUURZ7DyhEdNaG+2UUpSiMTpJk+Gg6PXyy+lp01QFY0Wvd+vWrVu3b3POGymrtp3N5sxoRwORJracEksoVCVy0SnlKSNIDKA3ziMNoofGOWWcdVpQVjer1WoFlABA16nj8zNKI6SiLFsaZeNiUte1giZN0pjHBJEhSyLqje86JaWOIs4YW65WIhHGGGW0cFEsoiSKu7Yry7I36FPO1ut1WdWBxUYZL8vSGJNlmYhixtjx4cmwP1isVZbFW5NBW6+MtW3bWuuNRUKEEDFlsfG17BygL8tyOp02TVMUWZZlCSYebChbEb0xSlmDCIxRD8Z5XMxn1xuWAaAsOEgxY7T33hvvrHZhzzWWCgiNhqZpwubrrOu6jnOexUkoW4OQiL+KGKGSE0IE1t5oNGKcM8Y4oUCJ9361WsVpYowJikAszwMBfjAY9LLcGFOWZaDaWWtBSRIqTkYJIUgI2o2aYRRFg8FIqY3+SYhkXacQMfDey7JerUpv1Wq1Mkr3+/3+oNiQ7SlPej0tZdNWi9UyjjeGK8aYoHVtvCNaGQ2EIiBBCmkaiyT23kMFWmvjNDqwznpjlA4WoEgpVeAseM55p2QgH8krtxVEVEotl8uDg4OHDx8Gc9HwWwQ6BVzZ0cLVUIBHIJQiUucceghe1c5bLbs0i/M8F5QoLQEgT5OQNVariiFJ8uztt9++det2kqZFUUwm24Qza4IeEsZpxr0L4a3tpLHOAxrrmrZrOxmU+JqmuQbJ7VU97r03V72D0HG4+t1pSM5ClEBEAEIpIYQpv+EibEpL2MT4cL3XnQhCCG7mORkCIIJH8J5AEHAiaB14sM57g+i9vb7fGDLvvHP2KrpvDuO+E7nCN2SanNsQ8QJXE5xX2hDrDGzy5rAtfndWxIdTRgs+CGiHqw9tCAdAEAEDjw8R3R/CCde5Al5JP20ygStcAQkCAfBglVLOWu+qrq3reraYSyllp5WUupPVulwul1VZrtbrkKM4b5qmWS7nTdNYq7Msa7s6z7MkSXZ2dkajwWKxkFIy2w4GA0R/Y7i3uxsRAs4b7613vXt3P2zqL9vGGp30eoNecWMw2Hl4749UM0DbX814Ed9+dD+1d+2je+u6rnWnl8vl5eVloxR6NugVRVEszi+924yweo9ANsiQiBg6zwAJ44QRKZ1XxnTy+Oh0uVhHSToa50cnx89fv6q7mjGRJpmUktGNjmExjotezlK4vJgLQZ1zVMnYw2CUpFEsEjPoF2mPZ4OYL0lKRRQV4HxZliZqm6ZpqrU1HXjrwTl0SKE/6nkD+3s7P/reT7755unscr63vd+1Kk57SZ5p6V6+ep2maduqJ0+ft418660H2tqqqXnM+8O+c7rpKsI9p3Ecp4geCYniBMFrLefzOUVs23Y6m4+GA6VM0zQ2iogybddFcZylaZIk4/HYex/md4qiuHv3rmD8+OCw67rdnR3O+Xw+j2N29/YdcnT8m48/PTo92d7d+/3nX3eWxEmufe0I0Z3UUhFCuqbpug6LAacotQL0jJJXr16cHB9++N671Xh4dHx6ePjaEbFD49CbZlQ40gFspJOcNeVq3TW1YJRS5hEEp3HEdddyIZq2ktqi37j5EEK01nVdr8tl3ZSAjiFwQb0n1moHzhjlnEFBLOhluZzPFlpbo21oITlnAlFMSr1crNN0g3rWdRvHsZI6yVMAkqZp05RJksQJy+J4UCRdkzTrhXc2iiJjFAI9Pz/nnNtQqBjTtEpKyTgH9IwRxkiep1HMj47Popilca8YDJO8uJwvnj5/3VRV3ajz2ZKlaZrnOXDaGeU4Fd7GaeIpsXXFYhF2LGMMeiiyLM/zsm57vR4AKN2ptvHeN237xddfMSrKqtzfG0ZxenGxYixKuGi6Fc8IExFSqjrtAeI0AgTVSQooGItEbO3cObchRRPs9XqMsbIsV6tVb9AXQsRRsrW1tVgshBDIaN21AGQQJ3ZdrlfrOBatbHxr80f3d3Z2Dl+/rKoqTdPt7e2Do0OjndYWPCIwB6iUWS6X1tpgWdY0TdM1ImKTych6xxgRQhgZKloXsOhqtQoWZ4hoPDhrQi3krQMA9MAJtZQEo0WGhLEIoaWE93sppdQaE1Jaa2yoqbS2WllEZIwiAKfgCcaCR0kUx1GSRExwqSJWEuNcmiZay+Fw0HSttbqq1uQKIddaN00jKJNS1nUNV+REAhjEGDZte8dCV3Y8HjPGFotFwPHyPA/pRdiC27ZdLpfee8EwKH1KKcsSKKVxLHp5KqXknKWYKqXmy0XV1Kvl/Pj4OBvmPIqCnA6jiEQwxglhZd1GURSYmEgJ55t2gLPQdV2QJRCcb8IVJWHIE8h3CIHzvlPKqkZK+d57713RMNE513VdHKchflzT/bz31jmK6K3TxjpKOaeEEKeNUmp37+b2eNQf9Jxz4Gwcx4wTa+3sYtbr55Px9g9/+MOHDx86D1prKiJEbDYFUBT0GZ31Fnwep13XBdZSkOIInqXBCf6qxbChiQCAxiAT+Z15Y1BGc84RvB5T3NBREfGaanqFTm+Oaz5UONy1CgIgbuRFbZDHIICEghCCWvR+gyeA94R8h8RcTx6GyYXrpph/wzpyk5RcS0EjAdxMVGrnVdtSSikTmzZQuBbvOYGra/fWOfQeER1uhBwAA9Hq6tz/cAT0v5grXCcKm7VSDozr6vpienmxmJ3PpqdnFxezaafkfD6/ODsvlytrPYYUy9jKWKUUgAvyGMbq4Oi9vpwzRkRCytn6+PyyKIosy27cuHFn514cx0p37334Q2uN9zZNU6lazvl7779//9FHiDibza5/sv2t7z9662eImGWZNaYsV9baiAtrLTjbNM304vLk5GQxmwUmf71/rpQONKmqbruuC9Ca1YY6gsQLwThjVivw3moz3rqxtbV1595bs+n86YuXjJP9mzcODw/TIj6fnnY26o/uaOgOjl8U60wIkfVBqYpysn0zp6Q3GAwGvb4xe2TDIW2kXXdts1wpCEKfyIEgi0TXNXXTOKNjEeV52sv784vlaDR48ODBf/zL/1StyvtvvX1xMRU8i6LIaN81kvPo88++PL+c/umf/tdIyZdff93K7tGjB3s3b5yenc1ml71ez2rnEbyDIB5MKNZlJeVpFPMoGC1C6Lxke3t7eZ6LmI9GoyzLnLWr1QqdL7J8PBo5bWIRWWujKKrK8sWLFxgyUmw4i14fHp2fn/+//9W//h/+yT/9m19/3MhOO+vsdX8KGGNZliVJtECW8FgpRQhhhNZtfXl5TvDxaDTwAP1BcXK+nM1mZVUhZSKO2u46Z3dt21brpVGq3+93SiFl2tuYi6ZpeJKuyrJTajwchDpHCEHo5vbePCPeWWuVUtYGdlnQTvaUUsZIlqeM8rbRddO1XWe1a5qOUso5VGVHKY2iGACiKKKUt22b94tgvDefXy6Xy514nOf5YFB0gsdxLLu265qyLLkTR8dHsjPKWECWRDHnjBAioohQNLoNtVagqw9HQ1q1SNi6rD/59PPXrw/7Ra+s28VKs+H+jksi45wixDrsbe3w/vDk4lIMdH8wyos+Ilq70aJBpGlUVOtytV60bc0ICMEFpxT8erWK+tHlurxclIQTR9xartJhbJSs6pYQQpF0XperllIaJyLvj05PT+9sPbzbT58/fSZ4upytu9r2R8np2XHTNuNJH0FdXhzFcXzjxhaAGvRHVVU51RljZucujaJ+NqlXUwBw1taruVPbH77/wenJiVRmdrmIaCyb0nZqAVNEKOc157xc1977fr8vBF8u510nGWNa25u3bxRFgoiz5bysVlW1NsY0TYNRosFZ1Vmrw5oqA0opQhgi9Q6sQXBIkXqDqnVNNxcR8xBlWUYIaZomjtJQ9wPjhBDfSUupEBETwnsPXUcIWoNaeYJusVgxzillk8nk9eHh0dFRFAtATyiV0gU/4pAIME6c001VSimtMV3XWeuMdgDE2jBOH2qyBhHjWCjVffPNV2HYRkqpte33+4i4XKy7tiUoGPVlWfKcp2lGKdXWQqd7vZjxRBqiHdHSeU+BJLUyy7qWEqJiCxmTBqI0I4QopUbDYbBy4Jwq1QVmH0VCgFrttDYeSCRE27aL+TwUChHjoaWPHvxmVIEwQqy1zhjOMkr4bFoaxybj/ePj47ruhv0JZxwRGUHgG3XCjVWaaReLRZIknFAPzjoTx+LBg+8/eOtuLKK8SNM0TYS4RtqPzxZKqaIobtx+K85GWmsgmjIRx3EcG22dtdYap6ztlNFar3zpva/b9abub9V10L0mnWxAAgTvvduIQLjAiBRCAIIxOooij2jRG2/CbkIJJYSAsYhIEGnovnvvgrebdZRS4sFpo50LgZxSquoO33B/8N6HzMB01yN5eP0GAGciuYnHjAKiJywodXPKCCUE0TsIYIjzSJBooJRQxhhwDpTaQDB0PoAQqFXYBCmlxBNCiHbhTIAQTwgC+EBHUd4E4diQ8QTESGsdx2k4PRZMb4kDD+iBMYLgvTPOW+8U8RAyxVXdLJfLw8PDV69evXr16vj4eDabNU1znahdZxvhiKO+tkYp5RlljHEgXpl6tSaECJIsLqeBAXPjxo2trfF4PL65sxWEaARN4iwOZPuQecsKB/lNABjkN8OHAwDwZB/+y0fAlgJ0FzAw5xzTa6VUuV6dn58fvHrx/PnTg1evF7PLSHBntJatlK3sNHWGAq6PTyff7//k/cdtZ/+nv/23Q9Hfu33rP/7yV6PB5PJ4NSx2mRCyoXk/VdDVnY+yFOoujwopZZYlnFBdKyM8Qe69d94eH5wWaTEoBk1THR8fR0JkHW0aaaUpXMoJaW0rIBY0M4qOtvcdT7589fzOe28nW+NnL15+/09+VPDUAVmslmSYHi5nOCzu7O/Oja7WbUPo/oNHP/zh99M46YxtpEKkVX2xt9efz+fgvDF8sVgQQkREAiophHjx/GWWZXVdd9Lcvn3blcAY+2j3tjXaKSjyHD30+/35fP705av5fM4EN84uq7VHHAwGt/Yf8H5/+uUXHlS5eF1fPP0//qOf/y//r//5bDF9fPfBi9eHjXLD8T4habEzLtd16uXJwev7d+7wuHd5fPj2w5+B2/r1b14y8Ij4T/7h/6FqGiqi/8f/83+iWn/54kQCFAWMB0NKmKPQ39o7Pj2Z1iqKok4aqZVzhHLeVA2lPBWJ7NxgMOi6rqy68XjsfV3VbRSnACCSGNFTBEJd29aE4Gg02ikSzvnl+bTtZFr0VrOF1S6KIgm2rKaUMsESKSUBiHkUM76GtQE53hkBmqZZOy/bcu1NR9GeHBz8/b//pxrcwdmZ93bZ1dbaVZcpl0pqgQKhtPEMGAPmOmttK7M0H/d7vXxwdnzBPAeJHRSvzxullmFyct5VSrN8d4858MoY42xrlAGkJgZGkzyLCaZ5EaepA4/abvYCwqxHlsU9MSpsAeC8s1p2Sssojp21RmmnnTFGGeW9Jx6TJA4bAVx5WYbtrKqqXq83mUzm83koIAKHMYD24eE0oY2jFAAZDAaCi+9ard4HYnxvOPLGaq1FnBLGiQNA6p1bLUsmOCJFisaBtsYYhwziOFWqQ0SkxCMJTy9ljFKa5704iwmnxqr1et0pqa1RVcU5D7JanNNQgcVxbK0nhCEQR4lSRimjpPa+jRkNNIhQDzVNYxLnnAv7YxzHV5Uc2xTcBK8KTcquDkrpYNir23YTeK6skpxzHlmISai8tSbw5gJ9xBjrrA6zltc1WdiRN0EUwNpNHx2Rhg5IEKn03lNKe70eOOUsBKshRHRu0wUmJLF2s9khUMZ8yIi1UwHeAACltFIqjuMoiqbreShStbbXwLIxhvEIrvgT10egvFyl21bEcVDnNMYY16VpmuTZeDyuqiqkX/5KalArHfCP4AQNAA50EG4yxoD0WZLu7Ow8fvvtR48eUYoRF1xQQdmGzYcQZyOlFBUidFIZY4xHiKiUCtHdWmscbCQQtKa44aJex6Sw2m+++GbY1lcFepA+vG69B5mm6zo+1NaEkFDBB8tKuNJEgvA6IQ7AAVjvXfhG7wEoQeIBCZLNXwQPCMYHxxHY/OOvBDzshk0J4BGReEOQIQLl4D0QRCAASIJaMwbVpD/kFV4/gP7Khioc12/DNzoXf/DTMxYgk2tcxHtf13W4+UPjmVDgnHPKpNTgfdgB6rpsyqqqKinl6eX85OTkyZMnr1+/Xi6X5sq6wnsMxdvVYm4u/A2i64ZHTAhhjAb9mDSNe71ecPPKsiTP893xMIzYhOQgMGrpldXnd1cUPh3A/SEH8/q4fqL5FWwWjkgMARC8f6y6uvzparlYrVZdU33+6SeMQNe1Z6enr1++mM/nSHwaxUcHC0G/FSKejPff/+B7RX/4+vXrF69e94ajQb/HI4EM4ogWfNDKZj1bjnqDoihsZhIReW+15pwKazUh2JQN9PpRlGitY5Fsjbe99+v57Hx2FgjRDj2LuFISwK9W68l491KcPX/6fDZd5PmgiFMnpRfF55999uzFS8rFcLLF4yTI1e/v7ydJ0uvl/X6fUtLr9Zqm4pwX8Y26bq31aZpqrcu6ccamado0Hed8axT/8R//6RdffBG8X16+fM1i+t47j43Vy8WiqSolZVPVWusgdB32bWONcQ4RldHT2azX6+3s7R8fH86mi6+++eYH3/v+T37287/5678lSG/evHl0Nq3rCll8eX7mkbS63t/b1qo5Oz3d2x4/enBfqe7ls9P57Byce/TOO3/v5z+jgjvTnRyfbI3gooQsjZM0JoRYZ8Pmj0AdeEppROLweFLKQ36fpGm4aa2VwajMOeeM5XHCOafoATwSG0URYzTPsv2bQyV13TGkbjjsZUV6eHx+Ob0AT5033oBRjbXWO9Daeodhpsp7T+mmeZckSZrFq9Wqa5um6VbLcr1eh059mqbEB90UTQjzznW6CSR35wx6pzowmdnb2/3qs0/rqjJa5/lIytYYE0D0qqmllJQSpsChlo2SZdt4QrlJaRzRJCKUkohbitY6SwAoEkYcoEX0VPCYonfeGS07b6SSlhAAC0gJGk8AnXXg/TXN21obprcgiPBrU63Lvb29OI5DEL2GFh1BEUfee4/Qyi4Mw8RaewClbZhuHxVFrzdIkiRJkuXiUrada0FbM18utZTB/Ons7KwoCqTEOIeUSKONMZGKjbNSG1+VVLJ1WWqjEmel0UcnJ1VbpXmitVLaAiIQ5tEIce07YIxBrW1QlJNSc4ach1lKag1YMNYai3Ad7I0xQRM6/LTBqyKKImstIgnbkDbXBj9ovdPGIfGeOCBMiFjEkXOOEBpQZQs+Diz0jeSOeTNchcMYi0ghum4bY5j2DPjtdaLgnLqug8OnhbcZpZxz3nrnHCWBfweEUO+dtcQ5e5WFUM4oo95IjYhB/scYI6W0xidx1nUniDQUeUGlOOQTCWF4ZW60+V7YAN1aKiklXOkoBwJm01aFK5wzRZHVdUkppmlstaYUtTbOKG8tYYQRYAH0pizP8zzPvbOEkMlkcu/e/ffff//27dtwZRuFG+IgRcS4h6HmQ0RtHCGUEDDGdEqHTCUIcF0XhQY2SYO7YoyG9W/b7rrpcNVQAACw1Ib7P1AZKGU0qCU6fy1ecN1qcM5v4j0AbF4NYgRASLBX8AAY3gkAzllKSVi96yz8Onhffwi+YRepdXjdIgTLSh/o2cZYQth3mgpAwxKBCP7dHsBdySn9wecjIqK7zgz+gEXxRqJwtTbBvtI5B+GcvUdKwTlwgBvKpCfAfF3X3rq2qxeLxenp6fnJ6XR60TTN6eV8Npudn59XVRXyxbC8Qojr1skbJ4anp6dhc48iEa4oiqI4EaGfyK+OQMDK8zwvetf5+iZFDg1nulHCejNLANiIR/3nx3VJc02w2NwPpqOUAgGMeR4X+WTnhvdg1A9+8ScAYKry9PT44NWrb7/99vnzp5fnF3lMTo9aQqTgxe7u/s7O1qOHtxfLs6LPbu4XSOl0vjQt5PE4SvO2bXe2tra2tkIWqJQyWuZ52nWdM8pIzXkUiySJ0jiOt7d3Oecff/Kr9etVTnPrjNSdEEK2jTGKc7aYnXtjZ5fL9aqckcvvfe8HH77z7l/86788ODpeVWWSZnRdUWniNOsXRVlVysimo2cX54IxIH40GsWx6Kq2Kavtnb2u6w4Ojy/Op977nR2OHrS3Dsjf+3s/Pzw8DrNLFxeXd+7u/+xnP1vOF1+8/jRL07qqTo9PghQNISRKYguecJYkmfVOSt1y+vU3T7q2ZiLZ29pal6028OjhO59/8c1iue4Pt6IoklXnQS9XFRcxowa0BQLvv/fwT372M7T697/5dcJJEomDg1fL5ez+vds379zeHfc//T30h0ApRFFEkXSy1cp4ghSJdd46S5ESQj2EfQeABPsRRA9b4wnn0Xq9bts2ieKu68xKWRUj8QBecET0PI44500711ozbjMm8h6Pk6EF9+zZQVlVUSScJW0tQ1VjjPEehdjww0IrBxHTNE/iqK5r7yBJsuAMjEjrus3zXrlYVU3trM96BWPCe88ojaKoKdfeg1JKcDooeovFwlprZBeNaA2WEp/E3IG3a+msipOMOYqNklXbSKNZzCwB5511ztJgIm+cd0HSxTkL4B0l1llntbfaGe2UMtY48FVdMY/EeeIQArkMMES4TaA11ntPCXHOuYDoWrtcLsMdcB0eaEwCiA2A0dU4+3i01TSNUiboG3rv1+v15eUlIgI4pRQ4D55Y652x1tqIccYjnqSEEOIsUkqcNc4KIQiJ4zRRxkgpRZLuT26OJkPn7GI5Ozo7t1bHsegNe/3JJNWyruuu1QDgvHWOAGyG+ANdTmvtHUUM+VDYHNnl5UWv12NMZFkW9qzQwI6iSEoZ8BJ4AxQx1nm48vUx2CkjjGaMVXW7XC2k1EAIMg+UEODsyvrIe+/BXocG51zbtgCotTXGUcrfzB7CzxGqnNCp9d4rJUMVSAkPEngh+FEk3iNAGM9Do52ShhAVcNSry0fETaQPooAbBp+FtpFN1MRx6hw4p4MpYhjxt9YabQmVCBCq83CShjLn3PWJXUfZq2Djuq5br9dbW1tSSqt1mqZJnhtjtFRJFGdZFgYTwPk0TTmLcdDf3t4Oe/3u1va9e/du7N/qFYNwDt57wO/0MTlljEdd1zVNo00jhDDOBw62cZtRBW2sMdZY55x31lwx0a4GF4MIhPrOc+E6anrvLdlMlxhjnAXBneOAgBgUbpCFn8/5TSNUvyGDcX1savcrKygXxBHCPeMDXgDfgQcb1eTN9oVXrIAQ2aUyV0mDu8IMQqEvN2KXGza+o5RS5xEZIUAskCu3qrBu5ArtuM5CyJX99JuJwh9GT+u9f3PpvPcRF4CeEMLJhtfZNA14L6UsV8uzs7PDw8Oj44Pzk9OgvSZtoLK6JMnC0oU+GmNBnWJTnIT0CxGKogjncy2/oZRqO/Luu++Ox+M0jZMk2dra6vV6WZZkWWYBEZAgsYCUUIeEEOoJsVcLiojgN9nBmwvy/+24zqg2jySyDdCDHpx3Dq0z4IjrDDjHo/zWo/dvPXr/j/7bf7i4vLycns9eHJ6eHlfNupMleD5fruNEDEcZISaOPee0qYl3GowWIo6yQimptXJWa62D2X3MmTc2uJxcnJ3Pp2x7a6tr2qqqbt682Z/0oiwWaWSUamXjnFmWS+KJt+TW/s00Tr4++3Z3a+/nP/vFO48eH7x4+ennX/R7g5s3bjWdXJZr1kmkrG1bJjghZL1er1ZLY5RWHSEkjkUm8jjPHrz99ovnzxfrlQYHztV1nSSJYIJSevPmze9///uffPLJarHsmna1mL98/iyJIsHpr3/9q6aqb968uV5XwfohENuzXpHluTa6LEtkqDtZl5U2wEXSG4wZjydb25PxzuHhedUcExEXvYxSEidMm65drxDcjd29B/fu5Fn86e++ePLNV28/fHDv7UeC02fPnrx4/m2vn/70hx89f/Z1Vakig1gwrQPsinGSAoDqpAMkFAkBj+AdIiUcGWc8BLL9/f0kSp88eQLOCyFU16HzAEAACSFJEqVpPOgVO9s7/eHSGDOaDKzDutWLxbQsV027thaKfGA0qM5wJiKRWOsBICj6WKuTJCmKYrFYoPdt2/b7/VhEo+G4bdt+b+y9ndeds0TKRrYdIYQCRoyGpz6LI3CJahoCbn9/fzGfluvl9njEOe/lsbGpUooy743hDAAJZ8CiKGo65cCLJE57vThNtHXGO0aYQ6CIlDEM7UMPxmysjJzVzqiQKPiNw5DdzHRb8NZ575FQRCRIAkrsrSOEiGBdqHRRFMaYo6MjrXUwsCGAURRJbDxBZJRR3u/3J5PJZDLp9QbHx8fWeu2sQ9Bal2V9cXHRNI2xKo7j3a1tFgtpjbM2jeMkTXvO5HlunPNaMcEjROMdY4wIK1xSluW6KrMsufvg4Xg8PDs/Wbe1rVTbKYueNq3SOtTHTdWFxaWUIgFrN/tsJGLnoNOdc8Q7ACAABIGEXolzEIpUzjlBBgAh+wskGkppKCuvKjP0iNZZZy1a68Az69frZaeVdSAYoYR7BOcRvA0ORgBwrf0XfLS994xxSqm7mpcLeEYo//CNMUK48g0KCoyCxwHbCH+L4HfThuHTwtsAwEMYNCCE4BUs4az1AZYwxgJAXbfOQa9nnQOtrXOOEkY4RSQEgVLQWuPV2EKIghpJeADA+2saZui5xHHs0UjVltXq7r3bRS87OTwCcINenxEk4HtFcffuXYp4enqKiPv7e4NBgYg7OztxHCdJEnQ4er2+EFEIUYF+769VkI0N8ck4b62xHrTWq7ION63zsMEVXLipwTsPSBin17V1+BwP6Dw4D957513YF7z3joRxancFifu/A0XgFb0/vCGga2+mCP6KD/Vm1Hmzsr8uXt88/k64eqOy36QNiOgQCEGHDpEickSKwAluHluClCBV1qMLQ+F/oD7+5vm8+aVCwH/xNK6zK7jCT8IbZNuFP0ecc86tNU3TdE1rrDo5On7y5MmrVy9ms5lsWymlMYZG8XXOaq9sQoOsyHV29eYSRYKFV4QQeZ73+/00Tbmg+/v79+7d29vbyfM8SZIAz163G8Jx3T4gf+ihhW/Yc5M/BAz+8+Pv/BYbZCIwOQlQAgQ2aT1eOW6FpyMfbg229h49/KFZLztZr8vLsp6enL4qBlvjrb2ua7pOtY3mlCV5xghaJTmLnG3bZtk0jZTSKMU57xcxo6Quy6IoppfnUsrtrfHl5fzLzz5vq3p4Mx2OBm3brtcrKWWR5fv7NxORgEPdmuVyvbe3/4u/98cfffQDo+zvfvspJUwaazpprKeEWw+L1XJVl4PRyDmntGzbWhuJzjLGtOl0pAtSiJTt39l7PH80m81ml1MEd3p2OB6P54vk668/392daNWcnR8dn8ycXf3f/2//1//dP/4nv/jFzz//7JOTk9kH770XDOIpZ0qbVlW1VLnSQfnNOHv37l3GxHy+/OyrbweDybpS9+72f/LTP2pa/cnnX0z6Ax7HVdONxsV8PqeC3bx589HD+6vV4i/+4i9U24x3tkUcXc5nu3t7PGKjyXgw6P/RL3729NnXT58+xRYZY0Z1WioeRZHgDEFrzZjw1jkg3nuPhCGLeCSiiCeMEJImibN+NpvNp9NeryeE8NZQQgAcRYwYz5IkEOrjOG6ahjLBeKxtNV+UWussy5p6Za11btOYZowheuM8ASAEvCdCiDRNT05OKIJSant7oolZLtdV2QJg4Hoh0iyJvTUOPCVonbFKG+9r7ygSRMzS5P7du7//3d8ao4peJhhP0miEvTBJp5TinFKKzmgmhLDGuzhhkcj6A5ZEVddxpQmSMOhEEBmhlNBNKaAdA6CEAUPjQROrnTfWpVFMrA80BSUlWCCEcMrjNL7eH9/YrVyQLDw9PaWUCsYDAgEA2hhApIyJKIqSmAnedG3VtCdnp1laOO+tcx6ARyLJUuNsuWzihPbHk62dva5p6rIyDppORUkWJRkaLa11QEiYnibUegdIjXPK2JSyKIkpZ51WUinKWZEMKMW6ldX00ljLBUMNbuN9bK/3O0IIKbizYK1HpIxGnEeUcELIzuQDQkhVNaHFkKap4HFwVb5CHULTgW82HcYJIQzRWq2DtIAHC55wkTDuMRgB4PU+GNCH6yTjGlFI05RzYY1vWxn2NWtt27ZCbPrxsKl0WYh8YaPvuq5tZFA6y7IMEVXXhfyGcx4iUVCepnRz8pSRq8931rqgoCdlaJtRKeum6bxHY4ySwTYz4BkhPeKtrOHqZgjNCI+EEKK15pQF8QbOOVyp/WuHdV0H97nt7e2j1wdaKqVUlqRCiF6e74wnjDFvXczF/fv3b927KaUMYo5pmvYHAyGiILHskbqQJwBa77UxStm6bcK3hzjUdEopVdc142JTifqglLLpECFleNW5COvgUDk0hAtPrDNmMyvrr4SWbSiyN5W692iMA3Ba22vs5M3wZtx3Zff1iwDAGF5H5esGP3jvLFwFbA/fqSzgm+nIH/z7OwEDgoDgA7mBIBJAGnADDwCeoCfoyVXvP3z+d5EvjFy+GZKvMgPyd1KH69MOrS7GGHmDl+O1Chcb2DZN0ywWi3K1Oj8/Pzk+fvbsyfn5uTFGUEYZUkq7VhFCguCSVjZgUQj0Oqm6tpAI6EpZlmE7zvP83r17Dx482N/fT7N4sVj0er00Ta/ng8J2zEV8nRwgogd0HsEjEvb/OwP4//O4RnfCiV5pWThrLV490YyxiEeb/0OBDQc5ZPnOCOzdrf2baX/yzrs/PD56dX5+cnBwoHQnIiToCCFpJvJhxDnKzkQCKCEUfRzRNE1fvax9zj3oLOHbk4Fsy7pZLZaXu3fu74y2Pv7447Ozxf7u5PHj937yw5/c3L89ny1/+Zd/dXhw9uCtm8iTv/pPv+U8ivNB1msup3NjTNYfWO9nsxkRbLK9e3p6ygQXEWeCi4hs5mTBtratL8unL588evDwRz/70bdff2Os1FLd7d/uFUUc8d/89lfvvfee96Yo0kcPtupqfnHufvubv/3Jj3/4f/k//Z//+T//54eHh2maXl5eDkdjEUedsWVTd8aKOHJItIf5YtU2XX8wXk4vv3ny/PzkkgL98MMPPaEGUDvbyM5Y2YvzNhWDcf/Gzb2bN29+/fXXLw5ejwaDm/s3SBQt65ZGJWGciTgt8mG/2N7ePjx8nXjsOtXUpTYuqMhbY7zVNEoA0DkPiAwJ54KziFMhBEtEVKSZMS7inBACzjNCHXgKaIw13nnnvHVd0y783NhqtVopbYveiEWx9wSApElPCFnXtTVojPVO+8iGDoiHhDPur8QhjDEijr2Xy+W6LqtPPvmsbduq7ABcJBIE5qFjG4N7CQAEwQMYpT0Cp5hnGWfk8uIi5gKcp5QIFmlivIWuVaEfDR46rRgaRwnJkzTKU5EkjqCgrEhSAx4JBeu9s94TRILOg3UJMIII4AC8dOCsN9YT5wkQIN5pY62VWjnjGWMAGF89UaEHEfZE9f/h7c9ibUmz80BsrX+KcU9nn/lOeW/eHCqzKiszq7JYVSJZLJZEtSg13ZIts2XIhoFuw4Zh2E+2Xwy/+s2wAaNbBNr90jAMobspthqSSImSyOJUxRqycs6883jms8cY/2n54d97n31vZrFlQGQgcfOcfWJH/PFHxL++tda3vqW1ty7Pc900VVV1slwpxZG1bdvaRUomhHPn87kxpizrsiwFV1rrUO7MuQzh682t7SSKkyxXcVwUpTbWkHWu2t/f7/UHjjwIYb3zAAxAKEnAjTFRkvY3BmmeNa2BWVG1WkQRY3GcRioSRTEzzjJvsiyj0lmnYRG9X3TTCWKL5BGAcaakjBgTQMx7unRpM8RCw0IZEv/r6+nKp185hcu4rURHAN56zy1GUQSLkj+zDHajJ4ykWD9gmKjAPRFCarBEzdLX98aYNI1DtGAJcSBwBoVQgSAyn5VVVQUVB8aY4IvoWTiLMWZZxraQ0iOPdOHKEgDzfkHeFEJY60P/J84EUWutZUyEpgfhKrXWq5z8wiKyC3MSUsiRUgHocM6jGIwVrWmm0/H29ubGRr+T5Vmabm5utlXdyfJOlqRp2s+zwWBw5cqVvUs7ZVkHH1HGUZ51rHeBvKmtsdaHph5Gu7quG902bUVEga4YHk7rPCAzxqCQIXZCRAQMEBAXdU3r8YAwvYE0ujLwfqmmQMCCBVpZ9+AvrtIT68kFRITlGZ/zjFcztmI/wAWJ4WJbWejPhxnCVxjwYK+W+sEs/Oc9IYbWDOG8mnPJmHdKLk/0DFB4bnirB9uDWZ1uHShIKW1oTsCBLS4HOFvUXlpr5/Pi/Pz8+OTw7OysnBcPHz6Yz2aj0aiuy0B19B6JyFgf1gchJACGuJoxoaTiIuISCBIA2Ol0AgUhyGaE/Z1zW1tbcRyHiuLQf0SpOCiSrXD5aibX3+LnDf/nyju/8PPV/PMFy2FBO13uhlKqQEWWMnBywRPMZjOkNk1T3bac8yzNN3ZvvrP1wlu6evT4wcHTR+9/8JMH924V5bhpyzSN0zwGbOqmLKuxEtI7W7dmPFHW5M43T548tFoPBoN5MSFvsjQq5pNyUu1vXfqEf7rTp3fe/MY3v/atX/zFX0o6Gx/85P1f+uW//rOffXj71r1//Qc/7PeG16+/eHQ+M9YHH9o5VzTNvCxS6AghmEgw5OeBjLOmapu2MsYkmSCijz57j3Hb6XQm89PWltZpcsy5quDi9mcf3L/36cOHDyMpo5iDS9/4cjSbTX7vn/+Lv/f3/t5Xv/rV9977IE1TzrkxxiHz3iMTyJkHhoBcqLPxhANKLvcuXRufnt36+E7bGkKxv7/7ve99789/8uPRvdtckDF1r5elKj06OXOEWtvtvUsc2WhWlK0GaybTea+b5odHu08Pmmaj1m3TaPCqbWvvLWPCe+u0cc5JKTmgDa8jcM4FJ07WWbIqYYhojPEeOlneyXIpJSOQKpWSm5YjgzzP0zQl8lpr3TIlerPZ+Xx+2ukNq9qUhSYQl/avHh2dGG04l0YbIsrz3LRacSTnQ89CY0xRFEkUAeB8Pm/r5vatu3meKxVrrdM0aRrdVjUAAJLVRjCe5hnn3BnryWZxmih1+/Zt50yaRG1ddfOMM9k2pq5aJKZEZJw1xqFHYVqNyGSslJAc0BqLAImKQHAHaI233i0lxNBzikGS89bZtm1t0zit0XlBqI1evYfe+5Ct4FIGJrm11kkLixpLDFOZ53nLeSjfl1JGUoUsfuBbrufXA3M4pNLDG9vrDba2tuq6PptMnHNBv68oitaaWCryUFRlp9cFzpxzVV0Z54Bh7OK6sWVZGtMiE23bPj54yhhMpqNuv1PXVWuaNIsdEHAGHowxiYwQERklSSKl5ByVUkmSRVFEHok4EDpHbePKsmqa5smTRmt9djaK4zjkzjkLKo7EuQyUq5BkDY6LNk4QCrFozRDUuJxzMlII5L3XxlmrV3Y0WBe33FbRUVqSRo0xAMwpF7R+Qyg1BPO990QYXDdEvsju64VWYyASxjLy3gfXdGkGlskjWiygzLOV1xvg6iLKwhb5FIZLEWXCgEsWq/cSKKxbPloLrYdrDJHk8K1erxcMQ5CCuHTp0t7ObhxFe9s7ptVElCVJv9sLfRk2N4ZMoZQySTLnHHAmhDC1aVsjI+YcNbq1xltrW6PrummaxngbwN/Cv0emOMRxPC1KvrSCnpDBgq8H5Nbd/RUcZMttPQbuA7zzEFz2VUTBGBvKOmAZTqBV1QDjBOTJ08o1X0g1h2pJT4RAC5K/8wDIwv4+TOSy0oEvqycWjZpCWhyRVn4wYjgsYdiBEzJiSBeFA54ROOOeu1PrIADW4G8obbXkVh+um0xrFt3IjLbhAQ4RI9INEdV1eXxweOfOnXv37p2eHTdVXZYlkA9vDSIFr8Bam/UGQUHELRW3aEGYveBMrPv6QVUlfDYajcIz32my0GmQcwwYQghhTDufz6M4J/CAHBBZQFGMI+P+oo3VMyCAX8C2Z7Z1PAdrSIut6UgE1mjorUlAnHHOuHW2ahrnKJT+e0aCCaFU3bTaozeAJKNk4/r17s2br79485Uf/+iPf/Lunz68f6tqqkbXaeSsqZ1teCwISBdVWUyAzOUru599/AkiSsnPT0+MMd1u3jTVwcOn77zzzotXb/R6vV/91e9tDrfns/qjD384Hs3OR7OHj57O5vXJ6Zk2QDx6/Pjx5vZmkmSNbpumBYDhcAsZG03Gcay0M56s4oxxIG8AQEo+nY12d3fPzk9+9NMyS5KDJ0+9dU6bNEkET7d3drxtZ/NxnMg0ijnnsZTb29tbG5t37tz5L//L//Ltt7/+7W9/88MPPx4MBkVZzWZzQpb1unGcOqK6bXyNN6/fyNPsD//1v9kbbnXy3rw7+/CDj733/+Af/OaXv/zlDz7+oCznSTfXptkebMdR9/T09NHjJ5zzy1dfyLLs6OBAN+2HH368t7O5vf9a3eq7Dx5OpuO60UyKIH/KOZdSKSGRkRCiI6KmDZQlJE4B8WltiUzSk2VZPnr0qK312dlZEMrLk3Q0PnOO66bmgllrvXXatMYY52B7eztL/flozJjs9zKpemmWA4hi3pAvwHOjK2SQJBFngOCbqq6qyph2MpmMRqMsSdq2ZYxJqbTWSZKkcTyZTKIomU6nHDFYz9DCNFaRlLKBJlZJt5PZtv3oow8iLpjgbdvGcSyZ1LUO7YGEEGY2c46kVCEE7pEZ8JohS2IpUWrnTeCpMWJOcCkZZ0bb1jTEuCULRBhHAL5pyqKcW9O6RgdNzaqtvPVJFIcKECV5rJRhqOvKGOMlj4RkoeNA23Ipu/2+EoIQqra1RGne894TijjNZRRbD0mSDoY7SdZptO70e3uXLxVFEUXRfD7vDvpCsLIsybZHh082ehuHT5+kcbK1uR2W4KPjk6qua6urut7Y2CwaA8Tr1jaNSZJYKFnMayInROw0RDLy3lVFzRh0k46PEmOMFqSyJFZJyNlLKROVRFEEAE3ZWNsopRhD7cqiHtd17XVnNpshMiVT730xnxXzZjAYhKxqcD1hwTDwnLM0W3ii3hkuIJMqcCM4QiAycKEsLHL2jDEDTd0sWmNrbTgXWZwwQs75fDory1IiJpHk0HryGx3Ry/KyLB1RkqWMsbKoddMgw9Cku26aJEt7g34URVwKAMhFHKCeM6EE3AghODLv7SIkCzwsbhwFE9yAYUxkWWc+n5+fn0dR1O1266ba2toCJEAyxqx6J3rvPWJYlyEUCDDmgGxTMSQeRUnEs1h415ZF4YzOYiXrUpBrJ6PpU/7K1b1vff3Njz79JO/ubl/apkVuO87zfNgfdAd9kUaNtSxJG/LAF8V4TkitNbWtcbQquG3axXgMASBHREeBbBc2K0Ox5WJ9p2CsiRwAw0DOJQqtngJOKqtmVRkBIbYMjADiiHNOodWTc957i0yE6lIHIZ7EgDNA5iE0gQ4WhC359Rjcz2WMihOBc3RxFnLr7v4KwQTPmK1tYQfFVhEGBOAIHjwRYChGIUJgHlnwpL3zEPpErAOFYNlC/dUFGgjJFu9D4xC/1HRawCm+zJUAOPIBxBjv28aWZ2cPHz68d+/e2enpaHQ2Ho+ttbFUHkgbE8KhFPSmAKSURVUQkSfvlrmPcCl1W8OSRsoYYwuyDaokMeQjzgbDjc3NzThNuBRpmoanOkvSKIrAkwCM4gQAkDvOgSMwcAyCljpnwMGvbgewtQt3fjWhX4wkFpjg2R2e+dNqBkPzC8a7abb6q8EkqNHFUcqICQkAQA4456Zudwb7v/Yrf+fK7gt//Ef/9t69O8XIO4jSrO/Qn40LJXA4HD56+OTKpWvMw2b/uq6stDm2yWZ3887stuJciviTj2/93f/of9LW+vRo9Oj+4WAw7PY2To4nDx4cHB6PhcoKi23ZHtx5sLu7W1v39OhkY2MDpfBtyxhrqnaYpuiZ9GAJgDnOpYgSxoHIGWjPx1NCPqvqoqpFkoWJ0K2dI7t3ekZSjltjPZzXDUfmJzOZZ9Om5nn86OQpfERf//rXHx3fv/fw0c7uXnE0FzLiirSrpvPZK196PUW7N0xG4+nu3ob3UCOxXl/E6c/uP4B/8bt/L+a/9B/89bPy9MHDu29+9XUiP567/WuXOJOn5+fTttm7dj3pbvzkhz9UcS5U4jW887V3Dh/efXr33gubm7S396CIdwZXprOZdjZSOSAXMqoaLfPEtq0lAkk1aONqFSdZltWtSZLkzr0HVjf7+7tZdhUZEXmZ0ZOnj5BDlMQkyDA/ayohWEdEp6enHsABTqZzYIJx1Yvis9NJpzswGqqiyeLcGzY5nSZRClSDbhKOvU7n+OnTneHQau29YwyBgUqSKvQG41g509vZOj6aJFJJhoy3xnnTtsNe98b+fjGf1nXtG8N9dPXazXlVIvJJIZBZBxFjqdbgHHKeJWkqgieKXCBwB0TWeQ+GvCEgRG11nCaAC852oNFR4ziAAzBto5vKWeuM1XUDnixgeJmZYJGMEVFrfXx8nETx0u+Eum4tW6gBtrhwjmFJetemNWTSNO10Oou88sI5s0KwsmzLcrHclGVZ13Xw8uM4Dp66lDz8LJVAxLqu67p0jpbl+hhFirO4bVui0Lci8l44Z5BBFEUhtWad9gv2FhK5rNcPYZLg80kmQwKlnFdVVRljJBMAEGJBzrmslwUEF1zGwWDg16rX2LNUcFpIGuAKRoQdggMdQtahpDtEU7z3xrHQuHfBmQdOhN5D09RaW0SulIpUvFqX8VmR48AA4EoGupOxNoqSNE2TJCGEEMta+WrrZLoV/TBoPYWheu9hGfpeLYVht1CTVlVVuFN2uaVZzpElMuKCcc4l40jgvCHrIiY4Y+S80do12jljCdCDECIoATe1Ns62bTuflaG/sxAqFD4kScI4D7nkIFNES3/d+oXIgfWwaNdkLgRwlj2MLprArm7N8wZybWHHpWjg6rurO7vafz0ItH7fww5sTVAS1ihybk3Vcd0CrX+yfpzwZK4PeLVzOO+qPC989wJSwEouEmnxOSMMKtBAHj1iePefO+z6eNY3Wsa0VvggIEvvvSOvtQ4vadu2IRdgtS6K4pMPPzw4ODg4OJjPZmU5L8uSiNCTsdoYgwREF6zJMD/LNcGvz0kIV+CShxse+/DaJ0myu7t74/oLW1tbIX6wikOsjkOLulNieGH417cvvNi/ko39HHyBnHPnXZwmX3r1tThW7/5s886tzyxNx8enzHpmPSDV81k9n1Wz2dUr108ORgjWWs15BxGzbqeqih/+8EdvvfUWYwLRHB4en52dzabvlWV948VX4kjubG4Rl1cu7ROTp+fTpqo0tZxB6F8jEIwnYS0iZnlirdBO+6UGOSJyLi4EUax13iMwzrngfJHms5aWtOgA+Hobg9YaybjKkizLmBSbm5t//z/+zf/sP/tHztv9/f2mNYxLGScvvfLyzv5ewuns7Gwym+7u7yme3Lv3oKnKK5cvHz21tz/5+L2re//L/8U/+J//5j/48U/+TApG5O8//TDLdNVYRF6Wk5999O7O5o7lHhPx+Phga2fQgLv00o1/+69/dzo673ay2HIppVSq1i2XoqobrXWra0JwzjOGjDFCEMQEQyV4JCJnjF9eV13XTVuB93GiXrx+Qwj2+Mmjxw8eDoeDKIoSlTx99GRjazOK036nPyuK6Ww23N7O02zCp0msOt2MIbZVq01VeG1sXdbneZ6HOHGn0wkxWq9pldZf+aKLwCfpujaGgeQ8imSSRJ1uNtzs7+5sPn78eDaeXLlyiQtyziiFzplQJNy2bdM0ZVkWdZUkSa/XE84zQOAEznoiC5K4UIpzYGhck8RZXddEXknV1k2epZ6hZLyuy7PpaDIeN3Wl67quKsWFFSQYV1yFAvpiXrZ1PZ/OkijudrvdblfJoArAwrvKka38ntXz3zRNkiREVBRFgCar1945V5Zza7UxbVBfMSZzzgAuyt7atl2o2Xq/t7frrGeMCUaMIVVUFEVMaSdPEIGIAL0QjPPEOUHgoyhKsziKpHPW2FDNj4iIctHDgxELXGgiMu2ikp6s16DDQpNEaRpjIIAEXMWWfOyg8htenqCztLre9SUYABaIZjknuNRKIqJAuDNgdNuGxYsvlXe995zzNE0ZY3EcR0KuHppQEBEE5hARiAGAiFQoXMRlB21ErJu6LEtTNKtQPCISeCmlI88YeO/8ov2BCLW8zoXO4dYvhQJDqj6YvSB0MZvNQpw5PL4CUCKLpAy9npUQROQM44qladpLM8kFoLNCImdSyrZseaw4Qw+MEPrDjes3bvZ6vayTh9RzEsWxiqVSRNAa7Rlf5WUcLWpDtF2QBrSxoanj4i1C5p4VCV7N/Mp++DVGIVxUDayv2EhLAeaV+Vn/K1vTX3rmvq8JEvjVn+iLgQIAhHsdPnnWwj3DIXgON6wOtaREEC4qAAEAPGCIXXhABEIC6xdKQiFyYN3zEGF1ybSsclydl4hQoHE2vAVcCuQMgJx2AGCtVUrFKvLej87Oj4+PJ5PJRx99NBqNxuOx0TqEexhjS8kaB568XwAFFvD0GtlzfZbCO/IcoMFl+U+osFVKBcEuSKLl3Vw8GKvLWd2s9Yt97o6vzzbg/0DVw7/j9vmzLD5fnDJQYzwRQwhPiGeRYg4AfDKIXht8PclyqdKjB++NDw+3tvvTSUtOt3XVTaKr+zu9LH3SPjaWqPHG5UVl4zQ6H58SYdua0dl4e3vbGX92ct62+oVrNz764MPdvcuT0bkn1snSOMnrqooEdxYGvW6eJUTknQKtKZKRVEjAQgmN89ZarUlIxiQLQCEsX8YYoFAItiibYktXcPUszebz6WzW6+ZMckI4OTu9c//eL/3SL339629//OlnKpLIGJeqqGopRV0U944eHx4eArC33/za9nDr+Ojg8f3RJFZgmiyJzp8+ufvxx19+7ZXf+N7fnM3Hn332WbeTNk097PeSNJ9Nk9mseHrwgKG9fHV/Ohk1pH/03k//5q/96u6N6wejY9PWcbwROPVQgiOPCIJjFEnrDUfGBCcg3WprnVRccuzm+dnZWViQrbW6befzqRD80t7145NDjrjZ74nNIWPgjO3mMbe7WZo/PTrsdHqxiAqqmnn52UcfOkfWQ6pEZ3vgnK/LSmuLnobpsN/vh9rXfDlvvV5vMplorS35KIpWPGtjTBzLpi6dwyROuKC6mY0nPFVsf2+HoD07P3rxxRed18h8FAtrbVFU5+fj09NzlcSd/gC4aLWu6lYQ44ILwSNizDNkILiQTElPyJgGorqqOOdJ3hnp8/HpmTAiyzLbNrPR+fHBU2taRt5aS0KAVFzGBkxbl8W8nE/mVVUhAEfWNDqO0yzrIGIkeZqm1lpnFsSiELMNr18grZydnQUpruFw2O/340Q5b/JOWpalJ0NguYCYSetaaxwR1XUJABQaNZFDpCRJxuOxczbw9a3T1aRIbcu5rJuy1bWQgJjHsSIQzrksS5M0iiLpvXcuRkRkRETT2QwAFFfBfpjWBqtfzorFEuMW7mCw6NYtrghCcGXNsw8XGExYWAdXcnXhZ7ZUAYJQLrhc8kLcPlg4kEiEnIvgnymlBDIrfSgvDD4cOL9qDVDXsyCVuCBzmYWCRcBqoRNROHhRlVVVMULrHQAsywh9UAPs9vJgmhyQJe89WGettcgWbsFKZiqMORAkkyQJGeLw+GqtY66SJEmjmHFkjAnGwZPjvK1rThDqd3TToPOMIXhK8yyKIutdKDC5tH9lc2un0+uHZLMUSkoZSnJq4733IIRb27T1i0aOSw3g1l6oIARq53O2gdYEB9cNUticuyjeoS/ys1dAAZekP0RcZaYREddiD6vjrDiJDJ/5fDWw9ado+ZUFQPz8YNYt5YoVFJ4Hv6jvZS7oOCFQ6ODgPSHH0K8J+Soj/xxOWs2VXyMzPgNWOPMIxNDjIrYUsFqSJG3dWMAoU6PR6N13371z63ZVVfPpqCiKqqq8c+Hl8N5rWkQygFZJFgphD7tsZ7XOEcFlItYvVTLDyBlj0pP3vq7r2WyGiCGe0evmK7rPOpy6uEdrQOHzM7/+L/v3gxN+7oaBfIqw8C2QkBDQe+e5ABDC6ZYzDoxdunwDMLpF1ejgeH+47ZuqasbWw5Xd/Te+8vrTJ8cIPo6jcNXz+TxQx/q9oUB1eHCyNdzZ2Bhe2i2fPj3UTfP0yROr7aPHh3He293ei5Ls6PCAeZMkSkUJOlM3rdHaOc8QnDOuMcAwIC9jjPcWLQojknTBW3LOeWvdgjF8IRYCz4JpywCREcOiqhiAdebu/Xs3bl7/a7/07fF0cufeAxVHeTeblbNbtz9WShW6qXXTVO0HH71/bf8q49TPE9cUpJu3v/7VLI7+6T/+x7+fxb/wjbe/851ffuG733v9na/9/u//m7Isnz49TJjcvbT94Ycf96Joc3OgLm3NppMPfvbuC1f2Y6mqUret6eV9760xtXWaC5FmMbEeIatbzQRnXNbanLeN8S6Noq2NjUx1jg6POecMqG0ap1tn9KAzyNK0m2VScvSOwHFkLIn2t7ZcbzNKk0cPHmvRKKUG3Q4Be/jgIRPCe591OpcvXx0Oh1rrs7Oz2ayQTAbuLWMsy7L5fF6WZZpnfllUGGhquNQKS1KpW2Do40RKhtV8fuJadHq40e31OmU1nc7O86yb5ZEnjYwm80lRF/NqnpCVkWhNY6xRkRDVrE7TNI6EShImpAUiAGeIyMUoqdHUaJWmCqgcTe7dvduNOpcuXepmaR7Hg06OlMeJEow75xIVSa7a1pyfnE31zBhHhAxFXTUTLPr9IkkyzjlZh8DrprTa1HVtrVFBFxPRORNnqTF2MpkcHx8G1duymhszZAz7/Z5zuigsgE/TmAtsmkYIxTmvy4qItKY4VpFSWZadnBydnp7OiipK4ihLpRRlGYLqnnOUkgvBlBJJkgCStTo4+kGthXPBGAY2tbVWMImIprUhkK7rNuAAwSRjHDhAkGVhnAFL0zTU8oW0SABASZJwzkNAPvDdwuoJS9LTKvUAy0Iy51ywu+HGB5OQZZnnxJEppZYKj8hoEd8ONkpr7bQJsoYhcxHEvMKS7ZcOdJgN6xzRgjbovJNSsmVAPZwdgFbBgM9bU8YYwUVL63VPdzweh8hYmqYBMQghtNbdJFdKScY9OSISjDNA720kZZ5mUnLTtkKISKoQdomTDufceGecJcQkyxV5oRRyQcgcEFrvkCAUznrvtHbOBTnFgOSMDa3Dyfil5gERIGcMgXAlEETLECh8zh1/Fgp8sUTBav9VLCf8NXT05vS8wUZEwIs0h/MLIwfk12HBcxaL1ipL6VkS6LqpozVff92sEhEgW/DsARZMRiIPoerBA+MIEDpcw7NXuhpJ+CE8e+unCAMwrV+BIe0sEaEnRFRCeunm8/nd23fu3bt369at48PDuq6lZAEHk1/IUgMAWRfSWwAUlEsAYPlngs/BF0Rc6Xctdl7OkkLs9/svvPDC9WtXQzghcM5DRm+Fy8P41wEiPJtgWpElVzvDX9GGAMAIYIE1PSEgLQIZBOQQddMyxmScX3/pS2x8/PjRIWPWOUSQROC9H4/HASdFURRFCRFOp/MAqNrach599smdgyfHaaxe/9KXr1+7cXJy9s6bbxvnxmfjjeHG22++UbXt3c8+nc6LLM47iSqKoikLZAIJnPPlfBalCZeMcZBMAgPtLirhQzIoiiIGoFsT3sp16gzCRbBNxEmiZNW2R6cnvU6ep8nZ+Ozd99/7G3/jb7z5tTefHj2dF0WaZ1tbwydPnhBlXolBZ3N2Pj4dnUoERUxF6Op6o59621RFoyKRJXHoD3LlyhUzGf3Dv/s/quv6X/zz3/vss1s4n1zuZicnJyJNru1cn0oujX782a08z6m1+ztXRvOawLW2FQLybsY4j5vYO3DkuYqQ8el8VtclZzDs5fs725ORruuaATrn55MpIknBhxv9r7z6qn3p+vnZyU9/9OfW6Vdeesl58/jB3UF/+8b1azeuXWradjKZDQaDzZ3tzX7v5OTk5Hykq8rqOoqkUBzHULfFvCalVL/fv3r16tbOTlmWZ2dnRVEs2lDJhau2CC3HcXM2c94QEYAXQkrFlRJxHG1tb37ptVdPjg7ruknSSEbJ48dPnXPDze2r2W6csdFodHzy6PR8urnZu3p9V9y//bDf7+/s0dZ2FPVULLjxznjnyadJ3LbtRtbNs4yMV9aNDw4fz+628/n16y+kUby3vSNw0ZehbRrOJAArZ8UYBRFyLtNUMRLOUNvq87OJNZ4xxpGUUkIw8kErjci5wJVDRFPXZVmORqPAABiNz46OjuI46nQ6SgltmlbXxtZSSsXjpql6WRJFkW5a732Q7RNccIGz+byqKgLnvWWMJUkyn8+5YEqJLEsYgziOpRJcLFI78/mcMUawkuuHUAhQ11UI47Rt21ZNAGuCyeCmS65gVdpObGWcVr7Uqo3CyoKG12PlZ4cFbgWog4O4ervCr3bZTyiO47JtAnVBCKV1pXVL1jVNI2UUGgUCAC2UIjki73TS8OiEno0MRRRFMo7KsiYiWIcsgjPGjLPWWcZY4CoGBWghhDZtGDkBeCCGyKUQShpdr8ySW9RcLNR5Q6g5JJJWPAlg6IE0OWeM914GiONs3unESYLktNZMiDhJBAOtNedcJWkvTVHwwXAzSuK61VobJgQCZ+gQDQcOAIF7XBu7CiavoggBIhCRXdIVGVuECgKbF9aqD9a9UnjWrQcAxsTq19XOKxD2nLUGgADyBC3SEwGj4LMhbiJCtgyD22fkmJ7b1mMVsLRkP8/CrOzZKjGEiFxEALAojSCyC0lHcgQIgN4jsKX3Ss8dDdYc69VIVr+G09XehjIlTkgUulYyDhhenOPDo+9///sP7t0TQighW2iCnNfyEVpmIYOCy0LJZRE4Wcw2XowH17bwYbhMtdyEEJ1efzgc7u3tXblypd/vh5cuzxIAWAGFpekCIlopG6zjsNUDvP6nv5ptSTCBZRYCATwgILDWaOcoiiIpU+8hCDZtX755/ZWvfPLZz7QTDmRjafTk4GQ0Ic+dZXHjB31ele2Tx4fO2+mkrgo/Hs2aqnZG97q5EtEbr3/51Zde5ZzfvXu/qWoVJZd2d6qqubS7JYCiSOxubRwDWWtlnLRGt9rWujWGOeJCcQj1gqi8tx5oRb0SQnBEzoTW2hrD1wS/YZlf985rrdMs01UZ1CvqtplMR0+fPr569fKXvvSl4+PDP/7TPzs+PnzppZc6nVwIMW0aQE+Mkiz26EfTkbH15rD34rWre/s7vW5+8+aN69evI0dr7d2nj/r9aF6NTo6O/9P/7X9y+uTJ/+P//v8c9pMr+6//+Q9/POgkRBBxNj0fvfjCi1/9ypunp6etbUTE4ySSUvb6PULw3ldt441jlhF3SBRxFrrWOl0Vs3Y6mURRZFtdlPMkUpLFkZCCs17e3+p3fFMdHTxuyulsNtNN3clyKWhz2J3NimI23tkc7G5tXdnb1S/d/PDjTx8eHLR1MxqNrHfj6Xxe1nsbu91ulzG2t7eXZFmgly2XJraqlocVRyH0eHO+aTTzhABRFKVpJqXc3t7++tffeffddyOV5N0+4oG1Ns1dp9ORUTfOrXNZeuQRSdNYtLNqrL03vqnawcZGmmcskogI6Dmw2en5xsaGtHR8fNxVyeXNnU9Ht2eT8elx0ulkCN45N2dMCOGNtca3rZnPi5Pjs9HZ1BijVJTEHc6kJzufl0GtWXKmlNjc3JBCxLESgjOA0HAIEafj+Ww2K8syXOF8Pp9MJlW11elkxrZ1XTIGUnJrNZFEtmA8SCWIqNKac+6c0boBAESSkltrrdNCsMBsa9raWO3JadOWZemcc24hthPMuTZNCAMEs2eaGtYcDo6CcY6IujHg0XIHAALFgoypdbhVIcy+MvYheR+SC+v+ehRFq0ACLP3y8MqsyhphjSCmtW6a1lunZOQd6daUZQWLhkOcGIVTMwGCy7DUrrxbQmCMSbGQPtzc3GyapijLptHhpIShkkLbtbY9jC3CG0RuZWwuWI1C0HKQsFQV5HxBKQ0GYxV4DLjBBgYQoGNgg1305Lyx3jW6tU5XRSmllJEixNrobn+71+vt7e+rJN7a2olUrK0jYOSRwHtg4L0mi0s1qtrY4J2vTLhe6oVfFP4Bw4Wd86vY8wrhreIKq5uybtdhTScA1jKsy1mi5/7q/IKsCkt2oaeFZSK24DnSUgLyOaixPoYVGWX9E3yW97AOL9a9/JU1ZcusByICMR/YOkTAloLQjCMSEgCwcCYGF0PCzzWAps9tyFkwD54xgQwWTwudn55NRuNPPvnk6ePHs8k04GzOeWsWIS7yPgCFAC8WYi8+dBshxhhbICr2hRcVUg/hzYqiqNPp5Hkex3GcZqHVXFmWQZYxSZIkiVZZvzCl1loAj4iRiNYncx3cw+cA3M/Dc39pW0gDAgB5QOAMgABFEIa1DohA5IOvfP0Xn5yfpMX5aKxrC5OqOZvM28buDC9V5TTPBk3VVoUpyzkRSpHMp5W1Oo2VFNEP/uQHTx882t7e/uVf/uWqKMHaWs90PU+ieG9rs5zPBsP+9WtXEdFai6EHjSRL0hlrvTMOUCATjIA8kHMOaOE5BM1LhCDhf1FDyxgDWjzb3jmRRMEDeXH/pSRSB08fcyFmpf3hn/9gZ3f7jTffODg6/PTWZ+PxeZLKptFpnjVVWZSFAnB1XY5nV3e2vvOdX7q6v/f1t9/u93uzsphMp2knv/bKi5cBTudPesPN9+59/P/57f/qa2+8/X/5v/1fJ+fjf/Lf/s4Lr16btjNrvSEvpIr6nbcuv/Pf/vbvBCOiVMw5ZwyNMZ4skuMMvLcMWBKrQb+nSmHa9vT4eDrF6XQ6HA6dc946njBn2mI+/fEP//za1f1f+sVvvnjtyns/+/GP//wHtqlfevFGZeqz08ezySl4iDjkiSqn426/P+h2+93uydl52+rT03NtXF23Kk52dnY6nU5ZlgcHB8a5siz7/T4h1HWNiLAsmIcgk2+t96BkyqTnTAAxIGEt1HV77+5D3borl689enhgjBFcbW3ujMdThwWPorSLeX9rb2/v6dHhxx9//PTwttjZ3NFaV9OiKsrJ+ajT72XdjlKKGCLSvVu3/QsvSMYf3L+/u7t7ZW9/c3MbAIRgQgjrtK6bxhjGmFKqLOuiqOqytdoxxqIojqKorlsGwJkMvZEYA5bERJIxoZTKskQp6YypqgVNoWl08H2FEN6HWgMMuoHaNIyxbjd3zs3mUxXJTqcjuYzjOJRI2FZLKbVrnXMh8cmkaLVFJVQUhfVlNBoFxnsgm0RRFASCAqcPANrGVHURQBkAsKXvEmKVDHhg5XTz3kK+zeMKB4Qtz3PGWIg593q9OI7DpYWyCERs27YsS+994BmsbMDqh+CGLgz1Wj4ikDmQANdI3bhcxWAJNcSynISI5vN5EDMWoYcekwCgte71BoiojVlWe6Fx1hjTNhq9D8opK4PpyCslmOBMcCIKvJCwWrNlH0i2VHRYLdNhkhdVvG0bgg2WKKAH07Zaax8yai0QQ88RPDLBkyztDgeRFKooggbz/v4+k1Iq1RittQbkKDgAoEdHzlnn9ELTN3RccEsxA0uwSDfAsu4A+YUgHgD5BRfVLVWB3bJqY/EALOtQcCEYZVcfwhpQWDfz6zZmpS618rkDXiEiYnyVj1ilHji7ABnrgGNF1lv3dNdByXPb6uu41JEMD4wnBFrWYAbYhAzCyJHBAhMwvwwnrIkCPaNSgM9SE1ZbFMWB3w4ewj0y1pqmvXv37meffPrkyRMA6Ha7p6enTpssyzzza2jYLMCQ8yFDggTrOY7VthrDOlZYxXXCXIVHEQCappnP52ElDdy6INayihZ47y2GklSU/iLLBmtQ4POA7K8cJaw28gAAJJgQjHkC7yH86yxJVNvXXrh0/UahJ+P5mUORdvqC84PHB1m325am2xlI1vR7A7I+TdO9yzeqcv7RRx853X7nl38xT1Mi/0d/+P37d+9Za0/PR3nWMU0bR6lgOJtMXnr5+tb28OzsDLytm6YqSlSKkUchLNmlphZjnNHK/C/TN3JZfsYY8xdLSCgAXmilR7GazGej8/P9/V3OeV3Xu9sb21v81u3bP3vvve9+97uvvvrKvCxOTk42t3fLsqxrkkJsbW11ogi0QavzTjrcHHzjm98IpD+UjEXcMDcup9Pp3HX1ycG9S6+9+Mff/6N373/U2d04mp383X/49731v/3bv9PUdri1PZvOT+eTl9/86le/+c5P/uzPhBCMgfWubkqjnXNWCDHoDYwxyEQUJe3QTCaTqmrapmkabBuDntiS3K2bcjqdXru0d3T49N2f/OSVl1/85jvfeOXFG9///h/eu3fPsNbaZjadpWlu2qqcj8uizbLs0f0HZyendVlZYE3rGuuIszzrjcdj7/10Oi2Kom5bAMjzvKyrlZZoeOwDAaVt27YxQqg8iXvdXCJ4rQVnRrv79x+enp6/9D9+qd/fePz4sfewsbGptfNwkHWEjFIiynoqnXMurPGtUDEJxVmj67oqxqXXU1OkQgT5fXly96Mb25lH1LODMc0YY6mUu7u7TPBpNfeEnV6mrQl8qGyYJBM5PZt2e1EkL01H06dPD6xzAblHcRSppKqqeTUXgo3HY7WzYzQ5axHZdKbbtt3d3R21c5UqC7ZpG+tNquKyru49uBvSDfv7+3mStroej8fG6I1evxsNRqNRFqnZbDabTZqm2tzcLOv50+NHAMAsY4IX0xYAFANTjHiWA7pWV3Vdc85DF0pkFHI8uFgveNM0wbb18jxYHnLApEySJI3BGKOUCm0PgzZ8MZ+1bdvpdBoj07zDOYeqiGO1vbeZ53kgXVvSZJmUkQe088oYzwRv2nEUJSG4KpXUWtd1G0L0jkDrUJfPlu4jxCpjjHEeOYfo0bZWSrm5MUySaGZmpmnBO1TKWF2XlTHGSnY+G6um3N7ayZIkvLoRj5QSxggpRFWWbduGlEQep9RlSi1IJ62unXPkHDk3mk72d3Y7edQ0jdFWcowRufecL1oOeqAoyVRPMcaCvgAieXB124YSReKAjKARgXNhAjfTeau1bozkvJf3ZJxHshkONm5cu5nFyXg8jjrdvNv1QkZx4oHNphUAAqNyVhORcc4vTSwiEoPWLJozOXeRAyIfSoaICIMUTPB+AaBtDCxjNhcMBoAk4St7DwDGXGCLlYXwa1t4M1dsklWoIFKh6yCHNQARoAwKiWvdBcOHPFIrVLGw8YwholkiVwpGGsB577znXKwPac16EeKi1I8t1Yit88AUXCg0rBnCAAgWERd3cZRnexms4M7Kyq5jI0RETYIYLjIEzrT6+ODw6cGTj97/4OzkdDIdCWSMsVgwC5yoJXsxsxhyZuQJiTEe2AlMcAAIBFtAANLk16mdjEgsrRFjjCMy71C3TrdOSa8U29rYvHLl0vbWnpSKAYtVpCIhGAfwBKDEWvQMkRGw0LcjQCUMSlXktGGMYaiICSAsTCwPe+ISSHwu8PBchOj/T3Tx7O4sHJ+vsynJhVkABBKkItXOpl//6jt5lBw+Or47fgJeConokgd3Hl3au8wAm0bXc5dEG9NxkWxV48lo9+pl7+3JZCIE89b96q//+q/8yq98/w//+Lf/u386rsaPTkZZZe4fnI4r/emth6++9tb9x4fTuh1PJxvbW/OmmsyncZpQeNo8Wm0QUSATIGy06OppvS+qmjGGDOM8c9oAgHXOWhvHcZ6nomGzmfG1F0Lsbu0fHZ4xDtnGxtx5r+ua+d/9/u+//uZrl6/tfXZLdZO9gydPu3GsRFrWlXOmcFbrVkfyqa3/+x/9WdWNh4ONTp4KZHkc94Q8OR9Np9PiZPrKSy+bpv3qq29+8OOfNF8qU5L/4r/+J7/+N/+D/9nf/g1E/PjjT+tOXtetOz3862+/8eiDW/28fzY6b42hNLXexVEGnLXaDgYb1nrOeduamEf5ICuK4sn5nWHKz8+ORZJ5mR9M2iSOZwYuX3/x0k7/D37/d99/9wdfe+edv/E3f+2Nb/zCp4+efHjvszfeeAM7nftHR71eb2x0ttE1ip3MRw+PHgIT2vmirLa2d7mS8/nE69QhGGta52QUAbF5UQNg4NUxBCRjtfGOGGNJBGCkMeb87IwT5GlWlnpruNnf2r/16WfOx3/0p+9a7edTPY5n+7t7I5hA2qsIhru9uh2dVp+aaJRuTZsaxGwyZRwBIFq0YOdWm6rQnU7W7w729/aSKD0/Py/nRb/bG/QHx6fHD+7dFpEabGxEqRpPJ8Rwb//y2dkZZ4z1sziSEiNE5sGpiTKklZLWWm0aT9Z7KyUXkSBybVs3TeWd63Q6WZ5wgU1bCa402EUPXGDOuWJecmQcWZqmZF3gRi0UVT2Mx+dtG4x62+3lnU4njqO6rrMsCZbekQ04K8/zbrc7mtdtW6OnbpYTUVNWjKDf7/c73eBq1EVhtBXImIok4ycnZ1mW5Wm26EFAUNd1XdeI1bA/IMLJZAYA4Elro9txnHe1tkIsTReRsW2ra20a5w2B51zlnQRZvyxr51w7a0Pamy2TduEHpVRIVzPGEBehJK01Yy7IxSilXJZ2nPPet1ozBm3bVnUVWRGiESqOVBxN6nIR91Mi1C4CQOikERofBwtH5I1tnTecK2OMdi3jC1XHpqrqug6Jsapqqqpo2zZSCrElojzPw0GY4EGagjFmydd1SeQ8kPfWOqO1bk3jnANMGC3YDAwxiiIpRcLjKErybreTZZzzreHm3qV9pVSUJsbDqqB05aB4IOdCb1Pv3FphAls48PBsyoCW8RUAduHZU2g20cKzZXJhSS6KYnWE1bZYu5fuJq3F5FfxHr8Unw77aB0KxP36QVZfX6UkQttxKSXihQ0IB18F+dc/XDn3q89Xx3zO234OEDzHLfhCj/mZ7dnPV1P9+X9xrWaSIRK5ttWjs/MHD+8/ePBgNBoV5cxay4UEAGSEjBAZfVG56ReP5Ods4YzeeyH4ulR2AH+7u7u9Xq/b7QaludVd4JwjEuOAjHGE1a15Luu0uuTVto7J1mfv5w4bP4cV/n1suEwDff50UZ5FWXrz5s0Xb7z00YfvHR+fdfO4qhpGUJZl27bgvHW6bcz56engUv9Xf/k7QogP3v+Z5IwBODS729vT8SSJ1bVL+2fn04f3H3zlja9e3t+7e/euUmpazJM0ZZNxa/RkMtHkxFI/e308Dp4fHq1RcVdvNCw5LmFVZ8Q554C0qpq3xrdNu7ExPD0+/ZM/+bM3Xv/yN7/57X/yX/83X/7yl3/2s5+Jfl8wDgCccVDKWtvW9eHh4R/90Z+89dU3bt54sSpm7x8cSs73d/e2Noem1k8fPZ2OxmSd4JGzcOfewz/4wz/+6U/e+/rbb//CL/zClWtXvYP5fG7JT+azLEnv3LkzGG6kadpozSNV1TWTgoAVReEczWYzADbo9S9dunT45OkdzqMoMmiDBGrIRLdVHULUs+l0Mjr5/h/8gbX2q1/7+t/5O39n8KMOAJwcnAzy7o1r1xkT83k5Ox/3Op1ep0vAmFQDZ6M4Nd5Fgw3OIu99WWpjNGIU8mVKqbLSiGxVVsUZE5JxzmVXBqOZZZkUMk3T3d3dKEniNKlM+/jwgDFmkCqrZ7qOutn7n/zx3Tvw9W98eWsnj1keRf72LbhydSA4USC6SymTOFFKad2Qtjsb29evXHWtjpgan4yK8Zy2fCfO23zGpGiaajo6yWyeJxEhVPORbktkrfcgIp6mkRCqcf1tM7zz8RNCb71pKogim6ZxlmWdTnb58mXGWFvXgQFbl5XWDSKBR/DIGcuSXHQE5xgpQQTn52PTtuRcWZaRVFxgCK2cH50wxqqqstZ2+z0id3Z2UpZlf7gxnU49WRcslWml5AB5r9flnNV1LYSIopgx5pxrmubk9Ihz7h00TRO0BxgTAJAkabfb63d7nPPQFr0q66qqhsPhYLBhjJnN5gDQzTtpmgVSIRGtqG3OOWuNtYZzSBKFiEkSpWk+9N3JZDqZTOo6SZIElm0OOOduWbkechyBnBXMj9ZaRRwQA18BEdNO7o01xtCivzA0WmtrkSgYrRYswELOwTrDOWfIAXzb1tZaRApZpNX66L01xnjnmAsdClAIEZpaOefatm1bzTlL03Qpnu8Y84hu0RFM10TU2pZxCGSAUGPCeYTca61Rcu+9bqxuGwAmiCspBVNCCpA86eQbGxs7Wzu9wUAgkypqW7MqCw7ZfWOMcXbR6XsZUVisRAhByC+YM1yR1REDq/QZbWUKlQ4L8qNdKgmu0tL4bND+OUuPS95A+GTFul+Z/xWfAxEZX2uczRa9oEJp3dL8wDJo5FcIYGWG1/HByk4szrXq1XTxOQEAX1NuBkQfaCqI3j1Ph3zuyOumCABorYID1qwmu+h9evEnRHTeSCkF48a4siwPjw4ePnx4//79pix02xIRgHfOr4I3nv5dx7M6z8rNXk3dyoQvanSZCHdNCMGVRMGJoXY2iD458t57FnEMba8YAwbAWNC4dmuaCqtbTGvk1nV8gIgelkrSi1LGMF3P2m9cDvzf67Y+jAsQgwxAAHObO3vXX7yZZvl0Omfge91+W9ch4T2fzK02g343TZKYiW997Z22rR989olAZnRbzIv5+TgVanY+Jm0lwOnhQfz21166eXM8Gs3K4tatWwFwKKWiJI4FZ1KMphMEDDTn0ADp+dtGhEtAsDYpQEvyHQAopbwFIkceEAlBMIZExjmcz+pIxZ9+cnt7uP2tb34DiM3nZZrmFkAsOMIkhciyrG3bVte3bt2ajic/+dFP63LeVLXiIo4iItocdl+4chURIy4mo/GVvXvjUZnlGw8ePjof/8Ht+49+8Rd/cW9vD4UoplMC+N73vvtbv/Vbk9E5SgUMr2xvG3emrauq0jnqdruzyZxz3uv09nf2Rqfnk8mksd46MNYzlUgpBSdj2j/70z+N/to7Alm/2zPGfPDee93+AIG/cvX6ycnJFLmUQgEzrUm43O5vqDienE3OzsdVoxlD22pE3N7ddshG55OmqVYpNsYoz/O6KUNbYoaAoV0LheATwlKIyDjLGBtubRZFodKkaZqj0VmSJDKNCqf9dBTH8ZWrlx4+ulMUs93d1JNhjMoKdrZ6Io3ikBwyTcsIJONZnKVx3FRVXdRt1TZlMx1NwTFwjJF4+823Nob96Xx26+7t0XhclEWpG+1spz+wvtXWEkoPkGRZlKut/c1hZ48xdnx8fHh4SERCcGP0ZDZJRnEI4Hc6HdO0o8kIEbv9rtPgvZeKJ1GcJAnn6MmiJ2O09z6wc32WdXt5LBVZF8USEY3laRYPh4OiKKbTMQAgknPGOcOlQMbqup4XU8ah099OkyQEyW7efPHGjRvj0eTDDz98+PBhHKeMsbqu67rlnGeZSJKEsTjPcylV0zRFUYUGo71e1On0BFdNrb0LCnSKMUaEUawYA08ekTgPfQqRcS8kRHEMAJyjkF4IZayq6tBMb6FkEG5EUVTOuX6/HzzUABRCuitJEmQiiiJtTFVVjEOeZkwKZzTnPEpiQrBaB0hRt23btiKRwb8KQlXeS8ZcqGnkHNM0DjsH2ME5N9Y674DIWhf0m/M0S9O0qiqilQKBoFAsD8AQhRBKRYSOyBE461rr2hCKFYrHSR78OWDonGsRvXVlWc9mM9MYzrnkkkOQ0ydimOadNOswLj1AlHQ4b4LxNsZ4WFSWaqOFWICn54ACXQgWXSxVq5U07LwEBKEUlq2AQpjwYMMCQ37dZtOSfr/mkl749Cv7YZcinuHOkrfB4Czzs5zxRd2EX5ZHwjKkD7CQS/w8Vli31l8YKvj8r3+B0V2PAcCz1RPPecnPudHrU/HcYcPOum0BAJibz+cHB08fPnx4dHQwn0/BeSIXTmSd1roNcRfCLyhewJ9fzbGCBetDXb8itix8iOM4y7KiLKWUQftE28gHegeidY4hEQNPxEPehYV2mg6IgBayVEDEghQ5AXgC5AwZhJ5OjMGih/tCCIsg0Cn4knL4bB+IvxxKw3MPgLOWCxbSOPt7l26++OqDe/edbRujB/3BjReuX7t85fGjR7qse3kGOQw2evV0enR0oItKRFEzL0+fPv2zP/r+2197p57P40gi5rdu3/9X//JfSqUY4y9/6ZXT8zPPSFstIglAs2KmjRGRIgy8F4/ICAkQ6HOFM7Qk5Hq/fKiWT1SIkrZN670zxjImOJcMlRRgOU3Go8v7e5Px5MMPPrm0d+m73/3ef/c7v3358mUnYytE3Ta1bpEzuYxtSCmFjByBcdhq23g9n5fW2u1+H5zQxjx4cu+D9z68d/ep5GJ8dt7vDQfDYV27Dz/69NPPbr/++ut7ezuXL1+mBt58640nT59OZnNLfjQ+H52dl1XDpKqrNkty5zx6KKaFtb6t9Xw+Rxl5YsY5hosuRYElA873u72jw9nW5qYH+tM//CMRxV957cWbV1+4srk/Go2m01kxL3vdwUa35zxtbwzbqp3PT5EzBsx451pTOj0an43GZ3GUAuSLnKnToWk154zIeQoo3HrPqKmFEN468BQEDJng0/mscQaVaJrG66abRIVuRsWs0+l0O0I35s6t28NhkneRge93QUVSBKZ6KAUMVQCDwSBJktFodHpyPp+VWZZtbe4ON7aHG9tKJknS8SSHGzu/0B8+Pnj645/+aHQ8SrLMJZaABHAUAoLiL5GKI84iAEiqKKsSyXi8MJY868SMpUmSKCFriVs4bNt2Vk7yPEeQ3vM4juNEoietrSUTKRErmeVRGsW9Xies41VdOttqrRnDbrcbp8nZ+Wmrm8Fg0LaN1q0xmgkmpQCIAcBac3JyEnrLdjrdzc2Nvb29JEnOR2dlWYYut0Q0GPT6/X6oun56cFpWVFdYVZXWOoqifq8XuplZp41tszyJogiR6rokorqZSclVJLI8yjuRVOipca513nChiKiqq7ouhRB13bZtvb29rVRc1/VCIZuIMREKBIy2AJAkSZ7nQZVBCFE2LWOsLEvrHZDjcpEar5qFoiJyJjgTSsZp4q3T1DLG4liJRa+KhXlLs0RK6axHxPl82rY1EUZR5AmM0eBpxUuXakFU5CgQubW+rtu6bIJbLITI8zTvpDLiQoLgKBWC4M4ZAh8aEwqJaRrLSAFAyzwDZowpirytWu8AiYEjzqXiSRwlUZQIobwHxoSU0pk2oARalPuvahNaWuMoLFYiBH7RpWdtAf1c1WKwNUQU+IvBtIcZDjAiTVNYs77rNum5z3HZYHD1J7+U62aMBRIlfqEWJ14cOVQ9rLmyuH7w5777jFnl7LkhLXZdajyFxXpl5y9m51kD83mrv/7hOmi4GP/aeFbf4px574qqOjw8vH3ns9u3b52dnBrTLupgyVmnlx0xaJUn+nffnoOAKxyzqit2znF2EV2IoihOkiRLQyYu1JoTA601MmIMODLHgXPOiAGAXLULXZSL+iW1FwLGCH9CZERBCQqXsCDcEVxDCV8EF/5ytgskh4HHQJ7Y3qVr3/5rv3x2cvrZJx+QMVube1euXBv0uo/vPyBvB/18OBx2su5nn358fnK60etJwaxpNwcb0+n00cP7r37p9f0rV85Hk7rRj54+9YRf/uobQnHOEcA7stZ64qBt2zrDUXgABGTIAkRAQL72DuICdy3fvpXU0nLkC+KzcsYYbcgYw7kUEhkqpaDfE9awqmo/+fhOvzv4h//xb97+7PZsNuWIyLnlQoO21i2BCDrnp9MZEnjvhYwFY3VZzcvik0/vHR2Pm7qdTebTwk7mOk8Uk91pYfp9Pqlra4+vX7/y8s2XOeInH3600R3+xt/+9ZPzsydPD0/G49F4KrhsjZUqvn//4cnhUTUvut1+W+v5tCjLOut2VJRqYLzWFkKPE++cT6M4TZIXrl57cv/OzReud3u9f/57/zJOsv3ht1577bUs63z88cfvv/8BWhCAJ08PtXMccDjYKOuWOMZpPp3PxuMxRZzIJUk0HG5sbW7VdVtVdShBD36Ic9aY1nkI7CaPnohWsuVENJ1OR6NRWZUhm6yXW1EUAIDUpKo3Ohm3pd3Z2iwqs7PZPTk8E7WupZTAgRh6otaa1hrulIxjS8SVipI871LTNKPJrNH2zz94N0mjfr//4ovXr79w/Tu/PHj//fcfPXmELfNETKgkyZRMvUdrLXoQiiuldvnOcDjodruDwaCXd/r9vnNOcuGcm0wm3vtup3N4ePjDH/6wrGZkHeccCL0lIodopYAsU3mqOp00z9M8TRmzbdvqtmLcI3NJKtNMGdsSmU4nGQ57DshTLiQ48oz7WAgAYIy8BWd1mkRZkp6dnB4dHJZl2bam1+3M53PyvtPpXL58udvtTsazw8NDAOecdkQEVkhUERcSkfl5MSfr2rYVQiD66WwSxM6iDgKLUtnLOyLNOWCrddvqEhGdQ+990zS6NYyJUEy3s7MTRUkACkmSMMbKsm7bdjwe86WucHi70jTt9XoDxhCxrerBYBDy6/PZrCzLRrfgSWutTcsBQ3UiCCGQL82YD5WlIWjvvUVUUgkVyVB0johxHNXa1LVBgDhRG4MhERXz+Xx+EqkkSFgrGRltm3aRdQuDZByUF8J6qRgy48AqJTx5D23TwGTimqYSMkJEkBRFEecyFkxmkWSSswg8MhKcyyxJOKDVxjMSSFZCqFwNqZwVUAiW2K/JHtCSo7CWAoDlDxTqr4JNeS6iYEy7yhQsaCh/YaH8ule0Mp+w7HABS8d6lb8I9nq1LAIEtWR0zgGnVXwClwrNK9u5Dhk+bxUuPv85EYXnLmEtYHARS1hHA5/30Rd/fdYOfR5VPPenNI3rup7OxodHTx8/fnx0dNBUtVLKORNabDnvvPecL9uFOPuFs/3zbsFzHI7VeIJWXbg7bFmWzBgbbA43hsNOrxfKHBz5VmvyoVCZJEcfUuIAPOSb2MXc0lr6fPXJ6mFbwLglYlxKHiwm7HNwAf5qEIMnAgQADlx0NoZvv/2Nw6cH8+m0mIyVituqPTNn0/GIc9zf271+/fr3v/9HD+/dd85euXqpLqvx+DxNU102B0dPdy9fao396bs/G88LHgvBxGhy3t63/eFG6PJVzOcxgyiK0m6nahsEJARCWKUePAJbPmyMsZBWC9MYkomwrBlePZBxrBhj1pHWQeKWGGMIKk2i+WyyMdg9Ozn46U/f+8qXXvvN/+k/+N3f++c/ef8TIvJAyLkQnAABuVKh6Z3RWscq7vf7SkTT0VgbYlK2FrXnnscyxZPz4nF1XhXll1979cc//SiR/D/8W7/2nb/2XUXig5/99MMPP3zp1Vd+4Rd+odPNB8MNAlZrYzyWdXN+Nq6q5sH9x22tRV9yJkdn43JeKaVkHCkRMWm1RymlYsRT0VT1dDTe6vfRUy/vvPnmmycnZ51O78qlS50sY8giIdMoNnnuCKuqaq21BMDQkW8a0xlsdHivqptCl0Lwfr+3sdHv9XqcV95TKGeTciE26hf99rgQTLPWOCsjpYNssfNPHj2ejidRFAnCmEvgMuGSMQ8q7kSJqYQSPaPHjx+dZWk8Go8j2Z9NRoLHCjlnRAIJkcs0Js61d5VueVkVTQu8Oj0flWUJAHt78UuvvdXv9yMpN/rda9e+tL+/+7U3v33//v3Pbt06Ojk+PR+1U2sbjUxYD0B80o7jeBnfHunZbLq9vR2lcTfLy7IM9TBxHA+HQy754fHh04d3kCEXwLgl7wFISpHGKkkUY+RdbQ3M5k1IRDnnIpUiqiRJlBKNroebgzS9lKbp2WgURcNOJ5sV86DwGB7NKM4RMYoi6/TTg8cnx2ehvebu7n4cx2madru9LMuIqKzm09m40+sSkdVmSR0w82I8mbrRaCSQBckBIURbN977brc73OxJKdMMhXLWlU3oCkEkuLBWO4dAyJggzxmyOFKz2SxNfcB0QQFGa13XdfChAcAYM51OV+WUKs8BQESq0++RNeGLSZpGSgQ5hKaugixjiDQkKbfWheAHLJv+GePm83nbtowJrbWQPM0SRFQyckipjzmKJEmcN1XZhNsExMB7KVQoORNsITqZ5Umnk+edmAlAZpBpQEbgAR2iZwDO66p2dV1jCLkLn8axFAqAcRQsyqI4UjKKoljyOIsS5tFUGsBo5AwaTc3KtFu/4EM58oyhX9vCKkkE3i8Vfy98dAJaENwW8iMrSuTaxteUE733oel2+JVdiCjA+s/rXw8AZWXDQtCDMRYpQYvWz894V4wxfyGKElouBEbFxTFhRWv4HCnsuZ9XeOXz4e11swoADC8gy3OHfW6EsKT8w+fwB67lLGANuCBiVVWnp6dPHj16/PjhaHQWqCHIyBmHBAQeFuEWAiBP9vMHX03sFxpC/NwMrF8pIoZsZr/f73Q6QUE8RGWNc6BbKSWRd97EKkIky5DRggIW+nGHfpEBTi2eHQLvA9TGEIVae8AIuFgOxiGyNbgQRurX8MG/zwDDF87A4umlQOpl4HFrZ+/tt78xG43/+A/+TUCow97wxo0b08kYyT28fxeABpsbjLGNjY0JAy7Z5cv70+l0uL3T6XWxrcfFTCXpzpUrWzt7BwcHTLIvfemV8XysEtWMGqa5NUQNqDgOFAmi0AOVGLJQS3PxOC3poiHUFl5qzi6qKL33CReMgZTSOUJg5BFQcAZGa/J8e2uvnM2nk9nv/M4/fe3VL73+2lf+7Ec/nc1L42zW6WSdnKFkDKVU5+fj4XC4tbnTNM352bhpdKyi7d39qihQJZJH3DMOajorqnlNho5PJlLEv/Y3/vp/+Lf/tiT7j/+r/6/V1dtvvvl0cvLuuz8ZDDcnszkB29rfJ2Nmk+nh4aG3pJs2IFSGeHZ6XlfNfD6PPMgYQiWKtTZJVCftxNwfHx5lYg88PX746Muvvf7XfuGb2vrPPvvs4cOHG4PNkH0+PD5O88727r4nODw7s85Z78bTicrzJEuZksW4CJlu51xVF2VZVlVprWUoAilhufJbxqxzwnpXFUWe51VVcWSc85OjY6VUxEVbVrGKtra2sjg5OztDMAmxoylZB8Pe5fFpO9k201GzvTPY2+kLFUdEpLX2QIIxLkSQTKlbLaOWgAFDLlVvEO3t7X3ta1/r7V5J09Sadnx2+qd/+uOymF+9fOXtt9989eWv/PjHP/6zH/75o4OnHhomFJeCC1W39So7G6za0Lk8z6Mkefz4sfd+c3Ozbds7d+40TZNlWX/QCUQkJNCmsdYiQyGBcdC6NqbUpgbwptWMsSSNptNp2L8sy9Fk3Ov18jzXWs/n88B2DmIUYR9r7XRc5nnOCEK9w6W9He/BGJMlkeAdImqb5vGDh+FV3BxsaDTGGG1qbVpEdJ45Y5umkVKkcRy6bDnvszweDAZ7e3sb+9IYY612vmlKY61lTEgRee+NNs4hghBCeccAkKEoy5IxscqRO+fqup7P57AI5PJgtILVSZLEV9WaQ0NOmyBNc/ny5aABrtumLMuiKCaTyXQ6BdSwNEhCsuBXca5D/af3ddNov9Sccc5xIbrdbiQVIg/deiRXvV7PmsWbLIRQKg4dLrz33oFz3lki9OQ04xbQOLSAjjHgUjDki8URiTEOTlvdOmOdcQicEs89QwWb/Q0pkkjEHNmigbCzxnsrFk1AnHPWLUULgLS2/ouqHlZkRr8mu0Qe+ULab610wgMRxbFa8RKC2FS4F36pY3HBLViWKa97+atTr/ud65/jChcsD4VMhN0sLcoyiQiX4ktC8HCx4QjrlhjW7DQ8G8+gtfDAOghYDeNinPiMRPTKwDz3A3wRmFi/OnyWnLEa4dOnT58ePH5w997h4eF8PnfOBV3v8AAALZ5bxCCRboSMPm//fh5K+As27xet/OI4DpHLfr8fmDFB+cc5h4wcUMw54xAS6ohI7JnCFsaeKSf5eYNZQ3MrXsJiIH8hVvj3tq1GtX6DiIgJ5h0Bw0BXEpF86eYrxWRcz+emKQVHRMyyzNm2aaqzk9NK243NoW7qoqpq3XLGQreOo+NjS/7l115/8eWbrbV5t/+Nb77z4PGT99//WafXreuaiOI43tzcrK2elwWsngpcG+Fzz9Varg2WiukMMTDKvfdaax4nCAwZiSC9ypgQEjx676Mo0doaY/r9zuHB+e/+7u9+5zu/tLW11bZtMbKMlVwK5JYJLgH7/b5xNBqNAFiS5Z0O9x6sNlmvX1fNdDav6zaLO1EGQAKdv3X/4Hvf/vqv/NKvZHH2/k9+8LMf/eTm9SuDrKO5m06n166/cHh8/O577/U3d46OTw+Pzz759JZSqdF2c7NLDoy2s9kMPdVt45ArYg542VrnnEsj1omMaw4EfenmC1/96lfjKDo7O+NS/ds//KMnT+9ubm5+5zvfvXHjBr9z5+7du93+4PLVF7Ju92w243Gyub09qcu6bQz4umk3NzeSJBNCRCqVMkLgjAmlovmsJCLv0FrbNG3T1GGxkoLVuk0xt9rEWZ6m6Xw6297cbKq6mM7EYLDZ7ScqGh2dtNOisvT2W98ytnz9yzcOT25v72afftI4S/1eX3Q6W2VZGuO0JisYNmDJMc48y8ZzH6e93vYVng7zvPvtb3/bE9UlnJ0eN01jTPvkeHrr1qd3n5yQSl577dX+7m4+6NCBiZSKIrCulsym2aZaaLBoIWMiGk3bT249yrKMyRyIzmZtkiRRN2thZnm7d+Xluq51WzvbcOcZh4izSAhdzEOynIwOU6C1PTudcKGiSDVteXJy0ui6108BW20Kxt10dgbAzkeTJEmdo7t3Hm5sbB6Nzm/0bpS60E3b63SKct5U9c7Ojm7nlbWhXJsQkTNigALduK3L0oPrdnLGWFXXHqnf6SqlimLW28xzE0dR9MLVK1mSCiHiPhwfHwMyFLKYleQoieN5YThI7xl5LpAj8lAWTki9Tuq9VoInUax1Oysqa32WZU2ry6oVXEVJSqwlD0JlRWW0q0O7rCSJQ1zEe8s5yjRyjDRpkYntjc1t2CyKYjaborPlvJhO595756zzhgEH7ztZwlFUVTWvZtTojorJQ13X/TQbDAZSqclkAjIa7u/HaSKEaJoqpMONadq2RWViBYyxphk1bsZMnMpYRRzAG6ut1pFMyJNvDS0rAxE8gIF43moveJzl3Uhl5NDYlohX2nREjjJ2wJx3QIyQPPqyWYjwrJckwDJ6yQA8eWetW6rH+5VTTR6WJsJ7AhIA4D2sjuO9J4+lM8F+e+8DTyVkIkJ0JxSU8rXGS03T4Bc52QAQcAYt6fewhh7WtwA4nHMAF9ECWmrOOLM06gSccSYWNReN0YjIF5K3S24mkbi43vDvwiZxLpcTtYoNACIa7+AiqbFYxAmAL1lg4aPwHyJytypJBc45Q3RE3jnkoXBjwbG2NgjB2bt3Pr5/997Dhw/DRAkOzpE3BiAw3QicJ6KQhggpoXX7t5rbnwdlDCEAASIL5RLgESwCjyMJyIWKVdJReV91hzzvg5QoEkPoUcZxJKXUhsi3QjLFkQnGUEjGw0wgEAJZEBzYItHuPQAhMgBGBN4SMAJGggFjCNYReuskAAXowxjjXDAWoBjCsj4wTP3yCu3yWtfRw+eQxGpWfg4FkrEvBh/MAwGCB8YFE8I5w/P07V/8aw9PDp88uOd1/ejs/M7tz8Ca4UZfGzvXZnx6+s1vfiPNsx//+Med4XDv5Zfr+3edc4/Pzg/++E+sta6uPGd/+nv/rKqqg9n8//WP/nNgOJpPuZKNM5WpeSSiSMlI1XXZto1MEimlMxYAirLZ2tpCBpP5BJGYWqiUIjGG3LXUWCdRRCxhQjDAsqziWAqRAFhL5JlvbeONJe87aaI42xpsnh4ebuSDj3/82a/94t968+Wv1OdF4Y9iEsKCZELxyGi3sdGP4/js7MwYx4FxoiyO51qfn8+BSILo9RIOqM+PzXmVx/DOjY3/5Df/9stXN/+L/+K3jg6fJv1spvWT0QgTMRpN0JjtTrTTkfc+/em81uPTcZ4IlSVR0iHrj44fvvbiddfMNiLL4k0hFTiKOKYxa4oCy7mI+5Kzmy+/1N/ZPR6Ps04+0/YPf/9fnZ+fe8LucGf36hUv2GBruHdt59ad22eTJ1n/hdaMnx4dARfDjfj23Vu7+/vTyWQyjbvd7v7+PhfQ6aRNU04nJ/1+X+vCe8+5FELESnmtrbWI3nNknCsZdzv9g8dP+72tbm9zMq2zJEky6HT7yFmpK5T0wsvXbt588bUvv7Ux7MQpSPVGWZ+Rs/fv37Y6EiFMzTkHsFprKVWv1+t0+4wJRB4lyebmVhRFjInRaHR2fv7o4RFjzDptTDufz5tGn5+Pfvb+h3mexol69UuvE9F0Oo6iKGSbzloMIn1BWINzHhIBRhPiQmzAu5YxU9eGs0i3ZVGUZTFHsIohR2wbZ/UkjSPySLDQD/YI1npjnJAsRIkZY0rGRKS1NcZtbm56D96DdcCY0K0FYMaYX//1v7O9vf344aOPPvjw/GzMGDPGjUYTDNqLIY4khVDSAxlj9i9vFkWhrZWRQg6yTQBAxdJaGyVJmsXFdBZJNRhudvPMOTdvTshjcG6SKGaMK5mSq8OaKBhXKuJcWuPbtnXG6sZxKYLsI6KRVau99p6xReITGBDnHDiEdgDT0biqCiKCSDHwAhkxLgQbn50DeMZYFMtuN0/TNJaK0nxzuOGta5qmrmtrnTFmNpsVRTUcDqWQSilAP51OAaDf729tb3aSrNfrKaUixTvdNMgwE3qtcwCwtKgLICIuuZRSSGjbuqoLrdumIS6IMYwTRRSIRRYgiAqI4DNr570jIN/U2pkGQcVCCsakiFZ8tNAiaT0g6dc6aCwDBgsb7NfaBF8ss2tBhQAUcEEdeEYliTx6Z1c9NXDJgMNllgGW/LiVAXNL9dnVicK/682lYM3jDN3GGfehBtV7j26BJPAZI/DMAS8SHEsqxnpEYaU9gM9SHcP3w//9sg5t3eNc5WhWh1r9df0a1/exNiR9llVty4yPtYZzbskTUegnXlXVkydPHty7f3x8XJYl0qJAlIcneTkhi9wBXtjC527cF9q/v3hDxLquozgNsmmh3kRKGWg9KhKRioLbSp7AM08Ypel6TuciQPBzNuccLXsXrbiLRATcI+KqgRaRD5dI5BH54sahXws5/KVvFwKbF/EeHprdzMdn6Exd1/PxqCrnHKHQlgl+69adS5cvX3/hRRR4enoa+vcW04l3bmdr88YLL4/OTj/+8MPZbJbtXh7NxnXbNK2N8khAnnIpk7hsau89OYhkjMB1a51zSqnNjZ26rKbzCSLm/Q55p5uGcx5wDuccGUrGAQi8c0vhMgAmhEDn26YFYlIIqaJQJtPtdsvpVErZaH337t3d3d26rrNMeqKiKJA1Wzvbg8GgaZo8z6WUnMskSSaj8Ww2IyKpVDkvIqmAw3g0stYMt+JEihsv3/j6N75x/8G9g+Oj0WT8jW98Y9DrHJwcXbl5o9Pt/eS99wW6K1evVW0rxvO5dtPTqQfWtBqIkiyNs1SDTaNuNzmazcamrdMkjqUQnPZ297782iu721uvvnzz6uVLB0+e/vjHP87zfDKebW3ufO2dt+/cufPee+/t7++//pUvG2dPz89G43Ha7Z2dnVV1vXfp8rXNLaFi49zjJwcixrqux+PxZDKZjGfBt4miCHFeVRU4L2UUVnulVJqmKo6bqiWC+WQWRcmDBw84skuXLummYozVdXl8ejTodV//yuuvvvzSSy+9NCtMr5cTVp6qOI63t3Ymk5G1WhwdHS1XW4qS5MqVK2+99dZLL796cnJWFNVoMqnrZjabCaEODw8Pj47ORvM8z7231pg0TdP0ivNmNpt9/Nmtl1568eWXvrSxMXz8+JGz2lp7dnYWcdXLNsIqGZq2hfr+kEqBJYfLWiuSqJdtnJ/qiS/KokFwIkuTJEJw3pmqbONESSmDwxYplWcRIiZZOp2O27aNVKIQOJNATMl4MpshcgSGyBG4lHxzcyvLsq+9/c5wOEyj7PT4rC5KKWUxm8+mRZ7nKouTNGWMRWmS5bkjP5/Ps6yLUhhnQ2OuDByXMooiY1vGWKKiJ/5xq+vWmbJt27ZlggsmvfdN01ZVzZlUnSzPOsfHp2RRcMe54CictW3TaK25SJVSsYqUip0rg9qIA2AcOQLjwAVGKBBRckbeRLHQhjdNU1WF1hVjLE6UEIlzLpQvWqcD2TusmEopEEAEdd0EYqAQqtsV3vu2rYmo2+2iJwDY2tra27s0PjsGtM47IaErExkp713b2iyPiZz3HCBCgYHPEUURcDsej4+P/XTWWmuC8QkeMpH3C7jAvbdSEudcJTlwYCiEiBlKzqI4TrM0DyWgwTI5bQGAiLwD662/qHRwK0wQ4OYXWEoAWK77ztGSCElL6ACr4yyAAtCFGHbozcpYMCorp5vW2gk+Z1BXsGAVMIDnrQ6Gwy5kqRhbNP57niKw9KSdxzVWhIclL1KK1ZFhvYTS/0VAYRU2uAiVrzzbL6prWPmpq0v2a0kQD6ENxAWiAmO0tyFHOxmPb3322cP7D4qisK1WSnGORGBp0UIbANiyYI4olM89c7/Wbx9bCvKsxrkMgKxN2QJKISIOBgNPWNf1wcHBvGyM80mab2xs1HXtvADnw01WkjPGpBSrGX4O861/sj42ejaQvtpoMarQXAuIkMiHiolQDgoQ4IKDZw/7l7StowQiCmKOCOzVV157dPduMRlz7pVS2jRFYTtZEsdpUZbv/vS9Vuvvfe97xpsPPnx/Oh1X1awoJ7PJdGuz98L1S71u4mwVx/G8cty287IoWy2SKBJ8XJfaWSDw6BiXXAgAZq32HrX23tZJEm/2dwk9cqjrkjGVpllblYBMCGCAHAjIEIEDxzExxiDyOI4FA2c8Ik/jhBGURSE4725u1vO5QFZXxce3Pvtbv/69Xq9HCFXTOPChMjaKIm0cIoa3P+gNRFGUpundJ0+uXr1aF+XdW7fR2Ss7e8NeN1HRf/q/+V9funH1t/7fv3U6Hc+ralQUKk9b5yz5Sy9cvf3JR508vvbijbPZZFQ1xJn1TnI01iCxRPJ53RSTqdwa6tnxZp5mO1tEpLVOomRvZ7OTpa+//npdlI+ePLXW33/wcGMwaJpGiGg6L1QcnZ6NAo36ypUrly9dLcv63r17UZq++eJLg61t5GJSVMiEipJb9x4KIZqm0a2ds5IhcsF6vd7+7u7HH398fn7OUMWRBHKcc/TGOZXFaSftnNfnmxvDk5MTRAz9Wvu9bhyLui6EpCvRbn/YiTMOshMl0hEBg0a3Rd2Mp3MAEEGwjIja1nii+Xx+eHgoVRyAyXQ6ffr04Pz8vNvta62n02kULTomI2NJmkvB6ro0xpydjpV8Gsdpmve3NrUxpm7KybQAw5wLUvncWrBWSykHg15RFM5VzjnOJWJIogNjUoqUMWkNGG0U6ljGSgpkXEkWx4t2DFLKvNvt9/tJkpR1UVdtXbdSxsiBCL2HKEqwKIHQEwiuOJdSql53I01T72E2K6bTeV21VdXkucjzbprm8/m8bY1xhfe+4yjNOojcOQLkUsXgHSEQB8aFkBIE6+QDjizP07KpT4+OR5NZpRrOeRx758Baz0gIlFXZODPqpr1IRA4pklESxaEAMpYqEpKcBR/IjLYoiroqnPOcc86lVEJwEUkBEpAzKblzLk0iIMeZt9bqtjXGlAUWUTTYCL0nwFk2B5SchcDA+dkIAdrGFEVlrVMqytM0VJnPJtMgYTns9wJf8uHD+xHzbb3ox62SOLKRMW1RlZ1Ox3vryDPGFCkvwBk0YEszL6vC+ZZzQMa8163W1to8zwCAMfQelnI1rfcy7vQlV1JEgkeIUoq0k/TSpMeYJEJrrdbGm4vOjYEu4J5txEBL8iCurd2L1ANeOM3erRINq44MzwAF74AJHgIkwTitsgwr3MD5hZbzyph93sysf7i+Z5C4DoGZhbICPBNvXx5hcSi2jG3Aov7hmeDBKqJwcS56Vm/gwvAvmi1dzEaASwzX0RV+juUAS5QQMBlfNgIGBAwaToAEC/2PsG81n08mk1uffHLv9u16XoBzUggpBIZyj9DHhBZH/jwmWP2wGsn6nV0fWBjxs/O2+GEymcRJlua9vNPLOn3O+XQ6tdZu7mzGTjECxIivdSRZklTCvXiGCPKFW2CXBOYyZxdKjmHgIaiAnhjzSITAQ8AEiS0gArEFxPlLL33wsGzjsag5AI6MXn3ty08e3DdNOTk7DkpUXGAUSYNCStnq+uTk5ODwiUc6Pz9tmuro+KCbZa+9/sq3vv3O1av7SSzyVDDG6sK/dO2qAX98fl61TQvw4PGT8+mMnCMmuBDOo3FGcOXAzWeFLqpLly5lnawx2hgDnkvknITgERAReViQjkKDddJNG6aanJdSbfYHDAVDlIxTa9I42R5unJ0cW2080L0H96fT6RtvvPEnf/anzjkRya2trTiOq6pKs07btlJKAFcURRzHGxsbk8nk6rX9JBLTcXX50k4aq3u370eK/R/+9/+7azeu3757+/1PPrLOyix9fHw0ravBYPDg8ZOvv/XmzVe/BLZ1BHGaEzBgQsRJp9tvWmuNcdaenp9Nz042NzdevjbsdDpNow+Ojvu93ne/99233v465/JP/uRPHj9+vDXcvHz5cpb3s86AYNq05kc//cml3T2tm5dffvkP/+j7N2/e3NrZPj47PT45ff2Nr7z2lS8fn57fe/T4+Oh0uLu9s3fp0dF5lKTe2LDOJGnqvRcMX3nl5ft379ZFwYiUjGLOnXPzyVimmMZJoOfO5/NepxtF0WwyMkYb20SK5Xly5ere/pWdKJHj6Xg8L6yr62bCRFtWo48+uzWv25deekkEz0wIISXMy/Lu3bv37t1jXO7tXVIqZkLM50VYo2ezmdZ6sLnlvNXOci4QsdXWOK/ihAv19ODYODfo9cuyYIz1er0Xrr9qH5yYxjVlEcexZJHWWtdW8ZiDJIu6sWSRMeZ0cCUpTXrDDUuWVeU8FixSUnD0TudZqpRiHLz3gstI5WnSy7Ls5Oy0KCrd+iSOkIOz1DStUpSlXa11VTXOUdu0UkK3G2vtP/rwk9AUe3//cl2UTdOQ93Esm0Zb65uyNsYglxvWS6UEV855ISQQt97ahdFrXOW9A4bU6/V2d/a98Va3iLi5uVkXo6Zs67odDAb7+1eno8n5+fl5fd7Ne0geQzPS1tZF6ZxTKiZiRjvvyrqui6JorZFSchYzBrEUSsk4icI6xRhrWzudTohIKSmlQIS6KYppYYyZznrdbh60H0LrXgDW7XaJOEMWRQvaP+cciUJVZ+BIxnGsuJjP50U50Vrv9HpBnYYJnjuD3mlnjW6KubfOWbcgiHAhgo1pqXTOee+QEefImHDeEPlAsVzQUv3K5kpvJRdxHHWlUIwpKZIoSjmXzhIu1Se9WbTZ9Ms2Pyuy54qmsLLuzy3xluzKI1+rioSl+ttCIXHR84kYXuguPFt2CLAKA8CSSxyYEGGfFfkt/LqekqBlZgQAwufMuVX8LHRK5px/3sVERLFsH7pIEyypds/Y0We1mJ45xMWvC4rlKoGy/MoXxGBwrbZihRLCbKtoSd5EdOSXDjxap0l7ItJtfXh4+Mknn9y+ffvs9JR74KF4zoP3Dj0xWrITPjdafJYzuL7POlygNe7kekhhPaIQ9uScx3Hc6/X6g0GW51EUxXEcxyqO4yhSfNm0nRqXKMWQnEPiuCRmfAE4Wz0VjCEuu2CvTocLgLAkLVIQshQEFhER+dJsM8BVm62/bKTgAQkIl486BIVZQH79+ovnx4fT85O6rq21CNC05aScqVjt7m4Buh/96AfEyDpNRINB/8Xr17721psvXLvy0fsfPHn8cG97Zzqdes2uXX9hY7iFn3369Pg4UqrbyUazqYok4wKEKMvaaNvZyInw/OzMtfbo6dMkSQabw0G/X9RVUc0tJyUS5wKJWIN3HBmXTHI+LZper6e4sK1ORLQ9GDLgk8mkm+a6amKpQg25B/IMx/PZD37wg1/5lV/58OOPpvN5qDBP07TWrZTSGLO5ucmYePjw4cbGxt7e3p07d4wvkySJOI+6sQD21S+/+iu/+EvvvPO13/rP/1EURVs7e0+ePNnZ3tnYHNZ1fXR23tb1yaWT17708sGj+5999lmadfb2LjmeFOZhHCfW1Va7qmp8i1JE165dr8dPrly59sKNm92Nza3t3c3d/c9u3ftvfue3f/KTdxlj3/rWt7YuXdm6fFkJkfW7w8EG4+7g4GA6Gyd5J4mzn/70p9/89rcns7n33jo6Pxv/+KfvPj48nBfVaDZ3S6+gtSYSEgCSJJmNJ2dnJ5vDQSdP00gJBFOXAJAliWRU6cZLTt7s7+189NFHvV4vzWIh+HBzezodl1Wzu7t19foLG5ub49n06Ojo01tjberj0wee6t39Xn+QvfTyy2+99bWFOFJIC0VRJIT03nvCx48f53l3uLWFiEmSxHHsve/1elknL8sSmY6iSMaJaxopom6nZ611zsxntdE0nY7bptne3t7f34+Vmk6ns9ms1+ttDYfdPK/r2rStEIKc003TVBVjbOXBCEmRyrc2eZMVZDR4bXRjtNPVVKqFcYqiyHrhHS9TfXJ8XpZVFEXdbh85q+uSPDiLTdsa7Yz2cZxBxJSKt7f2mqY5PDwxxsQqypLUKT+dFuS8FDFnCgERHWPgHbaNBRScq2I2j+NYKImEErhU0iNYa2fjCRENuxtKcsF43ZjKkOnaXm84OZ89Pn5azqpXX+1c2r+SRGkxrziX3jS21Q2ht65tGiKSQoTafeecD72bGY+E5IIBOSlYnKgk4t77ULVljAcyfukQA/k0jgRDa61SMkvSTpYHk1aX1VzOvHUbwx3vTNPotjUAwDmBt8aYqqq8dSGf64wRkg0GAylFPZ4Z44wxEoAceA+MQDHV1Np5Q0RcCBScLa249jqsR4H6xxgE9Y/pdMqQIeecSeRccBVFsZTSQRRHnSTuKBkLEUmRSCaBuLWe0AWgsBDSAHDeaPPFqYd1buMqB0FExpsLAjsuOkASQej6zZbivqsdAt8x2HW+VGuGtZ4IyyPQ6rzr4YSVDVuPIqxs3mpPWm8viTyM+ZlgwJpFfMaZhgUoCWwAWHIP4ecVCFyEClZEDQ9f5K8/F1RYTSA9Sx3Vy57j4btsiSrILfaZTqePHjy8c+v26clJ0OIhIr98niGEMACdDwBpMQC/NPfs2eDKKn6w7tyvf+IvkieLP4VtY2PDOmqaZjQaeeAyivNOLzyKsGbavfeWvCeIhAga3uHWe8Sg0YzocSmnDWuRKgqqwov5odU4mVz2LFhcDXmwiAi0alLKcQEXLsIhf5kbrRVcBN4nAkgA2N+7fPPmS2fHjw8Pu7rOra28t1xabdtEJoxT3WgZR4zTZDJ59dVXr1y5tLm5OR5Pv//9Pz58evCtX3jn9PS0LExva6s73DqbTG4/uBdl+Ww+b9s2Hwysc4wBYx7JRBKTJJn308IaIfDS/u5XvvoGl9F7H7w/nU7zYa9tGwyiYKECUvAkSZSS9djGSjHgrtURF4NujyxNz0e6qr020+k0z1MA4Ermva619v333/+N3/iNN954o9H68dGBtTaKorSTOw9BbTY8up1O59q1a4eHh2197L2PVXJyeGQI/qO/9/ev7F/5P/+f/o9PHz0ZDAY3X3z5+Pi40+lFKtHGJakadFKrHRg/Hc/u3bn/zje+cfPGi/3h7sHReDSZHp+MTKvr+Wx70LuyvfnC5avf/dr/CpjY3Nkjrj65c++n/+rffnLn/md370+adtAfVp4mTZ1vDk3bRErkm4OXr18VSpbz4cnJyQsv3nx88PTFF1/6ybs/qxo9Gk2YSo5Pz5wlJlRR1c45HsXBTeqkGSJ2u93jg6dNXXNkw37v+gtXTdM+fPjQaJ1GXDIQ3OSpuLQ3fOutr0USHjx4VMzGWZZ573d2dra2tvb2t7VxP/jhT46Ojk7Pju89LJJUjCdHKvLXb/7yr3z3u1evXs7zXLz44otah87ORIhCyJBGraoGkXc6naoKbREirXW/36+1XhS1Ox8K07Tz86oG55Mkdh6KompbN5vXrT6yDpGJJInLshiPR3me7e7uJknctu1kMq7rejabhghhSKiH+yqlZMgQBaBHBpx5z8hbTZ4Fx4eh9A7Lsq4r4xwpGXfybp53jDMhQCKl9MAiBZ1Of2//shBKimhra/v09PT0o08fP35azOcM0DQtAOxsb/d6AyKMojjNOgBADOfzsqpbIiLdRlwBA3AOORNMcM414Kwx3rrZdArOj09Gs8nUtPX49OzmSy8kSSqEfPTwSRKlb3zlzf29y3WnHo/HdVGbVkcySpKEMeG9D5mUwCQIbm4wJs46AkCGSnDJmSYXGFQMfKebBd8aAMKkeQfhqrMsi6IoVGwCwEKMAQR5vwgSMAjxA8ZYFEWCYdCCbOvKe8c5E0JEQqECyQRXUghFDrxDAKa4ssQsWCWiNM6UUoEAQDbYIUdERjshQztoznkFILwT3gGiRwUIUookjXtZkidJxrmQIlEqZp4b47wHgEXYAJz3ZInIGKOdX9mtdTLjqjUDLNMEy9jBReOZlSWmtbj3uuH0AMjZylrwtYYOYYbDWVb749q2+jCwFr6QJ4iIocw6VD+uXPOAjAnXncvFV9yyMdUiss2eD1EgIrKLs/8FQAGeZS+ujw0+x1F4ZlqW9EnOuUcXVHSAyJP3AJwQgJH3RL4si6ODp0+fPp7NJkhOMA4eANCRD/MGDEMCIjjqF1jh34HMv8IK+D9kXxFxPB4nad7t97q9QZp3iGg2m7Vtezm5bAzTWiNCqBZjnAeguIIOYUzeo3NOKUT2TMxgmZBafbzgYS5u0+LncAyHiEAMEViQEkAE8OBZqMUEor98UqMHQAJHtCy2WMKGpD+4efNmOTu1trgj/dnpEXjtnasaXdV1Vw0GGwOP0DRNt5uHMMz5+bipaueors3R4ejg4Fh1e5YLTBIvxKzWzM+190k3ZwLLskxZmqWRNY13TbczuPni1WlytrE53NrZVRLPR2fT8cS2xrTW21AzI4RgAiGO406eJklUZw0ikqc4jvvdXr/TNa3tJlnTNOhpMhqnaaydBURg6BHatv3kk0/efvvts9Gotrpt28lkIuOobrQxZjKZMCbqup5MJsaYt9566w9+7x/fvfO0k8pvf+tb3/7mt69fvX739r16PpOcl/MCAbpZFxFPT88RcW/vkrI6i+LjoyPbGLTeNmYw3M7y4c0bJz9+98NiPGeIYCmVcS/vCRQc1Kys7OHZ4en5P/vX//bOw6ed4dbl6y+3LEqy9OB8/LNPPpNJbGwzmY7GTbnV71y9eq1p6qPj0xdeuHrt+gvn49G8LD3gaDyOOp0kSTr9wWgym8zmvcGwqpvQT5VJIZB1u3m32+31euOzUwTY39nmjDmri9k0luLs7Cze6G3v9vJO1Ounv/q97/zzf/Yvnjw5CPUgr7766s7OnnHuZ+9/cnDwpG4qRCwq0g6sp8t7+6986bVr11/MOylDIdq2ZYzleS5lBIxZ65qmabVNksR7iKIIkSmlpIzqugYAxljA6YyxYKiscU3TcM4TYOPxtCiKLEnjKGWMWeOTjG1ubgP4zz777M6dmsj1ej1jjJQcMW4aNR7bpqk4xySJhBC93oYQArx3thFAaSwFJ/JW1zUXi6WNMYacew9a6zCYOI7Jo25NqK1QSnGpAIBzORwOibBtTF3Xx8end+7cmU6ncRRtbAwZQSiHOzw8vH79eohYMsamxXw6nznvOeeuYkkUM8G11ta0DBA5a5qGtCXvm3ldzmfj85Fumul4cnRwPBqdvf322/t7l0+Pz0bnk7Is+/0+AJycnBwdnbR1s7Ex3NjYAI/WWqsNY0wwLqIoqGxqrcuqKqt5ABOL5cwTLbRjiQOkWR46YSZJYow5Pz8/Pz8PfqvV2lsb1lpvbaP1bP4kT7MsTzudTvAMyTpjzGg0iqSIokgqgURpmjpniChNc6Ws955zBM5C8MJZn3c7bdtq3QAwaz2AtVZrrUkGPw+0Nlo3RD6QiQSPAcA5b4wjj85ywa0QvtPPlYoRGdCCZrV008WFR+scEjnnjDHGLXpr+WerG1Y8g7B2hzAAESF7XqEofGO1Q0AAS1DBkixdCEhwHtiUuCx5WJ0R1jzLAEourP7FKb6gld86pPBLHTpcNWh+Rgl/8bPzCxLlwolHoLVwxeLSPtdM8vMbXqQnLpDBysjBGppZmJeLpprP8jA4rHIfsAxyMMbaVgNAOS8ODw9PTk6s1gDQtm0mBBGBAwJPCAzQhrKRn1POtz6M9Sl97oe1uX0mArH6SmDXrpTOQ4n/xsZGnMVShVgXW93ZlSrGBVYg8h4YY1Kq1Tysh444F0iLOeH4fBQEAoVxERPyS6VsTxR4p35JU/griCgEOcTlVC0IMOBawxX2eoMrV65MJkej0yeT8YkzOC9GaZ4RcS69de3J+ZkQ8qtvvPX221/P4uxnP3338OnR6HzaNrpp2iztnpT1+bza0dYgb5zRRQMMZaSIUVUXKhL9Tr+sps7UaSz2di75re04jh8+evL7v//74wnJBIVS9+7d29u7hIiIkjOIJE+SJE2zJEmSZOStI+uzvNPv9/M0s9xuDAZt2yLi+eg0SOlrY8q6qOt6v5f/4Ac/+Po33tnc3Nwc/f+Y+69mSbIsPRRbW7n20CeOFikqM6uyVFZVd3UPBiMwwFyQuAYDScMT+RNofOAzafwzfKBdGs1gJEZAzEz3dM/0tKruqspSqU4efUIL11vyYUdERmZWNwa4ELOtLCtOHD/u27dH7LXWt771rc5gOJ6dzSspuFBxHKdp6nmB7/vPnz/3PO+tt94Sef57v/PAc/zvfvDgn/3hH/7yZ7/48z/5t0gBL8tWs40xDjzfSGOUNpjMp8lweH3z4OB6PCYYalF9NpnWm50oCj96/8Hl1XAwmPiOqwXf7m7Wo1iW1SefPM3ykvjB1Wh6enqpNAbijOcpYp4bRnmZn11fHuxvz/K017s2oA7b7Y12ezAYuK7juu7B/tGPf/xjKWW73Z5Mp/riouKy3ooHk3lvMKiE8oMQI7L6MjqOE4Zhs15TSnme02lu7Wxv3jw6zNN0Mhn/+te/Bp82G+FgcPG3P/nr3/n+78a1yBiV5+kHH3xQbzZnSXZ8/PT09JRzbtUHNrcPB4MroXSj2fSCsD8cSa1uHt6mDmWEkCAIwjA2CNIk00IabABhxigFVEmlQER+FAdhOp0Fra7neQ6jk8lkNp00m816ozYcDhljQHAQR1lZjOZT13UppdOzE5ePb95+48b+oYNvFkIgbAbj0e037jLGLq/O722/xUWp+0WVjdsxu31j34+coijA0Gb9oNlsUuqUZcWLUggBYLQSUnKEDEbAOc/zNL1Ia0Etmc8YNkrS+aRk7bBQouQVABBCvpg9lFLbxuGu6zZqATLigwcfdTrdVqvje8GPf/y3l5fXR7feunPnTr1eHw77V1dXCGiSJAhBrhEREklVVRUXZVHkSBsuSkIINfj69NxoiGg0EUqWSCt29vi07tZu3rz50TvfHQwGX336VbPZrNcaRVI6mAmQ4+FEVLLVanluoLWmzOMcpBQIg9bIaMKIF/nY1vEjBSANQ7QqK0RQI4hneOo4xPUoZaiqytlsNhqN5vOUMSb4ok/Ekg6Jjaa1kDBq5uOBlDKOw9D3hRDG6JBRKWWlOAiHUoqNAUAIY7fBLHfBxhZZkVFK43YNIQQaSQmgFVLSYKQwNowpxTHGCBhGGgHTRkipAPgyQKcOYwBgqzAYY76zSZFNRxKMsODGKGXhZCGErcswUlkvwUIWhBAwRglhtSDtLk8xwoAVGK21Esq2XkIIYYOQ1ga0weiFXDEsInXbf3KxjWKEEBLCtjQ0Whsh7OaurDjYCzOwEJpdZL5ecU0WANuSM2FT+yvzxitpDDJoIUGIMcZkYbQ0rIgOFGxpPiCtXhyJEFp2isCWt2HWcgT2jkSpLVMSbAnfCgl4TbPZgtKB65u1PIjR2tjbtKkWWMgQLcyMMRXnvu9jTJQUyBhGqda6yHIuSs55r391fnEynY0MNhShsqwKZOF3rbGxXgXCyArJWbO8WMPVBMDCP/aiaMUrhJeVBJYHALZCWAghoMggpCnGlFBHcqIVkwqVXBlSMF5pYqjPlBZE256ci6pXpRRwYFghIBgjbAESWCQgJBOANYABIAAGL+iPixIGs+hPsfIfkBZrKScECBHrwKhlHagtlkAAtkLCdoh48VxehVbw2k3/lwxjHADAWAMo66kAJsgAcRgYUNq7cee98Xj0w/w/UIcQh27EB0aZspDGeOlc8wxv7m/fuPHGzZtHw/HoZHj51fGTRKiLAi6/uuhu7BZI/PrkUgfNx0+vatHmbDZJ5nPUYEVVNjY3vdjlSOPAmeXJ2fkzhPbTuRn0hw8/f16rBYEvETAkqCjkoBjFcWyMcT0mUCV9VXfqQkomSZYXPqX7G807+9123U/nU1OaVmfvF78c4o1WIVRn4+B6nETe9qy8dp2w0755fp4haGYT0r8sGHXnaYFAk0rORvnm7e7orFfkxX7UKpub/6f/w/+x1ek0Go0gCn/18OH/50/+5OHx44ODg9pWu5IybsWbVSuZzagoIj9Qk0FcR73hsTGmPxqWvGoebNKYfvPkV3EY/f7HR3f33VGvL8qqFnkbG5GU/Vvv3pvNZs9OT56dfTVNLonvVyWquAgYzWeFlHrGy7/84lE9qu3tHX322cOfOY/H/R+9cbTTjNybey0+750++Yy6waSovLjp12o4UEILpIv37h2NJ8M6c6t5PyC6SAdxvVaU0xs3tpPRaHz9/I+//6ERVRi5v/Pgpsbq9OqM+LPRxYxVRdMJBs97f37x70fDOSrZ977/j/YPd9Ni/jc/+fHz5887m1ueF0ymSavVyqdjkaf1Jnvz7v7mZnB+9XA4pTcP69TuGjY8tEIujDGMaV5yqyVshRbsHk0pxUYrXilegZJGI1EWCCGPUS1FmaVFUcwnY845hCG4rhQi9n3bJfLOnTtcqsF4omeJ7zkAEAVhmae721t5Mh0mSVmWGxsbGolmo+04DsbUZR6lVEpVKDUcDl3XQaCrqiAE+YGnQUmtfN+1ve2LIsvydDabMIZc1y25xT8ookQrEEIhhDgvjTFRFG1vbx8e3mi1OhiR7e3tqhL1er3b7Xa7XUrxcDicTqeT6djzvCxP8izB2MaCRghR5mme557nEYSFUAiwS10AbXlvVUHG47Hrur4faq2TJEmSxPeGVlYWIWLbOuR5LrjCGDsa2RiI0EXPYkIIgGNLUZb1IIYQQikmhGxtbQmhbP8qwZWUkjHWarUAgGBmoyKltN0QBVdaK9shWmutlLHsNmOQ77plWdr2POtBc1EUSilECV3JEZJFihphbA0SZtRgZLPUBDNA2hhAiDDG6JIPrmwqCyHHdV3XD4IwiiLX8VcJb4ClpIGUC6qgWIg0gHqhmmBbA5i1zt3WBFqO4XoMutx/l8XuyqrlIADQgKwhl8qsEhaAEUJECWJTP6v6zKIo1jMRK+tibbPjeCt8Hi0LHFZwwvoLWLoRSymB5QnxCvd+gTeAWbIiXj4DWnIPF+xLG/ojgCVKbyheORArH0JKuR6jr79eYzWuTItZ8ypeZQZ4nieEUItiVKiqCgNyXbcos9lk2u/30zRF2uBl/YVatmNe0RrMWo/m1X29cpW//3hB+7CdI4yyz7dRj+2N131/e3ens7Ud12tKKSttvvqrFUawBp8ghBBeSmivFt+uiVJGIcsKXB6KzGpdAYDi/34CCX+/sd5dwg4DgIwChIEwBsi0250bN28DqhymkStHg5Hy3KrUpVaHezsH+3sBY1ky+1/+l//X+dV1s9ms5Nz3/YrL0WR4eLSVTabH3zxyMDo+OSt5QRjWQtTqQcbz6SRnjBKKarWaG/jamCiMf3b8C9+3TrkoC+77vkPdsiytxEjgOVzk29tbgesVRTHoXbZajTu3jo4O932XCllUVTGdjjUCpUVVVlkpVV5RTAwiroMNRllZlFU1HA4V6DiOKKUGZJ6nvkvC3U46G25vtv7w9/7xx9/9KI6jq8snf/e3PwKC//if/09pkgBSjVpcZPkbt273rwcXpyfD/qhMs1ajFUURpVTD3O7JnudNZtOvv/56OB6dnp7eu3O3Xq/fvXtXHd1A2hCEsyybjMfxdNrtdIwxlDqTycyJ4yTPsQbsMOawlBceYbubW0VRGKU/evD+rz77+e7mBkL05s3bUuo0zR+8/+Flb4jzwotbvOJKqI3tblmvXZyeRKE3zZJGswlaDibTSst6q9mo1bZ3d27fvh1FYey3qzLVQgY1P/aC73/nu0/rx0XOm60d17367LPHrhu89/5bYeRowz/91S/SZL5/sCe4roxp1ht5mgkhKCbI4PFomqZZ4EZ5Mf/l57+m7XZ7GaVRoeQq6et5HufScRxA2G6jFuhgrrsKXDjnk8kELXltxpgkSabTqc2XW8qY65PJLBmMxnfu3QvDiAs1n86S2aTZrO9tbZydne3tbite+IxgjKXi0zTpNClCJE0TIYTDXMH5bDbzfd93XW2k1AKDJdg7rsc2uk2tNSBVVYWcl1KVZZUTClxUi11b2HwKtzHNcDhPkuTHf/PXP//5zxEiCOgXX3wxHI6llEmSvPPOfcaY77uE4izLkiRxfYeXFYCO4iAKIyl5MpsMh/0gCKxeAmMu9mEV6bZaLaVUnueNRqter8/n8zRNwdhezI6N+C3LxnVd13WVRo7j+L7PnAW73hgDsJBttopvS5IdIoRIKRFaGGxb/mfDQiEEGGGVAyyzvSzLPCsJUbYJpI3JtYgRQhUvGrU651xKgRf7prIXklJjjD3KCGHMMY6r7IWQVVOloBEQYmv6tbb9jaTWWhswCBEwi75ydoMyGBHCXNfzvdBzA8fxLJCwslhKqZWjoKVa9BVc8hKUUlrxFyUAxphlHYRedm9aGR67cXOzgJetf7M0lBisDVuzWItN9GWy28pvXrhES9R94XysyT3Zv1q9s37Y6szLnw1GC/1my1dYXu4lUH3pGbxkSs2Sb7HiUizj9MWwDuW6q2fXbfUjfuGXAAAoJZZ+gz29WuU11nwX+69BCDFCqqKQxgRBgIwuK24XJ50nl5eXp6enyXRmv2irB7p+xRV0sV6o+YqvAN/moPwnx+K5IkwIWSwupRaJtY0eKKVCiDgKVjoZttaVYPTKw7JPgyz6W2or+7BkniIJtjsoWmItev3BUecflqOAEIJFcSYCMICMhS00ADEArgNSdLpbb771TpKOfJ988M7G118/mU6SJ49PGMP3797sbm+16mHv6urJ48c5F1terdPpECd++vSiEiIZDrTnC9d9/803k971aKobG616tzkp5lxhW9dsn78yOs/z73708V/+4K8CNwCgjuOURaWEpI6rtQatCWNGaS2V57ie607GQ4aF76BGI6jXfNcjnuNWBS3KTE8hivxSKlRW89mg3trMecrz+dPJtUGQldnx6XG/f805j+IgCug/+6f//HB35/337jsYsnTqUTzoP3/05TDncwNVt7sTR94PfvCXqsrfe/vN0XCSzaayLDSvbuzvPfr68Ww2BaVv3Lhx/537juNUVdXZ3Nra2TXG1BuNRr11eXHhOF7ghQ7zHEpd5hiDpuPZk0ffxGHgUXr/7p0vP3/Y3dn9+snTnEvmeCUXoyRpR/Wtvd3T09NsNtu7e/f//n/7fwyvL37xkx/95G9/3m1GHz549zvf+/ibJ09/+JOfbu0eVEoppQ7299o+e/bwq71mfD1PY4Kw4ziBI5C6uDp3CP7g3pt+4A4GA9SsVXnmOaRWDxkmb75xe29n87OHX3muNxwyoeZHO3tvv3tvOp3+6K//g9Ti+x8/uPnGnck4ubwaXl32nz456bY7jotKNetd9SfD2f337ijV5pxTzrleyNpTLkWxAPkxps6KZ26hhaIoOOchoZxzwTkCbbTMiwoWvYajqqowMg4jnufV4jAMQ6WUNGI2n4nHTxChe3t7BANG5uTZY7GzvbO1NR5cZ1Ma+t7W1pZRon91Pa+qNM2NRmmaci4YJnbvu3F0mJWFVCUoqZDJCuW6LK5Hkc+UUmFM5/OpgdLgijmYOcYFZLuOKoOU5mWVFXmllPLdaDaW5yenZcnB4FarE/p+fHD4yc9/dn7y/Ad/0YriwHWdLMvKLGWMzee55EIqro3sdNrb25vIqNFoUBQFIwIA2+Q6wlbG1SitOefT6TQIok6nEwSBMaYsKqVUURQIEQCo1Wqu62JElVIOecHiXCoOSSnVfD6nlHq+YykXAKCUUErNZzPfD636BcESAISQnHOzSHlbj8S1Vt9olOdTvVDeBK3BFtWMx2MlFgi5yxyr+bjYzRFG2hCpmIMcx9XalLwqqioIAg1IGQADyvaSRxgTShnCmColEDIIG6VUUeRlKYIgIIRR6gIgJY1SxrI218PllRXRWnPOzRJIAPUiTLdoluUQ2E/COo1xZS/xqm5tiecjgmGp46mMBVpeEB2sjbHDPhrLs1kUcTJmAYZVWAwvOw3rJEG9rLyANYcDVsQFjPFawwgbkJplQd5qczcrduHSM1icffl/vQR+rH1bHWC7g5plxgEtaZWr9Ae8XE2wupc1u/Ltltv+SlScILwQT0AIY1wV5Wwyve5dXZyeDa57ZVkySm09JXxbN+pXDP/qt8v3X+1n8duH3axsCgdjzAizhCRjjO/7zWYzCALbMzeMI8/zbAoPvxgIvVxRaZ8mwouJCSEMXpSlYNumF2lsKdSrFTN/X2/mf8BY1JAucAUD1knAhAEoAASgkR839o5uffX4S4pllQ232mG3UfMwrdc3Ops7nz/88osvHrY2N969f//x6ZnW8t79O0KzuLmBsMOyYS0MYt/pNmrf+fD9r776QlOseDUa9iVDjsMQQVxxUWliwKFsb3/n/fff+/TTz5CmGGPmQFUIUEngBc16QyqOjW41w1rgp9PJbDTc3mwiJEFWnWZtb28v9Nwvv5RZlgql2p0toQ0XYjKTgYuzNFc8q8eR41DQUlTF7vbmjRuH9968u7nRmYx6nksffvqzyHOePf2GGJkmcyn5OE8+/vjju2/cNIr/5Ec/KIrqvffecylgJVyCQs/98IN30+nk7OxsMMyPbuzbJmdJkrS7G5Zd57puu92+urzM8xy0MVwapTAg29Vpp7uBQY961xqgypLYdfc3N7lU01kqy6oR+I048DCAFGXBeZ5JrqqCv/POe8++efidj97/7nc+HE8nv/7sCynBc8Oj3R2ETOCxmoObISnTJGzGpeFVKbwoNMZc9nrNeu3mG7cjTDJeJUkS+04tDkErK2MCVDZbEcHu937nwUff/ch1ouPj468f/UqI2b/63//v7t27l2bV3vbO/s7eV/5Tn3qXl5fIpboSw+ve5en5m/duNZqtrCyoBQA8zwtDN/Y93wuklFqD1EAIJ4TwLAcAxlwbaV1fX9vQ3H4DLSLqOE6tVrM0e4SQLWa1sU6WccrceZJZ3ajNjU4ceslk9PTRl8moP+pdGWNu3bqVp3PG2Hg89pvtJM2FkARhhEhWVgDgO+zy+mo2mxgto8inDGstW43a9vaW6xEhVL0RYCK0qTs+FGVGCCGOwzlIoRUYQIRVUJZCSz4eVmVRGm2BUix4SYmDEPieoyQfDfvJnEVRpI3M8xxAp7wEo7IsGw4HtVp4eLR7eLg/nY4vLy8dxsBg12WOw5RShC5Syq7rGmNGo5ExptFoUEoBKowxY4wxd2XVBJdVVTWakZQyz3MuiI1iq6oqy3w8HtdqtSgOwjD0fV9rnWWJVevEGNvSBsEXNo9SmqYpGGzlCtBSOI8QEkd1KaVFdxyX1utNY8x0OieLnQO01sggY4BSxhiz5YJCCKGk53kOuFyKNM8dz9NaS62MMQQMogQRTDCrx7FSShuLdeuqKpRSnIuFyBB1CWbGEK2BYOY6vlBGL43EwlHQS8hav7AuK3tmlFSCG2MQRqANaAVaYTBaClhyAzEhBIF96TDP2gBltBDK9r6qhLQGTGl4kVagBGMKDluF1La6xxqTVc39yg+wfR+MeeHlvG4tVjbPrFEZMH6RwjDGnmBRHrmyWNZtAQD7uVw/s1la39WxBsHK49HoRciOlmS99WTEa5PUCK3QaQOgl//BCpWARb4DWbfDdV2tNS8rSrHnuDkh6Xx+evy81+sVWQ7IGGO0kFpKC9+bNfbluhv3G4CEl2CV9QX8VjuIXhv2/ZXGhu01pZSybSTtU1v3RRCyhOgVxUSjZSIDY6yNMPgFBELgRU9RvDjJoofyP1xf4aVhDGhbPGP7YClAxPE3Nne2tvdm8/Hl1Red9mZVVZTiza3uLEk/++yzWZbtzPYPbh71Z/Nao7O13VXgHt66M5kmOLl2Kbs8OXv67NHe3s7xsXMxvJrzXGIjwEiQtibUxZQxQik+vzj+3X/83U9+9anRQnBgFIwDtTi8+8ad2zdv9q6u5/PpZrfFHFqWeSPy2j4zxkSes7XRuX3jyLbHnc3mDWJxbQcj5GDEkHEwxC5TReFhjJTc3ug0Go1Ou51MxpPB9enJ09/9R9979PVXD957O/DcWtQIfOf89PnBwUGj0ciS9FoqXpXpTB8/eRwH4Vvf+ZgvbA1qNLwoumE02ujERZHZZjquyxgjSTIDbIIgaDbrlFLHoa7vJdNZ7+oqz3OHsVYtnE3GlMJ0Og89t391Ppsm/eEYO+5sMieul82mvavr8WTq+t5kOvrLf/9nvMzu3D7yXafT6f7608+lNoh5STZ8fn4VtjoImeFocHN/6869tx59+Xm82ak4z/KcOIJSqkFRSv3QQ1x2NzeL6Tiux1Kp8+Mn3Eg1HRkmwzA8O72+eaP1z//Vv7o6Pv31p78IPPL+g++HIfm7n/z1T3/6C4y8Wzfe0oZSApPRtV86iAoM/nQ8Pj89M2aHOi5dYaeMMT8IlLStcaTjOISwRai33Ci11rwq7LdIY0AIUYK01lJUs+m4LMsiT7VSYByjpVaglcgq2Wo1PKVEVfZ61z4jB3s7piqePH2cEHzn1o1ms9ne6PSuLsfDAUKovrlTFCUlThiGADCZTHhVSaMJY5RSpRVgpJQQoqqkp0BNZ+M8zynFZVUwBzdbsR6V2khKiN1gCQJCidae1pxWSMuVGcBFXmZZhqCsqqpWq+/t7XU6bQCQik8moySZlWXJAodgVpZFksyue5ej0UHgOq1WK89zgrEUlpqAGaPWc3Jo6HkeQshuXJRS3/cDPyzL0uolJElihbjrtWaz2QSE7CJzYaw5Xwn72JTESmhh5QEgZBUMuRQaISSEyPOyqipbR0AIMQaszGdZlnEUGAOu6zFGCSG1WoNSPJ1OPTeoeCGEbYJtMAbLiRPaKA1Kg+CKUQOYAqYIEYOwQRgAK6ONBqoRJgRjRKmLsdSaAhhtJEIEI4dgxisJhmKkACjBCAEFQABEa43MizI8pZT1F4QQGJDFrs0abUIbZXWgDRAApLQEZBijtn8dKEBrAskEiKoqu4Mro20ixkbYtmbHLMzoSojJ2A/z6gwroWWbSltRJSzYY4xha90OzRp2/fo7K0dhxUxbAgkrqvxLePvSCXhh/NAa0rByX5CVUl5qM6yyDCvjujrshYVbc1MQeuGHrYf+q7t4xUIThBmhXPGyLBkjgecZpdI0vbq4TOeJMYYiBEtNBbwUtFg5CqtAfuVdrV9xfayb899i915NaljajdIEs8lkIsEAJrVGPYoiq8xjH9zazYJe+XmApJT24dg2W4QQA1oD0kuhqtVUzdJ1emUa/+DGIueAYSFRantXaoOwNkAwEOYAgnpr49adt7/+6nOEI+L7k16vMmY0S54+PdPIOTjc6o372TePJ/Pp0c07/f51VukHDz5+fvLs8ukXNw4PWezM8xlx92jIQhnubR45tbA3HV6O+nmeOwT7nkspVUJ8+vCX//pf/+tbtzfLQkxG8ziou9Tf2drf2ujubLV4OQPIo5AQZAgg2gxjrJrNZuQH5yenohBZUTx7/FxJ5LDAGIw1EZVkxCnTDLgMXXdvu/3Hv/ePP3j3HaLl8fHxk2++Ho76eZ7fuXNza2PL98NGozUeDT7/4mvPIc3O1u7+Ub3WYpSen557xIk2MRIyn8+Tcf/OnTuMMUa556r9O0dbW1u+73/+8OsoihCWSTouRmW/3+9ubUbxQRS7VVUZ4I7L/IC6HqbMbzWb82JOPLq9u+2FAWZ00B/FcRyGoRdEn3/1dZKX47GIGs133nvHYIQpUUVmVNG7OPnOhw/a7eaf/bs/ffOdd6VG2vXOhyP/4przkhfzg6PD3/n9P8x59dXwOojCMPAqwZVQrkNH48Gf/umfvH379rt37satmhsGk2R81rtqtpvEIfX6ZhBEx8e9JC1BoyRJfN/7gz/83YODvT/9s397fHxcFYJXidxLt7pH1+cXcehwkWsleKFOT576Aeai3N07WDQ4tumGvCzKohqPx/N5urm9G0U1C8AihHzfz7JsPp+7rgtLvM5+D62dU0rZMNf6HLYPvVIKYcql8ijpbG8bWQHSnsNazUYt8APPOdzf3dnZyUu+u7Ntm3aUlQBEEDFSG6M0xhRhoZSy4bWWpRBlxQtMARFIklmRXo3HY8aIlDyKg7geMQdLCY5HtSEGJEaEECM94nJiQPGyFFVZSkMI00o7jEZRwB3H99zQ94xSvZ5tjyuQUYHnEN8PA79Wi7MsIYRcXJw1ahFlOAg9LRdLJ4jFxrHrusK8iPKFEJPJhHPuMNcYUxRFmuaTyUQpZZEG3/ens9QaM6kE55wxZvtpBUFQq9VqtQghtEoPua6bq8xCBa7rOswWC8iqqlzXRbBo6Gx3McspyLJMa+37vjFQlhXn3HVjx/HyPFdKmUXfANtW0VjtSYyxRiC0KniFMSaMeoEP9ryUUouNIytEZ7K00FZQwShjtFRcKWNdTCk155ISz3U91/WNQXlWIspg2QB6kUSQC8kEK3KrlNLLtAjGWEtllGX5gVZKS4UQciiTRCyiaq2VNsgABoQBlaK070utFs2N0aL7MEIIE7biClgyo0MX5IsVBmNXb70RyXpM/Ip5WHcRXrG4a8Z7odGzwr7RkmK5HhPbM+BXhATQ6sWykA9g1QDCHoaXytPrb9p01bfME5TWL9nslUVfZgHMgpEPthiBCSG0kAwTZMx0Oj0/PTs5fp4kiRICL6dEELYJLMsuXLf6ZklcWN3m+qzMyzD+38f6LlbV2nujbbab0cXCWppCGIaMsRUY84J5irE2VgLcaLs1LJrFLsEP8uKhrNxZ+2HGC68LEFpLM/1D8xYWqQf7g179z4JIWhjGiJGIuMGNm28cnzzvTaVMCxzEb95+4/Ji9M3xedzcaG9uTrJslqRpko8nQ+IEXJrLy+eD3sWTi2cV8J3uphe5V5M+i9y2137wvY+8RnzRuw6ePTo/P8+TlFeVqUQBoFCpdPnue3eHw8moFm92thl24zBOZ7NvHg1n0zEy+uJ8HAVOp9Mu0tQLam8c3cyy4m9++LdcaUqc0WQqhZnNUkqmUhts8Garm+QZxaQWRg/eunf36ODs0ePTR49/8fOfpVnR7LT3d3YVV/N54ge1vBTd7YO/++kvjVGbu4fnZ/08FR8+eF8USnLl+lRJLpWYDPrum290Nxtx3fN8VJVjIDWuKs+nG91mUXhCq0C6hJpGsx7H/ngkizKZjPsTwghCnk/r9fZGu4MpS+ZZmqdJNnccuru3fXTjVhDWzi+v42bj0dPjWZa/8+CDg5u3+pNRFEU1LL/5+mslirfevD2djuqtemdr+5uL3jQrkBPmgGZZUab5SX/0zr03P8yyh//v/6dmLPQ9jzl5VUoshsPhD3/010iK7Xb7rTu3KIEKqt2jfSdwo1pMZBQE0bvvfGdv91CXYjKZN+p1l5GyyHyP/dM/+oMH7380HqVFrsHYDhHF9fVVWkwwElky7feu2t12GEU0CAJr3Vcuv2UkgG0nuLHh+QHGeHt7t9/vc86FqBBCSimtFHUc3/MVJbnR9Xq9KIqUYM45RaAFF1oZY6jrTydzz2W3b94IXFplc60kMoogEFV5eX42m00mk1mr022325xzzrk1M9bCEYSDIDCgptNprR5hSsukLMrU9z2l1HA8Mnw+Gg0wBm0koRvNdp0xggl1XSYllxIpUMtedcYY5TgUwOOVZMzV2lDi2ExBFEVRFC23M40xLvIqyzLf1FyHbWy0t7Y7RVEkSQJa1ut1uwFVVaWkFXhDAEAIjppNy1UkhHHOh8PhbDarxXXP86qqEkIEQRCGYRRFWuurqyshF3uZkNzKitnmJVYmAROwhScIIYyR67rWWi3CXMwcxzEGpNSj0QgjahvWESIwxkKIquJKcryUQ86yLEtz3/cBYDqdU4qDIAg8H+EF/VAIIRdq9tgYw7kklBJCfY8JrTBGlC6K3lZRYylLKaVUXCmx2KgQMOZaD5Jg5rpeHMdRFCEgVVW5JIIlYVDYike1KIYkCIPFkNcdBa1Xu7z1AKxFr9Vq9gx25qtmCobiqqqqqsrLgnO5wJAx6XQ62Ho9lC64AggAMDKLjL7tVbaoQLHS02vWAtbs8evGDL3c3wHWIPclm+Gl+H7pf7zwCVaJDIpftprLn1YLrrVeqQEjhPBSHNssRahg6dGtImlYM8+rk5iXmRbrZMPVFQEAlBJSAoDv+0qL68urZ8+ePXnyxCxpoUopZMAul4Vf0LK+A9ZAi3UH6HV/6/X1/FbUAV5ypxBCiGLKGMOUxXEcxrXO1ubW1latVlt1CV/cyIoHusj8GCGEIdgYAwis+2bPv+7Jwcu+AloAJICWJAaEEPyDq3p4dWgAAK1h8eVijEkpGUaNdgdjQrxoOMtBuyyMHz/75LMvHv3TP/pnnc2d/njccjb48yeXl5cPvvPdWqM9T4rNbjvsfHx+enp8cdKO66cXp1EUzdNkMBnc2Gw2WvWtcns+n09H42xe+Bg8xw3r5PTsaaNTS9O02Qq7W80q47P52MHksn8een6n1bq+PAvC+OhwV/Bc5FW33b3W/dOTk+ksC+N6muVC6cFoNt/IW+0NRt2dnd1er8ddnxBy7+bNkLEf/fpX437PxdRp1ikmg97w5u0byTybJel/+I8/+Jf/8n/+8OPvzyZjg92yzH7ytz976+79W7feuHFw4/zsWJZia7v5xu0bGJk8ncY1j1FzcvpkkvaNMRg12p2G69F2vRXFcb/fH08nz0+eKik5L2bz8aQSoee3mk1MTMXz2ZRfnV9wLikmhFBC2PHxMcJUAezs7JRaPz05IQwPx4PBeHznzXsXn/8snY9+9x99/5137//bP/mT7/3O9wVxTy/PHx+fHd59y41qkGWZNl8+fnbr1o3Nwxvbne5kNlVKuVFIABFAXuDXXP/Ro0d73c7tW4dlVRlsujvbvVEPi2Iz2JtOZpT4WuEf/+jvvvryISHm/Pzq+cnjvf2de/fu3H3j9rnX++Ff/d2gP223Nt97562Ndq2UWdjw/IjV2vVup+UwQj0voNTByzoFAsTddXa3d+r1ZqcRyzIjmiNDxr0LWZbtWjCcyrIshBCOQxlFBrg2kjLjx44TMDfwyrISUgMi2hot6vhRFDh0OOg1a76H0fj6jGC02apFUeQHrlHSxtYPPvzgpz/9eZqmtVpDSsnLKggC1/XzNMmyoiLlbDz1XOp6LGBxmebpJDXGBH7WbtaHw75GOo4jRnEYeCVHYRAoKTnntk5aEIkxAcBuHOeTSaE5l8J3fI3INM0wIOo4g9EoT9PJZOR5ju/7SvIiy6ss9bY2upstP/Sms8l0PjEuQID8tpckAkvQXCpHhn7gOz5CSBUVcbRSuhS5MsB8opQYTPqIsmajFbebRcmTssxnidXsa0VNpZQQlbV5dieK4iCMXKU4JiSKXcqsCI8piiLwnKIopMo9j7XbHUJIrzdASFdVgTElmFHqEEIwoogRgpnrhwRRpUTgesqhWlSB40ElG2HMPA9jnBelMsA81/M8hDHIUmmwNALb5HiBHYPGGAxCSi1ywNbMsDAuy5JzSSkzxnDOMWZBEGCMHd+1GjigvaoAShElflmWNgwUspIVl1Ii0NiAgyFPpw5jvChkxWu1mBBijKxElifzer2+tbExT5LZJAmazVu3DylxiqI4v7zI81IpJZQ0CGbJHAex1BI51Cex62sAy2FgjhtqrQGw0Uwoo7WmlDmOg2hh91Mb7QPSBpRSq3JEbIN5hBAAtmyKFZi/blEscce8XKaolNIIAQJsMMEYENLGKK1AW43qxcF6LTvARbVYbwC9tPoYY2wAAdLGIFhUW65xKYwxAgEwYpZ+hlGiWJkKszZeARJW5zHyRbWCWcsdaKyAGkJIwdPzk9OHDx8+P32ujagkt4WsGGNldJlnAGBbicMyL/CSuVr+iNe0m9a9gXWHZn1hX3khK0kcIpUhBFzf0wo44FoQsjAMGo1aoxnV6q4fuNTFCnFe4cixadOl77K8a7RyCBYTWfimGgwoqTUhBAMQTC1SBdpgjAnCBCOCMAIE2hgwCq3V0Vh1DCua8EL0SMLyM2QAjPFWdwwAFs+A/5Tn9J8xXvgtFICitTekqphDAAFQFwirqureO3+Q/uAXCpJGuz4enDInf/udnf7weJqO8wrSXF71wAkQ/upq/yba3tvQIXKU6fXPu1stZqqaJ8b9r8dXhbzRCfnR3XsPIKXujcbTT3tMuaJShiNeBFfnxd7ezYvn0+lk7DpjLVWWJM1GI2r49biWlVnUaU6S9PnldXfvYJPjQb9/edmbZnxaadqKWKfdarSzoiykNFHz3lsHnkuaGx2jhUOpE6Ivnn5mHOnX/Lvv3k2KcjCe4Aqen1/0x6NbN4+EkH/zo781qtrZ3ry8vOz3zm8e7EWBE/j+g/ffdSlJZ/Nbt269/+73pJRBFF6dDqdDw8xGgDpFWXRrB7vhnclsPJmmzdubRLhJP9s/3Ot22oPBIJnNRr3BsN9HhUaOub6+upxlpeDD0QgI7u5sCV59NbginrfR7R7sHpUT3jyqo1iUejSaPP/iSzw9/fIP/tnvffTdB8ejJ5tv7UlDfv6TT3rJNKjVZSVPH5+6rhs5tck4/dN//1ebm20nCl2jer3rm5sbZZn5Pmk262WeZaAHMpmYPPDZZmu/ytJ20DRCbdSbxXyGiKl5kE3OfvrXf3r3zhuf/vIXYRCwZudWa2f2/LL36MnZ578Yj6bu4aFmdV/TRtTtbnZrrVqlef90dPLsklr7tMIJeSXslvf48eOrqyvOeZZlvr+A8hzHqUcRRShDBYAWstKGGABCSP/q2vEDRl3GmDFSG7SgjscxMsbByKVGcIWx9gK3EQcHe7unp6ftdnM6SyejC6XUgw+/s9FuXU0L0ErLRStqLQUhpB7HQlRaibIsyypXoizLXEqOEKqKme/7BtmeSTzPSiE1GKy1Rhg7jieEKCpRCo4Q8jxPKhpGvuUoEEIpopQ4juNVVVUURVFmZVEoJQhFBkEl+JK97xDCrCSWFLooivl8bvUhoiD0/dBlHtJGSkkc126LUmiplVIGI4qRKriwALgyuiiMUgJjz/O86+vLKIrCMHRcxnJKCIprYavVMkZxXhGNPM8TgnDOPc9pNpvjbKq1brfbruMTwlqtluN4lxfXlFKEiNGGc04I9cOQUiql9EMMGtnqWBv22dBZCGEwXvwIiGhmjMEIOY5rjeBi/fWCZ240Iov2QGhVemAxlRWpYhmkaillo9FwHHdFO1+ZCgMLYEAvRRW10lppyySygIGsqqIoanFcca6qQsuqHgf1KEqm08hzuhstl+DNzXZRltubG2VZXg/6vetBWZbdTnvOOQZA1PF93/dDIFgKrZTK88JaIltfqLQCpbEyDl1F+eRFagDwUnBpPdkPAC8UHuG1YP31F6uxbvnssKZ/CV+j9cNeOXIVQ7/yqxXgsX7Rbz1gPUB/Zc6vXEi/XCsBAICAEMI5v7q6evz48cnJyXw+x6/NcPX6dWTif+V4BX5wXNcmpGw1lhRaWdIHprVGM4jCZrNp0TgAAPIaPGAlGAEofdHNYX1ovXDOVn6VZavQNWFJtNaPe32sn9C8/OY/hGGnbZEnizk1Gg1KnaIoDOh6vR7Hca1RT2Z8lsxPTvte0LJgVZ6V88l8c3uzUWsNRwOHSIL4brdd87fH7VjLr05Pn+wd3Yxa3SD2dpzN9z54Z9jr93u9bD5PCwRIUgZKVUIWCCQGJXiezJTrOFk6ZYTUo3jGy812w3fdB/ffnqfZ2WUvDqN6a7PW6Z5eXJ8/f05dNwzD2WgktzffeOs+RajMUyn5dHxxcnLy8KuvhNSHN2516nGWzsf9q6Iq/b29bDapsoRCu7nRphjnaer7fhRFo9HoyXhcFMUf/uEfNuKa53mCl5YQ5rru5uam7/tb29uUUqYCQsCADCOfMqyE0FqDNkoZSp16ramEzuYFlzCdpZPJVGJ3nhTztDQICnWFGK23mnsHB/Miu7rsn59dnZ6e5nm+tbXlOM5gMFSFisLGeJoOJ5NCq8fPLz777OFwONzYuiGlnM0nQeBRAmD4cHg9nV1vb24ZpRPHzWfJfDz1PDeFmdGSgJkPJ0VatLdqDnUn6SCZJg5jWVp2N7a1Vj/72U+rSty9++YvfvbTt+6/9Y++9/HO9natVjs+Pvnqm6+u+70iy6/6ztGdzVan2dxoxY1aIbN8mmKCNzc36bIQX5dlKaUUXAIAIaTdbiOEbA9AS8GjlMZxjAlTSoGWxhipJUfaKgSk2dzh3PMCwlxKsTGIIACteFm5lGKMhJAVLzlSLgEVepgSIUSeZoHvff/j7/zJn/27dD67fefu5fhRWWRFUShlpCCaV0IISikliFeiKDIuSsmLqiqsgl7FciCUENcYlaS5MlpZbTuCMaZhSCuhpE4xLih1GENhreNM2GgwTuaZ1hAEYavVajU74/F4Np0KIYBAJYUxxvO8VXU1LLdRrUCIUhtZlpxSGkZeFMS+7xsFRZpXFSfKILBybsggbIw02AAAY7anA8U0ZozZVIWUXGk5m0+LMo/jsNGo+YHrOM5sNomiUCmFseP7vjFKa02pEwT+tJgTQprNJqNuWfJarYYQscUmjuNYaSnLZrCMvKIolNBCCIpwVZQOZdPp1JIhbEfjlbDBaoNDyIaV2nIjKKWUSq21Aw5zrDGlCClCFllztISazYIYSOmi2ZXzAudfBqnKah9JK8asEGgjlRBVWRZxFBVF7jEHAKQQzWaT83LWfyayxEWmUfP6FHiZijypsuT6vDo5Odnc2jk4OLh1+OFwMv7Rj340HV239g5sKI+AMGoIw5VWZVUgrJE2GqQBoo1WSmvASGksF88XoTW6vtErP2b525d4dqtoeB1aR2tNHdd3Z3jZ9KIlOdHA0oatnfbFg3jZF3ndD3glBH/pii9PwKzVTcDLdn3dtsFaBmQ1DQWaECKE6Pf7Z2dntpbH9xaqU+hbOAffnjL4VrsFi1KLbxm/8X5hQeFEBFNKMcXaIM/zbty+tbm13W63CaOcc4OKFYrw0q0hA4uuFgjgpZPb869cXljURCwqg4iz0qt4yat7fZ6/ZSnWHYj/zmM9l2cdfVvm9/4HHzLXmU7HQsmirEaj0fnFqOIornWDKEbEkWBm42klSsxoVAu0vtpo+0dbnaPdTQfMdiu8d/cGYVF/Vhmdt9rhaJxubrWGoysvoJTF4/Hk+OSZ5zmU4TD0tZagVaMW7W3v+I7rEZYmyd7OriirMAgajUYlpJLamp7Z9GqeZkqjo/298XR+++hgNOhpXjTiuMiSyWhY5ilKehjkZrPBpfCJAZFH1Gx3GmEYbm11jTG6Gdcj1yc4nY9DjxI36Ha7nucZY1zXbbVaHnP6/T6jWErZHw44rxaVEXk+Go3u3uxUVZGVMyklpprzyqVkMpn0er35POVFdX3ZO352Uovq9+69tbV7q5cVF/0xVyiq19wwwJQY5PT6k7PLC4xxmpXdjZ0sTRu19nA4/Ku/+Ov/6bvfjaPO82dXZ/0Ljcnf/t3Pvn70WCM2nU7jeiukCEAn6SwvJg41YeCEEPuE3T267Yd+yNw4CizMfHSwf7C7V3PD65Or6+MLzUWr0dzotpO06rRbg3HvyZPTrc3OO+99+OjRo8OjGx9857txEA4GveF4kPOqtdGObx6FYRjEQb1Z39rdjurhYDIcZzPCUFSL6eoDZAneKyX8+XwOy5wWYwshLdd1HQccRnyv4foOxiClRBiY6zYajaIoKi4Jxg5zFSAhRFmKPCtch3KXuRRR0JpAUfHJZDro96fjkTFmc3PzjVs3PWIef/3F3ds3AodyzkFKAkCMrqoqSRKEUKPRWNQEykoppQETjCglmLjGMExASZMkBefSIK2UKKqSMOZ5HkIYEWwAa2PAGNfBrkeZh2iFlDSUIoM112UQerO5cTwGmqbpXCgVhL7je5Q4CIgU2gAXQtlqPl7JMIgppaHvB56PMbVii8k8Q0JZjUJbasgrmzbGjuNIJfI898NgY2MDAIbDYa/fr/thURRFWTkujnFICBKiEqJ0XQcTsBr1tmUUAGRZodRSaVFVlLKqqsbjsQ1/GWPYoYQsxILKspxOp3HdEULaWMpxHCvA0Gq1OOeIUoQQcxwNyOAF6UxKSYhZaFCWlSVpOo6zhBA8ADAaEUJspw8hhNFoBbxjjB3Hsw04VmNV8ocQUlLa1k/Ygt5aKSUkFwgMY3Q2LikYrQRCaLPTllI+/bwkoMoiRUq0auG5rCaj3v7Odrse/fTq7OryfNi//L3f+717t2+cPz++uroKQ6/RaAihzi8uJqNpENfA4KpIGXM0hhctFZHGgLVBS3/gBXXA7qoWfXkZUcDfahhet76vjNXfw1ohgzGLbtGwLIm0g6xFwHbYGVvYD6/VL+jXhIzQ2t+u5rl6CrBmDl+ZHixpFnrZkOJFKoRz2xlkMBikaWovsdKrfh0zV69JWi3u678IaXjFxQEAqRVCSCjJMAOMGWEIUzfwwzhijEmtyrKUWvuAASODwEUvIUAG9JKVgmwHy0Xn7GW6BMFLS2Re8DnwckmNtfZ4rfBhNcNXPhirp4Bee6z/ncf6R2XFv2GMhVHH85vVYFLN0yTJpNS1Wq1W7yap1FJlWQaUub7jMo9hZhTqXRzvbW9RpGaj3mzYIwjfvHmbBZTlpZIFJt7J6ePry5OT40cOY51Oy/HC2XSUJdNbNw/3tjfzWcLLcntj897tN6q82Nrofv35F7sbbcnF559+Jre2BC2H49HFxYXiohFHfhhzpZthsNlsfPzh+5999hkDDVUx6l2ePz+Wkm+4sub78c0D685eD4YuUm/sb1FKteb9ft/zHI8iyXNs9Ifvv+fEcej5NraZZdmjR48acU1KWa9FjuMkWaq1tvynL7744tmzZ9PJpNFoaC3zPDdYMcZGo9HnP/j88PAGJW67vbG56Y5GWVWJwWheVEAbLTdott04rMVcVqUU0+GsNxzkeR7GQRSERwc3McaRHzTj1s7G9u7+TaXYyenVRf+6N5n88pOHQJ16s9kfz5kbSMk5AS5LqUqHEeb5FND+9s7R0ZEW8mtDGKMFy+/cvvXRRx9tbrRrUfznP/lkNOjfunFzb/uAIG9359Bx6S9++YnnRSenF/fefOP//H/5vzYa0enZeVmWJ8+P8zzvbG/6tej2jVtCCDfquK5bb9X8OBLU5KrMRaGVoasie/sZQkuJG1tkbxEFKfVkMinLMk3TWq3GGGk0GrUoJIRkecqlAK0dSkqElJRVwed6rqTGmDqO09nYtBUEGOPA8zwHG61nSZbMxlrwWhQMeleflOm9N27UI+/kyVc7WzeSJNGSC6EoNsIYpDUA0kKAWqhGEoK0JggbSqlLkTFISSMkCMUNYEJQVpVpUQJoTAkhREmTlYWUEiGSDq8QQvVm1Gw3CDCtoSxkr3ddq9WFFg6hQEAomZe54zm+7zuOizEWQgkl7WogijHGURQghCiixiBeibIsFz6BBK2FUgYAw7KOEQD7jHLOMc4NWlUPSNv5yHUthQ6l6UzKqtVqtVptTMB3fACYz+fGqCiKlFLD4XCaTTudTpIkGNH9/cOiKI6Pj1dyeFaoTi/bRQohtKbLhygxICsu5HlenudgzILnj7DQSkr5Ai/VSCswi2bASGsrroABOMCCwE8wc5hrmZuwtBDWgXCXt7QC6lcpaiGEktLqA1KEtdZKCiWFMQa0VJILgWXFAemizIQQhKCbN498z+tdXTmO0+1uIITjKKh4YRut5nl6cXHhOE6eJWWRRbK9s7XtOE6ZZ1VZBg5zvUBrnZeFMRoBhqUzYPduS41kjDHmrLInGsOSt0hW3wtYGNqXYu4V2rQQo3zNY0CvDfuW1nrVFGq9cbLFaf4LzMArTszrgMS6u/DKn6/M2LpJtsfbFnxPnjw5OTlJ09S+n+e5rSmAb/M8ftMM1y+39v5vu6PXbw0tCkZAKMWlIAYQAVepp0+fdjYSbtTWJmvV4iAKbfCzfvsAgEDbfo9KKTBWiMxgjFcaSoR++3ysKwwAABrwi1X9Vv8ALZED9HJZxP9AX2HFaAYAz2qiSEkIcd02IfF8LmfTREhab3SbLbrR3W21d/NCPT85n+dFIUpANjkoG/VOp7Xheb5DYHtzJ/K9ehSP0yyfzyZeD7nxfDwMPRz5JEvEbNTvtFsiK4pZ4jQ7tVpjpiCV2sGk5odOrYGkrkURMvD82fHV+cXV+UU2UmEceW5wdLhPqJOVZTmcyGKuK9w7P3785afJbK6rNJ3Nz87OPM9tvbHZv7o2xhwcHGx0WlKUGUPdToM6rN/vp/MR0jWMDMJIIoORGQ6HE0AYYzDG5tRsqHx5eWml7aQUs9ksTdOiKLrdrut5iFgxCqO1juP46PCwyMqsKNKqIsx3vbC5sTVLslTI8XWfFAIcrxnHRVE8O7maJVMFZp7O3nzzzZs3j2px3Gm2Tk6Of/3Jryaj0QcffHD7jXtJXgDgMGr0Hz2ZTmH3qCaUqteaQurBcBzFvu/TiMUAPMmy796/v7nR3dzcPDk5vjw750XZqNV3f2e34ceGgywUw16t1orrrbwU55ePa1EZRdF0Ot/c3uoPe+Pp9Hd/93fLKj85O//qqy9Pn5/4vv/BBx/gycyJ6slo1G20ldFJXk3SbF4kRckNwdIAtTIJq3rx1abfbrfsjmDtjW3YQwgBrWwiIAw8wkiazvM8xxjnZWGbAUoDRV4B4E5nY2d7L97oFEWRzmdKcsAEAFVVkacpI3Sz0w48h1dFHPgOo199+glC6O7hPWxMNp9P8qxQUmtwKMOUGGMMKFv8BwAAxtgOghiLSlKKjcZKg/IwIUQrrIwsioxzrtEiQmLMdRyH85I5nsc81/Eo9bRC6TwD0HmZGqPlQsl4ga8EkU+JSymzjHsw2HV921zGcTyllBRSSy2E4FwiRBzHI5iUZamVAYMJpRgrY4RNfFrBECH4dDrxPC8I/MPDg3QyLoqirDLP8+r1uh+4nu9gApbqKISYzWYIIc8LtDZZVkynU4sHUAJBEAghsixDCNlyOBv9CCG0AoxxHMfGLISJyqI0SoM2WVZstNrz+RwIcRzHcV1MmdBKaw0IeV4Ai20dM8aM7SVAwBgGgGzLDwAgBBBSWheccyv6s+Lb24+Q53mELBxQJRcfMGOMVsK+xgY0BivgCqCNkkWWI2SQUYA0MtC7us7zvN3eePfd96qqevTkCcak2WxRSudJNhgMpNGN+kKSfTKdzpOk1x/WO1sM4WbU6La7aZoz7CCDKcZaatCgkUZAAQxCxCACK89gGUwvI8gV8r/OyDMvPn2/IXZcHLdmP9ZfrByFxZNa6vboNedj3fysTo6WAlCr99dN+7o9WFnEdS6heXms/mqd9/DKFWGNkDgej8/Ozvr9Pi9KW0zxSonH+utvTd7/l41XfAX7gi8FRaxEN6bU8/y4Xguj2JYvcSmEEFJrRhbPd/3GMTKWP6j1AkqwEAICpBeamMsko9Zaa4xfLM7K30VrJZ3fipSsnvLf04v67zBWG7t9QBjjqqoAoLv7ZutkoOUX87l0vaDRIHlWJvPs4Rc/9Pxakpal4NN0poD7Ad7a7l5Pq04bO0gPJoN64OA2K/IRUD/2Igc7jLAbe7sH+0dv3bz9Nz/64cXZVTlKXdctJtm5OGnWG1EUcXAm/Ul2UOzc3Dk9OTk6vMUI/ezTL77z8e8Mh8M//fV/vHXr1u5Ou6GUUsqdz7HWhztbQRAgyT2CrueTsydPPc8ByUM3Lis1TzL7aMIw2Gi1W43a/v6+53mtZt3zvEqKmzdvul4wm81maTYYDpq1OiEkDILG5maz3oj84OzsrNvtOo6T5lmaplzJNE2VUp1O5+uvv7E0rCDwO+12luSu6966dbvXG3z+8Msnz54DdcpKuX5QazSVUtOrK8/zciEqKUolkqIsq7ws81/84hdZlhweHNSj0GUOxSgKgjdu3drZ2Tk5P221241u5+GjrxttHMTRZW9cb3QBUUIdrU2ecW1KSoTr4Nls1mg0ptNpVfAyK3tXAxc7HnGrXJyfXO7s7Ny88UYYR1EUVVJM5tlkfIwxSFWp0ty9+8btN24WVek4HqHOdW/06OlpGIaN9naelWcXw16vV2ucMpdyJSazMZdVY6Oxd7jX6TK6crottkkJs5+k6+trq8hmjPE8v9vtKqV832cIpOSMMYwRQeBQZnx/AU0boIRFjheFAIBbrXajWUuz1KJeUpg0TRPNeZYV6azTiJnrNJtNUbAP3nvnqy8+vzw73t/fH/SuPC/wXTYHKKsCDCaYOA6tqnKh04ANQguNekAaNDZGE+ZiSqpSaA2AsDaIc1lJVQlpjMKYEkapA5iSRlCjlAJgIaSqMkJcL3AxJYPrMaZISIkBOb4HAAUvPM+DRR9kbRUFCSWIIK211SFQXCohlTJKaK0WfHiL0ruu63guY2xFF7WRK9ICIeT5C9m4EdKXl3mazglBcRy2202M8Xw+tc2p7W6vlErT1BIObCRncbbZbGbzCMPB2PZfsDQIzkVVCsaY7/sGIWRwEASgtJbK5jKazebFxYXgXGttAIg20mgAIJSaZQLCLPLlVjXZUEqVFktoGq0MPyBlqzQXRRCM2dzw+gatlF6RH40QBjRorYwxwjYwlAQhREhZ5p7jMMYcSsIwZIx4LnPjNmKs226nBT89PTWAg1pzPJu5QYgm86vrfqsl/SjOSxnXWvsHmGfV2fFFOs2rgmNNp9O5VFooLYRSGgwimBoDGIxGCCO0EFYCAGPW+hjpF+bhtX/XFHheM9LfGjKuW/fVC71s2wgvQ9PGfEuZ5be6Ba9fa/23v8lR+C1WZHWhVeWClHI+n08mkzRNzVo/i/Wyw/8sROF1d+Q/+Vev+ApKKUIpYZQQgiihlHqBH9dr+3uHXhh6vo8xzquS5vlKSsGsAVovdCng26f9oooSAAAsJAlrnhm85nihl5Mjr9/X+hP8Hz601hZ0VEp5ngcy9twNpYLZXDaAen7o+ZQQ0u12m80OVzot0pMzeT2YJ8nQCxDCrUbzoBm5p0/P+5fjaT8FpN99/6N3PnpPAOuPp00vv3d06/6Nm1Bkj6OvdOVSSutRXUqpS+lENGQ+NzybZ1cX154bMNfPksT1g93Dw7fff3+rfjQeD4fDoZLy7t27zWb97ORUa00pmc1md44OeDrv9y5830+SpNNuh/XGjSh2HdrttCm12qnGBrdKqTfeeKOo+O7egdCmvbkzGo02d3fiIJxOp7yqOK+qqmo3mlEU9XpXV1dX/eFgPp9xJQeDQVGWm5ubnY29zsb25kZ3Z3er02pXVZGmqeM4nU6XMneaZkKZi+v+JEm54coYL6pPp9Pz/rXve5hRwEYZ2eq0b926YbRM01m/d316/IxXxc72Ri32uaqu+1eGgMIgtKg16sYYjGlRVr7veG6AsCmLSgge+oRF0ZNnzxBCruu2ao39/f0iyx3Hy7JiOBzPZsnOHgmimDiOxoS4JKg1v374xXQ6BdCPHj1qNutpkR4c7rdardFkPp4mT48vwiicpXJ3Z//hw4fj8VQKqLeaBulZMg7rwYMP393a2+VCUbRs6bTIQTBsCS8Y01qtZrcMGz1zzjHGoUvTNNVClGXpONQY4zrUDz2lapPJZJ7nrlO5bii4Okmen52deXEtjmPPd4wxZVkWWVpl87JICQIlT966c2s84LPZzGHkowcPhsPh5fnF3t6eFYHWaSb1Qpq3LMuiKMoy1wgIQYv9FBuEKMbAmAsgsxy4UExRKeU0mWtt07pUg7ETRgj5zGOOS4lTYVUWQpS5VlgIjZDBGFdVhgz2PE+IKklKY4zvECm1lLbvoEKEaGmkEmVZaq2NVFrazkbGSKU1KKmLorRcIS/wVyrXWtsKRqAOi6KoXq9bH8LznEajxhgJgoAxYq0pxtjCvJajaysy8jwvy7K52bQRVZbOLy8vDw4ONjc3nz45BgBbVeH7YZblVTmzsuSzZKqlvbXKKE0QphS7ruv7PhJW+eAFiGqTLPZrZrmiCCH76QjDUGlpdxmECEJgZQyC0HEd11oOWKgge5TSqqoIsUWAoOSLqjzbEBoAjNJSStAKIYQR8l0nSSo/CChGYRjv7u56rqu17l09Pz45+/DDD995/0GSFwBQbzTKSpydnY0nszRNXS84v7yezeb2EpEbP370yA8iz/N4ydNZIjX4YSSF1oAM0oCxAaTACilgQl4k/mFlMvEqIQ3wIvJerAaseQnrRgX9JjLjK77CazUOaK3YUktlvg1RWBm8lcVdxwNWF339Bbxs2NBv4Bu+4issPTx1eXk5Ho9tgY80YD/2bCnS8PoZfhNH4b8W0mCzRLaaUQghlcGUFUVxPehvQDeMoiAIHN+jlFqOwuKJrLkpr9wpetkbWwFLdq00ftFnfLWcr5j83+IlIPQPRZFJL7Um7Q2uIgFATqe126hvCg7DwdQPKSEoiiJCCMbgEoxZ1GrV03JijMzSaW8mbs+relivOPaIx9y4KrJOe/ud9z6aTea9619enZ1fbm69ceNov7sdEUpVYzweI4Ss5Pzgcpjnue/7ySx/8uinf/RHf/T110+Oj4/TNE1+9sm/+Bf/4sGDB6enp0KI66urdD5zGa2KzKLdSsoPH7znOfSXv/4VwbQWxUcH+4dHt2xXYYyMFpwreXlx8fz589u3byd59sadewqR68Hwy6++2d7d94JgNLwyG13GWOD7l6cnv352fFpvJEkynYy63e7e3l4c3yt4NZlMmq3WO++84wbtGweHmEAym/ueQynWQjLP/frRN34QRY2mMPDJZ5+dnl+E9Vq93vz0m/OsyKfXl9P5VBp5fT1vt93vfOfD/f3d+WzaadTjMPr1Jz/jRdlu1hlB/f71yclxIcu0LGyZoYLC932DnLLggk/DMAjDiGDXAOeVoI5TCTGdThu1Zru14XlnADAajUajUavVqtUal9fX12dnca3W6rRLIS8vTzc2Nu7fvx/XgocPH372+adb25vPjp+7ntvZ2A7CqNFoTSfpu+9tp/mvS26SucY0B2yEgChubO0ctFptAE1VVbquizHiUgilPce11DnOeZLMqOMGQSC14IJzLbGhWJFMKN91uNBlmTXiuF6vV1XFsahFjcA3UpvxdD6eTInj1hutGnWLrJRlxRihmIAUeZZWRTJVVZmRi+uLRi18dPIk8Jx3P/7oz//dnwpUIdf4Pt1krctfnURhjDEF0JRIMFwrTjBlhGmDlVKggPqh53m8NFqqyGt4jIisMCXko7zZabq+lxY5r4TjuYRFEoiPO4gTqcFI6QAzBHEpQVWUOggJACyVlkp7fk1rXZRc47lxWhoLJSV1KCO0qirFtcyl1ppzLspqrbEWpopijDBGo9Folsy73W4cx5aPiTGl1KGEaQV5VtpSgrBRCxs1m9wBhKQWBBGllEdchBCApsRURkzGY6XURqcB1AS+43thLYqMgSJPCYI3793rXfcZc9vtjUa9VZbV5eV1kiSMMVPWKl2VsyLyY631bDbrdALH9yspbTmJwxjG2FTCGEMwrkxptEKgEAIEmhKKGQWXSllZq1/muTHGshAcypRwuEEIkOs4C545AJLGZ66QnAAyxlRFhjF2EC7LkmJclaWUktiWA1ozh/q+n+d5vV63Dk3sMk2QxFBrNMZzn1Lny28eVZUw1GXMuexNucRZhbMSuAnHCZpm+XxWAOC4XisIxhrhpMBZhTFGfowASgA/rsOa+VzYCKUAu0oaKSQhZkWqsC4mQgihBUqEMcbYvqavBJfWeK8st3m5q7IQApMXBXXachOMoZSalX7e0jgBgIMRaGNTHtaGgTZaWyENZPszLW4BEBgw8GqkviDlveZb2KGkQMsOZNZFATDGKqNLYUmMgleEEK3kbDoRaZ6Np8V0jrR2MFEIcyWNFZMwYF6Wg5RaWXu68qJWVlkpYSti4EXU/lIranjZTfmNZagClbwkhBCHIYmZQx1EeF7u3t3a6HZ93zdaMgQOBawFhVUqxy6ZMQZpgxBChCAwyGjQFsVRWCuNMVCGEWikDSEAWAM2RkgkBZcEo4XSObBlsxKjtdArGizS2ihNKSWUyqJafXLA6jdYp9P6GdaBQAgAL0SXNQIAg9Y4Ioun/ErP6P/k+HaHzK6qDVqMMWzhbxlEzM7t/frGBoD37Pjk5o3dKHRkpWPfS9IxYVhjlFTz/nQ6LYqjWze9sP7zT56ddqfM2e4N+7TW+vj7f5hrbzYrRCU3GlG75n3+yd/eOujeun3wSJddGuj8quDFvOyVojo42o9qNSllknw9m/aK2c3h2dcXXz2Kw4ar6M//7V+4re5XX301Gg09z3v45LH46kvfdfzAdV3n5q2ju+/eCZq+33AYY8wh3W53a6crpfzyyy9rtVq9Xs+MyhAazufuZNJoNIzrGCWFKDa7DWyK54+fzscTH5lGoyEE3d7exhg/fvR0Pp/v7u7vHt1+6623NzY2lNRlWQKA4zjPLgd/9/P/X1VVp+cnN24c/tN/9k+EEL3ek6urqyzLonCSpwU2dLe+fXBw5Hnez785Sao8qwoEUAujuzdrlJCnXz6VaTUYDILAI8jMZ3mn09o+PAwajf7g4mJwdt0b5lwAYg7Bs1m6e3jjejBBhBqiS6l16XieFwQ113U32g4m9IM79/c3toqryX69+9Zbb3UbndPLC7rjPzk7+eLLLx8+fNhptwPX++LzhxzyvSK9Gvb+t//zv1AEPXv27NHx8ZdffBG4nqj4O2/eCVxvb3vncHfr+U7zq9nMiYjM8q39bQXxRqMTesFsXna3O9Sy27iUnEsAAQCUOQghpZRQGguplMKMagUAQMiiroYQLKV0GLWxr211KKW0Dn4URdqApcaNx+OiKJSoXJeFge8Q0ul0kKnzPA0DZzabEaST+fSte3cYIXEc+36bYWKkUtwyBEUQBM1mez6fV1Vp1vKvhFFKKXMcxghCYBCjDHme4wLBGHmBn1d5yStG3SCMCXMQITYvoLUGs9LGWTQgtiXFxpgsza1AMlq0DpJk2X2OIJuGUGbZoM9ItRrGGNCGZ9J+IRFCGBPHcWu1uuf5aZ6DQUpq5mCMiWUI2raCVtjAliwDgEXpg0bTomd8SZSzdqhWa1DiYIw9z+dczqbJdDpPkqzX6zHmCqGUNIQsihKLolgZLWs/VpHi6lqW4W8xG845DdzVlr1gGCxBDrPU7bYCWQBgyYYIGaO1WjR4BFBIARjQSkiNFcZg9WqEFpxzh2KH4lpUsxVKlvji+3672cryxHJm89y1vcrKstSY5Lyaz9PxeJxlhVZGGpBCe14gDBgEUilKMPNdjDFzHGH0Ouq+ZqjU6p31sWoGscxJL8y553mwDNzXQ7G1AsaX6gt+0yCEoLXyWoSQXmpBmleEdgEAQCm9ft3VtVYR7SuXW/EbVr81yzI/+A29FV6ZvH1tPyq2m/yKl2rlRFdLhJa64LZk+rfc9evzXL/o6rq/fele/60xBq2X8hpbm1MiSk5OTgzAxsZGEMe2HsdYwQCG7f2jtf4adkmWxTgvXUtrjQwgrDHGlqpiL2pxCQBijDEaXizIupzzWkXlyrkxxlidaliUd2orzLS8J70w7UiD+a9G7/j7j+lkFvruzs6OH3o27neYCcI4TedCc6IZdlnoB9vdDer5rXpjnBlklDEKCLguczyn2+28e/9eFAUzURJCsixJkmQ6nSKElBaT8aCs0rLIyiLf3d/53/zzP261GicnZ7Np0uv0i6xs1prIQDbP8mn5cPZl2NkeDvuEEEYYIKOETIVM0/T+22+ORpOf/PSXR0cH3/ne97XWjUbDcWhVFVrr+/fvWwYixtj3/QcPHthmYJzzIAjSNE2SxP7KxHI+nytlNjc3Nzc3tzZ3jg5vVlX13e9+D2PMuZxOp5eXl6cnZ+fn56PRiPm1Xq+nlGCus9FsXF9e/fSnP/3iqy8555vd7Xa7/fz5aRTWb926pTF6enL6q08fzqdTKQ0GM5vNsYZaFHqN5nQydynb7Gx6niOE4LzqD6ftjaLkqqrMLCkqqcJaEEYuN3SxESEDYLRWUlVCQsXBgCy5ievBeDa9d3RrmsyLovjss88qKb56/MiNw7Qqzs7O5vP5fDbjWTEajfzQO+UX23u7oVd/5+0Pvvry0c9+9osiy6MgSOeJrLgWsh7Fw/H47pv33nnnnc8enSdJMkumcSMCI87PTt/tvlflBSXMrapKSo0xNggqLpQ2rutKvUjscanpUiQHISQ5D4LAWpdmsxl6XpqmhJAsy9I850K5fsBcv16vSwOASJHleZ5XRRYEXhQGrVazUY8CjzoYfI8SrMuyeP78+Va3U+b58fExa8rxaBBGNc/zXMo455TSRqNmbuwraQAwYAQGrSzWJJ37vuc4FCPjOdR1KRgVBEle5mhKKHXiWs3xvZJXXApEGMWEMUYwXW6+mFFOCKHUQYhIKZXUVjGCMVaWJRcZIw5B1DYZ0lpVBeecVxUXVWVbb1v5F8tFaNfaCCHGmJWjTpLE8zyMcbfbLcsSY2yLAuwOgjGuSpHnudba932MqBDSyqEQwoRQyurjEsdzAyml0UgIWZYzQpjjOEVe9XqDyWQmhVLKVFWepqfDwTiOa5636N5ZKglgAIyUgnNeVWWeZ0kyZ4zmua6qihAMEBqjjdFSSqzZenhnlsQus2S8wtJlWUSrWkultFIAmmGEsGMQGNAgbUqLW2FmrbWQldKiKqXjOK4T1uLAGJOAUkobLed5arNLCKEsy/r9vm2dVZrKaMQ5r6rKGKDMtU+qUBI5lGgkCQAyzPcwxpqYlSu5Qo9fGQufb9k2SS+NwRI2eEk14RWbaoyxZv0VGqBZS6Wvmzd72hfsRetvIWwdl5WOwkuOwlpLSfhPpRiMMbbic/1NeFkA4JXgnqxJRL8yrObx6kEbY2y/Zquw8q1r8q3jdRcB1liur8/2t5/kW49/ZQLGmPl8nue5danR2oJrArCKztdqH5XSsLDxi3zQysHCgDBgrUAiiZDBBgi1ngbWSGO84HUR+hKMhNYqUZdUngWlA7RGL0iwAGDAaEBo6RkoY10HpC0j6rcv73/d0Wg1AfT9d97udDrtbqtejxnVCKksT1qdVnurIwwkRY4QUlKmaaoVM0Yl6cwlqCqzq8FVfzLw4w8Qo0DADz1EMAAk6azb7Rpjvnn0WbvdfvPDd/I8r1RVpslYinyWIAV3brxBkffu3QfFtLw8Gwihq1xUSeYTNwxDnzlCCOL4jDGhBQKys7crRDWep4TgqBY3u9uDYe/y7EJK2e12GXODAG1t7UwmM2NQvd4cj8ePHj2JosgY43nBbDYbjUaNKFbK1Ov1Wq3W7w3TNK3X691u9/PPP1dKjcfj4+OTs7Mzy9AnhLy5fzOfzx49Oq3VojJNyjT7+uEXXz58eO/+W1EUMdebptm8rNxGPTfw2eefz2aZNiTwa8RoyQVoQRChiJw8fb611U3n2WwyzdJKSvn0yWkyz9sbLQ2M0qCUFZfACNLaTOcpIA2ADAgDYAzVBgxgA/riYrLRbHHOB6PhvXv3vv+d7w6ue5e96wcPHvi+/+TJk/ls9uD9933HffjZ53mahkHLGDMfF7/65OHW1tb+zo3nz58zZlwn3Ly1tbe907u67Ha7QlY7O1v1en2aVQraZxfnewf7vVF/OhzsdrvYJZQ6LC8LqfQq46AMYMocTJRB2HbXJVQIIYXkvHIwQggRBHEcd7tdhrExJgzDXq8njdFm0ZKHUIYQVnoR2SCEtNZFUcxmSMkqd8nh7nYcR5SYQS+1EYDN3H/59JnWemd/7/bt2/u7O9N5EsRRGIYXV9dSSiG1UopLS6pQxhjkoEhEURR4joMlGKM5L8uimM6nnHMFejZL1WwupcSUeL4/y7IoinwvQIvCcVDSSCml1ITYhkyeRRRsdGWAL46UCgCEEHmeKyEopQpj21sPCDXGMEw44laIyRozMBojWosbcRwTh00mEyEExhQhIqXQWiNEpKk4t54KxRhXQhhjHMeZzWZCCABw2UJEwQZP0zy1pQcIEc75fJ6WZQnGCjiqsuB5nltxYkqZUgohY5nbSgkpuZS8qor5fEopJgQZo8oyR8ii7ohSR72s8ouX/aVWwAkA2I5fi31QCq01YKwx0kwb0BgwGEDYMIIrabguCSwq7+Mg9IhSSiHFNS+kNnk6z7IMYzybzQDTsizDMOS8Ojs7E0pGYaycBZyDHQ8ANOBKaa1V4DjUcwGw0mCMZpQCRqUSFL8KWaMllI2WkPtLwbp5qdZxna9gh17TIMLLHlHrvhSsm7iUuoUAAQAASURBVIRvqzA05oV2kzHGdpSWUr6oeli7HHnBtnsxVo9j/fViDgS/fiS8bFZfn9grfwIAtjcsLHnNVVWlaToYDJIkqapqVRy4Xhv1reOVlbdvrjs6KwfuN53h9bO98mLh/xljPXLP88IwbLVatVrNuuBCCMsyoXRh1Jd22hhYdI8E0IAWNBQ7O/t0YNFlGymljMHGGCCaKOK4DJACIAoppO3jJoQQocTrS71yss2K37BcNAMKAAEQMICQBmS7q9k3Fy0fjYFVXe5/6yGVmc/GyhjmuYeH+5ub9WQ2zLPpW/fvvv/Be0c3b55dXpVVddXrcaGqgldGGi2NpMbBWZENh5dRQFuN+P5b94pkzrURSpe8Onl+hg2+Pu/H9eiqd0kdsrGx4TN/MphQ6lBgb9178+zsajyc+U797bffb7f6j7958s3Xj+u1jk8dlxKRl5P5zCDDXKcUPEmSSsh6q56WlUGwS/BwOh1O5kEQ5Hk+n8+zLAOAKIru379v2XW9Xq/f70dRhBCyMFi32/3ehx9XVdXtdjHGn5x98vz5852dHc8LvvzyS4uOX19f5nlqJeMQQt//7nda9dqk3zegZMUZQgSg025rIQeDQSEl17pIs/lXX0Vx/ezyQimipAElidEOYUEU1IPApWxnY7Pb6RR5cX19jR1aa7TKQj784vHBrX3AXtRo6yQrhMhlJqQ2UCGMgQAGjDDGxBBiCFWU0XZ9AxGS58Vf/OCvvvP2e41GY3drW3/66/Prq7Ori2enJ1zJRlxr1huB57UaTerU4zgej8f/8c9/0O40/+kf/3Gr1f3880/7w0G73b7z5r29ne3nx0/T2dwY3Rv2O+2a1ArQVqtZm6WjwPVqUcCVpILLquRCCMIowUwZMFJyqV3XBaWUAS4V1kYDIEIR0VbBChHsui6lNE/TPM+jKGq1WlG9nhdVXlZcaiFVwauyEgAQBEEUeBiDUqrX613KiiA1Hw93d7o3jvayLKvVakKITqu1s7PTm8nZbOZSEvlBo9Gg1xQwKatif3+3tHIFSgNCxiCrATUrJxqEkJUBJSUhCBslpUL1uCWVSrP5ZD7RxoS1MAxCQvEwH7iu67nGFhFIqQlmGGOlpA3TPc+TUtrotigK32NKaAFSKYMxRhpRRBFBouKikkIoUAZjjBEG6hiDfDe0mK3t32yjtCAIKrkQTrZuwaqLMSXE1m0y5koppdAAgBFNksweU+DSxnaccymlYQQAypKnaSqEoIQRTG3jR8dxwiB2HMfzfNtmuigyyxuwWQy7kdn2u8YYK7602uAQQoQQtTQVsNZpaWUO7cxXZR2u6zqEGdAYY5cxSjAyYEAZpbVGlGFklOICUWy0dF23u9GuuWY0GmVFipECAMHzqkzBYKWEKMvrfr/T6cRxPYyjer3eaDRO+yMb8GGMtQYpJZdKGyOEoowiZKRUCCHACCkspayF0fquvbLxsGauYM2gSmXQWn7BLKEFSql1ec2iHNSqbRKlvuWcC4RgZbzXriKlBETWvRC0TGmv4eGLCSOEKH2JY78KT18xtC/s8Rrrfv23694MfJvNfmXYD+dqxax8wuXlpWUvrhZndV+/xfas/3YdGjFr1QevoAu//SSvXNd+cUBhjLW1AbYhu9ULt279umdjXTGMMSzRf1ikbBYugv1XL7W3bCpAa+tbI03QQu+ZEoQ0SIywMgbbPiCW3PrKkq6+L6+vg9VpBVAIsDGAgFhMCCECoA0AAuvpKvj7+VL/KwciyPWDVqv+B//kD4zMDvY6eTaaTQfz+fTdd9+qt9qn52fJfC65QEBlJRXCCBvHcaJ6pI3sD65/9flDxlit2XAJ1lwx6oFhg/60EbeVgsfHp0br/SNHAeldDHrX41s3br/99tsPP/v6xz/6O6NxHH9zsH+z3mq+/9GHt+7ey6bp5ubm0c0bnuf1hr3rfu+61zu9OJ9MJr/85c/3Dg+ObtzY2tkO/OjJkyfGmHnvstls5nlpjBmPx2madrvdshweHx8/fvwYIfTmm/e11kqZg4OjDz/8kGj85ZdfPnz4JQAMBiPOZa83mE7nBwcHGEO73dzf3w2CwBjz9OnThw8ffvPoKy54o1krilxWFQgVen4zjikhWtmCeXeSFf3eIIzmSZZqHSBMQRutJCG0UYv2uhuNeuwxh/MyTzNCiOcGCKGKCyV1ISRxXMgLjbEGYwC7gWOL1Y0xxvKVtNEGgSKgCGF0lswdjTAhk8nk2TePm41GkiRpmj55/iyryt3d3Uatvr+7Ww8jUVRVRULPp+2OMPLk5GR3e3tru4sxfPnlw9lsdn19/d7b94ejfhRFnVZzOBxubW6fnZ0ZbK4HV8Ph8I237o2HIy4F1YAQodgAsR14mWsWHeqQULooCi6V4zhBENiW0wEjxpjAcxljk8lkPBhMp1PXdQkhju8TQl3XBawQVpXVYy8rSqnvu45DpeBZmXHOjaqm02kceQihIAgcFmutGWPb29vjVF85TuD7ZZHVHIc5xABO0zQIQ0DaYIQRch2fuo4v/KIofPCwlXWVWkppEGWYea4bR4ExJgiiWtzwQi+uR1yUo9Go0WjU63XfC6wrYJZxCaUvQOmiKPJ8ZM2ht7mhhZIatNbUIQYThzqIIafWkJyXZWmxytXGpyQyxjiOI6gwgDGmQqiy5FxJrQEhoqRR0hDMAAAjKkShlKkqIWXCOS/SDAA4l77rAWCjdcm5LUOw243vRQCgVYYgR0AZcymlUuqiKARXYARCyOKbFi4mFFGKtUEIY88PgtBzXddxKRel47Iw8gHAqjNprYxSiC4+ACvXwdrOVTdCjLHtKm7tR+B7qzAdIWOU1MYYUEYqgjwpORjlOp5ACGPkeW5VDKaTfpJk9Xo9rjUaURCHAXOdIIiSNGceOzy4cXjjqKpErVYzxjz7//65UsoYIMuWjy51KaWzNGNMEUIsbIu4IIRg9Grpwer1er3AyzH3yta/qoWw+hGtDYxfVUSG1yLI9R+tH2AnD6+WSi7fWTPeK9cNXram9kGsT96+UEvVS/PyeO1aq39h/VfrC2VzTGw5rPdjdVbs1WEtFfKbbPy3vr+a3gqTX03p78l1WJ0Wr6ln2robm5ayqYc4jt2ljOnCz0MaY6wXDwWMMUYbAMBg7LaxWv4XabWFB2DxMw2UGmMIEWAMQswYBRoRgjC2at/GOi7rn5ZX/Cq0RjEBZMtiLTRFjLG3j5dCXsgSRWHhz+D/1lUTBgBTKo1ClFxf9zxPd5pB/WDvydMsS2fnF2c/+MFffv7rX5USgqhWFoXADiGENbzAj6qq8vwoq6rPv3p065efHe7ugFQGOQR7vDJakRsHb4z716PB6MtHp77b32htvP3W2wiRH/7V3wEgLqC70TUIC62y+bTb7b5z5/beZteavevRxWQ6qTXCo5sf/Q773hdffPHpZ5/5vp+l8zwJN7sdrA3GOAzDZrOpta7X661W6/r6mlL6zTffaK3v3r2LMZ5MJlrr3d1drfUPf/jD3e5ev7+ovGi1OkEQzOfz0Wg0m01arVYcx77v+4EbxzEgLRV/dPzk8PDw3v03z87OKMPMIXvbO0KIqF5vdDZrnc48LyfzglJHA2RFqakX+gHWpEwqXmVGBmHgdFv14aB3enZWctFptRTCk/kcE7q1u7N3sD+aTM56F0VVaIRsy/Myz5jnIgSwQLYUQaYEpLW+vLra39kVQrfjOCvyw8ND3/Mopc9OT0ajURBFW93N3d3dvZ3dPqZIG6Ors7OnnufdvH3j/OLZo8cP79y5849+93t+4PzqFz///OHD+/ff3NzaGY8Go8k0K0rHD/y4Vu+2n52fjibjm1rPkvnO/h617duFVoy6hBBMlVBSA7JZSr3cyzBhCGOjjJQyCIJWq0UptURFKaXloIW1mueHiFDHC8IwdPzA9YLryytjwUBkzMpnR8iWDMzn81qtBkZZR+TWrVuPTwaewzCY0WhkAIQQrufxrJRKaTCVkFIbqbWHASGEKaEItAbDQWrJK66FRgYzTHgpAGlCUBTV4jjEDKSoXNfdCDZc17UsBztstB1FNc6lpW5xzvO8tOQXhok9TCm1aHsoBCOkf33NOa+K0v4JwMIYKIkwxvV6PQxD2y06iiLP8+ajoV1J25fBbsRSyjyf2ZKHBUaqjTFmMpk0640wDH3fx8s2TvYSVSWs3+B5kePYqYHjOGCw/YI5jsMYQwhbVFZXleVPqKV6ozEmz3Mrt4KWxbF6peS93NdWFD/rMK0fAwBKKdud3GfUcRyKKcZIKS0Vt7RyKblWRCtFKY2iME/SLMtm4wmuJlkyq4oCotB1sOtFvh9G9dr+wSEXqtFoNNqtZqszmUyiKEryzKd+ZbjWGhmkpOKca1QSzDCABkkAOYwZhEAbhOH1uvmVc2OWfIUVQWG5cZN1OGFlDq1ntiIurIWnL1IbK7OnX66pe2kjNmZpipbew7IqYeWBWNYCfBsDYPV6naH50q/Mbwz0103st57zlbPZ27RP1jY3F0JYFs7KEK77T/+5Qy9v/HVT+pvG6weYZYEJppSQBWOxqipbPMw5x4wJYTO7xhjDqK0mQIuJmwWsAEsOo3n5GSmlDSaYIGPWq1GwEAojpBQBwAjZohIKgK3SNyw5v6vpWddZLzukw5r7uOZmLb0EpBDCxuDlrFbPSK+3g/xvNBhzymIaRdF4PNZiLqpGHLmYaEyA8zKZT4UQUdiIo3qS5WmlENKCmzSpJuM5Iq5H3dE0++Ff/93vfPc7SGohsJB4Oimvzsc7m1vUb+8cdBzHO312PBqd7+7d29loYSYeffV1rzftDZN/8kd/9NHH3+WcV6IazyZnnzyxajGMMXB0KbP+uJJSttuNt+/fswKAmpc8TTabDaUUd1gyz4QQ08lcSlmvNQGg2WgPBoPdg/0gCM7Pzw8PDj/++OM8z//Nv/k3n/3i4c7OTq1WK4pimQ2flWWptQzDEAAuLs+eHeOtrS3fd5vNOgvq9959q1lv/PgHP1RC1mq1o6Oj60H/6uKyEkYZXCQpaOMzz/WjyC97aUWxVcCvXIqiwIk8xqiZjnvJdOQEvueStKwoxWGj5niOodpgyWVliAFAZVU4WBe8Yp5rEDagsSHGgFbYaKI1diOfMJrOJmePnrbD2vv/8u3vfPTR48ePf/g3P37nnXcOb9zY2NhwKB2Pxx5z/uD3f/9vfvq348lVf5CkxTjNJv/xL/4sSccHR0dvvXUnjv1hv88Y++ijj379619fnl9EcX08m1daR368uXugqbO9u4cdN6rF1GBEmAsLyjo2CEBpIYT9Hvo+BgDH9R3H4ZyXZclFEYZhHMfGGM65hftsnyGU52UlMGV1TJutduR6nh92OxuWzKiUUFIQIzHSSiDbieDrr7/e3e4O+tetRu2Z60ZxYJRyKKWU5mVeFoHRGiFfCFEPY0yJq3RRlQYRjIFShgnRqmKAiEuU8Apc5UlelaXU0mWeEJxQhAjSWiKKmq360dHRrEyNMXlW2O+qlHqlG5jneVEUgi+kgXzf73Q6jGCMsRBCVFxLZYkCjBBLZQBt+Q3SKG0AEUpLoRDSUmpjEEZECDWbJXle5kVpjJFScVkSUlkhRc65MYXd5qSUnueFnq+1zvN8Nplub29vbW05S/0ia7rSaZplmWWSUkqtEk4UWhUpsM6N67p2lwcArDgAFEVmpRGsyeS85LxUSuT5oqLagreO42T8BUXL7mj2ADtDjLHrugBgmZtFUfiEhGFIMVnkU5SyOyJZGldKSeQHZZbPZjOt9d2Dei2KHEqjOAg8B1PMGPF9X0oJyMxmk9OL89k8vbi42Nnfw5iOhyODwLqzxA045ULJhTyDMYxQz3ENAltv6TKmFsr82rycd1jlDqxm88r2S/Win5OFSex2bwVuV9yRVcBNyAs6wipQXmc2rNs2s9b1cQUPrGoK0JIwD2sHrMbqHfvvspvlq8e84pq8jiu8amt/A01htWhVVQ2Hw8lk0u/3x+Nxnuf2Wdsj9Vot6G8yPK9Myd7IOr109f5vOsPqPK+8gKUeAKWUuS7GFC1X3uKdWuuqqjDGVEq78nSJhbx6ZvTSVfSSRyKlAmIoULPotb64WaWk1iudR7TMICyBrjVcAS2bnqw/hbVraWMAIfytHsByfQz895JfyCoZutTzgw8/+qjKB0rMGFVJMuWch6Fvt6B67TkQR2tdFYVBgTaKczmZzfuDCaPg1+Isy/KCe0FMpO7U2oabcX88Hs02mt07995jhEZhWIs3yiRz3fjicvTpp5/1r3sYo8GgX2s2pFZXg+tpMn/+/PnhZnM8HRtjgiAghHhe4HkBxl4Yhnkxr4rkwwfvAcDjx49HjFVV5fqhlPKNN97I8zxJko2NDSHE/fv37Z5mayYdx/n8888vLy8/++yzuzfv7u/vM8Z6vR4ARFEUBB6ltNNpeZ43m0+KMlNKScnTVIxGo/b+0ebO9hs3bz169CiZTJnr+oGbTGdSiFF/AIiqSiEDRZ4rQKBRFAcMI5Cq2Ypv7+0+ePvujd0dn+FPfv4ThBQYdd27mMyzuL3Befns+HjfTIVUwlSO51VCCcG9wA/DUINBBqwoHMGIMYcx16VOUZWDweBgY2t0ellV1Z/8+Z8hgKdPn3Y6ncODQ0Co1+ttbmyA1MJAp93+6MN3upvNzz///PjkfHe/m8wnz58/+/rx1x999NHR0VFVFJ9/8fAPf/8PLDbTaDQGs3Q2z3Iu2p3um2/f72xv9oa9bx4/poCIBmUQVhpJBUohBI7vhkooTBHDDECrqsyrHGOIfFoArZSUGF9fX08m03a7nWSF02j57XZVVYwx25Xg6bOvGWPNZjOktOl5yG+WhajyojKOR1zOU+bos6vnUf1mbzK6HAwUdqLQ+/zxs4owTPxCEOo25gq5bjDnyqvVgBE3CEZX557neYFbFIXDQCpOsGuM4UopIxSTLMY08MAorhK35mJkKp4oSYLQS6eTSa83zVOb/G4ENQcxxlgl1dXVlec5RVEANpgB8QnDzI1cv+ZDJous1FobA1JWWmvHcSgjGGNkVC7yXOSO52iN0ixTpXJRDACMMUqx57kEwbDfi2vhcDgMQ991XRcbBNrCFCAqLYyuFOJADcECAUUu84JGmKapUWg8nJpFcsRzmK+Uyqosz8okSUbDSRAsKJnJPLPbnBU+Qgj5vm8hWV1yAKDE0wrPZ3kYhp7nJfOi3x8v+F/OYvNFQKUAKVVRFJRS13VWcWRZ5dpITIAxxByEMcaE2eAQiDJYVjIHbiilvu8YpXlREkJ4ngLnRZ6FBwetw8Nprxcx1m62HMqu+/3xYFLlsr3Rrcr0/KR/cdnHjFWlmkznZVkRHJ0/GuV57rTrruNgjFPOlars7l9JjTFGCCuFZCVs0YqhpJAA+oW8P4DBGBCmGBED2P6nDVIa1NJurQoCzZLTZyGEFThhESDrZBBCtH6pG5BetiFelVGsXR0QQtQhVlfDgFK2hTGiGGMCCxUgS7DDeAlyIGzhbFj8cmFmtNEYY0uEXNldrRUyAMZoo1fG3v4VfVmHwJ7QGGO0segBwlZEAYwxCJCUiiIihCjSPE+yUX84Hoy0UJyXAIAxLF0ie1vISmvaua18IQBYfCpeY4ToZZdRAFixYmGZIINX8yOvuk2rgbABpA0orSWAJsTBBCjFjkO5KKXi2NCyLBzQvu8b0BgIXqN64IX8BCgwGjRGmKJF4QlXwkiOlBSEULVMRGKiQHHFXdc1AEJKpTHGmAI2SAplJMdLb5Ji/MIHWjF7DKwKYRHGWEuDEAFjUxz2g4QxQkYbMGCsNIadpq3F+A0Kkr/ZzXrBRF4/+DcdX3eoUooQXJX82bPT+/eOkvnAgMt5OZkUeV4SYI1aEwHlwuiy8rzIGKRzmedpw2/O53Pku3vd2/ls3gw35sPRaDjc6XRPvvg6N/p3P3jfN83RaHRydX122p9MZojG9Xo9kQjH9ZPT41ar2d7dcQI/bsRpke7tbaE8o9rs7u4ihIqiIADpdNLtdvN58uVnn9fr9Sj0q1IYpS97F0VRWH4bQzAej7vd7vHjR0KIRqOBEEJBkBndiMLJZKKqajoc/O73Pt7Y7Gb5rMZC5uobN25wzieTSZYWt2/fJog+fqzOT/phEG91alrBO2/eeXj25fOz03maFEJmlZil5STh1K83vDp23CSXQiNjEABMR+Mgjm5udK6vL0VZbOxt7+5uv/vBg6PDvb/50Q8h8MtkFkQRBtR2Q8rc4fUw9oOCo8kgMQXBBAJEgYTVrDAUIeJM8zlmJPRdYoxrNEEIJG+wlmNoOit/5/f+gJcVMfp03H92edJsN8ajXq0WRVGIdJnms8lkklbtm7cP33rvzbgRxI0vTk9PXS/Y6W4yxv7q3/9lnufGmKObN3Z3t+udxiyf3//gHeU6n3zyCVj0uiifHz8+OTlhjFG7JyJEMCJa2xgXYYzIgsy12CUBIaWEEML3/VqtZvF2xtjqe26/V9ajtztvWZZZlgUOc12fEg8ZghBirtPwG9r4RTHZ297Z3toihNSj+MbRbVnpbJb2y4oSQm3Oe9GvmWgAY4zDmMM8W2RoeXZlWQImxnI+NKIIG8YwQxgZIxXGGIxCNiLUSCvAhMVx3XV9x/EwxlblTSOMMe71esYYy5l3iYso0kLOZrMaCeDFRoOt4grBRGlpMwY2L2BjUK0hrIcWH1PKW8VSNlRljFlNobIsrSaE67oZL2yRxYoyZslZVigTVqIRhKzq1/Gylh2WLHSABepu8ybLGEjBks2+MooWAbJCeytQdIWaGmOUAd/3fd+3zVEAwPoWvu9XVcV5aSlvxhhKseM4vh9qbQBwFIWUYl6UgnOMUJYntSjmVdHptHzfLYus1a6laTqcToxGQRhTL3QdHyhN02IwmY5mM8fxpDKVUoZQ5vrUA+wwuVaKqfXC+lpjtxrrlvuVbXQ9oH/dIMHLkD56OZUO31YpsHIFXnlnPdeO1iCZ9W3arJVrYoxX2jj2jfXdfP34lc/xiiOyZF28ylowyzK/1XxeCanXz7M6wDJeF4RZs+iGYMU9V0m69c/J+glXP76ytua1UP718a3IxOvLvv6rhTNkCTQI22nb61rusHXp1s+wfqfrUzXG6G8rY1kdbJbaIUIIwAuoAGMMZlkVqcAiGYSQVaex1bcPoReNpRd3ivCyHnZ149rYOkm0eJoILTIOv2UdftP41jX/LefhXAvJw5CFYbi9tZMXlQakDfqLv/j/E/dfTZZlWXogtrY8+mpX4R46InVmZckWaAEM0GOwAfEA0gZmMBu+8ZF8wSuN/C8gH8gxGjmC0wM0uoHuRovq6uqqrNQZ2sPD5dX3Hr0lH/a9x697RFYXGuDMtrRI9+vn7CPvWt9e61vf+uPHjx/zwJ/MFpiQupZFJQEjY4wyWghRSRFQQhh19d6y5hcXF9UyrZbzkJIoCc9Pjn/y07/y+nuvXr1yld5FnX3z6Avf90cXw3a73YpjhvDw9MTHiFjwCK61effdd09PT110s91uz+eLyWTCGDs/H+7t7YVh+NOf/vT05Lzdbu/v7zs6nrX26OgIITSZTLa2tlxodmtr6+LiYjQaMcbKsszzvNPp3L9/v9vvPH/+PMvzZZpOptO6FGVZep53enoupZzOZ2EcJklMGK6LajqfDF88P47C55UMeOBr+Pyv/vrs7AJXdZC0oqRTKGW0LgoaUoZ9CH3v2YsX9+7cmk3Hp6enP/j4QyHEi6fPzk5Ox+Nxr9PtdrtFLaSyXhR7UQtjPMeySeZCU0QN2Frrcc59bq3OlkvNaMx50mq3wzDw/a1el2I02B48uHPXSoFkjcAMz86tVqoqBSNZtpyOhsjoW3f3xuOh7/N/8A9+O8uK49OTdjsOguD2nRuOEY+xLYocwC4W86+++vLGg1tJHFRVdXR0OJ/PKcKL6XSxWFCErGO2Y0yV1I4g6sKzdV0boxCyjDOMQUqwVnNOkySaz6d5njLGiiJzeIJSihAxxgihKKWeFwih5vOl7kRpWdW1rnLBMOm0291OFPpkb3vH83bu370d+YE1xiP+0ctTD1OCDcEMAdZu/W4MxtiCDqIII8IZK4pCSdOOosDzZF0Xq0izQcZaiwlCGANBGDOsZW01YEYJstoasI4ZAK6ogXEupa5r6b72SdzmzC/LuhK18tT6G4asstZYq601K66TVVYbWde1qGqGSRzEAGCtpggrpdrtNgCEYdhutx0sYIy12nG326UUM8ZcjsPV8xhjKOYNdcCRCRo2GQC4vEZj19wr5coompin+yvnqwCAWVO+3aG1Ug69aa2DIHBQw6lxOzDXWGq3o8ar3KqTm3SVq9pIp4+ktWzOxwEaqzCl1KMhJZwAwlgBCJekEKKy1t68c5N7+NHjw6ou8zI9OkdhGK8I6owLDWldL6u6UEphpS1WgAxGGFlARBPiCGhrIsXK4K45ZJcFC2sfb8hVL9g47E3P7cY1Z3kNOlxDFRu/XokZNH/dTFG7zzfJDdemWk+4+cn69Da87Led7Ro2rdMrTkoBuXDBKhaxeRVXfgAX2b5O27TWOl6glMJagxBYa7RWdmO80dNsjtcd0t+64+uO+ZcfwmW4tNawZpbAui04Y8w5LUII5avmpZuoxRjb8I6bHWEDvcFrKMoF/lY8x40un2ZdJaE2yKQYX/IWG/4KgisoxFyiOg1gwbozXMkuWWvWL5gGQIDRfwpHoTnuL9mGeRgTD8AoAwD4j/7oD/NsVpTz6WyaVXUcxwSzVqczmaW2UmErKQTSQteqllp54OjaFDCtRP30+bOYs3Q6Ytjs395LEp50w+/85sf2r8V8tmSE1SJ9dfQq8LxW3L51sPPWg99qt+L7N2/VZbGYT7BSN7o957YbaqqUMk3Tsiwp5efn51prVwq7tbXleV6WZb/26z+Mosha67o/cM4fPXp0dn4ShB4gE8VBr9fzfT9NUxdIfvL0KSGk1+sBgDGQ5tloNKKUR0nr1atXo9FomabGWj7m1lqhqu04Xp6ff/3l1w/vPfS98HA4thZt97cm0znnvK7EZHgxmc1KKQlnjLHf+p3fTpfzIk+3ex2M8R/8639DQIce/+5H33nvvfeENs9eHp2cDcuq4l4ktGqMGJgrNk0qbcAgCwQTRqjPuO95oR/0kiD0/J1eZzIag6SmKs5OjzmBrf42VkLKmnMeRQHSYkEwAnN+fra1tbV/sHtwcMA5H45HRtvj4+Ot7fZgsA0AZSUwMdrUnW6ctKLJ6ASsNLqo8lmVzznzCTYMI0dl19YSY4w2Smtp1xr4jBGEKCD3RQLGCMa+H/Aw8k/Pjp3olROmcIh78xsSBJG1SEqtkbUIpDW1UbUUaIm0qXwKZUbCkNy6sX335q2Ae0cvjvNlevvg5smjJxYbKVVZltoayjBCSEjZaseiqhZpmue5VgqszdLCGIM4QwawRQZhgpAFo7XVRvk+V8oYY3zOACGtNUaEct+ALYqiKGvPQJqmrq7GGNPrDrDFohRWaABwRbRSSmxXMWelnFUyWmtVK8ap6/fo+37TbU5KiYzHOb9x40av18EYl2UJAJTSoijG47kQwtVxdbvdIAiklJPRrKlCdOETa21DH2v4X7Be9jlVRLdZs/gj6yZMZi3kDhuuBa/jxa4eMk1T1yYY1sVjbhHm5qlr4VpLUEp6vV6327XWXgzP5vO5Uspa3ST4GSOUUitNq53EcSxELWTNKEW+XyoVhuHo4nyw1dvZ217MZxfjoTEqjuOi0gZkqcCa2lqkjE7zKi+ltNhoa60RAKtVstHKWmJW62OlFMBaePhN62x3segqIHDP7hpR4PVxbRJ7denfDOfdNx3n5jZ2I0vdfPj6Ua6dxnqey22amTd/aHZZJx1WQMElBZrPmxNb9728ckCEEBizzn8j55TdO+NUlazV1mqlhJS11tIY5Sirm/5704/C1bAHrLFsA2jgb3NXeF3FsDnPL0EMjNH1hWMXP3D4oFmWuVf62u7rCTc/XZdrbgDKa89rfZ+h+YptXri7KKfM2Dya5vI3YUeTt0IIGd3cf7AWAdIAyFqMkIsPuTyJ2/3vSFO4Buzs1ZDJ5skDgCNNaG0p5VvbN16dXKSLUVllUeRNZ8vJPHXie4s8L2SNEMLMQxpZBNpajbBFWBiNtSTcQ4S8//FH5WwcUPvBO2+FnLRb8faDB5jW49GUU97rhYGHAy+8c/PWvdt3eu3OYjZfTIZJHIKoTl68GI2G/Z3te/fuVVX16tWre/fu3b//oN/vp2n6J3/yH+q63t/fv3HjxnKRnZyc9Hq9g4MDz2OdTqeR0xBC9Ho9rXWWZU7pbrlcWmu3trbu3buX5/lH3/kgz/Ozs7PHjx9fjCeU8CiKbt66tXuwP8/SWqtWr+15XpiEzlreeOvg6dOnPGS5yGututs9KbQmNux3/CSeqtlclqkRNuB+ktBWdDa8KLMUE5J02p9++qkVJcdwcXb6e7/3e0kUX0ynjsJ/Pp70tnYBoZIbYwymBGuw1jJiwSIDyCJUCCVQHQVeK45jzqlFxTL1e92tTkK0RLLKF/KL6WR4fraztbW/tdWKwjy3i/ksXc5d4b3PaRB4nudauGlAinM8m82mswuLkZA5WJzly+cv8sCPWq2O57EXzx8hhLSycRwEgaelCYPAuxusWgJKKQGwUkYKJwtIXJ0CIUQqUVUFgGGccM6DwMcYGaMJRdyjQq4SBHUtCSEIkaqqjBFgCQLscX+eT0I/arfbnVYPGUsMGFPVdeFRSilxwrHucMvl0lrUilqMMaEVxhRjCMMQESyEELJaLjKtNKeUYFoWlVKq1WqJyhV/Y4oRIgislVJKLapKVLXEYCillBLArgk2ckKB1iL3n1IKLEaAKCGMMJ/7xjcIIc59a21ta0pY45W1lsaYUpaiklqpuq4xEJ9bUQsA8DwPAy5yFcfx9vY253QymYzH4+VyqY3UWruLdZ7ewQWtdV1KvJZrdOjERRFcBgfWLLzGCeF1ib8DE25ChJBLOLkNHBHPGYKIMfeF0VonSdLMH0VRQ0FvBiEkTVOXYOp02q1Wq9PpCCE8z1ssFlprjMFJUTnIwjlvBQlnlGDkcapkXRQFQTYMQ6lqA3b/xs2qEucXQ88PqqqMk/ZoVueVVlldVaIUtRSrBpzGACKACLXWFagjQNgSqmSJVtJYpjG4YK/oIK084fq3xkNfs9EIoWs2tDGXjS9t/vQ6Slj7gCtGeRNPNC4c1qRFay3Z0GJa7wOwijTa5iQbHwMb9v2NP5gNtqab2L5pXLs5zQU66uTrTqWp3DHGNBEvF1W6dls2J3/dd8KbUIL99tD35pybR/m2jTcdnrt1LqFm13LOFhHGmFm3bkJ0FcM3xlWfNUGI1YU3aSb3FNavyGVEyr1pWmvXNNUd10UUjDGIbDb4aNAbZoysb+8Vz23BAKC1OPRKfKkRXLKgAVZSCmvE8B8XUXgjwGru/+aza/5qjMHIApDt3T3uhZUAZVCai0qISlYYY8IoIAKUaGOUVQYZgwFTgjC2ANqAVBoQQozdvX/HJ7frbHbr7kHoUS3LVy+/BCj6/aAVtyMf9TsBw97e1i7DJF1MXx0+nYXRvTu3F7PpdDI8PXl19+GDnZ0dl0I9OjpaLtPlcjkej+/fv++ewnQ6jaPWD3/4Q8bYaDQKw8QYtVzm33xzijGez+dr5cDK9XsTQo7HQ89j5+enk8mkNpJzvre//7/75//1cDj+5tGTr79+9PLklPpBXpU08HZ3B4PtfrvT6nbbCKFsOq2Pnt/+4J26qK3Bv/4P//5X3zz9m5/9YvvGfkZsanWOoWYMGFNWl2VJde1x2mon/+yf/TMkKl3lT7784qTTDrmXLpdFmgXcI4TM06UmDDNaM2uUJWvEjDF2hW+UMq4pRRiMNUpZhEFpoNhKubuzPb4YImvbUSzr6sburhLy5NUxJqCUmkxmhNJbt249ePj2nTt3aiLOz8/n86mQlbX68OjQqebzwBsOTyeTmTbgeaHvhe12PpstTsav+v2+MQYBCYNAE8s95POAaq2kVMYYjKi1yAIYA0oJrSVjDJCt6zLLlhZ0q9UKAp8QtFzOMcaEYGM0IUTUCgDcqtQYrZWplRC1xJhqbX3KGWBQ2iiDLSBKPcYRhf39rSThSaedFwXGmPuesubpi+c5Dlxa3RidJEmn0+OclmU5nc/yIiUYI8uQQUqLPM/jMKyEdB6Oc84oa4yCE4oBjKW2FmmEkLaolrrM8zzPGas451UpjAbfYwhwWVRGKYZZJ+kQQqwxdV0bhJU0GBFXK+ViLS6S7yPfVRzUdT2bzQBMt9sFgDyTLr+wXBaz2cyRxquq6na7OztbnPOqqrIsWy5L93V1ezl75xCAq5Z0psf5J7LqVOQ8xKUlbTyHq35sjHKDMJwJc2lmJ/TkTsasq7ZcuqERaSCELJdLjHEURRjjNE2dhLaDBW5Lt0sQuOoKrx14eb7UMo/jmBE7zxYY436/XxRFt9tP2p2nz14+f37IORUSTs/G81SBXeEeobUT2DSAOPMtQmARWGwM1JW2Vhmw1F4uEMlaVNGYy0KDN/rFzYzDho+/dAwbjvMKenjdzjZbrl1m0yXk8ojNzJuueoXDiKOXXnG0zWzNo8TrLoXXgMLm+bx+FADY7Ga5OdCq5NLx55v/AK0UpS240wZrwViwrpuBVHVepPPFdDafZPlSSOHInk0A45c4frtmNrzuqP5Wx/+3btYM9waidVkv1RrWtB5HRHWvcROlu3ogu+KEbvb+WP91lSxYNZzcvOdgrVVKIXz5qrjVkUMM9hJtrHpxYYy1pqsX4zJ+4IooAAAsMhYAwKlAaosAAbVWA4I1XLh8ZP+x49rLvGkrXr/DBoBzZsEoLTwvipLuZPqZlFXUDqzFCDMN1gLCBFmEpTVaS20NYKQRaLDSgkEYYQqEjqfTFy9f3trraVnOlqPKp6OL87qaK6G0tvNyYRXaGSQeC1sh85ivqiJphcjavMqFLPZv3uh222mavnz5crFY7OzsxHEMgKIo2t7enk7nvV7PWrtYLMIwdHTF5XL5Z3/2H5IkAYD5fH7z5k2M0e3btwghh4eHs9m03W6HYTgaDVutZDabUkqHs9F0Or15cOcf/t5/eev2XepFeSn2bx60Wq2k2z46Oax0LawyRGMPGaPPl4tRsWy3ukLIVtx9+NGHF4uMhuFn3zziQbIo67xUBjNtiCwVFvWd3R2j5cuXL8uy/LWPPvz0b37y5NHjWwc3dra2RVnFYeTHycV0HgRBXpW6MCYgjHBCqMUI9NqwWKOVYoT6vkfAClkizrcHWwe7e5AvkjAaG4ss9Hq9XqeravH5Z58VVd3r9Xr9HcICQsj23m4URWGre3L4zXy+mExmw+EQYWuR9jzm+34pJPfY7u6u58fzWbpYpHVlp9OnkuOotT2fp4vZIkkSznxjTJEWjrGojLGY0rXSuzVWW2PxuvAHIcS512ol/X6fezhLsygKmhx2WZYICGNMa7sm9azieJ4XJARZi0RtqrIGAOIHnDPGaVrkCMtainmeLrI0zypghAbe408fuTw6YLS9vW2MYb6XZdlsNinrilKa5UvOOeM8yzKw2ItiANBYI7Oy4K7c0RhDKSMEG2OUAnd6WZbN5/O6qN3zkFL6nscIIwSVeS5qhREKwxBZLCqhhKaICV0bjI0xsnb6B9Rjvsfqgxv7PvfiOOYeVcK1O/KMMVpXTtUuTRd5nrugKEKoqqpOpxWGoVOwccsghBAGatfEAlcI7oiizTLFrIe1lhBizIp84L75Tt7AMR+bLZ0JcLbSCOGokY6m0LBQG5RgVzTMVWuowWDg2EAY48ViIYSIosgPuONAuJiK2eBXGl0xarUVZbVkzNs/2NNau0cQhuHxq7PRaDSbLimlykCWzaVYmXJMCQFmMQFrjbWEUGuQthYQBm20klJbay31ANYx3sblr1+zSxPvQIHzjk2aYBMoNCvjTaDwuo/fRAB4XRW5aYJds+lms81zWDmejRlgrQBtNtpNoXXdHLzWsHjTysNrKAFeyy9s7nJtvL77GpFcNrOADfBhjCnLcjabjUYjJ4tiX6vxs6+hqM1zQ68F/K/dpV++77Vd3gg43B5onSNwgkvu3jY5CG89HG4AuxkRcekHCwD2KoejuUbjcjRvJja+4WwuQ0Hrc2tw6tV9V28p4HWLKevaRJlVkAMZWHEg3IX/p8onbL5Fv/wpaACKsAHS7vTee/fDTz/9VKmqFqXUUoMmDGshqqxWWmOM47hFgSCKDFihZK2k0togFkZRLcXh0ct+1/OwTfOlF7QqlelqQQzKF1meVx4P4rBtAYrcdncTqUpjhFLKgGz32p1O5/T0VCmyWCxOT089z3MasGEY5nlOKT85Oanrem9v7969exjjTz/99LPPPiPUuAqvVqvlQO1kMlkul0mSzOdza+14PP7iiy/cLg8ePHjybx8XRVFWhZSSEj8MwyiJoyTe2tkOq/gXX37y8vjF1nant9Vpd6KiKPZ69+KkYwza3r5x5+b9upDji1k77H12+tiPjQISQOBRqgBZQpOkFbWidhzJsvj93//9fDwaHh1GQUAQfvzNN8enpwf37nlJMl3MgyhkQColJTGccY8yZJEBBYCRIQiZSkmELAFEMOKUJVE86Pa2B1uVUZPZnPt+q9PxgtDzoyKvo1a35/IvUbJNvVqK4/Ph6elpEIZegD2PYcTGs7Hv85t3bhCCl8v5YrHs9wZBkGiDsqywhgwGO1LAzKiiRoevRuPxuNPqNpZhhbgRQpRhjzNjQIgVgc5aixBQiqM4cLBua2tLqjLP806rW9e1EIpzXhQFAlfor4y2nHNKOUIIY4oxzqap74VJJ0IdSgARQgBJY6q6zmemrOqae97R0dGLZy8n40Wn02tyjXbl761rI5nnpZMBSJe5H8itMHQSbHaj/w3XHGErlJJSEoqdkxai1tYQsEbrqqop5ZJpI9WqWSKmAICs3drazbPMWpuELYxxxT2CUBiGj5++cD7JxfydMBRCKAiiKCr7/X4QBFmWLRYLhzy63a7neWVZTiYTpVSv1zPGuNYjCCGn71HXdafT9n1fCBFHsbVWSpnnuSMHNFREuw4qbF4jol5T3O8iBA4BOEvUCCshhNx5UsZcjqOqKmfFnD11HAUXLXD7ep4XhuHBYGs8HltrKV0125RSej5rwIFLDDdMharOdnd3lVKj0SSJw/fe/7Cu5ZdffimlXC7Tw8NXrU4nbvWGw2G73Y1jqqq1V0bIGgTaOAqtlMYCEMIwoxghSjglzo4Xq6DchgtpEi6bbmxtoFfv87XxutNtfjbrshS7wY5sLHsDI2BFE7nSLfDabNfcMFz14rARwIA3Le/QBtnt2sybF9t8jhAyq2Wo84BrMiPaLKpbcRHWuRm2DjM4hGasNQBWiCrLlpPJaDodF0XmaMvWkvks2zxD/FqByRvvKlz1/a//9fV9G3T1y70aWqeTzFrByX3oxB6aX+26tRVldONm2oYMYjbybrD5mFZhg80ntQJYGF/iTrIeaJ0/cqMR6nBGAyGE8JXSG3tJQbgkMDomKiC7WQrhuAv/WfQUfjlWcEWdUiuldBgl88VSyLLbTYSUgIH5HtLa1BWipNVuU8oBK8KotbZWsq7rWmnfwDxfAPh5WQRh6BGxyGa+D4vFbPr8cRBEUiqjcSsMPYaUrOqi/Hw8poTfuXdnMpmOpxMp5Z27d6Mk0pIul8tVz2GtW612HMfn5+ec+++8887BwQHGOEuLZ8+eHR0dKaUQBs/zqqra29vrdDrdbreqKrfU3N3d3dnZmc/nx8fH3//+9y8uLjqdTqvVevvtd2/dvhtF0en5cDwev3z58vjk7Ld+97eiJPKjcHtn6wc/+u5b797rdOPFYv7056dv3X6wmC3fefjOWw/eOXx69NlPPlESAmBMIjBAlNFGKqV4GLWSIGm3Hn31VcQZIeT8/PzzTz/9B7/1m7/2/e/9q3/1rw4PD6N2B/L87OzMMkY5o2CpTzzmc0KtMtoCxkAACAGkaFZldV37jPqMc0LLohiene+2k9PT8/39/X5vy/eDRZoPx5P79x+ORqPh2QWdTMO45XleLUxRKz+iHveLPKeM7O0ecI9IocfLkVJqf3/fsRqHw3mWVR9/54e/+zv/4Ozs4t//4otlrk9PZ0pZz7PD4QVGqN1u07okntdxd7YqNcbYaCuEcExGSonvt3zfT1qR53lpmhmjOQuHw7Fr9Icxrusqy7I4jp2z6XZb3W63iViqO+FkMuYERSETRU4wdKJICQDjEUJ0DR4JOYnnsyKOO7fv3KPRgFKa53lVVXEcA8G1UlEUOxjOOd/fu+l5XpIkVVUdHx9jKwlBAGCNELVepaMppZSq2gn5E0KIFshaTElAuTXGFrqI2+1W0nFNGYqyToXgcVzX8nQx+/DDD7e2toYXo3v37m0fHPz3//1/b6394IMPrLXOQRZPnx+fnRFCTi9GVVUVRaGUVUZ4XhD3WlLKWTELuzFCSBsTdcIwDIPQX2bLNE3TNOWUVXVR1UWe51EUL5aLuq4ppX7oKSN1pQCBRYZ51Cee53lN/psQwlnokAFjlCEotQKC/TBQSglllFYMIc6oRwkoWUkRhN7FaOj7ftJuaa0Jo9u7ewBQlqWqpDHWqdxZo7QhFpjIy9gLHLG8FSWhFygtq1JEUSSVAk0w93jok8DThNTWYmCLohZ17fl+lmVltijLEmTBUD1fZkkUqDrL8tIPaFYsrLU86ap11MQiAgQsArNOCde2BiUaywsYEWzBydxaMErpNYvt0viuu/ZiC0hbxK60bGhwJ8Z4RRpYF5radRHvJjhoDGuTrdj0KBhjQq1jgBmzSXxzLNRGVsiAXYc0ELIGW0CAVr4GXab5V3xDY4yDggihpuYIALQ2TVKcUgrW4pX4wmXEQkvn5wAAYbiEKUavlRUaYqYLy3tMa+2aWVsLShm3KJdSW4uMgbqWQii3YHBU3NdhzTUQABuO1sHcTa/vhqtgbHa5Ntvmr02av5l58wdDjDIKG0wwI4wgimpVp2kat9pGWQyEIoItwnp1K6QRzbPDFhCA1cYCULeON1avRJSbmA0GbAFjxBBBmCBiDTLGGDBAMbIYA7EACmkAZIytkSWE0FWmB2NtFGhsLCKYkDV/QRmLwWIghJBG7Mu6qIbj5+KVqVqFMghy9S8IGbhsjw5rtAcrrsObvb4Di05ao/m8yVlaN9X6nhPAWgOiIEvpc+/dB+8e7Nx+8uRRzXQr7k/nEy8KIp/LwviRL0tZgvT9ACmCgRHDWgHLp/PdTisKw3I6pHLXZGONyxDX5XSqlq9u3tg5H052dm68ODw7Hc/1KKsl0gpHYbK71X55MXv2zWOk1e2D/ZOXR2HgWyUXk8lHb98NovjsbEQwv33r7nfe+/hv/ubni2lxdvwLKWVZ5i9eHtZ1fefOna3dzvb21s2bNwPPr+uaUNRqx0HInj9/Tgkaj08Yo//4H//9s7MXnsdGo5f9Lm+3UZSon/78Dxd58dNPfvH1qycGobd+7XYv2vrB7/5IayOE+vlnLymlDx8+/O6v3yKAXr08Oti/9eTFs5eHL5bZ3Ci7u9vuD3YtIpPFcp6mi0wQVno4m01UkiRGisOTs7IQNQ2HFSxR9NFv/97U0mEpfEoUBotku5WMXp2+tf3BdDp97/sfHh4dvjg8pD7d2dkqioxQq6vapxxEFfnBoBNYWaaZ2vP5/vZ2L47b7faLFy8+++yzOI41yFar5cX8Bz/4wXg85py32j4lOs9zWWMMoTW4KOuiwhcXF4s0/cEPvt9qbYdRdHz61wZbP04efPABiYNX47P5aD4ej+uy5pyDgU6rp7WKwoQmSQQAUkopBQAQirXR1mrGfIxxGIb9fr/TbTnefp7nUoqqqhoVGgBgjLVarUbqx7HqXLi7LMtOv6WFsdxWeWW04YxJqZM4phj3ut0ojKfTaVUXD9+6W1dyvhinad3tdsMwdO/3SjXImKIoACCOY8f4W/cdttPp1JlR3/e9gFKKjTFGGyezWEvpfMBKgZjgKAgRQkncjuPYtZszBqSUw+HQiSsQQjAivW6/lbT39/dbUWd3/xfj8Xh3/873vve9drv99OnTyTxX42FVVWVZaK2BUIRwKcQiL3jEEULcY07B0PWcdIRNY0iz4HCAQ2t9fn7etJl2q2SyFn91SyKHThwJw/M8rZ2ypwEEmAChCGswRlvQmIDnM2stwtaCBoQoJcvl0veUMaYpc3COw6WNOOdJkhDCXNMKQojDf46HUYkakPU8L4giqWpXekAUMoaDsciC1YYHoTS2rIQ1aj6ZOoSRLdOiqKQy2tRVLUshPD80gIuykJA312XsJSFxtepFqCEkOqAAVl0auzVD0BjrODGUrtLSzqMTTFwo2Gx0OFyt5Da8XcPngDWGuLSz68XitdX8hju8PJsN+3yFsb9puJtoxDXn2nhQ2CikRAgZo9945q9HRK4d6I1Hef1wTTLr2j1xECoIgjAMXb+xFTohl9mH5ribAYDNc7jm8u1ViuJ/ymgmaXBbc8cc491V3qO1bEkTlmtiPO59a6YysIo9YHQFuxht1gcCRBByr9OqURQYWGUDXV9cjDFgcFm/NSxd6S4Igpqgw7WazKsXZa21b/jbeov/lIjCtUf8xqE1aKMIUM4585jv+2EYUoqHw6GX+dwnZVnKPA0CLwjD4XBIGFVKIqSllEAxpUwDiKr2GfT7/SRp52V5MT2Rxagd+xSj6SLTBs7Hk1JKMERIJS3hLIw7nRu3b3LK6rr+6V/++Hw0/Oi9d+/du4OUmC6WZa1u3w53dncnk/mf/tl/ePz4KWOe1FpKTQgKgqCdtPbe3rl//34QcWPMZDR2XaSllE4/uSzz/f39waBnjamqOk1TYyJrbZx0MPXyov7pzz49fHX09MUhUOoHASd0MZkus8xahC1MJlNCSOLR7XfeHU9nx0fPnz3+Cikghr777v3FbHH3zkNMWJrnN+/cqJT+65/+7PjsfGdnsLt74/jlURLHcRAPT8/iwCecf/7VVz/5yV9lZaWMLucTYSwlLCtKP4yW8wVjrNtqdT76ThyHw+GwzgqP0UqK2A85xoDwnZu33n/7baO0EqLX6xFCLi4uJpNJWZaDwcD3/aqqXKhbKZWmKQDMZrOTkxMAUCrb2dnJskwpVdTVxcX5YHvLtRd49eoVAPzwhz989913b9298/XXX//85794/uICAIIgSJLE9z1jNMYoCD3q+RwhhDAUZeq0gZMk6fU6eZ4TgjyfRXGAEJrNZi5jnSQxADjqvrPIhBDf913TRaccTNYtfSmlZVbLyvgEp2mGQEuUn2fp7VsHNw/2KcWM4/HkIssX7XZ7OD4fDofSdjqddhRFAI0pV3WtsiyVUmqtCMF1XcHKZIDnec0iBq95f8hoY1Rdl7USQRAQirUxCAzjNIoiz/OcWSGEGAPOB+/v33QuuSwqFyTgnI9Goyja+83f/i+LohgMeuej5TdPDqfTadLt37r34OLi/OjoaDIZaa09P4i7bcYY0rUzD446oIR0bTP73S7GjYgbbsru67pqGAnOxrmiR7xmwqO1jIzz63Utm41hnQuwjmy1TljARhm9UkoR1agkmRUlc7VodhDkMt1jLcaw4k5aLeoKERxFQSsO56nyOEVgKKEYAFltlDAYCmOUUqKufMYXaV6rE2yBMaYBEca1RQYpC9gAJoxpU9Z1jS7T/6sKdYtgtcZ17+JGsHTTMeBLfgBuHjdZl4a6vZxE4TVz6X7YDDA0CZoGZ8DahZANTUM33INYu731p1eBgotRv9Epvm6mnT/b9HbNzxi/QT6hOcYbGYXNJ7ABFzYQ2GWpp3sfrm3vxmayv7mlGGPzLT2bmhmuIYZrWGETkfwdxrfdz+Y5ojWZ0TWzRet8gUM5GGOELgMGdqNgxKhVpIHg5s5jAEDGYosJEOTahGKwFrC1hDGniqStAQvImBXXmK6A1CqUuRZnbHrHW4spJs38mzf/EgRcRX6/Aqfzbx/oMnD1SzfDgJt8h7WMsV6vkyRJGPmz5TSO24hAtswxRbgukySqpQCrEWACFlmLwQJAUVRRN7HIng/Hg7bXTSIjs5PT8yJf3r/zlpeET58dVspiZrTBWaUwKnKpkn7/O+9/8OCD9756/GgxmRiPkzjBVRG2kjwvHj17Wtd6Np0Lobb3dqtSFJNJXdceY4qg9lb/7q3b+7s7v/jy09ls5mjjxqxqynzf397eBcAnJxdFkUVRJJWZzpaz2ewHv/5rUiuEvdk8N5q89fDd9z543yLUjpJXr16NLy7CMEyiiBghSzE9O17s9Yo87XT8k8Ph+HxU5bKdtNvRVp6NW53eznb75u07cacbhPQv/vLHlMqL4cgAkspqY7OqbrVaxxfjFycn4zy9d+/eNJ2Pj17QMKScFbIMW20oNCWglfjRj370zoO7X3/99Vdff5EkSb5cYARFltd5NojbO91+nmaaeaPRCCF0cXEhpXSiy1EUZVmWpqnneWdnZ4eHhxjjLMum02m32z24ub+7u5vm2fn5+dePv8EY3759+/nz53/9s7/xg+DBgwdlWf71X//1n/75nznwYbQihGAE1mijJQJDCeKUUYKV1hoj1WlHvW7SEOyj0A+CIPD8uqzOTk7Pzk4xxr1eL10ssywzxjj9YKWU53lREIKx7k+rDKIx7plVeU0MYoQX2mBra1mPR6PtQZdSPBpdaFMLWShVzRbyxeGjsiyD2FdKaM2NUY7MjJBFyO7t7aRp6vt+GDrlHx3H8WDQK2QmxKotk1SiqktjDGCLCSIUAub1+90w8h2x3w8CRgL3FXI5MBc18f3wnXfe0lpz7juugLHaWnt+ft7uRRazrBTLoxNCiBAVxuTmnQfGqrutZP/27SxLZ/OJW4QBQIsja6020pE9KaWMEc/zXHtGAHAowR29rmuM2DreYNx1ueCH89+ux4ELTqyhAyBktZbNUpgQZC3SWrov+9r9I2u1UiYMQ0Y9t5R0w4XqPc9TUkspsyyzFrmzJYTESUApsRZzxZm3orAYYzxK/FYbAIxVxhhQ2oDUFjIjwWpjDImYH0acEiGEUgYAGUDGGD+IlK0qIX1EEXEn7Ayo1UKteJoIhFDIla9h0izxm24I7qIwoutCR9ysHfG6yZP7FTai1rD2ppvOrHEVDmQ0ObJNmgJc9dCNzXW+Zj3163Z546DrGZpKjeYHdz6b/nvzbBt3sjn5pXt7A5nxEjfA1UUk2oglbF6+263hIWmtXRNzJ6XqAnjN1ELIzVVpc/5aa/RaOAFeQzavX8uvODb3unqBK6dujXVAwZ2t02cUQmBCG6Bj1+Se1bguNeH+vSSKIoTMWglDaQsIuS+ARdZzOssIACxeqS2viCHNaG5RE9Oy6068zc943QV0fQLN+Vy/b+5u/h1u3X8UMnMiUQBgrAJEOt3222+/nReL58+fVVVV12WURJ1uS1sVhuE//If/8H/4H/87rSz3SOBzqSwAgLHu5hfpIp8Pqa1/4wfvRlH35Ojl8GJ8//7Hg+29n3/2ZFlUXojCVqfWYrEc14AOT48fvPv2nXfe/sHf+3vHr16F3c7FfHH29JtWq8U8vy7qyXy2TPOdrd133313Pl08fvxYVFUQeEWWz6eT50+fTMbnZ6PhZDKZTCYY41arZcAUWWEtVKVk1Pe9cL5Ij755+sWXj9rtdqvV+v/8j/8zpfT23TtnZxMeeHduP/iNX/9tZRXn/PT4uC7LVhj0WknMWV3XrSR5+eKREKIdJ4Pt9vMn3wxPh/Tm3ft3H9y/93A2T0fT2dn50QDkwwe3Ti9e/skf/1nv49/96KOPz05enZ+etXtd4vmffv1lHMeDnZ2w36mw5VFssNEIEGGYe0yJssyfPXn0D3/3tz545/vtMFiMLyI/OMmyOAi7PJRxst3pybxeTKbtOOltb2GMEQJXj9ZqJXt7u+t1lzk/P3/16igMwzAMB4P+nTt3xpPl6enxbDEfj8dB6DHqnZycrGhznW4YRv/uj//9dDoN4xgAsjQPw0hrLURdlhogkLLWWs/mQGVdLJdLrfXW1tb+/j7nfDgcnp8Pb968KaUssnQyGo5Go+Vy2e/34zCsqopiDE7ZHCG9ViRYzufHR0dZlrk0RFEUURRFQUDAUkaJNSHzCDK1lb1OezDocYqtllprhNxKzASB3+p2uNfxAypVWYsckMcRBqQAqSj2lK48jzKOpFRGCUCMMm5q7fCE1nqZLxeLhVKKchJFESGW+TwIie+71jCWcVSkaeMyMaa+71vggNTLo2eMsW6nzxgTQgtR+D4XshyNz5fpcr4YYYxv3Lixd6PPGIvj8PHjx5QS3/cwsQhbSmmep0qpqsoBQGnh1HDxuuPzcrnEawlYs/5q1XVdlalz29Zax9N2/aKaFSdes+Wb3K0zBE1wGK8FrR3CwBg7qRDnAChdJeMb5hdj1K0aXQvNJhnhPhSqVkYihBCB0A+0kVqJ5WKGMSRJEoahUirLUiGEW7UxRq3FSgqrJAbr+UzLuihySqmUuqhEt9tDVmfLBe1RRomQl6H4DUu6NpcbRAFnv9wysXE/679atIqii+bMXcCpsazX1rjNWtOugzFNnOA11/vLht2AA1c/tnZdMb9pstFr4YHN62p+bTIUGF/qdm+eW8NXfe0QV9zeG4/1OhZpXgkAcBWzDjLOZjMnGNoAgl/lnlw77us+/u88mt1f/8FdRVMJqdaj+XY7VgTBGl2+AZd5ffctgzWVr7lS7dig1liLsDEKAFuLsBVKroQXV2gAXEcGvD6r1SNDeP0cr4DCBrIQuiFItUYa8G18w79D6gEh2JjqKih5w1iDQkMpBgydTuvevTsXw5PDw2c3buz6oWeQioJgvlwYq956+0Ho8WWdWc08zqxVYFZkoDTPOaBaVC8OX7Vj7lNVVHpr+2AyKW/d6fYH+4ujV8us5Emf+YFO80LUry4uDk9OP3jn/bc/+vjG7dsM8HK59IkNPB8AIyDv+SHFjBMeRUk2SJVSBNm9nd3xePjq5YsiWypRzsezLM3KrIzjuNfuEc7quvZ8n3O/KCpjLaO+F8Sj0WSRXnjePO60vcCXwrZaHYOgqKrj0zML5u7d24s0nU6nlOIwDJE1SsipUmGHB4Ff1TUlZL5cMo9P5hP/zL//4K2f/vynX3z1dSXtW++8/U//2f/2X/yLfxFE0b/59ChPF8v5Ik2XhBDMMPV40Aqny9nwszGiGHOcZ4XF1gv8qiqYNXmVHh8fTqaje3dvc4oogKlqXdaEB9v9XpIkW73+bDabnI9bdxNApqorbaRb/Z5fnCJsO52OI9pThnf3tuM4ppSenJwMR+eelxRFBmA8j7317vdHo9G//+M//e53vxsEgWuXBRYHfhTHrbquAaisc0IIp0CQBaPAKIqtRwkdjc/c2mI0VnmxCIM4DMNbt25oLebz2WQycUoAjHkY47Is2+22q8t3i8KGAF9Vlcs7mHVpvvOUPkcYkFZVEvlKlmVebw/6PuNVVbVaLUqpEFLUsr81ePvdDwghXrjDGFsul0LWjFMLxliNMBRlLpWgjGijalGVZSmVSLNlUVe+7ydJ4nmsVrVbbWNjLUjuUeYRqYRMKyGEMQrnpMoVWpOTCUUIS6lyW8qXR6MoitJ0PhgMqqoqq6WQ+TKdjucnSZL0Bp7WuqzH1WjighB5MV0sFrPZzCE7SqkrEJienhJCMFmv7CldRQ7WHhpjjAhYWPH2lVwZu2YB2gR1nL1zfX4BYN36QSMElBIpXc26gw44ilYkRyeKRYhTvtJlWRKsGvafS+h6njefz9FlF+XL4pesSNEqqo8AA7JICKG1RBY8xsPAxwgoIMCIU+JRQgjS2mgtjABV55pZAjrgpN1uFUWVp0tdl8Tq0CPbvXZVibPxfDPs794iu65uB4w3GOUEMKJkZYWVUmCbcH3TtOkyWeNuF9lYzG2uHV3MBl4LyzfFIO6ThkTy+sbrf99scDeBwqbFbwI/m3gFIeQc1ab/s6vEB2pAYXMVzSebmGB9FLu5e3Nd9rXQyOa+ze1CCLn31nWULsuyEdXA69qZa1jq9Wt8ff5f4pl+9fFGlGDM6rYQggkhnu+7SkgXcHZiJGStNmaMsUa52+4IBM1pN1DMXA3JYEIsgLGWAFLWYAvUAFhTiRqDbdw/o3gdObgkT2CMkV13s/RXddGbz3fzscIGULBX81b/WVIP66n+lrlc4BYAOfMihDBWzWZTz/PuPrwNGIbjC2WVE0CUst7b3VrMZ2WeEhYQhA1YSimjntU1C3yfdSilWVpVSChhPI+dn86q9+zbDz/KK3s+nkiplbHagMWkruvj07MoanuYdnrbvXYHA5qddabT6cnxWb7Mep3+wY2bxtgXL14s54vh6Dz0/HYSI6vLIp9PZ57HJrNUKFXmReD53W53e3tXg42i6Mb+/jePnzx79kxq3UoG3EvOz8+n02ln0D8+Ph5PJ4tskbRaw/H4D/7w31ai+pf/8l+GrVZvd7e71adhmOf5ZDmfz+db252//9u/E/qRrsW7739w7/YDbLHPgycvXs6WedIZkKr+xRdfVUAePHw7q+RWuzU+PeEYdre6tZS7e4MHD+9ILf/8L//s9PS8O2hv724BUsZajHSWZ4xyGlDCyXg6PDs/vrg44wQbqR7eu1sWBWijajEdT0aj0Yrw7vtOE/nOnTuMseFw6AStnT8FABfpb7VaW1tblNIgTI6Pjz/8zkePHz29GI9+9KMfCSk///xLwngYxZz7vV5vsUgXi1QI0W51a5G6TtFlWZZScI/t7d24f/8+5Qy3khbnXAqdZ5mWcmvQe/DgwVdffRP4vN2Kfe7hlXRgoKUSVe08mBKrZkiiqvM0y/OcU4bDiCDsnAoYO5tMb/QjB9Du3L2Vp8tjK/Zv7BIMi9m0N+jXlbQGWUu0Qr3etpQy6bSUUpTiIPDcQgHA+D4H4JRi3/c9jxmjMF7F8GHVEEFQ6rs2RYAM95nWKgxDQnFRLKuqsqCRUzvRiHNOGabUAqqFlFIVGOM48aLIs6gytjK2mkxTC7UQYjQ+mi8Y55xRLwgCzwuqUhy+mC+XqRBKSsmoB9jWpVgsFovFYrvTAQBjFULI9/3A8130RkmJEDQmDK17JQR+3Cys4bWFxaZ9WRlobAghfsAtaKVcyyQwxvoBJxQ5AEcoMkYpLYxV1q60mddB+1UUoaoqglcEBUJWIWVjjFOUc4cGa52mLxirjJJ1JUrPWo2s8fm6VB2UttaL/HYrKRYTWWSEkIPtwQcffDBbpLIsyqpgjN/e33v44N7p6ek8rxvD7cTBAcCqVXy7IWNv+HLTDFhngp3XdxDHWusImAAAFm/et837idYlc00WANb+sklebK6hm73QBpPA/aExwpsG2V6NHl+z1JvTbj5f9KYocbP6bLzdCiet/3ptkk2U0GCIzWkb6ADrbo1NqqVBpQ6PmrXqg4MO5lLv+W+5rmsg5vUt/2NxwxtRQnMtxhiDjPNtrnXLJVNHCISQq9CGjcicozQ174PSajXhVaDgM7b6moB12yswKzEGDNaC1RZjrC0QsAasq55osGkDJhpuDQCshROuQ7orra5fS9nAr+Dm3zwQgl/5bq/zdMZaY6WSskYIVVVR1cV0Og1jv9ttz5azra2t3/3d37579+7HH31wenw8HM9C4jNGtcFKGYQQZR5Yyxj3g4Ayn2IkajKbZyEKRmfTh++8XUuzXxQaw3AyLkrZa3fiuFUW9eHhoc+8JEhkqTqdjh8nXUItkBfPnp8PL4IgCv1gNJlgY7vdLta6LnOP8YMb+71WEoZhZ55Za09PTxFhURCHYVjWEgFRCm4e3EaEPXny9PjklFLaG+zs3ri5WJ5PZ0MxlPfferh/8wBzdjI8P7hxh8VR1O21hPDbbRaE/U6vs3tjuVyG1B7cfiBr9dMf/9XTlyfnF8t8mVdFjRH3/OD2/ZtCG/n46adfPPrqyUtASNFgORkighGCTr83GZ2PxibL8047Kkrf8wlGhjOsjERWEWQEEpRj7JFX58fbW30hxIMHD4o029ve+fKLL5az+XK5DMOQebzn9wmjSglKsefFTjhysWCMEc9jSomyzI+ODo+Ojkaj0fvvv//ee+/duXPn7HzMORNCnJ4dv3x18o/+0T8Kwt/78Y9/0up0z87Onj17sbN7o9vtAqaTyaSu6/5WT0pZlHktKs/zOKcYgxAVvXv3NsY4CIJWq8M5twa5Rd729iAMY8ZYXddlUbsCfa11WdaMMYRQ06XQcRgppVtbW2gtzUbWveGtlVHIoyj47scfllka+uze3dvj8fDJkydVJZRSUZRIBXleAWZKGZfsr6rKvcculugavbjvjwtd4DVvMYoiKVfyA5WojFWEIMZYlqVB4BljizIrioJz5nmeBWtBAyIIGwtKSG0tMMYwYVJqIVCWCYyhruvxeGxBcc4xrnyfJAkTtZjN5r4f+l7g+0gr5nEiBKnKernIjQGC0aDXDzxe13WRV3Vde55Xeb5SIs/zbrt9aeOUNnaluoiRK4RbpQaaJYvLLDS6TK5MXGtN6Iq6Za112k1uuHS7u2nGGCGE2yWOEkY998jcE6zrWgjBOXepByklxtrp1SCEWknLrvPWStZuZkqwUggAhKiQBYSQz7nTsQh9z1rbjqPt7e3FZHj8csoZiaP973z4wXQ6ffr4m8OjY4+Q3UH/4Mb2bDxqtVpNEsQJkTkXuDK1xjhnuk4tu5LD1fYEY8aY53mUMicsTSlzoEdrzRjzePDtxhM16zm0ZlPajSpKtBEittY2VTzX/fq329yr6OTSEL+Ro9Ds0sy8OUlzbrAOaze8gc2TgW9xwNem3fS1DSHGjQa5ulhCAyXruna0U9cWqpnql6ckNr3gNSTxn2uQteyHBg0Atq7zPMeIZlkmpRZCUMbDMDQG3BeH0csbstkokq0Fu9DVEBG6HGCtNQiwW/xjvErnoytj84ZsXqzZqKq1YN/4mNy4hmj/swx0VRX7jXh0YxhtNFhLKG21WgcHB7du3VosZ2fD00TED966P5qObG0ODg46nfYH773/4x//ZDieIWwJoUgjo9wixyhrJLJFUV5cyMi3Pqe+53MZDs8nH30UPXz4bnfQFVZ/+egxYPrN40f9vlLSlHk56A4YYkVRhaEadNqAEODZ+cXo5eGh5wXvvfVep9NJF4t+v18sFmmadjvt3d1tDNutVtwdpwghxlielUEQMOYVRVUU1enpabvT9f3A9a3qDrY6nY4U2gtkGHmYkrfff2+yWLw6O5lMp9/94Q9enZ48fvbsmyePnSP76KOPPv7441arpfOMBfGLw8ffPDmMkp7HfEajKhCz2dIPWxfjWVmr3YM7Qad/dHyqjRmfHu/s7Eijgij8r/7xPzo9P39++Gx/f+f49ATwXlFmWbbURiIAo2UU8ryohDFSyhcvX967edsPvbsP7k8uRlpop5EThmGv19va3RFCaGNcysBFpk9PTx89epQkyXQ6jaJob2/v4cOH29vbn332WRiGLgNw+/btJEn+p//59//iL/7yR7/+68Ph8OeffNrtdk/OzrkfTKfTwdaOEMoYYy1SSiul8jzP85RS0u22AeD586dffvk5/af/6Hedt0iX+d27dyshhhfjw8Ojva2dJEmiKNHGFkXlEhDWolorKeV8vszz3CiT5zlY2BvsWGsIRRisVJVSwoJWSgRERjaAvI6i9vnzk7gV3b91P89LTELmt0fjpTIKYxxEYV3Xi+VcKeXp2vM8TFRZlkpjrTUgrTQgzCgza2dpMbaYIMqwVSVgnef5fCm4H2xt9a1FaZZRygCIUlZJQwhxagRVlROMlba10IwRz6ecU2Orul4QgtLSIoROhxdVVRkNWan2OnthkGm1nIxPjbbWkkLRuuAYc2RNwH2PYo94OEl8PwDAdV3nee4HvKrxbJIu57NWq8U9hqxRSnLuO9cbBB4Ykud5obUmzLmlIAgoxnVd16rEGFOflmVJCS1VSSlGDOVFSimNEW+3wtPTUwQkiqLZbAaAhRA89oxpYtTIKkKR7zOaq6wV4rQuD24cZFnBAEuhkzB2wSViLCaoqipAEmNa17VhIWMsz9Jut2sMGw/HURDWda1rxdscWdJJOkmSeIxzzsfjcZ0trFb9vRudINhtd0w/G/Q6iceZEfPR2bv377739lth0kKEnxy9HA9Pi5K76BRjjHIfALIsW6aZK9nX0EgRIGsRshQQIGQBHJCygAzCBpDFhCltla4RQpQxyhgACFX71L800OuchV2HIhpHCACulsSum3Y6c+PQElz1tU24RWtNyGqD1VJ0NVwi6XIhh5B2Qrxa001/gDY0G97obKw1AAYhF5RewUettavJ3/QuxihrrcUWYwwI0KUIMSYIIwTIrpSC3d10nH+CQNaVEMIoRQgBo61W2K2WtarLoiwKWdcEAacEIVRraVeu1LoDw1Vp4UsPZC0CIBtNnjb91jVX2vxw2bLi6vi2yAQyBAEQTCihLtRvpJF1fX561mq1+v1+HEa+xykBvAo5WHs1Z+9+KMtqBd3WHIUVjNN2detc0cO6isEiqxAiFBACiyxohJQxVgNiLv2KrLHWGgxAOWGMrlC7sdYCAowxwYhghAFjwE0rKQtAEMaArCumQC53YZ2KAlhwghCAGjaEca8IJtelG9f3a7UdvsqKJa9vvwrhEa0VIawo0zAKy7J+5+33/u/L/1uRV2GQFEX5/PkhIJym+R/90b8XQk1PRxgiimMCARhW5wXFhFuEwGKtKLE3BrtFNjx9dXbzxtZynPo93u/cUCEKvfBvPv3Z118/euuttxjYfhzf3d8DgEdPvsm2tvKdndPJq9/a/a3R/CzyoyRW+7tRbHd5uXz16Sf5PK2K+ng2J5wEUajqepHOT86O272urvw0Tc/Pz+tK1JVJOu1Xr175vh/GkTFme3urmI0DbOdnR/V81Gq1aBhub7VLsbg4P+QhFuWZleMnX/7k07/+s6++fpqErdF8fviLz+xsZsYXD9+63x20z9Lh7o3u4Ob2T/7qbwaDQdwJ2h1258G2KPLFrDy/mI1OL7Df6na7Z+ejdius0rlHMDMyypZvJYEJWHfQuXj5eDY8Ac4R49IQimhVmDiImW8BQFb1s+OX8Wef/OaPfm1a5bkWVZlLLdrtBBl7f//gxu7uX/75X7Tb7YTEkR99/vnng8GgvzX44MEHL49fBTRKgs5iku3cuDHo76VZPZvNzi+mo/HiOx+3Wu3e++99PLyYddrb1tDd7RsvD4+TVrSYTY0WnY4vRH74/BcHt25WVXn66iSKonbMlBIEysFg0Irh5OSE/sG/+SOEEKW0ruXJ6UWa5ovFYjieYvyN7weeHzolGWstAgIEewGPosj3/TiOkUWLxSLPc8YY5yzLlxZsHMd5np5fjLSW/W5bynq5lFLWi3ROCKmksNYSRo21eZEtlkttDWOM+S6w7wVJgjF21RMYr1yva4CE1ln8xpTrdcdkSilj1n3zKeWAUKsdY4yXeVrXtTbS9z0X6qeEuAWrUkIqxBgBpJSSjJPGnQCSmGBCFaGqqDNrkdbWGmQtYMQRkhgJSn2DFSBKPez7PIpCowFyjXForXVupchzQLaua1VXjBGErFbKBQ8QsrUUAGCgYfg3sW6yubB0X35CKELEWoQx9njg8cAtASnlhBCMKUIoCD0ldZqmUl722SIxyvO8zKunRXXv3gMALJCQSmCMOWWEYruucXLWfDIcuX5RboMkjnzfZwSTKNra2go9v91uh37gEgGMYkQ9UcFksbQIiOdv7e13kjgIgq8ePX15fDqdLSzCloy1JZPp9ORirL2+e2qEEMIqhFBVVU37iSb1AG5NBpecPnjTQK9VnV1b1G56/cYZ4A1+KKwXc01u28ELvEGrvGperzie5kC/ZDH9uuPc/Bxe86n4NeHkhjABrzldBw5X69/mDrhqfrjSDhFdZU02d8bdBydU2qQ8rqGl/9XHJsK4drsc4HO9ylzN8+YrYe0qmYK+nV9y7UCbY33oN3AD3XBRPWOMY0oaDBhjgoCt4kerM2xIQm/kvsDmO/MaDxHWv/8n3cFfNi51urTWAIZSVpalBu16a4VhCBg9e/bs4uIi8qOLydgQJK1BWlBGCJCyLJGRN3Y6v/6DDz/+8K10en788rES2QTBBx+83263n3z5xfn5+eHLV4yxkNP3H9yfX5x/9eknQRD4lDBQUOfTxeLf/A//7zs3w06rjY2NYxze6gTEq9JSLufJIEK+L2pdqJwRP+n3yXxxMV3+ve+9fffu3cePnwZR1Gp1zs7O+oPW0+fPHj/5klLKAuAeBGFQFJZ52I+w9eu8mrx4+VSa8s79W+2Otyzg7OxZK+n97/+bf4oQsrq+OH95cvz06fOXf/PJ+Nd+41+8+977ezcOfvDD70wmo7PzkzhEQlT9rf5iki8WeVlP0jyntjLEw6ySRSXqsjvY1lU1n4zjdlKWuR2PkjAIAt9QAoQgBGCsNkqKQlDpc8+R8Uej0fPnz7NuX1Z1N4x7g0HIvcV4+uXXX00mE+57zu0+e/EcU5KXRfby5WQ+6/V6u3t7cRwXRVFV1XK5rKqqERfZ29mtRL2/v08I+cM/+ANC6A9+9MNur33j4EBpWdaF1rKsC8qplMKC3rvZbbfbQRAghFyivygYZoIu0xohFIZEKchScXE+yavKGlpWoigVxjklnHqcMc4YwwZLLVy9AMaYESalZIyFYWidvyc4DANCkLEqCLz9vZ3Tx8/SPMurXGjBfb+u61oKi5Hv+1VdA0YBD3jgu3SjlFJnqbW2KAqtdRAE7vtljKkq4aLrZK1H1LQ5cKR3Qoy1VilFCKOU+gE3xuAyh7U1dF+8OGoZq4SoalFLCZgYQoBQq1QFyIIz0yARRhYVBkoha2sd5QlZi8BKhChYzIxEQKwlBDMPsAVSS5EV85PjMUXYWqu0NCBFLayTcCFAGcaY48vYLxhjNJKbZesbP1OMqQsHAWAA7KS1dVn2e1vW2tlskaa5011mjLmXA5B1SaIg8BDy8hwxQsFoz2Pz+RyMkkIjhAEQQcgywjmnGCFrtJHWWowhDiPOuBAiTzMEIKUMPd/n7O2HD1utlqhqSinBlmDkc8/0O9OF8nxSVnI8XwLxgMH5PBPDaVU9ny8XtdSAiNTGIiKlKqRGSDSFeZhyQohSqqpW4gqIXqonIYTA6oa89vq6EG0oZ9h1Xn/TxTYworHFeKPDVpPr2VzT2w1ORHOsTQhirko5XbWwlx/CRlkHXDX6r8+zmc9udA7sumZPb0gkNSinObRUq3wBfv0oV2dujtXcHLTmbVzTP3abrT+50nn5f7GxiWau+VS4wv1cdZpu+js4bjVcfus3pZo3ovHfclz0puE4KJvgqUGTWq9fZicCAi7Bh/M8J4R4jDBGVrkHgxFCTiLMrsuX3kDKWV8gWqUPvvUGAcCmVJPbEcElpNhEouhbJnI4BgAY88CiIIislTdv3vyLv/hzwhGjbDFP9WJOGKacW5uOp8vRYmEtMLBaa6ssYZgRHBCfIVwusy9//mkn4b/xgx8ZWT5+/HU1n+o8PT09LesqWy673S6u8x/96Ef1cvrHf/zHTKJ+Z5BNR8P52BgzyrKjJ9M4iXyKtKgo2F4n3u4N9h6ECKCHugiHZWV7vRtB2At6O5PZMumGN27taKSms8WLo6fD4XB7a1ca2el3er3OnXu3prOx7/NuPwYMSot4m8aKe/GNKAn3bx0ss9QLQApNKf3ud+6U1WIxO94ahAcHHe5rUXcPj44fvv2AcXjn3XtPnu4en31jEZI6O7moiDWDXdLq3ywVEL9bSHR2Pj56fHF+mnLPJq1Of9Da3t4Wuqytfn72UiuhLQFrEGBGmSUGI+eWADNKKQWMFmmqhKyyfOGFnThRQlaiXkxn5+fn77z19off+ai31fvs6y9b7dbz58+n02mv33/w8OGdO3eePXtWK+n6vjJCkySJ4zgIgucvng7623Vd+r6vrPrZz/5msD24eeeWEKKo8iDya1khiu+9da/dbYWhnxavOCeUOp9VIc92k7C/d4922n2lTBAEUmvCfKmxNcTzwyimdV3XSmFKCeZgsdIWGUOZzfM8TXMhROiH7vvpM66U5B6nGNV1zRi7f/9+p9OKQ398dEhrwJj0t/tbW1tFWZ4NLxaLhVQKERx7cRBHnucJqZxsc3ergxAOQo8QEoahlJJ7FGN8cnJiASxoQHjd8m6V6bdrCrfRpixLKV1FfODMR6vVErJaycggQwhB1q6XHRohSyhGiCitjNGwEkCV1oLShVSpAQ0IgAAYF5K0xhhAWGprLVLKYEQQVRqVi8ViNBpdDJdOQIJzTn0n3Et9L3RkSISQkYYQUpZlLYSUGlHEGDOGbjotYwxGhBIGFtWVUFJrDnlWlWW5UNXe3r4xMJvNlFL9/hZjXhh6ztZLKcFdEQbGabvTYgEyBm7cuFHkVZ7nRV5FQez7PsZEVLVGgnpe4DFjiNZSC5J0YxeNMFJprRPfj8PAY2Sr3/M8b6kVaClVDdZyEhFsKOVBEqyNGqpLcXwxm81mlHl5VWFMAKFKKIwNpgTzSG84YKOUXbekMsawNZGyWUO7vMO1Fd4maGgcZ4MFGwe86VZhw/81DtiNpvyyWUzbjYFfaxK9mXveNLibXha+xattmm9zhex2OWEjddxgF3cOUsrmWr6NKLA527VzQ+v708Cp5sTca+P6mqIN4a812/F/HaDQjG/zo81fXUDxdfLH+mW4Mpv9Fdblmw+xAQrXdm+APkDTFuRKJKYWNaWUIIYxGEy01ta4di22oeIihJoMAl63w4Zr9ALXn6JhO14Nm21ezAo0XFWk3vjhzUDBlSY7FXwA7IchIPveux/s7OzMljNtVJ4Xpaja3VbSDra3ty/SXE2mRVqAxUpoXYtulLRaETbKQyhg9PFXX718/vXnD27funmj323fPtj1PK8d+Fu7g6Ojo08++eTf/H//u8SnD2/tftHyl8slrn1cZ0LUQRC0OVpqrEQpATCRUuXnk/PZ8rnneRizwOvs7t0PWzsCi/Hw9Hw65iyiLLgYToO4/fyTL7766pted3A+Xnh+q6xUf3uvv7W1zBZe7O/v73o+m0xGf/X57wdBsLUzGGxH5xdHf/mXfzWeyHv3dh/ef3Dy6tnNm/1hPTF2vLfPT44fBRH0+p2iWC6Wo8DGlMsgUhpmiGYGVRhp4nlJFHWDdm9314t6aVb/fvbnk/FpWS1+84cff+/7HwHA7s29eZH99ZefYgwGWW00gMaUEGoRNoSxWitkbOj5xOPCqOXF+ej0fGew1XnrneF4rJRsdzoX5+de4H/w0UdfP/qKcR7FsTaGh8H3fviDnRt7lRRCq0bANwxDrfXZyWlZlr/2G78ehJ6sRV2XGGMhq06n9ff//u/8z//m96Wu+km3kEUtyrgdZ+WSB4QEthSpqpV7CXnFW61WHMdUGVxLaZEAwMZgbcA4iV+E9DoUpq1ytDmMMRbaWuvIhg7LK6Xm87nncQtaY0QZYoxorZfLZbqYaaRa3Van09nZ2eK+V8oKIWsRcJ+ZSldS6MxWtUAIYUZjrxXHK0fl3u+yLJ1IokseO7sGG96iKaG21iK86gEtlVJaxHFMOQ3DENdACGaMccLKKncllNYaC0opY8EC0oRYQGtVV7zC8sZc6vuuGreABQwAmjHmUubWWGNLqYxUmVSZH2GMgXCDsLTaaFCOOwnYGtBamKoSCKGqElUljDEYMCHEpULWZgmDdT0qGMZEKdDaUAoIEaOxkRohEsctY6AoqlZLutaRSZKskxHYWlsUGUKIMRIEXpZl9+7d293e+7d/8EecEWOVEtLzcFUVskYEWUoxo1QpKyhJ54sgCKIgVICZj7rdbiuJgiBYLmZJFCArGSNW2zwvUlWnabrMSi0iaXRZ1tpAlpezNM+F6SQB1ivNJWkl0hYZXQuJ0IZ3R5c+3pHs8KWukbHWIoxc8HZzNIu5TctuX6sFuDaaORvTubmx84tKqYZ1aNZaC+h6duP64nLTHNt1W6lLyHI1K3HFym+s7JvzaSozNzfecHtXcMwVCLXpGFx9jd248KYNqb2cpLlwd/PRRi8lu6Z5/q+VfLiGDzb/tHn3mn8bEYUmULR5e9+A7P6u19XcOqcr6vrTAgDCDomugkCbJNbNt8UF/ByyccFS94AYXl2pC6Ft3oQVSrjq+N8AFDb+v7nNLx9KGmOVMURUEnuACQKC7969e//+g7/8yY9rWQRBECaRF/A0za0dhVs7vd7AqlG6zFUtumHIKRFFYapibkU7+uA3fu2HSUjm44svvvhqb29ntlz0+30Ac8u78e7bbxVZ+s2jr/NssX9j6+bezs/PXmVzvL29XdVFURTImH57kBfLqszCEPlRoIyp67IsxPZWL6umCnb6Ow8A2PPDp69Ojm7s3SR8f7aUoi6qCvf7N3f39oqiODs7W+TlcDzdWW4ZYhGxi2xiM7lczm/f3eKc93odxu1scb7IJPeg220PtrcWWX5A9vrbN14ejctxuswhLUEKcfjy+WCnN0kvltlFq0OrfNxKEJC5BQU4BGJqqSczbJfTxbIsVIYo1Cp/672H23s7P//5Jzdu3gLK4lar1enW1pZSSCk1thoBWGUpN1IZo2sl5+lSK1VnRZqlt27eLETNPN5tt2Ut7ob33nn/vdF8+skXn5VKFLK+9/bDTqfzne9+/OWXX+Z5nmWZxzhCyPf9drtd1/VyucyyrMjyNE27vXYQBGWZv//+u7fu3nr0/PH7H76XFtmtu/uLbDGZjTyPZUVqsJIW1dZqAMoYwjiXMh2N8GRCi6JSWhPKMQZCaRAEiBJCSFEUymjKqOdxbY00NcKI+7wuRQPe3ZJda+1Ue9NsEXi8P9jxPDabT8oyDzyGkN0/2N3Z21XSnF6cD4ejsqoYI9pIDRYRDBgZBJSSwPeDIKjrDCFXu6+llOPxOM/zIAiCILDWGqMdUFh9zRBGsOrhyxhBK1E2hDCu6oJSSgx15gPAGmMoJVpLSjGhiAEx1jYrKIyJNY5ERCklAECpR0mACVtbZWMB2abzHjKUM0yR1hZRi5nxI9pSYRgwhJAxIIRAVjOMCEIaNKfYWFVUdZrmyIJrB+N5flXnzrIIIQCwq2l27XlcDsUaZAEY9aOwhYDKwjDKOeeMcowIXitYUErrupRSrHs6SADEuedkONtJvL098HzmcZ5nRS1KTrESlQIIA8aoj5ArpmD5eAFKg5LL5dJjvNNOAo/v9Hvj8chqraoCKDZKWy083yeJL4TIl7PZYjlfpFklCeXEC8K4JRWUYhXN1soJSqLm2TUu325EAsy6CtT5KgDAqxY2VwsUN0Kpm47cbYDXxev2aofoa59sbnntrw2DAdaqPnZjGbc23fbazK/Pg9Yn9LrV3oQO13wh2tDQtBuLe9fipNlXXxVGtNbCxtVpC5RStIEzmhQG4MvVNmwABReRWn9ZoFmm17X8W53N/z/GJhBshrWrXkdoI0nkKjldmY+7CrruuXoN2P2K401AcL3EvxpRQAhZu9JpMAgIIWCxtdYoxAgGAIkBIQvUkss+k5dy6dZatE6g4CaghS/fZ2utddoP64jCJh79tpN/4w/ftj3nWGsOAAgRKbVHOIDd2dk72L+F8U8IYZx7taqzrCjqijHmVerm3s3tVv/l8xfFcrG/s9dP4tl4mPS6t2/u3r59+97dg9/8zR8Rol+9evnNN1+//dY9zvnh4eEvvvz89u3bvZ0Bfkp+8dkn3/3+Rx9996Pj86Pz8/NKFel81RivPPUYx0HolpoFoLrXiwZb/boop9MpJS/bcT/wu8XyOJ8fZoFstX+nLKs/+dP/gDHdu3mAEGr32r/48hPG4WL8aj9vJT0WxPLo5JvZckwIvPN+hxBj8SQtVdwWv/abe0oTQFqiwvfj4Sy9devhvKiPXz16+P5bWTr/t7//Zbffe+udB6XKF8sR84yS0gtomS5DjoIo9n2eVXaxmI1mF6dn4/NJRkIIwxgxejYZffLll0fjySRNF1lZSK0plggkAoqxJRgwVlozznzfB2PzssAWep32/t4NiunJ+dnN3Rvb/cHx4ct3Pnjvnfff+/Ff/GVeV8ro6XLx8P79brd7cnb613/z0ySKXbVg6Af7+/sunr29ve37/mw2GwwGZVlShr/38UcWmT/+k3+ntfyv/uk/mS3H3UF3vpyenB2lRToaDwnDeZZIiytREU045wiHxmqlDV1mC9/3HZVPGim0qKpCalWWJaHIQz7W1mm7Okq4U30ipHSF1275xRhvLK/Wuq6N03rzOaWctDpJkkTnw/FiMZNaBqFPKMvL2vd95vmYUmvBGIMwBbQqhnYURRdFsdY6lHBtIeUoQlYbVytiLTIr2VPCPY9QxBgDjBhjhLrvmq2qCpDChBNCCQELmBCMiQEwWmuNCMaYYOoE3JD1wTJGfWdkARljTaPNr7Wl7hAgrdUAJgg8QjoEhxhRIUSaZkpozkKEiKqNtYgwxphyvh+QBWQopaZUxihnqR3+QbBqV88op4QCIEcK8bwAAGtsHFuCMa/VarWStlTCLVMIIb7vE+KCsdgYY0FbY7a2tqIw/Oarr5fzBQAWRd1ut4UQyIJrIYEQgDaEQOB5nVabcxrHsawFJSj0PY8xj7Fet42tkSWA1QTbOPJ3d3cDz4v8k6cvDonVvseENl4QBklSVGqZ5S4s5F4JjABjwiipaq3XGkRmQwDnde/ltsEb49J6XiUN2I1MxObq0V4lLlzboNmmmXyTN7B5iE27fN09v5ZZgDVWWOEE9GaOwqY3QhsEPRfYcBoGdsMtbTrOK/yJpjfEGmkhhIzrjGEvD3fNPaxe6XUuxlrrHO21YIbe1D/+X3bgb+GoGrhES45IsaoiM5dj01a4Bbp5jcz4bQmVjcNd+fjaiwHXFRrW/aNdJgKBRkAIsZoawywzlFKC3eKKbL57zdvg9C0QQu7UNpkov+w8Nz9x/7ss4F3DUzfHtyMll/3gnBslACHQJkna3/3ud3/81z8ejod5tZwtZ34YYIzjuIUN6oSt1vb+IG6fHx15BHdbrZ1OstVtvf/e2w8fPtja7ubZnHvkh7cOdm8d7O5tpWm6rKqvv/7yvU735p3b0milRaX0zs39e289nGfpeDaRdc05m6eVxnwQd/qDrs9Rlk+ybDydqKqYt2LfZzGxaHh21IrS7Q4hD/tbg3bS42Kc5fWUc594gzzPiYHeLq/FkjBhyEhhQD6N+0vWln5AlTpSBhFECCf9Xb7N46KA04vl0eib3e27qGI3WLhz+4OzcfXFo9nRq9PFou5vDSaTSWWyPM8Qkf1BW5aTre0WKGmMyvN8meNCBGBp4CcC5izyW73uJ199XlTSYPL1o2dfvXg2LapxmnqdtkVUYdCEGWo0IAMKEMKMYgtYWyA4TOLtwfb49FxLARiFcXTrzp1uv3d8dnp6fiZBGWN8EkitR5PJ4eEhpXT/5oGW6sWLFwasEGI4HLoYPMNkd3cnSeLHT59Mx5Pb9+/9/BefvDw++j//X/8v73749ovjw0oUoQm80BvOz2fZNM/zqHUXIW5MXtdSCOp7IcbYGE05561Wq9frlKKmDBuralGWdcUYoxQzThC2UtV5lXvWC03Y9jpJksRxK8syq+1KM1XIKArDMASjT09PlRKUYUpxURSxb41RZZkXZUYp7fe7jPtSKuYFFmEALLWu61pq5QP2fR/AKCXquuSccx74PldKMEaqqnJffADAmGAMhCBCELYreprWtqyFE2tilCatAYCrPTKEIilFmqZ1Vmldu0ZLgAzBhHPGPYqQdbX4AEAw0dYaDUZTJTH1fK21VcpqjTYIZVoIwhml1CAwxgDBjDHqcV0DIUhpUEpKrSjlRqm8rCK/RRnxo9AYrJUtS0cyv0T9SimMqTGGrG0EIYQQaoxRyihprEEIiENOxhjOeRQF/X5/sVi4bpRJEgEYF4yhDDtiS7fbunVwM47Df/2v//Xx8ZFVKPDDg4OD2WSOMfYD7vu+VUrqGhPGOYcoGAwG9+/fT5dzADi4sauEoAzT2t3VyHd8UjD9bpsx9uRROrk4y8qaB2HAGWVEKzUcDrv9gWserbVWUmitkTVWqyana10z5nXXqyY339h6+FYK+pXV0utpiNddcrMZvqq21LifzXqHa/NvTrV5lGuHwBuRg03H/8bZ3vjrtQOhtUwk3mhbtRlUWO3eRDhWatwWIWTBMMYaoLB5ZxC6svvm/dm8e8YYx+cnhL1+kv8LjGuA7PJs4cqNbbDC69tvAoX1dV0+mjcFLFb7rmdYnYZ1Ccc3bHM19rB+EGu+wpVwFFp3hW1OZjMMBhsRLECXqbTmcK/D018+fpVt3BDCAgBjyFpkNFitpay5x37nd37nf/rX/9N4Oq4r6XnB1tbWPF1MJpNbtwaiqpOtnRtvv0OkfPnkiYrCDz98r92Knj17Mhmd/m/+6T/GjH756OvBdj8IwvPxZDKZlFpSP9je3+t2u29LQShWYJMovPfwAQ98rUSSJFqrP/uzP/O8u+PJ8NHXh4zj3e3u9uCOkHmRTlNVb/W32lEwvbgo2Wxna/v+7cFg0J0uTs/HZzdu9afTaV6OClkdPXu+s9fp9G9ECezs+c9e/CKVpt3lkQ+1WAahKktFGfJiP6um84WxyE96iVJ4UY993fnm+fOD/buD/Yd/+P/4b2ez2dvbt7e3tx1QqOqC+7LfH7x8dlpi5RHKuU9IXNSmXIiz4ezkZOLHLV0Lg8mPf/o3nEa37tw7PPvJs5fHQaejEWHcMxhpVGmEhbHIGsKJ1EqXmhLCEalEPRyN6qzoJq2tbs9aSwj5/ve/f3p8/Kd/+qdJknz66RdbW1sHBwcGbFkUy+Xyw+98596dO9PpdDgcutoCKSWltJO0bGxPT0+rqmKMep43nU6Xy0UlxXwxXSwWo9HFdD4lPmY+C5N4e3swX7Ks5MwLWqxd10IpVSudZcVisaBVVQTBzsnJSX97q9PuLRefVaWoaoWAUcJljWng72x14qQUQng8DLyQIEoYWB5Ya30/BICqyDFgirExutfuHtzce/Xq5fZWX2tdi/N5umR+QCnN89wL/G63L2pJpahrWZVlLYUUyhijLdTI5nXurpMxVpal64+J8ZVmQnaVwiecc2Otc0gEoyT0MKZuEepxT6i6rkutFeEEY4QxoZhzr0JICSlduQRjHrI+AhSHSVWJMAx9359Op2mVSUHqCi0uSkppEEQexkLVhFGE0Hw2TdotK61SltIAUWsNlIXQWnNMtKjrorJKqlqklQINWqFCLT1CjbRgJUMUGAOllNIKGUBGm9paSylGiEslsqzo9XpSS21pbxBmWTZbvoqiqL/VkSU2oJd5+uDdm7PZ5HT0YjDYXi7nyNBOpzPodwnVy8XFZHwiVZokKI66Qi0//+LnXkCA6KIWSdTOZKWYZcQDSmohCEbWWqMkI3iv15J17iONQv/s7OyCIEKIWUIl6k6ng4LkLE21tmEcjc5nRVH8/EVa0u0cMr2QgR9i48lcJjTApfQNSKmMVj5hyhplDCLUaltUlTEGU+J0wbXWtVQII2m0qmu7ljwCsMgaBsxVwNnVAtdSSjGmCK3rz1Ym+9JIungTAHYJa2eWm9C9KyFprLbWGiFsjKuYR2jVR0O7MBXGGAA78ACrZLFunMSmXwEAJ3WglJJSWGsdgwcjC1ZbY123K0Kw4y1os+qD3HgaB4/qqsQYe3zVi9yusu/r+P8qSoHQ+myFUBYsgEEIuS6FsKop1crIJsaAMBgwFq1XrnhNbwQDyDSghBAGILW2UmprHaJ6s8v5tsWuMZf4Y9Njkas9OS/hy7d47E3SqFnrRRpjECUAgDBYZI01RlmtqbJKyprSlgEtVBmRkDAstbCIcrvWL3fhRljrJSAEYK3RZqM0xlorjUYIIYuwxQit/kWu45e1TW9qgglgCpggYQleaSkYo4xS3CeRHxijtZGiqjFYn3uMEveqR4FXFAVQSjivpcQYszgiBLueEwYAgdXWJSYAAUIMAGCzQbZdv4oAb3g42miE1lpSq4eEAcAai5oySLeBe2mxRQhZYwlBNAwQAk4YICO1+Of/9X/z9dePbt7wv/rqM39319dodnq+aHUObtxApBKy7nYZfbjX8rmoZ9PhNKLwJ3/wB3e2d7YHW77wFk9n58U56RGl1PsPPirn4vzFuJ7r08Px7u5uf/9mEHrdpMA32t///vcpJVmW+WyXb6HH3zx58ggYZqOz8yTeD0i30haEJbpPcdIbdJfp9NGrs06nTdudZfnly9k3wYFtdWsWRV4V11PGdsPeHYbJdJhdUD/CKkK5p6pUVkXJ97CvCLdaV0bUpgSKIAnbAnWU6X3zs8l4fnbngZpm2ddPq0Wq9npooVJSXNT1q50BssJcPJ/td94yAs1mabtzg4TB8y8/ffZiKWp+cWG82zzsxsN6SRMYm6Gc1efiiHZNRQvfDyslAYAYQFUZIaDYZspQjy/T9O79e9Pp1Fi9207aQXsran94763nXz+627uxF/cOF0/euv/W559/SS2KveB3/95vPXnyZHIxfPvhw68++zzknhTaKOsx31q0XGZaaGzxxcVFEnenw6zb7eva+4uf/CyIuB8Gf/zv/v35+FVrK8YhpFVeW0riPY77gx7xx6QoCmTBZ3o+n1dFagVgbWgQBCsqhOfdunVrpQOoVRzHQogiWxJCtre3t7e3Xf/oIk1dS8Pt/la/33eC0qKql+mirsu6zD2PMY7n87kUVRzHhBrP84bD4WKxCKKwKIrPP//c436UxEoZJwSJMXXB87IslTFaWwejCWFBELnVc1mWdtXJxiF0MAaUMoxxY4zWRptVYpVzjgjRWtdVNZ6O03SJMQRx4PmMEFIVZZIkAeerfscWZF07vne71e13uhhjWdUe5YSQMsuVBIJJ4EdlWabLgnISxeHBwa1ltnB5YawAqFsTaGN0ltdgUFXVQiitDQJCEMEYRVHEme97pCwn7tAuMu/xIPBDjHFVibKsXZ+qJGmnab412MYYL+ZLJXWvuxUEgRBCm4rQeLvbiSK/1wsvLi6qsqLEai2rMs8WxPMJMjSOO37QbXciJcPZeJLmr/JCJElMqFBaTKeTbrdrtcUIEAbPY0ErdgI7osykMUenp3meF5WolLEIur0e5yFQr6zr4Wi2yNIgCDCmrlAFIRQEgaEMI6TBIoQIo05CXzcBUbRihbs+0I1dbmDfNeICbFQ0NCZyM6iwyTC4XF6vfUzTuXszGrE5mu3xRtVMM/+1kMNV94au/no9AfH6Yu6Ny8HN2MDmlo5iAuvI//qdvyJ+8PpFbd5ABySunfy1C3n9tJuYir0eXfiV1qabgOD1Y30bqvhVJmxmcNeI1wqta06GlVISIfI8b7fbsA7aa6mklEZpxnFzORsXdf1Y6G/ruOisnyueQmuV67quA7iMBxAM1tqyqF2EVSmlpWjSN0EQUM4Wi0Wr1fI8L81zKXW32/L9IM9zyhla13TgdQHD5n34trfrTbfOvckrUS+AdY0rsuvSiM2q10ale4W53e3hnN+7d+9HP/rRH//JHxHCxuMxgN3dveFE3uqy3N8Z3L17P3yblotZsZhzziPfU0p9+umnezu7e3t7WVZ8/vnnUzG+d+/eO++8c/PgdiVUu9fdN7frul7m2XQ6Vdq2ux1tTbbMhRDKmI/e/s73Pv7hp5/84sd//uPZcPrl51/12h1Q0miJrBn0O/fuvy2lODp5MZ2Ov/76cXvvzmCw3e558zSrKr8oTJIknPOiKgirtZWACQLQoI1StZLlhUxavhRSVIpQv9ttUxxVpffk6WFRjh4/GT0/LPpffw6MTKdpqwvz8VgchFtbAbK+knXkJbqCZ89OHt7/iHJ8cpZOFq8eP1vOF2CszWrAtVSYetzvxVEYhkIoq8H3/VKSdVbPgW8MoBEQSleLllXvUwvz+RzyOgT0+Omj3cHA9/3T8/PpfDZfLpb58p333vnggw/m2TKrCqD48dOnw/Ho+Pxsf2/vYjxK0/TBvfvW2udHLx13bX976+PvfG86XxwdPqHEYKTqSvoeUaKajKplnZfKRO2+F3QHvAOYgRJhK5G1GA+HZV15QTAYDJRSNInDqiqs1UqpQbeXRAHCNs/J+elJ0m61k1a7lRCKXQrT9/2trb772ozz0Ww2e/bs2XK5nE7GSZJMp2MM6vbtW612fHZ2whnb29u5e6+3vb396PHTyXT64MFblNLxZNbp9gghUlsNlgBgSrA1UkqtpIFVBNXVgzkiggunNCXm7kP3tXFNgJzQARiEMbhoxNnwzBgVeD5CBmHLOcUYaa05ZYxQjLCWSilFMfH9MPD8JEk67Z7PvfF4vJjN4zgJ/WApZCvsZFmmhfVocPt2J4qiyWQEFnHqKSWU0dZapK211nWD1CVGAEJoKYzWCGPAgKxFolYllL1eHyGkjey0e9YiQghmtCzL2WwhpWy3OmGQ+L4vhLJGuEAygGtUHVDKRW2UWQAOeoNOuxVS2mJMfvPF44CFXhAJYYos14ohZBkJkzjqd7snp+Pz8/PJZOoFEfcCY6xUqqzSro0JJlrrqpA+RVHQ5owJIZbSGAOj6VJKyX1PGkQYa/e3K6mIF6paz7NqOJ4HQY0Iy7JMSRP6ged5SkhVC2MMWEspbepWLIBxPC57tez7qjtZOaoNTIDXmgeNKd+0hpsB202fR9YNu51LIOvxJnt6BVs06GRz7bvpOJst3zjDtStqcMA1lGCuVlduzobWVQ/NQc26oNH93Gy26e1W8QIMG6hoxYi0GwQOl3F4nd3ZDJfjaA7XPIJru/ySe/htn7x+H36V4c5q80pXDxSQXhciEozBIowxAUSd6hqhZN1Ygax6iMtN6HONxdIcy3xL1WszGhS7+StCiHmeqkVVVUoLACjLvMwLIas4jrUQUtUY4zgM3JoqSRJRV8wLATOtARBWepUrJJa66111pwSEASFA1urN8/zV7x5C7sG5N38DK0CDFSy4iM4lenAvLbiw3PbOzm/8xt/7wz/8A9/3q1Jwj0ZRVBQVQZZiLIQT14/9bj/k3nR4wTApK/H42dPxdD5bLNM0/fkvPrGRkVo/evJkOBx+8803mJJer+cH0dGrk0ePHkVR9PF3P/J6fZSm89PTJy8Oj6ev/vk//+c//MGv37t5Twv5//p//rez4fj0+FUchsPzie8del4QJ/52/0a/t5Om6aQUASNZVmoFWpssLdNl/tmXxz/40R4DiREinBFgmGKlDWAbB/sEjFUlxoQw0IotcjWdVhcX6XxRpGkRx3D/wb2bd/aX2Txq+XCxIEaUaVEWRV3p+PbOWw/fmm/hw8NxWePhdHR4Nl5kQDmz0OOR8qh/c/dgZ9ANGC7S9PDw8OJijDEHAAQEAbl8eQBZBBhjTijG2DGEMMJ5nmOq8PYexvjDjz8s6+JnP/vZk2dPX52e9Ab97//mx/fevv/JJ5+cnp5ubW3F3Var30Gcng4vKl0viux4dB76QVaXjHt3bt364bt3LQYzKQaDeJEOB4Mb/+V/9Y/+/u/9zsnweJovsuMiTwsNgScDay0miIZ+URTTfPlqeJ4vlrvb22ErUUJShJBWqpUkgc973XYURRhjqzQAbPUHnU6HeRyMLYtCCGGNzpYTBAQhRBF1gfo4jghGZ2cns9nE5xSQiaLg1q1bebbknBVV6QX+1taWlLLb70VJLJXp9PsY48U8XaSZUBKhVQNlA2CV0toqpaytm1IxrWuECEIWY+uggBNo0tpaWxmzUrO1FBmltRQAhhPqhQHudrSRSgmlRCUqpYVRqsxzRj2PMp9xn/F+p9vpdLTW1qLlfD4dj/Ms85lnuQfaaKmNsb4XdTrtnZ0dTGA4HC5myzDyDWIEIQQWwEop6kIURclxBwCsRtYYa4xjqBhtl3UmuOp2B27FwznnvMYY+2HsxBIwYpRyY6CuZV3LIIiWiwIAOPMxxmUhRK0JIcwTmBZRZKPQdlphlYffmMzDfLsfT8ZZlpZIK8oJwrZMxRTnF2fnoqo9j3uMIGQJAUKw7zMLknvEKlyXlWtbpTGui7IQUkrZ9YNOu1NLpYwJwlhatFjmiPBag0bUYh9IgAkBogipMSWUUqP0iq4B1oDVYDVYY63FK/9/2Ut6wzqvjdpaJtle8QoYY9jo3bxpyhsvCFeXiZvW3Hl9V1682VTp25a8m87A/buJOZo532i1N4HFGy37tZ+bngWbwKg5gU0m4y93xpuCVGZVnb+i7znRxjdetX1tNHTOJoDhFvDfhhNev7T1v2bzQM1wX9hfPs8bP29AxuUjNhZjRBHGBAMmPvcCz/d9n1OKrK3rmpDSCTVGQSjK5ea0zXu1ec6bB30jpgQA1zJ3JSFPiItcYoyNlMs8m8/ndV0KIfI0nc+neZ7TVRdQTRGO43hnZ2f/YK/f7w963fki5Zx3u904jtM0JwVqt9taGwDABKy12FpkLQZAANZsvKUb5/ltgMai9RMHQEAczkCIrHEABrSmN64mNhu9qZowg5VSMqC3b9/e3d1Ns6XnMWt1mmZBpxMmSRz42TJ9kuejJOi1WjGnw8l4Yq1QknsBYHQ6HCqlBru7BeTaotFkFkQJYd7LV6effP65C+B99dWXH37now++9z0IAqr1vCieHh3t7vXTZZlDBYghgNt37v2Tf/xPWnH0x//uj7756svRaPlnf/4Txsjbbz88ODhQslgu6qKsJWR+nCyX+vmz84vJrBT5d3+4T4FarAnlBKglgBGhnGOmZ8spoDrpBkKZw1cnZ2dVWcKzp1AW4Hlhv9vb6t2+f+sdS6QfsPa9xeOnn5RiTrBXlsVoWHbagR/t/PwXnxAvnKblNAcvjAxuVRWjQY/akgDTwgqlF9N0PlmCRlJojLmxCBCxCEDrNZ4HMBYAGKXGGEqpz7iVJgi8wXb/oLtlkfnq0RevTl9WuipkseVvEx+fT85+8eUni0Xa6rff++i9KIqePnn++PlTxGgh6kdPnty7d+/WvfvtdrvVaj0/epRlxc2bt/+P/6f/w88++fnnX31ZlIvnLx5HvZhLyhhDiJaFKqrMWs09NIP81cuXp8cnZZZHfkAYswi0NZQA+Ix1ksQgEEIgMB6nW1t9F8ebzsZGQ5IkhBBrdVUV+zd23bJDC21B16IEAM7p9vZgZ7cfcN5uJ4vFLM0WL1+8ODnhv/0Pvss974OPPty9sae1rZXcB2yM1dYggq210mhnIN0L3lS0N0sKa62TCoB1wBmvlRmllKHHATAmCAEFAKNlWSlUo8DnFtm6zPM8LUUJyGIMyEK/2xNCMOZFUVTXdZZlczQLPB9jvFxmSqlOq92KE0JoXVVlUcwXxcHBwb27dxBCWZqOJsPlbNnvd8uyXi1TCAENRipVg6oBsMVAlERGE6MBLAXbWHOEEIqjlqxkLUqp6rquMfW2t7fv3btXlvV4PF4ul67GgVIqhYYVpOAYE6UUISRMDOeGeSoIbKfNFgnjqK7LichjVRaqFggCz48ZY0ajbF5YpQeDAedcGS2EUNZgRBBCccS7rTYGks6XGBBYbaSVUvb6nbqu79y7u7e39/Llq0WWtrudoqiOXp1kVU0ws0CCIGReCACUSowxIkRqJZTUWruAsFarVkKrlSwCC6Ct1ZtKBva6SNGmKb/8YcPrbKIE9564hWAjmgQbColNc4dm2usmdWN1eC3xcc3rbwKUX7LA3uTbbyYv0JvC2q/jleagTXXo5nGvETabn/Gq/v4S2Ti+wjXE0xzw2uU3CAPWKZtGovFXGa+jmdcP8W0u7Vec2Y3L+wDI2pV5RQgRjOk61+DSDdZaTaVGWGHs0knNw712Mq9jhV9yqnajiStal7Baa18+ezwajYbDYZHlrn9KURRKVA6YIguYgOd53W535/lWu92+f/+ey5DuH+zdv38/CAKfcC8IiyxFCLlGVyv/gSzCdjOicHme8K0ZoTWNAgOAa5kLFq+TEQ1cIACb9RENtrCXsxioa9Hr9X74w1/76quvgiBAiIhaDaKEES6EMrJGvmc05Hm+nFRgcV5klHtBkuzs7gIAY4wSPi4mYRgaSyzgGzdv8SB88jc/Ozo5QgjlZfERY5VSi8lsMpm8ODnNhHx5fIF5mITRYjb907/4D2VRDHb3Bju7/yROvvPd711cXLx8+XI4HBLeHk6Kzz9/QoJI6CULVQ+3Ls4XL18OmR8c3NzShtbSEACsjAZtrAVkDcLz5XEtcsyR0BoTxkK/1fO8KmlPZknS6XfvVMK+ejEP+OjOvYOt3RuseDQY9EohC0GWWX0xLs7PPx+Nv35+XPgxqTQBmhCvO5uL+axqt9vFovr682/qPL11Y2d3Z+venQeFkF8/fYkQQYjY1ZtM7KqRqQXA1iKMqNVmFUG3dmt35+adWwnmn37xi4uLs9sP7nTSlMXsnfff80L22ReflnURRDzLlst8nhbp8xdPZ/O5taCUAIxuHNy8//DBcDj88ssvPT3W2qZff20wavU7lpjxbHzj5n9RWdGPIs18Gi3G83o8FUUtAu2fzk9fvHxZ5vmg3W23WohgoRRjjPYHXZdTmE0Xw7OzKs8rKYMgyLNlGCWh50spLWgA4nMvCILFYrbi4ddiJSqsNRg7GPRu7u/vbW9rUx8dHQZBEIZhp9tqt9tpmmpA83T56tVJWZbcCxjjdV2neVkriTGmhOqVKoxW0ihptLYYWc4Io8wBBSXd24ytBafk576qJGxIbdgYEErJWmmth2UOAFLWlcwBrO9z3/cpRVqqqigFlu5rwghtJcn21hbn3OozIUQURdbaoigzISkmu9u9tx/eZcSOp5PJZHJxMcKAfD8cj+YIISDYqTNJiYz2MEJVaQkyWoOSyFqCAAMga7Tv+5wxa+3u7i7FuKqqdrs9mUyErJWW3GOtdpIk0Wg0Go1GaZrPZkWr1QLAk+kiCIKdnR1K6WKeUr/QphJVynqhx0wS4n4nOH45vKiNNT7HHico8njYijFFUtUDGISRzxgrRS2kRNjWUldVFYdeGHEC1EpfCQ1GCQ1KyKAVdLvtKORR6CdJIGXNKKmqvCyy5YxyP+SUoTBBCAmtKOUYEBgrtJZaGbAIIzDQyC+6pIMF0MYaY/R6udlYXliBv6axwqVrdxuQtSgT2hiwLo5oPEFTGtDIADSf23Xt2abRv+aHrvEhNk8S1qe1XnOvfr202htsg2uRgGZcXxOvW1lunsMmZe91YHHFGWz8FW8QGNfz/+2O+RpKcPet0VH4FR38G1ECxgSu3sO/G1Bo7uq1Dy+fvtLIAhgQWCCEtJBaa4Kwzz3f9ymloI0oK9c1uokTbL4k8B8DZYqicA6eUqr1/4+4P3uyJLnyg7Hja+xx99yz9qqu7i50Yx8Ag9k4HxeJpJaRzPSif0Qy/RcyPetNZtLDZ5TIITnkpyE4MwAGaKAb3UDXXpWVlXvePfbwVQ9x89bNrOoecESZ3KzKbsaN6+HhHnHOz8/yO3o2m02n0zRNH/32N7PZbDabNUW6rbVNiY2yLMmSetxmk3FyfHTuuu6Lvf21tTVM4OmLF1lavPfee71+pyzrC9c1WGvBWABtwVqLDbwpOP77T2ZDXA2LmIOlD+LKBwvQWC0W0QmN79xexKxQytq4++Mf//G/+Tf/pq5L7jDPC2azpBWFoe+GcWtz0Nte71Ow89nYdVjGWBCFs2TeX1tvKgnPZrOqlpTpw6OTLMuiTtRf3xhsbr04PMQIEONJUb86Pu6srx+PRo9evJyXVeR7CvPD0eS//Of/9Jf/z3/zwz/4vrAgpIy6vQcbmw8ImZ+Nzs/PMSZnZ2fDpBQF/vSLV7VJBhNVlJTR2HOiTitSimiLEWikgRorhMREY4zm2aTb7WCKprOEu+Hmzs21jej8VH326/P5fCJlVJUqzeZKlHU5Oz/Zf/82WdvaPBsVe09fSuBuuH56ig9PMmXjWngGU0aRBVdrQ6n1PF8XtszLMq3xFnOZL7Vuh1EUTOaVskAAEdswY1ho9sYEkNVm+VhKWRNEdna2d3d3oazOjw92b+yurfUfPnm8vrX+jY8fiDqbjM93dzY3N7fn8+TZk0d5Vs5miVYqjtvf/OY319c37915r67ryWg6nc57EcnKcm2tk1cqK7Pe+mbcb+2fHCtkWOAJAMQdg2xaFfOk9iQ3SoW+32931vsDWdWTySRL0lYUUUYoI9RaC1YXZeZ53nw+l1IYYzvtOIpjIVQToMAYC1sxwGIrr4UuimI2myXJrMjyokwIBYdhxojSotWK+4Pu9vb2PEsPPj1KkkxKOZ+np2dnruvfvHnL9X0hlDHAOLcI4CKMXCnd2OyNsQhhQmgTwFjXYqkPEEKEUMdxCSEObcw4RFvQWmopGwqH6WSCKcIEEIKmWDNl2BhzfnrqOE4rDlph1Gq1Op3OxsbWYDA4OTlxOZd1XZellLKuBAHkO2486LQ7wcOHD+ezxHV9zqnnBbPJXEqFELG1lUYpqbW2ABghD4FFgBHgxilsDW6YXo0BoyGZZ1ubO0aJk5OTfr9X12Ul6sPDV+fnx5ub21tbW9eub2NilZZClpQ2uR4WIY0xMEYcl0hZS1nPZpOd9R4jdq3b/vjB+zKvk1k96A78oC+0BYpchzDPEQIpWVltFFKEIKJNJWqjtcMQIFVXOdJIKWktSCmV0EIIORldu3bt9asXZydHlLuz6TRJEkp5O44AAxjlcsYYCCGIApeSsq6aFcEYG9CAERCMGdVaW0AACJmFVUDbJnd/1XR/4RvGqLHoNjpuFQoQgt+pdZbn4JUSSsscyCX5/0VZjatbZHvZeLuIqP9qjgS8TJe/OL7U/W8jgytYBy6Ai13xYqzGTi4vdyU5c3VsXw0dmlD2N+O5MkWrR1bRzxU8tMpseOXe39neiRJW5+GrfvhV/VxpVyb2zXFYTCssDxprm2LoaMFAwBlr1LldofRePieNMWAVOlz58M7WGDUbJJdl2eHh4YsXL05PT8fDk2bD1GAXYwwhxGFcNrwoduHQqYXKiwojNByPJrO5UqI5ZzybXd/ZuXvv9rWd7cYw1BAyIAvIWjAGNUG7y4f/H4r20AvjAVh7kdlhm0wHsxKvgC4RSTRIwiJYhERcMIUTghS6d+/e97///V/84ud1VYdhmKZZ6AekKU5AueN4CBQhbJ5kUtQWkeF4GrdHiBBj4PT09HePv2j3uj/4wQ/WNje9wCOUt3v9eZpsbG5WWXpwctw/2vjGd787TbNHz56XUlVJ9vLoZD4Z/83PfzGrRCbE8WgYtttpMm9HLd/3g073RquHMW2v77bXd2dH+dOnT09eJ5PZqed32vFOKerJpLhxbwNIiUAj5CDFAbCF2iIqywh0Ryo5nabcMX5IhVDHp9OkFLMEev1qsB632rK/AcoeP3t53ut8+N76DTlG87JeW7/msJ2z4dxvh7UxlDFhc21KWUnPdV3mOJRa5CCm2uv+h/cfgFUPHz3y45br+PMyJUA0kGYhEOhF+S8LF1zDyBijlMW+c+PWTS9wp8lsfXONEqKU6Aw6zHN5wKej6eb6oNVqbWxseJS/mCWM0o219WSWbm5s7u5e63f6dSWPDw61kNe2d/qD+PHjx73N64WUnz985ofOXJWfPvltrYUTB9gJqdslrFMIogk3hAXIXev0wiCIgnBcDuuyqnReFQU9Pz9v5CkhxHPc7Z3NNE2FEJ1eBwBGw2GW54w5hJCmyp9GCixyHCfygziOoyjyPG8+nW1srgVBoLV2HOJ5HqXkBEApkaby6OhoOBxvbm5ubm0NR6Ozs7P1zS2nYVq0jcRsOOMQIW9q8i6b1rqJCF0GpjWlsZqiF3kyBACjwWrVMDUpIWohNjbXtNaVKIWorLVGa4Ostuq9u/du3Lixs3ONc968w/NZ+tmvP53P59PpvDHvSykRwo2sybOJqNdPTg6EUJuexymmFJ+cjDnzjEVKm7oQeSmNAUop547rOBiBUsqY5RNgjQEpNRg7mUwauZ+maa/Xq+taGZ1mcymlVLW1Oo5jzmm/3+WcNp10um3GmFQVJrbVimrFKeWirgEM57yzEYT8m9k033t5fOParh/1h6PZNE2lqrFGxug0Ta3V3GWO42RZNpmNHccZDAbImrLMjTRGAiPuQrhr0+20bt3c/dWnn52dnN65d9eoajKZrG9steMoK0qtJSMUELJKg1YM07kQTSYhIgQRDcY2G69ldJhFYOylBV0KqCs61VqLLuLUloofyBtVsapZrzgLzAW9Y1M2zHEWxUjRIl9RrmqdVd28RBJLQLCSdXlJlS5/DhdqbPn/lZNXm1nhXV7VSat3dHkG0JWD5oIgYXUAK/03M2ZXRn7Rib2kbgGu/rl63QYirF7o7Wv9Pk1fLs21vBf8FWmQX9VWZ3X1yCrso4QQyjjnDuPNOoqyQghx7nqeBwDoXVxVTZ9XxnPF5PN2a0LTG7vLfD4/PT199erV69evfZfChZ9LCFHmBQAw5siGlh5jht/UjTSAba1OT0+LogjDMHq+NxwOj69fd133+u61NwGGxlr0jkdu+ec713HZLqZ9iQxWsMKbKITmWng1A2L1ilprAFYUZRy1f/jDP3zy5MnBwX5ZVGsb65TS+XyeJ8Ax9jlJk9n+i2eT4TnBOC8LiklZV2fDESEEMKLcLfLq1q1bt+/eGY1Gk9mMOw7C9Hw8QcgqY6WxBuzx6cnB0eHa+npd61ppQ8hoNicOf/z8xZMXL7/7x38SxhGyIIQyFhzH1RrCVre3uQM7+O9/8VNN5Iv91/NZ4Xqkqqxbg+tEQCuCDQEPJDcyt1Ah6g261zll2op+J0QcS8nHk2KeFIhA3CV37m23W/7pSUb5zOBK2dPXJ+3d25vUdW7cvnPr9jeePZl+8fDx+JzduvVNTCkSMM+yui49NwIEyWziaDWbzjtxOOiuGSu1/LKq6iYB+WLBMIAGiy6IN5A2CvBFYRoMrutsbW0JIV6+fOlzZrV0XffHP/5DS/DRyfFsMr19+3ZRVEWWhoG3vbWJERFCxX4UBpGs6tev9ofD8XQyb2J79w/PNfCi0KPJOO70MYPfPvoMe6iQpc0SYF7U1t1BjLlPMLOEEoMYJlhbVQuKSavVElVVFAVFyNvfPyGEeJ4nBG63Bg8eeGVZai0ZY43KXxJHK6WSPA2CIEvm0/Go1+t0Op0g3NK7g06nI1V99+6dL7/8UiMYDocSMDCPISdL7PbmnXv37mVZ9ic//qcHR8dBEKhacR4URZFkmeP6jLEGjBNGKGfGGKNB1MZojTH1edvKHKzBComqVlh2o1Y3ailRx/HGdDoty5xz7nmcENZux0HgUYYB7GQyGg6HxiqOUeh7cRx/+5vfa4CRqOrGoZgksywrq1LXtQjDsKoqawEhq5S8d+9etxf+9Kc/J8pQA88fPXL9CMFUVQobgohTFeVsllmLAFMAcBzXgAGEDTYWGUQMZwxjYiS2Gqy1nbgzn6SqptiEv/v0hev6Es8G/fX5fA6WPn36stfrgbb9fl8RGcW+LKoyS0MMPsMqy4ye+63+TnDt5q2diHl1ob1OSCP49h//qHft9MXzw+PDPSHM5sa2EJIAKqraDy1CJI5iSimjthX4lFJjwKEMACNnkUaldY25DQLueZ4y+vbt20fOaVVKz+9o44JlgBzPd5TRtZRSCoWMIUaZKmReIz2pxZhwDVprbSxgQi1obIEAIEAAxlqCrVWUaa2VkkobAIMxthSQxUIIx3EYZ9aCUsZa61CXUceihQ7DGlOjGWPYYmttWVee5/lhoJSqqgop5HleEPhCCESwRaCMBikWygaBucg2xBhTjAjGFsBY08TtItTs3BCAbZQR54u02yY670LfIG1U4ytp4tPt4mvU9MyYA284+5pKhpeCD5Z6q9E6zZmNEloq7FWJv8RAS+yyqswaIIYRJgQtawoYqy60qUEIIUDNEJthUELyPJdScu4wQmUtlJBgLQLIs0zWgmKizMIOLaoa00sb7uW9XAE9S43eeMev2DC+pn3VCUtv0bIrc0FnYTEihCBKpNFKWkswAV7U1bzInNDnriOtsaLmnGOwgeM09i5onEdLp88KEEGNAb5xCVnMGUcIyVpovdicNJQNxtiqLs/Ozl6+fLm3tzc9OydaagnLeAUAcF238YkhAgDWgBa28R4sboRzBgAaGY3V6fjs5f7zs7PTbq99bXdzY30A2kRhQBFOk3kYBIQxISsAsA2AbhQ/XtgDFvwYzbAvwguwWUGZyDS5soARRhQADAZAi58vSEM1u1hHAGhsMNJa6/k8TaeeTx2Hf+tbH//bf9up6zLLsqwaxizCYFtxe2djnRg7Ox2rwsR+zxjT2d44GZ0ejubj2avBxrrrebmyg0H3i8cvw8HWLBNP9g6zLHPCfllXOzs7laIHJ/N/9+//9uDw1As3j0/TqBt9+Wz/i88+JUFXCqDB2je+90/KkmkTEUSxQznlABQsqFpVuWp1h9/9ow9JhFjQ+69/95kizw2CaG371Wvwwy3XY61WKykz37t169adV3uHJXuV1LjX3eh0Q87i8+FsYz1Ok2ft2Hn65OlPf/rT++9v37416HZdIZLNtc758+Szv3nInOj+B//69vWPDp7/LAyfxR2/EmORSwQEA/WYi7TJs9NBt7eBre70i6o8PX566727/+xf/fnPfvHLRKUuFXmdRjwGwHklKOUYMYJZyhLOnDrLZVqv97rpZPbDj76z1V87Pz37xWefb62v7e7uAtCTUVJVlTHwg4+/m5VFYerReF4KIYUus7JKiirJOmHbalPWlSXYIJtIlcwTOZUA8Or5M+Zz5uBC1t3+xihNN7ZvZ7WqFVI6KArwYxZ5AcYoSyqHMoMgr8raKMQpZX4c+ZRzvr29PRwOMcavD17dv39fSglg1tZ2Z7PZZDIuyxIh1JAcJOnMAJaUOY7TabV939VKhGG4sbGbpvMoitfXesl8vSgyBNKaOo7cl8+fr6+vt1qt8/Pzqqq+973v9dfWx+PxwcGBAdTQJ8BFuVhKaVXnS8FNEFnwziBqjKmqAgFyXVcpkSQJ59zlrCjSwHPjMNBGGWPiMHAcxhjTRna7ne2tjVevXp2dnfT7/ffeu7uxsTEaTYbDszzPKaVBEACAlLIsy4bWaTabVVXVbreMMYyxu3fvzuYjSnhdJVlRSmE8n3he1On6o/G8qupFbScNccePozZCaDIbU0oRtlYbSonrupxTZCynTl3XhJC6rq1GjWHcWht6vtFyrd8/G47jOC6zPAzD16/237t3t9fuWClm45FVkmPEI993vXidfPs7HwuZnp4eOe5m3IowItevXz89mRZFUVby7p0PJ+MZIcRqGfpuJZCoRYFT13URAKUEY6yUqMrScX3HcTCmzfoWeZUXOUFsPJ6mST6ZTV0nRIQhjAln1liAS6bspiDFakMrjLZfrx6WnSw/LI3zSyXUxNYhvCDXWmrTRmE0wa2rSrQ5Hy524cuyBcsrwspO942dYGWkq/pvuXO9oq7QmyrYq7rzajzm23e6+hV6K0bBXKQbvHPqrhxcNTysRuqtnvx2yMXqXDU/aTYxzdz+g4ERV9rqNveyCeTSsP/BJ+Gr2lf9EF2QJduFBQgtb6HJi27aKtUmWmnLdV/QqFwu2tQ8MFVVYYwJwk2qqjGGU8I5n8/npydHr169evny5fHxsawFY6RJblxyYsKKG2vZ7SrGqusaALTWVVVlJMuSBAN68uTJRw8+NFpurq0bY4RWrusSSrWUmlw8hAgZZBAQZJFFBmOKASzYBitgeIMVvr7Zryxd/Sb/FgCkFJ7nSSmtNevr6x9//K3RaEQIyfPSdRyKcFEUk+k0DsLB+ubO1vZp49LttOPz1jiZJWmaZVlVVWvrA0JIks6fPHpYS9mEahKKTaWzPHVc9/mTJ69e7dV1nZdFEAZffvml6/HD1wd1njmUeZ6rlHQ5K4RE2DJOMHMAEWyAICw1rrNybWPnX/6re//0n8Od9/7zf/nbvz09OysLORnPaum6JXWdoN9bX1/f3Nq8phX+T//pkyTJwmD08Te/u3ajgwmnlL93/87B4XPHA0RxFHd3d+4EAXny6HePD1+J1BmmBlM/Ec7hVD5+sZeWNdOkEoYxxgiuKmmt8n2HAHNd890HHyKEtq9dz+oyCry1nZ2f/eJnoecXZd1YXg1Qyi0CrAFpoxBFVVUxxizSRV51Wu3bt2+7rjc+H7quO5slCA42NjaKomjS7/lOB2prQBqsjK1rUWklOLOaGmxqa4BjiyhiFCNjtTHh2lZRZrWsEBCHcZ8zX6sAIMuyUhgN1AIt0kQp6wUiCKIwDBFCGuzykW4oAChleGNzq6oL13WHw+HHH38sZW2MStN0Pp+dnp5OJhPKcONW0FrXUgIgSikhKMuT0ei802n1B61evz2enD189Nnrg1dRFAQRzksQar65uVnW0lrrum5TZ1Jb6Ha7cRxbhKfT6Xw+NxZprfM8b95brY3WWiurrJZSE0IYZmmaTqdjh/FOt0Upret6PB47jKZFHkURxjjPU2ttFAVSOtrIMAxevnwZxcHu7u7m5rrjOI7jnZ2d5Xl5fn6epmkQBFLWUsrpdJpm81ar5bpuVZUA1nGcplh2qxX95je/yfPKGMSZ5zoUgE7G0yC0Sto8q8tCZmlZlrUfdhzHS9P8Qu4sXlqttVJAADUU1M176Hpet9sFY5MkU0LWdb3WX/ddryjr49GYICyl+MlP/vpbDz66c+vG9vY6sgaUJAiiKLr70W5eZrXI291OWVePHj8Jw/DWzdYHH3xQlfb4aHjj2u7h64N+vytkZa3ttII0BYwsxUAYNQisRVIi13UxhtlsNp+nABAGscWokmK73aKMaQzaWqEVxQwIRkCUNQANgzYCQGAuEv0vsg/MSsz/1yuApZZ6o/Auws0u9uILQFDXNaGo4S1vIIi+qMvXpBcuq0zBhaN9FTRc6BK8jGpcFd/vEqBvJPsynRLgSkrkEnBc4jxeOeHNbS4n48pxtOJ6uOI7bwiXlvps9YdvwM3KZF4AhXckj8AKN9RygZZVJK7EbbztdHi7zyvTtXKhVRPCpSOrwOiruvpvaujCYb9cr+ZhaPIhm0abCMaLALGlCefK+jbQ8wpWYIw1cKGxBi3RlVJqOBzu7e29fv365ORkMplwyny/VWaVWSHPhguc+s40S4SQEKLBH0KIilSNI+PZs2e/+MUvTo63f/zDHxGMtJBxHFpjqqqy7hJPN+YEgxACizHHAAgDQbAoTYO/Noh1sV5XXF0AyzBYu3BULZZJSun7gRCiqkSn0/nxj3/8y1/+cjZLABMLBCxOsmLv9UEvbq91O3F3gAh98uTR070XUkqFdFmW3cC/trObpkOlpDHi6GhfKRXEUV7MCdKdVpjnc9el83SuZ7q3NmASlCzXNwbN1hRjTBkRQnzx+WcMgef4BGPX8eK4HYQxGFTkeZYV/dCPW90o6vth76Pjyf/0k58j4mhLw6j7zW8+2NxcA4DRaDQeJYxMfa/9pz/6X/z24ZfPnj2bTgrfGyVZCmB6/a7rMc9n3KV1Zfb2hkrUL16cZqmsNTKTqbHJwVhHe9PJLBWWag1hFElRZflEiXznxo17t24c7b88Pjm0ohvH7WtbPY1w0O64cctIeXpyirjHGBPaADGYO0aDsUYp4zC3VmUcxrquqjRHXkAIOzw8PDk5bYXxdDoZj6aU8PXB2vb2NudcQgncxJ0w7Iaz2ey4yss6oQb1YxeU1kYDNHnRmChZy/pofj5L5nme+ZGzvrUetyPHa4WGHJ2eKG0QA2x0lc2LJDWtTkBQ43DXWjeConm2lVJ0NBq5rqu1NkYVRXZ6emyMmUzGTS0v1+Oux+u6llI6Dmu1IlHbJkDBcxyrpVGiLosim+1u363K6Ww2AlMFXqvTbXkuIIRctrG3f5AkSbfb1Vrv7+8LpXd3dzc2NrRdWO1qodI0bbgXW+2w4V7EGIO5qOaCoYkYauo7YYwab0iZZ5iYIosaix9CaK4qpVSeZ4SQW7dvfOODDxFCn3zyiyRJ+v2+MYY6lBC0ttZvtVpa6/FkiLBtt9sAljHa7XaVEtZqQpDjsL29vfF4wqjT7XiACaPePM2G57Oqnnp+BIA5dxEieWaFUGBJXQk/9q212khtVaPqpAQCyOWe67qN4zwKonar245bSZJNJns4jHa2N++/98Ff/dVfGa3Ho/PNzXWGBlWdvdx72onDu7dv3rp2p9Npx2GUw+yTn/x8MOi+/+Gtg/0XCNnr129mWbG+tvMH3ws+Zw+TdKZNTRnkWVGW+Xv3b0351FpLuSOlFkpag6zryLpChDbsXa1W6/qtm+vr69ba0TixlCJMmOMiwqnDEVBEsFVNcOJyhwpW26bYx3I3DBdb/K/XCo0UaOodLxXMKlBoDjbog9BFvTFY2as1kr2BAg1NwnL7uDxzqRLQCnHQij67cMeuBBysytDVzDpY0dPLCzWIYaGE3to+Lv+U8g3ggBVg0UTGLfvXb5VfeifqWh5ZxVuNTnxzX2h5poHLaAMAlpF3ALLRf+YiK3JV260M9d2w7wpeWZnbS8P7RwMFdNkN/855sBcMHISQIAiaZIflFd+JcpbHG+i5ZPJeYkGEEOe8QVSU4gY3NJWLnz179mpvL03TBjroRSE3WF2RppMl+LsyhuVaEEIs6Gb7hAElSfKLX/zi+bN2r93xvW9hC1VFFSFSSspoMyxA1mKEDAEEFmkwFBBYrAEQRgTesCJ8XbOXA2uaYxc3fsmiRjADWNCXSanv3//gxo1bJydncS90GddK1bU8n0zSJMuybJZmke8VlTg+Ps6K1Pd9zlg2T377m8/Wt3pKKUYpWKu1rvJiPBxhjIUQsha7u7udrDOeTgbdniiryXxGfPfs7ExLgcFyzpUWjx59GbrO/XvvESCglc85eC5Qx3Oolph2B1vxAIDNp1ldQ1mrbm+j1Yo5C7RCVa0czrc2ryPErLJCmI8e/Mhz+lmiOQuSpMiKDIjdjbYwA2U0s9wgdzI3o9P52ZkO/LUKdFEKCyQ3FUrOGHX9Vg+MFUqKupSiDH36zW/c+9Mf/+Dls/5nn2JZzKey+Hf/5sWte/d3bt4pDo9bUeTPstoi7vrzrLJaMZdZZDGlYBWymnNOMbGYIkSGp2effvKrg05vcn42Oj+Xteh1ukZrpUwct4uieLH3klDaW+v1+/0gcGajswwJhumgF+ezrJS1NKoslMakBp2XZW6jRJhJWpAyz5QJJ3OpRV7lAAAY+czljBdFVeQpx0hFPg5iKWUl6ka6LpOhKKU4yxLPcxhjg8FgMhlvbGxgjDGBVjva9Nc3NtaSJKnruokoBDCEYKVFURgAE8cxpShJZi9fvggjd2dnZ2NzgAlQih2HUUpn42QwGGitmxjJMAzTvDg4ONjf3yeMN/u2JkYhCIKGQ08pTSklmGEg1iJKqcc9Qshg0HMYF7Iajc7zPJVSFlkaxQ5CYbsdeZ5XVVVRFITgMFxvtaOPP/54Z2fniy++ePr0qeu6nU6nKIp8nBNC2p1YKrchYQWwjsPLsqzrst1uW+scHR11uq1Ot/X5F59lWWmMxRgXRYGREkIaY4QQjmscxwlDp6xkmuSNrGGMLXaGZjX/6s2uUQgxmUyypHC4hwEhhHY21v0w/uDue9eu7fTaHZ+xpEzm07G1+uR0DlrdvrkbRR+ub605jGhV7h+9kkqNpsO/+7vj45PXOzs7YOnz56/+xT/9V2HQsdbsPX+2tTEwqi6LGWM0cJkMnCaxK0lzJTQiLKTuqJoFfuA6fhy3w1bc6XQIIWVZvT4+oZSWZVWW0vOIiwmjLmBiQTVVKxcZHAasfSNxVlXC21L+iqhaSuQrm3hjjF3ZJTcijFzUcFoaCRrDchPiai/TdCKEGsjS/GqJIZbfLgMVl2NYxDBcJt1bjnMVHHytBF6es8C4q5vLy5d786EZDLpsXUBvsS+vTt2q+llBCWi5P2zGYBe2FvP2fa3abPRFseNmbpdYahWO2N/PoP32PX7Nkf+mrv7BmW/OaVCCMUZKadHCE7EKxeDyU9rEMsNKxefmUanrOggCpVRRFIwRzrmUcjgcPn369MWLF2enx82qMcZEVWdZhi6ycuAyVlhd91UFvMCs1hprtNYuZxQTIcT+/v7B61d3b92+/969fqdbVZXrOJxzbSRqKk0QjAxGSFvcxJ0oixG2GCwBpFc4T98925fftRUDA1rk+y7+sxasBYMYY1laNLnik8mk1xt877t/cHR4YnxTlqWxlrs+Yi6yZpZl0/msFcXrO1uUky+++GI0Gt28eRMhe3R05Ls75+fnw7PzMI4453Vd10Xpckcptb25+b3vfHeeJr/5zW8wxq7rdmxrWhdpOncYD8IgjkPO6TyZzufT8fDM5Y4svSKZnxwdBH7U6/VanU4+mQa9tXyS//KTz37x61+PJ8napltU9eMnz7Ii2dhcC31va2tna32HcMd1aJ7V16/f+uEPRdzykmzqpLwUeS1Ut7de1kYq0els3rvz3kH7OMuhqoRGFjBjlCHCtMGOG8ZxS9b1eHjCuetiQ20ty0KUqcfo5qBLzk4JQnVZnZ2dWeohJ9AKXOa73PVanbw6EtogIMYITqnFqChq3/OyrLBSxWF0OJo8ffyM3EGtVmc2mq5vrb/33l0tZSuMKSLnJ2ej4Yh7nHt+GIq6lloqh7KWGzSWAGOMtiYvKsUIuK4hvK6NpZR4njGqqI1GNYAxmtZSOI6DgDHMOFEMYauVrCvEvWX20+pbQ+M4ruuKc4Yx7nT6RVEM1npVXUwmSspaShKGoe+78/n87OwsTecEcaOZqGRdac4pBhB1fXZyroQE2y8wUIaVEgjB2elESul5a4N2lzHWpC00gQhJkpyfnzPHJYT4vs+4izFukhiFLFdftiY500gTRZHjMIrJyenRcDis67Lf76/1e4TU7XYr8F2tVVlkGKHrN67fvn17MBh8/vnnX/zmc0LInVt3EUJK6DKvgJqyquqzMk3TxjIhpRSiIoTked7Ug0iz+WCt1263f/Ob34jaeJ7fakcGrFKm3WnVdT2dZVmWOF7UGB7CyEfI1qJgnAK2S7Yca5udIiKAsjIDAOYza206H+dZWWR5VYkH99aVkKIqvvzNF77nOJzGrXB0Ptze2Xj/3oMP3r97/dputx3O5qPTo8OyKrRrbt29NR6fGVP9yZ/+kyJPnzx/Rix/9er1+prCAIDsB+/fffLkIWews7tWl5mVgnCgmCEwVmvGGHe9WCjXcwExnRezaTKfPSyqaj6fG+JaA8YAxhRh7roGEyPKGmMqjVYGjAatloQBeLm5Ryu28a8R7vYryAbMhXG86WdpFl7qzred+qvxbk0nzU6lkYmrjuoGjy77XH0B7EVWwvKps295KJahA8t7bAIb33RyMWx72aLwRnhftOUdrZ627HP55yrCeHtzvKr7V3+y+AotcyyvLgRCqIntsNYuK2Y1XzVA4e3zv2ohrwDElVG9feQf094GZ8u1sCtFn82CT2VhLF0Y9hFZuhuWMRlwGfZdAWRXlmBpbGhcA2dnZ+PxOMuyJsyoEWKaNDNmloauVWW8XI6rTwJGUkowFpCpqgpZYwiVtSYItBKPHj1KkuT6zm6SzJoHWKqyGTMGijBYhJDBCIFBCllijEXEIkDIXng6vt5b9FbYBFrYIJpfXYAqbKGJJ10U00IIk+9//w8OD49++ejXeV4Zo9pR7Huuy6isytlkNMvTKA66g/73vv+dxw8fpfOZw9j9O3dbYef06Pzxo+f9fn9ze4sQwpjHOS9ykWf100fPs7I4OjpDCKVpmmWZ0wsQwYDBcRilWFalpLgq88dPvoy9gCAynU6TWdrp9L797W8/ePAAO/zk4PVP/uvP/+Nf/dej47N2r9vudrzAPT7ZPz09RdieKH18fPrBe9XW1rX1/nqRzba6m/fv3VZWMYdSz5FS1FXVbq17Xrss61lSTpOyrFRRiyzNjUOtAUwRxYxRyinHQCginVZXlakQFQIiK1MmlRG25bd1JPqD9e/+4M8ePn2RJtXOrWujs7mUdn0w6K6tz6Z5UuQOoaLICaMWLBjAgEVdYWPisNVpdVut1s7OTjuMOlF89/YdbOGzzz4TQkRB2G13PH4nr8pkVtfVyXw6mk5SojHj/jwp5kmuLamBJsoooH7kG+JOjyeN3QtTBhghvGAUTY4PMWKV0IgoA8T1Q8ZYXcvKJsYYafTSNqyM1lrT6XRS1zXn3HEc1+MNOXmep1LWaTqnFHe73Sa8wFrLOe+0277vX7wVtiwLLLHjOLu714PAOz4+tmCEqFzXLQtTFHUcswaee55nrR2Px2UtXNft9/uASeNUbhLYGqcIdxbGQEY5GNQEwDfyaDqdKiHTLMUYh2G4tbW1u70l5dRxHCllliUYw9raYGtrI47DV69ePn36OMuK+/fvh2H46tWrLMt6vR4PiRB1w46CEPI8z3FY85lxkmWZ67pB4HFOtZZCVI7rra313n///bIWWVq2u/3z88nDR49f7h1Qh1e1NVb6vmNBZlnCGGOErrIFN9NNADU7HkopQqQuZZ7n08mkruXzp4nruhSTuN3yHNaOQwOwtbn2gx98f2drI/Cd6eT8s09/+uzJk2Q28X0XYufa9W1RZbdv3/jggw+++OKL8Xj63u370+m0HfaiKIrDgBHse87O7ubm5tr09FxJwSihGBjBhCJKKWNkQamLDCHEGCGEEFJSSksDtajBUsfh2oJBYAALLRzKkCHGKH1hMcEWN/7RpWKzF7l8bxvSl20llGFlimCpWi4C0RGijC1NBfaiXihcxCIstUijCZp94RWXRGNUaFQI5xxdzi9Y6PjL0nypQjC+xN/wtiliEU/+Lqv4am+rIGMJFJpnHl2ENK66VK7sROFqqOAlLHIFWCxGgpZOllVX0cWjeJHm16zD0g3ZfLhy6a9pV/DQcqKa2LrVHv7RcGEVel6agcuVphvraJZlrtELLc5deAvELFd8dT7RhRtrCRGWss73fWv1+fn58+fPD/ZfTadTYwznfOm+JYQghGsplssHK+Eyq6Bk9aZwE6aACWVYKVVZYxlHVjOCGaVNmOSD9z9oQijyPEd0kTKhrbUWI0MwtgYBAQrIGoww0IVjAjUn8q+ZUnvZhgcAKyxMsEyVRAgZbXzfN9oopeI4tgZ2d69/61vf+dWTz8tKSFFRygEAYx9TBpjWVbH/+nB7a+0Pf/RHt27c/PUnv/jNp7+bjEbX1X0t8WyUIuAODzEljuOCJTeu3x2Px3svDxElgdvCGHMaGDU8Gh1aZSoErcBXSiEpoyiKorDK0nkyLZJsNku0kJziyfD86PX+UZ49fvz0J3/z8ydP9msJfhAdHB1uba2vb60fH+6XVcoIvXnz9tr6oN9p13Up9Ww8VtpAJaTjuAN/0/HcF8/3jPXW1m+ORqPHT148f7FnlJ7NJ5RSxwJB2EqJEGYOlWWVlBIjRDHISlHLOnHcCrtaUi2p73ZsbN2gq8E5HSbDSYqcnpA4DGJkeOC2HeJ5WDJEcmWoMQhMQw3OGLNCFUUZuN76YAMMOjo4vH7tWuAGv/j5z371y0/CMEym6Q9/+MPN9VuHp4fD4ZkQs7rMpMDGQpLVIilnufCCWFM2rcYFVKGvhbF+wIUAVFopZV1WQiiEkAGgPMDUkQqZQnDOuechhLJS1FmBEGrI+NWCqFUDAG0wOCEkCL0m+7wxNzVVnhvjvDGmrHLGyVrYt9JSjDjnjaFP1BXmvNvuRUG8sbEBhhRF9vr167rIVQ0u92ezmRdETUmnuq611ggTQojrusrYBh9AWTY7e2ut57cawB6FLUa4UgZjHLhBnucHB/vz6cwP3MFg4Lq8iYikhDQWrcB32+321taOUmrv5cuXL1+ur61tb1EwtsyLKAitNkWWF0o7jrOsMkUpblQIJoCwnc1mGMNgMCCEzOdzrbXjMM932p2YZIUQottrU0qPjo8ZI4xRrRVC1g9cA1jIknFS13Vd18YqRijGGAEQghzucOoURTGfzKXURVZlaWG1CYJAy7TU+uDg4H/+jW8kWdrtdqWUf/7P/of1Qef4+PCTX/709OSAYiAUxZ3Y5RTHvu/7rkuFEE+fPj07O8uy7He/e7i9cX2jv+26blVVe3t7nNNBv+s4LGOM4YoQxChxORXcoYQwTFyXc9dnjusHgMh8NJk0tFqGEiE1xsjBCBFMOXNd92Kb2GzpwBjTVIEEwI2dSq+ENDZHllUQ3xZVqyrk6g4M3sjZxsCrtLAXbt2lZm3EKLrs42+0YJNn24SwNQ/SqtRe1RNLoPDOthyYuUyE0AwMLzIkl5oMrshf1JiLL4IP8EVxgcVFLzQ3XKg6uKzY4DIm+AfbKqZZuj+WWnYJCFY7vLIXf9ucsJiEr5igt4d6cYOXTlu1i/w+N/I1NwhLlIDedGgvsGme5wZsI3bRZY8DvAtg2QunVQMolwqeUjqbzQCgFcVVVb9+/frhw4enx0cNsGsQ3kKOWWig4/K5Wr3oKsRBl01WxhhEKGNMNY88AEJISul74fn5+dnZWVmWjdSdzWZxx110DqCtxdhaSxBCBmlkMQZskV6o+Wal/tvoKpbNNMGS1pqGJlVr7biuqEWe551uvypLx3E2NzcdxwOAqpaz2SxJIA4Dl7O8yKWojBDJfLre7d+8ce0v/uIvAs//5S9++bvfPVxbW3Mcj2GW56WUsr82QNqen+2Vom7KFFdCCCF835dS37hxg2FSFcX6oB84XORJt9dpd+Ka0eHJ6XQ6tUr3+4ONjXWtxN7L52ey9sLgL/7iLxgLnj5//fTZ8199+uv17fUsnzi+s729rpWq6zJN0/X+mue5YRTkeaENUloHPK41VKV+8uyVMajdGtSVOjk9lHXe6caexxG2HqHGgqilFRhRJSpZV4Jz1+XM5V477DJQjx+9fPjZF6PhaRz62x13Z/f6LHksNKLMf/psvy7NdD7eogFoa5UGY6lFyGoKgCwiGNV1HXC3MuL06NilTEr55MmTOi+6nc7h4eFvf/slISSO4xcvXqytrXHy/vQ8Gw5n1hqXU9cJVFXlWa2kFgYc4koL81KOK+mTpLbGl6DqWktprKGUc+5iQg0CQhihtAkQxARc6mit87KQuiaEAMFNgJ0QQlsDAHS935tMJqHnxn6oKjEYDKbDcTtsY4zzee770aCznmWZS8JW2xFCgCNc15vOkjwv/TBmbgtTxrzOaCajFgDiGDmtIHY5rhy+vr5u3Xg0Gr135z6lZH9//+mL51rrMI7arQ6xpizrsiy1wYRw13U4d0llUaWUlUVlwjBsyjWNTg+Oz06VrNqdkDFmEURx7Hh+Vom1bqCU8oNWt7eOEErSPM/z4WiyubXT8C2Ox+O1tTUXvKIqe73edDqv0gpTQyljlDsepxTnRaYrJaVCBtWZsNoMgt6Lz5/zihwev3Y8NpmNPTfClI2GE6VxEHZcJ6prRajrumSe5ghZQkiap9h1CGecuARhbDEYa7URtc2TohO3QFRlOaeIYWPzouCUEix83xX5BETRi8MffPs7UpvZ+UhV5cHR698+fhi3A0XMPJtuba8fnhytDdYGNwc3dq/NRpPNrZ2tjd3Z8eT85AyJAstiNhnJdJ4Z01/bKOaExrFShefFGIGpcctzfCpnaTI9P2WOP5ocC6AWueO0THNdS7escaULBNh1GeFBGPcJ9QuhpDZWaa21kJXSQulKKWWMNsYgrZvqDk3QS0PVbK2V2hhjtbEKwFisEbUEAAjFiy2gXKUKRgRjXFQlAA6CgFIqtbZSIErQZZKc5bZeCIEQYoyhi0qJC4lHmVUaEAZMlQErlLXAmQsWaWVhwYKAEEK2CWinC/5aeyG1V+MMEHrj6m6OULKSjmjBLvSvadLtLlCLNGZhVVo1JKzqNrtS23r5Q2MMvuDkQ2CgyZ8HQIDAgpKiKanKOacEITBKKo2qN9K+4cA0jd3YXNFVDdmzEHUz/0JIuHCZN1xq6IKcqtGgC7MNfkd6CFwEny7v8YpShMuaslHA78QWq+Grq/0saiWsTM4CqKHFJbTWhBDOqcMoQYBBOwT5nAYOixziMEQpIGSkvFQ3Eq809K4GSCKsjDFFOR8Oh4dH+6PxWV2XGGOtF6k+ixBICwDQ7LfwBVP48i6uQNLljS8xdFUKgjEAVtISBJ4T1JVuWNFOzw63tzeHo2OEZVU4nucSSrXSCCHKOSXUWmukQQgBQlgjhN5kPCotEUKIYIwIwRg1FR00QshajIjBza8AN+UekFUN0QIswnoBwFoE4DjUyJpi6HRbYGpkRZmXd27t/vGf/eD1wTM/aE/G4/l0Nj6p+71u7HuckkoaVdb/8T/8h//T/+H/GPrB/+5/+7//4rNHqUqBwPVbN6UydS0A2OnprKpEWZZSa2PM6DQjFPmh57aC3a0bt7/7fl1VW1trR4evtaymk3MctK0fp0mRYdLa2cWAEJCnB6ez6aON9a1r3/w4DkOtLOPenfu3Dk+O2+12Niun06obX5uN67rWm1v+ydkEETeKg04cCkFEWXU6ncNXL5XSrut+/vf/pRLSgD0+etVdGyDSrbWa5hloJBqAyBnCVtoCMUQ4QlAQzkxdz+ajXuC1WwM/6jiqOnz9GkuYFaNWu+8GXUpJmmVOxKrM8MCV1ji+bxDKstT32mVRcUaMqesy4xjarTixFmn76PETasmtG7cZb//9r36ncLh7+16aZiwIn74aHxz8vRK1UVU7DgI3AuLUhHg+Q4RU6iirSw3EIYRUCYznsR9kRjmeSx2eJFlepFJKz/MIYbKWDW2RiwmqTTGcY0Q9xDUILZUV1lpLLDCMQBqlFA0jP8sTpUVVF5RSQhHCFpBZW18bjs7m87mQ7d1r2029E4TQYCvQ2nTmUVZUnLsWUFnWVZ2srbfTdMoYEWA7nVanFQ3Pzuu6jmLmcOZ53vbu1q2b12/fvvnq1auT87OqzFw/aMcBIaiu5TzLCGGMMUQxIzhJEi3F5vpauxXt7e09fvSltiYMwyjwhFJZljmMYrAuZ0cnx91uN/ZaQso8z4MgaHXazOFCiDTPxtNJVuRenhljhJLT+SyKQsaIRcZiJEQlpdQaGsOGEkJrDUYzQhG2SlfzbDopZRyOk3nme20APJsltbBFVhLCkixFRDPqGGPKshJCcs7jfmi1aUgim20HJYRh1mq1tNKT6Wg+nQWu57rcYTSO47ZD5vNESbt/ePS//l/9bwzYk7Pzv//Fz+vXlbZqd3e31QqOz14XWQaw+eDBR9feu/btb303DkK0ezN0/JODw8Hm9myaH5+N49ZhmZXH5+MwkJa5vV6/zbnnB0IIsEYZm2dZlmVVVQLCyuiqqmqFDFGylqqWGDOPUUM4JczzPIfxRoVYi6zSQihjjFZyGfWGG/mIL/PEXfCWw+X9Maxsbd/YD1bSIy9wAF1EL2rd6DNOMbpsv30DLy7v+FdF/5Uzm2z7K1+hy7kMV9TeVwn6KzeFVvIb4V3tHarora6Wv7ULaij7zvtdVZkrm/h3WB1WZ/hNzwg1Crs5suIDendkyeJa7zIpoLfyRJYHr1z3CjhYhQL/uPb2ejWtMROu5r80ViWE36zmu5HBSrcA0BC0+L6vpaqqajabJUlS5U3hlTcGDGNMs1Bfk5No32Hnf+teVmJEMLI88BFCrutaaxljGAMgY6wyjTcbkAVtAQEChC2ARRgDMoAQvLkRDADIWMBm4Y2wAPitwFSjAaOmxKQFi+ybHq5MNVzQlDWOvB9///svHn35n/7qr7L51OMcc6/dCrA1QlR1XVJGBp0+ouhsMrJKt3ot0oqCMESYASgloa7rWkgpdV3XemFNsdpiVMj5LCeY5Z9+iTGUSYGJ8Vw3DuK6FGDQ9es3RSXTNE3TosjzPCuzJBP69FUy73a7rbhz/fqNdqu/tt6P40gpgTHM51MpJSE4TdPDw8Ph6Kwoij/7wx9du3ZNMF5VVRRFg8HafD6fTCaYMj8MgiDgjKVFWQmBjHUDHwuz+pA0S6mtAWC1FP04+ua3Pv72hw84MmdHh+Oz4a173f3Xx2lWxp11DUzuH56NUoTQ8PTsvffu3719Zzqdvj7Yr+sSA0FAMIDLHWRsVVUEId/zuu1OHMS+7+d5LoQosnwymeR5UVWVMUYyTikGpeZJapVk2FACjucm86SqZa0tYq7vBb5fS22klI7vhmHYrF6aptYgY4zWwvcC13Vd1weAsqzzPK8raYwRpAaCli8RQghT5LoudRzmuhxjzBgJAj+KAs4ppfj8/LTVinq9Tr/fZ4yNx8OyLNfW1n7z2596QRTHMeF2lk0o4UEQIUuKcioFubFzDXSFwQ08V7Xa0+l0NhrOx+Pa5XHottvx+qBdZK35fOh6ntCqlpXn0FYUZkVdFIUStUaqyPM4DG7cuOE4zt6zJ6Pp5NbNawghi6BxObcjv9PttiJfiGpzYzuKIkrp+dno7OwsDMNOp1PX9fHxcV3XSinOeTLPGteGFJpSXNUVIQQzrK3R0mjQRVliAGW0QcZIyRmxVhlQZSWDgBS5mIzmUdjPZsXx6dBaYgzyvEDURjY1KQx43HF8jxBSFaXLeOB61CfGGK2UkQqMrYrMaoNAc4rAKoQQJQiDFkp/7w/+4Mc//mPPC3avX/ubv/nb8XSaF+V0OqlV0WqFjLFBby2Ow9t3bty5c6u709/a2i6z0nHdMi+TXG7u3D47z14+e3k8TupSJEJLLOwkFYjToAUW1xpkLaSqy7Ko6pI7TrvdTvOirEUhNKWgylpUFeeuw11pmO8HcRx7bkAIAWMafr8mOEFpZaQyTVis1QAgyxpWVBe6yDRrKt+hC9v1KkpYxQpNW2r6ZboaunAoYP6G1RutGO3xCnfCUm+ttncMyVqzmjLQRCOudL78Cla4E650u/QXwAqkaLz+79QWX4UPlq6Bhdxemujhqv5e7Wc5quUwCLmUt7k6sas9LKfoynXRRXqhXqnysPqteUvbrcrNd2KUK3cK79LrXwOtvr5dgTXLTpYooUl8aD5gjJn3Jl4BrZh2lkGO6HJRCdSQjxVFw8HcbrejKKqLcpFWrvWirIMxC6/V13pUvgYiXFI/YLXWUktKY62153nGGMdxpASp3rBiN+afKzYYdAV7IQNAFvNjNBCMAIPR0KTxNoUeGiJn3djQmpGgBjK8cV1Ye8HZ2oQHMYwxELIRRz/+9rd/+j/9Va5qbepKVLmDojgwtla2otRt9Vvn89Gjh08wxpqTQXtDSlnkSgqVFzJJMlErcxH+jDAmlCBjjVYE59piXgilxbPfPvV8RonBVPs+31zf+NM/++P/+l//9uHDhxSzVtwOgkgkxeMXr8JefHp8IoSK47jX7Q+H42Q+DsOYEjg7PW3y7KwRDqNRHDRB9NevX8cY12XFGHMY11pHQTAcTwmjDatN8/xc5EzZhm3ZWou0Maip1mAalBO1Wzfv3r5x97YRZdSO3v/Gg/PTF8bieVrO8iOlyehsUpcVsbgVx+/duSul/N2jhy+eP9fScMo44z7TAXVEXVulXcKQBaOU73mh7x+8fn12epqmWbfbjcOooQ9grqPqajgc5em0HUXt2Hc5m+cFIcQSJpUAY8NOq7ZoluWcO5M8byoeQJMcUCtrrTHQ7fRc122SDOq6LsuyKVYsjaSWEkq1NkIKa21DZEwpxZ1Oq8lODIIgCDyEEKU4TfPhcGiMqaoijmNKsVLid7/7omZnhZoRvkEIq0QS+BF33bpSk9l0rTMwtpaizObJea3u3rqNLfzsF7+s69r33TKbcpdhDHmVU1TXZVXWIklLwNT3IlGruigs1xI7nuf86Z/80Xe/+90nT57Mk2mv39nY2HA8bzweCyEIo437GQDyPN+9cWMymRyfnY1ns0pKVFVyPM7zfP/w0Fo7GAwI50cnJ0VRbGxsdPp9o8pS1Bhjj/qMORYhZCTGwhjV2JnLqlRGlaLExBAG7dZgOp0eHBx6bjydpPPJPGr11gcbnPntVpUkydlwZJT2w8D3/KqqkvFUeE4QBA7j2ILR2hqDjDZaMkJbYYC1rooMAYo7nX6/+40Pv/Xee+9/+OCj0Wiitdk/OHr06NHzly8oxbUsEVrnDg7b/v3379y6dWN3d4cEnAIHK1++2H/y+EVdyjCMkdPqbN1kcZuGaLu15jgOIcxynmlcTHNjVF2LsiyN1UHQbnVaXhQeT/bSUlS1dh1Si1IUuZUSjAbuO44Tx3HohVohKVUDFASRyFqjQBojpZS1aKgwiyxfyl9CCCC8BApLibZkQb5wlL9pV7QOWkm4x5eZcJa6wVyYf2EFKLxxKFzWvhdBZ0gIAReEB81o7bssClfgAqyggWWfVyDIapDj2+1tmb761duYoIlCXx5ZvSO0Yr1YAoWlwrsCEa4o1NVzVoe9GlbyFuC4GnixOhvvVIHvnM/V9b0CDv4RWGH1vlbblcQHAGgEPb+Y9lWI8PYqL0fSEHlVVZXM5rPZTGvt+36r1UqSpJl3uwxuBYQQUkb+gwN+e1ou3c7FwaqSDUtsI9w451pLhIwxqvFkEUKUgqaux/L5v4gfWvSGgYDVCBAYYzFGBgEyC+MBmItakSugdoGWl46kZYGoBZZFCDVk0YAwWES1und999sfvn8Q+0rXZV3M5xNOfc0hjBzAGJj98tnjX3/xqecGSVX4Hp4neZrm1qK6kvOkUko5jqctBgACGAxF2Bpjawm0Mv1eR6rq+PVhQs14krseRDEYY54/f/p87/k8mwd+Kzk9xohjTFu9nsOwlDKbTs8OD18xBwA4d1wWZVVtZdUK/E47EkKoqvR7XafdTWbz0fmwCcXDGB8fHx8fH1+/fr2WmjOulNJCMkKZw4uqKrPcpxwsLElfEEKAEQDkea5FPRyP/vbnP/vkk19OhqcYzFp/0I8cA9gS+mr/4Hw4tcBdP+Kchy6LXJ4qpesaaR15PoCppTCgCCBiIArjVhgVaQ7GBtyNgvDl05fz6YwQ1mq1Op2uUsrzvDiOj47me69eTWfjWzeuh624UGZ8Pun1+4R7VSny+Zw7XqWklJJxdzabNSHSTfz+MqC+1+tJKbMsn81ms1lS1zXBjDFGGWOEUbZwuTaSoS4t7XTbnW4bAJRSSsssT621Qtau5xirAUFZFRbMzZs3r13fffToUed6dHRyMk1H1lpCKGLe6dn+06cvbl6/NehGJ8d7WZKPTsaz4fS927ddx8FScmtAVKPjJM3nwlTKyKzId65d63danXaUF3VZKmTVoN9ZX9/cbMV5WRiwv/ni84cPH/b7/fsffJAkyXA4zIqcEOJQmuZZcpY0QOn8bPLy5cssy6Io6nWDRt65TrizfSPLMtcJ60qMhjOMca+7fuvm3bKe2cPDNCuksJgRQilCBCElZKqk4pSUQlJWz7OZBh2ESEo5n9etaWqV9jyvFcXdbnd9ffDevfdfvto/OT0NAu98PKqqilHkdiI38LTWIi+TaqqV4ITGURR47mB9vSyyMssdHLC19vXdaw8ePLi+u7tzffPg4ODnv/zFfJ7WUjHGirJuluP2zTvvf3C7qLN5MkymyfnJaV0W19+767t6c337s08f/u3ffcLdsNsZEOqC18o1Q4jwMCKOY4wprR1m5WxWBkFgCM6R4k7I2t2akekkO5vm01yCsZihplSBlNLaHIhLADFMKKXIgLJKaU0AkaZAoQWrjdVmYV24iI97s1dbVD0Gbd+hGq+I+OWfsKKiGjgPGFNKOefNKWYlkQ8u9tPowlfdSFV0YYRYQo3lDw2gRTTAiouaINwI+iso4Y1EeJfpHuN3+A7evs13tncqy9U7stZi9EavX/ktvKXYlor/CkS4Aj4ufzBLfdnE5WVZ1mhEexljNYbrZf/v1P1wGUhdOe3KV3bFOPH7TNc72+pDtcRMjRVhtfolWjEerAK1rwJty1GVZdnpdMIwTGbzg4ODJ0+elGXpcecdw4B3dHKltyvT0jzrAPDmv4vjhFJloWHqhcuwRq9wezf/N0L83Vc3uunZYoIBAJTFGFsKRl9CCRcFIwCZxrCAAAFCYK+Y2JpQBgDAVhtjdNjt7yj1T/7wD5++WLNWKiv+81//lahzQKbVivKqLqry+fPnZ8NxHFmhIM/L2SybJRnGxGiQUhuLATGrlQZrAFFksQVMqQVsLJrP53Ec9fv9qMWVeUIZBBFZ32xpU5XVTKqCOy0LOM9SazCJ2+NR4nlepx13Oy3f9wkhQgilZBQ5N2/sbm9vB0H44sWLNJlNXJcx9p//82enx8c3b968desW5/x3v/3y008/ffFq79btu47vzdNEI6iEAgCttWwi/QDgIutqIesQDoMAhwGm6PXpcZXndZn6vltjq1XYbnWZ6yKMq6owugIAzt1kdH6w93w8m6bjYb8dDQaDWTI/PDx0GQGAdqt9/977N3euJdO5qOsb126eH59rISM/cIMQI6SUYoxxyoy1xiLEeRB1OoON9vqGlFLgeWUxw6yo1elwYgDXUohaCfPmfaSUhmHouUEYhpy7rVZrMpmkaTocDufzVGuNESWE8Ig3PAWu6wZeaGDhAqatVkAISdN0PB4XReE4DqW02aVFkde+SIaMIo9SGgTO85cPp/NZmqZa6yiKtFZpmk2mZ+3QTzsdgd3ID0XoJePx6fGhFDqdjCaTUSUrTCxhptWN4nbYajmqTjAKPM+JoqjTHfR6W+12L2p1Tl8fPnz48Oef/Kosy9lsduvWrfZgDQBOzodHpycIoX6/TwjRCEsh81poRPOycv2g0+snSXJwdCCE8Dyv2+1WQkptDKC43QmCIIxbBlAYtQkZV+U8yWptACFkwJZlrpWoyiyOfVEb60Ga5EoZyh1jbBTQzY21nZ0tKS0hlDNqtcLEjIZHWZr0B33u4KcvnpdVtdnZNHNQxtZal0VW5QXBoOpKBp5L0Xw6JQhtbW7ev3vnW9/61t07txhjn3z++a9+9avffvGllOrV/qE1aDgcamXDKLh58+aNG7fKKhmN3VYrdLl3dnw2zcvt7d3777vzWTGaZo5HgKmtnf60nMhKK6twZQAKZCwggxCiioN1GWMkCrBDS8LPJrOTk1NjqHVbPmd+4DHuUcc1UkkphTGyFmVegEFGY1VLKbW11uo3OhgjxChtCmm73FlVrvpivw1vJba9LUbfaLtmo4bfKH7CWCMlrVFLy8RqAJ25TMK4PKjMJVSxGLBdCPfGhrYMRkMIgbnqd1haCK7o2otwerJ6U78PRFjtZ9VufAkqvctov3ra0i/wNoK5cq2rIGylrUKKxpBQlmWapvP5fNk5esNa8cZlc+U2EUJXoNvKsL/OGXEF0PyDU/d2W529JUpYpm8AQLPEDe33O3mUV0eyegtLHdw8Wk3GOABUVWWkWtR9uMjueTP838/18PYcXhoJWvTUeBwAgDFmraWUGquVkkpJjPGF98wALAi2EboafKOXb4PWFmNDMCHM2iZDAyNkFxBhYWBA1jZBkQZQ45JYeBysBYQpoCam1gKgpgQuoeC53ve++wfWmleHz6s0lXU9T6au53luUOQlAj6fl6Cx1qAVZGVRCaGUwtgaiwAjZJAGq6wxxmhjLMIEEDWoWcGkmLsRaQ+iwVqclseU614/7PZ8xyW9rpvMzzDkG4O+agenJ6Nsfr7e21qQbFLU6/X6/Z4Q4uzsLIpia20UtrKsUKLOkzR3g3a7vbu9vb+///Of/1wJ2el0XNcz1vY73bqqonarFcVZVc6SjGhFMaZBYJpc94ttTxOQjBCaTCaEIAwakHI5D3ptx2GpFqfT8dl4hBDhPrt+e1eWEiHiMH97awt0cbr/MplOwiheX+v4HjO6PpqcGqmAOtQij3GvPyizXJTVwevXSshWFIdxazabzaZzx3N935dadXrd++9/KITodttSo0Ia7PjAuGWM+iHm86oShDk+d9FlarWG6dxxHErZeDyeTqdJ0tSaMsaYJr9skk4ZY40VLWqFnucxzg3XNMsTzvk8mQ5HZ1VVNRzDhJAmvm88Gd68efP73/++EOInP/nJ559/Ptavu90uYwQAceYQyoIguH37JkfMWNnrr0dubIRCeuv4+PDGtZu6qvL5rJKy0/X63fb2ja2tnQ3Hd5wg5K43nqXjSYrBFvl8f2//5Ox8c+tGqUxRq83taz/6oz89G55/8tnng8GABqEXtaSUQHl70I/jOM/zw8NDTFmr0w3DkDA2ns6OTk6tta1WywAyxjiYYMp6gzXXdctaHJ2cgtHn55OskAhTIVRZCSllVRdCVGky3t5a44x7QUtUmdLWAPJdx9nobW2vx1EwGk+NqiujzVwPhyfHJwcIY0Z7QchbkZvnaZ5OphNhrTVSYAuMYkao5zCXM2SN5zg725sfPfjg+u61MHCePn747NkzRfnpyTljjBDucK/T6ZRlPRyeCVkVRfn8+fNur7Wztb17bevGjZ29vb2Xx0d7L/efPNl7tneIqS81FgZP0loCVwQAI6FUWeYYWcaoMcbFXplJzm0QBAbRNKtmqSzB9cM49lqh47gMS7dw60rkebOzzPMcADyv4sRFgJsAO2PAWgvGYlgobNZYJilZClljjAWzpLpbaoJ3bukaf8TiHISMMQTTxgtrL0LGpJTkwhC+tLE3fa6GKawelFIt3Q3LYWB7KSQQmqxFY40xgN+9CX5beS8vcWUYq8N7txq4OHMVUa2e8PZVVoeBLjijvkIxX96wvoUSlje+enyJrpa1lBpWlss62AKARZc03FduZFcCTd7GLvAVmvIfgRVWB7B8Thp5pS8oqJfMWqvzszrPq3OyiskQQr7v13Wd5zlCqNvtdjqdPM9VLd5essb0g74CirwNEa7MgLUWLsJ+AUBr7XLsOE6/30cIcc5FXTmOo0xhQRurrEGAjLEIkMENGeOiHiZBgPAFq4eRarF8CJo9oiWLbEwAYzFF5gI441VUZ621CAFYbK1eiVRAWklrESEIAUFgIK8A6Ob27rXRyXg2nCWzjz/++MWrF8k8QwCiFK2AWV27LNQllEkNXqGMBGy11U3wpUaglDBG64YGAsAioq2SGmqJ17Z62MdhEIRdtzUICK5aHZpkJ2KU3b23ORgEo7PJdLSvJGYI1jZ6uzs3Xdc1Rs1mMyUEwdDvtRiFhqnv9OTo5ORsOh5Zi8BqTlktijzNKCZhu9lbO1LKpnQsJ5RSShAiCHHKLEEAUGqNAaxdPGzmwjcaRhECY0FpKwxGuazTOhdCkMCrqwoDGnR7t2/fWOv3OXGstteu3cCYPn3yO4q1w8FziMMi17t2Oj6thRifnX+W/2p4dLK9sVUV5d7ePkVUS6m1pphMkkkl5N27dwkhlagwo2ErLopimubp8Wkp6iAIXM9jjCHCW92BJ+s4jjHGk8mkUHJZfsgYUxRFWZZSaiV1k8nfvC/WWrBgrVX1gtO9rus0TcMwDOPAdV16dHQUBEGSJM27sZR3s9lsY2OjkbntdvvRo0fPnj0LgiATQRC2jNGEoF5vEATe8Py8rusPP7rv0WBzYz0ZJ+lsHob+5Hz83e9+u8Ock7Pjqs4x1Vk1T6rZweuXtVGz+Xz35h3Gvbwy3I19z+v1sTFmMpuvra0dnZyeng///F/8i+76+quDw0mSKqW44/lRrIyZzhLueAYTaRfV2IwxTQXITqcTRVGv15vNZo7jYIwbNggAIIRwzo8PDrOyCoO43e3VlZwlSVEUWsNoOJlOZLcjfdeP405iVA6pMUYbSSk1Ro0n58dHp6PROIg7hBDu8A8+uBdFURhHQoj+oH16enJ8fLy5tq61rOtaCsEIHvQ613d3NtbWtjc3EZhbN67t7Gzlaba/9/Kzzz79m7/5G0E4pfz+/fePDk9evXo1HI5d193Zuea6fDDonp0eKVkVuat03et1hBB//Md/+tOf//1/+B//X0VtsBNlhbCz7NXJFBDjrhe1YmN0mpWM4gATaw1gOpvOMMZrxHEBZ1llMR9sXa/KlGPkcoaNtJVEWFmEKyE1wWVZGqVUrYIAPNdnhCKKhFBgFsHP2GJkrEEWISQvQgRgRWsaY6y9pLfQ5bYqTK21zUNMLkIIzUX2oBDC5dRc5DXA5Q0lrCjs5cEldl6V6UvXg1lpWiqttRe4cFljNZe4ErS4PKHxjDSt6Xzh7XLdr1EYv49GfPser+i51ftdOe0dgOaKRoQVLbU6mZRS13XjOC7LuiiK1f6bdbP294rqf2tIby79VXjrH92WM38xyMXztup3WAVD6LLrZKHj36W53yDIRUUoXNf1fD6v6zpwvTzPjTF2xa6Dm06+urrC2/PTTCpAU3j9UlNKOY7j+35Dpk4prSvLOS/r6sr9NrepL4pdoZUAVXTBNtb8aYixlllr8QV0xlZbIGiBTpogvaamOHkH/rPWGq2UshZhRBeBEcwDVQO2t2/d8X1+NrrVagf/9i//8mc/+1nghZzOW1FbVrhAJk2KupAGMqWFtXph2sMAxhqjLLIILMaAKSEENUEmUolczGWScr9jMKeOEaIYz9Jf/+bnD96/u7XdY8S83nueZanPQ61slc32Xry8ffv2xsaaNfr4+HB/rwxCL01TrawQ4uxsPDwfAVCHe3maDe359u7m6fGJ67rtdhuMLcsyjuPrN27USset1snovJYiCALX9/OySLKUXTBzNO+DWRQAg9ks5w51XUYYlVJIUVKH+q1IycoPPVHV49l4bdDb3f6o1+mms7TXibU2ceQh1PHDiFGkDMShf+fmrTRNp+PJbDI/Q3h3Y9vlfHw+3N29LqUWVbW0ljHG0jSttGjYQvOizKtyOpkrozu9bsMp1+12Oac+Dxu4OZ/PW61WGIac8wY5KamLokjTPI5adoV+TQgBFiOEOnGn+TNJktls5nlet99ptVr05s0bh4eHAHptrZekM20qh+BalJ1OTCnZ2tra3Ng+OhwqQb7zrR8/fvy4w8tsWnmeRxyHoZYo4NnjMyNVedfZuXHt+HiKte2tbzKE795978nLp95a+/WzL589e4YZDYJAGSQUQZTvXHvv3offeP7yxd7hw+eH51r/emNz8/79+3fw+unp6fvXr0/nydHT5w8++sYfff/7z17ssZhrrdvdfiXqF3sv06Tc3d1dG2xVxawsaqFNVVWTeYIw5U6QZhUmjlQYAKRARlPmRWD5eJSmidzeuu77YVEJUUpdIVUCAR76MUb29Gy4de2BQGguhQ0caaoYQlGI09cjXaIXL17Ms/nHH7dCBzCRHBSqZyYXke94LebZuOva/aOZVuLeN+7cu3P38e++dBj/X/6rfyEqub6+bowtyvrw9dHZ+eiTTz/74rePFfHLOVrfHJweTV3uYaJv3Ol3uuGd+zelKh2Pf/75cDIfD9Z2p3pynJ0kkNcERmlaGTNJ0ih0Iq+lC+locFyHIFoMp0JJ7jnI2GyeME6sHboMOY5TzIfZ3LZaLd9vlWVpDamErivBAGvl1zWShgb9DjJVnudaGuK7DncxxkotWCyNlUJWTfg3bsQdgG647o3GGCOCLbYWWcAATVElhLHUWktjJSCNiTFyUVagMa4uSO4oNcbUda3UItwGU3phMJDGNCYKBmDworCkJYQoaay1jQjTCkmwGGPMFhJNggGtFlZoQoUQCGMLIFe4ijElRuPFXgqaGLGmeGNjx166CRa2WQvQvE5Le+9iZi6sBctd7BuNZQxuKm2Q5R3ZxpqylMaX/l9J94fLW/xVTbn8Lb5IXVtAlqYIB4BpSJyuCH4ARLBFoIwWSgKAtkYoWZRlLYQ2RlvT6FfCKF7U8nhHbMTbR5Z/LgkkrkAEay0AWjlzMedoZUt+Ba69feOrRxazSmnzFDVAsKke2eyQFj+0ejFnsCCqBGtNkzmJUKO3EUKwpN5CVCvteZ71oSGoQYjM5/OGYs4aJYTQWgFAQ0FpzDtiVq60VcTAELs4mQAYA2DAaGNch2dFfq978/4HdzGGPJsHvmtlpYTEgCgmWmttFEJICVkVZavVwowBWCFruEj6AAAOyFgrGi+hRohagpjFpAleUFYDGGQQIIwtRoCUaGJWoKEQs8sVIWgxqwRRRixStVLSSN/gusbT6dRxOnfu3UTk2U9+8pMqGxRJL080RWuHr+dSyqoqqrLgzCS5BWAWqEUX6U5gLWiEECEYIbBaAaKcOQghLWwoy07calEbclvkM+TKmbD+Wqh82D99rXUd9d3JdFbWc6RgPk0qqU9Ojq7tbE8mk2Q+bbfbUtaUUs8LACFKqR+HRaXAZf6gHcXtre01L+C/+fxTaYo/+NEPLMDvHj189OrLk7PTm7fuII4tgaLIuOuXuQjdOKulMSaI3VrkyirmOllVONg1hGAWSGFMWQcejxysVIHzJJO1IJRi4rmulOL45CBLZ1rIusqePn1KsL1xY0MqU+QjrfU0mQcG9/v9HnNnftTrDiiis6pa29qdlfX27i6hdDQZKwaM8+cHz1zfY9abz+cA4GCe1hkF5Ls+0XY+SxKlJuejOI7b7bYoBGMs8GNDM0Rwu93O87woCquskYYYINpqaVGloTSkMkwiYzS2oAlXyiLCO+0Q4YUAq2tNMWKtuEsIraoyDNoYg7VWCNHtxIQgaw0gg5AmFILQHax1Xa2n02lZltl8AkAYJlbZ9cHGy6cvxyeT29dv+NyRpTg8PUMWnx2ffPyt73zjGx9Lqf0o/OCDDx49eXx2fj7P0ocPH3PPL6sq8FulqJUsEWJao/F0rJQq6uqb3/oYU/L8+fM8zzc2NhrG6TiOuajjMNLWjCfD0Wi0s73uOg7GWNTWaqONMUYx5lhru92uUXaohoQQx2FFUY7HYzcMs7oU1taVnI4naZpLWRurKCa+63EGoE0ynaWzOWPIZa4RSmkByPQHXcZJnmc3r19zfO/LL3+bZZm1tt1ut1oRQkhUFSP42ua2VmprY9tzfM8NRF1/+unnk8mkKgVjjrJGSJ3mxflo6LjB5u712TA1RhGCsnz2zW9/9PE3PwhCZ+vahrXmpz//O9+LNrZ2j46OpJQU+Y+//I3j7mhFN9Z3KZ7V0ghRAVKEOhZVCAOhhgBYq1WthZJacyDyTfA/wQ2ddpakhBBkLMOEMEIx4ZgYz2BAZTZpVFEYx77vA4AAYaXVWpuLDEbUcBxpY4yxFF2R701r9tmrnvVG8zkOQwg1DD9wEVndRHo3cVTN7hBZSwgghDglS9W47A0Aaa3xynGE3oSwwZKoEVCDjjVSCwK+FYLhZlO4St1oL8ihrX1T06s5E+E3Nmq04hFYBQ2remJ1X/tO/bGqzOxlA/jqCJf9vFMDrZ68et2vObMhBWoci839NvlgX98/fLUW/P9jW94sukjyXI0BfGdCyuoirh5ZLkHzQEopKaVbW1v3799HCB282ltYksAsz7moyPzfYVrQIjwCNYXx4HJIynK0qw+DXWmrU0EpNhojgxBCFqAJZcAYW6ttE5qAtQWCjUVgDbIUNQctQJMJixauB20tXBCGWrBwYYgDhZAlBJVlPhqd//SnP/3JT35ydHS0TDlp4IXW2hpMMF8d6du3DBf+o8aTiBBCwIwCazCytN3upcXQc1Sn1et31h5lj2Uh1/pb3AbTk9nwdKwlRHGrSLM0zYwxrbjT63WzLKuqKggCqUylCq2sUsqU5XyWKG3y9cHte3d7a4OnTx+/2n9NOGu1Wn4YJnnm+F5InbwsonbLcVgch0IIl3GhgSBA1lhkGEHIYb7nV0XtMKJBg9ZGa2FqKXKCrcdiQhlGSAhzfD6pKxN4HrJw48aN4SxnjPNKj6az6XQehGEYttNsmheFHwZeGCVJ9tuHXwIirutrQJg0ZTwWOxDCOOdcZLLhpQ2CYI3Rpq6jRZBlmZSyKUdSlmUYhmEYur4HGDCgLMu01gSQMDoIPGOMUFJqLYxWoDVBGFGKMSE0EaXWiiLKqcM9jxDkOA53Gf388y8JIXVdai39wAVAZVlIobM8sQYRQnq9nus6Qhac48GgE2vPp9F4Ni1Z6ZHQWru1fv3B+x+sr63t772KgnYynU2G588eP23HnTwv//Iv/+M/+fM/973o1d5rSpzffvGQOVwZA5Y9evj8fDi0CNzAN8YcHw1Hw/lGa50xZzgc/tEf/3GWZQZMq9UyAEmSYIyVFnmWlGWJMQbuuNwRVQ3GWmuUkGA1wYQzFoZ+XUlGaClKSjFjnlKqqopWK+qsbYzHY2E0IlApUYmSMgzSIIyIsRgTUyvLsIc4AWSNJpzqymgwnU57MOjO5/MoCgDgmx89aBiowtAHgNPT09Pjw+FwWKSsqqpsMnO4W5Z1v9M9OjzLy9pa0LZK87KoSmFsLZTv+04QtrTgPOp0WsZu//n/8CdR7J+cnxwenA4Gvboyd+58+ODBg08++WRvb98opiX9v/yf/68IIUqp6wRegJSxRVWW9dzjIWBAxFKLKMUCgBpEMV9Q7CmNGSOYoEYgWEsRtmgRLo60EVVtlAYALQRCyHEczjmixCptrdV2YQYw1i5FlwKrjAagV17+Row2QWFwEUywFNla6GUA2rLyglLKdV0D1l5YVsEYYwBjrPWbUj36DdUxUUpRghchZsYA6GVM8jJIwiDcbPeFNkEQoBXKmaX8XRW1S3eDtYYQYu2biIQmcvNtqIFWvHXL48ve7OV2RT0sb3bZ1Vehjd9H2axeCP0ezo5mZt5Jn/D2mVc+fNXx3+e6bw/79zlhVa+8/SdcJEMu8cGbpUGLgS3b10STNN02j7rvuTs7O03ph9HZaZqmAIDRAhoaYxoGzP9WnGAvOT4WY2ts2pzzfr/vOA5CSGujlEJGL58Ne7m22ZXnaolHFQJjtLXaGA2YNCGDYLBWygJYTJBBiFgLuDFeNUEWeMGjYOwiOkEbi5CxBC0CGqw2VmujVCqm1iKDVCWLl/svP/visy8fPZzNZnHc1rrxqmujwVowBhlDrJVXUOyVP5fGsGZiq1K7LqjalpmM/c50fI4UVInqR5t1qqfnyVpnnUIkRVYVQDEr0qIoKgyorPKqqrJ5whgjnFHCa1U3xe0ItRZhqU1Z1j//5Fd//6tfTyYj5jr/+l//y3/xL//58fHxv/vLvzQIyrJUXEsjMYZalJhaWQqCHZDSKAtIU4ysEUWWi6pGBnGMMADBxuWEEMapw4jV2rcEY0IJBsC41JoY6zjOz371q7Ko1ja3fGtZEMSYlXX9dG9/t9d//fo1Yc76+iZxnCACoSDJiyAMMaXGGIsRxtRA4xHjpc7LsmzkEnOdBlnWUqyvrydJMk/T5nWu6xpjrK0hDDNMJsnIdX2lVFVV3W43yYtZmhhtpdI1SIm1RRZjSwgYW2OCGEeOSwKfUoczxhhj1GM+55xhaq3hDh6Nzs/PzxyXYTCNfVjLUtZlkc2l0GAkQ24n6vs8Yg51HGc4HFZVxYgTBZFD+XQym43Gk9Hs7OT8yy++dF33+OwMAG7fvvuLX/16b2//5PR0fXPDIMw4N0AJcYSUx0cjKeW9++1edzOZz3d2Wjei61LV83TWbne1VuPZFCzClMiqLIrCGsUdL/Bdzkg6HRNClNFVkSNjKQVkNAHkuCzLk9k04Zz7oXd2dsYc+u1vfztVNqtLhJDHvbQqEEWB58i6TuZTUSlrhKxqpx15vfWiyLIsBQ9ZZJSRiOO405FggDSvJe70O4HrOY4DYCkleZ5lWerjoHJ45PpS6/X++vbu9XmaYubPs3w2nR+dnUttwlYMhE/SNHlR/tkffez7/t7eXhj5Byev8+c5QqjX6yVJ8dGDb/X7fVmbKOycnnzyN//l77TWW2vX0jQdz8a1mHiBG8Sh5xILAmOhpalriwnnPOSMS8c43FN10Yghq7TWpia17/uRH1hrlZRIGymFEqLKCtXkZRHFOXcwsgi01hasxQhj3Fh0GisCAFgAhDGhVL+1S2s+NBujRrWvRgYIIRoWrDBuAUATWdIIRwTQ6NbVrsybssjQbPcJIYRgpRRGb0gGjVFL28AShVhCm2+1UotyBhcxXwsBjbE0iyA+colmgFyVaysUPasSH11YFMzlikrLkVxS4Csm6EYbLWs2ootN8LK64ztl69eon2Z67eX95TsauhRKsizG9lU9vxMcrB78Khjx/327cgt2xYqD3tphryK/VYgG9tIIr4z/ypjtIkl4wcoQBMH29vZoNNp/+aLZLhst0eUn4R8RfLH6GCCEACxCSCnlh+7GxobruoQQUeu6rkGrJb9Ts2WHC5/LpWdsBShIVVlrrUEGLGqcUdYiKRElzSOGGqMgYYYSjDEmDVNCk/WAkNXWWoMwACCwgAiyAFYprYSspZJlNldKaWWTJDk6Ojg5OanrmjEnSTIEzSUoRkhpI5ryPdQun8nVSVh9ZZZ3QQghhIdBx3V8K03sdzkK6ip9+XD/1vadtr9271sPdrduJOPsqfPUFA+n07msVTrPkvFUSltWkPpVp+N7YTCaTIpKlJVQgCh3EKGYMAtUg+KM8zAaDs9/8nd/191c29xc3762++XTx1mR6hy1WlFd10VZaq0tyHa7XZ1O8ry2VoWtwOU0tXWZV9ub277rq7pStdFKK1nKuqgRIOxoLY1VlBLPIYVySut7yt28vZGmaZJMz56crQ02f/iDPwyC6PPPf/v086elMf0oqrU5Px9yJ+h2+4gypXQlhLUWLAKEjdJaWaPBcRytdZOt4IVBGIaU0qIqHcdptVpeEDQ4crHDIdhqhS3kacapA8bWokSUGKOkVgghYIhgSok1xiBMEcYx8Tnnnue5rospwRgIMhYU5YxhhBzGXdcBpERVa6VCv+05LsbYdd3QDygmSsiyqMqyrArgnCGApr74fvb69PS0zovj1wcO4xDoQa+/tbZGrD0/OQ2CwHGcL7/88qOPPrp+/foXX3zR6XSa9Js4blOHawN+EHtBXFUVwTyZ58cHr8I4+MEPflDVNQBo0EKLKIoaJVEL6VCys7nhOE6el6dHx7rOPW9BXUABMAJZi7ouJ5OJBjSbzRhjZV0kSdpqtYIg0ID8KGzKbCjQeZYGridF5Y1pnjlgFXMdoMzzI6AMcyevUsuYACi17jLOfY87DiNkOh37hBVCDqcTh/M4jt/74IMgjo+enlLauXbthrHEC2LG/b2Do/EkYdyd50WWV4iSMAxbrZYxChEsTdXqrmUP53/0Jz9WUu7v7zdlQE9PjzudzsuXz8/PTzHGZ2evHcfZ2dnZ2Ow6LlI6m6UVBgCttDVKaEZRXlTzpKA8cHjIKEEGkNVVURZlQSlVUtZCOI5D19bCMCyLwmoD1lKMEaHWcRghBGGFasdxmONgjBuW40blK/uGlg4hRNBCpdVGrUrqpSDQFzWi7EogurU2ipzGhdTtD/I8f/Xqldba9YIr4h6TRTNGXSgAtLL5NuatsLWm1JlsAM3lNAR0EeSFMFlK+ZWN3aVASIRQQ6az7Bwh1FgXvkYXrsrBK9r9MlR4Y28wF/kdb/eD3orX+6rrrqrMps+lvnzn+UKKBq5JqYQQRVEURdG4gb6mLdcF3oUS/rtDBHhXiMPFily1Wq+u4FKnNqjLGNNQOK/2duVBvXLFJmqseWDKsvQ8r9frXbt2bT6fp2laFnJ1OTDG+ish1lfe1wXcWRxZWhQYY4PBoHmtlFLWKC3qWi3IIqWUVVVhjFfTPq88JA3QadAAXQTEgFbCgAWBL5Aos4yCtRQoYGzQhWtPL7jSLSAwCiFkEQarmy2GEKquKillJVSapnmen50NHz5++vrwMK9KhIixgJBFYBsNpaxRFtSFTc6+C79eeUEQQpzzVtzZ2NjiDrJQcmY3ehtJyo72zv72P/008PwbD+5ure0Wgby2/v53Pvqz+Tz5f/+Xv34i1Pn5KA79zQ2fUooJSYv85HhYVEIj6/iR4zuAmLYEDHA/midJkpXSokrpLC8rpYESPwwRQUaq996/p62qjk6KMkdAfM9gVGIjvZDtbneDKGTYzOdZvx0zwgQG65BBN4wj12HAHfzoyUlZ5aUwlELc8XqDThR7nFPPIywKsK+L43kmxpbXG7vXpL2djvOTk7NaaaHyeVm6hrCyKsoqz0tpgVKqTUNOjwEAyKJ4OrqgkGlEVlVVTf4Ccxzf95sap1prwKguKwQE2YVtzWqDrCaAKG5SVY0xCAihnuv7vud5MkkXH4wuy7IoaosAY0zLMjPGME48nxqjXc/p97vdbjfPU8fxXNen1LUWgSVgiVYwOx/5QVBVldU6DENZVkibPM3S+ezazo6uywSmoe8qJYajk047/PDD+7P5lDvsww/v/+IXP58lJcKwe2O33+9aTBDGd9977wc/+lGn06ulGI1G//4v/0dlzGQ2wxhvbm9SSofDYcOLMJ3OZ9OJlHJzY6vVau3t7U3HI1Xm6+vrhBBKEPVdQrnDqMNZURTdbtfE4XQ6Rxh3Oq1utztLpiju+62IUhq32oTTuswpxWD1YKdntOIET6YjLQVvRU6n7Xney73HNIwQIdOy8ou8EFITyhCO++txFNV1Pc8LUUnmaiDc8aOd68ZxvG6/a4EWlRrORrMiGyeT7Z0bcatd1aIWlSgLxXncCjudDnNUXtanJ+dKqeOTk6IsXc+jFBeF/PLhb+oyn81GYeR1O9QP2Gj0rJARxnSw5vfXolrqopSiFAT5RjlVobJUElZwNsekaOz5otJlXjiOY62tylLUdRgEvudpqazS1liDMbJAMbaEYowpQ004WJNthgEWeyljlDXSaNAGY4wIIo0zGN5EXK++/EtSRXxBo3kBAVxjTBzHQRQ3ptQ3Bn+wTaIyxhhfmJHt0hlxUQB3RXmvskEvsiQE1ktZvLrJXvzkglh6pS0s1faCWqQBCpTSKyGHi8vhq8kXV7Z0cBk0AMCV4ytjBnSBw6741Ff04hsk9A9qoFVB/FU/WWIIdGHN1l9dE/zK+OFdyODK7vxrMM1XDfvrv1od7erg4V2KZ+lMeQMXLF5+hVY8TfhyGufq58aLUZVFnueNaarX6wVB0BTOan679IL9I2wKdsWi0LTmMWCMtVotpZQQQghBMKi6lmZBjdVU28EYc87fOWmLg9giAoQgjBECssC+BrSW1iAgTMOCq9laaxGxaFmWrInLvHgCAYNRxphaaSFEE1QkpcyKcjrNZrPZy73Xz57vjcYzpRFC1nE8pZSUyhjRUHVhahmmSqorS7n6iK4C0CYrrVm4bF5QZvI0j70o8pyySF4934/8KOK/ewjPy0Jd37nd722tDdo3r92hmG2uTwaDntb67OxUa10TqREAILAYYwqIaIusMYiQUmoniHa6varOhKh+9stPgi9/d3S87zi8KQwRhkEYh/J17XjMGDsaP0dY37jWvnn7xs7OVlWJ2eh0OhZpMnJoAAbaQXh959aN65ut2PV8fvtGVpb52fj05PygqLNalDqptJaYk431zR/86A8QsIP9o/F4FPpRGIbf/cEP//qv/3p//8DzfNcLlNLnk7GSNs9zacFxHMdhlFIgGlHqut7R0dAYE4ah67qY0eZhaIxMqyZbs6h4R5p09wZMVEVplK6KkmCgYLVUSgpRVdZaDn4QBS3PGaZZA0zrskznaVYWTW/0w/fvVlWFsLXWZFm2uTaQUtaitMpShLHFZVbUiIpSaWFlpSlCDKHRbGaN1L2ug/GN3Z0g9FqtuNtpdVrxi6fPlKwZsZsbfdejv/3dZ0Lqra21tc2N6zd2Pu70bt6+9b0/+AFg0l/bODg6rIXqdDq+76631h48eLC92z06Ovrkk0+CINjY2FBKHR6+rirxox/96EJ82yD0wygIQj+OY01hd3vT87zRZFaWZVlX1mqGyebGehCF1OFCCNd3r13f6fXXlFJJljUEHcoo13cws9ZqSp2AhpTiIAjgiBVF0ep2LeggCLZACiHm83lSSyer6qpKaoktUIJaWeE5LvUDLdWsqIQQQkPY9ZU0R+cHRSmfvnz96tWp1CiZl9ICIayqyyyZl+lseHzgctZqtaSjfd/H1EvzajieCaFOTo/Go9N5Mnry+ItWKxistW/d3t3eWWu348PDg7T2prN0OslmsyJPqzQzlIbduD9PCmKAIWW1LctaykQZEeqQaE/VglLqEyoplVLmec45F1VtpEIWKCAwVksJTWwjJcpo0AjZRbJiXddlnjcGBoSQJdgipMFapQAAU7LcECwFlr2cT78q3LXWlNKqqqbzZDqdNoxjC5IAdGnn2kAEgi7y3BBZgo+mE7hwbTStkYmSmCaurdGCC2F0IVf1SuD9cmO3CiyWN2KMAVjFBItvCTb4cj6CeVfc2aoOuDIzS0OCvey8WG4UViftii58u62qzFX88VW/avhcmwiMJS75GpPA1yCDt0/7795WZ+NtILI6OW/PM6zgA7iYkys/eXuWlj8xxjSqMc/zZpvenLB8urRS/62oCN5CCRcPl22KWnHOi6Ig1gghGMXGNMkrDUGIbAw/rutKKRuzx5Wu7EXRrwtwaZpSDot3FwgCi6wBY43SYKzFBlFqAYBg1NQCQgstD1Zra6RQdVVlZWN2EkqpNKuKXNWVnc/K8XguhOHcbZxoSqkmmnyxUmABvVmIr5qKZWvogOq6Hp+fJel0Y6M1mZ69d+/6rZt3kDV5nr/ee/3q+SstSDKvnj8+JNgFwDvba+24E3hhv989OTlJktT3/VarTRzOsryUCjMHEFFKG0wwRczx8zwdz2faiHY7xpxXoi7rinJa1HkURa8O9hC2CNvvfOc7ZVmeH/5yezv6xjc+3L62DQAvnu/PZ6M8g9DJsEM4dTl3CLDZOHn98lWWz14+TfqDrkXybDjLRNJba29sD7zAQZR0O2u9zhYhzuS81tLG3mB39/qX1d7dDz6grj8cjs/PRvN5HoXx+vpmUdWNBGuyw4wGBIuarg1pEmNMaNVI2oZLUQghtW4KlDcxH5QzDKjBl3Vdi6rilCJteq22xzgYK0TVZENgbXSaVdpoaaqirqAu60oICdqqShd1RQkFxjFCtixrQGpzax1j/Pr1a4JZq9UBQHUlhajns0wINZ0m/Th2Xdcalc7nlGFjVbsVtdutLE+09j/66AEYlcym6XwSReHp6UnQam112nmRBsHN999/797993r9Ncflv/3dlwdHR//1b//u9cHB+w++8eGHHz74xkfvvx9du3at2+3++3//76uqOjk/XhtsRFHEeT0cDgeDwZ07t4yBbqsNAJzQKPA3djYfPHjAXWd/f//14XGapmVdEUKUhf39/clsVpZVEIXdbpc7jlJqrBobuBZaui6XUkgtPN9RhSQEYZdbRvx23Br05vN5KmVrsC6lrJSRFqQFjZm1oETte15aCouI74eA6tlspqVwPB9gCshqoZjPEQGgdndnV9R2b28/DlsOZYNeJ/BdpFVdlqoueNydzZOtra2nT1++ePFsZ2t9a3vr+rUthNXGejRPz0WdaJPvv3785Flalfna7jfy4nQ8nZc5qmtc5MYYXJYpZQGj1Pet8oZNAAEAAElEQVSQAQlWC1EbKwA5QpC6rht3BqdMCJHnORjbRDhSQJwyhjAYSwAhhBamy4WzceHRl1rzRtoS3JC1WGsbI3/Ag+XmdVUWN8HPzd5oWXASALShnufVdX16PizL0vd9x3GqWi5DEdFCXC3imwhdbMExfuO/MGbhWVjRi4tICA1vNMqyDAQG1Gw0G+YltEw9J0Rbu+AbucgDJIQQ0sRGvHGHw2XNvcQKF6EXl9DDcjbgItZtqZNW52eJEuyKzWO1h98TKyxPXlWQX3VaY5NspHpTVLohP/4H+39n+/8RPoCvVi3vVO3wFSgNLs/hFaAAK0/s8uSlKaLRfEKILMuSJFlap/BFDS1YoMmvq/Tx+7eG5g5jXFUVQyClpKRhJV+MrfE+WGubzILVl2W1n+aGljzmCCGMKEYIM7aoBQUAYEAja6xBBphGCGEAS5p/ABYAG2NAKyWVqOu6LsoqL4q60lpLgRBQShyjbVlUUijsECnlkrqH0osIYtFk1tDV4V1ZvtW1aybWcaEWZZrOd3Z6StS9TuebH3086HW01v/3/9v/Y+/lIWeB64Zx1I7Cbq83eL3/dDqd5kXabKld7kVRhCmhrieMVUWpAYSQpVSIEkK5LGsL2PH8dnv9vXu3trbXTs+OhpMh91yTJ67L9/f3tra22u34n/7TP0/T9HQfcc53rm1rI1/tvd4/eJ4XEAWglGQhG3QGu9s76+sbVZ6eHJ7uvXoesfvdaNvxWZbK8TRP57rTxY7DweLpuHzy+LWs7HSc7GxcJyiejepr128SyifT2ctX+4BJr9+XtTw4OOj1BoudAwJrUfPOWoOiKLLWNs6FKhPW2oZuuSiKJp5lGejaWChd7jQ8ik35eM/zAs/vd9ucc4qJMSZL0tFoNBtPqqoq5wnxYmtBa40s8tzAcwNGK2sR5bc2xTyZzWbCEsSjubYiySx22t0ewwRZGJ2Pz8/PUZOjzCDJz6P29r33b6RFenZ25nhudxAiZGytaEB/9eWn2KdHR6Pcg1TYnVv3P37/Ozdv3pzNZr7vf//P/rCqKt52Hz3/7WC3G8fxP+/8eG9v7+Tk5Pn+r4UdFuJoNpfW2n/2P/vnk+Go2+3Ok+kH33jf4/58Pi/LPPTC4dnZyeFBHMd5luxu7+zuDtqD9ng8qbU4H51MZuPt7e2yqvM8x4S6lOV1Ok7PiAEr1GCwHlqZJdnRyXGv1/O21rHRketQxCUGbe10PGfMo5T+f2j7z2dJsuxOEDvnKleh4+nUpbtEa4gBsBgsdneGGBJL2gjyH6AZ/xkK228kP++YLWdJYgbkzkDMzIKQDXSjuxpdVd1VlZmV8sl4IV1edfjhRviL97K60RhxLe2ZZ4SHi+vXj/yd35mvSg/cOhfFjKHfu30IAJoRRMJYZwT3xkSAOi8XyzJVUSfpV1QuJnOZlJ1Ohyl5dnx+cLD3zW/8IhPJ5WTx3rvvzqaLPC+VUr1eT8URETlnrF8ZM5RSItJrr71mrTYOKw3W+m/84m8iWWQuzxdn5y+LanZ68fR8mS8WRL7vAPJqXugyThCUYInnQncTBC6LqmGGGHAP4EnHiWh0cXFSCaGkEGiMtquyLJVSKkk558g5ERnrGqvTOPEWtXWInhRjjIFjAhVDQd6StZ7IWxuWnVKxdwyAhWyXtc4YZ61zznlP3oOz6CwaTdauMwLWmqooiShRUSSk994byxkj7xkwQCIg5zwicqE4Y14HeQbIwSMhkYc1Cg+YuzIXWIAfMO4AjPOOkHtcGxbgAELyRTsrEJRSXErGmAcoqzqKIsEEORscMsYZITnyQOsOgYjIBYZo7rYd05LzbEeDbygk3OJV3M4vtHT9N8aNg2ykP7ahRb8FfgQAjYAEPlAgIHgk8M46cs6EmBAiMrgCQzhXgvfMEfPAvJWIEVOSK9NY8BufMpyaeQQgWhuL2zYcfBkMcHOpLS8Cbf0FojUHF2wVngAgZzfDAGGbXWdJavfgHAF8CIdwzoJmYgy8t84Za7X3FsATOe/RWp/IBADIE5HzntATco6MOb+GJXLGYGPwAZEkZIwbY8D4mCvtdJOXujbkQEkZRQltQg7IhPdeySsu0e1ZsnbNI7K5hQ2DJ8/AEwFh6OPoNGfAAGIGB6PR0d5+vapc1SglVrrinHswy+W6xbBSKtguRVF1Oj3GiCHzDsPyJwwkzUDkybu2/xkReSKGHLZq7ULdJAEwWzImGHJOfF0HCUAGQ9EcM75erhaX07KpvfdAbFUZKWVelbPlihj3yPKqllLWukySBBivtXHOcs4BFTCGPryhjsgRuCtkBmNEpFSsZMdaJ0Uaq7G36aKai9pY8k8ff3JrPx1H1VFqBqyaLVa/+MZrD0b7j56+/N4PPhodHP3Sr/5a3PeRi1dFQYyez49t3bz79ff3dna+ePx0NptRYwR5wQDJWaylUFmsdKMvLk729of7/fTuAQd3nrLVMPGuLg56/eXFvNvtltNFL5KuXNzZ70dVpKK0Xs4WucuXePxcv/Hmu3lu68qrbn946/Y7X/9aR4k//vcf/eBHH3EGs/Kzi+nLbnc4ubgU0YDmyX78FemJoJm9PE1cHCcq4m4yffrp5+add97dVT1wl7o8T6W7+9pRtzN68fz8iy+eCUJwaK0dDMfee/D21uG+t9owiONEKeWMzcjXRVkXJSLqojLeeSBCDFQqgilE6clY18RJd7m4MNbsd4emybN011rT1OXXv/r+xdn5eJA85Xq1oizLnk10r9tjQpxPJo32B0eHHtijR4/Fg9fvf/+v/zovV9049cZPzy6m5xfz6Xx/tEPOkUddN1rrbrcbx8p6J2VnUebL5ZwJnvW7cRZ7hLqps35vvLsrlOBS7OwfHN65lXY6SZI0M7taFY+fPglY988fPtw/2M2ybHIxJaIkSd966+179+41RnPO8zyfTBaj0ajbHWRxkiRJp5NFUdzrdBmH6SUJyV5/40HTNJeTWRzH4/GYMayqylpzeHRw9+7dly9f5nmZdTv37t1zhJ988smjR198//vfL6viydMvyqrIRnshbVMUhUe/s7crhCiKIuumDMgDBX1DDBExpL5bqzf4h4QIgI7AOQ/ktfPeesV0VZTLvKBy5bxQKh3vHjFUwMVyuby8vEySFBlJyZ0zk8l5VRdlVVlrPVQAEKsoEG9Np1MhxHg8JPBvvfXGcNTd3R0f3dp/7/1bq9XCQ/S9Dz8r8ipK0n5vtH90eIgySXtcxIs8r2ttnSOGnKMQTAjR7XYvT1ZN0zjnIrlmv5dSxjKOoihwVsZx6pzL8xwJut1u0zRyE9Wv61oIIZBxzuu6dmad+gpxxhDqV63oab2VTYbYb0j4/RYaHzeB/W1VytpWkxtz2DnnvEZEFQidtvRvcNIZY7Sl1rxbC2slolY93wh14Favh/Y6WcsFuYFKOOc8uW0VtdaObTHb5vrbfbZPBK8Mup6XgS1owraOxE3++8bPXz3jNd1MQP4qneG9h+tdIbaPE3wR771jhJZaeqsQ3mxL/W64ff8Bo33cf6f9f5oV8tN+Eh5fazxtn7Q1buhG07KfEnJoR5vMorVdwnq93v7+/sXkrNFRXdfWrnPwAWwbWLm2H822XdUetr01zrmxGhkoJQG91ZZz3klTrbWUUggRxVIJzjkv8lVd11EiQkzOOdc0JmAtaYO7tH4dZmhtxytTAODV+6VN7OrVAFj4Olxr+K3167ggbKRBXel5bYOk0gGTvxnbET74WQugjZFs7E4IfCzWe+u90drECkWcdAZp1u8QF6eTy6dPn//1X/2AvOoND4bD8eGt24bgxYsX8kKpTO70egIZR+GN7aUpJ+wkcVkoVYtGWxJcCK6QAefOM2tskmRFUWVp99bRnUaX77zzzrvvfvD/+B/+x/l81usOlIx3d8ed7p2maR4/fvLugzceP3rWG+1pqh89/Jteb3A5me3sHtXVwnvf6/b7veH5yYtPHz6az+3t24P5KnU+NSYi6Az7e3fu3b51+Jql+q+//xfPT55mT0/SNHYesiwjF9+9Y8qmQS61s9PFPEr6WZf1hoPXRfT06fNQBC7jiHN0ztRGh9RD0zTkHG3A127TFQ+ctd45Iu+c9c6Fx6mE9W6+XCTd3sWTL0LRxEF9oOv68mLS6WXdTseRH+8f7B4eLWbzew86y+WSSXH3/j1PGKVZ3ZjOsCdmi+mjLx566x7cu+u1sbrMkuj27SNT1ZKroiguTvPuoPONb3x9PB6fnJwMdnthUXa73bzKl8tlnCZVXTtvFnWRXxa379zykmvAVMiz2Ww32U973dF49+LiYnfv4POHjyeXs7JqTk9Pj+4cvfvuu3t7e1Lxi8lkPp+fn11oDYvF4uzsjAOG+yfCfre3WCzqupGM37p1q2n02fmJ5KKq4qb2oeifS3H//v3BYPDhhx+madzpdF4cHx8fv9jdHf7Kr/zy4a2jp0+fVlU1nU6ttdPpFACSThIlcW/QH41GxmkKrVYC7cfmbfHegL8ZLgsygBCc88ZoR0Yz3ui61o2xvq4XACtjHBADwCJvlsu8qirw6wSn995ag4iRUlxkzjnGhHPU1K4sDIAhkETe+2dFsdKmzrL43v0777333nBw9/BQT6dTAFRxiiiRccGF9YYDIJInS5aInBIyTqIsTapMO6+JQAiGSE1TWWs5oBBSCEHO13UdZG4klRBivpxlaRrYrynAZPimF0DopYToNwIoLNyg824InVethDB1jHF/RVdArRxfFzkIBW39gnVEJFASEeFaryMibPx62CpMCGEGxhhTN0vMYathbiu+aYPj40KtXzZaS0/nnDNNFEVEPjx6IsJNgmD7jlpovfc+RAhelYy0BcHbti3W+Y4t4ELYDjnmL51P2FI/dOXCBk5cf+XXbsEY25mCLSXhnLPWG7OujQyjvdrth/XzKOxX73d740sn5Mbd/V3tEmp9esSQJvBbo72FbVOvjeVcC1Fc32gfxCa9FdiLiXFIs1gpJSXnHJ1bE1aGeIbZNHvcvotWZ29fxuaCrfUGPCjOkJG1VjBCRuQoy9KQ60fyQnIiMqYB5oK2DrGE8NDDIvHeB8Z+2NRMIpKScm0vXofRbN/m9usAcLUIbzzt1twP/zXGlGVZ1pYxFmCeLarGbzVeaZfZZnpfBdWuT9NGg5B5zkEqFBIK6502ZHOleKdkhXGTxer42cu/+sFHt47uoxw0FqWIlovlyYvjbr/fr2VPKgbcag3AXF1V1idSJVGcY2mtd0RMRci4I2hqr7WRMpovzp3zcZyu8kUUpTvj5Ojo7rOn3y0L8+jR48PDg699/b2msUDi6dMzGQ9v3XpzZxf++//+/6udmExzJbtZkh4dHb333nv7+wcPP/3s8nKVV3A5XaHbJZ9WDZtMV5Yw7qQvXp7zCPqDgxfH55OLFRc1kY/jXMr4zTfnD946ODrinU5vNptZw1bLusi14LExjryr63oymSglAPzkIk7SiBACrNUZ44OEFIJzHtSX9+CBPJAlj9YCYl0VxuiL2fTB3Xtx1kl7/fPz0+99+MNultV1ObqcDnZ2Ty4vtNa7u7urplqZynsfK5aXDXBxePeoNvbDj5eiaFbI/d7OeLw3ePL4i4vZpJdm/V7nvMz39vaSQp2cHdeujjoRKjy9PI2GKsuy27dvD4bDZ8+ePT87eXp2XNflwa2j6TSXUh7evVto/fz4Zey7MkkYF3HSefud96R6dHh4+Prb7zjnvnjy6OXp2SIviPDly5MkSYqi6PU6sUpmi+nx8fHHH3+suOh2u6+//joAC8HqptHPTs+apun1BqENzA9/+EPdVOPxeDQa5UUxubh87bXXHjx40O33FouZtfYrX/nK0dHR2dnZF0+fnJy8NMZYnoWk7P7+fr/fr6qqLMv9fl+vmrCAw7oPOBwAIEKAUFuCFOhaiXnySkpERt4SMA+IXHAZiThJRVaWZVnUWltjTGDY9taFaLlzzrs1KV4URQRpvayjKBoOuoPBcDTcT7PBdDq3xkoZTSb5YrFkHI6Pp59/fryY2/fee++1N76SnJwsl7kxpqwK74HzQhvHOdfGmEDE64iIvOGmbvb2dpQSWutYRc7Rcrl01linp9OpEAoJiqJijGVZFscJAFZVFSnV6XSCrQAA4HxohU5urSCdMUHWcM7d9Rwwbpqxto4ObNXz0HW3tRXowX4SQggZIWKQiQS2lVN0TT2woCGuRRToWh1E+FWrG1opFnAPrcp3zjEu1846+Jb53HtvrSVyIfXAGHOOQn2RdY3btJMIgLJwugCz/RkO69YMXKuDaD9sJewNTzT8DVcOG93f/pBz9uq5XhX67QiL0GjX6HVXmDDgpyj1n2Yp/DTt3n78pcr45sFfMa2+1Nj60rO3M/mlVsKNs2wPAPBrOOcV1HFrrWIwpZxz1ummaWaz2Xw+t1Zvgg2+fQ5tImn7OYbx0yJD1tqgO41tOEcukIiKYrUzHvZ6vdVqUeZzBNrdHYOnKIq0bbaUOgCAECJJkvZ2wgrn17ugvTpXrWHaXkxrKISn3L5EcAXmvRaTYKEy06wt7HAZUkqt9Zq25PpYP3EMhkKIfrGr19+v3/oQ/pQKhSQuPCOHZGXEZCx5JFCKZa1/8viLRdGMrb9c5FXlGuPrxtZ2FSXZ4iLf398na/PLOefcN0YplXV7qZJcoLVWW+BMiSgGxjwAkHPWAonJxXy1rMjzly/OGBN1ZYwm73ykMu/x4nw2P8qt1WKY/tov/trp+fT8YspZ6htz+/DOalUw1Luj5uJsMp/MfvLpw25/9GbabZrGYBL3O0kqGhpEMUv6POrCG2+88eSJODkfw5QcGECnfbUsLyszPz45S9MOIDfWN9oUZV2U9WCQ9voDa60jnxelaLgnq63pdruhvXNwSyzotvpGW2OMsc4FP4oxhowBZ876tNNxRJZovLfXGw0Z58+ePatn0yzLnp2eHty5pRFrosvF8nK1Eip25K1rlmVOwLgSr9+/W9SV0LbOBll/1C1NeXL+8mJ65uygqYujO4d3bh9FUvWGnaYx3UHmveUK58vZ2fS8NNVesaedGYz6s3w+XS2GZqfbG+wfHQ7GO/bhI0CuoqTX650/njTu6ZtvvqnSzulk6gHu3LvXGKMb+/Tp0x98/0dVXQTWiA8++KDf7+/s7CHyXm/ACBiTVaVXq9ViuorjeDabXZxe9Hvje3dfH492yfss6z787PMf/c2PsyzLy+KTTz771V/91Vu3brG8mVzML6aXd+/eHY/H4/H4i6dPBoMB5xyiftM0XIp+v9/v95Ez7/3FxYVQHDhjjCFHAPAbowHalt6b98eSp5D2Y+AQiCEgY0pGQiBnRd4Q81Gmsr4wup5Op0VdWaOTLMGSmsYFvkvvwDvQjY2jrGmai4tpXbk4TtOkx3diT/js2TPtrLa4O9wd7hwao4Fn00UzKfLFYhl8i6YxQskoieMUm6aJgSFya9Ex5z0heGsaIrCuJnCAEtABeOtMXWNRrrwDwWQUJQBQFCVHHtCz67atnAfHxbgmIBI8XfMY1s6NUq1oaN0dIgrgpjZa0O6wCcdck56w5WhuxAcXAhhjZE0QM61nwxA451pr2BLTbf61bYrTytbWUGhxfNtS1Vq7NibwSqOH4wD4kOYPNx3sA7/phtCaINtqfvumcCuQAK/oxfCGb5sRN6aiPcK2hXFD+odvt6tFWkEPW82iwnNrxb3fQk222o5eTT38h+IUf4aJcN2Uwe07vTF7P8Nc2J7P7azWtn56dfvGHP60awtevnUmkHY3TTWdTk5OTsoy17r23jLGAP1aBaLn19uPbb8g25fanpdxUDImcuQMokiShIHXdXV4eHBwuNc0ja7Lpq4QKY5kKHDYth2DtlZKhQenFA/WbXhnnVuHo2nLZg2BgaBaAK4tFUTcgn8QQMvxwAAgmMKbCOia66yFeXLOg6zArZLRG9MOAGtD4RWD80poUCBcENZVrqo55qNRdvfOrQf3dnsZu3O0W6+Kzx89ZEp6JovaNIZknDFRz5crxheHvSRLB4xA146xdT4oSZK8LKWUQjBHGEVR1uvxKEHBp2cX09lFpLIXL04mk9mbb75W13Wv1x8Odg8Pbz9/9nIw7DFUDx9+sbu7W1b5Tz5a/Nf/4J/tH4z/3b//H4qSJpPFPu9mcfbtb/3CfL78znf+6vnzl48/f+gdJHG8XJaRrGal7A0i41bWw8lsdfH9T+f1Bxfns5OLZ03TZFmUdZV1GCUm65k8LwWPpZRZ1s3SbhzHQHLQH1VVbYyra71YzLyX3ruynDpH491RJOM0ScFTnue2KBqjjbPOOeOcI4/AiCECC6QLurEIJks6eZ6PRqNHjx53u92Dg4PpdIpMfPbw89HO+O7d+0KIy8uLoTWJYi9PzpxjO6P+bLF69PjTtJP8wi9+U8zyKTFX2XKxwjhTr715f2cwBE+C+OnFcRzHURJpq5+/fJqm6e27Rz/+9CefPvzcWru3t/e1b3z9jdffLOrq5Ox0uci5UGdnZ+fn56dnF2+99dZgMDDGrKp6VdXvvPf+/dffePLkiVDpeHe/3xu+/95Xv/OXf67rZjqdKiUnk8nlZPbs6Yuvf/sXB/3dr33127qq8zyfXs6fPXthGp0kiXc06PWEiF6+OF+tVrs7O2+9+Z7T/MefPC7yxXhnZz6b/ft/9yfj8ZjQK6WePHv29ttvz77y9re+9a3XXnvrK1/5II5jL5O6rv/4T//kyZMnMpZvvv1Wfzg4PT9nAhlnQggmGAR+Ie+BKIQTwjp35C2Bc+QJ0HoidNYb5wkxYZxLLpHV80J7D4FKnSEXQkpOxAO9UhRFUacvhCRC9ITAB4PBdDqdr/KiaMiLKEm7vQ7nPC/1zu6obpplsVwul8iodub44my+WDpn0zRVkUAmpJShewKR5Rw5xxBI9h6ByOnmcjp1jhhjSMS5iBMlpeTAd3d3p5dzweR4PCaiy8tpWZa7u7sHBweRUu3732pu2Pj0bU6BiLYdDtqKq8OmDC+o4VZg0aZ5MW55+eHbNacCrflxW0uZtiTL2pVHQERjDG64FmBL/oXjbLuP21fur3MeEBEyE2J3BBSiqWv55T1jwNacDjxciPcecH3Sbcm77a7hdZ+YtrLv25/7rbI9eMVg+hIFdt3xvaZxr+/fHv8q8wDXyKl+mv99Q5vSf0Tq4VUFeePD1gb60nPh3xZXaA8VbvDGOmxvs500vwUi2T6Rv56mga14e+hoQETGmKapmqax1iIjzhCAex+iC1epIthaZl9qMWyOb4Eh59yT8956z6OIx6rzwQcfvPXWG0kaCUbkjfe2qiwiNU1DW8kC2FownHOGPJgIAevqnG1vcDv3sW0dtjuEF4rz7XVFRNRmXZmQTlFLAGWMCZUyZblmmFj3b9swk/qtnpZXzxE25sJ6nbZMIUjkCRwAcg5SAYGp62rUyY8G/Q9e37t//0Awd3gwnicyidWtB2+k6U6Zk7GNB+ZJ1Npdzpb7gx2ZDtM4cii8M0TWeWO8KXRJZFUkyGOsRNZJ0t5QKVUuqro6jlP+7OnxRz/6ye7urlKq1x39o3/02/t7d//lv/zdPF96B5PLy+l0YUxT5PDjT0/29vb+5E+/X5U+S4dWuw/ee3s+XfzgBz98/Pi5syAEKhWLqHNwa7TKpyCNl01vhKNxFsX0g+89PT5/6q2ochfHqVC8K6JYiSRjvUGka1jmq7KqgaG2hqqmaeyqKLwD54iANdpyKZjgdWmsc/1+nzHWGF3mRbHKnXNRFKk41taiDyWvCAChvy95SFU0m8663e5gMPilb//SH//xH5+8fDkejxnjUso4Suez5YMHQioVJenrb771/LMfvf/Wa8Px7nSZl/nq5dMv5vP5YDgWq3xOzJVN4a0VSvQ62ag/iFVcLJZ1WWmnnTbz1aX3QGzU6/UaXTHwDPz0YrKYzTvdbG+8M+j1z8/P5/O5Bzo+Pu32+7/6935FxQnnvLmFcRzvHhz2u72s0zs8ut3tpIhoG71cLvf29sq8KMrVd77zHQ64mi+/8xd/naZpHMe2sVLKItcIAogET1Sibt+6Xxb24ec/Ojs7S6L03XffXc6X1qC1rtsZMYyePX9a5EbFMo7jOO4AiC8ev5hczF4cv9zb29vZ2Tm4f38wGIxGo8ePHy8Wi7qux0JEUaR1IxnKTcwQ/ZXc35jeV61R1nFgB9Y7bQww3xijEAmh2+9VVVVVpbaNc0bGcsD75LwxztTGW0Li3oFtvLMEYFerE+ecZEpFKSJfLvK8rKqmuX///jvvvi1jeT45+/zhZ5PpRVHlebkgSAmkA2Y9kF+3UHLOAnoC65w1Rhtj19zBHqNYcs5DiZSScZKkjAkyVFVNrztAYoi8rutOp6OErGvdH2ZpkgTIW9uUT0pZVZXbdPJtK8iDe9EaE5v8tw2h3TU7wnUl17o7LYoQNvEJa63zcOUmIr/6+QajQEQhCNQKvk06dq3gW8rebWlIW0SNraBc2w3GsE1l/JrmmYjcOswQgH7BiQxQALjKB2MwmMIBW7vhVZWGW4GBdp8bOnJbDdwwEdqpvqHI252/FMwYDIW1NQZr3Rnu0VprjDPGhe3tPPT2ZQOu/37p+GmK/KelHraNSPiZVsjPtk62j9AGdW5A87atImiTiUS08YnxerDn+vGD/rabudKMQZqmebEUkhG0NFzrgD/batBMW7ktt9UU7fpZsG7qKJJxJBtd53nV6wxfu3/v/fffvXPnTr5aRLEESqSUZbFCpMDgzK+3saBXAmZtZCtkkXAD+oFNII02hZ3bRj8RhXapm6m4utRw/BA2yLKs0k3TNNY2AdVYVVW78uG6jYVb0GbGmKMvp/Nim44waZpGMe/3u0rJvGCvjc29/fjOOOpJXVariA0Odvp3bh0opZqmWayaVe4cWENMyswDnS7q/vmi3+nM56umyoGMs433Ni+Lumm0qbUHqhhbCWJooqiujPfMOzAaPvzwh4vF4ujo4N69k8OD26PRzte/9s3PPvssTgTjMJ8tAf3xcfEv/sX/9NZbb9679/b/7n/7CycnL3//93//D//g333729/mnCspUbE065dFczldxXG8KMu9/SjpRdkAj+6ODg72Hj48Lgtw2kaqI0XPNKzMiQlbVU4bevToi7Ksnz59ao2PpIiiSIpUKZWvymCtESFnUkhmjPEeUPAQlU2yNEmStrKpKIr2WSAibsK/O4MdsMAY66W9X/rWL/TSzj//5/9cVzUAeOvfe/ddIcR8ugiUIbu741E3/c2//+tJ2vmjP/4TsIZ5d/L86Wc//on46JMfMcAkSmOpBGAnSTtptjveaerq1ngsuXjx4oV21nt/enH6yaefFPN8d7wzHo9Pz8/ml9OXz19Mp7OmrNIkEUIgE5PJ5NmzZ3/6p3+eJMk3vvGN1998a2+8k8RpWdXj3Z3Du/cWk4vVcv5n3/mLs+OTJEmqojw7O3/+5PlwOAQA3fjlYuKcyxfLW7fu5KtaioSE6GQDrXW+qsryMoriD97/+unxyaOHjznGd24/uLi4KIqmrm1dmd2d7tHtwzzPG6ufPztBpCRJkMPHH/14Pp/feuPB22+/zTm/d+9ebeqLiwtCsN4TehS8jWSuHQsiCTzEdtdZPMLQyBeRE2Ohz7ohsp64I++h0rVz1pPVtvHWOau10bbRVVHbxprGkWXOITpQKo3jOOJ8tVpNZhfLvHTOySju9ntppxMl6mxy5slyyce7I6FwMOypOD5+sdI6MG0YQBczJYRgnJrGeh9IXxtrDBAGqmwPTAjhvCkrraQN+QTbWGPM7dt3vMGHDx/neb63t69ENJlMgFnOWBzHjLGgSLyxPvSA1mYN5XMuhAraqObG7cageAIhDLyiLbz363gkIm4C5t572kJNB6EGQcv9FE/xxmHDVtAKHMW2i9n+JDhGfDOoJXzcnK41AQEAvA3MEyH/EgyF0DTLuqtCidZQYBv6/VbIvioZf5rFAK8YBPhKzmJ7z9bSurKTvgzMSBvBDxtDodVY29vX7OD/1FUPX7rx6m7tk2oVz88/cAvq/6qhsH2/18ToK5P/6nmttXVdhuY6aRZHy8h7H1ANfkOoIIQAf22Jho0bS3T73qVUjXOI2O12Iadc13Ecv/X2m+PxWEq5XC4kR/DOkw3MOQFsu20VtYwjrWpnW1DK+XwOG1aotrrHOZemKW0Z6O17EUVX7+n2HNAGwBFFUbfbbaypqqos6tbO2H5wrbl/42YREWibaoK1fznnAOHgWa+f7u7uqEgsFtEbB+zerZ17BwMeYbWqnS6kzMC7j/7mQ+MygIzLvoykVEmcQtnUz08nIKJOqi4nZ/n8QggvmG+aMkkj633jfWMdlUVeV1meiyhaLAxjTEq1szO+nMw+//z8jTd2fvKTz62Bu3deQ2QnJ6dJqhD9+flkZ2dknfjs82ff++6HcSKaxnz26cfj8XB3d/f58+dSyk6nczldIGsQOSF3xIejzttfeePgqGPsNOmkw51x1oW6BkKloqFgSVEU1pYoChXxxawum3o6n2vnO/1eJ+srmRlN1nuuIkQU1qDgKDghNNY01lxcXAwGg9uHR0dHR92ss1qtXj5/cXZ2FhrE07oghzEG4BEQF9NZv9NdLBZPH3/x7Itn+7u7SgjOuYzlbDb7yjvvBMi2rpuyLBez+a9/841empxfnD978sXk4my5qjyTt28divrCMsT+bhxjPO4Nbx0cKq5cIc++mD/Xk/5w/I1v/erlFE7OTtM0tUBSmCdfPGtqf7B/+/nz57/zL37n8NY+ZwZZM5/PHn3xVClZVe7P//w73srv/PlH/81/+1/+yq/8GnaieVFAIcjOLqdndZX/z3/5p51EffqvPklUYgo3v8zzqcuyXmFW0+m00+kIFjWV3t89ODs7k1JOp9N79+7VTTOZXWZZZrxdVvl8Pr9za3c0HC/K89rMMeKg7LKap0XS6KZpTJylWutysSKi1WoVRRGLUFMz2h8mvdg4a4xubMMY6/U7DkjrijFSsUw4r72pqoolmbXWWQcAjKFk6Mgb3VS6CW21sm7qnKu9ts5LKSkCkJzLiFXkSl2V1Xy6XC1yp82aLRi4R08KnUIvvW+kj5N4JDDrAKCSceiR/dff+2gwGOzsjFQkyrIoS2NWy14PqnrBOUcmGAEQK1d1UxrJWd2UsZKpzKJeVFWVrkoixzibFwtBBpF7Xde6Yt5wHhltibjVutPpHx7uL5dZmmYMRbfXcwSnF5PpbJGmqRBCa+2sBoDa6DWHAZBHjwIJqNKVUHxV5kqp+/fvj8fj45enT58+BUYeiAmu4ijAOUNek4gkcOusdw4BOGcchQNHzlrjpJSMc2Br2PY67K+wLMtaN5xJKSVj6L01BqSUHtZ5Aeec0Z5zUEoYa4Jqp03PaNyEXgEgpFSccwHkwTmXEoAc0DrpEIQjF8oRomfo0WqHa0AWIgrgtnYGnOGAnHMuBRGRJ2st26pQZ5ssv4dgG4VkcFAqax4Fuj5gHSHY1mdroQ0A1mrGGJEPZDthKog8Xzd/4Y7AeAfWbQwmQIT14YjIb6wJh96hc20ZBTFOjDsuKOjBq4vEtczZ9hqDPUGbNPardsANZXNDib6qj2/o7PY4IRz16rdXDjHRWkc5R5yHq0QA8h4BGCIPgSK3CXIgQLD4EQHQubaKhAEAhUAtgEUOIgLg3oEh4TyzDo1hziJ5jqA4OkQPGNBiAsjSJpvGGXObCqA4yQIm1zlH3iNjnHHOubG21+9zzrXznaybJfGdO3def/31+/fvnx4/63e6gmNZLU2jh/1unufeSkRGDmrTMMa63W4kFZKPlTTGMAZJkgRkt3dGSQRgs9msaZpOp5NlWSANU0pxHjJ14L0l2PQ04egsQ0TkDDE0FODeAYAjQgDnyUspe72uA19VZVWV0JCQkSfMi6rRVkWJVHFRFAyvONDaElOttVID5yyR9qCJb6w6kBY4B2ENQ+KdOI6FSaTp7UdvPej3+n42eayi6M7hfl2aDz/9mzgec1kDi+dLK8F3x2lZ16XJ40yV2rw4e1nrqqoqpUQv7gKjQkNMMUoO4DVqY0xVNJVdSimtY7apL4scwdc1H0SRL+ISnVLqyeePrbdZJzbWEgMEMV01lKyWdql6kgl8/PxzEHwyXyne6Nof7OyspIFUeO+ddrd6o7Is+zv5/+F//xuz6ZTB64xYtzOiuRjL1KNy1gI0zFpDADzJl+nxc+yo8czUMfaGWcK4AMcFgiMQAsqyqPQq6fDaLAnccCer9GJyIRfz1Xy2nEzno9EI0E/zxaIuHPOOeUDgHMhbqzWEFr6xqJnVwoJi/9Of/MFXv/rVztHINnq1nHcG2YsXT/eHgyTJUmt9Vadxcru/84f/8l8bgmE6MKWdz/wv/9oHv/zrvyH6g+7h/sG3vvFtyfjk5HxV5GVeScani3kcp0Gq/te/+ZsPHz/W1hwczOrFzvnF9HRymfVHUaezmpSnlxe9fuyNm+eL2pbdzrgzECrmTe0dmT//zv8cJ/ytN9/znhntZ7PF8+dPl4spF1iW5cnLl+QADZ9PV92EOFcoUCkV2gmGaHaQBp1Op6qqoii01gGQn6Zp0zSPHj06Oqp2dkYvX54URXH79m3O+XRyWVSlECKUjTpv4ygZDoe9Xm8ymTaNYQxkpA4PDwfDYWO11poIAxTBOWf1ugVzi9xpU+Zuq9fRdjCwjfVFaeK0Ie+dEE6EjF2gFomMdlXZGOOISMaRFAnjsna2aZqqaoCw2+32egPOpTFmQF5r/eLkOI5VkiRxlgrBjHc3fOV1v0W3fi2bprHOaK3RrzOIo52xMcZZipIYgDny5I2MImPcYrXU1jamdmSX+dIaV9d1My2dc0rIbrebpmkQ1YgohADXJn2vMu5tCUAcx71eL1+VSZKEBtztCLMUnqa3V8FJR0Rk13fgDOccA3ajLV4AQrgZNfUtWJ2uqZZWzbfz036y7VO2sqy9POfWNWawccWCYbH981aleb+OQLT9phExROlpm11x41ZZd4WZoC1Kx/bytg0F+LLa9/Ys27fm1+xJjtFVMGbbZXRWtzF52gRL2r9beh03mRcNcDVv8B/SFhHa6f1bP/lbv3rV9X91h+2DtEhVtxnh3jl3AezJGCILCEQkIOddeExbBhAQkYOr2gFGfnv1ruEIhG3Qi4giKfyG7hMR0XsKeAJwQSYoAOuu2EE6WRLoSZRMq6oyupJS7u3uTyaTpjF1lTdVURU5Q+KcW6ujuNcGuuI4jtOESxElsfXOkQ/Jf8GZQMmlYIz1ml6IBQaWX+dMiAq01t7PeBbbg3OOPBRIr0+dZVmn0xEr1+l0Op3O+fn52dnZ2ekxIo5Hg8Vi0c7JZiaC/DSBKWGzuMIL5ZSKyZIQLI4jAF+WuZLRYNhflSdxFo0H/cF4R8jk5ZOTly9Pzs8X8/my2++l3bSu/Xy2cuCUilUcpykyxqJEZFkM6AFs3TRNU0WRQADGmFIy3EsUKSXjwhghRNM0QC6KIvLu4uJiOsU333yTM84E4xG3zjlwoX2uAOm809abRnsP5LziCgVzzjW6UpGIk6HV7vT0NC9mzrmjo36327faRUrtDMeXk1VdW2tWzkjvlFIdIuKME5LRbj6fH+zeKou6LGsgDohFWWjtpJTWgzEOPQqhwlPzaEH4sixDsNY5d3l5SeCqqqqqSso1TKpNQoYlJ6UAROuII7zx2muxkr/8rV/4w9//N4Mss3Vly9VPjp+98+C120e3Pjl58cE33z842P/z73zn4ZOnMusOh/2Du/1vfvsb77z3lrh//36/348S9eTx03//h/92ejaRQvQ6/VjG3nvd/PDjH3/yW7/1W6apuZBHB4d62HlwPDk/n6S9/s7BocwibavuTkaoK6hGfme0s0teGY12Xnth3nnrtThydT3t93bv373HmLh75+Di7PTbX//g+bMv/uJP7OxyLikSEDGQxpjFMg/UTFEUhbx1cBDTNM3zvKoqACiKIszUarVK4vj05OStt96KIrlaWYFstVjmVbm/d1gURZ4vm8Z0Op1ur7NcLi8m53KgiqJomiaO48Fg1O+T4pIkeOeQITDmnWs2DQNDAu8qUb2pLmubCGxLOr/J0FvvCIBzybkl5Ay54FJry5mIYy4jJELGmCMoy5rziHOJqI2xeV4GLgBrfBRFuc7zPBeCdTppnEThBUuHKW6SoN57CCgiBGTknPXeI4MsyzpJHFp3nswvm/lcOxtFEWO8qY0nJgUbDfqcCWKgEmXJr1ZFWZfGGGca5xx5GxkZeRlFEQ8kyOCBYQAneM+JgNZrcZ1rqOtaa80YC6QLRLBtJbQK0hmHiEJw7711bZ/ltSbdSH8f2PeAKChmvN4KYftv+0qs8yD8qmvUdiw0bG9wahgKusIltYZgiycPO7SPtRWyiNh6Y8S2DAgMOMer0erZjeN+pfZaMfqltkIrZ1sD9FUh3v7KOUcbLdXiDMJP2utnjIW8zFqJktvesz0X3ERK/u3qZNtEu7Hx0z78+ccN2MH2rd34b/t8WwOIro/tdbhtosEr1gYAEDLvPVkDoULSWmsaaxrOuRSMIUcUZI3BUGPJpBRX8xZAIdYCcO89ZwRr/k23ecreGQue0iRKVKSrstvtB9D3s2cvYsVWy8Xk7MyaJrwLgvOsg3ZDhDoYMAAWRUmadowxiJwInSNr/eYWeafTkVKGu9NaG+ODhd2+XLhV6QAAgAGG0j5xDwhADJEQKUB/pOJJEnW7XWNMZ1bmK+esztL47p1bSvLT09OLi4s4jtt5aCeciMJrwTkHBn7Nic7WbxJzUvIo5sio0ZVzLIrEaG9/9+Bgb38/ipLTi/nDR08+/smjyURH0ShJe8JK4yrrmVCSrG5qqyLunMuSeGfnqNfrGKun88vlcl4UhQNrnSdCIiS/biBujIsTVdVFUxmlJCFwjuPxUErpyBpnHBEBeYIQzGdcgXMQMNjOe0cO/Cbxaju9ZDQYWmvrZpnnuZA4Hu13OwNvAbzb3zt8/OhFXQNnlGWpbjBOIvJIjCzpqqpevjzZ6R+ZhshhnCSMiSJvnLGCqyrPERGRSyYZMQeOMYGIjiwiN8YtFqs2zcQ4IHIhOGNr3BjngjEUQljApm6894PO6HD/YDmbHb98fu9wvyrzycWy1k0ssJxdDF6/+wtfe/v9t+6XZa4innXizrB35/UH41u3927vn09ORdmUNHen553T85PL+WWpq4git5y9dv/1k5enZ2dny+Xy8HC/1+kPh8M4TV8+n+0f3Uo7/ThNeoO+FXR2+bImy5gz6FRXRVnsHS7yZd2UilKjFy+fP8wXy/3Du5GMh4Od8XCUKjkY9BjCD7/34aWbCSW6PUFWFEW1s7MDACFmgIihI0t4Y4N273a7eZ5rrZVSaZqO+lFRFNPppCxW/V5PKYFIiovFclbXteDKGJNlSaKkT1KBzCHEKpZMevT5cjURIo7jKFGNsZxzJsAGGCD64EOsihK3uiQ3TVOWVV3Xw+GwxRC5NWOxdY48EmOMq9iQLnxVVXVZN8b5oqikiIRQiIxxwRizhsqyGQ06WScSUi0Wq7Koy6pRKhZCMMHjJIuTBBE92UYbIXjLg0S09qbbZLTY5ClDfXOnkyVJIoQoyzJgsjjnUgZ0NGlTc96PYyGldJYIPIHlnLTmHGPnPAMMnUeiSHJktPaiYBMgWfumGzfJFUVxenrqva8r7ZxTStW1viE4WkXLGGfIW+XuPQFiFEnOBfKgcf3auBDcmvBOXunUUIrQzkEr5deG9FbC+1WN1bIRtMqViIIgZpsas/B5G0Pa1vSMsZBUoOu4+lBDS5t4wLah0F7GtroKwRXYsnjaudqGg22bF23t2Q2rYrMQrpV1vHrjV9+uoSHhUbrto90YP1vB/2e1EmAL24/XzfF2RW3v3JoIN2wCRCRyV5oJfMgBhQkL57lxzet25t4bY6xpjK611kbXSnKGCpxFRC/W0oCzQI8NjrYDRRxCaRQigmNIwICxdXEEA+hmWRrFeZ5bY+698cbBwdHJyZmzzc5osJwvL+fzWCqlpNWNlyCN9d4DMqmiJM3SrBMnqYpi5ylUaWpjrfPtjQvO1o0A6rosy7ouNz2EtpEN1yYP4AbrNgJ65z2CQPRhooQQcaw6nbTX7cymlwL8oJN2kygSvFwtJ+eTLI4ArgEXwzPi60aWjhg4cOHojKHzJo3jTjdREVeKcR6pCI2tysZ5lJWByWL64nhSNjgcHXV66SrHoqLlMifEbhqpWOWVLfM8zSSRyTqdt95+8Mab9xFpMp0sV/Mf/OAHs8Viucyt8YicHNPaeO/fe+fr/U73xYsXTx5/YWxTl1Wapjs7O51Op6jyqqkr3ThywBlg4JNnCCC5Wqc468pZr33DGDPeZnEiIpak6e7+0IP23iMoJbMkdpeTc2v9dDpNM0iintXce7suYAQQQnqwy0X+6OGTxXwFJMjBppQGjK49WQaCIwIx410b2wrBXLuBkwcjkDPhLDEOADzA2Bljwb91jkkVh95ms9nstbu3/83v/s5X3317r3uwl8a+rt64f+9HH/4ATPkPfvPXHz98+Jc/+ojIv/2VtwZ7+4Pdg8HhQaHtx4++EL/927/98OHDFy9efPKTj+f5ctDvd9KuQO45WbD7hwev37t/cnwMe65YLZ4+psbrOE57/W5/NBrujCqqnp+bsqikIk1GJQo4MGLGNJzzKIqq1XLUHdy/e+fO7dcRxWK6iON0NNwZj3a+9j5NTs4+G31WLorlrLANIeJkvlBKhdBZgO0EGM5qtQqQnCzLAIAxdnh4mKbp/dvjo6Ojf/uH/+6jjz66d+9Ov99P4vjZs2dlkfd6g93d3cv5bDlfmEYLIQaDwUU+b8sd61qvFrm1lrGeqY1XXpAARAc+vNKEEEeJ1rosqkBZioiM8SztSBFatoAnj4EQyJMjr7IYETkyKaN8WWrjylpb7RmPnMembLyHOEqzbhInUeK9EJJzzpgEYFnalTKKophzfnFxwTmXUjEGxoK1IKVMkpSLqy4DbN1kHJA85xz8Oi1SlmVTFiFpkvs6LBpjGkQMGZDVauac2ds76DCs6lobEydCqU5VVRByt2tYsrfWOADvrjQTbBoHbHy4NTXecrkEAGu81nrbvbih29YiHj2BI3JrkY3IuWIMnLehFhwRueBCcgTpAiPpJhjAmdigEa/50Gu7Y8NQ22qL8FV4z2nLg28NAn+dncm3HYqvc+Zc2Tp0ZSus5avz4cevph5oq4FeODjfIrLctma2d7txRgAIPiX8dNW7/VDwFQf66uBsk5vYxCi3ajfC9fwdVPvWqrj2yY1t2qo6uTG2LZvt0d749ifbB78xdW1KZXt470Pij7YoF2HL2rhxCiICcuQdeRu0nhQsjqR3CYcNnzF5YwzQpvQpLB4Izx1c6F4RPGfnwFnng0Bfd68edft1XZtGe2dG/cHt27ezJJ3P51WZe+/rMrcGRBYnWWaaChEDr3Mcx51OZzQa9fv94C2E0SZc2ntJkzgoklCbYIy+0ZY6POfrc+gh4DsQAHhYAN4TQ0e05vnkAuNEeUr7WXxrfwds8+TZ8ycvnh2/eOGdvXv7YJUXRITeOVrnswgYAJL3oV7PB28cPGOEhEkS7+6OdncGWSZ6XTkY9CLlja1rzTVFk2n1+aPHp+dzZ9XO3j2Cjn2+eH58fHGxiLPUM4ycUoqNdwYEutdP7t47uHN3fzjqaFPuiGxnL5vOTvCFr5scAIQQzrOmds4RgFdKdDrpYNjz3jdprZTIsiyIhI1kYIgcOQMAZjkBokcAoDVtfSDZAWNMychNmizLVBwxiXXZcBYLnjKsvAOl4l63f//+7aYCoIT80hMVVVHWNRNonCmKYnleBToKXdWWPIJHcEW+ipIYAYi8BUfOeEuMMeDQsnMGPxbR17X2vpZyzahBsG5AE3D3Ko7AkzXN6Yvnjz77/B/8+n/xa7/89/7mB9/75W9881e++Y1BN33y8POAaGma5o//9E8//Ozz0c7ua2+/0+12d/b3sn4fGn379pH4v/x3/+flcumcYyh+5b/4lXt3H1ycnV1eTAeDkff+/bffe/P1t/7yz/5cCQmeyjJ3aE5nFypODo72+/2MnxKBixPFJWUui6IIPJADXWkw0lbG1m4+XVXLmlNEjjeVQe8Y2I9efrQ7Hr733vvD/vD5k+ef/viz6cWs0UV4/bz34fVAxH6/771/+PAh5zxUcZRlGcexEGJ3d3fY66ZRnC8XcRIdHR6sljki3bt3J89LIWW/3/Xem7opq1xrrU0t+z1rfWjfKZAxgqqorfHASXkFERNyrX6d80Q26WRN0yyXyxDn6fV64/G41+uFlq+wqURqX0VtLSLGMk6SZDAarlaFabwuNRFIHjMUTIhO1ut2+8SwqqpiNXOWnHdSxb1+liYZY8wYNyas61Jr7YiQCRVxISUhRFEUPOBr4LFNfA8RCbwxpmqCP1EnvVQlCUpsGgPWIBdktK0rkgqtJVM3xaqqmkAgX5clF1IppYQkQk/aOo8E3gNDHhjltsj6GCJ6MuExIaIN/cs2ivaGSgsvYagYCPItmB2IBMict57QA7YJZiIOINeBU4sBjYeI4QXedvpbJRQU4rZRckNJbCvOoFPDq98epP0w1H1sq7cb3i1tpQDWPAXXbna94fwVAc72Vd2ov99W5/AK3gI2kQa4rn3DL9rgRBt+JyK5maUw3W2wYXOzV/e7eS4MADxRMBfWYemfDhHYvowbG9v//Wlmzc8ztm2m7QPeUPDh+ts2mK3uhDU9EQJgKKzZJvtijAHQWm9usi2IEKaKvGVIkRLQ7SrOdCez1hrdNE0TwKTONOBD1xfwtIneI2PkCbwDCKhf2gAdBILijEmZxYmpmzhJ337z9bt37+7t7NR1vVzmTVUSEQdkXDIhuVRCKO+9UHGapv1+v9vtdjodoWIPzDiyHhhjwNBZ7zwwhgwZEW2o2BqtNQAote7wsr26WiMVEUOUJUwJAGyIFBljyEJXanIAIDiL4xgARr0OQ2rqcnJxZnWTLxdNVXHOGV7P46whseDBEQGg894SeEQQgkkp9vZ3Dvb2Bv2UMZ1mYjTuCW7mi1JTVtZ8sVh9+vnJdJb3u3tSsca67mA3SVcqMR7o/OJM5vDa67dv371dlfPRaHD73m0u/JOnn15MToDRYNDnwiJvkHkuSAiOHkmhEOKLJ49mlxfW2iSJxuMxAyzLMqT5jbPOE2OCAABZmK0kjpqmaZoqJFg9WSmllGpttXtcFqVHxhg4QibUeHSoZCZEsbt7MBqOj46O7t69++zJGYJSkQCIAqt01k0JKeSzhoNemsZVUxujEXGlwNhScB/wtQiOIxJfI6CiKPHrahQZGt2VZdk0FWOJ945IE5GUMo5UHEeMsYbcxfm5FGxnZ+fk5fEf/fv/3z/9x//s+MmT7373e++8+dadO6//3/6v//eyMuODO7//R3/62ZMXhtjZ5VSdnBzef9Arm9PLx7OiAqnE6elJXTfL5TKO469+9aujneF0PpNJfHx6rIs67WTL1Xw2my0m01Gvj4iNnter3DTlxcmzZX75/MmTus6jrBOanzLAYl55wxcXS9cwYy7v7iWz8xeJ2BsPH4wGhxHPBCpT0fMnx3VZLuaToiiSNLp9+2jUH1xezqyLkiQJFQre+9VqZYzJ8zxEhvM8b12ri4sLKeX3/uwPV6vVw4cP9/cPZ5PLk/OzIi8/+OCDL6ony8XCOyeVunPnVogCKaUwWuf4EZEc1LVubCOEiOKYAtEpj4RkATHnrJnNFvP5fDKZzufzEA3v94eIPNQmbDsxfpM2ttaCg6Q/HO/smdrb2i9whR6TuBtFseAqy7oySoqibOpVUxsppZRR05j5fD6fLUKUcjQabd5XCLwmYWWwyARR5UNUfPPXWrsWUEDhh4Hsk7xzjWaMSQTwYMrSW59I2c9SMrpaWa+1a8o6N6GwPun14ziKk4g8hBYAuO5O20qZNbRwk/XgckMv36rDtgEEEW2olte+eOBXWBdBADEO5JkHb63jXAJjAcC+Ft+whhxyznHTZ6H1gP0GUoprwDvAhugJtjz+FoKwtfOX6OYQbfabQq9wrjZ5jxsoIrJW8W/Ft3+6F74dgbgx/Ctjo8B+lpalLchFaNsRxrZr3orstSXkbOtwuzVFP4etiEWIhjC2JqMAACL0fxfCpS+1Em7Mw895qPYI2yGE7Ql59VB0LcTV3sLVRhuO2kpJXG9N2Z6FHNDadpRSCoRIcOdcVRY1ZwDAkchxK+X6OM5elwPBcLRIDgAYArJ1/xjvPVqzXC6VUoN+9/bRrbfeeJMxfPnyZbHKOedFXmWdJIkTo71ubJZliEjk+BZDYrCHgpAJoLbW2A0bTdm0uzHGpORCCOdcSzSybSVcD/OE9RPSED5kxLcXthBMKTEa9o2u0yg62t+371kl+aefPTw5O1dxypkPxCnbB5dSco7IHKFyYKSUaZrGcbq/v5tlEaC3Vte1LgpMYi8Enp0VTXNyMZk+fz71ILJOLFjCeWQsT5POeIxFtVxVU8VEpxcPhnG3s5NmiXPN6dnzy+nZ5fSU0F9O0+VymecL5ytkyjljLCLKOI4EE95bRBScG2M4sqqq5vN5r9fzQJ688855Tw4Zl5xzIcFYD+idN843QeIF6Rr6eCFnjEvnHKAc74yHgx2l0iTu7O2MrfVFUTDGolg2FUkph8O9o6MIOT+8dRSnEQBVs1UUK0RsmkrrujG1dWUSofE1EQKhkIxz4T1Y451zSdZljIW0u5TSmGYtMcB77611RC6KojiOkzhFxHw1bXSFTt67dXT68vjf/tt/+1/+2q//1v/it/+7/9P/8cMffjTe2X95Pn/77bcKy374kyfzxgsVK6WMhS+evHh5sTi5mJxOpyikaJomjiNrUxXHvWEPBb+czy4vL6plGQnV63Xqoh72urPJLI7jYX9Q1HC4vwtCFk357OTZZDVXkZRSNroEj+jROYq4TFRqCU1VffHwORfRG685gTF52VTOKSsEOzy4c3L85F//m9/97NMfx0pkcSdWajFfSTXqdDp5nne7XcZYnufBXAg3v1qtAKDf74eOq6vVaj6fr1arBw8eeO+fP39ujDk7P0sfpaGJS6fTSdPUWLtarRApTdPLskySRDJJSFVVheSP7ElrLdfcRU4CMBSA5FxjrQUphRCdTgcRhRAh+tftdsuy9C1AbAtP3tnp5stVUCpxFCdJQoRFUaSqU1XVclFobYWKkyRtGjNfLLwvd3d3O52srGcvjk9Wq1Wadvr9/rOXL6SUSRLFccw5U5GQsYzSJBBHhlMirqW5JzJGr1MP5Lz36Nega9eYqswDcAGA6rpGxF7aQeeW00sikjLi5Ku6Mtoyxpw3jEMgXPLeh2AdbpXtBTebmAg5CMYkAGgdKkeIszUiV6m1FNtOGCPiJqJgvCdCYIwRIJF3zoXwLFuXMq5thSDfg6HQ6oNgiGxrhVZ5tAZKq4zb2CyFLtLXuRnajVbahuvUWuMWQzNsOBiE3OiwrXbJ+GXE/rRJIuJ1oANuyjVfNRdoK0R/Q4HduOw28hH4JnArXNF6iq1R5deI0YDH2iStNtcZzD5r3Y1g/M+j2n+GKfPTvv15jgCbCXzVVti+tm2DYLMy1xCQq8QTth3IAAGBGMNQ/0BttH19fCAC2iw/ICL0DkI1E8M6GKxInjHOuVJKcsYYsyYwGpHz2FqZm9n+kq6qhc6Pjo4Q8fj4eDgcjsej8K2UsmkqzmRo6wo1y3o9LrhprPVkPTXGekAum46x1pMjCG3zHMG6KTyRtVYXi9boDFwgQS5xHrcTFWbvan42JbvhMsM2kQtpM08u5AbDbzudzmq16mTJzs5OOHhVa0K2XC7dOosOsEnkAUC322UMuCBCR8zFSdTJummaZllCRFVVOldYawhWw4FKEpV1xlr76bxYFUZIPl9WslwAJfNFvcgL6x0yiBO5uzse7/a48EmcNU11cnrpfO2piSLZ2GqxnM7ny0bXnDMEYY2z1kshQ8A7EKI0jS7LPEs6SZIAQJZl1ruqqXXljXHAWSxZFEXgjECKBGNMco5BkoQprZo67WRpmnHOV6sVEYxGY6VihjyO036/d3by/PHjJ8vlMkmSr33wwWxaDodHzkFeloPR0IMry3Ln7u2iWFVVJRUSYF5URbms6pxz7ggYE5LHLJLeYYOGtPMeQjVfy5+tVBVFUZ7nztsghOPYtrLLOffg3v3z0+PFYlGW5b2j2z/+8Y+/9bWvf/sXfzkvmsvp8tu//Kuvv/760+NzTXxWWl42r722B0x8+MOP8rpxyIpGN8aI/80//V89e/bsww8/fPvtB+Pd3unZ86wjdB0bXd4+OtCgP330yarOL1bnt167VVL1fHY2GAyYZyqNcda4Kt8ZHNgcsIp2kp3HT46dzzq7R09Ov3j//a8+m3y0Z7q3ju49e3FW61La+WQ1lZHynvb29gZu1OmOhOiUVeO1joYdhNg6aDRxkSxXNSJWlbm4OOOcZ1mW5877iIhfXtZpykajEVGa9O88ePvg7Ozs+OQFS9Iold0eXlwWg75I4r6SqeTqxbOXg8Fgd7Q7Ho/92ZPGLAJlZVWX87yQUSyjpD/see+r3KK3nntAH0XRuDuYLleTi5y0PhrsZr1up9OJuFzO5kJKxoAr7o233mnfOCTgUNYFMPJIq6aygJjElES5d+cXJ/3uIFZR7ctyOomk6ve7B4fZdFUbLCxjUc91BugZIBaLVd40xpc+X4koipIk6WS9WIpGqjhWwkcx45rzxjZWW6218zpJIgqMcsBC1xvnjPPOKxwORuS81Y0SUmbKaSNcY5dFJBQielMroTqjkXOuaZpEpbSsy3oaJ2nCZePJOOeBWesRmaPQeNCQt4H8y1motAZgjGSVa85dFEVKKiAATwxQcsZRSs7WrXIrzZiSkiFa6z15Qgx9+CRDHoq5hZDBK7KNMU5v9ASG6njvfV3XSikOHBlDT2QdInDGgICso7aVVFB1zqMnDgiIZJ22blNpiU6bZquewm96VbSftBFs3nbOdGs2DES01gN4zpEj894DBnXOiMBRi8C/loJpjQO2her3W2kUJI+wrvzcNi8Y5+QsADAgAvLWeO+RiAjd9c5Gm2CDvjopR49rEB/33BvvQSOAEjKJ4qaqGySO4EJiPiRK19FoBLjqcrmtyLev7YYibz/ZNjX8T8Ei3Bi4FQv50h0CIoixbQsgyEPTNEVZLus6ThIZxcg4AYAzre2EDBkSgkciYDx4+qx9BOQ9EDSmZozJSDDGtAYynggZsIjHIImMRud4LBSXprFa17HgjTPeGQ5McsaQO6uNs3EcN8Y55wjQe2/cemkNBh1NJovStJNdzqZlXcYq2t07cN70ej0hBHne7fSTKGLEBMp4kGadjowz4ghMCBUjl865KIrqutZ1BQBKcOdcWed1UQKtk7Ybixm9B86ZMSQlE0JyLrZmE3yg9iOGwIEQAFmLKbEeiAQhAZGxDIAD5rbe2xkwBoQvi6YZ7e6NDmYnswU3rm5mjdVpnEgpyVnGKEmSZJh1Oh2puNYagNJMJUnEONUmb3TJOaZZbElOCuuizq3RrfmKzs7Onp+7pelyJ+uFU6pmTM+LedSJuPeK5GjvNiKePJ+7Rjl+4b1nnBBdbcuqKq2rAdESkyrWTje15UL1Rl3rYF5c9qKBjDkjMMxL4CqBSEGUJIPhMM9Lt7CNcZ5xJgRnyjpRQGWdtwQEzCEjLphSIo7neZGmiQdAwabzSwB44+0Hk8mkmhery8Xh4d6g0/1s9mjYv72Yf9LvjQF6Ozt71vjVctHppqZeHR8/S9JI4V5ZV9ZpAKqaepmvhGB37txaLpd1rYkgS6I46mhtmCfuoXGOiLS23heI5L1vGlPXOtQDAqAQ3Dkqy1rKKE1TnoiKdLY7AsDXbn8w3j/6olic/vV3b337W7PpQo927n/7V8uqrnx8YVWy/1pd5o8vzZ07B8mwszo/b6oqX5bek7DWMyaSJFMyRSbiOE7T1HRMscqNb+arqSU9GncidZRkcja9nC1X55dTRHznnXdu377LhVqt8qdPn+/s7huSitzpxdTk1biTfeeP/vKDD94Yd3tVVVnyn3/+6N3332uaRghJQN57qx0ipmna63YlSiaEMXMGa2ZZ771Sqg27ISIRhHgg5zIAHpfL5XJZGnPivT08uJNlyeT8fDabA9Bo2L+4uBCCVTUqJXZ3h5PJ5LPPPvGStLZMNIkn49Fab13D5bLXHwgupRIh88dAAEBjzWq1WuYrrXUcp0nHcymUitGbsqq4FEIwVCrg8I3RlrwzNvgVTV17i1rrSKpRfxCJKFYJUsAPx4JxY9xqVRjjGLNNY5rG1bXW2koZIUKaKCKSTIYsIxFWVcMxdywhIueMddaR9QEzgLwoqhA8ZDzgFhgR9x5iIdH5uiq9sXHKpBAGAV2TxHHI0XKOyDyRNs4ghHpFcIBcSMUE5xwYc4QhfuC2WjAEs9pvkpGtMwcbiH54jrRFOOE2yfJAkbvRQA6QB6nlXdA3LkAa3aZgHQBoq1p92xGEdZh6/W17im33HQDaTsq4Cb2yTXvi7Xhse7V1XbeXjVv1qJsYw3aOHIBxznlLgNiqT9oasKmh2FaB7VdwPWC7HR5gW92ntoMQrf7e1sfbZ7xyu7dmazv8wK4RVJhr8YTNU/7PPdgrwZgwfp5oxI3920m7cYQbn//sg2z/vXoKW42qGWOhIwnbEDq1a5IxxhGFEEJ412Jl1g8RGWOh12KapjKOuJIBcggAsYqyTsYYQ1qTsofURhRFKolCas86R84HCAJuepq0XmPTNGVZVkXRDce5nl94NYp2dWuvRLDCZfOtNujbOZ0QyZNSJknS7XarRgdG/Pl8EU4UwPa0ud9ut9vrdwJHvnNOKi4E8952ux1ZR95bIGxq2zSasyaNy3yynEwm+aqyBiwYa4ssxU6nt793JCX33hN4RFoulxfnl/PZcueWcs4h81JyZA4AOJOMMc0s5yTQ16SNMeSN9+Sd6w+6URQJZMY05LwUjDFBRE1Vh+y58wY8C9TInGEAXW2/eiGnMxoNjDHGas47wWhzjqIomcym3X5/WeSNqxy43rA33tv57NPHz1+eZ+kgibvOuaOjg53d/nhvt9vNlgvdGLdYLJfL5XwxresyTeNulCYZ03ZRlQYq46EBYsglV5yvYdMhfLUJGXrfxoBhk0XVWnPOTWOCKIiktI2dTCZFUQihjg4OmeCPnj6x1iVRPNof//Y//m8R8ccfffzkyZPdw4Pbt29/8ejxd7/7Xa310dGRePbkZDabVZWbTGc/+tHHy+XSGMMReoPu5fzyx5/9CAm+/s1f3h2NsyT9+Ecf/cVff9cYs7e3l3W63axTlTUS3j7Y/8H3f/iNb3zrt/6r/+rhZ48+/fSR5nh7rP7+L34zifvf//73J5OT7/7lX712/0EWJXVRTiYTW5mzs7Pp5LKqmm7WieLIWmuB5LoLS+M3pfBho2maoAKklEKoUJK3XC6Xq3K+yKWU3S41jcnLGrn0tlkV+UeffNLpxt1uFiXSgYkTPhx1NLC8LDygkJHkwiHT1jtPi9UyyzImuDaGWRSCUUPGmJ293cVqOZlNV2UBM8YER84DgJkBeg/BOLDaGGudcyrljDFw4I3VpraaIil7vUEad7XWxapoau29BwGuds45HkXecWeZ0d5orxtiCEpIHdx0QUQhMMi8B2Nc3hTBhSLrvPcWrEcgxjxxRpxxYIxxBACwZImcpJpZz6xPInVrf2/U7zlnkDwi1nVtrQXGm8bMFyunG0Agyqx15IkLybgkRAIgWjdsvKq2IEYBcyDWLR5wQzkVnO82k9pK4W39zRgjcpwhEENEYOvgp9/stl70SN61apVtqc+bifxt4b5lglzp4G1DYVuXtw0pwlfhyXrvw2sWgrdhbKsu2vpPgBM652DTnQpgbUggonfX8iPbwQP4KXV9rWDCTV6gtRL8q4xJgPRTdOoNTbk98Ho1RGuItNZVu9t/BPHSzzV+Hv3984ztqgfYQqhsG1I3tuH6FPktkI333m2aqjPGGNA68SSEt5yAe79pbrJJjRGsV4kQQimoGg1hMZMnWpsTtKEmy7IsjuNgEwRje30NRIAgOFdKpZ1Op9djAkP9l3UmcHy5Da8obPg92x4leL3fNFzlCq+ztG3NPG4N2HqDtqelHe0K4ZwnSdLvkyPY29ubzpeTyWVjdDvDfkMI1tSFiaUSgiNDBhwZOXAWwDFvWNO4pmlCVHzVs7YR3DlrgPMIAepaE2kpPOey0+llWcYYGNMURWHMrCxrIWyvJmOMBxfFQkqOjHkg75GBIAQpWCTQGiIixphSyhtNnIGUDMl6W9frRSJl1DSN1o0xDYFAzgkcMiZQePKh2kUyHgmZZnGapkLweVNXVeM63UhEFq1A1u90X758KaW0zpdluVwu5/NpkkacY5rGcRxJyauqmFyeezJFuZhOWVMJ731duaKo81XjHEnBy8J7L4xm+aoBMEnilYwAAICF3nWb57VpqMtYALlbawNhWLAdiQgdNHVDRJgAuXy5XAY3e7FY7OwdLMtib29PdRKRRgd7O3lZvP+Nr6oszrJsUa6eHj+/mF/WFZ1fngvOEuvnSdLVDf34J58/f/40SeLXX3stTpPzydl0er47Hq3qVcfGCnlvnI3G+/P5lKFYrYqqqKfTqbe21+nuDPqg9d39PWbsnZ19InzvK+8+efJs3oC1tlgWpy9Pnn3x9M6dO3VRexN6Si1D0bIxTivvCZJOl2EExHzTrIvnEImhR9BGBzyAcNY4Ct3SrLX94W4kJOdYVvl0NgcgxtjZdGq8YQzeff8rZbl6/uyJmPnxeDwY9y6XNQFW2jAmmIwS4qA1Aczny/CiBc2opBSCAUDcHSRZdwSMS8EYNLWpqkZK2+l1ich6A9YCkxAxKSMAcL4mQEfee0BPThvdWNPoOE5NY5qmqeuaiBSFHg0CQQIJBCl4LETMueZcCqHqynrvwRlEHjgqw/upbbV+n4GIrAcAhuR9QMCiDw3gg4svkJjyDWdMRFEniw7G/b3dMUcHQFVVrVZQa8OYyBktVxao4Vxyzq31zntjDBcGGDoiRyiECguREXDOETxwDwCOKFivRKSUgk3csnVKQo1GK2ik5N4j5+gcbpwxDBEFALAerLXeu1C93fq+2yo/eG9hAtqv/FWNwJVmuxG+bg/SamsiatHarYzbdr/a93D7w5B+ZVdu2YZuYVvpvlLOB9ftmPYCtmX6lpa64qPc1tzb366vFq66BN2wk26MdqLajZbGsDVKcBMyWR/wOhrgxjT+px3/kYfd9nppq85lW+e9apRsWxXbtkV7tJCbYFvdzznnnjbhBCJyNmx4Wj+ndk/GfPvkAnsF53w4GvX6/TTLojj2zgGQiFQURdoaRJRcJEnS73fH4/FwOEzTdLGYBfY5Ao8Egc2sKIr1pWxuiDEWx3EkZAit+XXDp6tih3ZaWrNgnU273ilj+x1pj7M9je27o5TKMjTOD4fD3d3d8XjcGG2MCbWXQgilVJZl3SzppHESq4CVxpAuszS/XFZlHaoz6soxiFLV78bjO4cj8jibzb744ulxcUrek4Gm0CuXSyal5HVZL+fzuijTKN7b2+Mq996D90gcSZD33joiX9ceEZG4FDGQI88YY1xKAkfgWEhDCUAT5KqwtvGuAXLgnCMLAMgIvAfvm8Y0TQUAUSTTLB4M+oNB7/z8HMBrXef5Mo5TKSXnChEvJ6cfffLjwTAztlrmC20bFfN33nvr6ZNT42oZKQ+uquuiKsuqqhqqCsY5b5qmKHWjPQArK2tMaYzJ86YojbW+rGwURVJEjDESV0zzQqzXoZQyMOUEHyo0qAuumpKyqHOtNTkQwhhrhRBpJ4vSpG7KsqqsN8DF3t7ewDSI+ODNB4UuL07PPv74408//wlyGB9mo/5A/OIv/crHH388X0yRs+Pjl1wmMkr64zHnXCZpEqvBzs755eT45GRvd9zpdN5447Uf/nA1mV6cnPRef3D//XffrcuyLsv90Y7gvKmKP/ujP24afffW3XK+/Ff/6l/df/fbTVn3Op0sir/3F99FC73h4Nbe0aNHj0jbndFumVfaGuudNi5KM/CMPHIpCIFLAQCSgHMes8ToNXe6cTY8eeSsKAo1HI52d/bEXt2U3pi8mEeJKvLF17/1td/4zd98+sXD07PjxWqp4sjMtXZyVRZ5Ucm6UXHqkFvnPKCKU0LugIDAWhPYIaUQP/nsM611v98fjIfhqaRpopQyxoXKIsGkUFLSGi9WFg0RAQEHgZxrsro2+bIwjauqxnsIVU+hWItzbBpjrVfKMiaSOLPGx3EcSYWIdV07bYxpGAMhmPcRQKCVDzqJAACQAIgheaudM+Q8gmeMScYFBxCQsTiKJHgtBVpTLecX3jWBJWlVlM4SE5HW1jtCLpSM/QaRxLbwbpzxNiTOGN/WrEavO0wGVRr47b33gTU2fBi0UVjQ3jnvcd0Od12gIWDt0YZ2OED6iltpS8xdZ0ikG+2V17uFlwWu62AAWEvzjSYIws57L/k1s6AtqWgdo02ZwNW9hKPSlepdYyGQbZkCmwO+qmhbgbutq9qd/QZ9SbS2hretpRvqEDeYyhtWwrbxES6WtoyqG2ZHuNnwfBlj3m+1M97YXa9abH/X8aqS3r79Gwf/UqX+s0erCNspap/1l579hulz9dsNCDQkDTZE3euDtFCVdtJaUC349WUzxjiHOI6BCSllaG5GjCul4jje2d9LkoSFkLcQQvIkikWkiDgDiKJoOBrt7e2Mh6NQdRx4FILpGbp/GWOQXNAKDKgNSyBDIaU1ur3lbbszfNJGBEPkjHNO1wCwV8syxAK3F9L2XIULUx6TJOn3+3t7e3t7e0VVrlYrbx3nPJJRr9cbDAYyiwLwfLlcLJd5cDaMtnEcG+PCyxXHnW63e/v2ndu3b795bz9SycuXL0+PL5yxQkggWi4WRb6qyoLINbpwznpnkjhJk8hizZF5z8AhiQCwEUTkdY3ICDlHdAAbJjxmddOAIysZY+AskeeMCw7WkmAgODEksM740ntrpKEG67o0VnOOnHlds7JYceYR/HDQI2+tMZYZAGwq3TTNcrH6vd/7vXffe9O6kgs4urW/XC3G4+gH3/8bhPloeODcGjDLuSRyaZLVdZ2vyvlsWZYV59xoYMxWZUNEDBVD6yxpsOQxtF3cjii0tqDf9LIRct15PCwVzmPw6C2BIQLnGm2NQQByvi7KKIoQoCxLyfD85Hg8Hu8Nsnw5Pb84rurV/sE4y7LxePzm62+I4XA3y/qErGkaoZLxzr4Q3HtZGlMbSLN49/Du3u6+bioVx3Ga9gfz8U5P15GUrNNNXnvzwWq+OD+9+PTHPz7aP5otVsenJ52s/+D1N//oj/74zoPXjl+eSikjpQSTXzx+/ODea6PB+OL49C///C+FUkxwrW2jTcPsssi73S5aROChjVB4GZzzjLHBYKCbNU9wW8pPRNbasl6VdXJ0uD8YdlerxWCn9/VvfPD06Rer1eLi4vLtr7x/fHb+B3/we6en5+PxGFSnrJqmMY2DyDOuFArJuex0OkxwT0jkmsYUVneSRPZ6yzw3xiDnQikhRKebdTq9fr+/KpYbwQFE1DRNVVVN0yghiYAIgRiAaNAhgPd0cnIGziNyziVjZEwT+gtoC+HVTdNYRaIDaRzHsZJVvfLeWqsJUBISSGQOmUMAIIZriwE4hNJnr02F3iF4wTFWIpJcCI6IPSXTSFknvGuMrZeromkqAJJSNsZYC1b7srK1AedkYzljoXeDCDIRglPLZaCJDb41bCK9nPNerwcbRRIi9oHrgjZA/da5CQTJTd0AAHGiYOSExkmIoSjUrVPjzjpNRODXwQPaqLuN+fIleiXIsvDmtNfjN5i7wFvQKlS/qUWkTXi5PdRVemXDXrBtGG2UEG1FFIiQSSnpepfGzeXdhFPcsCVubK+zjptuxbTVJrsNBrQ/uaFQv/TgNxTkdmgE0bV5ZWuvAstbO6zhIDfUKvxnGDcshr+robCtFMMIT3x7xl69i+310BoKG2RMmAgEWLOVb/+2fRZtaGHdlXud+KdYSGU9EUkVSSm5iuL1UOs3izOhVKykiiLGueQSAOSG9sB6l5eFtVZJCQAhHR7oOjjnHPm60srogBFGRC5QMs7ZuviijUXRFiEpbAJ+rdEDrzBVhLEd8dreaA8uhJAeoyjq9/vGUZ4XjdF5nq8Wy+3144xtoG6a5vJiOpvNgk1jrR0Oh4EqcV1CImA+uxCcFNRxHJ+fTebzqbU2SRIhePjVbH6R50vGaTDopakCdMvVrNNnodSOyAYZBQw5F510aANhLnqSLPTOiFQErvbWWCAlQr7HWesAPBAxhpyjkGAceee9bZg3qBP0qLiUkgkE01SzSbWcTw4PD0ejASOoaw0EVdU0hamqalUuPv7xJ/deu1XVC2PLTi8BRs7rg1t782klI5HwmIg8kfWOyMURcseiRA3HoyipqrKpa22tF0IIKSSic8Z7L8Qa/oVbMKbgMoQWP7Cm/MKAEQkRBSIi6yUTIJVkHD2Sg7qoqmUOAGknGw77mZTF5PLUupenJwcHB5nE+3fu7AyH/U5nOBweHh4WRZGmqfjoo4/KuhoMRs9evLicLoEEMLVYlaZpCCTjGeeZx1jGylp3Ma3ShL/91v3QzvH45BmgPT85f/b0RZZ0Xpyd0YtzmaUP3n67MO7x85dSRt3uLnmPiLo2VjtbN/2s86d//GePPv2cRzLr9pdVIZViSpVVDUJwA5zLxmhrLWEQ3BT5qGrqptahm0CkYkRsmqap6zffeMA5r+vy5clzLlDXzc7O6PD2rf393d/7vd/7H/+f/+9/9k//MRC/nMy11v3BHjgnhIoSJOTEmCeUKLgQVaNdWQWu7LLMm7rWw36cZf3haDabrYrSOMs5H5phkmahF5x3XpsGYIM/0sZbJ5AjQ+ewaUxVVcv5qlyVpm7Qk3UEYDcaiKrGELhAMl1VhYqYEAxRpamKpELmhUTBhFIqSZIo5ow7Tw1aHtYEAwrgeEDPEFAQDzk5JdJISSVCWJI7AkHOOU+WiHkGxAARPIJQkfG+Lk1Rg/HSEzeGC9QAjIt1/t4RMSGEYk2jw8prnUspZRRFB0e3Q6OHoigC3zYRBfhVCP/gGt4lGGPWWiFYANwAcPIbGl1ELhCBE0PnXNDaG7HLbsgptqbKAbimsdbbgYYdX0nTbodS4TpdQRstCEdobQvYgrDRVgUEbbCTG8FKbankDUVNRB6u6ilgEzO4cQE3lFkY207ediBhW73RFujsbx2bn19zDdvZwOvj2pz+pxh/q+LfDlr8Xa2E9vjrN9Fa7yX9lFrT7fHqzPurdr2hx8m1lbNtVWzPHuccGQIAX/N/MGBcekBEFcVRFDGp4jiOosiRFUJIpZRUURRJJQNaO0geIYRxbpGvtLORkIwxwa6qLpHWhqwBzzm31hqtW7xwiDUyJduluzYFtkIm7QW3N8XX3SiuHnp7uu37bbe1NrjBlQNAqEcj5ABY6ybP80uprLWCYZqmSZIk3Y4Qoq5rrQPhxLq8MNgwIaLAhdemOL/Q88XF4uwFIpZlreuy14mTWElOkvM4VsZyBoZx3+skUnLGodOJO4OUc+4WrizLxjoiUjKK47jbH5RlaUztyKNHyVnA6feygbWWIyolwJOu64CHM7UmAsFQcmYYbh6w50wgOmShOjoIXOsc1XXpXFdIlvHMGppMZroJvezZxfll0zR37tw6Pnn65MnjsiytYeNxfzHPPdkolkBMCIEs5pyXRe3JdLppr9crivri/NL7lRAQ+IiNMd5bpUQUSwDSWhOJNgoYnkgIQColaNMaLQjJsE8AOa6DSZ6YIzCurot7d+6maTydTHG0u5pM5scnzrlGqH/9O7/zT/7JP4niqM5XNk3ml5PvfPev5vO5+OzTh3fv33vrzXcWq6KqqjhLO51eknSzrPvm21+5d/tOEsfPnzxdzOacgVLqTtd1ukmv18uLha4Nka90NV1MG+vA4u7u4Tvvf5Amvf/X/+d3dw+PrPW2NgxFvzd01nLAPM/Hw52mqvb392tthJRc8yhJ4jRdlDnnnHsIaibMgrXWORJCnJ6e6sYQUafTiaIkvCdFUVzOzg8PD22hT05edrvd8Xi8szve29sry3JnZ+cP/uAPlFKr5ZKI93qj5bLMhhwRpYgsgbaWrLeOOFG9WOZliYhJJKuqqKsKkcbjsbvSEIIxrKrq+Pj47OxsPpsGU47ISSnjOA41God7AwSutV0sVtPLxXQ6v5wul4vVrVt3VquiqioghoyYEIoIAFQkqHSAnsghEucoJBeS7+6OjW3Qk5RSrPNSrmkqphJEBOZZKGbCgAZ0KFAylkQ8iWUSccYh0OxoK1xtqiI3tkaWqogb2xAReUMYNYaKxjYagUWcx0TMNivO5Xr9aastMCGk9XVrpQX5JYRkPMuynZ2d1WpVluVisQgeQ6jxDW0mmqYJzgdjLICw+j21ERMW/IYtETHk1ZB48AU51wBAsIEjXBfr23qk1Svh28DrzLawh8EOCBbMthHQbrSGwrZkDB+GVGsbXXDOBWudsatARyDFu6F+aGvjho7fNhSu/YRo+5qvuw7XwBatmIDranV7oq5NC1595Te1lK3zHdTeDUPh5zQ+/uPHtiZr7+4/+OzbWnz7UF9qedxYOWG0DyvgEnDTpnv7Cm/8d30XhIwx4BwRAbkjALem3GcbEsyAeA118FEUKSGF4EIILgSSk1KKTfwgEBOlaVqvVgGFxwULNVPee+1cmqbtqdc2AQdG1yp3Whu3zcK0qqX9RG1W2vYq2n4Rwthk3Ci80R6uKoaEEGmaAuDOzs7u7q4SkoiU4N1udzAYGOc456Us67IyjQ7CxzHodDrBaIiiSEVr8hXvvbG6KAprXLebcd7T2jpnQihXyt5wlGndcEGc42DQu3X7iHigoGjyvCxXRWNMFCXW+ijulEWd53lIUqtIBKEUx7HWGslJLjxzztmyLLTWAOhhnX3jAnmgjYG1h2Ctdg4j4lEsojhRis/nsyiKnKNONmAIurFa2yzryHTw4vjss4cPf+HvfS3rxz/84feJ4apcdfo9jy/LphQqtcZDWThvlFJCCO2M98S5bKx2QFmv2+30tdZVVem6aEwdc4Uics5op8Fui6wr2cX5OsgUIvFtADWJMwSuG1vVJRFxhkpKMpJ5OtrbF45+8etf6wjxySefOGDLyaVmrloVHNnOcDTs9n7y+Wd/9sd/k/RAyBjOzp+/YV7rDwf7e0fEOJHqdPacc7OZefzwr/Z2RvP59K0333j5/GmS0AzM/t5h0/Bu96Cz3x0OdusiehGXr99584uHT59/PhdCdlO5k93pJL3pdDqrmjt3br+cTF+//+D0sra8W0GkMRnt33OeZstFpDhH1dTQS8eMMQQXkgtKxd77oLACvxVjbDadXUzOx+OxtXY+n3POBYeffPJ5mB2llrqBTjb6q7/6m8vLyxcn87sP3n32cn55eWl94htxMZt2yjqIfo+QJEmWxc655eTUWquUlEJ6U+uyaHQzn8+fPn364PUHw25WFGVT5L1eLxZyOZ1Op5MfffTDXq9TVqumKXb3Rt1uxgV2OimZ271uX8rENMvJ5cn52dQYr6IoyJA4i+M49t5C6bNe1O/3Afx0NnHOqYh4Jqp8ZZzu9XtJopxHKaDf7caJappqOZvn+cKBVUpJLrz3ttHee8Z4JFNLMolVLBUCVZUFAAziMkmQI1AqyFOWVN7WVCKiA2e009pq4iDJGeNs45wrq5hzEsaG5BcAQqPrskLk5BwxZhhyJjmTKo5llBRFMZlMjo+PA8V1SKyui77iuNPphCLJQCSXZVlRzIPo5Fw6skTgPXkCL1gUJZJza5B8ZTQDYELKxlgiRoSwoUXwHrwPgRnfZtAR13T3nHNPdtNdJfDihYlwBITrwgSgwNVLKK10zpO9AiKEHWyjOaBD01i3fvGIyDoyFhnzsPa6AIAYQKhZFygYMgRAsLBx4JB58iFKyIAC/3Qw8AUyzhgxZATeOXKenK91TZtwQoiOwCZc7Db0grjFpmK9CZcXTMkWmci53EpOc1xz+JP3dQsypTVEX3mgtgys1T3eew/r2Mnmt1eeZZsDatXl3xYFuKb7W4vEWn/jw7Ad6EdbOdh+bo3H62dqVXawRNdFbpvCWtzkVtbmEVE4XChyo00VqA+Nznyo2wVExsCDI0/eWUvOCGHBaEOGo4+ElAjWCu+9trVbW5brwohA9MCRJ7FCwQHQec8IOKBAtjva894yZBEqgQI9oiPGwDlw4AUDrqSSSsmYoSCPFrhDwYRkjDmjtbGCMyl5nuccgQuUxIk8kmWeMcbAE2eMMy4YD+20N5bveoWHZMq60zZYJxxjjGCLO8QDETlyHtCDd+Sc87ol97TeNcY47wjX/bUJkAi82xkMXr93b9rNrDZKsNCJaulFv9/HpeTzqYgVGTSmUSKJkyxOlDNWSknkmqaRTFRNFXU7TMm6rp3zXPJ+JyPyWuskiRpdeO/iRDR1vlrmuzv99997O90VZ2fnP/mIzi6eOJf3kqQsVsY76HSGSSKBqrpWScylIOerqpqbEkg4B1VZNGXjnAeuOJfAnWNkvPMqQozzqmwMDvrj0q0aNMNhtpyc9pJOx+mDZJQlcnRrVzt/MZ3NX16IpHfn9mi2KJbFtDeQvV78+//mT/7+3/9Hv/qr/8A2e+Txwx/9zdnZxc7umxeXk1Vte73e5WS2KvKDg4PpshZClXUpBJ7PZqZp7ty5M5lMnbHDUR9jXK14r5txzvM8l0lKIJkQRV0zxrpZp2pKQrr7+oNytTw/PxdCDAejKFZN0+R5vlqt+mnXGh0pMdcNEXV6XVsXYE1dLncGveePH64uJ//r/+Vv9VP14YcfDofDZ/mLk4tHR9GtbMD27w8/flKmu3D/tX2xv7tnnD958VJy+Ru/8RuLvHj86ElZ5kVRKCG/ePLo6GAnUvLxw8+01rPLSf/W4PLyMo5Tjsx5n+d5kqXf+ta3BKrJxez4+LSX9S+biYxEVRXT6WT31v35fHp5eZHGKusk3ts8X87n8+DLrIq8rGreCGJICFJK5n3AyQdBHF57IgqtTRhjQf3Axl2o6zpJEsZYqAYxxhwfHz969Ojy8pIxFlgUhRDT6TRUxocenQCOS4kQ+sRGjMuyLJGzYIh5D1QWWtvLy1madRGRMS6lAGL5qry8nF1MJpxFk8lUSn7v3ut7e6Oyyosib2rnnHHeRswnaTQedpwzRdEY7bUppEIJzINmHPuDVCkVRWKZL4LI9eS8MQSOMwCgbi8D7+JE9btdFYm6kE4bIrcsPAMSggEwRl43jXXaW60kd85UZEJOcq0tkIwDznndlNbaqqqtNY2uOOeRSqy1WjtrQmIWnSPvr2jpQhwFcQ3h9psyMM45QxYeDSJqrReLRQg2BCACbopa+/11C93ZbFYUhfdeKRX1es65bVFORN6TMQaAeYAAcVjrpBvphS2N0uoo3KLjheve4Zf+6saoqmqdcJWyZezAKyzCJhC9SYFXVRVwGriVm6B13vdngeZabXcVOfwS7OOXjFePsJm0YIhgGxdpN2AT/2gjKO31860L5uveWk54B6B/9tn/1pn82eNL4wQ/495bS+JnXMaND/F6gW57yzdO3Z50e6P9tsW4MCDEQO7EiJi119qJtbYaCi48toaCEGtDoW06DhDSxlGcpXEcW2uJrgAoRIQeQxQNcV0BLwQLJeKhyI0xJhgIIWiDnmGMySgC7zxZa61zFjfMoS1qOLCTIQIyxkMvRLwCzBCu/3kXAndtkdF65TtHa1ynvYJ9eO/JeetD11dwANYzRwAAHlCoKPQWqrEADPYcQ091UZWrAgh73T6QD00onNXO+ChKdFUGuliPPomzsqyklJ2sv47nSemcU6pZLvMokpyT0Q2QyLIOADs5vnhz7+7h7uHqVhGrH+WsHo/2u6k12kmZMBSdTPUHQijVGM0lG4/HWRob7awmrY2pnDEGyDAJRbEgRo3RjgAwTlR3crkqlwsSSESz2SyNIsHV62/dv7077vcya5q8qFAlPes18aKxZW3tYnFxvjTGaAt/8Rd/sVzm89myNxzUtT5+ebKzt3v79t2mabhQaScrqvr09Dx4P0LIOI4PD/c5YFmWT548Ptjbr+qCC5Z1EgDQWiOCEAKQM4ZpGiOiNnVQf5zzJMk4l4gCUValnUxmwVs7OTlhjIU0kFuTzwKT4sGDB7dv3/mD3//973z3r4bD4S/98q+8//77v/M7v2OMe/Lk2d7B4d7uwdnZxWy6uH1752tf+4YYDcYvjl9+76/+ejAafvXr38ziCMkwFL1ucnl+oeuirkoG3nnHEfYO98t6vipyxsR4MO71cLFYCR4d7t9+9uT57sEuIk+i9M//7C/Ksu5lvb2DXRJUN3kU87JcZlm3KFbT6SSK5MuX58bZWlvjLGkQKopDNUHTGOdqvcbuamPWHCOccwAVx1EUxWlKRLKutdZSqV6vJ6VcrVaMsb2DfaXUbDHPy4JzbpwdDof7hwf94eD8/LwoCke2rmvvfZJlQWNFUZSmqbXWete27wSAqqryPF8uym632+/3syzTlXfOlXnjDY9lp8zL3f2jr773TSHY5w9/oisPzv549jej0Wg42onjTpqJW2qnqJq6boq8EipyzuWr0nmj4th5kReOMRYpBBCeam80IkgZCQbGNAzJW2adhsoWRVGWZVXUvc44JPy9tyRI+vCOQ5II7723zhsb1DACIGKhTRTJwP0upHXOWeviWDLGrSVryVjyHrwD59hmJV0JxIBqxA2ktFV1QZCF6qaARVhzS2wIB3u93v7+/q1bt6y1T58+DejUoIBDysa5IHHX+4fMsnGurpuWfcF7T3Azxt6qvVbut7bCq5pjex/Ysi3afa5SvK/obNoqYmxv3BoT6tK2AfB8zRn8JRp0W1W3qosCtoB9yQVv67DtcUPPtRYM41dYh/YItFVbD9cNBYI11dW29PdbEIr2LNtn/NKL/DuN7fm/8UC/dLQL6cavQirq1aACXLct2knzGzqWG1/duJIr9b/BfKwrDylk9LGdzO07CsqMiDwQAuecoxCcMQDGhAQmGGOh0YOMkzRN4zi2jSUAYMgEZyJE7cgTWec8ucZobQ2hB4aBXb6TZnzDAu6sJQp4V8YRwTvrtDfWOcsZC5buqy9C+/RvvDhhbNYnwLrg1zvvidA554FCPMFaa511jrz3gSbeOPIEHjb4CcAoinq9nvdeSlmWJTnLOHDAlMVxHHfSLng8OX05nVwS0XA4sJZAwN545/Ji0lRaSVWWZa/bTdNOyBEEjNeqqgKaj5zLssw5rOscwSklF/P8448/OZ4+zbJsMVvWlYll2sn6PkLvoZMN6rohY4RKOt1u7J1K4sOjw8M7d4qiMrUDB2SpqUvvrGB2cnlCYKw1znnAWBsYdxYX57O51Zz87GI12B14wN29w1t3j4aD3u/+zu8s8lXa6w139iUT3Sjlccbj6GJyLKW01n7/+x/+4IcfCyG63f7F+USoKO10kzhzHkytnaNIxUSU9TqLxSJJkrquZrOZaZqyLNMsKsrlYjkNLUNDNkdrXpYlMBMq4wCgruooiobDoRTR+ck5kGAo57NVXddN44yBoihKbkO5TRKn1hly3hMpFQOx3qAfJ8njL57+3h/+wT/8h//wl37pl374Nx/ty7uffvrpky+eMxG9OD1J0+6bO3vDwY4ABy++eH56fnZ+fu6ci6Joenl+dHQ0Gg2fP/n8tbu3wOtIsnu3Hzz89CdH+3tgoiKvqqoSSTQYDfO8bEpjvcl63elkFqUyTpOkk8wWcwMxc+A1BFwegfVkdnaHy9VcKjSmaYwmYEIw4zxjmGVpmmYLY9r0bRDBoalJeCcDICjIQdrA7OfzORFVVRWMJillr9cLumcymdR1HWxtKeVwODydnNWV9t5zGTWN8b5oGhOnCWOCrNfahHiGNZ48ksd8sWKEEY85Ca+Iczns7uwMdo1tFou93d0x8/Ll0+PZRS54orhybIlIztbOsjhOk1SpiJZLnaQZEWpNyITWHsBw5gXnvUFfSkHOFUVRVSA563XSOFaxioAcY8zWtnGuLhtyIGXU73adc1rXuqmdc8h8FAnOMY0jY4zR3iCQIxsUM/CyAiABIAAIQXJEGXW6na61FskBafLGO2dNqKr3TLDtYG8wFDjna0RxQDjaNesibHK6LaQu6N32E8ZYlmXD4XC1WtV1HUVRVRa0yaFa7611/3/W/vzHsu1KD8TW2tMZ7xQ3ppwzX76XbyL5OFSxVGJRNaiGbhm2ZdiA2jAs2RDa/43+gIYNGIYFyw3/0G60uiVLXapSFYtkFVlkkW+ecoyMOe54xj0t/7DvPXEzMskapINEIuLGuWfY+5y9vrXWt77lvScILa29ttYYC7DiqXgiay973NOLLQ+ubJuIobMxuJGl7nbYxA3dAenFVsVBAb6z6100fnPx7dLDYSnfjK5v4hjYYAt2SzPbqPK/Yrc2v9jBgk1Og9/YXjkOm/dylX5BlwIDL5uTlw8FL5pS+OsM/C/ZNm9/435ffajNSaSNLSh00SobcnX/kH1omkaqlSqa22gxunmPmzBi8yCsM6UUQhEvoJArg0NEwdgzAELOOGc8SLvyKHh4nAHjiCiEElJiINYjciW5kmtQ4oEx45y1JsyodFJ6R9Yg4oBzAAiB1bauQodrxkAJgeSRESNgDNW6a5SQUgghpGScQ9CEWd/b5Q/r4SMAIgzNh1aIyoEnCE+9X+FR8B6CUmoACo7QAzkiR9wDEiLnSMCiGPK+Y4yFSihvjTFGpf04jkfDoZKSeTc7uzg+PplfXCBinuf3bt0UjCkh8jStigUjyPO+Uso7WrbVclF45wKfA7xEEFrXTa0ZktZ6sVicnfF8KhBRMjnIR729Xr+3VZU6SdIkzZmo68lkMa+QR0meRXFKwGe1LUtjWyuY5B6sQ+88A9BNy9HFEfeMz2fzunIR4fYgQ5cmSoJtlYysN9NFRTxqLC1au2xsbztJeoNFVTtkWS/pG1M2RZZlg8FgsSiWy6WSSauNjGJn6eT0PM91EqetNZPZ0hgT+n41TRPHsfPG2JoxuH5j+9vf/vbjh48++OCDxXKiItbL+pFKgoSGSjhQqMyHfr93795rOzt7s+ni6OCoqmqAetgfkee6ra0lwWNjps453ZgoipRkXAEA5Hma9XsySt75yleOjo6mi+XPfv7B3dfu/S/+8f/qydnh8dH5o4fPFkVTts327s727t5wsC36+UA35sb+tel0+viLL/avX3OmBmqaejboJVvD4fbWIE/T0Wh0ephPJxdxRPmwHyUxlwIER87rdnE+m+rGVE25d2M/ktHO9e3Pvvwso7Qumzi11tXTyWzY6ydJ9LX33nn29Pn5+ZlUvDVeKsmFslW5XsS9I++BmOBhCVYvioRIKQmgrCvG2HBrxBirinIymRRF4Zyr2/bJs2dbW1ve+7pte73eYDSaTqdPDw6CItB4PN5850NEoaoqXhb9fp8xFqr/3VpnEBHTPMvTPIuTNEqFEJFK+v1+r5/FcbxYzMpy+cUnDx8+/CKK5f7+Hjio2qbmAoGs0ULiVj/t5SrLpZRRVTbaWSkj8qi1jqK4N+gzVIhYluX56dmCM8GYUooBJ+eNsVbruXXBceeAUkpvyWirW+OMBXBCMsaAMfBknTceHOOcSwFarCoKBOcqAgBwApnw5AiFB9loZy1oA9ahc+gBPTKPpITo6IeICOueQ1EUsbVSQiiA3vTDNgP1YQ0ty/Lo6Mg5t7W1RURRFIWgaMgWK6UAWN22TdO2bQvIAZi1vjUm1PGvcvMvig1sWv0rH24u4r8ET2za7/W3Lpl9V6z1y4eFDfGl7hrCXTPG4AWy5C+8NvJERMH5uwIUrpz3Clbo/tqFE7pTbFrfDkNcwQHh11CXvzomrNIQjnxdt79orP5zbVfw0OY9vryxF8meV45zZXxg1RfehdRs4KMppZRSHVDontXNi+nOsnlJ3V+pG+u1lsbmvIc/MsEhSFAgxxBXEhy4QETkTCglpULOQiGz955zDi82sezmBdaie7RmkBDRZDIBACRnjNFNbYxBIERK41gwVJGIhAwN2DbB3xVYwzZkNzdHbz10DIgBBFTkAzgM7wcReu9CxNE78D70fwj7IBH5lbwYI68JgTwCci6Z5Exr1uiWkS/ms6osski++87bSRz95Md/8fz5c2MM86ZYzOuy8LaVLJMMGfnJZJIkCQILmc00TUejUa+Xnx4fVlW1WC6cc0ywpmmcM3mejwaDptbOeMlFEmdCqKKYzGbLu/fy7e1txuXjg2eHh8fpIMvq3qKs5LIgB+BAEAciZi0nK4Sdzi7SWPb7YyD2bPr09GTSy7eGg+15rWOlru1dN23FUJ2dzyzxs1kxrxqQKhmMk15/oS05RMFlpOI4BWBZ2mdcMa7iOPYO+oP09PQctXcEwIVrrLF+viiOT87qah6IqAD+2rW92zdv9Pv5W2/fv31rH9B9/tmX3tv5fBpFjeChoQFHJOsaIty9tvPOu2/2e8OPPvr4xs29w6ODqmzGoy0Wy/OLSghx48aN+UXoPA6BZp6kcRjSNOt98fDL1vk0z3uDQW31n/7gh//kn/yTp+cnUkZNo4+OToizRpt8OHr77XfFa7fvvv7aa0RUlcv5+QXQ9s7WUAps6sWD+3eePXu29dqdvd3dpmy+8+u/9vmnn/3Vhz959913pVTT2cx5aGu9XJbE2NnRmVLx/Qf3B/mgbpvT0+Ms6z18+LA4XwohGHeeTN0skyR6fvik1aVSHCoSgqlYFhUYu4pjt9Z4BK5WCePw0Ashgi668aElhh4Oh/vXrydJMj2/aHTbGq0QrbXHpyfLsgiZiKqplVJZL0dE0bZVVV1MJ0KIKEoAQKmYMUHktNZV3ZLHrJfHccyY8B6081pbra3TmiMlkUyjWEglOSrBIiHRUy/NdFO1VcuBRyKqi2q5XGZDELgKYGRJvLM9yvs9a21VVfNF4ZzP8z4hb1uTxNn29narqSpK3bQAgB5bbdpaB5UE0+i2bpqmMU1LRFnW6/V6aeK9I0aCSxQSuQi0eluWZdM03oGUEQADFMiQMRbLRMXSWbLOa+uaWhtTV3VQ33DOWe89IiEi4xz5ukJayjWNDsIUhO7mIVgNaxd507Xt7Gi3Es3n8xC3ZIyFpu/WWrvW5w463N26tu7v4Tsz7ddcvE0L8fLPRC+kHjYNc2cdNy1u98WViXWXSuEBHnV46CVrBN57fLFwwHtPxAJQwEvnDTu+Qrc6X16zf2Epp41t8+66T/y6PfQVE3UFZGxe0iY7oTvXJrLpLGC3f7fn5vDSOqKwuf1yA/+Ltlfe5i/ff/N6Nj/HlyIK4XastSGjzzmP4tDOOA6407+YAIJXpSRW56Krh13N8osahZeHWtv8EDkAhig458I4S1xIRC6FEILWZQKrcA6A855WyTVHCIIhMARC611V163W4ewLYIwxHiRHAoE/tJjZCGhtRsVq3XoEFBw44wiMQDIOnJH3gAjrGAli+A1Wyuir6Qai0B8LvQMP3jnyvmvvTt6TsQ4RPaAjdKEFJ+OMC0FAhFxqbBrnaa0ZyRhpp8vZ5LSu6zSNe4n4yjtv3n/t1unxSa/X29keeNcUpBm4LFEcfVO3iCSEACQuGCJoXReFt1ZX9dJanWZxEsuy9IjR3v5OrKRgfDqdn52cMmCD3pAhnp4eJWl6r/dGPsjpuT8+O4YZT9I0TtTtt97oZf0ojpuqtXUbIUrFARznmCRyPB5prQmsJxvFSkWiOp04b5JIKRVFiltgIOLzk9OTyXJnZyyilLhSaU8QcSljY/LesK7rZVkRkW5tWc28B8GlI1BJyll0enZR122W5ZzLh18+unbt2re+9a29/fHTp4+5gNfu307TWOvq9OwwSeSNm3ta22rZtK0BYmmaVs2MiNq2JQJj6rJcat0sltO6Wdy9d6Op23I5rapma5wNh0NEd/PG7VbXURRVVRX6YhO5oq6Kunr45PH5+SnnfHdvG73/yc9//t7XvyElt1Z7b7314/Hus+cHp6djY1oxm0zfevDWRx99SNYpIatieevWtdY0Uax6WaybEsjGUpRG7+3cPn5+AAyjJAbA1hrjHZciTpJev2+tPTu9+PMf/fD2zTv90eCf/Z//WVXV//Jf/svDg+dkzWg0cNqoiC+L2dnZiRCMfFCn4nEcR5F0gDw0CF+zyULYMKzXiOica5om+EOBqs0511pfXFwsFou2bYP3YK0NFUQBK0gpQ6fp8XhcVdXx8bH3JKVijDHkDLmMFBBqY+u6EUoJISFQS6IksH2aYtLUUJdRnmYQSfLC6KZtRBADiVW0u73jrJ5MzqtimWXZb/6DfzgY9FQkrDVMin6/LxUnbxmDNImcIyKHALFUsRLgvWCqqprJZFYuyqZp27oxxnnrYhk755xh6CQQCmSRyLN46DSzDp3zXCAKjh4IDDnQxjRB5NFxRGYdEgjGlAuipt5ba7zHtm2rqmmaNk3TsHIFk8YFIgIRD5V+G7b/snQQNxrYdwvTZpJ7k0EGGwV4YXkN2ixtg2dnZ0VRcC7ZqvyBI+PeAyIHxnyoZF/97xFf8Pm6/3+JCbli82DDmr6MFby/zOsDQMe0CKx7vqGcH+4LV9+6RCTOrZpRbRRMMr6ubrhiejtT1FmaTTRwxYh2v9I6+/CiLwib19b92g17NyObN965qsYYY1wwroFf1o3b5lBfuZLur3/bbTPm8eLIvPpoG4AAYBPBvGpPgADsVitG0BQPvBzyuAkUrmR8uoNsgkt4FQzaJIfCBoXCMWQrag4jBB4yUUJIJriMhFolAgCYc86Slyj9Wtl5dSUIAGCMWcWaiNr2UvI8ZqHDMQohIimiKEqTWEqORIKF1/byysM7/EoMuvkovnBfNgTDOuwb/jnnyMOKoOAsWes7rSTkDIA5D4YAGQNOHACYULHMvG+axtaVIxQqynpMUJVn6oL7o8MnVVWlWTwebV2/fns0THfH20mScGaPSVvbJDH3rolixQUROSIDYK2zRamXhVOCp2mcZiLL4iRWccIAaDQaIrbOmkCJiOMozeJeL+n3e7P55PD4ufW+aSsZCSYZoDUOiuJ81E8iFS2ni/lsnipJTjHfMk7IfFEtTo5Oj09OlMrSLFvWy6aqjW3n09m13R1jKYrT2bL4/OHj1x68dePG9a3dHe0sAXiAsigWy6VurTV+sSyzLFNxAlonWQ7AzHTqCY3zdd0WyzLNe3l/0B9tcU5xoiaTycXknHNKItzeGTtnjNHj7SHjMJ+VDLhzhbUmFjG0IfgEnHNP7osvP5VSCinyXvSNb341juOf/uVPPvzwwyQDFbuLi4v9wb1+b5imqXOuLMuOVZblaVGUxrgoiWfzZRwrZOzh0ye/8u2vJmk0GPZOzs56/WzcjNIs/slPfyy+/2c//Hu//m2tdZIkSSpny6mnndnkYn9/9/T0OE0ib602rTbNxx999OzZs/HuThTFs9msauox50IpZ0lILqQ0xnz/+9//afKzJIr/+T//r9/75jcuZpN/+z/8j8+ePQsFMPv7u3GsAg9Oysh7yzjGsUrT1CNL07hpGCffrSyhlpSIQjFCKFsIFo6IZrPZbDY7PT5eLBZEFISJVuwPpcqyLIoCEUM2azQaDYfDnZ2dw8OjEEUPa0EopqjbJpwi4BIhRC+K0jRt25bSCBGzLIsiJaVgDAIPLE3TKJLO6SiWWjfPnh2Px/nf+/Vf6/UGo9EojlVVF9Ya55xrTFVVTdMIIRB5qw1jkGe9NM0R0VpbFGUxXzZNG/o/kXPesaKtOAqGnHMlhWCIUqSCJ9bqptFNWxO0cSyjOLSN9VJGiMY7b9EhMGuICKUQxtSMQfAGghMS1l8ppXPOewcAjPtL6hgBrSkgRNRVPQTDSRv5hZC4wbU73tmkzaKGPM+jKAqx7jzPb9y4UVcTxljbtsZcih4aY5SKiZB7b21Y69cswhcZf1csxysRw+bqv2lr4SWgAACM8e55C1SYYGLdhnJz9y3nnFjllS990/CD934zokAbeGXTMoXvXFm1OxRy5Tqv/LUDFrQBNWijm9Hmzn7N9t88OxExfln1gPiSP/3iIFOngfqfvG2OQPchIhK9gmZxZQw3P7wCFbpLtdYGL3YTwlprGYpNELZpO18+YzcrjDFcB36643e0D0REXD8buOKvQHiQ1l5+PhgwFFwKHrKDwDwC9yilshYQQ0kCAhAiQ8actYwzvlLsAPAIHgHAGw8Abq2S1N2ylJJB6Mt2+QBwzlFIGSkZqUASIgzqxZ4xFhDJKhTTVT2sRc86oLCG/hTIi97BJu/VOYeAHsgDek8IHj0gB611FEVMCFiXtkapiuO4ms0BPYFxvvWu1Q2dnR/Pp+dCiN3tkXdaCc4FVXWVqKhpGsdYAEzek4pEEsVSCiKyukmzTEqhIiY4IbPWakCrlDg/X9ZNlSdxrMRkcn58cti0uj8YnZ4ez5YLS/7a9b18OLDeOucYmCwV/SyeXXhnKmIJETqv8zxWSiwWiyfPnk7nswdv3NjeHT89OAIAa/zR0fF4PGqremd/9+jk7JPPv3jw5psyiiaz+en5mSOvlKrbtixLbwA5a9tWqVhrXVQ1l3FVVYzLptFJku3u7DExqaqmLGutden1+++/f3Z2cuv29W//2rdu37y2t7fjnEmT5Oz04s///EfPD47Kpa6q2lofqWRraytU+UVRJIWYzSZpmr777rt72zs7u+O9vb3X799W/z1+73vff34Io1F6dHQ0Ho/zPBdcMVYLoYiIc9br92fz+bIshlujyeRcCLG/v/v06dNvfOutfj/PeumTgyd1Xe7t79y/f+/x0yciV41eHseiHWQi7Y+3t8eRSNv67Pyi2dkaR4pXFdvbuffJB0+ePHp87/a9a4MshvTW7s7ps58u5x7QtK3vDeLWxYsGnp7MdXO0vTU+Or14+52v/rP/0z+/uXPtX/yLf1EuyhvXbr5x983DJ8fLSRXzrFw0kpLZWVmXPkpT5Gy+bIQQ9WKRZVm/39da67LOVCyUXD1529uhGC/Lcy7Esigm06kQMo4TY0zbakSWplme94gIoBJCWmulVIzx2Wx+dnaOiGkmsywBgLIs26YV3EnB+5mSUgoBkYCVBArnUqacD3Wzvw5gyFWsTygCZqxtTTObTcu6YAJ7AzUc90fj/o3r46qqlouSCWGtmx9NlFLD8ba3y7Y1xlrGIkaiqVprfRRFH//kw+VyWc/ntm6BmEIUijtOGkhKBcCqsjbOJUnGIukYmNYY54wl57i1qPUq649IpkGrtQlNqQiE4ESka3B6xTpERMFJCl8UiyzlaZpGUWqtreu6rduwWuRJLOVKH8n7VSrBGgvON0Yjcg4YSW6Mcd4yAE/oV72u2arvC6KUUVBBqeu2LOs8z6WMrPUXF9Ob1we/9u1fuTg/JY/Pnh9sb+8iCV0XBiCO4ySKA5XaWlssy6Iw3rQh8+tX0jEEQATMW0ewKuVHhLBaA4BgvjOrL5uT8GHnDiJiqMsPZikwNAMwWjXZWitBdQ46MGICCSB08YmiCLkgZI02nHMpGeMSmQBkobzcOQOwagQAAAGnrQwPdTQCHyr5rbVu1YXwhbhOiL114RkA6LJyWnftsNF7F9xpa62UqypBIrJ21ZuYhY7hhERgQoDVkwckFIQOGGdcgdOevHPeEQHwCEzweT0yT+gJHTACNG2jkkQKbowBZ6VgQgggF6GbzOteHlvnG2MJmCfM+oPlcomIIW6FiAierUJW4pVAIWydB78xdxxCUCbUzDAGhN6TECLIoYZWBlWplay8A6Xi0KU9iqKAqY1x3pvgloWmSLDS9QIA4oI8eedd0EtGIGAITBAy4Ip55kmQt56IM0REDhXnyLlgjBFyzqVUkVCRcAwFk6gEKkARegJb5xy1CIAIoaNkyHUhoJCCQsmB90iCEYULQqoFoORccrHBHIKyriPJpQzECHQIFogBKWIKhQQODgCQc86BgyVkFGDSGqoCEnlrOXFvnV+9DtzD6ho8YIgi6IAPPDlPzvmaODogXEEWRsQIyfk8FQ4845D2Uuu9cbYhjJjob40uTi+SZPTmG988OT47Ojy0VZuP49nZ0XO3+LVfeW9bKHPa9LiRgOlW74n2iKC1aWrNiCURRFx6b53SvRiFtEL6OOF56utaM661rrK0qau5Zq6xvio4i4qEU5zLh4+fZmn/K2+/M97a4Sqqq+b4+LiczOZPnunkvM/54M7e2dnZ5GK2v7N/cXFx9+5ry2V5PnFpvj/Y2m+MLcqlqc9d2/YzPD16fOPGNbD1k0ef97Mkj5VgjGybCMa5RMZ84x2QFymXwktoysZ7GGYDBjyN0uVyyWV0dhIq73wSp4yxNFIxlEdffsFF/N1f/e0/+If/CBG1bQ+PH3348w+SBLcGyY2d7KCe9IcsiXqMbJSJN966MZmeVs00SZWSy/395M0HcPNmMhrEWeK/+e43dvso9fz501PnPCNBi5mRDIrFdprsX9+dzKba6fnklKGLYzWZXQyGW8RwWeujs8lnB89KsM7rG6/tY9Qasov58e//3m+JyWTy+OkTxtitm3eI4aOnT4z3t2/fPjs7Y0xEUqVpqrVuW8OYODk/G5G7d2/svGdSzKYLLgURXMwunj97Nl9M00xyphh3z54/fPL0+tbW1mt3b/3u7/zmv/u3/9609c7O9t7OnvfWe+/IA2fMg7HOFIVFYkKEhqqBYBiWtjhNkiQxxjRGK6X6/b6x1hizWCxC7T4TLEmSJEnCktp9dzqdhor8S4cPUQjR1FpwlSRJLx8wxlbsegdZmrG1EkDIbiAiAkrJoyjq9XpRlKzUssJ5GUxnk6JYBvJE2+o8z69du5Yk8Yp7KGW/30/TVLe2qgrG2Hg8ao2eXMyIKM8HZd08fvzYOeO9BfCIRBDSlmStJQgBLeCCAQDn6L3VujGtJY+CK+90aEJPRAAU7lQpEcdxkFZERgCQJFmw+k3TGBOk0BIpIykjRB4ijatIBhEAdpGVEJMHAKNtqBxB3MyGhu4MHoGtUYVf1ZF731WZBl2s+Xxe17UQot/vW917991333zzzf/4x38ajpZlmeByUVZhFWPIumbqw+EwGBiAoNL4y0LfVxzQTScbNhxNuOIxr7fOGnUPTDDMeDXh8upIANugT3ZIhVbx8xdM3eUZNz/vwg8bufAr28tXvukZd9SEK9GFlyMQtEFx6C5m0+Hurmr1RALROu7SbXGWIZDW2hvNkJzjzmprLSrWS4XWGpAJIeIkm07ny+mERzGjzezDK3z6v8PW3ew6ygVd6iHwFUJRX8BMm/Bx82nZfAAAOqqNBwBa6xV1YRhgFFqis9XE8dBzlZAz5Mg4IgdgKFYvi1/tGVScGHuRpNk9Kl2Mc2OIAFbYFDcDO8hXvAQhmWCIiF2OgDEmkHfxpJeH6+UXxAKtBMgJATwRWfLWkwPvyBsi51cEcwPgujfo8oBdKJIhAudccKWUArvq2E4t9Xq90Wgcy3gxn3368YeffPzzx48/eeft13qpdK49PT9aLqd7O3t3b9/b2tr6gwdv7+zsnJyc/Pt//4eff/55EkOvF1dVEcVJmiZCAqB1TldF0epKSsmljCIaDEZZ2rtz796gP2qtcx7ef/+DshwPRuO7d26mWd62hlwbxUyqLe/9ZDJRSu1s7YyHI/RQlmWa5m3bGmMGg4Gz1DTNYrGczubL5TKO49FoFMexlFFY2ZSKlsslVzKsq0opIWWI6PSywenpqbV2uDXiTM5mM1NVWa9f13XGmPfknEuSZHdvhzycn58ryKTwcZJXVbOcF1vbI8ZpPBo9VcjApAl74/Wb9+9dE6BMa+uifn76RVUcersEX3pS9+/t3rl7y9nF7VujYW9QV4bs4mtfuf1/+a//qx/94Mf/7t/9zwBbSZIMB7nzWkRcSc4ZNItqa2fbOEczb7x1zgU8nab5wfNHBPr84qhuFplj+3vXy2p6cnQglmX17s7e7buvff7lF+fnk2F/yKWoddvPB5FUdVnNZotPP/n85OSk1+8VReG8H21tT+azfm94fHrCueBSuosLROjlyf17t85OD72zn336l9d2kzt37twaXP+t7/7aj37wfQQaDXOta20NZxFyLpCBB2Nd1bbG2aBKHToBWGu5FB6IiFprmrYRQiBjXAhkTGu9LIumaZxzN3ev9fv9wFdYLpeBNti1PguOYDBggQXtvVcqieMsjuNQUOS9j+Ms5CCICIB5j6F+HoDyLA8KzWmaAcBisQjs/aap5vO51m2ra2PMeLy1vb0dShy9JwAMrN2wWARJ9lBSwQUqJeNEVU1d12VdLZtWG2c8uNAd2pHz4IGBBwdAyEEwwSU6slVjvQ1LA7bWLZdlSMdIyYlISikZRyGRc/IeGTAhY6lCfSmwlWDiWhfBGWeDtKFQkgkeViXjLgX+hBDoqTG6bGqlYkSHiIG/3a01QQQxaC+GcSYg610/TcIxtTWhY1Ycx3Ga9Hq9nZ2db37zm3/2vR8oEQWgw5mIoihYr16/NxgMFovFfLYIY7iydbjB3IbLSsWXLUe3Jm4uuPiqNARASM5eWtDOcIbUQ9ffofucr2+f1tyCsAUPdXPpDz+vcj2wyZwgCO2mNiIKQRfQORfYZJu3s2nsu/Pii/n7K8imq2twa2GM7ux+Y6PLboc8MEyvwBHYRCRwaeEQCMi3WoOzcSTTJFKcee/Ju/lk8eCN28+PjlvjdKOHw+Fg0APkbdsG1udqnMETIv/P0U2CaFXPBwDOUchshu4i3vs4vhS1pHUwJowPvPi0rOACrvpvYWiIABBscLdPyCMAwIpLxWLGGPIVSuCcM7FSXsJ1n1UPBJesx6vtxS+f6hcnOnz3UjwKwcNqvpjgDJALBCQkoBXUx9Bt9YXH+6XXoZvQ8LkL/WeJCD0QWA/OeeMsATrntLc26KGtAIRbESwYdtkKxICNAIFzhlEUWWuZlYwxzoWUUnKhItbvRdevXdvZhp0t9/AhZ9Agd8ZWHvXuzd3f+Pt//1d/5dcGu7tn88XOrVvXTwdffvHzg6efertMomzQ789mE9Nqhkop4Zwma5kHTricV957cJDG8c54+9atO3GSyDh57e69n3/wgTZmZ9yzHoDo2v5od28wn7Ra6/OziW51WzdSqiztWWuFUHXdWmt3tvcAwDlf1wsiGo3G4/G4PxwAhLauXlvrvS+KUkSR966pNRCjdRua2WwWnM+zk1NElCpmjC1m05CaMVpXVaGUSuNEStk2dXlRxElfqfjP/uMPimXzm7/1nSwXdXX+3lfeWC5O5nPbZt407eTk4tnTp4cHR72BPTo4JvSM42zaxrz6+tde29raQrNMo37EONliPMp//dfeubadDHL3/f94tlgs5otTBMrTLEQqV8+6R2s9oSdCRixM4snJE86xKE7jmO9s9/auDT795IvHj4XYu3G9rKut7e37Dx6kR0enZ2flshqNRkVVBkmKYr6YT+bT2eLmrTtFWYso9giTixkToiyrRlulFGOwvTVSSqRxenHh2nrxyScXe9upEI0sWyXw2t7QOua8/tGPflSWS6EoTnL0zGvTyQgSkfO2aVrOeZTEQgjtbNU2vq6KohgMh8HrcuS5FKPRKAj+HB4eBq2PsD6GtqeMsc8//3yliLCmHYTlQ4pEt6VufZatFnchRBCgQADvXaut9yYkgYJURZd0h3XXGWNMUSzC6lOWZZ7n7733tW988z0p+fn5F1mW9XqDpm2rskFEKaM0zeu6rqoqmNW6rmez2Xyx0LpZlIUxxjgDyBAZeSBGHgEAHZD3zpFjKDxCgH7cRYyhtb6pdVVq512eJ/1+P8uSdRqbrHFEJKJIyaR1AEwILmNguJb1tdZaR8ZZzklKCchXMrTeg2nYmlHonEdEa70PkM0jkSdysGEzLFCwMZxz75y2VimV9/syilpjQqVD0zShv1RU1/P5/NNPP93e3r59+/Z0Pgs2sZf3uYpms5kU0fXr18fj8cXFxWeffn58fAxr0+vWq2RY94jWntArLceLi+OVv25+2HVVvgIUcN0du1usw9btdnlhLvTbXNW//SJn7sr2MlAIWfCAhzbvokMksOERdnfBN5oEbuaqO5ZJByw2b7Pj5a2SFhtW6gq6Wl3Axg0x8h5ZJLg34JHI22rZzrRjAErCr379zT/4gz/48JNPHj568pc//3Q+uai1GwyHfN2OOUAFhoSIHtnfFipcXtJLWzh4eFvbtg1AIXR1x42oQ5eR6Q7YDQIihltljHXWfJWYf3FYMJAUEIkJYJwxzoXgXHIphFDARKh48AjQNetmxBC9fQWP8mWUcPknxgGRWGAsQAAEGAQSEBBgRa9doxMGbDO41c14eD67s+A63GXW/Y88rFZXY6zxjhCcJe3WMkuWrCdHQJ7W9T0ExLq2K94DMGKMSRlFkQe2oscKJj02dbXUzcl4GN24noxHD958c/Doy4cc+O3bt+/RbfL49lfeHVzbgoiPIIW24Ghff+3W44c7n332GVB569atJGJFuWgbiiLpnCPrGDHTmF6SM8aatvXGX5xeSCZH2+Ptsbp5ff/g4GA2mxldV3XjnBvu7Y1GowO/BIBYxkeHx5OzCyFUnKWD3nAymXnvAXkcJ0IIY4xzxLm0HobDYZwmIXxrTOu9t9ZLuWJ3OuettcDQOQJgpm3SNDUtTc7PjPO7u7tRFLVtKxhSKDEj0k1VFYvBaLS9vYXGLRYLo/3jR58/P3x2+OyL0VDt7meDAddmylFnqUIi5+ZJbPb2U+6Xk3nDOdy5tz+bzT76+ZHEP/nVX/3V3cF2NS3ytCcEN9XSGf/arb3X/o//+zz5yx/+4C8+//xLGcVxhG1dOGuSONa1BqJYxsAZRxYMXCOkF7Pd3Z3RMOHCX9sfpgqMmT99WglLNF0sxm0zGo3iKO3lg8PDQ+eoqVoGPEtyQpjP5yqJ8/5QXUyFlAfPnz98/IgJqa0vy7ppNGPAEcmbVCFD5ByXUzh49mg4TLKGFotiOIj6gx1tiu//4E+aFhcnkwdvvRv4MsY75EwxwRiiJ66kECJKExVFrTXaGO8947xq6qyXJ1naNI1HyJN8JKW1Nrt7P7zqRVGcnJy0bRuq8vr9fjBRQTeQ1upMDGPvvTHOGLdeJ4EIp9N5eJcCtoiiKI7TJMm6BY4xFsdxlmWLxaJpml6vx2ssS++c07rtyrWr0kaKpUkaKeUsd855xwAwUplzTpu6WFbn5+cXFxehjWwsFRBjKGAVYA+sY48QahHJWeI8uIbgnLfaCYGeCDmXUSQhSrNemvWSNNW6McY4F3KOIAg8YNs2nTgV4mZmwXfrpve+qqogmz3IhyGM1rZt26zU/ZAJ6yhwqmC97gEhAKhIhahGlmUBNkkp9/f3Qw4iZGq6AI/W+uTkxHs/Ho/fe++9/+6////2egMppdY6ErJt27pqJ5MJ57yua8RVKXyIJJO/jBW/0vyvfnjJ8nR295W2ENYZjU07BBsGtduvE8jbPKBf13DaddfKTauzPvsLHl4XXXjl5jcIa1d2u/JruAAh+GZm5NLgvdhlu/uK3+gNuHmRm2GG1RAhAKzCXLhx2YjIAsQhx5AEgnNOIdy8MX799dffe/uN3/jOrz948ODp8+dbW398Opn//P2PjG6QiZWG6GqaGL3K2P+1W2fhNscBXtTwDlA4lHLY9daxOvyatdrdfjd04cf1Fb742LxYHtl9xTOOjDEuhIhkpKSIUKxCCz50eAXw4BA4IvpL8uBlfKJDpd3pNqc7cB5xhavAIxCRh1WkKrSsDHbbIzigmF9yU7pHFH7x82+RuXW1xSqK4KyxHhGNdyasE0B+3f+cgSRaES0BCSCwKZBWmtWMMZIy8oRh1bW6FQrIF9PZ02LZ1HWsJHpXDwbMan7t2rUbt16TIsp6Pe1tO1v0hluuroUQb731ltbNcNhHRrdu3RKCffnll6enp6HQOnSeq+t6MV0kSWKNWc6LYlGen17cuXOHA9/a3h3mvX7Wywf9k5OT49OT48OT0+MzsLmUKhZRmiSzybypWikjUNA0TRQlgksiAmBZ1suyXt22y2XNOW9qXVW1JyeEkh40syHBtJ4mJIIgvXbrxvbR0VHgGMZxzBCqqiJvyVtjjOR80Mutd+fn50KI3d3d5FbyyccfG2vTNFnMJn/2p/9hayT/wXff2xndHI7UoN/jjA4PDy/OHk0uZgDMNy6PWNbLf+Vr3+6Ptv6Hf/2vP37/wNTizftfn7gq3usneQZMO1s48P3x1j/+x79/8+bWn/7JD54fnizLcrFsnGeIgoiUjPO8Z5yz2ja6BQAkHMR2f2d32I8/+fT9timVhGs7o48+/lIMR+M0TYuyrsomEnJnayeN0o8++kihbKp2MExmi+V8WcZJlvd7d9+4z5g/OT8p6ipO0tHWdhTnxlhvXaR6w0EyyKPdra2ymBw++wKJLeeF6c2//OxDRu7NB3eRUVWVKLPDk/r6ncp6pq3V1gBA8Nm8typOIDDnrTXWeu+VUjJSRVEIIeIs1c4uppOzi/Pg8R9NloiYpiljLJAYtra2AGA+n4dlIhgq51yIQACJriWgc0ZrV1WeiObzeWAnAADnXCnBOSq1UgcK1H3GWJ7nVVUtl0vnjDZNkiT37t2bTi+ePHkC6O/evd1UdFheHLOpjFSIQVVVUTZ1r9dLkkhbWyxb3VKkcm18Vc5iJQAhxDmdI2Nc21qtLWehaQI4S857QB8SrrpplSIhhIySFDkRAZONNkV1Ya0l8F2ssm6No7qsGr8mDYTCkNC5otUVACADZIIhcaGQCaWUitKgBdQaZ3wTBBmRobYhk0qM8c1FFjkLKQbkDICABdU2b3WrrbHeAUMVR0qpNE2TNCVyw+Fwe3t7ONj6v/8//iURSimLZXn91m3nXNuYJ0+eHB0dIaKS0c2bN4OkY1VVsFq0vPfeukvT+7L5vIInaMMjfwklwFpUcV0UsFZa7ML4m/YJNmIGV1beLuDU2dqNS/Ih1R92QAbeeyFE9yGs09dEFIQp4VXYpbMrsDbtRIQbrH627iDVcdRpnVx42WcNy3q3/yuv/HIMgeGqO1SwTN7oxlmtOIslAy4H/fy3vvvd/+IPfu/86Fk5n+zvjF67d+fevXufP3yyWCyOzi7CkRE2WAIr0Pm33vDFLNJmjIHoBQDnnAu9OQJs3dDQXHUv29zgctG/alAR0W980sFBxhgXMSJyIYRUSkZcKQylkisTjiu/HzwAg7XO4+bMbkYUXvUnWAECRGDovTfOocNMJowj44AE3jtPpI0hgCyJ6KUNNupvO6QbNgur5yQImDgCb7z3joAFdBA4neRhpeF4ubGQJvPeexfcHEDkjCHnxLkIvGMkB143zeT8/FHbHJyfQ6aU0VguSbCRdzga34BsAFKqpmJRY0pgPJMKBsPk69/89Tff/pqQGMeqbevxzt6jR48ePXp0fHzcHw63traLonjj3l2l1HQ6ffLkyenZxfPHB/Pp4vxs+vY7Xylm9WBr1IuHdeqq2JZNXS4rahZaaxmpmEeDft9ox4BXZZmoJIpTxlhrLBEJFSulZGQET4wxy7JwzgEgF1JKklIyXJFwGRMhaB8MkJQyiiIksEb3t8db2zvn5+dffPEFE1IYk6ZpFMWuqhazqWAYK4lMjXfGs8nFeVv0MvXO22/cu7N1bT+dnx/0bg7Gw6HgNJ+AkpAkKLiywvzKt769tb1z7+5rUdK7eeO15wfnz54e//Effv/NN9/YG18PugNSyqYu5fRiuH3nd3/vOzu7wz/73l/88Ic/ratlFPWsc8gcekBiTuvKNsvl0oNzzqkMB73x2289mE7OJicXHOnB668fPz8W3/nub3z44YfPD56/9eBtgezhw4exitI4W8yWs/ksjtKjo6PZYnH95k3j/LXrN87OD2aLuYxE0zT93nYv35rPl7PJNInzOzfujoZ5nrD55GQ8GJyePEcnpufPp5MTAnn3zo3nR/PX7t+dLe3WdjlbLoRMgZgjdM4gQujI7om01shYElYWzoAhec+VnC7mHkHFkVDy8Pioqqo4jkXrQ2fCYMirqirL0hgTBMPDChIWiPC26NYyJqzVwYcORURaayFECKpHUZTneb/fz/NUBqqxECFi4b0PYXat9eHhgVR8Z2f79p3rN29ev7g4X6mdWzo4ODg5OeFS9Hq90N/FeOMdn1zMgWEc5zdujKqq0gfk7HK5aFdvsw+FhU631lpP5JWMOJcAjDy3BkMtt27JGs855wLDa0mEWtugp0G0UjxljDkH1vq21YwF2TUyxiIGhpfDVYg7CLAAIkuSOM/zttbBA2sabe26H5JHxkIyu2vosHJZjNFd8AAAOOdN0zx+/DgwOkMsIazO4bCn87PBYPDZZ589eOOtW7euGeM45wy5Umo0GunWNto45wKwCLCss3+d4V///AI/8ZfEG1781uXOuCG50y2jmx4ebDjffi18dCV2vTKBG3jF+xe4+ptAIaR1cKW78ELqYbOooTNInQN95co3b6dLkYT9wzi7tXzFZm6CNvgKm5PYoYpXGsgrUAzBE0AcSXRacAbeFYXxelos50ju2t7O9//se1vj7Zu376S9UZbEuqnqslaxonUb3815/LthhZcn168LdLvbDFv3WCJioCVt7ryJElbDy1Zj1cVjVimn8MAwBkCIxNfpKB5FsOI9SCYl4xIA3IqqyYAhArjV+22JiL+I/1626LCBEmDFFkZatxHxQNZaZGSUQmSMs0AXCBAkLE3rN/TyUPAiUNiMWnm67EPtgZH3gSKF4Ml79IieCAg8oQMkROBAAJcjhuTRo1cqDiU9DDlDz5jjqw5V2DblcnG6LI6sOZaCMa+8jSK1Nehvb4+vgcihQlDQGqGt7KVjiCVY7efnUmHW20LmjallnN5/IHb2b8R5XrU2SpK019eO/uMf/WGSJNb4ptGEXAjpNS1m5R/++z8ui2Z7d29v95oliKNkf7zjhu7g0bOzyRkK3uv1mGfgjGm1dSSj0GaICEFwVRQFADhHUqjAUQvyAwAgZZSmSOQ4k0AOwHjvnQ75H1pUk36etlvD50cn1trd7XGvlxVFobUOj0FdFUZrTz7o+jjye3t7cS1uXN/5xtfe/i9//7vjEf/5z753sixsw+tiCuBNW3F0SL6t67fevvfWV+6mSX+6OH/28QcXk5Osny+Xxfd/+Kfa1vdfv+to4E3D0bemTBCKxUm+vfvO269HUVQ3+vnRH1Xl0kFkbE3AtTYMuOLIGNONror69NA8/uLwtbv3/sFv/M7/9G/+u+fPDm/fvPX6/fuiqKr9vetWuziOnTZpnDAmvvj0s0VZTGeL0XinrZsgyH8+uUjzbLGcXVycJUnv/HwiZJZG/bpuT08nAsXta7dsSsmo16jlsL89OT0jzw+efWl17cFvj0aHJ7P79+89en7xBk8PDi8yHnMp0aJzzjEk4kII61e5HyEE8lXupGqbKIpOT0+rpn7w4MHdu3fruj44OHDO9dMsNB/DNXM4JJOCBFNYF7pahuArq0hYawmcVHw8HoeY+YpVx3me56PRqNfrKaUQkaEIlX6TySycKHjnWZYho8lk4klfu7YXOBAnJye3rt8/PZkWy7ZqJrNkmabJYDAYjrZHw+2Hj75w3r/19rXBYPDxxx8/eXJwcnLS74dOmBwAgJhzZE3Ie3kAFTGOQWTXB1r7Sl4iuEdRFEXxSr8ycIiaRgO0ALCupPLWuzSOAotTa62tCYHZPM+ttYCoranrOkTzkDPXOu8h0A+JVtjZk1dKBWebMUZ0yYlz4KMkttaWZck5T5KkLMvD46PhcMilUPFKEhsQHXnrnW7bR48eXVxc7O7s379//5NPPquqqpf35/M555xWjfjQOXdxcXF0dBQyfFrr4GqvJ/rSkF8xGK8M7G9ChO7DsNsVBcZNS3NpPEL36rXP7Te4iptHu2KoYA1SiS71DcMVhvOCdx1QAE8r6/4S+nmZGHHFQIasfPCbYd2yCzYUJDetxeYFI7LujuBF8LG5eVpVaGwObMjopZGII2Vq4x2YttZNfXJ+8t/+q3+V9Xq37r727te+6QC01oN+VjYtbFzMxg2+PI2/bMNX8U5gg8OxGTsJQxGYzmGiQ8ToZYiwefxubLvRYIz5lcscejytIk+MMSkiCpRVLhmKDmQAriBBGHFCIE/ee7XuitIdfNNsv+pi1o8lW42bI88ctG1LSjAuuwhiCIUFn4dvyNbBi0+1fzGHslI3W/MpkQg8QhhMj+A9EiAQI3Qh1YCAuOI/BqDpvWdMcM6JIQKDtbDE6tEFr3XjnU5jrnq9LPfMU7U0kuPW1tZ4vAcQl0Utc8VFyomTlVijs+R8ZL22jUNmAHE5X/QH+dbOzvl0+tOffTAvCmLs5PhsPNrinGvjsqyXJFlr7HxRLaaLNB9oibp2pyfTtjV5f+idsNYncWytNXWNHqeLeVk1cZYKoYyxy7JojYuTTCllvNPaeu+TOM3znIlOeA0ipZIkqet6xV8m5pyzzoXHIOLCWhvH8f7udhTHVVXt7Oz8/u//7gcffFTX9dnFRVEUwXwoyau6QCmm0wsp+P/2f/e/+Ue//1vjgXr25IPz08O2LqaTFqBiHNq6ybPMOayr9ujk0fd+WCVxD1DNZ1V/K9m/8e6TJ88Wi8XJ6cGXDz8kfyvPFGe+bZfEI4ZDs5zKKHvrV745n1V//Ed/dvj8SEU9bVDIWIgo7qUqjoRgF7MpY+zooP7xn//0q+++/du//90//MN/9+HHR3duHe3u7orljNcNqXhb+yjp5ztJdnT8HAdxXc9abj5++CkCF0IcPz87P5ke3jxeLCdF2ZqYCyY5IRdUN/Otncxx/fmzz+P+281xc3J41Mt7JwUsvbfTuNHwO7/zewdz+/Gjk2y8MzYM5CIb9j/74tHOcD8bDBcLgYJbR0neU8Ssg6KqkPP+aIACJ5PJfDljJUuzPMlyD7w/2HntPmtadnJ62jJfg22bdnu09fZXvvLG/Td00z5//vzs+KxYLgFAcVFbH3r6Nk3TCIgdpHGytXMjpJGQwBiTZJIDMQ6cs6qeW1NIJTjnbF2uGTSSAUBKqZSitoyipD/IGWPHx8dhGTo7nfzFjz5WShliRWXrtqo1GOKeKevnR8eLfr8/udB1W3lMjVXTmZMx297ellIuFgvvaTAYOGsvzqdcovdgjFVKSa6IyGpntHZNjcAFYwjeW21bQo/IuW69d5xAagOIAkCtChkI0AvJYmDgDVrbgAfJhDeIJIiobS05FsuMASsXTUhfGUs21GETc95prYGxNFVJmnaxmaZp6rqWXCEwJOTIrbalLcHTqDcUKLhHyQVn3JHzznFDjPu6jAWLj4v5xcX0H/3B7x49fUixkFLNpmWa91WSNdoUdWGNsWQsWnJIAmUauQbatvXOM8akVN5bBLdidiEAOOxW2rVlxg3v7cpy3P013M7m0hxWBLfm2EIw6gAAQYsGvGNCYMgzGO2EEFJxay1jwLnkHImctWZtD2LGQKwHjTbaMvG12+c9uTVocM5sepaIq3gvoghck87UBYhQtx4RuRQAoG1QIzTOu+A0C86BofXO25XYH1PR+va5914b01jrCbVZlc8HAj0AIYX6/oSInPeGPBASELLQuZjaBtC788ny9q3RH/zuP/zmN967/vbrg2qf7Qyenc312cnH/+Zf11V7OiniNOZMAjAPgRJLBG4laPSyedww/JvbaliYCQb78nMAIAggCSC0JwCtrXOEqOMk0cZxofLewHmoG20dJUnStEZKCRhY+1xIFXwAZ4k8kIMwCB44kPUEXCUhTe+JOBIKAUIg5w6dilQQaXbkkUKPEuQcgRgQ8oCHHCCEWMoqsc34ylo774kcIg/VicEMA64UlxUXAICE6AmZCO2iySNZZwkaT1JxwTgieus0tXmUL4qqbl0axUpKMlayFTMJiBEwAvJANvSQBtJ2xTr04TLX7EXdWr9+eyBcmfcI4JgFRAS+6pECBMCBrHZasIhxJE8kUPKIGLZtW4FjUdbWaUJ3/TKbHp/vbg8Sq0fD7Vja0/MP9uVCV+3kqE6iNEkSPfbVorYGlMyyNAOS1jJn2fXtu4wBA/b67bd/7zfNZHqeplEURcycWGuljKI4ffTk4Pjk4u4bt6ez4vDoVPWkAQdcy2HiJZ3Xc86llLEajr/8+GMxK4qqcs41xyc3bt3Ossx4hpITQweUJJkQ5mIyqarC2CaOYyBSkinFFWeMoUgTIqrKenZ2zjkfj8dCiOVsmeyM6ro8OzvZ29s1Rn/w/o+2trb+6T/9p7dvbP/L/+f/yzfLu9d3T47PqGkFl0VRqH5/Wl70B9mTwyd/+fHP8pR/8MGPf/LFs53twfFhaR89I9cqjmrVq0/YMtFT9vT5w63BVr/ff/OdNxpr2nJpfFnQ/Icf/CDewrd27n72wfsXh4fvvvXmcFTcvv+grapIwLtfed0zN9oenF3UQuatA44AFsjyrL+jkqG1tpdExweTjz98PBwO98d3v1SPDw9mr732VRFJ/qM//9mymH/961+7ef0dxrEqFndv3R72+ufnk+l0enh4HEfJnTv3OJfeWMZYL+uneVY1q07Qu7u7g8EgjmPdNHVdL01bliVnGMoXuTbb23s3bt49Pbn49NPPARWBSOKelPiVd7+2vbt3eHRyPpmnKkpiJZScnp23bbu1tfX2V97Z29s5Pz/3QNY7IYRQsXP2/PycPBKyGzdv7u7tFYsTf07WO5UlWb+X9nt5v8cEj9PMtJoDaq2nF7PVxTgrBTLGuiRC27bkvNZaSAbOerLeOwID3gXtYBmJbmlmHJylwFEAgCDeEJLooTmC1toa1u/38yTNk5RztMEqOO+dk4LFkTS2jay8trv31Xfe5UAkaDjsSxkBQNu2giMQj2IJ67o8ct567Rxp0xprkAjQERD5y2AjIQ/OWZAJ8mSNQYEghPCe2rYNOKbzUMO5ujxrWA/Wtylo3VmgK1ho25aIQlFJiFUEq2OtdRsLejBd3rq2bdM0bVsNAGmapmnKoyhcahSzqpn2e2mrl/3B3bfeev2zz76Yz6fes6ouq6a1johhkiRMph7c+flkTTRZhRCstUa7tYFfuVkbjual7/tKL/yVWxdO6H7tEIPfYCTAi1515xp2fvkmrWHDcYTNg7C1mh4ivHx5r/wZXyJRvrxPdxlX7uvKFmIPm9HprvzvlcfsDsIIPAJDDD5lU5T7N/bAu+P6vCzLN9544/r16xdnZ1wJwaUxMJlMtHYIMlz/ygcNY4Me6BfGBv5uWxdReOGaGQvvZlVVk8mkbdsoirIsCyHGl/MORMQ5d+RoIx2Dazl5WM9+yFp0pw7j6VdSRoDrnrdESCtiR/gKEBH4rrJmNa0baaxXRMLCm8vXuUZEHq5NEwnhQ2wDuVs/Ib6hKqxUgAqRgMiSZh4Q0WPgGdCqpQMQhQbZYT+ilW46raIOHYrzG8G2XzL+3b10EJxznveHs7O6rVtqa1+Vumnktd3da/vGQ1mWR0fPGY85i5C5VpdtvazKGoAxVAiNNRdN7QCE4PLatRuDQQ+H+b66vlieCwmcMy6QtbG1vmnNycnJ6fFhHOfX9na0tlrrZbUEFHl/y3solsskzcfjnY8+/GQ2m52dnWVZ7mGFL+u6fvDgwcnx2Wy5CGoKxqxKmbK0Hxa90Lc9RDdDwDuKouA0GmNCa1zGGHg36PWNbpNYKckF31ZKXZyd7u7uvvH6azdv3v7ZX/38yaPHe3vXzs8m4/G4Ba5U3NT6Z3/18+fPHuWZqKvZfLK0bTMapAiybVtNGslLzpIk6ceCc7Z/bUcQU5I/e/xotLu9New/Pz1czuaS4fNnx9LB0ydH89Mz5uTXv73flCWBKufTi0nFwTVNYXRLwIFFgklE73y7QsCSjUfXelX6F3/xYw/1/s3xV979ehTz5aIW//FP/ufvfe97vV4vTVWvn16/fp1zfufOvab5dGsLkiQzxhm9WkcYh+CZ6MbkaXrt2vU0TRE4eWjrxmlTVU05n50cHy1nScSj69f2t5Lbg/5w0N/6y5987/xiNhzsGmuK6qJuzGA0TpPe7duxjFLrXVmWWuskTy05rkSWJUkaASMAr5RojM6TvuCKEFrTCKFUJLlgRSNACRFHUZZGWSrTOBISBBdxorgQnE+nU2I8ZIkM+JghY4x80Lo3iOiM1VpLxdE7T5bIA1okjwwQkZtLlgoA1HU7n8/Lsuz3hnEcE0FXBBFFkXME1qHzSZYEfuVsNmtNQy62Te2dMbo5Oz48R7x9986gnw0H/agfbW1tCa4Ex7IsgZjT1iUxAENiREQenQtajNZbk0bJylQgMIaBNACceQdRFHEprHUBFQH4IOIeosTdq9shhi4XE9bEwMYIrQBggwfOGJNSJkkSRiAMWvgwy7LTs7NQNxLWZSICRitiBwGu6yy6dO+tO9uMwb17N9Icl8U5cj3e7r355puPHx01LRZl07aaGBJ6TtzDJb0uYJqATuxl8P9ySQ0KOcH80cbS9rLlvvKnK3Zx89438cE6ZrvCQ6GuenV6DKYXuoJ7ok27Dn5Da2i9KHu4BAqXTPgrV7K5EG/CF1ijmU0DeSWwfAU6hJ9DM5ROgMitmyNsDs76B0JEcj7Q8QCRA66HDACgLkqj20FP7u/sJnEsGd/e2/ne93/YNNo5WC61biFJGEMgv64uJAYbZSkvR/7/ztsmS4PWbcEZY9Y5RAzhwDRNg757N00dObRDcpu9RboHrzv+6o3Ay4ngjIX3qOP5M8aECJRJvDS6xGCVRVqRYBjDbuI6DuaVUwOAMYGeHCadd48BrPMFANpz3uVbwBrOueDMkpEB1xEaIoYMPPNAoUO09+BXxQ6se1Tc+tl2RB7DExaqO9eEhl88Xd2D181CGP+zg/MnDx8++vxxrtp+RFLGwKRUCRAUrT2fTD17JniMHgUysg56ob17U5V6Piva1qVJPhxuCc7qapkvZ2fnJz//q58+PzyQkisl6tnjXm/Q6/etBc75zs5OkiQHBwdVuXSGjNPOuTTpCxVXy8ViOnn06DHRSp4uz7Lt7e22bXuDIQCEyHFY6wLiT9M0zPXmItAl+0LxfJZlRVEEnpxSqlrO4q0tiWCb2nsbC1EuFz/7yx+/9957t67t39jf+4CIkxdAXrdNsXTJMI5jBrZcLNuyHPSjSGGsesw7XUO5XOq2GQxSIpou67Kqk11WLKeJSpbL2jbt2cX0bfbVdNB7cOf+zz/92GtIRZ7Ho1v7r1/rX+/3esuiOj49ybOt+eJisdR717aq1kZRsqwcIHHluXCAQEhCcBnFy+V0a2frsy/eP5/Mvv4rX7VWz5ezLB2KP//hnykJ77z9+sX5yb/9n/7H119/vSzLLMvOTy/quh6NRl9996uIqFs7nU45Qp6mQIxJ0ct6eZJ6YAzRGIMiNC8RSqlBf5Qm0Xg4unP7Nrbl1tZ2o/37P//o4nwOEE/n9bODo6qxUsXTRfHuV9977fX7k8ms0YfMQz8RbVvPFhfvf/jz/GlalmXRlDxi13f2b925nWW9umoJuPdQlqUxrvb9KEmcc3me80g6IE2uMRok50opqSLdJnkGnDVN0xi9Whes09puLscUQq6MITJkKBhywRhjQgZemHGWAMBb44zWTUuZNVoLzpWQXrigzR5JFaexlIKss6323i6Xc+dMnibZeJjFcZ5GVdMU5byuRoN+78Ebd2pnQmQiTWMO5D3UWIuGMwiGipED74zzoeOGFrB6apngnMdKCaUUCmmMUUpwoRiz1pqglUBECC4Yg7AUdqzvXq8XDEbHeO9Mjl8zxu26I1eo+wgKmBjEmowJVSRhN9ro/wSeQnAlUVEURQBQVZV3Ls/zwWDw9jt3hqP+vbs3k1QwoFaXd+7e/NpXv6XbH86X2hM2rSmbutE1cAC2aSAv6QIIfNPer40cwAZ0eKUDtLkWdyjhlX5SBxT8ujh2bYa7ddN36QxPIIRAvPSlVmyEyxD6Bq39Esp0IOaS7XjFYG9eamfSup9po6vky4s1bbS97g7L1ipAYRIDRWtTdKHbwoBT4FSssQIDDD7ocDhomqqpzf3Xrjlj/+IHP/zw5z9788Hr//F7P5hNC84AGTPogZDzkJcRAC+oFxARoAf6he2k/1bb5oht8ksAMTiC4YlN07SbXHyxiuQKtOoiLrTOCnWzwPCyP3WghiilQlHOykZyHrRNrV+HJYAjXAYiukBG9wC7dYPpF7EaAK0UzQEEgAXGGQ85DHRAWmvnuODYMRU8cwDgSFurLUdEZIBA62cDkIgseee9JSAi62V3496D6ZSpWNBQQgcEnYjoLw4A4fqWcM2rCJtgCj1nKJ3VmEbI8fHTo0++fJwOB7WGtH+xNZopFY96/Rv714bDwbOLaeh5rbVtWte2hqBCLpDDslzICT89Oz48PjyfnAWm+eNPPur1Brfu3N7fuz7c2h4Oh02rnzx5ImQ83ho1rZvOiyz2u9tbJydnP/+rn9ZGZFkWfJ4w5qFN8Ww2K4rCGNPv9weDQVFUVVUlnDdtE0BAiButM1wQ6reDaG+YrLV35JWQsZJCMueYlHwx14fPn33l3bdv3bz+5RdflMVi0O95a+7cvjmZTJaLkqxTEVc87mcqi6RuF/PJcmc8LJpqPmuEAOrFWmuh8jv37g2jC7LOVDqwv3d397/6la/2xuNpWXz6+UOv+Xjr+r2778A1L5ElMjqpjpSMs96w1YskHf2v/5f/6OD5mbb8hz/8ybxsi9p5xgGlBeRCxnE8qcmT7fX7yJk2Luv1T87Oh4Ntwbi+c+feO+8++PjjTx8/fnxyclLXdVFUbdsuFkshZL8/uHXrFiJ+9NFHp6enuoVev4+Ei3kxm35aNW2apru7u2VZVsWymM3bpk4iZVtbVOX56XnM6bvf/c2yas4uJkcn0+PzGYFojGFCamsePXkS59nt23dr3SZpFMVSMLMndpu2sq5dVj5O1F5/B0PUpd9Dzs1Sa20ROQpMomgv3suyzDm3nC+WZblYLOI4bo222hljFBdaa65khBDmdbW441pejXNyXkrJBYaIgvcuMIiDmZTCITkgFIwEl5IhuYwjplHsjQYVDXq9WKm2MWRdkiR53jPetVq3tnXO1XUJDCxpY9valImTUhFrqWoXo+3e219585NPP7Omreva6pYLxj015MlbYx0iR2TOOW+N1ZXVlW3tvGo453GcqDhSkWAcjDGmbQJpAFZmnqy1YZGS/FJ9JZi6lbTqOm1xJfLMVkV6q/57uBYzCJXoWZYJIZqmadt2uVzO5/P+YBBI9h3aQAJEtNa2tGow07YtAiRJAgCHh4eL5aRYTr7xzXdv3bxRVWUcFR9++H4I6IW3EQCs1QhMClHXLawsou9sdrA9sGHmac2f3zST8Cq4sLnor83g1W91e3ZraIechFBw1cyvxja0kVy7Uy8YrU2XccNF6066Ilh093IF0FwxXZ0hfBEn0SZKuPJJ92EIuRORMbqu6yBz7l8q+Lwck4BgVrUAAAihaUBVVf1e1lazSMg4ElEUHTx79lc//cvnx9O69mmqAHnb1M4RY8KuqA8vTAQygksT9Dfdrszm5dFeJTIBG4mVplm9I8E8bEZoNr9orYW1nuYqfLW24vhi6iFsfK1bwPlGQ5BXRUroVRLjL07fpohFV5zCiSh0l3NOkPASJAa04sl4Z611jEsJKgrt7QUSOCDjLDOMr3wD7tddUhyQJ7LkvQcPFJqe2nX/J+9XKQmOzAN0txzqIukX16h0IZxL9Ow9EfV7w9t37yWK5pMniTL9vqrqeTOftRBPysXx4vz5WclRbPUG02k5Gg6/fD5ljEVRlGW5lNICq4tqVlSTxVwqPhoNoix98O47990DIYS1enecHh4eTuaVSioVp9NlWdUGkCdpng9HiUUXWgQHVilj165dG4/HdV1zLqqmIaKyLI3z4/E4TVOPkGVZlmXLZVkUBSAGukrAE+GpCICgi8+FwOrqCUGUImqapm2NUhkhAiAAs9a3rbFm/uWXj87OLqxx5LUQsZRRP8q5QCQvEJEYJxaxuJ8MltPCulawOM+y5VIvFsW7X3vnD/6Lf/zmHTGfzj786fuJUMO8f//ea6OdXUuQzOe68otqUiycc6mUTDCJjO/t9pqm4TxTgvK8/xvf+c50Vnhi4/Hos4dPvnx0sKxbYsoCJ86FUG9vv3l6drS9Mzw+Pvzkk09+5Ve//sH7Hz1+/FT81m/+hrWWvLl5Y6+fZ0rFx8enVVGSo2F/EEl18PRZVZS7u7umbYvFYrm03hEAG4yGcZyGllZATAnZ27uWxmp6dsbRG93EcbqcT+dVmfezs9Pp1vaIgM2XVRTnQqXGup29/bo1nszF7Bw5kxHT2llvs0E6EH3nDCIBw7Zti2Lx7PnT+WIhZVTWzYpyRUwpxVe9xczp6SkSOW2vX7sWq8ihb8pKAyKiZByYd5wHnWZEJOeFWHnY4MkY47wBh84TAHlarV9EZEwLAJxzKbmU3BgQgnGO2jTGGAA2Gm2lSY5QC6HyvKe9IyJDFgC4YlGeeHAO/ayclu2SlxDHMTF3Mjkm4Ue7Q0BXN8ViUXjnlIwAwHlDXiNjCMQxLGckOcuSOFbOaIzjuN/v9wb9KIqMsYvFoq3r0MHFesdwVc8ZZCRbS13GIbhQXSeLsELx9RYKQ+bzIvASuoRLlmV5nnPOlVIBH4SiEsZYr9frDhIsEABwXHXQqIuy69nBGAv1qAcH50LwPJNxrLaG49lszjA6Ppp6K6uGtNEEDhmR84iA7HJt7Y5PRCGfEhaoX2JINg3/KxHD5ocvm9jNPTv+gVLxpvO0+hlD5eTqiQqopTMD3hPSZWhkJZ9JK/AAAAC+Y2JeMdVXEA+ugwovI5srKAEvcyUvfBguzzkXuiGEmNAmMti4d4K1pgxbJR2QaJU6ccZabTjCaDh8843XXrt7r1oufvbTnywqlIJzFlnnQ8NiJWUIs7+4vVqB+++8XbHB3Y24jR6nxpjlcrlcLpMkieO4m8HNgzjnV22rEHFDB4lzjpfztYpGhLepO8ImkgsH7jJ9QGwFttaCV+Es69kOBa6bOha0xjE8VG+QI/IQUDiiJEQfwhXggSNjzjtBDBE5AXnPjCfmvAOUniGEmARzK6YmeQdBZ8ms6yq9957WYiG4Cs0R/k2hXDcIiC+0uhax2t7fyzPxxWeFbWe98d5WcvOW4IvWlF88np9cLAsC20xm5tnh1GnD03FgRKVZPBgM8jzLsixJomqxdM7Mq3q8Pcrz3v541Ov1nHO3b+396C9+/Omnny606yn27PjiYjKtLKUoZstGqTjrjcqq+eLxU2v8/dffFDLb3d11zkVRPFssnHNt2wbz3+v1HFxeOWMsSRKjm64cOkBqpVRINgVpvvAzW8uEG3Ja67JqkzS3znMZx0kPEX/y0/d1a9M06/VHRjul4qOj462tLRCCnPfW1La19TLBrbfefP3+aze//70/fXLwkLwXQmnjVJzt7t0a7928dmtbxSfJF6eDJEtklOQ7TU3TokSVeIoWy/npRbGs3KiXE1ONg5hyckVTIlFqtUjiRG5nzvvf/s3vqFjOlzN7YbxgDpghAmZFHPcH8XC08/EnPz84fPIb6a8dnxyenJyJd7/y4OHDR0fHz9KkPxj2yqKtq/bg4HkvH4SGRs+fP3/8+PH+/p73Xio+HvWzLNPWXdu/sb+/fzGbG2Pqun7+/OiN1+71897FyUlZFk1d7o63XJLnO/29vZ2/+tn7e3u7t+7cOzw6AyaRiYvJYntvPF8uCVFIX7cVoCLm26Zp2pJzLiOBSGVZXlxcTOezMHN5f6hUrJgo66aqSqGF1+7s6HglxObJWy8Zv3PzViqjxpXeUxRFjnHtWkaQJymGQhfnO5EAZywittohcGTEGBJAR2ZkUAEgQyJvrfGmtdZocp7AcyHI+bZulIqTOJYiYoCV1UIyEAzAR2ki0qjVtWMeI97bGjCOJHySpGVZns3OZuWMI5Cz3hki8uSQgCPEsUIf+NueIXGGSRrnPOOcA8lQ15ukeXDutdatMd7XK9uAhhFwXClEwUqM2QVWQYizdeaQrTOsbNVmcKVlSaHRA4Z2eFIplWUZIgbx6a4SMs/z07Mz771cN/Ty3jNY9Rowou3ITWG9BoAsicqi6uf9i/PCaNjdudnL+74nHj08KIq2KIxxjsgD+CCHFVqAAkBQVkDEsizLot40Zut3+5K1cMXEbhqPzpasd7iKFWBtZTvj2q3lsGFHGXvBzBARwCVLoEt1B5rjy0Bhw0jQFaDQ/RC2K6CkW9YvYcpL8Kj7v7tmWHesCIPZpVq7q3rV+MDl2CIirWQBA1AYDgZVVdDKbuHHH398cHDQVo6xmHNurfOEQijvgDHhvAs0DAqM/lD6H9j0/zl0FLptcxy6tEuQvgjgtSzL2WyWpmm/399M4sDa9hNjsKFfCWt0xTknf+UBYB3NuZsptupxGtJSbOP4a8mmbuLWoSzvfQAKRJcVvwEi02UqZCUp7T1Z64k0+JDH8eszQkg1QnAJwJNn5Jnz5NGR596DR0+EnsASOfI+MBKA/IsX7BEwvBRsdembz9Yvgg0deF3f8gpIGfJV3TS6vliWi+lZoRsDVmZZZfHgZFlWLI5TyQURtM62utbONY1tmgrO53E8297eun3n+rVBP4p78/n0bFnMdRPH0XBRpFlsjFmeHT4+Pj8v2q3r6XDn2tOnB4dns2VjVW18OVexSdOsdTRdVJzL0WhULsqLi4u2bQeDYat1iHcy4du2Dc29gvkPblKWZdO2Xs+bD6zw8B6FPUMQtAOLjDEZZUwZEJFKc980+WCoktw5d3x8zLm8vr0XZ6OiKL0H4HGWZU9PDp3R6CyZttFVIiCP3/nKm29PT0+ns7NlWQgmk4Ttj7eu37w9XxR/8sOjar6cLEiikDx59OgkThIDftHMQcSE8fOTi/NJORrdYFHSVJUw/SjqCeb7OTe6YmCiXgbeSsXjSHhqtCkJrGXCEwKXZ+cHvX4mY7a9Nxz0srJcnpwdp1FP/Pgvfnh+PimW5c7OtdFwm7Noe7S1nC2apn16eoqIxpgkiZw2jMPWYOhc0usNlmVhtTk5OTs4OgxPahYnwV8P3S/iSI7HY2dtFKOKo9OzYyl5FEljWuQkIwZoT06Pnh8eb+/tqlgtl4usl45Go1E/Pz0/q+uSyzyKJBeoYrnFh03TECNj2lCy6L31ZBlTSRJNp1NrbRon3rpisZyeX+xtbed5nxEAgODcIHrnOGCepDyJMJQSabvGgOCcS5IEnLVOG6Odt57IOx/e2MBdc84xJrwDIUSaYRL3lIoRwuMCSZJorYui8D0JXIQgHgmMpALujDdFXXn02prG0VY64pIvyvlkMQl2WinhPQiG3oNUQrCsKpvAYTTGOudCxUEcx0AyRFOXyzljIiz4l7REEl2sFRlD5FmWhUyB3+AodCy8TVJb8C/DYhde+1B9HoQR9/f3A5QOTJ9QChEgRWdHrbVVVemmXRnadd7dGOOsDSyhOIrm86WS6WJeMVS3bt6dTRdV1dR1u1yUy6IBhkwKIZiDlekNWYkoikNXraIoqqpeO6kv9ACEF3s3bBrRTcfxF5mWlz/cxAFhAAPcQUTGYB08AEDf9WHaWFS7wDJ1ejiMsSsnDDuQpyunuwJfNq+/u5guxb550u6Hl28fEbUxHc1+A2q8EFHoztIdJHw5/BDsynw+F4IJAcV8cXZy2tZVv9+/f//WJ4/OAdBoK5RM4sxayxhvtOm87Y3t7xJUeOV8/ZKL50LAujAhJNSqqgppiCtH6NZ6WtewhFno0ABcMgZXo9cVDXnvO8wUvhs8eFiLH3jnupA+dWD2sl4mILYNTut6glaZ0g1s6ZzznsBzxi7RKhFZ652j0HBBYATEAJgHj55pCIRKDEUPnsh7dOSJwK0qH/0qZRDKWkKuYfVs/43mpcs8AgSpiRWK+un7Pz8+fOJ0eXz80NnqfDk9Pj+1TMa9UW0oz8dp1DOta5qWEyLGi6JVKhps5QBkXVO1ZlE2vVobcmXTWqepdnVda/1pWZaTycTVc+89EHv9TTnc3pssaiaexXmvNh4ASTvGnVTp1jgqm/p8Nu+puCzL8/Pzum4IkXNe17WKwRij0jhwtuxGA9K2bTstig4ZhOIyePHlCvfbaC2lRCaa1i2WZa8/JGSNbq7fuJPn+fHxaRTFdWPn8+X9+69PJ7Nr13Z00zLv0eticuGNXswm04uJdyYSUg5HaZbNqmI8Ht+4dTPJUm9jpGRrcC1lwKz9q5/+Zb/fl2l0dHHetLbW5tHjgy8ePxmN9/sJTSaTvsoH/R5TAP0eatfUdRYr8LScLxaL2WIxL8oZyJhUjEIIgigV1uvPPv+oP0j3b+w1Te2cu379uhhsvXHjdi6F0tp6D0mUpkn21a//vfPTsz/6oz86OjpKorgqW86Lvb295aL5yttvvv76G59+/tmP/vKnSZIBcgB44403l8vlw88ejkejN+4+UIJxBuAJHf+tX/vW+Zen+rw9Pj8b7++bunGo+6P+9Xzfebh9/9be7rU4zUKyZ9Dre1MsFvtlWQbxRA6tYj0AmBelNR5ti06gxZgjcWbqguJMJjFzzjskj7FInObHB9PtbZnEWwBQtbX2TGR527aLpt7GFAgZl3HEyaNzPhaQJXYxm8oEgUzTLq0GUMLYum3rFlXaSwXyuqzAQy/vxyrx3gflTsYYAVrvjHatay1ZVyCLeRRFEEXee+OBK07ex3E0n5tBvtU0TVu4YTpGLSdP5reu73s7jXlkQZfLRSxVLuWyqpkzVhvTamMBmQBUy8rMlo0QTMqIiNq2dY7iOM7SPO+PptNpXbec2aBN1JRGCBHHSts2yWJCT0RRouq60lrv7u4G6+uds42xhe2iCGWx6qEFCAROG9fqqmmNJxwOh3Eca+NavdofVrRQ7aSXImKMewfarDK1oaBRKaXSKLxOxPl0XsRpf1nUH338mVDRt//+d/6v/83/7eJ8VpSmNT5KkqptyJNDWBTNcCsLDSSDhHagZBJRFKl1KvTqMvayrd00it0OHb6hF9y+S5PjNxQMN7nxFwt789a10+OD7XE/Vuzi/PT6/o26qGNMyqLOB8NmUW3dHJT1jMDGiajACyHKZckI8rRXLwqr3e72dl1UWZYFmS9HHpF6w4HWWqBvmsZanySJ1la3Js8y5xw5JEIiQATOgBM5b7zxqLIr9jJswb/sBiGkuAHAIQuBnw7ndat8GL+Ncg8EAKYIEReLIoojLpU1RkWiLOssiZKIV8vq6eNHv/2dr//Wd39dN/X/57/9f3/48AiBZMwb3XrfciY9gVDCWns5V957DI1LV9qQ3dx107RhgC/v4hehvVc+A+FQzhvGWFW3YX1HRq2utWm0aQAzIVcVDQTkvLPWRgTeE/cAoT81ATAp2Cq27NbNOTsnEuPUhSiRQABwRH4VVwj9mdf9HhmQc945LlQ3UWvqyyqu02WF1i8gWudE6rxdnZdz7onQEhEVthRsnTTkGMJ+UkqhgUsOHrW2YQa9t7SuSNpEz+Hklsfek3PeEYT4hBAcmPAOgSFDEVAKIa6ba9vLeQoYZd04sW1XUm9EJAR3zi2Xy+WyLGvd1AVP4mpRFtNZnvY4k6bRPZXlglNV1G0TSc4Vb9raVLWSmUNb1UVZlkKI1jVnk7PhcJgkiRCsbU3TWG1sXXsP0bJRed733v+bP/yLzx/P9vevVyYlNnx2eBRFSZ5FtXHWLquqDiyrMp7Gcdy/ucOIW2srbUSetK3mkUBJjPtW18aKPEPvuNWz2XSZJMnpycWNGzeGw63z83PnXFmWUvIsSxiH8GCAM4AeGcXCXVwcRVEEIhqP0ru393Z29p4+ffb8+fPJWZVGwntbLs9Pjo6q4uL+/fvDfsRZtJxNTw7Peol/497d6/vDk5NHr7123djq8fNnxLmzvmoM4/H5RTk9evbg9Tectz/++fvPD55yZH3j0zotSz8/q6CRs+Pln/6HH0Si/+1v/73KYLX489oM7t2+4+v5+eREcJVRD3gap6N33v36F88OMJIQsYtiIhTl/aiFSHDOnPKKb+9fF1H0e//wd44ODwWg9tRIpbI8Z0xEIknTPI7jzz/7VJvWk2ttC2SVUlEkGYOyri6mk9CEO82y8Xhna3v84I23/vyHP5zMpsvlUilBRj95/FBKee/OHSB28PxoMBpu7e8mvf7t11/rb42zfr9s6jTrcRk58lJGvV4PEZtac+jvX7ve6ubo6OiDjz84ODoG8KOtbSGlVJwLyZio26YsamMcIaO2REQE9M4FP74olk4b78yNGzfG43FZ8qOjw7JaJklyY3vc1EtEZEwwrgBQOk6E6FkSj71rq2ppHRMsFhKdl20rNbNZliku8zTjgLFKAJhu2lVDW62d88gZl1HWy6WUy8pESaSUcOg8WGQIDAUjRKaUUnHEGIuEytLUe0qjuN8fWmsPnjytijoYJ6UU501YF5RSQgkhFOPCODLGeW+9twAscAzzvB+peFUJZn2SJMdHpwDw/PmhEGK5XCrHQ2Yh5BTC4judTqMo6irK/Lqzkfd+MyQbFsQAILz3RVGEEla3IWPcGBvMUohGdBJ4YcnuKBHdcp+mqSfjnENGddVWVXN+fg7Eb968+ezpcUgAEREyFAKMMYytJBFD5jKc629iLf4Tt1eiCgDoJdHs/CTiyMmS84nEcn6huIqFH+xtnZ+fR2k0u3guFIy28kW5qJd+2O8P4mQxm2tfpEpq8oJo2MuttbquskEWxWld19ZqAN82VZamRGiNB09AviiKtm1Ho1FwWwE8oPfeBfd2s76fNrbNe9n81XmH68aYfmP7ReNQVU0URchC9GIV646UjON4Z3s452eRoLfffjtLeydHhx988LHWYIzlXHAuhOArftwvjR3gLyjQf+WHf9utI9nBOhLj10Whbt1C8wWkskYY3Ve6P7F1tfAVls8rr7lDPFe2zTPSRkqoq1CFFzGutdZb5y/rgRHXEQsPLtya45cVoSRYR8ugdQKRMRa84e7a/BqkeJCrJ4Au/4Tg+brb5N8QnAkhkME6ibmKUBZFYZ0zxtS6dcZorZ31GjWCQxRBiVQoIaUEDqGlQp4PlFLOOWt8GJPQCPTg4CCOY84xkKw7vx9x1ZbFGt80zboyHPb396WMlIyttXVdt22DyKIoGgx6g8FgMBgwwsViMZlMnHPgqeMcSKmEkG1V69YyFGEwQ9GZ1KJpKu9tVVW7u9tEJHCVd3DOeu+NMb1eT4pKcCVFrLV+9uz5ZDI7Pj5J09S5lsARhWYF3rm2LOdcxZIJlWZ7N671s/TGjRtZv2+8f/T02aIuG+ucdShV0sviNE3zHjRL49vJ+cVkelrXpZS8aLxD0xv0xrujxZPHT55+MZmf7eyPZYx11VbnRwh89nbx1Xe/OuzvOWN8g0yKfn/vjXvZt742GW5db8lMllOLjkum8tQY41sTcbGzNb518+b2YFRXldi/OWJMREoJIeaT2SeffHBycqYb0zRtWS/SVMVxjIyyfuacm06nJ2fz50fH3nttTVGVMopZJL98/IgAhJJN3cznc2va45OT4XC4tT1+8923P3n4xe3X7skknc5nKs2yvGfJqygRKgaGrnWIVrcWAKqqImiJiMCJOB6Oxv3ZfLGYLctCJbHW1hitYq7SBKWqysYYY5DiOGLA27JFdBLR2LqtFlbXXjdtOc/SdGeYjfux9c45A1ACcABO1DKmGEcgARxSqTwBFxbQNJWx1gCg4DFIrbiIogikEsgEk02jW6PrugViyBmXAjlngocnhUvOlWSSe0/W+bXAWqjnZs5557wh19TatVQ7Y60dD8dPHz0tyzJREXhkKJy1zmpjHDmUCoXkyASRcQySJHPrttfWstC9WrcGkW9vb9+6devs7GwxLy4uLu7du7e9vT0rpm1rtDZRlGRZz1pLFBhtIaoJjCEAC7xoa72SGBz3kCkIUlpxHIcoXKgYho0UeMcPDyikc33CqtoxJTtrJKXUxmltGOehNm8+X964fvvWzduHBxfeG865tm2Qi9ZaC0zCkhcq3GhDXPlvu11Z9X65HbpiJ2gd7e+n/Pnzg9EgE8ByJa/f2efItgZb3/zGr3Iu3//5h+++97YHm/cij+0f//F/iHyEHqXi87JqfKn6/WY2rwHyPAfyVTWzvh6MRoAEyDxZchbJ60bPZgshVJrkiNjPcmvtWnvSE7jLuP0GUICXKI3h1838sYcVStj0ZX+JPWBcMi4Zs0RkWm20VpIj0f7u9rd/9RufffTBfHbe7w+Fkv3h6MatGyd6Ute1MVZxgYihCzcX6m84a93Fd5byPxEubEYpwgHdixt2QlhBLyHIDgGFf7AmaoQaRyY4Y4wJjpzBmmdx5eK78WSXSkqXF7MZ6dn8a/erX1egdFcLLzyEIXtAiJdFj52IExEBWcZYa7TU6xiDEIyx7rzdgxGgEsCqPNLRRqQAPGMMCLqa5L92nOu6RraiUTPGhJBhPAeD0bKYa1Ma0owJA2is994pKWyjiVVRRB6ciAQhIrAoiol8qGW1xiOQMaauWQABYQCdWymUO+ckk0F+TQhhjAuFPG2rQxLBYNs2pqqKYMKHwwGglzKOopSsC3lVY4zgvNfrzabT4MkoGS/90lpyDkZbfSGEkCglZwyUUgCeyGmtAT0yFbhfnLPAAUcQUsRJnGZZ1vDGOV+WlTHm7OwsjqO9vb3hcDgY5ta1h4eHn3w627uxwzkHcoohl/KiLC6K+Xw+d84Aw1LrShumRG3s2Wxyq9/3zB6fHR48eXo+PeUCVCJlRCLyy+osH4psyqdT27r50dkXn3wpGGPlwbIoisnZxXJe3rvz2tZgC33roc23tjLV6ycjBafet7kYqFSpJJZ9yRjrR32ObDjcLuf1l18+OXp+KJ4+/7xtmkjFW1u7RvtZcXYxOaorU9ctMkyyVAgR6Gna2aKuWg0EmPVyoWTZ1G42bZ09O58wxrb3dnXdaO9kpF57843r+9cevPXmrK5Pp9NWa1+VF5NZlJc0vYjilAlFeBEnWa/X84TFyQljPI7jOM1D15ORiltnL+bzw5Pjk9PTO3fuAIBKEiYE44IByjhikQSj4zRmHp02znnF0DtX11VbFno51+X81q0b12/sxypaLGbTaaGEJr9qhuasICYYckTemkZKGSfSWFHVvqpbjiCEiMRKZQg8ETKLvmnawPkH5KGlLDG0RN4ZshRlfSY4SE7OITFA4pwRI/DEhTLatY1unK6w9cYS0cnh8Z07dySX4CBoROq2DZ1syHnnPXecEQAQY6gEE5J5ssa2xbJyzoXaG631aLg1HA7btmaMldVSCO6cJfKDwaCua6VUnufOufl8zhjb3t6ez+e0ll0Ki0MIRAc+fEfnGQ6H/X4/TdOiKJbLZddwyG90SOLrXouw6o6zIkx0FYwrj4UIEY1xxhgknfd6iMwaDwCDwSAgeu+ASWFtLThjKIxxyFy3SnaxhFVrjL/T9jc0PPgLFJkmZ0/3tntff+/tNx/cHfSzfpoopZx22+Pdsqzf++r9b379HQcWuKmbxZNbO3/2+ePDw8Pd8dY4z7fHoySJj7nb3h7sX7sWZ2mk/OODZ/OZiZJkkI4iKc+m1XyqEVFJNujnQqjpdC5khIiBFO/RA4aQz7q51NoQbhoY3CBLbmIFjytg103KLx8QISPGJbCGkAF4FvolM4gU+9Y3vt4sp+9fnH788cdx/N6du6/9V/+Hf/r8v/lXZ2dni0UZNK21NsEDD9yOXzLOV8zYpr++CX3+tls3Alcw4gvB8w3/vrWGNnL2sGYMrEghiMjQI3QMP/UCm/UVFNSr47/enLuUZ7gyWfSCHijgRj1Fd0lhWQ77MPDd22Fa3zGUpdSduFZIq3c37r0PTgvhuswYu5TTqpY3EGs2ZuSXjbNzjsGK7UQMO5xkHRjttCFjPQWHBLx35BDqtm30gvHKAWW9bDDIev2tug0CiNo5z7mQUgIxZ2mjOrGr6HGMofHkjc2jWMagnV0U1bIqF2VhrQ3kRGttU2tERIFc8bJsZtOF1tq2ejI5Xy6XUoo0S5I0vjhfscFWo4qCMxnHQTLOW6vL0hhbA4CKRNNWK6QipRAhfiYBUGsvRMyYco44l9G6zs5aWxQGkbRunDf9Qer82DnHY4UE1pJl2HiaVY3VejpdAGdSyqWxVdsK8g+fPkl++lcopKvmy+Vyupx47nu9JE0igSBjV82LrZ3E8b3hcqmtfXr08ZPjT5VS5sLquvGWvvf9f/+Nr37rd37zd772lfd6eb9enBtHTrd5LFOuYq2yXtobDg7Oj5lgEYsZ4HxSFIvF40cHBwcH4vUHN54/P1wuy0V53jZ+Nj87m5xWVTvqjZVKgXC5KBFRRVmSJP3BuNXWeN9aQ8Bao5mKUHDGRVVVW4M+59zpNkqi4bB/7cb1fDj4+LNPJ/PZydnFoi4BmT/Hoqyv3bqZ9wdpkqeZ4FwCIHLOOJdSNo0BIKWUECpSWb8/3N+7lqTZfD7v9/tJkkRRVNftcll674WMvHVknPfgW2MazbkHQ+CdAEyTOE8Tr9tnDx86bwMeNK50NngVhCCEUFJGgqu2NYVz2lnnHBcsy9MwwUmCzjmrjXPOEDAmrPfIWZrk2vmmbY0xyJiKkyRNpVLaI1OcKw5egIxDo7mg4hdx3tYNQ0VACCxNEmfsbDK/ed2P+qMzcdI0rTNeIkNPcRTZ1mrrPPfIiCMwyVGpsm2sddZaY1vd2pCjjOO4P+hxjicnJ3t7O1LKN9988/nz50fHz+8/eDOO0xDbLIpl2NlaH/4hopQopQp6OG6tQBI6YQ6Hw1BoYIyZzWYryLyOJYTIgdVm0xMK8CLQWkMMwK47GbKV/oH3DhBAyggRJ5MJQ9425uzsoiwrZApDc23nABljPGQWN1kCfwdrARu2Z9P2r5bhv9kXw69CmN/97b//9tv3b1wbl8Xc6fbBa7fLZf2nf/KD+bz42te+dnF+VrfFYjnZ3hv9vW//2lb87p/+yR+nafyNb3z9xvX9s/MjKV2WJd/5zreiJAZsm3Z5MZ83dXvt+s6DBw/YW699+MFHk8msatrlYg6Ay0W5u3/NGk9AIXS5tlKXlomt+1GFi9w0PJuYAABWCtIbpuuVvm+31cYS49aDYoxzheA4krf2+ODAW5Pn+XA4/PzLL65duyaEGI13GOu6b1wWKLpXdfDqNnwxhk8bipN/LY75aze3rvvYhFO4LmS4pKNeFiBQNyy0DgzgBlulexpfBi6blh5eFdShjVbg3Z82sRG8GFHw3jtHDLCL28FaoMp7Kxhf4YBVzmQVYOAIDNARgPPWGwADAMjFi9MNHtAD+nVE6ko3tQ4orG7nr0NpaZoC8mCYPQJjPDSWe/786Oj4rCwnyLS1DjxIoYhTa50x5Lx12tVtY6xVUZTlifctETImglZbqNLy3hGRR884MMaF4MYYIktEjCttjDZOa6un86pui6pqjUZEZ1vmGXlkQbYfXGuawKUwtgVnndcqwjxPhsO0rKZFOWvauq5XZVZJkjDGelmaJAljo1AT7imp65rzlUQsEXUSTMF9YkhKKa31cjlHxDxPjTHn5+dlWRqjz8/51tbWYNjL8zxJEiKyAsW67UcsFQoFxGTa11oXrdaOEZeO2MnJGeEH+/v7o9x58CpVPZ6nWYxkKl2ZBkCYdChv9MY32E7V1F8+ejiZtBg32Zbss7gum8n0/Ad/+f8rmhNDi3/wnd+azaYemBD6+vUtpuTR2YlxrdVljNH0fPr48eOLiwuhZFAeGm/tip++/6P5fIkgd8d7HBMZCaE41thak2RS8chZTgSemNGAoDS11jouBFeCKenIL6tSCJUmSdnUjLE0Tawz0/ms3+/Xun349JmXctFW08UiybPFsp7OZiyOZZJv7+Yyji6mM+fcIB8AY+cX04NnzxaLhbU6z7MsT2IZ37111znz0aefWGuL6ZwNWT/rxVJNp/PlfAZOWNKcMbIEhpyxiok8Ssdbo1hF1tpnz56dnh4X5TLO4tFoBKwO721IHEiplIyZ4EQ4n8+rsomzdDQaJUkSKo6cs86smu0wAsYcY0JKWTY1AUPBExHzWKkoUUkshNB17Tl6BB96qQEYZ63VzliR9rwHxgQXKJkc9Ppta9riXNd6Z2fn8OD5yWIZZcIYk8ZJKPhs6woAlFIYoeCCCZ6wxDkXFhkbWyXjKIo4l5wLAl8slzdv3iaiu3fvjsdjY0xZ61ASprW+cf3m22+/Xdf1j3/8YymUs95ZZ7QVXAIgZwKBNU0jhOj3+7u7u/1+HwDatg39n4JTGPgKdt0/uqMOwNrR6VbVbuUKL9JKl4ZxIQQ5zzl3jp48eapUfHJyJmRpjImTeC1bxD0Zzrm/jCLgy8f8W230Kj/1l+y8+X+3hu7tj37ju792/frYmebs9NmXn38+m07nk/LDDz9ZLqob1+9JlUaJampvDb99+87J1gGP1NnsAhjtXtva2e997b030yz69NNPVRQt5ifOFowaY2hvPPrt736H6to29XxZTibTo8OzutGMQxpHk2YOABBqdwmRkScEWKk8wUsx9k184DcogR4wxCE20cMv2by1VkgIzqLXpnGKt+RhMavf/+lP4li99dZbjOPx6ckXD7+cTCanp6dVVeFa54dz7xwF/sorj9/NyGZQYdOO/s3m9pfN48sH6aAVvFg9CwDIGBB1LNmOyspWbaUvv9VRH37RSa/MRTgFw8tvdZE52CDPdsAijIPWWjDeAUEiH77ctrXkIrizgkFX89yFEDZfFh/U2ABg/Rp2V+W6MluOV6//b1O56r2HNfdlM6JgHBHjQkacMwDtHDHOAanVGpDJKJPAtHWNNvOiMN5xrkL8I3ydc+69JQKtDdGqZS6GejZgACCELIoSsQotcIP2UYfkQoAzXEyaxlEsvSXnDCIIBSlT1lKcYhTDxeSwaeeA2OpSm0bKSMqoLGsVqTRT/d5wsVi0bRvH6vTUIGKWpcYYXFODuxnnPMRNjbWac25s68kH9ca2DbQPRJBSRECmaZq2qViSKBV7BONQV9oYoy04FK1upEgYKEAPUC5m8/PTM4kJYyCjhDFgghNY9EiCrLeAlkWgFAMpRjt53K+2trZ2B9tpmmUqLWfV+z/94E//4q8I7Y3b127euAMkZMxm00Wz1EdHR1XTCCXBpueTi9nZfDKZa2viOE6zTEop6rpSSmRZHqVRU9u6KZfL5WRa7+1kSiVZOlDSWuOt9bOittZCwmQsuJJxksQZ1+2KUSIjNZ8VcRxzKaaLWVMV169f90A/++gD76hpddrLVZrFBHtJily2Rp9fTI9Pzk9PT72l0WikuFgulxdHF+fnp2W5zHtpfzSQkjtnCP1WfzCfz5vGmKoBlTJPrtHFYplCTzsdSRWRRCJwXiqR5kks0+VyvljMy3JZlLPy/8/bn/XKkW1pgtiet40+u5+ZQ5DBYEQwGHHne3O4U2ZWd1WjWtUtQRAEAQ3oWX9FkKCGAL0J0IOeVCmpurI6K6tQyrxj3iFuzBGceXjm47O5jXvWg7k7nWTEzcxCZm0QxDl+3M23mW1b+1trfetbZemm4PzyIuRk7RkAvHzyHQCcc+MsY4xSrNSSEA4hxKrO7QGEUF3vV6fgSpFTRlnoe35AfA4QscBpZxnnCGMH3VLbxJiyyouiQA5QyqtSIoCMAcoYhv00TaEUw8vxG7dudjud+WzSbDZlJQhHWZZVRV5VChHqrIbOQggphiwIlwUL1joHOOcIEmutUrIoCqnEeDzM8zIMw06ntbu7+/Nf/rYuWN3d3b19+/bbb79d695fXl7OZrMkSYwxVVWhVUnk2osCANTwqBZKWouRoZX2SE1lWBcWP9+KVmHVVRL0RaqdhSsRY2gNuLy87Pf7x0fn1lWEMIxpKRRGlDFSVAphwhhaH9htfMff34Stbd+XuoBf5Sm9ChHqE0nT+Wh60RtEYejdufuOEOK3v/rdw3vPIGAYBb/+9cevv3ErioJpMh5N86OjMWRRZ7t3Pr344um9ymatpn/96v7h8cUXX3x2/fr1Ip/NJkOHcL/d3+61W5H/4PCpz9m3v/mtvKj++q9/enh0Uo2mo9HIOQgxAg6CF8sBrP0S3xdsAIVNlABWHvY6Q7/xl68YmCLCnBRSSmC076PXX7vebTUuz09/85tf/fH3/6g/6CGEPr93//Dw8OzsLElsLTTknAOgJtM583eRGcEruG09rTWM+H2H+L0H34RQbhXheJXMuP6KzR+eA4XVD25DGPsloOBeHGCFftZPBHyxCcjmzVp/5OWIgjVooxRl/dWgLtZwTilFEKhlpJ1zDmLjLDQOquc5KQCAKyqw0oRYR9ettQ6uZdE3LtRKmOv3rowXRlmWywTZqjtMTYfUxiFICPcoJtYJACzGCEFIKYQAUsoxocoaaaS1oBTCowRjDAACDgFQAykIAKh7L9VzRgBZA5yFCCGpdVFVmFLjXCXlOlIklaKUMkoJpdYZiBBmhPlcZIUDum6AZWylTamNcYBDpNvdgBJPSSeFiSIPADSbTUejLE2TvJkuFlmNsWrz22634SqAWhtP4BBCyFiplbZON5phnXELw+D69euMsYuLi6Ojo3SRGw1arRbn3ONhkaSGWAO1dVBrYwAEAGDMqqKwFgaBX5a51jLivjH64uS007nheQxAJJQhBPlBEEYeZW40ukAYaidni/F4Nk7zQiiQnp7PkrTb6uzvXu3ubu8tiidPPr739Nn7H37W7u034qay7OnhxSxZzLMUIGitPX3ymXXO87xBty+Uqnk5WhvyvT/8rjHOKJdn8iwfWwiiKMIoaLY6UdgghFWlUdJVlZRSEsIgssbYJF2kRen7oQWIcx63mnlVamPqQtL5IuGMxM0GZSzLC8YY870ojguplLHc9y9Hw7KS0+kn4/EkyzJgIcaYYYYxLqap53nNZqyEPnn6LC8zz2ftdtPZViNq9ru8EuLwydPRaGQt8DyvKgooLQkxhggZqKVxGFDMLi6GZZkLWVpgCOPUaq2VgahINSGQMEoIgNAqp5TSSoMo4ls721euXKEeHw0nw+ElACBuNoixhBBKGSc16wcSwhAhXhAaBy1wyhojBGKAUMowZoQ67IwxTjtjgHG2psxQVPMHnXPOVFqW0mo3mUx9pBBCd995u9loN8JGq9keyyFcZZEBAIxgzjnnnDDKuVdoI6WUsrLW1gJKRjspZbPZLopKCHGRXwAAHj161Gq1JpPJdDoFANy5c+fWrVtCiKOjo3a7/YMf/OCXv/xlvY7zPK9DBbVRaDSatYVaLBZpmq7F+6IoqvVGloGBVT89sNGuaW3CrLU1OdlauyRjrwyfUgoiA1Y9oqpK7uzsXJxP0qxizKvp2cyjlHAAcgghobQmb66TDs45Y5as7H/ocC+mHn7/O1/6SP3/PHV/+R/+8oMPO4N++8c//NG11268/9uPo7h5cjw62N/+6OMv5vNqMp+kxfzKlYNWt3X9zZvd3a29Yh40ImGl0KgQxa9/+7dbg97bd25LJT79/DPgwMHediMOj549pQTFjXBnZ+vp0+OTk6PpdM4ZbcbhZJYAh+vW4RA+7za0Bgpo1b+gnvZL+GA97EpK6EV31qGvcI4BpohgAKA1gFN8ZX//xz/80Z23b//lv/s3f/k//+Q73xYlxh73P/3009lsnuYAQkBpTfrRzhlKOGPUWPBVHIXfc4/+UcZ6+a1Xpt2oeliTGes3Qwi1ex5lQbBuYeVq5SELHNhQuwIWQozwV4CYzZW2eSNEJTZRy0txnTWGeD6H1Vh9xNbOMufcmeWJaGfWAXBtn3/jZq5kzWWpET9aCUNBvAoBQmRXYlC1YsRmO6i/c6xYEdQY4yCs3Y8sy05OThfpjGAdBkBrBZyxBBFMOSPaLJ0xShmihDKOEKgjmhAuKdWY1PAXc76kPdV3rWZzI4RKJaTUGNMwZFLKdQu6+vQxQQ481+1mjCFonTPGCq1UWabaCO4DTEyj6fk88LxoPkuTeR7FnlZAqgo6U+vQJEnSbDa73X4QepTwPE8hxIwxjIlzzhoAocUYC5kXRYEQihs+BGg2W2CMB4PBYrHweNBstAn2KKWEeGUpsyzzOLbaVLqUymptEaaEM0KI1hYCzBhPkpmoqiBkUlRnJ6ff+O67HqdCllIbokFMvDCk3IPHx4eNpu97cSlzwnAv6mitLobpeJYZxxCeYhg12tu7BxfZrPzlrz9Sir/91t3jk/Hjx+dJuoCUYUYXSVbmldJaCGUhWKSpdIZ53AJArvV/eO/evUVS9Ptb4b6tZl+U06OQgVarm2dKSmEtMAZLiw1k1kKoGYSQYk9rW6XG8ygnTCa5UQopBZmoVMmg8zG4fnV7NjmfZ6Pt7V1UGVHlyEFqNapEYMHhZ5+XhagqhSEhBDsFsmIhhOm1A46BrnKrYDOO7rzx2sHBXrvZfP3GzdlshgGeTha/nP5qkVwa46wrFXMIEucWxhhCsB/7BtuL7IL6lHPuo1BrPZlM8yJ1DkVRI9MFx5xaqgsrpSQkaDab/Wb89ttvQ2SEqMbDWZGrIGgAYLVUiBNrnbEyl5m1mhCCSUS9BmFeVdnxcJ4VZa/fbXd96CywFnmtKAilKocX52kyIxj4CGCfckoDpkhop9OpsdYxl8jU79Cm4cls9G/+/F+/++67+7s75+fnk9F4b29vOBwfHBzMFwnzWXerI43WWknnKIqCVnB1/+pkPi8LASEcDsdlKcajw0WWCyH2964AgJ4ePorjRrfbvXrj+ptvvNHv9w8PDz/44AOl1Lt33/v6179+/fr1wPdHw6GzljGmpMEIKaUIwsYYo7QEAgCwbqs6LkZ1VFNpWauR1ECBMbZp8mq2Y40bIIRhGDrnhBA1caGqKg9XCKG8KsbDiee3PK8DUOAQ1UAoUzldIgqM1UVWckSAAco5ABDBfNXlziHoMFq7Vs/NKwD6S12g9Ta/BivrF9e749rSwxc/tfQLwXPRx6oEv/7bJ4F/8vqNa4x8fvXK/te//cev3Zr+u3/3lyeTR/1r/VFxnqmqv39FIO/x8XQhHiTJrBL5+dn4xvUr0e3XCO/8D//7/0M2n956/c3PP3nUiVplXl3Z2oqhe/jBBxVkP/qTP/vFb373yWf3j8Yz5oXDScJ7OzBqAefKqkLa+B5HxjrlAs4yUEIIEQaEIAidNW7FRSWrXQc6V8996USubp/dcEzXZXub1DwIAABCYt8DWhFqGjFvxHj3oPXaa/0/+oP3fv2Lnzy69xkjvpJ29HQehEELuznQzgLrbD0BY5WxygHkgHHuedweQrrcCOEL++LmZrl5I9avrG/Zq297KXIAlnjoBbY/2MgLeN4SmNbR45qZyyGFaLk2Ng9Yl/gaY4wF1gKEEIOUQgowW5MwIES15LE1FiFitFFKrkiLCDhnjDWw3lcstMsYwypI76xRAIDl9VDaAQAhNAgoa2pXCiAIHJJK19seqq/HCrhbgKWGhNbdy5y1DkKLMXQAYFB3q4QYYeuQ0q4u4wAAcLqMVSyPuYKb1tq18Ha996NlieyqY/iLdwesajoopZWSNR17Op2eTx8h6ALspYWr8pxRjBHVwBkpfU59HyOCAKRCCGxRHMalLWGtAAaBdtpKAzFEDCllMMbG2UqKeqoGGKOMqXzsgmRStNvtXmtvPp8LUXLOCaTAYFk4ABx0xCMeAVwWBiEAICOEFeWCh41sctzfOej0qTb5oB/t7LQePVwcHj9UZh6Hg0IsfK9dVZWR1c7BXrMZa62CBq/KkhGYpwtMWr7fAg6HXhyGUVkI7lyDxIRyY1xeFnHUQZj/8lfvR1F0//7DwWAQxc08z01eFkURBYHCkXMOAIsZYgGCEBqjpKoQNAiDskijMETQFXmOMQMOgww2o06lCj3XRsOFyhZQtTvRoLlTyXK+yHXFqelUC+2gv93sjqa5lSxN4f1yaDTs3HgzluDhxXT4/kcfnYzyRTWbLDrtduRHo8thMS3mecE5J5gk83mSZdeu3zg4OMjykjx9+rgoMsYJY4RS1Om0hvFoPJ4nyVRrBwBg1CMEQ0jKUimloEYYEwwRQsAZV6fSIXJVUShR5nnqe2Q2nw5uve574c9/8dNG2MAAckIhxFJojzLPC6q8Otjde/L4UFU2DAFyIMvyIAhev7E3mw4RApTS/qD35u2bX/vauzevv9ZoRqPRqNPpBUFQZmWj0bp9+82HDx9+9MHHYeRzzhCoO7gYIYQEQAo1GDRajUYcxx4PCCFlURwfHx89O4nCRlmWUkrfD65evbq/v7+1tRWG4SeffJQX6WQyXKRzjF2z1YjjkFIqhXIOA4AcMNroSiipAC8BgFJpiAAc9HpRGEohIurv7u3klZ9OFufnp8PRmawqxhGnGADDIk/lUAoUkAbzvTwv0iIFjgECLs6Hvu8HfrSztZsk6RTPW53et7/1nfkimaVpr7/dbLbH0ymAjjHPamyMyfN8MZuXZen7ISfUb/LTLPcIVgIm82lVybIQO1vBztbW//p/9d9DCI+Ojh4+vH///n2lVJHlTx4/un79ep4XdViCMYax8X0fOJTn+fI5NAYhVON6sGIvghc9pE1PaB0CqW1rrQi59thqc0kImY/GrVbD9/1WqzWdTquqStO0DofWBzQrzek60ogYf3nnf3G85I39Ew23iiQ3WrHVylg7X6SPHz8t8vxgb/drX/vG2enlr3/92zwvIaIOoTTPIChLqdLHc8YJQuD8bHR2fPTs6ZOq+MPI/9ZgsJdl8uj4bDLNuu12s9G5HE1/+rNfSoi3d3dns9kvfvkzbeC112602t3Do2Pf9wmhPmOUEeisMsBanRUK+WR9eRFCdfyTELKONIAN0PP3jKZsDhaGSimAsbVaCn3//pO//Mu/fOfNW/1+/7333nn08InHwsuL8cGVrSyvLiYJ8Og/9MK+NJ/N+M3vvx1/n7EOJAAA6uK9WgF9M/wOVr3QnHPrTkh2BRDredRbqwWg3vYhhK7+92U8D7ihVLGe8MtpoNUPsFbHhkuE8NIJLj+xyWAw6wzgczzkVr1I7IvICWy41+hFzaXNK4NWneKX8RLwD4jY1V/kVrwKuGwjx51zQRBcPbgmpUTAaimAsZQgSjgCoNKGEMKYRxhDiDgLrbVlWZWirKs6OeGILkNBiEAhhHVGW2sBBMs6MAwRRAiEob/2TKIo8DzmnDu/OPN9P45j3+eUerWGN8bYiCIIAmuNFYoxfrB7cHX/KvfsaFJwz0eYWgcRJdLqSTJbFFlrsG2tJQQRgmqqRy0B53kedAhBkue5s0hVSimFMQmCYDabIWyZx+veqQBhtVjcv3+/quR8PqeYOOcCL4jjOPC8ySJ1zkHkIMYQ1Zx+QjGE1ghRqVIBaDHAPvMJIQyzIPQZIwDa4WQsTxeNpu9xNBrDwfbAOYCgxyh2zsPEAIQRJXkRSCnTuQFIWIMC3+csiBv9NC3GdCHzqhQyXeRlVgohWt3OJBnPk5xX2vPCa1e3rly56nF/PJmTR4/vY0zDIJonYwgoJi6K/TzPtVYOWAgwJrauxDEWIow0QhgjiglCSFmplJAIIAyKoizSVMi83+toZW7cuBFFjWeHx81m21pLKYcAi0oFQRBFjXSRn54ettttjMlkkvoevHJlvyiKDz982OuAK1eu3L179+7dOzduXu80G0qp0XBCCAuDgBCilX3zzpt379798MMPy7L4+OETzwtq4qjn+UEQesw3xjQanf29fd/3x+PxfD4nCIdBs90ujdONVjOKona7W4sLnV2cJ0lyfPysKLK8SDFGvV6n0WxxzowxGCMEAYTAAeqAVUqWogROi8pi5BPKOPfCwEOIAWCqdF4MZ+cnJ8+ePcvzNIiCsBlzSAEiOlcOAuoY9+Nms52QFDsehqFIxkJajO1ske/s7EFAjYZ5Jra298pKD/q779x59+DqlVIKhHGl5H/6939T1+QUeaak5pQh6AjG21t9KTQZjxHCzW7rrVv9b33rW9euXSPYVVWZpYkUJWeIUWatTpLZ734345wvFolaUTWttYQ8V4AGS3XFZX/qTeL62plbR01fBQrj8VhK2Ww2u92uXckwG2PC0K8RQN3ptdVq3bt3L0mSrKiWFmdV9AX+rt1ic/xToIRNq72eSVVVZakQAEqez8aTRjP+4z/8gzt33/3a17/90cf3ZvOk3W6WwgohLEDWWj8ICUTOGQhEXtknT89E9Teff/bgxz/8/sH+LoRBGPWSTByfjRlj7fb2Qmfj8fj1Wzfv3HlrOJ4cHT7e2zu4drB1enquBYBeqBRQlSCE+NyzVjuM63hsHcjBiNZAQSmzmvbLbMFXr9jvuc51dIkQoitBCMkX+oPfffrk8WGv1fzud797cT7MFnl/0P1X/4v/fjyZ/et//ecT/Y9zI37PlL4KGroXcxabb1uv1XrDWHPl1nmE+oc1JfDVyaz3XbhihNT+95dGsdaR8zUsqINtxhiAV1Lfm1wH6OCLca+N0Ih1ztmNr0CrjEB9+CWOX83QrlqxrxOC9QHrmPz6sOuxPn0I0XqqEGzwgVZlNkuQgVYdx1fDvXgo5xzCS00qjPE777xnpCIEW6Pmk3G+SMo8WyzmRSYUtUZDz/McwhgTVhMVCWKMUM6cc6CCZVkKJZxwGGNlrDYaQAAAchA4awGCnDLGmLVWiLIuDVVSZnlKECYIo7q7iHUOWAOlsxqUFQtCoZWqBAhorzNohK1Sza1Bk1l6dHZ5/96TSZaGAbPa4pAqJQCwnheEYYgxri8nJayqJCXMGltVFaN+fYUHg4EqkvF4LLKMKCm1arRamNK6XiwIgjRNA8/nnCdJEgQBxThuhGvLidaC385SyqWUdU6QUQYYs9YqZT6/99ndO29v7Wxfv3792dFDhDAmtBJiNs0gwdbBSpisFEJqBzFCGIGOKBZGO4iJ0U4LE0cYI26UFCWwFmHKtQNFlnmMbe/uSK1OT09ns9l8nrJ5mqWlg2g0HpPxeMiYl+f5cDisKp1nQlQ6CHlVSYSAtcbY0khgjCEUcI9qyOqMmZJGS6iU0hjX56m1thaUhWi1W7dvv5VlRRBEnhcuFgtGuHMOAdSImkEQUExCPxgOx2VRtZsBhHh4eQ4Auna18y//xY8ODg5u3LjR7XYBAEVR1CV5EEJtTF4URVY2Gg0/4lEzaHUb5NAZW4pKFYXEiC4WC049re3Rs7MqE+12ezgcjsdjRqnneRCi1994HSFkjMuy7PzhZe1VY4y1BdoBgIgX+K1ur9XpWavTNA25V2vhEUQR4QAKFvQa9gABAABJREFUmYuylEoCn9kiT0VeUGC7vcZscv7p+yd4pheLRZYKgEHouY7XYBQKLbV0AQHOoSxLZJFbaz2CIophENy4cRMAcHJ82my0HEReEI6ns+F4NJnPuO9NpgvELrv9XqPdnh8dNdsNCGEzbg0GAwAAxng8mkgpO82Gcy70eRAE24Ot7e3dvb09o+W9Tz8OgqAZ8rtv3cbOPDs+kSKnBJVl6fvbzWaslKnzBbVFqPfvdSZ7vYifG5ENjv36nZtAoTbBzWZzNpvV0vq1bmPN0I7ieDQaeZ5XMyiVUrPZLE3z56zClVJ8bezUVyOAL90wvgox/P0xx6sHhxtRa+sgIcgY6wDKSyXl7OT4/OnjEylMVSqtgQXIWCuNJphBDIy2hSgRcEEQNZttY9Q8yUfDew/uP/32d77ZiuO7X/tuksy6/f3t7e1vfucPf/vhL3/5tz/XVr11+3X6hAwvzpXMoIP9bsMYgzFVSgmrnYPKyKIoAhJvIra6+PHvPLWXUMLv+YApS8RI3YbAOdhue0ZX/+bf/Ls/+cEfYUQJIZeX2Q9/+Pabb72+SPN/+xf/BmR/Xy7CS9d5c/yDUMJLmPJV9FDvynCzOfSqhfSaEv98V6yPAwGAANYuQh1shwAgCFDd9BEBBCFGAEGrzMvf9eL86xeXZAKta5dmfZpLwAGfQ7rl87W+Du7lC7LspmUt3AAK62OZVRPtNZcIPS+tfKEUdgVElgmdlzDEc4jwoibHV92XGmPZJW3T1pHmqqoefvHYGNVqNluNkBEOwpgg5IyNo6iWbMnz3FoACfa4bwiRWkAMiMOEEC/0MMO+8S1wSZJoreWKoemcM1YBAKjVzkCMEMXYAKC1EmUhirLTaSGEgLGizHUtcYs8SjAAhgIjjY4op4AijWeTbJqO5tlCGJNki1xIxMi8WADH+vv99CKhlDJOwjAEABDCKDU+C8bjcbvZcQ465yjDCAEIXRQFfivM83w6S/KqtMD1+/242cqLwvd95+Djx4/jMGo2m6PLUW0VQ68B8DoytIxCWeMwxoxwyFGthaqUEqUQsjo5Obl+/eqNm9cRcmWZT6aXqHLGQiERdcw6ICtXlVYZhhCxiDjpy1xCjD3iOWCK0hqjCHGERn4QA2cUqxBwxipIEGI4CLx2u+mcW2TZfL4YDqeUMc45MWYZt9TKTiaTy4sJADAKG1pb5yDCCMA6M2coxZ5PAQ201loqB4yxWmlJEDLGBEEgy9JHXJRFu30VQfz+b38Xx02ljVaGYOqcpZSHYegsrPd+Y5RUDoDSOed53ne+870f//jHN69teZ5Xy+/kea6NC4IgjuOyLAmhACCMMYDuyeHjjz758Pj06OBqT2uzmKdKSyVVJXJZCSltllbIwWazOZ/PZ7PZoN+/c+ftr33tax98+jspZZIkl5eX6SKPovjGjRsHBwfHx8eXlxcnJydCloskXTUXsBJYqYUDivncCzzucaGKqirbrabPg3yRLqajR/e+OOcYgApCsx861m1IbQpZAeYCnGXFbDZfxO1Ot7PfbHWOT88uLi4Y5ZSQ8WTSbu/xLa61nozGJycniJL9/X0p5YPHD8qy5L63yLP54+Tw9DiO4y/u3dvb2UcIRa24gZoIIWBAWVUwB37gxUHUaEa+7zei2Lrqi3sfPXjwIGrE7XZ7e3e30463Bp3jo8PZfOGMNg4g6DzPo9RIqaVWCGNQZwc37PXafOCNLnmbTs/amqzM7HPGOOe8blFdizXleW6tDbY61rp2u11jiHv37qVpihCIm63akkqlN/08t/JfwFcAgi+FC/8o49UjQwgBIIxTo7QFlhKMMD4+Gf6H//gTCOHlcEo4y/IiKwtCGKAOAlRncBCG0liZllpLjLDnN6bz+aefPrp16/U7b7/Z6g6Qz0azBBD69W9+bTqf3Lh5PQzDJ08fXNnf6nYaZ2cXlxeXQRB43DcOMF53PFdZnvIosCsxK7jK8oAlNaG+NS8gg0039/eb/uXAGGOsqooxlqalpqDbiX/20w/ffetO4HNK6bVr/Vu3bhZF/ujxfSEKAP5hqQfwZcDlP+PWrPezzaXiVuwM8GLZwrrow666/61rQDbJDptTWm+oL+2sm1d+E7FtHmH9p03eIlxVqSCEEHRSyk0/3rolItfOwLqpzLq2aIUP1k3XlmjjxbqMGiuADTrn84OvyIwQQgTR5sfRRuHM+omuL8vzFwFYWwnnnFvpedTvsNZaB9YBlfF4nC3SS3IWhR7BkBPse5RTLKWC1kBrnNUQwYD7UTP0fX90dF6VKM8YRMs2HEIrY4zneVprJWU9Q21MnVxDWkgpPM+rmwDkee6cYYwYY6zT0NZVHogxEgSe53laZxBYBOBWfwAQLEt5/OwiLaeTxdjzw929Kzv71yazxSefPJhNk7jZ4Jz4fsA501pLqZWqa7I8Sn3GPABgvWyKIpOy4h7uRI0ag6ZpSjmre94GQSCE8LyAc66UCsOw80YHQjibTBaLRc0DJQTXNhYaiKBFGEAYICSrqioKIURZy9KcnF2+/8EnWltCkHUYQY9yrxkEl5cjypADuBTQGs/jIfcCQuhiaimxhFKf+86VSlRKWaWk73vc9yB0FmgKgQNcqmoyH7351mvGXFfGlaU4P7948vhwsci4FxDPC9qtTqfTM8YBR9JFlczT1Oa10K/neYzVGji6DtlJB+vrpbWydlkgp5SKgjCKIsbYWGlK+P37D3/3uw+3twfWAqsdAlhr4zHOCB+NRoskefr4iDHUavhxHF+/fv299957++07/X5fl5NsUUIICeOe5xnttFTT6XTJp3UOMyJldXx+dHp5AinY3mspZXwfYgJEaRFiVqGykEKIZDFvtZs/+vEPr169apS8uDz7yU//k0ZWSl1UBeO8vxW2Wu1mu4UJu/3m261Om3J+eXlhnVmkBYROSlm4ShlpofOEjCyglDrIKYFh0PSYF3tRxNjZs4dHJ4uvf+3av/pv//m3e5hxP6uqJycnXzx99uT0LJ1ejscOMej59Lvf++Z7Qj18+BAAkCTJr3/1q/FsUhQFBhAAkGSzRhS3220AwMXoLG5Gt26/8e3vfVtIeXx2mqQLY9Xx8BwAkMoSIeRR5nkeoLDZbbXb7UYULeaJlLKShXPOQeMF7OmT+yeUj4bnOzs70OlOK4LQcS84Pj6dx7G1AKxUDq0zSkirlhV0ayu2NoUvJ1k3xtrWrI3saDQCAARBgBDinLfb7TpDvNUKq6qqj1k/P4SQslTr+Hn9LXXUFCH0+/exvz9c+PtsP1965JdGVUnKiJIKAmAlYBQ8O744Ox0FgZcL3fbCshTOQcoYQBAhSACyFtQSUrUeHCEEQLy1e4A5I15QCPn5F18IUXa77T/eGjCf7O/v9vtdTpkUBUWmSKetmHfar8VxczJdHJ2eOQARAtrZuBXbF3PkdlU3jxBZnQV66Zb9gxAV4VRJCZxrNlvT0ag0wPeiyk+fPTt649brr712860337hy5cpsNvvrv/6P6h/eheOlWcEXI/C/5/3gK1DCl57mJv+/Lp+rQ/FuVVa6driXShXweVCh/tU4CyGEGGG0qjFB0D7v6/iC2CV4Zb3BVcHh8z+tQgU1otBaA2fWUAPaZaBCIwAhdPi5lELdOB4hBFfzh6tcwxqdvIRaXrqAbhUatNYiDDfH5tvgZlxh85jweaGNtbau19VaU7SsPK9RTq0B3+u2IbDAaAidEJWujFIEQ7OYzxljmCKKnUMYIu1MpYTZ6feNtha49T0qpVrKt0ttKu0QAhgtOU+Y4TpYYh10gBBIMOSMcUbyPMeYULosGeOcIwiMrhvmCOdMo9XMimqRJcIU2glr8GS8kEb3Bv39nV1RmXtfPJwn473uvscDSvFsNsuyQivr+yEAqNloUcrWCAwAoLWazaanT58BACqhjDEexkmSCLVUnIvjeG9v7/z0rCiKW3dvaa2HFxcQOAgcggAtFbiXGh6McIcdsLDQpSil1g5Dhigdj8eiejAeJdevHnS73cBvSllBiLQmAGIDgJQIIEpIRFkIHGxGCCEEQF3sYRBChGALjIPaAIOQIxxyxgBROi2FLTTMKi2SeVpVWulSqnw0PncWEgRpFLUacacsRRy3drb3MJrO53PggFBSK+Ms8AOPc58QRAgRUisllJIQYkqpMcv1pJTy/cDzOAR9QtgXX9wHANS1p/WjaowLAh9COBwOx6NpsxkihF5//fUf/OAHt2+/VZfbSSlasZ9mRS0PggHGmBJCEKFFUThnpNE+4BZaoSoLTKMVUi4xQUriWDCMlZYgL8u8KPr9zt2333vnnXcghL/73a8Pnz6OovD69esaQQiFEAIAVU87z0oI5s1mkxLe6XSMMXmeGaO0NkprKSRAEBJXlVqbDFFCMEfEMxZKqQOMW1GsO+2Y67u3X/+T7//xbnpkHVQQNxotIe2Tw9N84RAEZSEfPzp6481Zu9sjLGw2m424+9GHn09m49lsFgTBa9euBb539fp+GIbT6XSRz+JGA2DLPLq1v9Xb7Y3G41LkF/NUa62gyReJ1rrTajXjxv7+vkeZ1SaXRVmWoedHUdAJWoiBZHzpEKzKdHgBMGXtZhNCmBWV5zFCsagkgIB7AaYMIVQUhXFLWtzaEamtBlp1o16TFeonBG2U1a3NLsa41WrVxnc2mymlOp3OtWvX2u22yWZVVU0mk729vSiK4jhWShWFqtHD2qZvujWbnuKX7hn/eVvg32e8ZPSdc5gQSjmwUGtNuAcBNFJY6GRaMOIBhEspGOeMk7woOOdJNpVSAoBCP4qiCDWWaeO8qISUSbp4cqQ++ewzP+BRMwqakdaZQ/Dxk0fddutPf/RDo+XTp08PDg6+/vVv+kH01z/5+b/9i/95kmRSVQ7i7Z296TQxq4Zea+NurYWrHMRaf2J9y17FeX/HMAYzZoxjlDil87zsdrZ++9vfXb929Y033tBaz+fT4+Ojk9ML+591+V8Fna+++Hs+tT61L18hL+IAQgjn3Pf9IAjWWbZ1OGFzMa83TrhKsdUO36sLfnN7Xkd3Ni/1GnNTSpV9fv2fo7qN1MPy9q3ZPxA55/SGsuSqszQELybFNk/8eaxiRcXAqyZPLy0G+1wrenN8JUehDtE4t2Y2QACWwQ+6jl7opZo753w6GqZpQjGiOCDIIeh8jikhreau73OMcV4WaZoKXcyTzDlX5MuyqajZ6HS6vu9ThAsHM6mRBlYoiDEhDFroEGIIE4wgxA6YeTLFGFOKOada63a7WUMNQggmCEJYZ0MA0JAh6BzmTKelcdgYjJlHgFrMs9Hocng5euPN269duQo1ePj4sdYSBYG1Nk0XWVZ6PCKYWeM8LxBlRQhhnBhjfN/jnCMMnHO+73t+2NsaBFFYVVVeVp1Op157u7u7abKYz+drHrcfBdbaNUusvqQQIEocIcT3Ahs7Sphd1fTyZuR5zBqwSOXWIFSmOHr2dD6fv/7GLYyp1ABjCRALghhAXCkdUOv5yBiXF6VUOaIEU2K1NtYm2YQg6PmUB0gZhz3c7rUabagnVZKdFbmMo85rN/e6vTYlPilyKYVOknQ8nqSL3POiRmwWSeoALMuiqqo8L1utRhzHwGPOaeuWVF6MIfGYcw5aiBAqy9Jvtiil29u7nsdOT44ODq4OhxdOO4wxcsBpwzCB1i1m8zRJDg4OfvzjH3/jG99ACDmjojiEEM5ms2mVIURCP6Dc08qUeW4BYoz5vg8QYkYGAdNOI4SybDGbT4RTjPoI0VY76HYDgkJR2EVSvvnmO6Ko3n//NyenR5eXl0apVrtRlFlzsOV5XtxoRHFTSg0BMs4qo49OTo1RaZpleZ7lGYSQEMS4r42klCIKlJFpXgCEwxCFnmctqISsZMmRbTWbBzutbrM9ubh4/9//W2NBe7AdD7ZkadOkrEoQNT0Ew9/85pPpQkVR4/jk5N13373x2mvKUObRuBE2m82Dgz1K8d1372itR+PLqiqDKHjy5HEli9tvv/XWnbcPruydnB55vUFdTZDm2Wg2KarSOLfjtitl8zQdzUYYojD0pdHJ+fnDR/et1XEYe55nnMYWt9oNRLAy4+UTaIyzjlrrnFXKpHke8ah+2ldd0cza0LhVqnXTeq5N1dqqrnOivV5vf3+fUjqdTqWUo9Ho+Pj45t6g0+lkWba9vV13tDLGxLFHuQ+WCgGgLnmoNZ0Aoa/uAV/qcf6jo4RXDw4hNNpobBAhHBMEsagqA2zIAmsNIlAKXUnp+b6xNk1TA2yjGUkplTTaqvliBiysWfeY0qdHzybJqNtqzpI5os2f//Lnn37+6X/1428MBn0A7CeffLKz1XdWi6rY2R5oKcJej3vEOk0oIpAwL7h27dp8/sl6vwEAoNVdsM/59i/km/+hl8s5hxkLPD6dTmKfRVFwfjmXMaHINpvNGzde+z//n/6Pt27dLIqs14uyLMtm//kX+SWP9u85vZd+fQlnrPmJ9Sv17uX7vu/7lNI1SlgDBV0v6VeqHqxzoEYSa0ywsSrqsbmzbvJ7ao+/no8Rcr03r+8RBJYQUqtmrs7Erue/3M9XEuloJa1Y10msT9yuu1G8mEdY/wm8CMLW81xu+Rthht9z6VcY022+tDyLDe3UdbVUp9vWRkhRFkVmtWAUNiIehoEf0DiOPY+XoirLhqkbXWKkKliWpVKGcx7HsQMwSZJc5dABZ6zTxgEMMagb7XiYGS0451LKNE0559vbgzAMhRAOmNWUVkJktVIyoY1WXJWGcGYh4L5HOUHUzE5G/Z1Wg0Zllc5G42bc2tvZLbL86f0zjLHHo9rucc4ppXW4vSwFIco5TxsZ+T73aJ1WYIwBiBvtFvP4/YcPSyF3dncRQuPxNIqibrd7cXHx+PHj2jWaT6brO7UCsh6lyCjFKQ2CIPR9a4DWOsuyLMuMlc1GLy8Wk/Gi3ZgB4NJFWeQyzwQhTmqbC4WJA6i0FiyyAjc86ByAVptSG0EJsE5IWTrs9KQgFG6xjnO8khVCoN1t7R+0ghDPk7ExmnEIAbPGBbxBktHkyNh+f0tUWpVSSZvMFgjQ8WQcRGG/38/zdDgZeRHvtdppmhjJGQkZghDCqhTOKkpIEARlWQIMsyqLWp6yBSAmrabEh4yg8XB4dJZACHn4+i9/85Ph+UWv3/iv//mPbly/3u2EECJRlLJInHMMGQtDzJiFMC2EdZp6lGGkrEAMz2azosp3vV1jDCau0Yovhudm0mSMYYxHl+etln733euEkKqozi4ejC9n4/FMKxDGTYo4wVQr7/Jw2G63Gw1ogat3KeFluecxxoqimCfzqiwYoYQQqVWa5tgWALIgaGaVaHe7SZYSHwNu5uVlwChChvhcCOOz8Iuji6uj/KePnp4NR1Gz19rae3Y5fjzXwo8zzYzBKNg5vMg515HfeXh4/vn9Z3me5Zf5W2/fKrL503uPm2321//x/LWbe1oe/uEf7P/4T/7sw08+hZCHTAFV5nMBhG3hzs5rO4vF4uL+xeJsQbs03m+qHJayyjLBceyAzQudZnI2m5USYe7H/Z3BYAAtTJIkW2QEsYPtK624GE2mnHAvjKyDwkqhpAVA6rrXKuXYgxBqa7Q1lNI0z8CK0rXWb8aUMEyWDKYVq5xRWovEQQAePnhQVVW9PPI8D4JgL0Df+Pa3vv+9bzkHt9qNfjPWoii1JcBZB+NmY3//oNVuG+PqdpSPnhyORqMkSWr/oM4rU0qh22hPUEdxl2yGr7RyXwovVu9fb6XPzejmm9ebgedR5xRwwDonDXDEIQAKkxtjCCCc8yAKlTJS6tBvQAPzTNQePyEEY2CttUAJZZVB7W7LGDOcJgB6k5nCOJjM9F/9u99+7zvf+q//9Mcdf7fMkuNnj0AB9ju7t27e/P/+T//2L/7f/5+zZ5ev3brxL/7Vf/fGm+8ki+zGrTf+/M//PJvllDAIIWEsTXNOqLUWQYwxlVIIocIwZISnaarhUuUGIWS0sxbWDQTgSpICAACgBasGlZAKq01W5thDEtikKgkni9K8c+f2aCHeCtt/8IM/ee/u20WxmC7m936aIR9SSqVSCCNtjVCKMCK1QBQBYJ1D0CHrLLIOWAgAME5t7GFrp/XvZrCu7svLf3npowjRGgEYY7IswzWvjRBjTJ32QgjVlClKqda6ptAbY+wqvlVH9SnGAACrdb3r4/XrPq9d1c1maWAVhFs79OucPTIAAWgdtAihdToPOKXUmjthrXVgWY2M9YYjD4FzwIDnxRr1QBA5uOpTaw1CiEDkgLPOIojqdIs0GkKIgcMAY4zrgANYxR7ASk+6frgQfoHTAGqJRAsghBp4q/vlVuEEA6DzPFaVOeeUkgDaimFCOGFYx4iNKzVoxr1+4/z8JMvnZUW1kelhQQjZ293ttkPuQL8XD7ZaSleW71BKlRBpmkqhjNamgjNT6CIDWjhnrLXKOQugNU4I1cQEGAgMwABS7DyGW00fIT/LFkmSlFXleV6SpAih69dvOOdK2Vmkaj6fK+OC0BPKxnEry7JOe0cVRillQHPmkMqHzjlXBdf6rzPMkkkyvZyFYTxotH0/nE6nURQJhhEkHqbGwYjGMW2WutQ4KYtsni58H0Ok02QqnTk/fzaZzpRSZRER7DDQ6XzUbbWBR+4/HhVFUadp6nK2fr8fBH63202SBGPUarWklElSYgKCkOu8DLC12F2Ohp/dX0SNEEf+oNfKrRJpAQGmmDtrgdTOOJMWoIOlNFVptSHWeXluEMb9nT6hCqGMkGr3QE5GH1VmXOQZoHY6d2Ecv3bTGjsS1bDT7+5dbTJqCUKoFmChlEIIZ7PpbDazFmxvb/th0O/3paxOL861cllaOUs5JUZJqTQAyDoDnauVBxtRpLRwzpR5DqGr993pdMopWcyTyWSilLEWBEHwve997/btt+7cuRMFoXOgqkqjFIQQQgwh8Dy+Vv2r/VrGGIXu7Oyi02n1twbPnj39m7/5mw8++CAr8k6nk5dCG9NsdbcGb+zs7HQ6ncPDw9Pjs7IUeZEaWyFMCTHAikoWeTHHkBljkiSp9ZVrC44IqR9pqVXdyt0YY9zSGxBCqCSxwJV54ZzL0yyZzW/ceK1YJPP5PKBdZ41zTkgJMVIYVwakk8kwl2fTJC00j1rMC7OqqrPUoiiN8FqNOPC8gDGV6FJUSbqAmPlhZAxBCLWb8ejyNJvP/sU/+zOl3TypbFUsprNb1/eHKZnOxsfHz/IiiaIgigLOSRT5UuJCZJhBn4cIgfl8XlUZhCYrcms1xtgYNZvNjg6PPC947frNJM2UUtZBr5Z/NwiAGqejSiotFcXE8zwMkdCmEDKKojqioJSCDhhjlkVHYCkDvI4lgI18gV01mawNMaX048++yMri7t27u7v73W633W4HcXRw5drjp4fOQoBRnudn5+dpmmNGW61WrQu59tjWJIkNj/lFXxm9EHt4dbwUAvn971y7jOsXX837brqVdlXOvv7U5mfXr8BVt4X11lIPhNA0Kf+f/69/+/HHH//pn/zwO994T4hSaznY3nly+OxnP/tZmqaDflwWmUfxjat7v/3d7wI/ZJTXXUW00LJcWGO8KK69WYQAtRgAyxiC0FqgVu2Gn6swQQhf2lxfOEH7AmACEENoHIKtVqcOmb7xxhuc82736qDXZ/ix1MYgBCE01hpjCMYIYaA1YAC45b1ZqVxZAF5wTf8pxrrWF7x4p15951e9vvnXtbO+5gTUymMrVaXna6bWBXmV07O5AW+QfupigecNOBxYFU+uFJDWT1Z9B+u8yeZ5rQ5u6ujFOpSCNoQpN+ewnu0GEllGJgC01lpgIUKrR9g5WIs8g6WyJMLL+AVCz+s1rAXGOKOdksYaYA3ICmEcXGR5pYqyquJmF1N+en7JMC3nqTEgyxpSZeeXuHkaKC3G6T1RVlmWYQgHg8FgMAAAhUEshS0vx5zwIG6Mp0khZLvT85nvlM6zEmHQanUIQUmysNa2O82qkkKoxSIzxsVx3O9v9Xq98/Pzw8PDuh9EEAR1eIAxz/cNXuB6r8EYe2HAOa2DmpAsoV59GevqLUppUeTOOeYRjLHRWhtprMIEZlXFPR6YAAIc8GB/f3+R51maB56fKjubzKfTaZokg94WAGgymcRx7JyTUpZl3SkpqJkZ4/G4vjtpmpZlCVeMrkZn11pLJGk0GtLK6WSOKIqbrSiKPB5gzGSl8qwUSHPqceYfnRwbC4DDFhAHDPdxp09296N2l9248W4UwsDH5yfPTo+Pnjw5VFmWx3x7r4dZcDGemtkC+1C5YjYdk0oUYRytZD6VUopQZA2glEqh00VOKfW9WEpZ5AZjjLCsRFFVFcOMYAYYthaAFQTe295VWoxGl57HQz8Idnma5rzvRVEjCIJGo3H9+o2D3b2dnR1KmRIyyzKtLUX4OTFkFbWrH8rV9uB6vZ5SajSa/OpXv7l37wEASEqdprkBJo7jvf2dfqdLCJlMJs+eHU9G435/q9kEYRgz6hHCVaWytChLIUudppXW2sFlfxTnnIWg5tyRWoIcAgAsQoBSbIVT1oCqYr63WCw6vW6z2eSczyYTK0RVVQjjbFGKwPc81t8ahFvbvtAiKwHl7a0tJm1aqMlkRHlAKKa1Xp4yVZbbqnLOYY9MF8loOgYoDCIYVFhKGQRBkae/+ttffOubX/e9uBWW83n2ZH7R6w6ara3J9GI2H2MCmq2IcYiJC0LmhzhJocuVdk6W8nJ8NplMLATX9q/MkvnZxfmgt3X16tUgCI4Oj46ODxvNNgbQWI2BA9BZbaxWxmgN9DoUVtNUa4NVB8rchrZPffWsMXXEzG0wDAAAxphanKQGXu55phacnZ21Ou1KqOFwSD2ezcuaZTNdLJIkEVJJKSulIYTz+ZwSvlb/rXdW8GKEGb7EYPgKWw9fJLv9/l3hq/YJsOqVAF7BGWu7vI42bzItwIsoob629cW0G/WlCCELcKPdenY2+h//b/+PR3/66J/9+AfXb791cnZ+fHz0+PApcGZva3c8mz7+4uNuFHz68Uc//c2Dh188CIIgbAWaamttrtTl5Xmv19NaQYitswAa65SzUqrcrAod69u1eXFeOuXlK8YuRXxcXcBfRx/Q8elJUSzKfHGwvz2+hO/cuU0p5xQYgMq8chAABCwA3OcIUUk0dLBufFirIkNgXV3j/k88lhrXK+rMeoN/4RxfpKG8dCngxnip5hBCqKRyG9ScNaKFq+zPS1hh/f8mXADAKaXApnoBWK0KYF/KI7gvQ8kbqNRtvrge4MuQsdvgKIAN9GBdTYZYHgMAYJxDS3KEXq5th2szAFbkD0IIxhQhQqkPIYUQe14kDOBhzD1kbKWyJC8qiAhnHoRUW5VXmuZSKi20qSwoyzyruJayKCpnrLMTCHCz0Wg1m51WN44bJ2cXizTzGSUYizxTZbXbGyRJihBoxC0H9Hg0KcsyiqJeb9BstoNg4pzjnGNEp5P5wwePlRaWcYKxVbYshbXaalNJYbVhjDGP12wDCKFSxhhX61wihKIoIoQJIepkkLXQ87jvexAiURkpq6oqIIRSG76M5Mk8z60xBCJRVrnRUmiKcRQEVhrOOfV4FEUimfs+RwhYa33fDwJPKTGblfv7+3XoK8uyosjiOA4CDwDrc5KmabfTfu3G9SSdn15eGOMY4aLSYej7LGLYUii0qNWpHPVibGs9U1tUmbJplpfzZPG1r33z7p19p8tiMbftpll0nhaP7n34+aLcw7zr+x6P2r52wghZzYUsCSW80WgghCpRVlURhN7elQPOvShsTGbzdJEbYxqNFsbYaFtVVVWMlCiBNYQzzjG3CABECPEZtRZ3uq2Ls9NkPiPdNvB5DeUwRMaY3d193/fffPNNI3WSJItFarUDzxuJrh5a6JxzEDrGSf3gaa2l1oQQBDFw+snjZ598fNjvhwTzQX/n7nvfrLW1hxdp3Y1ie3D9+sHbX3x+DzqfIAIcUhJK4YwBzqIw9JeN1BBc5903n9u1QTGrNGGv14uiRhTH9x48WMyT2WTKGPvRj38IjL44PRn02lNGtvq9dDG/9/TxsNQ5pInMisXMAISIBxDigU8po4RhCLVURiphlXQAWAfjKC1SoSoDQ4SptlYK3Yj9IIguzy9mkynuEYYhAsYjoEinsNflHm61I+tkkjhrTVYtFsU8iiKAndDVopR5nk7TaaEKQojQotlpdnCv22r3t/qcc4yJx3xtXFFURSU8z5NGA2eANRhCKURd0kMIKcuyKksIoe95eZ6DjS7Ptd1CEIIN+1tfwPWGalfJCLdUdKcAAOYxznlZlkIrRMmf/Omf/uQnPzk+Pbn/8NFikUkNHAS+zyilxtpKCCn0urf1JlNs00/atOz2q13UTUix6aL9nW/efHF9BcDG3g8AqCkdm/AFbhDdXx3uRXdzvcFoQBwEGHOLyt998sVoMu63G3/0ve9ki+mVq9cvzk8vzk873e5Wu3V5cvjFB79NpqrdiD3PE0VeFEWn04kGvdlsJqpCa40pgRA6YIyti9w0gjWkq+f55RMDm8lvAyBAwNVlqsgCiACCCJ4NR8PhxeXF+e2b19+6/brV7saVaz/4o/nZAn3w0YeVcBjWWXSklKGIagsAAMgBAC1cpooseKEa959mbCTsNyNemzv3+i6/upVu/gmuAMfmcdBGWRBc6ZKtY0WbrafWB9lcEm7TjwebTILl22qcXefd1ugEv9gde/OYnNNNPIQ2SiFe+sb1KS4vxYqyoLWGCFhroUGErFYmhMuHHikAoQPQAeIAss4haIHDa74kAKDuQwsBCvzYIViIykHcaoUOtFVVUUqvXLmSJLnvRUEYA2h1hcNm6Ee0MCPmGCaeg1iJXFmT5QsEDSbw6tWr7773x++//8H/769/QnnAPDYezTDjwHaVLIxRZeFRhhmFzUbc73U8zwsCb2drAACqIz7W2uvXrnFOue+tboQxxjgEKTacen4QNJtxHMeUMyGEtdZIJYSEEAZBFIYxhBABWCcICEIYY+dMWeaVyKGE2giEEKXYIQehk6IsimKxWPhhEHlemhdFVbIg6HW6PvcIIZTiqBkVqtxq9Osogu/7hJA8z/M8B9Ami1lVVUVRAADCyMcEUoaPnj2dL5L+1pYXcCk0dIhSbLT1vIBiprW2ylBMADVlViql0spqLRljQch87kutndYiXwzPT8XNXQIsEDDGDaLnEe5VCn326eTy4vNev9WMA4gbeTpCEOwf7JJWqwMAEkI0m00I4XQyu3J199q1a87CRZYfPTu+uBgaBzBGVVWk2YJShZHGCFECMXLaGgwshphxIspKS+EH3ptv3L569QBjSAixAGupkiSRUopKnp2ep0kyHk3b7XYYBFHYIIQ4a7VW9bKuYz7WWryWFrcQY5gkC4yxBQ5jOhg03nnnnW6/d/PmzariZX42Hc9m4ywM4ka/X+Xz8WWezCwjBGOotZJSAgMw5r4XUmoAtIQiWOcapAQA1skOraVxGtUNwTCz1iglIASLxeL09JwyNp+n165d2d7dOTg4+OPv/sHZ+cnh40effvppFAd37rwlVPXxZ5/eeufrvXny+PHTBw8fLxaZF9C9vZ2d7b0nTw6RQ1VVlUpqJTnndexrVGaVLlnAm+1W0IyZh5WDlAfd3na70yPUl8oNR6PhcNzu9pRS4zSjmDQaDaWEUqKUYrFYPHhwP4yjPM8zkVdVpY0OmiGPPGutRXbvYL/f7UmpK1lCAq9cOWg2W0eHx61Ww6skxETnGgBLCbHGVMqYVT/JWkLROSeEqFNom3pzepWvXXta9sVWQ8aY2rrVXkutqC+E9MNQW7u/v08529nZGU8nH3z8SVE4iIAXUISwUFJXsmaw5mnh3Msti17yz14wfL83TPCq3TcbipOvvnntY710hFf/3yTNwQ3/dR36dhvcsU3Lvj54/QplbJHMfZ/3B9tCi1/97sGNK51vf++7N95468r1aw/uffarX/1yZ3vw/T/+w/Pz088/O+pfe81nBCEkKMqzJJlPWp02dLougCAIIYIBcBACQogX+nVXcGuBtc6uVAa+6mI656BdoR8IjQMQOIiQA05ba4w5vxgX2eIbX3vXWnv71pu723sPLxfHx8+G46kXhFJpqWwlFeM+Wh7TQoeAMwBa6CwAwIAX7uw/+th0qetFWwsBbe7Wm/dx02d4CUBsooSN96D1p+yq10mdlFnzGddL4iUgsr7sq+/aUCUBK/KsXXb7XCOGl0APeGHXf1k6ab3e0IvFGq/+vDkZYIFzDmJkjFtiDoyXpcrQAlAXUNT4RzuHIbRlaeuwYv11WgPnHGPE84mxcjxJAIyasYcCH1pd5DMEYDOOue9PF0lZVe1eG3Ov0k7JEkBrgDBQUg+1O16r6SNopJxg0r/zzsFovP/Z5w+ApTev9+NGM0uLdosaAxgTjUa0t7u7tbW1u93J8xxYWRUZIQQAVGYlY952f2DAklqR53mNpJ3RyKEgiKIwaDc7rU4bY1gUhdUGAbRQeU2WWuaStCEE+T5XQljnsrRI09QZizEWooQQ9nsd7mFtoJA5IeTKwU6z2RyPx9YoAmPP9yHE2pAgCBjD88WUUtRqxTU/xvd9CCHGoNWKfd+fzcbj8ZhS2u12e712u93WWkuxgBACa0eXwzQvECZhEGrjoAMUQSFklqYQQqeNdabRCICks9ksyzOnAsqc0s6Uwkn7m59/HCJvp99rRw3s/HRsW/5BuLX7+YfvZ9NkeJy3O0EjxgiZ/lZjb2efQIimk1kURX/0R+9IKf/6r//TbD5sJeFwOAzCGCBhbCG0gRoaq8MIWW0lskqqsrKyKpQyzkLOWKsRN/rdMPAD32OMhKFPEIqiKK+MoVpJvUjS6XT65MlhLUBLKfeYDwBQSgEDatk+59xisaj7GjOPR1HEmV/b3GazmSRpslgc7F+9euXGrVu3pvPZ5cXkyZNhVUmEUFXqqkwWSXl8fDqbzHu9HqMMAKCscw56nMVx6HnefD5cm4/6wXbOYfJCQHKZhqyfOod83/e8YDAYbA22McYnJydffPJZMp3u7m7v7OxYp+I4LEV1dHZ6dHz4x9//s9lsdno+YV705sF1z/PLshwOh5RihEitWiSEQghZBCutCpFbpxvNOIgDQqmBoBBaOlwaGIate4enHg/OLs6rUu0d7OdKpWmqlLLaWAso5Qa4LMsuh8O8KjzP8wKfMeZHvKZrlVJ0om5R5WcXIk8LI00jajJMRqNxmmTOWM65ECLPFrKsnHNOK4ZZVVVZsiCE+L7PCZVSFkUWBMEmOXxpj8AL4gpri7O2yHX0bB2vE0JEvlf/EEThycnJ3/zkp7/67cdaA8qABVBrbYGuW8sqo21ZvOTWvxThf9XS1f0V/56bx6se5Kvv3PwIWEnhvvqGl972pdbcbQSN1+9/6boRDHq9TlXk5+cXlIBOJwibrY8+/vTKlStVnn3zu39w592748uLSgqM6dYWG14elaXmPu73+82mr6Te2e6F4fWHj56UUkAIa88HIGoBBA5DWLPBl+T91fc+n5JbBg9W17M+o1qnCDgAoHEOImABoAQDoMdTiSGZT+Y7/V4zjI7nVVlkWgFgrFEWAggBopiu8kcIAuugRc46CP5LpB5WA6yalwoh6piie+nMv5LuuryheDXgimJirYUrL7zO063br2/GKdcQZDOGZFe1KtbaVTnic2AB4DK6wDnfBBzrRNXm3Db/X7Ml1oihHnijd4N7JQe3vsXLa1K3egG4brRWZxMRsgghZ1cZqDWkAfU5OoStVtZaSSl1zjiAPR9fuTLY32+n88vx+FgUE4JcVVXZPN8a7GsJtKkWyTwXlXVdRAJEMODFzvaAoObZ6aEoh5awvav9rUFneH7+6af/8erV63/8RzetuUSY3X3nbSHU0fmEsYZzTkpJCGq3w0aDSDNVphqOF/e+eAAhbbc61qIwiBljUi7LLyfzmda6DjA7CILAs0oTwurHNsuyRZZJKZ1z9ZqprxUCrm50Z4yB0AlRClGGgef7vHY5SpFGMXdAGlv1m4O7d98MPP/nP784ffbo7t27b999J1lk9x8+IJx5vl8KLiX1Ay6EyIvMOk0I8Txvd3f32rVrYeQ/e/YMABDHsR9w63RZ5YHn6WazLMt0ngqtCOez2SzPcwvsjRs34tjzvJARJKvSAdvrtZvxtSdPnpyenjkLrHaitMo6pNx2u/fx+08/c4/eeP31/e29ZCah89LFpEE6rcirRHr2+DgJwe5+k7tIZnPSiDtJOi/LUimFEBAynx2Pzs4PhagaraYUqqoq6vkQ4DCmnU5vci6VKCtVOqMg8ygmkGBKse/xrUF/sVgIURKIHj+ccM739naanZ16uadpijF9/PgBtO7KlWv1s1SWQinFCfU8Tykzm83OR8+qqsIYtzptjDEEGCFkIfI8L/CjeZIiRE5Pzx8/fjqdzWaz2fWbN8uqxJhm+XyxyBEkmNB37t5WSiEApaosMITB0MdBgAjRhCDnjFICEbx6IAFCqx7qlICl5raFyBGKZGm01nU/j29+7RvOOVGWVVUF3Hvttdcwhk+ePjw+fnZ2cco42b929Wc//3VRFNbau3fe+eEPf4gx/uijj46OjvI8r9UeeRQgrrTWVaWEEJ6PEQoHg04YR9oBK3SSVe3KzjO1KBaZ/KLZ6aaLvNFqHV7O87zELnTOOQshwFLoNCuFEs7BspQWAB4EQRghSsqqKCsBAKiqUkrRbXW0Vc+Ojzzq9bsDWcrJaNJotCIvkFLKstJKQIillJxwigmgrobPmCJGKPZRVVa1ccQYY4oIWkKryiwJBGjFulob5RpYQAirqqKUIoS01oBCqZUfBlEUIYI/+OADQkCn28iyIivq+DTABHjc01pLqTzC19Ztve9uxjndS97wV2wYr1rG3w8UvmrPWAOFl/5a0yw2rTN4EcTUVv6lPNf6DesNoJhPtbUAgGYj4Jzn6ezzz548ePBkOBmrqvzhD/7oe9/5hgPw6bNTz2P/m//d//DXP/1JURTDybTIZ1bL27fe+Ff/y/9usLX7f/kf/6/T6bySBiOKGQeIKKWksMbVsfHneYf1JgFeQV0AAOgQgmiZMHDWWAcQtAAySoDVjYafqnmSJItG9Pr1a6Ionz19PEskBEAbKZQJ/FhpBR1a8SEsqBMQ4L8ESlif4DoMsBkMeOk9ry6G9Yv1fa/j/8t6xRW/BAJYR9o2UcL6+JuHXX9quRJWhZnW2lW9g1oHJCByS8aDW0YC1s9X/eua4buJEgAAdSP49Z/W+OalU3Mv5uBeWgNupQv5HNxAiFDd4G15WOAQJrBGkgAACB0hyFoNoEYYG6MdMJSBd999favfElX6+MEnx4f3inyBYeTdvBLF3fOL+XRRcOxAQAmFlIC44Tc5PLjSUbI4v1wUqnRoxoOcBbTZlhcXl5NpdfXK9T/503eUNBir2XS+d8C7nR6EcDgcLhYLAJHSWgioFYCwkjKTwrQacbPRjsIYYzyZps4BShnDBANIKU+SRCmVJInv86IoFosWhLAosjrshAGtO+i6uqSLYKVUXqRWaUI8hADGMAh83/eVUgghiKusmBuj280wDKjROXSoETFODEam1fAb7WCaDKfJHGHW3+moErTbrSzLIHSe50kpnTOexzqd1rVrVzCG4/G4qsr53CmlhsPhbDi11hZCAggb7ZZWMs2yUpV7eztxxBiHUch6/bazssyzdsePWJXNlRWY0ABTX8oOQjCMeKvtnxw9FmJhtCWMMo4YIw8fX/7Bu28Ntlra5OeXj4UaR03g0wpoR27duj2aDD/66INf/OIXlML5fAoJOjl92mw2k2yczFNKaXewpZWFEHo+arUaRZEVRUEJaTabgRdQwglhVVVMp+OPP/448oO33r7d6/Xa7ebe3t7J+bTMiywrZrNZFEVa64AHg8HA8zxCmNa6LEvInZR6PB4/evQoyYcAwEaj0e52VjrKWhqNMZ5Opx999NHHH396cnKitW53up7nPzn8dLFIe91Bt9Xt9na0NlLqdtdrNAZaqixbZDkEwIV+vVFJP/frjbzu3Fonzgmldd0H9706yielNM7WvgIAQEp5enR8cXExm82ePn0KAFBK7u/vvvv1d/f39wnDaZpI7bRVwBFGAyHL8Xj64YcfZln2+PFjqSrGmFASAGCcxYgSzrzYp5EfRYgQsjPoBz6tRKmVlcZoAB1mBkFAfUe8wuQc0MvLCeO+bzWl1PfDvKzyvBwOx17A+1tbcbMttQjjKG61lVLD8SjPc8/z4v72bDKFEA4GgyIroYW+74tC1E1KOp2Oto5xqt2yCNsYE8cxpbQsyyzLpJRxHMdxPBqNaisGNuSca5pPbYzqJP3aEtWwoMYH9Rs458YYKRdxHO/v7zfbrcFgcD681BpMJgtpAMGAM6KNVcpWVYUZ5YGHDVo7UnCVvwCrnhTgFaAg9Vf2Gvgq7/8fNNbCNeBFl3QtL7F5zM3daNOP/NIX6xFQ1PJDpZTVMpcVhLDR4karn//i0/29+N//h//4xRdfvHf3rbgRSqWuXnvtvXTcancODw8//PDjy9Fsd2+wtdXP82x3Z8daNF+kmHiUcoBwlhXWQmOXXY/dUgJuuVu8ioqW5wuW/Q7WkQfnnAWgEJXKhd8NCQHPnj1rBQGGpKqqJJkRBNrtQGlQlAUhBEJdliUmbMlOgAAsC/3AfwGOQr1NrntFruRyyd/9yS87Tj3cK5Eh8GIpwSYqXePm9b2ux6tAoQ7C1TkLhJdU6yIr6/nXXK51BqGexvrXTTgCVtiotm/1BzdTYC+h2DVQ2FwGCKE1lxwscfCyRqz+KghrRUFcf1Rraa3RWgFosAFSVgghY8XN67uEQGBgt/XOe+9czeZTj/G93Suj4fxXv/nk4dNjykmpDQRaqgIhe/vNqxC50ZOLQmS7+/jdr72+tRMtZmfTZPjWnSsXZ5fjyZP33v3mxcXot795PwgiGLSpJwghmAllFkgpz8eeH/BWuLO7lWXZ6cnIWJXnWVVKIRRGnjHG8zyMCaU04B4AqG674JxjLMcYOwTLstRaYwxrC+d5Xn0XCIJ19EKWFWPEOVcDXmutMQoAvLXVGg6HBNHdvYFR+smjeztbu7s7gx//yQ8ePXzy81/8ZP/qFQdUls2FKoI4qiojpW+tJgRhDBeLeZZlzpmtrT5jpN/vlmWutfR97nlssWCtZlNpa8AibjRev/1G1IiNtVHsMZ8BaA6fPaiKpNNhUewjhAjVHps34wo6EgQNQptloR0EURT85re/AFC9/vqVN9+67XMmbc65b0D+wz/8Wq/fCmMs1Dvj6eHZ5X3l5v2tiDw9/igMw26fT2enFlpAlNIKYjNbnHPOGj1irZhnjyiljVZDIbfT3up0+2enLpkXvS7lzMtS4SzAiE+n892dg063TXloXJGkJZskWioAkFKKc18L22kMEMIY8CKVoed8ToGx82SaF0mR5QCUGLO6N/RiljO82N2NKKVFMcO+FzfZ+flDxvOwWe7tbc2Tsed5Vzr7eZ4753YGca/XS9M8y7I4VJwVJMZhw2+UAGPKCc2yfDhMkixJ0pnvh0EQJvPcONpoNBFCUUystRgRzydClIt0ppSglCIUURqHIS1R+Ytf/8YYhRDc2toSsvzi0b2oHd65+072t3lRFBTR2WyGMPYZjhsdZeTvPvxICs0YazT6ZZUTwmrQjQnyfcI5RRhc293pdrvG2dFopIoKYISgJySUCvUHgyiMF0lhlJ1cDiFGuaj8ZgdgCxEhFBigjBEUh51Wy1oLAbYQ2NKqytgcuhJ7XgSEeevGG4SQNM0Hg4FSJl/kFsGg1Ujy7PrrN8/HQ6VUmWdx1CAQtPvx66+/QQipqqrf7zcajSwrqqr68MMPCaaPHz8OgoBSXhRFXWUEgKEU1+ellDLG1OZJyiqKAoyxUsr3uVIqz1NKabfbytLitZvXr1zZ/83772OMtQbcgxhDjEhWSAAQQRRBYkoLGDFIO+cQRozSmjjGMK5bgWw67mv+xFITvm5RvZFRrm3fuqCmRjDW2nUZ26aRrX9eu2J2o1Hv2g/bxAr1YcEKytSVxkslNc4hhGtPEaxcw5dIkc89RYkKrQFCDnEHLQSu0Aghn0Z6oZCYlqPs6fE4e+3a1V6n+/B4scd7bgF32/uzvaTT73zre+9VJv2f/sNfngzHpXaQc8o832NaVs4mnJe2ClYKTBYAB4CD0AFoKEF2Yzy/DsxZq6x2GNGGH2JMyrwoyzIKIgPU5SRnFP3isweg0QQffgghHJY5iMFpUsQNbnyw0AveDPIkN7CuhIDAOAgIcsQ5BwFA5MtTSF8FXL5qfBXmw6vumnUtXL1r1om5OjFsra37/dSZsvViqG99zbOpPRbnXL2duFWdsNbayJVT7hB0CFhbd09YVopY59xzlAAAEDJf3mizBAd1jKF+duo1YoyT0mhdGWMQpsYY4AxjLAxD3/cpQQAASimsCxABABCiVbILOOq0M85ACxFFxhlgrFl2HtdaO1Br5ZElVXxdGF8JUXfJwhivmStuVSIOV/0wOQYQIYIxQgABAJ2DgGGEIaFCYM7bEEKIWbO9rZTSNjs9ex84ky+Sdju8/cZbcEfOp0ML4GQ++ujDX7Rbnd2D7aKQl8NJh+++vnu96bksnfY9ufuN3atXm61WVsyfmSrfH0Rpenj1oHF6fnZx8TsDzM6BrMQ59YeXF59TwqHBFJdxMNjdIhABRPR4/Gxn3zu48pasyHQqyxzEjYazAmLmgPR8XFXFPMnDiPpBQ2tdlGVZlnEcB36cpVWWVo1Gw8OIc78oS4TwwcHB6ekxIMhoZAhyjHEAm70OdGAxm/ue53lePk4PenvO2EGzQwl5/PixFPlgMLh2fT8K/C8ePfjwd782DlrgqiJPZ7PeYOfw8F6r1fJ938Fqe79rTMsL+LPLw1IowujClIKjW7dueoHfvHKQHh+laf71/tf2r1y7c/ddAMC9+/e7vXZZpYdPH/32F798+80b2WxkSxw3fJ1PEhFw34NZEce4yMZVXmz3dx7d+2x2Oux1uslJXg3dp48/+eBXXyilOCOnj5/aon9wZRtBGOPGlcGNNL9AQpJf//rX7XbLGFNWudZSO00IDnzfOAqAMVIJLZ0zCBKrrRLq9OxiMNje2d3vdh2CfJGUQmpGeRyFDcKU1A6ASgoHkQM2LwtrQZ3jkVJjVGNbM5lNs2fpdDrt9jo+o5RRjKFxVq1SfYxaCGFVFaPJkHPunBvPxp1u++7du2mRFOWNra1ulqdlmRMeGWPyPIcWGqM8jxHS8HkQRTF0QFSqdiAYJgCAsow8nkNkpCo8yzwfY8SCkAOHMMbaGkKIz3kch+12ezafTCaTRqNRNxgF0CKEjFHGmLIsCUUnJyeU4jiOkyTJ89L3LWOs0+5nWZZlhda2TjgJIS4uLlqtFlySoUhNNPZ9v9lsNjtRt99JkmQ4vHjy7DCKGu12ezAY+F6gtUnTQkqppK17PCqlOlFPWmttVRYCOcJ5yLmPITHWtDsdY8wsWSCID/av1MHJ11+/1uv0fd+XUi4W2bPD43E5FpUihFDMFotFHWEriiKO4263u3/lCmNkPp+Px9PFYuH7flmWxri3335bCNFoNJ4+fTocXhLCIISNRiMt0npXBhuaLet8am1i1s4Qxng2Hfu+H0VRVVXPnj0zxjAGnHNCON9zlGKlDCEcQgwhJARbt2yXXNvB9S5SW7f11lIffJM4tt77a4tcRyDgxvj9u9Hm2+rvrSewuYnCFUdsnXHYJCXUr2xuEuv/f48HD4CtS9aXv0AAHXQQOAi01hA6W9mLiwstpH3NXrl6oK3hLNze3op7jS8eff7BBx+UUj07PMKswSh2lDLCEEJupV6cLOTmV2+e5uaLm5GS+g1ClpWoMCKcsjiKqqqCALbbrSDwHNDD4TBJku3tbUo8hlmFJLDQ6Zq7iHkYCiGAQwA4AC1wwEK7DCr8E5MZ10U39WaMEKqqqqqqNR7avDv2RQXD9YvrHX2N9upPQQiNlfWbtdbaaG2WgaVKVOs5bF5w9GUK0HZVpl/H4YwxQoi6wYFUxjmHoKv7CSmlAp/X2zlyS9z50pw3j1+f3DJqtSQTPNdqhBDWHIj69TruYq01FhhjMCV16MVa66wlBDHGjNEIvhyNqI+DEKpVTyKCGWNKqTRNv/j8yacff5Tlyf7u4P1ff9hscI8TVZVlWaa5DCI4z0olrXIIEBa0O702P9jbvyF3i+rY90tCc22dc8qClDJbqSwIeCnLZrOL6dyDDGLDqOIs0AhzbhGClSyVyqSwpZSUhFHMTECVMtZUzgE/CGoieSXFcDisqgJjKrRaLBZRGFLmQQiT2WQ+n1prIXTGiKpSns8XebZYTP3Ih9BV0rW7g1azQQjptTtFnj6VJYbQ86jPfCFThNBkermzs7Oz0x+Px5zTuNn49ne/df3Wzb/+m59++sXnQmov8D3PO78863Q67W5rPp93+r0f/ehHgR+dXZxPp/MvvvhiOkt8P+jFsU8JR+idN944wWA0GntBsH+we3CwlyyyVrfjBWGr2zk5OQqj9iyt0DDxPbTlCOPk4uzCOdhp98bjWZ6JTntwfH7x5NmhFwQ8DBCnJKK5KVNVYOi2tnpH05nk+GQxWaQjRKSxOSLy6kGfKCkWyazRanLOIXQMEmkUxjhfpBbUJfKMcowZlcLOVeYDymgWx4QSHwJKqW21gmajl2WFx32CbZ6nUiiEoZRVURS+F5WlyPO8qiQlXFkjqur09DTP87Isy6rodFph6AOrkySpqgohnKapVpZyRhg1xlSqqtcrYXiwvZU+SQhmUrkobkmlK5GHYWQdzxepkNDzPIwpANYYiQBGGHAPU8IoxoFhDREBeBwGFGPseYhxBBxWIltkebvdhRBTyiGEslLWWqdRlSufSyFLYzWE0Npl7XtRZoN4cHZ2cngI9/d3CaaUUgAQY14YxrXGg5SFs9DzuLUuy3JrrZJ1GtspJSF0vh8eHBxwKgCxWbUYTobTaVmnPzrtbhQ1lDSlVBgxjgnQ0jkJCIWOAAeAQx6Pu50BJR71OII8TUacBc5BVRmMMSKsDiYWuawC2Wp19vYOpNBlIZ4+ejocjoIg6LS60+mcENJqtc7OLheLRa/XazabWZadnZ2Nx2POeV0QDAAYj8e+79++fbssy6qqCCFKKYxhvWHbtb7sqkhynZ5Ye9i1Wbkc68GgubW1o7W9f/++UsrzaFkpsEy7opogCYA1xgAIuUdrRkJtxdau+drgvuSA1nZznSCok0foxaKJdWzgpUQA/IqSyDVcgKs2vuvxKl2x5jTBFZfiJfC0+c718Te/zgKHIHQQAGdqnph1dcMbKpUyxniUKSkv9Nj3Qy8IM1uRCU5ltzdodXrdjz/75PxyjBBTosQk8jnn3HfOSVlpB4Qym3OAL8arXzrrl65YnTmCEGACGCN5LggGi8V8OgPXru28/fY7lLK/+qu/Oj++rHINNbAaOA0csEIopRSsTw7aZSnF85TDPy1Q2Lwdz8MAxiil6mW5Xgb1PXopK7EOHtRJgTov8AK6si/EtNZcwpcCTqs922kt6yPXa2N9TACWDlXNtazVzaWUUhmEECWoxg3WWgRdEARaaww25RNA/YPbmLldNldbLTb4cqgGIVTndgEAiFBCSH0Rasxtra03fmstxNhaYK1lyGHgahy/uUjWALr+VF1eiDFezNWHHz4wWjiFDh+fNhvB9k4vmU0Xi7myvL11EEdNwhg8PRcOppWafnLZaXMLx8pcdjoyaggIMogLCwH2cJEVjMdFUXS6u5y2NLKlnikDnLRaOa2tcVYbUVYlpdTDMAxpFEBR2WRRkaJwDglppcoBVFpra2R/0Lt65Rpj7Oj0ZDyezudzpUSz2eh0WmUpiqKI41AIwQNGFYBIR4EvVDU6O/uzf/a/jeKgEcX9Tvfo8Nk8GZVZDokhXCVVyhFLysntzmsHV7bffz/FFJdlXhRFq9X6/ve/P9jZ/vVv3i+q8urVq8NsiCjgMe3wjjXq/v17rVY78KMyXci8zGdJkwegkpPjk7Is/Tt3yjI/OT9eLLJmu7Wzv5dkOfc95nn3Hzz47N4TwhvOEYQaUrlScG3powenV65c+8Y3/vhnP/vlw4ePXnuNX16OokbbISCAmqQzQ4BGGvjOQOP3ww8O712xVxxQZZVFIQXQEGQFFOTHP/5+LQKVpmklCGWsEmVt5YUQUlYAIIwYhdRqq4zBxJ6dD9kkazW77dYgjFoYeb4XZ3kltVFKOYgwJcboohRSVaLSUuqiKou8grACFpZVVc6mTpssyyazSRh4GEOpKimltTqK25PJiBDyxhtvUI9yQ9VCWWcgBmkxm8/n82Th+TxN84E/wIhT6jyPE4IJRHUljBBCa4UQooRgRCHESosy10VRKC1u37rBGJPazGZJkVeUcqUtJg5DDQlsNUOM6eHh0XgyI4QEQZSmiZRVXeaHMeOcQQgZJ57nRVGEMTbGIYQgQGVRGWM8ntRwoSxUVVUQIsZYq9VSSmutHTAQuqoqjPEwxt1uP8tPx+PR+eUF5ezq9a1er44lWEaDskhFpRjD1gJRWAsw554UhlLKGePU5zRoNnIpJTCo29kiiDoL4qhVVdXl6VBK2Wy27XZrNJycHF/UqH82m0OIfN+v1S/qmMH29uDs7KwOLeRFenZ+ejk8r/GBc5VSy+QRY4NerxcEQRgFtVb5fD73o9Cs2vXiVU96u2Lt1W7K2hc3xiAE4ri5vb1bFMXl5VAIE4aMECul4Zwr67TWtbSLtdBYRam3PjhYEddr/AE2dtzNnRtsVLLVamu1N+mcW2dqX/Kk4UZlI9gIGIAXgxBwpaPwErzY3FTAhurUep9YA4XNffrVXW0JL2AtdOfsMjPgrEMEYyulMZZR1Gh0KCazJP3kk89aPun2mshHUTd87eZNDWyS/er8ctLr7PmB7wUNSjwpdVEUlZDJIue8vZ7SevcCr9SCrqfHqCeldND5vo8xFEIUeZblC9+njTBywE4miZSVEGI4HH7yyafTYaqN8zjHgDJiEaEQIqk0pAQBu8QKy0vw6gX4xx91tUWdaQIbbSRriT2ykmStF+dLDNl6rIHCZv5ofbnqGoDNqEN9SetVt5lfeE5jXIlyrt9gV+UYZVnWQLwm0CmlHEAYY2vQmiFEyXJhQ/Cco7BeaQA6AJ4HDNZPk3MOouc31640TuxKxAxTVn+EUkooL4pikaZJkmqtPc/zgkAIBSHsNEJsTQ3vVlcA1ScCEamVAKTRs9nM9/2dnZ1Wd3ewfbWqiiAeFHmSFWo8LqrSIhxHcSPL9OXlUbvbEcoZZyxwAe0mk8miGAVRMRh4fmCEsBYCpYCzBiKgnDSOVcL4fnORlX7QVCoDjlpgMCVxI+x0W0HIaqeOMQNJ7pBlPG93IaMex1tpmtdFPNyD7Waj024ZZwfdniwrIcpWq/Xmm2/evHlTGX1ycnzvsw9cpoRM+1ud/av7fugPBoOw4Q92OoHnE0IqnU/ml0LkzCeeR+Ku70U4TXPtqkLmLnVFVVZC9frbp6fnAKPtnb07d+4cHj372c9+Nhpd3vne7du3b16/+tr9ew+On51MZ8Ph+cXl+WUrbvVbzZ1Ob29vDxgbhmGR5e/dutXpR0qXn997BKCZJ9Onz45n8yxJFqUQx6dD7sXaOaG4LCutKwTE3t7rpbRHx5PJtADIxySSZiKVbLQbi3RWVmllZdAOutvdvEgcMaVXTM3IWluJclzqRhQbpf7Tz98nP/7RDw4PD2ez2eXl5fHpiZIi8HzCaKfTmc/nw/G4LMtSa+tBQghwsCqtMSoMmIsJBDTwY85D4PD21n6e51laOOcAQFobrTWCJM0zY5wFzjgrSwEhlkpVVVVnuPM8z4tUiLIscwih77FS2aKsgsATWlVV4RA3RgEAPI8LJVud5tvB22EjPj8/536g7ezs/Hw8nmitMYBBEEEInYU+D6qqctSjFEkpp5P5dDwuigI49LV37u7u7hZF8cknn5W5DKPI84JmI8rysiyrTitudrrj8fDkJI2iRrfbnc4lxsj3vSAI6sp+7lGEUJZltYrteDwuy9I5QCkHQFWVxJgiSAAASmmEFAAIQmyMhtApaSByWltrXN1l3iGSFpXRbnd3n3MfOFgU1Xg0z/MyzwopdVVaKVWWZYxw0qKilNAhglxdvsioV1VVJYVSmlNunMPGYUhAFxJCeoN+4LNGHM9ms8PDwyKv1RVh3RaHUp5lhe/73W739ddvHB4ecc61VlmWSikRQtYti6Occ5RiKeWDB8t0Wp2UqXt/13awNl61IsXaLIJVvRZCqH4nRshZwCgfZeOqkloD4BBjjBAQRVEppLWWc0YpVbJWSVma1DWXu7bLay3Il7ZzKeX6lXpK9ai5C7UJBhvBhs1N5SXcsB7oxdLZtVFeT6b+arcx7EZTvk3TDH4vRHDOWWihcwAhC4AFDjpgAQLWKmUI5QhAgBAi1PeDsihGo0u33brzjbvbux1hLGHka19/l3n04YPDy9GcEuy0yUUuhKqEWjbj3QArm+MlzsTGPJG11gGgtdQaSFlxDwdB1Go3Lk7PytJyDpxz9+/fxxhDiK1xTd8njKdZYa0OgghTIoQAKxrj829cshr/yQfc0D+ooZ5SqqoqxhjYYK5s3qz1B90Gb3ENhdeLR2tt9XPd5c3IBNhIe60RRr0q1oKn6wfEWjubzYSoI6/V+q/Oubpls1JLsOKcwwhYa4MgQPAF1YflOkTOuee0jxVkBcYYhOGquvh5UKR2D+qnyW3IzSVJcnx6cnp6jhDa3d3d4hxjJ6UUghBnIXwesUN2yfSsNdmiKAIY1TpF7Xb7ymuv37n7zQ8+eP9iOGs3I6Or8TjjDENgSqPS9On5+enV69cIQc45LQsswkVyKu3ptRvMKF6kRVbmDgLGgNKABzSZa8qjNC9bje15JvOsyksNARRCl6KuOtHaCCFKiIx1gjkFAWo2EKNxFLZUGTHqKqk8n/R6nUF/BxM3HU7TRdput69evco4JZRUIg9Dv9dttnud7qA3HI0Orh80ms2Li4sbr7/W67U++eQj3/chsEVRPH74qMqLmzeue553Mfys3W5P5zME0Odf3FfSEsQZdkVRtXv9JEk+/PDDqNm489bbeZ7+6lcfjMcnTx4DaOV8Pq7KJJlAqwA0xkhRSY0AHAMgK2GttsZc29uKe82tQb/I8kGnyShyRh0fPfnVr3+nlD49OW80WhBCctPvtTtXDq60Wq0nD96/PDv55d++r5SBJDi5GFXSCmEiC5yDNQOGc04IqqWSbr4xMMZpBRhnRpFvf+fb3VZ/NPy/k16nGYe381JkWfbTn/70008/ZYxVRTUdT4WUwDiP+pBgCJAUxhjj+z7jrNHqYOpX0nZ5uHdwjbMwz8uzs7PhZJrnaSlLqSophe/70mhRKQQAIaSwlVs1Z/N8v94FrNUQuno1E4oAAHt7e1tb/UYrrlQFhI2iwAt4FEVCCC8M+ox5XiCVQYgwPp2MZ5RyIYQsKwixUgpC3IgaGGMCaV1YMR5PJ8NRPflufBz6EaW0EcWEIM8LAADWgwDSRTJRJovD7XYrCDzMKcDYbW33a03lMAwRQlEUhWGotZ5Op5zX5TQZpXRvb7fZbDrnRsOJtVbKCmMSBBEhRCtbliXjBGNIKUPYEYLC0AcALBaZdJCzqNtDxriqlLPZzBrSac+nk5mUGgGMKEQQcMp832vGIcWAYmitFsJgjD3P83ympBkOh3Ec5XkxHp5jSgeDfhQ1MMZpmhzsXWu3+mWhHj9+nKaZKEqtdZFXOS9q69loNN56663z8/M0TfbZVUII58xaW1UlAJBS6oAbj6eTyfTi4uzq1esHBwcA2DD0G43GxXCytkprq7rp2a+VD+phjMuyIs/Lp0+faa3dUl4UAuAopcaBsiwBsBA6TCCwsG6VWR+kzj6glUT02pqDjdCCWWk817/iVROgdru9yTCok8FKKfyieOIm7ACvbOrrPWNz/6jfvIlj1rgBvNJrYHMr2sQlbs1FhwBAsKTCAecgsNZiiJXUURRxSrVUWVoBRzCknh9X1ly7+fpbb984Pnownc8azaDTbh7s7z58+BQioSRaLKpSaQixcSCKGlkiNiewib1ePXG3lNXCNYFOyopx/PZbt7/2tXc9zzs5Ojw7O7u8vCyKYj6fAwAmk1nEMCO4rHJlFAQgTxPKfUqI1trVItAOAmAddMuIwj8xUlirAK1jWlmWTadTSmmdVq/j7etbs97ON+9arbaymUhaVygQ9DyBVXc9QAhZC+oUg3MOQocxRGiJa6XUaCUCvQk40jStBf9rNADXJZ21ksVK3bkObyxj++i5RCNc8Q+gMzVAreNw9UqsT4eyuvYbOWfqq1GzCtbttuv9XmudZsWTJ0+eHR+dnV3ULbm7/X4Y0mWeAkGM9Xr+wDiEULPZTLNCKUUI4YFft4EQQtx6681M5J8//Ozpwwe+f6MRhIWUoR+k87GRJUKWO9MgiHGS52k+PvfplThsRC1243qwO/Ah9f0Aep47vzwDDhAcQiABosroIA4ccou00MoyRq0BUuqsqKbJPFvMMHaYAOgDagmChFHGKMDYTLNxWZXj8cRouL93nXFstMYI1bpG169fB8idnBxNRudl7idJEsXxtWvX3gBud297kaYPHj28/+jhs5Pjsix7vd6g1+GMtdtt2G0NtrfKslwUImoh4gVAo0eHR8k0f/v2O34jnM6zVmdAMDs5OVGH4uvf+vq//Jf/zVtv31rIo6KorMpaPpsBI4vi6+9985//2X/z+af3fvWLX16eX2TzmTX6/PQYQugxKDAaz+Z5UXiMGaUHreb+dv8zj2qC77z9Zp6XxaKo8mKinRLK84KTs8M8LzXAvd7A2MXR0XEQBM6BZJpiArqNTrUoRFrms5RR6io7Pz9xFhjjCA58Ggyi9sHWFR/4JM/TMIyNMZ393cFWj96nO1vbSbJ4+vQpJZxzn3oMIWS0g8BAYBgPPc8LwoYxIFnk3Z6yDgqpPT9sd3rRaDKdzbIir5U6HATOuVpiGcJaL9liTACCaZpaa5WWwBpMIMJIKZUtsrhB4kYjbjbyPJvORs12A9OeQyavCmutL0UQxpUwjUan2WxaA6eTi6tXrzLGRpeXF+eX5+eXo9FoPjnz/VAJCRyEEJalqHLDGGE0rEo1Gc/jRrizs+P7PM/zyWzMuZdBi7CxphQiKasFJjaMqO+huN1ZLBb1FiWlrDvOlWW5Rt9RFA3627u7uwihPM9r3beqEhBiz2POQuUqAICzkHrc9z1KCSYwCDyj7dnpuXCi2WxS1sxms/k0zXPJma1KUVWSIMw5Z4x5lGjuUUoRgpQizimEsCxLYRxCgBBijdra7lNKk/k0zzPGWJ7yZDZN0/T61WuXl6MkST799PPDw8OqqjzKwjCcTscQgH6/75xrN5q3bt0KgiBN08ViLmWFMQQASekQcpwjrXWrHSlpfN9XSmxt9Q8ODrIs833fbpD71umGtZmrTc86Dl9bba11lhWffvq5FLqef5oXoI4PE1IUuZRSSmmtBQgCR+rD1qy6dRh/kyuw6VeBVXh5DVxqK59lWW33a/Y7xriqqnUkeXPAFSPhpSzDS55iffw1cPnS5MKX7jpf6tDDjfzFGjE4AKxzuCYuEGyt1RZARIwDVSXjIOy0e5kaj2bzQiri8Ytxcvjs0WQ0nk3TrX6PsjjNdJ4PF5OpcYAwtlnbCV5xnTfnuQY6zi1VBAhBSpcQuigKtrcHO7vbf/Kj71dV9Rd/8Rd/9Vf/YT6fB0FgNCDUSpERjAedhgNoNEuqMqPMBwBAhxxAABgAEHAbSg7/xMOueCr1KqotT6vVUkrVvSLXNwW+SFvZXF3rwEC9ZuofjDGY0bUvvk5kOOfq/b4+zpqECwCwVsKVGoe1tqYj5Hled7Jdp67WM8c1Z8I9r3isp1QUBUWQUlrHRSDAyy+ytiYqwBVz6DlvAbrVk/h8Ydc1zACAumiQEDKfz58dnTx9+nQ8neR5boyZTqetyYQxz/f99VV16yAZArU5queWZZmyxvcDY8xsNuvstN94+/V3vn43WUyUERB7jWYce2y3ez1Pphi5BgV7nbjRDCgdcM4nadPznbajy4sLIc+CuMAkI8wUFUTEU9pRFqZZGcWBhRpAFTcawELG/IxUQmlCCMGMEEIIwARgiAAAyAFjrRLCaRhFXd8jo+HFxeWQMc9akJfKWRTHzaqqqqpoNKNOuxlEQaMRzebBwyfHhNCrV/YhRl4YvXXn7nQ2DqOG0hYA4Ifx3vYOYyxPM4JZVSbXXnuj1ey8dfvr6bz69S9/U4ns/GysBGIkfvrkyA+474fP7j9175tvfeubN25c+91H965sD1px/LtHH58/e/bWG+81w2B8cTm+vDx+duScE1XZbjeZzzCGz549hdSrhAIYPfzsc2TB93/04/2tfq8ZPXz89Jvf+t7R0UnEebfVBADlWTobjTUDjiAFbFGWlLOt7d2qKI8Onwz6vUbkhdSfX848x7pec3d3xznjd3r9/tZ4NAOOBX4z0GF6mnk6IHWX68ePHw8G28l0hjF+4403sixPFlnd6respFKCEBoEge/7izJzFiBEnIVay9l0/oW4f3Z68Wd/9l81m83BYDAcXqhMEMLqejnoYA0UPC/QzjrrILTGGMS4UqosS6uVH/AgWEaGt7aDKIoQQkWZSy26pEMIqaQkhARBwDwPQpjnue+HRts8Ly8uLr/+9W/cvfueKMrRaHRxMfz/8/anvZJl23Uotvq12+gjTp9tZVZfdVvelhQlvScJkvXcCDZsGf5bAvTBMB5g2QYsSID1YFp6pEiKInlv3ctbfVVmZXPy9OdEH7tfvT+sOJGnsoqU9ERqf0hExonYsWPH3nPONeYYYz5/9uL87Gw+W2TLvK5rhGgU0ZC5KIq63W6vM8yW+WI+H4z6YbjllwW9fodQNCg77U5ijCzyBUSm1293u11Ew6apAADWaiGEz39FUfjSgdEgjlKfbp89e3Z8fNw0jbUOY8xYoLVW0jjnOOc+VDHGOGdByILAQx3TWZ7fucMoIYtFlpd1yIIoSqxdG0ARghCwyiohm7JQi7m7c+8WhABj5JyVjfBZsGkahGme57PZDBM2GPSCICjLMo5DQuhsOr+8vKzr2rsrFkVRlmWr1VrMp6vV6vz8HFr3zjvvjEajfr//4vRksVjgtVc8wBgQsu4mCFm3Wq35fI4QGo0Gs9kEwp4XjIEbi9QNI2HT8b3ZDqeUe5jn+PhYKYUxQAg1jUiSOI5jREhd10LMm8YAAAACcRRsEF0fnf28vpuuMuDreMZmxbb5L0JosVj4RaQHh3yXxFq7KSBeSZPg6xpIcANF2NQEN9/lORA3y6ZNleC+QZB035xldaNEsB5gcc4BByC0wCEHwzBumkZr24qTgHM/sZnz0PHWf/zTP/3o0w84URCKKECjwfDBg9fjsMeD9mLZfPjR5/KjL64m46YRECPfO99E+ZsF1ua73/wvpVSIxldpUhohzGeffQKRvXv3NifU+20cHBwQRK0Bnc6KNIVzYDDsvfX2e5CyX/36w6PzS865rrwKYD0aCgDkgFszHP8mN6UUcM4ag67nnPlk7OWI36zbNmvxm8+7a7OvDagArhtSQtZ2I6eEFq/tjZ1UL80fb25Sms094g+jKArvcL8pOMA1jGGMQcZLFc0G/PBIWF3XBq/HTGCMyfVg6HX/6vrqgteNBucctmhTtn7LL36tBy6K4uTkZLlcerzNWrtcLq+urjgP+/1+GrLNTbf+gtZCCJfL5XC0HYZhnueVaJwDnnE5mYm4FX/ne+9dnB6eHh6uMtMJuRLaBXE7Ce/e2tP3drdGPQytlI2x6mJSay1enDw6PjnujcDb78DBVoCw0tZFKSlL0em2suwy6fSVboyTEGIDnJAyz/PVqqG0SJLIOBBSiqADAFproQMQIAQJhEjKqtvtd7tpkVdxHBqjyiwnLDTGNNny6ipYrubT6bjVjnd3t2tR7ezsZGWBL68+//KL0dYWQujs/FII0el0qqq6uLjCAF5djvNV5hz0w8YRLH/0W7+jGzO5WDLccga0Wr3xeGqtvnPn1nA4PL9oaa0bUaHKddNocXXhGvng7p37t+4THD/76smXHz/Ksmq1Wr3z7tur1WpvZztOuFLiq6++SsM+4aydts7Oz8s8T8KAItjURSuJ3njtbjafSWEZAVprURbZqihovr+//86b75ydnB8dHadxCxqbRGlIGLKIOKTy5uBgC9x948GDu/PFDOLqjdtvfSVfaAV3tm9z0AIKf/eNH5I//8VnaZo+e3Z4+u/+uCyzre0hgPrW7a2Ts27TNE2DZrNyMplEYee1+7d2d3crIU9OzrSYW4u0UQh3i0J89Mkvd3aHjAVF1TgElXHaakqpg6TM8iTtaq0bISEigFihFMAkLyujHUIRiwjB1BoShb12685oqzNfFJPpklDU7Y7CoINRFDLKGIMQUkdFKetKOAltA1phanU46N2eXBVBEDDeffj6zusP3mOMPXv27MmTJxcXF8aYMi8uLy+11srpuZxijBvRzE+W43zGGDPOXS0KY8zOwR3n3LLMKccPHt4hBEpVLLJZt9Uti7pq6rTdtgaMJzOEcBK3EIQIaSHk8fGJc7YoCgdMFEW+7ddqxWEYXl5erlarNE2TNEDIlWUBYcIYy1YVhLDf74dW6JXQoA4Bjts9IWpTVxEjsioAYwQlgAXWWim0c4hz7hBslOylvelinpU5Y/zi4kIpxVl4dHTEOXcObg2Gu7e3P//8c611XjTL5Rw4NxhuV7Ws6hmAiHEOEO71h03TRHF6Pr46v7r88c9++vnnn/OraV1qzgilLGIsjlMpNdASAkJh0JSgnQ7LvPnH/+h/KIuiqopBp1OWZS0aCIAFQChpgSOExHFsrTXOIAcpJs4B2UghBHRie6uTJrTI54y6ugKMgHbMe/3e++++d3R0NCeMQJyEtKobYEFdlz6WeSt1CCHGkBB6HYXNmtB9ba/puaXWWk9W8JeNp7j7ssATudvttiedBEEwn8+FEFEUQbgualut1k0C2ibmboxrNtF2AzPcRCZuLiXt10nyNxevL0PtjaiNMQUAAOj8JAYEAYYWQ6OrIqCIUoyQcgCwgNLAKVjWFjhhy5not5Nhb1erOluy3a1hsRLAqJjSlAcBsMO0gyC9uBo7/nK8tQXXI38QtNcyEw+3GK2V1c4aJCHDCCMMHepEPedcndkP/uTzT3/9vCzzQa9FKdneGb3/3turxTJfTnAlZzPRCvH/+f/4v35xevHZpx9iAAhWEEoHEAQ3R0Y6BwAE6OZJuHk2NuUauFGrvVJy/Sc36ADGBEKopfLFa8CDgAfAOqO0n24kG8E5p4xbbTDGCDhntGzMdZNL+jrJWUOwxWtkYi2gkLJeVwbSNQ2+JkOs56RcV0VAr60bDYQvnRw9+uj9325iYP5bry9dbZ03xoJwA4p42ZG0xuMBG9gDISSNdeBrF60nMyKEpNbWz5THgBACrXEGQgv9EqLd7oRhOJlMnjx58vjx46IoeBgEQQAAqKpqPp/3eoNerwduIDS+HDcOWmu9jtq7m2ywVUqpXlJVVLujgyRsaaEtJbPFdPjaHQPV7t72/bcfRmFwdHS4qgpKw7K0YhUog2aXrdmcFdpmiu7cTl9764AFZrqcDQfprBZBMsxWYjF5giGqRcFoQChq9+KrSRG2AhQwI3kpUCuNldJSqk6LRymrijzPc87fthZoo3rDlIa212fzPFcmK5t6OVsauQhoFAUxa/DycHH37t2n06OLi4tfXfypQ3A5mZZNXTRFEAST8bjb7RZF8cXjryIeGOAuri53d3dJMgqC4GKWz66mjw9PjNB39g7yPB/0+k6rwydP9/Z2fvDu966uLkDl+v3u7t4/+Gz16fJ42WpxSGADxPZwOBpunV1crvL5F189Tlvd23GvQ5K6Ut1tNs9Od3qDUrko2atW9Pf/P79ECNxK9x58/+5v/+A73394ezybXlxd/Yf/+IuqWMRhqA3e6vTeeHA3ZnA2OWqqeRjg7/zw9s721qcffexoBCIoICgM+M1nz1erVWOaL1+stre3V6vluChHo0FRZgdvdcnnXzzp9XrLZQ4Ri5Ouc/TwxelXT57VtWmE0toBwCDkjTBX42Xd2NH21t7eAXD44mqmdMM5lxCEYfjJJx8RHlgDsiyTUmJKfBu4laRKGWh8EQ2kn62AMaU0X+RKyTAMfdxP03Q02q7lCgAQRVGv3+l0WoyToiiM0X6EUqvVCoKAUh5FEaMBxlhK6an7zjkjlXSgrmtr7XA4HA6H1to8z4+Ojp49e+a9BUPGMcbAWGOM085CCxCilHrTFaVUmqavvfn6cDj03KK40j7H+IWsNcAZay1o6vr6/veO+tA5Rwn390kURfv7+1EUeZmTH3Pud+LXxAAAT69LkkRKaYwKQkYIaZrKWuvnN27CZRAEGFHnPAJMIURKaa2c0UA6XVeqqqosu9TaMhbleX55OU7i9snx+WQy2d7LJ5OJM5YyXBSFtTZgAWNUa62ESNNktVp1u92LiwtvF2GMCcOAYGaMRRghBBACQeDHqWkIqVTNZDJZLpcQwsViEfDWRvWwxlTRWukAAAD2pW+xX8K20qTVajHGfFcFgppSeufOnXa376GasiyrWhK8jrDG+LTqNn5wPlXgr7vSvrL+eyUN38zc8JrO5uMdpdQjWD6LRFHknPOe05vPurmE+i/dNvD1N4/tWzd3Q3lx80sxxjCBN6UfAACttSVIKYXJGl9RSk2L4iv41Wg0Oju7uLyYPD88OT8/F42hQUAxgteUEQghcusOOEIoF2WSJIyxsqqqvAAIesV5Ocucc3ZNJUFo7dZkPHekqCu9FIwTAMD27s7b5p3x048gWXV6KWUoDClhKAzAclmiterQ+1/5M/DfqPXwzc05V9e1R5i89kFfW45ugJZvIkzu2rZ547Xstxs/08aVazNeEoBrbMnXnVmW+RJho7f0BcfmozefeLO+BNcOSJummxACe1qPMcYYCK5JM4iDGwAbhNDa9ezWG/fCy692Pbx3rdqYz+cXFxfL5dI5hwjGmKJryzJfoMScrBWu1lprCSEQU4SQlBIZp/2YX7rmEkEICcbQAVk3RiqIUBhHIYsGW9vIiv5oi8bxeDb96IsvxuNxu93mQfD4KG93ExxCGnFl6tmiLo3Ia8m4a7W5LGC3nTgNWp22I3oxm7c72toqLxvO0rfe3P3hD34aBJ35NDs7vZqNF1GQMEJtzRqJdMNubb02XxqnLQaoKso4TLSUnTRBiARBSAFqJ0mZ1y+eX4pKhDw0Wi5N+fTR41qK73z3u51e97NHXwaUfe/97xRleXp6mlf53s5ukiRPHj2+ms85wP/d//B3jl6cfPjBn54cvpicn/W7A6sKLcp2MspWmRALo6I02i4TdHn5/MWLL4hsc87TpIcpNcbUdd2IVZHZoqydCQIeIcfOTyfT6RwSCgDZ2hpFIWWtJNxOEKDOmTTt/PjHP41i8uGHH3a77Tt37kitb9++/eYb76StzvOL893d3W7aBju78r3vAGARBq0kRhhyzher5bPDw8VqtZgtOedSaGONkjZblUYDo+FstsrzfLkoyPnpoiqMscrfNUrqR1+8OL84i+PQGLNex5MWo1wJPJ9Wlbh4+PAhpdyezxEihHIA3GDQa6TWTaO1Ns7yMIiiyF89axCM4JAzz91tpLAGIAiFVo2QXj0ftkOIUVlVZZ1D6ABYW5UBQBhjCPEgCPwtba1bN7BdtVqtdkY7nPCQhRRRGnjgAWutrQXXHTV0//6Dfn/47Nmzp0+fckQJJpBCiTSGWCltrQE+5RttjAniKIyidqdjgEuEcNJpZb3tGgRISosQ0tp49pbnFjnnvAYQQmTMmtLspf/gOvT78UtCiKIo/LI1SZIoikLKFouFtXpnd8ubE2hNAQBhGG6Y0oQQL5J0zk0nGcY4WzWTycLD5ot5VhQFQshZ1NRqfDVbLsrjo4vpdBpFkVKqLMu6rLxvlZcLYkx8YyhJ0ul0Sgj56vHTJG6VRS2qOgkjjHFdizCIooivlGCMOQOAtRQD1TTjy/Myf21rMDw9euED5Zo25WmJ6FV3gY0SzFprDAjD0J8czjnndRRF9+/fvxxPj4+Pj4+P87zw7wo4a4S8EdzBKxn2myAq+EZ+dTf4BJsWrzHGd5F8egjDkDGW57knoPgX+EIB3ZCffTPZbP79K3L/BsS+2ZV4Bdn+5rbZ7Te36105X/FQGkG77suUVbNarCZXly9evLh9cKvfHxhnpWy2hoN2t1cW4uLiQipgLbAWmjVtEkAILQQch9Bg0RirIeOxlyrU+ZJibIFDa+WnsxD6SQ3arBfHZaOqpqGU3r179979u8HvvPnnv/hAGzienb84PTk5PwcYIAoABBBs+Pgvf76/6XLhlesBXEsZi6Lw8S1JEgihvl7AGGN8N+Qm9uPrSHfNJfSua/rrI6QBANc+117V+qplk1JKSr2hMa0XHl9XW3zzd9/sfJP7/aFCCDl5KT0wYE04IDzwtJKbtSlYa3OuJ0RYePOr3XRNGI/Hl5eXSqkgCPy39vWBMsY5uFwuX7tzQDmL49SvKzDG4HpXmxNCESTk+lBrRyDU0kRRsrW1tbs9iEKytbt3fnaUa3k+X16NJ+eLbJqXkvIWIXN1wdA2beFYpasSVrJqllAbTQnsdOyj+XEnJa2Q/sO//3q7m4ZgvIJ/0uuO5jOsFGu1RvvD14aju+l7o6Yw/+GP/3wxWyIHojAkGFyszs5fiKQbOAljli7BKmScArQ96BNCiqJKAppGnEO8HI9PL0+BhTHnl/kiTZJ7d+6Oen2A0bDT1sYgaaBS3TjeHwy2toaTq/Hi8tJZu9NtO7GCYlFMT0w9v7ff3x4NGTKlUVV+mnCyvx3GoTRqSnFFItBKuBVdrcxkNoMQJ2lbCmiFa7fjQb9nDFXKCK0QhISgNI3SdEuAK2tQxHhAuZJQVLIqyjjiJycnz55/Ndrur4rcWfjGg4eT2fL87Ezm5VidPoUIE9gO46QVO2eMVWVdC6WXy+ViVXSXRVmW7XYXIcQQJjR2gPUHI87pdDrNc5FlGUEoaRoEIIWQMBYiZOtalqXL80xrfd3QjRAKlaJSyqxa3b4FrHPGIsZjrU2jdLvXxXnVSNlUtWgU5YxojBCRWlFMAIIBD9ppy0fsRirVKGMtItgYk0vZ6nSHwyGldDFfQqx3d7d3drbDiBOCCEXWar8oX3MarJNSY4wZpUEQOGMnV2PV6Xh4kAXcF/Wc86Ioaimsta1WqzcctHvd4fZWfnrmnKua2hhDCCmbOi8zKRTBNIlTwkgcx+dXl/DqstVqtbud7GJqhCLAtaIQAiKZRQA3tZhPTwCC1gCllLHqmlJA/DBW3+eL47iqKn87+f7iJiT5W5cxFjE+m80IQd2m7alJniF4HZ7WBZM1wJszlnXDOYcQL5dLXyhIaZ3D1sKqbIAjSdyRUk0ni2538POf//zo7DgMY6u9Zlp7Vqm1llJOMAMABEGotZlP50+fPpVSLpfLMPTgikiSyAFVVpk2nHGmjSMUGGuybCmEuHv37ldfPSpr7++EAQDGOeCscdYjkAB4nhOENwwKlVJeV6nX4yUxAABjfHl5aYxbrVaMB0Iar2kEQvpM/a358mYM3UTGTfWw4RhuguyGVmmM8aCOD/dBEKRpijGezWbex9fP1L4Zal8J5Tf/Bd9WRtxMVOCapeGf2SzRvvX119/i5dv99ayUsuuF+Es0BUIIrDXGYYfqSjhjiqoRSivRfPjppz/50Y9u3b595/a9wWCkpP7Vr/5iOrkoKulPoOd5eQGqcw5AmOc5cI4yFrVSSilBuHKAYl/TIOcccJuZVwgDqLRxAGAMpJTHZ6dRmhwc7F+Nx8cXJ++8+z7ihIcsSulyqdKU5YUEAADkvDTPOd+CQAD+pWO+/xq3m6iSz/eeptdutyGEHpj0FbkvFDaNgM153vyOr9AUbv4WN2yIXrJb/Os9D9p7T92sMDaV6CYsvHJtX/8XbgoFH0UxxojRjRPzzbrk5jFvNvfSf8kihzYveEmwAKAsy/F4XFVVu932Sx0PJAghyrquazGZTFRdtLudnZ09j3pubu0wDLW5tqO49mYAAEDrAs6MMa1WSwxGPAyt07NF9vnjp189xzxkBrhKOxy3cuPyRZ5uBw0urUM0jTgismCYMM7al2en+VypBsiVJQM+6rx+dvzsyePjzgGJhzuFy5bz3NVgdmnaQdjdf70B1d/5na3J5dXpydGjLz5fzic8wE2tGA1yVcjaiMpYCVaLLG3FWglKMGA4Cuio18fO5ovl5HJ6fnaEo/SdN98ejYbnlxez+RwREDOezaZaa6yVATabAKzUG7dvK6VUUT799Nec0J0O74fDTtrCENV5wVo2W7wIt4atFIjmqqotY2R3b3dvb+/jj2eXF+N5Nt3fu71/sPXi+Gw1z3bg7s7OVhDx8/OzYjxzARoMOeOakBwgUBd1UepZNccuoDiYjceHz59sbfd3tnaDkC5ny7fffb/T6//pf/yzIAjqRSGLYjmettsp4cQqLVSTZdnp2dlymQmlrbJcyFIoKiVjQcwiSqK60nEEIHB51iAYYiQJhpFWBiEqjCUYhmGEkei0trIsM6A2mmiFrYYIAKNMXcvOsCUlMsZxlmBGi6opy5JQmrRiXBMLLCYCIQIAaFQjhKIR8TQcylmImXWOUS6E0lpPLicNl6Nu9+6du51eL8/zRsnvvPX6gwf3W6304vJstVp0uq00TaMo9Cs8QojWBl77f1VV1UraCOB22qnrWluDIaGEUsaUUgYARgOAoFRGmypJ29/57igb7CyXy/Pz87IpCSEawKqu/ZBynzBYwJqyKlbZMo5HzUgvyiIva6kgwAiBgPCgG6jEPn/+AloMIEQIGYOUttZp62AUEl8fXFxceEzbz1jyWLFnWSOEfKSo6xo7QCmNosDHpk6n0+12AADebtb7rntSpJQeq4DeVEpKWdfCg+eMBlVVOeeMsVEUc25Wq1WelycnZ4vlsq4aX3n4EAMAAAApZQgheVZ2O30ha0rpdDpL07TX6frxBBiCOAwwQXFEPdUUAgWcZgQbo/PV8tatW9ABL4haCxedg9b4WbN+tDQjFGOMb/gUcc673S4hpCiK1apqt2II4Xw+l1JiTKMoiuKkrhtfUmwC5To7XecpcGNRfgNQXYemVyL7t4IB/nkP+XqFmO89rVYr5xylVAhxM/j6DX4bhvHXu7kboPfNQO/j+ytgiXNO1LUxxhBstCwwxBi1u0MA7NXF2aOvniJI/t7f+3tvvv7Wr375QZHPITAYEHw9MNlCsFkZd3pd76bnACizXEnJOB/2+nmxuqbHXU8y9KcWIdlohBDEMC+LX/7qgyfPnr71xhvYXmV5OdzaOzo+e/zseRBG5eUKEAMQ9I7DEALPlH8VYfib3zb50hepHj/YnPBN+ehzt9soYOGr5eDmUnTXFl7Xv9JmwONGiQM2gsa6rptGenXDTbmNLz5uVgk3a4VNofDyxdffwjdwr2+E9ZWplHJgnaT9QW4OdXP13rxf/KdzzgEA0+n08vLSGNPttebzOTAGIegp2A6upZhXV1fK6ChKkiTx74KYXh/nt5BvoLYKqWKVKSGlVNNpXZer8/PzvKiVUtJIHgTtbodwVNd1I1SvG19cTYxG7c4wQnEjidaulqjd2avyeRxti2YlFWek+/jx6ScfP/su2xVb29UMrcZWBPjsMC+Wz6qcvfbaw+H2dpYtb93ZM7Y6emHefOshMHpyUR2fHhtVExQDR6VoRKMQBldXJwhgq3W0H3W73Vu3DgbdYRzHR+fTzz/59COlFtmyamrnTNpOdnd3w5BThgkOZFE2ZcEgYAyrIsttxgeDABgta5Gpuimbsmy3EoqrYnUuVFOJpt/vtrvbFpSL1floP71arE7Hj5bFlTBFWWiHaBDCLJ9EEcO4FnqCHen1B0IWRZnFrWGWT4zETe44aSVhVwgRMp4v82U+h8gKJfOs3D24VVXNqD8wRe2cw85C56qinEyuainmy9nJ6WkjJA8i7SzBlBCGEWc8XKxKiGez2Sw4PmeMlWXe7XadI0QZJKXkAZZSCW15LWbzedNUTdNobQCE3BJEKaZREIaYxtq66TynlFpAMEB5UVZ1mSRJp9duO7gNtrQDWpmmkR5j10JbA0xZamOSKOWcD0ZDY1y2XK1W+W7afv/dd4eD0fPnz798/Gg6nScxJgR1u535YmqMGo763W4XQuBr5yAIjLFlWTvn6koYY8qsmF5NDnb3nQMYYimUkho2wk83QATHQWycK8uyFjJxIOr2Jlk+ybLFYkEIapqmKLONHJli0mm3Rv1BXZfWWlOLdpooIZ21Rru6Kg3RLEgghBRhAyCCBDPqsRMIIULEr1AppWVZegqCvabrM8ZulDvaJ28vsGy1Eghh0zRx3E3TFFzLljYRA2OMsY9K1jnnjVmapqqqKo5TrTXjzIMoy+UiDCPGWJYt/8N/+JM7928bYxDChFDnAMYIIwohzIsVJzTPy4ODTl3X7XbXOdfrDY6ePm+aJk4iAF1eLAG01skgTBhLlK6NFZRw68DV+HJra0sIoa/H2vrjJIRAvJ7G9s2Q5xc0nvzhI+zOzk6v1/MK8m533a4yFjinEUIYQWVfshNuJtENOfzm5v+yObGbguDmompTc/jnKaVZlhVF0el0/Aopz/MNv+zmOvKbQfA/p2K4WWFsvsV/8i03CRnwGlTABG4EeJ6MaIxBACqtrIZVZZUWaZr2eh0I0Wh7v8qz86vL09PTbqcjVcMIcFogCCFwEDgHLHQAAQuRgc4U+cJY62t6gp2CVsp6PDmn3CsbAfC+Ds5BCwy0yCFjjXaAUEoIWiyy6TQDwP2d3/5Of3SbBe1//W/+1YcfPY8S6hwQ0gKHvRkQAABC5/FqDMHfNJ6wOeGbX2FTI3qqR13XQRD4n8lXvc6tX3xdKFh4AxJ75efbyHQRQj5T+pdsVv83C4W6Fh662FwVG+7C5mJ75RrbFL6b62FTUuBrX3NrrYVgI6Pw3pfo5dj3DS4FNwX05uQQTKy1XiF5dXU1nU43x+O83+KaeIRueplIKcuy9LwHHiJKqZTSAbTp7vn9I4SkEbIUZ2dnF1fjbLWAzlktEHQBTThzTdNAhJELnHLYuV7atq7KV9oC0OnzOAiEdMtlNp1Pd7ZGtnSIIlFqad0sLwxgPOrKcpQGD28d3LX2aDqfPT86NYfHp+cns/xKSvn4iy/vv3a31U3DaQwQHW7tDged2VKcnE3DsI9ITDEBEDJOjQWYkKyopvOZ1RBANNreSpJWqbGUjVJqa2ur3e1Yqy0wrVailSAEj4b9OOQvnh+OL044ZZ1O20hdr/K6LkVdwpDpplKqELXRVkudY0LihA23+4Ot3myxqGYlCStASh4BSAULESbJp5989ed/9osf/ehHrz24W1ZLSk0U414frzLloKTENPUqIGkcRRAgB8yg2/ve977X63d+/etfjOdXL46OPv7wk/v3H/7jf/S/evbs8MNf/srXlp1ep1Eyy5aUs7wqZaNCHnIeNo2kiDMMCOIBiSpTQYR6/b4npPv5QXVdEweRMpY6oIy10llglqtcKaG0wBhTHvAwwohiwgjlELlVsVouqjgOIQZV05RlZqxCGBtrOecsDBgixhihGqGkcdZBCAlSQjZNo5Tp9/uccSk1C/itW7c67d5wawsYtxn4/etf/3o6Hd+5c3sw7LXbaVVVX3zxxWq19Ay4TqfDeQAA4pxb6wAAom5mk2nTNMDaMIkD5xwAFjjCGbHGAme82J1grXXV1Ofj+dH5+dHF+XK5DChVShkto5BTiqscxDxgEO/0BtZ2rDZJksiqZBRjGBoDynJRVDmuVCONtVZbB5wljEKECQaYIMaYFMKnQE/Wc9duLeB6Po1zbjO2wMdlf/s556qqSpKIUuozltbaGGktIGsvamiMUVoB4fJiaY1jjBhDGUNSaoQYY7isHESOBySO0l6vK4QghBHCoAMIrSOLxzOMdjT00/OAUms2YpIk/V6Pc/7WW2+2WkmWL4WoGad7e3vz+fyzzz67OL8EziDgVoslo3RrNKrPxj7KeIE5psQT6/yKB94w0vdfsyzLy8vLPM/7/T7lbH9/HyF0cnKWZVmvN9Bay7oB1yrwIAiMFK/kS/8Y35jm8Eqh4K41YzcXfJvA53fiY5yv4fyJLcsSY+yFrH5lebOe2MTW/8z6YLPZa0HaprK5+eQ3N/cNGt3m+2L8UnrqD8NaGxDqjIUYO+ekMg4giIhUTcBY0u4EYfzixYsqz4A13U67nYRNAZyz2minHQAAYoQhJBSt8ioMKYJA6SYIWaebCiFWq/w6EWIIoTdfNs46a5C1lDFEgFKKB1HEwrKsK9F89vnRG2+89cmnz8/Pl1KDFo0ZL72+AjkEAQBgPXwBOl8k/M3Oetic1ZuP3bX4sGmaxWIBIYyiaHN7ro9uc1HBNWK/2a7ZNuYmEuCc2xQKNzsFG4GDH+JwsyN28/fd/Pcbx+z/DG+8GPgCl3POOfc3wkaDgAECN5xJEUL+9Riv50G/Uo4wxqqqIoQ4iLIs80Cg5zxpa4zRfud+1oNSaqvf8bM04bWpib9ijTEIvzSS8sHNB7G6rs8uLyaTiairgOCAE4LQcjmPoghbDBV2jZNSWweiKHE0DsmibOpy1YQJ7nQTGsDz8/zpi886SWQQbw2iMOVPDp+3Br1+scNYK04HvZBqpwuZSbNc5stCjQFvDm7d+f6Pv9dq9VTjIJtfXpkf/+Snk5NlVeOLy2z31ojQFPNI6QpS9uDNd6AD+TLDPEAEsShWDp1cjJdZtru7uxNxbQxhxAGT56vZbHbn7sFkfFkU2d7O/bfffD0N+Ww6zrMlC2xpCq0VIzxgnBKEodFOW6cBckk7YQGP2y0aBqDAjAe7e/tZVrVaV7NJ8/jRc057dWXLQj/68lApkxfTMLadDnNW11UGgSbUJTFN41a5sqJUCFkAAKf8/t3Xoig4fPFEa31yekoIuXvr9mKx2t3bW67m4+kUV6VSqhZNEIWU0oCFYRRhxBojGaQOA2SAaVS32+33+1tbWxjjXq9HCPnyyy9PT0/97w0QIoyxMORxHDZN7Wm9lOFOp8NZWFVVnudFUUkpu/0B42EQxULURVFUVW2BEVJShlkYhCJ2DuZFtVplQigAQLfVw5gKQuqiXMd3BKWUnU5nONjCkFxdXeV5kXbaP/3pT4VQFKnZbDKbzVbZIooCB0xVFUI0Xr4fRRHGBAAUxzFwaLlciqXgnLfbbYjRaDRiYUAodRDQgIdhSCnJsuxqMm6apt1uD0bD44vL8XJeK40wRZxhCDCBcZQWq1WNADS6kySL2RxhwDDhmJydnWVZEccpYRw6K6W0wmQeN5PaAkSdwx54hxgRSi31d3gQBF7X4DlTHif0q5bNDeZreZ/M1vnbGF8PIYSEEFVVG+PCMAw4klJWVcVCBqEDwCZpwnlPShmG4Ww6t9YyFrRaSZrGUsrjkxd+2pOHHYB1lFKInGeDrhsWLLDWelfj1dJ5QeN0Ou33+zs7O/fv370an62yZRyHt27dkvIgz1eTycRoDQDWWnc6nddee228KAghymittTLGQUAxghB6WNKrHvz6FwKAEFIW5HnunNvd3Y3zLEmS6XR6fHwMAIjjmHOutKEEYkw90dUTQjdRb1MobFoPaOMQtw6rLxWJ4OtLMX/a4bW1s6+N/KWIMa7rWggRBEG73UYI+QEom51v9vNfVCVswv1Nducmtv7V77qZPK4/92Y6WR+P1hpYS3DIA8QCPhwOu4N+lWfL1VxLOer3oii6urqaTydKNAe39rIXY62157sAADAilDFCyM9+9qPecLBarT744IOLqxVjRRwHlAHj1mcbeoc/87JZEASB0rWUrijKdjuyFiyXy2OJOt2djz/+VAoXBrzIBXDEQQcAshAgZwFA19OhAAD2v02h8Mq53azmhRBZlnlu0GYcFLyxprfWQrRO+TcLBY8R3vTzQGuDUf/Gl5jTK2/Z6FasfZUy+W1VwkuBAny5AQCArxIYY/4S9WsAjDEkZNN6gGuu4uYXfLmrzQd5dBNCiAnxXwchVDeNnwGmtb3eCUaIbEpe7k3gw3AzE27DxIIQehfl9e2D1lV4URSiriSCOgygM9AhhhhAgLGg1ep4uCVAAY3TNJoV+fnlxTTpVPdeu9PuhUpnxqwGW+nV+aWVoMhWCJt/8Pf/fpQmxdVssZq4XFVqkXZx2mnb83I8vXT0dnfQ/vHPfza/KpVgRvZ//atPZzMapQNEQoiD/YP7w50WD8FyNU5S/vTpIwRAWVQ7Owd3Du7J10y5qh89ejxb5kEUpq1kvly8ePHiYnIBgB0MO48elVHIe71et9vN3DxJIwAHnPOmXmgtrYEGuLquHdB+yFu315uv5lVTC6O6ShoHlbaOmBdHZ0K4O3cfRmH5+PPT6fi8393/7vs/hRACy1bzOuBJwFMEeV0rTEwAdaeTJFE6vbwoCxt2W/PZ7C/+4i8Wy/n9+7fTpH2wf2u+WP7e7/0eBPj+w9f/wT/+R8+ePfnoo4+sM3meex53WZaEEIopABBYxwh3BjptRNk0rjZWKS2aptne3u51B4eHh4eHhwRhESfI2KrVjtrtNg0Yj+/WdRlEfDQa5cXqcnypcHV4fDif17u77SgK6py0W1EcRnmeW9MY5+pKKttQLhirDXBKKW01oogQktUlRRghEiQp5YEwzjrEwjRp98IoxRj3nRtujdrttrV6sVhghO6/9UZR5YeHz6s6G25td7tta/T5+blsZJPXohKyUdm4qMtaCAEZrkvxP/6//u8Y4739A8ZYJSSllAfBwzffunvvHgBouaq11hAJ5zIUkv7OkKdhVZRGKWdsU1dVUcIwbqdJGMUa8kXRMMYI0nZRyoYnUdzUTd2ovd07xtmz88tKNIQ6bQxjpGoyqEmSJEprZCBnXGsNIZJSaZ0RQihlCGGMibWOEBqGnmwPGeMY40Y1qhHW6jDyjoFUCeOMq4qqzDMPVNarZRQHGGMr80qgg4ODN370I2PMbLnKs1Ip3Wv3mkbWtdAKSmmcw73ODoTQGDhs95hjZZVrPxTOAENsEEUqz1jILi7OopBv9TtKyHbEZVH0Wq1bu7t1mf+zf/bPsiKHGHW73Vt379y+ffu3/9Y/+PzLF9Or6fZotFosnj8+/Nlv/fyzz57mZc4p7WzvWGCUNdqYqqmbpsaQrFar0WAktKGYZ8vVYDAQsswqyeO2w3Q6y6U6hhCWtUjT9pqOSmkUhstlFnLinG3FkSdGQYwppQBAKaVQIo5SoaSxBmMMHDIOIIwYY6IsIHTWmna7l+c5Y6QsZRQF5nqKIMbQGFVVhQ/fjPGyLKMoopSGYbhcLu/evcs597NMrbU+Fm+KjG/mkldKh1dWbDdbJK9E/G9NYxgTY4y1DiGIkEe/KaXcOUMwI5hYpz2/lVJKKZGmlFaKvDnY21UK3N7dCoJAUBJYZ4w5fX783sO3333z/Q8//PDR51+8/fb37z9c/sc/+cV0Vu3v7pVVM1ksd3cH3/neew/evDsa9ZfzaZadFquV0wBLFUG6MgBj/xWMMQY6gzBECDhryqpJkjgK4qIoVksZBm2nceHq8/nk8PzMGoIQ5SxUuqYEMY6yPLcQhCEzGmpFEeJaW4zrzWm5eSZvnuFvPvnN7S8r4G6e9pv7gRDWdY0Q8n0He60msMABtF4ur38+Z5SQAADsAHZAaWOlslI5Y4C1hFPoIAZrSaQ3nDTAAQu1NHXT+BQohDIOAAQ988ZvmwWDu56o8s3vwhlSShlrMcYIYQAchGg9M51QgInxrQXKHSbCGITMhrR7vRqBGCOlzCaRQ4AhYBgFlASWkdZoQJJ4NV/kVekQttZqbYMAU4gRMhBiAgl3lECKEDIaIxhQEvrpwc5iP4QyCMKyqnwjNa/Kuq78x83Gk9VqZR0M47YyzjpQa+AcSJPUMX58cvT6g4eFKOq6JIRECV40F1EXdl1cVVUaR92knaZtbuIW6oq6ubc9MKJhBBAXf/nJ5c9+/CO1e7GS2YOH93fBzovDw4gHLz4/bbmBvLTRw7S+rHtRD3db2bzsdoPHT34z6BkNLn74W3fv3GpdXV2+mF0O+wMD8Fa6XRVlJeT0YiHLZ+2kzRhbZXOp1WQyefHixd27t1+7f7/bad25c8tY9cUXn22NdngYOcwKbZ6cnkEIO51ONT9stVpSg6ouDWKMhRBzguEyr3k4EKLGBpeZCFgWIFoVVcMPg07cGvQevnPvzXff+f3f/83hs0lQd0f9USWb3mhAA1U32WpZthOYZ8vp0V7TBK52o/7+zGWVLHkSHV8cL8vlo6eP/9bv/vb9+/fLqjg/P56MD68un33vez997423ZpfTx4+fOIlD1CUgbvM0y7Ko3bfW7vRbAWYKyqque73e/DTTRb64rBAln356hAhmnL+4WpEgZJ6izhglxLtzOAAtxhgTyBiN45AxUhSZ1hfOOa/vL8vaU4EIIdAhrbUSDcQIYwMwQghhQCACAAAeYKedtdpBV9e2aRpGOGMBwqDIVhjDKIrSJLJOAmjTVmiNZCEKAUlSZiHXWk5nk6IoFrM5QYRRFscMQyEbLZSua7WazpMkjniSFfnnnz4LIxomsXNua2c7y7KyKFrt9mgw9DDGcrWglBqjvDsyCUOrjdGqtFZLVdYVhDAvS084kM5BCFut1mQ+a5rm4ODg7r37y+VyPPHTHGQtRYQJxtgBKKVUyjDG9Nf9AT3Rb7OMAGADA67bkwA4KVWjTS0VANYC1OkNwjgCCDtIfFtAKeXb7FletttthCCAxgFjrVZaVFUjlMKIAmARggg5a50Dxi9G8nxV1pVSCl4vfZxzxmjGWFmW3W672+0uZ9MwDFer1Wg0fPfdt4fDYVVVazYlIWVZnpycGO0W0wXB9MGDB1a78eXlbDa5vLyMYl5WmdKiwzud3sgAN18shBAIQEKIx0uMMRhA55ynDWqtnz9/fnJycnV1tVqtGGPeVqGqqvl8CQDo9bqtVmKtLYoKYxiGESFMr4feasZYnCZlWdqX2hDr+81aa37tEo2uLXXruvYdB3CthtiQvDyi41WpURR5yetisfAX9jUH/qUybRPE/0txhf8Fm/+tbpj4YQghgBaClziHUtIzEL0q2JhmuVzGcYwAfPj6A+fcn/zx1XQ63dnZ2d/fPz1+8eWjz3/4w7f/yT/5J9Yhodx0tvzks8/PLs9++ctfHp883d3b6vU677/7ThKlv/7Fh/OZIsg48pKgR17aWsPBYDAej/O8HA367XbaNA1EbrGc94O4aaogCAgJrPGGH5IQnq1yAAFlwForpUYwIAQhRIyp/6bP5Ldur7QGNlQbIQSEDrqX/FnwUjb56gYhNEatlQtuTWWwxhng97w2ZdJabpQOr9Q9N+uhG8jEDUDCQQAxJsT7K6/BCePanV6StoMw9vsklGGMATQIrq/wzQ79Wh9dT3vyz3gQxd95XsoOAJjPlwgh704GXpKKjFIOWgcAoJQWZZbUsdYSAHttJ22MUdZSTCDCwFiljVRaFKWo67rO6zzPjdZxHBIIpGqMktaCPF8pUcVxHEXB1cUFxvDBg9fv3LljqDo6Ovry8VclYTt7u4NOV2oNoet0OqgHKUEXJ8dnZ6eMYAft5exeewCiuNXpDoC1vX5BEB5s7xw+PYQY7d7eSwZdpxzg6Pbdg1pUi2X2wQcf1HWZRIFSMkkjTLZaaZymcZ6vptPp5eWFUorz0ISqKGQcxxivfE+qrute2Lt//8H29ujx4y93dvbefvudnZ1tHtB+v//OO+94dOrevXthGJZlsVoxhAAk0DqNkFParZbTMOR3793uD7qz2cTzoghpOUuM1HkzUw29td0DCmhZOp0jYBACFBAgkSyckkhk9Hw6qSvR6dhedxgnHQgJwQHCHAIklb28mmndRHGyt3/HWPj48dMk7Y1GI4RdGNFut9vp9Lr9YV03GNHpfMY5v7y8VEqIRo5GoyzLbt8dKWlWeVaLqhFlzFphlKZJSJRqfBi1TlrHgbXGKmMUgAYhGIZ8SIZRFGxtDU9Oj7znYFEUQitKaS0aa7UFTilzDbKB9VW6vvwhhMAB44BzzkCAgUPWIefw2dlRvlphjHd2tnvdtrXWORMEgTZlXMbWWmkqhA2AxtfchFAMCXDEAoARS5NuzNvOuVpIz5mPgzZwJUGEEw4w6rS6O6Mt77m0yjJPhOx22/PVnFMGkwQBUBVlVVU+kXio3OcbhJBx1ii90jpfVlEUhQhVda2M3t7dmSyW5+MJpdRWtV0PsEeeMm2McXYdUDZVArmeqQhuyOh9l0FrDQi1DmllG1kaY7SDvemi2xsoiwxEDjNnIdBAAwgBNBALWRblki2JMaYRNUKWMqgtkLK2zmHisIVWaetbyxh7uzTnHMaAEAKQs9YoZTGGRZFtD0ej0WA1n4VRsJjPEEL9fndvb+fjTz+hlAIEgygGAGBEV6vs8uyi3W6//uDh1cX48vx8tVotFrM3XruLoFksszjkO9sjY0FZlkoZpUwQYMa4Py0AE0SgVE0UB1EUeY9YrbVnNQ4Gg6KoGGNRFAghq6rynNBWK7kOcNBCgDAAADNGecAJIWZdKBBjDJPCr9XQtUbLExEQQp6Z6GMfuHZ0RtdDcfyF4R2WiqJIksS7XOA1gAHA14Him5lmE+7/irrhL1sE/2XPO2cAsBujSYyxf4AQgsgB4Fno1Ji1cIYx5l1ufM6bTqdSynaa/vCHP6SUfvCLP//44w8//vhDynBdlGmaTq/G27v7aaursyaO49/6rR8cn40++/xjCGFRZAS7H//op3cO7uWL4smjw7oWtbHOGS9ztRBCiIF1wNnTk0tGIaeEEFLmmRJN0g0zB4xSdVUwSjBC2oFGKQAsgg5CwBjAGFpjGMUYEWeNURriNUf1P/N8/i/bXsnHm2d8Bz27jg8IIYiRcwYG65NvrUXf+KE2zQLwkiWjgPNsWWCM0dY6B5RSUq0HRmstjVHWav+yVw7sJlL1CpTifZT9peiLQm9D3u/3vfucL0F8TQwhhOBlm8xfNmuFy3V3AF6zIH2ZEATBYDBaLpdFUU0mE4wxAIgQZIzxlJjrI7HQj4q4Lg02Ck9/zB4O9K/3XQkhxGw2e/zZF6vVqqoqBKy1VtSV0gJDFIeB1jKNEwydEPX29uj9d995880383LaDUORFWeXVwljoqqWq5VQ8uzs9Pbtg3uv3U3TUDo5vrz4/KsvVlX2/ndfu3P71n0FoQMaICHkrTt3p/OFcjZOo6JYXF5N09YA04DF5OjLw8vLyyRJjBJHR0dhxCF0TYMHg57XpgWBH2oPpJQvXrzI8zKO09lsIUTNeWitnc1mGEPfPo7jOIoiiFy32/fFVpYt1fwFIWSxmGMEHLQQOqlq5wwm8WCQxHGYxBS4ptOOkoRACImMoqTlLD06np4cTpyie4MwW9VOLzCiGEIgYDnTYqEbIavSFtIUhYSuaSU4ClvOQeBwGNA4jk9OTj78zScYQ4isUmKxyFbL6t/9u//v66+/DjEZjYatVrvf2xpt72JMT05OT85OKaXL5dJavb+//3f/7t99cXT4m4/++M79u68//Pn+nbvGwkbp8Wz+R//hP5K6yZ1zhBCtkTHCIWKt9rIlrSVCKAzDwaC/tbUVBMFkMhkf51JKZl0Ueakx9A05ghkE2BjjPXUtWGOtStaMEGuAMTYIojRJMSRNU9dNIUSFCazqDCOttIAQxDqWeiFMCCGUWkhroXWcha1uByOipZONghYGEWu3A0Y4RWS5yHZGO5hAB+Hx8fFXT5/spa3/zf/ufxu3YkKIELJRTbfTanc7dV1fXV0ZqdJWHAWBKKvpdLqcLzAEnPMoCJ1RhBALHGZUKaWNbppGFHJ/fx8SPF0t9KNH3W63auputyuNbpQG6wYetVKubx7w0twUfJ0ej25IqjZOcFIbAKF2TkurtVImu5zOtna2q6aRykKIMYEOIB4EhBAHMba5UmK5mmptjYOUcuec1EqVAkKPTBrnjHUOAoicg5AiBJwDPuU4CJ2zAFjrgLXWAdPpdAjFzjkPHc3mk739naqqgiDQVemcY5SnacdaK4RqtZhzME7Cfr9blnlZZXdu79VV3lS5rEstGgOgFtJqo5QGABJC6rLykYsyYqyBkA0GgyzLAABpmvoT0u12ve9ku90uy8pLEwkhYRgiYiEEzhmM8cat0lqrtTLOOgusbay10lxP8AMv7Z48UOEVIjcRhZvgvy8CfKfWFxbex2KDPWyqBP8Wa79F0ffXntsghN4T00dnhAGl3rnPQri+uiB0WtskbXlD3xtHbjnnEDqlxHQ2ZoRqLTHGsm6Ubr6o53/wB3+4vXtw6/Z9beFguPWdd9/b298OQpzni5PTo5Ojw+FgJ02SMODzec0i6mkfAABveAwgRIDEEUII1UXVbbd++2e/tbOz04jqX/3Lf7mzP4qjtqrkbJop5TjGFDGGAWRQaSeEAw4wZp1ptEIEE/P1swe/bTTGf+WZfOWZm9QWb0cGIZRSBkGAMe70uoQgQxm4vnkR/NrQ8Fe7SMhBCCAEdq10MNdOpG7t2qyFNje8leyrjapXagX7dfarhxMARFIZKSXnfDDc2tvba7U7nHNMiAMIomuFjjO+rNl8x00sunl9bupgzjnnIWPMWXhyclbXwjOrRN1YawFwCDmMEcaEYkwwgdClrTSMOCbQOm2scgD78ZjWakSwc8ZayBhN01QppZQwRsmmaqqMUooRgMgFjMVhCKHDyGLorNX3793e2trCBNVNJZbZqNPdH40mlxfZfKZns7yuwjja29vZ2d06ONiLIp6VSxbg+WyyKOZHJxcBT2bzHDi9WBairh48vP+O1kqpvClefPno5PQ8bfda7X4cp8rWWVZgjMeXl3m+un37VrsTB0Fw+/Zdxlg7bQmh8jzvdvtN2VxcXOV5TlnHWjPoDQPGF7NlXqxCHkRBTGgaBTF0EEOsncpXBQCWUkrCkDGmtdJGQgwAMEJyZUXdgG63HUS0rLImU91um8FgMhnvD16zDla1KObLycVZFHe6nW2rgLMOI4QAdhqWlVZSNY3VgrJhjxBibeAcRzjQygKA4iQBAExny7IshsM+RG4yuep02rdvv/HwjeFoNErSlrVgOlleXFzOHi0o4RjTdifRWpZVhhC6d//u3Xu3rFNS3U7aCQ9tu0XSds8iGpyRfi8mNwtYY5xD/oIGdd3MZgsAACLYORiGYdPooqi0NhCiMIjSNBVSCiEsMBDCWgrjtBRG+9Fk8NpxzEmWdpwzZVl6yqQS8vLyvCxLghAGZLWa1dWKEEQp1aZp9SBllvAAUQAqqbSCGgWMxK1UVAoBTCAJWEAcFkJl5Qoh0+93y6oCADx4eE+qOk7C7a1+GEcWgqZpcAm11tlyUdd1U5UWQCWktbYsS1E3zrkwitI0VUI657wHFERIKmWsaZRMO+3pcoEQStN0ma2+ePIYI/raaw9pwIuyLorSxzXvVKqlouxr+KEvCDbZxSN7G1UhIaQsSu8lpaUyRlVVNR6P5/Md/0bfU/TLHcYYQiAka5+TuhZ+nk5d17PZDAIMobNgM1/RQQgdAAR4er+2FjvnqU0IIeSASdO4risAHOfU62UhhE1dTybj1WoBoLUWFHkJYVMLgzHW2k4ms6oot0aDNE3Oz04mk6t337o76rdOj9FidoUQAphmyxWBGAGIAOY0KExOKZVGAQCMUc4F3W738PCwqiqMaBzH3oZmMBhMJhM/YiqOY295pJRqxZGXlRrg1tN1LQDQNU1lHPAOuM4CgNe6LOheDvD1uTMIAm8AdTMhbSK+X1N6prcfIO6naG5cHDYx96/IQ3+tVYIFwPqGt89DAFrnHKXEGGMMgNB561J/XYlGaWUphRjjfn/Lah1FIQ/o5eX5nTt33n77zelkfHl52YgKGIAQ2BoNyrxazefNcLfXH07Gl1LV77z35iqb1uVCi6Yoin5XYQiapoIAePmcL5YgxB7dgBBCZxAArXb0+sN7v/u3frs/6CwWMymyNE3rWkRBNJt8rKVMkhZCRBshGpckIEmjNGlDSE5PrpSSSdRaNSvwjRX/f4POzqYL4CfBblTNrU4bvOLDAay9dh+6ySrw+9lUpQh6DiOwFiLsjLHXzQ3jK3K/KHfg2y8YjwnZa6LrBveiLPBmz0EQDAaD4XC4t7fnDeY9+xJdzxPxb7Rab04juiFW3BTH/kmPJYRhyGikFXDOHR0debuXdrvti3tn1yqhNSbhpBKmS3oeIvX/ohsqSmChXyw5JbXWq9Xq2bNnTZUJUWotMQLWOK0airBzJM9WEQ8ap63p/s7f/jvttLVYzi7Oj4kWo9Fo2GtzSoqqtghAp5yVg/5gMOjeunXQ7SVVs2x3wum0Vdf1zs4eQuTq6go6KxvlnEvT9OHDh2VZlmU5Hl8CALRR48nl0Ll2pxVHLaORFJbRKIpSJc10kl2cTzDmd+8+CIP4+PjYOUhAGYVpVYpeu/PGG2+0Wq2jo8OLs7Mw5PkyG08uvvOd9wLKgLXAwFbcaspqOp3Oy3mLS0wYDcMEdBkjELpaVNqIRKfaKgdYkvaappIGY4wxjylOnj07PDm9GE+XGJM4ThFhFjQ8CCHg2DENgRZSKOwsxxQgGHEKgKN1ZQjWSilKeCsNp9MpY6xpCOchpVi1TauV3Lv34Hd/9ztVWedldXx0enh4dHU1Pru4ms+Xu7u7t+/cubo6k7La3hn9+CffuXV7ezhqP3wjvLi4GE+XR0dfBXE7CNOqbBhFBELs269KmapqwDqTySyvEVpB6Aij2aqM41hK2dRr0TCllLNAGwuAgBBTiirRGOOU0gYYCKGDa6vUXrcVRbFSpqqENUAIVWTFdDo3xqRxKKXNsqU1mjFCKDLGjG7xOEp7vUGctBGheSa0thKpfmdYuMIoDRwgFBIIpJLaNdAKpfHV1Ym19q133v3B998+Ojn5wz/6/Xfefbfb7SJKZ9PxZ198vlwuB6Ptg4MDrXWRrcqyXK1WGONBr8cYJQgrAByCYRhCCClgNrOYEEKpQcAgYIBVwBJG01aLUh4lMaakO54aY72Dir+rjTEcsVfun02K8tHBwwl2bcNuobPM38nYccpKo4xorNZpHDurvcBS1g0lgBFgMESIxHHKWUhJUdSNaGRdizwr07RtrVXGae2sBc5CBwBwQJhaKgUAwNaHFUAIJRRBSCgmZZ7NF1PG2Go5D8K+dXow7JydnWT5ChEehmEjdFFUq3wCAIw4r5sSOssojgLKGFksZgF1tw62xuPt2SefX5ydaIfKSiNKOeEQYEo9kkmUkhA552y73U7T1NcEEGjOuVKqqiqtbZ7nXsUupfRBkBBinaYsjOOwkTLP8zzPAQAs4FprcM3Ehxgggn0oN3Izy8cAAPzKCd8Yr+yuVQwbkZtXmtR1nSSJf9L77G6C7GaNezND3Nz+itz2ly2O/7Ln1+IGaAG0AAIAkF+kQggJQdT7VyGntbZOIwOm0yml1AvZoyhSQjjnyrKcTqe//Ts/+6f/9P/07//gfw5DyjlP43Bvb+/dBw9H2zv/6l//m19+8NHf2jsYbI2eHT598tWjqimyfNrv928d7N0+uPXV8JlRqtuiOXR+JDR0EEGIEMEAAmeLqmQY7+yOOKfzxeTxo4+Wy9mDBw92dvZOTk6bUsYhVY2wunEQSlnHAfhbv/2zH/3oh+129+jF6f/v9/79o6+eAatfOXWbhPfXVSt8s5LbQPqbasD7efuUv5kq+QrO4bO4ud7cdenvnF1nYQSB9WnY46wIIuec78ka5yxE4OXA52/7gpvjuflf7UAlJMZ4ZzB86623dnd3/XwHwgP39U1r7YwBunGbkZLX/Qh3bUuKvPcdpUEQRFGUJAmgfjwNybMy4FFRFL5PwRhzwDpgjFVKO2D9HWSMUc4ZjCGlmHNKCLLASKUBAFAjP9K6lqJpmtPT0y+//DKfjH0z11pljVFNLZ3TqqYYOWBlI1fZbHtreOvW/nzSHg6H08ujgAdhyFutxCHnEAY1FFq9OHwOoLl/75Y2sqlKa3S3lXZb6bDXb+ry+MURQa7TjnvddjZbME4Dxpq83B7tDLe2lXYfffLFpx9/Nl8uMeDz2dJo1O12nSXT6WK5mJ0cn3/3u9/dHsEoskUurAUMhw9eeztNrpRSUqrJZDKdzq0BVdUcHR0J0QCAOp0eQkCqZjAYJEnSNPLo6AiPGIeYBnGEGaXYAGMwxUaFpNeISkpxenGxXC7393e3toawRB9//vj45Gy1LAFkhHdWFaiVgrRTNA4ChCCCjkiINcKAA0aocgTTSEo5na9qoZumcc4UdTEcDl9/883nh88q0cQk3Lt1gBDSFsym2fn5+dV4+uzZM2PsnTt3BqPtk5OT27cPHr7x+nx5ham2rq7qxdHJ462tYZ5lUZTcvt12EH7x5ZNHTw6LSrw4PiHOQmehsRY45SwGGGltlHJKW6UEpZQarGRZV4YxRnAEqZZSCqGqqqrqqq4biBGEwPh7ASFCGSHEAaC0kFLGURsjaqAOg9QasJivyryEgHS7HU6wddpaq5WAEGlltFEnJ3POpkVV7+8hymOpdVHUENRVKWSjoHVpFAcRDQMWRjhO6PjkGcb04HavKKqrq0NjASLq/Oyw12t99eRRnpXLPJtOZ2XVVGWjGrG9vS1kXZelMyZcWx1Yv6bEBMatVDVCa9lIwRizwC2Wy263a62dLxatVnt7d4cRbq0NgqDX61nr23V+rjTbVNnoJe1r3c70zDhfOqxHWBljjGlFQRSFVeWcxZxTAl0Y0FbMi5ACSwhBSinplBFKKCnqyhAC+5TzsCjqPKvqWijpCOHOImOcUlZr4xx0vmNqLILAz6HHGFOGCaWYrAcwStFwTs/Ozlpx5FFrjPH29ujp06f+yHnAjYWMBso0TS0xgNYAStByueTD3v7+/nwxnU2vhqPt7a3+l5Ss5rlyUCnEMXMOejtkCCFjTMiKYULI2pZxuVwmSWL0y8ztP30wGAghp9OpD21JklhY+zHHyhhrNSFIKVPXtdKC8TAIAs6dc9BB6F2eiFsHR3Nj8CP4OiHRh2B//gHQ3t0BIeSh3aZpPLHD/46bwG2v5/H8tSSwv3SDFrhNGeGhbGStlrIJgiBOQq/499WVEIKz0MvkpKillE1dIQS0Qi+Onn/00YeddjoY9H7ww//9aDSaja+staP+4Nb+Qb/bG18uv/zisx//5GejwbAoVsvVROl6NBpmy5XcbrYGQ2Oc0spS6I8HE4IQQQ5Za42xu1t7WTZfLGYf/+bDVsKzfPbhR7+izG2NRsv5nBE87PcwJNmqsM4FjNy7f/unP/nRe++9M58vj5zFCEAAjFr7VWxuk78hLGGz51cgpU0Fv0EBV6sVQoARuqGw+PFk17DKtX3C9YWklPQAvx+a7a7dCeF1UWCt3rzXOefdG28WIv5ff2lt+mKb3G8d5kE4GAzu3X/t9p277Xa7aZqiKPwQ502Q0Vo7bXxbA14LgH0rbfPp4AZHyscrzrkjLEnS1SqP4xShKcZYNMqvczCBABBjjFICEepxvlar5We1gI3Qd20g67Qx3mYtL/LlculHx8m6DMMwCkPf+IeOAesQglHIoQNRK85Xyy+++AxY1e/39/Z3L8+fvjg7upxdYYYxRdrYNI07lEiti3z16MvPi2z5+eefcUa2tobtdls3ZUgRdqYuypA42mt//tmnSRi9//77zoB+u8MRz/JVsSieP36+WGXchE+ePLFa7uxsn5+PMYaUkNk0+3/+P/4lp0wpBQBijPXaPUr56ekpBPKLzz5XSsVxNBj28jwXon748LX93YPbt+8Bq6ezsVFmcjWeXI2bSkTdrdZwSAjxfD6tpYIUOVeWubRhrcDVWBweX9aKCxNPJtnyYozDeNQaMd6qa3hyOp0uRZK2ilxABDnFjFGHkaIWAosYthpQyqXUvllmjMqyrKrL7Z2tbrd9eXU2m82EQACALMvG4/F/+OOrw8NDTEjTSGttGCwhgdvbo9HWEEDT7cUPX7/dH7SVyaraIdIa9LeV0oRyRNjR0WVd1mVRAgsJQsQ5gxBGiCCEESYIYgQxD2BdC0IIJqRpGinKMLTercljwnW99i7HYD0qEELISMAixjl3ADQNRbAxxuVVba0FDmstV6tCSxWGMYJYCGWMQphEEYbreSrccSO1qiuRVyVTbrFYjsczKez0atJtd27vH+yMBlvbg4DRusxLCrZ7rx8eHmKMu71oPJ46gEOOz8/P/+Df/1uMeNJqs2BtXqS1JYSB6yFAdV178aFR6hphQ4SQ2tW1FMs8CxmXUmIaFkXhWfRlWVZVFUXJaLhNGUuSRGtTlmVZ1n7EEWMMoZdd8M1Nu2HMeVZdEAQQwqqqyrIMAx5SohG00IWcaklDSgJGlWgQcMBiKUSd58YqAEBd14PhNqUMI1aVzXg8rcoGIYwRhRBbq7U2WjsIIQTIWWsdpAGCGCKE1tMaMbiOTa4sy0G3M5lMuq177XZba8k5bbfbZVmGYdhIJ4QoilorhxENQ2ykYowlMV8uF2kcvP7wXt0UX372KX4fO6OMUc4AjCCgNAiCcrF018GXUoIx5pxTiv3at6qqTqej5MuZOnfv3rXWhmHIecAYi+PYj+1JkmS5XJZlSRhvt9vtdnu1Ws2Xq263GydJHKf+I6TRRV7leV4uy5vcAvty5ec2tYL/da7/arznTL/fr+u63W57mZy7Jo1vovammvnWPPTXldJubOuk5j+6aRrPYuv1eowxrWVVVU3TPHywhTGsquroxXPf1eKcQoin0+mvfvWr/b2tDz/6i7LKj49fGCUfPnz4+PHjP/j9fz+erB482Pv880fzxepnv/OzN9984+w8VLomBD99+qTb7R8cHPQ66el5bpAHZgjBhEBiLdDaGKUvri7aacwo/urJWdL6i1Y7nM9Ko9V0fHVxfh5Hra3RKOSJlifW6l6/defWASP4N7/+iz/6oz96/OjZdNpAAMKAlY3enMP/Bh2Hb243rxNjzGq1IgQFbD1BjVIK8Poa2DQdfKGwXvF7EwJs4bUGwPs6+n1fv157aGFTH7xSJfhlw8aKwB+J3wjDw+Hwtddeu3fvXpIkviMZxzG69t3aXJ8+LJMbMcc3Ljf7d9dNtM1NASHEmKZJi9Kxbwj6DOdBBbq2ATVSGkhZksaDwWCwPWScE4KU8kAM8CyHIAiyIve+Ulm+XGWLqi4cMFmWIegwAsYY2VQAAIoxAsAYAoEdDbZOT08/+MWfnxy9eP3118uy/OTzz05OTlZFaaytRWOc2x7u7e3taS1ns1kchAS4Qb9rpCiWq9nV5RtvvH6wf5Cm8fnpsdGNkfLpl4/b7fY//Af/sCiqqqgfffns8mqRl00at9rtfjmTVSUX08l0OpeyuXXr4OGD+0argCfZajWZTIaDLYIDrUC/1+Vs3jQyjuPVajWbzQghdVPO59M0jZVSsmkYI5zzLFs+ffqsLMu33nprd787Go0QwXmei0Y1TWMxBwAAHMzPTuO494Mf/Q4LO6fn5428JIRsHezFcQvjJAh71gbCvrj69NmiyOK4hR0nKACUI6IgtM5KQ4wWmFBCGSMU9QZdrVMHnbHqxfGhBfsO2sGoTwipRXU5vggini/F6cnlu++/9957dy4uLs7OTmhAv//97+7tb1uofvSjHzCOKEPz1XQ8ObkcH+337n7y+WdVI77/w5989/3vj7YPTk7PP/zkU6KlqpoaQhiliXC6zArjvNqKwpBpYxshEcJef6WUmQgRp70gCKwxQFgIkRLGAasFdBAZ5IyzzgEeBjTggAQhxJZdjyCra90UzjkICSXIQODWfWsUhmGSJGEYKpoVRVEX9cnhRRKlQRDuD4damtvDkWrUdrdPIb46vdre2h12D4CerPIXYafjBYTd0aAoiroSt2/vvXhxTDCgCAaIorAToTRN2zGK2/3WcrlcZEspJSeUEAwhsNYKIaCEspLz8bwoipDGCCKKPZPJGK0MQBghRvlsMn325OmDB6+/8fpbFxcXz549M0YxxijDCAPZGIQc55yxEAEthDAQIEaVsFGQwhBKKa2GGGNgsWxMg2gStYKQzSYXEQWdKGEEDZJYbA8//vjjMIwRZU0ljQUAIOciKYqqzBAAF5dnVuk4TsbjaZx0GmmkVMYCzKi1VqoGYMQYs1bxiBsLIEYsiLOyqut6MBhUVd1ub9Wi7rR78+mi304arSOCJhdZJxlejqdRnGrrnNHWaYgIoxhFrN/vX5yevP+d97QUPE3f/e73n372y68Oj97/zg/Qn/1GuEY2loagKIW2qB21EEKcxXleMRoSSAPK77/+8IPf/AUPAl+iRUGYKR1iTix4cOf+48ePD27d6feGvcHwLz78ECKclyULwjDhSRreubt797WDKGZNU1POppPZxflsvixFY23lIDIQORRiCZWlrr89mE6n+3cPNDSrPBNSQggxWlsga22UMFprHiI/p2o8Hm9tbbVarSdPnkgpkyQB12umTZLwAfRb880rtYK70cO+mRI25ePGuu4m6RUA4CBB2AEIjV2vCwmlCFPGg9liWVR13egkifxc1jhOWmlcFIVWwhc6kGBEGQmieVZ+9vjoj/70w6pqDs8k50Gato+v+NXF2a9+9Ushxd6tbRnigsi9N2/98Aff3znbi3jw+PMvnj09rnPBgLBaQAAi1jbGVFWV9FlRFIQx5ar+dj8rVrksb+/c0oz+4sMXEIEf/eh7z87kH/3J/5tzdmv/YFUXmLvtg24cRlEUlUL88Z980Ch9eLY6njUAgU6nWwBkjId8NhMNHFjzoL+9l79Zqf9XbvjahxFeEwKcc76E5ZzGYeSVSkqpgFPGGLgeBmHWdsvI/xVghDBGiK47X9ZY46wxSiqnIYWEQqIAtsZSCyDElZacc7CecGsppQBiv09prDHArM8EDdNWmqaD3tb29vbBwUGn3YEOAueiIGSMFUXhEHJGWWOttdARigNAMNDSOedlmeDaxMwzMHyJ4KtJ/0W01oQaIZskSd5+++2PPvrID6NRymAMrQEQQIxpGNI4isMgYTRU0jAKGQlDnlASAMeg4QSECAbdVogQODx6/viLLy8vL7SRi8XMWVvXAgCEMQaIOW2UgwYhrAlC6PBkDkB0Pq7G86Mvvjonv/cnWXHeGw729vYwAEZozmncbaOALcfz3VsHd+/cgdrKWn752eeqEYwFSYtYWM/mK+vqgKHTk8Ot7X4aJ5cXp6usmEwXTw6Pq0Y5QBarjAVhaKNeKxb5KoqiWSMZ4nWhEOJpzFthZ2e475EkhPB8Pu90OkXDOeeQJHYxrqp6OBrcu7MPrNVNOT4/M04XRfHpF5+fnJ6+/73vPnj4kEepM1gpS2EStGm7DdhicXl1xWmXYC0awHhr/+C7mGwnSbK/vw91/Qe//4c//um7YdoqK/Hu9947Gl9leTFZHu4f3B5stzFEUkIOIuBCpZQpp3UJwjDsDYdhgKtGhhEvS12W5cnJmbV6tZxbazqd1u39oRYZBa2AtoDCP//pbx+dPP9n//wXq2x5fPb03uv333z7rZ2dHRwlZSN//enZhx9+zDnfGnxxeXk56I9GB8W913/4+ts/vji/un3wFomiyBiTFXnZ1JhQRLCDUCkVhvHancPXpNooaJ2xDiMAgJIyz3PV1JyxMAics9ABbY1DjmKCIbLaOOCQA0mS+vBaFLmPtlJK54ySBiHEGIOQ+46ap5WtVitgLQQEI2wN1I2GEENApZCMMISpsxg6oJQRSnMedtAgjtaXvtKCEp4m2lpPWSeUBACQbFVp0zjQWCeOj1845xACQcCgdVI2dV1LIXy80EZqI41V1mkIsHNGCOnPACGYc85ZACE0Jp3NZk+ffWUN2Nvbq6rK0/jb7Taw0H/Hsiydc15C48+kDzfu2s7Zs44RxwZaiAELGeYEWOmgdchhAoeDHqW8KGtOUcBjTDkhpGimeZ4raeqyyvM8SWAURXEcNo3UGkGngfVaJm+o7wxw7ShphMjz3AEYBAGKQ6cVoVgrAa1jjFoj/PFIrZfLZdU0PsNhAFjAGQ4ZDZJWqrVM4hjdOojjeCXFfD6P47jb60dR0m63D27dGs9yRHEQxUoDopH3OwpDDiDVWvq4vLW19fjx4yzLIAAhCwEAhJAsy7TWW9s7k8mkrPJsXE3nM4gcAA44a6yr8nKVTYXOHFIHt3bCOPzkk0+yVTlb5E1tjEZKwqoUtWg8DTBNUy/cUEp1Op3J5cQLHxz4mh2CX435lZEnTyyXSy8i2LzM3WhU/7WkqM3ebsJO/slrGONbyPC+FUIIEUJI2Rhjer1er9fL83yxWOT5CiEUBAxAV1XVYrFwzgEH/YgBrawxFuOr5SJ79OUXaZKOht26UlGY/u3f/e/ff+97V1eTxXx1VpxYa7e2tpbLuZaqP+g24uoqW3V6XSGhJ23k2SppJXfu3CKE/Oaj35yfnxtjophEUdTtdtvt9vnRc0qZR8uMEnVdH+zt37p1589+8efnZxdlrRChnGOpXFVVDuJvPav/DZoRG4QJfL0tJaXM85wgbIxptVpJkmwoIJs3GmOAdT7R4uvDd8AA51f5FqwbVWpj3w4RcMZttA/oehQ7QgjANTaw6RRIKRkP+v3+/v7+g/tvRFGUpul1ClfGmLquN/2FDRxy3RnBCEFCESEkiuMojhFCG1sRvweMcZK246QVxamBGELIOR8MBjs7W8+fP/ffV4jaGUsIoQyHYZimaRzHfmJkFEVhGFLOKKGUkiAIeBR6WLqRdV3XCEEHTF6WRV2ljFNOKCcQQicNIJBzHvGgKIowDClbT7iWRjZNJYTgiUMIQYQscAHjYRRYa6fT6VePHh8dvnj86BHU9uLspMjyQb+/PdqKwpY1KMvKfLlqt1qtuN1ukTRt/fKDD6UyBkClbBim1kE7XzVlk/LWzs7OuvMSBu1Oqo0EBoQkjFuJ7zzO5/PFbOUhJUwxJgBhu7+zPRx0rZZGN4Ot4b/6l//6j//wj773g+9Pp9O8LIZbO2nQunP34VfHp8bIMAwHwwHnPMsyQMu43Zkvpt1+z59YrWXSirvdFqbg5Oh0uZqdnx/v37nf63Upj27f3vvq6WEYch5QyjDFhDJMCCIIW2tfnM/LMs/Kotay16sppZSB0KEwZM5Wi8Xk4uIMOofRbhJvhRE/P31eNMurGfrk8w8W2SJKMGKJQ2CxmE/mU+PceDp78uzwq6fPRCO3trbm87lvMNV1WRTZwcGdfr9PCCEYwDAMAUK1EGY9zA05BwghjAYQQuikFlpq7e14IYVNXQNjyzyHwMZByCmz1his/eVOMCYQOQcCzsOwVSxzAEBZltPpOM9zuDbht1EQI4TW9qjGKi2VAAghiyHnUcgDSjmEkELCKKWEF1kZRRElYVVKpVReisl4jjF++PpOGIbW6vliulwuJRfeWX3/9i6ECEEipXZIaQcgFtIu85UKw+tGrxBGKOu09/JDiHhDM+cMhA4h4NayYWstcE5UFRJCNLUoy5KxoGmaOEq9qZ8xpizLui6jIPU6K9/n9pvWutfrecnDhqngi/qiWBWiQMA6JbgCRtQhx0KWd+4e9Htt1cgXh8eqrrQWq8kcY3Lr4S0IIees1+tV1doDQ9QNJhQhBJSz0GOhGCIAncWYbmh91mgpGgihRSCJ47oQnJE4DvNlY60FCNa1aKVRq9XiYRSnbSEVDULMKA+iyWSSJFGSpvv7u0HAGSO9bvu1115rlltnl1c0TL7/Wz86u5pfjWfWOQvd7u72eDqzwARBoo3UorEQaK1ns9l0Oo3jGAKAASyKwqs5mqYpywIAl+f5fLkUVzJKE8ZY3KaMkVCTLJ9NJhMhq9PzkzDkTdPUwkipnMOYEogIgIQwnmclAIBSmue5F/T6Zpldu2C9bFFvWsJeiX7r1q0oip48eWKMCcPQQ8rgP1vRcDPhffMtzn3tc8HXJ/uB64zonHMQbAqFm7WCEMI647OUtev0EwRBlRdlWWot/SVtrPYWrdZa4KC30PD0ZGtrZ2HAEoy40RAzRilqGnV2ejW+OqubsqnyfrvVarXiJPytH35P1Pk//+f/yjhDECQINlWZtFIIw9/5+c9+9NOfXFycHx8fXY6vIIQB41qqbLkKGG+321EUyUZQStM4Zqza2ztI03S5WM3ntXYgCAGjAcLOaKekIAhvvuPfXOvhZrV3E+DZ/PVmoaCUEHXjBTiUUs7WHhX+KlqDTNcPXk7nWtsurUkt1w0N7VVz6x/OGP8niF5KQDeXhLneT7vd3t3bf+utt+7du0cQ96/ZGCr4ALK5ftx1N219YHrNTuCcx3GcJIlfg3mvhXXyw9jPa8AYWwe11pTi/f3dt99++8mTJ5RhSrHW2DljHcCYJUnU6bSSJOKcAuQQgQB5praEBAMMEAKYckJQXi4b3STtVlf0DTCU4ZSxcK0Y1EWW13VdC2OMipIoCAKrdVHlHvAgGEMCwyjkQUApJQiGYYgxKrJ8tVo5a6UQxSqjmERRxClrt9tRmkgFCEbW0kYiM6+rwtdhS06YcTAIIs7SIIql1MBiay0IzM7OFsYQAJAk0cZmDSLkfK8IAAeAVMrTXfNivrO7lS0XVZF/9zvvvfvWm3VVTMfjn//k57PZ7Pf+p39XlmVvMChWzT/9P/xfQGWDdm82mymh24hRyjUiUasz2N4eVTvdbhsB++jRF1eTS9eoq6uLzz//NJ9d1E0+m497W4O0nTigh6Pe2cV5GPJ2J2IMO2usk8ZiCDAAAIZO17IWJZaq7VAat6IYGAuDwIq6ruoiSkqCYKsjkpbgHJBo2abK0NlXLz4ACI722w6CLC+ErqqmRCV9/OzpBx/8hlD23e9+/+79+0T76hacnJz84R/+4ZMnz66urn75i1+Rum4AACEP0jSthcqKXCmNKBO1dAZAgLWUUkroAEIIOmiszGsBIeSUhiHnhKpGKC2nk0kthIOWhwGihFDaGw467TRq94qiyLPx+GouVdPtduM4hdBpbQAwzq2h3Q0Z0AGAAXOWGAGsNRo5Zwjgbnt7nxBS5cVqtZJSykY454Ig6g6Tu3cHo0E3SNqInld1UdeFK0vKsFmPGHA8IaGhSpna5IRGEDmlhZBW1o2UEgDHGKmqCkIMoPX6YGMVckAb6W37tJa+T+w5eqKRQWCUWuY8j+M4jtLBoJ+m6dXVlfc1C8PQuwVsjPP8fvzUDC+9U0o1TVNAZ6ziBAcMKk0AMJQyzlkShcQ5E2kEbu1tb61W+enpOcGslUS1UEmS3L931xg3ncxlLRblYjjYgs46Z511iBJ6zdjnAc/yHELYbrchAMvlEkIXsUHEMIn41taw3+scPtMAAGtcLRrgMmU0hFAplRX5Mi8QxpQWDsHh9lav3QkjfnJ0JGRtjLpz9+75ePFnf/4rZfGbb7333nfeP7ucIBIghDFhLKB1XTNKTSUwQYQQYOH5+fnl5WWv3YEQAoe8HAsBWFXF46+mvnZst5NaNNrpKE6VyngQx0kAaauusYOuaRoDnJJGKWMssA4iCyGEiBICUBwD6a9YCOu6vry87Pf7Puvf5BxACNcqP+e8tMTP+fQz8TyxcZOqwY2WwV+GK9xMdTfXwd+a/NwNAsSrO/x6obDJB4wxqcS6b02wtTbLsqZpOKGegOK1bY2orbVBEOR5brT1i0VnvdkJhBD2+4Orq6tlpjrdzkIU/9O/+bdffvHYGvHZ50/eeLj7uz//qTEuz8rVasU5b7WBWAFgFeM4z0VTF5zzXr97dPj8yy+/nE6nCEBKKEG0aMqmaQAAW6OdJEnGVxeDweDh/dccsPfv3nv86KvVahVFGCAmpKqrKghTGlApX04SB99AVjYP/msKiG9rXqx1jzdf8xJnQtAY5RXU664EdFEUSSlvkgfBtcWqturGnq+1owBvPDAYY5Q6Sr1xFgDo5XQSv0F0zfFCOAzDra2t27dv7+0f9Pv9KIrKvPH0Ah8hvYQHIbTxGrmuHdcCTkf5pl3lJ/lRSgm1TdMQSiGE/uMQpg4gY4EDpmkqxthwOHzn3bf+4N//z0opKZsw5EpKazVCgDEWhiHnHOHNjDctjbQAUG4dBABBQpCFdpGtFos5Iag36LIQY4zrVbYxP5VGCSXKslwptR/uGWeU1Y0SxhiOAsQJghYzKo1W1sRhrLWWslFaW23SKEYIeUfdVhKVZamsKcpSX6gkSZxxxlBjAEQYQmiUieOWUwpA5rTTjdXScsSiJGqnaRzHdVMopdJOHIWJMoox1u518zxfLefWWshI2E51nldV1Ru2s2IVJVGv3z4+Pp5eXX7/e9/53d/920ZJgumf/emfN0qnjb26XLRb/eVsifspIoGxuqylcUAqhQhmAXfApmmMoIvjMEnDRuQXl1effvIJNFUr7RgrlqtpVuTGoaquEHadbjuKGERGSSGlhMhRTCCEyYDgOFYSJxHt90GnpQgxzkglZ2Fso9Rt77UYY602RnDZSLF/n1tLrQG1OcWOdvrcAYxZuFgVQuaoRlVdWGDuP3zt57/zO9vb208++fD4+Go+WyKEL87Pv3r0uCzrs9NzsjXcPrs4r7Jl2mkzSiHASgniVCmNIAo6JxrprGWEetN7BEBV10EQDLq9dpqsVqvFfFoUOaKEAKi0qYvSQhDEEdAGAxgFkRI6YGEat6xN4iCGDopGWGsxhoQQSgila10vpbRW3DlntTXOzxuklHCCWRzHVVUtVtlqtQIA+MnNyuiPP3kURq3haLfXpw7Aqs7ruiyrXIgmz1dlUTRNI5WGGELrlBEUB0LUZamllEaqdTjwwQhSAKzWUspGKemckUJQyn0/whofvrF3NMtWeVU1CBUeGOCc9/u9TqethFssFlVV+e5glmXOOe8sJKUsisK3JHw7xjmXpGldV2FAooCGYUAg73VbrVZLa301vpC+D5fGdV1bqzGn5+enUmojVW846nXai/kKQQAtaJrGGoMAMABABwiGXqdFCINQQuiM1lZJil0rSTppFHM66rfeffudra0titFisZhPps6CvCyEEBYgaZ0/WoiQQ/VwOKSULrLVxbh69MXnYcCWyziO49OTk6+enwbp4+Hu7dHubtju1o0cj6cO6E63xTgRdUMIwWHo145RlACAiqJCALaSKI5DQlCR5XVdlnX18OHD5eWiqKu01ZpP55Ti197aBgBI1SAFGKMsDLrddpIk0/kCC4Ow1hYhyCHAxABDHcOsLMvlctnv9ufzuYcHOOd+LB72ccQzzDECADSyDsPQR948zz2h0qeEm8v9/2RC2tQf30xvr3AUNi975b2bx5tCYVMlIISSJIEIXOvXXxYZBCJjDFxTadQGPQqCoCprjDGEQChlrQUANU2TL0tKCUfMOScaVdXN06eH2aqkFJydXn36yaM333htb+/2ZLw4OTkbDlq5rOeLZbsdtxIehuGD118fDfr/9t/+2xfHR02l2u1Y+7Z3rabjmTMAWN3vd2ezhZQ6CILRaNTt9FarlTHOGOes9g+apuEcAoC+9dzCr7ce/qbxBr/5IgBi71wEmqZZLpeMsTBg3rb1JgK0ASGs09fHti4AEMIIAYQBIYRz55yf1+oYY0KIWqw9mDcKGkKpN1ROO929vb1bt27t7OxQxquqOj8/73dHG04iuFZ1bs7STUTBExgR5xtZry8g0LWbgh9N5/fgDaERQphAIQQhKEyi27dv37q1//Tp0zzPR1sDrbxXhwHAQQSg99CgGFEMCcYYUk54GHBOAQZlUzZNc3p+cnRyjBDgnBJCkiSuq1VjaiCtcw5TEKTcQOVqM80mXdDmnEetAFjHONZG1KJMHS+qsmrqNE0xxlZrgtCw28uyJQCgaRotZZiEkGCKaNJtc0KzVb5arYQQnIcAMmdhluVCQqVUyAOnDcbYamOU7rTat27vUUovx2dV5TqdVtxKhRatTjtJWohgBaw1DmJkCaqUglo3Tl9NZ7vbW4TRRZ7duvP2u9/9XhxH/+Jf/Iumqt/77vdm0wXEhDH26ceftjptFJF2wDAOAwSQlgQ4IZr5ZVlWq9nVqbFqMhkbJZ3Vsi6aOu+1k+GojzFaLpeNUFI7zCOEAMbIWqu1BNASiiilIeeEkPYQQtSDTkJXU6QCsqRUEyiXiytOQasNAHKcA0KKqqqaLOu2b2utlbJlrZQiVrFGAIQCRMx4cgJm07yYRyEd9tsY6dPjw+PD5+fn52VZx3Ea8SDkPAkTTjj5yU9+8qtf/erLR1/VlegP061RkBeV1Aoh5CxomgZ4ig50Wmi/8Aooi1kQh1HAWG5dU9X5Krtz547gsmxKoSTAKGYBtC5bro4uT4wxzoHRaAusW4BZURTb29uME18jQwidM84BKbVS0ocGznmSJL5zzDmvinI2m8zmYyGUr+UddEVdqrE7v5yMdmZBwBpppXbaIuMwxExbmJdNlmXGGAgxAMgBqHTjm4vGGOeMNlprbbVptVqUIsYxJgAii7BD2EFkN9RlAJ210DlDCKOUR3GotXYOYgyFEMcnL/b399966y1Rufl8Pp1OAQCdTqfVakEIO52O70H4dap3T/KPAQ4NMhxz6oCWhhBkLcjzYtjvXVxN5pNpEESE0PlseXR6QghL2pEvTZKkFTBqtLTWhpzKpgLI+/I76AwAGEEHINDGJq3UajWfT6Ezb7/++oP79zACBIKdnZ3333l7MBgUq+WjJ19Np3PCA1XXDkEAICI4bqVJp8uj0AEUBAFAcDydrBbzsqlbrYRSWonGWLS1u2sBfvLsWbvbVUafXZx+8MGvu91umqZG2aaqsR9s4UC32/Ugf8i4lkpLKZt6tVpVRdnrd+7v3n399QdVUx4ej6Vuslxbd/n9zv1aCpE7gCEChFIKMXEAd9r9RmopLXAQkwAArKRRSs0u5845r03yiy0fha+lbniTtuG1lzYAIIqi1Wo1Ho99/N00pDcr/g3D/K9GFMA38tkrQLe7wXt/5TU3Hr/EG/D1RilttVMvipOy4Zz3er12u10X5WKxyLIlhDCKosBxL5v0X805gBB2zutjqVaWccQ5tdZigsIwNM5GYQIsDAKWr+Z/9mefzSbzmCXboyFnSa+73RpEH330kWxKa0E6Gvz0x781GAystQThNIaEoHJVcs6BBfPpdDqdAqu3trZE3cxms73tHYxpFMRBEN2799rTJ8/LWiVJjB0qq6YqK3CtPr1ZYIGvV1rffPBftG3e+8ov9c3f0V3LFP04b38VFUXRNO0NIoWubQk8tGOtBXizT+ecgZB45isAABMIEYEAe6VMEDCtw0ZCrbW6lkw65wjlniLQHQyHw2EURVVVISG9PBgA4C8Adz0aG14TYP0hbc7J2nCMRS81vZg4hCGhGGOACQsCjDGi2jmHGQeYWIgizrTWEDmtZRyH77zzzvHxC23WRAoIN9IJC4BFCEMCAIIAAYAgJhBRpJ1WtaqaOsuWq2yZ56tG1kkSRXEIEOAhK8uyFKUzllJKOOEuAMhxziFDFlpIIADQYkcJadE0SVMAgBeTy7qp65pTxhFxxgRB4Izx3RPnHMI4SdM0SWbLRdXIKGl1Oh2M6Hg8eXZ4tLO17YxpJQlGCBmjZWOt1aNe2oq0NphAGmBIoHEaYsAjrqwKkng7CgCAZV1bhLuE9rdGi+W0u4MAp5eLZdrr//f/8B/v3jr4H/9v/9e/+PiTTtryFhech00tLqZXqyoPiivfYi7LQhlJCLFWC9U4YJb5SmvpgJFSAq06rdZr9+4NB92AR7UQAIDBYIRpUApZVhJCzywh0EGCGaWEMU4pbR20Ak6Qa6rsqshOpckpEJiYTlsjII3V1gIICog4C3ULWmfnCFlMAHQCWCoEWMyrJN0igE9nSyEBMHhn2E5jNhufn56fTadXhKJOp+UcFEIoJRRQs9mEDPujNx6+6RxcZVkUxXHaSpNqvlzUdaOdBs5BgCklnKxHJGupkihOwkhUtZUCQ9RKUq1kU9dKa9UI62w7TdudjtXm9Oi4WnhPsV6v13POzedzAlG31Wm325igDRFXijWyV4qlh+4pJoxggiB0FlizzBYXV+dlVRJCtDGYBozSRpYOJOcXEwc/xRgKWWktlRJC1pSSpmnKSksFnSPXVTbSuoLQS5mQddYY54feWqcBtH6Cs/dQw35KnnMAWISAc0gZJYWWUlKqwjDuD2hZVD6IlGV9cXExGAwwCAkh3jlca/9iuklazjnOuS+WPWvaauUkQAQppU2tHIM5QifHF2kULxf5Kq8abSnlPE62bt2SUjOsMcayEdaoIAis0k3V8CCoyiYMESLMGGOdAUZbRCCEjZIxDjGmnozy8LX7P/nxDxBwVpt+v9vtpFpJT70sy3Jra2u5XNZ1bSwIHEjS1mB7ZzQaYcJanXbTVEopP/KnaZow4g6CVV5hwqtanF1cjXb3zq/Gq9UKEjiZXBGCZKOkEEmYlHkRRdG923eEEAcHBwSi5XxxevzCKD2fLzlDaZreun3AA/b9H3zv+PyoKFWng9rt9mQ+9oUdYwSRMAgCzgLMaJ03WltjLSEcY2wt9N/CDz6I/v/M/WezLGl2Hoqt16Yvv+3xfdpPjwHHAAM3AAiC5AV5v0pUSJcR92dJ+iCFpNClFJdXAA0gggA5IAbDHgymu2em7enjty+fPl+39OGtqrP7zAxNBC+gjB07ateuqszKfHOZZz3rWXHsxwdba8fjcbEqdq53h/oSQn3dwVrrS5U7GuPOl+ze8l8OKvwil/aSx/pZmGGTpCLu7LzHEryf2HmpHcdlN0bEbxu2Gm4UO7xT8W0TjDHOqZRhC4pRlecLrSFKAuegbjpCSC/NLi4vBlk6TMVq0XzvL37wa9/+ld/57X/w3//j9PPTZ4zTn/zkJ8uVlpxlSfzjH78/n14a1SoDWmujIctkkqAHZpqmUcpkWda27Xq9Pj09tdogogfYWq2FCACBc2OtQ8IAXkhTvHTefm6s4CUp/2u33RW8vq/r12K3X2ctIRRgs0g8T9mT+MBZb6N2r6eUIt0e2Iad4BABke6gHUo9OY96WCJFaa1F2PRZWGspE151MUozIURd14gYJ+lgMIjjeLUodpDA9ZPDNjLeZBfB+CdhOwsNAIIgkEEoZEApBWIYF5xzQpnz4pFArEMgjjKglJZl6Zz7ylfeeffd77dNI+UmhY2TMIqDF8EHAIK11jJmHQEE2zRN1dRlmZdlSSjN+j2z7CgjXMrOdIIRJpgIBFq3aRqigBSyQa9t26rZKNsKFGHW89MrwjAMo6hq6rIsKRCKkK/XdVnFYSQY19ZorVfrNSL2hgNKsG3brN+7e/eVOO0ppYExbTEOQ921SZoISsDoYtVq1WrVzOdTpZS1JopDY1S9rFblOkhjKUOgEIQRocxRohHGB/uDwSDOeuvVgjMC1tbF8tMnT5+fnQZpdvOVu4N+9tOf/vRgsieyEBzpXFsscvPg1BiT5/l0MWu7LooiGUsgxIGtVMuEiOKAMhbG0dFkf9zvt0qXZWkNRlF0+9adtDf6/MnzxbK0Bh0S54BTRiknILziEdBREAaCKTRWN4VxLaAlzqRx6CyqTbMLUkDJBY+F6lrBgIBUjQLJOaPrGVJspZRoa6uwlw77g2Esoa1mtl0X67UQMsv6lPCubc5OTglhZVHyk5OTNE1ff/31Z8+f153yBsg5PD09RURngTPmpTYIYUbpqirGg6GUsshzcHY4HO7t7VFGfPdtEARpGNy6fXv/6HC5WpVVHobhVlE89Pw+KcPRaNS2tTWuM0rpzs/Y8H0K+/uTKIqSJPGUQACom7JpyXI5XyxmjLE0jY1VaRamaUqobWo4PT87vzwzVgEgYwTBGqOstX7JMR6QjXrAhvROtzNYvTEFAC+5I4RottuuQYMAbFulKYCw1gJSSmld18PhsCyq1Wq1tzcZDvtKmYcPH0ay7wuNlNLFYuE/1lcr/H3uqUaI6LHuKEzAIKcBOCTOUGDOQF3UXWuNQ8qFsk6ZbjhOUtq/ms4jIEQIpRqyEW6yXQcydM4AADBOnC/cWEs9PYfSsizjKByNRh7munPz1nCQobWc86IoTs4uTk6fX15erlYrJCCEIIw6Zynn/eHg7t27d1+5F8hosVoCuF6v50lzi/VqMOxNJpPHDy+sxaptEuwNR6Mf/PDdZb48Pj5cLBZpmq7NimrKOfX++/XXXz9fXhVF8fCzByfPnq+W1cEksxYYYw8ePDLOWGv/0X//+wcHB7OPT3pDORoN7ty50zRNWZZ123hWByCx4FcpEEqECKSIlTYWXdO1fDtn7/T56d7eXtu2+/v7jz9/fN0lbNFa59EdXznyq8JfJl/vv26I/7Ou6D8Bkv/sn9dd1PUo4WffsnMDnvRe17VSilIghJRlWRRFW9Vmq9pb17XSGwqeJ7rvklHnvPZJw1gThAQRg0BSyhwipXy5XA/6o7aqbWsIuA/my7psjg9uvvPO2/fv3b1/7+7F2bmU+Ze/8s6dO3f+/Z9/l3OepmmnLaXUuZoRWtfKOZcmGWOsLMs4jBBxvS7q+sGDTz9rW3Vydtq2LSU8z/POgpBBHMV11/6XnNv/NbafxRV259xaZ51/YOq6Xq/XmxfjCxhg57yV23wFQgHdriphvUb7LqyhlHJOOeeUZYjIuPQ9UMYY2FYz/cXyj4UMlFLr9TqN+x5LeAnP8Eya6yHUJpgg4DkBPjTx9Vz/yo0U+obmsokg9aYsBU3TUAp3796dTCZGK0JIIGQUBx7+ZIx5lrfws3AZ4LZxQ1lV1+XV1VWrWi7oaDQwpg3iMOslVVXl6xkAeO4kpVS1nVIKCFkul148Owkjn4LXbSPDwMFmQHFZlkqp8WgQcjGdTtM4SeOkauqmaE1j2rYVYeB5RYyRIJTa6Wcnz5eLFWMsTpP9yV5d5kkQSAbEGup0zSAO5MXFhXPOgh2kaVGVTdH4uYDD8ahulTYGwFrAIAqzXm80GStkd+6/3lZlWax7vR4Dg6Z77e03f+3Xv9229d//h3//4uKiaRop5U8++uDg4KDXFMzaGMxeEraCGLREdcgpJSSVUoZBZ8xssQCAIAyttUTK5XJJmZCxC4IojtOu67SyiASBUEqYEFIGQRBEgZRSViUNZRSkcRB2SVIZarlbAaLTQEAGnArqgDBCuHHgrI2kYjRA4MUKhOQYRFNZMjSSEg6WWCc4CE66etXWFG07Go2qquKcjoaT2WxxfnYhZTgaTfi73//e66+9OV3Mu05LxucXU6CcahCOrdfLJIpu7u/3+qlg1BlVoZ7c3EPbTS+nXdeFYbxYmiSMbt+4OZ3OBOOHh8d7e3uqaecXS72ssjZa6RpRLy4upqen3WZSJVmvpoRikkRMsLZrlGqzLDs6nozHY7Dc22frdNvWbVt3Wlmr63x9tLdPCHEOsqR3dHwrSZJevP/8PEdEh0awENEBcQSc5KGUHMBZVMZpRIu4nRJridM2EBKNLfMmFDIJoqJYjwbDhElqTC8MLqu1EMzYRreKhD1kBNGBsYgoAomOAAJjrGu6LOkxIlVrBWA/6Usql8spIaMgYJPJZDy+JyW/uroiBBljcRxLKWfTxcX5lZSybVS/NySMKWW1rWXA28aOB8OiWC6rWiZJpy3n/PL8bDwcmjJ32rx+Y18De/ToEVJ2Pp3df/V1kUbtVPUChhEYARZti8YCSsZpEDLGXLkWnKeJCCm8+dorb929J8B5axEFYmXazx9+8smDj56dPh/u7Qc9aQrSWB3FiQyDg4ODGzdulHnRCRXKwMZxHMcikFGaUEGDLLGM9Ia9VrcXFxf379///NPPj/ZuXJ5OQcEwHj/67MmdO3fWi7PDg357epH0+9lkPF1Mz54+P3l2qpQZjvoO6XA04pxrW8zn1cXVPPyT7751/y2O7PHjp1/+jbfeevX+usg/+eTjuB+3ur26ukrTWJta8IAw6PX6QRDnZVvUVdU2Zd1GNF3lRa/XmxzeuDy/uHfvXtOa/cNjxljXdUZpSkkch4CotbbWCJHcvHmzLEs/9dU7Y48r7EwwuVac3jlguDZV0if6L/mb/7Rz2kbGL5T4/B4FB0BKPBMOGEGOlltN81WTpiSOw0AEWqv1cpVTJIQY3XpT7pw1VjuLiOisaWqVpj3/yVkvqeva2jZOWdfGWiMQW+SKEEIp153xvodwYoijlICgl+36z97/y1Kad27f+sqX3vjTf/uvKXbr5dPBiPwP/8M/evcHh3/2Z//+ydOyl4XWYlW2R3u3zq+uiKNNV8Vx1LRahMm60XvJSGt9Mp8ta4csRIqARHK0gI2uCX35dO3CMntt2uoXPTr92bjqPxltuJfAG7+5n9HDeLEvIL4mQoA5y/Ncd2pFWZRlSRDKtm06ZYAYwQUhQDXfYlT2i7tAX04FAEoJIQ7AOofGVYwKgmispZQJyZiXc2XgrYQQApB4cTYAaky7PSee0sGvw12UUgKMALN0IztNKGeUCSaEEFJwgmC1QUYoIUYrAlYIEQYBIQTRWKPAEs6oZ2tNJvsXFxdZlpVlT0opA+FZ2F5WyNfvCEFrteSZ4LStKt/GtV6tEFF1XdNUQHE0Gmmj8uUK0frmZ0Ts9zPOAyQuTqPRXv/8/Hww6SVJ5GN05wyllDAbCrFaLxCsiGVR5dNi1c96PI2FEKu2WS1XQRCAdSGL9gd79byM++Mo2nMAjKSXZ88//PijNE0Xs3kUBa+/+sr+aCQ4lYIAZ6O9STYea2zqthEBX6zmXWuMcVna7zq9XBTKGiCEhZIKRoWpbXW5slxGRbfknJOI5tY4Z4Mkxjj8aL0Ca8DYYDS+M97rZUm+XJ2cnFxd5YwxQlhVYFWZOEq5DJbT4tnJMwf4D/7R7w9jqeEkiMKL6UWnle3KIOtLGShnr5bTvK5W6ysZYlHkcZLxQBrTGtOhjBkjhiDIvUI5VThmqMEQIQAuQkGtWnPWcQ6cgHGAjsY85iJUtDC60x3s7YPuqqZsj8dBVVnbNvnc8YAN+gcORV4TJsJl3TYGIUgWnW3zgoUxzQZ53UDX8e9///uXl5eUS22NFGEQRVyGxpg33nij65pBLxv0+saqrqnByTAMW2NX69yT9dI0bZqmabrhENI0CUQQx6HW+urq6vLysmmartOD4YAQgmC11oi2sbbrOmxtkAQAVDBBYhKGYZYladKLo9Rq7LrOo9zGKqV0WZdN02htqqpCwrKsH8RJFEVhEJnYDkbCtyFthr47ba1CsJ3SAIhA0AdmG1sPCIZTXuUlJeRgst/Ule7UzYOjd770pTgOOaeXF+doQYTSiy1WhjnnZ8PszM119Nh3e260E3x4nue5D5Ynk4kvozDG8jz3nf1ZlnlRSK21b0xl3Ccizpc+vVmYTqeU0slk3yitmvaqmkkhhsOxVUp65jMCse5gvNdULSGUOSDWEUAOhDEmCLXaGKXTNNVK6a6L4rjf7x8cHw2HY8Go7erZcvX82elyuWSMjUaTyd7eaDB89vgsSZLRcJIkGWOsrWplDaW867q8zIuiMEp7Sp0xpigKpVSe50EQKKUePnyYpmmv13v+/Hma9Pb39y8vL8fj8XQ6jaJoPB4vFovvfe97s9msLKsgCAjZ+QM4PNoj6JyNu7Zw0L1y/+bF5dPZ4jRgb4Rc3jq+MZvNCMgbB4fG2aw/tNYVZb2aL6xZKI2IVBIWCV6uy7ZtiUPfWmKMQmdG48H08txP2dgp3yGi/yLWWq/DsRMO8fnZdS9yHW3220s+xl1Tu/tZwPyl4sVLoPf1qgSlGxB7B074cCTP865r8pwBcYiOcxonHqONdmQ3usGzEQD83FvnlcQI8RUW5xylgMwi+klCG7QcAHzYRBkAgFKuKKrz88vHjx+/ffumQ7xx48bJybOr2eJ73/ver/zKr/zTf/pPF4u8qt9VnUOANIvPr04B2NHB/rKwANB1XdU2uB3NtUFNtp7bu7ntmf2Ct/4vrPL8r7dtr8XmT9/6qDX4NIsyPxmBIvjQEPkvaIch17iou9+EEN/mtVtjQgghA8aEX1R+d4AOKCGECMF/0XG+CBTopurkn2d8q9cuBOe7QaObt1z/jv4B2c6U8l2UVVVZa+VmE96Le0m6HY4CW5DVaEcRvQ5VVVWeJsk4IwQRHKIFoIyD79H17e7+jgvD8Gtf+1oQBGEod1IQ/l/dum2axpsXv2KVUs7YsiwFYz7dIg4ppR4sKdYrP/dVtXXbtpKL8XCUxklTFUbp9XodB5wlkTGm6NqL89PBXj8OI0Hlcp23besIlWEoZXhyfsY4T3pZL4lkEBgwsBVG83c3IeD9CzpfmGbOWY0Ouq6oq0Byznm/319kKWNMMknDKB6ODvYOB4NR26nLP/xDKngYJzIM4riHxFVVN5vN9o5jzr1whS7yysaotVVKUcJhdzuQnYNA1bQMBfej4iw4B9razmnJJWFGcMcFlcCd5dYway0QoASEAMFFKJNQSikhbdhPPjwXAiaTOIlxVeadwoilUuq6ccY4ra3WinNRN2XdtIQAF4LOZlfj/T1rnQIYTcaEcQCXpnGvl46Hg1AGV9OLoigoOEpp3SjOZRQliKSua6WMjCVjbDwed02X53nbXl2dXxRFRSlFdHVVbfR/GOWcRxAyQpTRxBKKwJjwS1zKEJEoZVRrfEt917WIyATt9QbD4bioqk6fN03Xdrpp27KqCRUGQQSht0fMBtR0WmvUzDnjLHHOWOcQCXgMhzHKADsdRaFSJQekMatWhdX6q2+++du//puTvdFsNnv46WeRDAMeWEDJg8ZtWp+3NxghhFBCfXCAzlFKhWAEiJ8JKcPQQ8F+vErXad/Q0bbKGOPvJQ8jbxFL5JwiWICN1tuuJh2G4WAwAIfFap3neRxFQRAU6zKknBBKjWuLan80FlScX05LWhHjnNUUgAciEAFhfqg8t9RYax2AsUgoVc46sHXTPPj88w9+8uPPnzxeLtcOECzUdWuMSaJ40O8zKvI8f/z4MSEk7Q+MMXm+ms/nbdsKxpFz3an1cuVbOXq9XlmWn3/+6Pbt237CLCJGUXR2djYcjrxYwu3btx8/fjybLZyDfr8fBAEimk4RikIScLqo1oR0eX4VR69/5avvLGZPq/zyu3/yb7XWWb9XFEUcx4Kxy7PLTKafP3hIqOw6XdWdFGGSDdqqWlxcySCVlKiu0VoLTtumKdf53t6eJ/ft/JZ1DgB8wtR1nYf0EXE3R4dt50hd9+W4Ha6zcwy7IOALRdyf1+9w3cTDF+MAuJbpMkY9gk2uSUd7HNtaa612aJyzQjBCkVI6GQ92wAZcE+sNw8hbWw+B+Eb2rusYI4iADAAowIvRAwDgVzKl4BkOy+Xy+fPTn3z04SuvvPJ7/+Af/us/+len5yf/5k/+tNcfEi5++3d+Z//g+Ed//ePzyx8LwftZImU4W5wj2bSVIgFKqZDMd6siWiSwjYFwt+vdefjZs/e3tSHiizAGUWvt0PgZEIwj54xS6tA3CjopNqQW8gUixQudjJc2TjmjL5glfvNVAKAbxiIBSoD66gBa84sO0u+Lkhe9OZRSyrYjKjZWZPNNri/aLwQKhBhjKaVCiLZt5/O5bxTyS92LL1G66aMJgmBd59bapukopV2rHSHW2vV6rY1BcIwR5iiAM1Y7ZwjBIAzCULut9jmlXEqeJr0kzoRknpa4G4dhjAFHnIVGdVprQMq54FwCxbZVBBiiUcqgscY4wYM06a3OzlqlGsSqqtq6TKM4jWLByPH+XigZozBIkvGoR9Bp1UZBqLqWBQEAMELjOOZBSJnQxsGWNOrDAmc3MBvj4ACopYx/Ybl6qaEN9mWt0ppzlvSy5K03rcG2bqrL2bos7CqvLFVKvfalry2Xy+cnF1ez6Xw1v3HjRhb1wsNo1ZxRISnlXpI/iqKiKIzdEDbRe85tnYhySp1DYzVY5wxYoCApCylLCDOEWofaWaSUAFAAQGuRgp/V4nv7QLowJoRhksJRAOO9MIi6vFlTdA47BJ1l+9ai6owQPAxiwdl6xSjh/Du/9RsPHz6M46hT2hhnrVaqI4R4cSTBqJJBnud5nqO1AK6ozd07dwghJycnVbXOssznLgcHB6fPT6bTaV03zrnJZOR18U5nM78cBQjOGZW+68MQwtAR1IjEaWetRd3Z5XxtOyQELaCxlnPa7w9v3765f3R4dXV1cXV5eTWbzxfrvCJiZYChI3VnCSFAHAIi5cCQIIClnHDrNJoNR9I5QMIoUHAdY8GgPwGj0UIaJ4eTyTtvvd1P0ttHNwTjSZJFYUIYs51mjAPY3c25CeIJo8AQCTp0DgEIpZQg+Nlyfk5xFEVa67OzMy8VHsdxr9fzjqqpa0T05UNERGcoBWsMInXOaN1FoeScj8djrfXV5Ywztr9/OBpNmG+EM05QprUGZlfzBePicLKHFrI4K+p6nZedVsQ4ypExwiirqioKw0AGURRraywQIFSGoba2NbbqlFdXFUxIKdHYLE58hX5dFUVRIpIgDHu9AWOsUc16tdKd8omoV9L0rhcRV6tVnufz+TxJkslkYg0WReFHM3gu2HA4/OlPf+oAKWeEEA8aOWOEYIwKTez+3jAOxTqfzq+e54vJt775Fc75w89PhkmW5/mjTz6TIojS7PLqanG+ePa8vnt/DMjy6Wo4HO/dGgyTTFdNo5BHom1bcMAYb8piNr2cjIeCU84I2muDdynlQlBK27ZVSu0sqb9Fr1/0lwz0S+5/5xheMuL4RWH/n/uZ/vnrYYdfYrsnfYqJm8KEJcSnEs45QxkQQtokvN7MuYEUEH2WSbZdc7AtlFBKKCXOOUK9297k94hICCPAGKNSOmuNUma1yn/43vv90fjg+MaXv/p3yqb59LPz/88f/Ksf/+Tjt99++5133mk6+4Mf/vj8YioFBCGv6lWUhJxJxph2Vqm2KNbWotYdokWgG+oiet/28gn5Ww8RdtsO2PBUa2O9IgIXkkRRCOAQvH4Xvd7hef2B73GllBHix+jwTaDAJaOCckavSXc754QQuF0G1z7kFwpWXwtgv7AOyRc/ZLftame7t+8ebCPLsCzL6XTqnPOmiTIC4DtyKXg6BUCv16uqancAZovMMc4RHIBD4hA372KMaNP5GMZtoRRfiXdeJMdZIM439xhjuk7Vi03IDgCciyAICVDGaZKkkQwaWhNCOqOMzwERKIBumrKurbVWmyyNtWpXi2UaBq2BMBCMAqdUctlL4iQKnp+fNKZ2QKIo6o0SIKysm7wt++MRYZQy5sFgAOfjJG0VEuCcMy4F50CZEEJKDmhDwTljxDnOGFKCAFyIuH/UdbqdzefN8wdPn1gNIQuapjnaP6zrWlm3XC6100oZBNK11jlalV0UZmnac4B5XtZ1G4QhIcQ5IM55xolPYCil1KJpGgSDpgTTcuJizozgBATlUqPtug7RcGooEYCcUe4ArEGHDl1njDIGraXHx6QzTAbOQpEkOnDEYVtVa3QxZ4JwFMz1Ehly5jpdVx3/zd/8VS8JKSRljBmruk5nWR8RfabFCPVDelTbdl2nlNbKCCEEl0lMfJTgQ9HVauWxaEYI41RI7tDujcY74XGk6Iyx2hhlgiBwBrtGeRYrIiJa55wQIec8DIMojcbj4dHR4d7B8Wg0ipPsxu278+Xyk08+e/joibGoHaGUKtQ7iRqkBBkDQEIJAwArHBGUGPAhP6WEQdznQGjWTwXBkNK7X/vqV95+69bR0Xw+7/f7q1WuOuOAcyoJpzJO3arx2RrdtCQxHygAUN1pax0hzN+APir3QJxP5tbrtceRtNaTyb7v87TmsigKuoWJtdVcSNVpBObfmyYBISRJkjwvL8/OR4NxdONGFCWBkJeXU08sWjYtISTPc87F4VF24/Ao7Q/Ozi8ePXk2W8yV1g02TPnOIAcBdQ5kEHEZpv0BDYRj5Hw2W5Y5kyLOekXTMsaDIKSU3bx5s23bq6vZbLZYLdedsVmWLecrLgUhqLW2YCkjVhtOiZfhyrLMi+RnWVaWZZ7nd+/effb0ZL1eDwaD1WrNOb9//36e54vFwl9lba3VBgBCyQPBGaFKt02FV+drZwD1071h7+/93u86Z/rxMIqiH/31++PeEB1xBvb7+xdX0y+/deuXfunrq3Xxl+t3ddOiMgxIwPgsv0rTlDgjGWWMlk1ZVaVzfugXs8zufLa34ADgeyn9pC5/lXfKu9dNtn+865SDa0HDzkNfN80vvealjVyrR/ysiX/J3HvH78s0UkrGKGMkzeI0TZfLpbWWMbarQWyZ9pu4R0qJSDxYRQhB4mUBgRAEgoDUH90Wk3CI3Dp0Dpu6Wy7Wkrrvff/d4ah/9+7dL1XlxdXsk88eUS467VZ59eTJk6Pjcb4uj46O8+UKHXgmGiIapVerpb8pyEYQ0++CAXnBD/j/Q0Th2jF4IQHnk4C2bauKIDpKgVDLOReC73og/EKAa1eTUurjNELotZ+fsyo2i4FuEAhAAnSrCPsLtl2ry0bd9mc+0C+BDQBGt82TZANibVcX7uAoSmld14vFwjv+MAy1Ubvl7WtYbdu6plgsFohbIS9KvUXiQmij2rbrdEsIcsEYI845pYyzAEC2e2Ge20027EsE2HRgWlt6/Wwf1/pjoJQqpSghXvSJAnGel6u1J+c6azljaRS1nRKMB1HYNS1Dt5hOJaNyPARn6qIwjI6G/VAGsQw6oznlIoodZWXTVl3rANBZSsA6Z4xB5xgjURAC46prDTrGWOhsEATA/Ak0geR2c/Og09o5wykjhDSFZZQ6zoAL5VArhYIY656ePGeMWUBHIIoS68CT2eNx0irV742Ojo7qtn7+/Dk64iNL5xxYuyuVKqWIs11XUoIcLKIFjZ0xla0Bl8cHnPNACIqITikLSCgS4oAIII5ScM4BtZQRGQDlRGmMeSBE0ChiTdBorFvDCSADwUlnjdGts50UMgxYWzveH6S9fnw5u+QiCMPYWs04CQLh3Ka2RDYXKaEA1tp+f7hcrimlg8EgTdOiKFartfcTaN14PDZWNWVlrfayRU3dbtp/GRVCSM5JmnrY2Tls280ISovOI/NZgp5SMN7fOzy4ORqOlDJPn532+v0wiQ+PblWdW1dtVTcOCGGMR9eo5gjEMuqIz56Ilz7hHK6VlrOAl/m6MchDMdnf+9ovff31V+6U63Ucx5ez6dn5edG0GtEhRRGAiBFxe9P5W4v6CJ5sCWh0o3mCPhkliG3b7ojo3v1Mp9Ou04eHhwcHB9agz8W11kEQdFZvW/LQoW272rnEl9IJIcY4ROxaDUhHg3GatI3VYRiui1wI0SrTdp0yOk57URyLMBCB5FJYQgGIV0wLeIjWzud5FAXL9Xq+Wj58sshXi4cPP18sFgTBIXadkhI4Zc7gcDhcLBbFOi/zQinlLKKxhSoIITLgCEAZEKRoLJGbhDWKIn8Rh8Ph5eXler1+9dVXhbhM05QQMhgMiqJ44403PvnkE6UUIR4ZIwSoECIOQnC2axvBCCOslyRRyOuy+Oinn7795pcuz08Xq3KxWDgk/7v/7T+hLPiX//Jfn5xdUuSRTOKopxUGQlZNfXF2ppRaLpdgNEEbCIaIzlqwhjMiOHXGcMosY9u+NcGl8ObYqyYEQbAjLvg06+ca6N3zO5zgJTe/gxD8y3Yp/s9GDOTatnMwzjkC5KXPJIQEQaDUJtFhjO6K/kEQ7Bo0rLVdpxBRSpkkmTfNQgjnYNfd03UNokVwiOAZcoQQAOJj3I1BRwpAldJ5XnDhNOJ0teiPJ4c373z1a9/47LNPHj85UcodHh7euXNrOOzPLq++853vPHz48Ic//OF00fgGqCRJvCtSqrPWbmMFj2EAAm77CV8GbH6Ra/yb2XbXDr9YgAAApVRdoyeIUOaCIOCcWeuuX9yfvXAv/bbWIiGUvACifBuU39EuUPDjJ51zFH7+titbAL7MjdhGCZQxugsUtoHjS7Uw3FkqAKjruizLXQi74/N6OUjfkXG1nPrXSykFD3gQ+Fbw84sL78yss2EooyhEtG1bu92IS19QowwRldJxHHO+YUV0XeNDhPV6HdPYx5oAlHOJiNooyQUlXGu7WuV1XddFSSnFA8K5LNerKIqO9o+qtqvr2ll0yhwdHlR5IRgd9rLJYGR166xGq41qKZBekiLjtdLLVV61neU8iMJ1XvJACsqMMVopgs60XUWpI1xbQyntulCGwUY2g0G/lyJa4hDBMUIlp4GQnkIRZv3xaLiYDPtZVrgyDmMInFEGAJqm1tZoq2BNgkBkgz6SVuuGEBZFiXFICLXWtW3X6wW+Cww38h4a0YKjHAMpZBymgQRnkrqAfJXXldGTGCnnIuSUWd4ygmhBa62abrO0gBDCuCBCCECBjibJHmXZYm2K9bpclGVlHcqkF0oZmM6Udd2IIkn6ggGnjnddc3Cw//jpU+s0IVjXNTC2Wi8CGfmj9FSvtm3RAuc8TkdtUznngiCilFdVtVosmia6ceNGnCSDYY9SWhe5v3OqqrqcLnYKowQhiMI4iElI6q51zinVtK0vYaJXHUBTZlkWBnE/7fmezK5TTduyICyaNkzSrN8bDEfr8mRVVlmWEQk7HWjnAAkCEoKUWEo5gmMUN5Uev7XWsShpyzW3tD8Zh2mSV2XT1KPR4JNPPnnw+Mm6qrQjCMQgydtuhy4S52NJz29yjL0Q4kVEa4yvWuGWGE+28sB+inFRbCTNkzjzLL/VaoWIhCDn1Gu/W6vb1vomQD/ZXQgRhrHW2jkMZJimaVHMh/uTk8tz4EwE0ild1rUBssiL2XyhjJZRyENiLRrnAIATCgh13c7nywcPHvzovQ8+/fTj07Pn4GwYhrdu3AgJ4ZyHYZxlfSnlarVaLBY+spFcIiNSyjYvvSNB9CNvtpP3tiZSSqm13s2t8a3nPkQAIIPBgDF2fn7edV3TNaFgAMQas3HJxrVVt3/r+Mtvv/Xrv/btxfzqD/6X//nhg/l7P/jJdHp5vri4uoJv/8prBwdHnTKD/ujB58+u5upy/skqrxCgKusgCtu2JYTcuXePggvDUFtcr9fz+QLQBZwRh9ZaKQVQ0rUaAKIk9kStrml3vBDPRfVY386Y/lwf/4u2n33ZdWBg9wK/Nug1mttuR9ZauiXe7iw+IcQb9CAIsl4CgE1TefrY/t7In0a2jYGklH6ET57n/s5yzvnyCmPMOeOcL2QQgBe+cFc64ZwTgs5xIE5re3F1KcMgSuInT5996UtfeuOttwlnf/Hd7+X5g7/zd/7OP/7H/+jZsyff/9739vZH48ng1u3jv/rrzz777LPVaiUDHoYhOtI0jb8DtyfAq6ACguexspdCq7/dWGGH3Ow2QgiiVzk0QAwA+kABESklAsT1KIFQ6sMAdEgoRUfAz4zxP+AxhZe3DVq5vQSI26oypeB+fsD6gtoCXwhYd5/whfAUd9cXv3i2ffPF5vXe7PgboW3btmsJIW3bUgpRFA2Hw8lkcjY9r6qq67QxhhJOhfAcbSGEtBIxlqHY2xunWVLX5Wx2NZstAAARvDNxzpXFZrpKmqaIrqqq+XzedZ0HvWQmPTNaKdW2rbVWd4rGSVEUnNLlcllVlW67MAw9/2a9WhAcyQNBY8oJreu6tCYJe7ZtCdpAyiSNu8pVebOcL+bTqXFuMBohw+Vy2RqXDQZUBkXdICWUM89RbdsWjO6AICKVodIaKEmMNsYQBga99hQ4qwEglCIOI0qpRWeN25dynCWSycswsqrJF9OG5wQpInpWe5YlTLJWVcbxMA6sNut18fTp8/W6qJtG6ZbAZrKo0ZYKYIwFgdj0uHIWs0EchGkWp3HEmG7K3moVFEUgwxqhbtoSbSsZkUHgjDWdgi0n2yECGk4IIRSB9Xs9rWA6XZxfNldX7XzetS2EQUJoS1OOFozSqtWhNEZ3bVPx/f39b3wjXObr1ToPZEKXKwes63TTNLBV+/KSgoKyKIq4DEbDoTGmKNaLxWKxWHVN42OCpi7brg6CgAF68lRRFMfHx0qppmn8UmiqllA/ITdptUJEo6xWFv3kX2XbpsiyrNcbxFFqtO26LoiitN8TYXB+caWsi5M0jJNWGdeqOE6VbQEA3W7pE0IZAZBCeKfuHDoLm/Zn54r1+vjoEJyRnA33JpSzum1G49E6X330yccPHj9ZrJYKLQDRiHXdxJQSskHtrLWIzg/l8W1a/ma2Fv0gDQCo25ZstChcVVUAm4yBMXF2doaIb77x9ng87rru6upqO9WN7lBBX6fwyfF6vVZN2/U7ikAp01orZZRScZIwzpEAF8ISUrdN0bTK2LKqDbowiiyQpu66rjXGxCKM4zgMQwC4nM4fPnn84NHDi4szwcje3t7R0REAKKND55xzqu2eP3u2Xq/n07nYjMpkhBClVBRFCL6cCZxv5F98CNh1nRea9Ig9pXQ6nQoeeO87m82/9KUvTafTtm2rqgoikFHIgDjhoiga9oYMSR0nUZg+fvzs1Vdeq8pyf+9ob7w/6O9//smj+QIOD+mNGzf+53/+z9vWfPOXf3VyePOP/+jf5FXVdrooCm1dSEhelkmS3Lp94/jgEBHLsnz69Pl8PncOCIBSCoiTMiKMGe388ouiyFrbNZvBWrCtOBBC2Hac9I7E+nPxgN2D634OfoYsdv3Jl+KGXRywy2IR0eHLik+U0jRNg0Ds70+Ojg84Z8vlPC9WzrmqXPsIwJdUpQj6/f5kMinLGrdgiQdpfZfNtpEPEIDAi3y16zovM+rZdb7f01qrjTk7v2RcChEMh+Ou615/7Y31Kn/44LN/8yd/cni43+uld+/eCQLRNE2SBv/kn/xv/uAP/uA//IfvrddrrT1attF1APBf0wFcI238bbY4/JztJXe7ceGbHgdHqO9EoGD9ncu8EPh/ettdXEKIlJIA2whFb9msvtEAt5ASIPEENEqp+wWBgr/oiEi32JJfKsxtOHf+gDfkFQpetuELZx7Alx7YduL2juzii8Ubd2VMUax9KDAcDkej0XK5dK50znWq0VXVdV0YhpSxTrWIVoZ9L4TTdY3vPMJt669zDlEppZqGX11d7e3thWG4Wq1Wq5WvYI5Go2E43D2Z67Jpmq5tVU/XdT3qD4IgSpLMKm2tVZ25upzptitguVoNhRCccsZY0zR1WaVpqtpacpFGsSBgmqYs1qvVUqMDSg2Q1XLV3z+4f/9+rc3VJ58A3eA93gIzdJwLBqTqOq01oVRLIVxA6Yu73hdHhv1er9fjlPihSGA0dZZSZ3RX5uvZ1aVkkjERhiEQRwihnAgmmraKkySOh/PFrMirJ+pZ1z1Qprt161bSS4BuYinCqZeP84VFKniEPclFEkVpHMcRc71gNBJtE8+mnzrbrMpSt0WWkEAMGKWEkLSXoHXWaauNMWAtWtQEmSX26dOTD39qLi+BABgDiJSiSJDGYRKKJAqSfn+YJj0KrC4b3poqyoJvfvPr77/3k5PTaSQzzsKOKp/ICsYpJ2Es1oVqVJ2IZBCNRsO4qovTk/OyLA73B4yl8/m0a6YAYFSgtabArHWU8jt37lAmmqYJBLcmsUCcBaWMMto5qlvbNgYRpWDOGUC0HOdFldYL5ZogC+J+6gBbrXUDXVlHWVbWleuaRqvBqH8xvWpt0646D9/5FqMN29MBInDGGAsRiLbaOOPz/yAqTblKgmDcH9SNXddE0vDi4fT9996bzbpVjkrJUAgGnFGQYJYzPD7eXywWlAGlvGqrNI3btlXWhqFEhKZSujOSCUopWvBju33ZO45jQkjTNGEY1nU7HA6bpvnss8+Oj48Hg8Frr732/PlzLkWZNwGPVeuyaOScQxcCSdcrl8T76MpWs/5oHEn54WePb926dXhw8+zk8vbtu6vVqnFNEslVnlNKBSXgGtt0hlBroOsUdySWkQHM87xr1f7+PiEkIAl2bK9/I5QyixPTstlseevotbLMF7Pi1VdfffcHP1oul4EfsClZ16pu1aZpSDixxoZh2ClV1RoAAhkdH95fL/Pp2cwaF9BQVd3BaL+u6lY2N+4fP378WFIWcv5rv/zLf/RHf3T27Gx/f9S6vGmqgIterxfHqXZKIYokyFUFDv/Zv/gXfjq21vqDB89VZ6IkC6Lxuz989Oqrr/7ab3zr69/65tnpxZ/+2V+wRj1+epElPI5TySNwrFzWi6tiuHdzvV4vFqtHp5dlhyyMi9a0xjnHm1Z3nXYOJpPx/v5h27YX84uyLPv9vnfVSZJ4RMTbNU9Hd9ttF0PsUNmdIwcAxpiX1vZgr/8v59yjLL7rdQckwpZchtcaCP0yRkcREZBSIJwyzjgAOGvQmvFo75V7t8eTIaVwfDxaLufPnj1zmkkutdaqLSmlcRwTUHW5Xi1WaFUcSnSEOJKESW0b2zlCEkIsIRbRIRAAQOIQMMkibyJbpQUKxhgSdA7jSFpdxyE7PholCXz44Q/u3bn11ls3xmP2wfvv/7/+3/+3O3fuvPHqGx9++OHHH3/KGPvv/tHB7/7ur//gB3+udGcUZsnodHo+2tunQJqm7TQyJgkAGgRKhRBG1756Yq3xPZxaGw9Q7fzrzgv6M/dSZPafRSB+FtR5Kf/+wn8d21T9wSJYs2nfRQDKGCHAmloBABcUENFZTRshhBD+ogMAUgqeokQoUgaEIuOEMmQMKAWnDWOEUUoo5YxLP0OYcYIAiJsGMUIQAa2zzgAFQnfFT+LQOus839CitVrtQklP1YpYb+ebd4gU49T7bEK+sGgJIcY5ybzKrXn/Jz9utQqCgEmaV0VZlogYBNJPtovT3v3Xwq++88290Y3T09OiKNq2blVHCEgpmWSCE+u0atqPP/rIZ4aIqG234+I553SnCSGGWUbpxckVIoJ1TmMQyePxjddee63N1bNnz9pSSxIW+bqpas55sSp0p2oWhJKXVc0pG4/HWZIyTg5u3aOUNxqtA0StO9zrH1hjmKVgqGpUXVa6q8omny6vLs5Ps707/+4vfhDF6WtvvRMFo9l5RXlwlN2Zr5b5aq10IwRLkqip87KrGacOYsI3ZC/VtD7r40KQ1i8JGtK4Hw19HOCc62aXn3z0aVmW63Wxf7RPpdBaK6UZ55RyxpgWiAhh0ncAF1cLq3UaZcDoaHiQ9jJCiLLGGZRBxIRRWteNGvNAhJFxGsAGTieCpwJC1hGHgYAsPZByb29yXJWzn7z/7mJ59vzp4o3X97Mkms+mx0ljdEeQBzwImVNKgQVGYTaftbmLGBwMvaMcAEkApLE4O59GYUqJ7MXj+/feGqZL7DJOHE6nl/OreRiGk/GwyBvnmiQOg4h1nUHotG6UKR1UnV7rPN8fDMpat109mkS3707298ZVVQFtHn/+sNcbJHHa7w8psOVyzblANNPpoq5bQshwOI7TTGu7XhWuQT9wZTKZWKfruizL0rfDHh0N0jT1tas4TWUYUM4IY8zZfLV2gJxzzohzJpKCOOvvB9zcY14KjQLxWS8DAIcbYhqlQCkTcSoYQ4eLvDCfP768uEJjy3U+m15prS3hQZI659wm2woCWZTFusiXANDrp2EgqjJfr4t+mnVoGeGcIQhitfJ3IiNiuztPsGc+YI+iqK5rn4K3bVvXdRzHb7zxxsnFSVkXXdPGcRxnKWOs38uGo1GUJmkvq9umbprVallynue5QbeXBq3qIhIhYqdVW7RlXae9THkxAe5HWgC33FlAdEx4Oi4xRvX7/aJcNm3lD6zfzw4P943tuqaZzq4Gw/46X5Fte7eUkm57phExFpIQEgQBFwJg03vt5Qd85rFJaLZTQLXWfoahl4UOwzAMOfXtYbCBYTyvxTlkjHFC3XawBgA4iw4oMFpXRRyFh4eH6/Xqgw/eb1TnnOu0QgJZFsRx3Glrm9Zatyrrh4+fffr08Xq9RsS29cUgkqbpYDBgnGhlEa0QUgjhnFGq1br7RYAtuVYOuO6rrncz7giDu+++e9Inan7ar0/LvKV2W/LbL3JUdDuteFcLoJR6E6OUevz48aeffaxUSxkq1eZ5LjYYAPMActNUSrWMCUq4tZYQvmuVo9uO+d3BwzU4pCgKuMbKxO084lYtjo8OgiAIgmAwGBwdHYVhOBwOj44O2qZ5/71Pb926tVqtPvjgg+VyvV63g9Het7/97d/6rd/69LPPF4tVlmWDQbNYLNpGh2EYRYG1SCkPA6GMrqtSCOIDFLIZQP8zFL+/VVmF3WY30s4Am9IA9cdMNum48MO6AMC5TfGRkk39yK8gv3I4+zlRy8/GOi/c+Rdnju+2Hc9x968dKnY9DvBBqkO7RRRe7jShlPqqlnU6DMM4jruuy/OVfxIAfYVLSrlYzj748XucBc5ZQkiWxZwTrC2lNEmi+WrZtq21WgjBBAsC6ccaRTQwxvj24yiKvKZZWZaq63zfuG67xWKxXq8nk4lfAFVVzWYzQES0xM/SaxvJhV+9YRhmSToej5MoJnTTsqHbLs9zZzYKEMjpcrno9ZLBoBcEwXI1Pzs7L4o8jJL5fN6qzlFxdTWrGhNn+Y1bd1955ZXo6vLiik1nF13XodNKGWM0OEoYEgBwaJR2xlK2kU6nQLquA9fnhyyUAQVCgKdJ9PnDz8/OLhaLRadUp4xHLhkXdV0DUOvHuACjlHpuSRIPva2QQQQAddeCI5QQY4xXB/elfy6YRcMYaU0dRYJ6ZUzBmASt26IsBBe3bt632n360QefzYr10thONxX54feW9165e3AwtqY2qsnSMJTcaLdanH/tl45+7VcneeVm03Y2r6ezcrmYC5bFUVIUK3SsWAY//WC9WharZclXi9X56dnJyVnTdIA0S3ma9LIs+9EHPyIUKEcZsDCit25nDkNtzXr1FAs0VhFCHBkUVbdcLlf52WQ/6WUJIeyNN+9ORgcnJ6eUcMbEydlZ0zboCKW0VcpaUJ1hjNy6dcs515murpFzniRJkkSUs7OLsyAI+v3+cDjsZxlhzKIDSmqtl/PZcDIWnCVBYNpWEFJXJVLqeQmUCEqQ+MZFgoJzdMRaCw4JOEY3vB5wwhGGxDVNW1TTq+lctV1TFcaYUAYikMwy3bZKa0IIFywIOVg9GvQYI3fu3hoPR+fnp+v1WrW6qiqlakYDxolSxljDmDDdJpD3LciMCV97C4Jol4PWdW2MGY1Ge3t7MguB0ZOTk7KulOmiMMxISji5ml1WbVPUJQUgnHjKnp1eHPTv19XKWitkeHycdlqfn587QBEGKWUiMG1nQRlgASWcc0mFL+vEWRbfvXfDKAVgg1AEIWccrGu1brRpEfTNW4dFsfZ346ZcInjXKkS01jZN4yfcR3HMmCiKYj6fn56eBkI6Z6xF32/tB8cxRlarhZeUvXHj6OLizDkThtJaHQQBssPKsJwAAQAASURBVA2Z3BhjjCWe2yECNNZa65yvYwMSIIzGQjbV+qtf/XthGD56/PSf/bP/KQwjrXXddkGUAJNtU0kKMkxEbMvOUKbbVgtB0zSklBtjpJRZljLGFCjKIAwl57SuyzxftW1NKSfXQOAdeLDzl94S7azq9URtFyjAluS4ixJ2iIKncJMNjPyipvAFZ7BzP7Bt4CGbQQAARgjBOZtMJqPxQKlmvijatvbjyqSURrWUUkKQUobojNFaK8Q6DCMAynw+6geXMOLcF3a6ixJgy6rzCIpfsbhpUeNHR0dt256dnX388ceLxaIq1vv7+6PR4N69e6cn50VR3Ll559vf/vbZ2cUf//F/fPDg4f37r7322htVraqqKYqi1+shZYzWzvmxRkgIWmqBkDCKKPEX3XnGnA86r89I3EVp29//bYZF/dduzjlrX0yjdrhxw7AZQ298LEUpWmuNwSSJCCEEKOBmITkHiE7wF+d/FyU45zjnAGS3KHaST9fX2PWz4R359ZjP/+kbfX0Gv6VWuGtdD1/Q/ED0kwkZY3Q5zZum6bpNjXg+n3lAtG5KSmkaxE3TfP7550q13lxHUWSMarpWCME5EOKvmwJAwgSlxM/W8hx2D7Iqpfz9pZSKo2hjZ2LmxQM8kSuJMwAoisIaE4YyCkIA8OIucRxnaSylnIzGSZKotqurqlznvrDljDbGMIJV16i2cWg9J6DpuqIonp2cLRazvb29xgoHvGv1Yr1qjbuaL2plRBSWZb5erubTWdc1hDoKjhAEDtptECN/CTgTYRh4fSSfFK1W+YMHD58+fRyG4b1796ApT07Pu65Le1maCaUUAnRGEy420SL6qwAeqDDWCiHCMATG27btlHLOUc6UMUEoBPMBKBNCSMJFwG/2DmUYcsGs1Voh54KJKATgDOJkcOeWMDV99mj67HGumqskiVDff/SxfPjh0ugyTthkIhCa1Wp2//79piTLxbKqO+t41guTXnLj5vjhw9mwl1Z5ThkYVc2uzqMw3ZvEnBq31x/aTl1cXDnn0l4oJbTdPAi6IBDArbGlcRhIRtGBa0T0AlxVdlYvbNuoOIWAM0q6fL3YP+x/+1vfuLi409Td8+en8/lVFCVCyqJYz5YLSjjnAWyq/oQH8uDgIAhutm07m83my5mUwWQ4Pjo8Hg2GGzgXkQlOwaFWo16W9Xqqqjh1QSSqqrJEIPqOZOs7rwhQAGKMBiCeuMM2Wh+EENDOh6rOACAw66xyrnXEy9Q5R6x1nXXaOs45BRYHctDr33/1Xpakr7/6ys2bN0+fP7fW/tVf/fCzzz579uyks4pSro2y1lHue6apTyXDMORc+tvDeyzGmODC1/8IIdZalvLJZJQkUVmWdVE6NE1TTafT0XiYZEmcx9ZpAGecdmCLqvzgJz89Ojq6/+prAHA1n+q6DeMo6fWrqjKh450G0ljEkLIs66dZH0jnv74Q4o037z/6/OFwlPV6vTQOtWmePX9yfnEWh1GWxbduHf/VX536Oxm25XMfMVBKCWMeC3GIQgRd11VVRSk93D/wmkU7k+21Dquq6vf7N2/ePDo6evz4cdu2vV7Pa655c2+t9fVXQjljAgAcUO2HebptdxllXEpK6VtvvVFU9QcffvTg4XkSs3QwrJUGaY02VEgRJETwIOkxzkPeWWs5l/6c+ypGVVVCcGO4lCJJQwRTFOU6XyrdRmFvZ7J3UcKucLt7Zmeyd+qN10mIsO0y9//y1nkHt14r0H4BnNg5oet+zjrrYWRENMYw5usdYrVaxUnoux6klIQ6r48rGPFUdmM0giUEGSeMMSAagPp2L22se9ER9/O3HbOdbAsi3jsyQff3951zSRLleT4YDCSnVVWdnj4f9Pu///u//9Of/vTs7OzVV1/N8zxN4cFnDzn7s1//9V//yle+kqbZv/93f951pSN0OBzO5wvvUAFoXdeE8izL7LUePNgmytbaXSPAbtsd/8+GCP+Jr/bfMITYnUN/Ff19TS0SQpxFSpgQApE5h9Zqa0P/hQCAUo6OUMbItf7J7eF9AVHYxUP4M00xLwVDdjuleuf1YRtA7OyzD6qcc+BwizSgX5O7JghKNrPLT0+fP336uKqqMAziOFZKcUEBwPfUtG1tjFJKBSE3tiPUMQ6UUkKdNm1ROmDUoVG67VRDGrozI0BJmqZRFAFAXdfL5dI32x8dHs7nc611P81u3769Xq8B4PT09NXbr3vL02qNaHeAmGfbEEI8V5dzPp/OVusFR9KZlhAShiE4o5SyVnPB9vb2+720M1p0XdYbjMYHTavCKJMik13bNJ3gUkh5fnr26cNHnz54QCktyrxpKs5pHMpACiF8GzwyQgWTlFLOeCBlFERCCKW6IAjQwunzkw8//PDzzz9PkmS1WE96ycXljAoepj1CSGesNgYAkiRxBIjbDBk31hBnCSGdaoIgCI3yEKBFxwXngWzbGgihlPgkjTLCGLGWR8exF16jnDlwZdWFgUiSYSB4vmop9t5845fzhf2P3/vLB08+vHNrsLhSYYC9fsRZeHJ19fFPH/b60dHR3r/6Fx84AMKACwFUGofaAgAITHqRxo5Faapr3Yt6v/zLv8wJ5/PLyziObx8eHe2Nu64rmzKvll2bC9qFESXM5XXVtpWyFACVamORyoBG0UbSrqlaGQSMpsRRZ2nXNYTYTlVXV+ePHz95770P6qYcjvq9rFdVjbQQBFFddReXV75fVkahUirLMsZImqaE0dt37+zt7SVRpJrOV7mAokNjOuWsJtZYVVtVC+fiOA4YmbeGUkJ9pdUaQjmAJUBV68eoMEp9gyvxob1k3Dmnt4vYOmytbo0CcA6QW0PRUUqllIIyRlnXtAevvvbNr/2dXpbtj0f7e3v9IIrjeJT2JWXL2XK2WFmKiEgoZYwFPPTDV7x4M+fSO862VX7MlWducs6rqlqtVoODwcHBQZYlRrW5Vf2s98q9uwcHB2+/8WY/iZM4XC+Wbdt2bc0oECCL2SqQ0cXVLIxkkqRpL3t2euKRLueNCQNf8WSCJmkQJynn3N9maZoSguPxcDgcojUe8Vuv187qfr/fqebs/MQXDq57tU1nVBRVVXV5eam0jqLEz7UKw5ALGsWB0i1lAIDOWcGY0i3nPAjFV776DiISim1XD4Y9IM6PDN3YL6QEvCAuBcKAWqDMgdVoAYECMMbWVbu3t/fs4urPv/sfPvrk4zAiR7duz5cryrl//97eXhBFs9miVY1AUedrACDE1nXt788sS6TkYSg9Uy8Igq7r6rpUqt1OB30ZToBrocNL6PcuULj+rusRwO4B3Srpko0E7AsemV8A1+0+7rjom9fg9c05N5vNCEUpWVkV1mohKWNEa80pADhE51ABOC4oI5z6OoNFY23Xmq5FrTfBnx+FtTvO3S42QPr2sH1QSwiZ7PVu3759586dXi99fvKEUuia6oc//OF0ennj+Pi3vvM777zzzkc/+eiP//iPLy/z+/dvnp2vfvjD96Iovv/aq1KEPhGP0rAsSwAUciM0JCW3CG1XW70Zd2St3Tm/bUL8C5387vz/zcAJsNVQur5toBe0/qwaY7QygAQIWuuUMpSCc0wI6hz4fJ5eE1kiX7zG/jNffKPN90K41tTw0jfdPXn9Q37u2UB8EShsgNVN465zaJQiWpPnz5+v1yvnHGMUwfb6qXPOWs0YUaqbzRqf/BjbSCnjOPRDH3yGwBgTgvuVYw1a66zdaB1qo9brtedTO+fKvAiCYG9vbzKZlGW5Wq2stb6TVim1Wq0e28dt26ZpCoh1XfrBK1prq03btgSttfbo4NCvVSHErf1DLxcrJXeOWKOjNB4O+1EUyYBba5VxYZzcvH0n7fXH43GpaNU2eZ5zGRDKm6Y5Obm4mF55tmYahUEYMcLAokVn0Ia9nhASkbStQmzaVpRFDcRVVeWcM0Y75whhX/nK17Isi+PQqBYZV9oulmtjjAUcDAa9Xo8L4ddMp1XTVm3bKq29CQVCGq0Q9QYKEhwAgigMw9A6Y4wqy1Ib5ZxhjDwWiXNuNBrduHEDGC1Wa20gkFkkg3kxi7g8OLzxy99KVBM8+nxaFGwwTMMwODzYm0xGqm2enzxdzmYXJ+bVV36Fcz5fLp89P2+0SXsDY+xsPh+lbMaV1kFDyGI2u3fv3v1bb8znc87QVesV66WTyXC11o8fP50tpjIWZTkDGouIUTCMgeCESx7Fol4pnzBtBjwSXpZl13b7470k7u/t7QtJHz3+7L33f/TjH//044+K3/7tr/X7ww0FickkyWbTFV5e+BVeFMXV1ZW1Nkmio6OjvYP9/rDnKfRN06RpwgLRdU2dr0/Pztb58oGzIpTT6bQu1oJCGEdbQTPi0PoKN6WcEGesIsAodUL4+5s5NM45QdA66zlLiM6CA3DAHN8Io2xIw2DRWe10dzA5eOPVN+7cekVSAs4Wi5w5ElHxpdffPnl68vmnj7UiGpE7BEoEDxhyHyX4zNKnlP5G9YXqtlGedOacq+vaXXVZFGZZxoFGUg6z7Gj/4ObxURoG48FwmKVE64VW2qIH3Bbx+uTk5Pzq8tadm7/1W7/5ymuvMil+9N57q3xd1zVQlmQ9yljbtnlViFD2+8dCCE/1L8uy63QQRJTS58/Pq6pSqvP/DYN4Nl0URZHGe555ANfoygDgAUkf7EspfendxxB+fi7dzuRkjHVd57nTR0dHn3zySVmWbdv6fmtEJEjRbdjclHJE6IwWQBEJEuIoxY3rdYCEyWhW1P/y//unH3/8CVCWZX2NsCoLLgRjTHB6dHQQRVGZr9raCeKA8100gIhBIOI4DoIgjj02AwBgbOtQeVX8bWvti4z/euiAXwQAPEJ+HQfGLWt9q0OwoUN7VIlz7jU6ET0iba6HFC+Zdf+Ye+HILeeMMe6dQRAEdV0jSm9SwygJQ7larQw4xggXhDPfLg+EaodW8AjA2c50nVYKAQVjnADl+DJWvwsH/Rfxd7ffCCFHR0dxHO/v7/f72fOTJ3meo9XL5TJJksVi8Qd/8Ae/+Zu/+eUvf1kpNRpNmqYZDseXl+WPfvT+x598tr+/78fGMilm04UQQiu1rgoASNNQMNE0jdHW63n7vgxKqSd4e0rH9SPcOezdM+S/gMTw3yqKINekJL3HBR8roEVESqnWllK9i3V8CUAIQoihlCJyAOp7QTdr4NrS2vh7wO0ucHvcuPtAuIY3kC+WG146Sz8ndCAvClsvnUwZSD/OqiyLfr8fhKJt27ZtfRsLISSKIsZIXddCsMGgt1zNENFPtmxbhQQODw9v375tEaqqatuWUsq49LcAY2y6OD8/P/d69kEQdKK129Eq3pj4d3kgRAjx+PFjIcRoNErieLFgzmwU0vYPDkajke4aD+QQQjjnWZbpzhilndHIKAAiag+glFUeuYAQti7ysizLopZSakODKGNBLGTcqi6vyjCK7tw5EmHg45VekgrOrdJt25rOmE5pV3DOnTNesVFKCcT5ATdScs9WnkzGd27e2t/fl1IadKPJoRcUmM6vnNZAmEFoqpJSipRo3bVKtUppaxBxkI59U1LbbkY3e1jo8vI8SSKH1jnVH2RJEiNaQnH66KTVajQavfLKK2nSM8YMh0NG5b07d0ejQwbEKBLHo9u3XkMrHzx4/q1ffrvt6uenz5Rq79y585Uvf+OTjz97//0PLi4f/O//D//0zbe/9Bd/+f333v9xnPXCKBr0Z5GgQoimreqyXK3bulInp+eXF2f8q196e7mca6fB6eVi+vzpw9PL0zCWk4MxF4QSFJwC4cYaU/sOhR4AoGNaISWE0YAz00G9WKwYjXq9flEU+apExH6//86X2d7+hDFmtA0jqQwUxVrpttfLyqKJ4zgJE8ZYWZZN083nc+OslNIZNEqXZblYMM8761SzXi2sNeenp1EclFWVRAFazSEMhPRWGB06x4AiASCEM0Kds86iAYWWU2oRiXNOmRoABKOOoDHWoaLMhZEUlAIiaGuNQmUoIYIyIcWd23dvHN7q6q5W2jQNI2Q06K91kRdVW3SShaEMwTkCxCEByhhspMo2shBIdjIS3vbthsASQtI0nU1PYiljLl65fdvdvOmMKReLx1VFlFVtLQkbZX3TNu163U/ivb1JV9sgCKxzdd1+9OlneVUChe985zt/8Zff+/zxozwvuBRhnBp0piqEEGV/0O/3gyDq9/uAnDGBaBaL/Opy5scF9fopp+z4+EZV1YJv1Amvb94S+YwziiIESJKEc962rR/0xRjx6sKUUucgCETXEedM01TOmfPz04uLM0KwLHPGWNcZD36iBa8lbwFR2a7rgG4EgMDLlwJxCDxIy7KaffYk7U+yXnJ5eVnU5w7Bh2PWGmcVcYKCjkPGGKEk8P7YO5uiKIIgqOqCceI0egvVdS2lhHNmrd0GJV8IC2AreOx9AN3KGeE1Mjn8PKdlrfU1xV3edt314hfVmq+nidfMt5c8eiERhog+1Gg7NRweHw73q6oYjnqMkcVisUOpKaWUIaGWEEKoQzRACBCLYIEAY4QS4SgaILsD2/kSHwDBC+KI8Q17QgiPQud5HkXBcDh88uRRP0t/9Vd/dTIZLReLn/z4w+fPn79+//UvfelL0+n8u9/9bhTvZRmnhOd5OR7v3b179/z8/HI67/V6XddFUTgY9AFI0zSd6oJAjIeTGzduWGufP3/uafY7AOa6n75+aQBejnX+Zjb8gmsHf7o8Pc0TGNGLGTPGGDHaMgaU+knc4Fu1r0cF17/e9UABESklfogMAKJ78TWv8xXYF4eWXT9I3AJd1uI2nsBteLHTYvLynSDlRn5Da80FFUIURaF155u0KQVKuVccEZJzwSaTfevnXoogjrI4TY6Pbx4fH+dVCcgBeRjG/eHA4wfWuEYVYRj6OTh+WkTXdbPZLJDS59OmU3VdO+d2+aHHYqUQ1mrfAyUYPTo6Oj4+LtbLrus8KaGqKoemyBeI1qGptKKchGEoJFOqpZQ6JyiF+Xz++YNHWtu9vYOm0eneMSJyKZgIgkDfv39fhIEPfRgQY0xT113dGWWttoi4KqeMMR/cCMm8GFRd14zRKIj7/X6SRGEY6k6rViVREoQyihPGWFVVUZpUVUUI6UxX1Y2X3NRat22tlLKAlFKvJ9QZ3XSdr1f6rDJM4qSXckqEIMc3Dvf2JowRQjFA7tOwOErHe5PJZO/48GgymRBHspgywtuy6jptHbTaWIuffPwoCAIpguWivDhbTib7/Wz8W9/53dli+fDh47/4y+9//uThK6+9+pu//StVVVz++6f9yS3BmVvqqnWvvHbz6NZkVU57w5Br1fZ7KeW0bet+lt69c5tQs6rWXdfUraGC8IBTxggw337dtEpK6axrm5rQKpQBYyKOU+ooIhrtTk/PnXFaKyHYwcHBw4cPsqw/Ge8NBr35srg4n7atCcM4imKjXaO0FzwOw7DfH2ZZ6kUJ67q8uLiYzaZM8L298WDY6yVpGAZ5sUr7vTSN0zRer9dRFK1bbc0OAfb4KiIgYwSAbuY80A1NARGJUZxzRplxTqumKUvfyIRSorFaNbpuwbosike9Qb+fza7mTx89/eSnH5umYYhSiLu3bxHEs4urJ8+erhartjHKoSag0SGh/dE4jmPGmFcL8WiSlNKTFTjnvWzgqzY+Pr15MFiv15fn52htP+txAMnFuD9wqguEnAyGlIJu6jP1vC7KKghv3r41W8wXi/l8Ps+r/NGjz2/fvf3tX/vVt99+e75aLlf1tkMPjHEWTVGU/f6g3+/funXLWpsmvdVqtZivoihpW9V1ajDgURQfHBxdXFwQwjwzC7eT4nzdhDGWRrHXYymrqm1VHMe7SUXX3ed11No513Xder1erVb7+/td1/X7/aKoGN00hniLYJFQ0lVNzRj3mjOOUKTOo67TZX50dDSfz1kQtwY7ZUIZJL0kkoIzulrOry7Py0CuF3POCDAGNPRpje9TVapljGVZ5qlWVVU1Tb1x2BR1Z9g2AbtubckXWxmvJ23XzP0XCIk+SrgOKuCWBOqnMe16Q3ax1896u52JR+d2lWbnnJ+aA8QdHx+/9fYbXdf0+knTVGdnZ2W+8pGEsZoQxwUKyRhhDg3Ahr/GGBdcUMIsAQHkenTituR5rfVOacpuNaHTND04OOj3+34lDIfD8/PzizPb7/fffPNNSshvfed33n333UePHo1GI+fc7/3e7z1+PM/SftM0V7MpIk4mE6UU5VJrfXFxcXh09Ku/+qvW2j/7sz97+uwszcJ79+7dvXvXy9n60YUe2PBV7Z+9BC9tf2MRw7U1cj28A4c7yNDt0mIA4d/iLCLfVbUoIbAb73n9q72EKPhXbH59cSrpbuXsOkSuhy/XDvWFTrPHQrZh5wtJR/+nv9aMk7opF4sFY6wo1r69E9EiUp9OWGd0rZqm6WX9tm2DIJiM946Pb4pAKmUePXrCRHB5ebVarZMkMRbiOPa9XRezCy/i5I/QBxBaa48xCCHolhbqnCuKwiN2WmtrDKU0DiPGGAVUShFCPCTp7ed6ve5Us58MpYzKKp/N5kHIXnnl7mR/jOiUUkJISul8vkRCwyhMsx4ATK9mDnA4GsRpOorHk8lERuHV1RUhpK3qxXxe5UWZV8YYCpTiJp72itG741RKMUa8NsytWzf8HB9fAXFShGGYpikSyLJefzT0CfpsPkfYjNmsWlR2Y2PrpmGMUca4FAAQCOnLcA5NlmWM0Tjke3t7R0eHQjAEe9Db92ZBhsHh4eHt23fSKLXWobXMgQjiSEijLCGk3+8fHh7ePLpxdXGZ57mgoqpt8+yi2zd37tw5Pg6YoCzCwX48GPccrHmov/SVW598eNLv96t2Sbj+2je+cvvmERD35Xfe4kB7XIowknl1xkT6G7/191+fnfwvf/jPy6oejftAER12bRMEwbg/nE6nYRp1XYfGSSaZI6TBMBRpmmltB9HIOqebbv/o6NnpCY9lnMbT6eVyWRrdhEFmDBsPB61Wy/XCOSPCQNV52VRZOhZhJMP+4cH9/kHv4aMHbdXkdVfWNXFuPbtMojDrpYeH+2+++ebXv/714Whyfn7+h//qXyY8E92Kom8LBwPgkBpCgYBxLIzCcrUKpewN+lVZ1HUpOWeCEsGsNWVZtnUDDhgQVE63bSDYfLqqqure3ds3btwQlAFjJ+tH5+9eGWOaqkXEQdYbPH8UBTE4h0gUC0aTSDB+8vxUIIyG/bTfH4/HlNLZbNY0TRR5sWqDiHt7Q5+xxUnYH0QecFu3jcySapUv54uLZ8/efOXVveHolYODXq/3yv17/5f/+//1xq2b6SgOR9HCrt5648tJcOvDh4+F0dbqZbFsncmK4uT87Mtf++rRzRv/p//j/7kq6lAm+8ODPC/adfd4+el4lEUhu3P7qGmqcj3+/l/+uziOTk/O9vbG43HSVOuvvPNGIMnJ80f5ejbZO46iwFpLCHZd03YVIgmE7FRDKTHGRHEgJa/r0lpbFOteNlSKcp7mueKcEwpn53NKKaKJ4l4QppQFR8c3lVJKW8ZlOtwLw7Bt2yiKKKVd10VCOIB+f5DnuWpbX7bpjPZJxjgW5fRUIppKE0KGaeRV39u21cqFUVrVXd2oIOkDAGNMsjgIUwrIGaDtgn5089YeM2UqbED5IJl0yi1W6045h1KpeSheZGDeqnq8Zwf54lbO+fq/dtJMO1u8K9b6J71npZT6epPWele42dVodokdXmNIAHWEAFDnEJRx1HMbCXGA+3t7ypj+YEQITvZGz58/5SLcyupZgpQyjxwwABSCGWOYMEGklaoRayoCGQtbhB7PAgTngFGqtdVGScGNUY6AlIxSZ50TUvUH9LV7x/fv3vjBD35w+nQwHo8FBNPZvC3p+z98eP/+/bt3j3/pK7+9Xvz5e+99fPfu3W9849fW1U+enb13NV0LERRNi4x+9RtfefTo4bPnT4C3b79z7+vfeKdtu/c/+GHTFoC067rFYlZVVV2XvjsDAKxFAOeHIDoH1vhLQACAshfiRd4v7gr/Ozd5/QGl9Pqf/9moAkFvUK0vPP1ijDiAbx7ZjG9Goi04tMZzPxgwR6wFg7UTQkiUjqDzEpiUEsa61ghBKPEDDpjvNPad4Rvgx+M9ANYZX7feIFiIzjqADRXAOPAEesa4CMJdIsQcEI/J4Ubb02MIQCihjAvBvNYTIUAoEColl1KenJxMrxaU8JPnpweHe9Zuel6sM1YZIZgQwhjsVKOq+GDvMIqSpjDPH59lg6HqzGwxH4wncdwXMjm7OJ0tP3zttVf3jw5pAYkeEhH4u8M4goSFaa/HeK/X81MVDDMGmq6unUPBpNbOEVo2LXE2TjICzvdbAXGPHz7Y3xu/fv+uoHA1v0LTgFFPZs/2x6Omqy2qQbYXcsFaM+j3191aEjFfrPLpKgmitNdHSrTWLkx7vV7a76dpGschpXS5WNVFs1otV6tFWZZ5sSqKIg6DXq+HhIyig6aqAZo0SfpZwigpy1xpS5GEAQ8ErZqK1qw36NNINtwEHJSzs7x0zhHBIxk5aUUYShtaq13jQINF03UNpySIotVyOR6PkzhmhKO1vk5qtYnjkAuqmkYQoqomi+Llcs4ZadLTXjoCFxptq7WbnbRulIwGI+DONOtyNk+zUJuyM+XhzUmW9W8dHBJqqHBt21ptbt6+3ev1vvdX/2FvPPzd3/1d5w4fPXp09vD09MHZfD7/1re+dfvG8OLyDN3g8GBI6WGrhmkWN2rMpQw5Z0pbra21tq5rAuzu3VdOT58CUNW2cZr4CmKRV0Y7xzvCCAHamY4ilVx0Ttm66PcGNCDOURYwEXERCNIBcNy/eXM2XVwultas0YkoTbMs6w8n6/VKaRPGSZoNnBWq0wQhieJynefzZb5arhZz1bZxKDhnSRBkUajqKl9MnWrSiI0HcSLg8vRZyTQTkjAhCEdnjLFaKQcUCeWA1FmnO6MVIxgJGYZhW+eEoeAiDqO2rFfLBRobCNlLY92pNE2Pjw4ODw8ZkOl0WhRFVVdCCIJgreOEIoJXzheMDcfDMAgYEK21Mco5l2XZjXv3KKVeMt2rEZAt4cj3KHtpUh9ORlE0W8wwDPM8X+sFBzTWrvL8o08+/ta3vnXj5s29vb0/+dN/K+MgSZLTy9NynY9vh3ESTmdaqS4MQ87ZxcUFEKyqam+yf/fu3aurWZ4XzoJv4+j3++++++7BwcE3v/nNPM+XyyUhJM8LITbgQRKHiHh+fq6U8tJD3nJ5mAcAuk77PM+3KiCB7Z8AAF5ZxTtCRASySWWs1WEYepUFn0/716dp6vunR6MRpXS5XO4cLb2mQbQrQ/rfAOB/++zNbifh+kY+pZSU8vDw8PDwkDE2u7qazi5V1/R7yfHh5OaNw+Egu3XzoFMGgFZNa2w7na+drglshi3tnMp1L47b3shdBECvMeyu4wr+BS/ldvhFqiN+sZpOrzVN7N7o97/tud/tYaPF5AduFUUxGPT8OfeETUIIoZwxZIxQCn6qJG4GSNIgCChl25FDJM0Ca5EQL7qIgJuJANtjI+gIIYKAtQbaxgYyLIvqzTfe+vjjT6uq+Yf/8PcfP3r613/919/97nd/+MMfDfojxth0Ol+tVs+ePWfs3Zu373eqTp4FZ+fP5vN5VRdxfPPmreOLy7NXX33l8PCwKNcfffjJbHYZRcHx8fHJ8/P54rJpmsViYQxEkQhCsWnq25zwnXf3k6n/pisOfvtZSGNbJYHtsKsvcAh8oLkrFuwKWHzb8PXS+nHOEUp3t5WDF6MW6Pb5XUZrrXXwgqNw/XO8auo2Wt2AfDsIZHcwu6P1d+h8PuecZdko6yWU0vl8ulotOOdcsCAQYZgMBoMgFACwuJw3bd52hdLOWnRnJ1qbttMykgcHYxkGi9V8Ol13Xcc5j+P0BbyBGzEpQTfEJsG527YmOWM9Q0VZ7XmXFB0gMkplIARQRjDL4jiMVqsVGuusGQ6Hxpj8+fPLi5nRatTL0rRX5UW1WmmltFaUsSxLjm8eySRCSoByymG2WlIKUSTaFqpqXdd10zTaqLau87wsiqJtOwKMi4iLyDlXl7W2HWMsSuKkl0nGKCdInCMuiILeoD8YD5J+T4YCAVXXFOsVABjnkICMwjCKKBdeSSKKIhmIMAg2SgwIzqLXt0BEIQThnBCilNKdIgQD9NIRnHMeBEGapmEgdLfUnWbABYvTOIyjgINzuqYMm2rVNGvdom7zspzV5bKtiueffgoASZIQa54+fvjsyaPJZLJer/f2xn/+F39BCHv77bcd0B//+MdUyCcnp5UmWndxIoQkRbkEojuVAVi+Wq+zLBOSpFmPCUDQRkZvvPo6cbZqqyIvpXBRErWqAqW4jPOmllxQSi0aioRQCqA7bYUNQDGlTWNqhbpzbdOVlmgq+2vVzta5NRQd72lEIoJQxFnWtY0QQvCwa10Sh3EcL5azvJgtp7O6LFVdCYICkFojwFbL6c3jo3tH+zcmvWpxefnsWVvM2/XaphEHlJwQSgUhjlEiJReRNY7LIOHQdR1Vrc8biTX7g5GXQwk5N3Wzns+N0TIKnbGAdn9vPB4Mi6K4ujz3lTNnKXHIOZeUcc4Z2Ywz2N/f55Rq3bAwvHl4ePfeTT9V6K9//GlVVev1um1bIYT3fz6t9G/07R5BEPjf1phNiS6QIecGnUWHlDx49PDOK3dv3r7V/mlbdXXSy8DhcrmM43NEC4AOTS/rUQqr1eLqkpZF9Uu/9PUbN24RIqryCaU0jhOvIZ3n5WuvvUEI+dGPfvT8ydM4Sm2gvT552zZ7ewdCBKenp1rrJEkcvsiJ/UQrY1zbthb9wBJr0VGyaYtwznmi4q5DGjfFfcc5dVvRIbLl94VhmA3Hfijc/v6+t1a+RrMrdvgSnacHeobdzuDuPK611tMtfaUjy7LxeHzv3r1bt25dnD5qSrqiOgzJnZt7b7356nDYJ+jqglICnLNAhvmazlyDrg74RtvfRwbXAYNdVcVb/N1X2BESXzLQ9Iss/Z1Bvx5UXfc6jL3gtX3RiG8yRbctTnsfI6UsyzzP86urq/F46KPMrbvyr0HGiJf/84x0vxeQu0/zfkwjWMoVoQqIBWCEEsb8YEMf6zACFNBoRcui+3/8P/+nr3/967/x698Bwn7y00/eePOd3/67f3eV53/9ow/q+gQA4jjinGtl1nkxnc2+bCwhcHA4rpv1bH5xcXEWRXK1XrRtfXh42Lb1w4cPfvTeD9quPjw8vHnr6OTkZLFcOQdCgjFAKRFCAPiyHfFC6de983+5a/9vu+2iOvgiaOFFYH3RkxCC6AN0S6nXdCf+t48DEYHKwH1xe7EItiINhBC3bXbYhQh+/W/FpDejoPyavH6DSC5hF3Sy67SGjYLCjj3jn/eOarlcOOe06Q4PD7uu0bpr25px37bDsl4yGPZ6vZ4QgoPqOt00rTJNVXZ1pygRMozyYmndTc4jD3F54YQ8zwkCpywQEvxNoU1nOq31ejPphjBCKZBddy4qJyUnDn2riOBCciYFY0CyLCUE8+WKEkjTNJJBXqwkk5KLMOuN+n0hgjpfdk0N6G7dujUY9qMkjnqJOJfz5UI5FwVCz1bLlUZQw+HQObda5Z5GoJTKy6puWgDKhKA80JZ2ne7a1t9rQRRSwY1D5VAjKKWAU2s1gOMUGAXjNGrdtZfoR65RRkhISWyRtJ2WYRgEKSM8R2eUttYCMGNcGHBvJQIZcUq9ZfOUCEIRrHWOEkJ8y0kch6hMwjLOojhIsyhiaOpyoRvo2oJAE4VivZpVeVHl04NJNhnv67V6dnrCGAz2R1kaGWOUts7Zy8vLtlVZv/fa62+yIEz6gzfeeOPzh4+Vbv/u7/7OZDL58z//83ff/Y/3799//fXXHzx4yC2iA0CgcZxGUZgXCwbN/vCgOipX+WqxWDWNRuDW0KzfGwwGavrcL7JAxoiojTVWc87nxbo2ChGVs5YY7bqqKTvdOkFbZ3gcRjy1imhH87rlnTo4HHujvFqt0LFbNyaMuvff+yvdlUq1SRxGWSQ4SApo8WCU3jw+pBRtl5fLqTFKuLYn4Gx5CWxsXEdtRxgnwMIo7cdpkqVl1Vrr4kCsmgobK0VglXba1FotZnNPF6iqol6vKKUsI/fv3F6tVlKIslg/e/o0X642c4dBUGQMOOOUAtHKlq5mBJxzaPXZ2WkYBG3XHB8fRmnYqO7Bgwd1XXtdwiRJaj9vzjkvPemc8xU+f9tUVRUEQV2UURTtTyZozdVsSggeHr9Wd+1fv//e3ft3/97f//s//ukHV/PZV7/61ddff/2TB8+B6DiRDlUUBQDAmAiCqK7V2dlFmqacySRJoiiWUlZVpZS6f//+V7/61bpuP//8kVXdvXv3ELEqc8ZYnuc3jm/Gcdw0HSFMa+vtx051FQA8JACUMsadc8YaRjfwgIegdlECIYTQTdtYEAgA8JgKbhkJ3uH1er0NfLqdC+VPjt/vDuf3hs8/D1ss4Xpy5oEK7+C7rnv06NGzZ88Eqdq6jAI6HPTu3D68dWOfMbJczDhzgts4DmIS9JZhKKFlAAFYfJFsXQ8Uridh/su6L7Y4Xg8UXgK03VazaFdxv+5gdvYat8Mydt+LEIJorwUPLxjsPkltmub09PTOnVuDYW88HkdR1FYLSim9tn9ERDTwog0SgCAhDMBRSqytKacycNYRrjk6bg0wxlRnCSHoCDoGAASYVlA5m/Umt26/ioTvH9746/d+/O5f/eh//B+/9sZbXw6TP3bQaq2Nc6rt2lZRCtrZjz7+yWg0Go/HR0cHQpK2bR88+Hw+n1EGs9lstVolSVIU66OjvclkOBik9145ns1kFCXW4sPPn7StEkIAwW2m7n0n7E743zqi8FKssH3SLwMAAGuRkE31BMAiEgDqJRABKCcafEGEcdh+KQYeynrBgHHwAm3a3Qt+T5t1eI3sAttFRV7S+aDXq2ObQHwXW8AmstGeNyADXlUVom2axjnDuCfwaus6WkEYSiEYYpj1Q9kRhK5qNBCdpEEvG8VJGiWp1m3dsCCQe3uTNE3ruj47O0tSQSlNwogQ4klCXdsqrQvn0G5QQwYEnWOUCsqSMIpkoE0HGsEhI0AISMbjKHBGl2XnrBkMBkkUFEVRFMXeeP/oYC+No7oqmyJnjPmhtUWxBoq1bpZFfnF1dnpx6Sjp9Xr9LFqtVtNZwznNskwIRghah03bKaMJ45xzh6Qz6IjulNZOJVESJSGVola6qcuiyJu2DkPpADqjq7qgzMlWIFhrtRQ1YwIIcQQpU4RsSN5giepqpex8drVer4kjMpSMiV1e5C+ZtzxBEHBOKaVG65cypUikWZiFYRrJLODM6bprS0OtMSW6jiGbXjyhSDhRt4/3j46Om6USHNK0l2RpkVfrsmiVfvXVVz9//EiGejAan03Pi6JarFd1pzqthoPJv/2Tf9c0zeHh4be++WuDwUDwsKlzTmVgHFilGSAlVnemaxSAEyRKI5gMDirVWgJZLzy+eTPr9Rblqq5rQmgUJsaYZVMg2oQLZzV1gjMZhsJ3TgrJKaUy5L1+ksScQFgXRitkjPkcyAJqrao6N9rmRWoNnF88QdNMhoO90VhwEnJCnaEkeuPVO1/7yttWq7arTZOv1+umrnuRnKTxWbnCTjjO/WxcE0bCNLQt14t1q0wYZcV8qSyOhpO2aa21geCmrp0xhlKi7d5wcOPGjddeuc8YqfP15dlpU9VdWw/6PULI1dVVGk+UQ3CInBFCurZFtADu6uqqP8iEENqYzx48ODl7fnR05CdH+/p0kiTbadqOUtrr9XyHG6U0jmPOue8HC0XQUuodpDF6MZsrq4JIHhwcnE0vacCiJM4G/Y8+/eSdL7998+bNp6dXYSSCQCjFfV2Dc4mOBZJfXky7gWnbljFeVVXTNM654XA0HA61so8fPRVCHEz2Dg+PrbXnxvR6vSTJer0eAIRBhA7yPLcI6IiHAZRSu7ICAfDuaGe8Nsi82xo+n/uyFwbOh8Y+NvLmrGkaUZZpmjrnvDyld/Y+UIAtwr9Ls/yT9FrTQRAEURT5mojva/D8pqZp8jw3xoS8DqUMQ3+KutVqwTip6ypOQqAkDEOgPEmSOA6bprFordkaa89W38osXocK/IOdR4drUQJ5wWt74d53uSC9NmP6ussh27lT7hrjzG/GWF968L7HxxmwHQKklHr27Nnh4X6chFv6i38VcQ6cQ2McRS+7tD1mv1MKlHJKgXCPFckgsEo5o4nqgBCjOkMpRec1ygigc84pq+taXVzMus6GYXjr5v3lovjjP/q377//vuqMc+h7igEsY36wRVgURVmWy+U8jmNKeFNrpYu67qIomF4t82J1cHAQBNHBwcFgMBgOh3fu7Y/30iTOlsvi0aNHWkPXNWEYNLXedAYg9Ui8d4Z/470O/5ltq6T5co/GDgmArRf3Qa3jbIfGvbRgdkxGQgiSF8SLXfQA15Sbd3OorbVkS3/ZeZTNn/Q6k+MFgeb61rSVsaoo14wxzinnvGmqqi6dF1QHT8emURRlWRZF4fnpmRCSc65NB8SNx8O9ySGXwWT/CB1Rqo3j2Nulpqm07igKXzwihDBKAykF54xQpZRC5bQx1mkvcUao1SZgnIADh+j8MHTr78swlMU610YlURgFwhlL0A16WdYbC8aNVka1aLtQijSJKQUhBJdMCBaGYRRFURKzQI7Gk6LG65VQSmkQhYyKum2DKCTACGddp5kQ6aA/knKcJVmWZVnfObdcLhcLDpwl2OunCRcQSBYIBKfbqtRaGauyQcVoRBmzyJw1TlkARh1ZLFdMFF1r8mUBzgZBlESxEIEhG8DSOWe2+KXgIo5DyqBD9DnbVueXgUbOwyRKwyCWjDZd7UxDQ+j3gvOz89Pnl48efD6Z7HdNfX5+fvr8NKS9zz77LEmS0WQchUlelR9+/ElRlndeuV8UOZMiL9dSyropHz3+7Ojo6O03v/nBBx+8++67VdVVVdfvr+7cvtfvjfmTZyeDrDce9eIwQHSCBWDZdDbL88JRN5kcHgRyWeSGwGh0CJSsZoVSikvBSIAABDilhDJpnQZklDLBRSACyWXAJOecC2ICoQC0MhxcKIMgkMaYgDNkJAhgPMnKop6vzinlx7dGDNvxeDgZZVZ1gjjbWepsnNDhIDncf6WqKs55V1bTomiqymqdhoxTQik6p7U2pqzWbdEs4sUy1xb3D26W08vZIld7eV7WFqEoCq+tYVQHAAcHBweD8eHe5A//8A9Xq1VVlb48HwiOiIwAOm3RGOIYDYAQa40HdYMgsAY7YwghSmuqhDLYtHqX43oHs7vG3h/P53OPwPtuNEopWhfKgBAyWyyyJEp7GZPi4bMn67q8d+/uBz/9SVVVWZYQQj7++OOjo6PxuH95edq0pUPLGAmCRIqi61SWZW2rnANE4sevOecODg729w+UUtPpLAwDwYM8L6WcpmlaFrVn5ypl2rb10z4BoGmaQEb++LV9Eeq6bWmAWPNiZRsj+EagaVNZ3Lbewbb8CVtxBf8yL9T6gtMA4E9IVVU7O3u9BuGb9HYG1x+zL174UbPe3frWhjRN51dPuBBIXF40Dx89Xa1Ww1E/CQNjVVE1rGgJ5WXVApVURthWu2PYIJ8vYOAvGPqdNd8FE9fdw+7116OonTN4yYvAtT63XcGCXMMtdgn07sM3UCQhTdN0XfvBBx8wTkajQVmWmxcYALBAkAEiWEItJxKIIwTAEUIRYONLEAljjDBmLYJy1hqlTNfZLaoBAA6RAkEgiA4+/uQBAnvnnXcAIIiiJMv+8vs/mE6n1hGlrb9AQRDKICKEIIDRuiyr1Wp9cHAwHA6dI6buAIVWRGvbNjCfrQbDrCiqKEoQkUuT9bkUBJcd4wAEEFwQBG1jKKUEmB924M/3f40H/2+8vQQkvMjUN0nvi+m16AgS4iwAgkGHjgBaSixnjoDF4OdzKl+EdH6xUXZ9v7uQFLdii0h2PZDWuM3yE0Jcjyqun7Ed+cbaTeThV3IURdPp9OnTx1dXF2maRlEwHA2CUBbF2jptjGGMZFmWZWkcR1JKo1EKEcgkDFLJSb8/TJJMG2utrau2VV3TtW3bVnXBJHNo1quVtwN+tUsu0jRN0/Ty7ByNNcZQ2OKFxAFAJCRjlDMKyAPOwiAIOIujIApCjDWJgl6aUsC2bbI0vXf3bl6uP/no43KdD/pZFEhCnQxokiRlVTkH2rhOmU47ZRCdIusqDEf93phSmoRJ12mPpCZZugcYJQnljBBSNnUYhjdv3p5MJvujgRSBc26xWK2rUqPThAACkTyIRBLxgKOzje2AUispcEBqDUHwPSPIwADThqiuIdoC8jSNoyih4GtAkMTJzpwSQnbtkYwxIRlxTohNSkYIkVKGnEdhEidpIKQxpmqKspxHCpbrZr68Ojt7/vDJ4+fPzuez/MGnj40G7sKbd27/xm/8RlmWP/7pTx4+fHh+kd+5szebXfX7fcFgNpsdHB298epdREwTGYTuG9/80snpg48//ti6cjh6J+2R9XrFy7oJw5AJaRHaqpFcDnvD6eVlW7V5U+wfH92+e+/J6dnlYtYpty4LasQgzpjgWlkkkMV9JMRqSxx3lqIhxqFqrK4NaCJFoKp1KoJsMAnlkJKIiThfl89PnlK0USy4sTIgSSqbpiNAe71BwFUQCnBW65oH3OhuuV5cXGaE/VLd5B99/PGoPxgMBlIGl2cXdVH1BiP0XBjbMW0QEbTWXSuNiUR0czJcXs2eLlviLvOyIwQYoyIIQCnUZrK/98Zr948O9pqy+vzTz7yItwyltXY+nTFGR6ORVs45SxAIUiTEoXZuMwhgOp3meZkkCWOMM1dXarFYr1YrPw1PKeXl33fiox422LkiSqkQArX2/ZNd1/FAEmtFGDSNuZxeZf0sS2Mm+HA4fP31188vTru6ERFv2soYRQipqkopZ611jihlnAXBA2PMcHjYtm1RFL1eFobhYrFIkgQgzPNiPr3yUph+tGscpcvl0j92DrKsT7kIg9iLSHbaUxE1Y8xZ6509s2aX4ltr40jsUPSdd7TWSrkpIvhv5x28lLLT2rcVxXFMKfVMjiAI/MnZNRTsEm7/CTv37KMuzrnvv9qNL/eRh9Zaxj20TqPuurZs2qpplcWD/YlSZrmulTLaYlUrSwQXiaN6d39eBzNgyznYJXDX0zW4BgBcjyp2LuS6id9J6l5/wfVEE66libBJH30YsfkNW2U9xog2OorCR48eyYAfHOx51Zrt+5y1PvNGSlCbbpNEUkoJOgvWaeu0sRIRAHnXqbbVXWu7zmptHRJCOOPUGodoAYgfxWoNOTu9Wsz/f8z9WZNsWZYehq21xzP6HMONuEPOlUNN3YXqrkZ1NwA2YcIDKYIgjSa96UE/S3yhzGB6lEkARFIY1Y1q9FBdWZXzdPNOMYdPx8+0Zz1sd7+RWV1gm0gAOpaW5tfDwz38nH32Wutb3/q+n11dXQFAWZbx2jEmON+CQPvs0DknpSxyyhgblJPjowfOucvLy/V6XdeVEDLPYFNtpEwvzm+iPwjP1kIk2rSEhocPj9Okahu3Z7beCXgY/anhP5Ev9a9jQtsF4L517XD/Z98Fkyilcb4uhkxKKe7cmxCRbI2HXr5VILsexDddxKLHBCIGfPlZhJl9iWy34xjbFXT374/ZvNt2F7eNNslC29YR/EsSgQQmk8nBwfTFWei6brNZK6Wrag0QFstb5xxaBO+cY0U2DQGDY+tVrYy9XVTGGOudMhoxMDZGJvu+je18ay0GoJR6KYUQ3uxOQtiBiNZ6H9mLSAlxJNoexeQTAIBQTISkFIVgulfOasRECIZgjK2tbwJI7xzylHPKOT05OUmy3CHpLaTZYDgKPiDlgoBIExYVI0zfRysU730xHLBUJklCOMnqGhGzQiQF3zS1UvPVcn11O7+6ulou1rpvvbfeu7IQwaVOBIpW0DAqh1kqA0m0NZ0yzjog1Hk0vVo3fdQXEFyKJNXK1nVrrc1oFknZxhjvfJqmcVurVuu6rgfDghBirVksFpeXl4hBcDocDFmSMMEJZ+Btp7vF6pYQbVRN0HXKNG3/5PLKGqo1m06O2kZ1yj95fnZxcbFYVoPheLGsuq577/vvfe/7763X62fPnmw2FTjirP3i4wuR2d///d//P/2f/4u2/c+++uqrf/mv/sXt+pPvf/+77O233+acp0l6fXV2dXb28PRoPBzPJge3t7fVajM5PBoOxuzqtm46uljNV8vXH74+PZgZ784uXjR9V2Slcfby5rosSwo0OLDWmd6Y3oIDDqyu63vHo3ffeeu1V94bDo+cpb/66OPNetG39XBaWGO7fjOZTKYHk65T3nsqnQlWNY3W7WBwwCzp+vZmfvn5F58eTA8/+eSjJMkenT4ajcavvvr6aDT5tx/+W9P3Xae0AnDAKSSJZSzNE54Xgwcn926u54/ZBQleEBAycc7prq8qHQI8eiAOZweq6z/44rPvf++7xpim2cSk8sWLF23rp1NE4r0zxtq4/2utjfcEmdYakc5mh2VZ3s7n5xfXm7pbLpcH01maphF8i/dA3/d1Xd/e3g6HQ9iZMe4LBUKp1ppzHj2OX5ydTaajYVlsuu5XH334d37/p0Mh5svFYDDo23o8Hq91jRjyIjU6RKK4NYFzGW23tNbW+NlspvVWxQEAVqvVcDiMsieUMM75+fm5t66qKu99U3fz+Tx++nA4zMuCMxkTW6qZc67rVNRoiyXp3bDnd+P+e/FBwLBT7GGRwhlTopgrMMaqVb1cLtu2PTo6IoTc3NzEdCpmG9s2x+6z9kV5/Cfc0TWq67ppGmttURTRQbvruq7rjEHrPEFgNAVCrIO2t71yh4cnNrDlcq371nrkIuOOImn8HWOLu/GA7KYh9lF8L5O1jxb7LfhuoXk3luw36Lv/vBt+fj2uwJ6JhjE3IvvcJWZOeZ7P59uNg3MONgSIRKO7f0lQyjIW3wpCAOciVQIAGYncDxNVH5AxFjwxRlMSEAOg995FgztEbFtjrD9//GQ8mRBCLi+vZ4fTi6vLNE1DAMqFB2j77iWepCBNc0KYUkb1hnOBwKwJRoc8T6VMq2pDCVeqXy0rpb76zvdSQoR3riiS77z95mRcffbpk/PlWgpJkO0Thf/kHYe/9hIDAADZkxZDQAAMAUOI0xB7hCk4t/1Pax3XcyAv4avYJwqwXYG4JUUi3tHz2CcK8XHAl3IOaLcZOaXUUQcvx3Ze6kyEl82O7YIPITDGLi8v+74/Pj4OwY3H4/gyKeVkMqmqSqnOGK2UWlfLSJEp5RhCEyUJIZDFouo7bZwFGrV3AUg4ODh48OCB8XY+v01Ewgi9C7OZXtVYTyaTyOlDxOBc27axFxlLBaeVUopRCs4ait4kKedG9QTAa+W9N8Ysl8u+79M8nBwfhMOpVXpTrfI0TRIZdxLo+1bbm9vFZtMFTwgXhApGhXc6qgUmSQaEKtV1XS+zBMDzhCeZNE5vmup2edX0Ffe8qurbxXKzabQxSZamaRrANX1HiaVgNPeSWDFIinx0MJ14FOuqNnqpvUMk4KFpuuV8DZQGT5PM5XnoO7vZbCILO5KmnXMQSJIkw+GQELJaLNfrtUw4J6RX3WazPjs7E4KliRim4071UknKOROUMLTBu74fj4fe9p3qtQ3WwcHBKYXhvXsPVuuNtfbievHJ548Fp6+99gpQMhgUb73x2uF0orr6h99/709/9ieffPj+wcE0hAB0NTkgVJCTZPbbv/Pqd394j1KcTEfsvXfe7vrmi68+v17PL+r57deL+48eXPTLNdEN9RvVaa2HIp2xNOmD3vh7bxz8wU//oKo3/+Tqpu7WvEAgpEiSXHIg2HX1aDAczgbtp3UP3SAt7cIxwZCG8VF57/hgU/XFM9G1lbW+uq3TIhselWkquAhFQnulTGUIIZSlTNDVskEkB6evKe+/vFpDfnjeWFctbzq3mK8ePHhAKT04enC7XCjcJBl4H4x2NaGJyEIgIh0Yll+v+psV5Np4B0YZRRwgyJQcHU/vv/KwKGW1WVizHg0zipKfjPM0Z0y8+eDhk6+ePHv2Qma8yHNKed02WlkuRSZkQOICciFCgMV67QkrRmOPJB9P180GGEG6bUlGm7WoNtrWdTTzDUkiZKKtW97eSs7yPO+6ThkdNCRZuVg2600Xgjs+nH3x1ZPD2bhr6+Ege/jo9Oz82b3J4QjFcJAUg8knX355dn5VDkabpjfWSpmuN9V6ve5U//qrr8Ep6brus8++CAGzrPjii68QsRgONk07HI/n83k2GC7rqg+WZVlrLSHkajlPspRos7XCE5wIDowGSvI8i86OsYMQN6YQgg0OAJARJjlYjPuQByBMOOdu5ksX8PTBq13Xtb21tu+aZr1acc4xBMFYKmVd1zqSGLzHEOiOSmmttVoXgyHnnFFGAI0xulc9oZiCtTY4PyjKk5OTsiy7rqsZz9NsVa+btvLOeGtGgzwbZlKmJ/cfTqdTbeHicl6tO5kWIRCt+iwdbZbLOLsMANGPihAmZRrDN6WEsZd9X8aY1u0uW4ogsIuclbvsdETcR5YdH55wzuJWHmlrSm1zL9xOOnjvnfeeEQEAwQUbzN22RdiS2sj8Zlnmg3qtv9qcISJhtZRSckqIoCQwisE5rXok1DkwmgD6EAIClTxhnKX5QIhEigJBto1ZLurlommNIcIZ7ZxzDD0BtFYHEiijWS5W61vKodqsynyQ53m7Ua88eP3m+rpr+zRLsizTXW/AxEjhArm9WZ2envadfvL08Ww269UmzZhMBsYYpfVsNg0BGc2qtZrf1kwOf+u33pIJU101GAwAshdnN2mxFpz0fWu0J4RFS1hERggD//K07EPgryP5+8gaHeNgO/IB/4vtixDcX/v8Pk/dv/n2Q6MrOARAIAQBwIPzzuV5vv3DSAASMwdnvXGYIFDtEaxHEpAEQO/DtmW5TVgRgzMxqmuPW+7zrtPvvdfWp2mKiNYHYwyllFCeeNDGLU0d3QhTBBJilukoonOMwEuZUfAupo+BJRZ5MhjNEIfDklC4vb25uTiLo1i3q00iMwc4X7bT0VTwrNusKOEdoLOeUi6TjFMKLhDCmOCEEGSEAVvcrvu+BQOd1845wXhR5FEUQWtNGYHGH0ymRZFppZqmySS3ZW6MylmCiIFzW3jOeZKIGEoNEA202dSc0DJNAJLggbMiBz2592gyHDRt/ezJ0xDcZr1crzet1r112uOL86vOk4PjYwqU0MBS2TvlnPNowQdrWggmT7lWtaBErxbd3AH4IeXQW92sFrrP89ybtq3XWVp2nWqqdjId9V3Xel+koiyHGNpNXxcqPckOoWarq+WyXrM0q51adn0XnB/l2oe+s7Vpe4UMOU0YAaK7brW4SpKEEdI19fmLZjlfEEIo0LquV7QaloMXz6/fe+8d9HKYjcEm1bLTvUPn+mZxc/M8lWSzfjEs0yIb17X+8KPPlPFiWJxXl3/77/7BfLUeTwfvvvtda/3zzeWLF2dZ1/LhdBPIX356/v/4F381Go3KsqztAWT41UWV54VRs88/r0azEQS8enFx+vYby+rq2fqMSSnni5tnz549f/5ssZoHMIv1crmcc4rT6XQwGLRtGxXTLFrnXFmWUUYNACilSikDXghBCPPg4m7GGLPGJ0lmjEOkdd3MV8tYlXLOiyI7ODiI6OLB4fjV1+5PpsN1dfP8+ZOuraOQp7WWEACgwTokIcuyxe38c+tevHghKCvSTEhWVRVjrOt6pbTWJt66xtiADoGHALc3ixBCkiSzGXEWGZWbTSsHYjXX7/z2a+++99Z8fvvZZ1+kGb++nVfLrxAxkXIynh0d3RsUw++8+86j116/urqImlllpzvVG+cBiAt+03RN21vrCKNMJAAQnA8hRLnTWPxF2FkUPHKAASCqDMWRpDh/2Lcbzrm/I/0WWxJKea31Rx99RN57d7W8mU2/Wy0Xl5eXR6NZURSr9Sb26fddAGON1hYAoo1KBOTdzsA3Oq5GUmF8XJaltXaxWET0fgdFQtM0jIp4pWJr3DknhHDOWesiSiGkJDuZAReV3e4IB8GOZphlWVmWx8fHi8WCMRZpE7BzmYp8T9hNSG7Hqb3vus77sA+rTdNEj3nYARj7nTp+YvS1qus6TrL0VhnjBKeSMyll/CKTycRau9lsoo4sodQFECKx1jNGKEVKo/AAhACUonMQ7V7iR8XPRUQAf7ey3IeNPaYCd/oI8ad7GPhOKHo5Wol/XeC6G/nCHfrk/pn9/AUhJOeCIkDwRlnPAkFGWQBKtVFAgtmWpMA5I5ILydNyMBhMRsNpng2dJTfXq88/e1J9feYQLAaHIVL4HdnS9NpNSykMB+Xv/M7v/N7v/bSpu3/8j//xkydf3Ts+NcYopSBgr5V1VgopBA/eyYQ37UYplSTJeDzM83y5nMeFZJ0GAGu3eK9z7upydfbi+mB2VOTjROaXzbJtFCXCWo9AhGCMCQjEGOM9eO/wr1Nj/GtP4/50/aYf/W9y/KaPjusZduthj5BFkD9e1ngXxPvobnMK7gAAIsmttV3XOeeETLIs45wTtm1jw05n0+i+qaEjJCtyF6wHZ62OmwNFoJQa54IzdCscGXxsEhVFtVx758q8SATPU1ltltVieXN9tV6vCSFNtcEiMCbAeWu195YxFpCAi9BEgFgwIOm6DikJBIMxddt4CDHliuQnFi1LfAg7N6xocEApeuestVop51wIvm3beL8nXEQxqO3okDdRJz6KjCdSjIejw8ND4eq+7/te952JVAlgnvAELXT9prceqRwPytFw1imjtS0JSCmdM5RSJJAkiTE0eN/3PecsdjzjXx5ztihXL6UUQjhnGGN5kXpvk1QMioQxolRH0VCCUvLxeHy5XGhrjHaa9L3zWmsH4DxkaU7QOROHoMB736lOtR3Z0NFoJGXa973qTd9rIQQFSgi5vb29urgsyizLsh//+Me67w4PDzer9cnJMaD56uvPri6eceqbZu29/vzzT631V7dXBLkPfFO1H3/2yWKxHI+n4MP9k9Pvffe7nPKES60tAlFdTxC1UjddIxl/eHr/WlxeXl4+ffz1xeVZUaRvvfPa6+88SkW2cFjNN+z6+nq9Xkc7cM55AOj7PhJNJ5MJALx48aJe19ZaTmiSJE+fPnXO1W0Tu8ta69YoyhjJCCXMu5ZzqbXV2pZ54b2XaaatX682q9Wq73vO08PDwzdff/XZs2dCiEcP7v/gu98bjcuvn3wxv77ihAPd9tI4oRTAhEAABaN931+enZPgiyI7PT311p2dnSF47wCBxtgGAMFTv6VtkzhikOd5nue3N5vRYU4IMcQ+fDh66ztvjEaTJ0+ezOfzssiaWomksFo3rdZ6XjeqzAdCCELIG995O03T0WiUZXnb9+fnly/Oz5brTV6Or66vl+s1Q2B7+jGQ6JYUt4kI1mVJOhqNTk9P45hsZPNVq/VyuayqqswTAB99KFxcWM4hSO89AqWUt223WlUX51fHh7PY1j08PLy+mTPGIowZeZFCCGNcnMyMpyIOSsUzMBqNIicghJCm6Xg8zvO8qqqq2tyVFJRSaBtnfGFPJwwhRAoFfBOZ34cr+LWoCQBa69FoRAjJsuzi4sL7rd/afk/cIy6RahDJB3FeJmowRx9LAIw/gh2gGscs9x8UJVMirS9NUweBUiqFFBwRaVXVkgshEmNMtd70vZYitz5sJzvgJe30Lgsd7sT7fVwnd7Qc9t80/sh7L4Tw3xRO2P/oZRl35wTGjOdbT94Ngd/KFX69ho7NEa815RwDoTQwjIMNHggKKUUmYlaEiEzILMuyLEvlWMoUGVfWC56MZwezdTtfVbe3C4/eY5xKDB4BkXiE2UB4D+16Q52ZDsuHJ8fDMr1CLwThkgWPTAoGHgwlQgITXq3KsgghlGU+nU5fffXRwcHB8+fPv/jiC+eNMdsywBgdT/hyDk8e3+ie37t3z1t3dbFaLRtEHrbnMNJftLE6eIwJ6d0g/esdgf/Ix2/63H3DaJ8SRRpB3/eIGB9HpnO83WJSG3833HU8N3q9qZfLpVKqKIrpdJqkWQihaRou08jzRcZhZ3QeOiKlREqRUg6BIPGEhGAggPOOY6DIAMAED84Kq9rlyntPrQXvTdeubxerm5t6tQzWImOZ4GkiKOEYAIJHCECIs04p0ytNCHMehHABCRNcSkkFN84455TRQrAkS6OgO4GXSEx8FBN354wUAgDcVrwkrvQA4COBTylljFJKGdUG5521TquW0PFwdDCdZFny+smDs7OzqqqQWpkOO9VzkReD1K/WqaMSmMw9T7MsL43boIe6ruLH9X0fXYWVUm3bGqMgg61UjDPOOcoQEXsFAMAYGQwKpUwi0+D8crkcDIvZwYhT503NGOFMRKBx0/XKOOudMyTaPiEhGEIiBKXBKm+1RQAhmNK2qjfYkzRNs6yI7Pi4KjKZjcfjm5urq4vLcpB/8cUXr772yp/8f/7NYDD43vffOb1/SAmkWfLbf+u3vOv7drNc3fyTf/JPKKXWuzRJpUiTLEfmi0E2GY7Ae/Th9PgeB9p39uuvn65X6yIt8kQSgucX15zTyXg4LMv5zc38et509b3To8OjmW70V58+bfo1DRn7p//0nx4cTmN6yxkDZNoZznmzWcdmCadsVI5iyTgYDBpdVVW13qy06YWUnJLAiJDSWZ/mifdhPJrd3iyDJwA0qrgAkPV6/dVXX1GS3js4EYxOJqNnz56URTYejqQQ3jrwgRHOCDOu45QESpyzGJBRRBL6vq/XlTFmMBjMJtPZZNy27Xx+wzlnQhIiOI9tYIporAuIRGkjRMo5f+WVVx5//Wwxj551hcL+Rz/6kRDigw8+6DrFOV2uNpxlIUqlUA6Edsr2ehkL36dnFwAwGA0fPHhwdHQ0mh2cPHyU5+Vf/vzngMTH6IWkNzqEIASNhbsFjH33WJQrpZqmETvGXFxSccYPQ6yktwS6GB3H43HfF4zg/fsPVa/KYvTs2Ysf//aPHz0ExnAymUTbhYj0xO1GJlnf6zifGd85pgURKogZcQyKkTITdYX3HVPcKSzxIIx28Z9wh0gohKDUx+AUv1G85UUiw47JGBfJHsaQUlpr+76/urqSUkbRiNiV9N5vNhvGWOzLcM6Hw+FWvtS5OJMZuQ5Jku5Rk5hFxRfshx3iXhnjtPdeGQsQEGn8ldVqxSmBgJPxNI5xUyK1Dc6TAGitj0mCtdt3DiF4b723AH5f5FGKnFPGtidkDxHv43f8AyKbch/L40/3M2n7GBAf7N/817OEb714/xHfQtpjOtK3ClNgqWRUUBK8C8YaZfp8kGf54ODwsBgOKKUhsqw5sx33SNdN3zZrIbI0KZAzkaWeoKfBbyno6DyJs/pN1Q9yXqQkFXSzvG02y6ur55NpeXX9wnmUMrXWKGMIYT5g0/Zed1E9ejqdMg5d3xB6wAWVCY9uo/uMh5AIpMvlvNf9RbOxWZbNbzdaAaUgRGJB7/mt3jlCKRfU2/Ctc/Wtk/Yf+fhNn4t3mhSwi/0xPuHWV5ZIKfcTPW6n+rVnxsQHStt2U60X881msxaiaZo8zwkhhLLEOS8EUs6cowjgOQBUfV2WJaFAKCDhJHjQ3jnHOOWUBYRed4JzIZlS/fPzhTeaUswT6T1VunW2J+iKLPXBIWJRZGmaEsJ63nsHJCCh3HptI+kYPCB1AQljo9GIJxIJCRhc2I0RCU5gS1qE3U2xg9iwbVvnDB0OhRDBeaUUYhCUkK2WiaEUo6gGEmiaDSVAkVjXt1WHYNdVLm7g5tnjxWLhnBNSdtrOV831ciPTvByOkpLKpKh71fVqvanrppNSMkLjXtH3fazrAKBte2Oc3NE8nXPGKjQBACwK59xwOMrysm370WgkGP/6iU04HY5yDFp3fZExRpxS6vz8fFU3yrlAGVJKARPKNaAPRnV9kmRcMu0BA2Y8lZQFcAG2G7IXHgKx1sfFEJV4Tk9PszxZLG5fvHjxwQcfIOLz8683zZozKPPkv3jvf6fa9Yuu/vzLJ4tqE0KQMlXWeGKSNFemF0kSAl6eX2zW9eXFddf1RoeubqxWpCgE40mSLBJerder5e1kMvnRb39/ubhcrVa6Nbfn84/pl59/+SnhMBqX7P333/+9v/27x8fHfd9V9Rq3vNzAOY+b9f2T0++88Z1qve7qRjCZGpYk8mau1+u1TJIkz3wA76FpWplmjIl7904//PBDzmW9aRljTd2WBe+79a9++eHV5fw7b7w9mx7Xm5XgLBHC6P768mpQpBxYwqRuDASbSsEINk0XIKRCeh/athGccsZms2lZ5gSDs7ap67IsrUNr/G5kLwSPkf+tlMl4dn5+Pp0cpWl6cDDre4WIJ6+eAMDTp0+//vrrsizz7MBZBLTeO0KJkEwI4RGMiRMUlAhZVdXi2fnVzVImSQy6WZYVRamtiaiDsyFCTPs4igFibI7p6mKxuLq6YoTE7rv33moTwQDd91prG3wAIoRkQmRZcXR0lGXZxdlzqzVjgnCuut57eO21N7785P0oVRSpfHd5hTEAx9CrtQb0cXowhtiYEwBANAO8urqq6zo2OPdNBO89EoyCo3IvNUpInIILARhjfd93Wu0TlAgt7BOFuOtZayM5gxCilIqugPsUIeZPXddF6LUoijRN0zSN4wzxK8T2VswY9vtmXJB75Db+DWVZRniwaZq2bVulhRBd13nHJEejnbVeSjkajSjlXasQag8sAAlAI3iGO93ofcghuysVb1q2tfVi+6TqLkLgd2Px+2NPQUfEfaWIO1Mf/GYbYn9sg8qdn+KvPb774vh/a8DQQHkADEgoo4wjD4T7QEMQQBJKMkqZ8UFbjw76zua5ZIIThdo623SbtulMhwwxUAAbQgiAEVFwgLHTMx6MB4MBJdD3TbNRwzEh4JjkjEPbNcFakrBE8uAtSAxgrVPWqfn8dr1e9X3X9/1qtUQkeywkdugopc5JY8xise46PRwOKeWj0aTve+/dzoiccM5jjsE51VGX8A5N4T9VivDvP76FTm1Ze3fugngrRdgvKmTsAuhLg7H4MkogkbyusWkaY0ydpoSL4XBojaaMAwClNE1TJhJCiEYfOQhKyYQLKSUS8NYhCEawbZq2bfM8H40G7aZ6/PhxffssTVMhuJRcWbVeLRDCZDrknBtnOeeECQCQSlZVDQ4SygGI80AYI8iokEKmnAvCmXUuWOvBU0YCovEuOMsJDSGQqP4eWb2Aca/Qro8Jv5SSU0YptVZ7ZeLZ8+AIYYABMVAakATO2XBQgM+6qh4Ny7xItOm++vjTLMums4PpwaEHwvP5elMTIZZ1q4xLAtvUTacVEIztG4csbpnGGAhICHHWexe8wxAwTloAQAguum/rALFlCYRUVYXEzSYTxoOQxAflbW+tJoSVZWlN9/z589tq3RkTKCVMEGI5IAIqY7qqlkwKkXhqwCPjhLMUMfhAv9GJ7nXf953tYuNjUJRa95EG/tZbb9V1PV8u/ur9XzR1dXpy+N5337G6u7lebOpOyExrTZls2r5uK95pbV2WZbf9ejlZDwaDq6ubpmkpMI/+9OGDptrMl9V4PC7KrG5WZxfPgLh33nnn/slR2+tq3V5ezKlM57d1p7uL88V2Aj7OoNftRiYcKHJOgzOEkEFRvvfeeydHJ59+8olSCsN27DtN04ODgyzPA6Hrpo4BclM1eVZKmTa1okRWTcOYi+OxxriuXTRN541/eL+eLxZZKp1Xi5trrzu8d49zxgLtN105FrFcE4zE0+e9wRCGwxEhRHJmlK7rWrWNZDThrFbgHQZPAnpE2I0EuSwrtNZ/8sc/e/To1Zubmywr2raVMp2Mhr96/xfGuKIogkeljJTZ7WIznc62k0ucaK171UbTMy4lkylPMA4y9Fqvqmp/t0eDIsqQC0qcIxBnfAKElzVl9G5HxOC2ra+YKMTQniXUWt0bC0hDAOu9cyFN08PDw6PDe9dXF5PRwYe/+iWn7INfffzDH/wodnyibYQQAnA7VRi0JoQhYkwUlFIy4eyOSwLsoPKu625v3Xw+BwBCX8rRx5tHGd13Ou5uWZbFbT1qSyCSmCdBIG5njHm3JR92CoP75AMRI+wR1Z+iatDdYEkIGY1GkcMRK/6+7+PG2nVd27Zt2+0TkX0lvW+4kJ3+Aewan2maC7H1j6covAelTEzGKOXW+rZtKU99INZpa70UEXvwgB5JoJQSCoDeOeCcIqIxgVLkgkZbaiHSuxX/PlGISVv84ngHEv9W73n/pL+ju3AXVMDd8EU8vhUF7+IK8ZnOgzPetEZKHCT5YDgqRwMm+GJxK0XiFNmsDSE+ICBQRMDA0DGCiJ50dW1M3dUNBZScog/oaQjB+uCBBh8IIePRqGma9aZ7/uzslVdfnUwns1natl1R8K433ljBWOCBog22I95mZaK1pixE95MkSZwz4/H46dOncb5xD1bBFhQJMS4o1fc9GwwGUh60bXt5een8NpmmOzltrW0cj/x1RGF/Qv4jH7/pc2NDYf+asBPg2tIVdx3e/TP7+wK/qf4ZmTdFUSillkvVtXVszLWbigoZb14hk7iBc86L8ZAjBGM7bTznFEBKSRG8Vr3RzaZq25YDWCmr1fry/OLDv/rTSE6SmeSCAoHT03vvfe/dfFBWVUU5M8b02tZ12z950vcKAyGUJkki04xSThjnTDIpXZy9DJ5wygTjnLtgrbXeWEIIRYKIwbsQgg/B2sDIto8Ub59dF8ZEFREkATwYq43tte571QZrsjSZTIrJcMwPp6f3jg9ms/V6/antkmysnf7sy882na673gEthiPjQ6t0a22n+ths5XzVdd2mXnpvKeV9r7wL1voQMASaJBmjMoTtVkAIAQwhOI7UOVfXtfVusbgFcASt95oL0baVt8rpWmtalgcQkvMXz1d145wTlHkI3gEQAsEHbZ213mggDHxwzhqjGWOcMx9YLIGKrMiz0rnQtq1qVdd1SnVr541Vo9EghPDd73735uZmcvo7jNCz58+M6f7Vv/7Z9eXZ82dPFotb8DaEAOhcIMAoUsFIIEyILAdOgfFiNJwdHg0GI+fCeDwO1j1+/LgosjJ/Y764f3NzTRk66gbjw+N7J7eLOfLEeTae3E/63oNjiPj5558H9G3bOOcoFZJIzmmz6TnniZBt23744Ycff/RRKuTD+4Oj44eImBfFm9nbo/F4XTfLTTUYjf/iL/7i7PzywYMHL56fbzaNUgaBItCyHBBkjENZZM6FxWIhmDBKU8qt6lvY6LbxxgrONuta8iSVdL1eK6WESAghfdc455IkMVpxzq+urjCEIksil6frOiQlpeA9hOAIEkQE7hEpY8KDfvH8sq7bum4RaZz5uZ3f1M2GM5EleQBirTPOMpo0tXLBCyFEmqRMuOAp50CxLEaxwt40bQhBCJEVAwC4ubxijHnm4yz7vgDdxjC/VRV0O78AYwyGsCMTbbsMSZI400aMnVAMIcSoBkCOju5F+r1WZr2q8yz9+c9/8Ud/9PePj4+NMScnJ9e3q4jLAQDnPOzoEVEOsm1bLspYr8QPilV7DGl7k2vKtkIfsZWotVZaIRLvfdu2zrk9IcAYA7AttUWZxG9kjDHOxxGJsCNqxXox0hK11vP5XGsdoQvYzUqQnSF1TCzi6GZshXRdB7s6MssyY2wEEvaN/3iSYxUVcYhotRXztrwsGWPVet7WraAkhKCUqqoqtsYppc4Gxqn3ru+jNV9UxXXeR4k0ICR2FTH2GqJII6WEUuK9uxsbwsuRs7DPt2IScDf874/9C3DXpb77mj3wAH9dorAPh/uELD6vPToblHedVYGLpPAyI4TS4EVXu66pERtKGaMiZlQ8SY3yzrmogKu19g4k55yywDyE4GzwQDE4CAAeq8721iHjz8+vem3unZ68/vqrf/WLj4W0fQNFHr7/vbc5F0+fnJ2dXVoHSkDbQppC3VSIOBoPtNbOmYj3hhBwN7lujDHaITXeey4ooqubFaF+MpkMRweEQnTAugvYAAAjfH+67p69/39LFPDXWg/7bxFzgkj6iet8SzW4c+zfx+heSplIXuRp7DYaY4yB2LmLOtBSStV3fdcyLgAgaGOECN73lNpeMUa8sQDggyWARVHkUvSb6vbifHVzHUjQTltjetcxwfIiHc5Gb7zz5mA8ur29ZZx3qlfKzJfrq+WNAU8UMsaQcc4EYTwE9LBb+YwSyoXgXDJEjOCUalpKKVIWQoDgvPckQLDeIol0xRiGBeNxGxScRAFsY7VzJgSnTa9UF5z1Tvf9mEwGRZYeHM6GZbFc3t6/f++Nt96sNs1XT5989fSp9pgUw7FzIs0CAnBmextISHKRW2lsl/IkSYXgCefSaEuIiKikcw7RGdv3yhpjCPGEBsSQpunh0WwymeRFsa6WzhkpGONBcLSq4yxwIijD8XgsBb08P7POASXAuffBOBuHjpy1knFvrAEF3pGA0SQCICDyWOlFOtdW3hZopG0RJG+++Waep2mant47nk6nK9NlMnnnvRGFwCg5PDw5PnqQpfKf//P/mXMegi8FJ5xTStMso5SOs2mSJHleWmunk9l0NF0u14h4dXGpvAaFyDAwZClfLG4//PSj9qQDxtO8RMJfXNxyKRxIQggbDAbX19d5mQ0GZZ7nZZl3uqcUe0qzLOOUVVX19PHT5XL59u/87u//7T9gmb++vrm6vs6G5XQ2o0KOD2bfeefdy8vLrx4/IYR8+eXjSEHPkoxznibYdR0hbDabaW1X81VVrSlSb9vgPKXUGVWt1pEKOhoMKevruu66bjYTGEgkqQ0OSueCEMKZJeciz3NEnB1MqqpSNg4ub33bKY0Fmeu6Li8Hp6dHxrgsH9R1K4XIsuzjL3/5yiuvql7XdX1877TvzPXtajgcaWswhCTNhsMxUAgEPay997eLZZqmIkkBYhkNWuu2bY9O7qEPWmulu4gw7xizQQjhrYvrHhEZ2fohkV1x3/e9URp2RXCsvJMkKYqSCRN5BsaYxc1t33Y3VxdxTuHFxfrP/t1f/Hf/zR/N5/Ob28X55S3scPIsy7pexzZBDI2q62XCY3yNYTUeu93KRfJBCC6G8Bjate4Jo1Kk8U0ikEB3ZkhRaSoqKMeNvuu6gDxJksht3nKzhdibWTjnorIyIg6Hw6gcZ4yBHfsh/lYcCYngW4yCUZjZGNO23V5Y6S62H9OCSLaoqiqiuDtCkG2bvq7rYZlyLiilIeCLF+daa0pZ8J4xZmxwzjEqvO/jG+7n0/b7O/k12cQQgjb6W3FiH8LvFoJ34/q34se3qsbwzX4E+aZB5bcC0j5LeIlY8DSg1861rWq12dTt2fklIYQSiCgMQSaF4ExSygBgMBnGyOScC9ZZpWNblFPikQQMSIIHTwF9QBKA8gStTYvh109ffPn4yd/63R/94Ac/sE4tl/O6Xr39nVf+j//dPxqPpv/in//rf/Nv/vj6+rYYDThvRqPxzc2Nc4FSeju/ns/nfd/GOzRKlVJKrfEhWGtb7z1liQ+4Xq+7vi7L/OjoYDqdnp+fP39+tl6vnXMIlFFGKfU7XYG7p/SvTcv+4xy/6XP3MN4+/4tHvBfczuODcx75wpH0c5cwu2+xxew8LvWICGqtAUTEcXtjI2eoqipCyHo+n0wmRVFAzEIYiUDdsCxCCLPZ7PTocJDlZ2dnFy/Obq+ui0EpBOOcEkYa1TTN5sX588+/+vzd777X9E3BCsKZZLxwlmcJ6VtJJVKLSAWXAYnWVkUBV+eQErpzV9/mymTLxtjmSbFqss4HhwEymQghvHNN0xjG48lJBL9zP2KaZkNRhuC6pna2JwScs9r0bVtr1Tz++ktKwvG9g4Ojw8fPn14tF5tWa2Pmqzlp6/F4mgiurVoseiFJ3zRt2/BE+p00C8SOLdBEJlmeaN2vq0UsgQjxDBAxXF1dDYfD+/fvv/HW623bPn/+9Ob6wnvLeeItDAaFoCAZHY7KJJY9XMTzoK31HpAEEgARuRARUsWdvRwABAC244wrpRBqY9xqtWo3rRBiNJoIxn/0ox/N5zer1SoRXGudDwec8yJLdK+W85vTk2Mmsrwo/6t/+N8i4u3ttQ3eGNNrNRwOjbPVoi4no8OD482qMs7erpebZpPKrDUdT5MsS4BiytPB5OFoUiqlqrrNihy0toB1286Kge7qumvZuz96+OyJD1AVw1QKprqbjABn/MHrx53SzsKmbstR+eCVV++//kYLEGrV9Pbg3v2yLAmjRRqW6/WzLx5TD0ez6fz66nYx//GPf/yXP//50enMe9+1VVLmwfmrxa1kcjQdkUBUr711qZQMeV23XdcIxofDYZpmCcg//O2/+8tf/iJ0rmnXwzQXQgTrKOJyuaQ82bT90xe3RTm6utWMFYFBt2n7XgMh3nutDSFMJsm6ajatQ0KSNNtsNm+89fqrr7/+L//lv5yMTq4uVkmSZll+fXntXEgl12rNGFs3Vd/fJEl/cHQguaAkrNdVLjPdV721SALnFBE8MVKYq8vHRZFnWU4YU71Rqk+SVEqpekeQOh8nPAnn3PuglGkaVWY5o8wY07XGKC8ltwY8oUioAAjBKdVLzhORONt//dVnnPO27TddW9eNEHxwUPy//s0fn7wx+N3f+ckDp3755adkQ1ynVptFMRi6rtXGTw7H0Y2pqTbH6eHFi0uJoiiKsiy998a4+Xw+Go2yrKRU1HXNWJJIWdc1QVsUhTWGOxoAEUnGC85537da63ExXC6XCReI6JRqtKaUMvSZZDrwmNmkaRpBiH06MpvNRqNRXddZlrVte3JyAgBNv9ysFaVcSNJ2dVVVfdOt5uujoyPJZTnN67puqkoyyhhb6y41bJBPbWpW6/Xl/GZ2dLislrLMZ7NZCL7VnTOWcSxpwinjnPdWN5tNCGpQJGkirq8u7r/z1mq+ePHiRdM0lJK673S9YDJJCtqrPqWi73vvA+eC7GblkyQ1xjjnKaVFUYYQtDbOeSEEhi1VAnZznhApDjsiRXyeRB0V78NOV1FKmWdZ3+lNXQshpExDCLB9BcBOYIewnaLtN1MNCrBlPwQXgoedxUBwjhDKKUOWhBB059tN43cWEuTOEV8/3GyklCKRezwfCaGUtrYDSsA7a7VDA8JTRMSwXNbT8WjTLFwIX37x0Qd/df+H77354CC7unxx/8G9k5PD8YSVpZv9Nz84Plr+83/+r+6/9vDN19/SjXry1dcfvP9x2lUmYOuhHOTLuvOEsCQHIBy5ZERBOze6U11vMUuESKjS+vx6SXk+mUwevvLa2cXNcDCO8t4hBCGSdecAPcXAY6/ba6+V8z6RDALxEFxAH6gHcAEBgKH9Vr71LajmbpaGd0w77+JGv54WfCuZ+xZ+ADsoMS6GCMvF2B+7dSGECK1FLk6sE/Yhdh9LrLWB01r33nsMQWapDV6Z3nXGOwseESBlEEJA3WrVAIBTfb1ZR7oPISQO9HrvAY4ePXp0/+GpDuazx5+fn58v65WnwWmdpul/9Q//W0eg7VWr+mI0Tsr8qsLy8I35Yt629WCYzZtu06vOmWE2FdL1fd+j5Uw455zpOZGcovfWa6esphiFI7b65c65SGmMzCqKkGWZ5AwxKN0ZpRGR0UAJJRQc+LqttdZplg0GAyFE8Bac71x/ODqhClKXfu/Ru33dPHn81YSUh+8+enp+/cFHn5xfXlTNRjl3+vDRYlVr03Wq9msXAmrtzs7mIQSt7TgbVp2lqrPWWmWzrKDeXM+XszAtyrTMZd9KGox12nQ9ISQl/UAWYNab5Yteqa65bvs5glLaZkWMEiFN8tWy1r1ylliJAMG43nsIFJS3IYQ0y7TzxkQ0QmprPHGCJcYY6nySJIzxzWZzdXnjXEiSbDCaZFk2nU4PDw/rzlFZFIQ3GhhLdL3pvV9db2czPv7sK+/99bqOywYZRaSIDGVWKQRk4+Fxs7FfrhcAAOit7Zq+6fsbAG9EtvbgrV9t1hGRbdv2ZDS9uL6OPkSMMdZUTDJbd+z+wwdaa6OaNE0hGEqp5DzLk2JYHGWl1u76Zp6VJk3K65uLzz77tGsrQgiX4ic/+cm905PFYqGUury8/OEPf1iOhvP5/LMvPj87O4vWdpTSVx48vLi4mF/feA/G6c76YTk6un8qKFvczq3SZZGOR+WgKDnnVVXdO71//8HJcrn88usvCePWeMqgyLJltfYuSCmStLTWnr84i0oMlEshEuNAa621VUpr02HVcs7bTgGgc/Dud3/w5ptvnp2dcS4ROOeOEGaM09oCUuLBOWc9BA9IqXXBGoeEEmQQCONAKItBgQuKGCIN++h4GgI656wJEdC2VvV9R1B+q2rEnX5zTJn3dLz4Pi64XY2IO878lvFntIs1OueMc+lcUKp9/uzF299ZP3jwKEmyqqqMcZzL9XoZN7g8TwG23MMo+EgDxd3EdpQuODg4GA6H19fXkfMY3Sxj7wAAtLPEM0SkCBg8UJbkohwOF+sKKENCiKchBCCUciIArdo2VmJ9vyckCiHyPN9sNmma5nkeXbb7vh9PJxSpc8EYV29arQ0FgkibpiE5ckH3m7n31jkH6LV2AUEmnHPqnKGETMcjZ3SUcpNcJEIyAhQJpdSpnbA8otY6SWSs1SIHImzbDGavpb8Nz7tQCjssIUIa+wov4t7WWkq37LN9MI4H2U2O7C/9HmGKu388M5Rup0P3rAsfPNlRqADgWwXqPoztEYX9M/tUAL7ZBMGdh/W3omN8fHMzl1ImaRpNNKSUSZJJKQlhxkft5Ma5WisVezFpkhvjOuUA4OZ68f4vPlzc3gSvX3/11eN7s+lkIlJGqMwL+trr7/1e6x+98e7rr77GAv354K+ePH7a1C1w+fqrr5+8+uoHn37+5MVF11ZE5sCpdzaA4yQY9N5vPRGCt6rr1+ul1j3nD6bTqRAi9iBWq1XfVIkc+mCD83GqHhEpZ8T54BEgBMCtBBIAYtg//k3H/tzePcn/Pxy/Dh196yP2F2sPEsQFEwG2fStw34OAHSBHOd1d94CIaZo65zhlfd/7nXL53Y/bkTm0McZaHW/5LEtms9nJycnB4XQ+nz958uTs7Iwx9uqrr1xfP1O9OT+/GE+ns4NDZJxK4SBU9Qax7uq215pTupqvV/Nl3WxKx63djTihK8t0Nh0BkKZpYxuLUo4BjNlymARl3nvdq7ZtKYHxeOytadt2NDighAZwu3Q3Uoa5ajdCkMFglKap9e729uz29rarm/Gg5GAn5TBNpXH6ZnHTNM3R8UHG09uLG6c0mIA2cGTNuqmWq3I8Ibvz7IK3WkWEsmmaTCajUUkpWywWWnUEg2C8aTblICvLsuvqrq3rug7BlWWhLWrru66r1nWvu6ZunXYAJE0SDEFrDdaaTjmj+rY7OzvDabq7GSMkCd57wO3VtNaGgMbZPStFEGKdbtu2aTpjbJrkaSqj+ohMuDZ9ALMVZQ1WG9B9E2OHcW6Lr0IAgCRJAkG0OxXO3S2/Wjdd1zVN06vWhjhSYax3WZZIKUUa6fadEDwCVPPVxVZmF5x32GkmQDjoGE9kMSiNZSJNrfJM8sGgnE5GnHPGsGl6QDsa5AHJ6mZZbW4fPXw4HA6fPnt2c3O1XC8eP358+vDB7GAyn998/PGHq9UKCNZ19c57704mkzfeeOPpZ88lTUf5eLVaqLbL0vT05PD03r3NugKrjNKq7zmBN19/NBmNP/3009Wm+uHhb//Wj39n3dQhhCQRk8kEKHk3z/teK20ByJMnT25unhHCjO6vb5eU0uDROSSUF2VGkHnvO6ViR9kDOz19IHj2/i8+nN+upcyMMYz6gGCtT9OUCWqcBW+RMiQkJhBJJoXMmWgJCZQRzjlj8dJut13OudamaTqte6VVNNNDpMZGovJWEc8HJBDvBKs1eM+894wRRrkQQggGVuz3DkppCLFl7qNMXtd1zgXOZeQndp39/PPHb7zx/D//z79z//79X37wUZqKohjUTSOE5FweHEzrum0an6ZpXdcAUUrIxf+07r23nFMhmFId57zvrVKdlJJSjD/tjA+UEUKIDyZABPwtYG+dJ44x9D4454gPiAiEh6Ai3yryOmNzIbpMzWaz5XIZfefiAEhRFOPxpG/VcrnedROC1rau67Ztx6NBWZZ936muKUMcZNiYzgkh0jyjQlASrO4YhYPJuG6att4YpTORpGkK3mutoy5xbKYwgkqpKNpIKY1ZBdzRA45Dg3d37btV+L7yu1sjIqL1L40l7zIn9pQ02A0uxoAtJY/kj5c5CjhE7pzZpZHgwCEgQRYZ4b8eYOCOrsPuK3xD0GLfwt8FFbxL6YCXWsLbmZR48mOzxnoXB48dhAiNxmR21zjj2hjvAQHWq+bnf/XBkydPhmWKQDvlq41ijDEZta3K45O3rHPr9XqQlt5aq5VWRreGov2Dn/7ucDIsPvz46mZplbW6DzYQFnIO6MEaoOAQnANndV9XVV1VnFJvdVrm2eFsNBqUZd62bVNrb4MFF7z1wceMEOiWHOoBPWy/JwJGycTfdPym0P6/5vhWzvGtVJIQEvtu+2wyTiT1fb9PHPfk3C2HF7cmGmS3aNM05ZQxxrRS+wSR7OTOmmazr0kAIE3T09N7x8fHb775ZpYncfyq61rOWUxKRsPp9e3NF59/XY4Wo5sVE5wlCWXs3v0Tp8EZoJ5a7YyyHOUgI4T0zvWcc0p58H44mMxmh/WmYRQ3FRhjOWcBiAsao0YyjeaxcZo6WGvBe0rRe9+qzjmDAQCD0co5naYp44EyoNxqu2mapq2XGPo8p8G3TesSFpp2/ey5++yTT5tNnebJifKXz85V1YXeUk/SLHfGeue8c+C2XdfYV42lFIOg+naz5pzzvm1V3zurKKXeW2saY1TXtQEco6Lrm/W6pglVxlZ1R/nGWm2NoUg450bpTIrJaJwlsq2bqG82PZxe+W4Xvrd7SywGwsuRZhd3ktitJs56B4QELjJKZFEMptPZYDBgVBhjun7d9cAYRcQ4wr2vdrbcHdgmBIQzcHAX2owVabs0UbmnbVvj9H6vSIt0Op0Ocdh2fad6LgVjLCBu1IIQIqRA5/q+X7WGG96oht3O152xnEkPsNxUbbWyVjNOxuPh6naxWlVCpknKFqvK2vbwYEIImU6nPoSovoeIWussy54+ffr06dMQgjJaSvn2229PJpPhcPgCz3/7t36Lc/75Jx+/eP60LLJ7hwdpwvran54cCM6//PLL5e0iS/nrb7zaq/ZXH33RKH1y/2FejpfL5Ssnr/zRH/09KaUHjPre1tpEpn2nvPdPnjypNqYsS8a40cYaLyUVCSHIGA2YouDJbDYzNvzV++9/8dUzKYl1xHvPWAgBrfdMSGqd1toFzziEgNWm4ZIBKQGAIGNCCSGkZIzFW3EbbPq+BiCMIaBtmk3fQ57J4bC4utgQQuiuXHDOIaGwqzhhJ20EO1BuZ/hG2M43LMpu9H3vXNA66ruRnaU9PHt6/hd//ovvvPXea6++cXT0fuz0j6ejNE2Dx+lsbK3VWgsxWCwWgCTm0bCdH9keUW1zPB7HPyb+DXE8AaiI7VNOCBLiQ2jaVmnrPDgP4EIEA0J4SV+IYTVNU7+N1jaC5OPx+OrqKp6HaJ01nU6V6btOrVYrox0hNMsSRlnf91pr70zbNkp1jMBwmAvBCEVlG8J9AMYoG4+ycjjquo4GT7yjAQTBRAjBWNd1fd8HH8dNAuecIWjVDQaTrtlEZbA9qMsIS7KMEKK0DdrBNwv3PWK83+v31w4AtO73XJCIeeyBBEIgKjx670PYflYqBSUMnFdKOa/QO0QkNFZ+CBA8BEQKCAE94NYj4C5ysI9A+78n3OH53wUS9s/vw9XdLxVfkCY5AHgI1jpjlFKmU5pz7j2QqNCFjFHB2Nbl2SjnPQiRCEq8U/NFtV5XiWTX14vDw9l4NmWMUM6klAGh7zSTbZakB+NJs1y/9cbro9Hoq6dPGbi2Wrz66JRJ8cFHHz99/LxWDQOeyURyQklqjHE2KG9INMC0RhtzfXlBCEGwRVGkguMglSx06413FpwN3gMJ0egieATAgACR3AsekW5P3l+vyPztLOHfAwn8+4+7F+VbsMS33nB/TXeA2dYgPOYBMTF1uzJx/5rgYTcquW2rc87jyIDmnOxkXknYWoGg0pEkxBjJsuzw8PDRo0dHxweEkPV6vVotm6aJyF/b1uv1kjPGCJ/fLK5vF+azLy3g4fHR6YMHeV52XbOuVgCeUlxdLcGELEmnI7dELTinlHsP42E6HuZ1XVEatOmbukvSwPmWB2CM6ZpGax3paFp1TdMwgkWR9arVXeu9k4IhovUWieOCcqqM9doE54xVmjF7OCvLPDdKow9SMmV13TXIWT4sWZZ1TTe/WrjggwPwKJhsrU5Eih76XgekhG11aZ1zba8ogtKq3qyTJCkSORmUSnc3Nzdpmi6XTdu2iFAOciZk6DulLaHgNi2jy8igSpNcckEpSSUfDwdvvPawKPOnXz9+/NVXbVvHvWJLQInXi+A+/4t0DUpp9LvPsowxVi+vkySblZOyGCZJSilHoCEE57Wxqu8jgydSpK1zjhIZ8wMPYZsJE0REY7aa3yGEmPpHerjpvQuOS5bSjFuhtVZKWe96pdquS/IMKSGUOw+IRGR5MKMIBmutzWKhrLcBjGfsel5Z50sumk6dnV9fPH+SpeJgNvqjv/f3KGfj6USK1LmAGDinjEOaSgA/O5gg4un9+z/+8Y/W9UYIYb05eXjCGPu//Pf/PWI4PJxNJpN/+2//LQXpwQEgMB/QdmpzeauDdQSRIRkUQ0JIXdePv346Gk+Dxw8/+aTT9h/9o3/0d//o73/44YcYQBsADGmaDMpxng0QA2fi4ODg6upqvrh9dnOZpiml3Fjbdr22JguBctkbbYyzMkxm05ubm1/88pdJxpMkIYRprUMcovRgnAENneo9OOYJkqBMhzwABUTwxAtBhCBcAGKw1jqvQwiI1HuLSIVkw+HQe9hUTUzkAX0AF8K+jt+KKXHOGSWRawkOrLZKGe/9YDQMYTs87x2o3hhjEKkxsSVBEbfNaYI0SzNr+g8/+ORf/+t/83u/95NXX32965rF4vZgNpNSRhEwxkmWJUUx0sogJWpTa61C8EJwKYX3LgRfVWvvHSIkiXQuj8IDxmhELAdbJqMPgfEkSYWxCgIwxgl5iavvA2ScXMjzPPqexQHINE2j68nNzQ0ixjQihMAYmy/b4FGKFMEY4/ZyxoPBIE1ErDmSTIzGwyRJNvXK2eboaDIajb33MqGz6eHFxUW3WW9Wa68Vp5xT5r23xtmtJOfWKoIEv6/y93dv3K+jGD4AGOutfymRdDfE3o2ve4CBUpokkr70mfRh6/gX6NZEKcaJsJ+eMEoTiYwSlGI3GtNuHX1IhBQIkAiZB0Dw/mWM34ecfVzZpwv7B27noXU3Annv9zM4cCddAICtUhbBPYgVw9Xl5SXfdaD8XWUI4wWnhFIbHCKhQjhj1hvbq8Wy6uTz6yjjSAl3wRtjZodhVJSr2eGDo+O/8wd/+O677/7Jz372/ocf/MW/+9O3f/iDXDJvTST8IyVACVIvBAGg1qodiu7BeeJdLsVrr7/KGbRtvVxeL1fzqlqjTyl4oD4geIQQ0HnwwSPZkgc9xqj8EoD5a49vBXX4X9F3gDvTKPDN/AB3fav9NYpLcb8a4/knO73ReMr3NMYQQpz/hW0nZffmSBhjnLE4S4KIdG8/TQljLE455XleDvIsT6y1NzfXZDtqxPq+bdvGe59lWds0s9mMUC4SWdVNo3QiZJnnNxeXTd8o1SeCAXivTSZkKiUjDUeUlCSJRMpHw2GWSEHZul+3bV1VjdZGJpn34FwwxnSqzfNcJFzrPs/zyWQcnGmajbU6BM84kZITCs4ApQHRdarGAJSh5EQyZrUnYFS/oYCUUMRgrebD0WtvvGmNR55czVe9sciFA+oBtXXGeSETyqW1Vjct5zxJ87jy+7YxVkspy7QoimIwKCaTSdPWwZumaZJEJomM2dimmreNFkIsVnUITvWGy+RIJFlSetuDd1abzXq9XK6l4JRSIEEZ7b3HZDuqHVM+Ep1iGPHO7xcAITS2mWLxRghIydNMCi6sdU3b9p2OZaS11gcLGhBDvPKOvhyfjkt9i5pFnS7Y7gDGmE4rpZQzARGRk0RI7zk3jErKDedcmgDGhTwvHBLtHHrIi4G3edT2ttYaX8YutnOOzdcNOOsdCa6valO1WxaeBzIcTIUQi8Viva6m4/Hx8cn19XXbNldXl8bZuq43zSZN08Vq9fDhw9FoODmYZVn28NGDLMucs4xRaw2T/JMvP1R91/etZ7q1xrZtmiTeusuzy0Exmg4Pju8/aJS+ulkPBqN33vve++//Svyz/3E6PfjVh5+qXs8OTn7605/O5/PgqfPWGDUcDn/rhz+4vr42uiPFZ0mShIDr9Xo+n7e9VlYTb7U1i2Udwurw3mFd1+tqdXBw0HUdoUHrHgC4FIGAdT2hnrBACSIGoMAkoRSs14wTmTIukDKy46ttucoAIIRQykCAsizLYrxaVYv5ar2uOGeUkgj/bGfEowWwsp6yKNAUgkcExigAZVQ4cIQQRrl32lpvjKMUnNuVs0iDDx5CTGmLNKs267/6+a9+64c/euONN7788tMAjlJUqsuybLNZh+CTJBkMim4yCQg3bTRbbgI4yjBJBWW4XFWUYd1UIQTKENAD+iQVjLE840qptums6Ql6a701PaOc0cQ57Tz42HswBrynSIBuN744BBGntmI/Pg4uxhwitlE2m03TdYzxg4PDum4vLy+7rsNyENsWaSKc000btfEju7MTKX7/t9753nvfv7m5+Xc/+7O2XbXNmhLJCZGDUsqccamMic0/ACQEthMoxsU/IOYosT+y54tFXN3uBPn3273f2UnczRtwRy4jhFDGXgbRO5j/3do93CE3WN0ngglGCBcyS40xPrhOacSAGMCjBw8BPGIIiAC4Cyp+p5D9srj85uP4ubFM+Va1+q2Ydzf7ic1jyhnZDrITBBI8Gu0CWoDYX6OMCe/BWovME0a8t6prGYE8Sxij0PVcJK12Vbtx3gNllAlE9AFx3pbpWPXhcHby9pvvvPHotbNnL37xy1+21aar6nWvNqsNpVQWqTfgA7XeBEIsIQ4QKZVUUEDvDPH+ldN7/4f/+r+UCfn6q89//pf/Tlcvag2UOw/eBfCEeEAbCMK2vxe2Bk1hb8lBts7Uv/G4m43tTtTfPD34xrn99QdwB/XBnZg3vePJvl9yYSdWFn+0TyxC2JtDhbviXNFEEgHirDUism3vi1ApCIVYFMaatevaqlorpRjb5rvGKmMVpTRJ03FZHJ+eWBfG00nd9vPlYtO2N1eXSSqdc5IRyai1NpdckJxR3Cxa03uBQBKeiRw9Ou2kEIgkT1KTuwDogyWEAYEAfjo5ePDgAYKfz2+ODmfvvPP2anH7s5/9SbVeUBIYo4QCIYAckXhje0oxeA8QKKEEgQZ01mLwxngui4hkCJmXxWi1qp4+O+sWTRMArVUEWVF6wQUVIskcQtt3fd/Hmamo6N80TUFxNBqFLG3rtepqAm4yHb399hs/+9nP6nplfSjLMksL5/xqtdbauwScA+9hPB6PBkMGGByAC87Yy9sb3XfW9IRAORgYp6Pqy/7ShxAoopSSM9H0KvK0QgiU0D0qORgMGBNa68VigcCcC84G5xxARDF9COCDjVsiISTQEHODAGHfWENEF3bNCAQfvPEuggcYyJ3uGwmMCJrwRFImIrTgCCDjznmPhAvpgBmgBjIqaTrglNI4Y8jOzuaq77KEJZJaS6aTe5NxPhkUq+WmXtej0Ugpq5R++GA6mx12dffp519Op1Pj7OHhYZIkf/mXf/ni/PzRo0dH947X9SbLsldfffW1116jlMaYUetVtamcVjJhWZJ4SyXnZV6s5qtASavVmJDh7IhTiSIbTY4nR0dEyF9+8OFserBcbZzx/9P/+19tGnX/3nE3UnkmN/U6TfloOBgMip/+9KcP3v1B27ZRDv3y8vr58+fXN/O27fMiUVZb656/+JoS/ujV+5FbJ6XUtlPKMI7WWm0tYSAT5hECOClZXmZZnhBiAZELEqJsiAd42SeO3HjqvQngOUuTJIPAnSWct02tENGZl93rePPTXcm5Dz9RwDWGhIjfA9g94WX7WWFb+e27D4hciuzy8vqzzz578503vvjik6IolO7qui4Heb2oKJEBXIyLxllKad/3xpg0TU9PTweDwYMHDy4uLm5vb+MggJQyCkLHVFp3FUHMJEGkqSSOgiBJmuZt2zprvHUAhARHg6cBibcOaFSfZIxF24WIhUopx+Px0dGRlLKu65ubm8h5DoiM8Dhg6WzwDpwLzrm+b4s8SZK8VzWhaK11ztRNJVMTwBalzNKTq7PTm+vl7cVVcDYTPM1KwkTTm7btIgBjnZOC7kcrs1RuNpvJg9PIIc2yjAvSGdN2KrTtNp3H/Rl+OalIvqmvvN+1I2LknI0ksgAeSYisFR+21y4iKIwzIRildChKznnXddr2zJDtCvIu3ugevPfgEdCjR4OILLx0u95nJPu0YL8B7Z/HHfHtbsDb5xnwawB4vDqwq3T9juWfJInHLesCd+OpiBhAI7rgLVLwELTVjFAquA1eG2t8IMgYlYQKpIxTWuRiPDrGEKaTo/HowBkYlqMiySulbq5uLxfLzXojWQqBGm+lSLVE7xx4whOalzyhUvVtW60ZIyknbz46PTwYPDzMC9YOpf7yy8+fXIKB4J0PznlkAUOs1QNsrwSJ3QfvCEDYMhX+piH/b/jKX//FX88V4E7u6O4ord2Ff/xOqmsv8gF3couw0/fYLsgQwk6xMTiPiGznxBZCQEajdrunW9V2ussDnYMQHCHQdZ11Ouza9vEeefPVR48ePTy/vJKCaA2ckWazWlXVwcGBD5YTXIFv6w0Gxxjx3rHgJSsYJMFS1frz5zeULT0ip2I2O8wHw2pdK6MJQckFIs5ms7Isg7fe28PDw8lkorqGC2qt5ZISCsYowBC9z41Rw4FwXgfrdAg0+OAtQ8IpmwwGicwH5SwvxoKnPtBNay9ulvP5fHZw0KoeeToaDlvVB0qQMxoCMTyA1kpFfpLVzlrrGK1WS04JE2xdLV88//rRqw9PTo5nB5Ozs7P5zdxam8iMc45IlfE0pUgdoyIEtNqpYBgGSvDg4CCRdDAsOOdt3zRNs2mapml8KvfLIF6IoiiKvCTVRmvdd9p7z3YGud57kZaEEAjEmuC9gUAo5UKkOy4CAniECOWG2L/abg53lpnf5QIOQvDBe2+c1dZ0qudUbBdVQAAAgoQwJMQG7yD0xoa6NcYwJgjl1gWltXPOWBsHd6NZDKWUXd8sV8s5RRgP0iyh4+HwYHYwGecvXlzW69Wbb71+7/gUgXIujTHr9eby8nK5XD4/e/EP/sE/ePDo4Xq9Xq1WAFAMynxQrtfrOHVaDgbPnj378z//c8/0eDzMi7Rq6l51JHjO2M2C9U1n0XdNffvRxxST6eQYIOmVv3d8+oMf/ODP/uzPP/r0s0xk4/H0l+9/sFpWv/vjH73yysO3v/NmvB+stVVVzQ4my4AePRAymYwfPLj/4NH9y4vruq6fPH8xHI8JIU+fPh9Pxz/8wW91XRe2EmB0uVwSwqp6o+pWG09E5p0NwYuECEkoDdbZeLICvCQfxSACECCg954gY0wiUu+AMXF4cCyl/OCDj7Zgwp3Cwnt/fHzMcCty0Pe9t1vo3nmzo8SZSGvqe82Yo5R6t+1ZULq1RLLWLpd1kiRt3X788afv/eDdvu+LIqvqSqk+uqJxJpUyVVX1nem1iVtJtJJ69dVX33rrre9+97vX19fvv//+n/7pn7548QIAohl8BLsEV6PRaDoeDgaDshh670PAshz88v0PvLHBASHACfPUI1IhmCV8r3AQv1E8omfjw4cP+75/9uzZV199FT3GjA3WbOKcCCLGokcpRSkmSTIcloAWvEtT6YPlnGvX/OIXf9G3zffe+/73v/9dZ6Br2s8/+UJbAKRAzLpqa9VTJihnAQhAiNMc1lpKs2hG53a+O4zLTddt6tb1vZQyMuHupgVh12vYX+/91Q93GgFhJ/YQC8T4T611fHEUb4/Axsl42mvVNE21WiulCGORLRGHIr33DkK0YIqfxfAbOco+nN/96LtHpLbsM4lvQQvxt76ZMZDYQAUIsUZBuk18rTVbcVJKkRAIBELE0iwTVCaZUVr3xqCTkseZeMYY4ymh3DjvnCHEAVCj/XI5V73VvblWt1Y7BuzJ48/nm26+aZZVWxQTpZTRYTychoygMQGokHRcDHKZVIuVbpuE4Gq5OD97JsjBMGWvPzypF/f69dXz65W1zhkwDiyYQIlHDoTs2nfgIeydBQA8/obBh7udgrvn6n/zY788/I51uF9vbndEhfW4nParbot7iS0HiALuk78ASCmNiYLfDVtmWVYUhafbTkeUC9utUjw/P/feI9l6iVmrEVEIFoWxm7Zuuqaqa20t53RQZASsNXpj1Ga92qznaSIHgwGAn5bjPM8p4976TdtW51cAZDAaiiQrBpnMUtWbum0I80wwQokx5vz8nBKQkldV9f77719fnm82G8qiyBsarQE8IwwArdNNYyiBhDNCiNNGNY3VhiIZZiNrvbVea7vQaxfam+vFclWDlEcPHlzd3hhnRZLUN9faGturtMiBkkikc1p76xgjeZKTXpleIYHpZGR0++zFs2qzmM9Pf/KTnwyHwxfluQcEwBBwOJ4en6QX9ZwAFoMykVnwUTYehRC317dJyqNkRVWv664NIeRlsdbq7m0Yu66DwcAGaJpGKxvXwF7LRDe1FGnsiQePcVqZIGPUbr14IFAaDUF87CmEb/YigSAAcM49QkwYXNgK3mut/VZtliKBEMA7p3ZN7YCgjemVCgGHY0kY7fteB9/2bWuamMekLg3MU0rZi7MPppMJhWD6nidD6MjmejmT+fPPns0m00l2UN1s6rapq68ePHwo08wHoo1//bW3fCDVunnvez+8/+i1k9PjZbVqulaZ/pU3H/GMPL/6+v2Pf/704ovgTrJ0lKRio1qkNs2YC8pYLUcp6zHLpa1Du+5MXd+8eL48OxPHxfR4zNLA+9Db9nZlhgcDZfpN23TK3c7XZV44y5yFYXkaLJlhHxz29WZtFAhRTkqS87pXfsCrqkIkD6Q7uXevGND1enV0dLQ2yze+d+L9cd+q+Xx5dXW9qZrga85YURTDspCSh+AIeMY4peh2ri1R1sn5EDyGEFIuiVGcE8pcCG2WiTRNAGCQF5vNRmkTjEVA4jCVyWw8eHhwfHx83Kn27PKsbpv1erlplznLC3Anh0dXt3OwnKFHYqeHY2cRCHE2XF9dE9LnSToaDM8uLh6cnnYmVd4aTH/5yVfjf/mng/HxRx9/8PrrrxzfS88vFpRk3nGrQ28VBBYsrdpOJslicZunskzYe68/TLx+7XCiHj24ePyV7/Tz8+umCYRPehscBOFJ37Hf+dGPf/D9N4uM3FxfnBwdv3hx8fyLL9SmQ89FMmytbZ1BQsDDgeSQ59775XxeluXRwcHNfB4h1rquDw8P//iP//h2MR+MhlyKs7OzVJTee0BNKS0yQSkNYBj1iMb5jrP8/sm9EFwIGDxiINCNNlfsf/7kz0g9+J0f/63j6fCPfu/3p5l4cf6sVpuPPrsqJ6l2GCAnOGCicGStlZaMJklSLStiw6wYQ++EQW7w0cMHXz9+ZrTOx3kgDCnB0DMKcdLVQ2CUEAoeLKOCIfN+G3cIpSSOOwKhnBK6db+1zkUcOEk5JzSqSQJ4Z20m+Hg8HqYyUfDF4jrYsN40VEjrAAnzzgVCKeGxJvQQfBRhZSRAnJDfbgfW+4h0xOozz3NltDEmz/O6bj3XEOWZt2FpnyjE9ijbBZ5dVwJMlGqIBSogZZQzxrRqKCGMIQA4p73xAMARWcKNAa21cg4RWcIBwHgEoMijuU5LKaeUgrW91mfWr1fzn/zoB49eP+jDAr2b3R+2oV13zfMvH/NsoB0u63WR5YkMwJrXRqfn5+dV06dZgcreLK7aumZUbJpKZqWy2GlCGEuLg3fe+/FqpR/VHz19PrcORMIahcoRwiQSQila1wejwTkIhAZOAoZAPe1/PQdAhMhKvvtMPO5iS3/zBOJuVbB7w5dH2LWNnXMYgDHGBN+zgAOCMnqxWnIpAWDL8smzOCIxSEsmCE/EPodAH0IIUkrV9cYYzrlkXAhRFEVRFJ5Za22RlSGEuq6JoNb452eX5xeXqUiiAKsQyXA4jJ/11fPmqxcflaPh7PDgeMauFzctLKxUq7aWInWgRSEPy3tWG+tDkec+593O5IUKl7K867rbqlc368GgYYw5F/K0bNu20e1gMJgvrjjnQohO0cVqabVRSgHN0jzzxNlgCE85p4SjtVpb6jSkaUowUS54nxk20KbvOmUuzWiQXm9Wr7wymoyK86dPLdE/+sm76xZNgNnxEeXEescL0fad8S7Kti7myy8+/aJv7XA0EpRXVdUENxwMfTY4X/XPr2oDQ6LFL375rG7le99/9+Hrw6ubS5mL8mhwEuxoOnmtqimSrtmUMqXB67aVWdqtG+o9KLh5frm8vq1VEzSQIBgwE1SUqoswAHR90nYy1VKkg3LSd7bvNUVGAvPWEyQUi+Cps5QKzsXWUUHbbmvXR4EA3a0xQgjhhDZNAwQZY9ZaJASB9H1rjEFGAUAZ3XVd2/fWeyoolUz3SneaIHJCKZKUSSEEAgMgvfZd75BK1QawxgNrultElELGppXqm5qEAI4djEfgTd+1KHiWTo8PxsS7s/MnPiiREhu0Dvp6ea2c1cSGgH/n7/801mez2Ww0GRtUxnfK9oyRvtvczucYPA1etd0gHbzx6I1ew/ERT1JvfWd9KznP81SKnNO0XVpdQx+sVt6GftXfWGVzPnvttdf+8A//8C/+/Odnzy/L4WA6niUsQULqprq4gE1RHB5MsywJELyHw8PDrCgCxdvHXy5ur1mSgmDOmslkMp1OVddnXA4HA6QkK3JKKbGMCsq5LDLPuSSE5lkdnItK6ZySAN454xwSBAQSgglbUiEiEooAlACAUYoxNhwOB8XAWu+ca9t2vV6v13PvfZKILJXee290QO08jsbFZFoELNJSOOfmy9vHT54sl0suskZZIdMQAkKYFslkkh8cHM1vl4+/fnY4SOpGWaO0NZPZwe1qwxiLNpBt215cXExnIwC4vV3cv3/fGKeUJhiMcQAEAULwnPPgnVIqE5xzLrnggoKD2Wwym82ub1eEXIcQEIBTpJQKyopUDgflvXv3ZpNsNCyOD2dpmv/8rz6aL5q6a0F32theacoZk0yIsmmaWNMgpW3fR1pAURRN00QrbdX1iEiRQOQbkuB9CNYZ76wz4IP3djQepEIWedl2TV1XeZrFbkjd10obJvnl7c2f/OnPvv/e248ent5/9F/+s3/2/7xe3rzt4fGL68ngMC0ONq3z3iutjdKeYjAagiMEAdFaXbcbXice/WQ6Yql0SOpeMeRh1+MnhFC6HY+M8EAMEgS3QhS4my3cF4KxOGCMEkLSVILzcU/03jprjTFt2xYEjDHO+xDAhlhFAoEonfBSjREgREvlu4wH95KO4BhjveoASJy5hxDABwQPbgvhxJp1T2UgEBMYSwgJLytsCOEbjgN3MfA9hgF3lJoALSGUELpv9sdEOVJ3YwoCEMd2SAiUEA8ASZI450KgUoi67Y0xw+Fwfb0CgOl0mmQZhuCcOzg4cBBklkprjLOma7RWdd+1zWY6LFvdL6r1weGIaoeEDMfjk/unxbOnIcydg9GoSIpkuemVDTF6xq9DGKNAiWckoHfkN5EO8Dc8/x/62FUd38B7CCFKqU6pePU559Zv5RBYSvM8T7PMO7dH4KzSGKVOKYv4gWQ8hNC2bTkuOBVZlkXpFAQa24LGWKUqAMiyYjLk+07o5dUVAPBEHh0dlcNCWfX48eOrq6vJdKS8171yRgfnu6YN1jnnOoeRURTdzvbfwnsfhzzjBEdck0mSZHxr4x4bcxGy4pyD90iAAnIqkpQxRnrVWWu9Vt57FywGQEYEoRQlZbhaLw5nk4cPH37/u987OTn5Wz/+7Uij/h/+b/93QoASyTj1xgdwxihljfdeiiTPy8FgtF61i/maEOKN994yxlKZGDCIiMExioOi/OyTT9p2gyTITL73vXfKYXF+dX7x7MWmr9M0DcYIDCwE75RgKZeZ1e1gkAvJqmazXq/avgESvCeRNUIp9Q5idljXNSFM8MR7zzmPY7BxOJkQEtxWID92gmKi4HZylvtlcwfOdIRAdNMA8HYHHngTtrwEo40x1nsfrI+GW5wXAQiiN9pqZ4MlloTgkDJCKOcsEPTetn1rreudCiEEBETsuo5zhpQgInvzlftNuzFt4p2xqrJaZqlUfT2aZtmIVd2CJoKkgQay6tfj6WR6Ooh+gI1bUQUoTVoyRF2UcjTIMilms8Pjg5P1cqUKfX+6SUbm9TcOkxSvF7ra3NigATSnbJgPbGeMciA8SKeV6U2o2zqnh7027373u12rptOjIivytHDOzcYTRshqs1KqSzPOGIYQDAkuYJpl08PpaHFbd60LLljfdT3hbDKZ3lo3HA2yJDW9HgwGlLOMZpRSKVNGeJoWeTbo+54EIqU0Vule9X2rjXLOYIi0ZL69Th5C8AEICR6ASJHmWTYZTaRMlVJG6WbTLucrmXDJeJJIzliwzlnNOc8Sqfp1r3IhRJnzrBzPDofW9pt6qQJft3qQpV51HN1skr/y6OD3f+8nn376+dcffsETyDioEFZVPTk4vl03BQZChKCiqrrz80suGKPJYr6+d+/EGNs2PYD1LjAmKEVrbSqkM9ZpF0LghCIiQ9Lb3lubSCEkC+BDMARYCAAIVmkoJCIOBoOjw3GRsmE58Da88dort/PVetO6OIxre2SCIqWCevQBgpSSC2qsirzIh4/uW6fX1dI6zXikAWKWJ5JS7zFWz4hxTtQTQlIhGRNlWdabqq6aMh903YozmY1GzrnxcDTfbH7x4a+enz9/7dHJg/uH//v/+h9eXV2cX179D//Xf9xu1vWqAcjunTw8W9otIBECZ4xmKU84kCjNAzGfU0rLPMsSERDVLt4jIqXbm9N7z/nOmwDI3c097rDspaJRoLiNuH7ndmGtNlrHpsAkkxYCUIKASG3Y0pO8gxDVFbf3f0SUAWyEl++MNsTgkSQJIaRpmxBclJGmFCfj0Xy5CtGqDSH4AOAJblXwAJ0PFsIuIYi4ZHgpKrUPVHcDf7jD0ieEeI8ISBB8sBGEiGouTdNxTpEE5zwAIHGUUi6QUgYBpZR13QoMw6Ks6jZL8zRNrV1xj5PJdDAaVqv1er1yLiw2CyFEORpu6tbbkBQ5EPTepmVRNfXVzfX90wPnUVAYjkb3X3n06Oxssaq1Zq++8R4Rgy+fnH397Gy9qQIBHxx4v1VQ8D7OkP0vSS79BzzCNykL22d2ysHhjvJSpMvEQBvj316WLe71RVHEzDXLMkJI45xkfDKZCCFiSysQ9D50WmUuJVFw3UHwUb0ycC5OTk7atm07ZZ1bVOtG9WmaSynL0fDq6ur88uzR7cOsSDnnglMuaNu2GLxzTjDGOFUMkbF8mK8218ow47TQYjtZg55yQgwG9NYb593O8AldsG299aGNHRZnLKHAGReMBTDeW0aClEmWiaLMy7Js1ou4FCP6gkgYE4xhmorZ0fToeEY5Wa7mlFIP6c3VbdfVhBPgQAMJNABinGhbLJe90t6RttcAxAWPnhLG84RJwfq+b5vGam216VyjdC85b9aVst3scJYwXmappGzZNtZtAvfgvbPorHNdpzPGpDg4HJ+c3KMMv3j8hdIbQM84M94i8sg1cbDVM+i6DoAIrr3fjrfgbqKBUiqYIHdm6cOO6Hp3zdxdQsZqH6z36BxxzgHZiultmro3uu97ZbRzzu8wrdvbW0ppKhOZ8EDQgwMINljnAkNAgkRQQKQU43LUVsZfD95HdXfhLCGECeoq1aYCB+VQCnpwkJ/ePwE4efz4sXb1l89ujh88YCWTQiyqjbDicvW867qqqqx2RVEM8gETFAOzpp4ennrr6uX6rHXz+eLJ18+efPVUjDepgEevHN+bHB3PRk27Wa1WXR+W83o1b/sWnWUWaBCMCZJSwrh48eLF22+/++Of/N7ydvnkyZOu6cpykOSZ00a3PRKvdaeMjASwuu1za6x3RZENR2Xdq7rvbN9JzIs0u9Tncf9t+nY8GCqleCJDCNZ6II4QUhTFcDhOuIik/QYq7y0i+sC9Nd57JiDS/F1wGD2sgUCAwahMk4xzCT4wwrNhLmXqvV8v1iEEDEDAeTTIvRQhSel4miUJ+KB0r7ggQsosTxgBJkrnrBBJ125c11DqCtCvH09ORz/8F//sf3xyDfkAGE3qdXu7WslymBCLJBBCkyRp276p2yTJjDHeEWug7621ffCQJIGx0Ku2SDhQGu86a23XNJISZ+zi9gaDSzjjlEjBkFPba4TQtn2Ri8268kYTQgSjxijv7cnp4eDTIgoWcwqcIaHAeCCUZnm+XcqIe9ml09PTvu/bTU0BM5nEkjdP0m2XDhHJdjyPEYCoW+A8I5QSBh6KLL+9vqHIRMHBe5SyMybJCwv48eefG9e/8tpDzvmwLP7wpz9dLZuPPvpqudjY9dxZSxAFZdoaJHFImHkaCMckSwknXdd2ShVlCUC2ziuE7PhleLe2/muPONAhdqK8Wivdq31BsBNWshA9tyjtjI5TBsgIsRaBBO8DQWMMBI+Iwe28HkisieleJCgSkBAR0C9XSyl5maVFkUXrLAxmtaySnZuls875rYkGBo/BA2JMNhAREGKVQODljGt8EK9CbJCTXRjbAxuU8jiTrXVvrQ2BRIlAKUFIFrWkjInGJS/JlZSwuq4Tguwe41zeu3fv86cX3sGW0g/UubDZ1FdXVyzPDg+mQgh6dds0XSaTMB6lRcoxNPXqejnvjAYEKimhtBgO3njjjVXVbjbu8PAwLQ+oLICKs4vzqtlYB05vDQXCjhX+nwo5+OuzhC0G8w1EIV4RKaXZGapxzj2EyHSZHI3zMk/zNC/z6GfT971SPSJs2hrrcHt72zRNURTT8WQ6nRAkEKBrlbXR5I9Z7/OyODo+6fu+aurlcr1er5teAWUiTSAEJmj0jTs8nCWJaDcVBrdYLJz34C0hjLOtbZVIhLpVSqm9R/xOb5R672OJbIyJqyj6F6yq1cHBQTkYIRCAZaV7q30IQTAWtWi1sz5YxgZlmZfFIBEcALTpTa+s1QTBgQvUUcD56tp5c3193dRtIuXx0clisVBWEU8Ceg8uEAzghGBMcB9wtdos5iutXZqX00kGgM4GwRsEUm1W3oaDg1kipLd2s64APKHgO63b5vzFc6WaYPSgLKjTmeROq4QHQGeMJ9QB6CIXR4cTwvDFhYgevJSRvuu13yb3cWwtbizOuc520S0IANnOtp4QkogEd1zpeMQ8KRKevrURxXoAIGIVyjnHueRChBCW65X3AEAQqHfOeMe5EELe3C4BIBFdlkgW9WR9oNR7D84DMk+JpJRSTqiQlFLTBGsti+o+VHkIxjpEz371q1/e3tjTe+T+/d9+5dHJ8eFkPCoCWI8nrephHYbjzFFqga77xhHf9K022gbTm1YvO/CmyApvw2QwGw6Krz9//Bf/7hd9b7xBrf16sbK3NYdL3eD0oMgLUddwfdGtqp7RvK4MYEKoYJRRJoqioAzbVQcASMnh4aFWtqqqTVUXRbFaLbz31ihCQ6c7rignREqZD4qu6+q2AQycc71aNXVlnT0+Ph4NS0rAefTgjNOO+Ea3Oc+dCyFYijR49B4ICeAidcgHj5E54oNzSJwzlKJH75EEKhCRER47smVWci4F44g0k3Q6PWCMTYaTx19+1XWd7lvnDLBAkApKGLUYjDNdQPRBO2fG2fjRgwdt3bw498BMkiROiL4Jm83m7Ozsyy8+++nv/+Ef/uHvVv/Tn1kGktPZbHJ1s5oOByml0ZugLAeUkq4zeV6Ohsya4CwYbZVSccVQ6tq2RccSweO0glF6vV4XuSQYuKDT6XixqhLJJWfIURskBF0IEEKzWWvVgbOUEWs0wZBKmSVMcnRIGKM0IZRTnkjCcDgehBCWy6WxigvOBNW6F4KdPXu+XM6tUQieIDFGO6u1MrjlPCIjRHDKOeeURRBVdVryJJFZluXBeqsMzTJO2XpdWe0OZwflaPT86S1Qsmlq1de/fP+v3nnrna61q5ulbfvF1XOT8kRKgsF7Z7xhjFlvlNZplhlvAHF6OM17DeD7pvHO3VUO2LmLRXbY1vqBII27YdziY/hnNGql2f2+H79UHBYAoHFP11pfzm+klHXfcZ64EICEgICUBB0AQtTcwS1xPwAAkB31HX3MFJAAAimKxCrdKNO2HWeQJAmlgAAMHAIGBMqABxbF2ZwNSALiy+mprWkjIif8bqCKu3zsmMT/xzJxP0YRt/6uY31Pte7Djo43mUw450hCnDqLgzPGaASUCQcgwcftDxFoUQyc9YyBtf72dhGAEUKyrEAgr7/++vHxsXNuUzXL5ZoCZlk2Ho9N1/QdbZrGex9HxaIv83A4TNP02bPnq80Ho9n9bDA9PT3lUjw7e650o1qilQIHBCkhCIFZNP8h8oC/4XG3HLybN4SdowTiS0+EPRE4hMAFn0wmBwcHw3sD7z2gH46GiDifz5tmo4yy3tR1FS3sm80mmt0PJwOBnDCutbbWhQDKaAgkTXLCWcpKIqVMs9F4qq2JjYDL64u+743TH3/6EWWY5yklQAC9NdYYrTUEH5IkDvEqrWUqt/n9LkuIAY8JVpalMWa9XvfRnAKRMCIEG4+Hjx49YEycn0tnbVVVSqngHGXonNN917Zb/5Q0Ta0NAbxWXumthmMIIaZNi8Xi9mYxKIb1uiGE9X0ghFAurLVtr3rnkBAHgTAuOM/y0oXQtL0QMBxMiqwEB8bYTvk8yzjnucim48mwKK0xXd2Mx0PrdFWtlepYQhA8IZgKicA4BWU8p8AYlwh5yglgtV5W1YonUcnNAqC1plet4zlsaaox6m/ddwkhcbwrXme29eQL+/QxroQ9yBRrj2+smW0nlFEClLHIn/YOuk6vVtXtzVo7a63VximlPMJwKIo8Pzo+7btOKaW0NxiCsyEECjSEwCRkVDAumZAQRWAQdW+894xwCIgBggdnfAiOJXn25kx+773v/Pi3vzccpF27WqyvAfyDh8fK29eoyMbj28X6erEuiiLNMqvWzlhBJYitorhSqm/VK/dfy4q0N/rq5lq1TvBC8rQcToM4saY4e+FWi3WWM9VvlkttTMhSahrHBUuK1NNABc0zwaXoasNl+tVXXyUy6zqVZFm9aS6vrmbTCQZw4IzTm7YytmecFEU25JPeKaBQlKV29nY598Y6Z53VThsAsEZxTjnnyCiTApES4glSToVzoeuUUp1qFGPMWO09EGSEkRAcAUREHywiZYxxyqRM0zRPkiSm4CGEVKSUUkJYKhMAInniHZJAOJN5mghOOA2UIae03qycs1mWgYeuaeukJgGTJHP9RZlngrB0POsl03VVG/zwqxcnr9/+7h/+vZvG/eUvP26te3ByUlVr5hoqCgCvlB4Oh0II74CztCgGXWeshRDQWh+VMCi1XdeEHulwaLUhAZw2zabydgoAB9MZ53y+XAewWrcYEq1akaQUCYCvqqptW2sMo9QGlSRyUKaTyWg6mygXkAtgSAQDgABUCuG9h+VSGxNbdFELcr1eR5VZCigZd9pQwFTyuNHEuyURMpZQglECuFwu8zQVjAUbQkDnAkdSpEWzajmlzrnzq+uz88s0Yz/8rfcIQ6X6okir1SVF8+D08OLiSjkiKOGEUkRkTCSCCmq8OX3l/s3NTbVZSSkp5fPFEoJ75dEjxrm1dqeZ/3LiIOJDzrngX276sWzy3kfTlKh0HGfVjDEEMaKsAN4a07ZtXddVrfNBWXUhDUo5T3xU9QYSb/VdtIilJiI63NYNuP1/BBEDpVQWWWI053yzbpq6Hw7EqOTVRgMAQRACmUgIIcZZbeP4JQAQAHSAJDDEKAH1snaJn7ptZu90paSUdxkMbbcOEGQgPmAAau22s9O2LeccgGjtnEUIIurHOmu2zh2EZVluzJbOEWvl3oX5fI6E3Ts6Pvz/EvdfTZIlWZogdo7yy4w6Dx6Rkayqutg032kMFouewS5kMRAsfhwegHcAsrJ4wMoshvQMZLunq7t4chIZPMKJmRu7TLni4bp75VRTLADBfUjxdHezML9mqufodz5ycFDlxd54SkKqt3W92darTc8YmaWqyANgLqRpOtv1lSo5Y7rvObmicWy328vNmTrfzg9vE67W20361rAmxQQxQkL8uxGFvwc0+v/KdXOHv40o3PRn6VsvAANBRHbNEASAAek8ODg4Pj62zGy3W+ecUGpQILd9BwBCCO30YPJjvW9Xq5BS03WP7zwqiiI6r7U2xnV9H2MsRmOwFhFjQsblwWyOiC9evn7x4uV6s0gh2uX529PXn3/2yd7enndmuVx2XQeQfAwJyulstn+0L6WsxqOUDq5cT7JsWOw3bpKDwVqr222zHeTZyJBCGlI5pOTj8bidNgDJOWe14YQzJglE56zRbrXaMNYEnwDjEDAtOJOMUIo0xqIoulZv1jtrUlvrzWbX1n48Hk9PjmPqtLMxuAiAFDkgodDbXmbZ8cl+DEQwKaVkhKaE04Pjk6PjTObeWIoMfPDW7R8cAEbTdzJXlENKoeub1XbdNDtKIngXjHaIQGi0Rnd8oAednZ0hw91uFyNcearite0awE3MqffeuVAWo+uZwnC3rrgI4Lqh2bppGYf1+JuPyvXRZfjauUHSPICIYF3Y7OpXr16dnl0YF2IEIJASUMGUgOj53uG8aZqura21ybuEjECkjIWQOFNZPirKMVDmbPAhYUxBe4ABaSAcGCAIQlMi7Ls/+uH77z56+ODOuJDW7to+aqcFpZtmE1O6c/cBFRJSMo0usjzGRFJm6h4ACOU0RU65EMLb8Pr0rfdps9uWo8nefmU6r3tPKW9Msd7hrtWFSrNpxpmaVAd5nq8Wq+hDIoahAwIxaq17rSPBiZTy1x9/vLd3sD/bPzg69N63dc05d84Y02sdYjKUIadsMhmZEKWUXAqtdd81puudNSGEy8WyqiqMwTlHjAEKITihOE0MIRIyNMIQQoJE8CoTlTGWYoSYfIxASUJGwhV4O6RyIQFkyCilmZDWekRUV6HMbgiCqzd1TJ4AYMQUIZCgJGcUptMpJEIp09Zcnq9O3y7bXj97+qJg5UiMg+1UUWTZ8YZmjLFXK/Pf/eu/+IM//qMf/fE/++btZX92gcHsT7IQuhiVlHLwthvsBYWQmcq3m1Nnh9k8Dnvz1RomjACGkAjgUOGstSk4xFQURa4ESTFFn4Lz1iAipxxjqre7dle3bV0WIgUnudg/mN++dbRYrS+3jQNgmSSMdH0fIVpvCCAVlAAASYyRqiwkpwmClLxQ0hIclQVFEIwCuVpLmAZkmzBGOCWINEZYnC3k7dvOhrbuBJNlDolypw0l5OjwaL1eW62r8WSxXP2f/y//13ffvXty+5hJJhR7/ME7s9H8V7/6aPvklBASU0gp5FkxGo1kppxzVPBEsO371WqJSKXgk9H4+PAAGR8Eq4Mh5k3VbJruamR4ta6vrHaHc5iz9jofHCgSQkiCcMMZpPRK/uSc0z5kkAgFyhk6O1hMxxiH0/YVRp6u/HMAIEJKkFKIAEAgIiaSIEGqt21RsN/73R//yR//Z8vlxU//6q8ppePJ6M2rl3XbbLe19wmTQSJYwkSJtm4wIEpIUqJIaYo0IklwZVl9IwG9KWMDVjx4ftDrFDEfOiQoJAByQB+75Fzw0dd1TQhLCWMAREIph8QJAueQUnr96u3QSXjvORfDWOeqa/Gp703XXY2uT1+/uWYB18E5r80GMHkH3lLAzWq9vFhMS8mS6rWtRtXQVg6wx2q1Wtc6Ed4bjYz4oKOzMQSM1xwFAMr+f9oP/APX32xHriSO186+KSXw115M164qQojxeDyZTPI83zUb6020cXmJjLFWt4iJECIzIQRTSiglylFhjdFav3z1HA1OJhNjzGq7Mca0nWZS3LoFk+k8y3OforU2ROCcuRhW281ueaGqqixLiljXddfWgw5zOhpRSo13eZkdnxye3L6dFYO7wFVEyLV33G8cogBgmOUNzfR15XPLywtErKrxMGEZjUbOOcM1YwwwckKj8gApRm+0c4EgJgJ0SNXlTBCIMcJm21obt3UXA6dEpMjq1goFpcPoeQoRCAWMIQT0Aajb7XZKqWJUBusQQSkqpcQEl80OL1mzq1+/eN3Vje11DGFSVH3fBu9Go3L/YC4UjSl4b3ujc+4EZQSQJIg+BBcxEc5wPpnrztR92/eaIB2SfqTI7fW4IcUrhMB7b+0ggrjSLzjntNbX5f83VvE3HIWbDTxddxY3fYM16H0k1hLCEDljnBKHIJyFEIhS2Xg6yVSBjGZZoeQIiJAZ4Sy3ptdtY41hjGZSGWMYF5B4CBhDMjZEJJyzXMgYoyAcAIYcYYE8QWKXq9XZcrLdXlrbHO2P5uMCKTHBmNp47+/eQ0boqBzPpm4yP9jtGppA11rrDgEoMiWyUTnmRJ2dXRgdV6tVUZVHh7cvzte9XRPBTUtoICkR60LTWQqaksSQpOiDbV2ywXPCo/em1a02XfCPZrNZXbfHx1dj4LIsD/f3F4vzvu3Wq4uYgukLIVmWSSYwn8y4FN77Fy9efPHFF5erFSKlgkfnog95nre6d865GJqu5ZxjGk5FztNACAMAKSUndCCTx4DOYfKDQm3w4+MDZOScSwGiT9Y4yaQ3vu9NkZV0yijlMbphhz06uu29DV57p52p+2gRIudMa22NV3kePFrrYqIpIudyVmaTKl+t15pSLlTnQTC+WjU//+zJ+Oju937nu5O9/fPL1XJxXnDmILmUsiwbhAacS0KIEIpSrrXxPgzG38OsnRCSUlCiuLFvg8GrodfG9gBAORdicHtVifK2I0hgONYY02vdGWMKRb33nPNRUY7H46IoLrc7Z2zCFEjc7LZFdTScHgbbAExJKbW3t8evjegt5ynEwWGJMxYACVwp/a6EA4ApRM6BJFitVoeHh87aoZBIKcvjyfNnL7u6idPw+uUrgHh4MHn18tSYDZdx9v3v/eqjXxKg9x7cf3DvnfPzc3hy6p31JhhjKlJxyRCxM/3z58+kVCoTTdMIoe7cuae46Jrm9fmF1rptW+ccIUwIMQwXnAs3ZMZvm2XVdQ3XKxiuIPqIiCoTyYfBsEgpITivqkpKiT1UVdXUnVJ5DOB99CFcR1JcPfyqqFwxIgd6aYQEESLFBJgQyHw+apqd1vq99x7/sz/5p+8+fgcRD/cPPv7k5y9evPj8889fvXnbdyFGHQB9SClBHLh8V2blCJSSSAZbpxvA4IaUgNfRpnAdZzWMIaiInEtKqfKJMQIAqbXOeaXyGMEa730kiJQwSAkBlcLg9du3b9fr7bBwsizLVHHN5Rac8hjjZrNDTJiCbpv1em2MUVmhmGh0s1lemrquCpUJWq83m+WlPpqncRG9l1xMJhNK6bAe6zp40FRkeVnUzQ7QQ4qUEEYZBgohhfD/H4LC37y+DSN/G+RI1+Q1a60gZOhHhzJMKR2QOaWUc67r+6GzU0U+mG3ITCmlqqoqikJKuVgsXr548eLFi+1223XdxeXSOedDmsxnB0eHRVUeHBwESKdnF03fqaSUUrP5fHf51lrbAGCIpu8ypd599M7jR4+klBHS5ebSBT8ajfKyQEytbhUHa21d1zeqn2siMEXEuq4H018hxHA4Foxrrd+evs7W67IcSa6G3y+KwntvrI0hcME4Z97bGCMXFZIEEClGqSjnJAQPQGazvVEVrEkMRJ6NMUrB8zwbae29j2mg5pMISAaOTTUuhRDRB+8tp5hlgjFqrZ3sTUOCNxdn37x8SiORQhRCqaqYzida90ryRNNyddn3HeMEKPDgeM4lV3lecoSUxaOjozLLx+Px2eKi67QxLkZwPiSClLPg7dX8iA190gADuK3dKpULIRBh4HWSawP1b6/Ha5XTb+RL3+4VAICzwrvkXKQ05Fk5ns2LcgaJGZMSobPp3t37D6bTqbZuSB421ClZMCRG9+uIziUEZFxZF51NW980rQkACYjMCikyJaT3fiC0OeMRk6MMANh/81//N8H5zepCMPbOw7tS0PVqmYIxxixWy9OX3eHxiKfiBx/e9xH+6vlfLzfryWQyGs9PT0+7xkxGhWT7JAsx9E+enPFsev/WnvahWy/DCFJGY/+JoeVmFcbF0aYVwYnbt+e2q2uzmtyC6YxC2m5WPfo8F0cKuaPRxc39+9PN6rmuzzKpbNe5Bk3TvXz+/ODgsO37rJgkhNdnq1998mw8+qLruh/9kx+fn583vXUBmGCUKpWPnSdnp+vxeNppW0lBKcVEtb/s+14IKcXIORMDISisM1fzaQ8xYoosRUgBY4wIkGISTAyWU+2m1cTmebCd6TvTy66rG+8cQTaZTMo8L+8Vfd/u6vV6Y7T1Aa3H5JAqIYpcAETC6aPHt8ajmZRZDCBE9vbNmX8Jm80u9F1KpN7urHUM2NOvnprWjMv5/vyw6zpKaQapaa1M8WAyEpIJiTHFxfrs9r27rOCXpxe674tMCZZY9DxEmWLG2bSqwLuLs7e3j39n/+Bg22yVUsPOgom06y1zqcqr2cmk77V1tTbN/t7xZr32NkGUUmDw0Xh7eDSdTOXZOurkt22XCCWcCRqbrsnHYyDJ6JZznmfq4HD/66dPpJS73a61mnHhAFiWOQCRktZ6YPxyTmOEhEAYDcm/OjvjnAcC2bh6cfrm5ORkPp9f1uvVekk4WXdLWpJO+2Xv+eikmB0/e7XtuoUkPrntwfy4KuEHP7q9sF+nhKdvF89eJOoXofWxb9Zvz5mlmcpfPz3bn57s7+9rZ7968qxpGufhBoGnlFFkFBkjLK+KYT0bY4zu4ZrGSPA3gM1QhAmljFGn+yyXWmtrdV6MVZ58sITbe+Vca/3weC+EKJPte5ME7Y1Bis6DYCwA8y5yKRgTfd9XLDatUVKFhDFgnpUxxug8ROVN8+Szp3/2P/zb//V/9af/7A9+nCkmOH33SL158/DTByd/9bOP/voXv1puI5UAhBMARHJ1rElJUMJYICT0vb0yZwQghCilBpr90LoBQNu2wzeXy2VVVTnSTGUA4J0lKCAZrXfWOimyEAIS4AIRQ4Q2JQSMfdMWhayN+Q+/+GVxfPDhdz7QbdsrsjY7yqzXFhPzVrd1xpk6by417wghRKnW6pQSFSQFaEyLLCVgB/Ppar1bXbbzyf58cte5ZBepO+26xYaYWHBoXYTUWhuEIC5ykoARjjGFYCAMh9rfJH7d1Oa/edD/NrLyW8OCf2QH8Def7duPvWnIIKYYI1wbehFC4Fp1MuiBB+Whve6V87zUugsh+GACAcowQXQxAhKvY4qRRuL6SwjJGTvP+Bndne62MSKqKJSsRD6qZru67zvbtU5Kvjy9IBR73KQUwWmVD8LsyBWvJgcHBwf33nnn8M4tKSVl6J/H09PTutvSJQ5dS1nJYf0O3oXD6WjQ/hFCLs4vu9aMRlNjTNu249EY0IUQhr1ut21PTqZjlb15/TpAoAClzEBJ0zW7ehGTl1wkMM1md+furaOjg/PTV21djydVJsd1u0PKR3uTrnd16GyOgXitPHMuhND1DSKWowISmhhVLpFgllXVfNy29eV61VhTcNZDzLU/ODwe/864wPL0zVmVV5Oiij4E6/cmE2Pr/dmUkuRcfXyy9/z5UzbaM13o207R3Hj36P6D44PDR48eXlxcfPOTv7TRRoLr7SofVVKJTb1zlqILTBBKGFLwIfgEPgHn1ENEHxkhiAQCQU8Ykcg9MoKMINAIEVMIIcSIECMjHAnGGIP3eM1e2HZtXuTa+eVivVzXt5Cd3Lp7fHLv4eMP2153XRcxGm+YZAJI03SKlJnMENFpRKyEpCSBc7Tv+xgjQAIwCQKlNGeChwCC90avt5tBookRmqYbhpJEciXV8XRcHu7tGd2S/b292dR582f//t9//vnnne6P79xlUpy9fv3JJx+1rn733XcVF6/ffLPbbGezXNtidbn+/POfFtXo6Ph4Mme99tMZiRhT2k0mk1E1H1dIUq77pIPvdB26FdJYjqbTaQUx5SqlkJFQQOI8F9Pp1Bm7WaybdZ2rom/betsomc+nszIvrPEUsBpPm12LCRaLBaX0F7/4xXy+J4QYjSaUCZ+itV73xnvfddo4l+fleDzmnAcotNaMCcmldxBCapt+06yHsRBAQgIUASMApQAYLGf0ahullI5GI0gkxth1nTVea315eak7I6X03k8mE0ZE2zbb7Xa73WqtCfVS8hig7w2lnhJOKcQrVgsqJatqvLrcDAZegktIbLPZGbPJ83y73Q6N+bBre+/zPJvP9wYs3cagtfYhVHyUCd5sN9bavm8hBskZUBIBlVIBUtu2SBJlqWl7a62QrK5rpZRQ2W63CxEiQjUeSZEtViuig/c+AiGUE6TOOectYpLXbjBN0/TGIRLGObv2GYxXgUlKSpnJvKqqV69e6bYfaG6MxkEmEEKAIVETMc/zLMtCCMa4geI7TFJ2u4211nsLELNM9hc9XGfnwLUcYABLrLWbzSaa2urLF69uffDBvdFk+i//6/+t1vbLr544+xfWQ5Gptt5eLpez+cFkuu9CNM4yxvrdtq7rpmnKanpDIBqA38HVceB1D6DIoDa8VjSxK6w1QbrRNbkYY4oxCiapEmVZCsGoQ8l4cIkLKqVEoECwqhKl3PporaNMpIR9b9peDzY4k3HRN610CQijQFIMLnjGmOSZMXpv74Bh/I8/+WuA8C/+xf/89snhbrN+fPeIKzk7ONg/up1Vo5//6tOLy7XtnQ+AhAChSAAShuAAIiGkqipynXbx7do25MXFGAdobTAJlVJORHU1aMcYYyQEsixTKrPW4pUVXASgADAMX2KEGIi16eWL17/4xUdSZnt7M0g0RZISTSmkCDGATy5Q4h2E3F7Jya5muhj9lVkhBrtldLNrnPecSWTcG22c7bTRNiYCjCKG5ANE5wjlEFKIMcQAMUBKCMASGUic3y7ev1XXb6a/f2sf8G2Swd963ahM/+Zz/mOulBIkGOCceG3gaK1t23a1WjnnJgelAEaz3AYWoo0QCKGEsQQBkZJIMJEQY/DBRu+S29ufxEHLFxIkKkSeKSkkf3v6yrp+s9k8ffo0pfTw4f26rhfnpwnsfG/++PHjw8PDQcefF6Jutqu1SSldXJx5b8uy3NvbM8ZYa41pBs/pruucc8YM5B4QXHHON5t13/dSykEB1Pf9vfsnV42Ci23bBm+1SQBxVJSACVOMXkfOi6KiCEIyytSkzGdVRZPPpcToaUIEFER4G1xno42UZYqkkMD2XhtDrs1nB6gpxhicb0xvnO6MphSllJPJaDQaee8RaVEUjNiqKldSaq2XvfHWVaOsadKuvmQCfPDXOzbvdCsY54Jq3SkuIgRkxBhDOEkYvfdEUs454xQJcM4kCs455SwRJIgJCCINITlrKQ2CRc5lSpQQS5EBWMG59TFFh+hTSkOee0oJCRsogAOhCxGNMcYY4HnX6U6bvu93dbtcrl+8fDMaTYTKBzoVEyKKyBCVyjnndeeGlU4praoqy5X3Pjr/G0wRcbBNAYAYY9M0dV0PiBEdssc4l4yz8/PThw8fcpZv1pfe68ODfQ7ixZtXl5cL693h7ZO948OY/MXFWdPV09n4/OsX56eviiJjNMzn5WisVEbykv6z//z3E0LX92eLrxbLpYvh4LiaTqeT4kHf+edP3qyWfQLHVABinNfT8ShTVdu43W7HUCEkwfjefC4ZfffBYyWkIuLJl0+idUfzw1t7t2KMs+n8/Hzxtnm7o9tbx7dn40nyYddvDw4OzhcX77/3ITLqfWRcIqO7bV2NJ0Cod7Ezej7fn8/nlFKZCe/iQLpPKSGSzWbzCsh2u0UEIBFSAAwEI8QEmBIhgweWcy7Livl05pxbLlf1rh3CeTebXbOr8zxXSjHGlUramq43Tdv3uuWcUM4I5YwTcJ5zlMhdiL22QkaVsafPnj17/uLVmzdSZIcPj/f3jpumuzhfLJerum0ipL29vbwofAhN2zLBfAw8xggQnDemB0oUn+SZzJQ43J9tOKGAZV7EGINzQpWK5y5Gbb0Pqe5a4+x0Nm6apK0LITx/+UI7a6xHykWWxYQugQ1RG9v2xjiPjGeScUG3260LPqWktdbGESGj0SnJYZtL16oepVRZlANYOuwgNyenYew9KvKmaZxzhFHCaISkbb/b7Zxzg0Y8Qrh15wRIavvmfHGm+x4RCb0hhyfAmAi4YF0IvTVe933jz84v287evnVCFdXams4J9tf1rtnfU5TiycmByqqyVC6m1KS+79q2DikSdrUIbxBUuBYHDmemm+yWG/SP0yF1A0MMcOW4nFwK0TtDgDHMVcaQeuOtdVJxIQbxNI0xMjrAjImGkOVCcNX3PWdxNilSwoE7UlWVEMq4kBJqtH3fIlLOqTF9USjt9fnTiyznP/zR9x89ejSXWd/XgGRvb/49mWnrfIg//eVH690mywgQRhgb+tphL2CMdV1zrXRgVVUNHrSMsd1uh5hCCEJcqR4oxSyTQrDB9IJSSlngnEt5lXafyEDhGKJrhlTDyJhCwgn16037y19+WlaT73znw822j4nFGBKQQfgxHJFpiJRZwuVgYnZVOxESgrXWQ8IEy8t10/UDwd2mvrG+sVY7SBQSIwAhJcBEvB/OXBwwUTqc2hAA4jUn49uF/9v4Af5DDox/Fx3yb636N8/zdz3q7/q+9z5eD6qHtJ6hKefsSCklM84YaBtduAoT8SGSIcguxGCSSd5jjIyMZ4X33rsYIw4e8wQB0VrbX67sq1evvTPOueD1bnsZvbv3ztHe3t53vvvO4eHhYnHZ931RFISQi4s2hDRE3A0pd1IKKQXpnFLCGNY0nbU2peC9NcZt/VYppbUe6HfDH+Wc227XUmaTaiSrjCJ0Xd81jXPWBxe9G8KmUwxCMKYUJdQ2u/neVJIIzhxNp+Jgb/h7jQ6r9U7XzgenVAacuwCCKpIHSmkIbmA1BAgUkBBydHTU922wLnESQthttn3fG2O8j2/evOma/uzV6fpyLajYn8xm07H3lpArO7UiL3e7jdFWihyCjhCUlG3XKDHx3pVlmVe5DRoAettzJimnEYJzPkLI85xLJYQgTCAlIYIs+qxoCTIAQhJEn1KI3scmtLTvcyg450lKKSWjNPk0NGSZzCIiUAqJxAgxRheiC1EIOvRzQ/W5vFysN7vxeKqtZ4LneV5UVVEUqpDD6DCkGJ0FgAhJZkpE3nVd1+vZbOa9HyhXfd92TdvWzfpyNZuPMUH0wWg9rA7FhROC/eKXP12tFwBxcXExHlfvvPOo1+2XX345m00IARnDF199tVhezObz27dvv//+4y+//vybJy+ODg6/993vHx0djYoyy7L5eP/Z82+avmnqWreNd32ITvepYenW4ftd1yVqbdo2uvY2iGw8n0+UAiTC+5gX81Ex3m11CmkyGaGH6XhyeHCgmEQP9aa+c+tumRcXF4ujw5PF+RIixhAkk5KIMq90MFlWHB3ezvNS5WXTNBFgvrd/eHDChZrvHTRNt97WeZ4rmWmtNyuDiAMznVCoqmo0Gs3ns+12AxhTDDH5GH1MAWJKKUC8Sl5HjFLK0WhS17XWerut5/M5IsSYANA5X9cNY/zgQDEmhBAApO+MJimlFEKaTscppUQJAA0+De1hnl9p4uu61sw2TTMZX6m5ptPp4ACPiFVVTadTABBCeBe7qBNJgJEzopQqMgXe3j05SseH69XKWqtkvt1u15eXiJgVufc2IUQgy8vV27OLg+OjLM8vFovVdvfZF1/12nTaXq5XvbXr7QYgBqCbtv3m2fOj44OiyA4O50rwxXLJOd872K+qSvttAmjblvuQqdEAxuC1GpgQMvTCwwvOskyKbDKZSClDCAwJIu37fhAT53k+uM9q0yOi1rqsig8+fL/r26dPn6zWi6KaSS5ijL1uY/SEYkouxLTerUPwGWYiU9G3ddteXG7u3Lt39uqF976rO933zRa81ZDi++8+PD2/tKZrjavrtul1Smk6HQNA1/4nEscblt+A5YQQBhscALhhIw8tkdMp+BBIooRgwpQw+ZQAvY9953RfpxQFk7zAEJ21OsYYITBOU4zW9ZIoa2zXtISwvb1pSvjmzZvVqgvxkhC0PjHBBFcJLCJJwJQidXMZvKUU1tv1T376kyzjjx8/znJIIYTgilL94PsfFKNCKtF2/77XwUcbjA2J+Bh9AMaoECLPxY2qTUiawGvTJp2ct8Z2AIAkAgAlUFbZweE8L+hmsw0hDPRML7wx0HWGEEw0DZQ1AAKQhgRzwjLjElJBKV4sNx9/8nUE9vbtW+uIjzQCAhKgDFJMgBGBpISYKBJKMUAadnkCGLwFgN7Y5Xr99mx5vryMKem+P1utdp1xaSiuMUagFJWU2noh1OA1RFKkhHjvjemHwQr8p4jCb/UK3/7i24X8twYWf2uBJ9cOejfPDH/HPOLbT/tbz3XzqKFJdc75eEWi3S5WvsiLKkdBMCaEQfkbACARBEwhgaMxMaCMc2SEarQRaEQkgjJKMMYUvCkqEZwZjQSbll2nN9vLvmtHVfn9772fZdn+fMIpdM267/tcMcYkJ4gkKEGaRj9/+uT1y+dlWY5GI8YHy5OQgoPoOaFAGbBoe+3REoiCEUwJU6QIKfjF+blSClOoirLtunq7U0od7k9n0+l6vW7rdbOrERGigoh9St97/967774DMW03q3FVCMb7pqWUc1m8Zmfby9b1NQeGTDFKhcrLjKSUmq4OIVBAJWRZ5uPxGBm5bnCJi2nQKDnnrA/GmN2m3m7XTVNTREmJ4ERIRikyxlJCxgQC030oyiqjwhtLGeut6a25WC6MtVLKq3POddAGAJjBQwWQc54VFWWCMEaYmCKPKUmpUgDnnOn6rmn7rgnWxRidj0iSIkyKTErlB8aFjcZ6xpgg0nvfax2sQ6RSFNvdDgBDQgCilKrKsY/JGp8Ago8xpKsMayYZ5ZQiI1eppBgjSeB9CiEY74iLKUZCiBJSCe4LTwE5503fWOO8C8lfGUAN9ZAZ2/76o58LIaazmQ3+iydfxRiR0SfPnue5anTHGBWZEJJ99vlHP/3pTzGVdW3v3xq//+6Pbh+f7HY7TDAeV+xR9eTJV8uLWvHJ4byw0RCGWZ69ePVN1+lqwlW2f56lzeWmqMjDd+9a3QdnmyaWRQaYBa9H1eiD73xYL+sYoWv14eHhvXsPvmq/qut6ebFcr7acSe/jycnJfLaf5zkicdYLnq8ud0cnx8vl6vbdCoDWu11WjDnDXb3Oi4pRkWU5ALatXq1WQowopY7Yrm2s1eNJk+eKEGKdBogIMaVwdTBI6TrrEyjllKbBomez2V1erp1zIcTBpNJb51zw/tJan+Ul55wJgYQZG503MWGImOUjSilLNESCBEIC612n+wiJMJoAmq69WC6kKglhPgapsnJUee9d8Jyzo5PjrMhD9FyMnbOUYpFJqVhRZFWV276ejQul1LhUurfD7HC3qyNgSBiRGOs6q5+9ftk7XY6q2Wzy+ddPLi/Xr87OHEQP8WKxZLumqbuiypnIul5/9NkXjLGyzGUmQ5FlWbG3d/DgQf/Fk282TZeAxBgZQWstQTYY1N+cvIcYSZIwXZkcX/UEALBerwlnOSuFEGWZF0VBKY0xpJR29Wa5XOaVKkZZ3W0vN5dZlpWjuZRZp5uu70J0hNEAwXqn20ZQrh3hJEVKzxbrz794MplN8+Tbru/r7nB2oFgzHVW628VwwDjs6rrvrPMWIOVlXtFRSimG7cAgu6EL3bjPDp37wJv7TVWIUQjBKY0ueOdSIoQOiT2JEk4Agg0mWWuikEypQiqwLvjkKWfTrKiqCoCsVqum7kIIUjDGGMGUIEpBGIHD/ZFS6s3pBWcxrwilSAjG6KQUbaOFgqO9cVWqN6+efvxJkWWUHY8ppdSbhLQqsx989/2+qc/OXr8+vWh722kfUrIBfUycCZEpDE5KnmWDrJeE4Jwzg0mOtZoxRukVtUoIcXi4nxd8u912XatUdnV/GMboCWGDogITphQBbtyESNNsGWPT6bRutk+fvkgJttuttcG5EPxgbZkoI5CQ0hsZ4XUhJowRGFjGCNE7d7mpv/zmG5VlR4f7GNMvPv70dLl0CJxxCJDQMcoRqKCEIbHem64nBJRSEcIgEf12Cf97Sv7fWtH/wUYBvoVS/INwws2P/uYz39Ao4rU159CztrvWG2uMyaqMZZRxmpDEGJAiQIwAEQIhQBS/6uScRxopJxSJUhmjHCMGjykA52pvb/b86Utn9eJ8l4mMIZmMpnmeKyEhAQHUXX9xdsYYb5pmIEUaY3a7ndZaqbwoivG4HMxCrLXO+ZslHyOY6IN3MYDu2pQSgZSCl4JHHy4Xi81qZbR2zt0pbt29c/L973//+TdPP08BE8QQCCExYPQhk+r44BBCjFqXXDFKI2qKlAGRhCkuJBeEkJiSj5HEYIyLMTpjB+BBKSUyMZ1OQ/QJBkEONG07xGpTSpOxiCj25Hw0aZum3dUsoQ+W+Jiidc7WdY2YYgSVFUeHx3sjut1uGSEUGYG4Wddv355Op9Om6WOEBCRERMYiYkoOKbqQgDDKJCB1FmnCoiwHIjNwkCJm3CiVWz3yzgyHhxCC0d67mnPNOadEZgXvus7ZFKONMcZAEvKYkvex63ol84TJGk8pm81m2oTdrh5NJ9YFRCqEKstKZiqlGEKQSgwHG++ccy4EP+zGy/OzGAJFIoTgBFNKCGCNOTs701oPB1REQhEpMkTKskxkmZjM5kVVWWtDikRKJtiYYlFmijPddzHEkGyWyeOT/e0Kjw9vffc7P9ifHpIkBC0ZY5Rk0baZmrzz6Dsnt48m8+pys3zx+ul2uz69vLxYvE4JGWMyhwpo0y1fvKT7e8cx4qiaTyYzq91ytYvAts0uyzIuhcqLyXgSIjx/9bLKS6vNZrMTKrMu5EU1mc0oFyd376iyUFVxdnZ2sH9Y1+14PJ2M96RaEmSTyezFq5dFSbKszMoRAg0hEGQXix0AhOAuFxdn528AY5aJ4MzgN0cJEEII8gG9RMRrFkmSUjobTk/PF4vFblvPJpOU0upys7rcSCEA3BDAuFwu8zz33htjrQ3ORUYDgNlt+zxXlDAEBwk5T8OIfbFY9H0/bBCDAXCWFdMpa9t+8G01xiCK6XRaFEXbNcYxAMhzOZ9WZZGPJ0WmeN+3IQROoCoKyYU2oSiK2WxmXXAQEUiru1FIVutPPv10Mhndv3/30y8+b9uecR4SYUJ11qGLHqKNKDhzxiyWq6+efHP//u2TowPGSFbkkeB0OieEWN0jkzE4zvl21yuZO2tvZFEAYIyhlFIgA5G+rmsEaJompUSZuPEvMs6ajfXeGWPyXCFJ4+mIS6Ztv1xdaNvJjDvtOOfMs+BsjJ4ChmBijFQQSplx1gZLUrrc1J9++XUxqn733UckkuDi9z78TjGqhMw+//rrtl5zSkI0hKayLLj3CVgC0nX9kER6w7QYsF8AGDwWh91naBcGRYNERM4Q6eCyzAnlhAJAcJFnnHMGGCEGRrjiueJZVYkYQWudInIhy3JEkDkbml1LgMwm0xDC6nxBCBllBT8gkrn9w1m3Ax9joQzEKCXpTaRUywkUGR+PueQhpfb09Tc/g6599HA2m00mE0Q0WTadzB89uvNP/+h3/+2f/T8JaRIEn5hE5hMhTEipqpyXZTk0bUMnNFxaaxn5YBQ9/EhKOZ5UecF9sINJToyREEoI44IGP8DLCckQmEYAIgAghYRgvYuQCKW7ul4sV4QQH4IPIcQEBNPgKk0wkRi8i4wFQmKEAAmBIqWMsRgFxJBi0ta/fHVqtNubTZSSH33x1cVmlyhBwgGSFBQIdcYPrZ5zDlJSKjs43AOAzWbTNe1vlepvl/xvf/1b1f3b04q/p/b/Pc/wd13fRhRSuoq9HJqDmwuvZ3Y+QjDWQQwUR7KQXCKNHhxSiBhTCkgQJef0yhZsExAoIQicMpmxIisYEyRSiqwqp7qzv/z5r9bLTQzkYHrQddo7DJ6sVw0ASJkXxcA9lE3TrFarrmvLshyPR1mmfLBNu47JEEI455TwQeYXQhxMMq6EQpB8iJTwIQxiyFaAlIK3iCkFo/u61/WoVIwiIuZZFmPEiASZlDIGJMi5JBRoW2uKyfRWKdKZxjvHOS2qnKhCRzDamqhT4xAxeM+FIIjGmLar27atJlVwjnBmg3fWIkUpJY9Dai8iA56pssjceMQAMynqurbGDHi+c2E2PTg4Orx3797BjHdNiyk1JzvdtNvNKgFbrZvT0zdNZ1wg3gSMDBlYl0iiWam4UJxL58F6TxJQmwhD3VtCCCMAQCmRnAMhBGJiSmy328Vqsd1uIaaiqMbjcZ7nlMre9sYYTphSKqXUtm3TNJPxrCgKFxNCHSIqlRvrB9Xx0NUVoyrGQe0cnHMo2RACGwchKwJlTCk10GJ6YykgpGDtle1QoiSGcBNSHmOkgCQBM7afTKdVVdhga92VoyqG9Ozli6ZpHj28f/j4/nF21O920eu8Uj/4wfdXF31VVQf7Vd0um5oKKghk9Ua39W632a63y129vn33aDwrHz24b/3JbLkYj8uYvLdGKUVRfPPkeV+bly9fvvPog9/78e8fHRx//vnnL1+8WS4vf/rTn/7uD//o1mg83d+jQF+fnT55/vzh/fsUWGfdrms9pnq7JUIkwd55593xfG+3200nrq5bre2XX3yd56ULfrXZfec7v/PZ5184f6F7y5UcZrRN0+x2O0JAKVFWxUiXfd/CkOERPSIiEIKcMUEpZYQhIoHgbCAEOefGmPV63dRtnud5nuvebjabvjdVMUopdV0XA2zqnfHOObetW+MCIvERownbuqdcMg82aBkD4cw417ZtCEAYy6sykT6k1PRdSGit41xyJSNC3/c2+FyWqshFJlcrQymdzcazacUolHk2HZcmE5vNyjuLhMYY63obQihHle7tuu4ghV3b3Fd3ijJbfHH+5JtnCeHVm7dSZPcePPjm2SsmfCQphASR+hABIBGKlO+a9uunz44O9zlnBwd7Qx0dBiKCsWGedyOYhmvsdGgUOOck4ZWSzVpIaeDZHhweU8oRwTlnbK+13m43db2tRsV0Op1MRtvt5vJyGaIryswHu9lsZvszpRSlNPkYY4gAiRBKqRRKN22KaZQX1uxOzy+/fvLsbl7sHeyTRO/euXN4dORTvLhcvj59i1wa0wEyzqXxEIJLwIzRAPSGQnGDGKeU+r7H65gluN614XqKPKCYDMmgJnfWAkBZjkZl4YMLwYXglGAUaIqUoEjRdV3XNRoiEUK0dUuQx+jnkzkA1KudEvL2rbtaa3Bv/uDHH7778EBm6vjWUUQYj8dt31x1Vwm8sSEEvCKpGONd3dVZLrMsg+hSsLcO53/yx3+gdXd+uV3XHbKMyjwCjUgYl4rGYQZ0eXl5fn4+sJbgipQXlRLDT+vaeW9DcJtNu1qtdvWmTKXRjhCSZfm1OdV1r4AImAgQAHC+zwthjHHOwJVFjBsGTCklQIMkAF6BN4AspRRjCIGEkCICwcg5x+HtACSKMgQf0vli2TRNVVUBicgLa512IbioVE4pr5vGhyCEFIXgnB8dHRyeHHZd56IbkO1vv7PDf29IiL81cfgt5OAfrP2/9fCbz0m8Tgb/m13C3/oMIYSY0jDIiDFGn4ZTnSrGEYIJkTprvGCJc0Y5QUJhCP9ExMHJgyRIPoZhFIqIw6SpLMq8kFRNx3u7TfvqxZvkozNeySIGUuTjTEwZirM3S2PMdDbO5SgEf3iwzwg/ffPWWVMWGSHgPY9Jaq0ROACUeTEajYwxFxfLvu85oxRJjI4Rgsi89wQTZySl1HZdVVWAiVIsMmUZcoEE/Geffvz551++efUKkQoqIZH5dHT79p0f/uDxBx/+IAXf7to3z5939S54n2WuGk0QE2IiAkXOQ0wkGg9eRBauUmPAe9913XK1cCFIJZCCyBQhQDk7OjqYTaYxRiqMMcboLhjrvQeMkNAHpzLprKmqijDKGJtO5+PRNEVsOq9kZU0/quZlPnr88L0EAUJcb3vGirKQJpiEAIRCcpAYGSjwEYBQITklCoE7mxiVhBBGMKIPIQxMnZgCGaxfmUSgTdfUjV6vtkKI6XRGABljCanurda6aZqu6ziTiDQCSWmIkjKEsslktms6Sg0hhFEhhJAiY8xxzusUUvDEg3V2YJEzxiLnjDE36FoJZZRlQqaUMEHd64BsqImYUkgBEQlQNptNmq7Zterw6GQ0myeEXd0wKUNTf/HVl6/fPD863JuOKiXpuCqn49H+fH+9Xi8uX/bdVoisVCMhpNHOeUNZBAwXizd1tyhHWWfb9fpyZ9zh0f577z5ab1bb9ca5ruvr3a4f5QdFUVXltG70029eXq5Xs+lYSsmVJJz1RvetXlwu277rtNXtpuu1cZ4xtql3PsVE2bsffJgQzs7OOJfPn7+klH7x1ZM8z6ezvbfn548fv/tv/s2/29XtYrFI5MqK3BhTN32WZbfvHB/u7/lgY3RScYZQliUiUmRkmD4RjkARKGfBmpZSmmWZ995ZTymdTCZOu67r6rqOEfI8DyEMhXDgjQ/Ye0qJMQ5AnHXO+pQwRhg4qFJaY8xicZESGXLGBmRsKE7ee6XyocreQJpSSiE4IQkg7u9Nc8X6ZgcxMIqoBOdc6zom2rXdYrHwkaisvHlVdV1zLvb3pk+esPV2c7FYNG2b5eWtO3eev3yLSDgnCZLzBijzwRNEoVTX919//c1klAtJZ7NJCL8xKlZKIdbXKkfunbvCX65h/K7rOGGDyfHgspBl2dBDpJSGnMbBNpgxKgSTikvJjdGnp6uizAghs9lktVrtdrv5fDokTqHFMJigMmq0SSLzMVLErMgRTNv358vL1Wp9//6DMs8lU9aYRvdtWy+Xy7yq2qahXAVkbatjIozLm9eTrnOhbjgKA8YwKCEH6ddVqfbee297G6xjUgkhUoy9i3me783ms9nEWG1MH5wBAELobttqbXTv6l1njQkhlWXZ92Y8Hre7djKZVUXpjJ+Oxvfu3bu4uLDd6vvfe+TC/axQ84P9TrdSyuVqMRqNCCHRhxCSYFwwpnvb1HVIWdd1u90OALw1KcS8LIpSUQIIgUCSgvFcRaQuIBIWfOccaq03m/VyuRhskaSUu902pZRlinPmveu6NoSw2ax39WqxWBoDeT4gXinG1Pcmy7LfRtpJQkhtZybT0oegbT/MN3ujkRKVSyAJTAJMKYUUYwJI6PGmtF7PCAghjInoYiBOCFFlKsVgdRcSREhHxydCri4Xq6Zep0QUQc45RYqMVFVVVWVW5IeH+8Wo2O7WF4vFbxX+m0bh72oI/laE4B/TLtz0kf9IXOG3rkE4j9ehXNduh3FcTUIiBKKL0VgvvWeJcMYpg5QGp2HghEJKzjnvHKUckQJEIYSUMs/z8Xhcqmoymj998uKjjz5hTBBkzqbzt+d37twbj/YopadvL3Z1O5nMgg9N2486k+flQO/o+75ta2P7PFeMMQCWUqKUDkNDxDTMra7NywEg3nB6Bldmxkjf9yk4PhmNJ9XJyck777zz+aefvXn7arvdVnnlGIGInIv9/f0I1LrAEEPAzXbXbrdKcillniu6JdrrTpuoREAWSAgxUSqS98M+GX24ziWxKpMxhr5pe9szxrJMcs6ttY3xhBBGqcxzzGIKwfd6gOX7vq+qath1vQvr9ebs7Jyi3pvNdNcXKhuX5Yfvfddbs75cKFns7x8FiLu2MdECoyYApai1ZowjEUwUXEgmcoLCByjLnBFKCEAMlABFcBZiJM77PM/lLTkaTZbL5cXFxWqz1p1erdYHBwcHe4cppYvl4vJy7Y1ljC0WC0IY5ZIQal2IkVSj8Xy+/22TPe+i9354NwY0Yig6jDGCEABTjFVVYUzJ+uB8tME5F5yPMbp0NfySnNMre1smuGCS0mw2G0/GwXa7pismo1snh3v707fnb7/55puuaz758ivByN27t/f3Zn2KE9sBwHvvv/v0+bP16vSZ1iFFJllVVREiLd1qfXr6zelsb763t1e33cHB7VxkT7542vVbb3WMcX9WUpdGhTycVYUgzboR3s1ldnu2/917j/Zn0+dffz0aTabTaTkeMcGWm+W4mowyuVgtqnL8ox98/4vPv3z9/Nn5/YdN0/SL9ZvV6i///M/btp/NZtM79+8/3D8ppm65ffrRZ1RIQujpYlFUWaf1bG8+k5W3dvv2Qng3nZVSUiUwzxUSn2flbtdUxdTaOCpy54I1vu91VY3yPD8/P6eUEkZfvHp5eHh4dHT09NWzSCEv5KpdUUrVWPWxL2CqTWha02sfE0XCfEhta7i0vXaEihASILGWORet5dNJtd1uQ0gDSVXrLkbPuLC2RyQAJM/zFLFrvdENYyzaPnh7aZoVBK97ONh7dHT70bv3356dLi7Xb96e/uqLT1yCRBhh7PLysl41XDCWeL3p3n/0HomSpvz44MGb16uuiW/fXE4mB3UfY4jRaSol+o5z7q1FRM75+cXFT3/+a0bFeDKfzSa/+uXHr56+4ihcYwrCS8ZC4HXdlnnhbBjNJ6vVChGVUuPxGEIkBAc/oiK76nsIi8butNYhJKUypVTw4B32fZupUb1rynL85OtnSubL5dK5EER3Ub8pimLQ7zVNQxEFZxi9bzeCJO/9cu2yrOBl9vKi+/XnnxTT8Xw0e/P8TVUU+/PZHsqPX59N78bS18d3914vtl3TbzXk1UEiGVc1AFoTrDGMsVwWQ5fj/VAXIw4BH0gZEYIJ8AkAiMykGPKpKRNsOhdcUJELprgJtndX/oa2N0AWXIi27SaTPIZidbkLHiejSQxwdHKyWC3enD+vRlmTzp9dbN9//91q7/u67KbjimI0sIiod42ucg5haT2ECDGh8RQCiRTShNSXF1FED6Rva2Sc9Fx0hcqq6Z3jN3XdbjYUc0G87TrnXFaUjd+AI8bYiE7lMqTofTDGC17EEC7OVvSYxxhJhOOjk/M3p5eXl/uTvbbvdesQGRfERkQuto1WQnDKUoIUhgDRRBJIKkzjlch11zDGVKba9UYABEgEgFFqtPMxIRWDxyLS6JwjSDklQesIhmdKUAgYOGcpxt4aSqmOYKyTyLJSqVgK12fRWGtN7IzuUCBDRGKPjvcePXhojPnkk0+++eZZNI6JjBAyuAIP/vmUAeVUCNa0PaUQHDCORVnudnVMAIlnZUUJRucZQrAuRa+UNLEHgIQkAkQC8WroAuj+kyyJmxbkt9qFm/4jQEREhG/1JVdmnYkzxgilgAjIkBIg6ELbbbMs44JTAIjIUBWypIIRhsgHLmhCTD552/dt8iWW1tosU5DSreO7pusZ8Ezm56dnzXa3Wp6tFu3hwUG9qZvOhjQPrje9p5gghNPXb7ig3tvn3zx7e/pG645EHi1KlmNk0QIAz9UkhLBb6uXbJ865EByJzDqPXnPOMUBKXlLCOcMUESEQQrhUQC8XyyIno8PDSXFYyDknVSbGBwe5dY4yNpqMV/3y3//k302zo2Zj7h7PP/vo169ffCUErXty74N35N7e9myxDbRxtLuoBePMO73dXRQtpRQpuuC2y633nnKecV6Vo7ZtE6TD6YRSarZ+ZbfT6VRJo7W2AXyCEJ3kIgleNztOmRqVAQIFQiC9ffPSGEMAVHHyH//iLx4+uP1Pfvyd/ePDdbNqdnXX6c7GB4++r8pquVk3Zsdz7qLZ1hsXR4iotXYuUAqma2OssyyLDtzwkUgkRkhEImcYI7MhhURTKnLFDotCTVflerfbrZaXX3z69At4OipKpXJIVMkqz3M6Ohgo4c7aGPo8U0WRa91mkvjAE0Lbbb58snPBFqNqf3+fuDwiSCmRUa01Y5RJHr0FyeW4DJBM1zNIFa2sNl3bEgxKFQDgtCEk5XmOiEZr9qMf/ejw8HBbN7/46FevXr3yL0Jr7Ga3vXXruO/7LMsEo4hpyAkNIdx791FRFBfL5dM3r549e9Z1XVGVf/RHf/SjH/+TvCw++uijt28W9+6865x78c2plPKr+rPZtCI0Eeqn09Hh/j5J7GxvtVv155dP+dM4Lsf7d/Jaq9adf/Hsp5+/fNK1ejQazafzV6+/QR4Q42Jxdnh4dHCw1zQdAJycnPzlX/5l13WbzebXn3z63e9+93f/4A9//ctfvXl7ORpPV6sVZWy3242K/O1i5TxUBUnRM0zdblPyygfbdW6zQUqhGsmYvLV6Op3GCIQMg5lhaA1KKUCx2WwuLpZd1+V57n0UQpXlaLutu04P8QoAhBAmxHBCtSH4gfY/4AQxeu9TvWuV2l0zyIK9AqvLGOFKWnadSBQjpIhU0JTgCqBFTCk4F51z0fbBeYs6eMMRskzt7c/m86nKZaf1an1pjLnc7h6//6EsytXlZsBCBu5rnudFUSilJpMJY8y7MLj2GmPikGmmMogmeG+cKzJVTSqpeNO1v/r4I5ln40n181/9+vxyVY1HfW9UUQJhQwuPCQbHYgAYwnsopQB4/QXANUnQx5ugxcH2FBGpUmokSyllXe8GZ7c8z7qus05TwrTWA+JdFEXf9wNKYUyQkgziabhW4Xvvn789xZ/+bD6ZC8Ie3LuXl9XRyfHJyYE2Rgp+sLdPeIl0S1c9cmYteI+ccyk5HWSsfEh5iaOqiDEbmNIpJcmoEIQx0N2Gc84ko5QgJko9Y4RSSmjou/O+O63rum1bRFSZ5Jx/7zuP2rbfrVsCtChKCkLKTAgZgr9crwA9YFheXpZVdjCqrA8ffvhhmRfBma6pfQqDlzNQoo0BAEBKGQdCfEjGWOtcUZRaa2Ostl2M4ANQUjO+BUomk0nb99rq2AAi9dGtVsvJ/sQ555wfToFSRiGAIgs+UUIAYG9vj1Kqx5UQouvaoG3wHkOkmBBITCl675zL8zJY35qeEZ4pRQl3xui+H1VV13XBuRCSEEwI4VyIEfKytNYOfCvGKCHEW9v3/d5kjJiGNBbGCALEEIzuhGAxRmtdSkEIcTP2GlOWCVlmucttSslqE1yECJRdZWAywW+IFyrLI6QYfQwxpTicgAEgxdh3PUQgjFCJzoXNugaA2Xzc9o6T0LZtcJ4RqgRLMRrT/11IwP8E/ODbncTfz4TAK3cvB5FRSqWUSikhFOHIBL1qFFJMGCESQgxEpIKPx0VZ5sF5RrlD633s+76u26+++mq3bYUgxpiiqKZTZa398ssvZ7PZ8G8ppapRsd2uLy7OdtvaOkMIMJZJkQ0zEcbYZr2NKVzlFABQSoW4Co1E9MOvUXpl7kkI8T7p3iopDw+P5+NJ8okiW19unA0pIqMCkKaUQkiEMKXydx48+ODD9988/yrLJAAwxibzWQrh888/f/7s5W7bECYJo8OWyTlPiINngzFm2BkGnHLge91QjuAa7IkxckKHWVjfdV3X9G3Xdb3uujLLq6oiBOp6t9lsYoxKKbe8mFXF8XzabVdPm/XhwSyEsN203tjdptYmdF3vEhJBGM+qkr453w2GZogIgABXbO50HbEG8JusSABggt6AAYP+a9ioddc754YdlTEySJoZIzkVVzc/BAwhBm97HWMkjHIkIcauaSKAzBRHEp2XJYuQKCAgMiTReZPSoP0Oxg6NrQ8eY1JSFnneBzMajQRlw6QjxhhDsNayUVm9ffv2Z7/45ceffuJD4rkyLoQYttuts1YyCgPozblzrm2aT18+effdd5+evjAYu+hqZydZ/u6H30OWMV7cvf14XB1NJpPVavXJR5+uVqt1fL7ZraSi00kB6Le7peTy4KiC6HfN2fPX7Xy+39luY0/X68vzRmT8NgCsm3y1G/d2fXg0kyI7fX1WNyvOGUW6XF7cu/egKsqLiwshxNny8mRXP3r83sHRyX/4sz87X1xY7/70T//0Fz/9KaPkeH/SW1NW1duzi1HGyqqiWBUpSylmGRdCCKGCd3Vdv359mlIiyB4+fEcIxZlbr7eI+OjRo+fPn19cXAxvoXMuy7LJZPL8+XPn3BDkc83cvhrJhxBTAkLo9aeBEALWur4zeW7yPE8JrfWc87IctfUOEiIQRDrMJuKVeedQ/PCKmwQQQvTeC8IIR2910zSKIUCM0e/qTUwpJq91pzJhLgwArNfri4ul4FlRFEllAz1tAAzH47HW2hrHpLhitMHVnH7VdNPxaDyf5EoKSoAk1/ht3//kpz+bTEfniwUwWU33XNyUk1lKSQh9sxSHz3oIQQjhnEMkg9v/AN0Pz08SDmMg769m/5nK8jyvRgUArNerlFKeF4eHB23b9rpVeTnM0Qc3J631brcbNqOBvgtX8QpXgTqbXf/50+cHe+39k9tEyNFsigyObp08ffY1l6JUsusDT4l4R9FDIkAzihhTpIhCiKqqKIHpZHL6+lVZ5eOqGtIdGSfDu78vi+l0Op2NCSEDZ3Nvb29vbzagHbvdtm3zwYtm2Nnfvj4vy3J/sseYsC6u+7VzQSl/dHJ8evoGCBrjLzerozvf+S/++X/1wYfvffnFv4YIJKWEhHOOKViLEZJQcugZkDLCOPrkYvDeI5JBLoVIEqTgQ+/bFLXKi8lsbL27WK4TpKLMVQjbbd22rdZmt2uaTdN1OoQIiVASjHaMUiFYSmk8ro4O5iGErmtbtiEpEYRMSKDEhKiNSwkzKYnKh1i1Xd2mhEVR7B0cdbutt05KqUSGQINHSCwC0703zlsbUwJKKaeUK04I4SQGCN6FFN3Q8Dmnr4lUabjDw0dr+PwMlhuM0DIvKJJuMF/3iVLadO2zZ8+UUsfHx+PphDDqguecDpySAUQlVzbJMSWABHmWee9jCEOUDmfUt9vxvBztjaIPSuUUcLvdda1OBBMOQMJ1FU+/+foffw0dz9/aH3y7S8DryxttCFBKpRQMBx9HwAiCSSQQBw4oAQzBsoxzK4Uaj8dlWZpeU8rzvORcIlLv/du3p1rD3qxMEff3909Obp+fny8WC6WUtWa73RJCxpMKAOq69d5b4whFIQJiMCZo3aWUgv2NMyllZKjKlJIYI0EKcJVfgEgJYSmlQvIUIsREAOu63prV97/3O4yxt2/fXlxcSCkTgvfeOCulFJniFCiJn3/6Sd+sF4tFnonZ/v52U0MiEZExQTl3LnR9462NwQlGh25+MG4Z3t9rUNAPY8Rvl2qBXIfACJ/PZnE0Xq1WSxMwAaVMCKmUIohGOSn7EAJjbCZFWU5mGX399Zf1bv3hh++X5ejyYgNEnr050z72PkSKRDEUCCRtmraqKkqvzEgAgBA2qHggDbRCBCCUkkFERWJKKQ33kzHGimKgV3dNwxht27bIsqLMh48uIYQT7hA6Y4I1DBINwes+xtgbXeQVMuqNJpSW48msmhZFxbLMORdDTClRJDF5jEkJ6XVvYkJEQZmPITgPjOd5vlcdzGazLMvqur5cLAcvAEYFe/78+c9+9rOm677/vd8hXCzXKx+SKDLGSELIcwUpTiajIldnZ2/btr1sGoPxbHEupMonExPg1oOHsqjmeyevXrziJLt1tNf35v6t/aP5/V/96le/frGN3BU5M7pvd0vOiOJqf49st/XF2brvX5VlmZVZZCSfjyjFYDRjzMW2M4mIUM2zUT5OEEikMdjDg9tvXr6+ffv2e++/+4tf/OzevXs/+PEPPv38088///SP/uiP/uX/7l/qtvvko48zKR4+uPPRx78wWqtMvf/Og/v3bjVd23Ud8nGWZZwzypKUnDFaN9uutavLTdsaALA2QiLT6cxae3h43NTderXtO3P3zn3vfQyQqaIsRjEAZxIUEVyllLyLjgaPEdJVnBrA0CsMexzlnMcYvYsIlJKrQw/nHJETIiiNiBgDCT4lhgA4zIMBgLHEiAIAAB+jZyKTXEXJnW0VJ1kufdDn52+r8aiqsv39eaMNcmGtvbhYSCn7TscYEWCz2bRtK4RomzZeub91XElCiFLKOK+1Dgl2XTvb29s7OFSCe9uH5AlnAOn8cmVjunXvIfJTrgpVRhvSQJAeNk24Do0MIUyn0+FUPQiWhk7/N4xuRM55SjjgNzfj/+HhzjlELIpiWBu5VAMdlyTgXIzLymljjC0yRZAONr2MUs6uIpINbSAlWY1oljfWtMbGBMOdzJU4nM8zNe47v1v3xln0yUdOpaSEUsKVEJILRgBjUJKVSijFIIU8zw6P9meTKWPsuCzDzQA5+eEcNvSOg4psAB43m433XimlA9nfO9Raa23enl1sNrsYBlN3n+UyJJ8ITGAWEz59/rw3psoySQtCCAuOMrRWuxB8TGVZemOMs+A8Y4iIDDln0HbBWoDEhJQSKGUuNr2OfrOrOZNZWVTOORe4ZJIIwnBb6yEDeuhHQwjBe0hJcOWs9t6en58SmuT+njG6rrc0xYgJCGWCU8YFRCFE5uObt0sAUIpX1Xg6nUYftLZ126A1kILiglLaamOM8QkA6Wg8Heirxhire6s9gyTzPIReMgSA4ZRltYkxlmVpTT+QXoeRPQAMzj/O2KHRrKqqUJnOcudcSjj4Ctdts613B0eHKs9Ulllrg9cphRsm49DPM0IAIuGEEd53LaMwHld939frFQd4cGv6/e//cFRNUsKz8+XnX3715vVZ0+lv13sCECH+th/C/5vX30Vo+DbjgVICISZ/Jd8lAAjAKMUIiSAGBEQEQglmQkGWMqGKfMwojxwZVZzTlJI10ZpAkEkJUijO5eHh4Z07d5TMUNA8z+p6t91uEbEaFcMYNMuKKyZvujrAhIDeh1yWMQ7pjoNMbKCjAueSEIgxJgjDp2vYAxURlFMC6LSptztO2d5sv93ViitOOCJNkEJI3kUgwdkAyW3Wy0xymvL9+cwHd7R/cHj7wbIxb5bNrgsR0AWjraUIQ8rPsAYH2Gm4BteE4RUOLfsAoQNAplRT133dpMk0U1mZ5TvOCWHTUUYpdcYN+yGb7w8nnPt7e1WhJtOy22Q5dfdPDjNV+d6fL+uL5WK56wNlNJOeRBNdhJCNFKVs4GMRMlR3gERSGnbJwSr0hisD0bsUIcXfRNUPnc3e/kwq3nUdJ5RzPmyfiEiQ0RT74JOznFAKKTmLidi2Zwm5kixhAgjWJesZII+QEvbahhAIozRCSqCkNJQhAAFkUjJIve36vueE+hAoEmds3/dDgvwgamWFyqSU777//vd/+MNPPvv84nJ5eHg0mc96083n89G4bNuGM9L37Xa9rustGVd//fOfJZ8EV32rKeMffvBdRlXX9X1vUPDW6/PTi1ExIoS0W3P+WmeSqXm+qxuKYrQ36Xf61XZ15/Y7Auq3pxfr9W7XdPkoQyq6vie+YVFEH0KMzgVvoxTZZDbutr3Msr6v+76zun/w4N7nn3x8ubh48P579fby5cuXT77+/A9///c//OBxU2/r3ebxu4+ODveRkF3bPH/2DRdqMp+99/jdfHKrLAulVIw+QUgpDI7L3zz9erfbee+Ndq9evV0uLwGgqqqvvvrq8vIyz/M7d+68fv06xnhj4TLg3kKIQUo3fDS9czFG53wMEGNKaTDmTIyxGMFaP8wjEDFF9C5SwhmVlAwYT/R+oC0P2FQc+tB0nT8UQrDGSy4GI6Mq5/P5eDwug7cJXJaLLOcphQ8++OCTz76u6/bw8Pjrr55orQlgCG673Y7H481q7b2fz+fBR0KIoDTLsgi6aRoXYlEULvim7xAzRJIXFSLuNluuVEJ2/96j7a4/Pz9HwlYXayllJkWMEWJyzm02GyEEAkwmk9evXw8MwUEXEP0VXNnp9uZEEoe8ZhsYY9bp4Q/Uptc6Q8Qr12TnBOcpJd31oFKZF66yl3YtuYhXqDyhhGCC4HwKsRjPOGX5aLxtu8+++IoBZAI3m42UclyN3nn0QOXjyfSga93zl8uu73VSjDEpFSVAKfFWt7pr641gZDat7t29czCflWWeKeGc67omBv7m9flqtRqNRicnJ4KL1XK3Wp1++eWXzrkhpCeEsN1uU0plWUZJEJ68ffsWEfOyYozNjvbKsvQxjMfj0/O3XddOp9N62/2r//u/5pz/L/7ke+PxmFIEiHlV5IWSclSWuQ8uAImJwmBmgwxioIlzIULUwVpvWUix732ng7FBawvUAhIhlPXter1GRCp4DAQSYYxlWYHInL1KOxskFJxT6wxAHI1GQrK9/cmqs8noNKC40XvvQ4yMkg8/fDA4THStXpy/1DpVOZtOp8e37j958hSCTcgBIqV08K8fRlQhhkyKUvEYHCJwErw2h0dHs9ksK3Ln3PNnL86XixQ9YhKCMCbctfUQUuJjGI3Gw2yIIA3sCv0ekl3LsizL0Xg8vlyvL5ZLFwMV3NgaCTJKU4AYIUbPCKeUJR+cMRZwMKF2xibnFceHt8Uf//jDP/jDHx8c3tI2ffzJF4vF+dnpBVz5SgEAgQSAcVB9/E+jLsLfxpS86Wa+zYsUnCMiYMIh5itGSrgUKkGkSIHQGAnESCiXnBEiJDJGsxC8FDmjEgHevHm9vlxdLlaUciEUIs5me0rlRtvRaCTLbBiuZVkmhLjyXGeCsUApBwiUcsEVJULwDABSGPBOdQ2XxpRS8DHRlBIZcs4AIEWERADANN1oNOKMiZwqLk4OjxQXf/Xxx5lSQogAyVprnUNCWt33ff+z7ebs9dOj/eni/I239vLy8vXrt6P5MRkmhDLzgAoI51RwJihZXG6uoEpChk342vmY4JVAyd9ospxzUglJBCM8uNjYWte9pHJvsickSyFaa7UeHCev8q52mwWjk7vVwXc+eGxMf3S43/UOETGmrtPtrhHlSOTMx2D6zjiNgoSQrnUYQJBRMrjXJ0Ry0x8CJELw24oqAhQRYozeW+89Y6yqijxXBIbSMERwISVccLLbAiZPCYUUnLOUcs6ItRYoETJzISwXiwFomeGRlNIBxhCllC7GtmujdeOi7Da7LqY4AJUpWWs77JKx1nohRHBea+2MdzYEFxjn/PbxydHBIcSICd558PD2vfshxuVmub+3Rwi8fP68aXe279brdZZJiMTu+vl8v21b2/WT0YQjOX/99q+ev7h/5z4Htms1xdR1jTFmPp38b/70f//Rx784Odr78H/2Lybj7OzNm88+/XR/72hv/+TH3z3srP/1J5/+6pNfn52uCUuJwMnBiHNurRdZQaw3utHWlLKMGE9uH3/28WdScmO7sHMnt46ePXu2/sn5fD7/nQ8eeR+b7eVHlwvd72bT0Z1bJ7du3fr5L34RAFWWiZy/OT17c3b+z//Lu4MwYQgrA4yE8PF49nu/+4dd181ms6qqdrudc/bP//zPjdFvXr/WWpflaDyePnv2IkYQQjRN51xICQdYabAaQqTeuxivEnUZEzF6AAAESqNzfpgmhJAACGMcAFKKlApKPSGDGi2khEJcpZJf7RfkKsh1KJw2OmttwOCcixGRABeUMeZDoBRHo2q+N338+PHZxfr12SKGNGQYQkxFkYUQHj16tLxYcM6Pjo6MtlTwmJBzjtrgkBaslGBcyZwx3m430XtGaZ6XnNAYozFBsmy5WI9Gk+iAZzIEj4gxRe+91noQkZZlyTkP1qVrwyUXBiJuGpIUYoyEsJTAGJOiJoTEVN5oFFNKQ4BelmUxRillSslZH0LI83w+n1PKr+YyiJTSGKOzYVhLSo44Y86m9WrTbTa+bY8OpoSQ+XzOGEFMVVWMxyUh4IMhNGaCUxad7XV0meDZqBzNJ3uTUgp2dDC/d/v24f48pbC4uHj+/On5+fnmEvq+J4TM53ZTe+fcbrfrO/PmzQoAssw3HQkh9L0TQlDOx3sVAIAQ291u23YxRm3taFweHBwcHRwSSGdnZxjTdr1dr9fj8fi//7/9+Xx/VlXVaFrdunOyf8DykjkfhcgApZIMEa9ygbwNxjJRpgBdZ/uuMca0ujfWp5SYVF3bp5QoF86m1XbnnFMyd5YgopRZoUqCg16aUyRv3r7q2248rooyPzk5Orl11DS7lPxITrqu64221u7aXbvtXPBCsOXiNQBITidlfv/k3p1bt3/wgx9858MP/uo//FXTrN6eLmIglDBV5r3zztttvdGmowiTcVlkKjhvdG86Ox8X7zy68+jR4/l83nStt+b84qJva6TUMfQuGh8YY0gISzyEYLVp23ZIrwYAa4y9jv30KfoYt3Xd9/1utzPOur6nLBGCZMidjgkTSTDkaAIAYkxVViAEimkynzx+950/en/04NG9vUooGmyyut/Vu+12twVCI8JvIQh/S8zDP3TdiGu+3RncfP+3OglEdMZESCH4YXUARgAIYcwEzQjjnBNCrfcYkTCklHOgiCy4OBqPKMXdevPJx188++apNcYbDwm9j4SQ3W6329Z5nsOGDKrCqqrKsuy7frFYWqc55yEk50LfG++vFi+l1GoDv8FCbhIOr5DzoRQyxijljAmASAIGG+pNnUuV5/k7Dx/vNvXHH306RARLKWMAIVQ1mfR93xtNku/a7Xt//E9Ct52Nyvl8nmdlDOBDyIvRwRG3Pm7rTd8hRNcZPQwcb6RJ5DqcZTiXD1vNcGOHKPnkfM7V4f5hkeebzYYiu3VyUhRF27aDOuPVq1cvX75s21opVRRFKKiNgUoh1Mit3cXqcrNu1ut1IpRznmWZyDLOlbdd8sFbB1cBCmwYZA+3AuDKa/wmUx4SSTHCVTdyNRlJEEPwzjnrtLWaEOCcYsIYkZArQRBnPCUmBAsQIvjBPkEpBYjBuxA4YyxF1J3pur7p+5E2s9GYArYhZVwywI1ZN73OTw5JAquN1RpiiCEM8eJ5roZDFwAIIRghknPnHHvy9deTySQTMsb43e9+l3K22tUvXr6UgscY1+v1m5evEoSiyGaT6cHBnrHw3t1Hd27f++bJk49+/TGN8JP/8S8IYc74aVFlQl6uzjeXm4ODg0cPH4/H4ydfv9peXM4KtT+aFpn46PXPvvnsyfiHMxjBSE1uHe9RWvpIvn7xjfEdMpIVvCpKa21VjCSTm2KDHqzxk/GMUlpWlW761Xq5Xl4i0PV6GW29WbypqurenfsP75385Cd/HXxM3v3Ff/zz3/u93/vym6dcKO3D6cXSOCdU9q/+1X8/GIOQ6+BgSpFzPp/PrdOPHj26f/9uVVVFsS+EOD099T4QQoez/maz9T6kBBcXC2ud92Gg7BFyNUYNIQ6h4wP3JMbBgCgSQoZQPinVML4iV3lCiMgQ6IAueO8DS1KEGIASQoZllyCCDzEARsoQI8YIhCTAGKLr+7braoJpOp9FII/eecCzwkU/n88PD4+NtsfHx1muTK+n07G19uTkhBDSdV3XddZaxdnweoYJnxQSfUwhKMYLodqYggvjYjwpxpvNxmrz6a8+vnf/4WQ0F1zN7hxwLjqzTik5c0UdGv5S731RFF1sBij+ZmdMKQ0khhBCjDBIgYNPhBDv1c0hAACKosiyrCzLZCNFkiABpcE53XWc8/3ZvOu64Rw5bHMphCzLxuOxECQ412uDESjQs7ML9ProaFKN1OX64uzsDJnc1Jv19rLTNRMFZ4lg8N6mEFjG92bj48OjPJOZ4Jv16puvv/7y00903282K9P1iHheJ855kamt7V8+ubi4uNDaSilHoxEiBSEMQkCIggfONODi5av9/X01GlGpjO3bXe2i1dacn59jDLPxpFTZdrsNrTFE1Iu1yuYYw3a1K9beOXVx0fW299E9fHifUcoJhZictU4bb0MIwSId9GAhBBeDdQ4ACGdSpc1mo63J85xJIUUek7beZ2rEmVBKZSKTUiqVD8Z8Dx7eWy0XRZkb0yOm7Xa9XC52u93s6LC0NgRvg9/tNtV2DRiLIleC9F2TCf7OgwcP798t8yJ6u3zz5b07R7NRtlqBB/ApIIlcYEKopsUUCyk4g7DbrprtjlOYTco/+sM/eO+99w6Pj7IsW22208lISUxAxtN5SrDZbb23QzINpRhCqLe762SQgkvhnLPGJHCMsc1iJ8V6IDPOZrN3RuMnT56Q1AGkMGiNCMFIU0LvIkWiiMyktK4bj7L33n9069bJvTu3Hs82k7FE364X3fPXi08//uWbV8+cB3aduPn/2bThN+SDm7bgb6okbiQSiDiwniGmNqW+7y83l6vNejSZVOPRfH823ZtTxiikCECGOA5kMRBrPKUieNc03enp+du3Z7nKkh+i59N6vW5pmyIgUh27wfF3NBqFEAa9ft/3x8fHnLkhYKuzuuu6QcgD6So7g1xfV1Z1lA4FjxAiJFdKDYJJlWhKyfaGJeKtl0ycvTl12m63O8HE/v4BMJoV+dHJSd02y+WyDNuDvfm4KsfjMU3p0YNHdeefP3+5NbFLjHNlvG7bfrvdONvrbleo8Q1oP0C8Q0c1zKfgOrVrCDnrus7tH87nc4gpuOCtR4BBCU8IEUKlhEO6vfdeSsW5nBztIUld9Ahexxid37bdcrNuWq91JIRG79p625jW6Q5TQKCQyNC4DHYj6UqdeP0OD2OI37hr4dBG3Lz1KaUYI6UImGIKGAMicnF1bowYGWeRhpC8B2K9N87ITBlvCONIiU0hIcii4JkKMbpOs4TB2MvlcuI95TQYa/r+7avXg/Nm37YpeMG5ZNx73/cGANhgpA4QQgg2QkRmrT08PIwxvnz+ohhP6rZ5/uKVTzEkv9lsttv15eXlya2jvdn8YuHatjV1/NF3fnD//n29a7/mX1ZF+fbV66Ojk+1mc35+9sPv/04uxf/w+WeUpsePHz17/vWnv/4Eo6sy8eblU0g+6E4x0te7h3fvOGtWi+UoK370/R/ko+Ljzz9+/uo5xBElMgWAkk3G80wWzbZZnC5enb3+8rMv7t99sNmslJAAsayq73z4fuhWhBDngtXNann2q5//rJpMEanIqz/95//l/+H/+H+qZH777i2a5SGh9369vGyaJoSUZdlA1B8WwHK58t7udrtf/epXhMAPf/iD0Wj04Ycf/tt/8+cD3W84pgz37vLycoBDybXTOwAMszGC9Po78XpQd0VfUkqVZSmEgCGeeAjsCgSAxAhD/z4YLdznSk21AAEAAElEQVQwioffuVkJjDHfRUQUQhRFkWeEUhyIj5QiQTZQt/7s3/+PEeSwy+zt7SG58mkY+Erb7fbs7Ozly5fr1eaAH+WFHA761loulaBivdwo+ub2reNROeZjlJzrrmt2O92bp18/+/GPfv/OyZ2u7W/fudv3hnTOORd9uPGK2Ww26/U6y7KBjgtX4DAOm4tgou9751yMLiXw3qeIlNIB/0wpDkzAm98HitcGgmk4ExRFMZvuDSyHYWvbbrcp4XQ6PTw8rOt1r7tk/SSvcsaXpy9Xq9XJyYwQonWnbY8MheIBknZWUrHrV2VZFkWWCTkZj0ejkjL03n798tnl2UVTbwXjEP1ms9FdzzkPWbVereIyVlUlZVZMctrTlFJrWsZYJDFgIMiQk0RB+55l0qXYWSsl359UZVlGZ3e73dHB/ieffCI4P5jvzSbTD/7gP6OUfvP113/+V19dLhbrbgMAt+6ejmeVC857+/VXLxFTCtFqbdo+2IGBRfsUGWNFURSjajhYUEqZFJerer1et11XTcbHJ4fVaCKtbduWkquYK6113/eMdUqpTCrGiTGGcdp1nfeWQNRac0GXmy2lmOf5vJhM5yPvj4pS7c2ns0m5NxlVZb48e/uLv/6rb77+am8yffz4UZbdITRMx4UBXG0brRsiMqXEZrM6OTo8Otpzfbtbn3EG3/vw0R/8/u9/9zu/s7+/H2PsjeWUSMlHo5HKivfee2/XdC9evUqrjZSScZlSMqafVpOOsYG2xgi1AN57H1Jd163u2Vi43nddd/fB/e997/uP3n387/71fxddDDGmCIwwghQC+BgjhL3pBJLfNnqC+R/+3u/ff3BH6y71F5Jhngmz6xcXZy+eP93tfJVjH/6WDgHTb2MM//hG4ds9wd/sFW7+d6h2Q+XrW+23dnDoGzUTQkhRVCpnN79PKaXIQ4jGDNEG3lmfUlJKzefz7WoDQAiy1eUmE0op1bZ9oK4sy5RS17HBhHRoy4ZoKGsdIhpjYkwDHm61vqlncBWAcpX3xjkdGgXOrwhDMcagfZZl8+msqqr18jKEsFwuB/lSNRlPp1NgVOXZdDpNCJvNBnwcjcpdvQnOxgTHx8fd87efffbF1y/P8r2D8d5R7+zp+ak1LaMwpLwO/+JwDZqyG5bYcKtDCMaYuq4vLy9HQgHA0A00TRNjXC6XQ6I3E3RgQQ4bNeccEV+dX1jbh+RHlVKCKM42u+2XX3/VtYHwUsoRcd6mYIIlDEuh4rXpe4yRczlAGinBFe06YiI3OWQ4TGqGFzwgCjefhKIonLfG9DFGxgljFAC89y5YpCSk6KLjiUdIESMygpQopQhjXd9HBKmyEMJquym5bJvm4uz82ZNvZvt70+lk2EXfvHkFMTljnLEpegQgCbx1CJwQEm/C8EK8Kj2Ls8uP0xeHt25vWvPzzz+6WG93Xc9UhpSw8yYEJ7KJpflXr89c31ZV0V+m+7e/E3pjtv77j78Xg8P9uF2tDvPMb5bLl8/2D4//l3/yT7/48us3L54vF6vd8uw779w7ff7V5ZuvYtB929p+OyrYxemr0fSQCmaMk6AeHD8yjbN1aJafvbx8PRqNQF+a7VmeVU6j0zZYytmIYX60f6den+3vV3dO+Pv/+R/O2MWnn3z5059/st24n/yH5zTZZrM2Oty9e+fJV6/+V//8X/75X/3lL372y8Pjgx/++IdN0/zaktOLOsYolWo7z1AdHhxUZR5jLLMMMK7Xl31df/3Jkw+/8/7J3nEpaWBQlurzj34xrQpjTF9vgzG66wgAJhjahQF4jzFGaIWQhBBnIwBwLimlKcKoyqWorAGjGyHYeFJVVcUYI1L66OIu9KZDxLIomODaGiEEYRTIcMImlA3Iv5EKbOxsi8GnopgwdeDpyNuayLztahRYlEp3rTZ9oabV3uTe41uffvopQ7JdbRdvL/74D/5EiYphTlPWN0vThuOjWduaFV3PJlNtjav7XIq+qXdbOZuNQkq73u7WtfVcW5TZ4X/73/4//uk//ePt7nK9OM8LpQhvm52kHITquq5eb+eT2ddffPX48eO37RtK2XBzkNIIIKRcbNebtuvbfnBxts4SQMWV9T44xzmXouSs9IZuLnslpru+P7/YDQjN6enGW3f7diklUJQppXrXd50bl/t5nk9H00xUa7Mdl3PTdbtmB8GJInPRf/XN6z/8vd8t81uvv9lkbNWtd7dHVVfQ9XqD032WK1WU0YfJeBY9/uQvflrlhZISScElbfqu60znuENMLoFtBRUyk5iwXm+bpiGE5HlOCKTgTGct4hC5lBiLhGR0/Oarl++99x7n/Pz0LMYomLr/4P5sNHrn+EFw/evnz1fN69dffv7owb0P7t+vZvTly5fbnbhcbk9fnentKsurl6eXBEBlOVfSx6CdDcFdKcQg+AiMLbMiQ0o8xrwsptMpMux0F7wjfdc0TS5VxoWoqNUIETjheZ4Pu6dzhjGya3quMusTE0U5mgspC1LsdjtReK27VXPhkuIieadHbHb3zoO9+fRo/+DVqzcff/7lxVqX8/siH/Vx9k++96PVznz0yadnqxVRDXLhAS+Wi+ne2CedlVRK/sHv3H/84N6920fvPHx4+3A+n89jwuVyfXH+RjeNbfvQx1//7JOEJEWseOlDjMOEy/te2Wo6bk5Pt009poRxPp7P+t4UiHuMaW211kwKQsjBwcHJycmTTz9++/ZtDIZLbpxzwXKBKSXO2fn67d68KnJ663jv/r3bh/PZi2c7L4+2frJdwVffXPz1r1+8Ou9dAhekC5hwCOMBxITIkAASDDEmCEIIiMkYM+Defdcxzhm5PgwMHAtERBz0rt/GDOA/ZSfcEJKGh0hZ9n0fEwohFBdEkhDc6YvzvnH7swNOJAVmfEgEIqXaRSaScxaEvFjXjNDe4t7+3XbrJ+ODfusFZXvzeSbV2Zu3r1+9Pj46QplrbepGizkXE0oYl1wwQpWQlNLBO854a4MLMQQfOWPDkV2wYW5FCSEEcNBghxB0p30y0ANjIaUEpA6x8973l/XDh+9QRS/WS1UV5WS6t3dgTWwuG6MXH/31p5zLe/fu/Rd/8t7+/v52W2/6EEL86vWiNpGKbDYd50r1y/Om7seMs2wvABJF1rH2IQBSRrlLJLoYfUgplfmob1oABIfeBEg+9GF/vD+bjgglp29eVpMxkLTZbRPBXmvlVQGFM3673ZZZuTc/jjGen55NOBvl+9szzX354P33m1395mV/fhb29sYBCXCUmSAJbB9DgJAES6StuxQAKanr2jmf53kETEiBRMQr8xW4bgpZYpAgXhFxUnDR2mBt8sYDAoKiNEYfO+uu2KxUblvjel7mh5JzZ3eMYyYnKXQUqACmGHoXQq2BeSXE8+bZ+fkZE1z33ddffHH3wf3RaPRmsdyud5JxSIkiE0LlKqOIptfoImDyNrh0dcwb3m6WwHfdbrW+0AEQPGdYlJlQedO1hPAUsK3bV30HyUdvt9vd/fnDTrdSsOPbx85Mk7PeWyH4xcXFcr365vkzyiXlfDydFEXRdd39h/e0bndtXeasKkRRzjinzve7elVN9giJ1vSJ8aPDPcBH1vUvXy12u522hvWdC173jiSZwGeZ5JnIlezXSwxhPh4/uH337sktu9rev3u/7+KXX72u6/Dg3hRoSVN88fzJeDp9eP/Op599dMnp/Xu3M042tvPOSEG6ztfNRnFBFev62ppOCNa3uxBcAj+ZjA6P9gmBt29fv//++33fD8gVQBzyHlerZTma3MgCb0wVGWNa91cQHDIAMsj2UoRhyOfcwEWgWS6995wzH648swanigFOuOmL4VtesMN3dPSZlFQQ5+Nitfjm2TdSwWxStG0bkx+VOec4GuV20XZ9Xe8W5UyFENhgN+Z9CGEIcrw6f1AqhBiNRvv7+xGSCz4YF5MXgkjJo7dt2zqbUqIXFxfWxhjIZDKhlOa5MrZp2pZQGIwXh4Z3kHIM0zsAaNt2GG0OptdZlnXetm3bxmitHXqFYcLSti29Pmw1TXNxcaG1Pj09Tf8vuv6rybLsSg8Et95HX+lahI6MjEQKAIlCAVWoKhqbRdI4TbKtaTbWzbGxtumH+Rfzd3rGpruppnqKUxpVBSCRSC1Ce7hWV9+jtt7zsN0dARTplpYWEeYRfu8956y91rc+AchsNsMYB4uxwF2glDJCpJQAgKIodrd3iqKo6/ry8nK5XEacWqWMtZxgb4k1WirjEbx393631+l0Ov1+/3I8Ojg4TCJZWssxihldVNXx4X7EuGxrClGWpt57iH6d6RfGlIj/Oo0wCCDDQHMdowLhG/oOAAClGAAzujztdrtZQgGgRsuL88OLEzOfTHrdIqLkxz/6YbeTn5+cvHr1/B//6/8W/B5YLOq//bufTf7yo+VCISj6WcrjZDSZVm3jAUQYKusAAIyRKOHaGEwgIQQg6ENWCYQQ4DiOLSbee9Gq8FlprZ0lQRAYzP+8c945o3W30ynLMoxWEeeMsbqqqrKsRI0JsrpdTKeMwjTjSdTu7x/t7+0zxsp5ObqcSWWh9w0208Xy9elxJdu4k29n2UBbgImyJsuTxWJWZNnm6nq/kxQpvbOz1cuT1ZUhJkxIXdftcrmMomhnZ+f45PxyPK8WC4AwRARCaK+znRzwBoGsyMG1X3iapsO1Vcai4XDY6/cRIuPx+JunT9q2/fzzz2ezWdu2SpkgdQMAIASCgg4hFMVEax3H/P79+3meStl2Onk363gPXu4f/fKjzz755ItZ7SihjCeqEQAA5EPCQgATvPcAAuCMtdBcMZS9h9ATihFwb3IYbtCC0Dq82R/8F3978+uwVwriuPAohX6ivf5ywCujAYZXhiwIX8EPbcsIDXqcSZa1bWut7fQH9+7dS+PESBU4MYu2ChPIYrEIM7T3PkkSSmlYoYcqEVB9jDGD+Oa13YzCgWV51SgI4ZwzSoW9e29wtacPZo6c8/BBtW17eHjovaeUewestXnO8jydTBbOodHl+NWrfec9jYooSrJOUQwGK6ubVdO+ePFqOltoFyZ1T2IScAWlFAJQKaWltNaKpl0dDPM8Z4yVZdnUdbfb7ff7lDHvvbJGCAEQdBYwTj0ACBFGOKcRBhB6pJQkGGd5uj1IHz58KKU8Pjg8PDyMomh1be3OnYWH2AEICPUIeg8inkCMECWIkJs7IdAJpZTaWms9xOj68wTwmqrir91Ib4rMTad45XTu3c0RAwCYzuZpmhIKluU8CCIwxpPpKE1yhBDwEACIEBFCVVWDAGyA1FrzOAot3U2uEKUUQWTCy/PSGUsxNkoHJRq8dgi9aWrJP/7jP3AeSu1nZVmKqr2Y10JDyniStJWAEDDGquUcQc8JwZT2V7rHZ4cxZw5YD531xgFvrKOUVVV9cXm5s3t7c7iCGI+iqKzr1eHw+Oh1ksVNO59MjuKIJ3HqvG7asqnnlMdxhBSwTtVZSh89vC3MUSPFeHy5XC455918kKcdiikiPqEUOe21wt5ljA06nZxzMlxf6W2myVC2+JNPn0ipEfWjZj4Ybp4cvnz4+J3t9RXZLjYGHVEt6vl4Y63f76ZnZ2ez2YJRGHFodNsotbm2aq3VRgBnAUyMUZeXF1W93N661e93syzjPE7SlHP+8uXLtm2l0V5ZrayVwnuH8VV00E2Gr0fhqaUAgPDIOeeEEFK2lOIoZpxz56hWMlw2xliwE76pJmFfgK5N6cODwSIEsAMAEoKAt2W5qKrlxnq3aStKYb8/7A86Dx/edu5QNH65nI9GFxB6TGArhNKiFfVg0APAEYoYJwgDCH2axp1u7gGw3qllPRqfqVZjICfezmYziOjqylZ/0CuXYrmolBJStUmSGJvXddmK2l6HBQSwMWB9obWq6zq8uxuWIkaUEh4Sga/gOHsFyWBErPEIkrpqZ7PFYLBycTHywIXnLRzMLEROty1OkiiKer0eY2w4HCZJUpblaDQyRkDovbMEY0IJcFYp4ZU+v5jc+uHdIk+bpk2SuMjyYb/fLQqoWbfIunmiy8V0MsIQyaZVQlqrjTFKGSFlMEBllHpwNQkEiTZC6KblChUwPMz+Ok0KY7xcjCmBBNmdzUEU8Wo5d0YvF7NHD99q6/rOrR1v3cbGmtG6qqM1OJhP5ufn51GU/LM//uc//tEf/v3fffTZF19oLZbzWRzzGCBtnEcwY1cJfggBjDFGGEKMMIbIM8oJphY4SjiBRCnRNE25jFwKIITeGaVE21opr0AdALDWsiyNtTaKGKWkaar5XAkhOKfKEIxxq6SWltEoSzp1pX750Wfrq6ttK4AFWtu6lhAQyoCS9sunTy9Gl9JYHiV5lmBErbUU+1tba+trK9/7ztv9TppyksVMNWU5mcWdASVeCHl2MTk9Oa/qBhEGIRJKIUIIhpBgCKED3l6DtOFOa+q2aRrGWHC/0FrXde0caNs2IOebm5tpmh6/enXTat+cx97bqlJFFjFOtjbX3378Fo9oOZ9nedrrdZ98++yTj79+8nRPCBcxilhqrceYOu8BBle8IXC1UqSUNkYBZxCGGHrgDHDwv0hyvNqm/VfEDv+wpQhfNwXhpr8Jdx1445jBGAMMEUIOeKVUzDkAoG1bS0wWJ4PBoF5fX8znwWwgz/Otjc3paLy3t1eWZX910Ig2/LOB7e9hSFv2N5IBeB2GQilF9npXct1AO+e8ddZagq7gcXBtF4ExVsqEREBGoziOw4Mfatrl5TnGdDAYWOu1UcaYxWLx7fhyZ2enFWq5aFqhsuIsSbPZbLG9c+udd95mUUIY/eyLL5pGBGk15tQYE1S1CMAb6XKIq5VSMkLDi799+/ba2tp8ehYOQmV06JAIZxgSa+xsNnPaaKWcsfPxCEIIvB+QPsW+M+hPL0aj0YRQniTpw0ePq6Z1AEljylYgZzvdNMlSGvGT+Vlo6ewVJqRCexgahbDW8R6i68KO/K/bx7CzCOU07GG9995Z7z2A15RMWQNolZLOaYhRHHPOeQhF8x4YazGmjDEEmXcGEZKnbDabOee63S6h1DkXhOv9fh950DaNlsoqba1F1823fyMJ5aYjJ7u3txHCy6pJyrRs67Pzc0ggoRHEuFrOKKWrww1ZN84ZSqNOt8i78Yu9JwShJIqN0tB7gvCyKqVWiLJWqslynna6AMFFuTy5PDsQ7enx/rJaANuWoqYM9frdJI+sk2fnR4jQrd070rrDsxOh2n63+NEPfryzub2///rk5KhtKkpQGsUOgsWyVhYZTPpFjjtxP+/ElDmlraEI4Z3tOz/4PrWa1cKvb9ymUcHjrKplJ43/6Ce/++DBTpTEysjdrVWSred5vr9/8PXXX2tlESJlWQqhFss555QxBBGkFHHOkjSKEx4OsyDGBTBO0/Tu3bta65ev9wMuZ6wlhAUbgOAbSAj1Vx4aPkwV1lxd5muGi5NStm2rtVLSh94ihPFcwXqMhRIAr6MHQo3AGEcJklIppWLGszS7+VveeykkhL7XyR/ev31xPm6Z86A/rZs4jgNfwXtf1/VwOLTWpmmaZVk42kMJUFpDCCMOCfJZnq2uDpQW1XJ6OV5gSNK4T/EVC6Ft214v1yaum8UbbwoEJWS43efzecieDq8tSHK994CyYP0UgAdrrVHae08pJQgHE8ZgFvngwYP5fH52fhnHsbU2HAmM0PBeFCHr6+txHNd13bZtXddN02CMMY04Z9ZaBDzjDAAPpHBWvdw/vHXnLue8KsvD1/unZycrg8H29jZ6faGUUlUZIcgh0Foi6FtRai2dc8ZdNTeYBNAICdHeIHLhioR6HRqFUM1DeQ0YkndifaW/ujr8zuMHBLrTY7+zvTnsF7PJJM3iB/fuAwA+/fTT/+X/9f8MRfw//9nfxWmOEOn1hx988N0/+MmPdnd3v/7228+//NpY10phnCeUIkws8MBBo8N60yFrEcGEMEr5lan71UWJnHWhWPR6veWivAoBRyiKoiiKjNKiaeM4JoQkSUIxUUqJpoUQ5mkWVtQUs15vsDLsdrudo4PXX37xNPndrmwVpVwLt1i0aZpHcUpo/PrsuFzWWmvG6nCVCSGMkJV+cXd7o9/JuzFPY0aA94AAaxdlQ1jcCPXi+auPfvnJ+fll3WqpLKYUAuwgwAAABKEPJ7O7JjbiG7i+aRopdZIkrRBa23AclmX55MmTJ0+ecADANUvmpvBZa4EHCIE8T7e2NuKYaq0wgdbKv/rrn/3d3/3sFx99KQDodzoA8/G8VM4kSeGAd8B6j7y314e8jVlkFUQIUAwggMAjBDxwwbkBvXn2ewgB/AfCiX+QLvEP0YWbkzW02ghhQgiLo7DmQGGQxAhA6KxVRsNgj2GttI5AFEXR9vY2JWT/xauzs7Ozs7NbO7u7u7svXrwol8s4jq13V6x7751zxjmlVNM0ddtcrVOv9/2EENHW4bVheEW0ghCGLLrwgtG17WA4b0QjDbGc8yzGFDOCMMU05pG3AEFIMCQIGyWgd1K0+6/31hKwvU1XVwa3bjdnlxdKO13VrVBCSURJp9fd2N4UWiptQr5l6X0YzBBCYV0SPvRu0eGEvn79ut/tDYfDtml2dnYC4BHHMeccIqKdJoTnSYEpZYRcXl7MqrnVBjhjtcIQcs56xa5q60G3e+/u7aZpvvn2eSNlXvTrVkJCqlacnZ8v6ibs+3gS44JeQcL+N3LyjDEI4BvkwN/0Ug7e3MlhIrLOAACsMQFRgM4656zT16w4MBpdzOdLyjBjNJiIMMaaWngPVKsQ0hBgrbUxLo2j88mJbAV1XGvNOHcQQAjzPI+iCHlACdFStb5+o4f2v3UHhktMptNp3u0CAOKYxxHlxBNKrXfWyU4aee+9U9q0nU6HENIqSVN8eXjqLciT1GoXRxEnLER/J3lW1u23T569PjpO07Rp22fPnqVJcn56mMSk34k2ss08jqKEj8djzrLl4ny5aKIk7fYGVjZH+6+lUe9978P33v7+/dsPP/vs46+//EI3StHWaZQlCTIo4dHunXtFTLOUJTwDFmtDvYeMsk5n5a233+dRvrG541GkjBPSWO9u3d4qOsnB8cHKoO963WkL1lb7nBEt5Xy+UMoQDKXQFKM4jnlEGSNJlmZFTinGhCyXi7Ztw7CojAlXuixLCAEhiHMKNSIEE4K9txhD73E4O40xwZ4s9Ps4aF0Rcs4gdMVnVsp7R8KRyTkP8HW4SOH0fbNGhHonrLrCGDHyHk5mS8bIxtrw1u3N5UK0TUMQTNPYatlUTUSIBxZhYJQK2YBtW+9sbU+nU0ox59RaLWSDCXHOCNlY5yIjBt38B7/zvXfefWSd/uijj/7sz/96MZ2ciQnBkVaOdDIpW8qI93a+XHaLQggR7qewFwi/DX1rkiQBRwldlLXWQAkAwBBZ47USxhhwpQUF3phw6BnjRKv6veGgv3J+Mcrz3Hsfx3Ge5xSTgM5xzrvdbpqmUsrZZBrI4f1+v5FLSilUGgAHKMY+ZonGPm61/OKrr7vd7u2dnU/Pz148e37/zt3vvffuUsNnz54dnZxRSiNCsIeOUOcs58xYi4xzLljahCfchMJ9M+eFofxmMREqQhgLwnd2MoyAsapp6zmFTotFP7/z/fceffHFZ1Yb2c5///d/kqTk3//H/00Ivbdf6gaswhog+Pz84vTk4OGjtx8+evzf/et/8eDB3c+//Obbp88a4bzRpVaM0CLvNGrhvPPea20hcpQGzNPxJG6amdMmiiIp5Xi+qOs2xLtEnnvgrLUIQ8YpoRgAQChWSqqFDEuiLusEPXqnSNsWra70B4NelmWMIOjAbLpczCvgvDVICIkZH6ysZUW3LEthrEfQeGdVC4CzRnFKsiRZ6XXWVwYUAikaZDX2DgFAaSQAOTm5ePb85Sefff7F10+kAJQhrR3l3DjvrLHehXPZOn3DUwvdbZ7nnU7Hez8ajQAARaeTpnmWZYhewd29Xm92fm60gxBa6O1VxJr1HnQ7ifc6z7PtnU0eMYgshPbbJ9/86X/+m+WyilNoG1+LhnLAOFGt8d6GVAbgPQDWAweBgwCqpsLecsYxxs5oiAHGyBrgPIDeAXCF4np4tVgLqgHwD+Ilb1YSN1+/MVZeOwQYY7ynxpiUUgjhlQW7NWH1oIz2EGkpw7+glZ61Io6iXq/X1HWWZZPL0dHR0f2793q93qNHjxbzuUE+6IbChwMhtEFS3zR1UwcC481qFV/H1CGEwPUMA6791uBVaIUzxlitjTGc826eQAgjGqVxnEYJJyyNYk4ow6VZWbXWOqO0bCkhzqjZbLZyazVOk9t372XdXu/goBGttp5QjjEdj6ZV3YpWbu/sJll6fHxcVg1UIjyGwduNcx7IE2mceGM555ubm2tra/PZLNhBegQJZ9QDY60xFiPa7fSLopjP5xhiBLAHSkthjez1ezs7O3e21rE3TslukXU6HQihkBoLFaV5nOaZsx4TPB57CLz3ThtvYIA/AbpyhAw+04QQeGUvh95sIIjDAIDgih18opy3AAApJcIAIYSvz+xw80OItQ6Bxsg7ADBSSs7KOWcx59g5h7FnnNS1r6olwRAh1O33MCWhSQr43BVIgMlV0B2ExhgPoTM27EbebBTCjEr29o82NozUJoxrWRqHUFdMyerK6nyxqJYzq8XW1ltKqdls5qASplHSVE3pLehlXYqp9/DO7h2E0DdPn4wPD1ohsiK11l5eXt6/93aUdrqdCCBtbFsLuQJJmvIi6Usxq8tqPpoMBisrvf43X3/18a9+NT6pv/f99+7fv/Ng6+H0ZFzXdTfrxzSJee6Ez+Nsd/d2wokStcesVW7Q3RStuhhPDg9Oq1omjk2evDg8Ppdaf/d7H7I4UtPJ0cnJZ599JpU6Pz/vr9063tyhlLdlZbWBHiQ8JVAVRddaSwjKO0Waxsaq6XRRVZU1MmSgYUTny0VVVdY6KSVAiFIOITTOOm+cY8FvPJi2uWu2f+j6nQ3WQyFgzYV8XqWU1oqzDFwLHG4uoXuDChuKyM0UK5XO0wJGCHlUVmJ0dtbW1c7G+uO337LGAI+iKCEIIwAWsymlMYQ03CJhgpzNZu99512l1HK5rOvaAc/iKMtzpVRd19a5ZjHq97u3djYe3L2FKDJKPX/28vRkbEyTpKkkBkI4nU6l3qURT9P0TaTkRr4chrZQ4AJ2Aq8p3BBRhNC1t4lwznHGgtOztVeWCWELE/qzkE8Rbtwsy4DzIZYzjuOw2phMJqJpwRsiEQCAh8A5YDzwCGMeRwQTCI/OLiez+R/9wU/mk+mvfvELrUQepz96/31q7fj0RDRVnGYGOAQBwg5iDz0AGCCMECTQe6O9N5YwerNfCEhMeIYD/+PmqqEb71iEtFSiqcvZdGOlPyyy5fTi9YtnKSPfvnx6ePCiU2Qekw+++x7C9OT8rJn4vb09COGjR7elVp99+vHhwYsPPvzB2vr2P/3jn7z73qOPP/7s2fPX3gJGkVUtIQx65713FhhjkLVaWyGUBX6xKLWQnNO2bqbjajqdaW3ubm8RhBLOpJQYeOQdBt5aW85nYSyrlwtRV+F9GXMFpK8O1weDXl03ysGNjY0kSb744guMKYQwwXRjs7O5uSmlnMym3gLOYwgh8iCJuXcGOUuhz+Jo0Cm8lUYpi5AyimEEMZIGfvLpl7/4+Ucv9l47C4pOCgCq25IlFAEbgFeMkIfOO+CveTA3IFZouBeLRZ7nSZIMh0MI4WQ+U0plWYEQ0upKC+2c89bCKytcIGSDAIgT/vDh/Tt3dqxRF2eHv/zlR2lBf/JH/4Ty9G9++vc///iZl4u8KHKaSGmunNW9BcAB6BDwAHihXb8XD4eDtm3rsgRaoyi6Xi0HTybsr4GE31or/NcQhTe/Lezswi0XZtBwd4XgdWOM9U4YDTC03kmtEKHOGIQQp0xrLerGaM0gDuCzkWqxWDx58mRjdQ0AsLW15SlCBC8Wi7Isr9RGGEN4heFDCDnnAN8oyfFNkAH+TT2nMQb66/UKQsC5K/5T3gEAxHHcKboBtQpBskVR5Hk+nU4nk5m/YnFahMB4PJpMxvcePtja3rDe7R3sL2cLbV0zm52cnWHKL0cTB3wUJVXTsCij3kZRhK4d6P21asBaa7VOkgQhFA5gIQRjLM0KzuOqbufzRVnWmLBerwIOnxydirYhEAEHlnXtneoWW28/uJtGRFu7nI8XS3l2ehzH8SpPlQWHx6d5p8vThLKo6Pa0lhBCazVy+Aq9h1ddAmPMQ0gIDauHAEy6a/zGOeicM1YFhbNzzjrrvddaYw8JITg0HPBqIBFLk6U9zjJrbTfvbmxszGazzz/9rKoqCCFErtst7tzZSRJWNzMAFcGk2+16CEKMMIFXGhkpJebhamMIryxGw9MUNCw3tfSqjn38yy92dqbamtu3d1f6gw/ee18Z/eVXXzkI8pSdn1cQoyjGPEKtkEnO63ZpnGpkM5/OOYw6WYdFvCi6t+/dnU7mrVBK60a0BloAXJzFo8nCW2sAEmUdYe+xT7PO+nDTa6QENI5ELGaYMMITHmdRvDhZHCYnPd5ZL9Z/8N7vNE2VpjmjSb1ogKPeeuNQKz1EJOsOIfSQRFY2jZhalLA4XtRyNJ4um/rBW29/93c+7PeGo+lkY2f79t07T58/u7i4aFq4XCys8c4BbwEljCVRBdvRaNS0bZIkaZojxKzUwJMkLjDWgSkzr+ZN22ZZkee5Meb1wWGS4DiOrb+GBzEwVlWlQOiq+w63RThI3mwkvbdKeeecEK1PrwIL3DUv+qbrvJlTb6oDIcQ4RllitTXCKA1bYRfzZjpvyqVIkjxiJI6TLJnubO1cnE+19tZqIVzTVJRSY9TBwWv3+z/GBFbVUojGONs0aZLFADhrtQPeiorTXppwJWpkcJ7yXrdYLhuEo25ndbaopDKj0YUQIsvTlZWV6XgcXnM4+ENFDjUiMA+klGHXED4KRDmE0HoDgEGIIAQwosAjCLA2Js+jtm0xpk0jRqNJsLfqdDphmwghdN4558IAMZ/Pl8tlVVUBcgySJI88wAA6DACwHnoPAMQAc0wAwnRvb//rr7/N0+ydx4+VaE+ODh6/+72Uk8nF+dNXLyAGyljjgTdaIxIA1ysoyCMEg789Cc2fvbZnv6mVV+wqQkLCb2jVmZcRYxGNqsUy3926s/Ho5Pj1t199eWtno6nLJM3/9E//VDmf5/nDx+/8fpL/9E//dj49Pz+vT0/2ByuD737woKzrv/jzP7v34O4H3/3eWw93KYaU4qPjs7Ksy1bm/Q5wzjrjAYAQI3j1hFdlE3ZAQgjZtgAAY8BoNKLOFkVBCJFSOueUlGF7xRjr9/vhUw1W32H/JfWUYBZxvJiPj45OEKaDwUAIGap/uCf7/Z51ejIdSVV7j2hEPUQQgIhQrYwzElg2vjytdta9NZyyKI2EANKYpqqfHVx+8tnnr/YPnAU8jjCmiNA0D7EWCCLIMMaUBEJAcCwLA67WuqqqOI7jLI2i6OXLlx6AsAScTCZ1XSdJdXFxEcJNwtNk/BWzBCIgJUhjIGVrjLZOH58cTCajjc2VW3d2v/+DH+adflxEBtqXr46qtm6EZTRCwc8DInR1onsIwOYwf+/9D27duvX69euq+nhRCQQ8I1hqczVqB4O+K6tliPyvBc83p9pv9QrgN4Hf8H5vdpEYYwRxkAI656w1N2RGpRQw1lsLANBSBY5hXdeyambTaXg8m6Z5+vTp/qs9QsjDBw9WtzeyLMuyLGBy1lr4xr4jMBABRs658A2BtgwAgNcjjb8JrwNvNArXY5JRFiGEAMaQeAtkq5TQccw5Y3maiabVmSqyNHwCG2ur3syEaYIn/aycnp+fTqZLwrgy9ujkAnh0MRrNF6WyhrO42+1CCimlKKRxOm+tNRCG5kA0DUJoMpmMx+O2aULSTZjQyrIsy7JtNCJuMprMp3OjlDcWQ4QZS5MIepwmccxJL8+kMlWjzs5P9vb2Gg1o1Kla2UpVjUZswaI4JhQjxL33EHpEaSj7CONQpjDGLgD46AZRgDc2CgGAkUpela+rkPEAfXlrLbAeIQSDSwbGwEOCSZEPiqLYWt/Y3t46Pj4+Pjg5Oj4QsnHa8Ght99ZGnJDL0YnRenyxVEpBjDDGgzgOly9N0/l8bjDxzt1g1YwQ6EHTtKEjeRPWcs6RyXTBeNKKut8f7Ozcyrudpmm++earu3duT2azxXKyur4GEB1PLo9PTnZ2dqpqqa2Ssp3P3aBj+yvDnc3t2zv3jHFVU89mMx4zzrkHXltTFIW3VBpZN8opv7LWBUpdXo7H5zMCopPjyyTurK5skYPj/cPXk9F4bWVtLXnUltXhi8N7b93aHG5V9bJum3pZCeESHkmt5ouKM7y+2t+4+wBQ0h6PIbRR1tlNu3Ujvvzyq6ZVd+8/vHf/ofVwspwfHB1ubW//6Me/X/QHSZJcni5+9atfvd47aGVblQ2AFEI0Wy44izE2GFNn4WK+nM1mzttutzudjsL1ppQmEOZ5aow5OjpSWgKQY4wh8Na+YVJrbXjKwv/DfSBaiRDSGgSWn7UaG2iMadsGQR4Om0ACCkNqcCW6qSM34yljrNVEKV0uKqdhwniSZBD4ppZHh2d37+2giMZRmqb57du3Dw/Op9O5FbW1VkqJEGrbdrFYVFWltQ7nGTCaUhpFkXEuamrrHe8WW9sbd2/tdjp5LWqMsTG6qerRqGU0v+kJrNPe8/Dy2DUkEKY9jHHwSoqiSGsdZtMoipxzYQGBEArDxw2ZMWT/hPy3IDhEAE6nU3cd7gKuR8kwSyVJQggJ3g/omhQZSBgkIYwxBN1VcoS13nnr3XxRZ1nx4uWeqJb/+Ce/9/jx46PXe69f7d2/92h7Y+Pu7s7Z2ZlwRjuMPEJIRhEzHkCIMSKEMGCB99JZDK5bwPA4/dZoGN7+jWEGhPDy6JVscbWw+y+eMW9/9IPvlrP55cXJ0euXddv87o9//8//6q/3Ti6//+H3it7l5jbf3dl8+SLXRuzu7g7XVlfXVxCmu7dOT8/Of/b3P73/4NFP/uC/uffwwX/493/y5Vff9Hr5UnkAAAQIQICCoJxwSnErZRJnBKKyLDmPekWEMAi7rVAxw6cdjHu9951OhzFWVVVd1wihoO65vLxcWYsAsK0oZ3N1enpMCCuXy9OzM8b4xoYPutC6LqeL+Xw+j+PYCk8gUs47q52h0DhnLPQ2iXiRZVIKQhDEaDqfnZ6eXlxcHF3UJydnlNI8yy4uR7P5vNvN+/3++eUF8BASdD0pAg+BdT6484bklPB2+v1+lhUPHjygjDWNOD8/Pz09jeN4Y6PjnIPyCldXSml35c/hPYgi0Olk4/Hll1993pSTo8PXO5sb/+bf/BuWoU6n1+kO/uW/+udbO9t/8qd/8fNffCbOppiEDKiwnscQegQghPDDDz/8R//oH92/f/8XH/3y4ODA2rMoSawHcr54w/UZAAA8BNBfbfDf2Cz8toPCb6EO12ssH6bMK8gawV8vI5yVRkOCPARaa4AcDisJIxihjJCmrieT2WQ8vjw7c9pghJSQF3VDCBn0+7UWy6oMJmmhs8eU3pClQpXw6Ncw4a+n9jf8YwAEaZoGXWKohEqIcPLpGoWeJvgdhRsMQh+i3Ywx3W631+sppRCGW1tb60NalvW8nFyMzo9Oz45PTxCmncGQGT8ej0fjWSsUpVxqWy7nZdl2NxNrLXAeQhg6l/BhzWYzUTdJkgRzuXK5FEL0er3+IAYIl2UJAMjznPKIMdY0DQKgaYSRFUYGA4gxqpeL13sv19mdVmoDMMNXe5Y0TaO852k8Gk+nswWr6ySNw/iNMezQ/Ao2CC3pPwj6evMX4Joxc1M8b/6EEOJ8ACkNhBDhK+EDo9l0Oo042tzY3dnaZIxZe1Vyvbd1rYRsMAGMY+tU0y7XdzfiOJZaYYy73a7Qqq5rAICUkiLsr3MxEIDgWq9x0ybevEgIISFxfjFbeK//8m//8t7bO2nECWsfv7Nz6/bO0RE+Oux5D62PVAnvbH5ne2MX2ye/98F748tytqZ63fX7t9+5d/9BkiSHhweffvYlIthaq7XFiHhNgYqmap7GsfSw29lAJNOy/vbbc121RZIupouiqL/yv0rTPE7zAuXTycyS84ST0cXru3fXXx2+fvriudKGc+4c6Hd7O5s7q/3i/u37w60doDGYVXsHY2v13t4poiBJmccNTaWDU2XOXr+eLBblxfn49Ysn9x++vb29CyGWbH5vc4VD/+r10avnLy7GSx5ngJDd252Es14/Rxmt6wVgmlHQ2hng1iGvlF7ZHhBCJuOZsILEmAJkgTZeEcwA8EppCH2SRFKUAAQYwACIKMMQOW2M9U3C8/msxhhyntR17RyCgGHtsHZQGq+1lxp6CIwFxsaUGIIghARD74yxljFGCIpqhIxi2k4mk6lo45jjJPrm26+kapIsz9Le+cU8TYrd3d3p/N+dXZwJvB7Hsce6lg44DwCWGiXZwAAGaYqgMT5qFG4N0Y4771knWb13p7GgsCwCiIjR41sPmWMvoqN5fc7Tgic8iftnpxeq0bdv3ZULNV0sirVe1IlHkwmw2HukpDMaYEQXomZRkuZUCAExhQD28kwI4YwNo57W2moT8JI8TcOJZYwJAq233367lZUxVkqZpqkxpt/vL5fV2sYmIeTk+FQajzHTgHjvPYkAhAAapZ3UWsiWUpx3UwydaOuoixeLmYDtRTP5y09/9v533u3eu/XTv/rr3i//8oc//tHqer61u/L5t8/y4QaOMkmmBqBut+8caKrGAwiJR9ZHGcMYaq3bViOECWbQQuOBd8hZyHnMmHfQQh6hKG6MnM/ntTS+UVq2eZbsjZu7LbmQ8YtT9fajt+7eHf7V3371/MXcGPT5R6/mI78/uAToDLEmyuHm7bXf/dFPNjd2x9OZ0Gh392HdiouLi9H50VtvvfU//V/+1eef7vyH//B/LGdgZZhKbYRScTeNGXJKQppEjLWNlMICixEkELAkSvvdqJMwCGmWd42dUcaatkLELxfzJMJp3MVec+QIgPXlWAmxnudra2tSytOjM8ZYUXSllKEmZllaVWUjRWgTEUJxHFsHMAPCNK1uB73uvCzjJBIGzo01UfLl3uu1tTXk3fRitGzE06OT/f09RG9LmCVFb3x5zijdXk3WVocvX77MEWI8hpRqj4QyxiMPMsrTLAJtWSMACcUAw7Rb9NdWkjRVyty9e1cpdfB6HzM6Or+YjUdKSAIJpdRoXVeVV5ZQigGMEYPOydLmg84vfvbFrz76/Hd+58P7jz9k+dpa5GRtlZMrK2sfPHzv289ffo6+ThnV1jRCxjEFwBtntAZxDL77wXd/58OH3/3gfpRm3/3+20L/sz/5//7ZweGJBZAQAgAC3iNj0PUWH3llKHnzkLjpFcKod72HvirWzjmGMEYQOe8A4JRZ75SxUZJgwrSFWjoHgFMeSAsJjmBU1pUnJDguyFacjU8vzs4X42ldVQQiDFFwEU6iFGP8/NlLRCDnvN/ryVowwiqhGcWXp+fdbtcD0Ol0GCQIohZRhwxG2EN/JYXw0DuglXHOEYylVEH7EOiWcZRYa8tl1SA4HA73j8+khVHWBSBaLtXl5VIr1el00rTwzlYLZaxOkmTY2+xHdU7iP/uLv6Osy/BaTPH27dtSS4fahMgNHpVNWS5raDXGwBhVT1Cn09FGAui8d9VyjqDjnCKE0hytrhbz2RIIiYiTuubxkPN0PJo0tXYOYGK8aMvKYEyVEsAZCChyAEPEGReIzYT9dm//h7/74+Pj06cvnicRXVsbKgOsNy22JdTCGwqIakQrBSI4y7JmvmTdLoDIO+cpk1o3SmMeOdVARChlEeOEEAyId8BZb7wRWgtlgAUAIOu18/4Kh9AAI4QJtdYLIbxxGKNGTYsec1AcHL6cTM+NVE3T3L9/f2Vlpa3rra0NgmG5qJVQWZK9/857j+4/cMC3Uo6X88vZBDY2yfo8iXFEyvnCWxfoCwgiCrEUkkeRMUYqFSyzbjAtAjFXosYQYcTKZSuaJkvJ7Z3dnc0tUTXHB5Mkhbu7D+7cubt7+75odUwf3L3zVqdYHU9ao3GRDyiLMLySuUOAkixjkUGISqENgHGEEDQIOuPqshKukd2CD2+tIed73YhTlmSARybJPItwv2F7rw66/V6apkdHp3XTOAsvz0fvvv/extq6kcpbN52N9yB88ezJ2soqhPBXn37qnBmNLzqdjEf44uKikY0QDedxXnQHg2GRD5aVkFLO50vvvbImTtN+vz+el1mnuJwu6rZiUXw5HjFGopQlecwY88Bg6J0z3W4vOJdFPKnrmnM+HK7euoW/+eZJaMCNvqKwhbDTsBW7IqqQK1Y855bzKI5jpRQE+IYTFEUsj3NCqbYmRN0EjN05lySRRzDkHUCEwlTtve/3h2VZVs28LGshmqZpypJg6AEA29vbnU5n2B8QghirIp4Wec+DqGlq4CwlxDmnpZqMRrdu3QLaNGUpjUYYai0c8FZLZUy3R4s45YSWy3lbVqKtEx71Ot1t5/BkgmiEMOeUAmfapm6rCl+LtjHGEGJjHAAukA8AQN5Dra21XgillLHOEWpvdsxhSArSJiklI1fecL/FWQtqeCFEcOYIUUPee4RhmOwh9ABc2RwBiCCABPqYRSzmMWdaS2OM9RoRFMfUe3txcf4yZndv3+p009F8XCnBktgCz5Oo6OYr2zudlZWTi0uMsXeQdDOjdVNWAPk8z7USjBPgEfLIGCuUCangEMIoiijDxlngXV2XQoimqmiccIqbGmmtZ4v5vFy2bd2K5v33352OL6typqRaW+lkWXa0/+yzXzW9FdDppHnaH13Onj19tbl1j7IMYnZwfDEcDpO09+Tp3snp+M6t299598PBcOt//Y9/tqyrxWJsPVCtUEoRzvJup2pqpYW1FmLknGuN5D6mEc/yCEIoZcMjzCnB2ELo+t1sa3P1+9//frUsTw5PPv7ol61oOaHvv//ug+++9fTp09Fo5K51U1mW7e7uXqHlGN8IW5xzxjhtDYSQEBRQKwhhHMeU8rOzs/k8rus6WP5V9dIYw3l8MR6LpnYEt/UyjujKcOXOrS2K3bNne0a30GkHMAIYQ4QR8BCNJ+M0i1dXV5MsrZqmqsrRaNTRmnP++vXr6XQqpUzTGKyshJsKEwgsgBgEiY21NiABEMBl1VZ1G3HQ7+cB9BoOV2E7GXaH2oIXz1/96vOvv/z669lsIYTmUQThFSsikBoDezemmEATEb/a79y+tb29uTqZTBbLGgaSi0cOAujRdSD1b7gtvYlFyWsG4s0wF74taGcAQoQQTAm65iswxghEzjnrXVh6QecAAMGId9EsRNM2VTWbTGeTaVNWcRRhABG4iuMMewNrrRGaEFIUxerGel3Xn3zyCYQwL3LGWN00i8Ui2JOHqxaIyb81HzvnTHhHN8kPN0wr5yDC4TGfTifj8ShmvKqXbd0QghF008nl6Pwiilie53OMRV19+O7KZLqo6wq1vr+S3rm16wAwxkScA0wciPM8NQPnrEeIIUgOTg/ruq7rMooZZcgZ66DN8/T+nbsAOmttU7VJkkQ8CYGZTV1nmZTaaOUBRM4h561zkGKorAXAIoygN8DCAHlCDJ49e6adl1KmWffDH/xgOitfH51Erep3e1GSEUYBIspoBzyllKYwTjjCVFtjjHHGWA+od8YBjClwQcdKEAwgsbfaeu8xgA55G0AF7yFCy+USAch5jDBwxqpW1HVjrc2yFCLQ1s1yOsuStNPpDPq9teGAc65ku76+LtoGQt+2LWfwnXfeefzWu8enp69e74Hl/IoGzlg3LxAiEWVt3ZRmSQhx2ggl0XUkN7gWCoWnHkJI6kYiTAFyRsty2a4OUoZjJarL49OE0Nvr0TuP3/vOu99N06yV9i9//nNRj+uJ/aM/2lnJMyFhkQ+aRvAoGvQGlHKhFVaqahpCIkYj4/1iNsUI9bKsn6UZZzxmK73uzvq6EtIq6Z1rhdK6tR5ZaCFuzi4uKyHjOH65tw8Q1FpPF/PV4dqgO3jx4tnx0TcEImf1qxcvd7e2vfeTpYoiniRRmt0GBsVpurW7NRgOAUBaG4AIizisFaU8iqLZfPGLjz423lW1mC2WURLhiCxmpYKuHoksyzqDwrkimAg5Z7Sz1GHgkTWmsWK5rMqyDiFsAS5T0jjvIIQ3Hy4h3Htv9NUhhzFEiHDOGbvRdDlwzeyjlPb73WAYAoBDBCICjVFCi0YISmmaA8qiiFIAkFKqkSrmed2q6bysamkt0NBq4wmG43H5yadfEcw/eO87/X5PKieVqxsVFXi8XMQ84gRraz3209HZO4/u51k0m2rktJXNTFSYUUqpFnUe9Qf9fi/P6tnCK1PE6fb6GoRwvJg5JbUyhGloXSW1xGXOeQh4NcYE46ZQHTiPjXEQYoKZc85oa40PoYU3pgv+WsR1A7uFCnOzWV8ul4HNXl7H/GTZHQhhURRJEs1mC+89oRj4wAANChEAAfXeI8QIYVkScU5CGhunAFGMCMPI14vp0elrHgFhSkNSkkRru5voU4Ii4pApOimN+PHFEYAQYQygB9owjijlcUQ5chBC57yQWhtpnUMYY0R5HOV5Timt63pZLaQQUkqr9Wg26xa5NcYIuShLqUWWpVtba/fubr/76M76IP8//uQ/HR4eLmZL7P27D4cP3n/v1q1bNIp/+clnf/YXP52WejxZnF9MLicTa3wUhVRl/803r7a2tu7fu/d//R//7VfffPP3P//Z2egyIpglaWvUxcXZ6vqG9V4oZYGz3lsl9XxSijrGPQjhZCLW11ezNIYAjS7PiyxbX+l+8O4jLdXBsHe8/2q5XMZx+t77b+MsCVcEQuj8Fc0+TDzeXzH4/bWySxunjMQYR4xLKUO2b5Qw59xsURrnGGMeIu/9fL5slHYALecTrVsKKeM4iUi/l66t9tIYaVlKZaQ2ynjtgQXQeuuBzzbXV1ZXh6srPWum85m2XhmljJrOZ7PZ7PLyctAbJlEstHDaOGeE0EKopmmUVsYo7702gGGEEMAEYAiMBYyxvNOljCultDB5l41HF3/+F3/zF3/zt89eHixKGfgfwXGcUooggdBQyvM07WQUmlYLlCfxnZ3NO7d3Xh8eTefLYKPioQPgpksAYWdwc9C+iSiElcoNqRnd+DJBCLynlCZZGsfxFXP+KkrG3dh8kSCi9l6IJmwYl/N5tSxF04auPfyYq2xj7533wIHQiARgbyXPIYTBKi1QWEJfGA59eM16e2P1AG4eYQ9hIDP+ep9yFTLpnUXG2bAWkbLVRtTVrC6r/9v//D95Y2eTqXh0p63qZ8+ezcsq43jv5eT0fCwlbNpl09q8uzKazTUwnV4BCUDkiqKBr0h+SCl1dna2WMyKPM3yuCwXCNgiTSilSczH4zHGODgNRJwv5nPZtLJtw5YEQYixJxAjBIyx3lkILcUUeAyAIwhnWba+Woxn8yROCeUQk5WVlaqWk8lECIUxjDk1AALkOSPaGC2Fb6FOEuK9sS5oDR2CEHIMAIQeBAWMt/5K7+ut0uDauM9bZ40CwEOIIsa8984Z1ToptRBCiNYYa6VM01Q07XI5b9Iopmj13q3Hj+7VdTmbOSOX8+nIWosJjCNEiUOEQIzmy8WLVy+ni3nR63c5RwgNe32bFZPRSCybJIobUwkh0ji54aZc8X9v5F1CgZWVPoZ2LqRRbm11h2H79OCg18mSOH7/7bcfPri30omt0dPpReSF1tH0fKkr5zzQwljuykWFKNrf35/Pp1prSJAxxnqVdboAQGCGGKFeXhRxwSHkFHhLLs7Hbz+4H/NIKXl0cjyZTJumCcLTd7/7vZWVlfX1dWfBcrm8vLwsyzImUaDDlPPFYNCbTefaytHsfLlcZr01CwFLMsxgWS7LulpZW02S7GI0ttYrjapatK387gcrUumvvvrm4y8+Qwi3UlMWFf3eetu0ViPCPASEI4iB9Ve7Igihs8A5VNcyGIBTyuM4VcpMp+eEkJvsL3hNCU6SxBjvnEPoSix3cxBmWQYhMEY75xFCSguEAWPMehP+c8B755U1TjuttbGeUqqMNt6laRqksUqpciYuLi7G47m1hmAIAYaIRnGUZMXLvX0IMI+TO7d2WyGUdmXVZlw4o/NBj1LqKI6jHgQ2S9mjB7cJtsY5ytminGNGeRxdjnWWxkUcc8o0IZ0iZ4QiDMumLrKIYC/b1lhtWllqBxxkCOzef4tx0rY1ZRhTqrUNfU+SJBAgzq+oCdY6CBHESBl1xSe7dpoL+ArnHF9PUeEPpZTT6VQb6bxBGCyXJaW0KLKg1A3Gw2ma3hCCgIcIIWRhaJwxxoxQQjCGKJjlaSWFbjtpFKe8VfXByatqsdzZ3NDEdzo9HGFITNXOW7WAmFLqIw4xRnXdIqg73YRiVtd1RqGxRmvVNo22jrI4iRPKo6IoOIucc1I1EHhoLYWQRJGECBHivSdEL6vy8PAQO8UjAoHJs3RjY7C5Puh3osdvPex0OsPh0MXD9c2t1c3NpLOyd3T58vXh+Xi2c+vuUvm91/tZBgnCzvj53snnXz67c+f1u28/2Nnd+df/6l9+/NmnT54/r+eTOM8oY0oJTCCLODI45OABAIQQzuuIRwiSt9+6d/vWzmI++fSjJssS7I1YztM03Vjt/eEf/mg0nkopGYdPnj8vyzJYVijjAoPE2GttzvXBcH2fO+C88xbHWEoZRZExBmFSNyUmqNsf5EWHEFLXtbae8tRj0Rv2ik6SZzFDW9jbQa/oFMnmWpcTP18sJvNF3bRSOeW8BRBBrGhat5U8EYRHmGAaUxZRQsja5jpPojzPh8OhaNrJZOK8y7sd1dSNFMYbjCFJk+CgTDBsqmZtrY8hmE+n2joIsPdQKu2UOTw6+/hXn//nv/irTz/fMwAgDCjlxgEAgLUeY4cQJAQwTCilDEFrtKhKGnsELWeEopAIbTwgDkAAfsM6gbhfdwlvIgrgJqzhGqS5+nMEgQeI4DRNu/0eY8x6DwAIMqW2bd01MS6gC8tqIVsRqABaKug8J5TwSAsJIQrkCHiVu4oQvMpaPDo6qkUbCK1Zll2OR9bavCj6/X5Y+TVSBIJCWJEEoWYgCVlr0XWjcMO/xuDqvVjgq6bFEKZpnBcp41CbhmCNvOh2i93N+8P+4Pjg8Pj189aKhIJHb91fGQwBiMYzcXA8OTx41WpTdDvHRweIAMyCeA9ZA6z1zoJ5tZxPZ3VTiraWbeKccV63bauV8hGrqsoZSxBujdBaT6dTDKBSGgFHMArmKCA4QSDgjMeEpAkDDgFvkijO0tRBxKO4rBtEGeXR0cnp/tHRyckJ5SlEWGpTi9YDQCk13rVtmyCepjGIYuA9JshjjBFmBNM0CtFThBAIgbfOWOOMtVo5ayFw4Eo9Z6EHkLgoZj4cJRAzQikmFJO2bV0jnFAUgvXhoD/org37g2426GVnp3unJweLxWIymXBGOp2O1vqJlbPaa60vJuOD46OqadOQJLlYrq1tQEwXABGIsiwDzlXL0jmHr8bdK+v9m76BpFmH8xR6zWiymDd1KXmRFPmAU2KkvHvrbhGnppVxHDME/vD3fjib06oU3pGIxQiCiHFCEPLOO/3Ww/sOu063+/TFy6PD08V8KlrdS3oUYGiiyUXtpNga9te7KzFFZyfz1eGKdUA0EPokS4tOt0vusbaKGWO9ohdF0YtnL40yW2sbr57vleXSO6OliaNoMhklaWS9sV5vbA6ttYQBFpEu6wAECWetVKPRrNsbeICn03lTK2XMYll98+1TAwDCEMc8zvMkK2rTtk5HcYIpoZR2ehlhWCuDEUGAWuutAdaYuhLWAM5ZyA9V0ohWwWv3vXCEh5sAB/ErDqAckFIaq4xxjJGQYuecAwA7Z4O/9+XoAmNsnJNKWWuBANaBgMoqa8q6VcbVVRsYRlrr0eWiqiotBcbYOSC9ctBTRx0Ck+ksKy6PTk7CIlk5Dwj2wAxXepub6x5Y5MHWxuZwpZPl0YOHt1mMAEaUk+lsBglGBCYFHfa6EMKqKrXWScSSKGZzAqCt66ppKqEMIhSzNIoZhiTNOKWw00m1br3XmBCgDIDGGNW2tfOOEOwcDHsTa62zxgEXcJZQKwkhnjEAQNu28JoMmCRJSMe21rZtwzkLPqwQ+SSNEELz+RQAF5JdvPfGhNxwAyHw2jmrgbfQI2stMt4DRxlWShktlVImxgADWem5000NXp8eP3v1cnt3x0KNGfJAGVEWvf7qIGlqhaHrZLQFwbVSWCMwJR46jGwcwYxmSdHhUeoBslYpC7x1GMM0jiICnbHOOcMi1QoAUZrntq1Ojg4Tjh7evjWfTxlyp8f7VTm7tbP9g9/5sJMXdV3+L/+fv+gPVj748Ad1W+WdvDHWYsiK9F7/7ZWdbQjx3ou96XjCKGu1efF6/6tvnn74/UcPH73V6XXX1ldbrTDnyllC0O7arvf+8vJSKZUlKcYYAaj1YtAvBt2V73/v/bcf3J9eXpSXF1pJbM3Bqxf379+PI/aP/vAPXr3eOz27ODh4/dFHn2CMV1dXOefLqgnrM2tNoI7aNyDocNuHjMfQqAUAKXSKRTFYW10HCC6X1WS+VEpHSQYXZZJTa1iU8H6RNotZ1Vaz5WR97f729noUY4gsI6CWSirjAKSU1STz3vMkHqwM+yvDvCjyTjfJMgBA28q2bbM42d8/2Nvb00IhzAgjEIPgs0kRhRA6Y5VyDgAltfO6VWA8mn3z5On9hw82t7akVC9evvzbn3/09PmeAQAigDC1HmKEPUAAhhB5iDzQWi/m5fn5aGvndpplVdm+fn308uXebLaAEFgDPLQOwDDC3/QEyAc2pX+TqQB+M2baX9m1Ae+9cdY5BzHS1jjn9PXgobU27gqQCwjcdYKottZ6Y8POIgAUBCILIAhUzOuf6JyDGFJK27adzWYewW63WxRFsFVIkuRG3fMmChigU621FjIsBL33JKyfnLtBByEmQThjISrLkjHagzkhEALjnCgKfnl+UMS3xxeXJ6+enB6fAFk9urfz6O72W/d2F6trQoAtTarmi1d7+0nWgd4tZzNIPWNhvYiNAVpYrW1tJAKgSLOqWrZtm6WxtQ5B2O/2MIZWG61NRBlFmGHCMPHWRdeKHg8hQIFVasumlLJlFrXQAKetUZMJK7Ik3uhlafHk2SvK2K1bt2azWdM0hJA0TSHCqG2lbC3wEaeYkiRiLRDAe+QdRJAQAjACECEAk4gHh+2Qzu2MtVobY51RQQcHvAXOA+c9dMCitmkBAIzwJI2zLCOQ1HVblqVvzWw2cd48eHDv4aP7CDtt2ld7zy4uTqpqaa2mFENEICKY4Pmibo4OEEIewc3trcVi4YytliWltK0bZ6xoGm9dGsfAWCWkfwPuwtdeumGlS+I4XS5q4DTw9OWLw3pZfufRw52tTeCUFNX6xmqWRAiBTr9XCbW9vX12Yj795ItnL17du/cwy3sAeUaAFPWjh3fXNoq6LZXRStayrhBmkltiiZaqsWWzXFTzaU7oxvfX79269fzZE0rjdrlsG9MIZ4Ewvul2GYvQfD6ez8ecxMfHh21drdy7vzfft9INh8P79+8zhj77/HNMmPc2KQqIrZKNXNQA7ty+d7dtJQTEOA8RSZOC8qgV5mI0er13sL6+2ev1nh/tAa8RJdKqcno5Xc5YwnqrvTzPIcBRFGEAnbMIwgDuUcQQQkKotpWz2UIpkSSJtUEIh4L3HABAyMYaTyyKeBLoPNdEBA8AcMw5b4x2WsuwcUDXWelKtpxzjKhzVlvtnPcOamexd9hBqZVQallVCCFngVJKKA2QpzEHziqrvHXEYqHk2eVFI1pE8GgyfvbyaczZdDF1EPRXOoG44L2HwO/e2o0ZxdSzlPZXCwAhpohmGIQUSmo3e6uA4qqpsbMQc22V9SZOI8xgmsdAKm2tgZoA5Jxe1HM8OfbQSr30QCHmPTA8IhB5ISsIoYfYOuu8NlZK1Wrj4jRBCGH0a7lXUEBorcEbVrsIIWtt8GiKoggA0OkUodOaTEdSSkajazgHQegRAhB65yx0BkGPMMDQWS2MtVpLhJAQknHmgXEQOQesBwnnKyk9uzj9+tuvWtkAZzmFxuhyPlrp58zLs8vjiCerq5sA26aRUZRGRQraikcsT7nHjMYJSzJtXFXLi/E4YizmCWckIqlTTLZCKZVA6NoWU9bLk1LLpqqYizt5LhupMn1yfHZ6fiGEyoqP79+/37btspwsqylkYLQos37cifLO9nqURsuqTfsZ5/G6asqmnI7GNMbrqyuMguf7e3unR5s72ysbq5Aybc2yrqzzg26HUrqYzs4OjyfnlwmPrLU+s5xv37t3b3N9Y9Dr+1Zsr29MLy9jyi5OTnc2NvNOd3U4nM/nVdV8/PHHJycnwQnRGDNfVoFEAuCVbYu/bhGuJg8CgTdaAw9cFEWhY3DA553u9u6tvNM9PD49Pj4JjIe1tbVKSIcjB3ClVKTYvKm8qvmYbG+vU04pJ4xjxnAttGgrbQ0lfI58bzBY21gdrq1GcUoYhQQiAjGmiGBMEadRmibdfrdcLp1zSRpZW2it5vOFUvoq0deDOGLLusIQZDnzzvzq08+KotjY2OAQPn2x/3zvQGgQJwxAUjXKeZtRjBElBFmjAyemqsTx0dE3XXj3/nuA+IODy1/84vPPPvvmclpjggPaAgAEwEHwa3WMA/imtXqzUQDXqhljzLU0wwMAEKMOAOOc1Lpu2+BaGsaSG2GU1cZq44yFAFipEEIh9RlYF3YEztiQ4wADXxICcM0BQgjMZjOEUKfTKYqCUnpycpLnea/XQxhXVYUxzrIsSJSbprnmHrhgHHKl6IOQc+7RlabOORdiJuI4bpStqsoYPZmOprNhkRKra075/btbK7306PXIqFq383J61o2pV82L50/mi0obFseDtpwb2aAsk22FvIEWOOOdD1JLDDFBHs1FEOlEbVN56zDG1gKKCSGkrkvoPbDOQCOadjIaSynTOKahaGMEoYcEh5bp7r1t5D2jNKLEWW21ydO4KAqlrfXg9OwizwtM2PnxxXQ6XSxKgJj3vhUyOMthjGIYYwgpgt5bY5V3mAGHCUMYM06vV6sQAAQ9cBZ4a4Gz3jrntHPBXMcj4Lyz1gLvjNa6LqtqsRwMhmurG9vbm4Sw6fmlA7puyjSP+4NiuVw8e/ks6NXjNOl2+nFOm1oKjXiUiEY4rYJl7cO33hqNRkcHx+PxeGd7ez6eKKXqsgLXyQNJkoR777d6hauJTktTLZYYgTRm5xfjy7Pzumya6q0H92+nRdKIstW1cw6wJMp7kCVFnyzbunzxfH1rp9MftKLUTjX1PINRt4jbZmx1e2tzvZtk/cEaIezs5Oz1y1de62F3MGbKuWo2OxslaHtnjREqZR3FzHgzX0xfHx4QQhAjbSNWh2vYEehpHMeXl5eL+XxjdUNZdXvn7svXzyGkjdBpFhVRoXTlkCuX9fnoIi3y6WRZ1YKSGAB0eHzSNnp//+D49Hy+bL77wfd3bt352y9+eX56Ya2Ns1RqpYzOikKoNvExhNY4bLwPezsAgNJm99Y9762SZjKZGN1obb2/6iFCg04I4RFtWhyc7CCQ7jqqNdwbCIGg5bPWAuiSJMmyzFotpVRKEIIgQg5YB64fe2C9923b8iihFDvnrAPeG2OMlBpgQChxziklnbdxxKKUUY5nk7GDhka4bOZ7+6+2NzeHq0NCyM7d28PhEHhPCHLa8Jg4YGvVWKBYQhvZSO0gARAj43RcMJ4mxhkNXJREiOO6bgy0vZX+7/3R798ejRZ1c345KuelkWYxWy6beSKjTqfDONS6kgoZ4zCJjJGEMASJR8YDgJDjHANAUfD1YgzDq8h2hJDHV3l0CIRFpryxikMIUYoDzX57ezuOY2vNfD7nnDtvgAu+s8ZfM7ystbatMSWMEUShcVKJVlnlgXUQMMaMUkY7CDCmhNGEc+7bxcXFBaEYE0Sg10ZdnhxsrQxTirZW+3nWY4w5IYbraysrq1LK49fP8jyOksxCbBF2CLRKt6p2XltAIPaUYuSQNgYjGBGy1R2KJEXQMQTFbKxqWWq9nC+dA4t5/eLVIY+K4fpWLV0l7GRaOl9XdXt0SkbLKup0KzXvrK7Vom1dQ5F3xu7e22YMv3z2vEizjfV1wrB+6uq2bYxanp8yxta3Nu9v3P/ss8+Ojw4xQPVs5qV0zkFE8zhOU7+2uvHed94v0twK47S7s307AhgAcHhyuN/fv3X77vnpGYSYs7iqmuFwOJlMFouFMaYVKo5jY0yS5vZ6sXpTVgAAEECEcDicQpQwAIDHyebmZrc/LOv25d6rxaJsmlZKmRRdbUFUZHHEVVN6SjwmPM8bpU8vRwyDqqqMc4giTLED1hjlIdy5u7u5ubm1vcPiSGplrL2m+LRJkmGMHbCY0ZWVlTRNm7qOORusrHR7xf7rw+l0xgjlhFtrvXUIAQgB51xrsFioL776+tbPPxJV/eLFi/OLsQOQES61tR5AAKVUECIIsLUCE0QJaRt9dnZ2Muw937uwe+dfff3ks6++vrisHQAIEQSAD+MqcOH/ICT5/peCqW8Ahpul5w3AgCkBCGJKIEYOeHft33ezpCNXiy0SEE2jBL62FffAueukhrCSv/lBHl2Z7DjnhJDdbsd7H8fxo0ePAABCSWNM4FCHhyuO48BECeZplFIUX/HhrbUkpKMBCK5lzAyTOI6jKJK+ieMYeLNYLKbTy1vbA+tUVU5Gl8eDgm9vrW6srD7rdr765Ffnx/sJQ2/nu9PJtChWj4/2T48P8jRhFLOIbW+uCy2Fklpro7T3xjsEPOCUNVUNIk4IQch767RShJCmrM7PT5umccZjhCCEZVkGZXWWZVGSIAwgBoQQj7y1BnobRVEaRxFj2EMEXKA8pSkDAGzv7mDE2rZFlIQVYZIV3vskM51uARC8KmsYsl7EedRIIbWilDJKQEAyvPPWew+9twAgb66WdaFoeaNDVohFyBrjrPXeQ+dF3Vwul5eXl9WyvHv3/nA4jLP47oO7rWgssM9fvZjOJ4dHR9PFxGjXcQDgxDtcN4YmvF+sO1jiDHuCQufEFgvvvZKyrWrRyLZtgXXeg6a6UkRrra9oFG8kjITrTmTTQog5Z1HE27oxWh8enM4n09Fk/N67bwvRREkkVavPLzHGl9OXDPdf7r9uavXO++9vbG9BAgfDHq/dYjlO0441Elq9Nuh2szxLO95h5JuTI0kidP/OPV2tNWUl1PTgsFzMFmmaGucZjx/u3GmlOr8ceQ/Hy7lzhnF8tH+6tbrT73W00DubOwCAV69efec73zk9ueh1B5D6nVvbeZ59/u3fZVmGMFgu52dnZ6dnI+/QzvY9AO3hweFkMjMOQoiPj4831rfee++9ldXV10eH43GZiwZTwqIIYrRYzCywDLMkURgS3WpCCAIobAG11otF2TQCABAGWaUMhF5rDSEID2p45JRS5bIOy7xrcb8GAFBK85wHBni32+12u977xWLhnMMEemC1slJKa71H0BqntYWIRDHkPPIAWOudc0oZ58CimhFCjDGq1YSCokiyImWULJYIYUwpdgBwzu49uNvv9yPOa2t7vc58Ok2SRLZNo1tvdNWUmNMIR42uhWoo4QgD6xyPeVmXcRzlaRYnCUKwlY12Nivy9YSlg65DeDpfKGkYotWiwghBb9bXNp4/74/HY49wUwvGWCuVFBoiCyDGGBFCecRiEymtAYwYYyHnLSwaQuQ5hDBiLCwpguVzGGQ554FovbGxFvywjdErKyvhM/fuyooKXaWq+Fk5ZoTwCEMMTdsqLZRVCHkAgNGuaRpKUBpzglkj9HJZrgzzpq1nk2lRZBh6aM1sqoys79++NRis5Fn35ct9J+3bjx5vb+++erU3Pt1LIs4i1igtlZK2rVslZBNy6iIWIQCNUlpr5AEkpF4uIAScUwodI7RIeJFERd6VQm+srb/16PHq6mqaps9fPC2rtm7kxnp/2TTbd7ZXrSN58dNf/IqIdNkIwqJeP5tOlq6x/dXuj1d+WCTp6fHJaLFotZJWm+XMejgcDq13l5fn9+/fn1yOZtMJcH5lMNBSIQ+o94SwJMmGw1Uh1Hl5sRhNYp6kUVpV1eXZJUJP4yhtpFrd3OKcI4TzPD8+PoYQBgupMFZiwiilNwXlhvRkHUAEho0D53w+nwd78s3NzUVZX45Hr18fZFmWZLmHKESQjGoRZ6mFqGxFIyWG5GJ6WS2nq/2uNQpjTBjN8zyErnLOO90ui3jdVLPlQmiFKOFKIEhoxHkSY4ybplksZlVbOeO0McDrPOtQzq21TjmhJM4pgtA6l+VptSzH4xIAgBGAEJ6dnf/tX/51K0VZGusBwkoqBwBkEZdCAeAQgh54CDFjtG10XWtlyauDk1evDz77/MuyrhGmnEetlBDQACSgK8wfAuARAPofeCvdFOWbP7lZDAd+4m+0FBBAjLDDgWwYlhRt21bLZbBbdkY7ZN8kIQbTHymlf8PE6eapwRhhjLz3JycnvV7v/ffft9aeX17s7e3lRVEURYi5CSqtIGO5khoxHp5NYwwKPwXhK4qrtQSisBkEVZN3CgTd6HxhjOGcAqenM7n36tl33rrLKRlfnl5enKyvDh8/euet+w/SNUzJydrG7f/fn//s8PBwsLpVVcuVJLnz4MGiWk4mk9li3jZCa+8sdA4Ee8cwEzN6Fa2U8IgQUpZlWzfGuDiO19bW0jS9c+cO9D7LsjjmFnjjdIAhtZGc027R6XXyiHHOWMy4M1ZKSZCFEL/9+N2qbMpa9noDzpLpfAHQVQAexMh7b70JI/ioHhtrz0ej0WTsAHAQaeet1Ywl9jo8HFgbEs6cscQHNb0DEEIIEPQWAOfMYjqJoiSKWdOQalnu6/22ld1u9zuP3370+CGA7tmzb17tvZwtxhCjze1trS3C3FhvnGdRsbK+c/vuW7PZwsV1FEVGqaODw0AdM1qPx2MMiagbzjn0MBisdTodhJCx7mbtFbrVAFwRDqFUWhgAPEGsS2nPQH++lNNPDi9m4L//V/8CA21kM+imX3/56dnJCVndokmUMkRjY8Bs0UyLIvOwKrpsPLnsZEWLW4xxnsaL2cQ5gFr79q3b3SK+e3cduvZ4/xVB8O7tOx/94tNvv/ns/oPHP/idHy/K8uzi/PbdNQD8fXc7SZLZbHb/9tann36Oad1fH9b1dP/V3mCn++Lgk2+f/aLb6d/euhMRqqpGLYpGRv/in/53f/If/9Py/LBcLqOI/bOf/GMeR6Cd39rOhVb/27/7ikXg7z763//wv3l4+87OJ1/8amU1bdu6U6RRTCGS1lpda08IBbnysC4bBDHnsdN6sbxsmkbpUrnGI08wa4RQzkYRg8ApaBZiQSjCOcYUCdG0xoY2NiTuIIQIvHL8dcaur671+0NrLTDWa5OwWGoTkDpvYdmWCKGIMWgdY9iKkiQ00FKstbyfINQ9ubSj0aTf76yurIU4HAoIx9Hm6s50MlrM28eP35vNFsPVjft370ynUwnHilSGtwtZ5XkxnYyBdcu67HQ6+/v7yLNmOdczsb6+blpjrYXNqHGmTZjuFZBxxelUyPHJqXEeYkIYjRjPig4iuNPrAwCqSvsoe/zhD7WWSinnTXBWllJqZY0JsT5QCDWbLpbLpZES2IDGeIIoRgRijBD1jlR1a53mcRanmQnsROBjSmNaAAC+//4Hn3/15enJ0d3bd6bzGcY4kLMAANZ5b41zzgEvoKMUDPpdiMC8XtRtyyPKOaUYCtFQziEEy1YoLQghSZpJbLz3E1E5RmDScYps3OKVoFG6oiRcu7O+3h9+Bu3xi1/tf/v3AADg9MZqr9MbHp+OZ5endWuMR1hjhJFsDYx0kiQWmoUUSqmIUSVLHqeVpYynqLdae7eze+uiEvzsHEB37/6dnZ2t//e/+/dP9/bj/cPL8eztx8PBcP0P/8k/x5Qdn1/+8hdf2GnVjbPJtFw0Po1z5+F8UTdNAwDClMI02bh3t6qqpmqV0k2rzk7HnLJip/P48TvVcrmYj1++er660pNN27QVpAniWHmd8FjaUrjl0+fPgLMnR8dlO3UXprPfuce+w2bl3qujtrGz6cXacOABAAhbiKz3LM8bb7FFwCPoPIaYEcYohR5ZYAlzdV0PV1Zmi7l0pugMeZFJ5CfV7Nnec5oQwomQNYbu/PRICXl6OT15AR7cvdPp9mhvHQEIUVbNZ8256Hc63ShzzltTxRRzpL31xqLJeGl6OO8UhCeIEMqZNDpNc+cAj/jo4ODg6HA8Hh8eHmopoVNFUXDOkwwjkjrntGyFtFtbK9WyvPdwZz6fnxzN1taLeVXSOJppWbcGUoAgqqREECVJYo2jGBrrtVYccyN1pV0UcWvtnfff/6u//dsXL15QymDEhJCy1YRQaxUAAHjkrnoCDCG2EBD86zXwb/3iRlyArt2IrbXAAEpphAi23jTCOSfaNngsdjqdO3fulGV5eHiIsjyc6M7qANT92sPf+5uTHr5hEh/6DAyJ0Y7TSChJEWWYffDuBy/2Xs0m87ZtEcDWOgQ80L5qa0ophQhjQqM4aHyCjVJYQ6ytrQUt2Hg8XiwWWutGyy6i8/lcOTvI18+O5ou3EWYb0/npx18edgbf/J//1f/p9OC5R/Kd72z/3//n/346Hh2VtcH48+cvPvr2SUXo2fFpzAuJ4qy7nE0vMHR3NzYbKV7uv543c5akutJxyozTWZZhjJ2zw/7W6flsd0dsbd52q8ZYhaGN4+jO7fU0jZP8SpkFgEOI4jBAA6ulckrPpzNGaZZlsICUYgwJALnSejou27YlhJRSNEZbZDBBOJBPGQ+zYiB4Zin1kL7am56e7G9sbvd6w7oRHjOKqQZQKwv8VfWDngJgkBsz5FpjEKI0iqTRjdKcc0d51QjofCfL13sDDCBQsjo/i+8+KBycTKbtxbjPIkvoxfiCoq7RptXlcGVDNHI0Gd19cL+32q1kXYlIto4QFscDB85aYSACnFGnDY2oVlJrG0WMEW61gQAqKaIoEkI4azqdjlayXC6iKCIGehYzHiWMRVoZ44HTRkhpLT47vRhNFjubQ+hB0yptMGXpoJs9vLtLGSbYxxFVLVpML8/PTzY31nRbkoRnCSYEI+RNiqIowsNoMcfASYhMnPBb93a98RCD7334XetInvco4RCK5bIiUq2trXSzgdY6TiLuGMYQE3jr1k4cp6oVm5ubWZafnR6/fn3w5GmDEZ3NZh6l29ubbVUWRSaFmM/nGxsbJycXD966f//h46LbOb0431z/hMTcey8kWF/fKNJ8sZivra0B4KxycUKt0gQi6LzVxlqvtabIG6CklGW1CA8epcRahwkkhGOKhBAeekwgAIgxlsUZIQMI/eh8WlWVaBVBmBCGruZ82un1tNZGmuOTs7Zt67oRTWutHQ6HwYIw5EIF4C7ceUHB3O/3wy0YnP4Q9oNOd2tr6+GDR20rnj17Np/PKUHlsoVBkIVBnEZCicly6rFvl7VCqGka6DxDhBOKKFRaxHzIKYUQ5mmmrfHWtnUjpYTYCauV0cJq6EjwfRNKVo1ABCNCCSGEhRirCGOMCXHeaG2MMRB5AhHn1Puo3+9qba3xEGLvYF03oVuqfY0xBh4i5AFABBMAIIRUCHHtRGa9B1ora62xSqmEELa9vX1xcRFA18lkUrcNAADCqzQYjGlgzCmjrbVtK8uypIw45xghFGECwHRZIuwYxgAYwlDGkyiKeMS6WVzXtalt2svLpulEBXSQgujo1alW6vJk8ujh/cFw59nTg729Y84Iipi2fjKbPn/+fDQri84K51xbNZ/Ps6Krtb64PAMAxHFMGSnLklMOETXWC6HqqtFKnp6eE6+x0cNukSVxK9T6xhaO8q+fv5DaPH3yqtudfOe9/cfvfCemacziZdUQGneLwgHkrLYOIGgpxdZab6Vz/mr/xTlCGHmAr/SH86aq33//vZ3f++Ff/0387VdfOeeyLDNKy6YNeTlesqpqDo6O6sUSerCxsTFf1MCjKIr29/c/+uijuq6zLDHeaWu0cc4ZDxGCGCEIvI0iRggzSmspnJEUszDGQQin0+l0Ptva3Xn/g3frVh4cHEyn0yzL4iiVUgohgHOcc1YwcrGYzWav7H671vS63bRIrTF1I9qm0dIY69IkBphZqA3AHnhd10GPRzlj0XXGN0ZKqXA0VlUVXKjDVdDKIYKjJO50OpRSD+xiOhuPx8I0K1vDDz/8wDnz1VdfTccTr0zVTjnn1oaYXaS1CDChcy6JUqigc8YjiCHWRkPo8zwP/u7WAoSC7S4IkW/gOov5N+CA/3rGNHrDWh9fZ8SAN1SUAWYLMEAURTcLhTD5hVYAIWTNlQLiRgrhf1NecYMD+WufBs5Y4KCELCjO+a1bt7744ovZbFY3DYRwOBxGUaSMNsa0jcQYR5zGUZokSZY6XWjvfWBBTqfT4+Pji4sLKWUAkyKPWiWl0RG0nDMWR0mWoyk+v5g9+fbZs0cPt1a7f/iTP/rsFz/79usnG6srVdksl+VXXz47P19KC9j1Mby39wp4nSVca10Uxe3bt93h4fH5WZwNIQAIIa1VXSvvTFFkSZRiAobdbp6nxurlbNq0VSsWACoRrLXhFaACnQ/BjFYbxgilVAohZStEkyQJxphg7z1ECAVEjVISlmvKSBDcuKVA6Nc+7kIpa3y1rBfzuui0RR8ghJRREEJrrDMOeAh94G1La7UQIlzoYHV/nT9MKMKOAuxBzKNOmmVxwghFAJ6cnymrJrPxycUJItBam+edLCuwFOVkXlV1lBTr60kQuBZFsX/8RAgBvW3bq3xdRjElyCOMISIZssa0jWyqdrlcQu8N8N77cPkCPSUA4aRsKwCQAXBZNcbYNM0Zi+I4r5aLV5PDly9er6+scp5YI1lSRNav9nvTy4v5dP73f/3Ts4PD4HTx8OFdVUvdCgKcs1rpxnurlMT9/rzRnCGE7XRxCuY6z7Ju0U14yjRe1PU3T/aFRmtrG1WjRgcno8sJdE8IIRsbW1mWLcvFcjG/ffs2AODe/VtZWihl/vAP/+Dx4/F8Pj8+Pl6W05/85Pfeeeed6XR+/96d8/PLyWSCIPn2m6eT6WJja+f+vXdu3/tOK4hH8LMvPv/226N3vvf93d3dZ8/a7bWtql5qLRMWQePyKAmIGYI+IhgTijFS2k8mI0KIc0BrqVQID8QBDwzshChiSRLxhDNGCSGDlWHbtrJV3gPgkXOBPIVOTk6hJ6JVZVkZ4yhmgHCl2+V8Jpp6MBjcunULITSZTGazWVVVUghGMMUo5iyJOMY4TVNCyPFeM0yHd3Z333v7oZLm8vT01bNnHgIex4Qi67T3dri24pGvRT0YDPRFQ5Okm6VtJabjsTO2SLPlZOZX14GzXhsMgdK6rRuCUJTnupm3Ws2qOlnMmRKTxXyymI/nM2UNxAghEmjumNEoihjjKc20Ck2AhQgEjpWzRkmhlXEOIEgAQFpJ0dR1NXcOIoQgCJIkHIABhECe50EVTSkO6nGlBYRQCLGy0tnc3Hz58mVQ89d1bZxFCEEIvLX42sm/bdu6bbKIa+OqpuWaeA8ppZRA4HxMCUYeeq1kK4XSAIC4QTrxmK5kQwzhWtEfH422NjaMdsTixWRJKX11vt/NBhsbGx4lhHfjLAO0zfNcaecgQAgZZ9u2UcYjihCFHjjjNEIIEQSsU0ownjkHnEWEUB5nGFOAmG7F5Wjs3VsQMSHM6urm5q0HBxdjZdF0tsDUPH/2Oo57l6MJxcyZslrMk7xQykhljQeIEorIlSYNIIxQxDlB1AdUxUHo/HK57OTF4cFBHJF//sf/1Bvz85//nOBOgpHVRjRtW9daqSRJs6y7mC46RaGNT7Js987tw8PDg6NT772UrTICAOAB8hAgCBACgYOqrQGeAg+AtwhbiikhHgFDEd3c3GzadtGUq+trWzs7z1++ePHyZZqmWd4JyewQYAehEIoQEuEIGjC9mFBAOOExT4DHwNOmVUrOG2k73ZwQoto2MIfgcum9zzudLMuiJLbXp2BoMYUQi8ViOp02VR0KHMaeUIYpSfNsOOwnaVSWi/5l5/Bwv5Lz09nh/Tu3/+iPf7x/8Npamw+40KpVBmOAEQUAAAgA8hhiZSTE0BoLoY+iSFmNMdzcXJ9Op23bOgfslT8/eJOK6H8jzSGYbf92lwCvXVXctaVV+BeCvDC4ub/JCAknnHNOSlmW5U3wI/pNA4Y3+ZLgN9ccb7680GCFFuQmfa0oig8//PDFixfHJyfj8Xg8HkMIjbOU0pBNH2I+gtorGMSdnp4aY+q6Du7IwaDJe19JKYBttRRe5ThrlDYALEvTz8nh4clP/+Zn7zy8+8/+0U9Whlv7L48SmiRJkaXd4+NzKQEggBLuvW+aSjiHoGUEMsayLJPWMMYIobIVGOMsz3r9DucUQx/HLC9iCGWSdoaruVKibcbtbO691Sb2oIXXLuyEIHit7AXOApAAAJzVTeOFEFJKznkSY8ZYkkThs4fXMRb4jSQ27zW6TrBEmiMKOEsIxN4hTiMAkFyUiHkEHQQWAI8hMMBC13oj27ZK8gxhoIRyEDKMWBxxzjkAzljoQcRYliWdLI84xxAdHxyPF6O6LoUSEANIIGFYa90K1e/3PcCXo1HE85C9MpsusiRyRimlIHBxHCFgEfDWKOuMMTYol0QrmraiiCZpqp3OsoRRjBBKk4gQksQ8jmMCCKKEU8q90AiAKIoSnjhjBEJGmdlsUdcNymLO4pX1DR5FqtaXJ+NXr14BAF4/Pxpfjooi/+H/4yfOCGy51vXJ4eF0OuIR5hHGHkjQEl5gDC1UwCtAUhJRHPGvvnl+eHrx9PlRUqxsbt/Z3NhZzKv5vGqrRXAR2dzczvOUEQ4hePrs2zRKd3d3P/7lJ874ra2tt99+NJvNLy8v79x+azhcQdB/9913hVCvXr3ee7V/fjE+OjyratsdbK6tb969906S5Sdny8PDk8fvgw/e++7k/DLhccKoVsJaG+eUYKKsh9ZCADmhnDOIqTPGOEcIAB5ZQAOh1zlvnUEIYcziOC6KLE44JDBw8g2UHlkSY2CBtV4LJZQxyq5tblSlUHoOIPYAaOe9AwgzQlwUMUKQcyaKku3tzVu3drTW5+fnKysrjx8/3tzcbNt2NBqdn5+fn5+7csY5P9l7FUGYFrnXghJAGD08Ouv0eRJvbu9sPHx8T8r28PiglVVEWSBKGtGqpk6ilBPalktR1U5pYHQacWi91IpjzCgVGFdKgelYecsYmy+Xl5PLWTVnjEGAQji6thY2VxEPlreEhPirqxi0UMsshFpZ5wAhFEHitUDeEOikRwA4iAJwhxgjCBLnfBxzhEDw8gEAAOgg8gA4KZvhcBjiLoVWN1XPBX2I96HzDQQrrDCLk6Zp2kY6Zp1zGBLovdWCIMyJ73aKIttpm3J8ca6NjBB8dPfu9vZ2nqTj8TSCTpYzY9zr01PO47t37tuUK6tPL85qKVY21q13zuooiXMW7+7uAnw5XVSNsCzJ0zzTWhpnKacQuGU5bZpKGdl6hi32ABKO4ijFSZrlWWu1EBogXNdNLdrjs7N8sNLrDi9G835/M8uy588PLy5mo9GIc84gLpcLDKD1wBptrPcaUx5BRBBwCEUAAAwgRdBT4K0L8mtnoVH6m2++OXj94t/+2//hw+9/Xwqxv79XzReTi/PpZJIyQhHsFL3VtfW6arTWFtiIx0rbjz/9ZDKe553CamOgDmx/QghA0DhgjHHWMYy8aZSC3ntKKY+4Mapsmt7K6t37d7TWLI4Gg/58uZgt5t77tMidB5PJZL5YpnFipRtNpwCADCX9vDubzZqqubyY1FUbxzHAhMd5I2qxWLbWUYoDHzuKMBGCc55lWa/Xw5QsylIpZYEP5k7GmDB5w2uDoDiOgyfH3M8oR1lnY2tna3NnFWB5enry1befQCJ/9KPfvRNtDQaDpqmKbq7URCkAiAYYABAEn05rF3HugXMAYgK9Bw6C7qB/cXER0IvrUx+A69wW768iobz/L2MJbzYK+Nrg8obVGBaONxgDuvYoCxLEcLrPZrNgZhC0V4FKEvqGG3lbAETfhBNufrRzDmMEABBCIII556PRaD6fS6N7vV6WZf1+P+SuAQAwJb1eL4oSAIBSpm2noe0LQ3DYRBRFt9cbhD6maZq6rpXRPIkdQdZaTKiyjscpYdghMp7UH338+dMvv04x7yX8/GIxGX/88Ec/jHhutE8TKBxxzgGgjaGDTj6fjdq2tk7XdX1ycraYL/OsYzU0RjMEN1aGd+7sZnkEoPVe180CYiXVsm3rVpXaC2CdNtDaK9so56j3lGKMECAEIYApw4QiB0lQrGitnLMER3HM8zyHELZtK5QEPqRgYACgww7YkPl3ZYDBMEOEFEWX89hobaSy1jR1mSbMAQOddNbqEBbZtlIIZWUKE4Q8Qj6KeESZ9x4jpAgCzgPrgPfOyLrxUmDnnAZqtphCjNI8q0XtPMCQNq2ezRYra6nxwFvAOAEAQOeGK73prJKydUa3zhmpmqpUsm3KqtspnDYU0zROOklm+4Moijp5IaXs9/vhygZLYoFpFEUEYBT6VugRcNIqvWwX48vzhEX3793d2lhrm3o6Pc3TKE3jTq9wQm2t76rW7+7emk0XHB+en5y2Czxc2YJ2frk8OT0qy7K6dXuzSIqEF1nOjVFSG54QxhJl7JNXr5YL/dHffzUvRZyl1hNMko3Nbl238/ncdbL9/cOTkzMhVBQlu9s7nU73Z3//UVvXf/xP/pkxH//v/+uf7O6u/fEf//HGxsbv/u7vXp5Pvvj8k/FoiiHqd/rvvPMOgvThW4+n8/LFq/2/+PO/ITxiSXbrzj0PWF4Mxxfjdx+/++rp8+ViNuz3PItGF2dZEjdVY7QCGEGMgIcQUAw9hr7VwlrivbfOY4IookY7600UMUyvWvgoimhEvfdaKxRR55w1zmlnjMMYE2KN8uWyPDk9H52NrAUIEghxxOIsy0w9HfYHOzs7KysrnU5nY2Oj0+lIKT/++GMhxN7LV8+fPqvrOuwvZSuGRWKMOTp4dXJ82B8OpNDdLOFxNOh3rDdtvfj6q892b++EpXGnSJtSlmVZL0vRtDGPdzbWVwZDqzQw2krRVFW3P0y6naqpq7YZX5yjjLdSNkIsq4pSqrUWSoZZHyEUOI9BsGAUwhh7LKKIFUXBKHdWK9mE6ixl4KR7jQgEWEkFvKYEQMQIYRBC50BwDIUABmaotdZ7Zy1w3igljNHBU2F1dfX169dBCQIhjHhilJRSeg8gxoQQzj2lNE1TRLDVSgghrFMMOWcYQg57o1QS8yKLP3j/vR//7u9whl69eD6fz4si6+ZJ27acUqPbe3e2ZrNZJ4u86ZxdnDf1qNMf9AbR/uHhxey42xtcXl5S4ISQaaeHCK6aWinFoiiOeSuUMlpK6b1lnDBGooRD5LSwwFrtHFIKQowwtta2UvT6eZ7nCHsP/cXFxel4euvOvRd7r61zy0VzeHhcZLkQzTvfeZszmsVxfzgglAljx/NysliouvWYQIgh9VdkKAcRxBAhhwGCkEf86Oho0Ou2zfI//Yf/+N3vffCv/9t/+Zd/+edffvLLo1evj18f7Kyuam3394/39o8Xi9pa+9577xnjDk9OldRSK79crq4NS1ffKOKctsB7AhBAuNvvRVEEMVFKOeAxxlJrjyyLKOc0yZP7nBHOLi7Otda379313k8ms/myAh4yHmnlAMQR59TyguDJfBakd9P5dDAYOOcsgMZ67SxSOsYE04gwioL83XshxHg8Ns62UsZpEqVJ4AOFyThJEhD5EHTEKATeiUbNJvVyMbNG3r232+9333r7rbWNwXhyiRmcLkbL5Xy41kHUv/340Uv6KiSXAuiccdJoYD1ECFOEDEYQOGA9BBBDyun+/kXbtoHU7Jxz7ioYKeAC/r+SLg3f+LppfMNUeqN3CB95YEaTaytGrXV4UkJqV1jGgTfyHim5IvaG73fX8fTh8HbXjrw3+4jgNxQCnM7Ozn7605+enJw0UgRqVJwkoRF0zrVV5Zwj6CqOK/yVm/4j7EoCRAHh1dyfJAlkiDHmnBNCRFHE42Rjc7csSycay+tOr6vL2bcv9r//7jt5Z/XwYL/9/NvZUrSNppQ3jTXexHEcxxHjBGFYVdXp6SmP0+VyqbV1CKQskcASBBiBSYw6eYSw84DGkWMEtG0tlaCMZGmutQ7v+UpagjyhIeKRIwTwtRUSDWADRaGeN00TxzGEHmPkvQ3vDpNISXWD4mCMvXfGKGMUAAB6jyFyxs7Go04n53HknXaBoy5bIYRWSov2KlY3zTCBxlgEHEEAQS+FUKEtZpxgqpSSspWyDVcfEtoambAEc+Zlbb0njEPniu5gvigtgN1ul3H+zTdfzWazH/7wh4zgiHFntWwxRiCOopSzhLFBt4ch6hTF6spanqZa/v/Z+o8mybI8yxO7/PGnnBln7h7u4eHBIzKzkhRpUmgiM9ODGUwLBBAB8DUAfAwsIJDZAZvpBRbTXdXVVZ2VmZUZSSIyiJNwapwoV338vUuxeB5e0S2whYu7mZqpuprqu/ee/zm/I6uqMhpaFG9sbJRlOR6P65CLlohiSBpeg5ciS1JRcCWUpTEjNGSWqLIP7n9492hvuRp/++QrQuDRrb3RaBA0nNt39wqeMpv1er1OezTemAiFlvP8+mZ8cXk2m8aNZjja2G61fcZIojLbth3XMghMJpOnz16eHF+ul9ViJh0aSI0Xq3i1jg4P9putznK5jOPccbxWs80rGa3TS3ODoBVFCTJICL2zvf/227c3RlvTyXI6WfZ7GxTBxXSSZ8WThw8Jsba3dm9ubrZ39h3H4UJM54tCaITJIl5DRBzfG4+n7z54cOfWW7/9h18ZrggEuuSQMsAlUgpBCCAAQENTp5+1VBwZCQGGCDNKLcuBEHOhGo1GkqVVVSUJsB1muy6hiBCSFktjDC9FWXApNcYEA4wwUqIixnQ7rU6nFwZNJVS9oO4OPx4MBr1eL8uy6+vrp8+eLJfL8/Nz9V3zr+u6lk3rd1pepPMiRwhpICEARvMgdJlLjDFFWbY7XWKR0LYsCLJoeXV2cnH84vBorx0GgWWtF2stleGiHTS2t7Zubm6QBjcXV2mcNNstgplLrZVQRhpZSSErUVWEEAMhxsD3nPl8Xo/h/vHSppSW1bpMLYtBIyluGq1FlQEAKDZ5mmgF6qS30ZBzKaoSGlkHvI0BSimplZTaGFj3ZXBeKaUgMrUvEgBNCD06Oqwjsp7nOY5T25uV0YwxY4CpeezfY5IXWVqkmTFGSaK1lARZDCsJ0tk6jtN2Z7K7t2q1Ggp5fsvtDQaUiD8+/FWZZxTiBw/ef/XieTMM79+/f35+zqVqdNq9fmc8H1OLCi0MAVXBi7LUWqd5PpvNsOV2fb9SohKl5TrSyDhOmt3+g/tvQ2hOT08vj5eEYSAQJhBhbDHKHItZOKuStEhbDR8YjSm+mU5+8NMf94a9J0+eiKq0GNnf3/Vde2d7CyHAGOt2u8z10qJ6+vLk0bfPp8tIG4koBFBqqZRSNWmv3nlpA+rIuOe4EMjZZHL84mW/03333XdDjF6+fLmcTmaT6WQy+d3vfnd5edlqtQAAV5PZvXv3rq+voyS2LDoabZRlOZ4viqLI04xzTjFpNJr1WXNzc7M/HIRhmBdFkmcGAg2AECLKMmpTgHDQDIqKx2lCLStstiaTiQEgbDYsajnMwZj6fthsNtPZUkp5cmGUUpACKSU3QkiRJEmtwxsIEMG24+Dv4MFVVV1fXy/XK2W0H4a7+3t1gKJe+SiljUaDIAwhzNNU85IwhrQSRV5mKUYGAhmtm71ee39vr9UMlZJlXliWI4XOs9JyHESxNBJoYICpcwtAAWN0KUoAtIFASMlsNBgM2u1mHMdSakJeL8x1nfQ/SgjfawuE8B9dCP/VdqFevdB3hY1vZgTgOxCT/q4bBQBgWRb6rt6zfk5et5wAQCnFCLzRMN7cUb3Y18bGN5OReutQFuXr+h8Ez87OXrx4UVUVZnQ2mwkh/CCAENq27ThOmmdFUbh2UGO8a8BJ3ZIKAIiiBL2uvVY1zb32kHqeY1mWMjpNU2l0WXJMqOOGzf5AZNmw0wJlPl8Xl5PV/vZec7D5m89/f3Z5oxQ0CANg0HdNoVmWOY4lqmIymTTb3Y2NzbDFryZTnqe+72xvDHc2h82Gj5HWWkCsXc+ud2wQENsKlENSlRVFgbEywCitoCJKKWMUQqS2KygltdaUIJuyuvmWc85FmeWJlVDHcfRr2gE0xtR0FowpogSh1zhtwaXlWFqLur8xLzjQwnVCABqKV0YIJbko8iLLlZQAAAdTy7UM1FVVVGWhpQBSZUmqpWy1Wh6ljmNDoIWs6t+dNhIjQhg2GFSySsvKGENtDjDSGmBMIXw9TlKS53k6Hl9j4lMMHcaUa/d6HccetZsNBAzUBhhDEHYt12a2dnSRFnmeN8NGp9XOsmy9XBVKE4QJwghA0gjaK75USlqIYsZC23ZtKyBwPs1v7W0NO8FqcSKrdZHlaWojHEbFlFmuIdE33z7tdjabfufB+28RC0xn468ff3l5deK46ODWrb2DrYqnZZXnmWi3eq1uezqfnF9Onj8/nc4roDGxbIjsSpXzRXRxfbOxseH7oecG68V6f+/23bt3F4vFo4dPJuOx77XarUG/051NV5ubu//sn/6L/f2D4+Pjly9fLuaxb8GtjU3H8cqSR+s4SaOb8dXl9bWQ+mo8g5Tajl0pPVtMCLM8WQSOs15GR/tH3371TZnmLmOB7RIAHUoZwcRiGsFKCgRhvVfodluEUAix1AAhYjGbMRtALIQqF+V0OrYsi1nED0PKMISwHXQwxlqBqiy1qvP9SEuzU++jhXYc17IszmUaJ2VZXl9ezSZTQkgURePxWClV+1AghMKAJIrjdVQURS23hn6QLBe2be/s7YxGI9t2CCF1n0JZ5b3BoKyKy8vzZ4sbrZVWilBEMdnZ2rYpm89myTqSXK5Xy72trWYQ7mxtrBazPC/Hl9eI4MFg4NtWLCUEAAEMtJFcAAAgQcSyGEJCiJqWzzDBGNcnWcFlVWbACIINxrDIUi7Kqky1qs8xBECktEHQ2Bam2F0XygCgNdBaaV0n66BUGgBggHodwjemBi/6vr+/v399fZ1lGSGk1WplWVYfXwghAEDznaXrNVBWcIKwxV73bnAujYYGYEycwUZfaTGLqn/4w0PbtpWobNuerEoMyauzdL1a7GxsKNDIS2t8fTXodba37yhgbNcpS0Bo0GoO5+sVxr7tk/OrS8sLB4NRu3u5irIkz/KigogIUWktw4Z/69bRO++9k2XJOlnOJykmzGCJCIBQYWZZNmY2jZfL8fjKsTbHs7Ht0TSNVtHSDWzLZp1u886twx9+8rFvM8eiRmmEUK/Xs7xwGaXzWWRRF4MMImoRp9Svr/4UYUophkgCDbSpiqLf78frNSN4NBqtl6u//Zv/9OFH73/4/ns2o77rPX3y7NGTx1dXV1rroqi0Mb//wxd37769XK5PTs7a7eb29vZ6vVwuojzPsyQzGrRC33f8ndF2v99vNpudXtf23IJXgSgNBFlRrKK1T6Dr2WUlojS+vpksl+tWt5emsZLGGOj7AcHMKOB5ASUWBMBq2B4hdtOrqgrbpEhLgAymqNluvK5xEtIoCbRSWpVlSZgFIYR5XgmOKWl1OvX7pVbj6xXRdV3HsmuEl0k5Q4hgLG1LKYWEWEymq9mUF1ujjWEcZVG0chzrvQ/e852WqPTN+Nv5fM4zA8jrVhbbdkhAqoKDOuCoNBc8DIPhxiBsNuqM9BuMhzEA43ryBgEA8HsWxdcrNNDfnzt8/6vfRXzV9wcE9U+uJwj1q7oe+dXSgvxeN3EtSNQ5z/qf6rtWaPQ97OP3dyr1j61h27ZlJ0mSZVkQBNS23rgi3gwXIEae5wVeWFUVqIuavit8l1I2Go03G5r6Qda2ocBhCEClNAIYKbOcLSE0VcW57UiDzq/HDsUiT7hUfqu7v7fT7Qz++PVTBKkQklILM4YQWq7mFgb9XtuxwiRJtNbNZnN7rzPa3llcnLiuu7HZH/TagedKVVZc1VYSADEEFBitpBIcKUmBBhrW48u6sk4ppbioEEJAG6WEUopRSghhjGAMGSMW85RStfGCMeYAS0otJTegPuQYSiAhrN4oSCVknnPOtSkboc0sYFm0HkoncQSUroqySLMiLQAAnuO6jlsaKcoqz1NelpUGPC9EWTmWHXpup930fX8dRWWZV0qYetQKFHMtjKlUCmEspUryTGuwWsfNZttyHKFkp9k8ODhoNptlVc2nY845BKZ2rTaDsNVoQq2AMdAAXpRxtIoN8mz3tdSkpeBlniVJvNZaYwQZxQgBMr6+yeKUEtLyQwJglRfLyU00n9862um1Qosaz8KbG72Kp82G0wjYeL1q9rx3PzqQIN/aGK0X+eGdLdu21pFYrG9W8WRj5+jO2wcbO72z8yiL13nG4zgvuDi7OL2+mnAJmEWVYHFUIgcQakVp8ujhk8B1NjdGw+EwcINms9ludY1Go9FGEmUYW41Gq93uKwncwDk9ueCVarU6t47uSqmBhW3bTtP06uqmLKpuf9DvtCeL9fnF6XyVUMfVCHFjEGV+2ARYO5RNp9Pb+/vtZmtydQkIaQUtpSujFDGGECqB4UBoKevlhyKbEAwhqooqy4ocFbbtEmqtVqurq4vxOGo02GDYgxDW16ksTl3XtSgjtlcvhKKsqrw8uX6VROl0Ok3T1HG8VtjAGJdlyfNKKVUbFfM8FULUIqrWilLL910IoW2zTqezubnpOM7G8E/TNG02277v39xMonVsO8yyLEq7+7u7SRpNrs7H4ynCsN1u+rbda3dCzzdaD3v9g+3dxWy+Xqyffvvt4eHh23fvtZuti6vLr79+eD0eE4S5FMazIYQWoca8psNqCYDWvuMWJs25UUppYBA0xhitlM1olmVlXlRlbllU8DLPsqooy7JkzLZtG0GmNZBCvQbMY4oxRhBCiI2BGBEAkFKv++kRQnUJXf18Oq7tOM7z589rRGOn38uyLM9zy7HX6zUhFFNKKUWoPhwYhJAbepi8vg5mGTTGIMIghFleIYSmi9XVeFrzbmsJF2nfdx0Mjefo88skWpvHj85O3PF0lr377js2a51eXiSxVJodv7wpubh/q/Pw4UPXazx476NVnP7qs9/PZjMFIGEsW2QG6sOj/c3NTdtm02mSJAnGsIbblmUupZaiRKaK1/PD/c1Wt9kfdE8vj4fDoX18slzOgsDrDbp3bh19/OH72xsbUHECjORivV5DA5ABUEOjAFC1s8hz3ICXKwAAgYhSi1KGAERAcm0cx5FcMMY2R73pbJxlWVmWT548GX384f7uruM4X3798PT0tNloM9uaTCb7+/sYY88Px5NJs9lUSl1cXARBYCchBBRDhiHqtVubo63t7d2NwVBoVZVinYzLqsI208acXpw/e/H84M5hDUEaj8dPnz/XAPnNVpbFQsG8KoEywHDJle/7SpY3Nzf9YSMIwqDhgUSXskyTygtz27bffusuQiheR+PxlHOuhTTGlFmOlXZd13Ec1/ccz+12uxjj1Wrlum6dBahfOYwx13W1lEaVEEIEceB6iCDGmBBVlMWEEM91l/Plq1cvpDT7+4e9zqDbGkZxXPIKQIDJ67Ie27Zc1yNtUhWlECJNU5FzrWVNUatVg/owLeXryAN+Xf8GvoMm//9xF/5XikI9dHjjQqgFBoRQVVX18PvNd9WtYLWTsb5NHYys77cePZjvMAlvRg9v9hPge2aFOggTRRHn3Av810EACNM0zfPScaxuENSMZwhhEASbm5vIWEVRAIOqqrKYw6htjBG8ynjx+gQPoed5jo0gwFrraL40xghtDAQQY1FxZluYsChJXMYm80Uz8ELLyZVcZ1mclb3uKIkzKXSec6fp+r4vpUySgmPQbHC/6RtTfybZ3ju898571VsHi8UizaPlci5VDgnQhhsIMMaEUgiIVrIsNK8MAo7j+mV1A76bxUAItVFCaACAlqI2jUohGCOOYzHGLMtqt5ppmkrJtdaMMUJplhX1zql2MtazewCQqhH783MuRZ7nGENKsRTFcsWn0ymvJNCmqqo0zso8p5jY2MIuLstUcVFWuRICKlNVBcO42QhG/d5g0GeOLSWfzTXnpdDKGCN5yagNMSCYhmFYSVmWvOI8DMMky/KybLVaCKEsS7MsXSwWvt9TSjJKa5MZZaSqiiLNpKgaQUhw/XJBhGKooYDcSFXlRbKO0iiu57kGEwAA4cll4Nhpks/z2TtvP/j6jy/G1/Gd26N/+3/8nxDFv/zlL4OAZlEsRU4ViKZzpnqkaDFt/+jT/43fbJdSaYsJhuYqQy3PV50f/PSnxljLaYG4++KbiyItJi+XBpNcVKBqd4IAwzRKE9fTEFcew1KkaXH9639Y/rO/+Cf/6l/8y+n4qev6y8VyPj7eHrQg3/nkk4+eP33S7Xaafvm73/5iPj2N58c724d/8qOfuW5wffbq4sWrq5sJpdR2vG67tbe355xfnJwdD4btvKqUVulq3RuM3r27f3l5yaOzixfizk6n2w0F78ZxGvhhkZeaOQaoUlcASN91CAVSVTYAQLtFVCpjLIYBMWkeZ2VkOfYqiyuV3nuw47khIowx10J+mi5Uvu702tKA2Xx1cnZ1fnEjlQYAVVVFEMaIHh0d/exPfkAw/N1nv5GOWSistUTAGMUp1gSRQb+3vXVweHAbAOD7PsLq9p1dLiNmQQB5tFhujhr93rDXG33+hy+ePf7KsWm7E56dv2Tko2+//bbbaf3pj//bh988Wy6Sht1zPb1aXORJOR3P/vwnf7E/3LniDAtczMvudq9zq9Nv9ZDWCMoongpRdeTQ8dxC8ul6LQxwm6HBaFXkmCJiuTazijQtstzIgiKMEUhMxhgzqoqXi06z6RPMRZVHuW3biOdGCY0IqPVwDbTWqATMspQGjBAvaGoD4zQzBkCEGkFQlmW0XBCMt4ajaLn68x/9VKssS1eYaOaQ5y+/JRbjQlBjY4y5lEAWhGMIIQIYY2xhUhotMeZl9ZoyBmCZlEopzqWuM5wQEsIQoFJCIUSrWZay5FL+4osbZZG3jvbZ9enVxQllWlXLD957QEV59fLxYrkOAw9kxfnLhe+BfBkxSYf+kBYkWWWO71gE3Ez41o71ybv3j/Z2z0+uf/6ff3txdeM3+gFzQ8+iBLmMISlti/qdzsH24Wiwd3I+vnXn3bLMG55/fXZs2/ZbmweHG7s7nU2q8WKxdBw7CANWyagoTibT04urpyevpuurRRJT6YaoaSoHQii0phg4Ni2qPONrDBE3FSHE872M50KZrBIWdYvCXJ+fHhwcGYhu37797PnLZZy0mc0BuVlE/+f/y//p1dm54wcAji1mhYEPlGhhiJuBClytdRj6zdBlBBRlEiVJFEUAgDTP6sTj2dlZp9M5HO5Mji9Pzi9enZ5waA5uHzGLZJUpywQzlURxkWYIoKJAVVGmcZIs8YO37z+4dfv87CxeJySUJs57jX61zDe3d0GFb8ScYI9Yfikk9RhyUKEU4WIQthBC88lClHK1Wrqu+9GHH+Z5XkRZNFuleA0AcCxbMlcLuVqtur32ajF3W8zCrBQa5zy9ngWMkcKUOVieToqg7wb+u4dDz6QX55E0AFBgNGCk7DQ7GGLj2zdXY5Xx7W5bS/Xpvff/5j/+R0QVBqASQmuAKcAYQAwpQRgTKbQQ4vWqhGm9QhNMzPfyim+8BfVBvF7R35w6pJQ5aUHBqZAu1hbUQEiuVA4goi7CVpRzxhiCOM/zVuA2PauU+o0IUR/x69N/vb2oP1PbC4wxGGMBodtsxnEcXV4FQRC02kIIhGl/0BgMBpxzY2CvN4AQ9nq9jz/8mBKHc358fBzHcf04CSGTyQQAMJ/P8zwPgsBxnNrJCCHEsKjL7apKuIHfdJqcc6igFrpSlc0cgi0F0Vtv3+fG/P7hV7998tkNzINeY8/f0YpCQLMs29resmzkujSRRbMTVnm2Wl6V8XByki1mJ1wKwljYHGBGCaXUatu2XXIOACrSrMxSbHTDcwTnWZZgAKWUCigCEcRavinGBIBgbFuWTRnUJlquLMtyHCdPE8n5armuKlGVAlMWhE1suTKKkjyJ4qVcV7bHMIZRtJrMp1mWQ23iaO3YzGU4X59jBFCVxNMpRjQvRCk0sWxkwRylBFOV8zxJfdfjCiBgmq02AfDD++90Wu3FbEoY3R+NOr73xy8/P7u4YIy5nW1jDJAIQAS0Vlwgoz3XrjhvNYJKCkLRer24vr6qk40S2I5tKyFkVVFCMJLawgYy3/OlkBCbMGhCoyQXFJNet9VvDTudTpakw24vCAJjar8eIf/D//Bvjo5uJ+v4yz9+DQ0iBPk+HAwGxsCvvvp6fH2xMepGy6zXbyLorGc5cKhb5FwJBGCWJRpRBYrpfPr1N19dXJxtjHpBs+F4LM6T+XyWl0UuRCIFF6KQXGnAjSp4VVUCU0tyzigFWmdx1hgEw/7QcT3fdgik16fnJ89eGE2k0CfPjz3HzbPk5vr85bPnrVan0+4vFvOHDx8eHNxaJvNHz75ZRWsAwF/+i39l+3C03dakvL/cR5QprQEm6yjqdrtbO0Mtl4uJqfHdH3388V//9X9yXNcL/DRPmM3qZKkBXJuSUjTsjtrt5unFerEUlRSu55JKZHlJIGr4vW57e3urGgwGGEAhhJFiuZiURbE9HJ2eni4Wq/lyPV+s84oHfmu4MUzjNFoub9++9T/+9//m8GAnWsyjxWK9Xo96qCxzISrbob1ebzAYNBsdxlwI2GoZHT9/ajtk0A3eeXA7zRc3NxftdvvFs5ez6WJzM724uMiyTCu6XE0fPX758vnLMLTb7S4GVrvdRdByneD85LzZaBdZ8fTxU4f4791/f7S5YRRUXKzi2ETaQHX//judfufly+eX1xeL6+V8NcOUOq4NlVguJojRZruVV0VeZEpUQBuLIYOp5FyUnDALKFBVHCotXY9QBCFWQhDXhQBzzqWqqGW5jq8QLIpC6aISSmvgssD1qDYoK7Oq4lWZGy2TdZRlGcE4WUej0eDW4dFf/83/F0I4GAxs11uslgoYhxDXD+phRC1OaK211EopDWWhRG3aejP6ra/LruvK6nWUqzZ11ZSYslBSa6NhWfLr8c2H7z744IOPfj69Xq3WndB59eoVIgRCuLW1/c5g49G3zxosVxJ4YbuoOKYOJgwhlKbpeAY6TfDg/tvvvH0XAv3i6dPJzbUsS+BrwTlQEgKaC0WRKYtsazQsy/Kv/uNfKyXe/+BdhKDWuipKaMDh3bvNpi9VyYWSWkIMKs6jKBnPZ+Pp4vjs/Pj8cjxfVAr4AYCAEYQ9zzNGGa2UkAwjh1lSCmy0Q4kWYhVFRoNRf1CW5c3NlfPOwWS9LItqa2f3zv17X335zWw2sx170Os6zJpPZ2kU97rdLImXs/nezm5Widq7qpQqeXV5eSml7Pb7QoiLi4skS9M0rQX/wWDwo5/82PIbv/v950+ePwvazf2tTc8N6jkaIcwY6HgugRhoA7TmZaW1rvJCCd5qNrvtFjbaYRQABJGGSK2j+XK5UlB6nksdwo2AWiMDGWNQ6zzPjDFpkqxXyzzPP/nkI2PUdDqeTG5q2k9VVZ7niWjVaAaIkuV63e60BC+xRVvdlu+7zLF93+sNeoHgYbNVKRWNJ0dHR5hahL1aR4kGEAKMEC3LHELMCHNdO7OTvMqtuqRN1jw+WJ/VIaiNhK+TIBBohFCdwsffbRTQdx6CNxaEN3lI9F/2QtW3wUhjiKnRQHNhODam5iWX2gCgEcCknjdII4RIksTygu97IMx32Mc3EsKbr9Z7iHruUC8JdcUaAKCOTtTvr9pwwDlfrVYPHz68d+8+pbTbazeaQZ37Rwg1msHFxTps+JThLMvErGq3247jKK2B0cDIWjKRUqbxmlLa7zSKNDVaetjvNsOtzcH9O4cWJdPx9edPUCt0h8PN7c0jAK3ZdDkeG2b5EMmKp1rrej7iWe5ysa5KEccpodTxasgBZZaFvkPlvtGWEMD1c4oxFhJpDeoRqlYKY0wpgRBiiA1QSqlSlQAAjGGt8TQaDaUUJSxotmzLLaoyz9N1Ej99+jQvMwUUcwi/KdfxSilhO1a0XHuehyECSgutEcSB71vUxpBkeZlmc8GB7SCKidG6SnMMFcOQ57kS0rEdignFOEmSOI4Xi8VoNGq2Osz1sOUS5jS73Vma1U8mYxZCiBEqDIAAeWEDQZLnOdMQQqggthEBADDFiYQIQMVznstSlEGn2wyDRhASQlzbppiUeRZFUVUUSojZbCJEVVUVZkQaVQMVNNSk1fBGg85o0HNtmyIWrVZXF9dvvfWWbblff/V4tZjISuVZebDTD52B5znXyVlaFhpASnRRFZBKAvVsOVmsZ0m+bnWP4jRSmkWL2WIx0wRJC1dVlYlSKA0xBphgSgilGJIyzSBxet3Bzsbmnb2j/f19oLRPnFcvT371t784O71pt3t7uweXp2fNlq+1yIuUcx4EQRAEx68utX7heQGHVYVFYcpOp3PnnSNKKbF02LAOjkau67quK41eLBa9Xm9jY8Mh+a9+e1UIeT2d3bv79jqN8rwM2qEhOuUxxsZxGWVAS4AdsLG78eDd+zs309/+9g/PX5xgbFyv3ZTtJOFZpAiTvteyYFBVWVlU0XKRF0m32759+8Pj41NC0MHezuHhoZTq+Ph0ObncGG0Bnh/sbG4OB+v5Yjqe8kql6+z/8L//n4xRBtQ0aJTn+cnJ2cNXF5//4WtgiG27//pf/8tm4MuKZ1G2OdxMy/jq6moxXy0W0Ww6bzQaR4d7UpUG8Iuz8zt37tzcTP72b//2rTsPbCvo3R5lOgUaEESLovr8D19ARe7cest1vGgVQ4IRxpiA4cZgOBwMur04i0+fnT568vji6tw2ntcIEbbSsogXN5XgderMYgQow8tSC4kxHvZHnu1oLQmG/W4HaJWt00lWEkQJtQAASmkIlSAKYAIhJlRJnVeVgFhzHkBMoBEAKsoQqm07QmADjFZvv3XXtqxaehVCIM4ty6K2pbQWSmmtKcOUUiVlWZZlXgkhgNKGYoxrbBp8Y/N+M46t5UFj4JtKXKmA0JBRGxD95dePtoYbP/3BR+8+eO8Pv/65495qd/vD4TCXj2br9O7du9s7+6evHidJapD96vRKClhy1ex0ANLbDto/2j483Gt6/vHZ2fMnj7P1uiqBCKRWCmrDMMmrInCcOE7/7Ke3At999vJZt9UMgoAQvDEYzufzIAgGwzYhJIqWVVXVDrX5av3y+OQ3v/1dlJZxmhZcA00N59E8ilZZrz98nUuV3KFgtDFs7u7YDpNS9jpdKeWrVyc31xPbtjudQ9u2V8V6cjpZRWsa+Fvb22lZvXr1yki1XM6h0UBroM3+9o7gZVEUD95+p7mYHB8fAwAODg6klI8fP/7m0aOtra3z83PLsYMgKMsyKwvXdQ9uHe3u7r68uBrPpqvVqjsatdtdQ3FS5lKZ2pPoOK5FLaA0zwuEiIGIGJVHq/2tjd2P3yuzUgihAeRcvnj1sihjDbnnM4NFkhdcCOpYMs8t6vGyilYzjPF6taolYt9zBC+vry4m4+tasU+ShBK0SlZ+K7B9O46WQXPj5moJIKGURPESQGW7zCBoIEjLPC6y5WJNIQII2r5r4iTNMwARAEhr4zo+qrDWGjKgoO6N+sjCBc/frMQAAAhqpId5I/gbYwBA8Hvcgu9ISOb7G4La11z/uvX3EEkQQqgqLQVAutXytod7Dd8XQiRFdT1bISy4hpRgiqBQMM/zIs9C9Y92xdevcCm/74V88/Phd2SF7yhnrx9Ju93e29vb2NjAGN/c3Mxms9qfb4wpiqIGztaLaBRFdYK3rqKu36dxHNcBFiklpVRzUJaF1tqyLIdg26Ku6/g2fbWe8jz1bWojeTDqbvWaTd9jKncx3tvcDINWkSRSl1VZFkVmsB02nFW89D1qtEYINZtNQohlOb1ez3Fd23OpxerJO4RQSQMAAgAgRAhh0EgAgJISQVzLNhBipQzXEkDOOSaE1PQ5jDFAAGPsOF632x8OhzV9i1gUISBksVot87JYLBZFmWgllJZxJYSotOQQQSVkr9keDYZCiKrIy7zABPY6faVUs9GVUvd76zwvKbOlBpxzTGmRr6hNJ9cTCIDHMKTYsqzZekkwpY4T5+V//vVvkjRfrlJs+3n1uhRUaw0hwhhTiCizMKYQQqOMKbnUACFEpLFtAwAAyQIK27ZsChTX3Dak5zv7u9sY1Ocrw4uiSNIqyxUXCCFBdJysINKNRpCXRZQsESVaazIc9RA2FKHtnREyBBjBq8yxqOs4RVooDii2Oo2hkvTkxc27777b6feqkhujKMM5r1TJKSUUg7t3DhgxvV7r5PyVa1tGc4mU0wiiaG0gpsjBWgOE65es6yhRCqiN0fLOW2/97/7N/9hvdyzK8ixlGp2+PMnjzLfdXrO7MdiI4tXkZmyAvHv3DqUYIcKrkjLMGMmyJIOi09u8mSwwdZ69OPF9fzab+Z7zv/wv/25ra+v+/fu+72ZJ0vQCKFTT9QeDQZZlZZkjDFvt5mz+vKwSZsHR5rAoUwAkZQALRCi0bNxoeI3WcDxt30zPijIl3KEIY4irXEbrEnYpAqs4mhVlvF5NMJSH28PA88u8AAA4jB4dHe3v7//qV7/64osvx5evhr3hrf2d9XyGMd7f2b9zdCdNc1Asy6rM8ihOFpPJzdnZycXFzXqVGQGkgE2XehR5hETT+Wx8U/ieP/QwxicnJ+fn11XJbdve3hrt7m29+969P37+2/39/S+//PrseHxxcWFbQbczHO211uvYtdzDg1svnr367Hd/WC6S27fe6vf7zPIgNGWVGQWanaZFme/7O70N37WVLK8mV4Ln2CKqyJbJynVdSkhNK8uLXJVVGIbD4ehg963BYBB4jmXRVjMsszywPYdYNzc3vOQQEcwoMChNc4IZtS1qK1NJoUseCw0BwpbR2HNsz+6mcdZqNF1qAaU3BsOf/OhPnj/+9urqqigKLoUlJEBQaFWU5WK19n0fQQIhFJwXRVGVXEoJtZESUEoNod9ZqzQCsE7Sy+98CQAgBF5LvoAgWZQKqGa7f3V+8ZvPfr/R7/WHWz/703+yNWgRbBaLxXKxnq/WX37+5fV0Ol9MpDCNMKkq5XuNy+tZrxdYNvrgww8++fQ926FFnh4/fz65mrkWRUoArRUH0GgEiTEQM6qMPrh9J8+idqfX63fTJG80g06nd352Nuj3CZJKi6KUAADHb0sAr65vHj199ujbF0pDxlzHCyjB0JRlyY0CGAFgBMPA9YLNfnd3ONjcGm2Ohowxz/PSNDVV9fLJk6eXl3t7B++8887FcnExvonj2H/+tNvuvfPu/SDwfvOrfxgN+xvD/t07t3mRc84xIlroy8tr1nAAAFmWxWlq2zamdDabXVxddjqdn/zsp0d3bn/22WevXr1CCNmuM55OLi6uGmHr4OiW63lJllLHhQBXohD8temSUAtho4QmlFqWZatqNrnih7v3Pv6obi+kzE7T9P/5P1/EWcpFziwMCFClwEYjrDwLNzwrTQXS3LYchgAjsNtuIKCMFGWZl2XueR5jBCGgtZTQpEWEIaQWiuKFkCUhpIxLx7E5zwlFAGipVCm47XheA0yvbxREQatdKS1mszzPqUVCz4MGrddxmQmpge/A/dsHzLNEjbjSpuZjou9Rj2qtC2Osdb0/eL1Cay3/K4/Cm0P8dwHL71VNak2ULKvYscnB7q1/8tMfb20Mj89Ov370HCA8j/M4LTUASinJRZ5lgleV+se8g/mO1vxGY3vz8Ua9gBDWYIbaLa+1rokIxpjFYjGZTNI0rU2LBwcH77//ft1TU88dioIQghqNIAzDIPBWq5VS6uBgL4qi09PTNE1936/fnpyXBnjDQXtna2hbhFfZsBeUiQochlVeZfOr46dZGFyfHr9z+3a7M5jN07//5e/SQje7/ZoYEcUizbNWe0gQcBx3uLkVhM1+b4hBwGxLaJVmWcklQJhQixCsa3+0EVqb1880QAAASi0AECEMIYAArOcvlOG6NokQjOoJESGWZbmuO1+u61EO51We5/PFRAiRF0kjsDEL4ni9WmW9Tmt/f5/z6uTkZHu0ORqNsiybjicLoQAA2iAuVKs9cF13Y0NlWSalVkY7tttut/NsrrX+4vM/RlFEqVVv2VbLFcK03+8riBazVVFUmDlI49l86XiOAQAZYKTSUkMIHcuyLQcCADUylrAZo5QigJuNpjFGyyUlgGEtlC7TeD6fo7LUeWYxWyklpTZK15sqgjFGWANZFRxh2O60aIrinAEAsrwkO9ubvu+rSl0uF7PJLIlX/W5rZ3skysxzGYHh1mhrMBgs54tf/PLXAOAf/os70/lCFgU0WpZFmucEaceCD+6/1W66g0H/6voCURR67YoXleBClAAAZiEIsZSyzCsjKqQlkNy1CIaq12ncfu8BAGB5ckwIOXn+8sXLV612t9WihNL1ek0oXq0iy0adTstxrOVyWZZy0Ou2Wj0pqlToRnPUbC2XK/7lVy9sRiGEd+7c2d2+1+m09nfudrvdLMtarZZDKYN5q60cl1o2jpP5D374wTqaBKG9sdWBUBXjZRyvLJtSihhzGGMY07y4PDgaQPTg1avJeJxVAlBiY2xZlHoWQ1ojrYgGmgvbYZuDkVJq2OsjDIzRFjF72/3ZwdbFq29B6Bwd7m/22xenx9vbe1XBA68JXfr/+p//H3mROC7u9ZvaVMZU29ud9959ezGPP/zgh4Pe1pdffv3bz37JefnJpx998MH7UsperwchvLm5cWwXAJAkieu6e3t76+U0TdOdnZ1/+2/3jaarZeo4DtQ4jVK3Fx4e3p5OosuLm5enZ41Wb+/wqFLatRhlDkIEQcbLuIBF02t89MGHw0Hvm0dff/nNl9fXl4DCQSOAECpgTFVxrYHSnUbz6OjW0dFRwxttbW10Wi3btpqNQCkx6g0+ePDBP/zm10+fP7seTx1AbNeSFS9FhRCtgAAAYkQrodbrNYDU95qdTp9iNh/PfNsetjvXF5eddmtnY+MXf/d30+m02Ww2Wk1MWZKlaZHXGqkQQlIJAKjKsj7u1MHLSlQQQqDfDGghRK8d6QCA70xhtecMAgAKrhUgaZK32oP9w9vL8fVXXz++d7Dx7lv3mg42mj99+rQsy2ajPZ/PkySZjOcYWQjaCNIgaLguybJstdbzxbTT6bTa4enp+fnZRZGDfq/lOtoQCxgDgGGMackhwogwatk8Wg5GG0Crz7/846cff7i7vfvtw8eu5QnJMSIYY4s5XhBGSf7y9OL5ixOIqJa6qITUKQQYQTzsdjqdDoeVUhIC4NsUS351+ipbzXQa37t3z0VQQYC1hIoDVRV5vJiPS6mEAY1WJ03z6fTx4cHevbduHz9/tre9dXN9maSR1vLk/Np33LIs5+to/94hYSybz3//+9+HYej6XtAIb57fNDsGM4oxNhAEjZBSOhwOGWNZUfSHox4YFZInRUEMgIwqqYVWWgNSt0siggmjtuMHhhqRJMnNzc3Zxenezu7t27fb7e7VzbVt4fkizZPE8hyLOK7DKsGFKJBCQBRAlQQyApTmWSGyhMCqTClG0EiCIDS6XgezLA2b3jqJoDJbG535Yry3Ndzd3FwuZpyXeVlpxTkv65eK1GAVranjVmmKGB1sDt2mP1/NMcatVotARG/w5GbOlS5k5TbcnGeWb6My1VrVsGaIahMuhRBallUjfrUGUkopX9NHjPnHwCT6DtOD/stwxBs+EgDApcpySa8dHOwO793Z6/d6cbREpgo8u5QqL6qirIRSUlbAGKhBWZbfN06+8S3WcuAbS8SbexFC1itknUKM43g6nVJKz87O4jhWSgVBIISopzlBELTb7aIosux132Cj0Wi3261Wq4455Hle5yCSJOGcp2mqRW7Z1LIsjBWmstFiFsOrWcmoYA066rQULzGU0XpqRJ4mq0HYbfhBvi5NJXheioqXZVlpLoEAEBBGKSYuswhmRV5pZZTiUqtKqCTLAEa26yFEGLOAkkrqN2ZShAhCBGMKkAKvGRWgPkLUvlcEYd1qpYUsyjyKJKUzYyCzbcdxANQQwiwXXOQAaNuCWRbb0CHINENvd3PzaG8vjuP1fKGELLIyWierdRzFuTFGGpokyWQe2batuKiBj2EYtpt2p93+wSfvCiGMBi+PT5I4i9NEI6CAkUadXl5Ranle4Hh+DdDc2NhC2NSP2aYWAABoQyl1bU8LSanV8HxKKUUYQtxoNKSUWkHGWF2s2uu2k3XCmE2AUVUFIbQwRpQBAIQQRV4URQEtVYu4rk2hVkgLLkSeRWQxnVR5JoQ6OX4uhPrgw3dG/eFw1JlP5p7PjBYI60bo5nEkeamFTKIYaY2N0ZybqiyTqKCIUuxRb397M2z4vstqTXg2m02nU89GAAAMEUJEVgZxgCg0iFZGUd+F2vAqjcav8qxczGa9Xm8crSoEgWN7bugy3w/CdruVlllRxlEUGaAajYZFSiNAK/CLnOfL0mbuDz/5k1evXu3v7J+cnBwcHOxsHu1u3bq5uWGsqZQFgIHGTVKR5ci3SZFm60W1Xlzfv39/e6Pd77f29/fSNAWq1DwHAFiI2tjSlYkXCfXF5kbX8zzHc71wOp+Vy0W2XCZVqReLtc0caIBvWcpvNZvh/uhgNV90Oy0IwXo5jRbT1fR6o9f62Q8+DsPGahWdvnj25ZePz16eJZlAkCpljl9dpVnUHzQ2NgfdXrfZ8m2bbW3ttVs9RoMsFRXPMDHJMrq+vnZdZ//d/e3t7ffff//6elrTdH3f55x/9dVXq9Xq8vLSdf39nVuO3RgOgJIQozL0Wxa1bNsPw0ajUeYlv5pMHz761nHsDz94r+X7eVaNxSReL5vN0KNWJ+yMesO97R3fdf7mP/3VfDmHlgUREGWVFaXRsNXqbAw3drf3Bt2NptsPrRBpJEtRooIxsjXaOdg77HR66N//h+nkF0VW2m7AmF1WosirWEpKKQQ2oQAAJLjWEkCDjDRJFNudbq/dubk4N4In60WRRsPh0HGcoiqhVBBCxlhg264vi6KwmF0bvsqy1PJ11Im4ttZaGfCdnwtJ/fpcBQGozeFaAy1V/clc6Xavz1U8ma9bQSgNms3man/ryZNvkcxboY8RHY02v315fHJ1s7N3+P47D8qCc659r7G7uXGwvzuZjHWe+b4vlIzi9NXxyXgeMRdh5pe8DC3XmLpQm0BMq0pAhE/Pz5uh32g1r87Ojo9Pf/Dhx4f7BxuDUej7gmtkQQQxwlaaVy9fnT16/PTiZmJbvjJCK2kMQAZQShuevTXs9XaH05vr5WJGjMyjVRKvy6XXcdg5xbdu3VJVGc8mqiq2N4Y7uzth4F6u1xDiwWBEMbw8P7s8O711ePSzn/wJIeQXv/r7k5MTRu11FGNmIcepldysyLkUtutoYKbTaZJnvaErpTw+PSl5labpcDRSSjVbLcd1Hc+P4lhoBQnGklRSAqOlAQYgbZQCkCJcK+sEM2YDrDwXokUU/+JXv345OhZKb29vv3rxcr1cWpgMu528LGbX1zUWMPB9WJQEiqbHtjeHQRB4rmWMwZTlSRwtF3G0YhRCaAhGvmtLo28d7V+cnpRZ0m4ETOWffvDej3/wcRKtfv7zn5+cXSApLQi8wLepXeXVqxfHW7t7i/WKK94fdEedjUYv4FI6rhW4gd8KHd8ZX01Xs0Iofn5zQV2LUgohBEAZYxB8DUKozYnov1yw36A+vr8nePPBOf8u9fNfIJZdZjq93tHu5sH2kGFdJCteRAioPEtFJbQSUlRKKaQNY8wiJFf/mG54c0cAgBrd+P3dQy1d1I+qVhTqY/Sb2ggIoW3blmXFcZxl2XQ6vbq6arUaxiiMoW1blGJCULPZsG2WZajVamgt43hNKd3b23Ec6+zsDOBiY6NrWTRJI6CLqowwtiDiGlW+a4Ud38ENmzIEQSHLyig5m0+vJ3kFtoaDRmUyaeJkXQgetPzBaEQthyAQhM2iqKo88t3QZhVAUNQNh5hKoYuiUEpTq8YoUYIZxAAhJInAGFdclwVP07QsSyU5hLAunkbQ1MIJRVBKhRDilSjykkuJSR03BUJUghcQGqUFRApB1es2g6Dhuu50PFmv1xii+XJRcbmM1llWCAM0QJUypYKT8YwSIqW0Ket1rVLIy6ur2XyuFJdSrpOyKKUE0PZD23EcvwEhrqpKVFJrjSH2bZtgjBCSSgEAGKaOZRNCjNJv3KkUAICgULLklZJGAiM4r3jkum7LENd1meMHyIYAKIgaYUgIxQDWgyfbZr7rSOlbLqyjPRY0ikCidZJGIolIGIZaa4oRY4QS1L+132o0x1cXNrNuHe2mUYKR+vbpN1mcNZougvLRV1/v7+87lEEpXUKkzRxggNZVElmuk8dRt9PhnGdFUWdvWt5GURSScwiwsVloURUIAJDk9eNztS7/4dc/Rwg5truMpn/32W84l998/arXajXDdrfdaQR+Ei9Hg3aUxBYjvbYfLaM8S3e29kMn+PbqCiN7sDVwnTutVuvy4kTJUogCAHR5fVGUZaPRAABZtuv7/v7RLWd5Gi1vlMym0+nR/obDYLyaTixweHioxZZnsaqqOOeCq/n1EoqXdz9xpOSWTXb3hq1WczKNz85v3JvV6cnValkSbAVuw1AXamhjj0D/4uzxzfhmNZ8LkRZx66lH33tw/+N/86+ePXsxvrwc35xCJY9fvWq3NwwkQulmc0MbXFbi4nISpyvC1Pb2Ztj0/NCPo1wZ85Of/cl8vvziiy+SLFlFa3cyCf3GD3/4w+vr6W9+/dlqtVq2wt/9bhLF8+3NgWVZp6enD7/6djjY2drcR5A1A5as4zKtun3sOn6zrabTeVnx2WKFofnoww8dx7u6PIeKayWbYaPIuNbScWnDa/zgg09UVX3221+dX964LrEd2292CHP7g41be7eH3S2LOB5zEQC6UhDDClZVmhOLMdY6Orj19lv3T47PTi8vq7yijkswLiuhlVUpIIUmFnMcDxopBYyjjGHS7fQ82yYYdpuNjeFgMZtoyT3P45yfn58TZgWNUBoNESrKynVdizl1G7VlWRqbqqqk4a+ns+Z1uswYUJsWCWEEo+8SaEZUvM62Gcy0ga4fLBdrWS2BEItVlMTZ3Xdur2eXZ6fHd+/eef/TTzWmx+fXJycnnftvOxgrLTuNwGLYtSygue+iu3fvaq3nq9XFzTSrhOU2cgEFYFoqgKAxRlZcax2XBULwm0eP93Y3T1+dRMsFAEApo5TxXFdLDQGFgEoJ8ryKptHDR9+enF0ZQBChWBsKoMMsoyUCEAAOdVXmcZpE8+lYVpUFAC/yIvBDRm2IPEJzUaXR2rUwYmy5mJycvsqxhzUyUkllsAEnr46BlA8ePAiC4Pj41Xy5dDy/MkoQiAmuUTl1fG57e5s59uXlpWd0u9sJguDWrVudbldpHQTBdDqVUs7n8yzN5/NFxkvEKCAY2zaAoDbSG6MNMFproGEpZCW4FMIgpDDOuciKxXIdcan3tneKPE/jrNlsNhqNxWpepll/OLz91h3Lsop4mWUZpuT+22+1e10ptR80hFZPn714+vR5Gq0bjQBCLJWyrDbGeHM4UFUeLSCQ1aDbbTrWsNMaNPzPLeYxMmw1kn6feY1Oq93rdht+A2LiN1sAar8ZBqHrNR2lBUJIStnuNxEGQdCYNRZBO/zqj1+v0nX9MsMYGGMQfK1dKaWEkAQzjDGE/0hIrBfsN26A7w8IavMg/K7EGXwHVOg0/cOD7Xfeur2zPQRGzCazyfhqvVzMJktuqFKm1s9qtYwQUlYc/Jcf9X29iUJ8PyEJvuND15UNtm0zxhzHqbnXdfAyjmMp5ebm5t27d3d3d/M8BwC8nq0g5DhOjXKybfs190Ipy7JqKhelFIlpt9dGCKwiY4CUIOWCa1Rt7w0pQYhA5thZXtjYWi2XN/P5jt+fzpaI+cNet2VozHUmqulyYTvO1tZ2FC+xxs1Oez1d1z4QCGFZCqEVIrh2UShjLMvywxAAVO9+CKIQQo6E0TDPSyl1XVchOOacl2WptXZthxAJAGDMtixTM2Ft247yRK+V4CVCIEljKUtmEQx0GDitVqvd7iKE1qt4Pl+WJYcQagBKJbSBzPeRA7jU2PEsQHp+WK/rNiPNMABSRPEqSpO/+8WvldZCiKSUrhf4vg8AaDabaZqOXI9iUhWFqgTUpirLLE5cL1wsFsvpDGLieV7tC+FcNlpNbWBS5FVVZUUuhAqLUEq5TtaMMYrGnJdVWUouFBdSyk8/+ajdaPquB4zBAIRhONoYNhoNoHPbtieTCYSQUIwkb7o277TJ7du30zipba6nJydZFrcaIWWw3Qnfe/8+Mmg2mf7V//pXi8XSKP3NQ2i3xc7GZsPzpeTMc2yLYgwrUQnOFUaT6dSxrPlyESUxYVa33Wkyd7VaxesIAEQQ1q5jjMEAAwOjKGp1e9E6OT19GTYbw+HGs2+fjdeLTz/+wbcvTzk0F5ObdZoEjnN0uMscG1NSE91n02myyjzHavjt7c2W0VDxaGenP5stCJaz+XVZ7b48Pjk7O3EcywmsNMmFFn4YtNvtt27tRcuJZVmBY7mMEKC/+vqr3/063djYAADalhsEDYwxNFCVqkzK4+Mb3w9bnV6n3bAsZpDxA3b/7be++PybZ09Pi0RioHlRZkme0GI+WWutkQEImluHR712sF7OT45fBK496nW6nZaswLDv/4e/+s+txjDKouvxjCjlB63t3e72bscLgOvhg8OdO3fu/Pw//+rifHx2ev3uux/eOrpz+/btzz///WQy+dWvfnXr8PZPfvwzxtx//7/+hyiKXNddrqbz+fxwf/vBgwdpmj/86iVGjmOHaVJCXbx6dcws5+NP/8S2bc+TlCXtdvf9Dz949u3TvCh9x9VS2cySwJRZyQy2HQsbQiC6fXRn0Os0G8Hf/t1fjyeTnc2dnb0D1wu9oDXobzZaLQiw0Rpo6Lp2GIbMpnG0EoKrUkij+/3+4eHRIkrWUWIDRKiFkHJZM03TLE2ZIrZFIEC84mmUB4F3+9atMsu0kMPh8J0Hb4uqKvIkg7rZbO7s7Lh+ABC8noxr21R9GSWEeJ5n2zbFTAhRpNksWn0nNtapM62UqqrK8wKL1NAFJKXmZVVTehC0bqYz1/F9P/BcR+bp+Ob6888/96n6ySfvja8unj9/3uz0ep3+xx9//OT5i2/++NX+3l6r079z+8hxfd9zIIR+EBgAuNRB2AybHUKdLK/yQrY6/SJaYYw1UBhjg0CRV0HonF6c53l6/PzbXru1vbVr27aseBiGrmXbtkcIkVrklVouV5Pxoix4q9UJg0aaJZILgqGWEAJNkEZQCVE1GgGF20grB2OeZ0hrjECRJk+/fYwIvnV48OHHH2ZV9cWXXx0fH9PWVuj7eZKWWQoNWM7nq8V80O/a9l6j0XB9L68qgIkbBhARv9GAUG5ubs7n87wqO/3eX/7lX2KMV9H6m2++abfbg8Hg1atXdQ0Hc+w8zy8uLrIiz0VVCE4du7+5ySxWcW5Z2BijgJYaAClf1xNzgXQVr6PQD3rtTp6kX/zxq+nNdG9ntxGERuksSS1sffDgg5/87KcHBwfT6fTRo9+WZa6lAEYSBINWY2trR0iptX7x4gWEoNvtSinXUVKveVIIh1kFQpOrm6PdjdNXL58NOgyj6/OzZLUkEFGAdCVkUYWev7WxlSO92dykDpWyzIpUqBIijRAqiiL0Q8u1dvdb/d5wZ2/nl7/8h+k07jj2G3kAfOeQ1VozZhlUk44QIaRetAAAlkXfqAvf1xXeLOdvqAk1k2DQb21u9Lc3h51WAyqxXs3Pzk5evnieCYDtEGGXYKggNNpobfR3G4Lv7xLeGBHA93AOb/7CRVW7QWtGOwCgbplqt9u1Vlfjojc3N/f39xuNxmo9o5RCZKTiWmulhdJCKeV6dlEUnu8wa/D6vwC9vf0dD/ta67xMm8DHxNgOM0AYJEshi1LNy2pjMIIK9gdtjbAC2BWYQlCURVwWHFledzgYDFZ5YoxxPPfq+tyxbM8NcpYD13i+X2ZpmqYFr5htMcfmQipjMMZZVhBCoDb1kwq+s23WfsxOp+P7LkaoRqoAoDutNmPMsimBqDZv1gBjy7K0VnGaYAik4pZNw9BHCK5WK89zLYJns8VkMtMKUESklI7nYmoRqRhmlVBFVgptJACUMYxJofJsvYqiFQQaQxD6PoCuKvNWr+NWlet7ruvOlgtqMR/4l1dXPM8arm+EzNZxp91+/8E7dtB+9uzZixcvyoojF9T7M4g5Zcxy7EJybpSuoEYQUoIxQsJHFqm4WGdVmecIQGi0lOpmPCWEuK7LeRmt1jc3N5PpTSMIGZbD4XCxWFBMut1u6Lo2IZ7jkPHZeVFkVVVRBI72NzjnnBetnstc0Oh1Hj16NDzc+PP/7s9PTs7Wq+jq6ip6NtvcObn7tgUo7LTb6+XM8xwAYJWmWbF2m06SrhiCASJASAtSYFJqSt/WhABMlBCci7LGlXRsIPgpswShAOh0tVhujqzwn7zNsPqnf/npHz57eHycM5RvjTbRyToMts+vym4bAp3n3CsBevJy/OMfH1HLazabju0aY4qcO443GG5YljXodV+9eK5kIYqEV+nF6moymRweHhr01p37t8fj6XBrs9Hq/uCHfwaN8+LFyXqeWJRJYkxR9Xqdbrsjpbw5Hl+uytHIDawmxmE8Pl/dRIQQzwPDMIgDJ0fV1uYOBOzs9ApjeX39yA9I9/Bo+2d/Gnp+keWPH32Vr+Qv/+6XCKvRRv+9Bx989dWT3W2ymD3CyN3oObfv/eDTTz/xA4aJRlitoylhdlbCrBLHF2dZUf3yN7/+u7//hx98+pP33/9B+3pyBM5fvjj93W9+qSS8OjvPczAdj8fjy+ub5Wxv9uWXf3zx4mnQJJYLl9E1pU6SZYvV4u5b97vtJudSltVGK5yfn5r79442+yKNSgsxYtJsZdusVEUsaDfwoG1VWhhIG9uHP/lnbm/n8PPPP3dd17IcCLCSaL1OpICMMafpYIpdx15Uq2KdhUHQ7g0MBJbjDqvdB5+oAtBvHn67jBOCiNSgnGeY2ZbNpMFJSZUSvJRVtQ49+uzJH//5X/xpq904ubiMZPGHh9+kjltlRkK71aGiWhmVb7RKYHLLslfLpQc6GjKuqrwsC4gkVBUWBisAAEYME6SU4ZzX1GcMDcHAtlGv3fIDdzGbnZ2fKFmUGe2GtuKZg4hczYEQP3hnv99uKki+fn6Cws56NfvtN1/HWby9s6XMxrd5cjUdJ7zYnG1sbG80u6Q7sqeLqNExs+WrB9sffPrDd//hN7/2XJtSlwCSmAobbADUBtiWjwIyzwtLwFW8ihOcF/F/969+JAUs88q3HEaJyleAstBrKCXms6kCZrC1xaUpAEROQ4K0rDjBlpbicpEU+ubjro8JaA5aF+dnh3tHjdDP02Q+HT89eWLbtlKmPxx8/PH7BRd/+1f/fn7OaXGRB+HRnbuGOdub+4Sg8eXF8dVsONq8c7A7vzg5O70Y9DYun7zcPbitdCnc6t6dg6eq+PqrL3YH/o8/+POtrc3r6+vdDkUqOX86a1B6en4ThF2Z4sV1Pl2vESWl0mklGdOpUARLgZlUQCoQuL5UZr1KtIAQueto0m05mInZKqLMGY021rPZPImGhjf77en4Ugjg+Nb2re67nxy1Ok3oJuenzumrZ4PucHZ1o0t5c3n9r/+b/35jc3Pwbv+Pv/mDGJSGYgnV7u2trx9+s7E16Fm2SHBzf5fC3cP9g367dT3Lf/mLXyxmSeA2k1XpQmu9ii5XD2/3BqsnD68q0Oi273/wIMUmFwm2nSRLCNRIkyqXi8kCytndt+6v1rOg7QAIvMBPksTUFQ9a2Mx2XKssSwS042DPsymib+SE1351jDkXSZYCAGzHrQ/0lJEyj6Q0rm1pURgl9nZ27t57Z6elGCMUlpWskrS8iapIsMLYEhqMGCQYCIUtiogHMdJah/C1J7HefNQzBYQQRKiOP7yxKbw27zGKKSrLshJVDWPWRiuD4zSK4qjT6XDO/dAjDGOKTs6OqR8yjKAT5HmRJElVlLPVGgNTlZntEEog0FwrrYDhPKuKlFi0lv0cOxRVQRS2qe1bDjIAaEAdSgVtNFoHvaP+3V6elzOMv/j//L+Pj0+Z5STZqlPi7lZnWw04571W51kpg4bf67RuLk85yEu1kgissyTLsnar49iu5CIuqzItbdexLbfWSDQwnMucc651mclRv9fptIxRBkjXthyb1MJ76Aey4uenZzfX1xSRTqej/Ry3cK/XMb6dpJFSot1ud3u9vCyY40U5v15cFXlVGmgARABBy45NjpXCHoYY2RpTGxoFQs+VlUyjmGrdbXUt5vCqqkquOWi0jeTlah7brlfler1cFEW1ni0mkwnQphEEgkcYGo3VJF60slXLtgXWjUF7r9nsdwda69VynaYppowS5Hleo9kE08lkOpcIJGXRbvtRFFk26w07EPUIIXleOr7X6vX8wQi3mphzq9VWSiUAraQMM3H1/PTk5KQqsq2tDYxxVRWDfpcQgl63FxqpNeS8jONovV4HQXh0dDQajbqdntag1epQwhaLxYunjz0/fHVyJk11C96uRAkxKKvM991SK6Ol1hJBhBlGADCGeGUgopaNCMEASm0gQwhCUFUFwsh2me26AAAESc3saw8Hjh0OBgPbcnd2Jppji9oEoryaZkmcJOee65a6xNQIE3/16LPLy8skTmvVK8/zdrs9HPQazWCxnFk2ffHy2XvvvTfod6+uKtdxxjdX25OOKoRFmOUFQKoffPThsN3zvfD8/HI+X64Xy3oXORqNIISXl5fSLxmjsjJaoTBoSWHquqMH73xw6+j+apmkUZalFdpnjFkbGyMFc6jhKlpeXJzNx5OXz59tb/YhElqXV1cXnh/cvn07Sjg09s7O7Xa7H7SHp6fHs8V6a3soZBrFy5Lj45Pnl5cXaRqHQXt39wBB2/Ot8/OTq6ubjR1MKKKUDPqDP/2zn8RR2u12r29Oowg4jjMc9W/fvv3ll9/kef7+e/fm8zXUmNmOFwbPnz9fr+PhYNP1vE7HLsqMcx6GvtI+V7JWERljjuvVyEilJLOI67uB39ja2rl9+63xeHxycjKfLbU2lIp1tKxKYccuw6TZCoMgsJklLJ2mWVVVvYE13NymzM3y6uT0MskqXnAFDGUUYgi1MUrVnQ5lnhPIT08zXqwvLs+6vfe73e7rEDnGVbleTHLb1oxJ14KtVq/VanqO5/ntNBNPX56PJwtggOUwaIAxmrVaRkMAADKoqiqBBKCEIMw5xxjxqBRlsbk1ajQaNRAaYamVDhvevdu3NnqDbjPYHo14kTc81/VY4LlFkSklnjx5Ml/NZ9O5bdsXF4kGpq7hiKJkvY4hBGmSj7ZGdc57b2f38nKdRAUyhBHmOI4ByECkuQBC2ZiGnpes5lCJ0WiwPepPLk8vHOi5Fi/z3Z0DLlVS8Ol0Oh6Py7Ksle11kiJIBFdKyrKoiizzbDsIgrPTa9d17t294/th0/ekKBEijUZrb2e3rsYACErFLYtsbAwHw1fCcTEjm5sb3eFwa2dPG/nrX/zcdkiaxpPL0x//+E86naerKBuOupPJddDq9DrdRtjiXCoNIcCWEyBoOXZ4687bUoCT04somodhs9FoJUny6uS45mwaBRBC0ADJK8KYy5jUyibMIpCLipdxlqQOZb1+Y3553e92235z0O3tjDZBpReziax0K+zwghdlNuhvNML+9fVSa2bRJoSvS4yiKGo0WgCR9Xq9tb1NGTs4OJhHq2a3NVnOby4uR4Phvdu3dhqd7c2tbrvZ7bTee/sdBM0vf/7zq4sbhAAO8GAw2Nvz1nF8eXm9Wq2ePX+BukNA4Xq9Ai4iBBkA8jQr8pQZSLt9i+BKKC5KwqjtMOyBWrh+44/DGNdSfCNo1qTIigtoTB1qqFduwhilVANT1yJQSgkh2ijHcQiGGBkE7dBvHh4e3rt7fxjwoig8N6DEKvJ4Ml9E6xhCjBDAmBLGLJsgQjGtMfPKVEUtR7+ZaHzfkQC+h3iqLZzNZrP2Ida3rMON9beUZbler+s5Qo2ODYJgFsdSSoaJEGI6nV6eXwAtw9BnlBjNIdKMAEKgVlXFC87LggGjFEIQAICgafiBy0Jq0WYYYkhkJaaTZZoX3W5v//DWxk4jubis26RsxzPwtXC4sbHhOI7nebWzcjabFUWRpslqtULA5Hme5zmjFqUUEvzGCqqUqs8JECOlTD3ID8Pwu1pt4Do2sFjFC6A0NGA2m2khfd8/ODjglbAotRwbEIMQYYw5ngfLkguR53kleF6VgquqqkrBORdaAYAkhFBhCQkhhBGENQDSmLKsOBcYYCklMKaoKq0AhDBsBL7vn189T9NMCFkJDWDOuTQAMWYNB5tJkgCEtDYMYy1NlKQX5zcvz66RAXt7B+89eNd13bOTsyIvCSEGojiO11HS6LRrVJQxBmpzeXlZ/5brZDhljBDSCoNmM3Q9y2IYE+a4zLZZTdHAvIii6Hx1vcrmYnkjpeRVsZQpWUdzrbUQlZQSISil5KJcR0uEkDE6CLxaJXNdtxE2GWMHe3uYkdlsEqerRthcR8pizHVsLnJgFDQAQkgJhoAqpTRQyjBCGcYQYSglhxghqDAxQgFCCKVv6lONMUYBZXmW7VhB0Or1t/7kT2iZ86uzq7PTU4IqF6nVYg4quyw5QVSh5sNvX0ntIYhd160Pju1207KJ4MXGqH+wv/P73/9eKx74ThKtsySez+cuUv/yX/7L7e3dv/1Pf396evb2ex/32+3zsyuHsjSKF4tF3XK2v7+7tbvjBq7xTFmWjTBohB2L+VUBZpPTy8tLxiYff/xxv7f16OHjq5tJJUSj3eiPOgb5eZbxXBij2t3WXfrW9tYoL9bfPv5mOrvev3X7k09+9OOf/CQMepyjFy9O/vrvf75cLd57777j77x6fDqbj7d3RuPZDVe8O+jt7e2/c/99YPD4ZhnHsdewpotrv+EHzQASiAgpeJFXebvb7UUrRPFiFdVlVm7gP3j/g8ePvj1/dU2I7br+9dV4PJ4M+kNMIKaoLPMkSZpNX6mW1hpT6vie1tp1XKVFVVUQAiWNlppS1m51pZS7O/u97uDq6vri4mK9jmtpNL7J0zS1LNpqNF3XrufKEMK8ELt37naGo1uHtxH6G855kma+H0olIABSSi410LKqqiJLKFIaKyPKi4uL9957z3Xdq/F4uVxWVWVBPb46x4gPe344au1ubd25c8tibuB31kkRxcXZ+aXRBgIleamNBsCGAECIldHGAIgRI8xmljFKSlVkOee8kaa4EbY7PT8IkOtVZdFrd+7cvq2qskrTq+uzk+OXRRK//+67f/7nf661/Prrpw8fPtRa723vfTX9o+8D33WiKKqqqtfp4rv3LsfnkksCyVd//Prho+cIIVGVACDLYsUq0UhBBIGRCihdFa7r9hoBj+Z26Nw+3Dna33rx6Pev+PrDDx7EUZIkCcSkLMsoirI8RwhhhNO8iKMUEQK0MUYrbqQAdsffGG1vbo1W88XNzerm+rJIonbTPzrc6w83tjaGnufYzIqSCAAV+sE7D96Kk8XTi0gZsDHo7hzudXuDOFk3Am93e/TF57+dXF/cv3vr448/+PyPX19Pjs+vxhvQJJkvBby6nhW5KistBcwK6XhNP+ycnl1FcW4g7Q82K6FfHZ+en18aCBVXUkogVRGnVVUx17FsKpUKfN/wQvPKZkZAoQW3nbAfhk3XVUJu9jt/+qMfxXfv/e73nykhLcpGg43L64vAaTTCXp6pdVSVZenZfitsN5ttCDEwMMuyx48f7x8cYCF3d3a+/OoLhrAFIYXg/bff3t3c3giaPC/G1zdnx6+wNIf7u+1Wa2dnc71Y2ra9sbFx//4DTOnjx98qDRijgEBR5YvlxMdtx3UgQ67rFnk6m82arg+MMUZxXjme0243bx0N0mvFbBtCWJallrLZbLmum6Wp4IpQ6rmuFLqekWkpsyyrnQSUUql1jaJCiFgWhhAySjDSSlRaynarNRhsdDrdhicAiontEuoYgKQAluttbG5JDRGzICYAYgORqquABIff2x/o76ANEMJ65aj3CrWlt75Bo9FwXRcAUJZlURRpmtZTiaqqatVhNBp9+OGHjUajXn1tyjDGvu8DEK6Xi6IosiTRUhEMuCgpNkHgei7TRolSCK7KMqKUAgSM1sSyKAGMQkJwo92ACmdZDgnO0mK6Xl4vppkQFxcXzWaz3+9HcSqEIIz6vt/r9aqqyrLMcRxeVtfX10VRLBYLDNH25kaNsyyKohZICCHGwJJXBBd1Bz2mBCGitcYYb+/uEEKU5AjhOkZbFUxLMeoPKKXQgCzL4igSXNq27XneqppzzvOKQ4gNBHlWIpIAAKpSSG3qdg8DkAZGy7qUEhkFRMWV0FAbIZQQQpSV5YeOC4yGCIG8yusFjzKWFZxLw4VRVVmUvKo4sWzfgxRhy/YD3xdFXuZFUUgAmee3TFnkeT6dL548fVoV5fn5uWO5R0dHq2gdrZPlamm5juc4PAy1MSVEBFFMsc2c+WIxmd5orR3HgkiNNnpaFmVlIDKMMUyRBpoLLoBcmzwlPKNCmFxpKREnpiAAKAAlF0VZlp7nNFsBoUOEAKW0KLM8qyAsOC+FUFrrNMkihBkjeVEx5jrMWShTZlVd0A40xhg6zLEsS0pZllxK6Xo9CCGslSZYGehqzQHUjWYHYQ2h0VoLwQHgNYtkMa9SljUb3vbm1mC0B7gE8PeX4/PpZNwIXY2UxhoQ3Wq133nwoNm6evFqzmxnMBoyQg3QhCApS2NoI/A6raAROAhIiiCFRvPSJkRX1fXZGdEoi6PZePLN3/992OgsZpOb68nJ+UVdO5plyWQ+c3xvNps0cDtZp1mcr+ZxlhaLxWK1TItc//GLLybjpW3b1CKHtw4cx8mydLa8IrYGBmKHeNBhmPq+3RsNVkt45+37P+z86KOPPlESvDo5fvTt6cX5+NXLs0W6uHPn1tGdvVJkv/vD75araaP9p41Wc/9oD0Hiur6B8ur66uE3T9I073UHjTZax+tnz4+NJlWpEMHaQGY7d99+i1AmpBqONv7sL/ph0Ini9MXLk4Pt21fXkzjJhVIQwiTPWq2W5zklL9bJshkHlRhgihzHYdROs7iuSq8jiJRagteND1pKLaUhxNrd3d/c3F6tVqenp5eXl67TrPJKCyCEytNK8kWRlRjjNC2bzXbQaLbbbZtZyAAphO+7y2WMADJGaa0gggBojBElgBEMsVmv10KIQsgnT55c30w8v+k7xCKmGYb7O1ubG6397a1ht5cmBTQm8MJhfzTsj1dZkpYF5yWipO5RhEADDYRSUmiDldTKZlYUrSpeMYssVtF8udzaGH3yyQ++evqlANwPmOfTs/HFerW4c3Tr7Xfu/ce/+quvH35jgCqzvCzL7c2dyeQmiWLXtTGGGIHZZJoVedAM9vcP3MB1HG88nnz22Wcnx1f7+/eqqiLYrcrcc12tNTTS9TyMcQJEw7f3Rp3DzTaGqt8JjS7Xq/nsJr51a1dKfnx6YjlewdVsMY+iqNJIQZqVhYFQcKWUwggBABEhjLqOFySJ4Iq4QffgMMizaKPf8V26Ws53NreMgc1203ZtbaRl0zu3DoWsSvDs9OKyyBIgRbRe3txclUXSahwdn7xSZfL48cMf//inu3s7v/n8j1KVRZEJbqQBaVLajg8Mnc1iweH29rZUajpZrFdZpzdodHpPn784PT2VWuzsbq+i9WK1KoqirEpYYt+2+s0m57zh+0kaxUnsEIp8a71cRfPcFuDmfIqA3ui5ZT7t9xr7253Hjx9PbqLN7W0EKmPKVsMZbg6CMHz+/KXWgFKLUsu2HMuykiT56uuvf/yTn23v7d46POh3e+toBapqqzvwMD178uyEi9ViWZTZ5PJycT2+e+c2MFqUVVVVdSxLKTUYjba2tgxAu7u7awDiPFvOF9ilDb/nuO6gN8AGhNRthi1VcgppkeXz+fz0/CzNEkQCDGtLXSm1lkZrrYWUWVEEQYAJAdAoIQEAEGPCGDSqHuEzxqSUxsB6AQPAUMoogVwbiInrhgiSNCk9DDRgALKc65wbYrvd3sgLRKPVU9pwpaVSQumqqpIswxiv57PamfjG+lBvDqSUb9KYdaea1rouTQ7DUCkVx3GapgCAegABAPB937Is27a3t7dd15VSZlnGla6BgDWkyKL0erWejm/6vQ4j0LKoRalvW57jI4OkiGSZu1aIMVbAUIy0FFmWEMKklOvVknPlhgG27Pli9fe/+FVVVWup/TDwPO/hoyc3k9lgNPSCsK7VqOFOQJs0TevNQZ7nvV6vjkYrqbXWRkpjDAClgQAjWfswMCWUWnVGdJ3EQghe5GHgBb7jsMCxbOLY25tb7XY7juNHjx4VZdnt9nzfL4qCMCsryiiOHNdFiCijOZcAIaGk1sBAjCmkAEFstALGAIIh5zwv89oyaVFGCAXMUEoZoRhjoyEAeZaV62Rd8EIZrQ1UAAqpuFLCIKCgkEYhbSMKEVWASiMgcW1M/KA1Xi7TNCuKKs9LLWSeFk4/oMTqtft5VlKLMcYE54pXy2g9nU4t5tYbPl5VFBOv6bZaje2tzV6/bVmUMYYJhAQqJeN4tV6vX12dCyFW0QpTbLuWUVRpQi1C0myptc6yrOIloSZEXhC6SrcvL64uLi6qUo1GG73uCIDKGGPZbLaMpJRZnvQ7nU6nBwDgosjz1AADIIQGakWNsiEClEmINCPO6yQvUBaBNgRKCaWF4zADlFKVkBUikgGFMcQYx7HPrMC2e5h0gXK4KJnld7obUspOyy8LDo2hlFluI2wOBoL+4fPjJM48z3McVhZZxTMC4a3bh0WpVstJtxNaGAYOu3vnVpZEnUYTITGdTpM4Pb84rarys88++6f//C/jOP7666+zsjg4vNXpdG7GV2dnJ0kSzWaz5R+W/dHwvfc+2NzYvry8fvL45ddff3N8dtrpdJ49e9HptX/60z/56Z/9CBNwc3MlRDWe31DCGLbnk/VsOZ3eTNMiS+OlHzh+0Hjx8viLL786PbmZTNeLZbq9vf+nf/aTd955+9atw3U0397ZLKt0Op0GgVdPEK+ubm6ux5PJ7OZ6JqVMkshd0bIUJ8cXSqIwaLuux1wOibVarbJX52Ho7+7uv/Pex93ORhKX/+xf/DcBC8fT5aOvv6EUI0xvJuPt7a2dvd2z85MsS+bL2Sgdtpst13U1MFwoynntFmTMopQBgJWSSkLfa0oplRIIgyAIGo1GEHij0eDkZJJEseAlAtBhFqGIl2Wa5xuULmZTqbXv+xuj3tXN5Wq9AEYyRhDB1ChuAEAAY2jZ1KEQ66re/hNC4sXy+NXJOi8pc4kqDna2P/r43Tu3dyGofNdVEqRJibGKs6TIBaEWsxTRGjPLQE2JAwCAGkijNYClkKaoiqKyKCuLQkpBiooLoZVoNBpSm72t7fF4jBHKk/T88mI+m+zu7t7aO/i//t//bxenZ9cX59eTccsP7711l2B4+ur4wf27X3zxJef81uHBxdX11c01IejTjz7dv7X76uRlt9Vtfzi6deudVvO8KsDNZG5rk2YJZXRzYxA2/Hi9Chv+228d7u1uQiPPT1+NxzcIg7wsprOZ5zvLaGXidZwU51eX4+lEYYs5gTYQU0sZKYQ0xgADilLejCf2t88JdYosPTs7T+KVw+D/9r/91zt7h2WvYxDO0kJr4zhOmqa1h2tnc+v+XXxxcfXy22dhs93qdFeLBQJQS/X23XuijK+urp49e9Zo9999993iD18t1ivCKLMcxwuanTazvPF4phTY3NoVQli2v7G144cBJERrmVeJ1vJXv/pV3TRICKqkkFIawXVZDgYDJFQ6X82ubzY3Nt69+/aoPwjDsGXDKIrCwHNsixcJNOa99w4agbq+vn7rqGvRot0Omz6yCUcy5fkiWkaL6WI+W21ubu7s7EFEJpPryWQyHA6B0puDfplGzAsO9varqgBlKSrhu+6o13UQsihL4tiz7U67nWeZlHK1Wr18+XK2WExm873dg62d7exmbIxJosjPW24pIObAEALtsogm4zmDuNdqep43mUyi1dpybMv36xM8ItgIIITIUZlXpWVZECMuRVW8DhS02+3BaCiKnNoWgiQUvChK+R0fiXNuMQtBY6ixKAmDFiW2ECorkdZoFZeL+fjk7CpJKoioMno2mwupuZLaGA2MlLLglRCi0WhQSt+EId/AlNI0rWcc9Z+1jlvvFVzXfSPU27aNMX5z4yRJLi4uVqtVEAS1I9JiREmTpikCACHU7/am48liNvcsG4UuQTpeRUDrbqcBDdQSaAmkBFXFhaiUMkVRIYSCoNHr5dfXYwDgcLCFMZ1cj1fLqKrE1lv36ornsizrAdPJyclisWg0GrPZbGM4gAbU1a9hGIZ+UP9FCFHkJQCgHj0YYwilEKA6OmgEJ+R1yV+ZC2QAwdD3/Xa73fQ9qJVtUa3U+Obm9PT09OTEdhzLsTUw0+WiNWoKJQFEvu9TiymthVYV50VRaYA0AEZDA1HNStHQ6EpWJc+zQkrpup5jORhAjWSaxhBiQgjFBFPkeLaSBkCjjBZAK6AMQTZ2iEYQYA1Bq91Jo/h6PJdV6Xue44Vlnl5P593+BrPjMsvLQiKEbNfjXL58eey6NkJoNBjmZRFViee6UsrCzZQCVVmmRsuKNxqNra2NZivsdbpZkpY5xBgjCiE0VVWuVqv1en1xdgYAgMbYjFADNYBAQplxcn5xUg+rjFEASELqHaWO4vXLly9s29/e3tnd207iAgBk27Y7W0gpZ9OxNmYxj2bTZV2d6jiuMVRwlXJYMMmYBZEFISw5rgNECKF6IGeMFkIgDIxRCEtMJESKEGTZjDHW67QajYbn+EKo8/N5Gi+5BNs7e51ugKBer9dZktuWawA5Pr2sCuF5QRStoihar2UcLYQsbYb29kaBz5TIdrdHnssogdsbwy+0KvI00kWz3zUUY0YHm4MKCMToMloVvMjLYrVeGKCur6+rqhpuDj3Pe+vO7War3W21McBCqNAPHzx4/969dybzccmz23f2P/j4nVbbjbLlYCtst5s76XA2nk3HKwUqwjAkMCvSyXL+1cPLJ8+eLubLi6uV5ztJLJTG999550c//mEQeFmRE0Zv3X7renxzfnEdBN7zF8c1JDWOU8tywlYghc7L/OzptNsd2o5fFnqxjM7OJ553wxihDBVlAiH86qvnv/zlFxBYSqD79x/86Q9+wqh7PZ42mkGjESyXyzhNIAYQGmKRghdZlgwGPUwRlwJiBICp2fNaa62AxRi0MAS4yCvKMCFMSp7nuWXRzc3N7d0dox/lWXJ5uZ5Px7rdHA6HjmPZNrFsvI7mUgttet1eq9dtTmfX8XrO3IaBShvBeaVAJaUESjLEXNuyiBkOh2EYPnt5ulgs1nnJLEdhcfv2+/fu3et1G4vZjZAGE7s/2FrF1cMnL79+/GwWRZVRAigNSV7mGBQ11kZrLbgUSkquMFRJkniOixBJsrwRuM12Sxnw5MkTyfl6vcpj7tCg1ezZzLGdIOO81enu7+8n6+j4+bPVdN5uNcJrHwEoJYfQIGBarVYcp2dnZ9Eyopg+ffT8m8ffHJ+cOG6gJF4t8rzQ6+WKQQkA8Bus3fW2N0fa9ELP3T/YNFo4NgsbPoDqvY8+nI6vseMSx11dXsZpmhZCKeUGfsGN1AAAIITQBhkNDYIU24TwKKqevzhBjPY6LS9wHc/tNoPD27cObx0aUS3m4zha5hk3QCVJyrlQSmHEus1Ot9G+PL9i9kPI2DpeHe7vNYP2hx9+NLm6ePzw67/+q7/58NMffvzRD569ur54/PRqfHUQ7RGGKaVcyfli1e0NipJLqTa2tzqD/qvTk+uLCwMVc+hkPh6NRlJKrWXtMNdCBsxvMO+t3aP7994eDodKCYIxpTTLsjiOA08bhD2HrpfLF08fDfrt+/fubG32W6G7Mex6lFDboZqrZA28oO+7L4UEAF5dXBDChFC+7wuhHj16tLu7bYzZGI5Ekedp6kAULVfbo5HX7CyXS4yAKHKLss3h8N7dtx68886/+3f/jnPe7XbX6/X55eV4OvP8EGNsFMKQCQ2gpnkmFusCIRSt8+ffHhNl+p3O/tbBweFeJZXUutlqza6qNE25eo0tEkqqUldV5bZ8gyCl1LbdOohr23az2WxubRoEIcAaGK2NVEZKXUcSgFZVWVZ5Rgl2XR9jCgyeLqKqEtE6Obu4HN9MqOXari+lclwfSkE1hBgAAISSvhIQQi1NnudlWdaTglpdqPtp34QpalY05zzLMi/w692D67plWTLGiqKoc4b1Ob7uWa0H/5zzfrNZVVWtFjuWvbm5rYQOHLcs88DzLEIqXhRpEdU4CUMhCYUiaVoqIY3RUlUUk1bTpsTBAGkNsAFaKAzRcDBoNpvKDV+8evnixQtCyMbGRlGVy+WytkPWm546xIQRqBkPk8mkDrR/1/eGjTFKSQMBgv9YqFGHOQkhzHIJwqHv7uxsD4dDLGWeJUCqm8urKIpqTwZjbLFYAIQIowghxmw/aLS7XcZoXpXL9SJN0zQvDQQQIgOI0VC+doJAXuZKKaAMRcS1bIdZdcDnTSc4Iay+C0wwhMDyLVBCUBojtQEEK1BxkecVoVaNvhZVCQCwGKsqUQn10dsPxuPxxdlpEkVGSAJRlVfz+dxmVqfTaW23hBAYQC8MLMuyLev6alxkWVpVWZH70o2W0Wq1urq6qqpKgxo0DjDGEL8O7Ng2U0ohjBhhQEOkoEUsBBHxfEu/RoKboswmU27btsUc22a+729v7+7t7W5tbV1d3ihlwrDZ397BGN9cXU4nE9f1KLEdy0IAAoAIgoIYpaGGyGhmDAYAAIxcz3cdnxCGMaWUGg2EEFIJjDFCup5KIGxqRcFu+o7jYICSbL5cL5JkAU1JHZMtVxgZYhvH4EF/gIETZatus//xJ59OZ5PhoJMk66TpAKCKLELISF5YDA8HXaDBerVwmGMzWsiShs7xzeWtW3f+4l/9c8bcRw+fSKj+7J/9eWfQPz+/AEoTQsLQn80KoPSw11dGCV7xqgiCxsZg6LkhwpRQ9PzVs7OL406/hbBapzNEBETycvyiN9wGxGggMIOE4UpwneokzpIsb3c7m9tbkNppxuMkarZaw41Rt9u9uro6OXmVJNHLly/Pz67CMECIKKW1llojy3IIZggSKfP1OvLCdiX0cp0DQ2078KBdVdXlddRqI6U0paDdbi2WeRwtO+3hi5eXWH4Wx6nrB0VeNRoBIuTq6mJza9AbdH3frapKAYUZ1VoXRWnbNoaQUguA19XyEGKMCYTacchqtYrihePYm5sjq9cERR4tFvfvvx0E7qDXvr6+FLzEyGxuDDzPW6zmXPE0XRYibzfde3ePbAePJ7NlxDUwQlRlmUsDlDIYKG0hy3ag4p1OhzFWS6DGmLKsgo1ObzjAhC1X67QUvutxCSFkxyfHv/vi68vrGXbdTFSlqQzUqySzseGcK2kwJAAACDDGAEIIBCKEKG2MUdR2HNvTWs+XK6YthoKyMMtF4nntZrOTJuXz5y9ffPvi04/eG/X7n376KQUASFWmybPHj9Ik2tvdXq2yi7Oz8/NLSliel0+ePCUM19ejy8vLq8tZ2BgUuex0umU13dzc3NoYjkad0UYnDDyMMbPgch6VJW60Go7j9Aej9PCovpCd31xejcdxWrp+c9MO5lGR5EIBajKOMTYQVkJSwlw/xKiABtiuFTQb1KaOTTv9bhCGSinHshzbK7J8Pl/P51Ot9ebmJqWUEo9AfrR3+2b6+cnL09lqraF+cPftsqzSda64ujofT+cpJN90+rvvvfdBXqrZfFKIwg/cLMs4L7XWrVYrSRJIMMbQ9ezlav7y1VM/CPyAQSKfP3/BMHQcp9Vq3b11+52373/wwUe3Dg4RQkkUT6ezly9ffvvtt4vFopaCuSml4qLKVZW3Gi55x85judHfjOBSFQZJuLycmQzcf/tBaDUymCkh79y+SzBrtbuYkGaz7YXBixcv8vxn/V6v1+7Mb67W08n0IhtfXTAAKgkmk0kYeJ5lW5al/n9M/VeTJNmeH4gd7do9dEZqUZlZulrLq2buYAAMBjPYHQK2NK5RLF/4QfjID0CjGWlGGs1o5GIpFwM56uruvi2rumRWahE6wsO1+1F88LoNxFNaWkZEhvBz/ucnubCYsbLTvXNwOAsXtmlVQtSLeBiGZ2dnnGMEKVBScTCfxqPZzLLtKssN6ldlUmVSS+xajU6jN/XmtunYNk6SREqJGaVaaaFrAWkNjJum2fUCy7LiOE7zjCVxy3dKIZSqKDMty1Ia5nkppTRNU0tFEDUIFbzMs3I+iwBAi3C8WCyWYTQPl3GcWQoTZkFE8rzkUgIEodSc86xIa3KBEaOmDhljtUqxji13XbfORahniFrPWFMPWZbVYF6tYyCENBqNsixrIKROXaynjVrEUKsCJVdhGI4G4+V8YRq2looAYttOM2hkebSYLQDQlmVJjUsBl5HABFNJkiglmGuFkMZ1nqAoK8WFZ5iu63qu+8Xx6ePHj6+ubvygabsuQJAqXf9jtYcTalC/kFpuuVgs/tCp8Z8jsTkXCmiC4Q8BVj8kUQKMxR9eSDibl2kShwuTYM4rjHGj0dAIZnm+jGPTtvxmI1zGSqnaKVpynmZZWfC85KKO4wRaayU1VEpLBbTWkkvwpmOCMGZiTIXI4jiuP4u6xlNooZE2iYkxJghJgLFCCGipJdAQIQCRmoUTy7C8wMshElzoKkcY2q51fnEVxzGhVq/vAKmKJC2ylBDSCHzbtn03KMsyDMMqL7I0VkoQikyLaa2VFhihWuohpVwm8RsVCwQY43q7tywLuoRLRTEBlAmlEKOmYWBEydpaP8uyMAzjeDmbzbIsMwzL8zzBgQbK8zwAQBiGl5eXVSUajRbxXcXF1cVlkecNz/dsT3A+nYUbG1sGswm1JMSVABWXZSVLwQPfbbc6rVYHY6oVBABqDQVXUkpCEaUYQiBkVVWZkFwpVfG0rPKqqoo0UbDCVBVFlqcR15lpst5qo8yrdtNdhsV8PO33u57TAFBvbKxGkVNVPsLy9PjlMlqMh2mWJQhoodRsMgq8xtbGJgDgJB25rruxt0VsM46TlY21SorAMtc21us2l26rvb29fXp8YhjGzvbWJBwXRX5+fj6dhEUpuQSmaSKCKaWbW+v7tzcM2zg7P6Km6q+2yiq6uDxFhO4d7Dj24qsvnnz/7InvBgjA1fV1ZtoQY8Jog9rr63uU2OPZ9D/8p7+9GVxNJhMp+WKxaLQ6/dXezc1NUeRaa9M0CcNJktbKQUTJchkHftt1AimQkshz3Xt3Nxkj88V4MLi5uZm2W0RwYZnug/tv31yPXr486nRaK73+y1fPssxudxpJnmkt19ZWpZTz2YwxhjGspCh45fu+5rLGLbVCWkOlgBA8TVPXdV3XJQRJVSVJogHHGDHGABS3drdXV9rPn9lPnz5ZhJNe1nI9s9NtxWlaVDyOF51us7++eutw9+Wro7/5uy9kfeUiABXEGDDMarCR52l9aimKwnVdwMxGo7GxtdlbWUWE8pL7QcMyzPFkcXMzffby7OT0ElCr3+qms1EWxZDBQsi27wCNCllorSHEEGqAINDANE0uhVbSMAwIYRhHtsk6nc78OjVNc3ozPD+72txc393b4UxeXw3OqtMiT6EU77/1sN9u72ysP7h716R0Gg4ty3ny+DmEUGvQbDaLohBC+Q1ve28naLY++93n80XKDLJYRLfazXZ/5e23H62u9LXkrus2fK/I83A229nbGQ2HtusCAGSpOr1VvxGYppnns+lsPlmcYyEQtTDGlALMWFkprVGW5WmaIw0c27YsG0LQ6gSE4WUcezajBq0EH4xGjmEgpZQCs+n8+PWZ6zgba1u26UKEXFtsbe04j5+lnBPCTJOZzP7+u+fzyY1vG+fnyfqGO53OHz95/i/+6r+xneYXX/3t6urKbHfr7PiEEGK5VqfTGo1Gju8tFiGmKMniMAqZbbqe1eu1c85u3779/vvv3zm822o0oYJJFB+9PPr2q2+fPHkymcyEELPpQkpZNzvEUga+e3Md5nH61oMgXqrry3Ctu/XBOz9azOYnr46+efHUseb7G3fjyfBv/uPfhjy/fXjYbDYRZZ4XVJVYWVnJ8/zLL7/85OOPZ9PxZDhKwmWn1ew2Wprz2XQ6n80oQQSBJIoHlxeeY+/s7Ozs7KhzQCk1bafX73d6K81mc7lcchRIKQspomW6KPKr4ajT6UGlFQdlJufVcjyap2luWW6z2WUmdawVTEkSxfWRt6qEFpJzHkVJFEVFUQiu6j7A2vhwecnTohRC2I7n+wGAOI7TJM4ghIxSy2AUw6qo0ijmZcU5z6t4PJlmWYEQMSwHAFhWAmNauysxJRjDevWvqgoTlCdTIQQhJAiCWmHwQydkrWGEsMY5TNd1TdPMyyIMw5r7qKcKhFCn0zFN8+LiIk1TAEAcxxhjwzBqcwfG2DLMEvHZZPrNN99Ei9DzPIJhQYltGY1uF2h5dnKaZnG73eKyQgilWebaFqNqOo+VKFa6/Y3VVAilpS7THABAEIwX4fX5xbJSpmlub2+7XgAQaXc7pu2YpjmZTCilGEGDMmiyqsyVEnmer3Q7dYZVWVQ19SCE4Fyapkkwq4dRzjnnsqYe7E6QpWkiuEmwhZHmleICGyycx57nSS4KyRUEjUYDUJym6TScIoSqqhBaAaCm82kdn0UI1VpLBYQCQAOlNQAQAMQIrZ8I6jchFrXdoP4gqGnUKhDLsizLJIRcDMZKAaGkggpBQmzDtInWkHOJNCKEMIsx10YA5mma5ilJsmUYioozSrM4mYyHDOJer1MUZVmWjUZDS1Ur5wBQbdPc2NjK8xwAMJ1O6x+KoqoqYRBDgTf9ulIrrQgEJqNeWIUVrxyCNdBSA6QBBAhqRVZ6641G6ze/+Q1SyWp3++zsbDqOhzcxZWZvbaNQbJRUy6vB2Ty8Ggwd18cVNi2WJ/GLl0+/e/msKor1jdWDvV27b0kuFSxdy2fU4pVeLpM04psrK8ywijSTkDp+B1IjL4SgChFATcIoASqTcaLyWJcp1HJaDQlFCEqhUy7mVbWAqmQIrjQbBKBbtw6ePXllqmBj6/bfH//y+ugmCq8IIeFoRinZ2d48PT1WpVdqI47l6VFC4SJPi16v77ntv/gXf/773/8+nFuu6waN9ZOz65cvjm7fvt1ft+bpUmI9CsdFmu3sba1tbwzGw/3dvaqqilIGjbaQ+ouvv4rivLe62mq3t3a2N1c2KYXLePryxfM0WzQCOxvnEAFge6bJclD0e2t7u7MXa408L01qhcsZNdjwYppmJcSW31qPi+Li5lpcLbXWQolSVNCgSVW9Or+EEBqOrwCotNZaEy9AQGutLQBwroJmk8TRdDzpr3Y/fO+9+/furbRXXr8+/df//f+zXOYGtDWjFdfnJ2fEYIkuLcUlxq3uGqZGGutO04/miijHtZjd90ej6yLLNAKNpqVQSYAZLZa1HNpkCIEKqMoyYJkvCSFKVGVRGIQhZUCADGwBU84mE4OyW7d2o/nk5uJ8dnFlA40pqUoupPAte2t/P8ry6Xz23nvvPX92fHR80gucXtNZRplCqKh4xXm4zDDGBXWAEzw/Pu72Wma4WO/YP3/vQRrPh6ep4/rUsqVSpzfTv/n7f7i4GTLP9fxGniYqE5Z0dAr9EvJUEU0NBKqqAlARQhCAVVVJrSnAhJiIMikppSZA5jLWmWZCYGIFRMlwllr77MHtO/Fg9PzZ019f//Lw1t5jSMPtjURoy7IOPvikcfMUAIAsBBB4+OGtVy+PZuHirbfu9bprRSl++Xe/M1HDRmQxiG5tHty7dXfz0VqzGWRlgnRVLpPR6LQZtLIsGw4nZQEIxqXgpeRhNZ9ky42tdcPvM6+7SI6m0RRAGmVVf3UTEcOk5mAwAKJwbMBMBXEOicZQx9OCiSZBoO34bdf83T/87R/9+EdcFlWR24wgLaajq5ezWRqH77zzTqvVmufqfDBhtiOyzLfFeDxa63cXBknz7OzmanWnM59Of/5Hf/TowWET5v/s4wfv3LENA00cmVryrYP+B2/tlcsbnwETceTQ68Ho3u6uKsrffva7P/0n//SDtz9YWbtXK+YUBI+fPD07O4uS5Onz50mWKggUVgJKttHAGBdSLMsiQcloOTQDeO/BrhLlzfz8g/dv51Xy+Om3CJFYVlsP7wFi/M3Tb88urz7/5qtkHr6+unnrwf39/d2qyNf7zdnYy3OSJ7PZfLjM05so+ebFqeWMgUa251fVc0aMF6cnDrP/2T/6pwfb20k4p5Xiy/ndzZVXR896a+vvvvO2Yfi/+/03DCAhc5fRhuGKMvEU32oYRTWTEOZioSyBvWBWhNfTMQYQCOgQV0D59lv3kiRJs8JvBEqBMAyBhstlfHVxORlNF8vQd71Go2EZFiMsTHKMsWHaCKGiyAEAGCnPo1JKxhAAIsrzPM8hhEUqZ0VSZFVRVEoBShGzXUppbRT0m/4PGQlSysD334RAr4C6iJlXxTIU9Z5kmqYQosiLN3ZBCNMEE0JsyzAsFobzWrqIEIgiYdv26uqq7zfSNFcKNBvdaJkRbAoObNtejpN+v88QwbjyTN+z7HgxP379am2tn2i5u7d9cXU5nU4NxxlOp1wtNDKUEpRRA7GwKEstLULyOBJputPuDgYDUQq31QyLKkQqtx1l5Cv2VrvZjqJES2BZzuhmGC+XLb/Ji7wOq1ZCi1LyUkGEoFDMop1OZ7EIwzBESiMN+t1eq9UCiLx69UprbdpWHKe1n9MKY5OyaDkv58mdnVte0Pzys99QAmVZAFk9f/4UY3z79m1oGJ7n2RhjxxsMBpwXecW5EoAXUghUlkIrRDCBhAIgFCBS165sYuE8zzGSjs0MongRIV12Wz5jZpqmFNFWs+15QY06mKZZxulwNAZcmtQouSjzlDITYlxmeVUJ2Gi1Om0ltBTa8RoIknA5r8oSAKAgMl2viwiGkFoWJtixjKvrmyLPCAVlqjRQleQ+hI1GkKZ5ooHUqOSVZ9k2M0ouZvNwPEs1AISATs9jhoOJaWnZ9BzDMCHACgCMqU1ty7JJ4HcgQPdu399c2woXEaNWGEbTechMc7W/6gYtIeRkNJ1MZtEy0QpRMI0TQKlFGbRZY7238s/+7F+0G93lMposx8PRdZHfACAJVZQhxsj336eEGVwozJz+2mbQWoGYUkrzNIUKCqhklRbpvMojIEoMgWVSAJUSQlcCa2BRkwMteKWkDILG+dXli1cvt9a5aXndlZWqEjWqA6E2TePp82fPnz8FQB0cHMR5hi1jHIYAgK5BFlny/OSYeS5YsoKr45OLJElanW6WFa9evd7f3UMQXpye5WnW7/Ye3L3T6zSi5SxN02WyNCzWanXu3T/UmLleUAlOKIyzWEo+mw6ubwaz2QBBxRiCSCO7RRDSWq90O9PxOE1Tx3Jbzfb9+w8d28/SLx0XMsttd5o9ZFzeXFdC1jwaBJRgRDCAtagDoVpBLYQQUkhZB74pglgUJWmUtlrtDz/45Kc//tFaf5Vh2umsnByfKfV1lBRRlJiWbTsUEVoKVFu9Xc9uN5oEQsHLFy+erfU7/X4PE4AxjqKIWQaEkFIcjRcIIc/zDNsiBlMAcCW5kqZtSSmFVgoCzKhGMC3ysiyFzrRUrmkBSCDEeV6+fv365OTE87zNvR0NUBbGbqPtNHyCyPePn2QFz8uq3enfvnefGdZ0Pn/6/OV4PHZscz6ZqqrMk2R3c204uPYdw2Z4NBpBjDq+zyw7LYrZaHx8ejKezU3TBohwLsuiyLJMSk0IIxjX62CtduacSy7q99AyjJpGRABSShlj9QorZGEaGCHFRZWVxXQ6KcvtW7dulUX69OnTm5shAGg2W3z97ff9ft8wnLXe2uPHjw3TffDgHkKoEbQLXiFIkiT55tunw+FwMU8QsRzHw4h4XjA+H0WDie9ZgW+ZzAQEq0S4xG6aDUkQM6yiKAoNp/PJJF0QrgqhlstUcAAx4gpiTEuuDAyVAoiYzLJFWdVB7obBDIvhSgohMCM3V4M4XFCoWn5wsL+zu7WppAAS7N+6Ey6+/P7ZizQv+/1+yL2jo6P5bIIJSdNkc3OTELK3t8MYSpNohPHG1ua777//1v17QRAkUfz29vuTyaQsZBzlWkHfazmOWxY8ShOMaaPRSvLsYP/Qcb0/+qOftzudi3H2/fff//KXv3z9+vV0PnMDf2NjAxFcSQExUkArrYWskEZSK6mFKrLVXreIwrPXp3d21/7qL//ipx9/9OSbr//P/8f/07179/YP76ytrl1cDbMkefXiOQKi3Qh4md5cnzsO7TQb3fX+o7ce1NfL6urKwe1Dy3ePTo+iJAIIj85HSVwe3DpMCwkkI6x9fh1+8ZtfHBys2xbsIqkQjNKw4im1zKBhGpYquSmVklxQapqUSAi15BqCmJCyLONwAaE2LSarChOQJEvTchkmtm0jTC3DrIQ0DAMCVJbccZzCK5FGdbCBqESe577v1fD4Hzqi/lBEAqFSihDCGIN/aEjnnAMAal6glhnW1oMsy2q2HgBQRz7XPyOEKCW16az+gyRJkiTBGNfugBr6Vkr9EKUACKyN5fXdLcvo9XpbW1ta606ngzFutRumadYpNUIIh7DFfKKUchyn023s7+8RAniZIiAN04BI5VlSFonjOK5taK0wZUUlOedVxU3DbDRaTcfa2Nxut7omoQiSsijyyWS0jJFp+u2m5qyqBKUGYyJL8mgZc84dy55Ox57tNAIfIRAtF1zkCAPXtmrpm+u6SumqqrIsWy6XWVY0Gg2EACGklgjUeSSWZWEAlRJ1GGXtGnVdN44WvCohQpubm2maTqdToeTh4eHH778XijxOl+PTmzQjCqqyzKVWEkhCCAIUYgI0klppACFCEKM0ndiW63kexjhNwyzLCCHdjkepURYhJai/EmCMz88vlVJbW1t37tyhhE1mUwUgpVoBDRCruLBMk2JlMoMxpjGQkJdllUQzDHRRFLUyhRGjjpqllGZpoiUv8yzLY0YxQkADZfLSc+wkXZYldz2LmjRKYmYQPwiSvPAbfqu9LCrOTMu2bcMwENa27TBqMmYiRCDAhFDLcgxmEaztaBH1e9v9HhgNh53WyiJczmYzxw9sPwiX6auTs/Prm+ks5EIt3cx3X2epcO2uaTi84EWiTdzJIyOwdwrbmON8kS6LasFM6TcoNYy8AEhkWc41XGICK1kZzKQG00qqCgBVijIu80jznGKNCGIAaKVKLnXJsQKEMqikKCuowLNnLz7/7Csg0db2wXA6ibJICLWxtnp+fn52+nptY9U0TQn0cDgoJPcC3276acU9v6FMYxiFV999devWwbsffDoYDI5evsIAvv3Ww8Vs/uzJU1mIt+7fawW+sm2eJ4PLiyRc5HnWbDSa7SBou62OT0xacKGRBFoxi9za3kvTuNF0vIYTzlerMsUYUYaPzi6KotBSFZnNCN3c3Ox1Vlb760Lo9bVNISFj1mg851K3O8F8PoUc1PgPhEIDoZQCGiqAqkqAumpGI1gTwlApAAxkFkXBmLm1uXfv3oPNjR2DMlFVnXZva2trOBxfD4ZlmfuB3Wg6XMlKWowRAu0802kWOabh2AajaDIdBg3b87x+v08Zdn1fKWUYxrAMLcsiBsOMKgiEEEIrjaBGsCiqJM8AABpBruQyieM4LvMIaTAZDEc3N8+fPQunE6BkVVWtVivNi8vBMCnKly+PvGYzL6pFtLS81kefbNy/f18D8OLFy4uz83QZ+rZFMDJQ6/at7a5v/9GPPn7x/BkGuigziQDQYBGG40U4nofzKJ4vE8t0K6U1wH9wiUNGCGMUAV2BN4sgAhAoLbUCdcVvzSIqrZnGGBOEEYBSSkK1ZROOqRAJl+VgeDmdbe1sbwpeXF1djQbDLCswxklWBEFQlep/9d/+5b0774wnwxfPT3qrPWba4+l8Hi4GN5OnT1+0uh2pCQSsKsWLFy+iJL1zsIExWlvtzpFoBv7uztbgZiRKzTMCNJZyniRRWmTPnn97dPJ8b2/n4UcfD0dToaDruLKQCMHZdEmNsuSiLAXBlqZICc6FohQCjR3HZIQ+enAPI21gVCTLIi+1glUliiw3HWd7b+/J0xevnz7llaq4sppbRZZgjAxGV3qdn/+jP1lb6wshbm5u8jxtNBq721tra2uI4CzLhJKDm2maZv2VTc9qra1uTyfLZZi5rt/trGZFAbFJmHV45+F8sYij9Pzi6j/89uvj4+OnT59CjNrtdqvTkRCMF1MFNEUIQqS1VFpprSXQAMOWxQxZmSZrtINHdw53NzYNjIDg/82/+pdFUUzGQ6lVVZbbm2uPHzOg7cPtXcMwOu1m4LmCl+FiIngRNBqHh4eLJOKK3Trc/OCTtyFlJa8+++wzoYOcUwHdZSF/+9XTOJw8+fqr06uzIED3H+xTBhZpPJ7PGhopqKI0UkYnK3IkNXOZZZqAUCoFoqjdbmdZGi0Xm+t9RsHpxVUUhmmathqrWmtiMAShEEJLzRgjmJZ5VZcmKK4QAEDVpjDwQ4vjf9n1UNsTOOcYY/1fVDPUM26dNV7j/z+Q8Wma1ojCD/nE9YYhJa9lfYZh/OB9qM2E9Wjyw4BSP1SWZFprrQwEQe2pIRiazFgulwYlgee7toOg5lWhlBK81BjneVwURaPpm5Q1fGut3y2z5WIxC/ymY9JhmYyG155j8aKoMUhGqeW46xsr62tdioFv0M2VrgIwSfMwTiDCjuO5QhVKVEUJkCWFrEpNkQF0xYvcd4OdnR3FhcEohLrIsyRdFlmMMeQUJEnCLBMAYNt2o9GwLIsyZhjWcrk0bbfWJfwQSWlZFsKMMRL4HtZiOByYBNiOmWdkc33HD9x2p3l2dnp1dRWniWEZH378YX+1RahK0jlm1DCYgpUEAiEktICqghorCIBGACGMKMLYs1GWzdJ45DiO5wZmYGdZsZyPTMPybcMwjGgx5pwTIL3AsxkKXN8yDKC0lBJjRAlTGlZaQa0gUEryqihMajiuUzEhi2IZLfI8BxphBKAJgNZCaiUqpQQSquBFJUqICAZIaQGQHE2vIIS2bbdXekqp+PUijWPLoczArqKEehBiahiW5dTMFGm0MKaEEAgQAAhogjFBCBHJGS+w4kxKYZmewZwsLdfWNrZ2d84vbl6+fPnd0+cakKDRroScz0Ib806jVRZVw29Nx9fhYvpv/u3/hVfwv/6v/mW3bwXNW4MhmS9GWnOly7LIXc9i1GSsyIsqz5ZClAghwzBcz0ZQIiAFT7UoIeIIY0KhKkoAlK4qnmdClpRigxjYhjc3wyePn56fzxqBNVvMk7Sch9Fqf61hBUXZK8oUQngzuimqXEJdCA552V9d9yjZ2NoxLddoBmlR9na2b999OJ0np2dXZVHsbO12mp313qpvO1Cq23v7ZZ47jlXk2Ww6KctcSe5tdA2LOA2ngjIchWGcaIDaq82Lm9M4jpMk4mWuAOIaaQkYopsbq0IIinAQNCWXnVa30+xsrG998cXvh9dDRilCqCiyN1HkQFL2xpukNIcSKiC10gggqWqFDlYQaKgBBBBqDADQ0LYd3+3t7t7yvVaSFLHMKMKz6WUUJY7jbGyuE0IQwaaFkFQWN0TFtZYIAUbo2lp/Y22FUdxuNIsiS9N4c3MTcggh1lpbltNb7WOMmWVKoAteKaUgwYzRSoi8KgteUUo1gkKrGl3w3CZQYjqdjqcLANHm7l7D9zCEw/H46mZ4dn7RW+kv5/Gz58eMsZXVtb/4539ZlmUcx0+ePAnns41+X1XF6clrAsGD+3fvH+5trvfWVxrvPDiklJ5fnI3H4XwZvjo5PTu/irLcdANCTYQNLSqAEMEMmtpzRR03p4HkFddS1eWQpF4mtFZKiYrX6y/UQAlZc7Faa8ogJlJwAZFCWE0mo2fPnvq2ubW19dZbb32n1Gg0poR5nq81+PzzL7ot90c/+oRQdz4YlYKbtiG16vf7q+ubBeeT8WI4mgZ+03EpgOTo6KisEkrxcDa5OD9eW+v9q7/6V18/fXV8dAY1sywXQgiA9nzr9Ozs/OLKazUajUar0YyWGWFWHC2F0MswthxfKoAx9mwHAEtpCZTEBFJKteRSl9ubG61mw7XNyeBqOh5dXw+aQVBkebvd7HZWG602QDhOinmYNFlECQw8xw3cW/v7n378kePZf/3Xf/31N19KKTut9vbebqvThphOp9M0jaWGi8UCQfPgYKsRNEfDSVEU6+ubUiPb8fb3907Oz66vBr/53W9///VXECO2splVlaak2+tubGxoCJbLZVbkpmNCAjHGGiogtMaaQkg0DkriMXb39v07B/suJecnx7Obq2g+z7Ks2WwGgddqNRBJeJkd7O80261kllCKO52WY5sYY8NgaZYsw3mj0xiHo+miWF1f++kffxy0mhdXl9PFlZDx2eXLlU7Psq2vnn7hWcbG/qaU2auL6/W9za3u2ng8nYdCaI5RUys/jLO0yJlJTe4hppUCBGPbtjorXUJQHC4P9vc4L09OX5VZHkVRlYOyKPwgIIaJMYaI1LXFWZJbpskIyfK80oIgbDJan/VrYV09JfwgyK+J9lp4WNsQ6puWGiEgJfnhXvUe32yu1dhADTyUZVkPDYxAYLD/sgeynjw450BJqBVBEGJSPymE0LEYhJAxs06SllJSgHieaV7ZzDAJdU2DAK0UgBpgDZJ0WTtWqjLWWqdxBCHsdhpJPF9f6d6/e9htehSqJIosimzbNrx1iCFmuNVquV4AZFlWfDINVZY1vUArUJSlhMsiSyoAuBDIs5UAVcEZMymiFax6vd7dw9tYKSG5kiWhnTtkF2GlBJdSTodR/a6maRZFUVVVeZ5jTEejkeV4WZbleQ4Q5FzUBdydru86tm0a4+HN0dGRSaDlOF2kqcmm06lScrpcAIq7/ZVKybOrSzEpb8aXhUgNbFComAExQJUo6zgAhzohRwABAABJREFUCQHQSEOEIFJYQIxbtu87LtBQa1iVqixzCLFvNW3bvXXrYH19vcjL6+vr6XRalPngahAnejaZFmkhgdYAUUNDQpEGoiqriouyyqPEtu3V3qprGsKz8yIRgiulAVAacKl0keVVVQSeh6SWgGukIcMQa8BlHZNMCGEu8Rs2AKDRdjXiTmAQwnILGQVWCmBKGUOEYACAaTdqGSbBDCGsNVQSaA0Jw27g4aoEWZq6nhOG08HNTa/fzZP0/Pz0+OhVsgx39g7379wdTxaD66EqWu//5E/Oz0+rqhByvrG5fj36TZ4Xj5+jw4O7nfZaSzBM2kDTJC3zdMFhadt2QD1Kc62VKBMBNNasADkmmmCAgACwxFACKJWWUnIheRwv5/N5WWSOY6/0u/3e6tnJpajk3k7v6nL83TffPnr0tslouxX41C3L4OIK3IxulnG4vbvjNoJC8iRN4zQJ2h1I2WA63tre3W53V9Y2Hn/7mGHyj37+p4PLi+lwwuPUMRyLGLPJ3Dbs06PXvX7XMAyldVbxl19/043Xl1niNoNOr204Nry6HI4nYTQjhCTZsqgKCBVkCHLAlSg514BDLQzTRFADBH3HtSzLdZwH9x7+4pe/TtM8LyvTdjY3V5Mkw0TzggMAhFL1BY0xBggihAigP1APUkouRX0OkEo5juP7fqvVsixLKaWlZqZxcXE5m82riju2GzRyBXS703I8b7GohoObNFp2uq3tzdXdrS3HovEyfPjo7tOnT69vrlvtRiNoYUwxpo4doK5Ro/fgD71z9VpWLyK1JEr/IUPeNE0DEcYYQEQotb2743sOpVQL0VvfAI+fXE+myzhFzArcwHJdx3JOXh1dXFy8ePH8+NVRf6V7eLgvs0Qk6fsfv//jH3/abzcarjWdpu12mxmUGXRjF5ydXpxf3QynM0TNjtdQgFTxDBNTKaDUD/8kREhDIBkhNZLLCEEI1eHNQGlGaN0ziTGWnOdC1FS6JiLL07ra0TRZHscvXz43MHj/vfcOb+9bBn327MV4PIYQaK2Gw+nf/91vvv7q8c/++Mf7B9tX12dqXu7u79q2CQD6i3/xZ4LDb75++qtffvaLX365u7O1vbs1SeYVL0KeDKZj6FkxgseTyZOLS6CpbToAAIrhSr+15IVgGLnWdHAFRDUeXEHEwmXGbA9p0fSc+WIJtJQCAKgg1BgCKIEUXCGZllUSR6LMFgQMr68uz87yZHn/7gEzqWlbGiK/0SDMnIWLSsu0qPI0Wt/avHf//vsffQiRzvLkm2+/Gg6HaZrOp9O7d+/YnuvZzvPnzx8/fryxsdlstnu9VS3k2dlFHMf7+/vvvvP+IlrmZVFVVVWJJEkGg8HJyYkX+B+8+1El+CJaYkKjJF4ul1VVMItprQFQCCKCNYCg9ksDoB2JH9w+/Md//McGpS+fPR1f3TCKeZk/efLk4OCgv7a2u7tbSvXv/sO/xxgfHh5MB9PJZDJbzJOE7O/v93o9qdXVzeWvf/EPhmsyixVl3GxapgUtS3/wwf3h4G8Gg4gZ2LYCXmXNtm0abDScBz0v6LWtoBmeXj9+fhp4abe3LrSX8kxoRYAuOZe5LkVJKVYmCyczP3Ad224GjWW0qJ2HYRi5LKr3acO0AQCEmUEQIICSOBZlBTVAAEghJdQKCok4sdgPU8J/eX15nvdD5HA9atTb/M7Wdh0RWMvma2i91hj+wFwURZGmaT37Yq0AAHWIb22DrN004g+3H560Pm3XhoJ6bqjFmIQQBEGz0XAdBwDAqIkxrk/khJAMCkaQqDCEUFSlEtK0mGtZTc/3LLMd+Dtra03XOTk5WSwWhBAjWE/SdLqYHL06Pj09KfK0TCMHkQ/efuuth2+v9lYfP378+uQ4zJL+xma71cwxNgwDKo0BJAgblHWaLd/3gyBQUiAgLJt5jg2giKIwDMP6pSVJEsdJ7cjIi4IQ5vs+QIQxVlUVNZhtkzzPMcZKijzPgeZJElkmNS220msXmRVFYRgvkywOo0Wj0ej1V+bz+W8//22JojCcE4al5lGaW7aBCCrK2LBMBQGAUCklAYQKYVlpgOdjOZlMypJvbuw8fPDO+voW0CRLS4RIu93e2tgxTOoYr6L5l1nJG37r5mYgStFwPQVBUuRaaYowZjBPgZZSCF7KpEwTCoHnuFmaMoqVQaWUlBJKCUKyxFppmZWp0ISLEgIFsQIIVJorqTAgCuhpNMtEYZomoLqx0my1W+12tyiKNMnysqrHU8GFlLJIC9M0TYosZpqmjZFRq7aJrKBtNbJ0STAzmJVlWZYlZen9+je/SLJsbb1n2lZexK9fvZwtwtHommUtE3S7vjw5e+4w+2c/+dFkep2m6euT77jIVrpTBB0EfccKWtQwSDdOTxtBi1JaG2yqohBCGAZVuiIAYagR0BJqKbngsgBKK5UkyWKxCGfTJI1c12m1Wq1mZ6W9Mp/Mer3+7uaGVihw3GiZxPOZomQyG4+mo7OzM8dzV1f7vdX+9Wg0GA4hhKLi48Ho9OLStvyG376+uLYNoku4vb621mqpqrSoMbi8ePn9i7cePei0utEyi+IzQHElK2oYFYCj8TwvhBu0Hr3j9vrdVOQ3k8F4dtNotyqQK1ghBDHUVJKqEhxwz2K5VhQjUZVaQIxYlqThYmmaNgLYsqwoitfX1zc211+9PsJQIsK01qi+7CGQSgEJhFZvEEUIIEYQaAJJvS4QXUpZ5sWy4hnCwDAMAYQU2mCWkqjIK0wNRi2/4d25fbe/vrZcVuPRaDYeQKAci8XxYrkoIVDj8eDVq+dxnN6+fbvZbNuWW1V8uYxkxWvIhyKMIZJSSsE1klVelFnOi5IiXOVFVVVpFGOMC4gMKUslrSAwKJWSR2kCAFjbu1UhvMyrv/3bv8+5+vM//+dAo9evX1+dfxOFIaqKbmBjng/Pjz3T+Ff/4s/eefetg4ODIkuyzLq5uZktQkRIu91e21qfLaOciyjNvIapIRVCa4Udz8uLggvBq7IsS8YIRhhjbTpeDTZqqeolmHPu2k7N9TJCpJRFUUgu6nAnAECWlhRDhJBjOwYis+no7Oxstd9755137hzsv/XWwy+//PLx02dxHGMCojg5v7gYjK7efuf+7fv7e7c2NKiOTk43NjYAUkKCP/qTTx88ethd+f/eXE8uL6+thpvPM4HAzuF+e6WTiGqe5cixIWASEiHEMo34uCqKZVqJ8WKCZP7+23fD2RBTexmlaaEm0wVWhcOAEDzP84oXBEHDpBgjrUSuJAQgXk7mVcHLYjS4CRczz6FXVxfNZjPLrKwsLNswLJZk2nJM17ZMRj/96MOPf/Tp+ubGZ198ETSDO3duX11dJokuirgeE0vBB6Ph5fXVYDD+9NNPd3Z2qrxQSh0eHu7t7TmuNQ2nFxdnURoBjLa21+4/uH1xczadzS4uLizL2tne5pzneZZEsZScYKCVenP+gtrAuM5uV0pt9td6ftN3/CSOb66H8+m0DgJf29w7uxxMlsn9d9/fO9iXEL06PrJ89/bhgxKALE4ur2+Y7RmWbVqu5zZevjy6++BuVRWv06VQ3LRZkif72+vv3O3vrTlpUqVJebC3u7KyEoZhf31f6Wptr40RECR/df68EcwVBRKXpm/WtDeiQEqOtAIAi4ovF2GaJP1+TyngmM7O5t6IjaIok1KGs3mWZYyaRVVSanhBQAidDCdaa6AghJBiUoNbSsr6iP9GH/MH7qBuVyGE1FURtVuh/ptajVg7F+pUg3qnr+uef2imfpNVjLGBUX391rPFDw8o/3CrGYcaipBSIqDqGgiMMTRNIURVVWWWUxeblGJM6nSm+nEoMQLPBgBQYlgmwxincaSEmE8n3VZ7Op796u9/1W4G8TKaT2ZllkopZxezQvDFMsx4ThguyyKPSyLA4a0Dw3Qdy2WWKYTiVZUmERgp1ECGYQghgeamxRzLaDQakgslJQBASj2fhJPRGENdL5mEEAUBY6zRaNTmqWUUmaZNCMmK6ofXW8s5TdOMoijLEqBlmSerdw+2djYtkw7yGABNGFYZIAZRSBdVnvP8anjltphhmX7DT/IkSSKpJUEIESh0BQHUAEgA3yzdEiONeCG7/RXb8qSAv//q29/87utmo9ftrP7kJz9bW13v9HpZnlLD8xudxTK+vB55vmObjmGaCuh5GFWCI0qkAlmSao40wggZGCKkJC9zURVFWVSVkFJCqBEGGGPKsFRUaakAVEBBLCFDmCAMqBbVkqcGMmSlF0XqOA4hFEI8SxZOM5AYMs+irqW1lkLXJmFRGSYzGKEEYagBApBggjEhUmrJpZbSte00Xk5GA9ukgufHr58FjVav01RAD168vry64Uprxaej8a9/8ZuHDx+23K2Nldt765+ud3Ol4dHR66yoxgPVblmu53LOIZRewFx/zfd9qHSBqVayoqwqMoSAVBhBqYQoRZHncV6kRZELUWkIlstFlqac8yxNsyxrtUarq2uMGWnMUQe9/87b8TJxKDWbjSKOv331ut/vH9w+HE/Hk1l0enp65/69tfX133/9tVTAsn3P841bdr/V05Waz6cnz590W+1HDx9Sgk5Pr+ejie95P/vxT6IoWl/fevTWu9+/fDqcTdorPcOxd+88SKsoTdOrm0Gn38eGWZRlyQtNGISKGRgTgjEmCDguLUtWVZXNsOQVZVgJDTQEUg+Hw8UkqirR8H3TtheLhVQijiOgVJ5n1HalUgBpqQEQSmspgYYKai01BAAACAEiEAEMAAYAUM2V4kUZL5fzJIkc0+El51oixBgzMTaURJQartMAAE2n8yyTnue41jaCsirS89NXRZZsbaw/ffbk1dELBMnJyUm7tVJyBSGeTRcMKsdxGo0Gw0ToN2XthJA8TbM4KdLMpEwLyYsyT1KMcY6wTtKizAyDIpNFSR4lse/7uZSdtbU//fM/nycJBmhzZ/fZ98+PT88whjKLbAwa/a5tm2WRKSUbjkUhFFU1GAyEAgJgKGFVFpUOv33x8vmzF0+ePQ2XCTK8q5ubvNAKAK/RQkgQgiAAACjLZIwRhDUAxDCM2mldm5QoJiYzeFnWxzigNKpP5TUlUQIMgenaQnOtoO/7WRqlaTqZTMbjIQKy1Q4+/Oi93lp3sVgcHx8vxrntOaPx5bdPvn3wzsG9+7fTPMzK8PmLx/21dYIt27Z7ayv/8n/6X//+i283Lq5fHB+b3bbtWkGzESfpq2fPL8/Py0KVhUQAG4RWeUJEblm67RpMycX4/N33P/zGo8QwKJTx4noxSZaLpNvxCAQm4VgLjICBAcVIAr62vWUQ2ltpCl5CLU0DdLteK2jEydwPnKSIy4IThkyHrttrP//5z3utHpfipz/9qR14p6cnXFSWZf3jf/yPr66uEHpFKX3w6GG707m+uprMZ8RgIq9OT48ZIwd7tz755KNGvz84Pf0f/of/fjybQozeDd73/UaSRYtwijHIiyReJof7t9vN1tOnT8fDUbyMTItBABECWGusNYKAUWqYtN4IPdOdTebDm5EQYjoNX78+s2373n3vZjS9uLzur/efPH8pCe6trf31f/qP1DQXcWUYhuc40yiCF1eEGc2gAQBqBO1+ty9EMVkMRzeXhCGpOIPqnYP7puWUhTg+PcPU2NrZNEzit+wvv/5sbdsiBL+Lt66vRqYFzGbo9uMi7VBMINRclhBCw6AIwTLPCUKi5AwbcZhIKdutFYM5jh3kYTaZTaEGQog8zTJYVFXFK7FcxiYzbMMm6I1QAAKglOJFUR/ff0g4qE/8aZrWYYiEENM0bdumlCqlLs5PXdetD1o1eF7PBJ7n/RDJjDCA6I2sr2FZ9SBSIwR5kWVa15v9D9KENwkWVcU5N5lRXwj1wIExrvuTlmFlMMswDIkxIRQZBiRAK4EQhhBSTBm1GCW6UqXOMMCW4czjybMnz4DSVZFBDUzTLMv8dBwS0+BAIQM7tuf7fuVkWMokzcfTmQFRjWFoIWfjyWQy6t+1PbcPtEJYm4wZzGQED26upJQGoRqBQuWilIAgy3IoJXnGuZL121IHTSZJkiQZhDAvOQAgSRJDCgBQGIa2bROCeFWE4VwJXh/GFovFcHRzfX2VJFG4nFPGRBLHRWbbtuU6UhaVwMxEpmlWoqy7rahppGkMMQIYAQCkUlpDCCWE0CR6mWSzxUgrUpU6S8XV9TUAj3/567//+ONP33vvvV6vs77Vvf/ov7q+vv7uu28uT64554gQqZRtMgYMhFBa5M2Gr4Ssk7IMSg3KtNYYQUwQ5FpKnmW8xnQxRpZlSMkxQVxpgDRh2DAYwEpKrGyLYFZPnEHQ0FpXBQeIDmaTGlrCGBPMCCHENimEPAEYYSGqKOZKQoxpXcJJLMeRZa4VxIRcn1yfn582mk4UL2zbCpezebwsKuX7bn+tISG27evw9eVvf/PLd9569+7t9zvtdSo3KTUdr7m68Y+Pn3w3DSedZtuw4fXNWRTPXNde629JqcssL/IUY6yFEBUHQEAEhCwrnidZHEXzZbRI06Qsc81gEsUQQtc2NQTLOL65uWk3W+PhZHtz9e7h7dWVtZuL39vUfO+dd85OL47OF8w0Go0gaDYqwWuOKi+Ks+MT222srlrtZufth9ueF8wXy0iHxy9ffBsuXz39/sN33ttc7S8Gw7/5d//hq8+++JM/+RO0f+vBW2/nSlEv2D3cvxkNocVRSYVCANKSyyRNkyThinuWR01ENFMKAa0RglBhDQQXaj6bh4tIcoUQpdCo8vL09DRLy729/QcPHuVlWWtGMAJb2xtllV8u0ze8I5CYQIAJQwghpIB+s5MpCeR/liBRxgkhCKmKZ0kS+Y6PQa17KAhmlukAjIRQSZJ9/+TF6cV5WYn1tf7m+sq9u4erB9sUq6uLE9ez/X7v6uoqibOrq6te93xtA6yvbTOWhsMbqLRjWiXCtWC4TmIBUikutJAME8e0tJAUYaDB5fWNlFwC3et1TFdHRbbMc8t17Ia/mIW99dXN7e3Tk/Onz18+efIEQLycDrvd9s7mludbjm2F4Xw0GlGseVkYJmWMJVlKqCEAuByMjo5Pz05fzhbhMko2tjZW13eWy3wZhbbjC6EgxKbJKLG1dk2DQqQQhpPJG8+31rrI8prxfcNHqDfahRpdqJfvIq8Ihtpx8rwEQlg0QIhURfrixbPrm4vlfLa23v/0008fPboHIaQUnrF5ksT3H/x8pd+YzSbffPfl4e3doOGcXVYY6yQJf/P5r7c2Dw5u3Ts/P/71r3/7ybufNNsNhEGaZaPLy5k94nFS5cq1G1BBx7BLWSFZuYhihEFeXF+cfPLxe0DkjU5ra2Ntd2frxwI22h3fa1iOY5oGAABBTQiAWldVCalRVdX6al/yyvecKFwkSSKqotlsrq6uCgWkDCnDhOBOp/PTP/5py2nleb6+vv7577/4u1/8w50H9yte9Pv97e3t4XCIEFpdXfU8b75YnJ2eL5fL9ZU+QshxHNd1B8Obp8++HwwGw9GgEoIwmheJI22EgO+7zVbw7eNw9PjJ2w8f+Y4rK15kmcmYZztVmVOMMUQIaoKJaVDLNDCEQojBYNhut8uyWi6Xo/FkvkiiODOds739/ekyDpP83/z7/zCL4z/90z9h/4//+zwMl8VRr9Nd6/eXaW4a2fVgfHZ6fvzy5Uq/ffvw4Pad3f5Kp0wjoStMTJcyv7nuOM7G1rbDvvwf/+2/mU1Ge4ebw1mxiK6y0u02Ow/fXfV7SiuUZ1NBoqK0JKVKgqosGWOMeELILMnqViRGjOlkXoci9Lqrm9t74/Mry7K4FAiSuWkKpTHGSZwGQcAIJZCISvKKV0oRRBBC2GI/IP9vloIaCTCMOg8xz3OlVC1QgBC6jlWnFNe9d7X9gRBSe+LhH+bdHySQYRjWigfGmP4vSiNrA8WbWRm8EVBLKSFQAOia1UIAGpQZlCpmJElSsyOCcymE5KLMCwAAV9owjDzL8ryUvEqiCENYZCVQcKWz0mk0eVEmcYQAdBxruVxW1IEEx2Veqso0TdtzaattE7qzs7e2toGUbA6u6jpH33PbvTZk2PPtPEMQIoIRRjBJksuLi6bfxCaEGigFkriYTsej4XA2m7Q6Da5kr9dzXa/uEI7jGEKMEAKIvPnUTMM07TrF3DRZniWLxSzP0+VyeXNzswxnw8GgKIo4joejkRf4WZbNwmpto6G11rhSSjSbgR94WuuiqiTgCqpKcKwxBPgPucZaQwQhFCAGBFFCIcAAq1JwLrVSUCP9uy9+8f3zr/f2dt55962Dg33DoPt3Ns6OLtI05TV9ATQ1LA2BEKLT6aRpWkdlY4iUUqKsqjxr9bppmsZxXOSllFxrSggBCJelBEgrJYBWSkmEAaUYEog9M8/ztMwdjBWChBiOZQdeI03zoijSNC9LrrVmmNW5yTLRCCElQVUJwSVjZhA0PTcgWitAsO35osqur8btxhrneZrlQDKCYVoWo/FEY8IRiJLU8qzuO7cWi+Wvv/v7P/un/9xb3ZhVZxCQSQpXuv2gB5jnQlgoCdutBqMwTdN4ckMIYpRaEPI8q8oSKIEQ0kILpfJcLqJyPM8WyzytuFJIFUWeCoswXoo0TBCvUjo/Fl91W417O22moldPfv/Fr7787/7X/xvH2rpzuD0L6Wg0SmTuYJ+b6uX3p3cPh41OO8+U37QfvPNea6WrIJyW4bPLF8so4iV5+epqb/ceMb1Sqt3bB61e+/PPP399eTyNZ2+/+97rVy9KCQzNbm/d5pwTDxVlluQp0kArZRDaa7YDzzexoaWqZMVLASFGAINc6VikPIvyJabk3u1HokRff/7dZBwSaKZh9fDOu59//vne6v7Hn36QZvH14Pr21q34+DLLMgAxMGhRcgQBxKTi3LRYURRaS0ax0FIWOdTSNOjD9du+7weBZ9s24nGRjFwvgKQScClganikEfQWYRaGcbgIz86WSMW6UPObec9fvbV+50cf/pNXztOqzE1Kb63dS5PoyXff/D765Ycfvt+zWTkf5dV4cnxciu3trd0oWSZJYpp2lEiESBQvsyxrtuyLy0wI0Wq3hsOhidVkPmeEwsQ+uxp8/+RJt9X60eHDDrIBTJ7+9nOdpu/e3V/b7L/31mG310vgkCGzLIVjN2fjZRA0x+PxeDxUQBwPrkpZybz0/PZ0Hn/5zdPPvviCOJbBrNbqtmE4QiPMaNDwMCJQVYpXyzivs+SYQWrxhNYyjsJZVZUFhxCapkkxFkIwxnhZSS21BgQiBFFNCVu2UkrN5je2bU7mM4U1tpz923fCMDy5PN/Y2F7bvzdYKnqzvHt4+8c//8t7D88uL87m01mZJp9+/MF0MPz3X//Hfm/l/b2PNjc3Ty4uFzfP0uns6/mvOV8wqzr6/utWq/HWW2/d2tvaCtz5fNr+6P7oZrC6ttLr9d4I0KjpBv76+rpSKuaZZv3/7f/u/3B8dMwwswyzKKqL04tGu1WWJUQIM6q1Pj492drZTuKxIfl6t5ctsqqSBgyQ7oiKuq4rBFoubSn5Yg6KnG6uH9za32sFPYIFT/OnL75bXeu+987D1ycnzcBZ63V/+qOPtRJRFFGK4zT64puv5unS8qxHD3/6/vvvWiaL06XFcFVKislHH7y/WM7Pzs7OXh+v91epwTb7m7/9h89ubexFBrm4eH779m1MeZzMVlZXNdRRFjeDwDYM1zYZwbZFLYOVRZbkqUHN2wd3siRP4wwoGC3B+qZbCdDf2P3Nl99KBBCA2u389vmZtboPPffybKYMnKl0XuDl1Wzz4F6rY/z6s99nVfV//b/96//uf/nfPnhwn6Am1lLwKg2x46VBo71YnLue/LM/+yhoe/Nw2uuvzydXIsHj82Rze2un15iHkUjnSIJ8IaBNTNsyLAsQLDRRQGvLloYZx8t5Gjf7vdn1pRLSj6JHjx7t7Ow5jvf8+fPJZOx7jTr9t4zTWj/IAapHfAAAV5UWkCFFDWY59hv7rnhTUR3FWT3KAoCkgnlcSJkCAPr3uqu91fX1da11HMc1W6GVhuo/WyIxwAooLXSVVxiAmtSoihJjSOrY5px7rltVlShyWfsqlbIB9BjOyhwAgCDECEIFykxACCFAFGpRxKJA9VhTQQgA0lrbhpmkS4RQkcs68tkwTUQgY0xoFYZhhbHfbUMIkyRRlLrLOMuylutqrVtB486dO4bJ5vN5UuSprBhju48eHr73zmg0evbs2Xg8zuaTl2HYbbW73a6oUsUzAThRRaPBprNFp9cvYDW4WkZAxBYJGSmShcHMrmVMs9x1KC+k5feEUATh7e3t999/fz6dKSU+++wzDPEnH32yd7j6b/7Hfzufz6tMhXP+8tlwPhtVPNOgihMuFR0NEg0JQf7ohgCAMxhBCKdhstK3hAAVl71eEyKZxRwziLBWSkCkKMWEYKAVNdyiqPI8kyJFCBsuZjaWQmnNKTGKcvb0Zfj86HvO+e7urZ///Ocf/Ozd3/3udzfHp2/4oyqhlLqMwDJ7eHALIcRL4TiObdtnpxeDq0uirZVud6OxMV+EYZzwVKUxrwS3XMv3fdt1uagICwzH12VepLEGFDHqUo9ixpVBiUsZqySuJEKYWQ5kplBaACU1yDiAdW6YFFwrThCggKo8LiUhRVGYjJZVuZhNKKVSCEItxzMxI+eXF1oqkxkKQAKRTQ1MKEPcMhEEYjIe+m4z8FuO3RAAhGEohFBSIwwBQAa2iE8825sOzoWogNYmZZZlUUqRglLKsqoKXqR5lud5HQqGNJBaGyblJdJAQoRs2zSAASFYLBaKl41G4+T4rL+y2WxZtmOUVYYRazTtvDA3N7apgf7T3/2t7ZqGRdvt1qN33nrv/Q9WtzbCZawxyvM8WiaLefjku++2NtY3N9aicMFQw6Ck2Wy+/+57Nzc3NVjt+37J5WI+3dm91Wm1oKWXy6WWyrPcTtAxCBVKIqSAxmXFhdCGaWFMtQQNZrY7vbPLBEHbtQKMLETxSn8dCgNqgjFdzJfNZjNo+qurq5Rt7OztnJ2dXaT5fB5mZSGVhkgiiCFSCEoluEGgxkRLznlBoXJdu9lsvnNvf2try7Ss+XzOlWx5FiZgvphSqBu+yRjjVXZ9eTqZLBAxfN9QeQxl8slHH+5urRT5/PDWgyJdyZL0669+v4gntmkQC12NLppn3gefvDuYXF0PrhpBazyaxlEKIczzUmvtur5lWeFyES1jSimlRg1UFkXRaDRWVlYMygLXOz0+IQQrpYbDIeflyclJkiSPHj2wLMt07CzLrq+vu1ue7wdJXEWLbBmmq92N3Z1bzWYzzaOzy1NmGr12FxB2dHx2eXXuWNbdt99KkjRaJmXJKw59P+j1+kqCJEmKosyyrO7DxQRKKTkvmWERjDGihoHekM0I1fwugpAxmmXZfLmAEDabzW63M8smFGHNUO2obrVanU5nf39vpdP9+3/42zIvhJJVmnzxxRfPnz93XRfoFABADTNLk4urgefYttv4N3/971qt1l/91V/df/Rwd/dWnBfHJ2cb/Y1m0EonWTNo3Llz7+DgFsUEIi0rPp2OTcNot9u1861mUgkh0+n0+1cvzk7OOedFVnz5+Zenp+dFXsVxnCSp6zcWy1BpWPDKbzcXi0Wr3b27t1FV1dXldZ6XG+tbruvHccorkWVZo92ijCAEPM8JGo1Gq52XVTwf1Kcuiund23dM06SYFFlOEOo2Wof7B/2VlS8+//z3v/0scL1333339v1bkKpJOAnnY8tkQlSj2fTZ0fNGo7FYRj5CN+NJUVWD0aRS2rLdCgJRVjeXVyYzd3d34ygtirQVtAzDWFtbX+l0qjKPF9MiK2zbaTfaK8w1GRneXBVF0em0bh8WCsDpaJiny1YzWMQRhNJmVBS5rMrpTaQ04lUe64oQRCHCGPtN37KsPM+qsvju26eeF5xfXAvBbcdcWVkZjaKgUWoERIU77fW7D+7eDK/yMtvevIcATpIqDmGcppfXUwTxev+gyoz5fJ5EcdBsMMMoyhIg3fDcKFqWec45l0IURREvo/FgyDn3LQ9j3Gy1PL/RarUQQtQwalVyEmecc6MOOyoqhKDt2JzzLEs453VkAsKoLKosy6qqSNM8SRKttWEYhLA61O/o+FgoxaV0XRdCCCCsOK8TmfQfgIE6M0FrjZQiiteCA4AhxhhhDAGAEBZlKYWokcmafVAQQPWGlaj1lRBCrd+4Kngp3jAmEP9g5tRaS4Ity4QQLhYLzstOp9XqtPM839nZefbixfn5aRiGiOCaQ6GU9vrd4XBoUIYQyPP88upCa312/Hptvf/06dO33np4cnL85PHjNE2DwCMYlpliRLW6vffe+1BrfX15vVgspIZf/P4bLuVoFoZJnFXV9vbOpobTxbxaTiHElFoP7t7mlVAKZFnGS2HbdpIkv/31r9qdJiHYD6w7tz/Y3V579vjry9PXUPKVToMgpVX10UcffPTRO3mZ/Po3v/jFL385ms4wMhSgeSbLqvJ7fpZlVaGiRZamCcTAYMVsNgFQMkYABkoLADSlihANoAoVp5Qahm+YFEIshOBCaiUsy4YQA4BrJL8oCgj1ZDKSSmsIfN9FiNRyCoyw1opLHYaLsiwRwkHT7/d7hJC8SMfXC4SBFAWvcgQkBBIC6bmGFzhBy9PQ4YoHTd92DJICjSU0Sa1NwZAAgN6wXQBSSoGqv0Q/ZHpIrbVWSAGtEYSUYAABRJWWshSEEQoNI18uzs/Pq6q6vDovy7TTbTi+W1UVQajp+UXJmWE5zAYAmSwDyjAZjMLJdOw5po1szQguixIjSggEAGihAYQYGcQwTdOMlvlyuaxjfCxmJHkax3HJq6LKkzzL8rzgRSW4kEJqlWYxF5XWEHJOFKAGg1DFUTwZXa32ukevr0zTOTjYHY2ulUbra9tZvizKuNP3bW//y28/Z5adlUmSL7d2Nnf2dqaLeRguNnd2EUIGJuF0ZhDk2dbl+VkWJ289euA5VpIklJLDO7cXi3Bwfd3vrbheEMZRGi0nw8H69jrPKiiAKnWVCck1kKisZJollRSu63rNllJqPpkXRWFZlh+sNnxoUJMyDxK8sb5jsSAJM8dyuRSr62u+79q2adpGk7Yvr68VElznQlUIEWwgoCGEQCLdafhpEiVRJHnVduz+yvb25nqn09n1jNW2SyjFFU0LZehKC0VUNR/dLKfDNBcQMQxK0wCUAkGxY1iu6zzY3z5//ex6ePXw3q2t7dXJZDKeD7Iycfye5Ztn16fLLLIalqLq8uLaveunaRrHqed5dWBLp6MWi0UURTXgWZuvbNt0XRvYfqvVEhUPfK+/urKyssIImk7Hi8Xs+fOnjUbj7v07v/vd76aLqdaamcbl/+vc8wJGHYs5GDECjFv7u/1u//mryUq3Swxydn52M54madlqegYle3s7R0fHi8UiSSLOJcHU9/2iqOI4DsPFbDYra4jYIDVCYCtkMoOxN126hBCoNeelYdA0TqSUACjTYoQQiHS4nEtdtzdLSrFlWY1ms96/p4t5EATDPH959ApCqP5gxZ4nk1arsdrteZY1i2O/EfRX188uxqen4+HN/35tY/Od99772R//8c9+9JMP3vvINE2beXWAK0IoDMPFYhEm0WSRlOXsu+9ffP/99+PxmGLoOI5pmuPxuJDF7du3hZCz2ez758+ajbbU8PnXrzyfCVLEXC/CSCGcgvT8Or9tq0ZvbTAYSMLshruytbO1tR2G4enp6auL00SUhGDLsrzA73RXTMsbjac2kLwonj99enN5/ejRIyBUnEWJu3SYubG+7vv++euTb774cnNj4+OPP7Ztm+vs9HIYR2FRFK5jdzodqcVnn3/eaLcIYZ00j5Iyzou8KtOiVEAThC/PzqeTSa+7YmLjbHruew3LsLe2NwxCp+PpbDqRvFzpdvrd1W63i6Ioy+LxeAAA8Byz3WzMwoUSZbyY9xr+bDqUJbQwrKRY77Ymk0lnpevaznIREgrKNJtOR6vddlVVZVkFrndxNRhNFjfDeVUVnuclqfQIJGywWM4vLk/avdbG9j5jQZzJbncnTnPbMQ723x4MJq+PIl4p39qV1amsSqUkEBxrBqUoiopXldIaAEAJoZSazMgJWc4XFxcXntf0PK9OQSC54bm+12woBG3b1Vq32+3V1fUkSb799tujo6PpcrG1ui6EEKJSSlCKIcRC8ixPIYSMEcexhFAIIUpxLUUEkk+n07qOoY4KME2zNlTXu3uNH9SghZRSqArXSALGiGAEka7Lr4sKaK2UrHWMEACtNVdSAwQAgpDWLoxa71izEqAWSQEBIfxhXKh4oVE9XgBCECHINBkh6MWLZ0++f3J1dUUIoeyNP1NrmS5DhECjFVCCLy8vr68vu612b6UzHg+/+erLg/3dq8vzo5cvbt269Zd/8efdbveXX35lMLPX6zHTaniN9bWtV69eD8ZfMNsp4mQ0W8zmc6FBUco0z7jUjhaNRiNdFq+ev261Ojs7O28/2vzum2+XywXBej4daZVdXV0gDA9vbaTp7MWTr6bDc4s5tmlVSVymcbcV7GxuHJ+/rpkagqnjBhIYWhdCldPhWAgQBDYQrEi01/A8a2XKs9kiZEwAjJQCAGhCAMZaAUCAoBQyhiAUGkiEEGOEMbMooFICIuC4zHJsgGBR5eeXJ7NlvExiRDCEQHCFEFJQccEJYUopoTgCuuRlJSpEgBc44Ti2KF4sw2U4hRiVJS8Ft53uo4eHQaelMMiKlBoEEmjnpl/6AoH6MwVKaw21gkoprRQACAKMEHmDw6J6TlC8tt9hWH+HtNKci5yXBJsGQLCsqlKKZrfz/PXT0XhAbExtw3Gc1UYANIyipN1dcV2fc5nEp0M8MQ2HAFllWR4vDWRadgNBiCFCkEgpNdAAagCgUtpzXF4VcRwXRSGlDCFcRuF0PivLspIVF0JBADGov/RIAaEEY5RBokuulIQQKqXzvGw3O2VZttssisIH99/97PPPXe/oww8+dW1jYWCMQc7T/cNdZjuGTU3PNC17ES3G4zFlRpEms9GEIWhRcnBrO0mWl+ecUjqbzZ4+vbm+vm40Gj/5yU+m8/lwPP700x/7jeDm5qaqKlFlg8trBTSmSJQinC64FlrAqtJFxTHDCDOpYFnxXFSVEkBU2bJ0bJcgmmbCt+xev0GwJYqr1fV+o+kHQcBMWpZ5kmewgEIIxjClSCkEEYIcag0oJSaBj+7fPj89GYvC9zpvP7h/9/ZhM/CUkF6x4MkiV9rEmNqW4AWl1nqnNby+iWbj47Mr2wlsN1hfDepj95a7enBwACrxd//xP51fXf7P/2f/i82DfThfjKeL4Xg0GC+WizCrdHd1a74st7fvXLy+dl2/3W7bto0QGo0mQiilVH3WsSxLaRknWZoh0zJ6vV7g2ErJ2WxKMcIYGgbFEGZZNpmMXr9+vb29rbV8+vTJeDZdW1tzPFdW+uz4nGGr014t8+r06DwInHaveefu3rvvvw0ZGA6Hi8kIM+ej996ZzRdS8SQNEdKdbjNaplIJzksh+GQ6iqM0y7I6Zg5jXBdACSEkJvUvAQAayDr6jlIqZFWWZX0KMwyLUpwkUbvf5pzXNVTNZnNtbc1xnBfPn0ZR1GwGhmVlWYYJKXg5GA7SNKceKaRK0vzu/i1F2GAyX2k2V9dXeFakaXF1cRV4AcNGyWWSpb7vu41OWXAhRFGVSZLFcRxFURzHYRhurq8Ry13fsossF7wizOx0ernIzs8vvn/y7Gc/+5lpeYPB4Isvv9o/XJeQ5CVv9/qZQH6rM5jMd/Z3P/z0J8cXr6fTqQRkfW1jd39/f39/MpmEcYQISvOUUup4NrMty/GoaXClLdPIk7RK83FeHr16xYVwHCfPMohQYLtQgySOe93uw/uP7ty7e3p6ejk4iaLINE1EQckzZtJmr4OYkWZFq+NqTM6urkvBW51us9OOooiXRTidANlu7u1PTWO917/34H6a5NPhFAIleCml8F3bcbyqEoPBCGVzXkmEQV0+NF9MZVX2ux1R5v1e98XL55xXBoI8r/bW1qnWze0dCGGRJ1iDJJwPh8Pbt/YhRpSZzXY3y/lotBiNFgAjSK2kWGRMKCwXi8nRyWtyCda2V9zA1QDUkr3Vte3Dw/vNYDm8iafzJSPB1cVipdNMsyyMoywKqWUhzOI0tW2bvPEAYMdxGCZIA0pplCQKgDhOFouFO550Oh1KDa2B32wIrqhhaQhN293c3rUcT0qZh/PaTqkUqHcR33cJIY7jeJ5HMF0sFuPxuCiquo5BK15V1Txc1BR7r9frdDqWZdXyNAhAyd+0O9ZkOZSi3vIVBBDCuoFaCq6UIhDB2voPQO0ZkEpAgN+MBfW93oSTqXoyAAAADbXWEGqEEIQIAJ2ksWmatmMpZUglwnChlPrmm69H0wlC0DQNqZWQHFQ6zYTDzG63yzkvq8y2zTheEgru3L0/HNwUWbpczI9fviiyxLVZ03Pbgf/jj39SywVevzxBhLTb3aub4TKKb4ajKE067d7e3j7E9OLqcjKa9Xq99d6a5wa+511eXvY6K4Pr4d7m9tnJ6+Pjo1t7u4vFZDq9KrLoRz/6ZGuzZxjMpKDX8im2LNOk1FjttoAov/j8d5998duT85NlGM5nxWSaAGhwgXmlpAKUAt9qmtQu0pHBIM/weJhUFRCMIYSEAlprTGmNuBDCAQBKFUJwpQGlwLKYZVcIAQClYVAFcFYWnHOMoYIgyUH5B521EtI0TUCwqkSW57ZtO77HOZ8v50ILXpRJFjfbvuOZi6VAgDuOp3SZZJGSNoTcYAoQXApZ8EpKrTGwAhdj9gZAElpwVddeKCErXiBdk04QYww0AQBAKCoFlNJQA4ShQlBrLYDiSpKnz54qLRaLeSrKptXiUKe8LJS4HNwAghutVhanCKGVbnd7c0cKvVhAXQmEKMa43oDLTG5tWkBjCIgGXGmFEKKEKqA456KqajVcxXlZVVKrnFcC6EJwoaXGECFECIIQAiE0UIZhQQ0ZpBAYhAJKUTKf3VyPPcuCWu/s7M1ny6qqlBI3N1fT6Xg4mi6ipZC/++Wvf+sEHmJGs9vzWp21ra3pfCK1sIn9+We/O371emdn5/atvej4PIr4+tZ6EARplR1fnEXLpFDq6PzcsWzIyM7B7uvXr29GN7KqOp1OyZXWby4bjLHtuIgxsZgigqlpaA3n87nS0vd9z3Ns265ibjJDlCKJYmpQx7aK1AQYGCb1Gx4zKOccU4OLsiirjc2dt6g+tU+Hw2Ga5aoChsFWVlZ817t3+9DCsN8IdjY3Pnz/3VZ/RcbReDhqd/vRPEyylGIDUxNiQphpO/57j8hisghnyySLC80Vty3L6rWs7e7ugzuPvvr664uza8yMwWCyeXDH9Trvf/TTX/3qVxcXF2Wee25/rb//7//tL9959Fa/v7ZYLC3L6fX6cRxLKetVqdFo1BpACHVVVXEcX16ez+fTtxptpRhluCzz8/PzFy+febZz++DQdkylxSKcZVkGEbBty3WdJE3yLJqPJ7bteY5vGXYURVFYFvnSc6kfGKZt3L97IIR48v0L37EhhL97+vjy8jQImuur64yFs+nipqoE1/WIUMuXpJRvJgCoEKb1Gav+PVEIKi1ElaYKQu04Vp7nSRIHDXel33FcMylywzA453me122whJDBaJgkSZonjuNsbm52VnonJ8eTxbzl2Nt3dmzTGt9cl1IQxsajEZTizp07WioCUafdXltby4vq+++/Pz8/pwZrdDcZY1LDoiiTNA+j5TJOlVKy4t+/PBZl+fGH7/74008arnNzc/X1V1/OoznG+JtvvguC5r1799Y3N7578v3N6Obw3kNXoawUiNKb4ShOiw8/+jFh9s14ksWJ67pO4PutZtBqIAr3492T05evX7/OZ5Xv2xgBAOtKWVQUGYTw0aNHluvGcTxfLCzL4pzHSeJ5Qa+/0uy0m83m+dnl2dkZhNBxTcqQbTtxuJyOZ0EwbzU7n3zyyfPnLzzPM00zHw2JwSyLGiaOrufL+SxwrDJZjAeX2+ur6/0+s6xoMZ9Ph4wxhCDSQAkpecm5gSHIk7iqROD5e4cHvU63qKrBzQhgVBRFq91t+oEtuGtYNzc3CACRl3me13SSxQzTNMuyLHkV+M2yKIqKx+n09OpqNJ+XZSk1CBqNRTX2O8be7e3NWyvDyVXOo3y+8H0/yYpomXuBO5leS0W2t9fXVtfTotzb2MrL4vTs4vvnz8bTCaHYtPx2s5lkWVFVvKyiRVhmuW277WaHUhqnN3U9IQAgy4rhcKy1BhoBjWpC4E3MF4QAQABAXdhYM8FKCYRAo9HodEin03FdX2vNrmme50otEQJCVIK/qYKsH6eOcJZSBkFQx6jkeV63QNUuElmWSimIECGEY4gB1PAP0ABGCAApZT1DAFVXTkOltZJKKVW3FdQDAUIAQgAhrpV6AAKEIUYYYcUQglBWVYYQgpDFyWIwGExnQ8e1GWOcc6B1u91eWVmxLMuzfd+1//qv/3oymdy5fZAl+urqstnwbu3uxMsFL9M0ibpNH0rx+Juv282Wu7J9dHQ0nU6H42nJK6DRMonbne7BwYHWemdvf3V1NUkypHQaxg3H80w3i2LPsve2d7qt9u9/91sDo6oow+n0WRoPhmNKwI9/8u5bj+5PxgPGCKVYKzGZD/rdtW6rbZh4uZhfXp34trW3taklz5KjwQggXFKGCcBVCUQJlosFkMBkdq+54rtNIIgSUCOqEFEaKKm1pgAAJUEOi9pLAADUWvJCRkmldOX7wLRwgxpFBfKqME3W6fW63W4SVxpIId54ZQ2LaEgqXURxwpVmjGkhhdRCSqVUpaTnetQ27cBbZ6jb7Y6nIy5yDPh0fG3Y2PY9BIAGSkpFGLUsK8tKCDCEb9xhP1BLEGAIFQAQAgyAAgABAABAkEAkgFKKKwmUBkoDABDB5PtXL6QWXFQIi+g0vVlMR9FCXuqm77eazUrL6XIRxXFd+E0QbTiBRUyCLc5Vkac5SpCmSkjLsiHCleBVngmtGCMAgLIsZVUZhLqum+V5kqWlEgJqYhpMK1InFGsN9A8uX5lGGdTApo4BKZA6KarRcH55obC+2FjvNJstx/G++eY7121MZzdHr19S0traPGg0m8xw19Z3CyE3Nrd39g8gIkdHx3meO5b9/ePv8qS4d/v25tr6yWorrTLXtyvFoyJb29u+4zXyPK+A8myDUFQBNZqN5+G0LEvTN1wvWMZpUZQqA1bDbgcesth0OYd1H6hWQCuDkoZruY6JMXCavtZ6WWWlyJQWBkPEhO1eQAyEGVRQlZx7pim1kpL3emuSqGwWLidzripiWCsrq3fv3l1fXzeZsfSaRIFOp2dQE+SV5NJx3KoQ1Gwa0szLyqZ20OggZgIpVtprP/rwR932yuX1xeXl5XQ6JSZa63a3NnqEql/9+m+zLN3b3Ds+fX33rfutle7+4f6XX3+VJJkW8t6dW02v+f/51//vXtBBEJ+fXYSLJYK4ZhmCIKiBTSGqOsXFNM3FYnZ0dBRF4enpabPZbDWb4/Hw66+/fPLkyeba+ocfvBc0vNPT0zzP5ouZYRhc8f5qrxKclzPPRUoWg6vjXnfVdVzLMuaLycX5q8nswrSNjz75tOG5a/2VwHeTOJ4vJhoo02SWbZAlnC+m0TLTClWV0BpqDX6ATBFCCGKtgZSqqqo6TYFpgoGWUpZVDpQOAr/R9Mh6/969e57nPH78+GY4bDQajNAsS6bT6cXFRa/X84Kg1ekAqMIwPLk41xh1et07CFZVtdLt7O/tnDlWulwGDaflbrVcb2dj7d7tw7IsCSQQ49FoPJyNzq/Pj47P9Mmo1e1atgsglhBnHAlomI4pDGEzW1Z5Wshvv3sKgVppNx48fPtqNJjNZjeD8e+/+rrXX3Fd1zBor9fN0+Xe/p3RZHHvcP/J85eHt253m/71+RkEWEGQlVmWp1wUFc8ME+8fbBX5x2myuLwY2DZxbUqxxlogxSXQxGDrK91Ou3t0dDScTgrJdZaleba9v7+2uVFW1Ww+n4Rz5li2bXetNuc8jdPxcHJ5fiUq/f477/2jP/r5fDoXQvA8A4Izx4jjWbyclkW82m2+9dZbT5+/CGejd95+kGbF3//iV9P5wrKsIouLIrNNq9cLut2m57plWWaZmaQLVJaA0M1bt25NF7/81e9Gp0OpICTMMEzXbzJmziZzZhhJlE5OzgPfL9KCAkKJURTV5cV1JZVUYDAc+r6fFwVl7Pj8LCny3d3dfgtVQvTX+7cOdq+uT7lIZ+HMcEiSV5VMknQxGl4ZRuAHrlbYSIqHe4eDwSBw7GbDvby+ubi5jrIcamKbJgBA8nKxWIRhaDLLMAzGWL/b11rzglvMUkqVWSmEMAzj5vKmLnpGCNcrYRiGy+VyZ7NX5xdVvALVG+8DAEYcx8vlMkmS8XgaxzFEQGmZpHFdUlAnkgMEpVYKaIAgYZSZBuecS1HyCiBIEBRKFlXJ1RuRo4aKEQq1hhjVGJvWWgEttSIQawSl1rIslFJ1j8yb9CdEa98ghARCDaFWqpZDAIQh54XtubWrEGMcNLw0Ly4uzpQS3W6bUhpFkWGZ/X6/3+/3er0sSVutltYyyxIAdF4keZpFy0UauM2Gb1vGvduHrmObzCjSuDBZdHpyfXxydnF+cn52a//wvY8+lBp4QVNrPV+E6TL89csXFxdXs8k0DMNiGfkYSim/+PzEMIzPfs1ns0nT9+7fPjx6/nQ8GmsBNrd7D+8/mE2nf/Mf/r3v+9RGOReDkfa9ijE2nU7zLDVNQii0GO41g72NXmCnGhhSkTznusXm86RMM9wIHty5c//BQ8NyTo7Pjo6PtdY1g6OA1lJqgKSUEkkAAJAaQgiRBgAABCAAEBvMtFrtdT9wAQD9fu/+/btra2t/87e/SAtd8lwIJCWHacUKlaYlMayiFHGSA6Bc20GEQogUwNMkKSGUCDS6vc5KVwG5WMwh1DajFiGuaZgmqwDIeMmFRBoKrhACCBGtlFJAa1CHgDNmACml4lpKKbXgUkillECIvpEuKFVJgQAkCDFCScQL02KYmXkVPXn2JCxSxfBoMeut9bFlZKLKeYUJiaLo6OgII0p5GC+ylZVAcWVi0vSbhFkEQcUrSHSZZ/P5OMsTahqGwQBQlgY1Vcw5X0TLnFcQQ4CggG9mBC2lVBwojTGmmCDUtE3Ht9x8mV8cnSwn4yJaAg0ItqTAEBhrq90n371CJEuS7Le/efr2o4/2D/rNRvvhg/e9dpOYxubejuu3jo+Pr4eDPElFxRGAtw/3TAKTeLG9uzUNZ5fj62WcbmxuP3zwwA38m5vh+fn5cnDVbbWPL8+CXgsykCWxBirXRakzIWUZlRxKwzUNxzYMGk4WWR4Rgh2bMYTKLKrShdZ6ukwQQpJLWVYmsxjRvu3v3dpAiAjBLYtSgyigIcRCgTQveZyrTNiABSvrfhCsrq5vbm01Go35fF4UFedSK5AXVVmWSgnHtMJZSoiJTMdiCiHKhYE1UopE0dJ1G3cP766vrqx2uy9fPi+r3GUQm/kyvZklV3YLOE04Sy4H0xOO4u+e/u7p899P54uWxx7e23UMZWKeLG4sz1UKnJ6eC6F2d7eDIKgdhnEcQ6gppVmW5DnN85xS2l/tzcPZzeDKtZ3XL1+9evESQl2WeVmWjmNDCKIoStP4zp3DRbR8+PCh5dhxeBxHk2iZlrl07PW9vbXd3e0sS16fHUXJgtDg26+/chutRw8ecIEuz6+CwPN937IcznOItOPYVSm0xnleCqEEV7UMsDaXQwiFEAhArTUAdccrhBBoILXWCALKsOvaq72VDz96V2v94uWzGvGmlPqBmxdFVVWO7+3s7DSbTcLwycnJ0+fPipcvtre3KGNhEo+H11vr/ZVu6yaPfdfe3di0COZ5dn551u/3u6sdIcQyXa5tb9xOI8UAwyuB3yy4WMYZl0BoJADCzIFU5EVVCcg1miyW1+enJoGu42wfHLz/wSdlmX7x+e+++uqbP//zf/Knf/onUvIvvvgSQ8iFbrZXP333rf7q5nA8yxdTy6BVgbmoMATNhutYRGvZba2E8+7+7nrLM2/f3uu1ApMAkyCTIK0Rl+JmOCyEEFCXki+WYa9vm57DDANScnV5/vj5U99rbO3uTKfTIs0FVxdnl1enl3mST9F4tjF/9OCtTz74+NvH3xS8DHzH860wXrYa1q3dD9998GBja/ud9x48e/7SdVgch4KnpgEX86lpkv5Ke29v7/7dO81mMBoMB4MrAaAmLCn5s9fHQbubVVXKxTwC1nwhIZFSutiYzqObwWRtbc33W7NlDHVe5RVQGQBYCHl2dl6HuUEIe6v9MA6ZxVqdpu/7jmc6TmM4yU4vJqubO5bTIoLBZZSkBcIGhNQ0bUINxkwEcJpUVSV4nmKoO61msxns7u4+f/3qm28fH59ftNrduj1EcC4qWUcOMMY6wFgul1Aoi5gIoVznCtNGo4UxLstSlYIaGGMMsdHymg0nKMtQKVW3KpimCaCqTcjT6RQAlOd5kiQQYsdxEEK1/uYPVIWqwe0aWkiSpA4LSZKkzhys7ZRvpLtSgLIAUEkqEYBQK6hB3XvCpQRKAUIQQAihvMjenNYAQAjXPm2lNVSwVidIqev3FmOoNSIYVVVpWVa/v1KLILmomEF3ezt7B7fKsjRM5npenuePH3/XarU67bbtWL21Fduxms2gzFt0ped5TpwsW83AoOTB/XvxMiyyDEOtROUZ/v3DfYviweXl+kr3f/KXf8GVnIfR85evXgyuRsPJZDyuSn6ws7P+8ce+7xOGHMf5+uuvnz59Nri+cRxnOLje29n0XFvJYHN99YMP393Z3Hny/TdQI6hRf3OLmZ7jvj7Yu33v3sP5eFQWuWUyigFEqHt7/92HD3kF5mEeRyWv5LxaTqbzKIpW+mubm1v9ldY8XFY80UBpUGmgNNBKawiw0lAqSSxQe1MpxfW6ZJqmYbL6fLW+vu44lhCi3e65bl8pm0tUCagUQYgIgbJcVEiXlWq5LVlxpXMhRF4IpXOMsdZgtFxOk8RAWABNFjgvCs/zGCOM0CrL55xrQphtIw1FwXmhmMUQIgghJbTW/M3Mh7WUUr8hm9Qb0AgAAJASEgOIIJJaKqmABpggygg5v77s9TrMMuaL+XdPv+90A8tzF+EkKjO+EEmSLMMFhfT49HQxXQAFt5pWUVRrq9smg5Q5a/1VqSBBeDIbE2ZmRTqbjuM0dj272WlallHGJUC6bkyP47gQnFoMUVJUJYS1wkJDrRmltmmZptnfXet1er4VPP/26XdffHv0YhrYoNdiSsGz05tOp3N4cHd///Dx968ODw9fvjgeDuaj69lsnhSFjq4mu7dv2Y4XhvH55WUjaHVb7Yuzc89z1/r9cD6dSUlMhimJ01wB3Vld8dvNSihAMDZYnud+uxkX2dbaqtYSAHVzc+1ShQ1EiZlk2WA6dJvetrvX7jQHo5twNlOad9pB4NlQK0yga5n7BzsYYy1VnhZFkgqR5xWyDafVaJW8cDzXYx7nmjBqGDxNimyR6Ep1g/bm1k53rW85HkAwnC/LvOKloMTodlc6K/04XEwmoyTJuo1dXpaWZRlBCxTFeDCSUrYaQVmoJI6TdImw7nd7ebIcjq5FWYbJddNo7R70OFDNrum1cJwPr58euz5sd03Xarb85qNH+3kUBR6+vnwVdDfLsnz58uXp6WmevxcEwXK5JBTfurVrWRbG8OrqarlcSsUNw3i7/0jO0+fPn0/Hk+++fmKZ6O7tO7Li5+dn77//fpqm19eXZVk+ePDg1fHrMAzPLy/u3N2dTIdlnhqUddq+69D1tZ7r7uZlGL4YNzz3ZjKKk3xv987J65Obm5s0ix3bS5JoGI+EAJZloTaFkPJKpWnOq0JKhZD6gw5LQowxRAAAhP6zZRwAACEwDEopjeMYalkUxerq6tbWZpzwKIoAAO1O07LtetnlnI+nE8OgXuDfv38/SpM4SxljrVbr5urF4PpitdtjFAtRMgrLIp1NxttbG6bJwmgxXczTsti6tWn6NrVZNGWYknQ4jZJUaCw1smy/0elWRaEkwJ6/tr4li+Ty9CTPK9uy/u4ffvX577/62U8+eeed97Z3d7Isu3/vzu7OZuC711cDgGhVxp9+9PHNcHJ58hphEkaLNE2l4ACqVsO3TBZFcwjovTsHSFVQwW63m0RpEiVFHhMMhaqqqjq7OF/Ns42NDdf3i6p0fU9DOF7MeuurnW5XSBmmMaIEYBSHsRR6Pp4VWdnwAiDBfDQReXnv9p3nTx/HUcaQJhjk6XJnZ+tnf/LTt+7cn8ymd+78rN/v/f/+x383mkw3N/pcqBN9trW5s3drZ7W34jhWVeZKC9czzwaLIAgUFy9eH2uAlNBSw2YL2k4QJ5nWupTg8vJ6cDPqray2O71hqSDAWuM8Kz3boRQKoQK/kaSx69orK92vvvpKKH737u3Du4etVuv61cX06ujo5c321txxCWZ2kog4nvV6PV4BrQiCBsEWwJhgYlskjiIEgGUwBWCz02512mUlLq8HtUbnh1zFGi0ghJVJxEuBIWk3XdM0l8tlnpeOZXc6nclkWpalbduCq/lykaYpYywpwtrq4jhuo9HwfR8AkCa51trz7EajURt5CCF1DTEzzR/CFeqABKFUmuecc6GUEGIZx1VVmbZtQggxZqZRw8U1H1fPzQhoQghRpL5I3gBxCEJcXynoh8qJOk8JACD4m5KIHwaFWhrvBd5kPvM879GjRxDC169fK6U2NzcBAIyxurkqCIIoiq6ursqy3N7aqqqq2Qy21tc6rXa7GWAEBtc3CNAyz8q86HXaZZpog/Z7K4ZhMGTsdTb3tv//PP3Xk21ZfueHLbu9Of6cPOnN9ffWLdvVBg20ARoYEJghODOc4WiCQUohvjAUof9FoZBCCkkR0gNFDkkRwwEwABpt0FXdVV32Vl1/86bP4932ey+rh1NARkY+nJcTmZFnr7V+6/v9fHZYVdSajXg1r7i0bLvm2cv5LFku6oFvUqvXbjdD37Edr+EYhvF7v/s7rWY9S4tff/jhxfnZ9tbmvdt3lqt5GPqeGwyvRy+evWy321VVeEGwvbPvBa1es3f73t1xrXZ5+jqOZ46JhCib9XpQa2S6ypbTaJV7Xo3zKvBt2yK9brPRDAGUWRYBLSACEHEANYAaIgCA1BpoIBFWjWZje3u70WhQSteYDEIIRkQpFQQ1AMAiW0zGy7J4VZal4FArQgxiGAYuCs4rACE1YFly06S+bbOiSJIkL1LLNhzH4QAmaYaA5ppnaYq18B2n2ahBAJIozipGTaPV37Q8HyOqIaroWhGCJRBSKoSQVnC9IgOppND/SMHHiGCsq6wEGGOENIBKayA1JABDRK4vL8oyF4Ivk/nW1o7t2VwUjqvns0LJ1KHmcr4M3UCWaaPWhEq//eAn3f5GkmVRnG5sbVpemORZXPCsFMnkqsxTgpALUYjpzfbGZDjS2NEMUGC0Wy2uqslyzFWBEbYsqZRCiNjUYSVbLpPAq7/7znd27r5rEMshQd1+8+tPlqPTn25vedH8NTEZ9YHTml+ufrPzIHgyAEbT/q//j//t5PXyq6++fPju20ZZWpbR3/STZLS4XEqZYkIVNm7dv5ulxbKUbmPj7OzCznhv4zCo4ovh+ZdPf/vq6hkHoBY253msOMxK9erVZejWFov45OXL9955M0lSpVSelyGxujVfl2w5GW9t7x5u77+W4OTVSTKvtre3TWoURWFgECp2sLtn2SZFZWirIs6LtADAITJoNNtaaEAIgaAqc8fwykIY0Lx38y6iBFOCNKyylGulAIQQ+n7oOI7nr0MAdT8geZ5LyzB9F0EsFEcGafS7aZpO4xgYBjRtxCtWFFnOqeF7XrsocqENwwn+7b/9txBJxjMAGZWjgFSwUf43/9VPilwuF+nB7Z5WhOH/dD5bqZTUw9p0Mrq+vv7Z331g29C27R/+8IeUWJ/89vOyFFLor79+zLn8we/96OosN03w3Qfvfv75p+rm/vnFabwc+DU3yi6xdbvRhfu6Npw+uXm3pcH0o49/NZvNbu4e1b3OnLKw1pkV/OsPPh7F2b/5l//SscM7R3cvLi/7m9uKGuMkvkqjv/7kN0Bjx2ecSakBNewsraQGGxsbD7o7VxcX15fXrKwwBEoIoDRFuAJC/EPRaw3JdUzHND2CZBqljgXq9R7F4OT4ejRYnp0OHZ8WDHiBsUomtm13ev54dHJ2+kRrqBGs15q1WsPGdDRZmqbZ7291GvvTcU5RPJsvR9PJ3Ts3LgbDXnNDClvzgFfCxxuGUQledWyw2+i/SsaLxRyCwsTFfDimlhk0mqvBHACw3e9VVfXGG/uT0eDFMY5WEawgDTvbB3vHl1OWnzx5+vJ//1//l1fn47cePvg//Df/7eOnT37+y19Nl8uT0+cnF5fTeHDz9h0f1oej69Fo9PjZ8Yvjq0azL7STZYAge2vrdppEWVGNJpM0jsN6aHvUNDvD4dCx63FUnqtxLexgjPOMLRaLqqosw26320f7R+fn50++eNRoNDr1tgT6+uoqDMPJZMIYS1h8Njzd2dkRWDe7nfF4fHE+vnXrjWaz6dHOcrBMVsli+DxNCgNYJnQbW5uvjk+2Nw567Q2sjdU8ZTnjVTGZTNI4ApYqRMq5ZEA8P31tErO+0bu6uC4hyDhzXdcK7EhmJRWDeLjXOihWies6BsSCK8E5NikAEFPMON/d3R6Px65nv/ng/ffee4dzPhgMxrMXg8lrQJL/77876/f7u7s75+fDqqocq4dgOJvllxfjmzcaEIGijGzbldJVQvhhIGWVZokF9bfu3TBk/vL18fnF1UbN4rIwMBuOzxrN9o9//PvlZPjs2bNrFDGel1VaawQ9u51n5WwxxZQQTaqq1FJRpCnSFgWMO61ejxAipXStwLNDxtiyiCzL4GXFdCmlNDAiGGktoZZUSmwarkHXhz+pFNBKS24QNF9MOeeQAAPTuEgAhbVajU0r33en06njWtQ28zzzPUcIaWLMWe5admjXgVC2aSIAsxgL3xZCaIAQohWXRckAQKZpA8CklIIxLSUEgmCIudKKp0j7pgO4oshAxOACcoFen13v7m1zIaVStm0uFxOoSgMyKNNty17MFoTxcTR6fXbKhQrDUHKACb28mh3t3zYNz/bCXIjPXjyrNRssZrVGa3d312s1FcHt7kZQCz///HNWFP/sj/9oPh0Pr68kLyjJeT4Q2pxmziJaXF9fbm33D/caFn3481/+LIpe7+xurfJskURXY3A9GFWa+81QLEE6GchkudMNebX89Qd/5XmB45OisuZRWlXSsKwdf/P47KnUtpDFYDgUXiCl1MiYLctVdsmZXMZRvdEWGiJqaAA4lwoC0zQt2ybEaHWDNYGbUgwA4EpKznhVrhFQs3m2frHMdZzPIYQmdkLP54IVRaZ0RQ2odCV5VW910zSer5KyZFprDWCeZOPFxLADDAHUWrJKUGTbpuGaGunhZMg539jalkoYCBkAGJQoCRAy1DfKUA6AxBgoIJUSkCoBuBCl0gIBCCFSSkqhAt/P87zIsvW2WAhRiNLgBtEaTqdTKUUlKpArzjmlmECKITWpCSHc3drHCklHCCYcy2r1Wo5rYUq2drb8sDZbzIVgWsvVcqqUYqyaRUuDgHrNjeIVNtE6RQuRsCzaaobIkNPVLE4jiJGSgCsODGgQux5S23KqiiNiOnbNAO7mtvcHv/9H8+H18YtPeQkaTXj3Vr/X3pvNF41muN/fbgetlhvSbt14x3/n3beTMlFAAqony0mWLoCqDKLKMkrja0rpwa3bW1tbhpMsr1UyjoWSvX7P8u2iysfzlda63++PB7NVtHANazKZraMlUZqMxzPP8xAhCkDGRCUVIKm7jFw/3Nne89ywEdYajcbxq1ePHn0VL1fmk/BJt7uzvX2wvd9uNLGNiCIYojwvG20EtWJVBda3WVpoLYltUkqJQSFCCgIEEEYQIDgejxljtm1rqcokK8vSprTV73O+7ikBgjAgBCFM8kJrzapKKYUxBmvYu2X1ej1CCAOZ5waGiQDkWa64WN/3QNO0ASSCF0qBwWDQ39jd3t5ut7s//fe/sCzr4cM3ms1aFEVpnmxvb7c6dYjUycXr589mjgNsy2g3WycXx59/9enDN+/+4Ae/+8/+sz8djQZ/9R//1+HwivPKce08T2/dvrGzvQEAwBB02635tDEaXD/+8rONzV2L4PHwGs7ma6XparWazGbnpyclZ/3DQ2B6qyglhrW7f7RarRgTSZZyoShlSVYYpgOQ3tnfxhgrAObTaVWUQEgEAEQIqXUCfA2hW+8YpNZyDWVCCErJF3Hy+PHjbq8TBMEynmNI2s12WXmNRuPwxkGSJIyxnZ292Ww2ns4Xi6XgyvMCw7Cqioc1p173W03/jXv7kmfHr184lvUXf/nnt27c3esfSAEc216/i+c5vV53karBeDKfzw3D2tjYWESr+XxuGMbGxoZt26ZJCUW1WhD4LgTKtkzP2ihKlqVpkcR5hn714W80L0ejwe//6Afvvffe0c3bk8Xipz/7eVlk3Wbjq88/aW7ucc4Dz1VaXl1dDXe2lWAZUN1OEyFw48bhcj4dXV2cn59v9DpK8tevv+h0OoSgPM9d166Hvu25hmFMJyOEQBqtoJJKCcexCEEQ6mazTghJ9veEVkWRXV4uJuPh+cWpYZLAcyazmevalGLTpJ1Oa7GYFUCOxuMozaI4Oz05YUrXGvVWs86FKrJ0NLjGCGxtbTZqgeu6SvCIJZBA0zCAAkrooijWBEAIoWkYa7wxQqgqyjiOMYSMFQBITCCvWJZz13XrtZAQ2u7UK14Sit55563vfve7vu99/ejR8etXWVFmWT6bLRCBjHMh5WB4DQBAmNRqDaXA6empYZieW8uLHECNGGCsBJoghLAC1CCtWv1ob98glCIaeP5svuBFblLLN+2a4+69/x7FZLFYLZcT03Kj5Wo0nDhuIKWWMi+yEgBgmXR92S+5qNVqZVmuJw1a6+VyCSEMazUNJMYYKM05X9+mEUKADaSAQCmppF4bBcA3+pVWq8WlztKCCY4xdhzTNmzXchXJi6Iqy9K0jX+8UzAMIiQr0rTIVr7tOKZBpeAVWyxn0IEIE9t2DZMiioTSVVXmBfsH3rOCUEKgIYR6LQkXImcV43w4vCaG9fr1q5OzC8MyHz16BMAD0zQwJnkS53nuOM7uzp6EsrXR/vLVk9enJ81+FxF8cXmWreLbe4c//IM/ONzfk7xKIufi/GQ5nUzGwzDsnlxcnlye247XaLafHr+ixHx5clYUxf379w9v3x0Nrq4uL+bzaZkXXIB4em7b9v7uJsCo3W4pzTf7/dl80mzVbcMMe/2iKLTUR0c3m42uZXqBpyGEj754rDW8efOmknqexBrhrb39oijDejMuWC5UIjjDJOZien2ltVYAIYQwJYQYECPDtTb9Hd/3vaBm2hZCSGlYVVVVVRqL9d+KKw0QVABqiDQEUgEMMUIAQIggWv+HI4TKjCmlqqpgvCIEWLaBoQkASJKsqhjn37RVtYJCKCXB2oJrUINaJoQwLyshRJ7nFePr0Kth2o1GA2KSZ2VVFXGaIfQNahooqaRUQkol1jMkSfC6BaYlVBoABaqiqIqiLEshBOPl2vBCCCLtdns6nVqWXbcbq9WyTArkWBjT4cW439tYLBYJzUPXu3fn/vn5+enJxavXzyzL5pzfvHXHCz3fd9YrynJuvD55xVgJtKqHDdczh+OLRqORValhGNTSLiW23+hs1LaqbpJlaZqblgMhlUynaTmfrfJMzEZL8mpwsOs0Qx9h8r3f+1GyWv330UxUo/29zb3tTSRAtpi3PdvHgYyZiIq/+evfhmHoe/VGt+7XnNcXrybzwWQ+6PXbimVVmRom6vXqN271tnZ3Lat6JJZPjp8uVtOdg82trf5gOi6urlerhUmYYIxSurm5aZpmPXTSeMEqTm0Hm5ZQkgsFTWQYZsXlcDrd3t6u1VtKQcfzfD8k2CoLUVSyGkbpOBFLZglTZ9KxbdM0CUSmaQLLlmnKOaemoaCSUlADwzBcc1HWO811Z+QbCCuh5lq2JxUSiiAAFfzGbqARQBBgAjQA/8BiE0IIqTHGpu1ijLWWhKJlUqxr0OuYEtdECii4ns9WmFhVJZRSx8fHvtdwHMv3/c3tXlVVtaab5IvORr3Vbd++fXs+n/8v//7PJ/NZ0AS2jVnFch5JwhksfvHBh9TEb739sF733/3Ou8vFTpbHpkMfP/kSQqAk11rHydKxaC1wHYsmy3h/s+/7ta+eHUugt7a2PM97/Oxpu9tbRquWbdlOsCr4aDUFpgMgLctSKGkYxDCpWEexgJJAS62pY1mOrQAoOMNQE4QVkBhjiL7xSa4/jeub3bIs/+GnjFeL0WiACXrnnbdevRJQaYKQbVgbne7bb77zxRef1cOGadiuE1hmWeQCY2AaVEq9WMYKsiKfmHTzj37ynWgx/eiDX7Gs+vLRp6ysFJdJXHRa7TAMyyoNAn+6mAitWs1OXlRZXpqOXUdoMp2uVivf98NasNXfbNbqBoKMsaosgBb1dkPwqtlyYoQtCiaLZbSYZHn66tWrza2Nd955x6TkD378o5s3b/7VX//H7Y3er778Kk3LRuhUQP7lX/z5xx/+4sH9u3/4kz+QrEqTqO57WZKa1LBMahKarKJ4FSGELi4u4jj+/ve/3213NARpmu5sbnElgdbRaoUBxAAmqwgD2A6CdfWjqqrpaDwejyGE8Spa10Om4/E77723u7urte71es+fPz8dX5+dXy6iFTVMqFUjCELXLfPi2Yvnt2/ffuvBfUxQWZbT6Xg0GOZ5juoW51xyxUteZmWW5HleEogk4xBBVlUEhZSQsiwXiwXG2LQQxrrZCBFCy+XSsmgQWkqpLKuKKOu0mls7fWKgrx4/+ujTXy9mc6WF7fterY4xtFxPaACxYVlWUG8cHB0Z1Irj9Pj0deCHvu9TC9csR0kAgSQQagII0IbtHG7vbnY3QjcQUquCT/QElCUo82KxDA+2At/lZbGaTZ1AxEme5uX2jtntbuR5LhiXQhgGsahvWwYhRFOMMGCMAaiKMvtHKjNGdJ1O1wBJJaHUAGoIYVHy9YnwHyHuQkml1GqZJHnOmESImNSi2KxKMZsuQ8dO0xSgbxoWVVVlGBoUVkWKoO42m9tbG75tYQTyJKFWVamyKnlezrIcSA24hAAhhIjiCoB/iOMBoCHQGnAJbRtmcV6W5dXlxfbObr+3MRgMMEG1ICToG7iT1lpykabpZDQib73lBV5aZasi3q/fyPJyGS0sRK4Hl1qJi7MTAoCBoEVw0/eZFFmRRqvpfDFudzfG08mT588QJudX17fv3Du4eeuNo6P21kYFwOlwVJRVp9NBixEXOTWcyXjQbdcglEdHe3GclmWJIKn5QbRKpYRI09FgijGeja42NjauLsdbW1sPH743nc4n86cQ4yhnluO79c5wPh/G6TLJPC+gCjQb9X8MP3EhMMaW61m23Wq1AEJCKcYYl0JKIZHiQHiet34SAwAwRMgga+GfkBL9g9pbaLW+DdUQGgbhnCPEKdGYACVBURZZliol/vE6Valv2goAAAIAxdgwDIINIQWvqkxLg3yjDhlP5hBjRExEMGeScx5llWEZhm0RSgGCcA2axgZjTAOAEAEEScmV1gpBiSGWEhNIKMIQI2AQCCCE3xSDGWOe12g1mqvFcnQ1Nk3TolQIWUSsSoUm1UajT4nl20EtbP79B780DGO1Wj159uTu/Qf37t3bunvfKnMIRJZGtVotTVeT6eDe/ZsSVJBorLVTswhBUcySLIMYuqZpm1a32YmTYjZdxlG5mCeXFwMpYTTPublbdzdCp1PEWViv/9Gf/pmQZTy/2Oy6vJyXRdRt3NrZuCFihzOFmCYEBnW3rFKTtlqNxsUlqns1SikXfLWMISZHNx8cHBw0alsANG1rk6NFxtK0SqlNG91mzgvBq9FgKAVu1bue4663/K1mN6zXk2jZ7u8KIZLlQmrVDINWp8M5z/J8Nl9ijGfz+XAwzvoFl6JWq5mmmY0TCKGhUTSZJbNFu93e3NzklBBuLIfXk+nUdd3ORk9KUZaFhsD2/X+wEEgAEQRaSrl+CpsmRVDzsvAssxb6WshsPrObbQgRwBAABXilyooxpqTAGJdrYAg1TNMsSxLFy1USW44huOa8IBRICbUiUgDOgBSIC26ZztZWYxWlnDMJNaX6/e+9F0VRksTD6RWX7MbtA79h//SXn6RVZPmg3qz3+lsQ4iIvq4rHKQFYPXr+6Pjy5d07t959762Hb9+dz8bLxYQVxWg8WMymzVpdVCXAuCpzxza3jm7ev317lTOALMutHd25g4nx+OtHBEEmFAXo8y+/Gs5WyrC3D24tZiuMYRCEthNIDRarpKw4E2o4HggpOZerZJWUaSUqk1JMEJACQggABlB9szfC66KXFoIRjIoiKyvNqiJJ4yhaGoax1e/WQ282myECEdRQScHK+Xye57nScI1CqioupIIYOY5T8QmBgJqqrKKt7Xa7E/zVv/+PlayGkyvfD6eTRcGKDb2xWi3IDL58+ZzJWqPR2j84mq+WURSPJ1POebfbTdN0Nptt9FoYw729nZs3jmbjkVR8sowatfDWzcPVfEKRDFwjy5PxfD4YXdebtb/+m7+5devOt779nU67e+/evcvL67DX/elPfzqb5fWgDGt+VeZffP7Z8PL8f/e//a9a9QbQkiDU7/Ukr4DSVVFaBinTJFkuao36Gw/u9Te6r169Go9GUso0TWeL+bo/kmVZUK+1mvt5ljTqYeC7c1YaJum0GrVazbIMgqBrW91u+8G9O3t7e5eXl1xUSov5bLlcLllVbW/t3Ll127QdiPFyMQtsk0ItylJRAjVAAHPOkyQpeFJVXEtAMNVCl2UpGYcQSSmRRhVnlFLbtgnGVVFahnlwsEkIuXnzZrNZn0wmSilqkOl0ihfaNM2jg8NWt3V8evzhRx8maXTj7s1ms/Hs2bOiKNqd5vvvv7+9vX15eV5m+Xw+50J5vmmU5cuXzxzHefPNN7Mc9Ds+QhQAYGDCkS7LUkvgW85Gq6O5youqSLPZaJxmxep6+OTzT2U6v7oaAM6xVtPhoKy47QZYS9cxgZIxAayqBFvbm6BhkGWWGAbVWkZRxBhb9x4rxlzX/8bJBL6heiPElVKSqW82u0CvXTbrduXV1dXaJOk4zrqLhCAUnGtKKKX1VtMwDM4rYhqQYKlFXuWtRtDf6R4d7hoEKF7ifnhw1J1MR9Pp9PJ6uEoTqTUiFqGmhgQRpDXUGmqhpYZKaq410MgkAGO4tdVHmJZlsbu3tYwW19fXxLGEEJIzixIEIaXUwAbnPCuLRRqPF1OAQSVFnqe1Zu2NG7epBGWZ59HSs8wg8ELX4VUOtMJGsffmTSG1Qng8mcd51dzYYE0XynR0+Xq23z88PHznrfuLyeDJkydJMk/ihed5hwe3vvPtd7Z2tler1fVmn0v10Ue/lVIXRWkbZkmti9OLZRzblvt733vj6OgoWhXEMJK4nMxjgC3HD18dv755527Q2bheRiVAqQLtZtNut/MqNQyDULrOfEgpESWmaSasKMtyFcdrG47juZZlWb6lgOKScyEghAAjA2PwzXkFAYQ0hEIpKdd6CAUhdABFUBuGBSGFCBRFtlqtZrOplJKQdSPmGwr42hBCqSEBUEpzLhirWFEBqAxClRLL5bJgKsuywXDq+36r1arX677lGIZhUFNhyKVEWmOIAUa5yLUGGiANlAboG3ICgKwo+To9u/5loV7nxEmWx5xXBsWu4xgIy1JIAaFn1OxwfDmilO4e7d27fe/i4mI9/Pn4539RVdV0Oh9NJ5eXl1cXF7+X567rFllKIOq1W6+i+ePnT37n975v2vZgPFDYUBh6nkMNK0AIIq215kyen18/ffLiydOXVakgtspChEETbOB+dw9pw6C2MLSqGArDH//Rn0SL4ZeffPjo0RfJalr3LAJ8360bAZkMru+/cxDUa1VV5GyZFtZ0OhZSc6WlwkAGjXpvb/P9rY1Dy3BlDsej2dX0Chio3mlgC5csV0BgDAHU9dBv1EIhxPXVVbJa2hbhXF4NB9p2JNBxHkNEuJYSao0hJHg8G7daLcMy8zzPi9S2zXqzIcaTIl9tbGyENWeVLKbTaZTHgEKAUPwizvM8z/N33nmn3euut6hKqcCtVVUluFAAYIwQQExV67b0uqQ0X0ylYP2NPnQsVCSiLAzTBBABLouiKEu2zi0DAEzTNgyDIMB5pQDUACFs1kIPIZwkGcJ6HdkWCuRFwZhK0rzRMGuhl2U8z3POBUb01o27/a1+VRWWZ/z9r35xenW6/Hr+6aMvev0ucWmn333w8K4XhNPpfDyZlCUTu6gs87LIJqvFYDImBrQIqTcaq+XCdd08TTzPgwBADQjCDT/c8DcajY7XJMSpu/XO9t5+nGbnV5erxcTxXK3h2dnZ+dU4aHa3tg8Dx5nOZqmUySpJy6qsJCaGQcBqsczTglIquKKUQltSSgnCEgEgJAAKAAWRxhgTggkmCCFMkG3ZZZlzXkIElJKDweCrr74kKu10OmUx39zctC1Y5sujwx0hi5Oz8ywrKiYAMgCEhNIwqDWbzazIm61ws99ZLce2qe7euzUdjz/45ceT+bgWtoSEXIrFajmfzyRQg8lUKblcJc1O++bNW7P58tXxScH43t6eZZt5mmAAKUFbm/07N298VaTT8XB4cSWK4u7No263K3imeaEBwKZhu/Zf/O3f7u3s/+KD3/xf/u//j3/1r/6Lne09Sswf/eB7//RP/hBD9LOf/eyDX/1SMo4hSlfL/+7/8//+z//Fv8AQYgiUlN1maz6fhr57eTESQtimtdvfIho++errdcF1MpvN5/PhcIgJcRwniiLJBbqlDYMahGpLF0UBlG7WGzt7uwCALEn6/f7R0VE9rF1fX08mEwBA6Pndbo9SAyF4+/ZtPwyyLMuyvO7ac8ckEGTxcr5cRXG8StI8zxHCRZGuFhljwLOxbTprx5FWGgEINVBc2KZlfHOeKbXWt24fIYTu3b+9t7eXZRljZZ7np6enZ2fItu1Gp8GUGExGSZHVW+2d/QPDNm4gOJvNeFm9eHXy6uQ0S9J6vT4ejU/Ozw5293Z3t4kJiQmD0F4sZnNCAAAUE4pdaqCqUJwJahuL2VxVHHLpYOoQmrEoKvLnWSqKSGv9xr3brUbj57/6QHFWD31WFWWaEERqgU/X+nOpFBQIa9syAQCK4DXQ0LIs03IwxllZSSmziq29JBBRhKACGiINILBsyzCMPEmLIgMAuK7d63XWzxOlZJVnBkbQMCCEg/GSUtpqtbSWRVVSiqWUSZEoDcqyzIq8ZAWCEBPtuKZj+oFr2BblvNSAFyVTgDHOKsZN05YaAom0BloArSHSSGu9WhWe59++ccSkePrsBTUtBAHCUCuw9jHyik2n0yyOOGMUG8PhcL5almWlNby8vKz5te98+3tv37lfRonK8sAPk+UySRLfD5IsXcbxXs+/fXufGlbOxKlNZ6uk0291Q09BVMWTkyefdwNz9+bBP9W/C6vo0aNHX335qNVqNpuNbq99dXmOEKqqwrKc0HcxNZO4sC2rtFkURY5p5lny4W8+4lJBSIaD8X/3P/y75SoOm61ub0MTQkwLGabheLVWhwFo+TWM8XISaS01wIhig9jf4CgQSqJISIEINB1CKbUdSgjUWnPGtZZaCwgRgFIDpZSSQmkI1tGzdRl1TbgCABScrXGxhCCglRBMa40xVv9wO6C1FkJKqQCQEMI0yU3TRJAgiyiNuIZSAKGE1jpnEmfVahkBgLKssC2r1Ww2Wh1iUIhRyUueZ5JLCQEGAEOioAZKAIDXDTEMNULQdALOmBK6ECWrOEKIQK2RJnG80IApxZWoHMuq+x6GJHQC13aKKBOVkCXb3tyCUF9fX0dRhAiN5wtiUITIYDCcjCZ5mv3pn/7p7vbu08dPxqNRp9PJyoxL8fjx1y9evFiVkiLseU6n3Ww06kDL1XyxWKy++PzLLKuipNSaIGwZ1DnY69y/8/Bo90aapqxgju+WWUYFcLs9t1lfxvnlaDaexRdPL+aL5I9+/Hube5tJEQNbliCpZBEX2M0tiLXjONRw0rza2Drc3NrtbR1YfgcYjirL2YoP5kNgQgjg2eXZfDWFWtu21Wn0HLPmOTUDk6qqKKVcCoBgmhVfPvtybXTVEFcn1enViYbYNM2tft9yLMexPd82qWFSU2tpGAirvN/fQEidn5+cX14sy2UOMkqN66vhar5wXf/WndtCMIsaJKylaaqYBEJDBQkCCBKANJYEIaG1pJRqoKbz+Ww2EYJt9TcN16wKRhBCUpUly5KcMaakXo+FXNc1TcpFNZ8XJePUtP2wDkBeVTyKEqWE0rKqCmrRosyyLJsvFqySgmsAMed8fegxbUsIARC8++D+IloMx8OCs7fffzMIQ2zQjc3+xuZ2FCXzk0VSpn4YLAru1Zp+vRGtZq9OTi8vz3udZqvuI61v37q7v7WjpYIauZZrIqMqGAhNLknQbDuNLWhaxHJcTHf39inFQFSzydQimEDF81Tn2Rs3juy3714PB5dX11mSV2lCLZdQGwghZAGkQgi5psERVEoBJaBWEIE1RQ4hgAlECEEEANRVVRlkPWmACGNKiVJisVj0W6LXc0xzs9vfIIQNBs82t3fbbfvRV8PRZJ4VZVBrtTtdz/Vdn1oekIAMB5dVFR3ubYwnQ83E7/3e716eDV48PY+SbGf70HK80XS5WK6UElleUVMPrwaDyRQi0t/e+t7vfv/4+PhycN1q1BHQlmXVwxpUkrPy6uJsMop44U+u8+HF6c7upmCVbRpBLWw0arPpWAD0F3/763bL51z+n/9v/69/9k//0+3tXcdnlolrjeYf/P4P/uSf/FhL9ZsPP/zis8+KLFvMJqPBsFmv26YRhiEvK7veqIqsqirLILXAW8wmjx8/fn12urW15bquRcn+znar1TId++rqSgiRZ4nnthmrgNZAKYMQCMFGt4sQms5njUaj3W6fXpxHUVSv1znnpmkau7sbGxsAgE6rFcfxZDgAADRr/hWGnVpgOe56dGEbZrfb7W9s5gZ/8eLVq+ev4pXMQAIhgBpAADjn6/i9RY08zXhZMSGyJN3a7K1Hbpxzz/O0dqlhdXvVYDRM8+zs4sp13bxkrU6PUnp6ebW1u7WxvdPe6KdRPJvNnj17Pp/PN3tdTNB4OCjLfGuvf/fuHakqJrOT8xfpdOw6vmvboVd3bY9Jzjk3DKPdaoV+jXPp+37oB8towRjL89zzTdv13n73vWWUjEajV6/PHZMWZTUeXNeaDWddvAOAsZKXCgPtONZ4PK4q7gcBIYQJEYbh5s42ACgr8tlitVyuKsYkBEBDDZAGQkkVGkG324ksc7qYFUXhes7V+ZVlWY5l2567/iKEKKU8B2uINrf7AAAyogCAosi0wlUploJnaUUtu9trYiQUL7ngvU5PaxCt4vl8sSyWTAqNoNIwKSqtoNYAaII0QohgiCHEFjXLPJlNR7VGQ0k+vJ5BTFzbUUrFqyhJEsexFrMp1EADVZYl1iR0whu7NyerFbXMt95+9/e+9ztEgc9ff2xIRVz82eePlqv5d7//PWw6EpDBcJpXyvFcz6+XTM3ny+F0jhF1PF9U7FVZpavojfv3arXavVtHLsUOQWmapnH69Otnk8nk4MY+pRRDBCFwbTNPUiG1FGXg293Oxvn5+dNnL7obvVs37wign794JZS0LKvdbsPlKo7jp0+fJllmWVa72ZJcsLKyXV9rzaUGiq9TBQRCCLXvu1JKkoOiUFprLYWGQGvtOFQIIAT8BnCFBAQAIE3WnRKItdacfJMdgRBixjnnQnKlOCvziuUYg3rDp7S2vvKQUpZlWVV83bjhpVCEAAAIMSChQiFVVRIAoRiAhmG7ZlEqBRazuVaiSLPNHe54ru25AEKlgNIQQA0hsqgpgRICQcGU5JBLCCAA0DY9ioXgoCykFoAQYpmOSQ2SpAuCseBZGi8NBE1KV7NFvoiODm4c7uyNxoMsTipWvPP2W4TC58+fG5ajAAqDkCAsgVwuFh//5tM37r/1/vvvtRqd4WC4u797eHDDdbzFKtEIpymfjK7jZGUb1DaMPI1nEx7HwDRBuwU9yysqQYi5sdG/dXRrZ2cPSUixsVwu22YbUJhVRUADQHHv4OabjJcVX6XZLE1Pri9LmU6W16MlgBA6ll0CgUyCLdRoeUG9MZ0t6g272XecuizBKInYchVfzh9nLC9lBRQXvLBsErheWPM9M5BcU0objYZlmibBjFWYkk6v++XJYw44pVQCvUgWJasIIa1We2e3H2XzRthoNsMizxGW3V4DIkXNzDDJxcXF5eIkVdmSGWACLcMuZJXJ0kHeKl6+ePGiETbazbasuNaSAEQwVUCv8Slc8fVEMU3TqioYKyTjw8XYCd0wDJXmnHOlqqIoueBgrfcAYD2QpNTUEGBiaAjKinEpsmgEIFxFKwihlJzzykcBgoZlwjDUjuNNp/N2p2PbZhDUGGOCyzTNlvHKdq17D97wG7Vau95oNZ3QzYoiL4u0yBfpKsrTs+GluJA+3crN0jRomlWu61ZFrpSaz2Zvv/Fgb+9QFEUWJ1VeuLYvmZqOZhR1msPpvt+pdRsckYwpKTWiZGtri0IFBL9z48C1HWI6N/e27t578O53v3VxPRiOxk+ev/jrv/35V0+elmVMAMQQSlbJ9edNCsmY1pogTAyKIULoGyg4gEp90xbjeZGusc6cV7Zt3rx19O1vf/vmjrQci/NmyarhcHh6PovT8cXg2rSFFyCAkWFx09ZegAEq58s4T2ZffH78zltbnudEq/Lk/My3Fu1ubzEriop5QUgNu6ympuXGWcIUJADZrlOW5eOnz/OSHR4eOo7z2WeffP75555j13ybi29ppdrNumNZYVgA4hgG4XkazSeMF3e/8y3XoyVnTOmLwZADkAsVBI3hYPp/+r/+P//4j//k4f3aR7/5YDIc2bb9kz/4g2+/963/7M/+6X/5v/kvXj1/4Vi2Z5n1sJYmSZamSIM0ijFElmEGQeC7HlAaAVhm+bPHTxqNhuU6tVoNAhC43r3bdxBCtVoNQJlES8dxPN+xHXM1XAjJWvVWmsVlmQ8GV1kcBa5TqwV5niOEDL8mFReMI4Qkr/IkNg3Dt403790ruTg/PZkvV/1e7+ad+/uHRxsbGwuVPHv87OPeJ69fvU6jlJW8zKuiqATnhmH4rheG4TpfYhv0m+G8lNPxBABQr9cxIkxw07Qare5gMEizUmmEsNFsdUvOJtPpNJpLKZMo3dvbu3vnDsDkow9//ejxk26raVrkanB9fX313rfeWs4nJ6evxpMrGeQN1cxKI07TVtiSHPBccC44k+uWuZbC911CUFUVI8HqtcC2XN+2HMe5dXR0fTWMlwts2MPBlVTccrx1rVEJtr5P1NTI85wxZtumUipOU8YYJHhze8fxgiYmGsDFKqqqSnMhNTR0VhRFre60ezW/Zg2GZ2m24CKr1Z393b3bt2/v7Oy0m601d0FK+dtPfzsajfr9Xq3ebDSbSZLNZxPXdSfjgYG064X1WqcWNnmV5EJKwSEwA7fZafaur4fXYljkueFalmOneQEgXINuNCQIAoAgwtDzHAjhy1fPa/UmpZhQZJhmWK+/fHl8fX09mUSdlgu16na7vue0Wq2aFSoHK0W3t1B3Z/PG0S0t4Ucf/fbq9fn9GzeipIiyUgKyjDNs4oyLx8dDIS4b7U6r3TVtd1bg0XjheX55tfL9JSXENI8//er57tb20eH+O9/+AVT4gw8+KDNWFAwhPLoeS62EZKZpzucz0zKwEGUecaVX0ZSLfGOnRU3T8R0NAaFIAT2bTW7evtVqNbIse349sF2PmpaBcZYm1DCISdYZES4E5xwqjRDAGCstpJR5nud5BiHEEDgWNU2LsxxCBaFQCmgloMIYYkK+IWIjBJUGSAKl9JrUiQ0EEUZClpXI8yROVkoJ0yKO6wCw9ndgiAyMv5lAQw6FEGmaC6EAIlxKqSFCSCpYSS0VAJhAhMsym46mZZZfL7Jer9ff3qrVapgSAwIFAUYQGUgJCYAUQsqcc8YAABjoZca01llWpqs8y3KCcZEKQhAJa27g+aEXeo7nGHa8WI4vrlcLSeDrW7duCSG0Ei+ePkVIR9Gy02npqojjxDbsxWxuEpMS8+WL8UcffLSzufXuu9/6zce//uqrr2vN8Hd+9/vvvPOehmqVWa9ePDt5/SqL4yrPkIbhAQo8D2OMILkaTJK43N3Z+uEPf/+tt9/tbPSLrGh328tkvlotgloADUcCUAketjrvfqder9du37mRLi+qbPr66hTKcv/BO4yxeuBrJaht+XWv3g5anXAWXS+zSzNVXWDPo/jrZ8/H89nJxQkTlVDcc61+b+v2rRsmMh598Xg+XhnQBY5yHMc0DCn52clZu9PY2dk5np1qpLMywxgblmlhYph2ELizxaQqGNxVvU63WhbRcmEQUpTJ9fxMaXB+dZnLrLvTCYIaZyyXwAk8ziU1yfX19fByuLe1e//OvdFgeLTtUcPABq14mZc8KzMOBCIIY7xYzLIs8wPXCV1iGAIqTSCB64eOZIxpDSg1CDG01pRSJnhRlRBqx3EMao1Go9lsBkVqOTZjlWlRrTVCxHU8yzazLPMCnxBydnZBKF2tVlrr+XxOaM33fcuyR6Px9t4modR2PWzQVy9fm57FuFxEq1WcIII3NjdN0yqWlhYSIkBMq95s8SIFkq+Wy48/+iSaLZCS9SCsB6Hr+I7j+W7w8tWp6TWJ37IbXeqajLOCZXGSFfGiHliXlxdXl5cvX5wTShezxXwyvnP/TqsWHty+c+/O/cl49uzJc6bkZm9juliyvORSEITXEQ+EkGVQSAnUcF0KXyNL17q8IAhYVaw/e2VZuq5969atn/zkJ9Prvzk9fem67nyxKFlluwgTYdvwzbfu5RWbTBfTeQQgA6gCEDOeM8ZcF7z77rsYY8/zLMv6D3/5VxvtnV5v4/jltZIAQOx5Qa1ZG40Gy8Wqqnit1nA8bzqdfvrpp6PR6OGbD3784x+/evH84ux0MBjM5/P97f7t27e73W6RpxYNe5s9x7UWk3HB8u3Nflmmzz795MWrl0zpeisg2Hz8/MqgYG/36Od//+vV1HzzzTcbd25++fkX//O/+x8eP/ry8ODgW+++12jU0lV07969na3tr7/66otPP/M8zzCMrc1+HMeLxeKXv/g5MajjOHu7O6ZlnZ6e5nl2dnb66aefdLvde/fuHRwcCMk9z4miyHGcmh9cEzIejyejse/7hJAsy7gQzWbTsqzJcCSEaHU7EGMuKiGEZRjtViOO2kAKgJGypWVZUeIsozjP8yxJtdau6/q1mm3YrUZ7+GAwnc4HV8NXL47Pzy/X3Pd6o9Fut4sy813P9Z1Os2UaFoKYcx6tYkoMjLFQklLTcwPbiauq4lzGWQoBRggBTZbxlFKzZFWaZ4tlNJ8tKy4t2z27vPzWu29m6Wq+XASBV+bRaiUPj/bTZYIoKjkrCwYAAgyVSZWZOWxTg1CtNcTItu1v2HWSL+eLGEcvnpl+vWlZBquK8TzvbXUdxyaEKMEYY+tbXtM0TdMglmPum0IoCCHjUgGd5sXr168vrwe1Rsv1fKGklLKoKiEUAAAaIq8KBXWz1UAInbw+XkVLQvCbb35rZ2t7e3vbc12l1CpaMsZExeJk9frkle06lmMjRDjnZcUJtvb3jpQsHDtI4vy0yMtsZVDtOPZiunI8d2tzN07zVZoMxwMJtNbadiwp1io0oJUGGCAEEAZSsPtvPHj54vji/DRstDnnrhdsb26enp56nidYSQiZjCIIJ7a1EwRBlTEmRZGUezeP3nnzfSb4L37x9//+f/yfA8c+2j0UlXj41ru7+7sA6VUW5VKezZgsy8oIz2YlMXTJ4OtRatuyVqudvrrWUuzv7SiUD6fP4kI2N/ZrYfP18ZnrOo1Go9vdODs7EYIzWd27d+/09PTg8JBS//T0dDYZnb0+oRblGmV56njO0dHBzs7OaDz+9NPP8zS5e//BeDRbLBYUQllVEFNVVfV6fSLStZ1LCKMsS875emCJ0fqKCkOtyyov8lTwCiFEsfimyCqV1hpiahgWIQbSGkKpAJNCM6mEEGtgNq8yhAHGSIiqrPI0XQGoAbSEKP+RAbMWcEgplQIAWEVRrJaxhAgRgolhWY5pmkqBquRJVggmQ89Fvm8Q5Pvu5XLhBH4fQst1EEJ5lTPGpNQYQwWAFopXjBclKyutNYFISbx+R8ZklpVKSAi11poIuioQbzdcwyxvvnnzzoOtd79z+4tPvyxS8fXzz303ICX61S9+8/Offtjtdv/1v/43NdT81d/+dq28tExnsUgUBB99cbx/6/I7333/j/74X3/51Yeb27UkveL89Or6dGP/J3/4Rz+qiu8XUWkRczYavj5+wkRiWuBqcBlXMOjuvP+973/3R3/W6d5KEqmddLCcEwItQgEkFCAhFVQUE7MsweHth4Vgo2vy8ourWVL4gXVgbO/f3tVIj8ZXELH+flPq+Hj0eFq8yst4xYNSXbhe0wS5UfG9xkbd7UZRpJXwrVbL3yqKIlrls9lsZ8ttdFzHg45tPfrieDRY+W6nyjU1sNaaYkopNg1kWZZlEAPE8XSVRCtYXmWLZpkXWutareb7+PbB3mg0mhik5rmmYWhITNdynRrUuJzEKq3e3dj67DcfR4v48uwiWq7md5Od3V235pc8y3leqRJTZNvWYDCIRTKLJke3vrOYzT/77WfNoBWL2FLMcbzpZOh6NdPzoyRBBsrKipeVEEwrYSCkFbNh4cIqVanlE9smhmFprTFFjuMQ0wAAQG14bnBxPbANV0s8G6fXF/OyrEx7Nl3Ep6enaZ49O764c/8OxPrjTx5DhHac1snpq1UcJXlKDFoxmGQp0gZXrF1vcJZdTCYIiMC1G1vbqiyv54vFZLLR6/Xb3HCDB++9//mTFxN+/fXJWXPrHqsu9g92m21Lqezl4w/iZHWw92A8U/NlUPJWq+ZT03jy7LdffPHozbfeml2eJ2V+dHO7v1O7Gg0Nr8R5WnNpELShIrZhK6HOzy8VF4bhEYoqUXFRmpYFIMiZAJrKQiBk2qaRZ8ui4ptbrR/84B3HLs+/+lprNZlOuJJvvffup4++4KIECAwmg2a3Rx3DLq2KCWoY5+eXEMLx4OLwyN3cbiyjRRg0948e3H2Y/s1f/7xZ6yiTPDt/dfPgVpxn0+Xq9u3bQdgtBPrVr341evp4Y2PDqXtPXz69GJ7dPLpxeLC/udHL4+TTTx4d7R71Wnt7W3egdLDr5FWpqMSegQU6ny8GafHBo2dCCE2dVVGGDrl7b/vp15cEZN96+9b929sXFxfvvff+P/9Xbz99+jQry+fns8Hi703TqIUuEPxelm7tbY9/8deJTB/sPGjXyOir8UV0rSWwpGOwyixyiuhotuh2WmmaEaRfPn8cLcfTyfn7771XVWGSJJxXhmH4jn3n1mGZx5qXokxsilieZSvudjoUMMGL0ESVQ4qVogZFPgXUSYGKkmx7ezustaqq6iJjvIhPjo8pokd7u6vRsMEDv+K32p2H2zuGYVwNBr/4+199+GuZlUWcF4ajNOXLaGU07DsP3+jf3jehhNBRAEgpE5ZalmXZDiZYI9lo1KbTqVKq3+7OZrPlbOq7buD38rwM27ZvuovxjBWVazm8yLe2Np++eOr59NaD/ednX5kW3LnbefHySS4ntuZ5ztOFpNja6e1VjsiiOEpGRZb6geNYdlmtKsEhRvWGpwS9urqqKtjrlSpjO51u0y+pYSGTHh7spWU1WS0sz68ELxir9ze0LC1OkyznTLTrzTvNOwDAKE7XXWgm2SrKcIDbNT+OV9PpFBRmzwl6jidWsda861rW3pbnOfcONsPQjyZXz66uIUSO43HO86y8unyJWHb16mnTNxfzJZdKFFHOWLPZ1ArEUXF1taAEZ3Fqm0a32/Datu3VqB3cuW16fuOjj359/PolVIgzoZSyTK/iYrVMXNe0PTdZpN0N9ur505sHN/Ks+s2vP0GI7obd/Hr+YOvGub7A/d04jRHAlmMCg8RVpdEiyap6r9/Y3j8bR59+9uizT79QxHVrjVmy8ix65/7dvCrPL65bvQ0OljXfQKF1enIehnXbMkaDaxcqlsYlUFWSIAQUFw8ePJjMJynjJ9eXNtHQM4FjNrc2kiRRxDANu+5YhDhHB3ek5AUrQj+IFytpMF7xRJqPn5+/+RZrN7e0UgQ6+G1ndD4u4qooGcSQCQ4NksgSucZKFgywZbT0vMAgJqSmY3paSKgVhoqzvEwjIGKPypKtijw3DMKEBkAhDDQCnFcVLyHBhmnbticEZgJyASUnTCC+HqYq8Y/MK62QY9YZY2Wq/nF4tqbRG4YJIdRCJJEQAnIGpOQYK0AUK6RAWVVVjmGAEhjIjZaCUsu0fIA9YhmLiL18fSUV7vd7BrYA1NTErKziVZrFiZQaAxNBpLSCiLJsprXWgruWCXxvPcyDEBLD8iAmUZTkqBxc/f3NnRu9Xu93vvfd8fXEQDDwakgjCPDV1WCz1yCweuPewz/40R/++Z//OaU4qwqTUI3wZDD9H//7/8kkxu07N3qt7U4jeHn85XSSYuUNLo9ZGrfqPcs0izjlTFalevrsuGApk9z0ake33rx556EbBgIKYGibWAwzhABGSAOpwT94SgC0HQsAvpzNr6+vpZTNZrNR96azoedb9XrYqte5zpN4Nl9dlmxZlmUtqEOMvnr8Natg6PfqYb/edJ1c+L7PeaWkfH12mqyioih6vZ7tON2N7ubW5mQ4i5IVsRAgGkKAKbUsS3KWJMladhJ4DueVaVAEsOcGjh0ggIUQWsOSCcexNjb7xHKXq2SZ5JPpoqxAGDITO7ZlqUpGq4VpmtPp5Pnjx7PJLI15war+9mZSxtQkezf3DMccT4brXGu300uS5NPffnb84phX4j//F//CMIyqqjqdjpBr+aQRxzEASGstGGdVwSCgBNius72z0+62CNWO62ICi6IoigxgYNs2NWmeFxBChBShkBAoZJ5mUZrmAIDHX331dz//ebfb3T88iBbLaLW6OL0Im41uJaWAaVIu5ktiGlwKAIBjIUhMgihFOHA9y6RAVPPJvBF6tu2G9TqCeLFcXVnDDQlM07YMKkV5fnpCD+0ia14VY6EXgpcvnj0rM6WVRQkMAyfwbSmq6Wzwi7/7KQb64dsPLYKvz854mTUDN5pPGp5brzcf3H9IqTW6Hg0H42boE0JPLpYYwyDw3MDlkqdpJqXGGK3JE45rSI6UiZuNwLHNLI/v3bt3cXWuYx3PpqPZyHatssyDwJutosVi9vr1iZTAdgLLxrduHwoh3n67Np/OIKZcaS+sYyIevvnub37zaDiZJyvx9NlotSp9N7i4uEqy8s0337QJffvtNz7+uDg7f+37nmmRqion06FW1d2bt7Z3+4vp5Kuvvnxw996DN+/tH+yMlqvPPvuiYmxje2eRJE+fPn324lWRZQAR33HHi2w+WmxtlP/8z378ox/+Xq/dsQy9WKy2trYIpWFQBwheXFz8+sNfYQyj1bJMo78N3D/8/R/3+1tK8MV8aYVeq9mkd4zHj58+fvzkaP/o1q3bSIM8Dx3HefjwDa1lkSWtVsOyjZfHx1fDZa0WHB0dNZtN23Y77b5hEARJVUqpysViZVlWu92p1xtSzqbTWdDpUUyUUnmazefL2WTKGJNCtJpN1/FMevXSMJv1Rhh4Z6evv/zis1an2e/3fT+UUgZB0G63/8kf/tHNW7f+6m/++tGTp3EcV1W11gkCAHrtTpmtCDEQQnwNGYRwra65eXTjq6++9lwXITQZzxAA/X4/SRKINGOMc+44XhiGBHdYVQpeViw1CGrUakiD5WxZqzuTMktXaZaWLB9ibRtGACF0HKffa7K8GA8vIbAcx3EcByCoNVASEWLESVawrBjn2ACGhY+ODhAxiGGYtruME0ppo9Ycz+dRlhuWORiMAAZaa86lkgAWpZXntuU6jrO1taWUEpK1GnXG86LIPNsMPQ/nACJtUwJERQlu1IMwsMMwzOLo5NXL5XLpun673ZWcAaWajRoAO8PxiBLz8uw8z4uSiTCse657dnqKAayyBlTSIDhPM8cytJR1p2NaBqHU88P+xs7uzmw0iaIoIthmrKqqAiHiuD41jKxIkyz6+OPBw4f7G5ubrXqTMfH0yfPLq3Nq2kdHN5t5DRrE8S3LNxXQQgvBq3mV3Xnw5v2H75Qc/f2Hn3z19ReNMPj+e28jwSgQ7VbTIvi3Hz86Pjv79nd/t7+xMZtdryu4YRi6rucHHiHE84J6syGE4Lza3t7CBA6uruJ4VQs8U0WUIMuktm1/8cUXL18+b9UbP/rRjxzXEtyllLqu47su1HqxWAAA+k7bMAyI1Odffv782TOt9dHhje397Q8//E0puGm77Y1u2GwBgqVWZVkavgG0ppis644GJtgwCYLT8aAq0yiKOM8sA3AhGZcQYCC11gAThCgiGAFoAIwIplWpGWd5oRhHgsOKa1ZJwYEqs7VJhxCyxn1a1F4bD9ahAQPDNZ973c+s1WqMsaIo1q2Etd8cAGCaplKqLMv17GqdWpvP5wCD6XAGJWvWPCWZQaHr2hjQy/OTMi8Y4xhjDLDiUmuttJIKSKmE1ABiy3Ztx1tvWch8EdVrNWEC17Svzq4bTt02zHo9rHuBLIpmo4E0AgCsZte9lpMsr+/uvfGTH/3+//I//f8My5wti2bDQYgOriMCr//y3//lX/6v8vadg3/zb/8sWbDJdSRVdfjGZplMYRhu9HqVq0KvXlXi+PXVy5NJo9O+ffjmt77745t33iCWmfNYQESBjQDECAKgtFQKC4QwhjiKo9D3eFmMRoM0iZq1MAzbrkeTKYuWE9OA7U49r6qXV5enV8+JKbJisViQOI1Hw7lhBPfueu2O4bruxfAsyzKtdVkUk0mZrCIuxNbW1q2bd+bLBecCIbJ/Y4dVQkqBDbK5uVmW5WJZSim5UIIDarq1sGGbVmJHFCMICaE2JnodMGYsp5S2282wVnemyzjKqrIkEGEIfcetdFWWZb0eTgejoihqtXCxWr58+TLKE6GF4RhuI/RC9/Li+uXLl8evX7ZrjVPD/PzzL6PZokyzP/0nf9xshVG0bPc20rQUjDmuF60SCHGn3U7TOI4AY5XSkmBs+a5pG6XINNIKQEggNrACiqtKlCXjlUbQMGG94dg2pobyfGpanhBssZhdXZyt4c3n55e+7/tuwLLKxHbo1GIjXYLYgJZpIEIIRsZqtUphxPICBCFQGiiIEJJS27ZrUQMqXWZ5VUrDcu7cvT9eLuNV8frlU9dwuy0fkYKYea8dWlTLKrtz89bQjUYmhIhpVWrBzo9ffOqZZRl3Nzrtmtdv1NM82dnovvnWW6PRhGg2v55cn10Y1GyE9nQ8adSaaZoYhuE4zmI5Z6yyLJsQZAe+EpJihYnqdOubW51a3W22Qkm2p6vZMo9KXl4Pr9wwGI1HtVart9lexWnFUstyEZZ5sWo0WoPBNAitpKiuRlNWSUQCxWGj3TfsYDafUwqjVE9nqyBszZfx5fVvIKHf/53vfP93vqMk+8v/OBCiCkKP8wpC+dnnX2Mkfuf979iO8ezFE4rVRrdHiP/Z148H15derU4xLtJsMJycnl0QREfj3Ntx64E7K7Nuq/3t998jEL4+fvX3v/h5nue+H+7s7fZ6vRs3b944ujObzH/zmw+vLk+bYRjrLE2KO7eOxqNBVcm65UqhW412v7eRJfn+/n6n0xYVPzg4eH1yfOvmLSn5YjkzDEMItlhFBFOg8eB6PJsuCSGWbVJaS5Ki0+kVRZGlFUKAYEtrvVwko9Gro3ucMYYxDoIAKu3ZTgkRBnqj0y3LkiD4zlsPfd8XQozHY9c2f/XrD28d3djbOwjD0Pd9wzBqNWOz3799+/bl9UBC0Gm1AUah86osiuFg0Ki53+jvtP5mMi4BpOr88qpIMwzXYVa9tiTU6/VlOldKZVmmxIiVlWmajmX0Oq35QnGiNro9io08zna3NlbRzCJOBVMloef6Da9jWbZSAGkAoV4ul1WZlmXRaNQoNSGiEmnTspaXJ0UZBUFtY6Pj+2FZVhXXRcmEgq9OzrFphc32eB7ZhtvtbRRlNUqWpmnanmUYFqWUawBYhRC6HlwqJS2TBoFnGQ5FwMCw26g3qFcUGUJIVVlVSl1lgrFM8/l8Ph5N8jwPglqyXFQlVwpYlkUMjDXodburKLIMsyp5t9X2PC9ZRWmczKezKk0xglrIwHeVkL5TKCDDetMy3Vpj487ddzUMx+PpyckJJqKscoy17dkaCM6rnt9pHlKTWl989bllulEaG7YhlaBQzJeTrIw1gwBCP3BLVsTzlWGQe28+cH1nES0Btm7c3G/Uw3i+5FnSa4YOgr5Jh1eXs+FwMZ6+ePrsjYdvOY5DKT08PGy1OkVRtVotrcDm5maWZePx+OjoYH9/97e//Wi5mvd6vfOz13s97/BwHyHU6bS2+xtKlFJKg2LbNLlteY7b7bUd21wsZ1xUjmnNFUFaZVk0nV3PFuONjc3do13b9fQnqCjZcpkVWtQ5dzwXG5RQA4qMEEPLcr0JEJBRTDCEaZpXZZ6nRcUKZWENtNIEQKNiknOu12U6BKUUEmgAhRCqKHlWcM6A0kQKyLlkFTfW7gWMFQVQYs15BbiU0rKsNZECIUywgTWRUkOJAQJr2ccaRPaPpgal1PoVAMB657EmK0ADGAhCxYeX5+dV4ftuo1GvWJHFyTcwcmIqjJXUWmsl0TopvG7eGoZhWZZpmpRS0qj3MEKj4WIi5+kieVI8yVfxd955z3Wsbrdec716rbZarRqh4Vp6Nb+ajE+bDetgv0UIcB2xu7sFNHEd0G5tZulkNptxtpr8+DvT4WRwOd7a2rARzKtsdH1GNG3U+rVavdXePjh6cxbD/Rs3Hr757f3DNx2/wbXUWBgmqaIcQo0wgZpqrYAECEMIoGObAGqEges4Gxsbvq9NU0LNYZHFqwkrIyH6abZ68virs8uXjY7PVVaKPCskq4Bt52cXFxpY7dZGEATrQDVCSColgS45i5OMCV6IsmbWhBBMFRwIrnh/o+2izePjY6VEs9N2XdcwzcCv3bhxA0i1Ws6jaJmXhRCMYsQh1FpBDAXnSgHP9wm20qR0zaReb2VxWWQlECKLIwKIH7g3bxzubm3/5qOvP3/0JXn2xAm8QpS/+eRTL/Tmy9nlxfl8Mg0Dr+GH8/EMK7CYzSej8U6v6TgO0FoKkSWpFMoiVAEE1tg2hCilmBAABWdVKZllkvUekRjYcH2peJYl616ZaZp+YFKjrhTAGPle3TAMIWW9Xu92u91eWzA+GY1brVa/05+vlqEXxG5iU8uAholMDQCUEEquytLwnZoXWASXaUaxbtTqZZpwzjGAjPGSi7LiUoH2Rv+dt9/44rOny3l2dvbcdWF/q1awoRBpr1czCHr3nXv5bX3y+uxq8KpkwHZb/V63Hhqff/xLPwwevvXGH/zutx4/e3rz1p1mp82SSAM9KaJ4NqrXm0Wcnp7km3stzrmSEkNg2zQIO47tLZeR42BRKSXLRs17cP9Gs2mvFrO7d/YnMfXCQE+HtXaTK6GhZpLNltMt3yJUBaFTq9UrJharITXAxfkxk8Hmxhbj8tnz4ydPTotc9do7j5+eagza7S3DiCXECpBWt/fq1euPP/2sHlp3795tt2q7O/3ZbObYtLHZrddr29vd6Wj85Omjrf5GvR36dY/J8pPPPnn02SeWYW5t9GTFJ9ejUkgtgYHMdk1JJgSTjgUQ1KwoXlyel3mxvb17dXU9Hk8QJuPRZDSe/PjHP/7hD388HA5ZVfVa9d3tfr3eXK3iq8sBxrjT30jjDIfk1q1bh3uHCKEkidrt9tb2xq9/86vDw33DMPKiuLq+Xuc8knht6YSO43mep7Uui2q1jIQQSkvOebvdNgyLc06paRiW5CJeRXlepmHmeZ7veQalQOskXi0Wi8l42Ov1Hj58OB6PqzLb2uydDa+TPDu7uKiH4WKxuLy8bLa7jVbzcG//ZP+gFLxZrwMAdra2bWK8fPb88GhHSg01wBgjACGEBmaCUlXxu0c34zSVUu5s7T5+/PjZi+etVits+QahtmnNZrOzk2OMoee4jmvWA5cLtLu5ZSBcpoWBaJlUUEBe6m6nZxu+ElpzPRmNB+fXEKiqKqbz6TJC1CSdjqckZEIS07I8ajoGNqBCAhuwFbZdJ9CAPHtxbJqm1EByYRLTdcxWvZnmpdVqIYQoJgghJbkWkiBoUpIy6ZjEsilSglUVKzIopYaaMcHLXCmleUpNgoGEqsqiDErR77XXqADJmWtbUZI9f/4UU7K5uSkZB1oXRSE4T+OEYlILwjLNV6tllaW2YUAAijRZzOZh42anvxnW6kqS2TRarArOqdbW0c2HGOPlahZnCyHKVTwFQPQ2N3/43reFEIPr4XQ8VkITA8dxyiWDGGZ5WnEOEPTrdamkSWmn07lz77Zh2UrT2SodDi5PX56Mr6+qaPnP/+Q/uXV0sJiMXzx5vLvZ73b78zjb3tq6nl3i0YRSulwu4zglhOR5fnVxsbW1ebC3e/fWTWrgq7PTsBZQqJ8//rpp397e3HBdt1kPv/+73709PppOpxCpLI2LPMVIAVm3KAFSZHGkbGbXmr4fdrv1JO2torlfsxWQJ+cngEABZFbkuRBxWRqWTQyKMUZYt1otoCCEEGjIqwpCyBjLkkTIMolXZZlhoqGWlKJaDWcpW8tiCEWYUoiA1hpAqRQoSl7kXCkNMUSQQoUxhKHjrBd1pZRiWgj5zQIv0RpLCgmCEgEAKTSQgZdZth4k/KMebF2hXDvGTNPknK+3DpZlOY7z1dePer1GlaZZtAg812mGihUsy6QoCSEEYaWYUkhr+A84H8iFqpjQWmuAsFAACiE1uXd0N0mSy5QJXn3rrfdePH0yHUzUQ8l0ZVEjS2PHNqfja9uhCCuE0PnVE9937tzf5LwIwp3d3d35LHJ8aRDnxfOx5WipCoR1lmWfffb5xUUDmGBnZ08rdHx8vLUNu919J/Bv3H4wT9Wtuw/3D960rAartMbAsgxMcbwoCCEUQ4ShklIprjFFBqaUAKCAVAbBlZarRQR0hrAklVpMhxJoACqpZVXmGCLPdnYPb2ADZlU5Gk4Hw/lstoDgoiglsZ0oimq1BsIIIWSYtudDYpmVqJqdJqDw8ZNHaZrubm8zUJ1cvTA8GCczoSrLqXmeAyFmgmdZ3mjUPRFmWRYnKy2FVFCzAmkQNj0iJQTEsjwCK8e0clqYmAosBGQYkyLLKSF37ty6d+d+kiRKw8FwnJVZ0Kit4jjnlW1bFWdJrLotmq4yIjFQMI5K10affvzpwVan0esDLnhZXV5eJkl688btIKgNrq6qqmCSGQZ1TJMYJiBQIWU5SimlgVrzwThnUgtEoGf5WiqllGUhziXGGEFUFIWGVbfd2d/fb9aay+WyyPIiyaRgzXpoQFRlWbpaFUkEBFtXLcKwdmNv6+Bwbzi8FlUpoTKpgRHo9tqua2shlRKmZQEK0yLXCCIs42SugTkcXTgu3r/xrSjPqKGEzFarVVEtb9x8Y2e3+/LYWkaXUncEd/b26rPZ699+9JFB8l5/68ZezzGkg1QrsPYPb+1vbw0uLl8fDzAB9RDEcYShtG2jXnN29zsHB3uGbT1/dnx5diG5UJy1e9379+4cv/r6Zz/72e5OH0HU6W9eTketfn80HysEavX6fDU/OTuzHNt2KKUQAMSYCEO7061H0ewnv/8Hlxfj169fn56UCALPel3koBb6eV5JAZeLaGgPO50OhHo2m+Vp/OrFs42NjR/+4Afj8bDT6967d9dxLMH52etj2zJCz0+jyLSsMAgc33Es8+Doxs7u3ni+4hWzbM8gZRSnQMEkSgUXNc+sB6FnO7hR93Y2N7cPh8PR8fFxUZavXr26vL5qtVp3bt96++23LZOuZmPbtvM8X8ySNZmjSAvf8dMo9Tyv222fn5+PRsPbt28WRQahphRzXk0Xy9FovD5PiBK2Wg3P85rNZlmWg8HAcax1wJ5Q3Gq1tra2EELT6TRJEsMwWo02ULAqRtFyVRWllJIS5FhmGkeWQU1KyjwTrLIM6tqWbRrtdltrjRHJiiKKIoTQRpwJIYDWm72NSok8TufTaatW39ndlVpPp1OlgEmN0A9cx4UAaKFVyVTJijiFWtXDsN5sz2az16cns9ms1a3Xut1ut9tq1EajURKvyiLNsqXvmIHj3NjbC2y3SsvJ1WR6NdVah17rzs0HySofXs1qNmKMpUncbbcEI4yVacqyLGNCpkkZxblpWqXgTFSIYSYqYuAw9G3HKwtxcLAnFXh5cn5+fZ2nCTX5aDAUSpqtBmeyqFhZ5koKixLfcwgwCRCuZWIgVvNZmWeu4/ieRwiBcWpiXIiqzDjUJqYUaJnGCTUNg1rEMSFEgquw0SSGdXU1ME0TAPDq5DXCOE0z1wsur69mizlCqKhKCbRnO7VayMtqOLiaTqfIysNae3f3Vlnypy9fPX3yajZbJWlWlqXneRIITKAf2F13K6wHN2/eYBLt7B5hbJ+dXQ0GAy0kpdQL/P7Wxmq1itM8LwsMETVprVZ7+OABgtK2COfg+vz1f/z3/+HJ14OdnvX+wweA56Prs6uz89PXL3cPbzhhSwiRF+X3vvf9quJVVQEFHcejlE6GI8YYr9g7774FgJqNR3fv3KQGTlbR0d52v7+xs7PTaTctyzQo+uzT35Z5JoSwLAsBRbEWknm+EwaOVmy1TF2Cd/udZujOAsu2CONFWaVpHk0mo0oqw7IsxzUtG0JUFUxKCRT3LW9Nx6KYrkWdRVEgBClBSgIpAWNSSuk5FAJnPFkUJRMCWBZwXWKaBjEoRoQJhgEmmACMCKEYUQCAlNqxnDVDsyzLNVnfMAwAQJZlEOL1t5Raa0UIsSwTFcV6ErA+664rxAAAz/PW4tD1sIFSuqaAOxalhKyWyyDw7t27Z9n0/Px0fWdHKUUQV1XFhIQAK7WOWJL1Vd0/RDLlekpBtpq9BTRUo0AAvnH73vj0MlosVtOIYlhkZZ6nSZJdXV1RSrnUgeuWPJtdDsOmU1W6128aDozz+TwaUZJejeYIgrDe1JAMRouLS3BxuYDWi+3th51u6+z8cpnGXj01HKu71b2Z3NrdO2jU2wQaVVUoxJVUmACDWNTAFBOoNGdFyRkhBlAKQaIkZ0VSsSJLkjQdY1z5gVUlMaUytL1ep+mFnuOipIi7/bpXtxVQ51eXg+EEQmzbdlGyFy9eaELXwlZCjOVqiRBqddq9Xi9s1BfL2fnzs6vBhec7y3xWljlCKIBm2LDDum9Sk5qIYrpKVp998dnRwSFEOsnissxtxzIpElJCCOJk5Tq+7ZgIQlaUeRqXaVZiu9NoGe1+UZSWYduG/cYbb0gu/vqvP3h9dsqVxNTISyYVsi0fEyILZhrAdWrxYpaCvFuvFasSSPV3f/vT7e3mD3/4Y1Hx4XD46vmLi7MLIMGd2/cE41oqrTUTXOUCC4gIQgbN+ZJSSiiSTKR5UpSZaZqdWq0oitUqLvNKK2gYlu+FVcmHg4XlhoeHh995XxFCyrLMZpPZZGRapmlSKXNerkwimjUrCHyMMaWk297Y2tpqNGrnr5/NZ5Mg8ELPUIpblg+gVEhCpAwbQwyibGmaZlZGAIugXhOcxNlC6tINiICo1nSm88lHn/y9ZTvb29u7B712iTFlo+GyYOPDm22uNq6un1HKdrd7Qdishc3rq8t66N+5c//ly5PnL/+mSsHBfu1qxkPXcx1UssjhgBqqFthbmw0tq+H1pARKSTmbzV48f1WV2b07d28chAgZrtsImuEyy9Iq82u1QjChVBiGpmkmSUoIMk23VgsOj3bPz8sw8J6nr1bLsipAIwDLWdGsOQSR0WBCKbGowYqc4kY99JWo8rR49eJ4e2fnxs3Ddru71d8MgnC1WtZrwfe//33PcZfz6enrE8O2Gs1mb2Nzb2fLMmi0XKVxYlBqGTZWEVYoSTLXsjzX2Oi0dra2TYqJ52CgF4tZELg3bhxdXl/VGzVPeKvV8quvvrQs4+BgZ+FZhID5fEqQJhS5ruNY3ka3/4tf/IKx8nu/8x3XcTb63XanORqNbt+7bTr2+cuXZ2dngitqisUy7jZa7Xaz02m7rj0cXp+eHe/t7XU6HYSA1poQFEXzx48vP/300+vra4zxRrfnmtbe5vZaXchF5XneZm8jy7I0jZGSjcBHio+vL7Josbe1cfvOHcF5HKVJkliGsaa+nl6ce57Hyipo1BAAeZq5lr27vcM5X1UrqFHoB712J3BcwTgvmJRyMZlezc5uP7i30d14/PxFlqRvv/nWZDY9PT1tt9udVqvf729tbuRpenF2cn52XCRVze32ul3HMjc7fQMYrbCllLIdz3fCfCVEKbOsgAIgANvtNoK82+0uokUleJKlcZLPV6llOmlWpEUpNaq4JMSwHBsAkKRRvdm9d/9OUZUnZ6ec5QpIrirDsFanyyDwPMeyTAggtk1KDQVFSqEAEqZZPhuPgNQ1z23WQtt2pWFxUUURjpNkPQ3m3FIgGgzHhmF1N/oAkdPLs+z5a8/1Lc/r9tuWZS1X8eZW/zCoNdud05PzLMuazSa1zDJLDYpdx2ZF6aS+mWdX16sXL69qzeM0yT/57LdnZxcaQQCQ7docFkIIIIFYMS7FYLQ4P58c9Tdct9ne2P7DP/5njVrgWNbZ2dnp69eNRqPikgkllOJcYo1tx1CSYMnLOCpKTgTb2WgFJr5349b33nm7Ww+Pnz/b2uq12j8ZzFYJ46bvvjw7aec1xlhZMkppVbDFYmEaxt7OLsLg/PQkT2OEtWeZjmsd7Gzu7u50WhZjTAiGkGVScnbyyrIsx7KKrArD0KTEpGR7u18Wb8zm0+vra1UkiOXRZAB5ceNgm5jO9tYGVxJAhRECUAOlkQYIEZtiCTWE2LXsq/OL1WplWZYQwvM8zipCCCUWpbZS63F9hYlDqK8BAUAgpCA0tTaEIFojiUBVaSkBUFApxSUTkK8X+ETJIAhCL/SVv16VlVKMMY00V1xJpZFGAAEFhBYUUMdx1kCF9UpvGAYhBEJoGMZ8Ps+yzDRNx3HWUA0hxOHh0Ww2i6Nsd3e339+6Hl5dXY8d1zJsy0BUQ1hwlZclxgQAUJUcaiWE4FIppZDSXOn1ZJF0wxZf5REygRSPP/9qfDHkFbs8u6CY2LbdaW+OxsNme9M0TUAdAQ2KrKxYKED9WitKitFkcT2arOISI+x4BtDWzt69vDA/+vhls1lvNBpffHEdhJ+8/913avV6Jfir85emGVh2c+/mbi0IKKW26UCks7zkjEMDIaCUgpwzxmVeZpxJy3IoIn7oqFIyABzTcjY3IQyVSiHgqSx3tjYarXa726OGgbGOs5Xp0Vevno7mo+PTk9F4QUzXMGGWZZPJ/M7DB0oC3/fX6hTLdQBGeVVOF/PZbDpfzTv9FkLg6cuvPc+5/+CuHxgUUQAQ0JgiC0KSxWWRZhdX54HvZmmUZAk1gGXbGEFq4MlkoQEQQgJNijLDEGGoRVmY1GjUastFfPvmTdOwhWA//8UvP/7ktycnS9M2EcZVXgKIKTUBgpTalgGKooKQVHnlb4eRs/AsM4mixWKxWCygAhcXF8vFYjwe/+ynfzefLvb294lJEIEFY3mcQaK80PcC16AIIY0poNQCGJCCEISoYc2mK1YpralWCGjTtVuOBVezAlK71epgTBkr4zgusnQ+n+4f7CKkTazbrbDV8qQUlmUIISilodsiFPEykSwlSFKiCFYKyfH4Yk2WZZWAEDNZZVXaaDT2b+4JjfIUGtT/+vEXX794tH9UX6az3aNtSI1FMh0vh07NgpgRB2AKNg9qk9F4u9/u9N65OLvUStgOunvncDqLj4+fK43+7ODWP/uzfx5n4qd/94uLwcrynf5Ws9H04nRxdvFitriu1esYG/3uTryKNjptXlYnL8/SuIBQf/zhp2W+TSk2LLNismRKA4IA9oI6MXCr1ZJSPn/+3LE9Smme56ys7t4+EmXergd3bmzsbuiqQE/mgzzOmw3HswzHcQAABkaK8zyJMJBIIyDBdDxrNFo3bhyGYX00GH/51Rc3Dg+HrrucTwnCO1v9/uYOIahg7M7tm3GSLVZpnqSKC14xXnKKjc1eH2qgZbU+a15dnIe+rZVUReF5gVJqMhkwlrc67SyP5vNBu928cXS4tdFazMZFmrg2/frRmW1T33EpwrysEAKEkHo93N/fdhxLCHbr9g3OpOXY3Y0eJSZjvCgKAHWn29zZ2azValyUZZXt7m5TSiGEUnHboZhAxzVu3Nzf3tlACFVFEXie5ftrOXJVAdswTYIXeXZ1cVZV1dHBXhKtPv3tx5PJpNNq3r37cDabCX4Zx7GQ0jAMz/Ns215EqyRJ/DBoN1tJkjClPMvONajZNaiRazuUEME4L0qgoGfaVZoPrq5t2764uPjzv/jLebQ6vHGUFXkl8jzPF7NJLQg3t3rdXts2sW2h6XDIqmI1neNG7dbhURAEUIPRaPDq9PyLzx7Z1KsF9TTOklVUrwWraKG13tnZcpbOKo4mkwkXmDFWMVWvt1xvnGfly5fHi3l84+DG4eFRvV7nvLIsuru3eWu6hw3IpcKEAADqtn10Y7ff70klqqqgFGsgsyzzXa8oitl0oXlDKWAQWmQ5K3k2nWJKyooLITD1682W6wdMqukyzhkj1A6bIb2eZIvYDYx2u1tvB17gb+0fbO/t7u3tFyWrpJrP5xAhrxZSk2RJPF3MJRfUsfq7264ZxLH48qvnabo6vz5Nq5XrO0oppgqtECWWlHAV5xpSBGkUJy238fXT1xRDjOC9uzf39w77/c2Dg8OPP/54NpvnRYUxBQBLCVfz9PHXz94ztm7evN086jW8GmB8Mhj2O00DSVYleRaHtX6735vleRIvljl78uvTe3eOWq1OFEXz+bzISpPQ3d3djV6HlcUnH//G89xa6MbRUktnb2dzfH11fRktZnPf93/wgx+0Wq2dnR3HMhBC19fXlOCyKmbz6f7u1sHuzsXBPityimAr9Mt4ocp8d3MDm07gWb7r+K6TZlWc5llSlnZlWy7UiHPe7NRsw6zyIo0SpIHWGmkApCp48Y2oS2uMDUgJBJQz1Gi1OJMQAEopQugbA4/UUgIIEaWGlFJKoZRCawsN1pZn1Wo1IURRFIwxVrKc5QijqqqggsQilmutyUtCCdO01wd9jPF65rdubyZJwjlHCK1DHmtTued55+fzyXTuur7t+C9PzmfzkeW6putorZBhYkQUKQVAWmOlVCkkVkBKKdYFTq0ppaaJMMGESF33g/Dm7dAPfvXLX5rUyqPs+dNX/X5ve9u7e/8Bpkar00mylDFWcgYgwSSASLQ73ePXz2ez2DBrWztNy6hxdvnVo/Pvfrfm+5tJAnu9/SRJi9L6D3/x9xqRf/lv/pTJ9MXxC9vyej1je3MTSMEZqwV1g2LFSg1Q4DvTaYIE4BhyUeVl9g+bLA8AwDmPomVVVf1+s15vz2YX11ennXZ9a2c7DOpFJUbD69cXp4PRdaWK8ezqejIomKrVa47fYBUwTXJ4eFiWZVmW5tIuS1Zy5pM6BDiKIt93IYZREvs1q7fZLVhUlPnL4+d37+6N41RzEIZN2wx4pSTXju+YJjVNmqSqKLMk0wByAIWljSAItIJRtIIKYWA2aoEoRFXKyXjIczaZzN5+802CjQ8++ODf/bv/kXNATeq6rgYIMKEJkgoApX0vXC2XUKr97b3FZLRuNPjtVqfVOjo6sm1bMpElKaXUNIxff/DReDh55713G51mvVlTGGQsRSa0PMdybIyzPM8rphzHsT1bSjmdztPoGiFiECsMgirnVSWBpqEbdFoMOb7gMooijPHm5mZZpc+ePZnPxt2Nju2Q7a1OvR4qJSpWzudTrbVpoMHwmhCy2e8axmYUL4tyfQQnrmu5rl8UJedcSS0lpxRryOrNepLMWp0mE+xqcPm9H9wVcMl5de/B7eF12t1oUxMzIbIiZklEDeDXaVktB+Ox49mS67PzV0FQ29w8nM1mp2d/1+ruvPHWt/+TP/mnQpNPPv1cI3jv/o1bt46W8fTrx1+8fP1yOh02G72a34Ra37t7N0vyk1cvm40OhurLLx5TKyOEvPXu21UphFSW7RSi9NxgOp/4Xmi7jpLA8zzDsKLlajab1bzm61fPNvv7333/fdMIz1+Pq1TOJolFje3NvlJgcHlhU6y4z6vMNIzjVycAgLxcXZ5f7OzsVCV//uzlZ598fnVxWavV5tNJo1bnZYUhuXl0+P63vgOW0SKKjo8vs/wlr1jF0jRNITYAgGVZ+K55sH90/87d18fPyhxooGzbStMVxhRAtb2z+e3vfKfRaNRCfzC4Nihs12ueQ22D1kP30ZefAahZyU3T3Nvbb7ZqrmWPpqNerwOAdhz76OiIVeLhW2+naZ4X7OrqilViu9d46623kiSJosgwyPbOxuHRfpZlX3zxmVLKcRzHsSml9+/f831fKeXSOiZwNpuNRqMkSbSWRVEgBAAArVZrLSQcDAbD4fDy8vyDD/5+Oy4dx+l2u91u9+LsbDwe72zvvfPOO69evfrwo4+zJF2D86Ik1VorKYlNBJNZllVZzrOC52Xohe1ma393jzF+eX6R88q27QZGa6KiF7q8Kq6vr09Pjhfz7TffuN9u1QLn7olhvHzx7PmzZ5u97tHRUbvRpgRfnl/EcTq4HL37zvtv3Hvn5Pj0Sulms4kQev78+cGNvaLIXr16xQVynXpZqari9ZYLNPG8QAp4cnKWJ4XWoNvdaLQ6mNCDg32ptdRqsphrrVdxdHd/7+bedr0ejifDkuWEWLZtOsTLijyJllWRhr5LTVsrVJZMyoJVlYMxwEhBxIWqqipOsvFs7rjeeDafRVGt1X345jt370mAyGKxbHYbzWazu9FHCGGDnjx/MVnMy6p8/Oirer3u2GaaRMvZTHDuWqbnuotlMRwvhpOx51Pf91pd37JJnCyE1Aa1fa+xiqvJdICR06g3EFDPX5x4Xm13a3MyHf3VX/7t1189ef+9d27eOKwqvlxGAOJWq05NWwO4WsXXV+Omm+5ubvk7tmeZBGjFqvHgOl/ODne3/MDOsiStuO05nb7JZ0sdRcvlcn/v8Ozs7LPPPgu88I179xECH3/8sZaiqqp2q04gOjo6KIt8eH11eXm5XF5altXpdBhjk+lIS75cZsvlAiNUFEWSRPFy0e926vW6aVFMYLWKiRJFVsaLpWFbxOZbO/v37tz+6c9+jrFJiImR6ToBxjRN8zzJecW01LxinIP1NJ5zXhRFu92BGCsFCCGu62sFIYSe53HFgQvWSzgAiJVVWZZS8bIsCSEIAa2kEExKiTHEGCPLcQNPQR2lcZIkhBDbtd3AW61WmlcAAtOx6q0GIWTddECKMsaEEFVVrZ0M64lCHMeMMd/3wzAsiiJN03XU7OXxSaPVu3f/jmGQDz78tVRi92A3TvNao247PibEKnnFlAJICcH1WrHwza+5Hl2sdzyEJznM2e0bt1zbbfwk+CUJTk5OVqu4SuRynBQxO9y7OZmNWVlt724NBgNMVrhY/fD9bwVhM8urR1+esArcvfPQNj0hTvd26v0NH8O4GcLL00dhGFq+TlP5xZfP7t1/5+HDt3aaDaUAZm4yE/2NIHCsMouARnW3XVV8crXUqIIYKw7KsuICGoYDFMqSBGvuuRaBrOYbIs+fnFydnry6e/vIqWfTaJ5w3tnYMyE9/uiLX/3mC00gB6zW7PgNKyvLeJE3G90bN290uxsXr86fP3+uXJnNI6jUzq0er6os5zjTb+690TFaT58/zwH81o0/GE3GL168+PLXjwAAYehn81WtHhgGTZLE9/1GYwuiAuOo1/cn49lgeHn79l2EbRfZlFJOq2gZS8FaoUuUGl1PDSzSdLG3t/fF51+/Phk8+urpfAkwwUFzS0CRZpFpQYpEVRUGMcskNgH0rBoC9vbencVyioI6ada337gX1HYhDhQsDu/cMayT33768cZmezAY2I+fvfvOe6391vXgzLMVRcrgk7blM01M23QcaxXNF4t4Plu5dvDli5c3jx74ftMwLKQrw2RczX/24V97nhMYNzwTp6s5IlgBYDjurQcPjo+PWxubJ+cjoGWyylzbmk3H8f+fp//qkWxL0zPBJbeWps1ch4d7qBNxtEhRlVVJVlHNNLs5GNFEY4AB5mLmpw1A9rA5LFazyGKlqmRmnjwyTujwcG1uemu19hJz4STt3gAzwAxr7e973+eJk343rO2NZYrF+ro7MncOJsuVNp3Nwu7Q1AMhYBj0XMstigwoLmSzWs6Xl/pimR0eP5jNLzoj54uffoZpaJg7EFW6Th99uM1JPN3M0zQti1ophY3V9tYuwKC/NS7ydjPPDXf87iLzAufDJ/9ktUy++/1b3xj/7E8+l+nKFvEfX73Bsh6E3s//9E8++eDzZ8+elVXz5s1bApBnG53QZmVi6NDUQtu0ju8eL8vrP/nZj5wwHI6HWSsvrs4NR3d1r+Oj5TzGIMGcesT74PH7b9+ctHGDsKvpVEK5c6dnmo5m8LdnJM7KRkqEOo7THe/sE0Jmq/Tu3QdZniTLXNM0wJssywa9/v72zndffR06XjxbZ4u15IJHeYD1sds9Z7AqMp20umnvHIypbSldX6flNFnPl2m341a8rDbFncPdX/3674hkwd2DIkv/5E9+IpSkVP/ZT35kGCZAECgEIewFnYuz8zZn/e4AAp7ElUb8PG1LHlua1R16CCHNNCzLuZmuA3/gmL0qEb4fzC/mvJWu6xqKrpeLzLV/89s/UErrstxsNppGVstYI3R3++CX//nvPM/78Y9/vFzOG1wDpgaDAYKQM9FU9eXlOee8P+pH6Top4ztHd4f7O0rCmrXXUWL0RkPNeX6xuAan/W7v7l231w2x2cN6QwzHsFzDsAbdXlnmiDNV54GBCSsNwDDveYE3m15BCF89fZUl8fHxseu6putCSi5ns7ppkyiL0kw0iBBtBqadThD2QiVsLsrnL77ZGQ/uHd0ZdHS+FVy+/U5r9+/tdE1gzc6nzeZ6FWXD4cjuhdLE3f0xtyjnIpI4J/5NhotMF6rLebOaR6xubNsUjYYhCvygqhoAwNXN9Pzy+u6dOz/70z/v94e2bXcsZ38wDDWrrqplC7JGSWoYQaiXeRsv4iJigioggBIIFJTUCArU8qYWLWshxFRvN9EUY2JZdlPXb16/Y61qGwSR+fD4CcDo/PzcD4Pt7W0h2qaN87YUCbS7ge+Hf/ubX331zbd3j+7N4ihumyJa6zmhlGrdDm6bOE2vpldA06GSF1f1sON8+PDu3qRHVKO4BTj3Xc+wzKpky206Xayub37Iovzw0SfL9HT97HSxWHAmNsWqZHktudEJw8n26fmZTQBxjbKpu4cTBQDpmQumwqyMsjTN4qvrMyDVJx99Cqhd1sCyXCHh1nYvqJp+2NEFL6r2YGd3enV1eHhwfHzsOFaWpZez89B3hWoZqz96772u7z395tsfvvlGSlnzxurY+4ODq9fXdc0GwW6SRM6wU9cVUJC3MJXlm3dvgo5fizoc+G7XPz15vX/3vqmb3/7x+fuf/DiOeFZVnrt7c/NGIQNrlFoGpWQZz72eEYvi9c2pt9WDjgYAwhgLhPpbE8MwDMMwdQtCeHumUkwwgQaGSinJ6rwqEEKapoWeibFzu/W/jRfcbv1vM4lW4FRVJZjs+IHrulVZY0wtx55s7adpWjNu247juoQQ01ZCyaZaFQWApeIct3Vb5hUCSNd1JDEBtKlaxvjVzbRt2wdPHjaS6Ujv9XoWcV69ellEjW4ZVyeLrd0JFiZFNkEEyUQJSDACAEEhm7a6dVARCnVDp5QCKKo6J+cn7wzDaBtWSmUb5nvvvTfsj1ar1W9+85uiKF6+fB10/KzINslGNw3LciBGRXl9PZ15fv/Jkw8wMhbzuBv2MdKTOM3iDEOwXi9N2/B8J042wXjC6vzNq7N//+/+DQLy8PCoZfLV8+9Zww8Pj+7du9/rDgAmeRYXRVGWFRMrQ7e4FEGnZ5laURQU6whwXcNZvk6L5PTi3cmbZ5v1oq6yrNxgPBcc3nv0GBL3ZrFq23Zvb69RDaZAEQGgNE3ftl3X6QjJLi7fxVm+3Kw20VrX9XtHx/1B7+r84vT0nWNZZRAMej3tvfc0Sn3HcXS97/k/vKuvr69WdWI7pmnatmErDubThUFNz/PCoL9crvO8bpmc3cyPj+9PJmPO2ngdazoxXNe3g9BvTM3TNWd355AS59/+2//wi1/8Jo6ylgHBRc2Svb2dba1r2XSzmp2fpxAShKHluBRqm3jD5g0AvD8Ih6OJaVpRslFQ+J7T63U6oZskyX/467+RoG3bZmd3IiWXkmFMXc+2Te3i4pwCOhj0ZqczACRVpEnb6elJzxtcnU4NFECbNg0vynw5K149e2ea2ifv7/O6CWx3vppDik1KbNMrB/2qLHSMZ7Op4MzQSJkXhq7d8LWAbVXlRZNZ7o7nhrbj7u0f3j1+WOTs4vymKBpN04JgR4omzSIl5PW7M9s2snx9fX2V5+nbt283m4VhEgCVpgVAkapshYCBPxiPTIIp1Ba2bS8X67KqhJCu69pmB3JLKTEY95SCL168ePbsu8fv3/V8w7S0yXZnNj397X9pbccYjibvv/9oud5E0XqziU3TfPvqdZ7njx49QgB4jvvkyePzzdOH7z1wHCcrEt0EZxev+qPBaBwqQINwjCH0PavbDU1Tv3O4Zxo6pKoos7aWGgRciTKrWc2ABAY1AAdVXkAFep2uY2nbW+OrK74x9aZplFLdMDQtPcuyqio0TWsNKnlrWkYvDIJuQDRcNflsNX/36sWnn33OJNwk2YMHj1qBXr85S5Oc1Y1lGHcOdg1DW60WWbSajAd37x72BkOMsa4ZCsFNlLx9++7V2zeLxUInNE1jjIBrmvu7O+Nh33bcNImn0+nOzl5dsyxbB0HQ7w0ppU3T1nV9cvIWIZznOQSYEHJzc5Nl2fXiGmN8//79O/v7dV2/fv0yz/MP3/9genPFOdd1HSHkOE6WZS9fvkySZNDdtm1LSllkpW2boec33KjrGgOoEXp1OR2OJz/98Z9QYv7www/T5tpxnDjZfPft1x0/8D1nMBjYtj2fz5ngvV6nYjbGOC+Ktm0xJYZhaJRADHTLFKzJyizJs6IoqqrKFgvPcX/8+RdNK757+gO6vpkMhphSkNTvP3pycLBn6ASoNt0sWV0IqQbDLSGE4/jBYKQ7gSJGC0gjERNQIRoEw95oax2/fvn6pK7ZoD+yHBcTTbPN4WQMJC+ytKlr13W9jlucnEwXsyTOCCGu7QrJVnEU5WnQ7SBKdNMMOh3H86qqERBcL+fr+cyksC5zxQVvm5zVUvEsiZVSGGDetusoSuMMI83zPKGEruum47RMTW8uV6sIEw0RbXf/oNfrUUOP4riqqun0CmMMhEyW68n9UWA5m8Xq+vwi3kQYom4YLH2fEEQIIRhKyaVqEcFEw1mZ6BQSoCDiEHNCpU4IkeZiNlWg0WpDAeIExvuThx9/ZiJInr6czW9mSqmOb/FWiqYSvBkOOu8/eY8inBdpNwjvPXzUMBanyWq1yqJ8NVvb2Lg4uy6y0jFdBCEQMt5ESZZHUVKU1aAsCNXjNLk4f7d354Gu6w/u3UcICCGm02lRFAihMAyP79wd93uq5RdXl1yIyd4OQZj6CALkdoLZerlYLG6FI4ah7+5tAyiDwBGSQQibuvIc27XvLM4uFSZMCtv3hlswq/Lf/va3JxdTywu7g55hWVXTFHUpCoE1OpyMc94YhiGlHPSGmqbVdb1ebVjTDIdD3srbv7YQQnAFdaTrUCjQthwhZDmupmlSSiGlksrx/P/ujL5NJ2CqUaXStLjtEntBICWI4zgvC865aZqYUgihYRiEaK3gEHMCEFQGQsjUTM5llVdJm5Z5mWWZbdtFUZi2eSv90TTCWJ0kUVHlWk6bi2oTrRHFECkuWBRFW1tj0zQxhrqua5p2i5e2LBNw9N802ei/lzARIsS1HaWUanmS5VmSvXz2Mk1yz/OePHliGEZTVs+ur5lkhm0amrl3sD9bnZcFf/v2rNsZbW/vD/9svFhsbN1J0zJwvbpsHMeFSD158qSqqqdPnwJTGz16cHNzE2/mr18+3Rr1PS+QvPjdb3/77OnXvV7P9wMpQNM0mmY4jqPRBmDUtmJnd1c3rShJgk6INdzyKkk3ZZU8/e7LFy+/x0hqOk6ajcyW/eHk4896VLOvps/jNLdtm1fMsrWKZQCBXs/f3t6hxL66vDk/O786XwSh1+l0DKppBl6ubkyL3n9wd7GcahTePz6+f7hfZnldVURKl+L3H35UpawoMqp0DRodf4iBMZ1eJZt8erk0DCNJUtfxvW5ntVj57goLTglBEGNCLi4u1vPvdWJ1wuHD47tVIb/74dtf/eK3q1l85+BIKBRF0dbdwYcfvUcp9Bzz3enrsoos00s2RZqkhg44l01Z2Y6p6WYU56uvv6mT7Pj46KMPH2tEaxvx6MmDL7/8fRRFeZkAJPIiYW2tC3V6ek00kWWxw/Wtncn05ioMAyklxgRXQKr26uTK1zuZlUkpJeAQcdewAt+bXZ1DwcZb45urc90wvDDwHG970JdA9budMk1urpOsbSGEhmanSTVbTS3LTMu0al4Fnc69B8et4G/evL1/70mn0wKV6rppmnYaMwTp7u6dfndX1/XFcoY0YVrvIQSuri640kfjQW8wwYheX183Tbu93ev1xxhT0/KklOtlzttaAWJZlka1KMu//uZLKenh8V6ULs8vX3/z7e9sR79ztN3Q5sWrl2cn8dndbU0HXECguGObQKnNOi6rQtf1xXKuYfLxhx+GYXidCcPEQWhukhvH1bN8U55nugHv3r07mUwoonwsFBdlWYdBz7X9aJOwUuRRXcTNsEeHndH+1kG+EnkqWlaXdSm40oDSBl0NgH7g571+msXL1SqK1rPrK6WEZRlB6I7GXYyxZWhQSWoRYqKa1W8v3v7yN787uZof339gWkG+SO4/ePzjL35UFMVyuW4EODzYGU9GP//5nzVN9tGTR3t7u3XV5nkUp3maZ1GUrDabLCsgoGlecg6qpi6zcjAYlDU7OnqQZYmU7a3CrqnblgnkoNtHIk3Tbs3Fum74vg8hoBrudAPDsZIk0TSt1+vVdX16enIbPnj48OH1xSVC6M2bN5wzKYSmaa7rxusNbxhCME8zoARUyCB6LerxcKJp2lfTb4q0OLpPNERGg5FlWNM6hwowVq9Wi7rKUH8Q+HbLGSFkMB4VRaEZuuv6jDe6rpuOzRWvWUU03LYqSZIoWm+STRStj4+P37w5qYpia2tn2PvzumZlw7799lvf9u7uHN47vKsb1LGNokjydOO5TpYljhfWZe77vu73Mg4uo+zdYoOoM53HL16fW8GgaGRe8aZpOQBI02reCqAM3zIINmxdsNZxrevVjGMlCGJItFyaWFmh5wRuyaqirvS6qnlDNGraloKpBMLWaVPmyWbd8hopCaQCUiCgkEKMsaqui6Jcr5I0zXXdVELMorVje8MhVIAw1nIhMEEY02i9MQ3LJxhBWNd1w6putzsZj9M0Pd7d14n58upFvFgZEHccz3Wc2WyGMaYaphRDCFresI7PWWOZimDYNqWOONFEWqxkW0LZAsDzplZ1hiD2/K7l0eFgYLuObvi/SucXF5cYEoqokhAD1vGcUa/z2ccflEVWN226Xp9fXjUt932/zMqbixkRJI+LpmCiBa5jDTrdtm3LNGGcl6yFCNS8BQgFrm5ZVhRFnU5nf3fv8vKyZQwotTPZeu/ho/FwpES7XG+qqsSWoUu7qepVvEYIDfnkfD6N4xgqcH98z/NcPwzSbGNZFiZWXeZ1mXm+47me9+gRtmxi2K7jucNJkrfvpm+ibGMGLiJQIWnapuHoSinOuRu6lDtVVaVpqoS0LKuumrquJedv37wTQji23esOLNO55Sdyzk3DZ01S1QwjRQmlhNymDqNNZBiG4/i3GQIAwO2A4V6/yzm/rTAUZVY1jMtbtAJwTMs0TUIpF0xVDECJEKLIpkRjqCnL+r8CRSC4LTgAoHRdZ6ymFA8GPYiAAvKLn36KEHr27FlepYjgmlUYwzTdlE3ZtrWm2YZlaMXtOoMJ0QohWy7alt8OP6jQCSEQSoKh6nb7R4d3ZrMFK2qlFGNM04wPjo79IHjz7uTNl3+EGLz34WNCDaFQFBdpyZLrBQTayZuLXtBvW7G3sx/6nqHptu0CAFomvvjJZ6ZpjbaGTuA9ePCgLMuLi0sAgGmAwNXuHowM+gVruG2a3U4PIRTHMWPMMHQp29VmXZXN1384r2pWsaZuGsPSu4OuBE3NiqvZOTWR69oNKxtZjfs7H3/82eHR/SQvzs8uV6uVE5hKCcZKqVpDp46r6QZhVZEky9XyptPpdjodKaXt2ft39kPfZ3Ujbtqzs3dJvMFK7ownTVWbVOv4AdCNnJgdp2cRi1DUFpKX0jb8js+CIDgvzq+vZrbt7u0dKqWSOG9qdnH5TikYeCGF2nKxfvn81dZo/8HRB4ub9ZvXF2/fnEMJXcOMNpuWSdOwLRMS1FZlrusu1YBl64ZOlqLOSqZASamp27Zp21yBxWqdZolNdcOxhpN+6NvRajUZjz794tPNZlNk5fdPvzk+Pm6aamQHX339qmhSxzX8Sq6nZ5Rilq1Xq9X7H3743tHOL371Xz58eNzpmvP5vGIVwtKwSOCAjo+u3pxjWR/ujvLVTUGwDlulKwMIrFEDgb3xcDWfRZvMsb3FcsMYw9BijYyjch1Vr169dn0v6IQY4/V6fftTaxlvGS/LOopjJUDDC2IElqvtH+7qur5arQCkZdWORnvj8XZRFGV1maX1YICEoHUthFJSCi41olmKKwVQXmbnF5c311HgD/70J5PPv3j/+YtnN4uLPggG45CTyYvnX79+mwH4Vycnb4bj3cnk4PGjB0+fvSryKs9zhFBdVr0w6A97QraMSQQ1KZFh2Af7xz/9yZ9fTaeO1ZWcAmlirAWdYHEzf/t6enRkjwbjdFOOehOEwWazkRyGQfdo/262ar+dvQCCepa/t7PjeQ6AAvJWA+r+8d2mqV6+eZ2XxSZa9UZdz7eXmxvb8/r9vm2aV9cXm3SZN0Ml+SqZNwh88+KUOPbWBL58deJ47vHdAyB/9u7dOwj4P/0nf3nveP/O3mC+uN5k8fl/Oc8TJKUECGOMhYKE6LaDAABNbdV1xeoSKkmwfn09G42Gg8FIycoyHdfxeSvbtr28vDYMY29vTylVVVVRFJbFbdvSdZ3S/5qm7vV6t0KHwWDw3nvv3W4uT05OhBC33UgAZBxFUsqmabphT9MIxMg2bChBy5iCsiyKp99+NxiPdnd2GsZn17M0zVjdmLrVJEvfcW3bUUK2TRNF68B3et3OrXyorkuM8XA8AhgAAISSFSu5wFCBljdNU0CoMBS6hgRnWbopy1oJBjhQkoeudbC743acB4dHBtGj5RowC2KkFGatdNyAambDaoJpyVEcx++uFm8vFhz7SZwS/aUddoRQgBAMgICwZQ2CQPEWQYWQ0h0DAQ0hBDQy2t3GGFdFwVtpm45pmpZhcahm63mcpVVVcSU1rAEEFQQaVlkSNVUmBK+bsm0rjDElSCNGlhRlWeuavb/fbZo2z4qmaUzL41Ilaalpum5Yngersp3P54ZhVnXTiTpIw5xzyzJCP9jd3grM+1uTnavpTR3nfS/o+J1ht5OXReg6AiiEANUpoQhCAyOAMaY41Slu6kK2pWkhhaSEHBPFRXtbrAcA8VQ2olptlpTSrcH9O3sjW8dtw03T0rCmY/jDd18l69ViFS9vplUjHNevswIS4tvOPMnWq6Trd03dQgDXRUkCfzweG5peFJkACiKSFPn56cnW7s5nn31M7a0syzDGhmGYuu5atjKt4aCfJMlyvqib0rXswHerqgKM2FbAGimEII69ztPZcgWVuHN81CqRVsXNfOnaer/XzfM0TVIAAAKwt73jFkVeNJBqvuUl5QIi2e2GlqVzoBDRLMemula3rChLAQGllDGmJGgZL1XFGEOIOK4TxzFCxDTsbrdr225RFHVZEUK4AI7nSSk1TTNtG0KYZdlssej1ekEQDAYD3/chhEVRJElSlqWmmVLWVVPeykhbJjClt2FhhDHRMCFQAIgpVlBACJDSNKpzzZQyTlX634AKQAguFQeQR/GG6NS0tjAluu7t3tkFALy7euvWNsY4iiJISCtZmkV+4NiO6dhmXdtpKppGSinLsrwFLgmuEEKENBgTCCF5+vTp8fFxVjyM4xgSHHR6l9fzs8uL+++9F3S7xnwW9geabfbH20UrpvMFY2h7+07gZQjS7797ihSCAKjPhYY0JsTuzr5h2etNzARzdb83GvRH/V6/g3F/e2eS5zlnLUR8OOpMRt2bm5siKy0Ldjrh3k7/9lwJQuvmZm5Y5osXr+qmHY8n09nMCzzDNixXPzl9E6c3AugQySbKtnYmf/r4J4+evJ8X1a9+9ZuT07eAQqmoZesS1lTTLEvnbX19dZbHdbKJdUr6/T6ltK5r13V39nZd1704O5+tl5O9Hd9yiEavptfLm3kvCI27mmPZqIXb/R3Hc6Tkq/VMNtKw9UGnjzDe294TQvX7QwzRi1evbk3NvX4vXsdRtDaoOZlMht1JJxj0Or2z02kSxZ7j/uwnP2mYmE4X11c3juPNZlPXMzmvy8rNsgwACSFElHgBEhw3LSdEqzkvGtbpeqOdLUs33747z/P85z//iULIdOwvfvzFs2cv3r56+9133x0dHcVx+sX4k35/XF5kvAW+qykgBgM/KxON1HW+6BxOBiEedFFRXi5nJ5atEQsb1OyFjUZWvK55gX1D+SYsikSUlGUKYOTYvTJd9gcTx9DnvL2tztuW63j+ZrPYmmw//uDecLsTReu8zA4Oj4oykYK0vEEAaDr1PGe+uH7z5o2E5c7O3u7u7mAwuLqc6pr1+edfEKJJASDQlWhFi6SASpKmlnleJPlGKcFaqVGTgUYKKRUo68KwCUSCyWqyO6Ym4qyQirG2GvVCzzVtPa+y9OZ62jJQFtw0/YPdvfffe/+v//qvv//++8loqICbF/FoNDq++8R1+kVWO9ZQKvPBvU80/eTJkyfRJoHK5AxTzxVtNL2KPCcf9604i4MgeHDvqK7L68tptEnSNAVKdDthnQvHNo8O72o6jtcLILjnOKPhVl7ldVM2nHWCgDd101RJumGyUoh3uoFArIXNPJnplPgDr7c3EPPF5eIKUCRU8e13v7cM+/7xgwf3fm5o5MPH9//+t7/6+9/9OkmjnZ3tfr9r4T6mRNMMTCnjom7YbcqJc44g1DRNCQ4xYm07n8/zPN3dGmia0en0iqJar6PVatXtdsfjcb/f7/f7Uso0jaXkQRDUdQkAWMXRYDDQNE0IcZvKLooi3kS//s0v66IMw3A0GoWhDwG4vr6ez+e7Q5dXwvMdLwzrumoYJwQRqr96e1LUzc//wV8s19HTpz9ommbZLuf8/vFhVZQQQts1EYBStIzVcbzpdDoYQ6prpm1BSgCCZVnmeeY5Huect02ZpbxtKAYQKMe142i1XM6jKIEIZWUVR8l4a5tSemd3xzWsuqlYWUHH8DyfIiiB8H1fQcA5b4UAAEmg7e7frxj69ofXtmvbrl3XdVrkRV3wVi43Sw0TSjBBCirJJTQ1HUhRZEnQDynCQgg/DAxNR4gIJrIkdaE7n88ZY4ZmdoPQtm2M8WQyPnuTcM6VEgAAKYAURCkFAEqzoiyVYfh37hxOtnc55zc388VisUnSPC+AQgoQxwkcr9c0TFuvKaVCiDiObceybdO1bEvXKMK9oKMaUcSppel7WztcAccwkyi2da2oK1Y1StZAEEIQooQSBGWDoIaRkIC3rRJcCM40hErGMFAYEghBWVdpngt+AwAgwh2Ph6EfvHz2krNmPBlxLl8+f75erjAxoAKmRhFUgjMMAAZAQoAAQAQjBU3bGm1NRpMRImS8NdlkkaZpCpP5YrFKkp3d3YeP37Pc3Xdv3vKGSc6rqsrTzDRNTdNm0+WtrQYQvErj6XSqlNje3h4N94ssx7qV5vV0vnIsW2FNAHpydp1Ea31vR7dcGUeLTZLXDQcABaVmuhbUbMeBmArR+o7tev5gPOES1VxwJeMsi+NNKziCJI4K27Y7nQ6QSggVxzFjHEJ8//5DxphGdEr1W8R4UzWEEKkghLipmzTJgUIHBwf7e3eKoiiKwjAMJWGWFnVdR1G02WzyPH93eUUptWzDsixIMDV0CiQiWNN1QggXgksuhAAASAiE4Igpy7I0S6+q6lZfXpW1UooLlmWJUKysq+FkYDsmIhAAFaVz13W3dkbbu2MI4XQ6VUo1TYMpUJAryKmOHFtX0qAEOLa+IVrbtnVdM8ZuvbsAQikhGe9MXr87ofqvKdUn4y2kUw7U3sG+3+/XUlZSTvYPsEEU0aarte06WNHDo0cGwVABVrd5nFRpjoFaLmZN09iG4XbC9WaZZEXTMibY1fS6rCvXtX3Pm86n79689X1/OOgtFour84vL8wsp5fbWeGdn55b7VtZl0zSTyVYULS3L6fdCy9T8IBhPhsEwDHzr8vLNIprnVey57kcffHjvwUNMyffPvn319hXWsBc6CrKyqhxfIxoGUGZJkiZVXQnbCEZH24t5Fo7Cx4/f7w36puVwoUrOkW6MB8Ojw7uTwfD63fl8ub5aLFoFfMftdDqB7e/v7HLQ6gQz3qhW6LqeFqlt24HruZaZpvFquQiCoG6qquKdTqepWBbnFrX6YTdaJP/uy//fu7cXi+kSKAIh1aguhFqtl8sloGMnz2o/sIu8Xq02RV6XlUIQG4Yex03bck2BVrCiKbGOO/3ecpW8O33tOubh4WEYWt//8INlmHt7e8vZOk1ypeB8thQcvffo/aapnj//4fEn9y1bN0xSteXWziTJVlm6/uijh7/73R9uprMsLz797D3LRrbDuj6p67yM5wWtWbnc6hsLlWggJ1LXNUs0m/l8BYGgWGIM8yIty9J2vM06TtKMmr6u64NBzw306ez6q69/PxxsuU5HNyyTUs5rhEGnE2gEI91XEGR5Sakep6mUwPP7q+W6qhrTyqQEluNDTCBGNWuqpqa6LMuStZVl6Vy2GBEvdMY7Q1Gj7a07vUFXiNYwNOLqLa82G54nUT/o2U88oWhVqSwuWT3XjGrQ33Js9+ju8enpyenpCYAtodB2NGzvJnGVJIkfeqtlsl7Fm3VKsPHhh3ezLBOsdWzHsixd16NovVgs1vH84vot0fmD+/f3tW3GT6JseXl9JhTt9Ls744P9/d13b9+cnJyYBtnZ3eZhs5hO49VqtD3GSF5dn1V13h/1FeQKcYnE9u4EYVBVRc6y8f6IaXT5qwV15e7dAUDVZja/vsyyaPrzn/9D1+4lyfz5i+//8OXvqU6OH9x774MnPCNVVRV1U1ZFWdYVqxFBjulE8dq0bCVJmaWaSQ+HB8+e/vD3f//rzz784Pj4nuO4nIv1eh3Hsa7rcRw/fPiwKDIpeZrGcRxrOtF0YpqmC2kcxz/88EMax5zz9Xqd57nkglI6T5Kvvvpqd3f3wYN7YRhCCF3XZTVdr9dQIwDCtMiv57PBYNAdDpRGStYu402a51Xb7O0eHNwLqqqyev56uSyKgmLCGEvjREpe12VZapZj93q97d3dxWpZscZEsAWyY1kQqjRVN2Uu66Ys84uz0+O7h8fHx1Qj55dnNW/9sFPx5vTqTAgRLW9e/vA9woBzNt6e3Lt/FIahgki1nGgaxjoQnBASDsOO17+7f+T3OpvNhlKtrCIg5aAX3m6Uq6KsmtbUKdZoy+qqkAiALMsIlciwpZSEICllU2WGZlqmiTBseZOneUtr17QoQdjQLV3jrdQ107AtCCFuGljXTctqBpqWYo1otiuQUdQcKOSHA7cz0K+uGGMQYsFV2wrbdjzPE1zdEq7qpiQE27bp2g7BWHB+cXEFAIjjyLIsTEjeNFiCusxly5ssr5rcMnXDsSjSqFJUCUyQhrAAqG5lIYRsm5bVGAFNI1IBhTFQgLeMMQEAoERfLVPbCiFSq03CKtYJh7blQkjWmzjo9rr9LqZamhc1qyRrFuuFZmsUQaRj3rRBPzy+dxg4TtXUN8sFwtTyfIXwnhs4/cF4suX7fl0pCGEcx3VVbFbr66urXq8X+J5hGIZlcs6yIq/bWmDYSjVLNvV0CqTy3C4itqG7k63dsDPknH/3wytMgN/tZ41ISnY2WxIIJNGonwVBgIlOqd62wjGMh/fvmpaHdWO5il68eZvXTcnaMs8lBGVZFoWybVfXTc7atq0BQACg28ajbbue42qaXpYlQviWm4Q1HWEEMVIQSKCoroXdTtjtfPPNN3lZgM36vwKRNDocj4YA6L5JKb3FJAMguRS3lcs4jRhvRSPqlknJb/cabduSVtY146xdrTZxHEMI/cDVNK2qCl2nTVNblr6zOx6N+hKIoiiUDtMqsXyDEFKXpeMbGFEhBASgbVkcR0rKuq4BVI7jUEo7XfLfXRKMMd7KW701gRo5u7osq8YwLIAxV9Lthu998D7QaZkVRdsKDZd1g5u6rItScp0jqpmFFK5lWZZjU/2maXnb9DpempcIA4KQphHN0MumhBhBinXH0h2HCXl2efGHr7/qhP6dO3ccyxhvjztdv2mawPP7/S5rmtlsvkpj0zRfvXkluayq5tuvv+p0evEmKtJkUgw9y37/4XvnN06SRbpF7989Qgg+ffbdm3dvTFszsU402CjVlKWtsBJKAtwyIVpJkdHxu5PxrqjnSqB+ZxiGPc3QNcuU55cZ41WU7CNi+OHwAB8xcXV+UXFBJagvLhEClCBCIJQKKxmnCdWJblANo53tUdNyyMTBwQ4AqC6L3799df/+w0FnpGsma1gcx/P54vrqQrb1/QdHnuO/evn68nKh68CzgWbQROhxVOmaNZsvrqdXTQuqqjJ0SIjDWAsgxURTSqRFVp+WUbpuY7lZr10H/c1//sVnHz9JN+u2qmzDdv3QdQIhFaXmchENR1vbWwdff/U91MKkrJebhFDSCGHongIEE61lPI4K3QDbo0EcT6NF4ng0jVcE1VDqTbHY2wqbYtmKVDQIaWJ+HV9cTM/OT8P+NiHy5ZuXCulVy1gpEZZRNputTn+0+eAv/tGfB92jr7757mZ2npjJaLiDLbW5XnLWuq7bO9jpj/tJnEVRQggYj3Z4q9Kk+P3vv3JsT0nc6/V6vV7burdBG8sybB9lRVJUOcKirktdNx3P3toaRZvy4O4BgfTqaqqUcjQTI12j5iq5QUrTMF0s0zRrdg6GHX9oO/4v/vOvR+PxZGvw+PHjZz+I4ajbG3YA5Iv5cra4aXlzetEWdZGki7KKnj77w2jic5k4rgVh5fpweze4urr69ukmF7OLizNJagGrna3dO8d7j27uXV0uzt7cbG8djLfHo8noZjZFCM0W802yUULOFvM8TynZanmT55nrO8Odnh96fui2sqUUh51gvV6eX5xKKAa7Xm/b+ujJ43/0D//0+y+/3dxsphdXNxfTH777w9HR0d7O6IsvPg26fpylfiecLpaU6xBCw6CG5RmWHscwK4uat4Qg1pZCtC2vG1b1ertlleZF0jIhuNI10/fCLMvyvEBouVqt0jRt25YQApHK0xStgOd5vu86fu/y8vLi4iJar4fDoeM4nU4HKpAXaX07CbBtXde73e4tDi8r4CZNTNcxfXcRb67nC4mh3fFvFsuyLM9vbu4c3g16fWhoYb/H5nPXNm1zW3JRVVW02SjJkUKapiml6roOu50wDOfLRZZlNWtm83kuZjvbW55uBJY17HamTbma3eRpXFb59vbk/Poqb4o2UcsoVgCZpmkDjqAYjgaWa9V5kkdRL3CVAtdnZ54XEN1gDSeEIgfqGIem98Wn7z19/mw+XzAGHccbDoau61OqvXn1+ur8gjclcbyqqYqEu7bjma7QqanTBkhd03jDmqoOHK/j+kmUOroBTJ7GWbpZy9FESZlFcZwXWDeJAE1Tp1lRlnXZ1E3dep4HMaoYuphFp9crAEC32x0Mh3cOjzCmlNIsK9bLNcGa7/u3HXohBGvruq4Yq29rbVLym/mKc85aLiGAUlCKbccULQeci6aWNaM6tSk1CEJAEi7ivGxN3raN4Eg3iGEaQrcIIaJlEAAFAJRKAYSwcCzLc4Mkat+8u9YIHQx3DN0yraCoKsMgmNCr65s4Sydb234QHD84blpWVZVu0SSOkzLyLFt3dKRDjqVU4PnrV2XddMoaU+r6IdFsiegqKWcXF0mSXFxcmIamUw0DCKWK1xuuuFjJJIsBwXuHe1sHe2merKLN9GzuOA6v4bA7yaJyPNoVgs5my01SEoLOruem5zOoSWLGRbYpWu1m4ViuYRgE4bKpdI30+30uQNWwzWL26tkPVcttPwCEWqZVyWIwGARBIIRIy+xWwieEEEwkSeK6brfbu3Um8VYSRJumqdpSN+zdvS3LsqSUN7Orl6+eVVUVhuHtyNww7SDwbk9lpdTpzbtbwlJZV6z5r8kAiFEYdhFCjPO6rjmXAMFbgBJuaghLzlrGmO/79mhsWxbGeL1ebm9P0iL2And3d0c3qOMFPnNv0vnrk1cQIiHEarHEmGqaBqWyLKeuy02ElVJAKoSIbduEYtfzOOeGaTLGOJdt27ZtK4QiQRj+k3/2T3u9we/+yx8kgo8//MDrdJGm16wxXLs7HOA8TfN8tL1VVGWaZ0DCOE6X02nouXlW9gMXAlnlRUtIniZKqayokjwvGhbnhVKQU6IAqpuWEMQV8MMgCDtU13b3b/MB9W3TqdMNlFISiArKXth5+/pk0O0DCa+vr0f9EWPs26++ni+GH3/64eeffr51M96kSy5bnehJFn/33TfrOAk7nU2y4RXrDUPb7SEiIJIIIU3DtqUJToAirBGcccZaQzelVKtNZAtR1Y1CKOh2kaZlTQMQCoaDOM2S9QZoVKaiZNX1xYVp6b1hx7K9hpVYw3Vd6gbd3t5+/vLFfH7jhcFmHed5fnm5AAC1e2LcGwfdjq17h7t3f/Tpj/Kk9F3fsb3ZbCaE8Hzn4uLi/Pz83//9eZ6UhmHcTBdpAlyflAWnmik4wERDiBBdwxhKJMo6mS2WJncxIWnO/9N//MO9wzuYaC9ePd0ajAzD7AZdBLX9vUNWC0O39/fuHhzc9YOdZ8+/T7N0MulcXlz87E9+FHa2v/z9f0FIa2pgG8jUrcs4i+OZpY8IADtbA8c2LR31O92rc7BcpZlkSjSKcyCbm9n86P4jAfFX370IOnYcR00h9/bHvfEIUVbX5c3s2nJ0qZjjWvPZTVGU/c6wLGophFRMSdYooetm03Kqk6ATRFE6Pbt6c3Jq2w6ixHIdz/chUpRiw9AJIYTG19egaWpNh01TQQghBLqpmaYSsp0tlnGUbm2PLctYrZZZUWrU0rCBIPZs5PnmsLv15t358xdvhuNxGIZlWff7/X/5L/9nqqmz8xMg+XS9vJpetm3z9NnT8c6wN+iaNnj6wx9sFwAgjw/vapRiTQzG5uuT+fnFwt923MCkhnp7+qos05/86M9++rOfsBr+0XleZ+o3v/376XRWZrkXBoCAaL2YT29M29jf3dvZ2TE9S0Bu+baEfGdvZ/dg+/r68urmqkc7o8lwvppeX18vy/Xhg73t/aHj0s9/9MHyavm7Ku84dlU0ZZlOby6G46179+5tkhgbGiLUJw6EEBNNSimUBJC3bcMYcyxjvV4DKTUNS9ASA2uG9uT99yaTyS0cfjwes7Z++vRpnuc7OztVVXDODVPzfT/P8yxLlBK9XgdCeLt6kFKu1+tOJ9A0DUI4Ho8xgACAvb09TdNuVXWEEC5sQPBoe2c46le8fvr0+9lypbv2m3cnRVn/+rfnf/mX05/89E83adbp9Vsur6+vHMshhEghHMfxHbdpmrIsdYOmWUF1rSzLxXp1dnG+XC5fvHzZV5j+9KcHd/bG/cGDo7saQfP5TbcT5Hmum5oEYraYma5b1k3VclzQ2XK+vb3d7/uO7cXx5mZ6RbBijFmWY+jUdRymSSkUhUhyJVmb5WsEWOhbpu0qBaRoPNfc2tppy2p+fZ0nKdfNpqyaug5sd2s8KQ2RJMn06rrXa2zDbttmvV7PqhsNad1OBzogWW/qsqIQAExYVVu2Z9muphtp2aR5XVUN47yp27RcmbYVBEQHqKrrpmmqhjetHISubdth2PVcHwJEiOZYrhCibYVlGRCp1Wo5m17VRVlrOrdMQmlRlnlVcSF0yxyMhmEYYggNqmmYCAgowhoAivGa1UJwzaME64IrCAUEmlCKCwkBhMiAQBKEAQAQUARhGPaG/f5MNt989W0cx4Ne/86dO3FSJEkymZi6oTs+sTzX9b1NvInTaDAejbe3zmZnFzcXtmsG4WHNmxdvrjqef2f/cJVEy+U6ynLNsIm+2sRpfzJxbFeWom3bzXq9PRlPJpMyT3Vdz/OcGjROk0289johJoQpkZT5Jk0o1gnSJIe6ZlNi9rojz+2so8QLukWRL1bxKCu2d7Z2Dg6nN1dQN5qKWaazs7VlWdZiNl+vIwrQ5eVZ04q6yKHggLdlmkGqBUEwHgzjUgqhyrJO0xQhhDHO0mKzWt27d++Wf7xcLm+m8zzPpRCcc2TCtEjzKr/Ftd1GtcJOKIEEEEACJZRpkc6Ws9VqFUVRowpMCMb0VtNjGKamm7cQCKJpACApIRcSKYQQxQg1TWJZVqfTcV3fd1ydakWeR9F6MOgNhv0o2egm0XU9jjednt/pBK9u3pxdnnU6HYMaCirPc5RSbd1mWSK5AgBgTBGAGGMAJAASQEtKyblUCt5+JAixEIIoqA9Hw729vbysm4ZxwDSTQMSLvDZ0e+SFoBCKt+VpYluOzFijIycIt3YOnn7zB8O37K7bqTruyAZSLZPC050kWyzma9aq9SoGCMek2druEkNwUd05GhkOXy0WSKskKddZwVmreZqpGQzwumEKo1DXXE073t09fXehEfO9e48xpFkSvX19tl5HlOjvffhod2s/CIIkj7IkVQht7x4IdFrzUpG2KFIQsa3dSZ7GB3v7bd2Ymr5uVxijH394vFqtEysQQgyDoet7881Kp5ZLTMKhjY1JONjqjdfzRSnBsNszAMqyrM2u9vfvQECePXvVpPX9+/dt6OsUQS4Rb6PVtefrsxV4+uIHTXe3d/dav1zNNx/tdizbn91sDMBcLfjZj/5cMVUW7Be/+NXb03d/+OPvmWp1wzBMLS8r13XjpBLSCENPQQDhBmPHMKjXoaxpkywVQpimabs91vAFyHQduaavAfBunj45Pv6H/+Cfn7x4SWTTlAVnhWVq0+nVP/iLP0uS7H/+l//Pcrr54+/+P1gT1zfLTt9r9d7ZGvzr//Rqsy5MYxBD6++egijuFGX96vsIwMZBD5u06W+6h+HBRXGxaigvq7HpIgPsfbiVvzw5W2329x7p1rOiNB4/+qJQV6ZF/9E/+QtqwN/9/tdfffN2d3fX0LbmszWCIefkcr6SqvZ8c13fnC5f9LyRbbmeF3JRYM1BBGZF/tHHn0AId3d3F5u15dt5Fo/G/av5pW5QjW4ArfJ2la3ZaGsEUP16+jTwhkKDN/HN8+dvAdccrxt44yJZEhUi09zbw0pBoumbKJ0tVp5NPnhymBX5xdXz6fLF0f3D3vaB65mbZKGUsnrsvfHB86fPJxOvSC86R1rHKxxat8mpoe+cPksn44eQwIurZWc4CSZufF0EQaAJb+hvH0zuraYAKOt/+hf/9zv73z/74dWvfvWbv//jL8Mw3N+/45jdTLDGII3gf/rpjz774tPr6WVS5YIrrJMsSdeLpWhbIMVqtfB85/Dwzu7BXtXI169fp2kdZXXH9Us527p3GLoBlCgMgk7YjVYRq4vJqCel3KzjREMYYwoEAKgVAiCiYYIpkFxJJl3brOtKp1ZdtaZp2Y5/dOdelTedTgf7OHXTo4N7Nzc3y9lKcUSRdnFz+dMvfrbZbC4uzj3PQwgZRDvY2ZtRXUpZN83Z6aXvp91ut2lUK/FgPOpNdjHGi/U6bdTpy9d+Z1hzHmXZYLy1d+dB3aIoimbTPI7EixfnOyMbAff1i4vdnf3FNDVNezNf44BqmmpaBgk0bQsjJAWrJccG1g10fXFiY9nR8Vc/fHM0mXScTpxs/tN/erO/u/fpx59pmvHowXtPHn/UKjldzA7uHm7y9Go1BxhJBDOW7lpalaXxch2toj/72c+73e7J2ZlpmpPDg3W0fvbDq9VmqWlksjUaDAa6rqk8F1kmWkZ0DRGMMVAybdnGcoXlgSjLZlHZtm2/0xvemQy2x0pW3y1v3MDeFMn3J68M3cIAv33+uueET44fyJqJpqqhenf1ejgZS7MxPAOauAE8LvOIlUGv55v2ZpPWDauahiWVbSOCMAekrMUmLoo1b5oZxu983/d8xzAM3kLPczu9rmUZdV0bZuF1ukmS5HVL8wobRDFMAIEMqpbLolY5U2nFNimqWxvRLIp81wi6wXyV9oY9qZZAtbxOQ89Lko1lWVC2nuOxhmNspknetsqxfc7VasGQEmlSVi1Py8oVqhLKNIkehlHTGh0dAFSX5fW7y6zIIcB1u6Q05m3bM4frq2gGl7ZuVstmlWweHzzmqRgFY80wBcYA0q2tXdNyENE26fWbk9eMMaIjv+fswN31eukHQStUx7e8yahp5fllIi5iwZWUfusja9ArMds66FOjhih99vK/jHf2vL4fs6qq24tFrNveeLg9DntAtkCjZtANJ9tS8Xx2XVIuVX1T3rC2tXrOT//i86ur6c10YZqmAVnZlLYK6nVEEJn0Bi2XeVG6lmNs20f3Hnq2U5bl5cX1fDbDAFKKBee6QVzbwEBtlhvTsHYnh6ZhKwUJIXmeUg1TCqN4M59dR/E6TRPiIgrokycf6ZpRFIVtu0mSlUUdrZKrq+m9e/cwotlq3e12oeSsqrbCcGdnx7btsiyrqpgtpqvVqqoqCGFV5IvFoqqqXq+HMd4K9qOL+fn359vuNuBSMuZirY4jKbkCAmMMoZICKIUBplBRVkvRNsR2EEICq9vsDsSIsTYvSgKQWqyWJ6fvkiTZmmyXVROnydNnzz/84BPdAKZlffv9N6en5wTTyWT7k08+sQ1XMl5lbDjY6fimpaEiz1sGHNvo9Pus5UmSVE1ZVc18db1YrJz9/s3sTNNVpxv6js9loWQTuJ5tmwhCxYFSUMMUY4pQDSSUdb3ZbOI4tW17NNiyLKMo8jRN+/0el+18Ph/O+9t7Q9/3yzJfrzY3i/V0epXmBdaIrtkuxLpBAadJxKY40hBeFjEUYH97mzPimv2riz/u7O0CCaUEGFEhlBSorVsNa5pmmKbt+yES0DV9k1gGjcpqJTloWcsqvpgtgyDAGtI0w0JGC5oo2WR1iQgcbY2kQmWTl0n18PgR5AgqZGgmL3kwCE5O342HW9Pl/O3lu9Pr81UaS6xgXbIF26RUIahpWtXUqOWapkml2rY1bOtW8nH7AgBomEAdd2xdcoYRqYvy4uIq0K0/+eLTDx4/rrL4zYvnVVO/evVMKv7Nt19iSgHg33z3rW4aN8vlk93DDz/6KI6KOG7LWlxeVxhVu7sDjbqrVblepwq2EADI3ziO87s//KHb7789fXd+cTIcdxVqxluDTi/cP9g6O5+W7eTP/sEXf/fL33ldallblq25g7Btq6jI5ifTkrWPHj7WLcs0XNu2hWirOgeQrVeb8/Nz7dAIvbAXdpTECEDPcfb3doAiYRgul8tBf9ezLcmqm8ur169fdjrB8cOu4/q97iCvY4SQVLBu6tVmPe4fbJabt+/eEagHQTiZjIaTHsZ4dnE1pCPDsCAixFg2grOlSMvi8OiOANL2jJ3drbop48slInAw7GETAIU+/ezDzd29xeJsMDDPTpee79w9uuO7dw2ya5njKInHw4np9TBt/vr0f4/jlNVi2Bu7ruvYYZ03om1++tOfHN053t3e+e1vf3f69mSzmhuG2esEq+nM9/3VcnlxenZ+ecZZW7VlVqRX52c7+7udvg8EvLq4gVD1h73xeKxIAwkuynq12ZiGFfQ6pmn7jt+WTEpZVKVAAGqo5ZzqWm/UW8dNWZZ1zSBAOjVc15WtjDdR09Q7O3u2oX/73dfPn7287SIFfme+WpqmyZVECBi2dXT/nhcGeZ7ezOdRvH598na0NRmPx0SncRxPp9O0LC3LUgAIKTnnvu8PBoNbXIxpmr7jIgWqqlJKdTodpdTldFmW5YsXL25ubizH3drachznm2++22w2hoFv3/v69euLi4uGVXfv3k3imEup6zqXAlEkgUIEYgAFa03TyPO8LqumLh3TOjo6cmz7k/c/PD09rYqUUjifXxdZrJvG0+++qlij29akGxaH+xDwVRI1rNEE7/jD9Tr6w+9+t793h2C8Wq3/9m/+4yaJe/1+kqVxHEkk+8Pe9vbEtm0puSIVhBBTklxeSAj2dvd1XRdCTKezumIa1pGCommRQq7lbg+3Li7fdMOOiOOaiywtTleXSKH5xU0bNI8P7wdBuCiruuWM8Yvzq/Oz68ubTEpZM75cb4qy7g8nw9FEQmSbFoaobVsuGo0Yge9YlmUbZhXlnqdpmgYAXy2nt0eCrtPhcDgYDDzfGQ4G/V5vvV5FUQQUIEQry7LMK9VKCIBpmnEcn56fzeezNIuJTsLQzfOybplQCijEGgghlIIqSQg2lcKMty1TYdi7DcBzpqye3usOyrLeRIuq4r7vuq49HI6DwIMQV3UjhEzijCvZtgIAhJHGGMuyHEIcBjbWKK+rJElo4IfdjmXoEgLX93XT4EIxzjXd0HU9LfL5bLm6muZ5ijFerTeGafpeFyKS5FmZ5uPJxA+6UZpdXkw3UQIh1jStqsrVarW/MxqNRr6jL2ZzzjmQajwex0m+WK3SKE5Dv7Mzcly7KlKgRLKYn4iGUponMRLS8/yDnYPnz1/Ok9VgOP7pT35WV/zs4mJ2swCKxHHsOI6um0zwoigwQo/ee7i/v9+27bt375JoMxwH29ud9XKZpymlxmBnVFeMc6FRyzBsBHFT10rCRZK0bet6puvapm7s7ewf3b0DACA+EkL0uv00TauqKcsyjmPWcAhRv99ljCEoTVNnjGmaNh6Pjre3fN+fz+dff/v01q5yW7bMssxxHCml4wZpXimlTs+vZrNZHKe6TnWDQoiUBFICABBGiDXsFv1023NkqsWYEkLevX7j+77r+BBiKaVSUEPYtx0CADAMLU3FbeyZEDIaTVqmmqahtAl8u67LPE+bpllvVt1e8NGnfwmkQIDbHVPXAYa8G07ydGNbWn8wWS7nuqnBXF1Nz5areV7I3a6pGQoRpunKcrBbGWVh6DqK4hXFlCCKIBGghZAwxhhjEJM4Taqqnox3Or2wqorlcllVJaW4SNPr62tq4JoVtmdfXF+9fPb6q2+fNi1DFFu+qxmUS56lLFo3rh1UKUSmVRWcKPzq+ezf/qu/i6OauNZwNN5sIq6UEKJpGgyx6wRFXiebrPArAjXLdCVTpuHAEAfNTpJknPHQ6UBEMNA820IAaJggJBsJWFtBJF1PZwLWNTsc31U1uDmdoRFNVtnTr1+eTq5+9MWfaJYDdTw52GMEFkhAjLlszy7ORcIgRrppYEqkAAhjXdclBJxzjDFA8NYDhjEGAECl6kqwsuFUNHn58tXb67ML1lSff/j+3YPd40f3qQajfPXq9fNvvv/6k88+FqBZx2vb9zqy+fjTH3th8Itf/+YnP/3ZP/s//It//b/+u9PTeVUL0+kmiVxHEirQchD6yoL01evTf/4/WhjbTS27nZGugeUyevDkvSiKvvnuLbXcjz/72evzd4A2yDUqyf/d3/7N2dnbQS/8f/y//1+T0dZysbl7/D5GmuRqsZyBaIWQ5C0w6XqzivrdIWdtVZUqKUzDHXY6rutHUXL29s0nn3xEAZKMnb05uXh3itWBYe5Zjt2qVszaoqwhRkJJVleLzerVy/OXJy8ManMpbc8c9Pq+5VqOVVbrpEwhwPPl7HJ61XIRdHxqaHvbYyewbubXz199Oxj1jo4PhWibJi5ydrB7PCA9jBvblrZn372z1+v2bDMAXF8uFl99/f0yuhpO7LqNJVdRtonERsN64HZ63SFBhmkIqJBrG59/+uHezta7tyevXr06OTmZz+cd1wcSqJpRiKosf/r86XK9NCwdUXR5eXlw987O3vagMxwMe4PRSCn16sW3eVGVeXF1c0MINTXTDXzf76Ae3Cw3t35nJnmWrk3H9n3ftl0hVLRJ87zUqWZZDkKEEK0s67pibd1apvf48fv37h01ZbNcLoUQq9VKCDEc9h3Huc0cnJ+f6ro+GAxOTk4uLi7qurZty/f9brebF+VyucQYHxwcrNfri4sLxphpmrfG+rZtb8MNCCGDagSiKFpblmNZVtu2aZr4vuf7/ng83D/YjTZJEAS6TpVSWZZdX19TSm0dM8YgRgAAJRRjTEP01qzR73WrquiGHVabVVHe3T+oquqPX/3h22+/NXVjOOq3bauA2KyXv/37X4fdzoOHD+/eO94djx7evXszn63jqCxLtyYdr7NabXa2t2XLT9+dvn79tmasqJuasYbXiBCaVXCx1vSESynlJgxD03GzLKsb5np+2wqEyP2jB47mxFFUFXUWZ65hOcigkuqatjXaSvJKtEnbNDdXN4EXhn6o6yZjvNVEHGVE17K0VAjePbz3y1/969UmKRsAESAU6KbX7Um/49/c3AAAmqYqZ7mS3Pfd0WAgPJdVSaAHuomAQghjy3YopRjT+WLK2spJXM/zwjDs9fqu62GMc17fTOcANgAD0XIpQNs2m82Gc65RQ6hWKDBbrHVTPzw8MExPSkkI4TwBQAuCwHWsuq4sQ2OM8RZQrEHKdV3b2RoLIV6/rlmZDUd927Z9L0SI3Lb5EUJFUZQNF1wSomGkKynaViAEEEKdTkfx1jbMoNvZ2toSnCV5VnOWxuViud7EadjrT3Z3i7q5uLhQNUdYz6py8+JNklXH949MOygq7noUKBrH+SZKmqYVnK/Wy+Vy6QR6XSSCH+sacvvd6fUla6qyzK8uzufz2SZOqiIXrApto7M7EY1WbW4akzKsiOuIvIrTxKLm7mg3WReX06lg0CC27lFDjwVfN7W0dEMI0dSlaVn+9sT1vN6gHwTOm7evzi9elkXW6z/c2Ro6dlsUKPC8FtobGSsgTItSgtM0jTYZY1yjBiZQ0zTTNC1kWJbe6Qaapr2bvhJQJFG0Wm7atpUSsKqmVK+rJnC9JMkghLbtclaHvrs1Gpum/e2335+enkopw7CbJEmel6ZpGoZVlrWU0jTty8vr8XgshEqSLN7EjmMjaGOMRQsEVwhDCLESHAAIAL51RGJMNIoN3fQ1x3Vdx3ERJJxL1oi2bVslyWqz9DzPsExK9VZwLAWh+t3j4yRJqKZJII7u3T05fZtHSVm2v/rN394//Hww6GECiyLJosS0aL+3LYTMa27bel63SKPEQEkRURPdGXdMG0HE8mJTXUSLuV7XdZaksULj8RZGAGONUoIkkVIRQpCJizSxbDvwO4N+X4h2vpzlRWHZZlEUhq5jjNbLVVnmhqWv4/V8umwZdO2O63vUMrjiFWtqxttaQA5vkhWvb0a9fs8P3528OX9XGxTMo/Ls9PTvf/Xryd5O2B/UTZPnpalbvU4fIQ0ratsWEnhdrtsGBE4PBNlykSpFd3cOMcadINQtsskXgErTNw1XQxRdzudRmgJIiWYEJPj2h++3t3ZkpUbdrfU4//0fv7TdMOgN+sPRgw+fEM8OtoaT7e28qf7qr/4qS68Mx7Y818zSoqgQJUTXlFJ5nmu6Tgg1TfMW5tW2bV0z2+0opgxNIxaglGzi6I/ffLucTyeD7icff/DwweHjjx6ndUxMrJkaa5ukTPOmuv/ek8Fo59//zX/8xa+/+R/+xf/yo5/cZ4r+m//v/8ZZqzBpBATI0SgGrO4O9hhjrVSUhg8ffrpc5FWB37w+7Q5cJawkaVsOBNSpaX/6ox+/evNO1HKxmEnV+r3O8fuPAdVm60g37NH2Lm9UtNpkab1cZoYO60a1DK2r9fX1tecGhBgQYATKsqil4Cdv395MLy7e+Q8fPizitK3qjhu6hqUgtV1riFWSp6v1DGCBNQ1Rarim13Pvv//AoGYWly9OX8w317Zt3xnfTZqsrpmS6PTy7PnrF2HQfTB5RDRse3a/30mLFZ9xquFOJ6CGJnQqAWC8UgBBROJ0TSmVihsmUjJnrdY0HKh20B3sjPs3ywteN6qVQMEkil+9eHlj3xBsaJp+dOe+bbuEEEvHx0f7e7vj9UeP5/M537DFanVwcDDp991PP1eCX994FauW0XI5z2zbBQBWTXV4dHc03PbCIOx01R1Q5jlBeL5aYwAt3YizvBd2StbM1gvOBYCwrmtHNLVk2+FhEAT9fn8xXyZRKoSCEBqG5XmBErJIM9O0MaLrVQIBOT56oAFV1XVZVRKouq6EEJZthd2O5djj8XC92czn8+cvXpiW0e129/b2gm5nFW00XQ973bplCsGyLKMoOjs7a9s2z/OqLD3P832/yPPrq6vlcuk4VRj6vh/MV8sffvjBsiwpheu6aZrWTWk69mDQk1JSissyd4yAc47aVtd1TLBSSrScUgwVqOs6jZPjO4dJpK7OzhGArml/89XvptcXvu83VR4EwdZ4sF6vERDz68teNzg63D86PHj/wf08z6fT6WKxOHt1OYD9fqd/sHvn9OT07ck707Atz3cc34SKybYVnEkV56UJbMPQHacrIFYAm5YrZJmmZRIX/Q7SMB32R6P+yKImr5o0SaqsfPrVN95Atx1Lw8jWjH7YPaUXSkidmrPr2Qv91cO7x0HQMR07TcpVtHnvyaPt7b2qesOaStc1gGCySU7EW9t1RqPRaDTQNG12c311fZ4lG1YXGsXj0IlEGa2BlMAwjCAIodJKxo8O9yDEVdlslqsyy2+j+JqmAUtnrUSIKCVrwSRQmmk4vtfwttsJW8EgwZvNyod0NN61LAsCQ9O0NG0Xi3Rne9Tv7ynZsrZ89fyZFELXTQ2LaLW+si97nZ7nuG2DWykMw6CUVlUTRZEC0Pe6nEvGypZJy9SopmtUIcg1zYAQOI4DBG/qumJNt9/jjMxmMwZVyZqqZRzIVoq6rhlrAQASIUJIm4nVOgZE6w7HnufVtWxbsdlMN5tNVha27VqWRZOkyNPRZMuxrcBzizKnjum7jqYR13PTNDU0fdjvs7qO15vlfDEIXV7X8WxeZ1myWG5t77RltbpexKsUa8Z0vt5E6c11fHoykxgrCZIsv5kuJ9s7vBWU4v6g2x+OEUJxGk2vz66uz5RqbBvm5Xwd1YaBOx3H9ay3p5mUZVWWZZ5RaiqJMFIahQgKijVKEFCCM14BsRJt27Zxmd5ei9u2dV3/FlxhWdZisfB9v2kahAil2PfdBw/uOY737Osfvvnm+6ZpHj161Ol0ZrPZfD4XQtiGsVqt6oZVdZvl9fvDied3WCsVV23NakQwRlJKBYDiCnCFAYESAqWgAhqmhmHYtm1Zlq65CBJRM4UkhkRDSEilGCeGqa3WC8f24jglmLpueH114ziu74VBEGw2mzuH+2WV1027tRNiIlc3b2WzsSyL6BpGAEhFqB52BhCJui6WqzUhOOz0PvviM4whhPAinSIEAJCbTQKkMAwDY8yqmhCEMaYa0agGJWyZRAggoizXsVxHJzqGZLNaM1ZTCk1TB0Datkk0LIQo84oxDgTynXBvxwo6oR8EXIEoS2GWY8iUUt1u39HyOIo+ePJFnRf/2/e/iFdgZwd8+OEOlPLFy+cAo63tbUxRlGLDMMbjLcswb+mYWZss5qsqLd1d13ZCyw4QJN3uIE3TNKl9YksGESKIE8fzKDGytJ5db7hgYcdAQutZo2SWv/72Fx9++PHnn3xR5Ozp02c7+3ewbmANKwhc39/d38vKgjGGDa0FkkOlCBJKCiUVBG3LMcZCCADgf2d43dqxq6JkVS2aus5L3zMAgiVrLmbTl2+eT5fTZ2+2f/YnPz5+cH8y7kugpFK2b+06e/cevffizbtvnr4Ie77h2OGg8/EX77989/38ZspRLlBdtrnl9juhzxE4OT/v+P3lJvvJT/+SIHcxn16cLWfX5csfZgf7H6SF5rkTLq3Dex/kFVnDdUdJQ4OmRaMo+ru/+6VS4P33PnGsLlK4Yi2AGGMsgZISSAAIwkma1qwZd3uGZrZMrDbL9Wbputbu1tbLly963U4Sx8P+YGs8yfP85N3l9s7Y7zjD8XbByiRPAGuE5On0quT17uHO/s6d6XQKpEqz5Ca68i3fcAwn8OI0F0gSk2IT1aKyqHly9pqJyXhrpGCbVcl0cb29Pama3Pb8tMgdI/TD/maV7e/faerVan1dpLM6d4QIHYf4QacbdpJ4FXqhpmlBEHbCgWXYrBFXV9OLs8u//+Wv79170PGDq6ur9WbZ7Xb3dnbHA79QNQAcI/79d19/8NGH//gv/0EL5Nffff3yzeuvv/u6Ys1ovMVlCzGJ00Ii0lSMc1mVTV1VigsCiKFpK229Dtae7WRFXjWN4zhIJ1DDeZmXRjryJqPRYDAYzKfz1WqTRilnbV3XQCrW1hjj22K0ZVmO48CWTSYTKWVRFNPplBBy584dCOFiseh0As/ziqK4Ja48e/bszZs3H3/+OdU1otHp7CZO4uFwKKV8/eUfT05OKCGUEN/zdEJb3RCs5Q3TdbpeLxEC4/FWVhZVVaUZaZlYrZZFkVuWbZp6GI4QQo7j+b6/mk2FkpZldfo9jzpCCCm5UpgScnVxeXN5cbS7f3F++qv//It+Nzw4OLi7d2BRfTqdvnv3bnd3FyHk+/5nn3325ZdfQqmasuINM1yf2i4POlio0Y+2kyRrpdje2Xv2/NXOzp7fHUznc6JpTAmhJKREN3Xd0g3L1HWdlXzrN5IAAQAASURBVMtbWo6UUglNIzar+WaT3BInBWsNTA39FvnJCSVFEmka8R0baxQRPJ3OLk4vkjI52t4NPN9xHCgVJHi5Wm3iqChr33Ety2qZcDwfYtwI0bZtGiebzUaI9s7Bwc7OzmTUR1AJIeq6tHBT13VZlpTqo9FgMtlWCsRRWlWVZTmmaQIAMaZZlq1XEecceGbZMA0T3rKmaYqiABAOBoObm5uGta7rjbYmjhvUrAHQ9Pxh21oIg7y4ePXighJrNJQIws2mUFJnTW4QDWtwvVo9/+HlaDA0TdP3O6vNhjVcWRAA1DIOMTEMgzFJSQWBNE1TM0yEUF01CEHO27IsyzKfT29ubq5bKXzXzuuylQJRMtgabxEdEVrWTRzHbdtWTaspTbPMwWSr0+kAiMuqqVm7Wq1uH8cVgoRg09T293e63aBiyf7O2HGNi9N3iWs6ljEY9oSin33yUV4JxtokiWVde56nBBBCdLvd6XT69s1J3fCDO8fD/uRqtpienDIpMTUci1YtV630wk4Ydm3LreoiDLyw2zENY72cbeIkLzKheFXnnqtTTZei2Gwyz9UwsgHMHr73pKqq66v52zfnq/WKEgshra7aTjhAWLRtXVZcSi4VZQwmaYxNSg0NQ+ZYrmPZeV4CqeqyggpURalTDUJY5tmg1+11wuvr62fPnpmmORwO67o+OzvDGFuWdX19XRRFWZa3S+per7u1tcUYu7q6Mi1bKVXkDcLgdtcAIWCqhRAirDCGmCCEkJSiqkrGmipdAgAgxBgTjOjtOt7UDNId9BSEnU5vMd9wBYSCSZ5LgDu9USvFOlmPx6Of/OmPW948eHAvTRONxcvZXDftwWgUdnpcgsVyCgkOO52L6fX51QIi8cGHjz/8+LM8T9+evAn8gef2gtDDWK/yzLIsSkhDKwCQEIIxBgGCEtY1b9sWSOD6nlKqbXiWpTWrNFOXLa+bghDCGONMAgQRwhhiSAzbdCsGMSRKICEVbyRrBARY1835zdJz7L3t3R999vlmuXjv0ZZtmkeHd4Lt4PzyIskzy9aHgy4HcDq9blmta4hAoIRsWS2EQBBSDesagdDojrcRwq7pz5bJZrOipmGZAQdVUzaUKMvyR8Fe6vMkLfTWngz29sdHf/Mf/+Ob58ss/rUU+M7enfn8y//93//1z6rq089/tLez++7s9Pm337dcFutUNwzWti3nhFJICSJEgxARaRiGbHnb8rZtpZQUE41QQzOhYYHA03SyWS0tQ4Oi9bo+gbJuy02e2qt10B3YroswmM9nQRB8/tMvJMDrVfJv/t1fIU3/P/7z/8EJnZxtwhH95KeHm7UbjGVngs6nguOkMwoHo+75zauiXf3+m9/8X/6n/9uHHz25PA+k4v/qf/1X/+F//8X/6V/+nxnHb95ddyZHPex+8uM/fXb6ewvTvNisbpYQcMsyfdfHCGxWC6CI5BIS4Him61mmNdzeG8SrG865RHAVRwSltu3arn07n+SC/d2v/9YwNV3XP3jyPiHal19++WL5dr5cPP7wvud3HS+4WS3iJCnrtmVQSTIaaoavbWkTKfnZaZ5E0at3byaTLV0nUR43sjm4u9ftDRBCTFRxGpd12o0CJuuWl8vlnBCA/IwQb7VOKoN4ll436k5vGMVl2LGkqHlbdazxsDeAyNR16treT3/8UyEExZqhW47jEUKhkG9fvKprhpXwfTNJ9KurZFokUFbdbleWeLme3SzlbDGPixWi5PDe3Tt394hNvn353WI1L9vKdp24yL5/8QwTIqnazNfT6bQpSoqJZRgaoRRhz3a63W6SJLgsfN8FCGqGEUXRfD5Nko1hWEqBpuRlVUMkLFurK+bYduBZTdMQimzHNDSUponvOHcO9rMsm82mTdtqhqEZBldyk8TrOAIYQYIhwf2wG3a7lmU5nnvLWRJKGobRtq3kotPpDIdDx7b39vY824njuG2YaZpb48mmrhBCjLG3b19nZRGGYRh2K1FJyTnnjDWE4uFwCAAoiiKK1nGaZFmm6zrRNc9zMERCcSkEISTZRGWWV2V5cz199t33mkZWN3PHNn3Hr7yaVTxPytsn5v5wcvzwvclo7Hghb1UUZ0AqSozxYHJ6My/axvOCxWZzNb8RCjesXa+iRsisLISSYbczGA8I1pMyqZqmYxqGYSGosabUgNNxB0hqV2fTuioIhmWWNnVlaETH1HGcgdtRSglWeLbW8CbwnUcP7qqWNTn77POPx92BgfXZ9XS9iNZJnGXJs+dPg9C7s7fX7gjPD6hhYqJphimUsm2bUipky6oaAggwqvJyvYnq5LppGilBp9MJvTJPc6VgVZbLxbrb7QdBJww6nucZmi5aXpZlCclwMCEEZ0kkhJAQMMYc1x0Mh03T9IajTz77vCjr7394nsTVeGIOR0HbNoRaRcVZo1arTIomiRPP7a4q3jLQHfZ9u5tlGQG6rfu6HdaNghC7TscyBWsUF0rXzapKdV03TeR5jmbolGKlxO1NpW1bACTRtVq0qyRqAY/KXEKgMNAo1UyzbQVjDELo+74TQKUUhJASzXEcolEAgG3bmqaZjkkILsosSaKmqQGQtkM90jk42At9Z3VVrRdZcPfANsxGgMlovIqz1Tru93r9IAgdHcoWa5bdp3nT3qzi5SYfbnHDDvp9iog9W0WMS03XDROnRV6kiWVZvcB1wi0IYStEGm9my0WcZJBg29Ydy255IXlLKCII5HlSlZFuEL+33Ql9yxgh0E6vl3UlkiSNNhHFEmPa1PFtlaDf71q2YWgoTgvZSsaY4ziKq7ZuKMJN0yAF2qYOgwAhFCnp2pZoWbxZS8CrsoBYUUpvEZYAgLLOLcuiOsYYKyhGkwFAcr68adrKo37TNKxplFKEEEIgQkgB1TSVrlPDpBAhztq6KoVolVK+04cASynyvKqqWqPW1tZOv98ns9nUcTxKqWEYt2UPoFDbCs45ANIwDM7ZP/1n/3i1mgeh1zRV9PJ7gjkicr5ITQuG/eHVPJIAtlK8O7103MAPHIj01SqdL6Z52oyPDwg2oTI8l0CBhWgpJqbhUqIDpQAACCFKqFIQYQAVwlQXrGXtLSwTmZbe1uCW3VY3FYKYUp1zXpWtUorVAgBUFjUXEGGMEDZ1i2jUdd3DO/usqU1KkmQBQfOP/+mfD7udwPP//a//Q1mkd/b3+12vKJOSNdfT8yQrlJKmpWOi8jzlrA5cpwAgy5K62ACMLd12u/1OUuRlDRQVrRQSIUxFiQCmQ2ei3/HSpKyZ6Lp9COF7955QbDLGoISGZu5ubX/39Idn3z19//EHj4+PmzT/m7/9z4Zh7AxGVyVINhEi2HYdoYCpGwAAICHGuK5rIcpb8ugtTFfXzaypIFQE24ZJKUVJVMhlLXj78MHRxx++//Do2PL8uiJNVeRla1hqtDP+/rvnX3793dlV9sln9/cP9xAVN+tTRIp77w0bZigpPv3T/bi8GnS8hw+2jh/vaW7y97/85t3V71+93SHQWmwum3bj+UThdpMsARJlU8Z5VN7wveMD/BJenpwZJt6f7MbR/OTt29FoPPzTP5dClmVW13VeJFWT61KNuqPJXne9cG/vPa9fv14vN/fv3x8PhpZrtXXbqlZB+ertq0cPH/thwFsJIKxKdnp+7ff8u/YeIoS1bdnUrZB37j1QklCk16Iu2wIAyWQFqZJYlazmQLWC+aHX6fXHo4lS6uz81PGdokgup5edrtftdV3XUkB0+n0D2/2eW6RqtU7PTy8AjCRYCp5WuWC10wu4rpE4SoWkYcfb7uwkSbZeLK8vL03DPji4M+wPhoOerpuhb5sa7Xf9dS/crNZNXUAQuKG9TkC83nQH7sn5y+lieXL19ud/+Rd/9hd/Rl39/PJ6vlleP/uu2+t98vln4/G4KStHsyzNXC9XabwpspwTapuWZZidIFxYtlLKNM2qqoCUUKk0W80XFecSY0qgTgjtBYNBbxiGHZ1qdVm+ffv25cvn06n7+NGj7Z0t0XLdNOI0CbsdzdDLsmS8lUrVdZ1k6S0oJivy0WR0d2dnb2+PSXl+fp4VRYdSQsjy7BxDtH94p9vtAqVcy47j+PTknWEYjx4+3N/bi3h12ws/OTm5DUMYhnFLs3ccy7Zd13WpRuI4ns1ngstuGERRdCvMhRBCCHnTcigt3cAI+b6vhBSsdS07iTcnb9+Ouv3hkyePHzxGlHAlS9aMxuP+cOAGoWmalm1XQtRZThC2DMOwTKXHxDZ1zzk9PUOanmfFzXQBMEk266K+DQZW0TLJ4uy2Mg7NDsa15VqGpWuWphjO07Io0tV6Np70g25omwPbMtI4urq6uLx58/hwL00jbJrLxVUtwPbWEAppUovVZdOUAjS6Qdq2ubq+WCyXf/z67QeP70EILdNuWRXFG6EQwrRmLcE0z/PNZlOUuUaoa1sYY8ZYz4MQUl3XBUdnZ5dnp9eEEEr1fr8vJcjzHCOCMa7rJs/zJElwp9/t+AgqybltWgBKomuO7+0e7EdR4ocBwcbWZFAztYkSwaHl200rTMPu9Udhd5gkeVMXFNM8q6qq5Y00dt07+wdN2eR57lh20eJuxwQAIEgUFIHfq5q6rlnbtoRiSinVkK4TTcOEqqIospRxKXzf7fZ7dV0ijPOqLOrKdX0AAFCwamoIsB8GQacLJSpFLYRQClJKNWoQQpVSRCee50kphGwxES0vqzrlrMYYPnzv/eOjw61hV4NidnVJMUrTBGlGkkRF0RR51u90J6NRmW2WNzPbNLgGnM7Q6aVxUb0+u6LEwFQnuruzF0RRDBGiOiUE123t+bbrut3hYLPZrDcpoWA0GvUHIwCAQgoiUeYRazNNkwQ3jKmWMSnYL3/5V+PRTifsGyaabHlNBRzbMA2yXm00zQDKuAX42jY2jABBDgBkrJVSGYYJADAMYzgcFkWRpqmu6wcHB7qu34aIIYQHBwdNzZ89ewahGg77rmsXRQEA2Noa32IbblPGo9FosZitVqvhsM+aSikAAJQScK4gBBAihBBrCk2DCBEhOGtrBaRp6rZt7u3czbIsjtNWSF0Ztm37oecHAZnObvZ2jcVqDoFuGgZAsNPrZlkhlPQc1w+9759+s7M7fvn6hVRsb2+vyqeT7T1Cjd/+/lvdpAfHdxzXTLOS6JrlOj/+k58+eHDv4uzkyz/8Ltrk9+4/CrbGRZZirHt2QBEtisx17MD15vMbyVWrcQgQoFIpBaFCECCEJMGapinHJATJlhOICMK33hpKNIxpUdZpmgJMIIT/NQKZl6bjIEoIIbqum6aulNxsVqFrvX79YtgL3n98f71afPvd77//7pvJ9taHHz4O+4OmrfI0q6rCNvU8jVmvk8ZyfjPPNmmdFYv5PFrHjme4rtfvwl3b2j845K1Uiq9WM9PAjmtQZCCu6YblDwdNV1UlMyD+7rvvECAEkHDQ/fOf/TzJUsdxEMIXFxfffv3NeDga9vqYKx3ge/uH+XRdFAUkWCem2QpN06ACACAl5O13QQgpIf/b1Ah6rpXnqRDMsrUgCCiFBEPJ215/+PGnnz44OnrxwzOKYK8TeH63KluDgl/++lfLVfrxp0eHR0cKSt2im9Waq43ltLatWF19/OlRFM86/vDhg+MGzB59OHz+CvRGEOoZxUDCVMHi408fHDw4hjr+/McfVgoarr3J83fnb4ooSZfxVbY+efGcUNDth+PBII029++9F6VZURQIi3adVSzNShNrAlIy6vZ0XX/x5vVsNRtno8FgICQr6oIrcff4aDFbBp3Qcrwsy/xOuBOQ1WqeJlldNQogrFE/DIhmQAg1U6+K5nJ6mSWJ69q6rd85Ptgyj2/ntPceHE+2djiXnusbtvXdD99zUQvRTrb7e3vbrC3zMvNDL0uLGtODnffziCfR6hJq0+mNF7CWG1SjCEDTQoq38/kUYzoY9Io0a8oqjuM3b94IroAEum5ijGfXV5audbrBaDDYrBfr1axpKkqhbtJuP7xZTkf9Ud7k27vj5Xr27dOvBFHUoMiA756eCAj+0Y8+/eDjD6fT6fXFJSGkH3SohLyoeMFc39nd2anKsiyKLMuapjEMLc9TW9kASIikbhBPNy3TBRJKgQbDzp393bKs37x6XaSZrutb2yPDMFzXVkpYlrXZbGaz2fHx3clk8u233y4WCyHEYrGwbOPW6XB7RM1msyzLJvt7t/a865sbBEBVVbcbul6vt1mvl8vl1cXlmzdv+r3e/Xv3wjD82c9+lud5URSO4wCEOJcAACk5hPDo6Oj4+Hhrd48xtlqtbNve2toSrF2tVllZcM5v3XpFUXDBSIAghBih1WpVV9X+/n6ehW3Djg7v7+8eumGQZOmrk7dIp/sHRwd3D1+9eV1VVVpWKqswxhomWVETkhNDaxMxnc9+ePFyMt5uRXZyduoHnbZtTc3UTANDkmwiLoVhGK7rVhmPonnYDY7u3bU1K0+Kqqo0E1KqCyGSLIbIHgxDostNqkdR+e70dd20o7299XpRtWJv/66mId9zzECvi3q9TgLXU4DneWLZuiRNXRej0Wi8tdW0arFcM64AwjKH33z97e2TgG24CAEAMCG6bThVvnQcw9QcJdR8scqyIgzD7e3tJErLvMKYSC4t0xS8bVldFtkiypVSBEOl5HjY1yh2XTcMwziOTdOsK/bd0+/v3L03Hm1JhZVEcRxvNpuGC8/zXMefXp3VVT4a9s/fXfC60jWtLphrBiblrOYAYCmwZVpt29zaeRAGrGnjONZ0ijCBUHHOAKSOY1u25lSGRvEmjgAAuq43bV3WFcBAAJXkmWmaGGDBuU4NXdcZ43Ec17C2LNvzPcfxdM2UEpR5WRRFFG0wgZpGMEEYK0NDTqc3Gg9tQ4cQIABHo8Hy5mo2m93MF57f93ujXmgjRPqdruM4F29fv3jxYjzoP/rwsaYZ/ibZRAlXSEnY5FWeF0VRNIyZluE4DsSgG/quayMMVusZY9yyjcFo5LkBIkYruBAiL2IFWJos1qurvCwJ1rCG2pY5rl6zZLGqlaSsUayGUmDXtYIgsC1X1808z5umCUM3CDzdwLVsGWO3yumyLBljYdi5TaTZt9dq13Ucp22b6XS6tbVluc5kd3xri3j37l1oBf1+v6qqzWZDCLmd+W1tT7788svL6eX29naexAAADAmAQkmoBEAEa4QQRCjWMMRt23ImLFvfGo+2t7fLRqV5djOfAYX6/cHW1s5oNPYCl+xuPex3R/fvP4yjpCyZY3tRlIbBSLQoS1nTNNuTe21T3z/+hFLcNI0iB5uFMgzw8O7jmrGnX/4+6E46VrDcVDu9I8CCi3d1lNg7Bz8+vI/KuqSKbY/3r66uAseiWPVCn1KcZsVmXQAgDVNDGEvIIYRYg5hCxiqEkO2bVMOZglQn0pDpOkGSKoqEEAByS0fCgAJw0zQ1w2NcKIgx0eq2besWKAI1BaTq+b2w48XrxfnF6cWod+/46GwxV4Ya39nu7gwARCa2GJCH9+58+cevn5880z1ydXW1XM4vL854y6qqsCxroIaL+EaB+uvv4sB2RxN3OYuBqvNcxKnoTwbhwEya0jZRMO4iwZtrsMzrFy+fdQLf87yzt280CkFV3R33HuyO0+V1Ey9NyP+X/+v/+M3X3y43ESuXdbFSsvTDbgvqzXrteIFlOevFRnKFgMQYQsWlEBBzHaPrGIXeMM2XOzvdn//8Cymy5fLizv74zsFOUb/44/c/QIE3m2S4/ZO4mAZBeLWGOdMeffCJaWkQiSSb3UxNhcDJu5u9gy3NMdd50+lZ3t3x/SePsiK5mn1bFvyTP3u05T8Ybx16Wk9xu+bi3tgGGsvZRjcNHjc7jrtj+Oz6ZuQ51XrNiqJq6g8/+fif/bN/kuaZqLUkKjAiBjERpBg7sgVpQhE0uzpSBYBA67m7WF5/8/XJKqofPjoyQn11Nov4jIQgFqvT5alqUdY2oC3GYdhzui7phpOhDrs38xnjbL5aYlK2rVgtZ67tEQA5p+PRBGqkjGqRJ7Zuuob++vT/z9N//ci2pdm92PTLu/CRbufO7Y+tU6d8dzWrWbfZFxcgpasWIOD+SQL0qFcBfBMkQAIEiU1RxXZkVbPLHb/9zp0+fMTyZlo9RF0m8iEfEonMDMSa3xzfGL/xko2nHuzFpL24u6SOs10ZDZRCmCsZT88+HT0uC7mbVTbpUQVH0eecpw6QsMNcrByXrHavBr17USKbCsxuFzKajEaHvQZ6zrJtmt06IzjvR3Gbl7O7G941QRRuNmsJcNnJV+fXD8bMc5MHp0/mm8XRvXvJeFi01Xy1/Orl17uiqJuuleLDDz9+cP/R2xdvf/+7P+7Wi48//Ojs3v3JaDLsjeZ3M8E50rRMq9cvXquOC163DbIpKPKlbbMgDqWUEFKCMGE2Qw4wbLnY5WkBFcrzosxv4yQMPbftStu2ymLX7/cJNkryrtW+51xfX/u+n8RxU7XDxyOKWZbkvhtSQrNdweGbsqzDOGzqTgkzmEzbqvWDxHPc6dHx/+P//n9zYzfoe+HQUagCtLWVjPseGUfj2Bn3/MvrWy6V4x47nj8cHbhBWJRVlpf9yYHr+pZtp+tdLWDbguUyPTrUjuMsbrdhEBiHBqx3u6yvLzeeOxlP8f0zp67rtJJ/fH33wUeJMvaX3737H/+nvx4OJ//ym39BWPu+mzd5lu56vR5zXQjxYDCudf/qfHV3ezPpJaLaJJb8/Mnxy+fvTCWYl/iWjw2puOjqSnU1MJ0nXYhk1RR3q3kCBoaArpQAaoyhHfm///U/u4x+/xPx+P6ZJScB8FqxXK6Xq807x3GwVNXNZuJETVFKW69Wc8dxHn365PDhPezQuq4JIbF/iAi2bXc2nxdFdvrg7MmzD4qiwES+evW6qTZKmY8//GgyPjicTne7XbkIRqPR3d1dW5eH/X5OMWPYBrKr6pvFYnRwyHmzWC2iOLGD8CRJgpQ7yNqsV1WdY6NPTo9byT/7yQ8W2abWYlNnqK22X5fD4eTw6IR4NNuumAVO7x8eHQy3m1Wv11stJcFWHA8Cx26q8t27y3yXO64FtPE8T2NmOTa1LURlUWR5WTDLisf9VnCMMSXMQLhZ57O79b4jsOACMlsBenu3RNAEnt9WpQdsii0LWllWcKHDUaIFrsoKE4swC1m2hnbDjVQSQiiV0Qb6XmikoJhIDgfBcNKbYAzjOEak+82v/8vh4eEnn3wyOnu6We+Ok+lmszEEIaiffXBfSnl+9z4zHA8GhWUvinw4dFgcEaH90Zh3avbuUhlYtqooapA3rtvVTbXbbaI4OL1//+NPnlZVk2X5xdWNVNf7xQGhaLNZtm3G2wKAllGDiVZCa6mbNteWZRiDEAJtbJtYlmNbZjgYUGoTbBvtSwE4V1pxlxGH1haCdhy1ojKyWy8WvuNAhHebXVXV0+lUKbVeL5nLAAFlm/fCST8a7dubDscn2+12sVgUuzryelVVIU1PDo4kl7Hfr8K2TBvLxRBiwRWvhRQaQEw04ML4QWKARJhSpA9Pzp598DBOgigKVrN2u9z2wh4hNA76948e+n788vkb8vTZY98Lu66BCFRVcXd3FwaxbScIYQihlJIxajEn8CFlGKHKnva57BDS0gBUGwEboUuMgTKlF8RBAJnd+UrH1Jamq6pd1RIhhOt4EGKEiNJKd6Kqag2QUaprRUEqpRSl1LIpxsAYoQGA2GEMM0agBITRIPK1NKRhigsNASIMYoII9jzv4MGZ4EoDKDVIs3y+2JRNLYAeDwcGKMsiXVfZ/vGwH0GKpVbUcmzbEVxajk0YoxTHcei49Ne//se3754ThHbpqiyyJImODw9ubm7WKWOEvG0uob68f3Ty7NEzRqI4otvtNt2upcoIiW3f0Y0la4wQHY/9oswwxh999JGS3X/6T/+pFyd//dd/FYYhIUwbWOSt0sAPE4ycP/z+nyrFgHTbEjX5Li8rZrsOjrBidaEJBBhhghFjFtK667pZtvKHQ4Sbqlje3qRd8/Szz561zVCKWnDeCyOo4d3N7OL8/SiZVFXdi/oWJd/79KPr60sh7N///rdPnj6cjIar7eyff/O72Wz22Y8+tVigBeE1Wtzt2rYG2lGiQ4ZZzLdtz6JOGCWHxyfM19Qzx/FRVdeu03DFk7DnOCix3OFweDefffLJJz//+Z+FYbjarH//xR9/8rMfjycT13UZY/u0rux4vkupTZL+wCKO6MTNzSwrUmnkYNCbTPvj8YHiar1cS67KrFIS5mnRdM29e6PQH7aNYoCMR8e2E6w2y5PTB3mel2XpWp7jOJ4XFGnWVKWdTDzPWy8LCnGtlZSSSx6G4dHxgePZgOJG8KYp86aq6tr1qEt4EgwD38UYjscD33dev/7y229+a9tN3S3v3zt6+PBxmQvfS5DBq7vbJEw835oeTi7Oo+ff3ixnd8Ao3nbHB4dVVTV5iRF0mHUyPXRd13Ecgi0IYW8wrHjresHr168VBMy1dtt0W2TbXZYkycOHD1er1TfffLPdbp88fui6dp5nWiqpOgB1nqfb9cL3/aQXJr0AV0AbKUQDgMIY5WXGOxGG8cH4kBA739VCdJjGp2f3edv1hoOXz7+9uHjPOf84+dhyfIfqvTd2uVwmSbJYLPabAgjh69evlVKHh4cHBwdRFGmtPc/rUGdMVeRVXdaUUteytdZpmmIEbCcej8cWw8Ao12JCyJcvX263W0rp0dHxwdHRcDhcrrdIyNFo9PDxk11apllmAIyTUGrQdV29rSEwvuvMbm8oxW3bKi7qurYtC0IIALAdp67rtqp2u93aGMG5kKZpmpevvh0Oh48ePLx/fO/924tvvvrGdZ3Do+l0fBD78W63k1gfjg4DJ3j75ma73VRVgaBQggshqroVSnZCWgACgAxAnud5oSu1aGV9t7g7vndy/8GZ7Xt5U2x3WdWUQouTe8fb1ZLztkzXaXqw3viu7wjZuVYvciMAgAIQAOC5QRhHlmW/ePUGYsKFKYuWS0GYFRCbMbZZbDAlBwfuZDLplN5fDY0xh4eHTdNWRSmEGgwGnuftdrvnz58PfasoMmNEGPpRHIShK6Usq8zx3CgKbIvWTd7suIHakRwhBI2rtXRcywAXACA4z/O8apoPnn5wN5/d3M3rugUQV1WxWS8Jhq7nGGMCz22aqipzm5Lac6QSURQtZneia1zbuZndaSF7/diyrLyofR2EjLquO8TEcmwuxD5n7jjOPghT5oUQYu9W9/ohY8x1XNG1TV1iiIDWge9rBfYisevZjDENkOM4COLhIJJSOo5rWdZ2vVsul0kYPX3ypN9PPNuBQBmg8nRXN1WabiPfk4B88OwjxlielQBA1/Wapun3B5vNDmPMmC24qsoGQTIcjMfjcV1si7zGiIVRn1JL8M4LAgDwdHK0TXdVVUnJ27bFlAFIOZcXFxdV1WzTLM/zTkitAMAAIoOh1ppDKCyqEUQEQYgwxtRCDsEEGGQMMEYrpThvtdZv371EkCFEgSFGI2MwggQh1Cmr67q2Y1IDIbQQnRAdRMSyLG1MVdW27VDKCLbbRqa7enX3No5jjHFVVUqpXq/nOE4YhrPZzPM8QkiWZfu4suu6VVVpDZSUbdu1bQcMIkTv2185b4ej5Nmzp0nP93zq+ayqCs63TY0vL27zvHj08JnjeJeX10rebLc7MpvdPXkSLFd3u23WdSqOkg8+eJbu8rKsMd7L3QhCwpjtui6CDNJR1zWYGK7aat5synWpWtsKT+8/CcK+Y3t5niOcWxaFotNys97iuq5P75/sshIoiQkiBBPCxuOJMZpiSCnGGAOoEcYGIGopKSXCDDmI1KblHTIUYAS0BhAbbAimxPa8qLd3WhmGMVY2tSzbi4fjeHrQtBwA4AUuQqCpi4qXUejZDKVluS3Leydn49G0KKpOSDcAWuujo4Pj44O82KS7+b3To15/StCkP4h+9IPPv/jii/Y6dCy7yqvb61nqZukm7xoBDGIkoqgz3O4qqKVu65YAP+lH17Pzf/2vfyGl3G22F+/nD84enxwdG03yrMuydRglUtcHR0ee0xsOjoqMr/POskLXcduuk5Q5zFE1SIuNR1wEtJSi7TpNgGvbvue4jmWHwpj2yePB08f3T0/6WpSiLaFRRkCgPCkUb6SFbAIpBjjdZpxnk1G8Wly/f/vGpsx3vMCNGGO3V6um5k+ffWx7rNY6tIdYBi7zi92KNxi0tCpkuisFI0IIajFluqas7cjuhPaisMo4ZEaILltX3//B5/1R76NPPpFS/of/+Ld7YfzXv/71j370g4ePHzFCLEYhhIRiLrrv3r178hj14ilGTlOLzTIP/PXybh35/qg3CZ3gLTxva0mJM+oNkbGVNsenJ+PpSCi+TlfbdDPfzLbpdjxWQovRcDIZTpRSBGEKQVnWt9c3DrPevn6Tp9nJweF0PAQAvL88tyxq2bjsKs4byqhr6bouV4t3b7F59sSeju45lqcVdoE9nh4I9cFvf/t37y9n2Y5H/r3kUe9g+uD6av3m1T//4Y//7Re/+MWD+2fUJlWdr+aL0PPjIAx8DyrFudA1T5IkGfR7vZ7v+244zvLdrsgvZrcawJvbmZ8ED44OVnnGGLMs6+OPP3706NGXX3755s0bx3F26Wq7WSCDCMIY4LZu6qaoi9L37cl0EIbBq7cvFqu1ZTNmU6W7ljdlVTObERsiCKqmgKCL9XA2nwOADg6ODo9O3r55lWVZx9V2V56MBwCAOI73IZr1er0nMO7JPK9evXIc5+zsLI7juq7rugYG+W4AABCdIAhBCClGUnSbzUoLbluMEPTo7IGU3AB1eXk1HA7PLy7eX15ND4+UNPPV8ujk3ocffrhYrZerdSfUeHIQx8lyvXr17vz88iJyw8lk0rWt1rpta2w7SRzXdWlZ9OpqtZjP6rKyLItQkm13UnEIYRR7u/PFSvLj6fj89Zuvv/5yu92enJwgaUbRsBcM66yjgAFD1qscYTMa9+KeQ7CpqmK9WO/SEjELMVRzXq/WECHbs93AhlBJqaeHkwePzo5Pj2vRFF2qdMtFVba50n2IwaPHp1fn795dvnn96rsPnz6DQP/g6Yeu6+6yNMtyPwj8KLaYE0ThLm2UJk3bCgObTgJsO57NGBsDj0vhOA5ESEr5/Pnzt+fvMcabbZrnhWPZxphXr14Bg85OTx89euRjaVk0jL04CqLAxxg2Vfn8+XNK4XAUx/2krJpdlvseJdRgDJpOi65BxmAM67Labre73Wa1WQ/H4zAMH56dNZ3IsuzmdtZ1XRyHQEJjjG3bGAGgpB26lNLlcjlI4r27sJdEXdO2bet5HkBw74ljjEGE9u0D+zIC5tiEEMVlURS77RYh5Fg2IWC73SKEQGwwxnEch36geGcxdnV5AyH0/RBCuFqtACK9QT8MIkGhbTuMMclFURRlllOI8jy3KYl9v+0a17YwQchI0VUAem3Z7Zf6t1d3jDFgUFEUAIDAieq6Ttd5mqbbbdp1nTEGGfLw4ZlSKkCu62rOZcu1ZbkQ4JZ3WgOtAZdKA2gxFwJSFk3Hy7pr67puO641MBBiBJGBXElKDKOMUQCg4kpCKYwxGiOllDESAA0AgBAKISBsMaYQCAAQMAQhTAij1KaUNkXTdDlokQYYIQagMkZJqRzHKatmvdpKARhjBsHValUUmWv5x8fHQRBUVYUQsizHcTzH6Uajie/7Qoj5fF4UlWU5w+HYtouuZVKqpuksxvfAZqVU27aUYkppEAS9pIew6dqmKBrLoqtl4dj+aHj48OHjPCtubuYIEs/ziW3bCCHXdZqmcV0yHg+NEXmx7ToBoW+MllL8dzwgglSQiLCw1w8AFLuaX8xXIGt8T3z2ed/3IqWgNELyrgNKG+E6zmJX77L0WB7n+RYhZFnUtS0AMaHYYsRxbGYRCI0WUipujNGm7HiHkXaYTxgErVGSa4OVNgAjiBjCDFNCCKWOg5mjiNJK11wI2EBsWbZLbQcgxBizbCwMR8wK4ljwpsg2BqPDw+PheAIA2ntrAdSe71IChoMQgNJ30L3TqW0h32WjkfvxhyfkZIQQuXx3vd5cpPndNy/rthST4fHB9Pj4+JhaTGqwWa0BghYmkecTgnrjIaXUZpboOMNsPBoxaj9//vIPf/gjgPiXf/XXrhNXJU/i0b17Z+s3t3Ec+V7Qtq2NUNu2dZnVRdYfJMYoz0O25WEMLJv2evGwnyA35bwdjUaPHj5gDGfbmVbCtu2u7kpQYUCDIDo6xJblAA2ztFBGNk3z4OzeH37/27/5m7+ZTA4IYif37/fCKVAaG7+ppCx54E2aqivLImsw7zzY4C2o13Zmj0Pbc10V1NLM59eLbOsGzvFxf+RTDEhWpJh5Z4/OmMMwJYvFLMt2kPSqqpgt7p48eUQQcl0nbH1KKaKkaaqNAIRYUdA7GE+n48PlcnVxcdN2tR+4JyeH/d5YHIN0XURBcnBwPO4fVBJgjBmNHBe8e3/9q1/94+38Jur5xhilhG87jLGqKDHGdV5s1+uDwUkQevtMqdSKMCaVdIgNkMEYi64jSDsuJhIb47gu3S43b8wrgp0njxxKvbJqe4PB2YOjw+ODf/rHX60Wy3/4uy90G6uz0HN6zx59+P/61R8uL98lse+4NIqCbLOu61o23cn0yLYdBJBRQHIlG2EEdKjneL6C6PJutlyt13laNm0lOub7TdO5njccWsPhcLmcv3z5sus6y7IWi7mWyrUcz3ExhBiT8XgIRz3btn3fowxWdbHdbQaD2A1o13WOb0ktIVJFlSuJs7JwWSCNBBi9fXV+fn7xw89/8P3Pf3x9dZHneb/fF6Lal2GGYbgviNvLCYyxvWFKCAEAwBhrrbfbbaO6OI4dx9FCdV1ntAx81xhTFfluu1FKQIPG43tNVUsp12Qznh7WXBRFAQwySNuuxzt5eXnZdLyu664VSzgry7IsS941QMvrq4t+L55MRlrIy/P3CINHDx56nvPi5fP5fO5YtoF6sZorITnnZ2dnUlQnR1NGIW9aoOXbly82i/XpyT0G6Wa+fcvePHz8aNgbKKDTbVY1dQNbP8SeN7Qslue5UNxab9w49GJ7drdOs51lWZ3mRZ25gd0fRH/5i19oYHblLi8zBZoosbHVkaJrmy0lfDQYBv6ji/P3l1dXfmT7rmdTx3eC2d1yt87jaOBQr+k62sjp9IhZbpoXlDhNm7dcQ2y45FQZzvl2uxVS7na7LMtI01qWRQjZbrdK6DAMgTaCqzSO+/1+0guMMRTDMAwoQcNewqxpUabL5dKy7SBwxtNBUVSuHxZV6bpumuEsy5TRRolO8aZiTde+ff3u9mZ2/+GDwWgYhmEY+vtCDUpgEvlt27quFXqWEgenJ/cC1y3yXV3Xk+mIYeI4zoMHDxzHUUJijNFy3ev1/DAUUioAHc8llGJMIcFa6zIrhBBAGwhhHEZBECkKlstlURTQ6F4SIYTWu11VllqB8Xj84MEjqcC7d++FMgihtm13aZUkSV1W8/kSKP3kyZMoDEXbvHzxvGur9Wrm+W6V7yAySnAI1HK+autOKZVlGQSYMbZvWYxjxlsBDaLY6id9SimldDqdOo5/c3PTdAIAsNnsyrJCGGsNuq4zEGBKPBpYjrf/IULp9XIhpRRSAgARJdSilFKMIQSIEGQxAIEyquNCAw0RMELu6x8VxpAygvckMaMAgAbsjekSQAKgAshowIUEXFRSaA2wbfsASi6asuJN3RVl03VitdpJKZuOp2maZVkvjvI8Hw6H+yTkZrPRWhdFcXR0tCc37x99QggIoW3bgW8bY5Q0xhiEiDGmqqqiKJQSWoN0lwMAwL4xrG2SxAYKffj049FoTCltyy4OfNt299A/+vr1m4ODg3sn9znne5dWHPcIpnWTb7epbblRlDDGuq5br9ezZeoHthtOe32P2bOyflNVlVD8y2/eDvrjwIu1tqidSN51XDIWDwauMcYL3KarDVB1XVV1rrXUkruuEwSe61iUYoQQgMYA0/KmaSoIIaM2sbGjbCWA6gDxLKWgVsAgrA1QStWiawsRDiMDtdLKSE2RwZQiiACCTddCbEulMaYAo7ptlDLD4chx/cCPgtCzHcfyrKar0836/cW7Mt9MRv0iX89vO4L1YBh/82VFMXJgDQRQZGcFRZVtF9fvm1LZPn4Qnwz6Ewjx7G4BgcAEKVWV2Xp6OJnd3O2Jtp7jZ9tMSSihOTw4+fabV989f/k//00MAbq5vWWMHR8f1kBJKYXIGeahzYluE588PbuvlFC66497jx7dDyMvL3dCdp7nAaaM8QHQ795+KaWeTg+m4wMtdFV2UhhqW1HoK0GKvEIYD0Y9zydXV1cHB8c//1d/9vjpE6MxZW7dyidPP6ma3PWSXb6RhoRufPv25YvXL7jmFFLUwoJInyTHB/d8328Nn9+sb2bb2+XtoycPlWEHk2mWrrbb7aT30PHsw+OD8/Pzoq7OHp3tY9+2xTBGGGOXWMJ3McaEUpuSa4hFK4SQ/WTw+fc+dxzn4uq8btLVYu25VuB6cRCp1nRdVxclwbZjB2VZNnXnRw7nYrfLhJDDwbjMy6JMjRRKKd7WSRRDCBmCSRQCrQb95OHZg6aq/vjHL13X/Zv/+X/TNCXGuOGNVA0Qsi1KDJSNQcH1119+dXV52/yi+/5nP4z7fttyYcCTp9+LwsHt5fV//P/8x826fP/mP5+dPvIcazToNW2VpqlFWa/X2yw36Wa7StM0L8MwJpQ0TTtb7q7vNtHN6t79U3ddHp4cK0gs27+aXQdhPN8sXr15hymq2sb3/cvLy9VqlW13R9NpnhfJMMnTbO//11ozhl3XhUbVdf3ll186Lu26jhBSliW1ieu6DDs2cwi2mqrmHWCY2LathLSw3TTNd998y+v23/27f9fr9bq6iYOwyZq6rs/Pzz/99NOmaSilVVXtdru9VTaKooODgyRJuq7bt0G2BccAi1ZorTFEjJAoioCWUIk0TSnGAOg4TGzmYIiAhpbnP376gVJKawAxklLu0vzy8iqIItd2LMtSUvOusRgLA9+1LLs/mN3cOq5FMfniiy8sy/r4ww9Ho9G/3N40TUUpBghyKcoik1IqI3/xyz9vykYDVZcNxUhyEfqBRahRGmi1Wa1H4/74cLJczKu6OL53b5PNbc+GFBusqAfjUXgGz47OHtQNbP75j7v6CtsEUSRV0+uPvv/5p+Pj8atXL969fys1DyOXWMRxIISkabZpOmua3V/82Z/3+sH9s2ObWQihbZpjajUdr9tGSN1JIaXGmBjOwzDGxBZaAQ0RAKLr6qphGhoIOJdVXRtjRqPRnjrw7vyi1+u5tuN5wW6zVZYxxuR5HrqGc44h6LpWKcEYeTw5Ozs746LFhBBkxqOhbVtSqkWVRYFvO2S9adq2tS039P3IDwghQRAkvV5TVm92u+l0+vSDZ8+ePFhu1pTB0Pe07IAWmNLRsH/v9DjPdsYoA8x+/aGU+OyzT33f3242/X5/OJoQRoVWTds6fmC7jtK6qppk0K+qiuwLhBBumsayLMdxFAUAgK7rMAR7BaKu6yzLDg+ODw8P79+/33ElhCrrtm6bV+9fK4aLoujqbr1eH0ynx0dHo35vtV4u5rdVma8W8/ld17V1rx8dHx8Ggbfdbp8/fx4EQa/Xq6vWtu1er7c/QQkhURRpbeq6Rgh5njeZTPJKNp0WQvu+F8ba9T1CSN02XcsxxpDg/WnatSIvi6qqQAUhQhhjBQzEEACtjNRaAa0MwAYAZKSRAhhACUEYYqOAUQZgjBEllDIEodlng8E+zwahNkZILZUAAHSdJWXbcdV0UikjpW6aarncrJY7qYBlOZblVFVVNS2lVAhx8T7dbdO96VhrvR/rEUJGA6NBr9dDEGtltptd27ZKqfGojxBBNkIIWZa1b7/sum69XhpjICRSaISR0UyKrm1UEiUQwq5p8zQzWsZRoJQqix3Jdq1S2rZCALDWnDGGELBtwpi12Wzmi2vbciwbO26f82a9Xvz2t28pRUXZfvDhg6rRZS1221IIvP1vf3j86IPT0wf9ZOCF8Xa7Xu52TVtbHhyMEg2l41sYgrop8zwHRiMEpOFtZzpeYowtmzLGCIK2bQOALGoBALTWGGOMiADKYrYBGCECENHAcKG6ruukqLJKQ0AItajNLBczpiEyxtR1XVWN5Aoh1JTN3c1MSs7Yn3zdtuPsFRghRFlmdVGGYfjTH//k/N3z5eKGd6XLcJtnnueV5R8ty/O95NFHk7o0F++WN++Xm3qBHBP0XGBg2LluYO3BtHWdv3+f313fdJ0Y9ke9uB8EUZEV11e3jx8/ffDgQZpnCIHtbrVa3/X7/eEo3tbzbFdXohqN4uHwHgDAd73T0xNqo+12Qyx09ug0SvzZcnZ5dZGXWdasHceLo57nW7ttUdc1F2rQn9gOL/Imy6soSvrjMUIoHvY9zyuqeWfay7vLn/7Fz169easkPDi5d303++FPf7ZcL5jj47Zkjj0+PMjbohLN+3cLyaWoa0g6KSVCmlqEMpaXTZbzxar0k2K1LW4X67LIkAFGBlEvefD4wWw5q++ug8Cr6hJhGARBXRWr5dz1PaWUUgIh4NiMEczbpi4Ky7KePXsymo7evTu4W1xSipuqXixmvOHptkwC3Q+GTuQiRhzP0kYul/PtdkMIsizWtfV6nfZ6cT9JNqv13c1scXuXRPHBwUFTF5xzKUXHG601F8oFaLMr4jjkcn5zeSdEMxz1KKRQ6tX10o8eKaHP377bI60++d5nzHbKos3SKukdhN7g8t1dEve/+N0Xb7577gfu+5uLqqh7YS8Jk7u7+WKx9N3g/oMjYTBXqJOyVbBp9d3dTKi7Rd71JjtsWVwoYjvL7e743pHrBUILhHBZlhDCd+/elXn+9MkHH3744fn5+cFB/8V3z6u8QgATSByGjYFl3qzXq2+//XY6nXz4yTPP92fzOylALxnUuyawPYws3imojG87NqFd2ezKbRIGgzhaLWY3l++m02nsO7Kt4ji+vLz89ttvJ5MJQmhfobtYLJbL5b7a+ODgII7jN2/ecM6TJBmMJ8aY9XJBEHZDz/O8wHOKIsvyXdfUWgrOeVs3cRz7fhjHvS/fPQ+CAFNmpHQdP4hCP8iMAU3XRZFDCGt5ByE02oi2ybYbRmyEEMFwsZ13XfvDH/7g8PDw5vaKEDQcDgHQ221TFBnC6OnjZ9/7/mejg9Hz588N0dRCUksuO9tmTVNhAqM49ENPmfZ2/v7L519gis4+OBo6DqakbfNd0WkAmYdOk6PB4KSt4S4vrm5vFptl4LPDo9GTp2cff/L0+asXl7cXu2zjuFgCrHgtdYOIIkZx2XJRBIkdJ4dnD054J4UQxV0VmAjbhHpOp3leFlrrHhoWReG5QeA6GsBRL9Gd4Jzf5FVeVH4YAGD2wVHfYgaitm0/+eSTJ0+eeo779u35P/3DP2JMOOd3d3effvSviiKry6puO9G1VdVgjIfjkRBCKWUgCFyvqqqmaeuipIdQdHVbF1XZ2CM7Cn3XdSGmp8f3/TBoeVeWOQCatw3GyLWplt12syiKQkpJKcUYp9vNfH63Xa2Hw2FRlkqpwfi4Nxxk291iuWSWNegNpVaqbWzbjnp9Pwzmi8Xl5fVys1ZKyU7UdS2lNMY0TSOE2taZEGI4HEaB77m26DjGeK+gpGn6zTffNK1oWw4Q2Ww28/kcMJZu0n3gJcuy3/23f6EUC941VYmR3mxWu+0SIk2QSj562lRZke5ePf+u1+vRJ0+2223Xiel0Sim1LCf03SQK0jS9vbpaLBYY4/ntbcHx1dUFxng0HhKCNDCct5vNZnww5ZoDjiAiWhsu+J8eWY6jlOJKaqAQQpgihAEAyBgAEdB/+jDIAKQRRAgYAiEwxhgNlTKAawiNNoYxCwCwZxlBCI0G+4wPAMBxHK27XVZiZFFiI4SqqqjqmlDatsYYU5Z506i4Ry2LQQ3bli+X6z1YqG1bSqnv+zc3d9ttur9CLJfLqqr23gUhNMbKGCmEgLB0XcdxHMuyHjx4UNe167pKCwAQwYQSBYxVN3Vd10JwTCDnvCgyraXneeTTT36MEOh4s1wsMIbDUc9xrO1uTRlGWAPIm1bs0iXCxrYcy4YYo/ni7vlzlfRshGEQOnkBWt4gxohDFQQlb7FjIdsuOZ8vl64rp9Px1dW553lB6CECMAGWZfm+a1sMIcBFa6QwBmijhYYUMddhFmGc67JqLOqEngc8XFccEUQYpZRChIRWuAWoNS72EcQQEwOxUaopKy4Ul4IxUtcVMiByQ6naPM2ZhX3LsSyraSvbtdbrNbHJXsDp9/u2RRxmH0wOjRS8dcu8sShk2DKUl7xzAtfteXboKGBVHSja5mo+98JhEg7DZEwJ4U2bpqlW6uLi3e/+279oDX72kz+zqbdIl13Dbdu+vb1u2/p73/uorHbCVLPFRV4uqEXO7k03PquK+v7JvU8//mS/nOv1ekf3DrfpNi22GijJFUWUUUtJo4zEFPQGSa83XC7Si/Pb5TpLekeIEsRMpzpIWW80RAh1XZfl+eXt1eXt1W5XdMpc3t1YzL+ezRHBk8MDgVTZVcDGfuwRj45PxsPj/mCQLm9ud/N53/enR1M3cIlF/Ch0vNC2437/WEO7k+LqekWx8XzbQL3LtrZrYUr8MIAYUsYGg8Fw2JdSLhaLnoiVUlVVWZYVx/HR8WEcxYSitq0Fb7XsuKibuqIkQQg0VbFarKu8DtyAWRhjmFXbrmuk4mWdV3UGoCqyzXf54vhgOhr0JqOxqFti4Hq1ybe5Q+z7Ryec4sBzsu3u8ODen//5URwndcMhqCn2lSCbZUkxG/Rjx/F2q5xSHgeRVPrm5vpffv8b6rJHjz9gtgUhMQCVVdfrDZbLOdC86Xauqxzm7tbpxdurvFctF5uqEf0kiHoDjdg6rzfbDDHL88MGWfP1Ss9W7maz2GwhBq1smpbv8kwjKJS2LQoVlhrk67VjWaPxIAiCR48eRaF/eX6Tb+u2kQhI1/YhIMZAjK0w6B0dHZ+ePmx5Y1mW7dpR2IdVzogFIUZQY2QwtJBGmovAcT3LVvdPL9+f/9Pf/eqHn3//yZMnzCJt26ZpWhRFnueTycRxnOPj436//+LFC4zx3gzFOX///r0xJkkS2wkAALtdlsSh4zjQKCl519ZFunMch0O9mM/fvHnz/e//wHdch9lv35wnSTKZTAajoed5Qoi27TDGw+GQMdY0TbZL0zTlUtR1HYTe6nYzHA0sSoBWn3788Q8///z2+vIf/v7vq7b5yU9+FIZh2dTq9irpJX/2F3/2+eef/1//n/9+t94kUeK4roWY7dr9pD/o9xaLmeWQMLGhpRbbu1bniR9lfIWtDhHaqbxsCwWJlMghBFnw9PDe93746Ys3rxe/f9l30EcfPz48Gt3cnn/x7ReSdwYpBU3HS0IBNNwYHgaW6/iYksXydjo9xJgYqW5ubj0V3Kxn6+2qlW0jW9Fo3nV4RjmXgetSyhhjdDqNXXe93i5vb15eX48mY9f1jTGWZSFKpDaWZdV1fXV1ZZTmXN6/f/+DZx+Fvs85nx4ekNVS8LnRHGMKDNprhxAiz3M474SQdVFJIZTQCJGuy4xRBiiKEGOsLquiqm3brsuK2dZwOMQELpZzYxSEpusa6nm2RRVBWmsIdFUXbVfHSWjZFGHw8PGDp4+faA12eVG13dv3FzdXt3EvcQOfWda+g2avRd3e3oRhCJQpigIaMBqNMERZVriuCyEcDAYWJV1bp7tdURRKSoRQnuez2aLtJGO2gbjl3cHBwbYsq7KEELquu12tX373bVs3jm0lvaBrcwRVmm4di+RFCoB68eJ5lm8h0hBpypAfuOvNRXtRTiaTARu1bVmWadvWAEoAZVWXy9Vdye2u65hFlZG+G2ACq0pRB603M84lgNCybAhxy6WUGiMCMDZGGw2AAQZqA4wCAELgOpYxyiitlUHaQAClMkoqA7kxBkCjtdRaIwwgNABoKZQxBgAEIUSIQAiVNEopaluuaymZd+2WYOE6keM4xhjbxgjTrusApAhrAAHCknNuWcF/b3KSUlqWtcdpb7fb9Xq9V2sEl8yi+7kEQ2RZFoSwrPKmaQghvu/t3/Wcc611nufGGNf1m6YhpM2WN0oJxlgYuUqKMt9gAoeDkBxMj9++e/33f/93jJH7Z8eUkSjylFJVVVoWi+Moz/PNdqGUODo6GY37n3/f+eLLGkHJbNMf9IvqWEHe1N1kOp4c9imF22KDbURdakeO3sBttu6Pksuby9Ggb0CijVSqo8wyQGJCLYsSakuJCIJ7WMRumzuOAx3S1F3bdq7tJ4O+Y/sXF1cYE0wQItAgAKRRQAkttFEQQ2OUNloDDBGmhOz5x23d2A5J4pjzOvT8KPYpQdvdWmrhBwFXkkLq2W7T1K7tFeXum2++Gw6SyfiAQvPVF79ze5Hn+L3x8eXFzdXstq3nDPcCZxokw3TVfffqnedO2VnPcwIjzXqz2W1Sz/M450EQEcIIYcaYMAxRiCml33739WI5++Vf/WXZlEiKm9v3TVM9ePzgL37+Vzc3NzeXV4PecDo+cJizXq+7WuwWqe06oZvczu/W2aqVLe+A4lgZzYV68ep1VX6NkU2wy0X+xy++noyP+v3haBwwxoQB6WZdlNlisZhOe+E6AcT67sW3P/rhn2NkzzeLn/70zy6vrrb5DmAV9B1DwGI1X+1mcS+ejI9Wd0sppWVR27baruFSKEgMQE0rEbWrknuBvd2t4tgFDd9ut47j+L4fBAEhOE3To6Oj73/2aV3XTdMAoLXWe5AXNKprKtRBSmlV5Ntdend3u8u2y/nder3sD/wkcm3boRQTgi3Kuq7J04IzVFUVtUic+I+fPKi7FCJR10XTVldXV7vNtsqLfr8/nUx40/Z6g8Oj6Wq5GQx6GNHBYIAxOzo8Kct6s1p/9NGnGMLf/Pof16s50PD05GAyPLhZZoRhizFEEOfNaj0Pwtj3EgiYQ70gCD799NO//X+fB4HrO+bk+PD49OE//MM//P73Xz558ngyObh/+th1/bLqBNe3y/XV9SwZDs96E+bHYrndNSLPivdXl8f3jg7uHT5+/LiRLdfteruu2woRAjHebrdJFG02GwjwvZOTtuVV1eR5oYSsizrbFocHB7Lj08nBZDr+4IOnbmCvt+soihzPzvN01J9oqaQEWCsGjVaYQMowtSwn3e4OxiPR1t9981XoWgeTAYCmrLt9pH4PblJK3b9/fzwe720K8/n87du3Wuvr62uM8cXFxWKx6w96ts2ePnlkWUSJzmhXSukH7mgwnM+X2+32zZs3Tx897RKulIqSWBnNpaCUKqU2m03bduPxeLZYWJY1HA4ppU3TzGazpmkwxsPRoKqqruvCMHz27Nlsdvu3f/u3y+WyP+rvo19S8p///Ocff/zx0dHRr3/968VmQRANotBjbuiG2S6ljDienfTjvNh2krmU2S559tGD3rjHTaWqLGKx4yFNHIisvJLaiE62ButHTx/+8EeflcXu7PT4pz/9SVVt//Zv/zZtpG0zhDXPuVQ0TjyClWi7IHCrOq/z9Kuv/zCdTi2HZUX+xy9+/8Hx94rbq9ntnef5g9GQMtZ09e3d9ag/0kYajRlCNPAoBHVWiLq2bZsxtr/ttFJlZdF0nDE2X6zW641rO3HcWy6XbfPH0WBwcnLStLwsmu12ZzHqexaXcrfL6rp8++Z8ejA2xhBCN+sdZrTrOtFxRnEYeAAA13UppevVdrNLh4MxF7LlHcJAA6WhCUPXca2mqmLHiqJkvx3YL9dtyo6Pj/f+1sePH4dxdP7uoq7rOI5vb28vV5tHTx5T2yrK8v3Vdcu75Wp1ezv78c9+Oh6Pu7pVSlFMnjx5YlF2fX27yNb7LrG2royWFmVBEPSSJAoT27aVMtogKfXN3ZwBc3rv/t1qdXt7u9ls8jw3UmGI4jjuJSFva4zxcNCvqsxmGGOIEbq6fM+75uhwOhqNzu7fo5QCrXe73XQ8xphUZT6fAQhhHAWUoNVqJXhb19JAYCDQWtoO833P9ihzyMXFhQaCYGbblkFIKKU1oAyNozHnXcs7ITplpIYaIgOAaTmHRgFtINAQQaMRNMYYo7SCECIEIUR7ohREACGqlNAaaq0Q3OMRCYQGQbIvMTYG1DUAoIqj/n8vDRaiq2vFmKCUEEtggkQty2LnBx5jbG9BTZJk/y6+vLzcDwSO44QhZYzVdX1zfZtusziOPc9pmqYoM6VUWTqu667X6/3Nra5rpRSlxXa75ZzbsDVGYQK1GQyH/enBsK7LusnIV6//+esvvyr44nh4AHEzm63ndyKJ4iLP27bth24/dtbrDYQNxK3veZNT8SFK8qq4Xb+sTOyPw7E5efP2slTgar25uvxKStnvhQZwSrSEW6DFxfmb+/eOxuMx2wccgG7qajAYbNfLMittRi3LcimTUm6yDSU+MQExntHQIGP5sd/rM8uK2ni+WrRlbTnE820nsoDTNWrlxtPdNgs8r6saKbVF7DxNkyThbRUGVtM1eZlJLThGRw8ftW0rUA4AyLocM2NZRHS1SxCvq7v3N+UmJU+tX/7ylxDC3//hQph+1UaJTXqxebu61gJlxU64LjbuydEYacsA5XmOzdjV9YU0Tdj30zS1lPPo5Mnh4SFCSAlBbda2VZgMHzw8efvuxe9/9y+ffvrpu+dvZMEPR4ekJbVZnD4b3H86qvJqmV8nYWKHuCiq85u3hNFWtMvtarld5lVBLTKYDpevbtd32WA0/O6bl3lV/ujHPzaGC7F5P3v713/1P8b9seRScsUQXt0sFjezvvfgB9/7ya/+4R+sJEaJm5d1FEd2QstXm2/++DuM8YcffrC52yy3m2iUeMQDUA+G4+1sHfiHjj3JUxz4ngTGd5LdJuVGDA9GSEOsKdWeyLsrOQcuXqxmZ4fHR3Fvs1qktzdD36XYGJdeza6vFtdn906jnre4vZvPWiSw69kYd7/97X+uu6KqijCCy00teFY3fp6nxkBIyWx7i210e3vbH8QIIQjspmo8G33yyeMHj446Ib799tvvvntxd7ejBPai3ve//wMMAjucwE7PLm8JQI/unZ0cnwlpCESSi7/7+1/906///n/5X/4PxMfffff1cnN7kS5836Wesim1tA01HjnDRAV2iWDd+iH1lMI2Xd5uBofjgweH08MJsZiab7q2/vbb55PR8NGjZ/dOzr57/no+f6MVbgs+TCaiNsvz5epuHcFIbaQVjC3SCGlT4kyP7Kv5RZoW8TDK8lIjUtXCDwYQ0XdvL7ab9Ob2KluUQnSubZNA+yERbbVJb5N4jKmjhLVdyWLbMis2qr18f0cpYQaHYQy0qmUJKaEWFdoACAHR/jAWXYcYsV1/Nlt899W39+/fn62XURSdnJxADN68exVE3sXVOcQIEPP++j2m1m63u7udO55nU7uWdT8+AEqv5jtRf/fhR4/6g7Co8jBw752dtFV99vBUKP7HP3z97uqCaz2ZHIz7E9tmQRAYobO0aMuGEGxhMoxio0S+WT24f99j6Ordi67cGmPujz/gxP/q268+/Pij92/fv3j94m657A17kLLv3ryaTqeea//lv/kf4iD8zW9+85vf/Prk+NBxnPtHZ9CArqntkPo9R7B63c0LUTy896g37lcz2Rv0mGNf394E5mh7qzRgEPqIkggRqdH2avvsxMd96y/+/OfHx0dB6Hg979XNi4vNHQr9SrUMUtdmAtG8Bi5zGAvaCth0lG8WBebzm+Vkgq6vLiiB0hDKfIgsKTRB1KNWDZDhostybzBOgrgqW4hp4PaUuOUN0Fg0vGAOIrZlCaMBtqilAOz3fNf1KbNb1Y6OxwjiFou3s/cXt6s83Wne9mPfc2yjQFOVwzhYhl6+3TqOk253SS+aL9euF1DHv2f5B4PBq9dvJ8NBVVQ3Nzfa4IMjywn9MAy5qIsic2zquFRraduWBd226JIw6tqWt11Lqml/6FFaFMX9eydI8G//8Ns4jnfZwrbtzz44492plJIqfn17udlsAACru7lNcATVgW8XUG4swCihql2sbt++er7Jbge9PiYUgBJDPEmS0I+klB4FFjHQwlVVLXZLwneMki47T9wwZ8od+Eaq0XCIAAxcL8/zzXZ11Ot/8tFHNkLvLy+ePf50s6nu3XvWTwqIjEWJ6Frfpj/90afL5XI+n2PGLNdb5WtAWCtMWjbKcrZpRkhiuZ4CquKtZbFnT+57Ft3tVpPIfvn61SbLuQSAMG0BaONWSwM6AUUHOo0EhJAgCCGCRlFsKSUkF9oYgABAEGhgkAGIGaMAAABqCDVCAEGAIKp5tReQOtEZyQkhEGBlVAijqiw9x/UdYFtOuskf3Q+gIYpDAEgvCoGBsqs9yoBgDgm8Huy6bruper3QaCm5GJ/cu7q6sqgtOsE5BwBJYkSnESJx1MM24KZDEjLbspWXpnletEajqqq0Ep5vR1FggORN7fnO0cEIVE1ZllHsQ2iy9SwIXZdCYDoyu551bUsBqvPiIs83m9W+5LTX6xGCDYQIEz9OLMeJe4PheAQcvNguitV11my3VWi7ftspy2Uv3nx7cX01my0YI6NB4rk0CJ2iTE97E621AaKsdhgh13U9z/N9v+uaPUnKtm0MAcYYIGK7PsFB3fGmU5yLtuW3t7MsrbTW88XddrtWSvqBGwRO0osJQV0p3i3eB0GQ8txxvLatR6OoyKv5co0QIgRBCCmlbdV0dbO3bhlDtJZAAWggIQRDO8t2m80mK9IgdPuDeDiKsyxDRChTN53JspRZ5OmTR7PbVImmbsrD8bSfTM9OnmJg13Xd1p3FHKfnaK0pZangXdcRArSWSpmmEeWflDQPQLzZplqBn/30L376E9M0ze3t7eJug4HVi3qagSIt5uWCMXu/xvvuxfP3V++vbi5vFjeHxwc/+umPemEw6I/fn1+tN2kYJFEyWK8yY8z9+w/+8i9/aRGrbQQ0iDHbdcF0eug7QVGki9U6z3Nk21988UVZNT//s58nSdLr9Tjn2XYXRaHB5mJ+ra/A67evjCQutZmFwsiFQDd1QwlxAn88Hn/wwQfvrs6FUPfvP3QtuywyYLDrOkYARm3PDbxo4DnOd9999/bt27Onj6si3+12TdMgoHthHAQBNGCzTZeL9XC4Pj4++bu/+89VUyIMXNtr6267yizHVdLsdiljdRj0JpODrNhyJWnXhFoyx5ZaYEqTMDg8PoHE8oOLi4ur8cH0bj53mP35ZIywZTROkl4cDcqqpdRKkv42LaeTw1dvXr548frxswe9XvKP//Sfz8/faqHtyHGYXdetFtAJ3LCf9AaDMEwwolrrpmiMRmHQi5Lw5OgkiHwwznuj8b0HD30/sJhzu5x1smOOfXszBxjytjUap9n2cDpsGz6fz7XnSi2qqrib30Iqt+UaYxTGseBGaaMUhEbt0+cQAiXdN2/e9PrxwXTU64WebxOIJNe80+fn5zO2IJg9uH9mO7QuW621UiqIPEypQ5hlOwiRsmpAJ2zbbqpGCYmA6ff72WhwfXn58vVrx3GYa1NK4zg+nB5ooNu2BQCUZbnZbPr9PqaW47TAoK7jECKMUJnnEJp991IQBIQwAzpKrSSJRNu9e/f+wYMHwBDOOcZ4t9v4vu+6NgBgu91WVaW1rmuxXq9937cdliQJpfT4+PiTTz751a9+1TTNLbh5/OTJL375F4vN+ur6vD9KolF8eX1VtwW2hs8+fPLjH/848P3/+B//wz/9wz96nne3mj1+8NDy7LIouOZe4FGLFFURBH7cC0eDPjBGdbzK8qrM6zS9m99hjPrD3ngy5EIsFgsDcNybXl1dEOp4nnv//n1mIa4bSmmv11tVJXVdQohRWmtgWZbnOgRhybss3Smj5/Pl7e3daDqlxDo5PsWAGAiCJCbANIILoF3P46TbZDtvuRBSI0gD27Ftpz8YnD48W3y7wcgyGhsDMWKeaxNCEGEPo5hRG1MitdEaGGO4FF0rrs9nhBCl8G63E21J4PT0aHJ6elo31bt37wghR0dHYdIbjrP1ZjccDpFQuyzv9Xqe5/X6o6+fv6jruiiKycF0j/8jhOxXBnEcjgZ9ZNh6vTbGuK7ddc18PuddY9v2YNDPsqyu6yRJptPpZrnaK0Cff/7ZarXKsuzg4KDf71dVRSk1AHZdt9lslNG9Xo8Qkhfp7e11mm4BBHu3nW3bDrXCMHRsqyr1Pp27V5Fd143j2AsDy7Kw7U4GURgEFmW9KN5t0izLLt7VDcOBwxxK+lHYDgcUojLLm7KaTEdVUSol9m5/znld1w3nvmVpBSDEjDkGKVJ1kEDfdZbLbRKHEKPFbL6e371/9+Lp4weT0RBihBm1HNt1XcRsywBhEBdC8RxCQwgCmEEIjdFa632z9v7vApoAqAEAxmgAUNcpAxQABgCNMEAKYIwgUhhbABCtgRRAa220RggZA5umCYIIAhTHITC069R8Pt//Vzn/k1dx7/kAEHLOjVZxnPi+n6bpZDK5f//eZrPZ50ootRhjAAClzB4UTQipqsq27T3hd9+q1bZtWZau62pFADBt21KG4jg+OJwcHR1FFN/d3SgtiiLruFK646KWihNeVB5h2Pa6sq2qQoouSZLhcMQ593wPICIMgpRVrbiaz7mBs+1MAO4nHsQQYFCJAmFrcjhM+oPNOm1FTTGyPDIcx6f3D4FR6fWs67qqEXWTAQD6SW8Pq8rTTGuNKVXG8E4xhlzXH1qe6CjnHAAgVbXdzFerjRZSdBwhlGc7IToMjTHasZllWW1b+/0nP/vZzxAhvh+sV7u8bOuOL5brKA5c16WUGIDatrUsqx/3J5NRq9K6rrWRm01V143NcF3XLW8c39JANLzIys1mN8eUY9p1ot1uNp4bJnFvOS8ohpriw8PpMJliCLXS282GYHuQ9LQGN1fX8/kcq63WuqwwQgghQghBCGktgyD4/PMfCiHGk8Pjk3uWZe12mcHWPDuvU+kzwLBLsKzbyhjuee75+fkXX/xhuVm2vJFcxFFwcnDo+s790yeOHb2/vJjN5oobzws++uiTH/3oR7yT5+9uLEIPDw6URDfXs9ntrMzy09NRWTfT8YFE6N3llVRaK3Vzc4Mg5Jw3bY0JjPv9TvPZZrFaLKE2dDgah/3AsaAR0AiLwtGgdzO7Gw4Gs/Wy47zfH9ZFfX15Y5TyhqGUIB4M+vFg3B+eHp5sNyl12Xq5ent1kZdZMugxakMILcq0kPfvP5jP53e3cz8KHj9+JiV3HNt13f/6m18bkT56NGKRiyFVGgihwigpZdNst5uiWGUpV6KsKkxRmMRpmtddO5iMx5MDLU2el5hRP4rTNKtqcXAY+0Fclq3RmEtzfX1DiGVZ3ts3577vHR5OD8cnWiGt5cGzodbg9mquuD48O7r34H4c9jwrVMq0dUcI7PVGhNjEIlqgbF1D2ATD3vjeQZYVUnZfv3i+nC0W8w3Q2LMDtxf6dvDNN99J3gANXZcUbQGgNNAGUCMMLEq54rzqHGYJoQ2ExCYQGQiNbVHXtT/59EPLphhqbQSz/MlwBA3KsybbVavFYr64G/STvABtnUspLZtoCLSRmDDf9TEiXdfVFdeS9ntJvkuzXWqUsF1HiK6oSsd32k50/ythySJMS6WMhgYkUTyaTLK0SMLk4YPHxgCtwatXr15nL7XWhAKITNM0BkhmIWCg43hlVl5dXQ/6I8YYhqSqS0JIluV5nkII95VvQggIAcZ4s9kgDPbM9cGgd3h46Ps+Qujpxw8Hg+T7P/h8l6XnV5e2Z0f9ZLXdrFaL3qB/cnLSiOrVV99lxXYw6T18+PDwaDqdTsfD0XqF24phCynecdntdtskifI05byVdWMwgRgMvNCZRnVdXV5ePn/x9cHR4dnDxwCyu9na9SLbDt0gDKJRx6vtbCGEmEwmarOxLIsgJLlACNnM8f2QEbyYzaTUSTxs6pv1Nq+qpqxahJnteEKIIZkYwcuu8brWsploG0iJBIYbZTPb8l3XD+PxYHQw7d2OESJGg7oSUmhEsONQi5CqbGsoAIQAE9u2GbUJxhKDe/eOKUZFummKdNRPjo+njLHtdtt1nVLKsqyyLK9u7xouKLPzrDgYDDa71Lbttm21QQihg4OD4+Pjb777VgghZOP77nCQOC7FGGI8wgApJdqu7LqubkopJTDGspiUMt3tIDSTyWRvKi8Kvt2k+466zWYjjd5n7eI4juJ4uVw2Te14bhRFCKHz8+v3F+dFURwcDymx9rgFCjFERkpe1UVjAAAaUwsAEIbRcDj0w1BrTSzgUhN4nug41I2RBQGNTdTJdOgQcHf57u7isq6qNZtRSh1I97nZqiraui6yzEDIpXG9AGFLGqAMIIQgDYUQkGDPdR89nGy32/liJpoy9J2qbtO8cBznZjZfrbctF4AwCyBAKNRKig4jBCHEBEEDjDH7YUtrsC+k2A8NwECwb4DQGiFmjAFAQ2gQAghBjCDCQAEFAdJaG22M1kZTAAmCEBjU7w2k1CcnsGtl04iiKBzHEVxz3iolCGGUkn0qUGlplJZSWJallAqCgDF2/v5tVTaj0Ugpta/XUkruAUgIAQwwhLDrOtnJPUgbQtg1tRACI7Af1DxmHxwcPH788Ojo6Pbd1wZyTIztYEyp4xJtoBGaFGlGCLEJA1J6rus7/cF4lCRRVuTxoI8J0wghgle7dLnJWwGCvv346RNEnzRdc3F98fL1G8H1cHRkW4EGqteLLUqEqosyvb0Rbdf4Ggsh9w8Oo4Bt0boOlVKd4MZA03ZpVu52O4xpv993HCdPVb/fD/1gm5azxXJ2e4cMAEp7tiMbQRGBBgjeSq6IraEk+S7jTef6FsEsTHpN0yCMy7qCGERJGEWhlLytG4xAXRW7Ld5sbiEGhJAqL6BWOg7TIkeUnJ6d7dLV67cvmKXbroKI267RQpZFZTTBJrUps6jojw581+665uZyeXzwgFILGeNYrta6rpv5fOGx3HEcqToLOwgZjGEQB47jAYx++OMfE0K5UGlZybTgnLteFHSDppBbUPT6cRj2IMRN01R1gxj1woAUO5fAZNA7mh47lh26IQfeoD/FxDs5fvThx58cHp1UVXP5/vrbb79dLZaffPjRdHzStt3rV+fz2W2Z5eNx5DBHI7rJC0qp79s3NzeLm3kvDPJdapTGAMZRQJxjQMG7ywuoOdKSYCBF1dSU13y7htSxz9+/na3XWmvLdbpWbNa73a7QWjZvm1HSH3jJ3e1SVnw07DuWc3B0XLbV9Wx+dBR99tmnoR9oziEAom6rQiltyqp9+/b9YNDfrtbMcU2He8EYQjjsHSCC60o0XbtcrjBld9vNcrMqyxJiULfVNksJw4PBACG03W7H4+lf/Zt/8+//L/++F/eSfk8pdXO10IAyKyDMDWO3qur355ffffuiqouf/eRno1EfEyilOTo87Vr1u9//y9H3J5EXAgi7RgWRr5AWSmoICGGea3suJNjSAnHJ1/N8tVo5I4EgIYHrAPDw7CGy7H/+9X+rOk4QrbJm3B89fvi0qvNvvvlWS3VwcNA1wrIxc4jjMOYibTqetbvNNgwTbSCmzHEcAKWQrTFGG/nsgydZtlvMbze7tG4yZNRoMEmS6NGjM2QuMEabzSbLt8hITEDTlorDwI9cF0jJAYYQQiVkwdOmKC3KHIsS4viu0zZVWeZplgGAEEKKCwwgAmBvaDJKa63zNFvOV2GcKGUOpocPHz5WXJSblDFm2dRzGACAcymE4u1stVpVRTEYDIw2tm0P+4NvvvnOsqyjh0/m8zsI4XA47Lru4uJCKbk3om+2K8uy+v1+mqae5/3iF78oiqLlzWx9zdXT/jRMph/vspS69oOnp71e77uXL4qicAN7m27uVjMF1YeffvjRJx/tNtvr+W222yAAO9EiqEeTyWg6hlp1Tds0jUssKLSoeRzH3ihACHayfvP2eVZk06ODR4+fxb2B0rjr2roGUlvb7ebi4iLL0jAMqeu3bds2tWUT27KMgUIoipngyvcialPbDbK8urqeIWIzm9ieZ5rGCwPVtdl2V3YNpbTuWilVIziqyrLh1HGdIBRGZ3WZxOOu6/K82KR5WZYGIMdxLNdpO6G1ltoAhDzPC8OIMKq1xhq4jp3nueat5ztRFCmlVqvVPuSW7rI37y5evH4TJf2T09Oq4U2Wrbc7y3aFEJdXb6bT6dmDJ5brnJ6eCiHariQEea6lDc+y7ObmxqJss10qLlarFed8PB7bzKrrOs22ru0YYxaLxT5oBwC4ubkpiv/sed56vR5ORkEQbDabtm3r2QwTKGRHJQFAc86rOgNA9nqh74VG6bxIs+0O9TQAmhAKoRJcdl1nIUIpo9RWEu52RVN3Uq6LonBsuymLwPWgNjazfActZnNRuxAgyyg38nmWX63Wbdt+9ONPXZtRHDZN0wkVBIEXhELqoqo7ITmXUmrZ8aauHdd1AxdALHhDIDg+Ozs7PXYdq6yyr777Ls+zXZYrY5jrYos5FsMYSYEQpXutAmijlABAaw0gxvuCBmOgMWD/DUYZrQ2CFkB71BKAUGEEMYYIIWAkQghBACyq5J+4TQihOm+MgXleSqmMgZ4bXK5uMKbMI5xLpQHGECGK0D6gAR3Xy7LMcZyDg6nj2BcX7/O8DkM3zbZts09gAkoxIQwA0LatANwYI6WkiPq+73mBEKJAsOs6SqkBsm1bA+TGdS4v6W63q9LLoih83+1U23alRkxDxRxGGGO+7zuOBQBACFgUcyW32xRTAgymxI76g8FofFBX89UKIWQFwvYcKVuxW0FIlFB5Xjl2mRyNGbGQjZTo2qLBkjQYQqRtJ7AZoJQqabgUhDotF1XN9zYcCgmX7XqTZVmGzy+MMZKzDz744PBgulgt0zz3fT8JI8mFaFqCsEMZAqgpK6ANBkwbxIW8u7m9/8DSUt2/f+/t+TtmUwUUsUh/NOj1wvVynuXrLN387re/bspqs5kdHR8/evQIMmIMlFLt0hJhazAat7xZrBfguQCgsxgGQNoOnl9WDo2By549efJ186ofJxcX5+tFKVrYj4dJONytt9CAJExC3+vF0fL2CkNUVRXvpJRSGuN7oWVVXKoo7tme2+5yLgVXuhUdMMh1oqatpIaEuZZFs6pMy6LhTX/Q+96PfpBc9o0xR0cHCKHb21Vdy1waY4Bted/79IdPP/jo+mb223/56vnz5/PZcrNeBk4ceBFGcD6fd12HEOoazly76kot5cnhsQImXaeB47JebzwaNk3VtfV2vWKB43mOzUhZ5hiaYRKHQWAjIgGvyrzMC6P0mzevNCGHR0eb7WqTbahFyrxRWRNaTp5uZVmmjqe6+23VIoPqqjESdLW6ej8z+tZhlk3wYjavG/3w7MHx8fGrF2/Tdfnm1WtCSC+On37wAbXYsHc4m88vz6+X203Du+U6vVzN67ZBCPihZxAg1PJcLwiifYsM0LCt2tViCRW4d3xqEXp7Nz86OnH9gEvFCFutVhcXF2WV379//2c/+2m/n6zX6126sZlrURsDdvH+3cMHjweDQVPwNE3Pu7eRPxwlnUVdzw1tastOSqEptgyFGOLf/vG/tS1/9uwZUMBg5MX+YrPYlbvJaJIvdsS9d3Bv+NeDX1Y8v76+Tuu1pq4Tun5kG2SaquRNjTWwsUUBwRhgQgmAvBNVXRY5L3JapqllUdehUey3bX1x+VYrde/k4fRgjAyOo0HX8KIofIcZYHbpxgDmBD5AsG1b20a+53SVne1yTYwb+F1TbVbLIPTu3T8pitwQ6CCLUtqLk36/LyTPsmyf8OZtJzqhlNpttn/43R8/ePbhs2cf9vt9qXhgu2EYaMnTNIfQdLzO0u1nn30W+FEcx/tsS5qmGMMHD+5nbbuHwPT7/X0CM88z27b3YTCE0KNHj9J0q7X+1//6X19dXf3Xf/670/uHUexusrQ3HETQrzqOMBSSWxbjnC0W8++efztfzAe9PkLw9cvXq9Xqyy+/uL29dZh1cu/o8cNHTz54Nh7237873y6XFoX9OMIQ1XV9NDlobLbbbX3fffrsseUwxhDG4Oh4vFju8iKvshQguNlsZrPbsiwdzw5st20aDFASx67jyK6r8lpxLYRxA2+z2UgNN5vs3fnNn//Fz4MguLlZCyF812MYFWmmtMaUMMdu29xA0HRt15Z+GE0Rwhh3XQcAltI0TVeVTVm0BgKljFAmiEIAkNSKSymEyPPMQCiEIACEvpduVl1VhD5zGeqF3rjfm04O/ut/+fV6vZYalVUdJYOqbFabbH13I5Q+Or6X9Merr7/781/8JcLWP/3X/xrG0WAw8AO7LPMsy5TugsATQlRFen7+Ok/TNE0ppQBo13EIIb7vHx0dAQAWi8VqtXItmxBS1/W+pmi1WuVZ4TiO4zgYw07wfZd0lvNdumqaZrFYSMX9wGnbVgm5b0KilIRhwCjO8j2PFwK4l+tF02yaTrRtl6bnbd0MekkU+qenJ7Hvia5pyu1knBRpMZ8vKbWipI8BHCcJQBAD6FCrlKqpO8slnh8aCDfpChNLdrLjuuNSCAG1shgLfN8Y6+RgqsYjSvF6vSmKlFmk308wsRzPNxAQxigmNqOYMgyNAN0+qK+lMgZBCDHGECGlFIRQgz9lGYwx2ihjlBDdnpcAoEYIagKUMghp27b3oEaCbYGEEEoIBQGIooRSq6m7oiwZdfu9+O5ugYjte0HXiazI21YbYyjDCGNCEELIGEAIevjwrCiK29vbfj9ECCml/IAghDCi+2u5lFIpiSjar3gIIa7r+r7ftm1TlUKIvanSGEXIn+oaqqpyXB9TEkVBmu6KKjdtyyzi+QEZTiaWxRBCRktKKcVIFkXbSVE2ZcUdz9OAhkE/chMysrUGi+yqa5rVep5XqUui6eC0zt6v71JRXpV5aVsuUMYjfuLa2GiGMMX+/leEEEspmW1BCKs2r5rO8zzbJX4YB72qaNvtbpdlWeD0m7aumrpta9ezp6PheDSqi/LdqzdKmU5JoHXFWy0MQrJtW2N16W4dhZ8qzbN8+/zlt9S2IFGnj0+Svq9Ml+XrXbowQNVlVRbZMInHSX84GBBKJTQI0qrhbpBg5o0mRwqoTnAIjOdZXd05Sbic8UGCI3/4+NGHL7+5yrL8229eXF8sDyb3y3yXBKGQ9XbdMQI9n/Z7/vIGU2IbCQ0GSoGOC8c2Tdfq3W44KpnnWK4FKERSEoYNhLLFbhSEod8bDYTieocENcqg69XcdV07jBmhw+nJdrWZL5basGDUhxDGybAs6v/8q3949ebt2zcXUkpKrbaR8/kyCv3jo0NMYBB4jkVHo4lUqqg713a9KJ4vFl3b/tmPf3I8Hsu2nd/dcN4uVwtfhQAq17XLhYEKO8wxXLeqJRDZoXf/3vEq2/3qv/xdVYuoDdtF2bal59Msb+PIT3qe0l3XqQ4hwVuMcVU2juV/9OH3MLXKuqqKsjecGC6Xd29u5rff/+wH08nR9z76/Ksvvxz2JrPbeUt1uWuPjoeJ1+9iqSW+uZ5JoyAirVAYUcooNEQJoSXgrSjSOtWZyxwowe/++V8OR9Myr5iGvuUUVWMgzPN8tVrtabhpmu57b7/55quz05ODgwMMVdd1/PB48u/Gb9M/os5AqB3ENNdctfPtzZuvXmtFpuPpycmp5dgEIcoYJjCKojN6luflaDTinN/Mbr578e2L129kB1re+EEwOh6ykI5Oxo8/edDBer1ex5F/fHpweDRSst6uGwCAbVkYWZQwYtsIUwWU0cSzHWEAAGa5unnw4OzR41OM4Zu3r+6ubxixhoPJdNQ/PJxS4l2mF0qJMBw4LvN8xyDIbEosooWmGNm2qzthuBz0+hjjzWq5Wi/aLhiMBmF8ZFnUQ54QIs9zJSVvO9FxymgYhkDrqmkZs4u8apomy7Kbq+vlfNE0tZRdnqei64ToBoNer584ljsZHwAt1+v1w4dnaZpevH//ve9972d/8Rf/x//T//nu7kZK6brucDh8+vTpYjG/ubm5urrK8l2/36eUcs53u10URbe3t+luc3J8uJjfLZbL3W7HlRRGQQjXEFKLhb7//s07oMFPf/Dj4+Nji7Dn377o9XqyVe/fXDStdBz36RPWH0zCOFH6Kq851gAgx7VtCBijnjeIMAZFPYJIur41GvcgErt0KYTioqrqRhvYNJXSYs+zA9IYqWxmxVHk+35b1U1VQ4jqtnM6VZWt4/lFXS2WG22QNqhr2z/BqQiGEArOIYSj/oAYbFFWlvV6tYMG2cxZLFdFUeRZ03WdVoAxO4oItZjneZbjOJ5LKYUYSfMnLnKaZWWZIW3apkhXqzhwh8Oh7/sIQdd1Dw4OKKUQ4oOD6VADzCyljMUcx4ZIyL1QpLWOoujlq3e/+93vlNEffPDBcJTsdpvNekEoePr08aMHZ5ms54u77WrtOI7nOVWdaSVGo9Hx8eHB9GDPKXr9+rVRAAAURUkcx9PpFCG0WK2yLMuyDEIzPZzc3NwAoNM0Xa2WTdNIxRElQnYdVxgiCKFtW65rEwrqupzPb3zHhxB2UtWVEBI4to+pg5FFaACg0YB6fnR67+FoEC/uroTgN7fXWkjPtz549tF4PG0bqZTOygKFbtM0m+Xq6uY2GQxHk0PmOJDaBhEAudL7c5q4rht6bug6k8nJ/oVerubfPr+bL2YHB5OzZJiWFSZsH+RumqblneM4bdcwn0EIEQKQQAOgAQhjDI2BEGoFtFL7hIJSSgGFoJKS7yP3e5lBa7P/Yu9pgBACgJRSWgspJQTm8PA48CNKF3XV0siL46TX6zWd6iW9NM2LqhRCCGGUFsymEJm8KKLY7fUSynCe58aoKApub2ddpy0LEUKbphHCMIYmk8loOHFCd7PZLBaLuq4hhEqZvW+PEEIJUpprTS2Luq4bRZHv+7xeOY5/fHQ0GXeDwVgpoZS0bEoUQa3Rqm1FxymGlmUZoy3LYszebtI8q9pGZduiPxyHYWwAYDjkvM22XVl24+loFMMrs1wvlkj4g9700YOHnsMC35Zd/tvf/fpqfjEaGsZYFEvHcZhleZ6PMUZYNHXBhTYAWzYDCAVRBDFGhIyj8XDYHw6Trs07XgWJ3x8mvV50fXPRyrbtOqChodjxXcfxlNDz5W2Zb1er2bbIVrvt89cvIUVBHEnTvr9+W+bbPNsCKA4mfUaxw06nvYnlelEvURC1UikEBDeOHU6mJwbwpmnKYu1Ydhz1ynQjBTIKVIVWgohGC27qtraZkyTJ8eEYQQWg8j3a1K0UFaOW7eDDw3uj0QgA4Pk+QkganfR7bcvrpulk1/IGYWSUxgxZvoUQgiZQWluORTwGDeodDGlsA2gurq7SNLtdLquivpmtjYGe7QXx1PdpWVZKiTfv3lUNHw0nXavm80Ucx9ku3SN4XZdBaGyH3T89QQZWVS2FaOqubBez2UxwLjoeRRHDBAFDKVPAFNmukm2Rp3XZLu9Wl+51Zu1iJ+wnfYqoY9lHh9OTk6PFbsXbslSibjLk2tp0npeMRwmSwHRdXcr5/K6pm/VqhW0nSPrJcKQ10EIO40GR59PJujVyu90JoYbDsUM931HjAVRcrW43tuWtF9u4Pzg6Onn+5hUyUintMIcxBjBomqqoCs5rRGCZ5hije8cnnmvzukuCEHDt2U6b157nXV9fz+a3s9lMKbFbbxzXggb85p//yXPsf/tv/+2HHz0NfEcpVRY5IRFLntZFzbCDsd1JGdg+B+ZqfTu7W81u7pqqPXv4ABFY1IXWykDz9NHjqmqqqvJs59WrV19/85UXUovZg8Hg448/PX1wjwWWxCI57Fk3dkiin/78X4W+jwDfrIXr2hYjkgveyP3hQQjpRIstO4pdTAHXHSOjo6ODMPTzYidlB4Cp6uzy8r3sgMsijEWWZcYYQhFjzPV6Rfenvd5eyQQAMEIC3y3yXAjBKH306BEARiiZlZkpzMf3BoWUknPRdvuBwBhDLaaU8h3Xcj3Xqb+Pqet6z7/97nZ2d3R0sFqtuq5jBEHIPC84vXe/3+vd3d0uZrdd13z22WePHz8mGGIMX333Tdftb1SgqqrVatW27Wx2d3NzI4SglLZt+80337RtLaX8+uuv5/P54Wiihbx8e255frbZUtuK+wMG8Xy+WKyWp/fOvv/J93/02Q85513Dec2RQInfm/Qmo95kuV7xDiwX29l8IzjIy67rTFtWWi2G/YHoujy7wel8OOxNp5M0W1xeXRR1+fDx495ggjvjeY4ysGkFIci27bqumqbhAmBIMCZlUSONXMfxHF9rfXs722x2QmoHEW1g08rrq1nc7zBEBGGojeKizItd0/rEOjo8jMKQt+3ibnZ5eXP1/urq/ZUQcrVYCsUA1I5rUZtijPfuOebYeZlBpAhFjDAILUphWWdStYubOSUIKvX00elHH33g2bQtijAMCSHf+973pNSYOjezWdMp2/WCKLExbzoupG6aZjQapWl6e3t7cnIilAzD0LIsz/Pqyul4tfe7Jb0wTgKgVBDsTV0MQ0QIklJy3lqWEwQBpVRqKaXUWn/5xdd5Via9aDKZEIqbpjk/f2u71mg0EJKXdUkoCqhHSAAJdhwnTdv9RdEw5rgWADrNtrd3154V3LtHDZTv3l01jT4+Pg1jXBYVZp7lEsuJhCSbbYYQ6oSiFjs+Plyv113dNG25Wi2CIPr4e58wZn/3/m1eluPxWGoECKnrFhArjntFXVPLopRBiDAGtmUxSpVSVVlSi/iBH4QPXN+/uLio2/putuiEkgZ0gtd13XVNVJdhGErJAxru8c8YIYT+1G8HNUQI7QcFrYzWGkEFFDfGMEaMMQYoABDYpx4QgMhoI/eKGkYUY0wp3rtKgUFVVUupl8u1lKDrOoxo19UAAMZIFEUIgb3XRwPDeWdZpN/v+75/e3tbN2WchHVdS6ldF1NqYYxd1/H94PT09OOPPz47O/vim6/3DXB75x+E2PM8x3EYY7xruqpu25aLZokRRCZJEt+22lbXtWaWE0VjjGFR5py3BGBk2TYNfaM00BIaoKXEmI5Go7fmPE2zMiubslMc2NTBGLdCCKGqXZMVzXRsTYZHTx/xYVI8OH3kWr5j2bvNojVdVVbX727vbq+Xc2FZVq/X6/f7/WHP9xLLsi3LY8wt66KqZhpqIQRhxPU9TElgeXEcjidDKdrVet6JGlEwnAzvPzqb3c7X602Z1Q3vhAFcGyFUHAWL5ezv/+H/ty0yjSDz3eVutS23X30VlVWKtBwNkzDy+oMIGx1HQbUr67rWwPhxYlmWgrDrOCDw4YOnVZ1+983XZdHGfi+JB8UuFVyPhoe8M1Xe3lwvtQSW5Xz00UddqwIn7Hi9mF03dVdkZVNXcZzUTW4xj2Cbcw4hdRxXQ+04XtuJo+OD9Xab5jsvDILAC6Jw3wA2GhyneVo0hcLSi/zh4XhMpoiSweH03btzQq3dJt+udkpqywm5QF988cV6vekPRt98/WKXlT/56c8PD48+/viTr776Smudptu7OxiFzstX3x1OJx998LgoiqooMaJKyqypCKIaq9vb25/+8AcQQt/3D48PNNbXy9vNfJvnmVGozKsiq8OB3+8PJqOplDLLMtu2Hz9+6K68VbpuirLtSogkwmY8Hjx6fFbvimpXpPPNYjHHmGHLbvJqk1WtNMdHp0EUMEh9l3728Q8ff/wg3WwH/dHtxV2eV1/84auTw5P1ctXvDzfL9Juvnz/9+BkhzBhIqKU0WM9Xvu8Thuu6bERtWdRzHcbIZDIZJD2bWf7Y+btf/X3o+S515nez4Xjwhz/8oSiKIk8JQYyQiPnL+cKyqevaYehv16vFYnF4eNx1XbZLIVM2so5HJ1KC8/WlRvJwdIQVVe23RVnNF3dREgIEdsXOsu3+IMGQurbz5tXrDz/8kFLa68U/+MH/Po5j1/cPD482mw11CcCgNxloAphv/83f/O+ub95/+cXvZvNbZHS/FwMBtyLrOoUM0Fo3TWOgtDyPWRgKHSeOVPzy6v16vaiqKgg9JdW787dX7+dH09Ph4GB/4jZNU5SpZVGvP6y7VkrpMVcb2dRyz3juDF8ulxjD/nBQ1uViMTMI7jcF+9d9MBjgHSrL8uLiIk3T+w/OfvbTP3v06Ele1m3Ll8vVu7fvkyjWoFmtNbPIqD9o27Zt281mhxHy/eAfX7z2fOf169fDYf/zzz//zW9+/cUXfyyKYk98832/67o0TZumHg6Hp6enXLQAgHfv3g0GvclkMp/PJ5MJMTUE+G45ixKV19VHn3x6dHj07v3F65evmO04tl1mZa/XGyXjtVp/+YcvdaOWt8vNaht6oTFQdPLtm4vhYPoGvN9uVlqqumwQZIeHnu/HXd1c3Z2/fPmd0A0mCiE0X9xRizleEAQJZR61us1uV5SIUrr3fNWVCMNQdPx6tXYt++z+/eFwCAyKwuTt27fK6E6qqqnjJNlmaW84ci0bGUAQFk2dZ9lmvvSoNez1J8PR7Ha+H8rzpmjrzvP8XhTvSv6nmyVGGGPKGGMMUew4tjFGawUBopTZjkUpQQg+fvw48F3Psj754EmSJMvZTZPnTx6cnZ+fx3HMmH11My+Kyg9jz/O01gqoIAgMQNSyHz16lKZpnuc//elPn798wTlPU27bbDwel1WKEOKcD4fxw4cPs96urmujoW1bvOWr1UoIRRD2/VBrXZala7l7wP9kfPjtt99KxeNe74c/+sHJyclqtfB9X2vZNE1VF8YYz3MIQVxJrSVChFIKoAYUua67x0hgjPMiJYRIDReLxWqVY2RneXt7O+uP+/1+L+mNCZTbtFBa+DaZTKZ1lSulMoSEVjezG7/IHzx+hBBaLRauHzx9+vT43oPzq5vL2xtxeWP7/nhyQJjB1EIIaW0QQkrIqsgxYt26Awi6rgsxHk3GbdsqoDXU6H9dD+0lAYtghu22bfY9WJgQjMEerA4U0FoDA4AxYF+DBND+E2NsgNIaGaMBNAAYAAEAUAiutcaYYAtTShCyCGGO4+ynfMuytNZGQ0JIGIZF3VRVBQDo9eIwDIUQWZZlRSYEPzic7lvcbm5uXNcNguDd2/dR5GGMOZdd19m2TSkRQrx79+7m5qZo6vV6rZSKosjzPMZsQgjFKE1TwVsuGkKI59tJkozH49FoZGPr+vry3durtqshBFEUdLzpuoaEuEl8m4uKMGKMquv6H3/zj5blhEHU1KIqedfqwO+fHNyLrchxvHb3FiP16OREqcPlYtnvJ48PjtARMsYwZq6vX79/fy6lLMsSIDY5PFuuUtf1tQZS6rpsuoZHYZiudrxqBlG42ax9lyIbzxczgQDnHDKe79IbwHrR5Okpe/PmTb3DKPE/ePQRQ3jYC9+8edPVucVszksEsfa7i+tXY3z85NlHbhA3vMNMbYvt9e2LqOeudqvXl19azEt5c3J0enm5iWkIob54dXnvPnzw4MHtfHY4Olivl1Shl3/4tlyWk+hgEo94Dl06zpfVkyefX19eVYW8fH897g1CP+B150RRWzV1VWAL3jscXauuSBeU6OViJmsgZBFFUVl2LceMsbZZGWMK0kae+/Ll27tL9ctf/g+jZHhzdQOFbFsehz3WVgaCwA6FFuvlGhKsjAyD+Ec/PcSUvXnz5rtvX7w4f/0f/sv/N6YIQLhab0cHA2yjb1/97smTJ6UI8ubuJ3/+0bu3b3f5cruLXSe4eH+b/G8nl82b+CDcbDbHh4PubWrboH94jIGo6vzP//Jf/f3f/z11Qt/3GYlja9Ijt/WonozHWCPquIsyXZXZ4b1TR3W9g6E3iBevvrIsup3NPIg9A1UjnniTvrBkk0X9ITE4Lyri0IdPH80Wy5ev3nZ1Ve8ygtnNzaws6p/97GejIAx69vp6DpU8OZ5s1uOuy1uUa6eb7+bGF42+ZwdOmpcHR4cIIsxU2Wx/+MEP3rx5s1zcNYT0/fjs6KwXJ6enp77v1lX19OkHXdf1xsMkSd796nx2fiO0woQ4Tnh4ctR13eTQi+P4r375S9HxP/7x5cX5+x/+AFnQC+Ok2CGEEN8BxtjB8IBLvVzNijoHjOft+n50pEDTtF1/2GOM2bY926yur696w+GmyJJB8un3PxZCPHl6f7PepduVhdjiZn4wPRkE48cnH/7w+z/UatFU8zJdMgTjYGQhllYFEJAYTKEuykK2Vac5ofD4+EkY+uGgtR32+vWrNJ9tlpXjwGF/3HWcK/D+5m1eF3lexr3k7e1LIbvRaHhPVaPhNJme8lIT5FVtzbDHuwJ07d35qzzbPXr0aDZb3Du8v1hsbTaIRj292z09+mRVVBd3c+on8dRoK+CabYtOAgwhJMDotnx0OqaYbLalfd/abteiqRlGjkvLbP7y+W97vZ7tAttBL18+R+gjrUia8qZBQrQIQdu2mqZ+/fpVx5vhcNjrT/r9pKoqKSWlU0rx1dXFdDrN87yqCs65lnyz2j774MNHx4+rqmu3PLL6lu1t74qqqbWkcMTeXVxj5jgOyJv8+Ozo7NkpN7yo8oY3357/FmE4nU4Ph6PZbVvW6017dzQ9uP/w8PKfvrm+ekspnR5NVdt0FSTatZHvULvIUihbwLvdcq0qefj/Z+rPemTL0jNNbI17Hm02N5/dzxxzRkZmMskki1XVVLUECFBBQKOBBgTo50h3+gNqlErobgkQoOoGu1lJZg0ki8wx8kTEiXP8HJ/dbTbb87gmXVh2QXbtV+Zmttf6vvd9ns5B3x+XKp/P50TDlm0QiDqdjmM6D/fT6f2ccxAn+WBvEgQBJAQSd7WpCLCBwnWDsm3LmV3m+Ptvrg/6x6HZlY1oi6oTuI1tZlkmoSagCjvOfL5sBbcdRyjUtLVqUc1az/PSLN+d4bI4nT48lmUZmI7voCAwRoM+JWo2f1gtFnG0Gc2G+/uTm9ltsXlgQJR8DlmareXe/j6XkNXZ6cnTvGxm01lRtqambi6+swiZL2e9flfXdKVhy+3oBtlmq72+3fU8RzcxplmSlUWLdSNPMujg7SppC15VFeRgnayk4q5jWV7X9jv/+I//6fL6dtjvHR7uPz09wgQtVtssjX3H9n23bGqqa7pCs+Vi+1AMhj0EJVYKcoAVGHY7P/7yi+12W1Zbr9PHthBZrQ/xNltu1aJr0IyVl7fJaDh0c4iRW+dAN0LO0Hisv3zld3rhYrGABFWq+vbbb3/7q2//7M//9Nn5k7wquWqKOtqmiU7Jcnnx9NlL0SbL5co2XNXApCpd27mrr9M0F0LYtqtrJgLIspwwDAPDMwwDQDmdPr79/rv1/XL7sEEIdA+oME1ZW4TqAAAFEKIaJboALaJYo1gwCZhQUkkoheIY6wBISOCOjykVk1JKJSAhlBClFFAaJpQz1jbKtnRiGHXd1lxYjpfk2XQx7/fGWVlgTKNY2ZbpuY5SAkFZlUk3cP2gp+t6FEWYaAhTiMhwPIrjmOoaksJ3veFwaNt2UWTT+UPbtoZOs7RoiszpdqPtAgAAANrb23M9q6qQrwUHBwc/+fFPu93uw8ODpmmr+fv5ah0EgWY4jDEJ9LpplbJJUVS9Xi+wQojAdrueTqc7npqSINpmVcmfP/v4s0+/evr0RRIXl5eXxFOe7SGFpJQa1mzbdhxLKfX+/fvp/fTm5qooCtu2CcSmZhiG0Vas47lt2waO/ezpszRNL99euK6NpAgcT0PIcSwAOauqNIvbsnJ1WZd5gTTHsGzLGPW6rmVCJcLABUpoBPc6YRLFUAkoZVmVtWpOjs+//OFPXL9z8f7y7uG+qDPftwSvCTQDz5a8yuJoes8NDCCgGHDHcShETVm1dRN6/nA45Ly9v79/mE3ruvE9p2K8ZoJxWbZsvYhsy7+/f2zrxjHMl89fXL2/kkJ4bmhq5vXV7fXVnaE7QafveYFSMF2Xo9FI00he5jsvOCEIa1QI0TRVr99P0/zdu3d11eqm1bCWsUZKnuUpIpAFtgSirUqhZJxnrusSiN5+/+av//oXt3cPu05O1tSWZZmmfXR4Mtk/nM1mnMs8LaQEnhf88U9/Jpo22mwxpvv7hxBiCPDhwXHgd5qmSZJks4mqqur1em3Lqypar9dAQc7E7hdqOBwuo02vO2zLarOM8iTVNM1wXMf3PB4qpeI4Rkj2+70yi6sqoxpsWJsVuWGZYdDdxikhRNf1NE2jKCrLEkFyeGLGcbraLl+//mabrj86PRZCfPHFF3EcD0bDZy+e/7f/7f8Q9qzlcrlYLiWGf2SanvAGg55haIZhtLRQSg32+xIyosMyrwxD0ynpdDpAKt/2Pnv12dHe0dXVVWj6miTfv3lXlU2WFf3RDgJY7+9PfN/v9Xrr9fLnP//5fPaIAZzsDztBSA3sB0FVVYggRLBQqqqq1WZ98eGKakaWpdPpFGJKCIGY7FyLehcjqaCQeZzEcZznqRTi8X6qUSNaryzL9uxg3B8a2Bj1+rqm/epvf02wtj/Y71pdDRtAAJazjKUAgDAMvdCLku02iQRrl7Pp3V39xY8m3Y5/dnoMZX2BL/KshBA4rp1EranT45PDaJuleUII0Q2SprFpnneCoBOEBWwVEIw3CBFdpzFvCAWM13GyKcu0bFNClR9Ym+2y3xsKJa8uLuaL1YsXr372Z392d3f37vvv/+7v//bd2+8C3/VtK08SjPH08b4/Ojk8POz0O1fXF0WRO77TsnKx2myiCCGkGdb+0XHY7UMIP/n8szwv4r/5+ZMnT46Pj1er1Wq1Go2POp1Or9fbbFa6rmdZliQJIUgptcNIG3gnkKe2YWGM67omhO7v7z99/jxKsrquAYKj8cDzvLqqLt69ZU3V6/WOz44PDvcNS9vEm6vb9/P5rNPpSNYK3uqUIkPt2LQY49PT081mE8dxVTUQYt/3fd+3TbOpG53QFjMpBARAI/jgcPLk/Nk0evjmm28W09nkYLQ/3ut1+kIoyzSLLJdC2KYTuh7SdKkgYIpV9bsP1/t7I9e01+t1XVa+40pWT6fTftjZbDbr9ZpSSjStqiqi6fu9nsLWZrOp2qquayZF3TZcipbLLMssyxoMRp7tLNmSMVYVpRKSQC2O0zxNjw+PlJSLxaosMtYK1wkcxy/LGggBAeUMFEW1mm+++uqrum4tyyprBqCSUnLW1A03Hds0dEpxURQIy24v0AzKGHt39aEX9ka2HUcZF6DbdxzLbZpWSlkV5WKzZoxRShzP3WxW6/Vas0LHtbo9X9NAXZfrzXy9WTHW1G1TliWTAhIKCYYIcN4CgA6fnhRp1rA2dK0oy79+/W2v452enp2dna83Eab6k7PzTtg/OThYLDeCcaXEbsIPITRtK+x2mqJaJXNMiOt7nW736PDID8P1el1kRRzHUkrbdoFCZVmenZx+8cMvPlxf/d0v/4EAY7mYpslW1xCADCKua1pRJ4UqdoaFuq5ZK3Sq67rJOW+rOggCquHDw8PTk6M8T6+vr5fLeVI8NjVDeYkIpZQapml7FBOoGrWDJSjAGGtY2wIgqQaZbBFCACgpBBetUgJCiDGkFCOEOZeccyVhVdVAodpoM1j4fkjIVkrpOC7BmlJqNNpbrVaWZbVtu9lsbNtUSui6PhqNdNPbLUcQApRiIRjnLaXYMDQpedvWaRorJRBCnU6gaVqWpk3TwEJxzndGD13XHMf59NNP+/3+ZLJPKQUAvL+4/Ou//uvtdnu075dluVtR2bataVqv19N1nbiOv1pt2rZeb1ZFUXDenp2dVVUzm85fvnxJsKkU/sv/+X/6q7/6+Wg4+clPfprnMTEIYggqaCJzdvNoGEbTVJfv30+n0+VqbpqmrHmaxkKIjuN9/OLZ8fHJHzKWXOqI2H6XajhZbxtas6opBAdAIqkC2+14bpmL9Xw6re4ebm4N3S7LMk3W6xWdTEZlkRqUuLZhahQAuHswb/L1sD8e9seMyzKviizHGhj0u3E6w6ANPb3vT7abDW+UZ0KMYTHLHE3jLb9++1ay9uzpk8O9iYYJUBAoFHR7w8mEYihkQ6hWMVHX7OTkZDGdT6eLjuubnzlcys06EhzZE7/h2XYTG2ad1y0itNsbGJj1h/3ZbPb69Tdt24T9cDQajEYjCVTL2Wg0QmTz7u1Fy9inn35OKZ4+3u1A3KZteJ6FCazzvGHt0cFB1dTXlx9e//Z3bVV/8vHHjuMsluvZ5ZVt22mSX15cdgd9jejpJplOp57npZuke9IBRH+4ewyD7tMnTxDVbNvt9QYY0zzPbdvd7irgeZlnBee8bViel4RonucZBjWM9vbum6oUTVbIhlGIDEPleZkWeZezXr9zcDyBij99cvZ4e11EkanTpMjpemXbrusLhKnjuL3ByDTtQX+03sQCyIeHOwVR2WSAyrvZTZNv4zhGOrYNc7PZfPLp55j+D2Gnx7iM0gwv1rqlG1wbDHtciP6wJ8vy5Pj41SdP5ot7LppeL/BcK062Z0fH4/7gYDQZBL2OFZ4Mjzab6NvX32zXCQI4ikAY8rZm7958n2WpH7hJFve7naxMBqOebRlEBxVPERd9t1PxVkKQVcXD9DGvyrKooyQeDodcik0cOb7n+WFVFVEUrddrMyZCiCrNm6apqso2rfFg72B4eDA5+PDhStdNz/aQUPF6E63W71vZcRzf9wf9geoA0zQBABrSok2cZLHrWG7Hc1MbzWDdVggDWfGriwvfNnvdAJ4cx9E6T3MumWU5lNp5xqhOXN/aRvOySHv9wLYsSzeqqnp4eJA1MAaaYxpNWWRZ3Ok6na7/8HiZF5ugY/UHQZLmmqmCwGtZOV+uAFCM11fXF3sHe0fHk+nDzXz+kGeb/b29KjN4W+/v7w0Gvels1u93Xdc1DZtSure3LwSr6/by8tK2XcOwXNfdbrdJkpydPcEYf/LJJ3t7e7Zt39zc5Hl+eLRvWZZSyrbt3cq8LEsIVRAEAADP8wwqSVl7QWc4mhwdnViOF0d527J//MUvqqqCGPf7/adPz01NQwjmea4RhDGEUEElbdPSDdzWeV2WrmPpmOiEdgKfNVwI/vBwv1gsIGzqqiGYEogEQgjCpqqzJKWaAZRoilLD5OnpaZTkJtVdy3RyOukNsuWmH3Zcy765vtawphFKMNaIYZh26AR1y5M8e7i8VUrdP26QFKNeX7TMsx3LD5sy1SkN/YBivJwvoihCABqaRhA2NF0gyjlvygpC2CpRVbUECiF0fX0dBIGlG6P+4OXzF50gfLi/j+MYsUa0IovLRzjPo2yzXtdFfvXh9ujgtCl5XXDDsvfGR45jbaKtpmkYIo1QKaUSQgiBoDJNXUoJFbdt0zC0vEilgrZtU4qbqoEAV1Im6+16sdZ1s+t5huXoQuZ5WaVZnKcAgE4ntHRa8RrnacszhPXxXk/XYJJuAGSEIACo5ZiGYay325ZL0zAw0gDGg/5keHj0cHcvWD3sdVVdzR7uuWi73f7ZyWlVNhCT08Ojk0N8+vRZy4RS4Pdvfp/GCQAgCLyDo6PAsR+K2zhNN6tVJ/Br1iKqIQRWq81sNru5vhsMx+8vLi+vr03bOH/xTCd0bzz64x/9eLlZ3z8+cJZr1BC8lYBjAvMiyZvaMCxCCGNCcEUxFUJkWRZ6vm3bjDcQwm632+t1EEJB4P3mzYpzwdpGqpoQotdt2bQ7DaOumVTDGqRSCqA4EAAhBATDGCKEpQSMEaUwpVTTtB3ZTEqpJEIEYaRjTA3dZUxQonMG8rxCUC+KyjDawWAghYIAbaNNkiRScs5bjHG3FwJkllXORUsoohpumkZIRjWsgCB0t8zCUnHWCgAA480ObAUhBFACKHcUJkLQq1ev7u/v/+qv/pd37y52WKcdjL8o6Wg0chyn0+nsKB0QQgAhmd5Pb++uT06OFJNH+wfdblcpcXf3gBUiCK0WM8fuDLodywyiKP7lP/z96fHJql5G0abT6Ugp33z9bdu2jDV1XedZokNsINLmJSsq27YDy8mr9PHuJgzDuKj+l1//pWU5X375ZZqmDw93k/294bA7HPYsW4uTIMsSCJV7GgKF86zGSHOcgHPOeYuw2KxmirPpal6WZa8f+l7P0B1Ktdult5iv/t2//zsp1DaJm4ZhwZuy3B+Phch0JMbDXtfV8yR3TVDmKZWeSalO6Hz2+HhLjo+PO0GIEGpYreumbdvd7oCxBkJgaARAmq1SHduj4WG8LSDQOAcH+2dtpRbLbV3LPC+lQDVL87rVbY8r/GTydG88ztIizcs42SJKev0+1bWDgwMhZRAEAMH379/PlvPBcqbrOpAtodTQkKURA0MmmGoa0dSz25v3l5d/94+/vLm7O3/y7PNPPjZN+612gRoehuHXX3/9u9/9/tWrV8+ePaOIfnj3wdasy/dXH959sG3btexnz168fP4CAhwEnbpuGRMIESmgaVg7ocbDw3QwGJyePnVdd8dNe7iffvvt98t1DAQmAPqW01b17Pa2lSKus2DY8cPgj//4pwqwF0/O7iZ9xVrB2zdfXymCVtGWA0gNw0A0CDq93qDXF9s4XkfbD9cXB0dHDDT9vc4mivI6a0SdFEnYDb7+D998+aMf/+yf/un9/T01dAVAWVcIoYY3Qcev2/roeF8vm4OjseebXFRNmwHHdm27E/RCPxiEPcdwqrjQidlxe/mm4AU7Pz5ZbTZF9oc1+WI1v7i48EIbY+gFzni/HwSeFDyuYwtovuVyoBQGAsooTx8WU0q03rD/0tAAIo3gpu34QeB5Lsa4ZrZRlY5BTNMcj8eWZW232+n9dPGwqLP229989+7de84Fq1hd10pBpZSaSA/apWyAhZVSRbxmgld5hTEOw1DTdYNqoR8URbZY17KVBKAyKpq0xiEKnSD0Q02b7VbaSqn1enl19d73Q8c1Hx4Tta6fnp9BIKuiyLY5EtQ1HddwFOAKsF5vOBh1tHeQiXqvO/Q7NlNNw4u8iO/vHufLxatXH0swvr29z/P08PCQUNTthUqIKN5sWbtezNfruW0ZErgXH967ro0xHu/td7q9ljWff/HVp599uV4v1+v1b3/3+5ubm06n0x8OgiB48eKFUurt27dJkkAI7+/vd66aMPQppbvsJGPtLp8VhqFmINsJDMMwbDvNy02UbbZxFMWj0ShJksVi8fDwcPHu7fn56ZOzE9+17+9vbdvWCY23Gwyl7dmuZQWWRSQUbYOU7IedXbFi1jIhRBxFvBUIQ4KpoZuMsWgTXbTvjk/PVsvNxYdL1wlevfrY0ZPL69tvm7pqMw0hCpFvWoHjXsQXgkNKdd/xhIKeG+rUbMq8LZoqrZum0R0vS1KqoIaAbVpICKiApRmmph8/31/PF79cr5WQ/W6PEp3VjRE4BEOIFMZQB5RrHGGsGabneW3bZlk2mz70+/3Adx3jKWPs/vpKpxqlNE+zum5DNxSm29Q8iwvJQVNz1wlGfdf1fYItxtjj48z3fdtWCEDFmZCt55q6QaWCAEjPd3UNIYp8x+VKNk3j+KEAIM0LibAbdjgA6yimlCZ5KoEyHFtKKTGUCNq+N4LSpETXse2EgldpHBsmPpjsm6bJBK+qhgkVZxlGVClIqT4Y7XnDvsKItY2JgDC0E8PAQEVJlpeVEMqg9GBvQnTNNbUKMcOwXjx7fnV1RSB68uTJ4eHherl8f331/bu3nuOMLLtu2O3dg21aZdE2DcOYlmW1WHyvW+arj1+WefrLf7wtq6o76H70/Fkn8NIkWq23uq5DpGpeACJkJXcQvKpqgEK70UKaplmcrNdrQlFR5L/+VSolhxBKyQVXSkEI8I7EWNdtUTVKqd0iMvA8z3FN09SpprgAAPCi1jSCMeYcCsk4F1JiIYSum0pioBiCmq6bugYNw+r3B2XRMiYAgARrRVE1DTN0k2AaBMGut6WUSpLEMDTbMTnnQtUQAl3XhBCEYM53cAYgBCcEG4ah6xoAgP2hsAowxEoJXaeWZTDGNI1iApVSv/rVP75792693nLOHcfZZSNs2z48Odrf399sNmXTSoiSvNjRHglv+XYd/fEf/ZRSvDcZjcej77//XiNUcG5o9HD/wDTdqmxNww+DgBLt4vu3O4vlFz/4DAAwe3isqiIIgiBwDQ394f1VYtALe71OEATLFLUtA0D2+91+vz+fL+fzpW3bZdk+3M/qupaKHx8fhGG4m6K0batrVhAEEBDLcqSUUbQRQkyOJn7X+5u/+fl0Of/88x/0uuMkLlopDo7P76ery4tLAJDlmAAiBaGu60eH+4vFNZANhkC0TVPnlmbIthoMTkf9nuU6WZkLqRCEGiau67ZbZlILYCAFAhw5nu37vhRQZTCOioPJSR5XabS9urw/Oz5BWIOIzhcbzqUXhK4fPn/x8d7B/vdv3l5e31JDx5oeBEHVlhAjznmSppPTUyAlaxvDNsb74+nj/M2b7/b29ohGdR1SAhWr4/UiSaPbm5uqaebLVcOYBtS4Nwg8P4+TOE6bspICQkAw0gQHTcVkqzzL3x8daJhst9uvv/56MBj8i3/xL54+eWFa7mq1IhpfLdeaZrhOoBR0Xd91/cVicX//2OuOTk/OdV1fLtdF3rx79/4Xv/j3euA+PTsf9vo8r642l1mWCSURQnGSMCyCTgiRwJT0hwOd4Pvrq9H+hDFx9/g2LaqXz1+1Lc+ywg84Y9ww9DRPHd/BOmxAdTX9wEV7/OzzL89/dHxwmESpE/hv3198+aOvLt5f2lJhSiDBZV1ttxtdJwpSz7NKZK+WU8+2Bt1Oc3CQR6mukT//2Z9QpAMlWVWbtkcQrrICCfXFp1+0sHn79q1lEA6F71rPnp+XTfnp5588ff4UELlczhlrW1YrLPK2XGdraaBaNoyrVjEndAO/M5ns121TFnW330MIeUEHIVRVtevaGEPLxpxz3rbYcpuSXVxcrRZLwzAp0Yu0oJSWZZnnpWs7uq7fXF3/5j++7na7p6cnmKIPlxer7cp2bdPUmWh/+ctfUkM/Oj6wTTd0mFIKImTgoE7Y4nbNIWtyJTgu8root4buOr7HAcdEffnlZ6apZrP7XtfmbeU6umpFXeWX7y9c00FAWRpeR1vf9/f2J0mSLFZrBYjl+IblSCkdx9GTuCxLxtguPz+dTvv9/o9//ONos6nK/ORgf/74UJalbRlFQ5bLZZJEo9Go3+/7fnh/f08IOT050zWjE/Z//ZtfVVXT6XSqqoIQdvvD9Xr97t0727Z93//m299jjPM813W6yzY6jiME3Gw2u9S9H+x5ns8Y+/D+arFaE6LZlosxEXXNGIuibZZlSrRZsgm7Xds0Dg4mlmlKyTlnEEgoOGiYrJplMhNCtEXV7XaDICBEk4IhjCaTA00ziiIzdcvznbIskzStixJKuNlEt5fX3W7v5OCkyvNktVJNg3RICCmTjDF2dHTEOLi6vLm8ueUScC6UAoJJBLBvh03TGMSyAh9AWWZ53pSyLBEX+XatCXk/HJmUAiEpJrZuaLouABQtowRbpuHajmlZXEoIFaIa1bS6rrFuEEKSaDt9eCyznBDiOa5BqJSSIEwxhgpYtqVR2g173W6/KIrp4zzPS1xhzlSyTcum9Fw0GU16YUcKhaCSrHWD0FGqaVndNq5t+oGzK8ixsqSIzqarvb091+toWmuYFsFaLTmrGs4kNXRP13hTC8mrphaSEY1qGiEU+77L226ZpzuzK0RGkWZCKEqxZZlYI3XDMNYc18K6Zno2zEGWJBSos9PTUa9XZSkGyLIcTdMs29ZMI47TDxfv4iy1XF8jqNPpKAhmi/lsNkuKkgNlOM7k6JA1bRrHTEjDsEK3yyqxiMruoI8Qmj48rtazhjVSyvVqcbi///z86bfffnv54XpvfwIATLLCNE0XaJqmFUURRRFQiCBCCEvTNN5si6I4f3La7Xaur68Xi6Vpak3TOj0CIUSUUF2DECFEAFBSyvU6klKutHWv0w+CQKcaBhBC2O0OdxJtIURZ5m3bQoj/YHbgAKhKcCA4YK2EQDS1UBJNHxcQ0PF4sl5vHdsdDEZlWWqakSQJQigMwySJfN/3A7dt26xMNE3zA7ssS4SlbmCTUcYYhAohpQBrWQkhJBTohkEpTaOCUrLzkFGKNY0QohGKyiqP41hKbpp627a7PV2/3xsORpRoy8Uqy7LdHXIwGBBCyPHx6Xa71XWTsebu9qGqquVyaVlWr9eLoxhjPc9LoKjgsNPpKQW30fr1N18jhJ4+O8vz3HVtwyQvXz7jvF2uFlGUc966nn14NOp2O2VZtqz84Vc/LPImz6of/ehH33779vLDtWFYUoDb2/urqw+Xl97nX3x0eLSXFxkhqNcfatTASNtu0zZJdV2XEJm20xvuma529XBXMJYW9Xz1Li/qwO/AsmmUwqYpJZCISqkMyx6MxxBRiDSqI0p1waSluy+fvTBNW1RdCLDl2J7nrKIoTWNCqem4g97w+dMXQgiq4QaVvhV0nQBzrLVGmqajwQFv1PT+IY4icYRZqzy3E3YpgrhuWc2VZlmmZa+20dub91ESB6EnAWxq1rRMKJllWZEmhKDtdptlmR96m+12E611U+OsCsMQSIkAKPMk3qznDw9SgslwsH90ZLlhCwAHIMrym4dHxlhvOJJSWpYzHu7lafH6d68n473jyVGRZhQQR7eH/dGTs6e27SZJNl0sEaw6nU63q0kJGBNAYUO3XMfnXKZprhRcLNavX79u2/b9+w9p2k5GOAx9SvE6jynFH3/6yYtXz83QIbpWVmkcbTWKgFSakq1qV+sYm+7d9OZuOvNd/7CsonWcGHndtG3baqYRx9tnnzyrRW0FRlZXZ0/3D072f/iTHyCA//oX//7o7Pj199/983/6X5w9e75er03HdTx7upjePz5UTdm01XT5mGSr0Pcxl6JqTEyNIHRNCwHF28ZzLcPQmqZCAJq2HnRD27bTZpFnG4jb99dXcbLqjnpPnp7+s//iz3VbF4q7oVdWRV2XWZat18v4oRBd8gelvFKTo8NerwcAqvMMa7jn9ziXuk7rqk3TZHet32w2URQVRdXrDaJtMpsuLcsZDifLxcL1u7quW2bj2DUmMEmS+/v7Ue9MYGj67ngywo4+iDeGZRCCDFPbbDZKyOfPPnr+5HmZF1VRJknC66VuGXkZCQB8Z3B2jCEhZd3mRXM0mACplsvZp588+/iT592OMRwG0SznLXu8W5QpAwx4TlhnBQTy/MnJ2cnk/Oz5xcXFuw8fVuv8p3/8pweHZ0JBKZDj5NE2u3u4z7JcQVLXNW/avb29brcn/eDF848DN1itFr7rff32EhJMdR0RrdMb7B8czRabLMuipNAMpzcczVdLRMjTF88VglEaBZ3eDl2+6xEwxgzDOD4+DkN/PB67rgshzPM0TdNut9vpdI5PT5uazWazumWO4wV+BwCQJOn9/b1lWRSTfr/bH3R1XdcpFoKbpqnrVCnMW0UhwkoZlPiOTbFK07TOs4csXZmWbpplWVLdwJAYhjEYjHzfV0osF+uyKDqdznw6syzrcG9sGk6dZ9Fq2ZalFgQtbxvG0yz+5ptvLNsRCgsEECYNF0VRaUYtFQaKWJaDkYYUAISYGm3KahUnhIuO4wCltptNuolu0HWaJP2w47heVhaiZRjjNIqFELpGNUIgkIjYRNcwxju7t65ppmFDKZqmEhwLywjCcPrwyKrcNAwgZJpEtmkZGjE0EnieYGyTZQoC27aZ4BhgBJDrurZtm1mOEKAE9TqegvDhYdrURdPYDnWEknlalGWJEBICKInqul4t1oZmP//slRDqw8X7PM0UB/+51IcAlJxJwdpGNXULA9zp9Iosi+Pt4+NjFEUSCCWhYZKgFwBMsrzUNM2ytKQsGWOapknDElWFEXVsnyi0Wi7alidpGih5POjYgSOw2G63RSn29vY0Tbu5ufnbh7/dxbzC/iDOi6yqsQRCgKbmSgAICQKEECyE2GxWcbLVNM2yDKpT1rR3l7enT5+IRtZFQ5EpIJKipprdsSjncrcsUxLuMgpVVTHGVqsk7GyGw0G3262qQilV1y3nHEIMJBcSUAqpjolmEJ0qoW02m1VUpduZbacUEwwwxvj4yVjXgWkiQrBGfEMDmqZRqq9Wq7ZlRV4XRcOZLMsaIRxtU4TIcrnu9XqB361KrmnGrs340Ucf9XoDSjHViGVpQehRSqNoC4DUdQqAIQTbpR9s2xRCq+uaEKJpGiFkV81QSnHeapompWSMKSWVkgghTSOGoe3t7c3ncwhht9PfFVuapnFdu2GtqoDl2A1rJVCmaSgIGtYSpMCgO5jeP3Y6/jZar5dzAJRt2x/eXcxmix9++ZNBf1jX/Ob63jZsCKHfCQzbHAx6k8PJd9998+zVkzxPX3z0/P2H78WqaUQpJGs4LpsMJGyxWDAoX7x6Xlf8/cWVrrlVzaM4T9O0qiqE6WDUdz1DAriJtuvNwrb1sLsvJFQArqKkKplpmkWVB6HtbzY4VkgzBpOD2Wzx7u2VaXlFq65n3ymAECJFVVVCYKxEKRabWMI6TYswNBHWMNFdxz47fdHp9K6u4unjDJnEcCyYpsv1armNwrB7sH+0v3cApEySCDTSRJqmqKe7dGBauouxfrB/6tn+7e01a6VQqK7roedLAKOsnC9WUfpXQad3fX1r1tpqtR6NRicnZ0Lxfr97fHwqIb+9vXVdu6yrqikhhMNRXzM1JhmUAkgGlIIQ6Rh4ltkPA4zp4dHR+OAIElpwjk3LdPJtnMRx6vnh9OFRCOV5QR4nSZTuDyc9vysbARXa3zv46MVHnU5PCIWprlEjTaJBX5s+zjnneVYqBTmfa5qmFJjPF1VVtw2/u31IkiSO0r1x/+NPXlATrrezpimCjuuHTl7l0+1idLgHMZnNVp5naUTnTZFHESWW7jnXd7dRlnpu+DCdLh+Xw+F4uVxnWfJf/u//d5PDyXT6aIZG0HE+++HT82fnHvHvHu8RxNt02yl6ddve3t29+vijf/tv/y2hFCD09u3b+8fbTt9HCEbRpuu4nz7/tNPp/M38r+uq+fSjT8fDvTROLMMyTM20tOVijRCggauxEmrg5GS8Wo/jZIEhj+INsdAXgy90gz5Op4iirMijKGpYk+c5V0jX9ZrXmqZlSaYE6PS6VNeSOGt5qxQwbYeJFguioGw529Xn0iKzHNvx/L3xoe8XdSMtw3Ud7/MffKUTPS/SsiwpxmEYhh0fQriNWVWVh8dHp2eHQnEIQdNUt7e3SZJUecFqNuiMeQOKqBEMEmmZ2l5dVdPrm2U2r3nRIq5bZlyUmqY3TVPX9f3d9Xdvwr29judqCLGmzvNsffXhA1KGa3Z8O4zjOE2SMPR7YUdBzXK6vpu6Xu9g8hRh9+LdN5t1ZLuO2/Fm89XD/YeHx7VpmlApw3A+/fgjQ6cAwCTOBIcAICZa3TB830cEN4ynWf44nZVlOd6beEHYtjWhuut7QsnAsxFyHx4ePM979erVxcUFhPDTTz8VQgyHwxcvntm2nWXZYrHYAV6Ojo50XWdcRklcVHWn2/d9H0J8d3f/8PDg+75h6JvNuiyK2eM0jaPJ/pgQ0iihEQqkkJyZBgl9jwA47HYsaz/NEsb5Jo65EBohjFJC0P3VfacT4n5f2UJKWZel4iL0/NALTdN0TEdyBTijEHiWqWPUSmA55vHp6e39w89/8TcKUS4RoZrCqBGi5RK0TMPIJLqULeNq6HqUIJvqJgKDMDAxoUq4pkEJKfOiLivesjzNWtYCiKu6bFkLBNd1ijEWXGJKKKFcyZcvnj08THchDKsbdMJwMpm8fPa8qVIlWVvVGiG8YYIJxpuiyFarBWMtYw1vawEUIUjTNM3QTZNgiNI42W43RZYyKQiCTHDGK942TV1KKTnnWNN3rbzjg2NKaZmVkisMyWgwpgjP7x/vywYCqZSAUOkUUQ0jTQOSt5XK81qPMs+1u51+29ZSckoxIlqeZ5Zu9AchV5JokFKKKecFRwoYmg403rYCYxpFydtvv9lu191O2LA6aXJiaYNR//D84IScyJK6vte2bV3X17e3i9XKsqxuGPq+3zDGqlZyKdqGNS1WwDCsdDZtmkYw3u/3PdtJ07iIMyXUZhlhdL9ZJLxBRc4VJhBaSuimqwuhXNf1fb+p//+USwBYFplOpwhBxzYNw0jT1HVNDqrdkl4pyLlEGBCKCbbCwGtq0BYxa1RUN7wtoVSU6ov1mhBiGLqu66ap245p26ZhGGVZti2P47QqGyFUVdUYUV0zpcBAQQSxphmeF0gJNpsoz0sA0MHBQVUVTVsPBgMA5WazxhgPBn0hRFWVGCNCCOfcMPSdNR5CuMM87E4GTdMwxjw/gBAapmaZDsbYNE1dN1zXdRzLsqyd4ANj3Ol0dvTGKEocxzk8PLYsZ7vdIkTyvGSMEdt2O53O777+zcuXL/I8V0q8fPkiCIKyrNI039vbOzo8TZLi4t1ly2opQFnmQjRB4JkWjZP10fFemjHGK0qxbmhB4CklGG8222UUo+Vy7g26l9eXw8HecDyaPq6jJO72ep1ubzab+YFzcDBOsvXjbMofSs6bZ8/PFqttt9sf9Hthp6dpjYQg36wb2Vgz+83b1+vNcjTZ4wBLhCGmd9NpI6UXhEChpCg1jTqOVVbRzf2Dpk+SNMcEZG6TFY1kdLHaxlGx3LLpcmaH/nh/LBFuW3Z1c4UxTeJsNBhohKabpCpKAxET6QhCCJHjuNE27nW749FECBFvIyklAChOs22caprhh+HV5c2btx900+jBQAixtz/xfdfxbNOkw/F4Gy0BABAj3/d33dFuv2+77uPjo2nqVMO8biFQBCJdI7ZhYkKSKIKYrON0naSD/QMrDE3TdBzrP/2n/zR7nJZ53u/2KERBELi2k6WpaRhFntu2PRwOCSFCyclgACG8vys7nc7FxcVyuSaEAACiKBqNxlmWNU3T7w0PD0b39/fv3r1XCvp+KACfzR+qtOwFoaNby9V8td0gQ5vOZ27HT9PUMDQmxGqxXU4fv/ziB+OT/VZw07bcwF+tNtP5LAy70+n06urqv/pv/usf/OAH/9f/2//lxz/7ShH1yRcfcclZw/7df/jF0dHJ0xfPP7y9PNw/+d3r3//wix9mWcZYwxW/+PA+y+LhXjfoBoSin/zgy5OTE9a0OqKGox3s7x/tH3e7fQJx27KiLF3PdlyvzePFdnlwfh4Y/aPjyfurt8cn+7qreZ3w+YtzQhChiAnBGGOC67qhIGqaZqeuCFyvqqrtNi7ryqhrCUSn112vNxhDAACl1DCMoiiEUFJKomndfg9BzXJsxtVgOB70RqZpl2VpWZ6hnIaJzWb17v2FYWie5335039aVZXp2yVvy6awHVNquBasapvBcGxphkXNeJPlaXmwd9DvDnQMNvE6TrIoTwveQA0hRIHCpu3mee5Y5mg0AFDxpnFdqlNUN4Wm651uaFDPIt5oNKKIctYAgKbzBeccAHx2/nKydzwcH81ni9ubB8/zPv/sS8fxojh///763fsPTdMgBaqy7nd7x4cHSVxkWYkxLPLadd2iKAghTdNcXV29/f7dz3/+N6ZpUqp/+eUXEOHnz5/nRZqkWyllnqfffPP9Z5999vLly6urK8MwvvjBZ69fv26aJooiKeWuQ++6rmEYQRAsl8v315dCKE3TDN16fJxNp9PZbF7mBed8NBpatoGJqusyjreCt4TgZRobmmbqukahZ5qsqiyDQKjmUSwkp4aOAUCa5jgW1XVE8PPn7nK5vLi4yLLx4eF+GIZFUSAAP/nk0/l8fnN1m6XF/j70XY8grGkGpma/398/PPa+e/Pu/fvH+Xq9SXTT8f2ughBQjDClum6YZlU2SrLDw8M0iaiNTg8n/cDfzhf5dj0ejpqmgVIWaXZ9dwsg6vV6fthZp6k16JmmiTXKpagaoZTkUpR1U5alUuLw8LDf6W5W69vbWwzUqD8Y9j3fsZdlsVpvRd1ahqkYu76+Ojs90XWdUsI54W1blWVd17Si4/EJhGiz2cwep6vVCkIYJ1FeFIRgw9QIIW1TtUwEhrWTCsZxTCG1dGtvqEvGLy8uKSarxVIJqWsEIcJ4w9pWCogx1KnGG16V7VrGBGHTtMKwSzE6Op5sok2eZ1w0QrZNy7loMAF5nhDSwRBJxqPNhgDkur5o6m+++64uy06n43heksXfvvkGXWI38H3fC7WhfHgYjkc//qM/Onvy5Ldf/y7P8/FwzzbMuqqm23teN21e5mnmWTYE+OzsjLEmzzIhBMbYdX3H8RzH04g+my6jbaokjrY50nTDtIXEO4liv98HAOVZCRXk/A8ey8lkMl9MF4uFd36KMc5zNpm487gyDK6bBqEUY0qprmmWRs2iqIDSTMNDAEsJa9gAITXNWEePUrYAlggBXQe2Y1qWqevUdV0hRNtwCLFhUEo1Q7cGg4HgyLIcXdfbtiVE0zStrtodNW5XUYni7f7+qCizxWJxdHQ0HA539CSEEKV0pwMNgmA0GpVlGcfxLn7oOE6320UIsUbtIA2e50XbYNdxCIIgjuN+vxuG3aIoZrNZ01YAgLJUQnEuRLfXK6vq7v4+DENd1w3DILZlPDk/ty2jrivOGoyha/tNxQSTQMLZ42y7jiGgGqH9bg9CuFqsu17vq89/pGMTA0qI3uuP3l18yKu8aFugaQow3dI3WbzerPr9/rZSb2+nSasODiZcrz7Mv4YQ7u3tDQwtCJybxw8Pt1NCNCWRYdhx5FHoKi9scpysCsZaSkC53QKkFpDMr9eW5//ki3+yjaOi+KvVdrNN4sZ1t5tFN/Q7Bz5rCqEJi6Dru/calcNuL/D3JTMxJuso/dXvHp4+efJhkY6PX0QN1i0XOO2Ht983GG82K2no96s5hrjMcowQMHXquE1dayAkGBMEHu+3h/ujveFxus4/fvn573//O8DgsDNI81wJcLB3sFpvbdv94uOnJycnTZs0Lfr441dlWUMBPnn+RVVVWAJHNzsH/sPddRj6aRZbqp+Wq+X8VgHU7Qz0sGNgrdYi3fSoZnEFHm+SxaoBRutxhrh9dvAqeVRvfvlGCmb3DN5Wp+d7FLC9wfDh7v7Z0Z5DQLFY8L39sm7OuntJA6hSVCkDoXS76vf7GOOWgDhaPs6mL168sH0tr6Mf/ckPXr/9bTi0HceRSXG/Xruuax/uE4rrbc3qbNQZjrruajVfXF8OHOP3/3D51R99dXi0Z9v21d3jD3/yRx/eXSKsPXl6dPPh1rG940P929ff/Hf/6l//n/7P/83QCy9ef/eTP/2xofAmjovNCrFy4NuRyFiZnh5OVrP52zffd8PuN6+/6wTBdrrt93su6H15/sN4G+nI2szTb7/9btQ7HgxGk97Z3uCoqdlqEw+Hw7rgQegBpNVFHLgeqGpguZPx+Zc/5P/3/+f/A2nuD3/6TzalANssZ7BuRdkoJWlTSpOYrulACAMS8Iwf9E4OuihOk1Wx9TuhgR2dNIJjLqBu2Ov1+nGxOjg4MABohFhvCoSatxfXcZSWZfn+/funT59blv345uF3v/29bbuB59/fP0IIbdte5jgIgh/90Ik2manRZJZKwXABQMotjxbLGNmy6ziyqLM4qos8AWmWZffJPG4rSWmSbh/mD7Zn1YXSMGia6tnz027o52nWCKgRHpccATg4OOv6I8fsQIUnfg8q2dYFQixbzQfdzmgwNiiK1x/S9cpG+sDtWIgQoIo0cRwDIi5h2ypZwfpqcUt8XdM0EDqbOG7TYp1mRweHmmZ0/PDm6vbN778zdWf6MP1Oe4Ml+ov/7T/HGN3cXRojsyyLsmL9jnd58caz9R99+dnbt29vPrzXEJxOpxSCxN5SSncRh5OTk4eb++12u40S27YRA7P5ajabxXFsWvr+pLe3N2a8OT876Ha7aZq+efPd3d0txrjnGc8Oj5++eLrYrlveGK65Wq+RhlNVaDqxbUUp2RuN8zRBiDd17TpD3zu5v9Pqsr65nFZV49quqJ1oKfeHr5a96ubD32GwPj0+tHQDIVQxHK+Ko5Pjf/bHf/7k9GmUJm/fvf/L//mvbIOOB+Fnn74yDHu1jKCEYc9jrXAD1/XM//jvfhFa5pVSk0F3uXoUbea5ruu623RdVHl/vJc11UG/e7ecY9589OJFWdeP07nSDAUR1Q3YZICjuq63d6s/+uiHXWzLqGRle/X1N9X+sM1ryJRsmE41AECcxZiorIhtd3BysjedTl2h7YxWxweHQW+IICmLZjZdthXHlNxd3RuGZdgWBTJdx03bUkqpr7qOy5jQtJpgAABjslmv6+n0g5RAQeA4jkKiUZyjViiGIEQAQagwFVGypjQEMMCE9ru9Mk8fbqebzappGsHEQ33fCqZpGrBtAEAhxGg0IoZB98Zt26ZNSikN90erxSzjDYpr3wvj9RrXKlmvp9lN1AjLss5PTj/6qA2C4POnnyCEgiAoiuLdh/dffPWj2Wx2eX11M7tXeXR2djbxOsvlEpvcs+1NtAYmevXRR5phnpw9yYryH99/bwOFNC0IO3leMyauL5cfffTR3umZRm/u7u6EEEQTUnKlbIiA3wnaqtxst3t7++tNFCfFeBy0bcslAAojZCBiGLrtuGHLIFB5XuWSSUp23haIcOVQulNgIIQEUBBiHesUaSYJmWJWZ2RbrpSyrmvbtvvDgYKgbVveMoR1qhEFpFTcsqw0jTt+gAHuhV3RgiJqxt1DHTrFtjaRdbZ/jjFu2zbPc8YbWYsqKTlrNIADy0UIYozBHwxRfH9/VJblZrPxA1tKwDk3TBwEAcJKqdZ1LaX6RVFgjBFCitPHx8fvfv8/cs51XS/Tmvo6NXWilIqiaDemaGomFX98nOm63jQMKHR3d1dX3PMCKVVZ5re3903VdEJ/sj++uHgbx/HN5dXkcBJ4YVVVGtHLKl+uNgII09Q7Xr8fjpDt2bZDqZamea83ePHi46+//rqu+YsXHysJsrS27UxwkJVFWbRze5lG5Taau5Z1c3tVVYVl6qvVQgJJKMQYTmcP//pf/6tNFM/Xq7DXc10XaZRz+AdglkIa1Q2d9MUwybLxeE+zrDjL3t9dl2kJqdbNBhqhdVnFacIYm87nd7f3VNd4y9I0XVerpqoVF4N+HyFkWZZGaZ4IhZRuapj4ENOmqt3APzk8aqr6t7/9bZJmvd5gfzIpi9q13JOTkx98+nK9Xn///bvhcPjFF1/ujcL1ej2dzkeDvpCsLGrbpBrVbdsVQtR1nRSwKArTcAFAUkoh2sViSbT8h1/90Wq98TxP0wyE0HT2AAAYjUaTyfBnf/pH3/z+6/VmYekaF/XB4alrmXXVwRgRgltWvbv4jmpGv98Vsh0Nh0pKx7Z/8tWPEMa3t7eyYd+9+yYMw/Fw5Fn2dD6r6/pwb2IYRtu2Uby0LGcymZyfnEoIdrPiJMlev37tOE6/3xdCPT7Ofv0Pv+50OoPB4DHeDvvDumbZNs3q8sd/8tMfffXjLMv+6q//6vb+bh1HP/7jP3nz4ZvZYtUfdhAmv/v6aynBn/2Tf6bp9otPPhrujSYHB8vlGlNS1G29Woa+s3e4NxgPiE4a2UKIkyStqmo4HD579nLQHwqu0jTffRV3KSHVNEopy7IAIbN3H8anJ0/OFSFktVnVdRvl+W9++1vdNDzP03UqhSAIU0OzLRMD6OsBY0xBwDknCjMu8ygr4jzNM8BFmqZtWV9eXn64ulSs6XQ62/VsOBwfHBwEnv37r1+vl1OlYL/XS+MkiuLVcpkbBQLYsqxut9ftdh2/izHcrjdNVYxHfQRVlWd5mq1WCwzBZrV1bfv46FRAcHd/c/n+fYUqw9QklI7jGI7e7QdRusqrTNO0+/tb3aCK87+bTg/293/yo69ms1kaJ6tVhKH19FSOB1QwJVphaPSLzz5OkmXblLqu67puGZbrulIAySkiUEIJgOz1Ov1B9/nzp1TXwl437HRGoxHGsCyyqi4ohqZrs5pJxiGSHdc3z59SCZFCB+NRmqZJvF1N525g87qpWc0E81w3Xix2KUXXdeM4/v777weDwbNnz/r9/mKxiKJICEEpjeM4juOmaTSNACAxhmEY7m57VVs1TYMx1g1nf39/tDdeLhf3j/dJluq6/vTZ6Xhvr9MdcAi3SSy4NAwr6Ab7k4O7+9um5qEXdrtDVrNaVLaj70+OKaWO619f36RRwTmP08S0nPV2lWVJt9v9i7/4iyxLkzQqyxwIOY1y07G7/U7X7AElbNt8+epp1eT309lo2Dl/OnFsP/SN7Ta9u7mbL+aP9w4CsMozURa2hjUoG9Y2jF+8f//q5cvzJ8/Onj4DiBRlJQRTSu1a6VlRIIR8z7EdTzctx0mLoqhLbGikbbI03+gG8Fyn3++H/X5ZV1mRY0qIrgGpmOB5XUVpEnSDvK4WyyUhxHbsycH++HA/zsodQhEhpOt6w9qqqiilmkYsK6BUb9pWKRUEga7rTZPe3T7ouqFpmlKwbXgrpBRKKUWJBiEEUEkIMKa76ZpUwtB1Smme53kaD3qnvutdfUjfvHmzC6tSg+qmsQv97BRGQsiyrHVdR4goxeqqNQ17PJ4ErmfqxnI6Y1UdbbYGoQRhKVVTFgSC29vr9XrZ6/X29/cHg0Gep2mR+74bhF6SRk1TbVdLSuns4T6nG8938jxPslgzDKDkdrsdjvcIQUVRACDD0IdI44pDoohOHN23fJdamoKQS6mgJAghgib7o6ZpDA1J33RtZ9APez23LEuCMNJ0TDSqW0KBpmlLkGNMJReWofe7XcH4LoW/y5n5vr/LClBKAUZKwd3vPKWlruumaTqOs+MTUEoRwQ/TRyklAnAXFCjL3DLNMAzDMBwNhoZhCMkghLxldV1zzsumZIwppRACOxAn0TRNJ1jDAhAgmNyFESRACOwayHme53lumqbjeLsKUrfbZ4xxzqUAAqldP0LXdU3T401BCLFte2cwSdNUSimEIHe3D1y0hJDZbH57e2vblpJwNBrFUToYDCHEabKMokjXDSHZYjFbLzemaZq6tlosJ+MRhqit2rvHO8dxyqzOyiZPG6G4qbmD/ujs9KnQ0dvvLz7//PMw6Are/unP/nw82r++vq0rpusmgnpdMc4lwRpXcrvd4p6satjr2HuTXhQhCKRuoNVme3V1EXQ7AMjH6X3TctM0bduiVIO82YWDCIKQEIQIIZrrBZvNJi2r+Wodb5N1kiopV3lycXd30J8ozgiCEiKCkKlToum2aZmmqWHqu55O9NAPdjRJxqXuOUpIgmib8el8sZw+sqY+nBw+e/pScfj4+KiUGgZ9/8hPsty2bYyplKBt2zwtos0WdSHGGEhl27ZUvMyypmG7Qg5rWigRb/hysTbNKk0KKdBk/8Q2rSDs39/e3Nw+JFmOCGFCMsZ0y4yi6OzJ4SefPfv3+/1vX7+2Db1p6s1m9eFiNewPlBItKzebTVFm/4d/+S9d172+ubBcY7GNm7zc258ghMBkf9wfLBeLF89f7A9G2TaO5qs8z8edvmEYSZJ0O65pWycnJ5PJeD6fK66EUJKLt2/enT05992wzMvtNsrz4ujoqKra7x9uHrxZW7N+0JtO56Lm3324+OyTTz798our26tv377rj8f6w7WQUNPdpszGh/tSgr2jg9UyGu2xydGB7QWX729YKwRU283GMDSv40MNAQq7g7BtRJrkSuLhcNIJu2VRK9VAiAnROJcIISllWhSMCaLrQKnx0VGyWgsh/6v/43+9SqOiqdbr7bPT55t4gyRKNlmexoHnT7qjUX8IIezhbl3XTdPksuR19PgwjaINl2K3+atZNRoNHA0HhiaKrEYg2sybKqWYE6Ltjbq8rdarbRytfvTVT+u67YY9x/F63QHnglJ9d44hFFGg5uuV4m2WbB4e79q6KYqcSb6aL+fzuW3/ihK9LIrFYhF2LNt1Dg7GnUEnb/JNlLVt21a13g9M0wSSb7fRdtPuTxSEOI7ToihEy7zA9BxXSrndRq7lTiaTbrdb1wmlFCEEEFQIcik1TUvySCQi6PqYEkSB59kvXz3r9LoQ493iFkIYK06Q4lhRDb148jSKomS7uZUAA0iACnz7aG+4Wq2yLP3w7k3Q8YsiS/I0yzLNoJ0w3PkFOp1Ov983DGN/f//8/Pzq6mpHVoAQapq2OzEMh8MdjkIpFYbheG/oenZRVQBKomOlVFEV8+X88vLy9v6OUnr+9My03DgpqvdXaZ7UTeP4ju/3xv0xkwzIKWdCSU0pygSJ0ibL0qxCnbDX7XQsxxEK+r3OerEuqnw2fyzLcjAYdILQMIxO56xpqzdv3hCMlWAYIgRAnkYciKAbnJ/vf/bFC0JIt6Nl2RqifG9i+d7+yWmHYAcB+OT8pC2L0HWoAho6sU3j6Ojoi88+39ufcM5//duvhZKL5ZKzZrPZOJ5bFEVepHXLkyTp9noEwo5ntSaajEe2i00LHJ8OCSGB57aAcKAEBNQ0DNMsiiKrSwkEJJiaOtU1w7UHg8He3jgMQ4jwYrHYhdgJIYPBoGpqjHG/3y/qyjRNy7KyIi/LkvGmbkBepFBhBDDFBtU0y4ScCyWhhEDTNM45l4wSSikWQLVt3bbCNrFrW3mep2maRFsCFABAIzTw/KqpKaWIkrIu2oZplGFKCNUFUwIJXdchRJwrirV+b1gadlsUaZoXEuRxxk3DMk2KcMfzia4BpbI0rapqu91CCKu20XW9LEvN0E3TPNjf73W7uq7f39/vD0ZHJ4e//e1vP1xdOTpJ0vTr17/37x8Mx54ulgABzaAtFwAAquO6zaFtAwoFAhJJpCFKKFQSN0rTEYBQCQ0jHQJBiJwM+01T1ZLXdQsV0ghpmJJty0CLhFKsBVIpzuuqZHVDEA48x/d9x/YxxkqppmnSIq+qmnMOIHx8fPQ8z5cCQWKapmEYnPMoinYFSEwIhLAsy/l8blsWpdTznN3td/eV4ZwLIRhjeZK2bcs5RwhRDWuaZlkWhKRpas65lByAHYsaYowhhE3DyrLeUdWVUhhjAFBVVYSQtvmDlrosq6ZphJCUNnleAQBc1zVNsyzLuq53xz6CEMrjkjH24f1N3ZRHRycA4Pl8JQQgRIu2SRRFO76EUlIqJgTrdMaMNZvNand4XK/Xt7f3p6enq+WWK25bjoKAEoMSwzQc7FEI4Xq9lRJURbm/vx/4AyUf/v7vfnV29qQqWZ6XddX2egPTpIwx2yGmDTo9e3LQSbMOVODwaHj3cC+E8MJAINWwOgjNivM4jnVdt7s+AIogLKUESkgppYCaYbuukArXXCFNN12vLMukrHC83Q8HimPHMCtQawQbmk516rlBEIS2bXfDnkE13grbtBAilILpcu06TjjoW5Y1u7tfrDbX7y/mD7N/8qd/dn5yPhnuzedzz3Z6QagYL7NsOV84lnt6bFVVdXNzF21ix3EopbdXt65nZ1miUbRarNerFWNtXZdZmWlYGw9GedlmWV7mxWI6u715DDrdi6vrJE65VEyI0d44gOri/k6J+osvvvjo4xdJvOz6YZ7GaRqXZb5YiCRJdp88QtBo3IvjuCij0fhJmeeMsXffvyWE9Hq90A9+8uVXe3t7bVE9Pj66rkttFyFUFAUWyjBMUzeaqr54+/7du3dvvn8LIXRMZxtFSZR6nlcWrWeHSikC9c0yNnQnTfNhf3T29GlZNP/pu78vqvro7MTrdU41xBTo9IajvQPLsY6ODou6+OjsaZYVYae32eZpWRmW9cnx2XB88Hj32Ol17+9vN+sl522SxUwwvxsggU3D64RS16yrq7ur91f93vCjjz6SErQt1zTCmMjzctdybsuCNVJKBAHt+L3+YI9a2s3dbSO4rNlytbq+vEyT+Pz8nDzRdGTUdb3Jk7ZtEYY6NgxsVmnZFHWv1yuKTEKsIeSbds/zqFK6rtmWrQG+fLjRoXxy/uzzVy8+ff787mFeVe3p4YHg8HDvSNdtSrQ8L7ebOIuL6eOtYWhUI4vF7Pz8NEm2d3e3RZEZhpGXWSOb+WaZ39+aphmG3dHJvksBY0w3qOs623i5mM+Fak3TjDfxeDiMoqitqxfPD0zN/Obr13d3d23BJAcYIs55keW8abVA241q27a1Hcd1bdu2MaJMcIVgXuUQwrzMojRREJiOabuOG/g7T6Dv+wQijSAC1M4ZEZhetNpsl6urtxempju23RYZHQ2haB5uLu+u3/VHw4ODfcd1WFUqIbfbbVVV0+kUY/z06dNer8cYu7u7++6775bLpWEYCKE8zwEAg8Hg6dOnnues1+uiKBDBEMKdIB5TYprG/eNDlER+xy+bUkBZV/nDfCa6uwuQ4lJYluU4QTfssRbc3N5LBvvhyA183Qw8r47jaja7/dXvvt/b2/v8888Xmy3G+ODwWCqFIVlu1sv5fL1dn5+e9Xo927YNw9gb77OHKWMN4BwCCZXUKfRdq9/zXc82TDMILQQrBImuY41gTUMdZ6CUGne9eLMddrtpFLdlzZrKdz3NMgFCEqDFaun7fq/X3d+fAKxMS1+vW84ZQRABDkSjAMjzqmkq43iMIev2nX63kxdpUWTTVRaliYSAGjogmHHOOJfYOH1yHnRCrGt+GJyen43Ho6qqZrNZlmW7e8iu6/8HCmFTWpbpOJZhGEWVSckQUrpOTFPraT1CNF3Xdc0EAHKhAAAY07KpOc8kVxgCqGGshOJKtKwQdd3UgjW+O3RtuyxLyzA//vhjztumaTAlbduWZVkUhZTANE2/p2EA25YDgKBCGqEIYd6wMi9MTQ+8EHDWlhWGhNUsrdLB3lBw7njuaDQiuraJtpso4pxjirzAZYzVZRFtQFVVo8GgytLg6blp6ppGCEW7q3DdNtvt9vXr10wqTSMAoVa0CnBE8HqzhiJcblcBr2vRGo7uWjYCCkCOCLSphaGUXMTRJo63tqP3B2HD0SbaCi4dtyMBVHJbFs1mE7UNZ23LOYdSGZru2ma/3w/9wHb8XTdytVpt4qhpGtM0HddFiNR1vV6vN+tos9mMRqNOp2NZloKAMUYx2cWhdqcBKWW/3/d9f71e73gknPOdqFpSgWpcVZXYiduV4kLUbZMVOQAAQoUx3q2JAIIAQtYw27aVUlXVJElGqW6aMokzjChjjBAqhCjLMkkSpaCUEkqilKKU6rq+c2r/gdHUNA2EcAfJeXh4WMw3nueNRqO98eH19fVqdbWzkO3v7xEKqAYPTw5evnwBsOKAr6b3tudSSv3Qi/OkFa1pWxCjtm05VIvtOvvNr4+eH/zZP/nZ9HH+8HAHJPz1r3+73UYQ4Kpqmpr1+/1XLz+OoghjvJvJOF4DoYrTWcuttm1s0zo4HNme/ubNG8aqJNmsVsuzJ0/2R0fz5ZoxhgwKIWzblvEGKkAQdRxv0O0VQdEJwvHeEGN8c/nh8vJSw7QTBF3f51IghArWypbpBFuWvTcaB0FoWZbnBUihpqoNzUREkw33XJtSohGiIBoNRl98+nnX8QnCWZQTjHTdnowmuq6LluV5LqVsGua7lJpOVdSb1Vq0O4MIz/PcMvXb20vHtZuygEg0TZ3n+XR5hxBy3WC7SYu8PTl+dnZ2Fqcl1Q2KCed8E21Xq1WWJ8fHx1mRc1EvV1Pb0U3TJIQYhtFU9dOnz6cPj+/fv8/z/MnTszjZ/u3f/fuiKPzAff7yJZcyybLLy8sdXdvQtK+++gpIGW02WZKcHB3tvHCCMWXbddvG1XY5myd5vlgseMsty+JMtFVT541jQ0Oz9vcO27Y1DPvx8TFh9d7e3tMnzzRqGIYBIKzb9pvvvvt//3/+v5999pET+mVRI2jMpusir09OTuzApMRaLrecqdUySpLc94Rtu8vlynXdpmFJkiEkMcZCiLKsoTQAIEBpy8X27duL3/36t2dnZ5blvnz+QqMEY6T+8EKSsSwrirSdHOwzxm7zRxvQw5ND1/DuHu7j5dYAescKe054tn8aOiFkkJfcIg6QUkpJEUEKNWVDET0/OYVQuZ7TNBXGuKoKDSDPsE8OTwbD4PHxcTAYDYdjJlQS55Ap2YgP37/fRilrlVJIQSwlKPKqaRrHo3VTVbVomkrTie3ahqHlhYiz+N2Hdw1rORRhz9c1ExFgedZHp0dJkui6lsbbNImA4FKxzTr+7LNPjo8PpeI61UzTTLZRlmWBF16+eZ8kxWa1lux9P9zz3NDUjSiKZo93Wb5BUCoghFAIEs4FRnSyv9/pdIZ74/TNdxCjgTXQTUNCYNs2hLAuSgihYHynJMIQGYZhGZrvOmWaHOyPjo+Opvf3q+WMUIQQz5KEPdSOY3qeo2kaY+zt+4tduLppmp126P7+/he/+MWu5+a6bl3XjDFd1y3L2uWkbNtumqqoih3VWDM0Xdccz2X3rBVssr//+XjUG/X/w9/+x199/esvP/5hGIamacqmKeu6yCtKstls9u7dO93U9sb7k70j13WXqwgQAxCt4rxVikOYljlj7PD4qD/qW5aVZLEAAiF0eHzUDbvff/99EsUH+0dF2dR1bVBdx8S1nYZXyWY9vbulBj47O7NH3VJH62W6miXr9Roh5E7GSilTt0ydWJZR5ZQTlm7KtuVUN8q6sW27KIqw23Vcy/O8z7787OLi4pe//GWapicnJ6PhnqHpSZIk2/VqtVgtBnVlEYIgUhKI6fxhE5tt22KMpVS7fP4Ote77PsaYMcYYq+t6R0wvy/Lw8Lhpmvl8vl4v67puWCskq5uyO+hhglpW1U0pJJOKA8gxUU1eN6BuKs0wGoiIEBJCTCllbVsXZd2UumVgApRSkjMlJGNVFsdSiKPjg2fnT96++V5IMR4Orq5uJFcYQylBWdRpnAGADGq0bSsl3z1ldF03Bn0l+HIxN6h2dnp8dnCgpLy7vKyrKt5s59N6PV/sHI+8ZZpttpwBAAzLTNNU0zRd14s2T6KoKsqO5xuYRptlU5VScN+2k20ECR70hqvNOolioaSuES/wnaq6u79njGXZBmO8WkyL3MjzHGPk+67jWp1uQDEiCD3e3wMpNcMMPB8AaZomjyspcMsBoWbQ6bveIIrTtmVRlBiGHQaYUmrqVKcaxWjXoRBC7G78QRD4fmiapmXbjImqqoJux3V8SukfVFtA7WaiEkpN03zf7/f7hq6HYRgEwU4ZtfunQwV2tknDN6uq2rUZAQAIod3MQNM0jCGhGsEEISglEIIDADyvo+s6xhSArG1bwRVQCEKlFJBSMcbaltd127Z8Z99G6g8ByV2T4j+3J0iel1VVUaoPh6PlcvXh/fVgMAj8rn/U8f3o6dOnbVuHYWg75vX15WazcoNhydusqV588lFaZISQx+m0hm3LG2wjzaVxHG/iTQCCtm3X6/U0OgwCdzjqHx0f3N8+fP/991VVbdbRoN/fbree5z1/8XS1WkynU8MkR8eTwVgqpZqyQghKxZNsUVZxlMTvLr47e/JsvDc0LPPHP/mjl5988vU333737fclbgFAVVU0VU0p1bDW83tPT55WVaWE9A1/MOiN/M4w7LOm7na7eRqXZakZFmtqnRLf8xCmmqZNJhOMKSW6YAxDQrAWbbfv3r3TMK3LahZ2OkF31Ou/ePbR0egIKhBv145lYwKXy4WmU930Nul2tVqljxVreKfTkQJIJqFSQRA4lp3n6WI5+9U//IpQ5Hl22PF1Ssq8HPaG1NCBgOvVpr6fDfp7R8dnBCGCMAbQtkwAw+VysZhOx+PRi2fPzs7OhBC2bZ+cnJRpDiWEioyG+1CS0ejhw4eLaJsw3qRpyjl3HGu1WS/Xq7KuDMtMsvTy+qobdnqDwWw22+lfq6qKosgwjMDzQz/gJthut8vlEko17PV7YSfNy8VsiRBSChOoOY6BoFaWtWhhXYiizI0D3cC677iHBwcvnr08OjqaTCb9Xtc0zaqoBVOnh2evXxfzu/XL80/uru8wpknyoFEz2xZ3V1MsrX53AAQu0vqXf/+r2ezx6bMz76NQI0ad547l6hrRdQ5A47nh3viwEw4EVxBiXTc5bxkTBGuYQMZY27ad7qCpuRDy7OhcQrCerTVD/+IHPxn3x4+Pj+v1GmOom0ZbsCqtEULmwMAtiuM4T6IkT2rWUgwp1Z88OaMErVarJIldO0AjCpSAiux19/NNZSKLFWI2W909PN7dPaRZNRpObu4eMDEQwlJBXTcZEwihoDe0LMu2zarNP9xebbfLKI2ITvIqXW0roBDnTEpaVrmUcrw3dG1T00iaJavFAmN8fn4eZZuLi3cfPnzYbFa9brff70/vp3Vd743GnuOLrEn9crVI0yhmFRA9RRCtqiJPo5blhknVimvUsCynbZmumR+/+uz4+NiwTE03G866nS6EkEvZNpy3LQDANi3bsG3dYowJxj3Po4ZuWCbRyf7h/vOXTx8er2fL+1evXr149TTLssV6VVb5fDnL8xJTcn5+niRJURRpmuq6vlqt1us15zzLMtM0d/DBuq41Tdtut998802v41FKuZSsaZumxrvzpqUXdWE7TmBQvxc4nt0bDZ4+fxZ2O7puBv2OhkkxL2aPU4RQ6Ps7glBdNgiRXm/Ahdhs47JlQa8/Aerps2dnz54rjFbzBTaI77gE4d6ogzDQiC6lhAh5blDkVZSkkCOdGk3VptsMcFXm5fZhfXFxoVu6Tg3btNbr9Wq6XK/XdV0fTA55k0OIqKnZlqbrtNMJuOcBAAkhYb/f7fUnk72TDx8gUnGcz2aPP/zJ50I0dZ03TcGaWrHW8lxb73uWqWFkaxZSpCiKu9sHRPB2kyBk6oQqpcq6ggDYlqVr2v5wfPX+w8HhxNYNnWppnORpKoSQXOjE/s81Oc65AkLTNNu2oihqmspxLAUBIUAp1jRVXZdpHAEADMOEyCVYaxomJNhx+ioNCYkMDWkENJwL2QrZDjq+Zep5nusErxbLb37/2vO848PDIs0aznZXZJ3oluUEXtjp9LZFSiltmgogCJGUikne1HXeVIpgNRgMbN2QdbleLlhV+K6FEA7DEOta2dSSt51OqFlmw1qAFMTAMLQil0WWbtdrwFmWpM+fHnFW7/V6gWP/7re/z5PUM10NIB3itCg1g3Zdl/Y699fv4+1a1KVLhk0Z8wa3bavrplTcdz13fEAp5Yw9PK4dz3n6atINw912zO8KhcyH6TxKG0VKy/TCnquU6o+AUkpxoYDUCKaESNbWdQ2lqKqqbVtK6WQyoVRv27asqrIsMcadfm9vvK+Ums/nWZbVbaMZel3XBOFd9nkymVim2e/3kySBUjHGdkpoJaRhGJRSjjEhlqY5nPPdEKVt26oqIYSEEErxbmGxMztIxYUQq+UGAGCaFmOsKArDMHu9HgBAKdg0bVVVgivLdAzDkFJWRV2WJee8rmul1O5YjxAiruuWZXl5eUWw1u0Mbct3XX8+X1N6nef50dFx09RCNpy379693UbrbSWxqTud4NnHrzzPXUdrbJPVerHcrOJF1MKqUW1Wb9u0tCwrGDlZvv3v/vt/9ad/+ueff/YDKfkXP/gsz+p/+29/Pp0+xHGMCQSyzYukqnOpmodH2B+PO90OszXHsTjvXr6/Wq9Xi+VqMhmenh5Pjk7Lqj46PneCoCzL1XoRHg4QxGWJEQIEEiWUZJJCajlGHEVVlAE3OBru7/ndIs+FYN99/evH6ZxoVCiICIGYZmlBiHZwcGgYCClWFrXkyrJwWVTLxeof/ubfZVn24unzLz77ATjhiZ4s54u2qvf29hwLSQWKsmoVM1zT8lyHtUVcJtvINuxuGAKheMsll57nGVRjTYMQAgBQrAG5M5OCh4eH0WiPQv3w4CAv/hDK28bJ7e2tEKLb7fZwv2maJIsdy+iG/mq1SpJk4/iD/mjWTJGiod/FSNM164vPf6gkXG+Wtu0O+sOWNcvl8vHf/7uiKIqi8LudVbQFGH38xWdEo5toy1smhCjruqqqqqpIv48QQoAYmm7qBm8ZRARCyBnYbuPj4wPbdKqy0TUbCGRQg3NOiFYlxepx8bfr//DVj380e5jF222/20UAfPHZZ2mazh5m79++n+wdgBYkq2Qz2xRFgxDbLlPbVpbuPdzMHL0TuoPJ4EApAQTSoO7ojoFNLImh2a4TeJ5nGg7nstcdnZ88tSzLNM3NOkIIZVkiBPMcUye6lBJjXBSN67pSCiEU57xpGEUUlO147zjfxCmMdapXRX1zc11VVRiGi/nUcZw0Teui1DTt8PiAt4xLuVptCCGrVVTmxcHBQeB3V4tlEpW//83r16+/7XQ6o+Fe1TYtFzb1iGOzRmZx4XjEMDTORauaqmqorv369W/DMDw+Olhu12WVV1VmWZbnu5ginWq6rm82m6ZhQkjHso4PD3YXHSm5aZp9v6tb1PXMMAzevn0jhfjw4cPj3X0QBBrRq6rpd7q9Xs80G4LseFNKQXeLzyQBoe8AqFmWmecpQNDzvKqqESQAEoS1NCubhidZ7gddroTrunmeZ2lZlqXveuae5bs+kIoxVrdNWVUSyoY1aRmvNvObu8siz1rZeB23O+y4HU/XTaoZxDDDMHz+4unj4+OHDx8sy9pdeYMgePLkyfX1teM4QRCoP2h5wWq1iuP4k4+ej8cjxzIVkEwwTCkAsizLsqkt1wq7HS7F7f3NOlr3R/3TZ2fJMjJNs8wLhAChyLT0/f09z3OgEo/zmZR8G2+KqkzyxLQNao0aCGzPbwX3wwAh4HiupuuP9/dJluzMTL/89a/7nd7L5698P/z669c3766CTsiYQEQJxZGODM2ajA/fvP3OoDcEUM/zzk9e9sL49vYWKc0gkFLNMnSDaoZGQm9s2+7e3kRK2el0LMPsHx2enZ8kSRKGvpT8629/1zbtyfkR4MrSrSJPDUId2/NsbzLYP5gcm7bx/ur91fXd3t6IS4pYK1vGOW/LyvM83/XqusYQPd7dH072As/fLlZtUWGClFJt2z48fr87jbVtq5RoeB0EQbfblVBy3hqmZpgmwmA3xyYUEioJ0cLQ6fcHBGtRmmZpISVXgGEkLZN6nm1aelUVgmGowKDXc1337uZmtVzeljd39zcvn73c8bXquq7rVjN027II0WzbNTTTYDkkEmFKKcUYsrZAnjnsdz68f3d/f1NlSdcP7u4vbz5cFmmGEGpLYRmmaxoYYwHUbiCPKXn9+jUh5Pz0rCmr9XodrzeOYZ2fnOoAVHmOdHMQdM6PTxardeh4GMBtEusYQaV4WRzvnz09PdKwmM1qEyOkBAZQw5jXdbTahF6n1x0hTIXEs+U24OjFJ5PucCzJCmN8dnT6YrG4vrmbzpdlXUuAEEBCguFwyBkriqwuSqAEIhhhjUJqAPa/PrOpYZoIkd1IIAzDtm2TJMnSYjcJsG3b8dxtHAEAdqe6P4zVLUvX9aqqDKphjLvdruM4TVVjjHVdzxpFsKCUQQh3zu6mqbIsm8/ngnDB/4B7V0BBSCjWsrQoimI3tyOEmqblOI5p2rvwQVOzHUxi9wdCiLZmu6OGEAIAgHeVUF0n9/f3L1683N9v7u8eV6sbx/G++OILpdR0Os3zsm3bpmlczxwMDu7ur01Ln+eybJlCkAPVSB4XycHxwavPXv1Pf/k/zhZzosPxZIRNmNZVby/8wQ9+sJ2u/vIv/2H+3/+//v7v//7Zk+cfvfo02iaWZdZlFQSeaepFmTPW9PuhlHI2v2e/vD47O9MIOT4+DsMw7Lgtq5ng3klgWjpCACE0m82K29sduw1iYOi6ZVms4UiRtqrjbbRdbX3b0ZFmEsLLpklKjeLAdACUL168KMtyvYmyItcs23Y9IZkQ4u13b8KwS4i23cQI4OPjkyAIzs7O7r79/rvV+uHm1tXsIs51avCGeZ73q3/8FZe82wsd3zKVHl1FSokXn7ykjH7//TsIYa/Xw4jc3929f/9+s9nwpnFc6/DweDTsWpaZpFGaxlmWnZ8/dRynbbhl2FW93q42dVnVZZXFSScMu70eV7Jt6yQLIIRXVx9M11utVm3NvvrBDx8fZm3VTsb7Hz7cIKA6ne7p6Zlt21K1i8WK87YoikVRjcfjireGaWZFjjB+9dFH13e3v3v9+7aqJ5PJs2fPPM97eHjYRFFdlsgmmqZpmqbrJoTQsu1OpzeZTGzPF1ytViuMKefCdV3GmGs550dnZ+dP3r592xTVw919Wzdlni2XS0KI5MKx3Lps8iQniLqWjyVWHAGMTd3RiX04trabJF6naScTLdibTD5/9XnTVv1+BwPUlq1jm9ttTKleVU1RVEopCFFZVovF0rVtiFQUbaTkOp0YpqYUpJS6QbeqqrKsQz+AED48PMRtQrEmhHj35mK+XBwfHxOd8JbHcQIhaqLi8PCwYXVWZkN32B/1F9PF1c314+Njvz+s8mKz2UCIwzCsWxH4PitAWylWgyypNtutQtiy7V6nc/84XcwWaZoRzajrFhPCOfd93z100iJdRmuB5HA8MM0DAGSepTsXs+/7QgjFxS6E67vuq9Pjuq6TPJGA5VX+9v2bqinPnz352c9+dnJ69G/+zb9ZzRd/+id/JoRYzubaaHx8dDSbrhSne8Pjfm+fEmOziRaL2U4Gs8PaG4bhBT6AiDFhGjZGtCwSJlSaF3leNoz5Xsha8fg4u3z/AQN4cnR8enIShqGGCTZN23V6/Q6hsNvrSSgBAvuHEz/0HMdxbC/s9Rzb4wqs1ttut6tp2osXL3q9nlKqKIodlc9xnC+//NLzvE6n8/DwcH19vbNDMcZ+/etfPX36tNPvZVlWVpUbukTDvBUCClt3Xdf1fZcaNMrT1WoVp4mBabKM8zTTqRF0fMsypORZlqRpnCTRhw8f7maPDWuLpt472OdVa7telCSr362gFJapt4KBRr2/vLy9vFrOl+PB2NLd6w9XAJAXz15ijDVNdyynrdpNvAIEeKGrGdqov5dnRZmV795cnp2d/fCHz/bHx7wBdV23TYUgZE0tAUzT1DI9y3VM2xl0Bxjj9WapqooJznl7cHDA2/qb7177vu/77rA7JJBeX97c3t42VWuZNqZE143RcPIwnW/WSX84ocSMt2nTNEIIzpmlGaEfrNm6SNJRrx+4XrzebDcbQshg0Nd0XdQtIdqOpe26LmNNGRcAqCAIqEGjaCMVV0AohZqmFkJUVWXZhmEY/X53MhkjRBAGbdtUVbVcTOu2MU2dYN+yDIwVgNxsqaZpQRDcAZCmiW3a5+fn+/v7dV3rup4kSVOWSinBlRCiKAohRGc8VEoBjGzb/s82o8n+ECPp+Y6QDWNVmsaPj/eCNePxOBwOdI1qGFPXSetys1nXou31+4cnR7/79W+yNKnSvOP5k/EYAZUmMYHDKi8267WpW0d7+1DBpuW8aZP19vTJCVcijWPftn705Q9C3ynyBMqWAOoYulIwitPNZuO6frc71HSLCTVfxUnOZssIEjsrea8X5BUb758cn7+q6zrJi7yosqIs8vL+/p5Qw/WooTuSt1Lypi5bqQwEbNu2LItz3jSNlM1O7LJcrjVNMzQqBdgN9neu1N09HkillBJCYAyllG3bMtbudkkQKYzxDt5jmmYhdyLiRghBCMEYF0UWRVGWZYRgSimllFC0W9NoGqaUdrtdwzAAgFKy/6x6AgDsQg7/q0oGti2vqipN091EYdfL0HXdcRzf94mNuj17EvHYd9of/+hgG0cXV9cvXj3vT8a1bKllZ1UtoWGa4dHhS4LxoiSGbdXM3qYibUuns1e22bfXH94/3A6PR0JWUbakpuzaIEun0wdL8wcvvpxsNlHKs9fvv716uBMtzIp8G8e25RdJAyysYw8wlUbRdl5/95ub2VP41Y++/O3i0fY3H338QkBvEX+X1Py7q+960zVAWlE2UpCq5N1wz9LsJM6yeU2Q5XlelmWrdXp9/3B8dGTpmm1oZdtYogEQt3VlmiZRmCj48umT7777RrJSk5RqCpRLJ9Su3nwwDMew3LPTZ5CXlts5mIyefPRqmUSObT+up4v17L/8F3/x/OxJkae/+93s7uoDwvuGNcoz3PI2TpOqzk3N5qBebNYIt1WWWTphbXR/fd9U1XDY3594WbYlmhN2LMNG/ZHf747OTs+Lovr699/86IuvINaghC+evcRI7w36nU6napuDydhxnJrVt7e3RFFLEt/3A9MY9nxCyN3ddRj6H66v49cxoej8/Lwoiv6of3V15QXh5cN8DVCv14MQPjk7n0wmlmXNOT8+Odl9UBolHNd5+dkn33zzzSKJRK4Fng+EuL5+gFL9yZ/89MnZ2fThfjqdlmX29HC/KLOS17ZmL6JVlS8lhEo0H796sd3GH3/0xdlJy6Xq9w+jSCjlur43m04Nu/P5Vz9++/ZNDaRr+Ofn551O58O7i7dv36bR/Pe/21oaOzw8Biz7i3/+s+VyKYTQKQWtahSDGl9tZ7PZ7P3791VRappuGIalG7w3CCrXdV3OpQS0KEESpbqus3K+WCweH++/+OILRBBE7fvLi1/83YNlG7PFFGtknkKIQTA2vOFoE21BIRaz6XA80nV9upiP9yfrIr66ujo+Pk6aXEmJEbpf3E9Xjwd7k5pl+y+eAtd6eHiArjnpnU4X82W63d5fTuezw2f7XMmqqceTLiIEAOD7ftC3m4blZek4juX5vhdyLgB2sgI1LVbK2R92qzLPkzTbpL/529/ogBqGMRjsEUIefvvb17+8Xa+Xl98uPv74pc7dL579SD5tAVOh71y8Wf3yl9OvfvTTYNDrjw48q5vGZVu0k36P8CaJ12EnvLm5EUIM3MF+72jaLB6iecq40Oj1dPrmw4XvuyUr5/Op55tJvFSyoVhuFhtWFC6loWFzxD3X6tiWbdujL3+8o7/9b/7FvxRCdDodQshwONxsNgihg8HAczdKKQg0CE3HHziuv42TyfFHmzgaHkkolaET09AINrpewFkTb9dNVWWrWbJZrfMM6XS9WV3ft4fnp+fPn+mWfXB6DDHhnNuej3B8//Curuuj3iCK8qZphQ5934dI+903b6LtOoqT6XQOhNR1nXPe7XTiy9tOJ5Cgns+WnLedXhh4FqEMEv7skyMO8u4o2K7iUuSbNP75L37ech764cmzJ2VZEggcN5zP520tgiCoqkJl2EauxjQHOjrXq6rSJRWcdUZjIUTbqKZpqzLnDdKw1e32NdOr0rTf2bv8cNXWOE35el3qZv8wPK/rWgiRrmtdV77X5QwwEXMoIVRSimi7qbL8fP9I5+DT85ez3jTP89VqwxoFSJ1kc4CEbmAF2uGgr6T4x7//e99xCcSi5axhSDLftrCSURQJoPYGe57nTB8eFWDHx4c1K+fzhyB0Pd+SiAeH/vXVva4TADOq1f3BoGF4uU5tF2sG1XWnrJr1ZipV7fu+7waMc16V8Wo57g83m814MDw+OL69f5SAnD1//s2HSwBhz3OirFonkcpgGIbAE51Ox3I8zitiUIngzcMjqxvBwXoVH+6NMSTb5cLSyHgyGg2HNceTg/00Ta9urz/6+NP3V5frm7t/+pM/MgXchvfv3r0jEBcM2IaJdH1TrNaD8eW7OyllsWXj8RgJFM3W+4cH2TLCtdQJhoSur+c/+emPm3V6afZuiwYIrGFDQRIGo5rJ+TLxu8lo7H74cDUa7UsBZndTJCDV8O1mXffHg8HAMIxOp7PXDbnva5qW5NmTw8n9/f3944MAkuq6lBQAhLDu6H+IjGw2kRDCcRzLcjCAWKNl3VqW1emEbdvO54vdzEkoZhgG1amUAEK4U9nppoFb3Q6C06BXFIWU0nR7Simia4e6Np1Oy5ru4muccwihBLLT62hE+//x9F9PtmTpdSe4hWvtR6vQcePqm6JSoFAFoACqJgDSGmwb40OPccz6vf8dPvTDjM3LmM20mOaQDZBAkwQKVYWqzMq8mXllxA0dJ452rX373nseDoljdp7jmJuH+7e/tdZvIQ5IXdO8hg01FKWtu3JLj6IoTVOEkCDIRdmkWcE5FwSBcQ4xUDQZYhBFYbEpAACGCob9/0IH6PV62/Gl1WoJYbS+vDmzbfvd+9fD8aggNRbR9fXleHdnMB7tjke7u7t5ll1cXF1f347H4+P9iawoUIRp7lclVZWWoSJDRev1HcKN27Z7fV1RpKrOkjCom0xiRZSs0oyYuizKsqJCIKMiA+NJ23FkJFRVXQNBViTZdiRRajesOju7uLq5MS3tp3/wu4psqkoCuEAIh0CoKpLmWZrlum5butFy7bSsAGCqoiCMRVGklHvrdZ5mvuc9PTkxFFk39KqooSSsluumJv56QQiFHLTb7TAK6qIUFbHXbWOMDF03TKvbGWmyQhtQ5gUE4NGDIwlD1jTr2WKzXi6nU5HzNApvry5ZU+uqXBZ5UZaaaZRp/tWH88cnTy1D67ltWzdATeqqImWVRfF6vbYMXdfVPEkDz1cUpSJ1HIehnyRxWjfU87yqoYBjTTe7g6GmaVmSWpZ1eHgIIU/zjHM+6g8oAWEYaprmOM5wONxmnzRNefz48fX19Wq9ODw8jOOYUh4E0XaM3ZprtmDd8XhMKV0ul2EYtlqtfr+fZdlmszk4ONgeAUUV5XlWpBngfDwe26Y5vb379ttvIeTtdvvm5ubBgwc9RWaA/uxnf/jm3duZH452D3TNLMsqSfP5YlOWtWk7b968WS6Xoii0O+4D4eDB0QGpirOz077b8cOgqMqFt16sV9P5rNfrWY699laqqmKItu/soijCMCzrmpQFhHi1Wq3Xa0JIr9NttVrD4bDbanc6XcYYqWrO+WazSZLEMAySZd999+2LFy8UTa2qcjgevPz+W8/zKqI3TcMASNKUAwoA0Ay93+/XSVnX9XJxX5FaVdVhv7NcWepS+fSzT1bL5d3t/Wq+MHT96OBYMk3DsufTtaZppm1tzcmdTqfd6bCLi4rUk71dCOFyvaKUcgC2w7jvB4SQvCybhiVJslH8pmnKvCzLUpEku9Rff/dyd7IjI+Hu9nbw+ed1Xfd6vbIs37179+7dO1VVB4MB4/Vf/dX/Ob0/Pnl4dHC4o6iYMQYga5rm4uJ0b+eQNeDD7PT+dgkIbLudOPAtUy2KAkI+Gg36wwGEnAJm2+bueKQrap6nooBUTYGQr9fLy4vT44PDKPYaWmu6NOh3J3s7/WG3KirEgSrJvKEbz0+SpGmabdRqvViWZbllIZRliTE+ODjY2dkJswQhVDfUspzZfMkAbxpm27Ysy6whiizuDAfDh08pKRhjkFHQ1EmecQjiPHt38eFmNo3C5OXL7x8+eiQqsq7rCKEoik7fvGrK/NmjR9dn56Io2qqlaZpjWhjjJIm2gbHj40NNViil1zeXs/n0s88+e/Dw5Ks339ZFGWdxXZeU1QAww7YYaZ49e2bqlrf0v/nq+x8Wb0Qk+b6/N9lDQIii8L9avJ0tzTBJki1FXxRxmuS//frb65vLPM+Pjo7evXs3HA4t09nG5YMgWBueLOtWkkRh3O/3tzWYl5fnW2Ty1iO2Wq0uLi4QQp1Ox7KsbTBKluWtIiPL8vHx8TZi9/mPvvA87+bmZrPZUEop5YIgGJoOAFosFr7vy5rmui4AIEmSuiGqqhJC/kuPEcaqqmJRoLQhDU3zLIp9hFBVVRyaw0n//PwcIZCmcZJEWZZ1N73AjzxvwyDstHtVVZRVqciaaRqyLFNKFU2WiMY5p4ArqoEktWwopbRhdDgcOqbhhQEldcu1VU2er5b3tze0tljDB0iCSKANwQCrkmpZ6jfvv746v/gX/+xP958+FZHY6/T29vZkSQAlOz8/J5RjJMZxPJ/PIRLCICaEMAAgRJRSRgEDqKpIlmUfLi8Jo6Sm94t5kKaapmm66bhtUZIWqw2lRJIF27Wurq4YBS9evGCzpecFZV4oqqaqMhbB5vb+61//5uRRvFyu0zSVBLEoszQORVEkhLScuq5rAEAcxwghyphtuwJElm2cqCc7OztBFN7d3W+h7IyxRbI0DEOW1U4Hb++i7WerLDRNs9lsyrJqmkZVVVEUIeNhGEKI+/3+aDRSFGX7DwUhLLKc0GZrd8AYU8qaukqrijVUFsV+t3t0dAwxKuoqSRJCCGcQUAYBQBwgADmHnIMPZ5dbBK0oihywbXrCNPW6braSNOec8YbxRpSwIAj9tjMajbb5C8MwEEKEEFEUheGkXZOkO9yfr26B0IiKrOra+dXi6OQoy5LvX79q2S3btjd++OHDxcuX3+/tjSeTyc7hnuVahqNrMv5wffa3v/xrVYYIC5IIJUmwbA1j3TS1TqfT6g5Xy7kkRKZuIyiZpixipcrz2XSW5RDDRsC4wrisBUppXVad7ghA0TA1t2NLonF5Od14yyyty7rgFDEKeEMhB5qiKqoOIVAkmWmMMSAIgiQqZV7FYVRkOebYNG0MBctyWF1pqpbG2Ww2kxAv8vr16/euaw56A4BYEATBxnv6YlQVVZ5klZb5DbdMFxAqIeHR4d7haPD27dub0/c3F+dVEqnSj23TkkTU644O9vaSLK3ybNBpG4p8d3P1l3/+Fwd7O5999pkqSgKADeUyFnRFxe2uIoiAsiJJwyQcDAaGqmZRWJNm43uUco7g7c2douk7imFbbsPo2dkHPwwRQlGafPPN10mStLudltkqy0LTtG63u82xpFHcVPXh4WG/2w2CYH9//+3b1zc3NyLClmPoAAEABEHiHHY6PcOwXr9+fXNz12q1AEC27QIA0jSnlMuyyjn8Z3/8x77nffvtt4Cyj549n4x2fvGLn3/3zbdN0+iW/vjx4+VybbnOZHcMIP7k4x8dUqwoynw+P724/vD+w9u3b5umOT4+zuIkj2MI+XjQAQ3RFGVvZ+dXv/y5IilHD0/SNP3Pf/M379+/Pz4+7o2GRVNrugYgqikVIcKQNoxwyAQFG2qLc14ULUJ2tpYax7IVRUmLXMtTQighxHVFXVeLIkvT+M3Ll3d3d5/86BMIQZKliiZrpkYp5ZwrmiEIyLZtWVWSJGmaGkKcRitFUdJolRcV6LSieF2TvNN1r26vNU3/6LNPry5v7m5u7lar28ViPB4/GB8mWbrVHdMi39bfYVFo97qtVosBXtZVWddpmm58L80zRcUcAk5pXVdFkcE43g7puqICxrI8XS2WrmFxTT88OLZ0i1KuKNo2zwIASNO4qgtZFlVNOj09X63vN97xs+cPP/nkI8o+ubi42N0bTybDq/PZu7O3WVh0nG7dFAXJaFQC2ORlJquSZZmyJldVdX9/P1/cU0aiwNM0VVNkRcYt17y6PAu81ZNHj//5P//j4WCgYFlT1CzOFuupv4kEQSCEvHv3brFYbEPV3W7XMAzLsuaz2bYCgzE2Gg6nd3cE/Zf23tV6/vVX34Rx3O30Hz161Gp16rKqy1wRpSAILj+cW6ah63oYB7Ztj3fHO4Nxa7L7UZ74kb/arLMiD1erVBAwhkVR0CziRVIEa8sxFUnebDb393fbPLquyt1ulwG6OxlTSm3bbrXt3/72t/P57KuvyIfpze30JgxjWcMjrw8A28f7giDYtskpjNLo4vK8KFh3bJckK5u81XKbhlBK0zQ9Pz9fLIp2x1VVNUmSskyrquIc7u7uuk47TfLLi+tury1LKkKCquq97kiRdYzFqqoW8+UWfc0Y0zSt1eoghBACEsS6rjPGwjDchty2xvKdnR3LsgghFxcXnuc5jtPtdg8PDweDgeM44+EoSbL1ej2dTn3fD8NYVfX3Z6fVtvJYVZqmQYKkGVZJCa0h5VDA0tZvL4piTUrNUCVJaQhtAA/XG0mVPvro+Xw+X61WW89aEHqzxX3TNHXdyKre6boIYcoAAIBzEEaerpnj4VgW7NvrGyA1Dw8fjYbD64tzCmBd16/e/FDVhaYppCrSMAcAqIKgtduQsyovYz8WRVFAIgQihIQD1pRVEUfBahNPIkPXLcNUJCVN05//6ivO+YOHj4uqfvv+rCjrwaB9eX39ww8/0IafPHyMIMzirMjyKi+2tlnLsqIwqapKVWMsiYqsvj8///FPfqwoCmmq+fx+tfGLH14JgnB0dLzhOAmjklSNIDY1EbBMSXO/uG8aWhHKOVewSMoqieIt7Tj0A0WSFUVhjHIOOOcCxhwgWZarIsvzUpHkw/09XVevrq4Wi0Wr1dqilBljpmnbti2KYkNYp92TJIlSFgTBdvJACBFCJUWu61AUoeu6/X7/v7p9c9M067rcLp8oqSmlURRFURRvojhJoICfvXj+xZdfapa98jZYkFTNKIqirhtBEDRFRQhVRZ3n+efj3aqqOKd1XS9X8+VyDgEyTTvPU8ZYnucNrSEEW4ITAFBRdct2ZVmO47goawBAURR17QuCCNb+xna0TsetaKmIWpKnYej7oXdwdLhZeVEUffhw8e70DCBBMyxakPX9Mo6iilQVrY22wQVepIRkoKBFnBYcNO1uy3VtSpmisL299rC3p0lJluXBJoKNqGtCmhZF0QiogFBURAkjUG7RirTRFOnk8eOHD090U7u6/vDnf/6XQRwgBAxLFwQB8CLJ0rpqiFEYhsEoQUhGAAoIcsqKpgAAuLbDOXjy5Emv3QsDTxgJWESGZglYZoQdPnp8P7ubvvx6Mhk9fvJAFNG792+SJLI0NdfV2A9Cz4MgcXSnTLMkySoBTSYTTcS2oQ46Lds0RqPhZDSmtFEUZTgcir53c3Mzn84ZY8HK99eZIszqJ6WMBcXWGkXDCMjSriAIW0iJqRsYw1F/YNiWgODh04/zPN9s/CTJLi5vWlCsaVMUxWAwFMXrIss3Gy+KIm/lhWFYVSQ3cwCAquaUMkmSthlLy7IWi0Vd1+/evdveoxBizuHV1U3e0O1bR1VVTVElQWxqsr+753ne5XxRZPmLFy9M3YiC0DLMttuqi1LEgipIogYd22akKbPStu0gCPqd/u/+7k9FWWIAGLpzeztTFOV86bVaLdo0qiQfHex32y3OmGvZeZ7m+b6qyYqi0Kq6OD/f29uRBPH69mb/8CCOYy8KxjuTP/3n/2w0GpZFUVOi6zogIMkiL6irquKcW6Z5fX0JAIjjlHKmqpIo4rohaZ6OBkPLdeI4TtOYEA1j0Q+Dm5ubNI2fvXjGGFus5pvNxo/8TqejGbpq6AAAJMBOrzcYDC5vrl69enV5fTuyEBYkyipBBJzTd+/ffvP994KkRFmepNlgMCKEUoR11727uZWj5Kef9t++fTtfLcfjMeZssVzarqPret+2ZFWhlB4cHRmWOZ1O37x5E8exINqiKKoy3kqSDW8wxhKWAEB1XqwXmWkaeZoByv6H/9v/cH5+3tTE1A1N01RZA4w3hEiCOBz0k0Tyg7WA4dXlB0KSvd3hyYMj1zHm3hoLiKMGC9xp2YZpYgEihLIq02QJIdQ0DQXcMCxCm9dv31CMP/vsUwGjXqdV12VRoJ3dSRgFnFPD0r1o422WZV7piooA2mw2MjKePn3a6XQ0Tbu+vs6ybDabxXG8BSVtufEIoTzP4ziO49jpG3VdAID8zcbzl0EQNXWdpdEf/dE/GE9GkPFOp/Pu3bv/9Ne/sCxL07Sr+xtJkW3btmyj3+89f/705OjR45PHRZkKAlou57d312Uc26rGqmI9mx4//9gwjLqpZov7ipRijnirtWvstrnDAPNDvyaFrMnj3THGWNZkUzc0RS2kVNzGwQUBIbD1ey1W88VyXpF8vOO22s7rd28Wi8WXH/3k5uYGQmhZluM4hmEIglCWhSzL/X5/s9nc3k4ZAwcHB4PBRBCE2exWkqSqqsqiabldRVGyrEriNIlTQmhRVGHoi6JombasSFmWVakviqJlWQ8ePNia15qm8X3/7u7Otm2EkO/7cRx7npfn+bNnz+qSFFlZllVRFGkUV3kFGaSUYiREYQwhrAn1gwghxACXVSVYpXVdU84ESZRVRZZVURYEGbuug0QgKvJmsV5t5hDD5Xqzu3fw8uW3sixvD46bzQoAZBiWrqvb5TOAcJvro5SWVZ5msTXcN1tVt+U+f/YkicOb+9nN9A5hkKbxluIMBWyY5nA4JIxXVfXw0SNS10VRVHmFJFDTPMjWcRCCun728HGWpFmcPHn8rCyK2d307u5O1S3P88Iovrq6uputHNeSFf3Xf/fVxcVVp9PZneyIWMpB2VCeFuUmCAeTYX9n1+03CKE8L26u7+4X6+k0oxD90T/4Wbs/yKry9esfWm1H09SHj564jtVyXAYgh5izRpCUtmuXZX1/dyfLimFYjJIiTyGj27boxZIiDDTVsIm9vcjbxUBRlrpuWqYpimLVEEVR9nf3RFFcz6//3oWDECI1raumqiqEMMZse2bbjpJ5XjIGdLNj27bjOL1ejzG2WMyCIFAVBUHYkApwCmmTxFEYhovFYj6fK1DBoqAaBqtIkeclabIssyynoZxBVDNOSAMEqqqSZEmCock1RgLBGALAGg4I5YahTXYnURQwCDiCdV0CyJumZowhDCRFL6qmYbBueFGUeZ6naVoUhfDNd98vFovHT54NRsP7+dI0TRonu/sHvd5gb/dAU437+/n11e1quWm32512W+XSaDTYPTqoSPXdm+8/3FzUvOKCWBWYMBk0vGmAImMMYJxk3pq42i6iBmZgOV1dfAgcO3ZMZzbzLEMrCtjUhSBWIsKSJBiGoTuOoWqKJtW0EGoYxNF0NiOEKKrUbrctx95GQlerVVXU3Va73+9PFysJC5qmEUKDIAKMm6alKyprWBwnq8X6we6RIgmUsJbdEY9FCcu6bBiqJUAhT1IOmiJNIKdZErRdGwPMKY6jQmCgLCp/sSKY99vtrtP6nR99/vTR4zzPCWlevXs3vb/P8zwpipbjMMaTJN3d3f3pT3/v4vyKUbLt2M2yrMoLUpeDfncymSwWDCGoaGqaJ5qmCYJg2/Zitdxs/M3a5xwAgCy3xRn88OF8MBibhp0kyWbpkaYa9AaOZXPOB92hJEkIIUkWCCF3N3eUUgHhD2FUVdVvfv3rs9PT8Xi85SuEfqA7bQRQGqW84WmUzum8Lup+p399cR0HcZ7k+zv7siwnWfLxxx9jgFlNdkdjCeFg4zVF9fbian4/63X6/+xP/vmjJ4+Xm3VZ1DVjcTYdjUZ3s/tPXjyRRWl+P7s9W3nrtWNao3G/5bjbgJbl2BdXl0Ecnb598+DwYDweh3GxWm4QBv/oH/2jXqfbarlnZ2dFnuu6+vDhw7Zrk6YIQi8MfcZYnkVuuw0hBBitVqvlcs4YcF13b2dXUmQkIFVVqOtwBFfeygs9CPmzF08fP34chuHL774B/5Vi64UBR1yQREEUq5JUhDYNTdPM831HcXiSBGG8e3QwGu+cXV6tvUBU1AYKs9kySPJWqwM5qhtaEIJlCWBUkjrPcywILV3PilxRFN0yt5iXhrH9w4PBYBAlsSCJqq7VZQU5EGVBQBBjyBsuYiArQhREWZoywsq8yBGWJUnX9clopyJZx21Np9Nf//JX11eXO5Pxw4cPdvfG33z7mzhaq5LY7Tl7OyMEmjjc3N9dv72+tCyn2+1++pmWx00WFrSmsqqIoiDJAhIlDsXlypOVeRLnlKEqLQ3VePTgoaTK55cfKKkcp3V4eHh5eX4/u/O99aDbGw3GLdf1lp4f+m1D2iLlp9Pp+fn5FpZQluWTJ0+iKGq1WpIk1XUtSRLn/OjoKCz8qsq2gfL93cnjk4eSKF5fX9/dXB7u73U7PcuyOIIcowbwME2ApCRFFST3zVWlKNJisXh4c6yqoiTi0WjYcZzO0xc1qYLAm06nnr8uqnw0GT7/6Fl/0CmKYr1eZ1nm+2tFUWpad7pOEASEEEkRG1L1B+2Hj56cX10ul3OAqeUYvWFPluUsyz58OI3jFGJwcHwgC2pTsflyESbo9PTUdd0nT55gjH/5y1++fftWlmXTNPf29jrtHqnpeuUVed0QLuiyLMmj0aTb7YdhvFnfX13eyLIqScpoOJYkRZLAtuwHQhiGIQBAVbQi3mx/oWmaw+HQcZwt8eb6+np7JYfD4cHBwXZ58Ld/+7fHJ0fbYyIhhFJuWZahm2EY6rquKJrjOKvVZrXxt65VCCGhjAEoSQoAQJQViAWMRSQKFWmSICzK2vNDAFCapr/41a9+//d/X5YVxpggYEURQUW2zWdpmr9/f2pZliSrlmUNBqNul9/eTL/+6pujY2l/d+/45DHC4g+v3pyefyiqcr5cui17srfLGAvCkHNYN6BhDCKp1xtlSYpRxJoGAVjlxWp2P7+fAcr6vV4cx73eQNOMX//614tNcDtbYkm9vFo0lFdNI0qiH8av373Ps7Lb7S7X3nKxERBWZNnSDYQlUjMmCLrb6hsGxuLlxXWY5xShwVifr9bLjccR3D88ipJ4Z2c8vb/1goAUhSIJummLsgYQVnQLY5EQOpvNMEJNXVUIAkKJnKuyIstyDnMBIlEUs8SRVAUjsalJXddIEDBCooDqWgQAyKK4bffWJL63t/ejTz9fr9fX19e3t7dhGG11h7Kst2e2brdbFrWiKLZtQ4mJsrT1G2ZZFoYhqWvbMhipOSWKIHCM8iwWMOi4FuKNDCTbbeumoWvy+dnZ2vdFTXn89DkFEGOsqjIhNM/zLehJkTU/iLaGYlkWBVExTEdVFSxIvf4QCVg3jW1BaFVVVV0yxkzDTtMcYyxJEkJC0zDGAMaiEATZdLr64dVZw8QgzPaO1MHQ+eRHnx0eHt7d3Z2ensqiopuWYVtVVU3v71tmxz9Lb701h/B6ejNbbwRV1AzVbvfruqppnRepiEwRGawusqIpUmooraZEgAqcgSymVeYnMWg5IkJIURVTN7AAIeSKpummgYWm5sl8Xem6XlaJKGHT0iGE796dHhwc7O3tjYaTumriOE6iuNNqAwAQBq5uQ4jzJN/khSCICpZvLm9EKMhYMQxTkyRO2f7+oSLJqiRf31xqumqYkuPqnFV5FlRVmUbRaGjsjIbeJi6iiuSlrRmgP0YkK7ICANjqdHYOjjgEWZYt12u43Cw29378/Wg0mU+XjDHHHQyG+6btblaLBrB14N9dXSEAl/P73d1Jp9ulnHEMOQR5VYZJrJuGrqubJIuzVFKV4XD00Sef7+8f0obf3c0Mw9jb2U+ShDLCGJuMxhjjLMv6/aGiKFvsGiEk9CNNU548eeL7PoSAVHUcx4NefzQacf7o8x99UXJQFMXd3V2v1+t0OtPptCxLCNH+/sGDByeMMVXVZFm2LPvx4yemab35/mXkiUUSy6LUdluaorbd1nA48sNgsVh+98MPi9V652D/+UcvLKf12HKuLl61Wo4GSUdFaZMVXrqqwkAUHz96ChCiWVTGIavqNEniOB0MRqSZu47Tbrc1Sbm8vPyrv/wPq9Xq5Ojw0eMTzJksCrIo1FXmLRdVVdm25bZbtmMripQk0enp6d3dXb/TbZrGdp2iyERR1jSNNTQIfFVVXrx4Meo4eZ5zxN+fnQ2HQ7fd+vkvfxFFUZy2HNd1XTeII44ggPjR46dN0yT+zHRbAy4enTyXdY1eTgVZ0wyrqmmr3e1NIur1AABjFklEQVR0esPBeD6bBUGQxnEax2EUdbpdz/cJIbppdLtdgFHDaJIkSZYVVQkxCuPow4cP6/XasCzIACEEAAYxAJwyWpOqQZBKIqKSKOvy1ftY328gg/+///3f/jf/5I8/PnlOSPPDDz+sVqvtNnI0Hjx//rSqk/OLN6omPX/x9IsvPxFE8OHD+9PTd4rtRnGKAQVAxFgEsKaclRXFEKZZyTkvq+bduw9XV8uqors7R48fPh32dgDkooyqsqaAaJqk63oYhhdnp7c308Vi5XnBaDDGEKmGvg78H96+sW37/Prq9ft3tm23Wi3NMjdhMBgMBoPBb37zm+Vy+eDBA1mW14FPecMoUhSlKpZNTQeT9ieffOL7z3/729+WRRIn+Oru8u371zUvVaSWtFZ0DUtYV9q6prCmLrL87P0p4mwxv9d0+eHDB1988dl4ZyRgMa9qjvAsDfIi0TTNsHSAuJLJNcnzKpM1SdWUg4ODzWaTxhEhZDqd+qFvaG0JCy3HERWsW6qiKLRpsjzx/HUUJmVZcwChAHVZffr8SdttsVggTbXxVqZh93o9QRA450VRRlEMAEzTTFUMCPDtzfR+Ondd90/+2T9sGuZtIkIa2uSUQs5QkmQIVYIgtNvdfn8YBN7XX3/NOX/y5NFwOPR9f5tYgxAmSZIkSRzHo9HIsqw8z7es306ns/3ToR8JgqBYMoNUQILdtrEgbD3qWKKd3sBpdfLZjDFGKKvrGosCggBAhBASsEg5J5RDzjf+YrG8lyQBYqHf72CBX1xc7e3t7e8d39/f13VtGCpCKiGEUR4GSdMEGMmG0battut0qqoipNmsg6p8Y6n6/d1tFPrnH06PD/d7vc7s/q7VcfcPDqAgvnt/dnl1O52vi4rouvnyu1eKIlmGaZg2hhxQpqqqZVl5kn44Pz85eWQ49lffvvz5L39tuQ6XlR9eXfR61njn4OrmGgkiIE1ZEEmSDcPkHFV5UZGGMaDIhq6Zw9GEQPT+8pIziBDK0wIIomE6/U43L9Jvvn8ty+If/uz3nVZHt+zFS//u/m+wguM4dlt5tzfaOTiynE4SF5Az17QgEiDntKoJ4yJUGGooRDWtAh9SSqMoUFVdluWq024IcxzH970wDBVFsSyLKUqWJXle9jstTdP6fbff749G493dvfV6XVVkuVzGcUxqqsiqaIjMYrIs27adlvFWhtge3GVJkiVBESUsQJEILcdWFMlQJEEQWm2HMfa3/+ffbrzNbH6j6DrAyItiJEpRFL745BPDtA2n1VCeJFmR5SQrQN202+26LrdBR0FEgoA4Z1VFOBcgwIbmyoooSeKWWMg5T71w4wWcc9M067rO8hIAJIiygKEBmOpvMtXQGRXznBmG3u0Mzi9v3rx5kyeJaZrr9XrbJcU5v1xcNIyqkSpIYpInSAWyIQCR1XlVNnlZVVmWGIYKoSEgzhra0JIDkXHSbTuGpmRZFkepZcPxuIcQMAzdsg3GmrLIMIaKBtLMtywLQJaVpRfOw3hjQ4eSRhAQY8w0zZ/87u999unn79+/tywLAXx+9qEoiuFw3Gp3t20oAhI4BW2n7dotU1NbdkuTpTgITUOHjPt+cHZ6DmBD6ty2VNuxLcNc5omIYJbEli6QIgecrhbLbgsAwpKkyLK5aVkMQ46K9mDg9oYPP/rRn/3L//7i/HKxWCiSdH56cXV5STheb6L+2EECBIxXpL6d3R/s7ydFHibxfLOSZZEjWNEGyxLkrKKNAoGqaSahmqZLskpo8/Lb7yllWVZgJG4ZHbIsc0AFQVBVXdO0Iq9ITQkhuq4DyObzeafTYYwVRbFYLKbTaRRFg8GAUTCd3pdVjhVt2ykyGY0lSYrDqNvtOo4ji5Ioit1u1/M8VVaGw2EaJ5Q0e5Oduq5ZQ0WEWdNkSRJsfMb4mzdvojjtjgZ7uwePnz57cPzw8ua61xucTDpxEGbeRm4yR6SA87YmqLoBqkRRtSBIDF3Fkux2+tPZ3LJbs19+tT/eQZR///Llr3/1d4TUk53Rk5OHlm6UWZpFooyRqahNXZdF2nasoigcx2l3OycIcs4NQzMMa7QzchwLYzHP8zosBUHSTcNtt0aTkQjIzdkNAKDTbXX7HQq4pilIgDWhACDVMBES4jSzLGt3v9s0TWhZw8lYX61Mtx/GUV4BzWypmjEej4uiuL2ZcgrKLJ/0R3wwpBXxfX8wGARBkJcF5xyJQpKlVVUhQZBVJcnSm7tbZb3a+D4DQBRFUNa0ppxTUcIIAABZQ2tYNqIock5EpKoaIISkcXL27v2f/tM/7bTab9++DbzgwfFxr9fx/c37t+9sS0+T+GBv//jBXsux18tFGHmkKQ72JxFTVktvvYiqjEEmu0bHNh3O4pJUCGFFUURBr+tM15TxsGvYraODA9bA5XpZ1JkXeoomxnFs2MbzZx9zBpTpHWDc90NvEyqyjDG+PLsXRfGnP/3p049ebMJgPp8Tb6Np2s7B/s7BPgCgZpRjpFlmt9sNw5AQVmREEtU0KefTOWL04cmxrimqIoqYqZqY3AVJGii6jBSgyUpZkuVmFSM87HclJAAIZF3vtt22bV1dXX391bfX17ejybDV7SiqirAkKmKUJlmZpWmaRnFdl6quCBgx1hRVcXV9maYppaTX67345EVdlN99+21W5rIqWcgQKoRyyBGllHa7HUXT/LVPCKd1QzHudFtVVbfttiiKjuNUVeV5XhiGmqZvVwK25UqiEsfz29upqqofffTRp59+QQhdLBZbf7GqmAgJDaGr1SaKIlXRNdUYT8S6bmaz2VYi3B05lmXZtr2Nzm/x/ttPr9fDGM9ms+Vy6bruaDTa29uDDGw17x9++OHi+urB8cOHDx8qilKWdZqXR4e1pMiCJEMI67KsqkpBuK4JpVSQJQkCzkFd1w0jQRCFQWw4mmmaEELKuG4aa89/9uBxEudpmqqKVZEacFHXdUFWAIfdbtewbM6E5cILw7DIabc7VJAWBZvTt3mRRYG32tuZHB/tQUR7vd58ub67u5+vPVm3FEsKb2fr2aaocsuyRoO+KIqmoSu6optGVRW6rlYN2T/aL0izjkIoy5s4dV33x7/zo3/wj/8RAPD0f/qf6oogQURYHE92MMattqaOFEaaMAgo5VAWO51ersGV78dx3BDWa/dGO7tN3Yiq2rHNy/ML2zbni1WrbemG9fTZi5vry+XmLouTMIw2fqDohijKSRxWReGYRkkaUjcMMSyJWEAYQcBZURTb2gvGWJHngihusQeev56Md0ejERaFqqrqulYUxbHNoiiSJPU8f6usHR8f7+0dMMaurq68TbDZbKqqwliEEBJC1mtPsyRFUSCAXuBXRYkgFEWJMiJBRRKwbRqttmOoiq6re3t7IsLff/31D6+vb++T/lDvdPuKhBvWLKY3w26HdipdFB3TNgQxF6Rt/Nir1lsAlyiKsiI6SqtpCCHEC+KiLBljHGEOAeMCwoIgCCmIAUCUNnGcxnG8xUBxzoW721UUVw3Dimo5br/IyXoTvn93/vb07cXFha7KgoBvb26wgA1DC8OQy/V4POoPBht/fffmOsniFuqYthnESwCAICLDFNsds9c1G5KkcTO9vdgGNyVROTqehGG4XALLsgxNLcq8rIPaC6q6KIpM01UgtHf29i3LSuIsjlPS5BwQLIC6arbXGgDw4vnHg8FgvV4XaT5d3/m+l2W547QUSbYsy7IsWZTb7W631R0MBoooMQoQFKqqghwsZrOX337/3fffPH50iATS7Vm9ntXrdzFknPM4jCATeUN1Rb27XS6nq80mtE3DcuzRzm4QRb/99oeCfM0h2jvYPzw+kWW5P5gcHx8/e/Hp+el56AdnZ2dhHIkCIoxUdRNE4RNNQ6JQM+qHwc7OWNIVWdf2Ta0sy9linheZ291TVd22HW8T3N3df/Pbl71e/3D/qCiKJEk0TVMUJYnz5Wxumma73VZ1M03TKIpkWSaEvHz50nVdSZLyPK2qahsE2Nperq6u+v3+2/OvXdfNsuzx48dFUbx+/frzzz93XXfrmfrRj340m80ePXrU7/d//etfV1Vla4pt28cHh5IkZVlxcXFx/uHCtm271d7dO8gJ2T84sFrtr7/+hiOYFSVcf//h9Kwu82F/cLI3RgAihCAS2qbe6g3ZbDGa7C2CqDfe2YTx7v7eeDjCEM2m9/fXtxjAg6Pj/YPdYb+3Wq2aqoSM7u3v7u7szG5vIG1swxRFXBQZIRUhxHLsbrfLOayqamstjuN4tVju7u5++umnW8LPzfm7NE055z/67DPLst6dnf7RH/1REIVv351utUCAYJIkeVkSSheLBRIVuAovru9rhg3TfPTsI7vdK4qiqqqnB8eKpJKyuv5wISLMG4YQSpKk3+/LqkJoIypyWmzBpmKaZQ2jYRyFYei0WqIobqHIDaVNQzCFADKEOGCcU0Z4nSVx6BeZmO/s2LGXBCX94kdfYIzPz8/LsjRNE0BumqbnrS8vL/IisWxVN1RJFt69f6tqWDcU2zYZY74XSJLGIaqqqkhL1xrs7x217c6H89OW49i2Y1qOaXWfP/3UafeKvNYxCvz4zav3F1dnfrQxbK0i5cHx/t7+zmbjh0HS6bYcy0YcQQ4ghMMdMJ/PGQLdbodjOF/7rZbpdtv98fB6evvu3TvGmGrqZ5fnQECSJPWc4V12jZDU6XQBayBn52cf7mfTxWr+6MnDZx+9qJraizw7jArSiKK4mm7qsqAQItDFEGRJChqCOdM0bTKZZGXhh97Z+dUBBwdHh1gUEfE5p4ZhmaYZqnKRZWVZJklcB5XruuvNMk+zMAzX6/Xz588X6xXnXFEU2zJNQwMi2LKeBAEv1qudye5ksitA4eb6Po1z1dBXq6swzCmlW08GQsiyrCRJr6+vjw4faJpWFBWldDSanJyc7O3tAQDOzs6qkrRarXar1zQsywpSM8YAIUTXDABAnpUY406nt/WOfffd9e7ubrfb3UbYt2Y3RVEuLy+jKBJFcTudbGNKdV3/+LMvDcPY2uY3y5WhW47jeIE/mewuVqskSSjllFLKWVbkeZEjLCR5RgjRua6qKgCA0Kauq6qqGAScc0EQZ4s5xvDgYI8xZhiWYViSpKmquvECCMFotLOzt4exuF5707t5VVWyrBJCKKWW6ewPjzinEoJAke/vbj+cveWAyKoync08P175wcYLoKiapmq1up3hDqQJZU0YR6auSrLYMJqVWRAErus+ff7k8y+/gBg9efbMcNz/9f/7b9qS/Ojhyd7efhhFH33y8Xg8Pru4/PblS0ppp9PblodBiAVRpqQhhFSMNbIEEXJbnSCIGs6gIESe3zA66PVlVRmNRqfnH8yZWhRFr9dFWDzcP5gp84ur28vzC0N3ipIsZvOqKBHCjDRVUTYYy6IEGN+SLssytyxDFLeROloWxfbxwjlfLpeEENd13XbLMMztaW3LWAQApGnq+z6lXNd1x3E+/+zLOI7v7+/Xa29bzuR53nQ6JYzasgwA8PxN6AeqIlm6VlGAAKCkQgjJopRjCBknZVVxynkDeGNoQNeVhpYcYtOwBFm5vrrI00wAUN4RbNNRBaxglOd5HCWCIHCAWN1wQEVRqKpqWwXSEEYpLWFDG7gNTGKEW+0ORHjLhmI81lTVdty6rgVz1DKG7nSz4DIsSB7eBfv7+3/x7/8P3/cppbksYozTLFcURVRURW96Tm9//8B1bQaN0ag8vz4LowxgwbTcKAzDIHtwdNxuTQ72Hupy7/zD5Tq4s2372fOP93d2fW9TpKkqoGg1vwlzy5YkSSqqvK6ZrAlMluI4JjWUBbN3MCElcdXee/nd/H5hItPp2GVe7erDgeA8sHd7D80t5fDR5Nnh4WFelpIkNU9q/zPfdV2IYdM0hFS306uzd98eHh4eHx/neZqBjFRMROrsfsNBtbu7G4dEEuzxSM2LtOXwPM8rlvY6g9k8C+PA6RoTy/rkk491w3r3w28PBq37xfK3331/dfb+1z//W8dx/5s//qcC4EVRSQLmnHe73bZlCILw9u1bWtcIs7enr/zYJ5z89S9/9a/+1b9arlayrHZ6bceRb27W797chtVby7IePXqSRLG3ua+LoMqF6V3z5PHeYr6WRMcLUt/3BUnkJTWYwSsQhqFuGg1nAIv2cHB5e9O+v1FV2TK1p88PR5uWZap/8x//CqOqqiqMakXmL55/qsrCr37xW9Y0r79/myflg8PHIrq+vZqpqloXDWS43+l/+PBhcHSURHHalCDO7m6uQF3/3hef7+3tvX///vzN6+cvXtA4WgTe/fmlIImHD477ckNb0s3NvFiXMnEEUZYUrQGI1NpsXkmyLkvYtUzEuAhQ4sea1UnSShTVf/wP/8lmvWB11es4tiTM06iRJFC3yiQnRYNF4+TxDkICK/LL2xtKiSzLmqJIvFrOF21NvDt7d3R4nC2XGsAKwZffnxq61W538zLd3Z/EcezHHhTBJz/6OIjCJM+6g64oK3GaVITkeXl5e7ellEiSEUWR4zi0Yp1227KstuZczD1OQbSM+s4g2HgnB48IIRxzSunl/c1P/uj3316cUUHYhBFE4myxcdx2FGVFkQGOJEmyTWOxXsiyjAWW8EDRFVVWBEFAEOqqjADEAJdSIbMAcTAaDK/qC4yxaXEAw+tpaFnW0rvhnJmNSFi2sz9ECCyWSw5oGGXr9bLVch4+fHgfVVhQdN26ubljDX/4+AlkuMjrhGf2qH2iPWtqKiBEEVB06fTyO3ttdjptYqr/7uf/bjm/twyzrL35+w/j8bAIvF9fXWRFygH9cHHhRYHba3/yox8dHx8/yJV2u71YLA73D58+fioJEmPs6Ohwi4jp9/s//vGPTdOEkEdR1DTN2enXqiMs1jPAESGF50UACA1hpjae32c/fHfe6rSenLx4+/7N/f0dYIzE64GtJWEWzVe6Zi3nK1EUR909U7eePH/wt7/6j6TZ2C2VwVun42q6JIba7e2NLha7B/urhbdaL0c7uw1XSc1qhgSp/ebNub+OWw5Lg5s8K2WBQgg5bZqmQQjIumhZhiXpjY4Hzjgt8tVqTQjhAiegGh9Nwrv06uqGxlTIJUYR4ngTx73xvtsd3s5WWVYYTscyTFXTsSAGYXT5YZHnebfbHfb2HUsz9cq27SovwrZDCLFUwVKFy8tbTIkty6Yo5nl6e/Z+dXttOa0HDx48f/4CQPj+wweE8GrtVQ0Z7R3Jhnd1cx1XzcV02R7eKIoSl6Vs23a/vww86e5ub29PNQ0lzdK8sCwLQlimZRZndV0LWhPnma7rUJQWm3Wr1XIsMwg8WZYt3QCsqcvSlFVBAHkU5RE4m1/M05UiqYPRiCogTVOjrRdNTgoSp0FN0oYSUDcCwhDzPA1fvT8ri6LVaqVxoiqDblsPPfTpp8+qqlrevffnxZsfLgzTFNTFzt6uY7c2XjQc9HVFddqtyWh8c3Y2vbiNfQ+U5cBxaJHZw97Bbl/X0OefPp4t5vP14n//d/9mPB67Lefk8cMnHz27vr10O+5wd/Dtt98iAWOML9bXtm3vjHY3m0242AiCgLFoaCrnPE/Sdru7bc1+ePJoNB7Kc3GzWdzNpl64OXywf7+ZD/YP4oo2cPny1auspAwiWZQaCg3NlCUtz8uyYu2e5bpulhUuqmmTCrIsICxiyBhqmlRV9CzLN+usKqPVYvrg0UNRAKvVynXbTYI0Q2saFqQRlsR2t4VFYeFv5t5akiS753YGHQRgURSapsiymBZ1EmaMMdvoWqpLqgJBrmtaU+fdVr/Mq+nNFCEw92eXFxf7+/sa1k8OnlxeXnp3HkfAtC2JSyqWKa8wI6E3D/yF2+5MdnaABKLIr9MMyjKDkHMOVR0xOYui5XyJMdZ13XHcbWcQAAAgCCFUNdl1u1VVRVHU643qui7LklIiMNYoimIYumEY7XY7TsIkjbIskyTBNN2qqjabtePYnU5na83lshbFnmaoDNL5fE4IAYjVZVVWOULIdV3btgEA2+q5bq+tW2w8Hp8cHRuaXhdlv9NjVdMURBaqneGBbdvr9TpKE8d1TdOklLISNyVEutSy260ng3574m88jIVPPvrUNmxJEEejkagaOmdZlvm+3xn0y4YEUYgQMgwNC8J8OS+KotVyzs/PXr161dBaFNCw1y3LfLNcUNIAxv2NjzBdzjaQM9+bN03V7tj7+/vtjuN5eUVYfzSWVXs+n98up4/h883y9u++/buDowe/9we/3x53zy8v/vpvfm6a9mins1jeZUnaarU4BUkSk7y0LEtVTASrTrsny/JWsg2CYD5fQQhns3m/N5Ilrd8bxbvZ2//zLxZo3hSk1WohDnVdFxBWZYU1lFGa5zmEmJQVBhCLMmhI6PtRkjRljgRpvLPz0eMTVhdpGJQp7tiHIpYO9g9Dfy1gWbP1hycP7E5+fn7+/fffZ2lhWRZjXFV0Xdc/+uijLX6g3++enJy0Wk6SJMNRP1ivoiiSRcnUVde2GKkDb71ezi8uLmbLBQDs8PiIUNpxrSTLNrN7VSkYx7JiEEKCuLRtue22Hbc73NmP8tJPCoAwY1WSxUEYVqQGAIRhnGVJkUQNqWQJR0lRNVNVM/M894I4Sou8qjESKeV1XTJaxHGKMVYlVZG10XAii4phGIw1i+WcUjqZjIsq+3//v/4/juO+ePHx+LifZZkXBqIoAih4XnA3nfq+77puw2hZlmmWRnFclqWmSY4zLEomKwJjbL6YxpFvWRbnoCxzDFEQrAmhtCaWZWwjbWmaiiK+vb3OspQ2TRQVgiCVZX5zG9q2zXg9HPV9XwjCtWvrnPPQX2mabBi6rusCEjHGmqwYmq5I0nqxMQ2D1qTVaimStLe77zrO5eVlDVir1dJ1fb1eXV1dWZbRH3QBAN9++9uiKKfTaZ5XvV6vLOuiyB88eEAUqcyrPK9FjPOy9j0P0QtRUCmhh/tHB3t7AIDA2+R5LoriFpURh+GHDxeWYbq2eXh46Lqupml3d3dBFJakJJwZmu7ariQIZZ4bWPPDQFYVp+V+8TtfHhwdlmXZdltRFLquCzmY3t7Vdd1utzFEeZ4bZnt/d78s6unN9PzD7eXFLWfoYG8/K/L5/F5SRUkRJElomno2n5ZVpUKlaZhjt0RR8r2wKGtNM1arVbvbEgX15METu2UwVDagyLNKNywAclGUOYcIipZlB2HCKJAkCUHOGbw6P18slozAg4ODbqu9ajYQkKoq4jhmrOGQ1l7ZNKRq8oqUdV2bjjUZj83HT+q6Tot8uVwqSnN4eNgQ6vthQ4CpWcPhUNds27Z937+/v6eUlm5LFPEWvVdk+dY75jrO/v7+aDQaDAah50+n06qqMESEkC3YX8RYUZTnzz+6v79fLpdBlBBC6oZiQbibza6vry3bBRh5vsc5f/78OYQwjuM///M/393dNXQrjuMsy7YM/21bjyAIpmlyzmVZDoMIACBJEiFc123HcURRZAwURcU5bxoWhqHjmJouVXWWJKQiBaygZRu3l1dRlIxG6qDX63U6m+WG02Z6exMFIUJCVRZJkimi1O32FGXb8ISLuPBXsySO8zzPE62pi7tuq9vvQ4FrhtrrdRiGUBA1w7Ace+XNARIhEvKCrNabOEt1XXcMfTzst1puEoVnp+873Z6la19+9vF6M/nm12/b7XZTV7fXV7qqfP7lFz/9yY85g7ujUeB7VVWlWWYoarfdHnZ7oKFVsSU5cEHAgiBCiP9LAWNDAWBJYpCGYSxWZb1lDiq23G53Hpwctbu9u7t7QQR5WSPM8zRpGsYZ3H4pqQBkiiokIVORLECZcwgAFEVJlVVJVqIohhCTmnIBLu4XeV5uMRiqJpRVSimVREhh4/sbVddMXWcQYIwRBwAgSZJN03Yst98fXlxeE8fyN15Z5pQQAICmaQgwyEHTNFVZNHUJAFvM5lfXF3/5l395tHsoy/JkMun1+1VDyroqiiIrcsux5/P5cr2SFAUgbFoWYyz0fNvubZ31W4wHFmVN01zHquoGQkjrmkrStiISIYQxhqzeShXbiFDTNFt0oxCE3ng8fv7i6cOHDyGEr1//8MMPPyiKlGVpVZWMMUFET589Ho/Hv/jFLwxTawD0vZXns+F4UOSpqekAc1038zxXZEVRpLpuPO82T1LOIWvo8cHEddw0is/fvr+/mwkQKVh19K5r9Cy1rYtW+3gyHu+cPHrY7XYph47W2VIt67ouiqzXWm83chDC8d4uY2AdRcuzs1evXt3e3vq+X3O8ZQplaazrugBhUWa9Tvf4+PD8w+l6PlM1OVqv7q4ugiD49ptvzl/e9ftdW9erqnj17ffvZBSGm/6gd3szVxVnvLOHRY0heTDuQGH96t0pIByqSp1nQZ6zu5s/7bY/mQytrjPfLBhjQKDL1e352QfbsnRdDzaepVq2bauqCgDe3z9SZK3dbne73bKsHz58yDn/+c9/EYZpUZAwTOuaPTo+iaIIUu6aNmzgoN0HABiqIUCB1iRIM0GQOGOiigQA0zACVYbrnKZNVjew0zoYDqskTvKsrmtvvry9vHj44IRSgkUNIyEIijzNHp08dN325eX1d99+H4bRk8fPHMv9f/4//u+qqvZ6vfFwsL83xhgXeaKrari8hYxosixhVKbJaj7L04wxxmlTxPG7V981VVLW9fHJgyZLp4sbbahgrDru0A+DvKoUJnCoSrr96v2Z0+kXNcnjOC1rgAXPX13fXitKT1VVjMWKsDTOVUWQFVUVtNn8ftjvj3YOCCHB9ZUgyVXdRFGkYFAktWUZgiDLomqbDmcsSZINRJKkrP1lVqS9Tp+wUjVkP1rd/3apaYokSZKsJ3G2WC3TNO6PhlhE0/u7+WoJEDctVVZBEIahtyI10jSNNLUfJB6AoigjhDAWHctWNdWRVbpVfAWIsayokqELG28lyUgU5aapDUMfTwZBEOR5ut4sdEMydHG1DgxzwFhTkwxXkiAgSmpCCKBA1zTbMFVZC4NAwtJgOBr2+7P7uWWahmGUeYGxwCkbD0fj4ahpiG3blJGzs7PryxvXteuiqUtACUiiVBCEyWjvYjkXoIQ4gRypsuaYxLIsWVQ2K282n1JSbaGcgiAIkmSa1puzN5Zl7U52er3ejz7+ZDjqbyWGhtHXr1/HWToYDk8en4wm46Iqr84u7tkiTdNnz555UahomqrreZ6vNmt/4yVJcvb+VNd1x7Q6nU7TNKvVquQEQ8O2HSxqttMWFWlbfoMxCiMvf59giX3x4y/+8Ge/J8nw22+/JRmsK6ZYAqNoudqwhnU6Xc+PLi+vRRmbrtrpjgqSeuF8uUxsp08pJgTezzZI1NKkjKMc4szQzbIoEORRlJR5KWJ5Mhzt7u5NRjuz6Y3vb6qmghDrmpZXICxLWrPN2iP1O91QHz5+fKLbtu1qmlFmpbXb8v3w7nba6/UU2bi/nWHMbds+OzsrioIQYlmWLMtbX8JqtXr86KFlm03T3N3dUtoYhj4Y9CEEeZ6ladqQOgj9m9vrMArajss5KypCKJUUjXO+WK2X6w2AMCtLURTzPBVkxbFtUZJEUUySpKkr13W3+UnO+e7uriRJoijP53OE0HrtbWvit0S17fOdAQlBnqU1xg3nuG6aus45B5PdQwjpYnmX5VGv31aUVpKGUZK1FRUBmMfp/O6eUuZ5niyIQRCsVpvRYKhJYo2BLKFey7FtO8uyumDJpiqKShYa0RA5oFWdrNbThlclyVs99+NPP9qE0XK9SpLk5uYGYrmsaJVHWRRXWVplhWHZtq50u11dVaPQ5wF3TCP2PJKlmR8g1uxPRs+ePfeWs6//7hdRuFFVXZCEPI54WWqy5JhDAUDdMm3DoO12UaMkScqiEkRJlTUAAAeAMcYhz8siCAJCaoTFhtEkzuMoqefFcDiUZUWWFdvWBVkkrBElUTO6QRCVBUENIE0TxT7nFAAgiaoi6xiJVV5xzjVD4AzRmlqW0xBGG4pl0fM8z/OGw3Fd1G1TV1VVM0wOYVHVFSmrEmCM0zQVRZlSSqpGkiTLMAWI6qLc3xtrmracz6uqKLK8rPKWbSdpdHdzDTjftqlJktDpdBAGW/aGoqrdXk+3TIRQVuRJVtQNKetK0XQsCoTSIsvvp9Ntm5SIkYAghwAAzBgDpGKs2ZKeK1LXdS3VNRLF7ZQgiqIM0db5yDnfOhsEQZAkSZBlkXM6Hg8nk9Fms5YkUZYl09IBpPP5vN12J5PRZDKyLFMUcbvtboINYpWsKrvjwXx+BzCM0yThsSBInHNS0y0lTRRF0zQ1TXv+7AhjoUizWcPLrEQAa7KImOj7EWzUwx89/cOf/aP9kxMAAKkIQogRVOeENFRRTMWxZ/fe3/7iN69fv3r49CGS5fFkImhKhfjVYn4zn6mqut54giCIGG02G6cyRQEFm7WmyIv5vYjhweEuRiCN/bevXgaBN735ICDR0OT1ZpnkiSQJR0cHDeFNjTESlqvUtKqWO5QVpdPpdLpDQZRfvjxtgMqwevLkk4rUDZcU2RqM958//1Gappbp5knKGOQMWZpdKtV3L1+bpnlycqJpGqXg1dkbXTfb7bbrur4XW5YV+NG9tNB1/fLyJo7jnd6OqdpN00hYZU20JSDZtr0lwSVJhqGgKoqhaCIWyzg1ZaqLgixLTV4UwQY3TcdUTUXceEGdZWkYkpI8ePBAEbXFevXdD28ePz3pdHq2bZ+eftgSvjDGVVVwTr/88vPd3V3dUPM8i+IgCII0jUFVlEVRYQFzXhYZ4HQ0HuxNdqIoEjGMolBCQJSxhrisS0VQRCHVTcMwnJIAgAssqOsgydmiJo2k15dXN1hWDo+Pjx891jTtN7/5zevXrx+fPBxPhqIo3lOapHF5v1wsPV1T+qMdRTU0HZmmX5Yl4Pzm+trVzcCPFUmt8iaXS13XBSyFYVik2XA4XCxmaZr/8T/9k8nu8OBot9Vq+UkTJfFq7UuKenB4bDnubDmVVSlJ4zAJkzw0TV01RFzTICJlHbXdybb7WBAhJU2e54Ko7O9PXMeRRKVpmsiP8rygVLBtV9d1hCrAqGtbm42va4qmSx3XGfRat9Mp42WRxy8+fqbqiFJSVSV1tG2otSiqLImrogwDvBFEBDAlzDbMk+MHn3zyiSzKVVFyykajUVJVnheYpq3r6jaukmXp+3dn3W5/Mpns7jZFUYxGI9u2DUNTFYPXvGN3Bu3R0dERgsLt9W2el3EUGIb24fT0/PT04cOHiqRMp1NVVR8/fowAPjl+yCmYz+fffffdxYWpqupg2EvTPAzjJEkmk0mv1W2Z7rV3fXd+s1xHTdM8evqEEGJYZlmWTV3XVTMr5mfvP8xmix9/8eX+7sF6vfa8oCqaRRR/9+qDAFFDag5F02rNZou0+O7x45Od3XFdl+/evWl3rJ/9gz/SrX+oqNJ3vzmLo2y98uq6SZKcUuB5gevas/kyTqPhpOd2DCjzNGVh5BtWlBSlHxZhHImSKUpSRUAUprJkYCQWaSlgqdPpHu4cWJYFOaA1ef/+NE0TCLmId2zblBoV0JAxOOrvJFl8P11CIDYNmEwm3W7XMp0GMEr5HZi2Wq1+bxx6kecFWzDitvaXUhrHcRQFg8HAsizGmuPjQ0mSVovlZrO6v78bDHoQ8V6/AyCDHBRFFsdhWeZFIS2X86pKyrK0bdu27ShOVqvV1j3gui6AEAMuKzKDoCpzCFi75UiKnCRJFCYY40ePHu3s7CAkxHGMMfa8YL1eby0Of89CIAXdOu9Mx2y3O2WVr9ersswbRoejjmm5eVnkRZmVvCaVbdtVWiGKqrye3c3zPE+ieMt9QozKInZd2zI0TVH7vZau65oqnv7wrsoD1jRuq60qelYUWV4u1jNRk6PE98M0r8liuVxvNgCjmpSqZsVhwupClyVTUTRNlxCUBVg3jQZBHMeWYa7ms4uLi/PLqyAILKO/mN7tjUfDXjv2fdA0bccuimI1n62Xq06nM+qPRIgo57Zmtkwnbe4hECDIBEGQVRUAwGmzhSFiBIqyAoAjJAHANN2mlAZxcDu9hxAOh0PN1DqdXlmWACAEhaLIwtCnDQQAVkUeM9o0bNSfYChQwqqixqIgYklAYtNQQzcKUCR1LYtSU5M8z8syX60WmSV99tlnB4eTJMsXqzUUpKqsbjcbLEimacuiQhFralKkmSRJEELMmakqsN9WJXm5XLx//35dFUHgJXFomToAYLPZRFFkWYaiKLKkckyRIAGMmqZhgBNKRVGUVaWtdBRNlySpqMowTquiVCTZdh0RK9vo7H+pnmJckSSo87phVVXVTdGoEucabxpKKeKcS4hzzhhDCG1/oSiKmqYJBwf7lmVZtj69v3n37h2EcHdvFEWR41j39zeWZdi2uVjMwmiTpIHneVWZCQh1XXtvZ3J7exNlybaOs9Pv/P1aTBRFQRa2kIfp9F4QBEu3Dg+PHaNdZCWniBRNr0sOD45/53d+srd7srr3X799F0epZVmT0d4PP/zw5s0rt2P96EcfA8Q5glVDrm9uoiwdIJClxWy5WqzXaVGImtHrddI0JVU9Gg9/98svRAH9/D//pzjxFQW1bEvV5DSO4jCII58yYtl6bzIZjUZlE0dpAJA42Tk4eagRQq6vr5dTX8Sz8WQYeFGW5IeHh3/yL/4vCP41giqphEcPP2Gcq3KLVCD0q0H/IJRDAYm2ZezvUsswx4OhIrn31zNd1/f29ra34Onph83mZrlYm6Ypy/LJycOPP/5kMBh0u31dN4ui8GZ3454BMWp1OpxDw7ElSdJNgyHc6vQkRTNVjVOmq6qhqSGjN9fv8jy3bTuKojSNsyzLihwgwXZdTTYG/S7mTJFkx2mZpj2fLZ8+frZcLpfzlYBwp9X21h5G4Mnjxx999NHOzni5XHietymzzWaFMQSc9zqt++k8yxJBECRFtmy30+72BqNWpxcEQVVmjmN1Oy2ImKEquoRuLk+LMgOAMU41TdNNM0jyKC1avb6um5Ryf7lMkjQOI8O0//AnP+n11rquM8riMCrLUsCibhiGph4fHw6GOy+//S2EHCOYJknbtbMoLPwoTVMMeJ7GReYaqiIJgq6oJan9KJyvlpKoyJp6dnG+8vw/+7M/0zivG5IkkecHt7e3DDVxmrC4ma3u0zwRBFhU2d00KkkJIeyP+lVWBWGZpUXTNHVJPM9znJYo4clkslwub29v/bUvimKvN1BVVZbl1WxhaZYqyds6RN/3kyR5+PjR7mS0vzv6zTe/pk096Hd8fyPJkLKq220nUVQVmaJIkogZYZQ0DW3qoo4a6m82TU1c20lxSkqSxdlwZ7xardrttu/73tp7/f3r4bDfa3e33W6qYe6M9uq6rnLiWuriftN2eoSQIs+bsjFNtdNqM4c1Dfvw/hQAZlqm41oYCtsjAmMsDqLRaDTsDZaz+cX1TRRFALCDgwOMMRZFS7eyOHv525eaImdZloexLIiIg6aqto/pLMuytCjLcrlc+0Gka2a3P1Y14276fRhGuq7v7J8QBuarRVlkvXarNxhwTi1b1zTtd37nC1HE//lv/vO333zndtqPHj362c9+xioNi9L3334fp7FuKcGmnC1WhqV1Oq3lZlnelGnVdjsWbVBRgdUi8tM0zyvSYFW1e8NB3cCqqhRZ73bsl799ySnvtNp/8Ad/GIfJuzdvLs4uwzDEGGu6oip6y+7kSlpWuc51BhkhJKLJarWp6yZLC85QURT+3N/Z2dsmOERBbbValPLp3WKrpW4/siBCCIui2PoNW61Wu90WseD7/vZxbBrGaDSq67ouK0opxnjLTQmCoCgTSqkgSlgQwzAMw7DhTBCEOI673a6iqlmWlVWh67rlOgihu/kcALBVvrIsS9P06OjBs2fPFovFeu0hhEzT9DwvSZKqqqqqwjLGGFuWNegNdMtcLKokzdIsWXsbxnlDq7xiQACUNXXdiApI4lRVVVEQEZQNXWYEMgqzrIBAzLPSNBvHcXRVY4yWZYEQnC+mSRw1lFqWhQTIIa9pHa5mnUFfFIWa5Lqstm2TNY2KMSRkNZ22W07bsdst17YMCQEqYsgIoc32hrRdZ3o3TdM88ELDMAGnURTVdS0Jom3bpma0W60kzYuC9LsD3TRISaqyUVVVERUBS6KgqbpDuUBpwwASMQQYA8A4ZUgQG85kQQQANA1XNRsA4OCCkCpN06woKQcNoxAjEQuW5XBGNusVbYBlurbpmIbNGMvijDccYAAh1BTV0HSIYVGUGaMMcAQY5JQDKohIkcVKEgJvQZpS0yUOmRSKRZkVRZElaZxkrVaxVevqMs8Rch3LcRyBA1aXuiTbjrlZL2b3N4Hnk6YqikJRZFPTGWPbI5wkSVmW1XVNCN2+ZxvGMMaGZZum+fjpkyLNIhpBCCUsYIwBZXmU6Ka45fpIksQ4Ak2NAJdlkfOaNXVVVaRSiVyWlHLORVmqENwuqBhjW88mhFCSJEHTpS++/PTgYO+rr756//5tv99ttVpJEm28FUSc8SYIvdn8rtfrKYpMSK1JiAJA6nSzuKekYg3RVVVWVQEKHMPt1iJJkvV67SEoCQLJ20mSqbLSdtoYioqmt52uqTsttzccTHTdfP3+/fv3Z4vlmlIOsfg//y//Zr1e+fFmNBqUtBQkeHN3k9X50eDQdq26rk8/nL1587aoiGG1GgIeHO4wxooi3xlP/ugP/zD0NmfvXy/n89ni3jAlXLO0iAUZCaJQVdwQdCxzycJ7D3f1lkEJF2TJclur+Wo2W7VaLVO3Ok777u7a36w39/dd29gf9Fu2LTCmiEIYRUkQQgFPr++63a4iqYSQfqfbaw8FhBVFoRQ+evRoa57YbDbb7s1erzccDquSVFWVZammqYwxhEC77WaZNHRbgiQKsiSIotFqC5LQAE4IqTm1ui3DMnVVK9MEQWg5NqXUaLWSqipJU5JGqnIAmGFokiSlSSTK6mh/d7ZaJ1EAGMMYK5JU5UXbceu6SaykzMqnjx7/+Isv//if/OOrq6tf/u3PP3w4ffDgqNfvxCEAgJGGzO59SqlmWgDhoiIlacqGpyUhVQGQUFZks9m4jglYbfXa+3uT++sfwmADeVUz7rR6iiRwShjAdVkWWR57gRcE0+l3//Ev/sOTJ09+9rOf7Q67AKA4jkFT06pSNPXoYP/BgweMElkSijwNNpvxaLCa3tjy4aOj3endoiE4z6I09Tgng2EXY2hZlg54EmdRmChKE8WprBieH6834XS5BoADBDx/dX59yiEzDE3WlJu7WyxwDnlVFUmREEKwiGRZNmW3aRpFlUyjSwmTZVlVdVEULy8v0yQnhIqiDCHKshzBUJaLm5s7w3B6vd5qvorkiHNeknp/f980DNu1pO+kl7/9xu067ba7t3MQJ2EcBFHkV0Vp27ahGnVJ6pIIEFPCHNNp6uq3X31dltWgO/j7Vo4tXN1xHFVV1+s1QihJstvbaVEU/X4fI+nDhw9Jknz55Zc7OxLJSkqp7/sCxJqstl3HNE1VVVlT9wddAUvdbtc07fFkUhQl59xfrVlDW63WRx99TChdrhbrzcYw7bLIEBIYBXEYVVnhWrZpGOPu8OGzj7IsMySlyjLothDjGCIBiYIgaJoBOIziRNFNJoiCphWUCpQxSmRNBYABDDXL6LBOr+NiDKMo0nVd03Tf99++OVUV4+jo6Hd+8qVp6VVVRn4oiuL52UUQZIKMVENFIdz4vqSLqqliUWkZJgdSWYKmETiEomy1W0MsaIw1sqjkeV6VDUKCACGG6N3rN0mUkJrYZkvTlIqUq8UGAMB4nVel45qSIgKOKGVlVUVh3G4VgAJAge/7sqwCAOq6Xi6XkCHbtltu7/zsfAvGhxB2u11JEmazGWNsPOpSSsoy1w1VknuyIiZpBFizbbNkDW0aIknCcNhvu26e50mmDIfDyWQSBMHl9e39fKkbaqvVCsMQQjCZTHYm4zRNF4tFGkfbGBfnnDNYVVWappeXl5KkOI4jCMI2gLe7uxsEwbY1WJIk3dYghLZtYAHe39/d3t6WZW2YTprG93PP9zdFnbZajmGbhtmz7N7+yYAQMp/N7mcBhJwQIiAQx6Uo4Y0fY0kWJKVhIIhCRVEcx7GdtiRrfhTXjJc1YQDqhkV4mud5u+UmcWxbjmOYAmNREMdxHEZJV1d3et22a9d1kReFgICIUV1VDKHhZG8wnJyeXbitvqxMTbMNAGq32zs7e9//8HqxWNYNV+5XWV4CLE529lVVD6OE1FxSxOlsc3d3V4quKMm6IeRpQkgFgIAx5JRWVYUEDBsAJIFSnpeloSoQQoQVQzERVgCQAj8RsCTJgm3bx8fHs/vFcrmJ/RQDKEBBlRQA0DpZM97IsizLooghaSpa0SJPuKbqui6JclHGpCoURUGA2KZyt06vzj9Ioqxoep5kSZQyxh2npaq6qqq6JokYRbyoygwLWq9jFkkRhRtFURCkZZ4JGALW9Dtd3/cYaSDiW5iHLMuKoiRJNF8t66ohhGzzO6IsWZazZXqKIoIQFnVFyvrvTQaiqHLOiSRtGUoQCFvdIYaxjzFgjFNSVUVRVNt2yhqB7ZSw5XdtGQ8IIcFxLNe137x59d133waBF0VBu+NKkrRarShtbm9vMMaKKvkBwgIcjvoaboqq5hD7m7WuKVVDRFwxxtbrjW4ZlmUZuhVFAaXUspz+oHu8d3R7e5tGSZjmgEFTMw2TqZxVDfGiYOX5s9kyylKOUZrFYRjfrqZ1XWZ1fnp5ere8YLChlCiKdHZ++hd/8X+Ionw/Wy0XG4SkdsvNsoKXpN/rzcsqCfzvf/vbd+/fnL57jzA3Td2wjDgOZ+t5t9tWVDMnOZIRUMjr8+8YhYPeuNcdS0hZe5t//5d/KWFxf3cPAZCFMabc0bRgtfz+668tswUIaluoLmmRlsH60rCczfJyMb8oy7KuGufLL5989Om2qKIm1qPHx0mS+F54e3tdVVWn4xwdPRgMBmEYxnGKMVqu5nESNg2xbSvLUtKw+XqORQEIGMlikcUcwiRLBoNBXleh7/c7Xd3QNFHGkrjxvU2cVwC6tiNzjiQRiXSbz6+qjALaci0KGG1KEYtnF2fL+Wy9HvX7w6qq+v3+p59+enz0gHMehuHP//ZvVqtVWaZPnp4gBJIkBpAlSXJ7dnVwcNCWlTgtwySXDcfp9QVZ/eH1m6bhV3f3kFWyJExGXVEUR4PBztitypDTrCqqXBLSxEhiX9YcyzDzJD1985ZS5mi62PC7s4v/7X4BNXdnb/fk5OTx8YFrGrPl4v72htaVZVl5HHQcZ3l3peJhuJrfY/bpp59KgpimSZqmSRbblq5IYkVqWZYFWaINf/joadMwCMU/++/+5Xy+IA1qtc3T09OsLBRFkmXRtA0k4vningOKRYkxVjWUM2iYtqZpCAEJypxzjMSmYVVZqqouy8p8toyC1DAsy7Qdo53nZZ7nRe4LgpDGRejFL55+rIjaZuNtHWTLhVe3mjRNHz58+ObNK1LWrmV3Wi3TNPM0wggYprq/u2NZduCFZVaZmqmr+tH+kb8JNiuPlLUoiookm4bprf2ri+vQjz7//HNowUFv+NVXX4VhSAjp9foCEtdLvyoaXbUMzQYM397ctdtty7DHw8nezm6SxP7GS9P0k48+CoLAD0MI0La6Pori2WxxvLPv+f7FxUVWpqZtcwA1TVdVdTabFUUlCEK31e622r1WW5Wlsiw3s5mumxfvTiVFBoTFWb4FD5c1CaPYC4O4LHd398+vrxFCRV4NsVTX5aDX6Tt9y1AhbZLYmy0Xmiz9x//8n0RRVGTNNJ08rk/fXzW1sHswHE36P/npl2ma+mufM7LZ+IIIb6ZXdV0WZVbXdZZloiL2Bv2iqrKCMgZ1w9QNx7Tamm5RSpIkuXn3PksL1nCEhNnd7PTde0ro4eHxcLA7Ho/KMvdDTxRQ1VBFVEzD1k3NMh3HcbIiz7JsPJy4bjvP7/M8f/XqlWXaum5whnkDTNN69PCZYznffPPNeu0FQWAbZqvVStP04OBgd3cXQhhFkaoopmnGcTydTtuOm2VZURQYoi0bzXXdw/39oijCNHv06NFkND4/P7+4uKCUuq5rWnqappQ2AIBer2Oa+mq1SONUU5RGEPM8FwXZcRzTNLMsy/M8CILtPhlj7DhOp9PZipWSJDEkpmnKQVNV+XxxHwSB7bZabjtOc1lUev2JFwbT+4Xs57/7k9/57LM/2B90qqr6Fn57dnbqeWva1O22KyimLGNBEhgU0pKkWUEZ6XREQVFHO8dN05DLi6KuijCBgmg7js7Rdu2xmM1nN3eQwzSOq6LUEXYHnXHLdBSRlulqfl/XpeM4uqFijBlAh8cP8qyMk7rb2xMVe+lFvXanqqkfx1c3U8Np7+0fr31/dr/oDYagoKRBSJAVw0nTajabvX//vr3/eDIZmYZNCCmqHCEIIaob0jTN1tUoSQqlvK4oNCSEUJ1hVdM1VeCA1IQxBkRBVmRNVfT9/QNSsNlsFYU54kJTc0apIgsQMs4IY4g0JaeSaZqdru37PhYo5zwJI9u2h+NRWZYNrXudPudwcT8DSCIMcChiLAEORFHMsoQ1lSzjwF8tV/dxNEewRKX44cMHTdMIIWmeSRhLktjrdTVN3a4NZFneQqMRQoZhfb63u+XoNIxuD6KMAYzxarXSVB1CWKRZVRFJVbazxRbYACHEEKiqIgiiLMuapi2W6yRJWENEUUQAsKbmnItY2y4Stu6ELV+VUgohFJ48eTKfz//1v/7XNzfes+e7URRd38CHD09kWeKc3d5d27Y90SfT6RQANhwOHx8eMIBEWTHdliQr78+vXr56PVutAAC6bmxH3TAM0zSRFUFVdzeeTxm3Xdc2bAwFDAUG+Gy56A938irP0iIu0jAOvDCSJGmw09872WOsqUmlKIJhqmG0ub4+j5MojpJXr79nFGmqhaHQEJqnRRKn11XKavLqzSuEwHq1mM2mpqEfHO3LMj44OPjuh29upzcVKRgYUEZs2954y9ls2e0MRVUq6+L85spUrLquJUVQRGk5X2xm01639ezJwygKsiBURXp/NRMliUE47Kl5VYpSmaSL2XxBCM3y8ujBBOCnZZHOl/PVcnNz+n4wGFi2MRoPtkqYZTpFURBSiSKWZdn3N7qup2l6eHj47v2bZJ2/PX0vKnKr33707CmUBKfrWrKIZHHlB+en70ldP3nwwHadPEmvrq4aVd4/efyHf/B7pMjrKk8CP0viKPBtx4rCuCiSu+ur86vrnd29IIz3dsZlWUZRRAh59uzFwcEBZ+Df/tt/++7dO9LUkiTIsuR53sXF2eXV+cnJiSQJjx4/2d3dFUT5brYpCeuP2hBJpxfX51e3Rwe7FEDLsDRDPzl51O+1DE3d3+nF4ZoBXJZlkSe+t9ws13YbnJ2++/ijz0zdWMwWTVlLggAhCjee4qLV/N617aMH1qDfzfJkPluul/OnT59en3u2qd1dXT0+2uekvr28+MmXn3W6rm4oSiTohry3v9Prd1brNeeQASRr0p/8yZ/Iso4Eudvp39/Pp9PZOoyTIhIEwXbMNE8Zp7IkiZJkiVa312OQ0ZubhtHhcPzpp58eHh6upqvZbLZcrmezme8FnU630+7leakoWl2RoIlMzVRVXZbVLM3LspREtapIrzewLOfi4ooBruv6xdk5FFBZpf/9//VfSpKwXi8ZA6fvP5CmkkQsCEiSFMexVFnd0HVNSgBMWZYNXc/TwrZtpSMbmr5t6Hn04kVVVR8+fPjiiy9ms9n9/f319fVHH33SNM3Jycnbt2+vrq52d3efPX1hGMarH1677ZYAhboo0ygts5wxJsmCifTFctYQZhmmaTkACIEfiaI4mUw8zzs7O/twfi6rChIEQpuyLAEADx48ePzo0aDddWyzTLIkCOMwCsNQluXJoL9YrBDGpK7TKDZsy7btXq/nh5GgqIKs+HF0ez/r9/vtfpdDFsXBoNfaO9j7+NnTTtt+/f23N1eXN5dXddUwClxHb7f7nIE8qzabQNZBXqQ7+yNG+Kvy+1bHlWTB9/3ldNZutyHkAPE4S1GJdzgvy3IxX5uWPt7ZOdh/sLuzn2bxfH5/ezs9Oz33PY8SNumPOeeUsvPzTRYn+QPS7/d7vYGqyRUp54t0tpj5kT8aDTVD1TRD181ETRVFCf3g1Q9vVqtVXTcttw0AUBRl1B8jJPy9vWt7dHMcp9ttbxWi4aivqNI2JiZJQlnmjDWuZXa77TxPEYR5nsuKaJhap9Mqy7JqwGq1ms/nt7e3AOFnz56ZphknYavVEkVBEITlfA4A2N+dsMmIUno5W1ZVZehWu92uqmqLiNgm1BaLxTahGsdxURRbAzhWpDjxdVOz7LbrOk3T6IZFKW23u0VZCrI01ixZMZAANa2V52y+CieTyZNnH0uKdn5+tt6sNN02TE1RRE2VNV1irMlrImCsaIbtthc3IRaEivCNF5V1LSkyluS8qCgN9nb2t72FuqYpABZY1GTl8bPHqiaLgG68tbdcIBFZps6YbFsuQNB0W2/f/Xa+9g5Pnhimu96EN/n9gwcP+oNJtz9QdZMw5vmhYdlYUM4vrwRBOjg8VhVzsQz8MJEU3Q+i8WSi60aWZQAgDgEAgFKKRAEAUBSVJNYIIAjxdlMY336glKdpzJoqyyNdVwEAAKDT09OiqHu9AUaKADcN4RBgAJimKU3TcM4AJxhrtm2enBzv7O/9L//r/1zkDeVNVReTnYdPnz5++/btPA1+9NGnjuPmRXVzN6cMdntuTenddHZwcEDrCkKuapJWShiyskryIiQRvrm93u4OBUHodrsQAITQZrMhTbUFdCZJEkWR5Tiqqv7m268AR1uD4bbNq64bzjmhDQSIUipJ0ni8M5lMHMdpmma1Xm/LybavfEXTtn6Fdsf1faeua0mRIMTbtYGqqmmaQgi3xsbtHUUIaZpGyPPyP/2nv+YcD3ouyaGt9dI0nd34Bwf772ZvLaM7GA4VXT+9vux0u5ssvg2cXr+jurbadj5cnK+SDYG81RsO+juk5rqiV0WiCg6ElY6l1Ftzx7y8/KAp9mRwcHL4jNV8MZsPen1AWVWkX3/1y9PT0ygpaANstyNgeVNMdV23dEuSJAGJnPKqYCzDLAZGywKQiYSaGJZNjtKkjSGj1bu3t3VWdPqDJFzKIjo4OBjtTI5PjvI8jqJMMyxRke2WvVjN/cTTbDQSzX5fD4vTi7uwzJv24OOdh3aZcsFSIm9TZOntevXg+VO916G0/n49293dr5vm/Pxc0XTXdess/cf/3b949f0PjIHzs4swTYIg2GLsZFHKcz8MwWS8c3K8kyQJrfMiA4iDNPCOjo45Za9+eEMp7bgd73b68OHD8+rdjz87+eHNa5nLJFns7O3KsDZa5nq97hpiM2hvFrcvs6DdahVFQYWmM9r5x3/6x8PDQ57nv/7VrzYlXfqpolh6q0/4/f/2H38eBAHnfPn2jaqqzXqxR/afPXtxcnJSFNn7969P37+/u7uzLEmTLQCArIj+YlFVlaNamZ90Op2d4/1+b0AIb16dVUnx5tu3T548O9o5fv3NG1WxRnuHqiFIw658MIBtsxRg+6OPq+n08vR9z3GK2PMX/tDtnV+/3T95MV8t50HojMdRFEiq7M3vOy3nX/6P/+O2822eZeu1N9zZ3T151Ol0rs7Os2p5efl2er/5+qvXP/29f/K3f/Ofk4jZMrg9v6qb5tGTp/1eO0rCvCrTrGAQTcY7HMG0SAddYz67rotKFujpu+86tqqoKkTEtGTKWZz6WOKj0WS9XpZl2eu3Tx4d7I4nk8lIEeBHX3yx4/tRELz85ttX330/Hg8O9vdvr24jb6VphuvYWVYUNTMtJ6tKRiBVhKDOaxG6k8EkP4qi6OnjJ0EQVFVV1vX333/4yU9+93Avf/Xq1XS6iON4f3c06T33PG92k0NY3F5ver2BY42KvLy4nq/mC0PTd3aPIIRAEC8uLiY7B45uAsZfvvxm43vfvXn56PnD3k6nLEsv3si68vjJk/3dA0MzLk8vby9u93f6T08OKOfz1Ww6nY73dyVFyYoizYvr+1vHcjnSlrOpKms7o8PLy8ub2er6fmm2u6KIHz5+ghA4P/+Q5zlnjabLgJPZ3e39zW2WpKSuszjRgdTRzJOjQ0FRl/7GNPXRZEJAs3e0v3N8iDB+9+795eXl8cFuHMeGKqVpiFm9nF932kq463QH8uGjUQPC6eoDLUleZlEm8wXvdHreLBh1xy2lBVT24fKCc9py7N7vfEpo/Ytf/KKoIy7UQRiIM9jrDkzbUZEqmcrOcYUx/unP/mD/8EgUxU5r8Obl22//7tv1/VoTdd1SOKe/+eqXNckme5JtW5vF9Ne/+GtRFLEo2LadFnnsVbFfi8DKslXDmeXY7XZ7USSX5ZpX6oPRU0EQ0jS5+nD17PkTVUC6rq4WV4F3ZxvC7G4NKVwtZ9PbG4SEqqiTOOCM1HWdAiC1WgKC3nqlilL/2bMXT57+h//wV61Wq+2017PVJVYVRVmvFrPZjFJ6eXk5Ho+rory7uz8+Pt7b25tOp2lS5HkuCIJhGJRyy2oNLB7xCFOIGtDk9fp+qWjqk5OHtm7M76dlWe7tTOIoWm8WnueNRiN7cGC4g4KAq+ms4kx3bYQRluThsL3YeFmW9fr9ycMDt9VxXff87i6vwt70+vc+//z508fe7Lq3O3jw4CDOwiD1vWC+c/xpGIbRej3sD9LYm9/hCkez+4WfLvqDNiHN2zfvg/VqZzwRRTePo45rbtvv3IE9Vnuu6+qi5PkeACAMI8PQRFEOg7TXnYhIa7ldzoTXb96Lil7UpL8zzpsmXi1VVX/79m2r1bq6udlsVpRxTunN/QYI9H59h5T68PiB1YZOJYi6ali9NJzdlB7Cgtt2hpOxIIlXN3er1Wq5DiRJib3AMIxet58SoktSEnF/E0kY6XrbNbvBmtcpJgW+u3pnmXqr5dYNqmitaYos09VqJVCo6Jq32Tx9+riu66u7s89/9wWp4yRYpHEkIvxHv/97O932WMKpJNxcX5Gd8YNnD/7i3//V1eUNFOXZ3eVyEyiamiernZ2dk6NHsoQmPevRwXC9miPePDp2WdZ69eZUYAUCSuBXmq7Gic8RlxRNVnXXbUVJDjhyLMfzvCanuq5UReUtNpxzjiCEUJZlTVMNw9j2uRPafDh/X1UVpRQiBWMMALi/u9V1vdPrOo4jSZJhW4xRy1I456omd9rjIAjupxeQI9M087yscgEhxBgTMfZ9T/j5z//6w4d3eZ5KggxgA6GgKKLrWs9fPE6L4Oz8dHpfHyi7OztDp+VEUfTh/P3L71LCyGgyTPPMdruj8UA3WpbZCfwYACAgbW9vT5a5oYm+79/c3FRFY8quv/ZPy3eT0WR3MhoOB0G49oPV3fx67t03jGMkwwI1Da1BWpM89DeEUMC4iCXEEWsob+iyqSAHuiqLIuascV2n3+8jWM/n87KoEWeEkCxN67qUJAEwSgiZzaZJGLfaNkKCaRhxkrT65rjfEUXxze3U2/i7o/3ReDCfenkc27b56OnJbHp/c3Px+vR1q222O7YkiAihTqtVjMc3d9OXL1/mWfHllz/+9NNPWcNVQREQbpqmYEyVlcFgMB6P0zTlgB0eHeVZ9uqHN57ntZ12u91mjJKaNE1TliVvQ9XQOYKu63IEHz5+tHd4oKgqEnCe56RpsiwzNd148IBSKksShHA5myOENovlxekZKcokSdbLVZqmmEMBoMV0JiL8yUcfn5+fB0Fwd3fXdlvtdvtnv/cPZVn5xS9+dX19DSEEjMqSqmmaIglb41VeFFEYpmkaBLHvR1Gefk+/35nsI4R6vd7f/eprwzAcx/nkk486vVaU+bKGHMfZis1IQru7u6PRaH57QzmjnGUZkeXCcRzf9zvDPdsx67rEGMdx3DTMsKxPPvrYtu2K1NfXt7/4xa+2c25Zlk+fPn3x/Ol6Pvuq0765urJt+9GjR0EUOi7c3d21XXdnb59D1IS+JElyw+Is3x7+IMSKqACOBFGUZdl124ZhKJpKaJOWBWsqSVIFUWkIVWTdNBzHtQ3DUGSjLBpS53eLHxAAW1XY33iuZe+OxoPBYDlfdlrt8WTX98O7+8X9dEoZ6HS6e7vDTqfz4Oj4X/y3f/abX/0dQqjVatVllaTR3d2da1mwYQJE9v+/pzNpkiO70uubffYYPIaMiByQyEyggCxUFQE2VVYkW23G3rTJ1KalzPQv27TRRpS0YJNdAwtEAQWgEjlnZMbk4fPwBn9aeFvvw2IZcd+95/uO622YFfMIQZKlxXKxrry61+tZllOW5dnZmes4Wus0TRupsiwzDKOVCkrJV6vVh7NfANJ7B7tffvmlEHyzWa9W6zROHMf7zcuvDw8ef/j5w5s3r6+vb3/9m1PGWC8IANZxkrx5/TcuBTNNz+uURWFQsxHSMAzLsqSUknNqsP4gwIwQgoRSWZpCgpnFEIGb7fbhYR5v4u16QzC2DRMb7Ppqbt90jY4bjHeiOH5Yr3NeWZ4fDEZ7e3tNA+5v7oJO93C2F0WJEOJmu82LjDFkUqaUXj4sqzpn1PzyxVeLh9Wb1z+VLB8HE9/3RS1ub68blc9mk9lsVtellBwiUItqOp1mWZZXZZIAx8nNPXMymQyHwzwvT09PT05Onj9/rpTSmqZJslgs2sN2med1LUwT9no9RqjW2rbtbF06jgMAaF9NhBDbtlErACzLSvAG6PbATxhro4ZxHOd5ppQqy3I+n89ms8lksl6vWw68lPVms5JCd7tdrXWS/LuPmHNeFEVbt6yVPjo6SrZR26DaNA2E8OrqSghxdvkpz/PZbOa6dlnm5+f3SqnhMFitzIeHeVmWVVUxxoSom6ap63K2MxNCFHWllAqCYDqdzh/uv/vuu1evXmGMW5NQURSt+eXq6mqK3OF45DhOVXOkgUZISpkWZRht86ICGDmOc3h4eHz8pNfv+77/x//zv1RZ/5W8plLkWe07VAitIbm7XzkdByHKpSTMoIRtNuFqsaE2q6qKYeL7vmlagquqqNtrJsPk+PhYa60EhxC6nm3bdr6N0jSLoqiu6+l0t9/v390v3r1798//9b+5vt++WZMkybLMcZzj48d5r7sON4TR0WgEEHr3/ue7u/v+ILAcp67r9XKpte70+hBC3/MAAKvVajgcdjodpZtgMPzyq6+UbjbbOIqiFjgt0izLMskFI6QNdNR1ySFElDjYQAgq3XDOwzAkBPV0T0q52WwWUk7GY9/3i7pqFLRMu8jrNEsAAMtFOB3vnJ1ddj1XoYYyixC2jTKAmJKwSIvzs7MsTTFEdVGZPfdwb4+a5v7+fq/XjTfbPN3OpmOKEFBNrxsMh8NBMPL9+6zkWZZVdQwwM520USBOs7woEcJCiEpwwzAePXo0mUwsyyrLstVVV4ILITDGXsevqipN0+VyqXRDKbUsy7Zt3RCMsWwapZTWusjyNkk7lFIj+B/phjY74/t+VdTtYNE2h7aMAiGE/PDjX7KM97qeY5kUQ4iEYUK/a7z44jPHp4OxezO/bUBp2rAX2LaLf/z2B6XUJpLnVwkm4L//jxf7+8e/fLopykwqDgBQUhJCHIcpWc7nc5FWlFgJjIpkMSf3e6Pp7mxqWOQv376VoKp1SmxJEUEEWi7gErrYb6Qqy1pIIRsFoaaEIAipyURVUkIaKLngSgkhLYQbz3JDygjGGGOlVF2XZZWLuqx5admmqDkhmGGqNdwZ71KyGXbYcBQwxniedV1vb/dwGPQ931kvtg0Qw/EgybZmaJzdnO2CMXFglpWNVIQQ17afHJ/s7+/P7+4ZY3/+858fPzqaTqfj4Wgy2om2W1Fz0Ci/67UNykJy23G8jus4zvHxkzAMtdTujocIXt6v+v0+xujy8vzZ8ydCCGYa4/F4tV6LUpqOTSl9eHhocTYlZVQUeZJeX15dXl4y231t28vb2+1228ZMLMsSQFvUFEL0/d6oP5yOJgZmw2DIGPvbmw9Znrx79y4Mw+PDx3t7e67nuo4ThiFjFChd5LLmDaEWQVgKsFw+pGm+M54Nhn2GTS6qzWZ9d3eNMEhShBCYTqfPnp32hzsASS04QHAymcx2dy2Car84O7tcblYSGoA2mOHJZOfi+jYIgsuLiBpkNpuNRiNimsnNTRLFoNGu7bS/oWoyPdidDcej0xefn338aNrW17/9Jg63kKTdfs/v9pVS96uHqhZepzvYcbqdXpLlRV4VWVlUPE/SzWazWCwpsW2r47guF0IjagJt2pZhGUVV+Z5CGCCE8oxvFtecc611mMZHR0fDYPDk6Ljn+gjCh4eHeJt0Op26rpeLRV5USAPGWF0JzjmDnoHY5mG90wuKJNVaX5+dSSn7QY9qoKW6vbrO8zyNYs+wjNmu1rCuedMAy3SenHzW8X3Oebha86oO+gPYQIKw1ipN481mtV4vGfsVQmizWa3Wi+efP3v8+PDNmzcXV5eTyQRDtLMzPXi0Z9tmVZWmaT579lRyUZZlgFAQBBqAeH5X1pXrup7nMcYk53EcI0Acx3EsazAYVABanisEb4AqRXl5fQlII6oyykBV5qjRSjfYpia1mGkiZvzDP/0jNVhDUCmqVRR+/+MP+vVfEcH7e4/+/u//YTyeGBiPu0EQBKQB19fXh4eHd/PbRjSLxRK90QDKjutMZ5PHh8ee3f3+2x8Wi5XndKFGdS3Oz88ZfYEQ0Ai6rq21Pv90bljs66+/dhzn8ub6/NOKc66UAgDEcbxabb76xz+8fPmSmMZ2sajr+v37D7/88kuv15uMn93dXK0WSwghowTq1sUF2qJ+pVS7lOZKKqU0AEVRtCnu9gOc845lBUHAmibPc9O0LMtqmubDhw9lmX/xxRdpmta8HI9HhNA4StM0p4ykaYxJYdu2ZVktIu66bp7nSqlPnz4t7x/af/F2Ibzdxoyxly9ftkbpNmnJOQcA+L7vum77JS202LY5aa1vLm92d3fbuHUrFkIIua57dXXVTtjv379vYXiEUJJwzjlBmDHGawEAoJRqAIQoVps1FwpREkWRlsp13W6nYzAGNUrj/P3bj1SrOs3qHGXFR9lwvzs4ODns9MeLddTtDDyvt7hdLheLp8+ObdOUrmtZlud4k52dbRgvFosvTr8YjUZpFv/w3fdNI23b3t37ajKZ/O93vxwfPzk4OCiKIksLCKHfHRDM2oaf1iYAQCOlbMMjo273X/7lf8Zp2urgw2jLpRqNRq7vR1F8dXUVRVG8jXq9nuM4UsqXr46ARllZlGWFMHRdx+92X72qV6tFXdtSSsVrAKAQQnKOEBoOgzzPhawxgQq0NiLORcU5L4qq5pyZhm27m80mKwrTHmCklQTd3rgo1f39Rin1f//fvz45Prq7W5F906JAcKgBExx4br8XjOY382gbb9YhxUYDtYlQv9vBhOmqjpfLX84+rJeLl6++pAS8efO63+1O/vm/aIRtx3OcuihFlMRFWUFsYMriOAYa0qdPHdc9P/90dXvjuq6WqtPpEEJ6vZ5lWe2FxTTNMNpuNpvtdhtFUcVrxpht24Zh+F4PAEIwppgQggFspKi11kkStR5tDXHTSAS1ZbJe109RaRhGWZZKNe2sgBBSShFK4WBgT6cTgnCZF3VdAwA1qAlVxyd7AFd+YK7DVZxtG110uvbz0yPDMKI0up3fVVw8OpyZJj379F4r5nuDXq9XyTpOthDaBgUQQpe6Qsh8m9pWJ+gPHx8+Mgz69qe/Xt6eTfcGuwcjswvTsqqFdDzqIEwlhhBK2fCSK66gRo3UiktKCGeo43kIISlqgoCGzWqzTAEuyxJjiBEAEDKCeFU+PMzzPHl8ciylBFqvV2HTNC9evLCYHXgOEg1G+vnRk1U31A2ty2LQ74T9LCli5pgSysmjaVZsoAWLprRNs67r7SZMkmQ4GO/u7kour69uj46OdidTBLBBabTdbjchw8TpdI6Pj4fDoRAiTpKWCkEMcFl3Op3NZuP3Oq1by3XdIq8uf7k6+eyIGsxynbworm9vbM/tDoK2VpNznmfZdrut8kIJmWcZBIBn2eLmpqmqMIwGg4FslCa0qnmvG8y30VJIg9DRaOSYzna7TZLk3cc7kxlSAMfuEGbnZZ1lRZZVRZF5josxVhq5Tsf3fcuylFKb5Kbf73e7PsNMCXBycmgY1jZaFUXhp65ohO+7s9kMmBaoMimbRqiDx0d1ntVlJsoiTdPr+aYqawAwhM1wPJgvFi++/JwyzAj55ne/10otb29/+O67zXrbPscRQkWalWWZ53ld5q7v9wfBzfzuxfNnmFGIDcvxGqCvb++ubm79fjA7eNQfjKlhcqGlAMgnfrdTVTVXquQ1QojXABMplFISE4NZpms7jga5VDxJkjgO43ibbKPW47J3tEcxSaNYCbm/twcbvbh/wBh3PX+13MxvbqMksx1/OBxHOkvjBPUGTa1EWc0ms8O9g6oukFagkXd3dwihJNz8bbWazx/KkjOMEELaohBpzrnkajye+G6nUSDLMseyu92uyRiGwHXd5XK53W6Xy+UmXEGkW7A03G4gxtc3l5eXV71e79HBwe7uXhxv3/3088XFRbfXOXp8fHy820pMDNtwHYdhskpShFCvGxCIMMaNUqZlm5RJLizD7HS7hmkWZaqAiJPwfjN3XTvLYyEEhqjndzzfUpXDiNH1upTS3/7n319eXt7e3haaQ4Z6QX+z2dzdPFyd32VJur97UJalYzomIV9+/uKb//T1n15/r5VaLhe313eL+wdRF6PxQCvQcOW6Xsf1i6xsmoZi+Kvf/Hq73c7nc9NkDQTtRuHu7s6w2Onp6e9+97udi/OyqJMkJYS08qTVatNaEnhRYkwf7hd//OMf3737+dWvXhqGQTBrGsB5WWtQV1XTNJRSy7SVUk3TQIyklFLJ9rVEKZ3NZn6va1hmXddVVRmWZRgGU6Db7bquSympefn27Vut1WQySZIoSaJutx/0h21Xne+7jRae1+/3+67raqlbG0un08nT/OzsbL1YGobVggWe41JKj46OsIkuLj9dXp1HUTSdTgfDfp7naRY//exkOAqU6jKDzOfzssqLouCcY01t20aUtGGWJ0+ezB/uW3bEcZx2tjAMYzgcGoaRpqnvuFLKJIqrqpZKUQghAAahFBMldZ5mF+fnEELGzOHO+M2bN5vF2iCUGSDo9YZ7B6aBMUXMYcFOf+/xAW8qfH5JfYYwraUSsoEaOJYNGii5iEQEIex43nazCcOwLIqu36mqCkOwWa2vL6+WD4u6FnUl8qxGiPb7gyAI8rKez+dZlhuWVRZ1W5naAvaMMcyo0nq5XFa8DoJgMplgytI0dV3XNK3WvSmqWillUGabVuvvPjg8/NWrl72gf307/9e//Pn86hIAYFmGaXZ8x1VKUUKqvACgGU9GbZNmo5USknPeNBopRQ2W5lkYhrPZ7OTJU3p1kWXZOtwSbDVNM50Nw3AtFaqq6tt/+2sc5YPRVEmgKb2+ve93BoN+rygL2+q+/fjaNE3BVb8/CAZDjGC8WtmOl6zXRZltNisMAdWQaJRF6dWny2EQZEWhNOr2AtmQMC3XUVJWhWlbUjWYEtO2nNpugF6tVnG89R2/bdaCEBZFwZVsg7u9oO84zu7ubr/fz8uidXMAABCAUAMMEaaEUkoJxRg3EDRKEYxN0ySEMExaLoEx5vu0HeBatVjTNP/usNjdm7UMRZmVVZlroKSUZZktlndc1rd3F5io8U6PRKrmGYDi1d99vt1u+9w5eXawXG2kKm5uwzgOEbR6vcDznUbylv3xPHs4HGbXW6EAJvBXX3x5cvL09PT0/OLjj6+/B1CZFvWJWWtbACFAJUGlVFMVkFKDQIQZhhACBZpGKtDARkKC/G5HCQktYzweZ1kSheF6ubBtkxEqpWwaZZoG1M1yMS8r//DxAcWEUVOJJk+qRiEIqMWsON6uHlZuxy+ysuY5hnbH7+3tIYgQ5xxRtDebhTHu9hzDYg508yRVXGjVPDw8XFxc3D08YETD9VrWMgrjLIqTKOFltbe3NwxGh8fB/uG+krouy6IoIIZZli8W9/3+4HZ+BwBoGrBYLzGltmNShj+c/XJycmLZdlGVWVl43Q4hpMVPOOfRdhuu1lo1jm33O12GSRLFBACtJMXQtU0hhFZNkecIkjIviqIaDAZJklmOd3VzBxCxLcf3fcO0hBC260CIFdCYGoahqGFCCIkQjDGv02srWnEOGaNplsx2pps0Ony8XxQV5zwYdCEGsuZVVdV1TSOFKaKGC7A7HAZFsr34+B4hMNgZNBgtw+J2lcdZiJmDGTn94oXruo5p/vab368fllESJ1EMIZQ1f7ibB6Nhv99vmoYarCxzKeXzz0+FksRgxDSqItIAykbXQmoAbMdhli0bFa3Xq81aysYyHYOZlDHbcvu9QZYnQkgeZRWv60YQg3GhkjSvpdBabzZhGIZSyqZBtu2Ox2PbtOq6jjah5MLa33cNy/M827BFLaaTSZpVq+Xb+2jOmIkRoQgv7ldJlJwcH7u2YztmnkaubXJenT77TAhxfX3Ny4oiTGyrKnkUy55jWpY1HDiM0M1m67uruq6XyzXUwLZtg1LPcVs+2bZtz/Nc193bm52eno4n4+l0p9PvXV6dY0Lm89uTo+NOp7N6WH369Mvd/G4ympiW4ZjWw908SpPZ3nQ8nUAAqrzQWn86OyuLejLxbdvuur5Wze3dnRCiBDgvUqUFswjCACFAGWoKiVCDCUIUyUYmRWab+tEweHTw2Ov7bGVUvORSOI719LOT1UOvyPJRMNyfznodzzbNJEnevn3r+/5wONyd7g2DQRRFSkmKdV6kJmMdrx9H+WQ4Gwaje34PlASgefz4kainf/rTnw4PD9yOv1jcE4J2d3fX4erDhw9/+MMfmGVK0VxdXdeVoJQihDqdjud2qpJblmWa9sXFxesf32w228VisV6twtU6TeKmaYBqGqnak61LvKIoAADMNLTWXEkhhJAyCIKDg4PeICjrKk3TpmkqzqMoYgpQaoxGI0LwNtowRhBCSgnLMsoyb/Or680KQeL5Vkslh2GYZVmyTQghnU6Hcx6uw3ajPhrZSqntdquE1BpqrdsfX611r9f75ptvGGPX19e+7y+Xy6Ioer2e53ntTbCl1fr+AAAwn88dxzk9PT06OsrL4sOHD5TSXhAMBoN2ET1/uK+qqh330yiWUhLDYIZRlxWXQintmBZGlHOeRvHV5aXr+jthePbhY7haTkZjzwkGg8HB7q7r2Y5vBZMhYMALvMurCwkgQ2CbxFmRQqhXq1W/PzBNU3GVJLFtu/1+vyrK8/OLvd3dZ0+f2KblunYSRWkU32d3sDE/fbr66af3EML9/f1fv/o7w7aiKB6Nxh2/t1qtdnZ2AIJS8vV6KSU/P+PXtzdREleSM9M42jnKy+rnn39WWg8GQ89xy7wQnNdFafjUMa2OF4RhWJSZECIMw+9/+Pant28RZQAAAIFpmhRjIQRodA2h1rqWHFFsEquRvAINAZgR0zCMONlmcbLZRtPdvdHOJIqizTos8tKxkdINJgwi1un0fN+/vLyUQj1//vnZh4+W6Vxe3T3ae7SzMz0//zFKktVm63leWdZSg6dPn2VpfnFzsbOzUxTFfD7vdDoHB/tBt+f7/snhSbgM//Jv31uW0w36rtfzutDdbMk6EqryCDMsohGMoqgW3Pd91AWGYTBIGWPtnYtzvgo3bWvn8ZMTz/PaPG0teFVVbfsybgsZAYAQgEZJ3jQEI9RepYBlMkopIhhD3TQN1IpSo92N/UebQpub+P9mz0Fh6kOt6AAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Show image\n", + "Image.open('img.jpg')" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Preprocess image with TensorFlow\n", + "tf.enable_eager_execution()\n", + "\n", + "# Constants\n", + "MEAN_RGB = [0.485 * 255, 0.456 * 255, 0.406 * 255]\n", + "STDDEV_RGB = [0.229 * 255, 0.224 * 255, 0.225 * 255]\n", + "CROP_PADDING = 32\n", + "image_size = 224\n", + "\n", + "# Helper function\n", + "def _decode_and_center_crop(image_bytes, image_size):\n", + " shape = tf.image.extract_jpeg_shape(image_bytes)\n", + " image_height = shape[0]\n", + " image_width = shape[1]\n", + " padded_center_crop_size = tf.cast(\n", + " ((image_size / (image_size + CROP_PADDING)) *\n", + " tf.cast(tf.minimum(image_height, image_width), tf.float32)),\n", + " tf.int32)\n", + " offset_height = ((image_height - padded_center_crop_size) + 1) // 2\n", + " offset_width = ((image_width - padded_center_crop_size) + 1) // 2\n", + " crop_window = tf.stack([offset_height, offset_width, padded_center_crop_size, padded_center_crop_size])\n", + " image = tf.image.decode_and_crop_jpeg(image_bytes, crop_window, channels=3)\n", + " image = tf.image.resize_bicubic([image], [image_size, image_size])[0]\n", + " return image\n", + "\n", + "# Process\n", + "tf_img_bytes = tf.read_file('img.jpg')\n", + "tf_img = _decode_and_center_crop(tf_img_bytes, image_size)\n", + "tf_img = tf.image.resize_bicubic([tf_img], [image_size, image_size])[0] # ok it matches up to here\n", + "use_bfloat16 = 224 # bug in the original repo! \n", + "tf_img = tf.image.convert_image_dtype(tf_img, dtype=tf.bfloat16 if use_bfloat16 else tf.float32)\n", + "tf_img = tf.cast(tf_img, tf.float32)\n", + "tf_img = (tf_img - MEAN_RGB) / (STDDEV_RGB) # this is exactly the input to the model\n", + "img = torch.from_numpy(tf_img.numpy()).unsqueeze(0).permute((0,3,1,2))" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# Load class names\n", + "labels_map = json.load(open('labels_map.txt'))\n", + "labels_map = [labels_map[str(i)] for i in range(1000)]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded pretrained weights for efficientnet-b0\n", + "-----\n", + "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca (82.79%)\n", + "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus (1.52%)\n", + "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens (0.37%)\n", + "American black bear, black bear, Ursus americanus, Euarctos americanus (0.23%)\n", + "brown bear, bruin, Ursus arctos (0.17%)\n" + ] + } + ], + "source": [ + "# Classify with EfficientNet\n", + "model = EfficientNet.from_pretrained(model_name)\n", + "model.eval()\n", + "with torch.no_grad():\n", + " logits = model(img)\n", + "preds = torch.topk(logits, k=5).indices.squeeze(0).tolist()\n", + "\n", + "print('-----')\n", + "for idx in preds:\n", + " label = labels_map[idx]\n", + " prob = torch.softmax(logits, dim=1)[0, idx].item()\n", + " print('{:<75} ({:.2f}%)'.format(label, prob*100))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This is the result obtained by the TensorFlow implementation. " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/simple/example.ipynb b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/simple/example.ipynb new file mode 100644 index 0000000..6af47a4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/simple/example.ipynb @@ -0,0 +1,144 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example\n", + "\n", + "In this simple example, we load an image, pre-process it, and classify it with a pretrained EfficientNet." + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from PIL import Image\n", + "\n", + "import torch\n", + "from torchvision import transforms\n", + "\n", + "from efficientnet_pytorch import EfficientNet" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "model_name = 'efficientnet-b0'\n", + "image_size = EfficientNet.get_image_size(model_name) # 224" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAArgAAAJlCAIAAAC+AhaJAAEAAElEQVR4nHT9WY8sW5odiO15b5t9jjnOOffcIe+QebMqs7JYlVVsNptsSNCLIKEF6J/oTwgQoEe9S4AgoNESSUENkt3VLGZlsoac7nzumWMOn2y2PethR/iNvEXaQ8DD3dzcbJv7Xutb3/q+Df/P/6f/Y1EUm6asu5ZFwgD06PG7/+K/+d8cn74DPAUIDlcX//DLv1ovz/IEY+icQHGUZkVuPeyGASCkrVsul33fE0IAcNPJeDTKL87evH39SgihPdlsNqPxeDabHR0dLxaLq+vby8vrpmnenF84C6qqghDFaSKlXMz368327Oxstth3AK5XmzdnV0LEt7e3QsR5kjJCOaGcEaM0Ai7P85//+Z+cXZz/q3/1r/b29tqh18rMFvOLq8tyW3/4ycfFePTs2bPxdP7+++9vt1vC6NnV18bYtu44i7S21oA8LZqm2dvbqzbbpq0Q9nmaZFmSpJEQnFiMCL68vFxvN0cnJ6ePH5Vl+fLN6ydPnv7o40/evn17cXb++9/8djGfHx4eVtty4rhzLs/zOI7jNGKMNX0HADg42v+bv/mb6WL+L//lv8xHmffeObdarX75+3+4uLiICB/l4x99+PHl+dV//9//DwcHB4RRC0Fvhj/6059MDhbZtPAEAgTXy9uubRlj3377bd+0R0dH2/Wm3pZaa2etYJxzTiAihBhjyrLce//Jp59++u/+3b+LomQ+nX3++Rec8/Xt+mD/iHMOHKzrZhgGAJD3Xmtdl6brOkyJc2a13QAATk5ODg4OxqPpxx9+SDHbm8y6pv/rv/oPb1+/qaqKzMef/vBHl5eXRZZ89rvfz8aTvb29YpRfXV31fS+1TvNkb29vtrd4+frVL37xi+OjJ2kae+/Wm2VVraNYzOfToihubm6urm7KbSN4GseZdxBCwqhQ8poQMpqOEIJlvY0iHqWiqipljIfIGm8cpFzEUc4YBwCgZAAAOOes8cYYpYxW1lq3vLkVIqaUG+2UUt4DSimltFuvkiSJ4xhCb4xxzmEEEELG6JgLiPwwDN6aoigWi0We59ATQoj1DgDw9uzs/OpSRKzve4QQxrirG4rJ3nxhtSw3W875wUG62WyA81maCEqSWBRpRjE5OTl57733Xr9+e35xVRSjQZmLi6vzy4s4TrMkff36NbLeWfv45NF//V/9s8vLy9/97rNtWW7r6vD4qOr6LM9ne4s/+skff/v8y81mU9e1994YI2WPEIqi6Ob2CmMshEAIMMYWi0Wapm3bwsYqpYz13gOlTNP2UkpvwXw+F5yqtgfOeGv+1//yX/7yb/4jhBBCHMfxer1u23pvb288HkdRNJ2OvfcvXn57cXFBKbXWnpwcG2MQQhHGP/z0x998803b9cenjy9uVtc3twCAvYPDsixHebZYLA72987evJJSDsPQdY0Qwnv/5uwCQjiaTLXWIooJZ6v1tus6kaQHBwe//d3vwt2cF6ht2+Pj4yiKOGVZlimlpuPJaDRqmqZcb25vb733x8fHo9GobZqX1zd1XTvnFosFgoQxNhqNrq9vvPe//e1vx+NpFEUffPDBe++99//5f//rf/Nv/s0HP/qAMtE0jYNIG1d3PYCoLEspJULovXef/ulPfzIqsrPXry4vL4G3xfS4aRqttfceQpimOaW0ruu6bouiuLy8xIimaeqcG41GWuu+H7TWWmvvHaU0S+M0jaMoMmrQWjnnvDNd122362EYIIRphIo8PzjY04OczWbWKELIarnBGFMublfbuuny8QQi1g9SCKFVixB6/OjR+fl5GgnBueoHa21VbYui+PrbZ9bbKE2qquJCAACkNAAArXWe59PJfLvddl23WCwIYQAAIQSEuGkaay1CSCkFCEYIaa3btlNKEUKEEJRwjDGEGCFkjOm6bhgUpTRJEsKdcw4AQClljHmIwuW3/UAphRDGUZpkqVKqbVuMsdceYwwhlFJWVdW2LQCAc44QCscBAFhrrbWUUiFEnKfX19ec8zzP67bV2hBCejlAgKuqqtrGGAMhBgBACBHGnDHnnLUWAIAQghCGYxJCwl/GGADAe08I4ZwTdvdeIcSomKRpGi4QQrxara6vr5umBQBgjO+nHei9996HwzrnjDHWWu89xjhchdbaWgshRAh568JEFI4QnsQYSymNMRBCQgiEMAwaAEApRSkFAFgtvfeUIO+9kZJRzCnjnApGIITAeeet995jrLV2zkEIwxQRjrk7w92HQgghhAWGSqko4pwzaSShKMsySjHBmJd1M0gZx7HD8PbyhvGk6xqAoGlrp3Tdbi00ZbMtt52I0HRSSKuQV8b5bpBERABiCGwkKEKoqtrXr7dXF7gsN5vNhnM+nh8654zWQzfcXF03TbdarZq6W222ThuMqPdAayVXUg46T4pw5+q6HgZV1U1XNxDiKEqg80IIb13TVoyO9/bnVumuaReLRd02URQBABhjo2KcF8Vqs/a+Ojk5oZy9evUqz/Nw+6fTKaaPzs/PL8pLRqVzoNyU4MAfHh6O8xwBB5F11njvh2HwwBpjgPZt2749P2vb1gFAuZjv7/38538phIizNE1TCOGPfvSjruuklEmWwrrTRi43Le/4ITscz4s4F23XYQx/9rOfxlmqtfz22+s4jq216/X69evXt9c3o6zAkDjnTk9Pf/zjHzPGPvnRDwHB5zcXx09OJbBqkIiTtu8E533XSSlnsxkYT4QQGKIizc7evtXeU0oXi8WkGJVl+eybb16+fDlgzRC8ePPaGHOVjdbr9Wg0cUq+ffVtkY/ztOAYYs611l3X920LveAU56OiGI+OpWyHHiFUbus3L9+sb273ZnvL+UJL0zQ1QihNU2uH8ubsaDY2Su9Pi5hToPsIFxGGGjoL7e3lRVlt3l68zUfFH/3Rpwjy9XpV1VutpfO26xrnitlskucpQsjoc+8dAA5AqJSUUiYiOj46StP44vKsLluCaJqyOMpNVdVtu96UwyCjJJvPbVEUhHJvDaWcRzHG2GjX97JpmmFQSZJgTDEAHnqCkPceeueMFoJhDK3V4fdDMAQAWGsixo1VTjuCMBPce//27du+7z9672MHgRAiHxVJkuR5LiLmnAvTCifYGauNVMOAKcnznFKQ54k1SkQoYtTYYVvKWAgP9qt6XdVrDzSAVuneOkkoMMYABPM854TKfojT9NWb1998803TtgCA8XjMOefWee9Ho9H+Yu+bL37PEM7jxDnXdd2gnbISOR/TyDlDPOSEC84iyiLKALfr9bptW20cIXRQpu5apRSjQhntnPHAMYKNUW8uzy9urrMkoZgtFgvnnJQ9hDBMZwCAsiwZY0VRKKWurq60VlEULRaL09NTY8x2uwUQYYwZY0IISun5+Xld19V2Mx6P0zQlhGy322fPnsWxSNNUiDhMo4HdNl1LGD+7ONfKPn3/vTxPoyjqui6K8HvvPX779u1sNiOEbFbrJEmyLOv7PvwAkQdJkmw2mxcvXmRZJji/vr4GAGRZJoTouq5ZN9batm26rk/TNE3jrhuGYQAAUIYpw1prALG1FtG7K424IFMipfTWOecuLy/Xq9tyvbLWxhF31iIIGaWB+gPngfPQA6sNAnCUF1EUMSa6rqOECM4xJk3TDEOvtcYIYIzDJayXUilV17UcOqUGrTVCiDFGsEMI3d7eAuu6pt0/WDBCp5PJYm+v64btplZKaa2TNIEAQYwcxtbatm27ukHeYYSEEISQpqmyLDs9Pb2+vQbOp2lqnVNKIYTCmSulhmFQShljjDGj0WQYBoQQQnfMINz9pu8D2gXkCzAWOCIA3nsfcC4cVmtNBdkBM0YEkjscZYwxxsKPLqAUQogQ0nVdQLUApbt9GGMBX8NNCTtjjJVSzrndh4YNQqik0lobY8LOgV4YawnGAdfDJ+5oh1IqsBnn3A6zrbXQgh2QD8PQtm1ZltvtljER4J9zHkYjXLvxbkcUApbt/oZxDlcXnnTOwTBq3geiCR5sO/DeDeDunMOhvPeBO4WDP6Q+u3/NA0IA7j8rjO33PiU8sB4ZB5QBiADvEfAEAgwAIm/PLhGCxSQ7ODhs+vaLL74cBvXFl797/8OPCAaO+umsePL0dLV5dXl2aTAl11sAMabcOCetS7NCpBnU+u3ZRT4qIITe6kEPQ9fXTbVcydtlNQxDGiWcUmBBtS2rba217ppWKVNVy/Vmm2UFpbSpqjevXu0vDqaTudb25cWbrhuklKnzgjKllLfOaeOM9c4QiNI8LfL05OTk+vaGMTYMg4OAj3i4YYvF4smTJ3XbhKmq7/vVahUl8cHBQZqmUZQU2UgI8eyb503T9k2dCO69pQQbYMNtwAAyTITInAN5NiJMOADPLi8ARtPZjAuxXC7rru3l8OEHP3jz8tWb16/TNHWu7YauqirGGBGexMA7uFwurTs+ODpp29YYkyTJwcHB3/6nv//FL37xZnlOIIr3DpRSv//8s9lknuZZkiTamvFkVOjRdDq9WN1UVTWZTaw2Td2ORiNvbJ6kwzA4Y9PRuOu6vb29oeullF3d6EFeX10tl8skSW4vzoi30Ejb9ZUakANDtTVSYkRVW/fOMsYxxsB77q2DvvXeGeO1phDSOKGYtP1Q13VT1b+++e10NMYQ9W1HMU7jZD6Zer/ZXnw7j98fmmaRIOBNtTwvsV1fX22rUqRZ31bGqufPvjk8PhpNJ/sHY6XbQaKuH8pyBaDbP5gwjtIsX62T6Ja1zWCsxIgSioBHs3xxtDjhnA+d0YMzSjflkKRRxBJrvBaaIBwJzghwZpBKMuQIgBBTiimlCFoHREQhYhB7742xEADvjNbaaugRSmMe5gUEAKOYc4Y8UEpZpwlmYbKrqkorRSnlnJ+fn2utJ/NZkiQYoTQW2agwxgjBKKVWcOs0AhAh4I3lMTGujBKqJYDIUEbbpu+HXvDptrxt2s3bt+eECoTAzWq92W61lta4NE3TNM2yTA/y8PhouVyuNpvxeHp9fW2kVdYUxRhhDKy7ubwyg0QWMEg88g5TEKeMsSxLAnZiAqMQHUDqlXPSSmt6rZSx2Fqt7WC0hwBT0gw9RiCmHFIKHblZ3jZdSxjNkhxj3DRN13VFUczn881mc319Heb6ICc455qmCTN1nuc3y9VyucyLUZiGwnz98uVLCGHLqLUWYxzwoCxLY4wxLk1diOEQusMAxqgxRqohjkWaxmkmjFWU0uvr66qqwrSOEHr33XdPTk5ev3xVVdXZ2ZnqBwjh3bVjPB6NijRxzo2yfD6ZVoSWvow4t3F8fXnJCaMIY+C90W1VR4x/9MEPOq+8c9ZaRAH0AAMohMgZ6/s+zAk3V9fOG2+04BwA0Pedc45SGsI1bWQIMDCGO7zXWjdNAyEcj8fGaAA8o5QzEkURpVgNw1ZLrZUchq6tpewhAJQQIRhjbGi3UspyvWKEKqXyLHHGChHvLRbXN0sIobMWOh9xoZDp+/76+joQF2stRshIBawbuh4BGMfxu/k7bdtsq1IkcVmWXdfl+TgErMaYEHoGWGKMtW07DAPn0S4SpZSCod+BIiFkd+GUUufu4mbGmPcwIF/AOa2Msx4CBC3W2g7DgAgmmEEItNagl8MwSKkhxCH+DviNEOKc72AsPLkDvIDuRrlwbg+R2Fq7YxWBQwRYtdbuQupAKXY7h3g9XFd41w68ww7GGGdBiOm11nXdcs4ppYQQKWW40RjjHYSH89kxht3zO3gOp4rA3UsPw/rdPnfgbW0gJYGvhGfur8jvrj0cYSe93A3a/ePd+YT3Boa3G9vvPgsQAzEByEECsMeEQkwgwORg73BTrrarrbFKOyOHtm+7v/vlX//pT34yKSacc0BhksdJliCGGedo6I2RTklpndSWIswYgx6Uq6VRQ5DcrUXQO+g8sG673jLGEIBt3a3lxlmw3m60tovZXlrkFxeXfSMpJnlaAAettXXVnpyceAhfvXzbAcUIRwhZa6EHTdMQBDineujP3r4+PNz/4UcfM8b6vg/fIWl027Z12wAAQsiepumHH34Yp/lqtbq5uVlvN3+e/vjP/8nP/1f/bR5FUd/3/+F//g+//vWvq6qqSmSMcc4g4AAAGBLGRBynjMZCKR61xkElTVnfdr20DmCMr66uOCMEoiiK3v/wB1prpVTOLYsAi8B4PE2LZNustLJ132yqCDP8xRdfFfn4L//yn84m06os37x+TVIWCbGYzbM401oDBLMi996fXVwo6KVWEKPLy8vfffbbyXzaD0MU83/6T/+pc+7m6rouK+QBH43qqro4OzfGMEzGWR4JYa3t6gYAwFPXleuT+TyKIgiwlLLaVk6wJMm0NN45Bjwn1AMLMXIQKGA98gxB6Lx3ximtu0E2Q5EWq5sXyPq6rOpyc3RwGHEWJ0w4vF2V5cWL6XQ+XsyiJC3LOk6yN99+tb4+z9TEOT+aj7/+atm3lUhiCEGWZXuLaV3fyqFJs0hwZHQHgU4TNp/l3pm+6yARo9Ekz0cfHn+UpilA4KP3Pjyc7/3297958/ztaJzHaYQ8jJjACDFKCXDYWw88MtB5paz3SmJEMYKp4KngFayMA1pr7+zQ66FvvPeE0s6hJEkYY85ZbxX0mFCKCe9bG36QRum27rTWxYjEcby6XHVd570fj8eB9gUpHmPsnHEMUhRhjGXC+7bTziDXZxH3DhjdA8gwcZDj8SQfZFvfNjc3V9P5YpDddrvuug4R3Esptc7TlEeCUpqNR9erZZQlVHBpdN/3bddNJjNGKMekLauYxV3X9X0b1O8iyeeL2Ww2Wy6XZbkJEQ8AwAyqq5qyLB1GIo681H0/aOtEHFFKMaZd00aM+ghZ7+I0BQTn0zFnPOBuSG0IIay1VVU5ZyaTCYAu4PFisWCM8jvg7LfbLSEkTdMoihJlIYRlWfZ9Px6PCUaEkKZpbm9vR6PR3t5e3/fDMHjvtbWcc8bYaJSPJmOp7Xg8ur29HWS/LTcAeCGoEGJot3meB7U8SRLOedd119fXIZHhjDHGbDabtm2LoqCUHh4eVlVFCMIYRhG3Ns2yBGNYFAUAAEAfjvDFF5+V5ebjjz98c305DEopxQglmDkA0yimgmutCUQAeqUUcIZRDD0Yul47hRGlCEMArFLSGEuoloNgxGpNEEgizlnktGnKrdMKEuqdgdAjhDD0Rqlt31qjhRDOGc4IZxml+C6odb6RqnW1GqSnjjOmlILOq15tt9tys7XaUITDBrVp21YOg/e+3pbAeYaJc85q03VdiM32DvZjIVarlZEqZGQ45xhjY8xDMSAAatu2ASsDeAdQTJIkiOcIOYwxpdQYY7S7E97vsZnzu4DNO+cduBcJvHOm7/uuHaIkxkgjRLy3DsigtHt3FwoHxT4gepjbvxcB2/tNORNkgzuiprW1tu974BEhhAMOIXT3J8AY2x1/F9MH+AxqfBiBMAjhb5BSQrrBaBdEFMYYhHdqipRqGAZjTPj3PtD/TjbYyfs7OSFcQvgXY/Jf2v8hwwi0KZzVjiWEw4XzBN4+pBr+niE9JAq7exQ0mB052F0shNB57BD1mEBMMMCEYAwxdJ789Mc/ev321W9/9/e/+bvPALJJkogkLW+uv/ndbz/++If86Ah4GHE+n86G/lBEXGyZMhogYp2XxsbpKE0LA+Cjk8eX11dXF9dZllmrm6rO4uRo/2A+exykv4ury6vzKwhh38leaQzJe0/fG+eTtmpvb1aGytlkaow7P7tYzBanj5/8yU9/en5x9dkXX3VNzxiz1hIEIKTA+67r+q7B0O0v5l9//fXz588RQnEcu6YehmFblUmWNk1zfn7+g48+/PGPf3xxdfPq1Svv/XK5/Oqrr/70p3/y7pN30jR9/vx5P9RG93FEIbIUewyhtcQZ660Lf4kQADEpbVk3LBLFaJqPMkyodWa1WSdRfHpytNqsf/jxJ9vt+usvv/QJ5GmUjPIPPvjAe/j825fFePToyRNnAfTw6urq6urq8ePHcRwboz755KNWdm/evHn5/PnHH/3w3Xffnc/3Lq5uvvjqS+Odu74WMbfW3lxdf/XFl+PpWCTx3sFivVxtt9vr62tvLGOMEqKlcs69efmq7zqt1Afvvrc3m3d1U1XVbCyUUlkUzedzSnld1wygohgRzKqyHHoVRVHMY6WUUxpwy6KIUjqb7yXZuGpaowzywBkLCEQAC8bpKE8ienC4mE/GxSg9zZg+nL98+fJwPhqkgrpPGaLIvf/0VA3tYLV1bpyISSocxlki1lfnQI0YI5M0xoeLOBG5YF62EET7kyJjLKH04vwaI3I0nxwfH//g9J2Qv2QiHqXJ5cXZt988e3FzO9+beeS1loMaCEXWjGazSZamADjgAdTOGQWIJYRgghEiDEEMPUMEejt0AAEDERQcE+uzOMqybBiGvu+1HICzEELgrVJ66JW1NkuSMHveXt1ii5wxTdWU601VV93QjscFZdR766GjhGCMteylHgCy3jnGWJ7nwLlV2zRNgxFIkqwoxta4vlOMiSTOMaIQYko5wtT7HmPMo4gwhgQarJbO0EjcbtfKGhYJ5AEhJBbRdDxZLBbL6ytrbVU1AbMBAHJQXduHOQTCuwk0yNHr9VaMsiRJIKZd1zvniqJgjA2DCvEipXTo2ul4lCTx/v5+W9WbzYYQEsdxkkTDMKxWq6qqDg72IISYkGEYkiRJ0xRCACGUUi6Xy6ZpxuPxeDzGGOd5XhRF3/fz+bwoCuAsQqjruqZpiqLIsgwh1DSNlNJ6vwsECSFN36VpfHmpXr36drO51dYwxjzAIXK6vb09PDw8OT65uLhYrVZ9233++efj8fj44DDEXn3f39zcQACScdH3vTW8Kjd11TZNg4CXUo+L3Hs/DKruu9fltq5rAMB0PFnMZpuyrqqKICwYp5SnaQoAwABijIF3CGPKaBxxAICWvVKWMeCB9d5bp61FEELvnbUAQjcej05OHo3H44uLi9/97nfnF6tHp0+8sbLrPbDeRkkkKCOYMW8NJQhFEQYeYwygG4ahHwaEUNe0nHNG6aPjE4ZJFEVKmeuLy9vVRkpJCDFK902rrDVanxwe1XVttbmLKQFM44QgfH6xvby4YIwFIFSDRB4kcRLgUwgRYmJKaYC9HehyrnewGsT58HVyzt8RFAit8RhjrVV4CSEEwL0GbpxzACESUhjeQ+BDyE4QIhhjjAglzBIHAdql6neYGmJ0Y0xIoARIgxAqpXZJh5CD2L0rpFHiKIUQEs6MMcOgQkKBC/Gd0H8vGOxyH2EfQsgu2gYADMMQcmdCiM4OSqm+740xRTHeqRG7SF0p5f0dA3iYCNilBr5HAh7i9/fSCrsBfyhF7PZ/KFQAADDG3vndYRFCEAOIvvuIsNvDv7szQQ82CKFS/jtxwjnkPLIee09++/e/XOzN/+TTT+YjfnFzEUV8VMwQZr/9T79IEJ0VIzAei3Q0G8+r7bZptnWlIcBZkaV5lhGapFmSZ86C7XZ48fKsLmsCOac4jdLJuJhOp5hmSimCKEOMEQoAksAQby7enr+cvCSEYEgiIabj2cHBwfX19bkFl5fX77334V/+xX/1/PnLr778tizLdG9PDzJKEwS97FrvDIagLrdffv5Z3bbPnz+PoijP815J4EFVVTwSWZZZa4Mi+vz58/Pz8yzLRBxdnl3+23/771er1U9++kdy6NQwdH3ttBGzmQMOQQQgACj8xjD0yHpMMKNMUJ5keTaZTJig1vrxeJJlmZbKA3B1e/Ou7KIoqts2E1QIIUQyXxxKqZm4eufpuz/85JNvvv62rtskiW5uls6bpi0nk3GWpcZZ2Q/Lm9tv2bec8yjNRBwleaa0rtuGpyIAWJIksYj253vFaPTtt9++ePHCG3t8fAwBaOsmz/NPPvzo5uLy7O31Z7/93c3l1d58sb/YOzk6plT2fW+tXy7XBJK6rq11eZpNp/MrzLbrNQRYa22tL7J8MZsDIqwDIk488EBbhnAWZ8Ag412R5d5ajPFkPCqyZDzJsixJU5am6XZTJWl6ff282mLCeJKYJycn3prldtspeTgff/DuIwf8bLqou76pSgXtfJwfztK+b01b1tDi0SgbT9NR4XtFjOcsfnRyeHhw2De31tqurerSx3H8zunxdv3uF198gbzjTOCIK8OdNwxBaI3XiiAYvvIeAGBsUFOtByKOvPeI4DQWguMsFQihJE3n48l0OhVCVNtyvd52Xdd1XVN3nPMsy9LI1nXrnEuiOEyR68sNxaRv2+1mI5VarpcA+XycY4o8cGkaQ+NXy6WUfSIiQrE1EEFGcFRXXbmqxqNRwvO+MxhR7wjFqXe075yWCDg+KAsAwJTwSCRZmhV5ua0gwSTiclsOVuu2GSUZQbjIc90Pt5dX5+eXTdOUZamUIgQBAJwDZVmGWQyTuxmQUgoh9t7LfkiiOOI85oJhJxiHEAJt0ygWnHFEWqkizr02EWGrrscA1nXNOR+PC855CEP7vscYT2fjEIcNw9B1bRzHwYtwu1rPZjOl1M3NDSA8qBrHx8daay0HKaVSyntflmXTNJTwoOgigsKTVVVpq5I8K0YZZXC1vr25vUqzTAiBMR62lVIKQvj48ePFYvGb3/xmvV5/8N77r169IoQEzfzx48f7+/td152dnRV9K4RIothqU5WbqqooIVLKpmkEj4dh2K5XVdUAAIqiqKsSxQIDGMAGY8oJJRj3fU/uEu8QIsg5jWMBrBuAU1ZhCDAEHiAEICaIUQwBa5rGGEMxWd3eYAisVkPXd027LdfGGAQ9YyIWnAuGIXLOeOiJR95rqZR1Gjgv5TAMAwG+bdtIjNMonkwmXVsLzrMkl/3QVJUcBgBQVzfWQsYFJ5RSGovIORdzkcZJtS0BAMA5b+x6vaaUAgQEY72U42IUJbFy7mGAG/AjxOg7P0ogEBhjrbVydqeE7xAlfA0Cou8g+C43wZH3HkKMEIEQIwgYE95DAIB3wAFPcIhoqbU+hP6BCgSADIpCgNIdsO1kAAhhJKLwQeHVQCz8fYYeQODvXYQhUYIR2p1/QMcdHu9SGDvwDt6LwADatu07GYaFUlpV1T24IkopQihQK+DJw5EMxw/Du4v4w8fBezHA/+e28LkPOcfuveGZcKVBSMAYe+jvBZK7s/LA7a5xdz7foy8PUw/hSWOVtRo4AozxTkFARMQZw+QffvUf3//g3R99+uGf/8mPN9vjpmkgIMCzFy/PPvu7v+OQPn3vXaWG9XXVlXK7boeWUM5zPs9me5xFIo4458YqY19SnM2m8ZPT0zxNCPIYASnl737z+7Is9/f30zSdjKZGO+hQUYy8g5vVWmqTRFGe5CcnJ6cnj5CH600TaPJkNL1KbjjnURRhjFEYawQg8t55hECSRLPpuOu64KKSUlpr4ygJUlVRFHEcv3z5crvdVk2XJMlkMjHOnvfl29dvBKOnJ0fT6eTDD96Tff369WstB+ccAAiAkHzw4YuyvrySSillCGEAoM226lUPkd/fXzRd3zX1uCweP3o0DEM+Kvb39zMRbKKU0KwfaudpkhRpNo6ihFL++J0n2+12PC7G48I5c/b6TRqnn/7wR2/Pz7dl/fW3zxDj77z3/kcff7zebt6cnyVZutlsttttnmZKyvV6fXRyuFwujVTGGEpIFEVt3UyKkfe+KIoP3n1EKW2quuu6+XxulKaUHB6eXl1dXV3dOOv7ts2zkTYAQZIkGcYcWNe23TDUNGLFaEoIqapKtq11EHk/zos0gYJFZVXPxpO3Z68ZwVk6NkZhDDFyF8uS1wNPcmXAaLqXJYlzLo7jsq7HRS6NzGA6ydOnp8e3y2VEQDabrKBXuhcIEui1M0rL3pmU0w5ChBg0al5keT5JGemrzc3NldbWGOM97Lo4TrIPnj72TnkAoihCDGstpR4AcBAANfRMxAgABABAd2TcGuOcQ4IrqxBmWZqOxpnWM4BQFEXH+0fBWzfMZ1oZNcjz8/PXr9/mSXp4eIwQOntzvl6vkySJokhrvTxfMcLbvvHG5Xl+s7q5uLhohxZRIGU/HhejcQ6Q55w5ZLd1rawdFQNwZOh9tamzeMpp3jW261o5OGvI0EFrLPARxaTv2yQjnHMPgQM+zfPPv/zKWrttah4JTMi2XY+LEUIo5uLy/GK9Xr+5PN/lHYGF/aA8rAclCSGUYuDgMPTWmslkIqKI8ch765RO0ujk4HBQRikzDAOGKIki4CxGiGMSU351eQG9k203Libhhs7ncyFEmFO2220cx1EUhQnaGCOlLIpiMplcv317dXWVJIm1drVaSQvKqgYARELUdQ29q6qKM2qMCUG8MSZY6gQXCKGha7fbNaYozmIRCSGE8bZpGmP1tmyVUgVL76s50NXV1XK5DHry4eHhMAxa6yiK5vM5AGCz2Sgp0zQVQjBGEYLhXYyR5fLm/PwyJH0jkYzHBYRwPB5D6J2xjLGYi15pay2hnEAU6CbDhHFGCaIEIQ+s9/geJAAA3luE7mzzCJm+x1rrtms232wuLy8xxsPQxbHQUlFKRcyiKGKMeWel7JVSGHqEgNZ66FulFAChZkcZY6w2EMKDgwOKiffeSLU/3z+7uKjrWnaSRrFSWrsmQ5gQ0lQ1ISTmIs8yjHFd14wQKSVjzPZ2vV5nRYYhwhAW43EA/nD+QUXYodfOBRIwiTGGEJJShuoPhFDQkPy9LcAYY8xDA8Gd2zH8Fr13Ic3v4Z2X0BhnUABmjKm31oYJXCkVclg7ZrDD/h0XCSMD7osUAurf+SUDfAIQtA0LfGA54VS11vje9PAwQAcPfILfC+UJJYEZ13WtpAl4xDlfrTb38sadeeKOoHi/kwR2bAbcux8eIv1dFA/gjvf4+/fulJvA2HY3ZXdiD4gCuD8MvicKHmOMALTOBKLzPRFiRwv+MUsAAEBgMPTQO+gUUIoQVMR8lCTkow+evnnzYnv79tMff/L4yfHepKirflt2n3744dXN9qvPPu+b3ns76CGLi0Qkt5WOomTv+GR/fx8iBABw3tgWjEd7R4ddJOjTR6dFFmHky+36zctXSRxfX101TbNY7M/i2BinZrP5fLHdVi9fvep7KVhUt83N9e3+3kEURcfHx03TJCLZbrer1SaJ43tZDPR9zxlJRKSGvq278ePxX/zFX7w8uxxPJ9Pp9Pnz58GUEFLO2+3WWntxcdE0zQ8++iTP867rzi8vxuPpo0cn77zzThzH3vvHj0+LInnx4uhXv/yl885Z5y2y1gFvAEDew3pASqlBaQ+ANd64wUOfpSmlfD6flwRTSk9OTzdViQE0ziISU54YqZar8ubm5vXr8ywrlDRJkuzvL37wgx/8/re/C0FVnqTW2l/+4hfHp6dZltVNF0xY3nvGWCeHuq7zUVaWm7Issyxdrm6Xt7fDMGRZ9vjx48uLCwBAxEW52SqlppOJEIJOp5PRuGma6XQ6m0xfvXq12m4ODo8Y5UZbABBAJIoSQtjZxWUSpcdHJ2maXl3dvHj2bVN3GJWjIlGDBBDl2UQkBWXJtqzlYJrqghLirYuLZG8+hTAQwYEhcn51MxmNL26Ws8mYiOjt6zfGXBdF5pxrygpRIocmZtjIdqu68WixGBXWRmW1qbYNZWQURRBCAaHpur7bysHGcca8r5bL9WoTj9h6vWVURGnW1Ju2rTGmh3t7vVLO2V4OdVsbOzBBIxZhjCLO70IQAAAEhCDOqUewqispJeEsinhWpBgnAADMqJQ9AI7zKE3TvccLzsR8Pg+q0MHBEfQAATwajQAAl5eXX331TYZjyKnUQ5Ik+0eHq+3q7OpMqr4t2/W6aZoqit+dTMac0+1m8/btW+lot6+SOMaIO4viOB+P9owydbVFiBAcIcg9hFnKIpEAuInHPo5jpVRZ15O+/+bbZ5zzq6urw/2jPM8RhIeHhyFE6PveG+sApAGajDLGaGuwQRAhEUUi5gCAum2MMR4CyihhFDrkreOUzecLpcz5xdVgu5hxTlnfNdiBiHFOWbutCIbO2KZpJpPJ6enpYjEryxJjnGVZ1zU7IxuEMIoiCMF8Pt/f3++2WwCAEGI0Gm3KutxUIT0hpey6LolE0zQIgr7voygaj8fL27VSCmMsgMD3GyGkLMvYGmtNkiQQesJoWfZKKYvtfD4fj8dSyq+//tpae3h42Lbt8+fP0zQdn5zGcbzdbsuyBABMJpO9w4NhGIZhIITEcTQajbz3wVp4fn6e5/nTn7xHCFmtVowxQggSIqO87/tNVQOIhRDBs1aWJcOEcx5HHHir+k4rZa3Bd6kG552DAGAECMIA+SQSSRQLIW5vb5WUjDFGaZqmk+ncORPS/EoOQbSG3jZ9zxiBIIjMHgCAIcQQamOEEBHjp6enfdsRQoZhCJy1a9pBGh4lCEKrjdUGIRQcpghArXW1LcuynE0mRVEoNUAIy6aO1J1dXwhRVRUW0UN/3E57b9s2JCOCOyEQhaDlhHsUBGx/7wzQWgfzIiHEOQehDyiO8XdVFdZ4B8HdHaYUABCkBXhXueecM+HLsFPL4X194M48sbMKhspMd59cA3dWhvvI2BgppfFOCMGYCCxHKUXujY3ggaofptyH0v0u+N6ZG+I4JtgAAIIbJsuy+8JFsxPJEELW+N3B/QORf6e+7F4NFwX9H5zGw1d3pOF7UsRDyHf2Pmdx770Ip40hCuWRCOOH1Ofh0R7KJ7uNYWy9xwgAZyGwHKM8iuaTghyfZFdXhjLuPCyKAybSN29/27Q+KZIok1W/Pbt5NZ9Ps1FCOGOMRdl4NBoxxtrKOq+atjRGJUmU5sXP/vwvr2/OnRA9YZvNioskf/zOB0f445/+mXG2lxJRNhuPkixFiPjV6sPD/WdfPdtuNsvNplOq/uXfPH369M9++rM3b9788q//Ksuyb775StW3SqksSyj3jGEA/NDVCCERR/lo+ukf/8n18/+76RvS4hTaw1Gqus0oE5ii5fb2P/7NX3dyOD4+DYJkHKevXr2pVf3JTz5ZLBbS6bPXF3roP/nkE2ls8ezF7dV11w+T0eTmZsmYePzk6dXF1VANcZooqiAj0um94/1tW27a7dXmcjYtLBwWi3xasBfXL89fv3q8tz90dKhVVVXV9vdSStmqr7/45vZqub+/d3h4CBH58ONP/uZXv6ya+vT0tJhNXqzOPj97bpQeZaOnT55GlNQ3t5TyRZTl77zfDP1ifvjROx88e/Fs6OWjR4+sZ3mcPn3nB0eHj5WUcVQsph4hogb/T3768xffPj86OL69vUUedZXluNj2/d//6uv333//ZF8URfH8+fNxsV9thlevXj1+/Hg+3rfKq14ZZRHAGKJRlJe3q/VqZfqazmbG+76qUtAT9dYP8L2DZDqfzUaZNVBtlfRGcTo/er/v++Vyfb66IoR4a4Hzq7bO01gZvlmu33nCVzc3urNPnzwZz+e3t7dl00PAEEm7XlpPprPxat0lIrLaya6l3sOIYSVte3VdY865dUPVDyJOszhS2iKrdFdRxoAabNdAACBCccr2D/bigg/D0HWd9zDLsnxUIEiGYTDOr5ZrrXVB8wRlgYlHJELecc+wI6Zzt9dVlLgkm3/yR8UwDE1Vd22bpvmkGHV1Vd/c5gQr4w8Wcx4Jb+0kSz988gjqbjafbrfr08mk67ruensy2psmC9xjNdUHhdifzLRxk2I6SucHB08JH2srZ/vRm/Ozjz/+uCxL7/0kiry3nan20nRo2h+8/8FiMfvr/+V/+cnJidX64+l0Npudn13KUUEo7bvqzbkSUWIZTotFNwyD1h4xh4mCDiBGRNJD5CzEEMTZGCMAPG6qxltPGEnzTERsvVoG8FuMs2pbAqlnScI5GWpzsL//2W9+W/eDsQDpnhCQJHwYupevniGEptPpdDbalqt0Fc3nk4uLiyRJJpP96XS8tze/enM+KWZ7i8NRMWs7J6geqI2iLE1yq0Db1U3VO2Wnk33oHfJEECoItdZiD7y14yxHAF7dXGZFfnV15YwTgl5V7Ww2jePUGFcspp0ccNcsnDk4Oujr9gfvvvvs2bOjvT2tNWUEE/T67E3V1CcnJ1VVi7LinAOAb25WSZKMx6IqW4x4XV3n2WRvsTcMSuvWOYgQTZIMAH59eW0Hd7J/XNbVersBPrhcTT80xSjN89x7L6nQWqthAOSuOJAxFidZ3/eDkowJDyWEcJCKi6jruna7hQh1vZRnLymPOOcO+FCwBwDwwDpnrHFtU6Yx80Bq2Y1H+XJZYdTPpvPj44mxrbISEHp4+KiS3pE0mx7JzbY1gLIIOSedZkQkmGmtHQBS2s7ayWxKI7F/eKicNauV2mylshCQIk2Wl0vBBKBx1yoEGWcJ5xEAwNp+NJrsZAaEAKXYWj0M2nsbJbFzLhQKAgA4ExhjgHyUxH3fa608ZAgh4J2HDmJgjOp7eddxIWIAAO8ghDAVwlpHCIk489p6qYFS2lpnLOSAUkIRkVJ2XR+ECsqYc8574DywABjvvXPYOcq4ts4NMsuYEFFdt1prRrjRDkPCKYl4ghAa9EA8JhgT6wGAAITIHjhnvXPWO0QZsM4Cv9Mwwt1JYmGMwRAJxjEkgWMBAPq2C+5OCBHFxBNnjPHOQ3iX2nhowwzZiodAvkN95z1AEEEMALDeOaMDePdygAgywR8CPIQQ2oFQ4D0MCTIIQQB974OcgBFkEGLnvHcII2blAJ0VhGith2EgiGBK+76HEEOBESXAIw8IRCTwrdblSlZZljOkMTJFRuIU7R1NyA9/+MP33/vBYrE/mc6HQX/97OV6uxl6UzUtj6LFYn9vby+OYwcdISTLMsoTAHRZb733UcQJAUqrppWE4Lar4ziGEHDO5/O9m5sr58DR0ZH3/vr2ZrvdYkqKouCUOecmo7EQYlaMjTIXZ+dqGG6vb25vbw8Xx3mRTiYTxtgHH3zQ9/3rN68ghMFL1bZtmqaUUq211vL165dvzy+/efZtVuTzg/333v/QYfzy7ZtOKgeJcb5qu826xOTSOD8ejzmPpqNF18j3//zD5fJG8CjmQmsNIZ7NZkPbKaWarkEIpGnsvIoz0UvokaOM5NPCQjceF1HKAV7cXl8ZNUyK8eHh8XbTvHp5ZqUzFldVFUURpTTLkq7rmiiCEFZVyRh99s1XaZoeHu6PRvlyeYMxfHz66N3Hj5q6Xq+23timKm+uL7abVd/3xXg0nc9PnhwfnBzcVOdvlt8WhJ8+3VvM5sEuVOR529Zd19R1ncSZkSVwbjqdT2bTq6urN+dnyIPJZJLn6TAMRZE9enTijPkPf/1XZ+dv9uaL58+fd10ztE2WZVJKiMF4NJpOp9WgsslcZCPnjAIeYUQEA1rySGCMOYtG01GcJbe3y7OzC/VcHb3zjtTK36cng+ldDUORpc5PrHdSys12GyXxJ3uffPDBB2KUq9//vhq6NE2SJOn7LhQOcM4F40ZG0AOGCYEIYMqpeHN+ub+/n0SJc4BzniSJb3ulFGMMQBhFUVaccs6VVZSRKIqwd9A64ByGBHnglQEMC87TJNdSrdcbOQyUNABCjCl0nhWp9Y4BEAnuOeu9gxDiOKaETJN0bPX69nazXFFGj9594gW9fLmME0EZLEa5ECJJklFRUEqfPHkCgGv6DmOcpjklZDKZZFmRUL+32HcOaIuaunMA123f9oPg8Wy+ByAZT2ZXV1dcQIhwlKR1XUcxhxBW223f95NRMV4sptPpzc0N53zQbdd1Wpum6ZS+bds2Ge2DB+GIdbbve+/dZDIKIYK11juP/J1hSknT1J01fuj7pmlGWT4ej/mctXVNKAYwyMV3wqaUMs+Kvu+vrq6iKEKQMEaklMvlCkKwXC5Ho/FqtTLGZFkWJkQh+OMnJxhD67SxUuthvhgrbT3QlEHh2MH+nohYuV4zxrI0ZQQ753olOadBPWaMjcdjD+/mx+C0D/mFWERN08wnU+D9bDwJHt6u60L4e3NzE6SLsixDiBzajgEAlLpreqO1btt2u90GC8LDyFJKCSGE0IR9PILaGmNM0zTd0CdJEkKxkIYINXIBS0J0izEGwO8y6KE3g7NeKRVEdQAAQtBZF04pEAUbolvoAHAQIGOMUkArJYehadAwDI+P9oGHYeo7P78mLKKUvn37tmm6EMV+T0P20EJ818Ei3JGsyJMstd5BjAijiGBntHFWWwMtiikNlkBrbUjNBLPekydP+r4Pj7XWfd9LKZ1zFgKEiHOOIAwhxAg6a43W1hirLfQAOAshwhA5Y7umNU5b+z3nHbk3LUIIodZaKRPqb51zu45h4dJCUiMQhYfifCgx01pTrbqu45zf7UZpGIcsFaG8syzLYDkM32qLYVBNgm5x54FAd44KjOBOz7jDZWtDDYgxJuRT7kf7u8zFbvydcwR/RybAHzoHHw4C/C/bG9EfFknuvqIP5QT/h8eFEPoH/Rh2b3z49dgd+d6Z8Qdawu7VCEKkYczxOKYpFkd74/3ZeJxnZFvW0+kcEwYh0tYnSfLzn/8lofz585dpPto72B+NRoNSZVn2nXS+4gIyxuLIAwCE8FK6oddGK6tB27ZxItbr9XYdpWmyvF6ORqPtdk0I0XLQUhpjuraGwDVNl6bp0NRt089nsx998nESJV9++eWLFy++efZVcC1Np9N3333nxYsXgkcH+wfPX3w7m82iCIQpqSiyn/3sp/v7i6+fvbIAW0hev72Ots3jd9999Pjppm7aF6/athmkVav1tmxvb1aLxaJp2jhGf/e3v/3LP/unTdlFLJrOxt47COFqdTvoIc9TrTWAbNDNs+dfF0WhnGWIF9Pk+OSgrCupeozQ0dGJ7oYX3z4Hqe9aff329tXLyzzNotuSUxH8EIvFrFxvnFF9329WK0qQMQYBb5Ts27YpN7rvtsvb4/msjaNcCGttnhWEuqq+vV0t907+iGW+M6svn1+9OPvCwOrwyeTRe/OcTuu6FpxkacQIvrq6chYIxgEAQzss9vbSNEUEBwIRZ/F4Vgi2yPK4rNYQwqfvv0MRNsZ4aAfZXi0v19VKUDaZTGbz8XhcbFo4TiMhhFJD29R93wLGaZooCIB13vs0S6JYxL2gHA/KGOBVyG0nsWDMWospkVJaowCE0/1FMRn98U9/Ml9MEYJSys/fvnl2fna7vjngBxHLgWe2t8Y7TBmPRMQ4RZhhMilyBGAWJ9OT08lkYpy7vLiWRksptZFS9g4iYy1EPk5HSRqZRvd9X1XVdJxYpaEBhEJgbN923Pg4jp1RwGhoLTAGek8wRtAj7xzCSlvIQCYiysSgNMYoiRIA0DB0aZJ1Xffq179e3l5naayUct40TcUYQwg0VamU4pxZrTDOEKKxA8aY9Xq9Xm2FELPZLB9lyWhOCHE4qquWYQYABkgqpbjIrq5vZ7NZVXceQu/9er1lujnYf282m9XlhhEKve/7vqlqqw2lFAPoERZZVDWd7aUQInQSwxh75D0EzhmIUegbBT2wzmqlgDWAMQQ8xYQlcZikdjNgmF739vYwRNZaNUjnLABea9W2zdN3HvV9//r1W4TAMAxFUQT/gbV2s9keHh4yJsqyDGmIpmmU7vJRxjiezyeD0TfrFY+41FXVlJSTQVrj1TDoTnZc4LJee48QJQx4661se0xplmVRsnhz9rZvW6m1EIITapUOhWqrm9vFdIYQStN02d1SSruuc8bGcRyqVdu2lX0/mkwIwqIoCCLW2uDk38VznPPRaAQAEkKE6fIOZb0PfRG2223dtVmRB9oUVG4AQGiqASE0SltrofeY4l28GAC16zpr7xiDubPX+LAPxlQPOrBqF1xQ1nrvIfJaS0extVYpF/yCod+Rcy6OkhB7NE2T5nQ+n59fXAfke9jUL0z01Y6UDEMIZxutKyl75wzGUAjPmFbKQ2gJ8YyFoCtAZkgxMMaCNyIMl793wATy0WsFoUcIEBKwlQQ0Ch3qtLZSytBxBEKvtZZahUrCcHo7yX2XxZdStm3fNE2wHeyMhwHLjXc7wN45JYNRN4C38U5rHXysYZwDvWCMBWYjg8XsvtjSIhBuFri/65RSRO6qaQBGIaFwB9sIhbrNUGSxYzwIIYzJvTBwV14YEPpe+fhu+x5dgA8cA+A+y/CQJTwkCg/fvrsdd8f/w8N+//H9v7tjhpPEiASeFBowgAdyxZ0EogcHTEz9JBOzhB4txvNRmiWMbLeVlPr3n39Z5JM0zaM4e+fxO5TyR0/eM9YhhBAlsdYQ47Lc9n1vvYniKWWoqprNVVNVlZQy+A2tU8tlc35+7pzLsgwAkGX53/zN34RebGkcee+3y9Xl27Oqqv7rf/bfSCkrObTl1vJ4f7H34Yc/GI2Kv/3Vr7766gspJUIgTdOuGwBAbdtPJwuC8fXmOkmijz766E//9E/+4i///Msvv9Qezvb2sjx/c3b+5tuX20HuHR0jjJt+6KXiLAIIWm2apsV4ba1NeLFdVuW2PTw40krOJ1OEPaP4m2++GobBWp2P8/l8Xpb1l19+GTkujaGIjkbZbD5ebW/Pzs7jOH765J00TtMo2ay2n/3+a2ictVgbeHG9Pd5faGWVUlJq7WyapuOiEJxcnJ3v7y/GRVaXm7oqsyyLIl6W61gw6s3BfDqfz5MsrbvWI61h0tsaNHbVmlZ1PHGP3t13znz+1d9+8jhxzjnLESMAeuh9FEWcR9Za5zqEyGa75Zw/evL47OysG3q77g8PDwfV1221v7//85//2TAMn3/++Sc//Cik2YauhxBSQTHDDjqWjhHnNI7iYkSTBG5X1mohWDEZd123Xq6kUsgMLOFHp0fee0mE8a7teucAIjiKo/00jTjHGKZJFHHWds1oPqWRuL6++ru/+7v/+NXndV1DCBVCpZQEAeA8IzThnBASU84iQRGGmFJCUoiTmDkLtqtVXdfKOkp4kiRPnz69ur1xwA/DMMgOYQ8hpBRD6J1yVlqrDcVEDVLrZuCdtxo20GqVRCLP8zRNPAz+aqB7o7oBSidxNLhmtVpZ60ej0XhcXL15c3p6/Gi+94UQX52dtRHPsowTAr0TjHJCm7YyWqZxooxs6wZjbJy11sq6beo2y7KYR6PZYjCOAEBFOiYpxtRZQLiqqspb16y3idJJXjjgOKVa2QgjIQQGcL1cqX6orFaDvIDno9FIS0MImeQjJqK+e2Ot3dvbawYHnLfWQucxo1GScM4JwR5YCKE3TinljSYYc8EFY3GeBWBghDHKOaXO+UEN46IgCFtrZd8ZoyjDSRpJlYZekOiuNc3dxJcm+Wa7sta2bYcxUUr1vZzNsuVyLfWgrYoSYbyByAFommaLKTFOF6PRINu6LiH02vTaidvr6zSZOO8Jpcr6XkqkNY9EwqPgDIAQMsY450pL5xwntMO1Neb09JG3DgFICN6s1gQiTih0nmHSdN0wDBjC0NmM0rteSSEG9d7neX5yctL3vdZ216/3LglNyM5A0CuZZCkhBCC4q80zxvR9H94FAGCM8UQEZArH7/t+uVxut9Xjx4+NMUbf1awH6wAhDHmlzHeB6R0AeCel9A4754zx5N5/l+f5er1+8sfvHBwciSgJcbP3fjQabctaCAEwQfe1CXcQy5kLaoe6ayIEMPIIIoQwo4QzSLCDABLMIhElSegXFI5gjAlGDWNMaE0RviphC5eAabD3WwAAhh5DTwgDFEDnlNLaSTUM0PMkihhjljHfAefADqj8vdPQOYcxAQDsOkJ67wkhcSzuzCKht6McvPf0ztBw50UA960VH54YAMBa6xwIupHRdXBUBOAPLRfv4fWuWbu7Pw4jOOwDMNppQjtUBveVDgh9ZybABD8kCvdhOmCMBQKz2/NB5P+dUxI8kAe+F9b/YzvC7nHwjd49/4eEA/yhorB7jCCE91QGY4zRnQfFGLU7JnjQYdqCChM3Svg0j6cZHedRntA0wiRKkjhOm05qa6q6HZSvqoqwOIoTpZQFkFpGGc7znDCqlDK6Ixh2XXt9dXZ7e9s0XfhCnxw/ypL0+fPnEb+LcpIkqTb1i2+fpWkausRLqVa3Sykl57wvS4wx9f7V8xdffPHF6enpyclJnucffvjBmzevNtfrzz///Ozsom36UTF5/fr1ZDLBGB8dHr/77rucU6n69Xr1u9/9Np/NNlXdW7t3egqj6PXF2c1mm01G2lnrXJSI0KEiipIQdq+XK4TQr//hH/7Fv/hnz58/Pzt7NV9M8iL95//8n19dXX355efvvvvuH/3kj+u6Pb+86IaeiFQZva3KUH1Qb0vkgdPuaP9of7r3xWdfrldrQZlIUgBJ18vb21trbT+0TdNAZyHyk8lkMi4wBBh6b03MWA9BW64jOjs5WPR9jyHYO5h/9MmP4jQ5u77MpgWNWTd0ry/elPVaJPzo6MB4vVxe36yqb7/5fDqdequaitVVPww9J5GUsusGRBmLxPq2GY3Hj995NJ6O3r59mxfRzfJ2UHJcjII8a62dzWZ5nqdpiiEMXD50KVZKbfoqTVNptFXSeUUIccCvtyXGEBEGKVODdFIBAAAjCOI4zowxVg3We4sAFmxUjObz+cFijhHo2/of/uHv//3/8ld937Zd89VXXy2BT5KEUrrtOgvBbDZL0sgo6Qi2GHmKIUZW223XYQ+s1diwsiyrukUYMEyathqUFCJCCERRJGV/dXUlhJjtzYqiiCI+dINSylgTyqy7roMYSSnzPCeExAkREbFO9nIwxmCMkXdD0wwODtfrrutvr66HTnJOT09PL87PRNfv/ejjdybT8vAIE0QpJvOptTrLMh6xpq0owjzLtGHDMBBKI0QAAH0/KGkBQM4BiFjXa2Ok1ZZRwTkFDlhrtdTeO855vS0RQjc31/t7c0rQKM2bsnr2zTe//e2vt6v1Yj4VjEMAy80WQpxmxfHxsTK2bb64vrhkmHicdnKQUgLk0zRNkiiJYsqI1pJhop2lCEMGkygusoQQYjxuVe+9p5gIERMErLVyGMqyTKIYY0gIkaqPIl4U2WiUh2qdKIqSJPHeDsMQ2qvLQQseX15cU0aiKFZKLxZ7b9++hQhBhKjgn3/5xXK9pZwNSkYpY5HQzlLBizSD0Fcl0FoDjBAlAsecc61lL4dQDOy9HeUFwsA5F1biKMttCDGtMbLr9/b21us1dE5KtSqrLMvkMMRRBCEc2s5bBz0wSqF7TAoTZYB5glkIbJQKWQlljMGYhNS7lNZaizEmhDHGeim10h6CUKTQdV3wDEL4Xfr53vNvKKXBNF2WdQh5lbqL+wOk+dCDyNsgvAXBX0rpvLHWOgwhhMZoyhkhBHib57mh8OTkhFLedqEAe3jx4kUoL8AYY+eDo3ynpRvtjHFaW2sthKGFdsx5hBCi1Emp8d2iDIRSzHkU3PUBnEJhQpAWIIRhuAKChleHYUAEhtidEEQIxpgEET6kLZyzoZd5HItQvKC9C20MAogCAIx2gZGg++7Ru5EM/CDUPgTvpJQyKBlxHHddF7SNcEqBbHl0xxUC/CNEAveSgwyYiu5bL9yhrNU7Rc1DGPSYwI0ghGC32z3kBngNPMNaf5+DsPiBSXAH8P7esbi73f7elrjTDB4KCd8jCt/b/AMz48N/v0cL/uCE/9EGEQT+uzPcmVUBUDtxAvq7W0MIERwTFk3ypMiThANBIKMoYoxstxVAOEnzvpcXV1daWWXAYv/AGsfjJE4SSqn17q7wFANOBXDQaheL5NFJYo0Pa6JwyiCERpnJaHxycjIMQ9M0n337mVLD1VVVlps0zpqqXi+XWZq/88471XrlPey6Tg9919RVVXVdRymNIv6zn/10vd5+9eWzX//61/PZ3p/+6Z+9fXv+b//t/7i/v/j5X/zZk3dO//7vf/WrX/1S6+HV629X5dYCbxSKrAIUYkF4KozTiAHVDk4b2UiEUJJyC7z3/oP3nqxWq9/++m8//eEPumarBoQJ6PqGUlyMR3sHh0+evnt4cHpmz+J0tFqtolisVqvzi8ttWVVVgyGhmKlB7u3tHe4fJVH61edfhFV5HHIWWErp6elpmqaCUwBcuVlVVeWdmk/H1XYj+8Y707elUmqUxmIyar1VRvVKSj0QLyAl873F3tGhiPntv16fnZ9DiE3vrm5vGKEfP/noq8/epAmvvAWeaAUwJEKIYdCUsPl8fnR4TCnFGGZZko0ySKDqe+ARowJAfH5xdXl5WeT548ePb25unAOE0SQrUucZY5QJTChUOmVESnn2+qVzZrY36/r22fNvhBBZllngIRWEC2tt36m+b1IuMIQ8iTmPxsWIc87jKE4TT1DdNZvNetvWwGnpDKRkspjHIk6SpJfddrtlcXx8+ijP02pbMkJSwWMmCIBea2cssI4AgJzmnO/F8Xg8ldq8fn1+cXltrRVxwgQ3Rtd17ZxBaCYEI4TUSivrlFbaGmvtoBSAznijgcEYc8t62Q9ahRQy55TIXksjpbTKWm2QdcJZ36qbl8+BUcPm9urVc6y79x4fDEN/dXXFkxmAUBtpmqFra+MN6IFSA4sEwRgj4iFkFHDKgINDF+zQUA1SawMEEFRACIDzzmiKySTPzs/e9H375uUraFRoXrRZr5u6bsrKWxfa8kPny7LkMV/M5of7B9uySpJEMN73fTGdAgCQBw66KIryJBVRhDD0RmOMIWM4SSlB09E4zWIM0boe5KC7riMYW6sJRJRhq00SRRFnGFNCUd/3EHrOKQDOGleV9Wq5nkzHo9FIKbNarZbLpTEmjmOt9cnJCZxiSmkwD3XdcPLoyWK+/+btr6u6e+fd965vV7c3K4BQ0zTT8TgfTTD0m015c32dJInUCgDAkWCCBx9AQHSttbcOQdg0TTDhD8OAMbZKr+pmu950TVtkWde0XdOMikII8fTp0zRJVqvV3mw+nU4ZY0mS3K7W3nvO+S4Br1UTrPUhQRBUZQhRXddt267XbUCRxWKeJMmgVIiAu64LBgU1SCGEM7aua0WpAXcdPL33aZqERbPC0hUAAK2Mu28cRAjBmA7quyA1TPdSSmMVY6EzIFRac0ogAGGEH7/3hDG2XK4220oIARC9vr5O0iKE+855fN9FAOCQ7HDWeKOdtQ4h5Am0xg+9yvPcI08wgxACjyDAzlmjHUmIMSaMj9Y6JA6yLAumjZ0G4O/bdmHrg4cgTVNKeEj/OwcsMpSgIs+DxS/AG0aEMbYziNyhqbtLZOwOju6bJoUdwjF3Yg/GOIqiOI5DCiBEybtg2t73JgoQzjm9i5ixC09qbSGEoemn917L76ouKedRFAkhCKOhEMbdYye6b08ZGhXAuxoEsENldN//YMcVdvrQDs4fZhAesgrwjzwK/5gZwAedHB8yjIdUY8cSHqoX/1hU2B3/IV3YZYKstRDAHVeDBCHnEIIYeues8dYBDwAgTdfWTZvnI22c9147//XXX1/frI4fPc4xYTzSVvZ938kujM7hbOGs6VqNIJ9Op3Ec13Vd12Xbdk3T9H1vrQ2/0tVqtdls4jiGEHrn+q4ZusYZyxCiED778qurq6um74rRxFmLgF+tbt++fS0i9uNP//jw8Lip+7//+1/nmU7T/Od//qRpmuvry7Zt37x5MwzDH//k0ySNvvnmq2U5nJycQIyul9fLchMl0ZOnjzs53K5uYeWNU11XF0WxfzAP3DlLo8vL5uKbV//j/+9fSz0wRuNEQIL39vYenT55/Oidx0/fhZhVjVzMD5zFdQ+UQVU5ILTUUo1GI06F1U72anVzO5uM/vgnP/7Nb36zWt8KIQDymLLHjx9nWQaBowh+K/vrizPknXeqrStrlHfKaT0u0vEoF5xWXdv23XB1qQAQaWadY3HSKjuaTqwh88lRmsUpK8Bw6zFxHY5ijLDtulrwLI5TSpI0KYwpnfVt297c3GCCkyzuZb/Y3/PQ/z//H/+v999//+j4WCnV92oymUMILy9uMMZtJ+UQjIQCICS1c87EGPuhA0oJDAZnu7LsZccRsdpJbbU2SluAOcTcIWWAWa3XoaRexLFDcLVZ366W63JLENBaQuDiNH385HjommHo9w8PbqUJfXs4i7I8yYo8y3NEKHAWY4IoIwQhF3lrMUSEkO2b1wA4hChjTMTJYiGNtW3b8zgSkciybH9/EfDVWCVVL43WzvbaGHMvPBJsIWj6FhOivMIYa62lUh5Y5YhreoxI1zXVtsxE+uj0NE/SoWshhJShrIikaSD3zJOy69bdbXVBnXMAOMxwHMcIw2HouqHHGHvMFFBK6WFQQyc9RFXVNFUdRRGBSBkj+05zgQG0chCEeKu9tV5L3dZ7s3Eq6A/ee4dYVZWbSTF6fPqoqipGcSKi0K44SbI0TQNWnR4dcxY1TSPimDPGGHPQpXGCMdaDHGTvvYUeMIJ4HKdJVOQFZyQAsJR6GBSCXkpJIMiLVDAe7A6EIOCFcwZCaKzu+hZ5UZZl6IM0nTZKy+22AiCsAWjH43Ge50II59ztzUoOum4lpXyQRsTZCHFrPQBwWzYAAGvtYu+QED4MnRBx32vOAcTGGIMpEYLxSCCCEULO6M1mIyJGCGnL0iaplipUAAIA0jh5+/rN8eEhxUQPMhIiLI02m82ub2+A94vFYjaeeAQRgCGupZQHO0LTNH23bds2iiLvYQhetdaM8R0WDsPAOZ9Op9a7YRgAgs659XqNEBqGwSgdpteu6wwhHoPQrjjAWJqmRVGE0BMAMPQySBrhq0gIBUKE9vxxmgAAtDF93xurdnN6mLIJJs7qPM8//PDD59++2G6rvpdxTLMsa+oB4X7HNoKrwHsvjR6GIRGRIZThOxcnwcQbO7Rd6PTsjUUYE4gowk4bq3TQEQMSh0Uyw1JnFxcXoefgTgkPSgOhACEUx/Fd4nII7WdsqCgkBCVJ5D0MPb+TJAlFkgEpwykZ7YKKgPGdwh+gPUj3InTLJiQQO2l0kCh26QCEEAyUKNRTGhNEnVCiyXlE7jZ3ZxMxf9CpaXctcRxHSXInhIC7JR4C9O5IjHPOaPUAd/+gjfT3YDhs5n6tyB0hePjqDvL/s8AP/lFfqSCGfe+NO5LxkCuA7/Ve/Eduyh3h+M6vcF9x+pCuWUKs0oNWvZQCE4QwxNRhSASPlsvleDp78s47jx49hpj/+jef1VVDKbUO9H0PcUi9COeMtRZ42rXDxfnNarWaTFYHB3txHKfJ6Pr62c3NTVhKzhizv793dXXJOYM0H4/HkRAEYdX2fdtyKuIoYpTY8E11Rim13W7rttlutyKiJycncrDaKITQ2dnZ//w//dVHH3306Y9+/M2ziBL+2WefleXqv/s//G/3DxYAgNFkpK3ue5lkaZ5nFnpIEEc0SaIkiTDGjJEnjx//k5//LI1ipdSr3305nRZFkVxcvr29vZV62N/f3z/aBwA8fvIEYzr0crXc3tyuRJR5sNJaJUlmZz5ifNOtvYcIEQJRud68fP7s3Xff/eCDDy4vz1+9eiES0XXdzc3NdltWVaXVMClyLRXnfG825Qx9+83XbaPTfKSF6Ltms1kJRo5PTwalByUHKXtjAaFI6VVZszfnfafSZOyUgo4vxkeCEmtMnibQ26HrEpFmeYJRFImIsf75ty/KsmaCPn33ncePT7flcrE3PT4+juMUIbLZ3KVORlkexxEAgDNWVVVnBgAQhMFFpQAABUkuVjeUkv3JVFt1u1lzyj790Y/qfpBKvz6/uLpeRa3K8kI76BHNiiRN0yxJoyiRUt4sl9v1Oooi2Xfz6Xh/f8E5s97drlbWmdFoJGgcRZHUarxcKq211m3fOwAA8FIrrTUjCDgPnCMIB+hyzi2Xy/V6SxgPOcigdiqlrLVJksSJaJpmW667rts7OjXQK2sGOVBKWRJxTr33POIYw5D+VFaBDnZd1w2DMZhzXqruenOrRuaJeBJPUoedMUoDW6sGeuSoW6/WZ+vLzg92s4HQO+dG43w8GcVxXFVb3jSIYM65Na63KrS+xxAhgGXfJhEnlDbWtl0PrfMe1mXVN21Tl33TKD14az7+6Aec83efnDbb7e3yejKZIOCB91rL0Oe4LEvOI2PM+duz9WYrovjo6OjNmzdK6zuPHgBa67qum6bZlpvFbIoAJLEQgodmABA4p81mU7Ztb60llGFsKEZZWkzGBfR3EzFjBGMcxUFJdsGBRCmN41gIEepNRqPR5aV2zs1mszhOAWj6vn/58qUxhlLWtfLq+pmHGALy+tUZ5VH4Eu7v7//0pz97+/Zt27anJ0/KbYUhirM4LNId9OQQqCmji6IoRllYiWA6nVprrDZhrp7P59eXV48ePVKDXN3cpmmKIQKUjsfj169fl2UZok/tbF3X84MDYwwhLIAWIUQOOkCjMfddgO49Ct77oljc3t6GlT9DrRZh1Prv9sQYCyEMwpRSgrG9X4ZAKVVVVciIh94DQa4PDokA5xj70D8bIZQVeSAKXdcB6LSWzkGPYPgIzjAELs/z6XT6n371txhTznnbth6Stm0HaQjlEGKPsIiisLJir+QwDEYqf28/hBBa541U1lpgHWNMDxILQRGmCFuIgHVBRfD3jQTCyDDGdmtn+/uCpqDAZwmz1mIInLHaOi2V0dp7TwmRvTJOG6W11nVZYYxDP1PG7iLsO4+k3dUH3kXb905P7L2nlHjvAynZdVKy1urQF8GYgHAQftenCNwLHg+dAYHiUEqDGyToE+FmBdIWWieFodtZMcADZ99OOdhF7ejeCYgxDu1E/YNVIsP2sNPD9xINO8D+z2oJu8c7FWF3IQ+P/73D7rgCgg9Ywh8e86G0sHvvbhghhP6BlZJFUW+Vc85DQBmL4ljEEaOcuLydT4vWbC+6Nwuxt3+w/x5+/9vnbyGmPImdA9bqOI1igq3VGOPSLCeH4+NkMWuSels23a3gk+fPXvzd3/6DUur08aO6rl69Uc9fvwAYCSEEjKfTaZoXNzfL2ewwH4+urm5EkmAIPjnYx94hZ/48jryRn//+d19/8+vJ4tH/7f/yf/3gRz9ct73Cjs/yZxevr7bL4+niw6dPOQFfXm9OZpM5z29fvLVV1wNCMFTScw6dpdty/RaeA+Trujza2/viy9+dHB8hXWdUPz7IrbXrN/TXv33R9/3+wRHnOEmKw8PDyXjy4tWbs/OrOI6LoojT3Fm9vL3crK9H8/fKstSmvLm95JRc3a7Xm0obN51MiqIot+1XX35bbfp6O6jWTqfjySi5urjOk9hYqYeeMhyn0bYtiUIkiYBRCrN4Psb9MFi7New0GaMMtW1bVQ2jbDbfY4y3vWRxcmXZZ199/cM/+iMieDYvuqEfhmE2Svq+T0SWiPHbb9+GxCqldH1zEacJ8FLJinFYTEafffXlwcGByPj/9Nf/fjqaPn78WMR873Dv6Ojk4u1ZVVWRSL755pu+GKSUkYg5paPRiDktjWybrdcqHxWTdCTSZP/4eLldX94u+75Ps/jo0VGvpKyq0yeHxWw0KUZVVV1dXwxdr3QjTWu6gWIsnbpcL9M0h9s4mZ1oYzRC2+HiprbK2PF4Sn08GM8I7TtJiaAk8GjjnIYQeGC97hFwiGPXmtV2DQBiQngPlVfAoaFV0losBCSR9w6SzHgXQYKRI0nmosR65z3kNC5GIw/BoJV2zjuCqKCxQJYa20kJnSfrdXd1WzqAL5a3g9VWyWEYKOWtchJEQqTGzKGzMV/AyFFKnTabzarvzIfvvWMWsuu6vu+bptLYTLJs4+zQG8pxkiIibX211Eq1bWutVXQTigua9Zoi/Pz1i8VsDiEEg336zmNVyZvXZ8O6edF/s7+/gM56azABWktCYdtVTVN5j60ByEEEyVhkJYRd1xGKIITAGY6jdDZLI4YQevrk0fX1dS8HYfjF8oZSent7q5wVUdT3QDlTjCbT8SRLEgi9lurycpPEYjGbt3VXbjRC4MmjD+v6VmvY96asb8bTx8pAiBlC7u3Z8yh+39jBOrnYm719+1ZE0WeffXZy/Gi+t9h2jQZGIUtG5Ojx4farLw+PF+88Om3M5sWbL2XbJfzRyf5MSxXl6Y0evFUYEAKB1sp7TzAu8nw+nZdlSTHT0nAabZpNnufz6ej6+vrpo8dqkNbayXwmu14avVgszi8vHPDj8fjg4GA6njx//jyJ41lWcM4HZZbLZXDRxkKw+RxCKLXFGB8cHEgpN9tyW5aEkM36pqmrYjwq15vtuqSIURwhgh1zBAqRxHLoq205LrLTw4NtuYYAI+Occ7aXgPqYcc55XTZt34RClQA5ASOZgB2kCoM4EZAJa3WcpWM9DOc1sKrXxiDIKO3bluLsw49+dHhw+vXz57ebLSGEUt4NstdKGZMJ1Ko+EgmlxDrZShn6GBFCgA2BqZdBUQjmQWtVVRFGAQDbrkUIGS0Jp8a5ZpBxXkBEGI/yYlxWzTAoCHHfS4pJLKKQnTFSYYgYpKF4LY3SJEmapuu6IZQKQIgJddqaTlpjnANUW78sWwKJByB0IAAeEET9nfZApLlbG8J6F4yHEEJHKCTYIlsPNQAgH6UAoF726MGijsBBgigkSHsdU8A5t9Z2UlupelBBYyJCLByklBCALGLOOSkbSulslA1K7Hyy98yYhfLIuq4hwWGxwGEY4jiezWZ91+xMnRjTXX4hkAaEUGjzEFQN5xyAbpdA8Q8sC4GjeP+ddLFTU3Zgv2MJO5jf+TSDL5VzrgaJEcYIAGecU96b0EPau7CgtAPQQEghAoRCAKAwygLgzGA9jCl3CFnrMeVAO+O89wZixCkm2AOvgMG+kwlmxHoBGcMUQix4pJ0nqgcZiRgjRqOvvnz+7Nml85iymDE+KF3XZd/3Ug5ltdFyyLLs4HAOrGmaptmWwILDxwd5nNwul8++/TrLsv3Dg9vl9fOXL6SUT95557333iurTvdy78CnPBpnOfBAD1JCdPj4NE9iwSkCADi1vLpcV9Xlcrvcap4k+wdHpGq+efl2u906BSJMg65+vDf7+c9/7qAZTcZvLl71SvOMp2kKSbhPJji0274JN342W4xGo4O96cX5FQawbZub5e1sb//999//9NNPm6bZ1s1isRiPpv/kL/7yq6+++vzLr65ubg4wzUdFmmfKaK36tikZAo+ODjDG5WbrrFay//gHHxAMX7169ftf/8Nmszo5XLz33nsIocV4Wtd11beck1YN/bbG2BejDECQT0bZuCiyfH9/H2PcdR2EKEsjBMlEG621EBHnvK7bTdtdn59d3Vxvtqv15lYMCU/jLI0oRqpZt227Xq+rsiw3VZIkoXNtWNlv6GVd11VVWXDXKf2f/Oxn3tq26auqur68evn8xcHBEfLg6dOngkUffPDB4eFhU9feAc55URT1+dlkfzH2sBuGVg7BCtAPnXMujaOD+WJTV31dLddbpZQZjbuOrdfr7WptrBqPx6ePn+SjcVVVzjkaxYxzgGBZV8vVxgGfZVlrewDQMMhtVVLCESJ936/WK+B8xHgcCYqhU0oNnZKDMerjxVGRj+eL/eVyfX5+vt5WiOA4SpxzUZpgZZqutcalRb5YLIqiiDIRBTMzwcYYqRTECGLUy8FYa7yzxnsbuskCTAllofpfLBbzx49OFouFVnK12nDOvVcQoqZpCGHz+dw58OL1K4qglD1FOE3TIHVwRiaTSVVV6/Xy8vrKOAcJDLkYzrl12mozDIPWCnngCeYR5zRJk6it6iePT0O0N5tPilG2XC5/9atfAQAWe7NQqVtV1cXFFYQwjuMkThEiZVlrrdM0Z1R0XYe8jyIedF2tVTDYRy7CGJ+dnY1Go5ubm65riqJ48+ZNlmXAOs45gl4ppaWq61r2rdWmSDMAXSiQCylzQkgURTfXYZbERT7O0qKpO2eld/6nP/kZ57zc1hDgSCQEM+BRHKVpHBulp5MJZvTt1cXrl6+ttXmUVKvN31/dxEw4Y2bjCcGYEbo3na2aKkBp0LTCfBrm3LDuRjClhqATQhgaPwMAQr1VeBycaJzzyWQShLGQXw9G5pDzDtWzzjlEqPeec960fciFh6gxiOQ7aA9WfGststZDEEJeSgmCIERiAALGWOjEh8KCyJiEj0PobknSKE5CE4UAGFJKEhNKacQ4IcQaBSGMomg2m93eGGA88A4AADEVIkqSNM2zf/jbz1ercjQaIYwwooxxQkUUZQApCIlSxloPAUYUAgAxJh55DwBGkEDgvUcYW++scUoqjkAoOHTGAo0xIYzz0AbUORcsGuFUTViECd2NuXMOeRC+GyJmweEYFCAhBPAIYtT3MgTTu5DaA++cs8Da+7L/u+z4/apO/l7wh+C7fL9SCoJdQUGIgBEA4L67UUB3uIuJu87sxjy8Hd2vTOHvqyTgXedH0/e9Azh05kAoEDxnrZVa7Roi2fsVFowx/n5R6TurI0C7QFwri+7bLIIHEbnzfxD3f0/w3xVH+Ae9F3ex/kPxYKclfE9+wPerR4IHfaAf7vDwc8F9twQIQrfPu6yEf7Dzw9PbKRkIoSjmcRzhu8pbQIDkr66vN9sqSYvZ/lEv7fnVVdMNaV5YYKXskzR++uTk6PHhqMhGo8JL7ZxLIlEkad+0y9XN8tZjhv/3/93/brlcchaNRvkoz3pJtVJffPGFbzRjbHO7Ojo8yaIYYsoAoBA8/+bZfD6dzsYEoUF2t5vSEDY7Pq5X/fHJkydP34/W64M3Z2Xde+3GWQG9X61vU05+9OnHxgzamFdvztquG8+Zc4ZzmuWjQavleuWci6Ioz9Nyuzk+Pp5NR7Pp5Pnzb9XQee+fv37TdO3sYM9CND84HM9NWuRRlERRtC63by8vXr16XX/79Ww267rOAVuubnTbYAgZBsDqLCYI0GZ1a1VzeXX96tm3SstREi0mo8Vk5L3//JuvqqqKIj6e5MYo7/Rib1JMJ8+ffY0xaprGe/+kqR89esQj4ZzHaQEAoB4klHLOjVTter2pyzdnZ7erlXPGaKkQiGLKEOlsD+/X+BmPx97eNfeOomizLhHBCCOt9dnZWdnUhNI4jjnnk9HYaR8gpCzLtqofPXq0t7c3DGoERvv7+2oysdaFqSFlJOQF5xg774UQVERK6//4i19oazdlKYQYz6bU+9Vq06/WGtm+7z1wlEfaeh7R0WyOubi6upLWYQA5ZdqDTbP13rOIWefrervebKfTaZrm2rhIDwhBbY2FxjljAehlW603m/WqbdvCoKdPn84nc+BJ20vKY23MoHSUJBjTtt9cXl4ChD8o8sfvPEqSZLNZQggRwQCAduhtXUulWjlIo21I51kPADChDO5O6vOU0sVicXp6Op1OL87Ot9syyzLGGGN8GIbNZnN4dLK3N79eXmstMcZpklprGSPOmaYZ+r4Nyi3nLGY0TpMoipIsK0aZ6UxVVQiBWDAAAMVkPB7NJtPL8/O+refz6fmbtwcHBx9/+APv/S/++j+kWZzfb5zzZJmkSUopDXMCY6LIiXdl27YDUsYYHgmEOISwbVutrHfGO2O1EYy2bQsLwChumoYzxigVnNdlJ4QgXEAPEHCMIIKwg1aqHjhvjOn6ZrPZrFarMA2NRqOmacB9Pf0ddiIE7+tlAADvvPPOaDTabDbGGKvN6vY2ShLgPFQWGLu6vF7s78dxXLW91F3KI06obLq+bWPGMcZJcrc+S7DcY4zDai/h8c7PH9qrYIxCMNA0Tdu2SZIIykKyfLNeh5UIQgQmpby9vaVCJEniANrlgL21AIDQvjM0TQqJ6jBrCyE4H0I/hp32HoI5SmkUCYxi54x3xty1H8CBx8RxbD0IMn6IIMO0HkJGY4yU0gGYEXqXj3deSmmVZowVRbFc3UCDnHNKWwDMoEzby7Lurm82EHLGUkYjD4x1BHhotKckstZbbYx2mCLsEHQOOmigAwA44DVwHnjjvDFGaiWVtATt1n400HuCBIaJ98G70HXdMAwhXbJer8H9qkj4vjFRqFPl/O4adwJ76An4j2V/D+4sFw+Jwo4udF13V7SJcZDN7yHT7nD0e5I+gvieU/qQsNgNb/BXovtVoQNRCKmKcHfQfaPJNM+CgREA4JQy9+tr79Bz16FBa20CJH/n5fSh2iJ8EL5r4Pjdsg7gv2Ah/B5R2HEC73044R1ReHicoC64BytQw/uciLUW+n+0QOUD4Ifgu7cEohDyKkG78Pf8A+1uwf1mrTWq0zoO1NxarbQkhpHDvXcnI13WrfbOId7LUjuKBK56aazebG5ZiWeL8Qk/st6eX579+L2P+75f9UvCyN7e3tdff/3i+XMI8Hxv8d50Opstkjxr23a1Wh3uH+R5fvX1WdN3Z8/frK9Xt9fX4+mUcuadMdZu15umadquXm+3bdtaC0+evndD13E2Wq7L2+UWYz4umFEaAS8Yt8NwcXE2GmcfffJBr9Rg7DsffFAZuFzdZEWxf7Dolby6uRyG7uTR6f7+Qskhz/MiH2231atXb6Dzp6en+WTaDPLVm7P41/+wt7c3n+/N9vb3Dw6urq5EFO3t7Z2fn5+dnSklsyzLkqTd9PNpgSFSSllrDo8Okij+4osvfvfrv682W6XU0eF+zAVjuKs22+326+ffzmazxXhkobvdrIHT2svr5TVnZG8+LaYT79zB6fHp0yeU0r4blnVvjMEIxQApL7u2LdtOOX+9vNbGzOfzNI0pxZwh6HVfb6uyads2y7KTkxNvwfn5eWiDIwftIYiSeG9vbzQZW+CV1k3THEwXT58+nY6mWuvJaHR9fU0JF0KkaXp19eL8/LyXQxzHECJOWVVVs/l4uVymabq/v2+1vl4vGWNRFD158kQptei6JEmm02lVVQKivu+brm+bhnPunb7alJRthYjLpnaQ9NpRD+K8EIx0Q1tutlVVYW7Xq8319XUoo91syvF4fHR0dP72bGiV7junZFs329V6s1r3bXsTjfb3juLUWgD2Dw6zrLhZ3n7+2Zdv357HaYIxPTw+5lE0moy11pvNxmLovXdahWouaY0FHkAAMUIAOuPDUu0U47sZQRsAAICOMR6c8BDiJM7koJ0FeYaNMev1JcJ4NptNp2NjFQCAYbJcLq21WZ5U2/Lly5cnJ8eTyfjo5IhFLPTSEUmc53m72mrZWY0gxNYYa5V3WjCMMcizmGHy9vWLxXySpOLs7Kypt1mWiTiGGDMRH508ms73AjA7525vVxzT0ycno6o6e3vhnFscLNZdG2YGS6CGADmrurYtt07JPM/L1brIcuwAduCHP/jo+voae+eUVEqFHrQgTiiFgCDgvHNGazkMgzHaIwAhtNZE0STk8rfbLcYb50Cej5xzZ2dnhBClVFU1VdVkWWatN8at1+vQV1QZ44B/dHB0u1x2q+0syQ+fvIsgbKsaGkcoZYSWZYkSsatPC0QB3XcJDL6BcJsC7gIARqPRdrsFD6zywdBXlqXWOsB53/d92wX0CstdEiaEENPp1DkXsDMsJpemaXBgSKWHYbDWBkRs+64fVFjsIESofd9TStM0juPIOaOGzhlLCPEOBrDhnA9K930fyEc47eG+28+dpIywGnpHiOLMUa0HqbXCBDoH4ii1lBmjvLXeu7ZTV9e3AGLryGgyz/IpIcQDqbWGAHuHAcTGKK0MAAgBDB1y2lrlQIysczqUFjiHEHIIAowIZ4hgiBFBFHoMETLOSq3uxufeIxJk/Jubm2EYgPvOPUDv6++VuqtRNMaEfpHeQYCgte4hNCKEAAQQQheWwXhYKXBv2XtICOD9CguUMe/+ICYOtiTvvXX3Cy/dr2F958OwdscjQ2mMECKsg/XwlML3ZDKZBJi31mprQbBSIhqUqjuLjFLhq0jQdytfB7Oquy/mDI2gwP06TPC+h6Pz33kMd9f4MHbfXVf4UX/X/emeHIRtpzcEorOTCgghgb+Cu6H7TqL4A1HBP5AW7onCTlSw97cDobuVO3ZcwaPd8mAeQu986MnmyXo9ECaUIbeb7eCa1pjBIgOh0noYJCB8tpjn+aiqmo0eEIC/+dvfxFn67NkzpdTPf/7z0XS2/E//sNlsEKGCRW/Pr9q6iUR2uBeNshGC6NNPPw2OJEgwIQRiVDX1m5evD4+Pzt6+Xa3XTdcb7wilIomjKCnGex6Ll28ublbrXupERIhBoAyBAHHa1tVnn/3un/2Lfyqi6OTJo8XJ4f/w//13AHiEYd+3ddfufqgh1jHaRVHy+e9/V9fter394IMP9/cPCWHGmKurm+vr29nsSgixt7dntO67zlmbxGKSZ/uzaRRF2+UtH4339vZExAVlWus0TijDV5fn29U6inmeJd7Ydbs+OtyfTkZD3z55+s7h4eH+/sI6lSTRenO73txslrcnp0ciFsD5pmmMtVyIw8PDJElu28FBgAAw3vV1u7y+Wa9WWuuwoMbJ6WkSJ5jhWETGWcH4681bhNAwDJeXl9fX1+FHu16vrfFN1w5Kzhb7Qoj5fL4ty7quD/LpfDSBDi6XyxAlK2mCGBvsV9ba2WwGAGRCVFXVqlZK2SnZDL0eZFjRZzIee2BHRba3mHGKMcYRGy3GOef812/fnBs3SOmQJ4AwIhChCNIoFdZqhFmWFeMiqcvt1cVlW5d5wZmHHBMrlZXSqcHK3msp21o6562Dxnnr8zjJeOStOzp57CGuqxYiNsoSEUVJNxT/f7r+s0mybMsOxI4+V1/XoSN1ZlU9/YDu12hMA2hg5iuNf2HG+Ov4gUYBI402BAYEehrT6O6nSlfKyNAurxZH8sNx94x6PXRLK4vy8Lh+5dl7r732WqOhJaTve21VMkjTNMWE5EWhlFLkT58xwuj+HU20AdahzSVCsu8l0A5NNEZlmwJqSAh7/Pjper2uqkopgxlVSqzWiyD0kjRyAnkUwSgKMIYIoTiOptNJHMdxEkZxbBEs66ppqratjVF1vi7Kdde0Dnnu2hZqEQWsrrKmqhBCnKEw4t9/99X19fXnX7yQyjx++gghEiUxxKTp+iIvEUJxHDetkMpyr5RSUc4YY0mS1KpXSkEIAp8jaCGEQgij5XpVEQwBAIHPPeZHUeTzoGv6xA+rqsqX67LKfd8nCEjuOSEAY4wUvVZSKeVaD9SjWZY5Lp4L4ePxOEmSPM+n06mjvFVV9e233z579kwp5fu+EOLm5qasK0ppOhxGUbABEFtQ58VkMIQAcsqiIAjDMI2TPM+XdWGtZYy56+UEkuu6duWgu72rqtrDGI6wySkbjUZRFCmlZC+klIvFYjadDgaDPM+/++675Xwxm80ODg5Go5G1VhkDAPB9H0IolHbdOuYFLh2x1gKIHGTS1h2lVFe6rmtpLETYFa9ZllFKlYqR80SWn8bonQ6B+3mLVBvgClwLoMt7tks2Jg401lIigK21EACjbN/3o9EEAAOsVUr0fW+Mqpr+9n6pLeJ+ZBEGiBEGhQLc8/wwFm1nhBZtBwCCxkIGEQCcUusRp/bo9opSahF0KkAAbU0HMCX72tdVxvs4ASFUUroA5mZM9lF2j+1DCPe6TE3TAIswJUoZo60BFgC9r4m11kbvA/2PamWEkGuPAgCAhft9wNiaH2cSxlitNefcyVzCB6TX/V0BPykTA3cXOT4p2rs/g60APKXUmYRZazGlO26BbdvWWssJd5932QClFKNt3qOUslbBvUwCwtupVAD3dAQIHSf4ky7TnyQH8EErwe4ZErtzsifGwh+7ae+zBPCwswAgAC4ZgxDCPxFScL99kDfsEJr9B7ZnDO6hO3f2fJ9rQRAGAACILEKYEEQpJd+8fh8msTC2VZr6IaFAdLKsquPT48164VP05PyxFvBv/uN/zZb3J8eHRKIwDLu+55x/vLjLsux2vg684G//6z8+e/LEGkgQVsoGPDyYnBwdHc0Go8FggBDCGEopCUUIoTdv3y5Wy6uPlzcfryFlk+mM+V4v1LrJY39iEa9q0e9UUBgm3CcI2NDzoFV5mVkIV3mhjE3H49FoQCmGhH68vCjrhnECAMiyTMre9auSJMnzIgoTNxXWtb3oZV2W7jIs7+f5cv3dl18/e/ZMqv7V48cvzs/n83kUBRDCiLEf3s0pZx73P/vsM6v1mzevq3VFCKO+39VN4IXTw1m2WittDw6PX7z87HJT+r5PKQ4jj/7spx/evf7t7/6bVXI8HBGEN/nm9vZ2vVopKa0xz54+9ynCylqrdK+W9/dvf3i9Wi4xgP/yL/87z/OCKHr37h0ASBADAEzi0fn5ue/76/X69va27/svvvii67r3799ffPgAEExAen19vdqsLYKUsSAI7vFdEIZlXiznC8bYeDzWISjL8ssvvzQGAIyKuhpOxlrpgJDheKSVHA6j+Xx+e3v79OnTn7982XXdajkfDBKKCcJASZHndVEU2XpTFIUgvMmLqusboauup14wGE14GGHAjLXAQIpwEqXT4WgeR2WRy7o/mh1wQhGCDKDE87CR+fxO1AWygALiMT8dJqN4EEepx1jA4yzLqrqP41hIvd7cCa2evXgJIXz34cPd/N4N5rjbnXme7GpCKd5ThCBQSkkhGGOEUEAohJBzTgghAFmpsYAYY4qgErptWwzwMB0cHRyMRpOrqysAQBAGCM0sNMaoIAzc0O/zJ08fPXrU1JXWihDy/MVTt0Kt1otOCLPV/Ndt26qm1KpH2GJoMbKcIsoQJVCJ9ubmkmD8q1/+7M//7NeXl5fBip6dnfEgPTs7W6+z+/v7i4vL+7t5FCWTyUwpkCSDuq6//fZ7KfuDg4Mg8JqmQkCLrsYY+77PcKC1xsAO4qiwIFutnz59qoUJguBwOrt49160XRwFZZEp2TNCA8+nGDGKMWZKbaWBlBJ1XZdl6ZDVyWBMKZtMZmEYO8CWc14UlbXQGJCmQwjxfL6cTg+stZRyhIwwWmo9GI04585VYTab+b6f53nTNOPhKIxjKaVQUkMAAHBdfIchuxS273uXHHC+pV84bNnzvPl8vlwunTEjpbQsy6as6rqeTCaj0ajve9dBd6GIc35+fp5l2WK1cYrICCGpjSsNITQ7aV6t9CdxnjAMy7rSujAWQLAFDHzfd7baSimtZd/3ZVlWdZHEg73l8T687Rd9h3DoT2YB2uee1hIYDQxCAFqILTDWQsbYbipPu70CAEgFPM4Z571QCFOMsZQ9pRQhgDG2wGijtDIIWoKhU2itgEIWGIQlxtZaF/+VBUoLKbXBmBHKELEQaK0tsIyRh8U9pZQz5uKZo4tuuz9gO3PvjtTJEuxo84gQ0nWNNcCJiFtrjQHaGgCA1dbAbarxsHjdt3WstRCifTjUWmm1/aJdv8BIKaMoMnob3h6yT+I43rMKtvmN1o5VsC2RrXVH4ZIkR3xxJAzXR3A+XvucY38qdlxF4ZgrlFKtrZtegRACi3ZyEdtjcVdfG71PC/Y3A/oxgfEhALBvJex/6/Z5P3axRyPsrnfm9g1seSBgn3/86PXQHuIhxvBjaWeMP81JuvcRQpi4KZ5OCOZzBKFVWpB/+Oqr2eFBNBhCyvqu35RVp3SUxBeXt2W+8RCkFocMrW9XXdV2QU8Azdbzn//859OD2cWHqw8Xl1qhOBlZwHoB//j7358enf70s89Pj09E15VZ9fbt+zzPLz+8b5tKK/H06dP//q//Ogj8L159VpdNW7UG4SgaCG1qIVVvayCFhnVb9aq3QKu+N5RQxtyJS5KE++yHN68X+ebN5UWcJtyjdQO6vp3P5waAdDhUKi+Kou9bj/MwjCnlwKIoCtqqev/uQmpQZkWWZRjjwGNN182vbr7+wx+TJJlNxn/xF38xSKLr9+/dLVht1tyPMeObqirrxlr74eO16JpOSs8P5qurKB188ZNf3N5eX11eNr14MjtA4eDo6KhpK6X68Wggu/qbrzklpMhyBEGaJLEfKKWKTfbl7/+QrTZJEpZlDYyllNbrrK9KYmESxeenZ0EQFGX9/v2F54dhElNKB8PRwcGBkwmjhIwG8NmzZ47wdfnxmnnc9/26rlebtRcGk+l0S8kxJvSDQZICBH3fr+t2tVpVVTU5OFRKLZfL2eFB3wllNCEkDcPxYFhmeY/w47PzJ0+evH3zQ13XZZlrpRBChGDGmBf6iZWtqD98/4EwbpXuyiYvq7y99aP54fHJ2aNzgggFBEiDtB3F6fPzx1KKrs7DMLR97wXecJjezoXRKmbMDgYYQIZZFMSjeJjGaeRHhBANqJSq6zpljGpE0/WUszCIsrIYjIaUM2W0MaYTomoaXZZRHCAIEYTGPXXGWqlk2/V14/hlAFgoFMOUeAEbQgaptbYjVPYKQ6KUEkIZDSjhg3SkjaQeRQg0Xd2LLojDoijyPMMEHRzOsjWtixIjEATBcrkUoqvbppeSB1vMvGlq27cQWoIhhIBiSDwWBb7nselszCiOw+jf/Jt/nUaxz+jRbFrXdd52t/d3FxeXl5fXeVZQ4p+eTaezw7puo3hI6Ga+WDRNRxiN06SqqtV6XhU5pZQRrJRp2xYAEPoBxaRpOopJHHppOuza9v7ujhDSVXVfNwzBdDiO0sQR9MIwxBh7HnOY7Xw+70VLCPEDniaDoiiU1Jx5wMI8z6Mowojc3twJIV68eHFyfHpxcYEgNtYwyjvTxKNBEIbhIEEI2caGYejUuhxYLawuu2axWLiFyQV7FwlcBx1CGEWREzjaDxk6ho2raN3S5rjobkm11p6fn3PG3r9/zzn/5S9/OT+8NztHADf/5kIvQkhb4Frv2lonA7BHFKy1cRwz5gkl86LqldbG7mlATi+hqiqtJTDK8Rg4850KEMYYYrLNbEy/zUsw2YcNVxwDq60xWimjnX2XxQRBCPteEmIJAQAYABBjWxMKL2CY0TLLPI8RSnspcN9K6QNgKIWh77kzwyj1fR4EQVuWACKKiU+Y1hoBpJQ2UmILtLEIWqC06renwtEDlVIAGHcajTHY2TcbA4zdN18cf8VaS9i2RiduqoIQN3ABIUQYWQishS6Aoa2foQW7ih/uSIXAeZXBbWTaqxFrrQE0Lu7vC3GXFX0KkA9GEPcokXayFkp5nudysoef3Ed9a62QW26sy+FckqqU2mkvfhKDcihCU9fuRt2nI9sYvwvq+3Rk2xrQ8GG++CMM4MdTjg9zSr1zsgA7/IaQLTH24f0DHthRQuQ0G7aND7Pt8Owyhh8nChYACJHLFdy23G7vj3T/FUoJhBCEVqpeyh4AzxjTtjX52a9/+ebt2y/fvE6Gk0cvXkSDIWwbCxEiNAqTJl9fXFw+OTx49fQzoCQEBgJa1/Vnn/0UIPjb338VBHEyGF9dXb18+dnp0XFTttCC25vFh3eX89s7igkMUZUX69Ui8FjX1BcXF+/fv4/j2PeD5XxRlaUGuOuktph7wXg2aS0RSpZNCzGIohAowQmmGGst+66bTafD8eEPP7y5Xty+u770omA6nV5fX3dC9VIEUQIAaNsWQAiARwlhjDkDNAdgvn79OghiAABByEhVdG2e51KIJIk1Z9lydfXh/T2lX/7hDxDaR48e9X2fjk8mk9G7N2/fvr8AAKw3OWcEU960vR+E48nUC0KAiQVovcn/+OXXWdWcnpxEQXh5tQg4i4JwkKbZZqWl7JuWIhz6gaMLlXlRpeXq7qooCozxdDwjhJweHMZxcnb2aHFzlwxH88Xq229+GE8no8nMD4MkHS2WC3e/zmYzLU3TNEVRuP7rJs9ubm4I86YHs9PTU0LpYrHwBcAADsfTNE0hhEEULpfr+/n8/Px8cnAIAFhnmyiKrKmKuhJCJN6j0+OT8XCUF5ujw8PNen17e+t5XlUV2phOdEVRdF0znU5Pj0+G4/EvPv8zgPAqL4pWGESu7tfXizllHqceYyT0fKuN7gUl5GAy9TiDWt3cXOlezM5Op9Ph3e0lMurJ+VkSRghCCqnHfJ8ECCCggVKmUvbo6KSq6+VyqYwZjUYG2PV6jSiJ4wQRvFwui6pyy6VbxxljBKL9cuBRhiy4vLzEBpAIGmMMkhgiz/PiMJJGK2mgAZxaaFHf9G3bFkVZVZXve5xzoURd15t8HfR+lEZpmjRN3bbt9fV1U9Vx4IdBYIxZLueuxqLQuDKlapqiKGBX7p9w1QtCSODzuswpxX/+5//8YDrzPPZ//3/8Xw8PD3/605/e391QP2naqixzKSWl3ONe27Zv374HFmKCtip4WldV1fc991jfNXVVMMY8xruuy/OScx7Hicf5IBlm681Pf/rTwWD0D//wD1Ybofo6X+f5xvVxGSZNU7vJgaLIXDeBEMI4iaIojKLpdIokms/nrqNvrd1sNqenp2EYWmsXi8Xp6emjR49ce9h19xfXi2Q4YIRKpThj6WhorRVC5HkepQn1uJRytV5neT4YDTnnXd00TWOtdUWqC117BR7XwUU7fqJTNHKiDs4Cyqk7UEqvr6/Ho1FVVW7COVtvNpuNlPL6+rpt276X+zBj7dbkSagtBc/1CBwakaap74d124ThCnS9kE6wB9R17XkeAEZJQQhKogABKFW/p8QrpSgmrqvVbxWUP+nzuKrUQtRWpSsfjTFVWUIIPd8HALg4sW1SOFekLcAulBJtW+s0DlmIMUQYME7aqiYU+V7i+z7FxG7VCDAnFADAMSE7ejwUQlkZ+gE3WyknYKyyhhLie15R5Fprhokrr6WURmsXpVyCtW2QG+t2DxHQ931d1wCjhzMFhBAEsYU7SgGCwGitNdDAbn2Qt+oOZtd6sGjX0UAQ7nQRGHdTEdsuCUKIEOsuPQQI7QSe9/CPi/F2J5a1B+0d2cKlknumhTFm713p8AC7n1jZ6bIYY5yKA0JIOGtmQtBWT/qT+JLTuMT7GYRd+Henbo8i/Ela8LAN4V5qJxvlTiZ6QC180LX41LDYk2SB+aSa8L+LKLiX3X0CgE9ZgvsB/njYAeygFMda3bF/CKFIG0NmsPrn/8O/+Jv/9e++/OqHUvWTw7PJYFzXYr0podYjHAYeW95lOpUnx4f3N9fA4L/4i7+4u1u/ffv29nZuAZhMJkCDIluT89Ozx2f/+N/+3mOcUUJ81PUtEqO6Jb3kGOGD2dnzx2eXFx9/82f/zCL7+PykE+37y5u+VV484HHI06hdrYrVMkDAKMkVSdO4b2tErISWJAGM+eT8CEBzO7+2daOEuJUQk4ADFQaJHwaEkNFguM42Xdtyzv/h978/OztDUXJX1nwwIYwRwiil7XpddgVh1ASBRED5XuMF3mh0J+HybvndvObcX8A82xQHM6C+VR73v33zIUmS1kDOQ6sEBAARfPnx4n/OVxTa49l07AMfNDphb95+DaFNksTzKUSDk+Oz/+1//Vvf53Ecd00LjAXQHB7Njo8POaGlj+Mkads2y3MlpU99Tvn97d1yuQ7m8w8XlwGhIeWfP3s6m816qQ9++i826/VqtSDImx4NVqvV4n7JOeecnx4eNXWntX48Pno0Pl0sVuu3d2/lt9Pp9K/+u3/94sWLpmmElTxiyTDhATdGhaEfhxG04Pzs9I9/+DKJ4yLP5/f3vs9ns8kf//jb45OjtiuiJF5eznupjYZCKIwiPz5Kxk8498tieXx8jDfZEcZhEg/fvfPfqbIsTXP/8snPPc77tv3w9ffWwudPnx0dHM3vP0wHB+NfHQzHozBJN2tzfXuTjl6EQVyW5XKzuXj3g7X29OSEEep5nrKmqiohu+Vy7vucxX5dl51pmKZ902OMzw4OyiC4v1+Itg78kNGgKisNIUDIjWL7vm8hmN8tRoMx5Z4DhDshNoul53lUIiC1hwhgACEU+lwIcb+5bpse5MAYM5nMuBet3l5Upfhnv/pLhpYBD5u6e19+PDiY9tKsru9OTo9nBydCCGs1aBqPelrrD5cfsmzNAAEAzGYzrSWnkHPeZpYcJBSD2egxMvBv//M/5CtpRNbX33qet8jqKIqOTl9FaV4URVVVP7z/frVaHR4exkm4Xq+llIeHh61qv/r+m+Pj49PHpyxgTd2xgI0mw+PTI4SIzz0hxHK5no4HBNl8M8dQ+xxPRuPb+TVl4/v7+8Ewxhh3ff38+fP3799zSss8n00mTV3Pb+dJklBI7q5uZ8Pxsyfnq9WKYuB5/niYeJ73ww8/XF68Qwi9e/O91WI8Ht9cXXDOHz9+bOsOchuOotFwpKzJsqztOkopYlQqNRgM2rYFvTgYjpMwIoR8XK+rPGOMRb5HCJnP546GxhhTUjSiJ4RMR0NjTJ7ndZHzwP/ss89Xq1WZ5ePx2BnXEYRWWbZYLCmlQqqLj5frTdY0LSGEB/73b988f/4SQphl2XA4zIqKYmys5b6HnH8BI74XwBoVVZm1xTxfdlIMxgNQlLDufIQxRIHnO4IkIQhZZBBFHjaUa9FrBBACliDqUT/yIYStaLFxoLfck93cihx5SV3XQEGKqU8NAIAjbq01yFCyDXgEkoAGxpi6rM+OZtfX16GfMMPzZRZ5fhiGfVNzTgHgWuuu7zUFnhcYC8ta+LGzYyY+4EqppumE6jSQ2mKAIcIYEQwAxMZoY+qmgQpyTDHCVlsMkUe5Mbora89zg8pGNF1nFKU09HzOeNfLuuoJ9rgygR+JXuV5HiWpAxWUNkpJhBBlFBioGi2MJohYa50uEyQYQmggABjugzdAW5MFqaRBjFIKjKm7Dknl+74xpu5atmXNEAoJdCMJRkktbUUQ6j3PC9ORrZypmw6CUBrTCSE1AIQDAHoNTCeZgcOUQ6it1Vr3EGIAFSYWWKC1IhQIJa3VnkesNXWdd6LX2rStm8FhnEMIIYKEcdedAW7wUCnjioSqqhACFmoENQQG2J1Eo8V7RoGLzwAg94aFZoeawB02AAAAQiiECELOHGs74Wh2WgsOi3A5EGPM8zxrlBKS0i1SouUubaJESaW0RphSzgjGQhmhJLQAQmghAAhaBI21FgJAMVagLuvJy0eDQYoQIBiKrvPjmCyW93/93/+70XiMKfvq2x/uv1ql6UxZiyAZDAZaaiMgp3izWkOtgsB7dPKka9qsyG9ubu7v7621AJokSbRWWso4CZ49f/zNV18mcWyt5h4u67ppSy37aDw5OJxOp9PNelHU1cXFRVHWCoAXL174caotLrsuyzb3d3d936ZpysMkDH3GmBIdhND3/dPjo8ePH49Gg/u7m72cVr5aMcYQIVrruqwQwRDCQZJ+vLp08AshJFtvnDTsPtPCGFFKEYaMEWu5wz+FEDfXd34Yvnr1omm6sqxns1kcxzc3N3VdbxtgRvm+H/iDcZKUm5HHcMgJMmo8myZpCqxe3S4/fHhXVdXPfvrTOAoBMD7nR0dHQnST0VgIcX9/FychRbjYFGWZR4MhYywZTNLUGmWAsQQzhGkQhpuiNACdP3785NnTyeywaGoppUTo+voaQjscpsYY17i9uLhQSgVR5PGAEHJ0dHRychLHqRDi4/zjYDAIPV92fb7OXDP7YDq9u7u7v7m/vLzM1ysM4NHsoHveQAjfv3739VffRHEYJeHBwfRgdlzk5WqzzrPq6uYOAHR0eJoMYyl0WdYY89D3i6KoyyIZpG3bZlkGoR2Px5PJ5PjksMjKRVGsF0uC8PHhEdDm5Pjsw+XHNE3Hk1kj+8FovC7KL//4rXPqW9wvv/rm21E6GI0m0CcBZrqvv/vuO9G3s9kkieM8z1fLeZJEgACGie/7STJIksRjftM0jHJEkWNoGmNKQrbgJEa/+MUvlFJFUbimZtc00No4DD1I+77vus71xbeqhYQsl8uTkxPnKGitdaoJNzc3UezFcTwcDvu+xxA5/2IXhJRSnsf6vm+axtWgq9UKG3RwcMA9726z6bru7OwMI/T2/bvRYPjDm9dhGAKE/DjCnGlriqrshGWMuD6oq4Q8z3MmCJzzOI7Lsmya2gHpSZKMTqYnJydGAwghMFYIYQyQUvq+n6apR5kjGPq+72qR8Xic5/nR0dFsNpNSNk3jVHvn87mL02VZUkqPjo6CIOi6brVYhmFIONPArvMMQnhycnL2+NG7iw9lWZZNXTb181cvz6JHWZbdzu9//etfr1arsq4XiwXAyPGEXAXveR6CUEqpjdRa930vhAiCwJ1kN6MIIWzb1kks7zEGV59Np9ODg4Pb+b2rvQaDwWg0yvO8LMvSGGtt37R45xPh8AbXsQZgO9HnWPpt27qijcJtzdf3vRTKNTVcpMGYBkFAGK94I4SEmLovxRhba5zh4cMizKHffd87r4Q9E808mBvc1obaGKsYZZRiYylCKIoijLFbYZTSEFmIrDYSAEAZdi4MGKNtqmGgUsoY5z1BAABGA0KkUoYSZq0NWez2SmqllNHaGgOsgVIqgNGuusQGAGsNhBbuBCTc0Kk1RsrtmIYxxun8uw32ShoImq6Nosi1k/KsXK/XDjESQhBCINw6MO3LWvRgfg9CiHe2kBZ++tX+5KCdDOLD1MrdPw6id9sxO5cvhJDRdk9XNA9GGffsE4dOuf1x8M++4of73geAe9rEHmRytJhGt/sDcgil7/sYUceH3TFweynlTkoSPLzce+Rg33nZ/XZX8cNPRISH/YV/CkVsz4y7LLtEwe5OFyHEavMnCMEeingAMWwvhMc5QohSh6wYR35EW0aLcfoivu+CJtBaE4Q1ROqXv/jJcDj8T//5v/5///PfSlXNpkddJ4ZJuFgsqqz2fb/YbBiCj87OZ+PJfD7v2y6Jw2ePnyglCKez6aiqCu7Bn56/+sXPPsvze6OkMejJ+aMsQ103a+sy5BRCW/eVH3nj6WSdbTZlEUbx8fGhMDArSwQ1hhpj63ksTWPf9z1GIIQNMBiRxf088Hjg+Xm2Xi6XRVEopZAFEkBCMNSg7bpW9JSzJEl83x8M0iAIyrrq20aIPmIRALZpakg15wxC6MAFp+8bBEFelRiTdZ4dHh//5V/+5evXb9+9+/CrX/9CCZll6+VyzTk3Vjsn2TDwhFbKGqlsBywlgDIPUdaURdfW93d3q9Xy7PSEUhoFXhB4GOMkjp88eaaU2KyWw2QItppcsOrEgPlBkvjM10oZaaAFhLC6V9c3r4uqCuPYi2I/jjZlRf2AUnpzc1OW+Wq1SNL47OxsNBp9//33g8FgmKYIEoegrtdrZ9p2cHBACK3rRsrb1WoFIQyCqK5ra23dlMaYKIqs1QCYNI2bpvO8aLFYZVmBCJyMD+qq7zs7v8/axkgBlFJXV3fXV4vZ4dFwMJs8m8p+4xqZZVmuss3N9bU2BsLm0dk5tKDMs/n8brPceIxXRVZWeZym2aby/LiTqm3k6fnTIBr88Pr1//wf/tNnn30mu76quzRBq2X21e23GOMXLx5RQrrWvH/3Toru1auXAaN5vsEASm2UkMDY0A/gENSMAwAJp8aYre8c3PYLoQVJFN8v5thgionLllw3SiPmSM5KqTzPr69vHT0lDKPT01Pf99frzBjz6NEjsFN9qarKKK2UMj631lZFcfnhIk3TIssRhH3fl3mRpul4OGrrqq5bwlmYxGFTt12njaGet9lsjk5Prud3URT5Pg/TGAAgoa37tigFQshRHKzVEFrOeRQHEDkTXl+I3jX7EUJtW6/X2Pd9Rj2lVN91QggIMYbIiRFBCF30cqtJ13WYwaqqBoMBxthxv5fLJca4bVvO+Xw+b9v2/Pz84ODAtW+dUN1ms3Epqdugtfbg4MBNySqlsiyLomg6nUIIjw6mfd8v12vTWy8MMMbgAXbqFn1kwV48B+50e5wpjMuQXIwHu5k0RzRzsBmE0PEcxweH0+nUkQrdI6zFpxl6p6gzGo2CIBgMBq517ZoUrjvwEJTum7qpS4dCKymFEBibKOSck64VQshPYwsQGmOllG2L3HYY9RAklFIEkehV2/TWg0YDYBGwAEGCEYTA7PIGK7vOjd7tz4bLh4wxW8rCrjvucOysLDopCMCt6I3R0mjVd1q7I0UAgF5I0/WtkB73Mca22racldFdK/tOaGUgRBBiANButBJaAIyxxmifEifS7w4QEYIQdIfGFd0ZW2v4YNxOSrlcLuNBSghxNwamzGWumGwFmrTWQjulkm3SsI++Lmw7McRP2Pju9KoHgwz7b0QPdJTd1tyb1tquFcaYveIc3hlNuRvJ9RHgVo7MUQ6D/XgnRts7EO/GJR6Gc845YVT2yjlNbMciODfGCFH7Xqi1JgQzFriHghDEWJDn+T71QTvjKGuctNSnkc79MZr/PWPJP8kbHmZa+6cJwk+hHWMMLLHa7M2393/18KTtNgL2j4nTazdGWW0opb7vHw4CKVq969EobSkGCBFy+uhUyp4y9IuffzGZTI4OZt99+7oT6urqFpvWx5ZQGvgkZJODg1nI6dXF25ubG2UMpng8HASRX1R539ezg2E6CM5PZ57HPv/i2Zd/+EPbtc9ePf6zX/5rIcT8/rbYbH74/lshhIW2Fe3kYLIpiyhOuc/urq5X62I8nYxOD0POmqaJ49B1hozSCAIAAEEwW2/KIvMoS9P01fMXR7MDpdRSCillVVVS9qHH/Sj0A49Scngw5Z5HCPIDL+78MA4QQlXZ1UIpxV2GqJSgDBNCEQZCdIQYhAGlpOub5XKe5xtrdRR4aRrXde3W5b7vN8UGY1gUWV9XHCOPk0EUGgC0AY2U1pog8MuMbFaL5fwuOD8PPJ8gfHh48OT8EaV4eT8/Pjq6vPxIEB0kw3ndSgOlAlp3dVH3bUcpjfxIaqUAkFov16vlen14emKx1VYxxqbTadNUHz58ANB2XTcZDdM0dXaI7pGo67qua0r5cDjMmkwpVWSl7/vAQGNtkeVZlsVxHPqBd8K11l1bf3j/1hjQNA0hpK7bvs8sAN98/cP7d5eXl5dZkYfh8NXLA0LYarW6u7tfrzZffvn127fvj2fBaDIGAPRKirY7Pj5MkmS1zu7vb7USm83GKh2Fvs+4UmK9XFzf3i8WKwvROiuZx8+fPomjxPPCs+Ozk8OTm+s7owHGtK7b+Xw9Ho9F3T5/9GS5nP8v/+k/9E39L37z5wcvnl9dfey6LluvRd93deMR6nPPaiOENMZACxCA0Do5eoW6DiHUGGO1IZR1XUeUGg6HEMLr62vmYwBAFEXunabpuq6L4/jZ0xfPnz/nnPv+vYtbbdu2bZtEseyFBFJrDYzxOE/TNA7DzWo9n889TpfLZZ7nLhJoKU9PzwEAxoDp9CDPy3VenETx7Oj46ctXb77/wVjox0mvTdu2GBPEuLWtMQoAgzGklAJoILKUYaUkIRhCHschhNBN5fi+75zYKOEIob7t+r7HmAae3zSNtRBoY4xxtbtbT4XWdV27GO+Sg7quCSHD4dAxBJ0fkpTy7du3bdt6nnd/f79cLieTyfn5OSFkk2cOLk6HAz8Muq7Lipwu2PHx8fHRUV0WThUxCgMe+E3TZHnuntA4jn23mgNI0NZ0x1XSLglw8M9gMNBah2HoCOou1dNabzYbrfVwMnb5DSGk7/s8z4UQ/o7CBncCNQ6S9X0fABAEgStz3diCS0H26yZjDPed62e7GNZ1nVKNFBogXFetQxQc5YXybfjZztkTQim21jLGHJ+u73u4M5PcswWt/WRPrJFwsx3GYACM690rJRAC1mprNcYYIeCimnMiCMMwCP04DgEwQjoTJtO2LWMcAKCk6ftOKaOVYYwBigihCCGplMPJlDLWQs/z4E6BGEIM4NYQi3NurRZKSq0wxpwyyggllBFqLTNbw8St+qHW+ihNiqLouu6LL77wvdCloUK5afstNW+rJAEsBNjVo/sztq9r8QNHIrOznkIImZ3mhLs99lDBvvluHoo/WisFMLthB7LT5kIIufVwr7MJdyOde+7hwwobwK3Fhksl3cNCKWWcDwaJ7/tSSqWEEApCK2Xftn0YhsYqAKjve8ZoKbdTo3vexgNKwJ86O+xhEgihVWYPOTxMhtDOC8pxBdwHtNbGbk+OU7mAALqJDGC13mEkDzOzP/l2CCEAn9COh3gDpTQMw1cvH0nRAIwgRhaCvpeUUogRGaQjhzHe3t4DbX/1s8+Q0f/49789PxjEUfTq0fFkNuWcO7jp7du31apCUElRAcRHw6Mnz58IJTf5ejweYqhX6zlC4MWLp3/88h8vry66vrK2hVAFHp48O2UcVGW5Wq3my/v1ZrPaLKnvOfw/CnkaB2kad03di8YajSBhlBgMCIIYgul0ioBpmkYjnSTJZDRumqaqKlzly/WqKArf8+IkiaKo7tpNloVhiBHinEdBqJRy9KuGMgwRIYgyrJQSqkeIMraVdNVaB4EnRPfdd9/d3Ny0bf3999+P0wRDG4a+w2ndjdj3ve8xZQ2BiFDuBVGYJtFg0BuF53cUQ2PVcjn//ttvZN9hjH/6xRfj8ZAQoqWejmdHhycfP37MNzkhZDAYOAik77q+b6uq5pxjAAEwz188y7Lsw8VF05YWKMbwcrnyvfBXv/rV4eHst7/9h6vry2+++WY8HKRp6moyznxCCMXOwifAGDdKRBFL09gR2l2ZSCnN89zs+L3OSSiKgr7vMYrdo5Kmad9LY1CWVYR6Vdm2InPFuiv13r59vVgsXj6bPX369Oj4eDo9SJIEYsT9AGM8n8+ttR4js+mYEwohJgiv1+v5YhMEwXq5yosPx2enlNKLy4938+Wjs/M0Tu7gfci98WCIAPQpPz08noyGnPHA96MgLNabr3//R/vFZwfjSZZlsu2gthRhThlk2BpjtZVWMUwcOUvuuEgWAt/3rdYWgLZurLUe43GaxHFMIXGBKgxDrXXb9n3fJ0lydnY2GAzCMPS8QAjhbrbFYtGWmUPy1+t1lq+jIJxMJoezWZHn1hiEUByEFOHZeKK1ropsPBiVZVlscgBAlZec80GcTmazMIgwZZhS7ge4bnTTAoS5H6SpDcPQ8xkAgFAkJfU8vgtjCAAThmEYhtyjbuGm0N/T+qDnuTaKG6KBO+k3tSOoM0It0m4LDjsdj8cuNI5GIwCAmzWIoqgsy+vra7d4rVYr55KQJIkbQx8MBgAAF7BdIwZjnGUZhFB1rVJqOByOpxMDQV3XLiNxOK0UW7kqDBHY8b3dyWeMuSjrfnZyy27p3wVvtYcKnJrCYrF4//49pRSPRk4RwS2scDeXCCFcrVZCCLf/bgl2YQAh5GwLHPDu6lEhhNAuotuu6wDaqi4iQty3U0qhsy3YRTstnUQgBQC5kVHnsialIGQ7tevQYmOUlJoQsifN7YvOPSQO/4kegLU2SZIgDCFGWuteCCEEJrDtO4iIu7cNsMAYA6wB1k3FQwi1hZhsy3dsncK0w9CdwzKz1vZ9b7WyEDq8A2ggoDTA7MB5QBC2CBkLpNFSSaC3JmGuCzZIR+fn56vV6uLyyjlTS6XNTsYAEkwI1jvOHXxAndsXxH8Sw/b5gTsD7vZwaZa7rPtPfrqxrXNz3QbafQ29Xq+dYJfbFGPMGXu2de6+nRACdx2iPZzgM+bWDWflBSGMkygIAqMtQsj1lQBAlFIhOvcZz2dufbDWunxlBxj8CPm31roI7VIESrdZi9kpUe67FQ/Pz58c8h5ysDt/SPjgtf+VfSDkAP7/vBw7RO8Uu4HZnvbp9GAwGAFZI8IwwaIrNbBdL0nbqTdvPizm2XI+F10/TAdId6+enuweSzaMCUI2jQlCcn73/ocfvrXW1m375NnTOCII9MNBqDTnHun62mQyDMNf/foX7z+8dYP+r7//oxu2CaMjQqEXsCmZNE0zX6+80IMY9H3rcwJhCIEui1W+WeabjU0SxgaceVpjQhGBUGvFPA8BaK2tirJr2qqquq7Dw4ggGPqeF/jHx8d+GNzf35dlHgRe4HvW6nQQS9VTgoy1QeDFYUIoCoLAGNWJPo6jNE2VUkVdVWXtzMuzbOMHnBCyWNw3xWY4HMahX7W2KIokGRBCDARSq14ohBBAmHAGKYOUeX4QeNyo4Pz05OzsLIqi1WIxGAw+//xzguDdze18PndWK6EXdV2HEdXAaNF3CGohEbQ+xwRZCBTB4GQ06prC6L4uN02dD9OwLNaO8u0Ko8Fg4OwAjDFhGEopKeGEEI97jnzb9/1kNE3TOI7jvu+rol4t1hDaKIoQgEbpot64yswYQwkSove5n6ZhkkRRnMZxypi3XC6NBff3F7f3C4jscDgEwHZNRyk+OJzM7+6N0kKIg4ODeJBcXl73i4W19uTwKI5jbIwQihMqhRB9WxRFVzac4Jubm8vrK4YQQfDm8hJjnAQ+toYYlYTBNE2hBXkajZIQAYAAiPxgGCfvl4t//Pt/uL76eHx4MJvN2raF1hqpurqxGrR13Xc9DrkrIzxrZddjAAFGxpgoCN3DHMex1DrLsrZtfd/HCAMA3NrtuAWOoJ4kiVP77/st8uzuk7uri6qqojCsyrKtm9APEIBt02CMgTb5epNlmbW2LArG2HQ84ZgODo+rqprP51CZo/PDx+ePCCG319d92xKEjFLbbqsxjLEoDgjBO6AeYowAIAiBLTNaCwCcjiRTSjkNCSGEVpYQ4jEex3Fdt1ppz/OM+SSVs18cDQQu1jrZGZdDNE3jing3bOY+PB6PB4NB03XKmLpty7ruhECE5GUJMR6OxwAAp7WMCIEYu38u8AdRNB6PW9FzzpMkSdPUFRucMWstgcjjHtkKvPgIIedZvNeEJoTslBAjl8gaYzzPcym1CyFFUWw2m67r3DiG3gnsuJ13pSGEUAjhlm+0m6Fw1b87mw78IIy6qtHxKOM4RohIoTshKTHGWGWAW9YZYxDavu8t2AaYum4opRC6ctAihDl3ftb31gIIkVuxhZDuH0VIawUhMMaFQKiUhBBKKQCw7n0IASHYDVRKowFGnRRd2UjVF0VWVRUhCEIotQ2CwEDgPJkQJYiSusld9g8RohTHSWCttRCIXiECrDVGG0yQ5xEIIcZ2vW4whpSQ/TyhbFt3EX2PQc8DuwbQ9lbJm9FopLX+m7/5G4yoA88QQmmatm2rm3ZfMWNnMA0+RWKyE/ZxsRzuqQwP6uCHP5hdKrmPo+jHQwEIIWDp/sNuGsJt3PWY3FLg4mIQBFEUyb7eTxgCvE3I7E4vmWK8n4N1o7Pu2BFC3GMIQ4wIpTRN46KoKMXWase5AdAoJaXqnTOF3c4n2t2xwH1eiHeq5NuXwvvkBsIH7ZhPf7tNF1z2pnf0EWi11hqjP6FEfPor8ECXaf/an2FXDe63r412zJteCkSwNYgxjzEs+sYC1DQd6Vr7u99+NUxSYLVsmwW+CT3/6ZNHaRpjAuu6Zh7zfR9RIFR7ejq4/gARpUGI/s2/+c3J+fnbD+8HXnR0PG3bdq92sFpunj55/pMvfnF2/vQnz84RQuv1mlBQlOu2l0kyCBA6e3TKuY8RbZoGIUAxNEoYYzCBjGLOCCXIgXJWaciwg5QhhF3XLe7vHeWEcx5TOkxT3/fDMJzOZm6xC4IAIIQpIYQkUZznuVtHBmnqBj/C0DfGQAyTJJlMRn3fA4ziOF4t1/eLpegVAEhpDSxqlEjTFKGthBalmHIeBWFZlkVVdh0mEBirnYhsWzfWyOEoPT45fHR6Zq29v79vW8pI4vvJavX244fL8/Pzoqg44X7ij8fjy81NmW/Wc2GVhsAiABUAfV2GYdgQePnhjRY1NP399cXp+RnQndb68vJyvV57njcYDAghomuXy6VSClrr8ZpS6k/9MAwp5QghiIjv+0aauqjz9eb26rqqqiD0HDWh2GSMMa1VWZay6wAABa6n04PJZPbh4qNSilJOCMmLcjQaBXE0Go1Go2Geb+7mt9pIQgjVnrX67ub6+urqFMKiyHopJpNZEPpB6NelZ3WtleiaWgjVVjUQfb64W95f677zOQ4IwkrI3o7jyFqr+77NNl2ZHxwcPDo+OhyPXr/9/ujoyPe96XiiRD8cDUTXff/tt1VRcM5DP9DSrFcrKTSGkBLGGbbaQAz2EmkIAAts0zQEYamVMcbnXPZ9W9eU0iSNXbHiFHz3vUNHiKuqKs9LrfVsNptMJoPB4Hg6ur+/D4IAAACAOZjOMMZKy5PDozSKyzJfLhbGmDRNfd8fjZ7OPyyePHmyWK0YYo9PHz9/+eJ4enRxcbFYLYGySugyr6w2yCIjDSRuvEo5E8XdWim3QA7car/XdYkQQAhJ2XeVMMa0TV9V1Xg4mk6naTossryua60t8oDneWSnZYQQUtq4mS7XTMnz3KHiWZa9fPny4OCgKIqiKBhjw+EwSZJ0PHKlap7nmBJEcN02hNGTs1PXHVgul2VdDYfDZJCOp5MawaZpMKVa67quHUSRpqlLg7YQMdqOb2utGWZ7jHSPuzoxZrzTgbE7t4U4jhEhrmZt+9plCXtZJ+e05FKBNE5cfRIEAcK4rlsHM0AIuY+ttW3bEqlcsUGtoYQ5IiQkOIoizv31KpN1AyxCCNmdirYQAiEnVfQp/rkMw+ycEZwm9PX1tVuyHcK8Bzm07PyAuyZI27bGKoQBY7RpK0wg0EBpASBhjGECpZRBFAdB0LZN0zQIg31hvW+vUAoxotZCjAkAoKoyV7MyjzPGGGcIITfbCLB1HHAANGEIIdSLVikFId5yHN0wgBSOJ849igiGFhhrtTEQIYyxz7g70o8fP0qh3X1ydnamtW7bdt9z2acLCIH9O+7hcn+OgN2OPPyT18Po+LDi3/8v2aU1CCFrtlOO++aOu5GSJNl/o0ssHLC03zhCCO7kGRye7yAlR8TZ7gaEQnRS9oQQY4DDouIoCcPQpch93zdNtU9ShRDG7it7Ax70HRw09SfVv3skH6ZH+5D/MEvYf+BhC8Y9LejBCK7b1P6QMcAIIWQRQhYa+yeQg9sAxs6Am/Sgs9a6koNRT6oOQGigo9GwtmsIRsxoVNdtVeRAyYPxsK6Kd2+/H6bJdDpe5+soCvzjo9XibrlZGyvOzg+6rtsU+enZjDH41Vd/eCHbw5PTwSChzCuK6v37999///rR2fl4PHn69BmwzXqVvX33YTSaKCXCMAhDf96sEEKPHj0KguD66tYY41BHSinlvse4e9iUUl3d9H1LKbZGVVVVFEVdlg4tjON4NBrdLudufRGyU1rESYIIjqLAQGCAJRRh7HQ1NeeB58VlWWutuw51XbvONn3fStk3TROlyXQ6k0Jd3dwKIaXUxtjpdJrNF01bQ4AQZZRhz/P8MJpNpgCAIlsrKYq6KquiyrOLOG6b6vHp+NHpGUV4tVo5nxWP8Y3aDIdjDJG7L+9vbouiGI/H4/F4+mg2v7u7vr5sqpoSwglVQrairSvdd9VqcTs9PJiM083qHkP94eOFsokU4vDwcDIZffPt1443bq39yU9+ggDoWuE6tWgvpoFp23ZVVTqaNIQwy7L5ovM8bzhM67rWWnVdd3NzU9WF7/vcj56nz58/f/rV118DgCjh0+kEE9L3gom+77vr60tM4KPHZ57HyrJ89ZtfE0Leffiw2awY5642bZrK87zQDwDcknillNAaTKDUsq3aiPPTo8MXTx5zz2vK4rsfvpdtMxwO89VC9Z1REltTblbfZOuybYCxJ6dHT58+ffXqRRpHy+W8KLP53T2nDEIo2i7LMin1YDCIo0TvqhBjjJZb6RVrbSuE7/vNev1hsRgOh07kPwqCruuiKGKMuZrJrQVKKdcRr+t6sVg4sb+964zneUdHRwihuiy2g/4KAABGo9F0MppOJm457uoGGptE8cF05jE+G42H41GSJJ0Qy/kCQOBzzxpbFyXGGLiaBgBCiTuC3fK607IFmhLqeZ4QWyTAaQUqiBhjWm3u7u4oJk+fPo3jlG6tKKwzTHIVlYugVmnXrHXNuK7r3KPUNM1sNnvy5MmbN2+Wy+Ventkg6OYer66uHEXUtaWstY4hCABwAAzGeL1eH4/HEMK27/M8v7m9+fjxo9jF0SAIOGMAAE5o6AcUY6UUD7f0AvdoOCTMWhvHsUvUHHjmmmue51kE3W60Ve0W66qq3r97NxwOcbxdCj3PS9PUkU7WZY53nHm3miNCMMbD4bATUmvtcCPsJgiMub29pZT6vlwsFussD/woDCOXOEoptTUYw77vMdlGPld97hdxhzq4N92NBCF0056u8u7byjV3AADr9Vpr7YxXNpuNC5zuT1ybw+VMTdNsNus830RxsCupQZqmaZpGUWQM6IgQQmltlFJh5O2ir9sFqZRRxjBODQBCaCEbpRHAGljUNA2Ejp23tUcyO1UAtXNL2s/rb9vqxt7c3HieNxyPgEV5nq9Wq7ModltwTYd9vHdTAA+Dk3s2hRAM8Ydw+v5P9u+g3UwE2AlpmN3LWot39gfGmVrtHb92nFknAe4eW6fz4d7hFNofg/b74ltrbRwJGuNtmQ4gIWRPhGzb1rmOpWla13VVFXW9m6DZjbpASOyPcRG7s7vcf9H+5Lgd2EMaD7GB/UE9zBustWCHmcFtWrBd6rV1W9tRPXYhAOoftTD2O+YaNxhTxhjGEBiLMQIAMN/zwkCJUmurlDIWUM6IlCSBQYVjCuDRyUHfNdYa3+cW6KySXgTOnn6x2CxfX93HaczSCQPgeDJZr9cGWKTEN999Vy7v/sPbH/71X/8bqUw8SC1ATVUOUm+V3R4cHyzWV7PJ07//5itK+d2Hu7PT4y+++PyHH34Ik1hZI0SnlIDIjkejvpNt20OIIo94h2y5nHdNGccxJiBI/HSYdF3XtP3p9GyzydabPJlMDg+Pj46Omrdvrq6uOqnS0bDqbNasDYYQob7veyXSQWzqUgENkNGmW60zS8ByuXx8dnr++Kj4ar6cX43iwIo+JtxDxAitWg0BpoQrZYxl/nC0KuuuE0EYQx5nrQqH8Tzr6x6PZi/KLG/7VgtZNzppNYEhbnxTsXVWjUZsuShHo0FTqrYtp6Pm/cVHTNlgMlqv19+9/2FUTU5ePHoyPuSYxmG8XC5Xq5U2RmNbS2WFlFLGw2lVi8U8u76+fvbMjJPpy9OD5XLJjQRCnY8PHO3g5Fe/4YRXVTXw067rEj/2ML27vkUI9aj++PHjwcFBmIRhEtZdPT08CMPw7u7u9evXm03myqw4nTDuIUzDmB+djv/+t/8F0+72/u7g4OB2vnDj+x8/Xp2eno7H4+VyicDw5//yr37zm998++673/7297fZ/POTn82LTV6UWhuL+Xp5MRlXgzjpWxF7QQcqKZRqewtEEHCEbOgxpPoAe+eTyeX332+uPg4ZzW+ux2FY3t/JODocDd68ecOTadfqNJ7FaYQxZh5rBJbAiyUBAHYSAmMI8wk2Pvd8jwMfAcA3m02e58BCzogxxvMCyvlisSg3VdeJpVirHhweHvosNmArmAMtiIIwjROMcdc02ere8zxsVOzhkAXEitXNRykl9WJKaddLJ8AidAWhHQwGQsr7zdpa++WXX85m0zAMLQTzqo6o1rhFnvI9TDxddpuu6wAWShtoNQQQWCJEX1UVQiiOA2uxg6wRgpRSuzObAUKVTVc2klIKsJ/VUliapmMAVNd1g/gw8id93354exMEmZSSYW6RjeNkMBhIIbRSXddJJQiislcnL08vLi6U0MN0VFWVUTaJ0rpsfvePv3eGQIeHh5PR9Jtvvgk4O3316vrudpoOi6Lwff/zFy9ns9l6va6KMgiCIssHg8F6vf7973//6tWracSN7opsrayNg3Aynr159/Z+8TpNUycan0Th7GBCA4owAO3WFEdK6UgznHM3aeakfiCEcRy7ZTQIAillSK1VfUQwDQKYJHXT9VJ/9rNfCqG2A5bW+HEIOW9k37btx5urbcaMkCwFxsTVo0oZSqnvEZAmQgijhFXCo/jR0TFH+Obiwlp7djA2AGqrm7rElAFoter73gghfBIhzLTW1kDkxOkJGY1GVumP79/7vp+GYd+3om9lW3fWgJ0tshfgVohifT8cDoFqZ+Ppb37zZ10rmjy/vLzmnE/ScZZlou7TZCCUkKrooRgmHjQ+JYxz3ledsRYD3lZ930it7H4CEwBge+eIDY01GFvfZ9pIUTeIQWgtRZJhY61GSlJKWehpaay1WkvVK623mkgQAMpYW9a6l5z7RkNgsJTaKKm19MOYEGI0wBj6vi+lnt/dV1Wl1FZeEEGCITbSiL5Bfoh2OspSSgOBSxT8MNgX0MZumU/WWmQlAQgarURvgfUYgRC78lxr40ZQlAaMIc6557O27fM877puK+LZC2st55wgpIwRUkJrKUTM97EFUBs/Cq02UkoEIIbIAKiMhdoABBGAVmnZ95YQZ5wBIexk6yilLqVOkqTv+y+//HK1Wj169GgymX38+NGxmowx8/ky4tGWfojwNrRDYCF2RwoQtBBYCCwADnrAELpJCkfi+VRWQejyFUfndEentbbAWgiU0VJ01lofUm1MJ4WWPWaka2plGUFQI8AJr+u6kz3G1BgAEEQIdKKVQnHOge45QYzgMs8o99q265U8PD2H2txefjyaJAGjSPcB521Vxn5I3ASzVL1SijKGEfQ85gpxN3vddZ2xwPN8IHHXtcoAxoMwDHsB2kYgSMejA2swhkT3VllpJYDK9m1XbqpiXYZsM4iDqiqX97d1ue7aqq7rQZyU+UZ2vdYGIz4YjJhHbu8Wd7e3Z6ePMCW+l2AkRG/LolMGEhQqodertVGMkTiOKGdJW5uPH+6Hw2HVNkLq0/OzME6u53fz5QIitMo2lJK+Z4ZijEnbdqu27bvGIKw6ozXsWgUMlFKvdiZMdmWbqgx8GoRx08nlcplv9GAwgMZ2AABjEYRWmyzLNpvcahv6AaGI4MBQhRF0DVRpxWAyvL6+/sPXX3k++8u/+svpdHxx8QEAILoeQnswnh7PZuv5vKqqu4srVVar1WrvaRuGIfYxBciFB9l0TdN4mFKARN0aypummc/nVVV17XZC3WW4fd+vVquT4+Dly5c//PDD999/PxpNlstlUZVaa6xM3bWuM31weBiH0du6dqlJOhy4WfbJbDqdzbpys7xdre83DPKIx9WmbttWqp4xdjybNkWu++6LL774t//230ZR9P/5f/8///DNt03T1F23vL1HhKZxXOTl/c3t6fHZlrMmdaPrpqyAsZwQigOCmRNP/fbbbznnQoif/OQnRVFwzl+9eoUQevXy8+fPn4/H46qqoJdKrdM0rZry/v5+sVhUVZXnOaXU517gcwxtWzdd3QCw1YwzALjpQcY8IqXS1k0HpGkaBMH8fnl5c71er4XoCCFR5LnGuaMEHkynzlj55uqaYuKmJ6w2GG7n3XultVauSTJf3LkV0E26a62DIBiN0jD0ATC+73FOm3VZVZWT2W6ajlBKKU2SwWK19DxPCJXnOaY0iiKttcvbPM/zIs/10Y12sIhyq5sjBlprmcfjOK6qikuitfY43w7+EeyGJwfjEUXYjbc5Ud6dNQ6t6zLL1pTiMBxQSrWWCIHj40Nr9WazQgg4Sq8QdDhM+6ZdLpd3d3cY4+l0OhwOtTXr9fr7779/8uzZdDpVSn377bcWgq7rjo+PoyR+f3l1dXtzcHh0cn5mCFkWBeYl5UxK2beNNDJKQ8SIzz0LwZ5M4IppV/S4uAsA2MPFcCfXWJa5UyPVBjDfp5RyPxyOJ0qprVolJW0vLy8vtZYIQs/3HiC1VuvtyiuEchCRY6Lpnck1pZRz7lpLzjkaYkoIgZjUba+UMlLZ3VwlIaSrGlfVbcGJthOi7/seb+tLiBAycju7DwDQChJMpWiLonI1JWd+HKW+H7qU3VrrdgNhaK09Pjx0OhaEkL7bDu5DjNbr9XYfMEOUOOczAEDbbwnXeGcBxRCB0LZt68h9EGIpJSWulCRV1VhjgIUAQWC2oLy1tu07jLHUqhNbFWcHujDKXSdbGQ0htBa6eppQZoHzZQYWGAQRxAhjBB+A//sevMsOP1XzOyMJay3Z9VastRZsrzsAQGoNAKSUWgWM3kZTu9NVdMx/Y0xrgbuLnAKSI0bw3YsQkmVrpZTbmT1hQkrZK+naRq5j5VAutwWEkOd51kI3G1+VdVmWp6env/71P59Op69fv379+nXfS8dURbsX3CUKEEILsX6g1Omyh10L6RNmsAeDHY3mE4qwJyho7RqR+9TQPqAj7GEtdzNjALTWdNeSePjtO0jmRzgH3pm47u/VXXuLcs7JarOWWrVt23WdR4nou75vGSdh6Pdy3EmxWW6Y73Hq+X7Y86Berzzfnx0cVVUlFZxNT87PHyeDge/7iODValNuGgb50TBhzCtW5Zuv/1+PHj0axLyfJFKKu6sPAEBsFWNEqr4sGoxpEEQAEWVNK1WWZZgShJDnBVprgj1MEWfRbBrf3q261nDOtZJaYU2g6OWrX75AlMyXC855GEdgDqqyCePIuZwtF2ttJCEIGGMtiOK0E5YTTwpwf7fqWkUgsRoM04HsO6O0Vk0UstlsKJXGUAkhMAKAEIrcMwEoxRCArmnd2DfnXPaiV6IVUnRtEASPvjiZnR3P8/XV1zez8QRSGMZRGEdVmatecE45Rp7nBQTfrpavvxbBz391/eHq5ubGGDMYDJ4+fXpydMzP+O9+9zuOmU887KFxMhLjniOW+HESxVqqu5tbRr0XL14EQdD3/dnJaZ7nVVHWVdW1bV1V11dXRtk3b97UZfn06dNxnBKAuq4r6jZbrtarlc94mqZ+GHz22WevPvvMC4Pb29u3b98epaMiayj2J5NRXdeXlxdFVjBO677hnA7T5F/9q3/16tWr9Wb523/8u//yX/4LJMFwOEqnwySItbIRDySRmMN8uba9lFFV54XPmahqawwA4G5169SBXHZydHR0eHgcRVGSJEmSHh4eXl9fp8lwvV7P5/OiKK4WP1Dn4cRw0zQWWqdR+Ld/+zehHyRxyCmWvZBdTzASQgCGMMZAAwQQQghj6jzf0jTl3LMQUEqVFpt1bqQqNmstvSRJIIRtXVlroyA4nM3qMFR9DwAoyxJDgAjmlCBglegxwVVZlhRtstXN9UenAWB053D4vivSJBikYdM0g0EAIcybrm17Y0Bdt0VRMc7H4+loNMKUzGaHy+Xy9vYWYhJFsTH2/v4OAHBycjL2x9ACA7TjzTsJW6N0mRfL9UpKGQRBk9RhGE6CkSuhPN9H2MfOBdGaKIqiKHIsBN/3nQyfUsrzhnvtAZceYYwnk4lDv52TqjHm6uqqLMsoipzstBunRAi1bct9r2kat+y6qOYkrQAAdV3Ps3xTFo2SgDFLyKYsNlVBOdMQdEpu8g20IE6TMwixx6Bibdu6SODaEy7I7RWNXOx0y3pd11prYkQvHCqOMSHaYoAwAAAinBdl37Wj0QhKmeVrq3SapoA402ezjWeEALDlQ2hlAXYFrnKpiYvWYRh2vXCkCqE0YQwhpC1oOrG9EPsMwJi+7zmk7mSqnc4PIcQiZIxyLR5OsFM3EUIwP0nTtKqqPCubTtwvVl9+83UQRL2SGlijLVLSIggwkkYroJ0AVJIkVmkjldBKa80JoR51dTYkW6ljbY3ZyQZorYWUABiMEaFbrWVXp7opOIKZ1loIuWeBYIztA+E/17q01rqJWZdU9X0fBNsfrN31BSBGELsuPkJaa2mtdWLEDgPfBzCwUxFwAWkP/iO4jWR212Lf9hjAJz+Fsq7BLp+Q4pMjg8t7HnYQXKq9Bd7d9nd6Bm6rGGM3I+aGXV03zf1VEAQWgrIs3VwuY2w0nrhBFTc47RwsKaXT6cHh4eHJyUnXicVitVgsHhIOHnYZ3B4A57bwkIuwbalsqQz7fXO7ve9H7D+vd699foAevgix/2RWAiGEoVMJ+zRSAXZDKPsBzoeJghvxdbmCAcbs9KDIcrNM0zTCsQWaEdp2jSvX0jT1vCDgPsUcaCA6rW1fVRXS1PPCOJkslgWC3vnZ85/87GdlUeV5ThhazNfv3nxI0/TFq5cY49VqdX3z8Z/9818GgReHbLPZfLj8iBCqKugFIeceAKCXphUthJhSMhwPtMmlsJ7nc59oBQnTddvOF9fGgLLcDIfpdDooKhwG0XA49H3/+PjQQjBfLd9dfEiy7G5+X9YVjwJMGKG8rPKua6PQdwabqRcySjCGDNOqzAMeeEnqUTJMUik6RoBPUV22VbZgnu8xK3uh+x4TyjGxRlsAQp4eHR8NkvTDh49ZvukoM0ojABgjWmsL9P16vio3yCOTw4PpdLzOsiDwPZ/97revu7YOWLKeL4DRdV4iZUzXv/7u+6urq6ZpkiQB2izv501ZWWtF1zHGkAU+45EfTEdjIUQchNPR2A3gTUajn37xBSHk6uoq9HygzbPHT7777oe3P7xOo/j06Pju7n61WPzi1ReBH16/v7i8usrK4m5+//Kzz07Pz0bD4b/8i38BCZZa3d/fe2FggA3jiAHPQ/z88HwwTLu4wwZ5mHd9c3x8RCjinP7ln/+Z1vr/9n/5P1dV9ezRedOhw8Pjo+PTw8PDrKz6TuSrfDoY1XVLgKUAJWGQhGGzXuV5LvreAjAcj5wyoHvSOPfzPAcI3s/nRVl+++23fS+zLJvP50EQXN3nYRwbYw6PD8IwDONwOp36NQ+CAEHU972WAJitx4kxhiEe+EGhy9VqhYvK8zwNbN/3k8mEUsIYwxApIR3rVshudXE7m82SJAFWG6mKfLMJPNn1GFghhOpaYLXveZyRpi7X63WUhE1VxQFjCHoEcoY9hjECaehfX1+vN8uf/OQnozSyqifAtk0VRZHbMdGrvu+1AXVdh2EMISaEAAvrqumkwBgbA/K8DDiDxlKELbb7+a6maxljCAAI4WQy0VpbCAaDwWQyAT1wE2thGAK4Xei5UlVdD4ZD0LZFXUGC3XLg2tuDQRIEXlEURZFZa8/Pz09OjoQQWkeEoNls0nXdx49KKTOZjDCAs9lsOBm/f/8eQth13XA8cuqWEMKqqjjnZ2dnQRSuVitrbV6WxONhOiA+F0Yvs/zq9pZzPhyPe617baDRrRQKQIuxQajvW7cyumDm8i1XLzqWktP8GI/H6/U6yzKkJYQQIkI9jzGvarqu7bSxEMI9MAAA0coSwoIg6GS39yzYzU4CayBnvjFGSt33Uu5ciT3P930vCIKg3erudV2HtQUACKWdkIaQW2cH17GuqsoC3y2yjjtFCI7jGEGolPB9fzabhR7Psuzq6qqu66YXnIW+F3dcGQOk1B8+fOScV1XlDClcIuIY6UqprioxRB7jFBMSMfcrzjkiVAihjMGUYEqMMUJIKSVnRGstRNeLVilBKQmQByF05uBt28ZxOh6POfOXy+VyuSKUW2uNtcZagKC1W3FhwqhrAEklAYSUMWOMkNIzPibMx06LYmvA0TU9QgRihAhG5pMRkX0wubBlI8KtgoIr6x/G8i2v0Bjnz+XuWLNTTTDGIIS3gDx+qEe07bhvDSAAdHx2JcQexnDNJseicDmEwxvAzlsBY8wD39HVXUq318lwouNt01VlLXpJCBkMRg7AWCxWVdVcX1+LXnHmGw1Er4zYjrlC8ICnua3gkd3Nd+zDMwRbfSqwU31w6NSe6Ql/rJu0D/YQfDK7QghBiPXOcnOXORDOOdIC7Vwn9ptyVwTu8jP3eW00ACDLsqZppEystcYaaSSABmFK7ub3zONRECKAGaFOqjb0vThKEcBaQc78dDg4mMzWWbbpciMNZ4HoddX0ykBMvbaT7y8+3tzeUkrvFvO8bpDnrfIsiiI/jn7xz3/5+MWj+/v7VbG+W9wtssXBwaGX8GyzTtMhohBaLXRjLYBERTGPoxYiEoYRRvD2ZrXOLlab0hrkeVEY05/87OnR0dGbd2+Xy+X9ch3HcdMdUU4AAJfXV3S5lMpIo9tOKGUAJl4QYUp9z2vbpm27Dao54Y7I1rU1Y8TnNM+WG2iePHkynU2mo9G3331XlhUChhpju0YhSkPCKFbSdI0EqTk9PkySAQDg+uOllFKqPvKDJA7zPNfWYM4AgtODCQAmieP7xf397VXbNN99+3VXl0kYzOd3CMA4DKLHjxljXW8CHnjUc8PoXd31TQ8AcKpzWmhrbbbKrLWqV0aaoijcI9E0zdXVVdd1i8XC4z5jzAnprFYr95nlcul53ngwNMa0qn/66DHE6B9+/7thmp4cHY8m47ypirLcbDb3iznCOEkShrDc+tPTu9v74Wjw+ec/GQ6HX331R2vBs6cvtJF3d3Nj1PnZY+7Rqqo+vF9DiBGERtkkjI0H3nz/Ztn0fdeEnu9hDJTKRHd5eVmVOSXo3/0f/w9Pnz4/Oztrmma1WlVN/eHj5e9+9ztKaVO3DkDuOpGm6WKxcCLKxpiiKJhHjTGEka7rmqZ59OiRUVqKTvatlspI5YQBEgiTJAUGzO/m1tqjkyPP89xZGo/HdDDQWpVlsZzPjTFxHDd16XNvEMZpEGVZdnXxcT1fOFQz8gOGSRJGGGOojBUKSJ1v1hBCTgkjkZhMLTBSSmtMFPqib4sst1p1TS261ga+x/jTVy/rur67nd/d3cVxHMZJnpdFUQGImqTlnB8fnxZ1hTFFyE4mkzQKkyRBCEFpGSY+4xbBvu+d5+94OByPxwaAsizDMDw7O2vydr1eK6V6JbWR1lpkgVCyqopjYwiliGAAAOEsiCOMsTFqMBgEQeDcs9I0da5OzlrM5WduBNeR6nGCEUJGSbDjCY5GIwjh3d0d3UksCyFMZV2wtBAM0pFQxhoYBNFkNqOULteZF8UIwSiMEYZ+EFlrlbbGWvxApW4PTbv6HmNcVdXHjx9PT0+fPHniJCIWd2tKOCQEagukbntR1Y0wgHPOgxAAoI3t20YZi61tOsE9bgxwYsZSSudQZIwJgqgsy7quldJbfGJHX3Avl3O0bYukRgj1Uu0FGNweuvEQt+aiB/I4jlyCEQoCL0mSKIpk12ZZVpalUoqxQCrr+RwiamwnhWoWKyfi4uYJXTdEKqWNsQB4jDvpMAwR5Z7SGhrbdR2mpm5bA2yAMMLYGCO0UkbbzoEHvZDCAeG+7xOy5cPvm5UuC5FSSvWJSefUlXZBZTstuUcCXJ8eY7K/TMYYIVTbdl3XhUFsnRIlBm4bW1UJ/CPjIvNAnxE+mAbcNtqMsciAnVKk3fpI9dZaSqmDyiFGBLN9CKaUu0RQCCGlhHa7q2SnvEQpjYKQc75FzT3iLp9SyklTuARlP//ZS6F3ttTGmOurG6dfByGM4zQMwyAIOPeFEIvFQvRqvV47DIlzj/O+k/XD2h3sBJ0QQi5RePgrl+iYnZ6B6x4qpRxLdx/F9+jCj1CKB1MMxhhot+2SfRbijt1atf8u+2B6QkqJECaU7q+Cu6CbzcZpoCGEMMDWKPcnRGpVtxWE1miNLCAIj0ajwPOFELc3FUILA3SSDBj2MaiNBEIYKfVqtel72QtV1M23r3/4/vUPACFltAAyOhgKa37/+ts0TZ69fBFx9vr66vXr1xcX76u6hBb4aZROR63oDJJlXVVF7XtxFMVxSigNw0D4QZSmozxrPl4WBrTcg0pCpZuTo8fDUSRku97cv3v/pu/709Pj8c0JAMhAYK3tRE+Zx4zXNE0vNWkFRABC2gkFAAzCxA8CqCUEimBbl1kmukma1mXFEARGe5h6o0k2nVmpAUAUYQxgLwXUPkcEM9A0RVvVWqowDE+PjvP1pus6rRTGEGEsRAehnY6HBEEDMLBGC7FeLdaL5ffffXN0MKOUUoa1VhBjPwoJBNbaZDQo6qrv+2Q4cOJ0cRy7MLncrIu6stbWXetcEzWw7968Xc4XouuvN1e31zdlWXqeNxqMXrx4cXN9/ezpCwTgv//3/74oioODo5/95KeeH7Rtu7pd+WEwmUxCz0/jZJimBOGbi8tNnqVp+ujopKyqrmq01poF880CY7xaLX/2s5+dP3skgRJf6j98/U00SAnFF9c3AJiDg4N0NLq8+SqMUwTJx8vrb797fXx0+vLzzwAm2XKVpnEQRwcHBx5njCAte07J0fHh4bNnEKLvvvvuu+++Wy6Xp6enQqjvv//+r/7qr9JksFwufd/PssJVM8vlMh4cZVmWZVndVnmeCyUIIW1Xl2VJEAbWuLmsrm6qsmia5sjAYTLEABOIDAJREA0GA6ll3/d9017XddvWbVMZLeu6ruri6elp5HHOiLXWaFluypVSTdMczg4ORqPZbNY0zXq16toaQjhI47wqMcbFpqjqwk1A+Mw7OJgdHx7fXt3m683ibnH98frw8PDk6Ojo6IiB+Le//f3FxeV8Pn/+/LnnecvFqqqq0WTcdcL3g2fPntVtn2VZXde+FwaciK7vmlYIwX1vC8W3XVVVfduFYejI8FrKfLNhhFiDV5u1lNL3fYx2gnfAcs4BgpQzSmkneqSkMlpq1RYrIQTNcNNWSZLMDiZ+wK+uLp1KY9NW2sjj4+Oj44PNZnN9c4kMGQwGBgJjzOXlJWNsMpu6DoW2Vu28+7TWDtWY3y4msymlvCzrzTrTUoVBTJhntUGIhmEYBUEUxEqZpqqlUG1VueJSKeX8P92K6RJWp51weHg4mUxWqxUh5PDgWAPb98IY0/Zd3bSt6BFlFiBCKEZAKKV6aSEwANZtB6CRUmplAQBGA0ghpdwtsn0vmqZ1Ao4IEs45Z75SwmEJbi02xmgpXd4AIWSMWbDVBXdTkT7lDup3S60UQmtFCOm7Lo5Dl5Ct5vfX19eOCEXDYd32AJGiatbrHGFgjPGNCYIgCPxtcxpBraUFFjOckAghZJSyABCE9wv6rvdsIEIGgn4b9PWueAUuzLv6GCHgXLkxxlLK+/t70SvXyqm6xn56fQKuLcTKGAghwMj1QYRWBoKm71ye4U4pcNA5Iv02lYQGIg20NcDpBBInMOzEqbRWO14nYfRhdesivdaaEuCSOa21az1sFbABUEpLKS0EGFEHpztMCOw6DowxaLcKhmiXmmw5pJ7n+jLmgewjxK47iQkhrejdNreyklo3TSOlzMvaGOOapOPxmFK+xe0tUtL1emjfi74Xrnfmck1CCIB4nyhYiLZQP4L7VtouZut9IrWHB9BOlsr+eFoSQqiUdudt3yfawgNWu7bXHorbHukuM0A/TiyMMU78Y89gcDvW1m5iFlJKGQTWWDfrR8azMaLEQqC01kpNB6PxcODyMiGytu3zMhNCNU3nMvokiRCAy+XSPTmYYWX0wekh5qxum8VmVTV10ZWrYnPXrCustOnDH75r29oogRAIQ78QtVpeEx/7oQeptUAhIDEWYRjFsW80M8ZUVZVlhVb2+Ogc06DI2vl95vH45mq1WCxu7udKwigcPHv6edd1RdVUTQkJNMoYCDAlTScgRp3oPc8DENZ1SSkepannedjWnLM4CbTp14u72Cdx6E9Go8X9XLZdGIZIAwZp3fR925leA2i06Knnp1Ekhe7a+vb6Sin15u0P1zeXURQBYBBCjJMgCMbjcbZaf/P118ja1Wo1HCRAm9OzY0qQx6js2/F0kgzSssxXmyVCKIqifLHK8jIIgtnB0WAwcJS6rpfGwryotAGe51dVRSjXWvdCLbLbLMvcveiwRFcxI4QcJhEEgeMKPX369Isvvri7un589MQA23ad5/tRFNV1/Q//7e8RwU3TQAjn9d16vQYATA9mk+G4t7qosjgd9jL0Qw8i5AV8Mpsyjy/XG4TQ3d3dfD5//Pjxz3/+8+OTx2XRWoi6plmuszAdIEzTwWA0HndtbaxuZJek4Ww6rqrMaBWE4bc/fH99fXt1dfXx40eEEPX4dHKQDod+GHZdv86zzWYjhHJVWtU250+GddtGUZSXWdM0vew/fvwoVR9FwTAdhIHnEoWmrh3Qcn+3cKTFg4MjQsggTpIk8TwuhLib368XS6VUHIdh+NhVGI/Pzp0UZl4Uou0YYwThuqwYYx7nlBBrjGg7lxN4lDWYEUKW89XV1VUYhqPRIEmSk8MzSvAwHc4mh9k6L8vy17/8Z4fTkyRMl/eF40szxgCAWhmlFOd+38vVasWZ76b+nKWCe56rsiyrSikVBIHshZNUYpjAKCKEdE0LMXIqCNfX15SH6zwjEHmehzCxALiKxPe5K4O0NX3bkZ04sVtKmqZxxUfXdVmWffz40Vr79OnT0Wjk4FaM8WKxWK1WRgBjDGYUYzyfzxlj9/f3s9nsxYsXHz5+XK1WYRgmSUIYHQwGvu/P728JoroT88Wy78X97RwB/Oj4qO07a63H6XCYjoZDiolVFgGAMVZ7DRkIHbXIqTRKKdM0ffny5bNnzyCE8/lca312eiqEyPKyrFshZNd1QirGDUTaLXAEQ8w4MRpBYIxp6s480MRFiDgNx7puXBTxvdDJMu4rLbnTsnQLsTJGKaXt3gsAuT10pZ5se7dS71vLLuw1TaO1RAilaQp29D2M8WqVr9fL4+NjCCHGNAg8AADjzvoBu79FCGDM3O6NokgpxSgzwHqeBzEaj8eQYKkNQEhb4/m+GwVXZnsIrjCFyGIM3TiJ2xN3UO7hapteKYUQdp4gu9ePTBZ2s6/c4XyuJdR3WyVms3WT8ghhCIm+d4mCUTuLbfjJ19vuI5+RW6Vn9EA/YItm7JIbCLbNHWcc5WK5Mqbr+rqutTWMek6Qw4lAoK3WIeWcc8q2KMWOMbAfWNjvxp7GAXb+1w+rbbWzF3E/uEHWOI4Hg5ETaRVCILh1mSKEOshTSqeEBPZhHjyUZQQuzH9ydga7qVGjtVvM3aV3N4nv+07WZc972D8g7ibfpyD7xAI84DTsswe8nfOEWmvwYz0Gh9C446XGAABcq4JJ5horGGNg5B6ZIFrrsiw1l0Zp47JmysuqEN0W8JG9ur6+NQaMp9NH509kX1NKN3kGCeYenUwmxKfTw4PWyrKp//jtl/fffYs88ujFk07082pd1zU0NkmiOPIhtI0Sbz9+6Nvm8enJ8LPPnh0/6g67PMvqsmraHEPFOVqtN1XZZ5tGdPDo8JjxVLT3v/j5y/Fk2nUizy6VoBRHGDFGYy/yb+f3DmzveqUhosxDCAFgMabD4RhjeKe1FF3VdnmeH0w4Zd7h4ezo6KCviySOOKGDwQBaK7q+RyQOEzm0Rm+00HGQrptWScm5H4eRNHaxzObzeVHVVVFqrQPP7wAUWjRNUzX1YJAGLFzd3wMAqqpSXWut/ezVi89evnj/9l3f1tPZbDgZr/P17WIeB+HscHp8fFK3jbtrLQTaGqmVhQBipK0JY1cQK8qZqGsnsy+6LkmS6Wg6G09c/ogsWM0Xn798tVptPMb+3V//9eXl5dnxCbLg+PxsNBr93T/+w3/8j/8pigIhxKtXr1w3dzqdUkoX9/dd1z169Cjm/u3F5Ys//ylk4PPPP2+aZjyeQoa8KHz26mVZ1nmeTyaT4eSQvX0bxImyOAzTq+sFpTQdDkaT6cHhoRf4cZq8evWqa+vry48AQUxJOhgQzt6+/vD1119KyoVQaZo+efKkLEu3KIdh+OWXX/p+4NDd6XR6c3MTx/FyuXTVahAEWbEpy1JquV6vlRYvXz7nlFGCnEljXZQIAkrperVaLUbsiLuY57hOTpjS93mapkIIhAHzvDRNkyRp8wxCmOf5ar1om+b4+JhSqmSvZL9aLZqy2HsiQGCMlqKTXuIDA42yGCLRybXY5Jui7eq+FYNkKHuBAHrx7IWW+g+/+4MUUGt9enpKKXPRHQAwm83avnMJMUJoNh6l6TBNh0IIpDtjDATA1TRbwWyj3XKpgc3zPAxDNyWxXC4PkpFT70mSBCHQNE1ZVUWeTyaj+WrJCXULrh8GHuNVVWkOXaeWEGytyfOMc6a1YoxNJuMoCtu2VUo2TQ2ATdOkytqiKJq+m06naZoOBgN3FEmStG3rdA4AABaCOI5ns9nhZCqU+mb97WqxDOOEYUIJYZQ6OxJK0GQ0ngxHnHNrlDF8MBg5U0d3pO6/WusoipxQpmMN39/fX19fTyYTpwbtgOIdYG5cT6SqKmMMiwKPc9V3QggEAff4HrY1Bjh4lpBtyYUg2VMptbZaW8rAroMLXd4glLHWAoSlVBhjiLBrcgdBwDn/+O6DNnBHcfAIIRACSuloNGqaqm3bwWAwHo+11tfX13d3d/H4TCnj+2GSJIQsrbV5vtls+uEo7bpOayWlDALPgYuEEKCNi9BCSWUApkQIwbDnjhogCDBy1TPACCMkipIxxjljnBCCPI9DZI1Rbu6073vitJYQcqnAjpLo+uVo3wYywAopjDHK2F4Kd94AAH2nXO2OMYaYIIKttRYgu42yWmnlQhRBEBGst9ZQW4llB/U7dMFdO/3AFwNjLMXW3UMpBRB0bXuMcRqGGBOttdSKs22i0Pc9pXyfnBFCINtOSRDG3Le40O50egghAOhP9H5rjDFulqGTws1NuAyVMeYETI9OzrpOAAA451sWsIFB4JpTBgCDMY6ixDFkAQB9XW0zrQeiSXurzAdwwtYOyt05+0lIa20QBL7vOzalMZ+ojvbB5ILbkjsQF9etBg/4Cj8Sft7bTNoHREtKqTGf5C/dzebQtb1Em1LKaIkJ1FqTKJ3Kru/6pm+7wGOagKwrNtWKMYYD3HVdfDyO0zSME98PZcCUNstlPpocK6gXt1cJIf/iL/4SBfjj9cdG1jpgg5MDIbqyLnrZh0l4MDzaNVcAQBBAY6HCKMolu93Iw7Pp08fTfL1a3N2XRVHl+eX3wPMO1ov71TI/PTqeeKOmaZhonh2Pnzx7+uHy5oPPigpe36+zolrkEnGUFTnGGHFP6Vo3PaQMM4ABpBwt1jeMEW3atq9Ho2RwNH3yNI2jdJOXq6zxo4kXTRmyneiHg2gw9K+vb6IgtlZjzF48//n7dx9XfbbKq0aJHpRV3faS9IqMgokQ5Bez8/l8jqBNoxgi8OLpF5PJaMzRbX4rZDeJpj+8/mY2m95cXZ+fnrj5tE1eGUAJHxydvGKMU/9gNJs+ffnqzZs3ELOm1UpTIbXVxGrvJ5/9qm3bpqlOj84pJMN4qJQgFD05OdNaQ4wDHrup3Kot7xa3EsgoTYCGKR6EsxhCKKwIAZ/fXb18+Xw4TIuiMgY49k6WZZDAtm9602EPosDWOrvPPz4X0599djoYEnQwBZC0fSmsffTiGaV8tdokYTJM01evPnv9/Q/vX7+Zz+dK2hcvXjDmaaM263USx2Hozw5GXeMvl3ME4CovyeUdxuHTFz99//bdfXa7XK6Wd5uj2XkanIrM076PRFCUKzzA/6f/6X98f3HxX//uf1PA3s7XQZRWRR/ypKwKD/ss4RbomPubrDGdOD08UFoQlFRVfnX74ejoCEI75YPhKDK2BxbMpjOlRF3XuveB6k9nMwaAUqpuyjrPp6en52dnX5f93d1dmqaPH71s6+bk5AQhRFGQZVldScPQZHzcNM3t7a0UcDQcctrkm5XH6K9+8RNnqx0EYZmvbq6v4jh8/vj05GD82ecv62L15s2bxWKx2IjpdBqGYZjEvu83XX/y6Gnfy6IuAQDJcBiNDoXlVV5pjf1wAvuV5ws/EBAR98ALIUQvPc/jnudyAoSQFhr0xkf87uKCEELi6Ohwts5Wzaa5ub8JQr9Vfdu3Svaz8UT2HUMDoGTqcxQfNk3DGIviAYSwruvrm/sXLz/3PK+sWgih0uDmdg4ACMKk7eTLnz1drVZB3/dC/PSXv3CcOAvhdz/8oLXmnK9WK8754eFhU9UruDwepgShn372Ik1jQMimytu+WhdYZaatq0EcDVJebcyqrhmhh7MJZsbISrRl22uMCeM+pRxi3Pc9YYE0NvTY3f1tVWUvX55DCAFlQRBCxuVy2ZciHUSBlE2TtxQo2Wut86IvIHRYdBAEGkKllHborux7LVvZM8bqsgIQIArqpuCcc4+2bTtfXFMWElJFURQEQVU2GHHZFYSwvheB7wMLhZKUEx9jpBQQKhnETdOIrsOEWACUNZhSP46LovCTgTHmdrmmlBZFnfdKAnyYss+SZ1r3dVukw6SuW879MIw9z0PW9r0E1kBtdN9xj47SNF9lo9HI+b4CBF1HXGgFAJJ9r4yB2pniAKaBEjIZp1rLWjSuGCVdzxghhJTr0lhtLRJCCtE9CJxwj7hgDCGCECBnV6G1tQYCAq1BUhhrsed50CiCCSN8a8LZK4dROUF0SrAUvdGKM0oJtuaTx7ejCu5jlRMx23fcP6HoAigr4M5TlHnc5TRd0yBgOSVC9G1TSkJcMqGUcXCdC5NuT/q+n00mjkfstEfdZIQQgkeUhd5msymKQkpJKQvDUGjV97LrtvkQZ34cDJIkYYxRxA0GUsq+6a21DDNEEcXYyVA6+LZpGgQ0hqZvK/CJDfDAewkijLEx1uy4CBhRA6zLqNTO2NqJQjpMC+xkpvYwgzteQtg2tBtjDdTEKmWkVBhAKTRjntFISYMgEUJy7vUEme2Aj9UIaAgsBgjDXgrHQ2rqNo5DIUSWZVbJeDTK21ZawILQNoJ5mCCoe0HW6810OuEeXYnepSfKaNdlOD45OTk99cPID2NtTeX6ppxx3/dDz2AYNYkfBgADY8zNzc39ejGfz1erhTGmF60xxjLaK2dxayCEEFkIoTEKWl0VpRCibZosyyhhjx8/zrPs3es38/kFhHB+f9c1dRIPmqbrOiGNXqyWz16+evXZi16bH969E1oBYLRuGQ1n02FZV63ooihQ1vRN60dh23WiV8YYTjyPR0k0fPr0ue/7vt/VVbFerpu2AtJWecEoDMapEKIqmvl83oSi74w1zPf9JEkGBmhlXPFXlLXUNum6PXcGQTsejxnBm9UyCYJH56dpX158+EFrqYRJ46ipsu+++Wo0iKSUTVX4vm9USBE8mA4RIn1bvX73VinFPI/6npEGERglsZOILssyjIPj0yNKcVHkebEhEmMLnPJMs/PQU0bXTRPHcdu2FkELgRDCAuCkrPObFed8OBxaa8MwhhA2bZ/neRj5ZV1zzs/Pz+umqKpiOBz+6le/CsOw7WVzc6MNoH4IIJXaVqQZDEbQgros+rqGwD55dB6HQRSExpjHT86qqvrym68Pj46ePXsMgf77v/u7R48eBT7rm1YKHYRc9M39xfV8fvfV+3dHByevfvZoOjwmiA2T4Xg4aPtmvrwDReH7/qsXz/7mb/7mYDpmXjSdHKzuC6UFoWAwjAAAUgqENWU2CBkm4Ob2djIdSdX6PqcU+b6vGwWAghADoLquKopitVrVdXlwcHBxcbFeZwcHB5yFq2V+8eE6z2pMCaWYcyplX9fler1M03Q0GkjZJ0nCCKGUaEO5R41VWm+7vH3fI7SF/uq6vrm5oZzdL5YG2MPZgRS6yCs33hnHE0qxhQYiY4HpZNeLWipNGWyabrNZa60wJm3XUUrtQNfLuSsyOOdo54YshHCZvrW2LEuEkCNVAQAQwZTSKE3qur6/v7+4uBBCjCcjCGHX911bl4xXRckpC30/icIPHz5Ya6fT6R4bcOW4ECLPcydpbK3dyxxNJhMAQFmWTtjOobjGmCRJ3CRVEASOZ+6wyR+Wc4iRMlvgOvKDo6OjOE3c5FUUBNPJgUdw17R128yXa4BHvdIGIIQs2DePofUYcR7cbVvnRSWEStLhaDSiXooxruvaWqiU6aWAECJExHb2j7n7XwjRd1IKfTAbu/Kx6zq5gyuEEGjn0uuUB/eRqarbIAjcCZ9MJhCjjx8/lmW9beUC6OBfdxWk1A6vclPKzmOWMebmIxxtfj8fwTl35iwIuMsKXd3mmuhd1zHimhpOxcEA4BSjt3Q/iBEhhFCqtNbAMo9bCJQxmG4VFBDGBEJ3+eDOfIgQgjFFCLqCuJedOznGKFeJQr1XPASOzIGQQYg45j+C2FqotdLaYowIIVoaFyZc4H9Yze+73ftKFyHUdh3cDfvtm/EQQrmD0K212po9mXGfOsAH0gLWWu57ruT1dpiE22mp9H7S0qUgDoNsmsbhXtPpFIGtNXmapmWX53m+Xq/d/O2em+l856WUnHPfCx2O1fe9k+90SdWesgoAcCpMcCf8vOe1ELrN7+0/0aIG4FO5b60FECCEzD8xxwK78cWHWdT+nOx7NA/fhBAqKZVSRiullFMZxxA9VOJyPpRoa7K1veQYbYcs3A1QlvU4oAgRCDGEEGIKrHBfRObz+WQy8b0wGQwDj0ul26Y2Fv7mN38xHA57KZbrdSuVx33P8xjzurrykyAcJAYDv4kMsOtsI626uLjIm7KqKq2Usx1TxlhtpOi2iBN0HT4LIYRWC2Pvb24vwsCKkyQOFSKr+er6+rauW04ZJQz4UFuQFZW1Ngjjjx8/nj9+9Pjp86PjadXmVZUBKPu+Dzwax7FUXdvWgZ8IKbOugibwOBe9QghT7Blo0jAdxhNjzJvXv8vzsliXq8WSYY9BHAf+wXjUtSrf5FJo7emyrLoecu/m/cePxos1gJRQznxKVS+bMi/uyI0QQksFIUBA99CKvpZ9oGW32tz0oiIEKW1fvHyktcQYRVEoZLdcib6HbVsppaMoiuMgy7Ksrj0eIIaUUZ3sgUUYorrvLEZN12IMD+MgCP28ypquA9BQC6lHGWGd7IwFnFOkYdXq27trtuHJcBDHsdTK8bE557qUw+EQEgIA4Jz6fkh524t2va6iOAhDX0pBKAwiP01jY8zt/f3JCfP8sKvb+fJqkxVCGI/7STL47LMvZtOplJJAxBk7PJg+PT9Jhkmapl998/X/8p+uLOiz/PnZ+QlCaLVapIPw7fzm4v0HTmHXtJdX743qfvLP/uLpo6fjwQHUZHG/7rVRAE9nR3m51qYbDAYQ2jLf+GHMEETAGNB2faOUohRrLYsqq+vaGM050aa/u786OZ1xj4SRZ4GEiFXNZp1R3/fTNFU6hMg4YwtjTFFUCCHO/dXq5v5uiSClxPvu9TeOLDIcDpWQLoUP/WA0Gs1mMwRAVVUAgDRNN5uNy+5dsuuau1prjKG1VgpFCTs+Pn3+/PnFxcUf//jH77//Ph7Ev3zxmXvUCfF54AstuEf+f3T9Z49kWbYliB19rr6mXUV4qBSVWZklnu7X5JuZHgwa4JD/cQB+IPm5ARKNIYbT6OmeV0+XyKxUEeERrkybXX2P5odjZhlVPW1IJMLd3Ozqs/dee+21KCXG6bYr27qWqvZaLmmSRyHz3UZjlJQHH6PhMB+Ph23bAuAtanqMsXN+3SQsyD3z4ObmZrGa393dZVmmteac+zEQXyw2TUMQgmmSpqlv93garDEmjuOrq6t3794ppbyP6Kn36bW9q6ry/Ak/OD2bzfq+n06nXu3ARwVfCWmtu2JPCAnTDEJYlKUGzpua+vIlDqMwSZOAQwiN0kkcWmgAoYg5Z4XVxlqhlPFUGyllVRW73abve8qIUmq12pSNHo/HvlhM05Qpz63SytiD6o6DPkhACIMg8KCxZ0faYzMbAGDUj71ev9b7qs4PfPpcMEsHHsJhjEmjrbXY98u19qoSCJETqO7DRpZlXlL9w9CFDm5S0APmDngtRXDi1h1p5+4DrplWWlhrL8ezOI6LqrTW+uXcgCOSfCQBaGPsEb2v+95vGmPq5wmdc8Yc4GsvA2AOysrQOeicRYj4oGy0M04RwiCEWitnASReVdr62vfEBmBHWx9rra/XfV5rjlP+9scZvMN5OA1Q/FHA84nCKZ59OLUIPnRswsh+wDUBR6sqB7C10LfgtQYAY22UsarrvH05xdh7mhvnDGMEC9g1Td+2CCGMEEU4ZLynDGNqjNFUI4SiKAnD0HN9pHUelj+02NxRvPKYIjjnvFS/9aQNwg5X0MHTbeCOXg8/cjWchRhhjLX6Y33lP8oA/ihROOVbf/Q3pxP1YZPCZyfO2Q9kFLC/6NY4a5xFFkLsZ4I8+mImOeXc0zwhxk45gAgAgKTZYDAcZmkynY2jgJfVXms1OztLslRb87hY3d7exkn6/NXLqyfX4/H44fGOMDo5mzaiX5fboi6+e/1dK9pdubfQpHHIA0wpUULUdQ2hcxoq5yB0yAHkLILIUzrCKKjrer/ZX83ORSvul3eL+4e6bJ5ePZnNZuQowVa1HYQwSZJdVb65udHAYYwpgZNRytilMaa1zlgXh1QpBo3CCIaM9l3DeIghpIRHPOjbXgldl03ftPvtilI2GY9sr2VvvPzIYDA6m43mi2Q8O2M0kOpWmkYoaZxuO6WkwYQhRCilhFBrbd92GCMMAUZ2v1l6+8q23X/zu1///Hn+V3/5891+W5blk6tzIbrz83NrbcC41QZRB4HdbVeib/IsHY9yHMcQwqptatEZpYMgkEpUVXF5ecki7qxW1vRSGgRISBkm++XaeG4tIQQjTAnEyAPCdV0750LGEcFWm65tlZCgd4QQwpk2UkkjhBBKAwC8spkQyAHNOaUUL1fzt2/fPv/0GUAwDEOtLYGtU1rUvemV6qS+fm5EL7oOEGpVJ3tBCfru21+zMOi67vmLi7Ozi/X6QanGGDMYDJ5ff7J8uJN9o2SHoHVGTqZjcvXUYl4LBZTdFNV+tanr+unlBAAwGo2Ade/ev9FahpwySrHTxnZ1s5FSpmmKMQRQIWwYR4S6tiuMlZi46XRY13tj1H6/nUwzxoG1su/rfYGVMs5C67QfnYAQv3nz5v27W2tBHKech0oJKeVyOU+SpOuaMAw5pxeXZ/pW8oA6bYToKKWDQdY0ldYyzVJKsYfc9vt93/fZcBAmadM0ST4I46QR4l9+97u7h4dWyhwzaZQXMMGIQAL7vu77FiAYRbxpqHOWc6q11lIZ3Rjd+Oa0V+b2K2aapmmavn371vfsvZShj47OuYDz5XLZK7nf7yEGPgxsNpvJZOI5RkmSREHYt51f187Pz9u2bdt2sVicFv39fs8YGwwGfiHb7Xa73c5Pan399dceN4YQ+qmH4XDok4aiKJqmCYKg6zqfbCVJEoaB9b4YZV0UhQYAIGSNCYNASqmUKasaAIBYyGMcJ8lutzIAaeC0ccZYjGGIIaekrStCSN/3XScwJoTxqq7n87kDB3jAWtA0TSd60UspJQ04pZxS6gCQUmrtSQm90dJLajoHEMScBT572NRrv/pDDKy1Wku/yH7Yvz8Yi3ddkiS7soCHouwQIRA6eDCu12tvTDocDgeDQZIknrrhsw0AQJIk/ixVVXV2cQYd9DWoly3yPQLGmD0MXJgwYBD6XgBW5lAnCiUdQFXbAAAIo/uyhBBCjBxAPntACEGCKQt8CHEAGGutg9o4AA8uiMZZhAkB0BhjnBVKAocJAYQQCLA1whjnfIQDyAFgjAEAGWOAQ14ngFHq80J7JH6eSAb+vJ1mLvxbhNJTfP2QM4iOIkinmOeHDqA5HLj/Qntsrvd9f8obwAdeXEk68E/EAZB3zmeZThuvQbJerz033wNR4Djl61MBD+eAg8QIOiVDH5bspy16bM9DZRBCr7VgjPG6uj6tx+igZ3DCdSCEACBrLcY/Wjf50Q70wWDwh4jICUj4LxMF++P86o8fP+Wj9gPDMC9RgY5enUY7o51zzreXIITWAm8wK6UkhCVJ1nVCWxuGsQPQOogQ0tYxCAHC5Pzy4urqKWOk2O8QIUmcIYTyNH39w1sWBm3bRnEynk2TNO97+TBfXL24jtIkmoy7Yvuwedg/lOVqvq9LQhHljFKotWacyr4jCDrnAEc/zndCiDBAAAIABlkOjY3jWPZKVF3f9GEQX5yxIb26uroKo6goivl8fvtw37Ztp+VgNBZK+/E50TZa9IMkPDs7W9bNzc3NII3jOLh7fISIZGk0X676pqZBGDAaBwRZ2vdtsVlJKa+fXlxfPycw+N//49+9++HOaE0Io5RfXFwZY4piL5QyzgqteiXz4VDVCICu64Q1sBM9RThN0zzPnFHAmbPppG+rqtxDp7p6v7Uy+Gz06qPr3/2uxDjlnK7Xyy+++GK3K+7u7qqqmU7Pnjx5Upblw8ODL84GowkAYLfbAWCN0c45TCAhpKoqTKDVaLfbIgyUEkHAKMaUE2UkQC5OYuT9yqBL09havVwu15sGQJumKXQmiYIoikxvOedpPuSc73b79+/udsUeAPDixbO7xwet1dWTC4TAcr3Y7/fOWYjxervdbQutXRTEl+cXBSuaqj2fTHar5d2bN0KIwSBHCNVNCSFMhlyWXZqm/+a/+5tXH3+0WCxXm83t7eL92zdJGN3cvNmsl13TYgB/+OEHLdXm1z9kST4dXXAcAgWQA01Z1VkghPj5z/4ijsN//ud/BtZVRZkkgFKqdOuAihN+/exyOh1LKYXoCUVVVRmjBoOs6xovnt12dd3UaDgwxgBgug4AADCmlHIpRRCEGMO6rufzudb6yy9+9vLl87puz87Oqqry60UURcaY+XyupDTGLJdLaN16vc6yzFP6tVRZknLKGtL6GlTZA8UdE7Ivil/9wz8bYx4fH19+/Mn5E6GUsg5YB4x1AFpvw6itcgbSiAacBpymaS6E0EpI0ZbFNsAcYxzFIQ+YX5LartFGEYrbroEQZnnqHZu01t5h2fO6tdZREPrFse/78Xi8mi+Mlj7MV0WJIWzb1sHDoulXXu/W/bd/+7effvqpb1GVZeltnSeTSRzHUluMcZqmZVn2fe+p0dPp1HfNvTGVc26/33sEixAMnIXQIQJ94c54GIfJIB9p46qq2mzLsmqs0whAxlgUcy2VaIWSPXHQ06niMCp2O98fSbJUa015GCcDyiI/3y+Eapqma3vjLISQ87BuWwB6Z6EFP5ZTxhjOiJTaIwrQOgixX2ml1P7eoJj4IIqgwwxfXIR+pfY8eY/KeFgCY0wJcxD4qOBfxlkPa6dpSin1zDgvwqiU8m/5FqFfr4MgOCYKQkrpHPQ3mzEGOqO1xphhjAlBDuAgCN7f3c5ms7bvnHMCq77vAYIUga7rMCUYEGuAdke5Q2sw4845L/QLHIAQIOQggEIZqZVPSgCCvotiAcQAY0QpoRhZ56AQylqrpAYASSmNcRhLSjxRAPa9CIPgQHOz1msa+oLbH51HTQ7zgcdLcMIYnHMO/pg0nMpfPwbpQ52W8hQm/VuH11Hz+DQY6cNkGPEkiX0TrWkaZ70LVKiFpJQAYJUSWqpTO89YjSEaZHkURUVRIIigA7IXBFGMMYTIGquE9A0pRuixL2NPTUAPk5z6LO6oSOFJG+BHjsLRlglC4P6gZQDhwf35BCZ9mCV8mD/9117+Gz7MEk6vD3EXYwyCBEEAHNJaS6mUNRBCSjjGFABgjIMQKWWCgPjmdd9LTKgxTlnLEVRaOxhAiMlquVks18CZN29+wAhORgNntAcwSU2U1oQziEjXibrtnHPT52cBssApi12cZ2mb69JyJSwkhCBjlJLCasAwSeMQAQgt9VTVU6LknAPWIQCyPHfaffPV77umPRtP8jSTvazlel/QXoZKmTAJZ+dnq81GayOk5CFo277ruqoo1/NFEASzwShPE6NkGIV5kC6Wj9rqJCQhg70USRiHzKUJTWLyeF9V1SIMw88/++jjj3/SVfpv/+Ov9vt9EmZxnKRJvlkXfS8dwMAZHsZSrt+8fYsxgfzMOoARRYQyBxACFGGrNYbQWvPy+dOA4zfffXN3+87p/nx2Cazab1eb1eLq6qnqxX6z7xsRUP7mu7dSymE2+OjFx9W++uHbH/75H/5xNBr97K/+OooiRkhAcCP7UipCKEF4MX8IKEMYNlXvnKF+SNy68WzadR3GOElTY0wnD8O7npGA+h4B6Iy12i89gBOOMR4MsouLKz+sX7eVv+OTJOYBNUY5BybTUT5Idrvdvq8GgxEP48jCOIiBhU4YbCBUut7upJQA2LYslJFa6yBkGHLKmNaya6rvv/2ubhvGeJ6kqhfFbv94f9+2vXNuPJkFLHx7P69YP41DJJv7+V0SpB+9fMUxqLebvq2++OlnENj3794q2ZdliRChjHNOR6PBZDL5/POfXF5eaK2bpnHAfP/990KI8Xi83e69VaDWOuC6bfog8KRx5Bz06ijWOELwZDrKByl6AFrrLI+l6qRqEAZByIKQRXGAISqK4t2728eHu5cvX87nDyHjm82679s0jTmnwJmi3Akh+k5aa/M8Twc5RlRpK7q+aZpOqKZpskF+fnHVtu3jYsnDmAWRXzFZEHHeEsKEEF3TKqWiKMqyTHb9Gi7rtqscYMORLwUwhlr7ok30PQrDECHgnHXOQOggdBhDSnHb9UEQQIKDINBWeap2GAV5nhNCRN+u12tO2W63m47HAADPvfIovZTSmzX87ne/2+123pzGm0b6ajvLMoipMSbLsvl83vf9YDBo2/b8/Nx3oPu+99O5vtUVhmGWcgCAQzCKovFwtN5spRAUYSW0ErrrhLYdJtA65+Whnjy51FJpqazxKKju+xYajTHUWtrG9p1wEABipbJKOwytc85oSzANQ2iPyyUiB9VCbQ1C3vbJt4ecH3D3caXvez944qMaYyxg/BTtwjDkYeRTSc45o3w0Gmmt27YjjDLGMCLKaISQR9oxxoTS0WiU57m33vCtfU/18CKAQoi2beM49m63lFJgD90K34DwE6Faa4oPwcMYI2UvVe9n5CCELAwQQpTwoipb0Qut/GAUhNBCBxzwA1NaGkTJMf76OAQxJgghjBWQSCsF4KGjTwglhBJAMMZ+yBAhh5BRUjunnYNSaiUNIQQGhHMCjo4D7qiP5KOmr6R9sY4Q+nB+z2exp4/4G8P/w7cbDt1x8IE+4FGR4pRYHKLGUdPw1M7w5/+oKmm9OTt0gDHqnOUB8yafeZYrITebjbE6SWMrjJ9o8BgPZ0Ge54vFwh71FTxB0t8PCCHP4ACHhuOPDgj+qMMw9G6TP7Yk0Enz4A8YCfBH/o1/A/1RivBhlvAh9PLhP049BQAAgj9++SmxcEelZwihte4wCQIRQoepCmUccN7rkgBwQICkMpTwQT6qytvVZi216qTQWgcEG+Osg9A5stpufv2736Zx6De33++F6OqqaNvWWOucG06m05lBBGfZeDgcCiuLpq5k20thkQ2TsFUtEbjreoyJs8YYRRHEBDFCMUTOYWwRNsg55xMra61zpi4rqG2h9OPdvZWKIQwd2K7WUBnrBOOh0IbQwDiICLZaP8wfMaZtVWul0ihK46xr2sXjkoxi2bfWacZYlsRVU2Nkn1xNq6bxjl5piofDoey3XdednY1fPHs6HY0euxUwFkMwyPOr86vr62eb9SofjIYIIMqmZ1eY8F//5muIEUTEQ5qekmqM6bqu61tGoTUSOIMhAs6kMU+j8csX1xcXw6ZpPOHr9va+rpu/+9U/EUK+//5tkiRV2RkNx6NZHCd3d/fz+ZKlWZ7nWtns+cs0iKTUSghjAceEYIQBNABaB5CzFBMMEeSUcoIAJAQ5Zzin1loI3WCQBQHzaLPWuii01VLLvqu7ruvwwXzMMUaTJJFSvr15kw0GSou7u/eUwY8++ShN0+12jTEdTyfnswsPqvR1n/AwGtLlfMU5HQ0GlOK6bRAms+kwDMNdu/eT4lGSvr+7q6p6NjvPs0xLU2yLh9vFdr15/e2biEZxkHASPP/o6acfvWQ0btdb6kRMgNUCIFjvNwTY77/5vZYiDMM0Gzx9do0gRr3FGI9GI4xxWVZ1Xdd1ZYyxBkCAgyDa7+dCqBcvXiyX2yBI8nzodenruq7rNgxjhFRRFNrIqqrTND2/mFprm3a/2y8Hg8FoNCrLsq7r3W4Xh1EYhhcXF6PBEGP8+PgYMm6tXa/X0+nUP9Xb7RYehVyCIOBR6MdxHxdLHoIwJQ7CME7e3Lxre4EQaloZBAHCVMpOCCOU6ztdlg10QAiBMcUA+rBNEKaY+GjtWxs+gHlGt6/mTxisZ8k1TSMV8MVNHMdVU3oKgl+/fBuiKIqAca+3MRgMxtPJmzdvvBWyD2B5nj8+PuqjT4GfHfdkSa21Zyz6aiOOY49AIISur6/3+/18Pvd75ae5+r6n3DIaQAjzPMeY1027enicdALs97tdsVpvhNUsZEEUYowdAPP1BjhHnAsIAQQZa6uqqo2lmNRlpYyt295AhEnVCbmvynGWedTBOdc0XVGVXddZA+IsRQhRyq2Sh/rVQuAQJURrjaCL0ogi7Gt9hA5cRuegMU5KLYRCxyfdxwAhRFN3Xv7yzZubTgiMMQTIx3WvfGUtIAFljI3HY69Q6S9lURS+xPSFfhiGaZr6QKi1RuCQJTjnPCjtA4D36TiqXEhtJABgMBryKKy71jmHKWFhoJ011hqjiXOHaQ5PMwTOaK2kOqZOECGEMAYIOAgAwgBBgCCEBELonNHGOAAwoVpbrYVz0FqLEYEMeVFFCDFGfgs+1B1sOHz886iAP0C/5pxwBXBsLiCEeiH+MPodBYnBB+YFEByj/oHa+SOicATwvYHRqS7318jbeftkBULIOUfgYLpojt5afrMndWQMkUcRrDYIwNFweDabzR8fpdDOOqO1x5gRgMA6Tyg5dQFO9Ah/yTy65kEU+If6SKeC32dXSh9cUk+9LYww+CDzAB+wE/4oM/ivJQqn/OKEr2CMzR+oNR+SM99MMcZ4XqpzznNrAAAQYEYDpVtCSBhGfS9u35cU2SaNtHEwoNaHbADJbHZeVcV0NP6zP/0lo+jt29e7zWo0Gi0Wj5TSKM3Oz8+n06mXDLPWTqZnUsq6req28QSfvu+rqtJaIhQghDihjBMfPgGwQEPrflSgNNZoZZwxQoi2qoGxBOEgjRnh0EJCmNE1ZQBhW+12vbDaIqG0NbCuWiFlV9VWm5fPnr989uL25kYIsbp/aNvWtA2hNInDXjXWyYvLS7C0ccLquuYBurwYl8WoLosXzy+cM0W5m8/nu/1GiIPspdY2zTIArTGKhcF0dllW/WZbxnFW9elmsxFCNE3XtjWEMI5DRlHXdAiYotit581ut7k8m6VROJuMgLNVWQaMF7vy7v3dZDK7e3/fdeLd29vxeHTz5vbFswdC6MXZ5XK+KMvyh2+/S5KE8/DF0+cX5zNn4Xa7r4vSGNPXjVLCo4gIOs5pEATBIPUYl58DDsPw9PwcKNZN49lnfj2tym3btvuqvL29pZQbY5SWUomrq6tduWu73g9bV1VVlvu2bW1IdrtdU7Wy6UMShCQwSsleDLK0ruvdZmucVlpMzyZJkkglnlxdSaXW6/WLfHDswaE8GzqLnXaTyRQBpLWFAM+m530nRVN25Toc4lEaUMSt7PqunU6HH796MR7mv/ntPw2HQ21dlo/TNJ0/LoI08AvQYrFo23a/3/uKME1TQjCl1BqwLfZ/9qdTKTQhZL+rlLQAuP1+Twg5m13k2dA5FwTBu3fvtOlfvXr19OnTpmnu7u6ePr34/ocbSj2f2QUBHw4HGOPL84u2bW9v3ydpXJRsu9vUTYUxttacJK26/qDFFkVRNsjDOHl3e1e3TVnXmPLXb28AAC9fvoSQBEFMKXYOcMrCIAqCsG27YrcXoo+CwBiHEAlYmKZ5kiQe4TxGEXD6d9u2nljnce84jpVSm80mCLO+73f7nScl+B4BAGC/3/s1XVnl198sy8bjsXPOC3Z5yl7f915kYjqdelkLzvnZ2ZkPbLvdbr/f+xgQRdGLFy8+++yzzWaT57nXRXj37h0AII5jAIAvsouiSBNgjMnTPIqyd+9vpfAj+EgptSuLoippwNMsCyIOCX6cF5yyQRLTQUYZ4wiZ3kohmro6m50PRjmrmsfVuqq7IIwvL54UmyUAAEJsj4YIcZRyzou6CoKIUmSB01p7KxxKqdbKT3YQQkIe+EWMECKF8FFZS1VVlSejBUGgrfHJWVVVq+XmxauX0+l0s9mt3r7BGBt8mESIoigIAqWMgdbPNfgiezabMcYeHx9PshCEEM8yqapqu92meYohPpV9J7DaWmuMM0Y5Z13AOOdZkAyHQw+BrNdriNBsRryGAYDw5v0tQggD4CAimGBGfRGmlT6EIowRggAAo50D2s9Z0IMt1mEwxDlBQnrsGni7Ck96h85BZ6HHq40xB6MGCD/s65/KekLIdDr1CJM/HJ9vAQCatj0FUefcaRoToB9/6T4Ik/4uOgVjeGQPUM4AAFprZA6aIr7HwQLOGPPHFccxhgeJ7rauJ5MJpXS32ykh0zT1A+on+Q2/t4yxNE0ZY9YcSBXgSF30JTjiDBzJlQcs5Ph/n6D/2F84RuhTonAq8bU2nHNrD5oGB+oA+IPA/+Hrj1KE/8NEAfzhu9Tvs/HoEcYYQwcAAP6Jxph6NS17oDc655yzEOKD/JSHOfq+31SrYRz0MndHiU9rgbWaZHHktKKU9kI5wGcXT0fTi7IsewMhhPv9vmz63371dZ7nl5eXcRgqSsM0qYxuyh0mxALX9Q3FKGCxEJ2W0jmkJbYIaR1IKSHsCSEIAaVU11RKSIZJQFkAHKTEQXd++ZQRWu5Kpw1CJDs7u7h+rpSrWrNYz7tWchavt9u2lbe3j//j/+3/EsXs/HyqTW9yt1wu6kbFIE2SyDkXsADGgzRNF4u7s9n48WFhDJSd+O1vvs7SUXI+Voo/uXxBMH/nloPsbDYFRmOlEWVJVe8vL68e5vcOMu1Ip8Hg7HIyPgcPrTcT00YjyBgnlIbWGUrjMODv369FUw/ysXFJWVuMhxu3Ds7On/90Wm12t+/v+7JAovn47PzZf/Ovt9vt7u0b8q/+EmL0+atPFncPVsP9m/evfvazZ5fP3Hp7dfUkzbLf7NYaqWSQFoV+mBed6HkYIISMdoxEGtqXz57O5/OmLiGEnNDhcLhZrzHBIY+KogDWxmG422z2m82rV68wcY9393mep2kIodhtVp2Qr169Ms5ZKFkwcc69u32/Leqzs7M0m1re7XfLiIcY8VbZOE14yOZvt13dZ3E6nc2cNtvtFtpAd7CtNUpIHOckcN9+f//11988efLEOXb/sJxMJs+fPz9/cvXDD69//etff/vwFqb4+mcf2UrEcSqlPruYYkyUbhyzpajPnzzdld1f/MX/mbLoH/7h76qqqHeL1eNbuwuSJPIEuv1+jxBKklhrjRA0VvV1C5FL0uDb7756/uLJfD5H2gCoFosFxlhIsy/WQjYXFxdBEABg379/7wvxJ0+e/PSnP5VSRiHHOGrbttGyKApCSBCEX3/zzW5b9EI3vZldXEMSC43zOG9FAYhxmERZHqbO60FRgiiCBOpRyimQ0bPLuq7PU04pDW3/3W//7vr6+sWLF8R0fV9OB2lVbYAWgyzc71Uch7e376SUs9l5GEdCCKW6pmkIwpwx36HzzXIaZcoaYxyh3GFaCdkqbRnvZGehTdN4vV6ORiNnnRECc/54836QpH3fC4CNta8+/XSx21w8e7qZvy/KddeXGNPZ2dgB/e23v/eMmfF4SknY1I/OgoCnCCFGZRyb3W733Xc/XF9fT6YjxogD2T/986/CMNpsF0HIvFbSaDTJsqExQPX2bnP/5Ze/UNJ0fZvk6fR89Li874yquu7i8uk5pIvlSilEdCpbCWnQCAmRyFIkHbbaBEE0m0x3641ygHJORF8UOx4Gs9mgl8IoBDmzGjat6HtNsSeyuTiICSH7qvRhvus6D2hznrSdchCl+WA0GuHtVq3XSmsUBI7ibV36WG4ZXtfFrquHadI0TRQmAAAWUK/onGXJ2WSslFGiZxgHPMQWqLZ3zmnk0iTZbreMMYLxu5ubOI6dtQTjuqr8SJ5WqmtbKYSSsqt6jLEzwBnb9g09QH0qTWMIIYMBYyzOsiRJIIRtL9+8ffChi1Ky2pSMcz8oGwYJhBA5jAhBEFmhjTFOGwwURZRgBKHx6LRvwVAAKKXOEZ/QIOMCTCFkSmhKaRhGjHFffiillLKEEO+bPBkPAABN0/k2fxxwpRQELgy4B5y0NZxgaI2SQgtBIIAIUQSDgLMsbdrWY7Eey4EE+/jqOziHUAR+VP6RUHnBH0+9BEc/JOydRyiyurNKE0yyNCNDoo+ZhP9LaJ3/cZjkXrY5CeLD1LgFFNPeNBjjoiiSJPFNh6IojHZBEACACGE+jhLCOAs4kwr0fd8rKSDGAEGCnbPOWAuAbZpKGpmmqTRSaDEYDJq21cAZCCzy4dxZ78GhTdd1lHKKKXQAWIcIhA4Y/SNiccoXT7EfHR0xfGriEzIpe+9KChxSSgJAKcXGGGhdEATOWGct5yGltG9apQyh+NjNsVofTEmapgrjiFKitfJplpQCY2StcX2vsNN9B4FVSvCYK2AJpeTq6irPc2PVzbs3vm7gnAJgnzx5oo0qyv3t7W1d19fXTy4uzgaDrClL2ePdZrFZLiByXVsxgoNBVpdFwKhGUPZC9O0hhQ84JRQAoIWsur7YlX3TYog4ZVfnV5SQPMmeXz9bLzfv377PsuzFi1cvXjyFEBfFknN+Pp1tdqWUGlgXRVEch1arOB4653744Yftdo0J6ktRbcs4iF+8eAmhy7Lsq29+HwXxZl1cXT27f5x/9fvvESJnZ4IH4aefzspSXpzPzi6uk3iw3nyzWdfTyfknn2zjJNBWDUeDummc7iCyj4/3Wuu6Jh5WpZQCoHztzhhbLxctJVWxCSkZDlJKqTPi/v7+yVOe55nqxWa1FX1PWBDwEEL4+Rc/LYrq2++/r9smytIwTs6fPNUAXo2G8SDXCEhgWy2Z1QrYTvRPRs+ywYCGwWKxAAj6ES8WBIPRaLVZU0onkwmEsCzLm5ubKIrq/f758+effvppVVVv3rzx0xxBEADKR5NZGIa+EkqyfD6fL9frKIqevXj++Pj48HibZZGUcrNZXF9fh3kYxwmEuGt6zsKz2VmxK/u+z7IsCWOEkKNwOBxEURRFAee0B6pu9hCZ0Tj76OOXjLEkjYBLmqb53e++iuOYc/5Xf/XXr1+/vri4+Oyzz7RAEMK+k33fAwOcc5zzQZZThqUG3/7w9e++/qYoqjAMozCOo8SGPM8zzvl2u22aZjAYpGnqVyiEgX/ardV+xTk/Py/XWw85+gfMTy4ZYz766KMoirzM4n6/t9Y+PDwopRaLlTHGN5WzdIAQEmKxWq2UUpyFYRhyzofD3FpbVcV2u50Os8lwdHV11bZtXZRtWzdlYZVUSrVV3ZQVpXQyHOVJWtd1XZSEESHEfr8vywoA4JkBHqMOw1AptdvtPoQ0feMcOiCFllJySjHGCBLEkXbWWOdlTpTRQhnnIA+46SwC2MsqaK05D66urpqq8qqdfd9KKZbzR2PM/OERO4gRAw73nRC93G52QijvrZwkkTGuKIOmaZTuGWM8IKQNRK+chc4hrUCx3z083i0WizSNvdthFCaMBeRop+TteqWUXSsI49batm0Ho4nWuutayth0NoQY3D08FuUKQuhUzyimhCgtRI8xY8A6L0LgF8o8z2ezGaYszbPIGGwDrXXbdcYYr7RjnFV9H0URDXjqGQnAGWf9rUUZHgwz/yx4nrxfN5VSxiiP1sAj+VEpVRhnrSVY+oERCPFqtZrP587Bg9syxKe12znXK8MDQFmAMJZS9kI50PVCCakdQJhgB5CQmlBDKB8Mx15xQWqFKXEQAAiNtcZZbY1vYZydnTHGdrvdcrX0TUyPgkgpjbUAQj8XMx6P/Q54Oq3HRY0xQcROMPtJjcNau9vt/AGeAH9f/jLCjqwIeaqAPbDNOU/TdDAYAAAIKTwNxUvIwA/YfP7UVVVljhaRfgd8F2Y8HvtkxTnHGLPw4JGhjfZ3C8bY80N9JuGn/4+7AU5ohNei8OfBJw3+9+zoTGaMEUI4dxghSaL4YJxxtO3AR5soQhilB3sRKZW1tuu7i/MrQrqiqKQ8kEusM5ggITUAFiGAEHLoSGIwxs8Ter8MKaVWPMOLxgABAABJREFUVvTqZJYBIXQOnAieJ/zjxK44QMLGeM7vfw1ROOETp+wBY+xNA9yxxaO1tsZQHnw4Xgt+5FL8AfDgKZYIIQ9Mns6Mx5kO3ZkjXHSipBBKSbHf+ovk1Si7rhOyD4JgMpkEAQPWEILjMMjTBFiznD9mw3Ol7G673iwfIXRKCSGaJA61EgBY6ACjkGBmrYUWOi13u6Lv+6Ioyl3Z933IwulkMh4MP371acSDgPM0Tua3j2VZU8y0VM4AQtFoOMlSI8Xt/vVGGRCGnBmitf53/+7fOWDOL6baql/84md/8ie/+Md/+TWHDBMSQF637YsvPv7tb74xAjy5eP7ZT7/YF//bt9/+ejiaxql6d7+WBj99NgpjvSvUpuiUQXmSvb9/+H/8v/6ff/2v//L6+vL6xZNWdZhga/W7dzfL5XKYfyRERymJkrTrOt8+5DzO85xgoGX/7Nn1l19+nsSh7NuqKEyrkkk0f3wsN7uAMo+orXbbT/hPX316sSp2jtJ0NIqz9G+mo9V6/er6aVmWPnOM4rjv+13d3Nw/PH35Kk3TfDCqm845F8fxiTLz8PDgwd4sSSGEr2/eMsYiHmBCRuMxCwOAUT4ZPXv2DABgcViWpXMOYWit3hXFrijyPK+7KhFR19fWKsZx07Z1U1l3Np1eFrt9XbeDwehsPKuqarFYJElyNj3jfhhdmSCOOKeE0XE+Wdk9ACAfJ8DBuqvfvn2LGc6z4b4snQNF1WBMf/GLL4MoW61WX/3+++nFK8ZYmI4GOWqapi4rowFVdhDGr29uf/ubr9+9v4XApSkzxkEIrTYhD4b5AFjnjE2TFCNU1PXt7W0QMC9/BID1TlrsUIXDyWgcxzGEcDgcHmad6yYOo4Bx2QslpOyFozaO4yxJD5IYFlFMgHXAWgRgGid5nlOCtBJxFDnnKquHg8QpobpWNHXI2E8+euWFWXa73Xa7vZjOzsYTv2BVVSUgMhCxKLbaVEXZNm0YhoxQTRngEEKYpwPnHHQgieIoCI3SwDoAUBjGSsiuqZRSnHBnoYW2rhtECURIW9f2fdf3QitrLR0MlTQQQkJ510trvBPuwBlTFAWlGDqAAKyqYjweE4IW99uuU1kWMuZFcI3W2td8CCGELEJWiKYoPI0fhSGnlELkpBS73aaqCwDMbDY2VkUxJ2QAnKdEEUys0i0n1LtQ7nflYDT06rkIoX1R9H2LCCQF7fraul4bBSFMonA6Gg/ylAFACInDiCEoRUcIccYqoxFhUZK2fbcvK2ttEMZCCOxhgGNCgDFGlPgxubKuvLqD1ppSCoCN4zDLEudMWTZ1XSul/MqolPLLvXMQY8o5NMYwzgAATdfZthsOhw5BqXSYpL5OtcZ5YSWllBc0QhT73M4c1YL9Wo8Q8vtzmpP0wtteu8K/6yO3309PqEJHxuVutyuKAgDAKIYQak/6UwohBBCUWrVtCzDySYA8qv0wzuM4Oi0UShkItSeREMKMMVp75BmebID8H2utldI/IufAuw843+YDAHRd78/zKdR9qArgY88pyBljfHaOMQ7DyPd6fE4gjfbJ+o8YAIR+FOIU5E4yQf5r/Rd65oGfA/KX70QV9AmlByS0lH4/TywQ30fwFBwIIcbUJxn+khmjnWMIoaatgENd1wgh8jxljPhpZHA0bqaUOnToJmBjIMSU0uVmXRa1Mc634DEl7sjZt/bHBuJhEBf+aDHq709jDD6Oj/5RlnCCFtDRFMOTQk6Jgj0SJI0xWikYhP5T9hjdD4kRPjBJj72Mw7Xz/UTkDbSOiYuvr3yHAiFk7eGWoJSSfJAopXhAwjDEGL97964v2zxPy3KvFG/bOghGl1ezjz95NRhkZbU3K9R29dvvv72/vw04tU5J2XdxGEWBlMYZGwYB52Hf98vH+XK5LnZ7rbUxLmDB1ezJ+fR8OplkcdZWPQWs3u8e+/lms7u6uHz+/PlkMuk6QQgjGCthmrpumiqJ8+Ewe/7qo/F4/I//+Pe397cIofPJxasXH12cXT49K/JwdP9wW2+7qq03821XiTev766ffxIG6WA4G43PWRABxNtOvr+d/+e//c3z561TYDR9ko2W29W2ahrG0ZPry8snM4ccY4RQ2oo2SSNtXNc1Pk6PxsOyxEL4ZqREGBqjlRJZnnDOt9u16NrH+wfYcQbQ7e2d1ebFi5d1WbVNv9tsdlX52Zc/u5zPaRhoB4qqSfNsOIE/3L3fbrd++o6KjhFy9uQyGeZFWytr6qZW1pwo3NZazQDGZLvdFUVxeXk5GAxYENzc3FxcXKz2u6bvX7x4EWUZAEBamyTJ9cuPtttNWZbb3Vp2qulaoWSYBNWy+P3vv4ri4PrZZdM0GLmLyxkPSFt3Tdl0bX824YPBoGtk2/YYkKIowjCUfa+1DGWwWrfW2slkMv10mud5ng8fHxdCdfPl4+z87Cw6H9NJwMOvvvr9aDRp2n46O/vbX/3d/f3jsy/EdHL20cuXgzgVdVf7yWyp+u222KxWu722LuDEAqCsSZKkdYcJqzRN/RphrLbWTqfTLEuklNvtlnN6cXExnU6jKKL6IB7gnY0opV6ccfk4BwAQiOq6bqs6ZHyQZtdXT84mF8aY/X6/Wq2klGEQDLJsOh57P+WyLIuigJZbY5xWeRJP8jSKojQJ4zgeDoeXl0+ccx51QwhpZcqy3O12RVTkaSaEoIPYW3Z5yogXyANASSnD0AkhfJCQUvr5Qx4ySqlR2hjnZfn7XnjZRMICFnC/GBnjvE8bpdxa5+t52TRhGDqr27YLw7BpKgyRUgI4K/o+S+MkDucWIUiBw8BZBKE1wBjny1whOoShdaYXnVQdZSjLMh7ap88mnDMeoP1+KVSTDeM8T7uuOYUKv5b1fdt1XV0SQljT13Xd7MsdYeFkMun7tq5LxrGU/du332urgigkAWnbttx34zxD0EkhiKMwjSEEUsowSTvdtW1PudNar7e7x+XGWjvJp4wxyhnEqGkaIQThzAtTEkbDMKSCKmMo523f91JGAUGIGmPKsmyapmnq0xrqy2uMsdGOUsp5CCHkjHDO+34uREsIwZhyjsbjqVam67rdbrffl2VZekID5zyOR9ZBALFUUmmLMHUAEcqjGJ5UegAAxgJjgQNIKKmMhhhFSWyMcW3rqYiIYKtd1dT9nZBSNk3jIOCcGyUJIQGlnpOkjJZVtSsLIQRjjB0BAx+NMKP+ihwQCGP6PxRGNEfNA3BkGHwQloB/viCECB58qL35pG9Uo6NF9UlywJc3Pq6f+vQ+/nneBoQwdge9KZ829Up6TMKbVhwGLBH0yRal1JkfxRn98+vTKS9q6YO9R63w0R/B40NSyrZtjVReg6TY7T1jF0Hoz4M+eFwdYvkpIyEEExKt1+s8zyFyQchG4wEhpKp3WmsaMuQghJBQDPEBp3EAhGFMKS3rthQlQhhCpLXBlPqhD2utV+30mQrCB6NI8IFAAjhOQ5yuzn/ZgDhlQu6YFpzyiRPY8GO+5Q7tmz9AFPDBqOywCfgjK9N/yqM7nqR/eqKNMYQQZ5TnYIZhSD56+cLDsKvVChM4HCTjUTabTfx99nD/viy2SRzkafri2TOt9QZW1U4Wm227L4PxIOSMQpDwMGCMAGiUBto0fbnb7R7v7peL9XQ85mk4Ho6vr59fXj4JSFhVzX5bqNaWui33277vlZDeJSWKojiIu65/+/ZtXbVa6xfPnnPOm67fbZbns/FsNmWMAgCUkO/evncOvv7+NSGkrrrxeDR+On39/Q1G3Ch7+/YO4zBPBz//8hc37+6UNGfTyyzL7pf779/8xzwZjmcXv/zTv/j6q6+auhjmoQI2SpPletXUnUP4q69+H0apkkZ0AgAbBCxJIudM1zVKKeuMkr0UnVF927Zvb17PH+4ZwW1TrRzp23673V4/efrq40/Xi2VRFBbAZDhMR4N0OOqlaubz5XIZRPF2u/2Xr/65bdvxeMwIpZReX18/f/78yy9+fnNz0zTNvqik1GFIhTbGmCAIsnz45Omz169fGwv8qvTZ519QFiilQoSKulnv9s+fPxdabaoKMDbOh+PpeLF4XG9XhOGzi1lRbd++e3t5eT4YpaPhkDFSFPvJZDwc5Zzz5bx4enXt52uBhZeXl1XV3N7cHSZtCOSYs4BKA5XQ2grGWJ7nELqi2EHkrq+ffv75T54/+2i73Xed8ONFvneDEHry5LJq9sNxxmKMA6ScgBzko2yYZJvVYt8UUguAAWEUUiykRJyep3lVF4vHxyAInLUI4yRJB2n25s0b1Qsl5XgwzAcpcG6/3TJCfvnLX3oPJwCAn0zzXDA/S5llmbcg8uhlWZZ5PvZgJjpOrHlKDSEoSSIhOucMgIf/eEDOLyZJkoRh2NTd/f1t01RBEJVlcXPz3mtecRZMp9Pr62sIsZSyNp2XdLTaEoRF1++3u6pqrLVt3TVNE8ex7EW5rzzHEFPi1dMQQlEUE4T3ew/wDIRSQiiozKnRK4Qo6iaJYkwYxtBCNBqNdvvN3d3deDigmCgljNYEAtV3XVPv1+vJZOIHIqQUnqx+rLyBlBLhw/olhOy6LooiB9vJLMoHqZS90I4GiAeq7ZeYOqmEFppzPsyGYRjWtdGbar92fd/3agIA6tf9F7/4k3yY/cM//ToIWRiniOCB0xAjqeVmt9NaiEZI0TmdUIyCgKVRzCkhAPpmeVFVkQMQYetg3fXGGNU++hQtTGKAEWxbz3SzTW2tJYxhSrlzEKPtfieUPD8bOOek7LXWfd/5LIEQVFVNXdechZSQXvTGWEIohLBqah5EPIi0cQjTXgghZJyknej3ZbXabPf7vU8srINKWx+BfEXr+y/+N54P/+FLKeUne/27frDCKzIxxoqi8M0j70FwbEhLQhBhFCHkIADGOASttUjrNMsOnRdwUAHSzjqtuw54IiFCxN9FPtJ4Kckj0gwA8K2Tg9I2AMDLOB5CBUBKKQAOwRtC6APNKbqcOgInUOTDavgUxvwzeKTLQa21soZzPh6PhZL+BjbGQIB8+wBCqKXyOcfxSh20Lz2D27czfD/FZyTWHcZfDwMUzh309YX0KXiaJP7qHCWbqD/zPk1RSsVxTAiJonA0GoZhYIzJ88QYgzHQ2gDgvQKsUg4Y442UMKZhGGJMEUI+ygOAAEBGQ2cVOLSxDm6QAAAIsBfX8meMc04IOwIPh9eHiIL9QwuoDzMDz3vwf3dCO47Qy/+BgxT8gADhDkMnf5Aj9n1vnPWaHxD6efLDzWy0O7GnSbHbxHHct/UP331trX3x4kU+GlXFHkIYhuFokAVBkCVRHNLxMEWUpmhkhQpxSCBhkHLEsAMU0vXjljKMAazrerPZVVWFHH568eTnP/0cAhzHydnsajI5swp2lVK9jXiilIKW5OmgAsV+v3l8fMQA7mRhjNltC611HKd5knZS3N6u7m/f13WJMR2NRmVdF0Xz+LCYTc+//f4755xx9vzy/PrF86++/WY2m/1V+ldKm2KzzQajpxeXq8UGOMiCEDtQdkL0SurCWDKazv7t//hst1p8++1vhFYQo04I7YBT7v3tnLO0rlpGIULI95sopWma9qLzPWYETToeEIKE6Ebj4eXZLOB0fftDL2SaDaI041HIkiRwLlc6SfPNdu8V5SilVtlmX+6W609fvPK3zmq1qsrq9uad7HoC0WQyOQy3YMyDwC8Kg8GAZ2GWZfv9fjQaGWP2VXl9ff3s5YuvvvpqPJms1+v5eqUQCMMQQhgkcQ6F6NqyK+JBNBxk+/0evkEU0C+++OKTTz6qqur1D995H0WtFQAgDafGmKqqZa8e7h6tBc7Ys7OzrusgclIqAGxEg49/8vHFxcXZ2awAbRAE33333bvbO+DQ02fXo8mYMLzerpQySZYs149f/vxzxoJf/tmXX3/9tdRNyHWeIoL6rl91XVtXuto/yq5fru5aUSMGwyzABD4slmVV/PLLP/HYtR9Ic0cxnMlk8vBw17btxx9/fH4xO3gcjMdZlLZNI/reU9mdc0EQpEkSBsHZbHZ5cTEcDPysYMB5VZbA4rqut9tt2zQYYwRgXRZN00wmkzBgxX4n+nY8GgymwydXZ+Px+OHuXdWUnHMpZd/Jtm8YC8qy/Or3v9usd03TRFE8WU0Gg1GWZXEc96Zv69paa5XWULbWVfuibfskSfq27ZomT1OjVN+0EQ8YY20vdWiMtt7B1kjjJY2fPnk2Xy3ny0XX95hxgKCSRklTVA0E2DoYR4HSmgZh38ntZlWXxdOrS2tMHPAg5MBZ0bXlfkt4JlUHIYziYDAYcM7X69oYQCltmg4AK4VyFvSdKmANAQFUGKOF7IwRlOKrJ085J7v9qmlLaKwD0gKLaRol2EHS9ujqOi32VZTSgKdF2YzGqbFYKpEPBhDT4Xj07OULTNE//cs/3bx/b6yaTcdxwDmjMQvSMEAIOGsJIbvdrm17B6WxEBKaprlyyFpbb/dCK4fgRRQmSQIx8ot+GMcsCNI0bfsOIBjiaL/fR1E0Gg2rqmrbDsDDiBohLAzDpumsAZTSJEmBF8NysO+FQ7ZXUhrdin6xXvtFOU7zd7f3VVWVZe2cCyPvuwGl1s2+oJSORiOACYYIANBXtVIKUWb0QWqJMQYwUdaJXjinGGPUc/uVQoRYACwAiBDCmLW2l1JrnXBOKe37nlJCCNHOSqMBACEPKGOHYEwwhBB4UMRZvwxa4yBFCGKMnJfp9UCCNQ4CcBDlAw4cVZPRscQ89p4QhBACiDE2xp2KeIypb6NAd5DK9ifz1H04RaM/AtJ94+NoMGEdgn5YdH9/d+KLoKMUprU25MGpdeK/wecu3vDWYwy+5PWAEMT4lCIQQiA6eCyxOMYY+215eOM4V+yVrRE4KmEbYxAC5+ezs7OZlHJfbNuuFkIYq6zT1kGfTEuJDHBGOwgRJgxCDIBH+7TWvRQ6DEMpNYLm0A7QP4pIHiU9DiJIhPhi4NBQOGEA/sDdUW7hw1N66vgAcCB4fjhAcWoJfZhYnECIP4IrTi+PAymlAIKep+WcS5KEIOtHcvw3+ESQrNfL/Z5I2Wd5KqVknGKCqrpRSkHoGCPj8XAyGXHOu76xre0JF41SvVKdrou22O2bqggCVtdVHEWEkKZpmrImhFxeXp6fXV6ej+u67Vo5v32sdy1GvGslgXS72ltre9ETkkQ80GEMrSuK4vF+NR6PkzhGCGlj2rZWyhAEFsvHoijOz8+TJEEAZHF69+62q9teK2N1EIabYp+vF7v9BmAwGAwsgH0ntOyRSSKM26avmhYC/Laa//QnP704u1zN1/fz+89/8snls6vV+r7ru+2+mEwmSYzDKD+bPd1vO4TBCaWs6hIAQBmxjjVNZY1KovD6+okfV/3k448uz2bAmbZYSClfvHgBLdzXTSuFRZDHCSDYS0HEYQQhDBAVQgSIjsIsShKMMXMEzC4xxkLJcltcnF0O81E76YuiYGHkR/IgY0VRPT4u6rrmYYgQsgY0bf/s+ctNUVJKc+uqrn+cL4MgGIyGu7IaDOOqqnbVGlKLCGi6Ksnin3355fT8TCkDABqPzjCBeZ4LITabDafhzc33UujRaLRard6/vwuD+Pr6ummaYlu0bQuQQwS++uTl02dP4zgWNRK9+Ob3P7x5fcMYD4JwtdwY7bbbdRBEYUQhstqIUZpTBl6/+SabRKKJm+JOKbNZvHUWEtuu11tCSFVvGWdRGCR5whhlW8ZtoKXK02w4HOZ57h2lH3c7z2wqyp3WWgpBEA4YN8bEYTSfz29vbznnvoTyo/8Y47qusyzzYEBVVb6M2O12fvrLiyIghPyTZa3tRbvf79u2BsAyRvI89d3B94+3lNLxaJplWRJwQkgvRd13l9dPScDLsgYObsvi9nEeBMF0Op1Mhx7AMOZAJfOciTAMKbWerOcrOW+z5OUKDrNzkGinGWOT8cwLD7dNX1QlZpQF3PebldL7qhJa+XXKA7B1XWMIfvGzLxdKpGk6yLM0DJqmcc4gZH0jMopYnsenFcRTK4xxWgNCAoSUEKaue0TRXrYQWspglkdWhzQOo8CWRc8IpyQCAIgOVFD2vVECxbFFmGkrs5w6GPai3u87pYRSSnbCATSetkkWS6GMsVGYcEraril3mA7yDhrR1MBY5A6s707Ium3CNGNBxIWUyvggIbWSWgEJ27btpfSKBVIrZXRVVVLKbJCHcXR2fu6LWik9UE+MMRjDMAxns5noFUJ+JpABAAhmjeocNl3XG2OdA13XaWW9RFVVNSe74TCMCSEeD1AAOAsQxMpo70MNHAx4iCC2wFlrrHHAe7wC4IVkThA9hDAIAh+HvGC2x3HBceCZc26BsRA4e1zuj7VgK3onnTHGjwac+gIEeQEleLyghzj0R5XlCViA/wUDDiEEIcAYG6N92Q0A4Dw8hitACDkh/z7YeCjF/2iO/tH+a9mR0Od3yRjnGQMHPODI9UNHpeETL+E0rAiP1msnzMbfrh7GOO32KV/xIMQwHxhjvGWzR0NPtIZT9+R4Kg4hE2GQZnHXV+v1uu97Y5R1SmtonXbAAOeMdUoZACC2oCgKY5ynpHStsBYQwvq24eykR/kH1lZCCC8qehSkOvAqnDlkSB+2EjyacsIG/ihXUOpHHOL0/H748Q8TAuDMh3nbCXLw5yQIAuAQpgQh5E0+PTWh63r//YRw/z2EULxYzDHGz58/F0JgDAlB4/E4ioK+79++LSFylOG6Kcsy0lrXgLRtz1CQRmnEg7rRfSuM0qJXTlUeX8KIDrLBeDAbDydtVbV1W5WtVmC/rSkJnUXGOK9t0vf9frNnHAVB0HXi9evXH7/8KWehN6GRQgghgiA4O5tWbbPdbtfrNeXBZDL50z/5syzL9vvSIJfmg9FosN6tu2+abbG9ub0JgiDPhrPZOaNUdm3XNLvtzlpwcX71ZHhVNiVakSAMGKFFXRnRNV1blZuvv/n2p59h6+D5WRLwVMkmDDKCtb+6TdNACL3EhxCCEpRl2eXl5fLhVgiRprHSYrVYVk3NeXj9/MVisSjKUlmDGQ8hzvO8rdrz83OKyeLu8fbNzW6369r2m9/VWZYBADopPvnkk7OLc2csApAgNB6PLXDKGuec1KpqarGYKyPfv39PCKGcn52dJVlaFMVmvyOEzBeLPM+/+NmXt7e3u91OSjlfLa0tR6ORNv2vf/MvWlnoXBAE0+l4t9v9+//3/+fFixe//PkvNpud0XA6uVgvdv/0z//Q9/2TJ9fD4biuewhh27avX7/O81xbE8ZBGPIkjQjB2+32229/b+JECPH965vFakspnczOy6YeTSdnlxfGmKqtPv7Jq5vbN4/L+9/85jcW6uX9a2Q6qNum6d6/v7u8eJrFzxCUURjkg8Sjr4Sh0WTIw5/GcVjcF0kaeQGAoiiqqlos5ovFYr/fXz974mWGR6NRkiReeLguympfgDRVvUAIaUKhddA6honseoGw6oXqBY4TaF2x3WVPhmmUBJTVdd22rVKSIDTIkr7rJW8ZJnHIA0a6prm/v3t4eFjXu8vLyydpyKPw8fFxsVh2rRBCIYTevLlp6u7i4nIymeQjgBBKs8xrHyGEpFD+zjnGY30S8vLcaWPMfD4HjGqtrQXAHWhcAY+8QYMQhylwDAAEmBDIGNMYaaWkVB5w9ouvn2K4vDwvi40AhlIMw3C32yjB/uQvfvLmzZvVatV2zW6PMcbW6iwbaK2VMkY7jEkcpdZAYwyCVPVUaxhFQRQw0co33y8Q1kK2UrX5IAmCUGvZtW1BpNa669R6dctZXLeCUg5hsNks7x82ELrtdquMu3+Yz1fLyWy6K4ogiLIsBUK2dcMRGg9yRqhWAgAQBAdwxZSVc9B3aY0FUkplNKXUC5rtdrvVZkMImUwmy/XK0+UeHh6EkhY4a20cx9X2oSz3Shlfifq1lVKa58PFfOWZiV3XcxZCBq21RbXHGHtLTAhhVTaeWughOoyxF6vRR30qliaYEESwbFTbtWEYYkryPDfGIKU8+OzzBL/OW+sscMponwuGceQVlsq68oEwjCOPqzddCyFE8DDvhxAy1vpecq+kb0wIJX2ayzn3ti/OHurRUyBBH+gj2Q8ocgdKnZLH6G5OJ8cBp5TCmKRp6g3VvCdFXdchox4k8IXySZYAHpUHT9/vo5dnKnhWAQCgV9KvpZ4F7D/i8Qm/z8CB0+/V0WEZ/YHahPGtFh/deyl9EuDjsR+P9CNFHkL4MDcihHRdRyk9MfhOSUndlGkXR1EQBIHSUhtFGXHAGntQVkCQQOsgMNY6hLG3uYBHoy8f+I12kJ88LA4IjbXWGF+jA5/ZCCEoFcfwf0gsfGbwIYRw+vG0/wAAfGSw/lEa8WG68AdHjX68KOAPmJIHvQ0IsMeKvNyI1toooXTnmZj+LUII2T+WUGJlbdW1hCILwV6UiKL1chsl0Sef/ARREsfhfLmuuhZCuNpsHx/vy30BtAhgEOWZLvcIwGa/joajOExoxEMWIEjQXkvQKiyM0CmPDQVN06q+I4R5AfnVds8Yu7i6BMDePtyWZRmkwx7qmJGAJnd3D7vNPo5jbU21b9NwMP3k4vb2HWcojHDdrkaTcLF9d3l5SSm9ublZr9fT8TCLM90pivBskA8iPoro73//T2K3rpeL4XBYLPYmSAaDEQFoev6UhMHD/ePD4/3NzZtelBiSf/7Hf/nrf/V/Wi/3k3H2/bffIdEOLy7qunZGUxwJKcu2wRClg5wAI2Tf1GXAaZSGaciA1shJKbrnz572bZfG2evv3n388uPlcv38+ceD/CwK+/ni3hkxL+++n3/T9dVwmBPbNbK3Frx69SrJsNL1s+uzzX779vXXURxfXJ6tFqKuS23x2x++m81mVICfP796XKwowS/OZruqLstaNt3zp8+BQ4ThIAgGeRpwCqxZLpfbasV6ZZomVKDvZZQmGJL/7X/5X7tOTKZnWpqiqP7sz/58tV7//d//fVEU0rGPP//0o49fGqMABx998QwAe3v/PggC7YbGGGsdROib2++3v/7Ver2OhvGTJ08RkxZ0T6+f5X5SVBtO+LbcEgBXj3Nr9c3Nzds3r8/OzkAWjM+4IyWN1LOPMkrb+fY3VdNsCjSbnQUcVGW72e0os2madX3jWLct9vtqThCtqkrJajoZfPrJy76XcRxTFgxGQxomPCGVsl+9eWeKcnx2fnFxMZ/PH+7uswFbrFeerPDN99+9fPkyZHy9Xr5795YQcn19fXH25O3bt1EUv1vdYowRwkoJpUTTNMv54osvP396dbVer1UU5mn2XfktZTJP2W7z8O7td13br9f7qmziOJNCX8wu4AwBgFTnpJQQIgJEAAFHpBW9LwrLg2ZfMJ/Px9Oz8Xh8e38/HA57o+rVohYdFbIme1/A3dy8l1Ken1+Wbff4uGiaLggTymKttey8hXqYpMO7d+/PxtP9psUgFI0dZFNsiVX29t3jTz/54re//uevfvP1Zz/5VPe6hZ3u1Gw4rbbl25s39wA8e/lyNBw2TaO0DQLWtm3ZNJzzbJg2TSOl0FZ0slW26SQxRlGGsyzlQY5wQHCotanrvu8VJhoAK6XZF3Y8DrabjuDi2fVk/bg1vR6EsbSwrJvOyqbaAWyMtYTAXvbtZvfy5cs8zWgQ5ZPzcl9s12saBW9ubrXWWTqAGJVlefHkKkri//Sf/pNzDgLysFzUfYcx1tZwGmx2W+fcZrNZLpdRFKVxUuz2CKEfvvveKV3XMgp4FgzevvlhOBzmUeT6NhsMx0lQVJpYE3OKCOz6Zr/fa6Du7++fXb9IsnA4HJbV2yCKLQBV20BCAWNCKaUEJlRjpDHiNJBG7evm/OrSaPX4eG+Rsk5oo7RREEDGAoyw1c5agwHmPLMGWAcDypwzphPImTzguqlZGDRd28s+y3LjgNZ6MBiYXuveWqutARbAthYIIYBg0wsAAMacEgQhhAhjxDnjUAKjgXPOaOwsNVoDABAGnAd932stTzWrrymBdgghTuippieQeMcQ3zRxzjVN0zQNoZAnMQY2jVmWhU3b7asd8BO/yjgAHMIAIGONBQ5hCgDQxhDOAADKOSsExhhA5JztesEYcxxAgA7FtPVRFBj3o02lz56bpvEZc57nSZI457z0hac10LaTUmrZIYToMZeySlmjOaNt20DgB0BkEHBPK/aiy0pJdBAuM1L2URQ1defBvzwbbTYbSgjBdFeUYRhSSrWyWvcYQkIQBABRLEVnZS97EXCotdpv5wGlbeMVsqm1RirtjlqTp3lOa60QnbXak2E54sAaYA2AADoEAXRGaymgs8jTRhBUStujcHXTtAghZKFxiDMGCNeAYkYVJNpqiBEiUMsecESR6/uex4m1EACnHXIWQuAcMBYgCCzGUFuDkBNNvysLn2f8ZJTWjVVAQISEM42WgMCu68h8sfAnQkqpLSSEtF1ftdX19bXPQ+lRvXizWm82mzQfBIyT0UiKLgxDiuH57CzP849fvdpvd5vVFiEyHA6zdNA1bVVVrWnato2ieDAcZ2kghLIGAKA2m33X93QQJEk2HOY8ih8f79u2betadtKLopd1RQjNsny/33uFcyFU18v1ant2dnZxccE5f73YAQCyLJedkEJbaxkLKMUIEoJZ38s8HyJE2rafTs+s1cJYhNBms9nvas7jtukAAOPxuKxcWZZltb+9vX316pCNevqxT4o9BUxr7SBCwFbVPqS0mow5Ab0U2/mSEYyUZQH3Xai27a21nRQYYynldr/DEDDGwiju+ipNU+skIeTs+mq72XHOX370ymj39u07C8HV1dXf/upXz58/n51Nzs/P/+Ef3lNKKOV1Xb86f3Zx9SQfjlfr7ePj43K70wDsdjsHEec8jHjTtuW+kFJmWXJ2diarzeT8cgqhtGC5XEKH+r4vywpB0rfdaDA6m54xQud389u3txjjf/3nfxqETDUNJPBqOmtFhTH+N3/z31RNs9lthRA8jB1E6/VaSYdJEOfsbDp7+fwFI3wwGISMYwA3m02xK6WUlBFn7HQyebi79/Su/+F/+L9GUVTX9Zs3b+qqjQIMIQGGjMcTCDij4cXFWEkLAFbSOYtpMHSuC0I2yNK8q3hAMLLT6cQD9Ztt2XXb3YZagKp94ZTmEZ/Ops9ePkMIzOcPu2ILkEvT1BcuEEKHoG+yEEKyLCMMY4ruHm470Q6HQ8YoFA4RZ4GZzIbPXlwHAZdzKYoOYwwx+Mu/+FcQ4sViUVcN52Ge520jdrsdRtzaxhpoLbAGSKmiKBoMRhjjox6z8uWgb2kbYzAhvtfgZZv9Y1hti77v27ZXxniHns1223adtcDL4x2AUwKAx0uBHY0HSqlsOMQI7HYbjKCUcrOeP31yeTYdCaX8PjiItDV3d3e+tphMJspaf5MDeBBoM8AdIUdCKTXOlvXOWhtFQZrGAFqMEaFIyh5Cp4201mitEQaUUt+1SbIUQqi0EKKr6/rhYW4MHE/O4jT/6tvv+r4LMKzLspU9xDhn+dnFBcZ4vyuM1AjApqr7ph9kVimllTXGhAHHlCdxBmATBMFut/Pi1p6q5nFs717BOfeta4+peML8yRi6k4Jy5hEyjLEf0rPWAgSNsZ7klmXZarcYDodZlrV9t9lsAABZlkGENpvNCVf3CLCH3xEh1urTSQuCwHtY+N6E7JW1IAwiSjmEkEDiyAFq9q1xCKEXLfjLv/zL/X7/uJhvd7uiKIyDPAwAAP5wDkUhRhhjiBEA4NisdgABjDHBhCDfqD5UqOYosYwxxgSeeO/wqCiAMSSE6F6dbir4gZ6EX/l9xezXfwAAYyyNAs55FEUIE79XWuu67cCRN+ebAtjBE9rvjv5PWmt0nKI0RxEh+AHnDiEEIPZH7f/YQ9RSSm/L6XmIQRB4ZdLtduuFlk8HdcIz/Enu+94jbacJlLbtTtQ/d2Q+9n3rjcvbrvXSAEKI1WohpYSYHIzBHPLMAwghRsSCP5hWOCH/GP84Murf9Zs7tRI+6O/A03k4nXnwAbfg9JdHWMKcUIE/6i9Y56SUVv+BoZKXcvoRkDh+uf+sPzkWQE9T8LlXGIZpmhMKq1IjeuhhAQQNcERKGSUxpkRrjTGO45hoIrQwxlxcnE9mYynlbrcpd3uIHEJov90BYCml0ti2qjECzrk4DIf5gCCspbEWKKW8C+1+WwzORlo7o6ExUEkjek0Ii8IsDNKyriCEfad0AuMoyzNh9Mao7mHxIITyRxUnyfjsfFfV+7ppux4iLHq9XN7GcZqlA0YjI9dCCAIJY0Gx2yMEvC+O99jWCOaDUZrmzkGvh/r+7r1zbr/f73d1kgw4C5I0nk6nAMqqKoUQy+Xy+vqFXxcwxkIpqTXRWihllNJSAoy10cDY0Tj/+MXLNGTz23eru4csTZIoDIJAKGmc3ZeFNzSjjPuv2u02+9368mqWDQaTs1knm06K1VrMZrN0MCjqQklz8eTi8vKy6/vHx8f3799jiv7mb/5mMBh99dVvpRRJkjzCuQYQYgIQZAG/vLzc1/WbN2/I3X0QRUHIfDYDIWzrWmuNEWZhOhgMiroX0koplXZCmpDTJIyhha+/+WH9uHr79u36/rGqmsjaZ8+e7ffbqm0++cnH2SDtldS9VF0vG8HD4MWLjwMe3oS3lMYQ4iQHo+EYAxjxSEnHKXPW3t3eIoAppQHjVuk0jhkhnFBO6POrT5q2Xc8r2YEsnjx9+jSOkrquu04AgNJkCAE1qgt5HkVJURQXTz+Wso1CnMW8qZcGNdDJwQRDUJWrx9u791K4wWYW8gGh4SDNOQPaqvvHu8f1HHE8GAx4wB1yURImWcw5ZYw4hIU22oGqaopiyxi+v3/f9Y11Ko5DrTWE7qdffJokSZomq9WqqoooipIkOb+YMRput9vFfN22/XjECeYYU9HXnDMltX8em7rb7ws/PHl2diaECJNYSuX9lry74Gg0antprU3TNIoiRNRBoVkoAACy0EpJiPNaNBBCC6BfEAHGGCMGsV9W6rKMoqhuykEaA2BXq1WahF3XlGU5nz9eP7nwo2XWuThJwiDYlYVvoCZ5frCdNC4IAkqJtgACjBAyDlgAISbOSUoxADiMaBhxnyVYq4VoMEFexE9rTShi7Ij9QgwgjsJkMplMJqOHh0fVy9lsfHF1/f7utoxYlqW16LTowyjC0AkhC1dbpSGETdt3QvZS1E0znsz6vmece4enTvSbzWaz22ppDmw1FqRp7lV6pNQQYo9mexGFJMkopUqpcr+TWgOrhRDD4ZARWlUFBKCqKmW079HYtjPWYYR4FAZdkOc5xriqDgJZWuu6aUajkXZASulLCN+Yp5QqdyDiKaVIwDnnxiqvGuRDJkLar9TGGG00CanW2hntDILQAWcQdN403jk3HA6DMFwsllJbznnXdcZZY41/ogmCFjj4h51pDBEjNIhCPxfg4fpTTvNhUDl95Niqhwgh30v1x+KzRh/U/bedAs8peGNgfaKGCeWcE0J8MNZaA+dZCMYYA9GBfKC0PiUKJ5TixKX3KfKpeQEhdMD6uOVnSXySp7X2YjO+u3RxcfH8+XOt9e3t7XfffOuOQ4Y+7vrtaq29V5m/iN4qRUqpjs6fp4tyenHON9sVQsgXybvdhhCCKRMHm3jkz48xhmCKGfeX2B0ZhV4HkxxtM91xDPWUE/wRC+GUPJ1+eWJdnFiNp+sFjgqV6I9SBGuttQC4vu+dUe4DS62jOKRx7tiGsNZzfyCwjFFtnFESuoPOh++4YUoTnDb11hjjIHAOIkQQQiTNM4yxNNoBxwhmYZAF2Wg88Cdl8TjX+kDlqKpqv98bZQlBQRDUdQkRCBkF0LZ1s9/u2rohhERREvIQAhwEAWFt1yqMeBAkwGGlFHCE0QgRen5+XjV10zRagWLfIIqCIBoOYVdv1GBAKe2lJoxL65S2o+msV3a/3fEgojzc7N7f3T4mcQ6B2+2K3W5HCAHWCSHCMIyipO/bpmkZ41dXlx78+eTTbLfbmLb1zvEAACEEQq1WBkDnk2uEkJchgxBuNhvnXBSFgHPf0NJaa6WsMQgAp3SaJIPBYDaZPrs8SygpNsssTQZZslY7iJC1drVa1W0DIJxMJlEUQYyfPHlyfjGF0Pz6N//4m9/8Rqnu+YtraNR8syi75vLiKooSgOC+KLTWg9Hw9evXr1+//eUvf/nlFz9fLpdluZdSAozqtrEOtkImEJ4/eRpl2Q9v3nzzzTecc4ccJzSKAoQQozjLMkR5r3Xd903X1W1HKU2zrGvFxdl5HIZWm+1qXe+Lel+EjLMBGYQcKdGX5W69uKVoOB4Wbf3dd9/1QjhCXrz8KOJhlGRRmHAeWwN+95u/Z5Tv9/uqagjiWZTGYQS1VVpHPFBdb7Tu226Q5UWWn8/OCAiL9WozL5BmeTrEIGwqWRYdwTRNc87S5XLd1P352bPxaFqVXSNBGOQ0gI2uatEQDqKIhakLA1Y1iIe667uyADVsBtl0PBrFo7Asq7d3N9999x1j7MWr54yxtu60UpRzB4E2zteOEMKqbXbV5unTp9kPidmpXvUxDuMophRDjHfl/u37m4eHB+fM8zyzEAhlfvPVN23Tl2XlHIBFAQFWxmEatr1gDHIWUkqR0BYCiGkYx54s0kmBEPa2dZiQLMuCIBDKRFE0HA4hhJvd3hPHvGsAwrRpGmMcpbRuOoxxEETGAQAAcghihAFyvjDVkrPE2aBpC2AMwUDKHhN4cXEmhNjtdpiS2dmkaZogCMIoSvJ8tVo1TUMotRa0nUBCQUwdxFprRCjlwaH6hEBIBZEzxnj2BkIoigMAbFmWZ2dn2hzEbRjlGFNjjHMQOEoIzwfjwWA0mUzOz8/3u/L8bMopCgM6GuZplqmtinhAKW2ruu9tlqYEorput2RLMEaIVG0nu75pOgdhGIb5cER5IJXhLAzZQTZgkA8DHlZlraQGDopeIogJIRAgBLE1zmGAEcmHA9kLqXrK2Xg00ELWdamUUkYzxuIki9JUWie0ZpRZAJ89e5YkiXcrJoREUaSUaduWMSaN9bCic866Q0yyiPogWte1kt452gEIfACOAsdYQAnzRqBGGqd7ay10VkFgjILABpxiHO52u7qux9PJcDSqqhr0MgzDsiwRpto46yBwABjnB/LNEYh2yGlgKQXQIWCh74WfCIw+XCmltHFHFOEgVWStVUobY0Ia+NBy4vf56X8vDOUJAdbaozeE65vKORdFEQ9CTwE2R6XLg38QAJxzxkNP1mOUuiOh4Y8YiB/siTqhHcbqD3f+hHZ46MhH3/1+//j46BEFz1X0uYW/b32KE8ex36L/vZc5EUJ8YOp4CMOUUghDX093Xec7Mp4Ek6ZpLw/ZMKM/hnlwnFYwx6kNCCGCnktxuD8/hElOnzrFeP9Zv63TmTdHlYsTw8MejTfdia8A3CnPgEcmCgBOSgmdQdYQfFBh8r5UzjngLASH/xyAyDnkHKVUqdY3eoA1zmiCYBpHQqggpL1QbSectl7FByJCWBTWda2k5JwjBHa7XZrGWZ4cWakOAEAZRQgZKRqAdrstxlAGgVIiTZMoDI1Rzrk8zaADnIeDwYgT5iwkhJRlvS0KSjmjnQMYI5rksbVus92uVhvCqGecrtdrwlkYcs75ZDL56NUnUqvFaoMJL+vmzfvbs7PzJMvrtsMAEkIDnqTpEAGqtQZGt1UZBEEQBHEQxnHICCq6bi97adT51UVR7gghV1dXX33z1Xq99ifXp35aa6OtsVqILssDpVQYcX/3+7tWa80x9jxbXzRgCDHG1llCaVlV337/ndW9Fj2NgsFoSDEihMRxrIxeLJd120RRNJqM4zhWQj4uF4vFw2iQxHF8/ezZ7d3b27u7wSBwDiJNb+f3aZo9uXiy3W/m82VZlmma1XX97//9//xv/s1/+8nHP9kX26+//vrtzc1PPv/py5evGiE3+33XdZiyKIr8w2ytrlX5cN8CAD559fLllz9rtGzbdrPZvHnzZjlfPH369OryMs8yq83tu3fO2CeXl2Wx/eqrr8Ig+Ku/+qvLNP2nX/+LBzOqqjLI8SjkUYQYHU1nnPOmqquyWS4Wj/cPVdVUdVlV9XqxVMpEYXY2mVmppBC7bREQ2rY1gajY7p6cX8i2fXpx+e6H9w8PD7pTeTokiNy9fb/d75RSl0+uGYuAa7uu86Wql7n93e9+d3l5liSoLueiX46GLE7zMKKy7ygnw1EKARESia7XTkKkNGQX1xdBGnzz+ptNuduUuzROjNH7am+tK6oKOqeNSdMUQvjw8PDDDz/MZjM/8kApPT8/D8Owbdubm/dCdE3T9H0/Ho/HoxlCyNnH+4e5pwdjTIVsIITOQohxL1rrMIA0QJRQGkZJNsgHo+E4o3Vd99uDQGGapqPxeDY7V0o1nfDFnJfhE0JACGdnZw8PDz4pj5MsHwzablt3rXYAAr8WWWSQs1AarbVOIhYwMhmdPTw8KNkPBlmx22AMR4NJVZePi8VokI0nk6ZtOykgwedpPl+sirL2hEELkNa267pOSI/YI0yk0tYBjIhf9P1O+gUOIwqRQ4j5DouzCABsLZTCKaWlsAAGwDFKWNfqsmwgwFqazWYzHII0igmmDpOIheE4xpStNmutWweA0Gq92u735dX5RZakRru2l03XCaHKulHGGuvqrgUYX8zO/ZPIOfeSGJzz8/Pz0WjkV/lTXEmSJMsyQuF+v6/KPUUYYaysQQQjZyFGmBILgVDSWhvHcZYPpdTLxf1+v5dCn3yNCSGz2Wyz2Wht/AmhlGJyMD1iUaqUv47IF69ByELGfYUKLIQQa2Wk1BBCxthqvyaEUIyMs0J0wBmCE4zxeDy21iZJIpWqqqrpRBhHzjlljXbOnObgjQeQD2MOAACjjHA9MM5TWU+FpnP2BGg7a07HcrKU82rOCih3nNM7BTO/9PmuBz6+/DMiuwPx1htqe2Vin0N4rDsIgiiKGA/9UKKnOzjn/MXq+94TBby2rN/nUxYCADBWnyKrzyoQQmEY+r/3R7HZbLbbrf8xS9K2bT1UcEhbGQvDcDAYWGs9a/JDyOGD6tz460UppfQwXmiM8X3J5XK53W6dc3V7mKtEMWEBj+PYOUcw3Ral+WCsEQCAIMYYSw1OiMKPOcTxYp2iO/pAyPKPcghwVEc4ZVenDSGEwLHfcYITjDGefIqAJcCCo9Qm8hRYYAGwDhxdN6FD0EEItRCq782PItCHXE1bw8NEGi2UxhAa6yghAEBiIdBaG2sRxsbZXbFvmqrrU875dDoZDTJrrZcjjOOYUWqVxRhyzoXo4iSKw7BpKiUkzweDwWA4HBPCFvOlpxmPZ1MeJ9vtdrVekmIfRUkipRDqYb6QUgZRyDlXSgohWMiurq7yQfpkNqM8XK43eT64fPrs9nH5+u3Ncrsry3pT1NA6BjFn0Yvnn5zPpggAjVVV7hBCaRJiMqIUQ+ik7DabpRDNu3dJ1zcQIcrR/eP71Wp19elPfQaKEDLGxlE0mY7ruqybbd/31unhwBBC0jStK1GW5TCOrdYIAG9xBgnGlCIEpDMWw/vVYlduVouHkKGfffFFGLB9Vc4uzpuue1wu6qLBlAgliSCc86fX1+PxACJze/fWQgAwwgC/uXn4+OPrydn4/bs7oeRnn38BHZqv1r1Uzrmu67/77vuPP/6YUzYajH/62Rc/vPk2ybJ0kLuquV8uVjdvO6G2222SJFprQjh0oMXIGAOcWy2XPI7my1VZFvvt1iiZROH100vn3H63I8gJ0TmnV/ulAvLFk+sXHz+9//aurHtLCOOBgzZOwiRNWyUuLi7Gs7MwYOv1qqrq5WK12y53u+LTj5/c3z2sHxdGaQnFbrkOSbDfbK2ysheyF1EU9U17dX6Rxokz9v7dO+PsbDoejUZN1+w2a2dVHHGj+rLcEsIIQZQGj4vb7X4VRZFzDbC1FKAutn1fhjxrG2WUVqIDkOXDM4iaspbKtrUql8UCGx4PongQDadDGlLIUNlVohUAAgtM1dQEUQghDwJOAwSXN+9uZ2dvAcSYsNF4cnn1zDlXVu/rRgihCInCKNAGa4Ono+nlVf/9u/vttlTSeNw1imJGA+Bclg0QIs4CpZRz0JdTVdVcn11wzgFGRVF6RNcP5Hnd9P1+r5S6vb01Dvjuxs9+8XOp1f39o3E2y7Kr66fSWLXZ9EIQQqGFFvhq8iDWSRFp6mIyHoQBMRpEASuANcaWVdG29XpNhqPcIYwosQ4obcqy7KSQ2jqhMOWcc+OAcaBvGmMdRNg5VzZ1DGCSJIQHTnVBEIRhaK12zkVh6oDBqCqLzrfCrXUd7I+AKm6lssZOR2NCorqSXSMfHub7Xf2Tzz/TUjsNq6LuG5nmA07DAHcLWXRCEoQRJQhjykMeRqLrZ+cXaddVVaWkhgg3fecNUaMoadtWSu2VDzgPr66urq6u0jSdz+enjiEhJE3zyWS62j5WTS2ValVrre67jgUH+oKyRlSV3O2Ng09Ho+F4VBb1u3fvKKXOwjhN/Coxm51/+vTpv/zLv9Rd7+WDKKXWAQ+PO8yMUcaYvjcYQUIQJgwcZjKl7JVSpu+EN1gKWXjAxiGAzlpr/eAjOHhPGN+PP90bVVUhGh2qbV9HHtvtWZZ5flxbN0opJeShO2DNsQr/gHZwlDcF4KAf7JxzjjjnEEQnOAF9oLr4YfXsjiMDhBAj+0Po0sbj+T/qf5ODxUMQBAgfZiDruvaqTeg4Z+E3d5Jp8omCP3xrLcLQk5lOiYKXkPF5gCfhexDCayRYbU6dC58G+QfTHAWP/VnCR/8CjE9GMNoeNbMRAlLKuq4908WTErxUsW/HY4wZDVjAESLOOXMgpXglg6OaNUAIIWh+DOSnU+pxC/CBHKQ/CacLdEoRTqcdnuSZP2AeQAi9sP2HiYJPCg85BzzgEFo7DPDpqCGEGDoHHTx0npzR0jkHnQPmMLuBEVCyx2k+yEeMBRBCRDAACCBsnCXWWhYGHMIgDCmGlFJrtT9Nu/Vms1pQSvM0CcOwFWK5XD578jSKgjAMd/uNMdo5Y7XWUr6/uSGEMBZA2D88PGBMnz17lmWDXnYYw76XjIdd188XD0oaB9xgmHvbEAdY10ELAYAaY6iUWSzeL7e7KMtCiGjAEaHz1Xq/L4tdIboeaDPJh1k6CFmMEIpCmsRca804CsIYY9z2XRAiiAzC9rvXX19eXmKAV7vlYJIbqD/77LO3b9/5q9j3knN+cXFRFMHuu0UQBIQiz+vE3tTcGKM0dAAfRdEJIRZDra22BkZcYtCLZt2Uocb3+3VR7GIGkyQpi7aqqrJqAEbz+dyLkZEa7otNnPDz8/N/+2//rVStlP3//vf/a9u2QRz84k9/QUjg5cx/+sUXGJHlchmFIcb4hx9e/+yLLzjnT66uP/npx8Chsq4eFsuqqparzWK12RX7i4urIAjOZ2dePosgDKD9/tvvwpg3TZOm6eVs2os4CZlWXdd1V1dnT56eKaVWqwWN8UefvxyNRrWuWws/+vJLKaUFJklizOB2v1Pr5WA8whTVbWV02fTCmI4hEAX4/dubxWK5WiwQJCgl2+2W08Bqk2W5MzbkAQIQGEsgGuUDIcRklGOMWcQItgS74SBCWAstCbVJjNu+E0pTwte7R6Pd9fX1bMIx6IzQUcgCNsSO1HvdU/jy5aeU4qKq5/O5Xq5WRbmYz28e340vL4VVWZzEg+T6xfX59Pz+9kFKHfGg2BVt3w0yRjhz2sRpcnF1WdetMe7zz75crVZZliVxXtc1cFgrV1ddkiTG2PnjPSUBBKRrtQO47aRSCiCkDaDMMo6NQ0mSYESMMQAgDgDESEh9e3e3ffxhOp06BKvqIHwWRpFnDvruXl3X8/mch5GHuPyQUpqm+6r0JjRKKQcBJZQQaiFw2h1KAwgscGW5F4Jrcfb5Tz5pmkZrmWVJVRa73W48HiIEAITz+VwpZSFQ1tzePypp/NR+23dxnCZBABA01lkhlDVt2y+X68FAe7Ledt04ixCkQgilpbXIObfbNoNBBgCBAANntNKeGxRFEWmElGKQnw0Gmey7MMjybEQI220KygJOoSRuPEiiJMOYuDFDKVdKEURnk7MkjPI0Y5hZ7TDGWZoHPDTO0oA7CykL4iSzxnrqz3A49MW9L1hvbm78+LT3Do7juG3b9Xp993izXq9nkzHG2Fotu/5sNpFW9koaY7peFlWDKXPOtW17e3urteacuyPJy0cjv62yOTQuMcZdL7ypLCDc0wopxVma5HkKoNVCGmOapumaXgilpDbGYYwVOWjiGmMQcAgheJyXe/PmTdM0UivGeZqm2h7aGTQiGB566lYpDJGlADEWhnEURcA6YGFb1T6iW2ANPAzOAeA+DPbgWDv6xU1rbYyGEGJ2aEn4vbJHXwZ99AIA4Ed2pD1qByGEfE55KJcRtNb6KOuTJKU9FadTde3FDMCRQOCnW71+5anLcNoKwuQEs9sPqme/Yp+aF76PoJRihB44BBh7xUafolFK67oWQvhBCXsUbPiwyvdbhEdfhtVqZa0NgqCqKufcdDp1zlkAfcaglXUQ+BTfK7md4vdBocEnCkqdIr1PWP3Djg+WzRYeR1XR8XVKFPz1Qsdx1hMBAh5pj/YDQqI92DFY38nyYMNpAtNZQ7yUFnAOOAecv98AAAQBiBAAIODUOoi9wCTnDOO+7ynlcZYSRiGmjBKIASLYGkC0NTwMKCYYY87pgHMArFaibds8SQLG+rZ7eHhgjMVRMB1P0jgZj4d5nnNGimLfy84TgtI03e12Xd1YiHxRtdls6ro9v5zmg/Tzy6fnl1evf3j7q1/9nTZyOJrM53MeBqPRKMsSxtBmv7u/vyuK/dPpdLPbkiB0iL55/4/z1VZZJ43tpQQQa227qsmjxFpQljV0QCnhgNJGOBvwKCaEGKDSLBn1w8Ew+/033/zsZ1/QgEopv/zys91u99lnn+12BaVLCFXTHExivJ75aDTiAU3T1FsOWmvDMCQQIUz8mDJCCGDonJNGQ2eKpm57d342/vlf/tnZbIQR/N3//M2L6SgfDnpxGPJhjK22K4xxnuf5cNC05c3NzeP8ti7LFy+fPn169d//9/9d03SDfLjbFVq64WC0mm/m87nPZEPOi6L4D//hP/ziZz93zi0Wi8fNXZYP83z42WefffbTn373/etf/f0/Nl27Wq0++uij58+fU4zv72+zJI2jQPbi3dvvEEJPL86fPLnc7/dSyzdvXz88PIxGg060g8Hg8vLy6bPL1Wb99Xe/e//+PezGjBGt5dPrq+nF+d389v37m/PzGUCwaWrbOEIoBIBAQBkMDVktGn0QUacE4bqqNmTjjL04u2zbNo1DTzJHCF1eXKzXa6Fa05t+02pgBoPBxeUsbcOH+ZwzeH4xfVysHt8+MBZ0QkopqzojCG/7hhE6Ho/zfAqB9TfqelXxIKjarmxkK2QrxbrYFEXTAYMIfHL5NBvmT59dJ1FaVVUSxW3VejGi0WCIEGpF7xsN8+UOYvbqk0+DOGnb9n6+2O/326KEmCsDeZhSSrf7uqz71aa4vb1nNIij1EPQfS+FUIQoKWXX9nEccx76LlhTd2VZtm1LmHLOKaW9l8R0Og2jyJMkRqORL7astV3Xbbfbqqom05l29vLpEwsBcEhKqa2J4xhBgih1zmlkLXCnaX6rNEEcE/jnf/7nSomHh4d8kC0e53d376+vrzeblRDi7v6eUNT1HUJICxdFEee8E72oNedhGIaMB8DBsiwhxN7RwAEwGk+tg3GcxnHKORdCaeWMBghhBHES535F8+1hSul4PB6NRsqhm5ubMEgYDUUnR8PJeDQ9Oztbr9ej0QQTro0jPMKE91Jbayva/fDtd9vtNgrDkDDPnIrieJgPOedSKyGE1LYoCu/5OU6HWlsptXc2qqqmad7e3T34ktS7gIZhSCnf7Yq2fbydv+2aZjodx3EMnGm11lpXVeUHUA+QOzJCycVi8duvv8JWKaW6VjgIfKW7XC57IeI4Lurm1HU+Qcdd10HoCCFxHE4mk4uLsygOgLEeuIYOce6MtlJqpZQ1ltNASimVJAhijNDR4xQh1LbtbrdL0pQxlmWZMjoMQ2ktphggaNXB4NFHi77vOWUfRju/VyELD8NZ7jBl4JyzzngtUX10yzTGWKsRQhSSYw16CNs+3Pol8Y8inL/tfXHv4fQPMwxfPfvct2l7v7kgScDRKcqH9lNf/4/weX8+T5tzR+UAd+zoe+gCQpgkyXA4BACUZVkVpU8UTjoibdt6stpJ+fEEiiCEhJCn4P1BNW+dc9vtthethyuklBh7OS/lIQqllH/ufJP3FLAP9TrGEBzMGE/ffDpF/jfuA2boaa9O58EdFS9OaMqp3XDqm1hr4QcNi1MKBQ5DFofvds5ZaxzBp80dUhBgHUQeXbD+NgbAHXMd55xove0zgwBDCBFBCDmMsXWO7LbzkwDF5eXl2fnIGPP4WFioQYANQhIDRaB2Ogz4k+vrKEnny2VRGB5GMaOu3IfW5cNRHMerdcnD5Pr5C87T71+/mS/3Bjjx5lvOOUVqt7uXUv/Zn39qtHt3e7vZSQC0A4G0cDAZIga//v23i9WyF/Unn3zy9Prl7d3j7b/87mG5SdJhGMVWucFgsN4sZ9dTxNFv735zcT7dbtcDAtI0qep9EI6ns2y1WTFuJtO41yEE7tWzlwnPMAyhkdPwVb++/f79m49+8lGv7Jvv3zeNeLy7H6bpdDb85MUr6xQLqNJCavHnf/WX/9P/9H//6JPPuwJGaX55fuWgvb17r7VOU4ZDiBEAQkutG1B/8eSTIUwGcfoqfzGeDZQgfWeAw0kQno9GYl8Yq9vdkrpuNb9ZzW/nj3er9fz9w3cY42wc/fVf/7UC5pOffPr+3Z22Oh1lRV29vn09G82uXz7///0v/18esV/906+ur58sHh+Fsbt93bbfWgOSJN1ut+u7u74s27b/vutVVb969UoptawXw/EoCGMeplVV3T48xnF4+/5d11cvn12LYn23nishHp1jX/78808/Xy6a1W/f54yjkP7n//yf//RP//TJ5KzdFpt3iwHPIhCVj7vpbGad1v9/tv7jS5Y0yw/EPm3aXLuHfvF0qqos1VXdQAPoaQwHwOE5JGdFbrDiv4X1kIc7niEHQEO1QIuqyqrMysrMly+fDO3a3bR9mosvwjO6OL7IE/nCw93Cwtzuvb/7E1zlZVnVRRoEjGrTGa7n2cHoZDI4aFsluKY28TxfN5RI//e/eQmhefzkgSirkwdHB+PB1cX1L3/9D8pIgO3V9dvhuOfFjARy22z1VR2FiTRZvl6URTsa7l1evINe4JHAWtXUhRgkPvOVkJhIqTJErIFGarDZwMtzXmRxJz3hVi1yuSzehWEQ9/cEIqum7KWdyI9Gtr+332vzMmDmenm2unmDAFisixD/ZH1zOj+/yPJ6MBhaoT54/JwyP8uyMIkXi4UfRnEc5nmedjtekrbCLpdLYkmnlypl6lZASIQCTENEoNRWVny92ZZl6fv+Xi96cHr813/91xiCOAiapq7KYjgYTCbjMPRfvHipgR0Me27diyDJskI2elpNkyCh1GtKHpKwrmtttax14EfQ2LquvdCPvchWBhD2/OHjDx49yZdLa203jLAGDw4Pjvf3Op0E4w+m0+vY92azGbEQYai8QEoJIPa9pG1bo6EURvBKtJJixigbpF17dNLtdodpyjmvLRoNxkEUpmkXAJCkKUJocnh0eHh4M5vmeR54t/T4uNvt7+1VRXt0isMk8aMoHR4aY4qiQFFSTDOqsObcWtv1AqVabVSSJACRTqd3fn55fT0dDofdtOPUd6u2gLxsmqauKtdws5CuVqs07Y4O96fr5c1izhgDBM+XS6BNlmVpkji+AsZYS8WVapsmZmHiRUAhn0XGmKRDGwFZ0JsvFs6ooNejw+GwKcuyLNtikyRpHMe+F9ZtkyRJVVUO7B+Px7Plyun00jRdb7YOeI+AZYyVZVmVJSH0yZPnFOHvvvtuerMKPL/IawghI4Q3lSvP7aLwfd9nzG3Wu90uxsF6nRnjdbt7jLEyazEIOmFcVVWII0p8Y525kwtfINRjiMC8Llrdaq3LMrfWxkkYBAGyEBTCpwT7ngF2V78RDoUQkDKNbl19IMHWUmsthsBaAwyAEGAIjNGibYy61Uwaa5WwhJDQ97TWvKkZY1GcGHPbrxhguTBaaw0g506+3iqlEMKMeRAiCABGiGAMrEXwe8Y/hBAjZO92EO44A993pU4b43BcBKDVxgLge55ouaOeWG0Ws3kcx/uTPYyjb775BiMAAYEGpGkaMk9K2Y2TFRdBp4csVAZiEixXm/39/W4I27ZtWkEI8Xx6W0QRMMYAhIMwVtoCoVwf6QWAMqQVaJTEGENgqyqrqqptBRccIcs8orRVUltz64wptcIEewS71goR7LjtGCFggbVgF3lqjEXAWmHdKmQn63W7lV2rtGsybhEX4LqK247KPYdRhhASrVJKIog9RjEi0JqmaXDIIITAWqO1NYZQShC2xhqpwjBUSnGlESEOvQnDcFm/uV4aAXMSYIMACyIDIPEwOTk5dTpvR/uUUrkNDUIo8CM/YK73kVJWVXV9fc1oFIZBHKRVkZf5tiwy3tTDbm8xnZV5FgVBvt00dWm1AhAha7nii/XiZn4zGk72DvbDMNTGjPbGVzc3mJKa1xrY4wcn471JWVeb7TbyI0bDKIoPD48//LCIk2lZi6KsmqqOo3B/sheEtN9LEYC/+dVnlJLO83EQoSilFjV1u1YqhxggYpIYVWWVFcvLawoBE62iVC7XK+pNrNDDbv8CnnfiqJ90kjBKwujxk4+++up319PrtNsbdfvMT/vdHjYgSEmc0CACQimEOZAtBBAiTRkygnsMFtX8xcvfxn4wGvS1rSkdcyWFEAAgbUBVt9T3+mF/sjderWaM+ZR4p6en787eUkq0VUHqvX37Xqk3x/vz7TYHAI4H+0+ePl8tN2/fvkOUHh6dhGHIhXr37swYo4yu6zpJEmPkq7evAQDEI9PZLPD9pqnevXvjedTZFK5Wq8vLy8GwGwTBYDDodvvXN1c3b2dW6bpuVstlr9P1CL2Zzq3Bs8USE5qkqQD0008//fM///MkSV69emmAHQyGSRJfT2+wy6v1iPOto5Qul0utC8/zPBb2+32lQFMLSj0MiVJqtVzmeb6/Pw7DsCjK1Wo1Go2ePHv63dvvrqZXEFhM6KYos5t1q2TaTS6up/3hOOn1h5Pw9cu3eVkEQbSZLf70n/7zyOu8+Prlm+z16fEDpeTF5bvhqNPKerw3XKwWi8XyNr0JGGvhepU3TSVk27Z8fzzmbX20N/ng8VOGmWrFkitI4WCy11b1ZrX95JNPhuMRgFAoOZ1OL66v86yuatHv9+fzRZjEWqsgCHq9nssFHu9N9saDN2/ezedz3tQY4zjwGWNFUWBorZbZpiiKItsWlNJuEgOLmppjRK21SbfDOVdKL1crTEiadk8enhZFsd3k6/UaY2yAFULUTamUMlbFEbTWAKi1UcDCMIy1VmVVKmmM0XWNIQQfffqjw8NDg/Crd+dZtnELXafs7/Y6cRxLyRELBpODu4EPIYQcWb1tWydJV0ptNhtHMRNCjCeZw9ubpjl7/14ZzTkHEBpj6rqOO+ne3t42z5y57+4hhcjzXEvgkBul1K3gU8qqqtwy+3YyM8bJRNu2ndf5ZrV2dBbRcpACa+3tOsbzMEKMMWfgvVqtsiy7uDgLgiBJIkKQUsIYE4Z+4HlcNMvlHELorv8s2zqcI45jh1dvt1unAHcei1prx2x3I52jsp6cnBhjpZRV2VRNXde17/t13Sqti6JwzHl4l5foJtSqrtwm3m3TB92e+1ZbNxQTYwwwRhiDMY7jmBBSl5XzAHVnu9vtIoSqqsrzfJd15OIV3E1Y1JXneWkQaY/dCi4Eb1sDlMI2ppTGfmCt9bEHpJZCkDuXaIiRc7ZGCAEIlVI7wjyEEKDbPbcy1lqLgAUOpIYIQAic089dlqAxRigNIfSCsK0rKaULrIHYMfwR8RivKgCEvVP3EUIBAIgSfedZ7sZu5894+9a3yId18/1u1+Aup92CYAe8g9u4Mg7uOInWWgANRNoYm+Ub2uulSWiMSuORlNIYEQax54frLDNGEQq0kUqh3Vt8vzqh+O5KUHfLiFumiwbQcdqs1QBBB8MQQpCSCAGMoQXWJa072OP+7wLviPP3PyY7GMB1ALvNy27u3y01dl/ssAeMMYbunFv9j20VzD2HBoQQQhZZgBBCFgB0T5x5R53Ed5AYBv/IBBoomG3rIq+FML3hYDIaNnWZb7fk2dMPlstlWZZBEERxIKWsqsZaYIyt68a567tk7iD0AEAEqOAgEK3YLDe8rbUyCACKab7NmqpeL5fAmMVqxduSMR8C0EqOCCKEpv308PgAYrrabDFFf/zP/slkf3+9Xr99d+YFzI8CFnqhDreb6s2b93UlKQt8Ej58+Fgru8nzr7/+miA86PUptg9PTsOAff27L8fDPqQ2CsO+SS3QVb0RsgTSGqsQasMQ9Pt+GIKmLotqM52DvCyyYjPsjZ49/WjYHzw4OI3DBEPQ6yfjweBwb5Jt13WeZZvt0XFv2OlZoyQUXIFtgZqmquotwoDQIAw8q5WG9mBvpFseBARTs1jflO0Gk2fnZ5dvX78WQu3vHw4GI2AtNPby4ooy9MnHP2QBCyLvV5/9utNL27aZjA9OHzzmXHbS/v7hqeLq7Zuz3335YjQaMS/qdIdPnjwiCH/99dd1U47H47otjTHYYzTwe0oul8ui3Ho+jJNgb29c102nmxwfH89nS631drt1n3dgEQvC46PT5WKVl41V8MHpk9FgyDlvqna6XCmlw07CosAK4mBwLoUbqtJup21bznle1Zxzxkiapl7gKy5Wm+1isWhqjhFTSmFMg8AjhCGApZRVXWgtO90kSZKy2KxX2zAMs6pcbNbT5SKIgu6oY6FRACdpUjVtUavyYqq1TZO+0MBasykWB8Mj2Upu2yAIatvezK+t1ZgBA0VWrmLuTZfXq+2G6yaIAyHrAHWauiY0UMZe38yapoZW13Xd7/cTPzzaO4jjtNqWRdmwII06g/3J2Fi42mbKAESJboSFwFrb6XTfX1xcz6Zaa89j2+0WAEAICgNv2OuyZ48DhpfLJYKEMQYA6CYHvu83TdNWhVWSYhuHLI39R0+fdgeDwXh8eXmJymqz2TAvTC188+ZdbzCq67ooisFoTJjnduQ0iIq6EEIoq1zqj7JKA314dDgcjo0x69Wmbdu2Fa65x35Ew4SGIa65b2AY+QCALMtG470g8BClCDEvRlAIIUQtRK/bc9lmxhgXa1m1vGkaZYFHKGaeR2gMIHVqQKWjOOWcC6mJx4Rs19mqbNrJPprPl1prAwBU1iILAGgbYfQ2ihJ30yzLUikVhqGT3SOEiqLY8YjF3WNTZU1dY4SsMbPpdLNeu8rNOY+iyPf9TqdDKXWmyEEQRF5c13Wv19us1kIIx/6Jw7AsS960roq7dbXDsRmj7mbtdKd3JisKY+yWjA7zd0+YTCaUsuvra1fAXLvj+6HLBXVv5zCSPM/dClxZ7XkeABYaK5pmvV4pKTfrFUQg8JgI/LqulZKSC46xREhr1TS1ECIIAq1VVZUIIaVkGIZRFKZpQilxbYfnMc9jYJ2laRpGvlLKgRBaa60VhghbQICFEFtgkZM7CAMxuX/fvy20biVvza6QONcmYwxQt3R3V7F2sn73V3CrMQfFO+aH1pJLoYwhkAJrOecW4oAECCF159yAGUWUALegpNTchbI6lqKTBaI7IyYIodsauK2E1rcUhPuEStc2uRrctq1bQzi2BKNg0O8QBFerlR+Q4ahrlXry5MmLFy8Q0HHih1G03CwRVHHkAyuUIrvi7Uqsu1qcCnQHb+ze3RlnNU2jlMDUoQaIUgp56+oxBgBYbO1tmjPEyG1AwP/fY3eS7/MbnEkGunNcAPf4Cn/QK9w2KMD1H/oPvmtuI75uyaoYWez6BKuxhQRC7ag00EBgAEQIASd/gBAia4A2CDKKsDW0KmVT681iG7HADABBFBhDuFStkAZAwjypwHK1cWk6aRpXdauUQoQknU7TNEIaLvRmMy+ynCDQ1lW/l0aBJzS0Skd+UPtM8ma5XDZ1TRBkjAAAJPaOH57GcZJ2Ol4UVi1vpYCUPH7+9NMf/eTi8uJiOruaTTWYvXr/Vms9CY9Wq2qzea0NENp4fggJKYoiDMOmaRACg16HYoIsOto/8pmXFRpRhHBc1TmGxhrK27JpKsdpYoQiALVRbdvmZYYxbrZlDmmx3gaUJFFYZJmWvMzXScQ+fP48CL1f/uqzt999F/lJEke8VS3EdauqZtlWJReik8QBCZMwVoL7cfrJhz+MQ7/f7URh8Jvf/Ga12mw22WaTTafLMEr2D48xZRaQuim5EKN0ILS5vpmlnfjo+MEPfvCxNNoP6Hi0t1pt6loQbGaz9e+/fnl9Pf0hjcK0pwDeFC3GOIg7Xpj4YZLXWRj6RZk/ePDgn/8P//Szzz771T/UP/nZp8vlcm/v4M2rt1zUZVmUVd5Je4eHh9RjfsCG/YHHwoOjU4AYAmYwGPS7HSnl7OZmvV57ngeMraoKANDr7r19/+79+bvDw0Mv8P0wCMNQax0laRTGRlshVZZXvtCEECWt0ZBSZjQoy5Ix3+3OKWZNU3keTZIY3zp/qc1mK6WSSLVSAITLuoY5RR42hoRpf3FxITVqWiGENCAc9IZRFGutj3r7m8VmLpdx0k3TbpZtjDUHBwdFuUa+2VSLRhUGST8kH374fDZdZcLWjUg7keeHVZMboIDV62ydl1knjD/64ONxfxQP+92iQhZR6sUIbrfbqq4J85JeD9K6rMX51VV3MJBCU+L1eqEQ4v3b95eXF5vN5t/86z/76KOPHhwdImswcE71nrsVImAIAqNB7+GDY98LPc9L03QyHlBKR8MJF2o0GjFvGgRRUTXL5VZbzKUoi+rZs4MkToWSLReHhwedTiqEdJIwCHG323LOR8OJ4/z3ep3NZuP7YZqmWuvfffOtQWQ4HBIv6Iax51EppdBgtS1IeRswTynFzEOWW9UQ6hFKMWHQGAtQyx1jnydJ4nkeJgxoTaintGmLqmkaYbUExgCIgDXAcinqVVO3jRDC8e/d1hJCKIRomqYsa6ddrKrKDc3uuup0OpvNxhVmR21zfjhhN8UhIBPkEZplGbxdJN+mGQkhNpvNzXTKPE9K6fn+/nBydnYWReFqOccEMo94Pm2aijFyfHzY7XbzPEcIuG1OXddKcYwxY449SqIoYIxw3vR6nW63W9clAECIFiFUVcVoNBgOJ4vFAiHkfB6d1s4VS8d+N8a0bet5ntvWIy0JAkkUWGvXq8Wvf/UPnPPpdJqEEWMkCDyMgNa6MEoqzjlnjHk+7XQTl/Gx3iw9z6OMKi1aXpMGSSld0KXneRCGEIBOp8MYq6pKcY4Zc+WEUuwKkpTKGOPGR4aJK+2EELd6cCw/fGcLaO54/vAuGxASZrW21ioDHPkDACDvopuY1AkiAGIDkIWYML/b7WqtOZeYUq11K4QxABHMfE8KLY0GEGBHLDBGaR2F4Y6XsCtprhF0rAjHrNzRHTh3GY/fuwvYuwxDJ8V0vSbG2F0/Bso0CTqdpKkyo4XkNUJgbzK4uY4ur7TPsO8RaKXn44iwum2MhvfLsLvenCOTQzhcvXe+VQgDgsmOywkxghDfRwgghC5Cy+1i7F3oJbxzikR36Zp/8Lh/Nm4n+ztJyA5ZgXdWS/Yee9He0T93xMb7iAK6k5ZgZLFjyxqJEXXHY63F7pghgC6KAmoELYJQGwOAIQRh6CkBrIZn72+qbelhPOinjHqkqrnSAGFiLKirerMtqrryfT9FVGqdpJ3Hjx93OsnZ2dnZ2RmACGKwyda+R7tJMhyPMDI1NphR4jEviDjnEKMwjgFCGFMLQdxND4+Ptdbz9WpbNsz3LKGE0E1RbMoyq1quTZp2me+xIFwtNzk0nAtjrVI6r0qpFJdiu92ORiMlRa+Thkl8eXVT5Nte2lmtlxcbPRxogq1oZSdicRTXwqpGJX4ihOBNzbGJg/7hQRQnnSRJ8KqQUr96/ZKiIPDFdHbdSzsIG0LI3t7YIvPu/fuiaLJsgxAA0ATBUGtdFYWQNImSTpoSGJiW+sTrRFEUDNM46nZ6h4f719fr9++neVF7lKXOp9KYb75+IbmgjPjM41x+/c23RdFAjCb7R1LDvCivb6qvvnoNAEyTrtbTv/rLv53PFz//+R8TEqZpNFtmRS339sZ7J48QBnVdZ+d5q7mUvDvofvCzn/RHveGoc3R0dHVxrbW+uDi7ubnK8zzPqtGoIZDczKYPHjzgQr349nXTNBSjhw8f/ujHv5jNZtn0uqglogEkrOU1d+gWRcynXIrr6U1d19paoQyEsOHCD40ytmn4cr0FAHS7XWVAEnc6ac8YgMAtktaqRoBWabG/vxdFIQCgqioEiVLq/PyytZUb1Fbrddk2nUFfW5ttawj8pq6NxYcHRwDA1Wq73VSTycSnfjrqVm2rpCEEjfeGjSwqUayqRdzzq6aMO37Da0jIT372wyyrPvvqbJNty7qWqqnqDLNunPiE4Nl2fjG9WGyXRwcP/sU/+Rcf//wnIYu32+3rzz63EEXdbpd4LIjzvASQrleb1Xrr5p44ShvUQAjDMJJSbTabpmk6qWnbdrFYtG0bRQljTAgRRYmTYyVJ4hLo1+u1BRoAYCF6ePr0k08+uZ5N1+v11dU1oYwyLwgTCzCXSggx6I+ePvkwHSYdzrW6jQLSWnu+DwDyPG8+nw8RZB5DBD98fPrpp59mWTbLCwPAtizCMMQYF01trSUsyNaZqwcOfg9DD3sYG7ja5BDCsuYIoaJqlQaU+gFmEFOAaMOVEEIp7aq+UqqVDUIIWs11gwAMoqSqqvlyTSml1BBidsOQewAg3I3elXkAQNM0hJA8z+9yfonDGNzNlyvp+z42wAo17g0+/uEPMMaf/+6L9uzM1YmyLM/Pzx1FUQgxm81cXXGgvRsH881WCNFNO6enp+/fv0d3BoW+72+zAkBigaaUEUKC0COENC2ilBKKADQAulkNEYqc1fEtvRze/lLuRu95nh/FbnXCGAuj2C34GEXGhTm5fqLlbVX7lEkhmroWnDPGAs/3mWetFUI4zL/T6fi+v16vsyxzesKzs7O6rh0Q4jbxd1RECaF2Cn+lWueUBwDwvPC+du5+kbhtYvStgsYxPHbV7hbSv3N6pvg24WlXwl35ceIv55fsmg/XCyIMDATaWurqHLDGlas7/2+AoAFW3oVF7QwSzF3Ik72zN96VRvetW8T8Dt5wV++O+b+rvo5R57ZgUsqWl3EcYgCNVovZDUFmf7KHCfQDBowVovWkp5VkmFCPumC/uwJ/64K9K7fmThmBd1YEFkCws5qmhFJ8G93kNIrGGGdpATFGxgCEEJcC3guoxPeSo+93CfeXCztY5T4z1N5zXnJziHumEi7Ey5p7CgjXoEBrEb69BhA0CFiMMbQQIYAANAQBCyFCjsZpCNl5KgBgrFEQYEoQwYw3XHF1dXlZ59HpycFk0LeEkB988qlr7oRsN5sNAKiua4SAklpK6XmKUm8wmAjhmMa2TcJttj7Y2z+YjAmGZb6tGpTVxc1yzlsBAAi9AGPaCtkKTSk9mBywIN5ut4t1AUDR7Q0AQHm1ymrZyi8Wy2XF9YcPHk72DqaLbdl8ud5yZSQhRGhbciUURwjRwN8UebeTHpye7O0fvvjmq5cvXj5//vzi8mbVmJtpkcaBke3R/jD0Aqh9jwCfEFFvsaZRkBw+OFXalnXFmP/Jj0+vz6d///e/2h+ftKLBhFCfRkkQJmFRl0VV9gZ9QKq8KfKy4EpzwzGEQhpK/dFg0k0jK6WRRijZYv7t1y8pJZThp08fzxdrC/BktNc0TVPX1sKmlmcXN4FHKSaDQY8oI4V59PjptlgX89mbd2+lFOtVlud5vz/0/YUB+PxyWhTg6mYRxL2DgwPix+O9UZIkXhhorf2QHZ0+KIpikk6ibrSZXowm/Z/80Y+/++47TG2WbeMkABFMk14/7Wtt6rqFEFsLfT+2tgGWhGFoLF2uinfvr2+mV01ThaFflcV8Pue8DcPwYHRwenra7XbLug7jOIqSvb09ZTQ/Fx4LQEqoFzR8vlgtuTQQQt7UaZoiBJTQhFgArBCt1WYwGDw4Oa6qyrnaEQSttUqabLUSTWu14a1spaIkqIWc3qyjNA1oqrXmpazKhjGmgMrWhX/o9/vDESbbPOeyLZosK1etybWpD8dHsDCU4/dXZwR4FqhHj49eXW/9yG95JbUw2ABmcAD9gLK4Y6QWQv/u5ZcsDH724z968rhPTfTd2zfdbn/v6EEaRfPFuqwbQlkQRHXdcqnausGYKiV8L/zRpw+YR6azC6GsNJYFoRdGWdGIrPA8L4qStNtPkgQAYIx1v/itwJoQhFArqq+//VYp5fnhhx99kuc5F8oYcxglXGkDjAZQA2gsgwgBpJSxSiinvkEIBVHiprogSroGBFFUNc378/NONynLUhsJoDEGOv1eFMHxeOy4Am0rON9kWeFkXQGjUkovL9wUBSEMggAhtNwsdiXcFYmmaTDGlEFKkeC8qipKaRSESpvNNhsOh5wLpe5S77S5FdBT7HYKbl50WjWMcZZlzuu3LMvNZoMxPjk5GY/Hb9++hczTQpZFkcTxh8+eSyn/7u/+zlprpLIEa62dG1UURdRjlQGDwUBIzhgFAHied3Cwn4ThF1980bR1EPqdbmqMmV7fQAiHw2FZEXcYuwLgKnGWZa5ZkVK6xbnLG+TtYqfvh3e0DKa17/vul3LUfWOBK3v9fteZPadxGEVR0zQQWUbxdrsty7wuq263iwKfIEApS+OQa1VVlZScEMQYSZIIAKCU6Pe7u6oJgFFKVJUyxhgt4ySkDCMMCHUMeS2EQAhYC11FRwhhTF1D4yKUjDEGWHmXE+EuIaDvod/6NpTB6O/jBoyFEBGMMWUsCGOpjJRSKkMZQphqY4qyNqYpq0op4/IwscueQIhz3gqpjSaIaGu0sQAAiKBrd3YtF7wLlR4MBu6A3ZXmulLXcrnzD3duUXcthfvb3UaK311jVitooJIyjpOmrLJN/uThI4/6WhqtFG8ERq1SmhACDQTaYA/vuhCEsCuxjhRyv8DfVfHbVGjP86wliGDnEOiytrWy1ipjDQTmXqN8izTchw3APT+r3T7F/S3cigfubCruAAP3avehIHd+XMdgLbx/im5/BFh4j9XhHgQCiiBCCGhye+QYQuQsNKCCECMAgQFWQwswxgjasszbtl4vlgxZCCEjmFeK7B0euZlvu117QbR/eGyM4aJZr9dN0zRNdXF1rYzFGA9GE0LIxfVXtlJx1496AW/aqJMoq8ssny4WEGDGfIOVMXq9yRAiw/FYKrTaVlICyuKsKNS6lFrN5ot4WVzeLOuWbzZZmLxbbevZYls1Ku50oRAAGlXXpZJSyjD0vTSy1vpp3Bn2s6YyEEHK5usN10ZalFW11ko2pU9BJ2CybUMfl3lmjY5i3/cpMHK1Xl5eX0dR9HR/UstGKIUIlMAM9oaXZ+d0hSaHo4HuCitZ5PPlajOrsjqnJKjazGNE6NpHhHnWY1AZjaA2Um43xZs3s8GgJ7WcL86EVsvl8ujBabXZZts88HzWCQHCUdKBxqbdXhh52zwrivLq6ma9yTCig70BsF7TyKxofvvFK0qw1iBOyHyxTjrrw6NT4lM/TLOivp4vlFJRFB0/OFmv1wSD5Xr169/++uBwItvm1esXVV7M58vIT0I/icIo9rvbbeZ5AeWqKOu8qAghzPcBJJdX8yxvOp2EkLDTDXq9DheNhqCuawjBcr0u6xpTihCaTCbK2MVqaTR4/PR5mnYJIdRjV5c3X375ZVVVUkqlDMbUFSFCiDO8whRRhpUS1moX11uXFWNsOBz3Bv58uUCEXE9nb88veCN5w/O6MRrv7R2Usi22Ncb4pz/8aRiGnHNoUL7JicfCMDSNmp3frIpZZ+gn/UACDolqiyor271R8Jvf/fpnP/15t5sOBv11phtVAKtrnqms6oAkSSOLLERg3az/6ld//dV33/7iZ398eHjM/KCoyuubG63MxfXNm9fvMEC81YHvU+LFSWcymRhjiiLr9XqS89HePmYeC+KHT56zIH779u12k7nhNUw6QRy7Aa5voBMyZJsFQqhtRFEUL7977fv+aDR59vz53v7hxcUVoujZs2c3N7OiKDCmztjGGK2Ui9nVnCunRoNwCwBar7dKmX6/3zT817/+zZs3b7iS7o0U5xhjzoUxhjeNu/2VRV2WpbW384q1Njw6lFwYITWAXGlrrbLAWrvNclfFlVJunK1bHgSBklIDK6RoBJdGu5nJ3T25EFhr3/et0s4NiTGGAN5ut6vVyvlXQgjruo6iKAiC7XbrRsbNZuPynKIo2h9POOckik+OjgkhV5eX2hgMYBLFXArFFUKIQCiEsBAE1jw5OTk9Pf3iiy+gdTO0H0WB1RITCIDxPAqhdRFWSilXjOtaGKMIQY7/6EDxpqmMUWHoV1UFgJGSU4qzbFPkt8w7x2Nw90ZjLeecCenkf03TuGBAAMBqsXBz4VQIz/MIwm6l7TPP3XmjIGCEbJpGKZWmqUtwcH2kE/W5dcZkMtmhF+CeN2IlK2WEtp6FFhGEAdFaA2WFksYYCLEjFQIIzZ1Tshu1HQvB9TrCZTFbs6uRuzHUQqKdRhFCCyHAGGAMCTEQYsakMVwpJCWlVFkrODeq1lpbC6SUFkGAETTQXRJS6lt7D2O0ujVpxgjvhuldDQZ3/s0Y4zAMwzCsqqooCheQtusndtXOfaZcZ+NcDdzXEEJCAgix4Obk8MTHtG3qXnfQlE1btRBgaJHVliJqLdbaEuy7LdIdBnbrLWHbW3sDY/Su+gIACETGKvfu1moDrLMdgxD6vm804FxprYHVACCXK+38K9253a0Ddn0nuKcLdd/FCNk7pqfTO7hztTOx0HfOlfrOyNm1NPieHdadvOVenIS9BRt2rQ8hmmoIIb77xRGAEFkLADR3tFCCoQXSak4J3m6XRjeL6TSbDOqyIG/evKGUums3TdMHD46cVwlEVghxeXk5n0+rqup0OmEYQQgBBcoIBRXyYC9Kjw4OtdBvXr/7+sW3GEMFIBe6rMVynYdRfBSk26wSQhCPaQuzrDS2CKKYYH+x2vQhoYRlRfl3f/fLMIqKosDYmy7WEFk/DCFjLAyhJCxiGEOCcKvleru5PL8IPb8zGGbbLfODYRpI3mJky6zKc7vNmFVV5Hd7vcj3mRd4Btj54uLt+fttlj/qPvlv//2vF9NFq0Rn0N+fHD5/8vzf/3//P8bqr1589c/+2Z/u7e9vy+pmOW9bAxAM0hggwgi1pjSykaoUyhjNfY8GDK43xXpzc/JgZACqxLauy9Vm+u7dO95KBDEht1f2aDTpd9Lrm0uMO5zz5WbBmH9ycgKAWWfb4WA8X67TtNM2sq7FNss7nd7Jg4cnJ6dpt//mzbu64VJKz6MGAqFMmFhrzOXshvNqf2+0XM3aOieUYIa1lp1u2lZqOp0e7lHG2GSyD6g/n8/TNNXabDc5hJIxf+/gaLVaNUIyRgAmSadDPJJlm022SdI4juNut9vr9ZJO+tvffnF9fT2a7B89OEEIhXE0Ge8z6l9fX9d1U9fNg+PTwWCw2a5kywEwGDPGGCVovV4W20xKORqNAEBXF5dxnB4eHoZ+MOj1H5w+evy48cPP86LCns/8YDpfGmBVIwa9HqXUalNkJUIobzJ3G/OTSGODKaIeMdAGUTBbzoRsNquNEGA8Hn733YvDw8Mf/ejPMIXv3r9U55UtG0wRZrAzSCgl5+cXBLDBqK8F+O71O0wZYnSyv//27fvFYjEa7XV7vdGo9P3ISmuMCbwQAJAkHaWEtbbX6dR1bSTKy2Y6X3S7XUQopr6BBQCgP9yzEM+XayllFCWEENdI9eOQKy6lHO9N/DBCCL1///7i8vrZs2fr9frg+GQ0mrx+/XY2nydJAhHKs9pFSlprjVFN07j7/vXV1PO865tLz/P+9E//VGtzfX0DAMTIYEaM0dvNylpLCHPiwNs+o63bplFK+b4f+BHGeLPZFEXhoHI39Lua5Dg9RVG4AunqDYRQqMo5L/lBwCgljPnGdHrd2yHpbmxyd21r7d7eXlEUWZa5f3G6dlcYqqpyDgGDwcDzvLIsX7165RN6fn7e6/cPJntX05u/+m9/mXTSXq/nhcFsMW/qygHpO5Ld3t7e0dHRP/zDP8Rh5IrKarV6//atUipKUt/3nRXgeDxu29aBOu7dwzB0qLUTebmQ38FgMJ/PIYSOdrDdbm9uFr1ez92dgyAwxkjZEkqFEAGAbr+DMUbaMMbCMLSqGg6HGOPlctlUZRiGYehTgtzCxWfxeDxECC0WM9eseFFojKrruqqqpqkhDMMwZYxst+sgCMIw9Dx2By1QpWjZbJXRrWjqtuJCMsYQwYhgd0hKaQuANgbjW2KaO2Y3L7r4JWOMNoZSqqzZeRiA28ggsEP73f/uZlwHZbmJf7cU0FoTjD3PU8rlZmsHEyqunUWxOwZjDFcSAGAgQBbsVh47IoIxZj6fuyVdt9t1rFWnMm3b5n59vb90cDQCuDM4ckzDukUVNMo+ODq22sym18iii/OrthEUUcY8QhglnrIAWMCYhzDW3yc5uebAaKFcrpXT0cA7l2WICIK3hlTGWGfmaK2FEDHGeCvdM4HVEGIAkFtRAXDn2nlP33ifHeIerlcgBO/AjPuEht0awvWLDhxydR5CCAByiId7Ze1SIO76hN1Sw1rr9gsEY3ULeNjdX9NYd3oNsvfCKq2lDEWRV9XFejn97tW3+/t9ihA5eni82WyyOocMYR81qpWV4pz7PgMEPv3w2enTU+eN6vqacrEdJD1iwP5wb29/bK0mhABqB7/uKwmU0lJwi9Rg1E2SjrGy4fVsugqibpZJrdNOb7zeZFxRC7yLi6xtWwwYBKSYW9VGXMOaQmRslVWiLbRqGbGWG0sgwERrNZ8tzy9nSoGHj5904z0pteBn/U4ItfDDLrVy2y5GvU44jB4/eej7PoBQatNw7Q/3t1mBMaW4Za3yUPb7V1/MVmeDMX3waLCaLzZr+dtff350/LjY2HwD8kJ2B/tFLhvhhQExrZ/EMOkwCzYwqI8fHXsMqXdiH4TAEz//2T+rK/3Fb1+EUeCn8fgofvfunfFNoTNVVD8ffwqoiIbk4uY7GkLWWIYxBKDXGyCtDvefrK7X0+lyr3/y/J98opT68usvJ8Pen/zikzLfAH7FGxiHSS8cbTZ5HMSklNOr6wdHBx7d400VQr+SEgqgKrwXjUzR5qvV4ckxxO3ewWQ06VVFtFllv/n1Zy2vhWg5b549f/LhR/uPHnavr76a3WwD9kGvN2iLBkg87uz3Bqf7TUAI2t8/ev7scSfs/6//6/8bK3X97p3FZDAeNW1bcSkxaSAyQchgHZC41G2zXaIo6XlhiANrbFtW59PLbr8zkyJJkrTjjyadslm+Wy6AgZXRSdzdH+/3Uz3ojweDkYHg25cvi3HZGXa90OeyzctsnRd7g0dWGw9j0UpoZaBxrUAISZp0Gt28en2Jib+R4OXFshONL66r7oNv/+mff6z+Jv/85ReLXD4YnRRV8dXr6aPTIz/qNUXJAPQYezAZ6217/uXbB8cTv0dpiKSqHz08poSEUbotK2VxJYSxNscw8JPYC6tWQRgY27x7f9m0Kk5626x6++5svd4SQozFk/F+EESEGohI04qyapVSUZwuNhtjTKWAA7pZlHLOr2bzLMv8OLmZThfLZVEUvu/XdY0hV1ICY5SGjVBFrZoWWMustcu8ADh+8PBxGHTfv3n/+ruXvu+vW+ncfIVUCKEwJJrrosi01tT3IGYoQFhrg3FtrJbcAimUMG3tK+H7PoKgbRvetFmWAWsxxj5l2yy31kIL2ob7HtVcCVlDCFEYJn7YH432x+O3b9/6FHmeN59dDwaDThpMp9Nf/PzHhw8fX91c52XhBT6EEAIEEfY8L4xSiDZppx8EkbGV54fbrG6aRiMiBHj91cu9TTEejyuTN1k13h8Fvp9qZSmp66oTR0EczedTrniN0e/evqGdzs16PfIGsshfvHkT+EwYHXc7f/Gf/+MnH31MCLm5vi7L/MHRcbGViYc557zYWD90woEkSQe9IULIKh1FUZ7nnPO6rjGmFS9JTSmliMCqqaOwEycRhDDPMgS9NEogggqJMGHO6zdgDBojpfQIOXh4ut1mcRzXda2VtdZqa96+v4QQYhoQQspaWGu7cTLuDzabzWw2swbyqm04Xy6XcRxXXq21RQgFUehKTlO0KtIMASOsVQYzhBEWFjRlgTFmBFurtBRK1G4klRK5Hui28ABEMMHIQgiNCzc1hgICLTRaSWEplW5HbbSBADCEIABWKS4luKtVTSuAC9xCpBUyihKIVbFea63TtBNHHudcSYOBaZpGtyaKEkI9rXXohdpwN3/vHALsnceiY6pyzg8ODlwP597OFS3XE7ja6UAX1xfunKDcj0CCqroeTYbTxbQqCmMtN0pZs9yso26qjSmqUgMttYYAS6MYwBAZYC3CFiIIjIUIIIhvQTIufT90vWbTcEKYQDpkVFmbJOliMet1uq6PJ4QRiAkhRENtgXL6UoDBXXx2EAS3XcidHdZ9/sFuO+A2Dt9TUI1x0mXXojmrAntPS0kIuiU23OtCdh2V6/ixxxhFBCIIIaS+hgQACImHLNJaA4ghZVIIY41BRGnrxPwWkNWm6HYioYzUBjP68x/+nPPm5uZ6MugTp5+eTCbWWggt59xRjq293T4KycuyvLm5cRBiGMVB6ENM8rKIq6TbTUkU+bX44ac/nt4s1uutNRgNqFYWAGQhYChMYtPtjw4O4ryoVtuMqyXnzTpfKSWgsRh70FBjMCKMeV7RlAZoaDSDUkNphWilIghyAKIo2SzmBFlCqIfx4ydPpdTXN+KjD58/e3SaBES2xXY1laJOk3A86G23W8LY0cFhI1XdtIv5attshaLQwr3hXlPkVurPf/WZT9n+3t7xwWHLNQRCijKOWBD4zMNlucYIBkECgdW61sJPuqzfiU+P9gfD3rDbybbN5dX8t7/6ewADguEvfvqjxjSDYU+K5vLycjzsP3v2dDwYnr1/RzGCEEqhx4NhGicWaIqJzzzG4kePTluuLs5vMKYHh8f7B5PxeHx2dma1gBj5gQcRghgQQoRSlAVB0ukN9iLfm82nZdMqBcIw/D/9z//Tt99++/b9exp2Hj1+Qj2/rKvrm+V8Mfvqm6+32WY8HkYBsxXg2kyXiziO426HBqGwerZaQog7g2GSJEiztNd7/+7td999p7X+8IPnf/SLP3nx7dcPnzwtiqKoqu9evqwbnq832BqK4Hdvz6tGta1QFglpt3mNLEQIKQ3STr/bHwBkAcZFUdK8whTlRZPGSafbT6NuU8t6u4XrtdDG9/1+v//k2dPBeFDyer1djSZjjKFocVPXQGqtRL7dLhYLDniPpU3D5/M1wT5CjBJwdbUxA+rTRfoWdbt9SkjTNMiCxWwpFJdS593CY5hg1lRN3Ta6sQ0UaEkxlpP+6PDoSBt7fn6+WWZhEEPrHPcQIVhKWW6zYrURZQWMpSmJomAyGR0fH+d5HgSB51WHh8dSysVy1jYCIdTr9QhxA59ar9dVVTmSoxuqhBBFUbhbiduyu/l7sVg0TQMArFshlDYWKmMbLhoujTGMsawsRoPesw8+WC7m59c3CsDlemX9pKqbppVOw+2s27S2CBEjjVKqahpjjLsNWWtFpW6dNBGWAGqtRcullGmSuBkIQbRbhWqty6p1Axwjt/HTAFNKMCa0bVsLFUDEACSV9oJwOJ7EcUgIKYqiqqrhcHh8fMBb0TQN542LZt1u14Swtm3LskySpNfrbzYba60WUutb+9u2bbMsU0pQStO04/t+XZRlVg6Hw/dv3nLOp9c3UkoKEcXY97zIDylAURAUiFZl2el0hr0+sqAq6s1mE4YhY6yu2+V6W1WV5/lhGC0Wi07ac17C2+1WCOWm2/28gAArpRhjGN+mMDgtpcsRCMMwCHxjlLtxWygb3jojhKblLnWlKAop9d2YfhtS5ebpzUoEQeAIiZ1Oz0LoTCeti3lUyrH/mqYihCmluJRSGQsQoR5EhHkBAABhQdmtgNAYo7RQSittpTJBEFmgpeKEEGMhNLfLbwAAocgDFNxFMXk+pQwLxeEdb+C27KBbJwBwJxHEhLE7tYU1t/JLpwp2VhCU0unNfKct3OHhUkpjlbrLl9rlHUAIHRXGWYmv12t452zRtvgPSukO59B3/s34nnLSWkt9r2kao6SWsj8cHB8fz+fzrMg9zyOUWQullFxIRJi5pxRwOMcucsLJOHfvaG+rsfE83zEn6rrspp0d6dKdVYSQtcoYC4EBAFprMbg7ewDs6vf9Ef8PSju820fgu7gNN+474qC5M5aA3wsozD0A6PuHe9U/ICiAO2Ps+7QGe5stuTuQ75+mtRaSY4yjKEpi/+joiAANISyKgqC7jHCMsdbS7W/cSsZ1cLEXj8djl7v46NGjz3/3l/tpHHQ6XtIxlOZcttn066+++bvPfosQ6fUGg/5ESrVcrLW2QewDDY1sN8vs6EESx2S13QZRreDqg6PUWq2UEo3NM94WFiMQel7YC3lbG2MJ8pWEVSGbVmAAPS+IAnpxeUMA5aKdX553wni1Wl3dvI1p2Atitj9myLcKL6ab2eX05Vf84uri4Pjoj37+81ap2Wy23W4b3qajRw+PTged9IvPP2MI5tt1ur8/7KaLxXKzWOdVvc0qn6qjk4ed/uhg3n17NvdoY0VtbVNmUpSKgSECx6NOup7ODsYDIM0244Hf+eJ338xvLv/8//g/zWZX6+Us2yw8DK4vz9+9mYxHwyovHj98ItqaIOyMz0Rb9zvdOBl/S15uN+uW19vttqhKxsjJg/3xeLxe3Wit8zwXjaLIsxCWVf76/QYjVLSqbMR0sWUUd5NOGEXvL64hCRALPQolYPv7D/oGKG1L+Rb6pOsNT589aduaX8l1mf/uxYvFYoExxohqrX0/fPLkWd/3i7b1ka8hoUHkJXKZl9+8erPYZIQGQRhXdb3dbjebDUAQQax5Nb++JogmjcKQWOQtN2Wei8FgcHRwmJ2/Q14oDSirKk5jQLzeZP/Ro1P4Iizy/PJmSdC6zKum5mVZ3cwXxlrf96VWrWy3eVbUZZTESZL4ASkLLtom9IM0TQdyoHAaBtHV5fTN6/fpaMBYyCiazkwvNMtFNs48j9DD/YNO3DG6LLMaIAARWK/yw4Nxp+OXADWGKymE5OtyCQ3v9/tRmvCiKcq8rsu2bberdd62hPnU8602VZaXeWa5JAjPLqfGmDzPO51Or9c5PT113LT5fOk+L4z6bdsiJBBCg0HfMfjcnRHcKcJd6PPe3p6r31mWIYQ2m03btl6Stq3gSgKIAYAGGkwRgVgpCZA9fXz68PGjr198fT2fDXv9bLWYdA/cShhCDCySQltrrYEWQG2MlBpogABCFkEDgQUOXMUYW20kF04iD8GtXbEL79vRrCCEkHoAY2tMzTVXZSNMmBWEEIOoUJUy1kDMleFNYwHWAEspKcWMkc2mdQsUTG7pWlEUVFVT13W/70spyjIfjQaB7+cEIgwwgRRjn7Ga15vVuixL5nsIAc55hXFVVRiT09NT6oVX5xdtXYuWbyAaD0cPjo6H/QHDxBp1c3E1vZrWWWWt5nWjPa/b7UdRVBRFni+2eQ4A9DwfQii4ckNk03AhlNsuB0EwGo3Wqy3n3K2chRDWArdEz/PcAr23twchuCWACxHHrG3bsqiKvNTKFEUBIc6ywhgDwK0ehGDGGAAWWQMhANpC1zv2h0NCiBvAKKXWWheA6U6+e31Ebr2AHD7f6902Ny9evHDVxSkMd+sDJ8DZcRHMXe6ze8FdxXXvcn9f8H3pBdC1R8B8LwpwjaNSCkHgWhkXK+qAgR0Rz/7jIqa1BvBWMXhfTeB+8Fawd6cJvG++5I5z993dqznWoavr7qcgRoEfVE3T1qXV5vj48Pj4+JtvvnG+TL6UABFtjdSaYesconaX965dcIf3BwfvnuZWHs6L5eToeL1aQAAwRIxgjhEGEBgNjDUWQgsBAM5T5P5jtyC4V9Hhri0jmIA74+rdokHfpWP/wU8BALRUAACXNn2/J4AQIgR3hEprLbgXLuXoLDuOiLkTnjhW5q4juT3/1lirrbVRFEUeieOYAE3c5WiMSdM0jkN3oNbaPN+6ZVUYBWmaOn5Tt9t9/tHHrkMv6mZblHXdaq0vb+ZlowgBuGwbfr1eb7IsOzg4enL4hHFfCH129o4yiH1rYTkYE5TJpNdSChmlRhNRhdDGo/7xaHjgIVaWuRSCYSJEu1mvy6IGxq63RbYtFkAT5vkE99Kw3M7WsynF9NW3r15/+7IbB/t7IwJ0vpm3vITQTqcLgLyLy9k6z65uFpiyThB7SfePfvpz2ZS//81vCQSPnzw73B8ncdhPH3Q78cXldVluPWbHo/SjT54i8vHbd5dXV5dXl0I0RrRZXpfHe92jyWQ+u/ntr35JiacM7nZGP/7h87rcLpfr64vX8/kyDMOPP3hclc1sevNf/8t/+R//hz8/2N/78NnzxXz6+rtXX37+hZBtmRej0eD//H/5vyVhgIA9PjzaPzi6ns3LKnv//n3TZucXby7P3vu+77Po0aPHw86wzOoalr7PWq7zfJvX7aPT4wcPTo1Vm3X26NEjS3yuNMJet3tAvUBKOc/Lk0cPfd/f2x9fXp5bTMqmmr1eAgCePn0+GIzyrEyS9Ec//fnR0clsNpteZH6aPu6knk+tVu/O3s/zLOmkL968qptyuV60vB4M+oPBABOx3V4b5NMo6Hf71A/P3p2viyo0PRQFilDGaG1ALU3qBZhQQD0v7rKwUy2z+fSa161ROomiOEqN1S4p23nhedQnKUGU8IaXdbHeLOqsenBy8uDBg9PHJ7WuLhZnV5fzzdoEHdjrRVoB3wMQkDyr57Mbq1Xkh6PB+OZ6LVpgIej2PMWBbAwJESSU+FpqRT2SJnG2Ks+vr0aD0X5nsn+4Z6Ru67LItwCTti6dqK8tK1E3QGoMIAqBBbZuyt/89tdJ3Pnmm2+KonCKfGOU5wV+yuIk5JwXedk0jbuzSymLohBCOL96h5p6nucirR324Hbqa2F2O1RjjDbWOdVgDPf2xj/+yU+mi9nV9KYWsrWWW1TWzg1G3XKkgYEQMuwwXm2MgQBgjN062RhDQ6KBxQBCAIzWUghn7Mo51+J7vfvulp2kfbex5opDrQ2EXClkre/7FjELLcTUAKQA1oq/P7+YzyFv6/3JnkvryTZrjGnge4SQ2WzO2yYK/TDwWgADz2eEct5ACzpJPOz30jjivBGi3W631PfGwxEAYDqfGWUZ8UiIkyhhXjDFmBFPYyO5skr7zA+YTzBaLXLFVVtWouU+ZaPR6OnTpw3fSimXi9VmvVXW9Pv9KEqUMnEcE8K0NgihKEwIIRjRumoJIU3T5HnOmIcQaRsVxziKotVys91ujVVBEFBKyjLnnBtjmNet64ZLYYAFCDM/iJK4aYWbuBxnjhBCKCWMYkzCII7jWC6XnHN+O4GbbrcbhqGWMsfQammtdUIMDK2fdCnzuVCc8ywvq7rFGHPOMWEuIBlC6H+/dEdaCXeXd5p5rZXWEACDMbZWS8nRbX6x6xEhpgSj7wd0c+9h7wZZB4M7m4EkDhzE4uKbnSkfxphg5vQgu+H4VoqJsLuQdlwTVw4dtCaESNN0OBxCCG9ubtbrdbfb2a3q7xdU91OOQLPrgbTWCCPqsapolVK8abmSQRiWVRWEoTFGaIUtdHgeYbdxXOaeiaHTjOwm/vvV/RYAsNookdUCQQus9n1fttxzsdQWAKMt0NbCOyOjPwx0sHfS0D/AD3a/l+t43EmWUtZ17Viut3jVPS0lvEc+AADZf3zM8B7L4e5pOxWoe0cM79Yfu5eF90iU7mul3NjQCoF8n6VJGAQeNIK8ePGCUur4OFpL5192S2C+TegibsNkjMmy7P37SwghY2w0GmXb4vLyEkJU13Un7QMAGPMcmRljPBj0Op3kqHOapBHGAFHdyC1GOgpwzU3bLhGmmIUhDmDox2F0ejw8OT4JTVxVlRYSY2y1raqqbQQA6PLi5usX3xIMu71hnPSeffDR67dvZ4ejxx/88PXrV1/89rPzq2tg7fHh3tHx0yCkbdv48UUYdaeL/GaxaIVNoxggZDSglJ69mWZZBhT/+IOng8Gg20l+9uOftFJcXt383S9/9fnvvrq+OYcYdPqDh6enx0cT89NPmnp7+f7tu3ffag1fvny9Px6PhpNut7deb19++41S4vLy6vT0EY3Jo9NPrbbz+fLm8uLx6ePlfHV9frE/GC2ni1fffvebX3/213/9l1WRQWg//Oj5v/nXzcPT0+dPn0ZJ9/kHn8zXq9dvX718883F5Wsha4DBcDj0/bDT6w1H427fsBTf3FytthurOQB2na2jlecs8Jbr1XQxO33wNE56VcM3N6ssy3//5puszLsEXE5v3l2c14IjyoQp4zjujyeD4bgRV9ICCUAj1XKbl7w6ODjAGHLOCWEPP3qeTroXl++3oopSbxLtldWWEahx5XfA5KSzrhHwVdD3kuEhDOD0aqaRvlxcHT488TzPC3xp9GgyvLi8nK2z7eeff/f6ndaahl2IW143YdI7PjnqxMnN1XXTNC401vd9gMO6rlfLtWZSa6uU5q2CgEwm49a0L9+/Wa8KRhEELIo6UprJZMAwK9smW9dt20LgP3/09Le/fuFjmhfSJrQp5LWZez4KAkowNFgTRuNxQixe5et3F+/3h3sP9h/MLmf5NmME11w2QraCa2195sV+YIWSXNSmcD56Usqj44Ms3wg+opT+0R/9tCiquq4nk9HTp8+FEF999dXZ2Znz7THGuLsAAMB9mtwnFiE0n88BAHmeu9a+MdyVFmBvc3oYJYHvQQifPn368PTBv/t3/+7q6opQtlyttbEVb9q2cYF+lFLfY3Ecp1EohFBaGKVdvQEAQIggJNpaq60TYyqtRNNaazGAoedzAMWtSlvvlqluxm3b1mgZBIHv+5RSaExd1y6a1o13CKGqUK9fv/axbppmPB5HRwfb7RYA5DEyHo+l0C+/ecEYG0/2MMbQ2EGvA61WsmEEDHo9ZxbpwlqVUglN9kZjPwp3dSXLy6vLmUZgneUOitdGrbPcvHn37t27hycP4jD60U9/spzN66qCEIZhjDHmrXAdmBCCBaHz8K+rNumkAADG/Ci0wDpbQIsQcAsad9/cDb5pmh4dHRljKMOMMd/3tJauYjUNr6pGa4sxBQBgRD0WQAgxpne1BmhtINQASIyNtcZYWFaNO8/KqqKoILQYYwyt53ndtIMwxBg3TdMq5aT/rqwCAJqmcf20uwk7JBjc5fo4edt9eOB+Tb2FKBDyfT9JEmeTQCgFiPzjImQdYOBACYyxBehOlWfdkeyWWcZY51Bp6S0M4JADfee4jCAkhFBKwzB0AIajSRpjHC9k56EJAHDqU3jPoQjdGQnsXJt2xc8tyDCm2hpEMAt8KSXnnCvpbL+3Rc45x8RaBKnHnEMUvNs+uKMFd17RuwP+g+oLIcQItE01Ggw5b4a9bpZlWmvOpTHKAoMsgNYgiCECCGGp9G598H23cXfM/7uP+12avnu47grdi4yy99Yxrguw9w53t7aAd/RGDNzX9v6V4J7uTp3LfHGXsb1TYVRV1Ulj3/fDMOj1OknoGSvbqiLv3r1zBu9VVdV1WZZlr9c7PT0NAm/XygEA+v2++7zdXM+CIDg8PDw5fqj2TRh0IIRS6uvr66Ojg5OTE4jsxcXFbHbDGDVGC5HHEfvko2etai5v3rRiK7lg2EfYYmiA42ebBlRiuUGexxN1YIwBGkAIoUUYoJBRa+GzJw8BAJ98Qo4fPBTadPqD7958zdv16GDQ8GI627NWYo95UXTw4PTk8Oj1uzeFsHlVbi83rTZ+3IU0baRgDF1dX799/wYxutzMX755FUbe/sHkaj49OT794S8eDCb7Xtz967/527/5+18aY8aj42dPnjx5+iiOBoMhhxBZyWc3+ds3l6HHDvZPjo8enl9eX1y+V1b/6Kcf7R8f+X74ze+/ffd2o3gLLTBSLWerv/yvf1Xlxdn7t8vZbD1fEwIePTz56NlHl+fnTVn7jH747Nnjp4/zzzNGMYYWUTYad8LQ73a7WVYtVgsvCI8OT+M+efntl+cX705PDgjTb159s5y9p5RS6i2X6yxrev9zn/nBX//t386mK0zYi/NvMIZxHAshNptVHMedTidJksePn2JMpzez2Wwx6E+kMPP56uXLV9tNDhmK41AoLpVQVpVtfra83mbzp49PHj48HJFenq2aKsPUHHVGA+m3ddOgZSfqTU7TdBA0tSjzxusQqRSjEEGiEGiMWed5NasNYgBYRFDAgn5/dHpy8uT0QRh44/G4LitjDCEMQugFYS/tRX707cXLNOlhTXmr3727qFoedILpzWI2q9Ihc/Iw3/f6/S7SlBKi1c1iNg39/t54H1uMod9UcnZRagC8CAwnoDc86vfT9XadZdlsM//h4x+W20woVbcVQmC5XCxuloPhHq8bYwECkDIS+L6PqeKiQWBv8mg2m2mt2rZJkvgXv/iFY7zzVvZ6sq7rOE6DwAtD/+joAEKrJAAAOIK3+5C7QcpBmi7Q1t00726CBGNMCYbGGqgQ1B4AHjS9Xu+Dhw/L7eb3X/xOa5P2vCwrgiAQWhFGEEHWWo+RbrczGvQ7nU5d5lJwKaWRQkppjXJmf6t1YZRCCEFrkbGSOzG6UZQ6kZs7SAMBxghgxNtauTZNKaC1VcpZDVpr0zg0jpputU+gomS7Xo0HcdvWbVu7fshaC6FVSmGCPJ92O70kicuyRAj6vqe1VrcLadvWZVmW6/UaYqqlAsa2bcuYL1tZ1vVoNIrDqCpq0onSTs9Yu16tpFTa1EVVaqmkMj/9yY9+/JOfvv7u1fXVZdu227KYff4F0I3v+xjTbrcPye12BiGb5yUAqNsNwjAuiirPyySBvV5PWhlFUdu2cZxYC62pXT2O49j3fYisI5c4q0dCyPnVuQEAEYYhVgZIo53uFGNsNNDGOLddpAxCzqnXoLzgnI8Gg25vYJSezWaL2ZQgFASexyhLIgKBtVY0tZZiUbZcabebcMsRrbUGcL1eE6kIF64mueaSUtqNPUqdPYCEEFJKGCMQWq2lE4haa32fpWkqpcyyTCht79wGHdJu7K3oQN9N3tZ+v+p2V697poOW0zS11kqhXY9l74IibxEXeMtGdAXMcQJ28IDWuixLp4CQUoZh6FoZcGc8sPvBnRTiPgCwey9KCMIAAIAJKYqCBT6iBBR5yzkB0ACL7kosvEuOsPc2Aver7K5RcP8SBp5WIvBYkkRW6V6343tsu82augZGAWMRsAQB53ZogbmPydl76RXonsXC/cq9wwLBHVPBdWAOVMD3vBrvYxL2Tv14V+C/39FYa401EFgNLABAWHXHpbCuDVFKSWPc3xFh6nSnUkoIsVJqMEiGg56WjTs8hBCwiFJM3OrLrVfrunZhuHEcIwScoEhIrrV2puuz2aybpJQyqFFbCkq9cX8SRZEQCmrw6OTxeDKaTq/X8+VmufIIUbx9O/sqK8r+YEB9jxKP4GC7yajH8jL3uDEa+B5GEEpgy2I6wwLhxPd9aFHd1FJqipk2lnMZRslg2JdKxZ3gy9///rMvfvUPv/yrm/lchJHkomgz5GGDzCrLwcV12crFfLOtZFYIQLEXJhbHjfGJFwslv3v3qmjq0WRIsL2Zz062x6tt+dU33z1+snny9DkN0tPHH7x4fX52vS7q6vK3L168eL83GfW6MYV22E8/+fjj549P82LdNiXEdrQ/efj40duzt3VeAWw6abjdbn0PPjg+PJocfvv1q2+/eTW9nMlWQgsYQUVWJFF08uDgj37y4+Fw+PLFN0mnG0XRw4cPoQVfffXVcrPwfb+ot61ARZVt86zMKyk0Y8Gzpx8gW2OokOEBhQwjz9OeZ61t8nxTFjmExOhKNvmrF18bi0bD/SQMjDFVXtR1jRFkhCAA4iDmdTNvpkVer9dboPDl2aXvhZtFZohseBkmzIuD19++/eL3v1FGZOXm5Xdfv1u+/gn/8MnTE7+LvIh51EuSqMz5bNoC3SiCrIe6STxG/Wxb11W5WGUoZ3UjMKLr7cYLIml0mo7LvCjrliLb6XQm+3uD8chKgft9n3lSSim1UsZIwzkvtlXbqMhHwFIl4fVyscnK8fFoucirEiQDGMcxxrjXS+qm8LGfJOkya37z688m4wceGwTUD8IuHNNtuW4VQBAYA4y1fhx0SLfg+bJcWQI6g34xW19cXBwODkTLp9OpFBphRsOQUCKl3FY1hghZYI06P988e/bsk08+GY0mVVWt1+tvvnnx9ddfTyaT4WAMAJjP55eX574furGgbVpnOeXilxwVwN1NjDGbzSaKIsfodm72slXcCA4tAtYoabQshTC8PhyNAkb/+r/+NysVIdRIY5T1WTDbrPr9fqfbcbFyCJmqzps2N0pBayiGCFuoJJe10RgCSSFWCFNK4zBSSjVNU7oYiLpWSlkICCGIkl3hIcDEcQjiUIlWCGE0l9IYJaMoslppZTzPQ1ZbAztxoDXrdbsQAEpI2zRu1cJbPru5HgwGH3/4UZqmm81GcO4mEN/3uZTWGGCtFNwtepKEuvKwXm4AQEKIYlt0k27gBVKWkOBOt1O2TdFygmHoe9YYyvz3VxcWwU63vy0ygLEGwCKIPa9crR24miZdgJG11o2hiICmaSjxMKNa67blYRiFYdTI2q2EoijCmFqDXUvnXkQb6QZxFy2ttUaIAIDcCkAp5SwrKKXAQoCMMcbZIjnyI0KIMNpwQRAhjEmpALDO0vFu6IdAK4UQRgAj4AcsJpQSpqRWUmtlpFDur9bvDe7Pgg5PopRCqNxiy7llh2HoQrAQQu6N8jzfsRNuWavwe0vjHSzPORd3ADiA+JbCAoCzxzDGFEXRtq2DkN12zBhzR9Qvm6YRQoRhSKi3oya4I3evY+5StbTWLr7BfV6sNa7h0P/Y9Xm3BHHAg/uaECKsaYUgoS+FjKIojuPFaulkC8rs+KTABTbaO5RoJ6kAd+QJN7vfg1VuNZlxHJd51u/3KUbGakLIZDJx9FsAAHT4DQAAGAsg0NDcM0zcLXHAvfUBupN9ur5ESeU0Hbu33jUQu1K9+1kIob1byoC7RgRCCMD3KMjtH9EaZCGEUFu9U5O6X1xrvVux2Ht6CvcnSDvxYDAo823blMYqazXBkEURmUwmYRienJxMJpP1evn111+v1+urq6vDw33HftRGGWOSJFFKzWazXtpN07TfHw66PQCQqOX8Zv7mzburq6syKx+cHgOgB71eJwn9gCkhinL9zYtvxqO94WhfAwsBy7c8ShBlMcESuxNhJG/KpWzLanP08KehHxljiqKqqgbalguV5zmhnh+Fr969n62X//4//cXVzXXNuReib15+5fs+BjhMoiCMa8GXr199++o1RlRZK4zFFnGryvUNpGQ4HkG8li2PA3/Q6/cH3fdvvmNBsFyvtkXxuy+/+ua7d0GYbItmucw1wHHSf3jy4N37N21rPK/Dq+LFt++hIb1O7wc/+MH11dvZ7Prw5OSf/9mfpb/vfPPNV9pqaJXVKkmixXQ1HI3+cvo3dV1Lro00P/zw49PT09989g9NXaVRTCn98ssvFYD/+uMfHB6cDAaD86srF7Th6CCYms2mlFIChJVS62ydlQWul1FE44g2TaZNk6b+3ri3XM21rAMfGovrKj/af/Dk0clochwGkX/pbTabm5sbjxGnowPGMkbXy83x8cnh3vF0ulwuVmfvzh+ePn725KnfpfuHBwaabVlkxXaxWQFqs3o7Ph6tqvWvv/7NVi4ODvoEaUZhamLADfZEFKYEoWLTGohYGMXA2z86pP6NBWS52q63Odeqn8QGglffnnkUR4GPCNHKci6kVhSjqqqsMVEUBV5oDCiK8uLs8vPPP0e9ANpMVLLX6VuL60rUFW+aNgpBEATD4ZAQFCfRt1+dJT7DY4QxOD8/pySZjLqPHz0d7z1IO73r2fXL17/ntiqb/O3bK4BVZ5im3cQS8+bd26PRwXy5tLX62Sc/7g+6DKM8z7d5MZhMev0hQpD6rNftHh8cDnr9rN1GUXR9fZ1l2ddfvzg5OVmvV+v12sHvvJXz+dyR5AEAQigI6Gazcas6h9u7W5IrPE43YYyJ43i5XHqel6CQ80YrQTEihACDgdFG6aaqy+3mL//zf4qDkCvVVDWGmCIKkO0PeyfHhwCApirbusnzbbZd9ztpGPhxkkZByHlT5EQpQSkyigEAPM/rJKkyumma28mPMYSQAZYQAjDa3XEwtp0k6qaJKxJaa4IRQqiuay055zyJAkqJlqKTdKIoMlZ0u2kYhptNZq1ljBkNqqrinH/66Y8ZY7/85S+VEhhTznmn02k0hxAS4gVBEAY+ACBNU6VuRWJxGB3sHzmGZtvybL29np4/fPgwq8uGt4NeP0hj0XIC0clkeP7+7Ndf/JZiRBGez2ZxHD969Air2mgguDLGMM9Zmxgp2+F4lGflZrMJkxhC5Aqnq2du3CSE+H7AWy2lKsvy0cMnRVHUTQkhbNtWSu6qbKsllwJiZKyRyhhrCaWeH7RtayEwABpg3YwOLTbWes5hEMGqql69ecMIBlbHcex5HgKgqqqmLBCAYeRTTKIgrAkGAHApjDEAwTCOoiROOmme58roXTVCCGlrDLA+w5QRK4zSEhqICfJ85nqyMAwxQWVVNG1dN5UQYpttIKIW4n8EgNvbmdvVY2P+0eod3lk1cM6rqmqa1q08et2Bq8Ft20qpmzutDQDe7sTu0AhXutzqzdlRuI0G5zwI/J2K0j3T3iMr7DZi+O4h1C3pr67rOI69MFiv1/PlYjQaGWNcxLVRVmuDACCE1HXjKqIDDBx9wYkv4D1u4G6+9yjRWg+6vZY3BGEhxP54ooUEACAAMYQYQWOABQAYa4DZLa3uwx67ir7jEOwwDCXkDhrZkRnBndAD3nu45xvlNgjfV3pjzF3bcK9XsMZCZK2FCLqmHyGi72koXDeJMHFne9e+5Hm+N57EceQxnKapR5DglVaC5HnuWj9njXm4tx8HoRBCNq0/GFZZ7nkMYZwvVgghH2Ib7Ac09Fk/L2VWFsvNkhDy0R/9IN5LrTazzbzf6Z6ennqUVXmxXW9Y1Ds4frjeLLfn31ogyyaztqkbUK3z0biPSZRXnHOLiB+EHtPs83dfJUkSeaESWrbcKgQ0AAius/zt77/aFvUvv3zx4s153O1pP/TTTlvAohI+kkmCoFh9dHq4hkW23mBMNUDKC6xP1m2ltWV+fD3bYqCBgb3TcSfc+/mnHz+dPLo6f/Py/KYocz8Mk3730ZPnEDbzzbmwRa/bLxtTtwhif7W1EMWSqlezdfL2NerDm5uX08Vb/vn1k6en/+xffdQ7NBIvpiJQzIAEjx/3vv3qJU1wOopDmmiuBuMBJaCu8mE32R/18tVNsbnuDh+ul6s/+ad/+tW3Ly6vbjbrWS1kq1Qgw1ZwSgcEB8aorBBjbabLaRLri8urVbamgep1PSSb681XSjcoUMNxVyvz3bu/B6z96McPykIengxfXbwHCu4N95bLpdEo8Lqc68gfDjpJnQvT8geHT/7kZz/v9bzeIOz1klffnSO+ff3uzWe/+5J14mcfPP/tV19ClgZRpLNu25ibdyy7bkzLfUx7HTI6QowxDxMWkYqvt+q61OevX78+OT7tjIdJPOgeUNkMAv+hFODy4po869Rl5WFNAcCmUFXerjabqhVCYEzrtuB6K4zeltVltW4SGtugldAwepGvIAGA4Kt375XXZd6eByYXXzcffPDg5oUgbao0HaWn2iM30/nF5aUfdf783/zxu7OrpAf+6NEHH326H8Xsb//2PyHYfPnl7J/8cfPDp4ef/Xomm7OpmFqDa2EW+fLTP/njv/iv/wAlRDYB3N/vH3kBvZpdGCAKtTn/7s2rzy+22frf/tt/W2/Kjx5/mHaT54+e+pidXV5QzL56+eLJs+dGAwjR2dmF7/umzpRW6+lC1S3zPYzxeH9PK4UxLpvSQHP86MT+d3B9fY0IXpWbNIqEVpEftE0FKFEGtC3/6U9/enR09J/+6q80wlxy5vlGKAtNVucfP/k4jmNsiOd5NMBQIeUr2qWex8aTkVGaeBELUkB8znlb1VEIMbLWNlxixli/H2Isy6rRWlvK7PfoKNbAWimTgAZIhxQCABgk/f54NBohhG5ubvI8d6J2AAAC9HYu5HWapvl67RH04OnDuioyVR4/2T97+252Hjx//mG7XWQ3Vz/+8U+Xy6Uucmas7/t12+R54RFcc3H17n3S7Syurzrd7vn710oZRqEUNSGk14208GUpGGTD/lBKmRcVhqhWDcAA++zb714c7u8/PD7Z2xshADpJiHu9N2/e+Mxbztb7bMIABhBpo2EtTNtaCGwQxGkMCRJWX6/nStc4ACH0pG15VXMtEEEWkP3DzuvX5WiYMOYtFgurjZQySTpFVnb8BCFU1iVUejwatWUNtbEaKKmlEFIrayAhBBOEEGorwxsb9oL1qiQUNWWxtz8GiALsIUp5o2g02G4yL+1JY5jPDruKc24tc6EC3ZgxRrNt4QGtlXXrD4ARAMgjNPSDXpQ0TcMgQ5ZopZuSi3YZxbHRdrvdGmu7vZ7WepOti6qyGDCCa95KKQeDvsMhOr2uUgphbLk22lprrIXGAIJ9z/PCELuodEIoxkTdZRBoIwklfsCEpAgDY7HWAGGjjRFSciGcnBJA6NSilNK6riFCFgCldRCGzPO2WYbx7XZ/ZzHkzCs55253thNJug4pohQA4GHqER8D4nvx2fl0van8oOP5KVZKttIYQzAmxiAAENS+hz1KrIUAQAiwVVBo43kBZhREqK5KikngMUyg4FWd5/00pRBCSqy1QMnNet3tdrMstxYagJQGWkOLISQEQeJ7juUjbzcO0FpgrDUIoTtvJYsRoARBCJQSxAPb7eqoewSAtVKGESvKjZPUWuDkyIAgZAAwGmil3d/CGogwthZqbRBEhBIAEMYUIWSNgRa5RCqtrcLIIKIAsloraxGlDkcCAFBKHZuwrgoITBQyRkC93RrRpmGwLPM6z1maIAuiKCFBEJydna1Wq/V6bYwKPf/g4CCKIiV527bX19edTjoeDm/huzQNIsAYIwRoLRG0aRRTnwae/6d/8qcYQi2l4Lyt6iJvtLReEDdb1U/3O9FQQyVEXfNC6QZgm+WrbqfTNPX0agYh3D+YUBPUm+bddA7BHBpAMeuEaRwmQNu6EvPlti5tmWulcDceeV68zouQdeJxUGTb9fS6G4cnRwdPPvhY84c319fG2nVe3WzybZ5nrdHUN0ZZC25mZ2kcv3/f7A+COGFHx5Nvv/kMGWUBb1oZaEY9Qz3DqFYyu7papRGIezaJYdHOmrYKfcq82Fi+Wc+zLFtMV1aItmgfPnxsZVDnvOpIKYwWgteWkrjXHYoa6sauV/lstiAIIUSUsVyp4+OHDx8/Sbr7dV3/5tefnV9d5XVjlFZCQAiX80UQhRST9XLVNM1kb9RU7bdffQNJYbU+PX0UhmC7uarrTbcXjfZGAADKwm++fmWMFEIJoa6vr1ebQrR6s8nC0D99/DiKotV6W5RZp9vvpP3pdC6lIpjFaYqpvZku3r17k2fN2eXV+6sLYcHj0wcpRr3L67KpgYbAYqBgXckmb0VV+pjK1kS9oeelECQei0cj3DaFkI3h9PW3ZwjdxEHqsRhZr5MOO53BZDjYH3eN0gELDJdVXmotr6+vV4u11tZYIJTmWmmINAK81YyG2gBsgVK6rppWcUiwNsZoQKmX5+Vms8m2RZaXYRA8efJkMJrQyEhhlbRxELc1/+2vf9twYaz+l//yX/zxL/7s0YO97fa6m/yFENuzt5e9lNjEo9i/erfqJ72//Ku/+Lf/1//7/v745myxt7/f743iOC7rcrPOpBYs8NtGUoYfPnyotT4/P+/1Oo+fPhoM+6cPH7z47uVsdsM80jTN/t5h2/IHD46bhm9FG/iBhRZ5tGqbqqrCJB6NRkprZBHQQLVcCdlUtUOSa1h/D8YiGARBv9+llJ6fnzsWlQHwVgaNoMMGmqbJ8k3btkpwYwzCgCK8WMzLKg/9gJB9hz93Op26KFc3U3WXoezGPldslFLulnY3VzkFFnDcN611VVVu6CnL0gHduwnJYaduf1FkG2U0I/TWJcL3h3Tc6/Xm8TLLsvl8Pp1O/TBwhMfZbMGiSBkDCcUAtpwrbSxCddP6QUSY5wWRB5CpSgCAMCYvyqwVXuC7AcxaCy1ABFBIV6uVERxDuFmvkyBMw+BHP/z00aNHX332K865aGUQxdQL5vN5lVdprztdzBshAUYSIeZ7rVJKaykUZQgCSjAAxlpzm8AkhJzN5knSMcZ4XpAknWyz9bzAwfhFUbikUAsBQkhKucOHEcEEAmBv50gNLDQGIeBOlzaWMhyGAQCg20udY6Yrgdvtutvtx3EMaZt00iRJgIUum8Nt55jvGQOUUlIrAADBzFjbcp7n1hjDvNuRXWuttMaEVFWltMSEOLJFFEWEMWOM4treyeqk2ypJ3rYCIWSM0hpCqK0bcCFysL+TasB7W39He3cXkgubJYTsQkQdtOC27xhjNzS7/3UXkrt+HPljt8T5AxWl+8V38MmOLRgEsVt8VFXVNI0zHnWXN7wnBDDGVFWllPIDBgCwELpGwRpHSnB6K+Z5nlTcOTAieztku3d3G4eiKNpW7NgD7gxbe8cksN+flh2ocH/RsINtdo0OtCbwfAwRY55oeVEUCCEnrLXWamD/EYDw/WbEHfY/Ai3s/84/GRd3uYMx7B1hwn187iMr7jdFCDVNK9qGcy6F0lojiAmjpK7r5XJ5Z7VroA+done73QZB0O12CcFOyuW4slmWWauaumiF0Nb6YRD5AcNEC818348Cj0orUVnwqhRKqfW8DgIvjGLCIEqtBbJusqrKWBoq3vqA/OSDX3T7Pcem7HuoLoyUUgtpNRItzrioqjrPaqlMHA02mWibst+ZsDDarpuQJv/0X/757Ob6v/zFv2+kzhqVcfXg8NTrDM7Pz02rFEICQupTRIiQjQZw2PN9D12evfwuRT/54cNJP2155nuE67LY5ttmwU2xyjIpVx99fOz5/uuzBrSShgJRQ32SRkG3x8IYN20OjcTWtGUzvZj1k6Gq9OX59G9//aVWwCoIDLKC1KXMs6bcVN2kqw2EgAZhooxerfP9PdEbDn/9y1/dTOdeGOztH3hemCSd2XxV1lWYpE1e86rOsqyuQSeKiYWqlZtq1U0jYElZ1Nm21cpmoM03pdZ6ONqPgv77s5s46m7W2dnZWVmLH37yZ8+ffSgU/+7Nd2XVDCd7UWfAwuD85mq52NZldTO7/tVvDQRtHLH+IB30R1988cXVbD45Pq6LViEEFDICWoOJ8SEE2DJoFYGIEIpQ5/rGrNZFFIvnz9JOetjWM2S9B8c/GPZ7Z+/ez65nl5tZXbSdtP/8+YcnRw9WedbWNTCQAMIwgX7Ehao4f/Xda2WBVEAoAwnGngcJNcZAZbxAa2AbLuqmNdhaBJW2QRgDAHQLrm6mUmrF7PHRg9Fw8uyjB5PB+WpTfPTBR0Ja0fIyy7NC/Mf/7T9++skHP/npx9k21aL83Re/nN6cHR/vzzbXccCSDohTXK0qY5uHjw+SIIyDPka0KIrlapXnNSK0qY2SbpxSf/u3fzOfT09PT39h9c3NzTfffKO1Wi6XYRjn+fbw8JDztt/v39zcFKL1kS8RwEZLpfKimC8WaaeTxDFCyGMMGGiE1rUAGDPGGtsQgjjnjjPoed7BwYEQ4vXr10VVQgghgMYYjCmmhBAym8201tpId9OBEAJlhLGj0Qii7zMAtZFxHN9cX8fMc3curlRZlvCOWQ0hNOY+QHpLm8KUBVEshKiallJaNe1yvYEQ9no9t5C21rqVitSmbtq2EQVq0g4WUs2XK6sVIURpvbe3hzHmUh4eHi6Xq6op9w72IUaLkrdVRTwPMbYuKgUgC8Kq4Rpq2/K4z4IgEO6GK0TdiKLOtVXo1hVASykxYQjYqsgwRJHnGalE21jP63aSm+tLo0HgR449oA3AhKV9f7K/d3V1hQgFCLZcCK2VAa3gnHPPc85UUInbcqi1bkX95vVZt9PJtgX3JARYSu37YZ6X2pqyrgCCg2iIMa6quhUCACClNLepilADANwCQgOjFIBAKCEUD6k3GA3H+3tGiiAIsjxnHinyClOMCI7jsG3byEPOmMjo21t8EAS9Hthut+7O7GiwOKBOYWsERQgRepuIba3lglsApJRSCSBvxRoI3xIUjLEu/NBaC4BxZBprrVIGAJcDAAAwCAFCXBxz7jZQuwIM7gIhnQTDCYARujWxZr7n9jiEUYQx0EpqJaXU1lBKme8RQrQ1DW+lVoTdcuvsvdLofgvXeewofrvC5jgTnU7n6dOnR0dH3W73l7/85Ww2c7ty123s9gthGHLRWGuBRQB8zw10dddx8hAGbdv4PiMUOacAx8EcDod1Xbd1k2XFeDIJw7AVBt3TMlhrtVaE3jIizT37pt3W5g/aCADAjtccBCGE0KVdKHXr57hbKLgr6T634K5XsLvn3D8Sx4yw1gCAd7/m7r/ozivFtXS7ps1aqw3YbvOmqWTTFnVDKA4ZhRAT56CZpmmv1+t0EopwURR1XQOrDw8PR6PRfD5zvWHTNGdnZwCYMAyVNVXZYIzjMAwow5BWecFrHvhRGIbd/ph6kUMmm7y21q5Wq7rOKUNhxJq2XK1mZZX7Pk3iuDNOhnHv+vp6cTEFAFxPsyiK0rjDGC6qRgkhuFTaHBwe7R8eCymXm3U/6Hb7ndkVZEBFSfIwfDSe7CveXi9W5NtXcXeQdvrb9s3Neruta2Mh86kw2gJ4fHT06OTRZrNBJttur88vXvXTD5Oe3xTF4ycnb9+/KZsSwvbRo71f/NMf9Yej716/upqtr6/e5lkzGe11uwkwIi/WecY6aZ95ejQMJuMhMDb0tAm1Fpts2wJt20aJVvkkDr243xt6JLJCKWuI73W6faM582OLWdGIV6/frtfruNNJ0h6SZjIc+UFImCe0mk6n1tpBr18URewHyECKcJJ0wjCwhvDWMJJGnb6xfHZz/f798vmHwZ/9i3/5+9//P61BZZm1jRiNRj/4+Ecff/zhN99+9d///u/8JPjBj39SC/Xim1e1UpAhQNB0tdhu5r4HP/zg4ZPBiDJfK4gxk1y/+e6dAqApBUWBEhYaD0NEkU8w8pCmCFnLptdVU208n0z6T0bd/SbLsm2OIGsQDHDvwX5qx6DIcmhgCH1RNKtZOR6PT49P+t1BW7eXZ5dffPX1N9+8CPzEQmwh0sZqC0HFASIAQWR0IBXEWCmlrDHKKmOkFgDBMAwhhBAOFotFXbeE0dFk3EnR8TFGZI4h4U0FDYjjVKn1dl2fn13+7CefduLBR88/lZy3VZttOURAyOrwaGy5/uGnzy4u354+PHh49PDVi7P5bNa2UhuDIIaAaAXDoPvkCb64uCjLfDAYbLPN9fW1UuLm5ioIgrIsKcXFJru6uqqr1t0xLbZcc6kEUtCL/VSnRV2cn58/ffJkOBwOe32feEAaaCFBBBlojKHUcyOXI8x7nnd2drZcLgmjnudxpY21BGPGGMbEJS0RijzPYwRrrVtey5YLIbSR0ILr6xIAECfhaDRSShlCIYRRFFGtXdQkIQTjWz3k7R7hVgp/O0tBeCubjONYa+1sBMMwdGI2F53gMOG6rokXCm2cm2Tb1sZqSrHVZn9///DwsKnqP/rjX/wv/8v/Y7vdQoifPn16mU1rIYdpByEk9Y0FKIrTRq6F1IhZRHAQhUJrY4wCQMFboxitldZKKg6gQdgwQvvdntUq8oOQUoYggrbKi7/9m//e7wyHw72maa6mN1zp8WSSpB0AgKGEEGatFUpKaQGCSpmqaqqSJ0lCCOFcW2sppcbYtm0uLq486jvTSSGkNUBw6XtBI1tHU5dGN4LXzmAYI3mbLXjLZoMQQowwxhBjDIGU3AKNCOwNer7Ptm3dZOuizBhjDa+jKPZ9HxF89u5tUrAwDHt1TQg1t243MfX8rMgpYcZa6LKkCQYQCiU97JyaqFvwE0Ka1jRN4/u+sVpp7UZ/hEErBOe8E3cIpgDcum4wj8Rx6HbVAFpjjLLCQqQVgBAbe/sXd/SFXaNwX4u/K2y3akZ0y+50vZ2+C5tw1kk7eeeOeeCG3duVyr0IJQc83B9/3RdB4DlvzTiOkyR5/Pgxxvjg4MDFljoDUCfgVEo5GY611mhtrYEQodvkCOIwIUopgMb3Pd9nABqnqrXWFkXhXpAQImXpPgt1q1yVhRZbhPRtlNT3NfgPire9l5e94wTEd/wMoA0GEGjDIta25a4FsdCJIYGDJu6/4B/0Cn/QLhhj7oInzf2uZdey2DtRxu6UGmMspFwoayGApKwaaDTsdgDEJEmSp0+fOunjZDJKwsglvh8e7LnXYoxFQQAhfPv27WefffbkyRO3QdFSaqFk3UomNDR+FFoDtZScS4QQwqTb6w9HZNgLmqY5O3+7eHGxup4TbI0VdVn+8NNPfvqjTzGGy8UioOSDJ48Txmaz2WbTJFHY7caM0KbhlRUeRAFiUYelHQ+gNltfMSqTBLf1/PJ98Z//4j+cnjzQWgdRyBuUV3ye5SyKLPNKzmvBEcHQQCHaKIn/+GcfPTgKXr58MUh+9PXvv9hu53WzP9kbf7OY9oe9/vBnFlkv8I21B/v7o8k426wDSAJYEEgoTHwSIiggtIRC37NGS+qJvcMQGOsl0o/Iw2f9yvSkVPPZ8mJ9Pd9uGcEUe1Ahj1CIIfUpN2q5WCqjqe+Bc6sBYGEYxEkjlag5IGQ02XPKZj+IfOYFoVfXteKt1nq7zowvkzAhxJMCrVc1b2gYeRR3k6jtdw+fPf2hVv+vr79++ez5B3Ec/6t/9X84mTwottnV1dV4PN4/3U/6cb3aCiiEVVlTLDbzKs89jyadSCj18vWbn/3ojzq9IaSBQXi93CLmecgnzKvqFkIMLVASQAi0tEVbaCkR6rQNEo1qcpQE+0cTM7/cfvW7L43m42H3w6fPTx8c+8zzPD9gnlLq4bMOIaTIyt998dX5+eV2nUGAPv7hT25uZsZCDYBWVmojtQUQYISgVa1srQIWAM/zSEARxhaC2XJetfVqtTo82kcZUlwRRk9OTnh73ukNhYTzxebd+4u8qsMwtABxaT7//HcffvT88cODtDP88Y/+WAj9d3/z13uP+tcX6/6RpzWGEM8X0x9/9Cdnr26qOl+uprzVabfn9qlCiP39/aPhkzSNgyAIQv8//If/7d27Nw8fPhwOh5s821Girq4uoiiRUh4fH0nU1nXLm9YYQynu9XplWeZ5vlgshsPheDQxxihpKPE85pu7WcrNQFpJY0ye5/P5XGvNkIcwEfqWoGyMQchGUSSllJILITB0YcHKGrNcLjGBRunlcs4YO8T7URQlSZItV9baOI7jON5utw7NBsD5B2h9pxYjhFgEAQANl+ttXhSFUhogYrSV2gIALMQQQ+oFABFrrVCmbXnTil4caK051w6fJpQhBIUS787eI4TSNP3w448++OD5559/sdquPmQfW6ut1ZRihy1bYBkjlOJWC8actA8iDKVSQnAIAaXEQqu1llporbWWwDAMbTroGS77aeJhhCEKKGnLYrOYy8bEnXSxWm+zPE27aW+ICH7z9n1Wc8assdCNdJ7nIUgwolobjAmlTAmltIR3PHkpZVmWjx492m63UmZB6BdFkaapgLo3HHieV9d1lmUIkziOAQAGAOdIILUyBiCEGMEYYz8ItNZNUyFCMKNB6FVNfTO7lqLdbDaT8f5g2IvCZL3eLlYLLvn67MYNx/3+IAxDV+puszqp5zB/h9vvUG7neyGUcoaMxhghpfM2uEMRDKHEkdKSNCKMFQVqea2NZChw1kx3Ln7KWKUUFFwKIYxR92dWeM9+YGfc55YFO3/Joq593zcQACVvvRc95hNSVZULLEKUEEqNMRBYAKyDKwAADtg2dy4CUv7/2PrPJ0nW9T4Qe236zMryVe3HmzNnjr8WBHDhCWI3tNqVQhG7EdJ+kBQKfdJ/o5C+ahkSueAuyQV3QRDLC4C87tzjz/jpaV/eZKXP1+pDdveZC7BiYqKnp6o6u6oyn+f5PT/Dr/kK9c+67j/qBPDxePz555+PRqNaZFTThHu93nA4jOO4ziqilHJRCSGqkguhtFYIX37aaxzl2kuKEMR4KYQQUFuWtU6zi4uL9XrdbXeUUlEUEUKvmZWXtR8ijH8jkOm6M/iH36lLNcbYtS2MsRKSM6akpJQSiKqq8jyv1mkAXZsvX+avXcISEAIItf7O2entI4Fv9RNvNwrqN2Ud4Er1cL2g0VpjTBGxDMOqEMrzklfMcRxVR1LUIvvNZpMkSegHtZj15OREa33jxo39/X2T0sPDw+fPnydJMhsvqpwbhlFkuZQ6j8uZvdBaN1sdvxF0+j2DoIKzNCsIIa7ph532XsO/ee/mwc39Fy+eTifnWbIxDBqGoW27VZFLoSmiFjYbfsu1g/e//xOCLi07kiSJ4/jKXpsgDALf3d0ZNpphv9Nses5qFa3mI9+hZZEBbWmgmZYX4ykxDa4kscyeYyKsANByGSO+6Qak3bK0yGwT5Gl0evpme9jVWiNqnJ5MtreHjVaLGLQsyyIRG1pohsq4GLT6oe9BpCxMh8NOI9wdbDm+LwlNZ4uiYgxhkKRpoxHu3Ni3AldwNRlNTBMevjguMglRaXuu59jNjhu2fY1kFK+pSTplWZblKo0xMbx2k7q2YjLJizjaHB4fWaZjWRYm0CS00fCDRigE0wp0eh2LUilAmetoVSyma4RBniZBw3ftTp5K1wlfPH/9h3/0j1++eDMcDhXjRZWHzeC9Dx+VqpotJ5bvP3z/PmNgudy8eX00PT+3DTTsNikG8WZpWEGj0daAZhUzEACAMi4rXlJETHo1Q2gJkAZQMcksLmxo5Hl69PzNw5t37j14Fwv8+skraiLXbJi0BZRXFDpLKiGyLE6OLj6HEKZpzjm3XceygpJVXConaJSsUhXXRAOkKYWUmKZt8SrlQgghkEGCht8ddNv9rhv4X3/95Zvjo/lqvndj2O020zRthG6j5S2WZtjsOl5r/Ktfz1Zrw7GJZTsNOV6sv/z22f4vf9Xt/xPLhM324PbtR5999m2ZFXEEVlZ1Z//B8Zvx/vCW47gvX76M4xgAZdmGZZGSszhJgsJ1XMMwSLPZDJuNMAx2d3d93zVNmucp5xWAqqpKwyRFkdh2hxr44Tv3LQ+9enVYEpzneR4nlFKKMSQwiqLZdL69tVNyUQiGCEWESg201nXpIoQYlCilptNpVVXX2+v6b8YYQFApALSQUtbSJAUBQgjhesLDmMAazIQQ1vyGTqeTraOyLBFCrusCANI0ZYzlRdVoNIRQ1xACxrgWExZlzmbzujKt1lFZlmXFbNvO8qI+MS3LJoSUZamKUmnABcCECAmkkpZpGJRowDGl8/n85etX77zzbp7nf/QnfzxfLF3XzYrUtSgrkOAlY4yzglJKkCYY5tmm0wkFr5KYl1WZZVmRZa5pZLIUgl1ByhBjDJGGECTRxrOt3Z2tphfMRxee7czHUwR1WbI4mxwevQlbnUa7Y7neMlqP5wvDtgSATAguBFJKAawUgJAYhiYEIwQhArVGl/OK88owDC6Kbq8ZNr3xGKdxEm1mShtaa8d1DcOIk2QTx74fAIxqA0EFtAIaKaSABggigolBFZD1n6DhhWEjbLWKMsMYJlUFIKQG3r9xAwBwfHbKCra7u3t4knElN2kiAfBYBTDCGK/XG8651FAIIYFmgquqhBBq9N2kWJex+laD+RjXhpy1RQGpLX2EEKZjGiapGEQIWJZhO2ZREq1NoRWEUEkIuOQcXEIOlFybEFxXo5rfUPcNWuu6QblUExBommbdOkitamQFIAgxklpxKYSSUKHaSlxKqfISvmUXeI1YXG3HFADg2qTBNM04jmtYwnEchNDJycnXX3/NOa/1/FtbW+12u4blashhvphyziHAVcWlVNfwvhAijisIoWlRw6B1cGtRFJ7V6PV6SIMoiuI4dm2HMVYul6ZpXQMVNUMTYQQJBFBdox1vD/rwN2/XdXoxnTUaDdswWVFmWWZQWrethBCltVJAKAk1kFrXgZ/4SocC3uoHroGca7gCQY0AqJeJ6h9kZ7wNbLzdygAAuJBCSGoYrBLL1caiuN1sFEVJDMOYzWZ1Tzqbzdphs9FoRFH0s5/9rO6jPc+lGB8fH49GI4RQvCmrUmOIhBBaymgZ15ENftjoDwaU4kYz8DxHQ8WkkJox7RhOu9PrDA/u3X3w+Oz0ZDI6jVZL37Zn8+zi9EwJ6RjtWbJeLJa9Xi9eiUbDDbwmpdQyMtfNXNdttVqbdbRard55+P7jdz+yqGFZVpXwFy9eHDy45/t+Eq3jOOaVyIF6+fqVApKLyvOcVtOvWK5ELjkti0xXawIdJctoFWGkMYSc88l00fA7o/FyE5eNRvT48eP7dx+UZXlydPLNFy91lRmE+I5JKXY93PCoY2mDIEpJ0OgqZaU5Nk23VMi0W5ZlW86CUmpZPQUyBKrFbGUSp+G1bMu3XEJM0B20DYPevn3n3u07URSJzz7PyyJsd5hQioJCctf3HT+oHTPjOHMs8s7Dh82yIAi02+293YP1clmWueeFW8ODJI7iOKrKJEOSYFdwGPgtjSCCpNlsjkYjH2FEYa/X8aT92befLfPNex99PGj1ZvPV0O1XVRatJqwqsIE911DaZZXkHBY5LyteFSyrkqwobddtNBqWbdbr84oJhJXjUtNCYsER0FKrxXj82S9/FVh0q7v98fs/AloqpVynhVBjOp2+evHi/Px8vV47tk8IYVI0Go12ZwthnC/mSVE6jsOBQkCbAFjQgBjVtBjOIGOs4kxqDanEBvADu9MPt3cHi9W03XFb7cbBjd3DV6/DdqMSJdcgY9J1veVmM10tNMKlEpbn375PlObPX70uODdsV0Jqes3+9n60PL994531bC2YmaVqPFq98k/6/cHJmwtCsZKyYpkCmhpA6SrN1ivFV+tllqfdbvsnP/kd07a4qFzPTvLsEmvVtK7BtYf/vYObT7/4ipelQWhcFZILw7Bsy9IAvjk+Miy73W5zDRTFAmMFgAFo7fwIIbRtqyiyyWRaM8allBromv/FpRZCQIhr12FqYNM0ob7cK0MAiqKQigvbgRD6vp9lGQDg4cOH8WJZCzivNWnXNKvrMfH60gYA0AAkSVJfalerVZqmrut2u936cpFlWX1NrJmMZVlKjjqdDpBCSA5tm0tRFrlhENt10zQ/OjqyLOv9x+/1er31en1xceGG4SaS0XKaZZkoE89qUawdAzVce2fYs10zimJZldlmvUmTMAx1dikkQxgQRDGGSklWVbzIqZa2YW4N+zLLdoaD1y9edsOWQEGcpkoCw7SF1Ks4idKsktK1rKqqmJBMSASUkiWEGEGCiVBaCKlqNF5KoZSkBrRsImSVZpudnZ2KpfFm4fuOENUmLsDVS6cg4EoWRVFWlVCyLopIYyS+47hVVQWAAgB4vm85pmUbQpa25zJehc2G0prxEgJsWUaSJLZnt9ttcZlJUYirkOUoimsLh3oIllJqDWsvNZlm9TIeVhW68mKqSynCEClVVVX92UB1AjWvDNuAENaulJZlmCaFENaTNwCAEAQgBgABDS3LVFdZiNfg+dtAen2rCYk1w8akXg1y1OvwGuGo1x/wqmu59leAEGIprxmO+spIqibGXqUVfld6EUKuG9T7BUppu92GENYmB4vFgjG2XC7rRUbdfBdFIeSliwnGGEIEwXfOj5xXNQ/Ptq0k2WitayuRra2tbqv94uWzOI7jOBZCaADqA9ZXfglAKagUggQidL000VfQf42y1PD+NR5T/+7JJu53e7du3boYj+fzOTENIYRnOzVrW0ONgBZKXeMK9W+tAdTokntYv5K/gW1cNQoAaADU9Ut9fZrXPxpehYFdAxIQQiEU41LrapOk0ToOG64CiHFJut3umzdv6h5wPB5v9QfD4XA8HteSByHEaHQx6PXq14gxFu5sS6GFEhhADLGSqqpYWZZlWaZpzCXjQHS3BgppqUXORLN5oxKkjJhlmZ3BTc/tdNrDaLmosrQqysn02aunz8925gQb89nMNE9tv3Pr7p0HDwLHMdbLMs1LgzQ9p5vGcr06t83Wuw/f0VonceRaTS1opx36vn9rf3c0nizWERdiMpk0Gn6r3fA9x7LNaD1WsnQMbEDDMRGGuuG7WZI8ePBgf2dvd2f/my+fbg9382ziOlQrslzE89lXR0dHmJBBZw/ena1WUcMX29v9oOFVooyTBTVYxWEQuBAFAISW3VOSQtjV2iZ07HtOq+lZNjIIHJ3NMDQCv21iZzmP4niFEBhs97e2hqZjp+ORREAi4IWNvGK6rLI8t/2A2lbQaqZxEqfA8yi1zIvxKI2Td9995+XL1/E6smyjFTbDsG0Q03Ua7WavqopuZ7C9vUepOdjams1mOzs7y+VSGd755Mxwye7dHUThOlrMlhOrzM9GY9cOmaoA1n7D294Z+LaBoJzN16tVtFyuC8bTLF9vIibUjVu+UopgSAnO8ipJ1hpIwyAYY5sYCOHQbxmG8fr5syqL33//fcdyms3m4eHh4eGZba+Wy+X5aAmAtbd/L4viRqNRVGWa5YevjzEljWbz3v0HpxfnNgSGaVKT1GdaTbEB2uCiyvI8ztI4jipelixfxt042XDNhjvDTqd18+bBdHLuB3ZZpYbjMiFNoDdptk5SJoWS3PU9Jqr1YnNxEf1v/svRQ/8OpJbfCB+++/70pPXg3oN/8f/781cvzi1o/+qXX7IU/F/+2//rz//Dr6RUWZaBIm60mq12QClcrsaZsmpewvb2cHt7eHp+xkX1zjvvLNc/Y4whRLjgjuOmaVw7393c7m2idZ4VW1tbnu0AjSBBCCHLdl8fHSFiHCgNEESYKggQJSax05TX0149VK3X6+3tbQCA4FIDTSjBBKqK1xNEbacDoKqqSnIG6pBAQk3TTNJadIoxxqPR6Pz8fG97h1Jag881bc1xnEajgYmRZdnf33dehQxlWVY3bavVKs/zGs7tdrsAgLOzs9paOI7j1WpVlqVJeLPZhBAoBTDGQkrGuW2b9+/fj6JoNplMp9PJbNob9ur5xMRQ82q9WiZxKgUzDWRRUpn0wf3b7737iAn+RryRqqqKPIui0HXBpX+twhiTOpGVcyFVt9ms0jTerHnZpxQfHBysF8u9nd3RnPWHeJOkCsHTi/O4KpXWJauSvKh58kgjg1AAkEmJaVkKFFrXO3JkGEQqbUBCoCWE4Ly6uDjb3R1iDIsyaYTuer2OoqhevSsIam16lmV5UdS/PsRXUQVacykB5wDoGjbGBFVVFcWbZBPlRQYJ7na7x8fHz1+8CINms9lcraL5fK6ANizT8zyDmlpriFCW5+tNZBiG1qwu2IhgwzQbYdjtds9fvqqrHab0svRibJgm55xCUs9+UkqpCKYUAECxSQjBGFYVZaysq3iSJMvlsqg4AMQ0LEQourIcyNL0kq7/FmRdf8ZqHmLdU9aliFLKJKsZl5zzellQuyn4vl9vDQgh9fq/vj+6MoG+nsjrX4cQstlsahP0t8MjTNO8tDCpqtVqVRTFarUKw7BmD9TMA9M0a+JdmqZcVAAAoBEAqAbxwRUZE0Jo23a73a7TQbPcrkkerVbLtezFcpYkSZakACDLsjgXl45kXHFR112kITSuDDmuO29xFXZ1XZKvewUpZafdvnXr1ve+973PPvv8+PhYKVUxRohRwz8KKFHf/woOQP8AUbhuPr7r1QC87OOAhm9Zev/GAHAV9HCtH6nBAmpYGGOtleCKK2matuv6lJrk/juPgkZzPp+OJxdCsOPzIw2Y1urOndvpJj57c7ocR8P+tmU5RIaObWVF2mm3EQBZshFMACWJFm3XzPMcFlBnpdhUGx0VZWW49nC4XcrCtz2r4QINAFcQYaVxmpbLycJ1rJ/89u90m81vPv9yE604561GICA7O2Mffnj7+cuneZ4PtvoCRIcn3xSsKnQcp/yf/cWXZVm2m80SpcARm9k5ZKEuYkMWTQtOxusGAT5SIUW+3/n1558qzaRkUlVB4AkGiSSQIV2gZFFubP58c7pZKgszCoNoUcSr4rd++Fu+Z7iWfv365dnxWXvQvXv/7iaJkLFWqOCqDJpkFU33w93jN6dbrYFP7Gw23d3aluXY0E6BtJClkCxsur1+U0swPovu3n6v4fb++f/3X8o8r3JaFdXh4eHZ+dHrw+dpQu7ev4cVjuZRb2s7DOInXz1zHJcgo6qE74DAD/O0YpVy7EBKWG4ypODh89dsb6gV9337T//s987PjhkrISwOX3wjis3R8wVi+SeffHJyeKh3ITbYN0++RZbx+O7HF2fJVz973QibUspxOg0cZ7vZeXj3DoUgXqyyeZap11hxzyRQMq5FN3Adx0GixIXiUChM+2HoQjKfz7vNbrfb9R9Yq9VKMm5SQ00zXm48C97dv8PKajPG8yxXae5A9uCgq7RcLBbYRwXMrNAymy0AgOXYSqmf/fIXiGBCCCZGwSXnKedcKgAA4LACANSf+oqrTbacLxfkNQVAOa6VK9buBHv72/ce3LoYn9y8swMKWnE2vxi7lpluopJXfrN1dHHmurbGBCL+13/5N9udbRe5qKR3tu6ATDtG8F/8Z//V//w//cXxizeyKkenx9Fy6hh4U+QGwZgaeVYhbPihn+eAENHtds/PT588eUIp5hVTSr/78PH5yQXn8sWLV0hC3/BBmfmGf/rquN9xH7//7s9//vOCFb1+e7lYIQQw0KIsdnt9JITOCw8TI/QQQm8OjyUcb+1sNxpmnpez2UhruLWzW1QcY4oo4VLxSnBZbxaJFNowdZqmSinbtiFFWZZKRRzTjDarwWBwdPim1+tFUbS/d+PJt8/u3LxzePFiPB97LbtddiAERVpgTXu9QGKVyyqOksFWXwLJdCG4bLVCClxbcsOgCigmmB84YdMXsuKivHFzb7GcAgg3yWaVRFGW2LbthWGlJVLAa/gKIcZ12O4EgUeodffuPd/xWq1Wt9XfrJKf/OQP4ziGpmktYlegKBOW4TKJX7853dnb7fW3s6y49+D+eDwtisqxrFltIayI5TlKcFZmeV4Cwfqd8M7N/abvGwjyvDx8+aTf7n7+618mm5izYpWuyopv7TefPH2+s7uvMVysNs2wIZXUgld51mw2GS9FWTRbg/V6RaW0Gg2L4EpVGvHAM9vtsBEEUbQqiurOze3p6BRKdP/WgxcvXnXDgdgLzs7OViLyAh8jg9Vpx0hjigtWKVFr+aCWUjCuJTZNl3NGMRZCNEJ3PhtLyS2LYGxOV5NVsu72Bu98/Lio+Nlq8eTk8FZ/2/PDXmc4mk7TNN3e2TMd128ZBEGMseBVutloJVqBv9UNt7a6s6M3CIFkExNCXNMCACjDK8vSIjbSCCGtMK5EpRmyDM9xHK8VEGQzxnhVAuXzyooWwqbtbLOAkCKEpFKQSJNSghAU30Ua1l3CNYfAMIw6PL12Gq0xjCRJoGlKrhDAJsVS8rRIlFImpZJzRKkUlZJMCkEw0EqxKtXAwFo5nksMzFgJoUYIVEVCXNekulCl4zhKgc0mQRqUMSekJ4SwbTuO4263e35+3u1266IYxzGo5R6c27Zdx1s4TiiEYEJyJTWo6aUUIaQdw3BNRfGrs+ODvb1NllDTaDSDfLUc9Dqji8kmySzHL5iGECPLoSZg2bxgSkICiQZSQSUt2yJSAgQlQqUUXCgAMUQGUEJIaFETagUVMzGyKc2yNFosguFwb2/vt377H01mM68RCCECwzgbjUMaCiEARJZhVJyzPJdSE0IkwoBQrZSWAgBlEo0xxIBRpLHmtYMLQAAiAoCSGiJkVgpjASiBQAGpeJ12wYqi5o1KoTExFBCFgIBSAxNkmFryoNVSiEqIoiTzgwaxLW97Z7/b7VuWk8TZZDxfL1ZRtDYIDcPQdX1eCaEYxk7Y9Gzb5iAnVAItMZGEQNcOfM+rfaABxr3B1s7Nm3FevPzq6/KcSYw/vHsLY8irCgLFijKON4vlcjKbLeczk+LQ9xph+I9+77ddy14t50VRzKIojuOvv/5yNJ0YhoEInE6nTPAsy5rN5oeffAwA+Pbbb8/OzkrG9m/cOBudLVYriBG2DN82y6qKomi2mAZNTyUiSTaWTXr9zmVUHq/ORxeffvZrzjQE5Pz8fDjYoYbx9OnTfr/farV2t4eWZRNqhM3O9nbleUF3u+F5XsVTDQXnBRcFobbtkDyLPd/UQpZsY5iIGDxZrggJWFUli816vS6K6uTNRRJXvEDLvQlpWRamJjXMgD5/+s34nN++fdBttbf6nU4YKilsgxoI2pRUeSY4r7JMlIxVQHFmUgIts8jy2eii8h2EUKfTAqDOe3XiON7b2/vss0+DwGs0Gvt7By9fvnzx4sWdO3cePHjw5mK8s3fw/AX/t//zv9zev0khqpLVP/3X/0O71XUdhxXlzRv7+M6tk8NjwZltOstlHgRBv9+vqmo2nwshfN9HlHzxxReuF3iet9XvbQ8Ht2/eCMMQY6y0eHjv/mI2X0xn3WZruzeoY0HevD6cLxYQwkYYXvq6awURtuSli3vJxDULGmM8Go/rRqGG1KSUGiAIIbLA1YJNScmFEpRiAhFXsqoqJWTtDbe9vV1V1XKxTldx2Gp6nut5Xg05EoTbzVZZ5kDLXi/4xS9/9vid+9bHH4WBR4g3n8ywgu89eu/urbsvvnruur5lOb/4xS+///0f/vN//udho1VW3LBsJXSZFCVn6yq/efNmt9uHEI5Gozp+5uLiYjgc5nlJqfkf/8PP6jQNANRyuSxy5rkN3wvTJCtyxjlvNTuGYSgFlJJlVUTRKkk2zSai1MZEm6ajtS7LsjZo0/rS149zLpSS+pJZfY0xFkVhmjZCQAihpLRtR2uVpmmddt3pdIQQnMvpdFoTGG/evDkdz5aL9bA/GPSG5ycX9aWfMdbv94sq32w2pmNqqLzQZ0wAVAfcQaWU7ZgYoixLFnOdZ0kjDCGEWZFuNomUynEc1/Nqkh0mCAJUcVZVlWVQ13XX63VV5kIr07RLVlFquq57fn6Obbu+lF8z1Wu+57Nnz4bDYcXZbDbjnDcajXrfAbXK0wRDYBoGhlIj1et0H967f/PGQRqtn3397f7OLtRgdH6RJvFsMQ96fS6SvCzr3Tm13fUmWUUR46J2/SvL0nYs13Zs246iyLTdkom8LKEWlm20Wq1bNw+2tgaiYnmW5EkOJd/uDdvtdhEvnz17drEStmWWrNxEbL5eQYTvP3xQchbFm3pcpphwLqu8qHOnlASW5XiObVuO4IBpTggyTavT6eRHR+tVivAqz0qtIUbUc/w6dLSqqmiTgKv8J6UUl5cGGI7jQKAMy9EQVVVVFWm9CcIG7ff71DAB0rZnZ1mmAAQa1PwtxnhWlgrjfDa7DqGuh8v6bK2XX9eYAbjyDKg7gBqUumYycs5ns1m95KqXblVV1ckAdQBUvcWQUlzxGy4nY6W+s+Ko1xyyVGVZrlYrjGFVlJc+2UDV3EYhRA0q1JQLSmiNkYCrWKmat1gfeaPRqE0jDMOoAQbP8xhXhBCICdZKK6gh0BDVHAuMcZSlUrCpaUohqrIoy3wQhjUaQSktipJzTghECNmWTchaCMG4vNZqMsYQghB+t3TQECr9XXokQgi8BfUjhHq93p07d5bL5dnZWf2L1ApJ/ZsSBlgrNOB/QhL51pd///Y2HwL8pt4BX6VIyCtmdE1evjTJuHo98zyfTqeCVeT8bAYAsG0zbPZu33oAtCjKjFXi9atXlmW02804jqFWtkdM6vi+f+POA9u0tJTRas3K3DJM1/UNw7Bs52I6m6ynQ3Tw8L13tG0dn5wavn1+ftpsNj3XZawESgcNf2t7sF7N/u4/PLt1Y//Gzb3+sEsJ8n0/iQZVVZmO+8033yBKalcQBOF4Oo2iyLKs7e1tivBoNErTdDgcttvtNE03Mqs4bwbB3t6ebblxHI/H48VsMluMe73O9t6g3+/dun1js1mfnBxJrUzbWa43luVYpnl6ftZsde/evyOlrlW8W7s7y/XGKoyKQ4xty5ZSyoozSikxqGEQJgVjTAN5cXG2NejZluHYhu+4rkNXsxIqqgq1WcTzyXy5XE4v1kBD12obCB7s7jR9b7NavfvokUVQ4JnbO/2L0cmrF+e9doNC2mr43WZjb3srbPgaoMlkslqtlsulaRDfJJLYDkG2Y7W7nfl8qhRar9cY6TzPf/aznw37/bJk5+ejfqf/0Uef/OAHP3rz5o2U8OmTl9iGBHLb0KvZaL1eBGFo2t69g92iqP4f//f/2/np6ae//NWzL7/J0jRw3cOTk+H2bc6qlDMAQLsVNpvNWvj3k9/5R67rGoaxf+NGu91er9fn5+dPn3zd7na2+j3Oq3W0bAXhJl5//vk8iTYIwNVqBSGsmAAIGobRarV6w63Rcl1fNbKiStP0+pN6fbIBcEkFQphijDfZulYJAgTr/FxIEITItdyizDCmVVFiiPb3b7x48SxNU99366pJKW01GyVnrMwLVnHOe+2O4Czn+he/+MUf//7vzabjwPO//70f/r/+n//v9WLzu7/zB48fvPf5rz6/eHPy4OGjVtj+F3/+ryzLLuMUAcxyluJMKVVlSVlUdciqaZrvvPPg7t27f/M3f/P8+fMbN2794R/+4Xy2IMQQQi3mq6Io/vk/G1uWZVu+bflSSk44AGC1iuoLk2EYrmf5geO4JsYQE9VsNjUEZVnWjuxKKam+01m9TVkCV179QRAghOI4poR0ui3DMIo0Wy4XmyhxHYex0jTNNE0Nw8KIQo1cx1dSzmaz/Z39RsNP42w2GfeHg36///z506xIh/4OxtjExmoTQYg927FMahi0GQZJsok3mypLEUIQ3iiKbLPeTJcrAKBUQGOCQaV1ZNmmsk3HMkzLEkpWFWdSiESYhrFYr6J1nCTJgwcPDGqdjUYQwp2dna2dnc8///z4+Pj+/ft5WdRQ7Xg8TtM0iqKaol8UxaDTXi4XgldQal4UvMq0EqZpvnj6rCrzV69eGZQCqYqyaHe70+l8Npv5QRi2OxUTWuvNZhPHcRRFGsBut1tHbIRmIwiCmksPBQAAOK7VaXcankcJWC5WrMhv7e/dffTO0eHhbDKp0lX3YHur469C+/mbo5UEfsM1sOlZpgB6dHEKEEaEYEwwRHUTVhSF1ppSqqU2sGUabsVlvIyyLKEG8X3fMD1quI2wxQr9zVdPkzidzWZhGLJkzbRkzK1rIUZAA2ybRrvbMymBELKqUJybBqm4XEWbW7d3EUKj0aioOKWQi3IdR4ZpaQzzolIKYEIUJVJJZRim76+mF7W6tS6o4K0Y4ustwHV9qgtefbfalLPeMlwrR8IwrANN6odXVQWulvRKKSkvjRCuC9jfo1sCACzLqqqq/rGmadY1XrAyyzLXteueBkJYVWVRFKXKQ9OtqaONRiMMw+tjzrLMdd16eVc3GYwx13WBljWMgJSsVG2XpCRCZcUtamw2EUZgPp0GnluWJVCSdNrPnz+PNymEsF6C+H6j1emVFXOcFSGECUUpRQjLS3AFvKVJVAoAqS9TMQEAFCMEvhNPUkoHg0G32/3iiy+eP3+epqllWUVZ1lJYXasi38qUghDqK34ieKtRkPIyvOO6b6gfe32heKvbQODqCevDgEBcv/gQQqAl5xwoIYXQEhVFsRQVKzKCqZUkSVaUUKtub6vXbSvJXce3LKcqc8bKalEqpRBud/rh1tbW7bs9kxoEojRpxusIKI0QLiueFFGr3VCYlizfZGsvsPdu7w2G20qoQbcDIXxy8ma9mN+6eYCBzso4L1Ni4rDbgBokyYZtFnESa63T+XQVLZVSRZZDqAlBQGsEYbvV4hV7/fLw9PysEtzzG0XJNln+5ux8s9ncvnXr4PadVr+9tb892B2+fvm8qop2u+k1fYShUIpJ4QaN3f2biIqg0bRt17Ls6WyVV8XNO7dtz/3lz365ipabNBlNLhoNXymZ5zlEIEl5WQEpMaWOZfiB51RVgTFG0tbcwoZNobmexlXM4lVGobmZRZtFBCplAtpp+rbV6Da3tvt9AuTO9sAm0d1bNy0D2RZtNoLx6BQCvjPo+Y0gilOkxfagl2dpxdnBzlBrHW3WySaWkksuKIaNRmO6XiOENpsNJnDQ7ziOlaTrxWLheZ7r+kmS1du+yXjh+1WWpnm1sg0a+u7jB3dWcVwyvt1pPrh1YzKZBQbe7nbQB+9fnF78+ptvIISOaUt5UtOOTNNstVo2pVrxTbTc29vb2dlBCGTJ+hdPvx6NRlLKNEmChl8WeeC5O1tbrGTL+UILmec5UBoAVFVsk4wRQu1eN1CgVqPVDGTHkxDCrMhrXKHOxRFCCH6ZcIMwvdqfAQAAVBporTTgTGIsms1mxYqG17ANGwAYeL5g0nP8XreZJFnJWRj4g15/Npsl65UEGmNsEbKMVgYBgpWMl1orgEAcp8enJ2fHZ9Px7L/8z/+3f/KP//Rf/vn/uEqSra29e++8s5gtAUBKSCE1ywrbdksNF4tVHEdBw+e82t3ddRwHY7xarfr9/mazdlx7PltwLoVkoROMJ4tGo8HYpc2cUsI0IcY4z1OlsO+73W47L2IhSg2w51vNZjPJ0qIotP7uYlpHDQEptaw5zLCerjQAWkMhlNaSMeZYtucGrVZLtVme57YNZtOJZVlaXwrWnz9/jhG1Tce27eM3b0xiOra1WiyllEWebu+87/uuYdHNeq0htCyn3+lHaWE5rmubge9aBGklTAL2d/cmkwnUvCwKKaVl2QDikgkAaVGVUbxxXbvX7bTbTS2NyWScpqnrWK7rQkTSJC/L8uL0wrbdra2t12fHZVlub2+3Wi14RcM2DKPT6fi+P5lNCSF1xk9dim7cPBiZdLValHmiWGUb1KIkS5OLi7OG59++fXu5XGZx4jjO7u5+nGZff/bsJ7/3B+99+NFguP3rz744OTlZbxKlgZDf2fjU5LskSQEA3eFwE0Ul42lWKaUUK5WsbIothO7eunnv1k2RJ5PxBQJ3Agd+76OH01gKoXYP9j//+qsyy7b2hovl0m+1DNPSGkqhGGNVXiilHMv1fR9oSinlUmZZNltGURQBoExztUnybrfruc3lfHF6dJ5lGUGo3WhVWAEAGr6LMZYKcM4RppZlFUUBgO1YZqPRsCzLdxxKKQTq0e3HhJCXL19PZ7Og2ZrNl3K5QibN0nwVb0rGDdOSUudV6SpgeB7CtJZhYGIQanLOueBS6qJkNZGwvmGMEUaYwCLLrikI9TtSj541gHcdcQkASNN0s9lQ171GI6T8blt/vbCvz/or1wFF6WV8GqVYMF7fTfKqdv+ry6BpmoZBAQAYQEJpzfEMgqDRaNTNd62GqJGPuj+rwcU6uQpCqCFUQkrOGecAYYSQZRmEEIyQYxpxtIFaScFqjPbi4iJLCzfw69Jety9FWdU/i8tLHwINQO1WAN/SEVylPIJL7R64dPKunyoIgq2trcVi8fOf/3y9Xn9nIIFx/fV1lvRb8gR1/cRv/aDfMHf6h3DCdzxHiPQV7VEppYUAGl1LSzDGBAHJK8EqraS2KNRSKSClJAhTw7CEYFEUxdFKa3jzxv4nn/xgMNhaLWZlkezs7ChRGQYpiuJ8fIKMCEPkOA6SusgykxoIkXgTb5JkuL0vmfzm68+PTo+6gy1qO9PRmShkHm1Mk87Ozl6/frmajRuBNxmN9vd2tJZnZydCcM5KzvnR0eF6vS5L4dkOxSTP8zzPJeO8yImGWsjx2Uhu6Z2dnayonr548fTZM8MwiONixp1G02s2Dc+zLVsjaDf8zSheJ2m7FWqtFSKcw4qp1TphOkeGlRUlMexGq7FK1mcXZ91ud7jfj6PNKp4yxoKmhTA0bGhZthYBY5yxSgloYNMyG0QxyzIGTTq6OM+Ykol+/uzEdQzbMCwIeK5kqRpe6FoOpWYz6Az6uwd7u0lc9rttA5vtTtMwSbMZGCZarOar5UKKvCwgUDzJkotzfHb6WgOUZVlNyfE9gxAbKCUEMyhQSgVBsNmsKaKc882m8jwPY2JZVq87WC2jL7/8crOJlVJ7ewdxtAk75maTpGm6WqyIQR/de7B34wAhcvfmzePDF77X+OSD99uNMFovT96c2Lb9/NmT3d3de/fu9Xq9OuQaY2xT9O23X88vTmotsmGQYbthWdZ6vT45Pwt9b9Dvm4bx8umzZBP3O7042lBq9vt9BfRysdYIIkwXq/XR2XnBq0ajYZomhJeZcjUvSWrFGBOX4ngJIcREKaUoolJLrSWCiCkluZCCcVa2WiFQ0He9VquluMqTLIvTMAjSNC1YaVnOzs7W/u7O02dPWK6aba+qytV8QTCeTMTu723/x//4H3784x97jiW5fvjeu2cnp1988/Xe/v4/+ZN/8tFvfX89W82S1Y9+57f+u//unzLFCKCGSWoPOCn1ZpMAABHEFxdjxv6j69o1RSCKop/+9KfNZiiESJOsLMuiyG7dusU5XywWdQuf53lN8242m0VR1JeYLMvm83mz2awv+hWvEyY1hlDKul+44tXXeOYlVVsLKQFUSZLUzwl1NZ8vOee2ae7s7PX73b/6y3/rOt5iOddaVxX/8suv//M/+8On374gkPBKCM6bQeNF8qzZbEpe2qaRp7ECKM+zW7fvQo3m00WhdJ7ntmlIEVKoo+Wi12198N7j56axWK3G52cCYGI5EBMNFcJEKVRU3LZtqZSU9cu1qV8913Ul0LbtbG/vjsfTpy9eeo1wd3f3q6++Wi6XRVUxxg4ODgzDcAwXYzyfzyeTSe0F53kexBgh1Gl4qmoiWc2qrOFaN28e3L1zG2lV5KlFyQ9+8IO//fd/u0mT5XpDTdsyDNd1wzAMw5ALhb/8uvbPgRBBCGu/qfqKWZZllmWUUs5lXlRVmUMIJTd5lbEiw1AO+/2jo5Pd7YGCIM/ToszHs7Hv2r/74+8/fPdRXpST0fH4IiVA/PAHnzTC9vNXrxmXDAiliG1atul4nhf4YRRtqqpiWZqVhZCKGiaTImdiMl1qhaPVIkvSOzcO7t++QxHutJtChJPJpCryMs8RMYCUCgCtYZ32KaUEwLEdgg1DQyiY2OSV6xrIdKwgtJyAWIXlBWGnS+xCEaMsKmJajHO1SRQEBassw5BSSgAqITRCnHMupVIKGwYAQNSSNq3JVYG/mpO/0zvU/USNGdSKwRrZrkmRoW3Lq1BppcQVy49c0/2+m3SvtirXps61yQeEsNY71FYW9R3qh3i2o+vmhnN9KXGMy7JUStWinkvTCEJqEiWEEGkApJJa10CdQQjEFGPcabUYY8J2CEHzWUYQ7LTCg929Vqt1ydjNsrKs0qzYbJLZYoUJrYXHEMJL8F5rABC8DHSAV70QqkM66l+KEGIYBEENlKaUmo67s7Pz7bffvnjxwnVdBQEhxLSsOMv1lVBZfwcAfAcoaq2vHKF/I3Oynqr+3iZCX9pRIAABhFC/FYGttIZAXqNHhBCENNQKQWCY1LFMiyKMIMWQLFdTy7KC0F+vl98+fXp4eAjAbz9650HgNwkhrm1KUUrJknTz6vmzaLNaRzBPM9MwkAZVmvueQ7ExG0+mi2UcJVKB4/Mx1/rg9j3Pb8yWC5Zr17U//PDDna0BUaKscl4UBGjXNLJN9IYVaRqnaZxl2fnFKWPs8Xsf7e/uKS6KPBcFD3zPIrQWAhGEBZNZVpRCFHm5WK0IIbGJuKIC0lVaRumF0kxU1cV0djGaClZ+8skn/W6v2+0iaOWlOjqe+C3DdoLD8WGS5a7rpnlyfHFouui9jx5OJiPHwm2rGbYczjkuMSE6W0shtUEdrSGCNoX2eLoqiqLVbC4mqUFRw3UM3QjtpkVJYPY2cCWEhgALoQjRGkgAVZxE8aaSQGIDTedzz3cgwXGWQIIdx3j28htCzWana9nOaHq6jud+EDKZb1IJAHBsuxu0CEFZIiqWtlotjKHr2ZQSSqllGa1WmGfJ1tb21tb2y03S7Q7ef/+jne090zR/+fNfCVQtVwXn0rHDRjPodfqBGyCEgiBwHcuxvSjarNazn/zkd2bvLF69evXjH/12zYSXUq6j5Wx0Eceb6WxcleVZtFgul+12896dO1vdru/7DgXQ9m4c7G0NhmfHZ5yVUGvbsTbriBDieC5nksm5kjDJiqqKN0mswGWQASJGFEVM8Jp+DxAkhACIkbgk/BNKEUIEUaB0jSYAqRVXTDKI9GYdAwQBQEDB0flFyaqLi4s8K4ila7/6RhA8fvedyWRS66p5IUzP0EoMegAjNJ1Oj44Ofd/3/PDZ4bP1atUO23/5d3/lNNwbOwfrLJ4l89uPblPPSKucI247VlHleZIMutt5nhOKMKatZlsp8OLFK8ex6oE4TTcfffSRZTmLxaLmWPV6rfF4nGUby7IoJYSgWh0XRbFSKgwxIYYUME1K02Cm4VZVVevZOJdaSkKIAppxyRhjQgil6y3M9axAsFFHxRBsCCFGo9F0OrUN84MP39va2tnbO6AUX1xcWJYFARJCGph6tsuqqiQkidbb28Mw8FeLueXYSvB+tzdfrsqS2bab5/nh62NGsYHR9lZ/iFpaMaClQRBQ/NaNPaWEkrxgDCugiSwrAYiBheKcY0orITZxiok2bPvG3p6S3DDtaL3JaLG/d8Oy3VcvX9qW84Mff284HAIAXr16NZ1O66Hw1vZtpdTR0VG9G261Wr1ebxVF4/G41epAzihUUDLPMm/t728P+uPR+dnZ2WI2f/TokUZQaT2ZTYuqfPz4vUePHkkpP//88xrNajQahuVEcdLp9uqVcO3fhyGq/7lYLJJkcxmcyAVnZVWUkpUFg/Mo7W9h6gQMGReLZam1LNnt/XB70Pr6myd/9o//+J1HZ18/fTrotLkGVZ6VXAgJIUCmaSoFBFdJkqzWcyllxQSTAhNqOraJoBACaiC1qqoKKOC73na/59hWt9U2bZXG0fHpeZYXze7AcRwFIeMSUwMAIISouGCMpVnBGEuTzXS18H2/TjqQRFSSGG7DtAOpSRMZAADbtqXQ601UliUhBLKyRq2vQ5jqIlfLMq9T0a/tk+sdELiSHtTNSt0H1Ah/rY+rUcN6oXNJUZJSqUs7B3nlAI3Qd37MCCEItYXtuvMoiuzSBNowag5BTZn0PK/f75+dnc5ms5TEje5QSpnnue/7dcdQ1+faQKlG12sswbZtrTUCUGkNlEYAWoYJMcKIAowYY3macs7LnCkhHcva39/f2909fPV8vV4DjZIkqSomFcjzLM1L07IvdRzi2gkbQYjfnuYvv6PBNSfgEkGRXHFhWaZrO67rPn36dLPZbG1tG4JzzgGE9VuDEbpGFK57grdXCVc02d+wtXgLx7hsHb5bgyB8fR/4m1qJa5klAgpjQgl2bTPwHKiV4ExyRtzAAQAYBnV9F0L45vgYIXR2dr41GNgWAS4tqwxo1Wn3zcc2F1UUX5SFlBICrYRQSkAlZLpOFufjpu3fvf+QIvy//OVff/PZFwc3bjiOt3/wkCrtINwNAiREXmSLxWx6draOlgqoMAySdDMeX2RZZhjk4b27XsPTUM2WM9d0XNf2PK+E2DadsixbHR9APJ0uBAaO5/teY7WJSmR7QdttdCCxBS8QsgY7/b0bN7787LPPPvtssYrTpJrNI6gRxi7neJPkQdi2nGmRpwhjqRiXJC83xBz0txp5lrR6NkSVEDlXaZpUeUIpMQI3VBKZFkJQj8bnx29O7ty6HXp+GkVVnO1u7d462F/MJw2/VcJhkiSUmsvlMk3TaB2v4wLBU8Gw7/bareHR+XFRFJPJuCiTTqd1++buxcUFolSoSmOrKgvTJsgAP/rtHyiliqIgEHmeU5VFJQoJiEgZY+z+/ftaK0rQ9vYQABWt1k+fPNcKnp6eF3mVZ2WWFb7fvHnz9pPXh2kW39jff+fRA0LQaHL25MkzyzK2trYqwZ/PnkNEVtFSIeg2/EcfvJvO89VyvlzMAABVVSIAKUHdMMzyxPd7w24rzZJXz765ODn88MMPb+3vmalohkEax0m6cRwni5PValW7eGkF1/FmvYmVBElWYEo8z4vTTT0WEMOqqkpDoJSquKg4k1fxvugqUrY+FbXWSnGgNNQAQngF3QHPceNo8/r167IspeSnp6eHr9588MNH9eBOCLl58+Yf/xFsuN6LFy8meuR7/ni2HAyDNE0++uj9V69eKSA50qlItAEitpm9WX4wO33w/jv5UVmk+Q3j1rsfv/v1F98k67hSBrawbVi27TLGtNJxHO/v77qes9lEhKA7d+6Mx2Ol1GQySpK43W67rtvv9/uDoZDlt0++nM3HjuNaltUMmhDC09PzIAhd1yfYRJBapougmcRlxZXl2IZhSFkBKQkhAEGpmJQKIQQvLWvqtXE92CFCUH19EJfbXqa4GI+mQohWq1VfQ2vrQM9x1utNq9V58/rVsNfP83xne/j4vUeff/45Y8zznP/Tf/t//PyLL//yr/79t19/E7ZaW4PBrCwtk/b7w8FgoFjuUNT03fls0m63h/3BnTt3pqtNVkkOqIQCE4MVZV5UjAmQFoZBfdemlDY7bc92Vsv5JonTOGu3uwBAANHZxfmDzb2bN2+Ox+PaJXcymQyHw9qIwjTNwWAwm81qGecmSdI0zeM1u5o0LMtst0Lfc86E2NvZdRxnOp1algUJtj13vYlXUfTJBx+u1puXh288v7Fery3LCludIGwGjXCxWOirlMJOp0MpXq1WrWYIgJrP56PRCGPsO65j2XbgFgIwbYyXSaExR3Selh//1k+Wy3kZbb789a+iePPh977/0ScfuoF7Pp6lBQNaScYZ15Qa1DCB1mVZpnFKCaCUQAxkpTQEmGLDtCCElmkGnmdiVOUJ1OLs9KgV+MNWEAaha1uubSJMGr5rmqZGiBpguY7qk8W2bWpatushlJdlmXBUbJjW2HXtCpja9CwXK2TGaSSlQghwyCGEFsGAYCkFgJgahta6KAql2HUmk+M4mEiICETsMo1Ja6CBYZDr8xRC+HYmcp1JUS8uayUkAEC/5T5Uk3BrEOJtGgT8LnkZOJZz/V81c4JSqiVvNpvNZtOgdGtra2dnh7FquVzqq6yH+pCuj6EurvWx1UuQml5QgwpaSKWBhhADCDSod7XrKmeMGQTHm00tkmyFzdls9tlnn3meN+hvRUlMCHEtRykA0GVs5jWmgmpfRogVqyD8bvS/LsOX7EWgJYZSCC1lrRDJ83w+nxNCanyRMcY4v+qiLpmP18+jlCJXDA905Zeg1H9i73D9kOtG4XJhAS6xhBoEusQqwHeu28SoOxSFaztIJRFQGBsEYTWbzZJ0YxrmrTu3MMZxHH/66ae2aX380fs7Ozu+4yjNwoa/Xi+jaD1bwJIpxzAt0zAgbYcNrMHCnbbDFitKz7I/fvTe9Hz0+s1Ry/Vc3xt2O5zzyfnZyeHr9Wa9s7uVZcnXX32htYJIC96DUBuE2u3m7t7eD3/4w4vFKk/S4+PjdqMVuj7Lqvl0bhjW7u6ubZjt/qDd7882q+Pzz04vzgGC3YPbYRg2wnbF1Hq1AVB4ntfr9frDHcP4FiNjFcWnJyPGBNQQY9zoGo3Q397eWa9XRZkIIYQsp4uxfl72B504jtpdp8hyznkliyiNPLOLkYa4ZBUXyretQOmirJLBsHPn4ObP/vZvn7960e/8yKAoiSPWbnzvBz8ImgEhwLBpkmzyvJRSRukqDAbreGU5vtTq7v17tmcenbxptJvUovs3972g8ezFSy6ZF7hhp82EMCzqOA7GHQRgUeTjyfnJ2XGRp5IHCKHvf//7Fxfnz55+ixBYLBZQq3pPYRp2rzsYDreVAlEU5Xnx6PFHb9680YCuVimAcrWKp9OpBmo4HDYajS+/+mpv7+DG7ZsV4xqokhVAy5oeLwS/ODtPsk3Dczvd1gC12+227zrdbhsCVUcMFEXmOI3NZnN6dCy56vf7q/kijZPbt+6UJSuKojYBlFqneWZbztZ2k0tW2yEDdGk1mGVZFCde4OO6lTVoPeIwLjnn6BLZk/UUQjHFGBOKGl7Da3gvXjyLomi+nNUGNScnJ7fe2bFtF0IohLQoeXjvPoGo1+t8+umn4/EFAuDpN/G775q3b99Osvjw8PDboxduw6NWBQHmIvniydc7ezvUNUeT8TrffPT9T2az6Ww+QTkIg0YQBNE4siwbYzidjYMgIBRPp9Mg8LTWk8nEdd3RaDQYbN27d+/o6CRJkvc/fG8d7ewf7MznS4SQknq1Xsxny3v3HoRhK2w0i6JKkgxCSohRFqxgG0xJ7Y6gtSYEAwQxlrZNEOdISK117ZpcZ8RcmzYyVkEA6vw9pNXp6emLFy8++fjDNE09LwAAvHnzWnJu27ZlmFEUvfvuu2m8SZLk1q0bvu86juc4zo2bNzWA1HT/6t/9r3Wybac3UKIUQkymU1kkjoGkoGfHJ8eHb9q9vm1btlVt8lgCJJUqmQg8L4qivCyEwJ5vC0E3cbpYLK2hkaQ5hJgxtlgsbNseDodJkozH49u3b89ms6qqut1u7bxbLwgwxp1e9+joqF6EdzqdwWCgpARaOpaN2h3Pd4f9waDXn88mglc723svXryQUldVtb29PZ8t6slMSlkUhWHaeZ4bjtfr9RzPH0+m9V62nlNv3LhRlv3Xr19vNgmltNNuV25wuTBWKi2L0/OJYVm//OzXtomTdG243h/92bvRp7+qpss8T+/euXdy9KY73P693/ntf/Gv/sK0naKSOEritKTUsExbawg1LGSxd7BLDLOs2CLaxFkOCbVdxzDMdjMkECDNQ9+xDbqcjIvNcnfYN11CCOn1ekwobFhSMIWIaTnNZlMIITXE1KxLe5oXURSBYJimhdZaYUvqqip5UVRpUYbNluBcVJWQDCiNIbKIwTVPy6ymBxVFUVVVDRjUr5u+chOHV5aL9a3G8+vvX39dlxkhBCEkCALDMGopRG9np+4bEEJKoZoxUP/Eq5L3nQWQUrreB9VPWMMASqkaTkjTdDadUkobjUYduxD6AYe4Nga9Jj3Ux++6bh3uUBtaSynrc4rnXGtdezJyzYWSeVEVRYEJrCkXQohWo1EzuD/7/NM0Tff29g4ODo7PTuM4oYZRszI1gDXiUhtXY0zqYoy1BlfO09fV+vrvupUhGEKqgyBot9u1iMP3/aIo/LCBMWacB0Ejy7Kao3ANAFxCC995WwGEEATq7ef/h7fvtgyXlkqX37l2qkAI1SNHPWQY1AIACC7KUlsEYgQNgm3bJo6HnYwSSoVU0iDBYBgn+fnhEeaVeTY1e6Pbdw4M4p3FsWkF2w92Taedp79iZVFJ0u/v2haZjy/uPP4IWE9t2+ZEWyZ47+PHnWEbU+L7/rBhNDvDX3/+2SrexHn281//dPdg/52P3z09P1+uV9q2EMY7/WGn1fr+979vGMb6IrOp0bF6Igc0aGLqKiqCbi8TJB2v50m5u7+FtXaQfOdgeHR0ZCkWjU7XJgwPdkCeLmaTdDx6/ZWVl4Wq5Gg04UJJDebrxLBsADjDdBXFv//7f/jk6ZeHr54Tw9QARStWFWvJCQRgdJZ1mq3FfBrHXGuH88bOzt679z5Yr6Ojo6PhoPHo3o9PDzfzeeba0e1HH1A/iBSYlOzVYpVSmiDW7Q7CMPj26RE1umyzLAr13/zX/2fPbWYpv7gY+b7n+ZYTwIPbnaoq281dhMByNaG2scmWLavlNzyC/TAMF/PYoPZoMjs+ebNazR+/92i5nLteCyHwzetv3xy9PD09MabI913Gq513tvYf767E0g+8jYzLimVR9vTps3RtIgQMg/hHh3kROxbd2e0lSXw+HlGK7965pbVeTEYY01evXnW7/X7Q5VIYWCTRCiN+786Nbr/f63dOT0+JQTMhSFn2BwMCwOHhYVEUu8PWcrkUeYEpNSm8desGooaU2nCNKFnnZWYYxELIRy4AYDa+WCYpQkjKypGg1WoxzuNNorgQjBNCtGS8LDSso8wJIVAqBJUmEHieL4FO07QscylAVSpKpOd2hBAGYsk6gdB9+fzihz9EDdPH2ImS6M38/O6923s7+62wvb+79z/+y39hW7PtoTYgqbLSxvaTL540dwajeESUSQ3MaXVxcvw//Pl/79p2mZbRYvq9j7+/dbBzcnGOMWEQRrziZlnpEjDt9fx1Ho2ejBzT3kS5bQXxpjg/mw62hmo8w6b76vUbqZTTevH9j3/w5mRUsqcQQoJgWZYPHv7w8ePHk9k8TrJnx08SmQIL54SnMBOigiXt7W+Xk9lyNrENT0plW37FheCqqngQtqRWaVYQy2Y89kyAEGCMacUQxgiIIkvjOO602hDyp8++IYQ0Az9NU89BrVZjcXF8e3/4H/82kyJttp2//uu/+tM//dM7d/cWi8VideLNsOTrP/79j//Nv/z/UEoRIvl8ub29rYr1q9Gb3rDX2TpQppmkeRCEn79+cxZVCjhhy91EGaxSXqSiDYmpMQVCcmLQIGwdn55+++SZFHodrRAAjucaFuUYNDthUkTnq2V1hLTnWO0msSyDgtb+1qvR6XI+C/3gbHSmFZOswJL3fHdjW0Tb+1s7v/78U9ezDQEnJxer2Xwxm1vUOj09JcTIi8yyvSQrLTdYrNOT8YVlWb/7+7/7r//NX2Qs1SZYrC4aOszzFQAlrwQF4mDQ//DRo3gdXbw+EoaV5JkQlRvY9WoyyzO7YcTl4q9/+uw/+9M/ipbLKt74CP7P//S/54I5Er1+fRgtUoCx4rD1TrfjhX/113+zd+cOakCMccmUhJzaduAGrnRNP4QQWthqQ9MyciWBQS3TsG3gKqmajR1CkJQyMEIAwDezJEJeQTrCdO3A9DwPY8w5r0qhFSCQWpRYmFY5K9KyLEsEDYQyyaNOpxNvxt2d3TKO8/Wi1+spXqZxDBAyLLNglZAaW4a2qI8JAKAs86riaZoTwlzX9bzANO2aqG+aRg0e1NN/kqwhQrX7pL7KE6rrWcVYlucQQsuyoBBlVWFCOIcY2xgjCCS1COccAOVaXlmWnuNMxqN2u62VJIQApQnFgnHHcdbrtWEYjDHHcaDWd+/eRwjEURJv0tfl6yxOiqJAGgIA283GdDqHWguhvvn2WaPVWa83iNoKUtdzGWNFUQWe222GDoEQg4KoJM0cz81LpgVHGJdF4joO59wkMI83oefZlEzPzk5ePtdKGcgwkPEHP/m9v/7r//VUnZ2dXQAuhZDNsJWqolAlEgpogABEAEEIqWkCBIWQQiuIIECEiUtk1LZtg9TRnVgpgTEOw3C1XAKtWVnZplVmuUmoECLfJJZtKa01lwRhSACXQitNIAIAAKWV4BBjAABABEKpAeIaYgAkxLDWZyIiLlWahYGRhKgUEiBuQoQwwBJgLaBkFCDL0ABgLCWEyoYKSJVn2aDbidbLzt7eYjoyqDvs9ohSghqYGgRxjYCOk4gLsL09zPNytVp9+eWXVZHdPNilBrm4GH/99bdt2261WuOL8/V6WeRJK3CDwBdKvPf+h0kcMSbyPG+2W47nR1G0WC6/98FBu9tdJht/tcxZ1ex12r0uk+Lo6OjuzVumaU7Hk5s7e/du3uZZ8eKbJ4fPDg8ODt774PFoNEEEe03/3UFvHUVJXnDJLhbjF6eH680qitbEdpvDba21H7jUMvOiqtvGoiguzs+FlKZpF0UhFTBtr9vpNFstx/MXk9n2zrDT6d28eXd8cXZ2fuzYuNnweCXzTDi2aVse5+riYpJs4iAImj6KNouj41dSaAiVVAwT1es3yzJbR4tut90IPUKQbZs3buynWfzkyRNCsBDs9eHLDz5479atWwgaEEJKzcViPJ/PPc91XReittJunmeDfrcoMqn9XhGWF8nF6HQ6nbpOCCB1nVaWFnESNZshxsA06c7O1qvXp1G0qqqKi9J1XQAVAMig5sX5aLo3M017NJot5iuEMCFGnpdVpbTWGhhtI2yEw+1h75137knFnj39NklYEARVVc1mCwBAu91+/Pjx+avTRjPc2tqCEEZRtFwuhVKMMct0NAQS6LIsZ7PZfD7P0txxnPVmVVR5q9NeLBZh2Pze9753ej568uTZfLbkUiilAUBMijrUCkKoAZYKCKkrJnLGhRBcaQVRVlzNBJem9FhpqLU2TdtxHCk555xJUV+wGOdKqdryvU4r4JwTQuI41lpyzuvJxjRNxlhVMYTQ+++/73p2VRVJkhCCKDGAhoyx3d3dJNkcvTnTGvguMQyzKDOgRLpJ33DRabUbYWN7Z3j05kRy6bp+nKbbwy2T0GQTSyld37MMK62KX376K4ig5ft5xSaL5eloDDDKssywHULIycnZcrkWghmY7O7u/uiHPyaEfPrZl89fvIDY1BAJybP5Ks9z0zSLXHCmPDdshpXkQikGkYZaBZ4tpaiKVGNEDWhY2KwQoZgQYttu7SeIMYZaN5vNPM/r0CCMcbvXv3379mAw8DwvL6qw1dzfv/H11986jjUaTb7++uvd3e1nz55t7WzXw32el81WyJlQSlnYnI4nk8lk/8Ze6Afbg+F0MddKeJ7TbrcrIctCZFmuIUAYE0prT+jabN+2bWoa3W4XQ11xFgSBZ9tRFDHGkiSp8c8iSwfvPPQbwd/8zd9oXgaeO59OyjzNko3mzKak12n3Op1uM9ysI86K2XzFRXn/wd0sS4Rgk8nk+OxYSvmT3//d8dPn8/mSUIoQkbLSWmCMv/z1Z7//h38YNhrDXr8sGStKAMBoNNIKep7num5RVH4rjPNsulpwoC4uLhqNRrvZFEppJTzXBVKs1stbB/u3d7fDoDE6PtFCmtSajsZlkb1z87YmOM4yiEjJhFQKYYooEaJWElpCMQlATdoVUq/Xa4wxRhQA6LougsQ0bcOw6g4AAEAIJoQAcPnxnkwml0y+BjsAAQAASURBVMo6StFVgILWup5E33Y+qO1x8yohhIxHozAMl8vlcDgUjNUCSIwxvkoThjXmLES9a6gxfHAVY13Hm9V9ALqM65RXEBdRStXM/+uptFYuvD2k1qNzrTKglBqUKowwxkoJVvIkSTCGSsgwDIPAy5KUYFLwTCrY7QxraoLneYQgBIAbBLdu3UJAj0aj1WpV5UXNWBRCGIaRbGJWFphQQpCSosyLmnAthGAQCMmv95j1MquerYuiSOMMEuJ6nmVZddArxhASRQmyHRNqhZE2MFGKr9frL7/8siwLKWVZllIBx3Hrd8RxHABQUVZaS6VgzcGsTUdqpB9eOcxev1MIIAR1nVpJKa3XOtdz/29QDQDQGmhwbbn4HWzwn8QP4Fu36yd5+59v3w1ojdCVPSW8XABd60QulzgmqVMntNYkjTdVXrBKaEAglHme5Hnl+c0g8DZRdH5+QSByLLsVNmaj5bNnz+7fHN65dasoslEaLRYLCMTO9nA5n5Ss8sMm59VyueRc9reGnzx4GIZhvMmxbXthkwHgAeA2QkLIeDppB61eo7PZbHzD/eTRB7dv33767ZPjF28E4D/9u79xXRdT2h/swJQMAlsY4NWL14gaSZoWvIg2GwV0o9VIEVxfnN69e9e27dVqtVrOm81wb6/f6w1evXi5s7f75s2bvKw4U93BcHtrCxNy/Pq4KuXF6cX+7sHs9juj83G2KRoeVRA0g75hkjSqCOXtsG8Sd7lcFunJYDDgInddF2EHE7G71+/1/9iyzTzNhGBpvsRYS1X1B52zXx9TAoLQZ7xoNJvUtBEh7VaPS/3sxYvxaBFFsTo6fvH6BSZyb3+ogXj67IuizLQWmCjbwat1hTFuhB5CwPOs9XotpXAc6+R0nj9b37ixHyerwzev8jxttVqEYFaJmr+zWsWHhyfDwbZWGwAwpWar2UGQtMPd5XKZJBvXtQGUVVWt1+u8SJeL9WI56/V6AADGWN3Cf/HFZ4uz1d7+vu81irzK0qJAVVZU6/VmMBi0Om3bdrMsW62nm80mz6uqkvPZcZ7nt27dWa/XAKOw1VquN6ZtSC2CwMOGmaX5ZrNhrIIQAwQVJhBCLpTkFS0LDYCCAGCETeMafsQUQ4NohGqa0rXCWyNYay5KVtX70XpzcU3DXi6XtQc7Y4xSbBhGfbXFGB0fn4aNVk2/z/MUIeJ5gUHt87Mj33P2dnpCsDAIqqJMNzEQ3Avcw9fnm83mz/70z37/D39vPJpOx7OTk5Nok52ML6AGnuM4lh3nuQCg1evGSSaESNK1T6jperPFfDAYlFEihY436XqTzOarg/2dBw8e2AY9Pj4eTWevD4/SNHcCUymVFZXUACAcJ2VZyE571W23Pc+PVuvavoZS2m21NOCLaA0BtmzXNhAzke/6NUIOIaz99RBCjhdUVTXYGtq2naVJVVWG3X/n8XuDwaBajqlhvfv48c7ulpT8l7/6xZMnT773vY+//vrr9XIluRCMf/v1V+vlCkJ45/bdZ4eHYRjeu3Xwkz/4fQAABCperxqt9mwyzfIUQogIlkpxKRTQhOC8yjSCcZYihEpW1ZnUBa+q6XQ46FmuQ7NsvlyslyuMMedV6JgG1oNO0yJwsd4EgceV2tkaths+r1hZFK3AMxDIs/irLz87PT42zIYf2D/40SfffPON1lZ30F2u19TAlJiGYSGEGOOMCc5rhx+9FQYGwlWaP7x33zTtz7/6su6J3xydtDpd03U0JhLD4/PTk5OTXLB7d+8yzqXiiHNCUKPhdwLfMcjk/OyP/5v/+g9+73d//NHH0/Hk3q2b/9O/+tdFVn7x5On5+Xmz3W22OobjEcsxHNf1g3W0wdQAAFJKKUTUtAAiUkouqrp0EUIxJhhdMrWv9W81+RxCrZSqmNgkMQDAsixiUEQwvExaRkVVaq01BFIrJTUXnAnOpUiKJAiCPM8D1xMKOJa1u7t7dPiGlRUimFIqGK/yQkGg6i2+hhhjBXSdWQUhrJ9TsuqSq0+IEIIJrrWmNTtQK4AgggjXfEbOueD1OaiAFlJIrYSqd2SQM4YRUkoKISiGFGOBgJQyDMKKlQAAVlZllXf8jlSXdAettWUZgpW26RdF4VhmlsQHBwe8Kia+O8mS8XhU5HkYhq1miDCWvMrKAgiOCTEN6jl2zciRUgKl6+IdxzGG2rIsi1qUFvUes057NCgmGAoEoFIYAc+xfNfhrNKiIhR5nocQePbsCca0qirbtiEi2DCrkl2+uZRXDAqhIFSwzneQ17YQl+sGwzDUVbiDAIoSVNdmQkiR55cUEKXqJU5dzpVSCgKgEYB/v8D/vW7g77UCf28T8fZ/XfYNACKE4BX7EiF03ShcdVSqplHWGg1MoFKC+I4PAdlESckz2zJ6ndZ8GVGMqqo0TQMqXebF6HzM85Iz7bthnTa2u7ubxCvGK8uyGOcFF5Ojk+3tbcMkyLBbTbfV6zMJT0ZTDeEijSfRqm7BZKpWq/nJ0UmzEeab1MLG44/f2RvuJKtNtFy3g2ZjpzOaTr559qTT73EC9WL8+YtvhNLEMoFWFVHQsKBmVVkmUsVCeoHfbLeoYczybLmKOJdAQgCAY/smsatSUGpyxlqNsBk0vvzmay3kYjqbTUb/h//9/+7Rw8fjs/Ojo8N4nbdbzSxleVJYJm23W/3ebjPIi7TClO7uDZst13EcwVVVVaYNBluD+XzeCM3VKlsup4ZhQC22trYajYZpoBs3bmigEEJFUYxGk7RfLRdpvUfwvEBrPR6PLRt1e608jxsB1bBOqyNWYhgGMSgxDMN13fqE2T/YzrIky6M054NhQ+mK8SwvEqcyHewgDIDGWgHLdMaj2f7e7U6n1+tu2bZNiOG6PlQ0jiHCihrYMCzDMNI0v7i4+PbbJxDCXndg27bvuPVQcnh4KFNtmGYQBHEc10pfBZEQYrFYOZ6PIDk7HS/Xq7pyr/OESZUW5SraZEU5fvIsL8qq4kVR9IaDg4MDQoznL14sVguIkWWZWutKEIQQgJxxkTOOMdYIQwJd79LxhiuJiAEx1QAIJQkAWmvLshuNRiV4lmVVBTAlhBAphWEYlNL6wgoA2Gw28/l8MBiAK0KvFJfM56IoGo0GZxxoRIkNNLFt1zTt8cXIts0kiZuNxve+90m72frqiy+ePXnqena741CMKaWWa/X6nVarhSlJSxEtV4wJ03V8z18v1kmWbtKs0Wl9/dW3UZwIjRrNZpJVAZPNbu/gxm3Ttra2djzP+/4nn9zY3/vmm29+9otfTyYT03bCVo9xkWW5UAATQ2NomW6WZfPZptPsNcO25CLP0jxPKfHaLd+xCUK65IwY0DSgY1LH9auq4kIqIbM8SZIEIeQ5tuV40/ky2awxxjdv0jwr8jw/PT2loji4dXtnZ6/f6756/eLg5o0g8AAAf/zHf/z68GUURQcHe1WZAyXjhBV51vZ9yzR8398d9k3bWm9izsrNajmaziqmpEYAUg0RMQwTYtO0sEQYgc1mAwmuu7Qyz7I0Rgj5vuvlLpMiSZIkS+v1dFN7ZyfHrVb48MG9r7/+Ks+zvb299x49WK1WaZyMz06hknG0LpJ4fHa6ibL+lm/ZREp+cXFmO1an39nf38eYTKeLwG8Y1B5PJ1k2t23HcZyyLH3XW87mugsPdvdMy5nMZ2maSqAVBELJrMhLJowiz0bnp+MLAIBRVev1mlDkeY4ULF6tmi3/zs29j/+r/+KTjz4IXb/yw+nFpCx4WQmp4Nl0WSkkEdGYlkwtokRI6AbNxWqtMcB1uLxhWY6tNSyqCmGglFJSSykBgEoCrWs7/0tK4OWqG2PTNCHSEiFCiOu6dTpiva2vvRcvSW3oMtO1nggt01RSBq5X5oXf7c5ms7u37yzni/F4bJqmNCXjvG4aFMZKyrxkNesQXLn6KKXqJ6/n4PqQ6mIGAADgMhRKa13TAGtFw9vYQy2RuIQupARSaiG15ABQwyAEeYQgADUrq7oOebbTDhuKs6IoBCubDR8hdHx8bBCshEjT9PMvfu27ttba81xK8CZax3EchmGrFd6+ecvA6Pj0DEONIcBQ51m6jladdldKWY/jQogoiqSUDd+1LMfApJA68H0NYJEXJWdaayClhMqkpNHw251WlkZlHpdFIpVsBo0oikzTTNK402lhaqyimAsmlWZc1vZxGF86PWkp37ZC0kpBjK7Lf125McYIXFqh1PyJ6/tflvYroeNvsCJ/8z7wLbeGv9c0XAdYaK2v/RxrMqXWumZBQqAJITWFFFzlRkgp64ahfseF4PVJKiQnRcGBxFLqNM2Bxs1GKITKsyLZLMOw5fkehiiKVgZGrWaz024jsIEQ9vvd0UUoWNVstdZpDBAKO123EWZZFucFdV1ETWxbgev96otPJ5NJkiS7W9th0OBSZEWutS7LcrNaP3jw4PF77y2j9cuXL8uyPLhz6/OXT2/cf5By0e50EMGGa79+8qTRav/2D394MZ6ejcYSaCKoZCU0cdAJLUTiLC6rvCwqxw8YF68Oj6qi2NvZjaLYpNbewb7W+pOPP+x2u7/4xc9czw589/T09OzsrNfp3Lv7EEP81RdfSA4W03mn1frhj77fbTams3FVVWGjJYFGCH7zzddFmTmOM5/PlZCO4ziO8967j/OinC9mhJCLs7MkSYMgmM8n5bPnEOrFcgohXC2jLOd59goC0zS8Gwe37ty9RQxSVrFUqmT8VmcoJCvLfLmcZ1mhFBBCRVHs+7FSsNttO77z4uXTO3cPgoZlO8bFRHb7nuNiw6AGRZblGYbFmazZNDU41usOtra2jo6OPvvsUyVgVVWmaQ4G/cFgEAZeVVXL5boq5Ucff/CjH/1ICJGnybXnySJeJUkynU6TJCuKwnV9wzIJNgihs+mSM7labtZRTAhxHA9Bs8hLjKzpbMW5Wq7i6eyz3nDLcZxG0+n1h4ZhHh2fAAAtx3IcJ89zUYoavWSMoaKowTcAQB3YWlUVV/Kax8s5D8NWTUoaDAYlZ0dHR0VRmLZVnxWEkBpmJwTV6YVnZyfvv/+YEFQvIJRSCJKsSKqKVVW9ocCEGBhT32vevfPwf/npv2k2HV7ySK5Xi9W7999p/6S1Pdw+PjoFeprmZZwmaVYUJTMMy/G9n/zkJ0qp06Pjl89fnZyduZYTBCEAYBOnRcX2dg9KzuaLdV6Uecnfe+8hlyqZLnf39nvdj1ut5pvjo7OLUdhqY2oIBZI0KyoGMSUYMSkl0xRaeVaen1+EzaDXbiEEKcWsTJFnIag67ZAQNJ0v0jTVhFIIOBd5XuR5JqWsqqqquEGxUDpdR+vlgnM+GAyqqnr6/Fl9BW8Y+MXLVxfnZw8e3PvX/+Yv0k307uNHJ2enf/j7f0AoKvOi19159913R2fnZ2dn44sTzwuINlq+u5nPbde5uBhRCD3XtqghJGeVEFpLoE3bsmxCLdMgPsKAmoZtWpbjSM6FQalpOJatEczLwjRNx/OEUkWWaS3LspzP52mafvjhh9Pp5Pj0ZGtrSwiRpmldbMqyRAA2m83hcNjrd7yw2e931tGCGijP809/9evdg/1OpzeeTBREAABKTMt0bMfp9/uGYVTr1Wq1slzvxetXZ6Ox63l5FFVV1Wy1AMYlZ/WUVVPeKKWbKuNASS5FFGnFtwadjz58/5OPPnz3nYd/9zd/++c/+2d5ns8m80ePHr85vYjWMTQMO3RLBd6cjxZp+fWr16vlehVtLMejhqUglEJjjBHAlWBlngMM6gkcQmwYJsGw1tTVTp3wirpbZ/kYhiExopQatoUoEVrV1DmEkNAKaFCPhhBCpZVGEGBkQANIhRCimLCyzONE7t/Y2dpezhcIQCUkvBpJCUQK41yIS4UipfSq3iilGOcQY6l17YfKa1mBUtTA5IrGWFWVUhJRQi2zPioopdBKSwkwQrVXoAQQaUwgwSbFSEqptaSERlEUhkGnFXa73TxPLdNcIaSkaPheGDa01vU6nyCgBEs38dnpiW3bCELLMsLANwh2bQsoXaSRlpwAWZWZlFJyZhCkNQVaAa0uBRcSqNoAVkOCMQJYVBmlVDCxWq0AAARCrbVBoWVYge912g2L6GS9iOO0qqSUkvHarElYtgkRtiyjDmfnohKCIwwRwhBhCAHCWF1GsaDLdlADjC+BIAgBpYRSAqTinNdg4dt4Ut2uqUtrxd+wS/qu+bhqIdRbWa/g7dUD+I0H6toGHl5SLCGCdbMCYb2AQkpdSiqEENSiCmohWZ6nZZlrLSEkSinyxaffNpvNRrM96ARcSjMvkiQbjUamSS0Du65JCbGo4TpGt91ot9sAhBgh07Ecz23wZhA2o9XStO3Dw0PDMi3H6QwGrU43LauT8Uxr/dnX3zx//hxqXTGxt7OLIfSCcAcbi9k8q9g6SddJGsUbYJqmYVQYvXhz1u5144Ld6g2oRf2GdzoZ3bxzY2t3mPN8vp5KAIsKEqoNigzT5DnfJLESkiJqYlMKaVne7ta+SY04jrXCWZI7nm1Rw7XsVrM5mUzanT2p+uenx2kctcLm+48/mI4nx28Od7aG+/sHW/2BEExyhQDUUnUHvYODgzhaxUlUVcV8PjWpMRz2McZpGo9Gk81mM+gPu80ewVavu3V0enx6cd4ftKlpE0LsihPD8rCbpXyTZLPlaqfaKxg7Pb/QWG1v94tcaKCzlMWbUivabg0s0zMNL45Tx264rquhWizH/WHY64fPnn3rN9AObnMuy5KVBcdEASAqllesuHFwqyiy6XRMKb1xc38dLT/99S+77V6j0WiErmUZGEPB1WQ8++rLJwjqQX/HdRoIg06zhTGez6dJkuA+hYhAiGuQEQAUhs1WqzWZzE7PLggh29v7fqN1cT5K4tx13SjKgyCI1hshBDU9gLiScBNnfsDjTUEMwQUwTNe2HIyIFEWZV5YFBZOs5AhUylB1o1CP/pcbO6GRRoQQqKAQIssy06S1XKo+86+I8UpK2W63i6KwLCvPc4zxaDR6ezaqJzCt4Hy2lELXa2lEsZLQtrwffP/Hf/fLv8bQ6HWaUbT65c8+bTU6P/rRj6zH7nCwv1rE0Tq23Ubgt6SIkiSTCgrBGl5jd3e3Ktj4YlSlRZnllWHnadEK27u7e6PRJE3mg8GOYzqKgTeTYyHE/v7+6dnF8+fPT09PWVE0Go1Gs5NlmdIo7Aw1gGcXY5bHjWZLMthsNjUQZZlnOcFItTuNslgzXpyfHg/6W77vT8ez2WTuuZUCIE94TdHQWiMNHMfxPM/3/bPTY9v1Oq7b73bOz8/Pzmc3b2zfuHGjZRtMKMaFBmg+W55POTWeS1nZlnHn1m2KyXQ0llIe7O/ev3fn+fPni/G03+/fuXOz0wxLyYskogSFQSMKEqaSdZIlaVkwKTWmpmkhDKBWXFBKNQQaAmqaZZnbjtNsNjEEleCDbg9TZFnWaDRSWlDToqa1juJ1FM/XGwWgAvDVm5PNekUQ1thgsgj9ABmm6Xq+6wIslWbzxfjdd9+ZzVb//qd/02i179x+ECfFsxcv0jT1Ah9CXKsndnZ2tGu/OT72uh3HdcfzmdBKSNnqdSsm0qKozfUAABhAirAW0u90AghWy7nW/IPHH/7oBx/fu3Wz1+n89O9++tN//9PDw8Nuu68xma+jjMmU8Tyv+v2+1jit8nS+nK4ixoRUwHQQwBgDLHTFuSREagG0ALZrYYwFVlpDQghGuJ4wr6uFlEIpBaFGCGkgFdA1ks+lgBBKXSv7QJ1LCQBQWmulpJRSSQW0FhJjnCXpztbWfDoLPP/09HRveyf0gzhLiywHCEKtJRcMMaUUlwJJDDECCGJ6KUTSWl3/XC2B1EpDACAECGoIAIIQIQwJBbo+jLqfAwhqBQCCQEOAoKqLGpNaKgh1zXtI4kgpSUm70wofPXpYVZXn2oJXaZpYlmma3VpgWVa5ZRDByqqqCMGddivLMkox1Mqx7MDzgRIIoeV82g7Mpu+UrXA0nk4mE6WRF4SDQWs8nderE0opohghZFsmIQQj6vt+WZaKy6IoJBd+4FFComhFfRtoxaqMFTnQwiCQYmhZRlUVjuMEDS8vC4xxxbjv+/W7XDCGsUIQIEQ0BForCC49ITAEUkkpZN3PXTYKGtROl0qLazihHniuJZEY4zpUWv+9Yq90DdhcpztcchquQIW3GwX4ViqEUgoCqIGSEiqF9ZVuokYULr8G8FpiijCQklesFEJgAiklCAEyGW0Ws2z/Jr5x40ZgEssqkiTBJyfb2wPLsnzXc22bYmJZBsYAYwAhqZNnCSGmbTEpqGlEm9Xd+/dWUfSrz7/kUty8fVcqcHJ2KqU0GxbTsszyo7NTxliv03UsC1uG0/Abio+Ws8+ffWs5tmGZF+NJlmWdrYMkSTKmciYark0Ny7DN0/OT8FUjiiKtK4yISbRnEdsARZHbXoAQyZM0TdOIJUWcdVrtTrsPAUAIVxV7/epNf9D+8vMvmu0WBroqUlZmNw92ORO1I+TNGzf+0Y9+bBn00cMHN/a3OefjiwulhOe5q+VsNptXFRP80i9svV4P+713331nuVzPZovlcu06Xhh2PvnwEyXB7u7ecjM7PNQff/S9oOFGUXR4+KYsROC3CXY500qB5XK5juZxHDPWtizHc8OyzDHirtPC1COYmqaNkZXlpVJgNpspxBfLiQSWApuL8ZtOt4swIcRhlVrMN1laJOlmE2WNoNXuNLVWjmMrxYsilZL1+k3Xsra2er1en1BUlqVt2K7rep43n83yvHzy5Fmv39rdGu7s7NTeTT//68+m89l8Po/jVAghhSqZLEvGKpHnpet4BFOCTYRInrEsXTl2U0OqQBnFWRAEEIjDN+e2be/v3XlzdFYUxXy+EgqlORMiz/NSFBxgA0kAhRZMQA0lF1prqCAAACuopNZMaiwRJBjiOI6zLKul7cQ0EEKO49RhcQjBirO9vb3j42PbdhFCjUYjiqI67b6+4mRZVu8Ib926lWWZadqUmhhjKXVZltvbu//4j/703/27fzdPFqZJi5xVpRCVenN4MpstFssNAvirL59+9um3q1WUpjkhhEg+HAwGveH+9tb+cPvi9PzibDSfzizDJYicHJ5IDaCCnulqAV+/fN3tdaIkgqfnr1+9QAj4rocwOjo99dyg0Qw73b7puElaKAUgJs2w3Ww0m60QKEYxEjwzMOy0wmbDefn8xeuXL1hZvfPoPVGJxXStWogQY1JsLMsiCHPBtNbENOoP6mC4fXpydHFxUU8MBAMp5fn5ee/B3fv3H7733ntff/OFFzRuWTnE+OnT57ZleI579/at403EymqziQJ35w9/7/ezdTSbzZIoun37Vthum6b56y++nI5H6/U6zcqyzPOSZRVXGlMppQJFUZVlHgSBKArLMn3fT/LMtUwIYcUrAHXYbtm2qbWm8zkmBjFdSKzxfLXJvl6tN37YOB1NlWCV1EmWhZ4rAbb9xibOok1qOa7m+WrN4yS7eeM2hHhra4tgA0JsUKsoqs0mMSxba5Vl2Xq9DoIgzZLxcr5TFR/91g+PJuM3x0eQ0CjeQICzOCGNhoYk28SUmg3XU0qlrPQ8F1nGVm/449/5cb/T/PmvfjE6O4VA2Z59885tzhSxYVyWitBWf6ucj72whTE23cAwDCZUWZZaA6G1S0xCCOeyYqwOF6+9gyilGAGtYW2MI6XQGtY2RFdwMaq1D5xz0zbraa/GV8Bv2uNcF4wr8RuEGmipCEK15xJC6OL0zDGtOhknyXNqGlprcWWgVJNP0ZVeH77l8HPdZ1+iBRASQrTmdXo1QqjW5dYR0vWRCCGu2Xn1P6nCZZkjAJEHIdJVVfqOuz3sd7vt+3fv/fuf/vV6BQnBZZbbtgEhzNM4DLagNi3Lms/nlGCNoG3bGILA8w1CBSsxxgCYeZ4vl0vjzsHB/t5gMLBtu3r5Kk6KMk9LVkENMaaXeg2ICCEKwDzPiSK9bldrvVgseVXV5nJxHEGoLcuCUC8WM8FygyIAlO1QCi1Kze3t7bJkCAFKcbTZaFRyXmkgKSVKKS6U1hJAjBBSXKnv0jU1AN8FZ7y9fYDfubNrhBBGCCGMBMcYI6000grommwAtH675CulgNIAXL7vAACof0OK+TaicIVAAKkUBKqWp14vLxBCCNWh2BARWHtr1pzo+vAwgSa2anIrgYo++fbZ0ZvT2bur3qDr+E5Nb2m324Qiz3Fdy5RC8CrbxNCgsGLpYNhbLBYV50qpKIoMg4xH08lsRqmJKVEIL1fRYrXMivLGjRvvfe99CPHp6SnPy8l0jjEtLLssS9/3d/YOzkcXF/O5Ajot8hcvXwZhwzEHlm3fuv2QC5CX7MHW8H52/1//xb9CSGGD1qaeeVGaBPu2qXk1WSyEEJvVRkvUcAIJdBynFxfjD957//bt25v16he/VDt721VVnZ4cua4bBE6SrO/duxN4jThOkk1sWcZ7773Xabcg1IwVRZ5MZ2POy+GgRymWCiZJMpvNDYO6TZ8xtl5viqKqWcoH+zcGfTabrqJ1XJbcNJ2HDx/GcXTz5s3+oDudTpbL1fPx67LQDx9sW6Z3cTGeTuemBXd3dyFAX3311aPbj+M4yrIEIkVMixCKoEmp6ftmmqbr9bI7DE2T5nnKhPA8i/FUa+iaju/bWmshVkWhINTdbrvVChEi+we7Sqlos5Cq+uCD91w72Nvbc2y3LBln2vO8ZrP13nvvb9bR+++/Px5fpEl+dHSEMDRN+sEHH5jS//zLL2azGcbUMIzhcGh7HgSYsZISI8uyb755Em2Sen+52SSlKtI0pZSmadHtDDFF5xeTu3ce2I5/fHx8cXHBJKtXqlVVKaUwdKAGGCIEINEQaQCV1krVKBgEQAoguRCYE4QRQgBrCGEttTBsq1bY13sKQjDn3HEcKWXNVPACv1Y2Y4x93693k5TSIAgNwxiNJmXJPM9rNpuGYZRlGYatmzfuVuVfCiGbYXsTjaPVBiFiGM5yuV4tI6VgsimHw+0H9x/NZovpdGoRpIR89u2TzTq6e+vu40ePb+3fOj+bLKYLLdSbs9Pbt+9yJous8BuB1npvd//k+LTKi7Kq2q1WbYNvWywvi64xUAqcn11kRamUosRkgjdb4b17dxAQ69X84mwNFXAcK2z4jdA/PJz6/oZiA2OqBACaEGzm+dI0TUxJVVVFmWutr51rTcvpmuZwOFwul8v1qqp4HMfVrZtHJ8cP799bLJa7u7vHR6/W682dW/uDwWA8Hr/z4P6dO3cwRC+ePvv1rz/9kz/5k63BMEvSyWSipFzN55JVVVUEre6vv/pGQoIQdhwHUikkRJhggype1WKwa/dczjly7KIspWCmQYMgqAhOksSwTNs2q0roouQAlFy0esNWOzw8PNze3tZMxElm2y7ExAtCprSA0PUa7VarLMuy4gCqMAw/+eSTrOBZlm02G9O0Wq2WZVmMlfVSX2vptZvNXud8Mv7m22+/+uZrTOn7H75zfHxcFEWdG4QAz4uy2Wi1ghBjzEMPQ5BlSc7Lw5PjX3/6819/+gubEKj1h+9/RCg9H51vbe1xCRTCiKDe1rYXNtfrdZImpmkLIQQTAIBm2A7DkBCDMZFlWS6y2jpsuVwihGoOIwBQK1hLEK6pCVdhSKSqqrJSdWrz29D0d2Pib5rr1Ytw0zSzLOu0WpPReG9nN8/zLE3Pzs4ePnw4nU6TPMMQQUKv/YKuq8J18wGu2BL6LbHD5WYdIYhI3QrUjULNVIjj2Pf97yrZ1f5CCOFbDq8YE1wqbhDDtezt7e3Hjx+XeWZZ1vHxsW1bt2/clGallFoulwSbW1tb9Qp1Mh512i0AgBJcG7TZbEIt0zjabDZCCMZKQsi//bf/9sMPP7xz7/67777T7Q0mi+Xp2ejk7KLRbBkmJYQKIYQU9UWjqgrlqN3d3Yqx+XzBObdtCyE0n893t4fNZgiB2kQLXmSB55gW9l0vKmQQeIPB4Msvv4yiqNPplWU5mc8NagF0CR7UrRpCECHKJVNKXfkw/Ub9vm7p3l4TXLYUGNe24pc7C6hqBOd65/BWt6CBvmwUfqMX+Ae3tx9RP+Sqobz8JF0dPIAQa1hrUpTQef1eYwwNw7AoMS1KKSbLcnPn8cOf//zn8Wd5s9l8cPcepXg9WRn30cHejmng1XLq2RRQvVodWTRtuO7h0888z9sZtlYrkJfFeDx+9uY1Maht2zdu3vbDxnQ67fQ7727veJ5XVgBTr9keGH0aRdEqL9qDrXu7OxY11utlt9+bTyePHz3SWnf+/2z9V49kWZ4niB199TVt5tpDZWRWypJd1VXdPdM709gZjliC5GBflnzhVyD22xAgAYJskAPs7GBJDsjpaVXVpbIqdWboCNdu2uzKow8fjrtX7gztoZCI9MowN7v3nv//J4PoxbOncQDK+Vw7yUA4OTzuhHZ3FI/79PTN16PJbrc3qitpFUqCblOBxVSi0louR2EvSiMhOEMkZGQ6PfnsC/nDH/7w/Y/e2T0YvXz50hgDYXe5nB/vP2ia5psvvjk+PFytVs7ocr2kGKYhdc4sti0F5O37756enBzvvjVF06yDvvj1L2W5Fc6tL087lCFpXnzx9c7OXoTIo3v3Pv74dz/9yQ/bVhjI834Ew3hy+NbVqsZxR8NwvH/ver66ODtdbjvHh/fm86cvXrzMk16/P3z88PFu93iw0399/urp0yeehk/zTp7HSZgDbeKM7R8fnZ4/BwRBjBabmbXy4e7BarVqqwXHAcJwd7+bd5Pr6wULbcOXBAaPHzxczVdivX60t/f2wYEOZdtwgDAmoCiqi8uzPM8fPNyfXoOXrz/N8zQMwySMyrJdr6qz08X+g+PuZHh6etrJe9ra58+fd/LeeLxzcnZxuZjPNxsAsQJguyn9o5YlUXeQNm0Lqd1Uy8lk53t/9H0A4fnl5XS5aKQwxnHeaGshhFKqvXH/+vr6Jz/5o5NXr6ezS8ZYEFASUkyR1EpqbZHFNEARg1GAWeAAUNLUTWtREXOBgaOEBBD3k/TZsyf3Hz4Ypum404HKOKVDRIpt7SyL07QoKmN0lmXT2VW3l1X1+uh49/z8XEggVYAFnExGs9llng3u33v8248/7WSyk6dSNG2zvTx78c1nv98ZdAed0RdffOM643ZVu0q7ykIE804edRJYw9dfv37z5avDw+MffO/7jw/v/fq3Hx/vTupimYTBeHfn2bNn//Qv/iLuRqOdzvn5JY0ZDLAmkHPZ3ZnEcdzUHFrZG/RoVdV1qWVbrWfboverX0+Rc8AI0TYBgbMofHDvaLta74zP63oLoOgM4sMHw/V6UW5tnKSbzSpN006nBzGq6xaTFiK2XC6TJEqSLBsMTq4uSMxcCPJub13MPv283tkbXl5fNk2V90YkjIpaRuno7Ozk6+cn//pf/quy3LK89+rLJ3/197/Ms2C2WLzzzjvRoNs0zTefPr+cXpMwGXR7z1+8Udo5GHCh06RLAyLrKghwr38wv5geHR0xQE6evD7cO+Cci1qFYdDrjKuyBQBZgEeTPcYYQk1ZVHnOwiS9vpqdnZ31+32t7HpdYkzns2Wexsv1artdjybDTbm+9/DhB/c+Wv2Hv5pezx8+fPv+cfb1V89+9fe/YkGUsiyiCSKkPxjHaSq1KMoyJuHR0duL1fLvf/7bLB8GQVCV7Ycf/eiXv/ylw+zJyzdRFB3eOyaDbpAkaZrmnbjcbC+RXczm//e//H9874P3v/v+D148+cZZ++bl6bDff3h0aI0zkg8TUpZ1wzUPgjyMUxYmSYIQWq23ZVlSDOqmdM5ZJ7M8woRJKauqjnHoz5tiW8ZpJoWO06wVSkljLADYEkIMVo5ilqbDbrY/6my326qqtFIYUWuMc04ZraWOoogQopTWVvvV2VoLMEGIcKmHO7sF57zlgNKC8/l60xuNoyxvm6ZpmpA4ay1wgLGQ0uBW4WQxBAhCiolTEkJIgMNOO6WgMSGNO0lU13VV1ISQkEVWGtMKamFCgghRpZTTLgAYIQQBdBBRAp2DnU4HQAuMxdAe39s/Pj50Vr54+SzN4of37j979uLkzSXGmCAKTWSd+eKzz9977z2KCQJItAJjpLRRXG3Xa9FyilkcRkKIo919pVTZbk6vry0hewcHvX5mgOFt3dQFgJZYSSCkAXGOSK0tAITFhWvCQWLrzbRY9AY9AMDl5eWD43uj0YgSJHgdBmkUUICtNJDQII6Y4K7YNmman19cnZ+fUkYoRQj7akcPFSCjnTYOIwchYowAALSzEDrKoLZaae0AwRgjdAPZeBclQliI2jllrIWIYEYdJkpZEkVWKWCtgxY5BwBkACinrVWQBs5YP8AhhCB0wELnjLWWYEZJ4M0jhFBjjBDSGgMppgjB2+nNOQSchQgTipzTXhwTRAmEWAgNLM6TGCOYxklTVt3xcDQYEgwJAGBnZ2d/f3+xWOj5/NH9B5Rij+r8+Cc/ULJdr5EQAhN0sH+ktNhuC62Ntc4XOhsH4jR/9913n798dXp2UdXt8YOHGGMH0Ww2Oz09XRd1URQIoULrothMhqNBvzcajSJG37x+PhmNHz66PxoN0jTdrBcf//ZXsw2nIU3SMAjjvBOnnWwP7fzoRz/6u5//Pee8qWpK4zwPGY3aRiqlFtOZXyNWq+VyudzZ2Xnr8cOdnZ08z50zb968cc5FUaSUMMbkeb4pV0KJVrTL7UYobowpqjJL4kYKimHe6dR1jSl59M7bBqJayNmrGca05TbLsjCCRS0wxl8/f9rfnVSq/fmv/35TbL+XfLe/twMAwKHdVFNta+tibeo4IQ8eHkDAe91ESXVxccYYeeedtzGky8X65cvne3t79x7tHh0cQGjXmyWluN/t0CBYr5fGuNVmPZ3Bi6sXs8VZnEOIYBzHV1fTqqritIOMbFo12Tkcj3NrUbEuX7165QxoqgI6FBA6GY8RAtfX17xVWToYDfcCls6vl5988sl8fnV4sFuUK8ZIFEWj/iiKMghh0zTXV4skSZS2s9Vqs9lcXE3rVrXKF6vD44ePHtx/9NVX3/wP/8O/Kyv1zjv39/Z2RqOJsXaz2WCM8zyP0jSKoi+//NJfghgDaxm6DWIris1kd3xwcFDX5XqzCEMGkYuiyHkGFEHsgLvti3IQEUq91kprXdeaQNDvdfI8c1Z1u12vb/BYqF9ujPPB74m7EWBLznlVVd1ut6oq788OgqBtWy+EfvTo0T//5//1YjmdXV/FCdtui1ev3gCHjLEY27ZtfdwChHAymUipIW9vSvwoiZJYcrVarb765ututx/H8ZPnz/JOt5tn0+l0NB7/7Gc/+/2Xn3c6napq5vO511J4sxbnXEkTRVGSJEmSdDod7zt/8uSJcy4gJI1oQAnLUoiJBWgwGIx39qSUytxItBhj1aYNIoYTZLS+vrpwFiZJAh24urhkjDhjlZSvXrwstmtKEEHYan12dpHn6fX1dHo9M1bfP76ndffs5DWC5L33PuCt+B//3b//8KMPfvzjP67L5vDw8NNPfjlfLt9+++08z/M89/EVnPN33313vamqmnNp06w7Ge9uy8ba0oPPXllpjBJC+Mg/hKAx2H8XHmHySLvUWmjFlaTWsjCg2ljnuBRlU/d7nSSKsbMAwuF45Iydzq6jKNrd2f/e974XsQRCfHV5NZ1ON5t1twf7ozFAcFuWxpgwYo7bsiw3dUEp5YJ7dZiUcrPZIMLW63VZN4yxvb29nZ2dJEmllOfn5/DSMMY2m01TV5NB/yc/+UkSsPV8tl6ufKIwwcxCG4ahUsYYB7AeDfp5nkttjDFlWSpeKcmVZBBYB5BVimAcMRIGlBKkW4GsacuNMwZZoySvSyelHI4njeBWC0whQwQ4hSCOgzhJEiFE0zRt2xrNjTFed4gxuYMZbg8eH653A/7fUQDer1+WZb/fz/N8s16729pMzvnd//2Wor7ZdD1icYdb3DEREHpPIHXO+d4+AAAhxOOF8Nap6wl4IUSvEwFAIHCYuCgMwjAWQsxms9VqNZ1el2VprcbERVGQRDFj4atXL87Pz322sXXOWkspUdAQCKfTqXPOH5MAAGVN2sm39aqum+l8bpxzFlZtU9W1tRZhbG87KQhhwS3OH6WhRzd9FYUxpthum6aJokjwhhAyHO4FFF9enZZlCbrdbbX1jzVfAcUYo0HY6/Vms4V1SBmnlFUaQIAJpZQwJf/wkd6BB3/Y77/1r+wfYpX/gDp4GMb+IYnhJvECAOycs9CZbxVUAmDBDctwE5QJ4B/inCGEGCPg/oBF3SEN4FuxGd9mIgghAaGMMQhuFDN3YBIxzi7XK2ttmqbW2ovrq16e5d3OdL7grSzKrXPQAWylSnY70+mVarmUClSN1lpqk8TZcDIejifGouls8fr0zDgUxpFn1IRWQqjRaNTN8vl8utUGQQCMlW1z9vpys17uTEZaiN/97mME3f379//7//7/8B/+7u+qqpwvZ9q1cRIEIXaVzbL0+Ph4uymFkISmDDOnjbU2DILhcJDnuf/9d3Z2/ujHP3zvvfecc5988juf5b67O+n1ev5ms9bmo06cJ9YYFGDRmqosZut5Pnj0aO9tp818MX1zcVaW5dH+watXr16+fLG7s7O7O5FmFXcGR0dHo8N7ELnPPvv0yZuXw1G/app8pzOvZ4t2qrUqimLQ7StT16375LNX6+Wq0+lA5yAyEFkuakzgxcm5US7Pu9tiPV9M337/HoQuS0PBAwh0y8vVanV5MW24btt2NO5FUTSZTCDmQjqjNHa45dYBgbBtGmEt6PUG9+7BF+pNVZVa6mq7IRjHQZilYafT8RdrWZaCn1elOHl1JoQYj/uLxQJhu7+/2zTNx68+Vspp5bKs8/g7HwJCDIRCKA1wkOTSgMVqu1iuwyS//+BxbzBI8ot7Dx8JIUaTie9FBAiWVQkcquvaQkAIFUIAYMMwvHVJaS+Tccrs7IwJQ0kahgnr9Tt1XYdJWDcNwIBihizQwDkELHDQaqUQAMDDYlK0AcVhOB5P+pQATNxqvWWM3N6QmHOpsd5sNuPx8NvArOdQ67r2IS1BEEihtdZhGK5Wq6Ojo06n8+bV1b37OwiSTz75VNQCIcJobIxLkkRK1bZtvzuQUsq60tYgAMMgTvojZwxv2rJpV9s3w+GQBeF4dwcA4AD43g9+CBD++uuvy7L078SrKIIgapqmrmsptDHG5+fc2duyLBNNo5QqrW4RqKqqruurq6vxeNztD+I4JiyklBpjsiwzxmx4AwCw2kFrozDKkohgRhF0wPCmgi5cLaaU4jAIOlma5+nFs1fROOt2+3neff369SpbP3781uOHj8uyXiwWWkitxGAw6ve7YRjVdTMejwljvV7PN1nM53PnHOfNd7//g6vr2bPnr42Tx8cHg8G4ePpSSO4PCXgT+mv8xKCUattG6xhjjDDw/hcfDoEDginxykccMGYsQBABHKUJDlh30Pfnaz7sM0K5FkoaANC9e/fTKGsaXpX1aNzjnC/XG351FkZxmMTOmYuLCy6EAxYA6IcqrY22Vim12Wy41Lu7u2A688b35XIphBJCnJ+f7+7vdDBO804YBG3bfP3114cHe0f3HvCm9R++708PgsBBDDGmdR1SpJUsNpu2bY0xeZoc7O2WZQkQFkJY1UKLYRRkUdxJojcvnkcMhgRlSdTrdjp5ksQZImS+XCYUpnl/d393OOpTSgEGGGPfjOXnKmNu2G5KqSfgEUKEuD8cKrdJDHeyBnNX2iRVmqa+KslbJwAAd7oH/zPOOUTw3bV6d954igFC6O8jPyh4AdBdzImvKvBKPf8fVEq1bWvVJooCQjFFkDEmuLq+ntXVJorCtm2N0QhBIbiUQrRNt9sd70xOT99UVeUpQkppEkc+Xun8/Nx3QmIMwzA0xoRhaCxsmta6jdEOYKKUUspARCwEzhghJYAYYIQRdRAYayGEl5eXm83GP5p8c1tICSGktVZJYYxhSRTHcV37VqrQOcc591ORtRYgXPveSAB9jjJCALibQerb3NDdl3KnJvn2YPdt2QGE6D/7Af/Ec84B6wD4g7DROQOAAzfJzdA551OcMcYQ3TonAQDAQQgIIc5ahL7FYdyaNoE19n+e7+T/Ad+WV90xIzfDVlUXFxemFZxSiixarlZKKWfhcr35/WefI6CTJGIU1U1p3pwUm+3Zm5dN07Ag6Ha73W4/6/YQprxqWRjSIAy1swCuVpuyqX1B+KN7j3d2dsIgIAhXRV2V9ZMnT87PT188f2qMQgi+9957H33/o9/99te//u2vjo+PHz4+3G63mOkgYUkeleX27Ox0uVwZ7bbbUiigJCQ4QjiMozQ/OFzABcbYWDXZGe3t7b399tuEoBcvXlxcXCAEwjAkhKRZzAKy2a4Wi4WNwWQy6WSpUmq+nF3OrwwwNAqXqxUl5IvPPn/9+jUwdjTeGe3thFkCJEQo0o5whcK8/+jgSFu1FrxpShBHxUJcXV5/8fzTOA4hMoSQ/f4IArzZdi4uLtbLzf37D5Mw2a63zuI87SRJNJvNri+XH374XsQiY+D8+qrltRAcYMAwXi1n52dXJ6eXvFWEsDAiDx4eIpxvquuq2m6L7bA/CFjeNBJCTVm8WtdSnTIW0iAsipIgPJsvtBTQgdG4Lw0/OTmJoww4BlzDW+OtAcPhcDHXccIePHhwenr69Rdfl2WrpE3TfO/eo/K03W63AOE0TVkQbDbbuuFV1eztxZuqXBUFIuyf/MVfKGOvr6+DhYvjUBmbZZlPU19tirIsfDO9T9r3Xi9/CYZJbJF69fpZWW7DiHa7Hc5rC7RUrUPYQugghgBZX1LirObcaIcQQsCbjmAQ4jSNIdBZHrWioRRjAv3PKKUgBuv12t1qyAkhcRz7RDa/YzVNE4aRD6oDALRtHcfxzs5O+7j57kffN1q+fPmyrWqtAIyx1bDT6fFG1nUppdxu18V6EwVhkiSDXhglMSOBioXWer3e1E27s78XJ9nF1eWjt9/5/g9/+Nd//3e+DdLv4v4Z6h86YRhq1QghiqK483k755IkljePFQgxtc4UdSukWqyLKIqO7j3odDp1XSspev1Bt9v93dcvAQBhGPZ6Y0qCtm0pgUeH94uimM6uopA1tUmTxBjFKBwOelcIrdfrzz/70hg3mUxWy83nn3/xvQ8/7HW7T58+H/a6EIAvv/zye9/73qA/urw6Hw76LAzbtv3Lv/xLzvmLV6/6w3FRFGEUvPveO7/5+HfrbfPW2++sVouyXFurrQV3p4V/xNxVC1JKfd6Xd7JUVaW1RkxY6yhjXtOgraE0yPMuDYPNaiGdGfVGwGpuFGaUpnGadoqiFFwBU5VlvVovV6vF+cWbJM0JAUFIKEWtFOtia5z1HYZBEEDeSqmsn2C0adv2Ljxgu91uq/LRo/yuaWI4mjjnltfXXz17zuvmR9//3uMH97u9QRgn1tqm4UqpgEUIEcZCoJXVqqnK7XrjR8/xoHtwdG+xWAAAWi6KwteEdkaTnSzL9gfdLMuU0kIZSgOpTBTF2rjDg4kxRltjnF3NrxverFar9XpJwvSuY4lg6C9say0A5qbi0loPnvlDwtx2Mt2g0w54HyPFxNcH+LnNYwOEEIscvPVkOuc08OH/3mTvS5sQQTfTw53j/2628Aq4u9X51mUJfTQqxjgM4zRNoihgBGd5lKZJy2sppdbq+vqaiwYTiAkEDkEEojjs9UebzQpT0u33NsW2LisHAcQoJOFmswqiCDsXRfFwOGzbVigJIdbaNlwHkcXUWgcxYWEYQkKU1tYCabSRADjl1UtcNAsHgEN+1NBadzqdfidvmgYhJKW8vLy0epimqVKq3G435crfuU1b+Wg7iAljTAjl7QwYAwA99WCk1gEj7laLcHPSf1uLAP+gG/32wWyMcQA5CDFGhBBEqBe3+igI/yFrrY3WANGbeQLau7gGCB0hHjlwEN6MFd6V6gxGCEBrnf0DeuGcIxj7L8gPiHcvpRQwGsGbb9P/IUKItELEaRrE0ezqmhAy6PW1MbPlIkui84vL+/eOhVCbTSV4c3Z6Ya2FEDuApDRCWePAYrm6/urJi5cvpdHrddnt9br94Xa7VQamWa/bG0zGBwTT9aaoamEs2m5Wy+U6DBkXNbB6td3mve73f/JHypl//+//3V//X/8vg/3xYNDr9XoHRwdJms8W89lsaQ0KgqST953DBIebdb0p5oyFYRC3ZeOVq4wxreWrVy8wxrPZ9d7ezng8NlYZY9I0jaJosZhPp9cNaliADg728jxP0xQRMp9Ov/766z/72Z90Op26bRDBddt+8fVXo/5IKTXpD8uyRoidnl87SncP9nFAdg4fWKC04WHW0QgAgt9+/HCxvCaEyIb7DNFOp3f/+NG7776/mq0/vvrddrPFIJgc7x0f329r3jRNFif37z9eLtdNU0nFsyzud7ohhdba9XIFAXNQXZ1PIxawGEqtAtKJAycle/To7fOLi/Pzy/3DURJnm00VRTAIEm2XvW43DEOnlORCKHV5eam1btu2qUtnSZr0er0e53y5XJ6eniJsCUFv3ryZz+f9/hjGJM+73rc2nU4RpgDsOgvbtgEAHR4ertfbsm7H43Gn08WIrjZrraV/ykgp2rbVWgKA2rY2RjHGEIYQASONUgogyBiBkGIK15vFRVPFcUwYVIZLLSAHBhhjjHXQIgQQgdAZba2GTmNgDSLAV6k5oJRuW15xUVNKMUZNW4Hb0DFrHKN0u93Cb7mDfAqpt0sopdbrNQBwPNpxzlVVNRj1OecPHjzI0ywM44uzVdsoSiJnKyWt0y4IoqbiSimlRBiyLYIaQkSoMm652hCMMSLAuazbKco6DKNtWYwmuz/+45+SMHrx6k1AKeecMZJl2V3Abb/fJ4RAgJ1z/l21LffFNpQgSpC11C9q1lpntUX4erHudXQr9SSM8k6XUgqBS+Jo0O8ghJwDFKOAId5aKVqtRJpEZHcnjFjbbEXbWKeBNfPp9ePHj7Ms46IRos2yrKlLrdV777138uZVliXW2mdPv5lOr/b2dnZ2xwDaOIRBFDMWrFaroig4BwBYY9V8Pr1//55SQgiglHjx4lUrFAuCzbpxziEElFKUYg9xSyn39nbTNPa5mf7I8fyLUOtut5emqV+Cq6pCTUMIo5S2QiwWC2e0MyrP0zRJ5vPZW8PJ61dnEEIMUVEUq9XCAUsoTLPwrXfeDsL4+avXRbnqdLoQo+VyHbn0blHzHJYVVgmhtc7zPM2ypmlq3voTDgBQtyIMY6Pd1XTWSjAYT/Jur274wfE9aF1VlL5DiFLNGIPgJh4YIXSn5qvrerFYrFarIAggwh6W9xRMGIbO6jSJTk7P25ZHSRaGcV1XV1fXy/UKINQ0TVEVYRJOJpM4okYnq5L7qCWMsXE3FmIpJcbUx5YDcNP17J/sflDw1z9CKEpTf2x3845HGjqdjnNuvV4rpYIgcMCAW1jeWmvADQ0RBfQ/W4U9Gwhu46jdbdAvhNBLjG8wDGP85xwEgTGmqgpCACYAWNQ0zlrV8rqqqn6/J4SgJIgj6IuIgIPWOKGUQ7ARXGillFquV9pqCMBgMKAB8zAAi8LuoG9Xq9Vq5SDl0lpeI0IZY5hQjDGLoiAMpdRcSeOsc04bKbUQSkjFrTZJnGVZxjBjjPmlvq7rTp52Oh2lhNa6LNuqqjjn1gIPjXjFtBcopmmKseBCy1YoZa3DEGCMMYLQWuMbce9ohTuJIsbYC7Tv2ByEUJIkQRBwqYwvqkL2bqf6//tyztx2RflHnLlRJqIbsABChxB0wHiTJETo7ln4bZDgVkwJ7/7ZXzbGKuccRMhn2VlrpZRKUZJ3O0VVCs4tBFEUSa2YYXmnZ7SMomR3b//q8myxWCMEmlZZp8fdPIwzf5UUVV1ezl6fvHlzdspoKKQGmEiphbQsiLK0E7Do/HpZ1/V0Ot1u18AZ7YixkkISZb3FYrqq269fvop+0xlMJv/mv/vfSYh//dtfLZbb42M3Gu9VpVjMys2qFUJkaT+NB1GUKKXP8bVSpirr9WLZ7w6MUVpLhIAQrVLC9443bVVWweXl5Xq9stY8ePDAWss553NOCDnY29/dGR9998Pjw/2/+U9/+8UXX3yRd9569Mga0O8OOlm33Gy11sY4Xlda8TQKi/n8xYtXm6rpj0dvv/8dbZWx4tHjXhAAYNWjh8fz+TWCDtVNXddJkhZFpYUrt2q5aIqNOth/2M07xbYVXI/Hu7u7uxGjm80mYIxShqFua6GUGQyGvbwHjHMIAos2y2qebnv9LM06R3v3pOLbqv3uRz9l9MnzF1Mtg3vvvesvOK01gsHjhw8OD3YZxUaK1XLx+vXLSRQgSEXqpHBSOM55WZZNUzRNMxx1x+Nx0zTQwqOjB9bAyWQ37WabFU0ipqQBTgUsDBgJgohRiKBx2mrJRVs5CzeL2ezyvNuN4iQkDHPebDYba20QRzuDnfl8rq2xliotIQJREg0GfYzIbHnZtq1zpsMSQgIDVdqNAQAMUi6VFNJqhBhEEGlnjTEMpt7s65zWRlkjq2pbVuvL87MoihxQ8/nMOWOts8ZBiCmlRVH4G0BrbYx2twlohBAf9Q0h2t3Z9w84KSXn/NGjRzvjyfPnL05OTsui3p1Mut0hcEio1lmgpBFCrFarTqfDhYEQ0jAwzm43JYIwixPnIKUaYmQRxoT883/xvzi+f/+Xv/4NCyOrmqIoMG59Aa7fAofDYdM0HgoeDAbW2jhOvJQnYNhaGyhFaAAhbKXUDhoLst4AInQ9WwyHQ0qpT49o63LU60ZRtFisivWi3xt0krgsy6uz0ygKJpMJoTiPo6urq24v3RkO37x5lUzYvfvfEUKcvHkpVZumUbeXD4adJ0/K3b3R7OqaBbjbyy6vzkaD4Q9+8F0EdNnU3W7XODudXn/2xReboliv11988cVovPPBB+9pB4aj3pMnklKitXDOCSEoxW3bQhgSQpS+idKKokAIAaCNoghjuN1uAQCdMB73BrvDcZzlUVkvpovr66uqqPuDAQKg2Gwvzk4RsPfvHaVpenFx/pQmQoiDg4OAkrZtGSNvv/1wd3f09PmL9WaBMG3bUmvZtBVlYRiym8YB4BgTyhh/ytatODg4iNMsCEOfQjGfz4UQUkpizHK9XqzWGNFeLwYAbDYbkHc+eu/d7WothArD0B8egkvCaFNWVdNqexO93DTtaluVjVgsFlGS+rW1bVtxPSenF4yxDx7fE0peXl4iStO8m2Rpsa3qtlHGEAgBggDf1gNihCmh1HoUWghhzU0vMwDAi+QRQl6s4A9vxhilxDc3+rMtYjelDH4UY4xFYQgh9HSGtRZT/J/hzLfn/Y0M3lrrzE0/5B2GYW8dgP7s9MwIuIlgv0HO/KyQJVGapmkaRwElBEPkELYIDnxPI0bYWsi5woiKVhBc1UohjKWUTdNYCJQ1np5TWiOEhJJhGPqHmNZ6vlqGYbdpZV3XXOggCqMoitMUEay1VdYYZ40x9q4+EUGrtNGac66U6nf6YRheX11dnE57vZ4x0aDfy7KEN9XzF99Mp9M4DIMg96csxjiOY+ecBTDP8/39znK1vZrO1+tCa40RRIRgQgSXAH2LaLhdV9C3Xnf8AoTQPwSkNloZAwBExhjjEZ5b6sHeCQggsErfALSeuHDOAXhXJ62dQxgTCKxzxptvMULAmf9sagF/yPX6Q6anLw41VgGjMYLAGo83cM4DRkiaxrPZDGPc7/d7vd78elo1TSfL1qvVtqyfP39ptGi4VEp081RKcXJ+cbMlKMUYs8AhRPb3Dpfr1agzGQxGQqrlchVEodS6rOvpdGmMms8XQrbdbs6SQBtOooiLelk0xZMXwjpH6XsfvLuzs9Od7Ny//53r6+vVqlmvWwc209l2uaq3203IyjhOTOYAQJ0sHw9HWtv5fK4kuB2iHaU4DEOMcd2UWuvVajWfz/w4X5ZbCN39+8ez7fTq9OzzTz5pq+qdx+/s7Oz0OllbVi+ePw9IkMdJWZa9tCtrURU1tA6GaNTPlptyNOhqRKwBxqLzszkOWJJEQiqtyHa5XM+f7O/tJEkEEQwoC1l+XdVff/EUoVfIIeCivd0HzthPPv7d9XRxuL/XyXta8tlsdv/wPoHIIGOsKLe11cCXDmtpKUsEt+VGAIAhzOyQiVauVvLNq3lZWWfi8/PVwUGzf3RorW6bDQAs7w7f/+C7GMEoJC+fPQUAfv38ay54ng2TSXc2XW+WhceXMMa9Xq/f719eXkZRhBCazxZBEGEMneIhgcCCXpZ0u93tar6YnWNMGQ2Vc69ffBOG8dHRvSgk1oi2BXEc+/tQSi6EMMClaeo3Ff+lpGnaHfQnkwmltJYrqRrGQhaSYX+QZVme59Pp9PXJidRKa20gCkBACAEOWOAQQBBAhJy1WkoOnQDQEgIXy3kURd1e30dNG62ttZ7v94OCd3MZAxC6saRTSn3/HudcSulzspVuKaW7u7t1ml9dXUOApdQQUIIt5wI46NxNnfz19HJ3dzdOU865A9BBBADAhCHCpJRt1eR5DjEihIRx+vzlq6+ffIMpydJeXdda38RI+zo7rXWWZQAA3ykHIazrxhOc1XbBpfbnASQkQCSEkIVRyCgEtqya+XKNoet2uwhYhFCvE2dZBoxlGPW6HSnVatFaazF0bV1Zqx89uCd5FcfR8eF+U21G437eScKwF0c/zfIkieLr68vPPv8dY7g/6CYR+fCDd8KIbTYbTBxlEANGlbTWjsdjSkkr+KeffzGdz968efXpZ7///g++iwj7/ItvMAFxGm23Zb/f55zfZNVDGMcxFwAAUFVVFAVSyjBiw+EQIeAf1qOM9HuDTpp3u4NRd9hUbV1VVhtkXRbGW6Fk02IMRcujIOzmnaYWy+V8NBpheHM6pllsnWYBevr0G6lMbziK4mCxWiNMJ5PJzZ7kc4Vvl2CqbdM0rZAIY2stpbSu6yAIjo6ONk3dcM45D5OYt/XF5ZVs+eOHD5S227Ju25bQIIqRKkouBQOwbITf8oMojqJIaGut0wDTKNUW1FxDCB1iDoiyFbqsj8pBmnfnm1WW5tP54vxqVlT1ZrMJolhKhQgeDEeUEa51WwpPAcBbjRuCntXx3RDkhjwm1I8O6KZT4CaOzBPqfiPknJes9FSgVsrXo/iPxeI/hP1hjBG8cef7Y8yPIL6G+G77vL2zbkgHdNu65G41leDWbIkxRthwsYVIEpw4gDyyYrRumqapecSipuFRFPd7uaMUo7AsiyzLhBBcCK8ehQRDr5yAoOFtHMdVXZ+enfV6PUIphNhaoJSRykCkAVKQcmpp1TYWOH/eOucMuPkFBRdRFGllN5tNnuSelPSizrqugdMQujQOPbVhlLq4uLgRP2ZxnCSUUsKCo6MjYxwXmrEtxtjYm0wkY40xBoJbRSiEEEB3aze9+wy/rWT8A0Nxu+jfsTw3MMC35AUQQgeMb3Jyt8pIAD1mYKxFDhiIMAI+AdMzEH/4fu9gDGtv4pzsrYjVv0OMMYFIcetHEGutNMoPlIRz3uv1Op3OYrGCEAZxpLg8PT0FAHz11Ve///g377//LoJgOp3meQ4AfPjwIef87Px8s9kgRDAlFgBCWFU1k739o8Pj1WZ9fn4hhKjKxrl6szVpmgZhoq0jNMIYNkromistB5O95Wp6cnGVPnn2i9/+djjsQwgpiRlN+r1Jrztu22YxL+pKSOEwtGVZbzZF0zRxlN6/fz9N06apFk2ZprE3zft7Q8i2qqrj4+MgYJ4UGo0GHkfa3d0dHfa/+err5WLR1DWv6p/97E+6eWc8HK1X6/n1dNgfXGwu8yjr5Z26bqIwylM23hmuNpv+oBukvXXFjQL/8Itfj3b37t07atr63uGB0ezzz57Vhdqul3x9qbU9PDwWXHl6uJP34nG0WbWb9fL8/Oro6PD4+N6bNy8oxvfu3evlg7LcQgejMBNcXV2+urqaYUQtBMjB9bZoGo6Qyzvxer3ebFaff/Py17/6dP/wEKHw/Ozqr8U/HB8fctGu18tis0qT5P7xseT13u6OUq7T7e/t7f3DL3795erZeLSPURjH8cHBQZqG280yTlhVVfP5PCBBp9M5O71cLpdpwoDTUciSOHz86N7u7v7l2fnTr7/p9PqVKcuyni9WcZxkSZTn+bjfM8gyRgCC/X4/SZK6roU2Nzo1B706GgDQ6/UGgwHGeNyMmrammDhnBqP+wwcPjo+PP/3si5OzM+ectspBijGkYYABhBJBCRGECDltnNaaEhcEtNvLjVFKoU6ns1xswzDUivvHkw9GhbfabAgdRM5ve74mLo7jqqqLogiCIMsy7/jfbrfXl9fbTdm2YrlcZ3EmueRcRixgLAzDMAiC1XIZxzGm3DaN1AoxBAmmlAIEhRBpmjsIpJTL9fb/8x//v6vN2loLEEIa3E0wAIAwDJ0r3S2G7C2mfsPz1EMQBBBTf54J46SUAGGIVZZlaRxNxWVZlkq0jLE8zxljkIVpmkdR5CxECH311TfrdTEYdMbjUdvWLa//8Z//SVVtV+tFmsb7+/tZlkjZZFnEAvTq1YvhoB9FkWybpi2GqHv/wXEny54+fWqt6Q86//DLv79/+EhZpZSazmeE4O985zsWgqfPnpd1++mnn/7Lf/XfFFX7y1/+QmswmuwAAKJgWBSFUr6hOEjTFBPoSSIPzsdJOBwOMYaLxaJpmoRBYG1T1Z1Ob2eyK4XWQrZSRXGstNBK7YwnhCKKcEjZg3vHelHP54BgmmVZr5dB5IbD4XR6PRj0G863ZeX/3m43F1JXVaUBbNt2vd1sNlsaBHEcd7vdrAO11kJp51y329XOTqfz6+vrMAyllGEYGucW87kR/GBnEoXhdD47PT29ODvnTZNnXYSwEKKuW8Y0IhRDSKM0z/M4jquqarlkQdRyWbWtELXvGIzzbmCMUurN6VmUpNaBNO9oY+arFQvCx+98p6iq5XqltA5iZp0tioIQ0u/3HZR3gg/gkMfJpJRhGN8uiDdzsH/QV1Xp/8GvTAELlFJeGtLr9QAAbdNUVeUpOedcY9W3cWmMoD/17wYCa5GX2t8Mr14DT4gn+D1+cMe7/5dnUpYHUkqEbRhRSqk2CkCHEOx2+whuCWRFUTe1TGLTVK1WEMZ8MOhxzv19Ya314/Ww1w+CQCmVJImUfLPZ7OzsjEaj7VpjjIMwTJKEhj6cCgihtLUO+YxJezclIIRaa7MsaxtR13VZlhjjJEl2d3cRQk3TXF2eXV1F3/vog3feeSeO4xfPnsFbOxXnCGHMOYe4Lctyuy3Xm7KqKmstxgRBYgHwsxP69pzg/hCtePcC3zI+tK30E7MDN20RGNs7LMdaCxyw1jr/aRv9raHhf0ZJfFspAgH0MwAAANzqS8C3EAUIIcIY3cbX+jHU80oIA/8X3QhZjPIzJUGGEGcuTy7X6zU7JKrhSqkwoFkSKcHzXqdcLSRveVWcPXt6cHCQh6QTZSHaC4CeL5dOS4qwbOp+xqhpQ6Q7ASJOKK7q7fVquVEQLxeSBmEYRxVfsjBIsrAVXEmnpej1xwElr5+9ChiBXEEI9waT4+99JHm7nV0hDLLAJZO+tbm11monpVy0YjudvSrbTqdntI1IeHRw/8mTJ+OdSX/QaQTHCGV90ij76vQlhG406L589VTwllIMDBMY7R8cMcJWq9WvfvM73pr33/3gpz/7R7/6h19Op3NCSJ7HOwfDTppBCKqqClx7cLA3mIxZ1vnq5aun56+evT4J0yxRycvTZ9Aaq8pelnz4zuNBJ1fd7JSw4bB/cnKymM3vHR0RQtbLhdLxdrPhTR2lLIzo3t74m2++eH56SgghoRFAdEf9+WKltV0WzcefPesNeghSDDiJQ6VAvzs8u7iSroqiTrWVETPPvno1GAwORsfr1frJ8rmWYltsIDQf//zTw/HhZDL667/6t69fv6zL7U///MfHx/cPdwx1GFqXxqmQ8sWTpz//zRf/9b/4R3/853+S9PNu3oFSYthsl6uXL7621l5dTfvDgbMfFuVcmcrAZlvKvf1DgDmJu5QEV/PXNDr68EdvQQjPz89Xyy0NwiBMALSm4lpZTBhFyADnnKUBsbDdVlOEkAWAheFoNIIQOkylgwrSKO9LhyTAEAeMUsYYAQZCSCnecGGk4G01GuVZls2vz5Ww3aSLHGY4OD+9iOK03+9uywsDVBDRMOm8fnE6u1qPxj2nIUEBYVgbaaypVSVafrh/IISUklNMEIBNCdK4E+f9T3/z9S/+7tfr9SYOotVqrZQa9kdBEBjgOr0uAKA76F/NpgTiTp5YawFwcZ4ghLUDYbcTJNnu7u7O3j4hrNhWQdzRWkOICTUs73POrZZ+nUq7HcowpMgKWbbcLiVjDCARpwhjbFSgqkppAaBGAEDTOu0cDJAmvKr6nVDp1kJrEWR5hzEG27obkXgwbriYzWZZgh4/mqR5lufBeCcri/rVyevBeHR0/96rN+dZJ1dNuZkbrFUYkP3xEADgOOdVebR/gKwRLWeD4XA4qstSK4sgiZ3o9LuvT89UUZ3O5wcPHkmJW0GvFytA6W8+e/rRRx/9+E//0bNnzwaTISuDbtjF0JQ14FJlvZ5F6Gy22tkZW4wWZbMpClQU8Le/oQyX2y1CSBZtJ+/GcdyKF4yRvclgPu0sFqvxoPvrX/32vQ/eV8qcXpxri4ej47Y1qmn8kay1DoKoKrevNi8Ho+GoPzk/m+7vHJEwni9W3XSIKF1vCsJwHEcNr5WSFrokSSgJgFIsTvlqY6wd9Ear1coIe3B8iB0Bmm8W11nMZk7n3Y5F+Gq53t/d/fjLb6zS3U4nyDtSyoqL5WbNCK0lN8YkSTIcjw6ODj/+7e/jOIYOPn70+OLiqtxsnYOmMYDpPIodimez5cmrC2TJ1dkVpgRYG8RJWaxXy2XTtEEcUQQBZIN8QBgFFhjtNuuV74XyB6cnd24RfubPaEKgc1YpTgFqiwpCCJ2jAGmpCMajwdAY40kWAAALAo88W+dCFkgptQMEYeec1do5xwhBADhjIUQMEwOMNcZ58AA4CIHgbdM0zhoIAIIgyVKjlT9pjFabpjbG+EF8eXm9s7Pzw+//8P333xdCfPLJZ998803Tihq2yCFtZLfbMcaU1YJSigPpcHB+uYAQEuiMMZiEUkoojVBWG1wW5dffPOt0OlnWXy4KSum6mNGIBdByJS1EBFN1d+YRwgIaUGKc1kY6Y6GzxDnR1AggBOxqPdOGA4BYEpVl2bayVlA19s11YVlP4wHJDnXYSANbxChN55U0xiRJ9tWz07IsnQWUsjBCRVEhpBhjDmlCIWVMSqm0ITRwDnKpKaUBCSmAVtlWNEkYTsaDYb/X76VX12shoZQAUoec005DZAi1xmrr9M2YY4zVADrCEOXq5uxHwNbtZjzsFdt1GEfQWBYiijAw1mJKWAy8iNVizpXRLmQEI6q081pF41zokLXAwSCKIgAZFyDGjGrFK6606Hc6YRhvt0sYBMI6UhRbQki325Gc12XZtFWWpFFA1+slo8QYV5YcAdjp9wa9fhCGm/VWaLVer5frTcM5YyyMkywK4zY5PD7Kull/1IcMffrZJ2dnbzDGOOh6V6j2ueEIOgellHcCHMowolHACMTYKFUUxf7B3t7eTp6nELl+vycVb9v26dffpGk62dk72D++vLzebArrXBhHB7sHO7uT8/NzIURV1w7hwaDXH/Z8jiyELggC3mrBZdPoCrhZVToHBr2+F1crZZ4+ffry5assSXGSMBbOrqfPn70c9ruMMc7b9x8dZFk2Pz8/ffn6s2+eXC9WYRD0ul3JBW9qUTfVctHv5I/uHabpbrHVnW7XB//1ej0MUdM0mJIgjpqrK+fseGeijMGUvPfB+1VT14IPx6OdnT2h1M7B4XpVzBbr4c5ACtMb9q2DpycXEFIhtUFgUxRCqbzTKcoSANC07QcffDQcjV6+fFlW2zzP8jyt2vav/uZv9/fGeZ5+8OF3i3JDKDo6OgLKLK/m8+v5cr6s6vZidv3RR4/eeecd74DYnYybzfrevXtPqnKzKcqy9Pahs9NTTEhdVoNu3xiDICSIZiElLEAIUYy3i81g0smyhHO+XK4228oa0OkNx+OdV29eWwNoRAklLa+lauu6slaTMBoOhwcHB3XdaK032/Lk5OSrr74K47SPaZ5bL8tRSmltAQAIUgOUMZa3WgkFEQnChLIsiDJlnOTcAZqkEYQYQYIxUUZra6qm7ruOBc5oDaFxwBFGrTUQI0QwBcz36joIMEHaKC7apq39w9c455Vu/X7fo7hWaaUUhDgM46ZqhRDAoTxPRpNJlmXWACGUt8xpIa2yCNg4DD0xSahL47gsy+Vyfkvxurque71eEASU0k6nExBa1WVd13dKcs+m+b3QSyvu9FB325sXtyNoIUZCaWvtcDiM07SuawcBxpgwSikNGCGMYYwhdFWxFcWCMRaG4XB8GIbhbDbbbreiqQ8PDx0wTdMsFgtKqbZmuVweHx/3Bj1E8Lsfff/z5/92vi3NxWWY5mGc7B8enZ5frJfrqqo/+uCjqqiMMtv19vXVyWg0SuPw4vLs7Ozk4VuPO1l6cnJyeLhvGQ4ohchVVSVFq7Xqdbu8FQTXxpgcd/wS7JVx8/n88ePHTVVzKR7dfzCdTueLWRAEhARxlOb90Wx2/fLkhDf1wcHefpQlWZ51ehYADwmkWQwccp0syMOdnZ2dySSO0/l8Xtc1pTROM8EVhA7c8hcsIFrrsiyttVEUBYytVitob9QtZVkO+wNt7B0mfCeCSZKEEBKwyGPXXgOBILy8vLTW5HnucSPGGCFEShnHsef7vw01CyGyLAMQ0TAghGjjvLiMEKIt0Fp7vv9OSeAp87v13SdD38wQin9bbXC36Ptf8w76NrcsOMLwjmuw1sI7+gDe/AmC316Gb1L9/BvwONkdAuH//M486T+fo/v3mqZ5/upllCaT8e5wPO7P5+JqKrQyxtzGsSKIICTYQdS2rc8MxgBShge0p5RqqtoHNmCMjVFt2xJCAsrCMEySpG6FMYbSwH+ShNE0TZumsU5LaZWSykhjfQIEBgBIKYui6na7xpjT09PRaKKUWq1WSZx5V+Tz589nszkAoCxLb5XyvOFtNAX03ZUIYvStPk+Mcdu2UjkAldbGWIOwde6Gr2nbFgSUYBgFQcDI3s7kvffeCyieXq+CgELojLPaWWidMU4KraQxBhgDnbHAIAeA75UiRmMCMb5ti4YQ+BhIQoyDQisECXEQY6yUaVsBCIijEFoqRau1TaMwoAg4o3jrnLPWQKudYwha57Qx0iFACHG3ydOe/cFeCNM2FWPMAeOsjChiBKyWs16epWnqrC7LkhDU7/fzbtdBeDFdVFVVlmUrhXPYIRYl6WDUx5QmabxYL4aT4Qffe/9qefHFky+GwwENQgsgItRB6CywFrg7ISjBjJE4jjF2FCNrVFW3om6Hy1HN6+kCW6tFWzPGrDVH9x54s2mx3S6W67Is799/+P7777fKdDqdJInKuirKGjPa73fzPB8O+71uHkVBN8+UbHhTA+iAsYWWFxcXddlIqbWyVVkroQRXCLRJGAguN5uibVsEQK/XwZgYB7hQJ2/OPvnqm1dnF2HaiZMYaksJqrngbQ0UCZAb9D788IMPivXqV7/7rOEtpmQymXDO/a0IMXr4zuOq3O6OJ/P5HBA82t3pDgdhkiy31b1HWTFdTHZ3LKDLzZYF0XI7+1d/+idtI9dlXZV8ulpEYbzcbgBAWYq4Ev3esCgKQLAQelPVmFDlHI1jKZvz65kG7lGSdOLENvV0Oh31BqLi5+fnq/lKcimktNb+9Kc/PTzcn81mSoqQUVmVWZ4QQg739p8/f44ccNqdnZ1pZauyiljAheJla4QMwpg4GJAQWzS7vKrbhZSqbTglYH9vFEWJ0ma1ng4Hed3yrNOJ43ixmnPOAXLOgdVq9aMf/einP/3Z2cX5l198/fXXXxsAnzx5ksQZYUEQBJQwKTWXyjjfuguNBkojLqwUBpKIRT2AAxrkm81qvd4qS2kIACTOIaOh1K1PdNndG1lrm6ahFGLiVyznD+A7X7g/uZUSTcOFECyglHVkK9q2dRb69UsKDazPeDFSyjTrUCaEEFLr5XyxWW2VUoKr0WhUl9W1uwbAw7TIn/QGKJ/vtF6vJRdhGAIAlJBWGwghRRgD6NE/Y4xRN+qEO4URISQIAq/PhbfmeP/09zsis6Ysa6m1d4R7yRUicL1eIwSSJLLW7oyHlNLxsN80zTaAnprx0U8+FYcEjDGmtKhFfT2bDvsDzvl6vX7r7cfzusYYH48mUae3fv4KlvV+ZxTG6fT0jCK6mq9+/fNf/bN/9s9++sM/Qgh9HXz96/NfrWZXo/HO9z98/+p6JtsqzTIrW6CVBYZiGIZBFDKjeCt5UWwyDHw9x57Z999Lsd164P3o6N7l9BpBGIeB1Uo1DTJmvq2EEEGWKdFy6aSBq23z+0+/+NGPfhQlufPsLMJSqdevT3hbF5s1QRBCHAZ0NBiWdVW3XLSNg4hSCgDivLXWhmGojeScO2IIIUme9/v9tqqB/UMk9h157AdKxhiwLgxi/7H7JGOInFLKaM0YG/SHeZoul8uW1xAZAG3La+8F8IoZ/0qSxNsugkBjSiGEHjT2JzEWCt2mKd9p4jxxcHcl3IwUWvtksLsp4dtEwLcVi/bbL63vsPE7aNpa6+6UifgmjviWTb/5HDxN5m7LJvzVDm5bqj2iwBiruKhbLucL/PTZ6eXVdrtdbraAYGOt8Up9CJxXPBhjtbz9bAl0zquI6rrcKgGABdACYAEAYRiEYeCA5aL1ZSLW2iCIrLUFr7Q1YRgSgiChlBKMYStQUXIhWghhTAOvWvO9MOv1emdnT0pZVVUSZ97Bu9lslsuV14R2hxnDASEEWAesi4KQENY0DQKw2+kkSbJYLIpN2daN1cYobQxQ1iprtHHIGGV8baWJoogShKDzXGeSJPePD09ev2l5BYEhFBrpwxWgNcBYTUiIIEDQGme11dA5B7A1IGDeZQmcMxhDgIFDwDh9eHxfCCE4N8ZgCAlCFiKKML7du6y1BGFjXKs5Rg5B6IDRhhtrEKAOQGulUhLD8G6y9F6eNE2BNQQC2zSVMyHQwkGDEKDIiHrTOZgwijlXAFhtXNMKA7Za66uL67qunXNhHFBGlQNVq4JWqaZ5dXYyX84/+Oj9w3vHR8fH470dhADGFBGLEQUQOagdABYCgBEJGEI0zbMkizAE0OmmFUK03aSz2izPLxs/MhflhhFKCDk4OEijGEBgIGJRyLRSVtW81gYoLcKIaRtJrSCm3qcuBAQWMBbGcUzShI4nQUCdMYVWk/E+cM5ZuN0WTdn4eP+ry2lAaJYnbcOVUsW2Ag51uhkXajqdn5ycbVabgIZZkhYNn0/fHB4ct2VJIUyDABmHnBv1e4NO+vFnX52cvAbWTSaTIAg6vW5dlcvlcnc82W7tbLlYrJaffPl5tS2kVpM82xbV9XQxWy46gxFXerHe1I3QxhkHFLAOwZq3FgJAURpmxhhpGsJCErCD46P9o+OnT5/GnU6WJcvlcrZaGWcGg16jzZdPXzx5+Wq9WU464Dq+LpbbzWwVEEoxDcN4POhxzl+/PhGKG63SOOnFcb+TP378zu6BMwCy6JxSWhY3oicIMMVQSgkdilgAHIpokCVZXdYEA8iwEtAZ0+9EO3t7LZfnl1dBlNiVCkIcJyyoA3/3AgDGewfdfn++XDx98vy3v/9dXbfdbjdg0aYow1BBgCCiFjjgEHAIAiy4ktoABx0gDlASBJBEiy0XGha1Xm0bGirrEARUK1BVDQiMtmY6nzn4HmFUFRpTEgWhsVYoCSHElFDCEMYOAGV0QJm22rtmOp2MsbCtmvV6HUdpGIbAoTAkaZR6ATDGGEIShpFPAijK2lkbhqHfYIzyp/jNDsEYi1jAjfTPU0YoTUgcRgAASnzNouOcW2MAAFIKawxCCMKb6iD8LRLRN/fcsYngNlLNWptQom/PsE2xLcstQmg0GgVBEMUBY2w2m/G2bhvHGFOSI0q8/WxdbH0DBYCQUvr81UvOmygI/JsPopAGrD8czNdbGqDr+frP/qt/sqkFRISwMIzi9XqdZR3R8q02bVntTSbO2uC9D9R2+/Tp0+n5m4f3/3QyGnz+5ZdNuQFGCV475whFWRyMhoPd0eD8/PTy8jLsZtZZ46yUcrlc+rjAnckEY7yYT5E1YRStFtcI6IDCOCQn0/rs5NQAd7i3PxwMeVVK7S4uLliQBJTFcQwx6OVZIzgEOmCk2JZXZ+dVXRtjaBhJZaQULgiTJAMAcKnXm6U2zhi9LUvOOQ1hXdfsNmXBD2oIobZtCUReCOwX6CiKMET6ttQAOheG4XA4rKqqrZs4jtMsiaNosbDO2TRNunkHIUewkx6LDgLGWBRFcRy38ia/kkLgZw6/pn+b2/a16f7M9sbIOxWhV7f4xMyExeBbdse7/8IfWPNv9xFDqG9Vbx6y8gSE1hq6O8HdjTLR3cYE+evwTrLgh1ePbznnttutdz8Nh8MkSTSQcZYmSWIAvJrNV6uVcy7Pu5xzYIw1zgGgtRZKGuMQEgw7hIDWURyGAFgh2rquW14TmjnnOG8hhFmWZVm23WyWy3m3N8AYRXHAKDPGRZFRWvv7xc80CCGEASEIwoBSOumPlFL7+4ej0agsSw8IxXEcxzG8TZQKgsADP9ZaIYSz0Bjj6R5vDVBKlWXZ6XQHgwFjbDZdXFxceKDOAAgtMBYY46S2xhilDIQqyzJjlNEKBpRRMhkNhoP+V599Vm4Xoi0hpkZrZxwIAADAWWwNhIhQBCDUCFMIgDHKKg2s0kYboxxyxnIAE+1Eq/XBvf3VarWcLUyjnDPQIYxAQLBzWraat3UU0CiKoJWi5WkcIOi9EdZZA6BBQFljJZCMsDvdQxzHWRb1+/26LEjbltYoRiIBreQVwaC3O+h953EcB3VdS95A4JTS55cXxkEIYVk0SqkoihKWOuQ25WaxKa6Wq+l8WhQbGtLBzt6r03ON2Gj34OLiAlijlbXUIQi0ccA6bK11EGIQBmGURmEcOGuAdcwQGhAah8tiU1XV8cFhmuaboryer6ADeWcQhKkzVmqbdHOWRM6Bi6vLOE7DJEYIRFEAcRcgpJT2GhOCIETOSEEwSuKQMSKadiNVr9c72DsIw3g2XZyfnmrpJhObRGlTVozSo6MH1mpC8HZbUhoIbZbrbVFUlAQZQ8gCXtWy5MvpdV1VvU5Kkni9mH/y8W/3J8M8iZuyujw9V0phiPr9fhQEDBMj1cnJCaN4sd0WRbHZrDabzR//8R/v7O0hZ8uitgYVRVUUldFuuSjH4+FiseBSBSxq25YQttlsBoOhMYYhCjCaz+d//uf/5MGDB6/fnO7vHTrnWBApLaq2QiTQFm02a6VEURSyUL2swyAZDXe6WW6UxYTtHR++eX1uEXzw1n0lbbGtI0TPiqu6ElHaffjoUZZ35/P5crkyxjAaWmdpwBxEGJA0ybU2COE0TvZ39xbb1w8fPEjSfDabFVVjFA8YGQ27RVmLtvWsPOdSa9s2EkI42E2fPHvxN3/38/l8XpZ1nCRSGS5VnCQIYc9SGQctBNpZrYGUyhiHWUCD0DjHQiosen1+XQkrLNYAKwcsIJDcRMUlQYgxXi6XhBCMI2stxjiKIn9JuG9lIHpUEEDrnyOU4iAIwjDQQnoENY5SSqnWliLqnPN5cIzeTAlaWSkEBBhFJGTBdrvFEAaEQQiVlFIIZK3F2GuCCMIhCzzsCazFN5FqwN3F5BmDEaKENrfRqnfLn7ftecP6HR6YpqkXlCUB9f6xbrcrhHj2rC7LMss9n02tRU1TzWdXXgVmrd167yiEQgjtbKffg9b50lspZafTqarq2YsXRwf7o9Ho6uoKQrhZrV+/Ovnzf/oX3//+9z/5/Web1WJ+dTUZDI0xWqu3v/OOaZvzN2/evH65v7//pz/5wcOj3b/5278/ff5k9+hokKfrYkugWc2uCCFRHIooQM7u7u5QAtqmaprWO18YY6enp0KITif76U9/ooT827/9W9E2SvB+v3//6PD6+npydGgDFge41xsMB92Qkel2O+h0jw/vvXl1kiQRAKAoiv6gSyhdL5ZRFO2NR03bOq3WRSm2G6ENC6JOliZZ1gppy3K53iitAUCtFACAOOvUdY0g9FgLZIE/pDnkSRj5Q8J3kiGEKKHW+MNABZRGUbS3t3d+fm614bwpig2vq6JcEQp290bHh0dN01ycnVdVo5QKwxAR7HNCAQDb7VZIFQF3hyf5MGy/qfuT/u7qBTcuksiL9oUQ/kISQnSTmzDWuzSkO9Lqjob49stY821nP2PU/46St7c/fENheF4MoT/AWn5QuJtZza371A9YngiIeynnXGgTI5j3uhDTsiwbzjEmzjoLjXNQOyDNTayhArKqi06nk0S7jDGlhHMmyzJrTRgGlFIuGm2kVLyuy7ZtldEAEQAAFw1GNE1jbZwQoq4b7SzGMAipcRoAl+d5r985GB9IKR8+fOvy8tLjCkVRMMa8H8qPAoSQMIw8KKhNSxgjhDBCrLVpHCNICETrxZIi0ss7SZIM+/3VYuGHRamAhVZpa325M0TeA9HWNW9LAuGgmz+8f/zud97pZimCBmilFScAQOcAgMA6BADEmHOOKcEYAwQCihEGnDuueCdBUjplLKKIheFop4MCRSmWupa6VpZbJ7QxziotlZHaOnX//n2jOsVqDZwKQ8YQiALW8hpY32VtgTXOQmM1AgixGwOnR4z8TOaMJmHAthu13iyBFqN+NwrozqiXRmG32726upqvC6HMpqzX17OGCxpEmIYYUcRCi4hSalvy7XZrodPWKAVZmpycLxfrX1CK6xZyQViMACaYUEQw8dJTQomFLW9REvtpXWiOkQ3iMO91KCGrzbIoS0BomudRkrZCE0KCMLEOXc+uLy/O4zhOkshiZ5DlvCmKLeetttZaQGlUVRXnwWAwIAiGQWStU1ZrZZVqV/PFinNK2XZbLhfbq6vperFlNLx/75FR+uriXGvd6+RcNLJtiqJoW77eFISwlgsIIbTQGdtL84PJ/nq9BAELMGUIyqZ59vVXfxOQNI2vr6v5+SWldJPMgVB5nodhMOr2r6+vD4+P2rZdLpfr9ZL00Hcev922rWjlYrNWSpXrerFYioYbAbp5b9QfaW31PduUjZTy5aIpyEYKlYQ4z3Op3HgwXM6W15fX3iaktY7zFNMAQWosjMKM0bgs2s16M+xM3nn87sF4FwO8XW8AxL3h8Gqx2dYVxbEGoKlViYVsGysBb9uDBw8mh4dPnz4reFvMV0XTQoDTKHXOGQg0cK3gs/kSItLpdNazldo5CHssTzPnnFcJiLYxxmACy6oBiHV7Q2PBtqyF1C9evGiaZrlcsyDI8jwM4+VyXdX1gAVKGe1abQFCCCCMCAUAUGaRARgTFgYWWBYHXJvl2VmtFWKMxYkDSGoBgHXIGaWExkKri+sroRREblMWgNg4ixElUGG/ZFkIvD8IIWiMxhgTiqzTbVsrpYqiKIrCaJcleRynQjTrxawoSm8o3dk72pTVYjrzFXYEYdHyxWzezXNIKKaAYWylc0o6CJANedP6XMI4DjGEPuqHYqSUwgQyTLw02mfp+NP6bv+7g3y9fs0/6z25MBqNvCvVSMk5D0I62d3BGLeiOT15rbUOQ5bGSZxEdn/HOdc0jTOCYOzPjKZpfAt8GidpmjrjumkKgO32e9bas7Oz/qB3lCanF+cxDs5fvz47vwwwefjW26Iq21YoUccBFcJaBpGVz59/OR4ML05ezK9O/9v/5b/sPrrXFJuL6SxENiDW8DqiuGokZAqEVPBquZgNOulo0PvO22+9eXXSNI3WWgjF2CKO416nm0RxMhruTsaff/5501QfffDuwcHB6xdP04h97/De4+MJsI4gymterWfUqNFgnIZBxAIuRF2U2sg0y7TWANooYBgiMHC7Ozt5p9dKMV2uuFB5mrAwgBiXdWWdM8AZo/I839nZ8Z9/GIZW6Tvvu+ca7tT+8LZ5j7HQk00+oKbX6y2XyzSNy7LcbFYYOqVEGIZBQOM4DAL68vkL/1UGQeAgCMOw0+nUvD07OxNCYkZ9yrKfTiCE2gJ/KqPbWH5w61H0nJS9rSr29eJ3RMONI9TctAb7OKn/UqNg8Q3X4AEzAKj/fe/8dcbcOOhuORf8benDHbvhnOOce7K12+36CWa5XFa8shakadpJOzSkIWMtxEpJAjEwABgAgcMAU0QBAgCAgPrmaxCGjFIshIUIRFEohNjdnYSMXF1daa2bpgLADga9TVESAqwFUkpCbBhgY1VZbYUQiJBON+t0Mm1NUWy5aIoCllHpGb3ZbLbZbPr9flEUSZKA2yCpOI45523beoVQwKCHpT3QEkWRc7CN47ZpNqv1xdn5zs5OEsW7k52qqqSUQhmHrXHAJzL6K4fSgFLCW58tQR8+uD/q954/e3JxfppnyXxOrLMEYQeBscoYi4ljIUEIOaAB0CQMCEFCt9KuMEwIkQ4aTLACmoYm7dI4jjfNgusKYkkZQE47rRCwiLo0i/7kT76vuPrbv/6bpl6n0YgworWGzodPY+wcsNBoZ62F+GYcVEo11q7XawD0YNh3zhFCiNWyEnUnSQ73d41oeFVAxXdGg4BRgh1XRmsltZLaQmIJIcCCRsh2oVopylI00llrs043SqFz4M2bmXY2juMwDK1LCKYAIsaYN1J6B4iFVltLCCIMa6ebpqIExXGUZnHIsrpt61ZwJeu2hYh0uv08zabzJZe6aThlQZKlSolys1VaD7KeVNwYw4XQxgCM7h61t8iY8UlZbcvn8+XFap0meRp1OJfbbVmWFbBOKZslqTWuLCpgXVluMXSbdYEg6XaDIPQmbEwpSdIOxjjPumkYOGeA02nM8N6O0VxruVo262nJAIoIM62o3RYbR/rdLE5E3tFcxiyUUdJWpRby5NXr09PTfj4MgmBbldPpbLXc8Ir3OmE3yVTLCQ0iyh4e39tsNqKVjDHGGNAiz7sqVREJP//dp6vZ3CqbZdnr09fD8chBZ6QCDTBWQQykNHmQ59lgONjN0oFoRK8bGms3qzaOekLDpjZx1MnSQRzFB5PDeluct8vR/l4QBDgKoyx/8uTJ61cnSppS8oAEzhqpdSPV8zevZuvl/fv3CWSvXrx++uS5NHpnb/foXr/hgnO5WW+BQ1pZBOmD+48dIF9986Su1i1vkiQJwpZzXhRVf4AoY3ud7vV84ZxjNDTxzcOX0YAQgrpGCGGBQxQDg1kYGOhmy4XWikUB5qzm9Wqz1FIBqAHUymhtzGw+F0pC6NbbjYM6TqMsyzAlTmsLnPRCKoccAM4ZRqhztm3b1WoFIW6rtq5rgpnnMpVSWhslTauFs3C9adu25U0DAIjDIIoigjC89btjAAnGAWNW6SAIkihet23EAu9QhxAmSQKss1Zvt9sooCREAABnLAAOQgicY4z9AQcm5C5lrygKD0hYaz2xba1VSlVViTEWmp1dXISMKaUoYxjDnZ2dw4P9OA4f3DuYz+cXF2dv3rwxxkCWCCHmy4WS3G9Uo9GIEWyMCSPWkTLr5JPdnU63u7Ozs1wuwbrOMGXafP6rXx9Odj94+/GTp8+G777zzTffpFk4vH8AEZhereODncmk980333z2+4+jKHr74b13331nud7wr7k1KmZ4NNyp6jpOojyJjRSr5XyX7Q77fcHVq1evvLp5PrejwTAKwtOzk36n6yO8A0ac1REl3Swt1qsgaA/G46bkq9XScavKzaJqZFU5B0WUEEYZY1GYDAaDJEkgwVpyYE1ESW8wPL5/Txirvvzy+avX63UAMCIUB0EAMVLGcc57g763ODLGBoNBQKgS0hkbBUEcRhELCCEIwCiKQsqUUkZpA8mNmADjzWaTpulms8IIhRHTWmFMkiRGCK1WSwiMvekDk/6bFUoCAMIw9D2QQEj/jPbFPx63UMZ57d4dnuQPfq/4uxMf+B9QtxkJfkrw6bxeZ+fHjv9yUHDghsPmnGOMGUE+JgfcpgsDZ7x3zt5kRWt8m/57p/vxTIQP8rsbWfyRUy+aTqeTxklAGa95VVTAuizJtdYIGOsQhBAi5whFAAIAwhBFUeAv77ZtheRCCGcsY6TX6zFChGjTNLHWCswZY9CbIzDudHLGAiVNu92UZRkEAWU4y7Jer8clr5uiaVpr9aXB3W7Xx9H693+XGdU2DSEkyzLG2Gaz9eORNlrrxn+SHh2M4zSO4729PUpYFEVFUdR1G8cxpXSz2WClMMbOQgMsgNg565y21mIMMcZZGschOzw8xAj86pe/ePPmzWRnDwKrhcZBgADWSmutWcQQgQAYDRSlsDsK8jwRLU3mRq02QQAZAMLyphZFNZXWZCxIOwxiBoGStTCtxBBGWZjF2XivMx4lF+dXWlcAOmOF4RpYCADAADPMlHXOQaudAxBiAm5TOOu6Pjs7m89J3sn63Q5ZLGYQuk6nM+zmURScXJ1q0QyHw6oulqvNcluWrawaWTTSQAQwtq7lXDRN00qhtXYA+GLTxXKTdXLOpdS63+9jHDSNYizwhBYAN/CUM0AoZazCGGNGCSFSSM65xiAICMa45hUimFLaNM3MrhbrDXLIGEcIkVIr4zALCCFluZVSRnFAKaaUBiH1J4S/E7xghCBsjHFGA+skkWVZLhYLZYCUerstt9vter2RXFDMrLVBECRJVpZl0zRFUfQ6HYwpAIhQSghFiCCHkjjtdbp13a6Wy8GgFwS0LjcsQP3eYRSSJIlW6yUaRtbdLBwQIi0Vr2oQmmG39+rVi8Fo6IwhELVV/fvffjydTn/8o58e7B2IVjRlVWw2GOCDnYM8y8ptkWWduqw6WV5ui16WcyGyOPngne9uiqosS6v0cr4Y9oZlXTdVTRFVQgPkGGNGqrKpO91sZ7y7k+EsHXCuz7aXohGjwdgC9/z56/HBXpqCuhKdrOscOnlzUXfbb774/PGffcTiKIrjB289Ojy+n/X6Upn1eusMCMPQahtmiTK2auq64d3eQCwuIYEQwv5w0Ov08qwzX75+/eLlqqijrLMtmiDq5nnXASyl3ayrZX15eHjonMOYhnHU6/W2223NW3+3Q4wQItZa4CBGgBCSd6KqAlIrCB0AFlNioa15ixAISQCQK4uCYoghsNZACBhjzt3E4EN4c2/f7VW3zzWntb51pXMcx56DLKstgkQL6xxo23Y+nxdFBSFmmHpB4mKx4vKmVooR4hwUQjliA0p50wJjZcujMDRKSikRsFqGGNwU2BTFxjmXRDEAoK5L34nppwRjDCY3K2mA0N2W5plsjzx7b5tXXbVtyzmnlLZtq60eDYYY47OzE601ApYxkibR9fVVEgcIDY8P941RTVW+0kZKLoWVUgZBEMXx3SPSWqSlGk+GLAo7aeYfkZvtdrFcdls17nQuMDl7/fzF11/+8I9+/Nknv7t3/N52s9Za/+ynP6aUfvXVFxhZSpHW7vTVy9Fk/Od//uf90ejp85c0iMI4mi1Xe0eHL1+/IoSMRyPnjB9x/BnpyeztZnNy8rrYbAF04+Hgm8uLq8vLIKCD7vji9KzfzQ+P9mfXV5vVZtz7U6D47OKkLflyetXtDoJev9sfWQctBBUhXiBiIVBKZUmKMa7bpiy3L168EFIuFgut9cuXL8Mkzno9raWvRLDAhWFYlmVVVWmShGGohXTGekldEsUEIoQQxSQIAoqwL7Pd1txrSoyWs9kMQjidTge9PsK+MQHHcSwVX60XxXZtjCEkJMT6wUI1qrl9+e/iBgbwtgOMIIQWoJtb4zbYwB/DXqngrwTvi9FaF0UB1B9c+36A8NdVXdd33LP7L15eGIsQkoz4i82PFxBCa8BdHQCEUCkJAGA3PpqbgD8vXYzj2BjjS7E9hhHHcT2vJRdNVVNMva9ESkkw6/f7VhtgDQDAAWch8YGGnFfeCFpVFcI3udG8qSFMPRroE0S01qvlfLUWRVFYCHrdwXg8TtNstdystxufY2ahresaY6iM0lonSdTr9XRjkyRZrVYQwl6vhxDKsghCKISoqgoh5FOFMCY3Salh3DS8bVs/LXmSYjgcGmN63T7G+M2bN8vlMo7jKIrCMAw0pZQCKIHW7qbAVkEH1rIRvOllCaV0bzLOsmy5XIaUEoSVUpyrCBOAsTHaGIUQKNsSAAuIidN0POkd39sFwKw3w+2b6yBiEKFtU5xenXcHHUTg7sHusNdfL7GVoi03nHMK0bCbj0e9wSg+O3/1+adfbLfLg90DAkEj2jztKK4gxAg6hHyGIwQOQ4ijKCIEOWAKztfrNYSmqqrxcECaqgDWLVYrUVbVqkjCYG/nEaN0NptvG1PUlhuEo6zXYRYhB4GVPIlImGdCRQAgCGjb8qKse72eMppSHIYMAW0NQNAKLq6VzvM8YdgZA6yNwiCJEiV5kGa6EuWsQgjIGloIWmiVEnm3b6QELri6WCk1N8ZgTFqhoyhargtCMGNsuqqNxUlvYlACcTAZ72plGQ2XyyXUPMDatOu4GwKr21JB6LIsQ9g4oMKESW5ns+vryyulXJpkUZg0Db+8mAFHMA2ENnHAMGXCyLyXSyU7+cQY3TYqjsM4whi1lNbWCgQQw/H1dtbNc65NYPPVtpDcCmQooYQShgnQyunWtAYgG1C41+/ylkPndCOrdS2E7HUHH7z9QRCFJ69O5peLTp4/OJ4QQo4PjmazmY4SCOx6M6/rcjTpZ1n2/vvvE6l0e6II7Xe6URSpsqU0sg5k3UxKCR2EKoSWRBZRkQyHg6MJHvT689mCN/VwONyK1cXFRd1utUidEARn8+liNl0JpX7z1UvnnPv8aUyT733vB5Nh9+rioizaYtsuFwUCWPLNuDdWFKoNfjR+z1kL1mR0b+/ly5fvvvvuT//0z5I0f/LsRZD39x48xrPlclsARrJBvuXr+XK9baesYybZSBqBKZFaUBJuNiVADCOaJUQZDQFCECNCnLMWOmX1crnodDrWWKM4sqobh3y9GacpAKCpm5yGhoSCqyzLIA6lbm1T+TT7+Ww9GPbuHT+8uDhTHK4XFYSublqtt3EcEwSNMcBIGpPW6i2vURhKRxgKFutlnvUoCVYrPhp10ji9vr5uyioMQ4IBQw4ayxAmCPKmJYREQQ6hM8Bo5zOoOSEEMiCxKXjZljWFgIvGSgkJvLw6CYLgvQ/eT5Lki8+/XG02O5O9kIVVVUsBqMUkkFEUCSHqukYCwRp6HfhsMRuPx/mgu23KTV0EaUSBWW5XSZ7My9X+/j5D8SDLlstlkqXj/f2mrABNAI3+6ue/6Wa5I+nk8K0vv/xy2y729/fnyxVscLfX21bNqt7EQail6nbN4/t7Voo+680uLwq6wFyuZMEYe/DW/evlvK6qpqrvHxwFGP03//xfnJye8UbuPzqaDhZX0+tHjz/8+Pdf/+r54qFNj59dHDVmXdQI6Ht7w14Eyu31D986nM5noFlzwXuDQb1ZxpNxWV0nKQyoPjyY/NN//Gcf//aTF89evXl2miTJW28/kso0Skikf/G7X+TdDMR2drXt9IbDcbCp6zevT7/30w93d/Y51w8fPPr88y94K3YmXcbCarls23Zvb28D0cn5BWOsEUJvuVTuyctLgCAlUVWZVbMghECnnbO7gxHStq2FVaYtGpLhtmw3m20nzZI4a+q6l3d405aqHg9HkICybpU2hIJhP1+tVkeHu5vN2ml+vL97g9UTUjR82ypMGcJMa6C1SwNqCOLG8VoEUd448MXzV5xzi4OsF2itWy496O0MkEpaBL0exc++PgXc1yJ7hMkf6l7iaq0lEW3btmkbY0wYhlmQamiU0jQkPraZEIIhustUyIOQ89pKGVOAkFO8sgrf5KQ5hxB2EGCMfIYCQshiFiWJaGsHbBRFjOI0ctfX14NufzgcPH/1sj8crLcrRPB4Z2c6ne7ujRhjmAELNKbQOCUMZ3FQy0paLY3ylAcOCHTIWQt1F5D47GK+uztBwFV1df/ewWIx25blV19/MR4Pv/+DD/f2dn/5Dz/f3RtWVfH0dP7BBx90Op20H/zjP/3p1dXV5b99c3gwCsOwrlunbSeMjXG2VgEJ5EpLUSysAQDwsiIIDbpdxljTcqco2um1jaAEjIbdIMRBSPI878X02bMXm6KOgkAA2zSVtmo0PsIYP/7OOx9//PE3T7/hnKciTWRirT3e2edClE3dCse11M44AjgQUcgQoIvtcl+M/vbv/vpf/7N/9r//7/63v/qHX3z5xRcJRAhhpOFmUzgSpkm+XVQogA4756RD7WLzJlpVf/zTH/b692Zn09evX1MWPwjv5a97m22VpT0jUJ5MTl9cL+ZFSAPaRaJpWAoO3hru5uH19aybEALldrPoHj/EnAhncEC51VprjAkLmDHKWI0IhUb3805bFqPhkPMqzfLJaGisIgjATq8bM4oRyvN8f2d3b2dS1/XVbO73sCAIIGXcWi2ltsBqASHWWkttIKQQOI9KIYSw78z2dBe6Me0MhkMIISYoCP0ahoKARiFBCHBgtdZKibquKbpJ8fONovCmGAN5KY3gCkGitHDOYYwIQZRhL/c1jEEI0zT1CJhWgGAWR2mWdvzPK6W8zd0YhxARvMCQVFUlGkX2cDbM6rKZz6eMYMaY3ywhhG3bOufqqpjNZt1ux5eRUErH4/F4MjzcP3j69JvFfE4pvXfvXhylwIL5fF7WNQ1oGNA4jBCwgisjDaKQRVQ7LY0EAARRFCdJEATGARaGhAUAIKXMw8dv/eiHPyaE/N3P//4v//IvAUKPH78Tx/Fqtbq4uPjOd77zT/7inx4fH/+P/7e/XCwWDx+/fTWbel3VcrOlYYgI09Ygn/qOAAlu+NTpoknSYZT1rcPd/kQ0tXUIODyfzie7O0672exSadvp93ZHw+12+/U3XwJoV6tVnudNw58+++bNm9dlWXWyLrC44fVi6SRXcR718l4UhDqcv//+hwCRf/jFrwDCo929w3sPpILfPH+dJFmvP7IWfvXlN69Pz2bzeZZ1MHGIYGchcAjC21wwZcqytABSSgmR5KbPxLfRU7/iQAg9BurhQT/vUxL4J6bfxqSU2bDPRaO1fvb8yYfR+1K0zpqTk1dJGkVRAABwzkrVcmuNURGISJRqI4OAluXWGMWV6fRyBJAQwlqw2WzauhVCOATvmOlv47d+qYIO3KRAAnCLEt/Av1fXZ1WdJUmitWrLOozje/eO+91OkqWj8VBdXG22K+AQpUGW5UEQpHkeRZF35PtL0bOnBwcHHlFI09QfD4yxo6OjmnPoQFU0EAIlTRwmQRBRRAeDAQ1I0zRBEFxenb/11lvH9w4//ez3eZ6vlmvnXKeThyyCCZJS13U96HT7vR5GaL3eUAiPj48n4+HVxeX0evXw4UNC2L17RxAChMCPf/Kjb54+j6Lggw/e3z86jOLk2bNnp6en3/ve9/71v/7Xf/ebT7lSv/7dx0+fRZLXWRTtTgZH9x7MZrP1ehmHEZcqiZKmqt/9zvsXV5fdNMEYE8yiIEmTKGBE8KZpmjBiSvK93cmTJ19utsuje/tOBVKKw6N7603R6XT29g9ZEG/WhVKmFfzy+qoV3DrQCm4BxD54X0qNSa/XW6/XCKGQMevUcDg8PT/rdSMMIEIAYwwd9CBT27ZVXQdB4ACo67pqar8Z+2dLVVVKKavNYrWEDnDOjTHdXu4T0P1Xprjwaj7OuUMIImKBU8Y6AC1EEEKN9V0SA759eS29F9P5B+Cdm9HHJ3jY32/2/i15TsFvsR4qa9u2aRpmg7tUgzvDJIRwu936N4wxxhDd4Q3+r/M8F7ytekK3bU93Fzy6qawEEBMWEMmhtdYojYkFhFBMjFb9fn+83U4X0zRN0zxjjGVZRgwIWMQIgw4AiPK8m2WdMIq32y20wDloLbDWWAsgIl6oG0VRr9ehlAJrCEGeeQlD1u/3q6qaTqdSiiAIrNXdbnc87CUR+8kf/aAs6//3f/ifRsPxO2+/9fOf/7zXG1BENdB1XRmtRdsQBDHGQghCSJ7nBwcHfuPXWvv+CCiF/xzCMPTQZpqmaeA6vZQwaByo68poUJfF5dX5ZDIRTStbYa2DEDmHlLJaa6SEA4ZSbEGAMFY3ymwLjE7jxMjm6urqSwi/+8H7b92//7M//bMnT79O0+h6cWpRk2T9iovVusmHfWWl1C2EKu8M63p9dt48Whw8fHRELTm/vDDa5N3u7q5erp5st+vxeO/k5M16vdZa0iiNk8gkYZzFANjxzsRYUFbtwcGBkqAsS0rpeDS5Or+C34qLNkpbZ5zvpUIAE4QQYIwRgqWUDhgymezGIWsIJRgyFmzKQmi12WwsBI2UTStQCAHQNedcKIcgAsY5ZYyV2kJoEfQ9wpBzDrE3zDiEIbituRTSM0kQIeCcKauSiypPsySNnfUx5jiOY4pwFCUQwqpuhFC+/uQGvVc3N4k1QBullEuSKAqTMGTWAq+z2NnZAQBMp1NPdiqltTbWQGeRNVBJa7RoGymFyZMcY+zCJKIGWHt9dbFcLqUQ22KdpnGexAC6NIuKosAYcs6jJGFRaIBTt/IWJaSU8k/+5E8+/u1vBVeT3d2yrC+nV9fX18vlcnw4aCtbtTAOwziK0ryTxHEaxYvFCjHcy/vD4TjrbdZlJS6virb+3SefffTRRwai9ab4/aefAgAoZY8eP66qZrlcLtcrJU0URQcHB8P+4Pz07OmL5ydnZ9//8Y9/8ctfbzYbYYCUMs6yum1uZLrWhIxFYRgEASS44vhqWQeMWY0Pjh9fX5wDeN3yVVVuHtx/VBRFsVwLIfI4Ge/slctVt5vXdfnZZ5+kaTqZ7GKMtbOc84ODnJEgTzvEIYzbOEzyPMYQni8qErDVasaCyADQSleUTSNVrzsMo0Q6s9mWXGohbZp287yz2SywgxBihJBzwDcoOucgxAR7+D3AhPriE+8V9EcyvbGY2yAIkjjy8asEI6+v9q3WENgoZJtN1bbt55/9rtdNPV07X1wZo/r9XpJGjBFtAgAthDBwuBENhDhPcgcdJrAs2v3JkLcaI0Ax00IXVckI9R02ADh6CwLfVbN8a1DwadPo7gHktNnfm1hr8zyyIGQ1mEwmB/tDY1slURSiXj9VyhgNMCMAqqriSdwNCI16fadNWZZ+EGmaJg7CWhsMIMPEAKW4kJhELOjlQyllU7dRyNbLLaXYOUfR4mc/+2Op+GKxyPP09ZsXea8zHPb/6m/+Y5jmz549ozSoqmp+NYcQZ0naSVJndLUtrs4vZleXWRIG49FqtVquF3sHRw7ioqoODw8///Kr/+f/63/6X/9v/s29+wcXF2dRkmKGGQuttdeXV9Od/QcPHgwml2/evHry4km3k0WUtHk+HI+y3oBQ1rTcaD3p9FerlZYiD+Nfv3oTdfDPfvanx8f3i7IhmAx63d3JcLsp+t2MIPf9jz5MInJy+vLe8b5S8nK72f/gw/lyJbXhrWi5hJhML68uL67ffTdEBLOAGeMcBCxivp6YUro3GFZVNV+tZovVtmh6o1EURcYYA5yvViKYWAvgrdHAb/BN0/igJMyoc44yxoXAACKCvf2PMeZN6h7bl1ImSdI6sFwub5ISwpAFCCCktTbWQUI9MODHDv+/d1bYW7IM3slX/XPcv1U/N3iDw82lfqvB8rfPXU4X5/zusjTGIHBjs/QO+5u53AE/glBKBed3OgZwm53gp6I71eRNaISXSULMMKGYIAQwgQihgLIkja+vr9955529/Z2L64vRaDQajTBFnTTZGe7Fcay1mc1m6/XaWWAtLDZly4W1wFoLDNDWagUwBoQQBZTSAmNclluMIACgaZo0i3hTZllW19vNZl1VW0ZvnE3f/ei9qqom4/5bDx+8efX89avncRB28+Rgb2y1mU7nSjZRGIYhgsBAALrdrnMuiqLxZKKUury8VEp1uj2/ZnjnyF1kGcbYwjbOgu4g08o2TSWlhk4v5ldRwDbrZVmWWlsHqbYIamA0grJFiFBKIUbYIGoMQIggwNsaQQgQ1do8ef7sP/7VX7t/ZD768IN/89/+r/6P/6f/c7ZZZt2BRcFOlmEWvDo9gcTkUdAbDf7xn//01etvZvOzV69eTHZ6D47fSfLs5M1Zq1RRNJvNGkE66A3H4zGGriq60CitBLUuShkO8X/6T/+pLOrZbBmFGQvZ9eUMQaoVsMaCm3YPB31KNPDDK7wT6gYhDcNQKeWAIZ00a9sWQnhweBwGdDGdKa2zXrcsK+tcqyQGwCHTCqGNYSz0xc1GO+MABM4h680wQogoiZMkYgEFAGinAbAQOs75aDTq9TpKtC0vOa+lAghahA0EgGDCAtJBPXJbRGbryhogpdTaQoi8h9UYx7n0uhWInHfk81bWTRkYvVqt9/f3b2o8kQHACKFms4X/bZ2FRmtjTLFt6kpkWdbJOjqyUsper0cpJRidnp5uixWGttPJAIBZljVNE0VBHMfD8ahp67Zt0yh88ODBW4/un528ubq4DBi5vr6ez+evT948ffZiNpuTgEEIr1eXRVFA6w72d78zfnt3skMxccaUZ1UcpyQMpDVcG4dxI1WxWMfh+YPH79StlNoJZSilk92dnpTr7cY556sFnz198dv/H1f/9WPZlt4JYsuv7Y+NEz7S5828tizLsuib7JkeUaORoO6XbgECJAz0ogcBep8n/QkyDxLQGAGj0Uz3DHum2d3FYpEsslhVl9ffzLxpw0ccf872y+thRUSV5uAiEHkjMnLHPmuv9X2/72d+9au6rgeDwWq1YgFvpbwYXxZlxaKUR2EQhovV2lcJymhrtScoKSXjrD9btr0u4zSMkkHdHlMWH+zfe3D/zt3btz775JNlvKgAdEIwhDiELInLsri4uNza3n333feTrF9X4uz0AmPqtDFOY8IhBco2yjLEOQ1CSqmQZrjVw4Q1rfzy6QsHICBUO9RKXbYCQhiFsQVOKgshdA4C55cjBg5jCCElUYQQwX7zcshrtY1vYvxHSikGV6ZDN0v5agQIIUIoSZI0iRizaRxbrRfT2bMvv9w92EUYBAEDENMAaStkXVUtkLI1xkRRMNraY4xFUbR3a+/Vq9dNfQkwSNKI4SBJuutlvpgvr3KbihpCCJ3xhHOModU3dHTTNA0A1ssKfl1AWIexZpxoIyF0nSxOs6jXy7a3t794+kyqhmCXxJkDSCkjpdSynVwKislgMOikmf/5Qoi1kJPLcRAEFJNaG2BdwLgzdnI53tyJIcSyVVEQLxfzKGBGcmDsy5evjVHjyUW3m6WdLOt2vvaNr//RH/+jTz9/trW1k8SZ0+a8upStYB3y9sNHs8mlEW1V5JQSiklVFev1knHKw2C5Wllrv/71r1ei+dnPfva3f/fXnV6vKCpj3fHxYbfbDwPeybJ8tVZC7t7eE6odX54n3V5IidRqVdfdshn0ho/fDQ9fvX737Xc+//RjHSeiKkVRrIuqXC2j+4+GBxt1IxnBnPM7d297k2ZK0KO3Hgx7SRJxhAHQ4nIyret2Y2NjPp9rZe/evT8cbQnltHXKOAu01lYZp63VWrMwCILA60vzPJ/P52UlkD/XhbQQeMsizJm35kEI8SjGjENCMaYOIGehAwgzziFqW4kxwRDWrbDaRFHCOV8sFr2sI4QwGO7v78m0vbi4yPPcpzV6AB9jjDD0hYLV1lcJfubtp/tekuD5NJ6Lja6N/z1DEF4Jep0X+nvTAg8Y3BAIfDsutfKPzJVoAmt/BMaeLwmAbx9vFBAed/QwsL8w/2TdsBevKuBfWyloCCljhFEaUIYg4AFLVDSWommqx48fr1YLoZU2EiBqjJGNyOI0CPkSYaNdyCNKaV5Ws9kCAIAwhRBB64zRwFgMYKuldRo5W1ZFHAZpFlurgyBIkigMeZIkVZVTShVB1uokyYr1Yrlcvnrx7Du/9b3vfedbf/93f5/n69/50ffjMHn58mVZrMI+GQ23Is58El6nt+HJqvA6KsxzmMB1DpYQYjqdQowwpgihBq0QhZubI4RQVeeylXEcrleVaMq6zEXbAAAQIsZCbAllRGtJKYQQIOgQgAAiggmlOKCkKvIgCA4Odo9fv/7Ln/71arUiAf/Rb3//w08/PhtfzpZTzMIH7z5454Ov5XXxiw9/DqgLI/T222/tHfTrZnlxeXR0dDjavLW1t7sqq9PDs7bVvV4HQ6q0uDw/nYwvlvM5oaCXpUkcLMvl+tkyEs4A5yAcbW/10iFnMXTYWVysCm+bAZHDECAIAUAUE4yxA9abcsVBnCQRRI5zSrS2VjsPARkLDASEMRZwW5aIcR5GABEDAJHIt3hK+lLVWYAgNN4SASEURVEYBWEYEoq1lj6SCkK0MRrdurXPOJlPxqPRxsGtbW2k1cY5g4BXmRtnoZCmqSXGmLNQCm2Ma5oWOA+IYedu1DsAOGSMraqmbevlcmmK3D8Yp6enq1XOOackBA4VeYWQD467au/attUa1Osq4XFdVUqL995+/O6773788ccvv3oipQRGAqAZYwiB5XKpVCKlbqRYLBbGWR6FaZwM+wMl2jSOJ5PLpmnKujo+PXn55nUr5cbmJg+CxXqilE7iKO4kSS/lSWSVFlJVQkJqLqaL9fpoNl+t8loaVAoTxAkLIx7Fcdbp9AfQAQBxVbcQ4CxLs7TrEbzxeLzOl/1BN4rjh2899nz1RikPElZN3e/3HYLWWmgdItg5J6RsRSNd4SDeibKA48ls8frNYTeNvv319w/2dter5fn5eVVVSsiLs/M0TWXb9rYTJRQASEq9mOdC2bbRjdCyqefzeRTwzY2Rs9pqaYAZDvs8CEajUVFWSqlWKL8cMKGL5TrtkSRLaRi1UgCIpJTrsggIse5qbQAHAXAQIUpZ20oECXDIE6yNVTe6L79DEUJ8zSuEyJ31egQHgZ/OeliLEuq06KY95HDAwqdPnyVpxCM+GAyatkQISVm3ShCKjFHOOenUqpgnSbYulo/ffnB0dFKVjZBNOuhgx4KAiTAMggZTapX1dqzIOV8ocE6BvbJ6dMbVdQ2hg5yD34jPIRDJtu5maVEU2qiN0Wg+mxAM3378OI1C2bT5ag17mFJuteaUJcPYq7Ywxt452NP9kiQpisJvZ55H5qnpxpiL07MsS502SRjljHlPaAjhF188QQhMp+Moi/b29r58+qw/3Mi6g8OjE855FCUBZYzw6cUEGqeaNqIcQ+SsRcBqLZ3DWqu9/W1KeN0IpYUy8oMP3quq9avDF51lZ3//FmfhclWcnpaD/mbEg9VisZwtR1t9LfakaIIgQAAuF+vxdAEB4WF099Y9Jc3O3i7F+PLyHAF4sL93MT05enMc8OS73//tMAxXeXF0cnrnzp0k6xASvH5zVFe5knWWBN/77newBb989qIoijjpQsQg1saCOMlu3wkZC1qpIMCMQ+egtRZTknWz2qFnXz5ZLpfOuYODA+vw6eWlB5YdcNYaKSWGgJDAWxhxiLWxSmpEKGNca22tYzxwxmJMHIRSG9FKY4yQCmGyWq0uLi6apqEEZVkWDjeOjo68A7qyVggBsaU8oJRaiIwx0FzpG2+kB/Bau1iWpVeK+kRy/0b7/v6mFLDXaZB+uuGnCTcYMsaYgCsswTnnDZ2MMU3T+CneFQhxhRpYpRTB6Cr+57oa8EVJkiQ3Yt2bqYe1FkDkuCEYhYxRQrTWwGgA7GAwkG29t7eD0Hd+/ou/e334Jkri9Xo9hpd3794djUZt25b5CmUZIUS2DQLOQYSgIwQDZ4S4+vkkAFK2DBOPKDPGIDBVVezs7GAMe/3uYjZGGOhGb21t3b1356//+ifDfq+tmy8+/URryyhBEAYEZ0nUlFWdr+nGZhSERuluEidJ0hjkTS19ULWvGIQQURRJo03gfGUJEIxj6kctYcx4FCIH0jhpcRswqkLa68QBowQCSjFA1FiACGZx3KrcOQON00Zr5etQiCChmBsWcIqyrBtn2fF8/vTFy0+/fNLt0T/6x3+yKMr/5r/777MguRifgafwB7/7w7c/+OeNKl+9fipkVdf5cKNvTTOZjj9/+mUSxXGaBHGIqAlZqFrb1nUj5Go1b0V5b//Wd77zre2dzcvLs1evXt3ONq12rXg+nc2Wi3K9KLKshxwEAHiZCXQAAgetgxBgByC64roq3RKSxXFMKc6yhEwuJ/1+zzk6mUwAhBA5QOh8PF2uV8oBGkYQIGUsxqaVQpaNcVZr4xz0jrzOWYQAwjgMQ4SgkG1VS6WEdppQTCmlFBtjJpNZvlw+fOvuW48eSNmslkuttVF6OpmtVqUDwBirlGGUYAwRIhgTSimjQZp2KOXu2qvcoxQIX2mFkyQzph2Px73e4NWrN6vVatDfCIJIa5skmVJGtLJphDckoYRTEkUMzadTIcTW1ohC8Pyrpy+eP4FAc4YgMtrIbpxZa1slQVMTHiiji6pijOV5/uMf//jpk88RsO+9957n30IItbNBGmMbagREWwOKR6Phwd7+7ta2w2Q8mSmloMPLorIoIBgUtZGGEJ4ECQwFCKIsjDub2/urvC6qpm3bTKogCClny+Xy1atXnW76p3/6p3/4+38wHo8PDw8BANt7u5fj6Whz0yJUtVqDVmkDMHLWAgAwo5hiACFCOEShVGq1mFxGdNDpnB4fStGyQYYx/vFf/Iezk+OnX345GAy63a6UsmkaB+FoYzvgKWWJVPbLL55fTuZvjo6Xi3Wn06sbFced/nAIrJ2MLy6mk1W+jGIeJuHuwc7m9s50Nv/wo0/Kuto5uLUVbkpjhW6MA4ThME6VUmVTAWet8csGQWCdA8AhhIDRjlDnnDPKKus13NC6K7PbKwkWhP4EbZvas7g9iuP7ISGEVgACiyGLgpTR8NXLNxsbJ5yzg7t7mEIHQKu01gYzRBhHCARRVFRrAGxVVQ8fPL51a/fi/PLybOacK+pSCCVqra0BUhrjLATWGobRtTaa+v0dIWSMb0d+LWv0vjQEok40yOKszPPRxub3vvfd8/Pz14evPv30U9FIb8YHjAUIaG0RckEQxkFY5YUW0ru/bW9v72xubW2MLi4uiqJo2zbigbW2qirRtsgBY1trWMBpJ4uA29RW9fv9yWTS1K3DoJGmHK+SuPvq9fHF5QJjPF+sAs6X83xzuEEcjMOoEycnrw+3t0YMYStlvl6HER1u9JSVYRJSEoH5jCDmEBxuDnnET04Pd3a/s7u7ORhsfPzR5xfn51qaXqf79OmLT5pP04ORg6DKC+QQ57wRcrEsirKGhMZJlvUGy6K8/+itqq0YxvfferjIFw8fvL052rkczy8n03VVVUK+Ob3Y3N9/+PBBW1dffPFFwAkwGYKU8+hytpzP58qhW7duhZiejcfW2sFgA3MGCHYAYUKcc1o4Y4xytm3bPM/djTUhgFrrbtaDAFsItJb+mNSa2OupvDFGKIkx9o7BxlljDLCOcg6sFcawMPAYtT9967p2zrWtZIxtb2/funVrsVhQSkXTtK2E2CBCEbbKGiEER+w33Y59KXxjz1DXtT/j/fcgdPVQ34AQN00w59w7JV85hV8XB5gS+Gv3JBbHsX80fK1wNfvQ+uapAc56IaXX8nimAvoN6Y27FkD6m2OBBdZd1xlCC8lQhKCDwM6ns+V8uru3HX4WNk0TxSGEYLmcCrEZhjyJA2dV05YYo6pYdbqJUto5BxFwDiFotNHOOoxDb+6cprE1SikRhcF0Ou5m0WqlN0b9KA4QAkobB+x6vd7b2XfGIOvOjs/evHljret3e1oaBDCyrt/pD7p9VbeirDujrSiIX3z1wt+HwDqvy2iaRirtbyPB2E+RILgykwiyhBLSCima1hijZIudTePg3p19wiKIDIIWEQAcBBgQhpEBADpjtdMGWgMAgto6qAtRxSGXUpyfX9aN6I9GQRJfzuZHp6eP3n7367/1rVenp2eX41a163L91VdPf/sPfvjW/t3dg43p7OyTTz4LOLp1e/f+vQdPT84W65UWmocBhBJBlKRBlkTz6Wy0MTA2SZJwXS7cpWhFHWds//Ytp13dyMNXJ/PFQisXSBkHgV8A1hjkLKAIeJ2s8Uveer0YhI5xAgCI45gsZvN+v9/v91nAAbAGGETwui4XeREEAWFMims0zAGhDcQIAAQhQJgAgPS1oay11ihrpdFaWieBb6igK8tysZwtl3PZVsYqKZu6KZVqCCEIXOG0QRAaZCGQYRgLU3oqAEY0CKI07WCMpVABD431HB9nrdZGBkGQpqmrV9YCn0DaNrJphLXAOYgg1dq2bVuVTdtKhFCa0iBg/U6yWDlGaK/TXS4WL149Pz4+imLOOaeUM053d3dXeZllGcZ0OBymadqIlhACnX3z5s304gwjAKzb3B55XzAeBFvb2+uqao1ilDMOuoN+2ukaAC/H02JdYojTqFNWCoAmy8LuYCvO3GpdKrMyjmHO11XFeDDcGHW6WVPVJycnRVVxSsu6whgv5quvvvqKIOwzcAG+MlkL4mgTb82XBWKMBWFdt0LK67ZDl0ohBxinFOL1enmia701irnd3hyEAfnq+ZMvv/xcCbm5vb23t7e5vUUpNcYJIYq8WS1L4LAz4Px8usrrJO4DGFoLkrjX648GG9ucQkzJYj6u63p/1Ot048dvv/v2228fnZx+8dWTStaU4YAHcpUbYy2A2hhqqEMOIAcMck4jhCnhEGKtjXHQU1IwJgjh610JIQSAJZD8WiPOCfYyMCmuGin3G2HtGGNKkJO2zsVgNKAYY8xPjy60U/PV8oOvv0sIoYQba5W0UrfOGaWttpVSarlcAYfSTsI4VVbNl7OAppeLiZWW09A5SBAOw7BtJcbgN1kIV53WlYDt6jQC1875AAAhzGy2qCt5sJ/e2jsYDjaMtvmyXKyWRrk4yCgNjXZOWwNsUzYQQm8V5zOEbrhmXirpPdv9VzHGZVkO+6lzlmJAkNvcGq7Xa8bY2dmZMcYAaC0IoqQolBLozavzbr/3+PE70/FkNp0apYt1ub+9/aPv/+Cnf/HjOwe3pGiMajcGvW437fe7y9UUQvfFl19eji+6Wfr47be63SwIeduq09Pjhw8fDof9MOL+/frggw/G59Pz88u1KUeb26YRErdxmCRxRlmwXC6Pz87jJNkY9qVR/bamYUAw6m8O4qT//gffiJLssy+ffvjxZ8u8dSSshFaW0KjDghDz6J13HychXyyLspZ5URZltf7qeRjFe3sHQmmMMUDw8PhoMV/dAOlKqfW6uBhfZr2tq+mDMcvlsm6UTxJyFgKMlCJ1XQNr/JkNAECUEYQo5W3bSm0opka75SqHAERBaIBB0PT6QyVlXddKyCQIjDFpmoq2BgAEQZBlGbp24PaWNn7BeHYq5eSGe+tPaP84X1W6WvumyMsTPB2BEOKJijfk2aqq/Pf4ccPNagQAeDs/v3j8rQC/YfTp/1Fgr8wD3LUrM6XU+2PeABXep/9a3A48TQFjjLxjtWwFcAQiAG0Y8iSJ6rrEGD59+vRb8bfiKIzDAGMcR9Hi7BJYNeynQUhW6xnnfDAYpFlY13VRFI1QEDpAXMCRYYxzjhnVGlqrOU+Xi4Ix/OD+HaWa4+PDjeFgZ3dza2trvV5ujjas1efnp07ai4sLUStKaV3WjAVJmA26A+QQRbzf6UdB3NYNApRiNr6YnJ2dYYzTNKU88LCcb0G98BhB4jEbwmjTNGVZatiJorBSjZI1IZRA4qzt9zo7W1urPC/yeVWv0z7DmEPiDPBZGxYAhwnCCPlz00ilWoHiUAgxnQulRKeTQkLWVXU5WYxnP8t6/Vt37j4/PB5ubX/7O7/ViPq//C//y69/84Pbd3YppUmUTqant+8cbG/vLrQ9PTrR1oZxAB1kmA2yPkYo4kypQSuqui1evHgOkOmPepubm9PpdLlYT+azdZEjxLIsFq0gUNzMoZyzBEMMfx1bDaGPHLPXO1jjgCX9ft/7kW1sbizz5bPnX41n4+l8xlnIAo4QaWqplEGQBAGCEEmnnXMQIEwpAOiG8KK1hhhgghkLEA4QxYSgG05vHEcY6rouv3r+dLGYQWAxxpxE63UhWs2yCGMMIUYIBzSAEGltrBG+c2ob4Q00hBDGaoSAtdo6naZpv9+jlFtrO51Olnal8Lk4VAo1n88BQNYArT1vyBljnQNSyr2dXSGa1XJprLh9cBCG7OzsDGKMEADQdTqpUCaKIuegNz3M8xwgdO/2nfu3bw27WV0Ve3t7t27tb+3uxFmntzE8m06fv35dF7KbZQhjbdx0Pp+7ZVPVWqg4TCiJHUTLdaUM6fdj51ArDHC00xtKqauqcQjXonVL17R1nucbGwOrddu2/W5vsVj8wz/8w+Ry/ODBg/Pz842NDcpYp9cVs0WvF7XaAUK2dnYxYXmeCyEQQm1TLRYLqzQhRNU5J4AzTIDN1/Otjbsh5+OLs/v373r5tRAKIBinWVnXwGinzXK5UtJJZWezlTIgCFOthFKqEWa9KseTxXCQbm/v7Oxsibbe3+/t7+/fvnunEXUt6n6/G6VR21YEuNFouHfrQDvw1cuXy8UaYsQDbGqHECbEO9ci4IyzDgCAEL7ey6xSCiGEELDWxknomWXGGEiJt6KTovXE7xtQ1I8DkjhUtZnPZ920b4DZGm2tr4LjV8bp4Wa/081YgIOIhyGhAev1Oodvvqx0dX5+0dTqg3e/aax1zs3n8/t3h5eXU6dcEMQYYcY5QdSYHMMrWpnvKa+mm9b6uuY3mz/nHICII+gc4DxcLFZ/+7OfB0HgHCSEikYyGiBERCOrugEAURJ6Po1oW0oIo9RZO5tOgXNt2zrnOp0OZ6wsyzRJ9vb2yrJ88eJFFLHVaqW1FLLqDrpCCILRdDrd2t2TQocR7/eH08mMh+FiuRLSfe0bD+u6Fo3o9fqXJ+ds/9b77773s5/8ZH9v78VXXwoh9g+2h8NeI5ujo6Pt7c3ZbDafLZ4/fya1YMEfPXr08Nvffv+zzz77m7/5q/39fUIIw1jUzdZo1Ov1ZrPFnb2DrZ3t1WKpjCUId7NunCatUq3UeVlB5Dp3b705PulkSZnnNApu372Xl+3Z5XxVVo2yk8Vyvlht7e4ui+LZ8xedNFEOvv3+1wMEXr38CmH+ox/97mq1+uijj4qiaq8Jeqcn5+p6DOeFf9baPF+Nxxf9UoVJWpZlEMd9ytG6CtN0XeRp0iGUIgS01kZJdP3yfgBexaC1Dhg3wJXLpVZqc3MTOmCMSZJEUVrXtVJKQuC02djYsEb5UsDnae3u7nLOAcQQe2E9glBfb8S/Nuf2taZ3tvAHtj+V/e8SRVEQRb+uR631swNvou9RhOvt3oFri6QrFp61/rz3p2AcRR54gBA6cMWBcM4h6HwRkKZpp9PxI1ofHnFTW3gkw1885xR610gHAKFhwDqdThxGAadhFM3n89Vq6fvG+WSqne100/6gu72zKYS4vDxDiPT7PQjB5Vg3DYKtAohgAoOQeRGHQVQI0dYKISBli3H24MEDQtA//Orn1vastZ1OWpbr999/v23r8/Pz559/FYdxVTVNswhZSCk/Pj5ta/HwwVt13WppZKuk1GnaCYLo2bPnvun1FGkppR9AhFHs4Rxnr0ZC1FkIq7Is87YeDnrAmoCgQTfzDfGw1w0jPp62i+Usz2XUVYwHDlvjtIUWOQQhJIhAgoCxRjtjTBLHFBOEcBgElFKhTdmsBmX15//uP7z33nvfvXX3wcNHz1+/KerqyZMnUov5ev7Tn/70zvH+t779wbvvvvvkqZ3PVr/4+3+go24jBYJXMR+QQCnlbDoNGF2tF1WVQ2IpJzTgURQRhk1rj46OLi4m1bq5vX9/a7Q7uZiGLJKNRACa6z4fQgLhVdQ4QgiTG0GEattWKUXm9fJueJ9nwTRfTKfz8XQ+m636/U2tdVMKZfxwCxtjlJEWWuQghtgYo4VEHlOzVgphjIqiECBcNaUQbRCy3qAbRqwqW9FYZ4k1dDGrlRZNU20M+xDhr5697nQ6e3sHRV5ppaOY5fkcABPHcbfb3dyKeBCdn19O18ugE2utKWJIA9kKqxXFCBoji0pJu7GxgSwb9kaccIcgIWRt1wCh2XRBKQuCpJatVhK3oqjHZRjfi3urXLw+PL51e/fxO3tcOTVZFMUaQ7S/O7CGNHkr1s2t/dvtovpo9cnDB4+Pj96cnJ1ipy8uwf7e1pvT1zwh9+/fr9uKsqD/6IGRYlT2iqpyWoaOcqMBQLPVspf18sUyxLxcrThhFmNdzAihMazD0CVJqHW5OHv58NFbP3n+BSHszr0HlJPzywtr7cP7D756+mW/19FSVMVqOb/cHHbuPf4AIbIuSkIosLCfZPcO7u7u7mXd/mQymcymnU56fnmBgOv1ekVRCFjfufNWHMdJlF5eTsaTMgzccPSgaZpinZe1jsOwrcFMrgEAbd2WpFnXolzn1hijq7aqbLPssLASlVZVxoeXx8fI7A8ebY/HY0yjk3lug5Wi4ziO4+7gT/7J/+zLp0++ePKkPxz88Ic/HI5GH374UZ2v2moNMQJGG9RhnCKEpLXOaUgRMlap2gAJIMYEMg4huso1MABYIQjGnSjyQ7X5dJZlmedwQQgB5mXTUkqTThcAAAgxyFjMP/nyKwjdg0cPWnMUWguxm03XX355+ODR1mI5/Ud//Pv3Ht7jETs8fP2rn79ZrVZxnJTLVbX6NAx6UVAqUVS12BhtAYMAQAQyCJAFMMkGShUaobWQCFiHCYLIOYgIc0YhRCBC2upatUoJRHAQBEAZ72vbztrFerWxsdHpdKSUACBPROecY2gBAEbntXKiBTvbu23bSqG2t3Z87GSapt1utyjWSZLUdYmxu3fvVlmWdZ0Xy0snqijJYoaQaRk2L776MkljxoiDQCh5Nr50ENZ1bQlZt/WnH38lW5mlw8uLWRjGxtk///P/sddPP/74V0nIAgYIdL0knj4/STAhyp4U5bgoFaCfvj7vfn64vbX1znf+8bwOf/XxJz/4/Xb71uNffvTV1jCtnehuZffYrYPtpL/Bp1uBhSBOQaNU1rVlKZWp1znsdm9/+NHPfa/92z/8nZ/99V9PXp3Opovv/fCHFJAQU2zAre29um5so3Qlwu7gg0fvIQMWi/V8suhkSRZHqm2+8cH7mGKMXBSyVghMwLLI79y+1xsMl8v1dL4qi5qnarI6WcybLdph2VarNEIIIL2arvv9vjNO5Y00miAaxwnlDFECIPR5fNYASnmn08MQWWUJopiRYl1ShBGAy/kKWMsJZ5hB6FbV4uxy/sH7780XBXAnL18fJUmqteEshEBWTWuMi+KEE9w6cEO7uRr2uat0ym63e1MQ+DPAc8z7/T4Oo8VkKqoaYiTb1hgj29YmCaWUEAIQBAhihP3OHg9Gr18873Y6PIzrMldaIAf63YQgLIQKApYkSdUI5wDm3DlIQO0gAAg2ojXLhdZaahXGkX++lFJOA4AgxEhbo5ra6chajRCxBk7Wi8FgcDaePHp4f+/WbWd0K5qN4XAxn/bCCMXJ7t621ujTjz7/z/93/wep5Z//+Z/v7OxU9bpuciFrpVtAYBRGjIdKGWMMgtgh55zrdrtVvt7b3HZWnbx8eXt38wlB3SziAVrlLe+yj1589PjddxLUad84igPOs3ICpqXIQGAMHC8XX/yb/3Zvb2tnY1jXSxKHNGXni2kNAIZwd28PADCfTnwwWFnV66oyFihthTKttUEUBmFsMW4BOLt41dl4f3xxKuqCIbTR6fz2D74HrEMObI02h53eql8HmEYsWBUFQ9gq4BkgELogCIIodc4JIaxzhRAGgiCNZ7MqCFgWd6Ikzs/1iyfPf/C9H37j3ffXi/nRyfFP//ov337/nX/+T//ZF88++zf/9s+OT179L//X/4t7j9758tlnhZIbQPQodhYigtmgUzdyXq9REl5MZ0KKwWhDNWVZrKBFs/q8w7hU5vd+/4fVqjh9c5byZJAMhgidvD6igBLCaqC0tQAQS7EDoARmFMYAOWsQBCQIImdxW8mAxCQIAs75cr2+vJx4Hg0m1OuCICbQKgCAAU5bYA24ojFa69x1VLQXMQJDCGKMIeqIIcYQjK9KkiRJ1+t1UeYMIx2ZfF06YNKk0+/3m0omSRLwaKFygkmn07EWzGdjznmWdXqDISJ0Op1TRBjl0AIHHXQQMGghAs4oaWvbbPWGUsrFYuF1zJDgKIpGo5HUqm2kEFIbSQiyxjlnoigGGB4eHa2KFUBQGP3s5YtW1HEni9O0aRpp9fl0fDmfLvI1PD8r8/zg0e7Z2dnVWItzAuHF5SVn5MmTJ1k3q5oCqqbH+1HMtBUAcgyTOE6jKHIOUIhevXwDDKx4OOh1nAG7O1u9Tn+1yikmUsrTo6PN3VHTNF7lDIBcrVZlWWqtNzc3nXM8CueLxc725t6tg29+81sA2Ebi+XwuWlXlRZ6X3tY6YAEAyCiVxGEcRbtbWwQjYxTFaPf+dr8/3NwY1XVb162UcrlYn5+fO2PW67Voasbpzs7WW/fvMcbyYnV8eOicK9f5bDwJebS3vR2GkRAKOm9EgZRo8zyfLxfL1RpRohbLxbx48/o0jKIwjC1wq2UNQfDm9SnB/xAl2eHx0WpZAUQpCgDSjtLf9KNFPmrWuTC88sF1zmF89T3WWiubgDGjVNM0AIBulmIIVCu0kJzTiLPRoE8IoQiu12shxGJeWGut05yzfDVXWgjRYuKiOBkOw9PjS6HBcrnc2to2RjPCZ9Plem1FvW4roxWMghhCmGVd6BkHGAIHHDTQOeucd+oEzhmttDHOWAwBQwhCIqTknFLKKSNOQU+WvJKoIeSnv57EDq4Tou11wI8Hjf2+b630+IHnWBljbjz7wHX0cF3X3o8ZIaSVtQ5aa+tW4KYtyrqoGn8ICaWlMgARRAgEGBOMEGqaxigNgYPWMsacc8poHoaz2WyF3Afvv727uzscjV6+ebm1taWUOj8bLxaLTpZ0u73peDqbzIb93jvvvIOgOzl6s//97969e/DZp58+fHAvTvj2zlvYCGttJ8vm65W1ut/NHMaj0XCxWFhrX71+eXZ2hgAaDAZVse52sy+n406va4xe5/PlejGfzxxCaZoul3OM4eZo0DTl0dHR9PL8yRefpUnUvX1rOp0WVU4pBcDmVamUIpQrpSbLpUZUSQMRwSzkUdzpD+ftlbufFOImcdHvb/7MBtcKw9/kGPpuHkKohNTSm74aCIAC0FPAgLW+z04ijhEMA97rpMN+z2pZrJdNXTV11e31OefG0FaqtqkhRiFnN5OpG+WO/+j7Nnftr+DnDj670scbtm0Lrs08PGbg5wgOAv/N/ler1qsoDDil2BkMol6WxlHojBVCKCk4wUmahDzwWndjLLS/VjSYa59/j0bcwAk3F2OtVVJ5Ldug1wcArFYrY9ThIUnTtJMmCKHlcgkASJKk3+9vbm0s1/WPfvd3/q//9/9bmiaDwWA6n29ubdRN0+/3DXCuqK71HdQ5hxFZ10VdlqtFG4U8iiJK4GQ+m80vwyjKyzLPc855kG58/uVnx8fHW7vbUkoNDLTUWZCmnX7WLYtCicrbO3pgxl0DOVrrwWCQZZkyzjgIMXLaWGuNs0pbSmkQJZ7k4RHKTpoS7s5OzqGxwJGIhXlezmfrqijDIOVBjEgQJykmzFhnLXIWBwEFDhpjIMQ+INRvAlVVBQHz+qm2bZUShHYopSSh6bBbqjrppd/94Xd/FPzuu99498tnX2bdzttvv62d+u//h3/1r//1v/6Tf/xHSRQDYNMw4sRobbWFzjkGMQ0Zo0ESxdPxBQCOBny3u5eloWwbFkbvP3o7jbOzw5PDF0eGmOOTE92Yja3N8+PLVmnnHOXcWrte5ZSx/nCglKLsKrfWPyDW6jzPyQdf/+bjd967vLxsmqMoiTe3trWxUkqICfb+Yg4CrZ27yQgx/rlCV8MYCCFEEFlr4JUwAmICCbmqFTyb5lqxcyXTzfPaGGcN6HaGhJD1ep2EcZIk1gKGLMBktVycnp5bgITUGGMMMI8zSmkUhkkUp3EYRWHIA0JIPjmXUt67e39nZwdCV9SVUqJt21Ve9Afdqmw4DwDE4/G0aWrOuYCoWC+qpk67aavNej53wA4GvW63m6/WVutVWUCCN3Y2kyhpdNvrdl+8eBEy2un1vvG1D+7e3v/Vhz8/PTl6/M7jOItmX0yKujiwt5TIKbVbW5sR2cQQBUFUVdUg6VyeXPS7g8eP3v7008/39+788R/+MYTo3/7bf1eVzZ1be2kYztbL9Xo9nU4RQhjTMAz7/SHGeDqdEsIev/1uVeb7O9vAuadfPX/z5s3Dt947OjrBGJd5cXE2RpiKWmZxenh47D0om7xMO+lGrzufz4M04Yw4Y/f39yeTWafTGV9OptPpfD5PksQZ07a1XilM0cHtfcJZq+TqYrK1tRVBRh2+s7v/wQdfQ5gul8t5Z7WqKhYEcjxZl8Xx6dlsMSeEEAwmk9LLXpK0QyltlbQ2aKrq2Vcn2riiKJRxYRgoS5RyhCMEoLMOOoAgooR6oBLjqxMUWAcAggBaY7VSfhdHCHnlC0KIc0oI2toaGascMAgDALXSDSYuQMTalnNe1q1U8uLyOIxoFJMo5oNB7+DOuz/5yV+ZGkwvp2EYTyaXL1+8yZe2KoBVoK5K2YBu1ziH0yR1xiCMIcTQeVsY44xzTlOMAYAIUaOgc8pZq4w2VkHoEIQIAgAABo4iTDiJosiWDYTwxhvffx6GoXMuSRK/leR53jSNfzacc20rvLzes+SiOPSobxiGcRwbY5bL+atXr66YEAgFQYhpYC1Q0ihlnIOcBX7u5pzDntiMIEWYc66RF9pZiK4sAaBRESfD4bAqVt7UNs/z09PTLEtaJUOeUFzHYba9ubeazy/H58N+/5vf+vp3v/2tyeUYAvutb37t049+Uaxn7779YG9v79/8638lpVRKyrZmATUmPDk+rpo66/bTOJpOp1arVup+pyNEOxpthAnPeonUzWw5l6a1QEnt+sHAIVC1tTJyMp81deFUSwPaG/Q0wOuyWSyKpJNwzoUBjdDE4kYIfToez0uAsBSmbmXTiNbCKyGcNt56KAxDYKw3GwAAeNtgrbWDgEBAKXXOiqaxWiOEMETieoRPKXXWAu9I480JrLXGEIw4Rs6ouixMGpVFThF468H9Fy9eaNE4a0QrWqEQkYQQ4yAP2M0Aws+JPVfA2yYaY/BvBDt5kop3TaCU+rhTXxO0UvoS07hrQbwvFMp1QFCVL4wUCIIk5N/82gdBwH75979oy0LLhoAUUNjWsq1bKXUQ/v9FU/q93V7FOhgP1Ptf319nEAaEoDzP9/f3IQKr1co5t1wui6Kg+3tplq7Xa6XUer2mlELk9u48GI1Gn3zyyT988vH29mZv0F8sl9Za7wgHfHyJgz4OmmAXch5FEQB2a2ur2+9FAV0u52VTIU5HW5u7+3vTxfT27f3+sPfkyRPoHGNMtdaX1J00Ggw2CMbrhYlCBiEEDmJM60ZVVdXUZrlcdjqdomoBAEobAJGDgPOQOCdFTgjjQQAhbKUQTYshgg7oxpyeHG9uDCnGd+8+DBiNks7LF28ux4udvYOibDDi1mArIXAYQaqNhNcTed9Fe5JHGIZaS6UED1gcx9bqsixfvXolXfX49haKybScf/7yaZIk73/zg+3bO//jn/8P+/vbG6PB9ubWoNP9ra9/88HdO+cXp5evXkBtgIEIOOwAwQgjSgjqDHpRwKIoMFpgaJSWlHDGo8l4nN1JO51OkiQM89I1tWiFkN2NQV216/VaaM2ikDFGKL3Rzd7UnYyxphLj8Zg4CIqqvLgcjyfTDQcYY42Q1tqAIgsQdBBboK1z0DiIAMKMeaIWgh5AhH77c+DaroFzxjgOQxpFIed8PLnMsgwhUFc1JjAIorou37w+DIIgCgMf/lFVFTBWKalU29Q556Fqm9l0DBGO057V+ujwMIoighm7thLiBBOMIYQbaWCt9u5anU4at81kcjmZTOq6Yoywfqfb6WNKq6paLpdtwAxBOCBJkFJGHIZxnCmllkW5KistpLN2uS4owsPBII7iVDZJGGRx1Da1EhJCONwcvf3e+1rL88vL7sbg9r3bWZZ1OulPf/qXr1++aKrtH37jm00tCMTHl1MhRECw02prNMIffK3T6cVJ6AzY2dqs67rf7zGK10W+LNfz6SwMAkK5Z9kgSg5u39Fap1lS1+3lZBHFQTfN9m/fKcvaGnewv6ekXa3KJO1Iqeq6xr4GNGY+nQCrgpAtp5M0jY8W61534CvEK3K1Na0UoARJkvQ3hm3bSK0n85kBDlKShSmDNOHhwzv333373TRKp/M5xez+/YeNFAYiA9DlbLpcr4qyJoQkUWothoQ4F7aNW+eVUMpai0lSlrJpGusQQlHTQmuVMSZ2wl5TvgkhlGMv1FbSaK2t1tA6hKCzTgsphcDItrDGGHfSrG5KjEAU8k6WNE1VN3XdVEIIzmlTE84559Hu3iiOw/WaQ+TW6yUPqdRCNC3j/d/50Q+qcv3hR59Ya5FDGNDx+RRaSJCjCFltq7JFsKKUZlEHeF8OQm5qXAeNs9ZATDFhBEMHrBSqaZVspdT9Tka84ZizGIE4Dj1HeHk58f72fs+NosiHt/py4aZWmE6nbdtKKSEMb0Klfczglb+ItXEc+XGyDxj0vS/EPIwZodQ6bAH0iuIsyzDjEGBlLCIYAqSsQRBihFgUKaV02xKMsMN5VdZVvtHLAkaCOGpEWzW1tso6l2Tp/v7+xpOJKFuKKLSwzCvZqpOjo36W3r17u9tJDl8+393dPtjffvP62f17+wGFW5tDa+26KNJO0un3LHDz1Xy+nJWnOcK3i3IdMMIIogzPZxOl1NsfPN7f3xNOLPNZnMbD7Y3ZfGmxk87EGChgx9PJ1OrN0XBze2tjc3PcOOtwGCf3Hz6+ffv2Kl8+f/FiPJtHLCiadla2wEFlbFMrZSxCqNPpYIyNubIiiOMYOeBViNgn53lFldHEecQY1kpBb4lonZEKYxyFEULIXnMDMbjyPkISa9l6gWW+WmyPBtbq4XD47ntv101Zlc0qXwuhECYYUmutViqKuzdcV18T+EpFXXdRN1WCpwucnJxkWebRMnideHfTrflvvjndAQDQaMqDfFFao61W0/EFBO9tDbe3RsPFbFIXeRvFYRgHjCgNLYLeyPkG4btZb/4I919F1ynV1lqhBAC0bds0Ta+NDpFoqlu3bsVxfO/ePYzxcNj/xc//vq5rTCBi9Pj87PF77/ZGg9PTY2WNdpZH4a/HLsqJpjXGIYIpYd1Bd2dnp2mqwaC3zle1QI1o0m5GCLQYVqJ9+tWzVlbvf+29x48fr/PVatFoUbWtrMo2oK2U2llICGEUCVlJKREizjnP4kCQVHXrQH6FxxhLMKOEYwiCQLdSyLb1DhwEYSWQEKItBYfBap47o+PvZL/17W/NJmOI+cnJGWaxkBYgKrUFRlmDrAKtbjHy2k4Cr1/gymqo8ZTVKIo4p1EcdDqdg7c/eO+994YbG93N/jvk/fliKp3p9Xr/7J/9s+GwnxfLt+7fm84ufvF3v+CMnJ+fbmadVohGKgcg4Mw4IKQSUn3x2edSyqyTGCkBtNPxpbWWEhQ0Av4+3Bxubm5uHh+eDkcbbazGZxdNXRHKSMilUBjjII4ghFIIzjnCV3Wnxy/zlTo5OSGff/bFxfnleDyeLxeEcY/LxWlHKOOc09Zp64wxyhhtnXUgiThwyDlngXPO63AAgogyHIaccuKAhtBxTlnAMIFSlUJCTGCc8DSNkySJ47BY5x7cQwgpJRgjhIKiXI4nlx0GjSxCDnY3+3WjlJUEg143BRABAL3+db1eA2OdMxCAdcz8kyNku7e30xv2jDFNW02nY8YYY4FJU05YnATdXtLtJrOyJJQGnPmiI4qidZ5PppdtU3HOI86lEBGj3U7aSEQoIADEEZ9NL6Vsf/zjH//yV3+/u79blA1ETkhjDXr18o3S4vz8IkvSR/cfiFoZZfrdIafBfLq6e+fh4eHher2+d+8epXyxWCwX63WRTyaTdZGPRluUYqu09xkUQqyLvBbSaPts9TyKkrt3bx/cvrNYzjtZBxOUZdmbV0ckCHmcSOtqqbixeVmWdZ0liXMuCHhdl0o3cRyu10uEnLUSIVQUxXg8Ho/H0+nUWhuGYVmW2hqIOywMAHLz1ZKFQZjEabczX63qorxz5w7G9MOPPjo6OtndO3jr3YQHkUUo6WS8rqVWSZIgSupSU0oZZsDhulVFWUqpLQTWOSGUg5gxZhyoq0oZ7Y25bujWlFKEsN+n8rz02zTn3BO7rhQcppVSZlmWpJGda62ltZoQtFjOrdVBQG7d2u10UwhhEDBKab/b6/W7zhlK8dHpG2v1q9cvLicXq+Xl9ubgBz/4zosXX1ntppcLQjiCnOEAcpXFmVLKWm2VVUa1bcsYgwQ7BAEA1l0nUyPoIAAYQIIpBBZxgiGXxBnNOIUOIGAhcBgTHvFOlvR7HWKcc269Xvt+0X/8zR3kN0d4CCGlvJHEFVn9JhfKO8v6AKEoiryfz3q9ruuWUmodVNpqZaUx1jrOQwcxQppARAixDhhjgNHQGkeItRZiFAZBQDB01skWY1zU1eNHX2OMRXFMCUrTtCgKhFBdFVoJ2aLzo6PlYtrL0qoGVZnni/mt2wdSVCdvXn/rG1/78vMvzo6OIsbu37tTNfXrn70CGHW7Hcrpt77xwcbGYDqfcc5Oq9Ja2+t2O1nUtMXxm8P33vsg7XXni1VeV1lnkGRp0UhMWNO229vbUpnVumjqUimlRsO8qHLDxpeXjPNBZ/Duu++WVVXV7XSxjLKOcmvjBOFhBAkNRSsVIQwDrLWGAHhEh1LqguBGlWDh1TFrjXH+yLTO8yIxgH4SgQD0b4e9Hk84CK0DACNIcLFsqiJ3adoIlaQZhHBMxov5imDmYAMAoowTFmDKAACYehNxdyOMdNee9/+TU99dmy16cwUfs4Qp8XwXa627RqcQuOI0XA2nkOfqkCCO8tXy9evXH3/8sc8ylVJOJhPj4GCwARGmlDoLlbU3BfHNkvMo/Q3y4RehX6Wykv788EYL/uKtlsvlkhG8s7s9GAxGo6EPiOr1etP57PDwcLaY/8Ef/P7G5vCzzz7r9Lrz+VxKTRjt9XpaubaRShmMcRCECEBKqRLtar1+c/iaUpRm8Whr8K1vf/1v//ZvgojVTfOzv/vbN4ev7t69e/v2rS/0KymlllBKuVwuGaKyFQjosJ9BiLW2lELGOEIEY9Pt9huptIHWAWNAKwSixltaMcaKoqiaGkIYRVESRUEQOGN6UbKRdT///AsIwfjisiiKw+OTtNPdUDrr9qJVJaRp8to5YC2o6xpyeDMb8qMu/556VmmSJEpLY1wQsEePHr3zzjvDO/H+/r4SWklz7949QkixXh8cHIimfvrls9VqkaXR+fHkk08+vHP31rvvvrvdixeLhV0utbVREhEW1K1Yr/LRsH9xdj67GEshtjZGAY7SLBZCXBx9eXF2vtHf2Nnf+6u/+hm5zfZ3bo9Gmz/5i78OaAAh9GbbrZIIoSzLKKXaXMUv+5VW1/VisSBV04qLSyFEFCXGGKEMIiyO44vxxP+GyliptNZXowe/gCCExl5Z5kEIAUYBDQlBlBIAEYQGYaiNFNIMhkldNwCATtIhFDZtaQ2IkyiOQwTgfDFWSqQZz5IYQKF0ubE1Oh9PwijZ2d29uJy9ePkGED7Y2JpOlpgwhAGA0M+2/SQbI+eHjHmez2aMBhQiRwiRqq2bEiOKMYR4NBj00jQOw1Acn6zXa+dMkiTA2MV8tV6vq6JumjagQRREWkhrrTFKSxAELImpEpVs6+3tzeV69eE/fHo+mXqFEKYhJ3i5mmdJfLBzbzIZ/+rnn+6NtuI02d65fefuW0na/+Y3vv3pF19GSTdKuhjTxWp5dHb2/PWbk5OT4XAYdjo+tA0iEEXR6mK8WCyqRtatkEpVdbtcrXZ3d8/Pzvb3d/d39yzAL46OOAuFc09fvTo6O1sWhZSSIjpbzI0xlBKCYdnmlBIppXZq1MuSOH765MmTp1+9evVmMV92Oj0eBnGa5Hlu1nbENwLKlDVV2wAMDEaLMm9FWzt7Ppu8Ojyar9ajvb3nL16FSQIwWuWFc45QTsMIQtgUcwCwsko3WmmrtbbQAoCUUgA5xsMgCFopLLDGaEIwdABYZ43HDwwCyPsPEowthBjjgHNGmdYaQ0QQxjQQQlz5jGqFESAYKdkoWTNGsjR+cP/2aHNDyjbLsjiOd7aGYRiEaYgxvDMeCdG+9fhgMpn8xV/8hRD1rf1divDpyflPfvKXvd7g5PAcAosRIBhCgK3BXp8ppXYIM4QRchBCbZyxyAEIIMAUOeikFsYCAgFnLEgiTshqvgDWOGMRQhA61IIqLxhBBDN0bSDvzwOfUHqjhfOmgT67IQzD8Xh9Y7Pjo4T93x0Oh5zTw8PDtm2jqOvDR6qqCsIkDMOmaZS1xiHgkAOIUi+yd9ZahzGwwBkLIWKEojAwxrAw6Hc7FAICnBa1gxZjPNraLIvlarUC0JZ15ZxZLJcYmiggw14i2tJZFUe8mwZ72zsIQif13Vt3f/63P3v44HtW2idPvrhz687OztZ0Oi2LNcQIE3f79sFv/+hH2prnL18fHh6ul8s3b06UEhg4iuDFxdnG5i6PMmmMkDqva6kB4UEQJlmnv7t/u6obZTFEbDrPO1m/sfJ8fnF2fGKM62QJZThKI61EEASirpRsMSZZlkRxp21lXlZa62pVSyk5oYT4ztJcG114Vr9X3CALrs5mK5SXNGJPLAfQA+n+ht+M891VKi8Jk2SdF5WQ55P5ZLF2RhdlM5488RU5IowHoQWo1YpgGkShEtq/uf5c91jRjXLHnyvwN4ycvYzCg0z42tXAOWcBQAgxxpTRHpC4OukpketCasMY4GHUNE1ZCUjZYGNzsLG9rlqIiLSWYYYJghpwetU++ku6GZB5zoTHXXxF4mkKyCAPFadp6nMTlFIYgpOTk+n4Mk6iR48eZVkCAGiaJgiCi9UyrytxfvbTv/2bt996+P0f/ODLJ58vl8sgiBhjlHBGIUbUWu/CHtZtXVVVnufaKkRI1Im393a29nctgjQK9u/cTrvp3/7NX75+/TrLsk4nU0oZ4xjjSYythov5UivZ7yZbW1utqMqyVNI4B5fL5XpVN60UFgkh/Fi8bVvOeRzH1hkAQF23dVVxzmlK4jDinCsh5bqNYh4yfOfebQDAs2fPXr15ub29nXQySBFAzkKrjSGEEHIlZkHwuqC8To/0Rj4A+INJIATX6/XR0ZEQAj+VH3zwAUKoaUSv0/3888/DMOQo/OjjD/+7//a/iZMwjgOp2vW6eP7kL1ez8g/+8FuXs9nl5SXEaJuxUafHOUcAEoybdaEqMR0vYONWi+XovfcGvf4YPJlMJs65vb19B+EnX3yuLXr/nff+4I//6NmTr47eHIbOsTBwjXHIeY8WL7tFCHnPt7Isy7Ikw+GwrmtECQS4qBqlFAt4UdVCCIAQhNdZIIgg6GNGc4wxQgTCK3kuIQSSq74QY4wJghBDBKQWSgnMsTaVc7BpQdMWxhhnoVF6OBymSVTXubEyiikPIMJmYzNrmqWolv1udOf21mDYddCenF1enB0GUcYZJoQZ4+paCq2oIxC5IEowhlnWreu6qqrj42MeUKWEzy+mlKZZ3O93kyRr23Y+n3eSpFyvtVJ1XjZCNk2jjbPK6kZZ7TAkVulWmqooGuC6nTSOeNvUnJMw4nF6kHWHyujZbB6E0eHhOOQ8CljI+yGnk4v16dGRk5BHOcRBFCYIha2G773/jdOzc4fIi9eHx6enl5eTeVHEvQGJki+/esmVSdM46/WNhReXk7ZttbZKqSzrGuDyPD8+Oy/XBeOzbm/w7PmL8+ksSZLz2Wy2WlqCllURsjDudTppss5XVVUwzpdFLpXY2hha4Cil/X7/888/f/PmzdnZWVU2lAc8DKIkEUqFIQ/jKAw5xrASbWtE5Wwy2kghbJ3JZzOaxQfDQdjpFEUhCtsIpYzGmLKAI4KrRnSHHS+ObdrWARgk3FrbtC1mAFoHkQKYMo7ihFODMcbMV5nXXbXfGQEAPkzB/09jjDdetNZSijwIL0QjhNjY2Njd3T08fF3XpTGUcWyd0rqt67LTjYOQGNuu8wrTTCkpVWWdund3f39v8/Wb53W1Hgw29/b2Fp89e/rFUwjo5VmdxqHQChjtnIUIUXK1Uzvr/0MAIwAcghhgiBAi2FprnbXKaWchxRAhSggZjYayruq6Nloaa53VVYkgsARxjxv7jDRrbVEU3jDHW0f7Zs473sRxHAQC49AX5VEUAQCEEL6GQAgYY/zAwt+fpmkcCi1AQhljXBQ6BxFChDFGMFPGSSkxRBZajFAYBGmSgIjLtvXCIisFNFrUcL2YrBbzZ8+eSVmLtonjsCiKd955HATB3Tt7Z2fw/r3bZVmWZUogCgO2vTU6Pzm5OD/f3hoBC8fnM4YDp6Go1fNnT1ZFzhhBBFOCGCNFuQ7CcDDsVXXx4OG9s7OzYr2+HF/s7+06a87OL4MkoyxykBiHCcU8hACxOO3Haf/s5EnWHXbi6OWzp4QlASNgMu0kcV2Xp4dv/s6q3miICOml8fmkgtZYZ6xSzkoIDbCqrmsplVKKJGkQRv5AJRB5ipn11l9eHoagn0EYKb12zvMY4LUU1jnn3RJvCgWIEWM0iFIDSRxGdZl/9fJNGHCNeKurRlmpLXMWcQQQFloLa6x05jqLwY/ePHv3ppu/+RK8NhgOgsAXju7a/uhq0ODcFSlYOyGEB+oIIdY4KQUC0NgmicPh5rZyQGqQ9Dvb+weQcgcQQgQiYoADpJXN0j+DV1MVjP3PucG63LUDhD/22ralFPvEhMVigRCqqiIKWJIkFCOEUBzH/X5/e3v71atXEMLJbMoCThn71YcfNk31ox/9qNvv7986WC7Wzrm2bY1yRjtPUdJaO2PX63XTNMbpjc1RnEUbW5ssCv/DT3/SzdKkkzFO3n///enkMsuyN69eM8YBqAihLInbWsqqQQgPBhv37z9Uqj06fjOdzI2Fq2V+enrBw1RBWpYlpZRQJFqJMcUYQwuN1VEUBUHAGPHPnTFKa4mdK1bLJAru3b1NGCrr9dbWSDtd1mtHUNmU2sBWlAEACDNv9AnoVQwH/o0XpbRtr9y0wjBaLufPnz///PPP0wh99cnTJEn+8A//CMfw0198Sin9+OcfCy1evzzpdFJC0Mao1+9sTi7nz58e3n1rcz6fL5YLHkTdtjVOAwedc7JpOlHCN6nKG6jB+jI3dzQjuCjsdDoty7Jt2/fee+/f/Jt/98tf/aqqm3/yH/0pxtRBsJgu/IgTY9w0TRSFXooVhqHvYfzWRDa2NpeLNQDA7zuE0STJrLXGOQ++IYQQpgReGexbyPyWB5Fnh1JKKWbYR6rzgDJGMAEQulZChACA9WizTymXjbQGdLs9Y8zF2eV8Me717na6iRBN3ehWVIyhfj9LNCmKZVmtL86P+xvb3/vut3bPx//w0aevXp8lSZbEHQCQUtIYQxFGCPq5uwd1y2qtnd7c2uh2syAKEYJpmt6+fXtreztJsvPz85cvX3JG0iQRUuWr1WyxopQHYeiMHQ4GvU63m2V1sTJSWK2kEiCJW1FbI+IoOnr9hobJ+1/7hgO421saYy7PL0JGs3TQVmp+ucCAf+Pr3yPUFUXx4uUhYyyJ07IRv//7fzhbrI5OLi4nY8aCshUvXx8OhxujnZ3VapUylnU6w80tqTRCSAiljJVKTRfzgEdKGimq3qAfJmm32//Zz/5uLUQtVVkUECBIyXw8iyLV6ffWdblYLZfL+fbWSGuJCbz36GE3S6nSBwcHH3/8sR+Ka3XVNPinpdvrdbtdh5y12lqrhbYM7+/eYYydHJ1O18tO1u12u1Vd7+7tC63UdA4NCaKIBdw4UJb1YNgBAFRVJbVACPGQNqJtRa20lkZjSR1yURRlveyKM1U1fvP1FJOAX1nQ13XjuYrQAYSM0RpBSCg1sgFGCyXrpsQY7m1v37tz6+mXnyIHOKVxwHudNI3jMl+pthV1XWOtjRwMM2MFtKYoV91eShne3h7VTbnN9t//4N2ybCFgi/l6ays1CmutrDXOWEgAxti6q0EAxgRhCiEkmBhkMaIYY+NKDDFAABqCrq3BrdLffP/91WI2nVzmea6U8MAjIWQ+nS8WC611FEU3jDAfIJRl2fb2Nsa4KIqiKPzWnCQJY1e+EYwxKWVVVZTS+XxOKZZSDofDXq93eXnpeenrdWEtWBelMSaMoyu7KoB7vQwAUDY1Qsg3rN5oKNeilTJLEkqpNjqKIgLdcmabpjk5O+t14uVyKUQzXy2jNLmcTnZ3RvPZ5XDYHwx6WorT4xNrDScEQeicy1fFwcHtfJlzzt959H7E04vxC4bxj377h5Tx/uZGEIaffvqxA6htW8LZw4cPj98cnp+fGyV3dnbu378/XomyahIUMBZEcQpxAHArlBpPZpub6+lsube9tb+9/eTzJ6t1SQmKOdu8f4+HXFkjtahWqyCNeRhvbgwJWa2rVish6lpZIEQjmsoY7LttjzBbayHBjLH1em2M0c5CDzAQbICz1hqt0bUbEiEkjCJ/dvokT0KIg8BY45yjmGKMHUC7+7fv3bn98uXzslXK2E7WYTyAhM9ms7oVWLswZsyhuhVl3WAl/el7NTi4Rin8J+A6lslfQNM0w+HQFxNaa2yIuc5icL8h0/Cog68bpEPOgTiJjVKYMATdy1eH1oG7t+/M58tWSgix1MKPVtpWInvlAO1/8tXv6JxPA7LW+iqhLEu/03LCOecbGxueZ5MkiTEqSaI4jkPOsizr9/u9Xm9zc/P169dVVR2dnty7d+/4+Pju3TsnZ6f/8l/+y//8P//fv/Xgwb//dz9um6Yqq7aWEFxFiyEnpFFVUWqratWkJluv19NZkNeryWwax+HZ2VkSBQ8fPxptDDBEZ1pXTWMNUEpR6N0tbRwF/X5/Y2MDALNer6uyAZBQOncWJklSa9Q0DcYYI+KAdBAgRCA0cRzHcez9+41U0BkllFFqc3Pj5PTIGFWu1zhA/WHvnfff+9U/fKiBNkYB5KAFWkuJWmysMc7SK6qHL+4ZC/xT7/NLOedCXpnFDYcDzjkW1fnJ+QyR3X+6rYz75MNPHj58tC5Wq9Vqd3OvKNdNK87VJEnLr33wzcFgYCGiQZh2e0EQhHEEAKjbplgvZ5Mp0JYC0o3TLOquxwsg9MXRaVMBb+whxIu3Hj/6q7/627PTsqh/GfDkO9/+7h/+4R/+6u9/9erlSz+jrMuq2+1ACMMgUEpoI7yjRpqmpKolYbyqqrKsWBD68bAQAjpHMfJPF8XWWiilFLVI0qAsakphp5PFUZKmHcaYECIJ2zjmUlUEmfVqiom9dWvv4nKd9hIAgHVm0E21cSykYdAt2+Vy2TrsqqZtypbAMAm6wIB2aQt7tpLV3qiHI/f89ScER5ubd773rd9SzcdCYOIia0FdrHjAEQ5Xed516tatWxfHL6WoKTAP7t3WTqch1RZ1O/sQwr297TAML8ZnQjZBQPa37xH36uTsTIm2myWXk8nG5vDuvdudXvbs2bOiKUd7O9Px5cfPvrp/56ACmiR0uLcxnc4PDg5u3b7f6/WXq3wyb9qmZRzWzTpN9imCg34nz/NulnYGvGmay/F0Op1LrZyDf/f3v9TKTcbz4XD7/Pz89YtXDFMpxeuXL3u9nkOJtrTT2Vzka0PYdLnojzatNknaWSyWQqrFfJX1R++88+03h0eEdlS+zqXBJEYIadnGcQdDMBtPsiwp82LQ6a2Xq04Sbw22todbCKF79/b+/ue/bBsdx2m/Z6QwZV6EQZzG6bosvnr6+vXLo+FwyMMgCIJ+v58is7qYJEmSUIo2N7TWs/llmnYqWRRFYWzbKFGvlmmaUkoJak7PhG+UG2GNUahS1lrgsBICYYwBAspaoTGlEEJnHQ5ia60RQhuNkDPOYuesM0LWwFjCKacBhEaLRjYNADaIWC+JZ9O5Ee3te/d6cbYaL5njnaDfVi3qJ4uZaEWhTDhZtqtqEScyiqKh1Max1lDGh5fj+t6Dt771W3+8XpXrEm5sPgjj4/PzS+0oj7qTk2kYphhho4SoDNAiShNoMQZI1I1nADDOCSHGKCkawIjRkkIYEAqBM1JpqAMWLJaXO9tb48kZZYyyYLFY8QC+fHOxzseMEONcI2rGMmC0VTAJQqgtUG58Ps663ShOgyiL4zhI4rKcYIy73a536vXefM4CrczR4XEQBEaD+SyXwq1XdZqmlLerfA0d6KSZkxpqmwURUBIaHVLCaEoYraUoqtJhq5Bu6zlHEsHmcnK0uTFcVaswYrcf3uHswXo5/+rw1bDf394/GI42Xrw8vHfvDkP4P/tP/vHLly85D9MkO1baj6tbIVb5cv/Wzv6dna+eP/38xZf/6X/6p1EUHTfho0ePIEYAuCiKMMbZdjKdTrNsO4rTphGjrf5/8V/8X8Ju1gBjGO5kg+WiSDqjt955fzJfCocEbEAQZGn6qy8/GQ6yWTvb4v0aris7v721f5vfef+DD+qmXKxWyujTs4uqrSFxo41tjAhEq7rVVVEJaY20IQkVNAhRadpWEUxxq2uH+XB7UKuKuSumqnMOARNgjBlrEQiCwBijdYsQ6nbiTqfjm3ivQRCisd74yAKnbW+YWq2+evZZyLiBVgtVAjefTLXWjFJKCETOygYCR5ESSkJ7pX/xoMWNkZHv12/UEDd2jdABLVXIg7ZtVSO6yZV5YhyETdNoYyJMWdqtUAUACHhAwtRPh0kcG6WqqkaQHx1eiNoqpcoq55Rxzv2hZZXgUeas00YaAyABwF1FWmNEEKMQMIkMhNiqQEiopKa0TjdHd3b3Xnz1bNDtKKsgMohCaRoOASGgqVfIift3dqfnW6Ja8w4q5IrGcJ3Pd4ZblycXf/Yv/9v/7b/43/wn3//Dv/npX+0Nu69evTg+efnue+85UOdFXluCgNzfGQnRbA47rWoWp6/SNP6D737rzZsXH//9Xw2Hg69944Pf+u63ptPpl199Bl24f7A7uVhwTmeTOQLQQXd6fv7pl5+t1su6Lp2DRVG2TobduKiL0sU06jpnGq1pGFMeamsIQUHIrXZJxBAEw4Mt0VSfffYZ55yn1BG3u7dFGSYAYm1Pn79iCh4MtoVyg6x/Np4BTlpgITaOoABxC5GFCBIMCTbAaKcttJhhIfQyL6IogRhHCSkrJ6QJSBB0Nw8OdoNeRo2K+zFL2eKskBrkK2NMOBhuCVn2s95iupBt072/9/jrjz/8xS8Pbm2LujJtIPN8dnq2Hs86aa/VJi9biORg785KgVUhAATLRX56ev71D7aePvkyDvjD+2Gxrj/+1S/bvPwn//GfvvXgkVEgX+aXl5N79+6FAWnrti5bBCy0ADnc6wyllGS1WiGEPOf2hr/gM0J85evLVXOdr7VeFVprQtjNt3U6HUJQUa66vaRpeVWv7t+//+jRg6wTffX8iTQtpdQCqLW+uJys19O0IwEAm8MNv+MwxgJCrbLj8Xi1WOarajDYxHb/8rj95LMza939+7Tb2fjmB98N4+6Hv/poXaw6GaMMXo5f376971rdtm3AeRjxolgXRaGUUEqEcaRKFcRRWZbL5fLFq9dpmn77299ez5swCHqdznQ6lbodDAa3D25tbm/l5brX6wSMl2VurGorMJtPHj1+uL29bwzc2MwxYZwHxgKtrVLm1p3bzlirTZLFaRx1s6ypyrZtV6sZxnhjY2Mw2JDCKGXKskSI3Lt3TyjpBRpN09CAxXGcJMn4bLzDGWGYc04I0lo1VdlUVdM0GNMo4GjQQw5MJpeU0r29vTDt5Hm+Wq2KYg2soRTHcZiEAQQuyxIEXVm2D+7e2dndnlyOoyj65dnl6enpbDYHEHojBABQ0zRVVQFM/J54fn6pjblz586jt95mpH3+/Pnx8alDMMuyKEqksK1Q23HqHESUoVYKIQAg1mEHiLW/js01Nzm210MEv4S01gAhP+M00lhrEUIeS/cdtp+2enaID3lzV+b8CDpIMB0Oh9ubmztb24vF4uzsrKoqjyoRgrw0uWka5ngcxwQTCHiRNwCAgMaAu7quVasYY85ZJaVSQspWypZSHoZh2+n48TOhFCsJAJJaaWt0bjCjhJCmRaAoOOfdbnc47BeiMRgCqwxwFEJAsLaqrKrT8fjs4twovbW9U5Z11TYQY+M0IySKIuyBOYyVVMC5AASEEK2lyKW2tqyrtql8R+gdJ71YDiGUJIkfT3g5O2PM2/L4ubXnnCMEnLHWauuQsz61FQJnMIHWWGt1GDDKOq0Qi+mEcWwIWS/z9959GwO3v7uXrxZJlsWcdeLo/u1bWZTEYYAxbKr69PgszWILgXGwbtu8rAFG/Y2N7mBggKsawcO4kYoFQavkYp0ra/Z274Zh2jQVoQgi04hcSiHUKi8bxl0QBtq0ccyAAW2jqlJGYS/J0uFwpKaLuh3ntSjLkvKw1+tZay8uJ5OLUytlXbdZt7cx2n7v7sHd+/fG43ErzeV4vF7nAJHOIGtbqbUFgFz7FCgIEYJXWhsIod+swLWJxWg08jaLXpBilPpNfeAN01BK6UWqvqTwb5C9zi/1Cw9D5K0OjNJKSg/bAgAcghBCY63UqpHCY9EEEvQb3sk3ns3gOu7LXWcreE6i5y70ej0vjvXBub5kaZrGP2g3XAFKqfd6BJ6WeZUZgQgh/X6/003rur44O8/zPIoCzqkfKzjgn1wDoVNaKg08O8G3wko1EBKCmdZmna/6m5mUKk0zY8zZ2Zk0MgjZcrnsdrPxeDxfjM/O+a39rZ3dze3tjc8++2xzuIkxlmULrAvDcGtrS1TNj//yJ4/u3//BD78/mVx++NEv3nr8sG7yw8PDW7cPOARJOohi+nu//4MnT75wWJ89Ofv61z+YTuacx2VZX15MX786euft9zmLEOSDweDiYuyzNBFCDFMP/7x+/bqsCudMFCXOAc55FFlnoZYIACel1qrV0BJsGbHW4vVaY+h8fHbTCKUcY0EYRsZZxhjlFHreCqbGgkYKZKgwTlnjIECQAIgQIuDae8DXeX5eczNF9XMHj2/5PRBjPOxmFxc1xwRoY43uZtnF6Rmwttvtto0GCJ5eXlT1+vji9Lvfe/+3vv+9D35456snT6FDb14dWm1O35xENOCEZkmHE0qDEFtCSbAYLxsh8nWJIFASLObFfJY3tcaYMxo4jZFDp6en/9//5r9+/NbjP/mTf/Tq+as/+7M/K8uV1syXqv5+enStLEuilGCMUYqdY845QpBSBgAbhpxzDgCQEniLvGvPDeL56mVZjsfj2Wy2Xm9FcVjXZV4ESter1TwM73Y6vTCibz18W8haSjmZz+oqr4o6L0oIkA+M0VJpqynCGAflOr84Pz07u4zQvZ3RAXHbDrYH27au69VMTS9ODvbvhLwLrA4oUrpN02gyraVa9OhAShlGYRQF1mqKCecUEkgQlk4KIV6+eFFW1cXFeHt7+/d+5/embPXhh7+sqspamyTRYGNkra6qYtDrt3XVimY2mQJoMQZWm93tnbox1nGEA6Xdal2BQuR5XlT1F0+e7G5vj4aDoi6CkBigg5gt1rPd3W0hFIDQGHhxPj49PXWQjkZbi+W8qhtCyJ07d1brJUDIWHV2fiLXTdLJVqvFuqyappKikS03Si5W807WwZjGcdK29Yvnz9Kko7U2RkVRENBh00ZNU4m2Ltf5ajHd2drsdtKyzAMKHr71YHd76+TkBCHQtsJahxCCCIVhkCSJtRYgGPCIcCaEmM+Xs9nMWBeG0e3bd7766qPluqiqqtvtDgej/nBj2V065/YO7uR5nuflbDFv2nlVC4KdFE5Jo6G11ip9xbu21mpjIcQQIG9yZxG4NpKhFlpjDIIwCAKMoZTSakMwZoxZpX1+wg3dGkIghHKuRA70djZ3dnZE25RFfutg//DNS6FkGsXQgSovZtNpnCRJFMMgMhJfnM4Rwbu723EcQkcwprKRZZ4TQpVsnVVKtwCaEFASEIddq1WjpVQSAGAktNZGUYScM84ao/zBoLXW1ijZOueANc5ZQjBn1GmsRbvM14vZ/Jvf+Ma3v//dxWJxfnbZNE1rhAkQxthqgyEKObOEAgDSOBSt8FQMraW2AACgjSwrGfFEXL/8KP3qJDPG7yw3gYF+dE0Zp5gYYAhC0P0akW6ahgWcMGacRZTEnIVKVVVV5KvJZGy0vn/vXhIGRholNI4xI9wpHfFAaz2dTne2tjtpevj69dNnXw4HoziOR6MtxsBwuJEkyWq1opSdnZ09ffrUWBXHISUMQtjrDWotEeTGlAgba0wrVsY2lNbj6QkLTCcdXV4cG2Om09n58eT8eNofBRbRi8vJxXTeNA0hrJNmlAVayCSKlMDGmLwoMWHGouPzi+0sc+Docnw+Xy7LslbSsRBjRM/Oxk2rW20AoggSBIE1zpgr6xdjDLmmB0optVRhGPrb5X4jtsOrCeR1ZooxJs9zXyiYK8IaueEb+mO4LEtGKIRQUCZbIdrWnxZlWRrgMMZhFMVxTDgzxpR15TUFfoX7asBfg590+MrAH/+eK+C/yhjLssyPAzwN1iel3dAL/B+dc1K1/liCAjrngDF+rzdWbW9vU0qrIl8sZhCGvi0sqxwAT4CwAHCskZ9lZFnGKK/rtixLo6V11FipdGNMPJ8vm0as17lzMEmS3b3t8/MThIHSjdLt4eHLKECP3noIkZ1MzybrqJt1AsqsBYvFgkGqlPriyRdlvvzB9767f2cXQG2dWq5m2jRluere2hmNRt/73ve01oPhdz/78ot33nlnNput8nW/3y/ypmkaHlw4S+Oot7d7Fzr8xRdPQpaWZekHTBhjKaVSTGvtYwsIpWGopHBGu07gVTBSSW2Ntg5bRyCAWjtEkLYGWZcXpRRCageFlqsFC3DWSRGBtRBUChKEQZRN5nOlbVVLrYyPlQXOIUj8KcsY88DPjYLJm4v/phv3VRVoHFSmE8YMYuhQzMOTo9Nub2M2mwFIBxsbtak7G33KwMMP3n3ra+8d7PY+/eiTOwe3L87PGSSr9TzKQqOMqBuDzNaoQzLmNEDWybq1SnMaOAMWs+b8bJ6vG2sgCXkUIdUK6NDx8SEC7tGjt95+58Fi+VvPnj1DCEFCbvAzjHGYJj7KgHokym+FN42gd46z1jZN4/VC7lqq6zcsaxWlOE6ibi9NkmRndyilaFoyn09OTs7++q//BkJw//7dR48fnp6eXp5OZ4tF2yiCKIQYIdDWAvjCl3EJUV2VWkpO0aifUqynk2NIwHCjU1Xo5OQsX9daitOTY4xAJ47Pzqcuwff2b5XVerg/hBBGQeCAo5ikaZpmcdWUEGOKSVlWT46fIYTCIPYdWC9LT4+OGyn63e69Bw8wpePppKqLs9PjsiyGwz6hCAO4OQqDgOerJWWP8up8Nl8TxjElzlqpLOUMEdxKOZlNXzx/0ut2bu/vff1r79+6u0+ha5pZVbfGuHW+nM/nhAX9ft8Ys1jOlVKYIAihcVopu14vgYbj+eWzF88a0bZ1CZ0FzhAAPCPdKtlL0zCI1uu1JLQsy6KRjDEIndUKAhsHHMWBM9oaxViMgHvr4b2Dvd0oDg72dijFxTynlADgpBKEwiAIrIVCyUbI7eH+zu5od3f7zp39qmnDMHj9+tXz58+llL1eL4wTRCjnPMk6zsEwjIW0TBiCA62AkJoxZN3V7NZaq+xVG+TJfjcMrOt5P77ZDX2hTQiB0BljrDGEYo/H/ubw1ZPSB91BXdfL1Zwz4u5Z5xxy9uG92y9ebEkpkzisqmK2WK6WudGOsaXVDmNcNyWlOI06nMeMhtDiN68PDw8Pd3ZqhFCSRr1uYg2gBI42N31OYy2chRZCjAAAEHiWgNYaIcwY01I1dXl2Crb2thmhiGBnIUSIEIooVQhVRVFLVQs5XyystWHEyyoPIw90FYvVCgNIUBcB6LQRQjjgEED+vA8C6oABEJZlqcVVEqafdHp0wRcNvinxU0OfPAQAcAQxToBBAaMIIWe1tQ4jUFcFpThJIoCR0hoA18mS0cZgOg1ns1kQBePxmO1sf/X0GceorZsfff97k0s5Hk+rMgdGA+sG3d5wMHp5+ObFq5f7+7fu3LvfybpCKABAK0QUx62Qn3/5BGO4t7ezsbmljQ3C8NXTs53dDatdqxuhGmVyyiymyujcqFKb9Oz8iGAkW1OXLbQ0L9tWTg6PzlZ5ESZpb7hhEG5lo0SdJkk07Dvd7mxthpQOhqPVcvnxR5+ygK9WqzCJkzSN4xRTJoReLtdNqzVAQZTxgDNH2laLtkEM+IWHr0mI3tTLOUfQlSzQGOOu3a6MZwNcCwF86KLfG/1W6YGfm4rhRrNQVVWxztumsdYaqSCESkpjTKfbHY1GFgLn3LrIb1Rj4Prlt1ZwbYPoP/cbsn+glFI+taEoCu/c6pwbDoc+vvKG9uiv/1oxcZU45bRxwEAIX7+2m5sbe3t7XigPoK2b0lsL+6IHIQT8sXqNbEEIEQKUQq3butFaa4RV27ZhHCEEmqbe3t6MO+He3m4QYufUzlZvb2/n/Ox4Ort8cP9WnNAHD++cfn4kEE/TFFqwWq06UUID2lTV4elR93n2v/rP/ue/+4e/82//7Z8N+71//i/+Wd2Ux+vJaCu1oNna3iKE7d3a+6u/+puf/OQnRNpeF4vWWoOLvJXChWF0//5bXrIYb3T8IcUYCzgjCKZp6oA1Rnk6nfeMgsAGCTfGUOowNlrrkPEw5IxQhBDF0OcT+TYlCENjjDWiN+gORhtCqEVeyPXaUe4oWRS1UqZu2kYY612DDbDI+n+JXJ+1xhg/Pbx6O5zzhaB/swghphFZEPXTDrEAcy6bFkPkRSUAc6Fk0dS7G5sffPPdtz94L+yk/+Of//jk+Gx/by+O0k6c3No5eOv+W+dHJ/+v/8f/Mw7TXtLX0hgNAsa7SRoQWkrunGtqu5hVeS6a2nLmMGTWioDRjY2N5Wr+//mv/9//0R//yR//ye/1+vHkcqple0VDhLBt2ygMOOfEGOUcdc4Zo5SSfnVC6Bi7eoQwhmHI/e27UesiJBGCURR0u0mSRJTBIOCMYR7gKIraVp6djtu2Bg7v7Ry0lSrypi5aTBnlGENmoTJaUkoZJtaYssyFaDrdeGPQMXaKgqYx0mmjLJtOlnledbsbWUrnk8X21g5jjOzunZy+2d0bjfY2Br2+1hpTVJY5pzQKwyAI5vMpYbQsy8ViMZ9MoyjCiNZlNZtMB9lwNBoSwrSzjNFWyaZpIggvp2NjzNZoiAFcLWej0cb3v/vdx4/f6gz64omer9fdTj+grBFNrQSkTLT57PRYNvXZ6dnWsGjbOkqjjcGAAZPnubaWEp4kyWDQ184Zo5zTXtzPOVUqlkb6dcMDVrfVdDHudHp37twqiqpt5Eqo7dGw3xtAiPcOdijhlGBMibGyEpISQAhNYp5GO4zT9XIxvjwvy4IRKNvm4NZ7y/n47KTe3dterlZVvbCuDSOY0KDX73Q6KUJIKPn69WtMW+NQIxpEQJoxZcqXr54kWdztdwgh4/F0Xa6X+VoKJbVqRCta5SAyziJMAFC+jjbGWWuccxZ6B0OvNkfAIWM0AI5SSAjDmBpjlJLdNEEIuSt5mDXGuGu81+9xV/a0ntgI4WCw0c00xUQJMZ1NVNssl3MHDCMIIgqAbapKSxXxCGOWr3JVqzgO27aNkyDP6yiqgpAX6/Ls5PzNq0PRyN6gzwnd6G/4LTvJhlJKxgmhSLTqur0zTdMY4AigzjmgtfbIrAXYGE4ZY8RevQClhMZZlnV7vV5Z1z//xS87aUIJqvIijcLNfrcuszSJgAEB40oIq2yWZWmaEsLysiCcQYQgwb4IyNclvHZo8cm/NyJkd+3M430UKKVKKeMsoAxRGAQBBMBZa5UmCEuMpRBG6SzuCK3yolBKgQxUQhoLHIZKG6ENYbQ/7N/a3Tm4e4cw0rYtIShLUy3aZVHcunfHEfTRRx9BCCFAWlspJaW81xucnZ0Nh8OmaQbD7mw2S7P4ww8/bNt2Ml5sjXoh56tyJao1IgITIlvRzTpBEFilV/NFGsVlUDjnBoNBbRyGcDmfFVWNCSrmoGrqoqwPDg5CbDm0IcUBIXduHzx69Gh8cT5/8/rmrPXThDAMeRRtb2/P5utlUWmtOQCUUsaYklo7fVMfeLzXOWeNEUIwQn1j5znb/raL5spJ0ztVeMdMv+/fEA/htcsyIcQZQzCBEHLOG0JuEGbGmGdHeiqZdldql4AFNz/EX5gvF24qaf/We96Pv2AvuyjL0le0vtrO89zPi/1VXZ30AEDoKCUYe7swBKyPxNOLxez4+DgMuZCNr3vatgEAREkIALoZuIDrSCrv7EIIGQy7bRtUVSWloZzFUXD/wa2sEwOot7aHADshqyyL1vkq66RFsV4sFlHArLW9Xu/dd989znHTNEA7iCCGiHAWMqpNyyn+5Yd//53vfu23f+d7f/t3P97d2/jd3/v+Ol+t//Ynn3/x0fHJmw/e/8Yf/sGfKOsePHjw4YcfrdZFGCZRlK7XK6WcMY7zcHO0LZo2DENfB+jr648Cfl1g1QgRykJfUWGMGYGQUoJCioHnC3PKPOgCIDTAxkHEGItBnGRJnueAy163n3XSqhWZ0lKZZVnXQjbKCKlbZY0DECDoEPCpyr8hYfWe3D4k4X/invLrYtHYkPG6qk5PT7v9zmq57PV6SqnVek153GrZ6/UQJUrrv/zpTy3UYdRmWXZ6Nk54uMrLftZhPOQ8HE9FPzNSyul41tQSYxoE3FprakMptYY1tZECSGFFq8OAIUjzvIyigGD85s2bH//4349Gw9/7vd/7N//9/7BsayFEGIZpmnrhg1KKmOvksZurxxh7SqNfqTdKD28AQin3PvZ+aTZtdXF5VpZF01TbO5txHEOIup3+3t6e0kIIcXp0jintJAMpjHNOWWOVBdZxHg66PUpIvlyUtaAEbG9uDAaD2erp1lba63U6aeYcfPrkxcsXJwFGSQhdN4wCmsbpNz5497/6rw7Hp5dv3bsrGmGMYYAA5waDQRyEGF6BvW1d13XdSTOA4HI+N0q9fv06fhT94z/5Rw6g//CTv/zyi8/KqrEQDIbDuijTLEYIRCGfzwwC4P7dO7vbO2VTWqCDOAyzCGNsWgcxipK4lUIsDMLsO9/93sMH9yLOkjiknMdEpGkqlFotcwhdv9+XxmitpdRBEPgxZJJEiBLnzPb2Zt2Kuq63NrceP36HYXL05vjli1fYWcKDrdEwDGPr3HRyuVwtgUOtFM4aY2AQsO3N0YMH9wa9/snx4WfQKNFJYk73tt5+6+Hp2XGRr5To56sVC3QQBftxb7Q92NoadbsdyrCDoG2/HUTJZDJ99fKoqVUQJowFlPIk66xWq8vLy8nism1bZaTWVrSqrCsESafTYyyABFsIalErZdC1Rym8VnZZa+E1zHsDsZqrmyD9sjFXBBd301Fdb8RXNC6/dRKC5pNlliWj4eZiOamLUopKioYgqGQjpIIQxglNo9gCjAgVSmmpQBhFQZilHaedkYZ3gqLMQ55YC1arUkqbryoHoNZOCCXlzFor69ZJfcUAdxBBuLG1RSkH1pVluVyupZRBwNMolqulZU0URYwxDKGzhgZRp9PppHGWJtPLi/ls4pTWyoWURgHTUsVxnMUJgtBpU6xLxtju9k6SpBCh6XRat6JVkmHkKA4DRkkErxMH/NbvrRQ8zHBTUflyyjkHnQGOQAgpwdA6g7EBgBAyDHtlUzdtHZuYECyVGF9MtdbzWtRK5E21sb11MZuNdnbTJNraPxjP5tLaqNfrJPHGsH9xelZXRdzpjLa2oiRBhK6K8mIyHV9O/fhsZ+9g79bBmzevut3O4dHrME7LMg+iBKJJGFEeoKrBUmBnqWpBsZRRksoGTFfTy8uZlFY25uTotTXQ0bibJoMsTkKaxFHVVETLLkejbkQJNKIGolnPLuI0KVfzfLXQRnbSLuNIGTOZngup+xv90WiQpCkm561WrZRKtYQGvr2ZLa/SBNC11SZCyCCNMfaIgr/b7lp86JcxY6zX6/kBq9baZ5b6xt13Sjd/0RpDEPb2cbIVbdNACK1zdV0DjBBCeZ5PZtNWSY8/c849DcVep0X4B+TG+hD+xuvmubDWemDJ10aekXBDZbhGEawxphWNvzaEAEKU4itahgsDKduqqtq2AcBRSuraMsaMVRhjZ4FHxb00zitlOKf9frfbyyjFUoq8WNV1PWBppxuMJ8dlNb9z506URqenx01bR1EghHj69OlyMacYtq1M4u75+aWqW2QdRcg6IKQoy5x0u2kn2d3b+cu/+Pd/94uf/Yt//k//T//n/+NyMRtPj6fjSdMWJydH8/nyxfPX9+6+1e1vfPvb3/nbn/3y4vKXRjuPR8YuqutyY2OQdZJut7u1tSVqQylVQjRNA4FjBFMa34Ax1iHf0FsDiEIYY4JIwAJrjX+4IHQAWIicscLh0GHFGOsOe5AaS7hFZl0XQZgc3L8rlT27mJar0hFuFDBOA0QhdMghjDAlV6PVGyLCFdqqlCcVeacKXyn6R3i5XBJOLi4uvvoqDONAKrUxGM6WeZokNEj2et2v/9bXqnad18svX72gHP/W77z34NHjTz76ECjplJ5O55cXk3y25CHAjFPO11U5uZimSScIorquResIploZpSyEGCFijLMGWQviKKnq0ig5HIwmk9m/+lf/3Z/+6Z++++67x4evz8/Pvd9almUQuMViQQhFlGHGmAPGgWsKLgB5XnjwxG9PN4iCP06iKGCMUYbTNLFWF4XRRkIIGSO3b9/mnPf7fcbY0dFRkTdbW5007a1WudQCOWeM09oMe/3hcEgJgtZwggPG+v3esD8Yz1mv1+l20iSJAYDI8oBk+UoaSTaGach5HJNOxt97+95sPpbNSuSwaRrOKWIoieKqrpBC3awjlPSBbKNRRAipWxHH8XqxXExndx/cL8uSE8wpq1GTpFmaJKdnx3udHYLwgwf39ne3Dl+/+uijjz76+MPbj+8HCXu0+SAIoulsgZoqIBGlfJWXQup8vex2+5PpIomi9brq9XqWVbu7u1iQly9fnp9NsqwbREldN0HAEMGz2Ww2myVJcuvurf392/1+/+jsZDab/f/I+pNfy7I0uxPb/T79uf19nb1nvZube7iHR1uREUmySBECS6oipYlEQAPpHxEEaFjQRANBBQIaSiqiIJVEJqtYTCYzM5gZnUd47+bWP7PX3L45/dm9BsfMMgp6IwfcYM179+zz7fWt9Vvj4ejBg/sMk7ZuNssVQ7iuW58SRtB3T55dXc8wo8pYay1lgVEy2213myVwKvz+92/dOuun0XI5DwPv1umNwbBXFvuD6UgpRQkKxh6ltD+ID45G/UHIOWAccs9brUpE68GIef6ZEHqx2M3nV0Koj8Y/rZusKPcQacKQNq3Wlge8KPYQEkhwCJx1CkKAKXYQOEAgVMYYB6Gz2nQlRbibADrfFmxbibRFCGFEO1G9O9cQAowxYJ1z9q0D6O255hylNAi8ei+N2gch10IqIQkhnFPusel0fHk9k7Lt0SGFuKhaB1HkRxBojCEmiBJUFbkeJMPeUDTNeDSZDA8QQsYCLSFjHkS21RX3iTRSC1HledsKht/26yhDiWOEIO5bTwgHOKE+xoxiKSVAyOM+ZhwA4FEvDROjFMM0jVIGwHpx1VTVnds3EbCLxQwAYLVhzGOYiKYNPL8D+Hc3IShaazUElhHPZ5zESZdDq6qqaZqOEdnNBPZtiVR3EjnnOOfaCGd1Z2aEzgFnKEFdSSDjRGot2tpBoKWQTb3LMzKc9KJACymtq8pMW73YLrMq4wRTBEPOtVVCK4dhPBxWSgGEmOcRyuumqes2L4ssKyyABweT0Wh0dT2fLeab7dILeBAEddtw7upqX5a6qQUEzCpQVGK1kGxX5AFcbzJRmzRO5cApXYVhPBiMp9MpxxAA4Pv+YjlDKEzTNE58IYRQIqBQN9W+qa45u7q6SnziB+Mw7FVts9mttWm1llVdZHlZN7mQjZDGQcSBw5j4PkUZevPi/P+7wXddve9cAu92yd3qwfM8Smld1/BtKUO3Y36XjO/eARjjzmvSvQmklJ3fkFJKOMMYK60dBAY4AACm5F0MEr2lJry7XOK3RGf7FseEMRZCdL9h517s/sR3NdCd9vDOAtlp2m92KFZBCAniGGPnDEK0GyPeYjwoQsA5I6XyPA8TYgxUSrYt6B49znmSJEHoYQwm0+FoNMiy3avXL0HZ5MVKmwphI2Q5TQZJGjqg79+/u1oslvNFEIS7bY4ROz47/Hf/7i/mF7PJZMJ8rq3ClBFCjFUQoefnzx0ys8Xl85fPzm4ef/PN7//7f/OHbL/tnd1+//33/+ZvPgt8/l/+l/+n//3/4f/o8eDnP//Ti8vrsiyrqiKEeB77/Is/NG15//7dfr9/enr6zZePOeeyNVaZtm1V4L+7lf2PBy+rpHOd6dB2HwqIEIAIUAgRwsYoQh3A2guDk7NJmND5al41bdu0gLDeKCAa+VHbGwK5XENhLdQAWgQABA4BiGF35iEA/g5j1Y103ZD3Tjd995GrpbhxOD69dRb10t/9/rfU40VVEkIAgmVRCCW//Ozzps032TKI6Y3j27uyCno97EcvXr64eXQSDwcvnz6v9nkyTKCFLPC6lJMpck91Zq83tXOd15VRz1oghHIGhElMKb++uhgMAupHX335yBr0v/hn//OzszPO+Zdffrnf7yeTyfHR4WAwIJ1lppMxO0mq+7d13+Vu/n23n0MIGQ0ppVEUUIa7j3vTiKLMfN/vlIaDgwMhxH6/Pzo66feHlDJKmVWmqhqIAaUEQyiUSpJk0O9zgpE1YNAfDfqjwTDtxe4Rs1Y/f3J9/uq3TVn1er2D6fHxZFhVrZEAQiya6uWLRx9//J6xZ4+++wq4QVVVVWW9yBNCXF5eUoqPbhz5vt/1v4VxOplMBoORHwZffP7l+asXvu/v8ixN0wcfPMzLijLmhcEnn3ycpvHTZ4/TKLx982y1mG+2KyXkXuV3796/dTBuWrnerWezBcIUIWKtjaLEGEcIe/nioqnL/Wbb7/d7Xvbzn/9pFKdXV1fz2dLzgqTH2lZcXV4Nx6O2bfM8h8jleb5arRywx6cnADiPewDYtm2rvLDa9NMeAoVzTgm5Xiyrujqb3mqlMMYJaeM4VkpsVuvf/OY315cX/UEQ0M3QAAEAAElEQVTPaTUZDwl2URxcX16uN8u7d+9+89WXWuuf//0H1mo/oAirolxvdjWlKE4TiGGWb40Bo8lR4MeYoqLcImyePH30+tVF3TbcZ9QAIdpWyIHvCaeFFLjsqps082gUJYSQ66tNd9jZP4KRIdilvN7s6tq2hRh7nud5npSye4S6Z4ZSCqyTSrw1vvydStwd1oe3Jq9fn+e7rCizJA7CiAJrmrK4f/9+WZZF2VBMWmV3u53nRUk8QEgK0drWAmC1loNhz/d9zrnP/CTpMeYZ7ZylcRwDB7Os+OC9e1mWnZ+fy6Zty0oqwxBmhDZFaaWCQRj74TBO3+3dDqaD/X5vHUjSHiasbhsCMHKAMV8JbZXmjC3nC63kzb/3pwgBpdRisZhfzxgmg94QQmi1WSxmRVEMBoNGtB30ESGECcQEtm3bxRwopV3VTfet6MD78I98+13CDUgjpbTO2beLT4RQJ7xz32ulzIq8rCutddent4WOADc8mGRl3u+lVVWHgX89XyJgMQTDNLlezFXbPHjv3r1794qmPjs+Pjw8FK2CEA4GA8796+vrMAw77GNZlsaqfm8YRcloNLDWNm32+uJF0zSYoCiOldJFUW/mpUOo38N1rQf9ye3b95qbom3lZDKZDO/6vo+s0Vr6vi/KHUKon4QYA6mFVY1PkUIII5rEoRwNBmlwdnY6HPWrptFWXs8WdVMsn64Wy21ZtWUrIOYAQgAh5z5m+N2rwr3VA4wxRmlCCEB/14fUvXQhhG8act9SaOu67ngVURS9U+Y7WFb3QfUYE03bMeyKouhKEBil5q3pAWMcx7FDcLPZ7PPszsmdd3Yc+Ef4hO4c78YO87azEWNcl9U7D1n3k33n/unC+oyxLojRad0BDSBySinVWmstgahDpGS7vbG61093u13b1s5NuvhVmHBKCecepUYKihB513kIAMjz/XotGEenZwcOhMbo9fJiOBqdnh0cF+NXr86ZT+I4nE7HURQtZjOEkMeiy9evm0aMhpOL17N+rxfHMYIIU+/gaBr34tVmeXF5Ph71/uk//ad/709/SqDOst3Tp48ZJ++//54K0qpqfI8UhZjPXv/5n//FP/9fH/34xz9+9vzl119/Wdd1msaez/7jf/zlYnk5nqTT0Y2jo6NvvnwcRRGwWAsFnO141e+ifJRSbpFWEHfqpwEAAKedQ6CrQUYIWgsIhYhiP2SEovGk9+D9e03T/O1vG4CwVHkrxHa/lwpIbZJ+b7HaGtdpqAA6AC2wwELrGPe6h5dS+k7S6D5Cf7xxeLO2BOBwNPjw44/+3t//OcT2X//bP+OcX83mk+kxsA5j7DF+eXHhBwQYe3p0fDActx4YTKb90fBf/at/3U/Ss7MPnz1+1rSScNYUjUWQBT5hVBujjQmCQCtACJFSGqM9jxNClBZG6TgIF4vVycnJeDRV0oRJiDG7upr/7d/+7d/7xZ/cv3//8vJys9kghHq9HoSQhJToukr9oNVGlHWa9ttWcu5pa4MggMhVTemcDYLA8xiEMG+3zpje0PP9INsXzrkk6RnjCCEIOIJwkWerxVUQeF99/ipOIpemj3752WQySWKY53vPj7NsNxqPCHae77+6vGiFPrl5i43GLu1fFMV1BS4vruazq81iezAZCMs//cPn/TR+/8F7RydT3/fXm83/98/+bDw5+Of//J9vtbt49Kq0dZIkOPC+PX8+Xy7KsoRJhDEG1gZJOh6N0ygWVTl/fQ5EHfUGDttXr194PsnyXZqmAGIt2tBjy6tLz4EHp6d/8x//ehqF3/vge88eP07S4VlviBsJqnb5/EI0refHeVMRQrFlH7738Y3j4/l8/tUXX1aFrYpNkcCrpbnX60fJCVw1hEdBFK3Xm16vJ5oWaqBqbThUuXu5vGwO7D6Tw+G0KIqLy9Vms/EHgx8cHG63293X30T9eLlcBgmLkzGxra3ztmlQfLxcb0aDOAipxyl08ur83Gf+MBk/uPsxI/28mm/2xcu//neUgTD1k16MMOj3U9/3ijKzwFFGjAaNbCilQlS12I2nvZPTBOKjJEm++265Wru8LD3fg4BJYbnnW4PqSt04uyWEamolpQIIJkm/LCuHHCTQGvvGucMwoW8KqABwXSNfN3o6KSDBnh90HmwIIXQIWIgQ9LgPnYNB52lEUrWdfQ9jiog7Ojmcza9BS5f7XWyiIIx++bsv/+RP/iQZHjdqZR2J/KAfaSmUKltInBSGUqoay/yoyPQffvc1YdT30oBHz1++dM75AS+LHWNsNI7PTk9fvHgRheHZ8Uk/Top83wFGnLH9JM3znDM96MdN0ziH4jjtBQEFSmkbBcxBqJRtm/12pw6Oj2iAk+Fhke0O793Ki+z5Zv7RRx/1y10pa1pkQsoaqjiOAaartqCqWdU5QigOwv54oJTebrcYY6hVnq8hwJz7cRQ1tVBSGQWUsD4P2rZtmyaOQoIB97AxjUdgMupvNxuPB865qmp6g34jVCU1pmSdV5iHdaterxZhktqAcdt4nleV+Xqx9CihlPb7g+m9+01dX19czje5xzlB3pNXq+VWMMa+/PJJr9eLUq9tqqwsnTWDg4kFVkO1y7Znt0/rsuz1esC6+fViv83uH53Mz1eMc8IIBiTqDddbUzY87Y/PL3ZaubT33uXrpqmls3B+segPi122Pzo6Go0mr169qhVI0zhvtHMK89AZWMlKKR3G3qvLq7Ztdzabf1v+/Oc/r5TYGZtZ5/Fou8mxF7VZYwHxmWe0bnWJgcMITAaDoij2TSOlopQyj/rcAwC9iTk4TEl3oBsIIKMMwQooqK1RRvu+H8YRIhhTYoFDBFPIIIQQIwe7Ix8FlmMAMISmNT4JcMqUdIAC51DbtgcHB7tsf319ba2tq2YQDaSUnQwAEHwnMBgHNYAaQMa9KAxevnzpe8En3/tgPp+XefFu5dTtTboFyjvoQuc67PQMQojSpMhLSnkvTJUSUkojtZTK8yKGvaODo6+++CqOU9Wq8XBY1zVn6NbJ2X6fC6w//v7DXZ599dVXGOPhsP/42bdJHFBotwvSbg98in/20UN59/3JZLJarXv+kN7w9ot9hcv33nsvJhF3wTg9tNYeHeCvv3720Uc/idLR41dPhqMxD/x+Pz04nI7GPa2q9P79KAp+8dNf5Ms82+3u3bl77+RnL1++LNaOMytL2fdGRT4fHIS//vzXn/zskwfvvffBxzdfvf76T376sXa6qMrffHrB0uDP//ZXv/iRd3x6uz/6MvQjCOFus3/94nXoh7tVLSsU0FFAA6hgQDCJQVEUmntvMkQewggooJwxHGGAYN20WklnwGg0KPYNcuzixZNsv9ust0EUc4pnr15Q4kdxL8+3pt4bUWvZWgcxxpBijVANRWw86nGjbVOXlNIupMo5L+vCOJ2m6Wq1nE6n2jqtpdPGD/kPf/z+rbPDX/3qNyJrRSan8QS0DjlrRO08oow8GB0tXhWKpcYfN+sFVj5QPoX+i9ez6dHx3Y8++Pz3f1jO5v1BCsNAQWQhQxAzmmLACWuVUcwnrWyEai3SmEGKCGIWcXA5O++nief5m90iHQY3bhxJ03779Lv7d+/dvndnMB5iDPMyO75xRAhFHg/G42HHcUR/hP1CCDlgGGOcM9/nnufFSXgaHOX7zFoghJRScu4nceR5bLVaMUa11svlcrNe37hxDAAoikJb6BwCxlkDAy+Oo7QqG9FIp11ZFIvrxT4vTk9vJn64uJ5dz2Zff/XlyxcvoLWhR5zVu82mbVs2HiKEuvxFnmUYwDIvnj5+Ytq2LQsfY2RNGoVy0/TjREsVcu/Ro0cdALWY5gfTMUE4z/O6KA9Ob242m9VyvdntlTXAIQdBEARNVW+2qxtHh8M0ffHiPA6D+Xw+nR48/OFHWlvRyg7pihBpmiYIku12Nx5NPc97/fr1fD73fX84HM7n87a1V1dXx8fH7733XprG1to8zzv7dBjG4TiU0nDmU8rLvFmtVslJihEkGGmtsiyDDoxGo/FgCN5/HwBEKfV938jOxGTaVoZh6JzpbpOUYs45p954MD44OJjP503T7Pf70Wik1/W9+zdv37vp+yoIgiDwEUL9weDAm3JOLQQIId/3F8v1crl8+vRlVTZhGN1778OifHZ5Nd9ss7cWJ9g2QoqGMY4xJRhaI6VWWuvNZlOWJSLeOwOUfWtL7D48f6y/vVvKWqcBdAihDi0AoQMAIgD6g37btlYbyrDnJ0KItqp3601DG0xQtzqNIo9zbg3oTOCBH/X7GiHU/WgYpwihVtQdJUJr2So5m80ODibc9/r9Xl21nX5GMFNKVaqSSn3++093ux1w7vatM8bYerm6urqo8uLu3bu+5929c/b0u8dJGByMhqvVar9elZRbgCBGUEiHUS2Ecy4idLlYffHFNef0/ffu3blz7/Li1XK++kx9nniYENLr9aw2ntfhE4xo2toYzws8ylqEq6qSUslWAADeNEYSihDS2gDYJeiMMaqulVRt1x7peV4n6XmU+L7PKE2SxFmYZRnzfFK3eVlQwCGEDhjZtFVVQYI59zsbdhyEWkhOPaWEUkoI+d133+V5LuoGApskyc3TM0jwLs9iiqMwHI/H2W6/Wa3XywVjrJfG2PlpFB8dHGa7PQAuidM49DnnSRpq6/rDgcWgFcJaizBBlBVFsd3v6kogQEWrAMA+9RFBiOC6rufzubbWQTiZTEaTMYRwsV7BP0rBdFZwhNC23m02e6P/RmttAGTUcw44C7MsL4rKOIgxQ7Bz81GMaFVVAIAgCDzPYky6KzghLAiCsiybWoC32fcOXbrZlm/Wr3+kH3TEl3ciQRcg7BYExgohhNTq3VKDEEIgsM4BBJV5w8TsFIswDB1GgGCHEcQYWfvGIOkcC33nHKeYAhRQnkZRP46dUhfPHhtDncMAWOc6teMNFsJaq7U0RiGElFIAWEJQgEmZSaFrDJWUQgjBKOYeDkPf93kYBj/4wcf77dbzmDEmTWOjhTZKaEEIYRTHge95XpZlo36Pc44AxgjvtvsvPv8GI6CUCHicZXnTNE3TJEkymUy6ndF8Pp9Op9ba6+tr/IZY3I5Go48+xsfHx1fzK88frjfz7X72n/wnPzFaDgaD3W796Otv890eOvCTn/zI9/litjRW1cqURWM0cBYVRfnoyZPhcHh4eDgcDm+fna5329U3XxICFvPrqrxVZHmSJOPxuKnaTnEJw7BpmuODw06Qe2P2RC7gAWNslhdvziUMgHXGKGshdNjzmJS2q6Lt1i4IgY6LKqV0VdXUqqoqCGQrjRCGMUapIkQb+3f2LACAtcBaYIxTyljbPbmGUmiUxZAYZRHCQkhCqFKaMd7v9yGEq+3md7/73X6/F0JFiVe3ohJtnCaIM5EJIcTd23cmw4lo2zRKdWt95g16Q9no1WwzGvank0MKSMiCKIop4e/EMAFb6vE/dlB2Gw8HURepBW/f+J7neZz3er00DtbrNafs8PDw+PgYACvb1vM8QilNe/Hx8fF6ve2OpDexWgggcvqtrxhCB6CN4/jOg1uzq/lyuWzbjXXaGOWA8XwWRjyJYkxQlu0wxoPBoK7ZbrdbzHeU0rKQm00GrMEQAwvjoPf9D7//3ZPHT775rqirgAd//Ze//O7Rk4Ojw3y99SkZDYZKtAjAk+MjMxmWecYpB9a1dVMVZT9JhZSvnj8bj6efPHhYNvV6ve6HYZVlse9XeS7KWjdCtK2sGo5oyIPjw8Oje8ee533z/Gn3zl4st8zznIN5WTSNIAgb7Tj3ldJV2YwGw8V81U/To6OT5XKV+mGcqNMb89Ummy2WvV5/u8k6NniRZYSQo5uHAfdms9nV5Xe73X673d44/oAQ8uTJd7vdzhkrpRwOvSRM2lZZ88bY3DTN9eUFAo5SjiFq63q9Xi8Wi/FgeHl53ZmqtLZXs/lqtZJSWvAG35HnLQQGIxZF0aA3PDs5i6L0N7/5jZSykfXH339v6IZ37969ffemamZRFEnZ1nUVxUEQBJh2LyFLacRooyXdF2W2L+G60ebpYr7hLLx79wEEZLnaLeZr5yDhnpK2aRqlbdO0dV13snPVNIx1nvE3fPg/NoWBP6pGfBcnwxAB2OlvFkKAAEAQQgg4Zc5YgxXDBCEEjNUEGeOquuScO2cow5RyhLCUQin1m1//rm1bISSEuGuXwZi2olmul0op8iaDahFCURT1Bv3z8/Msy7W2GEPQ4e4dRgif3rjR7/UghP00YYyN0nQySNu2Xc7mpq3u3Dj+osxkVUAtl9eXs9lsbzCmpCwrbUHa70Vhcnrr5t0HH6RR/Nnnv18vF22jB73h4eRktZiVWeVDLwrCKIg79VgJmWWFErKtagKJsI4gXCMsperW284ZQjihyGiltYFvkbqDwaCuawBtFIUIgVbURakAsMeH0/4gHQ5vD3rDLMueP3tZVY1RgmJCMYmjwECEoDNat1XNMBG1sIGlPmeE1nWd7zOrnWxa4CAljMUEY+xzz0HgIGCM1UXZ1sIZSymliCqlgXUgiheLVRSE/V5vazb7zZZTOhoMgHWPHj+pquoOoYST1X7LPN4qDSBcrdf7PDcGYGe1sf00vX3r7nQ6pQxAjLf7XVbkxhgHQZ7nrXpTtM049zyvq0wUTSuEMBDWlf722ydSqP5oyLnf1Lqu5Xh0aA2W2gZB1Mm61gKldJJGWtnO7WGMAcB1FRtvJlcjIYTAoXeZiM5/0B213R72nRfy3V4ZvS0occ5BDEF3ZL4Vk6EDxuE4iaWktiNxYoAQpIAQhjVwDlrjnAXGAKONxgYCADhlVmmHkHWOI5d6LCTIBR4jyGMk9Lkxpm07BomDEFjoEAIIWOgMAsBjBCFGKW2MCRtCiNfr9YySTdMQijklo9GorHLZlpNRj2J36/bN/X6/Wa2vd7OiyhspgiBaLueEkGGaEOCsMZEXEIjSJHZGXV0ufd9v6jpNDfdD51wjlIMN5ZwQUlRV1TSnN28KLRbrhUc9zHAt6pt3bmZtvtleI6zCiChrAXCjcVpX1euLlxfnr5qiko383ae/vvGf/zPO+Xa3vnfve5Qn4OtHCDKhXH65+LN//W+ddv/Ff/ZPgiA4ODhAGABnxsNgnxWr1Uxr7XPe7/XmV981RaOFiuO4G9femfEtBEK0QL0BuzHGMIEQQqOkEABAizEuy9IPPEoSCN1yOVdKTCajPM+bpunGd2e1MUa0smokpT58G9FyoDvx3m4WAIEQA9BZ8brMqoEQt432fa6kwYA1Vdvr9YBtfeL99Kc/PTg43G72n3/5tbIOU+YFYdkKhAgirG2lVnazWk+OD2dX1xjjk7PJd1893sy3HAfL1ezbr57cvnUj9P0yqxXW+21WFJVSikBKCMIQgT/6ejvNOGutUpYQDBHqFprdzbNLf+R5brXpIpr9frqXsmkaghDwfX8yHSEErOvSjxZj0j1HSikLbNvWQjakInEc1nXbwWU766wDuq4LQgiARqoGaihlq7Vs2zbLiu12v1iWSZIgB1VrRduoWnKPpmcx0Ob6/LWHMfHC88ePr2eLXSZvnxzfenjv0aNHUIlPPvqAIJxtNw7AYX8EAPAor5UxrTocTZxz2MFJr/f9j9+7vL762khZFhFjDoKQeZvFEgIAHRz0Bz/6wY9/9KMfcUK/+ebRZ3/43dV2QSk/PT3t98vR9CAMw+Vqtdls8ixLkp7RLvCju3fvPXz/QdvWq/niqy+/tRY8/PAwCNF4PFUGv76YYYzTNJVSSqn7/T5jLPD8QS8ZjvqUSISAUnqz2VVVkedlWdYe49YC56CUSrSqbSUAxPdDKSVnhGAYBl7aiwfD3ma1unz1usqLLCv6/f79ew8ODg6urmZPnz598uTJd99demYDgG3rklFg4yAIgvF43Ov1Fov148eP1+tNf5T+5KcfPTh5EMfxq1evPFJK2QIAIMGIsLyshVCYUCk1cLvz11ePHz/P9qW1wA+ipsFBGMbJYECZ0TDLhNSgaTWC0PND6yCESFtrHAQQS22723x3tr5NZ735sm/pde90hW5c4Jy+mScsNMZYZzGEBJO82DNMGCbOmaZptFKcsTAI8qwkBCvtjHzj4BNCOueU0lVVta3sGm8NM1LopmnSftIpLt0wJKUsigogrJQBAEGIlNLdYpIxxqj3q1/9SmvNCE3TOI3iXpo+fO/ByfHh559/XldlL4lHvbTK9pnWxOrj8XB2tY6DkPrOSG0Byqr61etL3w/ff//9k+PTNE7asljMlpHveczfLjaDmPq+7/shxcQ51zQNtJBA5FGGIXnjOzPGaUMRDrhnoi74p9tGOOc48zEl1to4jjB0HsOj8aBpGqu0Fi0hJAj8IAiSJIEY5Hm+2a6KvIKYcM6Bs5xQ7Swj1KekravcGYdx2zScsaYRXaaDU+qMEaKpyrLX6x1MJlKK5WpTN2LQ63f1u8vl2vO88XiMEFKy5ZQ45xjzOOcQ4rZ9M2fk2Z6yAAqDGHcYV7UspVRa10JCynqDIWeeMXC1WO+K/Xw5s8Ad3phgj0GClTUAAoegdtY5RxnzPC+MoiRJuut7XddFUQhDhgNvv9+LtqgrWeSC0ibt9Xa7fZYVxnV2MgIhdG/rbTGBfsApw8YY4JBzVoi2quqOuhaGMUKorrWUwtTK9/m7Vs8u6uWc63px3lkKuqm3O3abvJbWKGcdcBZ0RkTgFIwxYox1RYjvvO4IIYShAQ6aNyRTp411FkFYVpUzFicR0oRD4BGkm8rJdjoZRlHEOa/rWslGSanVm/YmQjDnzPN4Z93vvnZVcXw0TpIk7cVKqbZuutOYMyAFtE7UVcEZPj05vnP79Pe///1qv3QOxpw5585fvuCch0F0OBrt93sEsRSKDzwDaVW2UegBoCEiVd1C5ChnYRxhyqq6rqpiMBiUdaWMHo5HbdsUVZGX+cHRwW8/+5vr66s79+8JVd66c/PevTvWSgjNo0dfbZars5OzKAhmV/PZbNb9oL97/BJRUlbKOOphXzn4+vXl61eXV5ezMAyHg0FHIQXAUgKkEE8fPzZCKiGt0gRji0wURUYZAxyEkAd+3EuV0VVTVlUJAKjdm+8ehE5LJVULIdQEG6tDFEAIlBJZ1kDkrq4v6rqUUkKAtNbAAYyxMbJtZBiSztlqre3cgu+uQxhjjIgBXXuMttYiByCFqhUB96yyHqF1XeMEIQN9Hnzw4CGl9OnTp+v1GmHsR5Fxjnk+DVErRdU2/XTQNPXyara4Bg8fPhSV/v1v/lBVhRFQ1WBZrjni0+GwKZWBti5a2QoEoO+xwOcIoVbad4rCW2OEdRABALtwr7OmOwwBAGVZKoGiKOKUzWaz/X7/4MF9q7UxhnRGnziOAbQQOoi6QgtqpeimD+dMVRdN0xCCdrvdt988Em1bVaXWmhCktRaygsgbDlPRts7ZJI0wdB1lYjqd7veuKCoCURwnvsfasvAIDhj77He/LTabD+7da5rm22+/HYbeOI5EvpveOxUnBx7j/dD3/TANA9G06/Vyt975B54RUjRNP+3101RrnfoBdiakZNJLn758GUTBar2NfS8rq2KfH984+/jjj89u3l6t908fP/nssy/Oz88HB6O6XkdhjzGPEtY2QkqDEAUOEszzvO71Rrdu3WkbYQzYbHZ/8ze/CcNQKOsAurpcWIicA6JVUZQIIaqqCn0updxvN/ut53lekiS9Xs/3vcV8KWTLmBfH1hlLCNNa523eNE1VCYK9fq9njRsP+xg6UVeiDuIgTJLo+fPnxhgA0GK+YvSxdmA0mQ7GEy+M5qu1gdT3uRIl56xT+Lua+UePHjdN05lmyrL84U+/5wfk8tHrNLa7LO/U77KYn7++WCwWGNH+cOJ5Qbavlsv9YrGBgEwmNAosQmZ2vSxriTGpW52mQz+wShrPCyj3GPMcxAAgZXQnLHcBYvf2C74l1b/Ld4G3mds3GhdjADhjjJTGdhBGgiiGomkhpwgyAAAEFgArpRTCaSOsQ+8eS2OM0dBa5ywAjvg+juO4S68hBPwgYdx/F2BzCGqtt9ttXhZhGPq+bxwUQrSNBgAo6VphlNBKqUo3681OtLWRKgy8fhIP0uTB/fvD/uB/+U//2Xq5evHi2Tdff/3pZ8vxwyln3DmHEaWUWgvqqnrx4sWzx09+9MNP7t+5DYGlECZRWBUjn3GMakqI1aYWEjqHEUqSxO9AvELXZQkhZIQgBzvfUJTaPM9X5aYuK0p56Ee+7wEHtVQQuMD3R4PhbHZFGeMMRVGUDvrbbLfbbYWQu812s9m2bRv4UTdUEc93AHoEj3u92XLVFGU0mrS1wLAyyvo+n46mBwcHCKHPPv+9UToMY0xp7PteEFJKDYS9JLUAFGUNIPTjZExJXZStqEdJwjxfGmusxdyjHtfOFk2DIbAAttJShIjnI4KhkT2Ak77TysZx4vHo4DDLdpm1YJNv6/OqVdILgzdreEoJpYzzzk/dmam7d2pnr1ZNSykVQkmprYPOWQDAeDxumtZaYJxLkh58g01kDoDdbtNVCadpijFWSlVV1dQtpYRzhjFJ06RbgXXzLqW4EzC6hs93zsE/TkZ0dNvu0yhkY4yBCNDOC2kdxpgTmm03HuPb9Srb7z3OMeceD0KPV1p3EBFkDbTGGu2cRRBBYChBaegxQpPQG/Vjq2urm1s3jnzfRwgVBSXA1BR1Xsssywjn2GGnWqmFfPsM0l4Y+TwMqdFCipYyTBBtqny/32qpkiiuqwJCeHXx6sGDBwfjiYGkCyKVWX59fa3qlgJEuOeUVroty3rXIdJr1QS6aXXqYJ6X1urJZHJ64yZEbrVaEUJu37795MmTtm1u37716NGj+eL6enZprT27eXR0PIYE7narO+j0+PhwNpvh0Asjf36lZrMZAaxtxPnL13/yJ7+YjA9+/8XLOI0gZhhCgDBj3mpVvHhx/qu/+XUvCgnmCDJRqd2mHU1iaODf/NUvN4sVRaSf9jjh+23WVG3VVs45SDDn3A8DLCWmxAmotbbOOOes1Qgh6zSEEGNIGfEwV0rUZeF5XpJGHcUyDAOttbNAma7EDnVT4x9NCW9v6m+VVICABdZY3U0J0BrrnLWaYAiBZYgQQg0R2NmA0cmg7xx88fz8V7/6jVQGYMy4v91nzAuss41QGNHBYLDZOIIQQO7B3Xta2W8ffb3Pc4gAg55zuN6rAovp6AYBbpCOenGa+TuCiHPGSGUt/uO9g7X2XSgdQgggJJRGUUQIMl23APbjOPYYV0qt1+vNZugxlqYpcc42TY0x7FIxEDoHLCYISKeUpAxjh+qmBMB6nt+2jdwYCEDbts4ZhEHTlk4bTNzB4a0iyztQiajFycmJtSBJkn7/8ae//+1+vw/4xKMsSJPTG0cH0/Hzp49EnUHd3jo5yDeL8/Nzq9tiVT37Bj54/wPG2Ha54Nw/OT21XvDtV19HYRhQvy4LKzV2Lg2j0OdWiOuLl57vHx2OZ8sZ9/1rJZJev1F6MBoNRkNl3edffvX46fOrq2vfC+++/8F2t9rv8/V6c+/+A+rxF8/P58slhDBNhlK1PqO9dLBdb/7qL/8iDP1iX5Ak2rbZL//6V0JphOmNs5u93oAxr67r7hApiiLLMtk2TVUJ0Q7SKIl7Uuoy2zPG4jjGGBdZnqYpJUxplSQ9jJtucldSF1nWUsY5h+Ph4XQihPj97z9br9enpzd3WfH1o++en1+cnp7evHU77fV//JOfvny9iuOQU3Dj5LCuCq31bDbDAAshRqMRhJumab7++utPfvzBzVvvA/whMPWXX39VFLu8EHVd73Z7iD3uhxDSzbbI9pVzOAgT4IhU7nq+EBcqzwuEiMeDVhhtAKMeZSTLCqldf+BBRDBhDoKiKJxzHvPfBb3ePTlvZ/Y3Y+y7RJC11iiFEIIOAOuMVcgBYDCEjjMCAdBaEkICz5cIr4rlarVCGHPOjbFvekoRVcgZo5pGUEp9n3cbHKUE5yyKon2eVVXFme95HiQYQugcVNJsmh0AwFj3Zu8AoTFWSNHWWgjprOac+2FPojrLi9VyczgdHxwcXc/ngecBhD/44Hu93qDI/7vSunK3kUJBTHxKDicHUZpACL/96uuXz576FB0fHUZRSCg6OTm6e/f2t1/+xlpb123dNAih0A+4xygh0DoCVVcxgCFy0IVBOB6ONOy6ghSAFmOI0Js1uXOOS4YQCEO/aSvrqHOubZu6Ll+8eNFWbfeb+z4vs3zfrL0g0tYFEDqIenHc6/U45+v1WhhXFIVVRmsV9PqDweD09ERKyZkf9L04TYwDo8Hwxo1jKeWzZ88Wm63nedKa1mrOOggBMBARz2udqYpWAuAnkUZoX5S1kNvVWmuNGPPDwCIAIVbastB//epyvlwa7frpIAgij3JGqRBivduGYRgG/pvzFzipFWMsjKOOhPPmRHaOEhL4/sAP6rruBGHnuukBQuSybJ9lWwsBRK7zEyCEjLWHh8fgTbO5AqBDybFucn07xb4JjzlnOPe6T+m7jEmHyunOxj+OIHYvjA7E9SZvYgzqOhgB1IRQTCLPyzD2CQk9j0AUYOJB5BDVThoIAWUcIoMJBBZjPB72MQQHBwcIAoRQmsaybR3QYRwHQcA513os5cm7qaWbnAAATdOUZdntVpRShZFSmMIZY4xqRVcPEfpB56Yqsqyrf/zii6/W6+1ut3MsKfPCWg2s9WiAmSMAaiEJhEJK5GxTlYSwzmaBEGLcHw6HTVsJpRerZVVVCIEPP/zwgw/fv7y+DOPwvffvv7o43+03s8V127Y3Tg8/+eST3/7+01ev1IuXzz2fS6GKPO+E1WxTedRnhH/36Pn3PvzRvbsPn19s86oMw8DIRmrtoDUWvDq/DAi/f/v2o68ev74419L24qQXj7JdzSkXTVM0Mo7TTgTqwsOr1Wq329VNgwjSxpR1rY2xznXtNgBYQgjBnnMGY8QpbZoKY8g593ymlKqqknN2MJ5EUZTt8y5jos2bWkj3NuWIEOqEK2PeBI4cMMYKqRtjBIQQdJg0BZIwAEATjAl2oc+cVb00unXjGABwfXF9fT1P0560jnJP28ynZL/bW+uiKJCt0kpFYc85gzU0BhfbZrfOAHIIA4xpuW8oZNb3OEVGWecgBhgAC4B1QHdgjv/xXQ508UYAHHSO+V7XmSmFUErhNKrr2ig9Ho87+ETetgcHB6RpK7iHUgqtJSHIGNNFh7tsSd9PAXK7fRMEfpIkUrVAou68sNZ2Qo2WIrZeUeyN1QHzOWeibqSUWlvG2PHJ4aPvQtmWEIG6KnqJf3w0SSPei4JhGjlVf/T+7Vsn4//3/+e/XS6XBwfTbL8ttqu03zuYTva77N/9238LAe7Sm0VRNGWRhJGHKUPwcDy9ePVyW2THN04BsFEcMN8/OD6M0r6C0GG6yfKL+Yr5QZj27yW9vKjWee6MjeNESROGYdUICLHnBXmeK5FD6G6dnmVZXlVNU4swDMfjaRCkWusXr147gKIkqesWo64VBhsjnXMIWEIIDUMMAcZICLHdbpVSSjTj8ZAx1p04YRB1e9A07XMWQoh838/zsq1qlpA4CEb9wXA89bxgOBxu3Ha1WiHCAADrzWa13vz6N58aYwil/eERQmg0Gv3iF7+4vrpYzq7X63Uv7g0GAwhhmqZX88u2bV++fPnx9x8eHR3lu32Ri/W6MJoADG7ffXD3zn1E8PX17IvPv3r67EVRVATzNO13SgAyYDAcp2nfWXxxcZlnOaKMUd8YV1YNZWWe560UhJDuqbP6rSHrbTbSvoHB/R2F6Z3u6pwryqxTSh0w0DrrjLXaGsMYcc4B65yxgDhCMSeUEyqd6k5zhABGlBAGoQAOxlHCOFVKbdY7pRvGqNYqL/YOvEmZC8GqtmnbFmMKESKEUUohJgCAN5WMDrZSgJg6VEvVAowhpRRSz2GA6GqTXc1XH330fYYRZ8SjjPvBJz9cPT2/3Frn9wdhlACMhqMB9fjl5eV01M/3m0fffpVtFowT6OzJ0eG9e/c8z3POQYg7/ybGuKqa/WYLIYQQA+sgAEoprQyjdNDvl21FMWKY4CBk1IPWQGt8RqMoEkIYoyI/QA4E3DNWQ+fKsuzcspxQCCGlBADbXWopBFEQtFKOhsPDo6MwDC8uLr58el4VpTMWQqiE0lorZfK8pJT6XoggybKMUjoej51zbSu3u81gMLAElaLBGIe+Z62tm4oFnAGrROsIYjzKqnqbZxjjOOlRSj0/UkZjSvZ5dnV9HSbx9PBAW5uXtQWuqEqBpe/7EOHZajm0Q5/zzl3lMc4455z30x6E0BnbgQiBc5zzIAicFFK2jJE0jaXRnudprRkjjCPPJ4iS4bD3zjxogQuDpGNVtW3btm9ancIwLMtSG+ks7Go18jwDAHQSQte51XkaOj3jTaCgg/ZYi94Cc5xz0AFtjBTCGWuMsdoQiBilZ2dnp0fH2LgqSqIoUq3oTJHTKOx8iOTNFGswAoyx0WhgnRmNBm3bQujCMFytFxyxOPSj6F0DMupmICnler3uYm+ydQhYjxFOQ2stMkq1IgqjOE67xUddNsbYOE5DL1wtN/1+32q32+yX81VZltCbNE1jtIxDPkgSz2cYYo9S6IxRpJdExkGIYNILrbWQQITQ2dlZnufX19dlWc/nM2vt7du3r66utNZh6BNC/IBDNFBK7HYbLzCEkLOzM8bYr379a4/7s9liMZsnycBoGAbJ4eQYWFTty6++fPSjH/z48OjJt3/1rYLOQgcpYIyfnp54BOdZmWdFWTaf/u4L2drx4BAAbETzve9/z2P8228eAQOaqh0Nxt05NpvNqqryg6BDkkktnDNCK2DeQKs8z+OMQAiUsko0AFjOAxJCjPHr169Wq+XB4eT4cNrv9/e7rBsWhZDWWsaYEG9srZ2ojxDqEs4IIYiUdc5YAaAmlGIItQQO6CAKst2eEyRF63OuRNWfjm7dPGkbOZvNnHM88FM/FMZODw8wZUXTiEI6CMuyhBZapYssf/b0qVGk3JXQItm2jFNIcd1UAVf79S4JeJ7XSmhjDAYIQ+AwNMKgt4zwt05ziBDqtmQYwu5G1z0f72iwnXe4u6KUQtR1TboKsq6+AgCglKqqohs8pGoh7AEIhBBB4DNOilI0sk3iEEKAMWYcMcYQcEEQVHXhMZ9x8tYPX6/XWwjwaHJDKTEajcbj4dWrTCsBndlu10a3P/zBRz5jd26d+r7/xWefLq6udqtFGPfu371zeHxCKP/662/XyxVC5OjwJPAjZ6w1YNgfcQIxRKNeevUK1k1V1+Xl7Fpr6ZHkzp07yPNrYy6u5xChII4o8w1wZVXv86Ku65Sz6eSwLMuXL18t19s4joeD0X6XKS3D0D86Onn58uVyMe86Cdu2tWUZhjGlNIpTRMhms6nqVho7HA63261zpp/GnHPZNp2FExjdtu1qtULAJkmCUCVa6XHfOVfXbV3UjAUIIWMsAMhaa6wyVrVtu9/vvTCA0PncS9N0X5Ra66IqlTQI4d2+0BocH083m43WctiP7t+/73G6WS6UUmEYYswghHfv3u0N017fa5pmvV5rJ30Wto2RwlSNFKLxePj69cX561fbzf7i4qqua8aYVjYrC0RYHMdpb1DXddtKjFgcp17QC/yQELbZ7ouiAgCVdVNVRRiGACDP8/Jd3i3nuhHbvI2hvwMAwLe82+6/y3IbBEHo+cgBCKG1zr6pVACcMoih0rKuFUU4SZIgCDb5FmPcNtK5t2YcByGEVVU1DSqroqqKo+PJw4fvU4YXi8W3j54YY7pCyG79wTlnnAuhIHznWSPdMoRYQ73YCyKjlGybpi7butRSYogRZi/OLwCkV7O5bOv57CrLsrYqD4fDKtubsspaUbdtwHgvPYk87+hgijHs99NeP6mqYrVcXFxfGKMSn3HOA8/3uQchrOt6t948f/48iqIoiD3P6/RwowyEMAiCsq26v3O3DZfdY4/xO2EmCL3BoHfz5s3OqNQAkaapMaYuytVqxSmllJ6cnJZliSAJ/cAY43PvYDoVQqi2/erZq3dMnrquyY54jC/XKwDQO+zgdrt9cX6epmmUJHVd8DBAlLStdsA5hKUUm2yflUXke9ooYB0hKNvu6roejvrTYJj2e9JoZWTAmZBqX+Q08Db7nRcE08NjhMhyttyst1VTU8yCIKCUCqWapgk8rzMHNE1T5QUhhKC3RMW/W2YZSvF0Oj44OCibmnOulBpPJ0dHB7PZTGgVBIFSqizLsipbKcqi8X2/Y2kIIYRQzjljVGeJ7/b63ZqmU2c7DawDzXVFpl3EpqMWdh/pDl3QxeXTpNc0jWhaYwy0DkLocR5wDzrQT1IzEirt9ZK0rmuPMsZYfzzu/jjqM0ppF1VgHgXQSdnGcbzLtgBY5lEAISKwMwFD6KzVxrjuvlQURTfEdEpymsbdxpNz/tXz87Isp9PpjRs3FovFHz79/cXlpVKaQJymPUIII2y322FElLNJ3M8VJcRqqUWjG9oCo1HAkiAAjhsuB8P+dp+3sjUm3Oz3Qgg/4NIY7WyYxO+//95gPPjtb3/7299/Kv62kUpMp2Me+HXbDoY96nGA0Ww2+8Mf/vDwex9CCKWUh4fHjx49VsoUeVUUpfNQEIQYMCDxH/7wxUcf/uD7P/z+X/zHv4TOWWi1tkrrwWCgpKIRn06Of/iDH//3/+bf7neFp61DcHpwEIcR5xy6zqCner2etWC32wshOusiIQRBgAhUVkmjrXtD4UQYUEoBcN0eIQx9KWVTl5yztm0xgdvttiiKDsnQCAkBzbLSWcx93LatsdAY4xzolAUCUfeWfavoA0wA45hABKyBzhCC2rZmFBqpPEZEW3k+HQzT169fP3v2rGkaaW1vOLKtPhwMpdZK2zwrrbWE0DgMrVZ1Wea7fLOqjXGD3rBpKkwJxlAJyYgHmPF9P/IDz/MoIdYqKVtrrXP8XR7nj7+CIADAkTctvugdlDOO4+Pj4+4iYa3t9XpdSQoxbesT0jaVtWa7W6f9MeFMGO0cQIhevL4CAFDIdoutLMXx4VErd0LUHiPMo8AqjKSX4F6PYQydE9vt8vbZ3f1+j1A9u14IoWSFsmVWYzjtDwghJzduWAKRjz78yff6aYig/ebll7Kt+wfeKgf3puYf/ex/cuvo5tXl1bfffptn5Z3JdDab/clHD6MoyrJs/PBB0zSDwaDf7//Vr38HAChN+z/89V+cnp1BCr2AE4qs1bdOz85fXW43e21cVjWD4bhqRa8/9KKIOMFCbxB7H3788Jtvv95ut9PDG3F45/r68tatW/v95X6///SL33qep2F7cHBw/73jb797cvPG6Od//x94fvTLv/nbJ8/OnWm22yUAmjHeyThFUfm+3xsMcdMc37rzWS077v3NmzdHw8F6vQbOIsiSiELjVtdXg17/4ul3nudx6olaNah5fX5hNIQQT8aH+/1zhgLMvbK0tSzDKBqP4rqVxrKsuYh75E9+/o+aqrl+dV2s60k8VpVsdD3t93fblXX6erkZHn/P8cF2s8ln5ygKN1eXl9vdZHywr/TVV88WiwUwIAgGQTDSWishrQVAm7astMKU0rpsymrNuZem/aYpX1+sZrOFx31MGUEwiWKMqe+HzgEv9DDGEAGttdTCWIMABAAoJXxGkbNSSGgNjcLA40ohK32MmHHIGKOMAxZZB62DhDAHEeO0A3z5geecW6/X/V406I+cw/PrTVW2QYDCwG9azTg0tu4xgDBUsjmYHPTSyesXKyuQNa7YNsR5URiFLILQUULX1/PBYKDatpUCxfHV6qrrGi6VMMZ4QZikyWB8AACoynK1WhnRooD/n//F/+WHH390MhkdHk6aah+P0g/e//Ds5tE3j74Nk/j45MQh2AoRBoDAVrbyyaNXTdUMBoNbZzd93y+3RcFUr9eHDgx6w+OD44uXl+WuSb2erLQBFkBnsXXa+IHn+16eZ+fPFqJFvtfHGBKCQGSlkavdjHOurQmC4NXV5e377wdxlGdl27a77Ro5pKSNo8lquanq9vT0pDUtYFrqar3feZ7//NmXWtbvP/hoc73102l5sQrSUKuWeLxW7bOXjzGGhFpCwcXlyzhOwyB++vTlndt3x+OxD3Fd1+vZMkpibUxuyigKsryWohn2Bx8+fLC4vM52WwJottqH2L8UiwrIj08/pphcXV3t13mzb7+7/o55PE4S/24gdbkr9rPtPOn31ovXpsjee++94XgcBR7E1EFY5FVZZ4gQZTRmOEkiwom2AmPo+74gqIZGELTIM5976/XWSOM0PDk6/vj9jz//4ovPf/2ZsgYTRimtqirw84ODg34UpGmfMbbP8uVy3daNz704TBhjYRBJrZSyu91OKQuxZR4LbLDLdpODSW/QW2/XyijmmDKqY5AHLuA+D8OQUtrDPqN0NZ/vd5tbt87u3DzL99vzp8+bun71+JuD6VE6mSiloLGY0DRNQ2RIyDorPiKQch8zCpBrpfSSBBE4GI8gQhaCKYH7LLOGWObRMNZaLxaLui6N0nVdG6uSKAYAjKajOI4JQpTSPM+zRTmbzdaXu/XlrmkaVTmk2Wa25nh1enriLH7x8qIois02l1IOBoNNtZmMBiQAZb4TEI8GaRhwDZUyLaHIGtWPozwHthEDL1jldbF9Mkp+fufmSRDGaW/w3vs3Hzy4+//4r/9lWZYE4dn1usyqyWi0ud7J5upnP/vZPn+5vV5csHi+yIhIX39Xxugu5uLRN48oDSsnfDI7OZ7MlueMosXmiT+9eeveg08/+0OSJBGlrahBo9M4wRino8GmLKcnN66ur4ur2Ycffhg4XC72ydERFBY6PekN1vPZe+8/KPMtG/WvZrO6yjmjlDNgrBVal7KxxWQy2eV7LTHrxXmeKyM9z8MeoYhKK+qmOj6cail831/NZ1fXG+cwcLRupNRAaS2BBJRDC6xsAUSqFYQQAkCdZclk6iqrnEm8cFs0xkg/9Mum7veSo+nEo3hxPe/1em0rb5zcHPSnzpKnj5+fv7y6uroeTKbb5d46oBrthYFuxY3DI6WEsZoQRD3/6MbRq1fPpVFJkvhhlGf7ptb7bXlyPEFCyryK0r5pFUceI7yqZNKLN5s1AB4mBFlslDVGEgg4w4xDhJ1SgjIPe0xqVbfKAGwAzspmbMDR8VG+3++rbL1dSSEmhyNCOCOcdTiRqmmMWwupgzgpy5JzboGrq8IqiTHmnNZtNQziqi7yfI/68Wjc88lQG2GE9KPo1q07F6+vLy9eeZwNhpOyqOMYUWyTiBGK+70oTW6fnR5gZPbb7Y2TaRjGgc+MMbv1brcv7txmH37wUZqmf/7nf/7ixQtjTJqmo9FoMHy/108YY1m+k6qJk+Dw8EBKKaXo9Xrvv/dz7n2eZYUXxft9fnyjvy8qxEgcp8tV5vuBgdQ5KIXOspwQElKLoO3FSRTwG0eHFLmQs6v14uT4MPK4sYoRxAgOGAt9H1hbVc1sthiMrOf5/X4fAty2bRBGDCHEmGgl6yUEYYJwW5fA2pOjwxunJ/v99utH3+bb/enpiXOGELTeLA/GEwQgcNb3OWOkrUxdl4fB0X6fSSGAhfPZzOMBgcj3vN1mmfZYGifOgLYVu13WtiZJwg8/ef/h+x8+fPhQ1qIzpqZx0qEzvTDYzq8rUYe9aDabPX/+/P333//3f/6vyrIUrcKIOgekUNZaxrxsu8OYItTlEzEhb0yIm83G98N32wQhRNM0HQcQAAChwwQhSAkhEKMOegshRG/DQtZarXRHsHDOCSGklF2+qIOqurd9EG98DAh1v3NZlhhDa32M4NtdmiMEMe55nocQiyIhhDbGMEbTNM2LXZz0EAJBEI1Hh69evfo6f7Lf733f76h2zrmmqay1lDNK6cnJSSvqXj+RUkIIge+PRyNrNYWgLPJst5H1ME1ThBAn5OHduy+ePbl765Zuq6tX57vZ5fc//CCNIynldMopHc4XOO6xP/n5h4DQJ8+eAlIfHB8dHBxBCL/+6tsvv/zy82+W9+/fv3v3HuO4ruvVYl3kuVPaD/wf/PBjYNxut8t2e4SQF3haawutlO1ydQ2AgdBiZD3O4jQiFBdFsct3z56fY4yPjm8MBgNj1NPvHl3NFwihIltKqQM/vn//xnAwvp5ddoIEhxhCv6Ni19Wu+0FQSp3VSegjaAOfe56X7zfaSM5pF0fUWldVhRHFGIum3u23u9WyQ+dK64xRAIAJGEdRVFg9GvSm4wmQshf6geeHFMdhlBvhnKuqqnMYxEE4SHuc8yzLuly/tqbLqXe3fKNCABlwhFICMakbUVVV3dQWAogBAbSsDNWEUGgdsLXab2XnVRFNLcq6LIqmrHabLUFYCBH4/sOHD/M8v17Mu5REWxaT4ajX6/l+kGVZU9WEIEppEveUNdaCDtTYlYlba4ssV0JyykAQIgARgIHny1YYpQnCiHFrLbBOtgJDpIkiWE/HE89neiWFrAiFjFHlJGGoaer54qppK8Z9hIjnceqx1rZOqM5NhighRlHNMKNVU1NNIUZaS+0sAEBIqVu1z/dKKehcXddlWXJO0zgBAHSdVd3f+erq6uriwvf9KIooc7fvnAAAjJGjce+9B3eyLHv06JHWGjiDIE2S5E0Ezlql1G6x5U5NJ+PjW2ejXgqc3u82bVV6nGqte73k7Oy0rttXry+bppkeDQ+Og96w9+jRo2fPX1xezz76/g8++N5H//gf/6P1ep1l2WqxzPc7Z5VoWkIQgPbw6OTqcj5frl9fzrf5fliWs9V8NlstNmuEodNKmbaWhWzLKPSevz7/8a2H77333sX1VVEUlLFubXTz5s082z158kS2ghAynU4vXr/ebDYmSW+dnmljGOfGmFoJhFBelZCSSrRSKweBhQBAiDBmnPsQyKrSRiVpRCmdzWZCNtPDCSGk3+9ZpyGwDQGmldrIopTn5y+E85qmaYVGmHYy8NsQ+LvoN8IYo3dVXoYYLYVWSilrEQA8DLzI93/20x/FYfAv/qv/6vrV8zCMsR3+7EffH8T+1WIutCI+Bxi1QihjqHUaOmNMr9fzOB0M+lZLitFyuSzCcL5Z3jo7G41Gy9kcAXjz7CgIgny3D8MQvGV+d2qKUApjAuHbXDrGmHDkLEJOax2FodayqiprQeQHjLHNZnN1dbWczwBAh4eHjPHVcrPorzinRVEQ5lHiEWVklEaU01JUwCFCEKXYOaOUaGWDAEyiqN/v+b6XEkwJkcwbR72z6TGlWJqW+Z7nee/dvru6Xl2vl9PxZDSZbFbr8XgaMW8yiQhBD+7fYRwFPtvtV1rxMsuPDiZ3bt8PuH/56mq13D98/5OPvveDeqsW6wXzGcaYerQ37P3kJz+5c+fON998U9TFYDwYjAcGqiDxbt49LYoCQDoYTtebgih7eXV9eHKrruV+ufV4kGdlglnAQ4BwEsaY0dFokvKWEEIp2S5nyInIp4NelG+92OeirRilSRhOR8M0TQdpr23bzS5rGlEW1WazC8K01xv00j6wEGMMAZailK0IPR9SmzVtmRfByeHdWzePDqe+x549fz4ZD5umwQhYq5USbdMQhCF0+2zrrDk5Ofn4ex9/9tlnxhhobFtUEQ+HvXS93PTjKImiOEQM4e0uU14bhWg0Gq2Xq/woe/ny5ezi+tWrV17gj6cTJeRsNiMeDcKwN+oLK6+urv67P/s3f/Uf/nKxvnoHhG8bnYMaY+z7YUWaLvcLAcIEvwt9hWHcbaCkUkrpbid3cHAwGAwo5RBhCGFX3aSNkVIqLZTqYEEWAIIQMAgaYxgjVmsHDCYQQFtVhXOGUooQ6bSv7qeA37TXAEYpAIBSQjDEGA8GvSgKRqPBHz77tMgrCKlo/o5PzD0MoAFQtm3NOf/pT3/65RffzOevMH5TctZ9dJ0jvu8fHk2Pjo4AAF988Rl0llE0Go3CMBwMBk+ePMG1QUkgpZ70k36/N58vd+siPIQHvV692wQ+nw77SRxC6GbzK6PUr/928T/9z/7Jydk/yevG99Wrq1eYip//gx9keZmOfYjQ8DiJZ37TNOlhevuDmx7A6/WWYpTvi322cWEPOLDdrtOkX+QZQg4TWDe10MJYsd3ZurZBEAQhZwwDp9u2yfLVernkHEMI16tZVeYI0zwvIITDwbghZDgcQ0Dqum5qkWWZ57HhcCJVjTEghBRF2TTNbrfbbrcQQlXnPkP79SJJYmhVK+o4jg8OJnEcdxyhsiylaqllm/Vyt10rC3e7TBvlRbGUqqkK5rF+vw+BDcNQy1Y1DcfoaDJMOQ6C4Kos6rIySksHYBcB4BwA0BnNgLFSycD3DyZT5nHnXINC4GjVaEYJxDbLsu1+p5QAGHKfcQ8BqJwzShtonAKgKozv+1EQJqOxaNpity/zIgzM/Hq2nC+CILh5+xYAN4bDYavkbre7ePWSUNRP0jRNnTaUUoJpVVVSGyS0UgYTRB3llDGCCYLxoHMJwN1ul+f5YDCIoqjrWeiawN6lb7oVtZK1kBWlcDwZTKeTXi8JfWrNe+vFMsuyvMwa0TLfY5TXqmltYyBQSmEMgyAghpjKAOg6JwRCoLOINU2jtXxDm4bIGMPIG2+N7/ud7UlK2Y1iHSdqOBwOBoNer3fsQBAEUkrRqjAMCSGejyn7YD6fr5ZrpRznPnCGUex7KQLwZx/dds4o2YDG8kHMKNUUOwJ9TmjkF+X+0XeF5/vEcxRiC/XD7z2Ik1BbMTkc0cC7e/e2cyYvdsvVQkopZF2LGkBtlPYRs9BWQmHuWYeF1LusmK2X+6potYoHqTHGaimBrZWeTA/SJNrk1aefflrXdXdz4J7XvYbH43GR71+8eCFb0UvTDz/80Pc8xthoMESc1kqEvSQvCqGUc+78+rIUjTVAOEMJBRRbDLVzAGNKvJE/2O02XhgMBr3NbillOxoNB4PeyxcvhGisUciBKODv3f/w9PQ08L0/+/e/pZQq7TAhHELrEEBYG/DWo/B3jeHd4KC01kZ3niQILHDG5wRB41QzSseffPjgVej1k/RgMjqdpMa0r2ZXrdVeFCJGaimU0tQBqdVsNhNNG3gMGYsRGA/7B4PR8XgcRt73Pnh4cnJ6fXn1/Onztm0JREcHh13IgmLWNE1dtX7Aq6qx1nKK3trMO0IIMlYqpdu29X0fY6yUqaqqS5Irpba7/JtvHydJL0kCAGDS7yFnq6Yk1MMQul2+SXrh9GC8ywprHfMwIbEQwmMoTQIMQRxGvTRmjJ0mCZqMMUZJL+4PexA5RHFvkJZ1bVvdFhUyLgo87FwYeAfT4UEvLoqQUnx0NLJGrNfLpip7SVwUxfx6cXJ0A2KOoFcWMg7dZ599893Xz7Jsd+PGjbIsRVUutxth9DbflW2VVTnmRDpx/uzlw4cPb79365e//OV/+KunnudpC6palE2LMU16/cvZ8uDomBBijeE+dRAxxgBEgzRJPaq1Nkq/fvXC8zxtJHKGYLBeLeqiPDg4iOP4cDLtpWmaptfX1xbzO7fvWgBfvniFkH/z5i2C+YtXrxljUsrQ80XdjHp9UVcR9z1MV7NrZ9TD+/eW89l6vRRNtVosKKWnJ0cdsW06ngBrv/nq6yAIHj58QABcXs/qRnieBwllhCsh1qsFwQQjxClTPt/vDQQGWivr+mf/8Kdp2t9ttvP5HGEcp2kj2otXrwEAL85fvvfwQRhHz7/5ElN0fHy42WzCoNehkJRSVdUqYZIkCcNo0IdKKSGE1ho4CBxyFhr7ZtXXtm3Xzai1TtN+GIZ3795FCFngjH7ji2mEqOvaGFNVVWHfgGgYY85Yay0hpEs/dpd70bSbzcY51+8PrUXOuY5ST/C7xTOVUgjZKOAopQ6oIPTCiLdtO58tm0YFXjocTrqCnG7B3IpKyrYoqs56RilFCFtgCCMdA1GKpigl32LPYxiCqsgBsL7v379z8/79+3du357PP/nv/uy/bZrk8vKy2a08oE2VES0Tjhf7cp1lt87OJtNxEgVVla83C0rpi5ebzWo5nE4gsM+fP3/66rVFqBDtbLW2Dkpty7LKmgojuq/Ll9fXdycT62QY+U1dl7vicDK9eXJWZkeff/45ANrzAmPEfr/VzgDgpGr7vcnx8dFwOFRKZEXWCjMcpL7PpJSDwWixWud5yTFO4xhCqGR7fHz8ox/9pMjr3//+i0ffPm5FPRoNkiQZDE8wBoyx5XK121ZVKR4/frxZZ9Vm6XNeGNFPpr042mwgAq5D0zvnwjBsqhoB5zEErXHWIB6pjlLLKAGm2Yrdbj/op5PRGEO0326laDzfh0ZTiLBzSRTLVjhtHIDOOQQhZ4xz3uv1pNEIoe164wU+9T2lFAYQ4bCsdS0qP9AY432etW3t+bw3TDinvWESx34r6uVmJWTLOSeYFHkW+/7d27eN1LOLS2f1sNcPfb7dbo1UyLjRdHhychRF0XK9+n/+32ezq+tn6eOzWzeFaDDEPmedWcEaZYzBGgLrtFFt2zrn+oMeJQQhJIWo67qp6yiKOGNVWSIIMULaOWctRijw/TAMPau0loTi27dvnp4cEQoo9h5+773rfrTbZbvdrm5E28p9lVeybWQDw8AY4/s+JRggVNalFAJjzAnpEmzWWo9xwHj3lEFOCSFxGHa+HAAsdKBTEHu9XtM0m80GAOD7vnPu+fPnLGR37txhDBfFnipIWehx3O9F+x12VjHqhZxiZyOPe16w2+02q8V0Oj2Y9g8m03t3bydJXBRZnueNqP2Av76+On99njeIcuYAYL733vsPeoP0w4++5/vhdp9fXF399j/8+6+++mYwGARBwDDpHjHDlLOmrMvr60vKPMK5AsARJIxGBPuxX5a1dQb7HFBsEOpNp2kcXl9ff/7NnzPG6rpGb1lAEMLtdtuNUNcAkmPUi8Lj42MAQBLFRVNrranvIdkqqbTW+9nMABeGMSQYe4x4HECoW2WBAwBAo7pSUAhdFEVnZ6cnJ0dKdfyMlhKOgeOcD4eDw8MDZ20YhmEYIswo87RF2gBtUSu01uJdYtYY48AbaIeU0gHjKPajAGiBEGQYa9mcv3giyt3/9n/zv4LAVfu8yHOgqzLL8rKwGHLODQTK6A7cVdd1P+1ZpWutnmy3AWez8/MwDDECo9NxmkSH49HD+/dk1RR53tbN4M7tuiiVEpxza5y2jmDWti0h2L6tVtdaW2eNsxBZ8KZ4j2CMrQVtK+q67axdwNLFYvUf/vKvDo8OjHEeD6Sq+/0+8X2ulLq8vAiCsN+POacAIAgwAKCuYMA9Rmie7ZBVaRiM+r0IAABA27abxXy3XTrn/NCXzQGipC5qIxXQqs6zNIhuHx+FnA76UVvtGYYecYCSti4wtIN+ihDKsuzy8jLyo17an4ynVdX86le/3a1BksD5YqO1Pjo+yIryD59/MRj04jgeTydCyYvL6+1uff76VRAEQsl9XqrNjvsedG56cAAQPDs7XW83vV5vPBnmZU2xs9BZZ8siXy2pClrOuce5dZoyjAwry7IoirquyzwfDAbGOM8LECTAISk09ehgOC2K6uXLC6nA3XvvUcqNVFE6KHQ2HU/ybDdMerqubxweplFcb6+B1giA0PeMVuvFQjTNjRsneZ4HXuQxdnR8CIx98uSx73sAw/l8XhY1YdRam+92/d7eAtAxKiaT0enNsyKv/Cf+brcL/CiKIinE5euLtmmscWdnZweTQ2MMxJgxVpTZ6elpGAcX1xfb3Xq/23mcG8SttVprJbUxEAMMAWc0AiEqy7KqGiEUApAQ0l3FW2UJIUmcQoze3Ve4HzBGIITaGuc6wVZrLZRqMaYYI8/jlL4p3tVKdS5xp3FnMtday7aVUkgpPa/zcjIAAMaIYgKABdBijBAGCEEEgQNKyqZpSoTQeDx2FjpXdqWOSimlpHVSqoZQMBwOX7w4/6u/+qvLi5nHk7puyrLsyAph5MuWN01lrG7rQik1HCRlmROoerE/HaUHk/77791W5QpC+OTJk5cvzzEkHNndLhPFbhAF33vvzs1bp0+fPnm5nZ+cHP2n//gfQuSwLDBNWoFagefLUkgIKf3DZ4/2RSm1tcAZAJtaaK2LSi9W+ddQR1F08/SW53klKLoIPoSwad6Y7BpRh2Hohb7v+3mxr/JsDu16OSvqQkoZRcFgNByNBpRyPwjqukYO9fvDomraVjRNwxjr9XoeD7u9SecErOt6tZ4RAkejUdsKz/PaxiyXy8V8Q5w4Ho1Hqffww4/iQS/PtvP1Wr887xY0HSU9CvxeklBKA99fVBIi14VaulBAnmdVVd08OjKiFUIO+qNRL9VKrtdbAKwOgmy7Y5jEUQQBwBgnSRJF0fHJSV6VQsnXr1+3bYsZdc55jBdOWO2g09Qw44QQwgHLfHJ4NEHIjifD0Wiw2W0226XV2k+iClhotGjruiyBtkBr3E2vSlKAlGyvLl/nxX7QH+nxaLtcU4rn8/n5udeZHI1xKu0D56AzSqm6qpoaS6Wy7a7I9iYI9vt9V8rV/fouc8E5T9M0DEPnXOf+7rI8hBAPYaeNdaZ7ke93G6tNL4lOTm8cHB5rrYu6Wa422+0eYZKm6VoI5IDnecwPILDE+AAThlGeZdZahnESx/1+P/QDAIC12hLknMMQ5nlelqUxilPWucwGg0FZlmWZK6UQhFrrsixj3hsMx+vl6rPPPuv3B//g7/390A8ePXqsWmG19AgOfB74LPb9KIqcEv344Ec/+tG9e/eiIIzj2Pe5lFKq9vLqinrk4MbRj372o6qpn798Jo2+995da+18seLcX212L87P/+V/8/9qGtEfDOKkBx1QVgWef3RyAp27uHj1+tXFarWy1rbCSGO78CHMQataaWQrW9/3ISUAY4S5cmiTvYnwdGNQ3TQIA+fckydPOCMAgPV6XeVF5PFemnaNWZbh7g3XtV1gRjGjoecZBwCCGGPCqHNOW9MlZrPN7uTk6PXV5Wy2+OGPf/Dhhx+WdfnFF19AANq2ZhRD64rdvq2bb799tFmt/cENKaVzAGOMCMUWYousQ0KoNyiCbhMBYQeuRlg6hwCwnHMLLaV40I9lW81ms2y3+sf/8O/fOrv55JtHTV1K2f7yl3/VjRfGGGON1hoj0gl7oR80QsZpvBNtmqbL6yulVJHve5NokMTOaATgnbPTwWB0fn5e7DPnnDPdRotgRBChBDNKqLPgXeJMSomB4x7h3IvjOMt2+/3e90OCcffBVkpZZznn1/N1nufcI0+fP0fQfvLJJ4RhZLTbLObe8Q2OEQl83w9Fq5y1uqp0XedVsVuuz05PPrhz6/79+1/++j8WRXF9fZ2Vhed5zONJkmhre71eEEXT0bitatu2x4fjyWRSFIVTYrWYR1E0HPTC0Pe55wdpFAVdWBxC6JBr2joIAmPMcDh4+OB2h2GP4/Dg4ODps8ez+SqKU8qC++8dXM8ut9vtaDSazZf7/f7o6OinPzv6wx/+YCGAAMRJ+Pr1edLvMUZfv37BGaqKXRj6YRxZiLRGWlXKAgqBgYB7nueHnHOtNfcCAIkD2A8TCyBhftlKTKR1CCICIaGUU2XLsn59fiGUthZIKWWr0jAGYeSMDT3/9Og4CILMlcjp3WpOgEkCXtaV75E48qwRR8cTYwxjyPOC73/y4f27946Ojh49evTehw8nk8l+nz/67jsHAfe9ycEYIXLn3u3b9+4++vbxZrcqiqo3HvQmvTD0d7vMKQ0w1Fq/OH/ZcUazLLNW7/P9J598/M/+8//i+Yunv/39b6fT6XZXm7eVtZRQSjgExGigFRCtqSsppWCUQoi7fafvhV3BrgUOIRRFURRFhLOmrRBC1rnOG2+t1UY4oLR2lGJ/kHYvciFEkeVV5ZRSzhpjTNvW1lrrNMIAIgfcmzBCd85SSglBCAOEAWOEc8oo1loZq8pq3/3NkySh1NcStm0rhKCUcA/3+31jm27PvV6vu3/gZrMhHpZSllXugHFGIwwDzsLQLwsZh3yzKmkQGi04gdvVHBj5vY/vxWF0fDw6nAzqqq3K8vLiervd3rt77/jGCeFEmtY46cX+j37yQyklEQ5TYoBztcsrLQ3pDwZsV+msbqUw2hHOPM9TSuV51pQlropbt+6cHt+KgnhPsvlslW+q+fV1kVdpkihlIEZHRydhEkopAYDJYZjn+X6/t1aPBoPTW6dnZ2fD4fhvfvW3xXx1+er64Oh4NJqslt+VZR2FCaX0+fPnTS27cOw+2+73+8Vicf7qWVXlaZoihCHgHo8ppaPRKInNaDJGBJ8ej6V149Fgl+27JgmEECM09L0oCAOfG6kIBB6FoUcAAEY1vu/306goCmgd5UzUjRBtOPAI9apWQ+JhjIumbOpaCqE9z0ilpHTOMUwoIUkYSaOttUYYp42Bxg+CMKKUYUIwIaiqs1Y10rQABm1bWqdZhjgnSkgCaehFSdjP0T4ZTygmr1++sEojAJMolHVdGjsYDAAARqrl1Ww5X/ivg+VyCYGNoygMAoxAo3We5865NOkL0YqmqovSAGeMM1r6Huv3kslwFIZhVVXQOqdNlRej/mCQ9nzGGWNCiNIBYCwwVrWi0KZs6jiOjZa75fra8zgjWgmtddfB6HuhBXC72+3ynDKvErIFiDGmrKmqyhgDrKMMA8CG0wPZ1M45L4qo7xkMVSuklK1qIYQU4w6WAID1GGeMPXr0qJ/2AADdTU+2befomp6c7bblo++evzq/Lgsxn6+Hw2HohQRh0bT5bssJDjnr9Xqnp6e3z06mZyNK6Wa7ffz0CUIIAbjf7/M8z4p9EAeDUf/49GR8OPzok4+kFpzz2Xz93aMnmFFlDOWhks7zo+n0CEFS17UUupeGN2/fIwgv5qvXFzMKnNR2nxVBFPnEk62w2viM+36YZVl3DwEOzZdL6MBmvccYN01DgOvSp57nh1HUFXZEUSSaViu93zdxFHXHkWgUQqip6rIsjXZJkkxHYz8KX7++7LJXzth3qwGEUL8/NAYiyD54eP/jj36YZdlX33y7XG7GwxHBHsGYEISIhyAVrYEQb7fbLMscwA4giDmABCDWnV2E2A5AjxAiGFNKoyhqhVZKWaMggcYojL3ReNzUweXrF01D/6//4v92986d+dX12Y3TX/ziF3/9q99h3pdSSKEsgFoZSKG1oBN667IiFGljHIKAYkQJYnQ6HP78pz+9eH35+sXzsqzff/AgieIvv/xyvV5zzjtWmNa6KmuIsHGgqx3DGFv7pkXsHZela4T2vKApq6IohBBBEDQCE8KMKbOyChRrhXFW7rYF0W3jM6qkiX1vh7Gy2qPE1MIa49q2Ksq2LBLf++GHH354/z2CsKUQcIwDxiznvh8EQRhHQRAIIfr9vse4URoHkGM8HQ6PJpOvPv/MKssIacpKi5YRcvPGKWPk/NULBCwnWLXNq8tXrW5aJc5un944+pBz/urVq14vqVstJFRG52UTVi3lnlAOIAYQ4z7qIcK8gFHSGyYQk+12ez2/FNpMDifL1eVml02m/afPH0tVJihgFPmjBCNKfQQQyooSYyyMiYOgWG0BYfkuI4gRHhgLqIfzooFIOOxhxDzPC+NeT2upjBAKQjgejPM8F03bVNWg15tdXQSev9/u6qyICC6z/cXFq7ptAo9B6DR0u+06y3NjVJ7nCMLhcKiFZB7d7TZSq/5wMJxOylYAiiElkGDi8Z/85Cc//PFPRtPJ1WIunZJOCdUuVvMTdOi06VKCQojNPivLklLaHw3zfP/0xfO6Lo+PD4fDoWrFxfkrGh87CyCEBHOCEQBYSlvXLbTQWgcAxJhy5nPOu6VDF/Kx1lZ11ekBjLFWSYwRhARA4IC1zjjgMEacs7YxhJDOqNjN18rzOuZJh1vvzIPd/30TTIJvPBOdYuF5jDJUV4XWglDgHLROa+2khB3Pqvv1VjvnAOc8CHzGUa8X7/Ziu936vu97sVaubWyapnE/Qgj4nkcpFo3J8qzY75aLK4phEPgIuhsnB6ItfU49hhEwUcylbIUsla4h1GdnxzdvnmZZ9vr16/NXz4TRXuzznvd6cXFjfyZE0251VRfzxaLRUlhFPF62ZV7vLRTWtcpIbHkQhgHHZdkURXt3coNSvt3mba21ctyjnheMRweb9XfWOANBHIaD/mRf7OfzheezOIyttpyypJf2+33PY23VLvX6s0+/iKJECDUZHkxHh9/ox3lWO0uHNn358mWeVc5hpVTXrt627fHx8XKJfN+HEIVBP44Ga5yZCIaEA4wQZbpt1ru97/MgCBChyuiusZMF/nDUT3x/06ydaiOfD9JASinrkmMYhb7RknOOEAnDuNCukSCvlVSABj1GaIDNrktbSdU0TVWUnYiitQ6SOIgjrVQlWuNs1TZpmhI+Yj4nBElVN7KGyPQHyY0bhwgDI12+29dFCQBgmAc8hBYFPp9OJwThKq+EtqN+L/Y9DAmE8PBgKoWywDkIyrKsq1q2YrNc9fv9MPS7lgRg3Xa7LorCGtCK1liFEbXQwrc3wtAP+mlPNC3FxGO8yPLdZhuGoVHaQGSUhg4wQhmhWqq6rFwtrEFaibrMPY95nGmptJYAgG7jiQkVyjRCUo0QaQ5OjobDYRiGSimlBKesy6cZq7orHWNUA9AK0YpWa11lGSEkCgLO+XA45JzGYWStTXtxHEZSSoyhMcYoFcex1voPnz5aLBab7UoJsNuWv//0y+89fDidHi3nK6eNkm2/l4wGibWWADPoRcjnl7PZ5esrBOGgP6yqarndF0WWJEkUJ3HahwhDiG/fOgtCb7ffb+fl1Wy5Xm+DKD45PUvSwfV88eTJq4ODA621Vq4s2t0255Q1Qitle2FMoBJUHowOHMJCmZKUhBCMaZnlqhWWe42pz1/k3f1h0Ot5nrcv8qZpuhuLc67f73ucBkHgEtuLk3K/C4PAWhuHkTQ6CII8zz1MJbCcseFgACAMPZ8CpKVSUkIInTYIQopw3Juu1+vJ5PBnP/sF5eSzz355NbtK02G2rxAGhTYUQ5/QXFdKKY/xIA3qupbKaq1lqx3AlIcO4HeO7C4x29154jjO88IaKbWGkCijpVaQUGMdYj4Lg9998dWz86s4jn/4s7+Hw56AjDrYKmOdhYgg4yzQDjrnXGcgqNoGM3J++dpqtS8LSvH3PniYxsn/8O03V5evV8tt6AfHxzd6aVqWZb/fx4gy5jkHi6JI01QphSntliMQvqkr63Tfuq57vbTf72ttr6vaOdcBTBHBTdNAjAjihPOHDz80Rl1eLUldFNPpQcjQaNDfrje50FBbI1ohlFXap2R8eHT/7p1/8Kd/6jP+7//8z+f1nFJKfY9Zo6zNy9I4RznvFj/b9brIssDj84urQRI/ePDg7MZNYOHxyaGUbZZvouHo+9//PmVkNB48f/48jHwhRBj6vn/85MkzxkmW1wi1ry9mWV62bU0IRxg8e/7KWHBxdTkY9LjnX88W77///tnNG59++unXn33aG/Qn49F+v724fBXGsbGyKPLpwfjmrTuPHn/rgBayBhqGQQSgrlsLAMiyrJekrdLKusv5It/tX7++HAwG4+kxQogx3mrrGqmNff369cHBUZjEddXmZYUQ8r2g27sjhNq27SU3Hn315fHR4cX5K+v0/+w//QnnfHZ5ZYDzfX8wGhFGy6b2fL+ua611V9C+bddCiKaql9sNAIDuvF22x4T0hwNEiRf4v/ndbyFj9+7fL+siSuIgCjEn14tZvtoKoaqm5swLw7A7hTtXOaX0xo0bbVV+9913P/7JD3/0gx9+9913wWDYofIRQtYaKbRzjXMujUJGvW7lH/oBY7SLd/u+PxgMGGNFVVprO6Vnn+17vR5CCCDYDQHWageBA2+qCjoqXLcJ63L/Xb9hF0DowuWdsaALQWACCUGEYEUQ4wgAlqaxlJRxzChmFgWB1+/3CcXnLxfOQqUcITBJ4jRNGaNKN2VZdplyzwsAAPv9vq70yclJ2WbD4fD2rVtHRwdStM+fP3357Gm23/qc3r51evrD79+9d/vrL744P395+9bZoJ+Gh1NXVdYcIweKogq8QApdlvliNe8NB0EQDA9G+yp//ejbsq1aKf7sX/+r4WRcliXiuD8caNNaa4eT1CHRHwQQAS2FlNIowbnEEIhWlUWz3ezQgPh+yIhfVyLLivHo8PDggFIcxhHn9Ktvv3n27Nnt27eXbEkpPT4+OT4+klo9f/784uKiqdt+fzidHBZ5s9tlnC9Fa3rJYDQazedzhIgxzhgLADg8PDw7OzO2vbw6F6I5PDw8PDw6mJ5SEjz69lmRNz6UddMGHgfWFvke8aCuS22hMrosSzB04+Fg0Ov34kgrkYRRruvQQ9CAfbnH0IRhjGGHhMP96SSOU2wBQEha0bZStkXSp845LaRgQkvVAQnquj4+Pp4cHAynk5OTk812K51pNxtCiKqtdj1KQNvWraijKLh1+/Tj739Qlrmom9VqtVyuIYRJ0kOQZvuqK/VJ42TUHzRVu19vpJS9JL2+uG6r+noxZ5QPxyNrbZHtynzftiaO4ySMKMVRNPQ8bz5bbrdbjwfOWkopxljXuimLXd5IKY12d+/etdb2+33f91++fDmfzzuSdEcK747UjumklBqmPWOMUibwI8ZYU5dKtlEUNU2TF1krFUAEICi1xZhsd/ur2fXNmzcnk0lHouzyJlIrY1SHbQgjv4O4YIypz4+jY601hrDz/SRJ1E971loHTC9Jl8tlx6geHx0RQp4+ffrtN08ppQfTG1qq2ez6+mp5MD7glFPKKaXD4fAHn3xMCPn897//+psvtNZtz9utN0KoyXAitcuzjHn+D95/OJ2OKcWnZ8eUo2+/+/Yv//Kvj48Pj04OAWJp2p8vN7Pr+auL69V6G/jRD3/44/1+DwDI99nV1eJXf/s73+eb9W46HQ+S/mwxL7Ly/r3w5Oxm2bRZVlwtFuPxFAJrteGUcc6rogyC4Pj4OI4YQujRk8fz+TwIgqat1pvN4eHh4cEEAEAQvnnz5vKabjeb/X5PED44OhwMBvP5XApR1W2nWuZlEfoBxUQoqYWEEFqlu66v9XrLuX/3zn1K+JdffHV9NY/iFEEspY7jmAckjcJeFO822+127Xn+wfExxjgvaqlM3TZKK+MwwqyjeXZ95R0hprtTAWMQeAMycsAKIaqm3ux3DiINUDoYawtGh8cPvvfJi1evIAuJg8gB6CDCb59iCDFEWZZFUdQ0+ujk9PXr8+F0vN1uh4P+4XT6+9/95r/5r/9l3TSex3/zm88PDi5u3bp1fHQ0nkz6/X6v1/N9P8+V53lN4zrXrXWIUkIZIhg559q29TjvNBtjTBAE0+m0LMuyLJXheZ73+gmlWIrmxtnNNIq//OJzMuwdEuhVZb1d7BlkQOfjfq/a7frjgc+R7/svXz4/vnfSIPnvf/kXj88fTw5OAKXCCWFMEASEYh4E4SBZLpeRSR1lN++/n6b9z759/vh88b+b3NbcP753L47D3X67XVxP0ygXggPrqHf3vQ/2+3y5zgf96ePHT6tC77f7/nT7zTffcM6ZJXlbcM4n/Ykq3DrPPY/tL653+81w2L8D7L6qvSTi4YEFFKHo/YeflIWSbdvm5d0bNxBC/z+m/vxXsizPD8POfvcl9rcvuWdlVXV1V3f19KwaikNyJEoEDesHGoJo2LIBA/yTCAOGbNgwJWokETSHM8OZ6WlO9Vp7ZeWe+dZ4scfd79n9w80qzvvp5UNkIHDj3nO+57PupsF/8Ye/+/U3TyGGACEAGylVXS8Y9QZh7DHkWlvMV6nrQo8eTqIkdQJW+F5YlqtxwoDFQiDkIKnqd9/76F/9q38VhJHneeeXL6MoCgIvCGnoe1998/nOXi9JnSLLkiQiodWYc1ivNhmAFDG63GYHB0ebTeZQ5FNFAJue35wcHnz+y08vLs8q7RVFcfv27d6gn6R7N4t892B/sSrmy8Vy+6fOn/+VlNJqgBFpy8Zn3nrd8Kb1fGc42fF9t24qxVuMoe+FZZEfHIym0+brLz8+3I/393qzG+dy+qkQKu0Nhju7yoCq1tqaWmte8DAKI3ewWi7bOh+5Axb6GhkIYQdRWGshgkIIDazregAiAFHdNNfX10m/t91mHXHriZoQpIHRWhkLAUSQYC0FdR1pDNecejhwXSlaYKt+z6nXsiq3DoqJ51gtPS91fX+zWfUHydVs/js/+VFdb4tsGwXuzihdLudAG6Ekb6W1UChbtaZqEReqLOtWCAs8l6ZPXj0PggDFeNEsetgNKclXV6MYxpFz7zj43r0fFvnadd1bt+54bvjNk1eQEKGJG+5kJW2aXBmlI+foBw9dTK8urp9+/bg0VX+vv7+/r5Rp22YyHNH3v/f151/5vo/cnctpxZgfMn+70sxBfogZU3hAIRYIGyVBUejNSmhpEIJFXTZKIIcahDnX08tr2Zo07ffTwebs3PM83/Oqqtps2tHkiLlJA+jh6b0PPvhgvV7/6qc/LcsyHu6rzQYRssjy8cG+BeDVxUsWQoSQpdwUgeu7EEkj+ajX3263n//6s34/ZTiIo54HozffnBfz8v79+yHTtdluNisNbF4VtZLCWozhwd7h89cXUpI43FUCxUEfAHB19TIMYBiT1Dut0urjj3/BG2VNW5Vqd3+/rOTNbLnNiij0CYJ1lm/XK6sVY+zskjMWC24Knse+f7B37DkuhvbhwYnreRevLj988O7ZdPrq8uLWnftFWVPSUwYmvp+maV3MPWaPx0lg+Wo1W6+3UoIk6GWFmF5z5jPXGYUYtSWBQgJjgdGhP2AJhhA+eKfHRdtKUVT51fV53VYWQovUeBQc7O+Mhr2ybhSAR/sHo+Hk7OyqyMuqacf9cd20i9klRqHPEIV+rcTNanFwsOf77mazScdRU5aQSOJgAxqIkOdThjEwCkDsOU7XpBqG4Wg0opR2EC7GeHIYdTbR1WqVZRkEXGvdFKVw2OeffQEA8AOvW9OttVLynZ0dzjlE1mithMYYd0mhFnlFUQyHfWOthTjLCsfxmqpsmsZz/MdfP0MAvPvofS1p5CX3bn1wdCcDALS8QYTuHh9qIy+WUyd1BZI09a+2SxC6QRSfrVev50ul1N34ADA/nqTz5eLNcqag/d73PwQRli745ae/+dWTL/f29m6uZ6vVKtuC5UJfz65+/stfvX6z3tmJq0buH55CTL969sJznOnV9aCX7O3tvXzxbGc09Fy2Xcykai+ml0mv12Lr9pNxfHyxWHzz8oUxxqEkdBnk9fHhzptq7UE8DlC92SJKIyUFxT945+6r89dTUweO2SyvwjDWCL188cRjnlYWWSoaNbtZWYOBJZi4ge9gRK+mS6VUy6XFThoOWq4QIcxLlVKUhpboO7fuikb82Z/9pTHG9+PtOouSsJeOl8s5sDa+lUx2jvb3Tl69ePHq1SvWy2ezTdNw3w99J+BWqlZxyTEm1EKEoCGoLnOH4iSJEIFGWKBw7Pmi5ZEbjwa9alNHftJUJZB20BteXFz8vd/7Q9nyJ18/1kIqkpaqshb4DsUuxQhqoFFrTCOkEYTi+c0scFwrDNLwR9//cWbQ//A//H90MK5FmdeNDSKd7L1Y1+XZ2enpbrgTDvbS/Wy3erxt6iryw6auKKIMQ4Is1sYaRYnjeQGldLPM1rNNFMZpMgy99FJeb5aFdeF4b0/wdjq9OTk6/OaLr377J7/17PHXZG9v/8mTJ5PJpDsXep53dXlhrDZW+76rlIxDP4mibLOZT2/aqtay4YYDLRE01nKHhYzCusoYslWxpRge7O2FYXx1fpZvVy+efcOtCX3fdX1GPYydxXzzi5//Oi82Uspbx0fj8TiKos1m/fXXX+Z5eXx8vL+7e3F2JqWcTaeMMQnAdr3u9Xqb7Ur6vu+7w37PddzlYtEF4wwGve12u1qtRuPB/fv3CUKDweD5i6fr9TJKQmttmsYWGkQJdVjLOUAVBLRpqjyrwoDHYdq27XK1Frz1A1db2DV515UAFgthKivG4x3HoZPJhFAmpQwCz3UZtKAVjRGtltw6uK5rCMHezq7RiHgsDNKm1hYiCHGWFXX9YjLeybN1XTdxGHnMMwZwLrNtUUnQNLIqFcVqm9eN4L7TZzBiiLskdZmPgWh1iyByXIfGlPMzgCB1KGaUuQ6m2ILAcxhlmDEcBR5hGEJoCa6leOd779WffH5+fj6bzTBlmLKibKW2AGFGnW8jSCEAsIMEpFBSlK7rdsmAECNjjLKmwy0AANACjDFFGANolZZSRmloDMDdGUfbLpEGY7yYzylG0GqhpG1qhxIGQFWWO73BwrQB02lAldHZZq5V5DFMEDjc3w2DgBIo2oYrPZsvLy+vlVWO43hu0PnRhFBKGi4VpbThQgitpYmjfq+XdAGjq22mrGHERrF/sP/w3Uf3x8NkvVqsVovADcq6KfPs+vxCNjpbbbwgSvbh/v5+4PnFltcQVet6cbV6/NXTo6OTfrw7ny2fPXvt+esgjlTjzLaV0gIT6HoMIZRlW2P5GMX9gY+wVrqWkkvBJUfAUodhgl0vGiplhAI3y41oeNUqRhxMWFaWvBEE5WEYEgQAIdhxIKFV1fz615+8eXPueR5C5OTkFiGkKKq3mlAhOedF0ZW0ESGUMXS73Vpr8zxvqjJNU8ehr1+/Pjk+/IM/+D1j1MsXL7755nGWZfv7u1LyssoRIWEchWkICLPEgSCPfaepNQagbcrlYmblVsnMdwe+553ePtFaZ5vVV4+ftEJSjLUUeZ5fXV3t7+76vlfkW6P0IE3CKNZaDyeJaLlqmyQMT/b3e1GY+H6/1/v666/Xefbq7NwyUnB+dXWJmIMILpViFIa0n/pJb7ITusgldLVYYoiCIJB5XZZl08im1QpA6jBlUCsqrQk0FluDEWobY7V6eP8BJjD2nbIsz8/fTGfNzs7O6P338rIZDodRlKjUWguiMD466D168OhXv/zk8dNns9m0aSXnHBKICIGdHg2hThjYOSSB1l18MiEEWNhpfrXWGCMIIXsb+Mi78bqu67qurbUHBwedpuHtV9M0HdIG/060ecfKdUmRs9ms3+9DZK+vr40xURRBaIuicKCvtBBftUpyY/TD+3cBANAa3/fPz8+VaOM49TyvK5oqiqwpM85525ZZvg08GgQexeF2NUvTOPBYHHqXF+fnUp29fiE57/cHk8kkc4rVdoMxHgwGO4f7rusWWf78+fOiKOM4Xs4XVVU1ZfV0+/Srr76a7O3GYXK4DxAi20357MmzXq9/fHx8+/btR/cefPH5p5vl6ujwpJ/G89kUGNsu13t7ew8evMMcbzZbbLclAODo6CjbbLfbbRKFeV0sFgvXdYPAC4LAgbBq6r3DvbAX/vBHH/bHvb/86V93X8pqtfKY5wy8uhZtq7S227yKEdxut92B3mGk8/gBALSWGEBtJIRI8nY02ONSAIAePHiglJxeXRdFHqWJ53lCCEqcPM8xpp7LtLY3N3PGSBBF7733vnJwnpXGZEIIIY01GCEMIVDKQggBQIy5ShlCSJIkTdOUzdbzvLxYYQI/+uhHjBHeVGVZUhpt1uvlMh8Oh0EcVVXz7NmLPC/cKOhoC4xx09TKaERRBxIzxlqlJpNJtll5nrderz/66KMwDPOyuLqacQkgYlYrXleYWIyAVbIbOgkCo9GIYrZdbQhGHXJgre344s4tn2WZEMIaABEmzMGI+kE0nuzOs6yuytGgd7o/fu/ROzuT8XY5w1aR8c7k8vKyM9T7fnfVYByHGFpj8GKxCDxfcrGYzYGBh/tHDjPWKuxhhziM0fGo1wWnu5HbtmWa9u/cPiKEBQGd3awff/3p3sk7oUc9N47CZDTMZrOr168uzi/e+D7LN9vRaEQI4qLxA8cPnNt3jpCxUCvN22K7PTo6ogxHPpsM0zigXfAkIURrrVvJ/Oho9+Bg7/DJkyeb5UoIEQSB3/ni3ob8NEHgHR3vCSUsgtQhy/VqW0GtFWaEUZcQwoWwAPV6g6qiUZz6fqC1Lus635ZaAc4lx6A/Gq5WK4yh1irP827V8DzPamWkQAAapbNy7VDiu6yfjLbbbL3cFnmNmSNkxWsBfXZ1NZUtH48n+/v7i5tZW3OHuv10wBeQEggNlVyX2+pmtoichAIv8Qax3/M8rwR1Xdm6aptGuC7qj0dlWVqgNQK1EkYrCC2yuKma3XhSCqEATOLkZrkyxnz44Yf/+B8ffvzxxxeX14EfMs/HpBUKAIiklF2zSefzFEJSagkheVZ0/m1E3uYuK6W4kkII3/e7+H1oQdfqBKRmQ69tW6sQJRQq1UoBoHEY2RmP6jyz0mLHodBEnuOliU6FaQqaUtcjDuZcC4Bp6AAvcDhvd3fGRVForVqufR8j4vlh4vuF67oIkboSeV7WTSmF1tr6XoAsoogaaSl2ECBWa8n1zu5+GPkQaKns5dVMG4GBZdRJo/TrL7969epN28gk8DGQL558leflhx/d9YSVQci5EFytl5vVdCUqZSUqVs1iXixvKse1SUPqVretRsRYrRFBjufQ1s2yarttIIZStUoLrZVSUAhqLUHQpdSTgAICDcCttMpgQBwFyKbiSoi6bAhEQwt6acp8ShkzmDZFXZZV29qO9KlrzRgrSwmh6myrdV1XVSWEYIwJAQlS/X7/vfff5Zz/8uOfKyWiKIzjuKqqyWQkpdzd3cnzbL1e7u5OEEI745FQEjOEMADYQAZ3BolWB29eXyGIXOzsj3qH+0PRZq5rNa+v3rwIohhK0RRboXQShYHnWwuyLCvrWiiV5xWGMEygB5i0/M3Vme96kef2dkbRoDedXv/y168oxGmaXt9MF4vF6OCgn/SmN4ttXrhe0FR1f3d8MBqNh5HWDUWCQqC1dQlxE6dpeFuXWgGplGxAGAeaUUwZc7CWSrZCNAoZDbW5Pn896KeTOLl7eNDzXKLkII4Px5M8EAihkFA3DvKyzldLZNHR6a3333tXSj1brBarddcErYzU1vmur7JteVmWQgj3rebLdRiDACllrNIdLQ0AEAVvGt4R1p7ndQnmWuubm3mapp39TAjVtl1mHdNad9O5MaYLzu/swXEcO47Tpbl3idGU4qqqZtk6Dv0syyC0DsN+GCIEMCRpL3781ddxHE5Gwyj0i6Koivzi7A3SXFSbfho7WLkubeu8FzrDfvrVV1+MR4Of/OQnWZYZTH7yox/c3Ny8ePHq6io4OrnVHw2TXiqV8uLwiy++enN+7rruYDCklNZFGXr+8HZPCCGa9vmr15S5kRtui3LUG663m93R+Pd/+3du3749m948/fLLNEn2d3YgMGAEMQLrcnt8fHpycuvi8vLrrx8rZfKizPPcc1ylJaWU+L6UPInD0WgUxcFWVIBY1TQAqvV6/sH33vvq6y82WcGYWxc1jiiyRBtAiUtDv5Mwl2WJKemlA9/31+t1zVtCaM9xgN0ACIMgOjs7E22KEUr6yaMHD16/ft3NakIIyRXGmBLSIuI7juu6QpnFck0wDMMwCSMDreu6jDV1JbTWBFOMqLBtV1P3dviD0FqYJL3Ly3Opyp7nc8nvPXznv/6nf7zdrF+8ePHs2TMjjOt5pGp2d/c9L8iKcrZYW9jF0rx1T8Bvi6a6xq+maYC1XfVSWZY7Ozvvv//+z37xl3meIwwC5mkNsywDVk8Gg5t5Dqz2PVpC23UmpHEPQsTzwiGYIgytVcBAALUF2tpWCCmVUoorXVQcIaKUkVI6GBTZNjkYP7x354fffx8ofv76he9Q0jbi3fff++KLL3zfCyP/+nK1uzexShdFAaxJ42hnPJa8KTfZ4c5er9fLi2spBcbYAE0IOTnai6Lo+voaQJhlNgpch0LOa0IAofBmdmVwX0vQ7w9Ho8FkvBdHaVVuB4PR69cvF/P1xcWl4zDXoScnR4yxyWQwu+BJ4DsYxYGzvzPAGN++fVpVFa+U1lpzaARu2xZqk4ZRGAWbzSoIvGH/9uHhoWj5dr3Jsm3TNOPxGEIIkUnS0EKlgTFAz1eV0m3gJ4cHd3bGp0Uu3rw4k7IYDMdpGidp6AdRXdeEMMf3rEGQiFZWnPNnz54s1yuMcZblw+FQCm4opRgpabXVqm2U4L0ocAkDGr55fXl5NoMYBSHdbtZtq/o9H0LspoOH9x/0kqjJy7LI27bN1pkxBAOteG6ZdbA1vJBt0QuCZ998VWS9pN9jjue6jDrE8zwviFiiuvWLUgwAkIIDYAzC2HEBodooN4yZH1RVBSAUxkZB4HmBMWC1WhHWYMoMQFy0GNOugs8ozTk3SjiOE/qBAVZqBQRH6u3s2VU+BrHX1Q4FjhtHUed8C8NQANMqqbX1CQYIGqOs1ZQyn7rVek6A3R+OXIKgVhgCg+gPvv89x3PrRry5vFplmvkeJqDcrm/fu//Oo3fPLi/qtoGQKI2qWrTcYowBQEqaLnsfGEspZQwjhNI4pJQCABCwxELXD9Mg2hZrAA1G9qrN8/W8qfYizz3am+zvHvziP/7i9ctXSZKenuz0egNjYRwF7Wr6qth0TbJV2cwWawPgyfGBVuKLr7+6vLqR2hLqzWdr4ji7uwd3xgdnby7aViCIPdcXQkih1sumrDKIDCGIYAdaRCAGgALjNlq61LEYWwMRdTyKrLJa6FYaDbsWKscgbKwRLW+kcixw3SAIoi7B+vJy6vt+JwXvjFhKGQgJY5gxhjELQ69uqs1ms7+/+8EH7y8WMyHEYNhL42g6nSIMb98+3dkZZ1mGCbx77zYQ5Xa7rtqKS1EWBaprzIJhEgT3bjW1rKri7vHh++8/EHyzWt+sV7O62GJoCbYuRUopZIGxKgj8k5MTgDBCaOwHCCENwDrPAQDpqOe7nmib8+vL7Xa1vJnOpzeR6+8fH/aNQczrjUbM8WaLTdNwn/kGiDQMYt/xHaaVch0W+o7S7TbPXTcgaG2tdRyXa1vxRvAKIy+kPvNdK1UjubAKWE2hffLVl4N+b2c8vHfn7jAOj3d2AADFci4MwRhjL/AJy+T28vXZi+evzs7Obt99uLsz7o+G+PnLm+VCamGMsVZ3ooqmwVLybhvojrPf+oYYxkYBAcBbZW4rhDJGKZWXJZey86FZa1+8epUkieu6WZY1nAOEIMYWwr+ztdi/UzFlq6pqmgaTt1R3lmUAGEqpMrrhUlsLgDna3e/UwZ7LPIdZLfePDwe9IUEGWdPwartZJB4+GJ/euXPriy8/advaQc6j+7fHk+HLZ1+2dSF5/dUXnyZRnMQJgpP1aiG1JoS0gl9cXECEYsEhhP1+/x/9o380GY6n0+nL5y9Q51TSJuz17t1yRqNRWZZXV1f37t2bzudBEOyPRuV6/fjTT+u8+IPf+R2t1MXFRb8/NMa8ub588uTZer29mS2m84XneUIqzrlRnQST3zo6FLwpy3ww6E2n14IXzHVW10uM8d/87K+/98F7D+7d/7M/+7PReO/k6IgSD0HihT4lgdYmiiIusu12C1oxGk6SJKnrGgNIIIzCsMpzrqTvsarMm7ocjUY/eP8913EgAAwTlzkIojAMmXSEbDHG2gDOJbQIMGYVWK3z1TIjAamqBgBECGEWUeppZTiXEFMIodbGaqOVUUq5rrtYrCAxGgjmkr//9//wwx9+cHV11YpmOr3627/9+aA/dBwnL4v5Ynn2+rwoKmv/Uz8OZYxS6jGqga6qajAYTKfXURxvNpv93Um22f7Tf/pPPc/75JNPzs4X1oIksoP+KPRY7DMlKyNbUecOBkaJ9WaZRoMoiU9OTuYX5105pJRSG4lI1ytmKGWEulpbKXXdSKVaY4yFOPWIICgO3NClSeAW2xpq8f333iFv3rz50Y9+1B1Bev0oTkIjpdKiyDeB6733zjt3bt97+s1TYO2o1yeESeYQiDDGymrKsOf4ruu6riuV8X1fGfPq7Hy7yeuWe4GvtG3btizLxWLVhUBHURoFUZr2hZBtVV1dny0WU4KQ6zFCSFVVcZTcuX263W6C0FNKAGAODkZ/+7fPLy8vGXWTpMeYK7iy2l4qYO1l3hSU0tFggBAyStdlRSk9OjryfbeVVZ5vExqHsWeQrduGy9z12WRvuH+4E4ep0YXjBzavs6Ia9CIAScsl5xJT0h8MGXPbRtjt3Fh1cXVeVU0cp4QiYK1seS6ly5iSsi3zWqvd0ej9dx4dH+6/fnWxmq8opYQyl3l5VouWT69uXJcFQZBv19iqwbDf76Xb9fLm+nw2PddaLxaFlj2MqdGrKrtkeAhtXmW1lJuo1/fjJAzDOI2TXg84NkrijuGDECotAQCMYGstwoByx3MDKSVhjCD87OnLk/1d1/H39vbrhkNM094IErJab1erTWeVcX2PMSY5/06720Gv9u/WRhsLjJVcKKWaqrZCIW0RsEBqRCA0FmhljDFKA62t0cBIzmUaR4nHdvr9fhwga9bLxc319Kd/+e+CMOZKbbKKuP5gd99x3TgKZrPZ3v5BnpdcCMFNbmrON8vl0hglpAYGdv4oShjGFFi0XK55W9WVbNsWIgsAiOMwjmOHgtAlUrTLm6l3sPvg7r3bp3cYsp99+mXb8tOTO0mSQAhv5jPX8cMwvHvat9bmZbXZ5spUiKrRePfho/uv3lzml9uS52lvFEZe1fJBv3fvwR1vSFphrq9mXFqhEMIBxhATCyA3RmuDEGLWUgugtUQbhxJiIWqFlq2EAHuOT5lDsOFcOi5hjGitt1kmFccQOQ6lYcyFEJsNhFAZAxCyEGJKZVkyxhzGusNoRxt12VMAmJcvn282qyQMgiBIe/HDd+5WRfn4m6/btmaExkmYF9v1ZvkHf/AHBPoaGVBAU5eyzHhZOo7E1L9369bV5Ww1u754/SpwIUKi5UVZloS5SptO1sqVLcvSEjKe7KZxooztFKwQQq11VZRlWaqU4uFQK7XeLlYIhq53+97dYX+APCfFA41xUZaAcwuRtRBjmvTcRjTT2Q3Ck+Eg7ichgKpe122rAFRGQ5d52PG50HnR1GUJMVpu1k1Vu4S6lMRxLOpKlGXX+XR5foEMSHtx4HoY46apMEa+73u+A6EFAABoizIrXjWrbUYdz4/ToiiqqsIY+wGGyK7XawgAhF2COPE8j32bV6YoRRBrbbTWWisAlDEGU+aSjhUSZd18V5BmISrrJi+rtm0Bwo7na62rpqUUftvL87bkDEKIEFyv1wAAP3DjOO5Eyl0AeZz08mxrgBVtO97duZnPLNBJvIsx7vd7cRhiaDarpRACWOUw1O9F77//7t7ezmef/Zy31WQy2t+bOC597913/uqv/ur5sydFvtVarzbrIAh+8MMPF1neNM18uXj+4sX+/n4jeBRFYRgmUeQ5LkGYYiy5mt/MNsvVcDhMepNRv+cQXOXZwWQcuU5VFaur85ubm8X1OdBy2Etd1xWidRwHAPTonffOz8+vr27KunGYGwZRKzilVHLR6/XiOP693/u9bLt88fzZeDLarpZBGjdNAwl2PO/m5ub169c//tFHX3z2ObDg9umtsmivrueDgQswaupmOB5LVXQV24PBoGtHE0IoLZhDLNB1nq2BJcCcHOydnJw8un/3Zz/9j/ObGbDWYw4kOO33uJIvX71qhYAQKmwwot0Wp7XgTSPbom2FNbAzbFlrtbZKKYJI1/jdcokQQhBTwrQynuOt12tK6eHx0XqTUeYeH53WVfs3P/s4z/PRcFcogzF+8fI1JiSMkrfiboy7e8wACwnsclY6sXwnHt/d3f3JT37y8ccff/PsqeOBJIyzbTW9vqCEuKRPED7cHTnMJpEfhp7vupRha63n+13p2tuMBYgopRhDbY2xkDiOR11rIRdSSg0AQpBQtRoMBlEUSSmrstBaDwaD7733Drmc3uzPbh4+fHh9fTEY9N55eO/Nq2dStFW27veig71dB6P1coENEK3kVjStNhBZaZSSiKurqwVb5XleIEowcepCvD5/WRSlNoC6SVd2xxgRoq2qCkIfQodgHPjxjz/6vfn0uiyrq6upgtLzfAgo55oCFURB1RRewKbTBWVkuZ43vAhjv67brFi7bggs0Qa2fNM0jQK8rppnT548e/YsDpI0Se7cuXVyfAthk5WrRmQAKMpA2RZ5sRAyG0z2mEdfnb2aX3/e1tBzUkqdPC9Lx1FKCcGrqirK2ncBhLhtW8JwK5qyLBFChCDXddu2NlL5aQ9aA42WLS+z7M7R0aOHD0aDweOvvnEY2x1PVtst5w2hqJdEZVM/f/F6bzIeDZK2yeMowMBOxv3f/slHR8sB51wJ7VLH98O2XLhUUtjeOhwt1tus2VikNdaAGt86AMcYEpfRLskYIaSVstZgjJuqyPNSCOFQ0kGpvu+/unh9/ebM8zxMqVKmKnKloQF4OptrrV3Hd5gXBAFCkeJCyNYa2xk63qbHQkgIwRB1iWNaSiGEaFurNVCat1w2bXLQdzBCBgMlrVRAK6iVNMqj5Pjk1Io2W69CBx7u7lCkZZNnMCibsmw4cVzH91abTSXN4fGd86vrb755ut7mZd1IKcMwtG/bmQGEqDN3UkIoYd1acHw4Wa4W63WhRDUY9Oq6NEpbDUaRc3Kyb43+zXpm2poAsJ4vbm6uf/63HyulgiiWGuRFLrUZDgNIHY1IFEUC0my+WhRl0Yrt9XRWtC/eXMyXSyksh2RTN1LoTVGu8m26G283hTEEExdCBGHXiqnjGAvRKi2kBFpDJS0EyALQ8hohDI1VygCjjbIEtsBYrSUlBFnb8koJDiFMe3Gvn7jEJYw2TVPVddM2EEIPQT8MWvG2LKPzSXc7k5Syyde3bt0ihCyX8yp3wsgfhYPBYGCN7lrjr+Y3q7VbVUXTNO+//+6j9x5t63q5zRohhFRK68RjadILPDeJApeyzSZ7+vS5sdzzMSbAo6htJWW+H6V5I7OyKFuZpIOR50VRVNf15eWlljJJYsvYosy+PL+8d/vO0cFhGodlXkgjDbbLbJM3VRBEFuPWGJ85XhTWUuksYzSmFrYAtMbMN9l6u0FQi6Zu27ZslNGkFw2khcAUhqumqoMkXa+yuZj1o+T44CCN0wqTtuaj3T2XYCkEZg6XVhvru4xBRFzH832FbFluSl6FaYQ9Txp7PZtzYQDCZxeXbV3GaV8Z3bZcQ9uVZRCCMMbGuh5jXWoyYwxBBID5NpJPa60VVx0X2ZnQulG7w5CllHVddz0pHcHHOWeMfjclfKsQAtbazkZhrekUJ12ECYRQGsOlCALP853bt29//LO/WS7n42EaxeHp6XHbNMCofJtJLhRXyNrJaEgw5G3z4ME9SnFZ5fP5fDDqa2Ncz0vS9IQ5TdMoqYej0f7+obOYQ0zGOxNCKYRwsVj0lHFd9z/8+V84hK1WK9WKOIhF3QS+f3p4VNZtlW1n08vrNy95sXYd5rouVDUF4p37t2Y3F59+8vGPfvyTg/3JbL4cDod3Hz5wXX+1Wi2Wq22RV1XV2akCz3eZE8fh6ekxsAejYS+KoqsLxvzg17/+tdFgs84c4vz5n/75v/gX/+L3f+f3/+2//Xfr5Wq2WF5c3gitCXXaVmR1prUCGAEAui6Mb02wAFqQRGGx3cxn00E/fXD/9mAw0Lx++fypUmpnPBJKv744N8BGaeL7gQHg7fOlQdPwt17ZONnki84LAyERUkOgrMXAoi4giVInL6okSTwv0Nq6ruv5/mpTeh59+er6+moxmYySJDk+uX3r1p3zs0vquJPRZDje2eQZdb0Ak+n1kjDs+76QsqoqC4EbuACA9XodBH622YyHI631+++/L6X80z/90zdn5wiBOA7rsomC8Phw/9bpQeCR+fxM6QZBrWTDHAIAOL88460iLccEGmu1kYwxQGl3es+rminruhBhog0yCAELLYQUojiOu/Swm9lc8cZ32XS2INba+Xx5dLAXBHcZhae3Dge94NNPfg2RGfRTq+WTb75eLZa7473p5RUjjqLA8VwhRNMogO06u5JSAgRHO5Mw8suGr7eNMsh1fUodx3WresOopRnExBIyJoS4jh9FEaW4yss4Gt6+9SCOfMZIVRVScW6l6zI/8pnH4n4Ux+Gbi1f3H92DAH3xxVfz2dpA5Dqx0ApZgKkTBp7nebxu+/1+5McIdtoWsH8wSpKobCLHR8wB9SbLixVlwPdZXZfLRbVYVaKlLpYAECFVXbcIuwSQTmRkLWi5zPMc+IRzbqwChjRNY5SuqioOwjQOqzxjFMdRmK8WBMGd4QBC2E1FAAKHEIARhuDo9GQwGFxeXfR6yaCXrpbz5Tx/+fL5eDS4d+/e7/z2bymlNssNMHAyHAfEsxaHUfrxLz+BditbCZDADUcO97hWxubr5dv8LUKg6dqcNUaoy2OpizJNYylkXddAA2Tger1O014Qxkqp9Xq93uQQ07ysrIHS012QImMMWdO2bdtU1Gfd23bIKrTAWKO1diiFEFJMfNc73D+oinK73TZNY1pBDLDGKKG0UhhCC4FRgrjU99xNud1slpFHF2uoRLt/tE88VdUNKivKfMTcelMpC/0oPj7xqoZTx82nC4QQJRIjRBCVCloDtTFaSwEhIdwYo3ibbebbzcICFfhkPPLPL26aeuM63Gp3dW13d3ffvX/b853tavnsm8eLxYIQVlZ8On+Tl3UQxu9//4NH3/t+FEXZ8nUL0LoErY1QAKXMl5ttO7uWEljaDyM/DCKjAQTGWrxaNS+v3rhOOBxOsEOBNRBgKXTL6zByEQQYYYKh0YAgZC3CiGqNgLUQIsKwVVZrDQCkhAyShDmEESylbKoCAJuG/iCJjAFBkDZNM51OyypTSrmSQpR4PqMUA6Cl6jScWirJBR8O+1VVdHg1JhAhtN1uf/HLnx/uH9y7d2+7XS/TWGt5dSUdxzFGa0yWWXE2nbZtLbv+BUQocbabDFjU6w2IQ+LEb9qcUAugSke71lpEaC70umxV3hZ1tsmyiZQuYw4lVR6Ilg+iwEYuRcq5sxdF0cHeflVVj9df5nnruq7RQBV5wBUhlAbe3vEJDePPHz/J2tZRbhwEOIoloVfXV5v5Teg6vSSGwIJa+0EQxcFqU6hGQQOhAtZga6EUtizbLCs94iFEwyiBEGFGnSBMh0Mh2uVN3gLo+56hQGAjtMjrshQtclmSxIQyQOjjp8/WmywvCkopJlDUugt/TJJkMBgAYMqyBND04jiKoqqqgLW8FV1R9bdbPKgbbiwEAEhlECLM8ZjjKKV4w6VUXCghOgMzRpg6ro+x6XiH77p/rbVK6U4MRBnu5HV5ngNgwjBcrVZCSERIPw0tAFVVzW4u0zj44fe/NxgMPv3VL0+OjqPQXzf1fHENDKQOQxgrbd5//4Pxzuj1m+dFUXhhhCjbOTiElLU8u5zOj46Odg6OgzgeY1gUldZ6NBq9fPkyy7Ik6U0mkzdv3myW69Vi6VAWukHkB1EQ7ownru/N5zcuQcNedH3+ynXZeDQMGDg8OuRS+g48e/k0ScIo7V1PrzBF08V6sVhqrTnnRVEhhJQyCAGIwHa79h3yxZefpXHc7yVnZ6+vLs6i/k5e1HUlpOC+456/ufjisy9vnZ6OR4PXL1/czBdSm/V6Eac9SPFqPfMoC8OwE1+3bQug6aRUge/6ntOWBcPo9PQUAbNZzD1KQj/YbrcOpZgSLWRZlmGcOI6DKYnCpBMO53kOlSGYMT/AeK2V1dpQCq2FQkhrFQDQaGCMiqJE65XneYSQzWYDALLWwzjyveg//MV/HI+Hg2Fv2Eu32UZIq5TBiKS93nq9Xq1WWuu6bimlAP4nVzmmpGs2933f932CUBAEw35669atp0+fXlxccA4AABrYg4O93/7oxw8f3MVWXV29/OLzyyTy6jw/e/P66uoi8NMsb9tGHvSSLpkKQAMJdrvit7ZpmqZpRZZXFmJrsQWw67vaCQ0idrUtyJBwqeu6BQA8fvKUBH5kjFkul/fv35lOz6bT6bvv3Ps3/9uf3Fxf3bt1p6qKi4sLl7Jekn72yeeMuuOTE4I9iayxREm92myzbOOFQdLfMRYrixBzYyckjgsA8IModKTvedaq7XYthCCYucxL05QQ0lTt8fHJu+++63v0/PzNk6dfF3lzfDoJQ78/TChDuwejNI3/7M/+/X/9W/9YcDVbzJtW7O3uJ8mgrqSSgBCapk7TNNvVhhDSNE1T10K0hGCIZG8YdRAigJbzRuk2SQOpRMvl7u7k/XePl/PmV7/46vr8gjFG6CSGYRQlhBApFYQQIyqlrEzb+ZuV1GWZY0iklFEYWmvruo48dzAYbBZzl7EgCOqqslpeXV1EUTTZ3aOO8/L1m53dye/+7u9y3my368X8Zrkw1sqb6UVT54f7k08/PaPEKbMiYH4/HEZB6jrBYLhzdbDWliG61phKg9erqubTslS+q76LNcQQdWsWxWg+vQEArBZLJSTGeDqdBkHgMGYtaJrGAlTXdVFUSpswTsMwZNT9rjAQY8wYdV3XGtPwGkIIje142W4hs1pvt1tGKGPMc9zA843SxTaTLUeVhRBqpQTnxhiKEMRYSMAYA8C4Lrt9+/bhwc5meVPW5Z07t3//7/8ugPjyevr67PJ6tirbKVdmsVxd3yxni+VkvLvdbuM40VJLI9u2bUXbgXJCCIyQ5zm+42JiAZeTneGdW4eHR7untw6//OJTKfne3s7Ns8Xr168JnPxXf/wPTk5O3rx58+ybJ9vVerXJAEKt0lXdsiAmjpfX7dVskS+Xq802y3M3jKgflm2T1QhTDzCSbxfbvModq4QGAA7SXhqnOwmzFlpDsm1VVxJjDICtK4Exgggw5vq+TwhVEihlgUX7QVgURVU1QkgNrOeyQTqcjEZxHDuUYGiLIl8uZlVVWKPapmBO0N20lOJeL+nKMpQSlGKMYZfSo5SEEBKCAKBpmhZFcXR0NBj2nnz9+Obm+uDgoN/vX1ycPXjwoGmaIPC0Zm3bpmmilHr28uzZq7OLqxvXpQ7FUpnValUWPNvWUtjlcj0cDl3fsQDVTV035aJ+HgRBrzco60Yqw1zXIOT7/ma1ohj10nhvZ2KUYAQHnnt6uO87pIt74qKZ3Vx3BR9cStd1y7qS2vhB4oR+qDX1XYMxCYLKqFVRYEpqIYtWGGMYdV2HGqP7bpgmg822FkJBgClmnEtKPBxg1fCr65u2bMf9XuhHvKmyqkYIJT3TSrPJCwmhlyQCSowsRgh5FEoMFESMur4/GI/A8+dCCOYQBAnQRkvpMkYp9X0/DEMhWmstALYrqoAQCs6buhVCCM4hxAAgrfXbJIb/pDaAQohOgNZll3U50ACALgECQWm/1aB3P1prY1T3JienRz/4wQ/KsvzpT386nV75vu+HgWf9qiiEEIvFgivZ9SAsl8so8K6urh49fGc8GhmpLs8vHMcpiuJ3f/d3lZHX0yul1Psf/GCzWflhsLO7N1utGiHS4WidV1KbvKotwi9fvHj16k1RFAeHh5zz0Wjk+77jOP0khcrqOInDcDweU4QpJsDYxXx6fnYmefvOg3vICght4FEM9cnx/jdPnjx8cNePYmVsVRWct4vlTd2gLjO0bnjNRRAEymilxMH+3rNnTzDGn332WT9NP/rR93/5q59fX1x60U4URRjT8d6EN1XoB7/424//6I/+6KOPPvp3f/pnSsvBYAiwQch4fjBfLnp+Gsex67rQ2G5Q6Ch5hJDibZIkg2Hv1snxYjZv2+b4+Oidd975i7/4i2fPng3H4+FwmPR7zPevb6auF2CMIaGIEsZchBBhDgBAK6uUghA5zLNACl4rpY0BiBBroeM4WmtCWLeHcs61IpQEYTj41S8/++CD91+/PlNKXF2chWGY9vq7u7vD4Xi+XAAEKaJCbJN4vM03TdOgLuw8jlrZ5nnued5isRj2+y9fvjw+/P3NZvPkyZPFYpGm7ocf/uiP/vM/Ws8W6/nip3/5V3m2EM2W8yo93OkPUqWU5IbExPf9OHKI1Z10FmLEGHNdF3AOAAIQS2WEqJRBEBFCnE51waXWlgMADvb3eoOh4zj9JJzfTEmUhAihME45V5T4CHhXF+tH7/xQNmhv/1a2zsOo99Pf/PVgMOwNYoe6FoKyri6urzbb7WBnHPXHAtGqrud5A9xIaFRUEmJ9794JQujly5eby0WXLSpgK1p+dXV1fHqyzmaYMAihNHpVVZPdnYP779F0/PTp00VZQJ9aSDALRsPh2dnrvdMf3mxQljWGDVu4vlxvrOdjF2sq3IDcfnA3z/Ph3s5ms5ndzAVSkle6UTakV69WnNcIm3QbRdEJo+PNZuN6o2EaIEh5a6WUSeS1gwhBhyIHAhZ4Q96CKDS+785mU0IRsxgHgTIGutQarDTwUDBb1UGys3f7nVfPnkSBJf20xKpxwOV8Oc1XDeBQY5TPZ7OZ53mUqaKYNU2DENxsl0LzWrSHp7cGgwH2wmwum6bZbJpGrP/i199c31yf3Dr9J//knzz8gx9e/Nmf7w7culXL5ZoqLYpWu2hZ2C7T8K1Klr611hRcS6k19edFizFGQZoJIcpCtRJj9ejdfRdiRGaS17ObfHfnyPVioxDG1PM8SqDWHEDpR0RtCATQ9V0IIedcAeV6zBLiMua4tK5LKQtlioavSjH3EtJLBkVRAIgxJXVdSykRgBQiAsD15fnOqPfsm696EZpMYmjKJMUORvPF8usvPoeItmWxWa/3Dk4uzy9cLxr2R2VZHR2ddk7dJEwJdmtx1VXyOo5LEKWEIUiRtaFHrGmN1IHrqLq6f/tYypLz9ta9k8VmxnWzczQJU//Zy2dXNxcAkCiKtlmJDB32egcHt630Hn/+erPJXLs8ObmFlKrbtm5EvalCGhLqCm0C5of9OE2ioiiAVnHiYai23DiOI7WoeGWxxczhnHPTeJZVeY0Q6ilCqe3qshhjnFshhFWAUZd6KAiC/d2dvf3d0PO1VMYobQRcI8Zc1/OVhlUjpptrIxWElmtbF4VSilHsUDbsp4HnM5cpgnzX830fIUAwPtodQwhnFxeB6/STI0qpaiWG9PL8uiv7VpI/fPABISTPxM125tHh0X5ACAHASMURQr7v1nxqCd856rsuqfjWWAUA8MOEWoQ4f/7pr/w4OtqJnz6/cZh++fI3SRol/e8Jw5eLhcvoo4cPQz+4ubmpb7ZK6kHPKSuupEHIq2rAWHIzzyyAQgizKObztec5k0FvsVhsy7yp6rasmrwiEFrjCEmkcXRlEaJpukMwKbaV5AooSTEkSmitPeZYz10v5/ly2RTj8aCfRNFwOMQWqFYkXjCM+4ubG0cBHfLB8clysdjmVV1riHG4G29X69ly7noQUWmEgtQx0Kcs5Fx5FpTb6qvVV8Nhv21LjFEUB01bQKQBssooA2zRCAhIrzeQhmNmGiGklBZCofVive4UJF1QLiQEYAysRZQaCDnn1LUIQ4wxRBgQaCGAAFCAmItl1VAGdnYHQgS9vr+YG4ylajW0wHKuaradZZNoXM0zntnFVfHZ1eMvvnx9fHL1w49+1Ns7Tlf5xfVVAA31k3K70gBnVeNF4f7h0W9+86vFYq5EUxUmTdM/+J0fCyHiOH7x4gVvxOuXbxhjTdqO+7sIEduC6Zubq6urIAjee+97WbahPq3rMvR97elqbTd5U2Sr5XYtpYBAxsMgHseVKPaOJuk4GU/2P//ym5evzqxRxTZrVWyt1Zqn8cBzQmMUQkC0Nc/K7z94b7GYPbj76Oj4YDFf1qVtOfR8UmwbhPBiUXQJ7tJ6f/E3n/7xH//x6e1ZXn7e1mBvb4wsqhb50EstskpxDGEn1gtDv3OLIAzGw9HBwUHTVNvt9unzZ1dXV4iQk9unq3wOIXRKp6rbg/39KOohicoqi1gU+r7BSuIWY4yQEarByO+lO0oprYwUSmttLXBd10ColFqtFnt7e3lW3r9///LyEljW8owQstkuwsh99vyJUtIYBa1er5eHh4eDYX8wjF+8fOZ5dr1exSl2fKSymnkEABlF7kc/vAsh/OwzpaV6sViyPjvdS3/3x9//67/+66dPnk9G6b0Hh3/4h3/v2dNnn3/2eDqd1XU92ZtwwKDXr5FTWlYoCANv3XIImOKtwRZjiCnQVjiQYD/ibVO1lZAAIJeyABtjjIJIEawIxX60GwSe0dXzs9cPHt1qdfHJV88kVySOgl6v9+TJY0bw7//e7+zsjl88fTqfzzGGFxcXDmWTyeTk5AQA0DTNcrne9wOhtVFcK17mGUIg8CjGQRQ4bZ3nWUmg2d8Z7e4M16ttVeTXZ28Gg4HWKgzD/mjY66eU0izLAACO50KAG2CVUtv1pmsSioOYQFaUxXad5duiaRrJ1fnrq+n0Sgi5v3MYxYHneVmW1SVHlu7u7kdRIoUaDcdx1FssFsvlfDabu647HKaMkcXyRkruunuMeRAWFgKpVFvXV5eLm+kCWLa7u4uxUxZV3ZTzxc1ms7ZWEwIgsgghoA2AFgGgtNRaaQUAQBBi0XLNqOu6SkloYRT2gIWuExgNKXGNhrxVrhOMRkNj4NNnL9fr9cOHD/cPjhn1Xr16tc3qshLbrM5bpyzzPM8xga2Qdc1Xq/V0Oj1ynTRO6raJIkoguJ7N+72kbUoJvu1eUgphzBizFgohLIJdrjMEQEsttepaUpLAK6scQP3jH394crr3y1/+cj5bd+VyAGBKPEIAoYwyjDAjBFHkdctc95RSTAhmGmul1CDsxXEoRY0xskBjDCnFzAF78TgvisvL66LMMMYEQ6tklmXvPbp7/86txc35wdH+znhweryPMfx//r/+34y5mDrSaEKY5wZFUTHmfvjhj2bL1dNnzzuOVgrtUo8xVtccY4whsQZaoI0WhhgMAbTKAsmFkQJyaYxWRdHmxXazmOdVjRD5xS9/6bnudDqdjEb93pAQVtctQPTlq/NyPbs5e+34gVLm5NHRT37yk//fn/359XSWVa0TRI3ki8XKcb26qpSUTZ2LpqWUYgSNMcZnnUCEENKVaXVQR+dNANpYa7sMSgghQZgxhBDSQlprESJdb9NmvZU+l1xYa9um6YIsu2uutDJSdUfMzp5qjAEARVGUJL04Dq02QBvHpY7jQAssBFEUMcZ6Q9mFBnbv0x1nO6cf57yz7GOMs9bUda2Ncl2300JDaB2X7uxNlFLWGoSBECLPt0VRSCkpRr1eb39/b7Q3thD0++knn316dplTXOyOJ3fu3HrDKLRgf7wjRDufXi+vrqI47W0yLq3j0rrly9W8FWY4mmBMiUuVUgAhoZWyBrsMY9il1dZ1TRG2CEqlFuvN3mQceL4B6OL66s3Feefiqcom9SOErBBCSwUBVqrdbFcUQd91bm5uBkkchyMhWoSA7/ubzebOyfHt0zsMT2c3X7eNFFwuZnNtTSt5nRdAm06brI3VimgFq0pCZLVWhALOG8pQV4d4dXXFudTKdpqDKIw7J5vqosX/TpvAd8YHrXWnJum6Ud6+BmAIELTAWmA0gBBgCC3A89mSYFjk9Yvnr6wxSmgA0GqZUSf1XQdD4HneYNAbJPF8Nr28vFjMZ45DPa9LQ7FJkuzv759fXfb7/fniBiHQdcuXZfn14y+/+OLT9XodRn4URa9fvy7y6v79+03TfPXV17v7px988MGzZ8+Wy2WvNwAApGlfax2GYZJEURQAqBkjk8kxJhBCe3l5fnNzU5dbP3AYha5DBDfz2cpoCDHZ3z+Mwn5Vyc+/eBJFyd7+Ydu2RVEQgkfDYa+XxHFsjNqsFhijsiiSJHn33Xf7/f6LZ8/ruvb9sENfOiqNMdbpPYuioJT+vb/39168eNEFYL948WJ/f19/W/btum4XXaWUMuptORLn/Pz8fDq9qqqqKssoiuI47p6RPM+n02mWl1GUTHZs27Zd2EAH/3SPT1eB1lTi7fdr33pirYWEEGUNoag7s0FkO02i41DL8VtSCsIuDgEhhyDEeWOt3W63jLHVcqMNQARbCBupwrivlJCKG4DCtPe9dx+d3Dr+6U//Ok7juq4fPnzox07cC0/uHFRV894H72GG//X/8ifbTQkhsgZGaej5TBvTXbEOo8IWWGiUUohShKDj0IZLhFBRVMYYhIgxwloNLLAWWgutthZDa21TFRTpwMP9Xkgw3CzmV2dvoEUEE1Bka96UCoE3b14EPo3iwPfZ3v7Oer1kxAGp6Q36NW8tgo5DCZCtqJPIH4/7cS/tj0eI0NV6XbVN07RaN0Hg9WIPQy1FYWTl+RQTK2QtDSUE7e/vxnGcZdvFYhEEoeM4FoK6KAuWua4bOp6ByBhQ1+LmZgHhvPtqX728MEZRSjEiZSG2m7qqKt/3h4P9tpEI0tFodHISHB+XV1dXz549aVsxu5lbqzGGdcVd1+EtwBhIDkpV+b5FkEqtiqocpDvpMC2L2vWosWaTbYoiwwTWnGj79uo5DoOEWoOUhtZCBClChItGaydJkmKzRohZiz7/7Jv5zXVRcUw9CGHdKmMQgO7NbHtzc+O67nhSU0rLxmSlbATEGvBVaTEREmmDkcUOC1zHF628vp4OBgPPYePh4PDk+Pr6+vJP/lfN8WJ6LW2Y9FLGmBaylS2nwlrbtryoSmutNdC8vUtxtzCdrV6Hode2dZwE/cE9bdqL86v5bMOcAADIKHJc4nqEEIgJwRjKllv7dskjhDjM8zwHY5xt15RS1yWrJl+tF01TxEmQprExajyJlK622awsa98PpIIIAKxBEEfMdRzP7Y+GvX5vZzK4vDonzE96vcnO/tdPXxhrtAVNVe8fjA9PTpkfnp1fFfnKcRwMLYQwjmNrsDEQoM5tjI0BSloNAUbQalSWYjbfFBU1mpfldrNdN5WBiDXSPH78LPBc13UfPXww7PcZddumocyNHRaG0WqT3bp1C2O6rfPJZDLo9bJBXtTXDIGSt1a2ElhkFFBAQSOVwAh2MR6VVQRh3/W6LQFCKIUgGBNCPOa8XWikcpnTDRPGKmuhlFqIFiOktW5rfn19Pez126aBEFCEhRAYgQ5HAdSp67JrInYdh/T70BhKyHA4HA8Hg16PEAItoJQiBLTWlqA4jn3f7xbHTlin1FuF3XczZYeNW2vdbd1RaX7gEUKk5NZaQhCAp9baTqlXlsVsNpvNZkWZMQkJIYNeL3DcKI4/+uAHqm0GyfXV1SVWKna8STqQUkKj66Kst3kpBGz5q/OLqmqalkulsjJHkApZKw46fSrB1AIrobYYaGkIJFbroigIwg4mSgq5bf0gcAO/bpur65vNNseMUkoNBJ7nMcakaBGAezvjIst5U/mBJ4RYLeceRRANm6bSWmICssXaygMK0CDuU4A8wgigy+VlEPkEosDzVaSRaavaIIh8zyXYV3SLMSKkU5pSABWEtqqKzWbDueylAyGEUtp13aJo2rZVQnUDwbf+hbeEXScK/q4RoBsjMMYIEGCMNsBAQJCFEAAEIAL9dASAmc0Wf/7v/wNCAEKYxH1t1GqTW+1DAFyXMca0bTsEMQi8ouMjlrOiKPzAPTzaf30+3pkMmqr0A7cq87reJkmEIfjtn/zkf/6f/6ciU6PBUHIxn82iMNyst/ObWZyO7969e3lxlWWZ5wXd56yqihBECBKSKyWEqB88vNO2zYsXLzbZNSLcDzEAsuWcEh8AY6za5hlCZDzZD+N0srfPXE9pS5jDtyUAllLaNd1IySmlxpidnZ2mrvv9PmNu27ar1cbzAqWUtbQ7onQFMd31tNZ+8sknv/Vbv3V6eooQ6mwF3QvWxdoYgwDoWADOGwRgNwp06XBdg0Y3iyul+v3+6enp1dWVEMYCxBjzfX9vb284HHp+2Ai+3W7LsszzfL1F1lrfjYwxRndqMAMhhBABaDDCjgMRQtZqa/U2WwrZMsYgwQh0AhYLIEQEMUIxhsoqKfXNbL7N8uVizrkkmGmthYRxf7jZrDhvZqvNm7OLD3/0g72jg//2n/+32+36f/yf/r+37x0xHz18/85v93706tWr/rAXRH7dVkkvJoRJoQjDEAMhOEKoMzsopTADBGFpldEAAMuo04qGUme9XjPqd2kfxmoIEQAWfNvUBywkwFjZKoS0gDeXZ5dvXm1WN+PBhAAjpJCugzFB69X85joYj8e375xeX1wiYIVQrRSnp8ebTeZ6Xjwev//Bu2/evBFa7e7v7R4ejCe7WVl89bh5/eZ5J+LVErx+83x2c1nXtVT1cDIghBioleZcVAeHO7s7e01TZdu14K3gLSXM8VyMAMGYMaYNqqqKQOIxr65rZAmldMtzhEjdis1mk2WZtZYx4uz52LIn37zUWidJ0uv1EIKdIOvw8Pj165frVUYZieNBmsbQurzlBEe1UVUt4yhIk0EU5Y7nAgjLsnQcD1irtYQYWWCrquoC1wBW2KMMYYMQIZhgxphLCMvXmRQCaJPnOUOQ1+o3v/xkOp1CghHCBDtcccH5elVss/rmZj4YjD797HFZ152op+YQYyhE6/na9UJjoZScMjoYjbVqF7N5eVzEUdDvxxevXzx79kw0ebo/hjujTQ4PJ5Ok3yuKIs9KgCCljjb2zZs30liplJTaQNCJRh3H6fdo2ovC0I+iYDzpQ6QGg96vf/VZXUlrAUTSWMG5blqpNbdAQ+NqZTDG1kKlDIGEENK54LqHDSLbNIU2Ioj8IKSvXr2EiJ9dnM3n58zxrXW0Br7rXd1MX756I5rm4OT08Ogkz5fPXr34za9++eEPfzwYjetWgGdvNLRVzRH1uNIvX7xSxgquEEKhHzSwAQC4rhv5SdelhiCm1IEQWqCBNsz1RAvyUrx6dQ2gAkADaOq68v3QWKBa6VC5u7N/7/ZJLwqhNnVVyloX6/neOL57967vhXt7e03D/x//+n/9N//m3xBKHt6/17btar1lyLz/6N7VdGY1t1Y7xIXWVFVlrR0MBth3uvMi/rblXSmFARRNCyE0ShulMcaO41CMtdaO5wkhtJBSYgiQMaaqqqqqgDZt03TjRSdHxRhLqTAi3dLmuW6/30/T1KWMEOK5Thfk7lCG3p5bgdYa+9T3/U4z/53P6rskzW7B7dRz3aZFnaqTTXXneK2V1rrrboDQdqfhpqnCKE17w7Is6+VKKqG1vrm6UUrtT/Y++uCH+nvqT/7kT8pVni/Xk95Ia5ktt/ObGcWM+ZEBcLXeaq2jKG7bTVMWca/PCFZto4x1qAuI4VzVbVNLrgRQAABrMYC6gwqUUlIstmvqMEjopswhI34Ydvtx0zSUUoRIFPiH+7v5NptNrzzPa+oKQqu0EKIhFHs+cxgmZH+z3D55/HQ03BsPJ5ttYS0s48QPPYtA3VZAm7aWkgBC3TjqeW5SGuM4lBDkeg7wCGU4CsNXr15I0SqhAt+HFoiWKynzbWaMdRyGEEQIdo5iQnA3nH2brQQQggBAjFHXg8YwBQBYawB4i1liCCCEUijPdYwxnDeMEM9jQos8y3u9pMjzqijbuvz0s98wiAeD/j/6B3+/yPKf/eynTVvVdc1Fw0UzHA77gx6hCBNrjGIOllJhDO/dv7u7O/nss0/yPN/f36eUzufLpmkWi0W/399sNkVRhGHYVbN6nmeMapoqTWOt9Xq9rKrCAh3HkbVqtZ45fnV3J42iKN9mN7PrMCSndybD0VgpIxW4np/fsw+//+G7ZVP+7OOfVyKP4khK1g0H6/Wyi1A0Shhj8jyHEP7bf/tvEUKr5WZv9+D169fMxcZqqYSulNKyS4klhHzx5eeTnfF/8V/+8Zdffvni5fP3v/f+zc3NaDwWVnSjcGcI4pwThD3Pq6oKAej7fpqmbduul0utdVmWnPOOj3MchzLXGNN1tZdlWTe8bOqiKLqVB1OktS6LqnvGu9xaY6y12lpLGOoyZgDUGMNOgYdxBADoOsbe4hBaSwgNwJQ4FuKibsqy3mYlgMgPYq21EiyK+3XTIIK22fL1xfl0Po3jgGs12Rv+/n/2u7/7h7/dNhXXZRCwrFo+f8W3m2K9tYzlceSmad8L/CJftbxVRtlOUmZt59pQEkjJgTIARIyxMAyn0xkMWNsKY4B9OyIQTAhhkDKEMXQI9CjCQKu2WkyvtKjTMNgbD4mRfDhIZ7MZhOTh/fcpw9Pr89PT07ZK+/30qy++/uzxZ48evstbaRG4de/ujz76vrJivlxFSTgaDbzIuZxdvTl/fT2/6nYmpbZnl2dAG9/3uxUfY4wxDSO3LDNjZBC6+7sTaO319fX0eiZA6zks8nxCsBK8lbgsakqdvd2DruiIUhqFSVmWGFHPDT03DMOwg2RvpitrZFEU1lqEQCd5dRwqBB/0J4Qi523SGlXSGu0QRBhSi8WizLXWVimU5ZXRuBVSGcuIAwBgnm+VrptGCNU0wveRaHVT5g1vAUBhnKYpZpjs7U+UkKJuBv30ZP/w3p1bnZBwuc0EV0oCTEkYu4S60mgvSABml9N5WTf9ft8L47zkQquiVVm1PDzcBwgXdRMjd9AfSVFzzi/Oz99998G9W7f/ly+/mF1cfO/hw1u3bllrv3x6FXiOqsoq2zRlRahjtNTK7u5MlDatFJxLLhQAABIEkU2i/ssXz548eTzeSX47+jEmVhshZE0os6ZbvCAAVkrJOdda+i6TSiiFtLZCKKusEKpt6yQOMcaOS4PAYw6GGDsOMZZvN1d5Nn/x6rysmt0khshKrqznHx2eCgV+8atP3nl4+/XFFaMg3+ZFIzZF6yfm6cvXjTQIM2WQx5zFfP2z7cdJ3NtsMgi7olWJAec1D8NQCNG2LWPAZQ4mUCmgrDEaAISl0otNXtclIcj3HKWAAUI0LYI2CaMgiKIoYYzWZa611lqevXl17949q3WahNl2NZvNp9Ppxx9//N/9d/9HhxKjeJmt9w+Pfvj+u5Ph4Kuvv1muNoM09IPo/PJ6ud5Ya3dvHSspq7JsMdZaKy6UUgAA0fJuh8AQuq4LreWct217NDyp69pIgzF1GaWUipYjSDCmGCmCaUdGGGCoz3yPAUa6Aoj9/f179+6NRqPO3uK67lshNGEYom4OMMbQkHVHzG6B6MYFakzXJNsZ7TqAtJsVeqn5dktDnamvO/JKKb7TrjLm+n7c748559lsboxxXLpcLpMwGKZDLTSl9KPv/9jznCZvQic0GmyWWbYuGHZ7qeN53qg/2N3dG/YHv/n1r68vzl0CB2mQmgAyBjFZZ9tZti6b2nEcih3OOYSQ+T6ApikrpTRxqLB2WeSt1UXLnSBEzGmLEiGEMWWMYYQGg/7u7q7nuLwpXUru3DpyGTVCQGAwhMtlUVfFcDh0gvD167M4Gh0cHK3WX1dl3XnPwiTq/IptWysFqWMBtNrwfhoSggEwWb4mBKXxeDzov35hy2wLAEEWUISRhcgih1DGHD/yurVICNEVCXbYr1LKWg2A6eCBTqAKIehaVKw10L4dFBACGCIhdZdn5XQGFoigBQThuq4BAEHgYQCbpqG+n6bxYDBoqjJJI4gO9vZ2KKWLxRxCUNd1mxVxeE8aeXywTyhSSlIEf/Hxf2yqgmKYhBGyoNgWyEIlBAZYKvPkm6ee56VpWhRVr9eDELoucxxHG5XnRVUXjkMog0HohKGLoD44CCeT8XbtBLEMgvDWvfGgPymrVlv6t//648nh7qPv/eD3/vPf+eTxp8vl7GRvslqtsqykmBBCHJcaY8q81loDY6uqOjs7Y4wd7h/cvXt/u81Xm2W38XfJ1t8GuNGyzL/88vNHjx7u7k7quhSiXa0W4/Gw1+s1TUMxfssUNFhJ1TVPFkVR1/V4PDTGrNfr7yqODw8Pb25uCHGDMBZCdZ3gGOOWy46u7bA9XrRVVTkk+I5FMsBC2D13SmlAiWOMIoyGYbRYLCAyhMKOKkUQoc74CqG2FhrjOA5hDoJEGmkhZo4XhqExpskFQoAx1uvHrocs0HVdDUdJmqZx6N29d/rXP/2Lbx5/jRD6Z//sn/3v/5t/+tVXz16d/81/83/4wyQe7Yz3OOdnZ2e//PmVNrIbaKy1Xd5DJ+0yCgAAjAFp0u/1BhC+6lqCIXAtQMAihBEhmFJEKcIYAiOCIGHM+g5A0I7HQwrQaNAn2/WN1U0/jXd2x8zBCMAw8oRsCMXD4fDy8tJa3YoGU0oYdX13vl4BgpJBf7K/N9gZ51U5Xc4Xm3WYJhBiA0DHvmCHuVEQRLFsVgijNA1PTw+zrMiL7Xa9FILfu3fHajO/mRVFwYMQGwCVaYtq28g8z5MkSdNUSbdthTAGd4XfhHmO53necDgWQpyfn19fTgEAnHNCECZIKVHkFXOIEO1gMNjd3XNd9+bm5mq9ZIwRzDjna91eX88JYXEcK4nyPF8ucqX0oNeHLtNaR0EIGKhbiSnBCqZhxKWsiu1mXQIEXSd0KQkCb7mYa8GN1ruT4bvvPUAW3MyuFrN5MtzZZNu2rtK03xsMXNdveMscl7ke18axyAnCbFvknFNKmR+ulhuLEaZYaqEBjeIAWKYE10I2ZVMV5eHOgWmU7/u8aD766KP33v/R5fX18+fPOSex33f9QBlTt3y53liIKIbQpZRioaQUqhZtvq2Oj0/6g+ji/OpfPv6XUjWu69Z1xSiwxkBAO9cDsAghAiEUonN5UADwt7XrBGPaCapbni/m10JGrg+DMO3109294Wq1dT08HidhGNaVrBuOiXd5MX/00E9743ceff/45F4cOZ99+ut1Vk8mcJOVT5+9sgBLbZjjeW5YVPV6sRLSCiFcx2vbtmkainBd1xhBo5UUHEGgNAOQSCm1llWrEQaQMGWsMhAjapAjjCVCaW0sAHnZvjm/aKraJVArodoGQng5nR8c37qazQxCVdWsVqsoihCEYeDN5/PNcgG0GPdjClTgYCOafLscjUZ7uxMDQVGVTdNYa9uq7i5aJ0ew+m3PrNXadrI1pblpjTFWm6IoN5tNlReEYgTC7hl2HIdSR2LRJS1yLjEEcRxHURT1k9FwEIbh7dt3Tk5OfN+XUkqhB4MBxpgQ9h3l0e3rzKXf/f4dhPAd9P3W4/otqGCtxextEtx3r+/+7nhBB0UopYi1no/CyCgtBunIcRyM4H6WY4QgtMASCMBv//j3iyKbz+e/OfuybVvYQU8sSEMnjeOTo8OT4+PIDy5evUpcjyGIjQr9IB2OLCZVVfKqBBakaVo3tAKVRbab/gHETVMhhDTGtRK6tFzrMI5Cz3ccJ47jwPUcx6UEua5rDcQYhmFIEdzZGUdhkK9X2WatjeSiUkrESTA5OJ0vNnXTDIa70uh1kVkEltuNQraVbaNabY2GUFtd1XVhqh8+uNOhVuvNvChK16Oe7/b6aZZljPllWbqumyTYcZzRaOy6roG6g3O6bJzurhBCdAfTblD77joDACTnAL6te0AAAAMBBhYC33FaXreSE4QR1NZgiGyShi+vLga9vkODtq5837t16wRD9M03XzuMvPPOO5w34/FosZhfXFxw3gBglsub4+N9yjDCgPO2qoogdPI8L4qibdvpdHp0dNTrrWezeVPzIAgAIMvl0vM8zwvKsu4IrDiO66Zq21obnmXrtBdfXZ25njMcDZrQdzxlbUWoTlLPcdyiKLbbpm4lRM7TZxcQ/22rycNH7+/uHeWV2GxWbVtba7WxommNdT3PS5LE8zyjNIRw0B9qrRlztDaO436nD+iozy69inNOKb25ufn5z38+mUyOj4/Pz8878c1sudBa+6773UPRbYSU4aqqIITHx4cdCWitzbKsKIogCJqmybJlrz+sqoYyTyijtdYGtFIopQjDruu6wPE8L1tX3agNIQSoMxAYIZSUEkKojQ7dYGdnXJa5MYYxogvdMVAQvwX+CMIIIQuRkppLZS2AEFsLpbZaG0aA0Rwjq2QjeMub7PFXX/Ami0IvjFxj1M/+6q/Ksrx797ZLCYTg11/86uz6zf/l//R/66UTCMnnn3w6nV5t8qyf+lKrTirRkY/YCmgBQh0Qi8fjSRQmGFFjAIIEIWoNBhaCtzoZBCy0FiDCAICCS0YwF2q4M/Rdp98fkaLMMAHvvf/OcNB79fJ5kkSnp6dZtsm3+XDY/+GPPjw6OiGEiVZ++cXXq9XKMutH8e6gP5zs1ko9f/3m7OrSCcJkNBJCcC66b9p3PcfxCCHWaTFEu3vjBw/vf/XVV/ObS9k2UmrXeR9jGAehaAS0FlgDLTJKC1kDqDhvyhJJKaRspdSdOgxj3Ta8u+ml1PP5PM9zz42MsY7jBL4HoKYUIwRaQsqy3m5z11FVKdvG8EYIUeZ5ORdN2wrfww4DAHlVsy22GUKo3xu3jdputyqFDmO8NZ4bIAC0JFZaoCiCThB4o8F4d7IXxV4v9pUQvG1/9IPv//jDH3z5+Re8LVtehlZRijm3Na+btjLAtlwqo7ereV5UrVByAWaL+Waz7Q8HBti4FzIH19oCoNq22WxWBEOHktFopJX6qz/7y8DzEcCb+ZZRerhz1GK+Wc8wkBRqgK1WldLaGht4TBqrjaUAGUhdy6TUUsqygE0tz4qr1ebqwx+++3/95/89hPCnf/2zn3/8GyUVsKppRPcUdcgzJebtI4odCHHkxR0mwzkvy8zCZru5ESqLYnc8SRLrjsYDACCwpGlB3RgpLELMaOx5SdNqY/Annz5erTYHeyMpzemt+0k6LIqiboTjBfPF0vFCLpXnh0Jaoy1GhDGmtcEAftupozGGjkMdhyJstOEGasyABcoAqIzU1mBKXM/HhCndtq0A2hgtr69vNqtF6DoIACtFtl2HYXhxcYVc33MDz3vueG5RVDVXR0dH5+fni5upFtwhWLXNF5//5ma22CznvK63m8VmM9ZCYogMArLl3ZrrOA7FREppjHEohRhLpaw2wFjR8i5CJ45jznmWZcU2832fQNJBo47j9Pt9BKCUEgHjuq7nuJPJ7nA43Dvewxj3er2dnd0oirTWShqEUJz2up0GQdKtiR20SfDbHejvbkgQQmssBLCbAkx3hO2YQaC/Ax6+HSCg1tpxnG4R/E5N9vb3EPu+X+R5zwkZJm1dhm5ijEqTYZZlCD7fZlUDTBzFvV4PUdLK0vd8oOzl6zMr5WY2CyhllDoQAiGBEF7sDpI4jsKm5Q4milDtup3AKogCNwo22+12uy15k7AYEGowZI4bxLHrev1+P1+v67rGCCghm6qA1iAICUGXl5eUQAcjjKHVYGdnMuinp6eni1I7QXC9WEhIhLEGgiTp5XUeJqGjXY1A2QpRagmUMUJroJWIoujgYA9B/eLF8+16paUY9nsOo5Rg3rRJnPgOwAg7HrMWKssRBoRgY4nWsEMXtIGEIiwhJpBYBCHs8E4AgLUSAohRx/UAhCACBkJY1RnBME2iIPARQk1V5Hle1/VkMsEQ5dl2Odv00yRJksB3CUSMYt93r6/Luq6ns+vz83Nj1J379zxoMQKB5zV1MZ/PrdVx5I0G/ZOjw80mU5IncTzo9euyHg9HjLHz+arf73MuMcZxHDdN47oMIlCWOecNc3DdlI6LP//is/393TSNMUp5rW7KddvIplaiVU1ZlqXICrFYl3kGPvvsTV570qT3Hvw4L+nZ08eEED/wlZB5nnfIfxzHGGPetHme+75fVdV8Pu/utzD0OyUjpdR13U5kkOd5r5cQQl68eNbvp//wH/7Rl19+uVgsKMVZlmGM0bc3cxchGgRBVRedYTWKIt/3yzzvMoLLsjw+Pt7Z2VmvnwEAoiiizFtvc9/3AcRMyaqq6raq69oADQBwHL9j5Ywx3aDQQQ5cKWuxMcpx6Wg8uLg845wrLTAhECELgP4uPhXBjghomsZWljEGEVXallUruQgTNwmdyO+v1nNkOKPo5vIMmiZJol4abzar8zdnd+/cuX/nflu1P/vZz37xq1+cHJ+2iv/s479ezLPNai2UePTo/vX0vGmqLuidEAIU0EBj5ECoHMcBAPR6vQ6q0RoQQoFF0EJjuqBJbblRykBkpMvW29xYHic7jke8KPaZG0QJcSh9+OBe4LvX19ecN3WNz85eWwsjP+jkLZ7nTKeLMIhHk7ExJu0PD44Ofd/flPmzr1/+7OO/nc5n490dgDAgBGgNIEKEQsosAEJKCozUBiEQhr7W+ubmZr3aQos8x7fKBqEneegySgghCFNCQt9zGdVa87aGEFJCtFJWmyQKHccrYLFYVKu61lprJXzXo8zJ83y5WGcZZIyMxsPBIEmSZDqdTq9njuOlaT+J+1lWTKeztlHaIteJIMR1JbXWSlplYMBca1BZVdPreVOLMIw55zambSsv1luMsTaSEDeO0i4o1PPpoJdk29XF2UrJpmmr84vXs9k157yqC9/3AQjzqry+vsaEaAsAIpttZiCixEEEur43JLTX6y3Wq9DzNRBNW5RtVZTb+UJRaIb9wajXe/+dR1dvLhlzD3Zjz3EVV23Jc7tSonI9miZBw9uqqY3Fged6gVtUTV5WLZcGQIDeYtFhGAtZQ4jyDJydXZy9uRiO+qent968nraNFtxCQBEiEFqMMWWYYKWUIsTFiFmDO1gbApRnpTZtlNAwCigleb49P3+zzRYMWc5b13W1Met1ro0Ng57nhsoBs/naaPmb3/wmiZwodN59dHdvb+dmPru+nlYNZ160yYvReHI9XQwGI4JpB+5hjLVSHZYhhYDYModEKHAcijHmkhMEGWPdEyul4FJCCDFhECGp1HYzJxBJ0QreOgSHgWelKLIWE8C22fU1wF88DuM4z/NBf7Rcr7RU/V7vP/yHP0/CiGCkrFktFxdXl4jQ0aCf9gfY8XnddJVjjud3LnnOOYaIeqTbnhFCdV23bQuMfUslKD0ajdyhEw36VVUZqaIoCoPQGMMoSZLk6PDEZU4XTW+sSuPk9PR0MpmM9kZdf3cQhFrrpml83w+isNvOgUWEEISwgcB2PYRGg7cDRIcxGqO1tba7RBC9VSd0HxJj3KgWo66KhnSURLdAK9WJhRmAuFvXtNYAcgUQdjyl8yiIAse1yrqhI3nDiBOF6WA4uvfoUV4UrWyttRaCptkEgedRtphdFdsscv07J8eU0nTQX642bV0FUTjqD3ZH44vpNN9sDekhALW1UkrC3F4SWgDW281bpZvn1lUhlBRStm1Di6LL1cEI1IQCI5IoHAwHke8wgrLtKo5j36PTy0vKcJIkQrRZ3hBGr66vs5K3nFuEWeAFSTw52BOKG4JWmzyvCqs1ogYR8uLFs5OTkw9/+D3Xo01TPn/+XCkVRdFwOISAQAjDMFQMFkVjtCzL2onwd/xOt0Z3BER3iu2Ko/4u5BM4DkKw87MgBDCEABgAzGiYGqOl5HVdWi07GRbG3vVmDYxllB4d7d6+fbvX60nRGqPaVm42q6+//urBgwfb7fbm5iZJkt/7z35vtxdXVZWmKUQ2igKlxPPnz1erled5Ozt7nPPXr1+fnZ2naQ9Y9PXXX+dl884775yfX3Z0/tXVFSHIWE0IgdCNkyDLVxjDq6sr33f393fLrdM2sqlF2yjBYRRSL06TiGX58uLsIvTHi3Xx5JvpZHL1D//4v3wSTMPwvAMIEYC+7wVBEATBW6EigOv12lorhIpD6jDv8KB3OX2rZHybn81Yt99ba33f75JA/8E/+AeMsT/90z9dr9f7+/sYYwxhRyRRipMk6ff70xvZ7c1CiDRNJ5MJ5/zm5ub58+cnx6eO41RVFcW81wsJdYQQWZa5XoAZ/Q6Q88PI9/3NsuqEk1rLblDoDE3fcn8aIeh5LgBWSqG1Yk4AADDadozkt/pH2I/6QihlrYcpQqQLZuZSjKg/GfXTNL64oINeGIUuZTCOo0E/vnXr5PHjx8f7h7dPbt06PmWYvnn5+v479//P//y/R4j9zd/87ZuXF6Ef5VVZl1utVNM0HeSJMbYGvc0DLZqO7XIcJ8u2TdMiyCCkECKocZfBC4DRFkCkrTWrpqYEpL3g4Oi4l/ppHLRVra0l9753LHGdtWvkQC/2qra+fjWDBp+cnK63zauXb7KsODo6GfVBFMWj0ah/ZzeKojfnF0+eviiL2k8GQ+gOBzuz2aytbdsaiqEXOFgxwdu6rvsp2N8/JMT95vGLqlBxsrNcbAihv/7k6cN7Dw+PH5X1N16cbCue57kxRoq2OxUpaYSSXckhACDf5MZkXElrLYKEuW6a9BljbgjS9M7V1cV2uxFCcLkGyN1kGWG6LGoLZMuJ1g7CJkl9qWpVGYyhtVpUjeAqZA6LIOdCtLytm64oljrMD/11vl6tVi5kCAFtJIbAdd0651DBiEU74z7PijRMlvP5J7/+NcLg7oNby/kCJGFeVdRlDvKWyzVjbq/XE1wRjOq6gcQgP9jt9Yui4nnRd32pWbZUjjM0crPZrNI0rSSHFS5B+GS6Tk7vtlpyCC+yDaFo+fnHAiz3dvbf+Z0PF/PV118/EdpS7DQS8FoYASInwbpmhEIIN5tNEgR5u1V1YSH44fcezlfL//u//B/feff91WY9m806iI8gYUxXiEcYoUHIugkaIgORaTivagAASHqxsT5C0HUTBC1m43JLZEtd5uR5WdV1UdV5iVwnoF7QKun7ociUx1h/sCN5S0n/iy+nf/WXXyXDvU5pvNpKjAMInDTuaanapiaEOIw2ba21Bghsq4IxxusCIkgdYoA1WmH8VszPWwkAQZA4xFFKbdeF4zj9dFS1XAHQ2qbkGGootA9x0PotgtZxnOEDmim52AoIfcFhODlZXM+sZJb0LPTSOM3z/PxiG4Y7lNKWOnfv3vU87+XLl1qJnVFvu91CFMRhLJiw1mptCMQWISmU5DIJk7dWBUI45wghxpzT8R6ouB6Mer1e27Z5vh0MBkdHBwcHB+NxIoRgjHqe5zhO11UxGI2/890hjMI4BABoIxBCmAAILQC6AwQIhgAAi2inM5BavhUtAmOsgcAAC2AnwMYQQmggNMAQwroJsrPtdUteB5gDAAA0CANjFRcKY+x6TCtgBO/3E2ttrTgIaKmEpoYFoRYtUmg4jAbWcs47A5gUR0VRcM77O/e9qJ5Nr3o2VKL1k5QJaqGlTuBjHnjMZ0gbTqlTNQ3B2GWMWeth2oviJIyMVLwRFBCHBYKbLKt816lb0QqqoR96tJJNENP94+EgdQdpSJGNQg6tbupmna/bRiLgRZH60y8/L4qqbfT8q2cIOkeHJ7S1ByePMLGxj68ulkZBXvJeHKlGuC6hMBENePnkxf0HJ7/90XsXb75wYBnHwU4/xShEyFccbPK84TIMQxJgbIxuWwuhR4jFVAghmkY0TVOUknNskOsE3ezYXW0HIMllUzYIQUopIZ0mVcKunKmuw9D3/bgoCgghc0gyGA+Hw/39fUpxU5fXiwsEbLZd37p1ApSBROXVKon90TDtJ8Hjzz599+4DpIESijGGkcuN+d/+zb9fZ9t79+8/ePSO6zvXs2tnEJJeYEDj7ITHzl5p22Q/EbLZiDnpy0rPLNAEgSgKGlDunIYOxdZaGqido3RT7z2/+ooQVmod9vvWDUqkt/m6QUJTgzCJQfDg/v3F7Pzlk68maXSBQNNUaZpabbKs+PDD70spy6zcbjcIQoShVGI46h3u77/77rvz+byVO12U8nQ6Lcu23/ebRhZF4zjBzc2SMfab33xeVXxnZ2ex2OR5fnx4jCnqDQdSy/V6PRqPx+PxarW6uJgDiJgXXk9XT5+9mYzGhJCDwzvV8nIUhbcPjr5yvzyaHJaV7PcnvXd2Z6t1WVdu4M4W1xAT32dc8cgJj45vvw2HtoA6DiFEGqvrRiqT9IJmI7wg5FLFvbSVwgIg2oox5lAGCDHGQggZcRzmtK3orKGcc60VJtBiiR2zyK4PzGRbcgUax7WOiybjIcYwiiKhZG8wfP/DHzp+cLnaDI5P7v3gR+vs5vWrV3fv3v2dn3zwv/snfywa8fOf//LZk6eKyzhOV8uyc/k6rgONrZp1SCmEUEo5GIweP/kGuS6XlhGvqITRJvCDuq4B0J7DqroAQGe4CVkwuXV7Xm2wj69fzQkwn37xKRmkkyzLsk2RhIlD3EYKZInvebxp87xMomBnPGaMMYr7vdj3GK9qqI1qOLLWaI0sQADUVRGFIcGYUIQBdF2XuRRjaBEc9+I0SKwBxTbTUgZu0Pq8LhvVyun15d7upJeGNzdXUoqOOiLU9f1QKVWUy6IoMKYIISWN1roRXEqFEPLcoJs0q6oqmrqu6+Vy6fte5xfvJJBFUXQzqTFmu91KqSHEcRxr1HYUjtYaIgkt7A7NCKEgCHzf97zA9b1OvRIEXlO1FBADQCva1Xq92m6EsV4YCQO5hqttMV+sfO9aa1lVNTdkfn5pre33+74fjkaYEOYwz5r6O7aYMRIEASGkg/saYbXWjJHRqBcnbq/XE4I7Dj0/P6+bcrtdU0r3D/YopXXdnp/d1Gp2dbFOkl6+LS4vryCgvR4jiGrFCXNcyqy1nue4jmOBRgj6yO/1etoaiNBgMFBGbzYrpXQURd3noYRhjKGxHaduLUSIAYCkUMYAApG1UCnFOU/imDHCeWuBCcPY910IYds0q9WqrBpCGIGormupDcGsqVediBphpa2p2qatKy6FUioMw47rVUp1m0qnhf5uAO9qLL47RnSxjEopABBCBiEE7FsVntZvj8vdl4gR2dnZ0Vp3+tYORe9C+Mu67ISxGCJMkOd5QRCEYWgGilIqpewOfwgh13WDIBgOh12z9u7uLoRwuVx29QoXy21H8DuEQmMBBh0NGXi+EGK72QwGg+Pj416vxzDxPG9nMmYOXSwW1trBoLe3t5MkyWDQG41GQehprV3XdV2nu1fDMGTUgd/a8eG3P2/hxG9rhIx+WzfQyey/ox6+pSSs/bZt6O9qFLoT7Xcu/+9Yc/ttDMB3/+z+77ciR/XdJ0EYUEQRQtZi13UpxcYYQgiAtsMkAACCa9/3pRAUYcFb3/cHg4GWAkBjCSrrWki9zYu6bq2FCLK6WBtrXN9jyFrNeZXXVSGaGlmjeCswooRAAKRSdQsIIb3RTllkZZNXxer/z9h/9tqWZdmB2JzLbX/8tc/HMxEvbPpkZmWX6aYaajaJLlJsQdAfoAT9GkHQPxCg/iB0gwSKFOiKzC5WZaUL755/15vjt11eH9a5N14WKUEbgcCLE+eds/c6e68555hjjlH0ot5oNJr0E0HOTg9Xq7Iqm/l0cX4+zbPBRVk9Pz779NMvlDTgxWy2FDTJs36v10vTPC+itq2dgzhOJ5NJlvbPz+blbLp/+9He/u7TF08fv3d3Z2drPB6v68qXLefcWOvBhvW11pZ11XXdejq/e/fu3t6tPM9Xq1VZnq9WpZTSOUCkiFYppaQJIjFxHC9WqzRNkzyVUi7WCwAYDHqD0VApmfd7O/s7SHyaprcjjohZlt2+8/D169cnJydnZyer5XwymUyGg3K95o+Swe64WlWMMc7Y7dtvjYaT9aqqW50VQ2vw8Py41+9HUZznOVC4nJ6mh3FvUBijgOrlvJWqHU9y7b3WWqkuoi5OImdwvdZlVROMnHOEIicsiRMhRJH3vSNp0kuSfLFYWIucMM6i6XReriUSHidR26jH7z64sb/9m9/87suvfn/z5m3GWFEUq9Uq4iKKIin17u7uwctfV1U1Ho2SJAn9BSm7w8ODy8vLNM9Go1G/31dKnZycnJ+fN01TFEUURavVKtBrnj179qMf/ejevXtffPHF4eHBW2+9BQTQ4Wg0ioSoq4pRGglBKQ1bwdHRUdu2+7t7o9Gov7/ftu3bb78dx//u5OQkinPnTBTn1mnvbegJSiMvLy+lkXmRKTuPomg46gOA9T5sREFOLWw729vbhOLDhw+Xy2VZVZRu9rHQ8ONcBPuPoii891K2Unahxgh8IPDm2bNnd27fjDjvj4aDfl7V62GvP7+cPvv2yZfffG2Mi5OkPxzVq+rJt09en736V//y3wohdra2Hz9+b9jrDwbDn//8v5qeT2WnAYgxjhIRRwk6j0Bsp7TWW9vD8IwPBoNOOQJJJ53DDfci7KWEEM5ZLOjO1hZnbD5dMAdtve6q8vjwgHWVlaVdNPVSNFEUOeeoo6rVla/A2HtvvbW1tVWXVZ7n4/G4LMuuLKn1uYiHSb6cLYiHIkkpZWXTBsqlB5C6U7pTSkopVZKRoecsUp3xUnemXs9npyfnhLBytfjFz3/81t0bhwdPLi8v2d7uYj4b7dxP8gwRl+tKmRVaBQBN01VV1e/3R6MhImmaZl02QogkSapWV1Xlve/1etbatqvruh6NRsaYSHDnXNM0stPWesZEcLC8btCGWYkoSiil2ijwmCRJUWQeITR7er2esga801JrZ6UzpZTT5SqdzcG6dasNxuu6O5tNtVRtW3ddB9xTypUyaQpxnPrNc6g2rDGwAEAphplDa/Vg3AsSfiy6QQgJT0jAA6I4IbTmIk6Sfhj4MWZdLmE5u/D+MsxzT0Y9AonWlqAQPGYRg6aSqrNOVc0a0Vdt0+/3kZKqqtM8FzSaz+dFfxAc6pz1YZYvtM873dZVR4qIUug6jYhRWoT6uC617LS3TmvtvCWAHIiImO7kcjavWzne2uacV01rjEtSwjnPsqzX66lWgPecM5+kkYhv3Lq1v7//9OnT0Dnrui6EJcbYteD5NfnIWtvrF3Vdd50yxlEaIh714AHA2o20ESWMEuasl1JKq51z1njwYQIbEQljrN8fcs6jiAduUUgNw/YdQmNgrXddRwgJ/dG6rler1WQyCRColBIAgoET51xw5pyrq8YYwwh99OiRlNIqvb+39/jRo5s3bwrOOefBELyq1ovFYm9v586dO3Es4jjO85wxFuZ0gkzKJo/ZXOQG27wO8CGKBy2EcMLX8f4/P/wbzp/Xn/DmbX9NZXgzUYArYuObh7XWeROI3AGfoBQBAqmFUrrJ6hjlzjkk4FMSx7EzljEGzgz6w7ZrwFmt9XZ5c7lczpezzrh+b2QsGGMYdt4TIYCi9bI2BKg1mWBOoZWdJMDzHjJqOmOtjQDO59O2rpzpjPMkSkmUL6ruxfxydn6ipXIOag0iH/a29zqpv3x5cH6xQkeSmDlLtce2M22rOmWgceVqOV8uqqZUSguuvbdCsLatj04Ok5h+/eQb9G2vV3z99Vf/+M//959+/E3dSCQUkYRhua6u6q61yl5czgk9GI1GlFIkDAkLSIkHAmgpAOM+dknI9nhGm6ZZzFaIPk5ixknZNZcvp+PxOFLdzs1d7/329mR3d3c6nUZRtF7V337z7OXL51VVtU1DUAzyESXJwevzfi+3hkwm48loDOBHg8F0Op3PytFwJ06jy9l8d//GcjXN+z0L2hPTtAskXdZLKIWyrjrZxHHcdrUF44mJOM+yDDEjpOu6tdKdt6n3pJGd99jLRL2Wr54fzeaLNOqTQdRJxXkyv5y/fnEQZ/lyUWWJUKq7sTfe3enfubsrYj9bHB0eHo/HY2s9CuqcOT4+/uijj0LUTJIk2DiNRqNeXmAYF0JcLBZh+iDwW8Nqh0KiqiqlVFVVi8Xi5z//+bNnz5RSUqvj42Pn3GQyCQw2QshsNrtSYUIAsl5XiBdSG1WeS+3+5M/+fjHo//Z3n4k4f3l4jIQVRR8oyfKEM5YlqZYFtqg7Va1neZ6HB1k7HfZzxkmcpFLKy+nF02dPmqb56U9/yjkveplqDQA6Z5wDBAoAgfsVpq5C6cI4YYwE3lV/ONKdHA6HFH0aR4h4dnia3IsHRX6weL2+WD148GA02drfv/nw3qO90W5Ff/6Xf/mXv//9J1ZdXpz/NQXkPCpX64dvPcyyvFcMx6PtqmyrskPvr5tft27dUkrVdR1FkTJSdYoxZgE3pYXz1xVakaW74x2v3PnJmWsVI66raqstm180WTao7PTsdJ4n6f7+bn/Sn16et3U3GPTQw2o5N8bkRSrbar2cKUqJ8yJKOEFVt6ZteZpSRgVBZFRQQghBAK0VehCMm1arWia91HmPjrRNg9YXaVYUBSCORwNKSb+XHR2+KqsFoOm00c6nadIfDqqmrqrGGOMRwmWEMlQp2Va1FYJ4Dwha636/773v9/vGqrDtxnFMCeu6DoD0+33GRFnWZVl2xjHGjNFh9gYpF4JFUVTWjXVWaVnXpFNyvV5zzofDoQZT17UzSmRxPuh7Sg5PT8+m04uLi15exLHQwDzLoqgX97eMMYy11njKuFRhwscbY7SxcZpY7whSB9Z6p61qulpKqYkCQCaZc44hq+q1McY7pJQSwnq9QZ4VStrlYgoABKMoHs1m86ZpOOej/tZ4vMs5Xy3mSVpwQeKId0ksVSO7VkRkb29HGntxcUE8DaiXiKHrujgNMzwUiANPvENPvLNgraeMEcoJMqM7AMSUUMIomL2dXaU69DYRkbNKt03pdFEUqtMUWRpRwSKlDSccKCOEOQfG+raRTVtrbSmlQDDLitAvfPHiRQiKXddtbHnfqGtDth7y3BChEZESLrgI+4UxTmtrrdUqMCo8pdQ5sNZWsqaUbARwgHqPhCDn0Xg4yvIkyzKKpOuauq5lJ9umAxuQCUsI6aQMvvIeYLFcLlcr6xwgeu/Lsmzbtu06wSgiplEUpq26pkZrKCU7W+NhfxDUym/t3wigUZqmpxfncRwjesbI7u72zZv71wYB4RmmhBNk4RyMsVyg94B4Hb83QV0p82bIR6TXje/rtODNdXsTRXizRx5a6X/nxf9irhDoC0g8Inq/UZt5881h0ZwLtkbOe0s8ESLyHr33nHMCPkoyYwwS75zbc7ZpmsVisX/74q37b89ms7IsTw+ercp127Z1W8tSybj2QFDLndFwua6sMl4pS2hZrb33eZ6rznddE0c0imNj8fnrw/nF2fnZUV2uwPk0zSnhlAve2aZRs6r1ThjruaOAsfV0PlvVdXt2cXr71i4SI4303jayJIzFmZhMtj/83juUuPGkV1XTo+NXsqul7qqmjFLRzStESgh1QChnVipGRW/Qs8aen11eDREAIhEiapqmaRqlZBzHRVEwxrquq+uackUoifNYSrlu1oyxXq833hmPRqOtrcn3f/TjUG+cT2e/+c1v4zi+tX37xYsXy2VZFIW3pKl1VUpG6auXx01dDorigw8+un3rxmx2yZm4cePG2dG8afVwsjXZ2s7z/OzypCiSsnJxJgS3QFrOiYigkZXWSyTMQxXFIvLEew3oCIkZ95yjN4wx5h3U62o5q8q8bWozvVyvyyXnvN8byqbb3eo/e/L69evjNE17gxHjJInZ43fuj7e2J1vDqqp+//vfewdt0wkeUcqiCMNzlOd5kAR1znAe3759czIar9drRL9cr4Jur1JKCEFpCLqm18u7bliW5Xq93Nvb+4//8S//2T/7Zzdu7HmE8/NzY8ztm7fG46Fsu9ViPpsuQNtaV+ghjpLJeMs4WzftdPYsoTrJjiil/91//w8PDk9XZdO0pTa+kw2PIuMk4YwxOhqNeMnLZcN4aq02RnVd5xDiWBRFFiVCyk4I8e7w3V4vb9taqa5pqvlyMR5sOee8B+8doSQ8yNbq9Vpaa43RQdMWkSF6RFxM50kSe49CRASIN3YyHN/Y3mvbFi1BCxzFarpGczY9ny/mK5hkVakEz1fLOkngrTt3Hz589Jf/7t9/8/XTnZ2du7fugycI1FrrjXUWGFpr7dbW1nw+X61WWX/gnOs6TUik7Gac8hqJdM5lccqpWC1my9kqonzcL7a3d999/Jg5T9Os37ZaKxfFcZwWu7v7QsTGqF6RKWO0Nf1evlqtLi4usizrugbR07Zt2ooSzJLIIenqRjAGzhtjwFrvHANf9Io0TW+lwhiznK+M85xSR9nu7v7t26LoDQBclhbSdFvbu+LZs7Jux+Phsmnh/LLX61kPRETO14zzwWBAd3erqjo6OtJaMkK4oNrI6axO+4n3Ps8nl5cXOzs7URQFGq1zTnCKaAihg8GA86hpOqWU1Npa5pxzzkDQPKFACBkMegEwNFZ3stVGxUmUJEmqEu8dIUgpFXHUyq696ELB5AhxhGmnpLYpj/KiEEJwJsPcPwAQROMVUJLGiXPOgkdEKjgyJJwQTsBA1ZSUcpDAOW9lJ+UshDdjjPdIkCLh0/ni9PQ0y4ooilZlqw0kcZGmuYhT65FY7wBX5RrBFr3MgSMERMR2dm/89Kc/GY8n//Jf/su6beI0uZzPbOsIIev1GpEKHnEeeeu0tg5cSIHjOI6iJJSbWhulFAE0WhGSxIIZbZ2RBH3CmZbd+XqZpP3t8bZHaqwr28Z7ZCSMXDLvQWrlLERpEkexUl2UZhcXF0mSXFxcDAYDzjfDmaF6ppSGZnkAGDeFNTKCjBIOzFJKEWmQfGjq9koi1xEC1npjZNd1KAiljHPBGDfGYFC2wc28AKMCwXmPV/Q+4sAYa5jgwQiYEmSRcAjz1dIhaGcv57PAYFLWKGsSLrTW6D2nlCUJA4zjeDweP7hz786dOzs7O3mWhesCgCSOh6OCUDeZTLa2tt56660Ay4URgxDUA68tsAsZja9l6f5OFA8fGB7pN7sS3puQGYSfzF8JK4WEAN+Ybrj+xusXr9OCvwMwhE+7+rowSEmvPyH8WNenHXj+3nvnCCL1DhGoJ+CAACFEMCEcUqK1jilLin4xHN+4c88515TVcrmcXrw4Ozu7vLxczFfrukJE53G9Xh8cn/hGOme9EIyLiFHO+aDf0xDPTcsRqfeXFxfr5bRczBG91aaqqkhIHifW+LKxaZIPxrvD0q9WpfOUEE4JV9bNzs4up8dn56/fun9jd3dr8Pi+0a4oBl7jaDT52c9/nCT84Ojp7s67Un7/8PDln/ziT05PpkmeWnuOKIw3QGic5so4HgnX+jzioehs23ZdLkMby6FHRsCCNJJ0JIq5I44ISLIkLHKURpyPR6PB7u7uaDTK83yxWDx98fyTTz45ODgIWeY/+Af/4MZw99WrV94fpmnqLazX1eHhSRrHjLGz04v0rTjLCs6itm3rcp0kyWA0vpheDrdHk53tumsvpxdpFrMILqcn+ze2sjSnREndWLuirGWM55Ht9SLwdLmo66ohjqvOoneCJ4IKZHxh6/WidJpwkrS1ZEx0tWQo59P1L/7o/tnp/OJssV6vOWHT84vBYHT//n1jnJZqvVycn548fvze8+fPOeda2V4/995//vnn1Wp9eXk5Hg2Gw+HOztbe3k4kxMnp0Xxx4YHkeRrG2o+ODrz3WZbduXP7zp1blOLdu7et1VmWvH798osvPnvvvccns7Mnz5/s7e399Kc/jqNoOVsKQtuyGQ4Gy+WSUKaVopQbQBFTad18OqXs5N/+u7/82R/9V1Tw+XKWpjkXsbVGNbpt2/5wIARvlXTOMcKDrZdSnXOO8DDlZLUNUqdm98b+2dmZUuro6GA47Kd5Updd6OKG0TEuKCHonA3K0NZp55yUxjnjwQFAL+W727veWBDeWq+aVvD49PQ8SZI4KbZ3blAWr5drQptutj49PVUXlHO+t7v/1ZdfIhJKxWSyrZSZjEZa28PDw4ODA6XUcNi3WodyKEqTJEleH74yxoT7ymrVdtYYo1FfP8teeWNMrxhIZafzlXHYdcrkMNne/9M//gWru/ZiNp3NZ1mWeYpHZ+dZkRPGKCXAoiRJt7e3Y8F+/etfv3z58t69e+PdIRBU1ogounf/bqfs9HJ+0U0FZdYrI5XVxoNN42SnN9y/sbs6OTg+Pq6bzhiX5sVqVTqP3uPNW7ezXnE2XSRZunvjbn/r5dnZSatwuS6X6zKezyPG27ZV1hRpMRqPEVFraYxqmibiPE3jNI0BICnSoHcbishQ552fn0sp9/duhsy06zqtbWgsOW+tA2uCzowLLV3nTZbkgfRxPciLiJ1sGWF5msci0lp3TdvWTehh7+3thSGfFXGdact2LW1HKZ30M9gg3si5CGLkWZbNZrPQB3LeOm8pI3ESIYFOK0qplCqKqLWubYLEKXDOV6syKI61bVvXNaVUaxka5wHnXK/X5+dNnAjvPXprrdZWMYZZ3o8TNpmMd/a2CVBrzWq1tM6XZck4j5O0rBqlVCRiIYRR1hgHSCglnPtONqxmhBDnDaB1XgOyKKblajYZj3pZ0bWQxLzXy5fL+asXlxAPBr0+ofz04mK1WnkgKWGyNXGabKqrOC6KnFGUS+0Jdm13fn4eoMUgIBiy2gAVvBm63qiJCSL13mltpdRt03Vdp7VGpNaGrMIDkBDsvAPvMVBbgvkCIYQywnlkjFuv11abruuMMZQh59wY0FoHfmW4ASilSqm2bYuiiOP4ugXgvaeURjGtjHZGU4QkjntpMplM7t6+896772xvb+dpRikNBBpnjOCMTsaEkHtv3YnjeHtncs29uFJJQgAS5NbDlQbn0v+8TRA6CVcNBXtd1l+TEq4TiOuWAbzhPnCdKxhj8I3j7/ASrhuW1z+E8xqBELLRb6GUIpI38g8alCGcgwDhdFYGEXHrfGCJAyXoAJkwiN6AAy5EEgmRpYMiH91/cHc2mzVV3XWdajsA8M5VVfWf/vqvZ7NZ1TaE0bprp7OZMto1LVADshEkihiHtmtKZaWcTCbGGNPZOMrSJC2rplrX4GkURVlWNLWyFjwQypmIotTFXLi9m7vvvf/o1u29W7duEkTOxOvXx3XV/uo3f71YTr3ttnd6WcZ7RYoMRRIxxjolEZl1HpDzCLU1mcgBwVrbdHWCCWFIOVWmU8YAcXFGqYjqulpVTWKjfr9f9PtOt8aYKI5u3Ljx4MGDnZ0dRGzb9l/8i39xeXkZENOYR4QQ3anDVweRQ0K991qpriiKJE7BOa3NfD4PHKCmaRbLGaWEAGmaNe2l59PzyXzw4UfvrddzIFbEJIpY261FtNUfxB5U3VRaVeiNYEwzHScInuJcGdVSzBljEY+aSjMqE8EIcGcJBS5oDACqMQDESlgu6unFHCybjHeKfHR0dLS1vfvBBx80pby8vBxvbXGWrldtlqquU0mSybapq9aD/fLLr95//Hi1XgghtrcneZ6v1gtwfja7bJqK0CgoN4d6IM/zGzduvP/++1EUzefzd999d71eHx4ejkajf/Nv/s2f//mfK2aSLM7z1Dj96tVpW9UJi4s0oZTncRbladOpVsvluorTBAgjXGjnf//JZ3/yZ//Nh9/7aL5Ycs7jNKvKRiuJSONIOA/lcr23t3fnzp15tT4+Pl4sFs45ZCSk4FEigmXrT3/64+Vy2e/3P/vsM2vtdD4TLArjDgSBMRIczBEwFKWMMO+tMRYQGYuiiP/ow/du37zVtq2zxhu9XlWz+eXJ4RFjrJUagNTKURHd3b+5tbN7b73+3Ve/Q8Qoisbjcdu2zrm2bXd2doiHOI6zPLFON22V5QlB8N7GcTyZTOI4Xi6Xob1LKWeMwdWYNGOMUhd2Cc551h+U63XbqTjNOu3m63JZVZ4yNl3PpdfT9ZwIggbLssyKHACWy/n21tbdu3dXdfv7z57/1f/6q9PT0yfPD372Zz98++23e71eVGSURsvFerFYJEkkKCcuihDROm9sxNkoy3aLYf+uL3o9aezlxQwJl8pp46u2++Vf/21W9IeT/Xv339IQi7hPWLmubFm33vt1VaMH723E+IYKVzchRoqIcULjOA46fZ7TMEXW6/WMMePx+OnTp6H1Za1VSjdN1xRNFCUBKTVyo8fZydY7YJEgBBB9cD5USmllPUIcC0So6zLP+kopb6xxYLV2zoFggpGzk4Oq7vV6OWNsOCyct0F+QIV6wnvw3jrXSam1zotCaW2spQDKSKoIAFhvPLqm09Z2y+VyOBwi0lVVR1EURnq0003bnU8vpJSEk7yftW073OohIo8peo/EGqutRUTY3t3yRouIOme2d0aT8VBrdXh4eH521nb1xcWFMtpYRxkLIflacifEEkoYAHgL6zp4YjFrjPcoVc2oTzgnDAWD/b3x9tbDIouUkudnUczd4VHnPVrrZKdlIx2hXBjnYTFfEc6kDo4ZDgCWizkhpJ/2yrIMPH+lVOBehHAV1NTDMpIrJUGzKZidMcYaZzcWRyZEQO8xABDuSmm/bhsbhI8IcQ4oYhRFcRwLzpRSZdkYJf1mggiNdtqZVnaUswTBhwl3glIr653UinRtSlAIYbQCgoPR0NXKW5um6a0bN/pFz3s/HA5v7t+4c+t2URQEwFobC0EpNUoTQjw1aZrs7e0ytpmJ8mC7zhhj2laGkEwpDzmTtR6IfzOQXydPbwIAbxIUrv/8dxKFa+zhmpBBrqwfNlDNmzbi3l+rPl9/8ianAfMdJrF5jSBSRPQeAwhkjTfaAQBBD0AQCRAEgp4gEOIJAkAURQDEE0tAW+ekcuCQswQ5yXusPyAMiZIdA88pM6r7wQffW61W89ViPp++ePXyy6+/en30uuvaNKM+5kUSUUIIAcJZB5gA7Yym2iGzoL033mnVYd11XSel9Y4ygUYZo4wlUcLG4+zHP/no7cd3CcjL2eF6vU7i7PjkfL0si2xyfHz41v0brw9eXV4cPX730dfffPvf/W//B8qYsRaJ00YbZ7mHdVXGSZoJsVqvlFKDQa/f72dZ5HwnpWyaOkxLIbF5IYbDwWDQF0L86MP/KuBDy+Xy6bdP/tf/+Mvlctl13fb2NkUyn86EEOPxOHS1D18fnL5+0e/3b9+5sVpWRVbEcd7VrerkxcU5IsZx1DSVs42ISJIy7Ewjy0auv/jqs+EkY9wnKY8Ttn9ju+kuBSdKtoz7OGJKkaZRXesIU1q2RndN03gPSRJryUpvm6rSHeoc0aGggrOIEq6Umk4Xo9FouVwvp+v/8G9/CZStlussy+Ko+OM/+q8fPnz788++ssa//+6PmrW5sXP32yfPQ+C31gcKs/f+vffe6w+KcrV2zs0X0+Vq3ssz7y2ltFNtJ5u2q6Vqh6P+/v5+f1A0bXV2ftK0FaBL0shYxTmv6+rps2/f+8n3Hj1+VK3Wv/nNb85PTntJsbu164xt604b2+MjqS1qWreNRaKMdtav1tXF5ezJs6ePHz/+5JNP5ouFb6HtmjTJpdFt21LCxsPRw/uPdnd38fLk/OLMOkMphaDTkMU7Ozt//ud/3h8UezduHB0dTbZGv//4t3EixuOhYEEvTjrrQ2/FWvTeWGcQkVKklHNOCcXBoDcajXppT3WSeZyvgnqsXcxXynmlbGdM1utN66bPY835cP9GNtm6uT6v61prfe/evadPn+Z5+vr1iyyPj14fPHjwYG9vuz/IV+u50i1Dok0X5YPxeGytnc/nAUuOksIYG4YnQ4HtXee9j6JoOOxbJHUnLZJ8MFwtprP1+tuXL3/5q79hHqE/HCijPSWAmGQZjcTFxUW5LnuDofHw+Rdf/cVf/KvT09PBYHBwcs5+/7vtvd3xzg5oK6WezmeLxYIhWy2X3jqOhANRRsuuXl/OpjyqzBIRgdCTs9PlqrqcrcbjbSSslXaxPjs8OYvSXt2269pEad8YI9UqiL3IrkPvWMG6Vp5351bLpqq9M8PhcH9nN45FuV7PZjPHwiC43Nvb7bpuPB5/8803QR48iqKmaZVSgXRalrW1Vi7nzrng47BpElPKGFuVlda6bVujXZTEeZ6HrDZLci2Xsu2stYJxAEAPdV0SCk21Qm/SNOaCUkbjOEqSxCsMtb6UnTEm1L6BrkjZZusP4Ed4nZJIq65rjS0wTlhgtIV9hBDCOOlko5S6cePG/Qf3yrJsVLtYLOaLTggRx3ww6FNKu7au6xLRd9JJVfcHGaH07Pj8k09+vzWZ3Lp16+LycjpfFEUBuMH2A3/CWu8cgEdKqbPgvY4iATQ4mnilOi0b1bYyEmkSnZweDIfJvbc+6mXZl1980nbl7vbo5aujtm4AN7QABwgAQggPgIwCgNa6lYqAd95Hcdy2bXDrCKdRFEXgD3rvlVKBtRQ4OIHbGP5trTXaXnffA1bGGFMqjBtZazylBDxhVBDcECGNMQ5QCEcIUVLXTd00DXpIkujaVtH4oA8trmI2ZVfui0EaISTagViws7Nz9vx1LPju9tY7jx5OJhOtVBzH29vbRZb0i8x733UdAe+tAXCIaK1mjIQstmlqShnnXMlWa911QUaNOQdaa+/AGkcYuW4x4H/GLXiDZbhBArRR1xjANXXAOZfneegLhLUNPZ1rJCDwD95MFLTWb+YZ4Ru99x50yEM2GMYV8pEkyUYp1pMrBIgQQoWICWNIiUfwSD0jeDVnwSgDTkQM4AG0AUQgFAhQnhNKwXlwS7SGRhEVaeRk3hvcuHMHjHz74f2HD946Pj4s6+rs/Hy1XCqlFrM5aJMkCRrmtRoV/aaqu06Vcqm07g2GRVG0lC87YExkWdE2XdM0nNE4SXf3Jx9/8tv56mg2Pzk7Oeo69f3vfX9nZ/fOvX1G+ju7Q6XrmzfGX31tVqvV8dHpz3+2yPIkiiJCY1TolPPeB9Ph5XJOCIzHw+GwD+jqZaOURHS9fqq19J70+1v33rpz8+ZNQqCqqlcvXx4cHLx8+TJIBPZ6PUZ5EsUHr16Px2OXpEqps5Nz59z+/n4sEs66P/6TXzgLf/M3vymXNYDz3nnv9vb26moZrMbrpqxqncYciY/yFCj85rd/o211+/Y+oVYI/ujRg+X6tKrWy9X5ZKu3t79NmQ9MndEwd861req6Dl2CQJUyq8Xae9p1KuaGIIuimHOBSLQ2+3s3e73e0dFRHKeLxer+/Qfz2aqu5a39u+PxTrlufvOrj3f3946PLp49fZ1nA8TDyWQSppCKojeZjIIthfPmy9l0sVgYq5IkGg36jNNu1QKyQK8OzHpjzPHx8fHxcaCoHx8fl2WZpunR0dFHH33029/+9k//0f/m7cdvf/XpF0evXjvniqJAcNaYiAul6rZtm6Zz4CnhIo6cgoQNV6tVVTWfffbZH//xnyZpCosFIaRt693d3cV8dXR01O8N3n33/f39/fl8eXD4+vT0tG3byWRCOV8sZ2HS9dPPPs7zHD/9VKr24cOHz58/F0I47995dLssS4B1KPdDb9R/J9CJzjnnLSMiyGY7bWYXl+PxeHY5DeYXZxfnk53dnd3d2WKRFL3y4LBx9nQ2G83nyujVeqm1Lsty2B8oJXu94ttvvwVv41hQitpIKVvO6XDY90Yvl9PQhV+tVvP5PM2TrpNRUlBKCXEhUei6TsnOOSci0e/3W6lqqYDxtMgXq7ns5NHJ8d/8xjFvyXy6unv7Pjr/6sVLpdQFuSSEIvBV3f3l3/ztl99+c7Sa0UE2j1BqN/3mWbH/ZTK+sb292zRLwvPx9i3TSiW9rFupJVDCYwKeNq66XJ4iK4bj0eHx6bevTs/OZ7fuvcVH261UJkre+/B7r89nNDn75qsvBOM3dveePvvWY2w9LasV8bi/dwMR54s5ARdHPIpTD/bxu+/fvLG3XM57vR6hdKXqJIoQ8dmTJ5Px+NnXT8f5EKTb3d0ryzoTPR8L8NFqJYXotaoEQvv9AaW0aRrnXJwmhNKma5XWy+WSc+6od+iMN1bZXq9HkXDK0jQnBLz3SksgGMdR27a8SEWcAWEsikPHXSnlhGpUS1NWFH2lVOoya610and/Z7Va1XUtg7y4MaF6bqVtuy7JsqppOqXy/kgptSxnvUG/btssKxarJVD2zctXo739H//4x+jMN998c3JykmVZL8uDyAZjrKpqIUTMRcQHB69mL56dg/OIyeWa3B3t3br/w0p9HUVRYBqP+ruMMcEiTpl22jgjjSOEsIT0mh54IEi0103dlqtVLdhw2GdEpGlP2+jZy5kzp69fz7qODId9TE6iNG5bWasGGWZRHDEOQESUUMpFHMXjrRByBr2Bt3h6eaGU4pRGbdPv9613gRxgjBGMM0KVUm3dMI+MEOvAWIeEAhJtTX80PD4+jaJIW6us5YRgzBMx9N4bgM5o6TWiVloKFFmW5WlkjTeAdScBwBGWpAVjIqjiOKcBKbN478a2EFwrlbEiTWNvrLcaFBRxlqUp9URPl9S5rEj9at1omabpvUcP9u/cmkwmw+Ew1OKYRmvdOec8egAHAMgQwLaaGMfC5uIcc8YZY4z2VVUzxhgTrXFgHCISSjoHwgSkx8IfcgwDynIVy0FrExJNZa9aFTYwFRwhhBLS1g2lVFBGCHEUPKGcc0Jpa22AAa7xg2s8xlyhN9dQEyKiloGLYJwPxBEeJUJQb4FQirDJP1gUbRoTSXbVNyJICCACIASTOg9hFMWBB04BwglYYJEFAApkOAEAEzKR6xlmxNHg7o8f/exDKaWU88uDuq6b9apcrZfT6Xw2s7LjlExnF4Slx8eHs8VcGTm97GbTE4+wXhNr7Wp6rpR0zuzsbv3o+z/98U9+cHR08Mmnv12vYdS/m+0n1douZq/29tth3+7sbP3ub3998+be7bsf/Opv/2pVm1cnp7cf3MW//lXXLZSieTpaLtpBOgBJJv3k7OysVKuIKi7Ezrg/uH/HE0zTmEdCCCGNXiwWv/v0s+l0Wte1MJHTjtGiV6TeY1MCIZYiCpLpxoFB4bGfZ5SALufncvXBR+/1s+SH3//B8vLsP/yHXw56fSXXzpjbN7edzS4vD1489R9++D6luJhPKaWf/fbTTz75uG4r7pPVZZkPop29gRAxJXyyt/X5F5+O9jOWYZ4lfZ9PRP/2Xtt1ZcVsSvhqZaeXry/O1LqEqoT33v1gNl8V/Z6gguV57V1pjde0W7TSiZ2bb7149erF8QWNs7IsH2yNs37+69/+JuqJSq6/ffnlx1/+pm3bPEs++fjj/f1tQshyNUWvd3a2hRAfvv/B86dP2q4mgBTJcrk8Oz0fj8dGbmxKRsVwPB5//PHH77333unFuZaGIjs9PusaabW7sX9rejnnLPryl7/+8MMPp9++nBovZdsuL248eHBi1jROPXRK19Ypqdzu9m6ntAWM8572ohhsHRyeEcr+x3/6T//5//I/zxfTdx7dWy+WwzwhYMb97Mb2aHZ2UlfN+YtDvW5Ruaaq4yJJipwwXLbr0sjy9WtEmorsz/70Hzn772Peb6vm9HyNiN7H3nvvkFJOgBprtDGcEOcQCVKKEeG2s4uL+cVqKYSYjHZu3nrr66+/XqzKrLdTN/6rr1522gyGdmu0V9ft+cEpSBdF0f1b7yyXC9seN7UdDnafvDgwwMAjTYqDk7NHjx6LKJkMJ7LqOKGpSPb3hs43hMCgP0ESEUp2JrfOzqY7O+PPPvusyHNtuizLtOzyrN/v7V6evGLGbvWHKUud9MRRwfOudqzX6w0Gg9FoRDyUZVmWpUdouvZiNl22datl1dTOOXQu7CNtA7/61d9WVf39D7+/Nd6WWimlrDVd11ln205iHO3f2C+KvKnquikBlGglUjqZbBtPhIi11sFhr20luC7Yl1EkgV5Qaw8AWRzv7e31inw+nzvnCIHFcrmzvUWBv3r16sWLF3W1LopCSjlbl9uT0f7+/v7+LYrEWouUIpLz83MRJcPhgDBmrFyXgbHs8jzf2dkpimK5XJ6fn9d1vdmR0TNORMS09aGXIaV0zpnYa62BABVMGaOkBe+opcWo1+v38zxHhlEUsUggQXTEtt8VvkKIwWAQQP5gCXhNJQuHc87b75hohBDBGCLGcdzWjUdwzvXywiNkWSalPDw8/G//mz8bDocvX748OjqaLuZOGyEEF2LAedifGW7CyabiJBCucWtrK0yyXXPQwpxx2IuFEAFfSRPRNI1ROo7jra2tPM+91ZzTote7OD87Pj785JNPopiDdXmeEkLqujbGtEorLUPjRltHCcuywoEH5b23nqAxKpT+VVU55zSluF6HK7Xeuc04PhIgAQ0inFHAKIqEEIHZdL2kAT+nlFNKCWPe/YFygOAizCARAM45Y94Z770viqKrm7Ztg5UAgU0zXlDinFXKG629t0oR3cm6rpMo5pwzzr1zjewCrcE7LPL+9vb2vbv393ZvhP4FZ1Ge52EUiiBx3l21A640XGEzFxB6XmFSIyA63uvQ9USk1loA69kmSF9H6/AHreSbxIJroKVtJCGE0Kt8AkkQd3YOEJ2lngB4hxYd2I2rjXMu3JPXsMQ1tPAHKULIE5x3sJFVsNYi5SzgDUwE9AUZC+/cJB9vAiEbEMIDXClHAniE0LLdKDy9MfDy5qMRoFF3RToFAM55kiRbWwOtO6u0laqpyma9csZySry35xenL168eHnw8vzyfLFeVVXVqbY3SUNzqutabeRg0IsEr8uq38t3trbzNCXURRHXRs1ms/l08YP3f3L37u3zk4PT06O2mvbSNIv4IE8ZGqtKq7xgqZFL1ZZxVEScVnXdH2TGGCHY2++8s7W1BUCo4J9/9uVitby8nK5WK6nt1XISLnzocCH4jSU8QQ9IOONCIAVrnEMw1lhnwONnn31y69aNwWAgtaKUOo9xkkzLGdLoxt62MUpqfXI26/Xyda1Xq+nHX3zcqMoRdXx+FBdk/+5jEfMXr18Vw+LdDx6wyGnTKGPuvXVrsj2ez2fOz7uuaTvlISqKFD3Kbi27mhIarJUYIYZSAOeMb+tGlud5njfN2lrNCCjdaWXTNL19+65S5vJydnkxF0IsFyV4jmAZM1HE4zjdaDMzsl6vv/zyyw/ef5dz7n1qrBoOx3fu3IqiqGmaRER1XXdd571njL37wfv//T/8h1LKX/7yl6vVqm3bwWAghAgstDRNf/e73733zuN33nnn6y+/2tvbM8Ycn56OJturdYmUMMYAVNt1Fqhx3gN0XccYr6tVmsbPnj197913xpPhqxdPOMHZ7HJ3d//dt98ejLbatj0+OQcAj8g5D+l9VzeeAWHMI9i6cwhNWY3vjqp6vb09qcumaptYJd6j1YYQwrjgnGullO4QkRPkIhKCUYoEfds2Xdfack0pvZieM8aiJOkDuJWv20Y7C+CCa30YmXn58qXWVtB3rbVF0ZNGFUXhveOcU+R1XTdNc3l5uV6vvTYRZ2mv1+/3hRCTycQ5NxqNAEWrXIgOZ2enxujQjpdSemuFEMvlXBBPKdFd1bV80M8SETVtbY1kGwkwIYwxZ5cXy+VSalVVVSulIyC1rlUHBI13qLX1rtePlqvy408+08p//8PvD3p9493lbIqU9IphkiRxJLb3bsVxvFw/my4q5GZdd53SIknz3sB5sMpyFu1u7wWGoHMuUHgQfd7rdU1XlmUc9weDgVTdfD5v2zaKeVnW49EIOb+4nK3Xa/DWOvDen50u1qt20N+6sX+vaarzs5P9ne3JeDuM8aR5z4Wpe7RRGk3i/rLi1trVahWoQJSzYPtrvfPep2nqr3Y9KXmWp1JqGtMkSXgcSSk9c957kcTDrUlRZHGabNq9FDV4jZ4gI+ilapu6Q0RGqZSybds0ipUyIR8AT8CTIJsTrM4DVd0QtNag90gAnEeA2WwaupWj0Wg5n1ut/d//swePHu7t7X3++edPnjzpuo4hCdCW3Uh4OE4oIzQEwul0Np/Pw0leg/mwmbhTQScnwNobXi7woG2QJAmPRGA1MkaapvEISZb2ej0RsbIsZ8vFslyzmIdnaTAYOA9to8MW3ykZ+PDGO8ZIGA2SUmZZ9p3NPGKYZUXEMDLjwXPBOWXee4tABRdC9Hq9uq4D/n+d3Gwsp71334kPeQDglDpj4LpJQah2Wmut2qbrmrqujNIhSQrAYCqEC6NghBLCnHPaaedc07U8EqHH4bwXUdzr9SZb4/1bd4uiuHnz5mAwCFRTQkgcx2/C+FeQo3HOacqMMVLpsNpamYAuMCaMCZRjwTwQsumVxIK/GaqvqZRBiuP6W66pCQ48gL8ylEdPACgFSo3WgIw4IMQjgHNovSMOgDj/h3MNIXGEPxyOuL4Wj9QBggfn0SMlhBHKCeXIGFKKjCHZeE8QSq8gBICrsU7wfhMNPQlZA3hARH+dEvg/yA/e/DO80WohV4f3lrGUc4TUZcMJWAvOAwUw6sb9R+//8KdKqUY2VVWtyrKTTVt26/X69cGrZ8+ezefTLEv6eQHWHR0evX7xcrGcx7G4ffvmeDLUneq6VnCapfE7jx4521T1Ko7o+fn5xfnRgwcPABoAXxTJerVQepEXPkmFdfbBg7cYY5Tw0Sidzs6fPn1mNDSdWq2q1bIEoGnaF0Jobdu2Xas1ABIfukuUEAIEw7C9Z4Qgs0AtAes9MsrTWNaLT7/4/OzyosgH+zdufPvtUwBS9AevXh9Zh4i4XCwOT6Z5njdNtVqtVs06SRJKhQHbaq2cn63WX3/zrO2Wg/GoqszLV6/4a1rVhgucTi+Iq5SyRhOKQAmREpFEUQx5nrdtm+X9QP/inNd1zQjxzmrVpklU9HPnzOXlzCE8vvuYc/7lV18/e/oSkWprD48vF6smpEFREqd5NppM8jSRUh4cvvr888+fP38+X0wHvSIMt4+3tvO8N59PHz/+4Je//OW3336rtV7XFaX05OTk66+/Dkyj1Wq1Wq0QcTgc1nVdVdXyYnF8cjSZTB4+fGiMKYpiejnbu7F/G2nTmShOD0/P3cGRMt4ZiUhbabI8Mc5qrb/84os//sXfe3T/rd//+m+auszS+ObN/ffff59H6cvXB03TcBFLKYEAY8w66xxSoBTRaysE7/UGJ63enmxJKe/fv//86Yu23YxigcM4jnu9PmdRXVVBGueaNkQp8c50Xad0J5yuqsY/A0KIMlYI4cAyxigllDNrLYAbj8dlVR0cHC0WCwYWEdMsdgBRFCmtKUOrVdgY1+sVXA15CcGiiGdZMplMTk/PjXHGdk1nlPGr1UIqI4QwRgG4rmvyLJ5MBtPptE+dBddWNekXD27fGg6HB4evqtWScSaQ0HVZVVW1rKpayk7JWikDvq4bZQwTPIkiZY2xlhIGgHHKOqW/+uZbSvkH776nrZmvljf294fjbbCOAEpHjl8ffvXt8/V6HedFwNqt8dahMyaKoJf3er0BAFhrsziZTCZNUyVZyiNhZyunDaW0qqrZ9LJpmjTPkihu6k4q44EAY73hkBBiAKq6jOLi/GJ6crrYmuy1jZrP1vfu3n94/57W8nx6XpalUl2cJsUg6w36hBB1NA13mDEmFKzOWefsuiqTJB6NhixiQfMnyPuvW0kICe/U1uRt6r2ngqVZyuMIONHGGCWJIQCgne0xAYB13axWa621YNwYU9c1HY+99yQ4iTnvPTjnrbUEHAFnvPHOO2lkx533umsBgEVCyZYQ0rbtYNibL6az+eW///f//s6dO3meS62QEgBQ1mitOdkok/grynooAT0SKXUcx0LE3nshCAC0bVtVVSgBhQjEfgx8BeU05SyNIsG4Up0ymnGSpmnbtlGaCEoc+Kqum6YJ86XOEu8NAIkSkcpUypX1BihYqT3x4IECZSLKoiyzmVKq1+uFcfMr+fc0JDTee2W0vaLUhZwgpjEA9Pt9xljoULgrZCu8x1qj1Ya6GCKK0Q48IYjg0BmLFMA6sG61XlptwDvKiGCMMaa11EpbA8pbjJxIEgCntbTW8igKCmO7u7tFloVJgclktL29/dajx6vVikexto6LKM1zY4zU5ppDsNFV0TokBO0VweKab+EceI+Bp+m9d06BJ4RsugmbtsUVD/GarKCMvg6cb4ZVFieISCBwTfNjCAABAABJREFUZ51z4LQlYeQBgASCpkfvLQCgQ+O+wwwopUgZYewagsI3hiYCNEKZCN9L0HtCuBA8jlmUAFJA6pARJNYDAliH6AFho/eAQBwCIoaUwV3LQhD0mzZEeNt/OVEI7nzXd3LAXQghWgfsDSll4LwD9OjBO62RsSge9mLwvatPs8ZQ04H3ZVmenZ8ul3OpwghRtVwsdrb2KbKyWpWrpsh6Mc84if/Tf/jliyff9nrF7mSnSsTFhXZKp4JPBv2379/jPHr44J2Tk7MXz1/neS9JMhLhZJwIETdN9/r465Pji7ZRRTHkxqBXWtZae+I9xyIinMWpsh14vAob3w2dKmWsdwDehwUjGCfxYDh89OP3IpF8/PGnb7314O7D+y9eHa1W5Xgrn6+qo7PLTfZfNvr0UmsJAAaiZS0Zo3Eszqb1r3/3ldLN4dHLNOP9r0+tY02btYtyvXxKKAB42XZpAllWEPRKrmULztIozimJm3pttSnX1YO37htjpheXaZx4C03TvP324zt37x4eHsVxnKa9d95596uvvn7y9PnZxfz+/QfgyWJZ1o3inCNgkfeTOIuiiHIBWoEn89XKE9zZ3tu/sTudXqxX1eHhYZZlPIqPjo+fPX9+cXm5s7Ozs7v74P79H/zgB1mW/fKXv1RtF0DBYLluua6s++ijj2azmbX2/Q8/+P3vPmY81gAnZ9PRZKvVxjOnjdPaSq2aqrHe8SQ11sZp7sEeHx9W1fqdtx+8/fCtp0+f3n/rwa2b+51sLufz84vpcr2Ser6u1ojIOImiNMljEXNrbStlkcS9PCvz1Bo9m13GiciKtJUpoRimHuI4Hgz6URRTAkp13ntvjdbSOSUiRgCcN845FIQ7ZrySbagtYmk0F5FwEQA29UZ4irPo4mLKOT86OmnbNoo4Mvruu++EJ2W1WBRFEcSpil6mmwa8B3CIhDFOCHny5FlZdcaC8VQkyjnHOEnSwigJ6DyY4bC/u7c9X1yqpgQAMHJ71H/8zsM4jsvllHrLKBdCxABgPIgocUhsSzkhTiunlfeeRjFjTHWt944QKrVOkoQCqZvm1eHBcDS+dePmux990DWy7lotDSFkuixfPH12cHBMKc2Btm0LDiMRA2XoPQLlhGptrLWMbSR3vPedVowRKWWwK6ybqlNyNBoNh0Ot9dAarQ1aH0UJ51zpbjqdtlU9Gt9pG9l0crZYr5frsqq7Tl3OZwDu8nK2rlbrqmRxBMRTSpFRIWIAEjgKTdOoSgWRUQc2y7LhqMBg7sIJEss4bPXHV6oUSIHzZMN/NNZqqzrVNrKTUgKQUH1GdFNQdl3XdZ2LYs55CIfXzDL7xgFX/QIPnoToACA4L6uqKIqdyRaLRNM0o8FwPp+vy/Wvf/ObL778UghhrZVd55zjnAu20TwmANa5ULw65whi1htGIg5PLCGYZZn3XnaKIBU8iuM4WJ4g4kZqNIquz6ppKiDonXHgHXjvvTRaOyso6fX7Ae5udOucZ5SKmDmP2hpnSZxmSDtKGWEUCVJKojQKOW8UJUmWhc3RWusRjDFS6zRNmZRaa2U0WuOtC+cTAI8gzLyuq2ssOuQTWmspdViETfVvNaM0NFm8sd4DIjJKjAbOKacE0HHOoygCK5KIg7fh9w2/mvc+jL9+/4c/6PV64/E4juOwmL3eYDQa7e7vhzPQ1hLGPKK21khJNiYxLtBUlVKBeFFrc/160CYCAERqPdow9EjQeEc8GO+N9yGSI4bi3BOyaV44j1dJwneERED0CB68A7DOO+Occ0h8kFG34KkH67xF571HDwBeG3vdeEIPxjrnzSZpeANLsJsGmsMgHY10s+yUI4uAUCDUIgMgzoHzAOARDSJ6C1dYCCWEIA0jkmj9FfXBodskBB4AGFyvyR9gCddJ0t+BFgRjgOA9OAveow+9HotRXljrm9Ah8j5QUBnhVb2M4zgrBneS9A7cI4Q0bb1czm/eutu0VV1X33zz1fHxMaAjtBNCOFnXlU5iVxR9KSVnab83GfS3er3hRx/9II7j99794PDweHt7N45jzsTZ8mi1uryzde/OW/c+++SL2dQWeYEIzrp+nyMUXacRnHet9Z33aJQGfz2V+h11NMsS9BvLwc1NK3iaZw8ePbp37/7nX3395NmzB48e33/n0ddffztdLkSWaOsIInDqCVBGMKZaa2Yz03WeEkv4bC1n6wtnVVn5JBu1XX7z1h4X2/PFdLVadK2klFqYWUwN5OBRGaOcocip4M26Iox6cFHE93f3Tk7Ophez0Wik2q5put3d3Z2tHQB67+5Dj+TsYvbr3/yubSWltO2U936YZePJdtu26DEax3GWauu79dooxSOBiGVZCSFaqY1xJydH55cXeZ7HcXz//v1WdjwSq3L9V3/1V0+ePLm4uJjP55PJ5OL0rCiKO7duc87X63XExa3d/STB5Xw2nU4fPnh078HD5y9eNdJ6333z/LfK2OFg3HWqVTKKIhs7rbUGrNtuPOipdmVk+9UXn/70h9/7+c9+nCb83t37QPhsOV/Xqq7rddPOpguPHWMsyrLReDAc9hFxVa5U11mp2nKN3r589tQax0QE6IQQSEFr7bwLekrO2SuqTShsNGUeiWAUnXPeWw2mNy52tveklHXdtp0ybSNolCRR16mqbYx34/GSEk6RRDwyHhizSmkjldbW+oAQE8owioRzJo6F6WpnLBIf6PNtK2ezGSAHZFGciijiLCrrinPaNpJzkmXJ1vaIUtCm1eUqjuPxaPDO2w8fPHiwXiysUkYqdnp66r1HRpu2Nei1d+uuqZoaCFHOamdRK2V04HAKIeI4bTsVwt5sufziqy+jJP7h93/w7Mnzw4Pjum7TONPKLKqOJQWlvLOG8JhznkYpRWa1o4SF/gIiFr18uVxShlXbeOLjNKnqstfrEYpSylhE/X7fOTedThljxvmIMmWs1MYY00nL4my1WqVFbq09OzvzYOI009Z89fW3dV1W9TpJU8IEpVFdlqVoA01GSimlJoQkSeI7r7V23hAKSLzfhA0LKIxRSvkootZopZRHQq8Oa51WSnmvtA10QgAQSRxF0drYwEptmq5tW9XpJEkYYzFHwhhBorTS2hpjnQPnALylBDjjlFLG+aDfo2EmmkCWxh5j55xgeZZEbDKKIs4isapKdD7P8zTLnHPeOaREG8OBEcYIMuY2wSbkmMEk2jmXpqm1NmgYBEbC9ZBkYBpnWcYprZfLYKMcbAy7rrEuWCFQxsloMNzd3c6ybDa/PD4+RtdRwrgQjAkE6gC9xyhK2i4O+V+rZAD/AcBardSmilLWOOe88kor4yyllHJOtA6te0AAgtqapmnW6/Xt27fn8/m6rsiVW2PoHYQKOERNxhgQhM5FjAsevKOAIgHiHTJGkAvmnLNKgjWMCB4JEVEtVVXVHmySRr1eL47j0Xhrd3f38ePHVPBIJEFsW2vNBHeErteVtb4oUkKI1qYs6yAcqbW+rsUDtBOQD3VV/wcAyW+GHy1jAikjxAeqpt8MDVCDgcXiN65Ob4wzAICH6+FJ8AQQwbtNiHVAHHUbkQNC7KZq98ya62iEiP5q0a5/+nCwK6rBm90HAFCbd7kAO2lP0DjrTUQ4Ae/B+SufUkRABG/NplymngIlAS25co4IHQdEDJQFfIOgcP2feCXr9HeQhoCQMcuQIBIABEKDhBmlQC2AJQCEIUUw1mjbNZUxhnq3qlZ1Xdd16b3nglGKzrmst8WTYrRFst7We10DAGdnZ21X18upEIIQQIadWrFolPf80en67n2S9/bPz8+//Ob16emptX68vbU12blcLaYXx/u7ZHfnzsVueXG2ns8XL18cjgYTRJqlseBcNVKpxjkHQEZ5L6zCdesBKSGE1HVJCAVnkVAkYJz11nnrnr98ZR05PjnR2s3m86opl+XSORAiIoxFJPLe85Tt7O6KiF1eXholOGcAsF4tVqsmSSJOo1bJVjmP8c7OHe+5EIOtLa1UV9d11LuhtbZaW2ujzPPIW6WNUtrISCT9frE13knjuCkrVcsKK6UMY4xjfHxwqq2/d//2weHhv/7X/3o+n/cGIyD06OiAiWi8uz0a9s5OVUTTPM+jKFJStq201jIRgzNam7Kqy7LMiv5k2wRqeWg3RGmSFjnxsFwuF4vFx598UpVlIiJKKadstVpRQESMGPfOyVaPtyaL1frbZ0///B//j09fHNSNnGzv1K+Pu04x0QauQ5Ik6HwHXoLvtKac6dJQCr/77a8/fHz/g/cfF0nUSv31N8+VI4PB9qpWlHMUzFviiY+iaDzo94pe3VReG+K807qrag7keHoRZ3kUJ73+kEfMOh38xI1VVbW21s+ns8ViEUURAUcIMNwI0RqjlFLIDRE94F51urOyka00qhcxSiJpLKJvmu7k5CSO0lBaK6mzNNdGWbDWWqTIGNvb22McsjjO85wS6KqSMFoUOaU0SNFHUURZ7DyhEdNaG+2UUpSiMTpJk+Gg6PXyy+lp01QFY0Wvd+vWrVu3b3POGymrtp3N5sxoRwORJracEksoVCVy0SnlKSNIDKA3ziMNoofGOWWcdVpQVjer1WoFlABA16nj8zNKI6SiLFsaZeNiUte1giZN0pjHBJEhSyLqje86JaWOIs4YW65WIhHGGGW0cFEsoiSKu7Yry7I36FPO1ut1WdWBxUYZL8vSGJNlmYhixtjx4cmwP1isVZbFW5NBW6+MtW3bWuuNRUKEEDFlsfG17BygL8tyOp02TVMUWZZlCSYebChbEb0xSlmDCIxRD8Z5XMxn1xuWAaAsOEgxY7T33hvvrHZhzzWWCgiNhqZpwubrrOu6jnOexUkoW4OQiL+KGKGSE0IE1t5oNGKcM8Y4oUCJ9361WsVpYowJikAszwMBfjAY9LLcGFOWZaDaWWtBSRIqTkYJIUgI2o2aYRRFg8FIqY3+SYhkXacQMfDey7JerUpv1Wq1Mkr3+/3+oNiQ7SlPej0tZdNWi9UyjjeGK8aYoHVtvCNaGQ2EIiBBCmkaiyT23kMFWmvjNDqwznpjlA4WoEgpVeAseM55p2QgH8krtxVEVEotl8uDg4OHDx8Gc9HwWwQ6BVzZ0cLVUIBHIJQiUucceghe1c5bLbs0i/M8F5QoLQEgT5OQNVariiFJ8uztt9++det2kqZFUUwm24Qza4IeEsZpxr0L4a3tpLHOAxrrmrZrOxmU+JqmuQbJ7VU97r03V72D0HG4+t1pSM5ClEBEAEIpIYQpv+EibEpL2MT4cL3XnQhCCG7mORkCIIJH8J5AEHAiaB14sM57g+i9vb7fGDLvvHP2KrpvDuO+E7nCN2SanNsQ8QJXE5xX2hDrDGzy5rAtfndWxIdTRgs+CGiHqw9tCAdAEAEDjw8R3R/CCde5Al5JP20ygStcAQkCAfBglVLOWu+qrq3reraYSyllp5WUupPVulwul1VZrtbrkKM4b5qmWS7nTdNYq7Msa7s6z7MkSXZ2dkajwWKxkFIy2w4GA0R/Y7i3uxsRAs4b7613vXt3P2zqL9vGGp30eoNecWMw2Hl4749UM0DbX814Ed9+dD+1d+2je+u6rnWnl8vl5eVloxR6NugVRVEszi+924yweo9ANsiQiBg6zwAJ44QRKZ1XxnTy+Oh0uVhHSToa50cnx89fv6q7mjGRJpmUktGNjmExjotezlK4vJgLQZ1zVMnYw2CUpFEsEjPoF2mPZ4OYL0lKRRQV4HxZliZqm6ZpqrU1HXjrwTl0SKE/6nkD+3s7P/reT7755unscr63vd+1Kk57SZ5p6V6+ep2maduqJ0+ft418660H2tqqqXnM+8O+c7rpKsI9p3Ecp4geCYniBMFrLefzOUVs23Y6m4+GA6VM0zQ2iogybddFcZylaZIk4/HYex/md4qiuHv3rmD8+OCw67rdnR3O+Xw+j2N29/YdcnT8m48/PTo92d7d+/3nX3eWxEmufe0I0Z3UUhFCuqbpug6LAacotQL0jJJXr16cHB9++N671Xh4dHx6ePjaEbFD49CbZlQ40gFspJOcNeVq3TW1YJRS5hEEp3HEdddyIZq2ktqi37j5EEK01nVdr8tl3ZSAjiFwQb0n1moHzhjlnEFBLOhluZzPFlpbo21oITlnAlFMSr1crNN0g3rWdRvHsZI6yVMAkqZp05RJksQJy+J4UCRdkzTrhXc2iiJjFAI9Pz/nnNtQqBjTtEpKyTgH9IwRxkiep1HMj47Popilca8YDJO8uJwvnj5/3VRV3ajz2ZKlaZrnOXDaGeU4Fd7GaeIpsXXFYhF2LGMMeiiyLM/zsm57vR4AKN2ptvHeN237xddfMSrKqtzfG0ZxenGxYixKuGi6Fc8IExFSqjrtAeI0AgTVSQooGItEbO3cObchRRPs9XqMsbIsV6tVb9AXQsRRsrW1tVgshBDIaN21AGQQJ3ZdrlfrOBatbHxr80f3d3Z2Dl+/rKoqTdPt7e2Do0OjndYWPCIwB6iUWS6X1tpgWdY0TdM1ImKTych6xxgRQhgZKloXsOhqtQoWZ4hoPDhrQi3krQMA9MAJtZQEo0WGhLEIoaWE93sppdQaE1Jaa2yoqbS2WllEZIwiAKfgCcaCR0kUx1GSRExwqSJWEuNcmiZay+Fw0HSttbqq1uQKIddaN00jKJNS1nUNV+REAhjEGDZte8dCV3Y8HjPGFotFwPHyPA/pRdiC27ZdLpfee8EwKH1KKcsSKKVxLHp5KqXknKWYKqXmy0XV1Kvl/Pj4OBvmPIqCnA6jiEQwxglhZd1GURSYmEgJ55t2gLPQdV2QJRCcb8IVJWHIE8h3CIHzvlPKqkZK+d57713RMNE513VdHKchflzT/bz31jmK6K3TxjpKOaeEEKeNUmp37+b2eNQf9Jxz4Gwcx4wTa+3sYtbr55Px9g9/+MOHDx86D1prKiJEbDYFUBT0GZ31Fnwep13XBdZSkOIInqXBCf6qxbChiQCAxiAT+Z15Y1BGc84RvB5T3NBREfGaanqFTm+Oaz5UONy1CgIgbuRFbZDHIICEghCCWvR+gyeA94R8h8RcTx6GyYXrpph/wzpyk5RcS0EjAdxMVGrnVdtSSikTmzZQuBbvOYGra/fWOfQeER1uhBwAA9Hq6tz/cAT0v5grXCcKm7VSDozr6vpienmxmJ3PpqdnFxezaafkfD6/ODsvlytrPYYUy9jKWKUUgAvyGMbq4Oi9vpwzRkRCytn6+PyyKIosy27cuHFn514cx0p37334Q2uN9zZNU6lazvl7779//9FHiDibza5/sv2t7z9662eImGWZNaYsV9baiAtrLTjbNM304vLk5GQxmwUmf71/rpQONKmqbruuC9Ca1YY6gsQLwThjVivw3moz3rqxtbV1595bs+n86YuXjJP9mzcODw/TIj6fnnY26o/uaOgOjl8U60wIkfVBqYpysn0zp6Q3GAwGvb4xe2TDIW2kXXdts1wpCEKfyIEgi0TXNXXTOKNjEeV52sv784vlaDR48ODBf/zL/1StyvtvvX1xMRU8i6LIaN81kvPo88++PL+c/umf/tdIyZdff93K7tGjB3s3b5yenc1ml71ez2rnEbyDIB5MKNZlJeVpFPMoGC1C6Lxke3t7eZ6LmI9GoyzLnLWr1QqdL7J8PBo5bWIRWWujKKrK8sWLFxgyUmw4i14fHp2fn/+//9W//h/+yT/9m19/3MhOO+vsdX8KGGNZliVJtECW8FgpRQhhhNZtfXl5TvDxaDTwAP1BcXK+nM1mZVUhZSKO2u46Z3dt21brpVGq3+93SiFl2tuYi6ZpeJKuyrJTajwchDpHCEHo5vbePCPeWWuVUtYGdlnQTvaUUsZIlqeM8rbRddO1XWe1a5qOUso5VGVHKY2iGACiKKKUt22b94tgvDefXy6Xy514nOf5YFB0gsdxLLu265qyLLkTR8dHsjPKWECWRDHnjBAioohQNLoNtVagqw9HQ1q1SNi6rD/59PPXrw/7Ra+s28VKs+H+jksi45wixDrsbe3w/vDk4lIMdH8wyos+Ilq70aJBpGlUVOtytV60bc0ICMEFpxT8erWK+tHlurxclIQTR9xartJhbJSs6pYQQpF0XperllIaJyLvj05PT+9sPbzbT58/fSZ4upytu9r2R8np2XHTNuNJH0FdXhzFcXzjxhaAGvRHVVU51RljZucujaJ+NqlXUwBw1taruVPbH77/wenJiVRmdrmIaCyb0nZqAVNEKOc157xc1977fr8vBF8u510nGWNa25u3bxRFgoiz5bysVlW1NsY0TYNRosFZ1Vmrw5oqA0opQhgi9Q6sQXBIkXqDqnVNNxcR8xBlWUYIaZomjtJQ9wPjhBDfSUupEBETwnsPXUcIWoNaeYJusVgxzillk8nk9eHh0dFRFAtATyiV0gU/4pAIME6c001VSimtMV3XWeuMdgDE2jBOH2qyBhHjWCjVffPNV2HYRkqpte33+4i4XKy7tiUoGPVlWfKcp2lGKdXWQqd7vZjxRBqiHdHSeU+BJLUyy7qWEqJiCxmTBqI0I4QopUbDYbBy4Jwq1QVmH0VCgFrttDYeSCRE27aL+TwUChHjoaWPHvxmVIEwQqy1zhjOMkr4bFoaxybj/ePj47ruhv0JZxwRGUHgG3XCjVWaaReLRZIknFAPzjoTx+LBg+8/eOtuLKK8SNM0TYS4RtqPzxZKqaIobtx+K85GWmsgmjIRx3EcG22dtdYap6ztlNFar3zpva/b9abub9V10L0mnWxAAgTvvduIQLjAiBRCAIIxOooij2jRG2/CbkIJJYSAsYhIEGnovnvvgrebdZRS4sFpo50LgZxSquoO33B/8N6HzMB01yN5eP0GAGciuYnHjAKiJywodXPKCCUE0TsIYIjzSJBooJRQxhhwDpTaQDB0PoAQqFXYBCmlxBNCiHbhTIAQTwgC+EBHUd4E4diQ8QTESGsdx2k4PRZMb4kDD+iBMYLgvTPOW+8U8RAyxVXdLJfLw8PDV69evXr16vj4eDabNU1znahdZxvhiKO+tkYp5RlljHEgXpl6tSaECJIsLqeBAXPjxo2trfF4PL65sxWEaARN4iwOZPuQecsKB/lNABjkN8OHAwDwZB/+y0fAlgJ0FzAw5xzTa6VUuV6dn58fvHrx/PnTg1evF7PLSHBntJatlK3sNHWGAq6PTyff7//k/cdtZ/+nv/23Q9Hfu33rP/7yV6PB5PJ4NSx2mRCyoXk/VdDVnY+yFOoujwopZZYlnFBdKyM8Qe69d94eH5wWaTEoBk1THR8fR0JkHW0aaaUpXMoJaW0rIBY0M4qOtvcdT7589fzOe28nW+NnL15+/09+VPDUAVmslmSYHi5nOCzu7O/Oja7WbUPo/oNHP/zh99M46YxtpEKkVX2xt9efz+fgvDF8sVgQQkREAiophHjx/GWWZXVdd9Lcvn3blcAY+2j3tjXaKSjyHD30+/35fP705av5fM4EN84uq7VHHAwGt/Yf8H5/+uUXHlS5eF1fPP0//qOf/y//r//5bDF9fPfBi9eHjXLD8T4habEzLtd16uXJwev7d+7wuHd5fPj2w5+B2/r1b14y8Ij4T/7h/6FqGiqi/8f/83+iWn/54kQCFAWMB0NKmKPQ39o7Pj2Z1iqKok4aqZVzhHLeVA2lPBWJ7NxgMOi6rqy68XjsfV3VbRSnACCSGNFTBEJd29aE4Gg02ikSzvnl+bTtZFr0VrOF1S6KIgm2rKaUMsESKSUBiHkUM76GtQE53hkBmqZZOy/bcu1NR9GeHBz8/b//pxrcwdmZ93bZ1dbaVZcpl0pqgQKhtPEMGAPmOmttK7M0H/d7vXxwdnzBPAeJHRSvzxullmFyct5VSrN8d4858MoY42xrlAGkJgZGkzyLCaZ5EaepA4/abvYCwqxHlsU9MSpsAeC8s1p2Sssojp21RmmnnTFGGeW9Jx6TJA4bAVx5WYbtrKqqXq83mUzm83koIAKHMYD24eE0oY2jFAAZDAaCi+9ard4HYnxvOPLGaq1FnBLGiQNA6p1bLUsmOCJFisaBtsYYhwziOFWqQ0SkxCMJTy9ljFKa5704iwmnxqr1et0pqa1RVcU5D7JanNNQgcVxbK0nhCEQR4lSRimjpPa+jRkNNIhQDzVNYxLnnAv7YxzHV5Uc2xTcBK8KTcquDkrpYNir23YTeK6skpxzHlmISai8tSbw5gJ9xBjrrA6zltc1WdiRN0EUwNpNHx2Rhg5IEKn03lNKe70eOOUsBKshRHRu0wUmJLF2s9khUMZ8yIi1UwHeAACltFIqjuMoiqbreShStbbXwLIxhvEIrvgT10egvFyl21bEcVDnNMYY16VpmuTZeDyuqiqkX/5KalArHfCP4AQNAA50EG4yxoD0WZLu7Ow8fvvtR48eUYoRF1xQQdmGzYcQZyOlFBUidFIZY4xHiKiUCtHdWmscbCQQtKa44aJex6Sw2m+++GbY1lcFepA+vG69B5mm6zo+1NaEkFDBB8tKuNJEgvA6IQ7AAVjvXfhG7wEoQeIBCZLNXwQPCMYHxxHY/OOvBDzshk0J4BGReEOQIQLl4D0QRCAASIJaMwbVpD/kFV4/gP7Khioc12/DNzoXf/DTMxYgk2tcxHtf13W4+UPjmVDgnHPKpNTgfdgB6rpsyqqqKinl6eX85OTkyZMnr1+/Xi6X5sq6wnsMxdvVYm4u/A2i64ZHTAhhjAb9mDSNe71ecPPKsiTP893xMIzYhOQgMGrpldXnd1cUPh3A/SEH8/q4fqL5FWwWjkgMARC8f6y6uvzparlYrVZdU33+6SeMQNe1Z6enr1++mM/nSHwaxUcHC0G/FSKejPff/+B7RX/4+vXrF69e94ajQb/HI4EM4ogWfNDKZj1bjnqDoihsZhIReW+15pwKazUh2JQN9PpRlGitY5Fsjbe99+v57Hx2FgjRDj2LuFISwK9W68l491KcPX/6fDZd5PmgiFMnpRfF55999uzFS8rFcLLF4yTI1e/v7ydJ0uvl/X6fUtLr9Zqm4pwX8Y26bq31aZpqrcu6ccamado0Hed8axT/8R//6RdffBG8X16+fM1i+t47j43Vy8WiqSolZVPVWusgdB32bWONcQ4RldHT2azX6+3s7R8fH86mi6+++eYH3/v+T37287/5678lSG/evHl0Nq3rCll8eX7mkbS63t/b1qo5Oz3d2x4/enBfqe7ls9P57Byce/TOO3/v5z+jgjvTnRyfbI3gooQsjZM0JoRYZ8Pmj0AdeEppROLweFLKQ36fpGm4aa2VwajMOeeM5XHCOafoATwSG0URYzTPsv2bQyV13TGkbjjsZUV6eHx+Ob0AT5033oBRjbXWO9Daeodhpsp7T+mmeZckSZrFq9Wqa5um6VbLcr1eh059mqbEB90UTQjzznW6CSR35wx6pzowmdnb2/3qs0/rqjJa5/lIytYYE0D0qqmllJQSpsChlo2SZdt4QrlJaRzRJCKUkohbitY6SwAoEkYcoEX0VPCYonfeGS07b6SSlhAAC0gJGk8AnXXg/TXN21obprcgiPBrU63Lvb29OI5DEL2GFh1BEUfee4/Qyi4Mw8RaewClbZhuHxVFrzdIkiRJkuXiUrada0FbM18utZTB/Ons7KwoCqTEOIeUSKONMZGKjbNSG1+VVLJ1WWqjEmel0UcnJ1VbpXmitVLaAiIQ5tEIce07YIxBrW1QlJNSc4ach1lKag1YMNYai3Ad7I0xQRM6/LTBqyKKImstIgnbkDbXBj9ovdPGIfGeOCBMiFjEkXOOEBpQZQs+Diz0jeSOeTNchcMYi0ghum4bY5j2DPjtdaLgnLqug8OnhbcZpZxz3nrnHCWBfweEUO+dtcQ5e5WFUM4oo95IjYhB/scYI6W0xidx1nUniDQUeUGlOOQTCWF4ZW60+V7YAN1aKiklXOkoBwJm01aFK5wzRZHVdUkppmlstaYUtTbOKG8tYYQRYAH0pizP8zzPvbOEkMlkcu/e/ffff//27dtwZRuFG+IgRcS4h6HmQ0RtHCGUEDDGdEqHTCUIcF0XhQY2SYO7YoyG9W/b7rrpcNVQAACw1Ib7P1AZKGU0qCU6fy1ecN1qcM5v4j0AbF4NYgRASLBX8AAY3gkAzllKSVi96yz8Onhffwi+YRepdXjdIgTLSh/o2cZYQth3mgpAwxKBCP7dHsBdySn9wecjIqK7zgz+gEXxRqJwtTbBvtI5B+GcvUdKwTlwgBvKpCfAfF3X3rq2qxeLxenp6fnJ6XR60TTN6eV8Npudn59XVRXyxbC8Qojr1skbJ4anp6dhc48iEa4oiqI4EaGfyK+OQMDK8zwvetf5+iZFDg1nulHCejNLANiIR/3nx3VJc02w2NwPpqOUAgGMeR4X+WTnhvdg1A9+8ScAYKry9PT44NWrb7/99vnzp5fnF3lMTo9aQqTgxe7u/s7O1qOHtxfLs6LPbu4XSOl0vjQt5PE4SvO2bXe2tra2tkIWqJQyWuZ52nWdM8pIzXkUiySJ0jiOt7d3Oecff/Kr9etVTnPrjNSdEEK2jTGKc7aYnXtjZ5fL9aqckcvvfe8HH77z7l/86788ODpeVWWSZnRdUWniNOsXRVlVysimo2cX54IxIH40GsWx6Kq2Kavtnb2u6w4Ojy/Op977nR2OHrS3Dsjf+3s/Pzw8DrNLFxeXd+7u/+xnP1vOF1+8/jRL07qqTo9PghQNISRKYguecJYkmfVOSt1y+vU3T7q2ZiLZ29pal6028OjhO59/8c1iue4Pt6IoklXnQS9XFRcxowa0BQLvv/fwT372M7T697/5dcJJEomDg1fL5ez+vds379zeHfc//T30h0ApRFFEkXSy1cp4ghSJdd46S5ESQj2EfQeABPsRRA9b4wnn0Xq9bts2ieKu68xKWRUj8QBecET0PI44500711ozbjMm8h6Pk6EF9+zZQVlVUSScJW0tQ1VjjPEehdjww0IrBxHTNE/iqK5r7yBJsuAMjEjrus3zXrlYVU3trM96BWPCe88ojaKoKdfeg1JKcDooeovFwlprZBeNaA2WEp/E3IG3a+msipOMOYqNklXbSKNZzCwB5511ztJgIm+cd0HSxTkL4B0l1llntbfaGe2UMtY48FVdMY/EeeIQArkMMES4TaA11ntPCXHOuYDoWrtcLsMdcB0eaEwCiA2A0dU4+3i01TSNUiboG3rv1+v15eUlIgI4pRQ4D55Y652x1tqIccYjnqSEEOIsUkqcNc4KIQiJ4zRRxkgpRZLuT26OJkPn7GI5Ozo7t1bHsegNe/3JJNWyruuu1QDgvHWOAGyG+ANdTmvtHUUM+VDYHNnl5UWv12NMZFkW9qzQwI6iSEoZ8BJ4AxQx1nm48vUx2CkjjGaMVXW7XC2k1EAIMg+UEODsyvrIe+/BXocG51zbtgCotTXGUcrfzB7CzxGqnNCp9d4rJUMVSAkPEngh+FEk3iNAGM9Do52ShhAVcNSry0fETaQPooAbBp+FtpFN1MRx6hw4p4MpYhjxt9YabQmVCBCq83CShjLn3PWJXUfZq2Djuq5br9dbW1tSSqt1mqZJnhtjtFRJFGdZFgYTwPk0TTmLcdDf3t4Oe/3u1va9e/du7N/qFYNwDt57wO/0MTlljEdd1zVNo00jhDDOBw62cZtRBW2sMdZY55x31lwx0a4GF4MIhPrOc+E6anrvLdlMlxhjnAXBneOAgBgUbpCFn8/5TSNUvyGDcX1savcrKygXxBHCPeMDXgDfgQcb1eTN9oVXrIAQ2aUyV0mDu8IMQqEvN2KXGza+o5RS5xEZIUAskCu3qrBu5ArtuM5CyJX99JuJwh9GT+u9f3PpvPcRF4CeEMLJhtfZNA14L6UsV8uzs7PDw8Oj44Pzk9OgvSZtoLK6JMnC0oU+GmNBnWJTnIT0CxGKogjncy2/oZRqO/Luu++Ox+M0jZMk2dra6vV6WZZkWWYBEZAgsYCUUIeEEOoJsVcLiojgN9nBmwvy/+24zqg2jySyDdCDHpx3Dq0z4IjrDDjHo/zWo/dvPXr/j/7bf7i4vLycns9eHJ6eHlfNupMleD5fruNEDEcZISaOPee0qYl3GowWIo6yQimptXJWa62D2X3MmTc2uJxcnJ3Pp2x7a6tr2qqqbt682Z/0oiwWaWSUamXjnFmWS+KJt+TW/s00Tr4++3Z3a+/nP/vFO48eH7x4+ennX/R7g5s3bjWdXJZr1kmkrG1bJjghZL1er1ZLY5RWHSEkjkUm8jjPHrz99ovnzxfrlQYHztV1nSSJYIJSevPmze9///uffPLJarHsmna1mL98/iyJIsHpr3/9q6aqb968uV5XwfohENuzXpHluTa6LEtkqDtZl5U2wEXSG4wZjydb25PxzuHhedUcExEXvYxSEidMm65drxDcjd29B/fu5Fn86e++ePLNV28/fHDv7UeC02fPnrx4/m2vn/70hx89f/Z1Vakig1gwrQPsinGSAoDqpAMkFAkBj+AdIiUcGWc8BLL9/f0kSp88eQLOCyFU16HzAEAACSFJEqVpPOgVO9s7/eHSGDOaDKzDutWLxbQsV027thaKfGA0qM5wJiKRWOsBICj6WKuTJCmKYrFYoPdt2/b7/VhEo+G4bdt+b+y9ndeds0TKRrYdIYQCRoyGpz6LI3CJahoCbn9/fzGfluvl9njEOe/lsbGpUooy743hDAAJZ8CiKGo65cCLJE57vThNtHXGO0aYQ6CIlDEM7UMPxmysjJzVzqiQKPiNw5DdzHRb8NZ575FQRCRIAkrsrSOEiGBdqHRRFMaYo6MjrXUwsCGAURRJbDxBZJRR3u/3J5PJZDLp9QbHx8fWeu2sQ9Bal2V9cXHRNI2xKo7j3a1tFgtpjbM2jeMkTXvO5HlunPNaMcEjROMdY4wIK1xSluW6KrMsufvg4Xg8PDs/Wbe1rVTbKYueNq3SOtTHTdWFxaWUIgFrN/tsJGLnoNOdc8Q7ACAABIGEXolzEIpUzjlBBgAh+wskGkppKCuvKjP0iNZZZy1a68Az69frZaeVdSAYoYR7BOcRvA0ORgBwrf0XfLS994xxSqm7mpcLeEYo//CNMUK48g0KCoyCxwHbCH+L4HfThuHTwtsAwEMYNCCE4BUs4az1AZYwxgJAXbfOQa9nnQOtrXOOEkY4RSQEgVLQWuPV2EKIghpJeADA+2saZui5xHHs0UjVltXq7r3bRS87OTwCcINenxEk4HtFcffuXYp4enqKiPv7e4NBgYg7OztxHCdJEnQ4er2+EFEIUYF+769VkI0N8ck4b62xHrTWq7ION63zsMEVXLipwTsPSBin17V1+BwP6Dw4D957513YF7z3joRxancFifu/A0XgFb0/vCGga2+mCP6KD/Vm1Hmzsr8uXt88/k64eqOy36QNiOgQCEGHDpEickSKwAluHluClCBV1qMLQ+F/oD7+5vm8+aVCwH/xNK6zK7jCT8IbZNuFP0ecc86tNU3TdE1rrDo5On7y5MmrVy9ms5lsWymlMYZG8XXOaq9sQoOsyHV29eYSRYKFV4QQeZ73+/00Tbmg+/v79+7d29vbyfM8SZIAz163G8Jx3T4gf+ihhW/Yc5M/BAz+8+Pv/BYbZCIwOQlQAgQ2aT1eOW6FpyMfbg229h49/KFZLztZr8vLsp6enL4qBlvjrb2ua7pOtY3mlCV5xghaJTmLnG3bZtk0jZTSKMU57xcxo6Quy6IoppfnUsrtrfHl5fzLzz5vq3p4Mx2OBm3brtcrKWWR5fv7NxORgEPdmuVyvbe3/4u/98cfffQDo+zvfvspJUwaazpprKeEWw+L1XJVl4PRyDmntGzbWhuJzjLGtOl0pAtSiJTt39l7PH80m81ml1MEd3p2OB6P54vk668/392daNWcnR8dn8ycXf3f/2//1//dP/4nv/jFzz//7JOTk9kH770XDOIpZ0qbVlW1VLnSQfnNOHv37l3GxHy+/OyrbweDybpS9+72f/LTP2pa/cnnX0z6Ax7HVdONxsV8PqeC3bx589HD+6vV4i/+4i9U24x3tkUcXc5nu3t7PGKjyXgw6P/RL3729NnXT58+xRYZY0Z1WioeRZHgDEFrzZjw1jkg3nuPhCGLeCSiiCeMEJImibN+NpvNp9NeryeE8NZQQgAcRYwYz5IkEOrjOG6ahjLBeKxtNV+UWussy5p6Za11btOYZowheuM8ASAEvCdCiDRNT05OKIJSant7oolZLtdV2QJg4Hoh0iyJvTUOPCVonbFKG+9r7ygSRMzS5P7du7//3d8ao4peJhhP0miEvTBJp5TinFKKzmgmhLDGuzhhkcj6A5ZEVddxpQmSMOhEEBmhlNBNKaAdA6CEAUPjQROrnTfWpVFMrA80BSUlWCCEcMrjNL7eH9/YrVyQLDw9PaWUCsYDAgEA2hhApIyJKIqSmAnedG3VtCdnp1laOO+tcx6ARyLJUuNsuWzihPbHk62dva5p6rIyDppORUkWJRkaLa11QEiYnibUegdIjXPK2JSyKIkpZ51WUinKWZEMKMW6ldX00ljLBUMNbuN9bK/3O0IIKbizYK1HpIxGnEeUcELIzuQDQkhVNaHFkKap4HFwVb5CHULTgW82HcYJIQzRWq2DtIAHC55wkTDuMRgB4PU+GNCH6yTjGlFI05RzYY1vWxn2NWtt27ZCbPrxsKl0WYh8YaPvuq5tZFA6y7IMEVXXhfyGcx4iUVCepnRz8pSRq8931rqgoCdlaJtRKeum6bxHY4ySwTYz4BkhPeKtrOHqZgjNCI+EEKK15pQF8QbOOVyp/WuHdV0H97nt7e2j1wdaKqVUlqRCiF6e74wnjDFvXczF/fv3b927KaUMYo5pmvYHAyGiILHskbqQJwBa77UxStm6bcK3hzjUdEopVdc142JTifqglLLpECFleNW5COvgUDk0hAtPrDNmMyvrr4SWbSiyN5W692iMA3Ba22vs5M3wZtx3Zff1iwDAGF5H5esGP3jvLFwFbA/fqSzgm+nIH/z7OwEDgoDgA7mBIBJAGnADDwCeoCfoyVXvP3z+d5EvjFy+GZKvMgPyd1KH69MOrS7GGHmDl+O1Chcb2DZN0ywWi3K1Oj8/Pzk+fvbsyfn5uTFGUEYZUkq7VhFCguCSVjZgUQj0Oqm6tpAI6EpZlmE7zvP83r17Dx482N/fT7N4sVj0er00Ta/ng8J2zEV8nRwgogd0HsEjEvb/OwP4//O4RnfCiV5pWThrLV490YyxiEeb/0OBDQc5ZPnOCOzdrf2baX/yzrs/PD56dX5+cnBwoHQnIiToCCFpJvJhxDnKzkQCKCEUfRzRNE1fvax9zj3oLOHbk4Fsy7pZLZaXu3fu74y2Pv7447Ozxf7u5PHj937yw5/c3L89ny1/+Zd/dXhw9uCtm8iTv/pPv+U8ivNB1msup3NjTNYfWO9nsxkRbLK9e3p6ygQXEWeCi4hs5mTBtratL8unL588evDwRz/70bdff2Os1FLd7d/uFUUc8d/89lfvvfee96Yo0kcPtupqfnHufvubv/3Jj3/4f/k//Z//+T//54eHh2maXl5eDkdjEUedsWVTd8aKOHJItIf5YtU2XX8wXk4vv3ny/PzkkgL98MMPPaEGUDvbyM5Y2YvzNhWDcf/Gzb2bN29+/fXXLw5ejwaDm/s3SBQt65ZGJWGciTgt8mG/2N7ePjx8nXjsOtXUpTYuqMhbY7zVNEoA0DkPiAwJ54KziFMhBEtEVKSZMS7inBACzjNCHXgKaIw13nnnvHVd0y783NhqtVopbYveiEWx9wSApElPCFnXtTVojPVO+8iGDoiHhDPur8QhjDEijr2Xy+W6LqtPPvmsbduq7ABcJBIE5qFjG4N7CQAEwQMYpT0Cp5hnGWfk8uIi5gKcp5QIFmlivIWuVaEfDR46rRgaRwnJkzTKU5EkjqCgrEhSAx4JBeu9s94TRILOg3UJMIII4AC8dOCsN9YT5wkQIN5pY62VWjnjGWMAGF89UaEHEfZE9f/h7c9ibUmz80BsrX+KcU9nn/lOeW/eHCqzKiszq7JYVSJZLJZEtSg13ZIts2XIhoFuw4Zh2E+2Xwy/+s2wAaNbBNr90jAMobspthqSSImSyOJUxRqycs6883jms8cY/2n54d97n31vZrFlQGQgcfOcfWJH/PFHxL++tda3vqW1ty7Pc900VVV1slwpxZG1bdvaRUomhHPn87kxpizrsiwFV1rrUO7MuQzh682t7SSKkyxXcVwUpTbWkHWu2t/f7/UHjjwIYb3zAAxAKEnAjTFRkvY3BmmeNa2BWVG1WkQRY3GcRioSRTEzzjJvsiyj0lmnYRG9X3TTCWKL5BGAcaakjBgTQMx7unRpM8RCw0IZEv/r6+nKp185hcu4rURHAN56zy1GUQSLkj+zDHajJ4ykWD9gmKjAPRFCarBEzdLX98aYNI1DtGAJcSBwBoVQgSAyn5VVVQUVB8aY4IvoWTiLMWZZxraQ0iOPdOHKEgDzfkHeFEJY60P/J84EUWutZUyEpgfhKrXWq5z8wiKyC3MSUsiRUgHocM6jGIwVrWmm0/H29ubGRr+T5Vmabm5utlXdyfJOlqRp2s+zwWBw5cqVvUs7ZVkHH1HGUZ51rHeBvKmtsdaHph5Gu7quG902bUVEga4YHk7rPCAzxqCQIXZCRAQMEBAXdU3r8YAwvYE0ujLwfqmmQMCCBVpZ9+AvrtIT68kFRITlGZ/zjFcztmI/wAWJ4WJbWejPhxnCVxjwYK+W+sEs/Oc9IYbWDOG8mnPJmHdKLk/0DFB4bnirB9uDWZ1uHShIKW1oTsCBLS4HOFvUXlpr5/Pi/Pz8+OTw7OysnBcPHz6Yz2aj0aiuy0B19B6JyFgf1gchJACGuJoxoaTiIuISCBIA2Ol0AgUhyGaE/Z1zW1tbcRyHiuLQf0SpOCiSrXD5aibX3+LnDf/nyju/8PPV/PMFy2FBO13uhlKqQEWWMnBywRPMZjOkNk1T3bac8yzNN3ZvvrP1wlu6evT4wcHTR+9/8JMH924V5bhpyzSN0zwGbOqmLKuxEtI7W7dmPFHW5M43T548tFoPBoN5MSFvsjQq5pNyUu1vXfqEf7rTp3fe/MY3v/atX/zFX0o6Gx/85P1f+uW//rOffXj71r1//Qc/7PeG16+/eHQ+M9YHH9o5VzTNvCxS6AghmEgw5OeBjLOmapu2MsYkmSCijz57j3Hb6XQm89PWltZpcsy5quDi9mcf3L/36cOHDyMpo5iDS9/4cjSbTX7vn/+Lv/f3/t5Xv/rV9977IE1TzrkxxiHz3iMTyJkHhoBcqLPxhANKLvcuXRufnt36+E7bGkKxv7/7ve99789/8uPRvdtckDF1r5elKj06OXOEWtvtvUsc2WhWlK0GaybTea+b5odHu08Pmmaj1m3TaPCqbWvvLWPCe+u0cc5JKTmgDa8jcM4FJ07WWbIqYYhojPEeOlneyXIpJSOQKpWSm5YjgzzP0zQl8lpr3TIlerPZ+Xx+2ukNq9qUhSYQl/avHh2dGG04l0YbIsrz3LRacSTnQ89CY0xRFEkUAeB8Pm/r5vatu3meKxVrrdM0aRrdVjUAAJLVRjCe5hnn3BnryWZxmih1+/Zt50yaRG1ddfOMM9k2pq5aJKZEZJw1xqFHYVqNyGSslJAc0BqLAImKQHAHaI233i0lxNBzikGS89bZtm1t0zit0XlBqI1evYfe+5Ct4FIGJrm11kkLixpLDFOZ53nLeSjfl1JGUoUsfuBbrufXA3M4pNLDG9vrDba2tuq6PptMnHNBv68oitaaWCryUFRlp9cFzpxzVV0Z54Bh7OK6sWVZGtMiE23bPj54yhhMpqNuv1PXVWuaNIsdEHAGHowxiYwQERklSSKl5ByVUkmSRVFEHok4EDpHbePKsmqa5smTRmt9djaK4zjkzjkLKo7EuQyUq5BkDY6LNk4QCrFozRDUuJxzMlII5L3XxlmrV3Y0WBe33FbRUVqSRo0xAMwpF7R+Qyg1BPO990QYXDdEvsju64VWYyASxjLy3gfXdGkGlskjWiygzLOV1xvg6iLKwhb5FIZLEWXCgEsWq/cSKKxbPloLrYdrDJHk8K1erxcMQ5CCuHTp0t7ObhxFe9s7ptVElCVJv9sLfRk2N4ZMoZQySTLnHHAmhDC1aVsjI+YcNbq1xltrW6PrummaxngbwN/Cv0emOMRxPC1KvrSCnpDBgq8H5Nbd/RUcZMttPQbuA7zzEFz2VUTBGBvKOmAZTqBV1QDjBOTJ08o1X0g1h2pJT4RAC5K/8wDIwv4+TOSy0oEvqycWjZpCWhyRVn4wYjgsYdiBEzJiSBeFA54ROOOeu1PrIADW4G8obbXkVh+um0xrFt3IjLbhAQ4RI9INEdV1eXxweOfOnXv37p2eHTdVXZYlkA9vDSIFr8Bam/UGQUHELRW3aEGYveBMrPv6QVUlfDYajcIz32my0GmQcwwYQghhTDufz6M4J/CAHBBZQFGMI+P+oo3VMyCAX8C2Z7Z1PAdrSIut6UgE1mjorUlAnHHOuHW2ahrnKJT+e0aCCaFU3bTaozeAJKNk4/r17s2br79485Uf/+iPf/Lunz68f6tqqkbXaeSsqZ1teCwISBdVWUyAzOUru599/AkiSsnPT0+MMd1u3jTVwcOn77zzzotXb/R6vV/91e9tDrfns/qjD384Hs3OR7OHj57O5vXJ6Zk2QDx6/Pjx5vZmkmSNbpumBYDhcAsZG03Gcay0M56s4oxxIG8AQEo+nY12d3fPzk9+9NMyS5KDJ0+9dU6bNEkET7d3drxtZ/NxnMg0ijnnsZTb29tbG5t37tz5L//L//Ltt7/+7W9/88MPPx4MBkVZzWZzQpb1unGcOqK6bXyNN6/fyNPsD//1v9kbbnXy3rw7+/CDj733/+Af/OaXv/zlDz7+oCznSTfXptkebMdR9/T09NHjJ5zzy1dfyLLs6OBAN+2HH368t7O5vf9a3eq7Dx5OpuO60UyKIH/KOZdSKSGRkRCiI6KmDZQlJE4B8WltiUzSk2VZPnr0qK312dlZEMrLk3Q0PnOO66bmgllrvXXatMYY52B7eztL/flozJjs9zKpemmWA4hi3pAvwHOjK2SQJBFngOCbqq6qyph2MpmMRqMsSdq2ZYxJqbTWSZKkcTyZTKIomU6nHDFYz9DCNFaRlLKBJlZJt5PZtv3oow8iLpjgbdvGcSyZ1LUO7YGEEGY2c46kVCEE7pEZ8JohS2IpUWrnTeCpMWJOcCkZZ0bb1jTEuCULRBhHAL5pyqKcW9O6RgdNzaqtvPVJFIcKECV5rJRhqOvKGOMlj4RkoeNA23Ipu/2+EoIQqra1RGne894TijjNZRRbD0mSDoY7SdZptO70e3uXLxVFEUXRfD7vDvpCsLIsybZHh082ehuHT5+kcbK1uR2W4KPjk6qua6urut7Y2CwaA8Tr1jaNSZJYKFnMayInROw0RDLy3lVFzRh0k46PEmOMFqSyJFZJyNlLKROVRFEEAE3ZWNsopRhD7cqiHtd17XVnNpshMiVT730xnxXzZjAYhKxqcD1hwTDwnLM0W3ii3hkuIJMqcCM4QiAycKEsLHL2jDEDTd0sWmNrbTgXWZwwQs75fDory1IiJpHk0HryGx3Ry/KyLB1RkqWMsbKoddMgw9Cku26aJEt7g34URVwKAMhFHKCeM6EE3AghODLv7SIkCzwsbhwFE9yAYUxkWWc+n5+fn0dR1O1266ba2toCJEAyxqx6J3rvPWJYlyEUCDDmgGxTMSQeRUnEs1h415ZF4YzOYiXrUpBrJ6PpU/7K1b1vff3Njz79JO/ubl/apkVuO87zfNgfdAd9kUaNtSxJG/LAF8V4TkitNbWtcbQquG3axXgMASBHREeBbBc2K0Ox5WJ9p2CsiRwAw0DOJQqtngJOKqtmVRkBIbYMjADiiHNOodWTc957i0yE6lIHIZ7EgDNA5iE0gQ4WhC359Rjcz2WMihOBc3RxFnLr7v4KwQTPmK1tYQfFVhEGBOAIHjwRYChGIUJgHlnwpL3zEPpErAOFYNlC/dUFGgjJFu9D4xC/1HRawCm+zJUAOPIBxBjv28aWZ2cPHz68d+/e2enpaHQ2Ho+ttbFUHkgbE8KhFPSmAKSURVUQkSfvlrmPcCl1W8OSRsoYYwuyDaokMeQjzgbDjc3NzThNuBRpmoanOkvSKIrAkwCM4gQAkDvOgSMwcAyCljpnwMGvbgewtQt3fjWhX4wkFpjg2R2e+dNqBkPzC8a7abb6q8EkqNHFUcqICQkAQA4456Zudwb7v/Yrf+fK7gt//Ef/9t69O8XIO4jSrO/Qn40LJXA4HD56+OTKpWvMw2b/uq6stDm2yWZ3887stuJciviTj2/93f/of9LW+vRo9Oj+4WAw7PY2To4nDx4cHB6PhcoKi23ZHtx5sLu7W1v39OhkY2MDpfBtyxhrqnaYpuiZ9GAJgDnOpYgSxoHIGWjPx1NCPqvqoqpFkoWJ0K2dI7t3ekZSjltjPZzXDUfmJzOZZ9Om5nn86OQpfERf//rXHx3fv/fw0c7uXnE0FzLiirSrpvPZK196PUW7N0xG4+nu3ob3UCOxXl/E6c/uP4B/8bt/L+a/9B/89bPy9MHDu29+9XUiP567/WuXOJOn5+fTttm7dj3pbvzkhz9UcS5U4jW887V3Dh/efXr33gubm7S396CIdwZXprOZdjZSOSAXMqoaLfPEtq0lAkk1aONqFSdZltWtSZLkzr0HVjf7+7tZdhUZEXmZ0ZOnj5BDlMQkyDA/ayohWEdEp6enHsABTqZzYIJx1Yvis9NJpzswGqqiyeLcGzY5nSZRClSDbhKOvU7n+OnTneHQau29YwyBgUqSKvQG41g509vZOj6aJFJJhoy3xnnTtsNe98b+fjGf1nXtG8N9dPXazXlVIvJJIZBZBxFjqdbgHHKeJWkqgieKXCBwB0TWeQ+GvCEgRG11nCaAC852oNFR4ziAAzBto5vKWeuM1XUDnixgeJmZYJGMEVFrfXx8nETx0u+Eum4tW6gBtrhwjmFJetemNWTSNO10Oou88sI5s0KwsmzLcrHclGVZ13Xw8uM4Dp66lDz8LJVAxLqu67p0jpbl+hhFirO4bVui0Lci8l44Z5BBFEUhtWad9gv2FhK5rNcPYZLg80kmQwKlnFdVVRljJBMAEGJBzrmslwUEF1zGwWDg16rX2LNUcFpIGuAKRoQdggMdQtahpDtEU7z3xrHQuHfBmQdOhN5D09RaW0SulIpUvFqX8VmR48AA4EoGupOxNoqSNE2TJCGEEMta+WrrZLoV/TBoPYWheu9hGfpeLYVht1CTVlVVuFN2uaVZzpElMuKCcc4l40jgvCHrIiY4Y+S80do12jljCdCDECIoATe1Ns62bTuflaG/sxAqFD4kScI4D7nkIFNES3/d+oXIgfWwaNdkLgRwlj2MLprArm7N8wZybWHHpWjg6rurO7vafz0ItH7fww5sTVAS1ihybk3Vcd0CrX+yfpzwZK4PeLVzOO+qPC989wJSwEouEmnxOSMMKtBAHj1iePefO+z6eNY3Wsa0VvggIEvvvSOvtQ4vadu2IRdgtS6K4pMPPzw4ODg4OJjPZmU5L8uSiNCTsdoYgwREF6zJMD/LNcGvz0kIV+CShxse+/DaJ0myu7t74/oLW1tbIX6wikOsjkOLulNieGH417cvvNi/ko39HHyBnHPnXZwmX3r1tThW7/5s886tzyxNx8enzHpmPSDV81k9n1Wz2dUr108ORgjWWs15BxGzbqeqih/+8EdvvfUWYwLRHB4en52dzabvlWV948VX4kjubG4Rl1cu7ROTp+fTpqo0tZxB6F8jEIwnYS0iZnlirdBO+6UGOSJyLi4EUax13iMwzrngfJHms5aWtOgA+Hobg9YaybjKkizLmBSbm5t//z/+zf/sP/tHztv9/f2mNYxLGScvvfLyzv5ewuns7Gwym+7u7yme3Lv3oKnKK5cvHz21tz/5+L2re//L/8U/+J//5j/48U/+TApG5O8//TDLdNVYRF6Wk5999O7O5o7lHhPx+Phga2fQgLv00o1/+69/dzo673ay2HIppVSq1i2XoqobrXWra0JwzjOGjDFCEMQEQyV4JCJnjF9eV13XTVuB93GiXrx+Qwj2+Mmjxw8eDoeDKIoSlTx99GRjazOK036nPyuK6Ww23N7O02zCp0msOt2MIbZVq01VeG1sXdbneZ6HOHGn0wkxWq9pldZf+aKLwCfpujaGgeQ8imSSRJ1uNtzs7+5sPn78eDaeXLlyiQtyziiFzplQJNy2bdM0ZVkWdZUkSa/XE84zQOAEznoiC5K4UIpzYGhck8RZXddEXknV1k2epZ6hZLyuy7PpaDIeN3Wl67quKsWFFSQYV1yFAvpiXrZ1PZ/OkijudrvdblfJoArAwrvKka38ntXz3zRNkiREVBRFgCar1945V5Zza7UxbVBfMSZzzgAuyt7atl2o2Xq/t7frrGeMCUaMIVVUFEVMaSdPEIGIAL0QjPPEOUHgoyhKsziKpHPW2FDNj4iIctHDgxELXGgiMu2ikp6s16DDQpNEaRpjIIAEXMWWfOyg8htenqCztLre9SUYABaIZjknuNRKIqJAuDNgdNuGxYsvlXe995zzNE0ZY3EcR0KuHppQEBEE5hARiAGAiFQoXMRlB21ErJu6LEtTNKtQPCISeCmlI88YeO/8ov2BCLW8zoXO4dYvhQJDqj6YvSB0MZvNQpw5PL4CUCKLpAy9npUQROQM44qladpLM8kFoLNCImdSyrZseaw4Qw+MEPrDjes3bvZ6vayTh9RzEsWxiqVSRNAa7Rlf5WUcLWpDtF2QBrSxoanj4i1C5p4VCV7N/Mp++DVGIVxUDayv2EhLAeaV+Vn/K1vTX3rmvq8JEvjVn+iLgQIAhHsdPnnWwj3DIXgON6wOtaREEC4qAAEAPGCIXXhABEIC6xdKQiFyYN3zEGF1ybSsclydl4hQoHE2vAVcCuQMgJx2AGCtVUrFKvLej87Oj4+PJ5PJRx99NBqNxuOx0TqEexhjS8kaB568XwAFFvD0GtlzfZbCO/IcoMFl+U+osFVKBcEuSKLl3Vw8GKvLWd2s9Yt97o6vzzbg/0DVw7/j9vmzLD5fnDJQYzwRQwhPiGeRYg4AfDKIXht8PclyqdKjB++NDw+3tvvTSUtOt3XVTaKr+zu9LH3SPjaWqPHG5UVl4zQ6H58SYdua0dl4e3vbGX92ct62+oVrNz764MPdvcuT0bkn1snSOMnrqooEdxYGvW6eJUTknQKtKZKRVEjAQgmN89ZarUlIxiQLQCEsX8YYoFAItiibYktXcPUszebz6WzW6+ZMckI4OTu9c//eL/3SL339629//OlnKpLIGJeqqGopRV0U944eHx4eArC33/za9nDr+Ojg8f3RJFZgmiyJzp8+ufvxx19+7ZXf+N7fnM3Hn332WbeTNk097PeSNJ9Nk9mseHrwgKG9fHV/Ohk1pH/03k//5q/96u6N6wejY9PWcbwROPVQgiOPCIJjFEnrDUfGBCcg3WprnVRccuzm+dnZWViQrbW6befzqRD80t7145NDjrjZ74nNIWPgjO3mMbe7WZo/PTrsdHqxiAqqmnn52UcfOkfWQ6pEZ3vgnK/LSmuLnobpsN/vh9rXfDlvvV5vMplorS35KIpWPGtjTBzLpi6dwyROuKC6mY0nPFVsf2+HoD07P3rxxRed18h8FAtrbVFU5+fj09NzlcSd/gC4aLWu6lYQ44ILwSNizDNkILiQTElPyJgGorqqOOdJ3hnp8/HpmTAiyzLbNrPR+fHBU2taRt5aS0KAVFzGBkxbl8W8nE/mVVUhAEfWNDqO0yzrIGIkeZqm1lpnFsSiELMNr18grZydnQUpruFw2O/340Q5b/JOWpalJ0NguYCYSetaaxwR1XUJABQaNZFDpCRJxuOxczbw9a3T1aRIbcu5rJuy1bWQgJjHsSIQzrksS5M0iiLpvXcuRkRkRETT2QwAFFfBfpjWBqtfzorFEuMW7mCw6NYtrghCcGXNsw8XGExYWAdXcnXhZ7ZUAYJQLrhc8kLcPlg4kEiEnIvgnymlBDIrfSgvDD4cOL9qDVDXsyCVuCBzmYWCRcBqoRNROHhRlVVVMULrHQAsywh9UAPs9vJgmhyQJe89WGettcgWbsFKZiqMORAkkyQJGeLw+GqtY66SJEmjmHFkjAnGwZPjvK1rThDqd3TToPOMIXhK8yyKIutdKDC5tH9lc2un0+uHZLMUSkoZSnJq4733IIRb27T1i0aOSw3g1l6oIARq53O2gdYEB9cNUticuyjeoS/ys1dAAZekP0RcZaYREddiD6vjrDiJDJ/5fDWw9ado+ZUFQPz8YNYt5YoVFJ4Hv6jvZS7oOCFQ6ODgPSHH0K8J+Soj/xxOWs2VXyMzPgNWOPMIxNDjIrYUsFqSJG3dWMAoU6PR6N13371z63ZVVfPpqCiKqqq8c+Hl8N5rWkQygFZJFgphD7tsZ7XOEcFlItYvVTLDyBlj0pP3vq7r2WyGiCGe0evmK7rPOpy6uEdrQOHzM7/+L/v3gxN+7oaBfIqw8C2QkBDQe+e5ABDC6ZYzDoxdunwDMLpF1ejgeH+47ZuqasbWw5Xd/Te+8vrTJ8cIPo6jcNXz+TxQx/q9oUB1eHCyNdzZ2Bhe2i2fPj3UTfP0yROr7aPHh3He293ei5Ls6PCAeZMkSkUJOlM3rdHaOc8QnDOuMcAwIC9jjPcWLQojknTBW3LOeWvdgjF8IRYCz4JpywCREcOiqhiAdebu/Xs3bl7/a7/07fF0cufeAxVHeTeblbNbtz9WShW6qXXTVO0HH71/bf8q49TPE9cUpJu3v/7VLI7+6T/+x7+fxb/wjbe/851ffuG733v9na/9/u//m7Isnz49TJjcvbT94Ycf96Joc3OgLm3NppMPfvbuC1f2Y6mqUret6eV9760xtXWaC5FmMbEeIatbzQRnXNbanLeN8S6Noq2NjUx1jg6POecMqG0ap1tn9KAzyNK0m2VScvSOwHFkLIn2t7ZcbzNKk0cPHmvRKKUG3Q4Be/jgIRPCe591OpcvXx0Oh1rrs7Oz2ayQTAbuLWMsy7L5fF6WZZpnfllUGGhquNQKS1KpW2Do40RKhtV8fuJadHq40e31OmU1nc7O86yb5ZEnjYwm80lRF/NqnpCVkWhNY6xRkRDVrE7TNI6EShImpAUiAGeIyMUoqdHUaJWmCqgcTe7dvduNOpcuXepmaR7Hg06OlMeJEow75xIVSa7a1pyfnE31zBhHhAxFXTUTLPr9IkkyzjlZh8DrprTa1HVtrVFBFxPRORNnqTF2MpkcHx8G1duymhszZAz7/Z5zuigsgE/TmAtsmkYIxTmvy4qItKY4VpFSWZadnBydnp7OiipK4ihLpRRlGYLqnnOUkgvBlBJJkgCStTo4+kGthXPBGAY2tbVWMImIprUhkK7rNuAAwSRjHDhAkGVhnAFL0zTU8oW0SABASZJwzkNAPvDdwuoJS9LTKvUAy0Iy51ywu+HGB5OQZZnnxJEppZYKj8hoEd8ONkpr7bQJsoYhcxHEvMKS7ZcOdJgN6xzRgjbovJNSsmVAPZwdgFbBgM9bU8YYwUVL63VPdzweh8hYmqYBMQghtNbdJFdKScY9OSISjDNA720kZZ5mUnLTtkKISKoQdomTDufceGecJcQkyxV5oRRyQcgcEFrvkCAUznrvtHbOBTnFgOSMDa3Dyfil5gERIGcMgXAlEETLECh8zh1/Fgp8sUTBav9VLCf8NXT05vS8wUZEwIs0h/MLIwfk12HBcxaL1ipL6VkS6LqpozVff92sEhEgW/DsARZMRiIPoerBA+MIEDpcw7NXuhpJ+CE8e+unCAMwrV+BIe0sEaEnRFRCeunm8/nd23fu3bt369at48PDuq6lZAEHk1/IUgMAWRfSWwAUlEsAYPlngs/BF0Rc6Xctdl7OkkLs9/svvPDC9WtXQzghcM5DRm+Fy8P41wEiPJtgWpElVzvDX9GGAMAIYIE1PSEgLQIZBOQQddMyxmScX3/pS2x8/PjRIWPWOUSQROC9H4/HASdFURRFCRFOp/MAqNrach599smdgyfHaaxe/9KXr1+7cXJy9s6bbxvnxmfjjeHG22++UbXt3c8+nc6LLM47iSqKoikLZAIJnPPlfBalCZeMcZBMAgPtLirhQzIoiiIGoFsT3sp16gzCRbBNxEmiZNW2R6cnvU6ep8nZ+Ozd99/7G3/jb7z5tTefHj2dF0WaZ1tbwydPnhBlXolBZ3N2Pj4dnUoERUxF6Op6o59621RFoyKRJXHoD3LlyhUzGf3Dv/s/quv6X/zz3/vss1s4n1zuZicnJyJNru1cn0oujX782a08z6m1+ztXRvOawLW2FQLybsY4j5vYO3DkuYqQ8el8VtclZzDs5fs725ORruuaATrn55MpIknBhxv9r7z6qn3p+vnZyU9/9OfW6Vdeesl58/jB3UF/+8b1azeuXWradjKZDQaDzZ3tzX7v5OTk5Hykq8rqOoqkUBzHULfFvCalVL/fv3r16tbOTlmWZ2dnRVEs2lDJhau2CC3HcXM2c94QEYAXQkrFlRJxHG1tb37ptVdPjg7ruknSSEbJ48dPnXPDze2r2W6csdFodHzy6PR8urnZu3p9V9y//bDf7+/s0dZ2FPVULLjxznjnyadJ3LbtRtbNs4yMV9aNDw4fz+628/n16y+kUby3vSNw0ZehbRrOJAArZ8UYBRFyLtNUMRLOUNvq87OJNZ4xxpGUUkIw8kErjci5wJVDRFPXZVmORqPAABiNz46OjuI46nQ6SgltmlbXxtZSSsXjpql6WRJFkW5a732Q7RNccIGz+byqKgLnvWWMJUkyn8+5YEqJLEsYgziOpRJcLFI78/mcMUawkuuHUAhQ11UI47Rt21ZNAGuCyeCmS65gVdpObGWcVr7Uqo3CyoKG12PlZ4cFbgWog4O4ervCr3bZTyiO47JtAnVBCKV1pXVL1jVNI2UUGgUCAC2UIjki73TS8OiEno0MRRRFMo7KsiYiWIcsgjPGjLPWWcZY4CoGBWghhDZtGDkBeCCGyKUQShpdr8ySW9RcLNR5Q6g5JJJWPAlg6IE0OWeM914GiONs3unESYLktNZMiDhJBAOtNedcJWkvTVHwwXAzSuK61VobJgQCZ+gQDQcOAIF7XBu7CiavoggBIhCRXdIVGVuECgKbF9aqD9a9UnjWrQcAxsTq19XOKxD2nLUGgADyBC3SEwGj4LMhbiJCtgyD22fkmJ7b1mMVsLRkP8/CrOzZKjGEiFxEALAojSCyC0lHcgQIgN4jsKX3Ss8dDdYc69VIVr+G09XehjIlTkgUulYyDhhenOPDo+9///sP7t0TQighW2iCnNfyEVpmIYOCy0LJZRE4Wcw2XowH17bwYbhMtdyEEJ1efzgc7u3tXblypd/vh5cuzxIAWAGFpekCIlopG6zjsNUDvP6nv5ptSTCBZRYCATwgILDWaOcoiiIpU+8hCDZtX755/ZWvfPLZz7QTDmRjafTk4GQ0Ic+dZXHjB31ele2Tx4fO2+mkrgo/Hs2aqnZG97q5EtEbr3/51Zde5ZzfvXu/qWoVJZd2d6qqubS7JYCiSOxubRwDWWtlnLRGt9rWujWGOeJCcQj1gqi8tx5oRb0SQnBEzoTW2hrD1wS/YZlf985rrdMs01UZ1CvqtplMR0+fPr569fKXvvSl4+PDP/7TPzs+PnzppZc6nVwIMW0aQE+Mkiz26EfTkbH15rD34rWre/s7vW5+8+aN69evI0dr7d2nj/r9aF6NTo6O/9P/7X9y+uTJ/+P//v8c9pMr+6//+Q9/POgkRBBxNj0fvfjCi1/9ypunp6etbUTE4ySSUvb6PULw3ldt441jlhF3SBRxFrrWOl0Vs3Y6mURRZFtdlPMkUpLFkZCCs17e3+p3fFMdHTxuyulsNtNN3clyKWhz2J3NimI23tkc7G5tXdnb1S/d/PDjTx8eHLR1MxqNrHfj6Xxe1nsbu91ulzG2t7eXZFmgly2XJraqlocVRyH0eHO+aTTzhABRFKVpJqXc3t7++tffeffddyOV5N0+4oG1Ns1dp9ORUTfOrXNZeuQRSdNYtLNqrL03vqnawcZGmmcskogI6Dmw2en5xsaGtHR8fNxVyeXNnU9Ht2eT8elx0ulkCN45N2dMCOGNtca3rZnPi5Pjs9HZ1BijVJTEHc6kJzufl0GtWXKmlNjc3JBCxLESgjOA0HAIEafj+Ww2K8syXOF8Pp9MJlW11elkxrZ1XTIGUnJrNZFEtmA8SCWIqNKac+6c0boBAESSkltrrdNCsMBsa9raWO3JadOWZemcc24hthPMuTZNCAMEs2eaGtYcDo6CcY6IujHg0XIHAALFgoypdbhVIcy+MvYheR+SC+v+ehRFq0ACLP3y8MqsyhphjSCmtW6a1lunZOQd6daUZQWLhkOcGIVTMwGCy7DUrrxbQmCMSbGQPtzc3GyapijLptHhpIShkkLbtbY9jC3CG0RuZWwuWI1C0HKQsFQV5HxBKQ0GYxV4DLjBBgYQoGNgg1305Lyx3jW6tU5XRSmllJEixNrobn+71+vt7e+rJN7a2olUrK0jYOSRwHtg4L0mi0s1qtrY4J2vTLhe6oVfFP4Bw4Wd86vY8wrhreIKq5uybtdhTScA1jKsy1mi5/7q/IKsCkt2oaeFZSK24DnSUgLyOaixPoYVGWX9E3yW97AOL9a9/JU1ZcusByICMR/YOkTAloLQjCMSEgCwcCYGF0PCzzWAps9tyFkwD54xgQwWTwudn55NRuNPPvnk6ePHs8k04GzOeWsWIS7yPgCFAC8WYi8+dBshxhhbICr2hRcVUg/hzYqiqNPp5Hkex3GcZqHVXFmWQZYxSZIkiVZZvzCl1loAj4iRiNYncx3cw+cA3M/Dc39pW0gDAgB5QOAMgABFEIa1DohA5IOvfP0Xn5yfpMX5aKxrC5OqOZvM28buDC9V5TTPBk3VVoUpyzkRSpHMp5W1Oo2VFNEP/uQHTx882t7e/uVf/uWqKMHaWs90PU+ieG9rs5zPBsP+9WtXEdFai6EHjSRL0hlrvTMOUCATjIA8kHMOaOE5BM1LhCDhf1FDyxgDWjzb3jmRRMEDeXH/pSRSB08fcyFmpf3hn/9gZ3f7jTffODg6/PTWZ+PxeZLKptFpnjVVWZSFAnB1XY5nV3e2vvOdX7q6v/f1t9/u93uzsphMp2knv/bKi5cBTudPesPN9+59/P/57f/qa2+8/X/5v/1fJ+fjf/Lf/s4Lr16btjNrvSEvpIr6nbcuv/Pf/vbvBCOiVMw5ZwyNMZ4skuMMvLcMWBKrQb+nSmHa9vT4eDrF6XQ6HA6dc946njBn2mI+/fEP//za1f1f+sVvvnjtyns/+/GP//wHtqlfevFGZeqz08ezySl4iDjkiSqn426/P+h2+93uydl52+rT03NtXF23Kk52dnY6nU5ZlgcHB8a5siz7/T4h1HWNiLAsmIcgk2+t96BkyqTnTAAxIGEt1HV77+5D3borl689enhgjBFcbW3ujMdThwWPorSLeX9rb2/v6dHhxx9//PTwttjZ3NFaV9OiKsrJ+ajT72XdjlKKGCLSvVu3/QsvSMYf3L+/u7t7ZW9/c3MbAIRgQgjrtK6bxhjGmFKqLOuiqOqytdoxxqIojqKorlsGwJkMvZEYA5bERJIxoZTKskQp6YypqgVNoWl08H2FEN6HWgMMuoHaNIyxbjd3zs3mUxXJTqcjuYzjOJRI2FZLKbVrnXMh8cmkaLVFJVQUhfVlNBoFxnsgm0RRFASCAqcPANrGVHURQBkAsKXvEmKVDHhg5XTz3kK+zeMKB4Qtz3PGWIg593q9OI7DpYWyCERs27YsS+994BmsbMDqh+CGLgz1Wj4ikDmQANdI3bhcxWAJNcSynISI5vN5EDMWoYcekwCgte71BoiojVlWe6Fx1hjTNhq9D8opK4PpyCslmOBMcCIKvJCwWrNlH0i2VHRYLdNhkhdVvG0bgg2WKKAH07Zaax8yai0QQ88RPDLBkyztDgeRFKooggbz/v4+k1Iq1RittQbkKDgAoEdHzlnn9ELTN3RccEsxA0uwSDfAsu4A+YUgHgD5BRfVLVWB3bJqY/EALOtQcCEYZVcfwhpQWDfz6zZmpS618rkDXiEiYnyVj1ilHji7ABnrgGNF1lv3dNdByXPb6uu41JEMD4wnBFrWYAbYhAzCyJHBAhMwvwwnrIkCPaNSgM9SE1ZbFMWB3w4ewj0y1pqmvXv37meffPrkyRMA6Ha7p6enTpssyzzza2jYLMCQ8yFDggTrOY7VthrDOlZYxXXCXIVHEQCappnP52ElDdy6INayihZ47y2GklSU/iLLBmtQ4POA7K8cJaw28gAAJJgQjHkC7yH86yxJVNvXXrh0/UahJ+P5mUORdvqC84PHB1m325am2xlI1vR7A7I+TdO9yzeqcv7RRx853X7nl38xT1Mi/0d/+P37d+9Za0/PR3nWMU0bR6lgOJtMXnr5+tb28OzsDLytm6YqSlSKkUchLNmlphZjnNHK/C/TN3JZfsYY8xdLSCgAXmilR7GazGej8/P9/V3OeV3Xu9sb21v81u3bP3vvve9+97uvvvrKvCxOTk42t3fLsqxrkkJsbW11ogi0QavzTjrcHHzjm98IpD+UjEXcMDcup9Pp3HX1ycG9S6+9+Mff/6N373/U2d04mp383X/49731v/3bv9PUdri1PZvOT+eTl9/86le/+c5P/uzPhBCMgfWubkqjnXNWCDHoDYwxyEQUJe3QTCaTqmrapmkabBuDntiS3K2bcjqdXru0d3T49N2f/OSVl1/85jvfeOXFG9///h/eu3fPsNbaZjadpWlu2qqcj8uizbLs0f0HZyendVlZYE3rGuuIszzrjcdj7/10Oi2Kom5bAMjzvKyrlZZoeOwDAaVt27YxQqg8iXvdXCJ4rQVnRrv79x+enp6/9D9+qd/fePz4sfewsbGptfNwkHWEjFIiynoqnXMurPGtUDEJxVmj67oqxqXXU1OkQgT5fXly96Mb25lH1LODMc0YY6mUu7u7TPBpNfeEnV6mrQl8qGyYJBM5PZt2e1EkL01H06dPD6xzAblHcRSppKqqeTUXgo3HY7WzYzQ5axHZdKbbtt3d3R21c5UqC7ZpG+tNquKyru49uBvSDfv7+3mStroej8fG6I1evxsNRqNRFqnZbDabTZqm2tzcLOv50+NHAMAsY4IX0xYAFANTjHiWA7pWV3Vdc85DF0pkFHI8uFgveNM0wbb18jxYHnLApEySJI3BGKOUCm0PgzZ8MZ+1bdvpdBoj07zDOYeqiGO1vbeZ53kgXVvSZJmUkQe088oYzwRv2nEUJSG4KpXUWtd1G0L0jkDrUJfPlu4jxCpjjHEeOYfo0bZWSrm5MUySaGZmpmnBO1TKWF2XlTHGSnY+G6um3N7ayZIkvLoRj5QSxggpRFWWbduGlEQep9RlSi1IJ62unXPkHDk3mk72d3Y7edQ0jdFWcowRufecL1oOeqAoyVRPMcaCvgAieXB124YSReKAjKARgXNhAjfTeau1bozkvJf3ZJxHshkONm5cu5nFyXg8jjrdvNv1QkZx4oHNphUAAqNyVhORcc4vTSwiEoPWLJozOXeRAyIfSoaICIMUTPB+AaBtDCxjNhcMBoAk4St7DwDGXGCLlYXwa1t4M1dsklWoIFKh6yCHNQARoAwKiWvdBcOHPFIrVLGw8YwholkiVwpGGsB577znXKwPac16EeKi1I8t1Yit88AUXCg0rBnCAAgWERd3cZRnexms4M7Kyq5jI0RETYIYLjIEzrT6+ODw6cGTj97/4OzkdDIdCWSMsVgwC5yoJXsxsxhyZuQJiTEe2AlMcAAIBFtAANLk16mdjEgsrRFjjCMy71C3TrdOSa8U29rYvHLl0vbWnpSKAYtVpCIhGAfwBKDEWvQMkRGw0LcjQCUMSlXktGGMYaiICSAsTCwPe+ISSHwu8PBchOj/T3Tx7O4sHJ+vsynJhVkABBKkItXOpl//6jt5lBw+Or47fgJeConokgd3Hl3au8wAm0bXc5dEG9NxkWxV48lo9+pl7+3JZCIE89b96q//+q/8yq98/w//+Lf/u386rsaPTkZZZe4fnI4r/emth6++9tb9x4fTuh1PJxvbW/OmmsyncZpQeNo8Wm0QUSATIGy06OppvS+qmjGGDOM8c9oAgHXOWhvHcZ6nomGzmfG1F0Lsbu0fHZ4xDtnGxtx5r+ua+d/9/u+//uZrl6/tfXZLdZO9gydPu3GsRFrWlXOmcFbrVkfyqa3/+x/9WdWNh4ONTp4KZHkc94Q8OR9Np9PiZPrKSy+bpv3qq29+8OOfNF8qU5L/4r/+J7/+N/+D/9nf/g1E/PjjT+tOXtetOz3862+/8eiDW/28fzY6b42hNLXexVEGnLXaDgYb1nrOeduamEf5ICuK4sn5nWHKz8+ORZJ5mR9M2iSOZwYuX3/x0k7/D37/d99/9wdfe+edv/E3f+2Nb/zCp4+efHjvszfeeAM7nftHR71eb2x0ttE1ip3MRw+PHgIT2vmirLa2d7mS8/nE69QhGGta52QUAbF5UQNg4NUxBCRjtfGOGGNJBGCkMeb87IwT5GlWlnpruNnf2r/16WfOx3/0p+9a7edTPY5n+7t7I5hA2qsIhru9uh2dVp+aaJRuTZsaxGwyZRwBIFq0YOdWm6rQnU7W7w729/aSKD0/Py/nRb/bG/QHx6fHD+7dFpEabGxEqRpPJ8Rwb//y2dkZZ4z1sziSEiNE5sGpiTKklZLWWm0aT9Z7KyUXkSBybVs3TeWd63Q6WZ5wgU1bCa402EUPXGDOuWJecmQcWZqmZF3gRi0UVT2Mx+dtG4x62+3lnU4njqO6rrMsCZbekQ04K8/zbrc7mtdtW6OnbpYTUVNWjKDf7/c73eBq1EVhtBXImIok4ycnZ1mW5Wm26EFAUNd1XdeI1bA/IMLJZAYA4Elro9txnHe1tkIsTReRsW2ra20a5w2B51zlnQRZvyxr51w7a0Pamy2TduEHpVRIVzPGEBehJK01Yy7IxSilXJZ2nPPet1ozBm3bVnUVWRGiESqOVBxN6nIR91Mi1C4CQOikERofBwtH5I1tnTecK2OMdi3jC1XHpqrqug6Jsapqqqpo2zZSCrElojzPw0GY4EGagjFmydd1SeQ8kPfWOqO1bk3jnANMGC3YDAwxiiIpRcLjKErybreTZZzzreHm3qV9pVSUJsbDqqB05aB4IOdCb1Pv3FphAls48PBsyoCW8RUAduHZU2g20cKzZXJhSS6KYnWE1bZYu5fuJq3F5FfxHr8Unw77aB0KxP36QVZfX6UkQttxKSXihQ0IB18F+dc/XDn3q89Xx3zO234OEDzHLfhCj/mZ7dnPV1P9+X9xrWaSIRK5ttWjs/MHD+8/ePBgNBoV5cxay4UEAGSEjBAZfVG56ReP5Ods4YzeeyH4ulR2AH+7u7u9Xq/b7QaludVd4JwjEuOAjHGE1a15Luu0uuTVto7J1mfv5w4bP4cV/n1suEwDff50UZ5FWXrz5s0Xb7z00YfvHR+fdfO4qhpGUJZl27bgvHW6bcz56engUv9Xf/k7QogP3v+Z5IwBODS729vT8SSJ1bVL+2fn04f3H3zlja9e3t+7e/euUmpazJM0ZZNxa/RkMtHkxFI/e308Dp4fHq1RcVdvNCw5LmFVZ8Q554C0qpq3xrdNu7ExPD0+/ZM/+bM3Xv/yN7/57X/yX/83X/7yl3/2s5+Jfl8wDgCccVDKWtvW9eHh4R/90Z+89dU3bt54sSpm7x8cSs73d/e2Noem1k8fPZ2OxmSd4JGzcOfewz/4wz/+6U/e+/rbb//CL/zClWtXvYP5fG7JT+azLEnv3LkzGG6kadpozSNV1TWTgoAVReEczWYzADbo9S9dunT45OkdzqMoMmiDBGrIRLdVHULUs+l0Mjr5/h/8gbX2q1/7+t/5O39n8KMOAJwcnAzy7o1r1xkT83k5Ox/3Op1ep0vAmFQDZ6M4Nd5Fgw3OIu99WWpjNGIU8mVKqbLSiGxVVsUZE5JxzmVXBqOZZZkUMk3T3d3dKEniNKlM+/jwgDFmkCqrZ7qOutn7n/zx3Tvw9W98eWsnj1keRf72LbhydSA4USC6SymTOFFKad2Qtjsb29evXHWtjpgan4yK8Zy2fCfO23zGpGiaajo6yWyeJxEhVPORbktkrfcgIp6mkRCqcf1tM7zz8RNCb71pKogim6ZxlmWdTnb58mXGWFvXgQFbl5XWDSKBR/DIGcuSXHQE5xgpQQTn52PTtuRcWZaRVFxgCK2cH50wxqqqstZ2+z0id3Z2UpZlf7gxnU49WRcslWml5AB5r9flnNV1LYSIopgx5pxrmubk9Ihz7h00TRO0BxgTAJAkabfb63d7nPPQFr0q66qqhsPhYLBhjJnN5gDQzTtpmgVSIRGtqG3OOWuNtYZzSBKFiEkSpWk+9N3JZDqZTOo6SZIElm0OOOduWbkechyBnBXMj9ZaRRwQA18BEdNO7o01xtCivzA0WmtrkSgYrRYswELOwTrDOWfIAXzb1tZaRApZpNX66L01xnjnmAsdClAIEZpaOefatm1bzTlL03Qpnu8Y84hu0RFM10TU2pZxCGSAUGPCeYTca61Rcu+9bqxuGwAmiCspBVNCCpA86eQbGxs7Wzu9wUAgkypqW7MqCw7ZfWOMcXbR6XsZUVisRAhByC+YM1yR1REDq/QZbWUKlQ4L8qNdKgmu0tL4bND+OUuPS95A+GTFul+Z/xWfAxEZX2uczRa9oEJp3dL8wDJo5FcIYGWG1/HByk4szrXq1XTxOQEAX1NuBkQfaCqI3j1Ph3zuyOumCABorYID1qwmu+h9evEnRHTeSCkF48a4siwPjw4ePnx4//79pix02xIRgHfOr4I3nv5dx7M6z8rNXk3dyoQvanSZCHdNCMGVRMGJoXY2iD458t57FnEMba8YAwbAWNC4dmuaCqtbTGvk1nV8gIgelkrSi1LGMF3P2m9cDvzf67Y+jAsQgwxAAHObO3vXX7yZZvl0Omfge91+W9ch4T2fzK02g343TZKYiW997Z22rR989olAZnRbzIv5+TgVanY+Jm0lwOnhQfz21166eXM8Gs3K4tatWwFwKKWiJI4FZ1KMphMEDDTn0ADp+dtGhEtAsDYpQEvyHQAopbwFIkceEAlBMIZExjmcz+pIxZ9+cnt7uP2tb34DiM3nZZrmFkAsOMIkhciyrG3bVte3bt2ajic/+dFP63LeVLXiIo4iItocdl+4chURIy4mo/GVvXvjUZnlGw8ePjof/8Ht+49+8Rd/cW9vD4UoplMC+N73vvtbv/Vbk9E5SgUMr2xvG3emrauq0jnqdruzyZxz3uv09nf2Rqfnk8mksd46MNYzlUgpBSdj2j/70z+N/to7Alm/2zPGfPDee93+AIG/cvX6ycnJFLmUQgEzrUm43O5vqDienE3OzsdVoxlD22pE3N7ddshG55OmqVYpNsYoz/O6KUNbYoaAoV0LheATwlKIyDjLGBtubRZFodKkaZqj0VmSJDKNCqf9dBTH8ZWrlx4+ulMUs93d1JNhjMoKdrZ6Io3ikBwyTcsIJONZnKVx3FRVXdRt1TZlMx1NwTFwjJF4+823Nob96Xx26+7t0XhclEWpG+1spz+wvtXWEkoPkGRZlKut/c1hZ48xdnx8fHh4SERCcGP0ZDZJRnEI4Hc6HdO0o8kIEbv9rtPgvZeKJ1GcJAnn6MmiJ2O09z6wc32WdXt5LBVZF8USEY3laRYPh4OiKKbTMQAgknPGOcOlQMbqup4XU8ah099OkyQEyW7efPHGjRvj0eTDDz98+PBhHKeMsbqu67rlnGeZSJKEsTjPcylV0zRFUYUGo71e1On0BFdNrb0LCnSKMUaEUawYA08ekTgPfQqRcS8kRHEMAJyjkF4IZayq6tBMb6FkEG5EUVTOuX6/HzzUABRCuitJEmQiiiJtTFVVjEOeZkwKZzTnPEpiQrBaB0hRt23btiKRwb8KQlXeS8ZcqGnkHNM0DjsH2ME5N9Y674DIWhf0m/M0S9O0qiqilQKBoFAsD8AQhRBKRYSOyBE461rr2hCKFYrHSR78OWDonGsRvXVlWc9mM9MYzrnkkkOQ0ydimOadNOswLj1AlHQ4b4LxNsZ4WFSWaqOFWICn54ACXQgWXSxVq5U07LwEBKEUlq2AQpjwYMMCQ37dZtOSfr/mkl749Cv7YZcinuHOkrfB4Czzs5zxRd2EX5ZHwjKkD7CQS/w8Vli31l8YKvj8r3+B0V2PAcCz1RPPecnPudHrU/HcYcPOum0BAJibz+cHB08fPnx4dHQwn0/BeSIXTmSd1roNcRfCLyhewJ9fzbGCBetDXb8itix8iOM4y7KiLKWUQftE28gHegeidY4hEQNPxEPehYV2mg6IgBayVEDEghQ5AXgC5AwZhJ5OjMGih/tCCIsg0Cn4knL4bB+IvxxKw3MPgLOWCxbSOPt7l26++OqDe/edbRujB/3BjReuX7t85fGjR7qse3kGOQw2evV0enR0oItKRFEzL0+fPv2zP/r+2197p57P40gi5rdu3/9X//JfSqUY4y9/6ZXT8zPPSFstIglAs2KmjRGRIgy8F4/ICAkQ6HOFM7Qk5Hq/fKiWT1SIkrZN670zxjImOJcMlRRgOU3Go8v7e5Px5MMPPrm0d+m73/3ef/c7v3358mUnYytE3Ta1bpEzuYxtSCmFjByBcdhq23g9n5fW2u1+H5zQxjx4cu+D9z68d/ep5GJ8dt7vDQfDYV27Dz/69NPPbr/++ut7ezuXL1+mBt58640nT59OZnNLfjQ+H52dl1XDpKqrNkty5zx6KKaFtb6t9Xw+Rxl5YsY5hosuRYElA873u72jw9nW5qYH+tM//CMRxV957cWbV1+4srk/Go2m01kxL3vdwUa35zxtbwzbqp3PT5EzBsx451pTOj0an43GZ3GUAuSLnKnToWk154zIeQoo3HrPqKmFEN468BQEDJng0/mscQaVaJrG66abRIVuRsWs0+l0O0I35s6t28NhkneRge93QUVSBKZ6KAUMVQCDwSBJktFodHpyPp+VWZZtbe4ON7aHG9tKJknS8SSHGzu/0B8+Pnj645/+aHQ8SrLMJZaABHAUAoLiL5GKI84iAEiqKKsSyXi8MJY868SMpUmSKCFriVs4bNt2Vk7yPEeQ3vM4juNEoietrSUTKRErmeVRGsW9Xies41VdOttqrRnDbrcbp8nZ+Wmrm8Fg0LaN1q0xmgkmpQCIAcBac3JyEnrLdjrdzc2Nvb29JEnOR2dlWYYut0Q0GPT6/X6oun56cFpWVFdYVZXWOoqifq8XuplZp41tszyJogiR6rokorqZSclVJLI8yjuRVOipca513nChiKiqq7ouhRB13bZtvb29rVRc1/VCIZuIMREKBIy2AJAkSZ7nQZVBCFE2LWOsLEvrHZDjcpEar5qFoiJyJjgTSsZp4q3T1DLG4liJRa+KhXlLs0RK6axHxPl82rY1EUZR5AmM0eBpxUuXakFU5CgQubW+rtu6bIJbLITI8zTvpDLiQoLgKBWC4M4ZAh8aEwqJaRrLSAFAyzwDZowpirytWu8AiYEjzqXiSRwlUZQIobwHxoSU0pk2oARalPuvahNaWuMoLFYiBH7RpWdtAf1c1WKwNUQU+IvBtIcZDjAiTVNYs77rNum5z3HZYHD1J7+U62aMBRIlfqEWJ14cOVQ9rLmyuH7w5777jFnl7LkhLXZdajyFxXpl5y9m51kD83mrv/7hOmi4GP/aeFbf4px574qqOjw8vH3ns9u3b52dnBrTLupgyVmnlx0xaJUn+nffnoOAKxyzqit2znF2EV2IoihOkiRLQyYu1JoTA601MmIMODLHgXPOiAGAXLULXZSL+iW1FwLGCH9CZERBCQqXsCDcEVxDCV8EF/5ytgskh4HHQJ7Y3qVr3/5rv3x2cvrZJx+QMVube1euXBv0uo/vPyBvB/18OBx2su5nn358fnK60etJwaxpNwcb0+n00cP7r37p9f0rV85Hk7rRj54+9YRf/uobQnHOEcA7stZ64qBt2zrDUXgABGTIAkRAQL72DuICdy3fvpXU0nLkC+KzcsYYbcgYw7kUEhkqpaDfE9awqmo/+fhOvzv4h//xb97+7PZsNuWIyLnlQoO21i2BCDrnp9MZEnjvhYwFY3VZzcvik0/vHR2Pm7qdTebTwk7mOk8Uk91pYfp9Pqlra4+vX7/y8s2XOeInH3600R3+xt/+9ZPzsydPD0/G49F4KrhsjZUqvn//4cnhUTUvut1+W+v5tCjLOut2VJRqYLzWFkKPE++cT6M4TZIXrl57cv/OzReud3u9f/57/zJOsv3ht1577bUs63z88cfvv/8BWhCAJ08PtXMccDjYKOuWOMZpPp3PxuMxRZzIJUk0HG5sbW7VdVtVdShBD36Ic9aY1nkI7CaPnohWsuVENJ1OR6NRWZUhm6yXW1EUAIDUpKo3Ohm3pd3Z2iwqs7PZPTk8E7WupZTAgRh6otaa1hrulIxjS8SVipI871LTNKPJrNH2zz94N0mjfr//4ovXr79w/Tu/PHj//fcfPXmELfNETKgkyZRMvUdrLXoQiiuldvnOcDjodruDwaCXd/r9vnNOcuGcm0wm3vtup3N4ePjDH/6wrGZkHeccCL0lIodopYAsU3mqOp00z9M8TRmzbdvqtmLcI3NJKtNMGdsSmU4nGQ57DshTLiQ48oz7WAgAYIy8BWd1mkRZkp6dnB4dHJZl2bam1+3M53PyvtPpXL58udvtTsazw8NDAOecdkQEVkhUERcSkfl5MSfr2rYVQiD66WwSxM6iDgKLUtnLOyLNOWCrddvqEhGdQ+990zS6NYyJUEy3s7MTRUkACkmSMMbKsm7bdjwe86WucHi70jTt9XoDxhCxrerBYBDy6/PZrCzLRrfgSWutTcsBQ3UiCCGQL82YD5WlIWjvvUVUUgkVyVB0johxHNXa1LVBgDhRG4MhERXz+Xx+EqkkSFgrGRltm3aRdQuDZByUF8J6qRgy48AqJTx5D23TwGTimqYSMkJEkBRFEecyFkxmkWSSswg8MhKcyyxJOKDVxjMSSFZCqFwNqZwVUAiW2K/JHtCSo7CWAoDlDxTqr4JNeS6iYEy7yhQsaCh/YaH8ule0Mp+w7HABS8d6lb8I9nq1LAIEtWR0zgGnVXwClwrNK9u5Dhk+bxUuPv85EYXnLmEtYHARS1hHA5/30Rd/fdYOfR5VPPenNI3rup7OxodHTx8/fnx0dNBUtVLKORNabDnvvPecL9uFOPuFs/3zbsFzHI7VeIJWXbg7bFmWzBgbbA43hsNOrxfKHBz5VmvyoVCZJEcfUuIAPOSb2MXc0lr6fPXJ6mFbwLglYlxKHiwm7HNwAf5qEIMnAgQADlx0NoZvv/2Nw6cH8+m0mIyVituqPTNn0/GIc9zf271+/fr3v/9HD+/dd85euXqpLqvx+DxNU102B0dPdy9fao396bs/G88LHgvBxGhy3t63/eFG6PJVzOcxgyiK0m6nahsEJARCWKUePAJbPmyMsZBWC9MYkomwrBlePZBxrBhj1pHWQeKWGGMIKk2i+WyyMdg9Ozn46U/f+8qXXvvN/+k/+N3f++c/ef8TIvJAyLkQnAABuVKh6Z3RWscq7vf7SkTT0VgbYlK2FrXnnscyxZPz4nF1XhXll1979cc//SiR/D/8W7/2nb/2XUXig5/99MMPP3zp1Vd+4Rd+odPNB8MNAlZrYzyWdXN+Nq6q5sH9x22tRV9yJkdn43JeKaVkHCkRMWm1RymlYsRT0VT1dDTe6vfRUy/vvPnmmycnZ51O78qlS50sY8giIdMoNnnuCKuqaq21BMDQkW8a0xlsdHivqptCl0Lwfr+3sdHv9XqcV95TKGeTciE26hf99rgQTLPWOCsjpYNssfNPHj2ejidRFAnCmEvgMuGSMQ8q7kSJqYQSPaPHjx+dZWk8Go8j2Z9NRoLHCjlnRAIJkcs0Js61d5VueVkVTQu8Oj0flWUJAHt78UuvvdXv9yMpN/rda9e+tL+/+7U3v33//v3Pbt06Ojk+PR+1U2sbjUxYD0B80o7jeBnfHunZbLq9vR2lcTfLy7IM9TBxHA+HQy754fHh04d3kCEXwLgl7wFISpHGKkkUY+RdbQ3M5k1IRDnnIpUiqiRJlBKNroebgzS9lKbp2WgURcNOJ5sV86DwGB7NKM4RMYoi6/TTg8cnx2ehvebu7n4cx2madru9LMuIqKzm09m40+sSkdVmSR0w82I8mbrRaCSQBckBIURbN977brc73OxJKdMMhXLWlU3oCkEkuLBWO4dAyJggzxmyOFKz2SxNfcB0QQFGa13XdfChAcAYM51OV+WUKs8BQESq0++RNeGLSZpGSgQ5hKaugixjiDQkKbfWheAHLJv+GePm83nbtowJrbWQPM0SRFQyckipjzmKJEmcN1XZhNsExMB7KVQoORNsITqZ5Umnk+edmAlAZpBpQEbgAR2iZwDO66p2dV1jCLkLn8axFAqAcRQsyqI4UjKKoljyOIsS5tFUGsBo5AwaTc3KtFu/4EM58oyhX9vCKkkE3i8Vfy98dAJaENwW8iMrSuTaxteUE733oel2+JVdiCjA+s/rXw8AZWXDQtCDMRYpQYvWz894V4wxfyGKElouBEbFxTFhRWv4HCnsuZ9XeOXz4e11swoADC8gy3OHfW6EsKT8w+fwB67lLGANuCBiVVWnp6dPHj16/PjhaHQWqCHIyBmHBAQeFuEWAiBP9vMHX03sFxpC/NwMrF8pIoZsZr/f73Q6QUE8RGWNc6BbKSWRd97EKkIky5DRggIW+nGHfpEBTi2eHQLvA9TGEIVae8AIuFgOxiGyNbgQRurX8MG/zwDDF87A4umlQOpl4HFrZ+/tt78xG43/+A/+TUCow97wxo0b08kYyT28fxeABpsbjLGNjY0JAy7Z5cv70+l0uL3T6XWxrcfFTCXpzpUrWzt7BwcHTLIvfemV8XysEtWMGqa5NUQNqDgOFAmi0AOVGLJQS3PxOC3poiHUFl5qzi6qKL33CReMgZTSOUJg5BFQcAZGa/J8e2uvnM2nk9nv/M4/fe3VL73+2lf+7Ec/nc1L42zW6WSdnKFkDKVU5+fj4XC4tbnTNM352bhpdKyi7d39qihQJZJH3DMOajorqnlNho5PJlLEv/Y3/vp/+Lf/tiT7j/+r/6/V1dtvvvl0cvLuuz8ZDDcnszkB29rfJ2Nmk+nh4aG3pJs2IFSGeHZ6XlfNfD6PPMgYQiWKtTZJVCftxNwfHx5lYg88PX746Muvvf7XfuGb2vrPPvvs4cOHG4PNkH0+PD5O88727r4nODw7s85Z78bTicrzJEuZksW4CJlu51xVF2VZVlVprWUoAilhufJbxqxzwnpXFUWe51VVcWSc85OjY6VUxEVbVrGKtra2sjg5OztDMAmxoylZB8Pe5fFpO9k201GzvTPY2+kLFUdEpLX2QIIxLkSQTKlbLaOWgAFDLlVvEO3t7X3ta1/r7V5J09Sadnx2+qd/+uOymF+9fOXtt9989eWv/PjHP/6zH/75o4OnHhomFJeCC1W39So7G6za0Lk8z6Mkefz4sfd+c3Ozbds7d+40TZNlWX/QCUQkJNCmsdYiQyGBcdC6NqbUpgbwptWMsSSNptNp2L8sy9Fk3Ov18jzXWs/n88B2DmIUYR9r7XRc5nnOCEK9w6W9He/BGJMlkeAdImqb5vGDh+FV3BxsaDTGGG1qbVpEdJ45Y5umkVKkcRy6bDnvszweDAZ7e3sb+9IYY612vmlKY61lTEgRee+NNs4hghBCeccAkKEoy5IxscqRO+fqup7P57AI5PJgtILVSZLEV9WaQ0NOmyBNc/ny5aABrtumLMuiKCaTyXQ6BdSwNEhCsuBXca5D/af3ddNov9Sccc5xIbrdbiQVIg/deiRXvV7PmsWbLIRQKg4dLrz33oFz3lki9OQ04xbQOLSAjjHgUjDki8URiTEOTlvdOmOdcQicEs89QwWb/Q0pkkjEHNmigbCzxnsrFk1AnHPWLUULgLS2/ouqHlZkRr8mu0Qe+ULab610wgMRxbFa8RKC2FS4F36pY3HBLViWKa97+atTr/ud65/jChcsD4VMhN0sLcoyiQiX4ktC8HCx4QjrlhjW7DQ8G8+gtfDAOghYDeNinPiMRPTKwDz3A3wRmFi/OnyWnLEa4dOnT58ePH5w997h4eF8PnfOBV3v8AAALZ5bxCCRboSMPm//fh5K+As27xet/OI4DpHLfr8fmDFB+cc5h4wcUMw54xAS6ohI7JnCFsaeKSf5eYNZQ3MrXsJiIH8hVvj3tq1GtX6DiIgJ5h0Bw0BXEpF86eYrxWRcz+emKQVHRMyyzNm2aaqzk9NK243NoW7qoqpq3XLGQreOo+NjS/7l115/8eWbrbV5t/+Nb77z4PGT99//WafXreuaiOI43tzcrK2elwWsngpcG+Fzz9Varg2WiukMMTDKvfdaax4nCAwZiSC9ypgQEjx676Mo0doaY/r9zuHB+e/+7u9+5zu/tLW11bZtMbKMlVwK5JYJLgH7/b5xNBqNAFiS5Z0O9x6sNlmvX1fNdDav6zaLO1EGQAKdv3X/4Hvf/vqv/NKvZHH2/k9+8LMf/eTm9SuDrKO5m06n166/cHh8/O577/U3d46OTw+Pzz759JZSqdF2c7NLDoy2s9kMPdVt45ArYg542VrnnEsj1omMaw4EfenmC1/96lfjKDo7O+NS/ds//KMnT+9ubm5+5zvfvXHjBr9z5+7du93+4PLVF7Ju92w243Gyub09qcu6bQz4umk3NzeSJBNCRCqVMkLgjAmlovmsJCLv0FrbNG3T1GGxkoLVuk0xt9rEWZ6m6Xw6297cbKq6mM7EYLDZ7ScqGh2dtNOisvT2W98ytnz9yzcOT25v72afftI4S/1eX3Q6W2VZGuO0JisYNmDJMc48y8ZzH6e93vYVng7zvPvtb3/bE9UlnJ0eN01jTPvkeHrr1qd3n5yQSl577dX+7m4+6NCBiZSKIrCulsym2aZaaLBoIWMiGk3bT249yrKMyRyIzmZtkiRRN2thZnm7d+Xluq51WzvbcOcZh4izSAhdzEOynIwOU6C1PTudcKGiSDVteXJy0ui6108BW20Kxt10dgbAzkeTJEmdo7t3Hm5sbB6Nzm/0bpS60E3b63SKct5U9c7Ojm7nlbWhXJsQkTNigALduK3L0oPrdnLGWFXXHqnf6SqlimLW28xzE0dR9MLVK1mSCiHiPhwfHwMyFLKYleQoieN5YThI7xl5LpAj8lAWTki9Tuq9VoInUax1Oysqa32WZU2ry6oVXEVJSqwlD0JlRWW0q0O7rCSJQ1zEe8s5yjRyjDRpkYntjc1t2CyKYjaborPlvJhO595756zzhgEH7ztZwlFUVTWvZtTojorJQ13X/TQbDAZSqclkAjIa7u/HaSKEaJoqpMONadq2RWViBYyxphk1bsZMnMpYRRzAG6ut1pFMyJNvDS0rAxE8gIF43moveJzl3Uhl5NDYlohX2nREjjJ2wJx3QIyQPPqyWYjwrJckwDJ6yQA8eWetW6rH+5VTTR6WJsJ7AhIA4D2sjuO9J4+lM8F+e+8DTyVkIkJ0JxSU8rXGS03T4Bc52QAQcAYt6fewhh7WtwA4nHMAF9ECWmrOOLM06gSccSYWNReN0YjIF5K3S24mkbi43vDvwiZxLpcTtYoNACIa7+AiqbFYxAmAL1lg4aPwHyJytypJBc45Q3RE3jnkoXBjwbG2NgjB2bt3Pr5/997Dhw/DRAkOzpE3BiAw3QicJ6KQhggpoXX7t5rbnwdlDCEAASIL5RLgESwCjyMJyIWKVdJReV91hzzvg5QoEkPoUcZxJKXUhsi3QjLFkQnGUEjGw0wgEAJZEBzYItHuPQAhMgBGBN4SMAJGggFjCNYReuskAAXowxjjXDAWoBjCsj4wTP3yCu3yWtfRw+eQxGpWfg4FkrEvBh/MAwGCB8YFE8I5w/P07V/8aw9PDp88uOd1/ejs/M7tz8Ca4UZfGzvXZnx6+s1vfiPNsx//+Med4XDv5Zfr+3edc4/Pzg/++E+sta6uPGd/+nv/rKqqg9n8//WP/nNgOJpPuZKNM5WpeSSiSMlI1XXZto1MEimlMxYAirLZ2tpCBpP5BJGYWqiUIjGG3LXUWCdRRCxhQjDAsqziWAqRAFhL5JlvbeONJe87aaI42xpsnh4ebuSDj3/82a/94t968+Wv1OdF4Y9iEsKCZELxyGi3sdGP4/js7MwYx4FxoiyO51qfn8+BSILo9RIOqM+PzXmVx/DOjY3/5Df/9stXN/+L/+K3jg6fJv1spvWT0QgTMRpN0JjtTrTTkfc+/em81uPTcZ4IlSVR0iHrj44fvvbiddfMNiLL4k0hFTiKOKYxa4oCy7mI+5Kzmy+/1N/ZPR6Ps04+0/YPf/9fnZ+fe8LucGf36hUv2GBruHdt59ad22eTJ1n/hdaMnx4dARfDjfj23Vu7+/vTyWQyjbvd7v7+PhfQ6aRNU04nJ/1+X+vCe8+5FELESnmtrbWI3nNknCsZdzv9g8dP+72tbm9zMq2zJEky6HT7yFmpK5T0wsvXbt588bUvv7Ux7MQpSPVGWZ+Rs/fv37Y6EiFMzTkHsFprKVWv1+t0+4wJRB4lyebmVhRFjInRaHR2fv7o4RFjzDptTDufz5tGn5+Pfvb+h3mexol69UuvE9F0Oo6iKGSbzloMIn1BWINzHhIBRhPiQmzAu5YxU9eGs0i3ZVGUZTFHsIohR2wbZ/UkjSPySLDQD/YI1npjnJAsRIkZY0rGRKS1NcZtbm56D96DdcCY0K0FYMaYX//1v7O9vf344aOPPvjw/GzMGDPGjUYTDNqLIY4khVDSAxlj9i9vFkWhrZWRQg6yTQBAxdJaGyVJmsXFdBZJNRhudvPMOTdvTshjcG6SKGaMK5mSq8OaKBhXKuJcWuPbtnXG6sZxKYLsI6KRVau99p6xReITGBDnHDiEdgDT0biqCiKCSDHwAhkxLgQbn50DeMZYFMtuN0/TNJaK0nxzuOGta5qmrmtrnTFmNpsVRTUcDqWQSilAP51OAaDf729tb3aSrNfrKaUixTvdNMgwE3qtcwCwtKgLICIuuZRSSGjbuqoLrdumIS6IMYwTRRSIRRYgiAqI4DNr570jIN/U2pkGQcVCCsakiFZ8tNAiaT0g6dc6aCwDBgsb7NfaBF8ss2tBhQAUcEEdeEYliTx6Z1c9NXDJgMNllgGW/LiVAXNL9dnVicK/682lYM3jDN3GGfehBtV7j26BJPAZI/DMAS8SHEsqxnpEYaU9gM9SHcP3w//9sg5t3eNc5WhWh1r9df0a1/exNiR9llVty4yPtYZzbskTUegnXlXVkydPHty7f3x8XJYl0qJAlIcneTkhi9wBXtjC527cF9q/v3hDxLquozgNsmmh3kRKGWg9KhKRioLbSp7AM08Ypel6TuciQPBzNuccLXsXrbiLRATcI+KqgRaRD5dI5BH54sahXws5/KVvFwKbF/EeHprdzMdn6Exd1/PxqCrnHKHQlgl+69adS5cvX3/hRRR4enoa+vcW04l3bmdr88YLL4/OTj/+8MPZbJbtXh7NxnXbNK2N8khAnnIpk7hsau89OYhkjMB1a51zSqnNjZ26rKbzCSLm/Q55p5uGcx5wDuccGUrGAQi8c0vhMgAmhEDn26YFYlIIqaJQJtPtdsvpVErZaH337t3d3d26rrNMeqKiKJA1Wzvbg8GgaZo8z6WUnMskSSaj8Ww2IyKpVDkvIqmAw3g0stYMt+JEihsv3/j6N75x/8G9g+Oj0WT8jW98Y9DrHJwcXbl5o9Pt/eS99wW6K1evVW0rxvO5dtPTqQfWtBqIkiyNs1SDTaNuNzmazcamrdMkjqUQnPZ297782iu721uvvnzz6uVLB0+e/vjHP87zfDKebW3ufO2dt+/cufPee+/t7++//pUvG2dPz89G43Ha7Z2dnVV1vXfp8rXNLaFi49zjJwcixrqux+PxZDKZjGfBt4miCHFeVRU4L2UUVnulVJqmKo6bqiWC+WQWRcmDBw84skuXLummYozVdXl8ejTodV//yuuvvvzSSy+9NCtMr5cTVp6qOI63t3Ymk5G1WhwdHS1XW4qS5MqVK2+99dZLL796cnJWFNVoMqnrZjabCaEODw8Pj47ORvM8z7231pg0TdP0ivNmNpt9/Nmtl1568eWXvrSxMXz8+JGz2lp7dnYWcdXLNsIqGZq2hfr+kEqBJYfLWiuSqJdtnJ/qiS/KokFwIkuTJEJw3pmqbONESSmDwxYplWcRIiZZOp2O27aNVKIQOJNATMl4MpshcgSGyBG4lHxzcyvLsq+9/c5wOEyj7PT4rC5KKWUxm8+mRZ7nKouTNGWMRWmS5bkjP5/Ps6yLUhhnQ2OuDByXMooiY1vGWKKiJ/5xq+vWmbJt27ZlggsmvfdN01ZVzZlUnSzPOsfHp2RRcMe54CictW3TaK25SJVSsYqUip0rg9qIA2AcOQLjwAVGKBBRckbeRLHQhjdNU1WF1hVjLE6UEIlzLpQvWqcD2TusmEopEEAEdd0EYqAQqtsV3vu2rYmo2+2iJwDY2tra27s0PjsGtM47IaErExkp713b2iyPiZz3HCBCgYHPEUURcDsej4+P/XTWWmuC8QkeMpH3C7jAvbdSEudcJTlwYCiEiBlKzqI4TrM0DyWgwTI5bQGAiLwD662/qHRwK0wQ4OYXWEoAWK77ztGSCElL6ACr4yyAAtCFGHbozcpYMCorp5vW2gk+Z1BXsGAVMIDnrQ6Gwy5kqRhbNP57niKw9KSdxzVWhIclL1KK1ZFhvYTS/0VAYRU2uAiVrzzbL6prWPmpq0v2a0kQD6ENxAWiAmO0tyFHOxmPb3322cP7D4qisK1WSnGORGBp0UIbANiyYI4olM89c7/Wbx9bCvKsxrkMgKxN2QJKISIOBgNPWNf1wcHBvGyM80mab2xs1HXtvADnw01WkjPGpBSrGX4O861/sj42ejaQvtpoMarQXAuIkMiHiolQDgoQ4IKDZw/7l7StowQiCmKOCOzVV157dPduMRlz7pVS2jRFYTtZEsdpUZbv/vS9Vuvvfe97xpsPPnx/Oh1X1awoJ7PJdGuz98L1S71u4mwVx/G8cty287IoWy2SKBJ8XJfaWSDw6BiXXAgAZq32HrX23tZJEm/2dwk9cqjrkjGVpllblYBMCGCAHAjIEIEDxzExxiDyOI4FA2c8Ik/jhBGURSE4725u1vO5QFZXxce3Pvtbv/69Xq9HCFXTOPChMjaKIm0cIoa3P+gNRFGUpundJ0+uXr1aF+XdW7fR2Ss7e8NeN1HRf/q/+V9funH1t/7fv3U6Hc+ralQUKk9b5yz5Sy9cvf3JR508vvbijbPZZFQ1xJn1TnI01iCxRPJ53RSTqdwa6tnxZp5mO1tEpLVOomRvZ7OTpa+//npdlI+ePLXW33/wcGMwaJpGiGg6L1QcnZ6NAo36ypUrly9dLcv63r17UZq++eJLg61t5GJSVMiEipJb9x4KIZqm0a2ds5IhcsF6vd7+7u7HH398fn7OUMWRBHKcc/TGOZXFaSftnNfnmxvDk5MTRAz9Wvu9bhyLui6EpCvRbn/YiTMOshMl0hEBg0a3Rd2Mp3MAEEGwjIja1nii+Xx+eHgoVRyAyXQ6ffr04Pz8vNvta62n02kULTomI2NJmkvB6ro0xpydjpV8Gsdpmve3NrUxpm7KybQAw5wLUvncWrBWSykHg15RFM5VzjnOJWJIogNjUoqUMWkNGG0U6ljGSgpkXEkWx4t2DFLKvNvt9/tJkpR1UVdtXbdSxsiBCL2HKEqwKIHQEwiuOJdSql53I01T72E2K6bTeV21VdXkucjzbprm8/m8bY1xhfe+4yjNOojcOQLkUsXgHSEQB8aFkBIE6+QDjizP07KpT4+OR5NZpRrOeRx758Baz0gIlFXZODPqpr1IRA4pklESxaEAMpYqEpKcBR/IjLYoiroqnPOcc86lVEJwEUkBEpAzKblzLk0iIMeZt9bqtjXGlAUWUTTYCL0nwFk2B5SchcDA+dkIAdrGFEVlrVMqytM0VJnPJtMgYTns9wJf8uHD+xHzbb3ox62SOLKRMW1RlZ1Ox3vryDPGFCkvwBk0YEszL6vC+ZZzQMa8163W1to8zwCAMfQelnI1rfcy7vQlV1JEgkeIUoq0k/TSpMeYJEJrrdbGm4vOjYEu4J5txEBL8iCurd2L1ANeOM3erRINq44MzwAF74AJHgIkwTitsgwr3MD5hZbzyph93sysf7i+Z5C4DoGZhbICPBNvXx5hcSi2jG3Aov7hmeDBKqJwcS56Vm/gwvAvmi1dzEaASwzX0RV+juUAS5QQMBlfNgIGBAwaToAEC/2PsG81n08mk1uffHLv9u16XoBzUggpBIZyj9DHhBZH/jwmWP2wGsn6nV0fWBjxs/O2+GEymcRJlua9vNPLOn3O+XQ6tdZu7mzGTjECxIivdSRZklTCvXiGCPKFW2CXBOYyZxdKjmHgIaiAnhjzSITAQ8AEiS0gArEFxPlLL33wsGzjsag5AI6MXn3ty08e3DdNOTk7DkpUXGAUSYNCStnq+uTk5ODwiUc6Pz9tmuro+KCbZa+9/sq3vv3O1av7SSzyVDDG6sK/dO2qAX98fl61TQvw4PGT8+mMnCMmuBDOo3FGcOXAzWeFLqpLly5lnawx2hgDnkvknITgERAReViQjkKDddJNG6aanJdSbfYHDAVDlIxTa9I42R5unJ0cW2080L0H96fT6RtvvPEnf/anzjkRya2trTiOq6pKs07btlJKAFcURRzHGxsbk8nk6rX9JBLTcXX50k4aq3u370eK/R/+9/+7azeu3757+/1PPrLOyix9fHw0ravBYPDg8ZOvv/XmzVe/BLZ1BHGaEzBgQsRJp9tvWmuNcdaenp9Nz042NzdevjbsdDpNow+Ojvu93ne/99233v465/JP/uRPHj9+vDXcvHz5cpb3s86AYNq05kc//cml3T2tm5dffvkP/+j7N2/e3NrZPj47PT45ff2Nr7z2lS8fn57fe/T4+Oh0uLu9s3fp0dF5lKTe2LDOJGnqvRcMX3nl5ft379ZFwYiUjGLOnXPzyVimmMZJoOfO5/NepxtF0WwyMkYb20SK5Xly5ere/pWdKJHj6Xg8L6yr62bCRFtWo48+uzWv25deekkEz0wIISXMy/Lu3bv37t1jXO7tXVIqZkLM50VYo2ezmdZ6sLnlvNXOci4QsdXWOK/ihAv19ODYODfo9cuyYIz1er0Xrr9qH5yYxjVlEcexZJHWWtdW8ZiDJIu6sWSRMeZ0cCUpTXrDDUuWVeU8FixSUnD0TudZqpRiHLz3gstI5WnSy7Ls5Oy0KCrd+iSOkIOz1DStUpSlXa11VTXOUdu0UkK3G2vtP/rwk9AUe3//cl2UTdOQ93Esm0Zb65uyNsYglxvWS6UEV855ISQQt97ahdFrXOW9A4bU6/V2d/a98Va3iLi5uVkXo6Zs67odDAb7+1eno8n5+fl5fd7Ne0geQzPS1tZF6ZxTKiZiRjvvyrqui6JorZFSchYzBrEUSsk4icI6xRhrWzudTohIKSmlQIS6KYppYYyZznrdbh60H0LrXgDW7XaJOEMWRQvaP+cciUJVZ+BIxnGsuJjP50U50Vrv9HpBnYYJnjuD3mlnjW6KubfOWbcgiHAhgo1pqXTOee+QEefImHDeEPlAsVzQUv3K5kpvJRdxHHWlUIwpKZIoSjmXzhIu1Se9WbTZ9Ms2Pyuy54qmsLLuzy3xluzKI1+rioSl+ttCIXHR84kYXuguPFt2CLAKA8CSSxyYEGGfFfkt/LqekqBlZgQAwufMuVX8LHRK5px/3sVERLFsH7pIEyypds/Y0We1mJ45xMWvC4rlKoGy/MoXxGBwrbZihRLCbKtoSd5EdOSXDjxap0l7ItJtfXh4+Mknn9y+ffvs9JR74KF4zoP3Dj0xWrITPjdafJYzuL7POlygNe7kekhhPaIQ9uScx3Hc6/X6g0GW51EUxXEcxyqO4yhSfNm0nRqXKMWQnEPiuCRmfAE4Wz0VjCEuu2CvTocLgLAkLVIQshQEFhER+dJsM8BVm62/bKTgAQkIl486BIVZQH79+ovnx4fT85O6rq21CNC05aScqVjt7m4Buh/96AfEyDpNRINB/8Xr17721psvXLvy0fsfPHn8cG97Zzqdes2uXX9hY7iFn3369Pg4UqrbyUazqYok4wKEKMvaaNvZyInw/OzMtfbo6dMkSQabw0G/X9RVUc0tJyUS5wKJWIN3HBmXTHI+LZper6e4sK1ORLQ9GDLgk8mkm+a6amKpQg25B/IMx/PZD37wg1/5lV/58OOPpvN5qDBP07TWrZTSGLO5ucmYePjw4cbGxt7e3p07d4wvkySJOI+6sQD21S+/+iu/+EvvvPO13/rP/1EURVs7e0+ePNnZ3tnYHNZ1fXR23tb1yaWT17708sGj+5999lmadfb2LjmeFOZhHCfW1Va7qmp8i1JE165dr8dPrly59sKNm92Nza3t3c3d/c9u3ftvfue3f/KTdxlj3/rWt7YuXdm6fFkJkfW7w8EG4+7g4GA6Gyd5J4mzn/70p9/89rcns7n33jo6Pxv/+KfvPj48nBfVaDZ3S6+gtSYSEgCSJJmNJ2dnJ5vDQSdP00gJBFOXAJAliWRU6cZLTt7s7+189NFHvV4vzWIh+HBzezodl1Wzu7t19foLG5ub49n06Ojo01tjberj0wee6t39Xn+QvfTyy2+99bWFOFJIC0VRJIT03nvCx48f53l3uLWFiEmSxHHsve/1elknL8sSmY6iSMaJaxopom6nZ611zsxntdE0nY7bptne3t7f34+Vmk6ns9ms1+ttDYfdPK/r2rStEIKc003TVBVjbOXBCEmRyrc2eZMVZDR4bXRjtNPVVKqFcYqiyHrhHS9TfXJ8XpZVFEXdbh85q+uSPDiLTdsa7Yz2cZxBxJSKt7f2mqY5PDwxxsQqypLUKT+dFuS8FDFnCgERHWPgHbaNBRScq2I2j+NYKImEErhU0iNYa2fjCRENuxtKcsF43ZjKkOnaXm84OZ89Pn5azqpXX+1c2r+SRGkxrziX3jS21Q2ht65tGiKSQoTafeecD72bGY+E5IIBOSlYnKgk4t77ULVljAcyfukQA/k0jgRDa61SMkvSTpYHk1aX1VzOvHUbwx3vTNPotjUAwDmBt8aYqqq8dSGf64wRkg0GAylFPZ4Z44wxEoAceA+MQDHV1Np5Q0RcCBScLa249jqsR4H6xxgE9Y/pdMqQIeecSeRccBVFsZTSQRRHnSTuKBkLEUmRSCaBuLWe0AWgsBDSAHDeaPPFqYd1buMqB0FExpsLAjsuOkASQej6zZbivqsdAt8x2HW+VGuGtZ4IyyPQ6rzr4YSVDVuPIqxs3mpPWm8viTyM+ZlgwJpFfMaZhgUoCWwAWHIP4ecVCFyEClZEDQ9f5K8/F1RYTSA9Sx3Vy57j4btsiSrILfaZTqePHjy8c+v26clJ0OIhIr98niGEMACdDwBpMQC/NPfs2eDKKn6w7tyvf+IvkieLP4VtY2PDOmqaZjQaeeAyivNOLzyKsGbavfeWvCeIhAga3uHWe8Sg0YzocSmnDWuRKgqqwov5odU4mVz2LFhcDXmwiAi0alLKcQEXLsIhf5kbrRVcBN4nAkgA2N+7fPPmS2fHjw8Pu7rOra28t1xabdtEJoxT3WgZR4zTZDJ59dVXr1y5tLm5OR5Pv//9Pz58evCtX3jn9PS0LExva6s73DqbTG4/uBdl+Ww+b9s2Hwysc4wBYx7JRBKTJJn308IaIfDS/u5XvvoGl9F7H7w/nU7zYa9tGwyiYKECUvAkSZSS9djGSjHgrtURF4NujyxNz0e6qr020+k0z1MA4Ermva619v333/+N3/iNN954o9H68dGBtTaKorSTOw9BbTY8up1O59q1a4eHh2197L2PVXJyeGQI/qO/9/ev7F/5P/+f/o9PHz0ZDAY3X3z5+Pi40+lFKtHGJakadFKrHRg/Hc/u3bn/zje+cfPGi/3h7sHReDSZHp+MTKvr+Wx70LuyvfnC5avf/dr/CpjY3Nkjrj65c++n/+rffnLn/md370+adtAfVp4mTZ1vDk3bRErkm4OXr18VSpbz4cnJyQsv3nx88PTFF1/6ybs/qxo9Gk2YSo5Pz5wlJlRR1c45HsXBTeqkGSJ2u93jg6dNXXNkw37v+gtXTdM+fPjQaJ1GXDIQ3OSpuLQ3fOutr0USHjx4VMzGWZZ573d2dra2tvb2t7VxP/jhT46Ojk7Pju89LJJUjCdHKvLXb/7yr3z3u1evXs7zXLz44otah87ORIhCyJBGraoGkXc6naoKbREirXW/36+1XhS1Ox8K07Tz86oG55Mkdh6KompbN5vXrT6yDpGJJInLshiPR3me7e7uJknctu1kMq7rejabhghhSKiH+yqlZMgQBaBHBpx5z8hbTZ4Fx4eh9A7Lsq4r4xwpGXfybp53jDMhQCKl9MAiBZ1Of2//shBKimhra/v09PT0o08fP35azOcM0DQtAOxsb/d6AyKMojjNOgBADOfzsqpbIiLdRlwBA3AOORNMcM414Kwx3rrZdArOj09Gs8nUtPX49OzmSy8kSSqEfPTwSRKlb3zlzf29y3WnHo/HdVGbVkcySpKEMeG9D5mUwCQIbm4wJs46AkCGSnDJmSYXGFQMfKebBd8aAMKkeQfhqrMsi6IoVGwCwEKMAQR5vwgSMAjxA8ZYFEWCYdCCbOvKe8c5E0JEQqECyQRXUghFDrxDAKa4ssQsWCWiNM6UUoEAQDbYIUdERjshQztoznkFILwT3gGiRwUIUookjXtZkidJxrmQIlEqZp4b47wHgEXYAJz3ZInIGKOdX9mtdTLjqjUDLNMEy9jBReOZlSWmtbj3uuH0AMjZylrwtYYOYYbDWVb749q2+jCwFr6QJ4iIocw6VD+uXPOAjAnXncvFV9yyMdUiss2eD1EgIrKLs/8FQAGeZS+ujw0+x1F4ZlqW9EnOuUcXVHSAyJP3AJwQgJH3RL4si6ODp0+fPp7NJkhOMA4eANCRD/MGDEMCIjjqF1jh34HMv8IK+D9kXxFxPB4nad7t97q9QZp3iGg2m7Vtezm5bAzTWiNCqBZjnAeguIIOYUzeo3NOKUT2TMxgmZBafbzgYS5u0+LncAyHiEAMEViQEkAE8OBZqMUEor98UqMHQAJHtCy2WMKGpD+4efNmOTu1trgj/dnpEXjtnasaXdV1Vw0GGwOP0DRNt5uHMMz5+bipaueors3R4ejg4Fh1e5YLTBIvxKzWzM+190k3ZwLLskxZmqWRNY13TbczuPni1WlytrE53NrZVRLPR2fT8cS2xrTW21AzI4RgAiGO406eJklUZw0ikqc4jvvdXr/TNa3tJlnTNOhpMhqnaaydBURg6BHatv3kk0/efvvts9Gotrpt28lkIuOobrQxZjKZMCbqup5MJsaYt9566w9+7x/fvfO0k8pvf+tb3/7mt69fvX739r16PpOcl/MCAbpZFxFPT88RcW/vkrI6i+LjoyPbGLTeNmYw3M7y4c0bJz9+98NiPGeIYCmVcS/vCRQc1Kys7OHZ4en5P/vX//bOw6ed4dbl6y+3LEqy9OB8/LNPPpNJbGwzmY7GTbnV71y9eq1p6qPj0xdeuHrt+gvn49G8LD3gaDyOOp0kSTr9wWgym8zmvcGwqpvQT5VJIZB1u3m32+31euOzUwTY39nmjDmri9k0luLs7Cze6G3v9vJO1Ounv/q97/zzf/Yvnjw5CPUgr7766s7OnnHuZ+9/cnDwpG4qRCwq0g6sp8t7+6986bVr11/MOylDIdq2ZYzleS5lBIxZ65qmabVNksR7iKIIkSmlpIzqugYAxljA6YyxYKiscU3TcM4TYOPxtCiKLEnjKGWMWeOTjG1ubgP4zz777M6dmsj1ej1jjJQcMW4aNR7bpqk4xySJhBC93oYQArx3thFAaSwFJ/JW1zUXi6WNMYacew9a6zCYOI7Jo25NqK1QSnGpAIBzORwOibBtTF3Xx8end+7cmU6ncRRtbAwZQSiHOzw8vH79eohYMsamxXw6nznvOeeuYkkUM8G11ta0DBA5a5qGtCXvm3ldzmfj85Fumul4cnRwPBqdvf322/t7l0+Pz0bnk7Is+/0+AJycnBwdnbR1s7Ex3NjYAI/WWqsNY0wwLqIoqGxqrcuqKqt5ABOL5cwTLbRjiQOkWR46YSZJYow5Pz8/Pz8PfqvV2lsb1lpvbaP1bP4kT7MsTzudTvAMyTpjzGg0iqSIokgqgURpmjpniChNc6Ws955zBM5C8MJZn3c7bdtq3QAwaz2AtVZrrUkGPw+0Nlo3RD6QiQSPAcA5b4wjj85ywa0QvtPPlYoRGdCCZrV008WFR+scEjnnjDHGLXpr+WerG1Y8g7B2hzAAESF7XqEofGO1Q0AAS1DBkixdCEhwHtiUuCx5WJ0R1jzLAEourP7FKb6gld86pPBLHTpcNWh+Rgl/8bPzCxLlwolHoLVwxeLSPtdM8vMbXqQnLpDBysjBGppZmJeLpprP8jA4rHIfsAxyMMbaVgNAOS8ODw9PTk6s1gDQtm0mBBGBAwJPCAzQhrKRn1POtz6M9Sl97oe1uX0mArH6SmDXrpTOQ4n/xsZGnMVShVgXW93ZlSrGBVYg8h4YY1Kq1Tysh444F0iLOeH4fBQEAoVxERPyS6VsTxR4p35JU/griCgEOcTlVC0IMOBawxX2eoMrV65MJkej0yeT8YkzOC9GaZ4RcS69de3J+ZkQ8qtvvPX221/P4uxnP3338OnR6HzaNrpp2iztnpT1+bza0dYgb5zRRQMMZaSIUVUXKhL9Tr+sps7UaSz2di75re04jh8+evL7v//74wnJBIVS9+7d29u7hIiIkjOIJE+SJE2zJEmSZOStI+uzvNPv9/M0s9xuDAZt2yLi+eg0SOlrY8q6qOt6v5f/4Ac/+Po33tnc3Nwc/f+Y+69mSbIsPRRbW7n20CeOFikqM6uyVFZVd3UPBiMwwFyQuAYDScMT+RNofOAzafwzfKBdGs1gJEZAzEz3dM/0tKruqspSqU4efUIL11vyYUdERmZWNwa4ELOtLCtOHD/u27dH7LXWt771rc5gOJ6dzSspuFBxHKdp6nmB7/vPnz/3PO+tt94Sef57v/PAc/zvfvDgn/3hH/7yZ7/48z/5t0gBL8tWs40xDjzfSGOUNpjMp8lweH3z4OB6PCYYalF9NpnWm50oCj96/8Hl1XAwmPiOqwXf7m7Wo1iW1SefPM3ykvjB1Wh6enqpNAbijOcpYp4bRnmZn11fHuxvz/K017s2oA7b7Y12ezAYuK7juu7B/tGPf/xjKWW73Z5Mp/riouKy3ooHk3lvMKiE8oMQI7L6MjqOE4Zhs15TSnme02lu7Wxv3jw6zNN0Mhn/+te/Bp82G+FgcPG3P/nr3/n+78a1yBiV5+kHH3xQbzZnSXZ8/PT09JRzbtUHNrcPB4MroXSj2fSCsD8cSa1uHt6mDmWEkCAIwjA2CNIk00IabABhxigFVEmlQER+FAdhOp0Fra7neQ6jk8lkNp00m816ozYcDhljQHAQR1lZjOZT13UppdOzE5ePb95+48b+oYNvFkIgbAbj0e037jLGLq/O722/xUWp+0WVjdsxu31j34+coijA0Gb9oNlsUuqUZcWLUggBYLQSUnKEDEbAOc/zNL1Ia0Etmc8YNkrS+aRk7bBQouQVABBCvpg9lFLbxuGu6zZqATLigwcfdTrdVqvje8GPf/y3l5fXR7feunPnTr1eHw77V1dXCGiSJAhBrhEREklVVRUXZVHkSBsuSkIINfj69NxoiGg0EUqWSCt29vi07tZu3rz50TvfHQwGX336VbPZrNcaRVI6mAmQ4+FEVLLVanluoLWmzOMcpBQIg9bIaMKIF/nY1vEjBSANQ7QqK0RQI4hneOo4xPUoZaiqytlsNhqN5vOUMSb4ok/Ekg6Jjaa1kDBq5uOBlDKOw9D3hRDG6JBRKWWlOAiHUoqNAUAIY7fBLHfBxhZZkVFK43YNIQQaSQmgFVLSYKQwNowpxTHGCBhGGgHTRkipAPgyQKcOYwBgqzAYY76zSZFNRxKMsODGKGXhZCGErcswUlkvwUIWhBAwRglhtSDtLk8xwoAVGK21Esq2XkIIYYOQ1ga0weiFXDEsInXbf3KxjWKEEBLCtjQ0Whsh7OaurDjYCzOwEJpdZL5ecU0WANuSM2FT+yvzxitpDDJoIUGIMcZkYbQ0rIgOFGxpPiCtXhyJEFp2isCWt2HWcgT2jkSpLVMSbAnfCgl4TbPZgtKB65u1PIjR2tjbtKkWWMgQLcyMMRXnvu9jTJQUyBhGqda6yHIuSs55r391fnEynY0MNhShsqwKZOF3rbGxXgXCyArJWbO8WMPVBMDCP/aiaMUrhJeVBJYHALZCWAghoMggpCnGlFBHcqIVkwqVXBlSMF5pYqjPlBZE256ci6pXpRRwYFghIBgjbAESWCQgJBOANYABIAAGL+iPixIGs+hPsfIfkBZrKScECBHrwKhlHagtlkAAtkLCdoh48VxehVbw2k3/lwxjHADAWAMo66kAJsgAcRgYUNq7cee98Xj0w/w/UIcQh27EB0aZspDGeOlc8wxv7m/fuPHGzZtHw/HoZHj51fGTRKiLAi6/uuhu7BZI/PrkUgfNx0+vatHmbDZJ5nPUYEVVNjY3vdjlSOPAmeXJ2fkzhPbTuRn0hw8/f16rBYEvETAkqCjkoBjFcWyMcT0mUCV9VXfqQkomSZYXPqX7G807+9123U/nU1OaVmfvF78c4o1WIVRn4+B6nETe9qy8dp2w0755fp4haGYT0r8sGHXnaYFAk0rORvnm7e7orFfkxX7UKpub/6f/w/+x1ek0Go0gCn/18OH/50/+5OHx44ODg9pWu5IybsWbVSuZzagoIj9Qk0FcR73hsTGmPxqWvGoebNKYfvPkV3EY/f7HR3f33VGvL8qqFnkbG5GU/Vvv3pvNZs9OT56dfTVNLonvVyWquAgYzWeFlHrGy7/84lE9qu3tHX322cOfOY/H/R+9cbTTjNybey0+750++Yy6waSovLjp12o4UEILpIv37h2NJ8M6c6t5PyC6SAdxvVaU0xs3tpPRaHz9/I+//6ERVRi5v/Pgpsbq9OqM+LPRxYxVRdMJBs97f37x70fDOSrZ977/j/YPd9Ni/jc/+fHz5887m1ueF0ymSavVyqdjkaf1Jnvz7v7mZnB+9XA4pTcP69TuGjY8tEIujDGMaV5yqyVshRbsHk0pxUYrXilegZJGI1EWCCGPUS1FmaVFUcwnY845hCG4rhQi9n3bJfLOnTtcqsF4omeJ7zkAEAVhmae721t5Mh0mSVmWGxsbGolmo+04DsbUZR6lVEpVKDUcDl3XQaCrqiAE+YGnQUmtfN+1ve2LIsvydDabMIZc1y25xT8ookQrEEIhhDgvjTFRFG1vbx8e3mi1OhiR7e3tqhL1er3b7Xa7XUrxcDicTqeT6djzvCxP8izB2MaCRghR5mme557nEYSFUAiwS10AbXlvVUHG47Hrur4faq2TJEmSxPeGVlYWIWLbOuR5LrjCGDsa2RiI0EXPYkIIgGNLUZb1IIYQQikmhGxtbQmhbP8qwZWUkjHWarUAgGBmoyKltN0QBVdaK9shWmutlLHsNmOQ77plWdr2POtBc1EUSilECV3JEZJFihphbA0SZtRgZLPUBDNA2hhAiDDG6JIPrmwqCyHHdV3XD4IwiiLX8VcJb4ClpIGUC6qgWIg0gHqhmmBbA5i1zt3WBFqO4XoMutx/l8XuyqrlIADQgKwhl8qsEhaAEUJECWJTP6v6zKIo1jMRK+tibbPjeCt8Hi0LHFZwwvoLWLoRSymB5QnxCvd+gTeAWbIiXj4DWnIPF+xLG/ojgCVKbyheORArH0JKuR6jr79eYzWuTItZ8ypeZQZ4nieEUItiVKiqCgNyXbcos9lk2u/30zRF2uBl/YVatmNe0RrMWo/m1X29cpW//3hB+7CdI4yyz7dRj+2N131/e3ens7Ud12tKKSttvvqrFUawBp8ghBBeSmivFt+uiVJGIcsKXB6KzGpdAYDi/34CCX+/sd5dwg4DgIwChIEwBsi0250bN28DqhymkStHg5Hy3KrUpVaHezsH+3sBY1ky+1/+l//X+dV1s9ms5Nz3/YrL0WR4eLSVTabH3zxyMDo+OSt5QRjWQtTqQcbz6SRnjBKKarWaG/jamCiMf3b8C9+3TrkoC+77vkPdsiytxEjgOVzk29tbgesVRTHoXbZajTu3jo4O932XCllUVTGdjjUCpUVVVlkpVV5RTAwiroMNRllZlFU1HA4V6DiOKKUGZJ6nvkvC3U46G25vtv7w9/7xx9/9KI6jq8snf/e3PwKC//if/09pkgBSjVpcZPkbt273rwcXpyfD/qhMs1ajFUURpVTD3O7JnudNZtOvv/56OB6dnp7eu3O3Xq/fvXtXHd1A2hCEsyybjMfxdNrtdIwxlDqTycyJ4yTPsQbsMOawlBceYbubW0VRGKU/evD+rz77+e7mBkL05s3bUuo0zR+8/+Flb4jzwotbvOJKqI3tblmvXZyeRKE3zZJGswlaDibTSst6q9mo1bZ3d27fvh1FYey3qzLVQgY1P/aC73/nu0/rx0XOm60d17367LPHrhu89/5bYeRowz/91S/SZL5/sCe4roxp1ht5mgkhKCbI4PFomqZZ4EZ5Mf/l57+m7XZ7GaVRoeQq6et5HufScRxA2G6jFuhgrrsKXDjnk8kELXltxpgkSabTqc2XW8qY65PJLBmMxnfu3QvDiAs1n86S2aTZrO9tbZydne3tbite+IxgjKXi0zTpNClCJE0TIYTDXMH5bDbzfd93XW2k1AKDJdg7rsc2uk2tNSBVVYWcl1KVZZUTClxUi11b2HwKtzHNcDhPkuTHf/PXP//5zxEiCOgXX3wxHI6llEmSvPPOfcaY77uE4izLkiRxfYeXFYCO4iAKIyl5MpsMh/0gCKxeAmMu9mEV6bZaLaVUnueNRqter8/n8zRNwdhezI6N+C3LxnVd13WVRo7j+L7PnAW73hgDsJBttopvS5IdIoRIKRFaGGxb/mfDQiEEGGGVAyyzvSzLPCsJUbYJpI3JtYgRQhUvGrU651xKgRf7prIXklJjjD3KCGHMMY6r7IWQVVOloBEQYmv6tbb9jaTWWhswCBEwi75ydoMyGBHCXNfzvdBzA8fxLJCwslhKqZWjoKVa9BVc8hKUUlrxFyUAxphlHYRedm9aGR67cXOzgJetf7M0lBisDVuzWItN9GWy28pvXrhES9R94XysyT3Zv1q9s37Y6szLnw1GC/1my1dYXu4lUH3pGbxkSs2Sb7HiUizj9MWwDuW6q2fXbfUjfuGXAAAoJZZ+gz29WuU11nwX+69BCDFCqqKQxgRBgIwuK24XJ50nl5eXp6enyXRmv2irB7p+xRV0sV6o+YqvAN/moPwnx+K5IkwIWSwupRaJtY0eKKVCiDgKVjoZttaVYPTKw7JPgyz6W2or+7BkniIJtjsoWmItev3BUecflqOAEIJFcSYCMICMhS00ADEArgNSdLpbb771TpKOfJ988M7G118/mU6SJ49PGMP3797sbm+16mHv6urJ48c5F1terdPpECd++vSiEiIZDrTnC9d9/803k971aKobG616tzkp5lxhW9dsn78yOs/z73708V/+4K8CNwCgjuOURaWEpI6rtQatCWNGaS2V57ie607GQ4aF76BGI6jXfNcjnuNWBS3KTE8hivxSKlRW89mg3trMecrz+dPJtUGQldnx6XG/f805j+IgCug/+6f//HB35/337jsYsnTqUTzoP3/05TDncwNVt7sTR94PfvCXqsrfe/vN0XCSzaayLDSvbuzvPfr68Ww2BaVv3Lhx/537juNUVdXZ3Nra2TXG1BuNRr11eXHhOF7ghQ7zHEpd5hiDpuPZk0ffxGHgUXr/7p0vP3/Y3dn9+snTnEvmeCUXoyRpR/Wtvd3T09NsNtu7e/f//n/7fwyvL37xkx/95G9/3m1GHz549zvf+/ibJ09/+JOfbu0eVEoppQ7299o+e/bwq71mfD1PY4Kw4ziBI5C6uDp3CP7g3pt+4A4GA9SsVXnmOaRWDxkmb75xe29n87OHX3muNxwyoeZHO3tvv3tvOp3+6K//g9Ti+x8/uPnGnck4ubwaXl32nz456bY7jotKNetd9SfD2f337ijV5pxTzrleyNpTLkWxAPkxps6KZ26hhaIoOOchoZxzwTkCbbTMiwoWvYajqqowMg4jnufV4jAMQ6WUNGI2n4nHTxChe3t7BANG5uTZY7GzvbO1NR5cZ1Ma+t7W1pZRon91Pa+qNM2NRmmaci4YJnbvu3F0mJWFVCUoqZDJCuW6LK5Hkc+UUmFM5/OpgdLgijmYOcYFZLuOKoOU5mWVFXmllPLdaDaW5yenZcnB4FarE/p+fHD4yc9/dn7y/Ad/0YriwHWdLMvKLGWMzee55EIqro3sdNrb25vIqNFoUBQFIwIA2+Q6wlbG1SitOefT6TQIok6nEwSBMaYsKqVUURQIEQCo1Wqu62JElVIOecHiXCoOSSnVfD6nlHq+YykXAKCUUErNZzPfD636BcESAISQnHOzSHlbj8S1Vt9olOdTvVDeBK3BFtWMx2MlFgi5yxyr+bjYzRFG2hCpmIMcx9XalLwqqioIAg1IGQADyvaSRxgTShnCmColEDIIG6VUUeRlKYIgIIRR6gIgJY1SxrI218PllRXRWnPOzRJIAPUiTLdoluUQ2E/COo1xZS/xqm5tiecjgmGp46mMBVpeEB2sjbHDPhrLs1kUcTJmAYZVWAwvOw3rJEG9rLyANYcDVsQFjPFawwgbkJplQd5qczcrduHSM1icffl/vQR+rH1bHWC7g5plxgEtaZWr9Ae8XE2wupc1u/Ltltv+SlScILwQT0AIY1wV5Wwyve5dXZyeDa57ZVkySm09JXxbN+pXDP/qt8v3X+1n8duH3axsCgdjzAizhCRjjO/7zWYzCALbMzeMI8/zbAoPvxgIvVxRaZ8mwouJCSEMXpSlYNumF2lsKdSrFTN/X2/mf8BY1JAucAUD1knAhAEoAASgkR839o5uffX4S4pllQ232mG3UfMwrdc3Ops7nz/88osvHrY2N969f//x6ZnW8t79O0KzuLmBsMOyYS0MYt/pNmrf+fD9r776QlOseDUa9iVDjsMQQVxxUWliwKFsb3/n/fff+/TTz5CmGGPmQFUIUEngBc16QyqOjW41w1rgp9PJbDTc3mwiJEFWnWZtb28v9Nwvv5RZlgql2p0toQ0XYjKTgYuzNFc8q8eR41DQUlTF7vbmjRuH9968u7nRmYx6nksffvqzyHOePf2GGJkmcyn5OE8+/vjju2/cNIr/5Ec/KIrqvffecylgJVyCQs/98IN30+nk7OxsMMyPbuzbJmdJkrS7G5Zd57puu92+urzM8xy0MVwapTAg29Vpp7uBQY961xqgypLYdfc3N7lU01kqy6oR+I048DCAFGXBeZ5JrqqCv/POe8++efidj97/7nc+HE8nv/7sCynBc8Oj3R2ETOCxmoObISnTJGzGpeFVKbwoNMZc9nrNeu3mG7cjTDJeJUkS+04tDkErK2MCVDZbEcHu937nwUff/ch1ouPj468f/UqI2b/63//v7t27l2bV3vbO/s7eV/5Tn3qXl5fIpboSw+ve5en5m/duNZqtrCyoBQA8zwtDN/Y93wuklFqD1EAIJ4TwLAcAxlwbaV1fX9vQ3H4DLSLqOE6tVrM0e4SQLWa1sU6WccrceZJZ3ajNjU4ceslk9PTRl8moP+pdGWNu3bqVp3PG2Hg89pvtJM2FkARhhEhWVgDgO+zy+mo2mxgto8inDGstW43a9vaW6xEhVL0RYCK0qTs+FGVGCCGOwzlIoRUYQIRVUJZCSz4eVmVRGm2BUix4SYmDEPieoyQfDfvJnEVRpI3M8xxAp7wEo7IsGw4HtVp4eLR7eLg/nY4vLy8dxsBg12WOw5RShC5Syq7rGmNGo5ExptFoUEoBKowxY4wxd2XVBJdVVTWakZQyz3MuiI1iq6oqy3w8HtdqtSgOwjD0fV9rnWWJVevEGNvSBsEXNo9SmqYpGGzlCtBSOI8QEkd1KaVFdxyX1utNY8x0OieLnQO01sggY4BSxhiz5YJCCKGk53kOuFyKNM8dz9NaS62MMQQMogQRTDCrx7FSShuLdeuqKpRSnIuFyBB1CWbGEK2BYOY6vlBGL43EwlHQS8hav7AuK3tmlFSCG2MQRqANaAVaYTBaClhyAzEhBIF96TDP2gBltBDK9r6qhLQGTGl4kVagBGMKDluF1La6xxqTVc39yg+wfR+MeeHlvG4tVjbPrFEZMH6RwjDGnmBRHrmyWNZtAQD7uVw/s1la39WxBsHK49HoRciOlmS99WTEa5PUCK3QaQOgl//BCpWARb4DWbfDdV2tNS8rSrHnuDkh6Xx+evy81+sVWQ7IGGO0kFpKC9+bNfbluhv3G4CEl2CV9QX8VjuIXhv2/ZXGhu01pZSybSTtU1v3RRCyhOgVxUSjZSIDY6yNMPgFBELgRU9RvDjJoofyP1xf4aVhDGhbPGP7YClAxPE3Nne2tvdm8/Hl1Red9mZVVZTiza3uLEk/++yzWZbtzPYPbh71Z/Nao7O13VXgHt66M5kmOLl2Kbs8OXv67NHe3s7xsXMxvJrzXGIjwEiQtibUxZQxQik+vzj+3X/83U9+9anRQnBgFIwDtTi8+8ad2zdv9q6u5/PpZrfFHFqWeSPy2j4zxkSes7XRuX3jyLbHnc3mDWJxbQcj5GDEkHEwxC5TReFhjJTc3ug0Go1Ou51MxpPB9enJ09/9R9979PVXD957O/DcWtQIfOf89PnBwUGj0ciS9FoqXpXpTB8/eRwH4Vvf+ZgvbA1qNLwoumE02ujERZHZZjquyxgjSTIDbIIgaDbrlFLHoa7vJdNZ7+oqz3OHsVYtnE3GlMJ0Og89t391Ppsm/eEYO+5sMieul82mvavr8WTq+t5kOvrLf/9nvMzu3D7yXafT6f7608+lNoh5STZ8fn4VtjoImeFocHN/6869tx59+Xm82ak4z/KcOIJSqkFRSv3QQ1x2NzeL6Tiux1Kp8+Mn3Eg1HRkmwzA8O72+eaP1z//Vv7o6Pv31p78IPPL+g++HIfm7n/z1T3/6C4y8Wzfe0oZSApPRtV86iAoM/nQ8Pj89M2aHOi5dYaeMMT8IlLStcaTjOISwRai33Ci11rwq7LdIY0AIUYK01lJUs+m4LMsiT7VSYByjpVaglcgq2Wo1PKVEVfZ61z4jB3s7piqePH2cEHzn1o1ms9ne6PSuLsfDAUKovrlTFCUlThiGADCZTHhVSaMJY5RSpRVgpJQQoqqkp0BNZ+M8zynFZVUwBzdbsR6V2khKiN1gCQJCidae1pxWSMuVGcBFXmZZhqCsqqpWq+/t7XU6bQCQik8moySZlWXJAodgVpZFksyue5ej0UHgOq1WK89zgrEUlpqAGaPWc3Jo6HkeQshuXJRS3/cDPyzL0uolJElihbjrtWaz2QSE7CJzYaw5Xwn72JTESmhh5QEgZBUMuRQaISSEyPOyqipbR0AIMQaszGdZlnEUGAOu6zFGCSG1WoNSPJ1OPTeoeCGEbYJtMAbLiRPaKA1Kg+CKUQOYAqYIEYOwQRgAK6ONBqoRJgRjRKmLsdSaAhhtJEIEI4dgxisJhmKkACjBCAEFQABEa43MizI8pZT1F4QQGJDFrs0abUIbZXWgDRAApLQEZBijtn8dKEBrAskEiKoqu4Mro20ixkbYtmbHLMzoSojJ2A/z6gwroWWbSltRJSzYY4xha90OzRp2/fo7K0dhxUxbAgkrqvxLePvSCXhh/NAa0rByX5CVUl5qM6yyDCvjujrshYVbc1MQeuGHrYf+q7t4xUIThBmhXPGyLBkjgecZpdI0vbq4TOeJMYYiBEtNBbwUtFg5CqtAfuVdrV9xfayb899i915NaljajdIEs8lkIsEAJrVGPYoiq8xjH9zazYJe+XmApJT24dg2W4QQA1oD0kuhqtVUzdJ1emUa/+DGIueAYSFRantXaoOwNkAwEOYAgnpr49adt7/+6nOEI+L7k16vMmY0S54+PdPIOTjc6o372TePJ/Pp0c07/f51VukHDz5+fvLs8ukXNw4PWezM8xlx92jIQhnubR45tbA3HV6O+nmeOwT7nkspVUJ8+vCX//pf/+tbtzfLQkxG8ziou9Tf2drf2ujubLV4OQPIo5AQZAgg2gxjrJrNZuQH5yenohBZUTx7/FxJ5LDAGIw1EZVkxCnTDLgMXXdvu/3Hv/ePP3j3HaLl8fHxk2++Ho76eZ7fuXNza2PL98NGozUeDT7/4mvPIc3O1u7+Ub3WYpSen557xIk2MRIyn8+Tcf/OnTuMMUa556r9O0dbW1u+73/+8OsoihCWSTouRmW/3+9ubUbxQRS7VVUZ4I7L/IC6HqbMbzWb82JOPLq9u+2FAWZ00B/FcRyGoRdEn3/1dZKX47GIGs133nvHYIQpUUVmVNG7OPnOhw/a7eaf/bs/ffOdd6VG2vXOhyP/4przkhfzg6PD3/n9P8x59dXwOojCMPAqwZVQrkNH48Gf/umfvH379rt37satmhsGk2R81rtqtpvEIfX6ZhBEx8e9JC1BoyRJfN/7gz/83YODvT/9s397fHxcFYJXidxLt7pH1+cXcehwkWsleKFOT576Aeai3N07WDQ4tumGvCzKohqPx/N5urm9G0U1C8AihHzfz7JsPp+7rgtLvM5+D62dU0rZMNf6HLYPvVIKYcql8ijpbG8bWQHSnsNazUYt8APPOdzf3dnZyUu+u7Ntm3aUlQBEEDFSG6M0xhRhoZSy4bWWpRBlxQtMARFIklmRXo3HY8aIlDyKg7geMQdLCY5HtSEGJEaEECM94nJiQPGyFFVZSkMI00o7jEZRwB3H99zQ94xSvZ5tjyuQUYHnEN8PA79Wi7MsIYRcXJw1ahFlOAg9LRdLJ4jFxrHrusK8iPKFEJPJhHPuMNcYUxRFmuaTyUQpZZEG3/ens9QaM6kE55wxZvtpBUFQq9VqtQghtEoPua6bq8xCBa7rOswWC8iqqlzXRbBo6Gx3McspyLJMa+37vjFQlhXn3HVjx/HyPFdKmUXfANtW0VjtSYyxRiC0KniFMSaMeoEP9ryUUouNIytEZ7K00FZQwShjtFRcKWNdTCk155ISz3U91/WNQXlWIspg2QB6kUSQC8kEK3KrlNLLtAjGWEtllGX5gVZKS4UQciiTRCyiaq2VNsgABoQBlaK070utFs2N0aL7MEIIE7biClgyo0MX5IsVBmNXb70RyXpM/Ip5WHcRXrG4a8Z7odGzwr7RkmK5HhPbM+BXhATQ6sWykA9g1QDCHoaXytPrb9p01bfME5TWL9nslUVfZgHMgpEPthiBCSG0kAwTZMx0Oj0/PTs5fp4kiRICL6dEELYJLMsuXLf6ZklcWN3m+qzMyzD+38f6LlbV2nujbbab0cXCWppCGIaMsRUY84J5irE2VgLcaLs1LJrFLsEP8uKhrNxZ+2HGC68LEFpLM/1D8xYWqQf7g179z4JIWhjGiJGIuMGNm28cnzzvTaVMCxzEb95+4/Ji9M3xedzcaG9uTrJslqRpko8nQ+IEXJrLy+eD3sWTi2cV8J3uphe5V5M+i9y2137wvY+8RnzRuw6ePTo/P8+TlFeVqUQBoFCpdPnue3eHw8moFm92thl24zBOZ7NvHg1n0zEy+uJ8HAVOp9Mu0tQLam8c3cyy4m9++LdcaUqc0WQqhZnNUkqmUhts8Garm+QZxaQWRg/eunf36ODs0ePTR49/8fOfpVnR7LT3d3YVV/N54ge1vBTd7YO/++kvjVGbu4fnZ/08FR8+eF8USnLl+lRJLpWYDPrum290Nxtx3fN8VJVjIDWuKs+nG91mUXhCq0C6hJpGsx7H/ngkizKZjPsTwghCnk/r9fZGu4MpS+ZZmqdJNnccuru3fXTjVhDWzi+v42bj0dPjWZa/8+CDg5u3+pNRFEU1LL/5+mslirfevD2djuqtemdr+5uL3jQrkBPmgGZZUab5SX/0zr03P8yyh//v/6dmLPQ9jzl5VUoshsPhD3/010iK7Xb7rTu3KIEKqt2jfSdwo1pMZBQE0bvvfGdv91CXYjKZN+p1l5GyyHyP/dM/+oMH7380HqVFrsHYDhHF9fVVWkwwElky7feu2t12GEU0CAJr3Vcuv2UkgG0nuLHh+QHGeHt7t9/vc86FqBBCSimtFHUc3/MVJbnR9Xq9KIqUYM45RaAFF1oZY6jrTydzz2W3b94IXFplc60kMoogEFV5eX42m00mk1mr022325xzzrk1M9bCEYSDIDCgptNprR5hSsukLMrU9z2l1HA8Mnw+Gg0wBm0koRvNdp0xggl1XSYllxIpUMtedcYY5TgUwOOVZMzV2lDi2ExBFEVRFC23M40xLvIqyzLf1FyHbWy0t7Y7RVEkSQJa1ut1uwFVVaWkFXhDAEAIjppNy1UkhHHOh8PhbDarxXXP86qqEkIEQRCGYRRFWuurqyshF3uZkNzKitnmJVYmAROwhScIIYyR67rWWi3CXMwcxzEGpNSj0QgjahvWESIwxkKIquJKcryUQ86yLEtz3/cBYDqdU4qDIAg8H+EF/VAIIRdq9tgYw7kklBJCfY8JrTBGlC6K3lZRYylLKaVUXCmx2KgQMOZaD5Jg5rpeHMdRFCEgVVW5JIIlYVDYike1KIYkCIPFkNcdBa1Xu7z1AKxFr9Vq9gx25qtmCobiqqqqqsrLgnO5wJAx6XQ62Ho9lC64AggAMDKLjL7tVbaoQLHS02vWAtbs8evGDL3c3wHWIPclm+Gl+H7pf7zwCVaJDIpftprLn1YLrrVeqQEjhPBSHNssRahg6dGtImlYM8+rk5iXmRbrZMPVFQEAlBJSAoDv+0qL68urZ8+ePXnyxCxpoUopZMAul4Vf0LK+A9ZAi3UH6HV/6/X1/FbUAV5ypxBCiGLKGMOUxXEcxrXO1ubW1latVlt1CV/cyIoHusj8GCGEIdgYAwis+2bPv+7Jwcu+AloAJICWJAaEEPyDq3p4dWgAAK1h8eVijEkpGUaNdgdjQrxoOMtBuyyMHz/75LMvHv3TP/pnnc2d/njccjb48yeXl5cPvvPdWqM9T4rNbjvsfHx+enp8cdKO66cXp1EUzdNkMBnc2Gw2WvWtcns+n09H42xe+Bg8xw3r5PTsaaNTS9O02Qq7W80q47P52MHksn8een6n1bq+PAvC+OhwV/Bc5FW33b3W/dOTk+ksC+N6muVC6cFoNt/IW+0NRt2dnd1er8ddnxBy7+bNkLEf/fpX437PxdRp1ikmg97w5u0byTybJel/+I8/+Jf/8n/+8OPvzyZjg92yzH7ytz976+79W7feuHFw4/zsWJZia7v5xu0bGJk8ncY1j1FzcvpkkvaNMRg12p2G69F2vRXFcb/fH08nz0+eKik5L2bz8aQSoee3mk1MTMXz2ZRfnV9wLikmhFBC2PHxMcJUAezs7JRaPz05IQwPx4PBeHznzXsXn/8snY9+9x99/5137//bP/mT7/3O9wVxTy/PHx+fHd59y41qkGWZNl8+fnbr1o3Nwxvbne5kNlVKuVFIABFAXuDXXP/Ro0d73c7tW4dlVRlsujvbvVEPi2Iz2JtOZpT4WuEf/+jvvvryISHm/Pzq+cnjvf2de/fu3H3j9rnX++Ff/d2gP223Nt97562Ndq2UWdjw/IjV2vVup+UwQj0voNTByzoFAsTddXa3d+r1ZqcRyzIjmiNDxr0LWZbtWjCcyrIshBCOQxlFBrg2kjLjx44TMDfwyrISUgMi2hot6vhRFDh0OOg1a76H0fj6jGC02apFUeQHrlHSxtYPPvzgpz/9eZqmtVpDSsnLKggC1/XzNMmyoiLlbDz1XOp6LGBxmebpJDXGBH7WbtaHw75GOo4jRnEYeCVHYRAoKTnntk5aEIkxAcBuHOeTSaE5l8J3fI3INM0wIOo4g9EoT9PJZOR5ju/7SvIiy6ss9bY2upstP/Sms8l0PjEuQID8tpckAkvQXCpHhn7gOz5CSBUVcbRSuhS5MsB8opQYTPqIsmajFbebRcmTssxnidXsa0VNpZQQlbV5dieK4iCMXKU4JiSKXcqsCI8piiLwnKIopMo9j7XbHUJIrzdASFdVgTElmFHqEEIwoogRgpnrhwRRpUTgesqhWlSB40ElG2HMPA9jnBelMsA81/M8hDHIUmmwNALb5HiBHYPGGAxCSi1ywNbMsDAuy5JzSSkzxnDOMWZBEGCMHd+1GjigvaoAShElflmWNgwUspIVl1Ii0NiAgyFPpw5jvChkxWu1mBBijKxElifzer2+tbExT5LZJAmazVu3DylxiqI4v7zI81IpJZQ0CGbJHAex1BI51Cex62sAy2FgjhtqrQGw0Uwoo7WmlDmOg2hh91Mb7QPSBpRSq3JEbIN5hBAAtmyKFZi/blEscce8XKaolNIIAQJsMMEYENLGKK1AW43qxcF6LTvARbVYbwC9tPoYY2wAAdLGIFhUW65xKYwxAgEwYpZ+hlGiWJkKszZeARJW5zHyRbWCWcsdaKyAGkJIwdPzk9OHDx8+P32ujagkt4WsGGNldJlnAGBbicMyL/CSuVr+iNe0m9a9gXWHZn1hX3khK0kcIpUhBFzf0wo44FoQsjAMGo1aoxnV6q4fuNTFCnFe4cixadOl77K8a7RyCBYTWfimGgwoqTUhBAMQTC1SBdpgjAnCBCOCMAIE2hgwCq3V0Vh1DCua8EL0SMLyM2QAjPFWdwwAFs+A/5Tn9J8xXvgtFICitTekqphDAAFQFwirqureO3+Q/uAXCpJGuz4enDInf/udnf7weJqO8wrSXF71wAkQ/upq/yba3tvQIXKU6fXPu1stZqqaJ8b9r8dXhbzRCfnR3XsPIKXujcbTT3tMuaJShiNeBFfnxd7ezYvn0+lk7DpjLVWWJM1GI2r49biWlVnUaU6S9PnldXfvYJPjQb9/edmbZnxaadqKWKfdarSzoiykNFHz3lsHnkuaGx2jhUOpE6Ivnn5mHOnX/Lvv3k2KcjCe4Aqen1/0x6NbN4+EkH/zo781qtrZ3ry8vOz3zm8e7EWBE/j+g/ffdSlJZ/Nbt269/+73pJRBFF6dDqdDw8xGgDpFWXRrB7vhnclsPJmmzdubRLhJP9s/3Ot22oPBIJnNRr3BsN9HhUaOub6+upxlpeDD0QgI7u5sCV59NbginrfR7R7sHpUT3jyqo1iUejSaPP/iSzw9/fIP/tnvffTdB8ejJ5tv7UlDfv6TT3rJNKjVZSVPH5+6rhs5tck4/dN//1ebm20nCl2jer3rm5sbZZn5Pmk262WeZaAHMpmYPPDZZmu/ytJ20DRCbdSbxXyGiKl5kE3OfvrXf3r3zhuf/vIXYRCwZudWa2f2/LL36MnZ578Yj6bu4aFmdV/TRtTtbnZrrVqlef90dPLsklr7tMIJeSXslvf48eOrqyvOeZZlvr+A8hzHqUcRRShDBYAWstKGGABCSP/q2vEDRl3GmDFSG7SgjscxMsbByKVGcIWx9gK3EQcHe7unp6ftdnM6SyejC6XUgw+/s9FuXU0L0ErLRStqLQUhpB7HQlRaibIsyypXoizLXEqOEKqKme/7BtmeSTzPSiE1GKy1Rhg7jieEKCpRCo4Q8jxPKhpGvuUoEEIpopQ4juNVVVUURVFmZVEoJQhFBkEl+JK97xDCrCSWFLooivl8bvUhoiD0/dBlHtJGSkkc126LUmiplVIGI4qRKriwALgyuiiMUgJjz/O86+vLKIrCMHRcxnJKCIprYavVMkZxXhGNPM8TgnDOPc9pNpvjbKq1brfbruMTwlqtluN4lxfXlFKEiNGGc04I9cOQUiql9EMMGtnqWBv22dBZCGEwXvwIiGhmjMEIOY5rjeBi/fWCZ240Iov2QGhVemAxlRWpYhmkaillo9FwHHdFO1+ZCgMLYEAvRRW10lppyySygIGsqqIoanFcca6qQsuqHgf1KEqm08hzuhstl+DNzXZRltubG2VZXg/6vetBWZbdTnvOOQZA1PF93/dDIFgKrZTK88JaIltfqLQCpbEyDl1F+eRFagDwUnBpPdkPAC8UHuG1YP31F6uxbvnssKZ/CV+j9cNeOXIVQ7/yqxXgsX7Rbz1gPUB/Zc6vXEi/XCsBAICAEMI5v7q6evz48cnJyXw+x6/NcPX6dWTif+V4BX5wXNcmpGw1lhRaWdIHprVGM4jCZrNp0TgAAPIaPGAlGAEofdHNYX1ovXDOVn6VZavQNWFJtNaPe32sn9C8/OY/hGGnbZEnizk1Gg1KnaIoDOh6vR7Hca1RT2Z8lsxPTvte0LJgVZ6V88l8c3uzUWsNRwOHSIL4brdd87fH7VjLr05Pn+wd3Yxa3SD2dpzN9z54Z9jr93u9bD5PCwRIUgZKVUIWCCQGJXiezJTrOFk6ZYTUo3jGy812w3fdB/ffnqfZ2WUvDqN6a7PW6Z5eXJ8/f05dNwzD2WgktzffeOs+RajMUyn5dHxxcnLy8KuvhNSHN2516nGWzsf9q6Iq/b29bDapsoRCu7nRphjnaer7fhRFo9HoyXhcFMUf/uEfNuKa53mCl5YQ5rru5uam7/tb29uUUqYCQsCADCOfMqyE0FqDNkoZSp16ramEzuYFlzCdpZPJVGJ3nhTztDQICnWFGK23mnsHB/Miu7rsn59dnZ6e5nm+tbXlOM5gMFSFisLGeJoOJ5NCq8fPLz777OFwONzYuiGlnM0nQeBRAmD4cHg9nV1vb24ZpRPHzWfJfDz1PDeFmdGSgJkPJ0VatLdqDnUn6SCZJg5jWVp2N7a1Vj/72U+rSty9++YvfvbTt+6/9Y++9/HO9natVjs+Pvnqm6+u+70iy6/6ztGdzVan2dxoxY1aIbN8mmKCNzc36bIQX5dlKaUUXAIAIaTdbiOEbA9AS8GjlMZxjAlTSoGWxhipJUfaKgSk2dzh3PMCwlxKsTGIIACteFm5lGKMhJAVLzlSLgEVepgSIUSeZoHvff/j7/zJn/27dD67fefu5fhRWWRFUShlpCCaV0IISikliFeiKDIuSsmLqiqsgl7FciCUENcYlaS5MlpZbTuCMaZhSCuhpE4xLih1GENhreNM2GgwTuaZ1hAEYavVajU74/F4Np0KIYBAJYUxxvO8VXU1LLdRrUCIUhtZlpxSGkZeFMS+7xsFRZpXFSfKILBybsggbIw02AAAY7anA8U0ZozZVIWUXGk5m0+LMo/jsNGo+YHrOM5sNomiUCmFseP7vjFKa02pEwT+tJgTQprNJqNuWfJarYYQscUmjuNYaSnLZrCMvKIolNBCCIpwVZQOZdPp1JIhbEfjlbDBaoNDyIaV2nIjKKWUSq21Aw5zrDGlCClCFllztISazYIYSOmi2ZXzAudfBqnKah9JK8asEGgjlRBVWRZxFBVF7jEHAKQQzWaT83LWfyayxEWmUfP6FHiZijypsuT6vDo5Odnc2jk4OLh1+OFwMv7Rj340HV239g5sKI+AMGoIw5VWZVUgrJE2GqQBoo1WSmvASGksF88XoTW6vtErP2b525d4dqtoeB1aR2tNHdd3Z3jZ9KIlOdHA0oatnfbFg3jZF3ndD3glBH/pii9PwKzVTcDLdn3dtsFaBmQ1DQWaECKE6Pf7Z2dntpbH9xaqU+hbOAffnjL4VrsFi1KLbxm/8X5hQeFEBFNKMcXaIM/zbty+tbm13W63CaOcc4OKFYrw0q0hA4uuFgjgpZPb869cXljURCwqg4iz0qt4yat7fZ6/ZSnWHYj/zmM9l2cdfVvm9/4HHzLXmU7HQsmirEaj0fnFqOIornWDKEbEkWBm42klSsxoVAu0vtpo+0dbnaPdTQfMdiu8d/cGYVF/Vhmdt9rhaJxubrWGoysvoJTF4/Hk+OSZ5zmU4TD0tZagVaMW7W3v+I7rEZYmyd7OriirMAgajUYlpJLamp7Z9GqeZkqjo/298XR+++hgNOhpXjTiuMiSyWhY5ilKehjkZrPBpfCJAZFH1Gx3GmEYbm11jTG6Gdcj1yc4nY9DjxI36Ha7nucZY1zXbbVaHnP6/T6jWErZHw44rxaVEXk+Go3u3uxUVZGVMyklpprzyqVkMpn0er35POVFdX3ZO352Uovq9+69tbV7q5cVF/0xVyiq19wwwJQY5PT6k7PLC4xxmpXdjZ0sTRu19nA4/Ku/+Ov/6bvfjaPO82dXZ/0Ljcnf/t3Pvn70WCM2nU7jeiukCEAn6SwvJg41YeCEEPuE3T267Yd+yNw4CizMfHSwf7C7V3PD65Or6+MLzUWr0dzotpO06rRbg3HvyZPTrc3OO+99+OjRo8OjGx9857txEA4GveF4kPOqtdGObx6FYRjEQb1Z39rdjurhYDIcZzPCUFSL6eoDZAneKyX8+XwOy5wWYwshLdd1HQccRnyv4foOxiClRBiY6zYajaIoKi4Jxg5zFSAhRFmKPCtch3KXuRRR0JpAUfHJZDro96fjkTFmc3PzjVs3PWIef/3F3ds3AodyzkFKAkCMrqoqSRKEUKPRWNQEykoppQETjCglmLjGMExASZMkBefSIK2UKKqSMOZ5HkIYEWwAa2PAGNfBrkeZh2iFlDSUIoM112UQerO5cTwGmqbpXCgVhL7je5Q4CIgU2gAXQtlqPl7JMIgppaHvB56PMbVii8k8Q0JZjUJbasgrmzbGjuNIJfI898NgY2MDAIbDYa/fr/thURRFWTkujnFICBKiEqJ0XQcTsBr1tmUUAGRZodRSaVFVlLKqqsbjsQ1/GWPYoYQsxILKspxOp3HdEULaWMpxHCvA0Gq1OOeIUoQQcxwNyOAF6UxKSYhZaFCWlSVpOo6zhBA8ADAaEUJspw8hhNFoBbxjjB3Hsw04VmNV8ocQUlLa1k/Ygt5aKSUkFwgMY3Q2LikYrQRCaLPTllI+/bwkoMoiRUq0auG5rCaj3v7Odrse/fTq7OryfNi//L3f+717t2+cPz++uroKQ6/RaAihzi8uJqNpENfA4KpIGXM0hhctFZHGgLVBS3/gBXXA7qoWfXkZUcDfahhet76vjNXfw1ohgzGLbtGwLIm0g6xFwHbYGVvYD6/VL+jXhIzQ2t+u5rl6CrBmDl+ZHixpFnrZkOJFKoRz2xlkMBikaWovsdKrfh0zV69JWi3u678IaXjFxQEAqRVCSCjJMAOMGWEIUzfwwzhijEmtyrKUWvuAASODwEUvIUAG9JKVgmwHy0Xn7GW6BMFLS2Re8DnwckmNtfZ4rfBhNcNXPhirp4Bee6z/ncf6R2XFv2GMhVHH85vVYFLN0yTJpNS1Wq1W7yap1FJlWQaUub7jMo9hZhTqXRzvbW9RpGaj3mzYIwjfvHmbBZTlpZIFJt7J6ePry5OT40cOY51Oy/HC2XSUJdNbNw/3tjfzWcLLcntj897tN6q82Nrofv35F7sbbcnF559+Jre2BC2H49HFxYXiohFHfhhzpZthsNlsfPzh+5999hkDDVUx6l2ePz+Wkm+4sub78c0D685eD4YuUm/sb1FKteb9ft/zHI8iyXNs9Ifvv+fEcej5NraZZdmjR48acU1KWa9FjuMkWaq1tvynL7744tmzZ9PJpNFoaC3zPDdYMcZGo9HnP/j88PAGJW67vbG56Y5GWVWJwWheVEAbLTdott04rMVcVqUU0+GsNxzkeR7GQRSERwc3McaRHzTj1s7G9u7+TaXYyenVRf+6N5n88pOHQJ16s9kfz5kbSMk5AS5LqUqHEeb5FND+9s7R0ZEW8mtDGKMFy+/cvvXRRx9tbrRrUfznP/lkNOjfunFzb/uAIG9359Bx6S9++YnnRSenF/fefOP//H/5vzYa0enZeVmWJ8+P8zzvbG/6tej2jVtCCDfquK5bb9X8OBLU5KrMRaGVoasie/sZQkuJG1tkbxEFKfVkMinLMk3TWq3GGGk0GrUoJIRkecqlAK0dSkqElJRVwed6rqTGmDqO09nYtBUEGOPA8zwHG61nSZbMxlrwWhQMeleflOm9N27UI+/kyVc7WzeSJNGSC6EoNsIYpDUA0kKAWqhGEoK0JggbSqlLkTFISSMkCMUNYEJQVpVpUQJoTAkhREmTlYWUEiGSDq8QQvVm1Gw3CDCtoSxkr3ddq9WFFg6hQEAomZe54zm+7zuOizEWQgkl7WogijHGURQghCiixiBeibIsFz6BBK2FUgYAw7KOEQD7jHLOMc4NWlUPSNv5yHUthQ6l6UzKqtVqtVptTMB3fACYz+fGqCiKlFLD4XCaTTudTpIkGNH9/cOiKI6Pj1dyeFaoTi/bRQohtKbLhygxICsu5HlenudgzILnj7DQSkr5Ai/VSCswi2bASGsrroABOMCCwE8wc5hrmZuwtBDWgXCXt7QC6lcpaiGEktLqA1KEtdZKCiWFMQa0VJILgWXFAemizIQQhKCbN498z+tdXTmO0+1uIITjKKh4YRut5nl6cXHhOE6eJWWRRbK9s7XtOE6ZZ1VZBg5zvUBrnZeFMRoBhqUzYPduS41kjDHmrLInGsOSt0hW3wtYGNqXYu4V2rQQo3zNY0CvDfuW1nrVFGq9cbLFaf4LzMArTszrgMS6u/DKn6/M2LpJtsfbFnxPnjw5OTlJ09S+n+e5rSmAb/M8ftMM1y+39v5vu6PXbw0tCkZAKMWlIAYQAVepp0+fdjYSbtTWJmvV4iAKbfCzfvsAgEDbfo9KKTBWiMxgjFcaSoR++3ysKwwAABrwi1X9Vv8ALZED9HJZxP9AX2HFaAYAz2qiSEkIcd02IfF8LmfTREhab3SbLbrR3W21d/NCPT85n+dFIUpANjkoG/VOp7Xheb5DYHtzJ/K9ehSP0yyfzyZeD7nxfDwMPRz5JEvEbNTvtFsiK4pZ4jQ7tVpjpiCV2sGk5odOrYGkrkURMvD82fHV+cXV+UU2UmEceW5wdLhPqJOVZTmcyGKuK9w7P3785afJbK6rNJ3Nz87OPM9tvbHZv7o2xhwcHGx0WlKUGUPdToM6rN/vp/MR0jWMDMJIIoORGQ6HE0AYYzDG5tRsqHx5eWml7aQUs9ksTdOiKLrdrut5iFgxCqO1juP46PCwyMqsKNKqIsx3vbC5sTVLslTI8XWfFAIcrxnHRVE8O7maJVMFZp7O3nzzzZs3j2px3Gm2Tk6Of/3Jryaj0QcffHD7jXtJXgDgMGr0Hz2ZTmH3qCaUqteaQurBcBzFvu/TiMUAPMmy796/v7nR3dzcPDk5vjw750XZqNV3f2e34ceGgywUw16t1orrrbwU55ePa1EZRdF0Ot/c3uoPe+Pp9Hd/93fLKj85O//qqy9Pn5/4vv/BBx/gycyJ6slo1G20ldFJXk3SbF4kRckNwdIAtTIJq3rx1abfbrfsjmDtjW3YQwgBrWwiIAw8wkiazvM8xxjnZWGbAUoDRV4B4E5nY2d7L97oFEWRzmdKcsAEAFVVkacpI3Sz0w48h1dFHPgOo199+glC6O7hPWxMNp9P8qxQUmtwKMOUGGMMKFv8BwAAxtgOghiLSlKKjcZKg/IwIUQrrIwsioxzrtEiQmLMdRyH85I5nsc81/Eo9bRC6TwD0HmZGqPlQsl4ga8EkU+JSymzjHsw2HV921zGcTyllBRSSy2E4FwiRBzHI5iUZamVAYMJpRgrY4RNfFrBECH4dDrxPC8I/MPDg3QyLoqirDLP8+r1uh+4nu9gApbqKISYzWYIIc8LtDZZVkynU4sHUAJBEAghsixDCNlyOBv9CCG0AoxxHMfGLISJyqI0SoM2WVZstNrz+RwIcRzHcV1MmdBKaw0IeV4Ai20dM8aM7SVAwBgGgGzLDwAgBBBSWheccyv6s+Lb24+Q53mELBxQJRcfMGOMVsK+xgY0BivgCqCNkkWWI2SQUYA0MtC7us7zvN3eePfd96qqevTkCcak2WxRSudJNhgMpNGN+kKSfTKdzpOk1x/WO1sM4WbU6La7aZoz7CCDKcZaatCgkUZAAQxCxCACK89gGUwvI8gV8r/OyDMvPn2/IXZcHLdmP9ZfrByFxZNa6vboNedj3fysTo6WAlCr99dN+7o9WFnEdS6heXms/mqd9/DKFWGNkDgej8/Ozvr9Pi9KW0zxSonH+utvTd7/l41XfAX7gi8FRaxEN6bU8/y4Xguj2JYvcSmEEFJrRhbPd/3GMTKWP6j1AkqwEAICpBeamMsko9Zaa4xfLM7K30VrJZ3fipSsnvLf04v67zBWG7t9QBjjqqoAoLv7ZutkoOUX87l0vaDRIHlWJvPs4Rc/9Pxakpal4NN0poD7Ad7a7l5Pq04bO0gPJoN64OA2K/IRUD/2Igc7jLAbe7sH+0dv3bz9Nz/64cXZVTlKXdctJtm5OGnWG1EUcXAm/Ul2UOzc3Dk9OTk6vMUI/ezTL77z8e8Mh8M//fV/vHXr1u5Ou6GUUsqdz7HWhztbQRAgyT2CrueTsydPPc8ByUM3Lis1TzL7aMIw2Gi1W43a/v6+53mtZt3zvEqKmzdvul4wm81maTYYDpq1OiEkDILG5maz3oj84OzsrNvtOo6T5lmaplzJNE2VUp1O5+uvv7E0rCDwO+12luSu6966dbvXG3z+8Msnz54DdcpKuX5QazSVUtOrK8/zciEqKUolkqIsq7ws81/84hdZlhweHNSj0GUOxSgKgjdu3drZ2Tk5P221241u5+GjrxttHMTRZW9cb3QBUUIdrU2ecW1KSoTr4Nls1mg0ptNpVfAyK3tXAxc7HnGrXJyfXO7s7Ny88UYYR1EUVVJM5tlkfIwxSFWp0ty9+8btN24WVek4HqHOdW/06OlpGIaN9naelWcXw16vV2ucMpdyJSazMZdVY6Oxd7jX6TK6crottkkJs5+k6+trq8hmjPE8v9vtKqV832cIpOSMMYwRQeBQZnx/AU0boIRFjheFAIBbrXajWUuz1KJeUpg0TRPNeZYV6azTiJnrNJtNUbAP3nvnqy8+vzw73t/fH/SuPC/wXTYHKKsCDCaYOA6tqnKh04ANQguNekAaNDZGE+ZiSqpSaA2AsDaIc1lJVQlpjMKYEkapA5iSRlCjlAJgIaSqMkJcL3AxJYPrMaZISIkBOb4HAAUvPM+DRR9kbRUFCSWIIK211SFQXCohlTJKaK0WfHiL0ruu63guY2xFF7WRK9ICIeT5C9m4EdKXl3mazglBcRy2202M8Xw+tc2p7W6vlErT1BIObCRncbbZbGbzCMPB2PZfsDQIzkVVCsaY7/sGIWRwEASgtJbK5jKazebFxYXgXGttAIg20mgAIJSaZQLCLPLlVjXZUEqVFktoGq0MPyBlqzQXRRCM2dzw+gatlF6RH40QBjRorYwxwjYwlAQhREhZ5p7jMMYcSsIwZIx4LnPjNmKs226nBT89PTWAg1pzPJu5QYgm86vrfqsl/SjOSxnXWvsHmGfV2fFFOs2rgmNNp9O5VFooLYRSGgwimBoDGIxGCCO0EFYCAGPW+hjpF+bhtX/XFHheM9LfGjKuW/fVC71s2wgvQ9PGfEuZ5be6Ba9fa/23v8lR+C1WZHWhVeWClHI+n08mkzRNzVo/i/Wyw/8sROF1d+Q/+Vev+ApKKUIpYZQQgiihlHqBH9dr+3uHXhh6vo8xzquS5vlKSsGsAVovdCng26f9oooSAAAsJAlrnhm85nihl5Mjr9/X+hP8Hz601hZ0VEp5ngcy9twNpYLZXDaAen7o+ZQQ0u12m80OVzot0pMzeT2YJ8nQCxDCrUbzoBm5p0/P+5fjaT8FpN99/6N3PnpPAOuPp00vv3d06/6Nm1Bkj6OvdOVSSutRXUqpS+lENGQ+NzybZ1cX154bMNfPksT1g93Dw7fff3+rfjQeD4fDoZLy7t27zWb97ORUa00pmc1md44OeDrv9y5830+SpNNuh/XGjSh2HdrttCm12qnGBrdKqTfeeKOo+O7egdCmvbkzGo02d3fiIJxOp7yqOK+qqmo3mlEU9XpXV1dX/eFgPp9xJQeDQVGWm5ubnY29zsb25kZ3Z3er02pXVZGmqeM4nU6XMneaZkKZi+v+JEm54coYL6pPp9Pz/rXve5hRwEYZ2eq0b926YbRM01m/d316/IxXxc72Ri32uaqu+1eGgMIgtKg16sYYjGlRVr7veG6AsCmLSgge+oRF0ZNnzxBCruu2ao39/f0iyx3Hy7JiOBzPZsnOHgmimDiOxoS4JKg1v374xXQ6BdCPHj1qNutpkR4c7rdardFkPp4mT48vwiicpXJ3Z//hw4fj8VQKqLeaBulZMg7rwYMP393a2+VCUbRs6bTIQTBsCS8Y01qtZrcMGz1zzjHGoUvTNNVClGXpONQY4zrUDz2lapPJZJ7nrlO5bii4Okmen52deXEtjmPPd4wxZVkWWVpl87JICQIlT966c2s84LPZzGHkowcPhsPh5fnF3t6eFYHWaSb1Qpq3LMuiKMoy1wgIQYv9FBuEKMbAmAsgsxy4UExRKeU0mWtt07pUg7ETRgj5zGOOS4lTYVUWQpS5VlgIjZDBGFdVhgz2PE+IKklKY4zvECm1lLbvoEKEaGmkEmVZaq2NVFrazkbGSKU1KKmLorRcIS/wVyrXWtsKRqAOi6KoXq9bH8LznEajxhgJgoAxYq0pxtjCvJajaysy8jwvy7K52bQRVZbOLy8vDw4ONjc3nz45BgBbVeH7YZblVTmzsuSzZKqlvbXKKE0QphS7ruv7PhJW+eAFiGqTLPZrZrmiCCH76QjDUGlpdxmECEJgZQyC0HEd11oOWKgge5TSqqoIsUWAoOSLqjzbEBoAjNJSStAKIYQR8l0nSSo/CChGYRjv7u56rqu17l09Pz45+/DDD995/0GSFwBQbzTKSpydnY0nszRNXS84v7yezeb2EpEbP370yA8iz/N4ydNZIjX4YSSF1oAM0oCxAaTACilgQl4k/mFlMvEqIQ3wIvJerAaseQnrRgX9JjLjK77CazUOaK3YUktlvg1RWBm8lcVdxwNWF339Bbxs2NBv4Bu+4issPTx1eXk5Ho9tgY80YD/2bCnS8PoZfhNH4b8W0mCzRLaaUQghlcGUFUVxPehvQDeMoiAIHN+jlFqOwuKJrLkpr9wpetkbWwFLdq00ftFnfLWcr5j83+IlIPQPRZFJL7Um7Q2uIgFATqe126hvCg7DwdQPKSEoiiJCCMbgEoxZ1GrV03JijMzSaW8mbs+relivOPaIx9y4KrJOe/ud9z6aTea9619enZ1fbm69ceNov7sdEUpVYzweI4Ss5Pzgcpjnue/7ySx/8uinf/RHf/T110+Oj4/TNE1+9sm/+Bf/4sGDB6enp0KI66urdD5zGa2KzKLdSsoPH7znOfSXv/4VwbQWxUcH+4dHt2xXYYyMFpwreXlx8fz589u3byd59sadewqR68Hwy6++2d7d94JgNLwyG13GWOD7l6cnv352fFpvJEkynYy63e7e3l4c3yt4NZlMmq3WO++84wbtGweHmEAym/ueQynWQjLP/frRN34QRY2mMPDJZ5+dnl+E9Vq93vz0m/OsyKfXl9P5VBp5fT1vt93vfOfD/f3d+WzaadTjMPr1Jz/jRdlu1hlB/f71yclxIcu0LGyZoYLC932DnLLggk/DMAjDiGDXAOeVoI5TCTGdThu1Zru14XlnADAajUajUavVqtUal9fX12dnca3W6rRLIS8vTzc2Nu7fvx/XgocPH372+adb25vPjp+7ntvZ2A7CqNFoTSfpu+9tp/mvS26SucY0B2yEgChubO0ctFptAE1VVbquizHiUgilPce11DnOeZLMqOMGQSC14IJzLbGhWJFMKN91uNBlmTXiuF6vV1XFsahFjcA3UpvxdD6eTInj1hutGnWLrJRlxRihmIAUeZZWRTJVVZmRi+uLRi18dPIk8Jx3P/7oz//dnwpUIdf4Pt1krctfnURhjDEF0JRIMFwrTjBlhGmDlVKggPqh53m8NFqqyGt4jIisMCXko7zZabq+lxY5r4TjuYRFEoiPO4gTqcFI6QAzBHEpQVWUOggJACyVlkp7fk1rXZRc47lxWhoLJSV1KCO0qirFtcyl1ppzLspqrbEWpopijDBGo9Folsy73W4cx5aPiTGl1KGEaQV5VtpSgrBRCxs1m9wBhKQWBBGllEdchBCApsRURkzGY6XURqcB1AS+43thLYqMgSJPCYI3793rXfcZc9vtjUa9VZbV5eV1kiSMMVPWKl2VsyLyY631bDbrdALH9yspbTmJwxjG2FTCGEMwrkxptEKgEAIEmhKKGQWXSllZq1/muTHGshAcypRwuEEIkOs4C545AJLGZ66QnAAyxlRFhjF2EC7LkmJclaWUktiWA1ozh/q+n+d5vV63Dk3sMk2QxFBrNMZzn1Lny28eVZUw1GXMuexNucRZhbMSuAnHCZpm+XxWAOC4XisIxhrhpMBZhTFGfowASgA/rsOa+VzYCKUAu0oaKSQhZkWqsC4mQgihBUqEMcbYvqavBJfWeK8st3m5q7IQApMXBXXachOMoZSalX7e0jgBgIMRaGNTHtaGgTZaWyENZPszLW4BEBgw8GqkviDlveZb2KGkQMsOZNZFATDGKqNLYUmMgleEEK3kbDoRaZ6Np8V0jrR2MFEIcyWNFZMwYF6Wg5RaWXu68qJWVlkpYSti4EXU/lIranjZTfmNZagClbwkhBCHIYmZQx1EeF7u3t3a6HZ93zdaMgQOBawFhVUqxy6ZMQZpgxBChCAwyGjQFsVRWCuNMVCGEWikDSEAWAM2RkgkBZcEo4XSObBlsxKjtdArGizS2ihNKSWUyqJafXLA6jdYp9P6GdaBQAgAL0SXNQIAg9Y4Ioun/ErP6P/k+HaHzK6qDVqMMWzhbxlEzM7t/frGBoD37Pjk5o3dKHRkpWPfS9IxYVhjlFTz/nQ6LYqjWze9sP7zT56ddqfM2e4N+7TW+vj7f5hrbzYrRCU3GlG75n3+yd/eOujeun3wSJddGuj8quDFvOyVojo42o9qNSllknw9m/aK2c3h2dcXXz2Kw4ar6M//7V+4re5XX301Gg09z3v45LH46kvfdfzAdV3n5q2ju+/eCZq+33AYY8wh3W53a6crpfzyyy9rtVq9Xs+MyhAazufuZNJoNIzrGCWFKDa7DWyK54+fzscTH5lGoyEE3d7exhg/fvR0Pp/v7u7vHt1+6623NzY2lNRlWQKA4zjPLgd/9/P/X1VVp+cnN24c/tN/9k+EEL3ek6urqyzLonCSpwU2dLe+fXBw5Hnez785Sao8qwoEUAujuzdrlJCnXz6VaTUYDILAI8jMZ3mn09o+PAwajf7g4mJwdt0b5lwAYg7Bs1m6e3jjejBBhBqiS6l16XieFwQ113U32g4m9IM79/c3toqryX69+9Zbb3UbndPLC7rjPzk7+eLLLx8+fNhptwPX++LzhxzyvSK9Gvb+t//zv1AEPXv27NHx8ZdffBG4nqj4O2/eCVxvb3vncHfr+U7zq9nMiYjM8q39bQXxRqMTesFsXna3O9Sy27iUnEsAAQCUOQghpZRQGguplMKMagUAQMiiroYQLKV0GLWxr211KKW0Dn4URdqApcaNx+OiKJSoXJeFge8Q0ul0kKnzPA0DZzabEaST+fSte3cYIXEc+36bYWKkUtwyBEUQBM1mez6fV1Vp1vKvhFFKKXMcxghCYBCjDHme4wLBGHmBn1d5yStG3SCMCXMQITYvoLUGs9LGWTQgtiXFxpgsza1AMlq0DpJk2X2OIJuGUGbZoM9ItRrGGNCGZ9J+IRFCGBPHcWu1uuf5aZ6DQUpq5mCMiWUI2raCVtjAliwDgEXpg0bTomd8SZSzdqhWa1DiYIw9z+dczqbJdDpPkqzX6zHmCqGUNIQsihKLolgZLWs/VpHi6lqW4W8xG845DdzVlr1gGCxBDrPU7bYCWQBgyYYIGaO1WjR4BFBIARjQSkiNFcZg9WqEFpxzh2KH4lpUsxVKlvji+3672cryxHJm89y1vcrKstSY5Lyaz9PxeJxlhVZGGpBCe14gDBgEUilKMPNdjDFzHGH0Ouq+ZqjU6p31sWoGscxJL8y553mwDNzXQ7G1AsaX6gt+0yCEoLXyWoSQXmpBmleEdgEAQCm9ft3VtVYR7SuXW/EbVr81yzI/+A29FV6ZvH1tPyq2m/yKl2rlRFdLhJa64LZk+rfc9evzXL/o6rq/fele/60xBq2X8hpbm1MiSk5OTgzAxsZGEMe2HsdYwQCG7f2jtf4adkmWxTgvXUtrjQwgrDHGlqpiL2pxCQBijDEaXizIupzzWkXlyrkxxlidaliUd2orzLS8J70w7UiD+a9G7/j7j+lkFvruzs6OH3o27neYCcI4TedCc6IZdlnoB9vdDer5rXpjnBlklDEKCLguczyn2+28e/9eFAUzURJCsixJkmQ6nSKElBaT8aCs0rLIyiLf3d/53/zzP261GicnZ7Np0uv0i6xs1prIQDbP8mn5cPZl2NkeDvuEEEYYIKOETIVM0/T+22+ORpOf/PSXR0cH3/ne97XWjUbDcWhVFVrr+/fvWwYixtj3/QcPHthmYJzzIAjSNE2SxP7KxHI+nytlNjc3Nzc3tzZ3jg5vVlX13e9+D2PMuZxOp5eXl6cnZ+fn56PRiPm1Xq+nlGCus9FsXF9e/fSnP/3iqy8555vd7Xa7/fz5aRTWb926pTF6enL6q08fzqdTKQ0GM5vNsYZaFHqN5nQydynb7Gx6niOE4LzqD6ftjaLkqqrMLCkqqcJaEEYuN3SxESEDYLRWUlVCQsXBgCy5ievBeDa9d3RrmsyLovjss88qKb56/MiNw7Qqzs7O5vP5fDbjWTEajfzQO+UX23u7oVd/5+0Pvvry0c9+9osiy6MgSOeJrLgWsh7Fw/H47pv33nnnnc8enSdJMkumcSMCI87PTt/tvlflBSXMrapKSo0xNggqLpQ2rutKvUjscanpUiQHISQ5D4LAWpdmsxl6XpqmhJAsy9I850K5fsBcv16vSwOASJHleZ5XRRYEXhQGrVazUY8CjzoYfI8SrMuyeP78+Va3U+b58fExa8rxaBBGNc/zXMo455TSRqNmbuwraQAwYAQGrSzWJJ37vuc4FCPjOdR1KRgVBEle5mhKKHXiWs3xvZJXXApEGMWEMUYwXW6+mFFOCKHUQYhIKZXUVjGCMVaWJRcZIw5B1DYZ0lpVBeecVxUXVWVbb1v5F8tFaNfaCCHGmJWjTpLE8zyMcbfbLcsSY2yLAuwOgjGuSpHnudba932MqBDSyqEQwoRQyurjEsdzAyml0UgIWZYzQpjjOEVe9XqDyWQmhVLKVFWepqfDwTiOa5636N5ZKglgAIyUgnNeVWWeZ0kyZ4zmua6qihAMEBqjjdFSSqzZenhnlsQus2S8wtJlWUSrWkultFIAmmGEsGMQGNAgbUqLW2FmrbWQldKiKqXjOK4T1uLAGJOAUkobLed5arNLCKEsy/r9vm2dVZrKaMQ5r6rKGKDMtU+qUBI5lGgkCQAyzPcwxpqYlSu5Qo9fGQufb9k2SS+NwRI2eEk14RWbaoyxZv0VGqBZS6Wvmzd72hfsRetvIWwdl5WOwkuOwlpLSfhPpRiMMbbic/1NeFkA4JXgnqxJRL8yrObx6kEbY2y/Zquw8q1r8q3jdRcB1liur8/2t5/kW49/ZQLGmPl8nue5danR2oJrArCKztdqH5XSsLDxi3zQysHCgDBgrUAiiZDBBgi1ngbWSGO84HUR+hKMhNYqUZdUngWlA7RGL0iwAGDAaEBo6RkoY10HpC0j6rcv73/d0Wg1AfT9d97udDrtbqtejxnVCKksT1qdVnurIwwkRY4QUlKmaaoVM0Yl6cwlqCqzq8FVfzLw4w8Qo0DADz1EMAAk6azb7Rpjvnn0WbvdfvPDd/I8r1RVpslYinyWIAV3brxBkffu3QfFtLw8Gwihq1xUSeYTNwxDnzlCCOL4jDGhBQKys7crRDWep4TgqBY3u9uDYe/y7EJK2e12GXODAG1t7UwmM2NQvd4cj8ePHj2JosgY43nBbDYbjUaNKFbK1Ov1Wq3W7w3TNK3X691u9/PPP1dKjcfj4+OTs7Mzy9AnhLy5fzOfzx49Oq3VojJNyjT7+uEXXz58eO/+W1EUMdebptm8rNxGPTfw2eefz2aZNiTwa8RoyQVoQRChiJw8fb611U3n2WwyzdJKSvn0yWkyz9sbLQ2M0qCUFZfACNLaTOcpIA2ADAgDYAzVBgxgA/riYrLRbHHOB6PhvXv3vv+d7w6ue5e96wcPHvi+/+TJk/ls9uD9933HffjZ53mahkHLGDMfF7/65OHW1tb+zo3nz58zZlwn3Ly1tbe907u67Ha7QlY7O1v1en2aVQraZxfnewf7vVF/OhzsdrvYJZQ6LC8LqfQq46AMYMocTJRB2HbXJVQIIYXkvHIwQggRBHEcd7tdhrExJgzDXq8njdFm0ZKHUIYQVnoR2SCEtNZFUcxmSMkqd8nh7nYcR5SYQS+1EYDN3H/59JnWemd/7/bt2/u7O9N5EsRRGIYXV9dSSiG1UopLS6pQxhjkoEhEURR4joMlGKM5L8uimM6nnHMFejZL1WwupcSUeL4/y7IoinwvQIvCcVDSSCml1ITYhkyeRRRsdGWAL46UCgCEEHmeKyEopQpj21sPCDXGMEw44laIyRozMBojWosbcRwTh00mEyEExhQhIqXQWiNEpKk4t54KxRhXQhhjHMeZzWZCCABw2UJEwQZP0zy1pQcIEc75fJ6WZQnGCjiqsuB5nltxYkqZUgohY5nbSgkpuZS8qor5fEopJgQZo8oyR8ii7ohSR72s8ouX/aVWwAkA2I5fi31QCq01YKwx0kwb0BgwGEDYMIIrabguCSwq7+Mg9IhSSiHFNS+kNnk6z7IMYzybzQDTsizDMOS8Ojs7E0pGYaycBZyDHQ8ANOBKaa1V4DjUcwGw0mCMZpQCRqUSFL8KWaMllI2WkPtLwbp5qdZxna9gh17TIMLLHlHrvhSsm7iUuoUAAQAASURBVIRvqzA05oV2kzHGdpSWUr6oeli7HHnBtnsxVo9j/fViDgS/fiS8bFZfn9grfwIAtjcsLHnNVVWlaToYDJIkqapqVRy4Xhv1reOVlbdvrjs6KwfuN53h9bO98mLh/xljPXLP88IwbLVatVrNuuBCCMsyoXRh1Jd22hhYdI8E0IAWNBQ7O/t0YNFlGymljMHGGCCaKOK4DJACIAoppO3jJoQQocTrS71yss2K37BcNAMKAAEQMICQBmS7q9k3Fy0fjYFVXe5/6yGVmc/GyhjmuYeH+5ub9WQ2zLPpW/fvvv/Be0c3b55dXpVVddXrcaGqgldGGi2NpMbBWZENh5dRQFuN+P5b94pkzrURSpe8Onl+hg2+Pu/H9eiqd0kdsrGx4TN/MphQ6lBgb9178+zsajyc+U797bffb7f6j7958s3Xj+u1jk8dlxKRl5P5zCDDXKcUPEmSSsh6q56WlUGwS/BwOh1O5kEQ5Hk+n8+zLAOAKIru379v2XW9Xq/f70dRhBCyMFi32/3ehx9XVdXtdjHGn5x98vz5852dHc8LvvzyS4uOX19f5nlqJeMQQt//7nda9dqk3zegZMUZQgSg025rIQeDQSEl17pIs/lXX0Vx/ezyQimipAElidEOYUEU1IPApWxnY7Pb6RR5cX19jR1aa7TKQj784vHBrX3AXtRo6yQrhMhlJqQ2UCGMgQAGjDDGxBBiCFWU0XZ9AxGS58Vf/OCvvvP2e41GY3drW3/66/Prq7Ori2enJ1zJRlxr1huB57UaTerU4zgej8f/8c9/0O40/+kf/3Gr1f3880/7w0G73b7z5r29ne3nx0/T2dwY3Rv2O+2a1ArQVqtZm6WjwPVqUcCVpILLquRCCMIowUwZMFJyqV3XBaWUAS4V1kYDIEIR0VbBChHsui6lNE/TPM+jKGq1WlG9nhdVXlZcaiFVwauyEgAQBEEUeBiDUqrX613KiiA1Hw93d7o3jvayLKvVakKITqu1s7PTm8nZbOZSEvlBo9Gg1xQwKatif3+3tHIFSgNCxiCrATUrJxqEkJUBJSUhCBslpUL1uCWVSrP5ZD7RxoS1MAxCQvEwH7iu67nGFhFIqQlmGGOlpA3TPc+TUtrotigK32NKaAFSKYMxRhpRRBFBouKikkIoUAZjjBEG6hiDfDe0mK3t32yjtCAIKrkQTrZuwaqLMSXE1m0y5koppdAAgBFNksweU+DSxnaccymlYQQAypKnaSqEoIQRTG3jR8dxwiB2HMfzfNtmuigyyxuwWQy7kdn2u8YYK7602uAQQoQQtTQVsNZpaWUO7cxXZR2u6zqEGdAYY5cxSjAyYEAZpbVGlGFklOICUWy0dF23u9GuuWY0GmVFipECAMHzqkzBYKWEKMvrfr/T6cRxPYyjer3eaDRO+yMb8GGMtQYpJZdKGyOEoowiZKRUCCHACCkspayF0fquvbLxsGauYM2gSmXQWn7BLKEFSql1ec2iHNSqbRKlvuWcC4RgZbzXriKlBETWvRC0TGmv4eGLCSOEKH2JY78KT18xtC/s8Rrrfv23694MfJvNfmXYD+dqxax8wuXlpWUvrhZndV+/xfas/3YdGjFr1QevoAu//SSvXNd+cUBhjLW1AbYhu9ULt279umdjXTGMMSzRf1ikbBYugv1XL7W3bCpAa+tbI03QQu+ZEoQ0SIywMgbbPiCW3PrKkq6+L6+vg9VpBVAIsDGAgFhMCCECoA0AAuvpKvj7+VL/KwciyPWDVqv+B//kD4zMDvY6eTaaTQfz+fTdd9+qt9qn52fJfC65QEBlJRXCCBvHcaJ6pI3sD65/9flDxlit2XAJ1lwx6oFhg/60EbeVgsfHp0br/SNHAeldDHrX41s3br/99tsPP/v6xz/6O6NxHH9zsH+z3mq+/9GHt+7ey6bp5ubm0c0bnuf1hr3rfu+61zu9OJ9MJr/85c/3Dg+ObtzY2tkO/OjJkyfGmHnvstls5nlpjBmPx2madrvdshweHx8/fvwYIfTmm/e11kqZg4OjDz/8kGj85ZdfPnz4JQAMBiPOZa83mE7nBwcHGEO73dzf3w2CwBjz9OnThw8ffvPoKy54o1krilxWFQgVen4zjikhWtmCeXeSFf3eIIzmSZZqHSBMQRutJCG0UYv2uhuNeuwxh/MyTzNCiOcGCKGKCyV1ISRxXMgLjbEGYwC7gWOL1Y0xxvKVtNEGgSKgCGF0lswdjTAhk8nk2TePm41GkiRpmj55/iyryt3d3Uatvr+7Ww8jUVRVRULPp+2OMPLk5GR3e3tru4sxfPnlw9lsdn19/d7b94ejfhRFnVZzOBxubW6fnZ0ZbK4HV8Ph8I237o2HIy4F1YAQodgAsR14mWsWHeqQULooCi6V4zhBENiW0wEjxpjAcxljk8lkPBhMp1PXdQkhju8TQl3XBawQVpXVYy8rSqnvu45DpeBZmXHOjaqm02kceQihIAgcFmutGWPb29vjVF85TuD7ZZHVHIc5xABO0zQIQ0DaYIQRch2fuo4v/KIofPCwlXWVWkppEGWYea4bR4ExJgiiWtzwQi+uR1yUo9Go0WjU63XfC6wrYJZxCaUvQOmiKPJ8ZM2ht7mhhZIatNbUIQYThzqIIafWkJyXZWmxytXGpyQyxjiOI6gwgDGmQqiy5FxJrQEhoqRR0hDMAAAjKkShlKkqIWXCOS/SDAA4l77rAWCjdcm5LUOw243vRQCgVYYgR0AZcymlUuqiKARXYARCyOKbFi4mFFGKtUEIY88PgtBzXddxKRel47Iw8gHAqjNprYxSiC4+ACvXwdrOVTdCjLHtKm7tR+B7qzAdIWOU1MYYUEYqgjwpORjlOp5ACGPkeW5VDKaTfpJk9Xo9rjUaURCHAXOdIIiSNGceOzy4cXjjqKpErVYzxjz7//65UsoYIMuWjy51KaWzNGNMEUIsbIu4IIRg9Grpwer1er3AyzH3yta/qoWw+hGtDYxfVUSG1yLI9R+tH2AnD6+WSi7fWTPeK9cNXram9kGsT96+UEvVS/PyeO1aq39h/VfrC2VzTGw5rPdjdVbs1WEtFfKbbPy3vr+a3gqTX03p78l1WJ0Wr6ln2robm5ayqYc4jt2ljOnCz0MaY6wXDwWMMUYbAMBg7LaxWv4XabWFB2DxMw2UGmMIEWAMQswYBRoRgjC2at/GOi7rn5ZX/Cq0RjEBZMtiLTRFjLG3j5dCXsgSRWHhz+D/1lUTBgBTKo1ClFxf9zxPd5pB/WDvydMsS2fnF2c/+MFffv7rX5USgqhWFoXADiGENbzAj6qq8vwoq6rPv3p065efHe7ugFQGOQR7vDJakRsHb4z716PB6MtHp77b32htvP3W2wiRH/7V3wEgLqC70TUIC62y+bTb7b5z5/beZteavevRxWQ6qTXCo5sf/Q773hdffPHpZ5/5vp+l8zwJN7sdrA3GOAzDZrOpta7X661W6/r6mlL6zTffaK3v3r2LMZ5MJlrr3d1drfUPf/jD3e5ev7+ovGi1OkEQzOfz0Wg0m01arVYcx77v+4EbxzEgLRV/dPzk8PDw3v03z87OKMPMIXvbO0KIqF5vdDZrnc48LyfzglJHA2RFqakX+gHWpEwqXmVGBmHgdFv14aB3enZWctFptRTCk/kcE7q1u7N3sD+aTM56F0VVaIRsy/Myz5jnIgSwQLYUQaYEpLW+vLra39kVQrfjOCvyw8ND3/Mopc9OT0ajURBFW93N3d3dvZ3dPqZIG6Ors7OnnufdvH3j/OLZo8cP79y5849+93t+4PzqFz///OHD+/ff3NzaGY8Go8k0K0rHD/y4Vu+2n52fjibjm1rPkvnO/h617duFVoy6hBBMlVBSA7JZSr3cyzBhCGOjjJQyCIJWq0UptURFKaXloIW1mueHiFDHC8IwdPzA9YLryytjwUBkzMpnR8iWDMzn81qtBkZZR+TWrVuPTwaewzCY0WhkAIQQrufxrJRKaTCVkFIbqbWHASGEKaEItAbDQWrJK66FRgYzTHgpAGlCUBTV4jjEDKSoXNfdCDZc17UsBztstB1FNc6lpW5xzvO8tOQXhok9TCm1aHsoBCOkf33NOa+K0v4JwMIYKIkwxvV6PQxD2y06iiLP8+ajoV1J25fBbsRSyjyf2ZKHBUaqjTFmMpk0640wDH3fx8s2TvYSVSWs3+B5kePYqYHjOGCw/YI5jsMYQwhbVFZXleVPqKV6ozEmz3Mrt4KWxbF6peS93NdWFD/rMK0fAwBKKdud3GfUcRyKKcZIKS0Vt7RyKblWRCtFKY2iME/SLMtm4wmuJlkyq4oCotB1sOtFvh9G9dr+wSEXqtFoNNqtZqszmUyiKEryzKd+ZbjWGhmkpOKca1QSzDCABkkAOYwZhEAbhOH1uvmVc2OWfIUVQWG5cZN1OGFlDq1ntiIurIWnL1IbK7OnX66pe2kjNmZpipbew7IqYeWBWNYCfBsDYPV6naH50q/Mbwz0103st57zlbPZ27RP1jY3F0JYFs7KEK77T/+5Qy9v/HVT+pvG6weYZYEJppSQBWOxqipbPMw5x4wJYTO7xhjDqK0mQIuJmwWsAEsOo3n5GSmlDSaYIGPWq1GwEAojpBQBwAjZohIKgK3SNyw5v6vpWddZLzukw5r7uOZmLb0EpBDCxuDlrFbPSK+3g/xvNBhzymIaRdF4PNZiLqpGHLmYaEyA8zKZT4UQUdiIo3qS5WmlENKCmzSpJuM5Iq5H3dE0++Ff/93vfPc7SGohsJB4Oimvzsc7m1vUb+8cdBzHO312PBqd7+7d29loYSYeffV1rzftDZN/8kd/9NHH3+WcV6IazyZnnzyxajGMMXB0KbP+uJJSttuNt+/fswKAmpc8TTabDaUUd1gyz4QQ08lcSlmvNQGg2WgPBoPdg/0gCM7Pzw8PDj/++OM8z//Nv/k3n/3i4c7OTq1WK4pimQ2flWWptQzDEAAuLs+eHeOtrS3fd5vNOgvq9959q1lv/PgHP1RC1mq1o6Oj60H/6uKyEkYZXCQpaOMzz/WjyC97aUWxVcCvXIqiwIk8xqiZjnvJdOQEvueStKwoxWGj5niOodpgyWVliAFAZVU4WBe8Yp5rEDagsSHGgFbYaKI1diOfMJrOJmePnrbD2vv/8u3vfPTR48ePf/g3P37nnXcOb9zY2NhwKB2Pxx5z/uD3f/9vfvq348lVf5CkxTjNJv/xL/4sSccHR0dvvXUnjv1hv88Y++ijj379619fnl9EcX08m1daR368uXugqbO9u4cdN6rF1GBEmAsLyjo2CEBpIYT9Hvo+BgDH9R3H4ZyXZclFEYZhHMfGGM65hftsnyGU52UlMGV1TJutduR6nh92OxuWzKiUUFIQIzHSSiDbieDrr7/e3e4O+tetRu2Z60ZxYJRyKKWU5mVeFoHRGiFfCFEPY0yJq3RRlQYRjIFShgnRqmKAiEuU8Apc5UlelaXU0mWeEJxQhAjSWiKKmq360dHRrEyNMXlW2O+qlHqlG5jneVEUgi+kgXzf73Q6jGCMsRBCVFxLZYkCjBBLZQBt+Q3SKG0AEUpLoRDSUmpjEEZECDWbJXle5kVpjJFScVkSUlkhRc65MYXd5qSUnueFnq+1zvN8Nplub29vbW05S/0ia7rSaZplmWWSUkqtEk4UWhUpsM6N67p2lwcArDgAFEVmpRGsyeS85LxUSuT5oqLagreO42T8BUXL7mj2ADtDjLHrugBgmZtFUfiEhGFIMVnkU5SyOyJZGldKSeQHZZbPZjOt9d2Dei2KHEqjOAg8B1PMGPF9X0oJyMxmk9OL89k8vbi42Nnfw5iOhyODwLqzxA045ULJhTyDMYxQz3ENAltv6TKmFsr82rycd1jlDqxm88r2S/Win5OFSex2bwVuV9yRVcBNyAs6wipQXmc2rNs2s9b1cQUPrGoK0JIwD2sHrMbqHfvvspvlq8e84pq8jiu8amt/A01htWhVVQ2Hw8lk0u/3x+Nxnuf2Wdsj9Vot6G8yPK9Myd7IOr109f5vOsPqPK+8gKUeAKWUuS7GFC1X3uKdWuuqqjDGVEq78nSJhbx6ZvTSVfSSRyKlAmIoULPotb64WaWk1iudR7TMICyBrjVcAS2bnqw/hbVraWMAIfytHsByfQz895JfyCoZutTzgw8/+qjKB0rMGFVJMuWch6Fvt6B67TkQR2tdFYVBgTaKczmZzfuDCaPg1+Isy/KCe0FMpO7U2oabcX88Hs02mt07995jhEZhWIs3yiRz3fjicvTpp5/1r3sYo8GgX2s2pFZXg+tpMn/+/PnhZnM8HRtjgiAghHhe4HkBxl4Yhnkxr4rkwwfvAcDjx49HjFVV5fqhlPKNN97I8zxJko2NDSHE/fv37Z5mayYdx/n8888vLy8/++yzuzfv7u/vM8Z6vR4ARFEUBB6ltNNpeZ43m0+KMlNKScnTVIxGo/b+0ebO9hs3bz169CiZTJnr+oGbTGdSiFF/AIiqSiEDRZ4rQKBRFAcMI5Cq2Ypv7+0+ePvujd0dn+FPfv4ThBQYdd27mMyzuL3Befns+HjfTIVUwlSO51VCCcG9wA/DUINBBqwoHMGIMYcx16VOUZWDweBgY2t0ellV1Z/8+Z8hgKdPn3Y6ncODQ0Co1+ttbmyA1MJAp93+6MN3upvNzz///PjkfHe/m8wnz58/+/rx1x999NHR0VFVFJ9/8fAPf/8PLDbTaDQGs3Q2z3Iu2p3um2/f72xv9oa9bx4/poCIBmUQVhpJBUohBI7vhkooTBHDDECrqsyrHGOIfFoArZSUGF9fX08m03a7nWSF02j57XZVVYwx25Xg6bOvGWPNZjOktOl5yG+WhajyojKOR1zOU+bos6vnUf1mbzK6HAwUdqLQ+/zxs4owTPxCEOo25gq5bjDnyqvVgBE3CEZX557neYFbFIXDQCpOsGuM4UopIxSTLMY08MAorhK35mJkKp4oSYLQS6eTSa83zVOb/G4ENQcxxlgl1dXVlec5RVEANpgB8QnDzI1cv+ZDJous1FobA1JWWmvHcSgjGGNkVC7yXOSO52iN0ixTpXJRDACMMUqx57kEwbDfi2vhcDgMQ991XRcbBNrCFCAqLYyuFOJADcECAUUu84JGmKapUWg8nJpFcsRzmK+Uyqosz8okSUbDSRAsKJnJPLPbnBU+Qgj5vm8hWV1yAKDE0wrPZ3kYhp7nJfOi3x8v+F/OYvNFQKUAKVVRFJRS13VWcWRZ5dpITIAxxByEMcaE2eAQiDJYVjIHbiilvu8YpXlREkJ4ngLnRZ6FBwetw8Nprxcx1m62HMqu+/3xYFLlsr3Rrcr0/KR/cdnHjFWlmkznZVkRHJ0/GuV57rTrruNgjFPOlars7l9JjTFGCCuFZCVs0YqhpJAA+oW8P4DBGBCmGBED2P6nDVIa1NJurQoCzZLTZyGEFThhESDrZBBCtH6pG5BetiFelVGsXR0QQtQhVlfDgFK2hTGiGGMCCxUgS7DDeAlyIGzhbFj8cmFmtNEYY0uEXNldrRUyAMZoo1fG3v4VfVmHwJ7QGGO0segBwlZEAYwxCJCUiiIihCjSPE+yUX84Hoy0UJyXAIAxLF0ie1vISmvaua18IQBYfCpeY4ToZZdRAFixYmGZIINX8yOvuk2rgbABpA0orSWAJsTBBCjFjkO5KKXi2NCyLBzQvu8b0BgIXqN64IX8BCgwGjRGmKJF4QlXwkiOlBSEULVMRGKiQHHFXdc1AEJKpTHGmAI2SAplJMdLb5Ji/MIHWjF7DKwKYRHGWEuDEAFjUxz2g4QxQkYbMGCsNIadpq3F+A0Kkr/ZzXrBRF4/+DcdX3eoUooQXJX82bPT+/eOkvnAgMt5OZkUeV4SYI1aEwHlwuiy8rzIGKRzmedpw2/O53Pku3vd2/ls3gw35sPRaDjc6XRPvvg6N/p3P3jfN83RaHRydX122p9MZojG9Xo9kQjH9ZPT41ar2d7dcQI/bsRpke7tbaE8o9rs7u4ihIqiIADpdNLtdvN58uVnn9fr9Sj0q1IYpS97F0VRWH4bQzAej7vd7vHjR0KIRqOBEEJBkBndiMLJZKKqajoc/O73Pt7Y7Gb5rMZC5uobN25wzieTSZYWt2/fJog+fqzOT/phEG91alrBO2/eeXj25fOz03maFEJmlZil5STh1K83vDp23CSXQiNjEABMR+Mgjm5udK6vL0VZbOxt7+5uv/vBg6PDvb/50Q8h8MtkFkQRBtR2Q8rc4fUw9oOCo8kgMQXBBAJEgYTVrDAUIeJM8zlmJPRdYoxrNEEIJG+wlmNoOit/5/f+gJcVMfp03H92edJsN8ajXq0WRVGIdJnms8lkklbtm7cP33rvzbgRxI0vTk9PXS/Y6W4yxv7q3/9lnufGmKObN3Z3t+udxiyf3//gHeU6n3zyCVj0uiifHz8+OTlhjFG7JyJEMCJa2xgXYYzIgsy12CUBIaWEEML3/VqtZvF2xtjqe26/V9ajtztvWZZZlgUOc12fEg8ZghBirtPwG9r4RTHZ297Z3toihNSj+MbRbVnpbJb2y4oSQm3Oe9GvmWgAY4zDmMM8W2RoeXZlWQImxnI+NKIIG8YwQxgZIxXGGIxCNiLUSCvAhMVx3XV9x/EwxlblTSOMMe71esYYy5l3iYso0kLOZrMaCeDFRoOt4grBRGlpMwY2L2BjUK0hrIcWH1PKW8VSNlRljFlNobIsrSaE67oZL2yRxYoyZslZVigTVqIRhKzq1/Gylh2WLHSABepu8ybLGEjBks2+MooWAbJCeytQdIWaGmOUAd/3fd+3zVEAwPoWvu9XVcV5aSlvxhhKseM4vh9qbQBwFIWUYl6UgnOMUJYntSjmVdHptHzfLYus1a6laTqcToxGQRhTL3QdHyhN02IwmY5mM8fxpDKVUoZQ5vrUA+wwuVaKqfXC+lpjtxrrlvuVbXQ9oH/dIMHLkD56OZUO31YpsHIFXnlnPdeO1iCZ9W3arJVrYoxX2jj2jfXdfP34lc/xiiOyZF28ylowyzK/1XxeCanXz7M6wDJeF4RZs+iGYMU9V0m69c/J+glXP76ytua1UP718a3IxOvLvv6rhTNkCTQI22nb61rusHXp1s+wfqfrUzXG6G8rY1kdbJbaIUIIwAuoAGMMZlkVqcAiGYSQVaex1bcPoReNpRd3ivCyHnZ149rYOkm0eJoILTIOv2UdftP41jX/LefhXAvJw5CFYbi9tZMXlQakDfqLv/j/E/dfTZZlWXogtrY8+mpX4R46InVmZckWaAEM0GOwAfEA0gZmMBu+8ZF8wSuN/C8gH8gxGjmC0wM0uoHuRovq6uqqrNQZ2sPD5dX3Hr0lH/a9x697RFYXGuDMtrRI9+vn7CPvWt9e61vf+uPHjx/zwJ/MFpiQupZFJQEjY4wyWghRSRFQQhh19d6y5hcXF9UyrZbzkJIoCc9Pjn/y07/y+nuvXr1yld5FnX3z6Avf90cXw3a73YpjhvDw9MTHiFjwCK61effdd09PT110s91uz+eLyWTCGDs/H+7t7YVh+NOf/vT05Lzdbu/v7zs6nrX26OgIITSZTLa2tlxodmtr6+LiYjQaMcbKsszzvNPp3L9/v9vvPH/+PMvzZZpOptO6FGVZep53enoupZzOZ2EcJklMGK6LajqfDF88P47C55UMeOBr+Pyv/vrs7AJXdZC0oqRTKGW0LgoaUoZ9CH3v2YsX9+7cmk3Hp6enP/j4QyHEi6fPzk5Ox+Nxr9PtdrtFLaSyXhR7UQtjPMeySeZCU0QN2Frrcc59bq3OlkvNaMx50mq3wzDw/a1el2I02B48uHPXSoFkjcAMz86tVqoqBSNZtpyOhsjoW3f3xuOh7/N/8A9+O8uK49OTdjsOguD2nRuOEY+xLYocwC4W86+++vLGg1tJHFRVdXR0OJ/PKcKL6XSxWFCErGO2Y0yV1I4g6sKzdV0boxCyjDOMQUqwVnNOkySaz6d5njLGiiJzeIJSihAxxgihKKWeFwih5vOl7kRpWdW1rnLBMOm0291OFPpkb3vH83bu370d+YE1xiP+0ctTD1OCDcEMAdZu/W4MxtiCDqIII8IZK4pCSdOOosDzZF0Xq0izQcZaiwlCGANBGDOsZW01YEYJstoasI4ZAK6ogXEupa5r6b72SdzmzC/LuhK18tT6G4asstZYq601K66TVVYbWde1qGqGSRzEAGCtpggrpdrtNgCEYdhutx0sYIy12nG326UUM8ZcjsPV8xhjKOYNdcCRCRo2GQC4vEZj19wr5coompin+yvnqwCAWVO+3aG1Ug69aa2DIHBQw6lxOzDXWGq3o8ar3KqTm3SVq9pIp4+ktWzOxwEaqzCl1KMhJZwAwlgBCJekEKKy1t68c5N7+NHjw6ou8zI9OkdhGK8I6owLDWldL6u6UEphpS1WgAxGGFlARBPiCGhrIsXK4K45ZJcFC2sfb8hVL9g47E3P7cY1Z3kNOlxDFRu/XokZNH/dTFG7zzfJDdemWk+4+cn69Da87Led7Ro2rdMrTkoBuXDBKhaxeRVXfgAX2b5O27TWOl6glMJagxBYa7RWdmO80dNsjtcd0t+64+uO+ZcfwmW4tNawZpbAui04Y8w5LUII5avmpZuoxRjb8I6bHWEDvcFrKMoF/lY8x40un2ZdJaE2yKQYX/IWG/4KgisoxFyiOg1gwbozXMkuWWvWL5gGQIDRfwpHoTnuL9mGeRgTD8AoAwD4j/7oD/NsVpTz6WyaVXUcxwSzVqczmaW2UmErKQTSQteqllp54OjaFDCtRP30+bOYs3Q6Ytjs395LEp50w+/85sf2r8V8tmSE1SJ9dfQq8LxW3L51sPPWg99qt+L7N2/VZbGYT7BSN7o957YbaqqUMk3Tsiwp5efn51prVwq7tbXleV6WZb/26z+Mosha67o/cM4fPXp0dn4ShB4gE8VBr9fzfT9NUxdIfvL0KSGk1+sBgDGQ5tloNKKUR0nr1atXo9FomabGWj7m1lqhqu04Xp6ff/3l1w/vPfS98HA4thZt97cm0znnvK7EZHgxmc1KKQlnjLHf+p3fTpfzIk+3ex2M8R/8639DQIce/+5H33nvvfeENs9eHp2cDcuq4l4ktGqMGJgrNk0qbcAgCwQTRqjPuO95oR/0kiD0/J1eZzIag6SmKs5OjzmBrf42VkLKmnMeRQHSYkEwAnN+fra1tbV/sHtwcMA5H45HRtvj4+Ot7fZgsA0AZSUwMdrUnW6ctKLJ6ASsNLqo8lmVzznzCTYMI0dl19YSY4w2Smtp1xr4jBGEKCD3RQLGCMa+H/Aw8k/Pjp3olROmcIh78xsSBJG1SEqtkbUIpDW1UbUUaIm0qXwKZUbCkNy6sX335q2Ae0cvjvNlevvg5smjJxYbKVVZltoayjBCSEjZaseiqhZpmue5VgqszdLCGIM4QwawRQZhgpAFo7XVRvk+V8oYY3zOACGtNUaEct+ALYqiKGvPQJqmrq7GGNPrDrDFohRWaABwRbRSSmxXMWelnFUyWmtVK8ap6/fo+37TbU5KiYzHOb9x40av18EYl2UJAJTSoijG47kQwtVxdbvdIAiklJPRrKlCdOETa21DH2v4X7Be9jlVRLdZs/gj6yZMZi3kDhuuBa/jxa4eMk1T1yYY1sVjbhHm5qlr4VpLUEp6vV6327XWXgzP5vO5Uspa3ST4GSOUUitNq53EcSxELWTNKEW+XyoVhuHo4nyw1dvZ217MZxfjoTEqjuOi0gZkqcCa2lqkjE7zKi+ltNhoa60RAKtVstHKWmJW62OlFMBaePhN62x3segqIHDP7hpR4PVxbRJ7denfDOfdNx3n5jZ2I0vdfPj6Ua6dxnqey22amTd/aHZZJx1WQMElBZrPmxNb9728ckCEEBizzn8j55TdO+NUlazV1mqlhJS11tIY5Sirm/5704/C1bAHrLFsA2jgb3NXeF3FsDnPL0EMjNH1hWMXP3D4oFmWuVf62u7rCTc/XZdrbgDKa89rfZ+h+YptXri7KKfM2Dya5vI3YUeTt0IIGd3cf7AWAdIAyFqMkIsPuTyJ2/3vSFO4Buzs1ZDJ5skDgCNNaG0p5VvbN16dXKSLUVllUeRNZ8vJPHXie4s8L2SNEMLMQxpZBNpajbBFWBiNtSTcQ4S8//FH5WwcUPvBO2+FnLRb8faDB5jW49GUU97rhYGHAy+8c/PWvdt3eu3OYjZfTIZJHIKoTl68GI2G/Z3te/fuVVX16tWre/fu3b//oN/vp2n6J3/yH+q63t/fv3HjxnKRnZyc9Hq9g4MDz2OdTqeR0xBC9Ho9rXWWZU7pbrlcWmu3trbu3buX5/lH3/kgz/Ozs7PHjx9fjCeU8CiKbt66tXuwP8/SWqtWr+15XpiEzlreeOvg6dOnPGS5yGututs9KbQmNux3/CSeqtlclqkRNuB+ktBWdDa8KLMUE5J02p9++qkVJcdwcXb6e7/3e0kUX0ynjsJ/Pp70tnYBoZIbYwymBGuw1jJiwSIDyCJUCCVQHQVeK45jzqlFxTL1e92tTkK0RLLKF/KL6WR4fraztbW/tdWKwjy3i/ksXc5d4b3PaRB4nudauGlAinM8m82mswuLkZA5WJzly+cv8sCPWq2O57EXzx8hhLSycRwEgaelCYPAuxusWgJKKQGwUkYKJwtIXJ0CIUQqUVUFgGGccM6DwMcYGaMJRdyjQq4SBHUtCSEIkaqqjBFgCQLscX+eT0I/arfbnVYPGUsMGFPVdeFRSilxwrHucMvl0lrUilqMMaEVxhRjCMMQESyEELJaLjKtNKeUYFoWlVKq1WqJyhV/Y4oRIgislVJKLapKVLXEYCillBLArgk2ckKB1iL3n1IKLEaAKCGMMJ/7xjcIIc59a21ta0pY45W1lsaYUpaiklqpuq4xEJ9bUQsA8DwPAy5yFcfx9vY253QymYzH4+VyqY3UWruLdZ7ewQWtdV1KvJZrdOjERRFcBgfWLLzGCeF1ib8DE25ChJBLOLkNHBHPGYKIMfeF0VonSdLMH0VRQ0FvBiEkTVOXYOp02q1Wq9PpCCE8z1ssFlprjMFJUTnIwjlvBQlnlGDkcapkXRQFQTYMQ6lqA3b/xs2qEucXQ88PqqqMk/ZoVueVVlldVaIUtRSrBpzGACKACLXWFagjQNgSqmSJVtJYpjG4YK/oIK084fq3xkNfs9EIoWs2tDGXjS9t/vQ6Slj7gCtGeRNPNC4c1qRFay3Z0GJa7wOwijTa5iQbHwMb9v2NP5gNtqab2L5pXLs5zQU66uTrTqWp3DHGNBEvF1W6dls2J3/dd8KbUIL99tD35pybR/m2jTcdnrt1LqFm13LOFhHGmFm3bkJ0FcM3xlWfNUGI1YU3aSb3FNavyGVEyr1pWmvXNNUd10UUjDGIbDb4aNAbZoysb+8Vz23BAKC1OPRKfKkRXLKgAVZSCmvE8B8XUXgjwGru/+aza/5qjMHIApDt3T3uhZUAZVCai0qISlYYY8IoIAKUaGOUVQYZgwFTgjC2ANqAVBoQQozdvX/HJ7frbHbr7kHoUS3LVy+/BCj6/aAVtyMf9TsBw97e1i7DJF1MXx0+nYXRvTu3F7PpdDI8PXl19+GDnZ0dl0I9OjpaLtPlcjkej+/fv++ewnQ6jaPWD3/4Q8bYaDQKw8QYtVzm33xzijGez+dr5cDK9XsTQo7HQ89j5+enk8mkNpJzvre//7/75//1cDj+5tGTr79+9PLklPpBXpU08HZ3B4PtfrvT6nbbCKFsOq2Pnt/+4J26qK3Bv/4P//5X3zz9m5/9YvvGfkZsanWOoWYMGFNWl2VJde1x2mon/+yf/TMkKl3lT7784qTTDrmXLpdFmgXcI4TM06UmDDNaM2uUJWvEjDF2hW+UMq4pRRiMNUpZhEFpoNhKubuzPb4YImvbUSzr6sburhLy5NUxJqCUmkxmhNJbt249ePj2nTt3aiLOz8/n86mQlbX68OjQqebzwBsOTyeTmTbgeaHvhe12PpstTsav+v2+MQYBCYNAE8s95POAaq2kVMYYjKi1yAIYA0oJrSVjDJCt6zLLlhZ0q9UKAp8QtFzOMcaEYGM0IUTUCgDcqtQYrZWplRC1xJhqbX3KGWBQ2iiDLSBKPcYRhf39rSThSaedFwXGmPuesubpi+c5Dlxa3RidJEmn0+OclmU5nc/yIiUYI8uQQUqLPM/jMKyEdB6Oc84oa4yCE4oBjKW2FmmEkLaolrrM8zzPGas451UpjAbfYwhwWVRGKYZZJ+kQQqwxdV0bhJU0GBFXK+ViLS6S7yPfVRzUdT2bzQBMt9sFgDyTLr+wXBaz2cyRxquq6na7OztbnPOqqrIsWy5L93V1ezl75xCAq5Z0psf5J7LqVOQ8xKUlbTyHq35sjHKDMJwJc2lmJ/TkTsasq7ZcuqERaSCELJdLjHEURRjjNE2dhLaDBW5Lt0sQuOoKrx14eb7UMo/jmBE7zxYY436/XxRFt9tP2p2nz14+f37IORUSTs/G81SBXeEeobUT2DSAOPMtQmARWGwM1JW2Vhmw1F4uEMlaVNGYy0KDN/rFzYzDho+/dAwbjvMKenjdzjZbrl1m0yXk8ojNzJuueoXDiKOXXnG0zWzNo8TrLoXXgMLm+bx+FADY7Ga5OdCq5NLx55v/AK0UpS240wZrwViwrpuBVHVepPPFdDafZPlSSOHInk0A45c4frtmNrzuqP5Wx/+3btYM9waidVkv1RrWtB5HRHWvcROlu3ogu+KEbvb+WP91lSxYNZzcvOdgrVVKIXz5qrjVkUMM9hJtrHpxYYy1pqsX4zJ+4IooAAAsMhYAwKlAaosAAbVWA4I1XLh8ZP+x49rLvGkrXr/DBoBzZsEoLTwvipLuZPqZlFXUDqzFCDMN1gLCBFmEpTVaS20NYKQRaLDSgkEYYQqEjqfTFy9f3trraVnOlqPKp6OL87qaK6G0tvNyYRXaGSQeC1sh85ivqiJphcjavMqFLPZv3uh222mavnz5crFY7OzsxHEMgKIo2t7enk7nvV7PWrtYLMIwdHTF5XL5Z3/2H5IkAYD5fH7z5k2M0e3btwghh4eHs9m03W6HYTgaDVutZDabUkqHs9F0Or15cOcf/t5/eev2XepFeSn2bx60Wq2k2z46Oax0LawyRGMPGaPPl4tRsWy3ukLIVtx9+NGHF4uMhuFn3zziQbIo67xUBjNtiCwVFvWd3R2j5cuXL8uy/LWPPvz0b37y5NHjWwc3dra2RVnFYeTHycV0HgRBXpW6MCYgjHBCqMUI9NqwWKOVYoT6vkfAClkizrcHWwe7e5AvkjAaG4ss9Hq9XqeravH5Z58VVd3r9Xr9HcICQsj23m4URWGre3L4zXy+mExmw+EQYWuR9jzm+34pJPfY7u6u58fzWbpYpHVlp9OnkuOotT2fp4vZIkkSznxjTJEWjrGojLGY0rXSuzVWW2PxuvAHIcS512ol/X6fezhLsygKmhx2WZYICGNMa7sm9azieJ4XJARZi0RtqrIGAOIHnDPGaVrkCMtainmeLrI0zypghAbe408fuTw6YLS9vW2MYb6XZdlsNinrilKa5UvOOeM8yzKw2ItiANBYI7Oy4K7c0RhDKSMEG2OUAnd6WZbN5/O6qN3zkFL6nscIIwSVeS5qhREKwxBZLCqhhKaICV0bjI0xsnb6B9Rjvsfqgxv7PvfiOOYeVcK1O/KMMVpXTtUuTRd5nrugKEKoqqpOpxWGoVOwccsghBAGatfEAlcI7oiizTLFrIe1lhBizIp84L75Tt7AMR+bLZ0JcLbSCOGokY6m0LBQG5RgVzTMVWuowWDg2EAY48ViIYSIosgPuONAuJiK2eBXGl0xarUVZbVkzNs/2NNau0cQhuHxq7PRaDSbLimlykCWzaVYmXJMCQFmMQFrjbWEUGuQthYQBm20klJbay31ANYx3sblr1+zSxPvQIHzjk2aYBMoNCvjTaDwuo/fRAB4XRW5aYJds+lms81zWDmejRlgrQBtNtpNoXXdHLzWsHjTysNrKAFeyy9s7nJtvL77GpFcNrOADfBhjCnLcjabjUYjJ4tiX6vxs6+hqM1zQ68F/K/dpV++77Vd3gg43B5onSNwgkvu3jY5CG89HG4AuxkRcekHCwD2KoejuUbjcjRvJja+4WwuQ0Hrc2tw6tV9V28p4HWLKevaRJlVkAMZWHEg3IX/p8onbL5Fv/wpaACKsAHS7vTee/fDTz/9VKmqFqXUUoMmDGshqqxWWmOM47hFgSCKDFihZK2k0togFkZRLcXh0ct+1/OwTfOlF7QqlelqQQzKF1meVx4P4rBtAYrcdncTqUpjhFLKgGz32p1O5/T0VCmyWCxOT089z3MasGEY5nlOKT85Oanrem9v7969exjjTz/99LPPPiPUuAqvVqvlQO1kMlkul0mSzOdza+14PP7iiy/cLg8ePHjybx8XRVFWhZSSEj8MwyiJoyTe2tkOq/gXX37y8vjF1nant9Vpd6KiKPZ69+KkYwza3r5x5+b9upDji1k77H12+tiPjQISQOBRqgBZQpOkFbWidhzJsvj93//9fDwaHh1GQUAQfvzNN8enpwf37nlJMl3MgyhkQColJTGccY8yZJEBBYCRIQiZSkmELAFEMOKUJVE86Pa2B1uVUZPZnPt+q9PxgtDzoyKvo1a35/IvUbJNvVqK4/Ph6elpEIZegD2PYcTGs7Hv85t3bhCCl8v5YrHs9wZBkGiDsqywhgwGO1LAzKiiRoevRuPxuNPqNpZhhbgRQpRhjzNjQIgVgc5aixBQiqM4cLBua2tLqjLP806rW9e1EIpzXhQFAlfor4y2nHNKOUIIY4oxzqap74VJJ0IdSgARQgBJY6q6zmemrOqae97R0dGLZy8n40Wn02tyjXbl761rI5nnpZMBSJe5H8itMHQSbHaj/w3XHGErlJJSEoqdkxai1tYQsEbrqqop5ZJpI9WqWSKmAICs3drazbPMWpuELYxxxT2CUBiGj5++cD7JxfydMBRCKAiiKCr7/X4QBFmWLRYLhzy63a7neWVZTiYTpVSv1zPGuNYjCCGn71HXdafT9n1fCBFHsbVWSpnnuSMHNFREuw4qbF4jol5T3O8iBA4BOEvUCCshhNx5UsZcjqOqKmfFnD11HAUXLXD7ep4XhuHBYGs8HltrKV0125RSej5rwIFLDDdMharOdnd3lVKj0SSJw/fe/7Cu5ZdffimlXC7Tw8NXrU4nbvWGw2G73Y1jqqq1V0bIGgTaOAqtlMYCEMIwoxghSjglzo4Xq6DchgtpEi6bbmxtoFfv87XxutNtfjbrshS7wY5sLHsDI2BFE7nSLfDabNfcMFz14rARwIA3Le/QBtnt2sybF9t8jhAyq2Wo84BrMiPaLKpbcRHWuRm2DjM4hGasNQBWiCrLlpPJaDodF0XmaMvWkvks2zxD/FqByRvvKlz1/a//9fV9G3T1y70aWqeTzFrByX3oxB6aX+26tRVldONm2oYMYjbybrD5mFZhg80ntQJYGF/iTrIeaJ0/cqMR6nBGAyGE8JXSG3tJQbgkMDomKiC7WQrhuAv/WfQUfjlWcEWdUiuldBgl88VSyLLbTYSUgIH5HtLa1BWipNVuU8oBK8KotbZWsq7rWmnfwDxfAPh5WQRh6BGxyGa+D4vFbPr8cRBEUiqjcSsMPYaUrOqi/Hw8poTfuXdnMpmOpxMp5Z27d6Mk0pIul8tVz2GtW612HMfn5+ec+++8887BwQHGOEuLZ8+eHR0dKaUQBs/zqqra29vrdDrdbreqKrfU3N3d3dnZmc/nx8fH3//+9y8uLjqdTqvVevvtd2/dvhtF0en5cDwev3z58vjk7Ld+97eiJPKjcHtn6wc/+u5b797rdOPFYv7056dv3X6wmC3fefjOWw/eOXx69NlPPlESAmBMIjBAlNFGKqV4GLWSIGm3Hn31VcQZIeT8/PzzTz/9B7/1m7/2/e/9q3/1rw4PD6N2B/L87OzMMkY5o2CpTzzmc0KtMtoCxkAACAGkaFZldV37jPqMc0LLohiene+2k9PT8/39/X5vy/eDRZoPx5P79x+ORqPh2QWdTMO45XleLUxRKz+iHveLPKeM7O0ecI9IocfLkVJqf3/fsRqHw3mWVR9/54e/+zv/4Ozs4t//4otlrk9PZ0pZz7PD4QVGqN1u07okntdxd7YqNcbYaCuEcExGSonvt3zfT1qR53lpmhmjOQuHw7Fr9Icxrusqy7I4jp2z6XZb3W63iViqO+FkMuYERSETRU4wdKJICQDjEUJ0DR4JOYnnsyKOO7fv3KPRgFKa53lVVXEcA8G1UlEUOxjOOd/fu+l5XpIkVVUdHx9jKwlBAGCNELVepaMppZSq2gn5E0KIFshaTElAuTXGFrqI2+1W0nFNGYqyToXgcVzX8nQx+/DDD7e2toYXo3v37m0fHPz3//1/b6394IMPrLXOQRZPnx+fnRFCTi9GVVUVRaGUVUZ4XhD3WlLKWTELuzFCSBsTdcIwDIPQX2bLNE3TNOWUVXVR1UWe51EUL5aLuq4ppX7oKSN1pQCBRYZ51Cee53lN/psQwlnokAFjlCEotQKC/TBQSglllFYMIc6oRwkoWUkRhN7FaOj7ftJuaa0Jo9u7ewBQlqWqpDHWqdxZo7QhFpjIy9gLHLG8FSWhFygtq1JEUSSVAk0w93jok8DThNTWYmCLohZ17fl+lmVltijLEmTBUD1fZkkUqDrL8tIPaFYsrLU86ap11MQiAgQsArNOCde2BiUaywsYEWzBydxaMErpNYvt0viuu/ZiC0hbxK60bGhwJ8Z4RRpYF5radRHvJjhoDGuTrdj0KBhjQq1jgBmzSXxzLNRGVsiAXYc0ELIGW0CAVr4GXab5V3xDY4yDggihpuYIALQ2TVKcUgrW4pX4wmXEQkvn5wAAYbiEKUavlRUaYqYLy3tMa+2aWVsLShm3KJdSW4uMgbqWQii3YHBU3NdhzTUQABuO1sHcTa/vhqtgbHa5Ntvmr02av5l58wdDjDIKG0wwI4wgimpVp2kat9pGWQyEIoItwnp1K6QRzbPDFhCA1cYCULeON1avRJSbmA0GbAFjxBBBmCBiDTLGGDBAMbIYA7EACmkAZIytkSWE0FWmB2NtFGhsLCKYkDV/QRmLwWIghJBG7Mu6qIbj5+KVqVqFMghy9S8IGbhsjw5rtAcrrsObvb4Di05ao/m8yVlaN9X6nhPAWgOiIEvpc+/dB+8e7Nx+8uRRzXQr7k/nEy8KIp/LwviRL0tZgvT9ACmCgRHDWgHLp/PdTisKw3I6pHLXZGONyxDX5XSqlq9u3tg5H052dm68ODw7Hc/1KKsl0gpHYbK71X55MXv2zWOk1e2D/ZOXR2HgWyUXk8lHb98NovjsbEQwv33r7nfe+/hv/ubni2lxdvwLKWVZ5i9eHtZ1fefOna3dzvb21s2bNwPPr+uaUNRqx0HInj9/Tgkaj08Yo//4H//9s7MXnsdGo5f9Lm+3UZSon/78Dxd58dNPfvH1qycGobd+7XYv2vrB7/5IayOE+vlnLymlDx8+/O6v3yKAXr08Oti/9eTFs5eHL5bZ3Ci7u9vuD3YtIpPFcp6mi0wQVno4m01UkiRGisOTs7IQNQ2HFSxR9NFv/97U0mEpfEoUBotku5WMXp2+tf3BdDp97/sfHh4dvjg8pD7d2dkqioxQq6vapxxEFfnBoBNYWaaZ2vP5/vZ2L47b7faLFy8+++yzOI41yFar5cX8Bz/4wXg85py32j4lOs9zWWMMoTW4KOuiwhcXF4s0/cEPvt9qbYdRdHz61wZbP04efPABiYNX47P5aD4ej+uy5pyDgU6rp7WKwoQmSQQAUkopBQAQirXR1mrGfIxxGIb9fr/TbTnefp7nUoqqqhoVGgBgjLVarUbqx7HqXLi7LMtOv6WFsdxWeWW04YxJqZM4phj3ut0ojKfTaVUXD9+6W1dyvhinad3tdsMwdO/3SjXImKIoACCOY8f4W/cdttPp1JlR3/e9gFKKjTFGGyezWEvpfMBKgZjgKAgRQkncjuPYtZszBqSUw+HQiSsQQjAivW6/lbT39/dbUWd3/xfj8Xh3/873vve9drv99OnTyTxX42FVVWVZaK2BUIRwKcQiL3jEEULcY07B0PWcdIRNY0iz4HCAQ2t9fn7etJl2q2SyFn91SyKHThwJw/M8rZ2ypwEEmAChCGswRlvQmIDnM2stwtaCBoQoJcvl0veUMaYpc3COw6WNOOdJkhDCXNMKQojDf46HUYkakPU8L4giqWpXekAUMoaDsciC1YYHoTS2rIQ1aj6ZOoSRLdOiqKQy2tRVLUshPD80gIuykJA312XsJSFxtepFqCEkOqAAVl0auzVD0BjrODGUrtLSzqMTTFwo2Gx0OFyt5Da8XcPngDWGuLSz68XitdX8hju8PJsN+3yFsb9puJtoxDXn2nhQ2CikRAgZo9945q9HRK4d6I1Hef1wTTLr2j1xECoIgjAMXb+xFTohl9mH5ribAYDNc7jm8u1ViuJ/ymgmaXBbc8cc491V3qO1bEkTlmtiPO59a6YysIo9YHQFuxht1gcCRBByr9OqURQYWGUDXV9cjDFgcFm/NSxd6S4Igpqgw7WazKsXZa21b/jbeov/lIjCtUf8xqE1aKMIUM4585jv+2EYUoqHw6GX+dwnZVnKPA0CLwjD4XBIGFVKIqSllEAxpUwDiKr2GfT7/SRp52V5MT2Rxagd+xSj6SLTBs7Hk1JKMERIJS3hLIw7nRu3b3LK6rr+6V/++Hw0/Oi9d+/du4OUmC6WZa1u3w53dncnk/mf/tl/ePz4KWOe1FpKTQgKgqCdtPbe3rl//34QcWPMZDR2XaSllE4/uSzz/f39waBnjamqOk1TYyJrbZx0MPXyov7pzz49fHX09MUhUOoHASd0MZkus8xahC1MJlNCSOLR7XfeHU9nx0fPnz3+Cikghr777v3FbHH3zkNMWJrnN+/cqJT+65/+7PjsfGdnsLt74/jlURLHcRAPT8/iwCecf/7VVz/5yV9lZaWMLucTYSwlLCtKP4yW8wVjrNtqdT76ThyHw+GwzgqP0UqK2A85xoDwnZu33n/7baO0EqLX6xFCLi4uJpNJWZaDwcD3/aqqXKhbKZWmKQDMZrOTkxMAUCrb2dnJskwpVdTVxcX5YHvLtRd49eoVAPzwhz989913b9298/XXX//85794/uICAIIgSJLE9z1jNMYoCD3q+RwhhDAUZeq0gZMk6fU6eZ4TgjyfRXGAEJrNZi5jnSQxADjqvrPIhBDf913TRaccTNYtfSmlZVbLyvgEp2mGQEuUn2fp7VsHNw/2KcWM4/HkIssX7XZ7OD4fDofSdjqddhRFAI0pV3WtsiyVUmqtCMF1XcHKZIDnec0iBq95f8hoY1Rdl7USQRAQirUxCAzjNIoiz/OcWSGEGAPOB+/v33QuuSwqFyTgnI9Goyja+83f/i+LohgMeuej5TdPDqfTadLt37r34OLi/OjoaDIZaa09P4i7bcYY0rUzD446oIR0bTP73S7GjYgbbsru67pqGAnOxrmiR7xmwqO1jIzz63Utm41hnQuwjmy1TljARhm9UkoR1agkmRUlc7VodhDkMt1jLcaw4k5aLeoKERxFQSsO56nyOEVgKKEYAFltlDAYCmOUUqKufMYXaV6rE2yBMaYBEca1RQYpC9gAJoxpU9Z1jS7T/6sKdYtgtcZ17+JGsHTTMeBLfgBuHjdZl4a6vZxE4TVz6X7YDDA0CZoGZ8DahZANTUM33INYu731p1eBgotRv9Epvm6mnT/b9HbNzxi/QT6hOcYbGYXNJ7ABFzYQ2GWpp3sfrm3vxmayv7mlGGPzLT2bmhmuIYZrWGETkfwdxrfdz+Y5ojWZ0TWzRet8gUM5GGOELgMGdqNgxKhVpIHg5s5jAEDGYosJEOTahGKwFrC1hDGniqStAQvImBXXmK6A1CqUuRZnbHrHW4spJs38mzf/EgRcRX6/Aqfzbx/oMnD1SzfDgJt8h7WMsV6vkyRJGPmz5TSO24hAtswxRbgukySqpQCrEWACFlmLwQJAUVRRN7HIng/Hg7bXTSIjs5PT8yJf3r/zlpeET58dVspiZrTBWaUwKnKpkn7/O+9/8OCD9756/GgxmRiPkzjBVRG2kjwvHj17Wtd6Np0Lobb3dqtSFJNJXdceY4qg9lb/7q3b+7s7v/jy09ls5mjjxqxqynzf397eBcAnJxdFkUVRJJWZzpaz2ewHv/5rUiuEvdk8N5q89fDd9z543yLUjpJXr16NLy7CMEyiiBghSzE9O17s9Yo87XT8k8Ph+HxU5bKdtNvRVp6NW53eznb75u07cacbhPQv/vLHlMqL4cgAkspqY7OqbrVaxxfjFycn4zy9d+/eNJ2Pj17QMKScFbIMW20oNCWglfjRj370zoO7X3/99Vdff5EkSb5cYARFltd5NojbO91+nmaaeaPRCCF0cXEhpXSiy1EUZVmWpqnneWdnZ4eHhxjjLMum02m32z24ub+7u5vm2fn5+dePv8EY3759+/nz53/9s7/xg+DBgwdlWf71X//1n/75nznwYbQihGAE1mijJQJDCeKUUYKV1hoj1WlHvW7SEOyj0A+CIPD8uqzOTk7Pzk4xxr1eL10ssywzxjj9YKWU53lREIKx7k+rDKIx7plVeU0MYoQX2mBra1mPR6PtQZdSPBpdaFMLWShVzRbyxeGjsiyD2FdKaM2NUY7MjJBFyO7t7aRp6vt+GDrlHx3H8WDQK2QmxKotk1SiqktjDGCLCSIUAub1+90w8h2x3w8CRgL3FXI5MBc18f3wnXfe0lpz7juugLHaWnt+ft7uRRazrBTLoxNCiBAVxuTmnQfGqrutZP/27SxLZ/OJW4QBQIsja6020pE9KaWMEc/zXHtGAHAowR29rmuM2DreYNx1ueCH89+ux4ELTqyhAyBktZbNUpgQZC3SWrov+9r9I2u1UiYMQ0Y9t5R0w4XqPc9TUkspsyyzFrmzJYTESUApsRZzxZm3orAYYzxK/FYbAIxVxhhQ2oDUFjIjwWpjDImYH0acEiGEUgYAGUDGGD+IlK0qIX1EEXEn7Ayo1UKteJoIhFDIla9h0izxm24I7qIwoutCR9ysHfG6yZP7FTai1rD2ppvOrHEVDmQ0ObJNmgJc9dCNzXW+Zj3163Z546DrGZpKjeYHdz6b/nvzbBt3sjn5pXt7A5nxEjfA1UUk2oglbF6+263hIWmtXRNzJ6XqAnjN1ELIzVVpc/5aa/RaOAFeQzavX8uvODb3unqBK6dujXVAwZ2t02cUQmBCG6Bj1+Se1bguNeH+vSSKIoTMWglDaQsIuS+ARdZzOssIACxeqS2viCHNaG5RE9Oy6068zc943QV0fQLN+Vy/b+5u/h1u3X8UMnMiUQBgrAJEOt3222+/nReL58+fVVVV12WURJ1uS1sVhuE//If/8H/4H/87rSz3SOBzqSwAgLHu5hfpIp8Pqa1/4wfvRlH35Ojl8GJ8//7Hg+29n3/2ZFlUXojCVqfWYrEc14AOT48fvPv2nXfe/sHf+3vHr16F3c7FfHH29JtWq8U8vy7qyXy2TPOdrd133313Pl08fvxYVFUQeEWWz6eT50+fTMbnZ6PhZDKZTCYY41arZcAUWWEtVKVk1Pe9cL5Ij755+sWXj9rtdqvV+v/8j/8zpfT23TtnZxMeeHduP/iNX/9tZRXn/PT4uC7LVhj0WknMWV3XrSR5+eKREKIdJ4Pt9vMn3wxPh/Tm3ft3H9y/93A2T0fT2dn50QDkwwe3Ti9e/skf/1nv49/96KOPz05enZ+etXtd4vmffv1lHMeDnZ2w36mw5VFssNEIEGGYe0yJssyfPXn0D3/3tz545/vtMFiMLyI/OMmyOAi7PJRxst3pybxeTKbtOOltb2GMEQJXj9ZqJXt7u+t1lzk/P3/16igMwzAMB4P+nTt3xpPl6enxbDEfj8dB6DHqnZycrGhznW4YRv/uj//9dDoN4xgAsjQPw0hrLURdlhogkLLWWs/mQGVdLJdLrfXW1tb+/j7nfDgcnp8Pb968KaUssnQyGo5Go+Vy2e/34zCsqopiDE7ZHCG9ViRYzufHR0dZlrk0RFEUURRFQUDAUkaJNSHzCDK1lb1OezDocYqtllprhNxKzASB3+p2uNfxAypVWYsckMcRBqQAqSj2lK48jzKOpFRGCUCMMm5q7fCE1nqZLxeLhVKKchJFESGW+TwIie+71jCWcVSkaeMyMaa+71vggNTLo2eMsW6nzxgTQgtR+D4XshyNz5fpcr4YYYxv3Lixd6PPGIvj8PHjx5QS3/cwsQhbSmmep0qpqsoBQGnh1HDxuuPzcrnEawlYs/5q1XVdlalz29Zax9N2/aKaFSdes+Wb3K0zBE1wGK8FrR3CwBg7qRDnAChdJeMb5hdj1K0aXQvNJhnhPhSqVkYihBCB0A+0kVqJ5WKGMSRJEoahUirLUiGEW7UxRq3FSgqrJAbr+UzLuihySqmUuqhEt9tDVmfLBe1RRomQl6H4DUu6NpcbRAFnv9wysXE/679atIqii+bMXcCpsazX1rjNWtOugzFNnOA11/vLht2AA1c/tnZdMb9pstFr4YHN62p+bTIUGF/qdm+eW8NXfe0QV9zeG4/1OhZpXgkAcBWzDjLOZjMnGNoAgl/lnlw77us+/u88mt1f/8FdRVMJqdaj+XY7VgTBGl2+AZd5ffctgzWVr7lS7dig1liLsDEKAFuLsBVKroQXV2gAXEcGvD6r1SNDeP0cr4DCBrIQuiFItUYa8G18w79D6gEh2JjqKih5w1iDQkMpBgydTuvevTsXw5PDw2c3buz6oWeQioJgvlwYq956+0Ho8WWdWc08zqxVYFZkoDTPOaBaVC8OX7Vj7lNVVHpr+2AyKW/d6fYH+4ujV8us5Emf+YFO80LUry4uDk9OP3jn/bc/+vjG7dsM8HK59IkNPB8AIyDv+SHFjBMeRUk2SJVSBNm9nd3xePjq5YsiWypRzsezLM3KrIzjuNfuEc7quvZ8n3O/KCpjLaO+F8Sj0WSRXnjePO60vcCXwrZaHYOgqKrj0zML5u7d24s0nU6nlOIwDJE1SsipUmGHB4Ff1TUlZL5cMo9P5hP/zL//4K2f/vynX3z1dSXtW++8/U//2f/2X/yLfxFE0b/59ChPF8v5Ik2XhBDMMPV40Aqny9nwszGiGHOcZ4XF1gv8qiqYNXmVHh8fTqaje3dvc4oogKlqXdaEB9v9XpIkW73+bDabnI9bdxNApqorbaRb/Z5fnCJsO52OI9pThnf3tuM4ppSenJwMR+eelxRFBmA8j7317vdHo9G//+M//e53vxsEgWuXBRYHfhTHrbquAaisc0IIp0CQBaPAKIqtRwkdjc/c2mI0VnmxCIM4DMNbt25oLebz2WQycUoAjHkY47Is2+22q8t3i8KGAF9Vlcs7mHVpvvOUPkcYkFZVEvlKlmVebw/6PuNVVbVaLUqpEFLUsr81ePvdDwghXrjDGFsul0LWjFMLxliNMBRlLpWgjGijalGVZSmVSLNlUVe+7ydJ4nmsVrVbbWNjLUjuUeYRqYRMKyGEMQrnpMoVWpOTCUUIS6lyW8qXR6MoitJ0PhgMqqoqq6WQ+TKdjucnSZL0Bp7WuqzH1WjighB5MV0sFrPZzCE7SqkrEJienhJCMFmv7CldRQ7WHhpjjAhYWPH2lVwZu2YB2gR1nL1zfX4BYN36QSMElBIpXc26gw44ilYkRyeKRYhTvtJlWRKsGvafS+h6njefz9FlF+XL4pesSNEqqo8AA7JICKG1RBY8xsPAxwgoIMCIU+JRQgjS2mgtjABV55pZAjrgpN1uFUWVp0tdl8Tq0CPbvXZVibPxfDPs794iu65uB4w3GOUEMKJkZYWVUmCbcH3TtOkyWeNuF9lYzG2uHV3MBl4LyzfFIO6ThkTy+sbrf99scDeBwqbFbwI/m3gFIeQc1ab/s6vEB2pAYXMVzSebmGB9FLu5e3Nd9rXQyOa+ze1CCLn31nWULsuyEdXA69qZa1jq9Wt8ff5f4pl+9fFGlGDM6rYQggkhnu+7SkgXcHZiJGStNmaMsUa52+4IBM1pN1DMXA3JYEIsgLGWAFLWYAvUAFhTiRqDbdw/o3gdObgkT2CMkV13s/RXddGbz3fzscIGULBX81b/WVIP66n+lrlc4BYAOfMihDBWzWZTz/PuPrwNGIbjC2WVE0CUst7b3VrMZ2WeEhYQhA1YSimjntU1C3yfdSilWVpVSChhPI+dn86q9+zbDz/KK3s+nkiplbHagMWkruvj07MoanuYdnrbvXYHA5qddabT6cnxWb7Mep3+wY2bxtgXL14s54vh6Dz0/HYSI6vLIp9PZ57HJrNUKFXmReD53W53e3tXg42i6Mb+/jePnzx79kxq3UoG3EvOz8+n02ln0D8+Ph5PJ4tskbRaw/H4D/7w31ai+pf/8l+GrVZvd7e71adhmOf5ZDmfz+db252//9u/E/qRrsW7739w7/YDbLHPgycvXs6WedIZkKr+xRdfVUAePHw7q+RWuzU+PeEYdre6tZS7e4MHD+9ILf/8L//s9PS8O2hv724BUsZajHSWZ4xyGlDCyXg6PDs/vrg44wQbqR7eu1sWBWijajEdT0aj0Yrw7vtOE/nOnTuMseFw6AStnT8FABfpb7VaW1tblNIgTI6Pjz/8zkePHz29GI9+9KMfCSk///xLwngYxZz7vV5vsUgXi1QI0W51a5G6TtFlWZZScI/t7d24f/8+5Qy3khbnXAqdZ5mWcmvQe/DgwVdffRP4vN2Kfe7hlXRgoKUSVe08mBKrZkiiqvM0y/OcU4bDiCDsnAoYO5tMb/QjB9Du3L2Vp8tjK/Zv7BIMi9m0N+jXlbQGWUu0Qr3etpQy6bSUUpTiIPDcQgHA+D4H4JRi3/c9jxmjMF7F8GHVEEFQ6rs2RYAM95nWKgxDQnFRLKuqsqCRUzvRiHNOGabUAqqFlFIVGOM48aLIs6gytjK2mkxTC7UQYjQ+mi8Y55xRLwgCzwuqUhy+mC+XqRBKSsmoB9jWpVgsFovFYrvTAQBjFULI9/3A8130RkmJEDQmDK17JQR+3Cys4bWFxaZ9WRlobAghfsAtaKVcyyQwxvoBJxQ5AEcoMkYpLYxV1q60mddB+1UUoaoqglcEBUJWIWVjjFOUc4cGa52mLxirjJJ1JUrPWo2s8fm6VB2UttaL/HYrKRYTWWSEkIPtwQcffDBbpLIsyqpgjN/e33v44N7p6ek8rxvD7cTBAcCqVXy7IWNv+HLTDFhngp3XdxDHWusImAAAFm/et837idYlc00WANb+sklebK6hm73QBpPA/aExwpsG2V6NHl+z1JvTbj5f9KYocbP6bLzdCiet/3ptkk2U0GCIzWkb6ADrbo1NqqVBpQ6PmrXqg4MO5lLv+W+5rmsg5vUt/2NxwxtRQnMtxhiDjPNtrnXLJVNHCISQq9CGjcicozQ174PSajXhVaDgM7b6moB12yswKzEGDNaC1RZjrC0QsAasq55osGkDJhpuDQCshROuQ7orra5fS9nAr+Dm3zwQgl/5bq/zdMZaY6WSskYIVVVR1cV0Og1jv9ttz5azra2t3/3d37579+7HH31wenw8HM9C4jNGtcFKGYQQZR5Yyxj3g4Ayn2IkajKbZyEKRmfTh++8XUuzXxQaw3AyLkrZa3fiuFUW9eHhoc+8JEhkqTqdjh8nXUItkBfPnp8PL4IgCv1gNJlgY7vdLta6LnOP8YMb+71WEoZhZ55Za09PTxFhURCHYVjWEgFRCm4e3EaEPXny9PjklFLaG+zs3ri5WJ5PZ0MxlPfferh/8wBzdjI8P7hxh8VR1O21hPDbbRaE/U6vs3tjuVyG1B7cfiBr9dMf/9XTlyfnF8t8mVdFjRH3/OD2/ZtCG/n46adfPPrqyUtASNFgORkighGCTr83GZ2PxibL8047Kkrf8wlGhjOsjERWEWQEEpRj7JFX58fbW30hxIMHD4o029ve+fKLL5az+XK5DMOQebzn9wmjSglKsefFTjhysWCMEc9jSomyzI+ODo+Ojkaj0fvvv//ee+/duXPn7HzMORNCnJ4dv3x18o/+0T8Kwt/78Y9/0up0z87Onj17sbN7o9vtAqaTyaSu6/5WT0pZlHktKs/zOKcYgxAVvXv3NsY4CIJWq8M5twa5Rd729iAMY8ZYXddlUbsCfa11WdaMMYRQ06XQcRgppVtbW2gtzUbWveGtlVHIoyj47scfllka+uze3dvj8fDJkydVJZRSUZRIBXleAWZKGZfsr6rKvcculugavbjvjwtd4DVvMYoiKVfyA5WojFWEIMZYlqVB4BljizIrioJz5nmeBWtBAyIIGwtKSG0tMMYwYVJqIVCWCYyhruvxeGxBcc4xrnyfJAkTtZjN5r4f+l7g+0gr5nEiBKnKernIjQGC0aDXDzxe13WRV3Vde55Xeb5SIs/zbrt9aeOUNnaluoiRK4RbpQaaJYvLLDS6TK5MXGtN6Iq6Za112k1uuHS7u2nGGCGE2yWOEkY998jcE6zrWgjBOXepByklxtrp1SCEWknLrvPWStZuZkqwUggAhKiQBYSQz7nTsQh9z1rbjqPt7e3FZHj8csoZiaP973z4wXQ6ffr4m8OjY4+Q3UH/4Mb2bDxqtVpNEsQJkTkXuDK1xjhnuk4tu5LD1fYEY8aY53mUMicsTSlzoEdrzRjzePDtxhM16zm0ZlPajSpKtBEittY2VTzX/fq329yr6OTSEL+Ro9Ds0sy8OUlzbrAOaze8gc2TgW9xwNem3fS1DSHGjQa5ulhCAyXruna0U9cWqpnql6ckNr3gNSTxn2uQteyHBg0Atq7zPMeIZlkmpRZCUMbDMDQG3BeH0csbstkokq0Fu9DVEBG6HGCtNQiwW/xjvErnoytj84ZsXqzZqKq1YN/4mNy4hmj/swx0VRX7jXh0YxhtNFhLKG21WgcHB7du3VosZ2fD00TED966P5qObG0ODg46nfYH773/4x//ZDieIWwJoUgjo9wixyhrJLJFUV5cyMi3Pqe+53MZDs8nH30UPXz4bnfQFVZ/+egxYPrN40f9vlLSlHk56A4YYkVRhaEadNqAEODZ+cXo5eGh5wXvvfVep9NJF4t+v18sFmmadjvt3d1tDNutVtwdpwghxlielUEQMOYVRVUU1enpabvT9f3A9a3qDrY6nY4U2gtkGHmYkrfff2+yWLw6O5lMp9/94Q9enZ48fvbsmyePnSP76KOPPv7441arpfOMBfGLw8ffPDmMkp7HfEajKhCz2dIPWxfjWVmr3YM7Qad/dHyqjRmfHu/s7Eijgij8r/7xPzo9P39++Gx/f+f49ATwXlFmWbbURiIAo2UU8ryohDFSyhcvX967edsPvbsP7k8uRlpop5EThmGv19va3RFCaGNcysBFpk9PTx89epQkyXQ6jaJob2/v4cOH29vbn332WRiGLgNw+/btJEn+p//59//iL/7yR7/+68Ph8OeffNrtdk/OzrkfTKfTwdaOEMoYYy1SSiul8jzP85RS0u22AeD586dffvk5/af/6Hedt0iX+d27dyshhhfjw8Ojva2dJEmiKNHGFkXlEhDWolorKeV8vszz3CiT5zlY2BvsWGsIRRisVJVSwoJWSgRERjaAvI6i9vnzk7gV3b91P89LTELmt0fjpTIKYxxEYV3Xi+VcKeXp2vM8TFRZlkpjrTUgrTQgzCgza2dpMbaYIMqwVSVgnef5fCm4H2xt9a1FaZZRygCIUlZJQwhxagRVlROMlba10IwRz6ecU2Orul4QgtLSIoROhxdVVRkNWan2OnthkGm1nIxPjbbWkkLRuuAYc2RNwH2PYo94OEl8PwDAdV3nee4HvKrxbJIu57NWq8U9hqxRSnLuO9cbBB4Ykud5obUmzLmlIAgoxnVd16rEGFOflmVJCS1VSSlGDOVFSimNEW+3wtPTUwQkiqLZbAaAhRA89oxpYtTIKkKR7zOaq6wV4rQuD24cZFnBAEuhkzB2wSViLCaoqipAEmNa17VhIWMsz9Jut2sMGw/HURDWda1rxdscWdJJOkmSeIxzzsfjcZ0trFb9vRudINhtd0w/G/Q6iceZEfPR2bv377739lth0kKEnxy9HA9Pi5K76BRjjHIfALIsW6aZK9nX0EgRIGsRshQQIGQBHJCygAzCBpDFhCltla4RQpQxyhgACFX71L800OuchV2HIhpHCACulsSum3Y6c+PQElz1tU24RWtNyGqD1VJ0NVwi6XIhh5B2Qrxa001/gDY0G97obKw1AAYhF5RewUettavJ3/QuxihrrcUWYwwI0KUIMSYIIwTIrpSC3d10nH+CQNaVEMIoRQgBo61W2K2WtarLoiwKWdcEAacEIVRraVeu1LoDw1Vp4UsPZC0CIBtNnjb91jVX2vxw2bLi6vi2yAQyBAEQTCihLtRvpJF1fX561mq1+v1+HEa+xykBvAo5WHs1Z+9+KMtqBd3WHIUVjNN2detc0cO6isEiqxAiFBACiyxohJQxVgNiLv2KrLHWGgxAOWGMrlC7sdYCAowxwYhghAFjwE0rKQtAEMaArCumQC53YZ2KAlhwghCAGjaEca8IJtelG9f3a7UdvsqKJa9vvwrhEa0VIawo0zAKy7J+5+33/u/L/1uRV2GQFEX5/PkhIJym+R/90b8XQk1PRxgiimMCARhW5wXFhFuEwGKtKLE3BrtFNjx9dXbzxtZynPo93u/cUCEKvfBvPv3Z118/euuttxjYfhzf3d8DgEdPvsm2tvKdndPJq9/a/a3R/CzyoyRW+7tRbHd5uXz16Sf5PK2K+ng2J5wEUajqepHOT86O272urvw0Tc/Pz+tK1JVJOu1Xr175vh/GkTFme3urmI0DbOdnR/V81Gq1aBhub7VLsbg4P+QhFuWZleMnX/7k07/+s6++fpqErdF8fviLz+xsZsYXD9+63x20z9Lh7o3u4Ob2T/7qbwaDQdwJ2h1258G2KPLFrDy/mI1OL7Df6na7Z+ejdius0rlHMDMyypZvJYEJWHfQuXj5eDY8Ac4R49IQimhVmDiImW8BQFb1s+OX8Wef/OaPfm1a5bkWVZlLLdrtBBl7f//gxu7uX/75X7Tb7YTEkR99/vnng8GgvzX44MEHL49fBTRKgs5iku3cuDHo76VZPZvNzi+mo/HiOx+3Wu3e++99PLyYddrb1tDd7RsvD4+TVrSYTY0WnY4vRH74/BcHt25WVXn66iSKonbMlBIEysFg0Irh5OSE/sG/+SOEEKW0ruXJ6UWa5ovFYjieYvyN7weeHzolGWstAgIEewGPosj3/TiOkUWLxSLPc8YY5yzLlxZsHMd5np5fjLSW/W5bynq5lFLWi3ROCKmksNYSRo21eZEtlkttDWOM+S6w7wVJgjF21RMYr1yva4CE1ln8xpTrdcdkSilj1n3zKeWAUKsdY4yXeVrXtTbS9z0X6qeEuAWrUkIqxBgBpJSSjJPGnQCSmGBCFaGqqDNrkdbWGmQtYMQRkhgJSn2DFSBKPez7PIpCowFyjXForXVupchzQLaua1VXjBGErFbKBQ8QsrUUAGCgYfg3sW6yubB0X35CKELEWoQx9njg8cAtASnlhBCMKUIoCD0ldZqmUl722SIxyvO8zKunRXXv3gMALJCQSmCMOWWEYruucXLWfDIcuX5RboMkjnzfZwSTKNra2go9v91uh37gEgGMYkQ9UcFksbQIiOdv7e13kjgIgq8ePX15fDqdLSzCloy1JZPp9ORirL2+e2qEEMIqhFBVVU37iSb1AG5NBpecPnjTQK9VnV1b1G56/cYZ4A1+KKwXc01u28ELvEGrvGperzie5kC/ZDH9uuPc/Bxe86n4NeHkhjABrzldBw5X69/mDrhqfrjSDhFdZU02d8bdBydU2qQ8rqGl/9XHJsK4drsc4HO9ylzN8+YrYe0qmYK+nV9y7UCbY33oN3AD3XBRPWOMY0oaDBhjgoCt4kerM2xIQm/kvsDmO/MaDxHWv/8n3cFfNi51urTWAIZSVpalBu16a4VhCBg9e/bs4uIi8qOLydgQJK1BWlBGCJCyLJGRN3Y6v/6DDz/+8K10en788rES2QTBBx+83263n3z5xfn5+eHLV4yxkNP3H9yfX5x/9eknQRD4lDBQUOfTxeLf/A//7zs3w06rjY2NYxze6gTEq9JSLufJIEK+L2pdqJwRP+n3yXxxMV3+ve+9fffu3cePnwZR1Gp1zs7O+oPW0+fPHj/5klLKAuAeBGFQFJZ52I+w9eu8mrx4+VSa8s79W+2Otyzg7OxZK+n97/+bf4oQsrq+OH95cvz06fOXf/PJ+Nd+41+8+977ezcOfvDD70wmo7PzkzhEQlT9rf5iki8WeVlP0jyntjLEw6ySRSXqsjvY1lU1n4zjdlKWuR2PkjAIAt9QAoQgBGCsNkqKQlDpc8+R8Uej0fPnz7NuX1Z1N4x7g0HIvcV4+uXXX00mE+57zu0+e/EcU5KXRfby5WQ+6/V6u3t7cRwXRVFV1XK5rKqqERfZ29mtRL2/v08I+cM/+ANC6A9+9MNur33j4EBpWdaF1rKsC8qplMKC3rvZbbfbQRAghFyivygYZoIu0xohFIZEKchScXE+yavKGlpWoigVxjklnHqcMc4YwwZLLVy9AMaYESalZIyFYWidvyc4DANCkLEqCLz9vZ3Tx8/SPMurXGjBfb+u61oKi5Hv+1VdA0YBD3jgu3SjlFJnqbW2KAqtdRAE7vtljKkq4aLrZK1H1LQ5cKR3Qoy1VilFCKOU+gE3xuAyh7U1dF+8OGoZq4SoalFLCZgYQoBQq1QFyIIz0yARRhYVBkoha2sd5QlZi8BKhChYzIxEQKwlBDMPsAVSS5EV85PjMUXYWqu0NCBFLayTcCFAGcaY48vYLxhjNJKbZesbP1OMqQsHAWAA7KS1dVn2e1vW2tlskaa5011mjLmXA5B1SaIg8BDy8hwxQsFoz2Pz+RyMkkIjhAEQQcgywjmnGCFrtJHWWowhDiPOuBAiTzMEIKUMPd/n7O2HD1utlqhqSinBlmDkc8/0O9OF8nxSVnI8XwLxgMH5PBPDaVU9ny8XtdSAiNTGIiKlKqRGSDSFeZhyQohSqqpW4gqIXqonIYTA6oa89vq6EG0oZ9h1Xn/TxTYworHFeKPDVpPr2VzT2w1ORHOsTQhirko5XbWwlx/CRlkHXDX6r8+zmc9udA7sumZPb0gkNSinObRUq3wBfv0oV2dujtXcHLTmbVzTP3abrT+50nn5f7GxiWau+VS4wv1cdZpu+js4bjVcfus3pZo3ovHfclz0puE4KJvgqUGTWq9fZicCAi7Bh/M8J4R4jDBGVrkHgxFCTiLMrsuX3kDKWV8gWqUPvvUGAcCmVJPbEcElpNhEouhbJnI4BgAY88CiIIislTdv3vyLv/hzwhGjbDFP9WJOGKacW5uOp8vRYmEtMLBaa6ssYZgRHBCfIVwusy9//mkn4b/xgx8ZWT5+/HU1n+o8PT09LesqWy673S6u8x/96Ef1cvrHf/zHTKJ+Z5BNR8P52BgzyrKjJ9M4iXyKtKgo2F4n3u4N9h6ECKCHugiHZWV7vRtB2At6O5PZMumGN27taKSms8WLo6fD4XB7a1ca2el3er3OnXu3prOx7/NuPwYMSot4m8aKe/GNKAn3bx0ss9QLQApNKf3ud+6U1WIxO94ahAcHHe5rUXcPj44fvv2AcXjn3XtPnu4en31jEZI6O7moiDWDXdLq3ywVEL9bSHR2Pj56fHF+mnLPJq1Of9Da3t4Wuqytfn72UiuhLQFrEGBGmSUGI+eWADNKKQWMFmmqhKyyfOGFnThRQlaiXkxn5+fn77z19off+ai31fvs6y9b7dbz58+n02mv33/w8OGdO3eePXtWK+n6vjJCkySJ4zgIgucvng7623Vd+r6vrPrZz/5msD24eeeWEKKo8iDya1khiu+9da/dbYWhnxavOCeUOp9VIc92k7C/d4922n2lTBAEUmvCfKmxNcTzwyimdV3XSmFKCeZgsdIWGUOZzfM8TXMhROiH7vvpM66U5B6nGNV1zRi7f/9+p9OKQ398dEhrwJj0t/tbW1tFWZ4NLxaLhVQKERx7cRBHnucJqZxsc3ergxAOQo8QEoahlJJ7FGN8cnJiASxoQHjd8m6V6bdrCrfRpixLKV1FfODMR6vVErJaycggQwhB1q6XHRohSyhGiCitjNGwEkCV1oLShVSpAQ0IgAAYF5K0xhhAWGprLVLKYEQQVRqVi8ViNBpdDJdOQIJzTn0n3Et9L3RkSISQkYYQUpZlLYSUGlHEGDOGbjotYwxGhBIGFtWVUFJrDnlWlWW5UNXe3r4xMJvNlFL9/hZjXhh6ztZLKcFdEQbGabvTYgEyBm7cuFHkVZ7nRV5FQez7PsZEVLVGgnpe4DFjiNZSC5J0YxeNMFJprRPfj8PAY2Sr3/M8b6kVaClVDdZyEhFsKOVBEqyNGqpLcXwxm81mlHl5VWFMAKFKKIwNpgTzSG84YKOUXbekMsawNZGyWUO7vMO1Fd4maGgcZ4MFGwe86VZhw/81DtiNpvyyWUzbjYFfaxK9mXveNLibXha+xattmm9zhex2OWEjddxgF3cOUsrmWr6NKLA527VzQ+v708Cp5sTca+P6mqIN4a812/F/HaDQjG/zo81fXUDxdfLH+mW4Mpv9Fdblmw+xAQrXdm+APkDTFuRKJKYWNaWUIIYxGEy01ta4di22oeIihJoMAl63w4Zr9ALXn6JhO14Nm21ezAo0XFWk3vjhzUDBlSY7FXwA7IchIPveux/s7OzMljNtVJ4Xpaja3VbSDra3ty/SXE2mRVqAxUpoXYtulLRaETbKQyhg9PFXX718/vXnD27funmj323fPtj1PK8d+Fu7g6Ojo08++eTf/H//u8SnD2/tftHyl8slrn1cZ0LUQRC0OVpqrEQpATCRUuXnk/PZ8rnneRizwOvs7t0PWzsCi/Hw9Hw65iyiLLgYToO4/fyTL7766pted3A+Xnh+q6xUf3uvv7W1zBZe7O/v73o+m0xGf/X57wdBsLUzGGxH5xdHf/mXfzWeyHv3dh/ef3Dy6tnNm/1hPTF2vLfPT44fBRH0+p2iWC6Wo8DGlMsgUhpmiGYGVRhp4nlJFHWDdm9314t6aVb/fvbnk/FpWS1+84cff+/7HwHA7s29eZH99ZefYgwGWW00gMaUEGoRNoSxWitkbOj5xOPCqOXF+ej0fGew1XnrneF4rJRsdzoX5+de4H/w0UdfP/qKcR7FsTaGh8H3fviDnRt7lRRCq0bANwxDrfXZyWlZlr/2G78ehJ6sRV2XGGMhq06n9ff//u/8z//m96Wu+km3kEUtyrgdZ+WSB4QEthSpqpV7CXnFW61WHMdUGVxLaZEAwMZgbcA4iV+E9DoUpq1ytDmMMRbaWuvIhg7LK6Xm87nncQtaY0QZYoxorZfLZbqYaaRa3Van09nZ2eK+V8oKIWsRcJ+ZSldS6MxWtUAIYUZjrxXHK0fl3u+yLJ1IokseO7sGG96iKaG21iK86gEtlVJaxHFMOQ3DENdACGaMccLKKncllNYaC0opY8EC0oRYQGtVV7zC8sZc6vuuGreABQwAmjHmUubWWGNLqYxUmVSZH2GMgXCDsLTaaFCOOwnYGtBamKoSCKGqElUljDEYMCHEpULWZgmDdT0qGMZEKdDaUAoIEaOxkRohEsctY6AoqlZLutaRSZKskxHYWlsUGUKIMRIEXpZl9+7d293e+7d/8EecEWOVEtLzcFUVskYEWUoxo1QpKyhJ54sgCKIgVICZj7rdbiuJgiBYLmZJFCArGSNW2zwvUlWnabrMSi0iaXRZ1tpAlpezNM+F6SQB1ivNJWkl0hYZXQuJ0IZ3R5c+3pHs8KWukbHWIoxc8HZzNIu5TctuX6sFuDaaORvTubmx84tKqYZ1aNZaC+h6duP64nLTHNt1W6lLyHI1K3HFym+s7JvzaSozNzfecHtXcMwVCLXpGFx9jd248KYNqb2cpLlwd/PRRi8lu6Z5/q+VfLiGDzb/tHn3mn8bEYUmULR5e9+A7P6u19XcOqcr6vrTAgDCDomugkCbJNbNt8UF/ByyccFS94AYXl2pC6Ft3oQVSrjq+N8AFDb+v7nNLx9KGmOVMURUEnuACQKC7969e//+g7/8yY9rWQRBECaRF/A0za0dhVs7vd7AqlG6zFUtumHIKRFFYapibkU7+uA3fu2HSUjm44svvvhqb29ntlz0+30Ac8u78e7bbxVZ+s2jr/NssX9j6+bezs/PXmVzvL29XdVFURTImH57kBfLqszCEPlRoIyp67IsxPZWL6umCnb6Ow8A2PPDp69Ojm7s3SR8f7aUoi6qCvf7N3f39oqiODs7W+TlcDzdWW4ZYhGxi2xiM7lczm/f3eKc93odxu1scb7IJPeg220PtrcWWX5A9vrbN14ejctxuswhLUEKcfjy+WCnN0kvltlFq0OrfNxKEJC5BQU4BGJqqSczbJfTxbIsVIYo1Cp/672H23s7P//5Jzdu3gLK4lar1enW1pZSSCk1thoBWGUpN1IZo2sl5+lSK1VnRZqlt27eLETNPN5tt2Ut7ob33nn/vdF8+skXn5VKFLK+9/bDTqfzne9+/OWXX+Z5nmWZxzhCyPf9drtd1/VyucyyrMjyNE27vXYQBGWZv//+u7fu3nr0/PH7H76XFtmtu/uLbDGZjTyPZUVqsJIW1dZqAMoYwjiXMh2N8GRCi6JSWhPKMQZCaRAEiBJCSFEUymjKqOdxbY00NcKI+7wuRQPe3ZJda+1Ue9NsEXi8P9jxPDabT8oyDzyGkN0/2N3Z21XSnF6cD4ejsqoYI9pIDRYRDBgZBJSSwPeDIKjrDCFXu6+llOPxOM/zIAiCILDWGqMdUFh9zRBGsOrhyxhBK1E2hDCu6oJSSgx15gPAGmMoJVpLSjGhiAEx1jYrKIyJNY5ERCklAECpR0mACVtbZWMB2abzHjKUM0yR1hZRi5nxI9pSYRgwhJAxIIRAVjOMCEIaNKfYWFVUdZrmyIJrB+N5flXnzrIIIQCwq2l27XlcDsUaZAEY9aOwhYDKwjDKOeeMcowIXitYUErrupRSrHs6SADEuedkONtJvL098HzmcZ5nRS1KTrESlQIIA8aoj5ArpmD5eAFKg5LL5dJjvNNOAo/v9Hvj8chqraoCKDZKWy083yeJL4TIl7PZYjlfpFklCeXEC8K4JRWUYhXN1soJSqLm2TUu325EAsy6CtT5KgDAqxY2VwsUN0Kpm47cbYDXxev2aofoa59sbnntrw2DAdaqPnZjGbc23fbazK/Pg9Yn9LrV3oQO13wh2tDQtBuLe9fipNlXXxVGtNbCxtVpC5RStIEzmhQG4MvVNmwABReRWn9ZoFmm17X8W53N/z/GJhBshrWrXkdoI0nkKjldmY+7CrruuXoN2P2K401AcL3EvxpRQAhZu9JpMAgIIWCxtdYoxAgGAIkBIQvUkss+k5dy6dZatE6g4CaghS/fZ2utddoP64jCJh79tpN/4w/ftj3nWGsOAAgRKbVHOIDd2dk72L+F8U8IYZx7taqzrCjqijHmVerm3s3tVv/l8xfFcrG/s9dP4tl4mPS6t2/u3r59+97dg9/8zR8Rol+9evnNN1+//dY9zvnh4eEvvvz89u3bvZ0Bfkp+8dkn3/3+Rx9996Pj86Pz8/NKFel81RivPPUYx0HolpoFoLrXiwZb/boop9MpJS/bcT/wu8XyOJ8fZoFstX+nLKs/+dP/gDHdu3mAEGr32r/48hPG4WL8aj9vJT0WxPLo5JvZckwIvPN+hxBj8SQtVdwWv/abe0oTQFqiwvfj4Sy9devhvKiPXz16+P5bWTr/t7//Zbffe+udB6XKF8sR84yS0gtomS5DjoIo9n2eVXaxmI1mF6dn4/NJRkIIwxgxejYZffLll0fjySRNF1lZSK0plggkAoqxJRgwVlozznzfB2PzssAWep32/t4NiunJ+dnN3Rvb/cHx4ct3Pnjvnfff+/Ff/GVeV8ro6XLx8P79brd7cnb613/z0ySKXbVg6Af7+/sunr29ve37/mw2GwwGZVlShr/38UcWmT/+k3+ntfyv/uk/mS3H3UF3vpyenB2lRToaDwnDeZZIiytREU045wiHxmqlDV1mC9/3HZVPGim0qKpCalWWJaHIQz7W1mm7Okq4U30ipHSF1275xRhvLK/Wuq6N03rzOaWctDpJkkTnw/FiMZNaBqFPKMvL2vd95vmYUmvBGIMwBbQqhnYURRdFsdY6lHBtIeUoQlYbVytiLTIr2VPCPY9QxBgDjBhjhLrvmq2qCpDChBNCCQELmBCMiQEwWmuNCMaYYOoE3JD1wTJGfWdkARljTaPNr7Wl7hAgrdUAJgg8QjoEhxhRIUSaZkpozkKEiKqNtYgwxphyvh+QBWQopaZUxihnqR3+QbBqV88op4QCIEcK8bwAAGtsHFuCMa/VarWStlTCLVMIIb7vE+KCsdgYY0FbY7a2tqIw/Oarr5fzBQAWRd1ut4UQyIJrIYEQgDaEQOB5nVabcxrHsawFJSj0PY8xj7Fet42tkSWA1QTbOPJ3d3cDz4v8k6cvDonVvseENl4QBklSVGqZ5S4s5F4JjABjwiipaq3XGkRmQwDnde/ltsEb49J6XiUN2I1MxObq0V4lLlzboNmmmXyTN7B5iE27fN09v5ZZgDVWWOEE9GaOwqY3QhsEPRfYcBoGdsMtbTrOK/yJpjfEGmkhhIzrjGEvD3fNPaxe6XUuxlrrHO21YIbe1D/+X3bgb+GoGrhES45IsaoiM5dj01a4Bbp5jcz4bQmVjcNd+fjaiwHXFRrW/aNdJgKBRkAIsZoawywzlFKC3eKKbL57zdvg9C0QQu7UNpkov+w8Nz9x/7ss4F3DUzfHtyMll/3gnBslACHQJkna3/3ud3/81z8ejod5tZwtZ34YYIzjuIUN6oSt1vb+IG6fHx15BHdbrZ1OstVtvf/e2w8fPtja7ubZnHvkh7cOdm8d7O5tpWm6rKqvv/7yvU735p3b0milRaX0zs39e289nGfpeDaRdc05m6eVxnwQd/qDrs9Rlk+ybDydqKqYt2LfZzGxaHh21IrS7Q4hD/tbg3bS42Kc5fWUc594gzzPiYHeLq/FkjBhyEhhQD6N+0vWln5AlTpSBhFECCf9Xb7N46KA04vl0eib3e27qGI3WLhz+4OzcfXFo9nRq9PFou5vDSaTSWWyPM8Qkf1BW5aTre0WKGmMyvN8meNCBGBp4CcC5izyW73uJ199XlTSYPL1o2dfvXg2LapxmnqdtkVUYdCEGWo0IAMKEMKMYgtYWyA4TOLtwfb49FxLARiFcXTrzp1uv3d8dnp6fiZBGWN8EkitR5PJ4eEhpXT/5oGW6sWLFwasEGI4HLoYPMNkd3cnSeLHT59Mx5Pb9+/9/BefvDw++j//X/8v73749ovjw0oUoQm80BvOz2fZNM/zqHUXIW5MXtdSCOp7IcbYGE05561Wq9frlKKmDBuralGWdcUYoxQzThC2UtV5lXvWC03Y9jpJksRxK8syq+1KM1XIKArDMASjT09PlRKUYUpxURSxb41RZZkXZUYp7fe7jPtSKuYFFmEALLWu61pq5QP2fR/AKCXquuSccx74PldKMEaqqnJffADAmGAMhCBCELYreprWtqyFE2tilCatAYCrPTKEIilFmqZ1Vmldu0ZLgAzBhHPGPYqQdbX4AEAw0dYaDUZTJTH1fK21VcpqjTYIZVoIwhml1CAwxgDBjDHqcV0DIUhpUEpKrSjlRqm8rCK/RRnxo9AYrJUtS0cyv0T9SimMqTGGrG0EIYQQaoxRyihprEEIiENOxhjOeRQF/X5/sVi4bpRJEgEYF4yhDDtiS7fbunVwM47Df/2v//Xx8ZFVKPDDg4OD2WSOMfYD7vu+VUrqGhPGOYcoGAwG9+/fT5dzADi4sauEoAzT2t3VyHd8UjD9bpsx9uRROrk4y8qaB2HAGWVEKzUcDrv9gWserbVWUmitkTVWqyana10z5nXXqyY339h6+FYK+pXV0utpiNddcrMZvqq21LifzXqHa/NvTrV5lGuHwBuRg03H/8bZ3vjrtQOhtUwk3mhbtRlUWO3eRDhWatwWIWTBMMYaoLB5ZxC6svvm/dm8e8YYx+cnhL1+kv8LjGuA7PJs4cqNbbDC69tvAoX1dV0+mjcFLFb7rmdYnYZ1Ccc3bHM19rB+EGu+wpVwFFp3hW1OZjMMBhsRLECXqbTmcK/D018+fpVt3BDCAgBjyFpkNFitpay5x37nd37nf/rX/9N4Oq4r6XnB1tbWPF1MJpNbtwaiqpOtnRtvv0OkfPnkiYrCDz98r92Knj17Mhmd/m/+6T/GjH756OvBdj8IwvPxZDKZlFpSP9je3+t2u29LQShWYJMovPfwAQ98rUSSJFqrP/uzP/O8u+PJ8NHXh4zj3e3u9uCOkHmRTlNVb/W32lEwvbgo2Wxna/v+7cFg0J0uTs/HZzdu9afTaV6OClkdPXu+s9fp9G9ECezs+c9e/CKVpt3lkQ+1WAahKktFGfJiP6um84WxyE96iVJ4UY993fnm+fOD/buD/Yd/+P/4b2ez2dvbt7e3tx1QqOqC+7LfH7x8dlpi5RHKuU9IXNSmXIiz4ezkZOLHLV0Lg8mPf/o3nEa37tw7PPvJs5fHQaejEWHcMxhpVGmEhbHIGsKJ1EqXmhLCEalEPRyN6qzoJq2tbs9aSwj5/ve/f3p8/Kd/+qdJknz66RdbW1sHBwcGbFkUy+Xyw+98596dO9PpdDgcutoCKSWltJO0bGxPT0+rqmKMep43nU6Xy0UlxXwxXSwWo9HFdD4lPmY+C5N4e3swX7Ks5MwLWqxd10IpVSudZcVisaBVVQTBzsnJSX97q9PuLRefVaWoaoWAUcJljWng72x14qQUQng8DLyQIEoYWB5Ya30/BICqyDFgirExutfuHtzce/Xq5fZWX2tdi/N5umR+QCnN89wL/G63L2pJpahrWZVlLYUUyhijLdTI5nXurpMxVpal64+J8ZVmQnaVwiecc2Otc0gEoyT0MKZuEepxT6i6rkutFeEEY4QxoZhzr0JICSlduQRjHrI+AhSHSVWJMAx9359Op2mVSUHqCi0uSkppEEQexkLVhFGE0Hw2TdotK61SltIAUWsNlIXQWnNMtKjrorJKqlqklQINWqFCLT1CjbRgJUMUGAOllNIKGUBGm9paSylGiEslsqzo9XpSS21pbxBmWTZbvoqiqL/VkSU2oJd5+uDdm7PZ5HT0YjDYXi7nyNBOpzPodwnVy8XFZHwiVZokKI66Qi0//+LnXkCA6KIWSdTOZKWYZcQDSmohCEbWWqMkI3iv15J17iONQv/s7OyCIEKIWUIl6k6ng4LkLE21tmEcjc5nRVH8/EVa0u0cMr2QgR9i48lcJjTApfQNSKmMVj5hyhplDCLUaltUlTEGU+J0wbXWtVQII2m0qmu7ljwCsMgaBsxVwNnVAtdSSjGmCK3rz1Ym+9JIungTAHYJa2eWm9C9KyFprLbWGiFsjKuYR2jVR0O7MBXGGAA78ACrZLFunMSmXwEAJ3WglJJSWGsdgwcjC1ZbY123K0Kw4y1os+qD3HgaB4/qqsQYe3zVi9yusu/r+P8qSoHQ+myFUBYsgEEIuS6FsKop1crIJsaAMBgwFq1XrnhNbwQDyDSghBAGILW2UmprHaJ6s8v5tsWuMZf4Y9Njkas9OS/hy7d47E3SqFnrRRpjECUAgDBYZI01RlmtqbJKyprSlgEtVBmRkDAstbCIcrvWL3fhRljrJSAEYK3RZqM0xlorjUYIIYuwxQit/kWu45e1TW9qgglgCpggYQleaSkYo4xS3CeRHxijtZGiqjFYn3uMEveqR4FXFAVQSjivpcQYszgiBLueEwYAgdXWJSYAAUIMAGCzQbZdv4oAb3g42miE1lpSq4eEAcAai5oySLeBe2mxRQhZYwlBNAwQAk4YICO1+Of/9X/z9dePbt7wv/rqM39319dodnq+aHUObtxApBKy7nYZfbjX8rmoZ9PhNKLwJ3/wB3e2d7YHW77wFk9n58U56RGl1PsPPirn4vzFuJ7r08Px7u5uf/9mEHrdpMA32t///vcpJVmW+WyXb6HH3zx58ggYZqOz8yTeD0i30haEJbpPcdIbdJfp9NGrs06nTdudZfnly9k3wYFtdWsWRV4V11PGdsPeHYbJdJhdUD/CKkK5p6pUVkXJ97CvCLdaV0bUpgSKIAnbAnWU6X3zs8l4fnbngZpm2ddPq0Wq9npooVJSXNT1q50BssJcPJ/td94yAs1mabtzg4TB8y8/ffZiKWp+cWG82zzsxsN6SRMYm6Gc1efiiHZNRQvfDyslAYAYQFUZIaDYZspQjy/T9O79e9Pp1Fi9207aQXsran94763nXz+627uxF/cOF0/euv/W559/SS2KveB3/95vPXnyZHIxfPvhw68++zzknhTaKOsx31q0XGZaaGzxxcVFEnenw6zb7eva+4uf/CyIuB8Gf/zv/v35+FVrK8YhpFVeW0riPY77gx7xx6QoCmTBZ3o+n1dFagVgbWgQBCsqhOfdunVrpQOoVRzHQogiWxJCtre3t7e3Xf/oIk1dS8Pt/la/33eC0qKql+mirsu6zD2PMY7n87kUVRzHhBrP84bD4WKxCKKwKIrPP//c436UxEoZJwSJMXXB87IslTFaWwejCWFBELnVc1mWdtXJxiF0MAaUMoxxY4zWRptVYpVzjgjRWtdVNZ6O03SJMQRx4PmMEFIVZZIkAeerfscWZF07vne71e13uhhjWdUe5YSQMsuVBIJJ4EdlWabLgnISxeHBwa1ltnB5YawAqFsTaGN0ltdgUFXVQiitDQJCEMEYRVHEme97pCwn7tAuMu/xIPBDjHFVibKsXZ+qJGmnab412MYYL+ZLJXWvuxUEgRBCm4rQeLvbiSK/1wsvLi6qsqLEai2rMs8WxPMJMjSOO37QbXciJcPZeJLmr/JCJElMqFBaTKeTbrdrtcUIEAbPY0ErdgI7osykMUenp3meF5WolLEIur0e5yFQr6zr4Wi2yNIgCDCmrlAFIRQEgaEMI6TBIoQIo05CXzcBUbRihbs+0I1dbmDfNeICbFQ0NCZyM6iwyTC4XF6vfUzTuXszGrE5mu3xRtVMM/+1kMNV94au/no9AfH6Yu6Ny8HN2MDmlo5iAuvI//qdvyJ+8PpFbd5ABySunfy1C3n9tJuYir0eXfiV1qabgOD1Y30bqvhVJmxmcNeI1wqta06GlVISIfI8b7fbsA7aa6mklEZpxnFzORsXdf1Y6G/ruOisnyueQmuV67quA7iMBxAM1tqyqF2EVSmlpWjSN0EQUM4Wi0Wr1fI8L81zKXW32/L9IM9zyhla13TgdQHD5n34trfrTbfOvckrUS+AdY0rsuvSiM2q10ale4W53e3hnN+7d+9HP/rRH//JHxHCxuMxgN3dveFE3uqy3N8Z3L17P3yblotZsZhzziPfU0p9+umnezu7e3t7WVZ8/vnnUzG+d+/eO++8c/PgdiVUu9fdN7frul7m2XQ6Vdq2ux1tTbbMhRDKmI/e/s73Pv7hp5/84sd//uPZcPrl51/12h1Q0miJrBn0O/fuvy2lODp5MZ2Ov/76cXvvzmCw3e558zSrKr8oTJIknPOiKgirtZWACQLQoI1StZLlhUxavhRSVIpQv9ttUxxVpffk6WFRjh4/GT0/LPpffw6MTKdpqwvz8VgchFtbAbK+knXkJbqCZ89OHt7/iHJ8cpZOFq8eP1vOF2CszWrAtVSYetzvxVEYhkIoq8H3/VKSdVbPgW8MoBEQSleLllXvUwvz+RzyOgT0+Omj3cHA9/3T8/PpfDZfLpb58p333vnggw/m2TKrCqD48dOnw/Ho+Pxsf2/vYjxK0/TBvfvW2udHLx13bX976+PvfG86XxwdPqHEYKTqSvoeUaKajKplnZfKRO2+F3QHvAOYgRJhK5G1GA+HZV15QTAYDJRSNInDqiqs1UqpQbeXRAHCNs/J+elJ0m61k1a7lRCKXQrT9/2trb772ozz0Ww2e/bs2XK5nE7GSZJMp2MM6vbtW612fHZ2whnb29u5e6+3vb396PHTyXT64MFblNLxZNbp9gghUlsNlgBgSrA1UkqtpIFVBNXVgzkiggunNCXm7kP3tXFNgJzQARiEMbhoxNnwzBgVeD5CBmHLOcUYaa05ZYxQjLCWSilFMfH9MPD8JEk67Z7PvfF4vJjN4zgJ/WApZCvsZFmmhfVocPt2J4qiyWQEFnHqKSWU0dZapK211nWD1CVGAEJoKYzWCGPAgKxFolYllL1eHyGkjey0e9YiQghmtCzL2WwhpWy3OmGQ+L4vhLJGuEAygGtUHVDKRW2UWQAOeoNOuxVS2mJMfvPF44CFXhAJYYos14ohZBkJkzjqd7snp+Pz8/PJZOoFEfcCY6xUqqzSro0JJlrrqpA+RVHQ5owJIZbSGAOj6VJKyX1PGkQYa/e3K6mIF6paz7NqOJ4HQY0Iy7JMSRP6ged5SkhVC2MMWEspbepWLIBxPC57tez7qjtZOaoNTIDXmgeNKd+0hpsB202fR9YNu51LIOvxJnt6BVs06GRz7bvpOJst3zjDtStqcMA1lGCuVlduzobWVQ/NQc26oNH93Gy26e1W8QIMG6hoxYi0GwQOl3F4nd3ZDJfjaA7XPIJru/ySe/htn7x+H36V4c5q80pXDxSQXhciEozBIowxAUSd6hqhZN1Ygax6iMtN6HONxdIcy3xL1WszGhS7+StCiHmeqkVVVUoLACjLvMwLIas4jrUQUtUY4zgM3JoqSRJRV8wLATOtARBWepUrJJa66111pwSEASFA1urN8/zV7x5C7sG5N38DK0CDFSy4iM4lenAvLbiw3PbOzm/8xt/7wz/8A9/3q1Jwj0ZRVBQVQZZiLIQT14/9bj/k3nR4wTApK/H42dPxdD5bLNM0/fkvPrGRkVo/evJkOBx+8803mJJer+cH0dGrk0ePHkVR9PF3P/J6fZSm89PTJy8Oj6ev/vk//+c//MGv37t5Twv5//p//rez4fj0+FUchsPzie8del4QJ/52/0a/t5Om6aQUASNZVmoFWpssLdNl/tmXxz/40R4DiREinBFgmGKlDWAbB/sEjFUlxoQw0IotcjWdVhcX6XxRpGkRx3D/wb2bd/aX2Txq+XCxIEaUaVEWRV3p+PbOWw/fmm/hw8NxWePhdHR4Nl5kQDmz0OOR8qh/c/dgZ9ANGC7S9PDw8OJijDEHAAQEAbl8eQBZBBhjTijG2DGEMMJ5nmOq8PYexvjDjz8s6+JnP/vZk2dPX52e9Ab97//mx/fevv/JJ5+cnp5ubW3F3Var30Gcng4vKl0viux4dB76QVaXjHt3bt364bt3LQYzKQaDeJEOB4Mb/+V/9Y/+/u/9zsnweJovsuMiTwsNgScDay0miIZ+URTTfPlqeJ4vlrvb22ErUUJShJBWqpUkgc973XYURRhjqzQAbPUHnU6HeRyMLYtCCGGNzpYTBAQhRBF1gfo4jghGZ2cns9nE5xSQiaLg1q1bebbknBVV6QX+1taWlLLb70VJLJXp9PsY48U8XaSZUBKhVQNlA2CV0toqpaytm1IxrWuECEIWY+uggBNo0tpaWxmzUrO1FBmltRQAhhPqhQHudrSRSgmlRCUqpYVRqsxzRj2PMp9xn/F+p9vpdLTW1qLlfD4dj/Ms85lnuQfaaKmNsb4XdTrtnZ0dTGA4HC5myzDyDWIEIQQWwEop6kIURclxBwCsRtYYa4xjqBhtl3UmuOp2B27FwznnvMYY+2HsxBIwYpRyY6CuZV3LIIiWiwIAOPMxxmUhRK0JIcwTmBZRZKPQdlphlYffmMzDfLsfT8ZZlpZIK8oJwrZMxRTnF2fnoqo9j3uMIGQJAUKw7zMLknvEKlyXlWtbpTGui7IQUkrZ9YNOu1NLpYwJwlhatFjmiPBag0bUYh9IgAkBogipMSWUUqP0iq4B1oDVYDVYY63FK/9/2Ut6wzqvjdpaJtle8QoYY9jo3bxpyhsvCFeXiZvW3Hl9V1682VTp25a8m87A/buJOZo532i1N4HFGy37tZ+bngWbwKg5gU0m4y93xpuCVGZVnb+i7znRxjdetX1tNHTOJoDhFvDfhhNev7T1v2bzQM1wX9hfPs8bP29AxuUjNhZjRBHGBAMmPvcCz/d9n1OKrK3rmpDSCTVGQSjK5ea0zXu1ec6bB30jpgQA1zJ3JSFPiItcYoyNlMs8m8/ndV0KIfI0nc+neZ7TVRdQTRGO43hnZ2f/YK/f7w963fki5Zx3u904jtM0JwVqt9taGwDABKy12FpkLQZAANZsvKUb5/ltgMai9RMHQEAczkCIrHEABrSmN64mNhu9qZowg5VSMqC3b9/e3d1Ns6XnMWt1mmZBpxMmSRz42TJ9kuejJOi1WjGnw8l4Yq1QknsBYHQ6HCqlBru7BeTaotFkFkQJYd7LV6effP65C+B99dWXH37now++9z0IAqr1vCieHh3t7vXTZZlDBYghgNt37v2Tf/xPWnH0x//uj7756svRaPlnf/4Txsjbbz88ODhQslgu6qKsJWR+nCyX+vmz84vJrBT5d3+4T4FarAnlBKglgBGhnGOmZ8spoDrpBkKZw1cnZ2dVWcKzp1AW4Hlhv9vb6t2+f+sdS6QfsPa9xeOnn5RiTrBXlsVoWHbagR/t/PwXnxAvnKblNAcvjAxuVRWjQY/akgDTwgqlF9N0PlmCRlJojLmxCBCxCEDrNZ4HMBYAGKXGGEqpz7iVJgi8wXb/oLtlkfnq0RevTl9WuipkseVvEx+fT85+8eUni0Xa6rff++i9KIqePnn++PlTxGgh6kdPnty7d+/WvfvtdrvVaj0/epRlxc2bt/+P/6f/w88++fnnX31ZlIvnLx5HvZhLyhhDiJaFKqrMWs09NIP81cuXp8cnZZZHfkAYswi0NZQA+Ix1ksQgEEIgMB6nW1t9F8ebzsZGQ5IkhBBrdVUV+zd23bJDC21B16IEAM7p9vZgZ7cfcN5uJ4vFLM0WL1+8ODnhv/0Pvss974OPPty9sae1rZXcB2yM1dYggq210mhnIN0L3lS0N0sKa62TCoB1wBmvlRmllKHHATAmCAEFAKNlWSlUo8DnFtm6zPM8LUUJyGIMyEK/2xNCMOZFUVTXdZZlczQLPB9jvFxmSqlOq92KE0JoXVVlUcwXxcHBwb27dxBCWZqOJsPlbNnvd8uyXi1TCAENRipVg6oBsMVAlERGE6MBLAXbWHOEEIqjlqxkLUqp6rquMfW2t7fv3btXlvV4PF4ul67GgVIqhYYVpOAYE6UUISRMDOeGeSoIbKfNFgnjqK7LichjVRaqFggCz48ZY0ajbF5YpQeDAedcGS2EUNZgRBBCccS7rTYGks6XGBBYbaSVUvb6nbqu79y7u7e39/Llq0WWtrudoqiOXp1kVU0ws0CCIGReCACUSowxIkRqJZTUWruAsFarVkKrlSwCC6Ct1ZtKBva6SNGmKb/8YcPrbKIE9564hWAjmgQbColNc4dm2usmdWN1eC3xcc3rbwKUX7LA3uTbbyYv0JvC2q/jleagTXXo5nGvETabn/Gq/v4S2Ti+wjXE0xzw2uU3CAPWKZtGovFXGa+jmdcP8W0u7Vec2Y3L+wDI2pV5RQgRjOk61+DSDdZaTaVGWGHs0knNw712Mq9jhV9yqnajiStal7Baa18+ezwajYbDYZHlrn9KURRKVA6YIguYgOd53W535/lWu92+f/+ey5DuH+zdv38/CAKfcC8IiyxFCLlGVyv/gSzCdjOicHme8K0ZoTWNAgOAa5kLFq+TEQ1cIACb9RENtrCXsxioa9Hr9X74w1/76quvgiBAiIhaDaKEES6EMrJGvmc05Hm+nFRgcV5klHtBkuzs7gIAY4wSPi4mYRgaSyzgGzdv8SB88jc/Ozo5QgjlZfERY5VSi8lsMpm8ODnNhHx5fIF5mITRYjb907/4D2VRDHb3Bju7/yROvvPd711cXLx8+XI4HBLeHk6Kzz9/QoJI6CULVQ+3Ls4XL18OmR8c3NzShtbSEACsjAZtrAVkDcLz5XEtcsyR0BoTxkK/1fO8KmlPZknS6XfvVMK+ejEP+OjOvYOt3RuseDQY9EohC0GWWX0xLs7PPx+Nv35+XPgxqTQBmhCvO5uL+axqt9vFovr682/qPL11Y2d3Z+venQeFkF8/fYkQQYjY1ZtM7KqRqQXA1iKMqNVmFUG3dmt35+adWwnmn37xi4uLs9sP7nTSlMXsnfff80L22ReflnURRDzLlst8nhbp8xdPZ/O5taCUAIxuHNy8//DBcDj88ssvPT3W2qZff20wavU7lpjxbHzj5n9RWdGPIs18Gi3G83o8FUUtAu2fzk9fvHxZ5vmg3W23WohgoRRjjPYHXZdTmE0Xw7OzKs8rKYMgyLNlGCWh50spLWgA4nMvCILFYrbi4ddiJSqsNRg7GPRu7u/vbW9rUx8dHQZBEIZhp9tqt9tpmmpA83T56tVJWZbcCxjjdV2neVkriTGmhOqVKoxW0ihptLYYWc4Io8wBBSXd24ytBafk576qJGxIbdgYEErJWmmth2UOAFLWlcwBrO9z3/cpRVqqqigFlu5rwghtJcn21hbn3OozIUQURdbaoigzISkmu9u9tx/eZcSOp5PJZHJxMcKAfD8cj+YIISDYqTNJiYz2MEJVaQkyWoOSyFqCAAMga7Tv+5wxa+3u7i7FuKqqdrs9mUyErJWW3GOtdpIk0Wg0Go1GaZrPZkWr1QLAk+kiCIKdnR1K6WKeUr/QphJVynqhx0wS4n4nOH45vKiNNT7HHico8njYijFFUtUDGISRzxgrRS2kRNjWUldVFYdeGHEC1EpfCQ1GCQ1KyKAVdLvtKORR6CdJIGXNKKmqvCyy5YxyP+SUoTBBCAmtKOUYEBgrtJZaGbAIIzDQyC+6pIMF0MYaY/R6udlYXliBv6axwqVrdxuQtSgT2hiwLo5oPEFTGtDIADSf23Xt2abRv+aHrvEhNk8S1qe1XnOvfr202htsg2uRgGZcXxOvW1lunsMmZe91YHHFGWz8FW8QGNfz/+2O+RpKcPet0VH4FR38G1ECxgSu3sO/G1Bo7uq1Dy+fvtLIAhgQWCCEtJBaa4Kwzz3f9ymloI0oK9c1uokTbL4k8B8DZYqicA6eUqr1/4+4P3uyJLnyg7Hja+xx99yz9qqu7i50Yx8Ag9k4HxeJpJaRzPSif0Qy/RcyPetNZtLDZ5TIITnkpyE4MwAGaKAb3UDXXpWVlXvePfbwVQ9x89bNrOoecESZ3KzKbsaN6+HhHnHOz8/yO3o2m02n0zRNH/32N7PZbDabNUW6rbVNiY2yLMmSetxmk3FyfHTuuu6Lvf21tTVM4OmLF1lavPfee71+pyzrC9c1WGvBWABtwVqLDbwpOP77T2ZDXA2LmIOlD+LKBwvQWC0W0QmN79xexKxQytq4++Mf//G/+Tf/pq5L7jDPC2azpBWFoe+GcWtz0Nte71Ow89nYdVjGWBCFs2TeX1tvKgnPZrOqlpTpw6OTLMuiTtRf3xhsbr04PMQIEONJUb86Pu6srx+PRo9evJyXVeR7CvPD0eS//Of/9Jf/z3/zwz/4vrAgpIy6vQcbmw8ImZ+Nzs/PMSZnZ2fDpBQF/vSLV7VJBhNVlJTR2HOiTitSimiLEWikgRorhMREY4zm2aTb7WCKprOEu+Hmzs21jej8VH326/P5fCJlVJUqzeZKlHU5Oz/Zf/82WdvaPBsVe09fSuBuuH56ig9PMmXjWngGU0aRBVdrQ6n1PF8XtszLMq3xFnOZL7Vuh1EUTOaVskAAEdswY1ho9sYEkNVm+VhKWRNEdna2d3d3oazOjw92b+yurfUfPnm8vrX+jY8fiDqbjM93dzY3N7fn8+TZk0d5Vs5miVYqjtvf/OY319c37915r67ryWg6nc57EcnKcm2tk1cqK7Pe+mbcb+2fHCtkWOAJAMQdg2xaFfOk9iQ3SoW+32931vsDWdWTySRL0lYUUUYoI9RaC1YXZeZ53nw+l1IYYzvtOIpjIVQToMAYC1sxwGIrr4UuimI2myXJrMjyokwIBYdhxojSotWK+4Pu9vb2PEsPPj1KkkxKOZ+np2dnruvfvHnL9X0hlDHAOLcI4CKMXCnd2OyNsQhhQmgTwFjXYqkPEEKEUMdxCSEObcw4RFvQWmopGwqH6WSCKcIEEIKmWDNl2BhzfnrqOE4rDlph1Gq1Op3OxsbWYDA4OTlxOZd1XZellLKuBAHkO2486LQ7wcOHD+ezxHV9zqnnBbPJXEqFELG1lUYpqbW2ABghD4FFgBHgxilsDW6YXo0BoyGZZ1ubO0aJk5OTfr9X12Ul6sPDV+fnx5ub21tbW9eub2NilZZClpQ2uR4WIY0xMEYcl0hZS1nPZpOd9R4jdq3b/vjB+zKvk1k96A78oC+0BYpchzDPEQIpWVltFFKEIKJNJWqjtcMQIFVXOdJIKWktSCmV0EIIORldu3bt9asXZydHlLuz6TRJEkp5O44AAxjlcsYYCCGIApeSsq6aFcEYG9CAERCMGdVaW0AACJmFVUDbJnd/1XR/4RvGqLHoNjpuFQoQgt+pdZbn4JUSSsscyCX5/0VZjatbZHvZeLuIqP9qjgS8TJe/OL7U/W8jgytYBy6Ai13xYqzGTi4vdyU5c3VsXw0dmlD2N+O5MkWrR1bRzxU8tMpseOXe39neiRJW5+GrfvhV/VxpVyb2zXFYTCssDxprm2LoaMFAwBlr1LldofRePieNMWAVOlz58M7WGDUbJJdl2eHh4YsXL05PT8fDk2bD1GAXYwwhxGFcNrwoduHQqYXKiwojNByPJrO5UqI5ZzybXd/ZuXvv9rWd7cYw1BAyIAvIWjAGNUG7y4f/H4r20AvjAVh7kdlhm0wHsxKvgC4RSTRIwiJYhERcMIUTghS6d+/e97///V/84ud1VYdhmKZZ6AekKU5AueN4CBQhbJ5kUtQWkeF4GrdHiBBj4PT09HePv2j3uj/4wQ/WNje9wCOUt3v9eZpsbG5WWXpwctw/2vjGd787TbNHz56XUlVJ9vLoZD4Z/83PfzGrRCbE8WgYtttpMm9HLd/3g073RquHMW2v77bXd2dH+dOnT09eJ5PZqed32vFOKerJpLhxbwNIiUAj5CDFAbCF2iIqywh0Ryo5nabcMX5IhVDHp9OkFLMEev1qsB632rK/AcoeP3t53ut8+N76DTlG87JeW7/msJ2z4dxvh7UxlDFhc21KWUnPdV3mOJRa5CCm2uv+h/cfgFUPHz3y45br+PMyJUA0kGYhEOhF+S8LF1zDyBijlMW+c+PWTS9wp8lsfXONEqKU6Aw6zHN5wKej6eb6oNVqbWxseJS/mCWM0o219WSWbm5s7u5e63f6dSWPDw61kNe2d/qD+PHjx73N64WUnz985ofOXJWfPvltrYUTB9gJqdslrFMIogk3hAXIXev0wiCIgnBcDuuyqnReFQU9Pz9v5CkhxHPc7Z3NNE2FEJ1eBwBGw2GW54w5hJCmyp9GCixyHCfygziOoyjyPG8+nW1srgVBoLV2HOJ5HqXkBEApkaby6OhoOBxvbm5ubm0NR6Ozs7P1zS2nYVq0jcRsOOMQIW9q8i6b1rqJCF0GpjWlsZqiF3kyBACjwWrVMDUpIWohNjbXtNaVKIWorLVGa4Ostuq9u/du3Lixs3ONc968w/NZ+tmvP53P59PpvDHvSykRwo2sybOJqNdPTg6EUJuexymmFJ+cjDnzjEVKm7oQeSmNAUop547rOBiBUsqY5RNgjQEpNRg7mUwauZ+maa/Xq+taGZ1mcymlVLW1Oo5jzmm/3+WcNp10um3GmFQVJrbVimrFKeWirgEM57yzEYT8m9k033t5fOParh/1h6PZNE2lqrFGxug0Ta3V3GWO42RZNpmNHccZDAbImrLMjTRGAiPuQrhr0+20bt3c/dWnn52dnN65d9eoajKZrG9steMoK0qtJSMUELJKg1YM07kQTSYhIgQRDcY2G69ldJhFYOylBV0KqCs61VqLLuLUloofyBtVsapZrzgLzAW9Y1M2zHEWxUjRIl9RrmqdVd28RBJLQLCSdXlJlS5/DhdqbPn/lZNXm1nhXV7VSat3dHkG0JWD5oIgYXUAK/03M2ZXRn7Rib2kbgGu/rl63QYirF7o7Wv9Pk1fLs21vBf8FWmQX9VWZ3X1yCrso4QQyjjnDuPNOoqyQghx7nqeBwDoXVxVTZ9XxnPF5PN2a0LTG7vLfD4/PT199erV69evfZfChZ9LCFHmBQAw5siGlh5jht/UjTSAba1OT0+LogjDMHq+NxwOj69fd133+u61NwGGxlr0jkdu+ec713HZLqZ9iQxWsMKbKITmWng1A2L1ilprAFYUZRy1f/jDP3zy5MnBwX5ZVGsb65TS+XyeJ8Ax9jlJk9n+i2eT4TnBOC8LiklZV2fDESEEMKLcLfLq1q1bt+/eGY1Gk9mMOw7C9Hw8QcgqY6WxBuzx6cnB0eHa+npd61ppQ8hoNicOf/z8xZMXL7/7x38SxhGyIIQyFhzH1RrCVre3uQM7+O9/8VNN5Iv91/NZ4Xqkqqxbg+tEQCuCDQEPJDcyt1Ah6g261zll2op+J0QcS8nHk2KeFIhA3CV37m23W/7pSUb5zOBK2dPXJ+3d25vUdW7cvnPr9jeePZl+8fDx+JzduvVNTCkSMM+yui49NwIEyWziaDWbzjtxOOiuGSu1/LKq6iYB+WLBMIAGiy6IN5A2CvBFYRoMrutsbW0JIV6+fOlzZrV0XffHP/5DS/DRyfFsMr19+3ZRVEWWhoG3vbWJERFCxX4UBpGs6tev9ofD8XQyb2J79w/PNfCi0KPJOO70MYPfPvoMe6iQpc0SYF7U1t1BjLlPMLOEEoMYJlhbVQuKSavVElVVFAVFyNvfPyGEeJ4nBG63Bg8eeGVZai0ZY43KXxJHK6WSPA2CIEvm0/Go1+t0Op0g3NK7g06nI1V99+6dL7/8UiMYDocSMDCPISdL7PbmnXv37mVZ9ic//qcHR8dBEKhacR4URZFkmeP6jLEGjBNGKGfGGKNB1MZojTH1edvKHKzBComqVlh2o1Y3ailRx/HGdDoty5xz7nmcENZux0HgUYYB7GQyGg6HxiqOUeh7cRx/+5vfa4CRqOrGoZgksywrq1LXtQjDsKoqawEhq5S8d+9etxf+9Kc/J8pQA88fPXL9CMFUVQobgohTFeVsllmLAFMAcBzXgAGEDTYWGUQMZwxjYiS2Gqy1nbgzn6SqptiEv/v0hev6Es8G/fX5fA6WPn36stfrgbb9fl8RGcW+LKoyS0MMPsMqy4ye+63+TnDt5q2diHl1ob1OSCP49h//qHft9MXzw+PDPSHM5sa2EJIAKqraDy1CJI5iSimjthX4lFJjwKEMACNnkUaldY25DQLueZ4y+vbt20fOaVVKz+9o44JlgBzPd5TRtZRSCoWMIUaZKmReIz2pxZhwDVprbSxgQi1obIEAIEAAxlqCrVWUaa2VkkobAIMxthSQxUIIx3EYZ9aCUsZa61CXUceihQ7DGlOjGWPYYmttWVee5/lhoJSqqgop5HleEPhCCESwRaCMBikWygaBucg2xBhTjAjGFsBY08TtItTs3BCAbZQR54u02yY670LfIG1U4ytp4tPt4mvU9MyYA284+5pKhpeCD5Z6q9E6zZmNEloq7FWJv8RAS+yyqswaIIYRJgQtawoYqy60qUEIIUDNEJthUELyPJdScu4wQmUtlJBgLQLIs0zWgmKizMIOLaoa00sb7uW9XAE9S43eeMev2DC+pn3VCUtv0bIrc0FnYTEihCBKpNFKWkswAV7U1bzInNDnriOtsaLmnGOwgeM09i5onEdLp88KEEGNAb5xCVnMGUcIyVpovdicNJQNxtiqLs/Ozl6+fLm3tzc9OydaagnLeAUAcF238YkhAgDWgBa28R4sboRzBgAaGY3V6fjs5f7zs7PTbq99bXdzY30A2kRhQBFOk3kYBIQxISsAsA2AbhQ/XtgDFvwYzbAvwguwWUGZyDS5soARRhQADAZAi58vSEM1u1hHAGhsMNJa6/k8TaeeTx2Hf+tbH//bf9up6zLLsqwaxizCYFtxe2djnRg7Ox2rwsR+zxjT2d44GZ0ejubj2avBxrrrebmyg0H3i8cvw8HWLBNP9g6zLHPCfllXOzs7laIHJ/N/9+//9uDw1As3j0/TqBt9+Wz/i88+JUFXCqDB2je+90/KkmkTEUSxQznlABQsqFpVuWp1h9/9ow9JhFjQ+69/95kizw2CaG371Wvwwy3XY61WKykz37t169adV3uHJXuV1LjX3eh0Q87i8+FsYz1Ok2ft2Hn65OlPf/rT++9v37416HZdIZLNtc758+Szv3nInOj+B//69vWPDp7/LAyfxR2/EmORSwQEA/WYi7TJs9NBt7eBre70i6o8PX566727/+xf/fnPfvHLRKUuFXmdRjwGwHklKOUYMYJZyhLOnDrLZVqv97rpZPbDj76z1V87Pz37xWefb62v7e7uAtCTUVJVlTHwg4+/m5VFYerReF4KIYUus7JKiirJOmHbalPWlSXYIJtIlcwTOZUA8Or5M+Zz5uBC1t3+xihNN7ZvZ7WqFVI6KArwYxZ5AcYoSyqHMoMgr8raKMQpZX4c+ZRzvr29PRwOMcavD17dv39fSglg1tZ2Z7PZZDIuyxIh1JAcJOnMAJaUOY7TabV939VKhGG4sbGbpvMoitfXesl8vSgyBNKaOo7cl8+fr6+vt1qt8/Pzqqq+973v9dfWx+PxwcGBAdTQJ8BFuVhKaVXnS8FNEFnwziBqjKmqAgFyXVcpkSQJ59zlrCjSwHPjMNBGGWPiMHAcxhjTRna7ne2tjVevXp2dnfT7/ffeu7uxsTEaTYbDszzPKaVBEACAlLIsy4bWaTabVVXVbreMMYyxu3fvzuYjSnhdJVlRSmE8n3he1On6o/G8qupFbScNccePozZCaDIbU0oRtlYbSonrupxTZCynTl3XhJC6rq1GjWHcWht6vtFyrd8/G47jOC6zPAzD16/237t3t9fuWClm45FVkmPEI993vXidfPs7HwuZnp4eOe5m3IowItevXz89mRZFUVby7p0PJ+MZIcRqGfpuJZCoRYFT13URAKUEY6yUqMrScX3HcTCmzfoWeZUXOUFsPJ6mST6ZTV0nRIQhjAln1liAS6bspiDFakMrjLZfrx6WnSw/LI3zSyXUxNYhvCDXWmrTRmE0wa2rSrQ5Hy524cuyBcsrwspO942dYGWkq/pvuXO9oq7QmyrYq7rzajzm23e6+hV6K0bBXKQbvHPqrhxcNTysRuqtnvx2yMXqXDU/aTYxzdz+g4ERV9rqNveyCeTSsP/BJ+Gr2lf9EF2QJduFBQgtb6HJi27aKtUmWmnLdV/QqFwu2tQ8MFVVYYwJwk2qqjGGU8I5n8/npydHr169evny5fHxsawFY6RJblxyYsKKG2vZ7SrGqusaALTWVVVlJMuSBAN68uTJRw8+NFpurq0bY4RWrusSSrWUmlw8hAgZZBAQZJFFBmOKASzYBitgeIMVvr7Zryxd/Sb/FgCkFJ7nSSmtNevr6x9//K3RaEQIyfPSdRyKcFEUk+k0DsLB+ubO1vZp49LttOPz1jiZJWmaZVlVVWvrA0JIks6fPHpYS9mEahKKTaWzPHVc9/mTJ69e7dV1nZdFEAZffvml6/HD1wd1njmUeZ6rlHQ5K4RE2DJOMHMAEWyAICw1rrNybWPnX/6re//0n8Od9/7zf/nbvz09OysLORnPaum6JXWdoN9bX1/f3Nq8phX+T//pkyTJwmD08Te/u3ajgwmnlL93/87B4XPHA0RxFHd3d+4EAXny6HePD1+J1BmmBlM/Ec7hVD5+sZeWNdOkEoYxxgiuKmmt8n2HAHNd890HHyKEtq9dz+oyCry1nZ2f/eJnoecXZd1YXg1Qyi0CrAFpoxBFVVUxxizSRV51Wu3bt2+7rjc+H7quO5slCA42NjaKomjS7/lOB2prQBqsjK1rUWklOLOaGmxqa4BjiyhiFCNjtTHh2lZRZrWsEBCHcZ8zX6sAIMuyUhgN1AIt0kQp6wUiCKIwDBFCGuzykW4oAChleGNzq6oL13WHw+HHH38sZW2MStN0Pp+dnp5OJhPKcONW0FrXUgIgSikhKMuT0ei802n1B61evz2enD189Nnrg1dRFAQRzksQar65uVnW0lrrum5TZ1Jb6Ha7cRxbhKfT6Xw+NxZprfM8b95brY3WWiurrJZSE0IYZmmaTqdjh/FOt0Upret6PB47jKZFHkURxjjPU2ttFAVSOtrIMAxevnwZxcHu7u7m5rrjOI7jnZ2d5Xl5fn6epmkQBFLWUsrpdJpm81ar5bpuVZUA1nGcplh2qxX95je/yfPKGMSZ5zoUgE7G0yC0Sto8q8tCZmlZlrUfdhzHS9P8Qu4sXlqttVJAADUU1M176Hpet9sFY5MkU0LWdb3WX/ddryjr49GYICyl+MlP/vpbDz66c+vG9vY6sgaUJAiiKLr70W5eZrXI291OWVePHj8Jw/DWzdYHH3xQlfb4aHjj2u7h64N+vytkZa3ttII0BYwsxUAYNQisRVIi13UxhtlsNp+nABAGscWokmK73aKMaQzaWqEVxQwIRkCUNQANgzYCQGAuEv0vsg/MSsz/1yuApZZ6o/Auws0u9uILQFDXNaGo4S1vIIi+qMvXpBcuq0zBhaN9FTRc6BK8jGpcFd/vEqBvJPsynRLgSkrkEnBc4jxeOeHNbS4n48pxtOJ6uOI7bwiXlvps9YdvwM3KZF4AhXckj8AKN9RygZZVJK7EbbztdHi7zyvTtXKhVRPCpSOrwOiruvpvaujCYb9cr+ZhaPIhm0abCMaLALGlCefK+jbQ8wpWYIw1cKGxBi3RlVJqOBzu7e29fv365ORkMplwyny/VWaVWSHPhguc+s40S4SQEKLBH0KIilSNI+PZs2e/+MUvTo63f/zDHxGMtJBxHFpjqqqy7hJPN+YEgxACizHHAAgDQbAoTYO/Noh1sV5XXF0AyzBYu3BULZZJSun7gRCiqkSn0/nxj3/8y1/+cjZLABMLBCxOsmLv9UEvbq91O3F3gAh98uTR070XUkqFdFmW3cC/trObpkOlpDHi6GhfKRXEUV7MCdKdVpjnc9el83SuZ7q3NmASlCzXNwbN1hRjTBkRQnzx+WcMgef4BGPX8eK4HYQxGFTkeZYV/dCPW90o6vth76Pjyf/0k58j4mhLw6j7zW8+2NxcA4DRaDQeJYxMfa/9pz/6X/z24ZfPnj2bTgrfGyVZCmB6/a7rMc9n3KV1Zfb2hkrUL16cZqmsNTKTqbHJwVhHe9PJLBWWag1hFElRZflEiXznxo17t24c7b88Pjm0ohvH7WtbPY1w0O64cctIeXpyirjHGBPaADGYO0aDsUYp4zC3VmUcxrquqjRHXkAIOzw8PDk5bYXxdDoZj6aU8PXB2vb2NudcQgncxJ0w7Iaz2ey4yss6oQb1YxeU1kYDNHnRmChZy/pofj5L5nme+ZGzvrUetyPHa4WGHJ2eKG0QA2x0lc2LJDWtTkBQ43DXWjeConm2lVJ0NBq5rqu1NkYVRXZ6emyMmUzGTS0v1+Oux+u6llI6Dmu1IlHbJkDBcxyrpVGiLosim+1u363K6Ww2AlMFXqvTbXkuIIRctrG3f5AkSbfb1Vrv7+8LpXd3dzc2NrRdWO1qodI0bbgXW+2w4V7EGIO5qOaCoYkYauo7YYwab0iZZ5iYIosaix9CaK4qpVSeZ4SQW7dvfOODDxFCn3zyiyRJ+v2+MYY6lBC0ttZvtVpa6/FkiLBtt9sAljHa7XaVEtZqQpDjsL29vfF4wqjT7XiACaPePM2G57Oqnnp+BIA5dxEieWaFUGBJXQk/9q212khtVaPqpAQCyOWe67qN4zwKonar245bSZJNJns4jHa2N++/98Ff/dVfGa3Ho/PNzXWGBlWdvdx72onDu7dv3rp2p9Npx2GUw+yTn/x8MOi+/+Gtg/0XCNnr129mWbG+tvMH3ws+Zw+TdKZNTRnkWVGW+Xv3b0351FpLuSOlFkpag6zryLpChDbsXa1W6/qtm+vr69ba0TixlCJMmOMiwqnDEVBEsFVNcOJyhwpW26bYx3I3DBdb/K/XCo0UaOodLxXMKlBoDjbog9BFvTFY2as1kr2BAg1NwnL7uDxzqRLQCnHQij67cMeuBBysytDVzDpY0dPLCzWIYaGE3to+Lv+U8g3ggBVg0UTGLfvXb5VfeifqWh5ZxVuNTnxzX2h5poHLaAMAlpF3ALLRf+YiK3JV260M9d2w7wpeWZnbS8P7RwMFdNkN/855sBcMHISQIAiaZIflFd+JcpbHG+i5ZPJeYkGEEOe8QVSU4gY3NJWLnz179mpvL03TBjroRSE3WF2RppMl+LsyhuVaEEIs6Gb7hAElSfKLX/zi+bN2r93xvW9hC1VFFSFSSspoMyxA1mKEDAEEFmkwFBBYrAEQRgTesCJ8XbOXA2uaYxc3fsmiRjADWNCXSanv3//gxo1bJydncS90GddK1bU8n0zSJMuybJZmke8VlTg+Ps6K1Pd9zlg2T377m8/Wt3pKKUYpWKu1rvJiPBxhjIUQsha7u7udrDOeTgbdniiryXxGfPfs7ExLgcFyzpUWjx59GbrO/XvvESCglc85eC5Qx3Oolph2B1vxAIDNp1ldQ1mrbm+j1Yo5C7RCVa0czrc2ryPErLJCmI8e/Mhz+lmiOQuSpMiKDIjdjbYwA2U0s9wgdzI3o9P52ZkO/LUKdFEKCyQ3FUrOGHX9Vg+MFUqKupSiDH36zW/c+9Mf/+Dls/5nn2JZzKey+Hf/5sWte/d3bt4pDo9bUeTPstoi7vrzrLJaMZdZZDGlYBWymnNOMbGYIkSGp2effvKrg05vcn42Oj+Xteh1ukZrpUwct4uieLH3klDaW+v1+/0gcGajswwJhumgF+ezrJS1NKoslMakBp2XZW6jRJhJWpAyz5QJJ3OpRV7lAAAY+czljBdFVeQpx0hFPg5iKWUl6ka6LpOhKKU4yxLPcxhjg8FgMhlvbGxgjDGBVjva9Nc3NtaSJKnruokoBDCEYKVFURgAE8cxpShJZi9fvggjd2dnZ2NzgAlQih2HUUpn42QwGGitmxjJMAzTvDg4ONjf3yeMN/u2JkYhCIKGQ08pTSklmGEg1iJKqcc9Qshg0HMYF7Iajc7zPJVSFlkaxQ5CYbsdeZ5XVVVRFITgMFxvtaOPP/54Z2fniy++ePr0qeu6nU6nKIp8nBNC2p1YKrchYQWwjsPLsqzrst1uW+scHR11uq1Ot/X5F59lWWmMxRgXRYGREkIaY4QQjmscxwlDp6xkmuSNrGGMLXaGZjX/6s2uUQgxmUyypHC4hwEhhHY21v0w/uDue9eu7fTaHZ+xpEzm07G1+uR0DlrdvrkbRR+ub605jGhV7h+9kkqNpsO/+7vj45PXOzs7YOnz56/+xT/9V2HQsdbsPX+2tTEwqi6LGWM0cJkMnCaxK0lzJTQiLKTuqJoFfuA6fhy3w1bc6XQIIWVZvT4+oZSWZVWW0vOIiwmjLmBiQTVVKxcZHAasfSNxVlXC21L+iqhaSuQrm3hjjF3ZJTcijFzUcFoaCRrDchPiai/TdCKEGsjS/GqJIZbfLgMVl2NYxDBcJt1bjnMVHHytBF6es8C4q5vLy5d786EZDLpsXUBvsS+vTt2q+llBCWi5P2zGYBe2FvP2fa3abPRFseNmbpdYahWO2N/PoP32PX7Nkf+mrv7BmW/OaVCCMUZKadHCE7EKxeDyU9rEMsNKxefmUanrOggCpVRRFIwRzrmUcjgcPn369MWLF2enx82qMcZEVWdZhi6ycuAyVlhd91UFvMCs1hprtNYuZxQTIcT+/v7B61d3b92+/969fqdbVZXrOJxzbSRqKk0QjAxGSFvcxJ0oixG2GCwBpFc4T98925fftRUDA1rk+y7+sxasBYMYY1laNLnik8mk1xt877t/cHR4YnxTlqWxlrs+Yi6yZpZl0/msFcXrO1uUky+++GI0Gt28eRMhe3R05Ls75+fnw7PzMI4453Vd10Xpckcptb25+b3vfHeeJr/5zW8wxq7rdmxrWhdpOncYD8IgjkPO6TyZzufT8fDM5Y4svSKZnxwdBH7U6/VanU4+mQa9tXyS//KTz37x61+PJ8napltU9eMnz7Ii2dhcC31va2tna32HcMd1aJ7V16/f+uEPRdzykmzqpLwUeS1Ut7de1kYq0els3rvz3kH7OMuhqoRGFjBjlCHCtMGOG8ZxS9b1eHjCuetiQ20ty0KUqcfo5qBLzk4JQnVZnZ2dWeohJ9AKXOa73PVanbw6EtogIMYITqnFqChq3/OyrLBSxWF0OJo8ffyM3EGtVmc2mq5vrb/33l0tZSuMKSLnJ2ej4Yh7nHt+GIq6lloqh7KWGzSWAGOMtiYvKsUIuK4hvK6NpZR4njGqqI1GNYAxmtZSOI6DgDHMOFEMYauVrCvEvWX20+pbQ+M4ruuKc4Yx7nT6RVEM1npVXUwmSspaShKGoe+78/n87OwsTecEcaOZqGRdac4pBhB1fXZyroQE2y8wUIaVEgjB2elESul5a4N2lzHWpC00gQhJkpyfnzPHJYT4vs+4izFukhiFLFdftiY500gTRZHjMIrJyenRcDis67Lf76/1e4TU7XYr8F2tVVlkGKHrN67fvn17MBh8/vnnX/zmc0LInVt3EUJK6DKvgJqyquqzMk3TxjIhpRSiIoTked7Ug0iz+WCt1263f/Ob34jaeJ7fakcGrFKm3WnVdT2dZVmWOF7UGB7CyEfI1qJgnAK2S7Yca5udIiKAsjIDAOYza206H+dZWWR5VYkH99aVkKIqvvzNF77nOJzGrXB0Ptze2Xj/3oMP3r97/dputx3O5qPTo8OyKrRrbt29NR6fGVP9yZ/+kyJPnzx/Rix/9er1+prCAIDsB+/fffLkIWews7tWl5mVgnCgmCEwVmvGGHe9WCjXcwExnRezaTKfPSyqaj6fG+JaA8YAxhRh7roGEyPKGmMqjVYGjAatloQBeLm5Ryu28a8R7vYryAbMhXG86WdpFl7qzred+qvxbk0nzU6lkYmrjuoGjy77XH0B7EVWwvKps295KJahA8t7bAIb33RyMWx72aLwRnhftOUdrZ627HP55yrCeHtzvKr7V3+y+AotcyyvLgRCqIntsNYuK2Y1XzVA4e3zv2ohrwDElVG9feQf094GZ8u1sCtFn82CT2VhLF0Y9hFZuhuWMRlwGfZdAWRXlmBpbGhcA2dnZ+PxOMuyJsyoEWKaNDNmloauVWW8XI6rTwJGUkowFpCpqgpZYwiVtSYItBKPHj1KkuT6zm6SzJoHWKqyGTMGijBYhJDBCIFBCllijEXEIkDIXng6vt5b9FbYBFrYIJpfXYAqbKGJJ10U00IIk+9//w8OD49++ejXeV4Zo9pR7Huuy6isytlkNMvTKA66g/73vv+dxw8fpfOZw9j9O3dbYef06Pzxo+f9fn9ze4sQwpjHOS9ykWf100fPs7I4OjpDCKVpmmWZ0wsQwYDBcRilWFalpLgq88dPvoy9gCAynU6TWdrp9L797W8/ePAAO/zk4PVP/uvP/+Nf/dej47N2r9vudrzAPT7ZPz09RdieKH18fPrBe9XW1rX1/nqRzba6m/fv3VZWMYdSz5FS1FXVbq17Xrss61lSTpOyrFRRiyzNjUOtAUwRxYxRyinHQCginVZXlakQFQIiK1MmlRG25bd1JPqD9e/+4M8ePn2RJtXOrWujs7mUdn0w6K6tz6Z5UuQOoaLICaMWLBjAgEVdYWPisNVpdVut1s7OTjuMOlF89/YdbOGzzz4TQkRB2G13PH4nr8pkVtfVyXw6mk5SojHj/jwp5kmuLamBJsoooH7kG+JOjyeN3QtTBhghvGAUTY4PMWKV0IgoA8T1Q8ZYXcvKJsYYafTSNqyM1lrT6XRS1zXn3HEc1+MNOXmep1LWaTqnFHe73Sa8wFrLOe+0277vX7wVtiwLLLHjOLu714PAOz4+tmCEqFzXLQtTFHUcswaee55nrR2Px2UtXNft9/uASeNUbhLYGqcIdxbGQEY5GNQEwDfyaDqdKiHTLMUYh2G4tbW1u70l5dRxHCllliUYw9raYGtrI47DV69ePn36OMuK+/fvh2H46tWrLMt6vR4PiRB1w46CEPI8z3FY85lxkmWZ67pB4HFOtZZCVI7rra313n///bIWWVq2u/3z88nDR49f7h1Qh1e1NVb6vmNBZlnCGGOErrIFN9NNADU7HkopQqQuZZ7n08mkruXzp4nruhSTuN3yHNaOQwOwtbn2gx98f2drI/Cd6eT8s09/+uzJk2Q28X0XYufa9W1RZbdv3/jggw+++OKL8Xj63u370+m0HfaiKIrDgBHse87O7ubm5tr09FxJwSihGBjBhCJKKWNkQamLDCHEGCGEEFJSSksDtajBUsfh2oJBYAALLRzKkCHGKH1hMcEWN/7RpWKzF7l8bxvSl20llGFlimCpWi4C0RGijC1NBfaiXihcxCIstUijCZp94RWXRGNUaFQI5xxdzi9Y6PjL0nypQjC+xN/wtiliEU/+Lqv4am+rIGMJFJpnHl2ENK66VK7sROFqqOAlLHIFWCxGgpZOllVX0cWjeJHm16zD0g3ZfLhy6a9pV/DQcqKa2LrVHv7RcGEVel6agcuVphvraJZlrtELLc5deAvELFd8dT7RhRtrCRGWss73fWv1+fn58+fPD/ZfTadTYwznfOm+JYQghGsplssHK+Eyq6Bk9aZwE6aACWVYKVVZYxlHVjOCGaVNmOSD9z9oQijyPEd0kTKhrbUWI0MwtgYBAQrIGoww0IVjAjUn8q+ZUnvZhgcAKyxMsEyVRAgZbXzfN9oopeI4tgZ2d69/61vf+dWTz8tKSFFRygEAYx9TBpjWVbH/+nB7a+0Pf/RHt27c/PUnv/jNp7+bjEbX1X0t8WyUIuAODzEljuOCJTeu3x2Px3svDxElgdvCGHMaGDU8Gh1aZSoErcBXSiEpoyiKorDK0nkyLZJsNku0kJziyfD86PX+UZ49fvz0J3/z8ydP9msJfhAdHB1uba2vb60fH+6XVcoIvXnz9tr6oN9p13Up9Ww8VtpAJaTjuAN/0/HcF8/3jPXW1m+ORqPHT148f7FnlJ7NJ5RSxwJB2EqJEGYOlWWVlBIjRDHISlHLOnHcCrtaUi2p73ZsbN2gq8E5HSbDSYqcnpA4DGJkeOC2HeJ5WDJEcmWoMQhMQw3OGLNCFUUZuN76YAMMOjo4vH7tWuAGv/j5z371y0/CMEym6Q9/+MPN9VuHp4fD4ZkQs7rMpMDGQpLVIilnufCCWFM2rcYFVKGvhbF+wIUAVFopZV1WQiiEkAGgPMDUkQqZQnDOuechhLJS1FmBEGrI+NWCqFUDAG0wOCEkCL0m+7wxNzVVnhvjvDGmrHLGyVrYt9JSjDjnjaFP1BXmvNvuRUG8sbEBhhRF9vr167rIVQ0u92ezmRdETUmnuq611ggTQojrusrYBh9AWTY7e2ut57cawB6FLUa4UgZjHLhBnucHB/vz6cwP3MFg4Lq8iYikhDQWrcB32+321taOUmrv5cuXL1+ur61tb1EwtsyLKAitNkWWF0o7jrOsMkUpblQIJoCwnc1mGMNgMCCEzOdzrbXjMM932p2YZIUQottrU0qPjo8ZI4xRrRVC1g9cA1jIknFS13Vd18YqRijGGAEQghzucOoURTGfzKXURVZlaWG1CYJAy7TU+uDg4H/+jW8kWdrtdqWUf/7P/of1Qef4+PCTX/709OSAYiAUxZ3Y5RTHvu/7rkuFEE+fPj07O8uy7He/e7i9cX2jv+26blVVe3t7nNNBv+s4LGOM4YoQxChxORXcoYQwTFyXc9dnjusHgMh8NJk0tFqGEiE1xsjBCBFMOXNd92Kb2GzpwBjTVIEEwI2dSq+ENDZHllUQ3xZVqyrk6g4M3sjZxsCrtLAXbt2lZm3EKLrs42+0YJNn24SwNQ/SqtRe1RNLoPDOthyYuUyE0AwMLzIkl5oMrshf1JiLL4IP8EVxgcVFLzQ3XKg6uKzY4DIm+AfbKqZZuj+WWnYJCFY7vLIXf9ucsJiEr5igt4d6cYOXTlu1i/w+N/I1NwhLlIDedGgvsGme5wZsI3bRZY8DvAtg2QunVQMolwqeUjqbzQCgFcVVVb9+/frhw4enx0cNsGsQ3kKOWWig4/K5Wr3oKsRBl01WxhhEKGNMNY88AEJISul74fn5+dnZWVmWjdSdzWZxx110DqCtxdhaSxBCBmlkMQZskV6o+Wal/tvoKpbNNMGS1pqGJlVr7biuqEWe551uvypLx3E2NzcdxwOAqpaz2SxJIA4Dl7O8yKWojBDJfLre7d+8ce0v/uIvAs//5S9++bvfPVxbW3Mcj2GW56WUsr82QNqen+2Vom7KFFdCCCF835dS37hxg2FSFcX6oB84XORJt9dpd+Ka0eHJ6XQ6tUr3+4ONjXWtxN7L52ey9sLgL/7iLxgLnj5//fTZ8199+uv17fUsnzi+s729rpWq6zJN0/X+mue5YRTkeaENUloHPK41VKV+8uyVMajdGtSVOjk9lHXe6caexxG2HqHGgqilFRhRJSpZV4Jz1+XM5V477DJQjx+9fPjZF6PhaRz62x13Z/f6LHksNKLMf/psvy7NdD7eogFoa5UGY6lFyGoKgCwiGNV1HXC3MuL06NilTEr55MmTOi+6nc7h4eFvf/slISSO4xcvXqytrXHy/vQ8Gw5n1hqXU9cJVFXlWa2kFgYc4koL81KOK+mTpLbGl6DqWktprKGUc+5iQg0CQhihtAkQxARc6mit87KQuiaEAMFNgJ0QQlsDAHS935tMJqHnxn6oKjEYDKbDcTtsY4zzee770aCznmWZS8JW2xFCgCNc15vOkjwv/TBmbgtTxrzOaCajFgDiGDmtIHY5rhy+vr5u3Xg0Gr135z6lZH9//+mL51rrMI7arQ6xpizrsiy1wYRw13U4d0llUaWUlUVlwjBsyjWNTg+Oz06VrNqdkDFmEURx7Hh+Vom1bqCU8oNWt7eOEErSPM/z4WiyubXT8C2Ox+O1tTUXvKIqe73edDqv0gpTQyljlDsepxTnRaYrJaVCBtWZsNoMgt6Lz5/zihwev3Y8NpmNPTfClI2GE6VxEHZcJ6prRajrumSe5ghZQkiap9h1CGecuARhbDEYa7URtc2TohO3QFRlOaeIYWPzouCUEix83xX5BETRi8MffPs7UpvZ+UhV5cHR698+fhi3A0XMPJtuba8fnhytDdYGNwc3dq/NRpPNrZ2tjd3Z8eT85AyJAstiNhnJdJ4Z01/bKOaExrFShefFGIGpcctzfCpnaTI9P2WOP5ocC6AWueO0THNdS7escaULBNh1GeFBGPcJ9QuhpDZWaa21kJXSQulKKWWMNsYgrZvqDk3QS0PVbK2V2hhjtbEKwFisEbUEAAjFiy2gXKUKRgRjXFQlAA6CgFIqtbZSIErQZZKc5bZeCIEQYoyhi0qJC4lHmVUaEAZMlQErlLXAmQsWaWVhwYKAEEK2CWinC/5aeyG1V+MMEHrj6m6OULKSjmjBLvSvadLtLlCLNGZhVVo1JKzqNrtS23r5Q2MMvuDkQ2CgyZ8HQIDAgpKiKanKOacEITBKKo2qN9K+4cA0jd3YXNFVDdmzEHUz/0JIuHCZN1xq6IKcqtGgC7MNfkd6CFwEny7v8YpShMuaslHA78QWq+Grq/0saiWsTM4CqKHFJbTWhBDOqcMoQYBBOwT5nAYOixziMEQpIGSkvFQ3Eq809K4GSCKsjDFFOR8Oh4dH+6PxWV2XGGOtF6k+ixBICwDQ7LfwBVP48i6uQNLljS8xdFUKgjEAVtISBJ4T1JVuWNFOzw63tzeHo2OEZVU4nucSSrXSCCHKOSXUWmukQQgBQlgjhN5kPCotEUKIYIwIwRg1FR00QshajIjBza8AN+UekFUN0QIswnoBwFoE4DjUyJpi6HRbYGpkRZmXd27t/vGf/eD1wTM/aE/G4/l0Nj6p+71u7HuckkoaVdb/8T/8h//T/+H/GPrB/+5/+7//4rNHqUqBwPVbN6UydS0A2OnprKpEWZZSa2PM6DQjFPmh57aC3a0bt7/7fl1VW1trR4evtaymk3MctK0fp0mRYdLa2cWAEJCnB6ez6aON9a1r3/w4DkOtLOPenfu3Dk+O2+12Niun06obX5uN67rWm1v+ydkEETeKg04cCkFEWXU6ncNXL5XSrut+/vf/pRLSgD0+etVdGyDSrbWa5hloJBqAyBnCVtoCMUQ4QlAQzkxdz+ajXuC1WwM/6jiqOnz9GkuYFaNWu+8GXUpJmmVOxKrM8MCV1ji+bxDKstT32mVRcUaMqesy4xjarTixFmn76PETasmtG7cZb//9r36ncLh7+16aZiwIn74aHxz8vRK1UVU7DgI3AuLUhHg+Q4RU6iirSw3EIYRUCYznsR9kRjmeSx2eJFlepFJKz/MIYbKWDW2RiwmqTTGcY0Q9xDUILZUV1lpLLDCMQBqlFA0jP8sTpUVVF5RSQhHCFpBZW18bjs7m87mQ7d1r2029E4TQYCvQ2nTmUVZUnLsWUFnWVZ2srbfTdMoYEWA7nVanFQ3Pzuu6jmLmcOZ53vbu1q2b12/fvvnq1auT87OqzFw/aMcBIaiu5TzLCGGMMUQxIzhJEi3F5vpauxXt7e09fvSltiYMwyjwhFJZljmMYrAuZ0cnx91uN/ZaQso8z4MgaHXazOFCiDTPxtNJVuRenhljhJLT+SyKQsaIRcZiJEQlpdQaGsOGEkJrDUYzQhG2SlfzbDopZRyOk3nme20APJsltbBFVhLCkixFRDPqGGPKshJCcs7jfmi1aUgim20HJYRh1mq1tNKT6Wg+nQWu57rcYTSO47ZD5vNESbt/ePS//l/9bwzYk7Pzv//Fz+vXlbZqd3e31QqOz14XWQaw+eDBR9feu/btb303DkK0ezN0/JODw8Hm9myaH5+N49ZhmZXH5+MwkJa5vV6/zbnnB0IIsEYZm2dZlmVVVQLCyuiqqmqFDFGylqqWGDOPUUM4JczzPIfxRoVYi6zSQihjjFZyGfWGG/mIL/PEXfCWw+X9Maxsbd/YD1bSIy9wAF1EL2rd6DNOMbpsv30DLy7v+FdF/5Uzm2z7K1+hy7kMV9TeVwn6KzeFVvIb4V3tHarora6Wv7ULaij7zvtdVZkrm/h3WB1WZ/hNzwg1Crs5suIDendkyeJa7zIpoLfyRJYHr1z3CjhYhQL/uPb2ejWtMROu5r80ViWE36zmu5HBSrcA0BC0+L6vpaqqajabJUlS5U3hlTcGDGNMs1Bfk5No32Hnf+teVmJEMLI88BFCrutaaxljGAMgY6wyjTcbkAVtAQEChC2ARRgDMoAQvLkRDADIWMBm4Y2wAPitwFSjAaOmxKQFi+ybHq5MNVzQlDWOvB9///svHn35n/7qr7L51OMcc6/dCrA1QlR1XVJGBp0+ouhsMrJKt3ot0oqCMESYASgloa7rWkgpdV3XemFNsdpiVMj5LCeY5Z9+iTGUSYGJ8Vw3DuK6FGDQ9es3RSXTNE3TosjzPCuzJBP69FUy73a7rbhz/fqNdqu/tt6P40gpgTHM51MpJSE4TdPDw8Ph6Kwoij/7wx9du3ZNMF5VVRRFg8HafD6fTCaYMj8MgiDgjKVFWQmBjHUDHwuz+pA0S6mtAWC1FP04+ua3Pv72hw84MmdHh+Oz4a173f3Xx2lWxp11DUzuH56NUoTQ8PTsvffu3719Zzqdvj7Yr+sSA0FAMIDLHWRsVVUEId/zuu1OHMS+7+d5LoQosnwymeR5UVWVMUYyTikGpeZJapVk2FACjucm86SqZa0tYq7vBb5fS22klI7vhmHYrF6aptYgY4zWwvcC13Vd1weAsqzzPK8raYwRpAaCli8RQghT5LoudRzmuhxjzBgJAj+KAs4ppfj8/LTVinq9Tr/fZ4yNx8OyLNfW1n7z2596QRTHMeF2lk0o4UEQIUuKcioFubFzDXSFwQ08V7Xa0+l0NhrOx+Pa5XHottvx+qBdZK35fOh6ntCqlpXn0FYUZkVdFIUStUaqyPM4DG7cuOE4zt6zJ6Pp5NbNawghi6BxObcjv9PttiJfiGpzYzuKIkrp+dno7OwsDMNOp1PX9fHxcV3XSinOeTLPGteGFJpSXNUVIQQzrK3R0mjQRVliAGW0QcZIyRmxVhlQZSWDgBS5mIzmUdjPZsXx6dBaYgzyvEDURjY1KQx43HF8jxBSFaXLeOB61CfGGK2UkQqMrYrMaoNAc4rAKoQQJQiDFkp/7w/+4Mc//mPPC3avX/ubv/nb8XSaF+V0OqlV0WqFjLFBby2Ow9t3bty5c6u709/a2i6z0nHdMi+TXG7u3D47z14+e3k8TupSJEJLLOwkFYjToAUW1xpkLaSqy7Ko6pI7TrvdTvOirEUhNKWgylpUFeeuw11pmO8HcRx7bkAIAWMafr8mOEFpZaQyTVis1QAgyxpWVBe6yDRrKt+hC9v1KkpYxQpNW2r6ZboaunAoYP6G1RutGO3xCnfCUm+ttncMyVqzmjLQRCOudL78Cla4E650u/QXwAqkaLz+79QWX4UPlq6Bhdxemujhqv5e7Wc5quUwCLmUt7k6sas9LKfoynXRRXqhXqnysPqteUvbrcrNd2KUK3cK79LrXwOtvr5dgTXLTpYooUl8aD5gjJn3Jl4BrZh2lkGO6HJRCdSQjxVFw8HcbrejKKqLcpFWrvWirIMxC6/V13pUvgYiXFI/YLXWUktKY62153nGGMdxpASp3rBiN+afKzYYdAV7IQNAFvNjNBCMAIPR0KTxNoUeGiJn3djQmpGgBjK8cV1Ye8HZ2oQHMYwxELIRRz/+9rd/+j/9Va5qbepKVLmDojgwtla2otRt9Vvn89Gjh08wxpqTQXtDSlnkSgqVFzJJMlErcxH+jDAmlCBjjVYE59piXgilxbPfPvV8RonBVPs+31zf+NM/++P/+l//9uHDhxSzVtwOgkgkxeMXr8JefHp8IoSK47jX7Q+H42Q+DsOYEjg7PW3y7KwRDqNRHDRB9NevX8cY12XFGHMY11pHQTAcTwmjDatN8/xc5EzZhm3ZWou0Maip1mAalBO1Wzfv3r5x97YRZdSO3v/Gg/PTF8bieVrO8iOlyehsUpcVsbgVx+/duSul/N2jhy+eP9fScMo44z7TAXVEXVulXcKQBaOU73mh7x+8fn12epqmWbfbjcOooQ9grqPqajgc5em0HUXt2Hc5m+cFIcQSJpUAY8NOq7ZoluWcO5M8byoeQJMcUCtrrTHQ7fRc122SDOq6LsuyKVYsjaSWEkq1NkIKa21DZEwpxZ1Oq8lODIIgCDyEEKU4TfPhcGiMqaoijmNKsVLid7/7omZnhZoRvkEIq0QS+BF33bpSk9l0rTMwtpaizObJea3u3rqNLfzsF7+s69r33TKbcpdhDHmVU1TXZVXWIklLwNT3IlGruigs1xI7nuf86Z/80Xe/+90nT57Mk2mv39nY2HA8bzweCyEIo437GQDyPN+9cWMymRyfnY1ns0pKVFVyPM7zfP/w0Fo7GAwI50cnJ0VRbGxsdPp9o8pS1Bhjj/qMORYhZCTGwhjV2JnLqlRGlaLExBAG7dZgOp0eHBx6bjydpPPJPGr11gcbnPntVpUkydlwZJT2w8D3/KqqkvFUeE4QBA7j2ILR2hqDjDZaMkJbYYC1rooMAYo7nX6/+40Pv/Xee+9/+OCj0Wiitdk/OHr06NHzly8oxbUsEVrnDg7b/v3379y6dWN3d4cEnAIHK1++2H/y+EVdyjCMkdPqbN1kcZuGaLu15jgOIcxynmlcTHNjVF2LsiyN1UHQbnVaXhQeT/bSUlS1dh1Si1IUuZUSjAbuO44Tx3HohVohKVUDFASRyFqjQBojpZS1aKgwiyxfyl9CCCC8BApLibZkQb5wlL9pV7QOWkm4x5eZcJa6wVyYf2EFKLxxKFzWvhdBZ0gIAReEB81o7bssClfgAqyggWWfVyDIapDj2+1tmb761duYoIlCXx5ZvSO0Yr1YAoWlwrsCEa4o1NVzVoe9GlbyFuC4GnixOhvvVIHvnM/V9b0CDv4RWGH1vlbblcQHAGgEPb+Y9lWI8PYqL0fSEHlVVZXM5rPZTGvt+36r1UqSpJl3uwxuBYQQUkb+gwN+e1ou3c7FwaqSDUtsI9w451pLhIwxqvFkEUKUgqaux/L5v4gfWvSGgYDVCBAYYzFGBgEyC+MBmItakSugdoGWl46kZYGoBZZFCDVk0YAwWES1und999sfvn8Q+0rXZV3M5xNOfc0hjBzAGJj98tnjX3/xqecGSVX4Hp4neZrm1qK6kvOkUko5jqctBgACGAxF2Bpjawm0Mv1eR6rq+PVhQs14krseRDEYY54/f/p87/k8mwd+Kzk9xohjTFu9nsOwlDKbTs8OD18xBwA4d1wWZVVtZdUK/E47EkKoqvR7XafdTWbz0fmwCcXDGB8fHx8fH1+/fr2WmjOulNJCMkKZw4uqKrPcpxwsLElfEEKAEQDkea5FPRyP/vbnP/vkk19OhqcYzFp/0I8cA9gS+mr/4Hw4tcBdP+Kchy6LXJ4qpesaaR15PoCppTCgCCBiIArjVhgVaQ7GBtyNgvDl05fz6YwQ1mq1Op2uUsrzvDiOj47me69eTWfjWzeuh624UGZ8Pun1+4R7VSny+Zw7XqWklJJxdzabNSHSTfz+MqC+1+tJKbMsn81ms1lS1zXBjDFGGWOEUbZwuTaSoS4t7XTbnW4bAJRSSsssT621Qtau5xirAUFZFRbMzZs3r13fffToUed6dHRyMk1H1lpCKGLe6dn+06cvbl6/NehGJ8d7WZKPTsaz4fS927ddx8FScmtAVKPjJM3nwlTKyKzId65d63danXaUF3VZKmTVoN9ZX9/cbMV5WRiwv/ni84cPH/b7/fsffJAkyXA4zIqcEOJQmuZZcpY0QOn8bPLy5cssy6Io6nWDRt65TrizfSPLMtcJ60qMhjOMca+7fuvm3bKe2cPDNCuksJgRQilCBCElZKqk4pSUQlJWz7OZBh2ESEo5n9etaWqV9jyvFcXdbnd9ffDevfdfvto/OT0NAu98PKqqilHkdiI38LTWIi+TaqqV4ITGURR47mB9vSyyMssdHLC19vXdaw8ePLi+u7tzffPg4ODnv/zFfJ7WUjHGirJuluP2zTvvf3C7qLN5MkymyfnJaV0W19+767t6c337s08f/u3ffcLdsNsZEOqC18o1Q4jwMCKOY4wprR1m5WxWBkFgCM6R4k7I2t2akekkO5vm01yCsZihplSBlNLaHIhLADFMKKXIgLJKaU0AkaZAoQWrjdVmYV24iI97s1dbVD0Gbd+hGq+I+OWfsKKiGjgPGFNKOefNKWYlkQ8u9tPowlfdSFV0YYRYQo3lDw2gRTTAiouaINwI+iso4Y1EeJfpHuN3+A7evs13tncqy9U7stZi9EavX/ktvKXYlor/CkS4Aj4ufzBLfdnE5WVZ1mhEexljNYbrZf/v1P1wGUhdOe3KV3bFOPH7TNc72+pDtcRMjRVhtfolWjEerAK1rwJty1GVZdnpdMIwTGbzg4ODJ0+elGXpcecdw4B3dHKltyvT0jzrAPDmv4vjhFJloWHqhcuwRq9wezf/N0L83Vc3uunZYoIBAJTFGFsKRl9CCRcFIwCZxrCAAAFCYK+Y2JpQBgDAVhtjdNjt7yj1T/7wD5++WLNWKiv+81//lahzQKbVivKqLqry+fPnZ8NxHFmhIM/L2SybJRnGxGiQUhuLATGrlQZrAFFksQVMqQVsLJrP53Ec9fv9qMWVeUIZBBFZ32xpU5XVTKqCOy0LOM9SazCJ2+NR4nlepx13Oy3f9wkhQgilZBQ5N2/sbm9vB0H44sWLNJlNXJcx9p//82enx8c3b968desW5/x3v/3y008/ffFq79btu47vzdNEI6iEAgCttWwi/QDgIutqIesQDoMAhwGm6PXpcZXndZn6vltjq1XYbnWZ6yKMq6owugIAzt1kdH6w93w8m6bjYb8dDQaDWTI/PDx0GQGAdqt9/977N3euJdO5qOsb126eH59rISM/cIMQI6SUYoxxyoy1xiLEeRB1OoON9vqGlFLgeWUxw6yo1elwYgDXUohaCfPmfaSUhmHouUEYhpy7rVZrMpmkaTocDufzVGuNESWE8Ig3PAWu6wZeaGDhAqatVkAISdN0PB4XReE4DqW02aVFkde+SIaMIo9SGgTO85cPp/NZmqZa6yiKtFZpmk2mZ+3QTzsdgd3ID0XoJePx6fGhFDqdjCaTUSUrTCxhptWN4nbYajmqTjAKPM+JoqjTHfR6W+12L2p1Tl8fPnz48Oef/Kosy9lsduvWrfZgDQBOzodHpycIoX6/TwjRCEsh81poRPOycv2g0+snSXJwdCCE8Dyv2+1WQkptDKC43QmCIIxbBlAYtQkZV+U8yWptACFkwJZlrpWoyiyOfVEb60Ga5EoZyh1jbBTQzY21nZ0tKS0hlDNqtcLEjIZHWZr0B33u4KcvnpdVtdnZNHNQxtZal0VW5QXBoOpKBp5L0Xw6JQhtbW7ev3vnW9/61t07txhjn3z++a9+9avffvGllOrV/qE1aDgcamXDKLh58+aNG7fKKhmN3VYrdLl3dnw2zcvt7d3777vzWTGaZo5HgKmtnf60nMhKK6twZQAKZCwggxCiioN1GWMkCrBDS8LPJrOTk1NjqHVbPmd+4DHuUcc1UkkphTGyFmVegEFGY1VLKbW11uo3OhgjxChtCmm73FlVrvpivw1vJba9LUbfaLtmo4bfKH7CWCMlrVFLy8RqAJ25TMK4PKjMJVSxGLBdCPfGhrYMRkMIgbnqd1haCK7o2otwerJ6U78PRFjtZ9VufAkqvctov3ra0i/wNoK5cq2rIGylrUKKxpBQlmWapvP5fNk5esNa8cZlc+U2EUJXoNvKsL/OGXEF0PyDU/d2W529JUpYpm8AQLPEDe33O3mUV0eyegtLHdw8Wk3GOABUVWWkWtR9uMjueTP838/18PYcXhoJWvTUeBwAgDFmraWUGquVkkpJjPGF98wALAi2EboafKOXb4PWFmNDMCHM2iZDAyNkFxBhYWBA1jZBkQZQ45JYeBysBYQpoCam1gKgpgQuoeC53ve++wfWmleHz6s0lXU9T6au53luUOQlAj6fl6Cx1qAVZGVRCaGUwtgaiwAjZJAGq6wxxmhjLMIEEDWoWcGkmLsRaQ+iwVqclseU614/7PZ8xyW9rpvMzzDkG4O+agenJ6Nsfr7e21qQbFLU6/X6/Z4Q4uzsLIpia20UtrKsUKLOkzR3g3a7vbu9vb+///Of/1wJ2el0XNcz1vY73bqqonarFcVZVc6SjGhFMaZBYJpc94ttTxOQjBCaTCaEIAwakHI5D3ptx2GpFqfT8dl4hBDhPrt+e1eWEiHiMH97awt0cbr/MplOwiheX+v4HjO6PpqcGqmAOtQij3GvPyizXJTVwevXSshWFIdxazabzaZzx3N935dadXrd++9/KITodttSo0Ia7PjAuGWM+iHm86oShDk+d9FlarWG6dxxHErZeDyeTqdJ0tSaMsaYJr9skk4ZY40VLWqFnucxzg3XNMsTzvk8mQ5HZ1VVNRzDhJAmvm88Gd68efP73/++EOInP/nJ559/Ptavu90uYwQAceYQyoIguH37JkfMWNnrr0dubIRCeuv4+PDGtZu6qvL5rJKy0/X63fb2ja2tnQ3Hd5wg5K43nqXjSYrBFvl8f2//5Ox8c+tGqUxRq83taz/6oz89G55/8tnng8GABqEXtaSUQHl70I/jOM/zw8NDTFmr0w3DkDA2ns6OTk6tta1WywAyxjiYYMp6gzXXdctaHJ2cgtHn55OskAhTIVRZCSllVRdCVGky3t5a44x7QUtUmdLWAPJdx9nobW2vx1EwGk+NqiujzVwPhyfHJwcIY0Z7QchbkZvnaZ5OphNhrTVSYAuMYkao5zCXM2SN5zg725sfPfjg+u61MHCePn747NkzRfnpyTljjBDucK/T6ZRlPRyeCVkVRfn8+fNur7Wztb17bevGjZ29vb2Xx0d7L/efPNl7tneIqS81FgZP0loCVwQAI6FUWeYYWcaoMcbFXplJzm0QBAbRNKtmqSzB9cM49lqh47gMS7dw60rkebOzzPMcADyv4sRFgJsAO2PAWgvGYlgobNZYJilZClljjAWzpLpbaoJ3bukaf8TiHISMMQTTxgtrL0LGpJTkwhC+tLE3fa6GKawelFIt3Q3LYWB7KSQQmqxFY40xgN+9CX5beS8vcWUYq8N7txq4OHMVUa2e8PZVVoeBLjijvkIxX96wvoUSlje+enyJrpa1lBpWlss62AKARZc03FduZFcCTd7GLvAVmvIfgRVWB7B8Thp5pS8oqJfMWqvzszrPq3OyiskQQr7v13Wd5zlCqNvtdjqdPM9VLd5essb0g74CirwNEa7MgLUWLsJ+AUBr7XLsOE6/30cIcc5FXTmOo0xhQRurrEGAjLEIkMENGeOiHiZBgPAFq4eRarF8CJo9oiWLbEwAYzFF5gI441VUZ621CAFYbK1eiVRAWklrESEIAUFgIK8A6Ob27rXRyXg2nCWzjz/++MWrF8k8QwCiFK2AWV27LNQllEkNXqGMBGy11U3wpUaglDBG64YGAsAioq2SGmqJ17Z62MdhEIRdtzUICK5aHZpkJ2KU3b23ORgEo7PJdLSvJGYI1jZ6uzs3Xdc1Rs1mMyUEwdDvtRiFhqnv9OTo5ORsOh5Zi8BqTlktijzNKCZhu9lbO1LKpnQsJ5RSShAiCHHKLEEAUGqNAaxdPGzmwjcaRhECY0FpKwxGuazTOhdCkMCrqwoDGnR7t2/fWOv3OXGstteu3cCYPn3yO4q1w8FziMMi17t2Oj6thRifnX+W/2p4dLK9sVUV5d7ePkVUS6m1pphMkkkl5N27dwkhlagwo2ErLopimubp8Wkp6iAIXM9jjCHCW92BJ+s4jjHGk8mkUHJZfsgYUxRFWZZSaiV1k8nfvC/WWrBgrVX1gtO9rus0TcMwDOPAdV16dHQUBEGSJM27sZR3s9lsY2OjkbntdvvRo0fPnj0LgiATQRC2jNGEoF5vEATe8Py8rusPP7rv0WBzYz0ZJ+lsHob+5Hz83e9+u8Ock7Pjqs4x1Vk1T6rZweuXtVGz+Xz35h3Gvbwy3I19z+v1sTFmMpuvra0dnZyeng///F/8i+76+quDw0mSKqW44/lRrIyZzhLueAYTaRfV2IwxTQXITqcTRVGv15vNZo7jYIwbNggAIIRwzo8PDrOyCoO43e3VlZwlSVEUWsNoOJlOZLcjfdeP405iVA6pMUYbSSk1Ro0n58dHp6PROIg7hBDu8A8+uBdFURhHQoj+oH16enJ8fLy5tq61rOtaCsEIHvQ613d3NtbWtjc3EZhbN67t7Gzlaba/9/Kzzz79m7/5G0E4pfz+/fePDk9evXo1HI5d193Zuea6fDDonp0eKVkVuat03et1hBB//Md/+tOf//1/+B//X0VtsBNlhbCz7NXJFBDjrhe1YmN0mpWM4gATaw1gOpvOMMZrxHEBZ1llMR9sXa/KlGPkcoaNtJVEWFmEKyE1wWVZGqVUrYIAPNdnhCKKhFBgFsHP2GJkrEEWISQvQgRgRWsaY6y9pLfQ5bYqTK21zUNMLkIIzUX2oBDC5dRc5DXA5Q0lrCjs5cEldl6V6UvXg1lpWiqttRe4cFljNZe4ErS4PKHxjDSt6Xzh7XLdr1EYv49GfPser+i51ftdOe0dgOaKRoQVLbU6mZRS13XjOC7LuiiK1f6bdbP294rqf2tIby79VXjrH92WM38xyMXztup3WAVD6LLrZKHj36W53yDIRUUoXNf1fD6v6zpwvTzPjTF2xa6Dm06+urrC2/PTTCpAU3j9UlNKOY7j+35Dpk4prSvLOS/r6sr9NrepL4pdoZUAVXTBNtb8aYixlllr8QV0xlZbIGiBTpogvaamOHkH/rPWGq2UshZhRBeBEcwDVQO2t2/d8X1+NrrVagf/9i//8mc/+1nghZzOW1FbVrhAJk2KupAGMqWFtXph2sMAxhqjLLIILMaAKSEENUEmUolczGWScr9jMKeOEaIYz9Jf/+bnD96/u7XdY8S83nueZanPQ61slc32Xry8ffv2xsaaNfr4+HB/rwxCL01TrawQ4uxsPDwfAVCHe3maDe359u7m6fGJ67rtdhuMLcsyjuPrN27USset1snovJYiCALX9/OySLKUXTBzNO+DWRQAg9ks5w51XUYYlVJIUVKH+q1IycoPPVHV49l4bdDb3f6o1+mms7TXibU2ceQh1PHDiFGkDMShf+fmrTRNp+PJbDI/Q3h3Y9vlfHw+3N29LqUWVbW0ljHG0jSttGjYQvOizKtyOpkrozu9bsMp1+12Oac+Dxu4OZ/PW61WGIac8wY5KamLokjTPI5adoV+TQgBFiOEOnGn+TNJktls5nlet99ptVr05s0bh4eHAHptrZekM20qh+BalJ1OTCnZ2tra3Ng+OhwqQb7zrR8/fvy4w8tsWnmeRxyHoZYo4NnjMyNVedfZuXHt+HiKte2tbzKE795978nLp95a+/WzL589e4YZDYJAGSQUQZTvXHvv3offeP7yxd7hw+eH51r/emNz8/79+3fw+unp6fvXr0/nydHT5w8++sYfff/7z17ssZhrrdvdfiXqF3sv06Tc3d1dG2xVxawsaqFNVVWTeYIw5U6QZhUmjlQYAKRARlPmRWD5eJSmidzeuu77YVEJUUpdIVUCAR76MUb29Gy4de2BQGguhQ0caaoYQlGI09cjXaIXL17Ms/nHH7dCBzCRHBSqZyYXke94LebZuOva/aOZVuLeN+7cu3P38e++dBj/X/6rfyEqub6+bowtyvrw9dHZ+eiTTz/74rePFfHLOVrfHJweTV3uYaJv3Ol3uuGd+zelKh2Pf/75cDIfD9Z2p3pynJ0kkNcERmlaGTNJ0ih0Iq+lC+locFyHIFoMp0JJ7jnI2GyeME6sHboMOY5TzIfZ3LZaLd9vlWVpDamErivBAGvl1zWShgb9DjJVnudaGuK7DncxxkotWCyNlUJWTfg3bsQdgG647o3GGCOCLbYWWcAATVElhLHUWktjJSCNiTFyUVagMa4uSO4oNcbUda3UItwGU3phMJDGNCYKBmDworCkJYQoaay1jQjTCkmwGGPMFhJNggGtFlZoQoUQCGMLIFe4ijElRuPFXgqaGLGmeGNjx166CRa2WQvQvE5Le+9iZi6sBctd7BuNZQxuKm2Q5R3ZxpqylMaX/l9J94fLW/xVTbn8Lb5IXVtAlqYIB4BpSJyuCH4ARLBFoIwWSgKAtkYoWZRlLYQ2RlvT6FfCKF7U8nhHbMTbR5Z/LgkkrkAEay0AWjlzMedoZUt+Ba69feOrRxazSmnzFDVAsKke2eyQFj+0ejFnsCCqBGtNkzmJUKO3EUKwpN5CVCvteZ71oSGoQYjM5/OGYs4aJYTQWgFAQ0FpzDtiVq60VcTAELs4mQAYA2DAaGNch2dFfq978/4HdzGGPJsHvmtlpYTEgCgmWmttFEJICVkVZavVwowBWCFruEj6AAAOyFgrGi+hRohagpjFpAleUFYDGGQQIIwtRoCUaGJWoKEQs8sVIWgxqwRRRixStVLSSN/gusbT6dRxOnfu3UTk2U9+8pMqGxRJL080RWuHr+dSyqoqqrLgzCS5BWAWqEUX6U5gLWiEECEYIbBaAaKcOQghLWwoy07calEbclvkM+TKmbD+Wqh82D99rXUd9d3JdFbWc6RgPk0qqU9Ojq7tbE8mk2Q+bbfbUtaUUs8LACFKqR+HRaXAZf6gHcXtre01L+C/+fxTaYo/+NEPLMDvHj189OrLk7PTm7fuII4tgaLIuOuXuQjdOKulMSaI3VrkyirmOllVONg1hGAWSGFMWQcejxysVIHzJJO1IJRi4rmulOL45CBLZ1rIusqePn1KsL1xY0MqU+QjrfU0mQcG9/v9HnNnftTrDiiis6pa29qdlfX27i6hdDQZKwaM8+cHz1zfY9abz+cA4GCe1hkF5Ls+0XY+SxKlJuejOI7b7bYoBGMs8GNDM0Rwu93O87woCquskYYYINpqaVGloTSkMkwiYzS2oAlXyiLCO+0Q4YUAq2tNMWKtuEsIraoyDNoYg7VWCNHtxIQgaw0gg5AmFILQHax1Xa2n02lZltl8AkAYJlbZ9cHGy6cvxyeT29dv+NyRpTg8PUMWnx2ffPyt73zjGx9Lqf0o/OCDDx49eXx2fj7P0ocPH3PPL6sq8FulqJUsEWJao/F0rJQq6uqb3/oYU/L8+fM8zzc2NhrG6TiOuajjMNLWjCfD0Wi0s73uOg7GWNTWaqONMUYx5lhru92uUXaohoQQx2FFUY7HYzcMs7oU1taVnI4naZpLWRurKCa+63EGoE0ynaWzOWPIZa4RSmkByPQHXcZJnmc3r19zfO/LL3+bZZm1tt1ut1oRQkhUFSP42ua2VmprY9tzfM8NRF1/+unnk8mkKgVjjrJGSJ3mxflo6LjB5u712TA1RhGCsnz2zW9/9PE3PwhCZ+vahrXmpz//O9+LNrZ2j46OpJQU+Y+//I3j7mhFN9Z3KZ7V0ghRAVKEOhZVCAOhhgBYq1WthZJacyDyTfA/wQ2ddpakhBBkLMOEMEIx4ZgYz2BAZTZpVFEYx77vA4AAYaXVWpuLDEbUcBxpY4yxFF2R701r9tmrnvVG8zkOQwg1DD9wEVndRHo3cVTN7hBZSwgghDglS9W47A0Aaa3xynGE3oSwwZKoEVCDjjVSCwK+FYLhZlO4St1oL8ihrX1T06s5E+E3Nmq04hFYBQ2remJ1X/tO/bGqzOxlA/jqCJf9vFMDrZ68et2vObMhBWoci839NvlgX98/fLUW/P9jW94sukjyXI0BfGdCyuoirh5ZLkHzQEopKaVbW1v3799HCB282ltYksAsz7moyPzfYVrQIjwCNYXx4HJIynK0qw+DXWmrU0EpNhojgxBCFqAJZcAYW6ttE5qAtQWCjUVgDbIUNQctQJMJixauB20tXBCGWrBwYYgDhZAlBJVlPhqd//SnP/3JT35ydHS0TDlp4IXW2hpMMF8d6du3DBf+o8aTiBBCwIwCazCytN3upcXQc1Sn1et31h5lj2Uh1/pb3AbTk9nwdKwlRHGrSLM0zYwxrbjT63WzLKuqKggCqUylCq2sUsqU5XyWKG3y9cHte3d7a4OnTx+/2n9NOGu1Wn4YJnnm+F5InbwsonbLcVgch0IIl3GhgSBA1lhkGEHIYb7nV0XtMKJBg9ZGa2FqKXKCrcdiQhlGSAhzfD6pKxN4HrJw48aN4SxnjPNKj6az6XQehGEYttNsmheFHwZeGCVJ9tuHXwIirutrQJg0ZTwWOxDCOOdcZLLhpQ2CYI3Rpq6jRZBlmZSyKUdSlmUYhmEYur4HGDCgLMu01gSQMDoIPGOMUFJqLYxWoDVBGFGKMSE0EaXWiiLKqcM9jxDkOA53Gf388y8JIXVdai39wAVAZVlIobM8sQYRQnq9nus6Qhac48GgE2vPp9F4Ni1Z6ZHQWru1fv3B+x+sr63t772KgnYynU2G588eP23HnTwv//Iv/+M/+fM/973o1d5rSpzffvGQOVwZA5Y9evj8fDi0CNzAN8YcHw1Hw/lGa50xZzgc/tEf/3GWZQZMq9UyAEmSYIyVFnmWlGWJMQbuuNwRVQ3GWmuUkGA1wYQzFoZ+XUlGaClKSjFjnlKqqopWK+qsbYzHY2E0IlApUYmSMgzSIIyIsRgTUyvLsIc4AWSNJpzqymgwnU57MOjO5/MoCgDgmx89aBiowtAHgNPT09Pjw+FwWKSsqqpsMnO4W5Z1v9M9OjzLy9pa0LZK87KoSmFsLZTv+04QtrTgPOp0WsZu//n/8CdR7J+cnxwenA4Gvboyd+58+ODBg08++WRvb98opiX9v/yf/68IIUqp6wRegJSxRVWW9dzjIWBAxFKLKMUCgBpEMV9Q7CmNGSOYoEYgWEsRtmgRLo60EVVtlAYALQRCyHEczjmixCptrdV2YQYw1i5FlwKrjAagV17+Row2QWFwEUywFNla6GUA2rLyglLKdV0D1l5YVsEYYwBjrPWbUj36DdUxUUpRghchZsYA6GVM8jJIwiDcbPeFNkEQoBXKmaX8XRW1S3eDtYYQYu2biIQmcvNtqIFWvHXL48ve7OV2RT0sb3bZ1Vehjd9H2axeCP0ezo5mZt5Jn/D2mVc+fNXx3+e6bw/79zlhVa+8/SdcJEMu8cGbpUGLgS3b10STNN02j7rvuTs7O03ph9HZaZqmAIDRAhoaYxoGzP9WnGAvOT4WY2ts2pzzfr/vOA5CSGujlEJGL58Ne7m22ZXnaolHFQJjtLXaGA2YNCGDYLBWygJYTJBBiFgLuDFeNUEWeMGjYOwiOkEbi5CxBC0CGqw2VmujVCqm1iKDVCWLl/svP/visy8fPZzNZnHc1rrxqmujwVowBhlDrJVXUOyVP5fGsGZiq1K7LqjalpmM/c50fI4UVInqR5t1qqfnyVpnnUIkRVYVQDEr0qIoKgyorPKqqrJ5whgjnFHCa1U3xe0ItRZhqU1Z1j//5Fd//6tfTyYj5jr/+l//y3/xL//58fHxv/vLvzQIyrJUXEsjMYZalJhaWQqCHZDSKAtIU4ysEUWWi6pGBnGMMADBxuWEEMapw4jV2rcEY0IJBsC41JoY6zjOz371q7Ko1ja3fGtZEMSYlXX9dG9/t9d//fo1Yc76+iZxnCACoSDJiyAMMaXGGIsRxtRA4xHjpc7LsmzkEnOdBlnWUqyvrydJMk/T5nWu6xpjrK0hDDNMJsnIdX2lVFVV3W43yYtZmhhtpdI1SIm1RRZjSwgYW2OCGEeOSwKfUoczxhhj1GM+55xhaq3hDh6Nzs/PzxyXYTCNfVjLUtZlkc2l0GAkQ24n6vs8Yg51HGc4HFZVxYgTBZFD+XQym43Gk9Hs7OT8yy++dF33+OwMAG7fvvuLX/16b2//5PR0fXPDIMw4N0AJcYSUx0cjKeW9++1edzOZz3d2Wjei61LV83TWbne1VuPZFCzClMiqLIrCGsUdL/Bdzkg6HRNClNFVkSNjKQVkNAHkuCzLk9k04Zz7oXd2dsYc+u1vfztVNqtLhJDHvbQqEEWB58i6TuZTUSlrhKxqpx15vfWiyLIsBQ9ZZJSRiOO405FggDSvJe70O4HrOY4DYCkleZ5lWerjoHJ45PpS6/X++vbu9XmaYubPs3w2nR+dnUttwlYMhE/SNHlR/tkffez7/t7eXhj5Byev8+c5QqjX6yVJ8dGDb/X7fVmbKOycnnzyN//l77TWW2vX0jQdz8a1mHiBG8Sh5xILAmOhpalriwnnPOSMS8c43FN10Yghq7TWpia17/uRH1hrlZRIGymFEqLKCtXkZRHFOXcwsgi01hasxQhj3Fh0GisCAFgAhDGhVL+1S2s+NBujRrWvRgYIIRoWrDBuAUATWdIIRwTQ6NbVrsybssjQbPcJIYRgpRRGb0gGjVFL28AShVhCm2+1UotyBhcxXwsBjbE0iyA+colmgFyVaysUPasSH11YFMzlikrLkVxS4Csm6EYbLWs2ootN8LK64ztl69eon2Z67eX95TsauhRKsizG9lU9vxMcrB78Khjx/327cgt2xYqD3tphryK/VYgG9tIIr4z/ypjtIkl4wcoQBMH29vZoNNp/+aLZLhst0eUn4R8RfLH6GCCEACxCSCnlh+7GxobruoQQUeu6rkGrJb9Ts2WHC5/LpWdsBShIVVlrrUEGLGqcUdYiKRElzSOGGqMgYYYSjDEmDVNCk/WAkNXWWoMwACCwgAiyAFYprYSspZJlNldKaWWTJDk6Ojg5OanrmjEnSTIEzSUoRkhpI5ryPdQun8nVSVh9ZZZ3QQghhIdBx3V8K03sdzkK6ip9+XD/1vadtr9271sPdrduJOPsqfPUFA+n07msVTrPkvFUSltWkPpVp+N7YTCaTIpKlJVQgCh3EKGYMAtUg+KM8zAaDs9/8nd/191c29xc3762++XTx1mR6hy1WlFd10VZaq0tyHa7XZ1O8ry2VoWtwOU0tXWZV9ub277rq7pStdFKK1nKuqgRIOxoLY1VlBLPIYVySut7yt28vZGmaZJMz56crQ02f/iDPwyC6PPPf/v086elMf0oqrU5Px9yJ+h2+4gypXQlhLUWLAKEjdJaWaPBcRytdZOt4IVBGIaU0qIqHcdptVpeEDQ4crHDIdhqhS3kacapA8bWokSUGKOkVgghYIhgSok1xiBMEcYx8Tnnnue5rospwRgIMhYU5YxhhBzGXdcBpERVa6VCv+05LsbYdd3QDygmSsiyqMqyrArgnCGApr74fvb69PS0zovj1wcO4xDoQa+/tbZGrD0/OQ2CwHGcL7/88qOPPrp+/foXX3zR6XSa9Js4blOHawN+EHtBXFUVwTyZ58cHr8I4+MEPflDVNQBo0EKLKIoaJVEL6VCys7nhOE6el6dHx7rOPW9BXUABMAJZi7ouJ5OJBjSbzRhjZV0kSdpqtYIg0ID8KGzKbCjQeZYGridF5Y1pnjlgFXMdoMzzI6AMcyevUsuYACi17jLOfY87DiNkOh37hBVCDqcTh/M4jt/74IMgjo+enlLauXbthrHEC2LG/b2Do/EkYdyd50WWV4iSMAxbrZYxChEsTdXqrmUP53/0Jz9WUu7v7zdlQE9PjzudzsuXz8/PTzHGZ2evHcfZ2dnZ2Ow6LlI6m6UVBgCttDVKaEZRXlTzpKA8cHjIKEEGkNVVURZlQSlVUtZCOI5D19bCMCyLwmoD1lKMEaHWcRghBGGFasdxmONgjBuW40blK/uGlg4hRNBCpdVGrUrqpSDQFzWi7EogurU2ipzGhdTtD/I8f/Xqldba9YIr4h6TRTNGXSgAtLL5NuatsLWm1JlsAM3lNAR0EeSFMFlK+ZWN3aVASIRQQ6az7Bwh1FgXvkYXrsrBK9r9MlR4Y28wF/kdb/eD3orX+6rrrqrMps+lvnzn+UKKBq5JqYQQRVEURdG4gb6mLdcF3oUS/rtDBHhXiMPFily1Wq+u4FKnNqjLGNNQOK/2duVBvXLFJmqseWDKsvQ8r9frXbt2bT6fp2laFnJ1OTDG+ish1lfe1wXcWRxZWhQYY4PBoHmtlFLWKC3qWi3IIqWUVVVhjFfTPq88JA3QadAAXQTEgFbCgAWBL5Aos4yCtRQoYGzQhWtPL7jSLSAwCiFkEQarmy2GEKquKillJVSapnmen50NHz5++vrwMK9KhIixgJBFYBsNpaxRFtSFTc6+C79eeUEQQpzzVtzZ2NjiDrJQcmY3ehtJyo72zv72P/008PwbD+5ure0Wgby2/v53Pvqz+Tz5f/+Xv34i1Pn5KA79zQ2fUooJSYv85HhYVEIj6/iR4zuAmLYEDHA/midJkpXSokrpLC8rpYESPwwRQUaq996/p62qjk6KMkdAfM9gVGIjvZDtbneDKGTYzOdZvx0zwgQG65BBN4wj12HAHfzoyUlZ5aUwlELc8XqDThR7nFPPIywKsK+L43kmxpbXG7vXpL2djvOTk7NaaaHyeVm6hrCyKsoqz0tpgVKqTUNOjwEAyKJ4OrqgkGlEVlVVTf4Ccxzf95sap1prwKguKwQE2YVtzWqDrCaAKG5SVY0xCAihnuv7vud5MkkXH4wuy7IoaosAY0zLMjPGME48nxqjXc/p97vdbjfPU8fxXNen1LUWgSVgiVYwOx/5QVBVldU6DENZVkibPM3S+ezazo6uywSmoe8qJYajk047/PDD+7P5lDvsww/v/+IXP58lJcKwe2O33+9aTBDGd9977wc/+lGn06ulGI1G//4v/0dlzGQ2wxhvbm9SSofDYcOLMJ3OZ9OJlHJzY6vVau3t7U3HI1Xm6+vrhBBKEPVdQrnDqMNZURTdbtfE4XQ6Rxh3Oq1utztLpiju+62IUhq32oTTuswpxWD1YKdntOIET6YjLQVvRU6n7Xney73HNIwQIdOy8ou8EFITyhCO++txFNV1Pc8LUUnmaiDc8aOd68ZxvG6/a4EWlRrORrMiGyeT7Z0bcatd1aIWlSgLxXncCjudDnNUXtanJ+dKqeOTk6IsXc+jFBeF/PLhb+oyn81GYeR1O9QP2Gj0rJARxnSw5vfXolrqopSiFAT5RjlVobJUElZwNsekaOz5otJlXjiOY62tylLUdRgEvudpqazS1liDMbJAMbaEYowpQ004WJNthgEWeyljlDXSaNAGY4wIIo0zGN5EXK++/EtSRXxBo3kBAVxjTBzHQRQ3ptQ3Bn+wTaIyxhhfmJHt0hlxUQB3RXmvskEvsiQE1ktZvLrJXvzkglh6pS0s1faCWqQBCpTSKyGHi8vhq8kXV7Z0cBk0AMCV4ytjBnSBw6741Ff04hsk9A9qoFVB/FU/WWIIdGHN1l9dE/zK+OFdyODK7vxrMM1XDfvrv1od7erg4V2KZ+lMeQMXLF5+hVY8TfhyGufq58aLUZVFnueNaarX6wVB0BTOan679IL9I2wKdsWi0LTmMWCMtVotpZQQQghBMKi6lmZBjdVU28EYc87fOWmLg9giAoQgjBECssC+BrSW1iAgTMOCq9laaxGxaFmWrInLvHgCAYNRxphaaSFEE1QkpcyKcjrNZrPZy73Xz57vjcYzpRFC1nE8pZSUyhjRUHVhahmmSqorS7n6iK4C0CYrrVm4bF5QZvI0j70o8pyySF4934/8KOK/ewjPy0Jd37nd722tDdo3r92hmG2uTwaDntb67OxUa10TqREAILAYYwqIaIusMYiQUmoniHa6varOhKh+9stPgi9/d3S87zi8KQwRhkEYh/J17XjMGDsaP0dY37jWvnn7xs7OVlWJ2eh0OhZpMnJoAAbaQXh959aN65ut2PV8fvtGVpb52fj05PygqLNalDqptJaYk431zR/86A8QsIP9o/F4FPpRGIbf/cEP//qv/3p//8DzfNcLlNLnk7GSNs9zacFxHMdhlFIgGlHqut7R0dAYE4ah67qY0eZhaIxMqyZbs6h4R5p09wZMVEVplK6KkmCgYLVUSgpRVdZaDn4QBS3PGaZZA0zrskznaVYWTW/0w/fvVlWFsLXWZFm2uTaQUtaitMpShLHFZVbUiIpSaWFlpSlCDKHRbGaN1L2ug/GN3Z0g9FqtuNtpdVrxi6fPlKwZsZsbfdejv/3dZ0Lqra21tc2N6zd2Pu70bt6+9b0/+AFg0l/bODg6rIXqdDq+76631h48eLC92z06Ovrkk0+CINjY2FBKHR6+rirxox/96EJ82yD0wygIQj+OY01hd3vT87zRZFaWZVlX1mqGyebGehCF1OFCCNd3r13f6fXXlFJJljUEHcoo13cws9ZqSp2AhpTiIAjgiBVF0ep2LeggCLZACiHm83lSSyer6qpKaoktUIJaWeE5LvUDLdWsqIQQQkPY9ZU0R+cHRSmfvnz96tWp1CiZl9ICIayqyyyZl+lseHzgctZqtaSjfd/H1EvzajieCaFOTo/Go9N5Mnry+ItWKxistW/d3t3eWWu348PDg7T2prN0OslmsyJPqzQzlIbduD9PCmKAIWW1LctaykQZEeqQaE/VglLqEyoplVLmec45F1VtpEIWKCAwVksJTWwjJcpo0AjZRbJiXddlnjcGBoSQJdgipMFapQAAU7LcECwFlr2cT78q3LXWlNKqqqbzZDqdNoxjC5IAdGnn2kAEgi7y3BBZgo+mE7hwbTStkYmSmCaurdGCC2F0IVf1SuD9cmO3CiyWN2KMAVjFBItvCTb4cj6CeVfc2aoOuDIzS0OCvey8WG4UViftii58u62qzFX88VW/avhcmwiMJS75GpPA1yCDt0/7795WZ+NtILI6OW/PM6zgA7iYkys/eXuWlj8xxjSqMc/zZpvenLB8urRS/62oCN5CCRcPl22KWnHOi6Ig1gghGMXGNMkrDUGIbAw/rutKKRuzx5Wu7EXRrwtwaZpSDot3FwgCi6wBY43SYKzFBlFqAYBg1NQCQgstD1Zra6RQdVVlZWN2EkqpNKuKXNWVnc/K8XguhOHcbZxoSqkmmnyxUmABvVmIr5qKZWvogOq6Hp+fJel0Y6M1mZ69d+/6rZt3kDV5nr/ee/3q+SstSDKvnj8+JNgFwDvba+24E3hhv989OTlJktT3/VarTRzOsryUCjMHEFFKG0wwRczx8zwdz2faiHY7xpxXoi7rinJa1HkURa8O9hC2CNvvfOc7ZVmeH/5yezv6xjc+3L62DQAvnu/PZ6M8g9DJsEM4dTl3CLDZOHn98lWWz14+TfqDrkXybDjLRNJba29sD7zAQZR0O2u9zhYhzuS81tLG3mB39/qX1d7dDz6grj8cjs/PRvN5HoXx+vpmUdWNBGuyw4wGBIuarg1pEmNMaNVI2oZLUQghtW4KlDcxH5QzDKjBl3Vdi6rilCJteq22xzgYK0TVZENgbXSaVdpoaaqirqAu60oICdqqShd1RQkFxjFCtixrQGpzax1j/Pr1a4JZq9UBQHUlhajns0wINZ0m/Th2Xdcalc7nlGFjVbsVtdutLE+09j/66AEYlcym6XwSReHp6UnQam112nmRBsHN999/797993r9Ncflv/3dlwdHR//1b//u9cHB+w++8eGHHz74xkfvvx9du3at2+3++3//76uqOjk/XhtsRFHEeT0cDgeDwZ07t4yBbqsNAJzQKPA3djYfPHjAXWd/f//14XGapmVdEUKUhf39/clsVpZVEIXdbpc7jlJqrBobuBZaui6XUkgtPN9RhSQEYZdbRvx23Br05vN5KmVrsC6lrJSRFqQFjZm1oETte15aCouI74eA6tlspqVwPB9gCshqoZjPEQGgdndnV9R2b28/DlsOZYNeJ/BdpFVdlqoueNydzZOtra2nT1++ePFsZ2t9a3vr+rUthNXGejRPz0WdaJPvv3785Flalfna7jfy4nQ8nZc5qmtc5MYYXJYpZQGj1Pet8oZNAAEAAElEQVSQAQlWC1EbKwA5QpC6rht3BqdMCJHnORjbRDhSQJwyhjAYSwAhhBamy4WzceHRl1rzRtoS3JC1WGsbI3/Ag+XmdVUWN8HPzd5oWXASALShnufVdX16PizL0vd9x3GqWi5DEdFCXC3imwhdbMExfuO/MGbhWVjRi4tICA1vNMqyDAQG1Gw0G+YltEw9J0Rbu+AbucgDJIQQ0sRGvHGHw2XNvcQKF6EXl9DDcjbgItZtqZNW52eJEuyKzWO1h98TKyxPXlWQX3VaY5NspHpTVLohP/4H+39n+/8RPoCvVi3vVO3wFSgNLs/hFaAAK0/s8uSlKaLRfEKILMuSJFlap/BFDS1YoMmvq/Tx+7eG5g5jXFUVQyClpKRhJV+MrfE+WGubzILVl2W1n+aGljzmCCGMKEYIM7aoBQUAYEAja6xBBphGCGEAS5p/ABYAG2NAKyWVqOu6LsoqL4q60lpLgRBQShyjbVlUUijsECnlkrqH0osIYtFk1tDV4V1ZvtW1aybWcaEWZZrOd3Z6StS9TuebH3086HW01v/3/9v/Y+/lIWeB64Zx1I7Cbq83eL3/dDqd5kXabKld7kVRhCmhrieMVUWpAYSQpVSIEkK5LGsL2PH8dnv9vXu3trbXTs+OhpMh91yTJ67L9/f3tra22u34n/7TP0/T9HQfcc53rm1rI1/tvd4/eJ4XEAWglGQhG3QGu9s76+sbVZ6eHJ7uvXoesfvdaNvxWZbK8TRP57rTxY7DweLpuHzy+LWs7HSc7GxcJyiejepr128SyifT2ctX+4BJr9+XtTw4OOj1BoudAwJrUfPOWoOiKLLWNs6FKhPW2oZuuSiKJp5lGejaWChd7jQ8ik35eM/zAs/vd9ucc4qJMSZL0tFoNBtPqqoq5wnxYmtBa40s8tzAcwNGK2sR5bc2xTyZzWbCEsSjubYiySx22t0ewwRZGJ2Pz8/PUZOjzCDJz6P29r33b6RFenZ25nhudxAiZGytaEB/9eWn2KdHR6Pcg1TYnVv3P37/Ozdv3pzNZr7vf//P/rCqKt52Hz3/7WC3G8fxP+/8eG9v7+Tk5Pn+r4UdFuJoNpfW2n/2P/vnk+Go2+3Ok+kH33jf4/58Pi/LPPTC4dnZyeFBHMd5luxu7+zuDtqD9ng8qbU4H51MZuPt7e2yqvM8x4S6lOV1Ok7PiAEr1GCwHlqZJdnRyXGv1/O21rHRketQxCUGbe10PGfMo5T+f2j7z2dJsuxOEDvnKleh4+nUpbtEa4gBsBgsdneGGBJL2gjyH6AZ/xkK228kP++YLWdJYgbkzkDMzIKQDXSjuxpdVd1VlZmV8sl4IV1edfjhRviL97K60RhxLe2ZZ4SHi+vXj/yd35mvSg/cOhfFjKHfu30IAJoRRMJYZwT3xkSAOi8XyzJVUSfpV1QuJnOZlJ1Ohyl5dnx+cLD3zW/8IhPJ5WTx3rvvzqaLPC+VUr1eT8URETlnrF8ZM5RSItJrr71mrTYOKw3W+m/84m8iWWQuzxdn5y+LanZ68fR8mS8WRL7vAPJqXugyThCUYInnQncTBC6LqmGGGHAP4EnHiWh0cXFSCaGkEGiMtquyLJVSKkk558g5ERnrGqvTOPEWtXWInhRjjIFjAhVDQd6StZ7IWxuWnVKxdwyAhWyXtc4YZ61zznlP3oOz6CwaTdauMwLWmqooiShRUSSk994byxkj7xkwQCIg5zwicqE4Y14HeQbIwSMhkYc1Cg+YuzIXWIAfMO4AjPOOkHtcGxbgAELyRTsrEJRSXErGmAcoqzqKIsEEORscMsYZITnyQOsOgYjIBYZo7rYd05LzbEeDbygk3OJV3M4vtHT9N8aNg2ykP7ahRb8FfgQAjYAEPlAgIHgk8M46cs6EmBAiMrgCQzhXgvfMEfPAvJWIEVOSK9NY8BufMpyaeQQgWhuL2zYcfBkMcHOpLS8Cbf0FojUHF2wVngAgZzfDAGGbXWdJavfgHAF8CIdwzoJmYgy8t84Za7X3FsATOe/RWp/IBADIE5HzntATco6MOb+GJXLGYGPwAZEkZIwbY8D4mCvtdJOXujbkQEkZRQltQg7IhPdeySsu0e1ZsnbNI7K5hQ2DJ8/AEwFh6OPoNGfAAGIGB6PR0d5+vapc1SglVrrinHswy+W6xbBSKtguRVF1Oj3GiCHzDsPyJwwkzUDkybu2/xkReSKGHLZq7ULdJAEwWzImGHJOfF0HCUAGQ9EcM75erhaX07KpvfdAbFUZKWVelbPlihj3yPKqllLWukySBBivtXHOcs4BFTCGPryhjsgRuCtkBmNEpFSsZMdaJ0Uaq7G36aKai9pY8k8ff3JrPx1H1VFqBqyaLVa/+MZrD0b7j56+/N4PPhodHP3Sr/5a3PeRi1dFQYyez49t3bz79ff3dna+ePx0NptRYwR5wQDJWaylUFmsdKMvLk729of7/fTuAQd3nrLVMPGuLg56/eXFvNvtltNFL5KuXNzZ70dVpKK0Xs4WucuXePxcv/Hmu3lu68qrbn946/Y7X/9aR4k//vcf/eBHH3EGs/Kzi+nLbnc4ubgU0YDmyX78FemJoJm9PE1cHCcq4m4yffrp5+add97dVT1wl7o8T6W7+9pRtzN68fz8iy+eCUJwaK0dDMfee/D21uG+t9owiONEKeWMzcjXRVkXJSLqojLeeSBCDFQqgilE6clY18RJd7m4MNbsd4emybN011rT1OXXv/r+xdn5eJA85Xq1oizLnk10r9tjQpxPJo32B0eHHtijR4/Fg9fvf/+v/zovV9049cZPzy6m5xfz6Xx/tEPOkUddN1rrbrcbx8p6J2VnUebL5ZwJnvW7cRZ7hLqps35vvLsrlOBS7OwfHN65lXY6SZI0M7taFY+fPglY988fPtw/2M2ybHIxJaIkSd966+179+41RnPO8zyfTBaj0ajbHWRxkiRJp5NFUdzrdBmH6SUJyV5/40HTNJeTWRzH4/GYMayqylpzeHRw9+7dly9f5nmZdTv37t1zhJ988smjR198//vfL6viydMvyqrIRnshbVMUhUe/s7crhCiKIuumDMgDBX1DDBExpL5bqzf4h4QIgI7AOQ/ktfPeesV0VZTLvKBy5bxQKh3vHjFUwMVyuby8vEySFBlJyZ0zk8l5VRdlVVlrPVQAEKsoEG9Np1MhxHg8JPBvvfXGcNTd3R0f3dp/7/1bq9XCQ/S9Dz8r8ipK0n5vtH90eIgySXtcxIs8r2ttnSOGnKMQTAjR7XYvT1ZN0zjnIrlmv5dSxjKOoihwVsZx6pzL8xwJut1u0zRyE9Wv61oIIZBxzuu6dmad+gpxxhDqV63oab2VTYbYb0j4/RYaHzeB/W1VytpWkxtz2DnnvEZEFQidtvRvcNIZY7Sl1rxbC2slolY93wh14Favh/Y6WcsFuYFKOOc8uW0VtdaObTHb5vrbfbZPBK8Mup6XgS1owraOxE3++8bPXz3jNd1MQP4qneG9h+tdIbaPE3wR771jhJZaeqsQ3mxL/W64ff8Bo33cf6f9f5oV8tN+Eh5fazxtn7Q1buhG07KfEnJoR5vMorVdwnq93v7+/sXkrNFRXdfWrnPwAWwbWLm2H822XdUetr01zrmxGhkoJQG91ZZz3klTrbWUUggRxVIJzjkv8lVd11EiQkzOOdc0JmAtaYO7tH4dZmhtxytTAODV+6VN7OrVAFj4Olxr+K3167ggbKRBXel5bYOk0gGTvxnbET74WQugjZFs7E4IfCzWe+u90drECkWcdAZp1u8QF6eTy6dPn//1X/2AvOoND4bD8eGt24bgxYsX8kKpTO70egIZR+GN7aUpJ+wkcVkoVYtGWxJcCK6QAefOM2tskmRFUWVp99bRnUaX77zzzrvvfvD/+B/+x/l81usOlIx3d8ed7p2maR4/fvLugzceP3rWG+1pqh89/Jteb3A5me3sHtXVwnvf6/b7veH5yYtPHz6az+3t24P5KnU+NSYi6Az7e3fu3b51+Jql+q+//xfPT55mT0/SNHYesiwjF9+9Y8qmQS61s9PFPEr6WZf1hoPXRfT06fNQBC7jiHN0ztRGh9RD0zTkHG3A127TFQ+ctd45Iu+c9c6Fx6mE9W6+XCTd3sWTL0LRxEF9oOv68mLS6WXdTseRH+8f7B4eLWbzew86y+WSSXH3/j1PGKVZ3ZjOsCdmi+mjLx566x7cu+u1sbrMkuj27SNT1ZKroiguTvPuoPONb3x9PB6fnJwMdnthUXa73bzKl8tlnCZVXTtvFnWRXxa379zykmvAVMiz2Ww32U973dF49+LiYnfv4POHjyeXs7JqTk9Pj+4cvfvuu3t7e1Lxi8lkPp+fn11oDYvF4uzsjAOG+yfCfre3WCzqupGM37p1q2n02fmJ5KKq4qb2oeifS3H//v3BYPDhhx+madzpdF4cHx8fv9jdHf7Kr/zy4a2jp0+fVlU1nU6ttdPpFACSThIlcW/QH41GxmkKrVYC7cfmbfHegL8ZLgsygBCc88ZoR0Yz3ui61o2xvq4XACtjHBADwCJvlsu8qirw6wSn995ag4iRUlxkzjnGhHPU1K4sDIAhkETe+2dFsdKmzrL43v0777333nBw9/BQT6dTAFRxiiiRccGF9YYDIJInS5aInBIyTqIsTapMO6+JQAiGSE1TWWs5oBBSCEHO13UdZG4klRBivpxlaRrYrynAZPimF0DopYToNwIoLNyg824InVethDB1jHF/RVdArRxfFzkIBW39gnVEJFASEeFaryMibPx62CpMCGEGxhhTN0vMYathbiu+aYPj40KtXzZaS0/nnDNNFEVEPjx6IsJNgmD7jlpovfc+RAhelYy0BcHbti3W+Y4t4ELYDjnmL51P2FI/dOXCBk5cf+XXbsEY25mCLSXhnLPWG7OujQyjvdrth/XzKOxX73d740sn5Mbd/V3tEmp9esSQJvBbo72FbVOvjeVcC1Fc32gfxCa9FdiLiXFIs1gpJSXnHJ1bE1aGeIbZNHvcvotWZ29fxuaCrfUGPCjOkJG1VjBCRuQoy9KQ60fyQnIiMqYB5oK2DrGE8NDDIvHeB8Z+2NRMIpKScm0vXofRbN/m9usAcLUIbzzt1twP/zXGlGVZ1pYxFmCeLarGbzVeaZfZZnpfBdWuT9NGg5B5zkEqFBIK6502ZHOleKdkhXGTxer42cu/+sFHt47uoxw0FqWIlovlyYvjbr/fr2VPKgbcag3AXF1V1idSJVGcY2mtd0RMRci4I2hqr7WRMpovzp3zcZyu8kUUpTvj5Ojo7rOn3y0L8+jR48PDg699/b2msUDi6dMzGQ9v3XpzZxf++//+/6udmExzJbtZkh4dHb333nv7+wcPP/3s8nKVV3A5XaHbJZ9WDZtMV5Yw7qQvXp7zCPqDgxfH55OLFRc1kY/jXMr4zTfnD946ODrinU5vNptZw1bLusi14LExjryr63oymSglAPzkIk7SiBACrNUZ44OEFIJzHtSX9+CBPJAlj9YCYl0VxuiL2fTB3Xtx1kl7/fPz0+99+MNultV1ObqcDnZ2Ty4vtNa7u7urplqZynsfK5aXDXBxePeoNvbDj5eiaFbI/d7OeLw3ePL4i4vZpJdm/V7nvMz39vaSQp2cHdeujjoRKjy9PI2GKsuy27dvD4bDZ8+ePT87eXp2XNflwa2j6TSXUh7evVto/fz4Zey7MkkYF3HSefud96R6dHh4+Prb7zjnvnjy6OXp2SIviPDly5MkSYqi6PU6sUpmi+nx8fHHH3+suOh2u6+//joAC8HqptHPTs+apun1BqENzA9/+EPdVOPxeDQa5UUxubh87bXXHjx40O33FouZtfYrX/nK0dHR2dnZF0+fnJy8NMZYnoWk7P7+fr/fr6qqLMv9fl+vmrCAw7oPOBwAIEKAUFuCFOhaiXnySkpERt4SMA+IXHAZiThJRVaWZVnUWltjTGDY9taFaLlzzrs1KV4URQRpvayjKBoOuoPBcDTcT7PBdDq3xkoZTSb5YrFkHI6Pp59/fryY2/fee++1N76SnJwsl7kxpqwK74HzQhvHOdfGmEDE64iIvOGmbvb2dpQSWutYRc7Rcrl01linp9OpEAoJiqJijGVZFscJAFZVFSnV6XSCrQAA4HxohU5urSCdMUHWcM7d9Rwwbpqxto4ObNXz0HW3tRXowX4SQggZIWKQiQS2lVN0TT2woCGuRRToWh1E+FWrG1opFnAPrcp3zjEu1846+Jb53HtvrSVyIfXAGHOOQn2RdY3btJMIgLJwugCz/RkO69YMXKuDaD9sJewNTzT8DVcOG93f/pBz9uq5XhX67QiL0GjX6HVXmDDgpyj1n2Yp/DTt3n78pcr45sFfMa2+1Nj60rO3M/mlVsKNs2wPAPBrOOcV1HFrrWIwpZxz1ummaWaz2Xw+t1Zvgg2+fQ5tImn7OYbx0yJD1tqgO41tOEcukIiKYrUzHvZ6vdVqUeZzBNrdHYOnKIq0bbaUOgCAECJJkvZ2wgrn17ugvTpXrWHaXkxrKISn3L5EcAXmvRaTYKEy06wt7HAZUkqt9Zq25PpYP3EMhkKIfrGr19+v3/oQ/pQKhSQuPCOHZGXEZCx5JFCKZa1/8viLRdGMrb9c5FXlGuPrxtZ2FSXZ4iLf398na/PLOefcN0YplXV7qZJcoLVWW+BMiSgGxjwAkHPWAonJxXy1rMjzly/OGBN1ZYwm73ykMu/x4nw2P8qt1WKY/tov/trp+fT8YspZ6htz+/DOalUw1Luj5uJsMp/MfvLpw25/9GbabZrGYBL3O0kqGhpEMUv6POrCG2+88eSJODkfw5QcGECnfbUsLyszPz45S9MOIDfWN9oUZV2U9WCQ9voDa60jnxelaLgnq63pdruhvXNwSyzotvpGW2OMsc4FP4oxhowBZ876tNNxRJZovLfXGw0Z58+ePatn0yzLnp2eHty5pRFrosvF8nK1Eip25K1rlmVOwLgSr9+/W9SV0LbOBll/1C1NeXL+8mJ65uygqYujO4d3bh9FUvWGnaYx3UHmveUK58vZ2fS8NNVesaedGYz6s3w+XS2GZqfbG+wfHQ7GO/bhI0CuoqTX650/njTu6ZtvvqnSzulk6gHu3LvXGKMb+/Tp0x98/0dVXQTWiA8++KDf7+/s7CHyXm/ACBiTVaVXq9ViuorjeDabXZxe9Hvje3dfH492yfss6z787PMf/c2PsyzLy+KTTz771V/91Vu3brG8mVzML6aXd+/eHY/H4/H4i6dPBoMB5xyiftM0XIp+v9/v95Ez7/3FxYVQHDhjjCFHAPAbowHalt6b98eSp5D2Y+AQiCEgY0pGQiBnRd4Q81Gmsr4wup5Op0VdWaOTLMGSmsYFvkvvwDvQjY2jrGmai4tpXbk4TtOkx3diT/js2TPtrLa4O9wd7hwao4Fn00UzKfLFYhl8i6YxQskoieMUm6aJgSFya9Ex5z0heGsaIrCuJnCAEtABeOtMXWNRrrwDwWQUJQBQFCVHHtCz67atnAfHxbgmIBI8XfMY1s6NUq1oaN0dIgrgpjZa0O6wCcdck56w5WhuxAcXAhhjZE0QM61nwxA451pr2BLTbf61bYrTytbWUGhxfNtS1Vq7NibwSqOH4wD4kOYPNx3sA7/phtCaINtqfvumcCuQAK/oxfCGb5sRN6aiPcK2hXFD+odvt6tFWkEPW82iwnNrxb3fQk222o5eTT38h+IUf4aJcN2Uwe07vTF7P8Nc2J7P7azWtn56dfvGHP60awtevnUmkHY3TTWdTk5OTsoy17r23jLGAP1aBaLn19uPbb8g25fanpdxUDImcuQMokiShIHXdXV4eHBwuNc0ja7Lpq4QKY5kKHDYth2DtlZKhQenFA/WbXhnnVuHo2nLZg2BgaBaAK4tFUTcgn8QQMvxwAAgmMKbCOia66yFeXLOg6zArZLRG9MOAGtD4RWD80poUCBcENZVrqo55qNRdvfOrQf3dnsZu3O0W6+Kzx89ZEp6JovaNIZknDFRz5crxheHvSRLB4xA146xdT4oSZK8LKWUQjBHGEVR1uvxKEHBp2cX09lFpLIXL04mk9mbb75W13Wv1x8Odg8Pbz9/9nIw7DFUDx9+sbu7W1b5Tz5a/Nf/4J/tH4z/3b//H4qSJpPFPu9mcfbtb/3CfL78znf+6vnzl48/f+gdJHG8XJaRrGal7A0i41bWw8lsdfH9T+f1Bxfns5OLZ03TZFmUdZV1GCUm65k8LwWPpZRZ1s3SbhzHQHLQH1VVbYyra71YzLyX3ruynDpH491RJOM0ScFTnue2KBqjjbPOOeOcI4/AiCECC6QLurEIJks6eZ6PRqNHjx53u92Dg4PpdIpMfPbw89HO+O7d+0KIy8uLoTWJYi9PzpxjO6P+bLF69PjTtJP8wi9+U8zyKTFX2XKxwjhTr715f2cwBE+C+OnFcRzHURJpq5+/fJqm6e27Rz/+9CefPvzcWru3t/e1b3z9jdffLOrq5Ox0uci5UGdnZ+fn56dnF2+99dZgMDDGrKp6VdXvvPf+/dffePLkiVDpeHe/3xu+/95Xv/OXf67rZjqdKiUnk8nlZPbs6Yuvf/sXB/3dr33127qq8zyfXs6fPXthGp0kiXc06PWEiF6+OF+tVrs7O2+9+Z7T/MefPC7yxXhnZz6b/ft/9yfj8ZjQK6WePHv29ttvz77y9re+9a3XXnvrK1/5II5jL5O6rv/4T//kyZMnMpZvvv1Wfzg4PT9nAhlnQggmGAR+Ie+BKIQTwjp35C2Bc+QJ0HoidNYb5wkxYZxLLpHV80J7D4FKnSEXQkpOxAO9UhRFUacvhCRC9ITAB4PBdDqdr/KiaMiLKEm7vQ7nPC/1zu6obpplsVwul8iodub44my+WDpn0zRVkUAmpJShewKR5Rw5xxBI9h6ByOnmcjp1jhhjSMS5iBMlpeTAd3d3p5dzweR4PCaiy8tpWZa7u7sHBweRUu3732pu2Pj0bU6BiLYdDtqKq8OmDC+o4VZg0aZ5MW55+eHbNacCrflxW0uZtiTL2pVHQERjDG64FmBL/oXjbLuP21fur3MeEBEyE2J3BBSiqWv55T1jwNacDjxciPcecH3Sbcm77a7hdZ+YtrLv25/7rbI9eMVg+hIFdt3xvaZxr+/fHv8q8wDXyKl+mv99Q5vSf0Tq4VUFeePD1gb60nPh3xZXaA8VbvDGOmxvs500vwUi2T6Rv56mga14e+hoQETGmKapmqax1iIjzhCAex+iC1epIthaZl9qMWyOb4Eh59yT8956z6OIx6rzwQcfvPXWG0kaCUbkjfe2qiwiNU1DW8kC2FownHOGPJgIAevqnG1vcDv3sW0dtjuEF4rz7XVFRNRmXZmQTlFLAGWMCZUyZblmmFj3b9swk/qtnpZXzxE25sJ6nbZMIUjkCRwAcg5SAYGp62rUyY8G/Q9e37t//0Awd3gwnicyidWtB2+k6U6Zk7GNB+ZJ1Npdzpb7gx2ZDtM4cii8M0TWeWO8KXRJZFUkyGOsRNZJ0t5QKVUuqro6jlP+7OnxRz/6ye7urlKq1x39o3/02/t7d//lv/zdPF96B5PLy+l0YUxT5PDjT0/29vb+5E+/X5U+S4dWuw/ee3s+XfzgBz98/Pi5syAEKhWLqHNwa7TKpyCNl01vhKNxFsX0g+89PT5/6q2ochfHqVC8K6JYiSRjvUGka1jmq7KqgaG2hqqmaeyqKLwD54iANdpyKZjgdWmsc/1+nzHWGF3mRbHKnXNRFKk41taiDyWvCAChvy95SFU0m8663e5gMPilb//SH//xH5+8fDkejxnjUso4Suez5YMHQioVJenrb771/LMfvf/Wa8Px7nSZl/nq5dMv5vP5YDgWq3xOzJVN4a0VSvQ62ag/iFVcLJZ1WWmnnTbz1aX3QGzU6/UaXTHwDPz0YrKYzTvdbG+8M+j1z8/P5/O5Bzo+Pu32+7/6935FxQnnvLmFcRzvHhz2u72s0zs8ut3tpIhoG71cLvf29sq8KMrVd77zHQ64mi+/8xd/naZpHMe2sVLKItcIAogET1Sibt+6Xxb24ec/Ojs7S6L03XffXc6X1qC1rtsZMYyePX9a5EbFMo7jOO4AiC8ev5hczF4cv9zb29vZ2Tm4f38wGIxGo8ePHy8Wi7qux0JEUaR1IxnKTcwQ/ZXc35jeV61R1nFgB9Y7bQww3xijEAmh2+9VVVVVpbaNc0bGcsD75LwxztTGW0Li3oFtvLMEYFerE+ecZEpFKSJfLvK8rKqmuX///jvvvi1jeT45+/zhZ5PpRVHlebkgSAmkA2Y9kF+3UHLOAnoC65w1Rhtj19zBHqNYcs5DiZSScZKkjAkyVFVNrztAYoi8rutOp6OErGvdH2ZpkgTIW9uUT0pZVZXbdPJtK8iDe9EaE5v8tw2h3TU7wnUl17o7LYoQNvEJa63zcOUmIr/6+QajQEQhCNQKvk06dq3gW8rebWlIW0SNraBc2w3GsE1l/JrmmYjcOswQgH7BiQxQALjKB2MwmMIBW7vhVZWGW4GBdp8bOnJbDdwwEdqpvqHI252/FMwYDIW1NQZr3Rnu0VprjDPGhe3tPPT2ZQOu/37p+GmK/KelHraNSPiZVsjPtk62j9AGdW5A87atImiTiUS08YnxerDn+vGD/rabudKMQZqmebEUkhG0NFzrgD/batBMW7ktt9UU7fpZsG7qKJJxJBtd53nV6wxfu3/v/fffvXPnTr5aRLEESqSUZbFCpMDgzK+3saBXAmZtZCtkkXAD+oFNII02hZ3bRj8RhXapm6m4utRw/BA2yLKs0k3TNNY2AdVYVVW78uG6jYVb0GbGmKMvp/Nim44waZpGMe/3u0rJvGCvjc29/fjOOOpJXVariA0Odvp3bh0opZqmWayaVe4cWENMyswDnS7q/vmi3+nM56umyoGMs433Ni+Lumm0qbUHqhhbCWJooqiujPfMOzAaPvzwh4vF4ujo4N69k8OD26PRzte/9s3PPvssTgTjMJ8tAf3xcfEv/sX/9NZbb9679/b/7n/7CycnL3//93//D//g333729/mnCspUbE065dFczldxXG8KMu9/SjpRdkAj+6ODg72Hj48Lgtw2kaqI0XPNKzMiQlbVU4bevToi7Ksnz59ao2PpIiiSIpUKZWvymCtESFnUkhmjPEeUPAQlU2yNEmStrKpKIr2WSAibsK/O4MdsMAY66W9X/rWL/TSzj//5/9cVzUAeOvfe/ddIcR8ugiUIbu741E3/c2//+tJ2vmjP/4TsIZ5d/L86Wc//on46JMfMcAkSmOpBGAnSTtptjveaerq1ngsuXjx4oV21nt/enH6yaefFPN8d7wzHo9Pz8/ml9OXz19Mp7OmrNIkEUIgE5PJ5NmzZ3/6p3+eJMk3vvGN1998a2+8k8RpWdXj3Z3Du/cWk4vVcv5n3/mLs+OTJEmqojw7O3/+5PlwOAQA3fjlYuKcyxfLW7fu5KtaioSE6GQDrXW+qsryMoriD97/+unxyaOHjznGd24/uLi4KIqmrm1dmd2d7tHtwzzPG6ufPztBpCRJkMPHH/14Pp/feuPB22+/zTm/d+9ebeqLiwtCsN4TehS8jWSuHQsiCTzEdtdZPMLQyBeRE2Ohz7ohsp64I++h0rVz1pPVtvHWOau10bbRVVHbxprGkWXOITpQKo3jOOJ8tVpNZhfLvHTOySju9ntppxMl6mxy5slyyce7I6FwMOypOD5+sdI6MG0YQBczJYRgnJrGeh9IXxtrDBAGqmwPTAjhvCkrraQN+QTbWGPM7dt3vMGHDx/neb63t69ENJlMgFnOWBzHjLGgSLyxPvSA1mYN5XMuhAraqObG7cageAIhDLyiLbz363gkIm4C5t572kJNB6EGQcv9FE/xxmHDVtAKHMW2i9n+JDhGfDOoJXzcnK41AQEAvA3MEyH/EgyF0DTLuqtCidZQYBv6/VbIvioZf5rFAK8YBPhKzmJ7z9bSurKTvgzMSBvBDxtDodVY29vX7OD/1FUPX7rx6m7tk2oVz88/cAvq/6qhsH2/18ToK5P/6nmttXVdhuY6aRZHy8h7H1ANfkOoIIQAf22Jho0bS3T73qVUjXOI2O12Iadc13Ecv/X2m+PxWEq5XC4kR/DOkw3MOQFsu20VtYwjrWpnW1DK+XwOG1aotrrHOZemKW0Z6O17EUVX7+n2HNAGwBFFUbfbbaypqqos6tbO2H5wrbl/42YREWibaoK1fznnAOHgWa+f7u7uqEgsFtEbB+zerZ17BwMeYbWqnS6kzMC7j/7mQ+MygIzLvoykVEmcQtnUz08nIKJOqi4nZ/n8QggvmG+aMkkj633jfWMdlUVeV1meiyhaLAxjTEq1szO+nMw+//z8jTd2fvKTz62Bu3deQ2QnJ6dJqhD9+flkZ2dknfjs82ff++6HcSKaxnz26cfj8XB3d/f58+dSyk6nczldIGsQOSF3xIejzttfeePgqGPsNOmkw51x1oW6BkKloqFgSVEU1pYoChXxxawum3o6n2vnO/1eJ+srmRlN1nuuIkQU1qDgKDghNNY01lxcXAwGg9uHR0dHR92ss1qtXj5/cXZ2FhrE07oghzEG4BEQF9NZv9NdLBZPH3/x7Itn+7u7SgjOuYzlbDb7yjvvBMi2rpuyLBez+a9/841empxfnD978sXk4my5qjyTt28divrCMsT+bhxjPO4Nbx0cKq5cIc++mD/Xk/5w/I1v/erlFE7OTtM0tUBSmCdfPGtqf7B/+/nz57/zL37n8NY+ZwZZM5/PHn3xVClZVe7P//w73srv/PlH/81/+1/+yq/8GnaieVFAIcjOLqdndZX/z3/5p51EffqvPklUYgo3v8zzqcuyXmFW0+m00+kIFjWV3t89ODs7k1JOp9N79+7VTTOZXWZZZrxdVvl8Pr9za3c0HC/K89rMMeKg7LKap0XS6KZpTJylWutysSKi1WoVRRGLUFMz2h8mvdg4a4xubMMY6/U7DkjrijFSsUw4r72pqoolmbXWWQcAjKFk6Mgb3VS6CW21sm7qnKu9ts5LKSkCkJzLiFXkSl2V1Xy6XC1yp82aLRi4R08KnUIvvW+kj5N4JDDrAKCSceiR/dff+2gwGOzsjFQkyrIoS2NWy14PqnrBOUcmGAEQK1d1UxrJWd2UsZKpzKJeVFWVrkoixzibFwtBBpF7Xde6Yt5wHhltibjVutPpHx7uL5dZmmYMRbfXcwSnF5PpbJGmqRBCa+2sBoDa6DWHAZBHjwIJqNKVUHxV5kqp+/fvj8fj45enT58+BUYeiAmu4ijAOUNek4gkcOusdw4BOGcchQNHzlrjpJSMc2Br2PY67K+wLMtaN5xJKSVj6L01BqSUHtZ5Aeec0Z5zUEoYa4Jqp03PaNyEXgEgpFSccwHkwTmXEoAc0DrpEIQjF8oRomfo0WqHa0AWIgrgtnYGnOGAnHMuBRGRJ2st26pQZ5ssv4dgG4VkcFAqax4Fuj5gHSHY1mdroQ0A1mrGGJEPZDthKog8Xzd/4Y7AeAfWbQwmQIT14YjIb6wJh96hc20ZBTFOjDsuKOjBq4vEtczZ9hqDPUGbNPardsANZXNDib6qj2/o7PY4IRz16rdXDjHRWkc5R5yHq0QA8h4BGCIPgSK3CXIgQLD4EQHQubaKhAEAhUAtgEUOIgLg3oEh4TyzDo1hziJ5jqA4OkQPGNBiAsjSJpvGGXObCqA4yQIm1zlH3iNjnHHOubG21+9zzrXznaybJfGdO3def/31+/fvnx4/63e6gmNZLU2jh/1unufeSkRGDmrTMMa63W4kFZKPlTTGMAZJkgRkt3dGSQRgs9msaZpOp5NlWSANU0pxHjJ14L0l2PQ04egsQ0TkDDE0FODeAYAjQgDnyUspe72uA19VZVWV0JCQkSfMi6rRVkWJVHFRFAyvONDaElOttVID5yyR9qCJb6w6kBY4B2ENQ+KdOI6FSaTp7UdvPej3+n42eayi6M7hfl2aDz/9mzgec1kDi+dLK8F3x2lZ16XJ40yV2rw4e1nrqqoqpUQv7gKjQkNMMUoO4DVqY0xVNJVdSimtY7apL4scwdc1H0SRL+ISnVLqyeePrbdZJzbWEgMEMV01lKyWdql6kgl8/PxzEHwyXyne6Nof7OyspIFUeO+ddrd6o7Is+zv5/+F//xuz6ZTB64xYtzOiuRjL1KNy1gI0zFpDADzJl+nxc+yo8czUMfaGWcK4AMcFgiMQAsqyqPQq6fDaLAnccCer9GJyIRfz1Xy2nEzno9EI0E/zxaIuHPOOeUDgHMhbqzWEFr6xqJnVwoJi/9Of/MFXv/rVztHINnq1nHcG2YsXT/eHgyTJUmt9Vadxcru/84f/8l8bgmE6MKWdz/wv/9oHv/zrvyH6g+7h/sG3vvFtyfjk5HxV5GVeScani3kcp0Gq/te/+ZsPHz/W1hwczOrFzvnF9HRymfVHUaezmpSnlxe9fuyNm+eL2pbdzrgzECrmTe0dmT//zv8cJ/ytN9/znhntZ7PF8+dPl4spF1iW5cnLl+QADZ9PV92EOFcoUCkV2gmGaHaQBp1Op6qqoii01gGQn6Zp0zSPHj06Oqp2dkYvX54URXH79m3O+XRyWVSlECKUjTpv4ygZDoe9Xm8ymTaNYQxkpA4PDwfDYWO11poIAxTBOWf1ugVzi9xpU+Zuq9fRdjCwjfVFaeK0Ie+dEE6EjF2gFomMdlXZGOOISMaRFAnjsna2aZqqaoCw2+32egPOpTFmQF5r/eLkOI5VkiRxlgrBjHc3fOV1v0W3fi2bprHOaK3RrzOIo52xMcZZipIYgDny5I2MImPcYrXU1jamdmSX+dIaV9d1My2dc0rIbrebpmkQ1YgohADXJn2vMu5tCUAcx71eL1+VSZKEBtztCLMUnqa3V8FJR0Rk13fgDOccA3ajLV4AQrgZNfUtWJ2uqZZWzbfz036y7VO2sqy9POfWNWawccWCYbH981aleb+OQLT9phExROlpm11x41ZZd4WZoC1Kx/bytg0F+LLa9/Ys27fm1+xJjtFVMGbbZXRWtzF52gRL2r9beh03mRcNcDVv8B/SFhHa6f1bP/lbv3rV9X91h+2DtEhVtxnh3jl3AezJGCILCEQkIOddeExbBhAQkYOr2gFGfnv1ruEIhG3Qi4giKfyG7hMR0XsKeAJwQSYoAOuu2EE6WRLoSZRMq6oyupJS7u3uTyaTpjF1lTdVURU5Q+KcW6ujuNcGuuI4jtOESxElsfXOkQ/Jf8GZQMmlYIz1ml6IBQaWX+dMiAq01t7PeBbbg3OOPBRIr0+dZVmn0xEr1+l0Op3O+fn52dnZ2ekxIo5Hg8Vi0c7JZiaC/DSBKWGzuMIL5ZSKyZIQLI4jAF+WuZLRYNhflSdxFo0H/cF4R8jk5ZOTly9Pzs8X8/my2++l3bSu/Xy2cuCUilUcpykyxqJEZFkM6AFs3TRNU0WRQADGmFIy3EsUKSXjwhghRNM0QC6KIvLu4uJiOsU333yTM84E4xG3zjlwoX2uAOm809abRnsP5LziCgVzzjW6UpGIk6HV7vT0NC9mzrmjo36327faRUrtDMeXk1VdW2tWzkjvlFIdIuKME5LRbj6fH+zeKou6LGsgDohFWWjtpJTWgzEOPQqhwlPzaEH4sixDsNY5d3l5SeCqqqqqSso1TKpNQoYlJ6UAROuII7zx2muxkr/8rV/4w9//N4Mss3Vly9VPjp+98+C120e3Pjl58cE33z842P/z73zn4ZOnMusOh/2Du/1vfvsb77z3lrh//36/348S9eTx03//h/92ejaRQvQ6/VjG3nvd/PDjH3/yW7/1W6apuZBHB4d62HlwPDk/n6S9/s7BocwibavuTkaoK6hGfme0s0teGY12Xnth3nnrtThydT3t93bv373HmLh75+Di7PTbX//g+bMv/uJP7OxyLikSEDGQxpjFMg/UTFEUhbx1cBDTNM3zvKoqACiKIszUarVK4vj05OStt96KIrlaWYFstVjmVbm/d1gURZ4vm8Z0Op1ur7NcLi8m53KgiqJomiaO48Fg1O+T4pIkeOeQITDmnWs2DQNDAu8qUb2pLmubCGxLOr/J0FvvCIBzybkl5Ay54FJry5mIYy4jJELGmCMoy5rziHOJqI2xeV4GLgBrfBRFuc7zPBeCdTppnEThBUuHKW6SoN57CCgiBGTknPXeI4MsyzpJHFp3nswvm/lcOxtFEWO8qY0nJgUbDfqcCWKgEmXJr1ZFWZfGGGca5xx5GxkZeRlFEQ8kyOCBYQAneM+JgNZrcZ1rqOtaa80YC6QLRLBtJbQK0hmHiEJw7711bZ/ltSbdSH8f2PeAKChmvN4KYftv+0qs8yD8qmvUdiw0bG9wahgKusIltYZgiycPO7SPtRWyiNh6Y8S2DAgMOMer0erZjeN+pfZaMfqltkIrZ1sD9FUh3v7KOUcbLdXiDMJP2utnjIW8zFqJktvesz0X3ERK/u3qZNtEu7Hx0z78+ccN2MH2rd34b/t8WwOIro/tdbhtosEr1gYAEDLvPVkDoULSWmsaaxrOuRSMIUcUZI3BUGPJpBRX8xZAIdYCcO89ZwRr/k23ecreGQue0iRKVKSrstvtB9D3s2cvYsVWy8Xk7MyaJrwLgvOsg3ZDhDoYMAAWRUmadowxiJwInSNr/eYWeafTkVKGu9NaG+ODhd2+XLhV6QAAgAGG0j5xDwhADJEQKUB/pOJJEnW7XWNMZ1bmK+esztL47p1bSvLT09OLi4s4jtt5aCeciMJrwTkHBn7Nic7WbxJzUvIo5sio0ZVzLIrEaG9/9+Bgb38/ipLTi/nDR08+/smjyURH0ShJe8JK4yrrmVCSrG5qqyLunMuSeGfnqNfrGKun88vlcl4UhQNrnSdCIiS/biBujIsTVdVFUxmlJCFwjuPxUErpyBpnHBEBeYIQzGdcgXMQMNjOe0cO/Cbxaju9ZDQYWmvrZpnnuZA4Hu13OwNvAbzb3zt8/OhFXQNnlGWpbjBOIvJIjCzpqqpevjzZ6R+ZhshhnCSMiSJvnLGCqyrPERGRSyYZMQeOMYGIjiwiN8YtFqs2zcQ4IHIhOGNr3BjngjEUQljApm6894PO6HD/YDmbHb98fu9wvyrzycWy1k0ssJxdDF6/+wtfe/v9t+6XZa4innXizrB35/UH41u3927vn09ORdmUNHen553T85PL+WWpq4git5y9dv/1k5enZ2dny+Xy8HC/1+kPh8M4TV8+n+0f3Uo7/ThNeoO+FXR2+bImy5gz6FRXRVnsHS7yZd2UilKjFy+fP8wXy/3Du5GMh4Od8XCUKjkY9BjCD7/34aWbCSW6PUFWFEW1s7MDACFmgIihI0t4Y4N273a7eZ5rrZVSaZqO+lFRFNPppCxW/V5PKYFIiovFclbXteDKGJNlSaKkT1KBzCHEKpZMevT5cjURIo7jKFGNsZxzJsAGGCD64EOsihK3uiQ3TVOWVV3Xw+GwxRC5NWOxdY48EmOMq9iQLnxVVXVZN8b5oqikiIRQiIxxwRizhsqyGQ06WScSUi0Wq7Koy6pRKhZCMMHjJIuTBBE92UYbIXjLg0S09qbbZLTY5ClDfXOnkyVJIoQoyzJgsjjnUgZ0NGlTc96PYyGldJYIPIHlnLTmHGPnPAMMnUeiSHJktPaiYBMgWfumGzfJFUVxenrqva8r7ZxTStW1viE4WkXLGGfIW+XuPQFiFEnOBfKgcf3auBDcmvBOXunUUIrQzkEr5deG9FbC+1WN1bIRtMqViIIgZpsas/B5G0Pa1vSMsZBUoOu4+lBDS5t4wLah0F7GtroKwRXYsnjaudqGg22bF23t2Q2rYrMQrpV1vHrjV9+uoSHhUbrto90YP1vB/2e1EmAL24/XzfF2RW3v3JoIN2wCRCRyV5oJfMgBhQkL57lxzet25t4bY6xpjK611kbXSnKGCpxFRC/W0oCzQI8NjrYDRRxCaRQigmNIwICxdXEEA+hmWRrFeZ5bY+698cbBwdHJyZmzzc5osJwvL+fzWCqlpNWNlyCN9d4DMqmiJM3SrBMnqYpi5ylUaWpjrfPtjQvO1o0A6rosy7ouNz2EtpEN1yYP4AbrNgJ65z2CQPRhooQQcaw6nbTX7cymlwL8oJN2kygSvFwtJ+eTLI4ArgEXwzPi60aWjhg4cOHojKHzJo3jTjdREVeKcR6pCI2tysZ5lJWByWL64nhSNjgcHXV66SrHoqLlMifEbhqpWOWVLfM8zSSRyTqdt95+8Mab9xFpMp0sV/Mf/OAHs8Viucyt8YicHNPaeO/fe+fr/U73xYsXTx5/YWxTl1Wapjs7O51Op6jyqqkr3ThywBlg4JNnCCC5Wqc468pZr33DGDPeZnEiIpak6e7+0IP23iMoJbMkdpeTc2v9dDpNM0iintXce7suYAQQQnqwy0X+6OGTxXwFJMjBppQGjK49WQaCIwIx410b2wrBXLuBkwcjkDPhLDEOADzA2Bljwb91jkkVh95ms9nstbu3/83v/s5X3317r3uwl8a+rt64f+9HH/4ATPkPfvPXHz98+Jc/+ojIv/2VtwZ7+4Pdg8HhQaHtx4++EL/927/98OHDFy9efPKTj+f5ctDvd9KuQO45WbD7hwev37t/cnwMe65YLZ4+psbrOE57/W5/NBrujCqqnp+bsqikIk1GJQo4MGLGNJzzKIqq1XLUHdy/e+fO7dcRxWK6iON0NNwZj3a+9j5NTs4+G31WLorlrLANIeJkvlBKhdBZgO0EGM5qtQqQnCzLAIAxdnh4mKbp/dvjo6Ojf/uH/+6jjz66d+9Ov99P4vjZs2dlkfd6g93d3cv5bDlfmEYLIQaDwUU+b8sd61qvFrm1lrGeqY1XXpAARAc+vNKEEEeJ1rosqkBZioiM8SztSBFatoAnj4EQyJMjr7IYETkyKaN8WWrjylpb7RmPnMembLyHOEqzbhInUeK9EJJzzpgEYFnalTKKophzfnFxwTmXUjEGxoK1IKVMkpSLqy4DbN1kHJA85xz8Oi1SlmVTFiFpkvs6LBpjGkQMGZDVauac2ds76DCs6lobEydCqU5VVRByt2tYsrfWOADvrjQTbBoHbHy4NTXecrkEAGu81nrbvbih29YiHj2BI3JrkY3IuWIMnLehFhwRueBCcgTpAiPpJhjAmdigEa/50Gu7Y8NQ22qL8FV4z2nLg28NAn+dncm3HYqvc+Zc2Tp0ZSus5avz4cevph5oq4FeODjfIrLctma2d7txRgAIPiX8dNW7/VDwFQf66uBsk5vYxCi3ajfC9fwdVPvWqrj2yY1t2qo6uTG2LZvt0d749ifbB78xdW1KZXt470Pij7YoF2HL2rhxCiICcuQdeRu0nhQsjqR3CYcNnzF5YwzQpvQpLB4Izx1c6F4RPGfnwFnng0Bfd68edft1XZtGe2dG/cHt27ezJJ3P51WZe+/rMrcGRBYnWWaaChEDr3Mcx51OZzQa9fv94C2E0SZc2ntJkzgoklCbYIy+0ZY6POfrc+gh4DsQAHhYAN4TQ0e05vnkAuNEeUr7WXxrfwds8+TZ8ycvnh2/eOGdvXv7YJUXRITeOVrnswgYAJL3oV7PB28cPGOEhEkS7+6OdncGWSZ6XTkY9CLlja1rzTVFk2n1+aPHp+dzZ9XO3j2Cjn2+eH58fHGxiLPUM4ycUoqNdwYEutdP7t47uHN3fzjqaFPuiGxnL5vOTvCFr5scAIQQzrOmds4RgFdKdDrpYNjz3jdprZTIsiyIhI1kYIgcOQMAZjkBokcAoDVtfSDZAWNMychNmizLVBwxiXXZcBYLnjKsvAOl4l63f//+7aYCoIT80hMVVVHWNRNonCmKYnleBToKXdWWPIJHcEW+ipIYAYi8BUfOeEuMMeDQsnMGPxbR17X2vpZyzahBsG5AE3D3Ko7AkzXN6Yvnjz77/B/8+n/xa7/89/7mB9/75W9881e++Y1BN33y8POAaGma5o//9E8//Ozz0c7ua2+/0+12d/b3sn4fGn379pH4v/x3/+flcumcYyh+5b/4lXt3H1ycnV1eTAeDkff+/bffe/P1t/7yz/5cCQmeyjJ3aE5nFypODo72+/2MnxKBixPFJWUui6IIPJADXWkw0lbG1m4+XVXLmlNEjjeVQe8Y2I9efrQ7Hr733vvD/vD5k+ef/viz6cWs0UV4/bz34fVAxH6/771/+PAh5zxUcZRlGcexEGJ3d3fY66ZRnC8XcRIdHR6sljki3bt3J89LIWW/3/Xem7opq1xrrU0t+z1rfWjfKZAxgqqorfHASXkFERNyrX6d80Q26WRN0yyXyxDn6fV64/G41+uFlq+wqURqX0VtLSLGMk6SZDAarlaFabwuNRFIHjMUTIhO1ut2+8SwqqpiNXOWnHdSxb1+liYZY8wYNyas61Jr7YiQCRVxISUhRFEUPOBr4LFNfA8RCbwxpmqCP1EnvVQlCUpsGgPWIBdktK0rkgqtJVM3xaqqmkAgX5clF1IppYQkQk/aOo8E3gNDHhjltsj6GCJ6MuExIaIN/cs2ivaGSgsvYagYCPItmB2IBMict57QA7YJZiIOINeBU4sBjYeI4QXedvpbJRQU4rZRckNJbCvOoFPDq98epP0w1H1sq7cb3i1tpQDWPAXXbna94fwVAc72Vd2ov99W5/AK3gI2kQa4rn3DL9rgRBt+JyK5maUw3W2wYXOzV/e7eS4MADxRMBfWYemfDhHYvowbG9v//Wlmzc8ztm2m7QPeUPDh+ts2mK3uhDU9EQJgKKzZJvtijAHQWm9usi2IEKaKvGVIkRLQ7SrOdCez1hrdNE0TwKTONOBD1xfwtIneI2PkCbwDCKhf2gAdBILijEmZxYmpmzhJ337z9bt37+7t7NR1vVzmTVUSEQdkXDIhuVRCKO+9UHGapv1+v9vtdjodoWIPzDiyHhhjwNBZ7zwwhgwZEW2o2BqtNQAote7wsr26WiMVEUOUJUwJAGyIFBljyEJXanIAIDiL4xgARr0OQ2rqcnJxZnWTLxdNVXHOGV7P46whseDBEQGg894SeEQQgkkp9vZ3Dvb2Bv2UMZ1mYjTuCW7mi1JTVtZ8sVh9+vnJdJb3u3tSsca67mA3SVcqMR7o/OJM5vDa67dv371dlfPRaHD73m0u/JOnn15MToDRYNDnwiJvkHkuSAiOHkmhEOKLJ49mlxfW2iSJxuMxAyzLMqT5jbPOE2OCAABZmK0kjpqmaZoqJFg9WSmllGpttXtcFqVHxhg4QibUeHSoZCZEsbt7MBqOj46O7t69++zJGYJSkQCIAqt01k0JKeSzhoNemsZVUxujEXGlwNhScB/wtQiOIxJfI6CiKPHrahQZGt2VZdk0FWOJ945IE5GUMo5UHEeMsYbcxfm5FGxnZ+fk5fEf/fv/3z/9x//s+MmT7373e++8+dadO6//3/6v//eyMuODO7//R3/62ZMXhtjZ5VSdnBzef9Arm9PLx7OiAqnE6elJXTfL5TKO469+9aujneF0PpNJfHx6rIs67WTL1Xw2my0m01Gvj4iNnter3DTlxcmzZX75/MmTus6jrBOanzLAYl55wxcXS9cwYy7v7iWz8xeJ2BsPH4wGhxHPBCpT0fMnx3VZLuaToiiSNLp9+2jUH1xezqyLkiQJFQre+9VqZYzJ8zxEhvM8b12ri4sLKeX3/uwPV6vVw4cP9/cPZ5PLk/OzIi8/+OCDL6ony8XCOyeVunPnVogCKaUwWuf4EZEc1LVubCOEiOKYAtEpj4RkATHnrJnNFvP5fDKZzufzEA3v94eIPNQmbDsxfpM2ttaCg6Q/HO/smdrb2i9whR6TuBtFseAqy7oySoqibOpVUxsppZRR05j5fD6fLUKUcjQabd5XCLwmYWWwyARR5UNUfPPXWrsWUEDhh4Hsk7xzjWaMSQTwYMrSW59I2c9SMrpaWa+1a8o6N6GwPun14ziKk4g8hBYAuO5O20qZNbRwk/XgckMv36rDtgEEEW2olte+eOBXWBdBADEO5JkHb63jXAJjAcC+Ft+whhxyznHTZ6H1gP0GUoprwDvAhugJtjz+FoKwtfOX6OYQbfabQq9wrjZ5jxsoIrJW8W/Ft3+6F74dgbgx/Ctjo8B+lpalLchFaNsRxrZr3orstSXkbOtwuzVFP4etiEWIhjC2JqMAACL0fxfCpS+1Em7Mw895qPYI2yGE7Ql59VB0LcTV3sLVRhuO2kpJXG9N2Z6FHNDadpRSCoRIcOdcVRY1ZwDAkchxK+X6OM5elwPBcLRIDgAYArJ1/xjvPVqzXC6VUoN+9/bRrbfeeJMxfPnyZbHKOedFXmWdJIkTo71ubJZliEjk+BZDYrCHgpAJoLbW2A0bTdm0uzHGpORCCOdcSzSybSVcD/OE9RPSED5kxLcXthBMKTEa9o2u0yg62t+371kl+aefPTw5O1dxypkPxCnbB5dSco7IHKFyYKSUaZrGcbq/v5tlEaC3Vte1LgpMYi8Enp0VTXNyMZk+fz71ILJOLFjCeWQsT5POeIxFtVxVU8VEpxcPhnG3s5NmiXPN6dnzy+nZ5fSU0F9O0+VymecL5ytkyjljLCLKOI4EE95bRBScG2M4sqqq5vN5r9fzQJ688855Tw4Zl5xzIcFYD+idN843QeIF6Rr6eCFnjEvnHKAc74yHgx2l0iTu7O2MrfVFUTDGolg2FUkph8O9o6MIOT+8dRSnEQBVs1UUK0RsmkrrujG1dWUSofE1EQKhkIxz4T1Y451zSdZljIW0u5TSmGYtMcB77611RC6KojiOkzhFxHw1bXSFTt67dXT68vjf/tt/+1/+2q//1v/it/+7/9P/8cMffjTe2X95Pn/77bcKy374kyfzxgsVK6WMhS+evHh5sTi5mJxOpyikaJomjiNrUxXHvWEPBb+czy4vL6plGQnV63Xqoh72urPJLI7jYX9Q1HC4vwtCFk357OTZZDVXkZRSNroEj+jROYq4TFRqCU1VffHwORfRG685gTF52VTOKSsEOzy4c3L85F//m9/97NMfx0pkcSdWajFfSTXqdDp5nne7XcZYnufBXAg3v1qtAKDf74eOq6vVaj6fr1arBw8eeO+fP39ujDk7P0sfpaGJS6fTSdPUWLtarRApTdPLskySRDJJSFVVheSP7ElrLdfcRU4CMBSA5FxjrQUphRCdTgcRhRAh+tftdsuy9C1AbAtP3tnp5stVUCpxFCdJQoRFUaSqU1XVclFobYWKkyRtGjNfLLwvd3d3O52srGcvjk9Wq1Wadvr9/rOXL6SUSRLFccw5U5GQsYzSJBBHhlMirqW5JzJGr1MP5Lz36Nega9eYqswDcAGA6rpGxF7aQeeW00sikjLi5Ku6Mtoyxpw3jEMgXPLeh2AdbpXtBTebmAg5CMYkAGgdKkeIszUiV6m1FNtOGCPiJqJgvCdCYIwRIJF3zoXwLFuXMq5thSDfg6HQ6oNgiGxrhVZ5tAZKq4zb2CyFLtLXuRnajVbahuvUWuMWQzNsOBiE3OiwrXbJ+GXE/rRJIuJ1oANuyjVfNRdoK0R/Q4HduOw28hH4JnArXNF6iq1R5deI0YDH2iStNtcZzD5r3Y1g/M+j2n+GKfPTvv15jgCbCXzVVti+tm2DYLMy1xCQq8QTth3IAAGBGMNQ/0BttH19fCAC2iw/ICL0DkI1E8M6GKxInjHOuVJKcsYYsyYwGpHz2FqZm9n+kq6qhc6Pjo4Q8fj4eDgcjsej8K2UsmkqzmRo6wo1y3o9LrhprPVkPTXGekAum46x1pMjCG3zHMG6KTyRtVYXi9boDFwgQS5xHrcTFWbvan42JbvhMsM2kQtpM08u5AbDbzudzmq16mTJzs5OOHhVa0K2XC7dOosOsEnkAUC322UMuCBCR8zFSdTJummaZllCRFVVOldYawhWw4FKEpV1xlr76bxYFUZIPl9WslwAJfNFvcgL6x0yiBO5uzse7/a48EmcNU11cnrpfO2piSLZ2GqxnM7ny0bXnDMEYY2z1kshQ8A7EKI0jS7LPEs6SZIAQJZl1ruqqXXljXHAWSxZFEXgjECKBGNMco5BkoQprZo67WRpmnHOV6sVEYxGY6VihjyO036/d3by/PHjJ8vlMkmSr33wwWxaDodHzkFeloPR0IMry3Ln7u2iWFVVJRUSYF5URbms6pxz7ggYE5LHLJLeYYOGtPMeQjVfy5+tVBVFUZ7nztsghOPYtrLLOffg3v3z0+PFYlGW5b2j2z/+8Y+/9bWvf/sXfzkvmsvp8tu//Kuvv/760+NzTXxWWl42r722B0x8+MOP8rpxyIpGN8aI/80//V89e/bsww8/fPvtB+Pd3unZ86wjdB0bXd4+OtCgP330yarOL1bnt167VVL1fHY2GAyYZyqNcda4Kt8ZHNgcsIp2kp3HT46dzzq7R09Ov3j//a8+m3y0Z7q3ju49e3FW61La+WQ1lZHynvb29gZu1OmOhOiUVeO1joYdhNg6aDRxkSxXNSJWlbm4OOOcZ1mW5877iIhfXtZpykajEVGa9O88ePvg7Ozs+OQFS9Iold0eXlwWg75I4r6SqeTqxbOXg8Fgd7Q7Ho/92ZPGLAJlZVWX87yQUSyjpD/see+r3KK3nntAH0XRuDuYLleTi5y0PhrsZr1up9OJuFzO5kJKxoAr7o233mnfOCTgUNYFMPJIq6aygJjElES5d+cXJ/3uIFZR7ctyOomk6ve7B4fZdFUbLCxjUc91BugZIBaLVd40xpc+X4koipIk6WS9WIpGqjhWwkcx45rzxjZWW6218zpJIgqMcsBC1xvnjPPOKxwORuS81Y0SUmbKaSNcY5dFJBQielMroTqjkXOuaZpEpbSsy3oaJ2nCZePJOOeBWesRmaPQeNCQt4H8y1motAZgjGSVa85dFEVKKiAATwxQcsZRSs7WrXIrzZiSkiFa6z15Qgx9+CRDHoq5hZDBK7KNMU5v9ASG6njvfV3XSikOHBlDT2QdInDGgICso7aVVFB1zqMnDgiIZJ22blNpiU6bZquewm96VbSftBFs3nbOdGs2DES01gN4zpEj894DBnXOiMBRi8C/loJpjQO2her3W2kUJI+wrvzcNi8Y5+QsADAgAvLWeO+RiAjd9c5Gm2CDvjopR49rEB/33BvvQSOAEjKJ4qaqGySO4EJiPiRK19FoBLjqcrmtyLev7YYibz/ZNjX8T8Ei3Bi4FQv50h0CIoixbQsgyEPTNEVZLus6ThIZxcg4AYAzre2EDBkSgkciYDx4+qx9BOQ9EDSmZozJSDDGtAYynggZsIjHIImMRud4LBSXprFa17HgjTPeGQ5McsaQO6uNs3EcN8Y55wjQe2/cemkNBh1NJovStJNdzqZlXcYq2t07cN70ej0hBHne7fSTKGLEBMp4kGadjowz4ghMCBUjl865KIrqutZ1BQBKcOdcWed1UQKtk7Ybixm9B86ZMSQlE0JyLrZmE3yg9iOGwIEQAFmLKbEeiAQhAZGxDIAD5rbe2xkwBoQvi6YZ7e6NDmYnswU3rm5mjdVpnEgpyVnGKEmSZJh1Oh2puNYagNJMJUnEONUmb3TJOaZZbElOCuuizq3RrfmKzs7Onp+7pelyJ+uFU6pmTM+LedSJuPeK5GjvNiKePJ+7Rjl+4b1nnBBdbcuqKq2rAdESkyrWTje15UL1Rl3rYF5c9qKBjDkjMMxL4CqBSEGUJIPhMM9Lt7CNcZ5xJgRnyjpRQGWdtwQEzCEjLphSIo7neZGmiQdAwabzSwB44+0Hk8mkmhery8Xh4d6g0/1s9mjYv72Yf9LvjQF6Ozt71vjVctHppqZeHR8/S9JI4V5ZV9ZpAKqaepmvhGB37txaLpd1rYkgS6I46mhtmCfuoXGOiLS23heI5L1vGlPXOtQDAqAQ3Dkqy1rKKE1TnoiKdLY7AsDXbn8w3j/6olic/vV3b337W7PpQo927n/7V8uqrnx8YVWy/1pd5o8vzZ07B8mwszo/b6oqX5bek7DWMyaSJFMyRSbiOE7T1HRMscqNb+arqSU9GncidZRkcja9nC1X55dTRHznnXdu377LhVqt8qdPn+/s7huSitzpxdTk1biTfeeP/vKDD94Yd3tVVVnyn3/+6N3332uaRghJQN57qx0ipmna63YlSiaEMXMGa2ZZ771Sqg27ISIRhHgg5zIAHpfL5XJZGnPivT08uJNlyeT8fDabA9Bo2L+4uBCCVTUqJXZ3h5PJ5LPPPvGStLZMNIkn49Fab13D5bLXHwgupRIh88dAAEBjzWq1WuYrrXUcp0nHcymUitGbsqq4FEIwVCrg8I3RlrwzNvgVTV17i1rrSKpRfxCJKFYJUsAPx4JxY9xqVRjjGLNNY5rG1bXW2koZIUKaKCKSTIYsIxFWVcMxdywhIueMddaR9QEzgLwoqhA8ZDzgFhgR9x5iIdH5uiq9sXHKpBAGAV2TxHHI0XKOyDyRNs4ghHpFcIBcSMUE5xwYc4QhfuC2WjAEs9pvkpGtMwcbiH54jrRFOOE2yfJAkbvRQA6QB6nlXdA3LkAa3aZgHQBoq1p92xGEdZh6/W17im33HQDaTsq4Cb2yTXvi7Xhse7V1XbeXjVv1qJsYw3aOHIBxznlLgNiqT9oasKmh2FaB7VdwPWC7HR5gW92ntoMQrf7e1sfbZ7xyu7dmazv8wK4RVJhr8YTNU/7PPdgrwZgwfp5oxI3920m7cYQbn//sg2z/vXoKW42qGWOhIwnbEDq1a5IxxhGFEEJ412Jl1g8RGWOh12KapjKOuJIBcggAsYqyTsYYQ1qTsofURhRFKolCas86R84HCAJuepq0XmPTNGVZVkXRDce5nl94NYp2dWuvRLDCZfOtNujbOZ0QyZNSJknS7XarRgdG/Pl8EU4UwPa0ud9ut9vrdwJHvnNOKi4E8952ux1ZR95bIGxq2zSasyaNy3yynEwm+aqyBiwYa4ssxU6nt793JCX33hN4RFoulxfnl/PZcueWcs4h81JyZA4AOJOMMc0s5yTQ16SNMeSN9+Sd6w+6URQJZMY05LwUjDFBRE1Vh+y58wY8C9TInGEAXW2/eiGnMxoNjDHGas47wWhzjqIomcym3X5/WeSNqxy43rA33tv57NPHz1+eZ+kgibvOuaOjg53d/nhvt9vNlgvdGLdYLJfL5XwxresyTeNulCYZ03ZRlQYq46EBYsglV5yvYdMhfLUJGXrfxoBhk0XVWnPOTWOCKIiktI2dTCZFUQihjg4OmeCPnj6x1iVRPNof//Y//m8R8ccfffzkyZPdw4Pbt29/8ejxd7/7Xa310dGRePbkZDabVZWbTGc/+tHHy+XSGMMReoPu5fzyx5/9CAm+/s1f3h2NsyT9+Ecf/cVff9cYs7e3l3W63axTlTUS3j7Y/8H3f/iNb3zrt/6r/+rhZ48+/fSR5nh7rP7+L34zifvf//73J5OT7/7lX712/0EWJXVRTiYTW5mzs7Pp5LKqmm7WieLIWmuB5LoLS+M3pfBho2maoAKklEKoUJK3XC6Xq3K+yKWU3S41jcnLGrn0tlkV+UeffNLpxt1uFiXSgYkTPhx1NLC8LDygkJHkwiHT1jtPi9UyyzImuDaGWRSCUUPGmJ293cVqOZlNV2UBM8YER84DgJkBeg/BOLDaGGudcyrljDFw4I3VpraaIil7vUEad7XWxapoau29BwGuds45HkXecWeZ0d5orxtiCEpIHdx0QUQhMMi8B2Nc3hTBhSLrvPcWrEcgxjxxRpxxYIxxBACwZImcpJpZz6xPInVrf2/U7zlnkDwi1nVtrQXGm8bMFyunG0Agyqx15IkLybgkRAIgWjdsvKq2IEYBcyDWLR5wQzkVnO82k9pK4W39zRgjcpwhEENEYOvgp9/stl70SN61apVtqc+bifxt4b5lglzp4G1DYVuXtw0pwlfhyXrvw2sWgrdhbKsu2vpPgBM652DTnQpgbUggonfX8iPbwQP4KXV9rWDCTV6gtRL8q4xJgPRTdOoNTbk98Ho1RGuItNZVu9t/BPHSzzV+Hv3984ztqgfYQqhsG1I3tuH6FPktkI333m2aqjPGGNA68SSEt5yAe79pbrJJjRGsV4kQQimoGg1hMZMnWpsTtKEmy7IsjuNgEwRje30NRIAgOFdKpZ1Op9djAkP9l3UmcHy5Da8obPg92x4leL3fNFzlCq+ztG3NPG4N2HqDtqelHe0K4ZwnSdLvkyPY29ubzpeTyWVjdDvDfkMI1tSFiaUSgiNDBhwZOXAWwDFvWNO4pmlCVHzVs7YR3DlrgPMIAepaE2kpPOey0+llWcYYGNMURWHMrCxrIWyvJmOMBxfFQkqOjHkg75GBIAQpWCTQGiIixphSyhtNnIGUDMl6W9frRSJl1DSN1o0xDYFAzgkcMiZQePKh2kUyHgmZZnGapkLweVNXVeM63UhEFq1A1u90X758KaW0zpdluVwu5/NpkkacY5rGcRxJyauqmFyeezJFuZhOWVMJ731duaKo81XjHEnBy8J7L4xm+aoBMEnilYwAAICF3nWb57VpqMtYALlbawNhWLAdiQgdNHVDRJgAuXy5XAY3e7FY7OwdLMtib29PdRKRRgd7O3lZvP+Nr6oszrJsUa6eHj+/mF/WFZ1fngvOEuvnSdLVDf34J58/f/40SeLXX3stTpPzydl0er47Hq3qVcfGCnlvnI3G+/P5lKFYrYqqqKfTqbe21+nuDPqg9d39PWbsnZ19InzvK+8+efJs3oC1tlgWpy9Pnn3x9M6dO3VRexN6Si1D0bIxTivvCZJOl2EExHzTrIvnEImhR9BGBzyAcNY4Ct3SrLX94W4kJOdYVvl0NgcgxtjZdGq8YQzeff8rZbl6/uyJmPnxeDwY9y6XNQFW2jAmmIwS4qA1Aczny/CiBc2opBSCAUDcHSRZdwSMS8EYNLWpqkZK2+l1ich6A9YCkxAxKSMAcL4mQEfee0BPThvdWNPoOE5NY5qmqeuaiBSFHg0CQQIJBCl4LETMueZcCqHqynrvwRlEHjgqw/upbbV+n4GIrAcAhuR9QMCiDw3gg4svkJjyDWdMRFEniw7G/b3dMUcHQFVVrVZQa8OYyBktVxao4Vxyzq31zntjDBcGGDoiRyiECguREXDOETxwDwCOKFivRKSUgk3csnVKQo1GK2ik5N4j5+gcbpwxDBEFALAerLXeu1C93fq+2yo/eG9hAtqv/FWNwJVmuxG+bg/SamsiatHarYzbdr/a93D7w5B+ZVdu2YZuYVvpvlLOB9ftmPYCtmX6lpa64qPc1tzb366vFq66BN2wk26MdqLajZbGsDVKcBMyWR/wOhrgxjT+px3/kYfd9nppq85lW+e9apRsWxXbtkV7tJCbYFvdzznnnjbhBCJyNmx4Wj+ndk/GfPvkAnsF53w4GvX6/TTLojj2zgGQiFQURdoaRJRcJEnS73fH4/FwOEzTdLGYBfY5Ao8Egc2sKIr1pWxuiDEWx3EkZAit+XXDp6tih3ZaWrNgnU273ilj+x1pj7M9je27o5TKMjTOD4fD3d3d8XjcGG2MCbWXQgilVJZl3SzppHESq4CVxpAuszS/XFZlHaoz6soxiFLV78bjO4cj8jibzb744ulxcUrek4Gm0CuXSyal5HVZL+fzuijTKN7b2+Mq996D90gcSZD33joiX9ceEZG4FDGQI88YY1xKAkfgWEhDCUAT5KqwtvGuAXLgnCMLAMgIvAfvm8Y0TQUAUSTTLB4M+oNB7/z8HMBrXef5Mo5TKSXnChEvJ6cfffLjwTAztlrmC20bFfN33nvr6ZNT42oZKQ+uquuiKsuqqhqqCsY5b5qmKHWjPQArK2tMaYzJ86YojbW+rGwURVJEjDESV0zzQqzXoZQyMOUEHyo0qAuumpKyqHOtNTkQwhhrhRBpJ4vSpG7KsqqsN8DF3t7ewDSI+ODNB4UuL07PPv74408//wlyGB9mo/5A/OIv/crHH388X0yRs+Pjl1wmMkr64zHnXCZpEqvBzs755eT45GRvd9zpdN5447Uf/nA1mV6cnPRef3D//XffrcuyLsv90Y7gvKmKP/ujP24afffW3XK+/Ff/6l/df/fbTVn3Op0sir/3F99FC73h4Nbe0aNHj0jbndFumVfaGuudNi5KM/CMPHIpCIFLAQCSgHMes8ToNXe6cTY8eeSsKAo1HI52d/bEXt2U3pi8mEeJKvLF17/1td/4zd98+sXD07PjxWqp4sjMtXZyVRZ5Ucm6UXHqkFvnPKCKU0LugIDAWhPYIaUQP/nsM611v98fjIfhqaRpopQyxoXKIsGkUFLSGi9WFg0RAQEHgZxrsro2+bIwjauqxnsIVU+hWItzbBpjrVfKMiaSOLPGx3EcSYWIdV07bYxpGAMhmPcRQKCVDzqJAACQAIgheaudM+Q8gmeMScYFBxCQsTiKJHgtBVpTLecX3jWBJWlVlM4SE5HW1jtCLpSM/QaRxLbwbpzxNiTOGN/WrEavO0wGVRr47b33gTU2fBi0UVjQ3jnvcd0Od12gIWDt0YZ2OED6iltpS8xdZ0ikG+2V17uFlwWu62AAWEvzjSYIws57L/k1s6AtqWgdo02ZwNW9hKPSlepdYyGQbZkCmwO+qmhbgbutq9qd/QZ9SbS2hretpRvqEDeYyhtWwrbxES6WtoyqG2ZHuNnwfBlj3m+1M97YXa9abH/X8aqS3r79Gwf/UqX+s0erCNspap/1l579hulz9dsNCDQkDTZE3euDtFCVdtJaUC349WUzxjiHOI6BCSllaG5GjCul4jje2d9LkoSFkLcQQvIkikWkiDgDiKJoOBrt7e2Mh6NQdRx4FILpGbp/GWOQXNAKDKgNSyBDIaU1ur3lbbszfNJGBEPkjHNO1wCwV8syxAK3F9L2XIULUx6TJOn3+3t7e3t7e0VVrlYrbx3nPJJRr9cbDAYyiwLwfLlcLJd5cDaMtnEcG+PCyxXHnW63e/v2ndu3b795bz9SycuXL0+PL5yxQkggWi4WRb6qyoLINbpwznpnkjhJk8hizZF5z8AhiQCwEUTkdY3ICDlHdAAbJjxmddOAIysZY+AskeeMCw7WkmAgODEksM740ntrpKEG67o0VnOOnHlds7JYceYR/HDQI2+tMZYZAGwq3TTNcrH6vd/7vXffe9O6kgs4urW/XC3G4+gH3/8bhPloeODcGjDLuSRyaZLVdZ2vyvlsWZYV59xoYMxWZUNEDBVD6yxpsOQxtF3cjii0tqDf9LIRct15PCwVzmPw6C2BIQLnGm2NQQByvi7KKIoQoCxLyfD85Hg8Hu8Nsnw5Pb84rurV/sE4y7LxePzm62+I4XA3y/qErGkaoZLxzr4Q3HtZGlMbSLN49/Du3u6+bioVx3Ga9gfz8U5P15GUrNNNXnvzwWq+OD+9+PTHPz7aP5otVsenJ52s/+D1N//oj/74zoPXjl+eSikjpQSTXzx+/ODea6PB+OL49C///C+FUkxwrW2jTcPsssi73S5aROChjVB4GZzzjLHBYKCbNU9wW8pPRNbasl6VdXJ0uD8YdlerxWCn9/VvfPD06Rer1eLi4vLtr7x/fHb+B3/we6en5+PxGFSnrJqmMY2DyDOuFArJuex0OkxwT0jkmsYUVneSRPZ6yzw3xiDnQikhRKebdTq9fr+/KpYbwQFE1DRNVVVN0yghiYAIgRiAaNAhgPd0cnIGziNyziVjZEwT+gtoC+HVTdNYRaIDaRzHsZJVvfLeWqsJUBISSGQOmUMAIIZriwE4hNJnr02F3iF4wTFWIpJcCI6IPSXTSFknvGuMrZeromkqAJJSNsZYC1b7srK1AedkYzljoXeDCDIRglPLZaCJDb41bCK9nPNerwcbRRIi9oHrgjZA/da5CQTJTd0AAHGiYOSExkmIoSjUrVPjzjpNRODXwQPaqLuN+fIleiXIsvDmtNfjN5i7wFvQKlS/qUWkTXi5PdRVemXDXrBtGG2UEG1FFIiQSSnpepfGzeXdhFPcsCVubK+zjptuxbTVJrsNBrQ/uaFQv/TgNxTkdmgE0bV5ZWuvAstbO6zhIDfUKvxnGDcshr+robCtFMMIT3x7xl69i+310BoKG2RMmAgEWLOVb/+2fRZtaGHdlXud+KdYSGU9EUkVSSm5iuL1UOs3izOhVKykiiLGueQSAOSG9sB6l5eFtVZJCQAhHR7oOjjnHPm60srogBFGRC5QMs7ZuviijUXRFiEpbAJ+rdEDrzBVhLEd8dreaA8uhJAeoyjq9/vGUZ4XjdF5nq8Wy+3144xtoG6a5vJiOpvNgk1jrR0Oh4EqcV1CImA+uxCcFNRxHJ+fTebzqbU2SRIhePjVbH6R50vGaTDopakCdMvVrNNnodSOyAYZBQw5F510aANhLnqSLPTOiFQErvbWWCAlQr7HWesAPBAxhpyjkGAceee9bZg3qBP0qLiUkgkE01SzSbWcTw4PD0ejASOoaw0EVdU0hamqalUuPv7xJ/deu1XVC2PLTi8BRs7rg1t782klI5HwmIg8kfWOyMURcseiRA3HoyipqrKpa22tF0IIKSSic8Z7L8Qa/oVbMKbgMoQWP7Cm/MKAEQkRBSIi6yUTIJVkHD2Sg7qoqmUOAGknGw77mZTF5PLUupenJwcHB5nE+3fu7AyH/U5nOBweHh4WRZGmqfjoo4/KuhoMRs9evLicLoEEMLVYlaZpCCTjGeeZx1jGylp3Ma3ShL/91v3QzvH45BmgPT85f/b0RZZ0Xpyd0YtzmaUP3n67MO7x85dSRt3uLnmPiLo2VjtbN/2s86d//GePPv2cRzLr9pdVIZViSpVVDUJwA5zLxmhrLWEQ3BT5qGrqptahm0CkYkRsmqap6zffeMA5r+vy5clzLlDXzc7O6PD2rf393d/7vd/7H/+f/+9/9k//MRC/nMy11v3BHjgnhIoSJOTEmCeUKLgQVaNdWQWu7LLMm7rWw36cZf3haDabrYrSOMs5H5phkmahF5x3XpsGYIM/0sZbJ5AjQ+ewaUxVVcv5qlyVpm7Qk3UEYDcaiKrGELhAMl1VhYqYEAxRpamKpELmhUTBhFIqSZIo5ow7Tw1aHtYEAwrgeEDPEFAQDzk5JdJISSVCWJI7AkHOOU+WiHkGxAARPIJQkfG+Lk1Rg/HSEzeGC9QAjIt1/t4RMSGEYk2jw8prnUspZRRFB0e3Q6OHoigC3zYRBfhVCP/gGt4lGGPWWiFYANwAcPIbGl1ELhCBE0PnXNDaG7HLbsgptqbKAbimsdbbgYYdX0nTbodS4TpdQRstCEdobQvYgrDRVgUEbbCTG8FKbankDUVNRB6u6ilgEzO4cQE3lFkY207ediBhW73RFujsbx2bn19zDdvZwOvj2pz+pxh/q+LfDlr8Xa2E9vjrN9Fa7yX9lFrT7fHqzPurdr2hx8m1lbNtVWzPHuccGQIAX/N/MGBcekBEFcVRFDGp4jiOosiRFUJIpZRUURRJJQNaO0geIYRxbpGvtLORkIwxwa6qLpHWhqwBzzm31hqtW7xwiDUyJduluzYFtkIm7QW3N8XX3SiuHnp7uu37bbe1NrjBlQNAqEcj5ABY6ybP80uprLWCYZqmSZIk3Y4Qoq5rrQPhxLq8MNgwIaLAhdemOL/Q88XF4uwFIpZlreuy14mTWElOkvM4VsZyBoZx3+skUnLGodOJO4OUc+4WrizLxjoiUjKK47jbH5RlaUztyKNHyVnA6feygbWWIyolwJOu64CHM7UmAsFQcmYYbh6w50wgOmShOjoIXOsc1XXpXFdIlvHMGppMZroJvezZxfll0zR37tw6Pnn65MnjsiytYeNxfzHPPdkolkBMCIEs5pyXRe3JdLppr9crivri/NL7lRAQ+IiNMd5bpUQUSwDSWhOJNgoYnkgIQColaNMaLQjJsE8AOa6DSZ6YIzCurot7d+6maTydTHG0u5pM5scnzrlGqH/9O7/zT/7JP4niqM5XNk3ml5PvfPev5vO5+OzTh3fv33vrzXcWq6KqqjhLO51eknSzrPvm21+5d/tOEsfPnzxdzOacgVLqTtd1ukmv18uLha4Nka90NV1MG+vA4u7u4Tvvf5Amvf/X/+d3dw+PrPW2NgxFvzd01nLAPM/Hw52mqvb392tthJRc8yhJ4jRdlDnnnHsIaibMgrXWORJCnJ6e6sYQUafTiaIkvCdFUVzOzg8PD22hT05edrvd8Xi8szve29sry3JnZ+cP/uAPlFKr5ZKI93qj5bLMhhwRpYgsgbaWrLeOOFG9WOZliYhJJKuqqKsKkcbjsbvSEIIxrKrq+Pj47OxsPpsGU47ISSnjOA41God7AwSutV0sVtPLxXQ6v5wul4vVrVt3VquiqioghoyYEIoIAFQkqHSAnsghEucoJBeS7+6OjW3Qk5RSrPNSrmkqphJEBOZZKGbCgAZ0KFAylkQ8iWUSccYh0OxoK1xtqiI3tkaWqogb2xAReUMYNYaKxjYagUWcx0TMNivO5Xr9aastMCGk9XVrpQX5JYRkPMuynZ2d1WpVluVisQgeQ6jxDW0mmqYJzgdjLICw+j21ERMW/IYtETHk1ZB48AU51wBAsIEjXBfr23qk1Svh28DrzLawh8EOCBbMthHQbrSGwrZkDB+GVGsbXXDOBWudsatARyDFu6F+aGvjho7fNhSu/YRo+5qvuw7XwBatmIDranV7oq5NC1595Te1lK3zHdTeDUPh5zQ+/uPHtiZr7+4/+OzbWnz7UF9qedxYOWG0DyvgEnDTpnv7Cm/8d30XhIwx4BwRAbkjALem3GcbEsyAeA118FEUKSGF4EIILgSSk1KKTfwgEBOlaVqvVgGFxwULNVPee+1cmqbtqdc2AQdG1yp3Whu3zcK0qqX9RG1W2vYq2n4Rwthk3Ci80R6uKoaEEGmaAuDOzs7u7q4SkoiU4N1udzAYGOc456Us67IyjQ7CxzHodDrBaIiiSEVr8hXvvbG6KAprXLebcd7T2jpnQihXyt5wlGndcEGc42DQu3X7iHigoGjyvCxXRWNMFCXW+ijulEWd53lIUqtIBKEUx7HWGslJLjxzztmyLLTWAOhhnX3jAnmgjYG1h2Ctdg4j4lEsojhRis/nsyiKnKNONmAIurFa2yzryHTw4vjss4cPf+HvfS3rxz/84feJ4apcdfo9jy/LphQqtcZDWThvlFJCCO2M98S5bKx2QFmv2+30tdZVVem6aEwdc4Uics5op8Fui6wr2cX5OsgUIvFtADWJMwSuG1vVJRFxhkpKMpJ5OtrbF45+8etf6wjxySefOGDLyaVmrloVHNnOcDTs9n7y+Wd/9sd/k/RAyBjOzp+/YV7rDwf7e0fEOJHqdPacc7OZefzwr/Z2RvP59K0333j5/GmS0AzM/t5h0/Bu96Cz3x0OdusiehGXr99584uHT59/PhdCdlO5k93pJL3pdDqrmjt3br+cTF+//+D0sra8W0GkMRnt33OeZstFpDhH1dTQS8eMMQQXkgtKxd77oLACvxVjbDadXUzOx+OxtXY+n3POBYeffPJ5mB2llrqBTjb6q7/6m8vLyxcn87sP3n32cn55eWl94htxMZt2yjqIfo+QJEmWxc655eTUWquUlEJ6U+uyaHQzn8+fPn364PUHw25WFGVT5L1eLxZyOZ1Op5MfffTDXq9TVqumKXb3Rt1uxgV2OimZ271uX8rENMvJ5cn52dQYr6IoyJA4i+M49t5C6bNe1O/3Afx0NnHOqYh4Jqp8ZZzu9XtJopxHKaDf7caJappqOZvn+cKBVUpJLrz3ttHee8Z4JFNLMolVLBUCVZUFAAziMkmQI1AqyFOWVN7WVCKiA2e009pq4iDJGeNs45wrq5hzEsaG5BcAQqPrskLk5BwxZhhyJjmTKo5llBRFMZlMjo+PA8V1SKyui77iuNPphCLJQCSXZVlRzIPo5Fw6skTgPXkCL1gUJZJza5B8ZTQDYELKxlgiRoSwoUXwHrwPgRnfZtAR13T3nHNPdtNdJfDihYlwBITrwgSgwNVLKK10zpO9AiKEHWyjOaBD01i3fvGIyDoyFhnzsPa6AIAYQKhZFygYMgRAsLBx4JB58iFKyIAC/3Qw8AUyzhgxZATeOXKenK91TZtwQoiOwCZc7Db0grjFpmK9CZcXTMkWmci53EpOc1xz+JP3dQsypTVEX3mgtgys1T3eew/r2Mnmt1eeZZsDatXl3xYFuKb7W4vEWn/jw7Ad6EdbOdh+bo3H62dqVXawRNdFbpvCWtzkVtbmEVE4XChyo00VqA+Nznyo2wVExsCDI0/eWUvOCGHBaEOGo4+ElAjWCu+9trVbW5brwohA9MCRJ7FCwQHQec8IOKBAtjva894yZBEqgQI9oiPGwDlw4AUDrqSSSsmYoSCPFrhDwYRkjDmjtbGCMyl5nuccgQuUxIk8kmWeMcbAE2eMMy4YD+20N5bveoWHZMq60zZYJxxjjGCLO8QDETlyHtCDd+Sc87ol97TeNcY47wjX/bUJkAi82xkMXr93b9rNrDZKsNCJaulFv9/HpeTzqYgVGTSmUSKJkyxOlDNWSknkmqaRTFRNFXU7TMm6rp3zXPJ+JyPyWuskiRpdeO/iRDR1vlrmuzv99997O90VZ2fnP/mIzi6eOJf3kqQsVsY76HSGSSKBqrpWScylIOerqpqbEkg4B1VZNGXjnAeuOJfAnWNkvPMqQozzqmwMDvrj0q0aNMNhtpyc9pJOx+mDZJQlcnRrVzt/MZ3NX16IpHfn9mi2KJbFtDeQvV78+//mT/7+3/9Hv/qr/8A2e+Txwx/9zdnZxc7umxeXk1Vte73e5WS2KvKDg4PpshZClXUpBJ7PZqZp7ty5M5lMnbHDUR9jXK14r5txzvM8l0lKIJkQRV0zxrpZp2pKQrr7+oNytTw/PxdCDAejKFZN0+R5vlqt+mnXGh0pMdcNEXV6XVsXYE1dLncGveePH64uJ//r/+Vv9VP14YcfDofDZ/mLk4tHR9GtbMD27w8/flKmu3D/tX2xv7tnnD958VJy+Ru/8RuLvHj86ElZ5kVRKCG/ePLo6GAnUvLxw8+01rPLSf/W4PLyMo5Tjsx5n+d5kqXf+ta3BKrJxez4+LSX9S+biYxEVRXT6WT31v35fHp5eZHGKusk3ts8X87n8+DLrIq8rGreCGJICFJK5n3AyQdBHF57IgqtTRhjQf3Axl2o6zpJEsZYqAYxxhwfHz969Ojy8pIxFlgUhRDT6TRUxocenQCOS4kQ+sRGjMuyLJGzYIh5D1QWWtvLy1madRGRMS6lAGL5qry8nF1MJpxFk8lUSn7v3ut7e6Oyyosib2rnnHHeRswnaTQedpwzRdEY7bUppEIJzINmHPuDVCkVRWKZL4LI9eS8MQSOMwCgbi8D7+JE9btdFYm6kE4bIrcsPAMSggEwRl43jXXaW60kd85UZEJOcq0tkIwDznndlNbaqqqtNY2uOOeRSqy1WjtrQmIWnSPvr2jpQhwFcQ3h9psyMM45QxYeDSJqrReLRQg2BCACbopa+/11C93ZbFYUhfdeKRX1es65bVFORN6TMQaAeYAAcVjrpBvphS2N0uoo3KLjheve4Zf+6saoqmqdcJWyZezAKyzCJhC9SYFXVRVwGriVm6B13vdngeZabXcVOfwS7OOXjFePsJm0YIhgGxdpN2AT/2gjKO31860L5uveWk54B6B/9tn/1pn82eNL4wQ/495bS+JnXMaND/F6gW57yzdO3Z50e6P9tsW4MCDEQO7EiJi119qJtbYaCi48toaCEGtDoW06DhDSxlGcpXEcW2uJrgAoRIQeQxQNcV0BLwQLJeKhyI0xJhgIIWiDnmGMySgC7zxZa61zFjfMoS1qOLCTIQIyxkMvRLwCzBCu/3kXAndtkdF65TtHa1ynvYJ9eO/JeetD11dwANYzRwAAHlCoKPQWqrEADPYcQ091UZWrAgh73T6QD00onNXO+ChKdFUGuliPPomzsqyklJ2sv47nSemcU6pZLvMokpyT0Q2QyLIOADs5vnhz7+7h7uHqVhGrH+WsHo/2u6k12kmZMBSdTPUHQijVGM0lG4/HWRob7awmrY2pnDEGyDAJRbEgRo3RjgAwTlR3crkqlwsSSESz2SyNIsHV62/dv7077vcya5q8qFAlPes18aKxZW3tYnFxvjTGaAt/8Rd/sVzm89myNxzUtT5+ebKzt3v79t2mabhQaScrqvr09Dx4P0LIOI4PD/c5YFmWT548Ptjbr+qCC5Z1EgDQWiOCEAKQM4ZpGiOiNnVQf5zzJMk4l4gCUValnUxmwVs7OTlhjIU0kFuTzwKT4sGDB7dv3/mD3//973z3r4bD4S/98q+8//77v/M7v2OMe/Lk2d7B4d7uwdnZxWy6uH1752tf+4YYDcYvjl9+76/+ejAafvXr38ziCMkwFL1ucnl+oeuirkoG3nnHEfYO98t6vipyxsR4MO71cLFYCR4d7t9+9uT57sEuIk+i9M//7C/Ksu5lvb2DXRJUN3kU87JcZlm3KFbT6SSK5MuX58bZWlvjLGkQKopDNUHTGOdqvcbuamPWHCOccwAVx1EUxWlKRLKutdZSqV6vJ6VcrVaMsb2DfaXUbDHPy4JzbpwdDof7hwf94eD8/LwoCke2rmvvfZJlQWNFUZSmqbXWete27wSAqqryPF8uym632+/3syzTlXfOlXnjDY9lp8zL3f2jr773TSHY5w9/oisPzv549jej0Wg42onjTpqJW2qnqJq6boq8EipyzuWr0nmj4th5kReOMRYpBBCeam80IkgZCQbGNAzJW2adhsoWRVGWZVXUvc44JPy9tyRI+vCOQ5II7723zhsb1DACIGKhTRTJwP0upHXOWeviWDLGrSVryVjyHrwD59hmJV0JxIBqxA2ktFV1QZCF6qaARVhzS2wIB3u93v7+/q1bt6y1T58+DejUoIBDysa5IHHX+4fMsnGurpuWfcF7T3Azxt6qvVbut7bCq5pjex/Ysi3afa5SvK/obNoqYmxv3BoT6tK2AfB8zRn8JRp0W1W3qosCtoB9yQVv67DtcUPPtRYM41dYh/YItFVbD9cNBYI11dW29PdbEIr2LNtn/NKL/DuN7fm/8UC/dLQL6cavQirq1aACXLct2knzGzqWG1/duJIr9b/BfKwrDylk9LGdzO07CsqMiDwQAuecoxCcMQDGhAQmGGOh0YOMkzRN4zi2jSUAYMgEZyJE7cgTWec8ucZobQ2hB4aBXb6TZnzDAu6sJQp4V8YRwTvrtDfWOcsZC5buqy9C+/RvvDhhbNYnwLrg1zvvidA554FCPMFaa511jrz3gSbeOPIEHjb4CcAoinq9nvdeSlmWJTnLOHDAlMVxHHfSLng8OX05nVwS0XA4sJZAwN545/Ji0lRaSVWWZa/bTdNOyBEEjNeqqgKaj5zLssw5rOscwSklF/P8448/OZ4+zbJsMVvWlYll2sn6PkLvoZMN6rohY4RKOt1u7J1K4sOjw8M7d4qiMrUDB2SpqUvvrGB2cnlCYKw1znnAWBsYdxYX57O51Zz87GI12B14wN29w1t3j4aD3u/+zu8s8lXa6w139iUT3Sjlccbj6GJyLKW01n7/+x/+4IcfCyG63f7F+USoKO10kzhzHkytnaNIxUSU9TqLxSJJkrquZrOZaZqyLNMsKsrlYjkNLUNDNkdrXpYlMBMq4wCgruooiobDoRTR+ck5kGAo57NVXddN44yBoihKbkO5TRKn1hly3hMpFQOx3qAfJ8njL57+3h/+wT/8h//wl37pl374Nx/ty7uffvrpky+eMxG9OD1J0+6bO3vDwY4ABy++eH56fnZ+fu6ci6Joenl+dHQ0Gg2fP/n8tbu3wOtIsnu3Hzz89CdH+3tgoiKvqqoSSTQYDfO8bEpjvcl63elkFqUyTpOkk8wWcwMxc+A1BFwegfVkdnaHy9VcKjSmaYwmYEIw4zxjmGVpmmYLY9r0bRDBoalJeCcDICjIQdrA7OfzORFVVRWMJillr9cLumcymdR1HWxtKeVwODydnNWV9t5zGTWN8b5oGhOnCWOCrNfahHiGNZ48ksd8sWKEEY85Ca+Iczns7uwMdo1tFou93d0x8/Ll0+PZRS54orhybIlIztbOsjhOk1SpiJZLnaQZEWpNyITWHsBw5gXnvUFfSkHOFUVRVSA563XSOFaxioAcY8zWtnGuLhtyIGXU73adc1rXuqmdc8h8FAnOMY0jY4zR3iCQIxsUM/CyAiABIAAIQXJEGXW6na61FskBafLGO2dNqKr3TLDtYG8wFDjna0RxQDjaNesibHK6LaQu6N32E8ZYlmXD4XC1WtV1HUVRVRa0yaFa7611/3/W/vzHsu1KD8TW2tMZ7xQ3ppwzX76XbyL5OFSxVGJRNaiGbhm2ZdiA2jAs2RDa/43+gIYNGIYFyw3/0G60uiVLXapSFYtkFVlkkW+ecoyMOe54xj0t/7DvPXEzMskapINEIuLGuWfY+5y9vrXWt77lvScILa29ttYYC7DiqXgiay973NOLLQ+ubJuIobMxuJGl7nbYxA3dAenFVsVBAb6z6100fnPx7dLDYSnfjK5v4hjYYAt2SzPbqPK/Yrc2v9jBgk1Og9/YXjkOm/dylX5BlwIDL5uTlw8FL5pS+OsM/C/ZNm9/435ffajNSaSNLSh00SobcnX/kH1omkaqlSqa22gxunmPmzBi8yCsM6UUQhEvoJArg0NEwdgzAELOOGc8SLvyKHh4nAHjiCiEElJiINYjciW5kmtQ4oEx45y1JsyodFJ6R9Yg4oBzAAiB1bauQodrxkAJgeSRESNgDNW6a5SQUgghpGScQ9CEWd/b5Q/r4SMAIgzNh1aIyoEnCE+9X+FR8B6CUmoACo7QAzkiR9wDEiLnSMCiGPK+Y4yFSihvjTFGpf04jkfDoZKSeTc7uzg+PplfXCBinuf3bt0UjCkh8jStigUjyPO+Uso7WrbVclF45wKfA7xEEFrXTa0ZktZ6sVicnfF8KhBRMjnIR729Xr+3VZU6SdIkzZmo68lkMa+QR0meRXFKwGe1LUtjWyuY5B6sQ+88A9BNy9HFEfeMz2fzunIR4fYgQ5cmSoJtlYysN9NFRTxqLC1au2xsbztJeoNFVTtkWS/pG1M2RZZlg8FgsSiWy6WSSauNjGJn6eT0PM91EqetNZPZ0hgT+n41TRPHsfPG2JoxuH5j+9vf/vbjh48++OCDxXKiItbL+pFKgoSGSjhQqMyHfr93795rOzt7s+ni6OCoqmqAetgfkee6ra0lwWNjps453ZgoipRkXAEA5Hma9XsySt75yleOjo6mi+XPfv7B3dfu/S/+8f/qydnh8dH5o4fPFkVTts327s727t5wsC36+UA35sb+tel0+viLL/avX3OmBmqaejboJVvD4fbWIE/T0Wh0ephPJxdxRPmwHyUxlwIER87rdnE+m+rGVE25d2M/ktHO9e3Pvvwso7Qumzi11tXTyWzY6ydJ9LX33nn29Pn5+ZlUvDVeKsmFslW5XsS9I++BmOBhCVYvioRIKQmgrCvG2HBrxBirinIymRRF4Zyr2/bJs2dbW1ve+7pte73eYDSaTqdPDw6CItB4PN5850NEoaoqXhb9fp8xFqr/3VpnEBHTPMvTPIuTNEqFEJFK+v1+r5/FcbxYzMpy+cUnDx8+/CKK5f7+Hjio2qbmAoGs0ULiVj/t5SrLpZRRVTbaWSkj8qi1jqK4N+gzVIhYluX56dmCM8GYUooBJ+eNsVbruXXBceeAUkpvyWirW+OMBXBCMsaAMfBknTceHOOcSwFarCoKBOcqAgBwApnw5AiFB9loZy1oA9ahc+gBPTKPpITo6IeICOueQ1EUsbVSQiiA3vTDNgP1YQ0ty/Lo6Mg5t7W1RURRFIWgaMgWK6UAWN22TdO2bQvIAZi1vjUm1PGvcvMvig1sWv0rH24u4r8ET2za7/W3Lpl9V6z1y4eFDfGl7hrCXTPG4AWy5C+8NvJERMH5uwIUrpz3Clbo/tqFE7pTbFrfDkNcwQHh11CXvzomrNIQjnxdt79orP5zbVfw0OY9vryxF8meV45zZXxg1RfehdRs4KMppZRSHVDontXNi+nOsnlJ3V+pG+u1lsbmvIc/MsEhSFAgxxBXEhy4QETkTCglpULOQiGz955zDi82sezmBdaie7RmkBDRZDIBACRnjNFNbYxBIERK41gwVJGIhAwN2DbB3xVYwzZkNzdHbz10DIgBBFTkAzgM7wcReu9CxNE78D70fwj7IBH5lbwYI68JgTwCci6Z5Exr1uiWkS/ms6osski++87bSRz95Md/8fz5c2MM86ZYzOuy8LaVLJMMGfnJZJIkCQILmc00TUejUa+Xnx4fVlW1WC6cc0ywpmmcM3mejwaDptbOeMlFEmdCqKKYzGbLu/fy7e1txuXjg2eHh8fpIMvq3qKs5LIgB+BAEAciZi0nK4Sdzi7SWPb7YyD2bPr09GTSy7eGg+15rWOlru1dN23FUJ2dzyzxs1kxrxqQKhmMk15/oS05RMFlpOI4BWBZ2mdcMa7iOPYO+oP09PQctXcEwIVrrLF+viiOT87qah6IqAD+2rW92zdv9Pv5W2/fv31rH9B9/tmX3tv5fBpFjeChoQFHJOsaIty9tvPOu2/2e8OPPvr4xs29w6ODqmzGoy0Wy/OLSghx48aN+UXoPA6BZp6kcRjSNOt98fDL1vk0z3uDQW31n/7gh//kn/yTp+cnUkZNo4+OToizRpt8OHr77XfFa7fvvv7aa0RUlcv5+QXQ9s7WUAps6sWD+3eePXu29dqdvd3dpmy+8+u/9vmnn/3Vhz959913pVTT2cx5aGu9XJbE2NnRmVLx/Qf3B/mgbpvT0+Ms6z18+LA4XwohGHeeTN0skyR6fvik1aVSHCoSgqlYFhUYu4pjt9Z4BK5WCePw0Ashgi668aElhh4Oh/vXrydJMj2/aHTbGq0QrbXHpyfLsgiZiKqplVJZL0dE0bZVVV1MJ0KIKEoAQKmYMUHktNZV3ZLHrJfHccyY8B6081pbra3TmiMlkUyjWEglOSrBIiHRUy/NdFO1VcuBRyKqi2q5XGZDELgKYGRJvLM9yvs9a21VVfNF4ZzP8z4hb1uTxNn29narqSpK3bQAgB5bbdpaB5UE0+i2bpqmMU1LRFnW6/V6aeK9I0aCSxQSuQi0eluWZdM03oGUEQADFMiQMRbLRMXSWbLOa+uaWhtTV3VQ33DOWe89IiEi4xz5ukJayjWNDsIUhO7mIVgNaxd507Xt7Gi3Es3n8xC3ZIyFpu/WWrvW5w463N26tu7v4Tsz7ddcvE0L8fLPRC+kHjYNc2cdNy1u98WViXWXSuEBHnV46CVrBN57fLFwwHtPxAJQwEvnDTu+Qrc6X16zf2Epp41t8+66T/y6PfQVE3UFZGxe0iY7oTvXJrLpLGC3f7fn5vDSOqKwuf1yA/+Ltlfe5i/ff/N6Nj/HlyIK4XastSGjzzmP4tDOOA6407+YAIJXpSRW56Krh13N8osahZeHWtv8EDkAhig458I4S1xIRC6FEILWZQKrcA6A855WyTVHCIIhMARC611V163W4ewLYIwxHiRHAoE/tJjZCGhtRsVq3XoEFBw44wiMQDIOnJH3gAjrGAli+A1Wyuir6Qai0B8LvQMP3jnyvmvvTt6TsQ4RPaAjdKEFJ+OMC0FAhFxqbBrnaa0ZyRhpp8vZ5LSu6zSNe4n4yjtv3n/t1unxSa/X29keeNcUpBm4LFEcfVO3iCSEACQuGCJoXReFt1ZX9dJanWZxEsuy9IjR3v5OrKRgfDqdn52cMmCD3pAhnp4eJWl6r/dGPsjpuT8+O4YZT9I0TtTtt97oZf0ojpuqtXUbIUrFARznmCRyPB5prQmsJxvFSkWiOp04b5JIKRVFiltgIOLzk9OTyXJnZyyilLhSaU8QcSljY/LesK7rZVkRkW5tWc28B8GlI1BJyll0enZR122W5ZzLh18+unbt2re+9a29/fHTp4+5gNfu307TWOvq9OwwSeSNm3ta22rZtK0BYmmaVs2MiNq2JQJj6rJcat0sltO6Wdy9d6Op23I5rapma5wNh0NEd/PG7VbXURRVVRX6YhO5oq6Kunr45PH5+SnnfHdvG73/yc9//t7XvyElt1Z7b7314/Hus+cHp6djY1oxm0zfevDWRx99SNYpIatieevWtdY0Uax6WaybEsjGUpRG7+3cPn5+AAyjJAbA1hrjHZciTpJev2+tPTu9+PMf/fD2zTv90eCf/Z//WVXV//Jf/svDg+dkzWg0cNqoiC+L2dnZiRCMfFCn4nEcR5F0gDw0CF+zyULYMKzXiOica5om+EOBqs0511pfXFwsFou2bYP3YK0NFUQBK0gpQ6fp8XhcVdXx8bH3JKVijDHkDLmMFBBqY+u6EUoJISFQS6IksH2aYtLUUJdRnmYQSfLC6KZtRBADiVW0u73jrJ5MzqtimWXZb/6DfzgY9FQkrDVMin6/LxUnbxmDNImcIyKHALFUsRLgvWCqqprJZFYuyqZp27oxxnnrYhk755xh6CQQCmSRyLN46DSzDp3zXCAKjh4IDDnQxjRB5NFxRGYdEgjGlAuipt5ba7zHtm2rqmmaNk3TsHIFk8YFIgIRD5V+G7b/snQQNxrYdwvTZpJ7k0EGGwV4YXkN2ixtg2dnZ0VRcC7ZqvyBI+PeAyIHxnyoZF/97xFf8Pm6/3+JCbli82DDmr6MFby/zOsDQMe0CKx7vqGcH+4LV9+6RCTOrZpRbRRMMr6ubrhiejtT1FmaTTRwxYh2v9I6+/CiLwib19b92g17NyObN965qsYYY1wwroFf1o3b5lBfuZLur3/bbTPm8eLIvPpoG4AAYBPBvGpPgADsVitG0BQPvBzyuAkUrmR8uoNsgkt4FQzaJIfCBoXCMWQrag4jBB4yUUJIJriMhFolAgCYc86Slyj9Wtl5dSUIAGCMWcWaiNr2UvI8ZqHDMQohIimiKEqTWEqORIKF1/byysM7/EoMuvkovnBfNgTDOuwb/jnnyMOKoOAsWes7rSTkDIA5D4YAGQNOHACYULHMvG+axtaVIxQqynpMUJVn6oL7o8MnVVWlWTwebV2/fns0THfH20mScGaPSVvbJDH3rolixQUROSIDYK2zRamXhVOCp2mcZiLL4iRWccIAaDQaIrbOmkCJiOMozeJeL+n3e7P55PD4ufW+aSsZCSYZoDUOiuJ81E8iFS2ni/lsnipJTjHfMk7IfFEtTo5Oj09OlMrSLFvWy6aqjW3n09m13R1jKYrT2bL4/OHj1x68dePG9a3dHe0sAXiAsigWy6VurTV+sSyzLFNxAlonWQ7AzHTqCY3zdd0WyzLNe3l/0B9tcU5xoiaTycXknHNKItzeGTtnjNHj7SHjMJ+VDLhzhbUmFjG0IfgEnHNP7osvP5VSCinyXvSNb341juOf/uVPPvzwwyQDFbuLi4v9wb1+b5imqXOuLMuOVZblaVGUxrgoiWfzZRwrZOzh0ye/8u2vJmk0GPZOzs56/WzcjNIs/slPfyy+/2c//Hu//m2tdZIkSSpny6mnndnkYn9/9/T0OE0ib602rTbNxx999OzZs/HuThTFs9msauox50IpZ0lILqQ0xnz/+9//afKzJIr/+T//r9/75jcuZpN/+z/8j8+ePQsFMPv7u3GsAg9Oysh7yzjGsUrT1CNL07hpGCffrSyhlpSIQjFCKFsIFo6IZrPZbDY7PT5eLBZEFISJVuwPpcqyLIoCEUM2azQaDYfDnZ2dw8OjEEUPa0EopqjbJpwi4BIhRC+K0jRt25bSCBGzLIsiJaVgDAIPLE3TKJLO6SiWWjfPnh2Px/nf+/Vf6/UGo9EojlVVF9Ya55xrTFVVTdMIIRB5qw1jkGe9NM0R0VpbFGUxXzZNG/o/kXPesaKtOAqGnHMlhWCIUqSCJ9bqptFNWxO0cSyjOLSN9VJGiMY7b9EhMGuICKUQxtSMQfAGghMS1l8ppXPOewcAjPtL6hgBrSkgRNRVPQTDSRv5hZC4wbU73tmkzaKGPM+jKAqx7jzPb9y4UVcTxljbtsZcih4aY5SKiZB7b21Y69cswhcZf1csxysRw+bqv2lr4SWgAACM8e55C1SYYGLdhnJz9y3nnFjllS990/CD934zokAbeGXTMoXvXFm1OxRy5Tqv/LUDFrQBNWijm9Hmzn7N9t88OxExfln1gPiSP/3iIFOngfqfvG2OQPchIhK9gmZxZQw3P7wCFbpLtdYGL3YTwlprGYpNELZpO18+YzcrjDFcB36643e0D0REXD8buOKvQHiQ1l5+PhgwFFwKHrKDwDwC9yilshYQQ0kCAhAiQ8actYwzvlLsAPAIHgHAGw8Abq2S1N2ylJJB6Mt2+QBwzlFIGSkZqUASIgzqxZ4xFhDJKhTTVT2sRc86oLCG/hTIi97BJu/VOYeAHsgDek8IHj0gB611FEVMCFiXtkapiuO4ms0BPYFxvvWu1Q2dnR/Pp+dCiN3tkXdaCc4FVXWVqKhpGsdYAEzek4pEEsVSCiKyukmzTEqhIiY4IbPWakCrlDg/X9ZNlSdxrMRkcn58cti0uj8YnZ4ez5YLS/7a9b18OLDeOucYmCwV/SyeXXhnKmIJETqv8zxWSiwWiyfPnk7nswdv3NjeHT89OAIAa/zR0fF4PGqremd/9+jk7JPPv3jw5psyiiaz+en5mSOvlKrbtixLbwA5a9tWqVhrXVQ1l3FVVYzLptFJku3u7DExqaqmLGutden1+++/f3Z2cuv29W//2rdu37y2t7fjnEmT5Oz04s///EfPD47Kpa6q2lofqWRraytU+UVRJIWYzSZpmr777rt72zs7u+O9vb3X799W/z1+73vff34Io1F6dHQ0Ho/zPBdcMVYLoYiIc9br92fz+bIshlujyeRcCLG/v/v06dNvfOutfj/PeumTgyd1Xe7t79y/f+/x0yciV41eHseiHWQi7Y+3t8eRSNv67Pyi2dkaR4pXFdvbuffJB0+ePHp87/a9a4MshvTW7s7ps58u5x7QtK3vDeLWxYsGnp7MdXO0vTU+Or14+52v/rP/0z+/uXPtX/yLf1EuyhvXbr5x983DJ8fLSRXzrFw0kpLZWVmXPkpT5Gy+bIQQ9WKRZVm/39da67LOVCyUXD1529uhGC/Lcy7Esigm06kQMo4TY0zbakSWplme94gIoBJCWmulVIzx2Wx+dnaOiGkmsywBgLIs26YV3EnB+5mSUgoBkYCVBArnUqacD3Wzvw5gyFWsTygCZqxtTTObTcu6YAJ7AzUc90fj/o3r46qqlouSCWGtmx9NlFLD8ba3y7Y1xlrGIkaiqVprfRRFH//kw+VyWc/ntm6BmEIUijtOGkhKBcCqsjbOJUnGIukYmNYY54wl57i1qPUq649IpkGrtQlNqQiE4ESka3B6xTpERMFJCl8UiyzlaZpGUWqtreu6rduwWuRJLOVKH8n7VSrBGgvON0Yjcg4YSW6Mcd4yAE/oV72u2arvC6KUUVBBqeu2LOs8z6WMrPUXF9Ob1we/9u1fuTg/JY/Pnh9sb+8iCV0XBiCO4ySKA5XaWlssy6Iw3rQh8+tX0jEEQATMW0ewKuVHhLBaA4BgvjOrL5uT8GHnDiJiqMsPZikwNAMwWjXZWitBdQ46MGICCSB08YmiCLkgZI02nHMpGeMSmQBkobzcOQOwagQAAAGnrQwPdTQCHyr5rbVu1YXwhbhOiL114RkA6LJyWnftsNF7F9xpa62UqypBIrJ21ZuYhY7hhERgQoDVkwckFIQOGGdcgdOevHPeEQHwCEzweT0yT+gJHTACNG2jkkQKbowBZ6VgQgggF6GbzOteHlvnG2MJmCfM+oPlcomIIW6FiAierUJW4pVAIWydB78xdxxCUCbUzDAGhN6TECLIoYZWBlWplay8A6Xi0KU9iqKAqY1x3pvgloWmSLDS9QIA4oI8eedd0EtGIGAITBAy4Ip55kmQt56IM0REDhXnyLlgjBFyzqVUkVCRcAwFk6gEKkARegJb5xy1CIAIoaNkyHUhoJCCQsmB90iCEYULQqoFoORccrHBHIKyriPJpQzECHQIFogBKWIKhQQODgCQc86BgyVkFGDSGqoCEnlrOXFvnV+9DtzD6ho8YIgi6IAPPDlPzvmaODogXEEWRsQIyfk8FQ4845D2Uuu9cbYhjJjob40uTi+SZPTmG988OT47Ojy0VZuP49nZ0XO3+LVfeW9bKHPa9LiRgOlW74n2iKC1aWrNiCURRFx6b53SvRiFtEL6OOF56utaM661rrK0qau5Zq6xvio4i4qEU5zLh4+fZmn/K2+/M97a4Sqqq+b4+LiczOZPnunkvM/54M7e2dnZ5GK2v7N/cXFx9+5ry2V5PnFpvj/Y2m+MLcqlqc9d2/YzPD16fOPGNbD1k0ef97Mkj5VgjGybCMa5RMZ84x2QFymXwktoysZ7GGYDBjyN0uVyyWV0dhIq73wSp4yxNFIxlEdffsFF/N1f/e0/+If/CBG1bQ+PH3348w+SBLcGyY2d7KCe9IcsiXqMbJSJN966MZmeVs00SZWSy/395M0HcPNmMhrEWeK/+e43dvso9fz501PnPCNBi5mRDIrFdprsX9+dzKba6fnklKGLYzWZXQyGW8RwWeujs8lnB89KsM7rG6/tY9Qasov58e//3m+JyWTy+OkTxtitm3eI4aOnT4z3t2/fPjs7Y0xEUqVpqrVuW8OYODk/G5G7d2/svGdSzKYLLgURXMwunj97Nl9M00xyphh3z54/fPL0+tbW1mt3b/3u7/zmv/u3/9609c7O9t7OnvfWe+/IA2fMg7HOFIVFYkKEhqqBYBiWtjhNkiQxxjRGK6X6/b6x1hizWCxC7T4TLEmSJEnCktp9dzqdhor8S4cPUQjR1FpwlSRJLx8wxlbsegdZmrG1EkDIbiAiAkrJoyjq9XpRlKzUssJ5GUxnk6JYBvJE2+o8z69du5Yk8Yp7KGW/30/TVLe2qgrG2Hg8ao2eXMyIKM8HZd08fvzYOeO9BfCIRBDSlmStJQgBLeCCAQDn6L3VujGtJY+CK+90aEJPRAAU7lQpEcdxkFZERgCQJFmw+k3TGBOk0BIpIykjRB4ijatIBhEAdpGVEJMHAKNtqBxB3MyGhu4MHoGtUYVf1ZF731WZBl2s+Xxe17UQot/vW917991333zzzf/4x38ajpZlmeByUVZhFWPIumbqw+EwGBiAoNL4y0LfVxzQTScbNhxNuOIxr7fOGnUPTDDMeDXh8upIANugT3ZIhVbx8xdM3eUZNz/vwg8bufAr28tXvukZd9SEK9GFlyMQtEFx6C5m0+Hurmr1RALROu7SbXGWIZDW2hvNkJzjzmprLSrWS4XWGpAJIeIkm07ny+mERzGjzezDK3z6v8PW3ew6ygVd6iHwFUJRX8BMm/Bx82nZfAAAOqqNBwBa6xV1YRhgFFqis9XE8dBzlZAz5Mg4IgdgKFYvi1/tGVScGHuRpNk9Kl2Mc2OIAFbYFDcDO8hXvAQhmWCIiF2OgDEmkHfxpJeH6+UXxAKtBMgJATwRWfLWkwPvyBsi51cEcwPgujfo8oBdKJIhAudccKWUArvq2E4t9Xq90Wgcy3gxn3368YeffPzzx48/eeft13qpdK49PT9aLqd7O3t3b9/b2tr6gwdv7+zsnJyc/Pt//4eff/55EkOvF1dVEcVJmiZCAqB1TldF0epKSsmljCIaDEZZ2rtz796gP2qtcx7ef/+DshwPRuO7d26mWd62hlwbxUyqLe/9ZDJRSu1s7YyHI/RQlmWa5m3bGmMGg4Gz1DTNYrGczubL5TKO49FoFMexlFFY2ZSKlsslVzKsq0opIWWI6PSywenpqbV2uDXiTM5mM1NVWa9f13XGmPfknEuSZHdvhzycn58ryKTwcZJXVbOcF1vbI8ZpPBo9VcjApAl74/Wb9+9dE6BMa+uifn76RVUcersEX3pS9+/t3rl7y9nF7VujYW9QV4bs4mtfuf1/+a//qx/94Mf/7t/9zwBbSZIMB7nzWkRcSc4ZNItqa2fbOEczb7x1zgU8nab5wfNHBPr84qhuFplj+3vXy2p6cnQglmX17s7e7buvff7lF+fnk2F/yKWoddvPB5FUdVnNZotPP/n85OSk1+8VReG8H21tT+azfm94fHrCueBSuosLROjlyf17t85OD72zn336l9d2kzt37twaXP+t7/7aj37wfQQaDXOta20NZxFyLpCBB2Nd1bbG2aBKHToBWGu5FB6IiFprmrYRQiBjXAhkTGu9LIumaZxzN3ev9fv9wFdYLpeBNti1PguOYDBggQXtvVcqieMsjuNQUOS9j+Ms5CCICIB5j6F+HoDyLA8KzWmaAcBisQjs/aap5vO51m2ra2PMeLy1vb0dShy9JwAMrN2wWARJ9lBSwQUqJeNEVU1d12VdLZtWG2c8uNAd2pHz4IGBBwdAyEEwwSU6slVjvQ1LA7bWLZdlSMdIyYlISikZRyGRc/IeGTAhY6lCfSmwlWDiWhfBGWeDtKFQkgkeViXjLgX+hBDoqTG6bGqlYkSHiIG/3a01QQQxaC+GcSYg610/TcIxtTWhY1Ycx3Ga9Hq9nZ2db37zm3/2vR8oEQWgw5mIoihYr16/NxgMFovFfLYIY7iydbjB3IbLSsWXLUe3Jm4uuPiqNARASM5eWtDOcIbUQ9ffofucr2+f1tyCsAUPdXPpDz+vcj2wyZwgCO2mNiIKQRfQORfYZJu3s2nsu/Pii/n7K8imq2twa2GM7ux+Y6PLboc8MEyvwBHYRCRwaeEQCMi3WoOzcSTTJFKcee/Ju/lk8eCN28+PjlvjdKOHw+Fg0APkbdsG1udqnMETIv/P0U2CaFXPBwDOUchshu4i3vs4vhS1pHUwJowPvPi0rOACrvpvYWiIABBscLdPyCMAwIpLxWLGGPIVSuCcM7FSXsJ1n1UPBJesx6vtxS+f6hcnOnz3UjwKwcNqvpjgDJALBCQkoBXUx9Bt9YXH+6XXoZvQ8LkL/WeJCD0QWA/OeeMsATrntLc26KGtAIRbESwYdtkKxICNAIFzhlEUWWuZlYwxzoWUUnKhItbvRdevXdvZhp0t9/AhZ9Agd8ZWHvXuzd3f+Pt//1d/5dcGu7tn88XOrVvXTwdffvHzg6efertMomzQ789mE9Nqhkop4Zwma5kHTricV957cJDG8c54+9atO3GSyDh57e69n3/wgTZmZ9yzHoDo2v5od28wn7Ra6/OziW51WzdSqiztWWuFUHXdWmt3tvcAwDlf1wsiGo3G4/G4PxwAhLauXlvrvS+KUkSR966pNRCjdRua2WwWnM+zk1NElCpmjC1m05CaMVpXVaGUSuNEStk2dXlRxElfqfjP/uMPimXzm7/1nSwXdXX+3lfeWC5O5nPbZt407eTk4tnTp4cHR72BPTo4JvSM42zaxrz6+tde29raQrNMo37EONliPMp//dfeubadDHL3/f94tlgs5otTBMrTLEQqV8+6R2s9oSdCRixM4snJE86xKE7jmO9s9/auDT795IvHj4XYu3G9rKut7e37Dx6kR0enZ2flshqNRkVVBkmKYr6YT+bT2eLmrTtFWYso9giTixkToiyrRlulFGOwvTVSSqRxenHh2nrxyScXe9upEI0sWyXw2t7QOua8/tGPflSWS6EoTnL0zGvTyQgSkfO2aVrOeZTEQgjtbNU2vq6KohgMh8HrcuS5FKPRKAj+HB4eBq2PsD6GtqeMsc8//3yliLCmHYTlQ4pEt6VufZatFnchRBCgQADvXaut9yYkgYJURZd0h3XXGWNMUSzC6lOWZZ7n7733tW988z0p+fn5F1mW9XqDpm2rskFEKaM0zeu6rqoqmNW6rmez2Xyx0LpZlIUxxjgDyBAZeSBGHgEAHZD3zpFjKDxCgH7cRYyhtb6pdVVq512eJ/1+P8uSdRqbrHFEJKJIyaR1AEwILmNguJb1tdZaR8ZZzklKCchXMrTeg2nYmlHonEdEa70PkM0jkSdysGEzLFCwMZxz75y2VimV9/syilpjQqVD0zShv1RU1/P5/NNPP93e3r59+/Z0Pgs2sZf3uYpms5kU0fXr18fj8cXFxWeffn58fAxr0+vWq2RY94jWntArLceLi+OVv25+2HVVvgIUcN0du1usw9btdnlhLvTbXNW//SJn7sr2MlAIWfCAhzbvokMksOERdnfBN5oEbuaqO5ZJByw2b7Pj5a2SFhtW6gq6Wl3Axg0x8h5ZJLg34JHI22rZzrRjAErCr379zT/4gz/48JNPHj568pc//3Q+uai1GwyHfN2OOUAFhoSIHtnfFipcXtJLWzh4eFvbtg1AIXR1x42oQ5eR6Q7YDQIihltljHXWfJWYf3FYMJAUEIkJYJwxzoXgXHIphFDARKh48AjQNetmxBC9fQWP8mWUcPknxgGRWGAsQAAEGAQSEBBgRa9doxMGbDO41c14eD67s+A63GXW/Y88rFZXY6zxjhCcJe3WMkuWrCdHQJ7W9T0ExLq2K94DMGKMSRlFkQe2oscKJj02dbXUzcl4GN24noxHD958c/Doy4cc+O3bt+/RbfL49lfeHVzbgoiPIIW24Ghff+3W44c7n332GVB569atJGJFuWgbiiLpnCPrGDHTmF6SM8aatvXGX5xeSCZH2+Ptsbp5ff/g4GA2mxldV3XjnBvu7Y1GowO/BIBYxkeHx5OzCyFUnKWD3nAymXnvAXkcJ0IIY4xzxLm0HobDYZwmIXxrTOu9t9ZLuWJ3OuettcDQOQJgpm3SNDUtTc7PjPO7u7tRFLVtKxhSKDEj0k1VFYvBaLS9vYXGLRYLo/3jR58/P3x2+OyL0VDt7meDAddmylFnqUIi5+ZJbPb2U+6Xk3nDOdy5tz+bzT76+ZHEP/nVX/3V3cF2NS3ytCcEN9XSGf/arb3X/o//+zz5yx/+4C8+//xLGcVxhG1dOGuSONa1BqJYxsAZRxYMXCOkF7Pd3Z3RMOHCX9sfpgqMmT99WglLNF0sxm0zGo3iKO3lg8PDQ+eoqVoGPEtyQpjP5yqJ8/5QXUyFlAfPnz98/IgJqa0vy7ppNGPAEcmbVCFD5ByXUzh49mg4TLKGFotiOIj6gx1tiu//4E+aFhcnkwdvvRv4MsY75EwxwRiiJ66kECJKExVFrTXaGO8947xq6qyXJ1naNI1HyJN8JKW1Nrt7P7zqRVGcnJy0bRuq8vr9fjBRQTeQ1upMDGPvvTHOGLdeJ4EIp9N5eJcCtoiiKI7TJMm6BY4xFsdxlmWLxaJpml6vx2ssS++c07rtyrWr0kaKpUkaKeUsd855xwAwUplzTpu6WFbn5+cXFxehjWwsFRBjKGAVYA+sY48QahHJWeI8uIbgnLfaCYGeCDmXUSQhSrNemvWSNNW6McY4F3KOIAg8YNs2nTgV4mZmwXfrpve+qqogmz3IhyGM1rZt26zU/ZAJ6yhwqmC97gEhAKhIhahGlmUBNkkp9/f3Qw4iZGq6AI/W+uTkxHs/Ho/fe++9/+6////2egMppdY6ErJt27pqJ5MJ57yua8RVKXyIJJO/jBW/0vyvfnjJ8nR295W2ENYZjU07BBsGtduvE8jbPKBf13DaddfKTauzPvsLHl4XXXjl5jcIa1d2u/JruAAh+GZm5NLgvdhlu/uK3+gNuHmRm2GG1RAhAKzCXLhx2YjIAsQhx5AEgnNOIdy8MX799dffe/uN3/jOrz948ODp8+dbW398Opn//P2PjG6QiZWG6GqaGL3K2P+1W2fhNscBXtTwDlA4lHLY9daxOvyatdrdfjd04cf1Fb742LxYHtl9xTOOjDEuhIhkpKSIUKxCCz50eAXw4BA4IvpL8uBlfKJDpd3pNqc7cB5xhavAIxCRh1WkKrSsDHbbIzigmF9yU7pHFH7x82+RuXW1xSqK4KyxHhGNdyasE0B+3f+cgSRaES0BCSCwKZBWmtWMMZIy8oRh1bW6FQrIF9PZ02LZ1HWsJHpXDwbMan7t2rUbt16TIsp6Pe1tO1v0hluuroUQb731ltbNcNhHRrdu3RKCffnll6enp6HQOnSeq+t6MV0kSWKNWc6LYlGen17cuXOHA9/a3h3mvX7Wywf9k5OT49OT48OT0+MzsLmUKhZRmiSzybypWikjUNA0TRQlgksiAmBZ1suyXt22y2XNOW9qXVW1JyeEkh40syHBtJ4mJIIgvXbrxvbR0VHgGMZxzBCqqiJvyVtjjOR80Mutd+fn50KI3d3d5FbyyccfG2vTNFnMJn/2p/9hayT/wXff2xndHI7UoN/jjA4PDy/OHk0uZgDMNy6PWNbLf+Vr3+6Ptv6Hf/2vP37/wNTizftfn7gq3usneQZMO1s48P3x1j/+x79/8+bWn/7JD54fnizLcrFsnGeIgoiUjPO8Z5yz2ja6BQAkHMR2f2d32I8/+fT9timVhGs7o48+/lIMR+M0TYuyrsomEnJnayeN0o8++kihbKp2MExmi+V8WcZJlvd7d9+4z5g/OT8p6ipO0tHWdhTnxlhvXaR6w0EyyKPdra2ymBw++wKJLeeF6c2//OxDRu7NB3eRUVWVKLPDk/r6ncp6pq3V1gBA8Nm8typOIDDnrTXWeu+VUjJSRVEIIeIs1c4uppOzi/Pg8R9NloiYpiljLJAYtra2AGA+n4dlIhgq51yIQACJriWgc0ZrV1WeiObzeWAnAADnXCnBOSq1UgcK1H3GWJ7nVVUtl0vnjDZNkiT37t2bTi+ePHkC6O/evd1UdFheHLOpjFSIQVVVUTZ1r9dLkkhbWyxb3VKkcm18Vc5iJQAhxDmdI2Nc21qtLWehaQI4S857QB8SrrpplSIhhIySFDkRAZONNkV1Ya0l8F2ssm6No7qsGr8mDYTCkNC5otUVACADZIIhcaGQCaWUitKgBdQaZ3wTBBmRobYhk0qM8c1FFjkLKQbkDICABdU2b3WrrbHeAUMVR0qpNE2TNCVyw+Fwe3t7ONj6v/8//iURSimLZXn91m3nXNuYJ0+eHB0dIaKS0c2bN4OkY1VVsFq0vPfeukvT+7L5vIInaMMjfwklwFpUcV0UsFZa7ML4m/YJNmIGV1beLuDU2dqNS/Ih1R92QAbeeyFE9yGs09dEFIQp4VXYpbMrsDbtRIQbrH627iDVcdRpnVx42WcNy3q3/yuv/HIMgeGqO1SwTN7oxlmtOIslAy4H/fy3vvvd/+IPfu/86Fk5n+zvjF67d+fevXufP3yyWCyOzi7CkRE2WAIr0Pm33vDFLNJmjIHoBQDnnAu9OQJs3dDQXHUv29zgctG/alAR0W980sFBxhgXMSJyIYRUSkZcKQylkisTjiu/HzwAg7XO4+bMbkYUXvUnWAECRGDovTfOocNMJowj44AE3jtPpI0hgCyJ6KUNNupvO6QbNgur5yQImDgCb7z3joAFdBA4neRhpeF4ubGQJvPeexfcHEDkjCHnxLkIvGMkB143zeT8/FHbHJyfQ6aU0VguSbCRdzga34BsAFKqpmJRY0pgPJMKBsPk69/89Tff/pqQGMeqbevxzt6jR48ePXp0fHzcHw63traLonjj3l2l1HQ6ffLkyenZxfPHB/Pp4vxs+vY7Xylm9WBr1IuHdeqq2JZNXS4rahZaaxmpmEeDft9ox4BXZZmoJIpTxlhrLBEJFSulZGQET4wxy7JwzgEgF1JKklIyXJFwGRMhaB8MkJQyiiIksEb3t8db2zvn5+dffPEFE1IYk6ZpFMWuqhazqWAYK4lMjXfGs8nFeVv0MvXO22/cu7N1bT+dnx/0bg7Gw6HgNJ+AkpAkKLiywvzKt769tb1z7+5rUdK7eeO15wfnz54e//Effv/NN9/YG18PugNSyqYu5fRiuH3nd3/vOzu7wz/73l/88Ic/ratlFPWsc8gcekBiTuvKNsvl0oNzzqkMB73x2289mE7OJicXHOnB668fPz8W3/nub3z44YfPD56/9eBtgezhw4exitI4W8yWs/ksjtKjo6PZYnH95k3j/LXrN87OD2aLuYxE0zT93nYv35rPl7PJNInzOzfujoZ5nrD55GQ8GJyePEcnpufPp5MTAnn3zo3nR/PX7t+dLe3WdjlbLoRMgZgjdM4gQujI7om01shYElYWzoAhec+VnC7mHkHFkVDy8Pioqqo4jkXrQ2fCYMirqirL0hgTBMPDChIWiPC26NYyJqzVwYcORURaayFECKpHUZTneb/fz/NUBqqxECFi4b0PYXat9eHhgVR8Z2f79p3rN29ev7g4X6mdWzo4ODg5OeFS9Hq90N/FeOMdn1zMgWEc5zdujKqq0gfk7HK5aFdvsw+FhU631lpP5JWMOJcAjDy3BkMtt27JGs855wLDa0mEWtugp0G0UjxljDkH1vq21YwF2TUyxiIGhpfDVYg7CLAAIkuSOM/zttbBA2sabe26H5JHxkIyu2vosHJZjNFd8AAAOOdN0zx+/DgwOkMsIazO4bCn87PBYPDZZ589eOOtW7euGeM45wy5Umo0GunWNto45wKwCLCss3+d4V///AI/8ZfEG1781uXOuCG50y2jmx4ebDjffi18dCV2vTKBG3jF+xe4+ptAIaR1cKW78ELqYbOooTNInQN95co3b6dLkYT9wzi7tXzFZm6CNvgKm5PYoYpXGsgrUAzBE0AcSXRacAbeFYXxelos50ju2t7O9//se1vj7Zu376S9UZbEuqnqslaxonUb3815/LthhZcn168LdLvbDFv3WCJioCVt7ryJElbDy1Zj1cVjVimn8MAwBkCIxNfpKB5FsOI9SCYl4xIA3IqqyYAhArjV+22JiL+I/1626LCBEmDFFkZatxHxQNZaZGSUQmSMs0AXCBAkLE3rN/TyUPAiUNiMWnm67EPtgZH3gSKF4Ml79IieCAg8oQMkROBAAJcjhuTRo1cqDiU9DDlDz5jjqw5V2DblcnG6LI6sOZaCMa+8jSK1Nehvb4+vgcihQlDQGqGt7KVjiCVY7efnUmHW20LmjallnN5/IHb2b8R5XrU2SpK019eO/uMf/WGSJNb4ptGEXAjpNS1m5R/++z8ui2Z7d29v95oliKNkf7zjhu7g0bOzyRkK3uv1mGfgjGm1dSSj0GaICEFwVRQFADhHUqjAUQvyAwAgZZSmSOQ4k0AOwHjvnQ75H1pUk36etlvD50cn1trd7XGvlxVFobUOj0FdFUZrTz7o+jjye3t7cS1uXN/5xtfe/i9//7vjEf/5z753sixsw+tiCuBNW3F0SL6t67fevvfWV+6mSX+6OH/28QcXk5Osny+Xxfd/+Kfa1vdfv+to4E3D0bemTBCKxUm+vfvO269HUVQ3+vnRH1Xl0kFkbE3AtTYMuOLIGNONror69NA8/uLwtbv3/sFv/M7/9G/+u+fPDm/fvPX6/fuiqKr9vetWuziOnTZpnDAmvvj0s0VZTGeL0XinrZsgyH8+uUjzbLGcXVycJUnv/HwiZJZG/bpuT08nAsXta7dsSsmo16jlsL89OT0jzw+efWl17cFvj0aHJ7P79+89en7xBk8PDi8yHnMp0aJzzjEk4kII61e5HyEE8lXupGqbKIpOT0+rpn7w4MHdu3fruj44OHDO9dMsNB/DNXM4JJOCBFNYF7pahuArq0hYawmcVHw8HoeY+YpVx3me56PRqNfrKaUQkaEIlX6TySycKHjnWZYho8lk4klfu7YXOBAnJye3rt8/PZkWy7ZqJrNkmabJYDAYjrZHw+2Hj75w3r/19rXBYPDxxx8/eXJwcnLS74dOmBwAgJhzZE3Ie3kAFTGOQWTXB1r7Sl4iuEdRFEXxSr8ycIiaRgO0ALCupPLWuzSOAotTa62tCYHZPM+ttYCoranrOkTzkDPXOu8h0A+JVtjZk1dKBWebMUZ0yYlz4KMkttaWZck5T5KkLMvD46PhcMilUPFKEhsQHXnrnW7bR48eXVxc7O7s379//5NPPquqqpf35/M555xWjfjQOXdxcXF0dBQyfFrr4GqvJ/rSkF8xGK8M7G9ChO7DsNsVBcZNS3NpPEL36rXP7Te4iptHu2KoYA1SiS71DcMVhvOCdx1QAE8r6/4S+nmZGHHFQIasfPCbYd2yCzYUJDetxeYFI7LujuBF8LG5eVpVaGwObMjopZGII2Vq4x2YttZNfXJ+8t/+q3+V9Xq37r727te+6QC01oN+VjYtbFzMxg2+PI2/bMNX8U5gg8OxGTsJQxGYzmGiQ8ToZYiwefxubLvRYIz5lcscejytIk+MMSkiCpRVLhmKDmQAriBBGHFCIE/ee7XuitIdfNNsv+pi1o8lW42bI88ctG1LSjAuuwhiCIUFn4dvyNbBi0+1fzGHslI3W/MpkQg8QhhMj+A9EiAQI3Qh1YCAuOI/BqDpvWdMcM6JIQKDtbDE6tEFr3XjnU5jrnq9LPfMU7U0kuPW1tZ4vAcQl0Utc8VFyomTlVijs+R8ZL22jUNmAHE5X/QH+dbOzvl0+tOffTAvCmLs5PhsPNrinGvjsqyXJFlr7HxRLaaLNB9oibp2pyfTtjV5f+idsNYncWytNXWNHqeLeVk1cZYKoYyxy7JojYuTTCllvNPaeu+TOM3znIlOeA0ipZIkqet6xV8m5pyzzoXHIOLCWhvH8f7udhTHVVXt7Oz8/u//7gcffFTX9dnFRVEUwXwoyau6QCmm0wsp+P/2f/e/+Ue//1vjgXr25IPz08O2LqaTFqBiHNq6ybPMOayr9ujk0fd+WCVxD1DNZ1V/K9m/8e6TJ88Wi8XJ6cGXDz8kfyvPFGe+bZfEI4ZDs5zKKHvrV745n1V//Ed/dvj8SEU9bVDIWIgo7qUqjoRgF7MpY+zooP7xn//0q+++/du//90//MN/9+HHR3duHe3u7orljNcNqXhb+yjp5ztJdnT8HAdxXc9abj5++CkCF0IcPz87P5ke3jxeLCdF2ZqYCyY5IRdUN/Otncxx/fmzz+P+281xc3J41Mt7JwUsvbfTuNHwO7/zewdz+/Gjk2y8MzYM5CIb9j/74tHOcD8bDBcLgYJbR0neU8Ssg6KqkPP+aIACJ5PJfDljJUuzPMlyD7w/2HntPmtadnJ62jJfg22bdnu09fZXvvLG/Td00z5//vzs+KxYLgFAcVFbH3r6Nk3TCIgdpHGytXMjpJGQwBiTZJIDMQ6cs6qeW1NIJTjnbF2uGTSSAUBKqZSitoyipD/IGWPHx8dhGTo7nfzFjz5WShliRWXrtqo1GOKeKevnR8eLfr8/udB1W3lMjVXTmZMx297ellIuFgvvaTAYOGsvzqdcovdgjFVKSa6IyGpntHZNjcAFYwjeW21bQo/IuW69d5xAagOIAkCtChkI0AvJYmDgDVrbgAfJhDeIJIiobS05FsuMASsXTUhfGUs21GETc95prYGxNFVJmnaxmaZp6rqWXCEwJOTIrbalLcHTqDcUKLhHyQVn3JHzznFDjPu6jAWLj4v5xcX0H/3B7x49fUixkFLNpmWa91WSNdoUdWGNsWQsWnJIAmUauQbatvXOM8akVN5bBLdidiEAOOxW2rVlxg3v7cpy3P013M7m0hxWBLfm2EIw6gAAQYsGvGNCYMgzGO2EEFJxay1jwLnkHImctWZtD2LGQKwHjTbaMvG12+c9uTVocM5sepaIq3gvoghck87UBYhQtx4RuRQAoG1QIzTOu+A0C86BofXO25XYH1PR+va5914b01jrCbVZlc8HAj0AIYX6/oSInPeGPBASELLQuZjaBtC788ny9q3RH/zuP/zmN967/vbrg2qf7Qyenc312cnH/+Zf11V7OiniNOZMAjAPgRJLBG4laPSyedww/JvbaliYCQb78nMAIAggCSC0JwCtrXOEqOMk0cZxofLewHmoG20dJUnStEZKCRhY+1xIFXwAZ4k8kIMwCB44kPUEXCUhTe+JOBIKAUIg5w6dilQQaXbkkUKPEuQcgRgQ8oCHHCCEWMoqsc34ylo774kcIg/VicEMA64UlxUXAICE6AmZCO2iySNZZwkaT1JxwTgieus0tXmUL4qqbl0axUpKMlayFTMJiBEwAvJANvSQBtJ2xTr04TLX7EXdWr9+eyBcmfcI4JgFRAS+6pECBMCBrHZasIhxJE8kUPKIGLZtW4FjUdbWaUJ3/TKbHp/vbg8Sq0fD7Vja0/MP9uVCV+3kqE6iNEkSPfbVorYGlMyyNAOS1jJn2fXtu4wBA/b67bd/7zfNZHqeplEURcycWGuljKI4ffTk4Pjk4u4bt6ez4vDoVPWkAQdcy2HiJZ3Xc86llLEajr/8+GMxK4qqcs41xyc3bt3Ossx4hpITQweUJJkQ5mIyqarC2CaOYyBSkinFFWeMoUgTIqrKenZ2zjkfj8dCiOVsmeyM6ro8OzvZ29s1Rn/w/o+2trb+6T/9p7dvbP/L/+f/yzfLu9d3T47PqGkFl0VRqH5/Wl70B9mTwyd/+fHP8pR/8MGPf/LFs53twfFhaR89I9cqjmrVq0/YMtFT9vT5w63BVr/ff/OdNxpr2nJpfFnQ/Icf/CDewrd27n72wfsXh4fvvvXmcFTcvv+grapIwLtfed0zN9oenF3UQuatA44AFsjyrL+jkqG1tpdExweTjz98PBwO98d3v1SPDw9mr732VRFJ/qM//9mymH/961+7ef0dxrEqFndv3R72+ufnk+l0enh4HEfJnTv3OJfeWMZYL+uneVY1q07Qu7u7g8EgjmPdNHVdL01bliVnGMoXuTbb23s3bt49Pbn49NPPARWBSOKelPiVd7+2vbt3eHRyPpmnKkpiJZScnp23bbu1tfX2V97Z29s5Pz/3QNY7IYRQsXP2/PycPBKyGzdv7u7tFYsTf07WO5UlWb+X9nt5v8cEj9PMtJoDaq2nF7PVxTgrBTLGuiRC27bkvNZaSAbOerLeOwID3gXtYBmJbmlmHJylwFEAgCDeEJLooTmC1toa1u/38yTNk5RztMEqOO+dk4LFkTS2jay8trv31Xfe5UAkaDjsSxkBQNu2giMQj2IJ67o8ct567Rxp0xprkAjQERD5y2AjIQ/OWZAJ8mSNQYEghPCe2rYNOKbzUMO5ujxrWA/Wtylo3VmgK1ho25aIQlFJiFUEq2OtdRsLejBd3rq2bdM0bVsNAGmapmnKoyhcahSzqpn2e2mrl/3B3bfeev2zz76Yz6fes6ouq6a1johhkiRMph7c+flkTTRZhRCstUa7tYFfuVkbjual7/tKL/yVWxdO6H7tEIPfYCTAi1515xp2fvkmrWHDcYTNg7C1mh4ivHx5r/wZXyJRvrxPdxlX7uvKFmIPm9HprvzvlcfsDsIIPAJDDD5lU5T7N/bAu+P6vCzLN9544/r16xdnZ1wJwaUxMJlMtHYIMlz/ygcNY4Me6BfGBv5uWxdReOGaGQvvZlVVk8mkbdsoirIsCyHGl/MORMQ5d+RoIx2Dazl5WM9+yFp0pw7j6VdSRoDrnrdESCtiR/gKEBH4rrJmNa0baaxXRMLCm8vXuUZEHq5NEwnhQ2wDuVs/Ib6hKqxUgAqRgMiSZh4Q0WPgGdCqpQMQhQbZYT+ilW46raIOHYrzG8G2XzL+3b10EJxznveHs7O6rVtqa1+Vumnktd3da/vGQ1mWR0fPGY85i5C5VpdtvazKGoAxVAiNNRdN7QCE4PLatRuDQQ+H+b66vlieCwmcMy6QtbG1vmnNycnJ6fFhHOfX9na0tlrrZbUEFHl/y3solsskzcfjnY8+/GQ2m52dnWVZ7mGFL+u6fvDgwcnx2Wy5CGoKxqxKmbK0Hxa90Lc9RDdDwDuKouA0GmNCa1zGGHg36PWNbpNYKckF31ZKXZyd7u7uvvH6azdv3v7ZX/38yaPHe3vXzs8m4/G4Ba5U3NT6Z3/18+fPHuWZqKvZfLK0bTMapAiybVtNGslLzpIk6ceCc7Z/bUcQU5I/e/xotLu9New/Pz1czuaS4fNnx9LB0ydH89Mz5uTXv73flCWBKufTi0nFwTVNYXRLwIFFgklE73y7QsCSjUfXelX6F3/xYw/1/s3xV979ehTz5aIW//FP/ufvfe97vV4vTVWvn16/fp1zfufOvab5dGsLkiQzxhm9WkcYh+CZ6MbkaXrt2vU0TRE4eWjrxmlTVU05n50cHy1nScSj69f2t5Lbg/5w0N/6y5987/xiNhzsGmuK6qJuzGA0TpPe7duxjFLrXVmWWuskTy05rkSWJUkaASMAr5RojM6TvuCKEFrTCKFUJLlgRSNACRFHUZZGWSrTOBISBBdxorgQnE+nU2I8ZIkM+JghY4x80Lo3iOiM1VpLxdE7T5bIA1okjwwQkZtLlgoA1HU7n8/Lsuz3hnEcE0FXBBFFkXME1qHzSZYEfuVsNmtNQy62Te2dMbo5Oz48R7x9986gnw0H/agfbW1tCa4Ex7IsgZjT1iUxAENiREQenQtajNZbk0bJylQgMIaBNACceQdRFHEprHUBFQH4IOIeosTdq9shhi4XE9bEwMYIrQBggwfOGJNSJkkSRiAMWvgwy7LTs7NQNxLWZSICRitiBwGu6yy6dO+tO9uMwb17N9Icl8U5cj3e7r355puPHx01LRZl07aaGBJ6TtzDJb0uYJqATuxl8P9ySQ0KOcH80cbS9rLlvvKnK3Zx89438cE6ZrvCQ6GuenV6DKYXuoJ7ok27Dn5Da2i9KHu4BAqXTPgrV7K5EG/CF1ijmU0DeSWwfAU6hJ9DM5ROgMitmyNsDs76B0JEcj7Q8QCRA66HDACgLkqj20FP7u/sJnEsGd/e2/ne93/YNNo5WC61biFJGEMgv64uJAYbZSkvR/7/ztsmS4PWbcEZY9Y5RAzhwDRNg757N00dObRDcpu9RboHrzv+6o3Ay4ngjIX3qOP5M8aECJRJvDS6xGCVRVqRYBjDbuI6DuaVUwOAMYGeHCadd48BrPMFANpz3uVbwBrOueDMkpEB1xEaIoYMPPNAoUO09+BXxQ6se1Tc+tl2RB7DExaqO9eEhl88Xd2D181CGP+zg/MnDx8++vxxrtp+RFLGwKRUCRAUrT2fTD17JniMHgUysg56ob17U5V6Piva1qVJPhxuCc7qapkvZ2fnJz//q58+PzyQkisl6tnjXm/Q6/etBc75zs5OkiQHBwdVuXSGjNPOuTTpCxVXy8ViOnn06DHRSp4uz7Lt7e22bXuDIQCEyHFY6wLiT9M0zPXmItAl+0LxfJZlRVEEnpxSqlrO4q0tiWCb2nsbC1EuFz/7yx+/9957t67t39jf+4CIkxdAXrdNsXTJMI5jBrZcLNuyHPSjSGGsesw7XUO5XOq2GQxSIpou67Kqk11WLKeJSpbL2jbt2cX0bfbVdNB7cOf+zz/92GtIRZ7Ho1v7r1/rX+/3esuiOj49ybOt+eJisdR717aq1kZRsqwcIHHluXCAQEhCcBnFy+V0a2frsy/eP5/Mvv4rX7VWz5ezLB2KP//hnykJ77z9+sX5yb/9n/7H119/vSzLLMvOTy/quh6NRl9996uIqFs7nU45Qp6mQIxJ0ct6eZJ6YAzRGIMiNC8RSqlBf5Qm0Xg4unP7Nrbl1tZ2o/37P//o4nwOEE/n9bODo6qxUsXTRfHuV9977fX7k8ms0YfMQz8RbVvPFhfvf/jz/GlalmXRlDxi13f2b925nWW9umoJuPdQlqUxrvb9KEmcc3me80g6IE2uMRok50opqSLdJnkGnDVN0xi9Whes09puLscUQq6MITJkKBhywRhjQgZemHGWAMBb44zWTUuZNVoLzpWQXrigzR5JFaexlIKss6323i6Xc+dMnibZeJjFcZ5GVdMU5byuRoN+78Ebd2pnQmQiTWMO5D3UWIuGMwiGipED74zzoeOGFrB6apngnMdKCaUUCmmMUUpwoRiz1pqglUBECC4Yg7AUdqzvXq8XDEbHeO9Mjl8zxu26I1eo+wgKmBjEmowJVSRhN9ro/wSeQnAlUVEURQBQVZV3Ls/zwWDw9jt3hqP+vbs3k1QwoFaXd+7e/NpXv6XbH86X2hM2rSmbutE1cAC2aSAv6QIIfNPer40cwAZ0eKUDtLkWdyjhlX5SBxT8ujh2bYa7ddN36QxPIIRAvPSlVmyEyxD6Bq39Esp0IOaS7XjFYG9eamfSup9po6vky4s1bbS97g7L1ipAYRIDRWtTdKHbwoBT4FSssQIDDD7ocDhomqqpzf3Xrjlj/+IHP/zw5z9788Hr//F7P5hNC84AGTPogZDzkJcRAC+oFxARoAf6he2k/1bb5oht8ksAMTiC4YlN07SbXHyxiuQKtOoiLrTOCnWzwPCyP3WghiilQlHOykZyHrRNrV+HJYAjXAYiukBG9wC7dYPpF7EaAK0UzQEEgAXGGQ85DHRAWmvnuODYMRU8cwDgSFurLUdEZIBA62cDkIgseee9JSAi62V3496D6ZSpWNBQQgcEnYjoLw4A4fqWcM2rCJtgCj1nKJ3VmEbI8fHTo0++fJwOB7WGtH+xNZopFY96/Rv714bDwbOLaeh5rbVtWte2hqBCLpDDslzICT89Oz48PjyfnAWm+eNPPur1Brfu3N7fuz7c2h4Oh02rnzx5ImQ83ho1rZvOiyz2u9tbJydnP/+rn9ZGZFkWfJ4w5qFN8Ww2K4rCGNPv9weDQVFUVVUlnDdtE0BAiButM1wQ6reDaG+YrLV35JWQsZJCMueYlHwx14fPn33l3bdv3bz+5RdflMVi0O95a+7cvjmZTJaLkqxTEVc87mcqi6RuF/PJcmc8LJpqPmuEAOrFWmuh8jv37g2jC7LOVDqwv3d397/6la/2xuNpWXz6+UOv+Xjr+r2778A1L5ElMjqpjpSMs96w1YskHf2v/5f/6OD5mbb8hz/8ybxsi9p5xgGlBeRCxnE8qcmT7fX7yJk2Luv1T87Oh4Ntwbi+c+feO+8++PjjTx8/fnxyclLXdVFUbdsuFkshZL8/uHXrFiJ+9NFHp6enuoVev4+Ei3kxm35aNW2apru7u2VZVsWymM3bpk4iZVtbVOX56XnM6bvf/c2yas4uJkcn0+PzGYFojGFCamsePXkS59nt23dr3SZpFMVSMLMndpu2sq5dVj5O1F5/B0PUpd9Dzs1Sa20ROQpMomgv3suyzDm3nC+WZblYLOI4bo222hljFBdaa65khBDmdbW441pejXNyXkrJBYaIgvcuMIiDmZTCITkgFIwEl5IhuYwjplHsjQYVDXq9WKm2MWRdkiR53jPetVq3tnXO1XUJDCxpY9valImTUhFrqWoXo+3e219585NPP7Omreva6pYLxj015MlbYx0iR2TOOW+N1ZXVlW3tvGo453GcqDhSkWAcjDGmbQJpAFZmnqy1YZGS/FJ9JZi6lbTqOm1xJfLMVkV6q/57uBYzCJXoWZYJIZqmadt2uVzO5/P+YBBI9h3aQAJEtNa2tGow07YtAiRJAgCHh4eL5aRYTr7xzXdv3bxRVWUcFR9++H4I6IW3EQCs1QhMClHXLawsou9sdrA9sGHmac2f3zST8Cq4sLnor83g1W91e3ZraIechFBw1cyvxja0kVy7Uy8YrU2XccNF6066Ilh093IF0FwxXZ0hfBEn0SZKuPJJ92EIuRORMbqu6yBz7l8q+Lwck4BgVrUAAAihaUBVVf1e1lazSMg4ElEUHTx79lc//cvnx9O69mmqAHnb1M4RY8KuqA8vTAQygksT9Dfdrszm5dFeJTIBG4mVplm9I8E8bEZoNr9orYW1nuYqfLW24vhi6iFsfK1bwPlGQ5BXRUroVRLjL07fpohFV5zCiSh0l3NOkPASJAa04sl4Z611jEsJKgrt7QUSOCDjLDOMr3wD7tddUhyQJ7LkvQcPFJqe2nX/J+9XKQmOzAN0txzqIukX16h0IZxL9Ow9EfV7w9t37yWK5pMniTL9vqrqeTOftRBPysXx4vz5WclRbPUG02k5Gg6/fD5ljEVRlGW5lNICq4tqVlSTxVwqPhoNoix98O47990DIYS1enecHh4eTuaVSioVp9NlWdUGkCdpng9HiUUXWgQHVilj165dG4/HdV1zLqqmIaKyLI3z4/E4TVOPkGVZlmXLZVkUBSAGukrAE+GpCICgi8+FwOrqCUGUImqapm2NUhkhAiAAs9a3rbFm/uWXj87OLqxx5LUQsZRRP8q5QCQvEJEYJxaxuJ8MltPCulawOM+y5VIvFsW7X3vnD/6Lf/zmHTGfzj786fuJUMO8f//ea6OdXUuQzOe68otqUiycc6mUTDCJjO/t9pqm4TxTgvK8/xvf+c50Vnhi4/Hos4dPvnx0sKxbYsoCJ86FUG9vv3l6drS9Mzw+Pvzkk09+5Ve//sH7Hz1+/FT81m/+hrWWvLl5Y6+fZ0rFx8enVVGSo2F/EEl18PRZVZS7u7umbYvFYrm03hEAG4yGcZyGllZATAnZ27uWxmp6dsbRG93EcbqcT+dVmfezs9Pp1vaIgM2XVRTnQqXGup29/bo1nszF7Bw5kxHT2llvs0E6EH3nDCIBw7Zti2Lx7PnT+WIhZVTWzYpyRUwpxVe9xczp6SkSOW2vX7sWq8ihb8pKAyKiZByYd5wHnWZEJOeFWHnY4MkY47wBh84TAHlarV9EZEwLAJxzKbmU3BgQgnGO2jTGGAA2Gm2lSY5QC6HyvKe9IyJDFgC4YlGeeHAO/ayclu2SlxDHMTF3Mjkm4Ue7Q0BXN8ViUXjnlIwAwHlDXiNjCMQxLGckOcuSOFbOaIzjuN/v9wb9KIqMsYvFoq3r0MHFesdwVc8ZZCRbS13GIbhQXSeLsELx9RYKQ+bzIvASuoRLlmV5nnPOlVIBH4SiEsZYr9frDhIsEABwXHXQqIuy69nBGAv1qAcH50LwPJNxrLaG49lszjA6Ppp6K6uGtNEEDhmR84iA7HJt7Y5PRCGfEhaoX2JINg3/KxHD5ocvm9jNPTv+gVLxpvO0+hlD5eTqiQqopTMD3hPSZWhkJZ9JK/AAAAC+Y2JeMdVXEA+ugwovI5srKAEvcyUvfBguzzkXuiGEmNAmMti4d4K1pgxbJR2QaJU6ccZabTjCaDh8843XXrt7r1oufvbTnywqlIJzFlnnQ8NiJWUIs7+4vVqB+++8XbHB3Y24jR6nxpjlcrlcLpMkieO4m8HNgzjnV22rEHFDB4lzjpfztYpGhLepO8ImkgsH7jJ9QGwFttaCV+Es69kOBa6bOha0xjE8VG+QI/IQUDiiJEQfwhXggSNjzjtBDBE5AXnPjCfmvAOUniGEmARzK6YmeQdBZ8ms6yq9957WYiG4Cs0R/k2hXDcIiC+0uhax2t7fyzPxxWeFbWe98d5WcvOW4IvWlF88np9cLAsC20xm5tnh1GnD03FgRKVZPBgM8jzLsixJomqxdM7Mq3q8Pcrz3v541Ov1nHO3b+396C9+/Omnny606yn27PjiYjKtLKUoZstGqTjrjcqq+eLxU2v8/dffFDLb3d11zkVRPFssnHNt2wbz3+v1HFxeOWMsSRKjm64cOkBqpVRINgVpvvAzW8uEG3Ja67JqkzS3znMZx0kPEX/y0/d1a9M06/VHRjul4qOj462tLRCCnPfW1La19TLBrbfefP3+aze//70/fXLwkLwXQmnjVJzt7t0a7928dmtbxSfJF6eDJEtklOQ7TU3TokSVeIoWy/npRbGs3KiXE1ONg5hyckVTIlFqtUjiRG5nzvvf/s3vqFjOlzN7YbxgDpghAmZFHPcH8XC08/EnPz84fPIb6a8dnxyenJyJd7/y4OHDR0fHz9KkPxj2yqKtq/bg4HkvH4SGRs+fP3/8+PH+/p73Xio+HvWzLNPWXdu/sb+/fzGbG2Pqun7+/OiN1+71897FyUlZFk1d7o63XJLnO/29vZ2/+tn7e3u7t+7cOzw6AyaRiYvJYntvPF8uCVFIX7cVoCLm26Zp2pJzLiOBSGVZXlxcTOezMHN5f6hUrJgo66aqSqGF1+7s6HglxObJWy8Zv3PzViqjxpXeUxRFjnHtWkaQJymGQhfnO5EAZywittohcGTEGBJAR2ZkUAEgQyJvrfGmtdZocp7AcyHI+bZulIqTOJYiYoCV1UIyEAzAR2ki0qjVtWMeI97bGjCOJHySpGVZns3OZuWMI5Cz3hki8uSQgCPEsUIf+NueIXGGSRrnPOOcA8lQ15ukeXDutdatMd7XK9uAhhFwXClEwUqM2QVWQYizdeaQrTOsbNVmcKVlSaHRA4Z2eFIplWUZIgbx6a4SMs/z07Mz771cN/Ty3jNY9Rowou3ITWG9BoAsicqi6uf9i/PCaNjdudnL+74nHj08KIq2KIxxjsgD+CCHFVqAAkBQVkDEsizLot40Zut3+5K1cMXEbhqPzpasd7iKFWBtZTvj2q3lsGFHGXvBzBARwCVLoEt1B5rjy0Bhw0jQFaDQ/RC2K6CkW9YvYcpL8Kj7v7tmWHesCIPZpVq7q3rV+MDl2CIirWQBA1AYDgZVVdDKbuHHH398cHDQVo6xmHNurfOEQijvgDHhvAs0DAqM/lD6H9j0/zl0FLptcxy6tEuQvgjgtSzL2WyWpmm/399M4sDa9hNjsKFfCWt0xTknf+UBYB3NuZsptupxGtJSbOP4a8mmbuLWoSzvfQAKRJcVvwEi02UqZCUp7T1Z64k0+JDH8eszQkg1QnAJwJNn5Jnz5NGR596DR0+EnsASOfI+MBKA/IsX7BEwvBRsdembz9Yvgg0deF3f8gpIGfJV3TS6vliWi+lZoRsDVmZZZfHgZFlWLI5TyQURtM62utbONY1tmgrO53E8297eun3n+rVBP4p78/n0bFnMdRPH0XBRpFlsjFmeHT4+Pj8v2q3r6XDn2tOnB4dns2VjVW18OVexSdOsdTRdVJzL0WhULsqLi4u2bQeDYat1iHcy4du2Dc29gvkPblKWZdO2Xs+bD6zw8B6FPUMQtAOLjDEZZUwZEJFKc980+WCoktw5d3x8zLm8vr0XZ6OiKL0H4HGWZU9PDp3R6CyZttFVIiCP3/nKm29PT0+ns7NlWQgmk4Ttj7eu37w9XxR/8sOjar6cLEiikDx59OgkThIDftHMQcSE8fOTi/NJORrdYFHSVJUw/SjqCeb7OTe6YmCiXgbeSsXjSHhqtCkJrGXCEwKXZ+cHvX4mY7a9Nxz0srJcnpwdp1FP/Pgvfnh+PimW5c7OtdFwm7Noe7S1nC2apn16eoqIxpgkiZw2jMPWYOhc0usNlmVhtTk5OTs4OgxPahYnwV8P3S/iSI7HY2dtFKOKo9OzYyl5FEljWuQkIwZoT06Pnh8eb+/tqlgtl4usl45Go1E/Pz0/q+uSyzyKJBeoYrnFh03TECNj2lCy6L31ZBlTSRJNp1NrbRon3rpisZyeX+xtbed5nxEAgODcIHrnOGCepDyJMJQSabvGgOCcS5IEnLVOG6Odt57IOx/e2MBdc84xJrwDIUSaYRL3lIoRwuMCSZJorYui8D0JXIQgHgmMpALujDdFXXn02prG0VY64pIvyvlkMQl2WinhPQiG3oNUQrCsKpvAYTTGOudCxUEcx0AyRFOXyzljIiz4l7REEl2sFRlD5FmWhUyB3+AodCy8TVJb8C/DYhde+1B9HoQR9/f3A5QOTJ9QChEgRWdHrbVVVemmXRnadd7dGOOsDSyhOIrm86WS6WJeMVS3bt6dTRdV1dR1u1yUy6IBhkwKIZiDlekNWYkoikNXraIoqqpeO6kv9ACEF3s3bBrRTcfxF5mWlz/cxAFhAAPcQUTGYB08AEDf9WHaWFS7wDJ1ejiMsSsnDDuQpyunuwJfNq+/u5guxb550u6Hl28fEbUxHc1+A2q8EFHoztIdJHw5/BDsynw+F4IJAcV8cXZy2tZVv9+/f//WJ4/OAdBoK5RM4sxayxhvtOm87Y3t7xJUeOV8/ZKL50LAujAhJNSqqgppiCtH6NZ6WtewhFno0ABcMgZXo9cVDXnvO8wUvhs8eFiLH3jnupA+dWD2sl4mILYNTut6glaZ0g1s6ZzznsBzxi7RKhFZ652j0HBBYATEAJgHj55pCIRKDEUPnsh7dOSJwK0qH/0qZRDKWkKuYfVs/43mpcs8AgSpiRWK+un7Pz8+fOJ0eXz80NnqfDk9Pj+1TMa9UW0oz8dp1DOta5qWEyLGi6JVKhps5QBkXVO1ZlE2vVobcmXTWqepdnVda/1pWZaTycTVc+89EHv9TTnc3pssaiaexXmvNh4ASTvGnVTp1jgqm/p8Nu+puCzL8/Pzum4IkXNe17WKwRij0jhwtuxGA9K2bTstig4ZhOIyePHlCvfbaC2lRCaa1i2WZa8/JGSNbq7fuJPn+fHxaRTFdWPn8+X9+69PJ7Nr13Z00zLv0eticuGNXswm04uJdyYSUg5HaZbNqmI8Ht+4dTPJUm9jpGRrcC1lwKz9q5/+Zb/fl2l0dHHetLbW5tHjgy8ePxmN9/sJTSaTvsoH/R5TAP0eatfUdRYr8LScLxaL2WIxL8oZyJhUjEIIgigV1uvPPv+oP0j3b+w1Te2cu379uhhsvXHjdi6F0tp6D0mUpkn21a//vfPTsz/6oz86OjpKorgqW86Lvb295aL5yttvvv76G59+/tmP/vKnSZIBcgB44403l8vlw88ejkejN+4+UIJxBuAJHf+tX/vW+Zen+rw9Pj8b7++bunGo+6P+9Xzfebh9/9be7rU4zUKyZ9Dre1MsFvtlWQbxRA6tYj0AmBelNR5ti06gxZgjcWbqguJMJjFzzjskj7FInObHB9PtbZnEWwBQtbX2TGR527aLpt7GFAgZl3HEyaNzPhaQJXYxm8oEgUzTLq0GUMLYum3rFlXaSwXyuqzAQy/vxyrx3gflTsYYAVrvjHatay1ZVyCLeRRFEEXee+OBK07ex3E0n5tBvtU0TVu4YTpGLSdP5reu73s7jXlkQZfLRSxVLuWyqpkzVhvTamMBmQBUy8rMlo0QTMqIiNq2dY7iOM7SPO+PptNpXbec2aBN1JRGCBHHSts2yWJCT0RRouq60lrv7u4G6+uds42xhe2iCGWx6qEFCAROG9fqqmmNJxwOh3Eca+NavdofVrRQ7aSXImKMewfarDK1oaBRKaXSKLxOxPl0XsRpf1nUH338mVDRt//+d/6v/83/7eJ8VpSmNT5KkqptyJNDWBTNcCsLDSSDhHagZBJRFKl1KvTqMvayrd00it0OHb6hF9y+S5PjNxQMN7nxFwt789a10+OD7XE/Vuzi/PT6/o26qGNMyqLOB8NmUW3dHJT1jMDGiajACyHKZckI8rRXLwqr3e72dl1UWZYFmS9HHpF6w4HWWqBvmsZanySJ1la3Js8y5xw5JEIiQATOgBM5b7zxqLIr9jJswb/sBiGkuAHAIQuBnw7ndat8GL+Ncg8EAKYIEReLIoojLpU1RkWiLOssiZKIV8vq6eNHv/2dr//Wd39dN/X/57/9f3/48AiBZMwb3XrfciY9gVDCWns5V957DI1LV9qQ3dx107RhgC/v4hehvVc+A+FQzhvGWFW3YX1HRq2utWm0aQAzIVcVDQTkvLPWRgTeE/cAoT81ATAp2Cq27NbNOTsnEuPUhSiRQABwRH4VVwj9mdf9HhmQc945LlQ3UWvqyyqu02WF1i8gWudE6rxdnZdz7onQEhEVthRsnTTkGMJ+UkqhgUsOHrW2YQa9t7SuSNpEz+Hklsfek3PeEYT4hBAcmPAOgSFDEVAKIa6ba9vLeQoYZd04sW1XUm9EJAR3zi2Xy+WyLGvd1AVP4mpRFtNZnvY4k6bRPZXlglNV1G0TSc4Vb9raVLWSmUNb1UVZlkKI1jVnk7PhcJgkiRCsbU3TWG1sXXsP0bJRed733v+bP/yLzx/P9vevVyYlNnx2eBRFSZ5FtXHWLquqDiyrMp7Gcdy/ucOIW2srbUSetK3mkUBJjPtW18aKPEPvuNWz2XSZJMnpycWNGzeGw63z83PnXFmWUvIsSxiH8GCAM4AeGcXCXVwcRVEEIhqP0ru393Z29p4+ffb8+fPJWZVGwntbLs9Pjo6q4uL+/fvDfsRZtJxNTw7Peol/497d6/vDk5NHr7123djq8fNnxLmzvmoM4/H5RTk9evbg9Tectz/++fvPD55yZH3j0zotSz8/q6CRs+Pln/6HH0Si/+1v/73KYLX489oM7t2+4+v5+eREcJVRD3gap6N33v36F88OMJIQsYtiIhTl/aiFSHDOnPKKb+9fF1H0e//wd44ODwWg9tRIpbI8Z0xEIknTPI7jzz/7VJvWk2ttC2SVUlEkGYOyri6mk9CEO82y8Xhna3v84I23/vyHP5zMpsvlUilBRj95/FBKee/OHSB28PxoMBpu7e8mvf7t11/rb42zfr9s6jTrcRk58lJGvV4PEZtac+jvX7ve6ubo6OiDjz84ODoG8KOtbSGlVJwLyZio26YsamMcIaO2REQE9M4FP74olk4b78yNGzfG43FZ8qOjw7JaJklyY3vc1EtEZEwwrgBQOk6E6FkSj71rq2ppHRMsFhKdl20rNbNZliku8zTjgLFKAJhu2lVDW62d88gZl1HWy6WUy8pESaSUcOg8WGQIDAUjRKaUUnHEGIuEytLUe0qjuN8fWmsPnjytijoYJ6UU501YF5RSQgkhFOPCODLGeW+9twAscAzzvB+peFUJZn2SJMdHpwDw/PmhEGK5XCrHQ2Yh5BTC4judTqMo6irK/Lqzkfd+MyQbFsQAILz3RVGEEla3IWPcGBvMUohGdBJ4YcnuKBHdcp+mqSfjnENGddVWVXN+fg7Eb968+ezpcUgAEREyFAKMMYytJBFD5jKc629iLf4Tt1eiCgDoJdHs/CTiyMmS84nEcn6huIqFH+xtnZ+fR2k0u3guFIy28kW5qJd+2O8P4mQxm2tfpEpq8oJo2MuttbquskEWxWld19ZqAN82VZamRGiNB09AviiKtm1Ho1FwWwE8oPfeBfd2s76fNrbNe9n81XmH68aYfmP7ReNQVU0URchC9GIV646UjON4Z3s452eRoLfffjtLeydHhx988LHWYIzlXHAuhOArftwvjR3gLyjQf+WHf9utI9nBOhLj10Whbt1C8wWkskYY3Ve6P7F1tfAVls8rr7lDPFe2zTPSRkqoq1CFFzGutdZb5y/rgRHXEQsPLtya45cVoSRYR8ugdQKRMRa84e7a/BqkeJCrJ4Au/4Tg+brb5N8QnAkhkME6ibmKUBZFYZ0zxtS6dcZorZ31GjWCQxRBiVQoIaUEDqGlQp4PlFLOOWt8GJPQCPTg4CCOY84xkKw7vx9x1ZbFGt80zboyHPb396WMlIyttXVdt22DyKIoGgx6g8FgMBgwwsViMZlMnHPgqeMcSKmEkG1V69YyFGEwQ9GZ1KJpKu9tVVW7u9tEJHCVd3DOeu+NMb1eT4pKcCVFrLV+9uz5ZDI7Pj5J09S5lsARhWYF3rm2LOdcxZIJlWZ7N671s/TGjRtZv2+8f/T02aIuG+ucdShV0sviNE3zHjRL49vJ+cVkelrXpZS8aLxD0xv0xrujxZPHT55+MZmf7eyPZYx11VbnRwh89nbx1Xe/OuzvOWN8g0yKfn/vjXvZt742GW5db8lMllOLjkum8tQY41sTcbGzNb518+b2YFRXldi/OWJMREoJIeaT2SeffHBycqYb0zRtWS/SVMVxjIyyfuacm06nJ2fz50fH3nttTVGVMopZJL98/IgAhJJN3cznc2va45OT4XC4tT1+8923P3n4xe3X7skknc5nKs2yvGfJqygRKgaGrnWIVrcWAKqqImiJiMCJOB6Oxv3ZfLGYLctCJbHW1hitYq7SBKWqysYYY5DiOGLA27JFdBLR2LqtFlbXXjdtOc/SdGeYjfux9c45A1ACcABO1DKmGEcgARxSqTwBFxbQNJWx1gCg4DFIrbiIogikEsgEk02jW6PrugViyBmXAjlngocnhUvOlWSSe0/W+bXAWqjnZs5557wh19TatVQ7Y60dD8dPHz0tyzJREXhkKJy1zmpjHDmUCoXkyASRcQySJHPrttfWstC9WrcGkW9vb9+6devs7GwxLy4uLu7du7e9vT0rpm1rtDZRlGRZz1pLFBhtIaoJjCEAC7xoa72SGBz3kCkIUlpxHIcoXKgYho0UeMcPDyikc33CqtoxJTtrJKXUxmltGOehNm8+X964fvvWzduHBxfeG865tm2Qi9ZaC0zCkhcq3GhDXPlvu11Z9X65HbpiJ2gd7e+n/Pnzg9EgE8ByJa/f2efItgZb3/zGr3Iu3//5h+++97YHm/cij+0f//F/iHyEHqXi87JqfKn6/WY2rwHyPAfyVTWzvh6MRoAEyDxZchbJ60bPZgshVJrkiNjPcmvtWnvSE7jLuP0GUICXKI3h1838sYcVStj0ZX+JPWBcMi4Zs0RkWm20VpIj0f7u9rd/9RufffTBfHbe7w+Fkv3h6MatGyd6Ute1MVZxgYihCzcX6m84a93Fd5byPxEubEYpwgHdixt2QlhBLyHIDgGFf7AmaoQaRyY4Y4wJjpzBmmdx5eK78WSXSkqXF7MZ6dn8a/erX1egdFcLLzyEIXtAiJdFj52IExEBWcZYa7TU6xiDEIyx7rzdgxGgEsCqPNLRRqQAPGMMCLqa5L92nOu6RraiUTPGhJBhPAeD0bKYa1Ma0owJA2is994pKWyjiVVRRB6ciAQhIrAoiol8qGW1xiOQMaauWQABYQCdWymUO+ckk0F+TQhhjAuFPG2rQxLBYNs2pqqKYMKHwwGglzKOopSsC3lVY4zgvNfrzabT4MkoGS/90lpyDkZbfSGEkCglZwyUUgCeyGmtAT0yFbhfnLPAAUcQUsRJnGZZ1vDGOV+WlTHm7OwsjqO9vb3hcDgY5ta1h4eHn3w627uxwzkHcoohl/KiLC6K+Xw+d84Aw1LrShumRG3s2Wxyq9/3zB6fHR48eXo+PeUCVCJlRCLyy+osH4psyqdT27r50dkXn3wpGGPlwbIoisnZxXJe3rvz2tZgC33roc23tjLV6ycjBafet7kYqFSpJJZ9yRjrR32ObDjcLuf1l18+OXp+KJ4+/7xtmkjFW1u7RvtZcXYxOaorU9ctMkyyVAgR6Gna2aKuWg0EmPVyoWTZ1G42bZ09O58wxrb3dnXdaO9kpF57843r+9cevPXmrK5Pp9NWa1+VF5NZlJc0vYjilAlFeBEnWa/X84TFyQljPI7jOM1D15ORiltnL+bzw5Pjk9PTO3fuAIBKEiYE44IByjhikQSj4zRmHp02znnF0DtX11VbFno51+X81q0b12/sxypaLGbTaaGEJr9qhuasICYYckTemkZKGSfSWFHVvqpbjiCEiMRKZQg8ETKLvmnawPkH5KGlLDG0RN4ZshRlfSY4SE7OITFA4pwRI/DEhTLatY1unK6w9cYS0cnh8Z07dySX4CBoROq2DZ1syHnnPXecEQAQY6gEE5J5ssa2xbJyzoXaG631aLg1HA7btmaMldVSCO6cJfKDwaCua6VUnufOufl8zhjb3t6ez+e0ll0Ki0MIRAc+fEfnGQ6H/X4/TdOiKJbLZddwyG90SOLrXouw6o6zIkx0FYwrj4UIEY1xxhgknfd6iMwaDwCDwSAgeu+ASWFtLThjKIxxyFy3SnaxhFVrjL/T9jc0PPgLFJkmZ0/3tntff+/tNx/cHfSzfpoopZx22+Pdsqzf++r9b379HQcWuKmbxZNbO3/2+ePDw8Pd8dY4z7fHoySJj7nb3h7sX7sWZ2mk/OODZ/OZiZJkkI4iKc+m1XyqEVFJNujnQqjpdC5khIiBFO/RA4aQz7q51NoQbhoY3CBLbmIFjytg103KLx8QISPGJbCGkAF4FvolM4gU+9Y3vt4sp+9fnH788cdx/N6du6/9V/+Hf/r8v/lXZ2dni0UZNK21NsEDD9yOXzLOV8zYpr++CX3+tls3Alcw4gvB8w3/vrWGNnL2sGYMrEghiMjQI3QMP/UCm/UVFNSr47/enLuUZ7gyWfSCHijgRj1Fd0lhWQ77MPDd22Fa3zGUpdSduFZIq3c37r0PTgvhuswYu5TTqpY3EGs2ZuSXjbNzjsGK7UQMO5xkHRjttCFjPQWHBLx35BDqtm30gvHKAWW9bDDIev2tug0CiNo5z7mQUgIxZ2mjOrGr6HGMofHkjc2jWMagnV0U1bIqF2VhrQ3kRGttU2tERIFc8bJsZtOF1tq2ejI5Xy6XUoo0S5I0vjhfscFWo4qCMxnHQTLOW6vL0hhbA4CKRNNWK6QipRAhfiYBUGsvRMyYco44l9G6zs5aWxQGkbRunDf9Qer82DnHY4UE1pJl2HiaVY3VejpdAGdSyqWxVdsK8g+fPkl++lcopKvmy+Vyupx47nu9JE0igSBjV82LrZ3E8b3hcqmtfXr08ZPjT5VS5sLquvGWvvf9f/+Nr37rd37zd772lfd6eb9enBtHTrd5LFOuYq2yXtobDg7Oj5lgEYsZ4HxSFIvF40cHBwcH4vUHN54/P1wuy0V53jZ+Nj87m5xWVTvqjZVKgXC5KBFRRVmSJP3BuNXWeN9aQ8Bao5mKUHDGRVVVW4M+59zpNkqi4bB/7cb1fDj4+LNPJ/PZydnFoi4BmT/Hoqyv3bqZ9wdpkqeZ4FwCIHLOOJdSNo0BIKWUECpSWb8/3N+7lqTZfD7v9/tJkkRRVNftcll674WMvHVknPfgW2MazbkHQ+CdAEyTOE8Tr9tnDx86bwMeNK50NngVhCCEUFJGgqu2NYVz2lnnHBcsy9MwwUmCzjmrjXPOEDAmrPfIWZrk2vmmbY0xyJiKkyRNpVLaI1OcKw5egIxDo7mg4hdx3tYNQ0VACCxNEmfsbDK/ed2P+qMzcdI0rTNeIkNPcRTZ1mrrPPfIiCMwyVGpsm2sddZaY1vd2pCjjOO4P+hxjicnJ3t7O1LKN9988/nz50fHz+8/eDOO0xDbLIpl2NlaH/4hopQopQp6OG6tQBI6YQ6Hw1BoYIyZzWYryLyOJYTIgdVm0xMK8CLQWkMMwK47GbKV/oH3DhBAyggRJ5MJQ9425uzsoiwrZApDc23nABljPGQWN1kCfwdrARu2Z9P2r5bhv9kXw69CmN/97b//9tv3b1wbl8Xc6fbBa7fLZf2nf/KD+bz42te+dnF+VrfFYjnZ3hv9vW//2lb87p/+yR+nafyNb3z9xvX9s/MjKV2WJd/5zreiJAZsm3Z5MZ83dXvt+s6DBw/YW699+MFHk8msatrlYg6Ay0W5u3/NGk9AIXS5tlKXlomt+1GFi9w0PJuYAABWCtIbpuuVvm+31cYS49aDYoxzheA4krf2+ODAW5Pn+XA4/PzLL65duyaEGI13GOu6b1wWKLpXdfDqNnwxhk8bipN/LY75aze3rvvYhFO4LmS4pKNeFiBQNyy0DgzgBlulexpfBi6blh5eFdShjVbg3Z82sRG8GFHw3jtHDLCL28FaoMp7Kxhf4YBVzmQVYOAIDNARgPPWGwADAMjFi9MNHtAD+nVE6ko3tQ4orG7nr0NpaZoC8mCYPQJjPDSWe/786Oj4rCwnyLS1DjxIoYhTa50x5Lx12tVtY6xVUZTlifctETImglZbqNLy3hGRR884MMaF4MYYIktEjCttjDZOa6un86pui6pqjUZEZ1vmGXlkQbYfXGuawKUwtgVnndcqwjxPhsO0rKZFOWvauq5XZVZJkjDGelmaJAljo1AT7imp65rzlUQsEXUSTMF9YkhKKa31cjlHxDxPjTHn5+dlWRqjz8/51tbWYNjL8zxJEiKyAsW67UcsFQoFxGTa11oXrdaOEZeO2MnJGeEH+/v7o9x58CpVPZ6nWYxkKl2ZBkCYdChv9MY32E7V1F8+ejiZtBg32Zbss7gum8n0/Ad/+f8rmhNDi3/wnd+azaYemBD6+vUtpuTR2YlxrdVljNH0fPr48eOLiwuhZFAeGm/tip++/6P5fIkgd8d7HBMZCaE41thak2RS8chZTgSemNGAoDS11jouBFeCKenIL6tSCJUmSdnUjLE0Tawz0/ms3+/Xun349JmXctFW08UiybPFsp7OZiyOZZJv7+Yyji6mM+fcIB8AY+cX04NnzxaLhbU6z7MsT2IZ37111znz0aefWGuL6ZwNWT/rxVJNp/PlfAZOWNKcMbIEhpyxiok8Ssdbo1hF1tpnz56dnh4X5TLO4tFoBKwO721IHEiplIyZ4EQ4n8+rsomzdDQaJUkSKo6cs86smu0wAsYcY0JKWTY1AUPBExHzWKkoUUkshNB17Tl6BB96qQEYZ63VzliR9rwHxgQXKJkc9Ppta9riXNd6Z2fn8OD5yWIZZcIYk8ZJKPhs6woAlFIYoeCCCZ6wxDkXFhkbWyXjKIo4l5wLAl8slzdv3iaiu3fvjsdjY0xZ61ASprW+cf3m22+/Xdf1j3/8YymUs95ZZ7QVXAIgZwKBNU0jhOj3+7u7u/1+HwDatg39n4JTGPgKdt0/uqMOwNrR6VbVbuUKL9JKl4ZxIQQ5zzl3jp48eapUfHJyJmRpjImTeC1bxD0Zzrm/jCLgy8f8W230Kj/1l+y8+X+3hu7tj37ju792/frYmebs9NmXn38+m07nk/LDDz9ZLqob1+9JlUaJampvDb99+87J1gGP1NnsAhjtXtva2e997b030yz69NNPVRQt5ifOFowaY2hvPPrt736H6to29XxZTibTo8OzutGMQxpHk2YOABBqdwmRkScEWKk8wUsx9k184DcogR4wxCE20cMv2by1VkgIzqLXpnGKt+RhMavf/+lP4li99dZbjOPx6ckXD7+cTCanp6dVVeFa54dz7xwF/sorj9/NyGZQYdOO/s3m9pfN48sH6aAVvFg9CwDIGBB1LNmOyspWbaUvv9VRH37RSa/MRTgFw8tvdZE52CDPdsAijIPWWjDeAUEiH77ctrXkIrizgkFX89yFEDZfFh/U2ABg/Rp2V+W6MluOV6//b1O56r2HNfdlM6JgHBHjQkacMwDtHDHOAanVGpDJKJPAtHWNNvOiMN5xrkL8I3ydc+69JQKtDdGqZS6GejZgACCELIoSsQotcIP2UYfkQoAzXEyaxlEsvSXnDCIIBSlT1lKcYhTDxeSwaeeA2OpSm0bKSMqoLGsVqTRT/d5wsVi0bRvH6vTUIGKWpcYYXFODuxnnPMRNjbWac25s68kH9ca2DbQPRJBSRECmaZq2qViSKBV7BONQV9oYoy04FK1upEgYKEAPUC5m8/PTM4kJYyCjhDFgghNY9EiCrLeAlkWgFAMpRjt53K+2trZ2B9tpmmUqLWfV+z/94E//4q8I7Y3b127euAMkZMxm00Wz1EdHR1XTCCXBpueTi9nZfDKZa2viOE6zTEop6rpSSmRZHqVRU9u6KZfL5WRa7+1kSiVZOlDSWuOt9bOittZCwmQsuJJxksQZ1+2KUSIjNZ8VcRxzKaaLWVMV169f90A/++gD76hpddrLVZrFBHtJily2Rp9fTI9Pzk9PT72l0WikuFgulxdHF+fnp2W5zHtpfzSQkjtnCP1WfzCfz5vGmKoBlTJPrtHFYplCTzsdSRWRRCJwXiqR5kks0+VyvljMy3JZlLPy/8/bn/XKkW1pgtiet40+u5+ZQ5DBYEQwGHHne3O4U2ZWd1WjWtUtQRAEAQ3oWX9FkKCGAL0J0IOeVCmpurI6K6tQyrxj3iFuzBGceXjm47O5jXvWg7k7nWTEzcxCZm0QxDl+3M23mW1b+1trfetbZemm4PzyIuRk7RkAvHzyHQCcc+MsY4xSrNSSEA4hxKrO7QGEUF3vV6fgSpFTRlnoe35AfA4QscBpZxnnCGMH3VLbxJiyyouiQA5QyqtSIoCMAcoYhv00TaEUw8vxG7dudjud+WzSbDZlJQhHWZZVRV5VChHqrIbOQggphiwIlwUL1joHOOcIEmutUrIoCqnEeDzM8zIMw06ntbu7+/Nf/rYuWN3d3b19+/bbb79d695fXl7OZrMkSYwxVVWhVUnk2osCANTwqBZKWouRoZX2SE1lWBcWP9+KVmHVVRL0RaqdhSsRY2gNuLy87Pf7x0fn1lWEMIxpKRRGlDFSVAphwhhaH9htfMff34Stbd+XuoBf5Sm9ChHqE0nT+Wh60RtEYejdufuOEOK3v/rdw3vPIGAYBb/+9cevv3ErioJpMh5N86OjMWRRZ7t3Pr344um9ymatpn/96v7h8cUXX3x2/fr1Ip/NJkOHcL/d3+61W5H/4PCpz9m3v/mtvKj++q9/enh0Uo2mo9HIOQgxAg6CF8sBrP0S3xdsAIVNlABWHvY6Q7/xl68YmCLCnBRSSmC076PXX7vebTUuz09/85tf/fH3/6g/6CGEPr93//Dw8OzsLElsLTTknAOgJtM583eRGcEruG09rTWM+H2H+L0H34RQbhXheJXMuP6KzR+eA4XVD25DGPsloOBeHGCFftZPBHyxCcjmzVp/5OWIgjVooxRl/dWgLtZwTilFEKhlpJ1zDmLjLDQOquc5KQCAKyqw0oRYR9ettQ6uZdE3LtRKmOv3rowXRlmWywTZqjtMTYfUxiFICPcoJtYJACzGCEFIKYQAUsoxocoaaaS1oBTCowRjDAACDgFQAykIAKh7L9VzRgBZA5yFCCGpdVFVmFLjXCXlOlIklaKUMkoJpdYZiBBmhPlcZIUDum6AZWylTamNcYBDpNvdgBJPSSeFiSIPADSbTUejLE2TvJkuFlmNsWrz22634SqAWhtP4BBCyFiplbZON5phnXELw+D69euMsYuLi6Ojo3SRGw1arRbn3ONhkaSGWAO1dVBrYwAEAGDMqqKwFgaBX5a51jLivjH64uS007nheQxAJJQhBPlBEEYeZW40ukAYaidni/F4Nk7zQiiQnp7PkrTb6uzvXu3ubu8tiidPPr739Nn7H37W7u034qay7OnhxSxZzLMUIGitPX3ymXXO87xBty+Uqnk5WhvyvT/8rjHOKJdn8iwfWwiiKMIoaLY6UdgghFWlUdJVlZRSEsIgssbYJF2kRen7oQWIcx63mnlVamPqQtL5IuGMxM0GZSzLC8YY870ojguplLHc9y9Hw7KS0+kn4/EkyzJgIcaYYYYxLqap53nNZqyEPnn6LC8zz2ftdtPZViNq9ru8EuLwydPRaGQt8DyvKgooLQkxhggZqKVxGFDMLi6GZZkLWVpgCOPUaq2VgahINSGQMEoIgNAqp5TSSoMo4ls721euXKEeHw0nw+ElACBuNoixhBBKGSc16wcSwhAhXhAaBy1wyhojBGKAUMowZoQ67IwxTjtjgHG2psxQVPMHnXPOVFqW0mo3mUx9pBBCd995u9loN8JGq9keyyFcZZEBAIxgzjnnnDDKuVdoI6WUsrLW1gJKRjspZbPZLopKCHGRXwAAHj161Gq1JpPJdDoFANy5c+fWrVtCiKOjo3a7/YMf/OCXv/xlvY7zPK9DBbVRaDSatYVaLBZpmq7F+6IoqvVGloGBVT89sNGuaW3CrLU1OdlauyRjrwyfUgoiA1Y9oqpK7uzsXJxP0qxizKvp2cyjlHAAcgghobQmb66TDs45Y5as7H/ocC+mHn7/O1/6SP3/PHV/+R/+8oMPO4N++8c//NG11268/9uPo7h5cjw62N/+6OMv5vNqMp+kxfzKlYNWt3X9zZvd3a29Yh40ImGl0KgQxa9/+7dbg97bd25LJT79/DPgwMHediMOj549pQTFjXBnZ+vp0+OTk6PpdM4ZbcbhZJYAh+vW4RA+7za0Bgpo1b+gnvZL+GA97EpK6EV31qGvcI4BpohgAKA1gFN8ZX//xz/80Z23b//lv/s3f/k//+Q73xYlxh73P/3009lsnuYAQkBpTfrRzhlKOGPUWPBVHIXfc4/+UcZ6+a1Xpt2oeliTGes3Qwi1ex5lQbBuYeVq5SELHNhQuwIWQozwV4CYzZW2eSNEJTZRy0txnTWGeD6H1Vh9xNbOMufcmeWJaGfWAXBtn3/jZq5kzWWpET9aCUNBvAoBQmRXYlC1YsRmO6i/c6xYEdQY4yCs3Y8sy05OThfpjGAdBkBrBZyxBBFMOSPaLJ0xShmihDKOEKgjmhAuKdWY1PAXc76kPdV3rWZzI4RKJaTUGNMwZFLKdQu6+vQxQQ481+1mjCFonTPGCq1UWabaCO4DTEyj6fk88LxoPkuTeR7FnlZAqgo6U+vQJEnSbDa73X4QepTwPE8hxIwxjIlzzhoAocUYC5kXRYEQihs+BGg2W2CMB4PBYrHweNBstAn2KKWEeGUpsyzzOLbaVLqUymptEaaEM0KI1hYCzBhPkpmoqiBkUlRnJ6ff+O67HqdCllIbokFMvDCk3IPHx4eNpu97cSlzwnAv6mitLobpeJYZxxCeYhg12tu7BxfZrPzlrz9Sir/91t3jk/Hjx+dJuoCUYUYXSVbmldJaCGUhWKSpdIZ53AJArvV/eO/evUVS9Ptb4b6tZl+U06OQgVarm2dKSmEtMAZLiw1k1kKoGYSQYk9rW6XG8ygnTCa5UQopBZmoVMmg8zG4fnV7NjmfZ6Pt7V1UGVHlyEFqNapEYMHhZ5+XhagqhSEhBDsFsmIhhOm1A46BrnKrYDOO7rzx2sHBXrvZfP3GzdlshgGeTha/nP5qkVwa46wrFXMIEucWxhhCsB/7BtuL7IL6lHPuo1BrPZlM8yJ1DkVRI9MFx5xaqgsrpSQkaDab/Wb89ttvQ2SEqMbDWZGrIGgAYLVUiBNrnbEyl5m1mhCCSUS9BmFeVdnxcJ4VZa/fbXd96CywFnmtKAilKocX52kyIxj4CGCfckoDpkhop9OpsdYxl8jU79Cm4cls9G/+/F+/++67+7s75+fnk9F4b29vOBwfHBzMFwnzWXerI43WWknnKIqCVnB1/+pkPi8LASEcDsdlKcajw0WWCyH2964AgJ4ePorjRrfbvXrj+ptvvNHv9w8PDz/44AOl1Lt33/v6179+/fr1wPdHw6GzljGmpMEIKaUIwsYYo7QEAgCwbqs6LkZ1VFNpWauR1ECBMbZp8mq2Y40bIIRhGDrnhBA1caGqKg9XCKG8KsbDiee3PK8DUOAQ1UAoUzldIgqM1UVWckSAAco5ABDBfNXlziHoMFq7Vs/NKwD6S12g9Ta/BivrF9e749rSwxc/tfQLwXPRx6oEv/7bJ4F/8vqNa4x8fvXK/te//cev3Zr+u3/3lyeTR/1r/VFxnqmqv39FIO/x8XQhHiTJrBL5+dn4xvUr0e3XCO/8D//7/0M2n956/c3PP3nUiVplXl3Z2oqhe/jBBxVkP/qTP/vFb373yWf3j8Yz5oXDScJ7OzBqAefKqkLa+B5HxjrlAs4yUEIIEQaEIAidNW7FRSWrXQc6V8996USubp/dcEzXZXub1DwIAABCYt8DWhFqGjFvxHj3oPXaa/0/+oP3fv2Lnzy69xkjvpJ29HQehEELuznQzgLrbD0BY5WxygHkgHHuedweQrrcCOEL++LmZrl5I9avrG/Zq297KXIAlnjoBbY/2MgLeN4SmNbR45qZyyGFaLk2Ng9Yl/gaY4wF1gKEEIOUQgowW5MwIES15LE1FiFitFFKrkiLCDhnjDWw3lcstMsYwypI76xRAIDl9VDaAQAhNAgoa2pXCiAIHJJK19seqq/HCrhbgKWGhNbdy5y1DkKLMXQAYFB3q4QYYeuQ0q4u4wAAcLqMVSyPuYKb1tq18Ha996NlieyqY/iLdwesajoopZWSNR17Op2eTx8h6ALspYWr8pxRjBHVwBkpfU59HyOCAKRCCGxRHMalLWGtAAaBdtpKAzFEDCllMMbG2UqKeqoGGKOMqXzsgmRStNvtXmtvPp8LUXLOCaTAYFk4ABx0xCMeAVwWBiEAICOEFeWCh41sctzfOej0qTb5oB/t7LQePVwcHj9UZh6Hg0IsfK9dVZWR1c7BXrMZa62CBq/KkhGYpwtMWr7fAg6HXhyGUVkI7lyDxIRyY1xeFnHUQZj/8lfvR1F0//7DwWAQxc08z01eFkURBYHCkXMOAIsZYgGCEBqjpKoQNAiDskijMETQFXmOMQMOgww2o06lCj3XRsOFyhZQtTvRoLlTyXK+yHXFqelUC+2gv93sjqa5lSxN4f1yaDTs3HgzluDhxXT4/kcfnYzyRTWbLDrtduRHo8thMS3mecE5J5gk83mSZdeu3zg4OMjykjx9+rgoMsYJY4RS1Om0hvFoPJ4nyVRrBwBg1CMEQ0jKUimloEYYEwwRQsAZV6fSIXJVUShR5nnqe2Q2nw5uve574c9/8dNG2MAAckIhxFJojzLPC6q8Otjde/L4UFU2DAFyIMvyIAhev7E3mw4RApTS/qD35u2bX/vauzevv9ZoRqPRqNPpBUFQZmWj0bp9+82HDx9+9MHHYeRzzhCoO7gYIYQEQAo1GDRajUYcxx4PCCFlURwfHx89O4nCRlmWUkrfD65evbq/v7+1tRWG4SeffJQX6WQyXKRzjF2z1YjjkFIqhXIOA4AcMNroSiipAC8BgFJpiAAc9HpRGEohIurv7u3klZ9OFufnp8PRmawqxhGnGADDIk/lUAoUkAbzvTwv0iIFjgECLs6Hvu8HfrSztZsk6RTPW53et7/1nfkimaVpr7/dbLbH0ymAjjHPamyMyfN8MZuXZen7ISfUb/LTLPcIVgIm82lVybIQO1vBztbW//p/9d9DCI+Ojh4+vH///n2lVJHlTx4/un79ep4XdViCMYax8X0fOJTn+fI5NAYhVON6sGIvghc9pE1PaB0CqW1rrQi59thqc0kImY/GrVbD9/1WqzWdTquqStO0DofWBzQrzek60ogYf3nnf3G85I39Ew23iiQ3WrHVylg7X6SPHz8t8vxgb/drX/vG2enlr3/92zwvIaIOoTTPIChLqdLHc8YJQuD8bHR2fPTs6ZOq+MPI/9ZgsJdl8uj4bDLNuu12s9G5HE1/+rNfSoi3d3dns9kvfvkzbeC112602t3Do2Pf9wmhPmOUEeisMsBanRUK+WR9eRFCdfyTELKONIAN0PP3jKZsDhaGSimAsbVaCn3//pO//Mu/fOfNW/1+/7333nn08InHwsuL8cGVrSyvLiYJ8Og/9MK+NJ/N+M3vvx1/n7EOJAAA6uK9WgF9M/wOVr3QnHPrTkh2BRDredRbqwWg3vYhhK7+92U8D7ihVLGe8MtpoNUPsFbHhkuE8NIJLj+xyWAw6wzgczzkVr1I7IvICWy41+hFzaXNK4NWneKX8RLwD4jY1V/kVrwKuGwjx51zQRBcPbgmpUTAaimAsZQgSjgCoNKGEMKYRxhDiDgLrbVlWZWirKs6OeGILkNBiEAhhHVGW2sBBMs6MAwRRAiEob/2TKIo8DzmnDu/OPN9P45j3+eUerWGN8bYiCIIAmuNFYoxfrB7cHX/KvfsaFJwz0eYWgcRJdLqSTJbFFlrsG2tJQQRgmqqRy0B53kedAhBkue5s0hVSimFMQmCYDabIWyZx+veqQBhtVjcv3+/quR8PqeYOOcCL4jjOPC8ySJ1zkHkIMYQ1Zx+QjGE1ghRqVIBaDHAPvMJIQyzIPQZIwDa4WQsTxeNpu9xNBrDwfbAOYCgxyh2zsPEAIQRJXkRSCnTuQFIWIMC3+csiBv9NC3GdCHzqhQyXeRlVgohWt3OJBnPk5xX2vPCa1e3rly56nF/PJmTR4/vY0zDIJonYwgoJi6K/TzPtVYOWAgwJrauxDEWIow0QhgjiglCSFmplJAIIAyKoizSVMi83+toZW7cuBFFjWeHx81m21pLKYcAi0oFQRBFjXSRn54ettttjMlkkvoevHJlvyiKDz982OuAK1eu3L179+7dOzduXu80G0qp0XBCCAuDgBCilX3zzpt379798MMPy7L4+OETzwtq4qjn+UEQesw3xjQanf29fd/3x+PxfD4nCIdBs90ujdONVjOKona7W4sLnV2cJ0lyfPysKLK8SDFGvV6n0WxxzowxGCMEAYTAAeqAVUqWogROi8pi5BPKOPfCwEOIAWCqdF4MZ+cnJ8+ePcvzNIiCsBlzSAEiOlcOAuoY9+Nms52QFDsehqFIxkJajO1ske/s7EFAjYZ5Jra298pKD/q779x59+DqlVIKhHGl5H/6939T1+QUeaak5pQh6AjG21t9KTQZjxHCzW7rrVv9b33rW9euXSPYVVWZpYkUJWeIUWatTpLZ734345wvFolaUTWttYQ8V4AGS3XFZX/qTeL62plbR01fBQrj8VhK2Ww2u92uXckwG2PC0K8RQN3ptdVq3bt3L0mSrKiWFmdV9AX+rt1ic/xToIRNq72eSVVVZakQAEqez8aTRjP+4z/8gzt33/3a17/90cf3ZvOk3W6WwgohLEDWWj8ICUTOGQhEXtknT89E9Teff/bgxz/8/sH+LoRBGPWSTByfjRlj7fb2Qmfj8fj1Wzfv3HlrOJ4cHT7e2zu4drB1enquBYBeqBRQlSCE+NyzVjuM63hsHcjBiNZAQSmzmvbLbMFXr9jvuc51dIkQoitBCMkX+oPfffrk8WGv1fzud797cT7MFnl/0P1X/4v/fjyZ/et//ecT/Y9zI37PlL4KGroXcxabb1uv1XrDWHPl1nmE+oc1JfDVyaz3XbhihNT+95dGsdaR8zUsqINtxhiAV1Lfm1wH6OCLca+N0Ih1ztmNr0CrjEB9+CWOX83QrlqxrxOC9QHrmPz6sOuxPn0I0XqqEGzwgVZlNkuQgVYdx1fDvXgo5xzCS00qjPE777xnpCIEW6Pmk3G+SMo8WyzmRSYUtUZDz/McwhgTVhMVCWKMUM6cc6CCZVkKJZxwGGNlrDYaQAAAchA4awGCnDLGmLVWiLIuDVVSZnlKECYIo7q7iHUOWAOlsxqUFQtCoZWqBAhorzNohK1Sza1Bk1l6dHZ5/96TSZaGAbPa4pAqJQCwnheEYYgxri8nJayqJCXMGltVFaN+fYUHg4EqkvF4LLKMKCm1arRamNK6XiwIgjRNA8/nnCdJEgQBxThuhGvLidaC385SyqWUdU6QUQYYs9YqZT6/99ndO29v7Wxfv3792dFDhDAmtBJiNs0gwdbBSpisFEJqBzFCGIGOKBZGO4iJ0U4LE0cYI26UFCWwFmHKtQNFlnmMbe/uSK1OT09ns9l8nrJ5mqWlg2g0HpPxeMiYl+f5cDisKp1nQlQ6CHlVSYSAtcbY0khgjCEUcI9qyOqMmZJGS6iU0hjX56m1thaUhWi1W7dvv5VlRRBEnhcuFgtGuHMOAdSImkEQUExCPxgOx2VRtZsBhHh4eQ4Auna18y//xY8ODg5u3LjR7XYBAEVR1CV5EEJtTF4URVY2Gg0/4lEzaHUb5NAZW4pKFYXEiC4WC049re3Rs7MqE+12ezgcjsdjRqnneRCi1994HSFkjMuy7PzhZe1VY4y1BdoBgIgX+K1ur9XpWavTNA25V2vhEUQR4QAKFvQa9gABAABJREFUmYuylEoCn9kiT0VeUGC7vcZscv7p+yd4pheLRZYKgEHouY7XYBQKLbV0AQHOoSxLZJFbaz2CIophENy4cRMAcHJ82my0HEReEI6ns+F4NJnPuO9NpgvELrv9XqPdnh8dNdsNCGEzbg0GAwAAxng8mkgpO82Gcy70eRAE24Ot7e3dvb09o+W9Tz8OgqAZ8rtv3cbOPDs+kSKnBJVl6fvbzWaslKnzBbVFqPfvdSZ7vYifG5ENjv36nZtAoTbBzWZzNpvV0vq1bmPN0I7ieDQaeZ5XMyiVUrPZLE3z56zClVJ8bezUVyOAL90wvgox/P0xx6sHhxtRa+sgIcgY6wDKSyXl7OT4/OnjEylMVSqtgQXIWCuNJphBDIy2hSgRcEEQNZttY9Q8yUfDew/uP/32d77ZiuO7X/tuksy6/f3t7e1vfucPf/vhL3/5tz/XVr11+3X6hAwvzpXMoIP9bsMYgzFVSgmrnYPKyKIoAhJvIra6+PHvPLWXUMLv+YApS8RI3YbAOdhue0ZX/+bf/Ls/+cEfYUQJIZeX2Q9/+Pabb72+SPN/+xf/BmR/Xy7CS9d5c/yDUMJLmPJV9FDvynCzOfSqhfSaEv98V6yPAwGAANYuQh1shwAgCFDd9BEBBCFGAEGrzMvf9eL86xeXZAKta5dmfZpLwAGfQ7rl87W+Du7lC7LspmUt3AAK62OZVRPtNZcIPS+tfKEUdgVElgmdlzDEc4jwoibHV92XGmPZJW3T1pHmqqoefvHYGNVqNluNkBEOwpgg5IyNo6iWbMnz3FoACfa4bwiRWkAMiMOEEC/0MMO+8S1wSZJoreWKoemcM1YBAKjVzkCMEMXYAKC1EmUhirLTaSGEgLGizHUtcYs8SjAAhgIjjY4op4AijWeTbJqO5tlCGJNki1xIxMi8WADH+vv99CKhlDJOwjAEABDCKDU+C8bjcbvZcQ465yjDCAEIXRQFfivM83w6S/KqtMD1+/242cqLwvd95+Djx4/jMGo2m6PLUW0VQ68B8DoytIxCWeMwxoxwyFGthaqUEqUQsjo5Obl+/eqNm9cRcmWZT6aXqHLGQiERdcw6ICtXlVYZhhCxiDjpy1xCjD3iOWCK0hqjCHGERn4QA2cUqxBwxipIEGI4CLx2u+mcW2TZfL4YDqeUMc45MWYZt9TKTiaTy4sJADAKG1pb5yDCCMA6M2coxZ5PAQ201loqB4yxWmlJEDLGBEEgy9JHXJRFu30VQfz+b38Xx02ljVaGYOqcpZSHYegsrPd+Y5RUDoDSOed53ne+870f//jHN69teZ5Xy+/kea6NC4IgjuOyLAmhACCMMYDuyeHjjz758Pj06OBqT2uzmKdKSyVVJXJZCSltllbIwWazOZ/PZ7PZoN+/c+ftr33tax98+jspZZIkl5eX6SKPovjGjRsHBwfHx8eXlxcnJydCloskXTUXsBJYqYUDivncCzzucaGKqirbrabPg3yRLqajR/e+OOcYgApCsx861m1IbQpZAeYCnGXFbDZfxO1Ot7PfbHWOT88uLi4Y5ZSQ8WTSbu/xLa61nozGJycniJL9/X0p5YPHD8qy5L63yLP54+Tw9DiO4y/u3dvb2UcIRa24gZoIIWBAWVUwB37gxUHUaEa+7zei2Lrqi3sfPXjwIGrE7XZ7e3e30463Bp3jo8PZfOGMNg4g6DzPo9RIqaVWCGNQZwc37PXafOCNLnmbTs/amqzM7HPGOOe8blFdizXleW6tDbY61rp2u11jiHv37qVpihCIm63akkqlN/08t/JfwFcAgi+FC/8o49UjQwgBIIxTo7QFlhKMMD4+Gf6H//gTCOHlcEo4y/IiKwtCGKAOAlRncBCG0liZllpLjLDnN6bz+aefPrp16/U7b7/Z6g6Qz0azBBD69W9+bTqf3Lh5PQzDJ08fXNnf6nYaZ2cXlxeXQRB43DcOMF53PFdZnvIosCsxK7jK8oAlNaG+NS8gg0039/eb/uXAGGOsqooxlqalpqDbiX/20w/ffetO4HNK6bVr/Vu3bhZF/ujxfSEKAP5hqQfwZcDlP+PWrPezzaXiVuwM8GLZwrrow666/61rQDbJDptTWm+oL+2sm1d+E7FtHmH9p03eIlxVqSCEEHRSyk0/3rolItfOwLqpzLq2aIUP1k3XlmjjxbqMGiuADTrn84OvyIwQQgTR5sfRRuHM+omuL8vzFwFYWwnnnFvpedTvsNZaB9YBlfF4nC3SS3IWhR7BkBPse5RTLKWC1kBrnNUQwYD7UTP0fX90dF6VKM8YRMs2HEIrY4zneVprJWU9Q21MnVxDWkgpPM+rmwDkee6cYYwYY6zT0NZVHogxEgSe53laZxBYBOBWfwAQLEt5/OwiLaeTxdjzw929Kzv71yazxSefPJhNk7jZ4Jz4fsA501pLqZWqa7I8Sn3GPABgvWyKIpOy4h7uRI0ag6ZpSjmre94GQSCE8LyAc66UCsOw80YHQjibTBaLRc0DJQTXNhYaiKBFGEAYICSrqioKIURZy9KcnF2+/8EnWltCkHUYQY9yrxkEl5cjypADuBTQGs/jIfcCQuhiaimxhFKf+86VSlRKWaWk73vc9yB0FmgKgQNcqmoyH7351mvGXFfGlaU4P7948vhwsci4FxDPC9qtTqfTM8YBR9JFlczT1Oa10K/neYzVGji6DtlJB+vrpbWydlkgp5SKgjCKIsbYWGlK+P37D3/3uw+3twfWAqsdAlhr4zHOCB+NRoskefr4iDHUavhxHF+/fv299957++07/X5fl5NsUUIICeOe5xnttFTT6XTJp3UOMyJldXx+dHp5AinY3mspZXwfYgJEaRFiVqGykEKIZDFvtZs/+vEPr169apS8uDz7yU//k0ZWSl1UBeO8vxW2Wu1mu4UJu/3m261Om3J+eXlhnVmkBYROSlm4ShlpofOEjCyglDrIKYFh0PSYF3tRxNjZs4dHJ4uvf+3av/pv//m3e5hxP6uqJycnXzx99uT0LJ1ejscOMej59Lvf++Z7Qj18+BAAkCTJr3/1q/FsUhQFBhAAkGSzRhS3220AwMXoLG5Gt26/8e3vfVtIeXx2mqQLY9Xx8BwAkMoSIeRR5nkeoLDZbbXb7UYULeaJlLKShXPOQeMF7OmT+yeUj4bnOzs70OlOK4LQcS84Pj6dx7G1AKxUDq0zSkirlhV0ayu2NoUvJ1k3xtrWrI3saDQCAARBgBDinLfb7TpDvNUKq6qqj1k/P4SQslTr+Hn9LXXUFCH0+/exvz9c+PtsP1965JdGVUnKiJIKAmAlYBQ8O744Ox0FgZcL3fbCshTOQcoYQBAhSACyFtQSUrUeHCEEQLy1e4A5I15QCPn5F18IUXa77T/eGjCf7O/v9vtdTpkUBUWmSKetmHfar8VxczJdHJ2eOQARAtrZuBXbF3PkdlU3jxBZnQV66Zb9gxAV4VRJCZxrNlvT0ag0wPeiyk+fPTt649brr712860337hy5cpsNvvrv/6P6h/eheOlWcEXI/C/5/3gK1DCl57mJv+/Lp+rQ/FuVVa6driXShXweVCh/tU4CyGEGGG0qjFB0D7v6/iC2CV4Zb3BVcHh8z+tQgU1otBaA2fWUAPaZaBCIwAhdPi5lELdOB4hBFfzh6tcwxqdvIRaXrqAbhUatNYiDDfH5tvgZlxh85jweaGNtbau19VaU7SsPK9RTq0B3+u2IbDAaAidEJWujFIEQ7OYzxljmCKKnUMYIu1MpYTZ6feNtha49T0qpVrKt0ttKu0QAhgtOU+Y4TpYYh10gBBIMOSMcUbyPMeYULosGeOcIwiMrhvmCOdMo9XMimqRJcIU2glr8GS8kEb3Bv39nV1RmXtfPJwn473uvscDSvFsNsuyQivr+yEAqNloUcrWCAwAoLWazaanT58BACqhjDEexkmSCLVUnIvjeG9v7/z0rCiKW3dvaa2HFxcQOAgcggAtFbiXGh6McIcdsLDQpSil1g5Dhigdj8eiejAeJdevHnS73cBvSllBiLQmAGIDgJQIIEpIRFkIHGxGCCEEQF3sYRBChGALjIPaAIOQIxxyxgBROi2FLTTMKi2SeVpVWulSqnw0PncWEgRpFLUacacsRRy3drb3MJrO53PggFBSK+Ms8AOPc58QRAgRUisllJIQYkqpMcv1pJTy/cDzOAR9QtgXX9wHANS1p/WjaowLAh9COBwOx6NpsxkihF5//fUf/OAHt2+/VZfbSSlasZ9mRS0PggHGmBJCEKFFUThnpNE+4BZaoSoLTKMVUi4xQUriWDCMlZYgL8u8KPr9zt2333vnnXcghL/73a8Pnz6OovD69esaQQiFEAIAVU87z0oI5s1mkxLe6XSMMXmeGaO0NkprKSRAEBJXlVqbDFFCMEfEMxZKqQOMW1GsO+2Y67u3X/+T7//xbnpkHVQQNxotIe2Tw9N84RAEZSEfPzp6481Zu9sjLGw2m424+9GHn09m49lsFgTBa9euBb539fp+GIbT6XSRz+JGA2DLPLq1v9Xb7Y3G41LkF/NUa62gyReJ1rrTajXjxv7+vkeZ1SaXRVmWoedHUdAJWoiBZHzpEKzKdHgBMGXtZhNCmBWV5zFCsagkgIB7AaYMIVQUhXFLWtzaEamtBlp1o16TFeonBG2U1a3NLsa41WrVxnc2mymlOp3OtWvX2u22yWZVVU0mk729vSiK4jhWShWFqtHD2qZvujWbnuKX7hn/eVvg32e8ZPSdc5gQSjmwUGtNuAcBNFJY6GRaMOIBhEspGOeMk7woOOdJNpVSAoBCP4qiCDWWaeO8qISUSbp4cqQ++ewzP+BRMwqakdaZQ/Dxk0fddutPf/RDo+XTp08PDg6+/vVv+kH01z/5+b/9i/95kmRSVQ7i7Z296TQxq4Zea+NurYWrHMRaf2J9y17FeX/HMAYzZoxjlDil87zsdrZ++9vfXb929Y033tBaz+fT4+Ojk9ML+591+V8Fna+++Hs+tT61L18hL+IAQgjn3Pf9IAjWWbZ1OGFzMa83TrhKsdUO36sLfnN7Xkd3Ni/1GnNTSpV9fv2fo7qN1MPy9q3ZPxA55/SGsuSqszQELybFNk/8eaxiRcXAqyZPLy0G+1wrenN8JUehDtE4t2Y2QACWwQ+6jl7opZo753w6GqZpQjGiOCDIIeh8jikhreau73OMcV4WaZoKXcyTzDlX5MuyqajZ6HS6vu9ThAsHM6mRBlYoiDEhDFroEGIIE4wgxA6YeTLFGFOKOada63a7WUMNQggmCEJYZ0MA0JAh6BzmTKelcdgYjJlHgFrMs9Hocng5euPN269duQo1ePj4sdYSBYG1Nk0XWVZ6PCKYWeM8LxBlRQhhnBhjfN/jnCMMnHO+73t+2NsaBFFYVVVeVp1Op157u7u7abKYz+drHrcfBdbaNUusvqQQIEocIcT3Ahs7Sphd1fTyZuR5zBqwSOXWIFSmOHr2dD6fv/7GLYyp1ABjCRALghhAXCkdUOv5yBiXF6VUOaIEU2K1NtYm2YQg6PmUB0gZhz3c7rUabagnVZKdFbmMo85rN/e6vTYlPilyKYVOknQ8nqSL3POiRmwWSeoALMuiqqo8L1utRhzHwGPOaeuWVF6MIfGYcw5aiBAqy9Jvtiil29u7nsdOT44ODq4OhxdOO4wxcsBpwzCB1i1m8zRJDg4OfvzjH3/jG99ACDmjojiEEM5ms2mVIURCP6Dc08qUeW4BYoz5vg8QYkYGAdNOI4SybDGbT4RTjPoI0VY76HYDgkJR2EVSvvnmO6Ko3n//NyenR5eXl0apVrtRlFlzsOV5XtxoRHFTSg0BMs4qo49OTo1RaZpleZ7lGYSQEMS4r42klCIKlJFpXgCEwxCFnmctqISsZMmRbTWbBzutbrM9ubh4/9//W2NBe7AdD7ZkadOkrEoQNT0Ew9/85pPpQkVR4/jk5N13373x2mvKUObRuBE2m82Dgz1K8d1372itR+PLqiqDKHjy5HEli9tvv/XWnbcPruydnB55vUFdTZDm2Wg2KarSOLfjtitl8zQdzUYYojD0pdHJ+fnDR/et1XEYe55nnMYWt9oNRLAy4+UTaIyzjlrrnFXKpHke8ah+2ldd0cza0LhVqnXTeq5N1dqqrnOivV5vf3+fUjqdTqWUo9Ho+Pj45t6g0+lkWba9vV13tDLGxLFHuQ+WCgGgLnmoNZ0Aoa/uAV/qcf6jo4RXDw4hNNpobBAhHBMEsagqA2zIAmsNIlAKXUnp+b6xNk1TA2yjGUkplTTaqvliBiysWfeY0qdHzybJqNtqzpI5os2f//Lnn37+6X/1428MBn0A7CeffLKz1XdWi6rY2R5oKcJej3vEOk0oIpAwL7h27dp8/sl6vwEAoNVdsM/59i/km/+hl8s5hxkLPD6dTmKfRVFwfjmXMaHINpvNGzde+z//n/6Pt27dLIqs14uyLMtm//kX+SWP9u85vZd+fQlnrPmJ9Sv17uX7vu/7lNI1SlgDBV0v6VeqHqxzoEYSa0ywsSrqsbmzbvJ7ao+/no8Rcr03r+8RBJYQUqtmrs7Erue/3M9XEuloJa1Y10msT9yuu1G8mEdY/wm8CMLW81xu+Rthht9z6VcY022+tDyLDe3UdbVUp9vWRkhRFkVmtWAUNiIehoEf0DiOPY+XoirLhqkbXWKkKliWpVKGcx7HsQMwSZJc5dABZ6zTxgEMMagb7XiYGS0451LKNE0559vbgzAMhRAOmNWUVkJktVIyoY1WXJWGcGYh4L5HOUHUzE5G/Z1Wg0Zllc5G42bc2tvZLbL86f0zjLHHo9rucc4ppXW4vSwFIco5TxsZ+T73aJ1WYIwBiBvtFvP4/YcPSyF3dncRQuPxNIqibrd7cXHx+PHj2jWaT6brO7UCsh6lyCjFKQ2CIPR9a4DWOsuyLMuMlc1GLy8Wk/Gi3ZgB4NJFWeQyzwQhTmqbC4WJA6i0FiyyAjc86ByAVptSG0EJsE5IWTrs9KQgFG6xjnO8khVCoN1t7R+0ghDPk7ExmnEIAbPGBbxBktHkyNh+f0tUWpVSSZvMFgjQ8WQcRGG/38/zdDgZeRHvtdppmhjJGQkZghDCqhTOKkpIEARlWQIMsyqLWp6yBSAmrabEh4yg8XB4dJZACHn4+i9/85Ph+UWv3/iv//mPbly/3u2EECJRlLJInHMMGQtDzJiFMC2EdZp6lGGkrEAMz2azosp3vV1jDCau0Yovhudm0mSMYYxHl+etln733euEkKqozi4ejC9n4/FMKxDGTYo4wVQr7/Jw2G63Gw1ogat3KeFluecxxoqimCfzqiwYoYQQqVWa5tgWALIgaGaVaHe7SZYSHwNu5uVlwChChvhcCOOz8Iuji6uj/KePnp4NR1Gz19rae3Y5fjzXwo8zzYzBKNg5vMg515HfeXh4/vn9Z3me5Zf5W2/fKrL503uPm2321//x/LWbe1oe/uEf7P/4T/7sw08+hZCHTAFV5nMBhG3hzs5rO4vF4uL+xeJsQbs03m+qHJayyjLBceyAzQudZnI2m5USYe7H/Z3BYAAtTJIkW2QEsYPtK624GE2mnHAvjKyDwkqhpAVA6rrXKuXYgxBqa7Q1lNI0z8CK0rXWb8aUMEyWDKYVq5xRWovEQQAePnhQVVW9PPI8D4JgL0Df+Pa3vv+9bzkHt9qNfjPWoii1JcBZB+NmY3//oNVuG+PqdpSPnhyORqMkSWr/oM4rU0qh22hPUEdxl2yGr7RyXwovVu9fb6XPzejmm9ebgedR5xRwwDonDXDEIQAKkxtjCCCc8yAKlTJS6tBvQAPzTNQePyEEY2CttUAJZZVB7W7LGDOcJgB6k5nCOJjM9F/9u99+7zvf+q//9Mcdf7fMkuNnj0AB9ju7t27e/P/+T//2L/7f/5+zZ5ev3brxL/7Vf/fGm+8ki+zGrTf+/M//PJvllDAIIWEsTXNOqLUWQYwxlVIIocIwZISnaarhUuUGIWS0sxbWDQTgSpICAACgBasGlZAKq01W5thDEtikKgkni9K8c+f2aCHeCtt/8IM/ee/u20WxmC7m936aIR9SSqVSCCNtjVCKMCK1QBQBYJ1D0CHrLLIOWAgAME5t7GFrp/XvZrCu7svLf3npowjRGgEYY7IswzWvjRBjTJ32QgjVlClKqda6ptAbY+wqvlVH9SnGAACrdb3r4/XrPq9d1c1maWAVhFs79OucPTIAAWgdtAihdToPOKXUmjthrXVgWY2M9YYjD4FzwIDnxRr1QBA5uOpTaw1CiEDkgLPOIojqdIs0GkKIgcMAY4zrgANYxR7ASk+6frgQfoHTAGqJRAsghBp4q/vlVuEEA6DzPFaVOeeUkgDaimFCOGFYx4iNKzVoxr1+4/z8JMvnZUW1kelhQQjZ293ttkPuQL8XD7ZaSleW71BKlRBpmkqhjNamgjNT6CIDWjhnrLXKOQugNU4I1cQEGAgMwABS7DyGW00fIT/LFkmSlFXleV6SpAih69dvOOdK2Vmkaj6fK+OC0BPKxnEry7JOe0cVRillQHPmkMqHzjlXBdf6rzPMkkkyvZyFYTxotH0/nE6nURQJhhEkHqbGwYjGMW2WutQ4KYtsni58H0Ok02QqnTk/fzaZzpRSZRER7DDQ6XzUbbWBR+4/HhVFUadp6nK2fr8fBH63202SBGPUarWklElSYgKCkOu8DLC12F2Ohp/dX0SNEEf+oNfKrRJpAQGmmDtrgdTOOJMWoIOlNFVptSHWeXluEMb9nT6hCqGMkGr3QE5GH1VmXOQZoHY6d2Ecv3bTGjsS1bDT7+5dbTJqCUKoFmChlEIIZ7PpbDazFmxvb/th0O/3paxOL861cllaOUs5JUZJqTQAyDoDnauVBxtRpLRwzpR5DqGr993pdMopWcyTyWSilLEWBEHwve997/btt+7cuRMFoXOgqkqjFIQQQgwh8Dy+Vv2r/VrGGIXu7Oyi02n1twbPnj39m7/5mw8++CAr8k6nk5dCG9NsdbcGb+zs7HQ6ncPDw9Pjs7IUeZEaWyFMCTHAikoWeTHHkBljkiSp9ZVrC44IqR9pqVXdyt0YY9zSGxBCqCSxwJV54ZzL0yyZzW/ceK1YJPP5PKBdZ41zTkgJMVIYVwakk8kwl2fTJC00j1rMC7OqqrPUoiiN8FqNOPC8gDGV6FJUSbqAmPlhZAxBCLWb8ejyNJvP/sU/+zOl3TypbFUsprNb1/eHKZnOxsfHz/IiiaIgigLOSRT5UuJCZJhBn4cIgfl8XlUZhCYrcms1xtgYNZvNjg6PPC947frNJM2UUtZBr5Z/NwiAGqejSiotFcXE8zwMkdCmEDKKojqioJSCDhhjlkVHYCkDvI4lgI18gV01mawNMaX048++yMri7t27u7v73W633W4HcXRw5drjp4fOQoBRnudn5+dpmmNGW61WrQu59tjWJIkNj/lFXxm9EHt4dbwUAvn971y7jOsXX837brqVdlXOvv7U5mfXr8BVt4X11lIPhNA0Kf+f/69/+/HHH//pn/zwO994T4hSaznY3nly+OxnP/tZmqaDflwWmUfxjat7v/3d7wI/ZJTXXUW00LJcWGO8KK69WYQAtRgAyxiC0FqgVu2Gn6swQQhf2lxfOEH7AmACEENoHIKtVqcOmb7xxhuc82736qDXZ/ix1MYgBCE01hpjCMYIYaA1YAC45b1ZqVxZAF5wTf8pxrrWF7x4p15951e9vvnXtbO+5gTUymMrVaXna6bWBXmV07O5AW+QfupigecNOBxYFU+uFJDWT1Z9B+u8yeZ5rQ5u6ujFOpSCNoQpN+ewnu0GEllGJgC01lpgIUKrR9g5WIs8g6WyJMLL+AVCz+s1rAXGOKOdksYaYA3ICmEcXGR5pYqyquJmF1N+en7JMC3nqTEgyxpSZeeXuHkaKC3G6T1RVlmWYQgHg8FgMAAAhUEshS0vx5zwIG6Mp0khZLvT85nvlM6zEmHQanUIQUmysNa2O82qkkKoxSIzxsVx3O9v9Xq98/Pzw8PDuh9EEAR1eIAxz/cNXuB6r8EYe2HAOa2DmpAsoV59GevqLUppUeTOOeYRjLHRWhtprMIEZlXFPR6YAAIc8GB/f3+R51maB56fKjubzKfTaZokg94WAGgymcRx7JyTUpZl3SkpqJkZ4/G4vjtpmpZlCVeMrkZn11pLJGk0GtLK6WSOKIqbrSiKPB5gzGSl8qwUSHPqceYfnRwbC4DDFhAHDPdxp09296N2l9248W4UwsDH5yfPTo+Pnjw5VFmWx3x7r4dZcDGemtkC+1C5YjYdk0oUYRytZD6VUopQZA2glEqh00VOKfW9WEpZ5AZjjLCsRFFVFcOMYAYYthaAFQTe295VWoxGl57HQz8Idnma5rzvRVEjCIJGo3H9+o2D3b2dnR1KmRIyyzKtLUX4OTFkFbWrH8rV9uB6vZ5SajSa/OpXv7l37wEASEqdprkBJo7jvf2dfqdLCJlMJs+eHU9G435/q9kEYRgz6hHCVaWytChLIUudppXW2sFlfxTnnIWg5tyRWoIcAgAsQoBSbIVT1oCqYr63WCw6vW6z2eSczyYTK0RVVQjjbFGKwPc81t8ahFvbvtAiKwHl7a0tJm1aqMlkRHlAKKa1Xp4yVZbbqnLOYY9MF8loOgYoDCIYVFhKGQRBkae/+ttffOubX/e9uBWW83n2ZH7R6w6ara3J9GI2H2MCmq2IcYiJC0LmhzhJocuVdk6W8nJ8NplMLATX9q/MkvnZxfmgt3X16tUgCI4Oj46ODxvNNgbQWI2BA9BZbaxWxmgN9DoUVtNUa4NVB8rchrZPffWsMXXEzG0wDAAAxphanKQGXu55phacnZ21Ou1KqOFwSD2ezcuaZTNdLJIkEVJJKSulIYTz+ZwSvlb/rXdW8GKEGb7EYPgKWw9fJLv9/l3hq/YJsOqVAF7BGWu7vI42bzItwIsoob629cW0G/WlCCELcKPdenY2+h//b/+PR3/66J/9+AfXb791cnZ+fHz0+PApcGZva3c8mz7+4uNuFHz68Uc//c2Dh188CIIgbAWaamttrtTl5Xmv19NaQYitswAa65SzUqrcrAod69u1eXFeOuXlK8YuRXxcXcBfRx/Q8elJUSzKfHGwvz2+hO/cuU0p5xQYgMq8chAABCwA3OcIUUk0dLBufFirIkNgXV3j/k88lhrXK+rMeoN/4RxfpKG8dCngxnip5hBCqKRyG9ScNaKFq+zPS1hh/f8mXADAKaXApnoBWK0KYF/KI7gvQ8kbqNRtvrge4MuQsdvgKIAN9GBdTYZYHgMAYJxDS3KEXq5th2szAFbkD0IIxhQhQqkPIYUQe14kDOBhzD1kbKWyJC8qiAhnHoRUW5VXmuZSKi20qSwoyzyruJayKCpnrLMTCHCz0Wg1m51WN44bJ2cXizTzGSUYizxTZbXbGyRJihBoxC0H9Hg0KcsyiqJeb9BstoNg4pzjnGNEp5P5wwePlRaWcYKxVbYshbXaalNJYbVhjDGP12wDCKFSxhhX61wihKIoIoQJIepkkLXQ87jvexAiURkpq6oqIIRSG76M5Mk8z60xBCJRVrnRUmiKcRQEVhrOOfV4FEUimfs+RwhYa33fDwJPKTGblfv7+3XoK8uyosjiOA4CDwDrc5KmabfTfu3G9SSdn15eGOMY4aLSYej7LGLYUii0qNWpHPVibGs9U1tUmbJplpfzZPG1r33z7p19p8tiMbftpll0nhaP7n34+aLcw7zr+x6P2r52wghZzYUsCSW80WgghCpRVlURhN7elQPOvShsTGbzdJEbYxqNFsbYaFtVVVWMlCiBNYQzzjG3CABECPEZtRZ3uq2Ls9NkPiPdNvB5DeUwRMaY3d193/fffPNNI3WSJItFarUDzxuJrh5a6JxzEDrGSf3gaa2l1oQQBDFw+snjZ598fNjvhwTzQX/n7nvfrLW1hxdp3Y1ie3D9+sHbX3x+DzqfIAIcUhJK4YwBzqIw9JeN1BBc5903n9u1QTGrNGGv14uiRhTH9x48WMyT2WTKGPvRj38IjL44PRn02lNGtvq9dDG/9/TxsNQ5pInMisXMAISIBxDigU8po4RhCLVURiphlXQAWAfjKC1SoSoDQ4SptlYK3Yj9IIguzy9mkynuEYYhAsYjoEinsNflHm61I+tkkjhrTVYtFsU8iiKAndDVopR5nk7TaaEKQojQotlpdnCv22r3t/qcc4yJx3xtXFFURSU8z5NGA2eANRhCKURd0kMIKcuyKksIoe95eZ6DjS7Ptd1CEIIN+1tfwPWGalfJCLdUdKcAAOYxznlZlkIrRMmf/Omf/uQnPzk+Pbn/8NFikUkNHAS+zyilxtpKCCn0urf1JlNs00/atOz2q13UTUix6aL9nW/efHF9BcDG3g8AqCkdm/AFbhDdXx3uRXdzvcFoQBwEGHOLyt998sVoMu63G3/0ve9ki+mVq9cvzk8vzk873e5Wu3V5cvjFB79NpqrdiD3PE0VeFEWn04kGvdlsJqpCa40pgRA6YIyti9w0gjWkq+f55RMDm8lvAyBAwNVlqsgCiACCCJ4NR8PhxeXF+e2b19+6/brV7saVaz/4o/nZAn3w0YeVcBjWWXSklKGIagsAAMgBAC1cpooseKEa959mbCTsNyNemzv3+i6/upVu/gmuAMfmcdBGWRBc6ZKtY0WbrafWB9lcEm7TjwebTILl22qcXefd1ugEv9gde/OYnNNNPIQ2SiFe+sb1KS4vxYqyoLWGCFhroUGErFYmhMuHHikAoQPQAeIAss4haIHDa74kAKDuQwsBCvzYIViIykHcaoUOtFVVUUqvXLmSJLnvRUEYA2h1hcNm6Ee0MCPmGCaeg1iJXFmT5QsEDSbw6tWr7773x++//8H/769/QnnAPDYezTDjwHaVLIxRZeFRhhmFzUbc73U8zwsCb2drAACqIz7W2uvXrnFOue+tboQxxjgEKTacen4QNJtxHMeUMyGEtdZIJYSEEAZBFIYxhBABWCcICEIYY+dMWeaVyKGE2giEEKXYIQehk6IsimKxWPhhEHlemhdFVbIg6HW6PvcIIZTiqBkVqtxq9Osogu/7hJA8z/M8B9Ami1lVVUVRAADCyMcEUoaPnj2dL5L+1pYXcCk0dIhSbLT1vIBiprW2ylBMADVlViql0spqLRljQch87kutndYiXwzPT8XNXQIsEDDGDaLnEe5VCn326eTy4vNev9WMA4gbeTpCEOwf7JJWqwMAEkI0m00I4XQyu3J199q1a87CRZYfPTu+uBgaBzBGVVWk2YJShZHGCFECMXLaGgwshphxIspKS+EH3ptv3L569QBjSAixAGupkiSRUopKnp2ep0kyHk3b7XYYBFHYIIQ4a7VW9bKuYz7WWryWFrcQY5gkC4yxBQ5jOhg03nnnnW6/d/PmzariZX42Hc9m4ywM4ka/X+Xz8WWezCwjBGOotZJSAgMw5r4XUmoAtIQiWOcapAQA1skOraVxGtUNwTCz1iglIASLxeL09JwyNp+n165d2d7dOTg4+OPv/sHZ+cnh40effvppFAd37rwlVPXxZ5/eeufrvXny+PHTBw8fLxaZF9C9vZ2d7b0nTw6RQ1VVlUpqJTnndexrVGaVLlnAm+1W0IyZh5WDlAfd3na70yPUl8oNR6PhcNzu9pRS4zSjmDQaDaWEUqKUYrFYPHhwP4yjPM8zkVdVpY0OmiGPPGutRXbvYL/f7UmpK1lCAq9cOWg2W0eHx61Ww6skxETnGgBLCbHGVMqYVT/JWkLROSeEqFNom3pzepWvXXta9sVWQ8aY2rrVXkutqC+E9MNQW7u/v08529nZGU8nH3z8SVE4iIAXUISwUFJXsmaw5mnh3Msti17yz14wfL83TPCq3TcbipOvvnntY710hFf/3yTNwQ3/dR36dhvcsU3Lvj54/QplbJHMfZ/3B9tCi1/97sGNK51vf++7N95468r1aw/uffarX/1yZ3vw/T/+w/Pz088/O+pfe81nBCEkKMqzJJlPWp02dLougCAIIYIBcBACQogX+nVXcGuBtc6uVAa+6mI656BdoR8IjQMQOIiQA05ba4w5vxgX2eIbX3vXWnv71pu723sPLxfHx8+G46kXhFJpqWwlFeM+Wh7TQoeAMwBa6CwAwIAX7uw/+th0qetFWwsBbe7Wm/dx02d4CUBsooSN96D1p+yq10mdlFnzGddL4iUgsr7sq+/aUCUBK/KsXXb7XCOGl0APeGHXf1k6ab3e0IvFGq/+vDkZYIFzDmJkjFtiDoyXpcrQAlAXUNT4RzuHIbRlaeuwYv11WgPnHGPE84mxcjxJAIyasYcCH1pd5DMEYDOOue9PF0lZVe1eG3Ov0k7JEkBrgDBQUg+1O16r6SNopJxg0r/zzsFovP/Z5w+ApTev9+NGM0uLdosaAxgTjUa0t7u7tbW1u93J8xxYWRUZIQQAVGYlY952f2DAklqR53mNpJ3RyKEgiKIwaDc7rU4bY1gUhdUGAbRQeU2WWuaStCEE+T5XQljnsrRI09QZizEWooQQ9nsd7mFtoJA5IeTKwU6z2RyPx9YoAmPP9yHE2pAgCBjD88WUUtRqxTU/xvd9CCHGoNWKfd+fzcbj8ZhS2u12e712u93WWkuxgBACa0eXwzQvECZhEGrjoAMUQSFklqYQQqeNdabRCICks9ksyzOnAsqc0s6Uwkn7m59/HCJvp99rRw3s/HRsW/5BuLX7+YfvZ9NkeJy3O0EjxgiZ/lZjb2efQIimk1kURX/0R+9IKf/6r//TbD5sJeFwOAzCGCBhbCG0gRoaq8MIWW0lskqqsrKyKpQyzkLOWKsRN/rdMPAD32OMhKFPEIqiKK+MoVpJvUjS6XT65MlhLUBLKfeYDwBQSgEDatk+59xisaj7GjOPR1HEmV/b3GazmSRpslgc7F+9euXGrVu3pvPZ5cXkyZNhVUmEUFXqqkwWSXl8fDqbzHu9HqMMAKCscw56nMVx6HnefD5cm4/6wXbOYfJCQHKZhqyfOod83/e8YDAYbA22McYnJydffPJZMp3u7m7v7OxYp+I4LEV1dHZ6dHz4x9//s9lsdno+YV705sF1z/PLshwOh5RihEitWiSEQghZBCutCpFbpxvNOIgDQqmBoBBaOlwaGIate4enHg/OLs6rUu0d7OdKpWmqlLLaWAso5Qa4LMsuh8O8KjzP8wKfMeZHvKZrlVJ0om5R5WcXIk8LI00jajJMRqNxmmTOWM65ECLPFrKsnHNOK4ZZVVVZsiCE+L7PCZVSFkUWBMEmOXxpj8AL4gpri7O2yHX0bB2vE0JEvlf/EEThycnJ3/zkp7/67cdaA8qABVBrbYGuW8sqo21ZvOTWvxThf9XS1f0V/56bx6se5Kvv3PwIWEnhvvqGl972pdbcbQSN1+9/6boRDHq9TlXk5+cXlIBOJwibrY8+/vTKlStVnn3zu39w592748uLSgqM6dYWG14elaXmPu73+82mr6Te2e6F4fWHj56UUkAIa88HIGoBBA5DWLPBl+T91fc+n5JbBg9W17M+o1qnCDgAoHEOImABoAQDoMdTiSGZT+Y7/V4zjI7nVVlkWgFgrFEWAggBopiu8kcIAuugRc46CP5LpB5WA6yalwoh6piie+nMv5LuuryheDXgimJirYUrL7zO063br2/GKdcQZDOGZFe1KtbaVTnic2AB4DK6wDnfBBzrRNXm3Db/X7Ml1oihHnijd4N7JQe3vsXLa1K3egG4brRWZxMRsgghZ1cZqDWkAfU5OoStVtZaSSl1zjiAPR9fuTLY32+n88vx+FgUE4JcVVXZPN8a7GsJtKkWyTwXlXVdRAJEMODFzvaAoObZ6aEoh5awvav9rUFneH7+6af/8erV63/8RzetuUSY3X3nbSHU0fmEsYZzTkpJCGq3w0aDSDNVphqOF/e+eAAhbbc61qIwiBljUi7LLyfzmda6DjA7CILAs0oTwurHNsuyRZZJKZ1z9ZqprxUCrm50Z4yB0AlRClGGgef7vHY5SpFGMXdAGlv1m4O7d98MPP/nP784ffbo7t27b999J1lk9x8+IJx5vl8KLiX1Ay6EyIvMOk0I8Txvd3f32rVrYeQ/e/YMABDHsR9w63RZ5YHn6WazLMt0ngqtCOez2SzPcwvsjRs34tjzvJARJKvSAdvrtZvxtSdPnpyenjkLrHaitMo6pNx2u/fx+08/c4/eeP31/e29ZCah89LFpEE6rcirRHr2+DgJwe5+k7tIZnPSiDtJOi/LUimFEBAynx2Pzs4PhagaraYUqqoq6vkQ4DCmnU5vci6VKCtVOqMg8ygmkGBKse/xrUF/sVgIURKIHj+ccM739naanZ16uadpijF9/PgBtO7KlWv1s1SWQinFCfU8Tykzm83OR8+qqsIYtzptjDEEGCFkIfI8L/CjeZIiRE5Pzx8/fjqdzWaz2fWbN8uqxJhm+XyxyBEkmNB37t5WSiEApaosMITB0MdBgAjRhCDnjFICEbx6IAFCqx7qlICl5raFyBGKZGm01nU/j29+7RvOOVGWVVUF3Hvttdcwhk+ePjw+fnZ2cco42b929Wc//3VRFNbau3fe+eEPf4gx/uijj46OjvI8r9UeeRQgrrTWVaWEEJ6PEQoHg04YR9oBK3SSVe3KzjO1KBaZ/KLZ6aaLvNFqHV7O87zELnTOOQshwFLoNCuFEs7BspQWAB4EQRghSsqqKCsBAKiqUkrRbXW0Vc+Ojzzq9bsDWcrJaNJotCIvkFLKstJKQIillJxwigmgrobPmCJGKPZRVVa1ccQYY4oIWkKryiwJBGjFulob5RpYQAirqqKUIoS01oBCqZUfBlEUIYI/+OADQkCn28iyIivq+DTABHjc01pLqTzC19Ztve9uxjndS97wV2wYr1rG3w8UvmrPWAOFl/5a0yw2rTN4EcTUVv6lPNf6DesNoJhPtbUAgGYj4Jzn6ezzz548ePBkOBmrqvzhD/7oe9/5hgPw6bNTz2P/m//d//DXP/1JURTDybTIZ1bL27fe+Ff/y/9usLX7f/kf/6/T6bySBiOKGQeIKKWksMbVsfHneYf1JgFeQV0AAOgQgmiZMHDWWAcQtAAySoDVjYafqnmSJItG9Pr1a6Ionz19PEskBEAbKZQJ/FhpBR1a8SEsqBMQ4L8ESlif4DoMsBkMeOk9ry6G9Yv1fa/j/8t6xRW/BAJYR9o2UcL6+JuHXX9quRJWhZnW2lW9g1oHJCByS8aDW0YC1s9X/eua4buJEgAAdSP49Z/W+OalU3Mv5uBeWgNupQv5HNxAiFDd4G15WOAQJrBGkgAACB0hyFoNoEYYG6MdMJSBd999favfElX6+MEnx4f3inyBYeTdvBLF3fOL+XRRcOxAQAmFlIC44Tc5PLjSUbI4v1wUqnRoxoOcBbTZlhcXl5NpdfXK9T/503eUNBir2XS+d8C7nR6EcDgcLhYLAJHSWgioFYCwkjKTwrQacbPRjsIYYzyZps4BShnDBANIKU+SRCmVJInv86IoFosWhLAosjrshAGtO+i6uqSLYKVUXqRWaUI8hADGMAh83/eVUgghiKusmBuj280wDKjROXSoETFODEam1fAb7WCaDKfJHGHW3+moErTbrSzLIHSe50kpnTOexzqd1rVrVzCG4/G4qsr53CmlhsPhbDi11hZCAggb7ZZWMs2yUpV7eztxxBiHUch6/bazssyzdsePWJXNlRWY0ABTX8oOQjCMeKvtnxw9FmJhtCWMMo4YIw8fX/7Bu28Ntlra5OeXj4UaR03g0wpoR27duj2aDD/66INf/OIXlML5fAoJOjl92mw2k2yczFNKaXewpZWFEHo+arUaRZEVRUEJaTabgRdQwglhVVVMp+OPP/448oO33r7d6/Xa7ebe3t7J+bTMiywrZrNZFEVa64AHg8HA8zxCmNa6LEvInZR6PB4/evQoyYcAwEaj0e52VjrKWhqNMZ5Opx999NHHH396cnKitW53up7nPzn8dLFIe91Bt9Xt9na0NlLqdtdrNAZaqixbZDkEwIV+vVFJP/frjbzu3Fonzgmldd0H9706yielNM7WvgIAQEp5enR8cXExm82ePn0KAFBK7u/vvvv1d/f39wnDaZpI7bRVwBFGAyHL8Xj64YcfZln2+PFjqSrGmFASAGCcxYgSzrzYp5EfRYgQsjPoBz6tRKmVlcZoAB1mBkFAfUe8wuQc0MvLCeO+bzWl1PfDvKzyvBwOx17A+1tbcbMttQjjKG61lVLD8SjPc8/z4v72bDKFEA4GgyIroYW+74tC1E1KOp2Oto5xqt2yCNsYE8cxpbQsyyzLpJRxHMdxPBqNaisGNuSca5pPbYzqJP3aEtWwoMYH9Rs458YYKRdxHO/v7zfbrcFgcD681BpMJgtpAMGAM6KNVcpWVYUZ5YGHDVo7UnCVvwCrnhTgFaAg9Vf2Gvgq7/8fNNbCNeBFl3QtL7F5zM3daNOP/NIX6xFQ1PJDpZTVMpcVhLDR4karn//i0/29+N//h//4xRdfvHf3rbgRSqWuXnvtvXTcancODw8//PDjy9Fsd2+wtdXP82x3Z8daNF+kmHiUcoBwlhXWQmOXXY/dUgJuuVu8ioqW5wuW/Q7WkQfnnAWgEJXKhd8NCQHPnj1rBQGGpKqqJJkRBNrtQGlQlAUhBEJdliUmbMlOgAAsC/3AfwGOQr1NrntFruRyyd/9yS87Tj3cK5Eh8GIpwSYqXePm9b2ux6tAoQ7C1TkLhJdU6yIr6/nXXK51BqGexvrXTTgCVtiotm/1BzdTYC+h2DVQ2FwGCKE1lxwscfCyRqz+KghrRUFcf1Rraa3RWgFosAFSVgghY8XN67uEQGBgt/XOe+9czeZTj/G93Suj4fxXv/nk4dNjykmpDQRaqgIhe/vNqxC50ZOLQmS7+/jdr72+tRMtZmfTZPjWnSsXZ5fjyZP33v3mxcXot795PwgiGLSpJwghmAllFkgpz8eeH/BWuLO7lWXZ6cnIWJXnWVVKIRRGnjHG8zyMCaU04B4AqG674JxjLMcYOwTLstRaYwxrC+d5Xn0XCIJ19EKWFWPEOVcDXmutMQoAvLXVGg6HBNHdvYFR+smjeztbu7s7gx//yQ8ePXzy81/8ZP/qFQdUls2FKoI4qiojpW+tJgRhDBeLeZZlzpmtrT5jpN/vlmWutfR97nlssWCtZlNpa8AibjRev/1G1IiNtVHsMZ8BaA6fPaiKpNNhUewjhAjVHps34wo6EgQNQptloR0EURT85re/AFC9/vqVN9+67XMmbc65b0D+wz/8Wq/fCmMs1Dvj6eHZ5X3l5v2tiDw9/igMw26fT2enFlpAlNIKYjNbnHPOGj1irZhnjyiljVZDIbfT3up0+2enLpkXvS7lzMtS4SzAiE+n892dg063TXloXJGkJZskWioAkFKKc18L22kMEMIY8CKVoed8ToGx82SaF0mR5QCUGLO6N/RiljO82N2NKKVFMcO+FzfZ+flDxvOwWe7tbc2Tsed5Vzr7eZ4753YGca/XS9M8y7I4VJwVJMZhw2+UAGPKCc2yfDhMkixJ0pnvh0EQJvPcONpoNBFCUUystRgRzydClIt0ppSglCIUURqHIS1R+Ytf/8YYhRDc2toSsvzi0b2oHd65+072t3lRFBTR2WyGMPYZjhsdZeTvPvxICs0YazT6ZZUTwmrQjQnyfcI5RRhc293pdrvG2dFopIoKYISgJySUCvUHgyiMF0lhlJ1cDiFGuaj8ZgdgCxEhFBigjBEUh51Wy1oLAbYQ2NKqytgcuhJ7XgSEeevGG4SQNM0Hg4FSJl/kFsGg1Ujy7PrrN8/HQ6VUmWdx1CAQtPvx66+/QQipqqrf7zcajSwrqqr68MMPCaaPHz8OgoBSXhRFXWUEgKEU1+ellDLG1OZJyiqKAoyxUsr3uVIqz1NKabfbytLitZvXr1zZ/83772OMtQbcgxhDjEhWSAAQQRRBYkoLGDFIO+cQRozSmjjGMK5bgWw67mv+xFITvm5RvZFRrm3fuqCmRjDW2nUZ26aRrX9eu2J2o1Hv2g/bxAr1YcEKytSVxkslNc4hhGtPEaxcw5dIkc89RYkKrQFCDnEHLQSu0Aghn0Z6oZCYlqPs6fE4e+3a1V6n+/B4scd7bgF32/uzvaTT73zre+9VJv2f/sNfngzHpXaQc8o832NaVs4mnJe2ClYKTBYAB4CD0AFoKEF2Yzy/DsxZq6x2GNGGH2JMyrwoyzIKIgPU5SRnFP3isweg0QQffgghHJY5iMFpUsQNbnyw0AveDPIkN7CuhIDAOAgIcsQ5BwFA5MtTSF8FXL5qfBXmw6vumnUtXL1r1om5OjFsra37/dSZsvViqG99zbOpPRbnXL2duFWdsNbayJVT7hB0CFhbd09YVopY59xzlAAAEDJf3mizBAd1jKF+duo1YoyT0mhdGWMQpsYY4AxjLAxD3/cpQQAASimsCxABABCiVbILOOq0M85ACxFFxhlgrFl2HtdaO1Br5ZElVXxdGF8JUXfJwhivmStuVSIOV/0wOQYQIYIxQgABAJ2DgGGEIaFCYM7bEEKIWbO9rZTSNjs9ex84ky+Sdju8/cZbcEfOp0ML4GQ++ujDX7Rbnd2D7aKQl8NJh+++vnu96bksnfY9ufuN3atXm61WVsyfmSrfH0Rpenj1oHF6fnZx8TsDzM6BrMQ59YeXF59TwqHBFJdxMNjdIhABRPR4/Gxn3zu48pasyHQqyxzEjYazAmLmgPR8XFXFPMnDiPpBQ2tdlGVZlnEcB36cpVWWVo1Gw8OIc78oS4TwwcHB6ekxIMhoZAhyjHEAm70OdGAxm/ue53lePk4PenvO2EGzQwl5/PixFPlgMLh2fT8K/C8ePfjwd782DlrgqiJPZ7PeYOfw8F6r1fJ938Fqe79rTMsL+LPLw1IowujClIKjW7dueoHfvHKQHh+laf71/tf2r1y7c/ddAMC9+/e7vXZZpYdPH/32F798+80b2WxkSxw3fJ1PEhFw34NZEce4yMZVXmz3dx7d+2x2Oux1uslJXg3dp48/+eBXXyilOCOnj5/aon9wZRtBGOPGlcGNNL9AQpJf//rX7XbLGFNWudZSO00IDnzfOAqAMVIJLZ0zCBKrrRLq9OxiMNje2d3vdh2CfJGUQmpGeRyFDcKU1A6ASgoHkQM2LwtrQZ3jkVJjVGNbM5lNs2fpdDrt9jo+o5RRjKFxVq1SfYxaCGFVFaPJkHPunBvPxp1u++7du2mRFOWNra1ulqdlmRMeGWPyPIcWGqM8jxHS8HkQRTF0QFSqdiAYJgCAsow8nkNkpCo8yzwfY8SCkAOHMMbaGkKIz3kch+12ezafTCaTRqNRNxgF0CKEjFHGmLIsCUUnJyeU4jiOkyTJ89L3LWOs0+5nWZZlhda2TjgJIS4uLlqtFlySoUhNNPZ9v9lsNjtRt99JkmQ4vHjy7DCKGu12ezAY+F6gtUnTQkqppK17PCqlOlFPWmttVRYCOcJ5yLmPITHWtDsdY8wsWSCID/av1MHJ11+/1uv0fd+XUi4W2bPD43E5FpUihFDMFotFHWEriiKO4263u3/lCmNkPp+Px9PFYuH7flmWxri3335bCNFoNJ4+fTocXhLCIISNRiMt0npXBhuaLet8am1i1s4Qxng2Hfu+H0VRVVXPnj0zxjAGnHNCON9zlGKlDCEcQgwhJARbt2yXXNvB9S5SW7f11lIffJM4tt77a4tcRyDgxvj9u9Hm2+rvrSewuYnCFUdsnXHYJCXUr2xuEuv/f48HD4CtS9aXv0AAHXQQOAi01hA6W9mLiwstpH3NXrl6oK3hLNze3op7jS8eff7BBx+UUj07PMKswSh2lDLCEEJupV6cLOTmV2+e5uaLm5GS+g1ClpWoMCKcsjiKqqqCALbbrSDwHNDD4TBJku3tbUo8hlmFJLDQ6Zq7iHkYCiGAQwA4AC1wwEK7DCr8E5MZ10U39WaMEKqqqqqqNR7avDv2RQXD9YvrHX2N9upPQQiNlfWbtdbaaG2WgaVKVOs5bF5w9GUK0HZVpl/H4YwxQoi6wYFUxjmHoKv7CSmlAp/X2zlyS9z50pw3j1+f3DJqtSQTPNdqhBDWHIj69TruYq01FhhjMCV16MVa66wlBDHGjNEIvhyNqI+DEKpVTyKCGWNKqTRNv/j8yacff5Tlyf7u4P1ff9hscI8TVZVlWaa5DCI4z0olrXIIEBa0O702P9jbvyF3i+rY90tCc22dc8qClDJbqSwIeCnLZrOL6dyDDGLDqOIs0AhzbhGClSyVyqSwpZSUhFHMTECVMtZUzgE/CGoieSXFcDisqgJjKrRaLBZRGFLmQQiT2WQ+n1prIXTGiKpSns8XebZYTP3Ih9BV0rW7g1azQQjptTtFnj6VJYbQ86jPfCFThNBkermzs7Oz0x+Px5zTuNn49ne/df3Wzb/+m59++sXnQmov8D3PO78863Q67W5rPp93+r0f/ehHgR+dXZxPp/MvvvhiOkt8P+jFsU8JR+idN944wWA0GntBsH+we3CwlyyyVrfjBWGr2zk5OQqj9iyt0DDxPbTlCOPk4uzCOdhp98bjWZ6JTntwfH7x5NmhFwQ8DBCnJKK5KVNVYOi2tnpH05nk+GQxWaQjRKSxOSLy6kGfKCkWyazRanLOIXQMEmkUxjhfpBbUJfKMcowZlcLOVeYDymgWx4QSHwJKqW21gmajl2WFx32CbZ6nUiiEoZRVURS+F5WlyPO8qiQlXFkjqur09DTP87Isy6rodFph6AOrkySpqgohnKapVpZyRhg1xlSqqtcrYXiwvZU+SQhmUrkobkmlK5GHYWQdzxepkNDzPIwpANYYiQBGGHAPU8IoxoFhDREBeBwGFGPseYhxBBxWIltkebvdhRBTyiGEslLWWqdRlSufSyFLYzWE0Npl7XtRZoN4cHZ2cngI9/d3CaaUUgAQY14YxrXGg5SFs9DzuLUuy3JrrZJ1GtspJSF0vh8eHBxwKgCxWbUYTobTaVmnPzrtbhQ1lDSlVBgxjgnQ0jkJCIWOAAeAQx6Pu50BJR71OII8TUacBc5BVRmMMSKsDiYWuawC2Wp19vYOpNBlIZ4+ejocjoIg6LS60+mcENJqtc7OLheLRa/XazabWZadnZ2Nx2POeV0QDAAYj8e+79++fbssy6qqCCFKKYxhvWHbtb7sqkhynZ5Ye9i1Wbkc68GgubW1o7W9f/++UsrzaFkpsEy7opogCYA1xgAIuUdrRkJtxdau+drgvuSA1nZznSCok0foxaKJdWzgpUQA/IqSyDVcgKs2vuvxKl2x5jTBFZfiJfC0+c718Te/zgKHIHQQAGdqnph1dcMbKpUyxniUKSkv9Nj3Qy8IM1uRCU5ltzdodXrdjz/75PxyjBBTosQk8jnn3HfOSVlpB4Qym3OAL8arXzrrl65YnTmCEGACGCN5LggGi8V8OgPXru28/fY7lLK/+qu/Oj++rHINNbAaOA0csEIopRSsTw7aZSnF85TDPy1Q2Lwdz8MAxiil6mW5Xgb1PXopK7EOHtRJgTov8AK6si/EtNZcwpcCTqs922kt6yPXa2N9TACWDlXNtazVzaWUUhmEECWoxg3WWgRdEARaaww25RNA/YPbmLldNldbLTb4cqgGIVTndgEAiFBCSH0Rasxtra03fmstxNhaYK1lyGHgahy/uUjWALr+VF1eiDFezNWHHz4wWjiFDh+fNhvB9k4vmU0Xi7myvL11EEdNwhg8PRcOppWafnLZaXMLx8pcdjoyaggIMogLCwH2cJEVjMdFUXS6u5y2NLKlnikDnLRaOa2tcVYbUVYlpdTDMAxpFEBR2WRRkaJwDglppcoBVFpra2R/0Lt65Rpj7Oj0ZDyezudzpUSz2eh0WmUpiqKI41AIwQNGFYBIR4EvVDU6O/uzf/a/jeKgEcX9Tvfo8Nk8GZVZDokhXCVVyhFLysntzmsHV7bffz/FFJdlXhRFq9X6/ve/P9jZ/vVv3i+q8urVq8NsiCjgMe3wjjXq/v17rVY78KMyXci8zGdJkwegkpPjk7Is/Tt3yjI/OT9eLLJmu7Wzv5dkOfc95nn3Hzz47N4TwhvOEYQaUrlScG3powenV65c+8Y3/vhnP/vlw4ePXnuNX16OokbbISCAmqQzQ4BGGvjOQOP3ww8O712xVxxQZZVFIQXQEGQFFOTHP/5+LQKVpmklCGWsEmVt5YUQUlYAIIwYhdRqq4zBxJ6dD9kkazW77dYgjFoYeb4XZ3kltVFKOYgwJcboohRSVaLSUuqiKou8grACFpZVVc6mTpssyyazSRh4GEOpKimltTqK25PJiBDyxhtvUI9yQ9VCWWcgBmkxm8/n82Th+TxN84E/wIhT6jyPE4IJRHUljBBCa4UQooRgRCHESosy10VRKC1u37rBGJPazGZJkVeUcqUtJg5DDQlsNUOM6eHh0XgyI4QEQZSmiZRVXeaHMeOcQQgZJ57nRVGEMTbGIYQgQGVRGWM8ntRwoSxUVVUQIsZYq9VSSmutHTAQuqoqjPEwxt1uP8tPx+PR+eUF5ezq9a1er44lWEaDskhFpRjD1gJRWAsw554UhlLKGePU5zRoNnIpJTCo29kiiDoL4qhVVdXl6VBK2Wy27XZrNJycHF/UqH82m0OIfN+v1S/qmMH29uDs7KwOLeRFenZ+ejk8r/GBc5VSy+QRY4NerxcEQRgFtVb5fD73o9Cs2vXiVU96u2Lt1W7K2hc3xiAE4ri5vb1bFMXl5VAIE4aMECul4Zwr67TWtbSLtdBYRam3PjhYEddr/AE2dtzNnRtsVLLVamu1N+mcW2dqX/Kk4UZlI9gIGIAXgxBwpaPwErzY3FTAhurUep9YA4XNffrVXW0JL2AtdOfsMjPgrEMEYyulMZZR1Gh0KCazJP3kk89aPun2mshHUTd87eZNDWyS/er8ctLr7PmB7wUNSjwpdVEUlZDJIue8vZ7SevcCr9SCrqfHqCeldND5vo8xFEIUeZblC9+njTBywE4miZSVEGI4HH7yyafTYaqN8zjHgDJiEaEQIqk0pAQBu8QKy0vw6gX4xx91tUWdaQIbbSRriT2ykmStF+dLDNl6rIHCZv5ofbnqGoDNqEN9SetVt5lfeE5jXIlyrt9gV+UYZVnWQLwm0CmlHEAYY2vQmiFEyXJhQ/Cco7BeaQA6AJ4HDNZPk3MOouc31640TuxKxAxTVn+EUkooL4pikaZJkmqtPc/zgkAIBSHsNEJsTQ3vVlcA1ScCEamVAKTRs9nM9/2dnZ1Wd3ewfbWqiiAeFHmSFWo8LqrSIhxHcSPL9OXlUbvbEcoZZyxwAe0mk8miGAVRMRh4fmCEsBYCpYCzBiKgnDSOVcL4fnORlX7QVCoDjlpgMCVxI+x0W0HIaqeOMQNJ7pBlPG93IaMex1tpmtdFPNyD7Waj024ZZwfdniwrIcpWq/Xmm2/evHlTGX1ycnzvsw9cpoRM+1ud/av7fugPBoOw4Q92OoHnE0IqnU/ml0LkzCeeR+Ku70U4TXPtqkLmLnVFVVZC9frbp6fnAKPtnb07d+4cHj372c9+Nhpd3vne7du3b16/+tr9ew+On51MZ8Ph+cXl+WUrbvVbzZ1Ob29vDxgbhmGR5e/dutXpR0qXn997BKCZJ9Onz45n8yxJFqUQx6dD7sXaOaG4LCutKwTE3t7rpbRHx5PJtADIxySSZiKVbLQbi3RWVmllZdAOutvdvEgcMaVXTM3IWluJclzqRhQbpf7Tz98nP/7RDw4PD2ez2eXl5fHpiZIi8HzCaKfTmc/nw/G4LMtSa+tBQghwsCqtMSoMmIsJBDTwY85D4PD21n6e51laOOcAQFobrTWCJM0zY5wFzjgrSwEhlkpVVVVnuPM8z4tUiLIscwih77FS2aKsgsATWlVV4RA3RgEAPI8LJVud5tvB22EjPj8/536g7ezs/Hw8nmitMYBBEEEInYU+D6qqctSjFEkpp5P5dDwuigI49LV37u7u7hZF8cknn5W5DKPI84JmI8rysiyrTitudrrj8fDkJI2iRrfbnc4lxsj3vSAI6sp+7lGEUJZltYrteDwuy9I5QCkHQFWVxJgiSAAASmmEFAAIQmyMhtApaSByWltrXN1l3iGSFpXRbnd3n3MfOFgU1Xg0z/MyzwopdVVaKVWWZYxw0qKilNAhglxdvsioV1VVJYVSmlNunMPGYUhAFxJCeoN+4LNGHM9ms8PDwyKv1RVh3RaHUp5lhe/73W739ddvHB4ecc61VlmWSikRQtYti6Occ5RiKeWDB8t0Wp2UqXt/13awNl61IsXaLIJVvRZCqH4nRshZwCgfZeOqkloD4BBjjBAQRVEppLWWc0YpVbJWSVma1DWXu7bLay3Il7ZzKeX6lXpK9ai5C7UJBhvBhs1N5SXcsB7oxdLZtVFeT6b+arcx7EZTvk3TDH4vRHDOWWihcwAhC4AFDjpgAQLWKmUI5QhAgBAi1PeDsihGo0u33brzjbvbux1hLGHka19/l3n04YPDy9GcEuy0yUUuhKqEWjbj3QArm+MlzsTGPJG11gGgtdQaSFlxDwdB1Go3Lk7PytJyDpxz9+/fxxhDiK1xTd8njKdZYa0OgghTIoQAKxrj829cshr/yQfc0D+ooZ5SqqoqxhjYYK5s3qz1B90Gb3ENhdeLR2tt9XPd5c3IBNhIe60RRr0q1oKn6wfEWjubzYSoI6/V+q/Oubpls1JLsOKcwwhYa4MgQPAF1YflOkTOuee0jxVkBcYYhOGquvh5UKR2D+qnyW3IzSVJcnx6cnp6jhDa3d3d4hxjJ6UUghBnIXwesUN2yfSsNdmiKAIY1TpF7Xb7ymuv37n7zQ8+eP9iOGs3I6Or8TjjDENgSqPS9On5+enV69cIQc45LQsswkVyKu3ptRvMKF6kRVbmDgLGgNKABzSZa8qjNC9bje15JvOsyksNARRCl6KuOtHaCCFKiIx1gjkFAWo2EKNxFLZUGTHqKqk8n/R6nUF/BxM3HU7TRdput69evco4JZRUIg9Dv9dttnud7qA3HI0Orh80ms2Li4sbr7/W67U++eQj3/chsEVRPH74qMqLmzeue553Mfys3W5P5zME0Odf3FfSEsQZdkVRtXv9JEk+/PDDqNm489bbeZ7+6lcfjMcnTx4DaOV8Pq7KJJlAqwA0xkhRSY0AHAMgK2GttsZc29uKe82tQb/I8kGnyShyRh0fPfnVr3+nlD49OW80WhBCctPvtTtXDq60Wq0nD96/PDv55d++r5SBJDi5GFXSCmEiC5yDNQOGc04IqqWSbr4xMMZpBRhnRpFvf+fb3VZ/NPy/k16nGYe381JkWfbTn/70008/ZYxVRTUdT4WUwDiP+pBgCJAUxhjj+z7jrNHqYOpX0nZ5uHdwjbMwz8uzs7PhZJrnaSlLqSophe/70mhRKQQAIaSwlVs1Z/N8v94FrNUQuno1E4oAAHt7e1tb/UYrrlQFhI2iwAt4FEVCCC8M+ox5XiCVQYgwPp2MZ5RyIYQsKwixUgpC3IgaGGMCaV1YMR5PJ8NRPflufBz6EaW0EcWEIM8LAADWgwDSRTJRJovD7XYrCDzMKcDYbW33a03lMAwRQlEUhWGotZ5Op5zX5TQZpXRvb7fZbDrnRsOJtVbKCmMSBBEhRCtbliXjBGNIKUPYEYLC0AcALBaZdJCzqNtDxriqlLPZzBrSac+nk5mUGgGMKEQQcMp832vGIcWAYmitFsJgjD3P83ympBkOh3Ec5XkxHp5jSgeDfhQ1MMZpmhzsXWu3+mWhHj9+nKaZKEqtdZFXOS9q69loNN56663z8/M0TfbZVUII58xaW1UlAJBS6oAbj6eTyfTi4uzq1esHBwcA2DD0G43GxXCytkprq7rp2a+VD+phjMuyIs/Lp0+faa3dUl4UAuAopcaBsiwBsBA6TCCwsG6VWR+kzj6glUT02pqDjdCCWWk817/iVROgdru9yTCok8FKKfyieOIm7ACvbOrrPWNz/6jfvIlj1rgBvNJrYHMr2sQlbs1FhwBAsKTCAecgsNZiiJXUURRxSrVUWVoBRzCknh9X1ly7+fpbb984Pnownc8azaDTbh7s7z58+BQioSRaLKpSaQixcSCKGlkiNiewib1ePXG3lNXCNYFOyopx/PZbt7/2tXc9zzs5Ojw7O7u8vCyKYj6fAwAmk1nEMCO4rHJlFAQgTxPKfUqI1trVItAOAmAddMuIwj8xUlirAK1jWlmWTadTSmmdVq/j7etbs97ON+9arbaymUhaVygQ9DyBVXc9QAhZC+oUg3MOQocxRGiJa6XUaCUCvQk40jStBf9rNADXJZ21ksVK3bkObyxj++i5RCNc8Q+gMzVAreNw9UqsT4eyuvYbOWfqq1GzCtbttuv9XmudZsWTJ0+eHR+dnV3ULbm7/X4Y0mWeAkGM9Xr+wDiEULPZTLNCKUUI4YFft4EQQtx6681M5J8//Ozpwwe+f6MRhIWUoR+k87GRJUKWO9MgiHGS52k+PvfplThsRC1243qwO/Ah9f0Aep47vzwDDhAcQiABosroIA4ccou00MoyRq0BUuqsqKbJPFvMMHaYAOgDagmChFHGKMDYTLNxWZXj8cRouL93nXFstMYI1bpG169fB8idnBxNRudl7idJEsXxtWvX3gBud297kaYPHj28/+jhs5Pjsix7vd6g1+GMtdtt2G0NtrfKslwUImoh4gVAo0eHR8k0f/v2O34jnM6zVmdAMDs5OVGH4uvf+vq//Jf/zVtv31rIo6KorMpaPpsBI4vi6+9985//2X/z+af3fvWLX16eX2TzmTX6/PQYQugxKDAaz+Z5UXiMGaUHreb+dv8zj2qC77z9Zp6XxaKo8mKinRLK84KTs8M8LzXAvd7A2MXR0XEQBM6BZJpiArqNTrUoRFrms5RR6io7Pz9xFhjjCA58Ggyi9sHWFR/4JM/TMIyNMZ393cFWj96nO1vbSbJ4+vQpJZxzn3oMIWS0g8BAYBgPPc8LwoYxIFnk3Z6yDgqpPT9sd3rRaDKdzbIir5U6HATOuVpiGcJaL9liTACCaZpaa5WWwBpMIMJIKZUtsrhB4kYjbjbyPJvORs12A9OeQyavCmutL0UQxpUwjUan2WxaA6eTi6tXrzLGRpeXF+eX5+eXo9FoPjnz/VAJCRyEEJalqHLDGGE0rEo1Gc/jRrizs+P7PM/zyWzMuZdBi7CxphQiKasFJjaMqO+huN1ZLBb1FiWlrDvOlWW5Rt9RFA3627u7uwihPM9r3beqEhBiz2POQuUqAICzkHrc9z1KCSYwCDyj7dnpuXCi2WxS1sxms/k0zXPJma1KUVWSIMw5Z4x5lGjuUUoRgpQizimEsCxLYRxCgBBijdra7lNKk/k0zzPGWJ7yZDZN0/T61WuXl6MkST799PPDw8OqqjzKwjCcTscQgH6/75xrN5q3bt0KgiBN08ViLmWFMQQASekQcpwjrXWrHSlpfN9XSmxt9Q8ODrIs833fbpD71umGtZmrTc86Dl9bba11lhWffvq5FLqef5oXoI4PE1IUuZRSSmmtBQgCR+rD1qy6dRh/kyuw6VeBVXh5DVxqK59lWW33a/Y7xriqqnUkeXPAFSPhpSzDS55iffw1cPnS5MKX7jpf6tDDjfzFGjE4AKxzuCYuEGyt1RZARIwDVSXjIOy0e5kaj2bzQiri8Ytxcvjs0WQ0nk3TrX6PsjjNdJ4PF5OpcYAwtlnbCV5xnTfnuQY6zi1VBAhBSpcQuigKtrcHO7vbf/Kj71dV9Rd/8Rd/9Vf/YT6fB0FgNCDUSpERjAedhgNoNEuqMqPMBwBAhxxAABgAEHAbSg7/xMOueCr1KqotT6vVUkrVvSLXNwW+SFvZXF3rwEC9ZuofjDGY0bUvvk5kOOfq/b4+zpqECwCwVsKVGoe1tqYj5Hled7Jdp67WM8c1Z8I9r3isp1QUBUWQUlrHRSDAyy+ytiYqwBVz6DlvAbrVk/h8Ydc1zACAumiQEDKfz58dnTx9+nQ8neR5boyZTqetyYQxz/f99VV16yAZArU5queWZZmyxvcDY8xsNuvstN94+/V3vn43WUyUERB7jWYce2y3ez1Pphi5BgV7nbjRDCgdcM4nadPznbajy4sLIc+CuMAkI8wUFUTEU9pRFqZZGcWBhRpAFTcawELG/IxUQmlCCMGMEEIIwARgiAAAyAFjrRLCaRhFXd8jo+HFxeWQMc9akJfKWRTHzaqqqqpoNKNOuxlEQaMRzebBwyfHhNCrV/YhRl4YvXXn7nQ2DqOG0hYA4Ifx3vYOYyxPM4JZVSbXXnuj1ey8dfvr6bz69S9/U4ns/GysBGIkfvrkyA+474fP7j9175tvfeubN25c+91H965sD1px/LtHH58/e/bWG+81w2B8cTm+vDx+duScE1XZbjeZzzCGz549hdSrhAIYPfzsc2TB93/04/2tfq8ZPXz89Jvf+t7R0UnEebfVBADlWTobjTUDjiAFbFGWlLOt7d2qKI8Onwz6vUbkhdSfX848x7pec3d3xznjd3r9/tZ4NAOOBX4z0GF6mnk6IHWX68ePHw8G28l0hjF+4403sixPFlnd6respFKCEBoEge/7izJzFiBEnIVay9l0/oW4f3Z68Wd/9l81m83BYDAcXqhMEMLqejnoYA0UPC/QzjrrILTGGMS4UqosS6uVH/AgWEaGt7aDKIoQQkWZSy26pEMIqaQkhARBwDwPQpjnue+HRts8Ly8uLr/+9W/cvfueKMrRaHRxMfz/8/anvZJl23Uotvq12+gjTp9tZVZfdVvelhQlvScJkvXcCDZsGf5bAvTBMB5g2QYsSID1YFp6pEiKInlv3ctbfVVmZXPy9OdEH7tfvT+sOJGnsoqU9ERqf0hExonYsWPH3nPONeYYYz5/9uL87Gw+W2TLvK5rhGgU0ZC5KIq63W6vM8yW+WI+H4z6YbjllwW9fodQNCg77U5ijCzyBUSm1293u11Ew6apAADWaiGEz39FUfjSgdEgjlKfbp89e3Z8fNw0jbUOY8xYoLVW0jjnOOc+VDHGOGdByILAQx3TWZ7fucMoIYtFlpd1yIIoSqxdG0ARghCwyiohm7JQi7m7c+8WhABj5JyVjfBZsGkahGme57PZDBM2GPSCICjLMo5DQuhsOr+8vKzr2rsrFkVRlmWr1VrMp6vV6vz8HFr3zjvvjEajfr//4vRksVjgtVc8wBgQsu4mCFm3Wq35fI4QGo0Gs9kEwp4XjIEbi9QNI2HT8b3ZDqeUe5jn+PhYKYUxQAg1jUiSOI5jREhd10LMm8YAAAACcRRsEF0fnf28vpuuMuDreMZmxbb5L0JosVj4RaQHh3yXxFq7KSBeSZPg6xpIcANF2NQEN9/lORA3y6ZNleC+QZB035xldaNEsB5gcc4BByC0wCEHwzBumkZr24qTgHM/sZnz0PHWf/zTP/3o0w84URCKKECjwfDBg9fjsMeD9mLZfPjR5/KjL64m46YRECPfO99E+ZsF1ua73/wvpVSIxldpUhohzGeffQKRvXv3NifU+20cHBwQRK0Bnc6KNIVzYDDsvfX2e5CyX/36w6PzS865rrwKYD0aCgDkgFszHP8mN6UUcM4ag67nnPlk7OWI36zbNmvxm8+7a7OvDagArhtSQtZ2I6eEFq/tjZ1UL80fb25Sms094g+jKArvcL8pOMA1jGGMQcZLFc0G/PBIWF3XBq/HTGCMyfVg6HX/6vrqgteNBucctmhTtn7LL36tBy6K4uTkZLlcerzNWrtcLq+urjgP+/1+GrLNTbf+gtZCCJfL5XC0HYZhnueVaJwDnnE5mYm4FX/ne+9dnB6eHh6uMtMJuRLaBXE7Ce/e2tP3drdGPQytlI2x6mJSay1enDw6PjnujcDb78DBVoCw0tZFKSlL0em2suwy6fSVboyTEGIDnJAyz/PVqqG0SJLIOBBSiqADAFproQMQIAQJhEjKqtvtd7tpkVdxHBqjyiwnLDTGNNny6ipYrubT6bjVjnd3t2tR7ezsZGWBL68+//KL0dYWQujs/FII0el0qqq6uLjCAF5djvNV5hz0w8YRLH/0W7+jGzO5WDLccga0Wr3xeGqtvnPn1nA4PL9oaa0bUaHKddNocXXhGvng7p37t+4THD/76smXHz/Ksmq1Wr3z7tur1WpvZztOuFLiq6++SsM+4aydts7Oz8s8T8KAItjURSuJ3njtbjafSWEZAVprURbZqihovr+//86b75ydnB8dHadxCxqbRGlIGLKIOKTy5uBgC9x948GDu/PFDOLqjdtvfSVfaAV3tm9z0AIKf/eNH5I//8VnaZo+e3Z4+u/+uCyzre0hgPrW7a2Ts27TNE2DZrNyMplEYee1+7d2d3crIU9OzrSYW4u0UQh3i0J89Mkvd3aHjAVF1TgElXHaakqpg6TM8iTtaq0bISEigFihFMAkLyujHUIRiwjB1BoShb12685oqzNfFJPpklDU7Y7CoINRFDLKGIMQUkdFKetKOAltA1phanU46N2eXBVBEDDeffj6zusP3mOMPXv27MmTJxcXF8aYMi8uLy+11srpuZxijBvRzE+W43zGGDPOXS0KY8zOwR3n3LLMKccPHt4hBEpVLLJZt9Uti7pq6rTdtgaMJzOEcBK3EIQIaSHk8fGJc7YoCgdMFEW+7ddqxWEYXl5erlarNE2TNEDIlWUBYcIYy1YVhLDf74dW6JXQoA4Bjts9IWpTVxEjsioAYwQlgAXWWim0c4hz7hBslOylvelinpU5Y/zi4kIpxVl4dHTEOXcObg2Gu7e3P//8c611XjTL5Rw4NxhuV7Ws6hmAiHEOEO71h03TRHF6Pr46v7r88c9++vnnn/OraV1qzgilLGIsjlMpNdASAkJh0JSgnQ7LvPnH/+h/KIuiqopBp1OWZS0aCIAFQChpgSOExHFsrTXOIAcpJs4B2UghBHRie6uTJrTI54y6ugKMgHbMe/3e++++d3R0NCeMQJyEtKobYEFdlz6WeSt1CCHGkBB6HYXNmtB9ba/puaXWWk9W8JeNp7j7ssATudvttiedBEEwn8+FEFEUQbgualut1k0C2ibmboxrNtF2AzPcRCZuLiXt10nyNxevL0PtjaiNMQUAAOj8JAYEAYYWQ6OrIqCIUoyQcgCwgNLAKVjWFjhhy5not5Nhb1erOluy3a1hsRLAqJjSlAcBsMO0gyC9uBo7/nK8tQXXI38QtNcyEw+3GK2V1c4aJCHDCCMMHepEPedcndkP/uTzT3/9vCzzQa9FKdneGb3/3turxTJfTnAlZzPRCvH/+f/4v35xevHZpx9iAAhWEEoHEAQ3R0Y6BwAE6OZJuHk2NuUauFGrvVJy/Sc36ADGBEKopfLFa8CDgAfAOqO0n24kG8E5p4xbbTDGCDhntGzMdZNL+jrJWUOwxWtkYi2gkLJeVwbSNQ2+JkOs56RcV0VAr60bDYQvnRw9+uj9325iYP5bry9dbZ03xoJwA4p42ZG0xuMBG9gDISSNdeBrF60nMyKEpNbWz5THgBACrXEGQgv9EqLd7oRhOJlMnjx58vjx46IoeBgEQQAAqKpqPp/3eoNerwduIDS+HDcOWmu9jtq7m2ywVUqpXlJVVLujgyRsaaEtJbPFdPjaHQPV7t72/bcfRmFwdHS4qgpKw7K0YhUog2aXrdmcFdpmiu7cTl9764AFZrqcDQfprBZBMsxWYjF5giGqRcFoQChq9+KrSRG2AhQwI3kpUCuNldJSqk6LRymrijzPc87fthZoo3rDlIa212fzPFcmK5t6OVsauQhoFAUxa/DycHH37t2n06OLi4tfXfypQ3A5mZZNXTRFEAST8bjb7RZF8cXjryIeGOAuri53d3dJMgqC4GKWz66mjw9PjNB39g7yPB/0+k6rwydP9/Z2fvDu966uLkDl+v3u7t4/+Gz16fJ42WpxSGADxPZwOBpunV1crvL5F189Tlvd23GvQ5K6Ut1tNs9Od3qDUrko2atW9Pf/P79ECNxK9x58/+5v/+A73394ezybXlxd/Yf/+IuqWMRhqA3e6vTeeHA3ZnA2OWqqeRjg7/zw9s721qcffexoBCIoICgM+M1nz1erVWOaL1+stre3V6vluChHo0FRZgdvdcnnXzzp9XrLZQ4Ri5Ouc/TwxelXT57VtWmE0toBwCDkjTBX42Xd2NH21t7eAXD44mqmdMM5lxCEYfjJJx8RHlgDsiyTUmJKfBu4laRKGWh8EQ2kn62AMaU0X+RKyTAMfdxP03Q02q7lCgAQRVGv3+l0WoyToiiM0X6EUqvVCoKAUh5FEaMBxlhK6an7zjkjlXSgrmtr7XA4HA6H1to8z4+Ojp49e+a9BUPGMcbAWGOM085CCxCilHrTFaVUmqavvfn6cDj03KK40j7H+IWsNcAZay1o6vr6/veO+tA5Rwn390kURfv7+1EUeZmTH3Pud+LXxAAAT69LkkRKaYwKQkYIaZrKWuvnN27CZRAEGFHnPAJMIURKaa2c0UA6XVeqqqosu9TaMhbleX55OU7i9snx+WQy2d7LJ5OJM5YyXBSFtTZgAWNUa62ESNNktVp1u92LiwtvF2GMCcOAYGaMRRghBBACQeDHqWkIqVTNZDJZLpcQwsViEfDWRvWwxlTRWukAAAD2pW+xX8K20qTVajHGfFcFgppSeufOnXa376GasiyrWhK8jrDG+LTqNn5wPlXgr7vSvrL+eyUN38zc8JrO5uMdpdQjWD6LRFHknPOe05vPurmE+i/dNvD1N4/tWzd3Q3lx80sxxjCBN6UfAACttSVIKYXJGl9RSk2L4iv41Wg0Oju7uLyYPD88OT8/F42hQUAxgteUEQghcusOOEIoF2WSJIyxsqqqvAAIesV5Ocucc3ZNJUFo7dZkPHekqCu9FIwTAMD27s7b5p3x048gWXV6KWUoDClhKAzAclmiterQ+1/5M/DfqPXwzc05V9e1R5i89kFfW45ugJZvIkzu2rZ547Xstxs/08aVazNeEoBrbMnXnVmW+RJho7f0BcfmozefeLO+BNcOSJummxACe1qPMcYYCK5JM4iDGwAbhNDa9ezWG/fCy692Pbx3rdqYz+cXFxfL5dI5hwjGmKJryzJfoMScrBWu1lprCSEQU4SQlBIZp/2YX7rmEkEICcbQAVk3RiqIUBhHIYsGW9vIiv5oi8bxeDb96IsvxuNxu93mQfD4KG93ExxCGnFl6tmiLo3Ia8m4a7W5LGC3nTgNWp22I3oxm7c72toqLxvO0rfe3P3hD34aBJ35NDs7vZqNF1GQMEJtzRqJdMNubb02XxqnLQaoKso4TLSUnTRBiARBSAFqJ0mZ1y+eX4pKhDw0Wi5N+fTR41qK73z3u51e97NHXwaUfe/97xRleXp6mlf53s5ukiRPHj2+ms85wP/d//B3jl6cfPjBn54cvpicn/W7A6sKLcp2MspWmRALo6I02i4TdHn5/MWLL4hsc87TpIcpNcbUdd2IVZHZoqydCQIeIcfOTyfT6RwSCgDZ2hpFIWWtJNxOEKDOmTTt/PjHP41i8uGHH3a77Tt37kitb9++/eYb76StzvOL893d3W7aBju78r3vAGARBq0kRhhyzher5bPDw8VqtZgtOedSaGONkjZblUYDo+FstsrzfLkoyPnpoiqMscrfNUrqR1+8OL84i+PQGLNex5MWo1wJPJ9Wlbh4+PAhpdyezxEihHIA3GDQa6TWTaO1Ns7yMIiiyF89axCM4JAzz91tpLAGIAiFVo2QXj0ftkOIUVlVZZ1D6ABYW5UBQBhjCPEgCPwtba1bN7BdtVqtdkY7nPCQhRRRGnjgAWutrQXXHTV0//6Dfn/47Nmzp0+fckQJJpBCiTSGWCltrQE+5RttjAniKIyidqdjgEuEcNJpZb3tGgRISosQ0tp49pbnFjnnvAYQQmTMmtLspf/gOvT78UtCiKIo/LI1SZIoikLKFouFtXpnd8ubE2hNAQBhGG6Y0oQQL5J0zk0nGcY4WzWTycLD5ot5VhQFQshZ1NRqfDVbLsrjo4vpdBpFkVKqLMu6rLxvlZcLYkx8YyhJ0ul0Sgj56vHTJG6VRS2qOgkjjHFdizCIooivlGCMOQOAtRQD1TTjy/Myf21rMDw9euED5Zo25WmJ6FV3gY0SzFprDAjD0J8czjnndRRF9+/fvxxPj4+Pj4+P87zw7wo4a4S8EdzBKxn2myAq+EZ+dTf4BJsWrzHGd5F8egjDkDGW57knoPgX+EIB3ZCffTPZbP79K3L/BsS+2ZV4Bdn+5rbZ7Te36105X/FQGkG77suUVbNarCZXly9evLh9cKvfHxhnpWy2hoN2t1cW4uLiQipgLbAWmjVtEkAILQQch9Bg0RirIeOxlyrU+ZJibIFDa+WnsxD6SQ3arBfHZaOqpqGU3r179979u8HvvPnnv/hAGzienb84PTk5PwcYIAoABBBs+Pgvf76/6XLhlesBXEsZi6Lw8S1JEgihvl7AGGN8N+Qm9uPrSHfNJfSua/rrI6QBANc+117V+qplk1JKSr2hMa0XHl9XW3zzd9/sfJP7/aFCCDl5KT0wYE04IDzwtJKbtSlYa3OuJ0RYePOr3XRNGI/Hl5eXSqkgCPy39vWBMsY5uFwuX7tzQDmL49SvKzDG4HpXmxNCESTk+lBrRyDU0kRRsrW1tbs9iEKytbt3fnaUa3k+X16NJ+eLbJqXkvIWIXN1wdA2beFYpasSVrJqllAbTQnsdOyj+XEnJa2Q/sO//3q7m4ZgvIJ/0uuO5jOsFGu1RvvD14aju+l7o6Yw/+GP/3wxWyIHojAkGFyszs5fiKQbOAljli7BKmScArQ96BNCiqJKAppGnEO8HI9PL0+BhTHnl/kiTZJ7d+6Oen2A0bDT1sYgaaBS3TjeHwy2toaTq/Hi8tJZu9NtO7GCYlFMT0w9v7ff3x4NGTKlUVV+mnCyvx3GoTRqSnFFItBKuBVdrcxkNoMQJ2lbCmiFa7fjQb9nDFXKCK0QhISgNI3SdEuAK2tQxHhAuZJQVLIqyjjiJycnz55/Ndrur4rcWfjGg4eT2fL87Ezm5VidPoUIE9gO46QVO2eMVWVdC6WXy+ViVXSXRVmW7XYXIcQQJjR2gPUHI87pdDrNc5FlGUEoaRoEIIWQMBYiZOtalqXL80xrfd3QjRAKlaJSyqxa3b4FrHPGIsZjrU2jdLvXxXnVSNlUtWgU5YxojBCRWlFMAIIBD9ppy0fsRirVKGMtItgYk0vZ6nSHwyGldDFfQqx3d7d3drbDiBOCCEXWar8oX3MarJNSY4wZpUEQOGMnV2PV6Xh4kAXcF/Wc86Ioaimsta1WqzcctHvd4fZWfnrmnKua2hhDCCmbOi8zKRTBNIlTwkgcx+dXl/DqstVqtbud7GJqhCLAtaIQAiKZRQA3tZhPTwCC1gCllLHqmlJA/DBW3+eL47iqKn87+f7iJiT5W5cxFjE+m80IQd2m7alJniF4HZ7WBZM1wJszlnXDOYcQL5dLXyhIaZ3D1sKqbIAjSdyRUk0ni2538POf//zo7DgMY6u9Zlp7Vqm1llJOMAMABEGotZlP50+fPpVSLpfLMPTgikiSyAFVVpk2nHGmjSMUGGuybCmEuHv37ldfPSpr7++EAQDGOeCscdYjkAB4nhOENwwKlVJeV6nX4yUxAABjfHl5aYxbrVaMB0Iar2kEQvpM/a358mYM3UTGTfWw4RhuguyGVmmM8aCOD/dBEKRpijGezWbex9fP1L4Zal8J5Tf/Bd9WRtxMVOCapeGf2SzRvvX119/i5dv99ayUsuuF+Es0BUIIrDXGYYfqSjhjiqoRSivRfPjppz/50Y9u3b595/a9wWCkpP7Vr/5iOrkoKulPoOd5eQGqcw5AmOc5cI4yFrVSSilBuHKAYl/TIOcccJuZVwgDqLRxAGAMpJTHZ6dRmhwc7F+Nx8cXJ++8+z7ihIcsSulyqdKU5YUEAADkvDTPOd+CQAD+pWO+/xq3m6iSz/eeptdutyGEHpj0FbkvFDaNgM153vyOr9AUbv4WN2yIXrJb/Os9D9p7T92sMDaV6CYsvHJtX/8XbgoFH0UxxojRjRPzzbrk5jFvNvfSf8kihzYveEmwAKAsy/F4XFVVu932Sx0PJAghyrquazGZTFRdtLudnZ09j3pubu0wDLW5tqO49mYAAEDrAs6MMa1WSwxGPAyt07NF9vnjp189xzxkBrhKOxy3cuPyRZ5uBw0urUM0jTgismCYMM7al2en+VypBsiVJQM+6rx+dvzsyePjzgGJhzuFy5bz3NVgdmnaQdjdf70B1d/5na3J5dXpydGjLz5fzic8wE2tGA1yVcjaiMpYCVaLLG3FWglKMGA4Cuio18fO5ovl5HJ6fnaEo/SdN98ejYbnlxez+RwREDOezaZaa6yVATabAKzUG7dvK6VUUT799Nec0J0O74fDTtrCENV5wVo2W7wIt4atFIjmqqotY2R3b3dvb+/jj2eXF+N5Nt3fu71/sPXi+Gw1z3bg7s7OVhDx8/OzYjxzARoMOeOakBwgUBd1UepZNccuoDiYjceHz59sbfd3tnaDkC5ny7fffb/T6//pf/yzIAjqRSGLYjmettsp4cQqLVSTZdnp2dlymQmlrbJcyFIoKiVjQcwiSqK60nEEIHB51iAYYiQJhpFWBiEqjCUYhmGEkei0trIsM6A2mmiFrYYIAKNMXcvOsCUlMsZxlmBGi6opy5JQmrRiXBMLLCYCIQIAaFQjhKIR8TQcylmImXWOUS6E0lpPLicNl6Nu9+6du51eL8/zRsnvvPX6gwf3W6304vJstVp0uq00TaMo9Cs8QojWBl77f1VV1UraCOB22qnrWluDIaGEUsaUUgYARgOAoFRGmypJ29/57igb7CyXy/Pz87IpCSEawKqu/ZBynzBYwJqyKlbZMo5HzUgvyiIva6kgwAiBgPCgG6jEPn/+AloMIEQIGYOUttZp62AUEl8fXFxceEzbz1jyWLFnWSOEfKSo6xo7QCmNosDHpk6n0+12AADebtb7rntSpJQeq4DeVEpKWdfCg+eMBlVVOeeMsVEUc25Wq1WelycnZ4vlsq4aX3n4EAMAAAApZQgheVZ2O30ha0rpdDpL07TX6frxBBiCOAwwQXFEPdUUAgWcZgQbo/PV8tatW9ABL4haCxedg9b4WbN+tDQjFGOMb/gUcc673S4hpCiK1apqt2II4Xw+l1JiTKMoiuKkrhtfUmwC5To7XecpcGNRfgNQXYemVyL7t4IB/nkP+XqFmO89rVYr5xylVAhxM/j6DX4bhvHXu7kboPfNQO/j+ytgiXNO1LUxxhBstCwwxBi1u0MA7NXF2aOvniJI/t7f+3tvvv7Wr375QZHPITAYEHw9MNlCsFkZd3pd76bnACizXEnJOB/2+nmxuqbHXU8y9KcWIdlohBDEMC+LX/7qgyfPnr71xhvYXmV5OdzaOzo+e/zseRBG5eUKEAMQ9I7DEALPlH8VYfib3zb50hepHj/YnPBN+ehzt9soYOGr5eDmUnTXFl7Xv9JmwONGiQM2gsa6rptGenXDTbmNLz5uVgk3a4VNofDyxdffwjdwr2+E9ZWplHJgnaT9QW4OdXP13rxf/KdzzgEA0+n08vLSGNPttebzOTAGIegp2A6upZhXV1fK6ChKkiTx74KYXh/nt5BvoLYKqWKVKSGlVNNpXZer8/PzvKiVUtJIHgTtbodwVNd1I1SvG19cTYxG7c4wQnEjidaulqjd2avyeRxti2YlFWek+/jx6ScfP/su2xVb29UMrcZWBPjsMC+Wz6qcvfbaw+H2dpYtb93ZM7Y6emHefOshMHpyUR2fHhtVExQDR6VoRKMQBldXJwhgq3W0H3W73Vu3DgbdYRzHR+fTzz/59COlFtmyamrnTNpOdnd3w5BThgkOZFE2ZcEgYAyrIsttxgeDABgta5Gpuimbsmy3EoqrYnUuVFOJpt/vtrvbFpSL1floP71arE7Hj5bFlTBFWWiHaBDCLJ9EEcO4FnqCHen1B0IWRZnFrWGWT4zETe44aSVhVwgRMp4v82U+h8gKJfOs3D24VVXNqD8wRe2cw85C56qinEyuainmy9nJ6WkjJA8i7SzBlBCGEWc8XKxKiGez2Sw4PmeMlWXe7XadI0QZJKXkAZZSCW15LWbzedNUTdNobQCE3BJEKaZREIaYxtq66TynlFpAMEB5UVZ1mSRJp9duO7gNtrQDWpmmkR5j10JbA0xZamOSKOWcD0ZDY1y2XK1W+W7afv/dd4eD0fPnz798/Gg6nScxJgR1u535YmqMGo763W4XQuBr5yAIjLFlWTvn6koYY8qsmF5NDnb3nQMYYimUkho2wk83QATHQWycK8uyFjJxIOr2Jlk+ybLFYkEIapqmKLONHJli0mm3Rv1BXZfWWlOLdpooIZ21Rru6Kg3RLEgghBRhAyCCBDPqsRMIIULEr1AppWVZegqCvabrM8ZulDvaJ28vsGy1Eghh0zRx3E3TFFzLljYRA2OMsY9K1jnnjVmapqqqKo5TrTXjzIMoy+UiDCPGWJYt/8N/+JM7928bYxDChFDnAMYIIwohzIsVJzTPy4ODTl3X7XbXOdfrDY6ePm+aJk4iAF1eLAG01skgTBhLlK6NFZRw68DV+HJra0sIoa/H2vrjJIRAvJ7G9s2Q5xc0nvzhI+zOzk6v1/MK8m533a4yFjinEUIYQWVfshNuJtENOfzm5v+yObGbguDmompTc/jnKaVZlhVF0el0/Aopz/MNv+zmOvKbQfA/p2K4WWFsvsV/8i03CRnwGlTABG4EeJ6MaIxBACqtrIZVZZUWaZr2eh0I0Wh7v8qz86vL09PTbqcjVcMIcFogCCFwEDgHLHQAAQuRgc4U+cJY62t6gp2CVsp6PDmn3CsbAfC+Ds5BCwy0yCFjjXaAUEoIWiyy6TQDwP2d3/5Of3SbBe1//W/+1YcfPY8S6hwQ0gKHvRkQAABC5/FqDMHfNJ6wOeGbX2FTI3qqR13XQRD4n8lXvc6tX3xdKFh4AxJ75efbyHQRQj5T+pdsVv83C4W6Fh662FwVG+7C5mJ75RrbFL6b62FTUuBrX3NrrYVgI6Pw3pfo5dj3DS4FNwX05uQQTKy1XiF5dXU1nU43x+O83+KaeIRueplIKcuy9LwHHiJKqZTSAbTp7vn9I4SkEbIUZ2dnF1fjbLWAzlktEHQBTThzTdNAhJELnHLYuV7atq7KV9oC0OnzOAiEdMtlNp1Pd7ZGtnSIIlFqad0sLwxgPOrKcpQGD28d3LX2aDqfPT86NYfHp+cns/xKSvn4iy/vv3a31U3DaQwQHW7tDged2VKcnE3DsI9ITDEBEDJOjQWYkKyopvOZ1RBANNreSpJWqbGUjVJqa2ur3e1Yqy0wrVailSAEj4b9OOQvnh+OL044ZZ1O20hdr/K6LkVdwpDpplKqELXRVkudY0LihA23+4Ot3myxqGYlCStASh4BSAULESbJp5989ed/9osf/ehHrz24W1ZLSk0U414frzLloKTENPUqIGkcRRAgB8yg2/ve977X63d+/etfjOdXL46OPv7wk/v3H/7jf/S/evbs8MNf/srXlp1ep1Eyy5aUs7wqZaNCHnIeNo2kiDMMCOIBiSpTQYR6/b4npPv5QXVdEweRMpY6oIy10llglqtcKaG0wBhTHvAwwohiwgjlELlVsVouqjgOIQZV05RlZqxCGBtrOecsDBgixhihGqGkcdZBCAlSQjZNo5Tp9/uccSk1C/itW7c67d5wawsYtxn4/etf/3o6Hd+5c3sw7LXbaVVVX3zxxWq19Ay4TqfDeQAA4pxb6wAAom5mk2nTNMDaMIkD5xwAFjjCGbHGAme82J1grXXV1Ofj+dH5+dHF+XK5DChVShkto5BTiqscxDxgEO/0BtZ2rDZJksiqZBRjGBoDynJRVDmuVCONtVZbB5wljEKECQaYIMaYFMKnQE/Wc9duLeB6Po1zbjO2wMdlf/s556qqSpKIUuozltbaGGktIGsvamiMUVoB4fJiaY1jjBhDGUNSaoQYY7isHESOBySO0l6vK4QghBHCoAMIrSOLxzOMdjT00/OAUms2YpIk/V6Pc/7WW2+2WkmWL4WoGad7e3vz+fyzzz67OL8EziDgVoslo3RrNKrPxj7KeIE5psQT6/yKB94w0vdfsyzLy8vLPM/7/T7lbH9/HyF0cnKWZVmvN9Bay7oB1yrwIAiMFK/kS/8Y35jm8Eqh4K41YzcXfJvA53fiY5yv4fyJLcsSY+yFrH5lebOe2MTW/8z6YLPZa0HaprK5+eQ3N/cNGt3m+2L8UnrqD8NaGxDqjIUYO+ekMg4giIhUTcBY0u4EYfzixYsqz4A13U67nYRNAZyz2minHQAAYoQhJBSt8ioMKYJA6SYIWaebCiFWq/w6EWIIoTdfNs46a5C1lDFEgFKKB1HEwrKsK9F89vnRG2+89cmnz8/Pl1KDFo0ZL72+AjkEAQBgPXwBOl8k/M3Oetic1ZuP3bX4sGmaxWIBIYyiaHN7ro9uc1HBNWK/2a7ZNuYmEuCc2xQKNzsFG4GDH+JwsyN28/fd/Pcbx+z/DG+8GPgCl3POOfc3wkaDgAECN5xJEUL+9Riv50G/Uo4wxqqqIoQ4iLIs80Cg5zxpa4zRfud+1oNSaqvf8bM04bWpib9ijTEIvzSS8sHNB7G6rs8uLyaTiairgOCAE4LQcjmPoghbDBV2jZNSWweiKHE0DsmibOpy1YQJ7nQTGsDz8/zpi886SWQQbw2iMOVPDp+3Br1+scNYK04HvZBqpwuZSbNc5stCjQFvDm7d+f6Pv9dq9VTjIJtfXpkf/+Snk5NlVeOLy2z31ojQFPNI6QpS9uDNd6AD+TLDPEAEsShWDp1cjJdZtru7uxNxbQxhxAGT56vZbHbn7sFkfFkU2d7O/bfffD0N+Ww6zrMlC2xpCq0VIzxgnBKEodFOW6cBckk7YQGP2y0aBqDAjAe7e/tZVrVaV7NJ8/jRc057dWXLQj/68lApkxfTMLadDnNW11UGgSbUJTFN41a5sqJUCFkAAKf8/t3Xoig4fPFEa31yekoIuXvr9mKx2t3bW67m4+kUV6VSqhZNEIWU0oCFYRRhxBojGaQOA2SAaVS32+33+1tbWxjjXq9HCPnyyy9PT0/97w0QIoyxMORxHDZN7Wm9lOFOp8NZWFVVnudFUUkpu/0B42EQxULURVFUVW2BEVJShlkYhCJ2DuZFtVplQigAQLfVw5gKQuqiXMd3BKWUnU5nONjCkFxdXeV5kXbaP/3pT4VQFKnZbDKbzVbZIooCB0xVFUI0Xr4fRRHGBAAUxzFwaLlciqXgnLfbbYjRaDRiYUAodRDQgIdhSCnJsuxqMm6apt1uD0bD44vL8XJeK40wRZxhCDCBcZQWq1WNADS6kySL2RxhwDDhmJydnWVZEccpYRw6K6W0wmQeN5PaAkSdwx54hxgRSi31d3gQBF7X4DlTHif0q5bNDeZreZ/M1vnbGF8PIYSEEFVVG+PCMAw4klJWVcVCBqEDwCZpwnlPShmG4Ww6t9YyFrRaSZrGUsrjkxd+2pOHHYB1lFKInGeDrhsWLLDWelfj1dJ5QeN0Ou33+zs7O/fv370an62yZRyHt27dkvIgz1eTycRoDQDWWnc6nddee228KAghymittTLGQUAxghB6WNKrHvz6FwKAEFIW5HnunNvd3Y3zLEmS6XR6fHwMAIjjmHOutKEEYkw90dUTQjdRb1MobFoPaOMQtw6rLxWJ4OtLMX/a4bW1s6+N/KWIMa7rWggRBEG73UYI+QEom51v9vNfVCVswv1Nducmtv7V77qZPK4/92Y6WR+P1hpYS3DIA8QCPhwOu4N+lWfL1VxLOer3oii6urqaTydKNAe39rIXY62157sAADAilDFCyM9+9qPecLBarT744IOLqxVjRRwHlAHj1mcbeoc/87JZEASB0rWUrijKdjuyFiyXy2OJOt2djz/+VAoXBrzIBXDEQQcAshAgZwFA19OhAAD2v02h8Mq53azmhRBZlnlu0GYcFLyxprfWQrRO+TcLBY8R3vTzQGuDUf/Gl5jTK2/Z6FasfZUy+W1VwkuBAny5AQCArxIYY/4S9WsAjDEkZNN6gGuu4uYXfLmrzQd5dBNCiAnxXwchVDeNnwGmtb3eCUaIbEpe7k3gw3AzE27DxIIQehfl9e2D1lV4URSiriSCOgygM9AhhhhAgLGg1ep4uCVAAY3TNJoV+fnlxTTpVPdeu9PuhUpnxqwGW+nV+aWVoMhWCJt/8Pf/fpQmxdVssZq4XFVqkXZx2mnb83I8vXT0dnfQ/vHPfza/KpVgRvZ//atPZzMapQNEQoiD/YP7w50WD8FyNU5S/vTpIwRAWVQ7Owd3Du7J10y5qh89ejxb5kEUpq1kvly8ePHiYnIBgB0MO48elVHIe71et9vN3DxJIwAHnPOmXmgtrYEGuLquHdB+yFu315uv5lVTC6O6ShoHlbaOmBdHZ0K4O3cfRmH5+PPT6fi8393/7vs/hRACy1bzOuBJwFMEeV0rTEwAdaeTJFE6vbwoCxt2W/PZ7C/+4i8Wy/n9+7fTpH2wf2u+WP7e7/0eBPj+w9f/wT/+R8+ePfnoo4+sM3meex53WZaEEIopABBYxwh3BjptRNk0rjZWKS2aptne3u51B4eHh4eHhwRhESfI2KrVjtrtNg0Yj+/WdRlEfDQa5cXqcnypcHV4fDif17u77SgK6py0W1EcRnmeW9MY5+pKKttQLhirDXBKKW01oogQktUlRRghEiQp5YEwzjrEwjRp98IoxRj3nRtujdrttrV6sVhghO6/9UZR5YeHz6s6G25td7tta/T5+blsZJPXohKyUdm4qMtaCAEZrkvxP/6//u8Y4739A8ZYJSSllAfBwzffunvvHgBouaq11hAJ5zIUkv7OkKdhVZRGKWdsU1dVUcIwbqdJGMUa8kXRMMYI0nZRyoYnUdzUTd2ovd07xtmz88tKNIQ6bQxjpGoyqEmSJEprZCBnXGsNIZJSaZ0RQihlCGGMibWOEBqGnmwPGeMY40Y1qhHW6jDyjoFUCeOMq4qqzDMPVNarZRQHGGMr80qgg4ODN370I2PMbLnKs1Ip3Wv3mkbWtdAKSmmcw73ODoTQGDhs95hjZZVrPxTOAENsEEUqz1jILi7OopBv9TtKyHbEZVH0Wq1bu7t1mf+zf/bPsiKHGHW73Vt379y+ffu3/9Y/+PzLF9Or6fZotFosnj8+/Nlv/fyzz57mZc4p7WzvWGCUNdqYqqmbpsaQrFar0WAktKGYZ8vVYDAQsswqyeO2w3Q6y6U6hhCWtUjT9pqOSmkUhstlFnLinG3FkSdGQYwppQBAKaVQIo5SoaSxBmMMHDIOIIwYY6IsIHTWmna7l+c5Y6QsZRQF5nqKIMbQGFVVhQ/fjPGyLKMoopSGYbhcLu/evcs597NMrbU+Fm+KjG/mkldKh1dWbDdbJK9E/G9NYxgTY4y1DiGIkEe/KaXcOUMwI5hYpz2/lVJKKZGmlFaKvDnY21UK3N7dCoJAUBJYZ4w5fX783sO3333z/Q8//PDR51+8/fb37z9c/sc/+cV0Vu3v7pVVM1ksd3cH3/neew/evDsa9ZfzaZadFquV0wBLFUG6MgBj/xWMMQY6gzBECDhryqpJkjgK4qIoVksZBm2nceHq8/nk8PzMGoIQ5SxUuqYEMY6yPLcQhCEzGmpFEeJaW4zrzWm5eSZvnuFvPvnN7S8r4G6e9pv7gRDWdY0Q8n0He60msMABtF4ur38+Z5SQAADsAHZAaWOlslI5Y4C1hFPoIAZrSaQ3nDTAAQu1NHXT+BQohDIOAAQ988ZvmwWDu56o8s3vwhlSShlrMcYIYQAchGg9M51QgInxrQXKHSbCGITMhrR7vRqBGCOlzCaRQ4AhYBgFlASWkdZoQJJ4NV/kVekQttZqbYMAU4gRMhBiAgl3lECKEDIaIxhQEvrpwc5iP4QyCMKyqnwjNa/Kuq78x83Gk9VqZR0M47YyzjpQa+AcSJPUMX58cvT6g4eFKOq6JIRECV40F1EXdl1cVVUaR92knaZtbuIW6oq6ubc9MKJhBBAXf/nJ5c9+/CO1e7GS2YOH93fBzovDw4gHLz4/bbmBvLTRw7S+rHtRD3db2bzsdoPHT34z6BkNLn74W3fv3GpdXV2+mF0O+wMD8Fa6XRVlJeT0YiHLZ+2kzRhbZXOp1WQyefHixd27t1+7f7/bad25c8tY9cUXn22NdngYOcwKbZ6cnkEIO51ONT9stVpSg6ouDWKMhRBzguEyr3k4EKLGBpeZCFgWIFoVVcMPg07cGvQevnPvzXff+f3f/83hs0lQd0f9USWb3mhAA1U32WpZthOYZ8vp0V7TBK52o/7+zGWVLHkSHV8cL8vlo6eP/9bv/vb9+/fLqjg/P56MD68un33vez997423ZpfTx4+fOIlD1CUgbvM0y7Ko3bfW7vRbAWYKyqque73e/DTTRb64rBAln356hAhmnL+4WpEgZJ6izhglxLtzOAAtxhgTyBiN45AxUhSZ1hfOOa/vL8vaU4EIIdAhrbUSDcQIYwMwQghhQCACAAAeYKedtdpBV9e2aRpGOGMBwqDIVhjDKIrSJLJOAmjTVmiNZCEKAUlSZiHXWk5nk6IoFrM5QYRRFscMQyEbLZSua7WazpMkjniSFfnnnz4LIxomsXNua2c7y7KyKFrt9mgw9DDGcrWglBqjvDsyCUOrjdGqtFZLVdYVhDAvS084kM5BCFut1mQ+a5rm4ODg7r37y+VyPPHTHGQtRYQJxtgBKKVUyjDG9Nf9AT3Rb7OMAGADA67bkwA4KVWjTS0VANYC1OkNwjgCCDtIfFtAKeXb7FletttthCCAxgFjrVZaVFUjlMKIAmARggg5a50Dxi9G8nxV1pVSCl4vfZxzxmjGWFmW3W672+0uZ9MwDFer1Wg0fPfdt4fDYVVVazYlIWVZnpycGO0W0wXB9MGDB1a78eXlbDa5vLyMYl5WmdKiwzud3sgAN18shBAIQEKIx0uMMRhA55ynDWqtnz9/fnJycnV1tVqtGGPeVqGqqvl8CQDo9bqtVmKtLYoKYxiGESFMr4feasZYnCZlWdqX2hDr+81aa37tEo2uLXXruvYdB3CthtiQvDyi41WpURR5yetisfAX9jUH/qUybRPE/0txhf8Fm/+tbpj4YQghgBaClziHUtIzEL0q2JhmuVzGcYwAfPj6A+fcn/zx1XQ63dnZ2d/fPz1+8eWjz3/4w7f/yT/5J9Yhodx0tvzks8/PLs9++ctfHp883d3b6vU677/7ThKlv/7Fh/OZIsg48pKgR17aWsPBYDAej/O8HA367XbaNA1EbrGc94O4aaogCAgJrPGGH5IQnq1yAAFlwForpUYwIAQhRIyp/6bP5Ldur7QGNlQbIQSEDrqX/FnwUjb56gYhNEatlQtuTWWwxhng97w2ZdJabpQOr9Q9N+uhG8jEDUDCQQAxJsT7K6/BCePanV6StoMw9vsklGGMATQIrq/wzQ79Wh9dT3vyz3gQxd95XsoOAJjPlwgh704GXpKKjFIOWgcAoJQWZZbUsdYSAHttJ22MUdZSTCDCwFiljVRaFKWo67rO6zzPjdZxHBIIpGqMktaCPF8pUcVxHEXB1cUFxvDBg9fv3LljqDo6Ovry8VclYTt7u4NOV2oNoet0OqgHKUEXJ8dnZ6eMYAft5exeewCiuNXpDoC1vX5BEB5s7xw+PYQY7d7eSwZdpxzg6Pbdg1pUi2X2wQcf1HWZRIFSMkkjTLZaaZymcZ6vptPp5eWFUorz0ISqKGQcxxivfE+qrute2Lt//8H29ujx4y93dvbefvudnZ1tHtB+v//OO+94dOrevXthGJZlsVoxhAAk0DqNkFParZbTMOR3793uD7qz2cTzoghpOUuM1HkzUw29td0DCmhZOp0jYBACFBAgkSyckkhk9Hw6qSvR6dhedxgnHQgJwQHCHAIklb28mmndRHGyt3/HWPj48dMk7Y1GI4RdGNFut9vp9Lr9YV03GNHpfMY5v7y8VEqIRo5GoyzLbt8dKWlWeVaLqhFlzFphlKZJSJRqfBi1TlrHgbXGKmMUgAYhGIZ8SIZRFGxtDU9Oj7znYFEUQitKaS0aa7UFTilzDbKB9VW6vvwhhMAB44BzzkCAgUPWIefw2dlRvlphjHd2tnvdtrXWORMEgTZlXMbWWmkqhA2AxtfchFAMCXDEAoARS5NuzNvOuVpIz5mPgzZwJUGEEw4w6rS6O6Mt77m0yjJPhOx22/PVnFMGkwQBUBVlVVU+kXio3OcbhJBx1ii90jpfVlEUhQhVda2M3t7dmSyW5+MJpdRWtV0PsEeeMm2McXYdUDZVArmeqQhuyOh9l0FrDQi1DmllG1kaY7SDvemi2xsoiwxEDjNnIdBAAwgBNBALWRblki2JMaYRNUKWMqgtkLK2zmHisIVWaetbyxh7uzTnHMaAEAKQs9YoZTGGRZFtD0ej0WA1n4VRsJjPEEL9fndvb+fjTz+hlAIEgygGAGBEV6vs8uyi3W6//uDh1cX48vx8tVotFrM3XruLoFksszjkO9sjY0FZlkoZpUwQYMa4Py0AE0SgVE0UB1EUeY9YrbVnNQ4Gg6KoGGNRFAghq6rynNBWK7kOcNBCgDAAADNGecAJIWZdKBBjDJPCr9XQtUbLExEQQp6Z6GMfuHZ0RtdDcfyF4R2WiqJIksS7XOA1gAHA14Him5lmE+7/irrhL1sE/2XPO2cAsBujSYyxf4AQgsgB4Fno1Ji1cIYx5l1ufM6bTqdSynaa/vCHP6SUfvCLP//44w8//vhDynBdlGmaTq/G27v7aaursyaO49/6rR8cn40++/xjCGFRZAS7H//op3cO7uWL4smjw7oWtbHOGS9ztRBCiIF1wNnTk0tGIaeEEFLmmRJN0g0zB4xSdVUwSjBC2oFGKQAsgg5CwBjAGFpjGMUYEWeNURriNUf1P/N8/i/bXsnHm2d8Bz27jg8IIYiRcwYG65NvrUXf+KE2zQLwkiWjgPNsWWCM0dY6B5RSUq0HRmstjVHWav+yVw7sJlL1CpTifZT9peiLQm9D3u/3vfucL0F8TQwhhOBlm8xfNmuFy3V3AF6zIH2ZEATBYDBaLpdFUU0mE4wxAIgQZIzxlJjrI7HQj4q4Lg02Ck9/zB4O9K/3XQkhxGw2e/zZF6vVqqoqBKy1VtSV0gJDFIeB1jKNEwydEPX29uj9d995880383LaDUORFWeXVwljoqqWq5VQ8uzs9Pbtg3uv3U3TUDo5vrz4/KsvVlX2/ndfu3P71n0FoQMaICHkrTt3p/OFcjZOo6JYXF5N09YA04DF5OjLw8vLyyRJjBJHR0dhxCF0TYMHg57XpgWBH2oPpJQvXrzI8zKO09lsIUTNeWitnc1mGEPfPo7jOIoiiFy32/fFVpYt1fwFIWSxmGMEHLQQOqlq5wwm8WCQxHGYxBS4ptOOkoRACImMoqTlLD06np4cTpyie4MwW9VOLzCiGEIgYDnTYqEbIavSFtIUhYSuaSU4ClvOQeBwGNA4jk9OTj78zScYQ4isUmKxyFbL6t/9u//v66+/DjEZjYatVrvf2xpt72JMT05OT85OKaXL5dJavb+//3f/7t99cXT4m4/++M79u68//Pn+nbvGwkbp8Wz+R//hP5K6yZ1zhBCtkTHCIWKt9rIlrSVCKAzDwaC/tbUVBMFkMhkf51JKZl0Ueakx9A05ghkE2BjjPXUtWGOtStaMEGuAMTYIojRJMSRNU9dNIUSFCazqDCOttIAQxDqWeiFMCCGUWkhroXWcha1uByOipZONghYGEWu3A0Y4RWS5yHZGO5hAB+Hx8fFXT5/spa3/zf/ufxu3YkKIELJRTbfTanc7dV1fXV0ZqdJWHAWBKKvpdLqcLzAEnPMoCJ1RhBALHGZUKaWNbppGFHJ/fx8SPF0t9KNH3W63auputyuNbpQG6wYetVKubx7w0twUfJ0ej25IqjZOcFIbAKF2TkurtVImu5zOtna2q6aRykKIMYEOIB4EhBAHMba5UmK5mmptjYOUcuec1EqVAkKPTBrnjHUOAoicg5AiBJwDPuU4CJ2zAFjrgLXWAdPpdAjFzjkPHc3mk739naqqgiDQVemcY5SnacdaK4RqtZhzME7Cfr9blnlZZXdu79VV3lS5rEstGgOgFtJqo5QGABJC6rLykYsyYqyBkA0GgyzLAABpmvoT0u12ve9ku90uy8pLEwkhYRgiYiEEzhmM8cat0lqrtTLOOgusbay10lxP8AMv7Z48UOEVIjcRhZvgvy8CfKfWFxbex2KDPWyqBP8Wa79F0ffXntsghN4T00dnhAGl3rnPQri+uiB0WtskbXlD3xtHbjnnEDqlxHQ2ZoRqLTHGsm6Ubr6o53/wB3+4vXtw6/Z9beFguPWdd9/b298OQpzni5PTo5Ojw+FgJ02SMODzec0i6mkfAABveAwgRIDEEUII1UXVbbd++2e/tbOz04jqX/3Lf7mzP4qjtqrkbJop5TjGFDGGAWRQaSeEAw4wZp1ptEIEE/P1swe/bTTGf+WZfOWZm9QWb0cGIZRSBkGAMe70uoQgQxm4vnkR/NrQ8Fe7SMhBCCAEdq10MNdOpG7t2qyFNje8leyrjapXagX7dfarhxMARFIZKSXnfDDc2tvba7U7nHNMiAMIomuFjjO+rNl8x00sunl9bupgzjnnIWPMWXhyclbXwjOrRN1YawFwCDmMEcaEYkwwgdClrTSMOCbQOm2scgD78ZjWakSwc8ZayBhN01QppZQwRsmmaqqMUooRgMgFjMVhCKHDyGLorNX3793e2trCBNVNJZbZqNPdH40mlxfZfKZns7yuwjja29vZ2d06ONiLIp6VSxbg+WyyKOZHJxcBT2bzHDi9WBairh48vP+O1kqpvClefPno5PQ8bfda7X4cp8rWWVZgjMeXl3m+un37VrsTB0Fw+/Zdxlg7bQmh8jzvdvtN2VxcXOV5TlnHWjPoDQPGF7NlXqxCHkRBTGgaBTF0EEOsncpXBQCWUkrCkDGmtdJGQgwAMEJyZUXdgG63HUS0rLImU91um8FgMhnvD16zDla1KObLycVZFHe6nW2rgLMOI4QAdhqWlVZSNY3VgrJhjxBibeAcRzjQygKA4iQBAExny7IshsM+RG4yuep02rdvv/HwjeFoNErSlrVgOlleXFzOHi0o4RjTdifRWpZVhhC6d//u3Xu3rFNS3U7aCQ9tu0XSds8iGpyRfi8mNwtYY5xD/oIGdd3MZgsAACLYORiGYdPooqi0NhCiMIjSNBVSCiEsMBDCWgrjtBRG+9Fk8NpxzEmWdpwzZVl6yqQS8vLyvCxLghAGZLWa1dWKEEQp1aZp9SBllvAAUQAqqbSCGgWMxK1UVAoBTCAJWEAcFkJl5Qoh0+93y6oCADx4eE+qOk7C7a1+GEcWgqZpcAm11tlyUdd1U5UWQCWktbYsS1E3zrkwitI0VUI657wHFERIKmWsaZRMO+3pcoEQStN0ma2+ePIYI/raaw9pwIuyLorSxzXvVKqlouxr+KEvCDbZxSN7G1UhIaQsSu8lpaUyRlVVNR6P5/Md/0bfU/TLHcYYQiAka5+TuhZ+nk5d17PZDAIMobNgM1/RQQgdAAR4er+2FjvnqU0IIeSASdO4risAHOfU62UhhE1dTybj1WoBoLUWFHkJYVMLgzHW2k4ms6oot0aDNE3Oz04mk6t337o76rdOj9FidoUQAphmyxWBGAGIAOY0KExOKZVGAQCMUc4F3W738PCwqiqMaBzH3oZmMBhMJhM/YiqOY295pJRqxZGXlRrg1tN1LQDQNU1lHPAOuM4CgNe6LOheDvD1uTMIAm8AdTMhbSK+X1N6prcfIO6naG5cHDYx96/IQ3+tVYIFwPqGt89DAFrnHKXEGGMMgNB561J/XYlGaWUphRjjfn/Lah1FIQ/o5eX5nTt33n77zelkfHl52YgKGIAQ2BoNyrxazefNcLfXH07Gl1LV77z35iqb1uVCi6Yoin5XYQiapoIAePmcL5YgxB7dgBBCZxAArXb0+sN7v/u3frs/6CwWMymyNE3rWkRBNJt8rKVMkhZCRBshGpckIEmjNGlDSE5PrpSSSdRaNSvwjRX/f4POzqYL4CfBblTNrU4bvOLDAay9dh+6ySrw+9lUpQh6DiOwFiLsjLHXzQ3jK3K/KHfg2y8YjwnZa6LrBveiLPBmz0EQDAaD4XC4t7fnDeY9+xJdzxPxb7Rab04juiFW3BTH/kmPJYRhyGikFXDOHR0debuXdrvti3tn1yqhNSbhpBKmS3oeIvX/ohsqSmChXyw5JbXWq9Xq2bNnTZUJUWotMQLWOK0airBzJM9WEQ8ap63p/s7f/jvttLVYzi7Oj4kWo9Fo2GtzSoqqtghAp5yVg/5gMOjeunXQ7SVVs2x3wum0Vdf1zs4eQuTq6go6KxvlnEvT9OHDh2VZlmU5Hl8CALRR48nl0Ll2pxVHLaORFJbRKIpSJc10kl2cTzDmd+8+CIP4+PjYOUhAGYVpVYpeu/PGG2+0Wq2jo8OLs7Mw5PkyG08uvvOd9wLKgLXAwFbcaspqOp3Oy3mLS0wYDcMEdBkjELpaVNqIRKfaKgdYkvaappIGY4wxjylOnj07PDm9GE+XGJM4ThFhFjQ8CCHg2DENgRZSKOwsxxQgGHEKgKN1ZQjWSilKeCsNp9MpY6xpCOchpVi1TauV3Lv34Hd/9ztVWedldXx0enh4dHU1Pru4ms+Xu7u7t+/cubo6k7La3hn9+CffuXV7ezhqP3wjvLi4GE+XR0dfBXE7CNOqbBhFBELs269KmapqwDqTySyvEVpB6Aij2aqM41hK2dRr0TCllLNAGwuAgBBTiirRGOOU0gYYCKGDa6vUXrcVRbFSpqqENUAIVWTFdDo3xqRxKKXNsqU1mjFCKDLGjG7xOEp7vUGctBGheSa0thKpfmdYuMIoDRwgFBIIpJLaNdAKpfHV1Ym19q133v3B998+Ojn5wz/6/Xfefbfb7SJKZ9PxZ198vlwuB6Ptg4MDrXWRrcqyXK1WGONBr8cYJQgrAByCYRhCCClgNrOYEEKpQcAgYIBVwBJG01aLUh4lMaakO54aY72Dir+rjTEcsVfun02K8tHBwwl2bcNuobPM38nYccpKo4xorNZpHDurvcBS1g0lgBFgMESIxHHKWUhJUdSNaGRdizwr07RtrVXGae2sBc5CBwBwQJhaKgUAwNaHFUAIJRRBSCgmZZ7NF1PG2Go5D8K+dXow7JydnWT5ChEehmEjdFFUq3wCAIw4r5sSOssojgLKGFksZgF1tw62xuPt2SefX5ydaIfKSiNKOeEQYEo9kkmUkhA552y73U7T1NcEEGjOuVKqqiqtbZ7nXsUupfRBkBBinaYsjOOwkTLP8zzPAQAs4FprcM3Ehxgggn0oN3Izy8cAAPzKCd8Yr+yuVQwbkZtXmtR1nSSJf9L77G6C7GaNezND3Nz+itz2ly2O/7Ln1+IGaAG0AAIAkF+kQggJQdT7VyGntbZOIwOm0yml1AvZoyhSQjjnyrKcTqe//Ts/+6f/9P/07//gfw5DyjlP43Bvb+/dBw9H2zv/6l//m19+8NHf2jsYbI2eHT598tWjqimyfNrv928d7N0+uPXV8JlRqtuiOXR+JDR0EEGIEMEAAmeLqmQY7+yOOKfzxeTxo4+Wy9mDBw92dvZOTk6bUsYhVY2wunEQSlnHAfhbv/2zH/3oh+129+jF6f/v9/79o6+eAatfOXWbhPfXVSt8s5LbQPqbasD7efuUv5kq+QrO4bO4ud7cdenvnF1nYQSB9WnY46wIIuec78ka5yxE4OXA52/7gpvjuflf7UAlJMZ4ZzB86623dnd3/XwHwgP39U1r7YwBunGbkZLX/Qh3bUuKvPcdpUEQRFGUJAmgfjwNybMy4FFRFL5PwRhzwDpgjFVKO2D9HWSMUc4ZjCGlmHNKCLLASKUBAFAjP9K6lqJpmtPT0y+//DKfjH0z11pljVFNLZ3TqqYYOWBlI1fZbHtreOvW/nzSHg6H08ujgAdhyFutxCHnEAY1FFq9OHwOoLl/75Y2sqlKa3S3lXZb6bDXb+ry+MURQa7TjnvddjZbME4Dxpq83B7tDLe2lXYfffLFpx9/Nl8uMeDz2dJo1O12nSXT6WK5mJ0cn3/3u9/dHsEoskUurAUMhw9eeztNrpRSUqrJZDKdzq0BVdUcHR0J0QCAOp0eQkCqZjAYJEnSNPLo6AiPGIeYBnGEGaXYAGMwxUaFpNeISkpxenGxXC7393e3toawRB9//vj45Gy1LAFkhHdWFaiVgrRTNA4ChCCCjkiINcKAA0aocgTTSEo5na9qoZumcc4UdTEcDl9/883nh88q0cQk3Lt1gBDSFsym2fn5+dV4+uzZM2PsnTt3BqPtk5OT27cPHr7x+nx5ham2rq7qxdHJ462tYZ5lUZTcvt12EH7x5ZNHTw6LSrw4PiHOQmehsRY45SwGGGltlHJKW6UEpZQarGRZV4YxRnAEqZZSCqGqqqrqqq4biBGEwPh7ASFCGSHEAaC0kFLGURsjaqAOg9QasJivyryEgHS7HU6wddpaq5WAEGlltFEnJ3POpkVV7+8hymOpdVHUENRVKWSjoHVpFAcRDQMWRjhO6PjkGcb04HavKKqrq0NjASLq/Oyw12t99eRRnpXLPJtOZ2XVVGWjGrG9vS1kXZelMyZcWx1Yv6bEBMatVDVCa9lIwRizwC2Wy263a62dLxatVnt7d4cRbq0NgqDX61nr23V+rjTbVNnoJe1r3c70zDhfOqxHWBljjGlFQRSFVeWcxZxTAl0Y0FbMi5ACSwhBSinplBFKKCnqyhAC+5TzsCjqPKvqWijpCOHOImOcUlZr4xx0vmNqLILAz6HHGFOGCaWYrAcwStFwTs/Ozlpx5FFrjPH29ujp06f+yHnAjYWMBso0TS0xgNYAStByueTD3v7+/nwxnU2vhqPt7a3+l5Ss5rlyUCnEMXMOejtkCCFjTMiKYULI2pZxuVwmSWL0y8ztP30wGAghp9OpD21JklhY+zHHyhhrNSFIKVPXtdKC8TAIAs6dc9BB6F2eiFsHR3Nj8CP4OiHRh2B//gHQ3t0BIeSh3aZpPLHD/46bwG2v5/H8tSSwv3SDFrhNGeGhbGStlrIJgiBOQq/499WVEIKz0MvkpKillE1dIQS0Qi+Onn/00YeddjoY9H7ww//9aDSaja+staP+4Nb+Qb/bG18uv/zisx//5GejwbAoVsvVROl6NBpmy5XcbrYGQ2Oc0spS6I8HE4IQQQ5Za42xu1t7WTZfLGYf/+bDVsKzfPbhR7+izG2NRsv5nBE87PcwJNmqsM4FjNy7f/unP/nRe++9M58vj5zFCEAAjFr7VWxuk78hLGGz51cgpU0Fv0EBV6sVQoARuqGw+PFk17DKtX3C9YWklPQAvx+a7a7dCeF1UWCt3rzXOefdG28WIv5ff2lt+mKb3G8d5kE4GAzu3X/t9p277Xa7aZqiKPwQ502Q0Vo7bXxbA14LgH0rbfPp4AZHyscrzrkjLEnS1SqP4xShKcZYNMqvczCBABBjjFICEepxvlar5We1gI3Qd20g67Qx3mYtL/LlculHx8m6DMMwCkPf+IeOAesQglHIoQNRK85Xyy+++AxY1e/39/Z3L8+fvjg7upxdYYYxRdrYNI07lEiti3z16MvPi2z5+eefcUa2tobtdls3ZUgRdqYuypA42mt//tmnSRi9//77zoB+u8MRz/JVsSieP36+WGXchE+ePLFa7uxsn5+PMYaUkNk0+3/+P/4lp0wpBQBijPXaPUr56ekpBPKLzz5XSsVxNBj28jwXon748LX93YPbt+8Bq6ezsVFmcjWeXI2bSkTdrdZwSAjxfD6tpYIUOVeWubRhrcDVWBweX9aKCxNPJtnyYozDeNQaMd6qa3hyOp0uRZK2ilxABDnFjFGHkaIWAosYthpQyqXUvllmjMqyrKrL7Z2tbrd9eXU2m82EQACALMvG4/F/+OOrw8NDTEjTSGttGCwhgdvbo9HWEEDT7cUPX7/dH7SVyaraIdIa9LeV0oRyRNjR0WVd1mVRAgsJQsQ5gxBGiCCEESYIYgQxD2BdC0IIJqRpGinKMLTercljwnW99i7HYD0qEELISMAixjl3ADQNRbAxxuVVba0FDmstV6tCSxWGMYJYCGWMQphEEYbreSrccSO1qiuRVyVTbrFYjsczKez0atJtd27vH+yMBlvbg4DRusxLCrZ7rx8eHmKMu71oPJ46gEOOz8/P/+Df/1uMeNJqs2BtXqS1JYSB6yFAdV178aFR6hphQ4SQ2tW1FMs8CxmXUmIaFkXhWfRlWVZVFUXJaLhNGUuSRGtTlmVZ1n7EEWMMoZdd8M1Nu2HMeVZdEAQQwqqqyrIMAx5SohG00IWcaklDSgJGlWgQcMBiKUSd58YqAEBd14PhNqUMI1aVzXg8rcoGIYwRhRBbq7U2WjsIIQTIWWsdpAGCGCKE1tMaMbiOTa4sy0G3M5lMuq177XZba8k5bbfbZVmGYdhIJ4QoilorhxENQ2ykYowlMV8uF2kcvP7wXt0UX372KX4fO6OMUc4AjCCgNAiCcrF018GXUoIx5pxTiv3at6qqTqej5MuZOnfv3rXWhmHIecAYi+PYj+1JkmS5XJZlSRhvt9vtdnu1Ws2Xq263GydJHKf+I6TRRV7leV4uy5vcAvty5ec2tYL/da7/arznTL/fr+u63W57mZy7Jo1vovammvnWPPTXldJubOuk5j+6aRrPYuv1eowxrWVVVU3TPHywhTGsquroxXPf1eKcQoin0+mvfvWr/b2tDz/6i7LKj49fGCUfPnz4+PHjP/j9fz+erB482Pv880fzxepnv/OzN9984+w8VLomBD99+qTb7R8cHPQ66el5bpAHZgjBhEBiLdDaGKUvri7aacwo/urJWdL6i1Y7nM9Ko9V0fHVxfh5Hra3RKOSJlifW6l6/defWASP4N7/+iz/6oz96/OjZdNpAAMKAlY3enMP/Bh2Hb243rxNjzGq1IgQFbD1BjVIK8Poa2DQdfKGwXvF7EwJs4bUGwPs6+n1fv157aGFTH7xSJfhlw8aKwB+J3wjDw+Hwtddeu3fvXpIkviMZxzG69t3aXJ8+LJMbMcc3Ljf7d9dNtM1NASHEmKZJi9Kxbwj6DOdBBbq2ATVSGkhZksaDwWCwPWScE4KU8kAM8CyHIAiyIve+Ulm+XGWLqi4cMFmWIegwAsYY2VQAAIoxAsAYAoEdDbZOT08/+MWfnxy9eP3118uy/OTzz05OTlZFaaytRWOc2x7u7e3taS1ns1kchAS4Qb9rpCiWq9nV5RtvvH6wf5Cm8fnpsdGNkfLpl4/b7fY//Af/sCiqqqgfffns8mqRl00at9rtfjmTVSUX08l0OpeyuXXr4OGD+0argCfZajWZTIaDLYIDrUC/1+Vs3jQyjuPVajWbzQghdVPO59M0jZVSsmkYI5zzLFs+ffqsLMu33nprd787Go0QwXmei0Y1TWMxBwAAHMzPTuO494Mf/Q4LO6fn5428JIRsHezFcQvjJAh71gbCvrj69NmiyOK4hR0nKACUI6IgtM5KQ4wWmFBCGSMU9QZdrVMHnbHqxfGhBfsO2sGoTwipRXU5vggini/F6cnlu++/9957dy4uLs7OTmhAv//97+7tb1uofvSjHzCOKEPz1XQ8ObkcH+337n7y+WdVI77/w5989/3vj7YPTk7PP/zkU6KlqpoaQhiliXC6zArjvNqKwpBpYxshEcJef6WUmQgRp70gCKwxQFgIkRLGAasFdBAZ5IyzzgEeBjTggAQhxJZdjyCra90UzjkICSXIQODWfWsUhmGSJGEYKpoVRVEX9cnhRRKlQRDuD4damtvDkWrUdrdPIb46vdre2h12D4CerPIXYafjBYTd0aAoiroSt2/vvXhxTDCgCAaIorAToTRN2zGK2/3WcrlcZEspJSeUEAwhsNYKIaCEspLz8bwoipDGCCKKPZPJGK0MQBghRvlsMn325OmDB6+/8fpbFxcXz549M0YxxijDCAPZGIQc55yxEAEthDAQIEaVsFGQwhBKKa2GGGNgsWxMg2gStYKQzSYXEQWdKGEEDZJYbA8//vjjMIwRZU0ljQUAIOciKYqqzBAAF5dnVuk4TsbjaZx0GmmkVMYCzKi1VqoGYMQYs1bxiBsLIEYsiLOyqut6MBhUVd1ub9Wi7rR78+mi304arSOCJhdZJxlejqdRnGrrnNHWaYgIoxhFrN/vX5yevP+d97QUPE3f/e73n372y68Oj97/zg/Qn/1GuEY2loagKIW2qB21EEKcxXleMRoSSAPK77/+8IPf/AUPAl+iRUGYKR1iTix4cOf+48ePD27d6feGvcHwLz78ECKclyULwjDhSRreubt797WDKGZNU1POppPZxflsvixFY23lIDIQORRiCZWlrr89mE6n+3cPNDSrPBNSQggxWlsga22UMFprHiI/p2o8Hm9tbbVarSdPnkgpkyQB12umTZLwAfRb880rtYK70cO+mRI25ePGuu4m6RUA4CBB2AEIjV2vCwmlCFPGg9liWVR13egkifxc1jhOWmlcFIVWwhc6kGBEGQmieVZ+9vjoj/70w6pqDs8k50Gato+v+NXF2a9+9Ushxd6tbRnigsi9N2/98Aff3znbi3jw+PMvnj09rnPBgLBaQAAi1jbGVFWV9FlRFIQx5ar+dj8rVrksb+/c0oz+4sMXEIEf/eh7z87kH/3J/5tzdmv/YFUXmLvtg24cRlEUlUL88Z980Ch9eLY6njUAgU6nWwBkjId8NhMNHFjzoL+9l79Zqf9XbvjahxFeEwKcc76E5ZzGYeSVSkqpgFPGGLgeBmHWdsvI/xVghDBGiK47X9ZY46wxSiqnIYWEQqIAtsZSCyDElZacc7CecGsppQBiv09prDHArM8EDdNWmqaD3tb29vbBwUGn3YEOAueiIGSMFUXhEHJGWWOttdARigNAMNDSOedlmeDaxMwzMHyJ4KtJ/0W01oQaIZskSd5+++2PPvrID6NRymAMrQEQQIxpGNI4isMgYTRU0jAKGQlDnlASAMeg4QSECAbdVogQODx6/viLLy8vL7SRi8XMWVvXAgCEMQaIOW2UgwYhrAlC6PBkDkB0Pq7G86Mvvjonv/cnWXHeGw729vYwAEZozmncbaOALcfz3VsHd+/cgdrKWn752eeqEYwFSYtYWM/mK+vqgKHTk8Ot7X4aJ5cXp6usmEwXTw6Pq0Y5QBarjAVhaKNeKxb5KoqiWSMZ4nWhEOJpzFthZ2e475EkhPB8Pu90OkXDOeeQJHYxrqp6OBrcu7MPrNVNOT4/M04XRfHpF5+fnJ6+/73vPnj4kEepM1gpS2EStGm7DdhicXl1xWmXYC0awHhr/+C7mGwnSbK/vw91/Qe//4c//um7YdoqK/Hu9947Gl9leTFZHu4f3B5stzFEUkIOIuBCpZQpp3UJwjDsDYdhgKtGhhEvS12W5cnJmbV6tZxbazqd1u39oRYZBa2AtoDCP//pbx+dPP9n//wXq2x5fPb03uv333z7rZ2dHRwlZSN//enZhx9+zDnfGnxxeXk56I9GB8W913/4+ts/vji/un3wFomiyBiTFXnZ1JhQRLCDUCkVhvHancPXpNooaJ2xDiMAgJIyz3PV1JyxMAics9ABbY1DjmKCIbLaOOCQA0mS+vBaFLmPtlJK54ySBiHEGIOQ+46ap5WtVitgLQQEI2wN1I2GEENApZCMMISpsxg6oJQRSnMedtAgjtaXvtKCEp4m2lpPWSeUBACQbFVp0zjQWCeOj1845xACQcCgdVI2dV1LIXy80EZqI41V1mkIsHNGCOnPACGYc85ZACE0Jp3NZk+ffWUN2Nvbq6rK0/jb7Taw0H/Hsiydc15C48+kDzfu2s7Zs44RxwZaiAELGeYEWOmgdchhAoeDHqW8KGtOUcBjTDkhpGimeZ4raeqyyvM8SWAURXEcNo3UGkGngfVaJm+o7wxw7ShphMjz3AEYBAGKQ6cVoVgrAa1jjFoj/PFIrZfLZdU0PsNhAFjAGQ4ZDZJWqrVM4hjdOojjeCXFfD6P47jb60dR0m63D27dGs9yRHEQxUoDopH3OwpDDiDVWvq4vLW19fjx4yzLIAAhCwEAhJAsy7TWW9s7k8mkrPJsXE3nM4gcAA44a6yr8nKVTYXOHFIHt3bCOPzkk0+yVTlb5E1tjEZKwqoUtWg8DTBNUy/cUEp1Op3J5cQLHxz4mh2CX435lZEnTyyXSy8i2LzM3WhU/7WkqM3ebsJO/slrGONbyPC+FUIIEUJI2Rhjer1er9fL83yxWOT5CiEUBAxAV1XVYrFwzgEH/YgBrawxFuOr5SJ79OUXaZKOht26UlGY/u3f/e/ff+97V1eTxXx1VpxYa7e2tpbLuZaqP+g24uoqW3V6XSGhJ23k2SppJXfu3CKE/Oaj35yfnxtjophEUdTtdtvt9vnRc0qZR8uMEnVdH+zt37p1589+8efnZxdlrRChnGOpXFVVDuJvPav/DZoRG4QJfL0tJaXM85wgbIxptVpJkmwoIJs3GmOAdT7R4uvDd8AA51f5FqwbVWpj3w4RcMZttA/oehQ7QgjANTaw6RRIKRkP+v3+/v7+g/tvRFGUpul1ClfGmLquN/2FDRxy3RnBCEFCESEkiuMojhFCG1sRvweMcZK246QVxamBGELIOR8MBjs7W8+fP/ffV4jaGUsIoQyHYZimaRzHfmJkFEVhGFLOKKGUkiAIeBR6WLqRdV3XCEEHTF6WRV2ljFNOKCcQQicNIJBzHvGgKIowDClbT7iWRjZNJYTgiUMIQYQscAHjYRRYa6fT6VePHh8dvnj86BHU9uLspMjyQb+/PdqKwpY1KMvKfLlqt1qtuN1ukTRt/fKDD6UyBkClbBim1kE7XzVlk/LWzs7OuvMSBu1Oqo0EBoQkjFuJ7zzO5/PFbOUhJUwxJgBhu7+zPRx0rZZGN4Ot4b/6l//6j//wj773g+9Pp9O8LIZbO2nQunP34VfHp8bIMAwHwwHnPMsyQMu43Zkvpt1+z59YrWXSirvdFqbg5Oh0uZqdnx/v37nf63Upj27f3vvq6WEYch5QyjDFhDJMCCIIW2tfnM/LMs/Kotay16sppZSB0KEwZM5Wi8Xk4uIMOofRbhJvhRE/P31eNMurGfrk8w8W2SJKMGKJQ2CxmE/mU+PceDp78uzwq6fPRCO3trbm87lvMNV1WRTZwcGdfr9PCCEYwDAMAUK1EGY9zA05BwghjAYQQuikFlpq7e14IYVNXQNjyzyHwMZByCmz1his/eVOMCYQOQcCzsOwVSxzAEBZltPpOM9zuDbht1EQI4TW9qjGKi2VAAghiyHnUcgDSjmEkELCKKWEF1kZRRElYVVKpVReisl4jjF++PpOGIbW6vliulwuJRfeWX3/9i6ECEEipXZIaQcgFtIu85UKw+tGrxBGKOu09/JDiHhDM+cMhA4h4NayYWstcE5UFRJCNLUoy5KxoGmaOEq9qZ8xpizLui6jIPU6K9/n9pvWutfrecnDhqngi/qiWBWiQMA6JbgCRtQhx0KWd+4e9Htt1cgXh8eqrrQWq8kcY3Lr4S0IIees1+tV1doDQ9QNJhQhBJSz0GOhGCIAncWYbmh91mgpGgihRSCJ47oQnJE4DvNlY60FCNa1aKVRq9XiYRSnbSEVDULMKA+iyWSSJFGSpvv7u0HAGSO9bvu1115rlltnl1c0TL7/Wz86u5pfjWfWOQvd7u72eDqzwARBoo3UorEQaK1ns9l0Oo3jGAKAASyKwqs5mqYpywIAl+f5fLkUVzJKE8ZY3KaMkVCTLJ9NJhMhq9PzkzDkTdPUwkipnMOYEogIgIQwnmclAIBSmue5F/T6Zpldu2C9bFFvWsJeiX7r1q0oip48eWKMCcPQQ8rgP1vRcDPhffMtzn3tc8HXJ/uB64zonHMQbAqFm7WCEMI647OUtev0EwRBlRdlWWot/SVtrPYWrdZa4KC30PD0ZGtrZ2HAEoy40RAzRilqGnV2ejW+OqubsqnyfrvVarXiJPytH35P1Pk//+f/yjhDECQINlWZtFIIw9/5+c9+9NOfXFycHx8fXY6vIIQB41qqbLkKGG+321EUyUZQStM4Zqza2ztI03S5WM3ntXYgCAGjAcLOaKekIAhvvuPfXOvhZrV3E+DZ/PVmoaCUEHXjBTiUUs7WHhX+KlqDTNcPXk7nWtsurUkt1w0N7VVz6x/OGP8niF5KQDeXhLneT7vd3t3bf+utt+7du0cQ96/ZGCr4ALK5ftx1N219YHrNTuCcx3GcJIlfg3mvhXXyw9jPa8AYWwe11pTi/f3dt99++8mTJ5RhSrHW2DljHcCYJUnU6bSSJOKcAuQQgQB5praEBAMMEAKYckJQXi4b3STtVlf0DTCU4ZSxcK0Y1EWW13VdC2OMipIoCAKrdVHlHvAgGEMCwyjkQUApJQiGYYgxKrJ8tVo5a6UQxSqjmERRxClrt9tRmkgFCEbW0kYiM6+rwtdhS06YcTAIIs7SIIql1MBiay0IzM7OFsYQAJAk0cZmDSLkfK8IAAeAVMrTXfNivrO7lS0XVZF/9zvvvfvWm3VVTMfjn//k57PZ7Pf+p39XlmVvMChWzT/9P/xfQGWDdm82mymh24hRyjUiUasz2N4eVTvdbhsB++jRF1eTS9eoq6uLzz//NJ9d1E0+m497W4O0nTigh6Pe2cV5GPJ2J2IMO2usk8ZiCDAAAIZO17IWJZaq7VAat6IYGAuDwIq6ruoiSkqCYKsjkpbgHJBo2abK0NlXLz4ACI722w6CLC+ErqqmRCV9/OzpBx/8hlD23e9+/+79+0T76hacnJz84R/+4ZMnz66urn75i1+Rum4AACEP0jSthcqKXCmNKBO1dAZAgLWUUkroAEIIOmiszGsBIeSUhiHnhKpGKC2nk0kthIOWhwGihFDaGw467TRq94qiyLPx+GouVdPtduM4hdBpbQAwzq2h3Q0Z0AGAAXOWGAGsNRo5Zwjgbnt7nxBS5cVqtZJSykY454Ig6g6Tu3cHo0E3SNqInld1UdeFK0vKsFmPGHA8IaGhSpna5IRGEDmlhZBW1o2UEgDHGKmqCkIMoPX6YGMVckAb6W37tJa+T+w5eqKRQWCUWuY8j+M4jtLBoJ+m6dXVlfc1C8PQuwVsjPP8fvzUDC+9U0o1TVNAZ6ziBAcMKk0AMJQyzlkShcQ5E2kEbu1tb61W+enpOcGslUS1UEmS3L931xg3ncxlLRblYjjYgs46Z511iBJ6zdjnAc/yHELYbrchAMvlEkIXsUHEMIn41taw3+scPtMAAGtcLRrgMmU0hFAplRX5Mi8QxpQWDsHh9lav3QkjfnJ0JGRtjLpz9+75ePFnf/4rZfGbb7333nfeP7ucIBIghDFhLKB1XTNKTSUwQYQQYOH5+fnl5WWv3YEQAoe8HAsBWFXF46+mvnZst5NaNNrpKE6VyngQx0kAaauusYOuaRoDnJJGKWMssA4iCyGEiBICUBwD6a9YCOu6vry87Pf7Puvf5BxACNcqP+e8tMTP+fQz8TyxcZOqwY2WwV+GK9xMdTfXwd+a/NwNAsSrO/x6obDJB4wxqcS6b02wtTbLsqZpOKGegOK1bY2orbVBEOR5brT1i0VnvdkJhBD2+4Orq6tlpjrdzkIU/9O/+bdffvHYGvHZ50/eeLj7uz//qTEuz8rVasU5b7WBWAFgFeM4z0VTF5zzXr97dPj8yy+/nE6nCEBKKEG0aMqmaQAAW6OdJEnGVxeDweDh/dccsPfv3nv86KvVahVFGCAmpKqrKghTGlApX04SB99AVjYP/msKiG9rXqx1jzdf8xJnQtAY5RXU664EdFEUSSlvkgfBtcWqturGnq+1owBvPDAYY5Q6Sr1xFgDo5XQSv0F0zfFCOAzDra2t27dv7+0f9Pv9KIrKvPH0Ah8hvYQHIbTxGrmuHdcCTkf5pl3lJ/lRSgm1TdMQSiGE/uMQpg4gY4EDpmkqxthwOHzn3bf+4N//z0opKZsw5EpKazVCgDEWhiHnHOHNjDctjbQAUG4dBABBQpCFdpGtFos5Iag36LIQY4zrVbYxP5VGCSXKslwptR/uGWeU1Y0SxhiOAsQJghYzKo1W1sRhrLWWslFaW23SKEYIeUfdVhKVZamsKcpSX6gkSZxxxlBjAEQYQmiUieOWUwpA5rTTjdXScsSiJGqnaRzHdVMopdJOHIWJMoox1u518zxfLefWWshI2E51nldV1Ru2s2IVJVGv3z4+Pp5eXX7/e9/53d/920ZJgumf/emfN0qnjb26XLRb/eVsifspIoGxuqylcUAqhQhmAXfApmmMoIvjMEnDRuQXl1effvIJNFUr7RgrlqtpVuTGoaquEHadbjuKGERGSSGlhMhRTCCEyYDgOFYSJxHt90GnpQgxzkglZ2Fso9Rt77UYY602RnDZSLF/n1tLrQG1OcWOdvrcAYxZuFgVQuaoRlVdWGDuP3zt57/zO9vb208++fD4+Go+WyKEL87Pv3r0uCzrs9NzsjXcPrs4r7Jl2mkzSiHASgniVCmNIAo6JxrprGWEetN7BEBV10EQDLq9dpqsVqvFfFoUOaKEAKi0qYvSQhDEEdAGAxgFkRI6YGEat6xN4iCGDopGWGsxhoQQSgila10vpbRW3DlntTXOzxuklHCCWRzHVVUtVtlqtQIA+MnNyuiPP3kURq3haLfXpw7Aqs7ruiyrXIgmz1dlUTRNI5WGGELrlBEUB0LUZamllEaqdTjwwQhSAKzWUspGKemckUJQyn0/whofvrF3NMtWeVU1CBUeGOCc9/u9TqethFssFlVV+e5glmXOOe8sJKUsisK3JHw7xjmXpGldV2FAooCGYUAg73VbrVZLa301vpC+D5fGdV1bqzGn5+enUmojVW846nXai/kKQQAtaJrGGoMAMABABwiGXqdFCINQQuiM1lZJil0rSTppFHM66rfeffudra0titFisZhPps6CvCyEEBYgaZ0/WoiQQ/VwOKSULrLVxbh69MXnYcCWyziO49OTk6+enwbp4+Hu7dHubtju1o0cj6cO6E63xTgRdUMIwWHo145RlACAiqJCALaSKI5DQlCR5XVdlnX18OHD5eWiqKu01ZpP55Ti197aBgBI1SAFGKMsDLrddpIk0/kCC4Ow1hYhyCHAxABDHcOsLMvlctnv9ufzuYcHOOd+LB72ccQzzDECADSyDsPQR948zz2h0qeEm8v9/2RC2tQf30xvr3AUNi975b2bx5tCYVMlIISSJIEIXOvXXxYZBCJjDFxTadQGPQqCoCprjDGEQChlrQUANU2TL0tKCUfMOScaVdXN06eH2aqkFJydXn36yaM333htb+/2ZLw4OTkbDlq5rOeLZbsdtxIehuGD118fDfr/9t/+2xfHR02l2u1Y+7Z3rabjmTMAWN3vd2ezhZQ6CILRaNTt9FarlTHOGOes9g+apuEcAoC+9dzCr7ce/qbxBr/5IgBi71wEmqZZLpeMsTBg3rb1JgK0ASGs09fHti4AEMIIAYQBIYRz55yf1+oYY0KIWqw9mDcKGkKpN1ROO929vb1bt27t7OxQxquqOj8/73dHG04iuFZ1bs7STUTBExgR5xtZry8g0LWbgh9N5/fgDaERQphAIQQhKEyi27dv37q1//Tp0zzPR1sDrbxXhwHAQQSg99CgGFEMCcYYUk54GHBOAQZlUzZNc3p+cnRyjBDgnBJCkiSuq1VjaiCtcw5TEKTcQOVqM80mXdDmnEetAFjHONZG1KJMHS+qsmrqNE0xxlZrgtCw28uyJQCgaRotZZiEkGCKaNJtc0KzVb5arYQQnIcAMmdhluVCQqVUyAOnDcbYamOU7rTat27vUUovx2dV5TqdVtxKhRatTjtJWohgBaw1DmJkCaqUglo3Tl9NZ7vbW4TRRZ7duvP2u9/9XhxH/+Jf/Iumqt/77vdm0wXEhDH26ceftjptFJF2wDAOAwSQlgQ4IZr5ZVlWq9nVqbFqMhkbJZ3Vsi6aOu+1k+GojzFaLpeNUFI7zCOEAMbIWqu1BNASiiilIeeEkPYQQtSDTkJXU6QCsqRUEyiXiytOQasNAHKcA0KKqqqaLOu2b2utlbJlrZQiVrFGAIQCRMx4cgJm07yYRyEd9tsY6dPjw+PD5+fn52VZx3Ea8SDkPAkTTjj5yU9+8qtf/erLR1/VlegP061RkBeV1Aoh5CxomgZ4ig50Wmi/8Aooi1kQh1HAWG5dU9X5Krtz547gsmxKoSTAKGYBtC5bro4uT4wxzoHRaAusW4BZURTb29uME18jQwidM84BKbVS0ocGznmSJL5zzDmvinI2m8zmYyGUr+UddEVdqrE7v5yMdmZBwBpppXbaIuMwxExbmJdNlmXGGAgxAMgBqHTjm4vGGOeMNlprbbVptVqUIsYxJgAii7BD2EFkN9RlAJ210DlDCKOUR3GotXYOYgyFEMcnL/b399966y1Rufl8Pp1OAQCdTqfVakEIO52O70H4dap3T/KPAQ4NMhxz6oCWhhBkLcjzYtjvXVxN5pNpEESE0PlseXR6QghL2pEvTZKkFTBqtLTWhpzKpgLI+/I76AwAGEEHINDGJq3UajWfT6Ezb7/++oP79zACBIKdnZ3333l7MBgUq+WjJ19Np3PCA1XXDkEAICI4bqVJp8uj0AEUBAFAcDydrBbzsqlbrYRSWonGWLS1u2sBfvLsWbvbVUafXZx+8MGvu91umqZG2aaqsR9s4UC32/Ugf8i4lkpLKZt6tVpVRdnrd+7v3n399QdVUx4ej6Vuslxbd/n9zv1aCpE7gCEChFIKMXEAd9r9RmopLXAQkwAArKRRSs0u5845r03yiy0fha+lbniTtuG1lzYAIIqi1Wo1Ho99/N00pDcr/g3D/K9GFMA38tkrQLe7wXt/5TU3Hr/EG/D1RilttVMvipOy4Zz3er12u10X5WKxyLIlhDCKosBxL5v0X805gBB2zutjqVaWccQ5tdZigsIwNM5GYQIsDAKWr+Z/9mefzSbzmCXboyFnSa+73RpEH330kWxKa0E6Gvz0x781GAystQThNIaEoHJVcs6BBfPpdDqdAqu3trZE3cxms73tHYxpFMRBEN2799rTJ8/LWiVJjB0qq6YqK3CtPr1ZYIGvV1rffPBftG3e+8ov9c3f0V3LFP04b38VFUXRNO0NIoWubQk8tGOtBXizT+ecgZB45isAABMIEYEAe6VMEDCtw0ZCrbW6lkw65wjlniLQHQyHw2EURVVVISG9PBgA4C8Adz0aG14TYP0hbc7J2nCMRS81vZg4hCGhGGOACQsCjDGi2jmHGQeYWIgizrTWEDmtZRyH77zzzvHxC23WRAoIN9IJC4BFCEMCAIIAAYAgJhBRpJ1WtaqaOsuWq2yZ56tG1kkSRXEIEOAhK8uyFKUzllJKOOEuAMhxziFDFlpIIADQYkcJadE0SVMAgBeTy7qp65pTxhFxxgRB4Izx3RPnHMI4SdM0SWbLRdXIKGl1Oh2M6Hg8eXZ4tLO17YxpJQlGCBmjZWOt1aNe2oq0NphAGmBIoHEaYsAjrqwKkng7CgCAZV1bhLuE9rdGi+W0u4MAp5eLZdrr//f/8B/v3jr4H/9v/9e/+PiTTtryFhech00tLqZXqyoPiivfYi7LQhlJCLFWC9U4YJb5SmvpgJFSAq06rdZr9+4NB92AR7UQAIDBYIRpUApZVhJCzywh0EGCGaWEMU4pbR20Ak6Qa6rsqshOpckpEJiYTlsjII3V1gIICog4C3ULWmfnCFlMAHQCWCoEWMyrJN0igE9nSyEBMHhn2E5jNhufn56fTadXhKJOp+UcFEIoJRRQs9mEDPujNx6+6RxcZVkUxXHaSpNqvlzUdaOdBs5BgCklnKxHJGupkihOwkhUtZUCQ9RKUq1kU9dKa9UI62w7TdudjtXm9Oi4WnhPsV6v13POzedzAlG31Wm325igDRFXijWyV4qlh+4pJoxggiB0FlizzBYXV+dlVRJCtDGYBozSRpYOJOcXEwc/xRgKWWktlRJC1pSSpmnKSksFnSPXVTbSuoLQS5mQddYY54feWqcBtH6Cs/dQw35KnnMAWISAc0gZJYWWUlKqwjDuD2hZVD6IlGV9cXExGAwwCAkh3jlca/9iuklazjnOuS+WPWvaauUkQAQppU2tHIM5QifHF2kULxf5Kq8abSnlPE62bt2SUjOsMcayEdaoIAis0k3V8CCoyiYMESLMGGOdAUZbRCCEjZIxDjGmnozy8LX7P/nxDxBwVpt+v9vtpFpJT70sy3Jra2u5XNZ1bSwIHEjS1mB7ZzQaYcJanXbTVEopP/KnaZow4g6CVV5hwqtanF1cjXb3zq/Gq9UKEjiZXBGCZKOkEEmYlHkRRdG923eEEAcHBwSi5XxxevzCKD2fLzlDaZreun3AA/b9H3zv+PyoKFWng9rt9mQ+9oUdYwSRMAgCzgLMaJ03WltjLSEcY2wt9N/CDz6I/v/M/WezLGl2Hoqt16Yvv+3xfdpPjwHHAAM3AAiC5AV5v0pUSJcR92dJ+iCFpNClFJdXAA0gggA5IAbDHgymu2em7enjty+fPl+39OGtqrP7zAxNBC+gjB07ateuqszKfHOZZz3rWXHsxwdba8fjcbEqdq53h/oSQn3dwVrrS5U7GuPOl+ze8l8OKvwil/aSx/pZmGGTpCLu7LzHEryf2HmpHcdlN0bEbxu2Gm4UO7xT8W0TjDHOqZRhC4pRlecLrSFKAuegbjpCSC/NLi4vBlk6TMVq0XzvL37wa9/+ld/57X/w3//j9PPTZ4zTn/zkJ8uVlpxlSfzjH78/n14a1SoDWmujIctkkqAHZpqmUcpkWda27Xq9Pj09tdogogfYWq2FCACBc2OtQ8IAXkhTvHTefm6s4CUp/2u33RW8vq/r12K3X2ctIRRgs0g8T9mT+MBZb6N2r6eUIt0e2Iad4BABke6gHUo9OY96WCJFaa1F2PRZWGspE151MUozIURd14gYJ+lgMIjjeLUodpDA9ZPDNjLeZBfB+CdhOwsNAIIgkEEoZEApBWIYF5xzQpnz4pFArEMgjjKglJZl6Zz7ylfeeffd77dNI+UmhY2TMIqDF8EHAIK11jJmHQEE2zRN1dRlmZdlSSjN+j2z7CgjXMrOdIIRJpgIBFq3aRqigBSyQa9t26rZKNsKFGHW89MrwjAMo6hq6rIsKRCKkK/XdVnFYSQY19ZorVfrNSL2hgNKsG3brN+7e/eVOO0ppYExbTEOQ921SZoISsDoYtVq1WrVzOdTpZS1JopDY1S9rFblOkhjKUOgEIQRocxRohHGB/uDwSDOeuvVgjMC1tbF8tMnT5+fnQZpdvOVu4N+9tOf/vRgsieyEBzpXFsscvPg1BiT5/l0MWu7LooiGUsgxIGtVMuEiOKAMhbG0dFkf9zvt0qXZWkNRlF0+9adtDf6/MnzxbK0Bh0S54BTRiknILziEdBREAaCKTRWN4VxLaAlzqRx6CyqTbMLUkDJBY+F6lrBgIBUjQLJOaPrGVJspZRoa6uwlw77g2Esoa1mtl0X67UQMsv6lPCubc5OTglhZVHyk5OTNE1ff/31Z8+f153yBsg5PD09RURngTPmpTYIYUbpqirGg6GUsshzcHY4HO7t7VFGfPdtEARpGNy6fXv/6HC5WpVVHobhVlE89Pw+KcPRaNS2tTWuM0rpzs/Y8H0K+/uTKIqSJPGUQACom7JpyXI5XyxmjLE0jY1VaRamaUqobWo4PT87vzwzVgEgYwTBGqOstX7JMR6QjXrAhvROtzNYvTEFAC+5I4RottuuQYMAbFulKYCw1gJSSmld18PhsCyq1Wq1tzcZDvtKmYcPH0ay7wuNlNLFYuE/1lcr/H3uqUaI6LHuKEzAIKcBOCTOUGDOQF3UXWuNQ8qFsk6ZbjhOUtq/ms4jIEQIpRqyEW6yXQcydM4AADBOnC/cWEs9PYfSsizjKByNRh7munPz1nCQobWc86IoTs4uTk6fX15erlYrJCCEIIw6Zynn/eHg7t27d1+5F8hosVoCuF6v50lzi/VqMOxNJpPHDy+sxaptEuwNR6Mf/PDdZb48Pj5cLBZpmq7NimrKOfX++/XXXz9fXhVF8fCzByfPnq+W1cEksxYYYw8ePDLOWGv/0X//+wcHB7OPT3pDORoN7ty50zRNWZZ123hWByCx4FcpEEqECKSIlTYWXdO1fDtn7/T56d7eXtu2+/v7jz9/fN0lbNFa59EdXznyq8JfJl/vv26I/7Ou6D8Bkv/sn9dd1PUo4WffsnMDnvRe17VSilIghJRlWRRFW9Vmq9pb17XSGwqeJ7rvklHnvPZJw1gThAQRg0BSyhwipXy5XA/6o7aqbWsIuA/my7psjg9uvvPO2/fv3b1/7+7F2bmU+Ze/8s6dO3f+/Z9/l3OepmmnLaXUuZoRWtfKOZcmGWOsLMs4jBBxvS7q+sGDTz9rW3Vydtq2LSU8z/POgpBBHMV11/6XnNv/NbafxRV259xaZ51/YOq6Xq/XmxfjCxhg57yV23wFQgHdriphvUb7LqyhlHJOOeeUZYjIuPQ9UMYY2FYz/cXyj4UMlFLr9TqN+x5LeAnP8Eya6yHUJpgg4DkBPjTx9Vz/yo0U+obmsokg9aYsBU3TUAp3796dTCZGK0JIIGQUBx7+ZIx5lrfws3AZ4LZxQ1lV1+XV1VWrWi7oaDQwpg3iMOslVVXl6xkAeO4kpVS1nVIKCFkul148Owkjn4LXbSPDwMFmQHFZlkqp8WgQcjGdTtM4SeOkauqmaE1j2rYVYeB5RYyRIJTa6Wcnz5eLFWMsTpP9yV5d5kkQSAbEGup0zSAO5MXFhXPOgh2kaVGVTdH4uYDD8ahulTYGwFrAIAqzXm80GStkd+6/3lZlWax7vR4Dg6Z77e03f+3Xv9229d//h3//4uKiaRop5U8++uDg4KDXFMzaGMxeEraCGLREdcgpJSSVUoZBZ8xssQCAIAyttUTK5XJJmZCxC4IojtOu67SyiASBUEqYEFIGQRBEgZRSViUNZRSkcRB2SVIZarlbAaLTQEAGnArqgDBCuHHgrI2kYjRA4MUKhOQYRFNZMjSSEg6WWCc4CE66etXWFG07Go2qquKcjoaT2WxxfnYhZTgaTfi73//e66+9OV3Mu05LxucXU6CcahCOrdfLJIpu7u/3+qlg1BlVoZ7c3EPbTS+nXdeFYbxYmiSMbt+4OZ3OBOOHh8d7e3uqaecXS72ssjZa6RpRLy4upqen3WZSJVmvpoRikkRMsLZrlGqzLDs6nozHY7Dc22frdNvWbVt3Wlmr63x9tLdPCHEOsqR3dHwrSZJevP/8PEdEh0awENEBcQSc5KGUHMBZVMZpRIu4nRJridM2EBKNLfMmFDIJoqJYjwbDhElqTC8MLqu1EMzYRreKhD1kBNGBsYgoAomOAAJjrGu6LOkxIlVrBWA/6Usql8spIaMgYJPJZDy+JyW/uroiBBljcRxLKWfTxcX5lZSybVS/NySMKWW1rWXA28aOB8OiWC6rWiZJpy3n/PL8bDwcmjJ32rx+Y18De/ToEVJ2Pp3df/V1kUbtVPUChhEYARZti8YCSsZpEDLGXLkWnKeJCCm8+dorb929J8B5axEFYmXazx9+8smDj56dPh/u7Qc9aQrSWB3FiQyDg4ODGzdulHnRCRXKwMZxHMcikFGaUEGDLLGM9Ia9VrcXFxf379///NPPj/ZuXJ5OQcEwHj/67MmdO3fWi7PDg357epH0+9lkPF1Mz54+P3l2qpQZjvoO6XA04pxrW8zn1cXVPPyT7751/y2O7PHjp1/+jbfeevX+usg/+eTjuB+3ur26ukrTWJta8IAw6PX6QRDnZVvUVdU2Zd1GNF3lRa/XmxzeuDy/uHfvXtOa/cNjxljXdUZpSkkch4CotbbWCJHcvHmzLEs/9dU7Y48r7EwwuVac3jlguDZV0if6L/mb/7Rz2kbGL5T4/B4FB0BKPBMOGEGOlltN81WTpiSOw0AEWqv1cpVTJIQY3XpT7pw1VjuLiOisaWqVpj3/yVkvqeva2jZOWdfGWiMQW+SKEEIp153xvodwYoijlICgl+36z97/y1Kad27f+sqX3vjTf/uvKXbr5dPBiPwP/8M/evcHh3/2Z//+ydOyl4XWYlW2R3u3zq+uiKNNV8Vx1LRahMm60XvJSGt9Mp8ta4csRIqARHK0gI2uCX35dO3CMntt2uoXPTr92bjqPxltuJfAG7+5n9HDeLEvIL4mQoA5y/Ncd2pFWZRlSRDKtm06ZYAYwQUhQDXfYlT2i7tAX04FAEoJIQ7AOofGVYwKgmispZQJyZiXc2XgrYQQApB4cTYAaky7PSee0sGvw12UUgKMALN0IztNKGeUCSaEEFJwgmC1QUYoIUYrAlYIEQYBIQTRWKPAEs6oZ2tNJvsXFxdZlpVlT0opA+FZ2F5WyNfvCEFrteSZ4LStKt/GtV6tEFF1XdNUQHE0Gmmj8uUK0frmZ0Ts9zPOAyQuTqPRXv/8/Hww6SVJ5GN05wyllDAbCrFaLxCsiGVR5dNi1c96PI2FEKu2WS1XQRCAdSGL9gd79byM++Mo2nMAjKSXZ88//PijNE0Xs3kUBa+/+sr+aCQ4lYIAZ6O9STYea2zqthEBX6zmXWuMcVna7zq9XBTKGiCEhZIKRoWpbXW5slxGRbfknJOI5tY4Z4Mkxjj8aL0Ca8DYYDS+M97rZUm+XJ2cnFxd5YwxQlhVYFWZOEq5DJbT4tnJMwf4D/7R7w9jqeEkiMKL6UWnle3KIOtLGShnr5bTvK5W6ysZYlHkcZLxQBrTGtOhjBkjhiDIvUI5VThmqMEQIQAuQkGtWnPWcQ6cgHGAjsY85iJUtDC60x3s7YPuqqZsj8dBVVnbNvnc8YAN+gcORV4TJsJl3TYGIUgWnW3zgoUxzQZ53UDX8e9///uXl5eUS22NFGEQRVyGxpg33nij65pBLxv0+saqrqnByTAMW2NX69yT9dI0bZqmabrhENI0CUQQx6HW+urq6vLysmmartOD4YAQgmC11oi2sbbrOmxtkAQAVDBBYhKGYZYladKLo9Rq7LrOo9zGKqV0WZdN02htqqpCwrKsH8RJFEVhEJnYDkbCtyFthr47ba1CsJ3SAIhA0AdmG1sPCIZTXuUlJeRgst/Ule7UzYOjd770pTgOOaeXF+doQYTSiy1WhjnnZ8PszM119Nh3e260E3x4nue5D5Ynk4kvozDG8jz3nf1ZlnlRSK21b0xl3Ccizpc+vVmYTqeU0slk3yitmvaqmkkhhsOxVUp65jMCse5gvNdULSGUOSDWEUAOhDEmCLXaGKXTNNVK6a6L4rjf7x8cHw2HY8Go7erZcvX82elyuWSMjUaTyd7eaDB89vgsSZLRcJIkGWOsrWplDaW867q8zIuiMEp7Sp0xpigKpVSe50EQKKUePnyYpmmv13v+/Hma9Pb39y8vL8fj8XQ6jaJoPB4vFovvfe97s9msLKsgCAjZ+QM4PNoj6JyNu7Zw0L1y/+bF5dPZ4jRgb4Rc3jq+MZvNCMgbB4fG2aw/tNYVZb2aL6xZKI2IVBIWCV6uy7ZtiUPfWmKMQmdG48H08txP2dgp3yGi/yLWWq/DsRMO8fnZdS9yHW3220s+xl1Tu/tZwPyl4sVLoPf1qgSlGxB7B074cCTP865r8pwBcYiOcxonHqONdmQ3usGzEQD83FvnlcQI8RUW5xylgMwi+klCG7QcAHzYRBkAgFKuKKrz88vHjx+/ffumQ7xx48bJybOr2eJ73/ver/zKr/zTf/pPF4u8qt9VnUOANIvPr04B2NHB/rKwANB1XdU2uB3NtUFNtp7bu7ntmf2Ct/4vrPL8r7dtr8XmT9/6qDX4NIsyPxmBIvjQEPkvaIch17iou9+EEN/mtVtjQgghA8aEX1R+d4AOKCGECMF/0XG+CBTopurkn2d8q9cuBOe7QaObt1z/jv4B2c6U8l2UVVVZa+VmE96Le0m6HY4CW5DVaEcRvQ5VVVWeJsk4IwQRHKIFoIyD79H17e7+jgvD8Gtf+1oQBGEod1IQ/l/dum2axpsXv2KVUs7YsiwFYz7dIg4ppR4sKdYrP/dVtXXbtpKL8XCUxklTFUbp9XodB5wlkTGm6NqL89PBXj8OI0Hlcp23besIlWEoZXhyfsY4T3pZL4lkEBgwsBVG83c3IeD9CzpfmGbOWY0Ouq6oq0Byznm/319kKWNMMknDKB6ODvYOB4NR26nLP/xDKngYJzIM4riHxFVVN5vN9o5jzr1whS7yysaotVVKUcJhdzuQnYNA1bQMBfej4iw4B9razmnJJWFGcMcFlcCd5dYway0QoASEAMFFKJNQSikhbdhPPjwXAiaTOIlxVeadwoilUuq6ccY4ra3WinNRN2XdtIQAF4LOZlfj/T1rnQIYTcaEcQCXpnGvl46Hg1AGV9OLoigoOEpp3SjOZRQliKSua6WMjCVjbDwed02X53nbXl2dXxRFRSlFdHVVbfR/GOWcRxAyQpTRxBKKwJjwS1zKEJEoZVRrfEt917WIyATt9QbD4bioqk6fN03Xdrpp27KqCRUGQQSht0fMBtR0WmvUzDnjLHHOWOcQCXgMhzHKADsdRaFSJQekMatWhdX6q2+++du//puTvdFsNnv46WeRDAMeWEDJg8ZtWp+3NxghhFBCfXCAzlFKhWAEiJ8JKcPQQ8F+vErXad/Q0bbKGOPvJQ8jbxFL5JwiWICN1tuuJh2G4WAwAIfFap3neRxFQRAU6zKknBBKjWuLan80FlScX05LWhHjnNUUgAciEAFhfqg8t9RYax2AsUgoVc46sHXTPPj88w9+8uPPnzxeLtcOECzUdWuMSaJ40O8zKvI8f/z4MSEk7Q+MMXm+ms/nbdsKxpFz3an1cuVbOXq9XlmWn3/+6Pbt237CLCJGUXR2djYcjrxYwu3btx8/fjybLZyDfr8fBAEimk4RikIScLqo1oR0eX4VR69/5avvLGZPq/zyu3/yb7XWWb9XFEUcx4Kxy7PLTKafP3hIqOw6XdWdFGGSDdqqWlxcySCVlKiu0VoLTtumKdf53t6eJ/ft/JZ1DgB8wtR1nYf0EXE3R4dt50hd9+W4Ha6zcwy7IOALRdyf1+9w3cTDF+MAuJbpMkY9gk2uSUd7HNtaa612aJyzQjBCkVI6GQ92wAZcE+sNw8hbWw+B+Eb2rusYI4iADAAowIvRAwDgVzKl4BkOy+Xy+fPTn3z04SuvvPJ7/+Af/us/+len5yf/5k/+tNcfEi5++3d+Z//g+Ed//ePzyx8LwftZImU4W5wj2bSVIgFKqZDMd6siWiSwjYFwt+vdefjZs/e3tSHiizAGUWvt0PgZEIwj54xS6tA3CjopNqQW8gUixQudjJc2TjmjL5glfvNVAKAbxiIBSoD66gBa84sO0u+Lkhe9OZRSyrYjKjZWZPNNri/aLwQKhBhjKaVCiLZt5/O5bxTyS92LL1G66aMJgmBd59bapukopV2rHSHW2vV6rY1BcIwR5iiAM1Y7ZwjBIAzCULut9jmlXEqeJr0kzoRknpa4G4dhjAFHnIVGdVprQMq54FwCxbZVBBiiUcqgscY4wYM06a3OzlqlGsSqqtq6TKM4jWLByPH+XigZozBIkvGoR9Bp1UZBqLqWBQEAMELjOOZBSJnQxsGWNOrDAmc3MBvj4ACopYx/Ybl6qaEN9mWt0ppzlvSy5K03rcG2bqrL2bos7CqvLFVKvfalry2Xy+cnF1ez6Xw1v3HjRhb1wsNo1ZxRISnlXpI/iqKiKIzdEDbRe85tnYhySp1DYzVY5wxYoCApCylLCDOEWofaWaSUAFAAQGuRgp/V4nv7QLowJoRhksJRAOO9MIi6vFlTdA47BJ1l+9ai6owQPAxiwdl6xSjh/Du/9RsPHz6M46hT2hhnrVaqI4R4cSTBqJJBnud5nqO1AK6ozd07dwghJycnVbXOssznLgcHB6fPT6bTaV03zrnJZOR18U5nM78cBQjOGZW+68MQwtAR1IjEaWetRd3Z5XxtOyQELaCxlnPa7w9v3765f3R4dXV1cXV5eTWbzxfrvCJiZYChI3VnCSFAHAIi5cCQIIClnHDrNJoNR9I5QMIoUHAdY8GgPwGj0UIaJ4eTyTtvvd1P0ttHNwTjSZJFYUIYs51mjAPY3c25CeIJo8AQCTp0DgEIpZQg+Nlyfk5xFEVa67OzMy8VHsdxr9fzjqqpa0T05UNERGcoBWsMInXOaN1FoeScj8djrfXV5Ywztr9/OBpNmG+EM05QprUGZlfzBePicLKHFrI4K+p6nZedVsQ4ypExwiirqioKw0AGURRraywQIFSGoba2NbbqlFdXFUxIKdHYLE58hX5dFUVRIpIgDHu9AWOsUc16tdKd8omoV9L0rhcRV6tVnufz+TxJkslkYg0WReFHM3gu2HA4/OlPf+oAKWeEEA8aOWOEYIwKTez+3jAOxTqfzq+e54vJt775Fc75w89PhkmW5/mjTz6TIojS7PLqanG+ePa8vnt/DMjy6Wo4HO/dGgyTTFdNo5BHom1bcMAYb8piNr2cjIeCU84I2muDdynlQlBK27ZVSu0sqb9Fr1/0lwz0S+5/5xheMuL4RWH/n/uZ/vnrYYdfYrsnfYqJm8KEJcSnEs45QxkQQtokvN7MuYEUEH2WSbZdc7AtlFBKKCXOOUK9297k94hICCPAGKNSOmuNUma1yn/43vv90fjg+MaXv/p3yqb59LPz/88f/Ksf/+Tjt99++5133mk6+4Mf/vj8YioFBCGv6lWUhJxJxph2Vqm2KNbWotYdokWgG+oiet/28gn5Ww8RdtsO2PBUa2O9IgIXkkRRCOAQvH4Xvd7hef2B73GllBHix+jwTaDAJaOCckavSXc754QQuF0G1z7kFwpWXwtgv7AOyRc/ZLftame7t+8ebCPLsCzL6XTqnPOmiTIC4DtyKXg6BUCv16uqancAZovMMc4RHIBD4hA372KMaNP5GMZtoRRfiXdeJMdZIM439xhjuk7Vi03IDgCciyAICVDGaZKkkQwaWhNCOqOMzwERKIBumrKurbVWmyyNtWpXi2UaBq2BMBCMAqdUctlL4iQKnp+fNKZ2QKIo6o0SIKysm7wt++MRYZQy5sFgAOfjJG0VEuCcMy4F50CZEEJKDmhDwTljxDnOGFKCAFyIuH/UdbqdzefN8wdPn1gNIQuapjnaP6zrWlm3XC6100oZBNK11jlalV0UZmnac4B5XtZ1G4QhIcQ5IM55xolPYCil1KJpGgSDpgTTcuJizozgBATlUqPtug7RcGooEYCcUe4ArEGHDl1njDIGraXHx6QzTAbOQpEkOnDEYVtVa3QxZ4JwFMz1Ehly5jpdVx3/zd/8VS8JKSRljBmruk5nWR8RfabFCPVDelTbdl2nlNbKCCEEl0lMfJTgQ9HVauWxaEYI41RI7tDujcY74XGk6Iyx2hhlgiBwBrtGeRYrIiJa55wQIec8DIMojcbj4dHR4d7B8Wg0ipPsxu278+Xyk08+e/joibGoHaGUKtQ7iRqkBBkDQEIJAwArHBGUGPAhP6WEQdznQGjWTwXBkNK7X/vqV95+69bR0Xw+7/f7q1WuOuOAcyoJpzJO3arx2RrdtCQxHygAUN1pax0hzN+APir3QJxP5tbrtceRtNaTyb7v87TmsigKuoWJtdVcSNVpBObfmyYBISRJkjwvL8/OR4NxdONGFCWBkJeXU08sWjYtISTPc87F4VF24/Ao7Q/Ozi8ePXk2W8yV1g02TPnOIAcBdQ5kEHEZpv0BDYRj5Hw2W5Y5kyLOekXTMsaDIKSU3bx5s23bq6vZbLZYLdedsVmWLecrLgUhqLW2YCkjVhtOiZfhyrLMi+RnWVaWZZ7nd+/effb0ZL1eDwaD1WrNOb9//36e54vFwl9lba3VBgBCyQPBGaFKt02FV+drZwD1071h7+/93u86Z/rxMIqiH/31++PeEB1xBvb7+xdX0y+/deuXfunrq3Xxl+t3ddOiMgxIwPgsv0rTlDgjGWWMlk1ZVaVzfugXs8zufLa34ADgeyn9pC5/lXfKu9dNtn+865SDa0HDzkNfN80vvealjVyrR/ysiX/J3HvH78s0UkrGKGMkzeI0TZfLpbWWMbarQWyZ9pu4R0qJSDxYRQhB4mUBgRAEgoDUH90Wk3CI3Dp0Dpu6Wy7Wkrrvff/d4ah/9+7dL1XlxdXsk88eUS467VZ59eTJk6Pjcb4uj46O8+UKHXgmGiIapVerpb8pyEYQ0++CAXnBD/j/Q0Th2jF4IQHnk4C2bauKIDpKgVDLOReC73og/EKAa1eTUurjNELotZ+fsyo2i4FuEAhAAnSrCPsLtl2ry0bd9mc+0C+BDQBGt82TZANibVcX7uAoSmld14vFwjv+MAy1Ubvl7WtYbdu6plgsFohbIS9KvUXiQmij2rbrdEsIcsEYI845pYyzAEC2e2Ge20027EsE2HRgWlt6/Wwf1/pjoJQqpSghXvSJAnGel6u1J+c6azljaRS1nRKMB1HYNS1Dt5hOJaNyPARn6qIwjI6G/VAGsQw6oznlIoodZWXTVl3rANBZSsA6Z4xB5xgjURAC46prDTrGWOhsEATA/Ak0geR2c/Og09o5wykjhDSFZZQ6zoAL5VArhYIY656ePGeMWUBHIIoS68CT2eNx0irV742Ojo7qtn7+/Dk64iNL5xxYuyuVKqWIs11XUoIcLKIFjZ0xla0Bl8cHnPNACIqITikLSCgS4oAIII5ScM4BtZQRGQDlRGmMeSBE0ChiTdBorFvDCSADwUlnjdGts50UMgxYWzveH6S9fnw5u+QiCMPYWs04CQLh3Ka2RDYXKaEA1tp+f7hcrimlg8EgTdOiKFartfcTaN14PDZWNWVlrfayRU3dbtp/GRVCSM5JmnrY2Tls280ISovOI/NZgp5SMN7fOzy4ORqOlDJPn532+v0wiQ+PblWdW1dtVTcOCGGMR9eo5gjEMuqIz56Ilz7hHK6VlrOAl/m6MchDMdnf+9ovff31V+6U63Ucx5ez6dn5edG0GtEhRRGAiBFxe9P5W4v6CJ5sCWh0o3mCPhkliG3b7ojo3v1Mp9Ou04eHhwcHB9agz8W11kEQdFZvW/LQoW272rnEl9IJIcY4ROxaDUhHg3GatI3VYRiui1wI0SrTdp0yOk57URyLMBCB5FJYQgGIV0wLeIjWzud5FAXL9Xq+Wj58sshXi4cPP18sFgTBIXadkhI4Zc7gcDhcLBbFOi/zQinlLKKxhSoIITLgCEAZEKRoLJGbhDWKIn8Rh8Ph5eXler1+9dVXhbhM05QQMhgMiqJ44403PvnkE6UUIR4ZIwSoECIOQnC2axvBCCOslyRRyOuy+Oinn7795pcuz08Xq3KxWDgk/7v/7T+hLPiX//Jfn5xdUuSRTOKopxUGQlZNfXF2ppRaLpdgNEEbCIaIzlqwhjMiOHXGcMosY9u+NcGl8ObYqyYEQbAjLvg06+ca6N3zO5zgJTe/gxD8y3Yp/s9GDOTatnMwzjkC5KXPJIQEQaDUJtFhjO6K/kEQ7Bo0rLVdpxBRSpkkmTfNQgjnYNfd03UNokVwiOAZcoQQAOJj3I1BRwpAldJ5XnDhNOJ0teiPJ4c373z1a9/47LNPHj85UcodHh7euXNrOOzPLq++853vPHz48Ic//OF00fgGqCRJvCtSqrPWbmMFj2EAAm77CV8GbH6Ra/yb2XbXDr9YgAAApVRdoyeIUOaCIOCcWeuuX9yfvXAv/bbWIiGUvACifBuU39EuUPDjJ51zFH7+titbAL7MjdhGCZQxugsUtoHjS7Uw3FkqAKjruizLXQi74/N6OUjfkXG1nPrXSykFD3gQ+Fbw84sL78yss2EooyhEtG1bu92IS19QowwRldJxHHO+YUV0XeNDhPV6HdPYx5oAlHOJiNooyQUlXGu7WuV1XddFSSnFA8K5LNerKIqO9o+qtqvr2ll0yhwdHlR5IRgd9rLJYGR166xGq41qKZBekiLjtdLLVV61neU8iMJ1XvJACsqMMVopgs60XUWpI1xbQyntulCGwUY2g0G/lyJa4hDBMUIlp4GQnkIRZv3xaLiYDPtZVrgyDmMInFEGAJqm1tZoq2BNgkBkgz6SVuuGEBZFiXFICLXWtW3X6wW+Cww38h4a0YKjHAMpZBymgQRnkrqAfJXXldGTGCnnIuSUWd4ygmhBa62abrO0gBDCuCBCCECBjibJHmXZYm2K9bpclGVlHcqkF0oZmM6Udd2IIkn6ggGnjnddc3Cw//jpU+s0IVjXNTC2Wi8CGfmj9FSvtm3RAuc8TkdtUznngiCilFdVtVosmia6ceNGnCSDYY9SWhe5v3OqqrqcLnYKowQhiMI4iElI6q51zinVtK0vYaJXHUBTZlkWBnE/7fmezK5TTduyICyaNkzSrN8bDEfr8mRVVlmWEQk7HWjnAAkCEoKUWEo5gmMUN5Uev7XWsShpyzW3tD8Zh2mSV2XT1KPR4JNPPnnw+Mm6qrQjCMQgydtuhy4S52NJz29yjL0Q4kVEa4yvWuGWGE+28sB+inFRbCTNkzjzLL/VaoWIhCDn1Gu/W6vb1vomQD/ZXQgRhrHW2jkMZJimaVHMh/uTk8tz4EwE0ild1rUBssiL2XyhjJZRyENiLRrnAIATCgh13c7nywcPHvzovQ8+/fTj07Pn4GwYhrdu3AgJ4ZyHYZxlfSnlarVaLBY+spFcIiNSyjYvvSNB9CNvtpP3tiZSSqm13s2t8a3nPkQAIIPBgDF2fn7edV3TNaFgAMQas3HJxrVVt3/r+Mtvv/Xrv/btxfzqD/6X//nhg/l7P/jJdHp5vri4uoJv/8prBwdHnTKD/ujB58+u5upy/skqrxCgKusgCtu2JYTcuXePggvDUFtcr9fz+QLQBZwRh9ZaKQVQ0rUaAKIk9kStrml3vBDPRfVY386Y/lwf/4u2n33ZdWBg9wK/Nug1mttuR9ZauiXe7iw+IcQb9CAIsl4CgE1TefrY/t7In0a2jYGklH6ET57n/s5yzvnyCmPMOeOcL2QQgBe+cFc64ZwTgs5xIE5re3F1KcMgSuInT5996UtfeuOttwlnf/Hd7+X5g7/zd/7OP/7H/+jZsyff/9739vZH48ng1u3jv/rrzz777LPVaiUDHoYhOtI0jb8DtyfAq6ACguexspdCq7/dWGGH3Ow2QgiiVzk0QAwA+kABESklAsT1KIFQ6sMAdEgoRUfAz4zxP+AxhZe3DVq5vQSI26oypeB+fsD6gtoCXwhYd5/whfAUd9cXv3i2ffPF5vXe7PgboW3btmsJIW3bUgpRFA2Hw8lkcjY9r6qq67QxhhJOhfAcbSGEtBIxlqHY2xunWVLX5Wx2NZstAAARvDNxzpXFZrpKmqaIrqqq+XzedZ0HvWQmPTNaKdW2rbVWd4rGSVEUnNLlcllVlW67MAw9/2a9WhAcyQNBY8oJreu6tCYJe7ZtCdpAyiSNu8pVebOcL+bTqXFuMBohw+Vy2RqXDQZUBkXdICWUM89RbdsWjO6AICKVodIaKEmMNsYQBga99hQ4qwEglCIOI0qpRWeN25dynCWSycswsqrJF9OG5wQpInpWe5YlTLJWVcbxMA6sNut18fTp8/W6qJtG6ZbAZrKo0ZYKYIwFgdj0uHIWs0EchGkWp3HEmG7K3moVFEUgwxqhbtoSbSsZkUHgjDWdgi0n2yECGk4IIRSB9Xs9rWA6XZxfNldX7XzetS2EQUJoS1OOFozSqtWhNEZ3bVPx/f39b3wjXObr1ToPZEKXKwes63TTNLBV+/KSgoKyKIq4DEbDoTGmKNaLxWKxWHVN42OCpi7brg6CgAF68lRRFMfHx0qppmn8UmiqllA/ITdptUJEo6xWFv3kX2XbpsiyrNcbxFFqtO26LoiitN8TYXB+caWsi5M0jJNWGdeqOE6VbQEA3W7pE0IZAZBCeKfuHDoLm/Zn54r1+vjoEJyRnA33JpSzum1G49E6X330yccPHj9ZrJYKLQDRiHXdxJQSskHtrLWIzg/l8W1a/ma2Fv0gDQCo25ZstChcVVUAm4yBMXF2doaIb77x9ng87rru6upqO9WN7lBBX6fwyfF6vVZN2/U7ikAp01orZZRScZIwzpEAF8ISUrdN0bTK2LKqDbowiiyQpu66rjXGxCKM4zgMQwC4nM4fPnn84NHDi4szwcje3t7R0REAKKND55xzqu2eP3u2Xq/n07nYjMpkhBClVBRFCL6cCZxv5F98CNh1nRea9Ig9pXQ6nQoeeO87m82/9KUvTafTtm2rqgoikFHIgDjhoiga9oYMSR0nUZg+fvzs1Vdeq8pyf+9ob7w/6O9//smj+QIOD+mNGzf+53/+z9vWfPOXf3VyePOP/+jf5FXVdrooCm1dSEhelkmS3Lp94/jgEBHLsnz69Pl8PncOCIBSCoiTMiKMGe388ouiyFrbNZvBWrCtOBBC2Hac9I7E+nPxgN2D634OfoYsdv3Jl+KGXRywy2IR0eHLik+U0jRNg0Ds70+Ojg84Z8vlPC9WzrmqXPsIwJdUpQj6/f5kMinLGrdgiQdpfZfNtpEPEIDAi3y16zovM+rZdb7f01qrjTk7v2RcChEMh+Ou615/7Y31Kn/44LN/8yd/cni43+uld+/eCQLRNE2SBv/kn/xv/uAP/uA//IfvrddrrT1attF1APBf0wFcI238bbY4/JztJXe7ceGbHgdHqO9EoGD9ncu8EPh/ettdXEKIlJIA2whFb9msvtEAt5ASIPEENEqp+wWBgr/oiEi32JJfKsxtOHf+gDfkFQpetuELZx7Alx7YduL2juzii8Ubd2VMUax9KDAcDkej0XK5dK50znWq0VXVdV0YhpSxTrWIVoZ9L4TTdY3vPMJt669zDlEppZqGX11d7e3thWG4Wq1Wq5WvYI5Go2E43D2Z67Jpmq5tVU/XdT3qD4IgSpLMKm2tVZ25upzptitguVoNhRCccsZY0zR1WaVpqtpacpFGsSBgmqYs1qvVUqMDSg2Q1XLV3z+4f/9+rc3VJ58A3eA93gIzdJwLBqTqOq01oVRLIVxA6Yu73hdHhv1er9fjlPihSGA0dZZSZ3RX5uvZ1aVkkjERhiEQRwihnAgmmraKkySOh/PFrMirJ+pZ1z1Qprt161bSS4BuYinCqZeP84VFKniEPclFEkVpHMcRc71gNBJtE8+mnzrbrMpSt0WWkEAMGKWEkLSXoHXWaauNMWAtWtQEmSX26dOTD39qLi+BABgDiJSiSJDGYRKKJAqSfn+YJj0KrC4b3poqyoJvfvPr77/3k5PTaSQzzsKOKp/ICsYpJ2Es1oVqVJ2IZBCNRsO4qovTk/OyLA73B4yl8/m0a6YAYFSgtabArHWU8jt37lAmmqYJBLcmsUCcBaWMMto5qlvbNgYRpWDOGUC0HOdFldYL5ZogC+J+6gBbrXUDXVlHWVbWleuaRqvBqH8xvWpt0646D9/5FqMN29MBInDGGAsRiLbaOOPz/yAqTblKgmDcH9SNXddE0vDi4fT9996bzbpVjkrJUAgGnFGQYJYzPD7eXywWlAGlvGqrNI3btlXWhqFEhKZSujOSCUopWvBju33ZO45jQkjTNGEY1nU7HA6bpvnss8+Oj48Hg8Frr732/PlzLkWZNwGPVeuyaOScQxcCSdcrl8T76MpWs/5oHEn54WePb926dXhw8+zk8vbtu6vVqnFNEslVnlNKBSXgGtt0hlBroOsUdySWkQHM87xr1f7+PiEkIAl2bK9/I5QyixPTstlseevotbLMF7Pi1VdfffcHP1oul4EfsClZ16pu1aZpSDixxoZh2ClV1RoAAhkdH95fL/Pp2cwaF9BQVd3BaL+u6lY2N+4fP378WFIWcv5rv/zLf/RHf3T27Gx/f9S6vGmqgIterxfHqXZKIYokyFUFDv/Zv/gXfjq21vqDB89VZ6IkC6Lxuz989Oqrr/7ab3zr69/65tnpxZ/+2V+wRj1+epElPI5TySNwrFzWi6tiuHdzvV4vFqtHp5dlhyyMi9a0xjnHm1Z3nXYOJpPx/v5h27YX84uyLPv9vnfVSZJ4RMTbNU9Hd9ttF0PsUNmdIwcAxpiX1vZgr/8v59yjLL7rdQckwpZchtcaCP0yRkcREZBSIJwyzjgAOGvQmvFo75V7t8eTIaVwfDxaLufPnj1zmkkutdaqLSmlcRwTUHW5Xi1WaFUcSnSEOJKESW0b2zlCEkIsIRbRIRAAQOIQMMkibyJbpQUKxhgSdA7jSFpdxyE7PholCXz44Q/u3bn11ls3xmP2wfvv/7/+3/+3O3fuvPHqGx9++OHHH3/KGPvv/tHB7/7ur//gB3+udGcUZsnodHo+2tunQJqm7TQyJgkAGgRKhRBG1756Yq3xPZxaGw9Q7fzrzgv6M/dSZPafRSB+FtR5Kf/+wn8d21T9wSJYs2nfRQDKGCHAmloBABcUENFZTRshhBD+ogMAUgqeokQoUgaEIuOEMmQMKAWnDWOEUUoo5YxLP0OYcYIAiJsGMUIQAa2zzgAFQnfFT+LQOus839CitVrtQklP1YpYb+ebd4gU49T7bEK+sGgJIcY5ybzKrXn/Jz9utQqCgEmaV0VZlogYBNJPtovT3v3Xwq++88290Y3T09OiKNq2blVHCEgpmWSCE+u0atqPP/rIZ4aIqG234+I553SnCSGGWUbpxckVIoJ1TmMQyePxjddee63N1bNnz9pSSxIW+bqpas55sSp0p2oWhJKXVc0pG4/HWZIyTg5u3aOUNxqtA0StO9zrH1hjmKVgqGpUXVa6q8omny6vLs5Ps707/+4vfhDF6WtvvRMFo9l5RXlwlN2Zr5b5aq10IwRLkqip87KrGacOYsI3ZC/VtD7r40KQ1i8JGtK4Hw19HOCc62aXn3z0aVmW63Wxf7RPpdBaK6UZ55RyxpgWiAhh0ncAF1cLq3UaZcDoaHiQ9jJCiLLGGZRBxIRRWteNGvNAhJFxGsAGTieCpwJC1hGHgYAsPZByb29yXJWzn7z/7mJ59vzp4o3X97Mkms+mx0ljdEeQBzwImVNKgQVGYTaftbmLGBwMvaMcAEkApLE4O59GYUqJ7MXj+/feGqZL7DJOHE6nl/OreRiGk/GwyBvnmiQOg4h1nUHotG6UKR1UnV7rPN8fDMpat109mkS3707298ZVVQFtHn/+sNcbJHHa7w8psOVyzblANNPpoq5bQshwOI7TTGu7XhWuQT9wZTKZWKfruizL0rfDHh0N0jT1tas4TWUYUM4IY8zZfLV2gJxzzohzJpKCOOvvB9zcY14KjQLxWS8DAIcbYhqlQCkTcSoYQ4eLvDCfP768uEJjy3U+m15prS3hQZI659wm2woCWZTFusiXANDrp2EgqjJfr4t+mnVoGeGcIQhitfJ3IiNiuztPsGc+YI+iqK5rn4K3bVvXdRzHb7zxxsnFSVkXXdPGcRxnKWOs38uGo1GUJmkvq9umbprVallynue5QbeXBq3qIhIhYqdVW7RlXae9THkxAe5HWgC33FlAdEx4Oi4xRvX7/aJcNm3lD6zfzw4P943tuqaZzq4Gw/46X5Fte7eUkm57phExFpIQEgQBFwJg03vt5Qd85rFJaLZTQLXWfoahl4UOwzAMOfXtYbCBYTyvxTlkjHFC3XawBgA4iw4oMFpXRRyFh4eH6/Xqgw/eb1TnnOu0QgJZFsRx3Glrm9Zatyrrh4+fffr08Xq9RsS29cUgkqbpYDBgnGhlEa0QUgjhnFGq1br7RYAtuVYOuO6rrncz7giDu+++e9Inan7ar0/LvKV2W/LbL3JUdDuteFcLoJR6E6OUevz48aeffaxUSxkq1eZ5LjYYAPMActNUSrWMCUq4tZYQvmuVo9uO+d3BwzU4pCgKuMbKxO084lYtjo8OgiAIgmAwGBwdHYVhOBwOj44O2qZ5/71Pb926tVqtPvjgg+VyvV63g9Het7/97d/6rd/69LPPF4tVlmWDQbNYLNpGh2EYRYG1SCkPA6GMrqtSCOIDFLIZQP8zFL+/VVmF3WY30s4Am9IA9cdMNum48MO6AMC5TfGRkk39yK8gv3I4+zlRy8/GOi/c+Rdnju+2Hc9x968dKnY9DvBBqkO7RRRe7jShlPqqlnU6DMM4jruuy/OVfxIAfYVLSrlYzj748XucBc5ZQkiWxZwTrC2lNEmi+WrZtq21WgjBBAsC6ccaRTQwxvj24yiKvKZZWZaq63zfuG67xWKxXq8nk4lfAFVVzWYzQES0xM/SaxvJhV+9YRhmSToej5MoJnTTsqHbLs9zZzYKEMjpcrno9ZLBoBcEwXI1Pzs7L4o8jJL5fN6qzlFxdTWrGhNn+Y1bd1955ZXo6vLiik1nF13XodNKGWM0OEoYEgBwaJR2xlK2kU6nQLquA9fnhyyUAQVCgKdJ9PnDz8/OLhaLRadUp4xHLhkXdV0DUOvHuACjlHpuSRIPva2QQQQAddeCI5QQY4xXB/elfy6YRcMYaU0dRYJ6ZUzBmASt26IsBBe3bt632n360QefzYr10thONxX54feW9165e3AwtqY2qsnSMJTcaLdanH/tl45+7VcneeVm03Y2r6ezcrmYC5bFUVIUK3SsWAY//WC9WharZclXi9X56dnJyVnTdIA0S3ma9LIs+9EHPyIUKEcZsDCit25nDkNtzXr1FAs0VhFCHBkUVbdcLlf52WQ/6WUJIeyNN+9ORgcnJ6eUcMbEydlZ0zboCKW0VcpaUJ1hjNy6dcs515murpFzniRJkkSUs7OLsyAI+v3+cDjsZxlhzKIDSmqtl/PZcDIWnCVBYNpWEFJXJVLqeQmUCEqQ+MZFgoJzdMRaCw4JOEY3vB5wwhGGxDVNW1TTq+lctV1TFcaYUAYikMwy3bZKa0IIFywIOVg9GvQYI3fu3hoPR+fnp+v1WrW6qiqlakYDxolSxljDmDDdJpD3LciMCV97C4Jol4PWdW2MGY1Ge3t7MguB0ZOTk7KulOmiMMxISji5ml1WbVPUJQUgnHjKnp1eHPTv19XKWitkeHycdlqfn587QBEGKWUiMG1nQRlgASWcc0mFL+vEWRbfvXfDKAVgg1AEIWccrGu1brRpEfTNW4dFsfZ346ZcInjXKkS01jZN4yfcR3HMmCiKYj6fn56eBkI6Z6xF32/tB8cxRlarhZeUvXHj6OLizDkThtJaHQQBssPKsJwAAQAASURBVA2Z3BhjjCWe2yECNNZa65yvYwMSIIzGQjbV+qtf/XthGD56/PSf/bP/KQwjrXXddkGUAJNtU0kKMkxEbMvOUKbbVgtB0zSklBtjpJRZljLGFCjKIAwl57SuyzxftW1NKSfXQOAdeLDzl94S7azq9URtFyjAluS4ixJ2iIKncJMNjPyipvAFZ7BzP7Bt4CGbQQAARgjBOZtMJqPxQKlmvijatvbjyqSURrWUUkKQUobojNFaK8Q6DCMAynw+6geXMOLcF3a6ixJgy6rzCIpfsbhpUeNHR0dt256dnX388ceLxaIq1vv7+6PR4N69e6cn50VR3Ll559vf/vbZ2cUf//F/fPDg4f37r7322htVraqqKYqi1+shZYzWzvmxRkgIWmqBkDCKKPEX3XnGnA86r89I3EVp29//bYZF/dduzjlrX0yjdrhxw7AZQ298LEUpWmuNwSSJCCEEKOBmITkHiE7wF+d/FyU45zjnAGS3KHaST9fX2PWz4R359ZjP/+kbfX0Gv6VWuGtdD1/Q/ED0kwkZY3Q5zZum6bpNjXg+n3lAtG5KSmkaxE3TfP7550q13lxHUWSMarpWCME5EOKvmwJAwgSlxM/W8hx2D7Iqpfz9pZSKo2hjZ2LmxQM8kSuJMwAoisIaE4YyCkIA8OIucRxnaSylnIzGSZKotqurqlznvrDljDbGMIJV16i2cWg9J6DpuqIonp2cLRazvb29xgoHvGv1Yr1qjbuaL2plRBSWZb5erubTWdc1hDoKjhAEDtptECN/CTgTYRh4fSSfFK1W+YMHD58+fRyG4b1796ApT07Pu65Le1maCaUUAnRGEy420SL6qwAeqDDWCiHCMATG27btlHLOUc6UMUEoBPMBKBNCSMJFwG/2DmUYcsGs1Voh54KJKATgDOJkcOeWMDV99mj67HGumqskiVDff/SxfPjh0ugyTthkIhCa1Wp2//79piTLxbKqO+t41guTXnLj5vjhw9mwl1Z5ThkYVc2uzqMw3ZvEnBq31x/aTl1cXDnn0l4oJbTdPAi6IBDArbGlcRhIRtGBa0T0AlxVdlYvbNuoOIWAM0q6fL3YP+x/+1vfuLi409Td8+en8/lVFCVCyqJYz5YLSjjnAWyq/oQH8uDgIAhutm07m83my5mUwWQ4Pjo8Hg2GGzgXkQlOwaFWo16W9Xqqqjh1QSSqqrJEIPqOZOs7rwhQAGKMBiCeuMM2Wh+EENDOh6rOACAw66xyrnXEy9Q5R6x1nXXaOs45BRYHctDr33/1Xpakr7/6ys2bN0+fP7fW/tVf/fCzzz579uyks4pSro2y1lHue6apTyXDMORc+tvDeyzGmODC1/8IIdZalvLJZJQkUVmWdVE6NE1TTafT0XiYZEmcx9ZpAGecdmCLqvzgJz89Ojq6/+prAHA1n+q6DeMo6fWrqjKh450G0ljEkLIs66dZH0jnv74Q4o037z/6/OFwlPV6vTQOtWmePX9yfnEWh1GWxbduHf/VX536Oxm25XMfMVBKCWMeC3GIQgRd11VVRSk93D/wmkU7k+21Dquq6vf7N2/ePDo6evz4cdu2vV7Pa655c2+t9fVXQjljAgAcUO2HebptdxllXEpK6VtvvVFU9QcffvTg4XkSs3QwrJUGaY02VEgRJETwIOkxzkPeWWs5l/6c+ypGVVVCcGO4lCJJQwRTFOU6XyrdRmFvZ7J3UcKucLt7Zmeyd+qN10mIsO0y9//y1nkHt14r0H4BnNg5oet+zjrrYWRENMYw5usdYrVaxUnoux6klIQ6r48rGPFUdmM0giUEGSeMMSAagPp2L22se9ER9/O3HbOdbAsi3jsyQff3951zSRLleT4YDCSnVVWdnj4f9Pu///u//9Of/vTs7OzVV1/N8zxN4cFnDzn7s1//9V//yle+kqbZv/93f951pSN0OBzO5wvvUAFoXdeE8izL7LUePNgmytbaXSPAbtsd/8+GCP+Jr/bfMITYnUN/Ff19TS0SQpxFSpgQApE5h9Zqa0P/hQCAUo6OUMbItf7J7eF9AVHYxUP4M00xLwVDdjuleuf1YRtA7OyzD6qcc+BwizSgX5O7JghKNrPLT0+fP336uKqqMAziOFZKcUEBwPfUtG1tjFJKBSE3tiPUMQ6UUkKdNm1ROmDUoVG67VRDGrozI0BJmqZRFAFAXdfL5dI32x8dHs7nc611P81u3769Xq8B4PT09NXbr3vL02qNaHeAmGfbEEI8V5dzPp/OVusFR9KZlhAShiE4o5SyVnPB9vb2+720M1p0XdYbjMYHTavCKJMik13bNJ3gUkh5fnr26cNHnz54QCktyrxpKs5pHMpACiF8GzwyQgWTlFLOeCBlFERCCKW6IAjQwunzkw8//PDzzz9PkmS1WE96ycXljAoepj1CSGesNgYAkiRxBIjbDBk31hBnCSGdaoIgCI3yEKBFxwXngWzbGgihlPgkjTLCGLGWR8exF16jnDlwZdWFgUiSYSB4vmop9t5845fzhf2P3/vLB08+vHNrsLhSYYC9fsRZeHJ19fFPH/b60dHR3r/6Fx84AMKACwFUGofaAgAITHqRxo5Faapr3Yt6v/zLv8wJ5/PLyziObx8eHe2Nu64rmzKvll2bC9qFESXM5XXVtpWyFACVamORyoBG0UbSrqlaGQSMpsRRZ2nXNYTYTlVXV+ePHz95770P6qYcjvq9rFdVjbQQBFFddReXV75fVkahUirLMsZImqaE0dt37+zt7SVRpJrOV7mAokNjOuWsJtZYVVtVC+fiOA4YmbeGUkJ9pdUaQjmAJUBV68eoMEp9gyvxob1k3Dmnt4vYOmytbo0CcA6QW0PRUUqllIIyRlnXtAevvvbNr/2dXpbtj0f7e3v9IIrjeJT2JWXL2XK2WFmKiEgoZYwFPPTDV7x4M+fSO862VX7MlWducs6rqlqtVoODwcHBQZYlRrW5Vf2s98q9uwcHB2+/8WY/iZM4XC+Wbdt2bc0oECCL2SqQ0cXVLIxkkqRpL3t2euKRLueNCQNf8WSCJmkQJynn3N9maZoSguPxcDgcojUe8Vuv187qfr/fqebs/MQXDq57tU1nVBRVVXV5eam0jqLEz7UKw5ALGsWB0i1lAIDOWcGY0i3nPAjFV776DiISim1XD4Y9IM6PDN3YL6QEvCAuBcKAWqDMgdVoAYECMMbWVbu3t/fs4urPv/sfPvrk4zAiR7duz5cryrl//97eXhBFs9miVY1AUedrACDE1nXt788sS6TkYSg9Uy8Igq7r6rpUqt1OB30ZToBrocNL6PcuULj+rusRwO4B3Srpko0E7AsemV8A1+0+7rjom9fg9c05N5vNCEUpWVkV1mohKWNEa80pADhE51ABOC4oI5z6OoNFY23Xmq5FrTfBnx+FtTvO3S42QPr2sH1QSwiZ7PVu3759586dXi99fvKEUuia6oc//OF0ennj+Pi3vvM777zzzkc/+eiP//iPLy/z+/dvnp2vfvjD96Iovv/aq1KEPhGP0rAsSwAUciM0JCW3CG1XW70Zd2St3Tm/bUL8C5387vz/zcAJsNVQur5toBe0/qwaY7QygAQIWuuUMpSCc0wI6hz4fJ5eE1kiX7zG/jNffKPN90K41tTw0jfdPXn9Q37u2UB8EShsgNVN465zaJQiWpPnz5+v1yvnHGMUwfb6qXPOWs0YUaqbzRqf/BjbSCnjOPRDH3yGwBgTgvuVYw1a66zdaB1qo9brtedTO+fKvAiCYG9vbzKZlGW5Wq2stb6TVim1Wq0e28dt26ZpCoh1XfrBK1prq03btgSttfbo4NCvVSHErf1DLxcrJXeOWKOjNB4O+1EUyYBba5VxYZzcvH0n7fXH43GpaNU2eZ5zGRDKm6Y5Obm4mF55tmYahUEYMcLAokVn0Ia9nhASkbStQmzaVpRFDcRVVeWcM0Y75whhX/nK17Isi+PQqBYZV9oulmtjjAUcDAa9Xo8L4ddMp1XTVm3bKq29CQVCGq0Q9QYKEhwAgigMw9A6Y4wqy1Ib5ZxhjDwWiXNuNBrduHEDGC1Wa20gkFkkg3kxi7g8OLzxy99KVBM8+nxaFGwwTMMwODzYm0xGqm2enzxdzmYXJ+bVV36Fcz5fLp89P2+0SXsDY+xsPh+lbMaV1kFDyGI2u3fv3v1bb8znc87QVesV66WTyXC11o8fP50tpjIWZTkDGouIUTCMgeCESx7Fol4pnzBtBjwSXpZl13b7470k7u/t7QtJHz3+7L33f/TjH//044+K3/7tr/X7ww0FickkyWbTFV5e+BVeFMXV1ZW1Nkmio6OjvYP9/rDnKfRN06RpwgLRdU2dr0/Pztb58oGzIpTT6bQu1oJCGEdbQTPi0PoKN6WcEGesIsAodUL4+5s5NM45QdA66zlLiM6CA3DAHN8Io2xIw2DRWe10dzA5eOPVN+7cekVSAs4Wi5w5ElHxpdffPnl68vmnj7UiGpE7BEoEDxhyHyX4zNKnlP5G9YXqtlGedOacq+vaXXVZFGZZxoFGUg6z7Gj/4ObxURoG48FwmKVE64VW2qIH3Bbx+uTk5Pzq8tadm7/1W7/5ymuvMil+9N57q3xd1zVQlmQ9yljbtnlViFD2+8dCCE/1L8uy63QQRJTS58/Pq6pSqvP/DYN4Nl0URZHGe555ANfoygDgAUkf7EspfendxxB+fi7dzuRkjHVd57nTR0dHn3zySVmWbdv6fmtEJEjRbdjclHJE6IwWQBEJEuIoxY3rdYCEyWhW1P/y//unH3/8CVCWZX2NsCoLLgRjTHB6dHQQRVGZr9raCeKA8100gIhBIOI4DoIgjj02AwBgbOtQeVX8bWvti4z/euiAXwQAPEJ+HQfGLWt9q0OwoUN7VIlz7jU6ET0iba6HFC+Zdf+Ye+HILeeMMe6dQRAEdV0jSm9SwygJQ7larQw4xggXhDPfLg+EaodW8AjA2c50nVYKAQVjnADl+DJWvwsH/Rfxd7ffCCFHR0dxHO/v7/f72fOTJ3meo9XL5TJJksVi8Qd/8Ae/+Zu/+eUvf1kpNRpNmqYZDseXl+WPfvT+x598tr+/78fGMilm04UQQiu1rgoASNNQMNE0jdHW63n7vgxKqSd4e0rH9SPcOezdM+S/gMTw3yqKINekJL3HBR8roEVESqnWllK9i3V8CUAIQoihlCJyAOp7QTdr4NrS2vh7wO0ucHvcuPtAuIY3kC+WG146Sz8ndCAvClsvnUwZSD/OqiyLfr8fhKJt27ZtfRsLISSKIsZIXddCsMGgt1zNENFPtmxbhQQODw9v375tEaqqatuWUsq49LcAY2y6OD8/P/d69kEQdKK129Eq3pj4d3kgRAjx+PFjIcRoNErieLFgzmwU0vYPDkajke4aD+QQQjjnWZbpzhilndHIKAAiag+glFUeuYAQti7ysizLopZSakODKGNBLGTcqi6vyjCK7tw5EmHg45VekgrOrdJt25rOmE5pV3DOnTNesVFKCcT5ATdScs9WnkzGd27e2t/fl1IadKPJoRcUmM6vnNZAmEFoqpJSipRo3bVKtUppaxBxkI59U1LbbkY3e1jo8vI8SSKH1jnVH2RJEiNaQnH66KTVajQavfLKK2nSM8YMh0NG5b07d0ejQwbEKBLHo9u3XkMrHzx4/q1ffrvt6uenz5Rq79y585Uvf+OTjz97//0PLi4f/O//D//0zbe/9Bd/+f333v9xnPXCKBr0Z5GgQoimreqyXK3bulInp+eXF2f8q196e7mca6fB6eVi+vzpw9PL0zCWk4MxF4QSFJwC4cYaU/sOhR4AoGNaISWE0YAz00G9WKwYjXq9flEU+apExH6//86X2d7+hDFmtA0jqQwUxVrpttfLyqKJ4zgJE8ZYWZZN083nc+OslNIZNEqXZblYMM8761SzXi2sNeenp1EclFWVRAFazSEMhPRWGB06x4AiASCEM0Kds86iAYWWU2oRiXNOmRoABKOOoDHWoaLMhZEUlAIiaGuNQmUoIYIyIcWd23dvHN7q6q5W2jQNI2Q06K91kRdVW3SShaEMwTkCxCEByhhspMo2shBIdjIS3vbthsASQtI0nU1PYiljLl65fdvdvOmMKReLx1VFlFVtLQkbZX3TNu163U/ivb1JV9sgCKxzdd1+9OlneVUChe985zt/8Zff+/zxozwvuBRhnBp0piqEEGV/0O/3gyDq9/uAnDGBaBaL/Opy5scF9fopp+z4+EZV1YJv1Amvb94S+YwziiIESJKEc962rR/0xRjx6sKUUucgCETXEedM01TOmfPz04uLM0KwLHPGWNcZD36iBa8lbwFR2a7rgG4EgMDLlwJxCDxIy7KaffYk7U+yXnJ5eVnU5w7Bh2PWGmcVcYKCjkPGGKEk8P7YO5uiKIIgqOqCceI0egvVdS2lhHNmrd0GJV8IC2AreOx9AN3KGeE1Mjn8PKdlrfU1xV3edt314hfVmq+nidfMt5c8eiERhog+1Gg7NRweHw73q6oYjnqMkcVisUOpKaWUIaGWEEKoQzRACBCLYIEAY4QS4SgaILsD2/kSHwDBC+KI8Q17QgiPQud5HkXBcDh88uRRP0t/9Vd/dTIZLReLn/z4w+fPn79+//UvfelL0+n8u9/9bhTvZRmnhOd5OR7v3b179/z8/HI67/V6XddFUTgY9AFI0zSd6oJAjIeTGzduWGufP3/uafY7AOa6n75+aQBejnX+Zjb8gmsHf7o8Pc0TGNGLGTPGGDHaMgaU+knc4Fu1r0cF17/e9UABESklfogMAKJ78TWv8xXYF4eWXT9I3AJd1uI2nsBteLHTYvLynSDlRn5Da80FFUIURaF155u0KQVKuVccEZJzwSaTfevnXoogjrI4TY6Pbx4fH+dVCcgBeRjG/eHA4wfWuEYVYRj6OTh+WkTXdbPZLJDS59OmU3VdO+d2+aHHYqUQ1mrfAyUYPTo6Oj4+LtbLrus8KaGqKoemyBeI1qGptKKchGEoJFOqpZQ6JyiF+Xz++YNHWtu9vYOm0eneMSJyKZgIgkDfv39fhIEPfRgQY0xT113dGWWttoi4KqeMMR/cCMm8GFRd14zRKIj7/X6SRGEY6k6rViVREoQyihPGWFVVUZpUVUUI6UxX1Y2X3NRat22tlLKAlFKvJ9QZ3XSdr1f6rDJM4qSXckqEIMc3Dvf2JowRQjFA7tOwOErHe5PJZO/48GgymRBHspgywtuy6jptHbTaWIuffPwoCAIpguWivDhbTib7/Wz8W9/53dli+fDh47/4y+9//uThK6+9+pu//StVVVz++6f9yS3BmVvqqnWvvHbz6NZkVU57w5Br1fZ7KeW0bet+lt69c5tQs6rWXdfUraGC8IBTxggw337dtEpK6axrm5rQKpQBYyKOU+ooIhrtTk/PnXFaKyHYwcHBw4cPsqw/Ge8NBr35srg4n7atCcM4imKjXaO0FzwOw7DfH2ZZ6kUJ67q8uLiYzaZM8L298WDY6yVpGAZ5sUr7vTSN0zRer9dRFK1bbc0OAfb4KiIgYwSAbuY80A1NARGJUZxzRplxTqumKUvfyIRSorFaNbpuwbosike9Qb+fza7mTx89/eSnH5umYYhSiLu3bxHEs4urJ8+erhartjHKoSag0SGh/dE4jmPGmFcL8WiSlNKTFTjnvWzgqzY+Pr15MFiv15fn52htP+txAMnFuD9wqguEnAyGlIJu6jP1vC7KKghv3r41W8wXi/l8Ps+r/NGjz2/fvf3tX/vVt99+e75aLlf1tkMPjHEWTVGU/f6g3+/funXLWpsmvdVqtZivoihpW9V1ajDgURQfHBxdXFwQwjwzC7eT4nzdhDGWRrHXYymrqm1VHMe7SUXX3ed11No513Xder1erVb7+/td1/X7/aKoGN00hniLYJFQ0lVNzRj3mjOOUKTOo67TZX50dDSfz1kQtwY7ZUIZJL0kkoIzulrOry7Py0CuF3POCDAGNPRpje9TVapljGVZ5qlWVVU1Tb1x2BR1Z9g2AbtubckXWxmvJ23XzP0XCIk+SrgOKuCWBOqnMe16Q3ax1896u52JR+d2lWbnnJ+aA8QdHx+/9fYbXdf0+knTVGdnZ2W+8pGEsZoQxwUKyRhhDg3Ahr/GGBdcUMIsAQHkenTituR5rfVOacpuNaHTND04OOj3+34lDIfD8/PzizPb7/fffPNNSshvfed33n333UePHo1GI+fc7/3e7z1+PM/SftM0V7MpIk4mE6UU5VJrfXFxcXh09Ku/+qvW2j/7sz97+uwszcJ79+7dvXvXy9n60YUe2PBV7Z+9BC9tf2MRw7U1cj28A4c7yNDt0mIA4d/iLCLfVbUoIbAb73n9q72EKPhXbH59cSrpbuXsOkSuhy/XDvWFTrPHQrZh5wtJR/+nv9aMk7opF4sFY6wo1r69E9EiUp9OWGd0rZqm6WX9tm2DIJiM946Pb4pAKmUePXrCRHB5ebVarZMkMRbiOPa9XRezCy/i5I/QBxBaa48xCCHolhbqnCuKwiN2WmtrDKU0DiPGGAVUShFCPCTp7ed6ve5Us58MpYzKKp/N5kHIXnnl7mR/jOiUUkJISul8vkRCwyhMsx4ATK9mDnA4GsRpOorHk8lERuHV1RUhpK3qxXxe5UWZV8YYCpTiJp72itG741RKMUa8NsytWzf8HB9fAXFShGGYpikSyLJefzT0CfpsPkfYjNmsWlR2Y2PrpmGMUca4FAAQCOnLcA5NlmWM0Tjke3t7R0eHQjAEe9Db92ZBhsHh4eHt23fSKLXWobXMgQjiSEijLCGk3+8fHh7ePLpxdXGZ57mgoqpt8+yi2zd37tw5Pg6YoCzCwX48GPccrHmov/SVW598eNLv96t2Sbj+2je+cvvmERD35Xfe4kB7XIowknl1xkT6G7/191+fnfwvf/jPy6oejftAER12bRMEwbg/nE6nYRp1XYfGSSaZI6TBMBRpmmltB9HIOqebbv/o6NnpCY9lnMbT6eVyWRrdhEFmDBsPB61Wy/XCOSPCQNV52VRZOhZhJMP+4cH9/kHv4aMHbdXkdVfWNXFuPbtMojDrpYeH+2+++ebXv/714Whyfn7+h//qXyY8E92Kom8LBwPgkBpCgYBxLIzCcrUKpewN+lVZ1HUpOWeCEsGsNWVZtnUDDhgQVE63bSDYfLqqqure3ds3btwQlAFjJ+tH5+9eGWOaqkXEQdYbPH8UBTE4h0gUC0aTSDB+8vxUIIyG/bTfH4/HlNLZbNY0TRR5sWqDiHt7Q5+xxUnYH0QecFu3jcySapUv54uLZ8/efOXVveHolYODXq/3yv17/5f/+//1xq2b6SgOR9HCrt5648tJcOvDh4+F0dbqZbFsncmK4uT87Mtf++rRzRv/p//j/7kq6lAm+8ODPC/adfd4+el4lEUhu3P7qGmqcj3+/l/+uziOTk/O9vbG43HSVOuvvPNGIMnJ80f5ejbZO46iwFpLCHZd03YVIgmE7FRDKTHGRHEgJa/r0lpbFOteNlSKcp7mueKcEwpn53NKKaKJ4l4QppQFR8c3lVJKW8ZlOtwLw7Bt2yiKKKVd10VCOIB+f5DnuWpbX7bpjPZJxjgW5fRUIppKE0KGaeRV39u21cqFUVrVXd2oIOkDAGNMsjgIUwrIGaDtgn5089YeM2UqbED5IJl0yi1W6045h1KpeSheZGDeqnq8Zwf54lbO+fq/dtJMO1u8K9b6J71npZT6epPWele42dVodokdXmNIAHWEAFDnEJRx1HMbCXGA+3t7ypj+YEQITvZGz58/5SLcyupZgpQyjxwwABSCGWOYMEGklaoRayoCGQtbhB7PAgTngFGqtdVGScGNUY6AlIxSZ50TUvUH9LV7x/fv3vjBD35w+nQwHo8FBNPZvC3p+z98eP/+/bt3j3/pK7+9Xvz5e+99fPfu3W9849fW1U+enb13NV0LERRNi4x+9RtfefTo4bPnT4C3b79z7+vfeKdtu/c/+GHTFoC067rFYlZVVV2XvjsDAKxFAOeHIDoH1vhLQACAshfiRd4v7gr/Ozd5/QGl9Pqf/9moAkFvUK0vPP1ijDiAbx7ZjG9Goi04tMZzPxgwR6wFg7UTQkiUjqDzEpiUEsa61ghBKPEDDpjvNPad4Rvgx+M9ANYZX7feIFiIzjqADRXAOPAEesa4CMJdIsQcEI/J4Ubb02MIQCihjAvBvNYTIUAoEColl1KenJxMrxaU8JPnpweHe9Zuel6sM1YZIZgQwhjsVKOq+GDvMIqSpjDPH59lg6HqzGwxH4wncdwXMjm7OJ0tP3zttVf3jw5pAYkeEhH4u8M4goSFaa/HeK/X81MVDDMGmq6unUPBpNbOEVo2LXE2TjICzvdbAXGPHz7Y3xu/fv+uoHA1v0LTgFFPZs/2x6Omqy2qQbYXcsFaM+j3191aEjFfrPLpKgmitNdHSrTWLkx7vV7a76dpGschpXS5WNVFs1otV6tFWZZ5sSqKIg6DXq+HhIyig6aqAZo0SfpZwigpy1xpS5GEAQ8ErZqK1qw36NNINtwEHJSzs7x0zhHBIxk5aUUYShtaq13jQINF03UNpySIotVyOR6PkzhmhKO1vk5qtYnjkAuqmkYQoqomi+Llcs4ZadLTXjoCFxptq7WbnbRulIwGI+DONOtyNk+zUJuyM+XhzUmW9W8dHBJqqHBt21ptbt6+3ev1vvdX/2FvPPzd3/1d5w4fPXp09vD09MHZfD7/1re+dfvG8OLyDN3g8GBI6WGrhmkWN2rMpQw5Z0pbra21tq5rAuzu3VdOT58CUNW2cZr4CmKRV0Y7xzvCCAHamY4ilVx0Ttm66PcGNCDOURYwEXERCNIBcNy/eXM2XVwultas0YkoTbMs6w8n6/VKaRPGSZoNnBWq0wQhieJynefzZb5arhZz1bZxKDhnSRBkUajqKl9MnWrSiI0HcSLg8vRZyTQTkjAhCEdnjLFaKQcUCeWA1FmnO6MVIxgJGYZhW+eEoeAiDqO2rFfLBRobCNlLY92pNE2Pjw4ODw8ZkOl0WhRFVVdCCIJgreOEIoJXzheMDcfDMAgYEK21Mco5l2XZjXv3KKVeMt2rEZAt4cj3KHtpUh9ORlE0W8wwDPM8X+sFBzTWrvL8o08+/ta3vnXj5s29vb0/+dN/K+MgSZLTy9NynY9vh3ESTmdaqS4MQ87ZxcUFEKyqam+yf/fu3aurWZ4XzoJv4+j3++++++7BwcE3v/nNPM+XyyUhJM8LITbgQRKHiHh+fq6U8tJD3nJ5mAcAuk77PM+3KiCB7Z8AAF5ZxTtCRASySWWs1WEYepUFn0/716dp6vunR6MRpXS5XO4cLb2mQbQrQ/rfAOB/++zNbifh+kY+pZSU8vDw8PDwkDE2u7qazi5V1/R7yfHh5OaNw+Egu3XzoFMGgFZNa2w7na+drglshi3tnMp1L47b3shdBECvMeyu4wr+BS/ldvhFqiN+sZpOrzVN7N7o97/tud/tYaPF5AduFUUxGPT8OfeETUIIoZwxZIxQCn6qJG4GSNIgCChl25FDJM0Ca5EQL7qIgJuJANtjI+gIIYKAtQbaxgYyLIvqzTfe+vjjT6uq+Yf/8PcfP3r613/919/97nd/+MMfDfojxth0Ol+tVs+ePWfs3Zu373eqTp4FZ+fP5vN5VRdxfPPmreOLy7NXX33l8PCwKNcfffjJbHYZRcHx8fHJ8/P54rJpmsViYQxEkQhCsWnq25zwnXf3k6n/pisOfvtZSGNbJYHtsKsvcAh8oLkrFuwKWHzb8PXS+nHOEUp3t5WDF6MW6Pb5XUZrrXXwgqNw/XO8auo2Wt2AfDsIZHcwu6P1d+h8PuecZdko6yWU0vl8ulotOOdcsCAQYZgMBoMgFACwuJw3bd52hdLOWnRnJ1qbttMykgcHYxkGi9V8Ol13Xcc5j+P0BbyBGzEpQTfEJsG527YmOWM9Q0VZ7XmXFB0gMkplIARQRjDL4jiMVqsVGuusGQ6Hxpj8+fPLi5nRatTL0rRX5UW1WmmltFaUsSxLjm8eySRCSoByymG2WlIKUSTaFqpqXdd10zTaqLau87wsiqJtOwKMi4iLyDlXl7W2HWMsSuKkl0nGKCdInCMuiILeoD8YD5J+T4YCAVXXFOsVABjnkICMwjCKKBdeSSKKIhmIMAg2SgwIzqLXt0BEIQThnBCilNKdIgQD9NIRnHMeBEGapmEgdLfUnWbABYvTOIyjgINzuqYMm2rVNGvdom7zspzV5bKtiueffgoASZIQa54+fvjsyaPJZLJer/f2xn/+F39BCHv77bcd0B//+MdUyCcnp5UmWndxIoQkRbkEojuVAVi+Wq+zLBOSpFmPCUDQRkZvvPo6cbZqqyIvpXBRErWqAqW4jPOmllxQSi0aioRQCqA7bYUNQDGlTWNqhbpzbdOVlmgq+2vVzta5NRQd72lEIoJQxFnWtY0QQvCwa10Sh3EcL5azvJgtp7O6LFVdCYICkFojwFbL6c3jo3tH+zcmvWpxefnsWVvM2/XaphEHlJwQSgUhjlEiJReRNY7LIOHQdR1Vrc8biTX7g5GXQwk5N3Wzns+N0TIKnbGAdn9vPB4Mi6K4ujz3lTNnKXHIOZeUcc4Z2Ywz2N/f55Rq3bAwvHl4ePfeTT9V6K9//GlVVev1um1bIYT3fz6t9G/07R5BEPjf1phNiS6QIecGnUWHlDx49PDOK3dv3r7V/mlbdXXSy8DhcrmM43NEC4AOTS/rUQqr1eLqkpZF9Uu/9PUbN24RIqryCaU0jhOvIZ3n5WuvvUEI+dGPfvT8ydM4Sm2gvT552zZ7ewdCBKenp1rrJEkcvsiJ/UQrY1zbthb9wBJr0VGyaYtwznmi4q5DGjfFfcc5dVvRIbLl94VhmA3Hfijc/v6+t1a+RrMrdvgSnacHeobdzuDuPK611tMtfaUjy7LxeHzv3r1bt25dnD5qSrqiOgzJnZt7b7356nDYJ+jqglICnLNAhvmazlyDrg74RtvfRwbXAYNdVcVb/N1X2BESXzLQ9Iss/Z1Bvx5UXfc6jL3gtX3RiG8yRbctTnsfI6UsyzzP86urq/F46KPMrbvyr0HGiJf/84x0vxeQu0/zfkwjWMoVoQqIBWCEEsb8YEMf6zACFNBoRcui+3/8P/+nr3/967/x698Bwn7y00/eePOd3/67f3eV53/9ow/q+gQA4jjinGtl1nkxnc2+bCwhcHA4rpv1bH5xcXEWRXK1XrRtfXh42Lb1w4cPfvTeD9quPjw8vHnr6OTkZLFcOQdCgjFAKRFCAPiyHfFC6de983+5a/9vu+2iOvgiaOFFYH3RkxCC6AN0S6nXdCf+t48DEYHKwH1xe7EItiINhBC3bXbYhQh+/W/FpDejoPyavH6DSC5hF3Sy67SGjYLCjj3jn/eOarlcOOe06Q4PD7uu0bpr25px37bDsl4yGPZ6vZ4QgoPqOt00rTJNVXZ1pygRMozyYmndTc4jD3F54YQ8zwkCpywQEvxNoU1nOq31ejPphjBCKZBddy4qJyUnDn2riOBCciYFY0CyLCUE8+WKEkjTNJJBXqwkk5KLMOuN+n0hgjpfdk0N6G7dujUY9qMkjnqJOJfz5UI5FwVCz1bLlUZQw+HQObda5Z5GoJTKy6puWgDKhKA80JZ2ne7a1t9rQRRSwY1D5VAjKKWAU2s1gOMUGAXjNGrdtZfoR65RRkhISWyRtJ2WYRgEKSM8R2eUttYCMGNcGHBvJQIZcUq9ZfOUCEIRrHWOEkJ8y0kch6hMwjLOojhIsyhiaOpyoRvo2oJAE4VivZpVeVHl04NJNhnv67V6dnrCGAz2R1kaGWOUts7Zy8vLtlVZv/fa62+yIEz6gzfeeOPzh4+Vbv/u7/7OZDL58z//83ff/Y/3799//fXXHzx4yC2iA0CgcZxGUZgXCwbN/vCgOipX+WqxWDWNRuDW0KzfGwwGavrcL7JAxoiojTVWc87nxbo2ChGVs5YY7bqqKTvdOkFbZ3gcRjy1imhH87rlnTo4HHujvFqt0LFbNyaMuvff+yvdlUq1SRxGWSQ4SApo8WCU3jw+pBRtl5fLqTFKuLYn4Gx5CWxsXEdtRxgnwMIo7cdpkqVl1Vrr4kCsmgobK0VglXba1FotZnNPF6iqol6vKKUsI/fv3F6tVlKIslg/e/o0X642c4dBUGQMOOOUAtHKlq5mBJxzaPXZ2WkYBG3XHB8fRmnYqO7Bgwd1XXtdwiRJaj9vzjkvPemc8xU+f9tUVRUEQV2UURTtTyZozdVsSggeHr9Wd+1fv//e3ft3/97f//s//ukHV/PZV7/61ddff/2TB8+B6DiRDlUUBQDAmAiCqK7V2dlFmqacySRJoiiWUlZVpZS6f//+V7/61bpuP//8kVXdvXv3ELEqc8ZYnuc3jm/Gcdw0HSFMa+vtx051FQA8JACUMsadc8YaRjfwgIegdlECIYTQTdtYEAgA8JgKbhkJ3uH1er0NfLqdC+VPjt/vDuf3hs8/D1ss4Xpy5oEK7+C7rnv06NGzZ88Eqdq6jAI6HPTu3D68dWOfMbJczDhzgts4DmIS9JZhKKFlAAFYfJFsXQ8Uridh/su6L7Y4Xg8UXgK03VazaFdxv+5gdvYat8Mydt+LEIJorwUPLxjsPkltmub09PTOnVuDYW88HkdR1FYLSim9tn9ERDTwog0SgCAhDMBRSqytKacycNYRrjk6bg0wxlRnCSHoCDoGAASYVlA5m/Umt26/ioTvH9746/d+/O5f/eh//B+/9sZbXw6TP3bQaq2Nc6rt2lZRCtrZjz7+yWg0Go/HR0cHQpK2bR88+Hw+n1EGs9lstVolSVIU66OjvclkOBik9145ns1kFCXW4sPPn7StEkIAwW2m7n0n7E743zqi8FKssH3SLwMAAGuRkE31BMAiEgDqJRABKCcafEGEcdh+KQYeynrBgHHwAm3a3Qt+T5t1eI3sAttFRV7S+aDXq2ObQHwXW8AmstGeNyADXlUVom2axjnDuCfwaus6WkEYSiEYYpj1Q9kRhK5qNBCdpEEvG8VJGiWp1m3dsCCQe3uTNE3ruj47O0tSQSlNwogQ4klCXdsqrQvn0G5QQwYEnWOUCsqSMIpkoE0HGsEhI0AISMbjKHBGl2XnrBkMBkkUFEVRFMXeeP/oYC+No7oqmyJnjPmhtUWxBoq1bpZFfnF1dnpx6Sjp9Xr9LFqtVtNZwznNskwIRghah03bKaMJ45xzh6Qz6IjulNZOJVESJSGVola6qcuiyJu2DkPpADqjq7qgzMlWIFhrtRQ1YwIIcQQpU4RsSN5giepqpex8drVer4kjMpSMiV1e5C+ZtzxBEHBOKaVG65cypUikWZiFYRrJLODM6bprS0OtMSW6jiGbXjyhSDhRt4/3j46Om6USHNK0l2RpkVfrsmiVfvXVVz9//EiGejAan03Pi6JarFd1pzqthoPJv/2Tf9c0zeHh4be++WuDwUDwsKlzTmVgHFilGSAlVnemaxSAEyRKI5gMDirVWgJZLzy+eTPr9Rblqq5rQmgUJsaYZVMg2oQLZzV1gjMZhsJ3TgrJKaUy5L1+ksScQFgXRitkjPkcyAJqrao6N9rmRWoNnF88QdNMhoO90VhwEnJCnaEkeuPVO1/7yttWq7arTZOv1+umrnuRnKTxWbnCTjjO/WxcE0bCNLQt14t1q0wYZcV8qSyOhpO2aa21geCmrp0xhlKi7d5wcOPGjddeuc8YqfP15dlpU9VdWw/6PULI1dVVGk+UQ3CInBFCurZFtADu6uqqP8iEENqYzx48ODl7fnR05CdH+/p0kiTbadqOUtrr9XyHG6U0jmPOue8HC0XQUuodpDF6MZsrq4JIHhwcnE0vacCiJM4G/Y8+/eSdL7998+bNp6dXYSSCQCjFfV2Dc4mOBZJfXky7gWnbljFeVVXTNM654XA0HA61so8fPRVCHEz2Dg+PrbXnxvR6vSTJer0eAIRBhA7yPLcI6IiHAZRSu7ICAfDuaGe8Nsi82xo+n/uyFwbOh8Y+NvLmrGkaUZZpmjrnvDyld/Y+UIAtwr9Ls/yT9FrTQRAEURT5mojva/D8pqZp8jw3xoS8DqUMQ3+KutVqwTip6ypOQqAkDEOgPEmSOA6bprFordkaa89W38osXocK/IOdR4drUQJ5wWt74d53uSC9NmP6ussh27lT7hrjzG/GWF968L7HxxmwHQKklHr27Nnh4X6chFv6i38VcQ6cQ2McRS+7tD1mv1MKlHJKgXCPFckgsEo5o4nqgBCjOkMpRec1ygigc84pq+taXVzMus6GYXjr5v3lovjjP/q377//vuqMc+h7igEsY36wRVgURVmWy+U8jmNKeFNrpYu67qIomF4t82J1cHAQBNHBwcFgMBgOh3fu7Y/30iTOlsvi0aNHWkPXNWEYNLXedAYg9Ui8d4Z/470O/5ltq6T5co/GDgmArRf3Qa3jbIfGvbRgdkxGQgiSF8SLXfQA15Sbd3OorbVkS3/ZeZTNn/Q6k+MFgeb61rSVsaoo14wxzinnvGmqqi6dF1QHT8emURRlWRZF4fnpmRCSc65NB8SNx8O9ySGXwWT/CB1Rqo3j2Nulpqm07igKXzwihDBKAykF54xQpZRC5bQx1mkvcUao1SZgnIADh+j8MHTr78swlMU610YlURgFwhlL0A16WdYbC8aNVka1aLtQijSJKQUhBJdMCBaGYRRFURKzQI7Gk6LG65VQSmkQhYyKum2DKCTACGddp5kQ6aA/knKcJVmWZVnfObdcLhcLDpwl2OunCRcQSBYIBKfbqtRaGauyQcVoRBmzyJw1TlkARh1ZLFdMFF1r8mUBzgZBlESxEIEhG8DSOWe2+KXgIo5DyqBD9DnbVueXgUbOwyRKwyCWjDZd7UxDQ+j3gvOz89Pnl48efD6Z7HdNfX5+fvr8NKS9zz77LEmS0WQchUlelR9+/ElRlndeuV8UOZMiL9dSyropHz3+7Ojo6O03v/nBBx+8++67VdVVVdfvr+7cvtfvjfmTZyeDrDce9eIwQHSCBWDZdDbL88JRN5kcHgRyWeSGwGh0CJSsZoVSikvBSIAABDilhDJpnQZklDLBRSACyWXAJOecC2ICoQC0MhxcKIMgkMaYgDNkJAhgPMnKop6vzinlx7dGDNvxeDgZZVZ1gjjbWepsnNDhIDncf6WqKs55V1bTomiqymqdhoxTQik6p7U2pqzWbdEs4sUy1xb3D26W08vZIld7eV7WFqEoCq+tYVQHAAcHBweD8eHe5A//8A9Xq1VVlb48HwiOiIwAOm3RGOIYDYAQa40HdYMgsAY7YwghSmuqhDLYtHqX43oHs7vG3h/P53OPwPtuNEopWhfKgBAyWyyyJEp7GZPi4bMn67q8d+/uBz/9SVVVWZYQQj7++OOjo6PxuH95edq0pUPLGAmCRIqi61SWZW2rnANE4sevOecODg729w+UUtPpLAwDwYM8L6WcpmlaFrVn5ypl2rb10z4BoGmaQEb++LV9Eeq6bWmAWPNiZRsj+EagaVNZ3Lbewbb8CVtxBf8yL9T6gtMA4E9IVVU7O3u9BuGb9HYG1x+zL174UbPe3frWhjRN51dPuBBIXF40Dx89Xa1Ww1E/CQNjVVE1rGgJ5WXVApVURthWu2PYIJ8vYOAvGPqdNd8FE9fdw+7116OonTN4yYvAtT63XcGCXMMtdgn07sM3UCQhTdN0XfvBBx8wTkajQVmWmxcYALBAkAEiWEItJxKIIwTAEUIRYONLEAljjDBmLYJy1hqlTNfZLaoBAA6RAkEgiA4+/uQBAnvnnXcAIIiiJMv+8vs/mE6n1hGlrb9AQRDKICKEIIDRuiyr1Wp9cHAwHA6dI6buAIVWRGvbNjCfrQbDrCiqKEoQkUuT9bkUBJcd4wAEEFwQBG1jKKUEmB924M/3f40H/2+8vQQkvMjUN0nvi+m16AgS4iwAgkGHjgBaSixnjoDF4OdzKl+EdH6xUXZ9v7uQFLdii0h2PZDWuM3yE0Jcjyqun7Ed+cbaTeThV3IURdPp9OnTx1dXF2maRlEwHA2CUBbF2jptjGGMZFmWZWkcR1JKo1EKEcgkDFLJSb8/TJJMG2utrau2VV3TtW3bVnXBJHNo1quVtwN+tUsu0jRN0/Ty7ByNNcZQ2OKFxAFAJCRjlDMKyAPOwiAIOIujIApCjDWJgl6aUsC2bbI0vXf3bl6uP/no43KdD/pZFEhCnQxokiRlVTkH2rhOmU47ZRCdIusqDEf93phSmoRJ12mPpCZZugcYJQnljBBSNnUYhjdv3p5MJvujgRSBc26xWK2rUqPThAACkTyIRBLxgKOzje2AUispcEBqDUHwPSPIwADThqiuIdoC8jSNoyih4GtAkMTJzpwSQnbtkYwxIRlxTohNSkYIkVKGnEdhEidpIKQxpmqKspxHCpbrZr68Ojt7/vDJ4+fPzuez/MGnj40G7sKbd27/xm/8RlmWP/7pTx4+fHh+kd+5szebXfX7fcFgNpsdHB298epdREwTGYTuG9/80snpg48//ti6cjh6J+2R9XrFy7oJw5AJaRHaqpFcDnvD6eVlW7V5U+wfH92+e+/J6dnlYtYpty4LasQgzpjgWlkkkMV9JMRqSxx3lqIhxqFqrK4NaCJFoKp1KoJsMAnlkJKIiThfl89PnlK0USy4sTIgSSqbpiNAe71BwFUQCnBW65oH3OhuuV5cXGaE/VLd5B99/PGoPxgMBlIGl2cXdVH1BiP0XBjbMW0QEbTWXSuNiUR0czJcXs2eLlviLvOyIwQYoyIIQCnUZrK/98Zr948O9pqy+vzTz7yItwyltXY+nTFGR6ORVs45SxAIUiTEoXZuMwhgOp3meZkkCWOMM1dXarFYr1YrPw1PKeXl33fiox422LkiSqkQArX2/ZNd1/FAEmtFGDSNuZxeZf0sS2Mm+HA4fP31188vTru6ERFv2soYRQipqkopZ611jihlnAXBA2PMcHjYtm1RFL1eFobhYrFIkgQgzPNiPr3yUph+tGscpcvl0j92DrKsT7kIg9iLSHbaUxE1Y8xZ6509s2aX4ltr40jsUPSdd7TWSrkpIvhv5x28lLLT2rcVxXFMKfVMjiAI/MnZNRTsEm7/CTv37KMuzrnvv9qNL/eRh9Zaxj20TqPuurZs2qpplcWD/YlSZrmulTLaYlUrSwQXiaN6d39eBzNgyznYJXDX0zW4BgBcjyp2LuS6id9J6l5/wfVEE66libBJH30YsfkNW2U9xog2OorCR48eyYAfHOx51Zrt+5y1PvNGSlCbbpNEUkoJOgvWaeu0sRIRAHnXqbbVXWu7zmptHRJCOOPUGodoAYgfxWoNOTu9Wsz/f8z9WZNsWZYehq21xzP6HMONuEPOlUNN3YXqrkZ1NwA2YcIDKYIgjSa96UE/S3yhzGB6lEkARFIY1Y1q9FBdWZXzdPNOMYdPx8+0Zz1sd7+RWV1gm0gAOpaW5tfDwz38nH32Wutb3/q+n11dXQFAWZbx2jEmON+CQPvs0DknpSxyyhgblJPjowfOucvLy/V6XdeVEDLPYFNtpEwvzm+iPwjP1kIk2rSEhocPj9Okahu3Z7beCXgY/anhP5Ev9a9jQtsF4L517XD/Z98Fkyilcb4uhkxKKe7cmxCRbI2HXr5VILsexDddxKLHBCIGfPlZhJl9iWy34xjbFXT374/ZvNt2F7eNNslC29YR/EsSgQQmk8nBwfTFWei6brNZK6Wrag0QFstb5xxaBO+cY0U2DQGDY+tVrYy9XVTGGOudMhoxMDZGJvu+je18ay0GoJR6KYUQ3uxOQtiBiNZ6H9mLSAlxJNoexeQTAIBQTISkFIVgulfOasRECIZgjK2tbwJI7xzylHPKOT05OUmy3CHpLaTZYDgKPiDlgoBIExYVI0zfRysU730xHLBUJklCOMnqGhGzQiQF3zS1UvPVcn11O7+6ulou1rpvvbfeu7IQwaVOBIpW0DAqh1kqA0m0NZ0yzjog1Hk0vVo3fdQXEFyKJNXK1nVrrc1oFknZxhjvfJqmcVurVuu6rgfDghBirVksFpeXl4hBcDocDFmSMMEJZ+Btp7vF6pYQbVRN0HXKNG3/5PLKGqo1m06O2kZ1yj95fnZxcbFYVoPheLGsuq577/vvfe/7763X62fPnmw2FTjirP3i4wuR2d///d//P/2f/4u2/c+++uqrf/mv/sXt+pPvf/+77O233+acp0l6fXV2dXb28PRoPBzPJge3t7fVajM5PBoOxuzqtm46uljNV8vXH74+PZgZ784uXjR9V2Slcfby5rosSwo0OLDWmd6Y3oIDDqyu63vHo3ffeeu1V94bDo+cpb/66OPNetG39XBaWGO7fjOZTKYHk65T3nsqnQlWNY3W7WBwwCzp+vZmfvn5F58eTA8/+eSjJMkenT4ajcavvvr6aDT5tx/+W9P3Xae0AnDAKSSJZSzNE54Xgwcn926u54/ZBQleEBAycc7prq8qHQI8eiAOZweq6z/44rPvf++7xpim2cSk8sWLF23rp1NE4r0zxtq4/2utjfcEmdYakc5mh2VZ3s7n5xfXm7pbLpcH01maphF8i/dA3/d1Xd/e3g6HQ9iZMe4LBUKp1ppzHj2OX5ydTaajYVlsuu5XH334d37/p0Mh5svFYDDo23o8Hq91jRjyIjU6RKK4NYFzGW23tNbW+NlspvVWxQEAVqvVcDiMsieUMM75+fm5t66qKu99U3fz+Tx++nA4zMuCMxkTW6qZc67rVNRoiyXp3bDnd+P+e/FBwLBT7GGRwhlTopgrMMaqVb1cLtu2PTo6IoTc3NzEdCpmG9s2x+6z9kV5/Cfc0TWq67ppGmttURTRQbvruq7rjEHrPEFgNAVCrIO2t71yh4cnNrDlcq371nrkIuOOImn8HWOLu/GA7KYh9lF8L5O1jxb7LfhuoXk3luw36Lv/vBt+fj2uwJ6JhjE3IvvcJWZOeZ7P59uNg3MONgSIRKO7f0lQyjIW3wpCAOciVQIAGYncDxNVH5AxFjwxRlMSEAOg995FgztEbFtjrD9//GQ8mRBCLi+vZ4fTi6vLNE1DAMqFB2j77iWepCBNc0KYUkb1hnOBwKwJRoc8T6VMq2pDCVeqXy0rpb76zvdSQoR3riiS77z95mRcffbpk/PlWgpJkO0Thf/kHYe/9hIDAADZkxZDQAAMAUOI0xB7hCk4t/1Pax3XcyAv4avYJwqwXYG4JUUi3tHz2CcK8XHAl3IOaLcZOaXUUQcvx3Ze6kyEl82O7YIPITDGLi8v+74/Pj4OwY3H4/gyKeVkMqmqSqnOGK2UWlfLSJEp5RhCEyUJIZDFouo7bZwFGrV3AUg4ODh48OCB8XY+v01Ewgi9C7OZXtVYTyaTyOlDxOBc27axFxlLBaeVUopRCs4ait4kKedG9QTAa+W9N8Ysl8u+79M8nBwfhMOpVXpTrfI0TRIZdxLo+1bbm9vFZtMFTwgXhApGhXc6qgUmSQaEKtV1XS+zBMDzhCeZNE5vmup2edX0Ffe8qurbxXKzabQxSZamaRrANX1HiaVgNPeSWDFIinx0MJ14FOuqNnqpvUMk4KFpuuV8DZQGT5PM5XnoO7vZbCILO5KmnXMQSJIkw+GQELJaLNfrtUw4J6RX3WazPjs7E4KliRim4071UknKOROUMLTBu74fj4fe9p3qtQ3WwcHBKYXhvXsPVuuNtfbievHJ548Fp6+99gpQMhgUb73x2uF0orr6h99/709/9ieffPj+wcE0hAB0NTkgVJCTZPbbv/Pqd394j1KcTEfsvXfe7vrmi68+v17PL+r57deL+48eXPTLNdEN9RvVaa2HIp2xNOmD3vh7bxz8wU//oKo3/+Tqpu7WvEAgpEiSXHIg2HX1aDAczgbtp3UP3SAt7cIxwZCG8VF57/hgU/XFM9G1lbW+uq3TIhselWkquAhFQnulTGUIIZSlTNDVskEkB6evKe+/vFpDfnjeWFctbzq3mK8ePHhAKT04enC7XCjcJBl4H4x2NaGJyEIgIh0Yll+v+psV5Np4B0YZRRwgyJQcHU/vv/KwKGW1WVizHg0zipKfjPM0Z0y8+eDhk6+ePHv2Qma8yHNKed02WlkuRSZkQOICciFCgMV67QkrRmOPJB9P180GGEG6bUlGm7WoNtrWdTTzDUkiZKKtW97eSs7yPO+6ThkdNCRZuVg2600Xgjs+nH3x1ZPD2bhr6+Ege/jo9Oz82b3J4QjFcJAUg8knX355dn5VDkabpjfWSpmuN9V6ve5U//qrr8Ep6brus8++CAGzrPjii68QsRgONk07HI/n83k2GC7rqg+WZVlrLSHkajlPspRos7XCE5wIDowGSvI8i86OsYMQN6YQgg0OAJARJjlYjPuQByBMOOdu5ksX8PTBq13Xtb21tu+aZr1acc4xBMFYKmVd1zqSGLzHEOiOSmmttVoXgyHnnFFGAI0xulc9oZiCtTY4PyjKk5OTsiy7rqsZz9NsVa+btvLOeGtGgzwbZlKmJ/cfTqdTbeHicl6tO5kWIRCt+iwdbZbLOLsMANGPihAmZRrDN6WEsZd9X8aY1u0uW4ogsIuclbvsdETcR5YdH55wzuJWHmlrSm1zL9xOOnjvnfeeEQEAwQUbzN22RdiS2sj8Zlnmg3qtv9qcISJhtZRSckqIoCQwisE5rXok1DkwmgD6EAIClTxhnKX5QIhEigJBto1ZLurlommNIcIZ7ZxzDD0BtFYHEiijWS5W61vKodqsynyQ53m7Ua88eP3m+rpr+zRLsizTXW/AxEjhArm9WZ2envadfvL08Ww269UmzZhMBsYYpfVsNg0BGc2qtZrf1kwOf+u33pIJU101GAwAshdnN2mxFpz0fWu0J4RFS1hERggD//K07EPgryP5+8gaHeNgO/IB/4vtixDcX/v8Pk/dv/n2Q6MrOARAIAQBwIPzzuV5vv3DSAASMwdnvXGYIFDtEaxHEpAEQO/DtmW5TVgRgzMxqmuPW+7zrtPvvdfWp2mKiNYHYwyllFCeeNDGLU0d3QhTBBJilukoonOMwEuZUfAupo+BJRZ5MhjNEIfDklC4vb25uTiLo1i3q00iMwc4X7bT0VTwrNusKOEdoLOeUi6TjFMKLhDCmOCEEGSEAVvcrvu+BQOd1845wXhR5FEUQWtNGYHGH0ymRZFppZqmySS3ZW6MylmCiIFzW3jOeZKIGEoNEA202dSc0DJNAJLggbMiBz2592gyHDRt/ezJ0xDcZr1crzet1r112uOL86vOk4PjYwqU0MBS2TvlnPNowQdrWggmT7lWtaBErxbd3AH4IeXQW92sFrrP89ybtq3XWVp2nWqqdjId9V3Xel+koiyHGNpNXxcqPckOoWarq+WyXrM0q51adn0XnB/l2oe+s7Vpe4UMOU0YAaK7brW4SpKEEdI19fmLZjlfEEIo0LquV7QaloMXz6/fe+8d9HKYjcEm1bLTvUPn+mZxc/M8lWSzfjEs0yIb17X+8KPPlPFiWJxXl3/77/7BfLUeTwfvvvtda/3zzeWLF2dZ1/LhdBPIX356/v/4F381Go3KsqztAWT41UWV54VRs88/r0azEQS8enFx+vYby+rq2fqMSSnni5tnz549f/5ssZoHMIv1crmcc4rT6XQwGLRtGxXTLFrnXFmWUUYNACilSikDXghBCPPg4m7GGLPGJ0lmjEOkdd3MV8tYlXLOiyI7ODiI6OLB4fjV1+5PpsN1dfP8+ZOuraOQp7WWEACgwTokIcuyxe38c+tevHghKCvSTEhWVRVjrOt6pbTWJt66xtiADoGHALc3ixBCkiSzGXEWGZWbTSsHYjXX7/z2a+++99Z8fvvZZ1+kGb++nVfLrxAxkXIynh0d3RsUw++8+86j116/urqImlllpzvVG+cBiAt+03RN21vrCKNMJAAQnA8hRLnTWPxF2FkUPHKAASCqDMWRpDh/2Lcbzrm/I/0WWxJKea31Rx99RN57d7W8mU2/Wy0Xl5eXR6NZURSr9Sb26fddAGON1hYAoo1KBOTdzsA3Oq5GUmF8XJaltXaxWET0fgdFQtM0jIp4pWJr3DknhHDOWesiSiGkJDuZAReV3e4IB8GOZphlWVmWx8fHi8WCMRZpE7BzmYp8T9hNSG7Hqb3vus77sA+rTdNEj3nYARj7nTp+YvS1qus6TrL0VhnjBKeSMyll/CKTycRau9lsoo4sodQFECKx1jNGKEVKo/AAhACUonMQ7V7iR8XPRUQAf7ey3IeNPaYCd/oI8ad7GPhOKHo5Wol/XeC6G/nCHfrk/pn9/AUhJOeCIkDwRlnPAkFGWQBKtVFAgtmWpMA5I5ILydNyMBhMRsNpng2dJTfXq88/e1J9feYQLAaHIVL4HdnS9NpNSykMB+Xv/M7v/N7v/bSpu3/8j//xkydf3Ts+NcYopSBgr5V1VgopBA/eyYQ37UYplSTJeDzM83y5nMeFZJ0GAGu3eK9z7upydfbi+mB2VOTjROaXzbJtFCXCWo9AhGCMCQjEGOM9eO/wr1Nj/GtP4/50/aYf/W9y/KaPjusZduthj5BFkD9e1ngXxPvobnMK7gAAIsmttV3XOeeETLIs45wTtm1jw05n0+i+qaEjJCtyF6wHZ62OmwNFoJQa54IzdCscGXxsEhVFtVx758q8SATPU1ltltVieXN9tV6vCSFNtcEiMCbAeWu195YxFpCAi9BEgFgwIOm6DikJBIMxddt4CDHliuQnFi1LfAg7N6xocEApeuestVop51wIvm3beL8nXEQxqO3okDdRJz6KjCdSjIejw8ND4eq+7/te952JVAlgnvAELXT9prceqRwPytFw1imjtS0JSCmdM5RSJJAkiTE0eN/3PecsdjzjXx5ztihXL6UUQjhnGGN5kXpvk1QMioQxolRH0VCCUvLxeHy5XGhrjHaa9L3zWmsH4DxkaU7QOROHoMB736lOtR3Z0NFoJGXa973qTd9rIQQFSgi5vb29urgsyizLsh//+Me67w4PDzer9cnJMaD56uvPri6eceqbZu29/vzzT631V7dXBLkPfFO1H3/2yWKxHI+n4MP9k9Pvffe7nPKES60tAlFdTxC1UjddIxl/eHr/WlxeXl4+ffz1xeVZUaRvvfPa6+88SkW2cFjNN+z6+nq9Xkc7cM55AOj7PhJNJ5MJALx48aJe19ZaTmiSJE+fPnXO1W0Tu8ta69YoyhjJCCXMu5ZzqbXV2pZ54b2XaaatX682q9Wq73vO08PDwzdff/XZs2dCiEcP7v/gu98bjcuvn3wxv77ihAPd9tI4oRTAhEAABaN931+enZPgiyI7PT311p2dnSF47wCBxtgGAMFTv6VtkzhikOd5nue3N5vRYU4IMcQ+fDh66ztvjEaTJ0+ezOfzssiaWomksFo3rdZ6XjeqzAdCCELIG995O03T0WiUZXnb9+fnly/Oz5brTV6Or66vl+s1Q2B7+jGQ6JYUt4kI1mVJOhqNTk9P45hsZPNVq/VyuayqqswTAB99KFxcWM4hSO89AqWUt223WlUX51fHh7PY1j08PLy+mTPGIowZeZFCCGNcnMyMpyIOSsUzMBqNIicghJCm6Xg8zvO8qqqq2tyVFJRSaBtnfGFPJwwhRAoFfBOZ34cr+LWoCQBa69FoRAjJsuzi4sL7rd/afk/cIy6RahDJB3FeJmowRx9LAIw/gh2gGscs9x8UJVMirS9NUweBUiqFFBwRaVXVkgshEmNMtd70vZYitz5sJzvgJe30Lgsd7sT7fVwnd7Qc9t80/sh7L4Tw3xRO2P/oZRl35wTGjOdbT94Ngd/KFX69ho7NEa815RwDoTQwjIMNHggKKUUmYlaEiEzILMuyLEvlWMoUGVfWC56MZwezdTtfVbe3C4/eY5xKDB4BkXiE2UB4D+16Q52ZDsuHJ8fDMr1CLwThkgWPTAoGHgwlQgITXq3KsgghlGU+nU5fffXRwcHB8+fPv/jiC+eNMdsywBgdT/hyDk8e3+ie37t3z1t3dbFaLRtEHrbnMNJftLE6eIwJ6d0g/esdgf/Ix2/63H3DaJ8SRRpB3/eIGB9HpnO83WJSG3833HU8N3q9qZfLpVKqKIrpdJqkWQihaRou08jzRcZhZ3QeOiKlREqRUg6BIPGEhGAggPOOY6DIAMAED84Kq9rlyntPrQXvTdeubxerm5t6tQzWImOZ4GkiKOEYAIJHCECIs04p0ytNCHMehHABCRNcSkkFN84455TRQrAkS6OgO4GXSEx8FBN354wUAgDcVrwkrvQA4COBTylljFJKGdUG5521TquW0PFwdDCdZFny+smDs7OzqqqQWpkOO9VzkReD1K/WqaMSmMw9T7MsL43boIe6ruLH9X0fXYWVUm3bGqMgg61UjDPOOcoQEXsFAMAYGQwKpUwi0+D8crkcDIvZwYhT503NGOFMRKBx0/XKOOudMyTaPiEhGEIiBKXBKm+1RQAhmNK2qjfYkzRNs6yI7Pi4KjKZjcfjm5urq4vLcpB/8cUXr772yp/8f/7NYDD43vffOb1/SAmkWfLbf+u3vOv7drNc3fyTf/JPKKXWuzRJpUiTLEfmi0E2GY7Ae/Th9PgeB9p39uuvn65X6yIt8kQSgucX15zTyXg4LMv5zc38et509b3To8OjmW70V58+bfo1DRn7p//0nx4cTmN6yxkDZNoZznmzWcdmCadsVI5iyTgYDBpdVVW13qy06YWUnJLAiJDSWZ/mifdhPJrd3iyDJwA0qrgAkPV6/dVXX1GS3js4EYxOJqNnz56URTYejqQQ3jrwgRHOCDOu45QESpyzGJBRRBL6vq/XlTFmMBjMJtPZZNy27Xx+wzlnQhIiOI9tYIporAuIRGkjRMo5f+WVVx5//Wwxj551hcL+Rz/6kRDigw8+6DrFOV2uNpxlIUqlUA6Edsr2ehkL36dnFwAwGA0fPHhwdHQ0mh2cPHyU5+Vf/vzngMTH6IWkNzqEIASNhbsFjH33WJQrpZqmETvGXFxSccYPQ6yktwS6GB3H43HfF4zg/fsPVa/KYvTs2Ysf//aPHz0ExnAymUTbhYj0xO1GJlnf6zifGd85pgURKogZcQyKkTITdYX3HVPcKSzxIIx28Z9wh0gohKDUx+AUv1G85UUiw47JGBfJHsaQUlpr+76/urqSUkbRiNiV9N5vNhvGWOzLcM6Hw+FWvtS5OJMZuQ5Jku5Rk5hFxRfshx3iXhnjtPdeGQsQEGn8ldVqxSmBgJPxNI5xUyK1Dc6TAGitj0mCtdt3DiF4b723AH5f5FGKnFPGtidkDxHv43f8AyKbch/L40/3M2n7GBAf7N/817OEb714/xHfQtpjOtK3ClNgqWRUUBK8C8YaZfp8kGf54ODwsBgOKKUhsqw5sx33SNdN3zZrIbI0KZAzkaWeoKfBbyno6DyJs/pN1Q9yXqQkFXSzvG02y6ur55NpeXX9wnmUMrXWKGMIYT5g0/Zed1E9ejqdMg5d3xB6wAWVCY9uo/uMh5AIpMvlvNf9RbOxWZbNbzdaAaUgRGJB7/mt3jlCKRfU2/Ctc/Wtk/Yf+fhNn4t3mhSwi/0xPuHWV5ZIKfcTPW6n+rVnxsQHStt2U60X881msxaiaZo8zwkhhLLEOS8EUs6cowjgOQBUfV2WJaFAKCDhJHjQ3jnHOOWUBYRed4JzIZlS/fPzhTeaUswT6T1VunW2J+iKLPXBIWJRZGmaEsJ63nsHJCCh3HptI+kYPCB1AQljo9GIJxIJCRhc2I0RCU5gS1qE3U2xg9iwbVvnDB0OhRDBeaUUYhCUkK2WiaEUo6gGEmiaDSVAkVjXt1WHYNdVLm7g5tnjxWLhnBNSdtrOV831ciPTvByOkpLKpKh71fVqvanrppNSMkLjXtH3fazrAKBte2Oc3NE8nXPGKjQBACwK59xwOMrysm370WgkGP/6iU04HY5yDFp3fZExRpxS6vz8fFU3yrlAGVJKARPKNaAPRnV9kmRcMu0BA2Y8lZQFcAG2G7IXHgKx1sfFEJV4Tk9PszxZLG5fvHjxwQcfIOLz8683zZozKPPkv3jvf6fa9Yuu/vzLJ4tqE0KQMlXWeGKSNFemF0kSAl6eX2zW9eXFddf1RoeubqxWpCgE40mSLBJerder5e1kMvnRb39/ubhcrVa6Nbfn84/pl59/+SnhMBqX7P333/+9v/27x8fHfd9V9Rq3vNzAOY+b9f2T0++88Z1qve7qRjCZGpYk8mau1+u1TJIkz3wA76FpWplmjIl7904//PBDzmW9aRljTd2WBe+79a9++eHV5fw7b7w9mx7Xm5XgLBHC6P768mpQpBxYwqRuDASbSsEINk0XIKRCeh/athGccsZms2lZ5gSDs7ap67IsrUNr/G5kLwSPkf+tlMl4dn5+Pp0cpWl6cDDre4WIJ6+eAMDTp0+//vrrsizz7MBZBLTeO0KJkEwI4RGMiRMUlAhZVdXi2fnVzVImSQy6WZYVRamtiaiDsyFCTPs4igFibI7p6mKxuLq6YoTE7rv33moTwQDd91prG3wAIoRkQmRZcXR0lGXZxdlzqzVjgnCuut57eO21N7785P0oVRSpfHd5hTEAx9CrtQb0cXowhtiYEwBANAO8urqq6zo2OPdNBO89EoyCo3IvNUpInIILARhjfd93Wu0TlAgt7BOFuOtZayM5gxCilIqugPsUIeZPXddF6LUoijRN0zSN4wzxK8T2VswY9vtmXJB75Db+DWVZRniwaZq2bVulhRBd13nHJEejnbVeSjkajSjlXasQag8sAAlAI3iGO93ofcghuysVb1q2tfVi+6TqLkLgd2Px+2NPQUfEfaWIO1Mf/GYbYn9sg8qdn+KvPb774vh/a8DQQHkADEgoo4wjD4T7QEMQQBJKMkqZ8UFbjw76zua5ZIIThdo623SbtulMhwwxUAAbQgiAEVFwgLHTMx6MB4MBJdD3TbNRwzEh4JjkjEPbNcFakrBE8uAtSAxgrVPWqfn8dr1e9X3X9/1qtUQkeywkdugopc5JY8xise46PRwOKeWj0aTve+/dzoiccM5jjsE51VGX8A5N4T9VivDvP76FTm1Ze3fugngrRdgvKmTsAuhLg7H4MkogkbyusWkaY0ydpoSL4XBojaaMAwClNE1TJhJCiEYfOQhKyYQLKSUS8NYhCEawbZq2bfM8H40G7aZ6/PhxffssTVMhuJRcWbVeLRDCZDrknBtnOeeECQCQSlZVDQ4SygGI80AYI8iokEKmnAvCmXUuWOvBU0YCovEuOMsJDSGQqP4eWb2Aca/Qro8Jv5SSU0YptVZ7ZeLZ8+AIYYABMVAakATO2XBQgM+6qh4Ny7xItOm++vjTLMums4PpwaEHwvP5elMTIZZ1q4xLAtvUTacVEIztG4csbpnGGAhICHHWexe8wxAwTloAQAguum/rALFlCYRUVYXEzSYTxoOQxAflbW+tJoSVZWlN9/z589tq3RkTKCVMEGI5IAIqY7qqlkwKkXhqwCPjhLMUMfhAv9GJ7nXf953tYuNjUJRa95EG/tZbb9V1PV8u/ur9XzR1dXpy+N5337G6u7lebOpOyExrTZls2r5uK95pbV2WZbf9ejlZDwaDq6ubpmkpMI/+9OGDptrMl9V4PC7KrG5WZxfPgLh33nnn/slR2+tq3V5ezKlM57d1p7uL88V2Aj7OoNftRiYcKHJOgzOEkEFRvvfeeydHJ59+8olSCsN27DtN04ODgyzPA6Hrpo4BclM1eVZKmTa1okRWTcOYi+OxxriuXTRN541/eL+eLxZZKp1Xi5trrzu8d49zxgLtN105FrFcE4zE0+e9wRCGwxEhRHJmlK7rWrWNZDThrFbgHQZPAnpE2I0EuSwrtNZ/8sc/e/To1Zubmywr2raVMp2Mhr96/xfGuKIogkeljJTZ7WIznc62k0ucaK171UbTMy4lkylPMA4y9Fqvqmp/t0eDIsqQC0qcIxBnfAKElzVl9G5HxOC2ra+YKMTQniXUWt0bC0hDAOu9cyFN08PDw6PDe9dXF5PRwYe/+iWn7INfffzDH/wodnyibYQQAnA7VRi0JoQhYkwUlFIy4eyOSwLsoPKu625v3Xw+BwBCX8rRx5tHGd13Ou5uWZbFbT1qSyCSmCdBIG5njHm3JR92CoP75AMRI+wR1Z+iatDdYEkIGY1GkcMRK/6+7+PG2nVd27Zt2+0TkX0lvW+4kJ3+Aewan2maC7H1j6covAelTEzGKOXW+rZtKU99INZpa70UEXvwgB5JoJQSCoDeOeCcIqIxgVLkgkZbaiHSuxX/PlGISVv84ngHEv9W73n/pL+ju3AXVMDd8EU8vhUF7+IK8ZnOgzPetEZKHCT5YDgqRwMm+GJxK0XiFNmsDSE+ICBQRMDA0DGCiJ50dW1M3dUNBZScog/oaQjB+uCBBh8IIePRqGma9aZ7/uzslVdfnUwns1natl1R8K433ljBWOCBog22I95mZaK1pixE95MkSZwz4/H46dOncb5xD1bBFhQJMS4o1fc9GwwGUh60bXt5een8NpmmOzltrW0cj/x1RGF/Qv4jH7/pc2NDYf+asBPg2tIVdx3e/TP7+wK/qf4ZmTdFUSillkvVtXVszLWbigoZb14hk7iBc86L8ZAjBGM7bTznFEBKSRG8Vr3RzaZq25YDWCmr1fry/OLDv/rTSE6SmeSCAoHT03vvfe/dfFBWVUU5M8b02tZ12z950vcKAyGUJkki04xSThjnTDIpXZy9DJ5wygTjnLtgrbXeWEIIRYKIwbsQgg/B2sDIto8Ub59dF8ZEFREkATwYq43tte571QZrsjSZTIrJcMwPp6f3jg9ms/V6/antkmysnf7sy882na673gEthiPjQ6t0a22n+ths5XzVdd2mXnpvKeV9r7wL1voQMASaJBmjMoTtVkAIAQwhOI7UOVfXtfVusbgFcASt95oL0baVt8rpWmtalgcQkvMXz1d145wTlHkI3gEQAsEHbZ213mggDHxwzhqjGWOcMx9YLIGKrMiz0rnQtq1qVdd1SnVr541Vo9EghPDd73735uZmcvo7jNCz58+M6f7Vv/7Z9eXZ82dPFotb8DaEAOhcIMAoUsFIIEyILAdOgfFiNJwdHg0GI+fCeDwO1j1+/LgosjJ/Y764f3NzTRk66gbjw+N7J7eLOfLEeTae3E/63oNjiPj5558H9G3bOOcoFZJIzmmz6TnniZBt23744Ycff/RRKuTD+4Oj44eImBfFm9nbo/F4XTfLTTUYjf/iL/7i7PzywYMHL56fbzaNUgaBItCyHBBkjENZZM6FxWIhmDBKU8qt6lvY6LbxxgrONuta8iSVdL1eK6WESAghfdc455IkMVpxzq+urjCEIksil6frOiQlpeA9hOAIEkQE7hEpY8KDfvH8sq7bum4RaZz5uZ3f1M2GM5EleQBirTPOMpo0tXLBCyFEmqRMuOAp50CxLEaxwt40bQhBCJEVAwC4ubxijHnm4yz7vgDdxjC/VRV0O78AYwyGsCMTbbsMSZI400aMnVAMIcSoBkCOju5F+r1WZr2q8yz9+c9/8Ud/9PePj4+NMScnJ9e3q4jLAQDnPOzoEVEOsm1bLspYr8QPilV7DGl7k2vKtkIfsZWotVZaIRLvfdu2zrk9IcAYA7AttUWZxG9kjDHOxxGJsCNqxXox0hK11vP5XGsdoQvYzUqQnSF1TCzi6GZshXRdB7s6MssyY2wEEvaN/3iSYxUVcYhotRXztrwsGWPVet7WraAkhKCUqqoqtsYppc4Gxqn3ru+jNV9UxXXeR4k0ICR2FTH2GqJII6WEUuK9uxsbwsuRs7DPt2IScDf874/9C3DXpb77mj3wAH9dorAPh/uELD6vPToblHedVYGLpPAyI4TS4EVXu66pERtKGaMiZlQ8SY3yzrmogKu19g4k55yywDyE4GzwQDE4CAAeq8721iHjz8+vem3unZ68/vqrf/WLj4W0fQNFHr7/vbc5F0+fnJ2dXVoHSkDbQppC3VSIOBoPtNbOmYj3hhBwN7lujDHaITXeey4ooqubFaF+MpkMRweEQnTAugvYAAAjfH+67p69/39LFPDXWg/7bxFzgkj6iet8SzW4c+zfx+heSplIXuRp7DYaY4yB2LmLOtBSStV3fdcyLgAgaGOECN73lNpeMUa8sQDggyWARVHkUvSb6vbifHVzHUjQTltjetcxwfIiHc5Gb7zz5mA8ur29ZZx3qlfKzJfrq+WNAU8UMsaQcc4EYTwE9LBb+YwSyoXgXDJEjOCUalpKKVIWQoDgvPckQLDeIol0xRiGBeNxGxScRAFsY7VzJgSnTa9UF5z1Tvf9mEwGRZYeHM6GZbFc3t6/f++Nt96sNs1XT5989fSp9pgUw7FzIs0CAnBmextISHKRW2lsl/IkSYXgCefSaEuIiKikcw7RGdv3yhpjCPGEBsSQpunh0WwymeRFsa6WzhkpGONBcLSq4yxwIijD8XgsBb08P7POASXAuffBOBuHjpy1knFvrAEF3pGA0SQCICDyWOlFOtdW3hZopG0RJG+++Waep2mant47nk6nK9NlMnnnvRGFwCg5PDw5PnqQpfKf//P/mXMegi8FJ5xTStMso5SOs2mSJHleWmunk9l0NF0u14h4dXGpvAaFyDAwZClfLG4//PSj9qQDxtO8RMJfXNxyKRxIQggbDAbX19d5mQ0GZZ7nZZl3uqcUe0qzLOOUVVX19PHT5XL59u/87u//7T9gmb++vrm6vs6G5XQ2o0KOD2bfeefdy8vLrx4/IYR8+eXjSEHPkoxznibYdR0hbDabaW1X81VVrSlSb9vgPKXUGVWt1pEKOhoMKevruu66bjYTGEgkqQ0OSueCEMKZJeciz3NEnB1MqqpSNg4ub33bKY0Fmeu6Li8Hp6dHxrgsH9R1K4XIsuzjL3/5yiuvql7XdX1877TvzPXtajgcaWswhCTNhsMxUAgEPay997eLZZqmIkkBYhkNWuu2bY9O7qEPWmulu4gw7xizQQjhrYvrHhEZ2fohkV1x3/e9URp2RXCsvJMkKYqSCRN5BsaYxc1t33Y3VxdxTuHFxfrP/t1f/Hf/zR/N5/Ob28X55S3scPIsy7pexzZBDI2q62XCY3yNYTUeu93KRfJBCC6G8Bjate4Jo1Kk8U0ikEB3ZkhRaSoqKMeNvuu6gDxJksht3nKzhdibWTjnorIyIg6Hw6gcZ4yBHfsh/lYcCYngW4yCUZjZGNO23V5Y6S62H9OCSLaoqiqiuDtCkG2bvq7rYZlyLiilIeCLF+daa0pZ8J4xZmxwzjEqvO/jG+7n0/b7O/k12cQQgjb6W3FiH8LvFoJ34/q34se3qsbwzX4E+aZB5bcC0j5LeIlY8DSg1861rWq12dTt2fklIYQSiCgMQSaF4ExSygBgMBnGyOScC9ZZpWNblFPikQQMSIIHTwF9QBKA8gStTYvh109ffPn4yd/63R/94Ac/sE4tl/O6Xr39nVf+j//dPxqPpv/in//rf/Nv/vj6+rYYDThvRqPxzc2Nc4FSeju/ns/nfd/GOzRKlVJKrfEhWGtb7z1liQ+4Xq+7vi7L/OjoYDqdnp+fP39+tl6vnXMIlFFGKfU7XYG7p/SvTcv+4xy/6XP3MN4+/4tHvBfczuODcx75wpH0c5cwu2+xxew8LvWICGqtAUTEcXtjI2eoqipCyHo+n0wmRVFAzEIYiUDdsCxCCLPZ7PTocJDlZ2dnFy/Obq+ui0EpBOOcEkYa1TTN5sX588+/+vzd777X9E3BCsKZZLxwlmcJ6VtJJVKLSAWXAYnWVkUBV+eQErpzV9/mymTLxtjmSbFqss4HhwEymQghvHNN0xjG48lJBL9zP2KaZkNRhuC6pna2JwScs9r0bVtr1Tz++ktKwvG9g4Ojw8fPn14tF5tWa2Pmqzlp6/F4mgiurVoseiFJ3zRt2/BE+p00C8SOLdBEJlmeaN2vq0UsgQjxDBAxXF1dDYfD+/fvv/HW623bPn/+9Ob6wnvLeeItDAaFoCAZHY7KJJY9XMTzoK31HpAEEgARuRARUsWdvRwABAC244wrpRBqY9xqtWo3rRBiNJoIxn/0ox/N5zer1SoRXGudDwec8yJLdK+W85vTk2Mmsrwo/6t/+N8i4u3ttQ3eGNNrNRwOjbPVoi4no8OD482qMs7erpebZpPKrDUdT5MsS4BiytPB5OFoUiqlqrrNihy0toB1286Kge7qumvZuz96+OyJD1AVw1QKprqbjABn/MHrx53SzsKmbstR+eCVV++//kYLEGrV9Pbg3v2yLAmjRRqW6/WzLx5TD0ez6fz66nYx//GPf/yXP//50enMe9+1VVLmwfmrxa1kcjQdkUBUr711qZQMeV23XdcIxofDYZpmCcg//O2/+8tf/iJ0rmnXwzQXQgTrKOJyuaQ82bT90xe3RTm6utWMFYFBt2n7XgMh3nutDSFMJsm6ajatQ0KSNNtsNm+89fqrr7/+L//lv5yMTq4uVkmSZll+fXntXEgl12rNGFs3Vd/fJEl/cHQguaAkrNdVLjPdV721SALnFBE8MVKYq8vHRZFnWU4YU71Rqk+SVEqpekeQOh8nPAnn3PuglGkaVWY5o8wY07XGKC8ltwY8oUioAAjBKdVLzhORONt//dVnnPO27TddW9eNEHxwUPy//s0fn7wx+N3f+ckDp3755adkQ1ynVptFMRi6rtXGTw7H0Y2pqTbH6eHFi0uJoiiKsiy998a4+Xw+Go2yrKRU1HXNWJJIWdc1QVsUhTWGOxoAEUnGC85537da63ExXC6XCReI6JRqtKaUMvSZZDrwmNmkaRpBiH06MpvNRqNRXddZlrVte3JyAgBNv9ysFaVcSNJ2dVVVfdOt5uujoyPJZTnN67puqkoyyhhb6y41bJBPbWpW6/Xl/GZ2dLislrLMZ7NZCL7VnTOWcSxpwinjnPdWN5tNCGpQJGkirq8u7r/z1mq+ePHiRdM0lJK673S9YDJJCtqrPqWi73vvA+eC7GblkyQ1xjjnKaVFUYYQtDbOeSEEhi1VAnZznhApDjsiRXyeRB0V78NOV1FKmWdZ3+lNXQshpExDCLB9BcBOYIewnaLtN1MNCrBlPwQXgoedxUBwjhDKKUOWhBB059tN43cWEuTOEV8/3GyklCKRezwfCaGUtrYDSsA7a7VDA8JTRMSwXNbT8WjTLFwIX37x0Qd/df+H77354CC7unxx/8G9k5PD8YSVpZv9Nz84Plr+83/+r+6/9vDN19/SjXry1dcfvP9x2lUmYOuhHOTLuvOEsCQHIBy5ZERBOze6U11vMUuESKjS+vx6SXk+mUwevvLa2cXNcDCO8t4hBCGSdecAPcXAY6/ba6+V8z6RDALxEFxAH6gHcAEBgKH9Vr71LajmbpaGd0w77+JGv54WfCuZ+xZ+ADsoMS6GCMvF2B+7dSGECK1FLk6sE/Yhdh9LrLWB01r33nsMQWapDV6Z3nXGOwseESBlEEJA3WrVAIBTfb1ZR7oPISQO9HrvAY4ePXp0/+GpDuazx5+fn58v65WnwWmdpul/9Q//W0eg7VWr+mI0Tsr8qsLy8I35Yt629WCYzZtu06vOmWE2FdL1fd+j5Uw455zpOZGcovfWa6esphiFI7b65c65SGmMzCqKkGWZ5AwxKN0ZpRGR0UAJJRQc+LqttdZplg0GAyFE8Bac71x/ODqhClKXfu/Ru33dPHn81YSUh+8+enp+/cFHn5xfXlTNRjl3+vDRYlVr03Wq9msXAmrtzs7mIQSt7TgbVp2lqrPWWmWzrKDeXM+XszAtyrTMZd9KGox12nQ9ISQl/UAWYNab5Yteqa65bvs5glLaZkWMEiFN8tWy1r1ylliJAMG43nsIFJS3IYQ0y7TzxkQ0QmprPHGCJcYY6nySJIzxzWZzdXnjXEiSbDCaZFk2nU4PDw/rzlFZFIQ3GhhLdL3pvV9db2czPv7sK+/99bqOywYZRaSIDGVWKQRk4+Fxs7FfrhcAAOit7Zq+6fsbAG9EtvbgrV9t1hGRbdv2ZDS9uL6OPkSMMdZUTDJbd+z+wwdaa6OaNE0hGEqp5DzLk2JYHGWl1u76Zp6VJk3K65uLzz77tGsrQgiX4ic/+cm905PFYqGUury8/OEPf1iOhvP5/LMvPj87O4vWdpTSVx48vLi4mF/feA/G6c76YTk6un8qKFvczq3SZZGOR+WgKDnnVVXdO71//8HJcrn88usvCePWeMqgyLJltfYuSCmStLTWnr84i0oMlEshEuNAa621VUpr02HVcs7bTgGgc/Dud3/w5ptvnp2dcS4ROOeOEGaM09oCUuLBOWc9BA9IqXXBGoeEEmQQCONAKItBgQuKGCIN++h4GgI656wJEdC2VvV9R1B+q2rEnX5zTJn3dLz4Pi64XY2IO878lvFntIs1OueMc+lcUKp9/uzF299ZP3jwKEmyqqqMcZzL9XoZN7g8TwG23MMo+EgDxd3EdpQuODg4GA6H19fXkfMY3Sxj7wAAtLPEM0SkCBg8UJbkohwOF+sKKENCiKchBCCUciIArdo2VmJ9vyckCiHyPN9sNmma5nkeXbb7vh9PJxSpc8EYV29arQ0FgkibpiE5ckH3m7n31jkH6LV2AUEmnHPqnKGETMcjZ3SUcpNcJEIyAhQJpdSpnbA8otY6SWSs1SIHImzbDGavpb8Nz7tQCjssIUIa+wov4t7WWkq37LN9MI4H2U2O7C/9HmGKu388M5Rup0P3rAsfPNlRqADgWwXqPoztEYX9M/tUAL7ZBMGdh/W3omN8fHMzl1ImaRpNNKSUSZJJKQlhxkft5Ma5WisVezFpkhvjOuUA4OZ68f4vPlzc3gSvX3/11eN7s+lkIlJGqMwL+trr7/1e6x+98e7rr77GAv354K+ePH7a1C1w+fqrr5+8+uoHn37+5MVF11ZE5sCpdzaA4yQY9N5vPRGCt6rr1+ul1j3nD6bTqRAi9iBWq1XfVIkc+mCD83GqHhEpZ8T54BEgBMCtBBIAYtg//k3H/tzePcn/Pxy/Dh196yP2F2sPEsQFEwG2fStw34OAHSBHOd1d94CIaZo65zhlfd/7nXL53Y/bkTm0McZaHW/5LEtms9nJycnB4XQ+nz958uTs7Iwx9uqrr1xfP1O9OT+/GE+ns4NDZJxK4SBU9Qax7uq215pTupqvV/Nl3WxKx63djTihK8t0Nh0BkKZpYxuLUo4BjNlymARl3nvdq7ZtKYHxeOytadt2NDighAZwu3Q3Uoa5ajdCkMFglKap9e729uz29rarm/Gg5GAn5TBNpXH6ZnHTNM3R8UHG09uLG6c0mIA2cGTNuqmWq3I8Ibvz7IK3WkWEsmmaTCajUUkpWywWWnUEg2C8aTblICvLsuvqrq3rug7BlWWhLWrru66r1nWvu6ZunXYAJE0SDEFrDdaaTjmj+rY7OzvDabq7GSMkCd57wO3VtNaGgMbZPStFEGKdbtu2aTpjbJrkaSqj+ohMuDZ9ALMVZQ1WG9B9E2OHcW6Lr0IAgCRJAkG0OxXO3S2/Wjdd1zVN06vWhjhSYax3WZZIKUUa6fadEDwCVPPVxVZmF5x32GkmQDjoGE9kMSiNZSJNrfJM8sGgnE5GnHPGsGl6QDsa5AHJ6mZZbW4fPXw4HA6fPnt2c3O1XC8eP358+vDB7GAyn998/PGHq9UKCNZ19c57704mkzfeeOPpZ88lTUf5eLVaqLbL0vT05PD03r3NugKrjNKq7zmBN19/NBmNP/3009Wm+uHhb//Wj39n3dQhhCQRk8kEKHk3z/teK20ByJMnT25unhHCjO6vb5eU0uDROSSUF2VGkHnvO6ViR9kDOz19IHj2/i8+nN+upcyMMYz6gGCtT9OUCWqcBW+RMiQkJhBJJoXMmWgJCZQRzjlj8dJut13OudamaTqte6VVNNNDpMZGovJWEc8HJBDvBKs1eM+894wRRrkQQggGVuz3DkppCLFl7qNMXtd1zgXOZeQndp39/PPHb7zx/D//z79z//79X37wUZqKohjUTSOE5FweHEzrum0an6ZpXdcAUUrIxf+07r23nFMhmFId57zvrVKdlJJSjD/tjA+UEUKIDyZABPwtYG+dJ44x9D4454gPiAiEh6Ai3yryOmNzIbpMzWaz5XIZfefiAEhRFOPxpG/VcrnedROC1rau67Ztx6NBWZZ936muKUMcZNiYzgkh0jyjQlASrO4YhYPJuG6att4YpTORpGkK3mutoy5xbKYwgkqpKNpIKY1ZBdzRA45Dg3d37btV+L7yu1sjIqL1L40l7zIn9pQ02A0uxoAtJY/kj5c5CjhE7pzZpZHgwCEgQRYZ4b8eYOCOrsPuK3xD0GLfwt8FFbxL6YCXWsLbmZR48mOzxnoXB48dhAiNxmR21zjj2hjvAQHWq+bnf/XBkydPhmWKQDvlq41ijDEZta3K45O3rHPr9XqQlt5aq5VWRreGov2Dn/7ucDIsPvz46mZplbW6DzYQFnIO6MEaoOAQnANndV9XVV1VnFJvdVrm2eFsNBqUZd62bVNrb4MFF7z1wceMEOiWHOoBPWy/JwJGycTfdPym0P6/5vhWzvGtVJIQEvtu+2wyTiT1fb9PHPfk3C2HF7cmGmS3aNM05ZQxxrRS+wSR7OTOmmazr0kAIE3T09N7x8fHb775ZpYncfyq61rOWUxKRsPp9e3NF59/XY4Wo5sVE5wlCWXs3v0Tp8EZoJ5a7YyyHOUgI4T0zvWcc0p58H44mMxmh/WmYRQ3FRhjOWcBiAsao0YyjeaxcZo6WGvBe0rRe9+qzjmDAQCD0co5naYp44EyoNxqu2mapq2XGPo8p8G3TesSFpp2/ey5++yTT5tNnebJifKXz85V1YXeUk/SLHfGeue8c+C2XdfYV42lFIOg+naz5pzzvm1V3zurKKXeW2saY1TXtQEco6Lrm/W6pglVxlZ1R/nGWm2NoUg450bpTIrJaJwlsq2bqG82PZxe+W4Xvrd7SywGwsuRZhd3ktitJs56B4QELjJKZFEMptPZYDBgVBhjun7d9cAYRcQ4wr2vdrbcHdgmBIQzcHAX2owVabs0UbmnbVvj9H6vSIt0Op0Ocdh2fad6LgVjLCBu1IIQIqRA5/q+X7WGG96oht3O152xnEkPsNxUbbWyVjNOxuPh6naxWlVCpknKFqvK2vbwYEIImU6nPoSovoeIWussy54+ffr06dMQgjJaSvn2229PJpPhcPgCz3/7t36Lc/75Jx+/eP60LLJ7hwdpwvran54cCM6//PLL5e0iS/nrb7zaq/ZXH33RKH1y/2FejpfL5Ssnr/zRH/09KaUHjPre1tpEpn2nvPdPnjypNqYsS8a40cYaLyUVCSHIGA2YouDJbDYzNvzV++9/8dUzKYl1xHvPWAgBrfdMSGqd1toFzziEgNWm4ZIBKQGAIGNCCSGkZIzFW3EbbPq+BiCMIaBtmk3fQ57J4bC4utgQQuiuXHDOIaGwqzhhJ20EO1BuZ/hG2M43LMpu9H3vXNA66ruRnaU9PHt6/hd//ovvvPXea6++cXT0fuz0j6ejNE2Dx+lsbK3VWgsxWCwWgCTm0bCdH9keUW1zPB7HPyb+DXE8AaiI7VNOCBLiQ2jaVmnrPDgP4EIEA0J4SV+IYTVNU7+N1jaC5OPx+OrqKp6HaJ01nU6V6btOrVYrox0hNMsSRlnf91pr70zbNkp1jMBwmAvBCEVlG8J9AMYoG4+ycjjquo4GT7yjAQTBRAjBWNd1fd8HH8dNAuecIWjVDQaTrtlEZbA9qMsIS7KMEKK0DdrBNwv3PWK83+v31w4AtO73XJCIeeyBBEIgKjx670PYflYqBSUMnFdKOa/QO0QkNFZ+CBA8BEQKCAE94NYj4C5ysI9A+78n3OH53wUS9s/vw9XdLxVfkCY5AHgI1jpjlFKmU5pz7j2QqNCFjFHB2Nbl2SjnPQiRCEq8U/NFtV5XiWTX14vDw9l4NmWMUM6klAGh7zSTbZakB+NJs1y/9cbro9Hoq6dPGbi2Wrz66JRJ8cFHHz99/LxWDQOeyURyQklqjHE2KG9INMC0RhtzfXlBCEGwRVGkguMglSx06413FpwN3gMJ0egieATAgACR3AsekW5P3l+vyPztLOHfAwn8+4+7F+VbsMS33nB/TXeA2dYgPOYBMTF1uzJx/5rgYTcquW2rc87jyIDmnOxkXknYWoGg0pEkxBjJsuzw8PDRo0dHxweEkPV6vVotm6aJyF/b1uv1kjPGCJ/fLK5vF+azLy3g4fHR6YMHeV52XbOuVgCeUlxdLcGELEmnI7dELTinlHsP42E6HuZ1XVEatOmbukvSwPmWB2CM6ZpGax3paFp1TdMwgkWR9arVXeu9k4IhovUWieOCcqqM9doE54xVmjF7OCvLPDdKow9SMmV13TXIWT4sWZZ1TTe/WrjggwPwKJhsrU5Eih76XgekhG11aZ1zba8ogtKq3qyTJCkSORmUSnc3Nzdpmi6XTdu2iFAOciZk6DulLaHgNi2jy8igSpNcckEpSSUfDwdvvPawKPOnXz9+/NVXbVvHvWJLQInXi+A+/4t0DUpp9LvPsowxVi+vkySblZOyGCZJSilHoCEE57Wxqu8jgydSpK1zjhIZ8wMPYZsJE0REY7aa3yGEmPpHerjpvQuOS5bSjFuhtVZKWe96pdquS/IMKSGUOw+IRGR5MKMIBmutzWKhrLcBjGfsel5Z50sumk6dnV9fPH+SpeJgNvqjv/f3KGfj6USK1LmAGDinjEOaSgA/O5gg4un9+z/+8Y/W9UYIYb05eXjCGPu//Pf/PWI4PJxNJpN/+2//LQXpwQEgMB/QdmpzeauDdQSRIRkUQ0JIXdePv346Gk+Dxw8/+aTT9h/9o3/0d//o73/44YcYQBsADGmaDMpxng0QA2fi4ODg6upqvrh9dnOZpiml3Fjbdr22JguBctkbbYyzMkxm05ubm1/88pdJxpMkIYRprUMcovRgnAENneo9OOYJkqBMhzwABUTwxAtBhCBcAGKw1jqvQwiI1HuLSIVkw+HQe9hUTUzkAX0AF8K+jt+KKXHOGSWRawkOrLZKGe/9YDQMYTs87x2o3hhjEKkxsSVBEbfNaYI0SzNr+g8/+ORf/+t/83u/95NXX32965rF4vZgNpNSRhEwxkmWJUUx0sogJWpTa61C8EJwKYX3LgRfVWvvHSIkiXQuj8IDxmhELAdbJqMPgfEkSYWxCgIwxgl5iavvA2ScXMjzPPqexQHINE2j68nNzQ0ixjQihMAYmy/b4FGKFMEY4/ZyxoPBIE1ErDmSTIzGwyRJNvXK2eboaDIajb33MqGz6eHFxUW3WW9Wa68Vp5xT5r23xtmtJOfWKoIEv6/y93dv3K+jGD4AGOutfymRdDfE3o2ve4CBUpokkr70mfRh6/gX6NZEKcaJsJ+eMEoTiYwSlGI3GtNuHX1IhBQIkAiZB0Dw/mWM34ecfVzZpwv7B27noXU3Annv9zM4cCddAICtUhbBPYgVw9Xl5SXfdaD8XWUI4wWnhFIbHCKhQjhj1hvbq8Wy6uTz6yjjSAl3wRtjZodhVJSr2eGDo+O/8wd/+O677/7Jz372/ocf/MW/+9O3f/iDXDJvTST8IyVACVIvBAGg1qodiu7BeeJdLsVrr7/KGbRtvVxeL1fzqlqjTyl4oD4geIQQ0HnwwSPZkgc9xqj8EoD5a49vBXX4X9F3gDvTKPDN/AB3fav9NYpLcb8a4/knO73ReMr3NMYQQpz/hW0nZffmSBhjnLE4S4KIdG8/TQljLE455XleDvIsT6y1NzfXZDtqxPq+bdvGe59lWds0s9mMUC4SWdVNo3QiZJnnNxeXTd8o1SeCAXivTSZkKiUjDUeUlCSJRMpHw2GWSEHZul+3bV1VjdZGJpn34FwwxnSqzfNcJFzrPs/zyWQcnGmajbU6BM84kZITCs4ApQHRdarGAJSh5EQyZrUnYFS/oYCUUMRgrebD0WtvvGmNR55czVe9sciFA+oBtXXGeSETyqW1Vjct5zxJ87jy+7YxVkspy7QoimIwKCaTSdPWwZumaZJEJomM2dimmreNFkIsVnUITvWGy+RIJFlSetuDd1abzXq9XK6l4JRSIEEZ7b3HZDuqHVM+Ep1iGPHO7xcAITS2mWLxRghIydNMCi6sdU3b9p2OZaS11gcLGhBDvPKOvhyfjkt9i5pFnS7Y7gDGmE4rpZQzARGRk0RI7zk3jErKDedcmgDGhTwvHBLtHHrIi4G3edT2ttYaX8YutnOOzdcNOOsdCa6valO1WxaeBzIcTIUQi8Viva6m4/Hx8cn19XXbNldXl8bZuq43zSZN08Vq9fDhw9FoODmYZVn28NGDLMucs4xRaw2T/JMvP1R91/etZ7q1xrZtmiTeusuzy0Exmg4Pju8/aJS+ulkPBqN33vve++//Svyz/3E6PfjVh5+qXs8OTn7605/O5/PgqfPWGDUcDn/rhz+4vr42uiPFZ0mShIDr9Xo+n7e9VlYTb7U1i2Udwurw3mFd1+tqdXBw0HUdoUHrHgC4FIGAdT2hnrBACSIGoMAkoRSs14wTmTIukDKy46ttucoAIIRQykCAsizLYrxaVYv5ar2uOGeUkgj/bGfEowWwsp6yKNAUgkcExigAZVQ4cIQQRrl32lpvjKMUnNuVs0iDDx5CTGmLNKs267/6+a9+64c/euONN7788tMAjlJUqsuybLNZh+CTJBkMim4yCQg3bTRbbgI4yjBJBWW4XFWUYd1UIQTKENAD+iQVjLE840qptums6Ql6a701PaOc0cQ57Tz42HswBrynSIBuN744BBGntmI/Pg4uxhwitlE2m03TdYzxg4PDum4vLy+7rsNyENsWaSKc000btfEju7MTKX7/t9753nvfv7m5+Xc/+7O2XbXNmhLJCZGDUsqccamMic0/ACQEthMoxsU/IOYosT+y54tFXN3uBPn3273f2UnczRtwRy4jhFDGXgbRO5j/3do93CE3WN0ngglGCBcyS40xPrhOacSAGMCjBw8BPGIIiAC4Cyp+p5D9srj85uP4ubFM+Va1+q2Ydzf7ic1jyhnZDrITBBI8Gu0CWoDYX6OMCe/BWovME0a8t6prGYE8Sxij0PVcJK12Vbtx3gNllAlE9AFx3pbpWPXhcHby9pvvvPHotbNnL37xy1+21aar6nWvNqsNpVQWqTfgA7XeBEIsIQ4QKZVUUEDvDPH+ldN7/4f/+r+UCfn6q89//pf/Tlcvag2UOw/eBfCEeEAbCMK2vxe2Bk1hb8lBts7Uv/G4m43tTtTfPD34xrn99QdwB/XBnZg3vePJvl9yYSdWFn+0TyxC2JtDhbviXNFEEgHirDUism3vi1ApCIVYFMaatevaqlorpRjb5rvGKmMVpTRJ03FZHJ+eWBfG00nd9vPlYtO2N1eXSSqdc5IRyai1NpdckJxR3Cxa03uBQBKeiRw9Ou2kEIgkT1KTuwDogyWEAYEAfjo5ePDgAYKfz2+ODmfvvPP2anH7s5/9SbVeUBIYo4QCIYAckXhje0oxeA8QKKEEgQZ01mLwxngui4hkCJmXxWi1qp4+O+sWTRMArVUEWVF6wQUVIskcQtt3fd/Hmamo6N80TUFxNBqFLG3rtepqAm4yHb399hs/+9nP6nplfSjLMksL5/xqtdbauwScA+9hPB6PBkMGGByAC87Yy9sb3XfW9IRAORgYp6Pqy/7ShxAoopSSM9H0KvK0QgiU0D0qORgMGBNa68VigcCcC84G5xxARDF9COCDjVsiISTQEHODAGHfWENEF3bNCAQfvPEuggcYyJ3uGwmMCJrwRFImIrTgCCDjznmPhAvpgBmgBjIqaTrglNI4Y8jOzuaq77KEJZJaS6aTe5NxPhkUq+WmXtej0Ugpq5R++GA6mx12dffp519Op1Pj7OHhYZIkf/mXf/ni/PzRo0dH947X9SbLsldfffW1116jlMaYUetVtamcVjJhWZJ4SyXnZV6s5qtASavVmJDh7IhTiSIbTY4nR0dEyF9+8OFserBcbZzx/9P/+19tGnX/3nE3UnkmN/U6TfloOBgMip/+9KcP3v1B27ZRDv3y8vr58+fXN/O27fMiUVZb656/+JoS/ujV+5FbJ6XUtlPKMI7WWm0tYSAT5hECOClZXmZZnhBiAZELEqJsiAd42SeO3HjqvQngOUuTJIPAnSWct02tENGZl93rePPTXcm5Dz9RwDWGhIjfA9g94WX7WWFb+e27D4hciuzy8vqzzz578503vvjik6IolO7qui4Heb2oKJEBXIyLxllKad/3xpg0TU9PTweDwYMHDy4uLm5vb+MggJQyCkLHVFp3FUHMJEGkqSSOgiBJmuZt2zprvHUAhARHg6cBibcOaFSfZIxF24WIhUopx+Px0dGRlLKu65ubm8h5DoiM8Dhg6WzwDpwLzrm+b4s8SZK8VzWhaK11ztRNJVMTwBalzNKTq7PTm+vl7cVVcDYTPM1KwkTTm7btIgBjnZOC7kcrs1RuNpvJg9PIIc2yjAvSGdN2KrTtNp3H/Rl+OalIvqmvvN+1I2LknI0ksgAeSYisFR+21y4iKIwzIRildChKznnXddr2zJDtCvIu3ugevPfgEdCjR4OILLx0u95nJPu0YL8B7Z/HHfHtbsDb5xnwawB4vDqwq3T9juWfJInHLesCd+OpiBhAI7rgLVLwELTVjFAquA1eG2t8IMgYlYQKpIxTWuRiPDrGEKaTo/HowBkYlqMiySulbq5uLxfLzXojWQqBGm+lSLVE7xx4whOalzyhUvVtW60ZIyknbz46PTwYPDzMC9YOpf7yy8+fXIKB4J0PznlkAUOs1QNsrwSJ3QfvCEDYMhX+piH/b/jKX//FX88V4E7u6O4ord2Ff/xOqmsv8gF3couw0/fYLsgQwk6xMTiPiGznxBZCQEajdrunW9V2ussDnYMQHCHQdZ11Ouza9vEeefPVR48ePTy/vJKCaA2ckWazWlXVwcGBD5YTXIFv6w0Gxxjx3rHgJSsYJMFS1frz5zeULT0ip2I2O8wHw2pdK6MJQckFIs5ms7Isg7fe28PDw8lkorqGC2qt5ZISCsYowBC9z41Rw4FwXgfrdAg0+OAtQ8IpmwwGicwH5SwvxoKnPtBNay9ulvP5fHZw0KoeeToaDlvVB0qQMxoCMTyA1kpFfpLVzlrrGK1WS04JE2xdLV88//rRqw9PTo5nB5Ozs7P5zdxam8iMc45IlfE0pUgdoyIEtNqpYBgGSvDg4CCRdDAsOOdt3zRNs2mapml8KvfLIF6IoiiKvCTVRmvdd9p7z3YGud57kZaEEAjEmuC9gUAo5UKkOy4CAniECOWG2L/abg53lpnf5QIOQvDBe2+c1dZ0qudUbBdVQAAAgoQwJMQG7yD0xoa6NcYwJgjl1gWltXPOWBsHd6NZDKWUXd8sV8s5RRgP0iyh4+HwYHYwGecvXlzW69Wbb71+7/gUgXIujTHr9eby8nK5XD4/e/EP/sE/ePDo4Xq9Xq1WAFAMynxQrtfrOHVaDgbPnj378z//c8/0eDzMi7Rq6l51JHjO2M2C9U1n0XdNffvRxxST6eQYIOmVv3d8+oMf/ODP/uzPP/r0s0xk4/H0l+9/sFpWv/vjH73yysO3v/NmvB+stVVVzQ4my4AePRAymYwfPLj/4NH9y4vruq6fPH8xHI8JIU+fPh9Pxz/8wW91XRe2EmB0uVwSwqp6o+pWG09E5p0NwYuECEkoDdbZeLICvCQfxSACECCg954gY0wiUu+AMXF4cCyl/OCDj7Zgwp3Cwnt/fHzMcCty0Pe9t1vo3nmzo8SZSGvqe82Yo5R6t+1ZULq1RLLWLpd1kiRt3X788afv/eDdvu+LIqvqSqk+uqJxJpUyVVX1nem1iVtJtJJ69dVX33rrre9+97vX19fvv//+n/7pn7548QIAohl8BLsEV6PRaDoeDgaDshh670PAshz88v0PvLHBASHACfPUI1IhmCV8r3AQv1E8omfjw4cP+75/9uzZV199FT3GjA3WbOKcCCLGokcpRSkmSTIcloAWvEtT6YPlnGvX/OIXf9G3zffe+/73v/9dZ6Br2s8/+UJbAKRAzLpqa9VTJihnAQhAiNMc1lpKs2hG53a+O4zLTddt6tb1vZQyMuHupgVh12vYX+/91Q93GgFhJ/YQC8T4T611fHEUb4/Axsl42mvVNE21WiulCGORLRGHIr33DkK0YIqfxfAbOco+nN/96LtHpLbsM4lvQQvxt76ZMZDYQAUIsUZBuk18rTVbcVJKkRAIBELE0iwTVCaZUVr3xqCTkseZeMYY4ymh3DjvnCHEAVCj/XI5V73VvblWt1Y7BuzJ48/nm26+aZZVWxQTpZTRYTychoygMQGokHRcDHKZVIuVbpuE4Gq5OD97JsjBMGWvPzypF/f69dXz65W1zhkwDiyYQIlHDoTs2nfgIeydBQA8/obBh7udgrvn6n/zY788/I51uF9vbndEhfW4nParbot7iS0HiALuk78ASCmNiYLfDVtmWVYUhafbTkeUC9utUjw/P/feI9l6iVmrEVEIFoWxm7Zuuqaqa20t53RQZASsNXpj1Ga92qznaSIHgwGAn5bjPM8p4976TdtW51cAZDAaiiQrBpnMUtWbum0I80wwQokx5vz8nBKQkldV9f77719fnm82G8qiyBsarQE8IwwArdNNYyiBhDNCiNNGNY3VhiIZZiNrvbVea7vQaxfam+vFclWDlEcPHlzd3hhnRZLUN9faGturtMiBkkikc1p76xgjeZKTXpleIYHpZGR0++zFs2qzmM9Pf/KTnwyHwxfluQcEwBBwOJ4en6QX9ZwAFoMykVnwUTYehRC317dJyqNkRVWv664NIeRlsdbq7m0Yu66DwcAGaJpGKxvXwF7LRDe1FGnsiQePcVqZIGPUbr14IFAaDUF87CmEb/YigSAAcM49QkwYXNgK3mut/VZtliKBEMA7p3ZN7YCgjemVCgGHY0kY7fteB9/2bWuamMekLg3MU0rZi7MPppMJhWD6nidD6MjmejmT+fPPns0m00l2UN1s6rapq68ePHwo08wHoo1//bW3fCDVunnvez+8/+i1k9PjZbVqulaZ/pU3H/GMPL/6+v2Pf/704ovgTrJ0lKRio1qkNs2YC8pYLUcp6zHLpa1Du+5MXd+8eL48OxPHxfR4zNLA+9Db9nZlhgcDZfpN23TK3c7XZV44y5yFYXkaLJlhHxz29WZtFAhRTkqS87pXfsCrqkIkD6Q7uXevGND1enV0dLQ2yze+d+L9cd+q+Xx5dXW9qZrga85YURTDspCSh+AIeMY4peh2ri1R1sn5EDyGEFIuiVGcE8pcCG2WiTRNAGCQF5vNRmkTjEVA4jCVyWw8eHhwfHx83Kn27PKsbpv1erlplznLC3Anh0dXt3OwnKFHYqeHY2cRCHE2XF9dE9LnSToaDM8uLh6cnnYmVd4aTH/5yVfjf/mng/HxRx9/8PrrrxzfS88vFpRk3nGrQ28VBBYsrdpOJslicZunskzYe68/TLx+7XCiHj24ePyV7/Tz8+umCYRPehscBOFJ37Hf+dGPf/D9N4uM3FxfnBwdv3hx8fyLL9SmQ89FMmytbZ1BQsDDgeSQ59775XxeluXRwcHNfB4h1rquDw8P//iP//h2MR+MhlyKs7OzVJTee0BNKS0yQSkNYBj1iMb5jrP8/sm9EFwIGDxiINCNNlfsf/7kz0g9+J0f/63j6fCPfu/3p5l4cf6sVpuPPrsqJ6l2GCAnOGCicGStlZaMJklSLStiw6wYQ++EQW7w0cMHXz9+ZrTOx3kgDCnB0DMKcdLVQ2CUEAoeLKOCIfN+G3cIpSSOOwKhnBK6db+1zkUcOEk5JzSqSQJ4Z20m+Hg8HqYyUfDF4jrYsN40VEjrAAnzzgVCKeGxJvQQfBRhZSRAnJDfbgfW+4h0xOozz3NltDEmz/O6bj3XEOWZt2FpnyjE9ijbBZ5dVwJMlGqIBSogZZQzxrRqKCGMIQA4p73xAMARWcKNAa21cg4RWcIBwHgEoMijuU5LKaeUgrW91mfWr1fzn/zoB49eP+jDAr2b3R+2oV13zfMvH/NsoB0u63WR5YkMwJrXRqfn5+dV06dZgcreLK7aumZUbJpKZqWy2GlCGEuLg3fe+/FqpR/VHz19PrcORMIahcoRwiQSQila1wejwTkIhAZOAoZAPe1/PQdAhMhKvvtMPO5iS3/zBOJuVbB7w5dH2LWNnXMYgDHGBN+zgAOCMnqxWnIpAWDL8smzOCIxSEsmCE/EPodAH0IIUkrV9cYYzrlkXAhRFEVRFJ5Za22RlSGEuq6JoNb452eX5xeXqUiiAKsQyXA4jJ/11fPmqxcflaPh7PDgeMauFzctLKxUq7aWInWgRSEPy3tWG+tDkec+593O5IUKl7K867rbqlc368GgYYw5F/K0bNu20e1gMJgvrjjnQohO0cVqabVRSgHN0jzzxNlgCE85p4SjtVpb6jSkaUowUS54nxk20KbvOmUuzWiQXm9Wr7wymoyK86dPLdE/+sm76xZNgNnxEeXEescL0fad8S7Kti7myy8+/aJv7XA0EpRXVdUENxwMfTY4X/XPr2oDQ6LFL375rG7le99/9+Hrw6ubS5mL8mhwEuxoOnmtqimSrtmUMqXB67aVWdqtG+o9KLh5frm8vq1VEzSQIBgwE1SUqoswAHR90nYy1VKkg3LSd7bvNUVGAvPWEyQUi+Cps5QKzsXWUUHbbmvXR4EA3a0xQgjhhDZNAwQZY9ZaJASB9H1rjEFGAUAZ3XVd2/fWeyoolUz3SneaIHJCKZKUSSEEAgMgvfZd75BK1QawxgNrultElELGppXqm5qEAI4djEfgTd+1KHiWTo8PxsS7s/MnPiiREhu0Dvp6ea2c1cSGgH/n7/801mez2Ww0GRtUxnfK9oyRvtvczucYPA1etd0gHbzx6I1ew/ERT1JvfWd9KznP81SKnNO0XVpdQx+sVt6GftXfWGVzPnvttdf+8A//8C/+/Odnzy/L4WA6niUsQULqprq4gE1RHB5MsywJELyHw8PDrCgCxdvHXy5ur1mSgmDOmslkMp1OVddnXA4HA6QkK3JKKbGMCsq5LDLPuSSE5lkdnItK6ZySAN454xwSBAQSgglbUiEiEooAlACAUYoxNhwOB8XAWu+ca9t2vV6v13PvfZKILJXee290QO08jsbFZFoELNJSOOfmy9vHT54sl0suskZZIdMQAkKYFslkkh8cHM1vl4+/fnY4SOpGWaO0NZPZwe1qwxiLNpBt215cXExnIwC4vV3cv3/fGKeUJhiMcQAEAULwnPPgnVIqE5xzLrnggoKD2Wwym82ub1eEXIcQEIBTpJQKyopUDgflvXv3ZpNsNCyOD2dpmv/8rz6aL5q6a0F32theacoZk0yIsmmaWNMgpW3fR1pAURRN00QrbdX1iEiRQOQbkuB9CNYZ76wz4IP3djQepEIWedl2TV1XeZrFbkjd10obJvnl7c2f/OnPvv/e248ent5/9F/+s3/2/7xe3rzt4fGL68ngMC0ONq3z3iutjdKeYjAagiMEAdFaXbcbXice/WQ6Yql0SOpeMeRh1+MnhFC6HY+M8EAMEgS3QhS4my3cF4KxOGCMEkLSVILzcU/03jprjTFt2xYEjDHO+xDAhlhFAoEonfBSjREgREvlu4wH95KO4BhjveoASJy5hxDABwQPbgvhxJp1T2UgEBMYSwgJLytsCOEbjgN3MfA9hgF3lJoALSGUELpv9sdEOVJ3YwoCEMd2SAiUEA8ASZI450KgUoi67Y0xw+Fwfb0CgOl0mmQZhuCcOzg4cBBklkprjLOma7RWdd+1zWY6LFvdL6r1weGIaoeEDMfjk/unxbOnIcydg9GoSIpkuemVDTF6xq9DGKNAiWckoHfkN5EO8Dc8/x/62FUd38B7CCFKqU6pePU559Zv5RBYSvM8T7PMO7dH4KzSGKVOKYv4gWQ8hNC2bTkuOBVZlkXpFAQa24LGWKUqAMiyYjLk+07o5dUVAPBEHh0dlcNCWfX48eOrq6vJdKS8171yRgfnu6YN1jnnOoeRURTdzvbfwnsfhzzjBEdck0mSZHxr4x4bcxGy4pyD90iAAnIqkpQxRnrVWWu9Vt57FywGQEYEoRQlZbhaLw5nk4cPH37/u987OTn5Wz/+7Uij/h/+b/93QoASyTj1xgdwxihljfdeiiTPy8FgtF61i/maEOKN994yxlKZGDCIiMExioOi/OyTT9p2gyTITL73vXfKYXF+dX7x7MWmr9M0DcYIDCwE75RgKZeZ1e1gkAvJqmazXq/avgESvCeRNUIp9Q5idljXNSFM8MR7zzmPY7BxOJkQEtxWID92gmKi4HZylvtlcwfOdIRAdNMA8HYHHngTtrwEo40x1nsfrI+GW5wXAQiiN9pqZ4MlloTgkDJCKOcsEPTetn1rreudCiEEBETsuo5zhpQgInvzlftNuzFt4p2xqrJaZqlUfT2aZtmIVd2CJoKkgQay6tfj6WR6Ooh+gI1bUQUoTVoyRF2UcjTIMilms8Pjg5P1cqUKfX+6SUbm9TcOkxSvF7ra3NigATSnbJgPbGeMciA8SKeV6U2o2zqnh7027373u12rptOjIivytHDOzcYTRshqs1KqSzPOGIYQDAkuYJpl08PpaHFbd60LLljfdT3hbDKZ3lo3HA2yJDW9HgwGlLOMZpRSKVNGeJoWeTbo+54EIqU0Vule9X2rjXLOYIi0ZL69Th5C8AEICR6ASJHmWTYZTaRMlVJG6WbTLucrmXDJeJJIzliwzlnNOc8Sqfp1r3IhRJnzrBzPDofW9pt6qQJft3qQpV51HN1skr/y6OD3f+8nn376+dcffsETyDioEFZVPTk4vl03BQZChKCiqrrz80suGKPJYr6+d+/EGNs2PYD1LjAmKEVrbSqkM9ZpF0LghCIiQ9Lb3lubSCEkC+BDMARYCAAIVmkoJCIOBoOjw3GRsmE58Da88dort/PVetO6OIxre2SCIqWCevQBgpSSC2qsirzIh4/uW6fX1dI6zXikAWKWJ5JS7zFWz4hxTtQTQlIhGRNlWdabqq6aMh903YozmY1GzrnxcDTfbH7x4a+enz9/7dHJg/uH//v/+h9eXV2cX179D//Xf9xu1vWqAcjunTw8W9otIBECZ4xmKU84kCjNAzGfU0rLPMsSERDVLt4jIqXbm9N7z/nOmwDI3c097rDspaJRoLiNuH7ndmGtNlrHpsAkkxYCUIKASG3Y0pO8gxDVFbf3f0SUAWyEl++MNsTgkSQJIaRpmxBclJGmFCfj0Xy5CtGqDSH4AOAJblXwAJ0PFsIuIYi4ZHgpKrUPVHcDf7jD0ieEeI8ISBB8sBGEiGouTdNxTpEE5zwAIHGUUi6QUgYBpZR13QoMw6Ks6jZL8zRNrV1xj5PJdDAaVqv1er1yLiw2CyFEORpu6tbbkBQ5EPTepmVRNfXVzfX90wPnUVAYjkb3X3n06Oxssaq1Zq++8R4Rgy+fnH397Gy9qQIBHxx4v1VQ8D7OkP0vSS79BzzCNykL22d2ysHhjvJSpMvEQBvj316WLe71RVHEzDXLMkJI45xkfDKZCCFiSysQ9D50WmUuJVFw3UHwUb0ycC5OTk7atm07ZZ1bVOtG9WmaSynL0fDq6ur88uzR7cOsSDnnglMuaNu2GLxzTjDGOFUMkbF8mK8218ow47TQYjtZg55yQgwG9NYb593O8AldsG299aGNHRZnLKHAGReMBTDeW0aClEmWiaLMy7Js1ou4FCP6gkgYE4xhmorZ0fToeEY5Wa7mlFIP6c3VbdfVhBPgQAMJNABinGhbLJe90t6RttcAxAWPnhLG84RJwfq+b5vGam216VyjdC85b9aVst3scJYwXmappGzZNtZtAvfgvbPorHNdpzPGpDg4HJ+c3KMMv3j8hdIbQM84M94i8sg1cbDVM+i6DoAIrr3fjrfgbqKBUiqYIHdm6cOO6Hp3zdxdQsZqH6z36BxxzgHZiultmro3uu97ZbRzzu8wrdvbW0ppKhOZ8EDQgwMINljnAkNAgkRQQKQU43LUVsZfD95HdXfhLCGECeoq1aYCB+VQCnpwkJ/ePwE4efz4sXb1l89ujh88YCWTQiyqjbDicvW867qqqqx2RVEM8gETFAOzpp4ennrr6uX6rHXz+eLJ18+efPVUjDepgEevHN+bHB3PRk27Wa1WXR+W83o1b/sWnWUWaBCMCZJSwrh48eLF22+/++Of/N7ydvnkyZOu6cpykOSZ00a3PRKvdaeMjASwuu1za6x3RZENR2Xdq7rvbN9JzIs0u9Tncf9t+nY8GCqleCJDCNZ6II4QUhTFcDhOuIik/QYq7y0i+sC9Nd57JiDS/F1wGD2sgUCAwahMk4xzCT4wwrNhLmXqvV8v1iEEDEDAeTTIvRQhSel4miUJ+KB0r7ggQsosTxgBJkrnrBBJ125c11DqCtCvH09ORz/8F//sf3xyDfkAGE3qdXu7WslymBCLJBBCkyRp276p2yTJjDHeEWug7621ffCQJIGx0Ku2SDhQGu86a23XNJISZ+zi9gaDSzjjlEjBkFPba4TQtn2Ri8268kYTQgSjxijv7cnp4eDTIgoWcwqcIaHAeCCUZnm+XcqIe9ml09PTvu/bTU0BM5nEkjdP0m2XDhHJdjyPEYCoW+A8I5QSBh6KLL+9vqHIRMHBe5SyMybJCwv48eefG9e/8tpDzvmwLP7wpz9dLZuPPvpqudjY9dxZSxAFZdoaJHFImHkaCMckSwknXdd2ShVlCUC2ziuE7PhleLe2/muPONAhdqK8Wivdq31BsBNWshA9tyjtjI5TBsgIsRaBBO8DQWMMBI+Iwe28HkisieleJCgSkBAR0C9XSyl5maVFkUXrLAxmtaySnZuls875rYkGBo/BA2JMNhAREGKVQODljGt8EK9CbJCTXRjbAxuU8jiTrXVvrQ2BRIlAKUFIFrWkjInGJS/JlZSwuq4Tguwe41zeu3fv86cX3sGW0g/UubDZ1FdXVyzPDg+mQgh6dds0XSaTMB6lRcoxNPXqejnvjAYEKimhtBgO3njjjVXVbjbu8PAwLQ+oLICKs4vzqtlYB05vDQXCjhX+nwo5+OuzhC0G8w1EIV4RKaXZGapxzj2EyHSZHI3zMk/zNC/z6GfT971SPSJs2hrrcHt72zRNURTT8WQ6nRAkEKBrlbXR5I9Z7/OyODo+6fu+aurlcr1er5teAWUiTSAEJmj0jTs8nCWJaDcVBrdYLJz34C0hjLOtbZVIhLpVSqm9R/xOb5R672OJbIyJqyj6F6yq1cHBQTkYIRCAZaV7q30IQTAWtWi1sz5YxgZlmZfFIBEcALTpTa+s1QTBgQvUUcD56tp5c3193dRtIuXx0clisVBWEU8Ceg8uEAzghGBMcB9wtdos5iutXZqX00kGgM4GwRsEUm1W3oaDg1kipLd2s64APKHgO63b5vzFc6WaYPSgLKjTmeROq4QHQGeMJ9QB6CIXR4cTwvDFhYgevJSRvuu13yb3cWwtbizOuc520S0IANnOtp4QkogEd1zpeMQ8KRKevrURxXoAIGIVyjnHueRChBCW65X3AEAQqHfOeMe5EELe3C4BIBFdlkgW9WR9oNR7D84DMk+JpJRSTqiQlFLTBGsti+o+VHkIxjpEz371q1/e3tjTe+T+/d9+5dHJ8eFkPCoCWI8nrephHYbjzFFqga77xhHf9K022gbTm1YvO/CmyApvw2QwGw6Krz9//Bf/7hd9b7xBrf16sbK3NYdL3eD0oMgLUddwfdGtqp7RvK4MYEKoYJRRJoqioAzbVQcASMnh4aFWtqqqTVUXRbFaLbz31ihCQ6c7rignREqZD4qu6+q2AQycc71aNXVlnT0+Ph4NS0rAefTgjNOO+Ea3Oc+dCyFYijR49B4ICeAidcgHj5E54oNzSJwzlKJH75EEKhCRER47smVWci4F44g0k3Q6PWCMTYaTx19+1XWd7lvnDLBAkApKGLUYjDNdQPRBO2fG2fjRgwdt3bw498BMkiROiL4Jm83m7Ozsyy8+++nv/+Ef/uHvVv/Tn1kGktPZbHJ1s5oOByml0ZugLAeUkq4zeV6Ohsya4CwYbZVSccVQ6tq2RccSweO0glF6vV4XuSQYuKDT6XixqhLJJWfIURskBF0IEEKzWWvVgbOUEWs0wZBKmSVMcnRIGKM0IZRTnkjCcDgehBCWy6WxigvOBNW6F4KdPXu+XM6tUQieIDFGO6u1MrjlPCIjRHDKOeeURRBVdVryJJFZluXBeqsMzTJO2XpdWe0OZwflaPT86S1Qsmlq1de/fP+v3nnrna61q5ulbfvF1XOT8kRKgsF7Z7xhjFlvlNZplhlvAHF6OM17DeD7pvHO3VUO2LmLRXbY1vqBII27YdziY/hnNGql2f2+H79UHBYAoHFP11pfzm+klHXfcZ64EICEgICUBB0AQtTcwS1xPwAAkB31HX3MFJAAAimKxCrdKNO2HWeQJAmlgAAMHAIGBMqABxbF2ZwNSALiy+mprWkjIif8bqCKu3zsmMT/xzJxP0YRt/6uY31Pte7Djo43mUw450hCnDqLgzPGaASUCQcgwcftDxFoUQyc9YyBtf72dhGAEUKyrEAgr7/++vHxsXNuUzXL5ZoCZlk2Ho9N1/QdbZrGex9HxaIv83A4TNP02bPnq80Ho9n9bDA9PT3lUjw7e650o1qilQIHBCkhCIFZNP8h8oC/4XG3HLybN4SdowTiS0+EPRE4hMAFn0wmBwcHw3sD7z2gH46GiDifz5tmo4yy3tR1FS3sm80mmt0PJwOBnDCutbbWhQDKaAgkTXLCWcpKIqVMs9F4qq2JjYDL64u+743TH3/6EWWY5yklQAC9NdYYrTUEH5IkDvEqrWUqt/n9LkuIAY8JVpalMWa9XvfRnAKRMCIEG4+Hjx49YEycn0tnbVVVSqngHGXonNN917Zb/5Q0Ta0NAbxWXumthmMIIaZNi8Xi9mYxKIb1uiGE9X0ghFAurLVtr3rnkBAHgTAuOM/y0oXQtL0QMBxMiqwEB8bYTvk8yzjnucim48mwKK0xXd2Mx0PrdFWtlepYQhA8IZgKicA4BWU8p8AYlwh5yglgtV5W1YonUcnNAqC1plet4zlsaaox6m/ddwkhcbwrXme29eQL+/QxroQ9yBRrj2+smW0nlFEClLHIn/YOuk6vVtXtzVo7a63VximlPMJwKIo8Pzo+7btOKaW0NxiCsyEECjSEwCRkVDAumZAQRWAQdW+894xwCIgBggdnfAiOJXn25kx+773v/Pi3vzccpF27WqyvAfyDh8fK29eoyMbj28X6erEuiiLNMqvWzlhBJYitorhSqm/VK/dfy4q0N/rq5lq1TvBC8rQcToM4saY4e+FWi3WWM9VvlkttTMhSahrHBUuK1NNABc0zwaXoasNl+tVXXyUy6zqVZFm9aS6vrmbTCQZw4IzTm7YytmecFEU25JPeKaBQlKV29nY598Y6Z53VThsAsEZxTjnnyCiTApES4glSToVzoeuUUp1qFGPMWO09EGSEkRAcAUREHywiZYxxyqRM0zRPkiSm4CGEVKSUUkJYKhMAInniHZJAOJN5mghOOA2UIae03qycs1mWgYeuaeukJgGTJHP9RZlngrB0POsl03VVG/zwqxcnr9/+7h/+vZvG/eUvP26te3ByUlVr5hoqCgCvlB4Oh0II74CztCgGXWeshRDQWh+VMCi1XdeEHulwaLUhAZw2zabydgoAB9MZ53y+XAewWrcYEq1akaQUCYCvqqptW2sMo9QGlSRyUKaTyWg6mygXkAtgSAQDgABUCuG9h+VSGxNbdFELcr1eR5VZCigZd9pQwFTyuNHEuyURMpZQglECuFwu8zQVjAUbQkDnAkdSpEWzajmlzrnzq+uz88s0Yz/8rfcIQ6X6okir1SVF8+D08OLiSjkiKOGEUkRkTCSCCmq8OX3l/s3NTbVZSSkp5fPFEoJ75dEjxrm1dqeZ/3LiIOJDzrngX276sWzy3kfTlKh0HGfVjDEEMaKsAN4a07ZtXddVrfNBWXUhDUo5T3xU9QYSb/VdtIilJiI63NYNuP1/BBEDpVQWWWI053yzbpq6Hw7EqOTVRgMAQRACmUgIIcZZbeP4JQAQAHSAJDDEKAH1snaJn7ptZu90paSUdxkMbbcOEGQgPmAAau22s9O2LeccgGjtnEUIIurHOmu2zh2EZVluzJbOEWvl3oX5fI6E3Ts6Pvz/EvdfTZIlWZogdo7yy4w6Dx6Rkayqutg032kMFouewS5kMRAsfhwegHcAsrJ4wMoshvQMZLunq7t4chIZPMKJmRu7TLni4bp75VRTLADBfUjxdHezML9mqufodz5ycFDlxd54SkKqt3W92darTc8YmaWqyANgLqRpOtv1lSo5Y7rvObmicWy328vNmTrfzg9vE67W20361rAmxQQxQkL8uxGFvwc0+v/KdXOHv40o3PRn6VsvAANBRHbNEASAAek8ODg4Pj62zGy3W+ecUGpQILd9BwBCCO30YPJjvW9Xq5BS03WP7zwqiiI6r7U2xnV9H2MsRmOwFhFjQsblwWyOiC9evn7x4uV6s0gh2uX529PXn3/2yd7enndmuVx2XQeQfAwJyulstn+0L6WsxqOUDq5cT7JsWOw3bpKDwVqr222zHeTZyJBCGlI5pOTj8bidNgDJOWe14YQzJglE56zRbrXaMNYEnwDjEDAtOJOMUIo0xqIoulZv1jtrUlvrzWbX1n48Hk9PjmPqtLMxuAiAFDkgodDbXmbZ8cl+DEQwKaVkhKaE04Pjk6PjTObeWIoMfPDW7R8cAEbTdzJXlENKoeub1XbdNDtKIngXjHaIQGi0Rnd8oAednZ0hw91uFyNcearite0awE3MqffeuVAWo+uZwnC3rrgI4Lqh2bppGYf1+JuPyvXRZfjauUHSPICIYF3Y7OpXr16dnl0YF2IEIJASUMGUgOj53uG8aZqura21ybuEjECkjIWQOFNZPirKMVDmbPAhYUxBe4ABaSAcGCAIQlMi7Ls/+uH77z56+ODOuJDW7to+aqcFpZtmE1O6c/cBFRJSMo0usjzGRFJm6h4ACOU0RU65EMLb8Pr0rfdps9uWo8nefmU6r3tPKW9Msd7hrtWFSrNpxpmaVAd5nq8Wq+hDIoahAwIxaq17rSPBiZTy1x9/vLd3sD/bPzg69N63dc05d84Y02sdYjKUIadsMhmZEKWUXAqtdd81puudNSGEy8WyqiqMwTlHjAEKITihOE0MIRIyNMIQQoJE8CoTlTGWYoSYfIxASUJGwhV4O6RyIQFkyCilmZDWekRUV6HMbgiCqzd1TJ4AYMQUIZCgJGcUptMpJEIp09Zcnq9O3y7bXj97+qJg5UiMg+1UUWTZ8YZmjLFXK/Pf/eu/+IM//qMf/fE/++btZX92gcHsT7IQuhiVlHLwthvsBYWQmcq3m1Nnh9k8Dnvz1RomjACGkAjgUOGstSk4xFQURa4ESTFFn4Lz1iAipxxjqre7dle3bV0WIgUnudg/mN++dbRYrS+3jQNgmSSMdH0fIVpvCCAVlAAASYyRqiwkpwmClLxQ0hIclQVFEIwCuVpLmAZkmzBGOCWINEZYnC3k7dvOhrbuBJNlDolypw0l5OjwaL1eW62r8WSxXP2f/y//13ffvXty+5hJJhR7/ME7s9H8V7/6aPvklBASU0gp5FkxGo1kppxzVPBEsO371WqJSKXgk9H4+PAAGR8Eq4Mh5k3VbJruamR4ta6vrHaHc5iz9jofHCgSQkiCcMMZpPRK/uSc0z5kkAgFyhk6O1hMxxiH0/YVRp6u/HMAIEJKkFKIAEAgIiaSIEGqt21RsN/73R//yR//Z8vlxU//6q8ppePJ6M2rl3XbbLe19wmTQSJYwkSJtm4wIEpIUqJIaYo0IklwZVl9IwG9KWMDVjx4ftDrFDEfOiQoJAByQB+75Fzw0dd1TQhLCWMAREIph8QJAueQUnr96u3QSXjvORfDWOeqa/Gp703XXY2uT1+/uWYB18E5r80GMHkH3lLAzWq9vFhMS8mS6rWtRtXQVg6wx2q1Wtc6Ed4bjYz4oKOzMQSM1xwFAMr+f9oP/APX32xHriSO186+KSXw115M164qQojxeDyZTPI83zUb6020cXmJjLFWt4iJECIzIQRTSiglylFhjdFav3z1HA1OJhNjzGq7Mca0nWZS3LoFk+k8y3OforU2ROCcuRhW281ueaGqqixLiljXddfWgw5zOhpRSo13eZkdnxye3L6dFYO7wFVEyLV33G8cogBgmOUNzfR15XPLywtErKrxMGEZjUbOOcM1YwwwckKj8gApRm+0c4EgJgJ0SNXlTBCIMcJm21obt3UXA6dEpMjq1goFpcPoeQoRCAWMIQT0Aajb7XZKqWJUBusQQSkqpcQEl80OL1mzq1+/eN3Vje11DGFSVH3fBu9Go3L/YC4UjSl4b3ujc+4EZQSQJIg+BBcxEc5wPpnrztR92/eaIB2SfqTI7fW4IcUrhMB7b+0ggrjSLzjntNbX5f83VvE3HIWbDTxddxY3fYM16H0k1hLCEDljnBKHIJyFEIhS2Xg6yVSBjGZZoeQIiJAZ4Sy3ptdtY41hjGZSGWMYF5B4CBhDMjZEJJyzXMgYoyAcAIYcYYE8QWKXq9XZcrLdXlrbHO2P5uMCKTHBmNp47+/eQ0boqBzPpm4yP9jtGppA11rrDgEoMiWyUTnmRJ2dXRgdV6tVUZVHh7cvzte9XRPBTUtoICkR60LTWQqaksSQpOiDbV2ywXPCo/em1a02XfCPZrNZXbfHx1dj4LIsD/f3F4vzvu3Wq4uYgukLIVmWSSYwn8y4FN77Fy9efPHFF5erFSKlgkfnog95nre6d865GJqu5ZxjGk5FztNACAMAKSUndCCTx4DOYfKDQm3w4+MDZOScSwGiT9Y4yaQ3vu9NkZV0yijlMbphhz06uu29DV57p52p+2gRIudMa22NV3kePFrrYqIpIudyVmaTKl+t15pSLlTnQTC+WjU//+zJ+Oju937nu5O9/fPL1XJxXnDmILmUsiwbhAacS0KIEIpSrrXxPgzG38OsnRCSUlCiuLFvg8GrodfG9gBAORdicHtVifK2I0hgONYY02vdGWMKRb33nPNRUY7H46IoLrc7Z2zCFEjc7LZFdTScHgbbAExJKbW3t8evjegt5ynEwWGJMxYACVwp/a6EA4ApRM6BJFitVoeHh87aoZBIKcvjyfNnL7u6idPw+uUrgHh4MHn18tSYDZdx9v3v/eqjXxKg9x7cf3DvnfPzc3hy6p31JhhjKlJxyRCxM/3z58+kVCoTTdMIoe7cuae46Jrm9fmF1rptW+ccIUwIMQwXnAs3ZMZvm2XVdQ3XKxiuIPqIiCoTyYfBsEgpITivqkpKiT1UVdXUnVJ5DOB99CFcR1JcPfyqqFwxIgd6aYQEESLFBJgQyHw+apqd1vq99x7/sz/5p+8+fgcRD/cPPv7k5y9evPj8889fvXnbdyFGHQB9SClBHLh8V2blCJSSSAZbpxvA4IaUgNfRpnAdZzWMIaiInEtKqfKJMQIAqbXOeaXyGMEa730kiJQwSAkBlcLg9du3b9fr7bBwsizLVHHN5Rac8hjjZrNDTJiCbpv1em2MUVmhmGh0s1lemrquCpUJWq83m+WlPpqncRG9l1xMJhNK6bAe6zp40FRkeVnUzQ7QQ4qUEEYZBgohhfD/H4LC37y+DSN/G+RI1+Q1a60gZOhHhzJMKR2QOaWUc67r+6GzU0U+mG3ITCmlqqoqikJKuVgsXr548eLFi+1223XdxeXSOedDmsxnB0eHRVUeHBwESKdnF03fqaSUUrP5fHf51lrbAGCIpu8ypd599M7jR4+klBHS5ebSBT8ajfKyQEytbhUHa21d1zeqn2siMEXEuq4H018hxHA4Foxrrd+evs7W67IcSa6G3y+KwntvrI0hcME4Z97bGCMXFZIEEClGqSjnJAQPQGazvVEVrEkMRJ6NMUrB8zwbae29j2mg5pMISAaOTTUuhRDRB+8tp5hlgjFqrZ3sTUOCNxdn37x8SiORQhRCqaqYzida90ryRNNyddn3HeMEKPDgeM4lV3lecoSUxaOjozLLx+Px2eKi67QxLkZwPiSClLPg7dX8iA190gADuK3dKpULIRBh4HWSawP1b6/Ha5XTb+RL3+4VAICzwrvkXKQ05Fk5ns2LcgaJGZMSobPp3t37D6bTqbZuSB421ClZMCRG9+uIziUEZFxZF51NW980rQkACYjMCikyJaT3fiC0OeMRk6MMANh/81//N8H5zepCMPbOw7tS0PVqmYIxxixWy9OX3eHxiKfiBx/e9xH+6vlfLzfryWQyGs9PT0+7xkxGhWT7JAsx9E+enPFsev/WnvahWy/DCFJGY/+JoeVmFcbF0aYVwYnbt+e2q2uzmtyC6YxC2m5WPfo8F0cKuaPRxc39+9PN6rmuzzKpbNe5Bk3TvXz+/ODgsO37rJgkhNdnq1998mw8+qLruh/9kx+fn583vXUBmGCUKpWPnSdnp+vxeNppW0lBKcVEtb/s+14IKcXIORMDISisM1fzaQ8xYoosRUgBY4wIkGISTAyWU+2m1cTmebCd6TvTy66rG+8cQTaZTMo8L+8Vfd/u6vV6Y7T1Aa3H5JAqIYpcAETC6aPHt8ajmZRZDCBE9vbNmX8Jm80u9F1KpN7urHUM2NOvnprWjMv5/vyw6zpKaQapaa1M8WAyEpIJiTHFxfrs9r27rOCXpxe674tMCZZY9DxEmWLG2bSqwLuLs7e3j39n/+Bg22yVUsPOgom06y1zqcqr2cmk77V1tTbN/t7xZr32NkGUUmDw0Xh7eDSdTOXZOurkt22XCCWcCRqbrsnHYyDJ6JZznmfq4HD/66dPpJS73a61mnHhAFiWOQCRktZ6YPxyTmOEhEAYDcm/OjvjnAcC2bh6cfrm5ORkPp9f1uvVekk4WXdLWpJO+2Xv+eikmB0/e7XtuoUkPrntwfy4KuEHP7q9sF+nhKdvF89eJOoXofWxb9Zvz5mlmcpfPz3bn57s7+9rZ7968qxpGufhBoGnlFFkFBkjLK+KYT0bY4zu4ZrGSPA3gM1QhAmljFGn+yyXWmtrdV6MVZ58sITbe+Vca/3weC+EKJPte5ME7Y1Bis6DYCwA8y5yKRgTfd9XLDatUVKFhDFgnpUxxug8ROVN8+Szp3/2P/zb//V/9af/7A9+nCkmOH33SL158/DTByd/9bOP/voXv1puI5UAhBMARHJ1rElJUMJYICT0vb0yZwQghCilBpr90LoBQNu2wzeXy2VVVTnSTGUA4J0lKCAZrXfWOimyEAIS4AIRQ4Q2JQSMfdMWhayN+Q+/+GVxfPDhdz7QbdsrsjY7yqzXFhPzVrd1xpk6by417wghRKnW6pQSFSQFaEyLLCVgB/Ppar1bXbbzyf58cte5ZBepO+26xYaYWHBoXYTUWhuEIC5ykoARjjGFYCAMh9rfJH7d1Oa/edD/NrLyW8OCf2QH8Def7duPvWnIIKYYI1wbehFC4Fp1MuiBB+Whve6V87zUugsh+GACAcowQXQxAhKvY4qRRuL6SwjJGTvP+Bndne62MSKqKJSsRD6qZru67zvbtU5Kvjy9IBR73KQUwWmVD8LsyBWvJgcHBwf33nnn8M4tKSVl6J/H09PTutvSJQ5dS1nJYf0O3oXD6WjQ/hFCLs4vu9aMRlNjTNu249EY0IUQhr1ut21PTqZjlb15/TpAoAClzEBJ0zW7ehGTl1wkMM1md+furaOjg/PTV21djydVJsd1u0PKR3uTrnd16GyOgXitPHMuhND1DSKWowISmhhVLpFgllXVfNy29eV61VhTcNZDzLU/ODwe/864wPL0zVmVV5Oiij4E6/cmE2Pr/dmUkuRcfXyy9/z5UzbaM13o207R3Hj36P6D44PDR48eXlxcfPOTv7TRRoLr7SofVVKJTb1zlqILTBBKGFLwIfgEPgHn1ENEHxkhiAQCQU8Ykcg9MoKMINAIEVMIIcSIECMjHAnGGIP3eM1e2HZtXuTa+eVivVzXt5Cd3Lp7fHLv4eMP2153XRcxGm+YZAJI03SKlJnMENFpRKyEpCSBc7Tv+xgjQAIwCQKlNGeChwCC90avt5tBookRmqYbhpJEciXV8XRcHu7tGd2S/b292dR582f//t9//vnnne6P79xlUpy9fv3JJx+1rn733XcVF6/ffLPbbGezXNtidbn+/POfFtXo6Ph4Mme99tMZiRhT2k0mk1E1H1dIUq77pIPvdB26FdJYjqbTaQUx5SqlkJFQQOI8F9Pp1Bm7WaybdZ2rom/betsomc+nszIvrPEUsBpPm12LCRaLBaX0F7/4xXy+J4QYjSaUCZ+itV73xnvfddo4l+fleDzmnAcotNaMCcmldxBCapt+06yHsRBAQgIUASMApQAYLGf0ahullI5GI0gkxth1nTVea315eak7I6X03k8mE0ZE2zbb7Xa73WqtCfVS8hig7w2lnhJOKcQrVgsqJatqvLrcDAZegktIbLPZGbPJ83y73Q6N+bBre+/zPJvP9wYs3cagtfYhVHyUCd5sN9bavm8hBskZUBIBlVIBUtu2SBJlqWl7a62QrK5rpZRQ2W63CxEiQjUeSZEtViuig/c+AiGUE6TOOectYpLXbjBN0/TGIRLGObv2GYxXgUlKSpnJvKqqV69e6bYfaG6MxkEmEEKAIVETMc/zLMtCCMa4geI7TFJ2u4211nsLELNM9hc9XGfnwLUcYABLrLWbzSaa2urLF69uffDBvdFk+i//6/+t1vbLr544+xfWQ5Gptt5eLpez+cFkuu9CNM4yxvrdtq7rpmnKanpDIBqA38HVceB1D6DIoDa8VjSxK6w1QbrRNbkYY4oxCiapEmVZCsGoQ8l4cIkLKqVEoECwqhKl3PporaNMpIR9b9peDzY4k3HRN610CQijQFIMLnjGmOSZMXpv74Bh/I8/+WuA8C/+xf/89snhbrN+fPeIKzk7ONg/up1Vo5//6tOLy7XtnQ+AhAChSAAShuAAIiGkqipynXbx7do25MXFGAdobTAJlVJORHU1aMcYYyQEsixTKrPW4pUVXASgADAMX2KEGIi16eWL17/4xUdSZnt7M0g0RZISTSmkCDGATy5Q4h2E3F7Jya5muhj9lVkhBrtldLNrnPecSWTcG22c7bTRNiYCjCKG5ANE5wjlEFKIMcQAMUBKCMASGUic3y7ev1XXb6a/f2sf8G2Swd963ahM/+Zz/mOulBIkGOCceG3gaK1t23a1WjnnJgelAEaz3AYWoo0QCKGEsQQBkZJIMJEQY/DBRu+S29ufxEHLFxIkKkSeKSkkf3v6yrp+s9k8ffo0pfTw4f26rhfnpwnsfG/++PHjw8PDQcefF6Jutqu1SSldXJx5b8uy3NvbM8ZYa41pBs/pruucc8YM5B4QXHHON5t13/dSykEB1Pf9vfsnV42Ci23bBm+1SQBxVJSACVOMXkfOi6KiCEIyytSkzGdVRZPPpcToaUIEFER4G1xno42UZYqkkMD2XhtDrs1nB6gpxhicb0xvnO6MphSllJPJaDQaee8RaVEUjNiqKldSaq2XvfHWVaOsadKuvmQCfPDXOzbvdCsY54Jq3SkuIgRkxBhDOEkYvfdEUs454xQJcM4kCs455SwRJIgJCCINITlrKQ2CRc5lSpQQS5EBWMG59TFFh+hTSkOee0oJCRsogAOhCxGNMcYY4HnX6U6bvu93dbtcrl+8fDMaTYTKBzoVEyKKyBCVyjnndeeGlU4praoqy5X3Pjr/G0wRcbBNAYAYY9M0dV0PiBEdssc4l4yz8/PThw8fcpZv1pfe68ODfQ7ixZtXl5cL693h7ZO948OY/MXFWdPV09n4/OsX56eviiJjNMzn5WisVEbykv6z//z3E0LX92eLrxbLpYvh4LiaTqeT4kHf+edP3qyWfQLHVABinNfT8ShTVdu43W7HUCEkwfjefC4ZfffBYyWkIuLJl0+idUfzw1t7t2KMs+n8/Hzxtnm7o9tbx7dn40nyYddvDw4OzhcX77/3ITLqfWRcIqO7bV2NJ0Cod7Ezej7fn8/nlFKZCe/iQLpPKSGSzWbzCsh2u0UEIBFSAAwEI8QEmBIhgweWcy7Livl05pxbLlf1rh3CeTebXbOr8zxXSjHGlUramq43Tdv3uuWcUM4I5YwTcJ5zlMhdiL22QkaVsafPnj17/uLVmzdSZIcPj/f3jpumuzhfLJerum0ipL29vbwofAhN2zLBfAw8xggQnDemB0oUn+SZzJQ43J9tOKGAZV7EGINzQpWK5y5Gbb0Pqe5a4+x0Nm6apK0LITx/+UI7a6xHykWWxYQugQ1RG9v2xjiPjGeScUG3260LPqWktdbGESGj0SnJYZtL16oepVRZlANYOuwgNyenYew9KvKmaZxzhFHCaISkbb/b7Zxzg0Y8Qrh15wRIavvmfHGm+x4RCb0hhyfAmAi4YF0IvTVe933jz84v287evnVCFdXams4J9tf1rtnfU5TiycmByqqyVC6m1KS+79q2DikSdrUIbxBUuBYHDmemm+yWG/SP0yF1A0MMcOW4nFwK0TtDgDHMVcaQeuOtdVJxIQbxNI0xMjrAjImGkOVCcNX3PWdxNilSwoE7UlWVEMq4kBJqtH3fIlLOqTF9USjt9fnTiyznP/zR9x89ejSXWd/XgGRvb/49mWnrfIg//eVH690mywgQRhgb+tphL2CMdV1zrXRgVVUNHrSMsd1uh5hCCEJcqR4oxSyTQrDB9IJSSlngnEt5lXafyEDhGKJrhlTDyJhCwgn16037y19+WlaT73znw822j4nFGBKQQfgxHJFpiJRZwuVgYnZVOxESgrXWQ8IEy8t10/UDwd2mvrG+sVY7SBQSIwAhJcBEvB/OXBwwUTqc2hAA4jUn49uF/9v4Af5DDox/Fx3yb636N8/zdz3q7/q+9z5eD6qHtJ6hKefsSCklM84YaBtduAoT8SGSIcguxGCSSd5jjIyMZ4X33rsYIw4e8wQB0VrbX67sq1evvTPOueD1bnsZvbv3ztHe3t53vvvO4eHhYnHZ931RFISQi4s2hDRE3A0pd1IKKQXpnFLCGNY0nbU2peC9NcZt/VYppbUe6HfDH+Wc227XUmaTaiSrjCJ0Xd81jXPWBxe9G8KmUwxCMKYUJdQ2u/neVJIIzhxNp+Jgb/h7jQ6r9U7XzgenVAacuwCCKpIHSmkIbmA1BAgUkBBydHTU922wLnESQthttn3fG2O8j2/evOma/uzV6fpyLajYn8xm07H3lpArO7UiL3e7jdFWihyCjhCUlG3XKDHx3pVlmVe5DRoAettzJimnEYJzPkLI85xLJYQgTCAlIYIs+qxoCTIAQhJEn1KI3scmtLTvcyg450lKKSWjNPk0NGSZzCIiUAqJxAgxRheiC1EIOvRzQ/W5vFysN7vxeKqtZ4LneV5UVVEUqpDD6DCkGJ0FgAhJZkpE3nVd1+vZbOa9HyhXfd92TdvWzfpyNZuPMUH0wWg9rA7FhROC/eKXP12tFwBxcXExHlfvvPOo1+2XX345m00IARnDF199tVhezObz27dvv//+4y+//vybJy+ODg6/993vHx0djYoyy7L5eP/Z82+avmnqWreNd32ITvepYenW4ftd1yVqbdo2uvY2iGw8n0+UAiTC+5gX81Ex3m11CmkyGaGH6XhyeHCgmEQP9aa+c+tumRcXF4ujw5PF+RIixhAkk5KIMq90MFlWHB3ezvNS5WXTNBFgvrd/eHDChZrvHTRNt97WeZ4rmWmtNyuDiAMznVCoqmo0Gs3ns+12AxhTDDH5GH1MAWJKKUC8Sl5HjFLK0WhS17XWerut5/M5IsSYANA5X9cNY/zgQDEmhBAApO+MJimlFEKaTscppUQJAA0+De1hnl9p4uu61sw2TTMZX6m5ptPp4ACPiFVVTadTABBCeBe7qBNJgJEzopQqMgXe3j05SseH69XKWqtkvt1u15eXiJgVufc2IUQgy8vV27OLg+OjLM8vFovVdvfZF1/12nTaXq5XvbXr7QYgBqCbtv3m2fOj44OiyA4O50rwxXLJOd872K+qSvttAmjblvuQqdEAxuC1GpgQMvTCwwvOskyKbDKZSClDCAwJIu37fhAT53k+uM9q0yOi1rqsig8+fL/r26dPn6zWi6KaSS5ijL1uY/SEYkouxLTerUPwGWYiU9G3ddteXG7u3Lt39uqF976rO933zRa81ZDi++8+PD2/tKZrjavrtul1Smk6HQNA1/4nEscblt+A5YQQBhscALhhIw8tkdMp+BBIooRgwpQw+ZQAvY9953RfpxQFk7zAEJ21OsYYITBOU4zW9ZIoa2zXtISwvb1pSvjmzZvVqgvxkhC0PjHBBFcJLCJJwJQidXMZvKUU1tv1T376kyzjjx8/znJIIYTgilL94PsfFKNCKtF2/77XwUcbjA2J+Bh9AMaoECLPxY2qTUiawGvTJp2ct8Z2AIAkAgAlUFbZweE8L+hmsw0hDPRML7wx0HWGEEw0DZQ1AAKQhgRzwjLjElJBKV4sNx9/8nUE9vbtW+uIjzQCAhKgDFJMgBGBpISYKBJKMUAadnkCGLwFgN7Y5Xr99mx5vryMKem+P1utdp1xaSiuMUagFJWU2noh1OA1RFKkhHjvjemHwQr8p4jCb/UK3/7i24X8twYWf2uBJ9cOejfPDH/HPOLbT/tbz3XzqKFJdc75eEWi3S5WvsiLKkdBMCaEQfkbACARBEwhgaMxMaCMc2SEarQRaEQkgjJKMMYUvCkqEZwZjQSbll2nN9vLvmtHVfn9772fZdn+fMIpdM267/tcMcYkJ4gkKEGaRj9/+uT1y+dlWY5GI8YHy5OQgoPoOaFAGbBoe+3REoiCEUwJU6QIKfjF+blSClOoirLtunq7U0od7k9n0+l6vW7rdbOrERGigoh9St97/967774DMW03q3FVCMb7pqWUc1m8Zmfby9b1NQeGTDFKhcrLjKSUmq4OIVBAJWRZ5uPxGBm5bnCJi2nQKDnnrA/GmN2m3m7XTVNTREmJ4ERIRikyxlJCxgQC030oyiqjwhtLGeut6a25WC6MtVLKq3POddAGAJjBQwWQc54VFWWCMEaYmCKPKUmpUgDnnOn6rmn7rgnWxRidj0iSIkyKTErlB8aFjcZ6xpgg0nvfax2sQ6RSFNvdDgBDQgCilKrKsY/JGp8Ago8xpKsMayYZ5ZQiI1eppBgjSeB9CiEY74iLKUZCiBJSCe4LTwE5503fWOO8C8lfGUAN9ZAZ2/76o58LIaazmQ3+iydfxRiR0SfPnue5anTHGBWZEJJ99vlHP/3pTzGVdW3v3xq//+6Pbh+f7HY7TDAeV+xR9eTJV8uLWvHJ4byw0RCGWZ69ePVN1+lqwlW2f56lzeWmqMjDd+9a3QdnmyaWRQaYBa9H1eiD73xYL+sYoWv14eHhvXsPvmq/qut6ebFcr7acSe/jycnJfLaf5zkicdYLnq8ud0cnx8vl6vbdCoDWu11WjDnDXb3Oi4pRkWU5ALatXq1WQowopY7Yrm2s1eNJk+eKEGKdBogIMaVwdTBI6TrrEyjllKbBomez2V1erp1zIcTBpNJb51zw/tJan+Ul55wJgYQZG503MWGImOUjSilLNESCBEIC612n+wiJMJoAmq69WC6kKglhPgapsnJUee9d8Jyzo5PjrMhD9FyMnbOUYpFJqVhRZFWV276ejQul1LhUurfD7HC3qyNgSBiRGOs6q5+9ftk7XY6q2Wzy+ddPLi/Xr87OHEQP8WKxZLumqbuiypnIul5/9NkXjLGyzGUmQ5FlWbG3d/DgQf/Fk282TZeAxBgZQWstQTYY1N+cvIcYSZIwXZkcX/UEALBerwlnOSuFEGWZF0VBKY0xpJR29Wa5XOaVKkZZ3W0vN5dZlpWjuZRZp5uu70J0hNEAwXqn20ZQrh3hJEVKzxbrz794MplN8+Tbru/r7nB2oFgzHVW628VwwDjs6rrvrPMWIOVlXtFRSimG7cAgu6EL3bjPDp37wJv7TVWIUQjBKY0ueOdSIoQOiT2JEk4Agg0mWWuikEypQiqwLvjkKWfTrKiqCoCsVqum7kIIUjDGGMGUIEpBGIHD/ZFS6s3pBWcxrwilSAjG6KQUbaOFgqO9cVWqN6+efvxJkWWUHY8ppdSbhLQqsx989/2+qc/OXr8+vWh722kfUrIBfUycCZEpDE5KnmWDrJeE4Jwzg0mOtZoxRukVtUoIcXi4nxd8u912XatUdnV/GMboCWGDogITphQBbtyESNNsGWPT6bRutk+fvkgJttuttcG5EPxgbZkoI5CQ0hsZ4XUhJowRGFjGCNE7d7mpv/zmG5VlR4f7GNMvPv70dLl0CJxxCJDQMcoRqKCEIbHem64nBJRSEcIgEf12Cf97Sv7fWtH/wUYBvoVS/INwws2P/uYz39Ao4rU159CztrvWG2uMyaqMZZRxmpDEGJAiQIwAEQIhQBS/6uScRxopJxSJUhmjHCMGjykA52pvb/b86Utn9eJ8l4mMIZmMpnmeKyEhAQHUXX9xdsYYb5pmIEUaY3a7ndZaqbwoivG4HMxCrLXO+ZslHyOY6IN3MYDu2pQSgZSCl4JHHy4Xi81qZbR2zt0pbt29c/L973//+TdPP08BE8QQCCExYPQhk+r44BBCjFqXXDFKI2qKlAGRhCkuJBeEkJiSj5HEYIyLMTpjB+BBKSUyMZ1OQ/QJBkEONG07xGpTSpOxiCj25Hw0aZum3dUsoQ+W+Jiidc7WdY2YYgSVFUeHx3sjut1uGSEUGYG4Wddv355Op9Om6WOEBCRERMYiYkoOKbqQgDDKJCB1FmnCoiwHIjNwkCJm3CiVWz3yzgyHhxCC0d67mnPNOadEZgXvus7ZFKONMcZAEvKYkvex63ol84TJGk8pm81m2oTdrh5NJ9YFRCqEKstKZiqlGEKQSgwHG++ccy4EP+zGy/OzGAJFIoTgBFNKCGCNOTs701oPB1REQhEpMkTKskxkmZjM5kVVWWtDikRKJtiYYlFmijPddzHEkGyWyeOT/e0Kjw9vffc7P9ifHpIkBC0ZY5Rk0baZmrzz6Dsnt48m8+pys3zx+ul2uz69vLxYvE4JGWMyhwpo0y1fvKT7e8cx4qiaTyYzq91ytYvAts0uyzIuhcqLyXgSIjx/9bLKS6vNZrMTKrMu5EU1mc0oFyd376iyUFVxdnZ2sH9Y1+14PJ2M96RaEmSTyezFq5dFSbKszMoRAg0hEGQXix0AhOAuFxdn528AY5aJ4MzgN0cJEEII8gG9RMRrFkmSUjobTk/PF4vFblvPJpOU0upys7rcSCEA3BDAuFwu8zz33htjrQ3ORUYDgNlt+zxXlDAEBwk5T8OIfbFY9H0/bBCDAXCWFdMpa9t+8G01xiCK6XRaFEXbNcYxAMhzOZ9WZZGPJ0WmeN+3IQROoCoKyYU2oSiK2WxmXXAQEUiru1FIVutPPv10Mhndv3/30y8+b9uecR4SYUJ11qGLHqKNKDhzxiyWq6+efHP//u2TowPGSFbkkeB0OieEWN0jkzE4zvl21yuZO2tvZFEAYIyhlFIgA5G+rmsEaJompUSZuPEvMs6ajfXeGWPyXCFJ4+mIS6Ztv1xdaNvJjDvtOOfMs+BsjJ4ChmBijFQQSplx1gZLUrrc1J9++XUxqn733UckkuDi9z78TjGqhMw+//rrtl5zSkI0hKayLLj3CVgC0nX9kER6w7QYsF8AGDwWh91naBcGRYNERM4Q6eCyzAnlhAJAcJFnnHMGGCEGRrjiueJZVYkYQWudInIhy3JEkDkbml1LgMwm0xDC6nxBCBllBT8gkrn9w1m3Ax9joQzEKCXpTaRUywkUGR+PueQhpfb09Tc/g6599HA2m00mE0Q0WTadzB89uvNP/+h3/+2f/T8JaRIEn5hE5hMhTEipqpyXZTk0bUMnNFxaaxn5YBQ9/EhKOZ5UecF9sINJToyREEoI44IGP8DLCckQmEYAIgAghYRgvYuQCKW7ul4sV4QQH4IPIcQEBNPgKk0wkRi8i4wFQmKEAAmBIqWMsRgFxJBi0ta/fHVqtNubTZSSH33x1cVmlyhBwgGSFBQIdcYPrZ5zDlJSKjs43AOAzWbTNe1vlepvl/xvf/1b1f3b04q/p/b/Pc/wd13fRhRSuoq9HJqDmwuvZ3Y+QjDWQQwUR7KQXCKNHhxSiBhTCkgQJef0yhZsExAoIQicMpmxIisYEyRSiqwqp7qzv/z5r9bLTQzkYHrQddo7DJ6sVw0ASJkXxcA9lE3TrFarrmvLshyPR1mmfLBNu47JEEI455TwQeYXQhxMMq6EQpB8iJTwIQxiyFaAlIK3iCkFo/u61/WoVIwiIuZZFmPEiASZlDIGJMi5JBRoW2uKyfRWKdKZxjvHOS2qnKhCRzDamqhT4xAxeM+FIIjGmLar27atJlVwjnBmg3fWIkUpJY9Dai8iA56pssjceMQAMynqurbGDHi+c2E2PTg4Orx3797BjHdNiyk1JzvdtNvNKgFbrZvT0zdNZ1wg3gSMDBlYl0iiWam4UJxL58F6TxJQmwhD3VtCCCMAQCmRnAMhBGJiSmy328Vqsd1uIaaiqMbjcZ7nlMre9sYYTphSKqXUtm3TNJPxrCgKFxNCHSIqlRvrB9Xx0NUVoyrGQe0cnHMo2RACGwchKwJlTCk10GJ6YykgpGDtle1QoiSGcBNSHmOkgCQBM7afTKdVVdhga92VoyqG9Ozli6ZpHj28f/j4/nF21O920eu8Uj/4wfdXF31VVQf7Vd0um5oKKghk9Ua39W632a63y129vn33aDwrHz24b/3JbLkYj8uYvLdGKUVRfPPkeV+bly9fvvPog9/78e8fHRx//vnnL1+8WS4vf/rTn/7uD//o1mg83d+jQF+fnT55/vzh/fsUWGfdrms9pnq7JUIkwd55593xfG+3200nrq5bre2XX3yd56ULfrXZfec7v/PZ5184f6F7y5UcZrRN0+x2O0JAKVFWxUiXfd/CkOERPSIiEIKcMUEpZYQhIoHgbCAEOefGmPV63dRtnud5nuvebjabvjdVMUopdV0XA2zqnfHOObetW+MCIvERownbuqdcMg82aBkD4cw417ZtCEAYy6sykT6k1PRdSGit41xyJSNC3/c2+FyWqshFJlcrQymdzcazacUolHk2HZcmE5vNyjuLhMYY63obQihHle7tuu4ghV3b3Fd3ijJbfHH+5JtnCeHVm7dSZPcePPjm2SsmfCQphASR+hABIBGKlO+a9uunz44O9zlnBwd7Qx0dBiKCsWGedyOYhmvsdGgUOOck4ZWSzVpIaeDZHhweU8oRwTlnbK+13m43db2tRsV0Op1MRtvt5vJyGaIryswHu9lsZvszpRSlNPkYY4gAiRBKqRRKN22KaZQX1uxOzy+/fvLsbl7sHeyTRO/euXN4dORTvLhcvj59i1wa0wEyzqXxEIJLwIzRAPSGQnGDGKeU+r7H65gluN614XqKPKCYDMmgJnfWAkBZjkZl4YMLwYXglGAUaIqUoEjRdV3XNRoiEUK0dUuQx+jnkzkA1KudEvL2rbtaa3Bv/uDHH7778EBm6vjWUUQYj8dt31x1Vwm8sSEEvCKpGONd3dVZLrMsg+hSsLcO53/yx3+gdXd+uV3XHbKMyjwCjUgYl4rGYQZ0eXl5fn4+sJbgipQXlRLDT+vaeW9DcJtNu1qtdvWmTKXRjhCSZfm1OdV1r4AImAgQAHC+zwthjHHOwJVFjBsGTCklQIMkAF6BN4AspRRjCIGEkCICwcg5x+HtACSKMgQf0vli2TRNVVUBicgLa512IbioVE4pr5vGhyCEFIXgnB8dHRyeHHZd56IbkO1vv7PDf29IiL81cfgt5OAfrP2/9fCbz0m8Tgb/m13C3/oMIYSY0jDIiDFGn4ZTnSrGEYIJkTprvGCJc0Y5QUJhCP9ExMHJgyRIPoZhFIqIw6SpLMq8kFRNx3u7TfvqxZvkozNeySIGUuTjTEwZirM3S2PMdDbO5SgEf3iwzwg/ffPWWVMWGSHgPY9Jaq0ROACUeTEajYwxFxfLvu85oxRJjI4Rgsi89wQTZySl1HZdVVWAiVIsMmUZcoEE/Geffvz551++efUKkQoqIZH5dHT79p0f/uDxBx/+IAXf7to3z5939S54n2WuGk0QE2IiAkXOQ0wkGg9eRBauUmPAe9913XK1cCFIJZCCyBQhQDk7OjqYTaYxRiqMMcboLhjrvQeMkNAHpzLprKmqijDKGJtO5+PRNEVsOq9kZU0/quZlPnr88L0EAUJcb3vGirKQJpiEAIRCcpAYGSjwEYBQITklCoE7mxiVhBBGMKIPIQxMnZgCGaxfmUSgTdfUjV6vtkKI6XRGABljCanurda6aZqu6ziTiDQCSWmIkjKEsslktms6Sg0hhFEhhJAiY8xxzusUUvDEg3V2YJEzxiLnjDE36FoJZZRlQqaUMEHd64BsqImYUkgBEQlQNptNmq7Zterw6GQ0myeEXd0wKUNTf/HVl6/fPD863JuOKiXpuCqn49H+fH+9Xi8uX/bdVoisVCMhpNHOeUNZBAwXizd1tyhHWWfb9fpyZ9zh0f577z5ab1bb9ca5ruvr3a4f5QdFUVXltG70029eXq5Xs+lYSsmVJJz1RvetXlwu277rtNXtpuu1cZ4xtql3PsVE2bsffJgQzs7OOJfPn7+klH7x1ZM8z6ezvbfn548fv/tv/s2/29XtYrFI5MqK3BhTN32WZbfvHB/u7/lgY3RScYZQliUiUmRkmD4RjkARKGfBmpZSmmWZ995ZTymdTCZOu67r6rqOEfI8DyEMhXDgjQ/Ye0qJMQ5AnHXO+pQwRhg4qFJaY8xicZESGXLGBmRsKE7ee6XyocreQJpSSiE4IQkg7u9Nc8X6ZgcxMIqoBOdc6zom2rXdYrHwkaisvHlVdV1zLvb3pk+esPV2c7FYNG2b5eWtO3eev3yLSDgnCZLzBijzwRNEoVTX919//c1klAtJZ7NJCL8xKlZKIdbXKkfunbvCX65h/K7rOGGDyfHgspBl2dBDpJSGnMbBNpgxKgSTikvJjdGnp6uizAghs9lktVrtdrv5fDokTqHFMJigMmq0SSLzMVLErMgRTNv358vL1Wp9//6DMs8lU9aYRvdtWy+Xy7yq2qahXAVkbatjIozLm9eTrnOhbjgKA8YwKCEH6ddVqfbee297G6xjUgkhUoy9i3me783ms9nEWG1MH5wBAELobttqbXTv6l1njQkhlWXZ92Y8Hre7djKZVUXpjJ+Oxvfu3bu4uLDd6vvfe+TC/axQ84P9TrdSyuVqMRqNCCHRhxCSYFwwpnvb1HVIWdd1u90OALw1KcS8LIpSUQIIgUCSgvFcRaQuIBIWfOccaq03m/VyuRhskaSUu902pZRlinPmveu6NoSw2ax39WqxWBoDeT4gXinG1Pcmy7LfRtpJQkhtZybT0oegbT/MN3ujkRKVSyAJTAJMKYUUYwJI6PGmtF7PCAghjInoYiBOCFFlKsVgdRcSREhHxydCri4Xq6Zep0QUQc45RYqMVFVVVWVW5IeH+8Wo2O7WF4vFbxX+m0bh72oI/laE4B/TLtz0kf9IXOG3rkE4j9ehXNduh3FcTUIiBKKL0VgvvWeJcMYpg5QGp2HghEJKzjnvHKUckQJEIYSUMs/z8Xhcqmoymj998uKjjz5hTBBkzqbzt+d37twbj/YopadvL3Z1O5nMgg9N2486k+flQO/o+75ta2P7PFeMMQCWUqKUDkNDxDTMra7NywEg3nB6Bldmxkjf9yk4PhmNJ9XJyck777zz+aefvXn7arvdVnnlGIGInIv9/f0I1LrAEEPAzXbXbrdKcillniu6JdrrTpuoREAWSAgxUSqS98M+GX24ziWxKpMxhr5pe9szxrJMcs6ttY3xhBBGqcxzzGIKwfd6gOX7vq+qath1vQvr9ebs7Jyi3pvNdNcXKhuX5Yfvfddbs75cKFns7x8FiLu2MdECoyYApai1ZowjEUwUXEgmcoLCByjLnBFKCEAMlABFcBZiJM77PM/lLTkaTZbL5cXFxWqz1p1erdYHBwcHe4cppYvl4vJy7Y1ljC0WC0IY5ZIQal2IkVSj8Xy+/22TPe+i9354NwY0Yig6jDGCEABTjFVVYUzJ+uB8tME5F5yPMbp0NfySnNMre1smuGCS0mw2G0/GwXa7pismo1snh3v707fnb7/55puuaz758ivByN27t/f3Zn2KE9sBwHvvv/v0+bP16vSZ1iFFJllVVREiLd1qfXr6zelsb763t1e33cHB7VxkT7542vVbb3WMcX9WUpdGhTycVYUgzboR3s1ldnu2/917j/Zn0+dffz0aTabTaTkeMcGWm+W4mowyuVgtqnL8ox98/4vPv3z9/Nn5/YdN0/SL9ZvV6i///M/btp/NZtM79+8/3D8ppm65ffrRZ1RIQujpYlFUWaf1bG8+k5W3dvv2Qng3nZVSUiUwzxUSn2flbtdUxdTaOCpy54I1vu91VY3yPD8/P6eUEkZfvHp5eHh4dHT09NWzSCEv5KpdUUrVWPWxL2CqTWha02sfE0XCfEhta7i0vXaEihASILGWORet5dNJtd1uQ0gDSVXrLkbPuLC2RyQAJM/zFLFrvdENYyzaPnh7aZoVBK97ONh7dHT70bv3356dLi7Xb96e/uqLT1yCRBhh7PLysl41XDCWeL3p3n/0HomSpvz44MGb16uuiW/fXE4mB3UfY4jRaSol+o5z7q1FRM75+cXFT3/+a0bFeDKfzSa/+uXHr56+4ihcYwrCS8ZC4HXdlnnhbBjNJ6vVChGVUuPxGEIkBAc/oiK76nsIi8butNYhJKUypVTw4B32fZupUb1rynL85OtnSubL5dK5EER3Ub8pimLQ7zVNQxEFZxi9bzeCJO/9cu2yrOBl9vKi+/XnnxTT8Xw0e/P8TVUU+/PZHsqPX59N78bS18d3914vtl3TbzXk1UEiGVc1AFoTrDGMsVwWQ5fj/VAXIw4BH0gZEYIJ8AkAiMykGPKpKRNsOhdcUJELprgJtndX/oa2N0AWXIi27SaTPIZidbkLHiejSQxwdHKyWC3enD+vRlmTzp9dbN9//91q7/u67KbjimI0sIiod42ucg5haT2ECDGh8RQCiRTShNSXF1FED6Rva2Sc9Fx0hcqq6Z3jN3XdbjYUc0G87TrnXFaUjd+AI8bYiE7lMqTofTDGC17EEC7OVvSYxxhJhOOjk/M3p5eXl/uTvbbvdesQGRfERkQuto1WQnDKUoIUhgDRRBJIKkzjlch11zDGVKba9UYABEgEgFFqtPMxIRWDxyLS6JwjSDklQesIhmdKUAgYOGcpxt4aSqmOYKyTyLJSqVgK12fRWGtN7IzuUCBDRGKPjvcePXhojPnkk0+++eZZNI6JjBAyuAIP/vmUAeVUCNa0PaUQHDCORVnudnVMAIlnZUUJRucZQrAuRa+UNLEHgIQkAkQC8WroAuj+kyyJmxbkt9qFm/4jQEREhG/1JVdmnYkzxgilgAjIkBIg6ELbbbMs44JTAIjIUBWypIIRhsgHLmhCTD552/dt8iWW1tosU5DSreO7pusZ8Ezm56dnzXa3Wp6tFu3hwUG9qZvOhjQPrje9p5gghNPXb7ig3tvn3zx7e/pG645EHi1KlmNk0QIAz9UkhLBb6uXbJ865EByJzDqPXnPOMUBKXlLCOcMUESEQQrhUQC8XyyIno8PDSXFYyDknVSbGBwe5dY4yNpqMV/3y3//k302zo2Zj7h7PP/vo169ffCUErXty74N35N7e9myxDbRxtLuoBePMO73dXRQtpRQpuuC2y633nnKecV6Vo7ZtE6TD6YRSarZ+ZbfT6VRJo7W2AXyCEJ3kIgleNztOmRqVAQIFQiC9ffPSGEMAVHHyH//iLx4+uP1Pfvyd/ePDdbNqdnXX6c7GB4++r8pquVk3Zsdz7qLZ1hsXR4iotXYuUAqma2OssyyLDtzwkUgkRkhEImcYI7MhhURTKnLFDotCTVflerfbrZaXX3z69At4OipKpXJIVMkqz3M6Ohgo4c7aGPo8U0WRa91mkvjAE0Lbbb58snPBFqNqf3+fuDwiSCmRUa01Y5RJHr0FyeW4DJBM1zNIFa2sNl3bEgxKFQDgtCEk5XmOiEZr9qMf/ejw8HBbN7/46FevXr3yL0Jr7Ga3vXXruO/7LMsEo4hpyAkNIdx791FRFBfL5dM3r549e9Z1XVGVf/RHf/SjH/+TvCw++uijt28W9+6865x78c2plPKr+rPZtCI0Eeqn09Hh/j5J7GxvtVv155dP+dM4Lsf7d/Jaq9adf/Hsp5+/fNK1ejQazafzV6+/QR4Q42Jxdnh4dHCw1zQdAJycnPzlX/5l13WbzebXn3z63e9+93f/4A9//ctfvXl7ORpPV6sVZWy3242K/O1i5TxUBUnRM0zdblPyygfbdW6zQUqhGsmYvLV6Op3GCIQMg5lhaA1KKUCx2WwuLpZd1+V57n0UQpXlaLutu04P8QoAhBAmxHBCtSH4gfY/4AQxeu9TvWuV2l0zyIK9AqvLGOFKWnadSBQjpIhU0JTgCqBFTCk4F51z0fbBeYs6eMMRskzt7c/m86nKZaf1an1pjLnc7h6//6EsytXlZsBCBu5rnudFUSilJpMJY8y7MLj2GmPikGmmMogmeG+cKzJVTSqpeNO1v/r4I5ln40n181/9+vxyVY1HfW9UUQJhQwuPCQbHYgAYwnsopQB4/QXANUnQx5ugxcH2FBGpUmokSyllXe8GZ7c8z7qus05TwrTWA+JdFEXf9wNKYUyQkgziabhW4Xvvn789xZ/+bD6ZC8Ie3LuXl9XRyfHJyYE2Rgp+sLdPeIl0S1c9cmYteI+ccyk5HWSsfEh5iaOqiDEbmNIpJcmoEIQx0N2Gc84ko5QgJko9Y4RSSmjou/O+O63rum1bRFSZ5Jx/7zuP2rbfrVsCtChKCkLKTAgZgr9crwA9YFheXpZVdjCqrA8ffvhhmRfBma6pfQqDlzNQoo0BAEBKGQdCfEjGWOtcUZRaa2Ostl2M4ANQUjO+BUomk0nb99rq2AAi9dGtVsvJ/sQ555wfToFSRiGAIgs+UUIAYG9vj1Kqx5UQouvaoG3wHkOkmBBITCl675zL8zJY35qeEZ4pRQl3xui+H1VV13XBuRCSEEwI4VyIEfKytNYOfCvGKCHEW9v3/d5kjJiGNBbGCALEEIzuhGAxRmtdSkEIcTP2GlOWCVlmucttSslqE1yECJRdZWAywW+IFyrLI6QYfQwxpTicgAEgxdh3PUQgjFCJzoXNugaA2Xzc9o6T0LZtcJ4RqgRLMRrT/11IwP8E/ODbncTfz4TAK3cvB5FRSqWUSikhFOHIBL1qFFJMGCESQgxEpIKPx0VZ5sF5RrlD633s+76u26+++mq3bYUgxpiiqKZTZa398ssvZ7PZ8G8ppapRsd2uLy7OdtvaOkMIMJZJkQ0zEcbYZr2NKVzlFABQSoW4Co1E9MOvUXpl7kkI8T7p3iopDw+P5+NJ8okiW19unA0pIqMCkKaUQkiEMKXydx48+ODD9988/yrLJAAwxibzWQrh888/f/7s5W7bECYJo8OWyTlPiINngzFm2BkGnHLge91QjuAa7IkxckKHWVjfdV3X9G3Xdb3uujLLq6oiBOp6t9lsYoxKKbe8mFXF8XzabVdPm/XhwSyEsN203tjdptYmdF3vEhJBGM+qkr453w2GZogIgABXbO50HbEG8JusSABggt6AAYP+a9ioddc754YdlTEySJoZIzkVVzc/BAwhBm97HWMkjHIkIcauaSKAzBRHEp2XJYuQKCAgMiTReZPSoP0Oxg6NrQ8eY1JSFnneBzMajQRlw6QjxhhDsNayUVm9ffv2Z7/45ceffuJD4rkyLoQYttuts1YyCgPozblzrm2aT18+effdd5+evjAYu+hqZydZ/u6H30OWMV7cvf14XB1NJpPVavXJR5+uVqt1fL7ZraSi00kB6Le7peTy4KiC6HfN2fPX7Xy+39luY0/X68vzRmT8NgCsm3y1G/d2fXg0kyI7fX1WNyvOGUW6XF7cu/egKsqLiwshxNny8mRXP3r83sHRyX/4sz87X1xY7/70T//0Fz/9KaPkeH/SW1NW1duzi1HGyqqiWBUpSylmGRdCCKGCd3Vdv359mlIiyB4+fEcIxZlbr7eI+OjRo+fPn19cXAxvoXMuy7LJZPL8+XPn3BDkc83cvhrJhxBTAkLo9aeBEALWur4zeW7yPE8JrfWc87IctfUOEiIQRDrMJuKVeedQ/PCKmwQQQvTeC8IIR2910zSKIUCM0e/qTUwpJq91pzJhLgwArNfri4ul4FlRFEllAz1tAAzH47HW2hrHpLhitMHVnH7VdNPxaDyf5EoKSoAk1/ht3//kpz+bTEfniwUwWU33XNyUk1lKSQh9sxSHz3oIQQjhnEMkg9v/AN0Pz08SDmMg769m/5nK8jyvRgUArNerlFKeF4eHB23b9rpVeTnM0Qc3J631brcbNqOBvgtX8QpXgTqbXf/50+cHe+39k9tEyNFsigyObp08ffY1l6JUsusDT4l4R9FDIkAzihhTpIhCiKqqKIHpZHL6+lVZ5eOqGtIdGSfDu78vi+l0Op2NCSEDZ3Nvb29vbzagHbvdtm3zwYtm2Nnfvj4vy3J/sseYsC6u+7VzQSl/dHJ8evoGCBrjLzerozvf+S/++X/1wYfvffnFv4YIJKWEhHOOKViLEZJQcugZkDLCOPrkYvDeI5JBLoVIEqTgQ+/bFLXKi8lsbL27WK4TpKLMVQjbbd22rdZmt2uaTdN1OoQIiVASjHaMUiFYSmk8ro4O5iGErmtbtiEpEYRMSKDEhKiNSwkzKYnKh1i1Xd2mhEVR7B0cdbutt05KqUSGQINHSCwC0703zlsbUwJKKaeUK04I4SQGCN6FFN3Q8Dmnr4lUabjDw0dr+PwMlhuM0DIvKJJuMF/3iVLadO2zZ8+UUsfHx+PphDDqguecDpySAUQlVzbJMSWABHmWee9jCEOUDmfUt9vxvBztjaIPSuUUcLvdda1OBBMOQMJ1FU+/+foffw0dz9/aH3y7S8DryxttCFBKpRQMBx9HwAiCSSQQBw4oAQzBsoxzK4Uaj8dlWZpeU8rzvORcIlLv/du3p1rD3qxMEff3909Obp+fny8WC6WUtWa73RJCxpMKAOq69d5b4whFIQJiMCZo3aWUgv2NMyllZKjKlJIYI0EKcJVfgEgJYSmlQvIUIsREAOu63prV97/3O4yxt2/fXlxcSCkTgvfeOCulFJniFCiJn3/6Sd+sF4tFnonZ/v52U0MiEZExQTl3LnR9462NwQlGh25+MG4Z3t9rUNAPY8Rvl2qBXIfACJ/PZnE0Xq1WSxMwAaVMCKmUIohGOSn7EAJjbCZFWU5mGX399Zf1bv3hh++X5ejyYgNEnr050z72PkSKRDEUCCRtmraqKkqvzEgAgBA2qHggDbRCBCCUkkFERWJKKQ33kzHGimKgV3dNwxht27bIsqLMh48uIYQT7hA6Y4I1DBINwes+xtgbXeQVMuqNJpSW48msmhZFxbLMORdDTClRJDF5jEkJ6XVvYkJEQZmPITgPjOd5vlcdzGazLMvqur5cLAcvAEYFe/78+c9+9rOm677/vd8hXCzXKx+SKDLGSELIcwUpTiajIldnZ2/btr1sGoPxbHEupMonExPg1oOHsqjmeyevXrziJLt1tNf35v6t/aP5/V/96le/frGN3BU5M7pvd0vOiOJqf49st/XF2brvX5VlmZVZZCSfjyjFYDRjzMW2M4mIUM2zUT5OEEikMdjDg9tvXr6+ffv2e++/+4tf/OzevXs/+PEPPv38088///SP/uiP/uX/7l/qtvvko48zKR4+uPPRx78wWqtMvf/Og/v3bjVd23Ud8nGWZZwzypKUnDFaN9uutavLTdsaALA2QiLT6cxae3h43NTderXtO3P3zn3vfQyQqaIsRjEAZxIUEVyllLyLjgaPEdJVnBrA0CsMexzlnMcYvYsIlJKrQw/nHJETIiiNiBgDCT4lhgA4zIMBgLHEiAIAAB+jZyKTXEXJnW0VJ1kufdDn52+r8aiqsv39eaMNcmGtvbhYSCn7TscYEWCz2bRtK4RomzZeub91XElCiFLKOK+1Dgl2XTvb29s7OFSCe9uH5AlnAOn8cmVjunXvIfJTrgpVRhvSQJAeNk24Do0MIUyn0+FUPQiWhk7/N4xuRM55SjjgNzfj/+HhzjlELIpiWBu5VAMdlyTgXIzLymljjC0yRZAONr2MUs6uIpINbSAlWY1oljfWtMbGBMOdzJU4nM8zNe47v1v3xln0yUdOpaSEUsKVEJILRgBjUJKVSijFIIU8zw6P9meTKWPsuCzDzQA5+eEcNvSOg4psAB43m433XimlA9nfO9Raa23enl1sNrsYBlN3n+UyJJ8ITGAWEz59/rw3psoySQtCCAuOMrRWuxB8TGVZemOMs+A8Y4iIDDln0HbBWoDEhJQSKGUuNr2OfrOrOZNZWVTOORe4ZJIIwnBb6yEDeuhHQwjBe0hJcOWs9t6en58SmuT+njG6rrc0xYgJCGWCU8YFRCFE5uObt0sAUIpX1Xg6nUYftLZ126A1kILiglLaamOM8QkA6Wg8Heirxhire6s9gyTzPIReMgSA4ZRltYkxlmVpTT+QXoeRPQAMzj/O2KHRrKqqUJnOcudcSjj4Ctdts613B0eHKs9Ulllrg9cphRsm49DPM0IAIuGEEd53LaMwHld939frFQd4cGv6/e//cFRNUsKz8+XnX3715vVZ0+lv13sCECH+th/C/5vX30Vo+DbjgVICISZ/Jd8lAAjAKMUIiSAGBEQEQglmQkGWMqGKfMwojxwZVZzTlJI10ZpAkEkJUijO5eHh4Z07d5TMUNA8z+p6t91uEbEaFcMYNMuKKyZvujrAhIDeh1yWMQ7pjoNMbKCjAueSEIgxJgjDp2vYAxURlFMC6LSptztO2d5sv93ViitOOCJNkEJI3kUgwdkAyW3Wy0xymvL9+cwHd7R/cHj7wbIxb5bNrgsR0AWjraUIQ8rPsAYH2Gm4BteE4RUOLfsAoQNAplRT133dpMk0U1mZ5TvOCWHTUUYpdcYN+yGb7w8nnPt7e1WhJtOy22Q5dfdPDjNV+d6fL+uL5WK56wNlNJOeRBNdhJCNFKVs4GMRMlR3gERSGnbJwSr0hisD0bsUIcXfRNUPnc3e/kwq3nUdJ5RzPmyfiEiQ0RT74JOznFAKKTmLidi2Zwm5kixhAgjWJesZII+QEvbahhAIozRCSqCkNJQhAAFkUjJIve36vueE+hAoEmds3/dDgvwgamWFyqSU777//vd/+MNPPvv84nJ5eHg0mc96083n89G4bNuGM9L37Xa9rustGVd//fOfJZ8EV32rKeMffvBdRlXX9X1vUPDW6/PTi1ExIoS0W3P+WmeSqXm+qxuKYrQ36Xf61XZ15/Y7Auq3pxfr9W7XdPkoQyq6vie+YVFEH0KMzgVvoxTZZDbutr3Msr6v+76zun/w4N7nn3x8ubh48P579fby5cuXT77+/A9///c//OBxU2/r3ebxu4+ODveRkF3bPH/2DRdqMp+99/jdfHKrLAulVIw+QUgpDI7L3zz9erfbee+Ndq9evV0uLwGgqqqvvvrq8vIyz/M7d+68fv06xnhj4TLg3kKIQUo3fDS9czFG53wMEGNKaTDmTIyxGMFaP8wjEDFF9C5SwhmVlAwYT/R+oC0P2FQc+tB0nT8UQrDGSy4GI6Mq5/P5eDwug7cJXJaLLOcphQ8++OCTz76u6/bw8Pjrr55orQlgCG673Y7H481q7b2fz+fBR0KIoDTLsgi6aRoXYlEULvim7xAzRJIXFSLuNluuVEJ2/96j7a4/Pz9HwlYXayllJkWMEWJyzm02GyEEAkwmk9evXw8MwUEXEP0VXNnp9uZEEoe8ZhsYY9bp4Q/Uptc6Q8Qr12TnBOcpJd31oFKZF66yl3YtuYhXqDyhhGCC4HwKsRjPOGX5aLxtu8+++IoBZAI3m42UclyN3nn0QOXjyfSga93zl8uu73VSjDEpFSVAKfFWt7pr641gZDat7t29czCflWWeKeGc67omBv7m9flqtRqNRicnJ4KL1XK3Wp1++eWXzrkhpCeEsN1uU0plWUZJEJ68ffsWEfOyYozNjvbKsvQxjMfj0/O3XddOp9N62/2r//u/5pz/L/7ke+PxmFIEiHlV5IWSclSWuQ8uAImJwmBmgwxioIlzIULUwVpvWUix732ng7FBawvUAhIhlPXter1GRCp4DAQSYYxlWYHInL1KOxskFJxT6wxAHI1GQrK9/cmqs8noNKC40XvvQ4yMkg8/fDA4THStXpy/1DpVOZtOp8e37j958hSCTcgBIqV08K8fRlQhhkyKUvEYHCJwErw2h0dHs9ksK3Ln3PNnL86XixQ9YhKCMCbctfUQUuJjGI3Gw2yIIA3sCv0ekl3LsizL0Xg8vlyvL5ZLFwMV3NgaCTJKU4AYIUbPCKeUJR+cMRZwMKF2xibnFceHt8Uf//jDP/jDHx8c3tI2ffzJF4vF+dnpBVz5SgEAgQSAcVB9/E+jLsLfxpS86Wa+zYsUnCMiYMIh5itGSrgUKkGkSIHQGAnESCiXnBEiJDJGsxC8FDmjEgHevHm9vlxdLlaUciEUIs5me0rlRtvRaCTLbBiuZVkmhLjyXGeCsUApBwiUcsEVJULwDABSGPBOdQ2XxpRS8DHRlBIZcs4AIEWERADANN1oNOKMiZwqLk4OjxQXf/Xxx5lSQogAyVprnUNCWt33ff+z7ebs9dOj/eni/I239vLy8vXrt6P5MRkmhDLzgAoI51RwJihZXG6uoEpChk342vmY4JVAyd9ospxzUglJBCM8uNjYWte9pHJvsickSyFaa7UeHCev8q52mwWjk7vVwXc+eGxMf3S43/UOETGmrtPtrhHlSOTMx2D6zjiNgoSQrnUYQJBRMrjXJ0Ry0x8CJELw24oqAhQRYozeW+89Y6yqijxXBIbSMERwISVccLLbAiZPCYUUnLOUcs6ItRYoETJzISwXiwFomeGRlNIBxhCllC7GtmujdeOi7Da7LqY4AJUpWWs77JKx1nohRHBea+2MdzYEFxjn/PbxydHBIcSICd558PD2vfshxuVmub+3Rwi8fP68aXe279brdZZJiMTu+vl8v21b2/WT0YQjOX/99q+ev7h/5z4Htms1xdR1jTFmPp38b/70f//Rx784Odr78H/2Lybj7OzNm88+/XR/72hv/+TH3z3srP/1J5/+6pNfn52uCUuJwMnBiHNurRdZQaw3utHWlLKMGE9uH3/28WdScmO7sHMnt46ePXu2/sn5fD7/nQ8eeR+b7eVHlwvd72bT0Z1bJ7du3fr5L34RAFWWiZy/OT17c3b+z//Lu4MwYQgrA4yE8PF49nu/+4dd181ms6qqdrudc/bP//zPjdFvXr/WWpflaDyePnv2IkYQQjRN51xICQdYabAaQqTeuxivEnUZEzF6AAAESqNzfpgmhJAACGMcAFKKlApKPSGDGi2khEJcpZJf7RfkKsh1KJw2OmttwOCcixGRABeUMeZDoBRHo2q+N338+PHZxfr12SKGNGQYQkxFkYUQHj16tLxYcM6Pjo6MtlTwmJBzjtrgkBaslGBcyZwx3m430XtGaZ6XnNAYozFBsmy5WI9Gk+iAZzIEj4gxRe+91noQkZZlyTkP1qVrwyUXBiJuGpIUYoyEsJTAGJOiJoTEVN5oFFNKQ4BelmUxRillSslZH0LI83w+n1PKr+YyiJTSGKOzYVhLSo44Y86m9WrTbTa+bY8OpoSQ+XzOGEFMVVWMxyUh4IMhNGaCUxad7XV0meDZqBzNJ3uTUgp2dDC/d/v24f48pbC4uHj+/On5+fnmEvq+J4TM53ZTe+fcbrfrO/PmzQoAssw3HQkh9L0TQlDOx3sVAIAQ291u23YxRm3taFweHBwcHRwSSGdnZxjTdr1dr9fj8fi//7/9+Xx/VlXVaFrdunOyf8DykjkfhcgApZIMEa9ygbwNxjJRpgBdZ/uuMca0ujfWp5SYVF3bp5QoF86m1XbnnFMyd5YgopRZoUqCg16aUyRv3r7q2248rooyPzk5Orl11DS7lPxITrqu64221u7aXbvtXPBCsOXiNQBITidlfv/k3p1bt3/wgx9858MP/uo//FXTrN6eLmIglDBV5r3zztttvdGmowiTcVlkKjhvdG86Ox8X7zy68+jR4/l83nStt+b84qJva6TUMfQuGh8YY0gISzyEYLVp23ZIrwYAa4y9jv30KfoYt3Xd9/1utzPOur6nLBGCZMidjgkTSTDkaAIAYkxVViAEimkynzx+950/en/04NG9vUooGmyyut/Vu+12twVCI8JvIQh/S8zDP3TdiGu+3RncfP+3OglEdMZESCH4YXUARgAIYcwEzQjjnBNCrfcYkTCklHOgiCy4OBqPKMXdevPJx188++apNcYbDwm9j4SQ3W6329Z5nsOGDKrCqqrKsuy7frFYWqc55yEk50LfG++vFi+l1GoDv8FCbhIOr5DzoRQyxijljAmASAIGG+pNnUuV5/k7Dx/vNvXHH306RARLKWMAIVQ1mfR93xtNku/a7Xt//E9Ct52Nyvl8nmdlDOBDyIvRwRG3Pm7rTd8hRNcZPQwcb6RJ5DqcZTiXD1vNcGOHKPnkfM7V4f5hkeebzYYiu3VyUhRF27aDOuPVq1cvX75s21opVRRFKKiNgUoh1Mit3cXqcrNu1ut1IpRznmWZyDLOlbdd8sFbB1cBCmwYZA+3AuDKa/wmUx4SSTHCVTdyNRlJEEPwzjnrtLWaEOCcYsIYkZArQRBnPCUmBAsQIvjBPkEpBYjBuxA4YyxF1J3pur7p+5E2s9GYArYhZVwywI1ZN73OTw5JAquN1RpiiCEM8eJ5roZDFwAIIRghknPnHHvy9deTySQTMsb43e9+l3K22tUvXr6UgscY1+v1m5evEoSiyGaT6cHBnrHw3t1Hd27f++bJk49+/TGN8JP/8S8IYc74aVFlQl6uzjeXm4ODg0cPH4/H4ydfv9peXM4KtT+aFpn46PXPvvnsyfiHMxjBSE1uHe9RWvpIvn7xjfEdMpIVvCpKa21VjCSTm2KDHqzxk/GMUlpWlW761Xq5Xl4i0PV6GW29WbypqurenfsP75385Cd/HXxM3v3Ff/zz3/u93/vym6dcKO3D6cXSOCdU9q/+1X8/GIOQ6+BgSpFzPp/PrdOPHj26f/9uVVVFsS+EOD099T4QQoez/maz9T6kBBcXC2ud92Gg7BFyNUYNIQ6h4wP3JMbBgCgSQoZQPinVML4iV3lCiMgQ6IAueO8DS1KEGIASQoZllyCCDzEARsoQI8YIhCTAGKLr+7braoJpOp9FII/eecCzwkU/n88PD4+NtsfHx1muTK+n07G19uTkhBDSdV3XddZaxdnweoYJnxQSfUwhKMYLodqYggvjYjwpxpvNxmrz6a8+vnf/4WQ0F1zN7hxwLjqzTik5c0UdGv5S731RFF1sBij+ZmdMKQ0khhBCjDBIgYNPhBDv1c0hAACKosiyrCzLZCNFkiABpcE53XWc8/3ZvOu64Rw5bHMphCzLxuOxECQ412uDESjQs7ML9ProaFKN1OX64uzsDJnc1Jv19rLTNRMFZ4lg8N6mEFjG92bj48OjPJOZ4Jv16puvv/7y00903282K9P1iHheJ855kamt7V8+ubi4uNDaSilHoxEiBSEMQkCIggfONODi5av9/X01GlGpjO3bXe2i1dacn59jDLPxpFTZdrsNrTFE1Iu1yuYYw3a1K9beOXVx0fW299E9fHifUcoJhZictU4bb0MIwSId9GAhBBeDdQ4ACGdSpc1mo63J85xJIUUek7beZ2rEmVBKZSKTUiqVD8Z8Dx7eWy0XRZkb0yOm7Xa9XC52u93s6LC0NgRvg9/tNtV2DRiLIleC9F2TCf7OgwcP798t8yJ6u3zz5b07R7NRtlqBB/ApIIlcYEKopsUUCyk4g7DbrprtjlOYTco/+sM/eO+99w6Pj7IsW22208lISUxAxtN5SrDZbb23QzINpRhCqLe762SQgkvhnLPGJHCMsc1iJ8V6IDPOZrN3RuMnT56Q1AGkMGiNCMFIU0LvIkWiiMyktK4bj7L33n9069bJvTu3Hs82k7FE364X3fPXi08//uWbV8+cB3aduPn/2bThN+SDm7bgb6okbiQSiDiwniGmNqW+7y83l6vNejSZVOPRfH823ZtTxiikCECGOA5kMRBrPKUieNc03enp+du3Z7nKkh+i59N6vW5pmyIgUh27wfF3NBqFEAa9ft/3x8fHnLkhYKuzuuu6QcgD6So7g1xfV1Z1lA4FjxAiJFdKDYJJlWhKyfaGJeKtl0ycvTl12m63O8HE/v4BMJoV+dHJSd02y+WyDNuDvfm4KsfjMU3p0YNHdeefP3+5NbFLjHNlvG7bfrvdONvrbleo8Q1oP0C8Q0c1zKfgOrVrCDnrus7tH87nc4gpuOCtR4BBCU8IEUKlhEO6vfdeSsW5nBztIUld9Ahexxid37bdcrNuWq91JIRG79p625jW6Q5TQKCQyNC4DHYj6UqdeP0OD2OI37hr4dBG3Lz1KaUYI6UImGIKGAMicnF1bowYGWeRhpC8B2K9N87ITBlvCONIiU0hIcii4JkKMbpOs4TB2MvlcuI95TQYa/r+7avXg/Nm37YpeMG5ZNx73/cGANhgpA4QQgg2QkRmrT08PIwxvnz+ohhP6rZ5/uKVTzEkv9lsttv15eXlya2jvdn8YuHatjV1/NF3fnD//n29a7/mX1ZF+fbV66Ojk+1mc35+9sPv/04uxf/w+WeUpsePHz17/vWnv/4Eo6sy8eblU0g+6E4x0te7h3fvOGtWi+UoK370/R/ko+Ljzz9+/uo5xBElMgWAkk3G80wWzbZZnC5enb3+8rMv7t99sNmslJAAsayq73z4fuhWhBDngtXNann2q5//rJpMEanIqz/95//l/+H/+H+qZH777i2a5SGh9369vGyaJoSUZdlA1B8WwHK58t7udrtf/epXhMAPf/iD0Wj04Ycf/tt/8+cD3W84pgz37vLycoBDybXTOwAMszGC9Po78XpQd0VfUkqVZSmEgCGeeAjsCgSAxAhD/z4YLdznSk21AAEAAElEQVQwioffuVkJjDHfRUQUQhRFkWeEUhyIj5QiQTZQt/7s3/+PEeSwy+zt7SG58mkY+Erb7fbs7Ozly5fr1eaAH+WFHA761loulaBivdwo+ub2reNROeZjlJzrrmt2O92bp18/+/GPfv/OyZ2u7W/fudv3hnTOORd9uPGK2Ww26/U6y7KBjgtX4DAOm4tgou9751yMLiXw3qeIlNIB/0wpDkzAm98HitcGgmk4ExRFMZvuDSyHYWvbbrcp4XQ6PTw8rOt1r7tk/SSvcsaXpy9Xq9XJyYwQonWnbY8MheIBknZWUrHrV2VZFkWWCTkZj0ejkjL03n798tnl2UVTbwXjEP1ms9FdzzkPWbVereIyVlUlZVZMctrTlFJrWsZYJDFgIMiQk0RB+55l0qXYWSsl359UZVlGZ3e73dHB/ieffCI4P5jvzSbTD/7gP6OUfvP113/+V19dLhbrbgMAt+6ejmeVC857+/VXLxFTCtFqbdo+2IGBRfsUGWNFURSjajhYUEqZFJerer1et11XTcbHJ4fVaCKtbduWkquYK6113/eMdUqpTCrGiTGGcdp1nfeWQNRac0GXmy2lmOf5vJhM5yPvj4pS7c2ns0m5NxlVZb48e/uLv/6rb77+am8yffz4UZbdITRMx4UBXG0brRsiMqXEZrM6OTo8Otpzfbtbn3EG3/vw0R/8/u9/9zu/s7+/H2PsjeWUSMlHo5HKivfee2/XdC9evUqrjZSScZlSMqafVpOOsYG2xgi1AN57H1Jd163u2Vi43nddd/fB/e997/uP3n387/71fxddDDGmCIwwghQC+BgjhL3pBJLfNnqC+R/+3u/ff3BH6y71F5Jhngmz6xcXZy+eP93tfJVjH/6WDgHTb2MM//hG4ds9wd/sFW7+d6h2Q+XrW+23dnDoGzUTQkhRVCpnN79PKaXIQ4jGDNEG3lmfUlJKzefz7WoDQAiy1eUmE0op1bZ9oK4sy5RS17HBhHRoy4ZoKGsdIhpjYkwDHm61vqlncBWAcpX3xjkdGgXOrwhDMcagfZZl8+msqqr18jKEsFwuB/lSNRlPp1NgVOXZdDpNCJvNBnwcjcpdvQnOxgTHx8fd87efffbF1y/P8r2D8d5R7+zp+ak1LaMwpLwO/+JwDZqyG5bYcKtDCMaYuq4vLy9HQgHA0A00TRNjXC6XQ6I3E3RgQQ4bNeccEV+dX1jbh+RHlVKCKM42u+2XX3/VtYHwUsoRcd6mYIIlDEuh4rXpe4yRczlAGinBFe06YiI3OWQ4TGqGFzwgCjefhKIonLfG9DFGxgljFAC89y5YpCSk6KLjiUdIESMygpQopQhjXd9HBKmyEMJquym5bJvm4uz82ZNvZvt70+lk2EXfvHkFMTljnLEpegQgCbx1CJwQEm/C8EK8Kj2Ls8uP0xeHt25vWvPzzz+6WG93Xc9UhpSw8yYEJ7KJpflXr89c31ZV0V+m+7e/E3pjtv77j78Xg8P9uF2tDvPMb5bLl8/2D4//l3/yT7/48us3L54vF6vd8uw779w7ff7V5ZuvYtB929p+OyrYxemr0fSQCmaMk6AeHD8yjbN1aJafvbx8PRqNQF+a7VmeVU6j0zZYytmIYX60f6den+3vV3dO+Pv/+R/O2MWnn3z5059/st24n/yH5zTZZrM2Oty9e+fJV6/+V//8X/75X/3lL372y8Pjgx/++IdN0/zaktOLOsYolWo7z1AdHhxUZR5jLLMMMK7Xl31df/3Jkw+/8/7J3nEpaWBQlurzj34xrQpjTF9vgzG66wgAJhjahQF4jzFGaIWQhBBnIwBwLimlKcKoyqWorAGjGyHYeFJVVcUYI1L66OIu9KZDxLIomODaGiEEYRTIcMImlA3Iv5EKbOxsi8GnopgwdeDpyNuayLztahRYlEp3rTZ9oabV3uTe41uffvopQ7JdbRdvL/74D/5EiYphTlPWN0vThuOjWduaFV3PJlNtjav7XIq+qXdbOZuNQkq73u7WtfVcW5TZ4X/73/4//uk//ePt7nK9OM8LpQhvm52kHITquq5eb+eT2ddffPX48eO37RtK2XBzkNIIIKRcbNebtuvbfnBxts4SQMWV9T44xzmXouSs9IZuLnslpru+P7/YDQjN6enGW3f7diklUJQppXrXd50bl/t5nk9H00xUa7Mdl3PTdbtmB8GJInPRf/XN6z/8vd8t81uvv9lkbNWtd7dHVVfQ9XqD032WK1WU0YfJeBY9/uQvflrlhZISScElbfqu60znuENMLoFtBRUyk5iwXm+bpiGE5HlOCKTgTGct4hC5lBiLhGR0/Oarl++99x7n/Pz0LMYomLr/4P5sNHrn+EFw/evnz1fN69dffv7owb0P7t+vZvTly5fbnbhcbk9fnentKsurl6eXBEBlOVfSx6CdDcFdKcQg+AiMLbMiQ0o8xrwsptMpMux0F7wjfdc0TS5VxoWoqNUIETjheZ4Pu6dzhjGya3quMusTE0U5mgspC1LsdjtReK27VXPhkuIieadHbHb3zoO9+fRo/+DVqzcff/7lxVqX8/siH/Vx9k++96PVznz0yadnqxVRDXLhAS+Wi+ne2CedlVRK/sHv3H/84N6920fvPHx4+3A+n89jwuVyfXH+RjeNbfvQx1//7JOEJEWseOlDjMOEy/te2Wo6bk5Pt009poRxPp7P+t4UiHuMaW211kwKQsjBwcHJycmTTz9++/ZtDIZLbpxzwXKBKSXO2fn67d68KnJ663jv/r3bh/PZi2c7L4+2frJdwVffXPz1r1+8Ou9dAhekC5hwCOMBxITIkAASDDEmCEIIiMkYM+Defdcxzhm5PgwMHAtERBz0rt/GDOA/ZSfcEJKGh0hZ9n0fEwohFBdEkhDc6YvzvnH7swNOJAVmfEgEIqXaRSaScxaEvFjXjNDe4t7+3XbrJ+ODfusFZXvzeSbV2Zu3r1+9Pj46QplrbepGizkXE0oYl1wwQpWQlNLBO854a4MLMQQfOWPDkV2wYW5FCSEEcNBghxB0p30y0ANjIaUEpA6x8973l/XDh+9QRS/WS1UV5WS6t3dgTWwuG6MXH/31p5zLe/fu/Rd/8t7+/v52W2/6EEL86vWiNpGKbDYd50r1y/Om7seMs2wvABJF1rH2IQBSRrlLJLoYfUgplfmob1oABIfeBEg+9GF/vD+bjgglp29eVpMxkLTZbRPBXmvlVQGFM3673ZZZuTc/jjGen55NOBvl+9szzX354P33m1395mV/fhb29sYBCXCUmSAJbB9DgJAES6StuxQAKanr2jmf53kETEiBRMQr8xW4bgpZYpAgXhFxUnDR2mBt8sYDAoKiNEYfO+uu2KxUblvjel7mh5JzZ3eMYyYnKXQUqACmGHoXQq2BeSXE8+bZ+fkZE1z33ddffHH3wf3RaPRmsdyud5JxSIkiE0LlKqOIptfoImDyNrh0dcwb3m6WwHfdbrW+0AEQPGdYlJlQedO1hPAUsK3bV30HyUdvt9vd/fnDTrdSsOPbx85Mk7PeWyH4xcXFcr365vkzyiXlfDydFEXRdd39h/e0bndtXeasKkRRzjinzve7elVN9giJ1vSJ8aPDPcBH1vUvXy12u522hvWdC173jiSZwGeZ5JnIlezXSwxhPh4/uH337sktu9rev3u/7+KXX72u6/Dg3hRoSVN88fzJeDp9eP/Op599dMnp/Xu3M042tvPOSEG6ztfNRnFBFev62ppOCNa3uxBcAj+ZjA6P9gmBt29fv//++33fD8gVQBzyHlerZTma3MgCb0wVGWNa91cQHDIAMsj2UoRhyOfcwEWgWS6995wzH648swanigFOuOmL4VtesMN3dPSZlFQQ5+Nitfjm2TdSwWxStG0bkx+VOec4GuV20XZ9Xe8W5UyFENhgN+Z9CGEIcrw6f1AqhBiNRvv7+xGSCz4YF5MXgkjJo7dt2zqbUqIXFxfWxhjIZDKhlOa5MrZp2pZQGIwXh4Z3kHIM0zsAaNt2GG0OptdZlnXetm3bxmitHXqFYcLSti29Pmw1TXNxcaG1Pj09Tf8vuv6rybLsSg8Et95HX+lahI6MjEQKAIlCAVWoKhqbRdI4TbKtaTbWzbGxtumH+Rfzd3rGpruppnqKUxpVBSCRSC1Ce7hWV9+jtt7zsN0dARTplpYWEeYRfu8956y91rc+AchsNsMYB4uxwF2glDJCpJQAgKIodrd3iqKo6/ry8nK5XEacWqWMtZxgb4k1WirjEbx393631+l0Ov1+/3I8Ojg4TCJZWssxihldVNXx4X7EuGxrClGWpt57iH6d6RfGlIj/Oo0wCCDDQHMdowLhG/oOAAClGAAzujztdrtZQgGgRsuL88OLEzOfTHrdIqLkxz/6YbeTn5+cvHr1/B//6/8W/B5YLOq//bufTf7yo+VCISj6WcrjZDSZVm3jAUQYKusAAIyRKOHaGEwgIQQg6ENWCYQQ4DiOLSbee9Gq8FlprZ0lQRAYzP+8c945o3W30ynLMoxWEeeMsbqqqrKsRI0JsrpdTKeMwjTjSdTu7x/t7+0zxsp5ObqcSWWh9w0208Xy9elxJdu4k29n2UBbgImyJsuTxWJWZNnm6nq/kxQpvbOz1cuT1ZUhJkxIXdftcrmMomhnZ+f45PxyPK8WC4AwRARCaK+znRzwBoGsyMG1X3iapsO1Vcai4XDY6/cRIuPx+JunT9q2/fzzz2ezWdu2SpkgdQMAIASCgg4hFMVEax3H/P79+3meStl2Onk363gPXu4f/fKjzz755ItZ7SihjCeqEQAA5EPCQgATvPcAAuCMtdBcMZS9h9ATihFwb3IYbtCC0Dq82R/8F3978+uwVwriuPAohX6ivf5ywCujAYZXhiwIX8EPbcsIDXqcSZa1bWut7fQH9+7dS+PESBU4MYu2ChPIYrEIM7T3PkkSSmlYoYcqEVB9jDGD+Oa13YzCgWV51SgI4ZwzSoW9e29wtacPZo6c8/BBtW17eHjovaeUewestXnO8jydTBbOodHl+NWrfec9jYooSrJOUQwGK6ubVdO+ePFqOltoFyZ1T2IScAWlFAJQKaWltNaKpl0dDPM8Z4yVZdnUdbfb7ff7lDHvvbJGCAEQdBYwTj0ACBFGOKcRBhB6pJQkGGd5uj1IHz58KKU8Pjg8PDyMomh1be3OnYWH2AEICPUIeg8inkCMECWIkJs7IdAJpZTaWms9xOj68wTwmqrir91Ib4rMTad45XTu3c0RAwCYzuZpmhIKluU8CCIwxpPpKE1yhBDwEACIEBFCVVWDAGyA1FrzOAot3U2uEKUUQWTCy/PSGUsxNkoHJRq8dgi9aWrJP/7jP3AeSu1nZVmKqr2Y10JDyniStJWAEDDGquUcQc8JwZT2V7rHZ4cxZw5YD531xgFvrKOUVVV9cXm5s3t7c7iCGI+iqKzr1eHw+Oh1ksVNO59MjuKIJ3HqvG7asqnnlMdxhBSwTtVZSh89vC3MUSPFeHy5XC455918kKcdiikiPqEUOe21wt5ljA06nZxzMlxf6W2myVC2+JNPn0ipEfWjZj4Ybp4cvnz4+J3t9RXZLjYGHVEt6vl4Y63f76ZnZ2ez2YJRGHFodNsotbm2aq3VRgBnAUyMUZeXF1W93N661e93syzjPE7SlHP+8uXLtm2l0V5ZrayVwnuH8VV00E2Gr0fhqaUAgPDIOeeEEFK2lOIoZpxz56hWMlw2xliwE76pJmFfgK5N6cODwSIEsAMAEoKAt2W5qKrlxnq3aStKYb8/7A86Dx/edu5QNH65nI9GFxB6TGArhNKiFfVg0APAEYoYJwgDCH2axp1u7gGw3qllPRqfqVZjICfezmYziOjqylZ/0CuXYrmolBJStUmSGJvXddmK2l6HBQSwMWB9obWq6zq8uxuWIkaUEh4Sga/gOHsFyWBErPEIkrpqZ7PFYLBycTHywIXnLRzMLEROty1OkiiKer0eY2w4HCZJUpblaDQyRkDovbMEY0IJcFYp4ZU+v5jc+uHdIk+bpk2SuMjyYb/fLQqoWbfIunmiy8V0MsIQyaZVQlqrjTFKGSFlMEBllHpwNQkEiTZC6KblChUwPMz+Ok0KY7xcjCmBBNmdzUEU8Wo5d0YvF7NHD99q6/rOrR1v3cbGmtG6qqM1OJhP5ufn51GU/LM//uc//tEf/v3fffTZF19oLZbzWRzzGCBtnEcwY1cJfggBjDFGGEKMMIbIM8oJphY4SjiBRCnRNE25jFwKIITeGaVE21opr0AdALDWsiyNtTaKGKWkaar5XAkhOKfKEIxxq6SWltEoSzp1pX750Wfrq6ttK4AFWtu6lhAQyoCS9sunTy9Gl9JYHiV5lmBErbUU+1tba+trK9/7ztv9TppyksVMNWU5mcWdASVeCHl2MTk9Oa/qBhEGIRJKIUIIhpBgCKED3l6DtOFOa+q2aRrGWHC/0FrXde0caNs2IOebm5tpmh6/enXTat+cx97bqlJFFjFOtjbX3378Fo9oOZ9nedrrdZ98++yTj79+8nRPCBcxilhqrceYOu8BBle8IXC1UqSUNkYBZxCGGHrgDHDwv0hyvNqm/VfEDv+wpQhfNwXhpr8Jdx1445jBGAMMEUIOeKVUzDkAoG1bS0wWJ4PBoF5fX8znwWwgz/Otjc3paLy3t1eWZX910Ig2/LOB7e9hSFv2N5IBeB2GQilF9npXct1AO+e8ddZagq7gcXBtF4ExVsqEREBGoziOw4Mfatrl5TnGdDAYWOu1UcaYxWLx7fhyZ2enFWq5aFqhsuIsSbPZbLG9c+udd95mUUIY/eyLL5pGBGk15tQYE1S1CMAb6XKIq5VSMkLDi799+/ba2tp8ehYOQmV06JAIZxgSa+xsNnPaaKWcsfPxCEIIvB+QPsW+M+hPL0aj0YRQniTpw0ePq6Z1AEljylYgZzvdNMlSGvGT+Vlo6ewVJqRCexgahbDW8R6i68KO/K/bx7CzCOU07GG9995Z7z2A15RMWQNolZLOaYhRHHPOeQhF8x4YazGmjDEEmXcGEZKnbDabOee63S6h1DkXhOv9fh950DaNlsoqba1F1823fyMJ5aYjJ7u3txHCy6pJyrRs67Pzc0ggoRHEuFrOKKWrww1ZN84ZSqNOt8i78Yu9JwShJIqN0tB7gvCyKqVWiLJWqslynna6AMFFuTy5PDsQ7enx/rJaANuWoqYM9frdJI+sk2fnR4jQrd070rrDsxOh2n63+NEPfryzub2///rk5KhtKkpQGsUOgsWyVhYZTPpFjjtxP+/ElDmlraEI4Z3tOz/4PrWa1cKvb9ymUcHjrKplJ43/6Ce/++DBTpTEysjdrVWSred5vr9/8PXXX2tlESJlWQqhFss555QxBBGkFHHOkjSKEx4OsyDGBTBO0/Tu3bta65ev9wMuZ6wlhAUbgOAbSAj1Vx4aPkwV1lxd5muGi5NStm2rtVLSh94ihPFcwXqMhRIAr6MHQo3AGEcJklIppWLGszS7+VveeykkhL7XyR/ev31xPm6Z86A/rZs4jgNfwXtf1/VwOLTWpmmaZVk42kMJUFpDCCMOCfJZnq2uDpQW1XJ6OV5gSNK4T/EVC6Ft214v1yaum8UbbwoEJWS43efzecieDq8tSHK994CyYP0UgAdrrVHae08pJQgHE8ZgFvngwYP5fH52fhnHsbU2HAmM0PBeFCHr6+txHNd13bZtXddN02CMMY04Z9ZaBDzjDAAPpHBWvdw/vHXnLue8KsvD1/unZycrg8H29jZ6faGUUlUZIcgh0Foi6FtRai2dc8ZdNTeYBNAICdHeIHLhioR6HRqFUM1DeQ0YkndifaW/ujr8zuMHBLrTY7+zvTnsF7PJJM3iB/fuAwA+/fTT/+X/9f8MRfw//9nfxWmOEOn1hx988N0/+MmPdnd3v/7228+//NpY10phnCeUIkws8MBBo8N60yFrEcGEMEr5lan71UWJnHWhWPR6veWivAoBRyiKoiiKjNKiaeM4JoQkSUIxUUqJpoUQ5mkWVtQUs15vsDLsdrudo4PXX37xNPndrmwVpVwLt1i0aZpHcUpo/PrsuFzWWmvG6nCVCSGMkJV+cXd7o9/JuzFPY0aA94AAaxdlQ1jcCPXi+auPfvnJ+fll3WqpLKYUAuwgwAAABKEPJ7O7JjbiG7i+aRopdZIkrRBa23AclmX55MmTJ0+ecADANUvmpvBZa4EHCIE8T7e2NuKYaq0wgdbKv/rrn/3d3/3sFx99KQDodzoA8/G8VM4kSeGAd8B6j7y314e8jVlkFUQIUAwggMAjBDxwwbkBvXn2ewgB/AfCiX+QLvEP0YWbkzW02ghhQgiLo7DmQGGQxAhA6KxVRsNgj2GttI5AFEXR9vY2JWT/xauzs7Ozs7NbO7u7u7svXrwol8s4jq13V6x7751zxjmlVNM0ddtcrVOv9/2EENHW4bVheEW0ghCGLLrwgtG17WA4b0QjDbGc8yzGFDOCMMU05pG3AEFIMCQIGyWgd1K0+6/31hKwvU1XVwa3bjdnlxdKO13VrVBCSURJp9fd2N4UWiptQr5l6X0YzBBCYV0SPvRu0eGEvn79ut/tDYfDtml2dnYC4BHHMeccIqKdJoTnSYEpZYRcXl7MqrnVBjhjtcIQcs56xa5q60G3e+/u7aZpvvn2eSNlXvTrVkJCqlacnZ8v6ibs+3gS44JeQcL+N3LyjDEI4BvkwN/0Ug7e3MlhIrLOAACsMQFRgM4656zT16w4MBpdzOdLyjBjNJiIMMaaWngPVKsQ0hBgrbUxLo2j88mJbAV1XGvNOHcQQAjzPI+iCHlACdFStb5+o4f2v3UHhktMptNp3u0CAOKYxxHlxBNKrXfWyU4aee+9U9q0nU6HENIqSVN8eXjqLciT1GoXRxEnLER/J3lW1u23T569PjpO07Rp22fPnqVJcn56mMSk34k2ss08jqKEj8djzrLl4ny5aKIk7fYGVjZH+6+lUe9978P33v7+/dsPP/vs46+//EI3StHWaZQlCTIo4dHunXtFTLOUJTwDFmtDvYeMsk5n5a233+dRvrG541GkjBPSWO9u3d4qOsnB8cHKoO963WkL1lb7nBEt5Xy+UMoQDKXQFKM4jnlEGSNJlmZFTinGhCyXi7Ztw7CojAlXuixLCAEhiHMKNSIEE4K9txhD73E4O40xwZ4s9Ps4aF0Rcs4gdMVnVsp7R8KRyTkP8HW4SOH0fbNGhHonrLrCGDHyHk5mS8bIxtrw1u3N5UK0TUMQTNPYatlUTUSIBxZhYJQK2YBtW+9sbU+nU0ox59RaLWSDCXHOCNlY5yIjBt38B7/zvXfefWSd/uijj/7sz/96MZ2ciQnBkVaOdDIpW8qI93a+XHaLQggR7qewFwi/DX1rkiQBRwldlLXWQAkAwBBZ47USxhhwpQUF3phw6BnjRKv6veGgv3J+Mcrz3Hsfx3Ge5xSTgM5xzrvdbpqmUsrZZBrI4f1+v5FLSilUGgAHKMY+ZonGPm61/OKrr7vd7u2dnU/Pz148e37/zt3vvffuUsNnz54dnZxRSiNCsIeOUOcs58xYi4xzLljahCfchMJ9M+eFofxmMREqQhgLwnd2MoyAsapp6zmFTotFP7/z/fceffHFZ1Yb2c5///d/kqTk3//H/00Ivbdf6gaswhog+Pz84vTk4OGjtx8+evzf/et/8eDB3c+//Obbp88a4bzRpVaM0CLvNGrhvPPea20hcpQGzNPxJG6amdMmiiIp5Xi+qOs2xLtEnnvgrLUIQ8YpoRgAQChWSqqFDEuiLusEPXqnSNsWra70B4NelmWMIOjAbLpczCvgvDVICIkZH6ysZUW3LEthrEfQeGdVC4CzRnFKsiRZ6XXWVwYUAikaZDX2DgFAaSQAOTm5ePb85Sefff7F10+kAJQhrR3l3DjvrLHehXPZOn3DUwvdbZ7nnU7Hez8ajQAARaeTpnmWZYhewd29Xm92fm60gxBa6O1VxJr1HnQ7ifc6z7PtnU0eMYgshPbbJ9/86X/+m+WyilNoG1+LhnLAOFGt8d6GVAbgPQDWAweBgwCqpsLecsYxxs5oiAHGyBrgPIDeAXCF4np4tVgLqgHwD+Ilb1YSN1+/MVZeOwQYY7ynxpiUUgjhlQW7NWH1oIz2EGkpw7+glZ61Io6iXq/X1HWWZZPL0dHR0f2793q93qNHjxbzuUE+6IbChwMhtEFS3zR1UwcC481qFV/H1CGEwPUMA6791uBVaIUzxlitjTGc826eQAgjGqVxnEYJJyyNYk4ow6VZWbXWOqO0bCkhzqjZbLZyazVOk9t372XdXu/goBGttp5QjjEdj6ZV3YpWbu/sJll6fHxcVg1UIjyGwduNcx7IE2mceGM555ubm2tra/PZLNhBegQJZ9QDY60xFiPa7fSLopjP5xhiBLAHSkthjez1ezs7O3e21rE3TslukXU6HQihkBoLFaV5nOaZsx4TPB57CLz3ThtvYIA/AbpyhAw+04QQeGUvh95sIIjDAIDgih18opy3AAApJcIAIYSvz+xw80OItQ6Bxsg7ADBSSs7KOWcx59g5h7FnnNS1r6olwRAh1O33MCWhSQr43BVIgMlV0B2ExhgPoTM27EbebBTCjEr29o82NozUJoxrWRqHUFdMyerK6nyxqJYzq8XW1ltKqdls5qASplHSVE3pLehlXYqp9/DO7h2E0DdPn4wPD1ohsiK11l5eXt6/93aUdrqdCCBtbFsLuQJJmvIi6Usxq8tqPpoMBisrvf43X3/18a9+NT6pv/f99+7fv/Ng6+H0ZFzXdTfrxzSJee6Ez+Nsd/d2wokStcesVW7Q3RStuhhPDg9Oq1omjk2evDg8Ppdaf/d7H7I4UtPJ0cnJZ599JpU6Pz/vr9063tyhlLdlZbWBHiQ8JVAVRddaSwjKO0Waxsaq6XRRVZU1MmSgYUTny0VVVdY6KSVAiFIOITTOOm+cY8FvPJi2uWu2f+j6nQ3WQyFgzYV8XqWU1oqzDFwLHG4uoXuDChuKyM0UK5XO0wJGCHlUVmJ0dtbW1c7G+uO337LGAI+iKCEIIwAWsymlMYQ03CJhgpzNZu99512l1HK5rOvaAc/iKMtzpVRd19a5ZjHq97u3djYe3L2FKDJKPX/28vRkbEyTpKkkBkI4nU6l3qURT9P0TaTkRr4chrZQ4AJ2Aq8p3BBRhNC1t4lwznHGgtOztVeWCWELE/qzkE8Rbtwsy4DzIZYzjuOw2phMJqJpwRsiEQCAh8A5YDzwCGMeRwQTCI/OLiez+R/9wU/mk+mvfvELrUQepz96/31q7fj0RDRVnGYGOAQBwg5iDz0AGCCMECTQe6O9N5YwerNfCEhMeIYD/+PmqqEb71iEtFSiqcvZdGOlPyyy5fTi9YtnKSPfvnx6ePCiU2Qekw+++x7C9OT8rJn4vb09COGjR7elVp99+vHhwYsPPvzB2vr2P/3jn7z73qOPP/7s2fPX3gJGkVUtIQx65713FhhjkLVaWyGUBX6xKLWQnNO2bqbjajqdaW3ubm8RhBLOpJQYeOQdBt5aW85nYSyrlwtRV+F9GXMFpK8O1weDXl03ysGNjY0kSb744guMKYQwwXRjs7O5uSmlnMym3gLOYwgh8iCJuXcGOUuhz+Jo0Cm8lUYpi5AyimEEMZIGfvLpl7/4+Ucv9l47C4pOCgCq25IlFAEbgFeMkIfOO+CveTA3IFZouBeLRZ7nSZIMh0MI4WQ+U0plWYEQ0upKC+2c89bCKytcIGSDAIgT/vDh/Tt3dqxRF2eHv/zlR2lBf/JH/4Ty9G9++vc///iZl4u8KHKaSGmunNW9BcAB6BDwAHihXb8XD4eDtm3rsgRaoyi6Xi0HTybsr4GE31or/NcQhTe/Lezswi0XZtBwd4XgdWOM9U4YDTC03kmtEKHOGIQQp0xrLerGaM0gDuCzkWqxWDx58mRjdQ0AsLW15SlCBC8Wi7Isr9RGGEN4heFDCDnnAN8oyfFNkAH+TT2nMQb66/UKQsC5K/5T3gEAxHHcKboBtQpBskVR5Hk+nU4nk5m/YnFahMB4PJpMxvcePtja3rDe7R3sL2cLbV0zm52cnWHKL0cTB3wUJVXTsCij3kZRhK4d6P21asBaa7VOkgQhFA5gIQRjLM0KzuOqbufzRVnWmLBerwIOnxydirYhEAEHlnXtneoWW28/uJtGRFu7nI8XS3l2ehzH8SpPlQWHx6d5p8vThLKo6Pa0lhBCazVy+Aq9h1ddAmPMQ0gIDauHAEy6a/zGOeicM1YFhbNzzjrrvddaYw8JITg0HPBqIBFLk6U9zjJrbTfvbmxszGazzz/9rKoqCCFErtst7tzZSRJWNzMAFcGk2+16CEKMMIFXGhkpJebhamMIryxGw9MUNCw3tfSqjn38yy92dqbamtu3d1f6gw/ee18Z/eVXXzkI8pSdn1cQoyjGPEKtkEnO63ZpnGpkM5/OOYw6WYdFvCi6t+/dnU7mrVBK60a0BloAXJzFo8nCW2sAEmUdYe+xT7PO+nDTa6QENI5ELGaYMMITHmdRvDhZHCYnPd5ZL9Z/8N7vNE2VpjmjSb1ogKPeeuNQKz1EJOsOIfSQRFY2jZhalLA4XtRyNJ4um/rBW29/93c+7PeGo+lkY2f79t07T58/u7i4aFq4XCys8c4BbwEljCVRBdvRaNS0bZIkaZojxKzUwJMkLjDWgSkzr+ZN22ZZkee5Meb1wWGS4DiOrb+GBzEwVlWlQOiq+w63RThI3mwkvbdKeeecEK1PrwIL3DUv+qbrvJlTb6oDIcQ4RllitTXCKA1bYRfzZjpvyqVIkjxiJI6TLJnubO1cnE+19tZqIVzTVJRSY9TBwWv3+z/GBFbVUojGONs0aZLFADhrtQPeiorTXppwJWpkcJ7yXrdYLhuEo25ndbaopDKj0YUQIsvTlZWV6XgcXnM4+ENFDjUiMA+klGHXED4KRDmE0HoDgEGIIAQwosAjCLA2Js+jtm0xpk0jRqNJsLfqdDphmwghdN4558IAMZ/Pl8tlVVUBcgySJI88wAA6DACwHnoPAMQAc0wAwnRvb//rr7/N0+ydx4+VaE+ODh6/+72Uk8nF+dNXLyAGyljjgTdaIxIA1ysoyCMEg789Cc2fvbZnv6mVV+wqQkLCb2jVmZcRYxGNqsUy3926s/Ho5Pj1t199eWtno6nLJM3/9E//VDmf5/nDx+/8fpL/9E//dj49Pz+vT0/2ByuD737woKzrv/jzP7v34O4H3/3eWw93KYaU4qPjs7Ksy1bm/Q5wzjrjAYAQI3j1hFdlE3ZAQgjZtgAAY8BoNKLOFkVBCJFSOueUlGF7xRjr9/vhUw1W32H/JfWUYBZxvJiPj45OEKaDwUAIGap/uCf7/Z51ejIdSVV7j2hEPUQQgIhQrYwzElg2vjytdta9NZyyKI2EANKYpqqfHVx+8tnnr/YPnAU8jjCmiNA0D7EWCCLIMMaUBEJAcCwLA67WuqqqOI7jLI2i6OXLlx6AsAScTCZ1XSdJdXFxEcJNwtNk/BWzBCIgJUhjIGVrjLZOH58cTCajjc2VW3d2v/+DH+adflxEBtqXr46qtm6EZTRCwc8DInR1onsIwOYwf+/9D27duvX69euq+nhRCQQ8I1hqczVqB4O+K6tliPyvBc83p9pv9QrgN4Hf8H5vdpEYYwRxkAI656w1N2RGpRQw1lsLANBSBY5hXdeyambTaXg8m6Z5+vTp/qs9QsjDBw9WtzeyLMuyLGBy1lr4xr4jMBABRs658A2BtgwAgNcjjb8JrwNvNArXY5JRFiGEAMaQeAtkq5TQccw5Y3maiabVmSqyNHwCG2ur3syEaYIn/aycnp+fTqZLwrgy9ujkAnh0MRrNF6WyhrO42+1CCimlKKRxOm+tNRCG5kA0DUJoMpmMx+O2aULSTZjQyrIsy7JtNCJuMprMp3OjlDcWQ4QZS5MIepwmccxJL8+kMlWjzs5P9vb2Gg1o1Kla2UpVjUZswaI4JhQjxL33EHpEaSj7CONQpjDGLgD46AZRgDc2CgGAkUpela+rkPEAfXlrLbAeIQSDSwbGwEOCSZEPiqLYWt/Y3t46Pj4+Pjg5Oj4QsnHa8Ght99ZGnJDL0YnRenyxVEpBjDDGgzgOly9N0/l8bjDxzt1g1YwQ6EHTtKEjeRPWcs6RyXTBeNKKut8f7Ozcyrudpmm++earu3duT2azxXKyur4GEB1PLo9PTnZ2dqpqqa2Ssp3P3aBj+yvDnc3t2zv3jHFVU89mMx4zzrkHXltTFIW3VBpZN8opv7LWBUpdXo7H5zMCopPjyyTurK5skYPj/cPXk9F4bWVtLXnUltXhi8N7b93aHG5V9bJum3pZCeESHkmt5ouKM7y+2t+4+wBQ0h6PIbRR1tlNu3Ujvvzyq6ZVd+8/vHf/ofVwspwfHB1ubW//6Me/X/QHSZJcni5+9atfvd47aGVblQ2AFEI0Wy44izE2GFNn4WK+nM1mzttutzudjsL1ppQmEOZ5aow5OjpSWgKQY4wh8Na+YVJrbXjKwv/DfSBaiRDSGgSWn7UaG2iMadsGQR4Om0ACCkNqcCW6qSM34yljrNVEKV0uKqdhwniSZBD4ppZHh2d37+2giMZRmqb57du3Dw/Op9O5FbW1VkqJEGrbdrFYVFWltQ7nGTCaUhpFkXEuamrrHe8WW9sbd2/tdjp5LWqMsTG6qerRqGU0v+kJrNPe8/Dy2DUkEKY9jHHwSoqiSGsdZtMoipxzYQGBEArDxw2ZMWT/hPy3IDhEAE6nU3cd7gKuR8kwSyVJQggJ3g/omhQZSBgkIYwxBN1VcoS13nnr3XxRZ1nx4uWeqJb/+Ce/9/jx46PXe69f7d2/92h7Y+Pu7s7Z2ZlwRjuMPEJIRhEzHkCIMSKEMGCB99JZDK5bwPA4/dZoGN7+jWEGhPDy6JVscbWw+y+eMW9/9IPvlrP55cXJ0euXddv87o9//8//6q/3Ti6//+H3it7l5jbf3dl8+SLXRuzu7g7XVlfXVxCmu7dOT8/Of/b3P73/4NFP/uC/uffwwX/493/y5Vff9Hr5UnkAAAQIQICCoJxwSnErZRJnBKKyLDmPekWEMAi7rVAxw6cdjHu9951OhzFWVVVd1wihoO65vLxcWYsAsK0oZ3N1enpMCCuXy9OzM8b4xoYPutC6LqeL+Xw+j+PYCk8gUs47q52h0DhnLPQ2iXiRZVIKQhDEaDqfnZ6eXlxcHF3UJydnlNI8yy4uR7P5vNvN+/3++eUF8BASdD0pAg+BdT6484bklPB2+v1+lhUPHjygjDWNOD8/Pz09jeN4Y6PjnIPyCldXSml35c/hPYgi0Olk4/Hll1993pSTo8PXO5sb/+bf/BuWoU6n1+kO/uW/+udbO9t/8qd/8fNffCbOppiEDKiwnscQegQghPDDDz/8R//oH92/f/8XH/3y4ODA2rMoSawHcr54w/UZAAA8BNBfbfDf2Cz8toPCb6EO12ssH6bMK8gawV8vI5yVRkOCPARaa4AcDisJIxihjJCmrieT2WQ8vjw7c9pghJSQF3VDCBn0+7UWy6oMJmmhs8eU3pClQpXw6Ncw4a+n9jf8YwAEaZoGXWKohEqIcPLpGoWeJvgdhRsMQh+i3Ywx3W631+sppRCGW1tb60NalvW8nFyMzo9Oz45PTxCmncGQGT8ej0fjWSsUpVxqWy7nZdl2NxNrLXAeQhg6l/BhzWYzUTdJkgRzuXK5FEL0er3+IAYIl2UJAMjznPKIMdY0DQKgaYSRFUYGA4gxqpeL13sv19mdVmoDMMNXe5Y0TaO852k8Gk+nswWr6ySNw/iNMezQ/Ao2CC3pPwj6evMX4Joxc1M8b/6EEOJ8ACkNhBDhK+EDo9l0Oo042tzY3dnaZIxZe1Vyvbd1rYRsMAGMY+tU0y7XdzfiOJZaYYy73a7Qqq5rAICUkiLsr3MxEIDgWq9x0ybevEgIISFxfjFbeK//8m//8t7bO2nECWsfv7Nz6/bO0RE+Oux5D62PVAnvbH5ne2MX2ye/98F748tytqZ63fX7t9+5d/9BkiSHhweffvYlIthaq7XFiHhNgYqmap7GsfSw29lAJNOy/vbbc121RZIupouiqL/yv0rTPE7zAuXTycyS84ST0cXru3fXXx2+fvriudKGc+4c6Hd7O5s7q/3i/u37w60doDGYVXsHY2v13t4poiBJmccNTaWDU2XOXr+eLBblxfn49Ysn9x++vb29CyGWbH5vc4VD/+r10avnLy7GSx5ngJDd252Es14/Rxmt6wVgmlHQ2hng1iGvlF7ZHhBCJuOZsILEmAJkgTZeEcwA8EppCH2SRFKUAAQYwACIKMMQOW2M9U3C8/msxhhyntR17RyCgGHtsHZQGq+1lxp6CIwFxsaUGIIghARD74yxljFGCIpqhIxi2k4mk6lo45jjJPrm26+kapIsz9Le+cU8TYrd3d3p/N+dXZwJvB7Hsce6lg44DwCWGiXZwAAGaYqgMT5qFG4N0Y4771knWb13p7GgsCwCiIjR41sPmWMvoqN5fc7Tgic8iftnpxeq0bdv3ZULNV0sirVe1IlHkwmw2HukpDMaYEQXomZRkuZUCAExhQD28kwI4YwNo57W2moT8JI8TcOJZYwJAq233367lZUxVkqZpqkxpt/vL5fV2sYmIeTk+FQajzHTgHjvPYkAhAAapZ3UWsiWUpx3UwydaOuoixeLmYDtRTP5y09/9v533u3eu/XTv/rr3i//8oc//tHqer61u/L5t8/y4QaOMkmmBqBut+8caKrGAwiJR9ZHGcMYaq3bViOECWbQQuOBd8hZyHnMmHfQQh6hKG6MnM/ntTS+UVq2eZbsjZu7LbmQ8YtT9fajt+7eHf7V3371/MXcGPT5R6/mI78/uAToDLEmyuHm7bXf/dFPNjd2x9OZ0Gh392HdiouLi9H50VtvvfU//V/+1eef7vyH//B/LGdgZZhKbYRScTeNGXJKQppEjLWNlMICixEkELAkSvvdqJMwCGmWd42dUcaatkLELxfzJMJp3MVec+QIgPXlWAmxnudra2tSytOjM8ZYUXSllKEmZllaVWUjRWgTEUJxHFsHMAPCNK1uB73uvCzjJBIGzo01UfLl3uu1tTXk3fRitGzE06OT/f09RG9LmCVFb3x5zijdXk3WVocvX77MEWI8hpRqj4QyxiMPMsrTLAJtWSMACcUAw7Rb9NdWkjRVyty9e1cpdfB6HzM6Or+YjUdKSAIJpdRoXVeVV5ZQigGMEYPOydLmg84vfvbFrz76/Hd+58P7jz9k+dpa5GRtlZMrK2sfPHzv289ffo6+ThnV1jRCxjEFwBtntAZxDL77wXd/58OH3/3gfpRm3/3+20L/sz/5//7ZweGJBZAQAgAC3iNj0PUWH3llKHnzkLjpFcKod72HvirWzjmGMEYQOe8A4JRZ75SxUZJgwrSFWjoHgFMeSAsJjmBU1pUnJDguyFacjU8vzs4X42ldVQQiDFFwEU6iFGP8/NlLRCDnvN/ryVowwiqhGcWXp+fdbtcD0Ol0GCQIohZRhwxG2EN/JYXw0DuglXHOEYylVEH7EOiWcZRYa8tl1SA4HA73j8+khVHWBSBaLtXl5VIr1el00rTwzlYLZaxOkmTY2+xHdU7iP/uLv6Osy/BaTPH27dtSS4fahMgNHpVNWS5raDXGwBhVT1Cn09FGAui8d9VyjqDjnCKE0hytrhbz2RIIiYiTuubxkPN0PJo0tXYOYGK8aMvKYEyVEsAZCChyAEPEGReIzYT9dm//h7/74+Pj06cvnicRXVsbKgOsNy22JdTCGwqIakQrBSI4y7JmvmTdLoDIO+cpk1o3SmMeOdVARChlEeOEEAyId8BZb7wRWgtlgAUAIOu18/4Kh9AAI4QJtdYLIbxxGKNGTYsec1AcHL6cTM+NVE3T3L9/f2Vlpa3rra0NgmG5qJVQWZK9/857j+4/cMC3Uo6X88vZBDY2yfo8iXFEyvnCWxfoCwgiCrEUkkeRMUYqFSyzbjAtAjFXosYQYcTKZSuaJkvJ7Z3dnc0tUTXHB5Mkhbu7D+7cubt7+75odUwf3L3zVqdYHU9ao3GRDyiLMLySuUOAkixjkUGISqENgHGEEDQIOuPqshKukd2CD2+tIed73YhTlmSARybJPItwv2F7rw66/V6apkdHp3XTOAsvz0fvvv/extq6kcpbN52N9yB88ezJ2soqhPBXn37qnBmNLzqdjEf44uKikY0QDedxXnQHg2GRD5aVkFLO50vvvbImTtN+vz+el1mnuJwu6rZiUXw5HjFGopQlecwY88Bg6J0z3W4vOJdFPKnrmnM+HK7euoW/+eZJaMCNvqKwhbDTsBW7IqqQK1Y855bzKI5jpRQE+IYTFEUsj3NCqbYmRN0EjN05lySRRzDkHUCEwlTtve/3h2VZVs28LGshmqZpypJg6AEA29vbnU5n2B8QghirIp4Wec+DqGlq4CwlxDmnpZqMRrdu3QLaNGUpjUYYai0c8FZLZUy3R4s45YSWy3lbVqKtEx71Ot1t5/BkgmiEMOeUAmfapm6rCl+LtjHGEGJjHAAukA8AQN5Dra21XgillLHOEWpvdsxhSArSJiklI1fecL/FWQtqeCFEcOYIUUPee4RhmOwh9ABc2RwBiCCABPqYRSzmMWdaS2OM9RoRFMfUe3txcf4yZndv3+p009F8XCnBktgCz5Oo6OYr2zudlZWTi0uMsXeQdDOjdVNWAPk8z7USjBPgEfLIGCuUCangEMIoiijDxlngXV2XQoimqmiccIqbGmmtZ4v5vFy2bd2K5v33352OL6typqRaW+lkWXa0/+yzXzW9FdDppHnaH13Onj19tbl1j7IMYnZwfDEcDpO09+Tp3snp+M6t299598PBcOt//Y9/tqyrxWJsPVCtUEoRzvJup2pqpYW1FmLknGuN5D6mEc/yCEIoZcMjzCnB2ELo+t1sa3P1+9//frUsTw5PPv7ol61oOaHvv//ug+++9fTp09Fo5K51U1mW7e7uXqHlGN8IW5xzxjhtDYSQEBRQKwhhHMeU8rOzs/k8rus6WP5V9dIYw3l8MR6LpnYEt/UyjujKcOXOrS2K3bNne0a30GkHMAIYQ4QR8BCNJ+M0i1dXV5MsrZqmqsrRaNTRmnP++vXr6XQqpUzTGKyshJsKEwgsgBgEiY21NiABEMBl1VZ1G3HQ7+cB9BoOV2E7GXaH2oIXz1/96vOvv/z669lsIYTmUQThFSsikBoDezemmEATEb/a79y+tb29uTqZTBbLGgaSi0cOAujRdSD1b7gtvYlFyWsG4s0wF74taGcAQoQQTAm65iswxghEzjnrXVh6QecAAMGId9EsRNM2VTWbTGeTaVNWcRRhABG4iuMMewNrrRGaEFIUxerGel3Xn3zyCYQwL3LGWN00i8Ui2JOHqxaIyb81HzvnTHhHN8kPN0wr5yDC4TGfTifj8ShmvKqXbd0QghF008nl6Pwiilie53OMRV19+O7KZLqo6wq1vr+S3rm16wAwxkScA0wciPM8NQPnrEeIIUgOTg/ruq7rMooZZcgZ66DN8/T+nbsAOmttU7VJkkQ8CYGZTV1nmZTaaOUBRM4h561zkGKorAXAIoygN8DCAHlCDJ49e6adl1KmWffDH/xgOitfH51Erep3e1GSEUYBIspoBzyllKYwTjjCVFtjjHHGWA+od8YBjClwQcdKEAwgsbfaeu8xgA55G0AF7yFCy+USAch5jDBwxqpW1HVjrc2yFCLQ1s1yOsuStNPpDPq9teGAc65ku76+LtoGQt+2LWfwnXfeefzWu8enp69e74Hl/IoGzlg3LxAiEWVt3ZRmSQhx2ggl0XUkN7gWCoWnHkJI6kYiTAFyRsty2a4OUoZjJarL49OE0Nvr0TuP3/vOu99N06yV9i9//nNRj+uJ/aM/2lnJMyFhkQ+aRvAoGvQGlHKhFVaqahpCIkYj4/1iNsUI9bKsn6UZZzxmK73uzvq6EtIq6Z1rhdK6tR5ZaCFuzi4uKyHjOH65tw8Q1FpPF/PV4dqgO3jx4tnx0TcEImf1qxcvd7e2vfeTpYoiniRRmt0GBsVpurW7NRgOAUBaG4AIizisFaU8iqLZfPGLjz423lW1mC2WURLhiCxmpYKuHoksyzqDwrkimAg5Z7Sz1GHgkTWmsWK5rMqyDiFsAS5T0jjvIIQ3Hy4h3Htv9NUhhzFEiHDOGbvRdDlwzeyjlPb73WAYAoBDBCICjVFCi0YISmmaA8qiiFIAkFKqkSrmed2q6bysamkt0NBq4wmG43H5yadfEcw/eO87/X5PKieVqxsVFXi8XMQ84gRraz3209HZO4/u51k0m2rktJXNTFSYUUqpFnUe9Qf9fi/P6tnCK1PE6fb6GoRwvJg5JbUyhGloXSW1xGXOeQh4NcYE46ZQHTiPjXEQYoKZc85oa40PoYU3pgv+WsR1A7uFCnOzWV8ul4HNXl7H/GTZHQhhURRJEs1mC+89oRj4wAANChEAAfXeI8QIYVkScU5CGhunAFGMCMPI14vp0elrHgFhSkNSkkRru5voU4Ii4pApOimN+PHFEYAQYQygB9owjijlcUQ5chBC57yQWhtpnUMYY0R5HOV5Timt63pZLaQQUkqr9Wg26xa5NcYIuShLqUWWpVtba/fubr/76M76IP8//uQ/HR4eLmZL7P27D4cP3n/v1q1bNIp/+clnf/YXP52WejxZnF9MLicTa3wUhVRl/803r7a2tu7fu/d//R//7VfffPP3P//Z2egyIpglaWvUxcXZ6vqG9V4oZYGz3lsl9XxSijrGPQjhZCLW11ezNIYAjS7PiyxbX+l+8O4jLdXBsHe8/2q5XMZx+t77b+MsCVcEQuj8Fc0+TDzeXzH4/bWySxunjMQYR4xLKUO2b5Qw59xsURrnGGMeIu/9fL5slHYALecTrVsKKeM4iUi/l66t9tIYaVlKZaQ2ynjtgQXQeuuBzzbXV1ZXh6srPWum85m2XhmljJrOZ7PZ7PLyctAbJlEstHDaOGeE0EKopmmUVsYo7702gGGEEMAEYAiMBYyxvNOljCultDB5l41HF3/+F3/zF3/zt89eHixKGfgfwXGcUooggdBQyvM07WQUmlYLlCfxnZ3NO7d3Xh8eTefLYKPioQPgpksAYWdwc9C+iSiElcoNqRnd+DJBCLynlCZZGsfxFXP+KkrG3dh8kSCi9l6IJmwYl/N5tSxF04auPfyYq2xj7533wIHQiARgbyXPIYTBKi1QWEJfGA59eM16e2P1AG4eYQ9hIDP+ep9yFTLpnUXG2bAWkbLVRtTVrC6r/9v//D95Y2eTqXh0p63qZ8+ezcsq43jv5eT0fCwlbNpl09q8uzKazTUwnV4BCUDkiqKBr0h+SCl1dna2WMyKPM3yuCwXCNgiTSilSczH4zHGODgNRJwv5nPZtLJtw5YEQYixJxAjBIyx3lkILcUUeAyAIwhnWba+Woxn8yROCeUQk5WVlaqWk8lECIUxjDk1AALkOSPaGC2Fb6FOEuK9sS5oDR2CEHIMAIQeBAWMt/5K7+ut0uDauM9bZ40CwEOIIsa8984Z1ToptRBCiNYYa6VM01Q07XI5b9Iopmj13q3Hj+7VdTmbOSOX8+nIWosJjCNEiUOEQIzmy8WLVy+ni3nR63c5RwgNe32bFZPRSCybJIobUwkh0ji54aZc8X9v5F1CgZWVPoZ2LqRRbm11h2H79OCg18mSOH7/7bcfPri30omt0dPpReSF1tH0fKkr5zzQwljuykWFKNrf35/Pp1prSJAxxnqVdboAQGCGGKFeXhRxwSHkFHhLLs7Hbz+4H/NIKXl0cjyZTJumCcLTd7/7vZWVlfX1dWfBcrm8vLwsyzImUaDDlPPFYNCbTefaytHsfLlcZr01CwFLMsxgWS7LulpZW02S7GI0ttYrjapatK387gcrUumvvvrm4y8+Qwi3UlMWFf3eetu0ViPCPASEI4iB9Ve7Igihs8A5VNcyGIBTyuM4VcpMp+eEkJvsL3hNCU6SxBjvnEPoSix3cxBmWQYhMEY75xFCSguEAWPMehP+c8B755U1TjuttbGeUqqMNt6laRqksUqpciYuLi7G47m1hmAIAYaIRnGUZMXLvX0IMI+TO7d2WyGUdmXVZlw4o/NBj1LqKI6jHgQ2S9mjB7cJtsY5ytminGNGeRxdjnWWxkUcc8o0IZ0iZ4QiDMumLrKIYC/b1lhtWllqBxxkCOzef4tx0rY1ZRhTqrUNfU+SJBAgzq+oCdY6CBHESBl1xSe7dpoL+ArnHF9PUeEPpZTT6VQb6bxBGCyXJaW0KLKg1A3Gw2ma3hCCgIcIIWRhaJwxxoxQQjCGKJjlaSWFbjtpFKe8VfXByatqsdzZ3NDEdzo9HGFITNXOW7WAmFLqIw4xRnXdIqg73YRiVtd1RqGxRmvVNo22jrI4iRPKo6IoOIucc1I1EHhoLYWQRJGECBHivSdEL6vy8PAQO8UjAoHJs3RjY7C5Puh3osdvPex0OsPh0MXD9c2t1c3NpLOyd3T58vXh+Xi2c+vuUvm91/tZBgnCzvj53snnXz67c+f1u28/2Nnd+df/6l9+/NmnT54/r+eTOM8oY0oJTCCLODI45OABAIQQzuuIRwiSt9+6d/vWzmI++fSjJssS7I1YztM03Vjt/eEf/mg0nkopGYdPnj8vyzJYVijjAoPE2GttzvXBcH2fO+C88xbHWEoZRZExBmFSNyUmqNsf5EWHEFLXtbae8tRj0Rv2ik6SZzFDW9jbQa/oFMnmWpcTP18sJvNF3bRSOeW8BRBBrGhat5U8EYRHmGAaUxZRQsja5jpPojzPh8OhaNrJZOK8y7sd1dSNFMYbjCFJk+CgTDBsqmZtrY8hmE+n2joIsPdQKu2UOTw6+/hXn//nv/irTz/fMwAgDCjlxgEAgLUeY4cQJAQwTCilDEFrtKhKGnsELWeEopAIbTwgDkAAfsM6gbhfdwlvIgrgJqzhGqS5+nMEgQeI4DRNu/0eY8x6DwAIMqW2bd01MS6gC8tqIVsRqABaKug8J5TwSAsJIQrkCHiVu4oQvMpaPDo6qkUbCK1Zll2OR9bavCj6/X5Y+TVSBIJCWJEEoWYgCVlr0XWjcMO/xuDqvVjgq6bFEKZpnBcp41CbhmCNvOh2i93N+8P+4Pjg8Pj189aKhIJHb91fGQwBiMYzcXA8OTx41WpTdDvHRweIAMyCeA9ZA6z1zoJ5tZxPZ3VTiraWbeKccV63bauV8hGrqsoZSxBujdBaT6dTDKBSGgFHMArmKCA4QSDgjMeEpAkDDgFvkijO0tRBxKO4rBtEGeXR0cnp/tHRyckJ5SlEWGpTi9YDQCk13rVtmyCepjGIYuA9JshjjBFmBNM0CtFThBAIgbfOWOOMtVo5ayFw4Eo9Z6EHkLgoZj4cJRAzQikmFJO2bV0jnFAUgvXhoD/org37g2426GVnp3unJweLxWIymXBGOp2O1vqJlbPaa60vJuOD46OqadOQJLlYrq1tQEwXABGIsiwDzlXL0jmHr8bdK+v9m76BpFmH8xR6zWiymDd1KXmRFPmAU2KkvHvrbhGnppVxHDME/vD3fjib06oU3pGIxQiCiHFCEPLOO/3Ww/sOu063+/TFy6PD08V8KlrdS3oUYGiiyUXtpNga9te7KzFFZyfz1eGKdUA0EPokS4tOt0vusbaKGWO9ohdF0YtnL40yW2sbr57vleXSO6OliaNoMhklaWS9sV5vbA6ttYQBFpEu6wAECWetVKPRrNsbeICn03lTK2XMYll98+1TAwDCEMc8zvMkK2rTtk5HcYIpoZR2ehlhWCuDEUGAWuutAdaYuhLWAM5ZyA9V0ohWwWv3vXCEh5sAB/ErDqAckFIaq4xxjJGQYuecAwA7Z4O/9+XoAmNsnJNKWWuBANaBgMoqa8q6VcbVVRsYRlrr0eWiqiotBcbYOSC9ctBTRx0Ck+ksKy6PTk7CIlk5Dwj2wAxXepub6x5Y5MHWxuZwpZPl0YOHt1mMAEaUk+lsBglGBCYFHfa6EMKqKrXWScSSKGZzAqCt66ppKqEMIhSzNIoZhiTNOKWw00m1br3XmBCgDIDGGNW2tfOOEOwcDHsTa62zxgEXcJZQKwkhnjEAQNu28JoMmCRJSMe21rZtwzkLPqwQ+SSNEELz+RQAF5JdvPfGhNxwAyHw2jmrgbfQI2stMt4DRxlWShktlVImxgADWem5000NXp8eP3v1cnt3x0KNGfJAGVEWvf7qIGlqhaHrZLQFwbVSWCMwJR46jGwcwYxmSdHhUeoBslYpC7x1GMM0jiICnbHOOcMi1QoAUZrntq1Ojg4Tjh7evjWfTxlyp8f7VTm7tbP9g9/5sJMXdV3+L/+fv+gPVj748Ad1W+WdvDHWYsiK9F7/7ZWdbQjx3ou96XjCKGu1efF6/6tvnn74/UcPH73V6XXX1ldbrTDnyllC0O7arvf+8vJSKZUlKcYYAaj1YtAvBt2V73/v/bcf3J9eXpSXF1pJbM3Bqxf379+PI/aP/vAPXr3eOz27ODh4/dFHn2CMV1dXOefLqgnrM2tNoI7aNyDocNuHjMfQqAUAKXSKRTFYW10HCC6X1WS+VEpHSQYXZZJTa1iU8H6RNotZ1Vaz5WR97f729noUY4gsI6CWSirjAKSU1STz3vMkHqwM+yvDvCjyTjfJMgBA28q2bbM42d8/2Nvb00IhzAgjEIPgs0kRhRA6Y5VyDgAltfO6VWA8mn3z5On9hw82t7akVC9evvzbn3/09PmeAQAigDC1HmKEPUAAhhB5iDzQWi/m5fn5aGvndpplVdm+fn308uXebLaAEFgDPLQOwDDC3/QEyAc2pX+TqQB+M2baX9m1Ae+9cdY5BzHS1jjn9PXgobU27gqQCwjcdYKottZ6Y8POIgAUBCILIAhUzOuf6JyDGFJK27adzWYewW63WxRFsFVIkuRG3fMmChigU621FjIsBL33JKyfnLtBByEmQThjISrLkjHagzkhEALjnCgKfnl+UMS3xxeXJ6+enB6fAFk9urfz6O72W/d2F6trQoAtTarmi1d7+0nWgd4tZzNIPWNhvYiNAVpYrW1tJAKgSLOqWrZtm6WxtQ5B2O/2MIZWG61NRBlFmGHCMPHWRdeKHg8hQIFVasumlLJlFrXQAKetUZMJK7Ik3uhlafHk2SvK2K1bt2azWdM0hJA0TSHCqG2lbC3wEaeYkiRiLRDAe+QdRJAQAjACECEAk4gHh+2Qzu2MtVobY51RQQcHvAXOA+c9dMCitmkBAIzwJI2zLCOQ1HVblqVvzWw2cd48eHDv4aP7CDtt2ld7zy4uTqpqaa2mFENEICKY4Pmibo4OEEIewc3trcVi4YytliWltK0bZ6xoGm9dGsfAWCWkfwPuwtdeumGlS+I4XS5q4DTw9OWLw3pZfufRw52tTeCUFNX6xmqWRAiBTr9XCbW9vX12Yj795ItnL17du/cwy3sAeUaAFPWjh3fXNoq6LZXRStayrhBmkltiiZaqsWWzXFTzaU7oxvfX79269fzZE0rjdrlsG9MIZ4Ewvul2GYvQfD6ez8ecxMfHh21drdy7vzfft9INh8P79+8zhj77/HNMmPc2KQqIrZKNXNQA7ty+d7dtJQTEOA8RSZOC8qgV5mI0er13sL6+2ev1nh/tAa8RJdKqcno5Xc5YwnqrvTzPIcBRFGEAnbMIwgDuUcQQQkKotpWz2UIpkSSJtUEIh4L3HABAyMYaTyyKeBLoPNdEBA8AcMw5b4x2WsuwcUDXWelKtpxzjKhzVlvtnPcOamexd9hBqZVQallVCCFngVJKKA2QpzEHziqrvHXEYqHk2eVFI1pE8GgyfvbyaczZdDF1EPRXOoG44L2HwO/e2o0ZxdSzlPZXCwAhpohmGIQUSmo3e6uA4qqpsbMQc22V9SZOI8xgmsdAKm2tgZoA5Jxe1HM8OfbQSr30QCHmPTA8IhB5ISsIoYfYOuu8NlZK1Wrj4jRBCGH0a7lXUEBorcEbVrsIIWtt8GiKoggA0OkUodOaTEdSSkajazgHQegRAhB65yx0BkGPMMDQWS2MtVpLhJAQknHmgXEQOQesBwnnKyk9uzj9+tuvWtkAZzmFxuhyPlrp58zLs8vjiCerq5sA26aRUZRGRQraikcsT7nHjMYJSzJtXFXLi/E4YizmCWckIqlTTLZCKZVA6NoWU9bLk1LLpqqYizt5LhupMn1yfHZ6fiGEyoqP79+/37btspwsqylkYLQos37cifLO9nqURsuqTfsZ5/G6asqmnI7GNMbrqyuMguf7e3unR5s72ysbq5Aybc2yrqzzg26HUrqYzs4OjyfnlwmPrLU+s5xv37t3b3N9Y9Dr+1Zsr29MLy9jyi5OTnc2NvNOd3U4nM/nVdV8/PHHJycnwQnRGDNfVoFEAuCVbYu/bhGuJg8CgTdaAw9cFEWhY3DA553u9u6tvNM9PD49Pj4JjIe1tbVKSIcjB3ClVKTYvKm8qvmYbG+vU04pJ4xjxnAttGgrbQ0lfI58bzBY21gdrq1GcUoYhQQiAjGmiGBMEadRmibdfrdcLp1zSRpZW2it5vOFUvoq0deDOGLLusIQZDnzzvzq08+KotjY2OAQPn2x/3zvQGgQJwxAUjXKeZtRjBElBFmjAyemqsTx0dE3XXj3/nuA+IODy1/84vPPPvvmclpjggPaAgAEwEHwa3WMA/imtXqzUQDXqhljzLU0wwMAEKMOAOOc1Lpu2+BaGsaSG2GU1cZq44yFAFipEEIh9RlYF3YEztiQ4wADXxICcM0BQgjMZjOEUKfTKYqCUnpycpLnea/XQxhXVYUxzrIsSJSbprnmHrhgHHKl6IOQc+7RlabOORdiJuI4bpStqsoYPZmOprNhkRKra075/btbK7306PXIqFq383J61o2pV82L50/mi0obFseDtpwb2aAsk22FvIEWOOOdD1JLDDFBHs1FEOlEbVN56zDG1gKKCSGkrkvoPbDOQCOadjIaSynTOKahaGMEoYcEh5bp7r1t5D2jNKLEWW21ydO4KAqlrfXg9OwizwtM2PnxxXQ6XSxKgJj3vhUyOMthjGIYYwgpgt5bY5V3mAGHCUMYM06vV6sQAAQ9cBZ4a4Gz3jrntHPBXMcj4Lyz1gLvjNa6LqtqsRwMhmurG9vbm4Sw6fmlA7puyjSP+4NiuVw8e/ks6NXjNOl2+nFOm1oKjXiUiEY4rYJl7cO33hqNRkcHx+PxeGd7ez6eKKXqsgLXyQNJkoR777d6hauJTktTLZYYgTRm5xfjy7Pzumya6q0H92+nRdKIstW1cw6wJMp7kCVFnyzbunzxfH1rp9MftKLUTjX1PINRt4jbZmx1e2tzvZtk/cEaIezs5Oz1y1de62F3MGbKuWo2OxslaHtnjREqZR3FzHgzX0xfHx4QQhAjbSNWh2vYEehpHMeXl5eL+XxjdUNZdXvn7svXzyGkjdBpFhVRoXTlkCuX9fnoIi3y6WRZ1YKSGAB0eHzSNnp//+D49Hy+bL77wfd3bt352y9+eX56Ya2Ns1RqpYzOikKoNvExhNY4bLwPezsAgNJm99Y9762SZjKZGN1obb2/6iFCg04I4RFtWhyc7CCQ7jqqNdwbCIGg5bPWAuiSJMmyzFotpVRKEIIgQg5YB64fe2C9923b8iihFDvnrAPeG2OMlBpgQChxziklnbdxxKKUUY5nk7GDhka4bOZ7+6+2NzeHq0NCyM7d28PhEHhPCHLa8Jg4YGvVWKBYQhvZSO0gARAj43RcMJ4mxhkNXJREiOO6bgy0vZX+7/3R798ejRZ1c345KuelkWYxWy6beSKjTqfDONS6kgoZ4zCJjJGEMASJR8YDgJDjHANAUfD1YgzDq8h2hJDHV3l0CIRFpryxikMIUYoDzX57ezuOY2vNfD7nnDtvgAu+s8ZfM7ystbatMSWMEUShcVKJVlnlgXUQMMaMUkY7CDCmhNGEc+7bxcXFBaEYE0Sg10ZdnhxsrQxTirZW+3nWY4w5IYbraysrq1LK49fP8jyOksxCbBF2CLRKt6p2XltAIPaUYuSQNgYjGBGy1R2KJEXQMQTFbKxqWWq9nC+dA4t5/eLVIY+K4fpWLV0l7GRaOl9XdXt0SkbLKup0KzXvrK7Vom1dQ5F3xu7e22YMv3z2vEizjfV1wrB+6uq2bYxanp8yxta3Nu9v3P/ss8+Ojw4xQPVs5qV0zkFE8zhOU7+2uvHed94v0twK47S7s307AhgAcHhyuN/fv3X77vnpGYSYs7iqmuFwOJlMFouFMaYVKo5jY0yS5vZ6sXpTVgAAEECEcDicQpQwAIDHyebmZrc/LOv25d6rxaJsmlZKmRRdbUFUZHHEVVN6SjwmPM8bpU8vRwyDqqqMc4giTLED1hjlIdy5u7u5ubm1vcPiSGplrL2m+LRJkmGMHbCY0ZWVlTRNm7qOORusrHR7xf7rw+l0xgjlhFtrvXUIAQgB51xrsFioL776+tbPPxJV/eLFi/OLsQOQES61tR5AAKVUECIIsLUCE0QJaRt9dnZ2Muw937uwe+dfff3ks6++vrisHQAIEQSAD+MqcOH/ICT5/peCqW8Ahpul5w3AgCkBCGJKIEYOeHft33ezpCNXiy0SEE2jBL62FffAueukhrCSv/lBHl2Z7DjnhJDdbsd7H8fxo0ePAABCSWNM4FCHhyuO48BECeZplFIUX/HhrbUkpKMBCK5lzAyTOI6jKJK+ieMYeLNYLKbTy1vbA+tUVU5Gl8eDgm9vrW6srD7rdr765Ffnx/sJQ2/nu9PJtChWj4/2T48P8jRhFLOIbW+uCy2Fklpro7T3xjsEPOCUNVUNIk4IQch767RShJCmrM7PT5umccZjhCCEZVkGZXWWZVGSIAwgBoQQj7y1BnobRVEaRxFj2EMEXKA8pSkDAGzv7mDE2rZFlIQVYZIV3vskM51uARC8KmsYsl7EedRIIbWilDJKQEAyvPPWew+9twAgb66WdaFoeaNDVohFyBrjrPXeQ+dF3Vwul5eXl9WyvHv3/nA4jLP47oO7rWgssM9fvZjOJ4dHR9PFxGjXcQDgxDtcN4YmvF+sO1jiDHuCQufEFgvvvZKyrWrRyLZtgXXeg6a6UkRrra9oFG8kjITrTmTTQog5Z1HE27oxWh8enM4n09Fk/N67bwvRREkkVavPLzHGl9OXDPdf7r9uavXO++9vbG9BAgfDHq/dYjlO0441Elq9Nuh2szxLO95h5JuTI0kidP/OPV2tNWUl1PTgsFzMFmmaGucZjx/u3GmlOr8ceQ/Hy7lzhnF8tH+6tbrT73W00DubOwCAV69efec73zk9ueh1B5D6nVvbeZ59/u3fZVmGMFgu52dnZ6dnI+/QzvY9AO3hweFkMjMOQoiPj4831rfee++9ldXV10eH43GZiwZTwqIIYrRYzCywDLMkURgS3WpCCAIobAG11otF2TQCABAGWaUMhF5rDSEID2p45JRS5bIOy7xrcb8GAFBK85wHBni32+12u977xWLhnMMEemC1slJKa71H0BqntYWIRDHkPPIAWOudc0oZ58CimhFCjDGq1YSCokiyImWULJYIYUwpdgBwzu49uNvv9yPOa2t7vc58Ok2SRLZNo1tvdNWUmNMIR42uhWoo4QgD6xyPeVmXcRzlaRYnCUKwlY12Nivy9YSlg65DeDpfKGkYotWiwghBb9bXNp4/74/HY49wUwvGWCuVFBoiCyDGGBFCecRiEymtAYwYYyHnLSwaQuQ5hDBiLCwpguVzGGQ554FovbGxFvywjdErKyvhM/fuyooKXaWq+Fk5ZoTwCEMMTdsqLZRVCHkAgNGuaRpKUBpzglkj9HJZrgzzpq1nk2lRZBh6aM1sqoys79++NRis5Fn35ct9J+3bjx5vb+++erU3Pt1LIs4i1igtlZK2rVslZBNy6iIWIQCNUlpr5AEkpF4uIAScUwodI7RIeJFERd6VQm+srb/16PHq6mqaps9fPC2rtm7kxnp/2TTbd7ZXrSN58dNf/IqIdNkIwqJeP5tOlq6x/dXuj1d+WCTp6fHJaLFotZJWm+XMejgcDq13l5fn9+/fn1yOZtMJcH5lMNBSIQ+o94SwJMmGw1Uh1Hl5sRhNYp6kUVpV1eXZJUJP4yhtpFrd3OKcI4TzPD8+PoYQBgupMFZiwiilNwXlhvRkHUAEho0D53w+nwd78s3NzUVZX45Hr18fZFmWZLmHKESQjGoRZ6mFqGxFIyWG5GJ6WS2nq/2uNQpjTBjN8zyErnLOO90ui3jdVLPlQmiFKOFKIEhoxHkSY4ybplksZlVbOeO0McDrPOtQzq21TjmhJM4pgtA6l+VptSzH4xIAgBGAEJ6dnf/tX/51K0VZGusBwkoqBwBkEZdCAeAQgh54CDFjtG10XWtlyauDk1evDz77/MuyrhGmnEetlBDQACSgK8wfAuARAPofeCvdFOWbP7lZDAd+4m+0FBBAjLDDgWwYlhRt21bLZbBbdkY7ZN8kIQbTHymlf8PE6eapwRhhjLz3JycnvV7v/ffft9aeX17s7e3lRVEURYi5CSqtIGO5khoxHp5NYwwKPwXhK4qrtQSisBkEVZN3CgTd6HxhjOGcAqenM7n36tl33rrLKRlfnl5enKyvDh8/euet+w/SNUzJydrG7f/fn//s8PBwsLpVVcuVJLnz4MGiWk4mk9li3jZCa+8sdA4Ee8cwEzN6Fa2U8IgQUpZlWzfGuDiO19bW0jS9c+cO9D7LsjjmFnjjdIAhtZGc027R6XXyiHHOWMy4M1ZKSZCFEL/9+N2qbMpa9noDzpLpfAHQVQAexMh7b70JI/ioHhtrz0ej0WTsAHAQaeet1Ywl9jo8HFgbEs6cscQHNb0DEEIIEPQWAOfMYjqJoiSKWdOQalnu6/22ld1u9zuP3370+CGA7tmzb17tvZwtxhCjze1trS3C3FhvnGdRsbK+c/vuW7PZwsV1FEVGqaODw0AdM1qPx2MMiagbzjn0MBisdTodhJCx7mbtFbrVAFwRDqFUWhgAPEGsS2nPQH++lNNPDi9m4L//V/8CA21kM+imX3/56dnJCVndokmUMkRjY8Bs0UyLIvOwKrpsPLnsZEWLW4xxnsaL2cQ5gFr79q3b3SK+e3cduvZ4/xVB8O7tOx/94tNvv/ns/oPHP/idHy/K8uzi/PbdNQD8fXc7SZLZbHb/9tann36Oad1fH9b1dP/V3mCn++Lgk2+f/aLb6d/euhMRqqpGLYpGRv/in/53f/If/9Py/LBcLqOI/bOf/GMeR6Cd39rOhVb/27/7ikXg7z763//wv3l4+87OJ1/8amU1bdu6U6RRTCGS1lpda08IBbnysC4bBDHnsdN6sbxsmkbpUrnGI08wa4RQzkYRg8ApaBZiQSjCOcYUCdG0xoY2NiTuIIQIvHL8dcaur671+0NrLTDWa5OwWGoTkDpvYdmWCKGIMWgdY9iKkiQ00FKstbyfINQ9ubSj0aTf76yurIU4HAoIx9Hm6s50MlrM28eP35vNFsPVjft370ynUwnHilSGtwtZ5XkxnYyBdcu67HQ6+/v7yLNmOdczsb6+blpjrYXNqHGmTZjuFZBxxelUyPHJqXEeYkIYjRjPig4iuNPrAwCqSvsoe/zhD7WWSinnTXBWllJqZY0JsT5QCDWbLpbLpZES2IDGeIIoRgRijBD1jlR1a53mcRanmQnsROBjSmNaAAC+//4Hn3/15enJ0d3bd6bzGcY4kLMAANZ5b41zzgEvoKMUDPpdiMC8XtRtyyPKOaUYCtFQziEEy1YoLQghSZpJbLz3E1E5RmDScYps3OKVoFG6oiRcu7O+3h9+Bu3xi1/tf/v3AADg9MZqr9MbHp+OZ5endWuMR1hjhJFsDYx0kiQWmoUUSqmIUSVLHqeVpYynqLdae7eze+uiEvzsHEB37/6dnZ2t//e/+/dP9/bj/cPL8eztx8PBcP0P/8k/x5Qdn1/+8hdf2GnVjbPJtFw0Po1z5+F8UTdNAwDClMI02bh3t6qqpmqV0k2rzk7HnLJip/P48TvVcrmYj1++er660pNN27QVpAniWHmd8FjaUrjl0+fPgLMnR8dlO3UXprPfuce+w2bl3qujtrGz6cXacOABAAhbiKz3LM8bb7FFwCPoPIaYEcYohR5ZYAlzdV0PV1Zmi7l0pugMeZFJ5CfV7Nnec5oQwomQNYbu/PRICXl6OT15AR7cvdPp9mhvHQEIUVbNZ8256Hc63ShzzltTxRRzpL31xqLJeGl6OO8UhCeIEMqZNDpNc+cAj/jo4ODg6HA8Hh8eHmopoVNFUXDOkwwjkjrntGyFtFtbK9WyvPdwZz6fnxzN1taLeVXSOJppWbcGUoAgqqREECVJYo2jGBrrtVYccyN1pV0UcWvtnfff/6u//dsXL15QymDEhJCy1YRQaxUAAHjkrnoCDCG2EBD86zXwb/3iRlyArt2IrbXAAEpphAi23jTCOSfaNngsdjqdO3fulGV5eHiIsjyc6M7qANT92sPf+5uTHr5hEh/6DAyJ0Y7TSChJEWWYffDuBy/2Xs0m87ZtEcDWOgQ80L5qa0ophQhjQqM4aHyCjVJYQ6ytrQUt2Hg8XiwWWutGyy6i8/lcOTvI18+O5ou3EWYb0/npx18edgbf/J//1f/p9OC5R/Kd72z/3//n/346Hh2VtcH48+cvPvr2SUXo2fFpzAuJ4qy7nE0vMHR3NzYbKV7uv543c5akutJxyozTWZZhjJ2zw/7W6flsd0dsbd52q8ZYhaGN4+jO7fU0jZP8SpkFgEOI4jBAA6ulckrPpzNGaZZlsICUYgwJALnSejou27YlhJRSNEZbZDBBOJBPGQ+zYiB4Zin1kL7am56e7G9sbvd6w7oRHjOKqQZQKwv8VfWDngJgkBsz5FpjEKI0iqTRjdKcc0d51QjofCfL13sDDCBQsjo/i+8+KBycTKbtxbjPIkvoxfiCoq7RptXlcGVDNHI0Gd19cL+32q1kXYlIto4QFscDB85aYSACnFGnDY2oVlJrG0WMEW61gQAqKaIoEkI4azqdjlayXC6iKCIGehYzHiWMRVoZ44HTRkhpLT47vRhNFjubQ+hB0yptMGXpoJs9vLtLGSbYxxFVLVpML8/PTzY31nRbkoRnCSYEI+RNiqIowsNoMcfASYhMnPBb93a98RCD7334XetInvco4RCK5bIiUq2trXSzgdY6TiLuGMYQE3jr1k4cp6oVm5ubWZafnR6/fn3w5GmDEZ3NZh6l29ubbVUWRSaFmM/nGxsbJycXD966f//h46LbOb0431z/hMTcey8kWF/fKNJ8sZivra0B4KxycUKt0gQi6LzVxlqvtabIG6CklGW1CA8epcRahwkkhGOKhBAeekwgAIgxlsUZIQMI/eh8WlWVaBVBmBCGruZ82un1tNZGmuOTs7Zt67oRTWutHQ6HwYIw5EIF4C7ceUHB3O/3wy0YnP4Q9oNOd2tr6+GDR20rnj17Np/PKUHlsoVBkIVBnEZCicly6rFvl7VCqGka6DxDhBOKKFRaxHzIKYUQ5mmmrfHWtnUjpYTYCauV0cJq6EjwfRNKVo1ABCNCCSGEhRirCGOMCXHeaG2MMRB5AhHn1Puo3+9qba3xEGLvYF03oVuqfY0xBh4i5AFABBMAIIRUCHHtRGa9B1ora62xSqmEELa9vX1xcRFA18lkUrcNAADCqzQYjGlgzCmjrbVtK8uypIw45xghFGECwHRZIuwYxgAYwlDGkyiKeMS6WVzXtalt2svLpulEBXSQgujo1alW6vJk8ujh/cFw59nTg729Y84Iipi2fjKbPn/+fDQri84K51xbNZ/Ps6Krtb64PAMAxHFMGSnLklMOETXWC6HqqtFKnp6eE6+x0cNukSVxK9T6xhaO8q+fv5DaPH3yqtudfOe9/cfvfCemacziZdUQGneLwgHkrLYOIGgpxdZab6Vz/mr/xTlCGHmAr/SH86aq33//vZ3f++Ff/0387VdfOeeyLDNKy6YNeTlesqpqDo6O6sUSerCxsTFf1MCjKIr29/c/+uijuq6zLDHeaWu0cc4ZDxGCGCEIvI0iRggzSmspnJEUszDGQQin0+l0Ptva3Xn/g3frVh4cHEyn0yzL4iiVUgohgHOcc1YwcrGYzWav7H671vS63bRIrTF1I9qm0dIY69IkBphZqA3AHnhd10GPRzlj0XXGN0ZKqXA0VlUVXKjDVdDKIYKjJO50OpRSD+xiOhuPx8I0K1vDDz/8wDnz1VdfTccTr0zVTjnn1oaYXaS1CDChcy6JUqigc8YjiCHWRkPo8zwP/u7WAoSC7S4IkW/gOov5N+CA/3rGNHrDWh9fZ8SAN1SUAWYLMEAURTcLhTD5hVYAIWTNlQLiRgrhf1NecYMD+WufBs5Y4KCELCjO+a1bt7744ovZbFY3DYRwOBxGUaSMNsa0jcQYR5zGUZokSZY6XWjvfWBBTqfT4+Pji4sLKWUAkyKPWiWl0RG0nDMWR0mWoyk+v5g9+fbZs0cPt1a7f/iTP/rsFz/79usnG6srVdksl+VXXz47P19KC9j1Mby39wp4nSVca10Uxe3bt93h4fH5WZwNIQAIIa1VXSvvTFFkSZRiAobdbp6nxurlbNq0VSsWACoRrLXhFaACnQ/BjFYbxgilVAohZStEkyQJxphg7z1ECAVEjVISlmvKSBDcuKVA6Nc+7kIpa3y1rBfzuui0RR8ghJRREEJrrDMOeAh94G1La7UQIlzoYHV/nT9MKMKOAuxBzKNOmmVxwghFAJ6cnymrJrPxycUJItBam+edLCuwFOVkXlV1lBTr60kQuBZFsX/8RAgBvW3bq3xdRjElyCOMISIZssa0jWyqdrlcQu8N8N77cPkCPSUA4aRsKwCQAXBZNcbYNM0Zi+I4r5aLV5PDly9er6+scp5YI1lSRNav9nvTy4v5dP73f/3Ts4PD4HTx8OFdVUvdCgKcs1rpxnurlMT9/rzRnCGE7XRxCuY6z7Ju0U14yjRe1PU3T/aFRmtrG1WjRgcno8sJdE8IIRsbW1mWLcvFcjG/ffs2AODe/VtZWihl/vAP/+Dx4/F8Pj8+Pl6W05/85Pfeeeed6XR+/96d8/PLyWSCIPn2m6eT6WJja+f+vXdu3/tOK4hH8LMvPv/226N3vvf93d3dZ8/a7bWtql5qLRMWQePyKAmIGYI+IhgTijFS2k8mI0KIc0BrqVQID8QBDwzshChiSRLxhDNGCSGDlWHbtrJV3gPgkXOBPIVOTk6hJ6JVZVkZ4yhmgHCl2+V8Jpp6MBjcunULITSZTGazWVVVUghGMMUo5iyJOMY4TVNCyPFeM0yHd3Z333v7oZLm8vT01bNnHgIex4Qi67T3dri24pGvRT0YDPRFQ5Okm6VtJabjsTO2SLPlZOZX14GzXhsMgdK6rRuCUJTnupm3Ws2qOlnMmRKTxXyymI/nM2UNxAghEmjumNEoihjjKc20Ck2AhQgEjpWzRkmhlXEOIEgAQFpJ0dR1NXcOIoQgCJIkHIABhECe50EVTSkO6nGlBYRQCLGy0tnc3Hz58mVQ89d1bZxFCEEIvLX42sm/bdu6bbKIa+OqpuWaeA8ppZRA4HxMCUYeeq1kK4XSAIC4QTrxmK5kQwzhWtEfH422NjaMdsTixWRJKX11vt/NBhsbGx4lhHfjLAO0zfNcaecgQAgZZ9u2UcYjihCFHjjjNEIIEQSsU0ownjkHnEWEUB5nGFOAmG7F5Wjs3VsQMSHM6urm5q0HBxdjZdF0tsDUPH/2Oo57l6MJxcyZslrMk7xQykhljQeIEorIlSYNIIxQxDlB1AdUxUHo/HK57OTF4cFBHJF//sf/1Bvz85//nOBOgpHVRjRtW9daqSRJs6y7mC46RaGNT7Js987tw8PDg6NT772UrTICAOAB8hAgCBACgYOqrQGeAg+AtwhbiikhHgFDEd3c3GzadtGUq+trWzs7z1++ePHyZZqmWd4JyewQYAehEIoQEuEIGjC9mFBAOOExT4DHwNOmVUrOG2k73ZwQoto2MIfgcum9zzudLMuiJLbXp2BoMYUQi8ViOp02VR0KHMaeUIYpSfNsOOwnaVSWi/5l5/Bwv5Lz09nh/Tu3/+iPf7x/8Npamw+40KpVBmOAEQUAAAgA8hhiZSTE0BoLoY+iSFmNMdzcXJ9Op23bOgfslT8/eJOK6H8jzSGYbf92lwCvXVXctaVV+BeCvDC4ub/JCAknnHNOSlmW5U3wI/pNA4Y3+ZLgN9ccb7680GCFFuQmfa0oig8//PDFixfHJyfj8Xg8HkMIjbOU0pBNH2I+gtorGMSdnp4aY+q6Du7IwaDJe19JKYBttRRe5ThrlDYALEvTz8nh4clP/+Zn7zy8+8/+0U9Whlv7L48SmiRJkaXd4+NzKQEggBLuvW+aSjiHoGUEMsayLJPWMMYIobIVGOMsz3r9DucUQx/HLC9iCGWSdoaruVKibcbtbO691Sb2oIXXLuyEIHit7AXOApAAAJzVTeOFEFJKznkSY8ZYkkThs4fXMRb4jSQ27zW6TrBEmiMKOEsIxN4hTiMAkFyUiHkEHQQWAI8hMMBC13oj27ZK8gxhoIRyEDKMWBxxzjkAzljoQcRYliWdLI84xxAdHxyPF6O6LoUSEANIIGFYa90K1e/3PcCXo1HE85C9MpsusiRyRimlIHBxHCFgEfDWKOuMMTYol0QrmraiiCZpqp3OsoRRjBBKk4gQksQ8jmMCCKKEU8q90AiAKIoSnjhjBEJGmdlsUdcNymLO4pX1DR5FqtaXJ+NXr14BAF4/Pxpfjooi/+H/4yfOCGy51vXJ4eF0OuIR5hHGHkjQEl5gDC1UwCtAUhJRHPGvvnl+eHrx9PlRUqxsbt/Z3NhZzKv5vGqrRXAR2dzczvOUEQ4hePrs2zRKd3d3P/7lJ874ra2tt99+NJvNLy8v79x+azhcQdB/9913hVCvXr3ee7V/fjE+OjyratsdbK6tb969906S5Sdny8PDk8fvgw/e++7k/DLhccKoVsJaG+eUYKKsh9ZCADmhnDOIqTPGOEcIAB5ZQAOh1zlvnUEIYcziOC6KLE44JDBw8g2UHlkSY2CBtV4LJZQxyq5tblSlUHoOIPYAaOe9AwgzQlwUMUKQcyaKku3tzVu3drTW5+fnKysrjx8/3tzcbNt2NBqdn5+fn5+7csY5P9l7FUGYFrnXghJAGD08Ouv0eRJvbu9sPHx8T8r28PiglVVEWSBKGtGqpk6ilBPalktR1U5pYHQacWi91IpjzCgVGFdKgelYecsYmy+Xl5PLWTVnjEGAQji6thY2VxEPlreEhPirqxi0UMsshFpZ5wAhFEHitUDeEOikRwA4iAJwhxgjCBLnfBxzhEDw8gEAAOgg8gA4KZvhcBjiLoVWN1XPBX2I96HzDQQrrDCLk6Zp2kY6Zp1zGBLovdWCIMyJ73aKIttpm3J8ca6NjBB8dPfu9vZ2nqTj8TSCTpYzY9zr01PO47t37tuUK6tPL85qKVY21q13zuooiXMW7+7uAnw5XVSNsCzJ0zzTWhpnKacQuGU5bZpKGdl6hi32ABKO4ijFSZrlWWu1EBogXNdNLdrjs7N8sNLrDi9G835/M8uy588PLy5mo9GIc84gLpcLDKD1wBptrPcaUx5BRBBwCEUAAAwgRdBT4K0L8mtnoVH6m2++OXj94t/+2//hw+9/Xwqxv79XzReTi/PpZJIyQhHsFL3VtfW6arTWFtiIx0rbjz/9ZDKe553CamOgDmx/QghA0DhgjHHWMYy8aZSC3ntKKY+4Mapsmt7K6t37d7TWLI4Gg/58uZgt5t77tMidB5PJZL5YpnFipRtNpwCADCX9vDubzZqqubyY1FUbxzHAhMd5I2qxWLbWUYoDHzuKMBGCc55lWa/Xw5QsylIpZYEP5k7GmDB5w2uDoDiOgyfH3M8oR1lnY2tna3NnFWB5enry1befQCJ/9KPfvRNtDQaDpqmKbq7URCkAiAYYABAEn05rF3HugXMAYgK9Bw6C7qB/cXER0IvrUx+A69wW768iobz/L2MJbzYK+Nrg8obVGBaONxgDuvYoCxLEcLrPZrNgZhC0V4FKEvqGG3lbAETfhBNufrRzDmMEABBCIII556PRaD6fS6N7vV6WZf1+P+SuAQAwJb1eL4oSAIBSpm2noe0LQ3DYRBRFt9cbhD6maZq6rpXRPIkdQdZaTKiyjscpYdghMp7UH338+dMvv04x7yX8/GIxGX/88Ec/jHhutE8TKBxxzgGgjaGDTj6fjdq2tk7XdX1ycraYL/OsYzU0RjMEN1aGd+7sZnkEoPVe180CYiXVsm3rVpXaC2CdNtDaK9so56j3lGKMECAEIYApw4QiB0lQrGitnLMER3HM8zyHELZtK5QEPqRgYACgww7YkPl3ZYDBMEOEFEWX89hobaSy1jR1mSbMAQOddNbqEBbZtlIIZWUKE4Q8Qj6KeESZ9x4jpAgCzgPrgPfOyLrxUmDnnAZqtphCjNI8q0XtPMCQNq2ezRYra6nxwFvAOAEAQOeGK73prJKydUa3zhmpmqpUsm3KqtspnDYU0zROOklm+4Moijp5IaXs9/vhygZLYoFpFEUEYBT6VugRcNIqvWwX48vzhEX3793d2lhrm3o6Pc3TKE3jTq9wQm2t76rW7+7emk0XHB+en5y2Czxc2YJ2frk8OT0qy7K6dXuzSIqEF1nOjVFSG54QxhJl7JNXr5YL/dHffzUvRZyl1hNMko3Nbl238/ncdbL9/cOTkzMhVBQlu9s7nU73Z3//UVvXf/xP/pkxH//v/+uf7O6u/fEf//HGxsbv/u7vXp5Pvvj8k/FoiiHqd/rvvPMOgvThW4+n8/LFq/2/+PO/ITxiSXbrzj0PWF4Mxxfjdx+/++rp8+ViNuz3PItGF2dZEjdVY7QCGEGMgIcQUAw9hr7VwlrivbfOY4IookY7600UMUyvWvgoimhEvfdaKxRR55w1zmlnjMMYE2KN8uWyPDk9H52NrAUIEghxxOIsy0w9HfYHOzs7KysrnU5nY2Oj0+lIKT/++GMhxN7LV8+fPqvrOuwvZSuGRWKMOTp4dXJ82B8OpNDdLOFxNOh3rDdtvfj6q892b++EpXGnSJtSlmVZL0vRtDGPdzbWVwZDqzQw2krRVFW3P0y6naqpq7YZX5yjjLdSNkIsq4pSqrUWSoZZHyEUOI9BsGAUwhh7LKKIFUXBKHdWK9mE6ixl4KR7jQgEWEkFvKYEQMQIYRBC50BwDIUABmaotdZ7Zy1w3igljNHBU2F1dfX169dBCQIhjHhilJRSeg8gxoQQzj2lNE1TRLDVSgghrFMMOWcYQg57o1QS8yKLP3j/vR//7u9whl69eD6fz4si6+ZJ27acUqPbe3e2ZrNZJ4u86ZxdnDf1qNMf9AbR/uHhxey42xtcXl5S4ISQaaeHCK6aWinFoiiOeSuUMlpK6b1lnDBGooRD5LSwwFrtHFIKQowwtta2UvT6eZ7nCHsP/cXFxel4euvOvRd7r61zy0VzeHhcZLkQzTvfeZszmsVxfzgglAljx/NysliouvWYQIgh9VdkKAcRxBAhhwGCkEf86Oho0Ou2zfI//Yf/+N3vffCv/9t/+Zd/+edffvLLo1evj18f7Kyuam3394/39o8Xi9pa+9577xnjDk9OldRSK79crq4NS1ffKOKctsB7AhBAuNvvRVEEMVFKOeAxxlJrjyyLKOc0yZP7nBHOLi7Otda379313k8ms/myAh4yHmnlAMQR59TyguDJfBakd9P5dDAYOOcsgMZ67SxSOsYE04gwioL83XshxHg8Ns62UsZpEqVJ4AOFyThJEhD5EHTEKATeiUbNJvVyMbNG3r232+9333r7rbWNwXhyiRmcLkbL5Xy41kHUv/340Uv6KiSXAuiccdJoYD1ECFOEDEYQOGA9BBBDyun+/kXbtoHU7Jxz7ioYKeAC/r+SLg3f+LppfMNUeqN3CB95YEaTaytGrXV4UkJqV1jGgTfyHim5IvaG73fX8fTh8HbXjrw3+4jgNxQCnM7Ozn7605+enJw0UgRqVJwkoRF0zrVV5Zwj6CqOK/yVm/4j7EoCRAHh1dyfJAlkiDHmnBNCRFHE42Rjc7csSycay+tOr6vL2bcv9r//7jt5Z/XwYL/9/NvZUrSNppQ3jTXexHEcxxHjBGFYVdXp6SmP0+VyqbV1CKQskcASBBiBSYw6eYSw84DGkWMEtG0tlaCMZGmutQ7v+UpagjyhIeKRIwTwtRUSDWADRaGeN00TxzGEHmPkvQ3vDpNISXWD4mCMvXfGKGMUAAB6jyFyxs7Go04n53HknXaBoy5bIYRWSov2KlY3zTCBxlgEHEEAQS+FUKEtZpxgqpSSspWyDVcfEtoambAEc+Zlbb0njEPniu5gvigtgN1ul3H+zTdfzWazH/7wh4zgiHFntWwxRiCOopSzhLFBt4ch6hTF6spanqZa/v/Z+o8mybI8yxO7/PGnnBln7h7u4eHBIzKzkhRpUmgiM9ODGUwLBBAB8DUAfAwsIJDZAZvpBRbTXdXVVZ2VmZUZSSIyiJNwapwoV338vUuxeB5e0S2whYu7mZqpuprqu/ee/zm/I6uqMhpaFG9sbJRlOR6P65CLlohiSBpeg5ciS1JRcCWUpTEjNGSWqLIP7n9492hvuRp/++QrQuDRrb3RaBA0nNt39wqeMpv1er1OezTemAiFlvP8+mZ8cXk2m8aNZjja2G61fcZIojLbth3XMghMJpOnz16eHF+ul9ViJh0aSI0Xq3i1jg4P9putznK5jOPccbxWs80rGa3TS3ODoBVFCTJICL2zvf/227c3RlvTyXI6WfZ7GxTBxXSSZ8WThw8Jsba3dm9ubrZ39h3H4UJM54tCaITJIl5DRBzfG4+n7z54cOfWW7/9h18ZrggEuuSQMsAlUgpBCCAAQENTp5+1VBwZCQGGCDNKLcuBEHOhGo1GkqVVVSUJsB1muy6hiBCSFktjDC9FWXApNcYEA4wwUqIixnQ7rU6nFwZNJVS9oO4OPx4MBr1eL8uy6+vrp8+eLJfL8/Nz9V3zr+u6lk3rd1pepPMiRwhpICEARvMgdJlLjDFFWbY7XWKR0LYsCLJoeXV2cnH84vBorx0GgWWtF2stleGiHTS2t7Zubm6QBjcXV2mcNNstgplLrZVQRhpZSSErUVWEEAMhxsD3nPl8Xo/h/vHSppSW1bpMLYtBIyluGq1FlQEAKDZ5mmgF6qS30ZBzKaoSGlkHvI0BSimplZTaGFj3ZXBeKaUgMrUvEgBNCD06Oqwjsp7nOY5T25uV0YwxY4CpeezfY5IXWVqkmTFGSaK1lARZDCsJ0tk6jtN2Z7K7t2q1Ggp5fsvtDQaUiD8+/FWZZxTiBw/ef/XieTMM79+/f35+zqVqdNq9fmc8H1OLCi0MAVXBi7LUWqd5PpvNsOV2fb9SohKl5TrSyDhOmt3+g/tvQ2hOT08vj5eEYSAQJhBhbDHKHItZOKuStEhbDR8YjSm+mU5+8NMf94a9J0+eiKq0GNnf3/Vde2d7CyHAGOt2u8z10qJ6+vLk0bfPp8tIG4koBFBqqZRSNWmv3nlpA+rIuOe4EMjZZHL84mW/03333XdDjF6+fLmcTmaT6WQy+d3vfnd5edlqtQAAV5PZvXv3rq+voyS2LDoabZRlOZ4viqLI04xzTjFpNJr1WXNzc7M/HIRhmBdFkmcGAg2AECLKMmpTgHDQDIqKx2lCLStstiaTiQEgbDYsajnMwZj6fthsNtPZUkp5cmGUUpACKSU3QkiRJEmtwxsIEMG24+Dv4MFVVV1fXy/XK2W0H4a7+3t1gKJe+SiljUaDIAwhzNNU85IwhrQSRV5mKUYGAhmtm71ee39vr9UMlZJlXliWI4XOs9JyHESxNBJoYICpcwtAAWN0KUoAtIFASMlsNBgM2u1mHMdSakJeL8x1nfQ/SgjfawuE8B9dCP/VdqFevdB3hY1vZgTgOxCT/q4bBQBgWRb6rt6zfk5et5wAQCnFCLzRMN7cUb3Y18bGN5OReutQFuXr+h8Ez87OXrx4UVUVZnQ2mwkh/CCAENq27ThOmmdFUbh2UGO8a8BJ3ZIKAIiiBL2uvVY1zb32kHqeY1mWMjpNU2l0WXJMqOOGzf5AZNmw0wJlPl8Xl5PV/vZec7D5m89/f3Z5oxQ0CANg0HdNoVmWOY4lqmIymTTb3Y2NzbDFryZTnqe+72xvDHc2h82Gj5HWWkCsXc+ud2wQENsKlENSlRVFgbEywCitoCJKKWMUQqS2KygltdaUIJuyuvmWc85FmeWJlVDHcfRr2gE0xtR0FowpogSh1zhtwaXlWFqLur8xLzjQwnVCABqKV0YIJbko8iLLlZQAAAdTy7UM1FVVVGWhpQBSZUmqpWy1Wh6ljmNDoIWs6t+dNhIjQhg2GFSySsvKGENtDjDSGmBMIXw9TlKS53k6Hl9j4lMMHcaUa/d6HccetZsNBAzUBhhDEHYt12a2dnSRFnmeN8NGp9XOsmy9XBVKE4QJwghA0gjaK75USlqIYsZC23ZtKyBwPs1v7W0NO8FqcSKrdZHlaWojHEbFlFmuIdE33z7tdjabfufB+28RC0xn468ff3l5deK46ODWrb2DrYqnZZXnmWi3eq1uezqfnF9Onj8/nc4roDGxbIjsSpXzRXRxfbOxseH7oecG68V6f+/23bt3F4vFo4dPJuOx77XarUG/051NV5ubu//sn/6L/f2D4+Pjly9fLuaxb8GtjU3H8cqSR+s4SaOb8dXl9bWQ+mo8g5Tajl0pPVtMCLM8WQSOs15GR/tH3371TZnmLmOB7RIAHUoZwcRiGsFKCgRhvVfodluEUAix1AAhYjGbMRtALIQqF+V0OrYsi1nED0PKMISwHXQwxlqBqiy1qvP9SEuzU++jhXYc17IszmUaJ2VZXl9ezSZTQkgURePxWClV+1AghMKAJIrjdVQURS23hn6QLBe2be/s7YxGI9t2CCF1n0JZ5b3BoKyKy8vzZ4sbrZVWilBEMdnZ2rYpm89myTqSXK5Xy72trWYQ7mxtrBazPC/Hl9eI4MFg4NtWLCUEAAEMtJFcAAAgQcSyGEJCiJqWzzDBGNcnWcFlVWbACIINxrDIUi7Kqky1qs8xBECktEHQ2Bam2F0XygCgNdBaaV0n66BUGgBggHodwjemBi/6vr+/v399fZ1lGSGk1WplWVYfXwghAEDznaXrNVBWcIKwxV73bnAujYYGYEycwUZfaTGLqn/4w0PbtpWobNuerEoMyauzdL1a7GxsKNDIS2t8fTXodba37yhgbNcpS0Bo0GoO5+sVxr7tk/OrS8sLB4NRu3u5irIkz/KigogIUWktw4Z/69bRO++9k2XJOlnOJykmzGCJCIBQYWZZNmY2jZfL8fjKsTbHs7Ht0TSNVtHSDWzLZp1u886twx9+8rFvM8eiRmmEUK/Xs7xwGaXzWWRRF4MMImoRp9Svr/4UYUophkgCDbSpiqLf78frNSN4NBqtl6u//Zv/9OFH73/4/ns2o77rPX3y7NGTx1dXV1rroqi0Mb//wxd37769XK5PTs7a7eb29vZ6vVwuojzPsyQzGrRC33f8ndF2v99vNpudXtf23IJXgSgNBFlRrKK1T6Dr2WUlojS+vpksl+tWt5emsZLGGOj7AcHMKOB5ASUWBMBq2B4hdtOrqgrbpEhLgAymqNluvK5xEtIoCbRSWpVlSZgFIYR5XgmOKWl1OvX7pVbj6xXRdV3HsmuEl0k5Q4hgLG1LKYWEWEymq9mUF1ujjWEcZVG0chzrvQ/e852WqPTN+Nv5fM4zA8jrVhbbdkhAqoKDOuCoNBc8DIPhxiBsNuqM9BuMhzEA43ryBgEA8HsWxdcrNNDfnzt8/6vfRXzV9wcE9U+uJwj1q7oe+dXSgvxeN3EtSNQ5z/qf6rtWaPQ97OP3dyr1j61h27ZlJ0mSZVkQBNS23rgi3gwXIEae5wVeWFUVqIuavit8l1I2Go03G5r6Qda2ocBhCEClNAIYKbOcLSE0VcW57UiDzq/HDsUiT7hUfqu7v7fT7Qz++PVTBKkQklILM4YQWq7mFgb9XtuxwiRJtNbNZnN7rzPa3llcnLiuu7HZH/TagedKVVZc1VYSADEEFBitpBIcKUmBBhrW48u6sk4ppbioEEJAG6WEUopRSghhjGAMGSMW85RStfGCMeYAS0otJTegPuQYSiAhrN4oSCVknnPOtSkboc0sYFm0HkoncQSUroqySLMiLQAAnuO6jlsaKcoqz1NelpUGPC9EWTmWHXpup930fX8dRWWZV0qYetQKFHMtjKlUCmEspUryTGuwWsfNZttyHKFkp9k8ODhoNptlVc2nY845BKZ2rTaDsNVoQq2AMdAAXpRxtIoN8mz3tdSkpeBlniVJvNZaYwQZxQgBMr6+yeKUEtLyQwJglRfLyU00n9862um1Qosaz8KbG72Kp82G0wjYeL1q9rx3PzqQIN/aGK0X+eGdLdu21pFYrG9W8WRj5+jO2wcbO72z8yiL13nG4zgvuDi7OL2+mnAJmEWVYHFUIgcQakVp8ujhk8B1NjdGw+EwcINms9ludY1Go9FGEmUYW41Gq93uKwncwDk9ueCVarU6t47uSqmBhW3bTtP06uqmLKpuf9DvtCeL9fnF6XyVUMfVCHFjEGV+2ARYO5RNp9Pb+/vtZmtydQkIaQUtpSujFDGGECqB4UBoKevlhyKbEAwhqooqy4ocFbbtEmqtVqurq4vxOGo02GDYgxDW16ksTl3XtSgjtlcvhKKsqrw8uX6VROl0Ok3T1HG8VtjAGJdlyfNKKVUbFfM8FULUIqrWilLL910IoW2zTqezubnpOM7G8E/TNG02277v39xMonVsO8yyLEq7+7u7SRpNrs7H4ynCsN1u+rbda3dCzzdaD3v9g+3dxWy+Xqyffvvt4eHh23fvtZuti6vLr79+eD0eE4S5FMazIYQWoca8psNqCYDWvuMWJs25UUppYBA0xhitlM1olmVlXlRlbllU8DLPsqooy7JkzLZtG0GmNZBCvQbMY4oxRhBCiI2BGBEAkFKv++kRQnUJXf18Oq7tOM7z589rRGOn38uyLM9zy7HX6zUhFFNKKUWoPhwYhJAbepi8vg5mGTTGIMIghFleIYSmi9XVeFrzbmsJF2nfdx0Mjefo88skWpvHj85O3PF0lr377js2a51eXiSxVJodv7wpubh/q/Pw4UPXazx476NVnP7qs9/PZjMFIGEsW2QG6sOj/c3NTdtm02mSJAnGsIbblmUupZaiRKaK1/PD/c1Wt9kfdE8vj4fDoX18slzOgsDrDbp3bh19/OH72xsbUHECjORivV5DA5ABUEOjAFC1s8hz3ICXKwAAgYhSi1KGAERAcm0cx5FcMMY2R73pbJxlWVmWT548GX384f7uruM4X3798PT0tNloM9uaTCb7+/sYY88Px5NJs9lUSl1cXARBYCchBBRDhiHqtVubo63t7d2NwVBoVZVinYzLqsI208acXpw/e/H84M5hDUEaj8dPnz/XAPnNVpbFQsG8KoEywHDJle/7SpY3Nzf9YSMIwqDhgUSXskyTygtz27bffusuQiheR+PxlHOuhTTGlFmOlXZd13Ec1/ccz+12uxjj1Wrlum6dBahfOYwx13W1lEaVEEIEceB6iCDGmBBVlMWEEM91l/Plq1cvpDT7+4e9zqDbGkZxXPIKQIDJ67Ie27Zc1yNtUhWlECJNU5FzrWVNUatVg/owLeXryAN+Xf8GvoMm//9xF/5XikI9dHjjQqgFBoRQVVX18PvNd9WtYLWTsb5NHYys77cePZjvMAlvRg9v9hPge2aFOggTRRHn3Av810EACNM0zfPScaxuENSMZwhhEASbm5vIWEVRAIOqqrKYw6htjBG8ynjx+gQPoed5jo0gwFrraL40xghtDAQQY1FxZluYsChJXMYm80Uz8ELLyZVcZ1mclb3uKIkzKXSec6fp+r4vpUySgmPQbHC/6RtTfybZ3ju898571VsHi8UizaPlci5VDgnQhhsIMMaEUgiIVrIsNK8MAo7j+mV1A76bxUAItVFCaACAlqI2jUohGCOOYzHGLMtqt5ppmkrJtdaMMUJplhX1zql2MtazewCQqhH783MuRZ7nGENKsRTFcsWn0ymvJNCmqqo0zso8p5jY2MIuLstUcVFWuRICKlNVBcO42QhG/d5g0GeOLSWfzTXnpdDKGCN5yagNMSCYhmFYSVmWvOI8DMMky/KybLVaCKEsS7MsXSwWvt9TSjJKa5MZZaSqiiLNpKgaQUhw/XJBhGKooYDcSFXlRbKO0iiu57kGEwAA4cll4Nhpks/z2TtvP/j6jy/G1/Gd26N/+3/8nxDFv/zlL4OAZlEsRU4ViKZzpnqkaDFt/+jT/43fbJdSaYsJhuYqQy3PV50f/PSnxljLaYG4++KbiyItJi+XBpNcVKBqd4IAwzRKE9fTEFcew1KkaXH9639Y/rO/+Cf/6l/8y+n4qev6y8VyPj7eHrQg3/nkk4+eP33S7Xaafvm73/5iPj2N58c724d/8qOfuW5wffbq4sWrq5sJpdR2vG67tbe355xfnJwdD4btvKqUVulq3RuM3r27f3l5yaOzixfizk6n2w0F78ZxGvhhkZeaOQaoUlcASN91CAVSVTYAQLtFVCpjLIYBMWkeZ2VkOfYqiyuV3nuw47khIowx10J+mi5Uvu702tKA2Xx1cnZ1fnEjlQYAVVVFEMaIHh0d/exPfkAw/N1nv5GOWSistUTAGMUp1gSRQb+3vXVweHAbAOD7PsLq9p1dLiNmQQB5tFhujhr93rDXG33+hy+ePf7KsWm7E56dv2Tko2+//bbbaf3pj//bh988Wy6Sht1zPb1aXORJOR3P/vwnf7E/3LniDAtczMvudq9zq9Nv9ZDWCMoongpRdeTQ8dxC8ul6LQxwm6HBaFXkmCJiuTazijQtstzIgiKMEUhMxhgzqoqXi06z6RPMRZVHuW3biOdGCY0IqPVwDbTWqATMspQGjBAvaGoD4zQzBkCEGkFQlmW0XBCMt4ajaLn68x/9VKssS1eYaOaQ5y+/JRbjQlBjY4y5lEAWhGMIIQIYY2xhUhotMeZl9ZoyBmCZlEopzqWuM5wQEsIQoFJCIUSrWZay5FL+4osbZZG3jvbZ9enVxQllWlXLD957QEV59fLxYrkOAw9kxfnLhe+BfBkxSYf+kBYkWWWO71gE3Ez41o71ybv3j/Z2z0+uf/6ff3txdeM3+gFzQ8+iBLmMISlti/qdzsH24Wiwd3I+vnXn3bLMG55/fXZs2/ZbmweHG7s7nU2q8WKxdBw7CANWyagoTibT04urpyevpuurRRJT6YaoaSoHQii0phg4Ni2qPONrDBE3FSHE872M50KZrBIWdYvCXJ+fHhwcGYhu37797PnLZZy0mc0BuVlE/+f/y//p1dm54wcAji1mhYEPlGhhiJuBClytdRj6zdBlBBRlEiVJFEUAgDTP6sTj2dlZp9M5HO5Mji9Pzi9enZ5waA5uHzGLZJUpywQzlURxkWYIoKJAVVGmcZIs8YO37z+4dfv87CxeJySUJs57jX61zDe3d0GFb8ScYI9Yfikk9RhyUKEU4WIQthBC88lClHK1Wrqu+9GHH+Z5XkRZNFuleA0AcCxbMlcLuVqtur32ajF3W8zCrBQa5zy9ngWMkcKUOVieToqg7wb+u4dDz6QX55E0AFBgNGCk7DQ7GGLj2zdXY5Xx7W5bS/Xpvff/5j/+R0QVBqASQmuAKcAYQAwpQRgTKbQQ4vWqhGm9QhNMzPfyim+8BfVBvF7R35w6pJQ5aUHBqZAu1hbUQEiuVA4goi7CVpRzxhiCOM/zVuA2PauU+o0IUR/x69N/vb2oP1PbC4wxGGMBodtsxnEcXV4FQRC02kIIhGl/0BgMBpxzY2CvN4AQ9nq9jz/8mBKHc358fBzHcf04CSGTyQQAMJ/P8zwPgsBxnNrJCCHEsKjL7apKuIHfdJqcc6igFrpSlc0cgi0F0Vtv3+fG/P7hV7998tkNzINeY8/f0YpCQLMs29resmzkujSRRbMTVnm2Wl6V8XByki1mJ1wKwljYHGBGCaXUatu2XXIOACrSrMxSbHTDcwTnWZZgAKWUCigCEcRavinGBIBgbFuWTRnUJlquLMtyHCdPE8n5armuKlGVAlMWhE1suTKKkjyJ4qVcV7bHMIZRtJrMp1mWQ23iaO3YzGU4X59jBFCVxNMpRjQvRCk0sWxkwRylBFOV8zxJfdfjCiBgmq02AfDD++90Wu3FbEoY3R+NOr73xy8/P7u4YIy5nW1jDJAIQAS0Vlwgoz3XrjhvNYJKCkLRer24vr6qk40S2I5tKyFkVVFCMJLawgYy3/OlkBCbMGhCoyQXFJNet9VvDTudTpakw24vCAJjar8eIf/D//Bvjo5uJ+v4yz9+DQ0iBPk+HAwGxsCvvvp6fH2xMepGy6zXbyLorGc5cKhb5FwJBGCWJRpRBYrpfPr1N19dXJxtjHpBs+F4LM6T+XyWl0UuRCIFF6KQXGnAjSp4VVUCU0tyzigFWmdx1hgEw/7QcT3fdgik16fnJ89eGE2k0CfPjz3HzbPk5vr85bPnrVan0+4vFvOHDx8eHNxaJvNHz75ZRWsAwF/+i39l+3C03dakvL/cR5QprQEm6yjqdrtbO0Mtl4uJqfHdH3388V//9X9yXNcL/DRPmM3qZKkBXJuSUjTsjtrt5unFerEUlRSu55JKZHlJIGr4vW57e3urGgwGGEAhhJFiuZiURbE9HJ2eni4Wq/lyPV+s84oHfmu4MUzjNFoub9++9T/+9//m8GAnWsyjxWK9Xo96qCxzISrbob1ebzAYNBsdxlwI2GoZHT9/ajtk0A3eeXA7zRc3NxftdvvFs5ez6WJzM724uMiyTCu6XE0fPX758vnLMLTb7S4GVrvdRdByneD85LzZaBdZ8fTxU4f4791/f7S5YRRUXKzi2ETaQHX//judfufly+eX1xeL6+V8NcOUOq4NlVguJojRZruVV0VeZEpUQBuLIYOp5FyUnDALKFBVHCotXY9QBCFWQhDXhQBzzqWqqGW5jq8QLIpC6aISSmvgssD1qDYoK7Oq4lWZGy2TdZRlGcE4WUej0eDW4dFf/83/F0I4GAxs11uslgoYhxDXD+phRC1OaK211EopDWWhRG3aejP6ra/LruvK6nWUqzZ11ZSYslBSa6NhWfLr8c2H7z744IOPfj69Xq3WndB59eoVIgRCuLW1/c5g49G3zxosVxJ4YbuoOKYOJgwhlKbpeAY6TfDg/tvvvH0XAv3i6dPJzbUsS+BrwTlQEgKaC0WRKYtsazQsy/Kv/uNfKyXe/+BdhKDWuipKaMDh3bvNpi9VyYWSWkIMKs6jKBnPZ+Pp4vjs/Pj8cjxfVAr4AYCAEYQ9zzNGGa2UkAwjh1lSCmy0Q4kWYhVFRoNRf1CW5c3NlfPOwWS9LItqa2f3zv17X335zWw2sx170Os6zJpPZ2kU97rdLImXs/nezm5Widq7qpQqeXV5eSml7Pb7QoiLi4skS9M0rQX/wWDwo5/82PIbv/v950+ePwvazf2tTc8N6jkaIcwY6HgugRhoA7TmZaW1rvJCCd5qNrvtFjbaYRQABJGGSK2j+XK5UlB6nksdwo2AWiMDGWNQ6zzPjDFpkqxXyzzPP/nkI2PUdDqeTG5q2k9VVZ7niWjVaAaIkuV63e60BC+xRVvdlu+7zLF93+sNeoHgYbNVKRWNJ0dHR5hahL1aR4kGEAKMEC3LHELMCHNdO7OTvMqtuqRN1jw+WJ/VIaiNhK+TIBBohFCdwsffbRTQdx6CNxaEN3lI9F/2QtW3wUhjiKnRQHNhODam5iWX2gCgEcCknjdII4RIksTygu97IMx32Mc3EsKbr9Z7iHruUC8JdcUaAKCOTtTvr9pwwDlfrVYPHz68d+8+pbTbazeaQZ37Rwg1msHFxTps+JThLMvErGq3247jKK2B0cDIWjKRUqbxmlLa7zSKNDVaetjvNsOtzcH9O4cWJdPx9edPUCt0h8PN7c0jAK3ZdDkeG2b5EMmKp1rrej7iWe5ysa5KEccpodTxasgBZZaFvkPlvtGWEMD1c4oxFhJpDeoRqlYKY0wpgRBiiA1QSqlSlQAAjGGt8TQaDaUUJSxotmzLLaoyz9N1Ej99+jQvMwUUcwi/KdfxSilhO1a0XHuehyECSgutEcSB71vUxpBkeZlmc8GB7SCKidG6SnMMFcOQ57kS0rEdignFOEmSOI4Xi8VoNGq2Osz1sOUS5jS73Vma1U8mYxZCiBEqDIAAeWEDQZLnOdMQQqggthEBADDFiYQIQMVznstSlEGn2wyDRhASQlzbppiUeRZFUVUUSojZbCJEVVUVZkQaVQMVNNSk1fBGg85o0HNtmyIWrVZXF9dvvfWWbblff/V4tZjISuVZebDTD52B5znXyVlaFhpASnRRFZBKAvVsOVmsZ0m+bnWP4jRSmkWL2WIx0wRJC1dVlYlSKA0xBphgSgilGJIyzSBxet3Bzsbmnb2j/f19oLRPnFcvT371t784O71pt3t7uweXp2fNlq+1yIuUcx4EQRAEx68utX7heQGHVYVFYcpOp3PnnSNKKbF02LAOjkau67quK41eLBa9Xm9jY8Mh+a9+e1UIeT2d3bv79jqN8rwM2qEhOuUxxsZxGWVAS4AdsLG78eDd+zs309/+9g/PX5xgbFyv3ZTtJOFZpAiTvteyYFBVWVlU0XKRF0m32759+8Pj41NC0MHezuHhoZTq+Ph0ObncGG0Bnh/sbG4OB+v5Yjqe8kql6+z/8L//n4xRBtQ0aJTn+cnJ2cNXF5//4WtgiG27//pf/8tm4MuKZ1G2OdxMy/jq6moxXy0W0Ww6bzQaR4d7UpUG8Iuz8zt37tzcTP72b//2rTsPbCvo3R5lOgUaEESLovr8D19ARe7cest1vGgVQ4IRxpiA4cZgOBwMur04i0+fnT568vji6tw2ntcIEbbSsogXN5XgderMYgQow8tSC4kxHvZHnu1oLQmG/W4HaJWt00lWEkQJtQAASmkIlSAKYAIhJlRJnVeVgFhzHkBMoBEAKsoQqm07QmADjFZvv3XXtqxaehVCIM4ty6K2pbQWSmmtKcOUUiVlWZZlXgkhgNKGYoxrbBp8Y/N+M46t5UFj4JtKXKmA0JBRGxD95dePtoYbP/3BR+8+eO8Pv/65495qd/vD4TCXj2br9O7du9s7+6evHidJapD96vRKClhy1ex0ANLbDto/2j483Gt6/vHZ2fMnj7P1uiqBCKRWCmrDMMmrInCcOE7/7Ke3At999vJZt9UMgoAQvDEYzufzIAgGwzYhJIqWVVXVDrX5av3y+OQ3v/1dlJZxmhZcA00N59E8ilZZrz98nUuV3KFgtDFs7u7YDpNS9jpdKeWrVyc31xPbtjudQ9u2V8V6cjpZRWsa+Fvb22lZvXr1yki1XM6h0UBroM3+9o7gZVEUD95+p7mYHB8fAwAODg6klI8fP/7m0aOtra3z83PLsYMgKMsyKwvXdQ9uHe3u7r68uBrPpqvVqjsatdtdQ3FS5lKZ2pPoOK5FLaA0zwuEiIGIGJVHq/2tjd2P3yuzUgihAeRcvnj1sihjDbnnM4NFkhdcCOpYMs8t6vGyilYzjPF6taolYt9zBC+vry4m4+tasU+ShBK0SlZ+K7B9O46WQXPj5moJIKGURPESQGW7zCBoIEjLPC6y5WJNIQII2r5r4iTNMwARAEhr4zo+qrDWGjKgoO6N+sjCBc/frMQAAAhqpId5I/gbYwBA8Hvcgu9ISOb7G4La11z/uvX3EEkQQqgqLQVAutXytod7Dd8XQiRFdT1bISy4hpRgiqBQMM/zIs9C9Y92xdevcCm/74V88/Phd2SF7yhnrx9Ju93e29vb2NjAGN/c3Mxms9qfb4wpiqIGztaLaBRFdYK3rqKu36dxHNcBFiklpVRzUJaF1tqyLIdg26Ku6/g2fbWe8jz1bWojeTDqbvWaTd9jKncx3tvcDINWkSRSl1VZFkVmsB02nFW89D1qtEYINZtNQohlOb1ez3Fd23OpxerJO4RQSQMAAgAgRAhh0EgAgJISQVzLNhBipQzXEkDOOSaE1PQ5jDFAAGPsOF632x8OhzV9i1gUISBksVot87JYLBZFmWgllJZxJYSotOQQQSVkr9keDYZCiKrIy7zABPY6faVUs9GVUvd76zwvKbOlBpxzTGmRr6hNJ9cTCIDHMKTYsqzZekkwpY4T5+V//vVvkjRfrlJs+3n1uhRUaw0hwhhTiCizMKYQQqOMKbnUACFEpLFtAwAAyQIK27ZsChTX3Dak5zv7u9sY1Ocrw4uiSNIqyxUXCCFBdJysINKNRpCXRZQsESVaazIc9RA2FKHtnREyBBjBq8yxqOs4RVooDii2Oo2hkvTkxc27777b6feqkhujKMM5r1TJKSUUg7t3DhgxvV7r5PyVa1tGc4mU0wiiaG0gpsjBWgOE65es6yhRCqiN0fLOW2/97/7N/9hvdyzK8ixlGp2+PMnjzLfdXrO7MdiI4tXkZmyAvHv3DqUYIcKrkjLMGMmyJIOi09u8mSwwdZ69OPF9fzab+Z7zv/wv/25ra+v+/fu+72ZJ0vQCKFTT9QeDQZZlZZkjDFvt5mz+vKwSZsHR5rAoUwAkZQALRCi0bNxoeI3WcDxt30zPijIl3KEIY4irXEbrEnYpAqs4mhVlvF5NMJSH28PA88u8AAA4jB4dHe3v7//qV7/64osvx5evhr3hrf2d9XyGMd7f2b9zdCdNc1Asy6rM8ihOFpPJzdnZycXFzXqVGQGkgE2XehR5hETT+Wx8U/ieP/QwxicnJ+fn11XJbdve3hrt7m29+969P37+2/39/S+//PrseHxxcWFbQbczHO211uvYtdzDg1svnr367Hd/WC6S27fe6vf7zPIgNGWVGQWanaZFme/7O70N37WVLK8mV4Ln2CKqyJbJynVdSkhNK8uLXJVVGIbD4ehg963BYBB4jmXRVjMsszywPYdYNzc3vOQQEcwoMChNc4IZtS1qK1NJoUseCw0BwpbR2HNsz+6mcdZqNF1qAaU3BsOf/OhPnj/+9urqqigKLoUlJEBQaFWU5WK19n0fQQIhFJwXRVGVXEoJtZESUEoNod9ZqzQCsE7Sy+98CQAgBF5LvoAgWZQKqGa7f3V+8ZvPfr/R7/WHWz/703+yNWgRbBaLxXKxnq/WX37+5fV0Ol9MpDCNMKkq5XuNy+tZrxdYNvrgww8++fQ926FFnh4/fz65mrkWRUoArRUH0GgEiTEQM6qMPrh9J8+idqfX63fTJG80g06nd352Nuj3CZJKi6KUAADHb0sAr65vHj199ujbF0pDxlzHCyjB0JRlyY0CGAFgBMPA9YLNfnd3ONjcGm2Ohowxz/PSNDVV9fLJk6eXl3t7B++8887FcnExvonj2H/+tNvuvfPu/SDwfvOrfxgN+xvD/t07t3mRc84xIlroy8tr1nAAAFmWxWlq2zamdDabXVxddjqdn/zsp0d3bn/22WevXr1CCNmuM55OLi6uGmHr4OiW63lJllLHhQBXohD8temSUAtho4QmlFqWZatqNrnih7v3Pv6obi+kzE7T9P/5P1/EWcpFziwMCFClwEYjrDwLNzwrTQXS3LYchgAjsNtuIKCMFGWZl2XueR5jBCGgtZTQpEWEIaQWiuKFkCUhpIxLx7E5zwlFAGipVCm47XheA0yvbxREQatdKS1mszzPqUVCz4MGrddxmQmpge/A/dsHzLNEjbjSpuZjou9Rj2qtC2Osdb0/eL1Cay3/K4/Cm0P8dwHL71VNak2ULKvYscnB7q1/8tMfb20Mj89Ov370HCA8j/M4LTUASinJRZ5lgleV+se8g/mO1vxGY3vz8Ua9gBDWYIbaLa+1rokIxpjFYjGZTNI0rU2LBwcH77//ft1TU88dioIQghqNIAzDIPBWq5VS6uBgL4qi09PTNE1936/fnpyXBnjDQXtna2hbhFfZsBeUiQochlVeZfOr46dZGFyfHr9z+3a7M5jN07//5e/SQje7/ZoYEcUizbNWe0gQcBx3uLkVhM1+b4hBwGxLaJVmWcklQJhQixCsa3+0EVqb1880QAAASi0AECEMIYAArOcvlOG6NokQjOoJESGWZbmuO1+u61EO51We5/PFRAiRF0kjsDEL4ni9WmW9Tmt/f5/z6uTkZHu0ORqNsiybjicLoQAA2iAuVKs9cF13Y0NlWSalVkY7tttut/NsrrX+4vM/RlFEqVVv2VbLFcK03+8riBazVVFUmDlI49l86XiOAQAZYKTSUkMIHcuyLQcCADUylrAZo5QigJuNpjFGyyUlgGEtlC7TeD6fo7LUeWYxWyklpTZK15sqgjFGWANZFRxh2O60aIrinAEAsrwkO9ubvu+rSl0uF7PJLIlX/W5rZ3skysxzGYHh1mhrMBgs54tf/PLXAOAf/os70/lCFgU0WpZFmucEaceCD+6/1W66g0H/6voCURR67YoXleBClAAAZiEIsZSyzCsjKqQlkNy1CIaq12ncfu8BAGB5ckwIOXn+8sXLV612t9WihNL1ek0oXq0iy0adTstxrOVyWZZy0Ou2Wj0pqlToRnPUbC2XK/7lVy9sRiGEd+7c2d2+1+m09nfudrvdLMtarZZDKYN5q60cl1o2jpP5D374wTqaBKG9sdWBUBXjZRyvLJtSihhzGGMY07y4PDgaQPTg1avJeJxVAlBiY2xZlHoWQ1ojrYgGmgvbYZuDkVJq2OsjDIzRFjF72/3ZwdbFq29B6Bwd7m/22xenx9vbe1XBA68JXfr/+p//H3mROC7u9ZvaVMZU29ud9959ezGPP/zgh4Pe1pdffv3bz37JefnJpx998MH7UsperwchvLm5cWwXAJAkieu6e3t76+U0TdOdnZ1/+2/3jaarZeo4DtQ4jVK3Fx4e3p5OosuLm5enZ41Wb+/wqFLatRhlDkIEQcbLuIBF02t89MGHw0Hvm0dff/nNl9fXl4DCQSOAECpgTFVxrYHSnUbz6OjW0dFRwxttbW10Wi3btpqNQCkx6g0+ePDBP/zm10+fP7seTx1AbNeSFS9FhRCtgAAAYkQrodbrNYDU95qdTp9iNh/PfNsetjvXF5eddmtnY+MXf/d30+m02Ww2Wk1MWZKlaZHXGqkQQlIJAKjKsj7u1MHLSlQQQqDfDGghRK8d6QCA70xhtecMAgAKrhUgaZK32oP9w9vL8fVXXz++d7Dx7lv3mg42mj99+rQsy2ajPZ/PkySZjOcYWQjaCNIgaLguybJstdbzxbTT6bTa4enp+fnZRZGDfq/lOtoQCxgDgGGMackhwogwatk8Wg5GG0Crz7/846cff7i7vfvtw8eu5QnJMSIYY4s5XhBGSf7y9OL5ixOIqJa6qITUKQQYQTzsdjqdDoeVUhIC4NsUS351+ipbzXQa37t3z0VQQYC1hIoDVRV5vJiPS6mEAY1WJ03z6fTx4cHevbduHz9/tre9dXN9maSR1vLk/Np33LIs5+to/94hYSybz3//+9+HYej6XtAIb57fNDsGM4oxNhAEjZBSOhwOGWNZUfSHox4YFZInRUEMgIwqqYVWWgNSt0siggmjtuMHhhqRJMnNzc3Zxenezu7t27fb7e7VzbVt4fkizZPE8hyLOK7DKsGFKJBCQBRAlQQyApTmWSGyhMCqTClG0EiCIDS6XgezLA2b3jqJoDJbG535Yry3Ndzd3FwuZpyXeVlpxTkv65eK1GAVranjVmmKGB1sDt2mP1/NMcatVotARG/w5GbOlS5k5TbcnGeWb6My1VrVsGaIahMuhRBallUjfrUGUkopX9NHjPnHwCT6DtOD/stwxBs+EgDApcpySa8dHOwO793Z6/d6cbREpgo8u5QqL6qirIRSUlbAGKhBWZbfN06+8S3WcuAbS8SbexFC1itknUKM43g6nVJKz87O4jhWSgVBIISopzlBELTb7aIosux132Cj0Wi3261Wq4455Hle5yCSJOGcp2mqRW7Z1LIsjBWmstFiFsOrWcmoYA066rQULzGU0XpqRJ4mq0HYbfhBvi5NJXheioqXZVlpLoEAEBBGKSYuswhmRV5pZZTiUqtKqCTLAEa26yFEGLOAkkrqN2ZShAhCBGMKkAKvGRWgPkLUvlcEYd1qpYUsyjyKJKUzYyCzbcdxANQQwiwXXOQAaNuCWRbb0CHINENvd3PzaG8vjuP1fKGELLIyWierdRzFuTFGGpokyWQe2batuKiBj2EYtpt2p93+wSfvCiGMBi+PT5I4i9NEI6CAkUadXl5Ranle4Hh+DdDc2NhC2NSP2aYWAABoQyl1bU8LSanV8HxKKUUYQtxoNKSUWkHGWF2s2uu2k3XCmE2AUVUFIbQwRpQBAIQQRV4URQEtVYu4rk2hVkgLLkSeRWQxnVR5JoQ6OX4uhPrgw3dG/eFw1JlP5p7PjBYI60bo5nEkeamFTKIYaY2N0ZybqiyTqKCIUuxRb397M2z4vstqTXg2m02nU89GAAAMEUJEVgZxgCg0iFZGUd+F2vAqjcav8qxczGa9Xm8crSoEgWN7bugy3w/CdruVlllRxlEUGaAajYZFSiNAK/CLnOfL0mbuDz/5k1evXu3v7J+cnBwcHOxsHu1u3bq5uWGsqZQFgIHGTVKR5ci3SZFm60W1Xlzfv39/e6Pd77f29/fSNAWq1DwHAFiI2tjSlYkXCfXF5kbX8zzHc71wOp+Vy0W2XCZVqReLtc0caIBvWcpvNZvh/uhgNV90Oy0IwXo5jRbT1fR6o9f62Q8+DsPGahWdvnj25ZePz16eJZlAkCpljl9dpVnUHzQ2NgfdXrfZ8m2bbW3ttVs9RoMsFRXPMDHJMrq+vnZdZ//d/e3t7ffff//6elrTdH3f55x/9dVXq9Xq8vLSdf39nVuO3RgOgJIQozL0Wxa1bNsPw0ajUeYlv5pMHz761nHsDz94r+X7eVaNxSReL5vN0KNWJ+yMesO97R3fdf7mP/3VfDmHlgUREGWVFaXRsNXqbAw3drf3Bt2NptsPrRBpJEtRooIxsjXaOdg77HR66N//h+nkF0VW2m7AmF1WosirWEpKKQQ2oQAAJLjWEkCDjDRJFNudbq/dubk4N4In60WRRsPh0HGcoiqhVBBCxlhg264vi6KwmF0bvsqy1PJ11Im4ttZaGfCdnwtJ/fpcBQGozeFaAy1V/clc6Xavz1U8ma9bQSgNms3man/ryZNvkcxboY8RHY02v315fHJ1s7N3+P47D8qCc659r7G7uXGwvzuZjHWe+b4vlIzi9NXxyXgeMRdh5pe8DC3XmLpQm0BMq0pAhE/Pz5uh32g1r87Ojo9Pf/Dhx4f7BxuDUej7gmtkQQQxwlaaVy9fnT16/PTiZmJbvjJCK2kMQAZQShuevTXs9XaH05vr5WJGjMyjVRKvy6XXcdg5xbdu3VJVGc8mqiq2N4Y7uzth4F6u1xDiwWBEMbw8P7s8O711ePSzn/wJIeQXv/r7k5MTRu11FGNmIcepldysyLkUtutoYKbTaZJnvaErpTw+PSl5labpcDRSSjVbLcd1Hc+P4lhoBQnGklRSAqOlAQYgbZQCkCJcK+sEM2YDrDwXokUU/+JXv345OhZKb29vv3rxcr1cWpgMu528LGbX1zUWMPB9WJQEiqbHtjeHQRB4rmWMwZTlSRwtF3G0YhRCaAhGvmtLo28d7V+cnpRZ0m4ETOWffvDej3/wcRKtfv7zn5+cXSApLQi8wLepXeXVqxfHW7t7i/WKK94fdEedjUYv4FI6rhW4gd8KHd8ZX01Xs0Iofn5zQV2LUgohBEAZYxB8DUKozYnov1yw36A+vr8nePPBOf8u9fNfIJZdZjq93tHu5sH2kGFdJCteRAioPEtFJbQSUlRKKaQNY8wiJFf/mG54c0cAgBrd+P3dQy1d1I+qVhTqY/Sb2ggIoW3blmXFcZxl2XQ6vbq6arUaxiiMoW1blGJCULPZsG2WZajVamgt43hNKd3b23Ec6+zsDOBiY6NrWTRJI6CLqowwtiDiGlW+a4Ud38ENmzIEQSHLyig5m0+vJ3kFtoaDRmUyaeJkXQgetPzBaEQthyAQhM2iqKo88t3QZhVAUNQNh5hKoYuiUEpTq8YoUYIZxAAhJInAGFdclwVP07QsSyU5hLAunkbQ1MIJRVBKhRDilSjykkuJSR03BUJUghcQGqUFRApB1es2g6Dhuu50PFmv1xii+XJRcbmM1llWCAM0QJUypYKT8YwSIqW0Ket1rVLIy6ur2XyuFJdSrpOyKKUE0PZD23EcvwEhrqpKVFJrjSH2bZtgjBCSSgEAGKaOZRNCjNJv3KkUAICgULLklZJGAiM4r3jkum7LENd1meMHyIYAKIgaYUgIxQDWgyfbZr7rSOlbLqyjPRY0ikCidZJGIolIGIZaa4oRY4QS1L+132o0x1cXNrNuHe2mUYKR+vbpN1mcNZougvLRV1/v7+87lEEpXUKkzRxggNZVElmuk8dRt9PhnGdFUWdvWt5GURSScwiwsVloURUIAJDk9eNztS7/4dc/Rwg5truMpn/32W84l998/arXajXDdrfdaQR+Ei9Hg3aUxBYjvbYfLaM8S3e29kMn+PbqCiN7sDVwnTutVuvy4kTJUogCAHR5fVGUZaPRAABZtuv7/v7RLWd5Gi1vlMym0+nR/obDYLyaTixweHioxZZnsaqqOOeCq/n1EoqXdz9xpOSWTXb3hq1WczKNz85v3JvV6cnValkSbAVuw1AXamhjj0D/4uzxzfhmNZ8LkRZx66lH33tw/+N/86+ePXsxvrwc35xCJY9fvWq3NwwkQulmc0MbXFbi4nISpyvC1Pb2Ztj0/NCPo1wZ85Of/cl8vvziiy+SLFlFa3cyCf3GD3/4w+vr6W9+/dlqtVq2wt/9bhLF8+3NgWVZp6enD7/6djjY2drcR5A1A5as4zKtun3sOn6zrabTeVnx2WKFofnoww8dx7u6PIeKayWbYaPIuNbScWnDa/zgg09UVX3221+dX964LrEd2292CHP7g41be7eH3S2LOB5zEQC6UhDDClZVmhOLMdY6Orj19lv3T47PTi8vq7yijkswLiuhlVUpIIUmFnMcDxopBYyjjGHS7fQ82yYYdpuNjeFgMZtoyT3P45yfn58TZgWNUBoNESrKynVdizl1G7VlWRqbqqqk4a+ns+Z1uswYUJsWCWEEo+8SaEZUvM62Gcy0ga4fLBdrWS2BEItVlMTZ3Xdur2eXZ6fHd+/eef/TTzWmx+fXJycnnftvOxgrLTuNwGLYtSygue+iu3fvaq3nq9XFzTSrhOU2cgEFYFoqgKAxRlZcax2XBULwm0eP93Y3T1+dRMsFAEApo5TxXFdLDQGFgEoJ8ryKptHDR9+enF0ZQBChWBsKoMMsoyUCEAAOdVXmcZpE8+lYVpUFAC/yIvBDRm2IPEJzUaXR2rUwYmy5mJycvsqxhzUyUkllsAEnr46BlA8ePAiC4Pj41Xy5dDy/MkoQiAmuUTl1fG57e5s59uXlpWd0u9sJguDWrVudbldpHQTBdDqVUs7n8yzN5/NFxkvEKCAY2zaAoDbSG6MNMFproGEpZCW4FMIgpDDOuciKxXIdcan3tneKPE/jrNlsNhqNxWpepll/OLz91h3Lsop4mWUZpuT+22+1e10ptR80hFZPn714+vR5Gq0bjQBCLJWyrDbGeHM4UFUeLSCQ1aDbbTrWsNMaNPzPLeYxMmw1kn6feY1Oq93rdht+A2LiN1sAar8ZBqHrNR2lBUJIStnuNxEGQdCYNRZBO/zqj1+v0nX9MsMYGGMQfK1dKaWEkAQzjDGE/0hIrBfsN26A7w8IavMg/K7EGXwHVOg0/cOD7Xfeur2zPQRGzCazyfhqvVzMJktuqFKm1s9qtYwQUlYc/Jcf9X29iUJ8PyEJvuND15UNtm0zxhzHqbnXdfAyjmMp5ebm5t27d3d3d/M8BwC8nq0g5DhOjXKybfs190Ipy7JqKhelFIlpt9dGCKwiY4CUIOWCa1Rt7w0pQYhA5thZXtjYWi2XN/P5jt+fzpaI+cNet2VozHUmqulyYTvO1tZ2FC+xxs1Oez1d1z4QCGFZCqEVIrh2UShjLMvywxAAVO9+CKIQQo6E0TDPSyl1XVchOOacl2WptXZthxAJAGDMtixTM2Ft247yRK+V4CVCIEljKUtmEQx0GDitVqvd7iKE1qt4Pl+WJYcQagBKJbSBzPeRA7jU2PEsQHp+WK/rNiPNMABSRPEqSpO/+8WvldZCiKSUrhf4vg8AaDabaZqOXI9iUhWFqgTUpirLLE5cL1wsFsvpDGLieV7tC+FcNlpNbWBS5FVVZUUuhAqLUEq5TtaMMYrGnJdVWUouFBdSyk8/+ajdaPquB4zBAIRhONoYNhoNoHPbtieTCYSQUIwkb7o277TJ7du30zipba6nJydZFrcaIWWw3Qnfe/8+Mmg2mf7V//pXi8XSKP3NQ2i3xc7GZsPzpeTMc2yLYgwrUQnOFUaT6dSxrPlyESUxYVa33Wkyd7VaxesIAEQQ1q5jjMEAAwOjKGp1e9E6OT19GTYbw+HGs2+fjdeLTz/+wbcvTzk0F5ObdZoEjnN0uMscG1NSE91n02myyjzHavjt7c2W0VDxaGenP5stCJaz+XVZ7b48Pjk7O3EcywmsNMmFFn4YtNvtt27tRcuJZVmBY7mMEKC/+vqr3/063djYAADalhsEDYwxNFCVqkzK4+Mb3w9bnV6n3bAsZpDxA3b/7be++PybZ09Pi0RioHlRZkme0GI+WWutkQEImluHR712sF7OT45fBK496nW6nZaswLDv/4e/+s+txjDKouvxjCjlB63t3e72bscLgOvhg8OdO3fu/Pw//+rifHx2ev3uux/eOrpz+/btzz///WQy+dWvfnXr8PZPfvwzxtx//7/+hyiKXNddrqbz+fxwf/vBgwdpmj/86iVGjmOHaVJCXbx6dcws5+NP/8S2bc+TlCXtdvf9Dz949u3TvCh9x9VS2cySwJRZyQy2HQsbQiC6fXRn0Os0G8Hf/t1fjyeTnc2dnb0D1wu9oDXobzZaLQiw0Rpo6Lp2GIbMpnG0EoKrUkij+/3+4eHRIkrWUWIDRKiFkHJZM03TLE2ZIrZFIEC84mmUB4F3+9atMsu0kMPh8J0Hb4uqKvIkg7rZbO7s7Lh+ABC8noxr21R9GSWEeJ5n2zbFTAhRpNksWn0nNtapM62UqqrK8wKL1NAFJKXmZVVTehC0bqYz1/F9P/BcR+bp+Ob6888/96n6ySfvja8unj9/3uz0ep3+xx9//OT5i2/++NX+3l6r079z+8hxfd9zIIR+EBgAuNRB2AybHUKdLK/yQrY6/SJaYYw1UBhjg0CRV0HonF6c53l6/PzbXru1vbVr27aseBiGrmXbtkcIkVrklVouV5Pxoix4q9UJg0aaJZILgqGWEAJNkEZQCVE1GgGF20grB2OeZ0hrjECRJk+/fYwIvnV48OHHH2ZV9cWXXx0fH9PWVuj7eZKWWQoNWM7nq8V80O/a9l6j0XB9L68qgIkbBhARv9GAUG5ubs7n87wqO/3eX/7lX2KMV9H6m2++abfbg8Hg1atXdQ0Hc+w8zy8uLrIiz0VVCE4du7+5ySxWcW5Z2BijgJYaAClf1xNzgXQVr6PQD3rtTp6kX/zxq+nNdG9ntxGERuksSS1sffDgg5/87KcHBwfT6fTRo9+WZa6lAEYSBINWY2trR0iptX7x4gWEoNvtSinXUVKveVIIh1kFQpOrm6PdjdNXL58NOgyj6/OzZLUkEFGAdCVkUYWev7WxlSO92dykDpWyzIpUqBIijRAqiiL0Q8u1dvdb/d5wZ2/nl7/8h+k07jj2G3kAfOeQ1VozZhlUk44QIaRetAAAlkXfqAvf1xXeLOdvqAk1k2DQb21u9Lc3h51WAyqxXs3Pzk5evnieCYDtEGGXYKggNNpobfR3G4Lv7xLeGBHA93AOb/7CRVW7QWtGOwCgbplqt9u1Vlfjojc3N/f39xuNxmo9o5RCZKTiWmulhdJCKeV6dlEUnu8wa/D6vwC9vf0dD/ta67xMm8DHxNgOM0AYJEshi1LNy2pjMIIK9gdtjbAC2BWYQlCURVwWHFledzgYDFZ5YoxxPPfq+tyxbM8NcpYD13i+X2ZpmqYFr5htMcfmQipjMMZZVhBCoDb1kwq+s23WfsxOp+P7LkaoRqoAoDutNmPMsimBqDZv1gBjy7K0VnGaYAik4pZNw9BHCK5WK89zLYJns8VkMtMKUESklI7nYmoRqRhmlVBFVgptJACUMYxJofJsvYqiFQQaQxD6PoCuKvNWr+NWlet7ruvOlgtqMR/4l1dXPM8arm+EzNZxp91+/8E7dtB+9uzZixcvyoojF9T7M4g5Zcxy7EJybpSuoEYQUoIxQsJHFqm4WGdVmecIQGi0lOpmPCWEuK7LeRmt1jc3N5PpTSMIGZbD4XCxWFBMut1u6Lo2IZ7jkPHZeVFkVVVRBI72NzjnnBetnstc0Oh1Hj16NDzc+PP/7s9PTs7Wq+jq6ip6NtvcObn7tgUo7LTb6+XM8xwAYJWmWbF2m06SrhiCASJASAtSYFJqSt/WhABMlBCci7LGlXRsIPgpswShAOh0tVhujqzwn7zNsPqnf/npHz57eHycM5RvjTbRyToMts+vym4bAp3n3CsBevJy/OMfH1HLazabju0aY4qcO443GG5YljXodV+9eK5kIYqEV+nF6moymRweHhr01p37t8fj6XBrs9Hq/uCHfwaN8+LFyXqeWJRJYkxR9Xqdbrsjpbw5Hl+uytHIDawmxmE8Pl/dRIQQzwPDMIgDJ0fV1uYOBOzs9ApjeX39yA9I9/Bo+2d/Gnp+keWPH32Vr+Qv/+6XCKvRRv+9Bx989dWT3W2ymD3CyN3oObfv/eDTTz/xA4aJRlitoylhdlbCrBLHF2dZUf3yN7/+u7//hx98+pP33/9B+3pyBM5fvjj93W9+qSS8OjvPczAdj8fjy+ub5Wxv9uWXf3zx4mnQJJYLl9E1pU6SZYvV4u5b97vtJudSltVGK5yfn5r79442+yKNSgsxYtJsZdusVEUsaDfwoG1VWhhIG9uHP/lnbm/n8PPPP3dd17IcCLCSaL1OpICMMafpYIpdx15Uq2KdhUHQ7g0MBJbjDqvdB5+oAtBvHn67jBOCiNSgnGeY2ZbNpMFJSZUSvJRVtQ49+uzJH//5X/xpq904ubiMZPGHh9+kjltlRkK71aGiWhmVb7RKYHLLslfLpQc6GjKuqrwsC4gkVBUWBisAAEYME6SU4ZzX1GcMDcHAtlGv3fIDdzGbnZ2fKFmUGe2GtuKZg4hczYEQP3hnv99uKki+fn6Cws56NfvtN1/HWby9s6XMxrd5cjUdJ7zYnG1sbG80u6Q7sqeLqNExs+WrB9sffPrDd//hN7/2XJtSlwCSmAobbADUBtiWjwIyzwtLwFW8ihOcF/F/969+JAUs88q3HEaJyleAstBrKCXms6kCZrC1xaUpAEROQ4K0rDjBlpbicpEU+ubjro8JaA5aF+dnh3tHjdDP02Q+HT89eWLbtlKmPxx8/PH7BRd/+1f/fn7OaXGRB+HRnbuGOdub+4Sg8eXF8dVsONq8c7A7vzg5O70Y9DYun7zcPbitdCnc6t6dg6eq+PqrL3YH/o8/+POtrc3r6+vdDkUqOX86a1B6en4ThF2Z4sV1Pl2vESWl0mklGdOpUARLgZlUQCoQuL5UZr1KtIAQueto0m05mInZKqLMGY021rPZPImGhjf77en4Ugjg+Nb2re67nxy1Ok3oJuenzumrZ4PucHZ1o0t5c3n9r/+b/35jc3Pwbv+Pv/mDGJSGYgnV7u2trx9+s7E16Fm2SHBzf5fC3cP9g367dT3Lf/mLXyxmSeA2k1XpQmu9ii5XD2/3BqsnD68q0Oi273/wIMUmFwm2nSRLCNRIkyqXi8kCytndt+6v1rOg7QAIvMBPksTUFQ9a2Mx2XKssSwS042DPsymib+SE1351jDkXSZYCAGzHrQ/0lJEyj6Q0rm1pURgl9nZ27t57Z6elGCMUlpWskrS8iapIsMLYEhqMGCQYCIUtiogHMdJah/C1J7HefNQzBYQQRKiOP7yxKbw27zGKKSrLshJVDWPWRiuD4zSK4qjT6XDO/dAjDGOKTs6OqR8yjKAT5HmRJElVlLPVGgNTlZntEEog0FwrrYDhPKuKlFi0lv0cOxRVQRS2qe1bDjIAaEAdSgVtNFoHvaP+3V6elzOMv/j//L+Pj0+Z5STZqlPi7lZnWw04571W51kpg4bf67RuLk85yEu1kgissyTLsnar49iu5CIuqzItbdexLbfWSDQwnMucc651mclRv9fptIxRBkjXthyb1MJ76Aey4uenZzfX1xSRTqej/Ry3cK/XMb6dpJFSot1ud3u9vCyY40U5v15cFXlVGmgARABBy45NjpXCHoYY2RpTGxoFQs+VlUyjmGrdbXUt5vCqqkquOWi0jeTlah7brlfler1cFEW1ni0mkwnQphEEgkcYGo3VJF60slXLtgXWjUF7r9nsdwda69VynaYppowS5Hleo9kE08lkOpcIJGXRbvtRFFk26w07EPUIIXleOr7X6vX8wQi3mphzq9VWSiUAraQMM3H1/PTk5KQqsq2tDYxxVRWDfpcQgl63FxqpNeS8jONovV4HQXh0dDQajbqdntag1epQwhaLxYunjz0/fHVyJk11C96uRAkxKKvM991SK6Ol1hJBhBlGADCGeGUgopaNCMEASm0gQwhCUFUFwsh2me26AAAESc3saw8Hjh0OBgPbcnd2Jppji9oEoryaZkmcJOee65a6xNQIE3/16LPLy8skTmvVK8/zdrs9HPQazWCxnFk2ffHy2XvvvTfod6+uKtdxxjdX25OOKoRFmOUFQKoffPThsN3zvfD8/HI+X64Xy3oXORqNIISXl5fSLxmjsjJaoTBoSWHquqMH73xw6+j+apmkUZalFdpnjFkbGyMFc6jhKlpeXJzNx5OXz59tb/YhElqXV1cXnh/cvn07Sjg09s7O7Xa7H7SHp6fHs8V6a3soZBrFy5Lj45Pnl5cXaRqHQXt39wBB2/Ot8/OTq6ubjR1MKKKUDPqDP/2zn8RR2u12r29Oowg4jjMc9W/fvv3ll9/kef7+e/fm8zXUmNmOFwbPnz9fr+PhYNP1vE7HLsqMcx6GvtI+V7JWERljjuvVyEilJLOI67uB39ja2rl9+63xeHxycjKfLbU2lIp1tKxKYccuw6TZCoMgsJklLJ2mWVVVvYE13NymzM3y6uT0MskqXnAFDGUUYgi1MUrVnQ5lnhPIT08zXqwvLs+6vfe73e7rEDnGVbleTHLb1oxJ14KtVq/VanqO5/ntNBNPX56PJwtggOUwaIAxmrVaRkMAADKoqiqBBKCEIMw5xxjxqBRlsbk1ajQaNRAaYamVDhvevdu3NnqDbjPYHo14kTc81/VY4LlFkSklnjx5Ml/NZ9O5bdsXF4kGpq7hiKJkvY4hBGmSj7ZGdc57b2f38nKdRAUyhBHmOI4ByECkuQBC2ZiGnpes5lCJ0WiwPepPLk8vHOi5Fi/z3Z0DLlVS8Ol0Oh6Py7Ksle11kiJIBFdKyrKoiizzbDsIgrPTa9d17t294/th0/ekKBEijUZrb2e3rsYACErFLYtsbAwHw1fCcTEjm5sb3eFwa2dPG/nrX/zcdkiaxpPL0x//+E86naerKBuOupPJddDq9DrdRtjiXCoNIcCWEyBoOXZ4687bUoCT04somodhs9FoJUny6uS45mwaBRBC0ADJK8KYy5jUyibMIpCLipdxlqQOZb1+Y3553e92235z0O3tjDZBpReziax0K+zwghdlNuhvNML+9fVSa2bRJoSvS4yiKGo0WgCR9Xq9tb1NGTs4OJhHq2a3NVnOby4uR4Phvdu3dhqd7c2tbrvZ7bTee/sdBM0vf/7zq4sbhAAO8GAw2Nvz1nF8eXm9Wq2ePX+BukNA4Xq9Ai4iBBkA8jQr8pQZSLt9i+BKKC5KwqjtMOyBWrh+44/DGNdSfCNo1qTIigtoTB1qqFduwhilVANT1yJQSgkh2ijHcQiGGBkE7dBvHh4e3rt7fxjwoig8N6DEKvJ4Ml9E6xhCjBDAmBLGLJsgQjGtMfPKVEUtR7+ZaHzfkQC+h3iqLZzNZrP2Ida3rMON9beUZbler+s5Qo2ODYJgFsdSSoaJEGI6nV6eXwAtw9BnlBjNIdKMAEKgVlXFC87LggGjFEIQAICgafiBy0Jq0WYYYkhkJaaTZZoX3W5v//DWxk4jubis26RsxzPwtXC4sbHhOI7nebWzcjabFUWRpslqtULA5Hme5zmjFqUUEvzGCqqUqs8JECOlTD3ID8Pwu1pt4Do2sFjFC6A0NGA2m2khfd8/ODjglbAotRwbEIMQYYw5ngfLkguR53kleF6VgquqqkrBORdaAYAkhFBhCQkhhBGENQDSmLKsOBcYYCklMKaoKq0AhDBsBL7vn189T9NMCFkJDWDOuTQAMWYNB5tJkgCEtDYMYy1NlKQX5zcvz66RAXt7B+89eNd13bOTsyIvCSEGojiO11HS6LRrVJQxBmpzeXlZ/5brZDhljBDSCoNmM3Q9y2IYE+a4zLZZTdHAvIii6Hx1vcrmYnkjpeRVsZQpWUdzrbUQlZQSISil5KJcR0uEkDE6CLxaJXNdtxE2GWMHe3uYkdlsEqerRthcR8pizHVsLnJgFDQAQkgJhoAqpTRQyjBCGcYQYSglhxghqDAxQgFCCKVv6lONMUYBZXmW7VhB0Or1t/7kT2iZ86uzq7PTU4IqF6nVYg4quyw5QVSh5sNvX0ntIYhd160Pju1207KJ4MXGqH+wv/P73/9eKx74ThKtsySez+cuUv/yX/7L7e3dv/1Pf396evb2ex/32+3zsyuHsjSKF4tF3XK2v7+7tbvjBq7xTFmWjTBohB2L+VUBZpPTy8tLxiYff/xxv7f16OHjq5tJJUSj3eiPOgb5eZbxXBij2t3WXfrW9tYoL9bfPv5mOrvev3X7k09+9OOf/CQMepyjFy9O/vrvf75cLd57777j77x6fDqbj7d3RuPZDVe8O+jt7e2/c/99YPD4ZhnHsdewpotrv+EHzQASiAgpeJFXebvb7UUrRPFiFdVlVm7gP3j/g8ePvj1/dU2I7br+9dV4PJ4M+kNMIKaoLPMkSZpNX6mW1hpT6vie1tp1XKVFVVUQAiWNlppS1m51pZS7O/u97uDq6vri4mK9jmtpNL7J0zS1LNpqNF3XrufKEMK8ELt37naGo1uHtxH6G855kma+H0olIABSSi410LKqqiJLKFIaKyPKi4uL9957z3Xdq/F4uVxWVWVBPb46x4gPe344au1ubd25c8tibuB31kkRxcXZ+aXRBgIleamNBsCGAECIldHGAIgRI8xmljFKSlVkOee8kaa4EbY7PT8IkOtVZdFrd+7cvq2qskrTq+uzk+OXRRK//+67f/7nf661/Prrpw8fPtRa723vfTX9o+8D33WiKKqqqtfp4rv3LsfnkksCyVd//Prho+cIIVGVACDLYsUq0UhBBIGRCihdFa7r9hoBj+Z26Nw+3Dna33rx6Pev+PrDDx7EUZIkCcSkLMsoirI8RwhhhNO8iKMUEQK0MUYrbqQAdsffGG1vbo1W88XNzerm+rJIonbTPzrc6w83tjaGnufYzIqSCAAV+sE7D96Kk8XTi0gZsDHo7hzudXuDOFk3Am93e/TF57+dXF/cv3vr448/+PyPX19Pjs+vxhvQJJkvBby6nhW5KistBcwK6XhNP+ycnl1FcW4g7Q82K6FfHZ+en18aCBVXUkogVRGnVVUx17FsKpUKfN/wQvPKZkZAoQW3nbAfhk3XVUJu9jt/+qMfxXfv/e73nykhLcpGg43L64vAaTTCXp6pdVSVZenZfitsN5ttCDEwMMuyx48f7x8cYCF3d3a+/OoLhrAFIYXg/bff3t3c3giaPC/G1zdnx6+wNIf7u+1Wa2dnc71Y2ra9sbFx//4DTOnjx98qDRijgEBR5YvlxMdtx3UgQ67rFnk6m82arg+MMUZxXjme0243bx0N0mvFbBtCWJallrLZbLmum6Wp4IpQ6rmuFLqekWkpsyyrnQSUUql1jaJCiFgWhhAySjDSSlRaynarNRhsdDrdhicAiontEuoYgKQAluttbG5JDRGzICYAYgORqquABIff2x/o76ANEMJ65aj3CrWlt75Bo9FwXRcAUJZlURRpmtZTiaqqatVhNBp9+OGHjUajXn1tyjDGvu8DEK6Xi6IosiTRUhEMuCgpNkHgei7TRolSCK7KMqKUAgSM1sSyKAGMQkJwo92ACmdZDgnO0mK6Xl4vppkQFxcXzWaz3+9HcSqEIIz6vt/r9aqqyrLMcRxeVtfX10VRLBYLDNH25kaNsyyKohZICCHGwJJXBBd1Bz2mBCGitcYYb+/uEEKU5AjhOkZbFUxLMeoPKKXQgCzL4igSXNq27XneqppzzvOKQ4gNBHlWIpIAAKpSSG3qdg8DkAZGy7qUEhkFRMWV0FAbIZQQQpSV5YeOC4yGCIG8yusFjzKWFZxLw4VRVVmUvKo4sWzfgxRhy/YD3xdFXuZFUUgAmee3TFnkeT6dL548fVoV5fn5uWO5R0dHq2gdrZPlamm5juc4PAy1MSVEBFFMsc2c+WIxmd5orR3HgkiNNnpaFmVlIDKMMUyRBpoLLoBcmzwlPKNCmFxpKREnpiAAKAAlF0VZlp7nNFsBoUOEAKW0KLM8qyAsOC+FUFrrNMkihBkjeVEx5jrMWShTZlVd0A40xhg6zLEsS0pZllxK6Xo9CCGslSZYGehqzQHUjWYHYQ2h0VoLwQHgNYtkMa9SljUb3vbm1mC0B7gE8PeX4/PpZNwIXY2UxhoQ3Wq133nwoNm6evFqzmxnMBoyQg3QhCApS2NoI/A6raAROAhIiiCFRvPSJkRX1fXZGdEoi6PZePLN3/992OgsZpOb68nJ+UVdO5plyWQ+c3xvNps0cDtZp1mcr+ZxlhaLxWK1TItc//GLLybjpW3b1CKHtw4cx8mydLa8IrYGBmKHeNBhmPq+3RsNVkt45+37P+z86KOPPlESvDo5fvTt6cX5+NXLs0W6uHPn1tGdvVJkv/vD75araaP9p41Wc/9oD0Hiur6B8ur66uE3T9I073UHjTZax+tnz4+NJlWpEMHaQGY7d99+i1AmpBqONv7sL/ph0Ini9MXLk4Pt21fXkzjJhVIQwiTPWq2W5zklL9bJshkHlRhgihzHYdROs7iuSq8jiJRagteND1pKLaUhxNrd3d/c3F6tVqenp5eXl67TrPJKCyCEytNK8kWRlRjjNC2bzXbQaLbbbZtZyAAphO+7y2WMADJGaa0gggBojBElgBEMsVmv10KIQsgnT55c30w8v+k7xCKmGYb7O1ubG6397a1ht5cmBTQm8MJhfzTsj1dZkpYF5yWipO5RhEADDYRSUmiDldTKZlYUrSpeMYssVtF8udzaGH3yyQ++evqlANwPmOfTs/HFerW4c3Tr7Xfu/ce/+quvH35jgCqzvCzL7c2dyeQmiWLXtTGGGIHZZJoVedAM9vcP3MB1HG88nnz22Wcnx1f7+/eqqiLYrcrcc12tNTTS9TyMcQJEw7f3Rp3DzTaGqt8JjS7Xq/nsJr51a1dKfnx6YjlewdVsMY+iqNJIQZqVhYFQcKWUwggBABEhjLqOFySJ4Iq4QffgMMizaKPf8V26Ws53NreMgc1203ZtbaRl0zu3DoWsSvDs9OKyyBIgRbRe3txclUXSahwdn7xSZfL48cMf//inu3s7v/n8j1KVRZEJbqQBaVLajg8Mnc1iweH29rZUajpZrFdZpzdodHpPn784PT2VWuzsbq+i9WK1KoqirEpYYt+2+s0m57zh+0kaxUnsEIp8a71cRfPcFuDmfIqA3ui5ZT7t9xr7253Hjx9PbqLN7W0EKmPKVsMZbg6CMHz+/KXWgFKLUsu2HMuykiT56uuvf/yTn23v7d46POh3e+toBapqqzvwMD178uyEi9ViWZTZ5PJycT2+e+c2MFqUVVVVdSxLKTUYjba2tgxAu7u7awDiPFvOF9ilDb/nuO6gN8AGhNRthi1VcgppkeXz+fz0/CzNEkQCDGtLXSm1lkZrrYWUWVEEQYAJAdAoIQEAEGPCGDSqHuEzxqSUxsB6AQPAUMoogVwbiInrhgiSNCk9DDRgALKc65wbYrvd3sgLRKPVU9pwpaVSQumqqpIswxiv57PamfjG+lBvDqSUb9KYdaea1rouTQ7DUCkVx3GapgCAegABAPB937Is27a3t7dd15VSZlnGla6BgDWkyKL0erWejm/6vQ4j0LKoRalvW57jI4OkiGSZu1aIMVbAUIy0FFmWEMKklOvVknPlhgG27Pli9fe/+FVVVWup/TDwPO/hoyc3k9lgNPSCsK7VqOFOQJs0TevNQZ7nvV6vjkYrqbXWRkpjDAClgQAjWfswMCWUWnVGdJ3EQghe5GHgBb7jsMCxbOLY25tb7XY7juNHjx4VZdnt9nzfL4qCMCsryiiOHNdFiCijOZcAIaGk1sBAjCmkAEFstALGAIIh5zwv89oyaVFGCAXMUEoZoRhjoyEAeZaV62Rd8EIZrQ1UAAqpuFLCIKCgkEYhbSMKEVWASiMgcW1M/KA1Xi7TNCuKKs9LLWSeFk4/oMTqtft5VlKLMcYE54pXy2g9nU4t5tYbPl5VFBOv6bZaje2tzV6/bVmUMYYJhAQqJeN4tV6vX12dCyFW0QpTbLuWUVRpQi1C0myptc6yrOIloSZEXhC6SrcvL64uLi6qUo1GG73uCIDKGGPZbLaMpJRZnvQ7nU6nBwDgosjz1AADIIQGakWNsiEClEmINCPO6yQvUBaBNgRKCaWF4zADlFKVkBUikgGFMcQYx7HPrMC2e5h0gXK4KJnld7obUspOyy8LDo2hlFluI2wOBoL+4fPjJM48z3McVhZZxTMC4a3bh0WpVstJtxNaGAYOu3vnVpZEnUYTITGdTpM4Pb84rarys88++6f//C/jOP7666+zsjg4vNXpdG7GV2dnJ0kSzWaz5R+W/dHwvfc+2NzYvry8fvL45ddff3N8dtrpdJ49e9HptX/60z/56Z/9CBNwc3MlRDWe31DCGLbnk/VsOZ3eTNMiS+OlHzh+0Hjx8viLL786PbmZTNeLZbq9vf+nf/aTd955+9atw3U0397ZLKt0Op0GgVdPEK+ubm6ux5PJ7OZ6JqVMkshd0bIUJ8cXSqIwaLuux1wOibVarbJX52Ho7+7uv/Pex93ORhKX/+xf/DcBC8fT5aOvv6EUI0xvJuPt7a2dvd2z85MsS+bL2Sgdtpst13U1MFwoynntFmTMopQBgJWSSkLfa0oplRIIgyAIGo1GEHij0eDkZJJEseAlAtBhFqGIl2Wa5xuULmZTqbXv+xuj3tXN5Wq9AEYyRhDB1ChuAEAAY2jZ1KEQ66re/hNC4sXy+NXJOi8pc4kqDna2P/r43Tu3dyGofNdVEqRJibGKs6TIBaEWsxTRGjPLQE2JAwCAGkijNYClkKaoiqKyKCuLQkpBiooLoZVoNBpSm72t7fF4jBHKk/T88mI+m+zu7t7aO/i//t//bxenZ9cX59eTccsP7711l2B4+ur4wf27X3zxJef81uHBxdX11c01IejTjz7dv7X76uRlt9Vtfzi6deudVvO8KsDNZG5rk2YJZXRzYxA2/Hi9Chv+228d7u1uQiPPT1+NxzcIg7wsprOZ5zvLaGXidZwU51eX4+lEYYs5gTYQU0sZKYQ0xgADilLejCf2t88JdYosPTs7T+KVw+D/9r/91zt7h2WvYxDO0kJr4zhOmqa1h2tnc+v+XXxxcfXy22dhs93qdFeLBQJQS/X23XuijK+urp49e9Zo9999993iD18t1ivCKLMcxwuanTazvPF4phTY3NoVQli2v7G144cBJERrmVeJ1vJXv/pV3TRICKqkkFIawXVZDgYDJFQ6X82ubzY3Nt69+/aoPwjDsGXDKIrCwHNsixcJNOa99w4agbq+vn7rqGvRot0Omz6yCUcy5fkiWkaL6WI+W21ubu7s7EFEJpPryWQyHA6B0puDfplGzAsO9varqgBlKSrhu+6o13UQsihL4tiz7U67nWeZlHK1Wr18+XK2WExm873dg62d7exmbIxJosjPW24pIObAEALtsogm4zmDuNdqep43mUyi1dpybMv36xM8ItgIIITIUZlXpWVZECMuRVW8DhS02+3BaCiKnNoWgiQUvChK+R0fiXNuMQtBY6ixKAmDFiW2ECorkdZoFZeL+fjk7CpJKoioMno2mwupuZLaGA2MlLLglRCi0WhQSt+EId/AlNI0rWcc9Z+1jlvvFVzXfSPU27aNMX5z4yRJLi4uVqtVEAS1I9JiREmTpikCACHU7/am48liNvcsG4UuQTpeRUDrbqcBDdQSaAmkBFXFhaiUMkVRIYSCoNHr5dfXYwDgcLCFMZ1cj1fLqKrE1lv36ornsizrAdPJyclisWg0GrPZbGM4gAbU1a9hGIZ+UP9FCFHkJQCgHj0YYwilEKA6OmgEJ+R1yV+ZC2QAwdD3/Xa73fQ9qJVtUa3U+Obm9PT09OTEdhzLsTUw0+WiNWoKJQFEvu9TiymthVYV50VRaYA0AEZDA1HNStHQ6EpWJc+zQkrpup5jORhAjWSaxhBiQgjFBFPkeLaSBkCjjBZAK6AMQTZ2iEYQYA1Bq91Jo/h6PJdV6Xue44Vlnl5P593+BrPjMsvLQiKEbNfjXL58eey6NkJoNBjmZRFViee6UsrCzZQCVVmmRsuKNxqNra2NZivsdbpZkpY5xBgjCiE0VVWuVqv1en1xdgYAgMbYjFADNYBAQplxcn5xUg+rjFEASELqHaWO4vXLly9s29/e3tnd207iAgBk27Y7W0gpZ9OxNmYxj2bTZV2d6jiuMVRwlXJYMMmYBZEFISw5rgNECKF6IGeMFkIgDIxRCEtMJESKEGTZjDHW67QajYbn+EKo8/N5Gi+5BNs7e51ugKBer9dZktuWawA5Pr2sCuF5QRStoihar2UcLYQsbYb29kaBz5TIdrdHnssogdsbwy+0KvI00kWz3zUUY0YHm4MKCMToMloVvMjLYrVeGKCur6+rqhpuDj3Pe+vO7War3W21McBCqNAPHzx4/969dybzccmz23f2P/j4nVbbjbLlYCtst5s76XA2nk3HKwUqwjAkMCvSyXL+1cPLJ8+eLubLi6uV5ztJLJTG999550c//mEQeFmRE0Zv3X7renxzfnEdBN7zF8c1JDWOU8tywlYghc7L/OzptNsd2o5fFnqxjM7OJ553wxihDBVlAiH86qvnv/zlFxBYSqD79x/86Q9+wqh7PZ42mkGjESyXyzhNIAYQGmKRghdZlgwGPUwRlwJiBICp2fNaa62AxRi0MAS4yCvKMCFMSp7nuWXRzc3N7d0dox/lWXJ5uZ5Px7rdHA6HjmPZNrFsvI7mUgttet1eq9dtTmfX8XrO3IaBShvBeaVAJaUESjLEXNuyiBkOh2EYPnt5ulgs1nnJLEdhcfv2+/fu3et1G4vZjZAGE7s/2FrF1cMnL79+/GwWRZVRAigNSV7mGBQ11kZrLbgUSkquMFRJkniOixBJsrwRuM12Sxnw5MkTyfl6vcpj7tCg1ezZzLGdIOO81enu7+8n6+j4+bPVdN5uNcJrHwEoJYfQIGBarVYcp2dnZ9Eyopg+ffT8m8ffHJ+cOG6gJF4t8rzQ6+WKQQkA8Bus3fW2N0fa9ELP3T/YNFo4NgsbPoDqvY8+nI6vseMSx11dXsZpmhZCKeUGfsGN1AAAIITQBhkNDYIU24TwKKqevzhBjPY6LS9wHc/tNoPD27cObx0aUS3m4zha5hk3QCVJyrlQSmHEus1Ot9G+PL9i9kPI2DpeHe7vNYP2hx9+NLm6ePzw67/+q7/58NMffvzRD569ur54/PRqfHUQ7RGGKaVcyfli1e0NipJLqTa2tzqD/qvTk+uLCwMVc+hkPh6NRlJKrWXtMNdCBsxvMO+t3aP7994eDodKCYIxpTTLsjiOA08bhD2HrpfLF08fDfrt+/fubG32W6G7Mex6lFDboZqrZA28oO+7L4UEAF5dXBDChFC+7wuhHj16tLu7bYzZGI5Ekedp6kAULVfbo5HX7CyXS4yAKHKLss3h8N7dtx68886/+3f/jnPe7XbX6/X55eV4OvP8EGNsFMKQCQ2gpnkmFusCIRSt8+ffHhNl+p3O/tbBweFeJZXUutlqza6qNE25eo0tEkqqUldV5bZ8gyCl1LbdOohr23az2WxubRoEIcAaGK2NVEZKXUcSgFZVWVZ5Rgl2XR9jCgyeLqKqEtE6Obu4HN9MqOXari+lclwfSkE1hBgAAISSvhIQQi1NnudlWdaTglpdqPtp34QpalY05zzLMi/w692D67plWTLGiqKoc4b1Ob7uWa0H/5zzfrNZVVWtFjuWvbm5rYQOHLcs88DzLEIqXhRpEdU4CUMhCYUiaVoqIY3RUlUUk1bTpsTBAGkNsAFaKAzRcDBoNpvKDV+8evnixQtCyMbGRlGVy+WytkPWm546xIQRqBkPk8mkDrR/1/eGjTFKSQMBgv9YqFGHOQkhzHIJwqHv7uxsD4dDLGWeJUCqm8urKIpqTwZjbLFYAIQIowghxmw/aLS7XcZoXpXL9SJN0zQvDQQQIgOI0VC+doJAXuZKKaAMRcS1bIdZdcDnTSc4Iay+C0wwhMDyLVBCUBojtQEEK1BxkecVoVaNvhZVCQCwGKsqUQn10dsPxuPxxdlpEkVGSAJRlVfz+dxmVqfTaW23hBAYQC8MLMuyLev6alxkWVpVWZH70o2W0Wq1urq6qqpKgxo0DjDGEL8O7Ng2U0ohjBhhQEOkoEUsBBHxfEu/RoKboswmU27btsUc22a+729v7+7t7W5tbV1d3ihlwrDZ397BGN9cXU4nE9f1KLEdy0IAAoAIgoIYpaGGyGhmDAYAAIxcz3cdnxCGMaWUGg2EEFIJjDFCup5KIGxqRcFu+o7jYICSbL5cL5JkAU1JHZMtVxgZYhvH4EF/gIETZatus//xJ59OZ5PhoJMk66TpAKCKLELISF5YDA8HXaDBerVwmGMzWsiShs7xzeWtW3f+4l/9c8bcRw+fSKj+7J/9eWfQPz+/AEoTQsLQn80KoPSw11dGCV7xqgiCxsZg6LkhwpRQ9PzVs7OL406/hbBapzNEBETycvyiN9wGxGggMIOE4UpwneokzpIsb3c7m9tbkNppxuMkarZaw41Rt9u9uro6OXmVJNHLly/Pz67CMECIKKW1llojy3IIZggSKfP1OvLCdiX0cp0DQ2078KBdVdXlddRqI6U0paDdbi2WeRwtO+3hi5eXWH4Wx6nrB0VeNRoBIuTq6mJza9AbdH3frapKAYUZ1VoXRWnbNoaQUguA19XyEGKMCYTacchqtYrihePYm5sjq9cERR4tFvfvvx0E7qDXvr6+FLzEyGxuDDzPW6zmXPE0XRYibzfde3ePbAePJ7NlxDUwQlRlmUsDlDIYKG0hy3ag4p1OhzFWS6DGmLKsgo1ObzjAhC1X67QUvutxCSFkxyfHv/vi68vrGXbdTFSlqQzUqySzseGcK2kwJAAACDDGAEIIBCKEKG2MUdR2HNvTWs+XK6YthoKyMMtF4nntZrOTJuXz5y9ffPvi04/eG/X7n376KQUASFWmybPHj9Ik2tvdXq2yi7Oz8/NLSliel0+ePCUM19ejy8vLq8tZ2BgUuex0umU13dzc3NoYjkad0UYnDDyMMbPgch6VJW60Go7j9Aej9PCovpCd31xejcdxWrp+c9MO5lGR5EIBajKOMTYQVkJSwlw/xKiABtiuFTQb1KaOTTv9bhCGSinHshzbK7J8Pl/P51Ot9ebmJqWUEo9AfrR3+2b6+cnL09lqraF+cPftsqzSda64ujofT+cpJN90+rvvvfdBXqrZfFKIwg/cLMs4L7XWrVYrSRJIMMbQ9ezlav7y1VM/CPyAQSKfP3/BMHQcp9Vq3b11+52373/wwUe3Dg4RQkkUT6ezly9ffvvtt4vFopaCuSml4qLKVZW3Gi55x85judHfjOBSFQZJuLycmQzcf/tBaDUymCkh79y+SzBrtbuYkGaz7YXBixcv8vxn/V6v1+7Mb67W08n0IhtfXTAAKgkmk0kYeJ5lW5al/n9M/VeTJNmeH4gd7do9dEZqUZlZulrLq2buYAAMBjPYHQK2NK5RLF/4QfjID0CjGWlGGs1o5GIpFwM56uruvi2rumRWahE6wsO1+1F88LoNxFNaWkZEhvBz/ucnubCYsbLTvXNwOAsXtmlVQtSLeBiGZ2dnnGMEKVBScTCfxqPZzLLtKssN6ldlUmVSS+xajU6jN/XmtunYNk6SREqJGaVaaaFrAWkNjJum2fUCy7LiOE7zjCVxy3dKIZSqKDMty1Ia5nkppTRNU0tFEDUIFbzMs3I+iwBAi3C8WCyWYTQPl3GcWQoTZkFE8rzkUgIEodSc86xIa3KBEaOmDhljtUqxji13XbfORahniFrPWFMPWZbVYF6tYyCENBqNsixrIKROXaynjVrEUKsCJVdhGI4G4+V8YRq2looAYttOM2hkebSYLQDQlmVJjUsBl5HABFNJkiglmGuFkMZ1nqAoK8WFZ5iu63qu+8Xx6ePHj6+ubvygabsuQJAqXf9jtYcTalC/kFpuuVgs/tCp8Z8jsTkXCmiC4Q8BVj8kUQKMxR9eSDibl2kShwuTYM4rjHGj0dAIZnm+jGPTtvxmI1zGSqnaKVpynmZZWfC85KKO4wRaayU1VEpLBbTWkkvwpmOCMGZiTIXI4jiuP4u6xlNooZE2iYkxJghJgLFCCGipJdAQIQCRmoUTy7C8wMshElzoKkcY2q51fnEVxzGhVq/vAKmKJC2ylBDSCHzbtn03KMsyDMMqL7I0VkoQikyLaa2VFhihWuohpVwm8RsVCwQY43q7tywLuoRLRTEBlAmlEKOmYWBEydpaP8uyMAzjeDmbzbIsMwzL8zzBgQbK8zwAQBiGl5eXVSUajRbxXcXF1cVlkecNz/dsT3A+nYUbG1sGswm1JMSVABWXZSVLwQPfbbc6rVYHY6oVBABqDQVXUkpCEaUYQiBkVVWZkFwpVfG0rPKqqoo0UbDCVBVFlqcR15lpst5qo8yrdtNdhsV8PO33u57TAFBvbKxGkVNVPsLy9PjlMlqMh2mWJQhoodRsMgq8xtbGJgDgJB25rruxt0VsM46TlY21SorAMtc21us2l26rvb29fXp8YhjGzvbWJBwXRX5+fj6dhEUpuQSmaSKCKaWbW+v7tzcM2zg7P6Km6q+2yiq6uDxFhO4d7Dj24qsvnnz/7InvBgjA1fV1ZtoQY8Jog9rr63uU2OPZ9D/8p7+9GVxNJhMp+WKxaLQ6/dXezc1NUeRaa9M0CcNJktbKQUTJchkHftt1AimQkshz3Xt3Nxkj88V4MLi5uZm2W0RwYZnug/tv31yPXr486nRaK73+y1fPssxudxpJnmkt19ZWpZTz2YwxhjGspCh45fu+5rLGLbVCWkOlgBA8TVPXdV3XJQRJVSVJogHHGDHGABS3drdXV9rPn9lPnz5ZhJNe1nI9s9NtxWlaVDyOF51us7++eutw9+Wro7/5uy9kfeUiABXEGDDMarCR52l9aimKwnVdwMxGo7GxtdlbWUWE8pL7QcMyzPFkcXMzffby7OT0ElCr3+qms1EWxZDBQsi27wCNCllorSHEEGqAINDANE0uhVbSMAwIYRhHtsk6nc78OjVNc3ozPD+72txc393b4UxeXw3OqtMiT6EU77/1sN9u72ysP7h716R0Gg4ty3ny+DmEUGvQbDaLohBC+Q1ve28naLY++93n80XKDLJYRLfazXZ/5e23H62u9LXkrus2fK/I83A229nbGQ2HtusCAGSpOr1VvxGYppnns+lsPlmcYyEQtTDGlALMWFkprVGW5WmaIw0c27YsG0LQ6gSE4WUcezajBq0EH4xGjmEgpZQCs+n8+PWZ6zgba1u26UKEXFtsbe04j5+lnBPCTJOZzP7+u+fzyY1vG+fnyfqGO53OHz95/i/+6r+xneYXX/3t6urKbHfr7PiEEGK5VqfTGo1Gju8tFiGmKMniMAqZbbqe1eu1c85u3779/vvv3zm822o0oYJJFB+9PPr2q2+fPHkymcyEELPpQkpZNzvEUga+e3Md5nH61oMgXqrry3Ctu/XBOz9azOYnr46+efHUseb7G3fjyfBv/uPfhjy/fXjYbDYRZZ4XVJVYWVnJ8/zLL7/85OOPZ9PxZDhKwmWn1ew2Wprz2XQ6n80oQQSBJIoHlxeeY+/s7Ozs7KhzQCk1bafX73d6K81mc7lcchRIKQspomW6KPKr4ajT6UGlFQdlJufVcjyap2luWW6z2WUmdawVTEkSxfWRt6qEFpJzHkVJFEVFUQiu6j7A2vhwecnTohRC2I7n+wGAOI7TJM4ghIxSy2AUw6qo0ijmZcU5z6t4PJlmWYEQMSwHAFhWAmNauysxJRjDevWvqgoTlCdTIQQhJAiCWmHwQydkrWGEsMY5TNd1TdPMyyIMw5r7qKcKhFCn0zFN8+LiIk1TAEAcxxhjwzBqcwfG2DLMEvHZZPrNN99Ei9DzPIJhQYltGY1uF2h5dnKaZnG73eKyQgilWebaFqNqOo+VKFa6/Y3VVAilpS7THABAEIwX4fX5xbJSpmlub2+7XgAQaXc7pu2YpjmZTCilGEGDMmiyqsyVEnmer3Q7dYZVWVQ19SCE4Fyapkkwq4dRzjnnsqYe7E6QpWkiuEmwhZHmleICGyycx57nSS4KyRUEjUYDUJym6TScIoSqqhBaAaCm82kdn0UI1VpLBYQCQAOlNQAQAMQIrZ8I6jchFrXdoP4gqGnUKhDLsizLJIRcDMZKAaGkggpBQmzDtInWkHOJNCKEMIsx10YA5mma5ilJsmUYioozSrM4mYyHDOJer1MUZVmWjUZDS1Ur5wBQbdPc2NjK8xwAMJ1O6x+KoqoqYRBDgTf9ulIrrQgEJqNeWIUVrxyCNdBSA6QBBAhqRVZ6641G6ze/+Q1SyWp3++zsbDqOhzcxZWZvbaNQbJRUy6vB2Ty8Ggwd18cVNi2WJ/GLl0+/e/msKor1jdWDvV27b0kuFSxdy2fU4pVeLpM04psrK8ywijSTkDp+B1IjL4SgChFATcIoASqTcaLyWJcp1HJaDQlFCEqhUy7mVbWAqmQIrjQbBKBbtw6ePXllqmBj6/bfH//y+ugmCq8IIeFoRinZ2d48PT1WpVdqI47l6VFC4SJPi16v77ntv/gXf/773/8+nFuu6waN9ZOz65cvjm7fvt1ft+bpUmI9CsdFmu3sba1tbwzGw/3dvaqqilIGjbaQ+ouvv4rivLe62mq3t3a2N1c2KYXLePryxfM0WzQCOxvnEAFge6bJclD0e2t7u7MXa408L01qhcsZNdjwYppmJcSW31qPi+Li5lpcLbXWQolSVNCgSVW9Or+EEBqOrwCotNZaEy9AQGutLQBwroJmk8TRdDzpr3Y/fO+9+/furbRXXr8+/df//f+zXOYGtDWjFdfnJ2fEYIkuLcUlxq3uGqZGGutO04/miijHtZjd90ej6yLLNAKNpqVQSYAZLZa1HNpkCIEKqMoyYJkvCSFKVGVRGIQhZUCADGwBU84mE4OyW7d2o/nk5uJ8dnFlA40pqUoupPAte2t/P8ry6Xz23nvvPX92fHR80gucXtNZRplCqKh4xXm4zDDGBXWAEzw/Pu72Wma4WO/YP3/vQRrPh6ep4/rUsqVSpzfTv/n7f7i4GTLP9fxGniYqE5Z0dAr9EvJUEU0NBKqqAlARQhCAVVVJrSnAhJiIMikppSZA5jLWmWZCYGIFRMlwllr77MHtO/Fg9PzZ019f//Lw1t5jSMPtjURoy7IOPvikcfMUAIAsBBB4+OGtVy+PZuHirbfu9bprRSl++Xe/M1HDRmQxiG5tHty7dXfz0VqzGWRlgnRVLpPR6LQZtLIsGw4nZQEIxqXgpeRhNZ9ky42tdcPvM6+7SI6m0RRAGmVVf3UTEcOk5mAwAKJwbMBMBXEOicZQx9OCiSZBoO34bdf83T/87R/9+EdcFlWR24wgLaajq5ezWRqH77zzTqvVmufqfDBhtiOyzLfFeDxa63cXBknz7OzmanWnM59Of/5Hf/TowWET5v/s4wfv3LENA00cmVryrYP+B2/tlcsbnwETceTQ68Ho3u6uKsrffva7P/0n//SDtz9YWbtXK+YUBI+fPD07O4uS5Onz50mWKggUVgJKttHAGBdSLMsiQcloOTQDeO/BrhLlzfz8g/dv51Xy+Om3CJFYVlsP7wFi/M3Tb88urz7/5qtkHr6+unnrwf39/d2qyNf7zdnYy3OSJ7PZfLjM05so+ebFqeWMgUa251fVc0aMF6cnDrP/2T/6pwfb20k4p5Xiy/ndzZVXR896a+vvvvO2Yfi/+/03DCAhc5fRhuGKMvEU32oYRTWTEOZioSyBvWBWhNfTMQYQCOgQV0D59lv3kiRJs8JvBEqBMAyBhstlfHVxORlNF8vQd71Go2EZFiMsTHKMsWHaCKGiyAEAGCnPo1JKxhAAIsrzPM8hhEUqZ0VSZFVRVEoBShGzXUppbRT0m/4PGQlSysD334RAr4C6iJlXxTIU9Z5kmqYQosiLN3ZBCNMEE0JsyzAsFobzWrqIEIgiYdv26uqq7zfSNFcKNBvdaJkRbAoObNtejpN+v88QwbjyTN+z7HgxP379am2tn2i5u7d9cXU5nU4NxxlOp1wtNDKUEpRRA7GwKEstLULyOBJputPuDgYDUQq31QyLKkQqtx1l5Cv2VrvZjqJES2BZzuhmGC+XLb/Ji7wOq1ZCi1LyUkGEoFDMop1OZ7EIwzBESiMN+t1eq9UCiLx69UprbdpWHKe1n9MKY5OyaDkv58mdnVte0Pzys99QAmVZAFk9f/4UY3z79m1oGJ7n2RhjxxsMBpwXecW5EoAXUghUlkIrRDCBhAIgFCBS165sYuE8zzGSjs0MongRIV12Wz5jZpqmFNFWs+15QY06mKZZxulwNAZcmtQouSjzlDITYlxmeVUJ2Gi1Om0ltBTa8RoIknA5r8oSAKAgMl2viwiGkFoWJtixjKvrmyLPCAVlqjRQleQ+hI1GkKZ5ooHUqOSVZ9k2M0ouZvNwPEs1AISATs9jhoOJaWnZ9BzDMCHACgCMqU1ty7JJ4HcgQPdu399c2woXEaNWGEbTechMc7W/6gYtIeRkNJ1MZtEy0QpRMI0TQKlFGbRZY7238s/+7F+0G93lMposx8PRdZHfACAJVZQhxsj336eEGVwozJz+2mbQWoGYUkrzNIUKCqhklRbpvMojIEoMgWVSAJUSQlcCa2BRkwMteKWkDILG+dXli1cvt9a5aXndlZWqEjWqA6E2TePp82fPnz8FQB0cHMR5hi1jHIYAgK5BFlny/OSYeS5YsoKr45OLJElanW6WFa9evd7f3UMQXpye5WnW7/Ye3L3T6zSi5SxN02WyNCzWanXu3T/UmLleUAlOKIyzWEo+mw6ubwaz2QBBxRiCSCO7RRDSWq90O9PxOE1Tx3Jbzfb9+w8d28/SLx0XMsttd5o9ZFzeXFdC1jwaBJRgRDCAtagDoVpBLYQQUkhZB74pglgUJWmUtlrtDz/45Kc//tFaf5Vh2umsnByfKfV1lBRRlJiWbTsUEVoKVFu9Xc9uN5oEQsHLFy+erfU7/X4PE4AxjqKIWQaEkFIcjRcIIc/zDNsiBlMAcCW5kqZtSSmFVgoCzKhGMC3ysiyFzrRUrmkBSCDEeV6+fv365OTE87zNvR0NUBbGbqPtNHyCyPePn2QFz8uq3enfvnefGdZ0Pn/6/OV4PHZscz6ZqqrMk2R3c204uPYdw2Z4NBpBjDq+zyw7LYrZaHx8ejKezU3TBohwLsuiyLJMSk0IIxjX62CtduacSy7q99AyjJpGRABSShlj9QorZGEaGCHFRZWVxXQ6KcvtW7dulUX69OnTm5shAGg2W3z97ff9ft8wnLXe2uPHjw3TffDgHkKoEbQLXiFIkiT55tunw+FwMU8QsRzHw4h4XjA+H0WDie9ZgW+ZzAQEq0S4xG6aDUkQM6yiKAoNp/PJJF0QrgqhlstUcAAx4gpiTEuuDAyVAoiYzLJFWdVB7obBDIvhSgohMCM3V4M4XFCoWn5wsL+zu7WppAAS7N+6Ey6+/P7ZizQv+/1+yL2jo6P5bIIJSdNkc3OTELK3t8MYSpNohPHG1ua777//1v17QRAkUfz29vuTyaQsZBzlWkHfazmOWxY8ShOMaaPRSvLsYP/Qcb0/+qOftzudi3H2/fff//KXv3z9+vV0PnMDf2NjAxFcSQExUkArrYWskEZSK6mFKrLVXreIwrPXp3d21/7qL//ipx9/9OSbr//P/8f/07179/YP76ytrl1cDbMkefXiOQKi3Qh4md5cnzsO7TQb3fX+o7ce1NfL6urKwe1Dy3ePTo+iJAIIj85HSVwe3DpMCwkkI6x9fh1+8ZtfHBys2xbsIqkQjNKw4im1zKBhGpYquSmVklxQapqUSAi15BqCmJCyLONwAaE2LSarChOQJEvTchkmtm0jTC3DrIQ0DAMCVJbccZzCK5FGdbCBqESe577v1fD4Hzqi/lBEAqFSihDCGIN/aEjnnAMAal6glhnW1oMsy2q2HgBQRz7XPyOEKCW16az+gyRJkiTBGNfugBr6Vkr9EKUACKyN5fXdLcvo9XpbW1ta606ngzFutRumadYpNUIIh7DFfKKUchyn023s7+8RAniZIiAN04BI5VlSFonjOK5taK0wZUUlOedVxU3DbDRaTcfa2Nxut7omoQiSsijyyWS0jJFp+u2m5qyqBKUGYyJL8mgZc84dy55Ox57tNAIfIRAtF1zkCAPXtmrpm+u6SumqqrIsWy6XWVY0Gg2EACGklgjUeSSWZWEAlRJ1GGXtGnVdN44WvCohQpubm2maTqdToeTh4eHH778XijxOl+PTmzQjCqqyzKVWEkhCCAIUYgI0klppACFCEKM0ndiW63kexjhNwyzLCCHdjkepURYhJai/EmCMz88vlVJbW1t37tyhhE1mUwUgpVoBDRCruLBMk2JlMoMxpjGQkJdllUQzDHRRFLUyhRGjjpqllGZpoiUv8yzLY0YxQkADZfLSc+wkXZYldz2LmjRKYmYQPwiSvPAbfqu9LCrOTMu2bcMwENa27TBqMmYiRCDAhFDLcgxmEaztaBH1e9v9HhgNh53WyiJczmYzxw9sPwiX6auTs/Prm+ks5EIt3cx3X2epcO2uaTi84EWiTdzJIyOwdwrbmON8kS6LasFM6TcoNYy8AEhkWc41XGICK1kZzKQG00qqCgBVijIu80jznGKNCGIAaKVKLnXJsQKEMqikKCuowLNnLz7/7Csg0db2wXA6ibJICLWxtnp+fn52+nptY9U0TQn0cDgoJPcC3276acU9v6FMYxiFV999devWwbsffDoYDI5evsIAvv3Ww8Vs/uzJU1mIt+7fawW+sm2eJ4PLiyRc5HnWbDSa7SBou62OT0xacKGRBFoxi9za3kvTuNF0vIYTzlerMsUYUYaPzi6KotBSFZnNCN3c3Ox1Vlb760Lo9bVNISFj1mg851K3O8F8PoUc1PgPhEIDoZQCGiqAqkqAumpGI1gTwlApAAxkFkXBmLm1uXfv3oPNjR2DMlFVnXZva2trOBxfD4ZlmfuB3Wg6XMlKWowRAu0802kWOabh2AajaDIdBg3b87x+v08Zdn1fKWUYxrAMLcsiBsOMKgiEEEIrjaBGsCiqJM8AABpBruQyieM4LvMIaTAZDEc3N8+fPQunE6BkVVWtVivNi8vBMCnKly+PvGYzL6pFtLS81kefbNy/f18D8OLFy4uz83QZ+rZFMDJQ6/at7a5v/9GPPn7x/BkGuigziQDQYBGG40U4nofzKJ4vE8t0K6U1wH9wiUNGCGMUAV2BN4sgAhAoLbUCdcVvzSIqrZnGGBOEEYBSSkK1ZROOqRAJl+VgeDmdbe1sbwpeXF1djQbDLCswxklWBEFQlep/9d/+5b0774wnwxfPT3qrPWba4+l8Hi4GN5OnT1+0uh2pCQSsKsWLFy+iJL1zsIExWlvtzpFoBv7uztbgZiRKzTMCNJZyniRRWmTPnn97dPJ8b2/n4UcfD0dToaDruLKQCMHZdEmNsuSiLAXBlqZICc6FohQCjR3HZIQ+enAPI21gVCTLIi+1glUliiw3HWd7b+/J0xevnz7llaq4sppbRZZgjAxGV3qdn/+jP1lb6wshbm5u8jxtNBq721tra2uI4CzLhJKDm2maZv2VTc9qra1uTyfLZZi5rt/trGZFAbFJmHV45+F8sYij9Pzi6j/89uvj4+OnT59CjNrtdqvTkRCMF1MFNEUIQqS1VFpprSXQAMOWxQxZmSZrtINHdw53NzYNjIDg/82/+pdFUUzGQ6lVVZbbm2uPHzOg7cPtXcMwOu1m4LmCl+FiIngRNBqHh4eLJOKK3Trc/OCTtyFlJa8+++wzoYOcUwHdZSF/+9XTOJw8+fqr06uzIED3H+xTBhZpPJ7PGhopqKI0UkYnK3IkNXOZZZqAUCoFoqjdbmdZGi0Xm+t9RsHpxVUUhmmathqrWmtiMAShEEJLzRgjmJZ5VZcmKK4QAEDVpjDwQ4vjf9n1UNsTOOcYY/1fVDPUM26dNV7j/z+Q8Wma1ojCD/nE9YYhJa9lfYZh/OB9qM2E9Wjyw4BSP1SWZFprrQwEQe2pIRiazFgulwYlgee7toOg5lWhlBK81BjneVwURaPpm5Q1fGut3y2z5WIxC/ymY9JhmYyG155j8aKoMUhGqeW46xsr62tdioFv0M2VrgIwSfMwTiDCjuO5QhVKVEUJkCWFrEpNkQF0xYvcd4OdnR3FhcEohLrIsyRdFlmMMeQUJEnCLBMAYNt2o9GwLIsyZhjWcrk0bbfWJfwQSWlZFsKMMRL4HtZiOByYBNiOmWdkc33HD9x2p3l2dnp1dRWniWEZH378YX+1RahK0jlm1DCYgpUEAiEktICqghorCIBGACGMKMLYs1GWzdJ45DiO5wZmYGdZsZyPTMPybcMwjGgx5pwTIL3AsxkKXN8yDKC0lBJjRAlTGlZaQa0gUEryqihMajiuUzEhi2IZLfI8BxphBKAJgNZCaiUqpQQSquBFJUqICAZIaQGQHE2vIIS2bbdXekqp+PUijWPLoczArqKEehBiahiW5dTMFGm0MKaEEAgQAAhogjFBCBHJGS+w4kxKYZmewZwsLdfWNrZ2d84vbl6+fPnd0+cakKDRroScz0Ib806jVRZVw29Nx9fhYvpv/u3/hVfwv/6v/mW3bwXNW4MhmS9GWnOly7LIXc9i1GSsyIsqz5ZClAghwzBcz0ZQIiAFT7UoIeIIY0KhKkoAlK4qnmdClpRigxjYhjc3wyePn56fzxqBNVvMk7Sch9Fqf61hBUXZK8oUQngzuimqXEJdCA552V9d9yjZ2NoxLddoBmlR9na2b999OJ0np2dXZVHsbO12mp313qpvO1Cq23v7ZZ47jlXk2Ww6KctcSe5tdA2LOA2ngjIchWGcaIDaq82Lm9M4jpMk4mWuAOIaaQkYopsbq0IIinAQNCWXnVa30+xsrG998cXvh9dDRilCqCiyN1HkQFL2xpukNIcSKiC10gggqWqFDlYQaKgBBBBqDADQ0LYd3+3t7t7yvVaSFLHMKMKz6WUUJY7jbGyuE0IQwaaFkFQWN0TFtZYIAUbo2lp/Y22FUdxuNIsiS9N4c3MTcggh1lpbltNb7WOMmWVKoAteKaUgwYzRSoi8KgteUUo1gkKrGl3w3CZQYjqdjqcLANHm7l7D9zCEw/H46mZ4dn7RW+kv5/Gz58eMsZXVtb/4539ZlmUcx0+ePAnns41+X1XF6clrAsGD+3fvH+5trvfWVxrvPDiklJ5fnI3H4XwZvjo5PTu/irLcdANCTYQNLSqAEMEMmtpzRR03p4HkFddS1eWQpF4mtFZKiYrX6y/UQAlZc7Faa8ogJlJwAZFCWE0mo2fPnvq2ubW19dZbb32n1Gg0poR5nq81+PzzL7ot90c/+oRQdz4YlYKbtiG16vf7q+ubBeeT8WI4mgZ+03EpgOTo6KisEkrxcDa5OD9eW+v9q7/6V18/fXV8dAY1sywXQgiA9nzr9Ozs/OLKazUajUar0YyWGWFWHC2F0MswthxfKoAx9mwHAEtpCZTEBFJKteRSl9ubG61mw7XNyeBqOh5dXw+aQVBkebvd7HZWG602QDhOinmYNFlECQw8xw3cW/v7n378kePZf/3Xf/31N19KKTut9vbebqvThphOp9M0jaWGi8UCQfPgYKsRNEfDSVEU6+ubUiPb8fb3907Oz66vBr/53W9///VXECO2splVlaak2+tubGxoCJbLZVbkpmNCAjHGGiogtMaaQkg0DkriMXb39v07B/suJecnx7Obq2g+z7Ks2WwGgddqNRBJeJkd7O80261kllCKO52WY5sYY8NgaZYsw3mj0xiHo+miWF1f++kffxy0mhdXl9PFlZDx2eXLlU7Psq2vnn7hWcbG/qaU2auL6/W9za3u2ng8nYdCaI5RUys/jLO0yJlJTe4hppUCBGPbtjorXUJQHC4P9vc4L09OX5VZHkVRlYOyKPwgIIaJMYaI1LXFWZJbpskIyfK80oIgbDJan/VrYV09JfwgyK+J9lp4WNsQ6puWGiEgJfnhXvUe32yu1dhADTyUZVkPDYxAYLD/sgeynjw450BJqBVBEGJSPymE0LEYhJAxs06SllJSgHieaV7ZzDAJdU2DAK0UgBpgDZJ0WTtWqjLWWqdxBCHsdhpJPF9f6d6/e9htehSqJIosimzbNrx1iCFmuNVquV4AZFlWfDINVZY1vUArUJSlhMsiSyoAuBDIs5UAVcEZMymiFax6vd7dw9tYKSG5kiWhnTtkF2GlBJdSTodR/a6maRZFUVVVeZ5jTEejkeV4WZbleQ4Q5FzUBdydru86tm0a4+HN0dGRSaDlOF2kqcmm06lScrpcAIq7/ZVKybOrSzEpb8aXhUgNbFComAExQJUo6zgAhzohRwABAABJREFUCQHQSEOEIFJYQIxbtu87LtBQa1iVqixzCLFvNW3bvXXrYH19vcjL6+vr6XRalPngahAnejaZFmkhgdYAUUNDQpEGoiqriouyyqPEtu3V3qprGsKz8yIRgiulAVAacKl0keVVVQSeh6SWgGukIcMQa8BlHZNMCGEu8Rs2AKDRdjXiTmAQwnILGQVWCmBKGUOEYACAaTdqGSbBDCGsNVQSaA0Jw27g4aoEWZq6nhOG08HNTa/fzZP0/Pz0+OhVsgx39g7379wdTxaD66EqWu//5E/Oz0+rqhByvrG5fj36TZ4Xj5+jw4O7nfZaSzBM2kDTJC3zdMFhadt2QD1Kc62VKBMBNNasADkmmmCAgACwxFACKJWWUnIheRwv5/N5WWSOY6/0u/3e6tnJpajk3k7v6nL83TffPnr0tslouxX41C3L4OIK3IxulnG4vbvjNoJC8iRN4zQJ2h1I2WA63tre3W53V9Y2Hn/7mGHyj37+p4PLi+lwwuPUMRyLGLPJ3Dbs06PXvX7XMAyldVbxl19/043Xl1niNoNOr204Nry6HI4nYTQjhCTZsqgKCBVkCHLAlSg514BDLQzTRFADBH3HtSzLdZwH9x7+4pe/TtM8LyvTdjY3V5Mkw0TzggMAhFL1BY0xBggihAigP1APUkouRX0OkEo5juP7fqvVsixLKaWlZqZxcXE5m82riju2GzRyBXS703I8b7GohoObNFp2uq3tzdXdrS3HovEyfPjo7tOnT69vrlvtRiNoYUwxpo4doK5Ro/fgD71z9VpWLyK1JEr/IUPeNE0DEcYYQEQotb2743sOpVQL0VvfAI+fXE+myzhFzArcwHJdx3JOXh1dXFy8ePH8+NVRf6V7eLgvs0Qk6fsfv//jH3/abzcarjWdpu12mxmUGXRjF5ydXpxf3QynM0TNjtdQgFTxDBNTKaDUD/8kREhDIBkhNZLLCEEI1eHNQGlGaN0ziTGWnOdC1FS6JiLL07ra0TRZHscvXz43MHj/vfcOb+9bBn327MV4PIYQaK2Gw+nf/91vvv7q8c/++Mf7B9tX12dqXu7u79q2CQD6i3/xZ4LDb75++qtffvaLX365u7O1vbs1SeYVL0KeDKZj6FkxgseTyZOLS6CpbToAAIrhSr+15IVgGLnWdHAFRDUeXEHEwmXGbA9p0fSc+WIJtJQCAKgg1BgCKIEUXCGZllUSR6LMFgQMr68uz87yZHn/7gEzqWlbGiK/0SDMnIWLSsu0qPI0Wt/avHf//vsffQiRzvLkm2+/Gg6HaZrOp9O7d+/YnuvZzvPnzx8/fryxsdlstnu9VS3k2dlFHMf7+/vvvvP+IlrmZVFVVVWJJEkGg8HJyYkX+B+8+1El+CJaYkKjJF4ul1VVMItprQFQCCKCNYCg9ksDoB2JH9w+/Md//McGpS+fPR1f3TCKeZk/efLk4OCgv7a2u7tbSvXv/sO/xxgfHh5MB9PJZDJbzJOE7O/v93o9qdXVzeWvf/EPhmsyixVl3GxapgUtS3/wwf3h4G8Gg4gZ2LYCXmXNtm0abDScBz0v6LWtoBmeXj9+fhp4abe3LrSX8kxoRYAuOZe5LkVJKVYmCyczP3Ad224GjWW0qJ2HYRi5LKr3acO0AQCEmUEQIICSOBZlBTVAAEghJdQKCok4sdgPU8J/eX15nvdD5HA9atTb/M7Wdh0RWMvma2i91hj+wFwURZGmaT37Yq0AAHWIb22DrN004g+3H560Pm3XhoJ6bqjFmIQQBEGz0XAdBwDAqIkxrk/khJAMCkaQqDCEUFSlEtK0mGtZTc/3LLMd+Dtra03XOTk5WSwWhBAjWE/SdLqYHL06Pj09KfK0TCMHkQ/efuuth2+v9lYfP378+uQ4zJL+xma71cwxNgwDKo0BJAgblHWaLd/3gyBQUiAgLJt5jg2giKIwDMP6pSVJEsdJ7cjIi4IQ5vs+QIQxVlUVNZhtkzzPMcZKijzPgeZJElkmNS220msXmRVFYRgvkywOo0Wj0ej1V+bz+W8//22JojCcE4al5lGaW7aBCCrK2LBMBQGAUCklAYQKYVlpgOdjOZlMypJvbuw8fPDO+voW0CRLS4RIu93e2tgxTOoYr6L5l1nJG37r5mYgStFwPQVBUuRaaYowZjBPgZZSCF7KpEwTCoHnuFmaMoqVQaWUlBJKCUKyxFppmZWp0ISLEgIFsQIIVJorqTAgCuhpNMtEYZomoLqx0my1W+12tyiKNMnysqrHU8GFlLJIC9M0TYosZpqmjZFRq7aJrKBtNbJ0STAzmJVlWZYlZen9+je/SLJsbb1n2lZexK9fvZwtwtHommUtE3S7vjw5e+4w+2c/+dFkep2m6euT77jIVrpTBB0EfccKWtQwSDdOTxtBi1JaG2yqohBCGAZVuiIAYagR0BJqKbngsgBKK5UkyWKxCGfTJI1c12m1Wq1mZ6W9Mp/Mer3+7uaGVihw3GiZxPOZomQyG4+mo7OzM8dzV1f7vdX+9Wg0GA4hhKLi48Ho9OLStvyG376+uLYNoku4vb621mqpqrSoMbi8ePn9i7cePei0utEyi+IzQHElK2oYFYCj8TwvhBu0Hr3j9vrdVOQ3k8F4dtNotyqQK1ghBDHUVJKqEhxwz2K5VhQjUZVaQIxYlqThYmmaNgLYsqwoitfX1zc211+9PsJQIsK01qi+7CGQSgEJhFZvEEUIIEYQaAJJvS4QXUpZ5sWy4hnCwDAMAYQU2mCWkqjIK0wNRi2/4d25fbe/vrZcVuPRaDYeQKAci8XxYrkoIVDj8eDVq+dxnN6+fbvZbNuWW1V8uYxkxWvIhyKMIZJSSsE1klVelFnOi5IiXOVFVVVpFGOMC4gMKUslrSAwKJWSR2kCAFjbu1UhvMyrv/3bv8+5+vM//+dAo9evX1+dfxOFIaqKbmBjng/Pjz3T+Ff/4s/eefetg4ODIkuyzLq5uZktQkRIu91e21qfLaOciyjNvIapIRVCa4Udz8uLggvBq7IsS8YIRhhjbTpeDTZqqeolmHPu2k7N9TJCpJRFUUgu6nAnAECWlhRDhJBjOwYis+no7Oxstd9755137hzsv/XWwy+//PLx02dxHGMCojg5v7gYjK7efuf+7fv7e7c2NKiOTk43NjYAUkKCP/qTTx88ethd+f/eXE8uL6+thpvPM4HAzuF+e6WTiGqe5cixIWASEiHEMo34uCqKZVqJ8WKCZP7+23fD2RBTexmlaaEm0wVWhcOAEDzP84oXBEHDpBgjrUSuJAQgXk7mVcHLYjS4CRczz6FXVxfNZjPLrKwsLNswLJZk2nJM17ZMRj/96MOPf/Tp+ubGZ198ETSDO3duX11dJokuirgeE0vBB6Ph5fXVYDD+9NNPd3Z2qrxQSh0eHu7t7TmuNQ2nFxdnURoBjLa21+4/uH1xczadzS4uLizL2tne5pzneZZEsZScYKCVenP+gtrAuM5uV0pt9td6ftN3/CSOb66H8+m0DgJf29w7uxxMlsn9d9/fO9iXEL06PrJ89/bhgxKALE4ur2+Y7RmWbVqu5zZevjy6++BuVRWv06VQ3LRZkif72+vv3O3vrTlpUqVJebC3u7KyEoZhf31f6Wptr40RECR/df68EcwVBRKXpm/WtDeiQEqOtAIAi4ovF2GaJP1+TyngmM7O5t6IjaIok1KGs3mWZYyaRVVSanhBQAidDCdaa6AghJBiUoNbSsr6iP9GH/MH7qBuVyGE1FURtVuh/ptajVg7F+pUg3qnr+uef2imfpNVjLGBUX391rPFDw8o/3CrGYcaipBSIqDqGgiMMTRNIURVVWWWUxeblGJM6nSm+nEoMQLPBgBQYlgmwxincaSEmE8n3VZ7Op796u9/1W4G8TKaT2ZllkopZxezQvDFMsx4ThguyyKPSyLA4a0Dw3Qdy2WWKYTiVZUmERgp1ECGYQghgeamxRzLaDQakgslJQBASj2fhJPRGENdL5mEEAUBY6zRaNTmqWUUmaZNCMmK6ofXW8s5TdOMoijLEqBlmSerdw+2djYtkw7yGABNGFYZIAZRSBdVnvP8anjltphhmX7DT/IkSSKpJUEIESh0BQHUAEgA3yzdEiONeCG7/RXb8qSAv//q29/87utmo9ftrP7kJz9bW13v9HpZnlLD8xudxTK+vB55vmObjmGaCuh5GFWCI0qkAlmSao40wggZGCKkJC9zURVFWVSVkFJCqBEGGGPKsFRUaakAVEBBLCFDmCAMqBbVkqcGMmSlF0XqOA4hFEI8SxZOM5AYMs+irqW1lkLXJmFRGSYzGKEEYagBApBggjEhUmrJpZbSte00Xk5GA9ukgufHr58FjVav01RAD168vry64Uprxaej8a9/8ZuHDx+23K2Nldt765+ud3Ol4dHR66yoxgPVblmu53LOIZRewFx/zfd9qHSBqVayoqwqMoSAVBhBqYQoRZHncV6kRZELUWkIlstFlqac8yxNsyxrtUarq2uMGWnMUQe9/87b8TJxKDWbjSKOv331ut/vH9w+HE/Hk1l0enp65/69tfX133/9tVTAsn3P841bdr/V05Waz6cnz590W+1HDx9Sgk5Pr+ejie95P/vxT6IoWl/fevTWu9+/fDqcTdorPcOxd+88SKsoTdOrm0Gn38eGWZRlyQtNGISKGRgTgjEmCDguLUtWVZXNsOQVZVgJDTQEUg+Hw8UkqirR8H3TtheLhVQijiOgVJ5n1HalUgBpqQEQSmspgYYKai01BAAACAEiEAEMAAYAUM2V4kUZL5fzJIkc0+El51oixBgzMTaURJQartMAAE2n8yyTnue41jaCsirS89NXRZZsbaw/ffbk1dELBMnJyUm7tVJyBSGeTRcMKsdxGo0Gw0ToN2XthJA8TbM4KdLMpEwLyYsyT1KMcY6wTtKizAyDIpNFSR4lse/7uZSdtbU//fM/nycJBmhzZ/fZ98+PT88whjKLbAwa/a5tm2WRKSUbjkUhFFU1GAyEAgJgKGFVFpUOv33x8vmzF0+ePQ2XCTK8q5ubvNAKAK/RQkgQgiAAACjLZIwRhDUAxDCM2mldm5QoJiYzeFnWxzigNKpP5TUlUQIMgenaQnOtoO/7WRqlaTqZTMbjIQKy1Q4+/Oi93lp3sVgcHx8vxrntOaPx5bdPvn3wzsG9+7fTPMzK8PmLx/21dYIt27Z7ayv/8n/6X//+i283Lq5fHB+b3bbtWkGzESfpq2fPL8/Py0KVhUQAG4RWeUJEblm67RpMycX4/N33P/zGo8QwKJTx4noxSZaLpNvxCAQm4VgLjICBAcVIAr62vWUQ2ltpCl5CLU0DdLteK2jEydwPnKSIy4IThkyHrttrP//5z3utHpfipz/9qR14p6cnXFSWZf3jf/yPr66uEHpFKX3w6GG707m+uprMZ8RgIq9OT48ZIwd7tz755KNGvz84Pf0f/of/fjybQozeDd73/UaSRYtwijHIiyReJof7t9vN1tOnT8fDUbyMTItBABECWGusNYKAUWqYtN4IPdOdTebDm5EQYjoNX78+s2373n3vZjS9uLzur/efPH8pCe6trf31f/qP1DQXcWUYhuc40yiCF1eEGc2gAQBqBO1+ty9EMVkMRzeXhCGpOIPqnYP7puWUhTg+PcPU2NrZNEzit+wvv/5sbdsiBL+Lt66vRqYFzGbo9uMi7VBMINRclhBCw6AIwTLPCUKi5AwbcZhIKdutFYM5jh3kYTaZTaEGQog8zTJYVFXFK7FcxiYzbMMm6I1QAAKglOJFUR/ff0g4qE/8aZrWYYiEENM0bdumlCqlLs5PXdetD1o1eF7PBJ7n/RDJjDCA6I2sr2FZ9SBSIwR5kWVa15v9D9KENwkWVcU5N5lRXwj1wIExrvuTlmFlMMswDIkxIRQZBiRAK4EQhhBSTBm1GCW6UqXOMMCW4czjybMnz4DSVZFBDUzTLMv8dBwS0+BAIQM7tuf7fuVkWMokzcfTmQFRjWFoIWfjyWQy6t+1PbcPtEJYm4wZzGQED26upJQGoRqBQuWilIAgy3IoJXnGuZL121IHTSZJkiQZhDAvOQAgSRJDCgBQGIa2bROCeFWE4VwJXh/GFovFcHRzfX2VJFG4nFPGRBLHRWbbtuU6UhaVwMxEpmlWoqy7rahppGkMMQIYAQCkUlpDCCWE0CR6mWSzxUgrUpU6S8XV9TUAj3/567//+ONP33vvvV6vs77Vvf/ov7q+vv7uu28uT64554gQqZRtMgYMhFBa5M2Gr4Ssk7IMSg3KtNYYQUwQ5FpKnmW8xnQxRpZlSMkxQVxpgDRh2DAYwEpKrGyLYFZPnEHQ0FpXBQeIDmaTGlrCGBPMCCHENimEPAEYYSGqKOZKQoxpXcJJLMeRZa4VxIRcn1yfn582mk4UL2zbCpezebwsKuX7bn+tISG27evw9eVvf/PLd9569+7t9zvtdSo3KTUdr7m68Y+Pn3w3DSedZtuw4fXNWRTPXNde629JqcssL/IUY6yFEBUHQEAEhCwrnidZHEXzZbRI06Qsc81gEsUQQtc2NQTLOL65uWk3W+PhZHtz9e7h7dWVtZuL39vUfO+dd85OL47OF8w0Go0gaDYqwWuOKi+Ks+MT222srlrtZufth9ueF8wXy0iHxy9ffBsuXz39/sN33ttc7S8Gw7/5d//hq8+++JM/+RO0f+vBW2/nSlEv2D3cvxkNocVRSYVCANKSyyRNkyThinuWR01ENFMKAa0RglBhDQQXaj6bh4tIcoUQpdCo8vL09DRLy729/QcPHuVlWWtGMAJb2xtllV8u0ze8I5CYQIAJQwghpIB+s5MpCeR/liBRxgkhCKmKZ0kS+Y6PQa17KAhmlukAjIRQSZJ9/+TF6cV5WYn1tf7m+sq9u4erB9sUq6uLE9ez/X7v6uoqibOrq6te93xtA6yvbTOWhsMbqLRjWiXCtWC4TmIBUikutJAME8e0tJAUYaDB5fWNlFwC3et1TFdHRbbMc8t17Ia/mIW99dXN7e3Tk/Onz18+efIEQLycDrvd9s7mludbjm2F4Xw0GlGseVkYJmWMJVlKqCEAuByMjo5Pz05fzhbhMko2tjZW13eWy3wZhbbjC6EgxKbJKLG1dk2DQqQQhpPJG8+31rrI8prxfcNHqDfahRpdqJfvIq8Ihtpx8rwEQlg0QIhURfrixbPrm4vlfLa23v/0008fPboHIaQUnrF5ksT3H/x8pd+YzSbffPfl4e3doOGcXVYY6yQJf/P5r7c2Dw5u3Ts/P/71r3/7ybufNNsNhEGaZaPLy5k94nFS5cq1G1BBx7BLWSFZuYhihEFeXF+cfPLxe0DkjU5ra2Ntd2frxwI22h3fa1iOY5oGAABBTQiAWldVCalRVdX6al/yyvecKFwkSSKqotlsrq6uCgWkDCnDhOBOp/PTP/5py2nleb6+vv7577/4u1/8w50H9yte9Pv97e3t4XCIEFpdXfU8b75YnJ2eL5fL9ZU+QshxHNd1B8Obp8++HwwGw9GgEoIwmheJI22EgO+7zVbw7eNw9PjJ2w8f+Y4rK15kmcmYZztVmVOMMUQIaoKJaVDLNDCEQojBYNhut8uyWi6Xo/FkvkiiODOds739/ekyDpP83/z7/zCL4z/90z9h/4//+zwMl8VRr9Nd6/eXaW4a2fVgfHZ6fvzy5Uq/ffvw4Pad3f5Kp0wjoStMTJcyv7nuOM7G1rbDvvwf/+2/mU1Ge4ebw1mxiK6y0u02Ow/fXfV7SiuUZ1NBoqK0JKVKgqosGWOMeELILMnqViRGjOlkXoci9Lqrm9t74/Mry7K4FAiSuWkKpTHGSZwGQcAIJZCISvKKV0oRRBBC2GI/IP9vloIaCTCMOg8xz3OlVC1QgBC6jlWnFNe9d7X9gRBSe+LhH+bdHySQYRjWigfGmP4vSiNrA8WbWRm8EVBLKSFQAOia1UIAGpQZlCpmJElSsyOCcymE5KLMCwAAV9owjDzL8ryUvEqiCENYZCVQcKWz0mk0eVEmcYQAdBxruVxW1IEEx2Veqso0TdtzaattE7qzs7e2toGUbA6u6jpH33PbvTZk2PPtPEMQIoIRRjBJksuLi6bfxCaEGigFkriYTsej4XA2m7Q6Da5kr9dzXa/uEI7jGEKMEAKIvPnUTMM07TrF3DRZniWLxSzP0+VyeXNzswxnw8GgKIo4joejkRf4WZbNwmpto6G11rhSSjSbgR94WuuiqiTgCqpKcKwxBPgPucZaQwQhFCAGBFFCIcAAq1JwLrVSUCP9uy9+8f3zr/f2dt55962Dg33DoPt3Ns6OLtI05TV9ATQ1LA2BEKLT6aRpWkdlY4iUUqKsqjxr9bppmsZxXOSllFxrSggBCJelBEgrJYBWSkmEAaUYEog9M8/ztMwdjBWChBiOZQdeI03zoijSNC9LrrVmmNW5yTLRCCElQVUJwSVjZhA0PTcgWitAsO35osqur8btxhrneZrlQDKCYVoWo/FEY8IRiJLU8qzuO7cWi+Wvv/v7P/un/9xb3ZhVZxCQSQpXuv2gB5jnQlgoCdutBqMwTdN4ckMIYpRaEPI8q8oSKIEQ0kILpfJcLqJyPM8WyzytuFJIFUWeCoswXoo0TBCvUjo/Fl91W417O22moldPfv/Fr7787/7X/xvH2rpzuD0L6Wg0SmTuYJ+b6uX3p3cPh41OO8+U37QfvPNea6WrIJyW4bPLF8so4iV5+epqb/ceMb1Sqt3bB61e+/PPP399eTyNZ2+/+97rVy9KCQzNbm/d5pwTDxVlluQp0kArZRDaa7YDzzexoaWqZMVLASFGAINc6VikPIvyJabk3u1HokRff/7dZBwSaKZh9fDOu59//vne6v7Hn36QZvH14Pr21q34+DLLMgAxMGhRcgQBxKTi3LRYURRaS0ax0FIWOdTSNOjD9du+7weBZ9s24nGRjFwvgKQScClganikEfQWYRaGcbgIz86WSMW6UPObec9fvbV+50cf/pNXztOqzE1Kb63dS5PoyXff/D765Ycfvt+zWTkf5dV4cnxciu3trd0oWSZJYpp2lEiESBQvsyxrtuyLy0wI0Wq3hsOhidVkPmeEwsQ+uxp8/+RJt9X60eHDDrIBTJ7+9nOdpu/e3V/b7L/31mG310vgkCGzLIVjN2fjZRA0x+PxeDxUQBwPrkpZybz0/PZ0Hn/5zdPPvviCOJbBrNbqtmE4QiPMaNDwMCJQVYpXyzivs+SYQWrxhNYyjsJZVZUFhxCapkkxFkIwxnhZSS21BgQiBFFNCVu2UkrN5je2bU7mM4U1tpz923fCMDy5PN/Y2F7bvzdYKnqzvHt4+8c//8t7D88uL87m01mZJp9+/MF0MPz3X//Hfm/l/b2PNjc3Ty4uFzfP0uns6/mvOV8wqzr6/utWq/HWW2/d2tvaCtz5fNr+6P7oZrC6ttLr9d4I0KjpBv76+rpSKuaZZv3/7f/u/3B8dMwwswyzKKqL04tGu1WWJUQIM6q1Pj492drZTuKxIfl6t5ctsqqSBgyQ7oiKuq4rBFoubSn5Yg6KnG6uH9za32sFPYIFT/OnL75bXeu+987D1ycnzcBZ63V/+qOPtRJRFFGK4zT64puv5unS8qxHD3/6/vvvWiaL06XFcFVKislHH7y/WM7Pzs7OXh+v91epwTb7m7/9h89ubexFBrm4eH779m1MeZzMVlZXNdRRFjeDwDYM1zYZwbZFLYOVRZbkqUHN2wd3siRP4wwoGC3B+qZbCdDf2P3Nl99KBBCA2u389vmZtboPPffybKYMnKl0XuDl1Wzz4F6rY/z6s99nVfV//b/96//uf/nfPnhwn6Am1lLwKg2x46VBo71YnLue/LM/+yhoe/Nw2uuvzydXIsHj82Rze2un15iHkUjnSIJ8IaBNTNsyLAsQLDRRQGvLloYZx8t5Gjf7vdn1pRLSj6JHjx7t7Ow5jvf8+fPJZOx7jTr9t4zTWj/IAapHfAAAV5UWkCFFDWY59hv7rnhTUR3FWT3KAoCkgnlcSJkCAPr3uqu91fX1da11HMc1W6GVhuo/WyIxwAooLXSVVxiAmtSoihJjSOrY5px7rltVlShyWfsqlbIB9BjOyhwAgCDECEIFykxACCFAFGpRxKJA9VhTQQgA0lrbhpmkS4RQkcs68tkwTUQgY0xoFYZhhbHfbUMIkyRRlLrLOMuylutqrVtB486dO4bJ5vN5UuSprBhju48eHr73zmg0evbs2Xg8zuaTl2HYbbW73a6oUsUzAThRRaPBprNFp9cvYDW4WkZAxBYJGSmShcHMrmVMs9x1KC+k5feEUATh7e3t999/fz6dKSU+++wzDPEnH32yd7j6b/7Hfzufz6tMhXP+8tlwPhtVPNOgihMuFR0NEg0JQf7ohgCAMxhBCKdhstK3hAAVl71eEyKZxRwziLBWSkCkKMWEYKAVNdyiqPI8kyJFCBsuZjaWQmnNKTGKcvb0Zfj86HvO+e7urZ///Ocf/Ozd3/3udzfHp2/4oyqhlLqMwDJ7eHALIcRL4TiObdtnpxeDq0uirZVud6OxMV+EYZzwVKUxrwS3XMv3fdt1uagICwzH12VepLEGFDHqUo9ixpVBiUsZqySuJEKYWQ5kplBaACU1yDiAdW6YFFwrThCggKo8LiUhRVGYjJZVuZhNKKVSCEItxzMxI+eXF1oqkxkKQAKRTQ1MKEPcMhEEYjIe+m4z8FuO3RAAhGEohFBSIwwBQAa2iE8825sOzoWogNYmZZZlUUqRglLKsqoKXqR5lud5HQqGNJBaGyblJdJAQoRs2zSAASFYLBaKl41G4+T4rL+y2WxZtmOUVYYRazTtvDA3N7apgf7T3/2t7ZqGRdvt1qN33nrv/Q9WtzbCZawxyvM8WiaLefjku++2NtY3N9aicMFQw6Ck2Wy+/+57Nzc3NVjt+37J5WI+3dm91Wm1oKWXy6WWyrPcTtAxCBVKIqSAxmXFhdCGaWFMtQQNZrY7vbPLBEHbtQKMLETxSn8dCgNqgjFdzJfNZjNo+qurq5Rt7OztnJ2dXaT5fB5mZSGVhkgiiCFSCEoluEGgxkRLznlBoXJdu9lsvnNvf2try7Ss+XzOlWx5FiZgvphSqBu+yRjjVXZ9eTqZLBAxfN9QeQxl8slHH+5urRT5/PDWgyJdyZL0669+v4gntmkQC12NLppn3gefvDuYXF0PrhpBazyaxlEKIczzUmvtur5lWeFyES1jSimlRg1UFkXRaDRWVlYMygLXOz0+IQQrpYbDIeflyclJkiSPHj2wLMt07CzLrq+vu1ue7wdJXEWLbBmmq92N3Z1bzWYzzaOzy1NmGr12FxB2dHx2eXXuWNbdt99KkjRaJmXJKw59P+j1+kqCJEmKosyyrO7DxQRKKTkvmWERjDGihoHekM0I1fwugpAxmmXZfLmAEDabzW63M8smFGHNUO2obrVanU5nf39vpdP9+3/42zIvhJJVmnzxxRfPnz93XRfoFABADTNLk4urgefYttv4N3/971qt1l/91V/df/Rwd/dWnBfHJ2cb/Y1m0EonWTNo3Llz7+DgFsUEIi0rPp2OTcNot9u1861mUgkh0+n0+1cvzk7OOedFVnz5+Zenp+dFXsVxnCSp6zcWy1BpWPDKbzcXi0Wr3b27t1FV1dXldZ6XG+tbruvHccorkWVZo92ijCAEPM8JGo1Gq52XVTwf1Kcuiund23dM06SYFFlOEOo2Wof7B/2VlS8+//z3v/0scL1333339v1bkKpJOAnnY8tkQlSj2fTZ0fNGo7FYRj5CN+NJUVWD0aRS2rLdCgJRVjeXVyYzd3d34ygtirQVtAzDWFtbX+l0qjKPF9MiK2zbaTfaK8w1GRneXBVF0em0bh8WCsDpaJiny1YzWMQRhNJmVBS5rMrpTaQ04lUe64oQRCHCGPtN37KsPM+qsvju26eeF5xfXAvBbcdcWVkZjaKgUWoERIU77fW7D+7eDK/yMtvevIcATpIqDmGcppfXUwTxev+gyoz5fJ5EcdBsMMMoyhIg3fDcKFqWec45l0IURREvo/FgyDn3LQ9j3Gy1PL/RarUQQtQwalVyEmecc6MOOyoqhKDt2JzzLEs453VkAsKoLKosy6qqSNM8SRKttWEYhLA61O/o+FgoxaV0XRdCCCCsOK8TmfQfgIE6M0FrjZQiiteCA4AhxhhhDAGAEBZlKYWokcmafVAQQPWGlaj1lRBCrd+4Kngp3jAmEP9g5tRaS4Ity4QQLhYLzstOp9XqtPM839nZefbixfn5aRiGiOCaQ6GU9vrd4XBoUIYQyPP88upCa312/Hptvf/06dO33np4cnL85PHjNE2DwCMYlpliRLW6vffe+1BrfX15vVgspIZf/P4bLuVoFoZJnFXV9vbOpobTxbxaTiHElFoP7t7mlVAKZFnGS2HbdpIkv/31r9qdJiHYD6w7tz/Y3V579vjry9PXUPKVToMgpVX10UcffPTRO3mZ/Po3v/jFL385ms4wMhSgeSbLqvJ7fpZlVaGiRZamCcTAYMVsNgFQMkYABkoLADSlihANoAoVp5Qahm+YFEIshOBCaiUsy4YQA4BrJL8oCgj1ZDKSSmsIfN9FiNRyCoyw1opLHYaLsiwRwkHT7/d7hJC8SMfXC4SBFAWvcgQkBBIC6bmGFzhBy9PQ4YoHTd92DJICjSU0Sa1NwZAAgN6wXQBSSoGqv0Q/ZHpIrbVWSAGtEYSUYAABRJWWshSEEQoNI18uzs/Pq6q6vDovy7TTbTi+W1UVQajp+UXJmWE5zAYAmSwDyjAZjMLJdOw5po1szQguixIjSggEAGihAYQYGcQwTdOMlvlyuaxjfCxmJHkax3HJq6LKkzzL8rzgRSW4kEJqlWYxF5XWEHJOFKAGg1DFUTwZXa32ukevr0zTOTjYHY2ulUbra9tZvizKuNP3bW//y28/Z5adlUmSL7d2Nnf2dqaLeRguNnd2EUIGJuF0ZhDk2dbl+VkWJ289euA5VpIklJLDO7cXi3Bwfd3vrbheEMZRGi0nw8H69jrPKiiAKnWVCck1kKisZJollRSu63rNllJqPpkXRWFZlh+sNnxoUJMyDxK8sb5jsSAJM8dyuRSr62u+79q2adpGk7Yvr68VElznQlUIEWwgoCGEQCLdafhpEiVRJHnVduz+yvb25nqn09n1jNW2SyjFFU0LZehKC0VUNR/dLKfDNBcQMQxK0wCUAkGxY1iu6zzY3z5//ex6ePXw3q2t7dXJZDKeD7Iycfye5Ztn16fLLLIalqLq8uLaveunaRrHqed5dWBLp6MWi0UURTXgWZuvbNt0XRvYfqvVEhUPfK+/urKyssIImk7Hi8Xs+fOnjUbj7v07v/vd76aLqdaamcbl/+vc8wJGHYs5GDECjFv7u/1u//mryUq3Swxydn52M54madlqegYle3s7R0fHi8UiSSLOJcHU9/2iqOI4DsPFbDYra4jYIDVCYCtkMoOxN126hBCoNeelYdA0TqSUACjTYoQQiHS4nEtdtzdLSrFlWY1ms96/p4t5EATDPH959ApCqP5gxZ4nk1arsdrteZY1i2O/EfRX188uxqen4+HN/35tY/Od99772R//8c9+9JMP3vvINE2beXWAK0IoDMPFYhEm0WSRlOXsu+9ffP/99+PxmGLoOI5pmuPxuJDF7du3hZCz2ez758+ajbbU8PnXrzyfCVLEXC/CSCGcgvT8Or9tq0ZvbTAYSMLshruytbO1tR2G4enp6auL00SUhGDLsrzA73RXTMsbjac2kLwonj99enN5/ejRIyBUnEWJu3SYubG+7vv++euTb774cnNj4+OPP7Ztm+vs9HIYR2FRFK5jdzodqcVnn3/eaLcIYZ00j5Iyzou8KtOiVEAThC/PzqeTSa+7YmLjbHruew3LsLe2NwxCp+PpbDqRvFzpdvrd1W63i6Ioy+LxeAAA8Byz3WzMwoUSZbyY9xr+bDqUJbQwrKRY77Ymk0lnpevaznIREgrKNJtOR6vddlVVZVkFrndxNRhNFjfDeVUVnuclqfQIJGywWM4vLk/avdbG9j5jQZzJbncnTnPbMQ723x4MJq+PIl4p39qV1amsSqUkEBxrBqUoiopXldIaAEAJoZSazMgJWc4XFxcXntf0PK9OQSC54bm+12woBG3b1Vq32+3V1fUkSb799tujo6PpcrG1ui6EEKJSSlCKIcRC8ixPIYSMEcexhFAIIUpxLUUEkk+n07qOoY4KME2zNlTXu3uNH9SghZRSqArXSALGiGAEka7Lr4sKaK2UrHWMEACtNVdSAwQAgpDWLoxa71izEqAWSQEBIfxhXKh4oVE9XgBCECHINBkh6MWLZ0++f3J1dUUIoeyNP1NrmS5DhECjFVCCLy8vr68vu612b6UzHg+/+erLg/3dq8vzo5cvbt269Zd/8efdbveXX35lMLPX6zHTaniN9bWtV69eD8ZfMNsp4mQ0W8zmc6FBUco0z7jUjhaNRiNdFq+ev261Ojs7O28/2vzum2+XywXBej4daZVdXV0gDA9vbaTp7MWTr6bDc4s5tmlVSVymcbcV7GxuHJ+/rpkagqnjBhIYWhdCldPhWAgQBDYQrEi01/A8a2XKs9kiZEwAjJQCAGhCAMZaAUCAoBQyhiAUGkiEEGOEMbMooFICIuC4zHJsgGBR5eeXJ7NlvExiRDCEQHCFEFJQccEJYUopoTgCuuRlJSpEgBc44Ti2KF4sw2U4hRiVJS8Ft53uo4eHQaelMMiKlBoEEmjnpl/6AoH6MwVKaw21gkoprRQACAKMEHmDw6J6TlC8tt9hWH+HtNKci5yXBJsGQLCsqlKKZrfz/PXT0XhAbExtw3Gc1UYANIyipN1dcV2fc5nEp0M8MQ2HAFllWR4vDWRadgNBiCFCkEgpNdAAagCgUtpzXF4VcRwXRSGlDCFcRuF0PivLspIVF0JBADGov/RIAaEEY5RBokuulIQQKqXzvGw3O2VZttssisIH99/97PPPXe/oww8+dW1jYWCMQc7T/cNdZjuGTU3PNC17ES3G4zFlRpEms9GEIWhRcnBrO0mWl+ecUjqbzZ4+vbm+vm40Gj/5yU+m8/lwPP700x/7jeDm5qaqKlFlg8trBTSmSJQinC64FlrAqtJFxTHDCDOpYFnxXFSVEkBU2bJ0bJcgmmbCt+xev0GwJYqr1fV+o+kHQcBMWpZ5kmewgEIIxjClSCkEEYIcag0oJSaBj+7fPj89GYvC9zpvP7h/9/ZhM/CUkF6x4MkiV9rEmNqW4AWl1nqnNby+iWbj47Mr2wlsN1hfDepj95a7enBwACrxd//xP51fXf7P/2f/i82DfThfjKeL4Xg0GC+WizCrdHd1a74st7fvXLy+dl2/3W7bto0QGo0mQiilVH3WsSxLaRknWZoh0zJ6vV7g2ErJ2WxKMcIYGgbFEGZZNpmMXr9+vb29rbV8+vTJeDZdW1tzPFdW+uz4nGGr014t8+r06DwInHaveefu3rvvvw0ZGA6Hi8kIM+ej996ZzRdS8SQNEdKdbjNaplIJzksh+GQ6iqM0y7I6Zg5jXBdACSEkJvUvAQAayDr6jlIqZFWWZX0KMwyLUpwkUbvf5pzXNVTNZnNtbc1xnBfPn0ZR1GwGhmVlWYYJKXg5GA7SNKceKaRK0vzu/i1F2GAyX2k2V9dXeFakaXF1cRV4AcNGyWWSpb7vu41OWXAhRFGVSZLFcRxFURzHYRhurq8Ry13fsossF7wizOx0ernIzs8vvn/y7Gc/+5lpeYPB4Isvv9o/XJeQ5CVv9/qZQH6rM5jMd/Z3P/z0J8cXr6fTqQRkfW1jd39/f39/MpmEcYQISvOUUup4NrMty/GoaXClLdPIk7RK83FeHr16xYVwHCfPMohQYLtQgySOe93uw/uP7ty7e3p6ejk4iaLINE1EQckzZtJmr4OYkWZFq+NqTM6urkvBW51us9OOooiXRTidANlu7u1PTWO917/34H6a5NPhFAIleCml8F3bcbyqEoPBCGVzXkmEQV0+NF9MZVX2ux1R5v1e98XL55xXBoI8r/bW1qnWze0dCGGRJ1iDJJwPh8Pbt/YhRpSZzXY3y/lotBiNFgAjSK2kWGRMKCwXi8nRyWtyCda2V9zA1QDUkr3Vte3Dw/vNYDm8iafzJSPB1cVipdNMsyyMoywKqWUhzOI0tW2bvPEAYMdxGCZIA0pplCQKgDhOFouFO550Oh1KDa2B32wIrqhhaQhN293c3rUcT0qZh/PaTqkUqHcR33cJIY7jeJ5HMF0sFuPxuCiquo5BK15V1Txc1BR7r9frdDqWZdXyNAhAyd+0O9ZkOZSi3vIVBBDCuoFaCq6UIhDB2voPQO0ZkEpAgN+MBfW93oSTqXoyAAAADbXWEGqEEIQIAJ2ksWmatmMpZUglwnChlPrmm69H0wlC0DQNqZWQHFQ6zYTDzG63yzkvq8y2zTheEgru3L0/HNwUWbpczI9fviiyxLVZ03Pbgf/jj39SywVevzxBhLTb3aub4TKKb4ajKE067d7e3j7E9OLqcjKa9Xq99d6a5wa+511eXvY6K4Pr4d7m9tnJ6+Pjo1t7u4vFZDq9KrLoRz/6ZGuzZxjMpKDX8im2LNOk1FjttoAov/j8d5998duT85NlGM5nxWSaAGhwgXmlpAKUAt9qmtQu0pHBIM/weJhUFRCMIYSEAlprTGmNuBDCAQBKFUJwpQGlwLKYZVcIAQClYVAFcFYWnHOMoYIgyUH5B521EtI0TUCwqkSW57ZtO77HOZ8v50ILXpRJFjfbvuOZi6VAgDuOp3SZZJGSNoTcYAoQXApZ8EpKrTGwAhdj9gZAElpwVddeKCErXiBdk04QYww0AQBAKCoFlNJQA4ShQlBrLYDiSpKnz54qLRaLeSrKptXiUKe8LJS4HNwAghutVhanCKGVbnd7c0cKvVhAXQmEKMa43oDLTG5tWkBjCIgGXGmFEKKEKqA456KqajVcxXlZVVKrnFcC6EJwoaXGECFECIIQAiE0UIZhQQ0ZpBAYhAJKUTKf3VyPPcuCWu/s7M1ny6qqlBI3N1fT6Xg4mi6ipZC/++Wvf+sEHmJGs9vzWp21ra3pfCK1sIn9+We/O371emdn5/atvej4PIr4+tZ6EARplR1fnEXLpFDq6PzcsWzIyM7B7uvXr29GN7KqOp1OyZXWby4bjLHtuIgxsZgigqlpaA3n87nS0vd9z3Ns265ibjJDlCKJYmpQx7aK1AQYGCb1Gx4zKOccU4OLsiirjc2dt6g+tU+Hw2Ga5aoChsFWVlZ817t3+9DCsN8IdjY3Pnz/3VZ/RcbReDhqd/vRPEyylGIDUxNiQphpO/57j8hisghnyySLC80Vty3L6rWs7e7ugzuPvvr664uza8yMwWCyeXDH9Trvf/TTX/3qVxcXF2Wee25/rb//7//tL9959Fa/v7ZYLC3L6fX6cRxLKetVqdFo1BpACHVVVXEcX16ez+fTtxptpRhluCzz8/PzFy+febZz++DQdkylxSKcZVkGEbBty3WdJE3yLJqPJ7bteY5vGXYURVFYFvnSc6kfGKZt3L97IIR48v0L37EhhL97+vjy8jQImuur64yFs+nipqoE1/WIUMuXpJRvJgCoEKb1Gav+PVEIKi1ElaYKQu04Vp7nSRIHDXel33FcMylywzA453me122whJDBaJgkSZonjuNsbm52VnonJ8eTxbzl2Nt3dmzTGt9cl1IQxsajEZTizp07WioCUafdXltby4vq+++/Pz8/pwZrdDcZY1LDoiiTNA+j5TJOlVKy4t+/PBZl+fGH7/74008arnNzc/X1V1/OoznG+JtvvguC5r1799Y3N7578v3N6Obw3kNXoawUiNKb4ShOiw8/+jFh9s14ksWJ67pO4PutZtBqIAr3492T05evX7/OZ5Xv2xgBAOtKWVQUGYTw0aNHluvGcTxfLCzL4pzHSeJ5Qa+/0uy0m83m+dnl2dkZhNBxTcqQbTtxuJyOZ0EwbzU7n3zyyfPnLzzPM00zHw2JwSyLGiaOrufL+SxwrDJZjAeX2+ur6/0+s6xoMZ9Ph4wxhCDSQAkpecm5gSHIk7iqROD5e4cHvU63qKrBzQhgVBRFq91t+oEtuGtYNzc3CACRl3me13SSxQzTNMuyLHkV+M2yKIqKx+n09OpqNJ+XZSk1CBqNRTX2O8be7e3NWyvDyVXOo3y+8H0/yYpomXuBO5leS0W2t9fXVtfTotzb2MrL4vTs4vvnz8bTCaHYtPx2s5lkWVFVvKyiRVhmuW277WaHUhqnN3U9IQAgy4rhcKy1BhoBjWpC4E3MF4QAQABAXdhYM8FKCYRAo9HodEin03FdX2vNrmme50otEQJCVIK/qYKsH6eOcJZSBkFQx6jkeV63QNUuElmWSimIECGEY4gB1PAP0ABGCAApZT1DAFVXTkOltZJKKVW3FdQDAUIAQgAhrpV6AAKEIUYYYcUQglBWVYYQgpDFyWIwGExnQ8e1GWOcc6B1u91eWVmxLMuzfd+1//qv/3oymdy5fZAl+urqstnwbu3uxMsFL9M0ibpNH0rx+Juv282Wu7J9dHQ0nU6H42nJK6DRMonbne7BwYHWemdvf3V1NUkypHQaxg3H80w3i2LPsve2d7qt9u9/91sDo6oow+n0WRoPhmNKwI9/8u5bj+5PxgPGCKVYKzGZD/rdtW6rbZh4uZhfXp34trW3taklz5KjwQggXFKGCcBVCUQJlosFkMBkdq+54rtNIIgSUCOqEFEaKKm1pgAAJUEOi9pLAADUWvJCRkmldOX7wLRwgxpFBfKqME3W6fW63W4SVxpIId54ZQ2LaEgqXURxwpVmjGkhhdRCSqVUpaTnetQ27cBbZ6jb7Y6nIy5yDPh0fG3Y2PY9BIAGSkpFGLUsK8tKCDCEb9xhP1BLEGAIFQAQAgyAAgABAABAkEAkgFKKKwmUBkoDABDB5PtXL6QWXFQIi+g0vVlMR9FCXuqm77eazUrL6XIRxXFd+E0QbTiBRUyCLc5Vkac5SpCmSkjLsiHCleBVngmtGCMAgLIsZVUZhLqum+V5kqWlEgJqYhpMK1InFGsN9A8uX5lGGdTApo4BKZA6KarRcH55obC+2FjvNJstx/G++eY7121MZzdHr19S0traPGg0m8xw19Z3CyE3Nrd39g8gIkdHx3meO5b9/ePv8qS4d/v25tr6yWorrTLXtyvFoyJb29u+4zXyPK+A8myDUFQBNZqN5+G0LEvTN1wvWMZpUZQqA1bDbgcesth0OYd1H6hWQCuDkoZruY6JMXCavtZ6WWWlyJQWBkPEhO1eQAyEGVRQlZx7pim1kpL3emuSqGwWLidzripiWCsrq3fv3l1fXzeZsfSaRIFOp2dQE+SV5NJx3KoQ1Gwa0szLyqZ20OggZgIpVtprP/rwR932yuX1xeXl5XQ6JSZa63a3NnqEql/9+m+zLN3b3Ds+fX33rfutle7+4f6XX3+VJJkW8t6dW02v+f/51//vXtBBEJ+fXYSLJYK4ZhmCIKiBTSGqOsXFNM3FYnZ0dBRF4enpabPZbDWb4/Hw66+/fPLkyeba+ocfvBc0vNPT0zzP5ouZYRhc8f5qrxKclzPPRUoWg6vjXnfVdVzLMuaLycX5q8nswrSNjz75tOG5a/2VwHeTOJ4vJhoo02SWbZAlnC+m0TLTClWV0BpqDX6ATBFCCGKtgZSqqqo6TYFpgoGWUpZVDpQOAr/R9Mh6/969e57nPH78+GY4bDQajNAsS6bT6cXFRa/X84Kg1ekAqMIwPLk41xh1et07CFZVtdLt7O/tnDlWulwGDaflbrVcb2dj7d7tw7IsCSQQ49FoPJyNzq/Pj47P9Mmo1e1atgsglhBnHAlomI4pDGEzW1Z5Wshvv3sKgVppNx48fPtqNJjNZjeD8e+/+rrXX3Fd1zBor9fN0+Xe/p3RZHHvcP/J85eHt253m/71+RkEWEGQlVmWp1wUFc8ME+8fbBX5x2myuLwY2DZxbUqxxlogxSXQxGDrK91Ou3t0dDScTgrJdZaleba9v7+2uVFW1Ww+n4Rz5li2bXetNuc8jdPxcHJ5fiUq/f477/2jP/r5fDoXQvA8A4Izx4jjWbyclkW82m2+9dZbT5+/CGejd95+kGbF3//iV9P5wrKsIouLIrNNq9cLut2m57plWWaZmaQLVJaA0M1bt25NF7/81e9Gp0OpICTMMEzXbzJmziZzZhhJlE5OzgPfL9KCAkKJURTV5cV1JZVUYDAc+r6fFwVl7Pj8LCny3d3dfgtVQvTX+7cOdq+uT7lIZ+HMcEiSV5VMknQxGl4ZRuAHrlbYSIqHe4eDwSBw7GbDvby+ubi5jrIcamKbJgBA8nKxWIRhaDLLMAzGWL/b11rzglvMUkqVWSmEMAzj5vKmLnpGCNcrYRiGy+VyZ7NX5xdVvALVG+8DAEYcx8vlMkmS8XgaxzFEQGmZpHFdUlAnkgMEpVYKaIAgYZSZBuecS1HyCiBIEBRKFlXJ1RuRo4aKEQq1hhjVGJvWWgEttSIQawSl1rIslFJ1j8yb9CdEa98ghARCDaFWqpZDAIQh54XtubWrEGMcNLw0Ly4uzpQS3W6bUhpFkWGZ/X6/3+/3er0sSVutltYyyxIAdF4keZpFy0UauM2Gb1vGvduHrmObzCjSuDBZdHpyfXxydnF+cn52a//wvY8+lBp4QVNrPV+E6TL89csXFxdXs8k0DMNiGfkYSim/+PzEMIzPfs1ns0nT9+7fPjx6/nQ8GmsBNrd7D+8/mE2nf/Mf/r3v+9RGOReDkfa9ijE2nU7zLDVNQii0GO41g72NXmCnGhhSkTznusXm86RMM9wIHty5c//BQ8NyTo7Pjo6PtdY1g6OA1lJqgKSUEkkAAJAaQgiRBgAABCAAEBvMtFrtdT9wAQD9fu/+/btra2t/87e/SAtd8lwIJCWHacUKlaYlMayiFHGSA6Bc20GEQogUwNMkKSGUCDS6vc5KVwG5WMwh1DajFiGuaZgmqwDIeMmFRBoKrhACCBGtlFJAa1CHgDNmACml4lpKKbXgUkillECIvpEuKFVJgQAkCDFCScQL02KYmXkVPXn2JCxSxfBoMeut9bFlZKLKeYUJiaLo6OgII0p5GC+ylZVAcWVi0vSbhFkEQcUrSHSZZ/P5OMsTahqGwQBQlgY1Vcw5X0TLnFcQQ4CggG9mBC2lVBwojTGmmCDUtE3Ht9x8mV8cnSwn4yJaAg0ItqTAEBhrq90n371CJEuS7Le/efr2o4/2D/rNRvvhg/e9dpOYxubejuu3jo+Pr4eDPElFxRGAtw/3TAKTeLG9uzUNZ5fj62WcbmxuP3zwwA38m5vh+fn5cnDVbbWPL8+CXgsykCWxBirXRakzIWUZlRxKwzUNxzYMGk4WWR4Rgh2bMYTKLKrShdZ6ukwQQpJLWVYmsxjRvu3v3dpAiAjBLYtSgyigIcRCgTQveZyrTNiABSvrfhCsrq5vbm01Go35fF4UFedSK5AXVVmWSgnHtMJZSoiJTMdiCiHKhYE1UopE0dJ1G3cP766vrqx2uy9fPi+r3GUQm/kyvZklV3YLOE04Sy4H0xOO4u+e/u7p899P54uWxx7e23UMZWKeLG4sz1UKnJ6eC6F2d7eDIKgdhnEcQ6gppVmW5DnN85xS2l/tzcPZzeDKtZ3XL1+9evESQl2WeVmWjmNDCKIoStP4zp3DRbR8+PCh5dhxeBxHk2iZlrl07PW9vbXd3e0sS16fHUXJgtDg26+/chutRw8ecIEuz6+CwPN937IcznOItOPYVSm0xnleCqEEV7UMsDaXQwiFEAhArTUAdccrhBBoILXWCALKsOvaq72VDz96V2v94uWzGvGmlPqBmxdFVVWO7+3s7DSbTcLwycnJ0+fPipcvtre3KGNhEo+H11vr/ZVu6yaPfdfe3di0COZ5dn551u/3u6sdIcQyXa5tb9xOI8UAwyuB3yy4WMYZl0BoJADCzIFU5EVVCcg1miyW1+enJoGu42wfHLz/wSdlmX7x+e+++uqbP//zf/Knf/onUvIvvvgSQ8iFbrZXP333rf7q5nA8yxdTy6BVgbmoMATNhutYRGvZba2E8+7+7nrLM2/f3uu1ApMAkyCTIK0Rl+JmOCyEEFCXki+WYa9vm57DDANScnV5/vj5U99rbO3uTKfTIs0FVxdnl1enl3mST9F4tjF/9OCtTz74+NvH3xS8DHzH860wXrYa1q3dD9998GBja/ud9x48e/7SdVgch4KnpgEX86lpkv5Ke29v7/7dO81mMBoMB4MrAaAmLCn5s9fHQbubVVXKxTwC1nwhIZFSutiYzqObwWRtbc33W7NlDHVe5RVQGQBYCHl2dl6HuUEIe6v9MA6ZxVqdpu/7jmc6TmM4yU4vJqubO5bTIoLBZZSkBcIGhNQ0bUINxkwEcJpUVSV4nmKoO61msxns7u4+f/3qm28fH59ftNrduj1EcC4qWUcOMMY6wFgul1Aoi5gIoVznCtNGo4UxLstSlYIaGGMMsdHymg0nKMtQKVW3KpimCaCqTcjT6RQAlOd5kiQQYsdxEEK1/uYPVIWqwe0aWkiSpA4LSZKkzhys7ZRvpLtSgLIAUEkqEYBQK6hB3XvCpQRKAUIQQAihvMjenNYAQAjXPm2lNVSwVidIqev3FmOoNSIYVVVpWVa/v1KLILmomEF3ezt7B7fKsjRM5npenuePH3/XarU67bbtWL21Fduxms2gzFt0ped5TpwsW83AoOTB/XvxMiyyDEOtROUZ/v3DfYviweXl+kr3f/KXf8GVnIfR85evXgyuRsPJZDyuSn6ws7P+8ce+7xOGHMf5+uuvnz59Nri+cRxnOLje29n0XFvJYHN99YMP393Z3Hny/TdQI6hRf3OLmZ7jvj7Yu33v3sP5eFQWuWUyigFEqHt7/92HD3kF5mEeRyWv5LxaTqbzKIpW+mubm1v9ldY8XFY80UBpUGmgNNBKawiw0lAqSSxQe1MpxfW6ZJqmYbL6fLW+vu44lhCi3e65bl8pm0tUCagUQYgIgbJcVEiXlWq5LVlxpXMhRF4IpXOMsdZgtFxOk8RAWABNFjgvCs/zGCOM0CrL55xrQphtIw1FwXmhmMUQIgghJbTW/M3Mh7WUUr8hm9Qb0AgAAJASEgOIIJJaKqmABpggygg5v77s9TrMMuaL+XdPv+90A8tzF+EkKjO+EEmSLMMFhfT49HQxXQAFt5pWUVRrq9smg5Q5a/1VqSBBeDIbE2ZmRTqbjuM0dj272WlallHGJUC6bkyP47gQnFoMUVJUJYS1wkJDrRmltmmZptnfXet1er4VPP/26XdffHv0YhrYoNdiSsGz05tOp3N4cHd///Dx968ODw9fvjgeDuaj69lsnhSFjq4mu7dv2Y4XhvH55WUjaHVb7Yuzc89z1/r9cD6dSUlMhimJ01wB3Vld8dvNSihAMDZYnud+uxkX2dbaqtYSAHVzc+1ShQ1EiZlk2WA6dJvetrvX7jQHo5twNlOad9pB4NlQK0yga5n7BzsYYy1VnhZFkgqR5xWyDafVaJW8cDzXYx7nmjBqGDxNimyR6Ep1g/bm1k53rW85HkAwnC/LvOKloMTodlc6K/04XEwmoyTJuo1dXpaWZRlBCxTFeDCSUrYaQVmoJI6TdImw7nd7ebIcjq5FWYbJddNo7R70OFDNrum1cJwPr58euz5sd03Xarb85qNH+3kUBR6+vnwVdDfLsnz58uXp6WmevxcEwXK5JBTfurVrWRbG8OrqarlcSsUNw3i7/0jO0+fPn0/Hk+++fmKZ6O7tO7Li5+dn77//fpqm19eXZVk+ePDg1fHrMAzPLy/u3N2dTIdlnhqUddq+69D1tZ7r7uZlGL4YNzz3ZjKKk3xv987J65Obm5s0ix3bS5JoGI+EAJZloTaFkPJKpWnOq0JKhZD6gw5LQowxRAAAhP6zZRwAACEwDEopjeMYalkUxerq6tbWZpzwKIoAAO1O07LtetnlnI+nE8OgXuDfv38/SpM4SxljrVbr5urF4PpitdtjFAtRMgrLIp1NxttbG6bJwmgxXczTsti6tWn6NrVZNGWYknQ4jZJUaCw1smy/0elWRaEkwJ6/tr4li+Ty9CTPK9uy/u4ffvX577/62U8+eeed97Z3d7Isu3/vzu7OZuC711cDgGhVxp9+9PHNcHJ58hphEkaLNE2l4ACqVsO3TBZFcwjovTsHSFVQwW63m0RpEiVFHhMMhaqqqjq7OF/Ns42NDdf3i6p0fU9DOF7MeuurnW5XSBmmMaIEYBSHsRR6Pp4VWdnwAiDBfDQReXnv9p3nTx/HUcaQJhjk6XJnZ+tnf/LTt+7cn8ymd+78rN/v/f/+x383mkw3N/pcqBN9trW5s3drZ7W34jhWVeZKC9czzwaLIAgUFy9eH2uAlNBSw2YL2k4QJ5nWupTg8vJ6cDPqray2O71hqSDAWuM8Kz3boRQKoQK/kaSx69orK92vvvpKKH737u3Du4etVuv61cX06ujo5c321txxCWZ2kog4nvV6PV4BrQiCBsEWwJhgYlskjiIEgGUwBWCz02512mUlLq8HtUbnh1zFGi0ghJVJxEuBIWk3XdM0l8tlnpeOZXc6nclkWpalbduCq/lykaYpYywpwtrq4jhuo9HwfR8AkCa51trz7EajURt5CCF1DTEzzR/CFeqABKFUmuecc6GUEGIZx1VVmbZtQggxZqZRw8U1H1fPzQhoQghRpL5I3gBxCEJcXynoh8qJOk8JACD4m5KIHwaFWhrvBd5kPvM879GjRxDC169fK6U2NzcBAIyxurkqCIIoiq6ursqy3N7aqqqq2Qy21tc6rXa7GWAEBtc3CNAyz8q86HXaZZpog/Z7K4ZhMGTsdTb3tv//PP3Xk21ZfueHLbu9Of6cPOnN9ffWLdvVBg20ARoYEJghODOc4WiCQUohvjAUof9FoZBCCkkR0gNFDkkRwwEwABpt0FXdVV32Vl1/86bP4932ey+rh1NARkY+nJcTmZFnr7V+6/v9fHZYVdSajXg1r7i0bLvm2cv5LFku6oFvUqvXbjdD37Edr+EYhvF7v/s7rWY9S4tff/jhxfnZ9tbmvdt3lqt5GPqeGwyvRy+evWy321VVeEGwvbPvBa1es3f73t1xrXZ5+jqOZ46JhCib9XpQa2S6ypbTaJV7Xo3zKvBt2yK9brPRDAGUWRYBLSACEHEANYAaIgCA1BpoIBFWjWZje3u70WhQSteYDEIIRkQpFQQ1AMAiW0zGy7J4VZal4FArQgxiGAYuCs4rACE1YFly06S+bbOiSJIkL1LLNhzH4QAmaYaA5ppnaYq18B2n2ahBAJIozipGTaPV37Q8HyOqIaroWhGCJRBSKoSQVnC9IgOppND/SMHHiGCsq6wEGGOENIBKayA1JABDRK4vL8oyF4Ivk/nW1o7t2VwUjqvns0LJ1KHmcr4M3UCWaaPWhEq//eAn3f5GkmVRnG5sbVpemORZXPCsFMnkqsxTgpALUYjpzfbGZDjS2NEMUGC0Wy2uqslyzFWBEbYsqZRCiNjUYSVbLpPAq7/7znd27r5rEMshQd1+8+tPlqPTn25vedH8NTEZ9YHTml+ufrPzIHgyAEbT/q//j//t5PXyq6++fPju20ZZWpbR3/STZLS4XEqZYkIVNm7dv5ulxbKUbmPj7OzCznhv4zCo4ovh+ZdPf/vq6hkHoBY253msOMxK9erVZejWFov45OXL9955M0lSpVSelyGxujVfl2w5GW9t7x5u77+W4OTVSTKvtre3TWoURWFgECp2sLtn2SZFZWirIs6LtADAITJoNNtaaEAIgaAqc8fwykIY0Lx38y6iBFOCNKyylGulAIQQ+n7oOI7nr0MAdT8geZ5LyzB9F0EsFEcGafS7aZpO4xgYBjRtxCtWFFnOqeF7XrsocqENwwn+7b/9txBJxjMAGZWjgFSwUf43/9VPilwuF+nB7Z5WhOH/dD5bqZTUw9p0Mrq+vv7Z331g29C27R/+8IeUWJ/89vOyFFLor79+zLn8we/96OosN03w3Qfvfv75p+rm/vnFabwc+DU3yi6xdbvRhfu6Npw+uXm3pcH0o49/NZvNbu4e1b3OnLKw1pkV/OsPPh7F2b/5l//SscM7R3cvLi/7m9uKGuMkvkqjv/7kN0Bjx2ecSakBNewsraQGGxsbD7o7VxcX15fXrKwwBEoIoDRFuAJC/EPRaw3JdUzHND2CZBqljgXq9R7F4OT4ejRYnp0OHZ8WDHiBsUomtm13ev54dHJ2+kRrqBGs15q1WsPGdDRZmqbZ7291GvvTcU5RPJsvR9PJ3Ts3LgbDXnNDClvzgFfCxxuGUQledWyw2+i/SsaLxRyCwsTFfDimlhk0mqvBHACw3e9VVfXGG/uT0eDFMY5WEawgDTvbB3vHl1OWnzx5+vJ//1//l1fn47cePvg//Df/7eOnT37+y19Nl8uT0+cnF5fTeHDz9h0f1oej69Fo9PjZ8Yvjq0azL7STZYAge2vrdppEWVGNJpM0jsN6aHvUNDvD4dCx63FUnqtxLexgjPOMLRaLqqosw26320f7R+fn50++eNRoNDr1tgT6+uoqDMPJZMIYS1h8Njzd2dkRWDe7nfF4fHE+vnXrjWaz6dHOcrBMVsli+DxNCgNYJnQbW5uvjk+2Nw567Q2sjdU8ZTnjVTGZTNI4ApYqRMq5ZEA8P31tErO+0bu6uC4hyDhzXdcK7EhmJRWDeLjXOihWies6BsSCK8E5NikAEFPMON/d3R6Px65nv/ng/ffee4dzPhgMxrMXg8lrQJL/77876/f7u7s75+fDqqocq4dgOJvllxfjmzcaEIGijGzbldJVQvhhIGWVZokF9bfu3TBk/vL18fnF1UbN4rIwMBuOzxrN9o9//PvlZPjs2bNrFDGel1VaawQ9u51n5WwxxZQQTaqq1FJRpCnSFgWMO61ejxAipXStwLNDxtiyiCzL4GXFdCmlNDAiGGktoZZUSmwarkHXhz+pFNBKS24QNF9MOeeQAAPTuEgAhbVajU0r33en06njWtQ28zzzPUcIaWLMWe5admjXgVC2aSIAsxgL3xZCaIAQohWXRckAQKZpA8CklIIxLSUEgmCIudKKp0j7pgO4oshAxOACcoFen13v7m1zIaVStm0uFxOoSgMyKNNty17MFoTxcTR6fXbKhQrDUHKACb28mh3t3zYNz/bCXIjPXjyrNRssZrVGa3d312s1FcHt7kZQCz///HNWFP/sj/9oPh0Pr68kLyjJeT4Q2pxmziJaXF9fbm33D/caFn3481/+LIpe7+xurfJskURXY3A9GFWa+81QLEE6GchkudMNebX89Qd/5XmB45OisuZRWlXSsKwdf/P47KnUtpDFYDgUXiCl1MiYLctVdsmZXMZRvdEWGiJqaAA4lwoC0zQt2ybEaHWDNYGbUgwA4EpKznhVrhFQs3m2frHMdZzPIYQmdkLP54IVRaZ0RQ2odCV5VW910zSer5KyZFprDWCeZOPFxLADDAHUWrJKUGTbpuGaGunhZMg539jalkoYCBkAGJQoCRAy1DfKUA6AxBgoIJUSkCoBuBCl0gIBCCFSSkqhAt/P87zIsvW2WAhRiNLgBtEaTqdTKUUlKpArzjmlmECKITWpCSHc3drHCklHCCYcy2r1Wo5rYUq2drb8sDZbzIVgWsvVcqqUYqyaRUuDgHrNjeIVNtE6RQuRsCzaaobIkNPVLE4jiJGSgCsODGgQux5S23KqiiNiOnbNAO7mtvcHv/9H8+H18YtPeQkaTXj3Vr/X3pvNF41muN/fbgetlhvSbt14x3/n3beTMlFAAqony0mWLoCqDKLKMkrja0rpwa3bW1tbhpMsr1UyjoWSvX7P8u2iysfzlda63++PB7NVtHANazKZraMlUZqMxzPP8xAhCkDGRCUVIKm7jFw/3Nne89ywEdYajcbxq1ePHn0VL1fmk/BJt7uzvX2wvd9uNLGNiCIYojwvG20EtWJVBda3WVpoLYltUkqJQSFCCgIEEEYQIDgejxljtm1rqcokK8vSprTV73O+7ikBgjAgBCFM8kJrzapKKYUxBmvYu2X1ej1CCAOZ5waGiQDkWa64WN/3QNO0ASSCF0qBwWDQ39jd3t5ut7s//fe/sCzr4cM3ms1aFEVpnmxvb7c6dYjUycXr589mjgNsy2g3WycXx59/9enDN+/+4Ae/+8/+sz8djQZ/9R//1+HwivPKce08T2/dvrGzvQEAwBB02635tDEaXD/+8rONzV2L4PHwGs7ma6XparWazGbnpyclZ/3DQ2B6qyglhrW7f7RarRgTSZZyoShlSVYYpgOQ3tnfxhgrAObTaVWUQEgEAEQIqXUCfA2hW+8YpNZyDWVCCErJF3Hy+PHjbq8TBMEynmNI2s12WXmNRuPwxkGSJIyxnZ292Ww2ns4Xi6XgyvMCw7Cqioc1p173W03/jXv7kmfHr184lvUXf/nnt27c3esfSAEc216/i+c5vV53karBeDKfzw3D2tjYWESr+XxuGMbGxoZt26ZJCUW1WhD4LgTKtkzP2ihKlqVpkcR5hn714W80L0ejwe//6Afvvffe0c3bk8Xipz/7eVlk3Wbjq88/aW7ucc4Dz1VaXl1dDXe2lWAZUN1OEyFw48bhcj4dXV2cn59v9DpK8tevv+h0OoSgPM9d166Hvu25hmFMJyOEQBqtoJJKCcexCEEQ6mazTghJ9veEVkWRXV4uJuPh+cWpYZLAcyazmevalGLTpJ1Oa7GYFUCOxuMozaI4Oz05YUrXGvVWs86FKrJ0NLjGCGxtbTZqgeu6SvCIJZBA0zCAAkrooijWBEAIoWkYa7wxQqgqyjiOMYSMFQBITCCvWJZz13XrtZAQ2u7UK14Sit55563vfve7vu99/ejR8etXWVFmWT6bLRCBjHMh5WB4DQBAmNRqDaXA6empYZieW8uLHECNGGCsBJoghLAC1CCtWv1ob98glCIaeP5svuBFblLLN+2a4+69/x7FZLFYLZcT03Kj5Wo0nDhuIKWWMi+yEgBgmXR92S+5qNVqZVmuJw1a6+VyCSEMazUNJMYYKM05X9+mEUKADaSAQCmppF4bBcA3+pVWq8WlztKCCY4xdhzTNmzXchXJi6Iqy9K0jX+8UzAMIiQr0rTIVr7tOKZBpeAVWyxn0IEIE9t2DZMiioTSVVXmBfsH3rOCUEKgIYR6LQkXImcV43w4vCaG9fr1q5OzC8MyHz16BMAD0zQwJnkS53nuOM7uzp6EsrXR/vLVk9enJ81+FxF8cXmWreLbe4c//IM/ONzfk7xKIufi/GQ5nUzGwzDsnlxcnlye247XaLafHr+ixHx5clYUxf379w9v3x0Nrq4uL+bzaZkXXIB4em7b9v7uJsCo3W4pzTf7/dl80mzVbcMMe/2iKLTUR0c3m42uZXqBpyGEj754rDW8efOmknqexBrhrb39oijDejMuWC5UIjjDJOZien2ltVYAIYQwJYQYECPDtTb9Hd/3vaBm2hZCSGlYVVVVVRqL9d+KKw0QVABqiDQEUgEMMUIAQIggWv+HI4TKjCmlqqpgvCIEWLaBoQkASJKsqhjn37RVtYJCKCXB2oJrUINaJoQwLyshRJ7nFePr0Kth2o1GA2KSZ2VVFXGaIfQNahooqaRUQkol1jMkSfC6BaYlVBoABaqiqIqiLEshBOPl2vBCCCLtdns6nVqWXbcbq9WyTArkWBjT4cW439tYLBYJzUPXu3fn/vn5+enJxavXzyzL5pzfvHXHCz3fd9YrynJuvD55xVgJtKqHDdczh+OLRqORValhGNTSLiW23+hs1LaqbpJlaZqblgMhlUynaTmfrfJMzEZL8mpwsOs0Qx9h8r3f+1GyWv330UxUo/29zb3tTSRAtpi3PdvHgYyZiIq/+evfhmHoe/VGt+7XnNcXrybzwWQ+6PXbimVVmRom6vXqN271tnZ3Lat6JJZPjp8uVtOdg82trf5gOi6urlerhUmYYIxSurm5aZpmPXTSeMEqTm0Hm5ZQkgsFTWQYZsXlcDrd3t6u1VtKQcfzfD8k2CoLUVSyGkbpOBFLZglTZ9KxbdM0CUSmaQLLlmnKOaemoaCSUlADwzBcc1HWO811Z+QbCCuh5lq2JxUSiiAAFfzGbqARQBBgAjQA/8BiE0IIqTHGpu1ijLWWhKJlUqxr0OuYEtdECii4ns9WmFhVJZRSx8fHvtdwHMv3/c3tXlVVtaab5IvORr3Vbd++fXs+n/8v//7PJ/NZ0AS2jVnFch5JwhksfvHBh9TEb739sF733/3Ou8vFTpbHpkMfP/kSQqAk11rHydKxaC1wHYsmy3h/s+/7ta+eHUugt7a2PM97/Oxpu9tbRquWbdlOsCr4aDUFpgMgLctSKGkYxDCpWEexgJJAS62pY1mOrQAoOMNQE4QVkBhjiL7xSa4/jeub3bIs/+GnjFeL0WiACXrnnbdevRJQaYKQbVgbne7bb77zxRef1cOGadiuE1hmWeQCY2AaVEq9WMYKsiKfmHTzj37ynWgx/eiDX7Gs+vLRp6ysFJdJXHRa7TAMyyoNAn+6mAitWs1OXlRZXpqOXUdoMp2uVivf98NasNXfbNbqBoKMsaosgBb1dkPwqtlyYoQtCiaLZbSYZHn66tWrza2Nd955x6TkD378o5s3b/7VX//H7Y3er778Kk3LRuhUQP7lX/z5xx/+4sH9u3/4kz+QrEqTqO57WZKa1LBMahKarKJ4FSGELi4u4jj+/ve/3213NARpmu5sbnElgdbRaoUBxAAmqwgD2A6CdfWjqqrpaDwejyGE8Spa10Om4/E77723u7urte71es+fPz8dX5+dXy6iFTVMqFUjCELXLfPi2Yvnt2/ffuvBfUxQWZbT6Xg0GOZ5juoW51xyxUteZmWW5HleEogk4xBBVlUEhZSQsiwXiwXG2LQQxrrZCBFCy+XSsmgQWkqpLKuKKOu0mls7fWKgrx4/+ujTXy9mc6WF7fterY4xtFxPaACxYVlWUG8cHB0Z1Irj9Pj0deCHvu9TC9csR0kAgSQQagII0IbtHG7vbnY3QjcQUquCT/QElCUo82KxDA+2At/lZbGaTZ1AxEme5uX2jtntbuR5LhiXQhgGsahvWwYhRFOMMGCMAaiKMvtHKjNGdJ1O1wBJJaHUAGoIYVHy9YnwHyHuQkml1GqZJHnOmESImNSi2KxKMZsuQ8dO0xSgbxoWVVVlGBoUVkWKoO42m9tbG75tYQTyJKFWVamyKnlezrIcSA24hAAhhIjiCoB/iOMBoCHQGnAJbRtmcV6W5dXlxfbObr+3MRgMMEG1ICToG7iT1lpykabpZDQib73lBV5aZasi3q/fyPJyGS0sRK4Hl1qJi7MTAoCBoEVw0/eZFFmRRqvpfDFudzfG08mT588QJudX17fv3Du4eeuNo6P21kYFwOlwVJRVp9NBixEXOTWcyXjQbdcglEdHe3GclmWJIKn5QbRKpYRI09FgijGeja42NjauLsdbW1sPH743nc4n86cQ4yhnluO79c5wPh/G6TLJPC+gCjQb9X8MP3EhMMaW61m23Wq1AEJCKcYYl0JKIZHiQHiet34SAwAwRMgga+GfkBL9g9pbaLW+DdUQGgbhnCPEKdGYACVBURZZliol/vE6Valv2goAAAIAxdgwDIINIQWvqkxLg3yjDhlP5hBjRExEMGeScx5llWEZhm0RSgGCcA2axgZjTAOAEAEEScmV1gpBiSGWEhNIKMIQI2AQCCCE3xSDGWOe12g1mqvFcnQ1Nk3TolQIWUSsSoUm1UajT4nl20EtbP79B780DGO1Wj159uTu/Qf37t3bunvfKnMIRJZGtVotTVeT6eDe/ZsSVJBorLVTswhBUcySLIMYuqZpm1a32YmTYjZdxlG5mCeXFwMpYTTPublbdzdCp1PEWViv/9Gf/pmQZTy/2Oy6vJyXRdRt3NrZuCFihzOFmCYEBnW3rFKTtlqNxsUlqns1SikXfLWMISZHNx8cHBw0alsANG1rk6NFxtK0SqlNG91mzgvBq9FgKAVu1bue4663/K1mN6zXk2jZ7u8KIZLlQmrVDINWp8M5z/J8Nl9ijGfz+XAwzvoFl6JWq5mmmY0TCKGhUTSZJbNFu93e3NzklBBuLIfXk+nUdd3ORk9KUZaFhsD2/X+wEEgAEQRaSrl+CpsmRVDzsvAssxb6WshsPrObbQgRwBAABXilyooxpqTAGJdrYAg1TNMsSxLFy1USW44huOa8IBRICbUiUgDOgBSIC26ZztZWYxWlnDMJNaX6/e+9F0VRksTD6RWX7MbtA79h//SXn6RVZPmg3qz3+lsQ4iIvq4rHKQFYPXr+6Pjy5d07t959762Hb9+dz8bLxYQVxWg8WMymzVpdVCXAuCpzxza3jm7ev317lTOALMutHd25g4nx+OtHBEEmFAXo8y+/Gs5WyrC3D24tZiuMYRCEthNIDRarpKw4E2o4HggpOZerZJWUaSUqk1JMEJACQggABlB9szfC66KXFoIRjIoiKyvNqiJJ4yhaGoax1e/WQ282myECEdRQScHK+Xye57nScI1CqioupIIYOY5T8QmBgJqqrKKt7Xa7E/zVv/+PlayGkyvfD6eTRcGKDb2xWi3IDL58+ZzJWqPR2j84mq+WURSPJ1POebfbTdN0Nptt9FoYw729nZs3jmbjkVR8sowatfDWzcPVfEKRDFwjy5PxfD4YXdebtb/+m7+5devOt779nU67e+/evcvL67DX/elPfzqb5fWgDGt+VeZffP7Z8PL8f/e//a9a9QbQkiDU7/Ukr4DSVVFaBinTJFkuao36Gw/u9Te6r169Go9GUso0TWeL+bo/kmVZUK+1mvt5ljTqYeC7c1YaJum0GrVazbIMgqBrW91u+8G9O3t7e5eXl1xUSov5bLlcLllVbW/t3Ll127QdiPFyMQtsk0ItylJRAjVAAHPOkyQpeFJVXEtAMNVCl2UpGYcQSSmRRhVnlFLbtgnGVVFahnlwsEkIuXnzZrNZn0wmSilqkOl0ihfaNM2jg8NWt3V8evzhRx8maXTj7s1ms/Hs2bOiKNqd5vvvv7+9vX15eV5m+Xw+50J5vmmU5cuXzxzHefPNN7Mc9Ds+QhQAYGDCkS7LUkvgW85Gq6O5youqSLPZaJxmxep6+OTzT2U6v7oaAM6xVtPhoKy47QZYS9cxgZIxAayqBFvbm6BhkGWWGAbVWkZRxBhb9x4rxlzX/8bJBL6heiPElVKSqW82u0CvXTbrduXV1dXaJOk4zrqLhCAUnGtKKKX1VtMwDM4rYhqQYKlFXuWtRtDf6R4d7hoEKF7ifnhw1J1MR9Pp9PJ6uEoTqTUiFqGmhgQRpDXUGmqhpYZKaq410MgkAGO4tdVHmJZlsbu3tYwW19fXxLGEEJIzixIEIaXUwAbnPCuLRRqPF1OAQSVFnqe1Zu2NG7epBGWZ59HSs8wg8ELX4VUOtMJGsffmTSG1Qng8mcd51dzYYE0XynR0+Xq23z88PHznrfuLyeDJkydJMk/ihed5hwe3vvPtd7Z2tler1fVmn0v10Ue/lVIXRWkbZkmti9OLZRzblvt733vj6OgoWhXEMJK4nMxjgC3HD18dv755527Q2bheRiVAqQLtZtNut/MqNQyDULrOfEgpESWmaSasKMtyFcdrG47juZZlWb6lgOKScyEghAAjA2PwzXkFAYQ0hEIpKdd6CAUhdABFUBuGBSGFCBRFtlqtZrOplJKQdSPmGwr42hBCqSEBUEpzLhirWFEBqAxClRLL5bJgKsuywXDq+36r1arX677lGIZhUFNhyKVEWmOIAUa5yLUGGiANlAboG3ICgKwo+To9u/5loV7nxEmWx5xXBsWu4xgIy1JIAaFn1OxwfDmilO4e7d27fe/i4mI9/Pn4539RVdV0Oh9NJ5eXl1cXF7+X567rFllKIOq1W6+i+ePnT37n975v2vZgPFDYUBh6nkMNK0AIIq215kyen18/ffLiydOXVakgtspChEETbOB+dw9pw6C2MLSqGArDH//Rn0SL4ZeffPjo0RfJalr3LAJ8360bAZkMru+/cxDUa1VV5GyZFtZ0OhZSc6WlwkAGjXpvb/P9rY1Dy3BlDsej2dX0Chio3mlgC5csV0BgDAHU9dBv1EIhxPXVVbJa2hbhXF4NB9p2JNBxHkNEuJYSao0hJHg8G7daLcMy8zzPi9S2zXqzIcaTIl9tbGyENWeVLKbTaZTHgEKAUPwizvM8z/N33nmn3euut6hKqcCtVVUluFAAYIwQQExV67b0uqQ0X0ylYP2NPnQsVCSiLAzTBBABLouiKEu2zi0DAEzTNgyDIMB5pQDUACFs1kIPIZwkGcJ6HdkWCuRFwZhK0rzRMGuhl2U8z3POBUb01o27/a1+VRWWZ/z9r35xenW6/Hr+6aMvev0ucWmn333w8K4XhNPpfDyZlCUTu6gs87LIJqvFYDImBrQIqTcaq+XCdd08TTzPgwBADQjCDT/c8DcajY7XJMSpu/XO9t5+nGbnV5erxcTxXK3h2dnZ+dU4aHa3tg8Dx5nOZqmUySpJy6qsJCaGQcBqsczTglIquKKUQltSSgnCEgEgJAAKAAWRxhgTggkmCCFMkG3ZZZlzXkIElJKDweCrr74kKu10OmUx39zctC1Y5sujwx0hi5Oz8ywrKiYAMgCEhNIwqDWbzazIm61ws99ZLce2qe7euzUdjz/45ceT+bgWtoSEXIrFajmfzyRQg8lUKblcJc1O++bNW7P58tXxScH43t6eZZt5mmAAKUFbm/07N298VaTT8XB4cSWK4u7No263K3imeaEBwKZhu/Zf/O3f7u3s/+KD3/xf/u//j3/1r/6Lne09Sswf/eB7//RP/hBD9LOf/eyDX/1SMo4hSlfL/+7/8//+z//Fv8AQYgiUlN1maz6fhr57eTESQtimtdvfIho++errdcF1MpvN5/PhcIgJcRwniiLJBbqlDYMahGpLF0UBlG7WGzt7uwCALEn6/f7R0VE9rF1fX08mEwBA6Pndbo9SAyF4+/ZtPwyyLMuyvO7ac8ckEGTxcr5cRXG8StI8zxHCRZGuFhljwLOxbTprx5FWGgEINVBc2KZlfHOeKbXWt24fIYTu3b+9t7eXZRljZZ7np6enZ2fItu1Gp8GUGExGSZHVW+2d/QPDNm4gOJvNeFm9eHXy6uQ0S9J6vT4ejU/Ozw5293Z3t4kJiQmD0F4sZnNCAAAUE4pdaqCqUJwJahuL2VxVHHLpYOoQmrEoKvLnWSqKSGv9xr3brUbj57/6QHFWD31WFWWaEERqgU/X+nOpFBQIa9syAQCK4DXQ0LIs03IwxllZSSmziq29JBBRhKACGiINILBsyzCMPEmLIgMAuK7d63XWzxOlZJVnBkbQMCCEg/GSUtpqtbSWRVVSiqWUSZEoDcqyzIq8ZAWCEBPtuKZj+oFr2BblvNSAFyVTgDHOKsZN05YaAom0BloArSHSSGu9WhWe59++ccSkePrsBTUtBAHCUCuw9jHyik2n0yyOOGMUG8PhcL5almWlNby8vKz5te98+3tv37lfRonK8sAPk+UySRLfD5IsXcbxXs+/fXufGlbOxKlNZ6uk0291Q09BVMWTkyefdwNz9+bBP9W/C6vo0aNHX335qNVqNpuNbq99dXmOEKqqwrKc0HcxNZO4sC2rtFkURY5p5lny4W8+4lJBSIaD8X/3P/y75SoOm61ub0MTQkwLGabheLVWhwFo+TWM8XISaS01wIhig9jf4CgQSqJISIEINB1CKbUdSgjUWnPGtZZaCwgRgFIDpZSSQmkI1tGzdRl1TbgCABScrXGxhCCglRBMa40xVv9wO6C1FkJKqQCQEMI0yU3TRJAgiyiNuIZSAKGE1jpnEmfVahkBgLKssC2r1Ww2Wh1iUIhRyUueZ5JLCQEGAEOioAZKAIDXDTEMNULQdALOmBK6ECWrOEKIQK2RJnG80IApxZWoHMuq+x6GJHQC13aKKBOVkCXb3tyCUF9fX0dRhAiN5wtiUITIYDCcjCZ5mv3pn/7p7vbu08dPxqNRp9PJyoxL8fjx1y9evFiVkiLseU6n3Ww06kDL1XyxWKy++PzLLKuipNSaIGwZ1DnY69y/8/Bo90aapqxgju+WWUYFcLs9t1lfxvnlaDaexRdPL+aL5I9+/Hube5tJEQNbliCpZBEX2M0tiLXjONRw0rza2Drc3NrtbR1YfgcYjirL2YoP5kNgQgjg2eXZfDWFWtu21Wn0HLPmOTUDk6qqKKVcCoBgmhVfPvtybXTVEFcn1enViYbYNM2tft9yLMexPd82qWFSU2tpGAirvN/fQEidn5+cX14sy2UOMkqN66vhar5wXf/WndtCMIsaJKylaaqYBEJDBQkCCBKANJYEIaG1pJRqoKbz+Ww2EYJt9TcN16wKRhBCUpUly5KcMaakXo+FXNc1TcpFNZ8XJePUtP2wDkBeVTyKEqWE0rKqCmrRosyyLJsvFqySgmsAMed8fegxbUsIARC8++D+IloMx8OCs7fffzMIQ2zQjc3+xuZ2FCXzk0VSpn4YLAru1Zp+vRGtZq9OTi8vz3udZqvuI61v37q7v7WjpYIauZZrIqMqGAhNLknQbDuNLWhaxHJcTHf39inFQFSzydQimEDF81Tn2Rs3juy3714PB5dX11mSV2lCLZdQGwghZAGkQgi5psERVEoBJaBWEIE1RQ4hgAlECEEEANRVVRlkPWmACGNKiVJisVj0W6LXc0xzs9vfIIQNBs82t3fbbfvRV8PRZJ4VZVBrtTtdz/Vdn1oekIAMB5dVFR3ubYwnQ83E7/3e716eDV48PY+SbGf70HK80XS5WK6UElleUVMPrwaDyRQi0t/e+t7vfv/4+PhycN1q1BHQlmXVwxpUkrPy6uJsMop44U+u8+HF6c7upmCVbRpBLWw0arPpWAD0F3/763bL51z+n/9v/69/9k//0+3tXcdnlolrjeYf/P4P/uSf/FhL9ZsPP/zis8+KLFvMJqPBsFmv26YRhiEvK7veqIqsqirLILXAW8wmjx8/fn12urW15bquRcn+znar1TId++rqSgiRZ4nnthmrgNZAKYMQCMFGt4sQms5njUaj3W6fXpxHUVSv1znnpmkau7sbGxsAgE6rFcfxZDgAADRr/hWGnVpgOe56dGEbZrfb7W9s5gZ/8eLVq+ev4pXMQAIhgBpAADjn6/i9RY08zXhZMSGyJN3a7K1Hbpxzz/O0dqlhdXvVYDRM8+zs4sp13bxkrU6PUnp6ebW1u7WxvdPe6KdRPJvNnj17Pp/PN3tdTNB4OCjLfGuvf/fuHakqJrOT8xfpdOw6vmvboVd3bY9Jzjk3DKPdaoV+jXPp+37oB8towRjL89zzTdv13n73vWWUjEajV6/PHZMWZTUeXNeaDWddvAOAsZKXCgPtONZ4PK4q7gcBIYQJEYbh5s42ACgr8tlitVyuKsYkBEBDDZAGQkkVGkG324ksc7qYFUXhes7V+ZVlWY5l2567/iKEKKU8B2uINrf7AAAyogCAosi0wlUploJnaUUtu9trYiQUL7ngvU5PaxCt4vl8sSyWTAqNoNIwKSqtoNYAaII0QohgiCHEFjXLPJlNR7VGQ0k+vJ5BTFzbUUrFqyhJEsexFrMp1EADVZYl1iR0whu7NyerFbXMt95+9/e+9ztEgc9ff2xIRVz82eePlqv5d7//PWw6EpDBcJpXyvFcz6+XTM3ny+F0jhF1PF9U7FVZpavojfv3arXavVtHLsUOQWmapnH69Otnk8nk4MY+pRRDBCFwbTNPUiG1FGXg293Oxvn5+dNnL7obvVs37wign794JZS0LKvdbsPlKo7jp0+fJllmWVa72ZJcsLKyXV9rzaUGiq9TBQRCCLXvu1JKkoOiUFprLYWGQGvtOFQIIAT8BnCFBAQAIE3WnRKItdacfJMdgRBixjnnQnKlOCvziuUYg3rDp7S2vvKQUpZlWVV83bjhpVCEAAAIMSChQiFVVRIAoRiAhmG7ZlEqBRazuVaiSLPNHe54ru25AEKlgNIQQA0hsqgpgRICQcGU5JBLCCAA0DY9ioXgoCykFoAQYpmOSQ2SpAuCseBZGi8NBE1KV7NFvoiODm4c7uyNxoMsTipWvPP2W4TC58+fG5ajAAqDkCAsgVwuFh//5tM37r/1/vvvtRqd4WC4u797eHDDdbzFKtEIpymfjK7jZGUb1DaMPI1nEx7HwDRBuwU9yysqQYi5sdG/dXRrZ2cPSUixsVwu22YbUJhVRUADQHHv4OabjJcVX6XZLE1Pri9LmU6W16MlgBA6ll0CgUyCLdRoeUG9MZ0t6g272XecuizBKInYchVfzh9nLC9lBRQXvLBsErheWPM9M5BcU0objYZlmibBjFWYkk6v++XJYw44pVQCvUgWJasIIa1We2e3H2XzRthoNsMizxGW3V4DIkXNzDDJxcXF5eIkVdmSGWACLcMuZJXJ0kHeKl6+ePGiETbazbasuNaSAEQwVUCv8Slc8fVEMU3TqioYKyTjw8XYCd0wDJXmnHOlqqIoueBgrfcAYD2QpNTUEGBiaAjKinEpsmgEIFxFKwihlJzzykcBgoZlwjDUjuNNp/N2p2PbZhDUGGOCyzTNlvHKdq17D97wG7Vau95oNZ3QzYoiL4u0yBfpKsrTs+GluJA+3crN0jRomlWu61ZFrpSaz2Zvv/Fgb+9QFEUWJ1VeuLYvmZqOZhR1msPpvt+pdRsckYwpKTWiZGtri0IFBL9z48C1HWI6N/e27t578O53v3VxPRiOxk+ev/jrv/35V0+elmVMAMQQSlbJ9edNCsmY1pogTAyKIULoGyg4gEp90xbjeZGusc6cV7Zt3rx19O1vf/vmjrQci/NmyarhcHh6PovT8cXg2rSFFyCAkWFx09ZegAEq58s4T2ZffH78zltbnudEq/Lk/My3Fu1ubzEriop5QUgNu6ympuXGWcIUJADZrlOW5eOnz/OSHR4eOo7z2WeffP75555j13ybi29ppdrNumNZYVgA4hgG4XkazSeMF3e/8y3XoyVnTOmLwZADkAsVBI3hYPp/+r/+P//4j//k4f3aR7/5YDIc2bb9kz/4g2+/963/7M/+6X/5v/kvXj1/4Vi2Z5n1sJYmSZamSIM0ijFElmEGQeC7HlAaAVhm+bPHTxqNhuU6tVoNAhC43r3bdxBCtVoNQJlES8dxPN+xHXM1XAjJWvVWmsVlmQ8GV1kcBa5TqwV5niOEDL8mFReMI4Qkr/IkNg3Dt403790ruTg/PZkvV/1e7+ad+/uHRxsbGwuVPHv87OPeJ69fvU6jlJW8zKuiqATnhmH4rheG4TpfYhv0m+G8lNPxBABQr9cxIkxw07Qare5gMEizUmmEsNFsdUvOJtPpNJpLKZMo3dvbu3vnDsDkow9//ejxk26raVrkanB9fX313rfeWs4nJ6evxpMrGeQN1cxKI07TVtiSHPBccC44k+uWuZbC911CUFUVI8HqtcC2XN+2HMe5dXR0fTWMlwts2MPBlVTccrx1rVEJtr5P1NTI85wxZtumUipOU8YYJHhze8fxgiYmGsDFKqqqSnMhNTR0VhRFre60ezW/Zg2GZ2m24CKr1Z393b3bt2/v7Oy0m601d0FK+dtPfzsajfr9Xq3ebDSbSZLNZxPXdSfjgYG064X1WqcWNnmV5EJKwSEwA7fZafaur4fXYljkueFalmOneQEgXINuNCQIAoAgwtDzHAjhy1fPa/UmpZhQZJhmWK+/fHl8fX09mUSdlgu16na7vue0Wq2aFSoHK0W3t1B3Z/PG0S0t4Ucf/fbq9fn9GzeipIiyUgKyjDNs4oyLx8dDIS4b7U6r3TVtd1bg0XjheX55tfL9JSXENI8//er57tb20eH+O9/+AVT4gw8+KDNWFAwhPLoeS62EZKZpzucz0zKwEGUecaVX0ZSLfGOnRU3T8R0NAaFIAT2bTW7evtVqNbIse349sF2PmpaBcZYm1DCISdYZES4E5xwqjRDAGCstpJR5nud5BiHEEDgWNU2LsxxCBaFQCmgloMIYYkK+IWIjBJUGSAKl9JrUiQ0EEUZClpXI8yROVkoJ0yKO6wCw9ndgiAyMv5lAQw6FEGmaC6EAIlxKqSFCSCpYSS0VAJhAhMsym46mZZZfL7Jer9ff3qrVapgSAwIFAUYQGUgJCYAUQsqcc8YAABjoZca01llWpqs8y3KCcZEKQhAJa27g+aEXeo7nGHa8WI4vrlcLSeDrW7duCSG0Ei+ePkVIR9Gy02npqojjxDbsxWxuEpMS8+WL8UcffLSzufXuu9/6zce//uqrr2vN8Hd+9/vvvPOehmqVWa9ePDt5/SqL4yrPkIbhAQo8D2OMILkaTJK43N3Z+uEPf/+tt9/tbPSLrGh328tkvlotgloADUcCUAketjrvfqder9du37mRLi+qbPr66hTKcv/BO4yxeuBrJaht+XWv3g5anXAWXS+zSzNVXWDPo/jrZ8/H89nJxQkTlVDcc61+b+v2rRsmMh598Xg+XhnQBY5yHMc0DCn52clZu9PY2dk5np1qpLMywxgblmlhYph2ELizxaQqGNxVvU63WhbRcmEQUpTJ9fxMaXB+dZnLrLvTCYIaZyyXwAk8ziU1yfX19fByuLe1e//OvdFgeLTtUcPABq14mZc8KzMOBCIIY7xYzLIs8wPXCV1iGAIqTSCB64eOZIxpDSg1CDG01pRSJnhRlRBqx3EMao1Go9lsBkVqOTZjlWlRrTVCxHU8yzazLPMCnxBydnZBKF2tVlrr+XxOaM33fcuyR6Px9t4modR2PWzQVy9fm57FuFxEq1WcIII3NjdN0yqWlhYSIkBMq95s8SIFkq+Wy48/+iSaLZCS9SCsB6Hr+I7j+W7w8tWp6TWJ37IbXeqajLOCZXGSFfGiHliXlxdXl5cvX5wTShezxXwyvnP/TqsWHty+c+/O/cl49uzJc6bkZm9juliyvORSEITXEQ+EkGVQSAnUcF0KXyNL17q8IAhYVaw/e2VZuq5969atn/zkJ9Prvzk9fem67nyxKFlluwgTYdvwzbfu5RWbTBfTeQQgA6gCEDOeM8ZcF7z77rsYY8/zLMv6D3/5VxvtnV5v4/jltZIAQOx5Qa1ZG40Gy8Wqqnit1nA8bzqdfvrpp6PR6OGbD3784x+/evH84ux0MBjM5/P97f7t27e73W6RpxYNe5s9x7UWk3HB8u3Nflmmzz795MWrl0zpeisg2Hz8/MqgYG/36Od//+vV1HzzzTcbd25++fkX//O/+x8eP/ry8ODgW+++12jU0lV07969na3tr7/66otPP/M8zzCMrc1+HMeLxeKXv/g5MajjOHu7O6ZlnZ6e5nl2dnb66aefdLvde/fuHRwcCMk9z4miyHGcmh9cEzIejyejse/7hJAsy7gQzWbTsqzJcCSEaHU7EGMuKiGEZRjtViOO2kAKgJGypWVZUeIsozjP8yxJtdau6/q1mm3YrUZ7+GAwnc4HV8NXL47Pzy/X3Pd6o9Fut4sy813P9Z1Os2UaFoKYcx6tYkoMjLFQklLTcwPbiauq4lzGWQoBRggBTZbxlFKzZFWaZ4tlNJ8tKy4t2z27vPzWu29m6Wq+XASBV+bRaiUPj/bTZYIoKjkrCwYAAgyVSZWZOWxTg1CtNcTItu1v2HWSL+eLGEcvnpl+vWlZBquK8TzvbXUdxyaEKMEYY+tbXtM0TdMglmPum0IoCCHjUgGd5sXr168vrwe1Rsv1fKGklLKoKiEUAAAaIq8KBXWz1UAInbw+XkVLQvCbb35rZ2t7e3vbc12l1CpaMsZExeJk9frkle06lmMjRDjnZcUJtvb3jpQsHDtI4vy0yMtsZVDtOPZiunI8d2tzN07zVZoMxwMJtNbadiwp1io0oJUGGCAEEAZSsPtvPHj54vji/DRstDnnrhdsb26enp56nidYSQiZjCIIJ7a1EwRBlTEmRZGUezeP3nnzfSb4L37x9//+f/yfA8c+2j0UlXj41ru7+7sA6VUW5VKezZgsy8oIz2YlMXTJ4OtRatuyVqudvrrWUuzv7SiUD6fP4kI2N/ZrYfP18ZnrOo1Go9vdODs7EYIzWd27d+/09PTg8JBS//T0dDYZnb0+oRblGmV56njO0dHBzs7OaDz+9NPP8zS5e//BeDRbLBYUQllVEFNVVfV6fSLStZ1LCKMsS875emCJ0fqKCkOtyyov8lTwCiFEsfimyCqV1hpiahgWIQbSGkKpAJNCM6mEEGtgNq8yhAHGSIiqrPI0XQGoAbSEKP+RAbMWcEgplQIAWEVRrJaxhAgRgolhWY5pmkqBquRJVggmQ89Fvm8Q5Pvu5XLhBH4fQst1EEJ5lTPGpNQYQwWAFopXjBclKyutNYFISbx+R8ZklpVKSAi11poIuioQbzdcwyxvvnnzzoOtd79z+4tPvyxS8fXzz303ICX61S9+8/Offtjtdv/1v/43NdT81d/+dq28tExnsUgUBB99cbx/6/I7333/j/74X3/51Yeb27UkveL89Or6dGP/J3/4Rz+qiu8XUWkRczYavj5+wkRiWuBqcBlXMOjuvP+973/3R3/W6d5KEqmddLCcEwItQgEkFCAhFVQUE7MsweHth4Vgo2vy8ourWVL4gXVgbO/f3tVIj8ZXELH+flPq+Hj0eFq8yst4xYNSXbhe0wS5UfG9xkbd7UZRpJXwrVbL3yqKIlrls9lsZ8ttdFzHg45tPfrieDRY+W6nyjU1sNaaYkopNg1kWZZlEAPE8XSVRCtYXmWLZpkXWutareb7+PbB3mg0mhik5rmmYWhITNdynRrUuJzEKq3e3dj67DcfR4v48uwiWq7md5Od3V235pc8y3leqRJTZNvWYDCIRTKLJke3vrOYzT/77WfNoBWL2FLMcbzpZOh6NdPzoyRBBsrKipeVEEwrYSCkFbNh4cIqVanlE9smhmFprTFFjuMQ0wAAQG14bnBxPbANV0s8G6fXF/OyrEx7Nl3Ep6enaZ49O764c/8OxPrjTx5DhHac1snpq1UcJXlKDFoxmGQp0gZXrF1vcJZdTCYIiMC1G1vbqiyv54vFZLLR6/Xb3HCDB++9//mTFxN+/fXJWXPrHqsu9g92m21Lqezl4w/iZHWw92A8U/NlUPJWq+ZT03jy7LdffPHozbfeml2eJ2V+dHO7v1O7Gg0Nr8R5WnNpELShIrZhK6HOzy8VF4bhEYoqUXFRmpYFIMiZAJrKQiBk2qaRZ8ui4ptbrR/84B3HLs+/+lprNZlOuJJvvffup4++4KIECAwmg2a3Rx3DLq2KCWoY5+eXEMLx4OLwyN3cbiyjRRg0948e3H2Y/s1f/7xZ6yiTPDt/dfPgVpxn0+Xq9u3bQdgtBPrVr341evp4Y2PDqXtPXz69GJ7dPLpxeLC/udHL4+TTTx4d7R71Wnt7W3egdLDr5FWpqMSegQU6ny8GafHBo2dCCE2dVVGGDrl7b/vp15cEZN96+9b929sXFxfvvff+P/9Xbz99+jQry+fns8Hi703TqIUuEPxelm7tbY9/8deJTB/sPGjXyOir8UV0rSWwpGOwyixyiuhotuh2WmmaEaRfPn8cLcfTyfn7771XVWGSJJxXhmH4jn3n1mGZx5qXokxsilieZSvudjoUMMGL0ESVQ4qVogZFPgXUSYGKkmx7ezustaqq6iJjvIhPjo8pokd7u6vRsMEDv+K32p2H2zuGYVwNBr/4+199+GuZlUWcF4ajNOXLaGU07DsP3+jf3jehhNBRAEgpE5ZalmXZDiZYI9lo1KbTqVKq3+7OZrPlbOq7buD38rwM27ZvuovxjBWVazm8yLe2Np++eOr59NaD/ednX5kW3LnbefHySS4ntuZ5ztOFpNja6e1VjsiiOEpGRZb6geNYdlmtKsEhRvWGpwS9urqqKtjrlSpjO51u0y+pYSGTHh7spWU1WS0sz68ELxir9ze0LC1OkyznTLTrzTvNOwDAKE7XXWgm2SrKcIDbNT+OV9PpFBRmzwl6jidWsda861rW3pbnOfcONsPQjyZXz66uIUSO43HO86y8unyJWHb16mnTNxfzJZdKFFHOWLPZ1ArEUXF1taAEZ3Fqm0a32/Datu3VqB3cuW16fuOjj359/PolVIgzoZSyTK/iYrVMXNe0PTdZpN0N9ur505sHN/Ks+s2vP0GI7obd/Hr+YOvGub7A/d04jRHAlmMCg8RVpdEiyap6r9/Y3j8bR59+9uizT79QxHVrjVmy8ix65/7dvCrPL65bvQ0OljXfQKF1enIehnXbMkaDaxcqlsYlUFWSIAQUFw8ePJjMJynjJ9eXNtHQM4FjNrc2kiRRxDANu+5YhDhHB3ek5AUrQj+IFytpMF7xRJqPn5+/+RZrN7e0UgQ6+G1ndD4u4qooGcSQCQ4NksgSucZKFgywZbT0vMAgJqSmY3paSKgVhoqzvEwjIGKPypKtijw3DMKEBkAhDDQCnFcVLyHBhmnbticEZgJyASUnTCC+HqYq8Y/MK62QY9YZY2Wq/nF4tqbRG4YJIdRCJJEQAnIGpOQYK0AUK6RAWVVVjmGAEhjIjZaCUsu0fIA9YhmLiL18fSUV7vd7BrYA1NTErKziVZrFiZQaAxNBpLSCiLJsprXWgruWCXxvPcyDEBLD8iAmUZTkqBxc/f3NnRu9Xu93vvfd8fXEQDDwakgjCPDV1WCz1yCweuPewz/40R/++Z//OaU4qwqTUI3wZDD9H//7/8kkxu07N3qt7U4jeHn85XSSYuUNLo9ZGrfqPcs0izjlTFalevrsuGApk9z0ake33rx556EbBgIKYGibWAwzhABGSAOpwT94SgC0HQsAvpzNr6+vpZTNZrNR96azoedb9XrYqte5zpN4Nl9dlmxZlmUtqEOMvnr8Natg6PfqYb/edJ1c+L7PeaWkfH12mqyioih6vZ7tON2N7ubW5mQ4i5IVsRAgGkKAKbUsS3KWJMladhJ4DueVaVAEsOcGjh0ggIUQWsOSCcexNjb7xHKXq2SZ5JPpoqxAGDITO7ZlqUpGq4VpmtPp5Pnjx7PJLI15war+9mZSxtQkezf3DMccT4brXGu300uS5NPffnb84phX4j//F//CMIyqqjqdjpBr+aQRxzEASGstGGdVwSCgBNius72z0+62CNWO62ICi6IoigxgYNs2NWmeFxBChBShkBAoZJ5mUZrmAIDHX331dz//ebfb3T88iBbLaLW6OL0Im41uJaWAaVIu5ktiGlwKAIBjIUhMgihFOHA9y6RAVPPJvBF6tu2G9TqCeLFcXVnDDQlM07YMKkV5fnpCD+0ia14VY6EXgpcvnj0rM6WVRQkMAyfwbSmq6Wzwi7/7KQb64dsPLYKvz854mTUDN5pPGp5brzcf3H9IqTW6Hg0H42boE0JPLpYYwyDw3MDlkqdpJqXGGK3JE45rSI6UiZuNwLHNLI/v3bt3cXWuYx3PpqPZyHatssyDwJutosVi9vr1iZTAdgLLxrduHwoh3n67Np/OIKZcaS+sYyIevvnub37zaDiZJyvx9NlotSp9N7i4uEqy8s0337QJffvtNz7+uDg7f+37nmmRqion06FW1d2bt7Z3+4vp5Kuvvnxw996DN+/tH+yMlqvPPvuiYmxje2eRJE+fPn324lWRZQAR33HHi2w+WmxtlP/8z378ox/+Xq/dsQy9WKy2trYIpWFQBwheXFz8+sNfYQyj1bJMo78N3D/8/R/3+1tK8MV8aYVeq9mkd4zHj58+fvzkaP/o1q3bSIM8Dx3HefjwDa1lkSWtVsOyjZfHx1fDZa0WHB0dNZtN23Y77b5hEARJVUqpysViZVlWu92p1xtSzqbTWdDpUUyUUnmazefL2WTKGJNCtJpN1/FMevXSMJv1Rhh4Z6evv/zis1an2e/3fT+UUgZB0G63/8kf/tHNW7f+6m/++tGTp3EcV1W11gkCAHrtTpmtCDEQQnwNGYRwra65eXTjq6++9lwXITQZzxAA/X4/SRKINGOMc+44XhiGBHdYVQpeViw1CGrUakiD5WxZqzuTMktXaZaWLB9ibRtGACF0HKffa7K8GA8vIbAcx3EcByCoNVASEWLESVawrBjn2ACGhY+ODhAxiGGYtruME0ppo9Ycz+dRlhuWORiMAAZaa86lkgAWpZXntuU6jrO1taWUEpK1GnXG86LIPNsMPQ/nACJtUwJERQlu1IMwsMMwzOLo5NXL5XLpun673ZWcAaWajRoAO8PxiBLz8uw8z4uSiTCse657dnqKAayyBlTSIDhPM8cytJR1p2NaBqHU88P+xs7uzmw0iaIoIthmrKqqAiHiuD41jKxIkyz6+OPBw4f7G5ubrXqTMfH0yfPLq3Nq2kdHN5t5DRrE8S3LNxXQQgvBq3mV3Xnw5v2H75Qc/f2Hn3z19ReNMPj+e28jwSgQ7VbTIvi3Hz86Pjv79nd/t7+xMZtdryu4YRi6rucHHiHE84J6syGE4Lza3t7CBA6uruJ4VQs8U0WUIMuktm1/8cUXL18+b9UbP/rRjxzXEtyllLqu47su1HqxWAAA+k7bMAyI1Odffv782TOt9dHhje397Q8//E0puGm77Y1u2GwBgqVWZVkavgG0ppis644GJtgwCYLT8aAq0yiKOM8sA3AhGZcQYCC11gAThCgiGAFoAIwIplWpGWd5oRhHgsOKa1ZJwYEqs7VJhxCyxn1a1F4bD9ahAQPDNZ973c+s1WqMsaIo1q2Etd8cAGCaplKqLMv17GqdWpvP5wCD6XAGJWvWPCWZQaHr2hjQy/OTMi8Y4xhjDLDiUmuttJIKSKmE1ABiy3Ztx1tvWch8EdVrNWEC17Svzq4bTt02zHo9rHuBLIpmo4E0AgCsZte9lpMsr+/uvfGTH/3+//I//f8My5wti2bDQYgOriMCr//y3//lX/6v8vadg3/zb/8sWbDJdSRVdfjGZplMYRhu9HqVq0KvXlXi+PXVy5NJo9O+ffjmt77745t33iCWmfNYQESBjQDECAKgtFQKC4QwhjiKo9D3eFmMRoM0iZq1MAzbrkeTKYuWE9OA7U49r6qXV5enV8+JKbJisViQOI1Hw7lhBPfueu2O4bruxfAsyzKtdVkUk0mZrCIuxNbW1q2bd+bLBecCIbJ/Y4dVQkqBDbK5uVmW5WJZSim5UIIDarq1sGGbVmJHFCMICaE2JnodMGYsp5S2282wVnemyzjKqrIkEGEIfcetdFWWZb0eTgejoihqtXCxWr58+TLKE6GF4RhuI/RC9/Li+uXLl8evX7ZrjVPD/PzzL6PZokyzP/0nf9xshVG0bPc20rQUjDmuF60SCHGn3U7TOI4AY5XSkmBs+a5pG6XINNIKQEggNrACiqtKlCXjlUbQMGG94dg2pobyfGpanhBssZhdXZyt4c3n55e+7/tuwLLKxHbo1GIjXYLYgJZpIEIIRsZqtUphxPICBCFQGiiIEJJS27ZrUQMqXWZ5VUrDcu7cvT9eLuNV8frlU9dwuy0fkYKYea8dWlTLKrtz89bQjUYmhIhpVWrBzo9ffOqZZRl3Nzrtmtdv1NM82dnovvnWW6PRhGg2v55cn10Y1GyE9nQ8adSaaZoYhuE4zmI5Z6yyLJsQZAe+EpJihYnqdOubW51a3W22Qkm2p6vZMo9KXl4Pr9wwGI1HtVart9lexWnFUstyEZZ5sWo0WoPBNAitpKiuRlNWSUQCxWGj3TfsYDafUwqjVE9nqyBszZfx5fVvIKHf/53vfP93vqMk+8v/OBCiCkKP8wpC+dnnX2Mkfuf979iO8ezFE4rVRrdHiP/Z148H15derU4xLtJsMJycnl0QREfj3Ntx64E7K7Nuq/3t998jEL4+fvX3v/h5nue+H+7s7fZ6vRs3b944ujObzH/zmw+vLk+bYRjrLE2KO7eOxqNBVcm65UqhW412v7eRJfn+/n6n0xYVPzg4eH1yfOvmLSn5YjkzDEMItlhFBFOg8eB6PJsuCSGWbVJaS5Ki0+kVRZGlFUKAYEtrvVwko9Gro3ucMYYxDoIAKu3ZTgkRBnqj0y3LkiD4zlsPfd8XQozHY9c2f/XrD28d3djbOwjD0Pd9wzBqNWOz3799+/bl9UBC0Gm1AUah86osiuFg0Ki53+jvtP5mMi4BpOr88qpIMwzXYVa9tiTU6/VlOldKZVmmxIiVlWmajmX0Oq35QnGiNro9io08zna3NlbRzCJOBVMloef6Da9jWbZSAGkAoV4ul1WZlmXRaNQoNSGiEmnTspaXJ0UZBUFtY6Pj+2FZVhXXRcmEgq9OzrFphc32eB7ZhtvtbRRlNUqWpmnanmUYFqWUawBYhRC6HlwqJS2TBoFnGQ5FwMCw26g3qFcUGUJIVVlVSl1lgrFM8/l8Ph5N8jwPglqyXFQlVwpYlkUMjDXodburKLIMsyp5t9X2PC9ZRWmczKezKk0xglrIwHeVkL5TKCDDetMy3Vpj487ddzUMx+PpyckJJqKscoy17dkaCM6rnt9pHlKTWl989bllulEaG7YhlaBQzJeTrIw1gwBCP3BLVsTzlWGQe28+cH1nES0Btm7c3G/Uw3i+5FnSa4YOgr5Jh1eXs+FwMZ6+ePrsjYdvOY5DKT08PGy1OkVRtVotrcDm5maWZePx+OjoYH9/97e//Wi5mvd6vfOz13s97/BwHyHU6bS2+xtKlFJKg2LbNLlteY7b7bUd21wsZ1xUjmnNFUFaZVk0nV3PFuONjc3do13b9fQnqCjZcpkVWtQ5dzwXG5RQA4qMEEPLcr0JEJBRTDCEaZpXZZ6nRcUKZWENtNIEQKNiknOu12U6BKUUEmgAhRCqKHlWcM6A0kQKyLlkFTfW7gWMFQVQYs15BbiU0rKsNZECIUywgTWRUkOJAQJr2ccaRPaPpgal1PoVAMB657EmK0ADGAhCxYeX5+dV4ftuo1GvWJHFyTcwcmIqjJXUWmsl0TopvG7eGoZhWZZpmpRS0qj3MEKj4WIi5+kieVI8yVfxd955z3Wsbrdec716rbZarRqh4Vp6Nb+ajE+bDetgv0UIcB2xu7sFNHEd0G5tZulkNptxtpr8+DvT4WRwOd7a2rARzKtsdH1GNG3U+rVavdXePjh6cxbD/Rs3Hr757f3DNx2/wbXUWBgmqaIcQo0wgZpqrYAECEMIoGObAGqEges4Gxsbvq9NU0LNYZHFqwkrIyH6abZ68virs8uXjY7PVVaKPCskq4Bt52cXFxpY7dZGEATrQDVCSColgS45i5OMCV6IsmbWhBBMFRwIrnh/o+2izePjY6VEs9N2XdcwzcCv3bhxA0i1Ws6jaJmXhRCMYsQh1FpBDAXnSgHP9wm20qR0zaReb2VxWWQlECKLIwKIH7g3bxzubm3/5qOvP3/0JXn2xAm8QpS/+eRTL/Tmy9nlxfl8Mg0Dr+GH8/EMK7CYzSej8U6v6TgO0FoKkSWpFMoiVAEE1tg2hCilmBAABWdVKZllkvUekRjYcH2peJYl616ZaZp+YFKjrhTAGPle3TAMIWW9Xu92u91eWzA+GY1brVa/05+vlqEXxG5iU8uAholMDQCUEEquytLwnZoXWASXaUaxbtTqZZpwzjGAjPGSi7LiUoH2Rv+dt9/44rOny3l2dvbcdWF/q1awoRBpr1czCHr3nXv5bX3y+uxq8KpkwHZb/V63Hhqff/xLPwwevvXGH/zutx4/e3rz1p1mp82SSAM9KaJ4NqrXm0Wcnp7km3stzrmSEkNg2zQIO47tLZeR42BRKSXLRs17cP9Gs2mvFrO7d/YnMfXCQE+HtXaTK6GhZpLNltMt3yJUBaFTq9UrJharITXAxfkxk8Hmxhbj8tnz4ydPTotc9do7j5+eagza7S3DiCXECpBWt/fq1euPP/2sHlp3795tt2q7O/3ZbObYtLHZrddr29vd6Wj85Omjrf5GvR36dY/J8pPPPnn02SeWYW5t9GTFJ9ejUkgtgYHMdk1JJgSTjgUQ1KwoXlyel3mxvb17dXU9Hk8QJuPRZDSe/PjHP/7hD388HA5ZVfVa9d3tfr3eXK3iq8sBxrjT30jjDIfk1q1bh3uHCKEkidrt9tb2xq9/86vDw33DMPKiuLq+Xuc8knht6YSO43mep7Uui2q1jIQQSkvOebvdNgyLc06paRiW5CJeRXlepmHmeZ7veQalQOskXi0Wi8l42Ov1Hj58OB6PqzLb2uydDa+TPDu7uKiH4WKxuLy8bLa7jVbzcG//ZP+gFLxZrwMAdra2bWK8fPb88GhHSg01wBgjACGEBmaCUlXxu0c34zSVUu5s7T5+/PjZi+etVits+QahtmnNZrOzk2OMoee4jmvWA5cLtLu5ZSBcpoWBaJlUUEBe6m6nZxu+ElpzPRmNB+fXEKiqKqbz6TJC1CSdjqckZEIS07I8ajoGNqBCAhuwFbZdJ9CAPHtxbJqm1EByYRLTdcxWvZnmpdVqIYQoJgghJbkWkiBoUpIy6ZjEsilSglUVKzIopYaaMcHLXCmleUpNgoGEqsqiDErR77XXqADJmWtbUZI9f/4UU7K5uSkZB1oXRSE4T+OEYlILwjLNV6tllaW2YUAAijRZzOZh42anvxnW6kqS2TRarArOqdbW0c2HGOPlahZnCyHKVTwFQPQ2N3/43reFEIPr4XQ8VkITA8dxyiWDGGZ5WnEOEPTrdamkSWmn07lz77Zh2UrT2SodDi5PX56Mr6+qaPnP/+Q/uXV0sJiMXzx5vLvZ73b78zjb3tq6nl3i0YRSulwu4zglhOR5fnVxsbW1ebC3e/fWTWrgq7PTsBZQqJ8//rpp397e3HBdt1kPv/+73709PppOpxCpLI2LPMVIAVm3KAFSZHGkbGbXmr4fdrv1JO2torlfsxWQJ+cngEABZFbkuRBxWRqWTQyKMUZYt1otoCCEEGjIqwpCyBjLkkTIMolXZZlhoqGWlKJaDWcpW8tiCEWYUoiA1hpAqRQoSl7kXCkNMUSQQoUxhKHjrBd1pZRiWgj5zQIv0RpLCgmCEgEAKTSQgZdZth4k/KMebF2hXDvGTNPknK+3DpZlOY7z1dePer1GlaZZtAg812mGihUsy6QoCSEEYaWYUkhr+A84H8iFqpjQWmuAsFAACiE1uXd0N0mSy5QJXn3rrfdePH0yHUzUQ8l0ZVEjS2PHNqfja9uhCCuE0PnVE9937tzf5LwIwp3d3d35LHJ8aRDnxfOx5WipCoR1lmWfffb5xUUDmGBnZ08rdHx8vLUNu919J/Bv3H4wT9Wtuw/3D960rAartMbAsgxMcbwoCCEUQ4ShklIprjFFBqaUAKCAVAbBlZarRQR0hrAklVpMhxJoACqpZVXmGCLPdnYPb2ADZlU5Gk4Hw/lstoDgoiglsZ0oimq1BsIIIWSYtudDYpmVqJqdJqDw8ZNHaZrubm8zUJ1cvTA8GCczoSrLqXmeAyFmgmdZ3mjUPRFmWRYnKy2FVFCzAmkQNj0iJQTEsjwCK8e0clqYmAosBGQYkyLLKSF37ty6d+d+kiRKw8FwnJVZ0Kit4jjnlW1bFWdJrLotmq4yIjFQMI5K10affvzpwVan0esDLnhZXV5eJkl688btIKgNrq6qqmCSGQZ1TJMYJiBQIWU5SimlgVrzwThnUgtEoGf5WiqllGUhziXGGEFUFIWGVbfd2d/fb9aay+WyyPIiyaRgzXpoQFRlWbpaFUkEBFtXLcKwdmNv6+Bwbzi8FlUpoTKpgRHo9tqua2shlRKmZQEK0yLXCCIs42SugTkcXTgu3r/xrSjPqKGEzFarVVEtb9x8Y2e3+/LYWkaXUncEd/b26rPZ699+9JFB8l5/68ZezzGkg1QrsPYPb+1vbw0uLl8fDzAB9RDEcYShtG2jXnN29zsHB3uGbT1/dnx5diG5UJy1e9379+4cv/r6Zz/72e5OH0HU6W9eTketfn80HysEavX6fDU/OTuzHNt2KKUQAMSYCEO7061H0ewnv/8Hlxfj169fn56UCALPel3koBb6eV5JAZeLaGgPO50OhHo2m+Vp/OrFs42NjR/+4Afj8bDT6967d9dxLMH52etj2zJCz0+jyLSsMAgc33Es8+Doxs7u3ni+4hWzbM8gZRSnQMEkSgUXNc+sB6FnO7hR93Y2N7cPh8PR8fFxUZavXr26vL5qtVp3bt96++23LZOuZmPbtvM8X8ySNZmjSAvf8dMo9Tyv222fn5+PRsPbt28WRQahphRzXk0Xy9FovD5PiBK2Wg3P85rNZlmWg8HAcax1wJ5Q3Gq1tra2EELT6TRJEsMwWo02ULAqRtFyVRWllJIS5FhmGkeWQU1KyjwTrLIM6tqWbRrtdltrjRHJiiKKIoTQRpwJIYDWm72NSok8TufTaatW39ndlVpPp1OlgEmN0A9cx4UAaKFVyVTJijiFWtXDsN5sz2az16cns9ms1a3Xut1ut9tq1EajURKvyiLNsqXvmIHj3NjbC2y3SsvJ1WR6NdVah17rzs0HySofXs1qNmKMpUncbbcEI4yVacqyLGNCpkkZxblpWqXgTFSIYSYqYuAw9G3HKwtxcLAnFXh5cn5+fZ2nCTX5aDAUSpqtBmeyqFhZ5koKixLfcwgwCRCuZWIgVvNZmWeu4/ieRwiBcWpiXIiqzDjUJqYUaJnGCTUNg1rEMSFEgquw0SSGdXU1ME0TAPDq5DXCOE0z1wsur69mizlCqKhKCbRnO7VayMtqOLiaTqfIysNae3f3Vlnypy9fPX3yajZbJWlWlqXneRIITKAf2F13K6wHN2/eYBLt7B5hbJ+dXQ0GAy0kpdQL/P7Wxmq1itM8LwsMETVprVZ7+OABgtK2COfg+vz1f/z3/+HJ14OdnvX+wweA56Prs6uz89PXL3cPbzhhSwiRF+X3vvf9quJVVQEFHcejlE6GI8YYr9g7774FgJqNR3fv3KQGTlbR0d52v7+xs7PTaTctyzQo+uzT35Z5JoSwLAsBRbEWknm+EwaOVmy1TF2Cd/udZujOAsu2CONFWaVpHk0mo0oqw7IsxzUtG0JUFUxKCRT3LW9Nx6KYrkWdRVEgBClBSgIpAWNSSuk5FAJnPFkUJRMCWBZwXWKaBjEoRoQJhgEmmACMCKEYUQCAlNqxnDVDsyzLNVnfMAwAQJZlEOL1t5Raa0UIsSwTFcV6ErA+664rxAAAz/PW4tD1sIFSuqaAOxalhKyWyyDw7t27Z9n0/Px0fWdHKUUQV1XFhIQAK7WOWJL1Vd0/RDLlekpBtpq9BTRUo0AAvnH73vj0MlosVtOIYlhkZZ6nSZJdXV1RSrnUgeuWPJtdDsOmU1W6128aDozz+TwaUZJejeYIgrDe1JAMRouLS3BxuYDWi+3th51u6+z8cpnGXj01HKu71b2Z3NrdO2jU2wQaVVUoxJVUmACDWNTAFBOoNGdFyRkhBlAKQaIkZ0VSsSJLkjQdY1z5gVUlMaUytL1ep+mFnuOipIi7/bpXtxVQ51eXg+EEQmzbdlGyFy9eaELXwlZCjOVqiRBqddq9Xi9s1BfL2fnzs6vBhec7y3xWljlCKIBm2LDDum9Sk5qIYrpKVp998dnRwSFEOsnissxtxzIpElJCCOJk5Tq+7ZgIQlaUeRqXaVZiu9NoGe1+UZSWYduG/cYbb0gu/vqvP3h9dsqVxNTISyYVsi0fEyILZhrAdWrxYpaCvFuvFasSSPV3f/vT7e3mD3/4Y1Hx4XD46vmLi7MLIMGd2/cE41oqrTUTXOUCC4gIQgbN+ZJSSiiSTKR5UpSZaZqdWq0oitUqLvNKK2gYlu+FVcmHg4XlhoeHh995XxFCyrLMZpPZZGRapmlSKXNerkwimjUrCHyMMaWk297Y2tpqNGrnr5/NZ5Mg8ELPUIpblg+gVEhCpAwbQwyibGmaZlZGAIugXhOcxNlC6tINiICo1nSm88lHn/y9ZTvb29u7B712iTFlo+GyYOPDm22uNq6un1HKdrd7Qdishc3rq8t66N+5c//ly5PnL/+mSsHBfu1qxkPXcx1UssjhgBqqFthbmw0tq+H1pARKSTmbzV48f1WV2b07d28chAgZrtsImuEyy9Iq82u1QjChVBiGpmkmSUoIMk23VgsOj3bPz8sw8J6nr1bLsipAIwDLWdGsOQSR0WBCKbGowYqc4kY99JWo8rR49eJ4e2fnxs3Ddru71d8MgnC1WtZrwfe//33PcZfz6enrE8O2Gs1mb2Nzb2fLMmi0XKVxYlBqGTZWEVYoSTLXsjzX2Oi0dra2TYqJ52CgF4tZELg3bhxdXl/VGzVPeKvV8quvvrQs4+BgZ+FZhID5fEqQJhS5ruNY3ka3/4tf/IKx8nu/8x3XcTb63XanORqNbt+7bTr2+cuXZ2dngitqisUy7jZa7Xaz02m7rj0cXp+eHe/t7XU6HYSA1poQFEXzx48vP/300+vra4zxRrfnmtbe5vZaXchF5XneZm8jy7I0jZGSjcBHio+vL7Josbe1cfvOHcF5HKVJkliGsaa+nl6ce57Hyipo1BAAeZq5lr27vcM5X1UrqFHoB712J3BcwTgvmJRyMZlezc5uP7i30d14/PxFlqRvv/nWZDY9PT1tt9udVqvf729tbuRpenF2cn52XCRVze32ul3HMjc7fQMYrbCllLIdz3fCfCVEKbOsgAIgANvtNoK82+0uokUleJKlcZLPV6llOmlWpEUpNaq4JMSwHBsAkKRRvdm9d/9OUZUnZ6ec5QpIrirDsFanyyDwPMeyTAggtk1KDQVFSqEAEqZZPhuPgNQ1z23WQtt2pWFxUUURjpNkPQ3m3FIgGgzHhmF1N/oAkdPLs+z5a8/1Lc/r9tuWZS1X8eZW/zCoNdud05PzLMuazSa1zDJLDYpdx2ZF6aS+mWdX16sXL69qzeM0yT/57LdnZxcaQQCQ7docFkIIIIFYMS7FYLQ4P58c9Tdct9ne2P7DP/5njVrgWNbZ2dnp69eNRqPikgkllOJcYo1tx1CSYMnLOCpKTgTb2WgFJr5349b33nm7Ww+Pnz/b2uq12j8ZzFYJ46bvvjw7aec1xlhZMkppVbDFYmEaxt7OLsLg/PQkT2OEtWeZjmsd7Gzu7u50WhZjTAiGkGVScnbyyrIsx7KKrArD0KTEpGR7u18Wb8zm0+vra1UkiOXRZAB5ceNgm5jO9tYGVxJAhRECUAOlkQYIEZtiCTWE2LXsq/OL1WplWZYQwvM8zipCCCUWpbZS63F9hYlDqK8BAUAgpCA0tTaEIFojiUBVaSkBUFApxSUTkK8X+ETJIAhCL/SVv16VlVKMMY00V1xJpZFGAAEFhBYUUMdx1kCF9UpvGAYhBEJoGMZ8Ps+yzDRNx3HWUA0hxOHh0Ww2i6Nsd3e339+6Hl5dXY8d1zJsy0BUQ1hwlZclxgQAUJUcaiWE4FIppZDSXOn1ZJF0wxZf5REygRSPP/9qfDHkFbs8u6CY2LbdaW+OxsNme9M0TUAdAQ2KrKxYKED9WitKitFkcT2arOISI+x4BtDWzt69vDA/+vhls1lvNBpffHEdhJ+8/913avV6Jfir85emGVh2c+/mbi0IKKW26UCks7zkjEMDIaCUgpwzxmVeZpxJy3IoIn7oqFIyABzTcjY3IQyVSiHgqSx3tjYarXa726OGgbGOs5Xp0Vevno7mo+PTk9F4QUzXMGGWZZPJ/M7DB0oC3/fX6hTLdQBGeVVOF/PZbDpfzTv9FkLg6cuvPc+5/+CuHxgUUQAQ0JgiC0KSxWWRZhdX54HvZmmUZAk1gGXbGEFq4MlkoQEQQgJNijLDEGGoRVmY1GjUastFfPvmTdOwhWA//8UvP/7ktycnS9M2EcZVXgKIKTUBgpTalgGKooKQVHnlb4eRs/AsM4mixWKxWCygAhcXF8vFYjwe/+ynfzefLvb294lJEIEFY3mcQaK80PcC16AIIY0poNQCGJCCEISoYc2mK1YpralWCGjTtVuOBVezAlK71epgTBkr4zgusnQ+n+4f7CKkTazbrbDV8qQUlmUIISilodsiFPEykSwlSFKiCFYKyfH4Yk2WZZWAEDNZZVXaaDT2b+4JjfIUGtT/+vEXX794tH9UX6az3aNtSI1FMh0vh07NgpgRB2AKNg9qk9F4u9/u9N65OLvUStgOunvncDqLj4+fK43+7ODWP/uzfx5n4qd/94uLwcrynf5Ws9H04nRxdvFitriu1esYG/3uTryKNjptXlYnL8/SuIBQf/zhp2W+TSk2LLNismRKA4IA9oI6MXCr1ZJSPn/+3LE9Smme56ys7t4+EmXergd3bmzsbuiqQE/mgzzOmw3HswzHcQAABkaK8zyJMJBIIyDBdDxrNFo3bhyGYX00GH/51Rc3Dg+HrrucTwnCO1v9/uYOIahg7M7tm3GSLVZpnqSKC14xXnKKjc1eH2qgZbU+a15dnIe+rZVUReF5gVJqMhkwlrc67SyP5vNBu928cXS4tdFazMZFmrg2/frRmW1T33EpwrysEAKEkHo93N/fdhxLCHbr9g3OpOXY3Y0eJSZjvCgKAHWn29zZ2azValyUZZXt7m5TSiGEUnHboZhAxzVu3Nzf3tlACFVFEXie5ftrOXJVAdswTYIXeXZ1cVZV1dHBXhKtPv3tx5PJpNNq3r37cDabCX4Zx7GQ0jAMz/Ns215EqyRJ/DBoN1tJkjClPMvONajZNaiRazuUEME4L0qgoGfaVZoPrq5t2764uPjzv/jLebQ6vHGUFXkl8jzPF7NJLQg3t3rdXts2sW2h6XDIqmI1neNG7dbhURAEUIPRaPDq9PyLzx7Z1KsF9TTOklVUrwWraKG13tnZcpbOKo4mkwkXmDFWMVWvt1xvnGfly5fHi3l84+DG4eFRvV7nvLIsuru3eWu6hw3IpcKEAADqtn10Y7ff70klqqqgFGsgsyzzXa8oitl0oXlDKWAQWmQ5K3k2nWJKyooLITD1682W6wdMqukyzhkj1A6bIb2eZIvYDYx2u1tvB17gb+0fbO/t7u3tFyWrpJrP5xAhrxZSk2RJPF3MJRfUsfq7264ZxLH48qvnabo6vz5Nq5XrO0oppgqtECWWlHAV5xpSBGkUJy238fXT1xRDjOC9uzf39w77/c2Dg8OPP/54NpvnRYUxBQBLCVfz9PHXz94ztm7evN086jW8GmB8Mhj2O00DSVYleRaHtX6735vleRIvljl78uvTe3eOWq1OFEXz+bzISpPQ3d3djV6HlcUnH//G89xa6MbRUktnb2dzfH11fRktZnPf93/wgx+0Wq2dnR3HMhBC19fXlOCyKmbz6f7u1sHuzsXBPityimAr9Mt4ocp8d3MDm07gWb7r+K6TZlWc5llSlnZlWy7UiHPe7NRsw6zyIo0SpIHWGmkApCp48Y2oS2uMDUgJBJQz1Gi1OJMQAEopQugbA4/UUgIIEaWGlFJKoZRCawsN1pZn1Wo1IURRFIwxVrKc5QijqqqggsQilmutyUtCCdO01wd9jPF65rdubyZJwjlHCK1DHmtTued55+fzyXTuur7t+C9PzmfzkeW6putorZBhYkQUKQVAWmOlVCkkVkBKKdYFTq0ppaaJMMGESF33g/Dm7dAPfvXLX5rUyqPs+dNX/X5ve9u7e/8Bpkar00mylDFWcgYgwSSASLQ73ePXz2ez2DBrWztNy6hxdvnVo/Pvfrfm+5tJAnu9/SRJi9L6D3/x9xqRf/lv/pTJ9MXxC9vyej1je3MTSMEZqwV1g2LFSg1Q4DvTaYIE4BhyUeVl9g+bLA8AwDmPomVVVf1+s15vz2YX11ennXZ9a2c7DOpFJUbD69cXp4PRdaWK8ezqejIomKrVa47fYBUwTXJ4eFiWZVmW5tIuS1Zy5pM6BDiKIt93IYZREvs1q7fZLVhUlPnL4+d37+6N41RzEIZN2wx4pSTXju+YJjVNmqSqKLMk0wByAIWljSAItIJRtIIKYWA2aoEoRFXKyXjIczaZzN5+802CjQ8++ODf/bv/kXNATeq6rgYIMKEJkgoApX0vXC2XUKr97b3FZLRuNPjtVqfVOjo6sm1bMpElKaXUNIxff/DReDh55713G51mvVlTGGQsRSa0PMdybIyzPM8rphzHsT1bSjmdztPoGiFiECsMgirnVSWBpqEbdFoMOb7gMooijPHm5mZZpc+ePZnPxt2Nju2Q7a1OvR4qJSpWzudTrbVpoMHwmhCy2e8axmYUL4tyfQQnrmu5rl8UJedcSS0lpxRryOrNepLMWp0mE+xqcPm9H9wVcMl5de/B7eF12t1oUxMzIbIiZklEDeDXaVktB+Ox49mS67PzV0FQ29w8nM1mp2d/1+ruvPHWt/+TP/mnQpNPPv1cI3jv/o1bt46W8fTrx1+8fP1yOh02G72a34Ra37t7N0vyk1cvm40OhurLLx5TKyOEvPXu21UphFSW7RSi9NxgOp/4Xmi7jpLA8zzDsKLlajab1bzm61fPNvv7333/fdMIz1+Pq1TOJolFje3NvlJgcHlhU6y4z6vMNIzjVycAgLxcXZ5f7OzsVCV//uzlZ598fnVxWavV5tNJo1bnZYUhuXl0+P63vgOW0SKKjo8vs/wlr1jF0jRNITYAgGVZ+K55sH90/87d18fPyhxooGzbStMVxhRAtb2z+e3vfKfRaNRCfzC4Nihs12ueQ22D1kP30ZefAahZyU3T3Nvbb7ZqrmWPpqNerwOAdhz76OiIVeLhW2+naZ4X7OrqilViu9d46623kiSJosgwyPbOxuHRfpZlX3zxmVLKcRzHsSml9+/f831fKeXSOiZwNpuNRqMkSbSWRVEgBAAArVZrLSQcDAbD4fDy8vyDD/5+Oy4dx+l2u91u9+LsbDwe72zvvfPOO69evfrwo4+zJF2D86Ik1VorKYlNBJNZllVZzrOC52Xohe1ma393jzF+eX6R88q27QZGa6KiF7q8Kq6vr09Pjhfz7TffuN9u1QLn7olhvHzx7PmzZ5u97tHRUbvRpgRfnl/EcTq4HL37zvtv3Hvn5Pj0Sulms4kQev78+cGNvaLIXr16xQVynXpZqari9ZYLNPG8QAp4cnKWJ4XWoNvdaLQ6mNCDg32ptdRqsphrrVdxdHd/7+bedr0ejifDkuWEWLZtOsTLijyJllWRhr5LTVsrVJZMyoJVlYMxwEhBxIWqqipOsvFs7rjeeDafRVGt1X345jt370mAyGKxbHYbzWazu9FHCGGDnjx/MVnMy6p8/Oirer3u2GaaRMvZTHDuWqbnuotlMRwvhpOx51Pf91pd37JJnCyE1Aa1fa+xiqvJdICR06g3EFDPX5x4Xm13a3MyHf3VX/7t1189ef+9d27eOKwqvlxGAOJWq05NWwO4WsXXV+Omm+5ubvk7tmeZBGjFqvHgOl/ODne3/MDOsiStuO05nb7JZ0sdRcvlcn/v8Ozs7LPPPgu88I179xECH3/8sZaiqqp2q04gOjo6KIt8eH11eXm5XF5altXpdBhjk+lIS75cZsvlAiNUFEWSRPFy0e926vW6aVFMYLWKiRJFVsaLpWFbxOZbO/v37tz+6c9+jrFJiImR6ToBxjRN8zzJecW01LxinIP1NJ5zXhRFu92BGCsFCCGu62sFIYSe53HFgQvWSzgAiJVVWZZS8bIsCSEIAa2kEExKiTHEGCPLcQNPQR2lcZIkhBDbtd3AW61WmlcAAtOx6q0GIWTddECKMsaEEFVVrZ0M64lCHMeMMd/3wzAsiiJN03XU7OXxSaPVu3f/jmGQDz78tVRi92A3TvNao247PibEKnnFlAJICcH1WrHwza+5Hl2sdzyEJznM2e0bt1zbbfwk+CUJTk5OVqu4SuRynBQxO9y7OZmNWVlt724NBgNMVrhY/fD9bwVhM8urR1+esArcvfPQNj0hTvd26v0NH8O4GcLL00dhGFq+TlP5xZfP7t1/5+HDt3aaDaUAZm4yE/2NIHCsMouARnW3XVV8crXUqIIYKw7KsuICGoYDFMqSBGvuuRaBrOYbIs+fnFydnry6e/vIqWfTaJ5w3tnYMyE9/uiLX/3mC00gB6zW7PgNKyvLeJE3G90bN290uxsXr86fP3+uXJnNI6jUzq0er6os5zjTb+690TFaT58/zwH81o0/GE3GL168+PLXjwAAYehn81WtHhgGTZLE9/1GYwuiAuOo1/cn49lgeHn79l2EbRfZlFJOq2gZS8FaoUuUGl1PDSzSdLG3t/fF51+/Phk8+urpfAkwwUFzS0CRZpFpQYpEVRUGMcskNgH0rBoC9vbencVyioI6ada337gX1HYhDhQsDu/cMayT33768cZmezAY2I+fvfvOe6391vXgzLMVRcrgk7blM01M23QcaxXNF4t4Plu5dvDli5c3jx74ftMwLKQrw2RczX/24V97nhMYNzwTp6s5IlgBYDjurQcPjo+PWxubJ+cjoGWyylzbmk3H8f+fp//qkWxL0zPBJbeWps1ch4d7qBNxtEhRlVVJVlHNNLs5GNFEY4AB5mLmpw1A9rA5LFazyGKlqmRmnjwyTujwcG1uemu19hJz4STt3gAzwAxr7e973+eJk343rO2NZYrF+ro7MncOJsuVNp3Nwu7Q1AMhYBj0XMstigwoLmSzWs6Xl/pimR0eP5jNLzoj54uffoZpaJg7EFW6Th99uM1JPN3M0zQti1ophY3V9tYuwKC/NS7ydjPPDXf87iLzAufDJ/9ktUy++/1b3xj/7E8+l+nKFvEfX73Bsh6E3s//9E8++eDzZ8+elVXz5s1bApBnG53QZmVi6NDUQtu0ju8eL8vrP/nZj5wwHI6HWSsvrs4NR3d1r+Oj5TzGIMGcesT74PH7b9+ctHGDsKvpVEK5c6dnmo5m8LdnJM7KRkqEOo7THe/sE0Jmq/Tu3QdZniTLXNM0wJssywa9/v72zndffR06XjxbZ4u15IJHeYD1sds9Z7AqMp20umnvHIypbSldX6flNFnPl2m341a8rDbFncPdX/3674hkwd2DIkv/5E9+IpSkVP/ZT35kGCZAECgEIewFnYuz8zZn/e4AAp7ElUb8PG1LHlua1R16CCHNNCzLuZmuA3/gmL0qEb4fzC/mvJWu6xqKrpeLzLV/89s/UErrstxsNppGVstYI3R3++CX//nvPM/78Y9/vFzOG1wDpgaDAYKQM9FU9eXlOee8P+pH6Top4ztHd4f7O0rCmrXXUWL0RkPNeX6xuAan/W7v7l231w2x2cN6QwzHsFzDsAbdXlnmiDNV54GBCSsNwDDveYE3m15BCF89fZUl8fHxseu6putCSi5ns7ppkyiL0kw0iBBtBqadThD2QiVsLsrnL77ZGQ/uHd0ZdHS+FVy+/U5r9+/tdE1gzc6nzeZ6FWXD4cjuhdLE3f0xtyjnIpI4J/5NhotMF6rLebOaR6xubNsUjYYhCvygqhoAwNXN9Pzy+u6dOz/70z/v94e2bXcsZ38wDDWrrqplC7JGSWoYQaiXeRsv4iJigioggBIIFJTUCArU8qYWLWshxFRvN9EUY2JZdlPXb16/Y61qGwSR+fD4CcDo/PzcD4Pt7W0h2qaN87YUCbS7ge+Hf/ubX331zbd3j+7N4ihumyJa6zmhlGrdDm6bOE2vpldA06GSF1f1sON8+PDu3qRHVKO4BTj3Xc+wzKpky206Xayub37Iovzw0SfL9HT97HSxWHAmNsWqZHktudEJw8n26fmZTQBxjbKpu4cTBQDpmQumwqyMsjTN4qvrMyDVJx99Cqhd1sCyXCHh1nYvqJp+2NEFL6r2YGd3enV1eHhwfHzsOFaWpZez89B3hWoZqz96772u7z395tsfvvlGSlnzxurY+4ODq9fXdc0GwW6SRM6wU9cVUJC3MJXlm3dvgo5fizoc+G7XPz15vX/3vqmb3/7x+fuf/DiOeFZVnrt7c/NGIQNrlFoGpWQZz72eEYvi9c2pt9WDjgYAwhgLhPpbE8MwDMMwdQtCeHumUkwwgQaGSinJ6rwqEEKapoWeibFzu/W/jRfcbv1vM4lW4FRVJZjs+IHrulVZY0wtx55s7adpWjNu247juoQQ01ZCyaZaFQWApeIct3Vb5hUCSNd1JDEBtKlaxvjVzbRt2wdPHjaS6Ujv9XoWcV69ellEjW4ZVyeLrd0JFiZFNkEEyUQJSDACAEEhm7a6dVARCnVDp5QCKKo6J+cn7wzDaBtWSmUb5nvvvTfsj1ar1W9+85uiKF6+fB10/KzINslGNw3LciBGRXl9PZ15fv/Jkw8wMhbzuBv2MdKTOM3iDEOwXi9N2/B8J042wXjC6vzNq7N//+/+DQLy8PCoZfLV8+9Zww8Pj+7du9/rDgAmeRYXRVGWFRMrQ7e4FEGnZ5laURQU6whwXcNZvk6L5PTi3cmbZ5v1oq6yrNxgPBcc3nv0GBL3ZrFq23Zvb69RDaZAEQGgNE3ftl3X6QjJLi7fxVm+3Kw20VrX9XtHx/1B7+r84vT0nWNZZRAMej3tvfc0Sn3HcXS97/k/vKuvr69WdWI7pmnatmErDubThUFNz/PCoL9crvO8bpmc3cyPj+9PJmPO2ngdazoxXNe3g9BvTM3TNWd355AS59/+2//wi1/8Jo6ylgHBRc2Svb2dba1r2XSzmp2fpxAShKHluBRqm3jD5g0AvD8Ih6OJaVpRslFQ+J7T63U6oZskyX/467+RoG3bZmd3IiWXkmFMXc+2Te3i4pwCOhj0ZqczACRVpEnb6elJzxtcnU4NFECbNg0vynw5K149e2ea2ifv7/O6CWx3vppDik1KbNMrB/2qLHSMZ7Op4MzQSJkXhq7d8LWAbVXlRZNZ7o7nhrbj7u0f3j1+WOTs4vymKBpN04JgR4omzSIl5PW7M9s2snx9fX2V5+nbt283m4VhEgCVpgVAkapshYCBPxiPTIIp1Ba2bS8X67KqhJCu69pmB3JLKTEY95SCL168ePbsu8fv3/V8w7S0yXZnNj397X9pbccYjibvv/9oud5E0XqziU3TfPvqdZ7njx49QgB4jvvkyePzzdOH7z1wHCcrEt0EZxev+qPBaBwqQINwjCH0PavbDU1Tv3O4Zxo6pKoos7aWGgRciTKrWc2ABAY1AAdVXkAFep2uY2nbW+OrK74x9aZplFLdMDQtPcuyqio0TWsNKnlrWkYvDIJuQDRcNflsNX/36sWnn33OJNwk2YMHj1qBXr85S5Oc1Y1lGHcOdg1DW60WWbSajAd37x72BkOMsa4ZCsFNlLx9++7V2zeLxUInNE1jjIBrmvu7O+Nh33bcNImn0+nOzl5dsyxbB0HQ7w0ppU3T1nV9cvIWIZznOQSYEHJzc5Nl2fXiGmN8//79O/v7dV2/fv0yz/MP3/9genPFOdd1HSHkOE6WZS9fvkySZNDdtm1LSllkpW2boec33KjrGgOoEXp1OR2OJz/98Z9QYv7www/T5tpxnDjZfPft1x0/8D1nMBjYtj2fz5ngvV6nYjbGOC+Ktm0xJYZhaJRADHTLFKzJyizJs6IoqqrKFgvPcX/8+RdNK757+gO6vpkMhphSkNTvP3pycLBn6ASoNt0sWV0IqQbDLSGE4/jBYKQ7gSJGC0gjERNQIRoEw95oax2/fvn6pK7ZoD+yHBcTTbPN4WQMJC+ytKlr13W9jlucnEwXsyTOCCGu7QrJVnEU5WnQ7SBKdNMMOh3H86qqERBcL+fr+cyksC5zxQVvm5zVUvEsiZVSGGDetusoSuMMI83zPKGEruum47RMTW8uV6sIEw0RbXf/oNfrUUOP4riqqun0CmMMhEyW68n9UWA5m8Xq+vwi3kQYom4YLH2fEEQIIRhKyaVqEcFEw1mZ6BQSoCDiEHNCpU4IkeZiNlWg0WpDAeIExvuThx9/ZiJInr6czW9mSqmOb/FWiqYSvBkOOu8/eY8inBdpNwjvPXzUMBanyWq1yqJ8NVvb2Lg4uy6y0jFdBCEQMt5ESZZHUVKU1aAsCNXjNLk4f7d354Gu6w/u3UcICCGm02lRFAihMAyP79wd93uq5RdXl1yIyd4OQZj6CALkdoLZerlYLG6FI4ah7+5tAyiDwBGSQQibuvIc27XvLM4uFSZMCtv3hlswq/Lf/va3JxdTywu7g55hWVXTFHUpCoE1OpyMc94YhiGlHPSGmqbVdb1ebVjTDIdD3srbv7YQQnAFdaTrUCjQthwhZDmupmlSSiGlksrx/P/ujL5NJ2CqUaXStLjtEntBICWI4zgvC865aZqYUgihYRiEaK3gEHMCEFQGQsjUTM5llVdJm5Z5mWWZbdtFUZi2eSv90TTCWJ0kUVHlWk6bi2oTrRHFECkuWBRFW1tj0zQxhrqua5p2i5e2LBNw9N802ei/lzARIsS1HaWUanmS5VmSvXz2Mk1yz/OePHliGEZTVs+ur5lkhm0amrl3sD9bnZcFf/v2rNsZbW/vD/9svFhsbN1J0zJwvbpsHMeFSD158qSqqqdPnwJTGz16cHNzE2/mr18+3Rr1PS+QvPjdb3/77OnXvV7P9wMpQNM0mmY4jqPRBmDUtmJnd1c3rShJgk6INdzyKkk3ZZU8/e7LFy+/x0hqOk6ajcyW/eHk4896VLOvps/jNLdtm1fMsrWKZQCBXs/f3t6hxL66vDk/O786XwSh1+l0DKppBl6ubkyL3n9wd7GcahTePz6+f7hfZnldVURKl+L3H35UpawoMqp0DRodf4iBMZ1eJZt8erk0DCNJUtfxvW5ntVj57goLTglBEGNCLi4u1vPvdWJ1wuHD47tVIb/74dtf/eK3q1l85+BIKBRF0dbdwYcfvUcp9Bzz3enrsoos00s2RZqkhg44l01Z2Y6p6WYU56uvv6mT7Pj46KMPH2tEaxvx6MmDL7/8fRRFeZkAJPIiYW2tC3V6ek00kWWxw/Wtncn05ioMAyklxgRXQKr26uTK1zuZlUkpJeAQcdewAt+bXZ1DwcZb45urc90wvDDwHG970JdA9budMk1urpOsbSGEhmanSTVbTS3LTMu0al4Fnc69B8et4G/evL1/70mn0wKV6rppmnYaMwTp7u6dfndX1/XFcoY0YVrvIQSuri640kfjQW8wwYheX183Tbu93ev1xxhT0/KklOtlzttaAWJZlka1KMu//uZLKenh8V6ULs8vX3/z7e9sR79ztN3Q5sWrl2cn8dndbU0HXECguGObQKnNOi6rQtf1xXKuYfLxhx+GYXidCcPEQWhukhvH1bN8U55nugHv3r07mUwoonwsFBdlWYdBz7X9aJOwUuRRXcTNsEeHndH+1kG+EnkqWlaXdSm40oDSBl0NgH7g571+msXL1SqK1rPrK6WEZRlB6I7GXYyxZWhQSWoRYqKa1W8v3v7yN787uZof339gWkG+SO4/ePzjL35UFMVyuW4EODzYGU9GP//5nzVN9tGTR3t7u3XV5nkUp3maZ1GUrDabLCsgoGlecg6qpi6zcjAYlDU7OnqQZYmU7a3CrqnblgnkoNtHIk3Tbs3Fum74vg8hoBrudAPDsZIk0TSt1+vVdX16enIbPnj48OH1xSVC6M2bN5wzKYSmaa7rxusNbxhCME8zoARUyCB6LerxcKJp2lfTb4q0OLpPNERGg5FlWNM6hwowVq9Wi7rKUH8Q+HbLGSFkMB4VRaEZuuv6jDe6rpuOzRWvWUU03LYqSZIoWm+STRStj4+P37w5qYpia2tn2PvzumZlw7799lvf9u7uHN47vKsb1LGNokjydOO5TpYljhfWZe77vu73Mg4uo+zdYoOoM53HL16fW8GgaGRe8aZpOQBI02reCqAM3zIINmxdsNZxrevVjGMlCGJItFyaWFmh5wRuyaqirvS6qnlDNGraloKpBMLWaVPmyWbd8hopCaQCUiCgkEKMsaqui6Jcr5I0zXXdVELMorVje8MhVIAw1nIhMEEY02i9MQ3LJxhBWNd1w6putzsZj9M0Pd7d14n58upFvFgZEHccz3Wc2WyGMaYaphRDCFresI7PWWOZimDYNqWOONFEWqxkW0LZAsDzplZ1hiD2/K7l0eFgYLuObvi/SucXF5cYEoqokhAD1vGcUa/z2ccflEVWN226Xp9fXjUt932/zMqbixkRJI+LpmCiBa5jDTrdtm3LNGGcl6yFCNS8BQgFrm5ZVhRFnU5nf3fv8vKyZQwotTPZeu/ho/FwpES7XG+qqsSWoUu7qepVvEYIDfnkfD6N4xgqcH98z/NcPwzSbGNZFiZWXeZ1mXm+47me9+gRtmxi2K7jucNJkrfvpm+ibGMGLiJQIWnapuHoSinOuRu6lDtVVaVpqoS0LKuumrquJedv37wTQji23esOLNO55Sdyzk3DZ01S1QwjRQmlhNymDqNNZBiG4/i3GQIAwO2A4V6/yzm/rTAUZVY1jMtbtAJwTMs0TUIpF0xVDECJEKLIpkRjqCnL+r8CRSC4LTgAoHRdZ6ymFA8GPYiAAvKLn36KEHr27FlepYjgmlUYwzTdlE3ZtrWm2YZlaMXtOoMJ0QohWy7alt8OP6jQCSEQSoKh6nb7R4d3ZrMFK2qlFGNM04wPjo79IHjz7uTNl3+EGLz34WNCDaFQFBdpyZLrBQTayZuLXtBvW7G3sx/6nqHptu0CAFomvvjJZ6ZpjbaGTuA9ePCgLMuLi0sAgGmAwNXuHowM+gVruG2a3U4PIRTHMWPMMHQp29VmXZXN1384r2pWsaZuGsPSu4OuBE3NiqvZOTWR69oNKxtZjfs7H3/82eHR/SQvzs8uV6uVE5hKCcZKqVpDp46r6QZhVZEky9XyptPpdjodKaXt2ft39kPfZ3Ujbtqzs3dJvMFK7ownTVWbVOv4AdCNnJgdp2cRi1DUFpKX0jb8js+CIDgvzq+vZrbt7u0dKqWSOG9qdnH5TikYeCGF2nKxfvn81dZo/8HRB4ub9ZvXF2/fnEMJXcOMNpuWSdOwLRMS1FZlrusu1YBl64ZOlqLOSqZASamp27Zp21yBxWqdZolNdcOxhpN+6NvRajUZjz794tPNZlNk5fdPvzk+Pm6aamQHX339qmhSxzX8Sq6nZ5Rilq1Xq9X7H3743tHOL371Xz58eNzpmvP5vGIVwtKwSOCAjo+u3pxjWR/ujvLVTUGwDlulKwMIrFEDgb3xcDWfRZvMsb3FcsMYw9BijYyjch1Vr169dn0v6IQY4/V6fftTaxlvGS/LOopjJUDDC2IElqvtH+7qur5arQCkZdWORnvj8XZRFGV1maX1YICEoHUthFJSCi41olmKKwVQXmbnF5c311HgD/70J5PPv3j/+YtnN4uLPggG45CTyYvnX79+mwH4Vycnb4bj3cnk4PGjB0+fvSryKs9zhFBdVr0w6A97QraMSQQ1KZFh2Af7xz/9yZ9fTaeO1ZWcAmlirAWdYHEzf/t6enRkjwbjdFOOehOEwWazkRyGQfdo/262ar+dvQCCepa/t7PjeQ6AAvJWA+r+8d2mqV6+eZ2XxSZa9UZdz7eXmxvb8/r9vm2aV9cXm3SZN0Ml+SqZNwh88+KUOPbWBL58deJ47vHdAyB/9u7dOwj4P/0nf3nveP/O3mC+uN5k8fl/Oc8TJKUECGOMhYKE6LaDAABNbdV1xeoSKkmwfn09G42Gg8FIycoyHdfxeSvbtr28vDYMY29vTylVVVVRFJbFbdvSdZ3S/5qm7vV6t0KHwWDw3nvv3W4uT05OhBC33UgAZBxFUsqmabphT9MIxMg2bChBy5iCsiyKp99+NxiPdnd2GsZn17M0zVjdmLrVJEvfcW3bUUK2TRNF68B3et3OrXyorkuM8XA8AhgAAISSFSu5wFCBljdNU0CoMBS6hgRnWbopy1oJBjhQkoeudbC743acB4dHBtGj5RowC2KkFGatdNyAambDaoJpyVEcx++uFm8vFhz7SZwS/aUddoRQgBAMgICwZQ2CQPEWQYWQ0h0DAQ0hBDQy2t3GGFdFwVtpm45pmpZhcahm63mcpVVVcSU1rAEEFQQaVlkSNVUmBK+bsm0rjDElSCNGlhRlWeuavb/fbZo2z4qmaUzL41Ilaalpum5Yngersp3P54ZhVnXTiTpIw5xzyzJCP9jd3grM+1uTnavpTR3nfS/o+J1ht5OXReg6AiiEANUpoQhCAyOAMaY41Slu6kK2pWkhhaSEHBPFRXtbrAcA8VQ2olptlpTSrcH9O3sjW8dtw03T0rCmY/jDd18l69ViFS9vplUjHNevswIS4tvOPMnWq6Trd03dQgDXRUkCfzweG5peFJkACiKSFPn56cnW7s5nn31M7a0syzDGhmGYuu5atjKt4aCfJMlyvqib0rXswHerqgKM2FbAGimEII69ztPZcgWVuHN81CqRVsXNfOnaer/XzfM0TVIAAAKwt73jFkVeNJBqvuUl5QIi2e2GlqVzoBDRLMemula3rChLAQGllDGmJGgZL1XFGEOIOK4TxzFCxDTsbrdr225RFHVZEUK4AI7nSSk1TTNtG0KYZdlssej1ekEQDAYD3/chhEVRJElSlqWmmVLWVVPeykhbJjClt2FhhDHRMCFQAIgpVlBACJDSNKpzzZQyTlX634AKQAguFQeQR/GG6NS0tjAluu7t3tkFALy7euvWNsY4iiJISCtZmkV+4NiO6dhmXdtpKppGSinLsrwFLgmuEEKENBgTCCF5+vTp8fFxVjyM4xgSHHR6l9fzs8uL+++9F3S7xnwW9geabfbH20UrpvMFY2h7+07gZQjS7797ihSCAKjPhYY0JsTuzr5h2etNzARzdb83GvRH/V6/g3F/e2eS5zlnLUR8OOpMRt2bm5siKy0Ldjrh3k7/9lwJQuvmZm5Y5osXr+qmHY8n09nMCzzDNixXPzl9E6c3AugQySbKtnYmf/r4J4+evJ8X1a9+9ZuT07eAQqmoZesS1lTTLEvnbX19dZbHdbKJdUr6/T6ltK5r13V39nZd1704O5+tl5O9Hd9yiEavptfLm3kvCI27mmPZqIXb/R3Hc6Tkq/VMNtKw9UGnjzDe294TQvX7QwzRi1evbk3NvX4vXsdRtDaoOZlMht1JJxj0Or2z02kSxZ7j/uwnP2mYmE4X11c3juPNZlPXMzmvy8rNsgwACSFElHgBEhw3LSdEqzkvGtbpeqOdLUs33747z/P85z//iULIdOwvfvzFs2cv3r56+9133x0dHcVx+sX4k35/XF5kvAW+qykgBgM/KxON1HW+6BxOBiEedFFRXi5nJ5atEQsb1OyFjUZWvK55gX1D+SYsikSUlGUKYOTYvTJd9gcTx9DnvL2tztuW63j+ZrPYmmw//uDecLsTReu8zA4Oj4oykYK0vEEAaDr1PGe+uH7z5o2E5c7O3u7u7mAwuLqc6pr1+edfEKJJASDQlWhFi6SASpKmlnleJPlGKcFaqVGTgUYKKRUo68KwCUSCyWqyO6Ym4qyQirG2GvVCzzVtPa+y9OZ62jJQFtw0/YPdvfffe/+v//qvv//++8loqICbF/FoNDq++8R1+kVWO9ZQKvPBvU80/eTJkyfRJoHK5AxTzxVtNL2KPCcf9604i4MgeHDvqK7L68tptEnSNAVKdDthnQvHNo8O72o6jtcLILjnOKPhVl7ldVM2nHWCgDd101RJumGyUoh3uoFArIXNPJnplPgDr7c3EPPF5eIKUCRU8e13v7cM+/7xgwf3fm5o5MPH9//+t7/6+9/9OkmjnZ3tfr9r4T6mRNMMTCnjom7YbcqJc44g1DRNCQ4xYm07n8/zPN3dGmia0en0iqJar6PVatXtdsfjcb/f7/f7Uso0jaXkQRDUdQkAWMXRYDDQNE0IcZvKLooi3kS//s0v66IMw3A0GoWhDwG4vr6ez+e7Q5dXwvMdLwzrumoYJwQRqr96e1LUzc//wV8s19HTpz9ommbZLuf8/vFhVZQQQts1EYBStIzVcbzpdDoYQ6prpm1BSgCCZVnmeeY5Huect02ZpbxtKAYQKMe142i1XM6jKIEIZWUVR8l4a5tSemd3xzWsuqlYWUHH8DyfIiiB8H1fQcA5b4UAAEmg7e7frxj69ofXtmvbrl3XdVrkRV3wVi43Sw0TSjBBCirJJTQ1HUhRZEnQDynCQgg/DAxNR4gIJrIkdaE7n88ZY4ZmdoPQtm2M8WQyPnuTcM6VEgAAKYAURCkFAEqzoiyVYfh37hxOtnc55zc388VisUnSPC+AQgoQxwkcr9c0TFuvKaVCiDiObceybdO1bEvXKMK9oKMaUcSppel7WztcAccwkyi2da2oK1Y1StZAEEIQooQSBGWDoIaRkIC3rRJcCM40hErGMFAYEghBWVdpngt+AwAgwh2Ph6EfvHz2krNmPBlxLl8+f75erjAxoAKmRhFUgjMMAAZAQoAAQAQjBU3bGm1NRpMRImS8NdlkkaZpCpP5YrFKkp3d3YeP37Pc3Xdv3vKGSc6rqsrTzDRNTdNm0+WtrQYQvErj6XSqlNje3h4N94ssx7qV5vV0vnIsW2FNAHpydp1Ea31vR7dcGUeLTZLXDQcABaVmuhbUbMeBmArR+o7tev5gPOES1VxwJeMsi+NNKziCJI4K27Y7nQ6QSggVxzFjHEJ8//5DxphGdEr1W8R4UzWEEKkghLipmzTJgUIHBwf7e3eKoiiKwjAMJWGWFnVdR1G02WzyPH93eUUptWzDsixIMDV0CiQiWNN1QggXgksuhAAASAiE4Igpy7I0S6+q6lZfXpW1UooLlmWJUKysq+FkYDsmIhAAFaVz13W3dkbbu2MI4XQ6VUo1TYMpUJAryKmOHFtX0qAEOLa+IVrbtnVdM8ZuvbsAQikhGe9MXr87ofqvKdUn4y2kUw7U3sG+3+/XUlZSTvYPsEEU0aarte06WNHDo0cGwVABVrd5nFRpjoFaLmZN09iG4XbC9WaZZEXTMibY1fS6rCvXtX3Pm86n79689X1/OOgtFour84vL8wsp5fbWeGdn55b7VtZl0zSTyVYULS3L6fdCy9T8IBhPhsEwDHzr8vLNIprnVey57kcffHjvwUNMyffPvn319hXWsBc6CrKyqhxfIxoGUGZJkiZVXQnbCEZH24t5Fo7Cx4/f7w36puVwoUrOkW6MB8Ojw7uTwfD63fl8ub5aLFoFfMftdDqB7e/v7HLQ6gQz3qhW6LqeFqlt24HruZaZpvFquQiCoG6qquKdTqepWBbnFrX6YTdaJP/uy//fu7cXi+kSKAIh1aguhFqtl8sloGMnz2o/sIu8Xq02RV6XlUIQG4Yex03bck2BVrCiKbGOO/3ecpW8O33tOubh4WEYWt//8INlmHt7e8vZOk1ypeB8thQcvffo/aapnj//4fEn9y1bN0xSteXWziTJVlm6/uijh7/73R9uprMsLz797D3LRrbDuj6p67yM5wWtWbnc6hsLlWggJ1LXNUs0m/l8BYGgWGIM8yIty9J2vM06TtKMmr6u64NBzw306ez6q69/PxxsuU5HNyyTUs5rhEGnE2gEI91XEGR5Sakep6mUwPP7q+W6qhrTyqQEluNDTCBGNWuqpqa6LMuStZVl6Vy2GBEvdMY7Q1Gj7a07vUFXiNYwNOLqLa82G54nUT/o2U88oWhVqSwuWT3XjGrQ33Js9+ju8enpyenpCYAtodB2NGzvJnGVJIkfeqtlsl7Fm3VKsPHhh3ezLBOsdWzHsixd16NovVgs1vH84vot0fmD+/f3tW3GT6JseXl9JhTt9Ls744P9/d13b9+cnJyYBtnZ3eZhs5hO49VqtD3GSF5dn1V13h/1FeQKcYnE9u4EYVBVRc6y8f6IaXT5qwV15e7dAUDVZja/vsyyaPrzn/9D1+4lyfz5i+//8OXvqU6OH9x774MnPCNVVRV1U1ZFWdYVqxFBjulE8dq0bCVJmaWaSQ+HB8+e/vD3f//rzz784Pj4nuO4nIv1eh3Hsa7rcRw/fPiwKDIpeZrGcRxrOtF0YpqmC2kcxz/88EMax5zz9Xqd57nkglI6T5Kvvvpqd3f3wYN7YRhCCF3XZTVdr9dQIwDCtMiv57PBYNAdDpRGStYu402a51Xb7O0eHNwLqqqyev56uSyKgmLCGEvjREpe12VZapZj93q97d3dxWpZscZEsAWyY1kQqjRVN2Uu66Ys84uz0+O7h8fHx1Qj55dnNW/9sFPx5vTqTAgRLW9e/vA9woBzNt6e3Lt/FIahgki1nGgaxjoQnBASDsOO17+7f+T3OpvNhlKtrCIg5aAX3m6Uq6KsmtbUKdZoy+qqkAiALMsIlciwpZSEICllU2WGZlqmiTBseZOneUtr17QoQdjQLV3jrdQ107AtCCFuGljXTctqBpqWYo1otiuQUdQcKOSHA7cz0K+uGGMQYsFV2wrbdjzPE1zdEq7qpiQE27bp2g7BWHB+cXEFAIjjyLIsTEjeNFiCusxly5ssr5rcMnXDsSjSqFJUCUyQhrAAqG5lIYRsm5bVGAFNI1IBhTFQgLeMMQEAoERfLVPbCiFSq03CKtYJh7blQkjWmzjo9rr9LqZamhc1qyRrFuuFZmsUQaRj3rRBPzy+dxg4TtXUN8sFwtTyfIXwnhs4/cF4suX7fl0pCGEcx3VVbFbr66urXq8X+J5hGIZlcs6yIq/bWmDYSjVLNvV0CqTy3C4itqG7k63dsDPknH/3wytMgN/tZ41ISnY2WxIIJNGonwVBgIlOqd62wjGMh/fvmpaHdWO5il68eZvXTcnaMs8lBGVZFoWybVfXTc7atq0BQACg28ajbbue42qaXpYlQviWm4Q1HWEEMVIQSKCoroXdTtjtfPPNN3lZgM36vwKRNDocj4YA6L5JKb3FJAMguRS3lcs4jRhvRSPqlknJb/cabduSVtY146xdrTZxHEMI/cDVNK2qCl2nTVNblr6zOx6N+hKIoiiUDtMqsXyDEFKXpeMbGFEhBASgbVkcR0rKuq4BVI7jUEo7XfLfXRKMMd7KW701gRo5u7osq8YwLIAxV9Lthu998D7QaZkVRdsKDZd1g5u6rItScp0jqpmFFK5lWZZjU/2maXnb9DpempcIA4KQphHN0MumhBhBinXH0h2HCXl2efGHr7/qhP6dO3ccyxhvjztdv2mawPP7/S5rmtlsvkpj0zRfvXkluayq5tuvv+p0evEmKtJkUgw9y37/4XvnN06SRbpF7989Qgg+ffbdm3dvTFszsU402CjVlKWtsBJKAtwyIVpJkdHxu5PxrqjnSqB+ZxiGPc3QNcuU55cZ41WU7CNi+OHwAB8xcXV+UXFBJagvLhEClCBCIJQKKxmnCdWJblANo53tUdNyyMTBwQ4AqC6L3799df/+w0FnpGsma1gcx/P54vrqQrb1/QdHnuO/evn68nKh68CzgWbQROhxVOmaNZsvrqdXTQuqqjJ0SIjDWAsgxURTSqRFVp+WUbpuY7lZr10H/c1//sVnHz9JN+u2qmzDdv3QdQIhFaXmchENR1vbWwdff/U91MKkrJebhFDSCGHongIEE61lPI4K3QDbo0EcT6NF4ng0jVcE1VDqTbHY2wqbYtmKVDQIaWJ+HV9cTM/OT8P+NiHy5ZuXCulVy1gpEZZRNputTn+0+eAv/tGfB92jr7757mZ2npjJaLiDLbW5XnLWuq7bO9jpj/tJnEVRQggYj3Z4q9Kk+P3vv3JsT0nc6/V6vV7burdBG8sybB9lRVJUOcKirktdNx3P3toaRZvy4O4BgfTqaqqUcjQTI12j5iq5QUrTMF0s0zRrdg6GHX9oO/4v/vOvR+PxZGvw+PHjZz+I4ajbG3YA5Iv5cra4aXlzetEWdZGki7KKnj77w2jic5k4rgVh5fpweze4urr69ukmF7OLizNJagGrna3dO8d7j27uXV0uzt7cbG8djLfHo8noZjZFCM0W802yUULOFvM8TynZanmT55nrO8Odnh96fui2sqUUh51gvV6eX5xKKAa7Xm/b+ujJ43/0D//0+y+/3dxsphdXNxfTH777w9HR0d7O6IsvPg26fpylfiecLpaU6xBCw6CG5RmWHscwK4uat4Qg1pZCtC2vG1b1ertlleZF0jIhuNI10/fCLMvyvEBouVqt0jRt25YQApHK0xStgOd5vu86fu/y8vLi4iJar4fDoeM4nU4HKpAXaX07CbBtXde73e4tDi8r4CZNTNcxfXcRb67nC4mh3fFvFsuyLM9vbu4c3g16fWhoYb/H5nPXNm1zW3JRVVW02SjJkUKapiml6roOu50wDOfLRZZlNWtm83kuZjvbW55uBJY17HamTbma3eRpXFb59vbk/Poqb4o2UcsoVgCZpmkDjqAYjgaWa9V5kkdRL3CVAtdnZ54XEN1gDSeEIgfqGIem98Wn7z19/mw+XzAGHccbDoau61OqvXn1+ur8gjclcbyqqYqEu7bjma7QqanTBkhd03jDmqoOHK/j+kmUOroBTJ7GWbpZy9FESZlFcZwXWDeJAE1Tp1lRlnXZ1E3dep4HMaoYuphFp9crAEC32x0Mh3cOjzCmlNIsK9bLNcGa7/u3HXohBGvruq4Yq29rbVLym/mKc85aLiGAUlCKbccULQeci6aWNaM6tSk1CEJAEi7ivGxN3raN4Eg3iGEaQrcIIaJlEAAFAJRKAYSwcCzLc4Mkat+8u9YIHQx3DN0yraCoKsMgmNCr65s4Sydb234QHD84blpWVZVu0SSOkzLyLFt3dKRDjqVU4PnrV2XddMoaU+r6IdFsiegqKWcXF0mSXFxcmIamUw0DCKWK1xuuuFjJJIsBwXuHe1sHe2merKLN9GzuOA6v4bA7yaJyPNoVgs5my01SEoLOruem5zOoSWLGRbYpWu1m4ViuYRgE4bKpdI30+30uQNWwzWL26tkPVcttPwCEWqZVyWIwGARBIIRIy+xWwieEEEwkSeK6brfbu3Um8VYSRJumqdpSN+zdvS3LsqSUN7Orl6+eVVUVhuHtyNww7SDwbk9lpdTpzbtbwlJZV6z5r8kAiFEYdhFCjPO6rjmXAMFbgBJuaghLzlrGmO/79mhsWxbGeL1ebm9P0iL2And3d0c3qOMFPnNv0vnrk1cQIiHEarHEmGqaBqWyLKeuy02ElVJAKoSIbduEYtfzOOeGaTLGOJdt27ZtK4QiQRj+k3/2T3u9we/+yx8kgo8//MDrdJGm16wxXLs7HOA8TfN8tL1VVGWaZ0DCOE6X02nouXlW9gMXAlnlRUtIniZKqayokjwvGhbnhVKQU6IAqpuWEMQV8MMgCDtU13b3b/MB9W3TqdMNlFISiArKXth5+/pk0O0DCa+vr0f9EWPs26++ni+GH3/64eeffr51M96kSy5bnehJFn/33TfrOAk7nU2y4RXrDUPb7SEiIJIIIU3DtqUJToAirBGcccZaQzelVKtNZAtR1Y1CKOh2kaZlTQMQCoaDOM2S9QZoVKaiZNX1xYVp6b1hx7K9hpVYw3Vd6gbd3t5+/vLFfH7jhcFmHed5fnm5AAC1e2LcGwfdjq17h7t3f/Tpj/Kk9F3fsb3ZbCaE8Hzn4uLi/Pz83//9eZ6UhmHcTBdpAlyflAWnmik4wERDiBBdwxhKJMo6mS2WJncxIWnO/9N//MO9wzuYaC9ePd0ajAzD7AZdBLX9vUNWC0O39/fuHhzc9YOdZ8+/T7N0MulcXlz87E9+FHa2v/z9f0FIa2pgG8jUrcs4i+OZpY8IADtbA8c2LR31O92rc7BcpZlkSjSKcyCbm9n86P4jAfFX370IOnYcR00h9/bHvfEIUVbX5c3s2nJ0qZjjWvPZTVGU/c6wLGophFRMSdYooetm03Kqk6ATRFE6Pbt6c3Jq2w6ixHIdz/chUpRiw9AJIYTG19egaWpNh01TQQghBLqpmaYSsp0tlnGUbm2PLctYrZZZUWrU0rCBIPZs5PnmsLv15t358xdvhuNxGIZlWff7/X/5L/9nqqmz8xMg+XS9vJpetm3z9NnT8c6wN+iaNnj6wx9sFwAgjw/vapRiTQzG5uuT+fnFwt923MCkhnp7+qos05/86M9++rOfsBr+0XleZ+o3v/376XRWZrkXBoCAaL2YT29M29jf3dvZ2TE9S0Bu+baEfGdvZ/dg+/r68urmqkc7o8lwvppeX18vy/Xhg73t/aHj0s9/9MHyavm7Ku84dlU0ZZlOby6G46179+5tkhgbGiLUJw6EEBNNSimUBJC3bcMYcyxjvV4DKTUNS9ASA2uG9uT99yaTyS0cfjwes7Z++vRpnuc7OztVVXDODVPzfT/P8yxLlBK9XgdCeLt6kFKu1+tOJ9A0DUI4Ho8xgACAvb09TdNuVXWEEC5sQPBoe2c46le8fvr0+9lypbv2m3cnRVn/+rfnf/mX05/89E83adbp9Vsur6+vHMshhEghHMfxHbdpmrIsdYOmWUF1rSzLxXp1dnG+XC5fvHzZV5j+9KcHd/bG/cGDo7saQfP5TbcT5Hmum5oEYraYma5b1k3VclzQ2XK+vb3d7/uO7cXx5mZ6RbBijFmWY+jUdRymSSkUhUhyJVmb5WsEWOhbpu0qBaRoPNfc2tppy2p+fZ0nKdfNpqyaug5sd2s8KQ2RJMn06rrXa2zDbttmvV7PqhsNad1OBzogWW/qsqIQAExYVVu2Z9muphtp2aR5XVUN47yp27RcmbYVBEQHqKrrpmmqhjetHISubdth2PVcHwJEiOZYrhCibYVlGRCp1Wo5m17VRVlrOrdMQmlRlnlVcSF0yxyMhmEYYggNqmmYCAgowhoAivGa1UJwzaME64IrCAUEmlCKCwkBhMiAQBKEAQAQUARhGPaG/f5MNt989W0cx4Ne/86dO3FSJEkymZi6oTs+sTzX9b1NvInTaDAejbe3zmZnFzcXtmsG4WHNmxdvrjqef2f/cJVEy+U6ynLNsIm+2sRpfzJxbFeWom3bzXq9PRlPJpMyT3Vdz/OcGjROk0289johJoQpkZT5Jk0o1gnSJIe6ZlNi9rojz+2so8QLukWRL1bxKCu2d7Z2Dg6nN1dQN5qKWaazs7VlWdZiNl+vIwrQ5eVZ04q6yKHggLdlmkGqBUEwHgzjUgqhyrJO0xQhhDHO0mKzWt27d++Wf7xcLm+m8zzPpRCcc2TCtEjzKr/Ftd1GtcJOKIEEEEACJZRpkc6Ws9VqFUVRowpMCMb0VtNjGKamm7cQCKJpACApIRcSKYQQxQg1TWJZVqfTcV3fd1ydakWeR9F6MOgNhv0o2egm0XU9jjednt/pBK9u3pxdnnU6HYMaCirPc5RSbd1mWSK5AgBgTBGAGGMAJAASQEtKyblUCt5+JAixEIIoqA9Hw729vbysm4ZxwDSTQMSLvDZ0e+SFoBCKt+VpYluOzFijIycIt3YOnn7zB8O37K7bqTruyAZSLZPC050kWyzma9aq9SoGCMek2druEkNwUd05GhkOXy0WSKskKddZwVmreZqpGQzwumEKo1DXXE073t09fXehEfO9e48xpFkSvX19tl5HlOjvffhod2s/CIIkj7IkVQht7x4IdFrzUpG2KFIQsa3dSZ7GB3v7bd2Ymr5uVxijH394vFqtEysQQgyDoet7881Kp5ZLTMKhjY1JONjqjdfzRSnBsNszAMqyrM2u9vfvQECePXvVpPX9+/dt6OsUQS4Rb6PVtefrsxV4+uIHTXe3d/dav1zNNx/tdizbn91sDMBcLfjZj/5cMVUW7Be/+NXb03d/+OPvmWp1wzBMLS8r13XjpBLSCENPQQDhBmPHMKjXoaxpkywVQpimabs91vAFyHQduaavAfBunj45Pv6H/+Cfn7x4SWTTlAVnhWVq0+nVP/iLP0uS7H/+l//Pcrr54+/+P1gT1zfLTt9r9d7ZGvzr//Rqsy5MYxBD6++egijuFGX96vsIwMZBD5u06W+6h+HBRXGxaigvq7HpIgPsfbiVvzw5W2329x7p1rOiNB4/+qJQV6ZF/9E/+QtqwN/9/tdfffN2d3fX0LbmszWCIefkcr6SqvZ8c13fnC5f9LyRbbmeF3JRYM1BBGZF/tHHn0AId3d3F5u15dt5Fo/G/av5pW5QjW4ArfJ2la3ZaGsEUP16+jTwhkKDN/HN8+dvAdccrxt44yJZEhUi09zbw0pBoumbKJ0tVp5NPnhymBX5xdXz6fLF0f3D3vaB65mbZKGUsnrsvfHB86fPJxOvSC86R1rHKxxat8mpoe+cPksn44eQwIurZWc4CSZufF0EQaAJb+hvH0zuraYAKOt/+hf/9zv73z/74dWvfvWbv//jL8Mw3N+/45jdTLDGII3gf/rpjz774tPr6WVS5YIrrJMsSdeLpWhbIMVqtfB85/Dwzu7BXtXI169fp2kdZXXH9Us527p3GLoBlCgMgk7YjVYRq4vJqCel3KzjREMYYwoEAKgVAiCiYYIpkFxJJl3brOtKp1ZdtaZp2Y5/dOdelTedTgf7OHXTo4N7Nzc3y9lKcUSRdnFz+dMvfrbZbC4uzj3PQwgZRDvY2ZtRXUpZN83Z6aXvp91ut2lUK/FgPOpNdjHGi/U6bdTpy9d+Z1hzHmXZYLy1d+dB3aIoimbTPI7EixfnOyMbAff1i4vdnf3FNDVNezNf44BqmmpaBgk0bQsjJAWrJccG1g10fXFiY9nR8Vc/fHM0mXScTpxs/tN/erO/u/fpx59pmvHowXtPHn/UKjldzA7uHm7y9Go1BxhJBDOW7lpalaXxch2toj/72c+73e7J2ZlpmpPDg3W0fvbDq9VmqWlksjUaDAa6rqk8F1kmWkZ0DRGMMVAybdnGcoXlgSjLZlHZtm2/0xvemQy2x0pW3y1v3MDeFMn3J68M3cIAv33+uueET44fyJqJpqqhenf1ejgZS7MxPAOauAE8LvOIlUGv55v2ZpPWDauahiWVbSOCMAekrMUmLoo1b5oZxu983/d8xzAM3kLPczu9rmUZdV0bZuF1ukmS5HVL8wobRDFMAIEMqpbLolY5U2nFNimqWxvRLIp81wi6wXyV9oY9qZZAtbxOQ89Lko1lWVC2nuOxhmNspknetsqxfc7VasGQEmlSVi1Py8oVqhLKNIkehlHTGh0dAFSX5fW7y6zIIcB1u6Q05m3bM4frq2gGl7ZuVstmlWweHzzmqRgFY80wBcYA0q2tXdNyENE26fWbk9eMMaIjv+fswN31eukHQStUx7e8yahp5fllIi5iwZWUfusja9ArMds66FOjhih99vK/jHf2vL4fs6qq24tFrNveeLg9DntAtkCjZtANJ9tS8Xx2XVIuVX1T3rC2tXrOT//i86ur6c10YZqmAVnZlLYK6nVEEJn0Bi2XeVG6lmNs20f3Hnq2U5bl5cX1fDbDAFKKBee6QVzbwEBtlhvTsHYnh6ZhKwUJIXmeUg1TCqN4M59dR/E6TRPiIgrokycf6ZpRFIVtu0mSlUUdrZKrq+m9e/cwotlq3e12oeSsqrbCcGdnx7btsiyrqpgtpqvVqqoqCGFV5IvFoqqqXq+HMd4K9qOL+fn359vuNuBSMuZirY4jKbkCAmMMoZICKIUBplBRVkvRNsR2EEICq9vsDsSIsTYvSgKQWqyWJ6fvkiTZmmyXVROnydNnzz/84BPdAKZlffv9N6en5wTTyWT7k08+sQ1XMl5lbDjY6fimpaEiz1sGHNvo9Pus5UmSVE1ZVc18db1YrJz9/s3sTNNVpxv6js9loWQTuJ5tmwhCxYFSUMMUY4pQDSSUdb3ZbOI4tW17NNiyLKMo8jRN+/0el+18Ph/O+9t7Q9/3yzJfrzY3i/V0epXmBdaIrtkuxLpBAadJxKY40hBeFjEUYH97mzPimv2riz/u7O0CCaUEGFEhlBSorVsNa5pmmKbt+yES0DV9k1gGjcpqJTloWcsqvpgtgyDAGtI0w0JGC5oo2WR1iQgcbY2kQmWTl0n18PgR5AgqZGgmL3kwCE5O342HW9Pl/O3lu9Pr81UaS6xgXbIF26RUIahpWtXUqOWapkml2rY1bOtW8nH7AgBomEAdd2xdcoYRqYvy4uIq0K0/+eLTDx4/rrL4zYvnVVO/evVMKv7Nt19iSgHg33z3rW4aN8vlk93DDz/6KI6KOG7LWlxeVxhVu7sDjbqrVblepwq2EADI3ziO87s//KHb7789fXd+cTIcdxVqxluDTi/cP9g6O5+W7eTP/sEXf/fL33ldallblq25g7Btq6jI5ifTkrWPHj7WLcs0XNu2hWirOgeQrVeb8/Nz7dAIvbAXdpTECEDPcfb3doAiYRgul8tBf9ezLcmqm8ur169fdjrB8cOu4/q97iCvY4SQVLBu6tVmPe4fbJabt+/eEagHQTiZjIaTHsZ4dnE1pCPDsCAixFg2grOlSMvi8OiOANL2jJ3drbop48slInAw7GETAIU+/ezDzd29xeJsMDDPTpee79w9uuO7dw2ya5njKInHw4np9TBt/vr0f4/jlNVi2Bu7ruvYYZ03om1++tOfHN053t3e+e1vf3f69mSzmhuG2esEq+nM9/3VcnlxenZ+ecZZW7VlVqRX52c7+7udvg8EvLq4gVD1h73xeKxIAwkuynq12ZiGFfQ6pmn7jt+WTEpZVKVAAGqo5ZzqWm/UW8dNWZZ1zSBAOjVc15WtjDdR09Q7O3u2oX/73dfPn7287SIFfme+WpqmyZVECBi2dXT/nhcGeZ7ezOdRvH598na0NRmPx0SncRxPp9O0LC3LUgAIKTnnvu8PBoNbXIxpmr7jIgWqqlJKdTodpdTldFmW5YsXL25ubizH3drachznm2++22w2hoFv3/v69euLi4uGVXfv3k3imEup6zqXAlEkgUIEYgAFa03TyPO8LqumLh3TOjo6cmz7k/c/PD09rYqUUjifXxdZrJvG0+++qlij29akGxaH+xDwVRI1rNEE7/jD9Tr6w+9+t793h2C8Wq3/9m/+4yaJe/1+kqVxHEkk+8Pe9vbEtm0puSIVhBBTklxeSAj2dvd1XRdCTKezumIa1pGCommRQq7lbg+3Li7fdMOOiOOaiywtTleXSKH5xU0bNI8P7wdBuCiruuWM8Yvzq/Oz68ubTEpZM75cb4qy7g8nw9FEQmSbFoaobVsuGo0Yge9YlmUbZhXlnqdpmgYAXy2nt0eCrtPhcDgYDDzfGQ4G/V5vvV5FUQQUIEQry7LMK9VKCIBpmnEcn56fzeezNIuJTsLQzfOybplQCijEGgghlIIqSQg2lcKMty1TYdi7DcBzpqye3usOyrLeRIuq4r7vuq49HI6DwIMQV3UjhEzijCvZtgIAhJHGGMuyHEIcBjbWKK+rJElo4IfdjmXoEgLX93XT4EIxzjXd0HU9LfL5bLm6muZ5ijFerTeGafpeFyKS5FmZ5uPJxA+6UZpdXkw3UQIh1jStqsrVarW/MxqNRr6jL2ZzzjmQajwex0m+WK3SKE5Dv7Mzcly7KlKgRLKYn4iGUponMRLS8/yDnYPnz1/Ok9VgOP7pT35WV/zs4mJ2swCKxHHsOI6um0zwoigwQo/ee7i/v9+27bt375JoMxwH29ud9XKZpymlxmBnVFeMc6FRyzBsBHFT10rCRZK0bet6puvapm7s7ewf3b0DACA+EkL0uv00TauqKcsyjmPWcAhRv99ljCEoTVNnjGmaNh6Pjre3fN+fz+dff/v01q5yW7bMssxxHCml4wZpXimlTs+vZrNZHKe6TnWDQoiUBFICABBGiDXsFv1023NkqsWYEkLevX7j+77r+BBiKaVSUEPYtx0CADAMLU3FbeyZEDIaTVqmmqahtAl8u67LPE+bpllvVt1e8NGnfwmkQIDbHVPXAYa8G07ydGNbWn8wWS7nuqnBXF1Nz5areV7I3a6pGQoRpunKcrBbGWVh6DqK4hXFlCCKIBGghZAwxhhjEJM4Taqqnox3Or2wqorlcllVJaW4SNPr62tq4JoVtmdfXF+9fPb6q2+fNi1DFFu+qxmUS56lLFo3rh1UKUSmVRWcKPzq+ezf/qu/i6OauNZwNN5sIq6UEKJpGgyx6wRFXiebrPArAjXLdCVTpuHAEAfNTpJknPHQ6UBEMNA820IAaJggJBsJWFtBJF1PZwLWNTsc31U1uDmdoRFNVtnTr1+eTq5+9MWfaJYDdTw52GMEFkhAjLlszy7ORcIgRrppYEqkAAhjXdclBJxzjDFA8NYDhjEGAECl6kqwsuFUNHn58tXb67ML1lSff/j+3YPd40f3qQajfPXq9fNvvv/6k88+FqBZx2vb9zqy+fjTH3th8Itf/+YnP/3ZP/s//It//b/+u9PTeVUL0+kmiVxHEirQchD6yoL01evTf/4/WhjbTS27nZGugeUyevDkvSiKvvnuLbXcjz/72evzd4A2yDUqyf/d3/7N2dnbQS/8f/y//1+T0dZysbl7/D5GmuRqsZyBaIWQ5C0w6XqzivrdIWdtVZUqKUzDHXY6rutHUXL29s0nn3xEAZKMnb05uXh3itWBYe5Zjt2qVszaoqwhRkJJVleLzerVy/OXJy8ManMpbc8c9Pq+5VqOVVbrpEwhwPPl7HJ61XIRdHxqaHvbYyewbubXz199Oxj1jo4PhWibJi5ydrB7PCA9jBvblrZn372z1+v2bDMAXF8uFl99/f0yuhpO7LqNJVdRtonERsN64HZ63SFBhmkIqJBrG59/+uHezta7tyevXr06OTmZz+cd1wcSqJpRiKosf/r86XK9NCwdUXR5eXlw987O3vagMxwMe4PRSCn16sW3eVGVeXF1c0MINTXTDXzf76Ae3Cw3t35nJnmWrk3H9n3ftl0hVLRJ87zUqWZZDkKEEK0s67pibd1apvf48fv37h01ZbNcLoUQq9VKCDEc9h3Huc0cnJ+f6ro+GAxOTk4uLi7qurZty/f9brebF+VyucQYHxwcrNfri4sLxphpmrfG+rZtb8MNCCGDagSiKFpblmNZVtu2aZr4vuf7/ng83D/YjTZJEAS6TpVSWZZdX19TSm0dM8YgRgAAJRRjTEP01qzR73WrquiGHVabVVHe3T+oquqPX/3h22+/NXVjOOq3bauA2KyXv/37X4fdzoOHD+/eO94djx7evXszn63jqCxLtyYdr7NabXa2t2XLT9+dvn79tmasqJuasYbXiBCaVXCx1vSESynlJgxD03GzLKsb5np+2wqEyP2jB47mxFFUFXUWZ65hOcigkuqatjXaSvJKtEnbNDdXN4EXhn6o6yZjvNVEHGVE17K0VAjePbz3y1/969UmKRsAESAU6KbX7Um/49/c3AAAmqYqZ7mS3Pfd0WAgPJdVSaAHuomAQghjy3YopRjT+WLK2spJXM/zwjDs9fqu62GMc17fTOcANgAD0XIpQNs2m82Gc65RQ6hWKDBbrHVTPzw8MExPSkkI4TwBQAuCwHWsuq4sQ2OM8RZQrEHKdV3b2RoLIV6/rlmZDUd927Z9L0SI3Lb5EUJFUZQNF1wSomGkKynaViAEEEKdTkfx1jbMoNvZ2toSnCV5VnOWxuViud7EadjrT3Z3i7q5uLhQNUdYz6py8+JNklXH949MOygq7noUKBrH+SZKmqYVnK/Wy+Vy6QR6XSSCH+sacvvd6fUla6qyzK8uzufz2SZOqiIXrApto7M7EY1WbW4akzKsiOuIvIrTxKLm7mg3WReX06lg0CC27lFDjwVfN7W0dEMI0dSlaVn+9sT1vN6gHwTOm7evzi9elkXW6z/c2Ro6dlsUKPC8FtobGSsgTItSgtM0jTYZY1yjBiZQ0zTTNC1kWJbe6Qaapr2bvhJQJFG0Wm7atpUSsKqmVK+rJnC9JMkghLbtclaHvrs1Gpum/e2335+enkopw7CbJEmel6ZpGoZVlrWU0jTty8vr8XgshEqSLN7EjmMjaGOMRQsEVwhDCLESHAAIAL51RGJMNIoN3fQ1x3Vdx3ERJJxL1oi2bVslyWqz9DzPsExK9VZwLAWh+t3j4yRJqKZJII7u3T05fZtHSVm2v/rN394//Hww6GECiyLJosS0aL+3LYTMa27bel63SKPEQEkRURPdGXdMG0HE8mJTXUSLuV7XdZaksULj8RZGAGONUoIkkVIRQpCJizSxbDvwO4N+X4h2vpzlRWHZZlEUhq5jjNbLVVnmhqWv4/V8umwZdO2O63vUMrjiFWtqxttaQA5vkhWvb0a9fs8P3528OX9XGxTMo/Ls9PTvf/Xryd5O2B/UTZPnpalbvU4fIQ0ratsWEnhdrtsGBE4PBNlykSpFd3cOMcadINQtsskXgErTNw1XQxRdzudRmgJIiWYEJPj2h++3t3ZkpUbdrfU4//0fv7TdMOgN+sPRgw+fEM8OtoaT7e28qf7qr/4qS68Mx7Y818zSoqgQJUTXlFJ5nmu6Tgg1TfMW5tW2bV0z2+0opgxNIxaglGzi6I/ffLucTyeD7icff/DwweHjjx6ndUxMrJkaa5ukTPOmuv/ek8Fo59//zX/8xa+/+R/+xf/yo5/cZ4r+m//v/8ZZqzBpBATI0SgGrO4O9hhjrVSUhg8ffrpc5FWB37w+7Q5cJawkaVsOBNSpaX/6ox+/evNO1HKxmEnV+r3O8fuPAdVm60g37NH2Lm9UtNpkab1cZoYO60a1DK2r9fX1tecGhBgQYATKsqil4Cdv395MLy7e+Q8fPizitK3qjhu6hqUgtV1riFWSp6v1DGCBNQ1Rarim13Pvv//AoGYWly9OX8w317Zt3xnfTZqsrpmS6PTy7PnrF2HQfTB5RDRse3a/30mLFZ9xquFOJ6CGJnQqAWC8UgBBROJ0TSmVihsmUjJnrdY0HKh20B3sjPs3ywteN6qVQMEkil+9eHlj3xBsaJp+dOe+bbuEEEvHx0f7e7vj9UeP5/M537DFanVwcDDp991PP1eCX994FauW0XI5z2zbBQBWTXV4dHc03PbCIOx01R1Q5jlBeL5aYwAt3YizvBd2StbM1gvOBYCwrmtHNLVk2+FhEAT9fn8xXyZRKoSCEBqG5XmBErJIM9O0MaLrVQIBOT56oAFV1XVZVRKouq6EEJZthd2O5djj8XC92czn8+cvXpiW0e129/b2gm5nFW00XQ973bplCsGyLKMoOjs7a9s2z/OqLD3P832/yPPrq6vlcuk4VRj6vh/MV8sffvjBsiwpheu6aZrWTWk69mDQk1JSissyd4yAc47aVtd1TLBSSrScUgwVqOs6jZPjO4dJpK7OzhGArml/89XvptcXvu83VR4EwdZ4sF6vERDz68teNzg63D86PHj/wf08z6fT6WKxOHt1OYD9fqd/sHvn9OT07ck707Atz3cc34SKybYVnEkV56UJbMPQHacrIFYAm5YrZJmmZRIX/Q7SMB32R6P+yKImr5o0SaqsfPrVN95Atx1Lw8jWjH7YPaUXSkidmrPr2Qv91cO7x0HQMR07TcpVtHnvyaPt7b2qesOaStc1gGCySU7EW9t1RqPRaDTQNG12c311fZ4lG1YXGsXj0IlEGa2BlMAwjCAIodJKxo8O9yDEVdlslqsyy2+j+JqmAUtnrUSIKCVrwSRQmmk4vtfwttsJW8EgwZvNyod0NN61LAsCQ9O0NG0Xi3Rne9Tv7ynZsrZ89fyZFELXTQ2LaLW+si97nZ7nuG2DWykMw6CUVlUTRZEC0Pe6nEvGypZJy9SopmtUIcg1zYAQOI4DBG/qumJNt9/jjMxmMwZVyZqqZRzIVoq6rhlrAQASIUJIm4nVOgZE6w7HnufVtWxbsdlMN5tNVha27VqWRZOkyNPRZMuxrcBzizKnjum7jqYR13PTNDU0fdjvs7qO15vlfDEIXV7X8WxeZ1myWG5t77RltbpexKsUa8Z0vt5E6c11fHoykxgrCZIsv5kuJ9s7vBWU4v6g2x+OEUJxGk2vz66uz5RqbBvm5Xwd1YaBOx3H9ay3p5mUZVWWZZ5RaiqJMFIahQgKijVKEFCCM14BsRJt27Zxmd5ei9u2dV3/FlxhWdZisfB9v2kahAil2PfdBw/uOY737Osfvvnm+6ZpHj161Ol0ZrPZfD4XQtiGsVqt6oZVdZvl9fvDied3WCsVV23NakQwRlJKBYDiCnCFAYESAqWgAhqmhmHYtm1Zlq65CBJRM4UkhkRDSEilGCeGqa3WC8f24jglmLpueH114ziu74VBEGw2mzuH+2WV1027tRNiIlc3b2WzsSyL6BpGAEhFqB52BhCJui6WqzUhOOz0PvviM4whhPAinSIEAJCbTQKkMAwDY8yqmhCEMaYa0agGJWyZRAggoizXsVxHJzqGZLNaM1ZTCk1TB0Datkk0LIQo84oxDgTynXBvxwo6oR8EXIEoS2GWY8iUUt1u39HyOIo+ePJFnRf/2/e/iFdgZwd8+OEOlPLFy+cAo63tbUxRlGLDMMbjLcswb+mYWZss5qsqLd1d13ZCyw4QJN3uIE3TNKl9YksGESKIE8fzKDGytJ5db7hgYcdAQutZo2SWv/72Fx9++PHnn3xR5Ozp02c7+3ewbmANKwhc39/d38vKgjGGDa0FkkOlCBJKCiUVBG3LMcZCCADgf2d43dqxq6JkVS2aus5L3zMAgiVrLmbTl2+eT5fTZ2+2f/YnPz5+cH8y7kugpFK2b+06e/cevffizbtvnr4Ie77h2OGg8/EX77989/38ZspRLlBdtrnl9juhzxE4OT/v+P3lJvvJT/+SIHcxn16cLWfX5csfZgf7H6SF5rkTLq3Dex/kFVnDdUdJQ4OmRaMo+ru/+6VS4P33PnGsLlK4Yi2AGGMsgZISSAAIwkma1qwZd3uGZrZMrDbL9Wbputbu1tbLly963U4Sx8P+YGs8yfP85N3l9s7Y7zjD8XbByiRPAGuE5On0quT17uHO/s6d6XQKpEqz5Ca68i3fcAwn8OI0F0gSk2IT1aKyqHly9pqJyXhrpGCbVcl0cb29Pama3Pb8tMgdI/TD/maV7e/faerVan1dpLM6d4QIHYf4QacbdpJ4FXqhpmlBEHbCgWXYrBFXV9OLs8u//+Wv79170PGDq6ur9WbZ7Xb3dnbHA79QNQAcI/79d19/8NGH//gv/0EL5Nffff3yzeuvv/u6Ys1ovMVlCzGJ00Ii0lSMc1mVTV1VigsCiKFpK229Dtae7WRFXjWN4zhIJ1DDeZmXRjryJqPRYDAYzKfz1WqTRilnbV3XQCrW1hjj22K0ZVmO48CWTSYTKWVRFNPplBBy584dCOFiseh0As/ziqK4Ja48e/bszZs3H3/+OdU1otHp7CZO4uFwKKV8/eUfT05OKCGUEN/zdEJb3RCs5Q3TdbpeLxEC4/FWVhZVVaUZaZlYrZZFkVuWbZp6GI4QQo7j+b6/mk2FkpZldfo9jzpCCCm5UpgScnVxeXN5cbS7f3F++qv//It+Nzw4OLi7d2BRfTqdvnv3bnd3FyHk+/5nn3325ZdfQqmasuINM1yf2i4POlio0Y+2kyRrpdje2Xv2/NXOzp7fHUznc6JpTAmhJKREN3Xd0g3L1HWdlXzrN5IAAQAASURBVMtbWo6UUglNIzar+WaT3BInBWsNTA39FvnJCSVFEmka8R0baxQRPJ3OLk4vkjI52t4NPN9xHCgVJHi5Wm3iqChr33Ety2qZcDwfYtwI0bZtGiebzUaI9s7Bwc7OzmTUR1AJIeq6tHBT13VZlpTqo9FgMtlWCsRRWlWVZTmmaQIAMaZZlq1XEecceGbZMA0T3rKmaYqiABAOBoObm5uGta7rjbYmjhvUrAHQ9Pxh21oIg7y4ePXighJrNJQIws2mUFJnTW4QDWtwvVo9/+HlaDA0TdP3O6vNhjVcWRAA1DIOMTEMgzFJSQWBNE1TM0yEUF01CEHO27IsyzKfT29ubq5bKXzXzuuylQJRMtgabxEdEVrWTRzHbdtWTaspTbPMwWSr0+kAiMuqqVm7Wq1uH8cVgoRg09T293e63aBiyf7O2HGNi9N3iWs6ljEY9oSin33yUV4JxtokiWVde56nBBBCdLvd6XT69s1J3fCDO8fD/uRqtpienDIpMTUci1YtV630wk4Ydm3LreoiDLyw2zENY72cbeIkLzKheFXnnqtTTZei2Gwyz9UwsgHMHr73pKqq66v52zfnq/WKEgshra7aTjhAWLRtXVZcSi4VZQwmaYxNSg0NQ+ZYrmPZeV4CqeqyggpURalTDUJY5tmg1+11wuvr62fPnpmmORwO67o+OzvDGFuWdX19XRRFWZa3S+per7u1tcUYu7q6Mi1bKVXkDcLgdtcAIWCqhRAirDCGmCCEkJSiqkrGmipdAgAgxBgTjOjtOt7UDNId9BSEnU5vMd9wBYSCSZ5LgDu9USvFOlmPx6Of/OmPW948eHAvTRONxcvZXDftwWgUdnpcgsVyCgkOO52L6fX51QIi8cGHjz/8+LM8T9+evAn8gef2gtDDWK/yzLIsSkhDKwCQEIIxBgGCEtY1b9sWSOD6nlKqbXiWpTWrNFOXLa+bghDCGONMAgQRwhhiSAzbdCsGMSRKICEVbyRrBARY1835zdJz7L3t3R999vlmuXjv0ZZtmkeHd4Lt4PzyIskzy9aHgy4HcDq9blmta4hAoIRsWS2EQBBSDesagdDojrcRwq7pz5bJZrOipmGZAQdVUzaUKMvyR8Fe6vMkLfTWngz29sdHf/Mf/+Ob58ss/rUU+M7enfn8y//93//1z6rq089/tLez++7s9Pm337dcFutUNwzWti3nhFJICSJEgxARaRiGbHnb8rZtpZQUE41QQzOhYYHA03SyWS0tQ4Oi9bo+gbJuy02e2qt10B3YroswmM9nQRB8/tMvJMDrVfJv/t1fIU3/P/7z/8EJnZxtwhH95KeHm7UbjGVngs6nguOkMwoHo+75zauiXf3+m9/8X/6n/9uHHz25PA+k4v/qf/1X/+F//8X/6V/+nxnHb95ddyZHPex+8uM/fXb6ewvTvNisbpYQcMsyfdfHCGxWC6CI5BIS4Him61mmNdzeG8SrG865RHAVRwSltu3arn07n+SC/d2v/9YwNV3XP3jyPiHal19++WL5dr5cPP7wvud3HS+4WS3iJCnrtmVQSTIaaoavbWkTKfnZaZ5E0at3byaTLV0nUR43sjm4u9ftDRBCTFRxGpd12o0CJuuWl8vlnBCA/IwQb7VOKoN4ll436k5vGMVl2LGkqHlbdazxsDeAyNR16treT3/8UyEExZqhW47jEUKhkG9fvKprhpXwfTNJ9KurZFokUFbdbleWeLme3SzlbDGPixWi5PDe3Tt394hNvn353WI1L9vKdp24yL5/8QwTIqnazNfT6bQpSoqJZRgaoRRhz3a63W6SJLgsfN8FCGqGEUXRfD5Nko1hWEqBpuRlVUMkLFurK+bYduBZTdMQimzHNDSUponvOHcO9rMsm82mTdtqhqEZBldyk8TrOAIYQYIhwf2wG3a7lmU5nnvLWRJKGobRtq3kotPpDIdDx7b39vY824njuG2YaZpb48mmrhBCjLG3b19nZRGGYRh2K1FJyTnnjDWE4uFwCAAoiiKK1nGaZFmm6zrRNc9zMERCcSkEISTZRGWWV2V5cz199t33mkZWN3PHNn3Hr7yaVTxPytsn5v5wcvzwvclo7Hghb1UUZ0AqSozxYHJ6My/axvOCxWZzNb8RCjesXa+iRsisLISSYbczGA8I1pMyqZqmYxqGYSGosabUgNNxB0hqV2fTuioIhmWWNnVlaETH1HGcgdtRSglWeLbW8CbwnUcP7qqWNTn77POPx92BgfXZ9XS9iNZJnGXJs+dPg9C7s7fX7gjPD6hhYqJphimUsm2bUipky6oaAggwqvJyvYnq5LppGilBp9MJvTJPc6VgVZbLxbrb7QdBJww6nucZmi5aXpZlCclwMCEEZ0kkhJAQMMYc1x0Mh03T9IajTz77vCjr7394nsTVeGIOR0HbNoRaRcVZo1arTIomiRPP7a4q3jLQHfZ9u5tlGQG6rfu6HdaNghC7TscyBWsUF0rXzapKdV03TeR5jmbolGKlxO1NpW1bACTRtVq0qyRqAY/KXEKgMNAo1UyzbQVjDELo+74TQKUUhJASzXEcolEAgG3bmqaZjkkILsosSaKmqQGQtkM90jk42At9Z3VVrRdZcPfANsxGgMlovIqz1Tru93r9IAgdHcoWa5bdp3nT3qzi5SYfbnHDDvp9iog9W0WMS03XDROnRV6kiWVZvcB1wi0IYStEGm9my0WcZJBg29Ydy255IXlLKCII5HlSlZFuEL+33Ql9yxgh0E6vl3UlkiSNNhHFEmPa1PFtlaDf71q2YWgoTgvZSsaY4ziKq7ZuKMJN0yAF2qYOgwAhFCnp2pZoWbxZS8CrsoBYUUpvEZYAgLLOLcuiOsYYKyhGkwFAcr68adrKo37TNKxplFKEEEIgQkgB1TSVrlPDpBAhztq6KoVolVK+04cASynyvKqqWqPW1tZOv98ns9nUcTxKqWEYt2UPoFDbCs45ANIwDM7ZP/1n/3i1mgeh1zRV9PJ7gjkicr5ITQuG/eHVPJIAtlK8O7103MAPHIj01SqdL6Z52oyPDwg2oTI8l0CBhWgpJqbhUqIDpQAACCFKqFIQYQAVwlQXrGXtLSwTmZbe1uCW3VY3FYKYUp1zXpWtUorVAgBUFjUXEGGMEDZ1i2jUdd3DO/usqU1KkmQBQfOP/+mfD7udwPP//a//Q1mkd/b3+12vKJOSNdfT8yQrlJKmpWOi8jzlrA5cpwAgy5K62ACMLd12u/1OUuRlDRQVrRQSIUxFiQCmQ2ei3/HSpKyZ6Lp9COF7955QbDLGoISGZu5ubX/39Idn3z19//EHj4+PmzT/m7/9z4Zh7AxGVyVINhEi2HYdoYCpGwAAICHGuK5rIcpb8ugtTFfXzaypIFQE24ZJKUVJVMhlLXj78MHRxx++//Do2PL8uiJNVeRla1hqtDP+/rvnX3793dlV9sln9/cP9xAVN+tTRIp77w0bZigpPv3T/bi8GnS8hw+2jh/vaW7y97/85t3V71+93SHQWmwum3bj+UThdpMsARJlU8Z5VN7wveMD/BJenpwZJt6f7MbR/OTt29FoPPzTP5dClmVW13VeJFWT61KNuqPJXne9cG/vPa9fv14vN/fv3x8PhpZrtXXbqlZB+ertq0cPH/thwFsJIKxKdnp+7ff8u/YeIoS1bdnUrZB37j1QklCk16Iu2wIAyWQFqZJYlazmQLWC+aHX6fXHo4lS6uz81PGdokgup5edrtftdV3XUkB0+n0D2/2eW6RqtU7PTy8AjCRYCp5WuWC10wu4rpE4SoWkYcfb7uwkSbZeLK8vL03DPji4M+wPhoOerpuhb5sa7Xf9dS/crNZNXUAQuKG9TkC83nQH7sn5y+lieXL19ud/+Rd/9hd/Rl39/PJ6vlleP/uu2+t98vln4/G4KStHsyzNXC9XabwpspwTapuWZZidIFxYtlLKNM2qqoCUUKk0W80XFecSY0qgTgjtBYNBbxiGHZ1qdVm+ffv25cvn06n7+NGj7Z0t0XLdNOI0CbsdzdDLsmS8lUrVdZ1k6S0oJivy0WR0d2dnb2+PSXl+fp4VRYdSQsjy7BxDtH94p9vtAqVcy47j+PTknWEYjx4+3N/bi3h12ws/OTm5DUMYhnFLs3ccy7Zd13WpRuI4ns1ngstuGERRdCvMhRBCCHnTcigt3cAI+b6vhBSsdS07iTcnb9+Ouv3hkyePHzxGlHAlS9aMxuP+cOAGoWmalm1XQtRZThC2DMOwTKXHxDZ1zzk9PUOanmfFzXQBMEk266K+DQZW0TLJ4uy2Mg7NDsa15VqGpWuWphjO07Io0tV6Np70g25omwPbMtI4urq6uLx58/hwL00jbJrLxVUtwPbWEAppUovVZdOUAjS6Qdq2ubq+WCyXf/z67QeP70EILdNuWRXFG6EQwrRmLcE0z/PNZlOUuUaoa1sYY8ZYz4MQUl3XBUdnZ5dnp9eEEEr1fr8vJcjzHCOCMa7rJs/zJElwp9/t+AgqybltWgBKomuO7+0e7EdR4ocBwcbWZFAztYkSwaHl200rTMPu9Udhd5gkeVMXFNM8q6qq5Y00dt07+wdN2eR57lh20eJuxwQAIEgUFIHfq5q6rlnbtoRiSinVkK4TTcOEqqIospRxKXzf7fZ7dV0ijPOqLOrKdX0AAFCwamoIsB8GQacLJSpFLYRQClJKNWoQQpVSRCee50kphGwxES0vqzrlrMYYPnzv/eOjw61hV4NidnVJMUrTBGlGkkRF0RR51u90J6NRmW2WNzPbNLgGnM7Q6aVxUb0+u6LEwFQnuruzF0RRDBGiOiUE123t+bbrut3hYLPZrDcpoWA0GvUHIwCAQgoiUeYRazNNkwQ3jKmWMSnYL3/5V+PRTifsGyaabHlNBRzbMA2yXm00zQDKuAX42jY2jABBDgBkrJVSGYYJADAMYzgcFkWRpqmu6wcHB7qu34aIIYQHBwdNzZ89ewahGg77rmsXRQEA2Noa32IbblPGo9FosZitVqvhsM+aSikAAJQScK4gBBAihBBrCk2DCBEhOGtrBaRp6rZt7u3czbIsjtNWSF0Ztm37oecHAZnObvZ2jcVqDoFuGgZAsNPrZlkhlPQc1w+9759+s7M7fvn6hVRsb2+vyqeT7T1Cjd/+/lvdpAfHdxzXTLOS6JrlOj/+k58+eHDv4uzkyz/8Ltrk9+4/CrbGRZZirHt2QBEtisx17MD15vMbyVWrcQgQoFIpBaFCECCEJMGapinHJATJlhOICMK33hpKNIxpUdZpmgJMIIT/NQKZl6bjIEoIIbqum6aulNxsVqFrvX79YtgL3n98f71afPvd77//7pvJ9taHHz4O+4OmrfI0q6rCNvU8jVmvk8ZyfjPPNmmdFYv5PFrHjme4rtfvwl3b2j845K1Uiq9WM9PAjmtQZCCu6YblDwdNV1UlMyD+7rvvECAEkHDQ/fOf/TzJUsdxEMIXFxfffv3NeDga9vqYKx3ge/uH+XRdFAUkWCem2QpN06ACACAl5O13QQgpIf/b1Ah6rpXnqRDMsrUgCCiFBEPJ215/+PGnnz44OnrxwzOKYK8TeH63KluDgl/++lfLVfrxp0eHR0cKSt2im9Waq43ltLatWF19/OlRFM86/vDhg+MGzB59OHz+CvRGEOoZxUDCVMHi408fHDw4hjr+/McfVgoarr3J83fnb4ooSZfxVbY+efGcUNDth+PBII029++9F6VZURQIi3adVSzNShNrAlIy6vZ0XX/x5vVsNRtno8FgICQr6oIrcff4aDFbBp3Qcrwsy/xOuBOQ1WqeJlldNQogrFE/DIhmQAg1U6+K5nJ6mSWJ69q6rd85Ptgyj2/ntPceHE+2djiXnusbtvXdD99zUQvRTrb7e3vbrC3zMvNDL0uLGtODnffziCfR6hJq0+mNF7CWG1SjCEDTQoq38/kUYzoY9Io0a8oqjuM3b94IroAEum5ijGfXV5audbrBaDDYrBfr1axpKkqhbtJuP7xZTkf9Ud7k27vj5Xr27dOvBFHUoMiA756eCAj+0Y8+/eDjD6fT6fXFJSGkH3SohLyoeMFc39nd2anKsiyKLMuapjEMLc9TW9kASIikbhBPNy3TBRJKgQbDzp393bKs37x6XaSZrutb2yPDMFzXVkpYlrXZbGaz2fHx3clk8u233y4WCyHEYrGwbOPW6XB7RM1msyzLJvt7t/a865sbBEBVVbcbul6vt1mvl8vl1cXlmzdv+r3e/Xv3wjD82c9+lud5URSO4wCEOJcAACk5hPDo6Oj4+Hhrd48xtlqtbNve2toSrF2tVllZcM5v3XpFUXDBSIAghBih1WpVV9X+/n6ehW3Djg7v7+8eumGQZOmrk7dIp/sHRwd3D1+9eV1VVVpWKqswxhomWVETkhNDaxMxnc9+ePFyMt5uRXZyduoHnbZtTc3UTANDkmwiLoVhGK7rVhmPonnYDY7u3bU1K0+Kqqo0E1KqCyGSLIbIHgxDostNqkdR+e70dd20o7299XpRtWJv/66mId9zzECvi3q9TgLXU4DneWLZuiRNXRej0Wi8tdW0arFcM64AwjKH33z97e2TgG24CAEAMCG6bThVvnQcw9QcJdR8scqyIgzD7e3tJErLvMKYSC4t0xS8bVldFtkiypVSBEOl5HjY1yh2XTcMwziOTdOsK/bd0+/v3L03Hm1JhZVEcRxvNpuGC8/zXMefXp3VVT4a9s/fXfC60jWtLphrBiblrOYAYCmwZVpt29zaeRAGrGnjONZ0ijCBUHHOAKSOY1u25lSGRvEmjgAAuq43bV3WFcBAAJXkmWmaGGDBuU4NXdcZ43Ec17C2LNvzPcfxdM2UEpR5WRRFFG0wgZpGMEEYK0NDTqc3Gg9tQ4cQIABHo8Hy5mo2m93MF57f93ujXmgjRPqdruM4F29fv3jxYjzoP/rwsaYZ/ibZRAlXSEnY5FWeF0VRNIyZluE4DsSgG/quayMMVusZY9yyjcFo5LkBIkYruBAiL2IFWJos1qurvCwJ1rCG2pY5rl6zZLGqlaSsUayGUmDXtYIgsC1X1808z5umCUM3CDzdwLVsGWO3yumyLBljYdi5TaTZt9dq13Ucp22b6XS6tbVluc5kd3xri3j37l1oBf1+v6qqzWZDCLmd+W1tT7788svL6eX29naexAAADAmAQkmoBEAEa4QQRCjWMMRt23ImLFvfGo+2t7fLRqV5djOfAYX6/cHW1s5oNPYCl+xuPex3R/fvP4yjpCyZY3tRlIbBSLQoS1nTNNuTe21T3z/+hFLcNI0iB5uFMgzw8O7jmrGnX/4+6E46VrDcVDu9I8CCi3d1lNg7Bz8+vI/KuqSKbY/3r66uAseiWPVCn1KcZsVmXQAgDVNDGEvIIYRYg5hCxiqEkO2bVMOZglQn0pDpOkGSKoqEEAByS0fCgAJw0zQ1w2NcKIgx0eq2besWKAI1BaTq+b2w48XrxfnF6cWod+/46GwxV4Ya39nu7gwARCa2GJCH9+58+cevn5880z1ydXW1XM4vL854y6qqsCxroIaL+EaB+uvv4sB2RxN3OYuBqvNcxKnoTwbhwEya0jZRMO4iwZtrsMzrFy+fdQLf87yzt280CkFV3R33HuyO0+V1Ey9NyP+X/+v/+M3X3y43ESuXdbFSsvTDbgvqzXrteIFlOevFRnKFgMQYQsWlEBBzHaPrGIXeMM2XOzvdn//8Cymy5fLizv74zsFOUb/44/c/QIE3m2S4/ZO4mAZBeLWGOdMeffCJaWkQiSSb3UxNhcDJu5u9gy3NMdd50+lZ3t3x/SePsiK5mn1bFvyTP3u05T8Ybx16Wk9xu+bi3tgGGsvZRjcNHjc7jrtj+Oz6ZuQ51XrNiqJq6g8/+fif/bN/kuaZqLUkKjAiBjERpBg7sgVpQhE0uzpSBYBA67m7WF5/8/XJKqofPjoyQn11Nov4jIQgFqvT5alqUdY2oC3GYdhzui7phpOhDrs38xnjbL5aYlK2rVgtZ67tEQA5p+PRBGqkjGqRJ7Zuuob++vT/z9N//ci2pdm92PTLu/CRbufO7Y+tU6d8dzWrWbfZFxcgpasWIOD+SQL0qFcBfBMkQAIEiU1RxXZkVbPLHb/9zp0+fMTyZlo9RF0m8iEfEonMDMSa3xzfGL/xko2nHuzFpL24u6SOs10ZDZRCmCsZT88+HT0uC7mbVTbpUQVH0eecpw6QsMNcrByXrHavBr17USKbCsxuFzKajEaHvQZ6zrJtmt06IzjvR3Gbl7O7G941QRRuNmsJcNnJV+fXD8bMc5MHp0/mm8XRvXvJeFi01Xy1/Orl17uiqJuuleLDDz9+cP/R2xdvf/+7P+7Wi48//Ojs3v3JaDLsjeZ3M8E50rRMq9cvXquOC163DbIpKPKlbbMgDqWUEFKCMGE2Qw4wbLnY5WkBFcrzosxv4yQMPbftStu2ymLX7/cJNkryrtW+51xfX/u+n8RxU7XDxyOKWZbkvhtSQrNdweGbsqzDOGzqTgkzmEzbqvWDxHPc6dHx/+P//n9zYzfoe+HQUagCtLWVjPseGUfj2Bn3/MvrWy6V4x47nj8cHbhBWJRVlpf9yYHr+pZtp+tdLWDbguUyPTrUjuMsbrdhEBiHBqx3u6yvLzeeOxlP8f0zp67rtJJ/fH33wUeJMvaX3737H/+nvx4OJ//ym39BWPu+mzd5lu56vR5zXQjxYDCudf/qfHV3ezPpJaLaJJb8/Mnxy+fvTCWYl/iWjw2puOjqSnU1MJ0nXYhk1RR3q3kCBoaArpQAaoyhHfm///U/u4x+/xPx+P6ZJScB8FqxXK6Xq807x3GwVNXNZuJETVFKW69Wc8dxHn365PDhPezQuq4JIbF/iAi2bXc2nxdFdvrg7MmzD4qiwES+evW6qTZKmY8//GgyPjicTne7XbkIRqPR3d1dW5eH/X5OMWPYBrKr6pvFYnRwyHmzWC2iOLGD8CRJgpQ7yNqsV1WdY6NPTo9byT/7yQ8W2abWYlNnqK22X5fD4eTw6IR4NNuumAVO7x8eHQy3m1Wv11stJcFWHA8Cx26q8t27y3yXO64FtPE8T2NmOTa1LURlUWR5WTDLisf9VnCMMSXMQLhZ57O79b4jsOACMlsBenu3RNAEnt9WpQdsii0LWllWcKHDUaIFrsoKE4swC1m2hnbDjVQSQiiV0Qb6XmikoJhIDgfBcNKbYAzjOEak+82v/8vh4eEnn3wyOnu6We+Ok+lmszEEIaiffXBfSnl+9z4zHA8GhWUvinw4dFgcEaH90Zh3avbuUhlYtqooapA3rtvVTbXbbaI4OL1//+NPnlZVk2X5xdWNVNf7xQGhaLNZtm3G2wKAllGDiVZCa6mbNteWZRiDEAJtbJtYlmNbZjgYUGoTbBvtSwE4V1pxlxGH1haCdhy1ojKyWy8WvuNAhHebXVXV0+lUKbVeL5nLAAFlm/fCST8a7dubDscn2+12sVgUuzryelVVIU1PDo4kl7Hfr8K2TBvLxRBiwRWvhRQaQEw04ML4QWKARJhSpA9Pzp598DBOgigKVrN2u9z2wh4hNA76948e+n788vkb8vTZY98Lu66BCFRVcXd3FwaxbScIYQihlJIxajEn8CFlGKHKnva57BDS0gBUGwEboUuMgTKlF8RBAJnd+UrH1Jamq6pd1RIhhOt4EGKEiNJKd6Kqag2QUaprRUEqpRSl1LIpxsAYoQGA2GEMM0agBITRIPK1NKRhigsNASIMYoII9jzv4MGZ4EoDKDVIs3y+2JRNLYAeDwcGKMsiXVfZ/vGwH0GKpVbUcmzbEVxajk0YoxTHcei49Ne//se3754ThHbpqiyyJImODw9ubm7WKWOEvG0uob68f3Ty7NEzRqI4otvtNt2upcoIiW3f0Y0la4wQHY/9oswwxh999JGS3X/6T/+pFyd//dd/FYYhIUwbWOSt0sAPE4ycP/z+nyrFgHTbEjX5Li8rZrsOjrBidaEJBBhhghFjFtK667pZtvKHQ4Sbqlje3qRd8/Szz561zVCKWnDeCyOo4d3N7OL8/SiZVFXdi/oWJd/79KPr60sh7N///rdPnj6cjIar7eyff/O72Wz22Y8+tVigBeE1Wtzt2rYG2lGiQ4ZZzLdtz6JOGCWHxyfM19Qzx/FRVdeu03DFk7DnOCix3OFweDefffLJJz//+Z+FYbjarH//xR9/8rMfjycT13UZY/u0rux4vkupTZL+wCKO6MTNzSwrUmnkYNCbTPvj8YHiar1cS67KrFIS5mnRdM29e6PQH7aNYoCMR8e2E6w2y5PTB3mel2XpWp7jOJ4XFGnWVKWdTDzPWy8LCnGtlZSSSx6G4dHxgePZgOJG8KYp86aq6tr1qEt4EgwD38UYjscD33dev/7y229+a9tN3S3v3zt6+PBxmQvfS5DBq7vbJEw835oeTi7Oo+ff3ixnd8Ao3nbHB4dVVTV5iRF0mHUyPXRd13Ecgi0IYW8wrHjresHr168VBMy1dtt0W2TbXZYkycOHD1er1TfffLPdbp88fui6dp5nWiqpOgB1nqfb9cL3/aQXJr0AV0AbKUQDgMIY5WXGOxGG8cH4kBA739VCdJjGp2f3edv1hoOXz7+9uHjPOf84+dhyfIfqvTd2uVwmSbJYLPabAgjh69evlVKHh4cHBwdRFGmtPc/rUGdMVeRVXdaUUteytdZpmmIEbCcej8cWw8Ao12JCyJcvX263W0rp0dHxwdHRcDhcrrdIyNFo9PDxk11apllmAIyTUGrQdV29rSEwvuvMbm8oxW3bKi7qurYtC0IIALAdp67rtqp2u93aGMG5kKZpmpevvh0Oh48ePLx/fO/924tvvvrGdZ3Do+l0fBD78W63k1gfjg4DJ3j75ma73VRVgaBQggshqroVSnZCWgACgAxAnud5oSu1aGV9t7g7vndy/8GZ7Xt5U2x3WdWUQouTe8fb1ZLztkzXaXqw3viu7wjZuVYvciMAgAIQAOC5QRhHlmW/ePUGYsKFKYuWS0GYFRCbMbZZbDAlBwfuZDLplN5fDY0xh4eHTdNWRSmEGgwGnuftdrvnz58PfasoMmNEGPpRHIShK6Usq8zx3CgKbIvWTd7suIHakRwhBI2rtXRcywAXACA4z/O8apoPnn5wN5/d3M3rugUQV1WxWS8Jhq7nGGMCz22aqipzm5Lac6QSURQtZneia1zbuZndaSF7/diyrLyofR2EjLquO8TEcmwuxD5n7jjOPghT5oUQYu9W9/ohY8x1XNG1TV1iiIDWge9rBfYisevZjDENkOM4COLhIJJSOo5rWdZ2vVsul0kYPX3ypN9PPNuBQBmg8nRXN1WabiPfk4B88OwjxlielQBA1/Wapun3B5vNDmPMmC24qsoGQTIcjMfjcV1si7zGiIVRn1JL8M4LAgDwdHK0TXdVVUnJ27bFlAFIOZcXFxdV1WzTLM/zTkitAMAAIoOh1ppDKCyqEUQEQYgwxtRCDsEEGGQMMEYrpThvtdZv371EkCFEgSFGI2MwggQh1Cmr67q2Y1IDIbQQnRAdRMSyLG1MVdW27VDKCLbbRqa7enX3No5jjHFVVUqpXq/nOE4YhrPZzPM8QkiWZfu4suu6VVVpDZSUbdu1bQcMIkTv2185b4ej5Nmzp0nP93zq+ayqCs63TY0vL27zvHj08JnjeJeX10rebLc7MpvdPXkSLFd3u23WdSqOkg8+eJbu8rKsMd7L3QhCwpjtui6CDNJR1zWYGK7aat5synWpWtsKT+8/CcK+Y3t5niOcWxaFotNys97iuq5P75/sshIoiQkiBBPCxuOJMZpiSCnGGAOoEcYGIGopKSXCDDmI1KblHTIUYAS0BhAbbAimxPa8qLd3WhmGMVY2tSzbi4fjeHrQtBwA4AUuQqCpi4qXUejZDKVluS3Leydn49G0KKpOSDcAWuujo4Pj44O82KS7+b3To15/StCkP4h+9IPPv/jii/Y6dCy7yqvb61nqZukm7xoBDGIkoqgz3O4qqKVu65YAP+lH17Pzf/2vfyGl3G22F+/nD84enxwdG03yrMuydRglUtcHR0ee0xsOjoqMr/POskLXcduuk5Q5zFE1SIuNR1wEtJSi7TpNgGvbvue4jmWHwpj2yePB08f3T0/6WpSiLaFRRkCgPCkUb6SFbAIpBjjdZpxnk1G8Wly/f/vGpsx3vMCNGGO3V6um5k+ffWx7rNY6tIdYBi7zi92KNxi0tCpkuisFI0IIajFluqas7cjuhPaisMo4ZEaILltX3//B5/1R76NPPpFS/of/+Ld7YfzXv/71j370g4ePHzFCLEYhhIRiLrrv3r178hj14ilGTlOLzTIP/PXybh35/qg3CZ3gLTxva0mJM+oNkbGVNsenJ+PpSCi+TlfbdDPfzLbpdjxWQovRcDIZTpRSBGEKQVnWt9c3DrPevn6Tp9nJweF0PAQAvL88tyxq2bjsKs4byqhr6bouV4t3b7F59sSeju45lqcVdoE9nh4I9cFvf/t37y9n2Y5H/r3kUe9g+uD6av3m1T//4Y//7Re/+MWD+2fUJlWdr+aL0PPjIAx8DyrFudA1T5IkGfR7vZ7v+244zvLdrsgvZrcawJvbmZ8ED44OVnnGGLMs6+OPP3706NGXX3755s0bx3F26Wq7WSCDCMIY4LZu6qaoi9L37cl0EIbBq7cvFqu1ZTNmU6W7ljdlVTObERsiCKqmgKCL9XA2nwOADg6ODo9O3r55lWVZx9V2V56MBwCAOI73IZr1er0nMO7JPK9evXIc5+zsLI7juq7rugYG+W4AABCdIAhBCClGUnSbzUoLbluMEPTo7IGU3AB1eXk1HA7PLy7eX15ND4+UNPPV8ujk3ocffrhYrZerdSfUeHIQx8lyvXr17vz88iJyw8lk0rWt1rpta2w7SRzXdWlZ9OpqtZjP6rKyLItQkm13UnEIYRR7u/PFSvLj6fj89Zuvv/5yu92enJwgaUbRsBcM66yjgAFD1qscYTMa9+KeQ7CpqmK9WO/SEjELMVRzXq/WECHbs93AhlBJqaeHkwePzo5Pj2vRFF2qdMtFVba50n2IwaPHp1fn795dvnn96rsPnz6DQP/g6Yeu6+6yNMtyPwj8KLaYE0ThLm2UJk3bCgObTgJsO57NGBsDj0vhOA5ESEr5/Pnzt+fvMcabbZrnhWPZxphXr14Bg85OTx89euRjaVk0jL04CqLAxxg2Vfn8+XNK4XAUx/2krJpdlvseJdRgDJpOi65BxmAM67Labre73Wa1WQ/H4zAMH56dNZ3IsuzmdtZ1XRyHQEJjjG3bGAGgpB26lNLlcjlI4r27sJdEXdO2bet5HkBw74ljjEGE9u0D+zIC5tiEEMVlURS77RYh5Fg2IWC73SKEQGwwxnEch36geGcxdnV5AyH0/RBCuFqtACK9QT8MIkGhbTuMMclFURRlllOI8jy3KYl9v+0a17YwQchI0VUAem3Z7Zf6t1d3jDFgUFEUAIDAieq6Ttd5mqbbbdp1nTEGGfLw4ZlSKkCu62rOZcu1ZbkQ4JZ3WgOtAZdKA2gxFwJSFk3Hy7pr67puO641MBBiBJGBXElKDKOMUQCg4kpCKYwxGiOllDESAA0AgBAKISBsMaYQCAAQMAQhTAij1KaUNkXTdDlokQYYIQagMkZJqRzHKatmvdpKARhjBsHValUUmWv5x8fHQRBUVYUQsizHcTzH6Uajie/7Qoj5fF4UlWU5w+HYtouuZVKqpuksxvfAZqVU27aUYkppEAS9pIew6dqmKBrLoqtl4dj+aHj48OHjPCtubuYIEs/ziW3bCCHXdZqmcV0yHg+NEXmx7ToBoW+MllL8dzwgglSQiLCw1w8AFLuaX8xXIGt8T3z2ed/3IqWgNELyrgNKG+E6zmJX77L0WB7n+RYhZFnUtS0AMaHYYsRxbGYRCI0WUipujNGm7HiHkXaYTxgErVGSa4OVNgAjiBjCDFNCCKWOg5mjiNJK11wI2EBsWbZLbQcgxBizbCwMR8wK4ljwpsg2BqPDw+PheAIA2ntrAdSe71IChoMQgNJ30L3TqW0h32WjkfvxhyfkZIQQuXx3vd5cpPndNy/rthST4fHB9Pj4+JhaTGqwWa0BghYmkecTgnrjIaXUZpboOMNsPBoxaj9//vIPf/gjgPiXf/XXrhNXJU/i0b17Z+s3t3Ec+V7Qtq2NUNu2dZnVRdYfJMYoz0O25WEMLJv2evGwnyA35bwdjUaPHj5gDGfbmVbCtu2u7kpQYUCDIDo6xJblAA2ztFBGNk3z4OzeH37/27/5m7+ZTA4IYif37/fCKVAaG7+ppCx54E2aqivLImsw7zzY4C2o13Zmj0Pbc10V1NLM59eLbOsGzvFxf+RTDEhWpJh5Z4/OmMMwJYvFLMt2kPSqqpgt7p48eUQQcl0nbH1KKaKkaaqNAIRYUdA7GE+n48PlcnVxcdN2tR+4JyeH/d5YHIN0XURBcnBwPO4fVBJgjBmNHBe8e3/9q1/94+38Jur5xhilhG87jLGqKDHGdV5s1+uDwUkQevtMqdSKMCaVdIgNkMEYi64jSDsuJhIb47gu3S43b8wrgp0njxxKvbJqe4PB2YOjw+ODf/rHX60Wy3/4uy90G6uz0HN6zx59+P/61R8uL98lse+4NIqCbLOu61o23cn0yLYdBJBRQHIlG2EEdKjneL6C6PJutlyt13laNm0lOub7TdO5njccWsPhcLmcv3z5sus6y7IWi7mWyrUcz3ExhBiT8XgIRz3btn3fowxWdbHdbQaD2A1o13WOb0ktIVJFlSuJs7JwWSCNBBi9fXV+fn7xw89/8P3Pf3x9dZHneb/fF6Lal2GGYbgviNvLCYyxvWFKCAEAwBhrrbfbbaO6OI4dx9FCdV1ntAx81xhTFfluu1FKQIPG43tNVUsp12Qznh7WXBRFAQwySNuuxzt5eXnZdLyu664VSzgry7IsS941QMvrq4t+L55MRlrIy/P3CINHDx56nvPi5fP5fO5YtoF6sZorITnnZ2dnUlQnR1NGIW9aoOXbly82i/XpyT0G6Wa+fcvePHz8aNgbKKDTbVY1dQNbP8SeN7Qslue5UNxab9w49GJ7drdOs51lWZ3mRZ25gd0fRH/5i19oYHblLi8zBZoosbHVkaJrmy0lfDQYBv6ji/P3l1dXfmT7rmdTx3eC2d1yt87jaOBQr+k62sjp9IhZbpoXlDhNm7dcQ2y45FQZzvl2uxVS7na7LMtI01qWRQjZbrdK6DAMgTaCqzSO+/1+0guMMRTDMAwoQcNewqxpUabL5dKy7SBwxtNBUVSuHxZV6bpumuEsy5TRRolO8aZiTde+ff3u9mZ2/+GDwWgYhmEY+vtCDUpgEvlt27quFXqWEgenJ/cC1y3yXV3Xk+mIYeI4zoMHDxzHUUJijNFy3ev1/DAUUioAHc8llGJMIcFa6zIrhBBAGwhhHEZBECkKlstlURTQ6F4SIYTWu11VllqB8Xj84MEjqcC7d++FMgihtm13aZUkSV1W8/kSKP3kyZMoDEXbvHzxvGur9Wrm+W6V7yAySnAI1HK+autOKZVlGQSYMbZvWYxjxlsBDaLY6id9SimldDqdOo5/c3PTdAIAsNnsyrJCGGsNuq4zEGBKPBpYjrf/IULp9XIhpRRSAgARJdSilFKMIQSIEGQxAIEyquNCAw0RMELu6x8VxpAygvckMaMAgAbsjekSQAKgAshowIUEXFRSaA2wbfsASi6asuJN3RVl03VitdpJKZuOp2maZVkvjvI8Hw6H+yTkZrPRWhdFcXR0tCc37x99QggIoW3bgW8bY5Q0xhiEiDGmqqqiKJQSWoN0lwMAwL4xrG2SxAYKffj049FoTCltyy4OfNt299A/+vr1m4ODg3sn9znne5dWHPcIpnWTb7epbblRlDDGuq5br9ezZeoHthtOe32P2bOyflNVlVD8y2/eDvrjwIu1tqidSN51XDIWDwauMcYL3KarDVB1XVV1rrXUkruuEwSe61iUYoQQgMYA0/KmaSoIIaM2sbGjbCWA6gDxLKWgVsAgrA1QStWiawsRDiMDtdLKSE2RwZQiiACCTddCbEulMaYAo7ptlDLD4chx/cCPgtCzHcfyrKar0836/cW7Mt9MRv0iX89vO4L1YBh/82VFMXJgDQRQZGcFRZVtF9fvm1LZPn4Qnwz6Ewjx7G4BgcAEKVWV2Xp6OJnd3O2Jtp7jZ9tMSSihOTw4+fabV989f/k//00MAbq5vWWMHR8f1kBJKYXIGeahzYluE588PbuvlFC66497jx7dDyMvL3dCdp7nAaaM8QHQ795+KaWeTg+m4wMtdFV2UhhqW1HoK0GKvEIYD0Y9zydXV1cHB8c//1d/9vjpE6MxZW7dyidPP6ma3PWSXb6RhoRufPv25YvXL7jmFFLUwoJInyTHB/d8328Nn9+sb2bb2+XtoycPlWEHk2mWrrbb7aT30PHsw+OD8/Pzoq7OHp3tY9+2xTBGGGOXWMJ3McaEUpuSa4hFK4SQ/WTw+fc+dxzn4uq8btLVYu25VuB6cRCp1nRdVxclwbZjB2VZNnXnRw7nYrfLhJDDwbjMy6JMjRRKKd7WSRRDCBmCSRQCrQb95OHZg6aq/vjHL13X/Zv/+X/TNCXGuOGNVA0Qsi1KDJSNQcH1119+dXV52/yi+/5nP4z7fttyYcCTp9+LwsHt5fV//P/8x826fP/mP5+dPvIcazToNW2VpqlFWa/X2yw36Wa7StM0L8MwJpQ0TTtb7q7vNtHN6t79U3ddHp4cK0gs27+aXQdhPN8sXr15hymq2sb3/cvLy9VqlW13R9NpnhfJMMnTbO//11ozhl3XhUbVdf3ll186Lu26jhBSliW1ieu6DDs2cwi2mqrmHWCY2LathLSw3TTNd998y+v23/27f9fr9bq6iYOwyZq6rs/Pzz/99NOmaSilVVXtdru9VTaKooODgyRJuq7bt0G2BccAi1ZorTFEjJAoioCWUIk0TSnGAOg4TGzmYIiAhpbnP376gVJKawAxklLu0vzy8iqIItd2LMtSUvOusRgLA9+1LLs/mN3cOq5FMfniiy8sy/r4ww9Ho9G/3N40TUUpBghyKcoik1IqI3/xyz9vykYDVZcNxUhyEfqBRahRGmi1Wa1H4/74cLJczKu6OL53b5PNbc+GFBusqAfjUXgGz47OHtQNbP75j7v6CtsEUSRV0+uPvv/5p+Pj8atXL969fys1DyOXWMRxIISkabZpOmua3V/82Z/3+sH9s2ObWQihbZpjajUdr9tGSN1JIaXGmBjOwzDGxBZaAQ0RAKLr6qphGhoIOJdVXRtjRqPRnjrw7vyi1+u5tuN5wW6zVZYxxuR5HrqGc44h6LpWKcEYeTw5Ozs746LFhBBkxqOhbVtSqkWVRYFvO2S9adq2tS039P3IDwghQRAkvV5TVm92u+l0+vSDZ8+ePFhu1pTB0Pe07IAWmNLRsH/v9DjPdsYoA8x+/aGU+OyzT33f3242/X5/OJoQRoVWTds6fmC7jtK6qppk0K+qiuwLhBBumsayLMdxFAUAgK7rMAR7BaKu6yzLDg+ODw8P79+/33ElhCrrtm6bV+9fK4aLoujqbr1eH0ynx0dHo35vtV4u5rdVma8W8/ld17V1rx8dHx8Ggbfdbp8/fx4EQa/Xq6vWtu1er7c/QQkhURRpbeq6Rgh5njeZTPJKNp0WQvu+F8ba9T1CSN02XcsxxpDg/WnatSIvi6qqQAUhQhhjBQzEEACtjNRaAa0MwAYAZKSRAhhACUEYYqOAUQZgjBEllDIEodlng8E+zwahNkZILZUAAHSdJWXbcdV0UikjpW6aarncrJY7qYBlOZblVFVVNS2lVAhx8T7dbdO96VhrvR/rEUJGA6NBr9dDEGtltptd27ZKqfGojxBBNkIIWZa1b7/sum69XhpjICRSaISR0UyKrm1UEiUQwq5p8zQzWsZRoJQqix3Jdq1S2rZCALDWnDGGELBtwpi12Wzmi2vbciwbO26f82a9Xvz2t28pRUXZfvDhg6rRZS1221IIvP1vf3j86IPT0wf9ZOCF8Xa7Xu52TVtbHhyMEg2l41sYgrop8zwHRiMEpOFtZzpeYowtmzLGCIK2bQOALGoBALTWGGOMiADKYrYBGCECENHAcKG6ruukqLJKQ0AItajNLBczpiEyxtR1XVWN5Aoh1JTN3c1MSs7Yn3zdtuPsFRghRFlmdVGGYfjTH//k/N3z5eKGd6XLcJtnnueV5R8ty/O95NFHk7o0F++WN++Xm3qBHBP0XGBg2LluYO3BtHWdv3+f313fdJ0Y9ke9uB8EUZEV11e3jx8/ffDgQZpnCIHtbrVa3/X7/eEo3tbzbFdXohqN4uHwHgDAd73T0xNqo+12Qyx09ug0SvzZcnZ5dZGXWdasHceLo57nW7ttUdc1F2rQn9gOL/Imy6soSvrjMUIoHvY9zyuqeWfay7vLn/7Fz169easkPDi5d303++FPf7ZcL5jj47Zkjj0+PMjbohLN+3cLyaWoa0g6KSVCmlqEMpaXTZbzxar0k2K1LW4X67LIkAFGBlEvefD4wWw5q++ug8Cr6hJhGARBXRWr5dz1PaWUUgIh4NiMEczbpi4Ky7KePXsymo7evTu4W1xSipuqXixmvOHptkwC3Q+GTuQiRhzP0kYul/PtdkMIsizWtfV6nfZ6cT9JNqv13c1scXuXRPHBwUFTF5xzKUXHG601F8oFaLMr4jjkcn5zeSdEMxz1KKRQ6tX10o8eKaHP377bI60++d5nzHbKos3SKukdhN7g8t1dEve/+N0Xb7577gfu+5uLqqh7YS8Jk7u7+WKx9N3g/oMjYTBXqJOyVbBp9d3dTKi7Rd71JjtsWVwoYjvL7e743pHrBUILhHBZlhDCd+/elXn+9MkHH3744fn5+cFB/8V3z6u8QgATSByGjYFl3qzXq2+//XY6nXz4yTPP92fzOylALxnUuyawPYws3imojG87NqFd2ezKbRIGgzhaLWY3l++m02nsO7Kt4ji+vLz89ttvJ5MJQmhfobtYLJbL5b7a+ODgII7jN2/ecM6TJBmMJ8aY9XJBEHZDz/O8wHOKIsvyXdfUWgrOeVs3cRz7fhjHvS/fPQ+CAFNmpHQdP4hCP8iMAU3XRZFDCGt5ByE02oi2ybYbRmyEEMFwsZ13XfvDH/7g8PDw5vaKEDQcDgHQ221TFBnC6OnjZ9/7/mejg9Hz588N0dRCUksuO9tmTVNhAqM49ENPmfZ2/v7L519gis4+OBo6DqakbfNd0WkAmYdOk6PB4KSt4S4vrm5vFptl4LPDo9GTp2cff/L0+asXl7cXu2zjuFgCrHgtdYOIIkZx2XJRBIkdJ4dnD054J4UQxV0VmAjbhHpOp3leFlrrHhoWReG5QeA6GsBRL9Gd4Jzf5FVeVH4YAGD2wVHfYgaitm0/+eSTJ0+eeo779u35P/3DP2JMOOd3d3effvSviiKry6puO9G1VdVgjIfjkRBCKWUgCFyvqqqmaeuipIdQdHVbF1XZ2CM7Cn3XdSGmp8f3/TBoeVeWOQCatw3GyLWplt12syiKQkpJKcUYp9vNfH63Xa2Hw2FRlkqpwfi4Nxxk291iuWSWNegNpVaqbWzbjnp9Pwzmi8Xl5fVys1ZKyU7UdS2lNMY0TSOE2taZEGI4HEaB77m26DjGeK+gpGn6zTffNK1oWw4Q2Ww28/kcMJZu0n3gJcuy3/23f6EUC941VYmR3mxWu+0SIk2QSj562lRZke5ePf+u1+vRJ0+2223Xiel0Sim1LCf03SQK0jS9vbpaLBYY4/ntbcHx1dUFxng0HhKCNDCct5vNZnww5ZoDjiAiWhsu+J8eWY6jlOJKaqAQQpgihAEAyBgAEdB/+jDIAKQRRAgYAiEwxhgNlTKAawiNNoYxCwCwZxlBCI0G+4wPAMBxHK27XVZiZFFiI4SqqqjqmlDatsYYU5Z506i4Ry2LQQ3bli+X6z1YqG1bSqnv+zc3d9ttur9CLJfLqqr23gUhNMbKGCmEgLB0XcdxHMuyHjx4UNe167pKCwAQwYQSBYxVN3Vd10JwTCDnvCgyraXneeTTT36MEOh4s1wsMIbDUc9xrO1uTRlGWAPIm1bs0iXCxrYcy4YYo/ni7vlzlfRshGEQOnkBWt4gxohDFQQlb7FjIdsuOZ8vl64rp9Px1dW553lB6CECMAGWZfm+a1sMIcBFa6QwBmijhYYUMddhFmGc67JqLOqEngc8XFccEUQYpZRChIRWuAWoNS72EcQQEwOxUaopKy4Ul4IxUtcVMiByQ6naPM2ZhX3LsSyraSvbtdbrNbHJXsDp9/u2RRxmH0wOjRS8dcu8sShk2DKUl7xzAtfteXboKGBVHSja5mo+98JhEg7DZEwJ4U2bpqlW6uLi3e/+279oDX72kz+zqbdIl13Dbdu+vb1u2/p73/uorHbCVLPFRV4uqEXO7k03PquK+v7JvU8//mS/nOv1ekf3DrfpNi22GijJFUWUUUtJo4zEFPQGSa83XC7Si/Pb5TpLekeIEsRMpzpIWW80RAh1XZfl+eXt1eXt1W5XdMpc3t1YzL+ezRHBk8MDgVTZVcDGfuwRj45PxsPj/mCQLm9ud/N53/enR1M3cIlF/Ch0vNC2437/WEO7k+LqekWx8XzbQL3LtrZrYUr8MIAYUsYGg8Fw2JdSLhaLnoiVUlVVWZYVx/HR8WEcxYSitq0Fb7XsuKibuqIkQQg0VbFarKu8DtyAWRhjmFXbrmuk4mWdV3UGoCqyzXf54vhgOhr0JqOxqFti4Hq1ybe5Q+z7Ryec4sBzsu3u8ODen//5URwndcMhqCn2lSCbZUkxG/Rjx/F2q5xSHgeRVPrm5vpffv8b6rJHjz9gtgUhMQCVVdfrDZbLOdC86Xauqxzm7tbpxdurvFctF5uqEf0kiHoDjdg6rzfbDDHL88MGWfP1Ss9W7maz2GwhBq1smpbv8kwjKJS2LQoVlhrk67VjWaPxIAiCR48eRaF/eX6Tb+u2kQhI1/YhIMZAjK0w6B0dHZ+ePmx5Y1mW7dpR2IdVzogFIUZQY2QwtJBGmovAcT3LVvdPL9+f/9Pf/eqHn3//yZMnzCJt26ZpWhRFnueTycRxnOPj436//+LFC4zx3gzFOX///r0xJkkS2wkAALtdlsSh4zjQKCl519ZFunMch0O9mM/fvHnz/e//wHdch9lv35wnSTKZTAajoed5Qoi27TDGw+GQMdY0TbZL0zTlUtR1HYTe6nYzHA0sSoBWn3788Q8///z2+vIf/v7vq7b5yU9+FIZh2dTq9irpJX/2F3/2+eef/1//n/9+t94kUeK4roWY7dr9pD/o9xaLmeWQMLGhpRbbu1bniR9lfIWtDhHaqbxsCwWJlMghBFnw9PDe93746Ys3rxe/f9l30EcfPz48Gt3cnn/x7ReSdwYpBU3HS0IBNNwYHgaW6/iYksXydjo9xJgYqW5ubj0V3Kxn6+2qlW0jW9Fo3nV4RjmXgetSyhhjdDqNXXe93i5vb15eX48mY9f1jTGWZSFKpDaWZdV1fXV1ZZTmXN6/f/+DZx+Fvs85nx4ekNVS8LnRHGMKDNprhxAiz3M474SQdVFJIZTQCJGuy4xRBiiKEGOsLquiqm3brsuK2dZwOMQELpZzYxSEpusa6nm2RRVBWmsIdFUXbVfHSWjZFGHw8PGDp4+faA12eVG13dv3FzdXt3EvcQOfWda+g2avRd3e3oRhCJQpigIaMBqNMERZVriuCyEcDAYWJV1bp7tdURRKSoRQnuez2aLtJGO2gbjl3cHBwbYsq7KEELquu12tX373bVs3jm0lvaBrcwRVmm4di+RFCoB68eJ5lm8h0hBpypAfuOvNRXtRTiaTARu1bVmWadvWAEoAZVWXy9Vdye2u65hFlZG+G2ACq0pRB603M84lgNCybAhxy6WUGiMCMDZGGw2AAQZqA4wCAELgOpYxyiitlUHaQAClMkoqA7kxBkCjtdRaIwwgNABoKZQxBgAEIUSIQAiVNEopaluuaymZd+2WYOE6keM4xhjbxgjTrusApAhrAAHCknNuWcF/b3KSUlqWtcdpb7fb9Xq9V2sEl8yi+7kEQ2RZFoSwrPKmaQghvu/t3/Wcc611nufGGNf1m6YhpM2WN0oJxlgYuUqKMt9gAoeDkBxMj9++e/33f/93jJH7Z8eUkSjylFJVVVoWi+Moz/PNdqGUODo6GY37n3/f+eLLGkHJbNMf9IvqWEHe1N1kOp4c9imF22KDbURdakeO3sBttu6Pksuby9Ggb0CijVSqo8wyQGJCLYsSakuJCIJ7WMRumzuOAx3S1F3bdq7tJ4O+Y/sXF1cYE0wQItAgAKRRQAkttFEQQ2OUNloDDBGmhOz5x23d2A5J4pjzOvT8KPYpQdvdWmrhBwFXkkLq2W7T1K7tFeXum2++Gw6SyfiAQvPVF79ze5Hn+L3x8eXFzdXstq3nDPcCZxokw3TVfffqnedO2VnPcwIjzXqz2W1Sz/M450EQEcIIYcaYMAxRiCml33739WI5++Vf/WXZlEiKm9v3TVM9ePzgL37+Vzc3NzeXV4PecDo+cJizXq+7WuwWqe06oZvczu/W2aqVLe+A4lgZzYV68ep1VX6NkU2wy0X+xy++noyP+v3haBwwxoQB6WZdlNlisZhOe+E6AcT67sW3P/rhn2NkzzeLn/70zy6vrrb5DmAV9B1DwGI1X+1mcS+ejI9Wd0sppWVR27baruFSKEgMQE0rEbWrknuBvd2t4tgFDd9ut47j+L4fBAEhOE3To6Oj73/2aV3XTdMAoLXWe5AXNKprKtRBSmlV5Ntdend3u8u2y/nder3sD/wkcm3boRQTgi3Kuq7J04IzVFUVtUic+I+fPKi7FCJR10XTVldXV7vNtsqLfr8/nUx40/Z6g8Oj6Wq5GQx6GNHBYIAxOzo8Kct6s1p/9NGnGMLf/Pof16s50PD05GAyPLhZZoRhizFEEOfNaj0Pwtj3EgiYQ70gCD799NO//X+fB4HrO+bk+PD49OE//MM//P73Xz558ngyObh/+th1/bLqBNe3y/XV9SwZDs96E+bHYrndNSLPivdXl8f3jg7uHT5+/LiRLdfteruu2woRAjHebrdJFG02GwjwvZOTtuVV1eR5oYSsizrbFocHB7Lj08nBZDr+4IOnbmCvt+soihzPzvN01J9oqaQEWCsGjVaYQMowtSwn3e4OxiPR1t9981XoWgeTAYCmrLt9pH4PblJK3b9/fzwe720K8/n87du3Wuvr62uM8cXFxWKx6w96ts2ePnlkWUSJzmhXSukH7mgwnM+X2+32zZs3Tx897RKulIqSWBnNpaCUKqU2m03bduPxeLZYWJY1HA4ppU3TzGazpmkwxsPRoKqqruvCMHz27Nlsdvu3f/u3y+WyP+rvo19S8p///Ocff/zx0dHRr3/968VmQRANotBjbuiG2S6ljDienfTjvNh2krmU2S559tGD3rjHTaWqLGKx4yFNHIisvJLaiE62ButHTx/+8EeflcXu7PT4pz/9SVVt//Zv/zZtpG0zhDXPuVQ0TjyClWi7IHCrOq/z9Kuv/zCdTi2HZUX+xy9+/8Hx94rbq9ntnef5g9GQMtZ09e3d9ag/0kYajRlCNPAoBHVWiLq2bZsxtr/ttFJlZdF0nDE2X6zW641rO3HcWy6XbfPH0WBwcnLStLwsmu12ZzHqexaXcrfL6rp8++Z8ejA2xhBCN+sdZrTrOtFxRnEYeAAA13UppevVdrNLh4MxF7LlHcJAA6WhCUPXca2mqmLHiqJkvx3YL9dtyo6Pj/f+1sePH4dxdP7uoq7rOI5vb28vV5tHTx5T2yrK8v3Vdcu75Wp1ezv78c9+Oh6Pu7pVSlFMnjx5YlF2fX27yNb7LrG2royWFmVBEPSSJAoT27aVMtogKfXN3ZwBc3rv/t1qdXt7u9ls8jw3UmGI4jjuJSFva4zxcNCvqsxmGGOIEbq6fM+75uhwOhqNzu7fo5QCrXe73XQ8xphUZT6fAQhhHAWUoNVqJXhb19JAYCDQWtoO833P9ihzyMXFhQaCYGbblkFIKKU1oAyNozHnXcs7ITplpIYaIgOAaTmHRgFtINAQQaMRNMYYo7SCECIEIUR7ohREACGqlNAaaq0Q3OMRCYQGQbIvMTYG1DUAoIqj/n8vDRaiq2vFmKCUEEtggkQty2LnBx5jbG9BTZJk/y6+vLzcDwSO44QhZYzVdX1zfZtusziOPc9pmqYoM6VUWTqu667X6/3Nra5rpRSlxXa75ZzbsDVGYQK1GQyH/enBsK7LusnIV6//+esvvyr44nh4AHEzm63ndyKJ4iLP27bth24/dtbrDYQNxK3veZNT8SFK8qq4Xb+sTOyPw7E5efP2slTgar25uvxKStnvhQZwSrSEW6DFxfmb+/eOxuMx2wccgG7qajAYbNfLMittRi3LcimTUm6yDSU+MQExntHQIGP5sd/rM8uK2ni+WrRlbTnE820nsoDTNWrlxtPdNgs8r6saKbVF7DxNkyThbRUGVtM1eZlJLThGRw8ftW0rUA4AyLocM2NZRHS1SxCvq7v3N+UmJU+tX/7ylxDC3//hQph+1UaJTXqxebu61gJlxU64LjbuydEYacsA5XmOzdjV9YU0Tdj30zS1lPPo5Mnh4SFCSAlBbda2VZgMHzw8efvuxe9/9y+ffvrpu+dvZMEPR4ekJbVZnD4b3H86qvJqmV8nYWKHuCiq85u3hNFWtMvtarld5lVBLTKYDpevbtd32WA0/O6bl3lV/ujHPzaGC7F5P3v713/1P8b9seRScsUQXt0sFjezvvfgB9/7ya/+4R+sJEaJm5d1FEd2QstXm2/++DuM8YcffrC52yy3m2iUeMQDUA+G4+1sHfiHjj3JUxz4ngTGd5LdJuVGDA9GSEOsKdWeyLsrOQcuXqxmZ4fHR3Fvs1qktzdD36XYGJdeza6vFtdn906jnre4vZvPWiSw69kYd7/97X+uu6KqijCCy00teFY3fp6nxkBIyWx7i210e3vbH8QIIQjspmo8G33yyeMHj446Ib799tvvvntxd7ejBPai3ve//wMMAjucwE7PLm8JQI/unZ0cnwlpCESSi7/7+1/906///n/5X/4PxMfffff1cnN7kS5836Wesim1tA01HjnDRAV2iWDd+iH1lMI2Xd5uBofjgweH08MJsZiab7q2/vbb55PR8NGjZ/dOzr57/no+f6MVbgs+TCaiNsvz5epuHcFIbaQVjC3SCGlT4kyP7Kv5RZoW8TDK8lIjUtXCDwYQ0XdvL7ab9Ob2KluUQnSubZNA+yERbbVJb5N4jKmjhLVdyWLbMis2qr18f0cpYQaHYQy0qmUJKaEWFdoACAHR/jAWXYcYsV1/Nlt899W39+/fn62XURSdnJxADN68exVE3sXVOcQIEPP++j2m1m63u7udO55nU7uWdT8+AEqv5jtRf/fhR4/6g7Co8jBw752dtFV99vBUKP7HP3z97uqCaz2ZHIz7E9tmQRAYobO0aMuGEGxhMoxio0S+WT24f99j6Ordi67cGmPujz/gxP/q268+/Pij92/fv3j94m657A17kLLv3ryaTqeea//lv/kf4iD8zW9+85vf/Prk+NBxnPtHZ9CArqntkPo9R7B63c0LUTy896g37lcz2Rv0mGNf394E5mh7qzRgEPqIkggRqdH2avvsxMd96y/+/OfHx0dB6Hg979XNi4vNHQr9SrUMUtdmAtG8Bi5zGAvaCth0lG8WBebzm+Vkgq6vLiiB0hDKfIgsKTRB1KNWDZDhostybzBOgrgqW4hp4PaUuOUN0Fg0vGAOIrZlCaMBtqilAOz3fNf1KbNb1Y6OxwjiFou3s/cXt6s83Wne9mPfc2yjQFOVwzhYhl6+3TqOk253SS+aL9euF1DHv2f5B4PBq9dvJ8NBVVQ3Nzfa4IMjywn9MAy5qIsic2zquFRraduWBd226JIw6tqWt11Lqml/6FFaFMX9eydI8G//8Ns4jnfZwrbtzz44492plJIqfn17udlsAACru7lNcATVgW8XUG4swCihql2sbt++er7Jbge9PiYUgBJDPEmS0I+klB4FFjHQwlVVLXZLwneMki47T9wwZ8od+Eaq0XCIAAxcL8/zzXZ11Ot/8tFHNkLvLy+ePf50s6nu3XvWTwqIjEWJ6Frfpj/90afL5XI+n2PGLNdb5WtAWCtMWjbKcrZpRkhiuZ4CquKtZbFnT+57Ft3tVpPIfvn61SbLuQSAMG0BaONWSwM6AUUHOo0EhJAgCCGCRlFsKSUkF9oYgABAEGhgkAGIGaMAAABqCDVCAEGAIKp5tReQOtEZyQkhEGBlVAijqiw9x/UdYFtOuskf3Q+gIYpDAEgvCoGBsqs9yoBgDgm8Huy6bruper3QaCm5GJ/cu7q6sqgtOsE5BwBJYkSnESJx1MM24KZDEjLbspWXpnletEajqqq0Ep5vR1FggORN7fnO0cEIVE1ZllHsQ2iy9SwIXZdCYDoyu551bUsBqvPiIs83m9W+5LTX6xGCDYQIEz9OLMeJe4PheAQcvNguitV11my3VWi7ftspy2Uv3nx7cX01my0YI6NB4rk0CJ2iTE97E621AaKsdhgh13U9z/N9v+uaPUnKtm0MAcYYIGK7PsFB3fGmU5yLtuW3t7MsrbTW88XddrtWSvqBGwRO0osJQV0p3i3eB0GQ8txxvLatR6OoyKv5co0QIgRBCCmlbdV0dbO3bhlDtJZAAWggIQRDO8t2m80mK9IgdPuDeDiKsyxDRChTN53JspRZ5OmTR7PbVImmbsrD8bSfTM9OnmJg13Xd1p3FHKfnaK0pZangXdcRArSWSpmmEeWflDQPQLzZplqBn/30L376E9M0ze3t7eJug4HVi3qagSIt5uWCMXu/xvvuxfP3V++vbi5vFjeHxwc/+umPemEw6I/fn1+tN2kYJFEyWK8yY8z9+w/+8i9/aRGrbQQ0iDHbdcF0eug7QVGki9U6z3Nk21988UVZNT//s58nSdLr9Tjn2XYXRaHB5mJ+ra/A67evjCQutZmFwsiFQDd1QwlxAn88Hn/wwQfvrs6FUPfvP3QtuywyYLDrOkYARm3PDbxo4DnOd9999/bt27Onj6si3+12TdMgoHthHAQBNGCzTZeL9XC4Pj4++bu/+89VUyIMXNtr6267yizHVdLsdiljdRj0JpODrNhyJWnXhFoyx5ZaYEqTMDg8PoHE8oOLi4ur8cH0bj53mP35ZIywZTROkl4cDcqqpdRKkv42LaeTw1dvXr548frxswe9XvKP//Sfz8/faqHtyHGYXdetFtAJ3LCf9AaDMEwwolrrpmiMRmHQi5Lw5OgkiHwwznuj8b0HD30/sJhzu5x1smOOfXszBxjytjUap9n2cDpsGz6fz7XnSi2qqrib30Iqt+UaYxTGseBGaaMUhEbt0+cQAiXdN2/e9PrxwXTU64WebxOIJNe80+fn5zO2IJg9uH9mO7QuW621UiqIPEypQ5hlOwiRsmpAJ2zbbqpGCYmA6ff72WhwfXn58vVrx3GYa1NK4zg+nB5ooNu2BQCUZbnZbPr9PqaW47TAoK7jECKMUJnnEJp991IQBIQwAzpKrSSJRNu9e/f+wYMHwBDOOcZ4t9v4vu+6NgBgu91WVaW1rmuxXq9937cdliQJpfT4+PiTTz751a9+1TTNLbh5/OTJL375F4vN+ur6vD9KolF8eX1VtwW2hs8+fPLjH/848P3/+B//wz/9wz96nne3mj1+8NDy7LIouOZe4FGLFFURBH7cC0eDPjBGdbzK8qrM6zS9m99hjPrD3ngy5EIsFgsDcNybXl1dEOp4nnv//n1mIa4bSmmv11tVJXVdQohRWmtgWZbnOgRhybss3Smj5/Pl7e3daDqlxDo5PsWAGAiCJCbANIILoF3P46TbZDtvuRBSI0gD27Ftpz8YnD48W3y7wcgyGhsDMWKeaxNCEGEPo5hRG1MitdEaGGO4FF0rrs9nhBCl8G63E21J4PT0aHJ6elo31bt37wghR0dHYdIbjrP1ZjccDpFQuyzv9Xqe5/X6o6+fv6jruiiKycF0j/8jhOxXBnEcjgZ9ZNh6vTbGuK7ddc18PuddY9v2YNDPsqyu6yRJptPpZrnaK0Cff/7ZarXKsuzg4KDf71dVRSk1AHZdt9lslNG9Xo8Qkhfp7e11mm4BBHu3nW3bDrXCMHRsqyr1Pp27V5Fd143j2AsDy7Kw7U4GURgEFmW9KN5t0izLLt7VDcOBwxxK+lHYDgcUojLLm7KaTEdVUSol9m5/znld1w3nvmVpBSDEjDkGKVJ1kEDfdZbLbRKHEKPFbL6e371/9+Lp4weT0RBihBm1HNt1XcRsywBhEBdC8RxCQwgCmEEIjdFa632z9v7vApoAqAEAxmgAUNcpAxQABgCNMEAKYIwgUhhbABCtgRRAa220RggZA5umCYIIAhTHITC069R8Pt//Vzn/k1dx7/kAEHLOjVZxnPi+n6bpZDK5f//eZrPZ50ootRhjAAClzB4UTQipqsq27T3hd9+q1bZtWZau62pFADBt21KG4jg+OJwcHR1FFN/d3SgtiiLruFK646KWihNeVB5h2Pa6sq2qQoouSZLhcMQ593wPICIMgpRVrbiaz7mBs+1MAO4nHsQQYFCJAmFrcjhM+oPNOm1FTTGyPDIcx6f3D4FR6fWs67qqEXWTAQD6SW8Pq8rTTGuNKVXG8E4xhlzXH1qe6CjnHAAgVbXdzFerjRZSdBwhlGc7IToMjTHasZllWW1b+/0nP/vZzxAhvh+sV7u8bOuOL5brKA5c16WUGIDatrUsqx/3J5NRq9K6rrWRm01V143NcF3XLW8c39JANLzIys1mN8eUY9p1ot1uNp4bJnFvOS8ohpriw8PpMJliCLXS282GYHuQ9LQGN1fX8/kcq63WuqwwQgghQghBCGktgyD4/PMfCiHGk8Pjk3uWZe12mcHWPDuvU+kzwLBLsKzbyhjuee75+fkXX/xhuVm2vJFcxFFwcnDo+s790yeOHb2/vJjN5oobzws++uiTH/3oR7yT5+9uLEIPDw6URDfXs9ntrMzy09NRWTfT8YFE6N3llVRaK3Vzc4Mg5Jw3bY0JjPv9TvPZZrFaLKE2dDgah/3AsaAR0AiLwtGgdzO7Gw4Gs/Wy47zfH9ZFfX15Y5TyhqGUIB4M+vFg3B+eHp5sNyl12Xq5ent1kZdZMugxakMILcq0kPfvP5jP53e3cz8KHj9+JiV3HNt13f/6m18bkT56NGKRiyFVGgihwigpZdNst5uiWGUpV6KsKkxRmMRpmtddO5iMx5MDLU2el5hRP4rTNKtqcXAY+0Fclq3RmEtzfX1DiGVZ3ts3577vHR5OD8cnWiGt5cGzodbg9mquuD48O7r34H4c9jwrVMq0dUcI7PVGhNjEIlqgbF1D2ATD3vjeQZYVUnZfv3i+nC0W8w3Q2LMDtxf6dvDNN99J3gANXZcUbQGgNNAGUCMMLEq54rzqHGYJoQ2ExCYQGQiNbVHXtT/59EPLphhqbQSz/MlwBA3KsybbVavFYr64G/STvABtnUspLZtoCLSRmDDf9TEiXdfVFdeS9ntJvkuzXWqUsF1HiK6oSsd32k50/ythySJMS6WMhgYkUTyaTLK0SMLk4YPHxgCtwatXr15nL7XWhAKITNM0BkhmIWCg43hlVl5dXQ/6I8YYhqSqS0JIluV5nkII95VvQggIAcZ4s9kgDPbM9cGgd3h46Ps+Qujpxw8Hg+T7P/h8l6XnV5e2Z0f9ZLXdrFaL3qB/cnLSiOrVV99lxXYw6T18+PDwaDqdTsfD0XqF24phCynecdntdtskifI05byVdWMwgRgMvNCZRnVdXV5ePn/x9cHR4dnDxwCyu9na9SLbDt0gDKJRx6vtbCGEmEwmarOxLIsgJLlACNnM8f2QEbyYzaTUSTxs6pv1Nq+qpqxahJnteEKIIZkYwcuu8brWsploG0iJBIYbZTPb8l3XD+PxYHQw7d2OESJGg7oSUmhEsONQi5CqbGsoAIQAE9u2GbUJxhKDe/eOKUZFummKdNRPjo+njLHtdtt1nVLKsqyyLK9u7xouKLPzrDgYDDa71Lbttm21QQihg4OD4+Pjb777VgghZOP77nCQOC7FGGI8wgApJdqu7LqubkopJTDGspiUMt3tIDSTyWRvKi8Kvt2k+466zWYjjd5n7eI4juJ4uVw2Te14bhRFCKHz8+v3F+dFURwcDymx9rgFCjFERkpe1UVjAAAaUwsAEIbRcDj0w1BrTSzgUhN4nug41I2RBQGNTdTJdOgQcHf57u7isq6qNZtRSh1I97nZqiraui6yzEDIpXG9AGFLGqAMIIQgDYUQkGDPdR89nGy32/liJpoy9J2qbtO8cBznZjZfrbctF4AwCyBAKNRKig4jBCHEBEEDjDH7YUtrsC+k2A8NwECwb4DQGiFmjAFAQ2gQAghBjCDCQAEFAdJaG22M1kZTAAmCEBjU7w2k1CcnsGtl04iiKBzHEVxz3iolCGGUkn0qUGlplJZSWJallAqCgDF2/v5tVTaj0Ugpta/XUkruAUgIAQwwhLDrOtnJPUgbQtg1tRACI7Af1DxmHxwcPH788Ojo6Pbd1wZyTIztYEyp4xJtoBGaFGlGCLEJA1J6rus7/cF4lCRRVuTxoI8J0wghgle7dLnJWwGCvv346RNEnzRdc3F98fL1G8H1cHRkW4EGqteLLUqEqosyvb0Rbdf4Ggsh9w8Oo4Bt0boOlVKd4MZA03ZpVu52O4xpv993HCdPVb/fD/1gm5azxXJ2e4cMAEp7tiMbQRGBBgjeSq6IraEk+S7jTef6FsEsTHpN0yCMy7qCGERJGEWhlLytG4xAXRW7Ld5sbiEGhJAqL6BWOg7TIkeUnJ6d7dLV67cvmKXbroKI267RQpZFZTTBJrUps6jojw581+665uZyeXzwgFILGeNYrta6rpv5fOGx3HEcqToLOwgZjGEQB47jAYx++OMfE0K5UGlZybTgnLteFHSDppBbUPT6cRj2IMRN01R1gxj1woAUO5fAZNA7mh47lh26IQfeoD/FxDs5fvThx58cHp1UVXP5/vrbb79dLZaffPjRdHzStt3rV+fz2W2Z5eNx5DBHI7rJC0qp79s3NzeLm3kvDPJdapTGAMZRQJxjQMG7ywuoOdKSYCBF1dSU13y7htSxz9+/na3XWmvLdbpWbNa73a7QWjZvm1HSH3jJ3e1SVnw07DuWc3B0XLbV9Wx+dBR99tmnoR9oziEAom6rQiltyqp9+/b9YNDfrtbMcU2He8EYQjjsHSCC60o0XbtcrjBld9vNcrMqyxJiULfVNksJw4PBACG03W7H4+lf/Zt/8+//L/++F/eSfk8pdXO10IAyKyDMDWO3qur355ffffuiqouf/eRno1EfEyilOTo87Vr1u9//y9H3J5EXAgi7RgWRr5AWSmoICGGea3suJNjSAnHJ1/N8tVo5I4EgIYHrAPDw7CGy7H/+9X+rOk4QrbJm3B89fvi0qvNvvvlWS3VwcNA1wrIxc4jjMOYibTqetbvNNgwTbSCmzHEcAKWQrTFGG/nsgydZtlvMbze7tG4yZNRoMEmS6NGjM2QuMEabzSbLt8hITEDTlorDwI9cF0jJAYYQQiVkwdOmKC3KHIsS4viu0zZVWeZplgGAEEKKCwwgAmBvaDJKa63zNFvOV2GcKGUOpocPHz5WXJSblDFm2dRzGACAcymE4u1stVpVRTEYDIw2tm0P+4NvvvnOsqyjh0/m8zsI4XA47Lru4uJCKbk3om+2K8uy+v1+mqae5/3iF78oiqLlzWx9zdXT/jRMph/vspS69oOnp71e77uXL4qicAN7m27uVjMF1YeffvjRJx/tNtvr+W222yAAO9EiqEeTyWg6hlp1Tds0jUssKLSoeRzH3ihACHayfvP2eVZk06ODR4+fxb2B0rjr2roGUlvb7ebi4iLL0jAMqeu3bds2tWUT27KMgUIoipngyvcialPbDbK8urqeIWIzm9ieZ5rGCwPVtdl2V3YNpbTuWilVIziqyrLh1HGdIBRGZ3WZxOOu6/K82KR5WZYGIMdxLNdpO6G1ltoAhDzPC8OIMKq1xhq4jp3nueat5ztRFCmlVqvVPuSW7rI37y5evH4TJf2T09Oq4U2Wrbc7y3aFEJdXb6bT6dmDJ5brnJ6eCiHariQEea6lDc+y7ObmxqJss10qLlarFed8PB7bzKrrOs22ru0YYxaLxT5oBwC4ubkpiv/sed56vR5ORkEQbDabtm3r2QwTKGRHJQFAc86rOgNA9nqh74VG6bxIs+0O9TQAmhAKoRJcdl1nIUIpo9RWEu52RVN3Uq6LonBsuymLwPWgNjazfActZnNRuxAgyyg38nmWX63Wbdt+9ONPXZtRHDZN0wkVBIEXhELqoqo7ITmXUmrZ8aauHdd1AxdALHhDIDg+Ozs7PXYdq6yyr777Ls+zXZYrY5jrYos5FsMYSYEQpXutAmijlABAaw0gxvuCBmOgMWD/DUYZrQ2CFkB71BKAUGEEMYYIIWAkQghBACyq5J+4TQihOm+MgXleSqmMgZ4bXK5uMKbMI5xLpQHGECGK0D6gAR3Xy7LMcZyDg6nj2BcX7/O8DkM3zbZts09gAkoxIQwA0LatANwYI6WkiPq+73mBEKJAsOs6SqkBsm1bA+TGdS4v6W63q9LLoih83+1U23alRkxDxRxGGGO+7zuOBQBACFgUcyW32xRTAgymxI76g8FofFBX89UKIWQFwvYcKVuxW0FIlFB5Xjl2mRyNGbGQjZTo2qLBkjQYQqRtJ7AZoJQqabgUhDotF1XN9zYcCgmX7XqTZVmGzy+MMZKzDz744PBgulgt0zz3fT8JI8mFaFqCsEMZAqgpK6ANBkwbxIW8u7m9/8DSUt2/f+/t+TtmUwUUsUh/NOj1wvVynuXrLN387re/bspqs5kdHR8/evQIMmIMlFLt0hJhazAat7xZrBfguQCgsxgGQNoOnl9WDo2By549efJ186ofJxcX5+tFKVrYj4dJONytt9CAJExC3+vF0fL2CkNUVRXvpJRSGuN7oWVVXKoo7tme2+5yLgVXuhUdMMh1oqatpIaEuZZFs6pMy6LhTX/Q+96PfpBc9o0xR0cHCKHb21Vdy1waY4Bted/79IdPP/jo+mb223/56vnz5/PZcrNeBk4ceBFGcD6fd12HEOoazly76kot5cnhsQImXaeB47JebzwaNk3VtfV2vWKB43mOzUhZ5hiaYRKHQWAjIgGvyrzMC6P0mzevNCGHR0eb7WqTbahFyrxRWRNaTp5uZVmmjqe6+23VIoPqqjESdLW6ej8z+tZhlk3wYjavG/3w7MHx8fGrF2/Tdfnm1WtCSC+On37wAbXYsHc4m88vz6+X203Du+U6vVzN67ZBCPihZxAg1PJcLwiifYsM0LCt2tViCRW4d3xqEXp7Nz86OnH9gEvFCFutVhcXF2WV379//2c/+2m/n6zX6126sZlrURsDdvH+3cMHjweDQVPwNE3Pu7eRPxwlnUVdzw1tastOSqEptgyFGOLf/vG/tS1/9uwZUMBg5MX+YrPYlbvJaJIvdsS9d3Bv+NeDX1Y8v76+Tuu1pq4Tun5kG2SaquRNjTWwsUUBwRhgQgmAvBNVXRY5L3JapqllUdehUey3bX1x+VYrde/k4fRgjAyOo0HX8KIofIcZYHbpxgDmBD5AsG1b20a+53SVne1yTYwb+F1TbVbLIPTu3T8pitwQ6CCLUtqLk36/LyTPsmyf8OZtJzqhlNpttn/43R8/ePbhs2cf9vt9qXhgu2EYaMnTNIfQdLzO0u1nn30W+FEcx/tsS5qmGMMHD+5nbbuHwPT7/X0CM88z27b3YTCE0KNHj9J0q7X+1//6X19dXf3Xf/670/uHUexusrQ3HETQrzqOMBSSWxbjnC0W8++efztfzAe9PkLw9cvXq9Xqyy+/uL29dZh1cu/o8cNHTz54Nh7237873y6XFoX9OMIQ1XV9NDlobLbbbX3fffrsseUwxhDG4Oh4vFju8iKvshQguNlsZrPbsiwdzw5st20aDFASx67jyK6r8lpxLYRxA2+z2UgNN5vs3fnNn//Fz4MguLlZCyF812MYFWmmtMaUMMdu29xA0HRt15Z+GE0Rwhh3XQcAltI0TVeVTVm0BgKljFAmiEIAkNSKSymEyPPMQCiEIACEvpduVl1VhD5zGeqF3rjfm04O/ut/+fV6vZYalVUdJYOqbFabbH13I5Q+Or6X9Merr7/781/8JcLWP/3X/xrG0WAw8AO7LPMsy5TugsATQlRFen7+Ok/TNE0ppQBo13EIIb7vHx0dAQAWi8VqtXItmxBS1/W+pmi1WuVZ4TiO4zgYw07wfZd0lvNdumqaZrFYSMX9wGnbVgm5b0KilIRhwCjO8j2PFwK4l+tF02yaTrRtl6bnbd0MekkU+qenJ7Hvia5pyu1knBRpMZ8vKbWipI8BHCcJQBAD6FCrlKqpO8slnh8aCDfpChNLdrLjuuNSCAG1shgLfN8Y6+RgqsYjSvF6vSmKlFmk308wsRzPNxAQxigmNqOYMgyNAN0+qK+lMgZBCDHGECGlFIRQgz9lGYwx2ihjlBDdnpcAoEYIagKUMghp27b3oEaCbYGEEEoIBQGIooRSq6m7oiwZdfu9+O5ugYjte0HXiazI21YbYyjDCGNCEELIGEAIevjwrCiK29vbfj9ECCml/IAghDCi+2u5lFIpiSjar3gIIa7r+r7ftm1TlUKIvanSGEXIn+oaqqpyXB9TEkVBmu6KKjdtyyzi+QEZTiaWxRBCRktKKcVIFkXbSVE2ZcUdz9OAhkE/chMysrUGi+yqa5rVep5XqUui6eC0zt6v71JRXpV5aVsuUMYjfuLa2GiGMMX+/leEEEspmW1BCKs2r5rO8zzbJX4YB72qaNvtbpdlWeD0m7aumrpta9ezp6PheDSqi/LdqzdKmU5JoHXFWy0MQrJtW2N16W4dhZ8qzbN8+/zlt9S2IFGnj0+Svq9Ml+XrXbowQNVlVRbZMInHSX84GBBKJTQI0qrhbpBg5o0mRwqoTnAIjOdZXd05Sbic8UGCI3/4+NGHL7+5yrL8229eXF8sDyb3y3yXBKGQ9XbdMQI9n/Z7/vIGU2IbCQ0GSoGOC8c2Tdfq3W44KpnnWK4FKERSEoYNhLLFbhSEod8bDYTieocENcqg69XcdV07jBmhw+nJdrWZL5basGDUhxDGybAs6v/8q3949ebt2zcXUkpKrbaR8/kyCv3jo0NMYBB4jkVHo4lUqqg713a9KJ4vFl3b/tmPf3I8Hsu2nd/dcN4uVwtfhQAq17XLhYEKO8wxXLeqJRDZoXf/3vEq2/3qv/xdVYuoDdtF2bal59Msb+PIT3qe0l3XqQ4hwVuMcVU2juV/9OH3MLXKuqqKsjecGC6Xd29u5rff/+wH08nR9z76/Ksvvxz2JrPbeUt1uWuPjoeJ1+9iqSW+uZ5JoyAirVAYUcooNEQJoSXgrSjSOtWZyxwowe/++V8OR9Myr5iGvuUUVWMgzPN8tVrtabhpmu57b7/55quz05ODgwMMVdd1/PB48u/Gb9M/os5AqB3ENNdctfPtzZuvXmtFpuPpycmp5dgEIcoYJjCKojN6luflaDTinN/Mbr578e2L129kB1re+EEwOh6ykI5Oxo8/edDBer1ex5F/fHpweDRSst6uGwCAbVkYWZQwYtsIUwWU0cSzHWEAAGa5unnw4OzR41OM4Zu3r+6ubxixhoPJdNQ/PJxS4l2mF0qJMBw4LvN8xyDIbEosooWmGNm2qzthuBz0+hjjzWq5Wi/aLhiMBmF8ZFnUQ54QIs9zJSVvO9FxymgYhkDrqmkZs4u8apomy7Kbq+vlfNE0tZRdnqei64ToBoNer584ljsZHwAt1+v1w4dnaZpevH//ve9972d/8Rf/x//T//nu7kZK6brucDh8+vTpYjG/ubm5urrK8l2/36eUcs53u10URbe3t+luc3J8uJjfLZbL3W7HlRRGQQjXEFKLhb7//s07oMFPf/Dj4+Nji7Dn377o9XqyVe/fXDStdBz36RPWH0zCOFH6Kq851gAgx7VtCBijnjeIMAZFPYJIur41GvcgErt0KYTioqrqRhvYNJXSYs+zA9IYqWxmxVHk+35b1U1VQ4jqtnM6VZWt4/lFXS2WG22QNqhr2z/BqQiGEArOIYSj/oAYbFFWlvV6tYMG2cxZLFdFUeRZ03WdVoAxO4oItZjneZbjOJ5LKYUYSfMnLnKaZWWZIW3apkhXqzhwh8Oh7/sIQdd1Dw4OKKUQ4oOD6VADzCyljMUcx4ZIyL1QpLWOoujlq3e/+93vlNEffPDBcJTsdpvNekEoePr08aMHZ5ms54u77WrtOI7nOVWdaSVGo9Hx8eHB9GDPKXr9+rVRAAAURUkcx9PpFCG0WK2yLMuyDEIzPZzc3NwAoNM0Xa2WTdNIxRElQnYdVxgiCKFtW65rEwrqupzPb3zHhxB2UtWVEBI4to+pg5FFaACg0YB6fnR67+FoEC/uroTgN7fXWkjPtz549tF4PG0bqZTOygKFbtM0m+Xq6uY2GQxHk0PmOJDaBhEAudL7c5q4rht6bug6k8nJ/oVerubfPr+bL2YHB5OzZJiWFSZsH+RumqblneM4bdcwn0EIEQKQQAOgAQhjDI2BEGoFtFL7hIJSSgGFoJKS7yP3e5lBa7P/Yu9pgBACgJRSWgspJQTm8PA48CNKF3XV0siL46TX6zWd6iW9NM2LqhRCCGGUFsymEJm8KKLY7fUSynCe58aoKApub2ddpy0LEUKbphHCMIYmk8loOHFCd7PZLBaLuq4hhEqZvW+PEEIJUpprTS2Luq4bRZHv+7xeOY5/fHQ0GXeDwVgpoZS0bEoUQa3Rqm1FxymGlmUZoy3LYszebtI8q9pGZduiPxyHYWwAYDjkvM22XVl24+loFMMrs1wvlkj4g9700YOHnsMC35Zd/tvf/fpqfjEaGsZYFEvHcZhleZ6PMUZYNHXBhTYAWzYDCAVRBDFGhIyj8XDYHw6Trs07XgWJ3x8mvV50fXPRyrbtOqChodjxXcfxlNDz5W2Zb1er2bbIVrvt89cvIUVBHEnTvr9+W+bbPNsCKA4mfUaxw06nvYnlelEvURC1UikEBDeOHU6mJwbwpmnKYu1Ydhz1ynQjBTIKVIVWgohGC27qtraZkyTJ8eEYQQWg8j3a1K0UFaOW7eDDw3uj0QgA4Pk+QkganfR7bcvrpulk1/IGYWSUxgxZvoUQgiZQWluORTwGDeodDGlsA2gurq7SNLtdLquivpmtjYGe7QXx1PdpWVZKiTfv3lUNHw0nXavm80Ucx9ku3SN4XZdBaGyH3T89QQZWVS2FaOqubBez2UxwLjoeRRHDBAFDKVPAFNmukm2Rp3XZLu9Wl+51Zu1iJ+wnfYqoY9lHh9OTk6PFbsXbslSibjLk2tp0npeMRwmSwHRdXcr5/K6pm/VqhW0nSPrJcKQ10EIO40GR59PJujVyu90JoYbDsUM931HjAVRcrW43tuWtF9u4Pzg6Onn+5hUyUintMIcxBjBomqqoCs5rRGCZ5hije8cnnmvzukuCEHDt2U6b157nXV9fz+a3s9lMKbFbbxzXggb85p//yXPsf/tv/+2HHz0NfEcpVRY5IRFLntZFzbCDsd1JGdg+B+ZqfTu7W81u7pqqPXv4ABFY1IXWykDz9NHjqmqqqvJs59WrV19/85UXUovZg8Hg448/PX1wjwWWxCI57Fk3dkiin/78X4W+jwDfrIXr2hYjkgveyP3hQQjpRIstO4pdTAHXHSOjo6ODMPTzYidlB4Cp6uzy8r3sgMsijEWWZcYYQhFjzPV6Rfenvd5eyQQAMEIC3y3yXAjBKH306BEARiiZlZkpzMf3BoWUknPRdvuBwBhDLaaU8h3Xcj3Xqb+Pqet6z7/97nZ2d3R0sFqtuq5jBEHIPC84vXe/3+vd3d0uZrdd13z22WePHz8mGGIMX333Tdftb1SgqqrVatW27Wx2d3NzI4SglLZt+80337RtLaX8+uuv5/P54Wiihbx8e255frbZUtuK+wMG8Xy+WKyWp/fOvv/J93/02Q85513Dec2RQInfm/Qmo95kuV7xDiwX29l8IzjIy67rTFtWWi2G/YHoujy7wel8OOxNp5M0W1xeXRR1+fDx495ggjvjeY4ysGkFIci27bqumqbhAmBIMCZlUSONXMfxHF9rfXs722x2QmoHEW1g08rrq1nc7zBEBGGojeKizItd0/rEOjo8jMKQt+3ibnZ5eXP1/urq/ZUQcrVYCsUA1I5rUZtijPfuOebYeZlBpAhFjDAILUphWWdStYubOSUIKvX00elHH33g2bQtijAMCSHf+973pNSYOjezWdMp2/WCKLExbzoupG6aZjQapWl6e3t7cnIilAzD0LIsz/Pqyul4tfe7Jb0wTgKgVBDsTV0MQ0QIklJy3lqWEwQBpVRqKaXUWn/5xdd5Via9aDKZEIqbpjk/f2u71mg0EJKXdUkoCqhHSAAJdhwnTdv9RdEw5rgWADrNtrd3154V3LtHDZTv3l01jT4+Pg1jXBYVZp7lEsuJhCSbbYYQ6oSiFjs+Plyv113dNG25Wi2CIPr4e58wZn/3/m1eluPxWGoECKnrFhArjntFXVPLopRBiDAGtmUxSpVSVVlSi/iBH4QPXN+/uLio2/putuiEkgZ0gtd13XVNVJdhGErJAxru8c8YIYT+1G8HNUQI7QcFrYzWGkEFFDfGMEaMMQYoABDYpx4QgMhoI/eKGkYUY0wp3rtKgUFVVUupl8u1lKDrOoxo19UAAMZIFEUIgb3XRwPDeWdZpN/v+75/e3tbN2WchHVdS6ldF1NqYYxd1/H94PT09OOPPz47O/vim6/3DXB75x+E2PM8x3EYY7xruqpu25aLZokRRCZJEt+22lbXtWaWE0VjjGFR5py3BGBk2TYNfaM00BIaoKXEmI5Go7fmPE2zMiubslMc2NTBGLdCCKGqXZMVzXRsTYZHTx/xYVI8OH3kWr5j2bvNojVdVVbX727vbq+Xc2FZVq/X6/f7/WHP9xLLsi3LY8wt66KqZhpqIQRhxPU9TElgeXEcjidDKdrVet6JGlEwnAzvPzqb3c7X602Z1Q3vhAFcGyFUHAWL5ezv/+H/ty0yjSDz3eVutS23X30VlVWKtBwNkzDy+oMIGx1HQbUr67rWwPhxYlmWgrDrOCDw4YOnVZ1+983XZdHGfi+JB8UuFVyPhoe8M1Xe3lwvtQSW5Xz00UddqwIn7Hi9mF03dVdkZVNXcZzUTW4xj2Cbcw4hdRxXQ+04XtuJo+OD9Xab5jsvDILAC6Jw3wA2GhyneVo0hcLSi/zh4XhMpoiSweH03btzQq3dJt+udkpqywm5QF988cV6vekPRt98/WKXlT/56c8PD48+/viTr776Smudptu7OxiFzstX3x1OJx998LgoiqooMaJKyqypCKIaq9vb25/+8AcQQt/3D48PNNbXy9vNfJvnmVGozKsiq8OB3+8PJqOplDLLMtu2Hz9+6K68VbpuirLtSogkwmY8Hjx6fFbvimpXpPPNYjHHmGHLbvJqk1WtNMdHp0EUMEh9l3728Q8ff/wg3WwH/dHtxV2eV1/84auTw5P1ctXvDzfL9Juvnz/9+BkhzBhIqKU0WM9Xvu8Thuu6bERtWdRzHcbIZDIZJD2bWf7Y+btf/X3o+S515nez4Xjwhz/8oSiKIk8JQYyQiPnL+cKyqevaYehv16vFYnF4eNx1XbZLIVM2so5HJ1KC8/WlRvJwdIQVVe23RVnNF3dREgIEdsXOsu3+IMGQurbz5tXrDz/8kFLa68U/+MH/Po5j1/cPD482mw11CcCgNxloAphv/83f/O+ub95/+cXvZvNbZHS/FwMBtyLrOoUM0Fo3TWOgtDyPWRgKHSeOVPzy6v16vaiqKgg9JdW787dX7+dH09Ph4GB/4jZNU5SpZVGvP6y7VkrpMVcb2dRyz3juDF8ulxjD/nBQ1uViMTMI7jcF+9d9MBjgHSrL8uLiIk3T+w/OfvbTP3v06Ele1m3Ll8vVu7fvkyjWoFmtNbPIqD9o27Zt281mhxHy/eAfX7z2fOf169fDYf/zzz//zW9+/cUXfyyKYk98832/67o0TZumHg6Hp6enXLQAgHfv3g0GvclkMp/PJ5MJMTUE+G45ixKV19VHn3x6dHj07v3F65evmO04tl1mZa/XGyXjtVp/+YcvdaOWt8vNaht6oTFQdPLtm4vhYPoGvN9uVlqqumwQZIeHnu/HXd1c3Z2/fPmd0A0mCiE0X9xRizleEAQJZR61us1uV5SIUrr3fNWVCMNQdPx6tXYt++z+/eFwCAyKwuTt27fK6E6qqqnjJNlmaW84ci0bGUAQFk2dZ9lmvvSoNez1J8PR7Ha+H8rzpmjrzvP8XhTvSv6nmyVGGGPKGGMMUew4tjFGawUBopTZjkUpQQg+fvw48F3Psj754EmSJMvZTZPnTx6cnZ+fx3HMmH11My+Kyg9jz/O01gqoIAgMQNSyHz16lKZpnuc//elPn798wTlPU27bbDwel1WKEOKcD4fxw4cPs96urmujoW1bvOWr1UoIRRD2/VBrXZala7l7wP9kfPjtt99KxeNe74c/+sHJyclqtfB9X2vZNE1VF8YYz3MIQVxJrSVChFIKoAYUua67x0hgjPMiJYRIDReLxWqVY2RneXt7O+uP+/1+L+mNCZTbtFBa+DaZTKZ1lSulMoSEVjezG7/IHzx+hBBaLRauHzx9+vT43oPzq5vL2xtxeWP7/nhyQJjB1EIIaW0QQkrIqsgxYt26Awi6rgsxHk3GbdsqoDXU6H9dD+0lAYtghu22bfY9WJgQjMEerA4U0FoDA4AxYF+DBND+E2NsgNIaGaMBNAAYAAEAUAiutcaYYAtTShCyCGGO4+ynfMuytNZGQ0JIGIZF3VRVBQDo9eIwDIUQWZZlRSYEPzic7lvcbm5uXNcNguDd2/dR5GGMOZdd19m2TSkRQrx79+7m5qZo6vV6rZSKosjzPMZsQgjFKE1TwVsuGkKI59tJkozH49FoZGPr+vry3durtqshBFEUdLzpuoaEuEl8m4uKMGKMquv6H3/zj5blhEHU1KIqedfqwO+fHNyLrchxvHb3FiP16OREqcPlYtnvJ48PjtARMsYwZq6vX79/fy6lLMsSIDY5PFuuUtf1tQZS6rpsuoZHYZiudrxqBlG42ax9lyIbzxczgQDnHDKe79IbwHrR5Okpe/PmTb3DKPE/ePQRQ3jYC9+8edPVucVszksEsfa7i+tXY3z85NlHbhA3vMNMbYvt9e2LqOeudqvXl19azEt5c3J0enm5iWkIob54dXnvPnzw4MHtfHY4Olivl1Shl3/4tlyWk+hgEo94Dl06zpfVkyefX19eVYW8fH897g1CP+B150RRWzV1VWAL3jscXauuSBeU6OViJmsgZBFFUVl2LceMsbZZGWMK0kae+/Ll27tL9ctf/g+jZHhzdQOFbFsehz3WVgaCwA6FFuvlGhKsjAyD+Ec/PcSUvXnz5rtvX7w4f/0f/sv/N6YIQLhab0cHA2yjb1/97smTJ6UI8ubuJ3/+0bu3b3f5cruLXSe4eH+b/G8nl82b+CDcbDbHh4PubWrboH94jIGo6vzP//Jf/f3f/z11Qt/3GYlja9Ijt/WonozHWCPquIsyXZXZ4b1TR3W9g6E3iBevvrIsup3NPIg9A1UjnniTvrBkk0X9ITE4Lyri0IdPH80Wy5ev3nZ1Ve8ygtnNzaws6p/97GejIAx69vp6DpU8OZ5s1uOuy1uUa6eb7+bGF42+ZwdOmpcHR4cIIsxU2Wx/+MEP3rx5s1zcNYT0/fjs6KwXJ6enp77v1lX19OkHXdf1xsMkSd796nx2fiO0woQ4Tnh4ctR13eTQi+P4r375S9HxP/7x5cX5+x/+AFnQC+Ok2CGEEN8BxtjB8IBLvVzNijoHjOft+n50pEDTtF1/2GOM2bY926yur696w+GmyJJB8un3PxZCPHl6f7PepduVhdjiZn4wPRkE48cnH/7w+z/UatFU8zJdMgTjYGQhllYFEJAYTKEuykK2Vac5ofD4+EkY+uGgtR32+vWrNJ9tlpXjwGF/3HWcK/D+5m1eF3lexr3k7e1LIbvRaHhPVaPhNJme8lIT5FVtzbDHuwJ07d35qzzbPXr0aDZb3Du8v1hsbTaIRj292z09+mRVVBd3c+on8dRoK+CabYtOAgwhJMDotnx0OqaYbLalfd/abteiqRlGjkvLbP7y+W97vZ7tAttBL18+R+gjrUia8qZBQrQIQdu2mqZ+/fpVx5vhcNjrT/r9pKoqKSWlU0rx1dXFdDrN87yqCs65lnyz2j774MNHx4+rqmu3PLL6lu1t74qqqbWkcMTeXVxj5jgOyJv8+Ozo7NkpN7yo8oY3357/FmE4nU4Ph6PZbVvW6017dzQ9uP/w8PKfvrm+ekspnR5NVdt0FSTatZHvULvIUihbwLvdcq0qefj/Z+rPemTL0jNNbI17Hm02N5/dzxxzRkZmMskki1XVVLUECFBBQKOBBgTo50h3+gNqlErobgkQoOoGu1lJZg0ki8wx8kTEiXP8HJ/dbTbb87gmXVh2QXbtV+Zmttf6vvd9ns5B3x+XKp/P50TDlm0QiDqdjmM6D/fT6f2ccxAn+WBvEgQBJAQSd7WpCLCBwnWDsm3LmV3m+Ptvrg/6x6HZlY1oi6oTuI1tZlkmoSagCjvOfL5sBbcdRyjUtLVqUc1az/PSLN+d4bI4nT48lmUZmI7voCAwRoM+JWo2f1gtFnG0Gc2G+/uTm9ltsXlgQJR8DlmareXe/j6XkNXZ6cnTvGxm01lRtqambi6+swiZL2e9flfXdKVhy+3oBtlmq72+3fU8RzcxplmSlUWLdSNPMujg7SppC15VFeRgnayk4q5jWV7X9jv/+I//6fL6dtjvHR7uPz09wgQtVtssjX3H9n23bGqqa7pCs+Vi+1AMhj0EJVYKcoAVGHY7P/7yi+12W1Zbr9PHthBZrQ/xNltu1aJr0IyVl7fJaDh0c4iRW+dAN0LO0Hisv3zld3rhYrGABFWq+vbbb3/7q2//7M//9Nn5k7wquWqKOtqmiU7Jcnnx9NlL0SbL5co2XNXApCpd27mrr9M0F0LYtqtrJgLIspwwDAPDMwwDQDmdPr79/rv1/XL7sEEIdA+oME1ZW4TqAAAFEKIaJboALaJYo1gwCZhQUkkoheIY6wBISOCOjykVk1JKJSAhlBClFFAaJpQz1jbKtnRiGHXd1lxYjpfk2XQx7/fGWVlgTKNY2ZbpuY5SAkFZlUk3cP2gp+t6FEWYaAhTiMhwPIrjmOoaksJ3veFwaNt2UWTT+UPbtoZOs7RoiszpdqPtAgAAANrb23M9q6qQrwUHBwc/+fFPu93uw8ODpmmr+fv5ah0EgWY4jDEJ9LpplbJJUVS9Xi+wQojAdrueTqc7npqSINpmVcmfP/v4s0+/evr0RRIXl5eXxFOe7SGFpJQa1mzbdhxLKfX+/fvp/fTm5qooCtu2CcSmZhiG0Vas47lt2waO/ezpszRNL99euK6NpAgcT0PIcSwAOauqNIvbsnJ1WZd5gTTHsGzLGPW6rmVCJcLABUpoBPc6YRLFUAkoZVmVtWpOjs+//OFPXL9z8f7y7uG+qDPftwSvCTQDz5a8yuJoes8NDCCgGHDHcShETVm1dRN6/nA45Ly9v79/mE3ruvE9p2K8ZoJxWbZsvYhsy7+/f2zrxjHMl89fXL2/kkJ4bmhq5vXV7fXVnaE7QafveYFSMF2Xo9FI00he5jsvOCEIa1QI0TRVr99P0/zdu3d11eqm1bCWsUZKnuUpIpAFtgSirUqhZJxnrusSiN5+/+av//oXt3cPu05O1tSWZZmmfXR4Mtk/nM1mnMs8LaQEnhf88U9/Jpo22mwxpvv7hxBiCPDhwXHgd5qmSZJks4mqqur1em3Lqypar9dAQc7E7hdqOBwuo02vO2zLarOM8iTVNM1wXMf3PB4qpeI4Rkj2+70yi6sqoxpsWJsVuWGZYdDdxikhRNf1NE2jKCrLEkFyeGLGcbraLl+//mabrj86PRZCfPHFF3EcD0bDZy+e/7f/7f8Q9qzlcrlYLiWGf2SanvAGg55haIZhtLRQSg32+xIyosMyrwxD0ynpdDpAKt/2Pnv12dHe0dXVVWj6miTfv3lXlU2WFf3RDgJY7+9PfN/v9Xrr9fLnP//5fPaIAZzsDztBSA3sB0FVVYggRLBQqqqq1WZ98eGKakaWpdPpFGJKCIGY7FyLehcjqaCQeZzEcZznqRTi8X6qUSNaryzL9uxg3B8a2Bj1+rqm/epvf02wtj/Y71pdDRtAAJazjKUAgDAMvdCLku02iQRrl7Pp3V39xY8m3Y5/dnoMZX2BL/KshBA4rp1EranT45PDaJuleUII0Q2SprFpnneCoBOEBWwVEIw3CBFdpzFvCAWM13GyKcu0bFNClR9Ym+2y3xsKJa8uLuaL1YsXr372Z392d3f37vvv/+7v//bd2+8C3/VtK08SjPH08b4/Ojk8POz0O1fXF0WRO77TsnKx2myiCCGkGdb+0XHY7UMIP/n8szwv4r/5+ZMnT46Pj1er1Wq1Go2POp1Or9fbbFa6rmdZliQJIUgptcNIG3gnkKe2YWGM67omhO7v7z99/jxKsrquAYKj8cDzvLqqLt69ZU3V6/WOz44PDvcNS9vEm6vb9/P5rNPpSNYK3uqUIkPt2LQY49PT081mE8dxVTUQYt/3fd+3TbOpG53QFjMpBARAI/jgcPLk/Nk0evjmm28W09nkYLQ/3ut1+kIoyzSLLJdC2KYTuh7SdKkgYIpV9bsP1/t7I9e01+t1XVa+40pWT6fTftjZbDbr9ZpSSjStqiqi6fu9nsLWZrOp2qquayZF3TZcipbLLMssyxoMRp7tLNmSMVYVpRKSQC2O0zxNjw+PlJSLxaosMtYK1wkcxy/LGggBAeUMFEW1mm+++uqrum4tyyprBqCSUnLW1A03Hds0dEpxURQIy24v0AzKGHt39aEX9ka2HUcZF6DbdxzLbZpWSlkV5WKzZoxRShzP3WxW6/Vas0LHtbo9X9NAXZfrzXy9WTHW1G1TliWTAhIKCYYIcN4CgA6fnhRp1rA2dK0oy79+/W2v452enp2dna83Eab6k7PzTtg/OThYLDeCcaXEbsIPITRtK+x2mqJaJXNMiOt7nW736PDID8P1el1kRRzHUkrbdoFCZVmenZx+8cMvPlxf/d0v/4EAY7mYpslW1xCADCKua1pRJ4UqdoaFuq5ZK3Sq67rJOW+rOggCquHDw8PTk6M8T6+vr5fLeVI8NjVDeYkIpZQapml7FBOoGrWDJSjAGGtY2wIgqQaZbBFCACgpBBetUgJCiDGkFCOEOZeccyVhVdVAodpoM1j4fkjIVkrpOC7BmlJqNNpbrVaWZbVtu9lsbNtUSui6PhqNdNPbLUcQApRiIRjnLaXYMDQpedvWaRorJRBCnU6gaVqWpk3TwEJxzndGD13XHMf59NNP+/3+ZLJPKQUAvL+4/Ou//uvtdnu075dluVtR2bataVqv19N1nbiOv1pt2rZeb1ZFUXDenp2dVVUzm85fvnxJsKkU/sv/+X/6q7/6+Wg4+clPfprnMTEIYggqaCJzdvNoGEbTVJfv30+n0+VqbpqmrHmaxkKIjuN9/OLZ8fHJHzKWXOqI2H6XajhZbxtas6opBAdAIqkC2+14bpmL9Xw6re4ebm4N3S7LMk3W6xWdTEZlkRqUuLZhahQAuHswb/L1sD8e9seMyzKviizHGhj0u3E6w6ANPb3vT7abDW+UZ0KMYTHLHE3jLb9++1ay9uzpk8O9iYYJUBAoFHR7w8mEYihkQ6hWMVHX7OTkZDGdT6eLjuubnzlcys06EhzZE7/h2XYTG2ad1y0itNsbGJj1h/3ZbPb69Tdt24T9cDQajEYjCVTL2Wg0QmTz7u1Fy9inn35OKZ4+3u1A3KZteJ6FCazzvGHt0cFB1dTXlx9e//Z3bVV/8vHHjuMsluvZ5ZVt22mSX15cdgd9jejpJplOp57npZuke9IBRH+4ewyD7tMnTxDVbNvt9QYY0zzPbdvd7irgeZlnBee8bViel4RonucZBjWM9vbum6oUTVbIhlGIDEPleZkWeZezXr9zcDyBij99cvZ4e11EkanTpMjpemXbrusLhKnjuL3ByDTtQX+03sQCyIeHOwVR2WSAyrvZTZNv4zhGOrYNc7PZfPLp55j+D2Gnx7iM0gwv1rqlG1wbDHtciP6wJ8vy5Pj41SdP5ot7LppeL/BcK062Z0fH4/7gYDQZBL2OFZ4Mjzab6NvX32zXCQI4ikAY8rZm7958n2WpH7hJFve7naxMBqOebRlEBxVPERd9t1PxVkKQVcXD9DGvyrKooyQeDodcik0cOb7n+WFVFVEUrddrMyZCiCrNm6apqso2rfFg72B4eDA5+PDhStdNz/aQUPF6E63W71vZcRzf9wf9geoA0zQBABrSok2cZLHrWG7Hc1MbzWDdVggDWfGriwvfNnvdAJ4cx9E6T3MumWU5lNp5xqhOXN/aRvOySHv9wLYsSzeqqnp4eJA1MAaaYxpNWWRZ3Ok6na7/8HiZF5ugY/UHQZLmmqmCwGtZOV+uAFCM11fXF3sHe0fHk+nDzXz+kGeb/b29KjN4W+/v7w0Gvels1u93Xdc1DZtSure3LwSr6/by8tK2XcOwXNfdbrdJkpydPcEYf/LJJ3t7e7Zt39zc5Hl+eLRvWZZSyrbt3cq8LEsIVRAEAADP8wwqSVl7QWc4mhwdnViOF0d527J//MUvqqqCGPf7/adPz01NQwjmea4RhDGEUEElbdPSDdzWeV2WrmPpmOiEdgKfNVwI/vBwv1gsIGzqqiGYEogEQgjCpqqzJKWaAZRoilLD5OnpaZTkJtVdy3RyOukNsuWmH3Zcy765vtawphFKMNaIYZh26AR1y5M8e7i8VUrdP26QFKNeX7TMsx3LD5sy1SkN/YBivJwvoihCABqaRhA2NF0gyjlvygpC2CpRVbUECiF0fX0dBIGlG6P+4OXzF50gfLi/j+MYsUa0IovLRzjPo2yzXtdFfvXh9ujgtCl5XXDDsvfGR45jbaKtpmkYIo1QKaUSQgiBoDJNXUoJFbdt0zC0vEilgrZtU4qbqoEAV1Im6+16sdZ1s+t5huXoQuZ5WaVZnKcAgE4ntHRa8RrnacszhPXxXk/XYJJuAGSEIACo5ZiGYay325ZL0zAw0gDGg/5keHj0cHcvWD3sdVVdzR7uuWi73f7ZyWlVNhCT08Ojk0N8+vRZy4RS4Pdvfp/GCQAgCLyDo6PAsR+K2zhNN6tVJ/Br1iKqIQRWq81sNru5vhsMx+8vLi+vr03bOH/xTCd0bzz64x/9eLlZ3z8+cJZr1BC8lYBjAvMiyZvaMCxCCGNCcEUxFUJkWRZ6vm3bjDcQwm632+t1EEJB4P3mzYpzwdpGqpoQotdt2bQ7DaOumVTDGqRSCqA4EAAhBATDGCKEpQSMEaUwpVTTtB3ZTEqpJEIEYaRjTA3dZUxQonMG8rxCUC+KyjDawWAghYIAbaNNkiRScs5bjHG3FwJkllXORUsoohpumkZIRjWsgCB0t8zCUnHWCgAA480ObAUhBFACKHcUJkLQq1ev7u/v/+qv/pd37y52WKcdjL8o6Wg0chyn0+nsKB0QQgAhmd5Pb++uT06OFJNH+wfdblcpcXf3gBUiCK0WM8fuDLodywyiKP7lP/z96fHJql5G0abT6Ugp33z9bdu2jDV1XedZokNsINLmJSsq27YDy8mr9PHuJgzDuKj+l1//pWU5X375ZZqmDw93k/294bA7HPYsW4uTIMsSCJV7GgKF86zGSHOcgHPOeYuw2KxmirPpal6WZa8f+l7P0B1Ktdult5iv/t2//zsp1DaJm4ZhwZuy3B+Phch0JMbDXtfV8yR3TVDmKZWeSalO6Hz2+HhLjo+PO0GIEGpYreumbdvd7oCxBkJgaARAmq1SHduj4WG8LSDQOAcH+2dtpRbLbV3LPC+lQDVL87rVbY8r/GTydG88ztIizcs42SJKev0+1bWDgwMhZRAEAMH379/PlvPBcqbrOpAtodTQkKURA0MmmGoa0dSz25v3l5d/94+/vLm7O3/y7PNPPjZN+612gRoehuHXX3/9u9/9/tWrV8+ePaOIfnj3wdasy/dXH959sG3btexnz168fP4CAhwEnbpuGRMIESmgaVg7ocbDw3QwGJyePnVdd8dNe7iffvvt98t1DAQmAPqW01b17Pa2lSKus2DY8cPgj//4pwqwF0/O7iZ9xVrB2zdfXymCVtGWA0gNw0A0CDq93qDXF9s4XkfbD9cXB0dHDDT9vc4mivI6a0SdFEnYDb7+D998+aMf/+yf/un9/T01dAVAWVcIoYY3Qcev2/roeF8vm4OjseebXFRNmwHHdm27E/RCPxiEPcdwqrjQidlxe/mm4AU7Pz5ZbTZF9oc1+WI1v7i48EIbY+gFzni/HwSeFDyuYwtovuVyoBQGAsooTx8WU0q03rD/0tAAIo3gpu34QeB5Lsa4ZrZRlY5BTNMcj8eWZW232+n9dPGwqLP229989+7de84Fq1hd10pBpZSaSA/apWyAhZVSRbxmgld5hTEOw1DTdYNqoR8URbZY17KVBKAyKpq0xiEKnSD0Q02b7VbaSqn1enl19d73Q8c1Hx4Tta6fnp9BIKuiyLY5EtQ1HddwFOAKsF5vOBh1tHeQiXqvO/Q7NlNNw4u8iO/vHufLxatXH0swvr29z/P08PCQUNTthUqIKN5sWbtezNfruW0ZErgXH967ro0xHu/td7q9ljWff/HVp599uV4v1+v1b3/3+5ubm06n0x8OgiB48eKFUurt27dJkkAI7+/vd66aMPQppbvsJGPtLp8VhqFmINsJDMMwbDvNy02UbbZxFMWj0ShJksVi8fDwcPHu7fn56ZOzE9+17+9vbdvWCY23Gwyl7dmuZQWWRSQUbYOU7IedXbFi1jIhRBxFvBUIQ4KpoZuMsWgTXbTvjk/PVsvNxYdL1wlevfrY0ZPL69tvm7pqMw0hCpFvWoHjXsQXgkNKdd/xhIKeG+rUbMq8LZoqrZum0R0vS1KqoIaAbVpICKiApRmmph8/31/PF79cr5WQ/W6PEp3VjRE4BEOIFMZQB5RrHGGsGabneW3bZlk2mz70+/3Adx3jKWPs/vpKpxqlNE+zum5DNxSm29Q8iwvJQVNz1wlGfdf1fYItxtjj48z3fdtWCEDFmZCt55q6QaWCAEjPd3UNIYp8x+VKNk3j+KEAIM0LibAbdjgA6yimlCZ5KoEyHFtKKTGUCNq+N4LSpETXse2EgldpHBsmPpjsm6bJBK+qhgkVZxlGVClIqT4Y7XnDvsKItY2JgDC0E8PAQEVJlpeVEMqg9GBvQnTNNbUKMcOwXjx7fnV1RSB68uTJ4eHherl8f331/bu3nuOMLLtu2O3dg21aZdE2DcOYlmW1WHyvW+arj1+WefrLf7wtq6o76H70/Fkn8NIkWq23uq5DpGpeACJkJXcQvKpqgEK70UKaplmcrNdrQlFR5L/+VSolhxBKyQVXSkEI8I7EWNdtUTVKqd0iMvA8z3FN09SpprgAAPCi1jSCMeYcCsk4F1JiIYSum0pioBiCmq6bugYNw+r3B2XRMiYAgARrRVE1DTN0k2AaBMGut6WUSpLEMDTbMTnnQtUQAl3XhBCEYM53cAYgBCcEG4ah6xoAgP2hsAowxEoJXaeWZTDGNI1iApVSv/rVP75792693nLOHcfZZSNs2z48Odrf399sNmXTSoiSvNjRHglv+XYd/fEf/ZRSvDcZjcej77//XiNUcG5o9HD/wDTdqmxNww+DgBLt4vu3O4vlFz/4DAAwe3isqiIIgiBwDQ394f1VYtALe71OEATLFLUtA0D2+91+vz+fL+fzpW3bZdk+3M/qupaKHx8fhGG4m6K0batrVhAEEBDLcqSUUbQRQkyOJn7X+5u/+fl0Of/88x/0uuMkLlopDo7P76ery4tLAJDlmAAiBaGu60eH+4vFNZANhkC0TVPnlmbIthoMTkf9nuU6WZkLqRCEGiau67ZbZlILYCAFAhw5nu37vhRQZTCOioPJSR5XabS9urw/Oz5BWIOIzhcbzqUXhK4fPn/x8d7B/vdv3l5e31JDx5oeBEHVlhAjznmSppPTUyAlaxvDNsb74+nj/M2b7/b29ohGdR1SAhWr4/UiSaPbm5uqaebLVcOYBtS4Nwg8P4+TOE6bspICQkAw0gQHTcVkqzzL3x8daJhst9uvv/56MBj8i3/xL54+eWFa7mq1IhpfLdeaZrhOoBR0Xd91/cVicX//2OuOTk/OdV1fLtdF3rx79/4Xv/j3euA+PTsf9vo8r642l1mWCSURQnGSMCyCTgiRwJT0hwOd4Pvrq9H+hDFx9/g2LaqXz1+1Lc+ywg84Y9ww9DRPHd/BOmxAdTX9wEV7/OzzL89/dHxwmESpE/hv3198+aOvLt5f2lJhSiDBZV1ttxtdJwpSz7NKZK+WU8+2Bt1Oc3CQR6mukT//2Z9QpAMlWVWbtkcQrrICCfXFp1+0sHn79q1lEA6F71rPnp+XTfnp5588ff4UELlczhlrW1YrLPK2XGdraaBaNoyrVjEndAO/M5ns121TFnW330MIeUEHIVRVtevaGEPLxpxz3rbYcpuSXVxcrRZLwzAp0Yu0oJSWZZnnpWs7uq7fXF3/5j++7na7p6cnmKIPlxer7cp2bdPUmWh/+ctfUkM/Oj6wTTd0mFIKImTgoE7Y4nbNIWtyJTgu8root4buOr7HAcdEffnlZ6apZrP7XtfmbeU6umpFXeWX7y9c00FAWRpeR1vf9/f2J0mSLFZrBYjl+IblSCkdx9GTuCxLxtguPz+dTvv9/o9//ONos6nK/ORgf/74UJalbRlFQ5bLZZJEo9Go3+/7fnh/f08IOT050zWjE/Z//ZtfVVXT6XSqqoIQdvvD9Xr97t0727Z93//m299jjPM813W6yzY6jiME3Gw2u9S9H+x5ns8Y+/D+arFaE6LZlosxEXXNGIuibZZlSrRZsgm7Xds0Dg4mlmlKyTlnEEgoOGiYrJplMhNCtEXV7XaDICBEk4IhjCaTA00ziiIzdcvznbIskzStixJKuNlEt5fX3W7v5OCkyvNktVJNg3RICCmTjDF2dHTEOLi6vLm8ueUScC6UAoJJBLBvh03TGMSyAh9AWWZ53pSyLBEX+XatCXk/HJmUAiEpJrZuaLouABQtowRbpuHajmlZXEoIFaIa1bS6rrFuEEKSaDt9eCyznBDiOa5BqJSSIEwxhgpYtqVR2g173W6/KIrp4zzPS1xhzlSyTcum9Fw0GU16YUcKhaCSrHWD0FGqaVndNq5t+oGzK8ixsqSIzqarvb091+toWmuYFsFaLTmrGs4kNXRP13hTC8mrphaSEY1qGiEU+77L226ZpzuzK0RGkWZCKEqxZZlYI3XDMNYc18K6Zno2zEGWJBSos9PTUa9XZSkGyLIcTdMs29ZMI47TDxfv4iy1XF8jqNPpKAhmi/lsNkuKkgNlOM7k6JA1bRrHTEjDsEK3yyqxiMruoI8Qmj48rtazhjVSyvVqcbi///z86bfffnv54XpvfwIATLLCNE0XaJqmFUURRRFQiCBCCEvTNN5si6I4f3La7Xaur68Xi6Vpak3TOj0CIUSUUF2DECFEAFBSyvU6klKutHWv0w+CQKcaBhBC2O0OdxJtIURZ5m3bQoj/YHbgAKhKcCA4YK2EQDS1UBJNHxcQ0PF4sl5vHdsdDEZlWWqakSQJQigMwySJfN/3A7dt26xMNE3zA7ssS4SlbmCTUcYYhAohpQBrWQkhJBTohkEpTaOCUrLzkFGKNY0QohGKyiqP41hKbpp627a7PV2/3xsORpRoy8Uqy7LdHXIwGBBCyPHx6Xa71XWTsebu9qGqquVyaVlWr9eLoxhjPc9LoKjgsNPpKQW30fr1N18jhJ4+O8vz3HVtwyQvXz7jvF2uFlGUc966nn14NOp2O2VZtqz84Vc/LPImz6of/ehH33779vLDtWFYUoDb2/urqw+Xl97nX3x0eLSXFxkhqNcfatTASNtu0zZJdV2XEJm20xvuma529XBXMJYW9Xz1Li/qwO/AsmmUwqYpJZCISqkMyx6MxxBRiDSqI0p1waSluy+fvTBNW1RdCLDl2J7nrKIoTWNCqem4g97w+dMXQgiq4QaVvhV0nQBzrLVGmqajwQFv1PT+IY4icYRZqzy3E3YpgrhuWc2VZlmmZa+20dub91ESB6EnAWxq1rRMKJllWZEmhKDtdptlmR96m+12E611U+OsCsMQSIkAKPMk3qznDw9SgslwsH90ZLlhCwAHIMrym4dHxlhvOJJSWpYzHu7lafH6d68n473jyVGRZhQQR7eH/dGTs6e27SZJNl0sEaw6nU63q0kJGBNAYUO3XMfnXKZprhRcLNavX79u2/b9+w9p2k5GOAx9SvE6jynFH3/6yYtXz83QIbpWVmkcbTWKgFSakq1qV+sYm+7d9OZuOvNd/7CsonWcGHndtG3baqYRx9tnnzyrRW0FRlZXZ0/3D072f/iTHyCA//oX//7o7Pj199/983/6X5w9e75er03HdTx7upjePz5UTdm01XT5mGSr0Pcxl6JqTEyNIHRNCwHF28ZzLcPQmqZCAJq2HnRD27bTZpFnG4jb99dXcbLqjnpPnp7+s//iz3VbF4q7oVdWRV2XWZat18v4oRBd8gelvFKTo8NerwcAqvMMa7jn9ziXuk7rqk3TZHet32w2URQVRdXrDaJtMpsuLcsZDifLxcL1u7quW2bj2DUmMEmS+/v7Ue9MYGj67ngywo4+iDeGZRCCDFPbbDZKyOfPPnr+5HmZF1VRJknC66VuGXkZCQB8Z3B2jCEhZd3mRXM0mACplsvZp588+/iT592OMRwG0SznLXu8W5QpAwx4TlhnBQTy/MnJ2cnk/Oz5xcXFuw8fVuv8p3/8pweHZ0JBKZDj5NE2u3u4z7JcQVLXNW/avb29brcn/eDF848DN1itFr7rff32EhJMdR0RrdMb7B8czRabLMuipNAMpzcczVdLRMjTF88VglEaBZ3eDl2+6xEwxgzDOD4+DkN/PB67rgshzPM0TdNut9vpdI5PT5uazWazumWO4wV+BwCQJOn9/b1lWRSTfr/bH3R1XdcpFoKbpqnrVCnMW0UhwkoZlPiOTbFK07TOs4csXZmWbpplWVLdwJAYhjEYjHzfV0osF+uyKDqdznw6syzrcG9sGk6dZ9Fq2ZalFgQtbxvG0yz+5ptvLNsRCgsEECYNF0VRaUYtFQaKWJaDkYYUAISYGm3KahUnhIuO4wCltptNuolu0HWaJP2w47heVhaiZRjjNIqFELpGNUIgkIjYRNcwxju7t65ppmFDKZqmEhwLywjCcPrwyKrcNAwgZJpEtmkZGjE0EnieYGyTZQoC27aZ4BhgBJDrurZtm1mOEKAE9TqegvDhYdrURdPYDnWEknlalGWJEBICKInqul4t1oZmP//slRDqw8X7PM0UB/+51IcAlJxJwdpGNXULA9zp9Iosi+Pt4+NjFEUSCCWhYZKgFwBMsrzUNM2ytKQsGWOapknDElWFEXVsnyi0Wi7alidpGih5POjYgSOw2G63RSn29vY0Tbu5ufnbh7/dxbzC/iDOi6yqsQRCgKbmSgAICQKEECyE2GxWcbLVNM2yDKpT1rR3l7enT5+IRtZFQ5EpIJKipprdsSjncrcsUxLuMgpVVTHGVqsk7GyGw0G3262qQilV1y3nHEIMJBcSUAqpjolmEJ0qoW02m1VUpduZbacUEwwwxvj4yVjXgWkiQrBGfEMDmqZRqq9Wq7ZlRV4XRcOZLMsaIRxtU4TIcrnu9XqB361KrmnGrs340Ucf9XoDSjHViGVpQehRSqNoC4DUdQqAIQTbpR9s2xRCq+uaEKJpGiFkV81QSnHeapompWSMKSWVkgghTSOGoe3t7c3ncwhht9PfFVuapnFdu2GtqoDl2A1rJVCmaSgIGtYSpMCgO5jeP3Y6/jZar5dzAJRt2x/eXcxmix9++ZNBf1jX/Ob63jZsCKHfCQzbHAx6k8PJd9998+zVkzxPX3z0/P2H78WqaUQpJGs4LpsMJGyxWDAoX7x6Xlf8/cWVrrlVzaM4T9O0qiqE6WDUdz1DAriJtuvNwrb1sLsvJFQArqKkKplpmkWVB6HtbzY4VkgzBpOD2Wzx7u2VaXlFq65n3ymAECJFVVVCYKxEKRabWMI6TYswNBHWMNFdxz47fdHp9K6u4unjDJnEcCyYpsv1armNwrB7sH+0v3cApEySCDTSRJqmqKe7dGBauouxfrB/6tn+7e01a6VQqK7roedLAKOsnC9WUfpXQad3fX1r1tpqtR6NRicnZ0Lxfr97fHwqIb+9vXVdu6yrqikhhMNRXzM1JhmUAkgGlIIQ6Rh4ltkPA4zp4dHR+OAIElpwjk3LdPJtnMRx6vnh9OFRCOV5QR4nSZTuDyc9vysbARXa3zv46MVHnU5PCIWprlEjTaJBX5s+zjnneVYqBTmfa5qmFJjPF1VVtw2/u31IkiSO0r1x/+NPXlATrrezpimCjuuHTl7l0+1idLgHMZnNVp5naUTnTZFHESWW7jnXd7dRlnpu+DCdLh+Xw+F4uVxnWfJf/u//d5PDyXT6aIZG0HE+++HT82fnHvHvHu8RxNt02yl6ddve3t29+vijf/tv/y2hFCD09u3b+8fbTt9HCEbRpuu4nz7/tNPp/M38r+uq+fSjT8fDvTROLMMyTM20tOVijRCggauxEmrg5GS8Wo/jZIEhj+INsdAXgy90gz5Op4iirMijKGpYk+c5V0jX9ZrXmqZlSaYE6PS6VNeSOGt5qxQwbYeJFguioGw529Xn0iKzHNvx/L3xoe8XdSMtw3Ud7/MffKUTPS/SsiwpxmEYhh0fQriNWVWVh8dHp2eHQnEIQdNUt7e3SZJUecFqNuiMeQOKqBEMEmmZ2l5dVdPrm2U2r3nRIq5bZlyUmqY3TVPX9f3d9Xdvwr29judqCLGmzvNsffXhA1KGa3Z8O4zjOE2SMPR7YUdBzXK6vpu6Xu9g8hRh9+LdN5t1ZLuO2/Fm89XD/YeHx7VpmlApw3A+/fgjQ6cAwCTOBIcAICZa3TB830cEN4ynWf44nZVlOd6beEHYtjWhuut7QsnAsxFyHx4ePM979erVxcUFhPDTTz8VQgyHwxcvntm2nWXZYrHYAV6Ojo50XWdcRklcVHWn2/d9H0J8d3f/8PDg+75h6JvNuiyK2eM0jaPJ/pgQ0iihEQqkkJyZBgl9jwA47HYsaz/NEsb5Jo65EBohjFJC0P3VfacT4n5f2UJKWZel4iL0/NALTdN0TEdyBTijEHiWqWPUSmA55vHp6e39w89/8TcKUS4RoZrCqBGi5RK0TMPIJLqULeNq6HqUIJvqJgKDMDAxoUq4pkEJKfOiLivesjzNWtYCiKu6bFkLBNd1ijEWXGJKKKFcyZcvnj08THchDKsbdMJwMpm8fPa8qVIlWVvVGiG8YYIJxpuiyFarBWMtYw1vawEUIUjTNM3QTZNgiNI42W43RZYyKQiCTHDGK942TV1KKTnnWNN3rbzjg2NKaZmVkisMyWgwpgjP7x/vywYCqZSAUOkUUQ0jTQOSt5XK81qPMs+1u51+29ZSckoxIlqeZ5Zu9AchV5JokFKKKecFRwoYmg403rYCYxpFydtvv9lu191O2LA6aXJiaYNR//D84IScyJK6vte2bV3X17e3i9XKsqxuGPq+3zDGqlZyKdqGNS1WwDCsdDZtmkYw3u/3PdtJ07iIMyXUZhlhdL9ZJLxBRc4VJhBaSuimqwuhXNf1fb+p//+USwBYFplOpwhBxzYNw0jT1HVNDqrdkl4pyLlEGBCKCbbCwGtq0BYxa1RUN7wtoVSU6ov1mhBiGLqu66ap245p26ZhGGVZti2P47QqGyFUVdUYUV0zpcBAQQSxphmeF0gJNpsoz0sA0MHBQVUVTVsPBgMA5WazxhgPBn0hRFWVGCNCCOfcMPSdNR5CuMM87E4GTdMwxjw/gBAapmaZDsbYNE1dN1zXdRzLsqyd4ANj3Ol0dvTGKEocxzk8PLYsZ7vdIkTyvGSMEdt2O53O777+zcuXL/I8V0q8fPkiCIKyrNI039vbOzo8TZLi4t1ly2opQFnmQjRB4JkWjZP10fFemjHGK0qxbmhB4CklGG8222UUo+Vy7g26l9eXw8HecDyaPq6jJO72ep1ubzab+YFzcDBOsvXjbMofSs6bZ8/PFqttt9sf9Hthp6dpjYQg36wb2Vgz+83b1+vNcjTZ4wBLhCGmd9NpI6UXhEChpCg1jTqOVVbRzf2Dpk+SNMcEZG6TFY1kdLHaxlGx3LLpcmaH/nh/LBFuW3Z1c4UxTeJsNBhohKabpCpKAxET6QhCCJHjuNE27nW749FECBFvIyklAChOs22caprhh+HV5c2btx900+jBQAixtz/xfdfxbNOkw/F4Gy0BABAj3/d33dFuv2+77uPjo2nqVMO8biFQBCJdI7ZhYkKSKIKYrON0naSD/QMrDE3TdBzrP/2n/zR7nJZ53u/2KERBELi2k6WpaRhFntu2PRwOCSFCyclgACG8vys7nc7FxcVyuSaEAACiKBqNxlmWNU3T7w0PD0b39/fv3r1XCvp+KACfzR+qtOwFoaNby9V8td0gQ5vOZ27HT9PUMDQmxGqxXU4fv/ziB+OT/VZw07bcwF+tNtP5LAy70+n06urqv/pv/usf/OAH/9f/2//lxz/7ShH1yRcfcclZw/7df/jF0dHJ0xfPP7y9PNw/+d3r3//wix9mWcZYwxW/+PA+y+LhXjfoBoSin/zgy5OTE9a0OqKGox3s7x/tH3e7fQJx27KiLF3PdlyvzePFdnlwfh4Y/aPjyfurt8cn+7qreZ3w+YtzQhChiAnBGGOC67qhIGqaZqeuCFyvqqrtNi7ryqhrCUSn112vNxhDAACl1DCMoiiEUFJKomndfg9BzXJsxtVgOB70RqZpl2VpWZ6hnIaJzWb17v2FYWie5335039aVZXp2yVvy6awHVNquBasapvBcGxphkXNeJPlaXmwd9DvDnQMNvE6TrIoTwveQA0hRIHCpu3mee5Y5mg0AFDxpnFdqlNUN4Wm651uaFDPIt5oNKKIctYAgKbzBeccAHx2/nKydzwcH81ni9ubB8/zPv/sS8fxojh///763fsPTdMgBaqy7nd7x4cHSVxkWYkxLPLadd2iKAghTdNcXV29/f7dz3/+N6ZpUqp/+eUXEOHnz5/nRZqkWyllnqfffPP9Z5999vLly6urK8MwvvjBZ69fv26aJooiKeWuQ++6rmEYQRAsl8v315dCKE3TDN16fJxNp9PZbF7mBed8NBpatoGJqusyjreCt4TgZRobmmbqukahZ5qsqiyDQKjmUSwkp4aOAUCa5jgW1XVE8PPn7nK5vLi4yLLx4eF+GIZFUSAAP/nk0/l8fnN1m6XF/j70XY8grGkGpma/398/PPa+e/Pu/fvH+Xq9SXTT8f2ughBQjDClum6YZlU2SrLDw8M0iaiNTg8n/cDfzhf5dj0ejpqmgVIWaXZ9dwsg6vV6fthZp6k16JmmiTXKpagaoZTkUpR1U5alUuLw8LDf6W5W69vbWwzUqD8Y9j3fsZdlsVpvRd1ahqkYu76+Ojs90XWdUsI54W1blWVd17Si4/EJhGiz2cwep6vVCkIYJ1FeFIRgw9QIIW1TtUwEhrWTCsZxTCG1dGtvqEvGLy8uKSarxVIJqWsEIcJ4w9pWCogx1KnGG16V7VrGBGHTtMKwSzE6Op5sok2eZ1w0QrZNy7loMAF5nhDSwRBJxqPNhgDkur5o6m+++64uy06n43heksXfvvkGXWI38H3fC7WhfHgYjkc//qM/Onvy5Ldf/y7P8/FwzzbMuqqm23teN21e5mnmWTYE+OzsjLEmzzIhBMbYdX3H8RzH04g+my6jbaokjrY50nTDtIXEO4liv98HAOVZCRXk/A8ey8lkMl9MF4uFd36KMc5zNpm487gyDK6bBqEUY0qprmmWRs2iqIDSTMNDAEsJa9gAITXNWEePUrYAlggBXQe2Y1qWqevUdV0hRNtwCLFhUEo1Q7cGg4HgyLIcXdfbtiVE0zStrtodNW5XUYni7f7+qCizxWJxdHQ0HA539CSEEKV0pwMNgmA0GpVlGcfxLn7oOE6320UIsUbtIA2e50XbYNdxCIIgjuN+vxuG3aIoZrNZ01YAgLJUQnEuRLfXK6vq7v4+DENd1w3DILZlPDk/ty2jrivOGoyha/tNxQSTQMLZ42y7jiGgGqH9bg9CuFqsu17vq89/pGMTA0qI3uuP3l18yKu8aFugaQow3dI3WbzerPr9/rZSb2+nSasODiZcrz7Mv4YQ7u3tDQwtCJybxw8Pt1NCNCWRYdhx5FHoKi9scpysCsZaSkC53QKkFpDMr9eW5//ki3+yjaOi+KvVdrNN4sZ1t5tFN/Q7Bz5rCqEJi6Dru/calcNuL/D3JTMxJuso/dXvHp4+efJhkY6PX0QN1i0XOO2Ht983GG82K2no96s5hrjMcowQMHXquE1dayAkGBMEHu+3h/ujveFxus4/fvn573//O8DgsDNI81wJcLB3sFpvbdv94uOnJycnTZs0Lfr441dlWUMBPnn+RVVVWAJHNzsH/sPddRj6aRZbqp+Wq+X8VgHU7Qz0sGNgrdYi3fSoZnEFHm+SxaoBRutxhrh9dvAqeVRvfvlGCmb3DN5Wp+d7FLC9wfDh7v7Z0Z5DQLFY8L39sm7OuntJA6hSVCkDoXS76vf7GOOWgDhaPs6mL168sH0tr6Mf/ckPXr/9bTi0HceRSXG/Xruuax/uE4rrbc3qbNQZjrruajVfXF8OHOP3/3D51R99dXi0Z9v21d3jD3/yRx/eXSKsPXl6dPPh1rG940P929ff/Hf/6l//n/7P/83QCy9ef/eTP/2xofAmjovNCrFy4NuRyFiZnh5OVrP52zffd8PuN6+/6wTBdrrt93su6H15/sN4G+nI2szTb7/9btQ7HgxGk97Z3uCoqdlqEw+Hw7rgQegBpNVFHLgeqGpguZPx+Zc/5P/3/+f/A2nuD3/6TzalANssZ7BuRdkoJWlTSpOYrulACAMS8Iwf9E4OuihOk1Wx9TuhgR2dNIJjLqBu2Ov1+nGxOjg4MABohFhvCoSatxfXcZSWZfn+/funT59blv345uF3v/29bbuB59/fP0IIbdte5jgIgh/90Ik2manRZJZKwXABQMotjxbLGNmy6ziyqLM4qos8AWmWZffJPG4rSWmSbh/mD7Zn1YXSMGia6tnz027o52nWCKgRHpccATg4OOv6I8fsQIUnfg8q2dYFQixbzQfdzmgwNiiK1x/S9cpG+sDtWIgQoIo0cRwDIi5h2ypZwfpqcUt8XdM0EDqbOG7TYp1mRweHmmZ0/PDm6vbN778zdWf6MP1Oe4Ml+ov/7T/HGN3cXRojsyyLsmL9jnd58caz9R99+dnbt29vPrzXEJxOpxSCxN5SSncRh5OTk4eb++12u40S27YRA7P5ajabxXFsWvr+pLe3N2a8OT876Ha7aZq+efPd3d0txrjnGc8Oj5++eLrYrlveGK65Wq+RhlNVaDqxbUUp2RuN8zRBiDd17TpD3zu5v9Pqsr65nFZV49quqJ1oKfeHr5a96ubD32GwPj0+tHQDIVQxHK+Ko5Pjf/bHf/7k9GmUJm/fvf/L//mvbIOOB+Fnn74yDHu1jKCEYc9jrXAD1/XM//jvfhFa5pVSk0F3uXoUbea5ruu623RdVHl/vJc11UG/e7ecY9589OJFWdeP07nSDAUR1Q3YZICjuq63d6s/+uiHXWzLqGRle/X1N9X+sM1ryJRsmE41AECcxZiorIhtd3BysjedTl2h7YxWxweHQW+IICmLZjZdthXHlNxd3RuGZdgWBTJdx03bUkqpr7qOy5jQtJpgAABjslmv6+n0g5RAQeA4jkKiUZyjViiGIEQAQagwFVGypjQEMMCE9ru9Mk8fbqebzappGsHEQ33fCqZpGrBtAEAhxGg0IoZB98Zt26ZNSikN90erxSzjDYpr3wvj9RrXKlmvp9lN1AjLss5PTj/6qA2C4POnnyCEgiAoiuLdh/dffPWj2Wx2eX11M7tXeXR2djbxOsvlEpvcs+1NtAYmevXRR5phnpw9yYryH99/bwOFNC0IO3leMyauL5cfffTR3umZRm/u7u6EEEQTUnKlbIiA3wnaqtxst3t7++tNFCfFeBy0bcslAAojZCBiGLrtuGHLIFB5XuWSSUp23haIcOVQulNgIIQEUBBiHesUaSYJmWJWZ2RbrpSyrmvbtvvDgYKgbVveMoR1qhEFpFTcsqw0jTt+gAHuhV3RgiJqxt1DHTrFtjaRdbZ/jjFu2zbPc8YbWYsqKTlrNIADy0UIYozBHwxRfH9/VJblZrPxA1tKwDk3TBwEAcJKqdZ1LaX6RVFgjBFCitPHx8fvfv8/cs51XS/Tmvo6NXWilIqiaDemaGomFX98nOm63jQMKHR3d1dX3PMCKVVZ5re3903VdEJ/sj++uHgbx/HN5dXkcBJ4YVVVGtHLKl+uNgII09Q7Xr8fjpDt2bZDqZamea83ePHi46+//rqu+YsXHysJsrS27UxwkJVFWbRze5lG5Taau5Z1c3tVVYVl6qvVQgJJKMQYTmcP//pf/6tNFM/Xq7DXc10XaZRz+AdglkIa1Q2d9MUwybLxeE+zrDjL3t9dl2kJqdbNBhqhdVnFacIYm87nd7f3VNd4y9I0XVerpqoVF4N+HyFkWZZGaZ4IhZRuapj4ENOmqt3APzk8aqr6t7/9bZJmvd5gfzIpi9q13JOTkx98+nK9Xn///bvhcPjFF1/ujcL1ej2dzkeDvpCsLGrbpBrVbdsVQtR1nRSwKArTcAFAUkoh2sViSbT8h1/90Wq98TxP0wyE0HT2AAAYjUaTyfBnf/pH3/z+6/VmYekaF/XB4alrmXXVwRgRgltWvbv4jmpGv98Vsh0Nh0pKx7Z/8tWPEMa3t7eyYd+9+yYMw/Fw5Fn2dD6r6/pwb2IYRtu2Uby0LGcymZyfnEoIdrPiJMlev37tOE6/3xdCPT7Ofv0Pv+50OoPB4DHeDvvDumbZNs3q8sd/8tMfffXjLMv+6q//6vb+bh1HP/7jP3nz4ZvZYtUfdhAmv/v6aynBn/2Tf6bp9otPPhrujSYHB8vlGlNS1G29Woa+s3e4NxgPiE4a2UKIkyStqmo4HD579nLQHwqu0jTffRV3KSHVNEopy7IAIbN3H8anJ0/OFSFktVnVdRvl+W9++1vdNDzP03UqhSAIU0OzLRMD6OsBY0xBwDknCjMu8ygr4jzNM8BFmqZtWV9eXn64ulSs6XQ62/VsOBwfHBwEnv37r1+vl1OlYL/XS+MkiuLVcpkbBQLYsqxut9ftdh2/izHcrjdNVYxHfQRVlWd5mq1WCwzBZrV1bfv46FRAcHd/c/n+fYUqw9QklI7jGI7e7QdRusqrTNO0+/tb3aCK87+bTg/293/yo69ms1kaJ6tVhKH19FSOB1QwJVphaPSLzz5OkmXblLqu67puGZbrulIAySkiUEIJgOz1Ov1B9/nzp1TXwl437HRGoxHGsCyyqi4ohqZrs5pJxiGSHdc3z59SCZFCB+NRmqZJvF1N525g87qpWc0E81w3Xix2KUXXdeM4/v777weDwbNnz/r9/mKxiKJICEEpjeM4juOmaTSNACAxhmEY7m57VVs1TYMx1g1nf39/tDdeLhf3j/dJluq6/vTZ6Xhvr9MdcAi3SSy4NAwr6Ab7k4O7+9um5qEXdrtDVrNaVLaj70+OKaWO619f36RRwTmP08S0nPV2lWVJt9v9i7/4iyxLkzQqyxwIOY1y07G7/U7X7AElbNt8+epp1eT309lo2Dl/OnFsP/SN7Ta9u7mbL+aP9w4CsMozURa2hjUoG9Y2jF+8f//q5cvzJ8/Onj4DiBRlJQRTSu1a6VlRIIR8z7EdTzctx0mLoqhLbGikbbI03+gG8Fyn3++H/X5ZV1mRY0qIrgGpmOB5XUVpEnSDvK4WyyUhxHbsycH++HA/zsodQhEhpOt6w9qqqiilmkYsK6BUb9pWKRUEga7rTZPe3T7ouqFpmlKwbXgrpBRKKUWJBiEEUEkIMKa76ZpUwtB1Smme53kaD3qnvutdfUjfvHmzC6tSg+qmsQv97BRGQsiyrHVdR4goxeqqNQ17PJ4ErmfqxnI6Y1UdbbYGoQRhKVVTFgSC29vr9XrZ6/X29/cHg0Gep2mR+74bhF6SRk1TbVdLSuns4T6nG8938jxPslgzDKDkdrsdjvcIQUVRACDD0IdI44pDoohOHN23fJdamoKQS6mgJAghgib7o6ZpDA1J33RtZ9APez23LEuCMNJ0TDSqW0KBpmlLkGNMJReWofe7XcH4LoW/y5n5vr/LClBKAUZKwd3vPKWlruumaTqOs+MTUEoRwQ/TRyklAnAXFCjL3DLNMAzDMBwNhoZhCMkghLxldV1zzsumZIwppRACOxAn0TRNJ1jDAhAgmNyFESRACOwayHme53lumqbjeLsKUrfbZ4xxzqUAAqldP0LXdU3T401BCLFte2cwSdNUSimEIHe3D1y0hJDZbH57e2vblpJwNBrFUToYDCHEabKMokjXDSHZYjFbLzemaZq6tlosJ+MRhqit2rvHO8dxyqzOyiZPG6G4qbmD/ujs9KnQ0dvvLz7//PMw6Are/unP/nw82r++vq0rpusmgnpdMc4lwRpXcrvd4p6satjr2HuTXhQhCKRuoNVme3V1EXQ7AMjH6X3TctM0bduiVIO82YWDCIKQEIQIIZrrBZvNJi2r+Wodb5N1kiopV3lycXd30J8ozgiCEiKCkKlToum2aZmmqWHqu55O9NAPdjRJxqXuOUpIgmib8el8sZw+sqY+nBw+e/pScfj4+KiUGgZ9/8hPsty2bYyplKBt2zwtos0WdSHGGEhl27ZUvMyypmG7Qg5rWigRb/hysTbNKk0KKdBk/8Q2rSDs39/e3Nw+JFmOCGFCMsZ0y4yi6OzJ4SefPfv3+/1vX7+2Db1p6s1m9eFiNewPlBItKzebTVFm/4d/+S9d172+ubBcY7GNm7zc258ghMBkf9wfLBeLF89f7A9G2TaO5qs8z8edvmEYSZJ0O65pWycnJ5PJeD6fK66EUJKLt2/enT05992wzMvtNsrz4ujoqKra7x9uHrxZW7N+0JtO56Lm3324+OyTTz798our26tv377rj8f6w7WQUNPdpszGh/tSgr2jg9UyGu2xydGB7QWX729YKwRU283GMDSv40MNAQq7g7BtRJrkSuLhcNIJu2VRK9VAiAnROJcIISllWhSMCaLrQKnx0VGyWgsh/6v/43+9SqOiqdbr7bPT55t4gyRKNlmexoHnT7qjUX8IIezhbl3XTdPksuR19PgwjaINl2K3+atZNRoNHA0HhiaKrEYg2sybKqWYE6Ltjbq8rdarbRytfvTVT+u67YY9x/F63QHnglJ9d44hFFGg5uuV4m2WbB4e79q6KYqcSb6aL+fzuW3/ihK9LIrFYhF2LNt1Dg7GnUEnb/JNlLVt21a13g9M0wSSb7fRdtPuTxSEOI7ToihEy7zA9BxXSrndRq7lTiaTbrdb1wmlFCEEEFQIcik1TUvySCQi6PqYEkSB59kvXz3r9LoQ493iFkIYK06Q4lhRDb148jSKomS7uZUAA0iACnz7aG+4Wq2yLP3w7k3Q8YsiS/I0yzLNoJ0w3PkFOp1Ov983DGN/f//8/Pzq6mpHVoAQapq2OzEMh8MdjkIpFYbheG/oenZRVQBKomOlVFEV8+X88vLy9v6OUnr+9My03DgpqvdXaZ7UTeP4ju/3xv0xkwzIKWdCSU0pygSJ0ibL0qxCnbDX7XQsxxEK+r3OerEuqnw2fyzLcjAYdILQMIxO56xpqzdv3hCMlWAYIgRAnkYciKAbnJ/vf/bFC0JIt6Nl2RqifG9i+d7+yWmHYAcB+OT8pC2L0HWoAho6sU3j6Ojoi88+39ufcM5//duvhZKL5ZKzZrPZOJ5bFEVepHXLkyTp9noEwo5ntSaajEe2i00LHJ8OCSGB57aAcKAEBNQ0DNMsiiKrSwkEJJiaOtU1w7UHg8He3jgMQ4jwYrHYhdgJIYPBoGpqjHG/3y/qyjRNy7KyIi/LkvGmbkBepFBhBDDFBtU0y4ScCyWhhEDTNM45l4wSSikWQLVt3bbCNrFrW3mep2maRFsCFABAIzTw/KqpKaWIkrIu2oZplGFKCNUFUwIJXdchRJwrirV+b1gadlsUaZoXEuRxxk3DMk2KcMfzia4BpbI0rapqu91CCKu20XW9LEvN0E3TPNjf73W7uq7f39/vD0ZHJ4e//e1vP1xdOTpJ0vTr17/37x8Mx54ulgABzaAtFwAAquO6zaFtAwoFAhJJpCFKKFQSN0rTEYBQCQ0jHQJBiJwM+01T1ZLXdQsV0ghpmJJty0CLhFKsBVIpzuuqZHVDEA48x/d9x/YxxkqppmnSIq+qmnMOIHx8fPQ8z5cCQWKapmEYnPMoinYFSEwIhLAsy/l8blsWpdTznN3td/eV4ZwLIRhjeZK2bcs5RwhRDWuaZlkWhKRpas65lByAHYsaYowhhE3DyrLeUdWVUhhjAFBVVYSQtvmDlrosq6ZphJCUNnleAQBc1zVNsyzLuq53xz6CEMrjkjH24f1N3ZRHRycA4Pl8JQQgRIu2SRRFO76EUlIqJgTrdMaMNZvNand4XK/Xt7f3p6enq+WWK25bjoKAEoMSwzQc7FEI4Xq9lRJURbm/vx/4AyUf/v7vfnV29qQqWZ6XddX2egPTpIwx2yGmDTo9e3LQSbMOVODwaHj3cC+E8MJAINWwOgjNivM4jnVdt7s+AIogLKUESkgppYCaYbuukArXXCFNN12vLMukrHC83Q8HimPHMCtQawQbmk516rlBEIS2bXfDnkE13grbtBAilILpcu06TjjoW5Y1u7tfrDbX7y/mD7N/8qd/dn5yPhnuzedzz3Z6QagYL7NsOV84lnt6bFVVdXNzF21ix3EopbdXt65nZ1miUbRarNerFWNtXZdZmWlYGw9GedlmWV7mxWI6u715DDrdi6vrJE65VEyI0d44gOri/k6J+osvvvjo4xdJvOz6YZ7GaRqXZb5YiCRJdp88QtBo3IvjuCij0fhJmeeMsXffvyWE9Hq90A9+8uVXe3t7bVE9Pj66rkttFyFUFAUWyjBMUzeaqr54+/7du3dvvn8LIXRMZxtFSZR6nlcWrWeHSikC9c0yNnQnTfNhf3T29GlZNP/pu78vqvro7MTrdU41xBTo9IajvQPLsY6ODou6+OjsaZYVYae32eZpWRmW9cnx2XB88Hj32Ol17+9vN+sl522SxUwwvxsggU3D64RS16yrq7ur91f93vCjjz6SErQt1zTCmMjzctdybsuCNVJKBAHt+L3+YI9a2s3dbSO4rNlytbq+vEyT+Pz8nDzRdGTUdb3Jk7ZtEYY6NgxsVmnZFHWv1yuKTEKsIeSbds/zqFK6rtmWrQG+fLjRoXxy/uzzVy8+ff787mFeVe3p4YHg8HDvSNdtSrQ8L7ebOIuL6eOtYWhUI4vF7Pz8NEm2d3e3RZEZhpGXWSOb+WaZ39+aphmG3dHJvksBY0w3qOs623i5mM+Fak3TjDfxeDiMoqitqxfPD0zN/Obr13d3d23BJAcYIs55keW8abVA241q27a1Hcd1bdu2MaJMcIVgXuUQwrzMojRREJiOabuOG/g7T6Dv+wQijSAC1M4ZEZhetNpsl6urtxempju23RYZHQ2haB5uLu+u3/VHw4ODfcd1WFUqIbfbbVVV0+kUY/z06dNer8cYu7u7++6775bLpWEYCKE8zwEAg8Hg6dOnnues1+uiKBDBEMKdIB5TYprG/eNDlER+xy+bUkBZV/nDfCa6uwuQ4lJYluU4QTfssRbc3N5LBvvhyA183Qw8r47jaja7/dXvvt/b2/v8888Xmy3G+ODwWCqFIVlu1sv5fL1dn5+e9Xo927YNw9gb77OHKWMN4BwCCZXUKfRdq9/zXc82TDMILQQrBImuY41gTUMdZ6CUGne9eLMddrtpFLdlzZrKdz3NMgFCEqDFaun7fq/X3d+fAKxMS1+vW84ZQRABDkSjAMjzqmkq43iMIev2nX63kxdpUWTTVRaliYSAGjogmHHOOJfYOH1yHnRCrGt+GJyen43Ho6qqZrNZlmW7e8iu6/8HCmFTWpbpOJZhGEWVSckQUrpOTFPraT1CNF3Xdc0EAHKhAAAY07KpOc8kVxgCqGGshOJKtKwQdd3UgjW+O3RtuyxLyzA//vhjztumaTAlbduWZVkUhZTANE2/p2EA25YDgKBCGqEIYd6wMi9MTQ+8EHDWlhWGhNUsrdLB3lBw7njuaDQiuraJtpso4pxjirzAZYzVZRFtQFVVo8GgytLg6blp6ppGCEW7q3DdNtvt9vXr10wqTSMAoVa0CnBE8HqzhiJcblcBr2vRGo7uWjYCCkCOCLSphaGUXMTRJo63tqP3B2HD0SbaCi4dtyMBVHJbFs1mE7UNZ23LOYdSGZru2ma/3w/9wHb8XTdytVpt4qhpGtM0HddFiNR1vV6vN+tos9mMRqNOp2NZloKAMUYx2cWhdqcBKWW/3/d9f71e73gknPOdqFpSgWpcVZXYiduV4kLUbZMVOQAAQoUx3q2JAIIAQtYw27aVUlXVJElGqW6aMokzjChjjBAqhCjLMkkSpaCUEkqilKKU6rq+c2r/gdHUNA2EcAfJeXh4WMw3nueNRqO98eH19fVqdbWzkO3v7xEKqAYPTw5evnwBsOKAr6b3tudSSv3Qi/OkFa1pWxCjtm05VIvtOvvNr4+eH/zZP/nZ9HH+8HAHJPz1r3+73UYQ4Kpqmpr1+/1XLz+OoghjvJvJOF4DoYrTWcuttm1s0zo4HNme/ubNG8aqJNmsVsuzJ0/2R0fz5ZoxhgwKIWzblvEGKkAQdRxv0O0VQdEJwvHeEGN8c/nh8vJSw7QTBF3f51IghArWypbpBFuWvTcaB0FoWZbnBUihpqoNzUREkw33XJtSohGiIBoNRl98+nnX8QnCWZQTjHTdnowmuq6LluV5LqVsGua7lJpOVdSb1Vq0O4MIz/PcMvXb20vHtZuygEg0TZ3n+XR5hxBy3WC7SYu8PTl+dnZ2Fqcl1Q2KCed8E21Xq1WWJ8fHx1mRc1EvV1Pb0U3TJIQYhtFU9dOnz6cPj+/fv8/z/MnTszjZ/u3f/fuiKPzAff7yJZcyybLLy8sdXdvQtK+++gpIGW02WZKcHB3tvHCCMWXbddvG1XY5myd5vlgseMsty+JMtFVT541jQ0Oz9vcO27Y1DPvx8TFh9d7e3tMnzzRqGIYBIKzb9pvvvvt//3/+v5999pET+mVRI2jMpusir09OTuzApMRaLrecqdUySpLc94Rtu8vlynXdpmFJkiEkMcZCiLKsoTQAIEBpy8X27duL3/36t2dnZ5blvnz+QqMEY6T+8EKSsSwrirSdHOwzxm7zRxvQw5ND1/DuHu7j5dYAescKe054tn8aOiFkkJfcIg6QUkpJEUEKNWVDET0/OYVQuZ7TNBXGuKoKDSDPsE8OTwbD4PHxcTAYDYdjJlQS55Ap2YgP37/fRilrlVJIQSwlKPKqaRrHo3VTVbVomkrTie3ahqHlhYiz+N2Hdw1rORRhz9c1ExFgedZHp0dJkui6lsbbNImA4FKxzTr+7LNPjo8PpeI61UzTTLZRlmWBF16+eZ8kxWa1lux9P9zz3NDUjSiKZo93Wb5BUCoghFAIEs4FRnSyv9/pdIZ74/TNdxCjgTXQTUNCYNs2hLAuSgihYHynJMIQGYZhGZrvOmWaHOyPjo+Opvf3q+WMUIQQz5KEPdSOY3qeo2kaY+zt+4tduLppmp126P7+/he/+MWu5+a6bl3XjDFd1y3L2uWkbNtumqqoih3VWDM0Xdccz2X3rBVssr//+XjUG/X/w9/+x199/esvP/5hGIamacqmKeu6yCtKstls9u7dO93U9sb7k70j13WXqwgQAxCt4rxVikOYljlj7PD4qD/qW5aVZLEAAiF0eHzUDbvff/99EsUH+0dF2dR1bVBdx8S1nYZXyWY9vbulBj47O7NH3VJH62W6miXr9Roh5E7GSilTt0ydWJZR5ZQTlm7KtuVUN8q6sW27KIqw23Vcy/O8z7787OLi4pe//GWapicnJ6PhnqHpSZIk2/VqtVgtBnVlEYIgUhKI6fxhE5tt22KMpVS7fP4Ote77PsaYMcYYq+t6R0wvy/Lw8Lhpmvl8vl4v67puWCskq5uyO+hhglpW1U0pJJOKA8gxUU1eN6BuKs0wGoiIEBJCTCllbVsXZd2UumVgApRSkjMlJGNVFsdSiKPjg2fnT96++V5IMR4Orq5uJFcYQylBWdRpnAGADGq0bSsl3z1ldF03Bn0l+HIxN6h2dnp8dnCgpLy7vKyrKt5s59N6PV/sHI+8ZZpttpwBAAzLTNNU0zRd14s2T6KoKsqO5xuYRptlU5VScN+2k20ECR70hqvNOolioaSuES/wnaq6u79njGXZBmO8WkyL3MjzHGPk+67jWp1uQDEiCD3e3wMpNcMMPB8AaZomjyspcMsBoWbQ6bveIIrTtmVRlBiGHQaYUmrqVKcaxWjXoRBC7G78QRD4fmiapmXbjImqqoJux3V8SukfVFtA7WaiEkpN03zf7/f7hq6HYRgEwU4ZtfunQwV2tknDN6uq2rUZAQAIod3MQNM0jCGhGsEEISglEIIDADyvo+s6xhSArG1bwRVQCEKlFJBSMcbaltd127Z8Z99G6g8ByV2T4j+3J0iel1VVUaoPh6PlcvXh/fVgMAj8rn/U8f3o6dOnbVuHYWg75vX15WazcoNhydusqV588lFaZISQx+m0hm3LG2wjzaVxHG/iTQCCtm3X6/U0OgwCdzjqHx0f3N8+fP/991VVbdbRoN/fbree5z1/8XS1WkynU8MkR8eTwVgqpZqyQghKxZNsUVZxlMTvLr47e/JsvDc0LPPHP/mjl5988vU333737fclbgFAVVU0VU0p1bDW83tPT55WVaWE9A1/MOiN/M4w7LOm7na7eRqXZakZFmtqnRLf8xCmmqZNJhOMKSW6YAxDQrAWbbfv3r3TMK3LahZ2OkF31Ou/ePbR0egIKhBv145lYwKXy4WmU930Nul2tVqljxVreKfTkQJIJqFSQRA4lp3n6WI5+9U//IpQ5Hl22PF1Ssq8HPaG1NCBgOvVpr6fDfp7R8dnBCGCMAbQtkwAw+VysZhOx+PRi2fPzs7OhBC2bZ+cnJRpDiWEioyG+1CS0ejhw4eLaJsw3qRpyjl3HGu1WS/Xq7KuDMtMsvTy+qobdnqDwWw22+lfq6qKosgwjMDzQz/gJthut8vlEko17PV7YSfNy8VsiRBSChOoOY6BoFaWtWhhXYiizI0D3cC677iHBwcvnr08OjqaTCb9Xtc0zaqoBVOnh2evXxfzu/XL80/uru8wpknyoFEz2xZ3V1MsrX53AAQu0vqXf/+r2ezx6bMz76NQI0ad547l6hrRdQ5A47nh3viwEw4EVxBiXTc5bxkTBGuYQMZY27ad7qCpuRDy7OhcQrCerTVD/+IHPxn3x4+Pj+v1GmOom0ZbsCqtEULmwMAtiuM4T6IkT2rWUgwp1Z88OaMErVarJIldO0AjCpSAiux19/NNZSKLFWI2W909PN7dPaRZNRpObu4eMDEQwlJBXTcZEwihoDe0LMu2zarNP9xebbfLKI2ITvIqXW0roBDnTEpaVrmUcrw3dG1T00iaJavFAmN8fn4eZZuLi3cfPnzYbFa9brff70/vp3Vd743GnuOLrEn9crVI0yhmFRA9RRCtqiJPo5blhknVimvUsCynbZmumR+/+uz4+NiwTE03G866nS6EkEvZNpy3LQDANi3bsG3dYowJxj3Po4ZuWCbRyf7h/vOXTx8er2fL+1evXr149TTLssV6VVb5fDnL8xJTcn5+niRJURRpmuq6vlqt1us15zzLMtM0d/DBuq41Tdtut998802v41FKuZSsaZumxrvzpqUXdWE7TmBQvxc4nt0bDZ4+fxZ2O7puBv2OhkkxL2aPU4RQ6Ps7glBdNgiRXm/Ahdhs47JlQa8/Aerps2dnz54rjFbzBTaI77gE4d6ogzDQiC6lhAh5blDkVZSkkCOdGk3VptsMcFXm5fZhfXFxoVu6Tg3btNbr9Wq6XK/XdV0fTA55k0OIqKnZlqbrtNMJuOcBAAkhYb/f7fUnk72TDx8gUnGcz2aPP/zJ50I0dZ03TcGaWrHW8lxb73uWqWFkaxZSpCiKu9sHRPB2kyBk6oQqpcq6ggDYlqVr2v5wfPX+w8HhxNYNnWppnORpKoSQXOjE/s81Oc65AkLTNNu2oihqmspxLAUBIUAp1jRVXZdpHAEADMOEyCVYaxomJNhx+ioNCYkMDWkENJwL2QrZDjq+Zep5nusErxbLb37/2vO848PDIs0aznZXZJ3oluUEXtjp9LZFSiltmgogCJGUikne1HXeVIpgNRgMbN2QdbleLlhV+K6FEA7DEOta2dSSt51OqFlmw1qAFMTAMLQil0WWbtdrwFmWpM+fHnFW7/V6gWP/7re/z5PUM10NIB3itCg1g3Zdl/Y699fv4+1a1KVLhk0Z8wa3bavrplTcdz13fEAp5Yw9PK4dz3n6atINw912zO8KhcyH6TxKG0VKy/TCnquU6o+AUkpxoYDUCKaESNbWdQ2lqKqqbVtK6WQyoVRv27asqrIsMcadfm9vvK+Ums/nWZbVbaMZel3XBOFd9nkymVim2e/3kySBUjHGdkpoJaRhGJRSjjEhlqY5nPPdEKVt26oqIYSEEErxbmGxMztIxYUQq+UGAGCaFmOsKArDMHu9HgBAKdg0bVVVgivLdAzDkFJWRV2WJee8rmul1O5YjxAiruuWZXl5eUWw1u0Mbct3XX8+X1N6nef50dFx09RCNpy379693UbrbSWxqTud4NnHrzzPXUdrbJPVerHcrOJF1MKqUW1Wb9u0tCwrGDlZvv3v/vt/9ad/+ueff/YDKfkXP/gsz+p/+29/Pp0+xHGMCQSyzYukqnOpmodH2B+PO90OszXHsTjvXr6/Wq9Xi+VqMhmenh5Pjk7Lqj46PneCoCzL1XoRHg4QxGWJEQIEEiWUZJJCajlGHEVVlAE3OBru7/ndIs+FYN99/evH6ZxoVCiICIGYZmlBiHZwcGgYCClWFrXkyrJwWVTLxeof/ubfZVn24unzLz77ATjhiZ4s54u2qvf29hwLSQWKsmoVM1zT8lyHtUVcJtvINuxuGAKheMsll57nGVRjTYMQAgBQrAG5M5OCh4eH0WiPQv3w4CAv/hDK28bJ7e2tEKLb7fZwv2maJIsdy+iG/mq1SpJk4/iD/mjWTJGiod/FSNM164vPf6gkXG+Wtu0O+sOWNcvl8vHf/7uiKIqi8LudVbQFGH38xWdEo5toy1smhCjruqqqqqpIv48QQoAYmm7qBm8ZRARCyBnYbuPj4wPbdKqy0TUbCGRQg3NOiFYlxepx8bfr//DVj380e5jF222/20UAfPHZZ2mazh5m79++n+wdgBYkq2Qz2xRFgxDbLlPbVpbuPdzMHL0TuoPJ4EApAQTSoO7ojoFNLImh2a4TeJ5nGg7nstcdnZ88tSzLNM3NOkIIZVkiBPMcUye6lBJjXBSN67pSCiEU57xpGEUUlO147zjfxCmMdapXRX1zc11VVRiGi/nUcZw0Teui1DTt8PiAt4xLuVptCCGrVVTmxcHBQeB3V4tlEpW//83r16+/7XQ6o+Fe1TYtFzb1iGOzRmZx4XjEMDTORauaqmqorv369W/DMDw+Olhu12WVV1VmWZbnu5ginWq6rm82m6ZhQkjHso4PD3YXHSm5aZp9v6tb1PXMMAzevn0jhfjw4cPj3X0QBBrRq6rpd7q9Xs80G4LseFNKQXeLzyQBoe8AqFmWmecpQNDzvKqqESQAEoS1NCubhidZ7gddroTrunmeZ2lZlqXveuae5bs+kIoxVrdNWVUSyoY1aRmvNvObu8siz1rZeB23O+y4HU/XTaoZxDDDMHz+4unj4+OHDx8sy9pdeYMgePLkyfX1teM4QRCoP2h5wWq1iuP4k4+ej8cjxzIVkEwwTCkAsizLsqkt1wq7HS7F7f3NOlr3R/3TZ2fJMjJNs8wLhAChyLT0/f09z3OgEo/zmZR8G2+KqkzyxLQNao0aCGzPbwX3wwAh4HiupuuP9/dJluzMTL/89a/7nd7L5698P/z669c3766CTsiYQEQJxZGODM2ajA/fvP3OoDcEUM/zzk9e9sL49vYWKc0gkFLNMnSDaoZGQm9s2+7e3kRK2el0LMPsHx2enZ8kSRKGvpT8629/1zbtyfkR4MrSrSJPDUId2/NsbzLYP5gcm7bx/ur91fXd3t6IS4pYK1vGOW/LyvM83/XqusYQPd7dH072As/fLlZtUWGClFJt2z48fr87jbVtq5RoeB0EQbfblVBy3hqmZpgmwmA3xyYUEioJ0cLQ6fcHBGtRmmZpISVXgGEkLZN6nm1aelUVgmGowKDXc1337uZmtVzeljd39zcvn73c8bXquq7rVjN027II0WzbNTTTYDkkEmFKKcUYsrZAnjnsdz68f3d/f1NlSdcP7u4vbz5cFmmGEGpLYRmmaxoYYwHUbiCPKXn9+jUh5Pz0rCmr9XodrzeOYZ2fnOoAVHmOdHMQdM6PTxardeh4GMBtEusYQaV4WRzvnz09PdKwmM1qEyOkBAZQw5jXdbTahF6n1x0hTIXEs+U24OjFJ5PucCzJCmN8dnT6YrG4vrmbzpdlXUuAEEBCguFwyBkriqwuSqAEIhhhjUJqAPa/PrOpYZoIkd1IIAzDtm2TJMnSYjcJsG3b8dxtHAEAdqe6P4zVLUvX9aqqDKphjLvdruM4TVVjjHVdzxpFsKCUQQh3zu6mqbIsm8/ngnDB/4B7V0BBSCjWsrQoimI3tyOEmqblOI5p2rvwQVOzHUxi9wdCiLZmu6OGEAIAgHeVUF0n9/f3L1683N9v7u8eV6sbx/G++OILpdR0Os3zsm3bpmlczxwMDu7ur01Ln+eybJlCkAPVSB4XycHxwavPXv1Pf/k/zhZzosPxZIRNmNZVby/8wQ9+sJ2u/vIv/2H+3/+//v7v//7Zk+cfvfo02iaWZdZlFQSeaepFmTPW9PuhlHI2v2e/vD47O9MIOT4+DsMw7Lgtq5ng3klgWjpCACE0m82K29sduw1iYOi6ZVms4UiRtqrjbbRdbX3b0ZFmEsLLpklKjeLAdACUL168KMtyvYmyItcs23Y9IZkQ4u13b8KwS4i23cQI4OPjkyAIzs7O7r79/rvV+uHm1tXsIs51avCGeZ73q3/8FZe82wsd3zKVHl1FSokXn7ykjH7//TsIYa/Xw4jc3929f/9+s9nwpnFc6/DweDTsWpaZpFGaxlmWnZ8/dRynbbhl2FW93q42dVnVZZXFSScMu70eV7Jt6yQLIIRXVx9M11utVm3NvvrBDx8fZm3VTsb7Hz7cIKA6ne7p6Zlt21K1i8WK87YoikVRjcfjireGaWZFjjB+9dFH13e3v3v9+7aqJ5PJs2fPPM97eHjYRFFdlsgmmqZpmqbrJoTQsu1OpzeZTGzPF1ytViuMKefCdV3GmGs550dnZ+dP3r592xTVw919Wzdlni2XS0KI5MKx3Lps8iQniLqWjyVWHAGMTd3RiX04trabJF6naScTLdibTD5/9XnTVv1+BwPUlq1jm9ttTKleVU1RVEopCFFZVovF0rVtiFQUbaTkOp0YpqYUpJS6QbeqqrKsQz+AED48PMRtQrEmhHj35mK+XBwfHxOd8JbHcQIhaqLi8PCwYXVWZkN32B/1F9PF1c314+Njvz+s8mKz2UCIwzCsWxH4PitAWylWgyypNtutQtiy7V6nc/84XcwWaZoRzajrFhPCOfd93z100iJdRmuB5HA8MM0DAGSepTsXs+/7QgjFxS6E67vuq9Pjuq6TPJGA5VX+9v2bqinPnz352c9+dnJ69G/+zb9ZzRd/+id/JoRYzubaaHx8dDSbrhSne8Pjfm+fEmOziRaL2U4Gs8PaG4bhBT6AiDFhGjZGtCwSJlSaF3leNoz5Xsha8fg4u3z/AQN4cnR8enIShqGGCTZN23V6/Q6hsNvrSSgBAvuHEz/0HMdxbC/s9Rzb4wqs1ttut6tp2osXL3q9nlKqKIodlc9xnC+//NLzvE6n8/DwcH19vbNDMcZ+/etfPX36tNPvZVlWVpUbukTDvBUCClt3Xdf1fZcaNMrT1WoVp4mBabKM8zTTqRF0fMsypORZlqRpnCTRhw8f7maPDWuLpt472OdVa7telCSr362gFJapt4KBRr2/vLy9vFrOl+PB2NLd6w9XAJAXz15ijDVNdyynrdpNvAIEeKGrGdqov5dnRZmV795cnp2d/fCHz/bHx7wBdV23TYUgZE0tAUzT1DI9y3VM2xl0Bxjj9WapqooJznl7cHDA2/qb7177vu/77rA7JJBeX97c3t42VWuZNqZE143RcPIwnW/WSX84ocSMt2nTNEIIzpmlGaEfrNm6SNJRrx+4XrzebDcbQshg0Nd0XdQtIdqOpe26LmNNGRcAqCAIqEGjaCMVV0AohZqmFkJUVWXZhmEY/X53MhkjRBAGbdtUVbVcTOu2MU2dYN+yDIwVgNxsqaZpQRDcAZCmiW3a5+fn+/v7dV3rup4kSVOWSinBlRCiKAohRGc8VEoBjGzb/s82o8n+ECPp+Y6QDWNVmsaPj/eCNePxOBwOdI1qGFPXSetys1nXou31+4cnR7/79W+yNKnSvOP5k/EYAZUmMYHDKi8267WpW0d7+1DBpuW8aZP19vTJCVcijWPftn705Q9C3ynyBMqWAOoYulIwitPNZuO6frc71HSLCTVfxUnOZssIEjsrea8X5BUb758cn7+q6zrJi7yosqIs8vL+/p5Qw/WooTuSt1Lypi5bqQwEbNu2LItz3jSNlM1O7LJcrjVNMzQqBdgN9neu1N09HkillBJCYAyllG3bMtbudkkQKYzxDt5jmmYhdyLiRghBCMEYF0UWRVGWZYRgSimllFC0W9NoGqaUdrtdwzAAgFKy/6x6AgDsQg7/q0oGti2vqipN091EYdfL0HXdcRzf94mNuj17EvHYd9of/+hgG0cXV9cvXj3vT8a1bKllZ1UtoWGa4dHhS4LxoiSGbdXM3qYibUuns1e22bfXH94/3A6PR0JWUbakpuzaIEun0wdL8wcvvpxsNlHKs9fvv716uBMtzIp8G8e25RdJAyysYw8wlUbRdl5/95ub2VP41Y++/O3i0fY3H338QkBvEX+X1Py7q+960zVAWlE2UpCq5N1wz9LsJM6yeU2Q5XlelmWrdXp9/3B8dGTpmm1oZdtYogEQt3VlmiZRmCj48umT7777RrJSk5RqCpRLJ9Su3nwwDMew3LPTZ5CXlts5mIyefPRqmUSObT+up4v17L/8F3/x/OxJkae/+93s7uoDwvuGNcoz3PI2TpOqzk3N5qBebNYIt1WWWTphbXR/fd9U1XDY3594WbYlmhN2LMNG/ZHf747OTs+Lovr699/86IuvINaghC+evcRI7w36nU6napuDydhxnJrVt7e3RFFLEt/3A9MY9nxCyN3ddRj6H66v49cxoej8/Lwoiv6of3V15QXh5cN8DVCv14MQPjk7n0wmlmXNOT8+Odl9UBolHNd5+dkn33zzzSKJRK4Fng+EuL5+gFL9yZ/89MnZ2fThfjqdlmX29HC/KLOS17ZmL6JVlS8lhEo0H796sd3GH3/0xdlJy6Xq9w+jSCjlur43m04Nu/P5Vz9++/ZNDaRr+Ofn551O58O7i7dv36bR/Pe/21oaOzw8Biz7i3/+s+VyKYTQKQWtahSDGl9tZ7PZ7P3791VRappuGIalG7w3CCrXdV3OpQS0KEESpbqus3K+WCweH++/+OILRBBE7fvLi1/83YNlG7PFFGtknkKIQTA2vOFoE21BIRaz6XA80nV9upiP9yfrIr66ujo+Pk6aXEmJEbpf3E9Xjwd7k5pl+y+eAtd6eHiArjnpnU4X82W63d5fTuezw2f7XMmqqceTLiIEAOD7ftC3m4blZek4juX5vhdyLgB2sgI1LVbK2R92qzLPkzTbpL/529/ogBqGMRjsEUIefvvb17+8Xa+Xl98uPv74pc7dL579SD5tAVOh71y8Wf3yl9OvfvTTYNDrjw48q5vGZVu0k36P8CaJ12EnvLm5EUIM3MF+72jaLB6iecq40Oj1dPrmw4XvuyUr5/Op55tJvFSyoVhuFhtWFC6loWFzxD3X6tiWbdujL3+8o7/9b/7FvxRCdDodQshwONxsNgihg8HAczdKKQg0CE3HHziuv42TyfFHmzgaHkkolaET09AINrpewFkTb9dNVWWrWbJZrfMM6XS9WV3ft4fnp+fPn+mWfXB6DDHhnNuej3B8//Curuuj3iCK8qZphQ5934dI+903b6LtOoqT6XQOhNR1nXPe7XTiy9tOJ5Cgns+WnLedXhh4FqEMEv7skyMO8u4o2K7iUuSbNP75L37ech764cmzJ2VZEggcN5zP520tgiCoqkJl2EauxjQHOjrXq6rSJRWcdUZjIUTbqKZpqzLnDdKw1e32NdOr0rTf2bv8cNXWOE35el3qZv8wPK/rWgiRrmtdV77X5QwwEXMoIVRSimi7qbL8fP9I5+DT85ez3jTP89VqwxoFSJ1kc4CEbmAF2uGgr6T4x7//e99xCcSi5axhSDLftrCSURQJoPYGe57nTB8eFWDHx4c1K+fzhyB0Pd+SiAeH/vXVva4TADOq1f3BoGF4uU5tF2sG1XWnrJr1ZipV7fu+7waMc16V8Wo57g83m814MDw+OL69f5SAnD1//s2HSwBhz3OirFonkcpgGIbAE51Ox3I8zitiUIngzcMjqxvBwXoVH+6NMSTb5cLSyHgyGg2HNceTg/00Ta9urz/6+NP3V5frm7t/+pM/MgXchvfv3r0jEBcM2IaJdH1TrNaD8eW7OyllsWXj8RgJFM3W+4cH2TLCtdQJhoSur+c/+emPm3V6afZuiwYIrGFDQRIGo5rJ+TLxu8lo7H74cDUa7UsBZndTJCDV8O1mXffHg8HAMIxOp7PXDbnva5qW5NmTw8n9/f3944MAkuq6lBQAhLDu6H+IjGw2kRDCcRzLcjCAWKNl3VqW1emEbdvO54vdzEkoZhgG1amUAEK4U9nppoFb3Q6C06BXFIWU0nR7Simia4e6Np1Oy5ru4muccwihBLLT62hE+//x9F9PtmTpdSe4hWvtR6vQcePqm6JSoFAFoACqJgDSGmwb40OPccz6vf8dPvTDjM3LmM20mOaQDZBAkwQKVYWqzMq8mXllxA0dJ452rX373nseDoljdp7jmJuH+7e/tdZvIQ5IXdO8hg01FKWtu3JLj6IoTVOEkCDIRdmkWcE5FwSBcQ4xUDQZYhBFYbEpAACGCob9/0IH6PV62/Gl1WoJYbS+vDmzbfvd+9fD8aggNRbR9fXleHdnMB7tjke7u7t5ll1cXF1f347H4+P9iawoUIRp7lclVZWWoSJDRev1HcKN27Z7fV1RpKrOkjCom0xiRZSs0oyYuizKsqJCIKMiA+NJ23FkJFRVXQNBViTZdiRRajesOju7uLq5MS3tp3/wu4psqkoCuEAIh0CoKpLmWZrlum5butFy7bSsAGCqoiCMRVGklHvrdZ5mvuc9PTkxFFk39KqooSSsluumJv56QQiFHLTb7TAK6qIUFbHXbWOMDF03TKvbGWmyQhtQ5gUE4NGDIwlD1jTr2WKzXi6nU5HzNApvry5ZU+uqXBZ5UZaaaZRp/tWH88cnTy1D67ltWzdATeqqImWVRfF6vbYMXdfVPEkDz1cUpSJ1HIehnyRxWjfU87yqoYBjTTe7g6GmaVmSWpZ1eHgIIU/zjHM+6g8oAWEYaprmOM5wONxmnzRNefz48fX19Wq9ODw8jOOYUh4E0XaM3ZprtmDd8XhMKV0ul2EYtlqtfr+fZdlmszk4ONgeAUUV5XlWpBngfDwe26Y5vb379ttvIeTtdvvm5ubBgwc9RWaA/uxnf/jm3duZH452D3TNLMsqSfP5YlOWtWk7b968WS6Xoii0O+4D4eDB0QGpirOz077b8cOgqMqFt16sV9P5rNfrWY699laqqmKItu/soijCMCzrmpQFhHi1Wq3Xa0JIr9NttVrD4bDbanc6XcYYqWrO+WazSZLEMAySZd999+2LFy8UTa2qcjgevPz+W8/zKqI3TcMASNKUAwoA0Ay93+/XSVnX9XJxX5FaVdVhv7NcWepS+fSzT1bL5d3t/Wq+MHT96OBYMk3DsufTtaZppm1tzcmdTqfd6bCLi4rUk71dCOFyvaKUcgC2w7jvB4SQvCybhiVJslH8pmnKvCzLUpEku9Rff/dyd7IjI+Hu9nbw+ed1Xfd6vbIs37179+7dO1VVB4MB4/Vf/dX/Ob0/Pnl4dHC4o6iYMQYga5rm4uJ0b+eQNeDD7PT+dgkIbLudOPAtUy2KAkI+Gg36wwGEnAJm2+bueKQrap6nooBUTYGQr9fLy4vT44PDKPYaWmu6NOh3J3s7/WG3KirEgSrJvKEbz0+SpGmabdRqvViWZbllIZRliTE+ODjY2dkJswQhVDfUspzZfMkAbxpm27Ysy6whiizuDAfDh08pKRhjkFHQ1EmecQjiPHt38eFmNo3C5OXL7x8+eiQqsq7rCKEoik7fvGrK/NmjR9dn56Io2qqlaZpjWhjjJIm2gbHj40NNViil1zeXs/n0s88+e/Dw5Ks339ZFGWdxXZeU1QAww7YYaZ49e2bqlrf0v/nq+x8Wb0Qk+b6/N9lDQIii8L9avJ0tzTBJki1FXxRxmuS//frb65vLPM+Pjo7evXs3HA4t09nG5YMgWBueLOtWkkRh3O/3tzWYl5fnW2Ty1iO2Wq0uLi4QQp1Ox7KsbTBKluWtIiPL8vHx8TZi9/mPvvA87+bmZrPZUEop5YIgGJoOAFosFr7vy5rmui4AIEmSuiGqqhJC/kuPEcaqqmJRoLQhDU3zLIp9hFBVVRyaw0n//PwcIZCmcZJEWZZ1N73AjzxvwyDstHtVVZRVqciaaRqyLFNKFU2WiMY5p4ArqoEktWwopbRhdDgcOqbhhQEldcu1VU2er5b3tze0tljDB0iCSKANwQCrkmpZ6jfvv746v/gX/+xP958+FZHY6/T29vZkSQAlOz8/J5RjJMZxPJ/PIRLCICaEMAAgRJRSRgEDqKpIlmUfLi8Jo6Sm94t5kKaapmm66bhtUZIWqw2lRJIF27Wurq4YBS9evGCzpecFZV4oqqaqMhbB5vb+61//5uRRvFyu0zSVBLEoszQORVEkhLScuq5rAEAcxwghyphtuwJElm2cqCc7OztBFN7d3W+h7IyxRbI0DEOW1U4Hb++i7WerLDRNs9lsyrJqmkZVVVEUIeNhGEKI+/3+aDRSFGX7DwUhLLKc0GZrd8AYU8qaukqrijVUFsV+t3t0dAwxKuoqSRJCCGcQUAYBQBwgADmHnIMPZ5dbBK0oihywbXrCNPW6braSNOec8YbxRpSwIAj9tjMajbb5C8MwEEKEEFEUheGkXZOkO9yfr26B0IiKrOra+dXi6OQoy5LvX79q2S3btjd++OHDxcuX3+/tjSeTyc7hnuVahqNrMv5wffa3v/xrVYYIC5IIJUmwbA1j3TS1TqfT6g5Xy7kkRKZuIyiZpixipcrz2XSW5RDDRsC4wrisBUppXVad7ghA0TA1t2NLonF5Od14yyyty7rgFDEKeEMhB5qiKqoOIVAkmWmMMSAIgiQqZV7FYVRkOebYNG0MBctyWF1pqpbG2Ww2kxAv8vr16/euaw56A4BYEATBxnv6YlQVVZ5klZb5DbdMFxAqIeHR4d7haPD27dub0/c3F+dVEqnSj23TkkTU644O9vaSLK3ybNBpG4p8d3P1l3/+Fwd7O5999pkqSgKADeUyFnRFxe2uIoiAsiJJwyQcDAaGqmZRWJNm43uUco7g7c2douk7imFbbsPo2dkHPwwRQlGafPPN10mStLudltkqy0LTtG63u82xpFHcVPXh4WG/2w2CYH9//+3b1zc3NyLClmPoAAEABEHiHHY6PcOwXr9+fXNz12q1AEC27QIA0jSnlMuyyjn8Z3/8x77nffvtt4Cyj549n4x2fvGLn3/3zbdN0+iW/vjx4+VybbnOZHcMIP7k4x8dUqwoynw+P724/vD+w9u3b5umOT4+zuIkj2MI+XjQAQ3RFGVvZ+dXv/y5IilHD0/SNP3Pf/M379+/Pz4+7o2GRVNrugYgqikVIcKQNoxwyAQFG2qLc14ULUJ2tpYax7IVRUmLXMtTQighxHVFXVeLIkvT+M3Ll3d3d5/86BMIQZKliiZrpkYp5ZwrmiEIyLZtWVWSJGmaGkKcRitFUdJolRcV6LSieF2TvNN1r26vNU3/6LNPry5v7m5u7lar28ViPB4/GB8mWbrVHdMi39bfYVFo97qtVosBXtZVWddpmm58L80zRcUcAk5pXVdFkcE43g7puqICxrI8XS2WrmFxTT88OLZ0i1KuKNo2zwIASNO4qgtZFlVNOj09X63vN97xs+cPP/nkI8o+ubi42N0bTybDq/PZu7O3WVh0nG7dFAXJaFQC2ORlJquSZZmyJldVdX9/P1/cU0aiwNM0VVNkRcYt17y6PAu81ZNHj//5P//j4WCgYFlT1CzOFuupv4kEQSCEvHv3brFYbEPV3W7XMAzLsuaz2bYCgzE2Gg6nd3cE/Zf23tV6/vVX34Rx3O30Hz161Gp16rKqy1wRpSAILj+cW6ah63oYB7Ztj3fHO4Nxa7L7UZ74kb/arLMiD1erVBAwhkVR0CziRVIEa8sxFUnebDb393fbPLquyt1ulwG6OxlTSm3bbrXt3/72t/P57KuvyIfpze30JgxjWcMjrw8A28f7giDYtskpjNLo4vK8KFh3bJckK5u81XKbhlBK0zQ9Pz9fLIp2x1VVNUmSskyrquIc7u7uuk47TfLLi+tury1LKkKCquq97kiRdYzFqqoW8+UWfc0Y0zSt1eoghBACEsS6rjPGwjDchty2xvKdnR3LsgghFxcXnuc5jtPtdg8PDweDgeM44+EoSbL1ej2dTn3fD8NYVfX3Z6fVtvJYVZqmQYKkGVZJCa0h5VDA0tZvL4piTUrNUCVJaQhtAA/XG0mVPvro+Xw+X61WW89aEHqzxX3TNHXdyKre6boIYcoAAIBzEEaerpnj4VgW7NvrGyA1Dw8fjYbD64tzCmBd16/e/FDVhaYppCrSMAcAqIKgtduQsyovYz8WRVFAIgQihIQD1pRVEUfBahNPIkPXLcNUJCVN05//6ivO+YOHj4uqfvv+rCjrwaB9eX39ww8/0IafPHyMIMzirMjyKi+2tlnLsqIwqapKVWMsiYqsvj8///FPfqwoCmmq+fx+tfGLH14JgnB0dLzhOAmjklSNIDY1EbBMSXO/uG8aWhHKOVewSMoqieIt7Tj0A0WSFUVhjHIOOOcCxhwgWZarIsvzUpHkw/09XVevrq4Wi0Wr1dqilBljpmnbti2KYkNYp92TJIlSFgTBdvJACBFCJUWu61AUoeu6/X7/v7p9c9M067rcLp8oqSmlURRFURRvojhJoICfvXj+xZdfapa98jZYkFTNKIqirhtBEDRFRQhVRZ3n+efj3aqqOKd1XS9X8+VyDgEyTTvPU8ZYnucNrSEEW4ITAFBRdct2ZVmO47goawBAURR17QuCCNb+xna0TsetaKmIWpKnYej7oXdwdLhZeVEUffhw8e70DCBBMyxakPX9Mo6iilQVrY22wQVepIRkoKBFnBYcNO1uy3VtSpmisL299rC3p0lJluXBJoKNqGtCmhZF0QiogFBURAkjUG7RirTRFOnk8eOHD090U7u6/vDnf/6XQRwgBAxLFwQB8CLJ0rpqiFEYhsEoQUhGAAoIcsqKpgAAuLbDOXjy5Emv3QsDTxgJWESGZglYZoQdPnp8P7ubvvx6Mhk9fvJAFNG792+SJLI0NdfV2A9Cz4MgcXSnTLMkySoBTSYTTcS2oQ46Lds0RqPhZDSmtFEUZTgcir53c3Mzn84ZY8HK99eZIszqJ6WMBcXWGkXDCMjSriAIW0iJqRsYw1F/YNiWgODh04/zPN9s/CTJLi5vWlCsaVMUxWAwFMXrIss3Gy+KIm/lhWFYVSQ3cwCAquaUMkmSthlLy7IWi0Vd1+/evdveoxBizuHV1U3e0O1bR1VVTVElQWxqsr+753ne5XxRZPmLFy9M3YiC0DLMttuqi1LEgipIogYd22akKbPStu0gCPqd/u/+7k9FWWIAGLpzeztTFOV86bVaLdo0qiQfHex32y3OmGvZeZ7m+b6qyYqi0Kq6OD/f29uRBPH69mb/8CCOYy8KxjuTP/3n/2w0GpZFUVOi6zogIMkiL6irquKcW6Z5fX0JAIjjlHKmqpIo4rohaZ6OBkPLdeI4TtOYEA1j0Q+Dm5ubNI2fvXjGGFus5pvNxo/8TqejGbpq6AAAJMBOrzcYDC5vrl69enV5fTuyEBYkyipBBJzTd+/ffvP994KkRFmepNlgMCKEUoR11727uZWj5Kef9t++fTtfLcfjMeZssVzarqPret+2ZFWhlB4cHRmWOZ1O37x5E8exINqiKKoy3kqSDW8wxhKWAEB1XqwXmWkaeZoByv6H/9v/cH5+3tTE1A1N01RZA4w3hEiCOBz0k0Tyg7WA4dXlB0KSvd3hyYMj1zHm3hoLiKMGC9xp2YZpYgEihLIq02QJIdQ0DQXcMCxCm9dv31CMP/vsUwGjXqdV12VRoJ3dSRgFnFPD0r1o422WZV7piooA2mw2MjKePn3a6XQ0Tbu+vs6ybDabxXG8BSVtufEIoTzP4ziO49jpG3VdAID8zcbzl0EQNXWdpdEf/dE/GE9GkPFOp/Pu3bv/9Ne/sCxL07Sr+xtJkW3btmyj3+89f/705OjR45PHRZkKAlou57d312Uc26rGqmI9mx4//9gwjLqpZov7ipRijnirtWvstrnDAPNDvyaFrMnj3THGWNZkUzc0RS2kVNzGwQUBIbD1ey1W88VyXpF8vOO22s7rd28Wi8WXH/3k5uYGQmhZluM4hmEIglCWhSzL/X5/s9nc3k4ZAwcHB4PBRBCE2exWkqSqqsqiabldRVGyrEriNIlTQmhRVGHoi6JombasSFmWVakviqJlWQ8ePNia15qm8X3/7u7Otm2EkO/7cRx7npfn+bNnz+qSFFlZllVRFGkUV3kFGaSUYiREYQwhrAn1gwghxACXVSVYpXVdU84ESZRVRZZVURYEGbuug0QgKvJmsV5t5hDD5Xqzu3fw8uW3sixvD46bzQoAZBiWrqvb5TOAcJvro5SWVZ5msTXcN1tVt+U+f/YkicOb+9nN9A5hkKbxluIMBWyY5nA4JIxXVfXw0SNS10VRVHmFJFDTPMjWcRCCun728HGWpFmcPHn8rCyK2d307u5O1S3P88Iovrq6uputHNeSFf3Xf/fVxcVVp9PZneyIWMpB2VCeFuUmCAeTYX9n1+03CKE8L26u7+4X6+k0oxD90T/4Wbs/yKry9esfWm1H09SHj564jtVyXAYgh5izRpCUtmuXZX1/dyfLimFYjJIiTyGj27boxZIiDDTVsIm9vcjbxUBRlrpuWqYpimLVEEVR9nf3RFFcz6//3oWDECI1raumqiqEMMZse2bbjpJ5XjIGdLNj27bjOL1ejzG2WMyCIFAVBUHYkApwCmmTxFEYhovFYj6fK1DBoqAaBqtIkeclabIssyynoZxBVDNOSAMEqqqSZEmCock1RgLBGALAGg4I5YahTXYnURQwCDiCdV0CyJumZowhDCRFL6qmYbBueFGUeZ6naVoUhfDNd98vFovHT54NRsP7+dI0TRonu/sHvd5gb/dAU437+/n11e1quWm32512W+XSaDTYPTqoSPXdm+8/3FzUvOKCWBWYMBk0vGmAImMMYJxk3pq42i6iBmZgOV1dfAgcO3ZMZzbzLEMrCtjUhSBWIsKSJBiGoTuOoWqKJtW0EGoYxNF0NiOEKKrUbrctx95GQlerVVXU3Va73+9PFysJC5qmEUKDIAKMm6alKyprWBwnq8X6we6RIgmUsJbdEY9FCcu6bBiqJUAhT1IOmiJNIKdZErRdGwPMKY6jQmCgLCp/sSKY99vtrtP6nR99/vTR4zzPCWlevXs3vb/P8zwpipbjMMaTJN3d3f3pT3/v4vyKUbLt2M2yrMoLUpeDfncymSwWDCGoaGqaJ5qmCYJg2/Zitdxs/M3a5xwAgCy3xRn88OF8MBibhp0kyWbpkaYa9AaOZXPOB92hJEkIIUkWCCF3N3eUUgHhD2FUVdVvfv3rs9PT8Xi85SuEfqA7bQRQGqW84WmUzum8Lup+p399cR0HcZ7k+zv7siwnWfLxxx9jgFlNdkdjCeFg4zVF9fbian4/63X6/+xP/vmjJ4+Xm3VZ1DVjcTYdjUZ3s/tPXjyRRWl+P7s9W3nrtWNao3G/5bjbgJbl2BdXl0Ecnb598+DwYDweh3GxWm4QBv/oH/2jXqfbarlnZ2dFnuu6+vDhw7Zrk6YIQi8MfcZYnkVuuw0hBBitVqvlcs4YcF13b2dXUmQkIFVVqOtwBFfeygs9CPmzF08fP34chuHL774B/5Vi64UBR1yQREEUq5JUhDYNTdPM831HcXiSBGG8e3QwGu+cXV6tvUBU1AYKs9kySPJWqwM5qhtaEIJlCWBUkjrPcywILV3PilxRFN0yt5iXhrH9w4PBYBAlsSCJqq7VZQU5EGVBQBBjyBsuYiArQhREWZoywsq8yBGWJUnX9clopyJZx21Np9Nf//JX11eXO5Pxw4cPdvfG33z7mzhaq5LY7Tl7OyMEmjjc3N9dv72+tCyn2+1++pmWx00WFrSmsqqIoiDJAhIlDsXlypOVeRLnlKEqLQ3VePTgoaTK55cfKKkcp3V4eHh5eX4/u/O99aDbGw3GLdf1lp4f+m1D2iLlp9Pp+fn5FpZQluWTJ0+iKGq1WpIk1XUtSRLn/OjoKCz8qsq2gfL93cnjk4eSKF5fX9/dXB7u73U7PcuyOIIcowbwME2ApCRFFST3zVWlKNJisXh4c6yqoiTi0WjYcZzO0xc1qYLAm06nnr8uqnw0GT7/6Fl/0CmKYr1eZ1nm+2tFUWpad7pOEASEEEkRG1L1B+2Hj56cX10ul3OAqeUYvWFPluUsyz58OI3jFGJwcHwgC2pTsflyESbo9PTUdd0nT55gjH/5y1++fftWlmXTNPf29jrtHqnpeuUVed0QLuiyLMmj0aTb7YdhvFnfX13eyLIqScpoOJYkRZLAtuwHQhiGIQBAVbQi3mx/oWmaw+HQcZwt8eb6+np7JYfD4cHBwXZ58Ld/+7fHJ0fbYyIhhFJuWZahm2EY6rquKJrjOKvVZrXxt65VCCGhjAEoSQoAQJQViAWMRSQKFWmSICzK2vNDAFCapr/41a9+//d/X5YVxpggYEURQUW2zWdpmr9/f2pZliSrlmUNBqNul9/eTL/+6pujY2l/d+/45DHC4g+v3pyefyiqcr5cui17srfLGAvCkHNYN6BhDCKp1xtlSYpRxJoGAVjlxWp2P7+fAcr6vV4cx73eQNOMX//614tNcDtbYkm9vFo0lFdNI0qiH8av373Ps7Lb7S7X3nKxERBWZNnSDYQlUjMmCLrb6hsGxuLlxXWY5xShwVifr9bLjccR3D88ipJ4Z2c8vb/1goAUhSIJummLsgYQVnQLY5EQOpvNMEJNXVUIAkKJnKuyIstyDnMBIlEUs8SRVAUjsalJXddIEDBCooDqWgQAyKK4bffWJL63t/ejTz9fr9fX19e3t7dhGG11h7Kst2e2brdbFrWiKLZtQ4mJsrT1G2ZZFoYhqWvbMhipOSWKIHCM8iwWMOi4FuKNDCTbbeumoWvy+dnZ2vdFTXn89DkFEGOsqjIhNM/zLehJkTU/iLaGYlkWBVExTEdVFSxIvf4QCVg3jW1BaFVVVV0yxkzDTtMcYyxJEkJC0zDGAMaiEATZdLr64dVZw8QgzPaO1MHQ+eRHnx0eHt7d3Z2ensqiopuWYVtVVU3v71tmxz9Lb701h/B6ejNbbwRV1AzVbvfruqppnRepiEwRGawusqIpUmooraZEgAqcgSymVeYnMWg5IkJIURVTN7AAIeSKpummgYWm5sl8Xem6XlaJKGHT0iGE796dHhwc7O3tjYaTumriOE6iuNNqAwAQBq5uQ4jzJN/khSCICpZvLm9EKMhYMQxTkyRO2f7+oSLJqiRf31xqumqYkuPqnFV5FlRVmUbRaGjsjIbeJi6iiuSlrRmgP0YkK7ICANjqdHYOjjgEWZYt12u43Cw29378/Wg0mU+XjDHHHQyG+6btblaLBrB14N9dXSEAl/P73d1Jp9ulnHEMOQR5VYZJrJuGrqubJIuzVFKV4XD00Sef7+8f0obf3c0Mw9jb2U+ShDLCGJuMxhjjLMv6/aGiKFvsGiEk9CNNU548eeL7PoSAVHUcx4NefzQacf7o8x99UXJQFMXd3V2v1+t0OtPptCxLCNH+/sGDByeMMVXVZFm2LPvx4yemab35/mXkiUUSy6LUdluaorbd1nA48sNgsVh+98MPi9V652D/+UcvLKf12HKuLl61Wo4GSUdFaZMVXrqqwkAUHz96ChCiWVTGIavqNEniOB0MRqSZu47Tbrc1Sbm8vPyrv/wPq9Xq5Ojw0eMTzJksCrIo1FXmLRdVVdm25bZbtmMripQk0enp6d3dXb/TbZrGdp2iyERR1jSNNTQIfFVVXrx4Meo4eZ5zxN+fnQ2HQ7fd+vkvfxFFUZy2HNd1XTeII44ggPjR46dN0yT+zHRbAy4enTyXdY1eTgVZ0wyrqmmr3e1NIur1AABjFklEQVR0esPBeD6bBUGQxnEax2EUdbpdz/cJIbppdLtdgFHDaJIkSZYVVQkxCuPow4cP6/XasCzIACEEAAYxAJwyWpOqQZBKIqKSKOvy1ftY328gg/+///3f/jf/5I8/PnlOSPPDDz+sVqvtNnI0Hjx//rSqk/OLN6omPX/x9IsvPxFE8OHD+9PTd4rtRnGKAQVAxFgEsKaclRXFEKZZyTkvq+bduw9XV8uqors7R48fPh32dgDkooyqsqaAaJqk63oYhhdnp7c308Vi5XnBaDDGEKmGvg78H96+sW37/Prq9ft3tm23Wi3NMjdhMBgMBoPBb37zm+Vy+eDBA1mW14FPecMoUhSlKpZNTQeT9ieffOL7z3/729+WRRIn+Oru8u371zUvVaSWtFZ0DUtYV9q6prCmLrL87P0p4mwxv9d0+eHDB1988dl4ZyRgMa9qjvAsDfIi0TTNsHSAuJLJNcnzKpM1SdWUg4ODzWaTxhEhZDqd+qFvaG0JCy3HERWsW6qiKLRpsjzx/HUUJmVZcwChAHVZffr8SdttsVggTbXxVqZh93o9QRA450VRRlEMAEzTTFUMCPDtzfR+Ondd90/+2T9sGuZtIkIa2uSUQs5QkmQIVYIgtNvdfn8YBN7XX3/NOX/y5NFwOPR9f5tYgxAmSZIkSRzHo9HIsqw8z7es306ns/3ToR8JgqBYMoNUQILdtrEgbD3qWKKd3sBpdfLZjDFGKKvrGosCggBAhBASsEg5J5RDzjf+YrG8lyQBYqHf72CBX1xc7e3t7e8d39/f13VtGCpCKiGEUR4GSdMEGMmG0battut0qqoipNmsg6p8Y6n6/d1tFPrnH06PD/d7vc7s/q7VcfcPDqAgvnt/dnl1O52vi4rouvnyu1eKIlmGaZg2hhxQpqqqZVl5kn44Pz85eWQ49lffvvz5L39tuQ6XlR9eXfR61njn4OrmGgkiIE1ZEEmSDcPkHFV5UZGGMaDIhq6Zw9GEQPT+8pIziBDK0wIIomE6/U43L9Jvvn8ty+If/uz3nVZHt+zFS//u/m+wguM4dlt5tzfaOTiynE4SF5Az17QgEiDntKoJ4yJUGGooRDWtAh9SSqMoUFVdluWq024IcxzH970wDBVFsSyLKUqWJXle9jstTdP6fbff749G493dvfV6XVVkuVzGcUxqqsiqaIjMYrIs27adlvFWhtge3GVJkiVBESUsQJEILcdWFMlQJEEQWm2HMfa3/+ffbrzNbH6j6DrAyItiJEpRFL745BPDtA2n1VCeJFmR5SQrQN202+26LrdBR0FEgoA4Z1VFOBcgwIbmyoooSeKWWMg5T71w4wWcc9M067rO8hIAJIiygKEBmOpvMtXQGRXznBmG3u0Mzi9v3rx5kyeJaZrr9XrbJcU5v1xcNIyqkSpIYpInSAWyIQCR1XlVNnlZVVmWGIYKoSEgzhra0JIDkXHSbTuGpmRZFkepZcPxuIcQMAzdsg3GmrLIMIaKBtLMtywLQJaVpRfOw3hjQ4eSRhAQY8w0zZ/87u999unn79+/tywLAXx+9qEoiuFw3Gp3t20oAhI4BW2n7dotU1NbdkuTpTgITUOHjPt+cHZ6DmBD6ty2VNuxLcNc5omIYJbEli6QIgecrhbLbgsAwpKkyLK5aVkMQ46K9mDg9oYPP/rRn/3L//7i/HKxWCiSdH56cXV5STheb6L+2EECBIxXpL6d3R/s7ydFHibxfLOSZZEjWNEGyxLkrKKNAoGqaSahmqZLskpo8/Lb7yllWVZgJG4ZHbIsc0AFQVBVXdO0Iq9ITQkhuq4DyObzeafTYYwVRbFYLKbTaRRFg8GAUTCd3pdVjhVt2ykyGY0lSYrDqNvtOo4ji5Ioit1u1/M8VVaGw2EaJ5Q0e5Oduq5ZQ0WEWdNkSRJsfMb4mzdvojjtjgZ7uwePnz57cPzw8ua61xucTDpxEGbeRm4yR6SA87YmqLoBqkRRtSBIDF3Fkux2+tPZ3LJbs19+tT/eQZR///Llr3/1d4TUk53Rk5OHlm6UWZpFooyRqahNXZdF2nasoigcx2l3OycIcs4NQzMMa7QzchwLYzHP8zosBUHSTcNtt0aTkQjIzdkNAKDTbXX7HQq4pilIgDWhACDVMBES4jSzLGt3v9s0TWhZw8lYX61Mtx/GUV4BzWypmjEej4uiuL2ZcgrKLJ/0R3wwpBXxfX8wGARBkJcF5xyJQpKlVVUhQZBVJcnSm7tbZb3a+D4DQBRFUNa0ppxTUcIIAABZQ2tYNqIock5EpKoaIISkcXL27v2f/tM/7bTab9++DbzgwfFxr9fx/c37t+9sS0+T+GBv//jBXsux18tFGHmkKQ72JxFTVktvvYiqjEEmu0bHNh3O4pJUCGFFUURBr+tM15TxsGvYraODA9bA5XpZ1JkXeoomxnFs2MbzZx9zBpTpHWDc90NvEyqyjDG+PLsXRfGnP/3p049ebMJgPp8Tb6Np2s7B/s7BPgCgZpRjpFlmt9sNw5AQVmREEtU0KefTOWL04cmxrimqIoqYqZqY3AVJGii6jBSgyUpZkuVmFSM87HclJAAIZF3vtt22bV1dXX391bfX17ejybDV7SiqirAkKmKUJlmZpWmaRnFdl6quCBgx1hRVcXV9maYppaTX67345EVdlN99+21W5rIqWcgQKoRyyBGllHa7HUXT/LVPCKd1QzHudFtVVbfttiiKjuNUVeV5XhiGmqZvVwK25UqiEsfz29upqqofffTRp59+QQhdLBZbf7GqmAgJDaGr1SaKIlXRNdUYT8S6bmaz2VYi3B05lmXZtr2Nzm/x/ttPr9fDGM9ms+Vy6bruaDTa29uDDGw17x9++OHi+urB8cOHDx8qilKWdZqXR4e1pMiCJEMI67KsqkpBuK4JpVSQJQkCzkFd1w0jQRCFQWw4mmmaEELKuG4aa89/9uBxEudpmqqKVZEacFHXdUFWAIfdbtewbM6E5cILw7DIabc7VJAWBZvTt3mRRYG32tuZHB/tQUR7vd58ub67u5+vPVm3FEsKb2fr2aaocsuyRoO+KIqmoSu6optGVRW6rlYN2T/aL0izjkIoy5s4dV33x7/zo3/wj/8RAPD0f/qf6oogQURYHE92MMattqaOFEaaMAgo5VAWO51ersGV78dx3BDWa/dGO7tN3Yiq2rHNy/ML2zbni1WrbemG9fTZi5vry+XmLouTMIw2fqDohijKSRxWReGYRkkaUjcMMSyJWEAYQcBZURTb2gvGWJHngihusQeev56Md0ejERaFqqrqulYUxbHNoiiSJPU8f6usHR8f7+0dMMaurq68TbDZbKqqwliEEBJC1mtPsyRFUSCAXuBXRYkgFEWJMiJBRRKwbRqttmOoiq6re3t7IsLff/31D6+vb++T/lDvdPuKhBvWLKY3w26HdipdFB3TNgQxF6Rt/Nir1lsAlyiKsiI6SqtpCCHEC+KiLBljHGEOAeMCwoIgCCmIAUCUNnGcxnG8xUBxzoW721UUVw3Dimo5br/IyXoTvn93/vb07cXFha7KgoBvb26wgA1DC8OQy/V4POoPBht/fffmOsniFuqYthnESwCAICLDFNsds9c1G5KkcTO9vdgGNyVROTqehGG4XALLsgxNLcq8rIPaC6q6KIpM01UgtHf29i3LSuIsjlPS5BwQLIC6arbXGgDw4vnHg8FgvV4XaT5d3/m+l2W547QUSbYsy7IsWZTb7W631R0MBoooMQoQFKqqghwsZrOX337/3fffPH50iATS7Vm9ntXrdzFknPM4jCATeUN1Rb27XS6nq80mtE3DcuzRzm4QRb/99oeCfM0h2jvYPzw+kWW5P5gcHx8/e/Hp+el56AdnZ2dhHIkCIoxUdRNE4RNNQ6JQM+qHwc7OWNIVWdf2Ta0sy9linheZ291TVd22HW8T3N3df/Pbl71e/3D/qCiKJEk0TVMUJYnz5Wxumma73VZ1M03TKIpkWSaEvHz50nVdSZLyPK2qahsE2Nperq6u+v3+2/OvXdfNsuzx48dFUbx+/frzzz93XXfrmfrRj340m80ePXrU7/d//etfV1Vla4pt28cHh5IkZVlxcXFx/uHCtm271d7dO8gJ2T84sFrtr7/+hiOYFSVcf//h9Kwu82F/cLI3RgAihCAS2qbe6g3ZbDGa7C2CqDfe2YTx7v7eeDjCEM2m9/fXtxjAg6Pj/YPdYb+3Wq2aqoSM7u3v7u7szG5vIG1swxRFXBQZIRUhxHLsbrfLOayqamstjuN4tVju7u5++umnW8LPzfm7NE055z/67DPLst6dnf7RH/1REIVv351utUCAYJIkeVkSSheLBRIVuAovru9rhg3TfPTsI7vdK4qiqqqnB8eKpJKyuv5wISLMG4YQSpKk3+/LqkJoIypyWmzBpmKaZQ2jYRyFYei0WqIobqHIDaVNQzCFADKEOGCcU0Z4nSVx6BeZmO/s2LGXBCX94kdfYIzPz8/LsjRNE0BumqbnrS8vL/IisWxVN1RJFt69f6tqWDcU2zYZY74XSJLGIaqqqkhL1xrs7x217c6H89OW49i2Y1qOaXWfP/3UafeKvNYxCvz4zav3F1dnfrQxbK0i5cHx/t7+zmbjh0HS6bYcy0YcQQ4ghMMdMJ/PGQLdbodjOF/7rZbpdtv98fB6evvu3TvGmGrqZ5fnQECSJPWc4V12jZDU6XQBayBn52cf7mfTxWr+6MnDZx+9qJraizw7jArSiKK4mm7qsqAQItDFEGRJChqCOdM0bTKZZGXhh97Z+dUBBwdHh1gUEfE5p4ZhmaYZqnKRZWVZJklcB5XruuvNMk+zMAzX6/Xz588X6xXnXFEU2zJNQwMi2LKeBAEv1qudye5ksitA4eb6Po1z1dBXq6swzCmlW08GQsiyrCRJr6+vjw4faJpWFBWldDSanJyc7O3tAQDOzs6qkrRarXar1zQsywpSM8YAIUTXDABAnpUY406nt/WOfffd9e7ubrfb3UbYt2Y3RVEuLy+jKBJFcTudbGNKdV3/+LMvDcPY2uY3y5WhW47jeIE/mewuVqskSSjllFLKWVbkeZEjLCR5RgjRua6qKgCA0Kauq6qqGAScc0EQZ4s5xvDgYI8xZhiWYViSpKmquvECCMFotLOzt4exuF5707t5VVWyrBJCKKWW6ewPjzinEoJAke/vbj+cveWAyKoync08P175wcYLoKiapmq1up3hDqQJZU0YR6auSrLYMJqVWRAErus+ff7k8y+/gBg9efbMcNz/9f/7b9qS/Ojhyd7efhhFH33y8Xg8Pru4/PblS0ppp9PblodBiAVRpqQhhFSMNbIEEXJbnSCIGs6gIESe3zA66PVlVRmNRqfnH8yZWhRFr9dFWDzcP5gp84ur28vzC0N3ipIsZvOqKBHCjDRVUTYYy6IEGN+SLssytyxDFLeROloWxfbxwjlfLpeEENd13XbLMMztaW3LWAQApGnq+z6lXNd1x3E+/+zLOI7v7+/Xa29bzuR53nQ6JYzasgwA8PxN6AeqIlm6VlGAAKCkQgjJopRjCBknZVVxynkDeGNoQNeVhpYcYtOwBFm5vrrI00wAUN4RbNNRBaxglOd5HCWCIHCAWN1wQEVRqKpqWwXSEEYpLWFDG7gNTGKEW+0ORHjLhmI81lTVdty6rgVz1DKG7nSz4DIsSB7eBfv7+3/x7/8P3/cppbksYozTLFcURVRURW96Tm9//8B1bQaN0ag8vz4LowxgwbTcKAzDIHtwdNxuTQ72Hupy7/zD5Tq4s2372fOP93d2fW9TpKkqoGg1vwlzy5YkSSqqvK6ZrAlMluI4JjWUBbN3MCElcdXee/nd/H5hItPp2GVe7erDgeA8sHd7D80t5fDR5Nnh4WFelpIkNU9q/zPfdV2IYdM0hFS306uzd98eHh4eHx/neZqBjFRMROrsfsNBtbu7G4dEEuzxSM2LtOXwPM8rlvY6g9k8C+PA6RoTy/rkk491w3r3w28PBq37xfK3331/dfb+1z//W8dx/5s//qcC4EVRSQLmnHe73bZlCILw9u1bWtcIs7enr/zYJ5z89S9/9a/+1b9arlayrHZ6bceRb27W797chtVby7IePXqSRLG3ua+LoMqF6V3z5PHeYr6WRMcLUt/3BUnkJTWYwSsQhqFuGg1nAIv2cHB5e9O+v1FV2TK1p88PR5uWZap/8x//CqOqqiqMakXmL55/qsrCr37xW9Y0r79/myflg8PHIrq+vZqpqloXDWS43+l/+PBhcHSURHHalCDO7m6uQF3/3hef7+3tvX///vzN6+cvXtA4WgTe/fmlIImHD477ckNb0s3NvFiXMnEEUZYUrQGI1NpsXkmyLkvYtUzEuAhQ4sea1UnSShTVf/wP/8lmvWB11es4tiTM06iRJFC3yiQnRYNF4+TxDkICK/LL2xtKiSzLmqJIvFrOF21NvDt7d3R4nC2XGsAKwZffnxq61W538zLd3Z/EcezHHhTBJz/6OIjCJM+6g64oK3GaVITkeXl5e7ellEiSEUWR4zi0Yp1227KstuZczD1OQbSM+s4g2HgnB48IIRxzSunl/c1P/uj3316cUUHYhBFE4myxcdx2FGVFkQGOJEmyTWOxXsiyjAWW8EDRFVVWBEFAEOqqjADEAJdSIbMAcTAaDK/qC4yxaXEAw+tpaFnW0rvhnJmNSFi2sz9ECCyWSw5oGGXr9bLVch4+fHgfVVhQdN26ubljDX/4+AlkuMjrhGf2qH2iPWtqKiBEEVB06fTyO3ttdjptYqr/7uf/bjm/twyzrL35+w/j8bAIvF9fXWRFygH9cHHhRYHba3/yox8dHx8/yJV2u71YLA73D58+fioJEmPs6Ohwi4jp9/s//vGPTdOEkEdR1DTN2enXqiMs1jPAESGF50UACA1hpjae32c/fHfe6rSenLx4+/7N/f0dYIzE64GtJWEWzVe6Zi3nK1EUR909U7eePH/wt7/6j6TZ2C2VwVun42q6JIba7e2NLha7B/urhbdaL0c7uw1XSc1qhgSp/ebNub+OWw5Lg5s8K2WBQgg5bZqmQQjIumhZhiXpjY4Hzjgt8tVqTQjhAiegGh9Nwrv06uqGxlTIJUYR4ngTx73xvtsd3s5WWVYYTscyTFXTsSAGYXT5YZHnebfbHfb2HUsz9cq27SovwrZDCLFUwVKFy8tbTIkty6Yo5nl6e/Z+dXttOa0HDx48f/4CQPj+wweE8GrtVQ0Z7R3Jhnd1cx1XzcV02R7eKIoSl6Vs23a/vww86e5ub29PNQ0lzdK8sCwLQlimZRZndV0LWhPnma7rUJQWm3Wr1XIsMwg8WZYt3QCsqcvSlFVBAHkU5RE4m1/M05UiqYPRiCogTVOjrRdNTgoSp0FN0oYSUDcCwhDzPA1fvT8ri6LVaqVxoiqDblsPPfTpp8+qqlrevffnxZsfLgzTFNTFzt6uY7c2XjQc9HVFddqtyWh8c3Y2vbiNfQ+U5cBxaJHZw97Bbl/X0OefPp4t5vP14n//d/9mPB67Lefk8cMnHz27vr10O+5wd/Dtt98iAWOML9bXtm3vjHY3m0242AiCgLFoaCrnPE/Sdru7bc1+ePJoNB7Kc3GzWdzNpl64OXywf7+ZD/YP4oo2cPny1auspAwiWZQaCg3NlCUtz8uyYu2e5bpulhUuqmmTCrIsICxiyBhqmlRV9CzLN+usKqPVYvrg0UNRAKvVynXbTYI0Q2saFqQRlsR2t4VFYeFv5t5akiS753YGHQRgURSapsiymBZ1EmaMMdvoWqpLqgJBrmtaU+fdVr/Mq+nNFCEw92eXFxf7+/sa1k8OnlxeXnp3HkfAtC2JSyqWKa8wI6E3D/yF2+5MdnaABKLIr9MMyjKDkHMOVR0xOYui5XyJMdZ13XHcbWcQAAAgCCFUNdl1u1VVRVHU643qui7LklIiMNYoimIYumEY7XY7TsIkjbIskyTBNN2qqjabtePYnU5na83lshbFnmaoDNL5fE4IAYjVZVVWOULIdV3btgEA2+q5bq+tW2w8Hp8cHRuaXhdlv9NjVdMURBaqneGBbdvr9TpKE8d1TdOklLISNyVEutSy260ng3574m88jIVPPvrUNmxJEEejkagaOmdZlvm+3xn0y4YEUYgQMgwNC8J8OS+KotVyzs/PXr161dBaFNCw1y3LfLNcUNIAxv2NjzBdzjaQM9+bN03V7tj7+/vtjuN5eUVYfzSWVXs+n98up4/h883y9u++/buDowe/9we/3x53zy8v/vpvfm6a9mins1jeZUnaarU4BUkSk7y0LEtVTASrTrsny/JWsg2CYD5fQQhns3m/N5Ilrd8bxbvZ2//zLxZo3hSk1WohDnVdFxBWZYU1lFGa5zmEmJQVBhCLMmhI6PtRkjRljgRpvLPz0eMTVhdpGJQp7tiHIpYO9g9Dfy1gWbP1hycP7E5+fn7+/fffZ2lhWRZjXFV0Xdc/+uijLX6g3++enJy0Wk6SJMNRP1ivoiiSRcnUVde2GKkDb71ezi8uLmbLBQDs8PiIUNpxrSTLNrN7VSkYx7JiEEKCuLRtue22Hbc73NmP8tJPCoAwY1WSxUEYVqQGAIRhnGVJkUQNqWQJR0lRNVNVM/M894I4Sou8qjESKeV1XTJaxHGKMVYlVZG10XAii4phGIw1i+WcUjqZjIsq+3//v/4/juO+ePHx+LifZZkXBqIoAih4XnA3nfq+77puw2hZlmmWRnFclqWmSY4zLEomKwJjbL6YxpFvWRbnoCxzDFEQrAmhtCaWZWwjbWmaiiK+vb3OspQ2TRQVgiCVZX5zG9q2zXg9HPV9XwjCtWvrnPPQX2mabBi6rusCEjHGmqwYmq5I0nqxMQ2D1qTVaimStLe77zrO5eVlDVir1dJ1fb1eXV1dWZbRH3QBAN9++9uiKKfTaZ5XvV6vLOuiyB88eEAUqcyrPK9FjPOy9j0P0QtRUCmhh/tHB3t7AIDA2+R5LoriFpURh+GHDxeWYbq2eXh46Lqupml3d3dBFJakJJwZmu7ariQIZZ4bWPPDQFYVp+V+8TtfHhwdlmXZdltRFLquCzmY3t7Vdd1utzFEeZ4bZnt/d78s6unN9PzD7eXFLWfoYG8/K/L5/F5SRUkRJElomno2n5ZVpUKlaZhjt0RR8r2wKGtNM1arVbvbEgX15METu2UwVDagyLNKNywAclGUOYcIipZlB2HCKJAkCUHOGbw6P18slozAg4ODbqu9ajYQkKoq4jhmrOGQ1l7ZNKRq8oqUdV2bjjUZj83HT+q6Tot8uVwqSnN4eNgQ6vthQ4CpWcPhUNds27Z937+/v6eUlm5LFPEWvVdk+dY75jrO/v7+aDQaDAah50+n06qqMESEkC3YX8RYUZTnzz+6v79fLpdBlBBC6oZiQbibza6vry3bBRh5vsc5f/78OYQwjuM///M/393dNXQrjuMsy7YM/21bjyAIpmlyzmVZDoMIACBJEiFc123HcURRZAwURcU5bxoWhqHjmJouVXWWJKQiBaygZRu3l1dRlIxG6qDX63U6m+WG02Z6exMFIUJCVRZJkimi1O32FGXb8ISLuPBXsySO8zzPE62pi7tuq9vvQ4FrhtrrdRiGUBA1w7Ace+XNARIhEvKCrNabOEt1XXcMfTzst1puEoVnp+873Z6la19+9vF6M/nm12/b7XZTV7fXV7qqfP7lFz/9yY85g7ujUeB7VVWlWWYoarfdHnZ7oKFVsSU5cEHAgiBCiP9LAWNDAWBJYpCGYSxWZb1lDiq23G53Hpwctbu9u7t7QQR5WSPM8zRpGsYZ3H4pqQBkiiokIVORLECZcwgAFEVJlVVJVqIohhCTmnIBLu4XeV5uMRiqJpRVSimVREhh4/sbVddMXWcQYIwRBwAgSZJN03Yst98fXlxeE8fyN15Z5pQQAICmaQgwyEHTNFVZNHUJAFvM5lfXF3/5l395tHsoy/JkMun1+1VDyroqiiIrcsux5/P5cr2SFAUgbFoWYyz0fNvubZ31W4wHFmVN01zHquoGQkjrmkrStiISIYQxhqzeShXbiFDTNFt0oxCE3ng8fv7i6cOHDyGEr1//8MMPPyiKlGVpVZWMMUFET589Ho/Hv/jFLwxTawD0vZXns+F4UOSpqekAc1038zxXZEVRpLpuPO82T1LOIWvo8cHEddw0is/fvr+/mwkQKVh19K5r9Cy1rYtW+3gyHu+cPHrY7XYph47W2VIt67ouiqzXWm83chDC8d4uY2AdRcuzs1evXt3e3vq+X3O8ZQplaazrugBhUWa9Tvf4+PD8w+l6PlM1OVqv7q4ugiD49ptvzl/e9ftdW9erqnj17ffvZBSGm/6gd3szVxVnvLOHRY0heTDuQGH96t0pIByqSp1nQZ6zu5s/7bY/mQytrjPfLBhjQKDL1e352QfbsnRdDzaepVq2bauqCgDe3z9SZK3dbne73bKsHz58yDn/+c9/EYZpUZAwTOuaPTo+iaIIUu6aNmzgoN0HABiqIUCB1iRIM0GQOGOiigQA0zACVYbrnKZNVjew0zoYDqskTvKsrmtvvry9vHj44IRSgkUNIyEIijzNHp08dN325eX1d99+H4bRk8fPHMv9f/4//u+qqvZ6vfFwsL83xhgXeaKrari8hYxosixhVKbJaj7L04wxxmlTxPG7V981VVLW9fHJgyZLp4sbbahgrDru0A+DvKoUJnCoSrr96v2Z0+kXNcnjOC1rgAXPX13fXitKT1VVjMWKsDTOVUWQFVUVtNn8ftjvj3YOCCHB9ZUgyVXdRFGkYFAktWUZgiDLomqbDmcsSZINRJKkrP1lVqS9Tp+wUjVkP1rd/3apaYokSZKsJ3G2WC3TNO6PhlhE0/u7+WoJEDctVVZBEIahtyI10jSNNLUfJB6AoigjhDAWHctWNdWRVbpVfAWIsayokqELG28lyUgU5aapDUMfTwZBEOR5ut4sdEMydHG1DgxzwFhTkwxXkiAgSmpCCKBA1zTbMFVZC4NAwtJgOBr2+7P7uWWahmGUeYGxwCkbD0fj4ahpiG3blJGzs7PryxvXteuiqUtACUiiVBCEyWjvYjkXoIQ4gRypsuaYxLIsWVQ2K282n1JSbaGcgiAIkmSa1puzN5Zl7U52er3ejz7+ZDjqbyWGhtHXr1/HWToYDk8en4wm46Iqr84u7tkiTdNnz555UahomqrreZ6vNmt/4yVJcvb+VNd1x7Q6nU7TNKvVquQEQ8O2HSxqttMWFWlbfoMxCiMvf59giX3x4y/+8Ge/J8nw22+/JRmsK6ZYAqNoudqwhnU6Xc+PLi+vRRmbrtrpjgqSeuF8uUxsp08pJgTezzZI1NKkjKMc4szQzbIoEORRlJR5KWJ5Mhzt7u5NRjuz6Y3vb6qmghDrmpZXICxLWrPN2iP1O91QHz5+fKLbtu1qmlFmpbXb8v3w7nba6/UU2bi/nWHMbds+OzsrioIQYlmWLMtbX8JqtXr86KFlm03T3N3dUtoYhj4Y9CEEeZ6ladqQOgj9m9vrMArajss5KypCKJUUjXO+WK2X6w2AMCtLURTzPBVkxbFtUZJEUUySpKkr13W3+UnO+e7uriRJoijP53OE0HrtbWvit0S17fOdAQlBnqU1xg3nuG6aus45B5PdQwjpYnmX5VGv31aUVpKGUZK1FRUBmMfp/O6eUuZ5niyIQRCsVpvRYKhJYo2BLKFey7FtO8uyumDJpiqKShYa0RA5oFWdrNbThlclyVs99+NPP9qE0XK9SpLk5uYGYrmsaJVHWRRXWVplhWHZtq50u11dVaPQ5wF3TCP2PJKlmR8g1uxPRs+ePfeWs6//7hdRuFFVXZCEPI54WWqy5JhDAUDdMm3DoO12UaMkScqiEkRJlTUAAAeAMcYhz8siCAJCaoTFhtEkzuMoqefFcDiUZUWWFdvWBVkkrBElUTO6QRCVBUENIE0TxT7nFAAgiaoi6xiJVV5xzjVD4AzRmlqW0xBGG4pl0fM8z/OGw3Fd1G1TV1VVM0wOYVHVFSmrEmCM0zQVRZlSSqpGkiTLMAWI6qLc3xtrmracz6uqKLK8rPKWbSdpdHdzDTjftqlJktDpdBAGW/aGoqrdXk+3TIRQVuRJVtQNKetK0XQsCoTSIsvvp9Ntm5SIkYAghwAAzBgDpGKs2ZKeK1LXdS3VNRLF7ZQgiqIM0db5yDnfOhsEQZAkSZBlkXM6Hg8nk9Fms5YkUZYl09IBpPP5vN12J5PRZDKyLFMUcbvtboINYpWsKrvjwXx+BzCM0yThsSBInHNS0y0lTRRF0zQ1TXv+7AhjoUizWcPLrEQAa7KImOj7EWzUwx89/cOf/aP9kxMAAKkIQogRVOeENFRRTMWxZ/fe3/7iN69fv3r49CGS5fFkImhKhfjVYn4zn6mqut54giCIGG02G6cyRQEFm7WmyIv5vYjhweEuRiCN/bevXgaBN735ICDR0OT1ZpnkiSQJR0cHDeFNjTESlqvUtKqWO5QVpdPpdLpDQZRfvjxtgMqwevLkk4rUDZcU2RqM958//1Gappbp5knKGOQMWZpdKtV3L1+bpnlycqJpGqXg1dkbXTfb7bbrur4XW5YV+NG9tNB1/fLyJo7jnd6OqdpN00hYZU20JSDZtr0lwSVJhqGgKoqhaCIWyzg1ZaqLgixLTV4UwQY3TcdUTUXceEGdZWkYkpI8ePBAEbXFevXdD28ePz3pdHq2bZ+eftgSvjDGVVVwTr/88vPd3V3dUPM8i+IgCII0jUFVlEVRYQFzXhYZ4HQ0HuxNdqIoEjGMolBCQJSxhrisS0VQRCHVTcMwnJIAgAssqOsgydmiJo2k15dXN1hWDo+Pjx891jTtN7/5zevXrx+fPBxPhqIo3lOapHF5v1wsPV1T+qMdRTU0HZmmX5Yl4Pzm+trVzcCPFUmt8iaXS13XBSyFYVik2XA4XCxmaZr/8T/9k8nu8OBot9Vq+UkTJfFq7UuKenB4bDnubDmVVSlJ4zAJkzw0TV01RFzTICJlHbXdybb7WBAhJU2e54Ko7O9PXMeRRKVpmsiP8rygVLBtV9d1hCrAqGtbm42va4qmSx3XGfRat9Mp42WRxy8+fqbqiFJSVSV1tG2otSiqLImrogwDvBFEBDAlzDbMk+MHn3zyiSzKVVFyykajUVJVnheYpq3r6jaukmXp+3dn3W5/Mpns7jZFUYxGI9u2DUNTFYPXvGN3Bu3R0dERgsLt9W2el3EUGIb24fT0/PT04cOHiqRMp1NVVR8/fowAPjl+yCmYz+fffffdxYWpqupg2EvTPAzjJEkmk0mv1W2Z7rV3fXd+s1xHTdM8evqEEGJYZlmWTV3XVTMr5mfvP8xmix9/8eX+7sF6vfa8oCqaRRR/9+qDAFFDag5F02rNZou0+O7x45Od3XFdl+/evWl3rJ/9gz/SrX+oqNJ3vzmLo2y98uq6SZKcUuB5gevas/kyTqPhpOd2DCjzNGVh5BtWlBSlHxZhHImSKUpSRUAUprJkYCQWaSlgqdPpHu4cWJYFOaA1ef/+NE0TCLmId2zblBoV0JAxOOrvJFl8P11CIDYNmEwm3W7XMp0GMEr5HZi2Wq1+bxx6kecFWzDitvaXUhrHcRQFg8HAsizGmuPjQ0mSVovlZrO6v78bDHoQ8V6/AyCDHBRFFsdhWeZFIS2X86pKyrK0bdu27ShOVqvV1j3gui6AEAMuKzKDoCpzCFi75UiKnCRJFCYY40ePHu3s7CAkxHGMMfa8YL1eby0Of89CIAXdOu9Mx2y3O2WVr9ersswbRoejjmm5eVnkRZmVvCaVbdtVWiGKqrye3c3zPE+ieMt9QozKInZd2zI0TVH7vZau65oqnv7wrsoD1jRuq60qelYUWV4u1jNRk6PE98M0r8liuVxvNgCjmpSqZsVhwupClyVTUTRNlxCUBVg3jQZBHMeWYa7ms4uLi/PLqyAILKO/mN7tjUfDXjv2fdA0bccuimI1n62Xq06nM+qPRIgo57Zmtkwnbe4hECDIBEGQVRUAwGmzhSFiBIqyAoAjJAHANN2mlAZxcDu9hxAOh0PN1DqdXlmWACAEhaLIwtCnDQQAVkUeM9o0bNSfYChQwqqixqIgYklAYtNQQzcKUCR1LYtSU5M8z8syX60WmSV99tlnB4eTJMsXqzUUpKqsbjcbLEimacuiQhFralKkmSRJEELMmakqsN9WJXm5XLx//35dFUHgJXFomToAYLPZRFFkWYaiKLKkckyRIAGMmqZhgBNKRVGUVaWtdBRNlySpqMowTquiVCTZdh0RK9vo7H+pnmJckSSo87phVVXVTdGoEucabxpKKeKcS4hzzhhDCG1/oSiKmqYJBwf7lmVZtj69v3n37h2EcHdvFEWR41j39zeWZdi2uVjMwmiTpIHneVWZCQh1XXtvZ3J7exNlybaOs9Pv/P1aTBRFQRa2kIfp9F4QBEu3Dg+PHaNdZCWniBRNr0sOD45/53d+srd7srr3X799F0epZVmT0d4PP/zw5s0rt2P96EcfA8Q5glVDrm9uoiwdIJClxWy5WqzXaVGImtHrddI0JVU9Gg9/98svRAH9/D//pzjxFQW1bEvV5DSO4jCII58yYtl6bzIZjUZlE0dpAJA42Tk4eagRQq6vr5dTX8Sz8WQYeFGW5IeHh3/yL/4vCP41giqphEcPP2Gcq3KLVCD0q0H/IJRDAYm2ZezvUsswx4OhIrn31zNd1/f29ra34Onph83mZrlYm6Ypy/LJycOPP/5kMBh0u31dN4ui8GZ3454BMWp1OpxDw7ElSdJNgyHc6vQkRTNVjVOmq6qhqSGjN9fv8jy3bTuKojSNsyzLihwgwXZdTTYG/S7mTJFkx2mZpj2fLZ8+frZcLpfzlYBwp9X21h5G4Mnjxx999NHOzni5XHietymzzWaFMQSc9zqt++k8yxJBECRFtmy30+72BqNWpxcEQVVmjmN1Oy2ImKEquoRuLk+LMgOAMU41TdNNM0jyKC1avb6um5Ryf7lMkjQOI8O0//AnP+n11rquM8riMCrLUsCibhiGph4fHw6GOy+//S2EHCOYJknbtbMoLPwoTVMMeJ7GReYaqiIJgq6oJan9KJyvlpKoyJp6dnG+8vw/+7M/0zivG5IkkecHt7e3DDVxmrC4ma3u0zwRBFhU2d00KkkJIeyP+lVWBWGZpUXTNHVJPM9znJYo4clkslwub29v/bUvimKvN1BVVZbl1WxhaZYqyds6RN/3kyR5+PjR7mS0vzv6zTe/pk096Hd8fyPJkLKq220nUVQVmaJIkogZYZQ0DW3qoo4a6m82TU1c20lxSkqSxdlwZ7xardrttu/73tp7/f3r4bDfa3e33W6qYe6M9uq6rnLiWuriftN2eoSQIs+bsjFNtdNqM4c1Dfvw/hQAZlqm41oYCtsjAmMsDqLRaDTsDZaz+cX1TRRFALCDgwOMMRZFS7eyOHv525eaImdZloexLIiIg6aqto/pLMuytCjLcrlc+0Gka2a3P1Y14276fRhGuq7v7J8QBuarRVlkvXarNxhwTi1b1zTtd37nC1HE//lv/vO333zndtqPHj362c9+xioNi9L3334fp7FuKcGmnC1WhqV1Oq3lZlnelGnVdjsWbVBRgdUi8tM0zyvSYFW1e8NB3cCqqhRZ73bsl799ySnvtNp/8Ad/GIfJuzdvLs4uwzDEGGu6oip6y+7kSlpWuc51BhkhJKLJarWp6yZLC85QURT+3N/Z2dsmOERBbbValPLp3WKrpW4/siBCCIui2PoNW61Wu90WseD7/vZxbBrGaDSq67ouK0opxnjLTQmCoCgTSqkgSlgQwzAMw7DhTBCEOI673a6iqlmWlVWh67rlOgihu/kcALBVvrIsS9P06OjBs2fPFovFeu0hhEzT9DwvSZKqqqqqwjLGGFuWNegNdMtcLKokzdIsWXsbxnlDq7xiQACUNXXdiApI4lRVVVEQEZQNXWYEMgqzrIBAzLPSNBvHcXRVY4yWZYEQnC+mSRw1lFqWhQTIIa9pHa5mnUFfFIWa5Lqstm2TNY2KMSRkNZ22W07bsdst17YMCQEqYsgIoc32hrRdZ3o3TdM88ELDMAGnURTVdS0Jom3bpma0W60kzYuC9LsD3TRISaqyUVVVERUBS6KgqbpDuUBpwwASMQQYA8A4ZUgQG85kQQQANA1XNRsA4OCCkCpN06woKQcNoxAjEQuW5XBGNusVbYBlurbpmIbNGMvijDccYAAh1BTV0HSIYVGUGaMMcAQY5JQDKohIkcVKEgJvQZpS0yUOmRSKRZkVRZElaZxkrVaxVevqMs8Rch3LcRyBA1aXuiTbjrlZL2b3N4Hnk6YqikJRZFPTGWPbI5wkSVmW1XVNCN2+ZxvGMMaGZZum+fjpkyLNIhpBCCUsYIwBZXmU6Ka45fpIksQ4Ak2NAJdlkfOaNXVVVaRSiVyWlHLORVmqENwuqBhjW88mhFCSJEHTpS++/PTgYO+rr756//5tv99ttVpJEm28FUSc8SYIvdn8rtfrKYpMSK1JiAJA6nSzuKekYg3RVVVWVQEKHMPt1iJJkvV67SEoCQLJ20mSqbLSdtoYioqmt52uqTsttzccTHTdfP3+/fv3Z4vlmlIOsfg//y//Zr1e+fFmNBqUtBQkeHN3k9X50eDQdq26rk8/nL1587aoiGG1GgIeHO4wxooi3xlP/ugP/zD0NmfvXy/n89ni3jAlXLO0iAUZCaJQVdwQdCxzycJ7D3f1lkEJF2TJclur+Wo2W7VaLVO3Ok777u7a36w39/dd29gf9Fu2LTCmiEIYRUkQQgFPr++63a4iqYSQfqfbaw8FhBVFoRQ+evRoa57YbDbb7s1erzccDquSVFWVZammqYwxhEC77WaZNHRbgiQKsiSIotFqC5LQAE4IqTm1ui3DMnVVK9MEQWg5NqXUaLWSqipJU5JGqnIAmGFokiSlSSTK6mh/d7ZaJ1EAGMMYK5JU5UXbceu6SaykzMqnjx7/+Isv//if/OOrq6tf/u3PP3w4ffDgqNfvxCEAgJGGzO59SqlmWgDhoiIlacqGpyUhVQGQUFZks9m4jglYbfXa+3uT++sfwmADeVUz7rR6iiRwShjAdVkWWR57gRcE0+l3//Ev/sOTJ09+9rOf7Q67AKA4jkFT06pSNPXoYP/BgweMElkSijwNNpvxaLCa3tjy4aOj3endoiE4z6I09Tgng2EXY2hZlg54EmdRmChKE8WprBieH6834XS5BoADBDx/dX59yiEzDE3WlJu7WyxwDnlVFUmREEKwiGRZNmW3aRpFlUyjSwmTZVlVdVEULy8v0yQnhIqiDCHKshzBUJaLm5s7w3B6vd5qvorkiHNeknp/f980DNu1pO+kl7/9xu067ba7t3MQJ2EcBFHkV0Vp27ahGnVJ6pIIEFPCHNNp6uq3X31dltWgO/j7Vo4tXN1xHFVV1+s1QihJstvbaVEU/X4fI+nDhw9Jknz55Zc7OxLJSkqp7/sCxJqstl3HNE1VVVlT9wddAUvdbtc07fFkUhQl59xfrVlDW63WRx99TChdrhbrzcYw7bLIEBIYBXEYVVnhWrZpGOPu8OGzj7IsMySlyjLothDjGCIBiYIgaJoBOIziRNFNJoiCphWUCpQxSmRNBYABDDXL6LBOr+NiDKMo0nVd03Tf99++OVUV4+jo6Hd+8qVp6VVVRn4oiuL52UUQZIKMVENFIdz4vqSLqqliUWkZJgdSWYKmETiEomy1W0MsaIw1sqjkeV6VDUKCACGG6N3rN0mUkJrYZkvTlIqUq8UGAMB4nVel45qSIgKOKGVlVUVh3G4VgAJAge/7sqwCAOq6Xi6XkCHbtltu7/zsfAvGhxB2u11JEmazGWNsPOpSSsoy1w1VknuyIiZpBFizbbNkDW0aIknCcNhvu26e50mmDIfDyWQSBMHl9e39fKkbaqvVCsMQQjCZTHYm4zRNF4tFGkfbGBfnnDNYVVWappeXl5KkOI4jCMI2gLe7uxsEwbY1WJIk3dYghLZtYAHe39/d3t6WZW2YTprG93PP9zdFnbZajmGbhtmz7N7+yYAQMp/N7mcBhJwQIiAQx6Uo4Y0fY0kWJKVhIIhCRVEcx7GdtiRrfhTXjJc1YQDqhkV4mud5u+UmcWxbjmOYAmNREMdxHEZJV1d3et22a9d1kReFgICIUV1VDKHhZG8wnJyeXbitvqxMTbMNAGq32zs7e9//8HqxWNYNV+5XWV4CLE529lVVD6OE1FxSxOlsc3d3V4quKMm6IeRpQkgFgIAx5JRWVYUEDBsAJIFSnpeloSoQQoQVQzERVgCQAj8RsCTJgm3bx8fHs/vFcrmJ/RQDKEBBlRQA0DpZM97IsizLooghaSpa0SJPuKbqui6JclHGpCoURUGA2KZyt06vzj9Ioqxoep5kSZQyxh2npaq6qqq6JokYRbyoygwLWq9jFkkRhRtFURCkZZ4JGALW9Dtd3/cYaSDiW5iHLMuKoiRJNF8t66ohhGzzO6IsWZazZXqKIoIQFnVFyvrvTQaiqHLOiSRtGUoQCFvdIYaxjzFgjFNSVUVRVNt2yhqB7ZSw5XdtGQ8IIcFxLNe137x59d133waBF0VBu+NKkrRarShtbm9vMMaKKvkBwgIcjvoaboqq5hD7m7WuKVVDRFwxxtbrjW4ZlmUZuhVFAaXUspz+oHu8d3R7e5tGSZjmgEFTMw2TqZxVDfGiYOX5s9kyylKOUZrFYRjfrqZ1XWZ1fnp5ere8YLChlCiKdHZ++hd/8X+Ionw/Wy0XG4SkdsvNsoKXpN/rzcsqCfzvf/vbd+/fnL57jzA3Td2wjDgOZ+t5t9tWVDMnOZIRUMjr8+8YhYPeuNcdS0hZe5t//5d/KWFxf3cPAZCFMabc0bRgtfz+668tswUIaluoLmmRlsH60rCczfJyMb8oy7KuGufLL5989Om2qKIm1qPHx0mS+F54e3tdVVWn4xwdPRgMBmEYxnGKMVqu5nESNg2xbSvLUtKw+XqORQEIGMlikcUcwiRLBoNBXleh7/c7Xd3QNFHGkrjxvU2cVwC6tiNzjiQRiXSbz6+qjALaci0KGG1KEYtnF2fL+Wy9HvX7w6qq+v3+p59+enz0gHMehuHP//ZvVqtVWaZPnp4gBJIkBpAlSXJ7dnVwcNCWlTgtwySXDcfp9QVZ/eH1m6bhV3f3kFWyJExGXVEUR4PBztitypDTrCqqXBLSxEhiX9YcyzDzJD1985ZS5mi62PC7s4v/7X4BNXdnb/fk5OTx8YFrGrPl4v72htaVZVl5HHQcZ3l3peJhuJrfY/bpp59KgpimSZqmSRbblq5IYkVqWZYFWaINf/joadMwCMU/++/+5Xy+IA1qtc3T09OsLBRFkmXRtA0k4vningOKRYkxVjWUM2iYtqZpCAEJypxzjMSmYVVZqqouy8p8toyC1DAsy7Qdo53nZZ7nRe4LgpDGRejFL55+rIjaZuNtHWTLhVe3mjRNHz58+ObNK1LWrmV3Wi3TNPM0wggYprq/u2NZduCFZVaZmqmr+tH+kb8JNiuPlLUoiookm4bprf2ri+vQjz7//HNowUFv+NVXX4VhSAjp9foCEtdLvyoaXbUMzQYM397ctdtty7DHw8nezm6SxP7GS9P0k48+CoLAD0MI0La6Pori2WxxvLPv+f7FxUVWpqZtcwA1TVdVdTabFUUlCEK31e622r1WW5Wlsiw3s5mumxfvTiVFBoTFWb4FD5c1CaPYC4O4LHd398+vrxFCRV4NsVTX5aDX6Tt9y1AhbZLYmy0Xmiz9x//8n0RRVGTNNJ08rk/fXzW1sHswHE36P/npl2ma+mufM7LZ+IIIb6ZXdV0WZVbXdZZloiL2Bv2iqrKCMgZ1w9QNx7Tamm5RSpIkuXn3PksL1nCEhNnd7PTde0ro4eHxcLA7Ho/KMvdDTxRQ1VBFVEzD1k3NMh3HcbIiz7JsPJy4bjvP7/M8f/XqlWXaum5whnkDTNN69PCZYznffPPNeu0FQWAbZqvVStP04OBgd3cXQhhFkaoopmnGcTydTtuOm2VZURQYoi0bzXXdw/39oijCNHv06NFkND4/P7+4uKCUuq5rWnqappQ2AIBer2Oa+mq1SONUU5RGEPM8FwXZcRzTNLMsy/M8CILtPhlj7DhOp9PZipWSJDEkpmnKQVNV+XxxHwSB7bZabjtOc1lUev2JFwbT+4Xs57/7k9/57LM/2B90qqr6Fn57dnbqeWva1O22KyimLGNBEhgU0pKkWUEZ6XREQVFHO8dN05DLi6KuijCBgmg7js7Rdu2xmM1nN3eQwzSOq6LUEXYHnXHLdBSRlulqfl/XpeM4uqFijBlAh8cP8qyMk7rb2xMVe+lFvXanqqkfx1c3U8Np7+0fr31/dr/oDYagoKRBSJAVw0nTajabvX//vr3/eDIZmYZNCCmqHCEIIaob0jTN1tUoSQqlvK4oNCSEUJ1hVdM1VeCA1IQxBkRBVmRNVfT9/QNSsNlsFYU54kJTc0apIgsQMs4IY4g0JaeSaZqdru37PhYo5zwJI9u2h+NRWZYNrXudPudwcT8DSCIMcChiLAEORFHMsoQ1lSzjwF8tV/dxNEewRKX44cMHTdMIIWmeSRhLktjrdTVN3a4NZFneQqMRQoZhfb63u+XoNIxuD6KMAYzxarXSVB1CWKRZVRFJVbazxRbYACHEEKiqIgiiLMuapi2W6yRJWENEUUQAsKbmnItY2y4Stu6ELV+VUgohFJ48eTKfz//1v/7XNzfes+e7URRd38CHD09kWeKc3d5d27Y90SfT6RQANhwOHx8eMIBEWTHdliQr78+vXr56PVutAAC6bmxH3TAM0zSRFUFVdzeeTxm3Xdc2bAwFDAUG+Gy56A938irP0iIu0jAOvDCSJGmw09872WOsqUmlKIJhqmG0ub4+j5MojpJXr79nFGmqhaHQEJqnRRKn11XKavLqzSuEwHq1mM2mpqEfHO3LMj44OPjuh29upzcVKRgYUEZs2954y9ls2e0MRVUq6+L85spUrLquJUVQRGk5X2xm01639ezJwygKsiBURXp/NRMliUE47Kl5VYpSmaSL2XxBCM3y8ujBBOCnZZHOl/PVcnNz+n4wGFi2MRoPtkqYZTpFURBSiSKWZdn3N7qup2l6eHj47v2bZJ2/PX0vKnKr33707CmUBKfrWrKIZHHlB+en70ldP3nwwHadPEmvrq4aVd4/efyHf/B7pMjrKk8CP0viKPBtx4rCuCiSu+ur86vrnd29IIz3dsZlWUZRRAh59uzFwcEBZ+Df/tt/++7dO9LUkiTIsuR53sXF2eXV+cnJiSQJjx4/2d3dFUT5brYpCeuP2hBJpxfX51e3Rwe7FEDLsDRDPzl51O+1DE3d3+nF4ZoBXJZlkSe+t9ws13YbnJ2++/ijz0zdWMwWTVlLggAhCjee4qLV/N617aMH1qDfzfJkPluul/OnT59en3u2qd1dXT0+2uekvr28+MmXn3W6rm4oSiTohry3v9Prd1brNeeQASRr0p/8yZ/Iso4Eudvp39/Pp9PZOoyTIhIEwXbMNE8Zp7IkiZJkiVa312OQ0ZubhtHhcPzpp58eHh6upqvZbLZcrmezme8FnU630+7leakoWl2RoIlMzVRVXZbVLM3LspREtapIrzewLOfi4ooBruv6xdk5FFBZpf/9//VfSpKwXi8ZA6fvP5CmkkQsCEiSFMexVFnd0HVNSgBMWZYNXc/TwrZtpSMbmr5t6Hn04kVVVR8+fPjiiy9ms9n9/f319fVHH33SNM3Jycnbt2+vrq52d3efPX1hGMarH1677ZYAhboo0ygts5wxJsmCifTFctYQZhmmaTkACIEfiaI4mUw8zzs7O/twfi6rChIEQpuyLAEADx48ePzo0aDddWyzTLIkCOMwCsNQluXJoL9YrBDGpK7TKDZsy7btXq/nh5GgqIKs+HF0ez/r9/vtfpdDFsXBoNfaO9j7+NnTTtt+/f23N1eXN5dXddUwClxHb7f7nIE8qzabQNZBXqQ7+yNG+Kvy+1bHlWTB9/3ldNZutyHkAPE4S1GJdzgvy3IxX5uWPt7ZOdh/sLuzn2bxfH5/ezs9Oz33PY8SNumPOeeUsvPzTRYn+QPS7/d7vYGqyRUp54t0tpj5kT8aDTVD1TRD181ETRVFCf3g1Q9vVqtVXTcttw0AUBRl1B8jJPy9vWt7dHMcp9ttbxWi4aivqNI2JiZJQlnmjDWuZXa77TxPEYR5nsuKaJhap9Mqy7JqwGq1ms/nt7e3AOFnz56ZphknYavVEkVBEITlfA4A2N+dsMmIUno5W1ZVZehWu92uqmqLiNgm1BaLxTahGsdxURRbAzhWpDjxdVOz7LbrOk3T6IZFKW23u0VZCrI01ixZMZAANa2V52y+CieTyZNnH0uKdn5+tt6sNN02TE1RRE2VNV1irMlrImCsaIbtthc3IRaEivCNF5V1LSkyluS8qCgN9nb2t72FuqYpABZY1GTl8bPHqiaLgG68tbdcIBFZps6YbFsuQNB0W2/f/Xa+9g5Pnhimu96EN/n9gwcP+oNJtz9QdZMw5vmhYdlYUM4vrwRBOjg8VhVzsQz8MJEU3Q+i8WSi60aWZQAgDgEAgFKKRAEAUBSVJNYIIAjxdlMY336glKdpzJoqyyNdVwEAAKDT09OiqHu9AUaKADcN4RBgAJimKU3TcM4AJxhrtm2enBzv7O/9L//r/1zkDeVNVReTnYdPnz5++/btPA1+9NGnjuPmRXVzN6cMdntuTenddHZwcEDrCkKuapJWShiyskryIiQRvrm93u4OBUHodrsQAITQZrMhTbUFdCZJEkWR5Tiqqv7m268AR1uD4bbNq64bzjmhDQSIUipJ0ni8M5lMHMdpmma1Xm/LybavfEXTtn6Fdsf1faeua0mRIMTbtYGqqmmaQgi3xsbtHUUIaZpGyPPyP/2nv+YcD3ouyaGt9dI0nd34Bwf772ZvLaM7GA4VXT+9vux0u5ssvg2cXr+jurbadj5cnK+SDYG81RsO+juk5rqiV0WiCg6ElY6l1Ftzx7y8/KAp9mRwcHL4jNV8MZsPen1AWVWkX3/1y9PT0ygpaANstyNgeVNMdV23dEuSJAGJnPKqYCzDLAZGywKQiYSaGJZNjtKkjSGj1bu3t3VWdPqDJFzKIjo4OBjtTI5PjvI8jqJMMyxRke2WvVjN/cTTbDQSzX5fD4vTi7uwzJv24OOdh3aZcsFSIm9TZOntevXg+VO916G0/n49293dr5vm/Pxc0XTXdess/cf/3b949f0PjIHzs4swTYIg2GLsZFHKcz8MwWS8c3K8kyQJrfMiA4iDNPCOjo45Za9+eEMp7bgd73b68OHD8+rdjz87+eHNa5nLJFns7O3KsDZa5nq97hpiM2hvFrcvs6DdahVFQYWmM9r5x3/6x8PDQ57nv/7VrzYlXfqpolh6q0/4/f/2H38eBAHnfPn2jaqqzXqxR/afPXtxcnJSFNn7969P37+/u7uzLEmTLQCArIj+YlFVlaNamZ90Op2d4/1+b0AIb16dVUnx5tu3T548O9o5fv3NG1WxRnuHqiFIw658MIBtsxRg+6OPq+n08vR9z3GK2PMX/tDtnV+/3T95MV8t50HojMdRFEiq7M3vOy3nX/6P/+O2822eZeu1N9zZ3T151Ol0rs7Os2p5efl2er/5+qvXP/29f/K3f/Ofk4jZMrg9v6qb5tGTp/1eO0rCvCrTrGAQTcY7HMG0SAddYz67rotKFujpu+86tqqoKkTEtGTKWZz6WOKj0WS9XpZl2eu3Tx4d7I4nk8lIEeBHX3yx4/tRELz85ttX330/Hg8O9vdvr24jb6VphuvYWVYUNTMtJ6tKRiBVhKDOaxG6k8EkP4qi6OnjJ0EQVFVV1vX333/4yU9+93Avf/Xq1XS6iON4f3c06T33PG92k0NY3F5ver2BY42KvLy4nq/mC0PTd3aPIIRAEC8uLiY7B45uAsZfvvxm43vfvXn56PnD3k6nLEsv3si68vjJk/3dA0MzLk8vby9u93f6T08OKOfz1Ww6nY73dyVFyYoizYvr+1vHcjnSlrOpKms7o8PLy8ub2er6fmm2u6KIHz5+ghA4P/+Q5zlnjabLgJPZ3e39zW2WpKSuszjRgdTRzJOjQ0FRl/7GNPXRZEJAs3e0v3N8iDB+9+795eXl8cFuHMeGKqVpiFm9nF932kq463QH8uGjUQPC6eoDLUleZlEm8wXvdHreLBh1xy2lBVT24fKCc9py7N7vfEpo/Ytf/KKoIy7UQRiIM9jrDkzbUZEqmcrOcYUx/unP/mD/8EgUxU5r8Obl22//7tv1/VoTdd1SOKe/+eqXNckme5JtW5vF9Ne/+GtRFLEo2LadFnnsVbFfi8DKslXDmeXY7XZ7USSX5ZpX6oPRU0EQ0jS5+nD17PkTVUC6rq4WV4F3ZxvC7G4NKVwtZ9PbG4SEqqiTOOCM1HWdAiC1WgKC3nqlilL/2bMXT57+h//wV61Wq+2017PVJVYVRVmvFrPZjFJ6eXk5Ho+rory7uz8+Pt7b25tOp2lS5HkuCIJhGJRyy2oNLB7xCFOIGtDk9fp+qWjqk5OHtm7M76dlWe7tTOIoWm8WnueNRiN7cGC4g4KAq+ms4kx3bYQRluThsL3YeFmW9fr9ycMDt9VxXff87i6vwt70+vc+//z508fe7Lq3O3jw4CDOwiD1vWC+c/xpGIbRej3sD9LYm9/hCkez+4WfLvqDNiHN2zfvg/VqZzwRRTePo45rbtvv3IE9Vnuu6+qi5PkeACAMI8PQRFEOg7TXnYhIa7ldzoTXb96Lil7UpL8zzpsmXi1VVX/79m2r1bq6udlsVpRxTunN/QYI9H59h5T68PiB1YZOJYi6ali9NJzdlB7Cgtt2hpOxIIlXN3er1Wq5DiRJib3AMIxet58SoktSEnF/E0kY6XrbNbvBmtcpJgW+u3pnmXqr5dYNqmitaYos09VqJVCo6Jq32Tx9+riu66u7s89/9wWp4yRYpHEkIvxHv/97O932WMKpJNxcX5Gd8YNnD/7i3//V1eUNFOXZ3eVyEyiamiernZ2dk6NHsoQmPevRwXC9miPePDp2WdZ69eZUYAUCSuBXmq7Gic8RlxRNVnXXbUVJDjhyLMfzvCanuq5UReUtNpxzjiCEUJZlTVMNw9j2uRPafDh/X1UVpRQiBWMMALi/u9V1vdPrOo4jSZJhW4xRy1I456omd9rjIAjupxeQI9M087yscgEhxBgTMfZ9T/j5z//6w4d3eZ5KggxgA6GgKKLrWs9fPE6L4Oz8dHpfHyi7OztDp+VEUfTh/P3L71LCyGgyTPPMdruj8UA3WpbZCfwYACAgbW9vT5a5oYm+79/c3FRFY8quv/ZPy3eT0WR3MhoOB0G49oPV3fx67t03jGMkwwI1Da1BWpM89DeEUMC4iCXEEWsob+iyqSAHuiqLIuascV2n3+8jWM/n87KoEWeEkCxN67qUJAEwSgiZzaZJGLfaNkKCaRhxkrT65rjfEUXxze3U2/i7o/3ReDCfenkc27b56OnJbHp/c3Px+vR1q222O7YkiAihTqtVjMc3d9OXL1/mWfHllz/+9NNPWcNVQREQbpqmYEyVlcFgMB6P0zTlgB0eHeVZ9uqHN57ntZ12u91mjJKaNE1TliVvQ9XQOYKu63IEHz5+tHd4oKgqEnCe56RpsiwzNd148IBSKksShHA5myOENovlxekZKcokSdbLVZqmmEMBoMV0JiL8yUcfn5+fB0Fwd3fXdlvtdvtnv/cPZVn5xS9+dX19DSEEjMqSqmmaIglb41VeFFEYpmkaBLHvR1Gefk+/35nsI4R6vd7f/eprwzAcx/nkk486vVaU+bKGHMfZis1IQru7u6PRaH57QzmjnGUZkeXCcRzf9zvDPdsx67rEGMdx3DTMsKxPPvrYtu2K1NfXt7/4xa+2c25Zlk+fPn3x/Ol6Pvuq0765urJt+9GjR0EUOi7c3d21XXdnb59D1IS+JElyw+Is3x7+IMSKqACOBFGUZdl124ZhKJpKaJOWBWsqSVIFUWkIVWTdNBzHtQ3DUGSjLBpS53eLHxAAW1XY33iuZe+OxoPBYDlfdlrt8WTX98O7+8X9dEoZ6HS6e7vDTqfz4Oj4X/y3f/abX/0dQqjVatVllaTR3d2da1mwYQJE9v+/pzNpkiO70uubffYYPIaMiByQyEyggCxUFQE2VVYkW23G3rTJ1KalzPQv27TRRpS0YJNdAwtEAQWgEjlnZMbk4fPwBn9aeFvvw2IZcd+95/uO622YFfMIQZKlxXKxrry61+tZllOW5dnZmes4Wus0TRupsiwzDKOVCkrJV6vVh7NfANJ7B7tffvmlEHyzWa9W6zROHMf7zcuvDw8ef/j5w5s3r6+vb3/9m1PGWC8IANZxkrx5/TcuBTNNz+uURWFQsxHSMAzLsqSUknNqsP4gwIwQgoRSWZpCgpnFEIGb7fbhYR5v4u16QzC2DRMb7Ppqbt90jY4bjHeiOH5Yr3NeWZ4fDEZ7e3tNA+5v7oJO93C2F0WJEOJmu82LjDFkUqaUXj4sqzpn1PzyxVeLh9Wb1z+VLB8HE9/3RS1ub68blc9mk9lsVtellBwiUItqOp1mWZZXZZIAx8nNPXMymQyHwzwvT09PT05Onj9/rpTSmqZJslgs2sN2med1LUwT9no9RqjW2rbtbF06jgMAaF9NhBDbtlErACzLSvAG6PbATxhro4ZxHOd5ppQqy3I+n89ms8lksl6vWw68lPVms5JCd7tdrXWS/LuPmHNeFEVbt6yVPjo6SrZR26DaNA2E8OrqSghxdvkpz/PZbOa6dlnm5+f3SqnhMFitzIeHeVmWVVUxxoSom6ap63K2MxNCFHWllAqCYDqdzh/uv/vuu1evXmGMW5NQURSt+eXq6mqK3OF45DhOVXOkgUZISpkWZRht86ICGDmOc3h4eHz8pNfv+77/x//zv1RZ/5W8plLkWe07VAitIbm7XzkdByHKpSTMoIRtNuFqsaE2q6qKYeL7vmlagquqqNtrJsPk+PhYa60EhxC6nm3bdr6N0jSLoqiu6+l0t9/v390v3r1798//9b+5vt++WZMkybLMcZzj48d5r7sON4TR0WgEEHr3/ue7u/v+ILAcp67r9XKpte70+hBC3/MAAKvVajgcdjodpZtgMPzyq6+UbjbbOIqiFjgt0izLMskFI6QNdNR1ySFElDjYQAgq3XDOwzAkBPV0T0q52WwWUk7GY9/3i7pqFLRMu8jrNEsAAMtFOB3vnJ1ddj1XoYYyixC2jTKAmJKwSIvzs7MsTTFEdVGZPfdwb4+a5v7+fq/XjTfbPN3OpmOKEFBNrxsMh8NBMPL9+6zkWZZVdQwwM520USBOs7woEcJCiEpwwzAePXo0mUwsyyrLstVVV4ILITDGXsevqipN0+VyqXRDKbUsy7Zt3RCMsWwapZTWusjyNkk7lFIj+B/phjY74/t+VdTtYNE2h7aMAiGE/PDjX7KM97qeY5kUQ4iEYUK/a7z44jPHp4OxezO/bUBp2rAX2LaLf/z2B6XUJpLnVwkm4L//jxf7+8e/fLopykwqDgBQUhJCHIcpWc7nc5FWlFgJjIpkMSf3e6Pp7mxqWOQv376VoKp1SmxJEUEEWi7gErrYb6Qqy1pIIRsFoaaEIAipyURVUkIaKLngSgkhLYQbz3JDygjGGGOlVF2XZZWLuqx5admmqDkhmGGqNdwZ71KyGXbYcBQwxniedV1vb/dwGPQ931kvtg0Qw/EgybZmaJzdnO2CMXFglpWNVIQQ17afHJ/s7+/P7+4ZY3/+858fPzqaTqfj4Wgy2om2W1Fz0Ci/67UNykJy23G8jus4zvHxkzAMtdTujocIXt6v+v0+xujy8vzZ8ydCCGYa4/F4tV6LUpqOTSl9eHhocTYlZVQUeZJeX15dXl4y231t28vb2+1228ZMLMsSQFvUFEL0/d6oP5yOJgZmw2DIGPvbmw9Znrx79y4Mw+PDx3t7e67nuo4ThiFjFChd5LLmDaEWQVgKsFw+pGm+M54Nhn2GTS6qzWZ9d3eNMEhShBCYTqfPnp32hzsASS04QHAymcx2dy2Car84O7tcblYSGoA2mOHJZOfi+jYIgsuLiBpkNpuNRiNimsnNTRLFoNGu7bS/oWoyPdidDcej0xefn338aNrW17/9Jg63kKTdfs/v9pVS96uHqhZepzvYcbqdXpLlRV4VWVlUPE/SzWazWCwpsW2r47guF0IjagJt2pZhGUVV+Z5CGCCE8oxvFtecc611mMZHR0fDYPDk6Ljn+gjCh4eHeJt0Op26rpeLRV5USAPGWF0JzjmDnoHY5mG90wuKJNVaX5+dSSn7QY9qoKW6vbrO8zyNYs+wjNmu1rCuedMAy3SenHzW8X3Oebha86oO+gPYQIKw1ipN481mtV4vGfsVQmizWa3Wi+efP3v8+PDNmzcXV5eTyQRDtLMzPXi0Z9tmVZWmaT579lRyUZZlgFAQBBqAeH5X1pXrup7nMcYk53EcI0Acx3EsazAYVABanisEb4AqRXl5fQlII6oyykBV5qjRSjfYpia1mGkiZvzDP/0jNVhDUCmqVRR+/+MP+vVfEcH7e4/+/u//YTyeGBiPu0EQBKQB19fXh4eHd/PbRjSLxRK90QDKjutMZ5PHh8ee3f3+2x8Wi5XndKFGdS3Oz88ZfYEQ0Ai6rq21Pv90bljs66+/dhzn8ub6/NOKc66UAgDEcbxabb76xz+8fPmSmMZ2sajr+v37D7/88kuv15uMn93dXK0WSwghowTq1sUF2qJ+pVS7lOZKKqU0AEVRtCnu9gOc845lBUHAmibPc9O0LMtqmubDhw9lmX/xxRdpmta8HI9HhNA4StM0p4ykaYxJYdu2ZVktIu66bp7nSqlPnz4t7x/af/F2Ibzdxoyxly9ftkbpNmnJOQcA+L7vum77JS202LY5aa1vLm92d3fbuHUrFkIIua57dXXVTtjv379vYXiEUJJwzjlBmDHGawEAoJRqAIQoVps1FwpREkWRlsp13W6nYzAGNUrj/P3bj1SrOs3qHGXFR9lwvzs4ODns9MeLddTtDDyvt7hdLheLp8+ObdOUrmtZlud4k52dbRgvFosvTr8YjUZpFv/w3fdNI23b3t37ajKZ/O93vxwfPzk4OCiKIksLCKHfHRDM2oaf1iYAQCOlbMMjo273X/7lf8Zp2urgw2jLpRqNRq7vR1F8dXUVRVG8jXq9nuM4UsqXr46ARllZlGWFMHRdx+92X72qV6tFXdtSSsVrAKAQQnKOEBoOgzzPhawxgQq0NiLORcU5L4qq5pyZhm27m80mKwrTHmCklQTd3rgo1f39Rin1f//fvz45Prq7W5F906JAcKgBExx4br8XjOY382gbb9YhxUYDtYlQv9vBhOmqjpfLX84+rJeLl6++pAS8efO63+1O/vm/aIRtx3OcuihFlMRFWUFsYMriOAYa0qdPHdc9P/90dXvjuq6WqtPpEEJ6vZ5lWe2FxTTNMNpuNpvtdhtFUcVrxpht24Zh+F4PAEIwppgQggFspKi11kkStR5tDXHTSAS1ZbJe109RaRhGWZZKNe2sgBBSShFK4WBgT6cTgnCZF3VdAwA1qAlVxyd7AFd+YK7DVZxtG110uvbz0yPDMKI0up3fVVw8OpyZJj379F4r5nuDXq9XyTpOthDaBgUQQpe6Qsh8m9pWJ+gPHx8+Mgz69qe/Xt6eTfcGuwcjswvTsqqFdDzqIEwlhhBK2fCSK66gRo3UiktKCGeo43kIISlqgoCGzWqzTAEuyxJjiBEAEDKCeFU+PMzzPHl8ciylBFqvV2HTNC9evLCYHXgOEg1G+vnRk1U31A2ty2LQ74T9LCli5pgSysmjaVZsoAWLprRNs67r7SZMkmQ4GO/u7kour69uj46OdidTBLBBabTdbjchw8TpdI6Pj4fDoRAiTpKWCkEMcFl3Op3NZuP3Oq1by3XdIq8uf7k6+eyIGsxynbworm9vbM/tDoK2VpNznmfZdrut8kIJmWcZBIBn2eLmpqmqMIwGg4FslCa0qnmvG8y30VJIg9DRaOSYzna7TZLk3cc7kxlSAMfuEGbnZZ1lRZZVRZF5josxVhq5Tsf3fcuylFKb5Kbf73e7PsNMCXBycmgY1jZaFUXhp65ohO+7s9kMmBaoMimbRqiDx0d1ntVlJsoiTdPr+aYqawAwhM1wPJgvFi++/JwyzAj55ne/10otb29/+O67zXrbPscRQkWalWWZ53ld5q7v9wfBzfzuxfNnmFGIDcvxGqCvb++ubm79fjA7eNQfjKlhcqGlAMgnfrdTVTVXquQ1QojXABMplFISE4NZpms7jga5VDxJkjgO43ibbKPW47J3tEcxSaNYCbm/twcbvbh/wBh3PX+13MxvbqMksx1/OBxHOkvjBPUGTa1EWc0ms8O9g6oukFagkXd3dwihJNz8bbWazx/KkjOMEELaohBpzrnkajye+G6nUSDLMseyu92uyRiGwHXd5XK53W6Xy+UmXEGkW7A03G4gxtc3l5eXV71e79HBwe7uXhxv3/3088XFRbfXOXp8fHy820pMDNtwHYdhskpShFCvGxCIMMaNUqZlm5RJLizD7HS7hmkWZaqAiJPwfjN3XTvLYyEEhqjndzzfUpXDiNH1upTS3/7n319eXt7e3haaQ4Z6QX+z2dzdPFyd32VJur97UJalYzomIV9+/uKb//T1n15/r5VaLhe313eL+wdRF6PxQCvQcOW6Xsf1i6xsmoZi+Kvf/Hq73c7nc9NkDQTtRuHu7s6w2Onp6e9+97udi/OyqJMkJYS08qTVatNaEnhRYkwf7hd//OMf3737+dWvXhqGQTBrGsB5WWtQV1XTNJRSy7SVUk3TQIyklFLJ9rVEKZ3NZn6va1hmXddVVRmWZRgGU6Db7bquSympefn27Vut1WQySZIoSaJutx/0h21Xne+7jRae1+/3+67raqlbG0un08nT/OzsbL1YGobVggWe41JKj46OsIkuLj9dXp1HUTSdTgfDfp7naRY//exkOAqU6jKDzOfzssqLouCcY01t20aUtGGWJ0+ezB/uW3bEcZx2tjAMYzgcGoaRpqnvuFLKJIqrqpZKUQghAAahFBMldZ5mF+fnEELGzOHO+M2bN5vF2iCUGSDo9YZ7B6aBMUXMYcFOf+/xAW8qfH5JfYYwraUSsoEaOJYNGii5iEQEIex43nazCcOwLIqu36mqCkOwWa2vL6+WD4u6FnUl8qxGiPb7gyAI8rKez+dZlhuWVRZ1W5naAvaMMcyo0nq5XFa8DoJgMplgytI0dV3XNK3WvSmqWillUGabVuvvPjg8/NWrl72gf307/9e//Pn86hIAYFmGaXZ8x1VKUUKqvACgGU9GbZNmo5USknPeNBopRQ2W5lkYhrPZ7OTJU3p1kWXZOtwSbDVNM50Nw3AtFaqq6tt/+2sc5YPRVEmgKb2+ve93BoN+rygL2+q+/fjaNE3BVb8/CAZDjGC8WtmOl6zXRZltNisMAdWQaJRF6dWny2EQZEWhNOr2AtmQMC3XUVJWhWlbUjWYEtO2nNpugF6tVnG89R2/bdaCEBZFwZVsg7u9oO84zu7ubr/fz8uidXMAABCAUAMMEaaEUkoJxRg3EDRKEYxN0ySEMExaLoEx5vu0HeBatVjTNP/usNjdm7UMRZmVVZlroKSUZZktlndc1rd3F5io8U6PRKrmGYDi1d99vt1u+9w5eXawXG2kKm5uwzgOEbR6vcDznUbylv3xPHs4HGbXW6EAJvBXX3x5cvL09PT0/OLjj6+/B1CZFvWJWWtbACFAJUGlVFMVkFKDQIQZhhACBZpGKtDARkKC/G5HCQktYzweZ1kSheF6ubBtkxEqpWwaZZoG1M1yMS8r//DxAcWEUVOJJk+qRiEIqMWsON6uHlZuxy+ysuY5hnbH7+3tIYgQ5xxRtDebhTHu9hzDYg508yRVXGjVPDw8XFxc3D08YETD9VrWMgrjLIqTKOFltbe3NwxGh8fB/uG+krouy6IoIIZZli8W9/3+4HZ+BwBoGrBYLzGltmNShj+c/XJycmLZdlGVWVl43Q4hpMVPOOfRdhuu1lo1jm33O12GSRLFBACtJMXQtU0hhFZNkecIkjIviqIaDAZJklmOd3VzBxCxLcf3fcO0hBC260CIFdCYGoahqGFCCIkQjDGv02srWnEOGaNplsx2pps0Ony8XxQV5zwYdCEGsuZVVdV1TSOFKaKGC7A7HAZFsr34+B4hMNgZNBgtw+J2lcdZiJmDGTn94oXruo5p/vab368fllESJ1EMIZQ1f7ibB6Nhv99vmoYarCxzKeXzz0+FksRgxDSqItIAykbXQmoAbMdhli0bFa3Xq81aysYyHYOZlDHbcvu9QZYnQkgeZRWv60YQg3GhkjSvpdBabzZhGIZSyqZBtu2Ox2PbtOq6jjah5MLa33cNy/M827BFLaaTSZpVq+Xb+2jOmIkRoQgv7ldJlJwcH7u2YztmnkaubXJenT77TAhxfX3Ny4oiTGyrKnkUy55jWpY1HDiM0M1m67uruq6XyzXUwLZtg1LPcVs+2bZtz/Nc193bm52eno4n4+l0p9PvXV6dY0Lm89uTo+NOp7N6WH369Mvd/G4ympiW4ZjWw908SpPZ3nQ8nUAAqrzQWn86OyuLejLxbdvuur5Wze3dnRCiBDgvUqUFswjCACFAGWoKiVCDCUIUyUYmRWab+tEweHTw2Ov7bGVUvORSOI719LOT1UOvyPJRMNyfznodzzbNJEnevn3r+/5wONyd7g2DQRRFSkmKdV6kJmMdrx9H+WQ4Gwaje34PlASgefz4kainf/rTnw4PD9yOv1jcE4J2d3fX4erDhw9/+MMfmGVK0VxdXdeVoJQihDqdjud2qpJblmWa9sXFxesf32w228VisV6twtU6TeKmaYBqGqnak61LvKIoAADMNLTWXEkhhJAyCIKDg4PeICjrKk3TpmkqzqMoYgpQaoxGI0LwNtowRhBCSgnLMsoyb/Or680KQeL5Vkslh2GYZVmyTQghnU6Hcx6uw3ajPhrZSqntdquE1BpqrdsfX611r9f75ptvGGPX19e+7y+Xy6Ioer2e53ntTbCl1fr+AAAwn88dxzk9PT06OsrL4sOHD5TSXhAMBoN2ET1/uK+qqh330yiWUhLDYIZRlxWXQintmBZGlHOeRvHV5aXr+jthePbhY7haTkZjzwkGg8HB7q7r2Y5vBZMhYMALvMurCwkgQ2CbxFmRQqhXq1W/PzBNU3GVJLFtu/1+vyrK8/OLvd3dZ0+f2KblunYSRWkU32d3sDE/fbr66af3EML9/f1fv/o7w7aiKB6Nxh2/t1qtdnZ2AIJS8vV6KSU/P+PXtzdREleSM9M42jnKy+rnn39WWg8GQ89xy7wQnNdFafjUMa2OF4RhWJSZECIMw+9/+Pant28RZQAAAIFpmhRjIQRodA2h1rqWHFFsEquRvAINAZgR0zCMONlmcbLZRtPdvdHOJIqizTos8tKxkdINJgwi1un0fN+/vLyUQj1//vnZh4+W6Vxe3T3ae7SzMz0//zFKktVm63leWdZSg6dPn2VpfnFzsbOzUxTFfD7vdDoHB/tBt+f7/snhSbgM//Jv31uW0w36rtfzutDdbMk6EqryCDMsohGMoqgW3Pd91AWGYTBIGWPtnYtzvgo3bWvn8ZMTz/PaPG0teFVVbfsybgsZAYAQgEZJ3jQEI9RepYBlMkopIhhD3TQN1IpSo92N/UebQpub+P9mz0Fh6kOt6AAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Open image\n", + "img = Image.open('img.jpg')\n", + "img" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# Preprocess image\n", + "tfms = transforms.Compose([transforms.Resize(image_size), transforms.CenterCrop(image_size), \n", + " transforms.ToTensor(),\n", + " transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),])\n", + "img = tfms(img).unsqueeze(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "# Load class names\n", + "labels_map = json.load(open('labels_map.txt'))\n", + "labels_map = [labels_map[str(i)] for i in range(1000)]" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded pretrained weights for efficientnet-b0\n", + "-----\n", + "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca (90.04%)\n", + "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus (0.62%)\n", + "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens (0.19%)\n", + "soccer ball (0.14%)\n", + "badger (0.10%)\n" + ] + } + ], + "source": [ + "# Classify with EfficientNet\n", + "model = EfficientNet.from_pretrained(model_name)\n", + "model.eval()\n", + "with torch.no_grad():\n", + " logits = model(img)\n", + "preds = torch.topk(logits, k=5).indices.squeeze(0).tolist()\n", + "\n", + "print('-----')\n", + "for idx in preds:\n", + " label = labels_map[idx]\n", + " prob = torch.softmax(logits, dim=1)[0, idx].item()\n", + " print('{:<75} ({:.2f}%)'.format(label, prob*100))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/simple/labels_map.txt b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/simple/labels_map.txt new file mode 100644 index 0000000..0c06838 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/examples/simple/labels_map.txt @@ -0,0 +1 @@ +{"0": "tench, Tinca tinca", "1": "goldfish, Carassius auratus", "2": "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", "3": "tiger shark, Galeocerdo cuvieri", "4": "hammerhead, hammerhead shark", "5": "electric ray, crampfish, numbfish, torpedo", "6": "stingray", "7": "cock", "8": "hen", "9": "ostrich, Struthio camelus", "10": "brambling, Fringilla montifringilla", "11": "goldfinch, Carduelis carduelis", "12": "house finch, linnet, Carpodacus mexicanus", "13": "junco, snowbird", "14": "indigo bunting, indigo finch, indigo bird, Passerina cyanea", "15": "robin, American robin, Turdus migratorius", "16": "bulbul", "17": "jay", "18": "magpie", "19": "chickadee", "20": "water ouzel, dipper", "21": "kite", "22": "bald eagle, American eagle, Haliaeetus leucocephalus", "23": "vulture", "24": "great grey owl, great gray owl, Strix nebulosa", "25": "European fire salamander, Salamandra salamandra", "26": "common newt, Triturus vulgaris", "27": "eft", "28": "spotted salamander, Ambystoma maculatum", "29": "axolotl, mud puppy, Ambystoma mexicanum", "30": "bullfrog, Rana catesbeiana", "31": "tree frog, tree-frog", "32": "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", "33": "loggerhead, loggerhead turtle, Caretta caretta", "34": "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", "35": "mud turtle", "36": "terrapin", "37": "box turtle, box tortoise", "38": "banded gecko", "39": "common iguana, iguana, Iguana iguana", "40": "American chameleon, anole, Anolis carolinensis", "41": "whiptail, whiptail lizard", "42": "agama", "43": "frilled lizard, Chlamydosaurus kingi", "44": "alligator lizard", "45": "Gila monster, Heloderma suspectum", "46": "green lizard, Lacerta viridis", "47": "African chameleon, Chamaeleo chamaeleon", "48": "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", "49": "African crocodile, Nile crocodile, Crocodylus niloticus", "50": "American alligator, Alligator mississipiensis", "51": "triceratops", "52": "thunder snake, worm snake, Carphophis amoenus", "53": "ringneck snake, ring-necked snake, ring snake", "54": "hognose snake, puff adder, sand viper", "55": "green snake, grass snake", "56": "king snake, kingsnake", "57": "garter snake, grass snake", "58": "water snake", "59": "vine snake", "60": "night snake, Hypsiglena torquata", "61": "boa constrictor, Constrictor constrictor", "62": "rock python, rock snake, Python sebae", "63": "Indian cobra, Naja naja", "64": "green mamba", "65": "sea snake", "66": "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", "67": "diamondback, diamondback rattlesnake, Crotalus adamanteus", "68": "sidewinder, horned rattlesnake, Crotalus cerastes", "69": "trilobite", "70": "harvestman, daddy longlegs, Phalangium opilio", "71": "scorpion", "72": "black and gold garden spider, Argiope aurantia", "73": "barn spider, Araneus cavaticus", "74": "garden spider, Aranea diademata", "75": "black widow, Latrodectus mactans", "76": "tarantula", "77": "wolf spider, hunting spider", "78": "tick", "79": "centipede", "80": "black grouse", "81": "ptarmigan", "82": "ruffed grouse, partridge, Bonasa umbellus", "83": "prairie chicken, prairie grouse, prairie fowl", "84": "peacock", "85": "quail", "86": "partridge", "87": "African grey, African gray, Psittacus erithacus", "88": "macaw", "89": "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", "90": "lorikeet", "91": "coucal", "92": "bee eater", "93": "hornbill", "94": "hummingbird", "95": "jacamar", "96": "toucan", "97": "drake", "98": "red-breasted merganser, Mergus serrator", "99": "goose", "100": "black swan, Cygnus atratus", "101": "tusker", "102": "echidna, spiny anteater, anteater", "103": "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", "104": "wallaby, brush kangaroo", "105": "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", "106": "wombat", "107": "jellyfish", "108": "sea anemone, anemone", "109": "brain coral", "110": "flatworm, platyhelminth", "111": "nematode, nematode worm, roundworm", "112": "conch", "113": "snail", "114": "slug", "115": "sea slug, nudibranch", "116": "chiton, coat-of-mail shell, sea cradle, polyplacophore", "117": "chambered nautilus, pearly nautilus, nautilus", "118": "Dungeness crab, Cancer magister", "119": "rock crab, Cancer irroratus", "120": "fiddler crab", "121": "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", "122": "American lobster, Northern lobster, Maine lobster, Homarus americanus", "123": "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", "124": "crayfish, crawfish, crawdad, crawdaddy", "125": "hermit crab", "126": "isopod", "127": "white stork, Ciconia ciconia", "128": "black stork, Ciconia nigra", "129": "spoonbill", "130": "flamingo", "131": "little blue heron, Egretta caerulea", "132": "American egret, great white heron, Egretta albus", "133": "bittern", "134": "crane", "135": "limpkin, Aramus pictus", "136": "European gallinule, Porphyrio porphyrio", "137": "American coot, marsh hen, mud hen, water hen, Fulica americana", "138": "bustard", "139": "ruddy turnstone, Arenaria interpres", "140": "red-backed sandpiper, dunlin, Erolia alpina", "141": "redshank, Tringa totanus", "142": "dowitcher", "143": "oystercatcher, oyster catcher", "144": "pelican", "145": "king penguin, Aptenodytes patagonica", "146": "albatross, mollymawk", "147": "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", "148": "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", "149": "dugong, Dugong dugon", "150": "sea lion", "151": "Chihuahua", "152": "Japanese spaniel", "153": "Maltese dog, Maltese terrier, Maltese", "154": "Pekinese, Pekingese, Peke", "155": "Shih-Tzu", "156": "Blenheim spaniel", "157": "papillon", "158": "toy terrier", "159": "Rhodesian ridgeback", "160": "Afghan hound, Afghan", "161": "basset, basset hound", "162": "beagle", "163": "bloodhound, sleuthhound", "164": "bluetick", "165": "black-and-tan coonhound", "166": "Walker hound, Walker foxhound", "167": "English foxhound", "168": "redbone", "169": "borzoi, Russian wolfhound", "170": "Irish wolfhound", "171": "Italian greyhound", "172": "whippet", "173": "Ibizan hound, Ibizan Podenco", "174": "Norwegian elkhound, elkhound", "175": "otterhound, otter hound", "176": "Saluki, gazelle hound", "177": "Scottish deerhound, deerhound", "178": "Weimaraner", "179": "Staffordshire bullterrier, Staffordshire bull terrier", "180": "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", "181": "Bedlington terrier", "182": "Border terrier", "183": "Kerry blue terrier", "184": "Irish terrier", "185": "Norfolk terrier", "186": "Norwich terrier", "187": "Yorkshire terrier", "188": "wire-haired fox terrier", "189": "Lakeland terrier", "190": "Sealyham terrier, Sealyham", "191": "Airedale, Airedale terrier", "192": "cairn, cairn terrier", "193": "Australian terrier", "194": "Dandie Dinmont, Dandie Dinmont terrier", "195": "Boston bull, Boston terrier", "196": "miniature schnauzer", "197": "giant schnauzer", "198": "standard schnauzer", "199": "Scotch terrier, Scottish terrier, Scottie", "200": "Tibetan terrier, chrysanthemum dog", "201": "silky terrier, Sydney silky", "202": "soft-coated wheaten terrier", "203": "West Highland white terrier", "204": "Lhasa, Lhasa apso", "205": "flat-coated retriever", "206": "curly-coated retriever", "207": "golden retriever", "208": "Labrador retriever", "209": "Chesapeake Bay retriever", "210": "German short-haired pointer", "211": "vizsla, Hungarian pointer", "212": "English setter", "213": "Irish setter, red setter", "214": "Gordon setter", "215": "Brittany spaniel", "216": "clumber, clumber spaniel", "217": "English springer, English springer spaniel", "218": "Welsh springer spaniel", "219": "cocker spaniel, English cocker spaniel, cocker", "220": "Sussex spaniel", "221": "Irish water spaniel", "222": "kuvasz", "223": "schipperke", "224": "groenendael", "225": "malinois", "226": "briard", "227": "kelpie", "228": "komondor", "229": "Old English sheepdog, bobtail", "230": "Shetland sheepdog, Shetland sheep dog, Shetland", "231": "collie", "232": "Border collie", "233": "Bouvier des Flandres, Bouviers des Flandres", "234": "Rottweiler", "235": "German shepherd, German shepherd dog, German police dog, alsatian", "236": "Doberman, Doberman pinscher", "237": "miniature pinscher", "238": "Greater Swiss Mountain dog", "239": "Bernese mountain dog", "240": "Appenzeller", "241": "EntleBucher", "242": "boxer", "243": "bull mastiff", "244": "Tibetan mastiff", "245": "French bulldog", "246": "Great Dane", "247": "Saint Bernard, St Bernard", "248": "Eskimo dog, husky", "249": "malamute, malemute, Alaskan malamute", "250": "Siberian husky", "251": "dalmatian, coach dog, carriage dog", "252": "affenpinscher, monkey pinscher, monkey dog", "253": "basenji", "254": "pug, pug-dog", "255": "Leonberg", "256": "Newfoundland, Newfoundland dog", "257": "Great Pyrenees", "258": "Samoyed, Samoyede", "259": "Pomeranian", "260": "chow, chow chow", "261": "keeshond", "262": "Brabancon griffon", "263": "Pembroke, Pembroke Welsh corgi", "264": "Cardigan, Cardigan Welsh corgi", "265": "toy poodle", "266": "miniature poodle", "267": "standard poodle", "268": "Mexican hairless", "269": "timber wolf, grey wolf, gray wolf, Canis lupus", "270": "white wolf, Arctic wolf, Canis lupus tundrarum", "271": "red wolf, maned wolf, Canis rufus, Canis niger", "272": "coyote, prairie wolf, brush wolf, Canis latrans", "273": "dingo, warrigal, warragal, Canis dingo", "274": "dhole, Cuon alpinus", "275": "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", "276": "hyena, hyaena", "277": "red fox, Vulpes vulpes", "278": "kit fox, Vulpes macrotis", "279": "Arctic fox, white fox, Alopex lagopus", "280": "grey fox, gray fox, Urocyon cinereoargenteus", "281": "tabby, tabby cat", "282": "tiger cat", "283": "Persian cat", "284": "Siamese cat, Siamese", "285": "Egyptian cat", "286": "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", "287": "lynx, catamount", "288": "leopard, Panthera pardus", "289": "snow leopard, ounce, Panthera uncia", "290": "jaguar, panther, Panthera onca, Felis onca", "291": "lion, king of beasts, Panthera leo", "292": "tiger, Panthera tigris", "293": "cheetah, chetah, Acinonyx jubatus", "294": "brown bear, bruin, Ursus arctos", "295": "American black bear, black bear, Ursus americanus, Euarctos americanus", "296": "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", "297": "sloth bear, Melursus ursinus, Ursus ursinus", "298": "mongoose", "299": "meerkat, mierkat", "300": "tiger beetle", "301": "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", "302": "ground beetle, carabid beetle", "303": "long-horned beetle, longicorn, longicorn beetle", "304": "leaf beetle, chrysomelid", "305": "dung beetle", "306": "rhinoceros beetle", "307": "weevil", "308": "fly", "309": "bee", "310": "ant, emmet, pismire", "311": "grasshopper, hopper", "312": "cricket", "313": "walking stick, walkingstick, stick insect", "314": "cockroach, roach", "315": "mantis, mantid", "316": "cicada, cicala", "317": "leafhopper", "318": "lacewing, lacewing fly", "319": "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", "320": "damselfly", "321": "admiral", "322": "ringlet, ringlet butterfly", "323": "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", "324": "cabbage butterfly", "325": "sulphur butterfly, sulfur butterfly", "326": "lycaenid, lycaenid butterfly", "327": "starfish, sea star", "328": "sea urchin", "329": "sea cucumber, holothurian", "330": "wood rabbit, cottontail, cottontail rabbit", "331": "hare", "332": "Angora, Angora rabbit", "333": "hamster", "334": "porcupine, hedgehog", "335": "fox squirrel, eastern fox squirrel, Sciurus niger", "336": "marmot", "337": "beaver", "338": "guinea pig, Cavia cobaya", "339": "sorrel", "340": "zebra", "341": "hog, pig, grunter, squealer, Sus scrofa", "342": "wild boar, boar, Sus scrofa", "343": "warthog", "344": "hippopotamus, hippo, river horse, Hippopotamus amphibius", "345": "ox", "346": "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", "347": "bison", "348": "ram, tup", "349": "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", "350": "ibex, Capra ibex", "351": "hartebeest", "352": "impala, Aepyceros melampus", "353": "gazelle", "354": "Arabian camel, dromedary, Camelus dromedarius", "355": "llama", "356": "weasel", "357": "mink", "358": "polecat, fitch, foulmart, foumart, Mustela putorius", "359": "black-footed ferret, ferret, Mustela nigripes", "360": "otter", "361": "skunk, polecat, wood pussy", "362": "badger", "363": "armadillo", "364": "three-toed sloth, ai, Bradypus tridactylus", "365": "orangutan, orang, orangutang, Pongo pygmaeus", "366": "gorilla, Gorilla gorilla", "367": "chimpanzee, chimp, Pan troglodytes", "368": "gibbon, Hylobates lar", "369": "siamang, Hylobates syndactylus, Symphalangus syndactylus", "370": "guenon, guenon monkey", "371": "patas, hussar monkey, Erythrocebus patas", "372": "baboon", "373": "macaque", "374": "langur", "375": "colobus, colobus monkey", "376": "proboscis monkey, Nasalis larvatus", "377": "marmoset", "378": "capuchin, ringtail, Cebus capucinus", "379": "howler monkey, howler", "380": "titi, titi monkey", "381": "spider monkey, Ateles geoffroyi", "382": "squirrel monkey, Saimiri sciureus", "383": "Madagascar cat, ring-tailed lemur, Lemur catta", "384": "indri, indris, Indri indri, Indri brevicaudatus", "385": "Indian elephant, Elephas maximus", "386": "African elephant, Loxodonta africana", "387": "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", "388": "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", "389": "barracouta, snoek", "390": "eel", "391": "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", "392": "rock beauty, Holocanthus tricolor", "393": "anemone fish", "394": "sturgeon", "395": "gar, garfish, garpike, billfish, Lepisosteus osseus", "396": "lionfish", "397": "puffer, pufferfish, blowfish, globefish", "398": "abacus", "399": "abaya", "400": "academic gown, academic robe, judge's robe", "401": "accordion, piano accordion, squeeze box", "402": "acoustic guitar", "403": "aircraft carrier, carrier, flattop, attack aircraft carrier", "404": "airliner", "405": "airship, dirigible", "406": "altar", "407": "ambulance", "408": "amphibian, amphibious vehicle", "409": "analog clock", "410": "apiary, bee house", "411": "apron", "412": "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", "413": "assault rifle, assault gun", "414": "backpack, back pack, knapsack, packsack, rucksack, haversack", "415": "bakery, bakeshop, bakehouse", "416": "balance beam, beam", "417": "balloon", "418": "ballpoint, ballpoint pen, ballpen, Biro", "419": "Band Aid", "420": "banjo", "421": "bannister, banister, balustrade, balusters, handrail", "422": "barbell", "423": "barber chair", "424": "barbershop", "425": "barn", "426": "barometer", "427": "barrel, cask", "428": "barrow, garden cart, lawn cart, wheelbarrow", "429": "baseball", "430": "basketball", "431": "bassinet", "432": "bassoon", "433": "bathing cap, swimming cap", "434": "bath towel", "435": "bathtub, bathing tub, bath, tub", "436": "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", "437": "beacon, lighthouse, beacon light, pharos", "438": "beaker", "439": "bearskin, busby, shako", "440": "beer bottle", "441": "beer glass", "442": "bell cote, bell cot", "443": "bib", "444": "bicycle-built-for-two, tandem bicycle, tandem", "445": "bikini, two-piece", "446": "binder, ring-binder", "447": "binoculars, field glasses, opera glasses", "448": "birdhouse", "449": "boathouse", "450": "bobsled, bobsleigh, bob", "451": "bolo tie, bolo, bola tie, bola", "452": "bonnet, poke bonnet", "453": "bookcase", "454": "bookshop, bookstore, bookstall", "455": "bottlecap", "456": "bow", "457": "bow tie, bow-tie, bowtie", "458": "brass, memorial tablet, plaque", "459": "brassiere, bra, bandeau", "460": "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "461": "breastplate, aegis, egis", "462": "broom", "463": "bucket, pail", "464": "buckle", "465": "bulletproof vest", "466": "bullet train, bullet", "467": "butcher shop, meat market", "468": "cab, hack, taxi, taxicab", "469": "caldron, cauldron", "470": "candle, taper, wax light", "471": "cannon", "472": "canoe", "473": "can opener, tin opener", "474": "cardigan", "475": "car mirror", "476": "carousel, carrousel, merry-go-round, roundabout, whirligig", "477": "carpenter's kit, tool kit", "478": "carton", "479": "car wheel", "480": "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", "481": "cassette", "482": "cassette player", "483": "castle", "484": "catamaran", "485": "CD player", "486": "cello, violoncello", "487": "cellular telephone, cellular phone, cellphone, cell, mobile phone", "488": "chain", "489": "chainlink fence", "490": "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", "491": "chain saw, chainsaw", "492": "chest", "493": "chiffonier, commode", "494": "chime, bell, gong", "495": "china cabinet, china closet", "496": "Christmas stocking", "497": "church, church building", "498": "cinema, movie theater, movie theatre, movie house, picture palace", "499": "cleaver, meat cleaver, chopper", "500": "cliff dwelling", "501": "cloak", "502": "clog, geta, patten, sabot", "503": "cocktail shaker", "504": "coffee mug", "505": "coffeepot", "506": "coil, spiral, volute, whorl, helix", "507": "combination lock", "508": "computer keyboard, keypad", "509": "confectionery, confectionary, candy store", "510": "container ship, containership, container vessel", "511": "convertible", "512": "corkscrew, bottle screw", "513": "cornet, horn, trumpet, trump", "514": "cowboy boot", "515": "cowboy hat, ten-gallon hat", "516": "cradle", "517": "crane", "518": "crash helmet", "519": "crate", "520": "crib, cot", "521": "Crock Pot", "522": "croquet ball", "523": "crutch", "524": "cuirass", "525": "dam, dike, dyke", "526": "desk", "527": "desktop computer", "528": "dial telephone, dial phone", "529": "diaper, nappy, napkin", "530": "digital clock", "531": "digital watch", "532": "dining table, board", "533": "dishrag, dishcloth", "534": "dishwasher, dish washer, dishwashing machine", "535": "disk brake, disc brake", "536": "dock, dockage, docking facility", "537": "dogsled, dog sled, dog sleigh", "538": "dome", "539": "doormat, welcome mat", "540": "drilling platform, offshore rig", "541": "drum, membranophone, tympan", "542": "drumstick", "543": "dumbbell", "544": "Dutch oven", "545": "electric fan, blower", "546": "electric guitar", "547": "electric locomotive", "548": "entertainment center", "549": "envelope", "550": "espresso maker", "551": "face powder", "552": "feather boa, boa", "553": "file, file cabinet, filing cabinet", "554": "fireboat", "555": "fire engine, fire truck", "556": "fire screen, fireguard", "557": "flagpole, flagstaff", "558": "flute, transverse flute", "559": "folding chair", "560": "football helmet", "561": "forklift", "562": "fountain", "563": "fountain pen", "564": "four-poster", "565": "freight car", "566": "French horn, horn", "567": "frying pan, frypan, skillet", "568": "fur coat", "569": "garbage truck, dustcart", "570": "gasmask, respirator, gas helmet", "571": "gas pump, gasoline pump, petrol pump, island dispenser", "572": "goblet", "573": "go-kart", "574": "golf ball", "575": "golfcart, golf cart", "576": "gondola", "577": "gong, tam-tam", "578": "gown", "579": "grand piano, grand", "580": "greenhouse, nursery, glasshouse", "581": "grille, radiator grille", "582": "grocery store, grocery, food market, market", "583": "guillotine", "584": "hair slide", "585": "hair spray", "586": "half track", "587": "hammer", "588": "hamper", "589": "hand blower, blow dryer, blow drier, hair dryer, hair drier", "590": "hand-held computer, hand-held microcomputer", "591": "handkerchief, hankie, hanky, hankey", "592": "hard disc, hard disk, fixed disk", "593": "harmonica, mouth organ, harp, mouth harp", "594": "harp", "595": "harvester, reaper", "596": "hatchet", "597": "holster", "598": "home theater, home theatre", "599": "honeycomb", "600": "hook, claw", "601": "hoopskirt, crinoline", "602": "horizontal bar, high bar", "603": "horse cart, horse-cart", "604": "hourglass", "605": "iPod", "606": "iron, smoothing iron", "607": "jack-o'-lantern", "608": "jean, blue jean, denim", "609": "jeep, landrover", "610": "jersey, T-shirt, tee shirt", "611": "jigsaw puzzle", "612": "jinrikisha, ricksha, rickshaw", "613": "joystick", "614": "kimono", "615": "knee pad", "616": "knot", "617": "lab coat, laboratory coat", "618": "ladle", "619": "lampshade, lamp shade", "620": "laptop, laptop computer", "621": "lawn mower, mower", "622": "lens cap, lens cover", "623": "letter opener, paper knife, paperknife", "624": "library", "625": "lifeboat", "626": "lighter, light, igniter, ignitor", "627": "limousine, limo", "628": "liner, ocean liner", "629": "lipstick, lip rouge", "630": "Loafer", "631": "lotion", "632": "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", "633": "loupe, jeweler's loupe", "634": "lumbermill, sawmill", "635": "magnetic compass", "636": "mailbag, postbag", "637": "mailbox, letter box", "638": "maillot", "639": "maillot, tank suit", "640": "manhole cover", "641": "maraca", "642": "marimba, xylophone", "643": "mask", "644": "matchstick", "645": "maypole", "646": "maze, labyrinth", "647": "measuring cup", "648": "medicine chest, medicine cabinet", "649": "megalith, megalithic structure", "650": "microphone, mike", "651": "microwave, microwave oven", "652": "military uniform", "653": "milk can", "654": "minibus", "655": "miniskirt, mini", "656": "minivan", "657": "missile", "658": "mitten", "659": "mixing bowl", "660": "mobile home, manufactured home", "661": "Model T", "662": "modem", "663": "monastery", "664": "monitor", "665": "moped", "666": "mortar", "667": "mortarboard", "668": "mosque", "669": "mosquito net", "670": "motor scooter, scooter", "671": "mountain bike, all-terrain bike, off-roader", "672": "mountain tent", "673": "mouse, computer mouse", "674": "mousetrap", "675": "moving van", "676": "muzzle", "677": "nail", "678": "neck brace", "679": "necklace", "680": "nipple", "681": "notebook, notebook computer", "682": "obelisk", "683": "oboe, hautboy, hautbois", "684": "ocarina, sweet potato", "685": "odometer, hodometer, mileometer, milometer", "686": "oil filter", "687": "organ, pipe organ", "688": "oscilloscope, scope, cathode-ray oscilloscope, CRO", "689": "overskirt", "690": "oxcart", "691": "oxygen mask", "692": "packet", "693": "paddle, boat paddle", "694": "paddlewheel, paddle wheel", "695": "padlock", "696": "paintbrush", "697": "pajama, pyjama, pj's, jammies", "698": "palace", "699": "panpipe, pandean pipe, syrinx", "700": "paper towel", "701": "parachute, chute", "702": "parallel bars, bars", "703": "park bench", "704": "parking meter", "705": "passenger car, coach, carriage", "706": "patio, terrace", "707": "pay-phone, pay-station", "708": "pedestal, plinth, footstall", "709": "pencil box, pencil case", "710": "pencil sharpener", "711": "perfume, essence", "712": "Petri dish", "713": "photocopier", "714": "pick, plectrum, plectron", "715": "pickelhaube", "716": "picket fence, paling", "717": "pickup, pickup truck", "718": "pier", "719": "piggy bank, penny bank", "720": "pill bottle", "721": "pillow", "722": "ping-pong ball", "723": "pinwheel", "724": "pirate, pirate ship", "725": "pitcher, ewer", "726": "plane, carpenter's plane, woodworking plane", "727": "planetarium", "728": "plastic bag", "729": "plate rack", "730": "plow, plough", "731": "plunger, plumber's helper", "732": "Polaroid camera, Polaroid Land camera", "733": "pole", "734": "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", "735": "poncho", "736": "pool table, billiard table, snooker table", "737": "pop bottle, soda bottle", "738": "pot, flowerpot", "739": "potter's wheel", "740": "power drill", "741": "prayer rug, prayer mat", "742": "printer", "743": "prison, prison house", "744": "projectile, missile", "745": "projector", "746": "puck, hockey puck", "747": "punching bag, punch bag, punching ball, punchball", "748": "purse", "749": "quill, quill pen", "750": "quilt, comforter, comfort, puff", "751": "racer, race car, racing car", "752": "racket, racquet", "753": "radiator", "754": "radio, wireless", "755": "radio telescope, radio reflector", "756": "rain barrel", "757": "recreational vehicle, RV, R.V.", "758": "reel", "759": "reflex camera", "760": "refrigerator, icebox", "761": "remote control, remote", "762": "restaurant, eating house, eating place, eatery", "763": "revolver, six-gun, six-shooter", "764": "rifle", "765": "rocking chair, rocker", "766": "rotisserie", "767": "rubber eraser, rubber, pencil eraser", "768": "rugby ball", "769": "rule, ruler", "770": "running shoe", "771": "safe", "772": "safety pin", "773": "saltshaker, salt shaker", "774": "sandal", "775": "sarong", "776": "sax, saxophone", "777": "scabbard", "778": "scale, weighing machine", "779": "school bus", "780": "schooner", "781": "scoreboard", "782": "screen, CRT screen", "783": "screw", "784": "screwdriver", "785": "seat belt, seatbelt", "786": "sewing machine", "787": "shield, buckler", "788": "shoe shop, shoe-shop, shoe store", "789": "shoji", "790": "shopping basket", "791": "shopping cart", "792": "shovel", "793": "shower cap", "794": "shower curtain", "795": "ski", "796": "ski mask", "797": "sleeping bag", "798": "slide rule, slipstick", "799": "sliding door", "800": "slot, one-armed bandit", "801": "snorkel", "802": "snowmobile", "803": "snowplow, snowplough", "804": "soap dispenser", "805": "soccer ball", "806": "sock", "807": "solar dish, solar collector, solar furnace", "808": "sombrero", "809": "soup bowl", "810": "space bar", "811": "space heater", "812": "space shuttle", "813": "spatula", "814": "speedboat", "815": "spider web, spider's web", "816": "spindle", "817": "sports car, sport car", "818": "spotlight, spot", "819": "stage", "820": "steam locomotive", "821": "steel arch bridge", "822": "steel drum", "823": "stethoscope", "824": "stole", "825": "stone wall", "826": "stopwatch, stop watch", "827": "stove", "828": "strainer", "829": "streetcar, tram, tramcar, trolley, trolley car", "830": "stretcher", "831": "studio couch, day bed", "832": "stupa, tope", "833": "submarine, pigboat, sub, U-boat", "834": "suit, suit of clothes", "835": "sundial", "836": "sunglass", "837": "sunglasses, dark glasses, shades", "838": "sunscreen, sunblock, sun blocker", "839": "suspension bridge", "840": "swab, swob, mop", "841": "sweatshirt", "842": "swimming trunks, bathing trunks", "843": "swing", "844": "switch, electric switch, electrical switch", "845": "syringe", "846": "table lamp", "847": "tank, army tank, armored combat vehicle, armoured combat vehicle", "848": "tape player", "849": "teapot", "850": "teddy, teddy bear", "851": "television, television system", "852": "tennis ball", "853": "thatch, thatched roof", "854": "theater curtain, theatre curtain", "855": "thimble", "856": "thresher, thrasher, threshing machine", "857": "throne", "858": "tile roof", "859": "toaster", "860": "tobacco shop, tobacconist shop, tobacconist", "861": "toilet seat", "862": "torch", "863": "totem pole", "864": "tow truck, tow car, wrecker", "865": "toyshop", "866": "tractor", "867": "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", "868": "tray", "869": "trench coat", "870": "tricycle, trike, velocipede", "871": "trimaran", "872": "tripod", "873": "triumphal arch", "874": "trolleybus, trolley coach, trackless trolley", "875": "trombone", "876": "tub, vat", "877": "turnstile", "878": "typewriter keyboard", "879": "umbrella", "880": "unicycle, monocycle", "881": "upright, upright piano", "882": "vacuum, vacuum cleaner", "883": "vase", "884": "vault", "885": "velvet", "886": "vending machine", "887": "vestment", "888": "viaduct", "889": "violin, fiddle", "890": "volleyball", "891": "waffle iron", "892": "wall clock", "893": "wallet, billfold, notecase, pocketbook", "894": "wardrobe, closet, press", "895": "warplane, military plane", "896": "washbasin, handbasin, washbowl, lavabo, wash-hand basin", "897": "washer, automatic washer, washing machine", "898": "water bottle", "899": "water jug", "900": "water tower", "901": "whiskey jug", "902": "whistle", "903": "wig", "904": "window screen", "905": "window shade", "906": "Windsor tie", "907": "wine bottle", "908": "wing", "909": "wok", "910": "wooden spoon", "911": "wool, woolen, woollen", "912": "worm fence, snake fence, snake-rail fence, Virginia fence", "913": "wreck", "914": "yawl", "915": "yurt", "916": "web site, website, internet site, site", "917": "comic book", "918": "crossword puzzle, crossword", "919": "street sign", "920": "traffic light, traffic signal, stoplight", "921": "book jacket, dust cover, dust jacket, dust wrapper", "922": "menu", "923": "plate", "924": "guacamole", "925": "consomme", "926": "hot pot, hotpot", "927": "trifle", "928": "ice cream, icecream", "929": "ice lolly, lolly, lollipop, popsicle", "930": "French loaf", "931": "bagel, beigel", "932": "pretzel", "933": "cheeseburger", "934": "hotdog, hot dog, red hot", "935": "mashed potato", "936": "head cabbage", "937": "broccoli", "938": "cauliflower", "939": "zucchini, courgette", "940": "spaghetti squash", "941": "acorn squash", "942": "butternut squash", "943": "cucumber, cuke", "944": "artichoke, globe artichoke", "945": "bell pepper", "946": "cardoon", "947": "mushroom", "948": "Granny Smith", "949": "strawberry", "950": "orange", "951": "lemon", "952": "fig", "953": "pineapple, ananas", "954": "banana", "955": "jackfruit, jak, jack", "956": "custard apple", "957": "pomegranate", "958": "hay", "959": "carbonara", "960": "chocolate sauce, chocolate syrup", "961": "dough", "962": "meat loaf, meatloaf", "963": "pizza, pizza pie", "964": "potpie", "965": "burrito", "966": "red wine", "967": "espresso", "968": "cup", "969": "eggnog", "970": "alp", "971": "bubble", "972": "cliff, drop, drop-off", "973": "coral reef", "974": "geyser", "975": "lakeside, lakeshore", "976": "promontory, headland, head, foreland", "977": "sandbar, sand bar", "978": "seashore, coast, seacoast, sea-coast", "979": "valley, vale", "980": "volcano", "981": "ballplayer, baseball player", "982": "groom, bridegroom", "983": "scuba diver", "984": "rapeseed", "985": "daisy", "986": "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", "987": "corn", "988": "acorn", "989": "hip, rose hip, rosehip", "990": "buckeye, horse chestnut, conker", "991": "coral fungus", "992": "agaric", "993": "gyromitra", "994": "stinkhorn, carrion fungus", "995": "earthstar", "996": "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", "997": "bolete", "998": "ear, spike, capitulum", "999": "toilet tissue, toilet paper, bathroom tissue"} \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/hubconf.py b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/hubconf.py new file mode 100644 index 0000000..dd0ea97 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/hubconf.py @@ -0,0 +1,43 @@ +from efficientnet_pytorch import EfficientNet as _EfficientNet + +dependencies = ['torch'] + + +def _create_model_fn(model_name): + def _model_fn(num_classes=1000, in_channels=3, pretrained='imagenet'): + """Create Efficient Net. + + Described in detail here: https://arxiv.org/abs/1905.11946 + + Args: + num_classes (int, optional): Number of classes, default is 1000. + in_channels (int, optional): Number of input channels, default + is 3. + pretrained (str, optional): One of [None, 'imagenet', 'advprop'] + If None, no pretrained model is loaded. + If 'imagenet', models trained on imagenet dataset are loaded. + If 'advprop', models trained using adversarial training called + advprop are loaded. It is important to note that the + preprocessing required for the advprop pretrained models is + slightly different from normal ImageNet preprocessing + """ + model_name_ = model_name.replace('_', '-') + if pretrained is not None: + model = _EfficientNet.from_pretrained( + model_name=model_name_, + advprop=(pretrained == 'advprop'), + num_classes=num_classes, + in_channels=in_channels) + else: + model = _EfficientNet.from_name( + model_name=model_name_, + override_params={'num_classes': num_classes}, + ) + model._change_in_channels(in_channels) + + return model + + return _model_fn + +for model_name in ['efficientnet_b' + str(i) for i in range(9)]: + locals()[model_name] = _create_model_fn(model_name) diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/npu_1p.sh b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/npu_1p.sh new file mode 100644 index 0000000..f617134 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/npu_1p.sh @@ -0,0 +1,9 @@ +export ASCEND_HOME=/usr/local/Ascend +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/python3.7.5/lib/ +export PYTHONPATH=${PYTHONPATH}:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/schedule_search.egg:/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/hccl +export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin +export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp/ +export PYTHONPATH=$PYTHONPATH:${PWD} +export SLOG_PRINT_TO_STDOUT=0 +export TASK_QUEUE_ENABLE=1 +taskset -c 0-64 python3.7 examples/imagenet/main.py --data=/data/imagenet --arch=efficientnet-b0 --batch-size=256 --lr=0.2 --epochs=200 --autoaug --npu=0 --amp --pm=O1 --loss_scale=1024 diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/setup.py b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/setup.py new file mode 100644 index 0000000..c4d3fee --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/setup.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Note: To use the 'upload' functionality of this file, you must: +# $ pipenv install twine --dev + +import io +import os +import sys +from shutil import rmtree + +from setuptools import find_packages, setup, Command + +# Package meta-data. +NAME = 'efficientnet_pytorch' +DESCRIPTION = 'EfficientNet implemented in PyTorch.' +URL = 'https://github.com/lukemelas/EfficientNet-PyTorch' +EMAIL = 'lmelaskyriazi@college.harvard.edu' +AUTHOR = 'Luke' +REQUIRES_PYTHON = '>=3.5.0' +VERSION = '0.7.0' + +# What packages are required for this module to be executed? +REQUIRED = [ + 'torch' +] + +# What packages are optional? +EXTRAS = { + # 'fancy feature': ['django'], +} + +# The rest you shouldn't have to touch too much :) +# ------------------------------------------------ +# Except, perhaps the License and Trove Classifiers! +# If you do change the License, remember to change the Trove Classifier for that! + +here = os.path.abspath(os.path.dirname(__file__)) + +# Import the README and use it as the long-description. +# Note: this will only work if 'README.md' is present in your MANIFEST.in file! +try: + with io.open(os.path.join(here, 'README.md'), encoding='utf-8') as f: + long_description = '\n' + f.read() +except FileNotFoundError: + long_description = DESCRIPTION + +# Load the package's __version__.py module as a dictionary. +about = {} +if not VERSION: + project_slug = NAME.lower().replace("-", "_").replace(" ", "_") + with open(os.path.join(here, project_slug, '__version__.py')) as f: + exec(f.read(), about) +else: + about['__version__'] = VERSION + + +class UploadCommand(Command): + """Support setup.py upload.""" + + description = 'Build and publish the package.' + user_options = [] + + @staticmethod + def status(s): + """Prints things in bold.""" + print('\033[1m{0}\033[0m'.format(s)) + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + try: + self.status('Removing previous builds…') + rmtree(os.path.join(here, 'dist')) + except OSError: + pass + + self.status('Building Source and Wheel (universal) distribution…') + os.system('{0} setup.py sdist bdist_wheel --universal'.format(sys.executable)) + + self.status('Uploading the package to PyPI via Twine…') + os.system('twine upload dist/*') + + self.status('Pushing git tags…') + os.system('git tag v{0}'.format(about['__version__'])) + os.system('git push --tags') + + sys.exit() + + +# Where the magic happens: +setup( + name=NAME, + version=about['__version__'], + description=DESCRIPTION, + long_description=long_description, + long_description_content_type='text/markdown', + author=AUTHOR, + author_email=EMAIL, + python_requires=REQUIRES_PYTHON, + url=URL, + packages=find_packages(exclude=["tests", "*.tests", "*.tests.*", "tests.*"]), + # py_modules=['model'], # If your package is a single module, use this instead of 'packages' + install_requires=REQUIRED, + extras_require=EXTRAS, + include_package_data=True, + license='Apache', + classifiers=[ + # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + ], + # $ setup.py publish support. + cmdclass={ + 'upload': UploadCommand, + }, +) diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/tests/test_model.py b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/tests/test_model.py new file mode 100644 index 0000000..0a03467 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/code/tests/test_model.py @@ -0,0 +1,124 @@ +from collections import OrderedDict + +import pytest +import torch +import torch.nn as nn + +from efficientnet_pytorch import EfficientNet + + +# -- fixtures ------------------------------------------------------------------------------------- + +@pytest.fixture(scope='module', params=[x for x in range(4)]) +def model(request): + return 'efficientnet-b{}'.format(request.param) + + +@pytest.fixture(scope='module', params=[True, False]) +def pretrained(request): + return request.param + + +@pytest.fixture(scope='function') +def net(model, pretrained): + return EfficientNet.from_pretrained(model) if pretrained else EfficientNet.from_name(model) + + +# -- tests ---------------------------------------------------------------------------------------- + +@pytest.mark.parametrize('img_size', [224, 256, 512]) +def test_forward(net, img_size): + """Test `.forward()` doesn't throw an error""" + data = torch.zeros((1, 3, img_size, img_size)) + output = net(data) + assert not torch.isnan(output).any() + + +def test_dropout_training(net): + """Test dropout `.training` is set by `.train()` on parent `nn.module`""" + net.train() + assert net._dropout.training == True + + +def test_dropout_eval(net): + """Test dropout `.training` is set by `.eval()` on parent `nn.module`""" + net.eval() + assert net._dropout.training == False + + +def test_dropout_update(net): + """Test dropout `.training` is updated by `.train()` and `.eval()` on parent `nn.module`""" + net.train() + assert net._dropout.training == True + net.eval() + assert net._dropout.training == False + net.train() + assert net._dropout.training == True + net.eval() + assert net._dropout.training == False + + +@pytest.mark.parametrize('img_size', [224, 256, 512]) +def test_modify_dropout(net, img_size): + """Test ability to modify dropout and fc modules of network""" + dropout = nn.Sequential(OrderedDict([ + ('_bn2', nn.BatchNorm1d(net._bn1.num_features)), + ('_drop1', nn.Dropout(p=net._global_params.dropout_rate)), + ('_linear1', nn.Linear(net._bn1.num_features, 512)), + ('_relu', nn.ReLU()), + ('_bn3', nn.BatchNorm1d(512)), + ('_drop2', nn.Dropout(p=net._global_params.dropout_rate / 2)) + ])) + fc = nn.Linear(512, net._global_params.num_classes) + + net._dropout = dropout + net._fc = fc + + data = torch.zeros((2, 3, img_size, img_size)) + output = net(data) + assert not torch.isnan(output).any() + + +@pytest.mark.parametrize('img_size', [224, 256, 512]) +def test_modify_pool(net, img_size): + """Test ability to modify pooling module of network""" + + class AdaptiveMaxAvgPool(nn.Module): + + def __init__(self): + super().__init__() + self.ada_avgpool = nn.AdaptiveAvgPool2d(1) + self.ada_maxpool = nn.AdaptiveMaxPool2d(1) + + def forward(self, x): + avg_x = self.ada_avgpool(x) + max_x = self.ada_maxpool(x) + x = torch.cat((avg_x, max_x), dim=1) + return x + + avg_pooling = AdaptiveMaxAvgPool() + fc = nn.Linear(net._fc.in_features * 2, net._global_params.num_classes) + + net._avg_pooling = avg_pooling + net._fc = fc + + data = torch.zeros((2, 3, img_size, img_size)) + output = net(data) + assert not torch.isnan(output).any() + + +@pytest.mark.parametrize('img_size', [224, 256, 512]) +def test_extract_endpoints(net, img_size): + """Test `.extract_endpoints()` doesn't throw an error""" + data = torch.zeros((1, 3, img_size, img_size)) + endpoints = net.extract_endpoints(data) + assert not torch.isnan(endpoints['reduction_1']).any() + assert not torch.isnan(endpoints['reduction_2']).any() + assert not torch.isnan(endpoints['reduction_3']).any() + assert not torch.isnan(endpoints['reduction_4']).any() + assert not torch.isnan(endpoints['reduction_5']).any() + assert endpoints['reduction_1'].size(2) == img_size // 2 + assert endpoints['reduction_2'].size(2) == img_size // 4 + assert endpoints['reduction_3'].size(2) == img_size // 8 + assert endpoints['reduction_4'].size(2) == img_size // 16 + assert endpoints['reduction_5'].size(2) == img_size // 32 diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/config/set_env_b023.sh b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/config/set_env_b023.sh new file mode 100644 index 0000000..7618849 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/config/set_env_b023.sh @@ -0,0 +1,31 @@ +############## toolkit situation ################ +#export LD_LIBRARY_PATH=/usr/local/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH +#export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/ascend-toolkit/latest/toolkit/tools/ide_daemon/bin/ +#export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ +#export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so +#export PYTHONPATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH + + +############## nnae situation ################ + + +if [ -d /usr/local/Ascend/nnae/latest ];then + export LD_LIBRARY_PATH=/usr/local/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/aarch64_64-linux-gnu:$LD_LIBRARY_PATH + export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/nnae/latest/toolkit/tools/ide_daemon/bin/ + export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp/ + export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so + export PYTHONPATH=/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH +else + export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/aarch64-linux-gnu:$LD_LIBRARY_PATH + export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/ascend-toolkit/latest/toolkit/tools/ide_daemon/bin/ + export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so + export PYTHONPATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH +fi + +# ln -s /usr/local/Ascend/ascend-toolkit/latest/toolkit/bin/adc /usr/local/bin/ + +export SLOG_PRINT_TO_STDOUT=0 +#su HwHiAiUser -c "adc --host 0.0.0.0:22118 --log \"SetLogLevel(0)[error]\" --device 0" + +export TASK_QUEUE_ENABLE=1 \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/scripts/run.sh b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/scripts/run.sh new file mode 100644 index 0000000..7a90b4a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/scripts/run.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +rank_size=$1 +yamlPath=$2 +toolsPath=$3 + +currentDir=$(cd "$(dirname "$0")/.."; pwd) +model_name=$(cd $currentDir/..;basename `pwd`) +if [ -f /.dockerenv ];then + CLUSTER=$4 + MPIRUN_ALL_IP="$5" + export CLUSTER=${CLUSTER} +fi +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "pytorch_config") + +# 清除旧日志 +rm -rf /var/log/npu/slog/host-0/* +rm -rf ${currentDir}/result/*.log + +#mkdir train job path +currtime=`date +%Y%m%d%H%M%S` +mkdir -p ${currentDir%train*}/train/result/pt_efficientnet/training_job_${currtime}/ +export train_job_dir=${currentDir%train*}/train/result/pt_efficientnet/training_job_${currtime}/ +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir}" +# device 列表, 若无指定 device 根据 rank_size 顺序选择 +eval device_group=\$device_group_${rank_size}p +if [ x"${device_group}" == x"" ] || [ ${rank_size} -ge 8 ];then + device_group="$(seq 0 "$(expr $rank_size - 1)")" +fi + +# get last device id in device_group, hw log in performance from the dir named last_device_id +device_group_str=`echo ${device_group} | sed 's/ //g'` +first_device_id=`echo ${device_group_str: 0:1}` + +if [ x"${CLUSTER}" == x"True" ];then + this_ip=$(hostname -I |awk '{print $1}') + ln -snf ${currentDir%train*}/train/result/pt_efficientnet/training_job_${currtime}/0/hw_efficientnet.log ${currentDir%train*}/train/result/pt_efficientnet/training_job_${currtime}/ + for ip in $MPIRUN_ALL_IP;do + if [ x"$ip" != x"$this_ip" ];then + scp $yamlPath root@$ip:$yamlPath + scp ${jsonFilePath} root@$ip:${jsonFilePath} + fi + done + export PATH=$PATH:/usr/local/mpirun4.0/bin + mpirun -H ${mpirun_ip} \ + --bind-to none -map-by slot\ + --allow-run-as-root \ + --mca btl_tcp_if_exclude lo,docker0,endvnic,virbr0,vethf40501b,docker_gwbridge,br-f42ac38052b4\ + --prefix /usr/local/mpirun4.0/ \ + ${currentDir}/scripts/train.sh 0 $rank_size $yamlPath $currtime ${toolsPath} ${CLUSTER} +else + rank_id=0 + #for device_id in $device_group;do + ln -snf ${currentDir%train*}/train/result/pt_efficientnet/training_job_${currtime}/${first_device_id}/hw_efficientnet.log ${currentDir%train*}/train/result/pt_efficientnet/training_job_${currtime}/ + ${currentDir}/scripts/train.sh 0 $rank_size $yamlPath $currtime ${toolsPath} $rank_id & + # let rank_id++ + # done +fi +wait + + diff --git a/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/scripts/train.sh b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/scripts/train.sh new file mode 100644 index 0000000..bc48baf --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/EfficientNet/pytorch/scripts/train.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash + +device_id=$1 +rank_size=$2 +yamlPath=$3 + +currentDir=$(cd "$(dirname "$0")/.."; pwd) +currtime=$4 +toolsPath=$5 +export YAML_PATH=$3 +mkdir -p ${currentDir%train*}/train/result/pt_efficientnet/training_job_${currtime}/ +export train_job_dir=${currentDir%train*}/train/result/pt_efficientnet/training_job_${currtime}/ + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "pytorch_config") + +export REMARK_LOG_FILE=hw_efficientnet.log # 打点日志文件名称， 必须hw_后跟模型名称小写 +benchmark_log_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils +export PYTHONPATH=$PYTHONPATH:${benchmark_log_path} + + +#source ${currentDir}/config/npu_set_env.sh +source ${currentDir}/config/set_env_b023.sh +# user env +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 +export HCCL_RANK_TABLE_PATH=${currentDir}/config/${rank_size}p.json +export RANK_SIZE=${rank_size} +export SLOG_PRINT_TO_STDOUT=0 +export DEVICE_ID=${device_id} +DEVICE_INDEX=$(( DEVICE_ID + RANK_INDEX * 8 )) +export DEVICE_INDEX=${DEVICE_INDEX} + +cd ${train_job_dir} +curd_dir=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +export PYTHONPATH=$PYTHONPATH:${curd_dir} + +if [ x"$6" != x"True" ];then + rank_id=$6 + export RANK_ID=$6 +else + device_id_mo=$(python3.7 -c "import src.tensorflow.mpi_ops as atlasboost;atlasboost.init(); \ + device_id = atlasboost.local_rank();cluster_device_id = str(device_id); \ + atlasboost.set_device_id(device_id);print(atlasboost.rank())") + device_id_mo=`echo $device_id_mo` + rank_id=${device_id_mo##* } + export RANK_ID=${rank_id} + device=${device_id_mo##*deviceid = } + device_id=${device%% phyid=*} + export DEVICE_ID=${device_id} + hccljson=${train_job_dir}/*.json + cp ${hccljson} ${currentDir}/config/${rank_size}p.json +fi + +#mkdir exec path +mkdir -p ${train_job_dir}/${device_id} +cd ${train_job_dir}/${device_id} + +startTime=`date +%Y%m%d-%H:%M:%S` +startTime_s=`date +%s` + + +# 根据单卡/多卡区分调用参数 +if [ x"$6" == x"True" ];then + # 多卡多机 + export CLUSTER=True +fi + +if [ x"${mode}" == x"evaluate" ];then + pass + + +elif [ x"${rank_size}" == x"1" ];then + # 单卡 + taskset -c 0-128 python3.7 ${currentDir}/code/examples/imagenet/main.py \ + --data=${data_url} \ + --arch=efficientnet-b0 \ + --batch-size=${batch_size} \ + --lr=0.2 \ + --momentum=0 \ + --epochs=${epoches} \ + --autoaug \ + --amp \ + --pm=O1 \ + --loss_scale=128 \ + --val_feq=10 \ + --npu=${device} > ${train_job_dir}/train_${rank_size}p.log 2>&1 + + +elif [ ${rank_size} -le 8 ];then + # 单机多卡 + taskset -c 0-128 python3.7 ${currentDir}/code/examples/imagenet/main.py \ + --data=${data_url} \ + --arch=efficientnet-b0 \ + --batch-size=${batch_size} \ + --lr=${lr} \ + --momentum=0 \ + --epochs=${epoches} \ + --autoaug \ + --amp \ + --pm=O1 \ + --loss_scale=128 \ + --val_feq=10 \ + --addr=$(hostname -I |awk '{print $1}') \ + --dist-backend=hccl \ + --multiprocessing-distributed \ + --world-size 1 \ + --rank 0 \ + --device_list ${device_group} > ${train_job_dir}/train_${rank_size}p.log 2>&1 + + +fi + +#taskset -c 0-20 python3.7 ${currentDir}/code/efficientnet.py > ./train.log 2>&1 + +if [ $? -eq 0 ];then + echo ":::ABK 1.0.0 efficientnet train success" + echo ":::ABK 1.0.0 efficientnet train success" >> ${train_job_dir}/train_${rank_size}p.log + echo ":::ABK 1.0.0 efficientnet train success" >> ./hw_efficientnet.log +else + echo ":::ABK 1.0.0 efficientnet train failed" + echo ":::ABK 1.0.0 efficientnet train failed" >> ${train_job_dir}/train_${rank_size}p.log + echo ":::ABK 1.0.0 efficientnet train failed" >> ./hw_efficientnet.log +fi + +endTime=`date +%Y%m%d-%H:%M:%S` +endTime_s=`date +%s` +sumTime=$[ $endTime_s - $startTime_s ] +hour=$(( $sumTime/3600 )) +min=$(( ($sumTime-${hour}*3600)/60 )) +sec=$(( $sumTime-${hour}*3600-${min}*60 )) +echo ":::ABK 1.0.0 efficientnet train total time： ${hour}:${min}:${sec}" >> ${train_job_dir}/${device_id}/hw_efficientnet.log diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/__init__.py b/train/atlas_benchmark-master/image_classification/MobileNet/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/download_dataset.sh b/train/atlas_benchmark-master/image_classification/MobileNet/download_dataset.sh new file mode 100644 index 0000000..a9bf588 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/download_dataset.sh @@ -0,0 +1 @@ +#!/bin/bash diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/README.md b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/README.md new file mode 100644 index 0000000..46314cc --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/README.md @@ -0,0 +1,25 @@ +# MobileNet_pytorch训练说明 + +### 1. 模型训练参数配置 + +在train/yaml/MobileNet.yaml中修改相应配置， 配置项含义: + +``` +pytorch_config: + data_url: 数据集路径 + epoches: 跑多少个epoch + batch_size: 单p默认768 2p 1534 4p 3072 8p默认6144 + lr: 默认参数1p 0.03 2p 0.06 4p 0.12 8p 0.24 + seed: 123456 + docker_image: docker 镜像名称:版本号 +``` + +------ + + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/README.md b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/README.md new file mode 100644 index 0000000..c69bf2e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/README.md @@ -0,0 +1 @@ +# MobileNetV2 NPU训练 \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/hook.py b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/hook.py new file mode 100644 index 0000000..6216f0e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/hook.py @@ -0,0 +1,29 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +g_feat_in = [] +g_feat_out = [] +g_grad_in = [] +g_grad_out = [] + + +def forward_hook_fn(module, input, output): + g_feat_in.append(input) + g_feat_out.append(output) + print(module) + print(input) + print(output) + + +def backward_hook_fn(module, grad_input, grad_output): + g_grad_in.append(grad_input) + g_grad_out.append(grad_output) + print(module) + print(grad_input) + print(grad_input) + + + + + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/main_apex.py b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/main_apex.py new file mode 100644 index 0000000..8ac73ca --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/main_apex.py @@ -0,0 +1,498 @@ +import argparse +import os +import random +import shutil +import time +import warnings + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.distributed as dist +import torch.optim +import torch.multiprocessing as mp +import torch.utils.data +import torch.utils.data.distributed +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torchvision.models as models +from mobilenet import mobilenet_v2 +import torch.npu + +# from torch.utils.tensorboard import SummaryWriter + +from apex import amp +import numpy as np + +from hook import * + +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter + + +# model_names = sorted(name for name in models.__dict__ +# if name.islower() and not name.startswith("__") +# and callable(models.__dict__[name])) + +parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') +parser.add_argument('--data', metavar='DIR', default='/dataset/imagenet', + help='path to dataset') +# parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet18', +# choices=model_names, +# help='model architecture: ' + +# ' | '.join(model_names) + +# ' (default: resnet18)') +parser.add_argument('-j', '--workers', default=4, type=int, metavar='N', + help='number of data loading workers (default: 4)') +parser.add_argument('--epochs', default=90, type=int, metavar='N', + help='number of total epochs to run') +parser.add_argument('--start-epoch', default=0, type=int, metavar='N', + help='manual epoch number (useful on restarts)') +parser.add_argument('-b', '--batch-size', default=256, type=int, + metavar='N', + help='mini-batch size (default: 256), this is the total ' + 'batch size of all GPUs on the current node when ' + 'using Data Parallel or Distributed Data Parallel') +parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, + metavar='LR', help='initial learning rate', dest='lr') +parser.add_argument('--momentum', default=0.9, type=float, metavar='M', + help='momentum') +parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, + metavar='W', help='weight decay (default: 1e-4)', + dest='weight_decay') +parser.add_argument('-p', '--print-freq', default=10, type=int, + metavar='N', help='print frequency (default: 10)') +parser.add_argument('--resume', default='', type=str, metavar='PATH', + help='path to latest checkpoint (default: none)') +parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', + help='evaluate model on validation set') +parser.add_argument('--pretrained', dest='pretrained', action='store_true', + help='use pre-trained model') +parser.add_argument('--world-size', default=-1, type=int, + help='number of nodes for distributed training') +parser.add_argument('--rank', default=-1, type=int, + help='node rank for distributed training') +parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, + help='url used to set up distributed training') +parser.add_argument('--dist-backend', default='nccl', type=str, + help='distributed backend') +parser.add_argument('--seed', default=None, type=int, + help='seed for initializing training. ') +parser.add_argument('--gpu', default=None, type=int, + help='GPU id to use.') +parser.add_argument('--multiprocessing-distributed', action='store_true', + help='Use multi-processing distributed training to launch ' + 'N processes per node, which has N GPUs. This is the ' + 'fastest way to use PyTorch for either single node or ' + 'multi node data parallel training') + +parser.add_argument('--amp', default=False, action='store_true', + help='use amp to train the model') +parser.add_argument('--opt-level', default=None, type=str, help='apex optimize level') +parser.add_argument('--loss-scale-value', default='1024', type=int, help='static loss scale value') + +parser.add_argument('--summary-path', default=None, type=str, help='event file path') +parser.add_argument('--stop-step-num', default=None, type=int, help='after the stop-step, killing the training task') +parser.add_argument('--device', default='npu:0', type=str, help='device type, cpu or npu:x or cuda:x') +parser.add_argument('--eval-freq', default=10, type=int, help='test interval') +parser.add_argument('--hook', default=False, action='store_true', help='pytorch hook') + +best_acc1 = 0 +cur_step = 0 + + +def seed_everything(seed, device): + random.seed(seed) + os.environ['PYTHONHASHSEED'] = str(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + if 'cuda' in device: + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + cudnn.deterministic = True + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + +def main(): + args = parser.parse_args() + + if args.seed is not None: + seed_everything(args.seed, args.device) + + warnings.warn('You have chosen to seed training. ' + 'This will turn on the CUDNN deterministic setting, ' + 'which can slow down your training considerably! ' + 'You may see unexpected behavior when restarting ' + 'from checkpoints.') + + main_worker(args) + + +def main_worker(args): + global best_acc1 + global cur_step + + # sum_writer = SummaryWriter(args.summary_path) + global_step = -1 + + if 'npu' in args.device: + torch.npu.set_device(args.device) + if 'cuda' in args.device: + torch.cuda.set_device(args.device) + + model = mobilenet_v2() + + # set hook + if args.hook: + modules = model.named_modules() + for name, module in modules: + module.register_forward_hook(forward_hook_fn) + module.register_backward_hook(backward_hook_fn) + + optimizer = torch.optim.SGD(model.parameters(), args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay) + + criterion = nn.CrossEntropyLoss() + + if 'npu' in args.device or 'cuda' in args.device: + model = model.to(args.device) + criterion = criterion.to(args.device) + + if args.amp: + model, optimizer = amp.initialize(model, optimizer, opt_level=args.opt_level, loss_scale=args.loss_scale_value) + + # optionally resume from a checkpoint + if args.resume: + if os.path.isfile(args.resume): + print("=> loading checkpoint '{}'".format(args.resume)) + checkpoint = torch.load(args.resume, map_location=args.device) + args.start_epoch = checkpoint['epoch'] + best_acc1 = checkpoint['best_acc1'] + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + if args.amp: + amp.load_state_dict(checkpoint['amp']) + print("=> loaded checkpoint '{}' (epoch {})" + .format(args.resume, checkpoint['epoch'])) + else: + print("=> no checkpoint found at '{}'".format(args.resume)) + + + # Data loading code + traindir = os.path.join(args.data, 'train') + valdir = os.path.join(args.data, 'val') + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + + train_dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ])) + + train_sampler = None + + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), + num_workers=args.workers, pin_memory=True, sampler=train_sampler, drop_last=True) + + val_loader = torch.utils.data.DataLoader( + datasets.ImageFolder(valdir, transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + normalize, + ])), + batch_size=args.batch_size, shuffle=False, + num_workers=args.workers, pin_memory=True, drop_last=True) + + if args.evaluate: + validate(val_loader, model, criterion, args, global_step) + return + + for epoch in range(args.start_epoch, args.epochs): + + # train for one epoch + global_step = train(train_loader, model, criterion, optimizer, epoch, args, global_step) + + if (epoch + 1) % (args.eval_freq) == 0 or epoch == args.epochs - 1: + # evaluate on validation set + acc1 = validate(val_loader, model, criterion, args, global_step) + + # remember best acc@1 and save checkpoint + is_best = acc1 > best_acc1 + best_acc1 = max(acc1, best_acc1) + + # save checkpoint + if args.amp: + save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer': optimizer.state_dict(), + 'amp': amp.state_dict(), + }, is_best) + else: + save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer': optimizer.state_dict(), + }, is_best) + + if args.stop_step_num is not None and cur_step >= args.stop_step_num: + break + + # sum_writer.close() + + +def train(train_loader, model, criterion, optimizer, epoch, args, global_step, sum_writer=None): + global cur_step + + if args.seed is not None: + seed_everything(args.seed + epoch, args.device) + + batch_time = AverageMeter('Time', ':6.3f') + data_time = AverageMeter('Data', ':6.3f') + learning_rate = AverageMeter('LR', ':2.8f') + losses = AverageMeter('Loss', ':6.8f') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(train_loader), + [batch_time, data_time, learning_rate, losses, top1, top5], + prefix="Epoch: [{}]".format(epoch)) + + # switch to train mode + model.train() + + end = time.time() + steps_per_epoch = len(train_loader) + for i, (images, target) in enumerate(train_loader): + + global_step = epoch * steps_per_epoch + i + cur_step = global_step + + lr = adjust_learning_rate(optimizer, global_step, steps_per_epoch, args) + + learning_rate.update(lr) + + # sum_writer.add_scalar('learning rate', lr, global_step) + + # measure data loading time + data_time.update(time.time() - end) + + if 'npu' in args.device: + target = target.to(torch.int32) + + if 'npu' in args.device or 'cuda' in args.device: + images = images.to(args.device, non_blocking=True) + target = target.to(args.device, non_blocking=True) + + # output = None + # loss = None + # with torch.autograd.profiler.profile(record_shapes=True, use_npu=True) as prof: + + # compute output + output = model(images) + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # compute gradient and do SGD step + optimizer.zero_grad() + if args.amp: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + + # sum_writer.add_scalar('Accuary/train/top1', acc1, global_step) + # sum_writer.add_scalar('Accuary/train/top5', acc5, global_step) + # sum_writer.add_scalar('Loss/train/loss', loss, global_step) + + optimizer.step() + # for name, parms in model.named_parameters(): + # print('-->name:', name, ' -->grad_value_max:', torch.max(parms.grad), ' -->grad_value_min:', torch.min(parms.grad)) + + # print(prof.key_averages().table()) + # prof.export_chrome_trace("mobilenetv2_{}_npu.prof".format(i)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + progress.display(i) + + if args.stop_step_num is not None and cur_step >= args.stop_step_num: + break + + print(' * FPS@all {:.3f}'.format(args.batch_size / batch_time.avg)) + hwlog.remark_print(key=hwlog.FPS, value=' * FPS@all {:.3f}'.format(args.batch_size / batch_time.avg)) + return global_step + + +def validate(val_loader, model, criterion, args, global_step, sum_writer=None): + batch_time = AverageMeter('Time', ':6.3f') + losses = AverageMeter('Loss', ':.4e') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(val_loader), + [batch_time, losses, top1, top5], + prefix='Test: ') + + # switch to evaluate mode + model.eval() + + with torch.no_grad(): + end = time.time() + for i, (images, target) in enumerate(val_loader): + + if 'npu' in args.device: + target = target.to(torch.int32) + + if 'npu' in args.device or 'cuda' in args.device: + images = images.to(args.device, non_blocking=True) + target = target.to(args.device, non_blocking=True) + + # compute output + output = model(images) + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + progress.display(i) + + # TODO: this should also be done with the ProgressMeter + print(' * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}' + .format(top1=top1, top5=top5)) + + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value="{top1.avg:.3f}".format(top1=top1)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value="{top5.avg:.3f}".format(top5=top5)) + + #if not args.evaluate: + # # sum_writer.add_scalar('Loss/validation/loss', losses, global_step) + # sum_writer.add_scalar('Accuary/validation/top1', top1.avg, global_step) + # sum_writer.add_scalar('Accuary/validation/top5', top5.avg, global_step) + + return top1.avg + + +def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): + torch.save(state, filename) + if is_best: + shutil.copyfile(filename, 'model_best.pth.tar') + + +class AverageMeter(object): + """Computes and stores the average and current value""" + def __init__(self, name, fmt=':f'): + self.name = name + self.fmt = fmt + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + +class ProgressMeter(object): + def __init__(self, num_batches, meters, prefix=""): + self.batch_fmtstr = self._get_batch_fmtstr(num_batches) + self.meters = meters + self.prefix = prefix + + def display(self, batch): + entries = [self.prefix + self.batch_fmtstr.format(batch)] + entries += [str(meter) for meter in self.meters] + print('\t'.join(entries)) + # 日志打点 + train_acc1 = str(entries).split("Acc@1")[1].strip().split(" ")[0] + train_acc5 = str(entries).split("Acc@5")[1].strip().split(" ")[0] + hwlog.remark_print(key=hwlog.TRAIN_ACCURACY_TOP1, value=train_acc1) + hwlog.remark_print(key=hwlog.TRAIN_ACCURACY_TOP5, value=train_acc5) + + def _get_batch_fmtstr(self, num_batches): + num_digits = len(str(num_batches // 1)) + fmt = '{:' + str(num_digits) + 'd}' + return '[' + fmt + '/' + fmt.format(num_batches) + ']' + + +def adjust_learning_rate(optimizer, global_step, steps_per_epoch, args): + """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" + # lr = args.lr * (0.98 ** (epoch / 2.5)) + lr = args.lr * (0.98 ** (global_step // int(steps_per_epoch * 2.5))) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + return lr + + +def accuracy(output, target, topk=(1,)): + """Computes the accuracy over the k top predictions for the specified values of k""" + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +if __name__ == '__main__': + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("pytorch") + config_info = get_model_parameter("pytorch_config") + initinal_data = {"base_lr": 0.1, "dataset": "imagenet", "optimizer": "SGD", "loss_scale": 1024} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/mobilenet.py b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/mobilenet.py new file mode 100644 index 0000000..356f491 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/mobilenet.py @@ -0,0 +1,179 @@ +from torch import nn +# from .utils import load_state_dict_from_url + + +__all__ = ['MobileNetV2', 'mobilenet_v2'] + + +model_urls = { + 'mobilenet_v2': 'https://download.pytorch.org/models/mobilenet_v2-b0353104.pth', +} + + +def _make_divisible(v, divisor, min_value=None): + """ + This function is taken from the original tf repo. + It ensures that all layers have a channel number that is divisible by 8 + It can be seen here: + https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py + :param v: + :param divisor: + :param min_value: + :return: + """ + if min_value is None: + min_value = divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_v < 0.9 * v: + new_v += divisor + return new_v + + +class ConvBNReLU(nn.Sequential): + def __init__(self, in_planes, out_planes, kernel_size=3, stride=1, groups=1): + padding = (kernel_size - 1) // 2 + super(ConvBNReLU, self).__init__( + nn.Conv2d(in_planes, out_planes, kernel_size, stride, padding, groups=groups, bias=False), + nn.BatchNorm2d(out_planes), + nn.ReLU6(inplace=True) + # nn.ReLU(inplace=True) + ) + + +class InvertedResidual(nn.Module): + def __init__(self, inp, oup, stride, expand_ratio): + super(InvertedResidual, self).__init__() + self.stride = stride + assert stride in [1, 2] + + hidden_dim = int(round(inp * expand_ratio)) + self.use_res_connect = self.stride == 1 and inp == oup + + layers = [] + if expand_ratio != 1: + # pw + layers.append(ConvBNReLU(inp, hidden_dim, kernel_size=1)) + layers.extend([ + # dw + ConvBNReLU(hidden_dim, hidden_dim, stride=stride, groups=hidden_dim), + # pw-linear + nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + ]) + self.conv = nn.Sequential(*layers) + + def forward(self, x): + if self.use_res_connect: + return x + self.conv(x) + else: + return self.conv(x) + + +class MobileNetV2(nn.Module): + def __init__(self, + num_classes=1000, + width_mult=1.0, + inverted_residual_setting=None, + round_nearest=8, + block=None): + """ + MobileNet V2 main class + + Args: + num_classes (int): Number of classes + width_mult (float): Width multiplier - adjusts number of channels in each layer by this amount + inverted_residual_setting: Network structure + round_nearest (int): Round the number of channels in each layer to be a multiple of this number + Set to 1 to turn off rounding + block: Module specifying inverted residual building block for mobilenet + + """ + super(MobileNetV2, self).__init__() + + if block is None: + block = InvertedResidual + input_channel = 32 + last_channel = 1280 + + if inverted_residual_setting is None: + inverted_residual_setting = [ + # t, c, n, s + [1, 16, 1, 1], + [6, 24, 2, 2], + [6, 32, 3, 2], + [6, 64, 4, 2], + [6, 96, 3, 1], + [6, 160, 3, 2], + [6, 320, 1, 1], + ] + + # only check the first element, assuming user knows t,c,n,s are required + if len(inverted_residual_setting) == 0 or len(inverted_residual_setting[0]) != 4: + raise ValueError("inverted_residual_setting should be non-empty " + "or a 4-element list, got {}".format(inverted_residual_setting)) + + # building first layer + input_channel = _make_divisible(input_channel * width_mult, round_nearest) + self.last_channel = _make_divisible(last_channel * max(1.0, width_mult), round_nearest) + features = [ConvBNReLU(3, input_channel, stride=2)] + # building inverted residual blocks + for t, c, n, s in inverted_residual_setting: + output_channel = _make_divisible(c * width_mult, round_nearest) + for i in range(n): + stride = s if i == 0 else 1 + features.append(block(input_channel, output_channel, stride, expand_ratio=t)) + input_channel = output_channel + # building last several layers + features.append(ConvBNReLU(input_channel, self.last_channel, kernel_size=1)) + # make it nn.Sequential + self.features = nn.Sequential(*features) + + # building classifier + self.classifier = nn.Sequential( + # p=0.2 + nn.Dropout(0.2), + nn.Linear(self.last_channel, num_classes), + ) + + # weight initialization + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out') + if m.bias is not None: + nn.init.zeros_(m.bias) + elif isinstance(m, nn.BatchNorm2d): + nn.init.ones_(m.weight) + nn.init.zeros_(m.bias) + elif isinstance(m, nn.Linear): + nn.init.normal_(m.weight, 0, 0.01) + nn.init.zeros_(m.bias) + + def _forward_impl(self, x): + # This exists since TorchScript doesn't support inheritance, so the superclass method + # (this one) needs to have a name other than `forward` that can be accessed in a subclass + x = self.features(x) + # Cannot use "squeeze" as batch-size can be 1 => must use reshape with x.shape[0] + x = nn.functional.adaptive_avg_pool2d(x, 1).reshape(x.shape[0], -1) + x = self.classifier(x) + return x + + def forward(self, x): + return self._forward_impl(x) + + +def mobilenet_v2(pretrained=False, progress=True, **kwargs): + """ + Constructs a MobileNetV2 architecture from + `"MobileNetV2: Inverted Residuals and Linear Bottlenecks" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + model = MobileNetV2(**kwargs) + # if pretrained: + # state_dict = load_state_dict_from_url(model_urls['mobilenet_v2'], + # progress=progress) + # model.load_state_dict(state_dict) + return model diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/profiling.json b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/profiling.json new file mode 100644 index 0000000..d0d9b1f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/1p/profiling.json @@ -0,0 +1,18 @@ +{ + "startCfg": + [ + { + "jobID": "123456789", + "deviceID": ["0"], + "features": + [ + { + "name": "task_trace" + }, + { + "name": "training_trace" + } + ] + } + ] +} diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/README.md b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/README.md new file mode 100644 index 0000000..c69bf2e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/README.md @@ -0,0 +1 @@ +# MobileNetV2 NPU训练 \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/eval.sh b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/eval.sh new file mode 100644 index 0000000..16d5c5e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/eval.sh @@ -0,0 +1,22 @@ +export ASCEND_HOME=/usr/local/Ascend +export LD_LIBRARY_PATH=/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/ +export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/te:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/topi:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/hccl:/usr/local/Ascend/ascend-toolkit/latest/tfplugin/python/site-packages:$currentDir +export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin +export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + +export SLOG_PRINT_TO_STDOUT=0 +su HwHiAiUser -c "adc --host 0.0.0.0:22118 --log \"SetLogLevel(0)[error]\" --device 7" + +export TASK_QUEUE_ENABLE=0 +taskset -c 111-150 python3 densenet121_1p_main.py \ + --workers 40 \ + --arch densenet121 \ + --npu 7 \ + --lr 0.1 \ + --momentum 0.9 \ + --amp \ + --batch-size 256 \ + --epoch 90 \ + --evaluate \ + --resume checkpoint.pth.tar \ + --data /opt/npu/dataset/imagenet \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/hook.py b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/hook.py new file mode 100644 index 0000000..6216f0e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/hook.py @@ -0,0 +1,29 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +g_feat_in = [] +g_feat_out = [] +g_grad_in = [] +g_grad_out = [] + + +def forward_hook_fn(module, input, output): + g_feat_in.append(input) + g_feat_out.append(output) + print(module) + print(input) + print(output) + + +def backward_hook_fn(module, grad_input, grad_output): + g_grad_in.append(grad_input) + g_grad_out.append(grad_output) + print(module) + print(grad_input) + print(grad_input) + + + + + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/main_apex.py b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/main_apex.py new file mode 100644 index 0000000..ace66d3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/main_apex.py @@ -0,0 +1,556 @@ +import argparse +import os +import random +import shutil +import time +import warnings + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.distributed as dist +import torch.optim +import torch.multiprocessing as mp +import torch.utils.data +import torch.utils.data.distributed +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torchvision.models as models +from mobilenet import mobilenet_v2 +import torch.npu +import torch.cuda + +from torch.utils.tensorboard import SummaryWriter + +from apex import amp +import numpy as np + +from hook import * + + +# model_names = sorted(name for name in models.__dict__ +# if name.islower() and not name.startswith("__") +# and callable(models.__dict__[name])) + +parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') +parser.add_argument('--data', metavar='DIR', default='/dataset/imagenet', + help='path to dataset') +# parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet18', +# choices=model_names, +# help='model architecture: ' + +# ' | '.join(model_names) + +# ' (default: resnet18)') +parser.add_argument('-j', '--workers', default=4, type=int, metavar='N', + help='number of data loading workers (default: 4)') +parser.add_argument('--epochs', default=90, type=int, metavar='N', + help='number of total epochs to run') +parser.add_argument('--start-epoch', default=0, type=int, metavar='N', + help='manual epoch number (useful on restarts)') +parser.add_argument('-b', '--batch-size', default=256, type=int, + metavar='N', + help='mini-batch size (default: 256), this is the total ' + 'batch size of all GPUs on the current node when ' + 'using Data Parallel or Distributed Data Parallel') +parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, + metavar='LR', help='initial learning rate', dest='lr') +parser.add_argument('--momentum', default=0.9, type=float, metavar='M', + help='momentum') +parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, + metavar='W', help='weight decay (default: 1e-4)', + dest='weight_decay') +parser.add_argument('-p', '--print-freq', default=10, type=int, + metavar='N', help='print frequency (default: 10)') +parser.add_argument('--resume', default='', type=str, metavar='PATH', + help='path to latest checkpoint (default: none)') +parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', + help='evaluate model on validation set') +parser.add_argument('--pretrained', dest='pretrained', action='store_true', + help='use pre-trained model') +# parser.add_argument('--world-size', default=-1, type=int, +# help='number of nodes for distributed training') +parser.add_argument('--node-nums', default=1, type=int, + help='number of nodes for distributed training') +parser.add_argument('--rank', default=0, type=int, + help='node rank for distributed training') +parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, + help='url used to set up distributed training') +parser.add_argument('--dist-backend', default='nccl', type=str, + help='distributed backend') +parser.add_argument('--seed', default=None, type=int, + help='seed for initializing training. ') +parser.add_argument('--gpu', default=None, type=int, + help='GPU id to use.') +parser.add_argument('--multiprocessing-distributed', action='store_true', + help='Use multi-processing distributed training to launch ' + 'N processes per node, which has N GPUs. This is the ' + 'fastest way to use PyTorch for either single node or ' + 'multi node data parallel training') + +parser.add_argument('--addr', default='10.136.181.115', type=str, + help='master addr') +parser.add_argument('--device-id', default=None, type=int, + help='GPU id to use.') + +parser.add_argument('--amp', default=False, action='store_true', + help='use amp to train the model') +parser.add_argument('--opt-level', default=None, type=str, help='apex optimize level') +parser.add_argument('--loss-scale-value', default='1024', type=int, help='static loss scale value') + +parser.add_argument('--summary-path', default=None, type=str, help='event file path') +parser.add_argument('--stop-step-num', default=None, type=int, help='after the stop-step, killing the training task') +parser.add_argument('--device', default='npu', type=str, help='device type, cpu or npu:x or cuda') +parser.add_argument('--eval-freq', default=10, type=int, help='test interval') +parser.add_argument('--hook', default=False, action='store_true', help='pytorch hook') + +best_acc1 = 0 +cur_step = 0 + + +def seed_everything(seed, device): + random.seed(seed) + os.environ['PYTHONHASHSEED'] = str(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + if 'cuda' in device: + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + cudnn.deterministic = True + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + +def main(): + args = parser.parse_args() + + if args.seed is not None: + seed_everything(args.seed, args.device) + + warnings.warn('You have chosen to seed training. ' + 'This will turn on the CUDNN deterministic setting, ' + 'which can slow down your training considerably! ' + 'You may see unexpected behavior when restarting ' + 'from checkpoints.') + + os.environ['MASTER_ADDR'] = args.addr + os.environ['MASTER_PORT'] = '90000' + + args.distributed = args.node_nums > 1 or args.multiprocessing_distributed + if not args.distributed: + print('dist param is not correct!') + return + + if args.device == 'npu': + # device_nums_per_node = torch.npu.device_count() + device_nums_per_node = 2 + elif args.device == 'cuda': + device_nums_per_node = torch.cuda.device_count() + else: + print('unknown device type[npu/cuda]!') + return + + if args.multiprocessing_distributed: + args.world_size = device_nums_per_node * args.node_nums # world_size means nums of all devices or nums of processes + if args.device == 'npu': + # main_worker(args.device_id, ngpus_per_node, args) # 需要外层脚本启多个进程 + mp.spawn(main_worker, nprocs=device_nums_per_node, args=(device_nums_per_node, args)) # 这里起子进程，就不需要外层脚本启多个进程了 + else: + mp.spawn(main_worker, nprocs=device_nums_per_node, args=(device_nums_per_node, args)) + else: + print('dist param is not correct!') + return + # main_worker(args.device_id, device_nums_per_node, args) + + +# first param must be the index of PID +def main_worker(pid_idx, device_nums_per_node, args): + global best_acc1 + global cur_step + + # dist set + sum_writer = SummaryWriter(args.summary_path) + global_step = -1 + + if args.distributed: + if args.multiprocessing_distributed: + # For multiprocessing distributed training, rank needs to be the + # global rank among all the processes + args.rank = pid_idx # args.rank * device_nums_per_node + pid_idx + args.pid_idx = pid_idx + + if args.device == 'npu': + dist.init_process_group(backend=args.dist_backend, world_size=args.world_size, rank=args.rank) + else: + dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + + if args.distributed: + # For multiprocessing distributed, DistributedDataParallel constructor + # should always set the single device scope, otherwise, + # DistributedDataParallel will use all available devices. + if args.device == 'npu': + loc = 'npu:{}'.format(pid_idx) + torch.npu.set_device(loc) + else: + torch.cuda.set_device(pid_idx) + + args.batch_size = int(args.batch_size / device_nums_per_node) + args.workers = int((args.workers + device_nums_per_node - 1) / device_nums_per_node) + + # Data loading code + traindir = os.path.join(args.data, 'train') + valdir = os.path.join(args.data, 'val') + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + + train_dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ])) + + if args.distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), + num_workers=args.workers, pin_memory=True, sampler=train_sampler, drop_last=True) + + val_loader = torch.utils.data.DataLoader( + datasets.ImageFolder(valdir, transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + normalize, + ])), + batch_size=args.batch_size, shuffle=False, + num_workers=args.workers, pin_memory=True, drop_last=True) + + # define model and train + model = mobilenet_v2() + + criterion = nn.CrossEntropyLoss() + + loc = None + if 'npu' == args.device: + loc = 'npu:{}'.format(pid_idx) + elif 'cuda' == args.device: + loc = 'cuda:{}'.format(pid_idx) + model = model.to(loc) + + criterion = criterion.to(loc) + + optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) + + if args.amp: + model, optimizer = amp.initialize(model, optimizer, opt_level=args.opt_level, loss_scale=args.loss_scale_value) + + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[pid_idx], broadcast_buffers=False) + # model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model) + + # set hook + if args.hook: + modules = model.named_modules() + for name, module in modules: + module.register_forward_hook(forward_hook_fn) + module.register_backward_hook(backward_hook_fn) + + # optionally resume from a checkpoint + if args.resume: + if os.path.isfile(args.resume): + print("=> loading checkpoint '{}'".format(args.resume)) + checkpoint = torch.load(args.resume, map_location=args.device) + args.start_epoch = checkpoint['epoch'] + best_acc1 = checkpoint['best_acc1'] + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + if args.amp: + amp.load_state_dict(checkpoint['amp']) + print("=> loaded checkpoint '{}' (epoch {})" + .format(args.resume, checkpoint['epoch'])) + else: + print("=> no checkpoint found at '{}'".format(args.resume)) + + if args.evaluate: + validate(val_loader, model, criterion, args, global_step, sum_writer) + return + + for epoch in range(args.start_epoch, args.epochs): + + # train for one epoch + global_step = train(train_loader, model, criterion, optimizer, epoch, args, global_step, sum_writer, device_nums_per_node) + + if (epoch + 1) % args.eval_freq == 0 or epoch == args.epochs - 1: + # evaluate on validation set + acc1 = validate(val_loader, model, criterion, args, global_step, sum_writer, device_nums_per_node) + + # remember best acc@1 and save checkpoint + is_best = acc1 > best_acc1 + best_acc1 = max(acc1, best_acc1) + + # save checkpoint + if args.amp: + save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer': optimizer.state_dict(), + 'amp': amp.state_dict(), + }, is_best) + else: + save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer': optimizer.state_dict(), + }, is_best) + + if args.stop_step_num is not None and cur_step >= args.stop_step_num: + break + + sum_writer.close() + + +def train(train_loader, model, criterion, optimizer, epoch, args, global_step, sum_writer, device_nums_per_node): + global cur_step + + if args.seed is not None: + seed_everything(args.seed + epoch, args.device) + + batch_time = AverageMeter('Time', ':6.3f') + data_time = AverageMeter('Data', ':6.3f') + learning_rate = AverageMeter('LR', ':2.8f') + losses = AverageMeter('Loss', ':6.8f') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(train_loader), + [batch_time, data_time, learning_rate, losses, top1, top5], + prefix="Epoch: [{}]".format(epoch)) + + # switch to train mode + model.train() + + end = time.time() + steps_per_epoch = len(train_loader) + for i, (images, target) in enumerate(train_loader): + + global_step = epoch * steps_per_epoch + i + cur_step = global_step + + lr = adjust_learning_rate(optimizer, global_step, steps_per_epoch, args) + + learning_rate.update(lr) + + sum_writer.add_scalar('learning rate', lr, global_step) + + # measure data loading time + data_time.update(time.time() - end) + + if 'npu' in args.device: + target = target.to(torch.int32) + + loc = None + if 'npu' in args.device: + loc = 'npu:{}'.format(args.pid_idx) + elif 'cuda' in args.device: + loc = 'cuda:{}'.format(args.pid_idx) + images = images.to(loc, non_blocking=True) + target = target.to(loc, non_blocking=True) + + # output = None + # loss = None + # with torch.autograd.profiler.profile(record_shapes=True, use_npu=True) as prof: + + # compute output + output = model(images) + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # compute gradient and do SGD step + optimizer.zero_grad() + if args.amp: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + + sum_writer.add_scalar('Accuary/train/top1', acc1, global_step) + sum_writer.add_scalar('Accuary/train/top5', acc5, global_step) + sum_writer.add_scalar('Loss/train/loss', loss, global_step) + + optimizer.step() + # for name, parms in model.named_parameters(): + # print('-->name:', name, ' -->grad_value_max:', torch.max(parms.grad), ' -->grad_value_min:', torch.min(parms.grad)) + + # print(prof.key_averages().table()) + # prof.export_chrome_trace("mobilenetv2_{}_npu.prof".format(i)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + if not args.multiprocessing_distributed or \ + (args.multiprocessing_distributed and args.rank % device_nums_per_node == 0): + progress.display(i) + + if not args.multiprocessing_distributed or \ + (args.multiprocessing_distributed and args.rank % device_nums_per_node == 0): + print('FPS@all: {:.3f}'.format(8 * args.batch_size / batch_time.avg)) + + if args.stop_step_num is not None and cur_step >= args.stop_step_num: + break + + return global_step + + +def validate(val_loader, model, criterion, args, global_step, sum_writer, device_nums_per_node): + batch_time = AverageMeter('Time', ':6.3f') + losses = AverageMeter('Loss', ':.4e') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(val_loader), + [batch_time, losses, top1, top5], + prefix='Test: ') + + # switch to evaluate mode + model.eval() + + with torch.no_grad(): + end = time.time() + for i, (images, target) in enumerate(val_loader): + + if 'npu' in args.device: + target = target.to(torch.int32) + + loc = None + if 'npu' in args.device: + loc = 'npu:{}'.format(args.pid_idx) + elif 'cuda' in args.device: + loc = 'cuda:{}'.format(args.pid_idx) + images = images.to(loc, non_blocking=True) + target = target.to(loc, non_blocking=True) + + # compute output + output = model(images) + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + if not args.multiprocessing_distributed or \ + (args.multiprocessing_distributed and args.rank % device_nums_per_node == 0): + progress.display(i) + + # TODO: this should also be done with the ProgressMeter + print(' * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}' + .format(top1=top1, top5=top5)) + if not args.multiprocessing_distributed or \ + (args.multiprocessing_distributed and args.rank % device_nums_per_node == 0): + print("[device id:", args.gpu, "]", '[AVG-ACC] * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'.format(top1=top1, top5=top5)) + + if not args.evaluate: + # sum_writer.add_scalar('Loss/validation/loss', losses, global_step) + sum_writer.add_scalar('Accuary/validation/top1', top1.avg, global_step) + sum_writer.add_scalar('Accuary/validation/top5', top5.avg, global_step) + + return top1.avg + + +def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): + torch.save(state, filename) + if is_best: + shutil.copyfile(filename, 'model_best.pth.tar') + + +class AverageMeter(object): + """Computes and stores the average and current value""" + def __init__(self, name, fmt=':f'): + self.name = name + self.fmt = fmt + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + +class ProgressMeter(object): + def __init__(self, num_batches, meters, prefix=""): + self.batch_fmtstr = self._get_batch_fmtstr(num_batches) + self.meters = meters + self.prefix = prefix + + def display(self, batch): + entries = [self.prefix + self.batch_fmtstr.format(batch)] + entries += [str(meter) for meter in self.meters] + print('\t'.join(entries)) + + def _get_batch_fmtstr(self, num_batches): + num_digits = len(str(num_batches // 1)) + fmt = '{:' + str(num_digits) + 'd}' + return '[' + fmt + '/' + fmt.format(num_batches) + ']' + + +def adjust_learning_rate(optimizer, global_step, steps_per_epoch, args): + """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" + # lr = args.lr * (0.98 ** (epoch / 2.5)) + lr = args.lr * (0.98 ** (global_step // int(steps_per_epoch * 2.5))) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + return lr + + +def accuracy(output, target, topk=(1,)): + """Computes the accuracy over the k top predictions for the specified values of k""" + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +if __name__ == '__main__': + main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/mobilenet.py b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/mobilenet.py new file mode 100644 index 0000000..356f491 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/mobilenet.py @@ -0,0 +1,179 @@ +from torch import nn +# from .utils import load_state_dict_from_url + + +__all__ = ['MobileNetV2', 'mobilenet_v2'] + + +model_urls = { + 'mobilenet_v2': 'https://download.pytorch.org/models/mobilenet_v2-b0353104.pth', +} + + +def _make_divisible(v, divisor, min_value=None): + """ + This function is taken from the original tf repo. + It ensures that all layers have a channel number that is divisible by 8 + It can be seen here: + https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py + :param v: + :param divisor: + :param min_value: + :return: + """ + if min_value is None: + min_value = divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_v < 0.9 * v: + new_v += divisor + return new_v + + +class ConvBNReLU(nn.Sequential): + def __init__(self, in_planes, out_planes, kernel_size=3, stride=1, groups=1): + padding = (kernel_size - 1) // 2 + super(ConvBNReLU, self).__init__( + nn.Conv2d(in_planes, out_planes, kernel_size, stride, padding, groups=groups, bias=False), + nn.BatchNorm2d(out_planes), + nn.ReLU6(inplace=True) + # nn.ReLU(inplace=True) + ) + + +class InvertedResidual(nn.Module): + def __init__(self, inp, oup, stride, expand_ratio): + super(InvertedResidual, self).__init__() + self.stride = stride + assert stride in [1, 2] + + hidden_dim = int(round(inp * expand_ratio)) + self.use_res_connect = self.stride == 1 and inp == oup + + layers = [] + if expand_ratio != 1: + # pw + layers.append(ConvBNReLU(inp, hidden_dim, kernel_size=1)) + layers.extend([ + # dw + ConvBNReLU(hidden_dim, hidden_dim, stride=stride, groups=hidden_dim), + # pw-linear + nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + ]) + self.conv = nn.Sequential(*layers) + + def forward(self, x): + if self.use_res_connect: + return x + self.conv(x) + else: + return self.conv(x) + + +class MobileNetV2(nn.Module): + def __init__(self, + num_classes=1000, + width_mult=1.0, + inverted_residual_setting=None, + round_nearest=8, + block=None): + """ + MobileNet V2 main class + + Args: + num_classes (int): Number of classes + width_mult (float): Width multiplier - adjusts number of channels in each layer by this amount + inverted_residual_setting: Network structure + round_nearest (int): Round the number of channels in each layer to be a multiple of this number + Set to 1 to turn off rounding + block: Module specifying inverted residual building block for mobilenet + + """ + super(MobileNetV2, self).__init__() + + if block is None: + block = InvertedResidual + input_channel = 32 + last_channel = 1280 + + if inverted_residual_setting is None: + inverted_residual_setting = [ + # t, c, n, s + [1, 16, 1, 1], + [6, 24, 2, 2], + [6, 32, 3, 2], + [6, 64, 4, 2], + [6, 96, 3, 1], + [6, 160, 3, 2], + [6, 320, 1, 1], + ] + + # only check the first element, assuming user knows t,c,n,s are required + if len(inverted_residual_setting) == 0 or len(inverted_residual_setting[0]) != 4: + raise ValueError("inverted_residual_setting should be non-empty " + "or a 4-element list, got {}".format(inverted_residual_setting)) + + # building first layer + input_channel = _make_divisible(input_channel * width_mult, round_nearest) + self.last_channel = _make_divisible(last_channel * max(1.0, width_mult), round_nearest) + features = [ConvBNReLU(3, input_channel, stride=2)] + # building inverted residual blocks + for t, c, n, s in inverted_residual_setting: + output_channel = _make_divisible(c * width_mult, round_nearest) + for i in range(n): + stride = s if i == 0 else 1 + features.append(block(input_channel, output_channel, stride, expand_ratio=t)) + input_channel = output_channel + # building last several layers + features.append(ConvBNReLU(input_channel, self.last_channel, kernel_size=1)) + # make it nn.Sequential + self.features = nn.Sequential(*features) + + # building classifier + self.classifier = nn.Sequential( + # p=0.2 + nn.Dropout(0.2), + nn.Linear(self.last_channel, num_classes), + ) + + # weight initialization + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out') + if m.bias is not None: + nn.init.zeros_(m.bias) + elif isinstance(m, nn.BatchNorm2d): + nn.init.ones_(m.weight) + nn.init.zeros_(m.bias) + elif isinstance(m, nn.Linear): + nn.init.normal_(m.weight, 0, 0.01) + nn.init.zeros_(m.bias) + + def _forward_impl(self, x): + # This exists since TorchScript doesn't support inheritance, so the superclass method + # (this one) needs to have a name other than `forward` that can be accessed in a subclass + x = self.features(x) + # Cannot use "squeeze" as batch-size can be 1 => must use reshape with x.shape[0] + x = nn.functional.adaptive_avg_pool2d(x, 1).reshape(x.shape[0], -1) + x = self.classifier(x) + return x + + def forward(self, x): + return self._forward_impl(x) + + +def mobilenet_v2(pretrained=False, progress=True, **kwargs): + """ + Constructs a MobileNetV2 architecture from + `"MobileNetV2: Inverted Residuals and Linear Bottlenecks" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + model = MobileNetV2(**kwargs) + # if pretrained: + # state_dict = load_state_dict_from_url(model_urls['mobilenet_v2'], + # progress=progress) + # model.load_state_dict(state_dict) + return model diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/mobilenetv2_8p_main.py b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/mobilenetv2_8p_main.py new file mode 100644 index 0000000..ff89201 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/mobilenetv2_8p_main.py @@ -0,0 +1,638 @@ +# -*- coding: utf-8 -*- + +import argparse +import os +import random +import shutil +import time +import warnings + +import numpy as np + +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.distributed as dist +import torch.optim +import torch.multiprocessing as mp +import torch.utils.data +import torch.utils.data.distributed +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torchvision.models as models + +from mobilenet import mobilenet_v2 +from apex import amp +from multi_epochs_dataloader import MultiEpochsDataLoader + +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter + +BATCH_SIZE = 4096 +OPTIMIZER_BATCH_SIZE = 4096 +# model_names = sorted(name for name in models.__dict__ +# if name.islower() and not name.startswith("__") +# and callable(models.__dict__[name])) + +parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') +parser.add_argument('--data', metavar='DIR', default='/opt/npu/dataset/imagenet', + help='path to dataset') +# parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet50', +# choices=model_names, +# help='model architecture: ' + +# ' | '.join(model_names) + +# ' (default: resnet18)') +parser.add_argument('-j', '--workers', default=32, type=int, metavar='N', + help='number of data loading workers (default: 4)') +parser.add_argument('--epochs', default=90, type=int, metavar='N', + help='number of total epochs to run') +parser.add_argument('--start-epoch', default=0, type=int, metavar='N', + help='manual epoch number (useful on restarts)') +parser.add_argument('-b', '--batch-size', default=BATCH_SIZE, type=int, + metavar='N', + help='mini-batch size (default: 256), this is the total ' + 'batch size of all GPUs on the current node when ' + 'using Data Parallel or Distributed Data Parallel') +parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, + metavar='LR', help='initial learning rate', dest='lr') +parser.add_argument('--momentum', default=0.9, type=float, metavar='M', + help='momentum') +parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, + metavar='W', help='weight decay (default: 1e-4)', + dest='weight_decay') +parser.add_argument('--workspace', type=str, default='./', metavar='DIR', + help='path to directory where checkpoints will be stored') +parser.add_argument('-p', '--print-freq', default=10, type=int, + metavar='N', help='print frequency (default: 10)') +parser.add_argument('-ef', '--eval-freq', default=5, type=int, + metavar='N', help='evaluate frequency (default: 5)') +parser.add_argument('--resume', default='', type=str, metavar='PATH', + help='path to latest checkpoint (default: none)') +parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', + help='evaluate model on validation set') +parser.add_argument('--pretrained', dest='pretrained', action='store_true', + help='use pre-trained model') +parser.add_argument('--world-size', default=-1, type=int, + help='number of nodes for distributed training') +parser.add_argument('--rank', default=-1, type=int, + help='node rank for distributed training') +parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, + help='url used to set up distributed training') +parser.add_argument('--dist-backend', default='nccl', type=str, + help='distributed backend') +parser.add_argument('--seed', default=None, type=int, + help='seed for initializing training. ') +parser.add_argument('--gpu', default=None, type=int, + help='GPU id to use.') +parser.add_argument('--multiprocessing-distributed', action='store_true', + help='Use multi-processing distributed training to launch ' + 'N processes per node, which has N GPUs. This is the ' + 'fastest way to use PyTorch for either single node or ' + 'multi node data parallel training') +parser.add_argument('-bm', '--benchmark', default=0, type=int, + metavar='N', help='set benchmark status (default: 1,run benchmark)') +parser.add_argument('--device', default='npu', type=str, help='npu or gpu') +parser.add_argument('--addr', default='10.136.181.115', type=str, help='master addr') +parser.add_argument('--checkpoint-nameprefix', default='checkpoint', type=str, help='checkpoint-nameprefix') +parser.add_argument('--checkpoint-freq', default=0, type=int, + metavar='N', help='checkpoint frequency (default: 0)' + '0: save only one file whitch per epoch;' + 'n: save diff file per n epoch' + '-1:no checkpoint,not support') + +# apex +parser.add_argument('--amp', default=False, action='store_true', + help='use amp to train the model') +parser.add_argument('--loss-scale', default=64., type=float, + help='loss scale using in amp, default -1 means dynamic') +parser.add_argument('--opt-level', default='O2', type=str, + help='loss scale using in amp, default -1 means dynamic') + +warnings.filterwarnings('ignore') +best_acc1 = 0 + + +def main(): + args = parser.parse_args() + print("===============main()=================") + print(args) + print("===============main()=================") + + os.environ['KERNEL_NAME_ID'] = str(0) + print("++++++++++++++++++ KERNEL_NAME_ID:", os.environ['KERNEL_NAME_ID']) + + if args.seed is not None: + random.seed(args.seed) + torch.manual_seed(args.seed) + cudnn.deterministic = True + warnings.warn('You have chosen to seed training. ' + 'This will turn on the CUDNN deterministic setting, ' + 'which can slow down your training considerably! ' + 'You may see unexpected behavior when restarting ' + 'from checkpoints.') + + os.environ['MASTER_ADDR'] = args.addr # '10.136.181.51' + os.environ['MASTER_PORT'] = '59629' + + if args.gpu is not None: + warnings.warn('You have chosen a specific GPU. This will completely ' + 'disable data parallelism.') + + if args.dist_url == "env://" and args.world_size == -1: + args.world_size = int(os.environ["WORLD_SIZE"]) + + args.distributed = args.world_size > 1 or args.multiprocessing_distributed + + if args.device == 'npu': + ngpus_per_node = torch.npu.device_count() + else: + ngpus_per_node = torch.cuda.device_count() + if args.multiprocessing_distributed: + # Since we have ngpus_per_node processes per node, the total world_size + # needs to be adjusted accordingly + args.world_size = ngpus_per_node * args.world_size + # Use torch.multiprocessing.spawn to launch distributed processes: the + # main_worker process function + # The child process uses the environment variables of the parent process, + # we have to set KERNEL_NAME_ID for every proc + if args.device == 'npu': + # main_worker(args.gpu, ngpus_per_node, args) + mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) + else: + mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) + else: + # Simply call main_worker function + main_worker(args.gpu, ngpus_per_node, args) + + +def main_worker(gpu, ngpus_per_node, args): + global best_acc1 + args.gpu = gpu + + print("[npu id:", args.gpu, "]", "++++++++++++++++ before set KERNEL_NAME_ID:", os.environ['KERNEL_NAME_ID']) + os.environ['KERNEL_NAME_ID'] = str(gpu) + print("[npu id:", args.gpu, "]", "++++++++++++++++ KERNEL_NAME_ID:", os.environ['KERNEL_NAME_ID']) + + if args.gpu is not None: + print("[npu id:", args.gpu, "]", "Use GPU: {} for training".format(args.gpu)) + + if args.distributed: + if args.dist_url == "env://" and args.rank == -1: + args.rank = int(os.environ["RANK"]) + if args.multiprocessing_distributed: + # For multiprocessing distributed training, rank needs to be the + # global rank among all the processes + args.rank = args.rank * ngpus_per_node + gpu + + if args.device == 'npu': + dist.init_process_group(backend=args.dist_backend, # init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + else: + dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + + loc = 'npu:{}'.format(args.gpu) + torch.npu.set_device(loc) + + args.batch_size = int(args.batch_size / ngpus_per_node) + args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) + + print("[npu id:", args.gpu, "]", "===============main_worker()=================") + print("[npu id:", args.gpu, "]", args) + print("[npu id:", args.gpu, "]", "===============main_worker()=================") + + # Data loading code + # traindir = os.path.join(args.data, 'train') + # valdir = os.path.join(args.data, 'val') + # normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + # std=[0.229, 0.224, 0.225]) + + # train_dataset = datasets.ImageFolder( + # traindir, + # transforms.Compose([ + # transforms.RandomResizedCrop(224), + # transforms.RandomHorizontalFlip(), + # transforms.ToTensor(), + # normalize, + # ])) + # + # if args.distributed: + # train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + # else: + # train_sampler = None + # + # train_loader = torch.utils.data.DataLoader( + # train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), + # num_workers=args.workers, pin_memory=True, sampler=train_sampler, drop_last=True) + + train_loader, train_loader_len, train_sampler = get_pytorch_train_loader(args.data, + args.batch_size, + workers=args.workers, + distributed=args.distributed) + + # val_loader = torch.utils.data.DataLoader( + # datasets.ImageFolder(valdir, transforms.Compose([ + # transforms.Resize(256), + # transforms.CenterCrop(224), + # transforms.ToTensor(), + # normalize, + # ])), + # batch_size=args.batch_size, shuffle=True, + # num_workers=args.workers, pin_memory=True, drop_last=True) + + val_loader = get_pytorch_val_loader(args.data, args.batch_size, args.workers, distributed=False) + + # create model + print("[npu id:", args.gpu, "]", "=> creating model '{}'".format('mobilenetv2')) + # model = models.__dict__[args.arch]() + model = mobilenet_v2() + model = model.to(loc) + + # define loss function (criterion) and optimizer + criterion = nn.CrossEntropyLoss().to(loc) + optimizer = torch.optim.SGD(model.parameters(), args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay) + + if args.amp: + model, optimizer = amp.initialize(model, optimizer, opt_level=args.opt_level, loss_scale=args.loss_scale) + + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], broadcast_buffers=False) + + # optionally resume from a checkpoint + if args.resume: + if os.path.isfile(args.resume): + print("=> loading checkpoint '{}'".format(args.resume)) + checkpoint = torch.load(args.resume, map_location=loc) + args.start_epoch = checkpoint['epoch'] + best_acc1 = checkpoint['best_acc1'] + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + if args.amp: + amp.load_state_dict(checkpoint['amp']) + print("=> loaded checkpoint '{}' (epoch {})".format(args.resume, checkpoint['epoch'])) + else: + print("=> no checkpoint found at '{}'".format(args.resume)) + + cudnn.benchmark = True + + if args.evaluate: + validate(val_loader, model, criterion, args, ngpus_per_node) + return + + for epoch in range(args.start_epoch, args.epochs): + if args.distributed: + train_sampler.set_epoch(epoch) + # adjust_learning_rate(optimizer, epoch, args) + + # train for one epoch + train(train_loader, train_loader_len, model, criterion, optimizer, epoch, args, ngpus_per_node) + + if (epoch + 1) % args.eval_freq == 0 or epoch == args.epochs - 1: + # evaluate on validation set + acc1 = validate(val_loader, model, criterion, args, ngpus_per_node) + + # remember best acc@1 and save checkpoint + is_best = acc1 > best_acc1 + best_acc1 = max(acc1, best_acc1) + + if not args.multiprocessing_distributed or \ + (args.multiprocessing_distributed and args.rank % ngpus_per_node == 0 and epoch == args.epochs - 1): + if args.amp: + save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer': optimizer.state_dict(), + 'amp': amp.state_dict(), + }, is_best) + else: + save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer': optimizer.state_dict(), + }, is_best) + + +def train(train_loader, train_loader_len, model, criterion, optimizer, epoch, args, ngpus_per_node): + batch_time = AverageMeter('Time', ':6.3f') + data_time = AverageMeter('Data', ':6.3f') + losses = AverageMeter('Loss', ':.4e') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + train_loader_len, + [batch_time, data_time, losses, top1, top5], + prefix="Epoch: [{}]".format(epoch)) + + loc = 'npu:{}'.format(args.gpu) + + mean = torch.tensor([0.485 * 255, 0.456 * 255, 0.406 * 255]).view(1, 3, 1, 1) + std = torch.tensor([0.229 * 255, 0.224 * 255, 0.225 * 255]).view(1, 3, 1, 1) + mean = mean.to(loc, non_blocking=True) + std = std.to(loc, non_blocking=True) + + # switch to train mode + model.train() + end = time.time() + if args.benchmark == 1: + optimizer.zero_grad() + + # steps_per_epoch = len(train_loader) + steps_per_epoch = train_loader_len + print('==========step per epoch======================', steps_per_epoch) + for i, (images, target) in enumerate(train_loader): + # measure data loading time + data_time.update(time.time() - end) + + global_step = epoch * steps_per_epoch + i + lr = adjust_learning_rate(optimizer, global_step, steps_per_epoch, args) + + target = target.to(torch.int32) + images = images.to(loc, non_blocking=True).to(torch.float).sub(mean).div(std) + target = target.to(loc, non_blocking=True) + + # compute output + output = model(images) + # stream = torch.npu.current_stream() + # stream.synchronize() + + loss = criterion(output, target) + # stream = torch.npu.current_stream() + # stream.synchronize() + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # compute gradient and do SGD step + if args.benchmark == 0: + optimizer.zero_grad() + + if args.amp: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + + # stream = torch.npu.current_stream() + # stream.synchronize() + + if args.benchmark == 0: + optimizer.step() + elif args.benchmark == 1: + BATCH_SIZE_multiplier = int(OPTIMIZER_BATCH_SIZE / args.batch_size) + BM_optimizer_step = ((i + 1) % BATCH_SIZE_multiplier) == 0 + if BM_optimizer_step: + for param_group in optimizer.param_groups: + for param in param_group['params']: + param.grad /= BATCH_SIZE_multiplier + optimizer.step() + optimizer.zero_grad() + # stream = torch.npu.current_stream() + # stream.synchronize() + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + progress.display(i) + + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + print("[npu id:", args.gpu, "]", '* FPS@all {:.3f}'.format(ngpus_per_node * args.batch_size / batch_time.avg)) + hwlog.remark_print(key=hwlog.FPS, + value=' * FPS@all {:.3f}'.format(ngpus_per_node * args.batch_size / batch_time.avg)) + + +def validate(val_loader, model, criterion, args, ngpus_per_node): + batch_time = AverageMeter('Time', ':6.3f') + losses = AverageMeter('Loss', ':.4e') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(val_loader), + [batch_time, losses, top1, top5], + prefix='Test: ') + + # switch to evaluate mode + model.eval() + + with torch.no_grad(): + loc = 'npu:{}'.format(args.gpu) + mean = torch.tensor([0.485 * 255, 0.456 * 255, 0.406 * 255]).view(1, 3, 1, 1) + std = torch.tensor([0.229 * 255, 0.224 * 255, 0.225 * 255]).view(1, 3, 1, 1) + mean = mean.to(loc, non_blocking=True) + std = std.to(loc, non_blocking=True) + + end = time.time() + for i, (images, target) in enumerate(val_loader): + + target = target.to(torch.int32) + images = images.to(loc, non_blocking=True).to(torch.float).sub(mean).div(std) + target = target.to(loc, non_blocking=True) + + # compute output + output = model(images) + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + if not args.multiprocessing_distributed or \ + (args.multiprocessing_distributed and args.rank % ngpus_per_node == 0): + progress.display(i) + + # TODO: this should also be done with the ProgressMeter + if not args.multiprocessing_distributed or \ + (args.multiprocessing_distributed and args.rank % ngpus_per_node == 0): + print("[npu id:", args.gpu, "]", '[AVG-ACC] * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}' + .format(top1=top1, top5=top5)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value="{top1.avg:.3f}".format(top1=top1)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value="{top5.avg:.3f}".format(top5=top5)) + + return top1.avg + + +def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): + torch.save(state, filename) + if is_best: + shutil.copyfile(filename, 'model_best_acc%.4f_epoch%d.pth.tar' % (state['best_acc1'], state['epoch'])) + + +class AverageMeter(object): + """Computes and stores the average and current value""" + + def __init__(self, name, fmt=':f'): + self.name = name + self.fmt = fmt + self.reset() + self.start_count_index = 10 + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + if self.count == 0: + self.batchsize = n + + self.val = val + self.count += n + if self.count > (self.start_count_index * self.batchsize): + self.sum += val * n + self.avg = self.sum / (self.count - self.start_count_index * self.batchsize) + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + +class ProgressMeter(object): + def __init__(self, num_batches, meters, prefix=""): + self.batch_fmtstr = self._get_batch_fmtstr(num_batches) + self.meters = meters + self.prefix = prefix + + def display(self, batch): + entries = [self.prefix + self.batch_fmtstr.format(batch)] + entries += [str(meter) for meter in self.meters] + print("[npu id:", os.environ['KERNEL_NAME_ID'], "]", '\t'.join(entries)) + train_acc1 = str(entries).split("Acc@1")[1].strip().split(" ")[0] + train_acc5 = str(entries).split("Acc@5")[1].strip().split(" ")[0] + hwlog.remark_print(key=hwlog.TRAIN_ACCURACY_TOP1, value=train_acc1) + hwlog.remark_print(key=hwlog.TRAIN_ACCURACY_TOP5, value=train_acc5) + + def _get_batch_fmtstr(self, num_batches): + num_digits = len(str(num_batches // 1)) + fmt = '{:' + str(num_digits) + 'd}' + return '[' + fmt + '/' + fmt.format(num_batches) + ']' + + +# def adjust_learning_rate(optimizer, epoch, args): +# """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" +# lr = args.lr * (0.1 ** (epoch // 30)) +# for param_group in optimizer.param_groups: +# param_group['lr'] = lr + +def adjust_learning_rate(optimizer, global_step, steps_per_epoch, args): + """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" + # lr = args.lr * (0.98 ** (epoch / 2.5)) + lr = args.lr * (0.98 ** (global_step // int(steps_per_epoch * 2.5))) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + return lr + + +def accuracy(output, target, topk=(1,)): + """Computes the accuracy over the k top predictions for the specified values of k""" + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def fast_collate(batch): + imgs = [img[0] for img in batch] + targets = torch.tensor([target[1] for target in batch], dtype=torch.int64) + w = imgs[0].size[0] + h = imgs[0].size[1] + tensor = torch.zeros((len(imgs), 3, h, w), dtype=torch.uint8) + for i, img in enumerate(imgs): + nump_array = np.asarray(img, dtype=np.uint8) + if nump_array.ndim < 3: + nump_array = np.expand_dims(nump_array, axis=-1) + nump_array = np.rollaxis(nump_array, 2) + + tensor[i] += torch.from_numpy(nump_array) + + return tensor, targets + + +def get_pytorch_train_loader(data_path, batch_size, workers=5, _worker_init_fn=None, distributed=False): + traindir = os.path.join(data_path, 'train') + train_dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + ])) + + if distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + + dataloader_fn = MultiEpochsDataLoader # torch.utils.data.DataLoader + train_loader = dataloader_fn( + train_dataset, batch_size=batch_size, shuffle=(train_sampler is None), + num_workers=workers, worker_init_fn=_worker_init_fn, pin_memory=True, sampler=train_sampler, collate_fn=fast_collate, drop_last=True) + return train_loader, len(train_loader), train_sampler + + +def get_pytorch_val_loader(data_path, batch_size, workers=5, _worker_init_fn=None, distributed=False): + valdir = os.path.join(data_path, 'val') + val_dataset = datasets.ImageFolder( + valdir, transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + ])) + + if distributed: + val_sampler = torch.utils.data.distributed.DistributedSampler(val_dataset) + else: + val_sampler = None + + dataloader_fn = MultiEpochsDataLoader # torch.utils.data.DataLoader + val_loader = dataloader_fn( + val_dataset, + sampler=val_sampler, + batch_size=batch_size, shuffle=(val_sampler is None), + num_workers=workers, worker_init_fn=_worker_init_fn, pin_memory=True, collate_fn=fast_collate) + + return val_loader + + +if __name__ == '__main__': + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("pytorch") + config_info = get_model_parameter("pytorch_config") + initinal_data = {"base_lr": 0.1, "dataset": "imagenet", "optimizer": "SGD", "loss_scale": 1024} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/mobilenetv2_8p_main_anycard.py b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/mobilenetv2_8p_main_anycard.py new file mode 100644 index 0000000..05bc36d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/mobilenetv2_8p_main_anycard.py @@ -0,0 +1,663 @@ +# -*- coding: utf-8 -*- + +import argparse +import os +import random +import shutil +import time +import warnings + +import numpy as np + +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.distributed as dist +import torch.optim +import torch.multiprocessing as mp +import torch.utils.data +import torch.utils.data.distributed +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torchvision.models as models + +from mobilenet import mobilenet_v2 +from apex import amp +from multi_epochs_dataloader import MultiEpochsDataLoader + +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter +BATCH_SIZE = 6144 +OPTIMIZER_BATCH_SIZE = 6144 +# model_names = sorted(name for name in models.__dict__ +# if name.islower() and not name.startswith("__") +# and callable(models.__dict__[name])) + +parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') +parser.add_argument('--data', metavar='DIR', default='/opt/npu/dataset/imagenet', + help='path to dataset') +# parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet50', +# choices=model_names, +# help='model architecture: ' + +# ' | '.join(model_names) + +# ' (default: resnet18)') +parser.add_argument('-j', '--workers', default=32, type=int, metavar='N', + help='number of data loading workers (default: 4)') +parser.add_argument('--epochs', default=90, type=int, metavar='N', + help='number of total epochs to run') +parser.add_argument('--start-epoch', default=0, type=int, metavar='N', + help='manual epoch number (useful on restarts)') +parser.add_argument('-b', '--batch-size', default=BATCH_SIZE, type=int, + metavar='N', + help='mini-batch size (default: 256), this is the total ' + 'batch size of all GPUs on the current node when ' + 'using Data Parallel or Distributed Data Parallel') +parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, + metavar='LR', help='initial learning rate', dest='lr') +parser.add_argument('--momentum', default=0.9, type=float, metavar='M', + help='momentum') +parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, + metavar='W', help='weight decay (default: 1e-4)', + dest='weight_decay') +parser.add_argument('--workspace', type=str, default='./', metavar='DIR', + help='path to directory where checkpoints will be stored') +parser.add_argument('-p', '--print-freq', default=10, type=int, + metavar='N', help='print frequency (default: 10)') +parser.add_argument('-ef', '--eval-freq', default=5, type=int, + metavar='N', help='evaluate frequency (default: 5)') +parser.add_argument('--resume', default='', type=str, metavar='PATH', + help='path to latest checkpoint (default: none)') +parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', + help='evaluate model on validation set') +parser.add_argument('--pretrained', dest='pretrained', action='store_true', + help='use pre-trained model') +parser.add_argument('--world-size', default=-1, type=int, + help='number of nodes for distributed training') +parser.add_argument('--rank', default=-1, type=int, + help='node rank for distributed training') +parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, + help='url used to set up distributed training') +parser.add_argument('--dist-backend', default='nccl', type=str, + help='distributed backend') +parser.add_argument('--seed', default=None, type=int, + help='seed for initializing training. ') +parser.add_argument('--gpu', default=None, type=int, + help='GPU id to use.') +parser.add_argument('--multiprocessing-distributed', action='store_true', + help='Use multi-processing distributed training to launch ' + 'N processes per node, which has N GPUs. This is the ' + 'fastest way to use PyTorch for either single node or ' + 'multi node data parallel training') +parser.add_argument('-bm', '--benchmark', default=0, type=int, + metavar='N', help='set benchmark status (default: 1,run benchmark)') +parser.add_argument('--device', default='npu', type=str, help='npu or gpu') +parser.add_argument('--addr', default='10.136.181.115', type=str, help='master addr') +parser.add_argument('--checkpoint-nameprefix', default='checkpoint', type=str, help='checkpoint-nameprefix') +parser.add_argument('--checkpoint-freq', default=0, type=int, + metavar='N', help='checkpoint frequency (default: 0)' + '0: save only one file whitch per epoch;' + 'n: save diff file per n epoch' + '-1:no checkpoint,not support') + +parser.add_argument('--device-list', default='0,1,2,3,4,5,6,7', type=str, help='device id list') + +# apex +parser.add_argument('--amp', default=False, action='store_true', + help='use amp to train the model') +parser.add_argument('--loss-scale', default=64., type=float, + help='loss scale using in amp, default -1 means dynamic') +parser.add_argument('--opt-level', default='O2', type=str, + help='loss scale using in amp, default -1 means dynamic') + +warnings.filterwarnings('ignore') +best_acc1 = 0 + + +def device_id_to_process_device_map(device_list): + devices = device_list.split(",") + devices = [int(x) for x in devices] + devices.sort() + + process_device_map = dict() + for process_id, device_id in enumerate(devices): + process_device_map[process_id] = device_id + + return process_device_map + + +def main(): + args = parser.parse_args() + print("===============main()=================") + print(args) + print("===============main()=================") + + os.environ['KERNEL_NAME_ID'] = str(0) + print("++++++++++++++++++ KERNEL_NAME_ID:", os.environ['KERNEL_NAME_ID']) + + if args.seed is not None: + random.seed(args.seed) + torch.manual_seed(args.seed) + cudnn.deterministic = True + warnings.warn('You have chosen to seed training. ' + 'This will turn on the CUDNN deterministic setting, ' + 'which can slow down your training considerably! ' + 'You may see unexpected behavior when restarting ' + 'from checkpoints.') + + os.environ['MASTER_ADDR'] = args.addr # '10.136.181.51' + os.environ['MASTER_PORT'] = '59629' + + if args.gpu is not None: + warnings.warn('You have chosen a specific GPU. This will completely ' + 'disable data parallelism.') + + if args.dist_url == "env://" and args.world_size == -1: + args.world_size = int(os.environ["WORLD_SIZE"]) + + args.distributed = args.world_size > 1 or args.multiprocessing_distributed + + args.process_device_map = device_id_to_process_device_map(args.device_list) + + if args.device == 'npu': + # ngpus_per_node = torch.npu.device_count() + ngpus_per_node = len(args.process_device_map) + else: + ngpus_per_node = torch.cuda.device_count() + if args.multiprocessing_distributed: + # Since we have ngpus_per_node processes per node, the total world_size + # needs to be adjusted accordingly + args.world_size = ngpus_per_node * args.world_size + # Use torch.multiprocessing.spawn to launch distributed processes: the + # main_worker process function + # The child process uses the environment variables of the parent process, + # we have to set KERNEL_NAME_ID for every proc + if args.device == 'npu': + # main_worker(args.gpu, ngpus_per_node, args) + mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) + else: + mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) + else: + # Simply call main_worker function + main_worker(args.gpu, ngpus_per_node, args) + + +def main_worker(gpu, ngpus_per_node, args): + global best_acc1 + # args.gpu = gpu + args.gpu = args.process_device_map[gpu] + + print("[npu id:", args.gpu, "]", "++++++++++++++++ before set KERNEL_NAME_ID:", os.environ['KERNEL_NAME_ID']) + os.environ['KERNEL_NAME_ID'] = str(gpu) + print("[npu id:", args.gpu, "]", "++++++++++++++++ KERNEL_NAME_ID:", os.environ['KERNEL_NAME_ID']) + + if args.gpu is not None: + print("[npu id:", args.gpu, "]", "Use GPU: {} for training".format(args.gpu)) + + if args.distributed: + if args.dist_url == "env://" and args.rank == -1: + args.rank = int(os.environ["RANK"]) + if args.multiprocessing_distributed: + # For multiprocessing distributed training, rank needs to be the + # global rank among all the processes + args.rank = args.rank * ngpus_per_node + gpu + + if args.device == 'npu': + dist.init_process_group(backend=args.dist_backend, # init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + else: + dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + + loc = 'npu:{}'.format(args.gpu) + torch.npu.set_device(loc) + + args.batch_size = int(args.batch_size / ngpus_per_node) + args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) + + print("[npu id:", args.gpu, "]", "===============main_worker()=================") + print("[npu id:", args.gpu, "]", args) + print("[npu id:", args.gpu, "]", "===============main_worker()=================") + + # Data loading code + # traindir = os.path.join(args.data, 'train') + # valdir = os.path.join(args.data, 'val') + # normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + # std=[0.229, 0.224, 0.225]) + + # train_dataset = datasets.ImageFolder( + # traindir, + # transforms.Compose([ + # transforms.RandomResizedCrop(224), + # transforms.RandomHorizontalFlip(), + # transforms.ToTensor(), + # normalize, + # ])) + # + # if args.distributed: + # train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + # else: + # train_sampler = None + # + # train_loader = torch.utils.data.DataLoader( + # train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), + # num_workers=args.workers, pin_memory=True, sampler=train_sampler, drop_last=True) + + train_loader, train_loader_len, train_sampler = get_pytorch_train_loader(args.data, + args.batch_size, + workers=args.workers, + distributed=args.distributed) + + # val_loader = torch.utils.data.DataLoader( + # datasets.ImageFolder(valdir, transforms.Compose([ + # transforms.Resize(256), + # transforms.CenterCrop(224), + # transforms.ToTensor(), + # normalize, + # ])), + # batch_size=args.batch_size, shuffle=True, + # num_workers=args.workers, pin_memory=True, drop_last=True) + + val_loader = get_pytorch_val_loader(args.data, args.batch_size, args.workers, distributed=False) + + # create model + print("[npu id:", args.gpu, "]", "=> creating model '{}'".format('mobilenetv2')) + # model = models.__dict__[args.arch]() + model = mobilenet_v2() + model = model.to(loc) + + # define loss function (criterion) and optimizer + criterion = nn.CrossEntropyLoss().to(loc) + optimizer = torch.optim.SGD(model.parameters(), args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay) + + if args.amp: + model, optimizer = amp.initialize(model, optimizer, opt_level=args.opt_level, loss_scale=args.loss_scale) + + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], broadcast_buffers=False) + + # optionally resume from a checkpoint + if args.resume: + if os.path.isfile(args.resume): + print("=> loading checkpoint '{}'".format(args.resume)) + checkpoint = torch.load(args.resume, map_location=loc) + args.start_epoch = checkpoint['epoch'] + best_acc1 = checkpoint['best_acc1'] + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + if args.amp: + amp.load_state_dict(checkpoint['amp']) + print("=> loaded checkpoint '{}' (epoch {})".format(args.resume, checkpoint['epoch'])) + else: + print("=> no checkpoint found at '{}'".format(args.resume)) + + cudnn.benchmark = True + + if args.evaluate: + validate(val_loader, model, criterion, args, ngpus_per_node) + return + + for epoch in range(args.start_epoch, args.epochs): + if args.distributed: + train_sampler.set_epoch(epoch) + # adjust_learning_rate(optimizer, epoch, args) + + # train for one epoch + train(train_loader, train_loader_len, model, criterion, optimizer, epoch, args, ngpus_per_node) + + if (epoch + 1) % args.eval_freq == 0 or epoch == args.epochs - 1: + # evaluate on validation set + acc1 = validate(val_loader, model, criterion, args, ngpus_per_node) + + # remember best acc@1 and save checkpoint + is_best = acc1 > best_acc1 + best_acc1 = max(acc1, best_acc1) + + if not args.multiprocessing_distributed or \ + (args.multiprocessing_distributed and args.rank % ngpus_per_node == 0 and epoch == args.epochs - 1): + if args.amp: + save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer': optimizer.state_dict(), + 'amp': amp.state_dict(), + }, is_best) + else: + save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer': optimizer.state_dict(), + }, is_best) + + +def train(train_loader, train_loader_len, model, criterion, optimizer, epoch, args, ngpus_per_node): + batch_time = AverageMeter('Time', ':6.3f') + data_time = AverageMeter('Data', ':6.3f') + losses = AverageMeter('Loss', ':.4e') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + train_loader_len, + [batch_time, data_time, losses, top1, top5], + prefix="Epoch: [{}]".format(epoch)) + + loc = 'npu:{}'.format(args.gpu) + + mean = torch.tensor([0.485 * 255, 0.456 * 255, 0.406 * 255]).view(1, 3, 1, 1) + std = torch.tensor([0.229 * 255, 0.224 * 255, 0.225 * 255]).view(1, 3, 1, 1) + mean = mean.to(loc, non_blocking=True) + std = std.to(loc, non_blocking=True) + + # switch to train mode + model.train() + end = time.time() + if args.benchmark == 1: + optimizer.zero_grad() + + # steps_per_epoch = len(train_loader) + steps_per_epoch = train_loader_len + print('==========step per epoch======================', steps_per_epoch) + for i, (images, target) in enumerate(train_loader): + # measure data loading time + data_time.update(time.time() - end) + + global_step = epoch * steps_per_epoch + i + lr = adjust_learning_rate(optimizer, global_step, steps_per_epoch, args) + + target = target.to(torch.int32) + images = images.to(loc, non_blocking=True).to(torch.float).sub(mean).div(std) + target = target.to(loc, non_blocking=True) + + # compute output + output = model(images) + stream = torch.npu.current_stream() + stream.synchronize() + + loss = criterion(output, target) + stream = torch.npu.current_stream() + stream.synchronize() + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # compute gradient and do SGD step + if args.benchmark == 0: + optimizer.zero_grad() + + if args.amp: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + + stream = torch.npu.current_stream() + stream.synchronize() + + if args.benchmark == 0: + optimizer.step() + elif args.benchmark == 1: + BATCH_SIZE_multiplier = int(OPTIMIZER_BATCH_SIZE / args.batch_size) + BM_optimizer_step = ((i + 1) % BATCH_SIZE_multiplier) == 0 + if BM_optimizer_step: + for param_group in optimizer.param_groups: + for param in param_group['params']: + param.grad /= BATCH_SIZE_multiplier + optimizer.step() + optimizer.zero_grad() + stream = torch.npu.current_stream() + stream.synchronize() + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + progress.display(i) + + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + print("[npu id:", args.gpu, "]", '* FPS@all {:.3f}'.format(ngpus_per_node * args.batch_size / batch_time.avg)) + hwlog.remark_print(key=hwlog.FPS, value=' * FPS@all {:.3f}'.format(ngpus_per_node * args.batch_size / batch_time.avg)) + + +def validate(val_loader, model, criterion, args, ngpus_per_node): + batch_time = AverageMeter('Time', ':6.3f') + losses = AverageMeter('Loss', ':.4e') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(val_loader), + [batch_time, losses, top1, top5], + prefix='Test: ') + + # switch to evaluate mode + model.eval() + + with torch.no_grad(): + loc = 'npu:{}'.format(args.gpu) + mean = torch.tensor([0.485 * 255, 0.456 * 255, 0.406 * 255]).view(1, 3, 1, 1) + std = torch.tensor([0.229 * 255, 0.224 * 255, 0.225 * 255]).view(1, 3, 1, 1) + mean = mean.to(loc, non_blocking=True) + std = std.to(loc, non_blocking=True) + + end = time.time() + for i, (images, target) in enumerate(val_loader): + + target = target.to(torch.int32) + images = images.to(loc, non_blocking=True).to(torch.float).sub(mean).div(std) + target = target.to(loc, non_blocking=True) + + # compute output + output = model(images) + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + if not args.multiprocessing_distributed or \ + (args.multiprocessing_distributed and args.rank % ngpus_per_node == 0): + progress.display(i) + + # TODO: this should also be done with the ProgressMeter + if not args.multiprocessing_distributed or \ + (args.multiprocessing_distributed and args.rank % ngpus_per_node == 0): + print("[npu id:", args.gpu, "]", '[AVG-ACC] * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}' + .format(top1=top1, top5=top5)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value="{top1.avg:.3f}".format(top1=top1)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value="{top5.avg:.3f}".format(top5=top5)) + + return top1.avg + + +def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): + torch.save(state, filename) + if is_best: + shutil.copyfile(filename, 'model_best_acc%.4f_epoch%d.pth.tar' % (state['best_acc1'], state['epoch'])) + + +class AverageMeter(object): + """Computes and stores the average and current value""" + + def __init__(self, name, fmt=':f'): + self.name = name + self.fmt = fmt + self.reset() + self.start_count_index = 10 + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + if self.count == 0: + self.batchsize = n + + self.val = val + self.count += n + if self.count > (self.start_count_index * self.batchsize): + self.sum += val * n + self.avg = self.sum / (self.count - self.start_count_index * self.batchsize) + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + +class ProgressMeter(object): + def __init__(self, num_batches, meters, prefix=""): + self.batch_fmtstr = self._get_batch_fmtstr(num_batches) + self.meters = meters + self.prefix = prefix + + def display(self, batch): + entries = [self.prefix + self.batch_fmtstr.format(batch)] + entries += [str(meter) for meter in self.meters] + print("[npu id:", os.environ['KERNEL_NAME_ID'], "]", '\t'.join(entries)) + train_acc1 = str(entries).split("Acc@1")[1].strip().split(" ")[0] + train_acc5 = str(entries).split("Acc@5")[1].strip().split(" ")[0] + hwlog.remark_print(key=hwlog.TRAIN_ACCURACY_TOP1, value=train_acc1) + hwlog.remark_print(key=hwlog.TRAIN_ACCURACY_TOP5, value=train_acc5) + + + def _get_batch_fmtstr(self, num_batches): + num_digits = len(str(num_batches // 1)) + fmt = '{:' + str(num_digits) + 'd}' + return '[' + fmt + '/' + fmt.format(num_batches) + ']' + + +# def adjust_learning_rate(optimizer, epoch, args): +# """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" +# lr = args.lr * (0.1 ** (epoch // 30)) +# for param_group in optimizer.param_groups: +# param_group['lr'] = lr + +# def adjust_learning_rate(optimizer, global_step, steps_per_epoch, args): +# """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" +# # lr = args.lr * (0.98 ** (epoch / 2.5)) +# lr = args.lr * (0.98 ** (global_step // int(steps_per_epoch * 2.5))) +# for param_group in optimizer.param_groups: +# param_group['lr'] = lr +# return lr + +def adjust_learning_rate(optimizer, global_step, steps_per_epoch, args): + """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" + # lr = args.lr * (0.98 ** (epoch / 2.5)) + lr = args.lr * (0.98 ** (global_step // int(steps_per_epoch * 2.5))) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + return lr + + +def accuracy(output, target, topk=(1,)): + """Computes the accuracy over the k top predictions for the specified values of k""" + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def fast_collate(batch): + imgs = [img[0] for img in batch] + targets = torch.tensor([target[1] for target in batch], dtype=torch.int64) + w = imgs[0].size[0] + h = imgs[0].size[1] + tensor = torch.zeros((len(imgs), 3, h, w), dtype=torch.uint8) + for i, img in enumerate(imgs): + nump_array = np.asarray(img, dtype=np.uint8) + if nump_array.ndim < 3: + nump_array = np.expand_dims(nump_array, axis=-1) + nump_array = np.rollaxis(nump_array, 2) + + tensor[i] += torch.from_numpy(nump_array) + + return tensor, targets + + +def get_pytorch_train_loader(data_path, batch_size, workers=5, _worker_init_fn=None, distributed=False): + traindir = os.path.join(data_path, 'train') + train_dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + ])) + + if distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + + dataloader_fn = MultiEpochsDataLoader # torch.utils.data.DataLoader + train_loader = dataloader_fn( + train_dataset, batch_size=batch_size, shuffle=(train_sampler is None), + num_workers=workers, worker_init_fn=_worker_init_fn, pin_memory=True, sampler=train_sampler, collate_fn=fast_collate, drop_last=True) + return train_loader, len(train_loader), train_sampler + + +def get_pytorch_val_loader(data_path, batch_size, workers=5, _worker_init_fn=None, distributed=False): + valdir = os.path.join(data_path, 'val') + val_dataset = datasets.ImageFolder( + valdir, transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + ])) + + if distributed: + val_sampler = torch.utils.data.distributed.DistributedSampler(val_dataset) + else: + val_sampler = None + + dataloader_fn = MultiEpochsDataLoader # torch.utils.data.DataLoader + val_loader = dataloader_fn( + val_dataset, + sampler=val_sampler, + batch_size=batch_size, shuffle=(val_sampler is None), + num_workers=workers, worker_init_fn=_worker_init_fn, pin_memory=True, collate_fn=fast_collate) + + return val_loader + + +if __name__ == '__main__': + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("pytorch") + config_info = get_model_parameter("pytorch_config") + initinal_data = {"base_lr": 0.1, "dataset": "imagenet", "optimizer": "SGD", "loss_scale": 1024} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/multi_epochs_dataloader.py b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/multi_epochs_dataloader.py new file mode 100644 index 0000000..28b6679 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/multi_epochs_dataloader.py @@ -0,0 +1,31 @@ +import torch + + +class MultiEpochsDataLoader(torch.utils.data.DataLoader): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._DataLoader__initialized = False + self.batch_sampler = _RepeatSampler(self.batch_sampler) + self._DataLoader__initialized = True + self.iterator = super().__iter__() + + def __len__(self): + return len(self.batch_sampler.sampler) + + def __iter__(self): + for _ in range(len(self)): + yield next(self.iterator) + + +class _RepeatSampler(object): + """ Sampler that repeats forever. + Args: + sampler (Sampler) + """ + + def __init__(self, sampler): + self.sampler = sampler + + def __iter__(self): + while True: + yield from iter(self.sampler) diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/profiling.json b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/profiling.json new file mode 100644 index 0000000..d0d9b1f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/profiling.json @@ -0,0 +1,18 @@ +{ + "startCfg": + [ + { + "jobID": "123456789", + "deviceID": ["0"], + "features": + [ + { + "name": "task_trace" + }, + { + "name": "training_trace" + } + ] + } + ] +} diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/resume1p.sh b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/resume1p.sh new file mode 100644 index 0000000..4a69466 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/resume1p.sh @@ -0,0 +1,19 @@ +source set_env_b023.sh + +su HwHiAiUser -c "adc --host 0.0.0.0:22118 --log \"SetLogLevel(0)[error]\" --device 0" + +export SLOG_PRINT_TO_STDOUT=0 +export TASK_QUEUE_ENABLE=0 +nohup taskset -c 1-40 python3.7 densenet121_1p_main.py \ + --workers 40 \ + --arch densenet121 \ + --npu 0 \ + --lr 0.1 \ + --momentum 0.9 \ + --amp \ + --print-freq 1 \ + --eval-freq 5\ + --batch-size 256 \ + --epoch 45 \ + --resume checkpoint.pth.tar \ + --data /home/dataset/imagenet > output_1p.log & diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/resume8p.sh b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/resume8p.sh new file mode 100644 index 0000000..0fdb346 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/resume8p.sh @@ -0,0 +1,27 @@ +source set_env_b023.sh + +su HwHiAiUser -c "adc --host 0.0.0.0:22118 --log \"SetLogLevel(0)[error]\" --device 0" +su HwHiAiUser -c "adc --host 0.0.0.0:22118 --log \"SetLogLevel(0)[error]\" --device 4" + +export SLOG_PRINT_TO_STDOUT=0 +export TASK_QUEUE_ENABLE=0 +nohup python3.7 ./densenet121_8p_main.py \ + --addr='10.246.246.57' \ + --seed 49 \ + --workers 80 \ + --lr 0.8 \ + --print-freq 1 \ + --eval-freq 5\ + --arch densenet121 \ + --dist-url 'tcp://127.0.0.1:50000' \ + --dist-backend 'hccl' \ + --multiprocessing-distributed \ + --world-size 1 \ + --batch-size 2048 \ + --epochs 45 \ + --rank 0 \ + --amp \ + --benchmark 0 \ + --resume checkpoint.pth.tar \ + --data /train/imagenet > resume_8p.log & + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/run1p.sh b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/run1p.sh new file mode 100644 index 0000000..761499c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/run1p.sh @@ -0,0 +1,18 @@ +source set_env_b023.sh + +su HwHiAiUser -c "adc --host 0.0.0.0:22118 --log \"SetLogLevel(0)[error]\" --device 0" + +export SLOG_PRINT_TO_STDOUT=0 +export TASK_QUEUE_ENABLE=0 +nohup taskset -c 1-40 python3.7 densenet121_1p_main.py \ + --workers 40 \ + --arch densenet121 \ + --npu 0 \ + --lr 0.1 \ + --momentum 0.9 \ + --amp \ + --print-freq 1 \ + --eval-freq 5\ + --batch-size 256 \ + --epoch 90 \ + --data /opt/npu/dataset/imagenet > output_1p.log & diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/run8p.sh b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/run8p.sh new file mode 100644 index 0000000..4c39128 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/run8p.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +source set_env_b023.sh + +su HwHiAiUser -c "adc --host 0.0.0.0:22118 --log \"SetLogLevel(0)[error]\" --device 0" +su HwHiAiUser -c "adc --host 0.0.0.0:22118 --log \"SetLogLevel(0)[error]\" --device 4" + +export SLOG_PRINT_TO_STDOUT=0 +export TASK_QUEUE_ENABLE=0 +nohup python3.7 ./mobilenetv2_8p_main.py \ + --addr='10.246.246.76' \ + --seed 49 \ + --workers 80 \ + --lr 0.24 \ + --print-freq 1 \ + --eval-freq 5\ + --dist-url 'tcp://127.0.0.1:50002' \ + --dist-backend 'hccl' \ + --multiprocessing-distributed \ + --world-size 1 \ + --batch-size 6144 \ + --epochs 600 \ + --rank 0 \ + --amp \ + --benchmark 0 \ + --data /opt/npu/dataset/imagenet > output_8p.log & + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/set_env_b020.sh b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/set_env_b020.sh new file mode 100644 index 0000000..00614d4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/set_env_b020.sh @@ -0,0 +1,17 @@ +############## toolkit situation ################ +#export ASCEND_HOME=/usr/local/Ascend +#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/ +#export PYTHONPATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/hccl +#export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin +#export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + +############## nnae situation ################ +export ASCEND_HOME=/usr/local/Ascend +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/:/usr/local/python3.7.5/lib/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/ +export PYTHONPATH=/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/schedule_search.egg:/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/hccl +export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin +export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp/ + +# pip3.7 install --upgrade /usr/local/Ascend/nnae/latest/fwkacllib/lib64/topi-0.4.0-py3-none-any.whl +# pip3.7 install --upgrade /usr/local/Ascend/nnae/latest/fwkacllib/lib64/te-0.4.0-py3-none-any.whl + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/set_env_b023.sh b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/set_env_b023.sh new file mode 100644 index 0000000..536616b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/code/8p/set_env_b023.sh @@ -0,0 +1,18 @@ +############## toolkit situation ################ +export LD_LIBRARY_PATH=/usr/local/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH +export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/ascend-toolkit/latest/toolkit/tools/ide_daemon/bin/ +export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ +export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so +export PYTHONPATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH + + +############## nnae situation ################ +# export LD_LIBRARY_PATH=/usr/local/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH +# export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/nnae/latest/toolkit/tools/ide_daemon/bin/ +# export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp/ +# export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so +# export PYTHONPATH=/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH + + + +# ln -s /usr/local/Ascend/ascend-toolkit/latest/toolkit/bin/adc /usr/local/bin/ \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/config/npu_set_env.sh b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/config/npu_set_env.sh new file mode 100644 index 0000000..55d3e6d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/config/npu_set_env.sh @@ -0,0 +1,12 @@ +# main env +export LD_LIBRARY_PATH=/usr/local/lib/:/usr/lib/:/usr/local/Ascend/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH +export PATH=$PATH:/usr/local/Ascend/fwkacllib/ccec_compiler/bin +export ASCEND_OPP_PATH=/usr/local/Ascend/opp +export NEW_GE_FE_ID=1 +export GE_AICPU_FLAG=1 +export PYTHONPATH=/usr/local/Ascend/atc/python/site-packages/te.egg:/usr/local/Ascend/atc/python/site-packages/topi.egg:/usr/local/Ascend/atc/python/site-packages/auto_tune.egg:/usr/local/Ascend/atc/python/site-packages/schedule_search.egg:/usr/local +export CUSTOM_OP_LIB_PATH=/usr/local/Ascend/ops/framework/built-in/tensorflow +export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/fwkacllib/lib64/plugin/opskernel/libaicpu_plugin.so:/usr/local/Ascend/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so +export PLUGIN_LOAD_PATH=/usr/local/Ascend/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/fwkacllib/lib64/plugin/opskernel/libaicpu_plugin.so:/usr/local/Ascend/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so:/usr/local/Ascend/fwkacllib/lib64/plugin/opskernel/librts_engine.so +export SLOG_PRINT_TO_STDOUT=1 + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/config/set_env_b023.sh b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/config/set_env_b023.sh new file mode 100644 index 0000000..7618849 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/config/set_env_b023.sh @@ -0,0 +1,31 @@ +############## toolkit situation ################ +#export LD_LIBRARY_PATH=/usr/local/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH +#export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/ascend-toolkit/latest/toolkit/tools/ide_daemon/bin/ +#export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ +#export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so +#export PYTHONPATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH + + +############## nnae situation ################ + + +if [ -d /usr/local/Ascend/nnae/latest ];then + export LD_LIBRARY_PATH=/usr/local/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/aarch64_64-linux-gnu:$LD_LIBRARY_PATH + export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/nnae/latest/toolkit/tools/ide_daemon/bin/ + export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp/ + export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so + export PYTHONPATH=/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH +else + export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/aarch64-linux-gnu:$LD_LIBRARY_PATH + export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/ascend-toolkit/latest/toolkit/tools/ide_daemon/bin/ + export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so + export PYTHONPATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH +fi + +# ln -s /usr/local/Ascend/ascend-toolkit/latest/toolkit/bin/adc /usr/local/bin/ + +export SLOG_PRINT_TO_STDOUT=0 +#su HwHiAiUser -c "adc --host 0.0.0.0:22118 --log \"SetLogLevel(0)[error]\" --device 0" + +export TASK_QUEUE_ENABLE=1 \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/scripts/run.sh b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/scripts/run.sh new file mode 100644 index 0000000..fdb7438 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/scripts/run.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +rank_size=$1 +yamlPath=$2 +toolsPath=$3 +if [ -f /.dockerenv ];then + CLUSTER=$4 + MPIRUN_ALL_IP="$5" + export CLUSTER=${CLUSTER} +fi + +currentDir=$(cd "$(dirname "$0")/.."; pwd) + +source ${currentDir}/config/npu_set_env.sh + +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "pytorch_config") + + +rm -rf /var/log/npu/slog/host-0/* +currtime=`date +%Y%m%d%H%M%S` +mkdir -p ${currentDir%train*}/train/result/pt_mobilenet/training_job_${currtime}/ +train_job_dir=${currentDir%train*}/train/result/pt_mobilenet/training_job_${currtime}/ + +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir}" +# device 列表, 若无指定 device 或大于等于 8p 时根据 rank_size 顺序选择 +eval device_group=\$device_group_${rank_size}p +if [ x"${device_group}" == x"" ] || [ ${rank_size} -ge 8 ];then + device_group="$(seq 0 "$(expr $rank_size - 1)")" +fi + +device_group_str=`echo ${device_group} | sed 's/ //g'` +first_device_id=`echo ${device_group_str: 0:1}` + +rank_id=0 + +if [ x"${CLUSTER}" == x"True" ];then + ln -snf ${currentDir%train*}/train/result/pt_mobilenet/training_job_${currtime}/0/hw_mobilenet.log ${currentDir%train*}/train/result/pt_mobilenet/training_job_${currtime}/ + this_ip=$(hostname -I |awk '{print $1}') + for ip in $MPIRUN_ALL_IP;do + if [ x"$this_ip" != x"$ip" ];then + scp $yamlPath root@$ip:$yamlPath + scp $jsonFilePath root@$ip:$jsonFilePath + fi + done + export PATH=$PATH:/usr/local/mpirun4.0.2/bin + mpirun -H ${mpirun_ip} \ + --bind-to none -map-by slot\ + --allow-run-as-root \ + --mca btl_tcp_if_exclude lo,docker0,endvnic,virbr0,vethf40501b,docker_gwbridge,br-f42ac38052b4\ + --prefix /usr/local/mpirun4.0.2/ \ + ${currentDir}/scripts/train.sh 0 $rank_size $yamlPath $currtime ${toolsPath} ${CLUSTER} +else + ln -snf ${currentDir%train*}/train/result/pt_mobilenet/training_job_${currtime}/${first_device_id}/hw_mobilenet.log ${currentDir%train*}/train/result/pt_mobilenet/training_job_${currtime}/ + #for device_id in $device_group;do + #echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] start: train ${device_id} & " >> ${currentDir}/result/main.log + ${currentDir}/scripts/train.sh 0 $rank_size $yamlPath $currtime ${toolsPath} $rank_id& + #let rank_id++ + #done +fi +wait diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/scripts/train.sh b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/scripts/train.sh new file mode 100644 index 0000000..7d82a0d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/pytorch/scripts/train.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash + +device_id=$1 +rank_size=$2 +yamlPath=$3 +currentDir=$(cd "$(dirname "$0")/.."; pwd) +currtime=$4 +toolsPath=$5 + +export YAML_PATH=$3 +mkdir -p ${currentDir%train*}/train/result/pt_mobilenet/training_job_${currtime}/ +export train_job_dir=${currentDir%train*}/train/result/pt_mobilenet/training_job_${currtime}/ + + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "pytorch_config") +export REMARK_LOG_FILE=hw_mobilenet.log +benchmark_log_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils +export PYTHONPATH=$PYTHONPATH:${benchmark_log_path} + + +source ${currentDir}/config/set_env_b023.sh + +# user env +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 +export RANK_TABLE_FILE=${currentDir}/config/${rank_size}p.json +export RANK_SIZE=${rank_size} +export SLOG_PRINT_TO_STDOUT=0 +export DEVICE_ID=${device_id} +DEVICE_INDEX=$(( DEVICE_ID + RANK_INDEX * 8 )) +export DEVICE_INDEX=${DEVICE_INDEX} + +cd ${train_job_dir} +curd_dir=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +export PYTHONPATH=$PYTHONPATH:${curd_dir} + +if [ x"$6" != x"True" ];then + rank_id=$6 + export RANK_ID=$6 +else + device_id_mo=$(python3.7 -c "import src.tensorflow.mpi_ops as atlasboost;atlasboost.init(); \ + device_id = atlasboost.local_rank();cluster_device_id = str(device_id); \ + atlasboost.set_device_id(device_id);print(atlasboost.rank())") + device_id_mo=`echo $device_id_mo` + rank_id=${device_id_mo##* } + echo rank_id is $rank_id + export RANK_ID=${rank_id} + device=${device_id_mo##*deviceid = } + device_id=${device%% phyid=*} + export DEVICE_ID=${device_id} + echo device_id is $device_id + hccljson=${train_job_dir}/*.json + cp ${hccljson} ${currentDir}/config/${rank_size}p.json +fi + +#mkdir exec path +mkdir -p ${train_job_dir}/${device_id} +cd ${train_job_dir}/${device_id} + +startTime=`date +%Y%m%d-%H:%M:%S` +startTime_s=`date +%s` + + +if [ x"$6" == x"True" ];then + python3.7 ${currentDir}/code/8p/mobilenetv2_8p_main.py \ + --addr=$(hostname -I |awk '{print $1}') \ + --seed 49 \ + --workers 128 \ + --lr 0.24 \ + --print-freq 1 \ + --eval-freq 5\ + --dist-url 'tcp://127.0.0.1:50002' \ + --dist-backend 'hccl' \ + --multiprocessing-distributed \ + --world-size 1 \ + --batch-size ${batch_size} \ + --epochs ${epoches} \ + --rank 0 \ + --amp \ + --benchmark 0 \ + --data ${data_url} > ${train_job_dir}/train_${rank_size}p.log 2>&1 +elif [ x"${rank_size}" == x"1" ];then + # 单卡 + python3.7 ${currentDir}/code/1p/main_apex.py \ + --workers 128 \ + --seed 123456 \ + --lr 0.03 \ + --amp \ + --opt-level 'O2' \ + --loss-scale-value 64 \ + --momentum 0.9 \ + --batch-size ${batch_size} \ + --weight-decay 1e-5 \ + --epoch ${epoches} \ + --print-freq 1 \ + --device ${device_single}\ + --eval-freq 1 \ + --summary-path './runs/mobilenetv2/npu_O2_ls64_c75b150_0909' \ + --data ${data_url} > ${train_job_dir}/train_${rank_size}p.log 2>&1 +elif [ ${rank_size} -le 8 ];then + # 多卡单机 + python3.7 ${currentDir}/code/8p/mobilenetv2_8p_main_anycard.py \ + --addr=$(hostname -I |awk '{print $1}') \ + --seed 49 \ + --workers 128 \ + --lr ${lr} \ + --print-freq 1 \ + --loss-scale 64 \ + --eval-freq 1\ + --dist-url 'tcp://127.0.0.1:50002' \ + --dist-backend 'hccl' \ + --multiprocessing-distributed \ + --world-size 1 \ + --batch-size ${batch_size} \ + --epochs ${epoches} \ + --rank 0 \ + --amp \ + --device-list ${device_group_mutli} \ + --benchmark 0 \ + --data ${data_url} > ${train_job_dir}/train_${rank_size}p.log 2>&1 +fi + + + +if [ $? -eq 0 ];then + echo ":::ABK 1.0.0 hw_mobilenet train success" + echo ":::ABK 1.0.0 hw_mobilenet train success" >> ${train_job_dir}/train_${rank_size}p.log + echo ":::ABK 1.0.0 hw_mobilenet train success" >> ./hw_mobilenet.log +else + echo ":::ABK 1.0.0 hw_mobilenet train failed" + echo ":::ABK 1.0.0 hw_mobilenet train failed" >> ${train_job_dir}/train_${rank_size}p.log + echo ":::ABK 1.0.0 hw_mobilenet train failed" >> ./hw_mobilenet.log +fi + +endTime=`date +%Y%m%d-%H:%M:%S` +endTime_s=`date +%s` + +sumTime=$[ $endTime_s - $startTime_s ] + +hour=$(( $sumTime/3600 )) +min=$(( ($sumTime-${hour}*3600)/60 )) +sec=$(( $sumTime-${hour}*3600-${min}*60 )) +echo ":::ABK 1.0.0 mobilenet train total time：${hour}:${min}:${sec}" + +echo ":::ABK 1.0.0 mobilenet train total time： ${hour}:${min}:${sec}" >> ./hw_mobilenet.log diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/README.md b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/README.md new file mode 100644 index 0000000..e75d8aa --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/README.md @@ -0,0 +1,47 @@ +# MobileNet_tensorflow训练说明 + +### 1. 模型训练参数配置 + +在train/yaml/MobileNet.yaml中修改相应配置， 配置项含义: + +``` +tensorflow_config: + # 基本参数 + max_steps: 1000 + data_url: 数据集路径 + epoches: 跑多少个epoch + + # 训练(train) 或 评测(evaluate) + mode: train + batch_size: 256 + #仅在 mode 为 evaluate 时用到 + ckpt_path: /opt/0908/benchmark-benchmark_Alpha/train/result/tf_mobilenet/trainingJob_20200905171017/0/results/model.ckpt-123125 + + # 仅多机执行需要配置: ip1:卡数量1,ip2:卡数量2 + mpirun_ip: 90.90.176.152:8,90.90.176.154:8 + + # docker 镜像名称:版本号 + docker_image: c73:b021 + + # 指定 device id, 多个 id 使用空格分隔, 数量需与 rank_size 相同 + device_group_1p: 0 + device_group_2p: 0 1 + device_group_4p: 0 1 2 3 + + profiling_mode: false + profiling_options: training_trace + fp_point: L2Loss + bp_point: gradients/AddN_30 + aicpu_profiling_mode: false + +``` + +------ + + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/__init__.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/README.md b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/README.md new file mode 100644 index 0000000..743826e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/README.md @@ -0,0 +1,211 @@ +# MobileNetv2 for Tensorflow + +This repository provides a script and recipe to train the MobileNetv2 model to achieve state-of-the-art accuracy. + +## Table Of Contents + +* [Model overview](#model-overview) + * [Model Architecture](#model-architecture) + * [Default configuration](#default-configuration) +* [Data augmentation](#data-augmentation) +* [Setup](#setup) + * [Requirements](#requirements) +* [Quick start guide](#quick-start-guide) +* [Advanced](#advanced) + * [Command line arguments](#command-line-arguments) + * [Training process](#training-process) +* [Performance](#performance) + * [Results](#results) + * [Training accuracy results](#training-accuracy-results) + * [Training performance results](#training-performance-results) + + + + +## Model overview + +In this repository, we implement MobileNetv2 from paper [Sandler, Mark, et al. "Mobilenetv2: Inverted residuals and linear bottlenecks." CVPR 2018.](https://arxiv.org/abs/1801.04381) + +MobileNetv2 is a mobile architecture. It is mainly constructed based on depthwise separable convolutions, linear bottlenecks and inverted residuals. + +### Model architecture + +The model architecture can be found from the reference paper. + +### Default configuration + +The following sections introduce the default configurations and hyperparameters for MobileNetv2 model. + +#### Optimizer + +This model uses Momentum optimizer from Tensorflow with the following hyperparameters: + +- Momentum : 0.9 +- Learning rate (LR) : 0.8 +- LR schedule: cosine_annealing +- Warmup epoch: 5 +- Batch size : 256*8 +- Weight decay : 0.00004 +- Moving average decay: 0.9999 +- Label smoothing = 0.1 +- We train for: + - 300 epochs for a standard training process using ImageNet2012 + +#### Data augmentation + +This model uses the data augmentation from InceptionV2: + +- For training: + - Convert DataType and RandomResizeCrop + - RandomHorizontalFlip, prob=0.5 + - Subtract with 0.5 and multiply with 2.0 +- For inference: + - Convert DataType + - CenterCrop 87.5% of the original image and resize to (224, 224) + - Subtract with 0.5 and multiply with 2.0 + +For more details, we refer readers to read the corresponding source code in Slim. + +## Setup +The following section lists the requirements to start training the MobileNetv2 model. +### Requirements + +Tensorflow 1.15.0 + +## Quick Start Guide + +### 1. Clone the respository + +```shell +git clone xxx +cd ModelZoo_MobileNetv2_TF +``` + +### 2. Download and preprocess the dataset + +1. Download the ImageNet2012 dataset +2. Generate tfrecord files following [Tensorflow-Slim](https://github.com/tensorflow/models/tree/master/research/slim). +3. The train and validation tfrecord files are under the path/data directories. + +### 3. Train +- train on a single NPU + - **edit** *train_1p.sh* (see example below) + - bash run_1p.sh +- train on 8 NPUs + - **edit** *train_8p.sh* (see example below) + - bash run_8p.sh + +Examples: +- Case for single NPU + - In *train_1p.sh*, python scripts part should look like as follows. For more detailed command lines arguments, please refer to [Command line arguments](#command-line-arguments) + ```shell + python3.7 ${currentDir}/train.py \ + --dataset_dir=/opt/npu/slimImagenet \ + --max_train_steps=500 \ + --iterations_per_loop=50 \ + --model_name="mobilenet_v2" \ + --moving_average_decay=0.9999 \ + --label_smoothing=0.1 \ + --preprocessing_name="inception_v2" \ + --weight_decay='0.00004' \ + --batch_size=256 \ + --learning_rate_decay_type='cosine_annealing' \ + --learning_rate=0.4 \ + --optimizer='momentum' \ + --momentum='0.9' \ + --warmup_epochs=5 + ``` + - Run the program + ``` + bash run_1p.sh + ``` +- Case for 8 NPUs + - In *train_8p.sh*, python scripts part should look like as follows. + ```shell + python3.7 ${currentDir}/train.py \ + --dataset_dir=/opt/npu/slimImagenet \ + --max_epoch=300 \ + --model_name="mobilenet_v2" \ + --moving_average_decay=0.9999 \ + --label_smoothing=0.1 \ + --preprocessing_name="inception_v2" \ + --weight_decay='0.00004' \ + --batch_size=256 \ + --learning_rate_decay_type='cosine_annealing' \ + --learning_rate=0.8 \ + --optimizer='momentum' \ + --momentum='0.9' \ + --warmup_epochs=5 + ``` + - Run the program + ``` + bash run_8p.sh + ``` + +### 4. Test +- We evaluate results by using following commands: + ```shell + python3.7 eval_image_classifier_mobilenet.py --dataset_dir=/opt/npu/slimImagenet \ + --checkpoint_path=result/8p/0/results/model.ckpt-187500 + ``` + Remember to modify the dataset path and checkpoint path, then run the command. + + +## Advanced +### Commmand-line options + +We list those important parameters to train this network here. For more details of all the parameters, please read *train.py* and other related files. + +``` + --dataset_dir directory of dataset (default: /opt/npu/models/slimImagenet) + --max_epoch number of epochs to train the model (default: 200) + --max_train_steps max number of training steps (default: 500) + --iterations_per_loop number of steps to run in devices each iteration (default: None) + --model_name name of the model to train (default: mobilenet_v2_140) + --moving_average_decay the decay to use for the moving average (default: None) + --label_smoothing use label smooth in cross entropy (default: 0.1) + --preprocessing_name preprocessing method for training (default: inception_v2) + --weight_decay weight decay for regularization loss (default: 0) + --batch_size batch size per npu (default: 96) + --learning_rate_decay_type learning rate decay type (default: fixed) + --learning_rate initial learning rate (default: 0.1) + --optimizer the name of optimizer (default: sgd) + --momentum momentum value used in optimizer (default: 0.9) + --warmup_epochs warmup epochs for learning rate (default: 5) +``` + +### Training process + +All the results of the training will be stored in the directory `result`. + +## Performance + +### Result + +Our result were obtained by running the applicable training script. To achieve the same results, follow the steps in the Quick Start Guide. + +#### Training accuracy results + +| **epochs** | Top1 | +| :--------: | :------------: | +| 300 | 72.47% | + +#### Training performance results +| **NPUs** | train performance | +| :------: | :---------------: | +| 1 | 1400 img/s | + +| **NPUs** | train performance | +| :------: | :---------------: | +| 8 | 11000 img/s | + + + + + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/__init__.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/dataloader/data_provider.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/dataloader/data_provider.py new file mode 100644 index 0000000..c0e0c99 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/dataloader/data_provider.py @@ -0,0 +1,240 @@ +# Copyright 2017 The TensorFlow Authors All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Functions to read, decode and pre-process input data for the Model. +""" +import collections +import sys +import tensorflow as tf + +from tensorflow.python.data.experimental.ops import threadpool + +# from tensorflow.contrib import slim + +InputEndpoints = collections.namedtuple( + 'InputEndpoints', ['images', 'images_orig', 'labels', 'labels_one_hot']) +ShuffleBatchConfig = collections.namedtuple('ShuffleBatchConfig', [ + 'num_batching_threads', 'queue_capacity', 'min_after_dequeue' +]) + +DEFAULT_SHUFFLE_CONFIG = ShuffleBatchConfig( + num_batching_threads=8, queue_capacity=3000, min_after_dequeue=1000) + + +def get_data_files(data_sources): + from tensorflow.python.platform import gfile + if isinstance(data_sources, (list, tuple)): + data_files = [] + for source in data_sources: + data_files += get_data_files(source) + else: + if '*' in data_sources or '?' in data_sources or '[' in data_sources: + data_files = gfile.Glob(data_sources) + else: + data_files = [data_sources] + if not data_files: + raise ValueError('No data files found in %s' % (data_sources,)) + return data_files + + +def preprocess_image(image, location, label_one_hot, height=224, width=224): + """Prepare one image for evaluation. + If height and width are specified it would output an image with that size by + applying resize_bilinear. + If central_fraction is specified it would cropt the central fraction of the + input image. + Args: + image: 3-D Tensor of image. If dtype is tf.float32 then the range should be + [0, 1], otherwise it would converted to tf.float32 assuming that the range + is [0, MAX], where MAX is largest positive representable number for + int(8/16/32) data type (see `tf.image.convert_image_dtype` for details) + height: integer + width: integer + central_fraction: Optional Float, fraction of the image to crop. + scope: Optional scope for name_scope. + Returns: + 3-D float Tensor of prepared image. + """ + + # if image.dtype != tf.float32: + image = tf.image.convert_image_dtype(image, dtype=tf.float32) + # Crop the central region of the image with an area containing 87.5% of + # the original image. + # if central_fraction: + # image = tf.image.central_crop(image, central_fraction=central_fraction) + + # if height and width: + # Resize the image to the specified height and width. + image = tf.expand_dims(image, 0) + image = tf.image.resize_bilinear(image, [height, width], align_corners=False) + image = tf.squeeze(image, [0]) + + # image = tf.cast(image, tf.float32) + # image = tf.multiply(image, 1/255.) + image = tf.subtract(image, 0.5) + image = tf.multiply(image, 2.0) + + return image, location, label_one_hot + + +def _int64_feature(value): + """Wrapper for inserting int64 features into Example proto.""" + if not isinstance(value, list): + value = [value] + return tf.train.Feature(int64_list=tf.train.Int64List(value=value)) + + +def parse_example_proto(example_serialized, num_classes, labels_offset, image_preprocessing_fn): + feature_map = { + 'image/encoded': tf.FixedLenFeature([], tf.string, ''), + 'image/class/label': tf.FixedLenFeature([1], tf.int64, -1), + 'image/class/text': tf.FixedLenFeature([], tf.string, ''), + 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32) + } + with tf.compat.v1.name_scope('deserialize_image_record'): + obj = tf.io.parse_single_example(serialized=example_serialized, features=feature_map) + image = tf.image.decode_jpeg(obj['image/encoded'], channels=3, fancy_upscaling=False, + dct_method='INTEGER_FAST') + if image_preprocessing_fn: + image = image_preprocessing_fn(image, 224, 224) + else: + image = tf.image.resize(image, [224, 224]) + + label = tf.cast(obj['image/class/label'], tf.int32) + label = tf.squeeze(label) + label -= labels_offset + label = tf.one_hot(label, num_classes - labels_offset) + return image, label + + +def parse_example_decode(example_serialized): + feature_map = { + 'image/encoded': tf.FixedLenFeature([], tf.string, ''), + 'image/class/label': tf.FixedLenFeature([1], tf.int64, -1), + 'image/class/text': tf.FixedLenFeature([], tf.string, ''), + 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32) + } + with tf.compat.v1.name_scope('deserialize_image_record'): + obj = tf.io.parse_single_example(serialized=example_serialized, features=feature_map) + image = tf.image.decode_jpeg(obj['image/encoded'], channels=3, fancy_upscaling=False, + dct_method='INTEGER_FAST') + + return image, obj['image/class/label'] + + +def parse_example(image, label, num_classes, labels_offset, image_preprocessing_fn): + with tf.compat.v1.name_scope('deserialize_image_record'): + if image_preprocessing_fn: + image = image_preprocessing_fn(image, 224, 224) + else: + image = tf.image.resize(image, [224, 224]) + + label = tf.cast(label, tf.int32) + label = tf.squeeze(label) + label -= labels_offset + label = tf.one_hot(label, num_classes - labels_offset) + return image, label + + +def parse_example1(example_serialized, image_preprocessing_fn1): + feature_map = { + 'image/encoded': tf.FixedLenFeature([], tf.string, ''), + 'image/class/label': tf.FixedLenFeature([1], tf.int64, -1), + 'image/class/text': tf.FixedLenFeature([], tf.string, ''), + 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32) + } + with tf.compat.v1.name_scope('deserialize_image_record'): + obj = tf.io.parse_single_example(serialized=example_serialized, features=feature_map) + image = tf.image.decode_jpeg(obj['image/encoded'], channels=3, fancy_upscaling=False, + dct_method='INTEGER_FAST') + + image = image_preprocessing_fn1(image, 224, 224) + return image, obj['image/class/label'] + + +def parse_example2(image, label, num_classes, labels_offset, image_preprocessing_fn2): + with tf.compat.v1.name_scope('deserialize_image_record'): + image = image_preprocessing_fn2(image, 224, 224) + + label = tf.cast(label, tf.int32) + label = tf.squeeze(label) + label -= labels_offset + label = tf.one_hot(label, num_classes - labels_offset) + return image, label + + +def get_data(dataset, batch_size, num_classes, labels_offset, is_training, + preprocessing_name=None, use_grayscale=None, add_image_summaries=False): + return get_data_united(dataset, batch_size, num_classes, labels_offset, is_training, + preprocessing_name, use_grayscale, add_image_summaries) + + +def create_ds(data_sources, is_training): + data_files = get_data_files(data_sources) + ds = tf.data.Dataset.from_tensor_slices(data_files) + + if is_training: + ds = ds.shuffle(1000) + # add for eval + else: + ds = ds.take(50000) + + ##### change ##### + num_readers = 10 + ds = ds.interleave( + tf.data.TFRecordDataset, cycle_length=num_readers, block_length=1, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + counter = tf.data.Dataset.range(sys.maxsize) + ds = tf.data.Dataset.zip((ds, counter)) + ##### change ##### + + if is_training: + ds = ds.repeat() + + return ds + + +def get_data_united(dataset, batch_size, num_classes, labels_offset, is_training, + preprocessing_name=None, use_grayscale=None, add_image_summaries=False): + from preprocessing import preprocessing_factory + image_preprocessing_fn = preprocessing_factory.get_preprocessing( + name='inception_v2', + is_training=is_training, + use_grayscale=use_grayscale, + add_image_summaries=add_image_summaries + ) + + ds = create_ds(dataset.data_sources, is_training) + + ds = ds.map(lambda example, counter: parse_example_proto(example, num_classes, labels_offset, image_preprocessing_fn), num_parallel_calls=24) + + ds = ds.batch(batch_size, drop_remainder=True) + + ds = ds.prefetch(buffer_size=tf.contrib.data.AUTOTUNE) + + iterator = ds.make_initializable_iterator() + + ds = threadpool.override_threadpool(ds,threadpool.PrivateThreadPool(128, display_name='input_pipeline_thread_pool')) + + return iterator, ds diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/__init__.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/__init__.py @@ -0,0 +1 @@ + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/build_imagenet_data.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/build_imagenet_data.py new file mode 100644 index 0000000..572bcd3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/build_imagenet_data.py @@ -0,0 +1,705 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Converts ImageNet data to TFRecords file format with Example protos. + +The raw ImageNet data set is expected to reside in JPEG files located in the +following directory structure. + + data_dir/n01440764/ILSVRC2012_val_00000293.JPEG + data_dir/n01440764/ILSVRC2012_val_00000543.JPEG + ... + +where 'n01440764' is the unique synset label associated with +these images. + +The training data set consists of 1000 sub-directories (i.e. labels) +each containing 1200 JPEG images for a total of 1.2M JPEG images. + +The evaluation data set consists of 1000 sub-directories (i.e. labels) +each containing 50 JPEG images for a total of 50K JPEG images. + +This TensorFlow script converts the training and evaluation data into +a sharded data set consisting of 1024 and 128 TFRecord files, respectively. + + train_directory/train-00000-of-01024 + train_directory/train-00001-of-01024 + ... + train_directory/train-00127-of-01024 + +and + + validation_directory/validation-00000-of-00128 + validation_directory/validation-00001-of-00128 + ... + validation_directory/validation-00127-of-00128 + +Each validation TFRecord file contains ~390 records. Each training TFREcord +file contains ~1250 records. Each record within the TFRecord file is a +serialized Example proto. The Example proto contains the following fields: + + image/encoded: string containing JPEG encoded image in RGB colorspace + image/height: integer, image height in pixels + image/width: integer, image width in pixels + image/colorspace: string, specifying the colorspace, always 'RGB' + image/channels: integer, specifying the number of channels, always 3 + image/format: string, specifying the format, always'JPEG' + + image/filename: string containing the basename of the image file + e.g. 'n01440764_10026.JPEG' or 'ILSVRC2012_val_00000293.JPEG' + image/class/label: integer specifying the index in a classification layer. + The label ranges from [1, 1000] where 0 is not used. + image/class/synset: string specifying the unique ID of the label, + e.g. 'n01440764' + image/class/text: string specifying the human-readable version of the label + e.g. 'red fox, Vulpes vulpes' + + image/object/bbox/xmin: list of integers specifying the 0+ human annotated + bounding boxes + image/object/bbox/xmax: list of integers specifying the 0+ human annotated + bounding boxes + image/object/bbox/ymin: list of integers specifying the 0+ human annotated + bounding boxes + image/object/bbox/ymax: list of integers specifying the 0+ human annotated + bounding boxes + image/object/bbox/label: integer specifying the index in a classification + layer. The label ranges from [1, 1000] where 0 is not used. Note this is + always identical to the image label. + +Note that the length of xmin is identical to the length of xmax, ymin and ymax +for each example. + +Running this script using 16 threads may take around ~2.5 hours on a HP Z420. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from datetime import datetime +import os +import random +import sys +import threading + +import numpy as np +from six.moves import xrange # pylint: disable=redefined-builtin +import tensorflow as tf + + +tf.app.flags.DEFINE_string('train_directory', '/tmp/', + 'Training data directory') +tf.app.flags.DEFINE_string('validation_directory', '/tmp/', + 'Validation data directory') +tf.app.flags.DEFINE_string('output_directory', '/tmp/', + 'Output data directory') + +tf.app.flags.DEFINE_integer('train_shards', 1024, + 'Number of shards in training TFRecord files.') +tf.app.flags.DEFINE_integer('validation_shards', 128, + 'Number of shards in validation TFRecord files.') + +tf.app.flags.DEFINE_integer('num_threads', 8, + 'Number of threads to preprocess the images.') + +# The labels file contains a list of valid labels are held in this file. +# Assumes that the file contains entries as such: +# n01440764 +# n01443537 +# n01484850 +# where each line corresponds to a label expressed as a synset. We map +# each synset contained in the file to an integer (based on the alphabetical +# ordering). See below for details. +tf.app.flags.DEFINE_string('labels_file', + 'imagenet_lsvrc_2015_synsets.txt', + 'Labels file') + +# This file containing mapping from synset to human-readable label. +# Assumes each line of the file looks like: +# +# n02119247 black fox +# n02119359 silver fox +# n02119477 red fox, Vulpes fulva +# +# where each line corresponds to a unique mapping. Note that each line is +# formatted as \t. +tf.app.flags.DEFINE_string('imagenet_metadata_file', + 'imagenet_metadata.txt', + 'ImageNet metadata file') + +# This file is the output of process_bounding_box.py +# Assumes each line of the file looks like: +# +# n00007846_64193.JPEG,0.0060,0.2620,0.7545,0.9940 +# +# where each line corresponds to one bounding box annotation associated +# with an image. Each line can be parsed as: +# +# , , , , +# +# Note that there might exist mulitple bounding box annotations associated +# with an image file. +tf.app.flags.DEFINE_string('bounding_box_file', + './imagenet_2012_bounding_boxes.csv', + 'Bounding box file') + +FLAGS = tf.app.flags.FLAGS + + +def _int64_feature(value): + """Wrapper for inserting int64 features into Example proto.""" + if not isinstance(value, list): + value = [value] + return tf.train.Feature(int64_list=tf.train.Int64List(value=value)) + + +def _float_feature(value): + """Wrapper for inserting float features into Example proto.""" + if not isinstance(value, list): + value = [value] + return tf.train.Feature(float_list=tf.train.FloatList(value=value)) + + +def _bytes_feature(value): + """Wrapper for inserting bytes features into Example proto.""" + return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) + + +def _convert_to_example(filename, image_buffer, label, synset, human, bbox, + height, width): + """Build an Example proto for an example. + + Args: + filename: string, path to an image file, e.g., '/path/to/example.JPG' + image_buffer: string, JPEG encoding of RGB image + label: integer, identifier for the ground truth for the network + synset: string, unique WordNet ID specifying the label, e.g., 'n02323233' + human: string, human-readable label, e.g., 'red fox, Vulpes vulpes' + bbox: list of bounding boxes; each box is a list of integers + specifying [xmin, ymin, xmax, ymax]. All boxes are assumed to belong to + the same label as the image label. + height: integer, image height in pixels + width: integer, image width in pixels + Returns: + Example proto + """ + xmin = [] + ymin = [] + xmax = [] + ymax = [] + for b in bbox: + assert len(b) == 4 + # pylint: disable=expression-not-assigned + [l.append(point) for l, point in zip([xmin, ymin, xmax, ymax], b)] + # pylint: enable=expression-not-assigned + + colorspace = 'RGB' + channels = 3 + image_format = 'JPEG' + + example = tf.train.Example(features=tf.train.Features(feature={ + 'image/height': _int64_feature(height), + 'image/width': _int64_feature(width), + 'image/colorspace': _bytes_feature(colorspace), + 'image/channels': _int64_feature(channels), + 'image/class/label': _int64_feature(label), + 'image/class/synset': _bytes_feature(synset), + 'image/class/text': _bytes_feature(human), + 'image/object/bbox/xmin': _float_feature(xmin), + 'image/object/bbox/xmax': _float_feature(xmax), + 'image/object/bbox/ymin': _float_feature(ymin), + 'image/object/bbox/ymax': _float_feature(ymax), + 'image/object/bbox/label': _int64_feature([label] * len(xmin)), + 'image/format': _bytes_feature(image_format), + 'image/filename': _bytes_feature(os.path.basename(filename)), + 'image/encoded': _bytes_feature(image_buffer)})) + return example + + +class ImageCoder(object): + """Helper class that provides TensorFlow image coding utilities.""" + + def __init__(self): + # Create a single Session to run all image coding calls. + self._sess = tf.Session() + + # Initializes function that converts PNG to JPEG data. + self._png_data = tf.placeholder(dtype=tf.string) + image = tf.image.decode_png(self._png_data, channels=3) + self._png_to_jpeg = tf.image.encode_jpeg(image, format='rgb', quality=100) + + # Initializes function that converts CMYK JPEG data to RGB JPEG data. + self._cmyk_data = tf.placeholder(dtype=tf.string) + image = tf.image.decode_jpeg(self._cmyk_data, channels=0) + self._cmyk_to_rgb = tf.image.encode_jpeg(image, format='rgb', quality=100) + + # Initializes function that decodes RGB JPEG data. + self._decode_jpeg_data = tf.placeholder(dtype=tf.string) + self._decode_jpeg = tf.image.decode_jpeg(self._decode_jpeg_data, channels=3) + + def png_to_jpeg(self, image_data): + return self._sess.run(self._png_to_jpeg, + feed_dict={self._png_data: image_data}) + + def cmyk_to_rgb(self, image_data): + return self._sess.run(self._cmyk_to_rgb, + feed_dict={self._cmyk_data: image_data}) + + def decode_jpeg(self, image_data): + image = self._sess.run(self._decode_jpeg, + feed_dict={self._decode_jpeg_data: image_data}) + assert len(image.shape) == 3 + assert image.shape[2] == 3 + return image + + +def _is_png(filename): + """Determine if a file contains a PNG format image. + + Args: + filename: string, path of the image file. + + Returns: + boolean indicating if the image is a PNG. + """ + # File list from: + # https://groups.google.com/forum/embed/?place=forum/torch7#!topic/torch7/fOSTXHIESSU + return 'n02105855_2933.JPEG' in filename + + +def _is_cmyk(filename): + """Determine if file contains a CMYK JPEG format image. + + Args: + filename: string, path of the image file. + + Returns: + boolean indicating if the image is a JPEG encoded with CMYK color space. + """ + # File list from: + # https://github.com/cytsai/ilsvrc-cmyk-image-list + blacklist = ['n01739381_1309.JPEG', 'n02077923_14822.JPEG', + 'n02447366_23489.JPEG', 'n02492035_15739.JPEG', + 'n02747177_10752.JPEG', 'n03018349_4028.JPEG', + 'n03062245_4620.JPEG', 'n03347037_9675.JPEG', + 'n03467068_12171.JPEG', 'n03529860_11437.JPEG', + 'n03544143_17228.JPEG', 'n03633091_5218.JPEG', + 'n03710637_5125.JPEG', 'n03961711_5286.JPEG', + 'n04033995_2932.JPEG', 'n04258138_17003.JPEG', + 'n04264628_27969.JPEG', 'n04336792_7448.JPEG', + 'n04371774_5854.JPEG', 'n04596742_4225.JPEG', + 'n07583066_647.JPEG', 'n13037406_4650.JPEG'] + return filename.split('/')[-1] in blacklist + + +def _process_image(filename, coder): + """Process a single image file. + + Args: + filename: string, path to an image file e.g., '/path/to/example.JPG'. + coder: instance of ImageCoder to provide TensorFlow image coding utils. + Returns: + image_buffer: string, JPEG encoding of RGB image. + height: integer, image height in pixels. + width: integer, image width in pixels. + """ + # Read the image file. + image_data = tf.gfile.GFile(filename, 'r').read() + + # Clean the dirty data. + if _is_png(filename): + # 1 image is a PNG. + print('Converting PNG to JPEG for %s' % filename) + image_data = coder.png_to_jpeg(image_data) + elif _is_cmyk(filename): + # 22 JPEG images are in CMYK colorspace. + print('Converting CMYK to RGB for %s' % filename) + image_data = coder.cmyk_to_rgb(image_data) + + # Decode the RGB JPEG. + image = coder.decode_jpeg(image_data) + + # Check that image converted to RGB + assert len(image.shape) == 3 + height = image.shape[0] + width = image.shape[1] + assert image.shape[2] == 3 + + return image_data, height, width + + +def _process_image_files_batch(coder, thread_index, ranges, name, filenames, + synsets, labels, humans, bboxes, num_shards): + """Processes and saves list of images as TFRecord in 1 thread. + + Args: + coder: instance of ImageCoder to provide TensorFlow image coding utils. + thread_index: integer, unique batch to run index is within [0, len(ranges)). + ranges: list of pairs of integers specifying ranges of each batches to + analyze in parallel. + name: string, unique identifier specifying the data set + filenames: list of strings; each string is a path to an image file + synsets: list of strings; each string is a unique WordNet ID + labels: list of integer; each integer identifies the ground truth + humans: list of strings; each string is a human-readable label + bboxes: list of bounding boxes for each image. Note that each entry in this + list might contain from 0+ entries corresponding to the number of bounding + box annotations for the image. + num_shards: integer number of shards for this data set. + """ + # Each thread produces N shards where N = int(num_shards / num_threads). + # For instance, if num_shards = 128, and the num_threads = 2, then the first + # thread would produce shards [0, 64). + num_threads = len(ranges) + assert not num_shards % num_threads + num_shards_per_batch = int(num_shards / num_threads) + + shard_ranges = np.linspace(ranges[thread_index][0], + ranges[thread_index][1], + num_shards_per_batch + 1).astype(int) + num_files_in_thread = ranges[thread_index][1] - ranges[thread_index][0] + + counter = 0 + for s in xrange(num_shards_per_batch): + # Generate a sharded version of the file name, e.g. 'train-00002-of-00010' + shard = thread_index * num_shards_per_batch + s + output_filename = '%s-%.5d-of-%.5d' % (name, shard, num_shards) + output_file = os.path.join(FLAGS.output_directory, output_filename) + writer = tf.python_io.TFRecordWriter(output_file) + + shard_counter = 0 + files_in_shard = np.arange(shard_ranges[s], shard_ranges[s + 1], dtype=int) + for i in files_in_shard: + filename = filenames[i] + label = labels[i] + synset = synsets[i] + human = humans[i] + bbox = bboxes[i] + + image_buffer, height, width = _process_image(filename, coder) + + example = _convert_to_example(filename, image_buffer, label, + synset, human, bbox, + height, width) + writer.write(example.SerializeToString()) + shard_counter += 1 + counter += 1 + + if not counter % 1000: + print('%s [thread %d]: Processed %d of %d images in thread batch.' % + (datetime.now(), thread_index, counter, num_files_in_thread)) + sys.stdout.flush() + + writer.close() + print('%s [thread %d]: Wrote %d images to %s' % + (datetime.now(), thread_index, shard_counter, output_file)) + sys.stdout.flush() + shard_counter = 0 + print('%s [thread %d]: Wrote %d images to %d shards.' % + (datetime.now(), thread_index, counter, num_files_in_thread)) + sys.stdout.flush() + + +def _process_image_files(name, filenames, synsets, labels, humans, + bboxes, num_shards): + """Process and save list of images as TFRecord of Example protos. + + Args: + name: string, unique identifier specifying the data set + filenames: list of strings; each string is a path to an image file + synsets: list of strings; each string is a unique WordNet ID + labels: list of integer; each integer identifies the ground truth + humans: list of strings; each string is a human-readable label + bboxes: list of bounding boxes for each image. Note that each entry in this + list might contain from 0+ entries corresponding to the number of bounding + box annotations for the image. + num_shards: integer number of shards for this data set. + """ + assert len(filenames) == len(synsets) + assert len(filenames) == len(labels) + assert len(filenames) == len(humans) + assert len(filenames) == len(bboxes) + + # Break all images into batches with a [ranges[i][0], ranges[i][1]]. + spacing = np.linspace(0, len(filenames), FLAGS.num_threads + 1).astype(np.int) + ranges = [] + threads = [] + for i in xrange(len(spacing) - 1): + ranges.append([spacing[i], spacing[i+1]]) + + # Launch a thread for each batch. + print('Launching %d threads for spacings: %s' % (FLAGS.num_threads, ranges)) + sys.stdout.flush() + + # Create a mechanism for monitoring when all threads are finished. + coord = tf.train.Coordinator() + + # Create a generic TensorFlow-based utility for converting all image codings. + coder = ImageCoder() + + threads = [] + for thread_index in xrange(len(ranges)): + args = (coder, thread_index, ranges, name, filenames, + synsets, labels, humans, bboxes, num_shards) + t = threading.Thread(target=_process_image_files_batch, args=args) + t.start() + threads.append(t) + + # Wait for all the threads to terminate. + coord.join(threads) + print('%s: Finished writing all %d images in data set.' % + (datetime.now(), len(filenames))) + sys.stdout.flush() + + +def _find_image_files(data_dir, labels_file): + """Build a list of all images files and labels in the data set. + + Args: + data_dir: string, path to the root directory of images. + + Assumes that the ImageNet data set resides in JPEG files located in + the following directory structure. + + data_dir/n01440764/ILSVRC2012_val_00000293.JPEG + data_dir/n01440764/ILSVRC2012_val_00000543.JPEG + + where 'n01440764' is the unique synset label associated with these images. + + labels_file: string, path to the labels file. + + The list of valid labels are held in this file. Assumes that the file + contains entries as such: + n01440764 + n01443537 + n01484850 + where each line corresponds to a label expressed as a synset. We map + each synset contained in the file to an integer (based on the alphabetical + ordering) starting with the integer 1 corresponding to the synset + contained in the first line. + + The reason we start the integer labels at 1 is to reserve label 0 as an + unused background class. + + Returns: + filenames: list of strings; each string is a path to an image file. + synsets: list of strings; each string is a unique WordNet ID. + labels: list of integer; each integer identifies the ground truth. + """ + print('Determining list of input files and labels from %s.' % data_dir) + challenge_synsets = [ + l.strip() for l in tf.gfile.GFile(labels_file, 'r').readlines() + ] + + labels = [] + filenames = [] + synsets = [] + + # Leave label index 0 empty as a background class. + label_index = 1 + + # Construct the list of JPEG files and labels. + for synset in challenge_synsets: + jpeg_file_path = '%s/%s/*.JPEG' % (data_dir, synset) + matching_files = tf.gfile.Glob(jpeg_file_path) + + labels.extend([label_index] * len(matching_files)) + synsets.extend([synset] * len(matching_files)) + filenames.extend(matching_files) + + if not label_index % 100: + print('Finished finding files in %d of %d classes.' % ( + label_index, len(challenge_synsets))) + label_index += 1 + + # Shuffle the ordering of all image files in order to guarantee + # random ordering of the images with respect to label in the + # saved TFRecord files. Make the randomization repeatable. + shuffled_index = range(len(filenames)) + random.seed(12345) + random.shuffle(shuffled_index) + + filenames = [filenames[i] for i in shuffled_index] + synsets = [synsets[i] for i in shuffled_index] + labels = [labels[i] for i in shuffled_index] + + print('Found %d JPEG files across %d labels inside %s.' % + (len(filenames), len(challenge_synsets), data_dir)) + return filenames, synsets, labels + + +def _find_human_readable_labels(synsets, synset_to_human): + """Build a list of human-readable labels. + + Args: + synsets: list of strings; each string is a unique WordNet ID. + synset_to_human: dict of synset to human labels, e.g., + 'n02119022' --> 'red fox, Vulpes vulpes' + + Returns: + List of human-readable strings corresponding to each synset. + """ + humans = [] + for s in synsets: + assert s in synset_to_human, ('Failed to find: %s' % s) + humans.append(synset_to_human[s]) + return humans + + +def _find_image_bounding_boxes(filenames, image_to_bboxes): + """Find the bounding boxes for a given image file. + + Args: + filenames: list of strings; each string is a path to an image file. + image_to_bboxes: dictionary mapping image file names to a list of + bounding boxes. This list contains 0+ bounding boxes. + Returns: + List of bounding boxes for each image. Note that each entry in this + list might contain from 0+ entries corresponding to the number of bounding + box annotations for the image. + """ + num_image_bbox = 0 + bboxes = [] + for f in filenames: + basename = os.path.basename(f) + if basename in image_to_bboxes: + bboxes.append(image_to_bboxes[basename]) + num_image_bbox += 1 + else: + bboxes.append([]) + print('Found %d images with bboxes out of %d images' % ( + num_image_bbox, len(filenames))) + return bboxes + + +def _process_dataset(name, directory, num_shards, synset_to_human, + image_to_bboxes): + """Process a complete data set and save it as a TFRecord. + + Args: + name: string, unique identifier specifying the data set. + directory: string, root path to the data set. + num_shards: integer number of shards for this data set. + synset_to_human: dict of synset to human labels, e.g., + 'n02119022' --> 'red fox, Vulpes vulpes' + image_to_bboxes: dictionary mapping image file names to a list of + bounding boxes. This list contains 0+ bounding boxes. + """ + filenames, synsets, labels = _find_image_files(directory, FLAGS.labels_file) + humans = _find_human_readable_labels(synsets, synset_to_human) + bboxes = _find_image_bounding_boxes(filenames, image_to_bboxes) + _process_image_files(name, filenames, synsets, labels, + humans, bboxes, num_shards) + + +def _build_synset_lookup(imagenet_metadata_file): + """Build lookup for synset to human-readable label. + + Args: + imagenet_metadata_file: string, path to file containing mapping from + synset to human-readable label. + + Assumes each line of the file looks like: + + n02119247 black fox + n02119359 silver fox + n02119477 red fox, Vulpes fulva + + where each line corresponds to a unique mapping. Note that each line is + formatted as \t. + + Returns: + Dictionary of synset to human labels, such as: + 'n02119022' --> 'red fox, Vulpes vulpes' + """ + lines = tf.gfile.GFile(imagenet_metadata_file, 'r').readlines() + synset_to_human = {} + for l in lines: + if l: + parts = l.strip().split('\t') + assert len(parts) == 2 + synset = parts[0] + human = parts[1] + synset_to_human[synset] = human + return synset_to_human + + +def _build_bounding_box_lookup(bounding_box_file): + """Build a lookup from image file to bounding boxes. + + Args: + bounding_box_file: string, path to file with bounding boxes annotations. + + Assumes each line of the file looks like: + + n00007846_64193.JPEG,0.0060,0.2620,0.7545,0.9940 + + where each line corresponds to one bounding box annotation associated + with an image. Each line can be parsed as: + + , , , , + + Note that there might exist mulitple bounding box annotations associated + with an image file. This file is the output of process_bounding_boxes.py. + + Returns: + Dictionary mapping image file names to a list of bounding boxes. This list + contains 0+ bounding boxes. + """ + lines = tf.gfile.GFile(bounding_box_file, 'r').readlines() + images_to_bboxes = {} + num_bbox = 0 + num_image = 0 + for l in lines: + if l: + parts = l.split(',') + assert len(parts) == 5, ('Failed to parse: %s' % l) + filename = parts[0] + xmin = float(parts[1]) + ymin = float(parts[2]) + xmax = float(parts[3]) + ymax = float(parts[4]) + box = [xmin, ymin, xmax, ymax] + + if filename not in images_to_bboxes: + images_to_bboxes[filename] = [] + num_image += 1 + images_to_bboxes[filename].append(box) + num_bbox += 1 + + print('Successfully read %d bounding boxes ' + 'across %d images.' % (num_bbox, num_image)) + return images_to_bboxes + + +def main(unused_argv): + assert not FLAGS.train_shards % FLAGS.num_threads, ( + 'Please make the FLAGS.num_threads commensurate with FLAGS.train_shards') + assert not FLAGS.validation_shards % FLAGS.num_threads, ( + 'Please make the FLAGS.num_threads commensurate with ' + 'FLAGS.validation_shards') + print('Saving results to %s' % FLAGS.output_directory) + + # Build a map from synset to human-readable label. + synset_to_human = _build_synset_lookup(FLAGS.imagenet_metadata_file) + image_to_bboxes = _build_bounding_box_lookup(FLAGS.bounding_box_file) + + # Run it! + _process_dataset('validation', FLAGS.validation_directory, + FLAGS.validation_shards, synset_to_human, image_to_bboxes) + _process_dataset('train', FLAGS.train_directory, FLAGS.train_shards, + synset_to_human, image_to_bboxes) + + +if __name__ == '__main__': + tf.app.run() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/cifar10.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/cifar10.py new file mode 100644 index 0000000..f0f5d61 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/cifar10.py @@ -0,0 +1,100 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Provides data for the Cifar10 dataset. + +The dataset scripts used to create the dataset can be found at: +tensorflow/models/research/slim/datasets/download_and_convert_cifar10.py +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from datasets import dataset_utils + +slim = contrib_slim + +_FILE_PATTERN = 'cifar10_%s.tfrecord' + +SPLITS_TO_SIZES = {'train': 50000, 'test': 10000} + +_NUM_CLASSES = 10 + +_ITEMS_TO_DESCRIPTIONS = { + 'image': 'A [32 x 32 x 3] color image.', + 'label': 'A single integer between 0 and 9', +} + + +def get_split(split_name, dataset_dir, file_pattern=None, reader=None): + """Gets a dataset tuple with instructions for reading cifar10. + + Args: + split_name: A train/test split name. + dataset_dir: The base directory of the dataset sources. + file_pattern: The file pattern to use when matching the dataset sources. + It is assumed that the pattern contains a '%s' string so that the split + name can be inserted. + reader: The TensorFlow reader type. + + Returns: + A `Dataset` namedtuple. + + Raises: + ValueError: if `split_name` is not a valid train/test split. + """ + if split_name not in SPLITS_TO_SIZES: + raise ValueError('split name %s was not recognized.' % split_name) + + if not file_pattern: + file_pattern = _FILE_PATTERN + file_pattern = os.path.join(dataset_dir, file_pattern % split_name) + + # Allowing None in the signature so that dataset_factory can use the default. + if not reader: + reader = tf.TFRecordReader + + keys_to_features = { + 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), + 'image/format': tf.FixedLenFeature((), tf.string, default_value='png'), + 'image/class/label': tf.FixedLenFeature( + [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), + } + + items_to_handlers = { + 'image': slim.tfexample_decoder.Image(shape=[32, 32, 3]), + 'label': slim.tfexample_decoder.Tensor('image/class/label'), + } + + decoder = slim.tfexample_decoder.TFExampleDecoder( + keys_to_features, items_to_handlers) + + labels_to_names = None + if dataset_utils.has_labels(dataset_dir): + labels_to_names = dataset_utils.read_label_file(dataset_dir) + + return slim.dataset.Dataset( + data_sources=file_pattern, + reader=reader, + decoder=decoder, + num_samples=SPLITS_TO_SIZES[split_name], + items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, + num_classes=_NUM_CLASSES, + labels_to_names=labels_to_names, + ) diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/dataset_factory.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/dataset_factory.py new file mode 100644 index 0000000..2cfb3f8 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/dataset_factory.py @@ -0,0 +1,59 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A factory-pattern class which returns classification image/label pairs.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from datasets import cifar10 +from datasets import flowers +from datasets import imagenet +from datasets import mnist +from datasets import visualwakewords + +datasets_map = { + 'cifar10': cifar10, + 'flowers': flowers, + 'imagenet': imagenet, + 'mnist': mnist, + 'visualwakewords': visualwakewords, +} + + +def get_dataset(name, split_name, dataset_dir, file_pattern=None, reader=None): + """Given a dataset name and a split_name returns a Dataset. + + Args: + name: String, the name of the dataset. + split_name: A train/test split name. + dataset_dir: The directory where the dataset files are stored. + file_pattern: The file pattern to use for matching the dataset source files. + reader: The subclass of tf.ReaderBase. If left as `None`, then the default + reader defined by each dataset is used. + + Returns: + A `Dataset` class. + + Raises: + ValueError: If the dataset `name` is unknown. + """ + if name not in datasets_map: + raise ValueError('Name of dataset unknown %s' % name) + return datasets_map[name].get_split( + split_name, + dataset_dir, + file_pattern, + reader) diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/dataset_utils.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/dataset_utils.py new file mode 100644 index 0000000..47e27d1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/dataset_utils.py @@ -0,0 +1,240 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains utilities for downloading and converting datasets.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys +import tarfile +import zipfile + +from six.moves import urllib +import tensorflow as tf + +LABELS_FILENAME = 'labels.txt' + + +def int64_feature(values): + """Returns a TF-Feature of int64s. + + Args: + values: A scalar or list of values. + + Returns: + A TF-Feature. + """ + if not isinstance(values, (tuple, list)): + values = [values] + return tf.train.Feature(int64_list=tf.train.Int64List(value=values)) + + +def bytes_list_feature(values): + """Returns a TF-Feature of list of bytes. + + Args: + values: A string or list of strings. + + Returns: + A TF-Feature. + """ + return tf.train.Feature(bytes_list=tf.train.BytesList(value=values)) + + +def float_list_feature(values): + """Returns a TF-Feature of list of floats. + + Args: + values: A float or list of floats. + + Returns: + A TF-Feature. + """ + return tf.train.Feature(float_list=tf.train.FloatList(value=values)) + + +def bytes_feature(values): + """Returns a TF-Feature of bytes. + + Args: + values: A string. + + Returns: + A TF-Feature. + """ + return tf.train.Feature(bytes_list=tf.train.BytesList(value=[values])) + + +def float_feature(values): + """Returns a TF-Feature of floats. + + Args: + values: A scalar of list of values. + + Returns: + A TF-Feature. + """ + if not isinstance(values, (tuple, list)): + values = [values] + return tf.train.Feature(float_list=tf.train.FloatList(value=values)) + + +def image_to_tfexample(image_data, image_format, height, width, class_id): + return tf.train.Example(features=tf.train.Features(feature={ + 'image/encoded': bytes_feature(image_data), + 'image/format': bytes_feature(image_format), + 'image/class/label': int64_feature(class_id), + 'image/height': int64_feature(height), + 'image/width': int64_feature(width), + })) + + +def download_url(url, dataset_dir): + """Downloads the tarball or zip file from url into filepath. + + Args: + url: The URL of a tarball or zip file. + dataset_dir: The directory where the temporary files are stored. + + Returns: + filepath: path where the file is downloaded. + """ + filename = url.split('/')[-1] + filepath = os.path.join(dataset_dir, filename) + + def _progress(count, block_size, total_size): + sys.stdout.write('\r>> Downloading %s %.1f%%' % ( + filename, float(count * block_size) / float(total_size) * 100.0)) + sys.stdout.flush() + + filepath, _ = urllib.request.urlretrieve(url, filepath, _progress) + print() + statinfo = os.stat(filepath) + print('Successfully downloaded', filename, statinfo.st_size, 'bytes.') + return filepath + + +def download_and_uncompress_tarball(tarball_url, dataset_dir): + """Downloads the `tarball_url` and uncompresses it locally. + + Args: + tarball_url: The URL of a tarball file. + dataset_dir: The directory where the temporary files are stored. + """ + filepath = download_url(tarball_url, dataset_dir) + tarfile.open(filepath, 'r:gz').extractall(dataset_dir) + + +def download_and_uncompress_zipfile(zip_url, dataset_dir): + """Downloads the `zip_url` and uncompresses it locally. + + Args: + zip_url: The URL of a zip file. + dataset_dir: The directory where the temporary files are stored. + """ + filename = zip_url.split('/')[-1] + filepath = os.path.join(dataset_dir, filename) + + if tf.gfile.Exists(filepath): + print('File {filename} has been already downloaded at {filepath}. ' + 'Unzipping it....'.format(filename=filename, filepath=filepath)) + else: + filepath = download_url(zip_url, dataset_dir) + + with zipfile.ZipFile(filepath, 'r') as zip_file: + for member in zip_file.namelist(): + memberpath = os.path.join(dataset_dir, member) + # extract only if file doesn't exist + if not (os.path.exists(memberpath) or os.path.isfile(memberpath)): + zip_file.extract(member, dataset_dir) + + +def write_label_file(labels_to_class_names, + dataset_dir, + filename=LABELS_FILENAME): + """Writes a file with the list of class names. + + Args: + labels_to_class_names: A map of (integer) labels to class names. + dataset_dir: The directory in which the labels file should be written. + filename: The filename where the class names are written. + """ + labels_filename = os.path.join(dataset_dir, filename) + with tf.gfile.Open(labels_filename, 'w') as f: + for label in labels_to_class_names: + class_name = labels_to_class_names[label] + f.write('%d:%s\n' % (label, class_name)) + + +def has_labels(dataset_dir, filename=LABELS_FILENAME): + """Specifies whether or not the dataset directory contains a label map file. + + Args: + dataset_dir: The directory in which the labels file is found. + filename: The filename where the class names are written. + + Returns: + `True` if the labels file exists and `False` otherwise. + """ + return tf.gfile.Exists(os.path.join(dataset_dir, filename)) + + +def read_label_file(dataset_dir, filename=LABELS_FILENAME): + """Reads the labels file and returns a mapping from ID to class name. + + Args: + dataset_dir: The directory in which the labels file is found. + filename: The filename where the class names are written. + + Returns: + A map from a label (integer) to class name. + """ + labels_filename = os.path.join(dataset_dir, filename) + with tf.gfile.Open(labels_filename, 'rb') as f: + lines = f.read().decode() + lines = lines.split('\n') + lines = filter(None, lines) + + labels_to_class_names = {} + for line in lines: + index = line.index(':') + labels_to_class_names[int(line[:index])] = line[index+1:] + return labels_to_class_names + + +def open_sharded_output_tfrecords(exit_stack, base_path, num_shards): + """Opens all TFRecord shards for writing and adds them to an exit stack. + + Args: + exit_stack: A context2.ExitStack used to automatically closed the TFRecords + opened in this function. + base_path: The base path for all shards + num_shards: The number of shards + + Returns: + The list of opened TFRecords. Position k in the list corresponds to shard k. + """ + tf_record_output_filenames = [ + '{}-{:05d}-of-{:05d}'.format(base_path, idx, num_shards) + for idx in range(num_shards) + ] + + tfrecords = [ + exit_stack.enter_context(tf.python_io.TFRecordWriter(file_name)) + for file_name in tf_record_output_filenames + ] + + return tfrecords diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_cifar10.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_cifar10.py new file mode 100644 index 0000000..f23618e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_cifar10.py @@ -0,0 +1,198 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""Downloads and converts cifar10 data to TFRecords of TF-Example protos. + +This module downloads the cifar10 data, uncompresses it, reads the files +that make up the cifar10 data and creates two TFRecord datasets: one for train +and one for test. Each TFRecord dataset is comprised of a set of TF-Example +protocol buffers, each of which contain a single image and label. + +The script should take several minutes to run. + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys +import tarfile + +import numpy as np +from six.moves import cPickle +from six.moves import urllib +import tensorflow as tf + +from datasets import dataset_utils + +# The URL where the CIFAR data can be downloaded. +_DATA_URL = 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz' + +# The number of training files. +_NUM_TRAIN_FILES = 5 + +# The height and width of each image. +_IMAGE_SIZE = 32 + +# The names of the classes. +_CLASS_NAMES = [ + 'airplane', + 'automobile', + 'bird', + 'cat', + 'deer', + 'dog', + 'frog', + 'horse', + 'ship', + 'truck', +] + + +def _add_to_tfrecord(filename, tfrecord_writer, offset=0): + """Loads data from the cifar10 pickle files and writes files to a TFRecord. + + Args: + filename: The filename of the cifar10 pickle file. + tfrecord_writer: The TFRecord writer to use for writing. + offset: An offset into the absolute number of images previously written. + + Returns: + The new offset. + """ + with tf.gfile.Open(filename, 'rb') as f: + if sys.version_info < (3,): + data = cPickle.load(f) + else: + data = cPickle.load(f, encoding='bytes') + + images = data[b'data'] + num_images = images.shape[0] + + images = images.reshape((num_images, 3, 32, 32)) + labels = data[b'labels'] + + with tf.Graph().as_default(): + image_placeholder = tf.placeholder(dtype=tf.uint8) + encoded_image = tf.image.encode_png(image_placeholder) + + with tf.Session('') as sess: + + for j in range(num_images): + sys.stdout.write('\r>> Reading file [%s] image %d/%d' % ( + filename, offset + j + 1, offset + num_images)) + sys.stdout.flush() + + image = np.squeeze(images[j]).transpose((1, 2, 0)) + label = labels[j] + + png_string = sess.run(encoded_image, + feed_dict={image_placeholder: image}) + + example = dataset_utils.image_to_tfexample( + png_string, b'png', _IMAGE_SIZE, _IMAGE_SIZE, label) + tfrecord_writer.write(example.SerializeToString()) + + return offset + num_images + + +def _get_output_filename(dataset_dir, split_name): + """Creates the output filename. + + Args: + dataset_dir: The dataset directory where the dataset is stored. + split_name: The name of the train/test split. + + Returns: + An absolute file path. + """ + return '%s/cifar10_%s.tfrecord' % (dataset_dir, split_name) + + +def _download_and_uncompress_dataset(dataset_dir): + """Downloads cifar10 and uncompresses it locally. + + Args: + dataset_dir: The directory where the temporary files are stored. + """ + filename = _DATA_URL.split('/')[-1] + filepath = os.path.join(dataset_dir, filename) + + if not os.path.exists(filepath): + def _progress(count, block_size, total_size): + sys.stdout.write('\r>> Downloading %s %.1f%%' % ( + filename, float(count * block_size) / float(total_size) * 100.0)) + sys.stdout.flush() + filepath, _ = urllib.request.urlretrieve(_DATA_URL, filepath, _progress) + print() + statinfo = os.stat(filepath) + print('Successfully downloaded', filename, statinfo.st_size, 'bytes.') + tarfile.open(filepath, 'r:gz').extractall(dataset_dir) + + +def _clean_up_temporary_files(dataset_dir): + """Removes temporary files used to create the dataset. + + Args: + dataset_dir: The directory where the temporary files are stored. + """ + filename = _DATA_URL.split('/')[-1] + filepath = os.path.join(dataset_dir, filename) + tf.gfile.Remove(filepath) + + tmp_dir = os.path.join(dataset_dir, 'cifar-10-batches-py') + tf.gfile.DeleteRecursively(tmp_dir) + + +def run(dataset_dir): + """Runs the download and conversion operation. + + Args: + dataset_dir: The dataset directory where the dataset is stored. + """ + if not tf.gfile.Exists(dataset_dir): + tf.gfile.MakeDirs(dataset_dir) + + training_filename = _get_output_filename(dataset_dir, 'train') + testing_filename = _get_output_filename(dataset_dir, 'test') + + if tf.gfile.Exists(training_filename) and tf.gfile.Exists(testing_filename): + print('Dataset files already exist. Exiting without re-creating them.') + return + + dataset_utils.download_and_uncompress_tarball(_DATA_URL, dataset_dir) + + # First, process the training data: + with tf.python_io.TFRecordWriter(training_filename) as tfrecord_writer: + offset = 0 + for i in range(_NUM_TRAIN_FILES): + filename = os.path.join(dataset_dir, + 'cifar-10-batches-py', + 'data_batch_%d' % (i + 1)) # 1-indexed. + offset = _add_to_tfrecord(filename, tfrecord_writer, offset) + + # Next, process the testing data: + with tf.python_io.TFRecordWriter(testing_filename) as tfrecord_writer: + filename = os.path.join(dataset_dir, + 'cifar-10-batches-py', + 'test_batch') + _add_to_tfrecord(filename, tfrecord_writer) + + # Finally, write the labels file: + labels_to_class_names = dict(zip(range(len(_CLASS_NAMES)), _CLASS_NAMES)) + dataset_utils.write_label_file(labels_to_class_names, dataset_dir) + + _clean_up_temporary_files(dataset_dir) + print('\nFinished converting the Cifar10 dataset!') diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_flowers.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_flowers.py new file mode 100644 index 0000000..7976e38 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_flowers.py @@ -0,0 +1,211 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""Downloads and converts Flowers data to TFRecords of TF-Example protos. + +This module downloads the Flowers data, uncompresses it, reads the files +that make up the Flowers data and creates two TFRecord datasets: one for train +and one for test. Each TFRecord dataset is comprised of a set of TF-Example +protocol buffers, each of which contain a single image and label. + +The script should take about a minute to run. + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math +import os +import random +import sys + +import tensorflow as tf + +from datasets import dataset_utils + +# The URL where the Flowers data can be downloaded. +_DATA_URL = 'http://download.tensorflow.org/example_images/flower_photos.tgz' + +# The number of images in the validation set. +_NUM_VALIDATION = 350 + +# Seed for repeatability. +_RANDOM_SEED = 0 + +# The number of shards per dataset split. +_NUM_SHARDS = 5 + + +class ImageReader(object): + """Helper class that provides TensorFlow image coding utilities.""" + + def __init__(self): + # Initializes function that decodes RGB JPEG data. + self._decode_jpeg_data = tf.placeholder(dtype=tf.string) + self._decode_jpeg = tf.image.decode_jpeg(self._decode_jpeg_data, channels=3) + + def read_image_dims(self, sess, image_data): + image = self.decode_jpeg(sess, image_data) + return image.shape[0], image.shape[1] + + def decode_jpeg(self, sess, image_data): + image = sess.run(self._decode_jpeg, + feed_dict={self._decode_jpeg_data: image_data}) + assert len(image.shape) == 3 + assert image.shape[2] == 3 + return image + + +def _get_filenames_and_classes(dataset_dir): + """Returns a list of filenames and inferred class names. + + Args: + dataset_dir: A directory containing a set of subdirectories representing + class names. Each subdirectory should contain PNG or JPG encoded images. + + Returns: + A list of image file paths, relative to `dataset_dir` and the list of + subdirectories, representing class names. + """ + flower_root = os.path.join(dataset_dir, 'flower_photos') + directories = [] + class_names = [] + for filename in os.listdir(flower_root): + path = os.path.join(flower_root, filename) + if os.path.isdir(path): + directories.append(path) + class_names.append(filename) + + photo_filenames = [] + for directory in directories: + for filename in os.listdir(directory): + path = os.path.join(directory, filename) + photo_filenames.append(path) + + return photo_filenames, sorted(class_names) + + +def _get_dataset_filename(dataset_dir, split_name, shard_id): + output_filename = 'flowers_%s_%05d-of-%05d.tfrecord' % ( + split_name, shard_id, _NUM_SHARDS) + return os.path.join(dataset_dir, output_filename) + + +def _convert_dataset(split_name, filenames, class_names_to_ids, dataset_dir): + """Converts the given filenames to a TFRecord dataset. + + Args: + split_name: The name of the dataset, either 'train' or 'validation'. + filenames: A list of absolute paths to png or jpg images. + class_names_to_ids: A dictionary from class names (strings) to ids + (integers). + dataset_dir: The directory where the converted datasets are stored. + """ + assert split_name in ['train', 'validation'] + + num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS))) + + with tf.Graph().as_default(): + image_reader = ImageReader() + + with tf.Session('') as sess: + + for shard_id in range(_NUM_SHARDS): + output_filename = _get_dataset_filename( + dataset_dir, split_name, shard_id) + + with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer: + start_ndx = shard_id * num_per_shard + end_ndx = min((shard_id+1) * num_per_shard, len(filenames)) + for i in range(start_ndx, end_ndx): + sys.stdout.write('\r>> Converting image %d/%d shard %d' % ( + i+1, len(filenames), shard_id)) + sys.stdout.flush() + + # Read the filename: + image_data = tf.gfile.GFile(filenames[i], 'rb').read() + height, width = image_reader.read_image_dims(sess, image_data) + + class_name = os.path.basename(os.path.dirname(filenames[i])) + class_id = class_names_to_ids[class_name] + + example = dataset_utils.image_to_tfexample( + image_data, b'jpg', height, width, class_id) + tfrecord_writer.write(example.SerializeToString()) + + sys.stdout.write('\n') + sys.stdout.flush() + + +def _clean_up_temporary_files(dataset_dir): + """Removes temporary files used to create the dataset. + + Args: + dataset_dir: The directory where the temporary files are stored. + """ + filename = _DATA_URL.split('/')[-1] + filepath = os.path.join(dataset_dir, filename) + tf.gfile.Remove(filepath) + + tmp_dir = os.path.join(dataset_dir, 'flower_photos') + tf.gfile.DeleteRecursively(tmp_dir) + + +def _dataset_exists(dataset_dir): + for split_name in ['train', 'validation']: + for shard_id in range(_NUM_SHARDS): + output_filename = _get_dataset_filename( + dataset_dir, split_name, shard_id) + if not tf.gfile.Exists(output_filename): + return False + return True + + +def run(dataset_dir): + """Runs the download and conversion operation. + + Args: + dataset_dir: The dataset directory where the dataset is stored. + """ + if not tf.gfile.Exists(dataset_dir): + tf.gfile.MakeDirs(dataset_dir) + + if _dataset_exists(dataset_dir): + print('Dataset files already exist. Exiting without re-creating them.') + return + + dataset_utils.download_and_uncompress_tarball(_DATA_URL, dataset_dir) + photo_filenames, class_names = _get_filenames_and_classes(dataset_dir) + class_names_to_ids = dict(zip(class_names, range(len(class_names)))) + + # Divide into train and test: + random.seed(_RANDOM_SEED) + random.shuffle(photo_filenames) + training_filenames = photo_filenames[_NUM_VALIDATION:] + validation_filenames = photo_filenames[:_NUM_VALIDATION] + + # First, convert the training and validation sets. + _convert_dataset('train', training_filenames, class_names_to_ids, + dataset_dir) + _convert_dataset('validation', validation_filenames, class_names_to_ids, + dataset_dir) + + # Finally, write the labels file: + labels_to_class_names = dict(zip(range(len(class_names)), class_names)) + dataset_utils.write_label_file(labels_to_class_names, dataset_dir) + + _clean_up_temporary_files(dataset_dir) + print('\nFinished converting the Flowers dataset!') diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_imagenet.sh b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_imagenet.sh new file mode 100644 index 0000000..b4b3866 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_imagenet.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# Script to download and preprocess ImageNet Challenge 2012 +# training and validation data set. +# +# The final output of this script are sharded TFRecord files containing +# serialized Example protocol buffers. See build_imagenet_data.py for +# details of how the Example protocol buffers contain the ImageNet data. +# +# The final output of this script appears as such: +# +# data_dir/train-00000-of-01024 +# data_dir/train-00001-of-01024 +# ... +# data_dir/train-00127-of-01024 +# +# and +# +# data_dir/validation-00000-of-00128 +# data_dir/validation-00001-of-00128 +# ... +# data_dir/validation-00127-of-00128 +# +# Note that this script may take several hours to run to completion. The +# conversion of the ImageNet data to TFRecords alone takes 2-3 hours depending +# on the speed of your machine. Please be patient. +# +# **IMPORTANT** +# To download the raw images, the user must create an account with image-net.org +# and generate a username and access_key. The latter two are required for +# downloading the raw images. +# +# usage: +# cd research/slim +# bazel build :download_and_convert_imagenet +# ./bazel-bin/download_and_convert_imagenet.sh [data-dir] +set -e + +if [ -z "$1" ]; then + echo "usage download_and_convert_imagenet.sh [data dir]" + exit +fi + +# Create the output and temporary directories. +DATA_DIR="${1%/}" +SCRATCH_DIR="${DATA_DIR}/raw-data/" +mkdir -p "${DATA_DIR}" +mkdir -p "${SCRATCH_DIR}" +WORK_DIR="$0.runfiles/__main__" + +# Download the ImageNet data. +LABELS_FILE="${WORK_DIR}/datasets/imagenet_lsvrc_2015_synsets.txt" +DOWNLOAD_SCRIPT="${WORK_DIR}/datasets/download_imagenet.sh" +"${DOWNLOAD_SCRIPT}" "${SCRATCH_DIR}" "${LABELS_FILE}" + +# Note the locations of the train and validation data. +TRAIN_DIRECTORY="${SCRATCH_DIR}train/" +VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/" + +# Preprocess the validation data by moving the images into the appropriate +# sub-directory based on the label (synset) of the image. +echo "Organizing the validation data into sub-directories." +PREPROCESS_VAL_SCRIPT="${WORK_DIR}/datasets/preprocess_imagenet_validation_data.py" +VAL_LABELS_FILE="${WORK_DIR}/datasets/imagenet_2012_validation_synset_labels.txt" + +"${PREPROCESS_VAL_SCRIPT}" "${VALIDATION_DIRECTORY}" "${VAL_LABELS_FILE}" + +# Convert the XML files for bounding box annotations into a single CSV. +echo "Extracting bounding box information from XML." +BOUNDING_BOX_SCRIPT="${WORK_DIR}/datasets/process_bounding_boxes.py" +BOUNDING_BOX_FILE="${SCRATCH_DIR}/imagenet_2012_bounding_boxes.csv" +BOUNDING_BOX_DIR="${SCRATCH_DIR}bounding_boxes/" + +"${BOUNDING_BOX_SCRIPT}" "${BOUNDING_BOX_DIR}" "${LABELS_FILE}" \ + | sort >"${BOUNDING_BOX_FILE}" +echo "Finished downloading and preprocessing the ImageNet data." + +# Build the TFRecords version of the ImageNet data. +BUILD_SCRIPT="${WORK_DIR}/build_imagenet_data" +OUTPUT_DIRECTORY="${DATA_DIR}" +IMAGENET_METADATA_FILE="${WORK_DIR}/datasets/imagenet_metadata.txt" + +"${BUILD_SCRIPT}" \ + --train_directory="${TRAIN_DIRECTORY}" \ + --validation_directory="${VALIDATION_DIRECTORY}" \ + --output_directory="${OUTPUT_DIRECTORY}" \ + --imagenet_metadata_file="${IMAGENET_METADATA_FILE}" \ + --labels_file="${LABELS_FILE}" \ + --bounding_box_file="${BOUNDING_BOX_FILE}" diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_mnist.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_mnist.py new file mode 100644 index 0000000..d6ae874 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_mnist.py @@ -0,0 +1,221 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""Downloads and converts MNIST data to TFRecords of TF-Example protos. + +This module downloads the MNIST data, uncompresses it, reads the files +that make up the MNIST data and creates two TFRecord datasets: one for train +and one for test. Each TFRecord dataset is comprised of a set of TF-Example +protocol buffers, each of which contain a single image and label. + +The script should take about a minute to run. + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gzip +import os +import sys + +import numpy as np +from six.moves import urllib +import tensorflow as tf + +from datasets import dataset_utils + +# The URLs where the MNIST data can be downloaded. +_DATA_URL = 'http://yann.lecun.com/exdb/mnist/' +_TRAIN_DATA_FILENAME = 'train-images-idx3-ubyte.gz' +_TRAIN_LABELS_FILENAME = 'train-labels-idx1-ubyte.gz' +_TEST_DATA_FILENAME = 't10k-images-idx3-ubyte.gz' +_TEST_LABELS_FILENAME = 't10k-labels-idx1-ubyte.gz' + +_IMAGE_SIZE = 28 +_NUM_CHANNELS = 1 + +# The names of the classes. +_CLASS_NAMES = [ + 'zero', + 'one', + 'two', + 'three', + 'four', + 'five', + 'size', + 'seven', + 'eight', + 'nine', +] + + +def _extract_images(filename, num_images): + """Extract the images into a numpy array. + + Args: + filename: The path to an MNIST images file. + num_images: The number of images in the file. + + Returns: + A numpy array of shape [number_of_images, height, width, channels]. + """ + print('Extracting images from: ', filename) + with gzip.open(filename) as bytestream: + bytestream.read(16) + buf = bytestream.read( + _IMAGE_SIZE * _IMAGE_SIZE * num_images * _NUM_CHANNELS) + data = np.frombuffer(buf, dtype=np.uint8) + data = data.reshape(num_images, _IMAGE_SIZE, _IMAGE_SIZE, _NUM_CHANNELS) + return data + + +def _extract_labels(filename, num_labels): + """Extract the labels into a vector of int64 label IDs. + + Args: + filename: The path to an MNIST labels file. + num_labels: The number of labels in the file. + + Returns: + A numpy array of shape [number_of_labels] + """ + print('Extracting labels from: ', filename) + with gzip.open(filename) as bytestream: + bytestream.read(8) + buf = bytestream.read(1 * num_labels) + labels = np.frombuffer(buf, dtype=np.uint8).astype(np.int64) + return labels + + +def _add_to_tfrecord(data_filename, labels_filename, num_images, + tfrecord_writer): + """Loads data from the binary MNIST files and writes files to a TFRecord. + + Args: + data_filename: The filename of the MNIST images. + labels_filename: The filename of the MNIST labels. + num_images: The number of images in the dataset. + tfrecord_writer: The TFRecord writer to use for writing. + """ + images = _extract_images(data_filename, num_images) + labels = _extract_labels(labels_filename, num_images) + + shape = (_IMAGE_SIZE, _IMAGE_SIZE, _NUM_CHANNELS) + with tf.Graph().as_default(): + image = tf.placeholder(dtype=tf.uint8, shape=shape) + encoded_png = tf.image.encode_png(image) + + with tf.Session('') as sess: + for j in range(num_images): + sys.stdout.write('\r>> Converting image %d/%d' % (j + 1, num_images)) + sys.stdout.flush() + + png_string = sess.run(encoded_png, feed_dict={image: images[j]}) + + example = dataset_utils.image_to_tfexample( + png_string, 'png'.encode(), _IMAGE_SIZE, _IMAGE_SIZE, labels[j]) + tfrecord_writer.write(example.SerializeToString()) + + +def _get_output_filename(dataset_dir, split_name): + """Creates the output filename. + + Args: + dataset_dir: The directory where the temporary files are stored. + split_name: The name of the train/test split. + + Returns: + An absolute file path. + """ + return '%s/mnist_%s.tfrecord' % (dataset_dir, split_name) + + +def _download_dataset(dataset_dir): + """Downloads MNIST locally. + + Args: + dataset_dir: The directory where the temporary files are stored. + """ + for filename in [_TRAIN_DATA_FILENAME, + _TRAIN_LABELS_FILENAME, + _TEST_DATA_FILENAME, + _TEST_LABELS_FILENAME]: + filepath = os.path.join(dataset_dir, filename) + + if not os.path.exists(filepath): + print('Downloading file %s...' % filename) + def _progress(count, block_size, total_size): + sys.stdout.write('\r>> Downloading %.1f%%' % ( + float(count * block_size) / float(total_size) * 100.0)) + sys.stdout.flush() + filepath, _ = urllib.request.urlretrieve(_DATA_URL + filename, + filepath, + _progress) + print() + with tf.gfile.GFile(filepath) as f: + size = f.size() + print('Successfully downloaded', filename, size, 'bytes.') + + +def _clean_up_temporary_files(dataset_dir): + """Removes temporary files used to create the dataset. + + Args: + dataset_dir: The directory where the temporary files are stored. + """ + for filename in [_TRAIN_DATA_FILENAME, + _TRAIN_LABELS_FILENAME, + _TEST_DATA_FILENAME, + _TEST_LABELS_FILENAME]: + filepath = os.path.join(dataset_dir, filename) + tf.gfile.Remove(filepath) + + +def run(dataset_dir): + """Runs the download and conversion operation. + + Args: + dataset_dir: The dataset directory where the dataset is stored. + """ + if not tf.gfile.Exists(dataset_dir): + tf.gfile.MakeDirs(dataset_dir) + + training_filename = _get_output_filename(dataset_dir, 'train') + testing_filename = _get_output_filename(dataset_dir, 'test') + + if tf.gfile.Exists(training_filename) and tf.gfile.Exists(testing_filename): + print('Dataset files already exist. Exiting without re-creating them.') + return + + _download_dataset(dataset_dir) + + # First, process the training data: + with tf.python_io.TFRecordWriter(training_filename) as tfrecord_writer: + data_filename = os.path.join(dataset_dir, _TRAIN_DATA_FILENAME) + labels_filename = os.path.join(dataset_dir, _TRAIN_LABELS_FILENAME) + _add_to_tfrecord(data_filename, labels_filename, 60000, tfrecord_writer) + + # Next, process the testing data: + with tf.python_io.TFRecordWriter(testing_filename) as tfrecord_writer: + data_filename = os.path.join(dataset_dir, _TEST_DATA_FILENAME) + labels_filename = os.path.join(dataset_dir, _TEST_LABELS_FILENAME) + _add_to_tfrecord(data_filename, labels_filename, 10000, tfrecord_writer) + + # Finally, write the labels file: + labels_to_class_names = dict(zip(range(len(_CLASS_NAMES)), _CLASS_NAMES)) + dataset_utils.write_label_file(labels_to_class_names, dataset_dir) + + _clean_up_temporary_files(dataset_dir) + print('\nFinished converting the MNIST dataset!') diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_visualwakewords.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_visualwakewords.py new file mode 100644 index 0000000..dc6aa69 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_visualwakewords.py @@ -0,0 +1,158 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""Downloads and converts VisualWakewords data to TFRecords of TF-Example protos. + +This module downloads the COCO dataset, uncompresses it, derives the +VisualWakeWords dataset to create two TFRecord datasets: one for +train and one for test. Each TFRecord dataset is comprised of a set of +TF-Example protocol buffers, each of which contain a single image and label. + +The script should take several minutes to run. +Please note that this tool creates sharded output files. + +VisualWakeWords dataset is used to design tiny models classifying two classes, +such as person/not-person. The two steps to generate the VisualWakeWords +dataset from the COCO dataset are given below: + +1. Use COCO annotations to create VisualWakeWords annotations: + +Note: A bounding box is 'valid' if it has the foreground_class_of_interest +(e.g. person) and it's area is greater than 0.5% of the image area. + +The resulting annotations file has the following fields, where 'images' are +the same as COCO dataset. 'categories' only contains information about the +foreground_class_of_interest (e.g. person) and 'annotations' maps an image to +objects (a list of valid bounding boxes) and label (value is 1 if it has +atleast one valid bounding box, otherwise 0) + + images[{ + "id", "width", "height", "file_name", "flickr_url", "coco_url", + "license", "date_captured", + }] + + categories{ + "id": {"id", "name", "supercategory"} + } + + annotations{ + "image_id": {"objects":[{"area", "bbox" : [x,y,width,height]}], "label"} + } + +2. Use VisualWakeWords annotations to create TFRecords: + +The resulting TFRecord file contains the following features: +{ image/height, image/width, image/source_id, image/encoded, + image/class/label_text, image/class/label, + image/object/class/text, + image/object/bbox/ymin, image/object/bbox/xmin, image/object/bbox/ymax, + image/object/bbox/xmax, image/object/area + image/filename, image/format, image/key/sha256} +For classification models, you need the image/encoded and image/class/label. + +Example usage: +Run download_and_convert_data.py in the parent directory as follows: + + python download_and_convert_visualwakewords.py --logtostderr \ + --dataset_name=visualwakewords \ + --dataset_dir="${DATASET_DIR}" \ + --small_object_area_threshold=0.005 \ + --foreground_class_of_interest='person' + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import tensorflow as tf +from datasets import download_and_convert_visualwakewords_lib + +tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO) + +tf.compat.v1.app.flags.DEFINE_string( + 'coco_dirname', 'coco_dataset', + 'A subdirectory in visualwakewords dataset directory' + 'containing the coco dataset') + +FLAGS = tf.compat.v1.app.flags.FLAGS + + +def run(dataset_dir, small_object_area_threshold, foreground_class_of_interest): + """Runs the download and conversion operation. + + Args: + dataset_dir: The dataset directory where the dataset is stored. + small_object_area_threshold: Threshold of fraction of image area below which + small objects are filtered + foreground_class_of_interest: Build a binary classifier based on the + presence or absence of this object in the image. + """ + # 1. Download the coco dataset into a subdirectory under the visualwakewords + # dataset directory + coco_dir = os.path.join(dataset_dir, FLAGS.coco_dirname) + + if not tf.gfile.IsDirectory(coco_dir): + tf.gfile.MakeDirs(coco_dir) + + download_and_convert_visualwakewords_lib.download_coco_dataset(coco_dir) + + # Path to COCO annotations + train_annotations_file = os.path.join(coco_dir, 'annotations', + 'instances_train2014.json') + val_annotations_file = os.path.join(coco_dir, 'annotations', + 'instances_val2014.json') + train_image_dir = os.path.join(coco_dir, 'train2014') + val_image_dir = os.path.join(coco_dir, 'val2014') + + # Path to VisualWakeWords annotations + visualwakewords_annotations_train = os.path.join( + dataset_dir, 'instances_visualwakewords_train2014.json') + visualwakewords_annotations_val = os.path.join( + dataset_dir, 'instances_visualwakewords_val2014.json') + visualwakewords_labels_filename = os.path.join(dataset_dir, 'labels.txt') + train_output_path = os.path.join(dataset_dir, 'train.record') + val_output_path = os.path.join(dataset_dir, 'val.record') + + # 2. Create a labels file + tf.logging.info('Creating a labels file...') + download_and_convert_visualwakewords_lib.create_labels_file( + foreground_class_of_interest, visualwakewords_labels_filename) + + # 3. Use COCO annotations to create VisualWakeWords annotations + tf.logging.info('Creating train VisualWakeWords annotations...') + download_and_convert_visualwakewords_lib.create_visual_wakeword_annotations( + train_annotations_file, visualwakewords_annotations_train, + small_object_area_threshold, foreground_class_of_interest) + tf.logging.info('Creating validation VisualWakeWords annotations...') + download_and_convert_visualwakewords_lib.create_visual_wakeword_annotations( + val_annotations_file, visualwakewords_annotations_val, + small_object_area_threshold, foreground_class_of_interest) + + # 4. Use VisualWakeWords annotations to create the TFRecords + tf.logging.info('Creating train TFRecords for VisualWakeWords dataset...') + download_and_convert_visualwakewords_lib.create_tf_record_for_visualwakewords_dataset( + visualwakewords_annotations_train, + train_image_dir, + train_output_path, + num_shards=100) + + tf.logging.info( + 'Creating validation TFRecords for VisualWakeWords dataset...') + download_and_convert_visualwakewords_lib.create_tf_record_for_visualwakewords_dataset( + visualwakewords_annotations_val, + val_image_dir, + val_output_path, + num_shards=10) diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_visualwakewords_lib.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_visualwakewords_lib.py new file mode 100644 index 0000000..4c3d200 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_and_convert_visualwakewords_lib.py @@ -0,0 +1,286 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""Helper functions to generate the Visual WakeWords dataset. + + It filters raw COCO annotations file to Visual WakeWords Dataset + annotations. The resulting annotations and COCO images are then converted + to TF records. + See download_and_convert_visualwakewords.py for the sample usage. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import hashlib +import io +import json +import os +import contextlib2 + +import PIL.Image + +import tensorflow as tf + +from datasets import dataset_utils + +tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO) + +tf.compat.v1.app.flags.DEFINE_string( + 'coco_train_url', + 'http://images.cocodataset.org/zips/train2014.zip', + 'Link to zip file containing coco training data') +tf.compat.v1.app.flags.DEFINE_string( + 'coco_validation_url', + 'http://images.cocodataset.org/zips/val2014.zip', + 'Link to zip file containing coco validation data') +tf.compat.v1.app.flags.DEFINE_string( + 'coco_annotations_url', + 'http://images.cocodataset.org/annotations/annotations_trainval2014.zip', + 'Link to zip file containing coco annotation data') + +FLAGS = tf.compat.v1.app.flags.FLAGS + + +def download_coco_dataset(dataset_dir): + """Download the coco dataset. + + Args: + dataset_dir: Path where coco dataset should be downloaded. + """ + dataset_utils.download_and_uncompress_zipfile(FLAGS.coco_train_url, + dataset_dir) + dataset_utils.download_and_uncompress_zipfile(FLAGS.coco_validation_url, + dataset_dir) + dataset_utils.download_and_uncompress_zipfile(FLAGS.coco_annotations_url, + dataset_dir) + + +def create_labels_file(foreground_class_of_interest, + visualwakewords_labels_file): + """Generate visualwakewords labels file. + + Args: + foreground_class_of_interest: category from COCO dataset that is filtered by + the visualwakewords dataset + visualwakewords_labels_file: output visualwakewords label file + """ + labels_to_class_names = {0: 'background', 1: foreground_class_of_interest} + with open(visualwakewords_labels_file, 'w') as fp: + for label in labels_to_class_names: + fp.write(str(label) + ':' + str(labels_to_class_names[label]) + '\n') + + +def create_visual_wakeword_annotations(annotations_file, + visualwakewords_annotations_file, + small_object_area_threshold, + foreground_class_of_interest): + """Generate visual wakewords annotations file. + + Loads COCO annotation json files to generate visualwakewords annotations file. + + Args: + annotations_file: JSON file containing COCO bounding box annotations + visualwakewords_annotations_file: path to output annotations file + small_object_area_threshold: threshold on fraction of image area below which + small object bounding boxes are filtered + foreground_class_of_interest: category from COCO dataset that is filtered by + the visual wakewords dataset + """ + # default object of interest is person + foreground_class_of_interest_id = 1 + with tf.gfile.GFile(annotations_file, 'r') as fid: + groundtruth_data = json.load(fid) + images = groundtruth_data['images'] + # Create category index + category_index = {} + for category in groundtruth_data['categories']: + if category['name'] == foreground_class_of_interest: + foreground_class_of_interest_id = category['id'] + category_index[category['id']] = category + # Create annotations index, a map of image_id to it's annotations + tf.logging.info('Building annotations index...') + annotations_index = collections.defaultdict( + lambda: collections.defaultdict(list)) + # structure is { "image_id": {"objects" : [list of the image annotations]}} + for annotation in groundtruth_data['annotations']: + annotations_index[annotation['image_id']]['objects'].append(annotation) + missing_annotation_count = len(images) - len(annotations_index) + tf.logging.info('%d images are missing annotations.', + missing_annotation_count) + # Create filtered annotations index + annotations_index_filtered = {} + for idx, image in enumerate(images): + if idx % 100 == 0: + tf.logging.info('On image %d of %d', idx, len(images)) + annotations = annotations_index[image['id']] + annotations_filtered = _filter_annotations( + annotations, image, small_object_area_threshold, + foreground_class_of_interest_id) + annotations_index_filtered[image['id']] = annotations_filtered + + with open(visualwakewords_annotations_file, 'w') as fp: + json.dump( + { + 'images': images, + 'annotations': annotations_index_filtered, + 'categories': category_index + }, fp) + + +def _filter_annotations(annotations, image, small_object_area_threshold, + foreground_class_of_interest_id): + """Filters COCO annotations to visual wakewords annotations. + + Args: + annotations: dicts with keys: { + u'objects': [{u'id', u'image_id', u'category_id', u'segmentation', + u'area', u'bbox' : [x,y,width,height], u'iscrowd'}] } Notice + that bounding box coordinates in the official COCO dataset + are given as [x, y, width, height] tuples using absolute + coordinates where x, y represent the top-left (0-indexed) + corner. + image: dict with keys: [u'license', u'file_name', u'coco_url', u'height', + u'width', u'date_captured', u'flickr_url', u'id'] + small_object_area_threshold: threshold on fraction of image area below which + small objects are filtered + foreground_class_of_interest_id: category of COCO dataset which visual + wakewords filters + + Returns: + annotations_filtered: dict with keys: { + u'objects': [{"area", "bbox" : [x,y,width,height]}], + u'label', + } + """ + objects = [] + image_area = image['height'] * image['width'] + for annotation in annotations['objects']: + normalized_object_area = annotation['area'] / image_area + category_id = int(annotation['category_id']) + # Filter valid bounding boxes + if category_id == foreground_class_of_interest_id and \ + normalized_object_area > small_object_area_threshold: + objects.append({ + u'area': annotation['area'], + u'bbox': annotation['bbox'], + }) + label = 1 if objects else 0 + return { + 'objects': objects, + 'label': label, + } + + +def create_tf_record_for_visualwakewords_dataset(annotations_file, image_dir, + output_path, num_shards): + """Loads Visual WakeWords annotations/images and converts to tf.Record format. + + Args: + annotations_file: JSON file containing bounding box annotations. + image_dir: Directory containing the image files. + output_path: Path to output tf.Record file. + num_shards: number of output file shards. + """ + with contextlib2.ExitStack() as tf_record_close_stack, \ + tf.gfile.GFile(annotations_file, 'r') as fid: + output_tfrecords = dataset_utils.open_sharded_output_tfrecords( + tf_record_close_stack, output_path, num_shards) + groundtruth_data = json.load(fid) + images = groundtruth_data['images'] + annotations_index = groundtruth_data['annotations'] + annotations_index = {int(k): v for k, v in annotations_index.iteritems()} + # convert 'unicode' key to 'int' key after we parse the json file + + for idx, image in enumerate(images): + if idx % 100 == 0: + tf.logging.info('On image %d of %d', idx, len(images)) + annotations = annotations_index[image['id']] + tf_example = _create_tf_example(image, annotations, image_dir) + shard_idx = idx % num_shards + output_tfrecords[shard_idx].write(tf_example.SerializeToString()) + + +def _create_tf_example(image, annotations, image_dir): + """Converts image and annotations to a tf.Example proto. + + Args: + image: dict with keys: [u'license', u'file_name', u'coco_url', u'height', + u'width', u'date_captured', u'flickr_url', u'id'] + annotations: dict with objects (a list of image annotations) and a label. + {u'objects':[{"area", "bbox" : [x,y,width,height}], u'label'}. Notice + that bounding box coordinates in the COCO dataset are given as[x, y, + width, height] tuples using absolute coordinates where x, y represent + the top-left (0-indexed) corner. This function also converts to the format + that can be used by the Tensorflow Object Detection API (which is [ymin, + xmin, ymax, xmax] with coordinates normalized relative to image size). + image_dir: directory containing the image files. + Returns: + tf_example: The converted tf.Example + + Raises: + ValueError: if the image pointed to by data['filename'] is not a valid JPEG + """ + image_height = image['height'] + image_width = image['width'] + filename = image['file_name'] + image_id = image['id'] + + full_path = os.path.join(image_dir, filename) + with tf.gfile.GFile(full_path, 'rb') as fid: + encoded_jpg = fid.read() + encoded_jpg_io = io.BytesIO(encoded_jpg) + image = PIL.Image.open(encoded_jpg_io) + key = hashlib.sha256(encoded_jpg).hexdigest() + + xmin, xmax, ymin, ymax, area = [], [], [], [], [] + for obj in annotations['objects']: + (x, y, width, height) = tuple(obj['bbox']) + xmin.append(float(x) / image_width) + xmax.append(float(x + width) / image_width) + ymin.append(float(y) / image_height) + ymax.append(float(y + height) / image_height) + area.append(obj['area']) + + feature_dict = { + 'image/height': + dataset_utils.int64_feature(image_height), + 'image/width': + dataset_utils.int64_feature(image_width), + 'image/filename': + dataset_utils.bytes_feature(filename.encode('utf8')), + 'image/source_id': + dataset_utils.bytes_feature(str(image_id).encode('utf8')), + 'image/key/sha256': + dataset_utils.bytes_feature(key.encode('utf8')), + 'image/encoded': + dataset_utils.bytes_feature(encoded_jpg), + 'image/format': + dataset_utils.bytes_feature('jpeg'.encode('utf8')), + 'image/class/label': + dataset_utils.int64_feature(annotations['label']), + 'image/object/bbox/xmin': + dataset_utils.float_list_feature(xmin), + 'image/object/bbox/xmax': + dataset_utils.float_list_feature(xmax), + 'image/object/bbox/ymin': + dataset_utils.float_list_feature(ymin), + 'image/object/bbox/ymax': + dataset_utils.float_list_feature(ymax), + 'image/object/area': + dataset_utils.float_list_feature(area), + } + example = tf.train.Example(features=tf.train.Features(feature=feature_dict)) + return example diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_imagenet.sh b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_imagenet.sh new file mode 100644 index 0000000..c780e17 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/download_imagenet.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# Script to download ImageNet Challenge 2012 training and validation data set. +# +# Downloads and decompresses raw images and bounding boxes. +# +# **IMPORTANT** +# To download the raw images, the user must create an account with image-net.org +# and generate a username and access_key. The latter two are required for +# downloading the raw images. +# +# usage: +# ./download_imagenet.sh [dirname] +set -e + +if [ "x$IMAGENET_ACCESS_KEY" == x -o "x$IMAGENET_USERNAME" == x ]; then + cat < ') + sys.exit(-1) + data_dir = sys.argv[1] + validation_labels_file = sys.argv[2] + + # Read in the 50000 synsets associated with the validation data set. + labels = [l.strip() for l in open(validation_labels_file).readlines()] + unique_labels = set(labels) + + # Make all sub-directories in the validation data dir. + for label in unique_labels: + labeled_data_dir = os.path.join(data_dir, label) + os.makedirs(labeled_data_dir) + + # Move all of the image to the appropriate sub-directory. + for i in xrange(len(labels)): + basename = 'ILSVRC2012_val_000%.5d.JPEG' % (i + 1) + original_filename = os.path.join(data_dir, basename) + if not os.path.exists(original_filename): + print('Failed to find: ', original_filename) + sys.exit(-1) + new_filename = os.path.join(data_dir, labels[i], basename) + os.rename(original_filename, new_filename) diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/process_bounding_boxes.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/process_bounding_boxes.py new file mode 100644 index 0000000..78c899e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/process_bounding_boxes.py @@ -0,0 +1,253 @@ +#!/usr/bin/python +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Process the ImageNet Challenge bounding boxes for TensorFlow model training. + +This script is called as + +process_bounding_boxes.py

[synsets-file] + +Where is a directory containing the downloaded and unpacked bounding box +data. If [synsets-file] is supplied, then only the bounding boxes whose +synstes are contained within this file are returned. Note that the +[synsets-file] file contains synset ids, one per line. + +The script dumps out a CSV text file in which each line contains an entry. + n00007846_64193.JPEG,0.0060,0.2620,0.7545,0.9940 + +The entry can be read as: + , , , , + +The bounding box for contains two points (xmin, ymin) and +(xmax, ymax) specifying the lower-left corner and upper-right corner of a +bounding box in *relative* coordinates. + +The user supplies a directory where the XML files reside. The directory +structure in the directory is assumed to look like this: + +/nXXXXXXXX/nXXXXXXXX_YYYY.xml + +Each XML file contains a bounding box annotation. The script: + + (1) Parses the XML file and extracts the filename, label and bounding box info. + + (2) The bounding box is specified in the XML files as integer (xmin, ymin) and + (xmax, ymax) *relative* to image size displayed to the human annotator. The + size of the image displayed to the human annotator is stored in the XML file + as integer (height, width). + + Note that the displayed size will differ from the actual size of the image + downloaded from image-net.org. To make the bounding box annotation useable, + we convert bounding box to floating point numbers relative to displayed + height and width of the image. + + Note that each XML file might contain N bounding box annotations. + + Note that the points are all clamped at a range of [0.0, 1.0] because some + human annotations extend outside the range of the supplied image. + + See details here: http://image-net.org/download-bboxes + +(3) By default, the script outputs all valid bounding boxes. If a + [synsets-file] is supplied, only the subset of bounding boxes associated + with those synsets are outputted. Importantly, one can supply a list of + synsets in the ImageNet Challenge and output the list of bounding boxes + associated with the training images of the ILSVRC. + + We use these bounding boxes to inform the random distortion of images + supplied to the network. + +If you run this script successfully, you will see the following output +to stderr: +> Finished processing 544546 XML files. +> Skipped 0 XML files not in ImageNet Challenge. +> Skipped 0 bounding boxes not in ImageNet Challenge. +> Wrote 615299 bounding boxes from 544546 annotated images. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import glob +import os.path +import sys +import xml.etree.ElementTree as ET +from six.moves import xrange # pylint: disable=redefined-builtin + + +class BoundingBox(object): + pass + + +def GetItem(name, root, index=0): + count = 0 + for item in root.iter(name): + if count == index: + return item.text + count += 1 + # Failed to find "index" occurrence of item. + return -1 + + +def GetInt(name, root, index=0): + return int(GetItem(name, root, index)) + + +def FindNumberBoundingBoxes(root): + index = 0 + while True: + if GetInt('xmin', root, index) == -1: + break + index += 1 + return index + + +def ProcessXMLAnnotation(xml_file): + """Process a single XML file containing a bounding box.""" + # pylint: disable=broad-except + try: + tree = ET.parse(xml_file) + except Exception: + print('Failed to parse: ' + xml_file, file=sys.stderr) + return None + # pylint: enable=broad-except + root = tree.getroot() + + num_boxes = FindNumberBoundingBoxes(root) + boxes = [] + + for index in xrange(num_boxes): + box = BoundingBox() + # Grab the 'index' annotation. + box.xmin = GetInt('xmin', root, index) + box.ymin = GetInt('ymin', root, index) + box.xmax = GetInt('xmax', root, index) + box.ymax = GetInt('ymax', root, index) + + box.width = GetInt('width', root) + box.height = GetInt('height', root) + box.filename = GetItem('filename', root) + '.JPEG' + box.label = GetItem('name', root) + + xmin = float(box.xmin) / float(box.width) + xmax = float(box.xmax) / float(box.width) + ymin = float(box.ymin) / float(box.height) + ymax = float(box.ymax) / float(box.height) + + # Some images contain bounding box annotations that + # extend outside of the supplied image. See, e.g. + # n03127925/n03127925_147.xml + # Additionally, for some bounding boxes, the min > max + # or the box is entirely outside of the image. + min_x = min(xmin, xmax) + max_x = max(xmin, xmax) + box.xmin_scaled = min(max(min_x, 0.0), 1.0) + box.xmax_scaled = min(max(max_x, 0.0), 1.0) + + min_y = min(ymin, ymax) + max_y = max(ymin, ymax) + box.ymin_scaled = min(max(min_y, 0.0), 1.0) + box.ymax_scaled = min(max(max_y, 0.0), 1.0) + + boxes.append(box) + + return boxes + +if __name__ == '__main__': + if len(sys.argv) < 2 or len(sys.argv) > 3: + print('Invalid usage\n' + 'usage: process_bounding_boxes.py [synsets-file]', + file=sys.stderr) + sys.exit(-1) + + xml_files = glob.glob(sys.argv[1] + '/*/*.xml') + print('Identified %d XML files in %s' % (len(xml_files), sys.argv[1]), + file=sys.stderr) + + if len(sys.argv) == 3: + labels = set([l.strip() for l in open(sys.argv[2]).readlines()]) + print('Identified %d synset IDs in %s' % (len(labels), sys.argv[2]), + file=sys.stderr) + else: + labels = None + + skipped_boxes = 0 + skipped_files = 0 + saved_boxes = 0 + saved_files = 0 + for file_index, one_file in enumerate(xml_files): + # Example: <...>/n06470073/n00141669_6790.xml + label = os.path.basename(os.path.dirname(one_file)) + + # Determine if the annotation is from an ImageNet Challenge label. + if labels is not None and label not in labels: + skipped_files += 1 + continue + + bboxes = ProcessXMLAnnotation(one_file) + assert bboxes is not None, 'No bounding boxes found in ' + one_file + + found_box = False + for bbox in bboxes: + if labels is not None: + if bbox.label != label: + # Note: There is a slight bug in the bounding box annotation data. + # Many of the dog labels have the human label 'Scottish_deerhound' + # instead of the synset ID 'n02092002' in the bbox.label field. As a + # simple hack to overcome this issue, we only exclude bbox labels + # *which are synset ID's* that do not match original synset label for + # the XML file. + if bbox.label in labels: + skipped_boxes += 1 + continue + + # Guard against improperly specified boxes. + if (bbox.xmin_scaled >= bbox.xmax_scaled or + bbox.ymin_scaled >= bbox.ymax_scaled): + skipped_boxes += 1 + continue + + # Note bbox.filename occasionally contains '%s' in the name. This is + # data set noise that is fixed by just using the basename of the XML file. + image_filename = os.path.splitext(os.path.basename(one_file))[0] + print('%s.JPEG,%.4f,%.4f,%.4f,%.4f' % + (image_filename, + bbox.xmin_scaled, bbox.ymin_scaled, + bbox.xmax_scaled, bbox.ymax_scaled)) + + saved_boxes += 1 + found_box = True + if found_box: + saved_files += 1 + else: + skipped_files += 1 + + if not file_index % 5000: + print('--> processed %d of %d XML files.' % + (file_index + 1, len(xml_files)), + file=sys.stderr) + print('--> skipped %d boxes and %d XML files.' % + (skipped_boxes, skipped_files), file=sys.stderr) + + print('Finished processing %d XML files.' % len(xml_files), file=sys.stderr) + print('Skipped %d XML files not in ImageNet Challenge.' % skipped_files, + file=sys.stderr) + print('Skipped %d bounding boxes not in ImageNet Challenge.' % skipped_boxes, + file=sys.stderr) + print('Wrote %d bounding boxes from %d annotated images.' % + (saved_boxes, saved_files), + file=sys.stderr) + print('Finished.', file=sys.stderr) diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/visualwakewords.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/visualwakewords.py new file mode 100644 index 0000000..41d4e67 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/datasets/visualwakewords.py @@ -0,0 +1,129 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Provides data for Visual WakeWords Dataset with images+labels. + +Visual WakeWords Dataset derives from the COCO dataset to design tiny models +classifying two classes, such as person/not-person. The COCO annotations +are filtered to two classes: person and not-person (or another user-defined +category). Bounding boxes for small objects with area less than 5% of the image +area are filtered out. +See build_visualwakewords_data.py which generates the Visual WakeWords dataset +annotations from the raw COCO dataset and converts them to TFRecord. + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from datasets import dataset_utils + + +slim = contrib_slim + +_FILE_PATTERN = '%s.record-*' + +_SPLITS_TO_SIZES = { + 'train': 82783, + 'val': 40504, +} + + +_ITEMS_TO_DESCRIPTIONS = { + 'image': 'A color image of varying height and width.', + 'label': 'The label id of the image, an integer in {0, 1}', + 'object/bbox': 'A list of bounding boxes.', +} + +_NUM_CLASSES = 2 + +# labels file +LABELS_FILENAME = 'labels.txt' + + +def get_split(split_name, dataset_dir, file_pattern=None, reader=None): + """Gets a dataset tuple with instructions for reading ImageNet. + + Args: + split_name: A train/test split name. + dataset_dir: The base directory of the dataset sources. + file_pattern: The file pattern to use when matching the dataset sources. It + is assumed that the pattern contains a '%s' string so that the split name + can be inserted. + reader: The TensorFlow reader type. + + Returns: + A `Dataset` namedtuple. + + Raises: + ValueError: if `split_name` is not a valid train/test split. + """ + if split_name not in _SPLITS_TO_SIZES: + raise ValueError('split name %s was not recognized.' % split_name) + + if not file_pattern: + file_pattern = _FILE_PATTERN + file_pattern = os.path.join(dataset_dir, file_pattern % split_name) + + # Allowing None in the signature so that dataset_factory can use the default. + if reader is None: + reader = tf.TFRecordReader + + keys_to_features = { + 'image/encoded': + tf.FixedLenFeature((), tf.string, default_value=''), + 'image/format': + tf.FixedLenFeature((), tf.string, default_value='jpeg'), + 'image/class/label': + tf.FixedLenFeature([], dtype=tf.int64, default_value=-1), + 'image/object/bbox/xmin': + tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymin': + tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/xmax': + tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymax': + tf.VarLenFeature(dtype=tf.float32), + } + + items_to_handlers = { + 'image': + slim.tfexample_decoder.Image('image/encoded', 'image/format'), + 'label': + slim.tfexample_decoder.Tensor('image/class/label'), + 'object/bbox': + slim.tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'], + 'image/object/bbox/'), + } + + decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, + items_to_handlers) + + labels_to_names = None + labels_file = os.path.join(dataset_dir, LABELS_FILENAME) + if tf.gfile.Exists(labels_file): + labels_to_names = dataset_utils.read_label_file(dataset_dir) + + return slim.dataset.Dataset( + data_sources=file_pattern, + reader=reader, + decoder=decoder, + num_samples=_SPLITS_TO_SIZES[split_name], + items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, + num_classes=_NUM_CLASSES, + labels_to_names=labels_to_names) diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/deployment/__init__.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/deployment/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/deployment/__init__.py @@ -0,0 +1 @@ + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/deployment/model_deploy.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/deployment/model_deploy.py new file mode 100644 index 0000000..3d8d5f8 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/deployment/model_deploy.py @@ -0,0 +1,677 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Deploy Slim models across multiple clones and replicas. + +# TODO(sguada) docstring paragraph by (a) motivating the need for the file and +# (b) defining clones. + +# TODO(sguada) describe the high-level components of model deployment. +# E.g. "each model deployment is composed of several parts: a DeploymentConfig, +# which captures A, B and C, an input_fn which loads data.. etc + +To easily train a model on multiple GPUs or across multiple machines this +module provides a set of helper functions: `create_clones`, +`optimize_clones` and `deploy`. + +Usage: + + g = tf.Graph() + + # Set up DeploymentConfig + config = model_deploy.DeploymentConfig(num_clones=2, clone_on_cpu=True) + + # Create the global step on the device storing the variables. + with tf.device(config.variables_device()): + global_step = slim.create_global_step() + + # Define the inputs + with tf.device(config.inputs_device()): + images, labels = LoadData(...) + inputs_queue = slim.data.prefetch_queue((images, labels)) + + # Define the optimizer. + with tf.device(config.optimizer_device()): + optimizer = tf.train.MomentumOptimizer(FLAGS.learning_rate, FLAGS.momentum) + + # Define the model including the loss. + def model_fn(inputs_queue): + images, labels = inputs_queue.dequeue() + predictions = CreateNetwork(images) + slim.losses.log_loss(predictions, labels) + + model_dp = model_deploy.deploy(config, model_fn, [inputs_queue], + optimizer=optimizer) + + # Run training. + slim.learning.train(model_dp.train_op, my_log_dir, + summary_op=model_dp.summary_op) + +The Clone namedtuple holds together the values associated with each call to +model_fn: + * outputs: The return values of the calls to `model_fn()`. + * scope: The scope used to create the clone. + * device: The device used to create the clone. + +DeployedModel namedtuple, holds together the values needed to train multiple +clones: + * train_op: An operation that run the optimizer training op and include + all the update ops created by `model_fn`. Present only if an optimizer + was specified. + * summary_op: An operation that run the summaries created by `model_fn` + and process_gradients. + * total_loss: A `Tensor` that contains the sum of all losses created by + `model_fn` plus the regularization losses. + * clones: List of `Clone` tuples returned by `create_clones()`. + +DeploymentConfig parameters: + * num_clones: Number of model clones to deploy in each replica. + * clone_on_cpu: True if clones should be placed on CPU. + * replica_id: Integer. Index of the replica for which the model is + deployed. Usually 0 for the chief replica. + * num_replicas: Number of replicas to use. + * num_ps_tasks: Number of tasks for the `ps` job. 0 to not use replicas. + * worker_job_name: A name for the worker job. + * ps_job_name: A name for the parameter server job. + +TODO(sguada): + - describe side effect to the graph. + - what happens to summaries and update_ops. + - which graph collections are altered. + - write a tutorial on how to use this. + - analyze the possibility of calling deploy more than once. + + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +slim = contrib_slim + +__all__ = ['create_clones', + 'deploy', + 'optimize_clones', + 'DeployedModel', + 'DeploymentConfig', + 'Clone', + ] + +# Namedtuple used to represent a clone during deployment. +Clone = collections.namedtuple('Clone', + ['outputs', # Whatever model_fn() returned. + 'scope', # The scope used to create it. + 'device', # The device used to create. + ]) + +# Namedtuple used to represent a DeployedModel, returned by deploy(). +DeployedModel = collections.namedtuple('DeployedModel', + ['train_op', # The `train_op` + 'summary_op', # The `summary_op` + 'total_loss', # The loss `Tensor` + 'clones', # A list of `Clones` tuples. + ]) + +# Default parameters for DeploymentConfig +_deployment_params = {'num_clones': 1, + 'clone_on_cpu': False, + 'replica_id': 0, + 'num_replicas': 1, + 'num_ps_tasks': 0, + 'worker_job_name': 'worker', + 'ps_job_name': 'ps'} + + +def create_clones(config, model_fn, args=None, kwargs=None): + """Creates multiple clones according to config using a `model_fn`. + + The returned values of `model_fn(*args, **kwargs)` are collected along with + the scope and device used to created it in a namedtuple + `Clone(outputs, scope, device)` + + Note: it is assumed that any loss created by `model_fn` is collected at + the tf.GraphKeys.LOSSES collection. + + To recover the losses, summaries or update_ops created by the clone use: + ```python + losses = tf.get_collection(tf.GraphKeys.LOSSES, clone.scope) + summaries = tf.get_collection(tf.GraphKeys.SUMMARIES, clone.scope) + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, clone.scope) + ``` + + The deployment options are specified by the config object and support + deploying one or several clones on different GPUs and one or several replicas + of such clones. + + The argument `model_fn` is called `config.num_clones` times to create the + model clones as `model_fn(*args, **kwargs)`. + + If `config` specifies deployment on multiple replicas then the default + tensorflow device is set appropriatly for each call to `model_fn` and for the + slim variable creation functions: model and global variables will be created + on the `ps` device, the clone operations will be on the `worker` device. + + Args: + config: A DeploymentConfig object. + model_fn: A callable. Called as `model_fn(*args, **kwargs)` + args: Optional list of arguments to pass to `model_fn`. + kwargs: Optional list of keyword arguments to pass to `model_fn`. + + Returns: + A list of namedtuples `Clone`. + """ + clones = [] + args = args or [] + kwargs = kwargs or {} + with slim.arg_scope([slim.model_variable, slim.variable], + device=config.variables_device()): + # Create clones. + for i in range(0, config.num_clones): + with tf.name_scope(config.clone_scope(i)) as clone_scope: + clone_device = config.clone_device(i) + with tf.device(clone_device): + with tf.variable_scope(tf.get_variable_scope(), + reuse=True if i > 0 else None): + outputs = model_fn(*args, **kwargs) + clones.append(Clone(outputs, clone_scope, clone_device)) + return clones + + +def _gather_clone_loss(clone, num_clones, regularization_losses): + """Gather the loss for a single clone. + + Args: + clone: A Clone namedtuple. + num_clones: The number of clones being deployed. + regularization_losses: Possibly empty list of regularization_losses + to add to the clone losses. + + Returns: + A tensor for the total loss for the clone. Can be None. + """ + # The return value. + sum_loss = None + # Individual components of the loss that will need summaries. + clone_loss = None + regularization_loss = None + # Compute and aggregate losses on the clone device. + with tf.device(clone.device): + all_losses = [] + clone_losses = tf.get_collection(tf.GraphKeys.LOSSES, clone.scope) + if clone_losses: + clone_loss = tf.add_n(clone_losses, name='clone_loss') + if num_clones > 1: + clone_loss = tf.div(clone_loss, 1.0 * num_clones, + name='scaled_clone_loss') + all_losses.append(clone_loss) + if regularization_losses: + regularization_loss = tf.add_n(regularization_losses, + name='regularization_loss') + all_losses.append(regularization_loss) + if all_losses: + sum_loss = tf.add_n(all_losses) + # Add the summaries out of the clone device block. + if clone_loss is not None: + tf.summary.scalar('/'.join(filter(None, + ['Losses', clone.scope, 'clone_loss'])), + clone_loss) + if regularization_loss is not None: + tf.summary.scalar('Losses/regularization_loss', regularization_loss) + return sum_loss + + +def _optimize_clone(optimizer, clone, num_clones, regularization_losses, + **kwargs): + """Compute losses and gradients for a single clone. + + Args: + optimizer: A tf.Optimizer object. + clone: A Clone namedtuple. + num_clones: The number of clones being deployed. + regularization_losses: Possibly empty list of regularization_losses + to add to the clone losses. + **kwargs: Dict of kwarg to pass to compute_gradients(). + + Returns: + A tuple (clone_loss, clone_grads_and_vars). + - clone_loss: A tensor for the total loss for the clone. Can be None. + - clone_grads_and_vars: List of (gradient, variable) for the clone. + Can be empty. + """ + sum_loss = _gather_clone_loss(clone, num_clones, regularization_losses) + clone_grad = None + if sum_loss is not None: + # with tf.device(clone.device): + # clone_grad = optimizer.compute_gradients(sum_loss, **kwargs) + clone_grad = optimizer.compute_gradients(sum_loss, **kwargs) + return sum_loss, clone_grad + + +def optimize_clones(clones, optimizer, + regularization_losses=None, + **kwargs): + """Compute clone losses and gradients for the given list of `Clones`. + + Note: The regularization_losses are added to the first clone losses. + + Args: + clones: List of `Clones` created by `create_clones()`. + optimizer: An `Optimizer` object. + regularization_losses: Optional list of regularization losses. If None it + will gather them from tf.GraphKeys.REGULARIZATION_LOSSES. Pass `[]` to + exclude them. + **kwargs: Optional list of keyword arguments to pass to `compute_gradients`. + + Returns: + A tuple (total_loss, grads_and_vars). + - total_loss: A Tensor containing the average of the clone losses including + the regularization loss. + - grads_and_vars: A List of tuples (gradient, variable) containing the sum + of the gradients for each variable. + + """ + grads_and_vars = [] + clones_losses = [] + num_clones = len(clones) + if regularization_losses is None: + regularization_losses = tf.get_collection( + tf.GraphKeys.REGULARIZATION_LOSSES) + for clone in clones: + with tf.name_scope(clone.scope): + clone_loss, clone_grad = _optimize_clone( + optimizer, clone, num_clones, regularization_losses, **kwargs) + if clone_loss is not None: + clones_losses.append(clone_loss) + grads_and_vars.append(clone_grad) + # Only use regularization_losses for the first clone + regularization_losses = None + # Compute the total_loss summing all the clones_losses. + total_loss = tf.add_n(clones_losses, name='total_loss') + # Sum the gradients across clones. + grads_and_vars = _sum_clones_gradients(grads_and_vars) + return total_loss, grads_and_vars + + +def deploy(config, + model_fn, + args=None, + kwargs=None, + optimizer=None, + summarize_gradients=False): + """Deploys a Slim-constructed model across multiple clones. + + The deployment options are specified by the config object and support + deploying one or several clones on different GPUs and one or several replicas + of such clones. + + The argument `model_fn` is called `config.num_clones` times to create the + model clones as `model_fn(*args, **kwargs)`. + + The optional argument `optimizer` is an `Optimizer` object. If not `None`, + the deployed model is configured for training with that optimizer. + + If `config` specifies deployment on multiple replicas then the default + tensorflow device is set appropriatly for each call to `model_fn` and for the + slim variable creation functions: model and global variables will be created + on the `ps` device, the clone operations will be on the `worker` device. + + Args: + config: A `DeploymentConfig` object. + model_fn: A callable. Called as `model_fn(*args, **kwargs)` + args: Optional list of arguments to pass to `model_fn`. + kwargs: Optional list of keyword arguments to pass to `model_fn`. + optimizer: Optional `Optimizer` object. If passed the model is deployed + for training with that optimizer. + summarize_gradients: Whether or not add summaries to the gradients. + + Returns: + A `DeployedModel` namedtuple. + + """ + # Gather initial summaries. + summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES)) + + # Create Clones. + clones = create_clones(config, model_fn, args, kwargs) + first_clone = clones[0] + + # Gather update_ops from the first clone. These contain, for example, + # the updates for the batch_norm variables created by model_fn. + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, first_clone.scope) + + train_op = None + total_loss = None + with tf.device(config.optimizer_device()): + if optimizer: + # Place the global step on the device storing the variables. + with tf.device(config.variables_device()): + global_step = slim.get_or_create_global_step() + + # Compute the gradients for the clones. + total_loss, clones_gradients = optimize_clones(clones, optimizer) + + if clones_gradients: + if summarize_gradients: + # Add summaries to the gradients. + summaries |= set(_add_gradients_summaries(clones_gradients)) + + # Create gradient updates. + grad_updates = optimizer.apply_gradients(clones_gradients, + global_step=global_step) + update_ops.append(grad_updates) + + update_op = tf.group(*update_ops) + with tf.control_dependencies([update_op]): + train_op = tf.identity(total_loss, name='train_op') + else: + clones_losses = [] + regularization_losses = tf.get_collection( + tf.GraphKeys.REGULARIZATION_LOSSES) + for clone in clones: + with tf.name_scope(clone.scope): + clone_loss = _gather_clone_loss(clone, len(clones), + regularization_losses) + if clone_loss is not None: + clones_losses.append(clone_loss) + # Only use regularization_losses for the first clone + regularization_losses = None + if clones_losses: + total_loss = tf.add_n(clones_losses, name='total_loss') + + # Add the summaries from the first clone. These contain the summaries + # created by model_fn and either optimize_clones() or _gather_clone_loss(). + summaries |= set(tf.get_collection(tf.GraphKeys.SUMMARIES, + first_clone.scope)) + + if total_loss is not None: + # Add total_loss to summary. + summaries.add(tf.summary.scalar('total_loss', total_loss)) + + if summaries: + # Merge all summaries together. + summary_op = tf.summary.merge(list(summaries), name='summary_op') + else: + summary_op = None + + return DeployedModel(train_op, summary_op, total_loss, clones) + + +def _sum_clones_gradients(clone_grads): + """Calculate the sum gradient for each shared variable across all clones. + + This function assumes that the clone_grads has been scaled appropriately by + 1 / num_clones. + + Args: + clone_grads: A List of List of tuples (gradient, variable), one list per + `Clone`. + + Returns: + List of tuples of (gradient, variable) where the gradient has been summed + across all clones. + """ + sum_grads = [] + for grad_and_vars in zip(*clone_grads): + # Note that each grad_and_vars looks like the following: + # ((grad_var0_clone0, var0), ... (grad_varN_cloneN, varN)) + grads = [] + var = grad_and_vars[0][1] + for g, v in grad_and_vars: + assert v == var + if g is not None: + grads.append(g) + if grads: + if len(grads) > 1: + sum_grad = tf.add_n(grads, name=var.op.name + '/sum_grads') + else: + sum_grad = grads[0] + sum_grads.append((sum_grad, var)) + return sum_grads + + +def _add_gradients_summaries(grads_and_vars): + """Add histogram summaries to gradients. + + Note: The summaries are also added to the SUMMARIES collection. + + Args: + grads_and_vars: A list of gradient to variable pairs (tuples). + + Returns: + The _list_ of the added summaries for grads_and_vars. + """ + summaries = [] + for grad, var in grads_and_vars: + if grad is not None: + if isinstance(grad, tf.IndexedSlices): + grad_values = grad.values + else: + grad_values = grad + summaries.append(tf.summary.histogram(var.op.name + ':gradient', + grad_values)) + summaries.append(tf.summary.histogram(var.op.name + ':gradient_norm', + tf.global_norm([grad_values]))) + else: + tf.logging.info('Var %s has no gradient', var.op.name) + return summaries + + +class DeploymentConfig(object): + """Configuration for deploying a model with `deploy()`. + + You can pass an instance of this class to `deploy()` to specify exactly + how to deploy the model to build. If you do not pass one, an instance built + from the default deployment_hparams will be used. + """ + + def __init__(self, + num_clones=1, + clone_on_cpu=False, + replica_id=0, + num_replicas=1, + num_ps_tasks=0, + worker_job_name='worker', + ps_job_name='ps'): + """Create a DeploymentConfig. + + The config describes how to deploy a model across multiple clones and + replicas. The model will be replicated `num_clones` times in each replica. + If `clone_on_cpu` is True, each clone will placed on CPU. + + If `num_replicas` is 1, the model is deployed via a single process. In that + case `worker_device`, `num_ps_tasks`, and `ps_device` are ignored. + + If `num_replicas` is greater than 1, then `worker_device` and `ps_device` + must specify TensorFlow devices for the `worker` and `ps` jobs and + `num_ps_tasks` must be positive. + + Args: + num_clones: Number of model clones to deploy in each replica. + clone_on_cpu: If True clones would be placed on CPU. + replica_id: Integer. Index of the replica for which the model is + deployed. Usually 0 for the chief replica. + num_replicas: Number of replicas to use. + num_ps_tasks: Number of tasks for the `ps` job. 0 to not use replicas. + worker_job_name: A name for the worker job. + ps_job_name: A name for the parameter server job. + + Raises: + ValueError: If the arguments are invalid. + """ + if num_replicas > 1: + if num_ps_tasks < 1: + raise ValueError('When using replicas num_ps_tasks must be positive') + if num_replicas > 1 or num_ps_tasks > 0: + if not worker_job_name: + raise ValueError('Must specify worker_job_name when using replicas') + if not ps_job_name: + raise ValueError('Must specify ps_job_name when using parameter server') + if replica_id >= num_replicas: + raise ValueError('replica_id must be less than num_replicas') + self._num_clones = num_clones + self._clone_on_cpu = clone_on_cpu + self._replica_id = replica_id + self._num_replicas = num_replicas + self._num_ps_tasks = num_ps_tasks + self._ps_device = '/job:' + ps_job_name if num_ps_tasks > 0 else '' + self._worker_device = '/job:' + worker_job_name if num_ps_tasks > 0 else '' + + @property + def num_clones(self): + return self._num_clones + + @property + def clone_on_cpu(self): + return self._clone_on_cpu + + @property + def replica_id(self): + return self._replica_id + + @property + def num_replicas(self): + return self._num_replicas + + @property + def num_ps_tasks(self): + return self._num_ps_tasks + + @property + def ps_device(self): + return self._ps_device + + @property + def worker_device(self): + return self._worker_device + + def caching_device(self): + """Returns the device to use for caching variables. + + Variables are cached on the worker CPU when using replicas. + + Returns: + A device string or None if the variables do not need to be cached. + """ + if self._num_ps_tasks > 0: + return lambda op: op.device + else: + return None + + def clone_device(self, clone_index): + """Device used to create the clone and all the ops inside the clone. + + Args: + clone_index: Int, representing the clone_index. + + Returns: + A value suitable for `tf.device()`. + + Raises: + ValueError: if `clone_index` is greater or equal to the number of clones". + """ + if clone_index >= self._num_clones: + raise ValueError('clone_index must be less than num_clones') + device = '' + if self._num_ps_tasks > 0: + device += self._worker_device + if self._clone_on_cpu: + device += '/device:CPU:0' + else: + device += '/device:GPU:%d' % clone_index + return device + + def clone_scope(self, clone_index): + """Name scope to create the clone. + + Args: + clone_index: Int, representing the clone_index. + + Returns: + A name_scope suitable for `tf.name_scope()`. + + Raises: + ValueError: if `clone_index` is greater or equal to the number of clones". + """ + if clone_index >= self._num_clones: + raise ValueError('clone_index must be less than num_clones') + scope = '' + if self._num_clones > 1: + scope = 'clone_%d' % clone_index + return scope + + def optimizer_device(self): + """Device to use with the optimizer. + + Returns: + A value suitable for `tf.device()`. + """ + if self._num_ps_tasks > 0 or self._num_clones > 0: + return self._worker_device + '/device:CPU:0' + else: + return '' + + def inputs_device(self): + """Device to use to build the inputs. + + Returns: + A value suitable for `tf.device()`. + """ + device = '' + if self._num_ps_tasks > 0: + device += self._worker_device + device += '/device:CPU:0' + return device + + def variables_device(self): + """Returns the device to use for variables created inside the clone. + + Returns: + A value suitable for `tf.device()`. + """ + device = '' + if self._num_ps_tasks > 0: + device += self._ps_device + device += '/device:CPU:0' + + class _PSDeviceChooser(object): + """Slim device chooser for variables when using PS.""" + + def __init__(self, device, tasks): + self._device = device + self._tasks = tasks + self._task = 0 + + def choose(self, op): + if op.device: + return op.device + node_def = op if isinstance(op, tf.NodeDef) else op.node_def + if node_def.op.startswith('Variable'): + t = self._task + self._task = (self._task + 1) % self._tasks + d = '%s/task:%d' % (self._device, t) + return d + else: + return op.device + + if not self._num_ps_tasks: + return device + else: + chooser = _PSDeviceChooser(device, self._num_ps_tasks) + return chooser.choose diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/deployment/model_deploy_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/deployment/model_deploy_test.py new file mode 100644 index 0000000..6687958 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/deployment/model_deploy_test.py @@ -0,0 +1,574 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for model_deploy.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf +from tensorflow.contrib import framework as contrib_framework +from tensorflow.contrib import layers as contrib_layers +from tensorflow.contrib import slim as contrib_slim + +from deployment import model_deploy + +slim = contrib_slim + + +class DeploymentConfigTest(tf.test.TestCase): + + def testDefaults(self): + deploy_config = model_deploy.DeploymentConfig() + + self.assertEqual(slim.get_variables(), []) + self.assertEqual(deploy_config.caching_device(), None) + self.assertDeviceEqual(deploy_config.clone_device(0), 'GPU:0') + self.assertEqual(deploy_config.clone_scope(0), '') + self.assertDeviceEqual(deploy_config.optimizer_device(), 'CPU:0') + self.assertDeviceEqual(deploy_config.inputs_device(), 'CPU:0') + self.assertDeviceEqual(deploy_config.variables_device(), 'CPU:0') + + def testCPUonly(self): + deploy_config = model_deploy.DeploymentConfig(clone_on_cpu=True) + + self.assertEqual(deploy_config.caching_device(), None) + self.assertDeviceEqual(deploy_config.clone_device(0), 'CPU:0') + self.assertEqual(deploy_config.clone_scope(0), '') + self.assertDeviceEqual(deploy_config.optimizer_device(), 'CPU:0') + self.assertDeviceEqual(deploy_config.inputs_device(), 'CPU:0') + self.assertDeviceEqual(deploy_config.variables_device(), 'CPU:0') + + def testMultiGPU(self): + deploy_config = model_deploy.DeploymentConfig(num_clones=2) + + self.assertEqual(deploy_config.caching_device(), None) + self.assertDeviceEqual(deploy_config.clone_device(0), 'GPU:0') + self.assertDeviceEqual(deploy_config.clone_device(1), 'GPU:1') + self.assertEqual(deploy_config.clone_scope(0), 'clone_0') + self.assertEqual(deploy_config.clone_scope(1), 'clone_1') + self.assertDeviceEqual(deploy_config.optimizer_device(), 'CPU:0') + self.assertDeviceEqual(deploy_config.inputs_device(), 'CPU:0') + self.assertDeviceEqual(deploy_config.variables_device(), 'CPU:0') + + def testPS(self): + deploy_config = model_deploy.DeploymentConfig(num_clones=1, num_ps_tasks=1) + + self.assertDeviceEqual(deploy_config.clone_device(0), + '/job:worker/device:GPU:0') + self.assertEqual(deploy_config.clone_scope(0), '') + self.assertDeviceEqual(deploy_config.optimizer_device(), + '/job:worker/device:CPU:0') + self.assertDeviceEqual(deploy_config.inputs_device(), + '/job:worker/device:CPU:0') + with tf.device(deploy_config.variables_device()): + a = tf.Variable(0) + b = tf.Variable(0) + c = tf.no_op() + d = slim.variable('a', [], + caching_device=deploy_config.caching_device()) + self.assertDeviceEqual(a.device, '/job:ps/task:0/device:CPU:0') + self.assertDeviceEqual(a.device, a.value().device) + self.assertDeviceEqual(b.device, '/job:ps/task:0/device:CPU:0') + self.assertDeviceEqual(b.device, b.value().device) + self.assertDeviceEqual(c.device, '') + self.assertDeviceEqual(d.device, '/job:ps/task:0/device:CPU:0') + self.assertDeviceEqual(d.value().device, '') + + def testMultiGPUPS(self): + deploy_config = model_deploy.DeploymentConfig(num_clones=2, num_ps_tasks=1) + + self.assertEqual(deploy_config.caching_device()(tf.no_op()), '') + self.assertDeviceEqual(deploy_config.clone_device(0), + '/job:worker/device:GPU:0') + self.assertDeviceEqual(deploy_config.clone_device(1), + '/job:worker/device:GPU:1') + self.assertEqual(deploy_config.clone_scope(0), 'clone_0') + self.assertEqual(deploy_config.clone_scope(1), 'clone_1') + self.assertDeviceEqual(deploy_config.optimizer_device(), + '/job:worker/device:CPU:0') + self.assertDeviceEqual(deploy_config.inputs_device(), + '/job:worker/device:CPU:0') + + def testReplicasPS(self): + deploy_config = model_deploy.DeploymentConfig(num_replicas=2, + num_ps_tasks=2) + + self.assertDeviceEqual(deploy_config.clone_device(0), + '/job:worker/device:GPU:0') + self.assertEqual(deploy_config.clone_scope(0), '') + self.assertDeviceEqual(deploy_config.optimizer_device(), + '/job:worker/device:CPU:0') + self.assertDeviceEqual(deploy_config.inputs_device(), + '/job:worker/device:CPU:0') + + def testReplicasMultiGPUPS(self): + deploy_config = model_deploy.DeploymentConfig(num_replicas=2, + num_clones=2, + num_ps_tasks=2) + self.assertDeviceEqual(deploy_config.clone_device(0), + '/job:worker/device:GPU:0') + self.assertDeviceEqual(deploy_config.clone_device(1), + '/job:worker/device:GPU:1') + self.assertEqual(deploy_config.clone_scope(0), 'clone_0') + self.assertEqual(deploy_config.clone_scope(1), 'clone_1') + self.assertDeviceEqual(deploy_config.optimizer_device(), + '/job:worker/device:CPU:0') + self.assertDeviceEqual(deploy_config.inputs_device(), + '/job:worker/device:CPU:0') + + def testVariablesPS(self): + deploy_config = model_deploy.DeploymentConfig(num_ps_tasks=2) + + with tf.device(deploy_config.variables_device()): + a = tf.Variable(0) + b = tf.Variable(0) + c = tf.no_op() + d = slim.variable('a', [], + caching_device=deploy_config.caching_device()) + + self.assertDeviceEqual(a.device, '/job:ps/task:0/device:CPU:0') + self.assertDeviceEqual(a.device, a.value().device) + self.assertDeviceEqual(b.device, '/job:ps/task:1/device:CPU:0') + self.assertDeviceEqual(b.device, b.value().device) + self.assertDeviceEqual(c.device, '') + self.assertDeviceEqual(d.device, '/job:ps/task:0/device:CPU:0') + self.assertDeviceEqual(d.value().device, '') + + +def LogisticClassifier(inputs, labels, scope=None, reuse=None): + with tf.variable_scope(scope, 'LogisticClassifier', [inputs, labels], + reuse=reuse): + predictions = slim.fully_connected(inputs, 1, activation_fn=tf.sigmoid, + scope='fully_connected') + slim.losses.log_loss(predictions, labels) + return predictions + + +def BatchNormClassifier(inputs, labels, scope=None, reuse=None): + with tf.variable_scope(scope, 'BatchNormClassifier', [inputs, labels], + reuse=reuse): + inputs = slim.batch_norm(inputs, decay=0.1, fused=True) + predictions = slim.fully_connected(inputs, 1, + activation_fn=tf.sigmoid, + scope='fully_connected') + slim.losses.log_loss(predictions, labels) + return predictions + + +class CreatecloneTest(tf.test.TestCase): + + def setUp(self): + # Create an easy training set: + np.random.seed(0) + + self._inputs = np.zeros((16, 4)) + self._labels = np.random.randint(0, 2, size=(16, 1)).astype(np.float32) + self._logdir = self.get_temp_dir() + + for i in range(16): + j = int(2 * self._labels[i] + np.random.randint(0, 2)) + self._inputs[i, j] = 1 + + def testCreateLogisticClassifier(self): + g = tf.Graph() + with g.as_default(): + tf.set_random_seed(0) + tf_inputs = tf.constant(self._inputs, dtype=tf.float32) + tf_labels = tf.constant(self._labels, dtype=tf.float32) + + model_fn = LogisticClassifier + clone_args = (tf_inputs, tf_labels) + deploy_config = model_deploy.DeploymentConfig(num_clones=1) + + self.assertEqual(slim.get_variables(), []) + clones = model_deploy.create_clones(deploy_config, model_fn, clone_args) + clone = clones[0] + self.assertEqual(len(slim.get_variables()), 2) + for v in slim.get_variables(): + self.assertDeviceEqual(v.device, 'CPU:0') + self.assertDeviceEqual(v.value().device, 'CPU:0') + self.assertEqual(clone.outputs.op.name, + 'LogisticClassifier/fully_connected/Sigmoid') + self.assertEqual(clone.scope, '') + self.assertDeviceEqual(clone.device, 'GPU:0') + self.assertEqual(len(slim.losses.get_losses()), 1) + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) + self.assertEqual(update_ops, []) + + def testCreateSingleclone(self): + g = tf.Graph() + with g.as_default(): + tf.set_random_seed(0) + tf_inputs = tf.constant(self._inputs, dtype=tf.float32) + tf_labels = tf.constant(self._labels, dtype=tf.float32) + + model_fn = BatchNormClassifier + clone_args = (tf_inputs, tf_labels) + deploy_config = model_deploy.DeploymentConfig(num_clones=1) + + self.assertEqual(slim.get_variables(), []) + clones = model_deploy.create_clones(deploy_config, model_fn, clone_args) + clone = clones[0] + self.assertEqual(len(slim.get_variables()), 5) + for v in slim.get_variables(): + self.assertDeviceEqual(v.device, 'CPU:0') + self.assertDeviceEqual(v.value().device, 'CPU:0') + self.assertEqual(clone.outputs.op.name, + 'BatchNormClassifier/fully_connected/Sigmoid') + self.assertEqual(clone.scope, '') + self.assertDeviceEqual(clone.device, 'GPU:0') + self.assertEqual(len(slim.losses.get_losses()), 1) + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) + self.assertEqual(len(update_ops), 2) + + def testCreateMulticlone(self): + g = tf.Graph() + with g.as_default(): + tf.set_random_seed(0) + tf_inputs = tf.constant(self._inputs, dtype=tf.float32) + tf_labels = tf.constant(self._labels, dtype=tf.float32) + + model_fn = BatchNormClassifier + clone_args = (tf_inputs, tf_labels) + num_clones = 4 + deploy_config = model_deploy.DeploymentConfig(num_clones=num_clones) + + self.assertEqual(slim.get_variables(), []) + clones = model_deploy.create_clones(deploy_config, model_fn, clone_args) + self.assertEqual(len(slim.get_variables()), 5) + for v in slim.get_variables(): + self.assertDeviceEqual(v.device, 'CPU:0') + self.assertDeviceEqual(v.value().device, 'CPU:0') + self.assertEqual(len(clones), num_clones) + for i, clone in enumerate(clones): + self.assertEqual( + clone.outputs.op.name, + 'clone_%d/BatchNormClassifier/fully_connected/Sigmoid' % i) + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, clone.scope) + self.assertEqual(len(update_ops), 2) + self.assertEqual(clone.scope, 'clone_%d/' % i) + self.assertDeviceEqual(clone.device, 'GPU:%d' % i) + + def testCreateOnecloneWithPS(self): + g = tf.Graph() + with g.as_default(): + tf.set_random_seed(0) + tf_inputs = tf.constant(self._inputs, dtype=tf.float32) + tf_labels = tf.constant(self._labels, dtype=tf.float32) + + model_fn = BatchNormClassifier + clone_args = (tf_inputs, tf_labels) + deploy_config = model_deploy.DeploymentConfig(num_clones=1, + num_ps_tasks=1) + + self.assertEqual(slim.get_variables(), []) + clones = model_deploy.create_clones(deploy_config, model_fn, clone_args) + self.assertEqual(len(clones), 1) + clone = clones[0] + self.assertEqual(clone.outputs.op.name, + 'BatchNormClassifier/fully_connected/Sigmoid') + self.assertDeviceEqual(clone.device, '/job:worker/device:GPU:0') + self.assertEqual(clone.scope, '') + self.assertEqual(len(slim.get_variables()), 5) + for v in slim.get_variables(): + self.assertDeviceEqual(v.device, '/job:ps/task:0/CPU:0') + self.assertDeviceEqual(v.device, v.value().device) + + def testCreateMulticloneWithPS(self): + g = tf.Graph() + with g.as_default(): + tf.set_random_seed(0) + tf_inputs = tf.constant(self._inputs, dtype=tf.float32) + tf_labels = tf.constant(self._labels, dtype=tf.float32) + + model_fn = BatchNormClassifier + clone_args = (tf_inputs, tf_labels) + deploy_config = model_deploy.DeploymentConfig(num_clones=2, + num_ps_tasks=2) + + self.assertEqual(slim.get_variables(), []) + clones = model_deploy.create_clones(deploy_config, model_fn, clone_args) + self.assertEqual(len(slim.get_variables()), 5) + for i, v in enumerate(slim.get_variables()): + t = i % 2 + self.assertDeviceEqual(v.device, '/job:ps/task:%d/device:CPU:0' % t) + self.assertDeviceEqual(v.device, v.value().device) + self.assertEqual(len(clones), 2) + for i, clone in enumerate(clones): + self.assertEqual( + clone.outputs.op.name, + 'clone_%d/BatchNormClassifier/fully_connected/Sigmoid' % i) + self.assertEqual(clone.scope, 'clone_%d/' % i) + self.assertDeviceEqual(clone.device, '/job:worker/device:GPU:%d' % i) + + +class OptimizeclonesTest(tf.test.TestCase): + + def setUp(self): + # Create an easy training set: + np.random.seed(0) + + self._inputs = np.zeros((16, 4)) + self._labels = np.random.randint(0, 2, size=(16, 1)).astype(np.float32) + self._logdir = self.get_temp_dir() + + for i in range(16): + j = int(2 * self._labels[i] + np.random.randint(0, 2)) + self._inputs[i, j] = 1 + + def testCreateLogisticClassifier(self): + g = tf.Graph() + with g.as_default(): + tf.set_random_seed(0) + tf_inputs = tf.constant(self._inputs, dtype=tf.float32) + tf_labels = tf.constant(self._labels, dtype=tf.float32) + + model_fn = LogisticClassifier + clone_args = (tf_inputs, tf_labels) + deploy_config = model_deploy.DeploymentConfig(num_clones=1) + + self.assertEqual(slim.get_variables(), []) + clones = model_deploy.create_clones(deploy_config, model_fn, clone_args) + self.assertEqual(len(slim.get_variables()), 2) + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) + self.assertEqual(update_ops, []) + + optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) + total_loss, grads_and_vars = model_deploy.optimize_clones(clones, + optimizer) + self.assertEqual(len(grads_and_vars), len(tf.trainable_variables())) + self.assertEqual(total_loss.op.name, 'total_loss') + for g, v in grads_and_vars: + self.assertDeviceEqual(g.device, 'GPU:0') + self.assertDeviceEqual(v.device, 'CPU:0') + + def testCreateSingleclone(self): + g = tf.Graph() + with g.as_default(): + tf.set_random_seed(0) + tf_inputs = tf.constant(self._inputs, dtype=tf.float32) + tf_labels = tf.constant(self._labels, dtype=tf.float32) + + model_fn = BatchNormClassifier + clone_args = (tf_inputs, tf_labels) + deploy_config = model_deploy.DeploymentConfig(num_clones=1) + + self.assertEqual(slim.get_variables(), []) + clones = model_deploy.create_clones(deploy_config, model_fn, clone_args) + self.assertEqual(len(slim.get_variables()), 5) + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) + self.assertEqual(len(update_ops), 2) + + optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) + total_loss, grads_and_vars = model_deploy.optimize_clones(clones, + optimizer) + self.assertEqual(len(grads_and_vars), len(tf.trainable_variables())) + self.assertEqual(total_loss.op.name, 'total_loss') + for g, v in grads_and_vars: + self.assertDeviceEqual(g.device, 'GPU:0') + self.assertDeviceEqual(v.device, 'CPU:0') + + def testCreateMulticlone(self): + g = tf.Graph() + with g.as_default(): + tf.set_random_seed(0) + tf_inputs = tf.constant(self._inputs, dtype=tf.float32) + tf_labels = tf.constant(self._labels, dtype=tf.float32) + + model_fn = BatchNormClassifier + clone_args = (tf_inputs, tf_labels) + num_clones = 4 + deploy_config = model_deploy.DeploymentConfig(num_clones=num_clones) + + self.assertEqual(slim.get_variables(), []) + clones = model_deploy.create_clones(deploy_config, model_fn, clone_args) + self.assertEqual(len(slim.get_variables()), 5) + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) + self.assertEqual(len(update_ops), num_clones * 2) + + optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) + total_loss, grads_and_vars = model_deploy.optimize_clones(clones, + optimizer) + self.assertEqual(len(grads_and_vars), len(tf.trainable_variables())) + self.assertEqual(total_loss.op.name, 'total_loss') + for g, v in grads_and_vars: + self.assertDeviceEqual(g.device, '') + self.assertDeviceEqual(v.device, 'CPU:0') + + def testCreateMulticloneCPU(self): + g = tf.Graph() + with g.as_default(): + tf.set_random_seed(0) + tf_inputs = tf.constant(self._inputs, dtype=tf.float32) + tf_labels = tf.constant(self._labels, dtype=tf.float32) + + model_fn = BatchNormClassifier + model_args = (tf_inputs, tf_labels) + num_clones = 4 + deploy_config = model_deploy.DeploymentConfig(num_clones=num_clones, + clone_on_cpu=True) + + self.assertEqual(slim.get_variables(), []) + clones = model_deploy.create_clones(deploy_config, model_fn, model_args) + self.assertEqual(len(slim.get_variables()), 5) + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) + self.assertEqual(len(update_ops), num_clones * 2) + + optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) + total_loss, grads_and_vars = model_deploy.optimize_clones(clones, + optimizer) + self.assertEqual(len(grads_and_vars), len(tf.trainable_variables())) + self.assertEqual(total_loss.op.name, 'total_loss') + for g, v in grads_and_vars: + self.assertDeviceEqual(g.device, '') + self.assertDeviceEqual(v.device, 'CPU:0') + + def testCreateOnecloneWithPS(self): + g = tf.Graph() + with g.as_default(): + tf.set_random_seed(0) + tf_inputs = tf.constant(self._inputs, dtype=tf.float32) + tf_labels = tf.constant(self._labels, dtype=tf.float32) + + model_fn = BatchNormClassifier + model_args = (tf_inputs, tf_labels) + deploy_config = model_deploy.DeploymentConfig(num_clones=1, + num_ps_tasks=1) + + self.assertEqual(slim.get_variables(), []) + clones = model_deploy.create_clones(deploy_config, model_fn, model_args) + self.assertEqual(len(slim.get_variables()), 5) + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) + self.assertEqual(len(update_ops), 2) + + optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) + total_loss, grads_and_vars = model_deploy.optimize_clones(clones, + optimizer) + self.assertEqual(len(grads_and_vars), len(tf.trainable_variables())) + self.assertEqual(total_loss.op.name, 'total_loss') + for g, v in grads_and_vars: + self.assertDeviceEqual(g.device, '/job:worker/device:GPU:0') + self.assertDeviceEqual(v.device, '/job:ps/task:0/CPU:0') + + +class DeployTest(tf.test.TestCase): + + def setUp(self): + # Create an easy training set: + np.random.seed(0) + + self._inputs = np.zeros((16, 4)) + self._labels = np.random.randint(0, 2, size=(16, 1)).astype(np.float32) + self._logdir = self.get_temp_dir() + + for i in range(16): + j = int(2 * self._labels[i] + np.random.randint(0, 2)) + self._inputs[i, j] = 1 + + def _addBesselsCorrection(self, sample_size, expected_var): + correction_factor = sample_size / (sample_size - 1) + expected_var *= correction_factor + return expected_var + + def testLocalTrainOp(self): + g = tf.Graph() + with g.as_default(): + tf.set_random_seed(0) + tf_inputs = tf.constant(self._inputs, dtype=tf.float32) + tf_labels = tf.constant(self._labels, dtype=tf.float32) + + model_fn = BatchNormClassifier + model_args = (tf_inputs, tf_labels) + deploy_config = model_deploy.DeploymentConfig(num_clones=2, + clone_on_cpu=True) + + optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) + + self.assertEqual(slim.get_variables(), []) + model = model_deploy.deploy(deploy_config, model_fn, model_args, + optimizer=optimizer) + + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) + self.assertEqual(len(update_ops), 4) + self.assertEqual(len(model.clones), 2) + self.assertEqual(model.total_loss.op.name, 'total_loss') + self.assertEqual(model.summary_op.op.name, 'summary_op/summary_op') + self.assertEqual(model.train_op.op.name, 'train_op') + + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + moving_mean = contrib_framework.get_variables_by_name('moving_mean')[0] + moving_variance = contrib_framework.get_variables_by_name( + 'moving_variance')[0] + initial_loss = sess.run(model.total_loss) + initial_mean, initial_variance = sess.run([moving_mean, + moving_variance]) + self.assertAllClose(initial_mean, [0.0, 0.0, 0.0, 0.0]) + self.assertAllClose(initial_variance, [1.0, 1.0, 1.0, 1.0]) + for _ in range(10): + sess.run(model.train_op) + final_loss = sess.run(model.total_loss) + self.assertLess(final_loss, initial_loss / 5.0) + + final_mean, final_variance = sess.run([moving_mean, + moving_variance]) + expected_mean = np.array([0.125, 0.25, 0.375, 0.25]) + expected_var = np.array([0.109375, 0.1875, 0.234375, 0.1875]) + expected_var = self._addBesselsCorrection(16, expected_var) + self.assertAllClose(final_mean, expected_mean) + self.assertAllClose(final_variance, expected_var) + + def testNoSummariesOnGPU(self): + with tf.Graph().as_default(): + deploy_config = model_deploy.DeploymentConfig(num_clones=2) + + # clone function creates a fully_connected layer with a regularizer loss. + def ModelFn(): + inputs = tf.constant(1.0, shape=(10, 20), dtype=tf.float32) + reg = contrib_layers.l2_regularizer(0.001) + contrib_layers.fully_connected(inputs, 30, weights_regularizer=reg) + + model = model_deploy.deploy( + deploy_config, ModelFn, + optimizer=tf.train.GradientDescentOptimizer(1.0)) + # The model summary op should have a few summary inputs and all of them + # should be on the CPU. + self.assertTrue(model.summary_op.op.inputs) + for inp in model.summary_op.op.inputs: + self.assertEqual('/device:CPU:0', inp.device) + + def testNoSummariesOnGPUForEvals(self): + with tf.Graph().as_default(): + deploy_config = model_deploy.DeploymentConfig(num_clones=2) + + # clone function creates a fully_connected layer with a regularizer loss. + def ModelFn(): + inputs = tf.constant(1.0, shape=(10, 20), dtype=tf.float32) + reg = contrib_layers.l2_regularizer(0.001) + contrib_layers.fully_connected(inputs, 30, weights_regularizer=reg) + + # No optimizer here, it's an eval. + model = model_deploy.deploy(deploy_config, ModelFn) + # The model summary op should have a few summary inputs and all of them + # should be on the CPU. + self.assertTrue(model.summary_op.op.inputs) + for inp in model.summary_op.op.inputs: + self.assertEqual('/device:CPU:0', inp.device) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/download_and_convert_data.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/download_and_convert_data.py new file mode 100644 index 0000000..e935780 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/download_and_convert_data.py @@ -0,0 +1,94 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""Downloads and converts a particular dataset. + +Usage: +```shell + +$ python download_and_convert_data.py \ + --dataset_name=flowers \ + --dataset_dir=/tmp/flowers + +$ python download_and_convert_data.py \ + --dataset_name=cifar10 \ + --dataset_dir=/tmp/cifar10 + +$ python download_and_convert_data.py \ + --dataset_name=mnist \ + --dataset_dir=/tmp/mnist + +$ python download_and_convert_data.py \ + --dataset_name=visualwakewords \ + --dataset_dir=/tmp/visualwakewords + +``` +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from datasets import download_and_convert_cifar10 +from datasets import download_and_convert_flowers +from datasets import download_and_convert_mnist +from datasets import download_and_convert_visualwakewords + +FLAGS = tf.compat.v1.app.flags.FLAGS + +tf.compat.v1.app.flags.DEFINE_string( + 'dataset_name', + None, + 'The name of the dataset to convert, one of "flowers", "cifar10", "mnist", "visualwakewords"' + ) + +tf.compat.v1.app.flags.DEFINE_string( + 'dataset_dir', + None, + 'The directory where the output TFRecords and temporary files are saved.') + +tf.flags.DEFINE_float( + 'small_object_area_threshold', 0.005, + 'For --dataset_name=visualwakewords only. Threshold of fraction of image ' + 'area below which small objects are filtered') + +tf.flags.DEFINE_string( + 'foreground_class_of_interest', 'person', + 'For --dataset_name=visualwakewords only. Build a binary classifier based ' + 'on the presence or absence of this object in the image.') + + +def main(_): + if not FLAGS.dataset_name: + raise ValueError('You must supply the dataset name with --dataset_name') + if not FLAGS.dataset_dir: + raise ValueError('You must supply the dataset directory with --dataset_dir') + + if FLAGS.dataset_name == 'flowers': + download_and_convert_flowers.run(FLAGS.dataset_dir) + elif FLAGS.dataset_name == 'cifar10': + download_and_convert_cifar10.run(FLAGS.dataset_dir) + elif FLAGS.dataset_name == 'mnist': + download_and_convert_mnist.run(FLAGS.dataset_dir) + elif FLAGS.dataset_name == 'visualwakewords': + download_and_convert_visualwakewords.run( + FLAGS.dataset_dir, FLAGS.small_object_area_threshold, + FLAGS.foreground_class_of_interest) + else: + raise ValueError( + 'dataset_name [%s] was not recognized.' % FLAGS.dataset_name) + +if __name__ == '__main__': + tf.compat.v1.app.run() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/env.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/env.py new file mode 100644 index 0000000..6e5fa0e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/env.py @@ -0,0 +1,182 @@ +import tensorflow as tf +from time import gmtime, strftime +from tensorflow.contrib import slim as contrib_slim +from gpu_helper import get_custom_getter +import random +import numpy as np +import os + +np.random.seed(0) +random.seed(0) +tf.set_random_seed(0) + + +class Env: + def __init__(self, FLAGS): + self.FLAGS = FLAGS + + self.slim = contrib_slim + self.num_samples = 1281167 + + def _configure_optimizer(self, learning_rate): + """Configures the optimizer used for training. + + Args: + learning_rate: A scalar or `Tensor` learning rate. + + Returns: + An instance of an optimizer. + + Raises: + ValueError: if Initializer.FLAGS.optimizer is not recognized. + """ + if self.FLAGS.optimizer == 'adadelta': + optimizer = tf.train.AdadeltaOptimizer( + learning_rate, + rho=self.FLAGS.adadelta_rho, + epsilon=self.FLAGS.opt_epsilon) + elif self.FLAGS.optimizer == 'adagrad': + optimizer = tf.train.AdagradOptimizer( + learning_rate, + initial_accumulator_value=self.FLAGS.adagrad_initial_accumulator_value) + elif self.FLAGS.optimizer == 'adam': + optimizer = tf.train.AdamOptimizer( + learning_rate, + beta1=self.FLAGS.adam_beta1, + beta2=self.FLAGS.adam_beta2, + epsilon=self.FLAGS.opt_epsilon) + elif self.FLAGS.optimizer == 'ftrl': + optimizer = tf.train.FtrlOptimizer( + learning_rate, + learning_rate_power=self.FLAGS.ftrl_learning_rate_power, + initial_accumulator_value=self.FLAGS.ftrl_initial_accumulator_value, + l1_regularization_strength=self.FLAGS.ftrl_l1, + l2_regularization_strength=self.FLAGS.ftrl_l2) + elif self.FLAGS.optimizer == 'momentum': + optimizer = tf.train.MomentumOptimizer( + learning_rate, + momentum=self.FLAGS.momentum, + name='Momentum') + elif self.FLAGS.optimizer == 'rmsprop': + optimizer = tf.train.RMSPropOptimizer( + learning_rate, + decay=self.FLAGS.rmsprop_decay, + momentum=self.FLAGS.rmsprop_momentum, + epsilon=self.FLAGS.opt_epsilon) + elif self.FLAGS.optimizer == 'sgd': + optimizer = tf.train.GradientDescentOptimizer(learning_rate) + else: + raise ValueError('Optimizer [%s] was not recognized' % self.FLAGS.optimizer) + + return optimizer + + def create_logdir(self): + logdir = "results" + os.makedirs(logdir, exist_ok=True) + return logdir + + def calc_logits(self, network_fn, images): + logits, end_points = network_fn(images, reuse=tf.AUTO_REUSE) + return logits + + def calc_loss(self, logits_train, labels_train): + base_loss = self.slim.losses.softmax_cross_entropy( + logits_train, labels_train, label_smoothing=self.FLAGS.label_smoothing, weights=1.0) + + reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) + total_loss = tf.add_n([base_loss] + reg_losses, name='total_loss') + + loss = tf.add_n([base_loss]) + loss = tf.identity(loss, name='loss') + + return loss, total_loss + + def calc_steps_per_epoch(self): + return self.num_samples // (self.FLAGS.batch_size * int(os.getenv('RANK_SIZE'))) + + def _configure_learning_rate(self, global_step): + steps_per_epoch = self.calc_steps_per_epoch() + decay_steps = int(steps_per_epoch * self.FLAGS.num_epochs_per_decay) + + if self.FLAGS.learning_rate_decay_type == 'exponential': + learning_rate = tf.train.exponential_decay( + self.FLAGS.learning_rate, + global_step, + decay_steps, + self.FLAGS.learning_rate_decay_factor, + staircase=True, + name='exponential_decay_learning_rate') + elif self.FLAGS.learning_rate_decay_type == 'fixed': + learning_rate = tf.constant(self.FLAGS.learning_rate, name='fixed_learning_rate') + elif self.FLAGS.learning_rate_decay_type == 'cosine_annealing': + current_step_epoch = global_step // steps_per_epoch * steps_per_epoch + learning_rate = tf.train.cosine_decay(self.FLAGS.learning_rate, current_step_epoch, + self.FLAGS.max_number_of_steps) + elif self.FLAGS.learning_rate_decay_type == 'polynomial': + learning_rate = tf.train.polynomial_decay( + self.FLAGS.learning_rate, global_step, + decay_steps, + self.FLAGS.end_learning_rate, + power=1.0, + cycle=False, + name='polynomial_decay_learning_rate') + else: + raise ValueError('learning_rate_decay_type [%s] was not recognized' % + self.FLAGS.learning_rate_decay_type) + + if self.FLAGS.warmup_epochs: + warmup_lr = ( + self.FLAGS.learning_rate * tf.cast(global_step, tf.float32) / + (steps_per_epoch * self.FLAGS.warmup_epochs)) + learning_rate = tf.minimum(warmup_lr, learning_rate) + + learning_rate = tf.identity(learning_rate, name='learning_rate') + # tf.Print(learning_rate, [learning_rate], '*****************') + return learning_rate + + def create_train_op(self, global_step, summaries, loss): + # Gather update_ops from the first clone. These contain, for example, + # the updates for the batch_norm variables created by network_fn. + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) or [] + + ################################# + # Configure the moving averages # + ################################# + if self.FLAGS.moving_average_decay: + moving_average_variables = self.slim.get_model_variables() + variable_averages = tf.train.ExponentialMovingAverage( + self.FLAGS.moving_average_decay, global_step) + else: + moving_average_variables, variable_averages = None, None + + ######################################### + # Configure the optimization procedure. # + ######################################### + learning_rate = self._configure_learning_rate(global_step) + summaries.add(tf.summary.scalar('learning_rate', learning_rate)) + + if self.FLAGS.moving_average_decay: + # Update ops executed locally by trainer. + update_ops.append(variable_averages.apply(moving_average_variables)) + + opt = self._configure_optimizer(learning_rate) + + from npu_bridge.estimator.npu.npu_optimizer import NPUDistributedOptimizer + from npu_bridge.estimator.npu.npu_loss_scale_optimizer import NPULossScaleOptimizer + from npu_bridge.estimator.npu.npu_loss_scale_manager import FixedLossScaleManager + from npu_bridge.estimator.npu.npu_loss_scale_manager import ExponentialUpdateLossScaleManager + loss_scale_manager = FixedLossScaleManager(loss_scale=4096) + # loss_scale_manager = ExponentialUpdateLossScaleManager(init_loss_scale=1024, incr_every_n_steps=1000, decr_every_n_nan_or_inf=2, decr_ratio=0.5) + if int(os.getenv('RANK_SIZE')) == 1: + opt = NPULossScaleOptimizer(opt, loss_scale_manager) + else: + opt = NPULossScaleOptimizer(opt, loss_scale_manager, is_distributed=True) + opt = NPUDistributedOptimizer(opt) + + update_op = tf.group(*update_ops) + with tf.control_dependencies([update_op]): + gate_gradients = (tf.train.Optimizer.GATE_NONE) + grads_and_vars = opt.compute_gradients(loss) + train_op = opt.apply_gradients(grads_and_vars, global_step=global_step) + + return train_op diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/estimator_impl.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/estimator_impl.py new file mode 100644 index 0000000..1d981ea --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/estimator_impl.py @@ -0,0 +1,133 @@ +import tensorflow as tf +from dataloader import data_provider +from datasets import dataset_factory +from nets import nets_factory +import os + + +class EstimatorImpl: + def __init__(self, env): + self.env = env + + def model_fn(self, features, labels, mode, params): + num_classes = 1001 + + summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES)) + + if mode == tf.estimator.ModeKeys.TRAIN: + network_fn = nets_factory.get_network_fn( + self.env.FLAGS.model_name, + num_classes=(num_classes - self.env.FLAGS.labels_offset), + weight_decay=self.env.FLAGS.weight_decay, + is_training=True) + + logits = self.env.calc_logits(network_fn, features) + loss, total_loss = self.env.calc_loss(logits, labels) + + # ### accuracy ### # + predictions = tf.argmax(logits, 1) + accuracy_ops = tf.metrics.accuracy(tf.argmax(labels, 1), predictions) + tf.identity(accuracy_ops[1], name='train_accuracy') + # ### accuracy ### # + + tf.identity(total_loss, 'train_loss') + + global_step = tf.train.get_or_create_global_step() + train_op = self.env.create_train_op(global_step, summaries, total_loss) + + estimator_spec = tf.estimator.EstimatorSpec( + mode=tf.estimator.ModeKeys.TRAIN, loss=total_loss, train_op=train_op) + + elif mode == tf.estimator.ModeKeys.EVAL: + network_fn = nets_factory.get_network_fn( + self.env.FLAGS.model_name, + num_classes=(num_classes - self.env.FLAGS.labels_offset), + weight_decay=self.env.FLAGS.weight_decay, + is_training=False) + + logits = self.env.calc_logits(network_fn, features) + loss, total_loss = self.env.calc_loss(logits, labels) + predictions = tf.argmax(logits, 1) + accuracy_ops = tf.metrics.accuracy(tf.argmax(labels, 1), predictions) + tf.identity(accuracy_ops[1], name='eval_accuracy') + estimator_spec = tf.estimator.EstimatorSpec( + mode=tf.estimator.ModeKeys.EVAL, + loss=total_loss, eval_metric_ops={'accuracy': accuracy_ops}) + + return estimator_spec + + def main(self): + logdir = self.env.create_logdir() + + from logger import LogSessionRunHook + + config = { + 'num_training_samples': self.env.num_samples, + # for 1p, just per loop print, for 8p, print each epoch + 'display_every': 1, + 'log_name': 'train_log.log', + 'log_dir': logdir, + 'global_batch_size': self.env.FLAGS.batch_size * int(os.getenv('RANK_SIZE')), + 'iterations_per_loop': self.env.FLAGS.iterations_per_loop if self.env.FLAGS.iterations_per_loop is not None else self.env.calc_steps_per_epoch() + } + + hooks = [LogSessionRunHook(config, warmup_steps=self.env.FLAGS.warmup_epochs * self.env.calc_steps_per_epoch())] + + ################################################################# + from npu_bridge.estimator.npu.npu_config import NPURunConfig + from npu_bridge.estimator.npu.npu_estimator import NPUEstimator + + self.estimator_config = tf.ConfigProto( + inter_op_parallelism_threads=10, + intra_op_parallelism_threads=10, + allow_soft_placement=True) + + self.estimator_config.gpu_options.allow_growth = True + + gpu_thread_count = 2 + os.environ['TF_GPU_THREAD_MODE'] = 'gpu_private' + os.environ['TF_GPU_THREAD_COUNT'] = str(gpu_thread_count) + os.environ['TF_USE_CUDNN_BATCHNORM_SPATIAL_PERSISTENT'] = '1' + os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = '1' + + run_config = NPURunConfig( + hcom_parallel=True, + precision_mode="allow_mix_precision", + enable_data_pre_proc=True, + save_checkpoints_steps=self.env.calc_steps_per_epoch(), + session_config=self.estimator_config, + model_dir=logdir, + iterations_per_loop=config['iterations_per_loop'], + keep_checkpoint_max=5) + + classifier = NPUEstimator( + model_fn=self.model_fn, + config=run_config + ) + ################################################################### + + classifier.train( + input_fn=self.train_data, + max_steps=self.env.FLAGS.max_number_of_steps, + hooks=hooks, + ) + + def train_data(self): + dataset = dataset_factory.get_dataset(self.env.FLAGS.dataset_name, 'train', self.env.FLAGS.dataset_dir) + + preprocessing_name = self.env.FLAGS.preprocessing_name or self.env.FLAGS.model_name + _, ds = data_provider.get_data(dataset, self.env.FLAGS.batch_size, + dataset.num_classes, self.env.FLAGS.labels_offset, True, + preprocessing_name, self.env.FLAGS.use_grayscale) + + return ds + + def eval_data(self): + dataset = dataset_factory.get_dataset(self.env.FLAGS.dataset_name, 'validation', self.env.FLAGS.dataset_dir) + + preprocessing_name = self.env.FLAGS.preprocessing_name or self.env.FLAGS.model_name + _, ds = data_provider.get_data(dataset, self.env.FLAGS.batch_size, + dataset.num_classes, self.env.FLAGS.labels_offset, False, + preprocessing_name, self.env.FLAGS.use_grayscale) + + return ds diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/eval_image_classifier.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/eval_image_classifier.py new file mode 100644 index 0000000..9a2b25b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/eval_image_classifier.py @@ -0,0 +1,174 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Generic evaluation script that evaluates a model using a given dataset.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math +import tensorflow as tf +from tensorflow.contrib import quantize as contrib_quantize +from tensorflow.contrib import slim as contrib_slim +from benchmark_log import hwlog +from datasets import dataset_factory +from nets import nets_factory +import os +os.environ["CUDA_VISIBLE_DEVICES"] = '4' + +slim = contrib_slim + +tf.app.flags.DEFINE_integer( + 'batch_size', 100, 'The number of samples in each batch.') + +tf.app.flags.DEFINE_integer( + 'max_num_batches', None, + 'Max number of batches to evaluate by default use all.') + +tf.app.flags.DEFINE_string( + 'master', '', 'The address of the TensorFlow master to use.') + +ckpt_path = './results/0526023335_train_hvdTrue_mnmobilenet_v2_augmentedTrue_mixedpFalse_lr0.4_optmomentum_me200_lrdtcosine_annealing_nepd0.3125_lrdf0.98_b256_me_param' +# ckpt_path = './results/0523130615_train_hvdTrue_mnmobilenet_v2_augmentedTrue_mixedpFalse_lr0.4_optmomentum_me200_lrdtcosine_annealing_nepd0.3125_lrdf0.98_b256_me_param' +tf.app.flags.DEFINE_string( + 'checkpoint_path', ckpt_path, + 'The directory where the model was written to or an absolute path to a ' + 'checkpoint file.') + +tf.app.flags.DEFINE_string( + 'eval_dir', ckpt_path, 'Directory where the results are saved to.') + +tf.app.flags.DEFINE_integer( + 'num_preprocessing_threads', 4, + 'The number of threads used to create the batches.') + +tf.app.flags.DEFINE_string( + 'dataset_name', 'imagenet', 'The name of the dataset to load.') + +tf.app.flags.DEFINE_string( + 'dataset_split_name', 'validation', 'The name of the train/test split.') + +tf.app.flags.DEFINE_string( + 'dataset_dir', '/data/Datasets/imagenet_TF', 'The directory where the dataset files are stored.') + +tf.app.flags.DEFINE_integer( + 'labels_offset', 0, + 'An offset for the labels in the dataset. This flag is primarily used to ' + 'evaluate the VGG and ResNet architectures which do not use a background ' + 'class for the ImageNet dataset.') + +tf.app.flags.DEFINE_string( + 'model_name', 'mobilenet_v2', 'The name of the architecture to evaluate.') + +tf.app.flags.DEFINE_string( + 'preprocessing_name', None, 'The name of the preprocessing to use. If left ' + 'as `None`, then the model_name flag is used.') + +tf.app.flags.DEFINE_float( + 'moving_average_decay', None, + 'The decay to use for the moving average.' + 'If left as None, then moving averages are not used.') + +tf.app.flags.DEFINE_integer( + 'eval_image_size', None, 'Eval image size') + +tf.app.flags.DEFINE_bool( + 'quantize', False, 'whether to use quantized graph or not.') + +tf.app.flags.DEFINE_bool('use_grayscale', False, + 'Whether to convert input images to grayscale.') + +FLAGS = tf.app.flags.FLAGS + + +def main(_): + if not FLAGS.dataset_dir: + raise ValueError('You must supply the dataset directory with --dataset_dir') + + tf.logging.set_verbosity(tf.logging.INFO) + with tf.Graph().as_default(): + tf_global_step = slim.get_or_create_global_step() + + ###################### + # Select the dataset # + ###################### + dataset = dataset_factory.get_dataset( + FLAGS.dataset_name, FLAGS.dataset_split_name, FLAGS.dataset_dir) + + #################### + # Select the model # + #################### + network_fn = nets_factory.get_network_fn( + FLAGS.model_name, + num_classes=(dataset.num_classes - FLAGS.labels_offset), + is_training=False) + + from dataloader import data_provider + preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name + iterator, _ = data_provider.get_data(dataset, FLAGS.batch_size, + dataset.num_classes, FLAGS.labels_offset, is_training=False, + preprocessing_name=preprocessing_name, + use_grayscale=FLAGS.use_grayscale, + hvd=None, enable_hvd=None) + images, labels = iterator.get_next() # label: [100, 1001] + images = tf.reshape(images, [FLAGS.batch_size, 224, 224, 3]) # (100, 224, 224, 3), float32 + labels = tf.argmax(labels, axis=1) # [100] + logits, _ = network_fn(images) + + if FLAGS.quantize: + contrib_quantize.create_eval_graph() + + predictions = tf.argmax(logits, 1) + labels = tf.squeeze(labels) + eval_accuracy, metric_update_op = tf.metrics.accuracy(labels, predictions) + + # tf.summary.scalar('top1_acc', top1_accu) + # summaries_op = tf.summary.merge_all() + + # TODO(sguada) use num_epochs=1 + if FLAGS.max_num_batches: + num_batches = FLAGS.max_num_batches + else: + # This ensures that we make a single pass over all of the data. + num_batches = math.ceil(dataset.num_samples / float(FLAGS.batch_size)) + if tf.gfile.IsDirectory(FLAGS.checkpoint_path): + checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path) + else: + checkpoint_path = FLAGS.checkpoint_path + + ##### evaluate ##### + tf.logging.info('Evaluating %s' % checkpoint_path) + saver = tf.train.Saver() + from time import gmtime, strftime + logdir = "results/%s" % strftime("%m%d%H%M%S_evel", gmtime()) + # summary_writer = tf.summary.FileWriter(logdir=logdir, graph=tf.get_default_graph()) + with tf.Session() as sess: + sess.run(iterator.initializer) + sess.run(tf.global_variables_initializer()) + sess.run(tf.local_variables_initializer()) + saver.restore(sess, f'{checkpoint_path}') + tf.train.write_graph(sess.graph, logdir, 'graph.pbtxt') + + for step in range(num_batches): + _metric_update_op = sess.run([metric_update_op]) + print(f'{step}, _metric_update_op: {_metric_update_op}') + + acc = sess.run([eval_accuracy]) + print(f'acc: {acc}') + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value=f'{acc}') + + +if __name__ == '__main__': + tf.app.run() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/eval_image_classifier_mobilenet.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/eval_image_classifier_mobilenet.py new file mode 100644 index 0000000..fc46e4a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/eval_image_classifier_mobilenet.py @@ -0,0 +1,181 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Generic evaluation script that evaluates a model using a given dataset.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math +import tensorflow as tf +from tensorflow.contrib import quantize as contrib_quantize +from tensorflow.contrib import slim as contrib_slim + +from datasets import dataset_factory +from nets import nets_factory +from benchmark_log import hwlog +import os +os.environ["CUDA_VISIBLE_DEVICES"] = '4' + +slim = contrib_slim + +tf.app.flags.DEFINE_integer( + 'batch_size', 256, 'The number of samples in each batch.') + +tf.app.flags.DEFINE_integer( + 'max_num_batches', None, + 'Max number of batches to evaluate by default use all.') + +tf.app.flags.DEFINE_string( + 'master', '', 'The address of the TensorFlow master to use.') + +tf.app.flags.DEFINE_string( + 'checkpoint_path', 'ckpt_path', + 'The directory where the model was written to or an absolute path to a ' + 'checkpoint file.') + +tf.app.flags.DEFINE_string( + 'eval_dir', 'ckpt_path', 'Directory where the results are saved to.') + +tf.app.flags.DEFINE_integer( + 'num_preprocessing_threads', 4, + 'The number of threads used to create the batches.') + +tf.app.flags.DEFINE_string( + 'dataset_name', 'imagenet', 'The name of the dataset to load.') + +tf.app.flags.DEFINE_string( + 'dataset_split_name', 'validation', 'The name of the train/test split.') + +tf.app.flags.DEFINE_string( + 'dataset_dir', '/opt/npu/slimImagenet', 'The directory where the dataset files are stored.') + +tf.app.flags.DEFINE_integer( + 'labels_offset', 0, + 'An offset for the labels in the dataset. This flag is primarily used to ' + 'evaluate the VGG and ResNet architectures which do not use a background ' + 'class for the ImageNet dataset.') + +tf.app.flags.DEFINE_string( + 'model_name', 'mobilenet_v2', 'The name of the architecture to evaluate.') + +tf.app.flags.DEFINE_string( + 'preprocessing_name', None, 'The name of the preprocessing to use. If left ' + 'as `None`, then the model_name flag is used.') + +tf.app.flags.DEFINE_float( + 'moving_average_decay', None, + 'The decay to use for the moving average.' + 'If left as None, then moving averages are not used.') + +tf.app.flags.DEFINE_integer( + 'eval_image_size', None, 'Eval image size') + +tf.app.flags.DEFINE_bool( + 'quantize', False, 'whether to use quantized graph or not.') + +tf.app.flags.DEFINE_bool('use_grayscale', False, + 'Whether to convert input images to grayscale.') + +FLAGS = tf.app.flags.FLAGS + + +def main(_): + if not FLAGS.dataset_dir: + raise ValueError('You must supply the dataset directory with --dataset_dir') + + tf.logging.set_verbosity(tf.logging.INFO) + with tf.Graph().as_default(): + tf_global_step = slim.get_or_create_global_step() + + ###################### + # Select the dataset # + ###################### + dataset = dataset_factory.get_dataset( + FLAGS.dataset_name, FLAGS.dataset_split_name, FLAGS.dataset_dir) + + #################### + # Select the model # + #################### + network_fn = nets_factory.get_network_fn( + FLAGS.model_name, + num_classes=(dataset.num_classes - FLAGS.labels_offset), + is_training=False) + + from dataloader import data_provider + preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name + + iterator, _ = data_provider.get_data(dataset, FLAGS.batch_size, + dataset.num_classes, FLAGS.labels_offset, is_training=False, + preprocessing_name=preprocessing_name, + use_grayscale=FLAGS.use_grayscale) + #tf.logging.info('iterator %s' % iterator) + images, labels = iterator.get_next() # label: [100, 1001] + images = tf.reshape(images, [FLAGS.batch_size, 224, 224, 3]) # (100, 224, 224, 3), float32 + labels = tf.argmax(labels, axis=1) # [100] + logits, _ = network_fn(images) + + if FLAGS.quantize: + contrib_quantize.create_eval_graph() + + predictions = tf.argmax(logits, 1) + labels = tf.squeeze(labels) + eval_accuracy, metric_update_op = tf.metrics.accuracy(labels, predictions) + #hwlog.remark_print(key=hwlog.EVAL_ACCURACY, value="".format(eval_accuracy)) + + # tf.summary.scalar('top1_acc', top1_accu) + # summaries_op = tf.summary.merge_all() + + # TODO(sguada) use num_epochs=1 + if FLAGS.max_num_batches: + num_batches = FLAGS.max_num_batches + else: + # This ensures that we make a single pass over all of the data. + num_batches = math.ceil(dataset.num_samples / float(FLAGS.batch_size)) - 1 + + if tf.gfile.IsDirectory(FLAGS.checkpoint_path): + checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path) + else: + checkpoint_path = FLAGS.checkpoint_path + # checkpoint_path = '/opt/npu/models/mobilenetv2_v0.1/ckpt/model.ckpt' + print(dataset.num_samples) + print(FLAGS.batch_size) + hwlog.remark_print(key=hwlog.GLOBAL_BATCH_SIZE, value=FLAGS.batch_size) + ##### evaluate ##### + tf.logging.info('Evaluating %s' % checkpoint_path) + saver = tf.train.Saver() + from time import gmtime, strftime + logdir = "ckpt/%s" % strftime("%m%d%H%M%S_evel", gmtime()) + # summary_writer = tf.summary.FileWriter(logdir=logdir, graph=tf.get_default_graph()) + with tf.Session() as sess: + sess.run(iterator.initializer) + sess.run(tf.global_variables_initializer()) + sess.run(tf.local_variables_initializer()) + saver.restore(sess, f'{checkpoint_path}') + # saver.restore(sess, 'result/8p/2/results/model.ckpt-3750') + tf.train.write_graph(sess.graph, logdir, 'graph.pbtxt') + + for step in range(num_batches): + _metric_update_op = sess.run([metric_update_op]) + print(f'{step}, _metric_update_op: {_metric_update_op}') + hwlog.remark_print(key=hwlog.GLOBAL_STEP, value=f'{step}') + hwlog.remark_print(key=hwlog.EVAL_ACCURACY, value=f'{_metric_update_op}') + acc = sess.run([eval_accuracy]) + print(f'acc: {acc}') + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value=f'{acc}') + + +if __name__ == '__main__': + tf.app.run() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/export_inference_graph.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/export_inference_graph.py new file mode 100644 index 0000000..ba5fb1d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/export_inference_graph.py @@ -0,0 +1,164 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""Saves out a GraphDef containing the architecture of the model. + +To use it, run something like this, with a model name defined by slim: + +bazel build tensorflow_models/research/slim:export_inference_graph +bazel-bin/tensorflow_models/research/slim/export_inference_graph \ +--model_name=inception_v3 --output_file=/tmp/inception_v3_inf_graph.pb + +If you then want to use the resulting model with your own or pretrained +checkpoints as part of a mobile model, you can run freeze_graph to get a graph +def with the variables inlined as constants using: + +bazel build tensorflow/python/tools:freeze_graph +bazel-bin/tensorflow/python/tools/freeze_graph \ +--input_graph=/tmp/inception_v3_inf_graph.pb \ +--input_checkpoint=/tmp/checkpoints/inception_v3.ckpt \ +--input_binary=true --output_graph=/tmp/frozen_inception_v3.pb \ +--output_node_names=InceptionV3/Predictions/Reshape_1 + +The output node names will vary depending on the model, but you can inspect and +estimate them using the summarize_graph tool: + +bazel build tensorflow/tools/graph_transforms:summarize_graph +bazel-bin/tensorflow/tools/graph_transforms/summarize_graph \ +--in_graph=/tmp/inception_v3_inf_graph.pb + +To run the resulting graph in C++, you can look at the label_image sample code: + +bazel build tensorflow/examples/label_image:label_image +bazel-bin/tensorflow/examples/label_image/label_image \ +--image=${HOME}/Pictures/flowers.jpg \ +--input_layer=input \ +--output_layer=InceptionV3/Predictions/Reshape_1 \ +--graph=/tmp/frozen_inception_v3.pb \ +--labels=/tmp/imagenet_slim_labels.txt \ +--input_mean=0 \ +--input_std=255 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import os + +import tensorflow as tf +from tensorflow.contrib import quantize as contrib_quantize +from tensorflow.contrib import slim as contrib_slim + +from tensorflow.python.platform import gfile +from datasets import dataset_factory +from nets import nets_factory + + +slim = contrib_slim + +tf.app.flags.DEFINE_string( + 'model_name', 'inception_v3', 'The name of the architecture to save.') + +tf.app.flags.DEFINE_boolean( + 'is_training', False, + 'Whether to save out a training-focused version of the model.') + +tf.app.flags.DEFINE_integer( + 'image_size', None, + 'The image size to use, otherwise use the model default_image_size.') + +tf.app.flags.DEFINE_integer( + 'batch_size', None, + 'Batch size for the exported model. Defaulted to "None" so batch size can ' + 'be specified at model runtime.') + +tf.app.flags.DEFINE_string('dataset_name', 'imagenet', + 'The name of the dataset to use with the model.') + +tf.app.flags.DEFINE_integer( + 'labels_offset', 0, + 'An offset for the labels in the dataset. This flag is primarily used to ' + 'evaluate the VGG and ResNet architectures which do not use a background ' + 'class for the ImageNet dataset.') + +tf.app.flags.DEFINE_string( + 'output_file', '', 'Where to save the resulting file to.') + +tf.app.flags.DEFINE_string( + 'dataset_dir', '', 'Directory to save intermediate dataset files to') + +tf.app.flags.DEFINE_bool( + 'quantize', False, 'whether to use quantized graph or not.') + +tf.app.flags.DEFINE_bool( + 'is_video_model', False, 'whether to use 5-D inputs for video model.') + +tf.app.flags.DEFINE_integer( + 'num_frames', None, + 'The number of frames to use. Only used if is_video_model is True.') + +tf.app.flags.DEFINE_bool('write_text_graphdef', False, + 'Whether to write a text version of graphdef.') + +tf.app.flags.DEFINE_bool('use_grayscale', False, + 'Whether to convert input images to grayscale.') + +FLAGS = tf.app.flags.FLAGS + + +def main(_): + if not FLAGS.output_file: + raise ValueError('You must supply the path to save to with --output_file') + if FLAGS.is_video_model and not FLAGS.num_frames: + raise ValueError( + 'Number of frames must be specified for video models with --num_frames') + tf.logging.set_verbosity(tf.logging.INFO) + with tf.Graph().as_default() as graph: + dataset = dataset_factory.get_dataset(FLAGS.dataset_name, 'train', + FLAGS.dataset_dir) + network_fn = nets_factory.get_network_fn( + FLAGS.model_name, + num_classes=(dataset.num_classes - FLAGS.labels_offset), + is_training=FLAGS.is_training) + image_size = FLAGS.image_size or network_fn.default_image_size + num_channels = 1 if FLAGS.use_grayscale else 3 + if FLAGS.is_video_model: + input_shape = [ + FLAGS.batch_size, FLAGS.num_frames, image_size, image_size, + num_channels + ] + else: + input_shape = [FLAGS.batch_size, image_size, image_size, num_channels] + placeholder = tf.placeholder(name='input', dtype=tf.float32, + shape=input_shape) + network_fn(placeholder) + + if FLAGS.quantize: + contrib_quantize.create_eval_graph() + + graph_def = graph.as_graph_def() + if FLAGS.write_text_graphdef: + tf.io.write_graph( + graph_def, + os.path.dirname(FLAGS.output_file), + os.path.basename(FLAGS.output_file), + as_text=True) + else: + with gfile.GFile(FLAGS.output_file, 'wb') as f: + f.write(graph_def.SerializeToString()) + + +if __name__ == '__main__': + tf.app.run() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/export_inference_graph_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/export_inference_graph_test.py new file mode 100644 index 0000000..42474f2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/export_inference_graph_test.py @@ -0,0 +1,44 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Tests for export_inference_graph.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + + +import tensorflow as tf + +from tensorflow.python.platform import gfile +import export_inference_graph + + +class ExportInferenceGraphTest(tf.test.TestCase): + + def testExportInferenceGraph(self): + tmpdir = self.get_temp_dir() + output_file = os.path.join(tmpdir, 'inception_v3.pb') + flags = tf.app.flags.FLAGS + flags.output_file = output_file + flags.model_name = 'inception_v3' + flags.dataset_dir = tmpdir + export_inference_graph.main(None) + self.assertTrue(gfile.Exists(output_file)) + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/gpu_helper.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/gpu_helper.py new file mode 100644 index 0000000..5dfaa5b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/gpu_helper.py @@ -0,0 +1,44 @@ +import tensorflow as tf +import numpy as np + +def float32_variable_storage_getter(getter, name, shape=None, dtype=None, + initializer=None, regularizer=None, + trainable=True, + *args, **kwargs): + """Custom variable getter that forces trainable variables to be stored in + float32 precision and then casts them to the training precision. + """ + storage_dtype = tf.float32 if trainable else dtype + variable = getter(name, shape, dtype=storage_dtype, + initializer=initializer, regularizer=regularizer, + trainable=trainable, + *args, **kwargs) + if trainable and dtype != tf.float32: + variable = tf.cast(variable, dtype) + return variable + +def get_custom_getter(compute_type): + return float32_variable_storage_getter if compute_type == tf.float16 else None + + + +def float32_variable_storage_getter_1(getter, name, shape=None, dtype=None, + initializer=None, regularizer=None, + trainable=True, + *args, **kwargs): + """Custom variable getter that forces trainable variables to be stored in + float32 precision and then casts them to the training precision. + """ + dtype = tf.float16 + storage_dtype = tf.float32 if trainable else dtype + variable = getter(name, shape, dtype=storage_dtype, + initializer=initializer, regularizer=regularizer, + trainable=trainable, + *args, **kwargs) + if trainable and dtype != tf.float32: + variable = tf.cast(variable, dtype) + return variable + +def get_custom_getter_1(compute_type): + return float32_variable_storage_getter_1 if compute_type == tf.float16 else None + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/logger.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/logger.py new file mode 100644 index 0000000..b5c583e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/logger.py @@ -0,0 +1,86 @@ +from __future__ import print_function + +import datetime +import logging +import os +import sys +import time +from benchmark_log import hwlog +import numpy as np +import tensorflow as tf + + +class LogSessionRunHook(tf.train.SessionRunHook): + def __init__(self, config, warmup_steps=5): + self.global_batch_size = config['global_batch_size'] + self.iterations_per_loop = config['iterations_per_loop'] + self.warmup_steps = warmup_steps + self.iter_times = [] + self.num_records = config['num_training_samples'] + self.display_every = config['display_every'] + self.logger = get_logger(config['log_name'], config['log_dir']) + rank0log(self.logger, 'PY' + str(sys.version) + 'TF' + str(tf.__version__)) + + def after_create_session(self, session, coord): + rank0log(self.logger, 'Step Epoch Speed Loss FinLoss LR') + self.elapsed_secs = 0. + self.count = 0 + + def before_run(self, run_context): + self.t0 = time.time() + return tf.train.SessionRunArgs( + fetches=[tf.train.get_global_step(), 'loss:0', 'total_loss:0', 'learning_rate:0', + 'train_accuracy:0']) + + def after_run(self, run_context, run_values): + batch_time = time.time() - self.t0 + self.iter_times.append(batch_time) + self.elapsed_secs += batch_time + self.count += 1 + global_step, loss, total_loss, lr, train_accuracy = run_values.results + if global_step == 1 or global_step % self.display_every == 0: + dt = self.elapsed_secs / self.count + img_per_sec = self.global_batch_size * self.iterations_per_loop / dt + epoch = global_step * self.global_batch_size / self.num_records + self.logger.info(f'step:{global_step} epoch:{epoch} ips:{img_per_sec} ' + f'loss:{loss} total_loss:{total_loss} lr:{lr}, ' + f'train_accuracy:{train_accuracy}') + + hwlog.remark_print(key=hwlog.GLOBAL_STEP, value=f"{global_step}") + hwlog.remark_print(key=hwlog.CURRENT_EPOCH, value=f"{epoch}") + hwlog.remark_print(key=hwlog.TRAIN_ACCURACY, value=f"{train_accuracy}") + hwlog.remark_print(key=hwlog.FPS, value=f"{img_per_sec}") + self.elapsed_secs = 0. + self.count = 0 + + def get_average_speed(self): + avg_time = np.mean(self.iter_times[self.warmup_steps:]) + speed = self.global_batch_size / avg_time + return speed + + +def rank0log(logger, *args, **kwargs): + if logger: + logger.info(''.join([str(x) for x in list(args)])) + else: + print(*args, **kwargs) + + +def get_logger(log_name, log_dir): + logger = logging.getLogger(log_name) + logger.setLevel(logging.INFO) # INFO, ERROR + if not os.path.isdir(log_dir): + try: + os.makedirs(log_dir) + except FileExistsError: + pass + ch = logging.StreamHandler() + ch.setLevel(logging.INFO) + formatter = logging.Formatter('%(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) + fh = logging.FileHandler(os.path.join(log_dir, log_name)) + fh.setLevel(logging.DEBUG) + fh.setFormatter(formatter) + logger.addHandler(fh) + return logger diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/__init__.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/__init__.py @@ -0,0 +1 @@ + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/alexnet.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/alexnet.py new file mode 100644 index 0000000..12c04d4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/alexnet.py @@ -0,0 +1,148 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains a model definition for AlexNet. + +This work was first described in: + ImageNet Classification with Deep Convolutional Neural Networks + Alex Krizhevsky, Ilya Sutskever and Geoffrey E. Hinton + +and later refined in: + One weird trick for parallelizing convolutional neural networks + Alex Krizhevsky, 2014 + +Here we provide the implementation proposed in "One weird trick" and not +"ImageNet Classification", as per the paper, the LRN layers have been removed. + +Usage: + with slim.arg_scope(alexnet.alexnet_v2_arg_scope()): + outputs, end_points = alexnet.alexnet_v2(inputs) + +@@alexnet_v2 +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +slim = contrib_slim + +# pylint: disable=g-long-lambda +trunc_normal = lambda stddev: tf.compat.v1.truncated_normal_initializer( + 0.0, stddev) + + +def alexnet_v2_arg_scope(weight_decay=0.0005): + with slim.arg_scope([slim.conv2d, slim.fully_connected], + activation_fn=tf.nn.relu, + biases_initializer=tf.compat.v1.constant_initializer(0.1), + weights_regularizer=slim.l2_regularizer(weight_decay)): + with slim.arg_scope([slim.conv2d], padding='SAME'): + with slim.arg_scope([slim.max_pool2d], padding='VALID') as arg_sc: + return arg_sc + + +def alexnet_v2(inputs, + num_classes=1000, + is_training=True, + dropout_keep_prob=0.5, + spatial_squeeze=True, + scope='alexnet_v2', + global_pool=False): + """AlexNet version 2. + + Described in: http://arxiv.org/pdf/1404.5997v2.pdf + Parameters from: + github.com/akrizhevsky/cuda-convnet2/blob/master/layers/ + layers-imagenet-1gpu.cfg + + Note: All the fully_connected layers have been transformed to conv2d layers. + To use in classification mode, resize input to 224x224 or set + global_pool=True. To use in fully convolutional mode, set + spatial_squeeze to false. + The LRN layers have been removed and change the initializers from + random_normal_initializer to xavier_initializer. + + Args: + inputs: a tensor of size [batch_size, height, width, channels]. + num_classes: the number of predicted classes. If 0 or None, the logits layer + is omitted and the input features to the logits layer are returned instead. + is_training: whether or not the model is being trained. + dropout_keep_prob: the probability that activations are kept in the dropout + layers during training. + spatial_squeeze: whether or not should squeeze the spatial dimensions of the + logits. Useful to remove unnecessary dimensions for classification. + scope: Optional scope for the variables. + global_pool: Optional boolean flag. If True, the input to the classification + layer is avgpooled to size 1x1, for any input size. (This is not part + of the original AlexNet.) + + Returns: + net: the output of the logits layer (if num_classes is a non-zero integer), + or the non-dropped-out input to the logits layer (if num_classes is 0 + or None). + end_points: a dict of tensors with intermediate activations. + """ + with tf.compat.v1.variable_scope(scope, 'alexnet_v2', [inputs]) as sc: + end_points_collection = sc.original_name_scope + '_end_points' + # Collect outputs for conv2d, fully_connected and max_pool2d. + with slim.arg_scope([slim.conv2d, slim.fully_connected, slim.max_pool2d], + outputs_collections=[end_points_collection]): + net = slim.conv2d(inputs, 64, [11, 11], 4, padding='VALID', + scope='conv1') + net = slim.max_pool2d(net, [3, 3], 2, scope='pool1') + net = slim.conv2d(net, 192, [5, 5], scope='conv2') + net = slim.max_pool2d(net, [3, 3], 2, scope='pool2') + net = slim.conv2d(net, 384, [3, 3], scope='conv3') + net = slim.conv2d(net, 384, [3, 3], scope='conv4') + net = slim.conv2d(net, 256, [3, 3], scope='conv5') + net = slim.max_pool2d(net, [3, 3], 2, scope='pool5') + + # Use conv2d instead of fully_connected layers. + with slim.arg_scope( + [slim.conv2d], + weights_initializer=trunc_normal(0.005), + biases_initializer=tf.compat.v1.constant_initializer(0.1)): + net = slim.conv2d(net, 4096, [5, 5], padding='VALID', + scope='fc6') + net = slim.dropout(net, dropout_keep_prob, is_training=is_training, + scope='dropout6') + net = slim.conv2d(net, 4096, [1, 1], scope='fc7') + # Convert end_points_collection into a end_point dict. + end_points = slim.utils.convert_collection_to_dict( + end_points_collection) + if global_pool: + net = tf.reduce_mean( + input_tensor=net, axis=[1, 2], keepdims=True, name='global_pool') + end_points['global_pool'] = net + if num_classes: + net = slim.dropout(net, dropout_keep_prob, is_training=is_training, + scope='dropout7') + net = slim.conv2d( + net, + num_classes, [1, 1], + activation_fn=None, + normalizer_fn=None, + biases_initializer=tf.compat.v1.zeros_initializer(), + scope='fc8') + if spatial_squeeze: + net = tf.squeeze(net, [1, 2], name='fc8/squeezed') + end_points[sc.name + '/fc8'] = net + return net, end_points + + +alexnet_v2.default_image_size = 224 diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/alexnet_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/alexnet_test.py new file mode 100644 index 0000000..b6fcdf4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/alexnet_test.py @@ -0,0 +1,181 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for slim.nets.alexnet.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets import alexnet + +slim = contrib_slim + + +class AlexnetV2Test(tf.test.TestCase): + + def testBuild(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = alexnet.alexnet_v2(inputs, num_classes) + self.assertEquals(logits.op.name, 'alexnet_v2/fc8/squeezed') + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + + def testFullyConvolutional(self): + batch_size = 1 + height, width = 300, 400 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = alexnet.alexnet_v2(inputs, num_classes, spatial_squeeze=False) + self.assertEquals(logits.op.name, 'alexnet_v2/fc8/BiasAdd') + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, 4, 7, num_classes]) + + def testGlobalPool(self): + batch_size = 1 + height, width = 256, 256 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = alexnet.alexnet_v2(inputs, num_classes, spatial_squeeze=False, + global_pool=True) + self.assertEquals(logits.op.name, 'alexnet_v2/fc8/BiasAdd') + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, 1, 1, num_classes]) + + def testEndPoints(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = alexnet.alexnet_v2(inputs, num_classes) + expected_names = ['alexnet_v2/conv1', + 'alexnet_v2/pool1', + 'alexnet_v2/conv2', + 'alexnet_v2/pool2', + 'alexnet_v2/conv3', + 'alexnet_v2/conv4', + 'alexnet_v2/conv5', + 'alexnet_v2/pool5', + 'alexnet_v2/fc6', + 'alexnet_v2/fc7', + 'alexnet_v2/fc8' + ] + self.assertSetEqual(set(end_points.keys()), set(expected_names)) + + def testNoClasses(self): + batch_size = 5 + height, width = 224, 224 + num_classes = None + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + net, end_points = alexnet.alexnet_v2(inputs, num_classes) + expected_names = ['alexnet_v2/conv1', + 'alexnet_v2/pool1', + 'alexnet_v2/conv2', + 'alexnet_v2/pool2', + 'alexnet_v2/conv3', + 'alexnet_v2/conv4', + 'alexnet_v2/conv5', + 'alexnet_v2/pool5', + 'alexnet_v2/fc6', + 'alexnet_v2/fc7' + ] + self.assertSetEqual(set(end_points.keys()), set(expected_names)) + self.assertTrue(net.op.name.startswith('alexnet_v2/fc7')) + self.assertListEqual(net.get_shape().as_list(), + [batch_size, 1, 1, 4096]) + + def testModelVariables(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + alexnet.alexnet_v2(inputs, num_classes) + expected_names = ['alexnet_v2/conv1/weights', + 'alexnet_v2/conv1/biases', + 'alexnet_v2/conv2/weights', + 'alexnet_v2/conv2/biases', + 'alexnet_v2/conv3/weights', + 'alexnet_v2/conv3/biases', + 'alexnet_v2/conv4/weights', + 'alexnet_v2/conv4/biases', + 'alexnet_v2/conv5/weights', + 'alexnet_v2/conv5/biases', + 'alexnet_v2/fc6/weights', + 'alexnet_v2/fc6/biases', + 'alexnet_v2/fc7/weights', + 'alexnet_v2/fc7/biases', + 'alexnet_v2/fc8/weights', + 'alexnet_v2/fc8/biases', + ] + model_variables = [v.op.name for v in slim.get_model_variables()] + self.assertSetEqual(set(model_variables), set(expected_names)) + + def testEvaluation(self): + batch_size = 2 + height, width = 224, 224 + num_classes = 1000 + with self.test_session(): + eval_inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = alexnet.alexnet_v2(eval_inputs, is_training=False) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + predictions = tf.argmax(input=logits, axis=1) + self.assertListEqual(predictions.get_shape().as_list(), [batch_size]) + + def testTrainEvalWithReuse(self): + train_batch_size = 2 + eval_batch_size = 1 + train_height, train_width = 224, 224 + eval_height, eval_width = 300, 400 + num_classes = 1000 + with self.test_session(): + train_inputs = tf.random.uniform( + (train_batch_size, train_height, train_width, 3)) + logits, _ = alexnet.alexnet_v2(train_inputs) + self.assertListEqual(logits.get_shape().as_list(), + [train_batch_size, num_classes]) + tf.compat.v1.get_variable_scope().reuse_variables() + eval_inputs = tf.random.uniform( + (eval_batch_size, eval_height, eval_width, 3)) + logits, _ = alexnet.alexnet_v2(eval_inputs, is_training=False, + spatial_squeeze=False) + self.assertListEqual(logits.get_shape().as_list(), + [eval_batch_size, 4, 7, num_classes]) + logits = tf.reduce_mean(input_tensor=logits, axis=[1, 2]) + predictions = tf.argmax(input=logits, axis=1) + self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) + + def testForward(self): + batch_size = 1 + height, width = 224, 224 + with self.test_session() as sess: + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = alexnet.alexnet_v2(inputs) + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(logits) + self.assertTrue(output.any()) + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/cifarnet.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/cifarnet.py new file mode 100644 index 0000000..1dae82c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/cifarnet.py @@ -0,0 +1,123 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains a variant of the CIFAR-10 model definition.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +slim = contrib_slim + +# pylint: disable=g-long-lambda +trunc_normal = lambda stddev: tf.compat.v1.truncated_normal_initializer( + stddev=stddev) + + +def cifarnet(images, num_classes=10, is_training=False, + dropout_keep_prob=0.5, + prediction_fn=slim.softmax, + scope='CifarNet'): + """Creates a variant of the CifarNet model. + + Note that since the output is a set of 'logits', the values fall in the + interval of (-infinity, infinity). Consequently, to convert the outputs to a + probability distribution over the characters, one will need to convert them + using the softmax function: + + logits = cifarnet.cifarnet(images, is_training=False) + probabilities = tf.nn.softmax(logits) + predictions = tf.argmax(logits, 1) + + Args: + images: A batch of `Tensors` of size [batch_size, height, width, channels]. + num_classes: the number of classes in the dataset. If 0 or None, the logits + layer is omitted and the input features to the logits layer are returned + instead. + is_training: specifies whether or not we're currently training the model. + This variable will determine the behaviour of the dropout layer. + dropout_keep_prob: the percentage of activation values that are retained. + prediction_fn: a function to get predictions out of logits. + scope: Optional variable_scope. + + Returns: + net: a 2D Tensor with the logits (pre-softmax activations) if num_classes + is a non-zero integer, or the input to the logits layer if num_classes + is 0 or None. + end_points: a dictionary from components of the network to the corresponding + activation. + """ + end_points = {} + + with tf.compat.v1.variable_scope(scope, 'CifarNet', [images]): + net = slim.conv2d(images, 64, [5, 5], scope='conv1') + end_points['conv1'] = net + net = slim.max_pool2d(net, [2, 2], 2, scope='pool1') + end_points['pool1'] = net + net = tf.nn.lrn(net, 4, bias=1.0, alpha=0.001/9.0, beta=0.75, name='norm1') + net = slim.conv2d(net, 64, [5, 5], scope='conv2') + end_points['conv2'] = net + net = tf.nn.lrn(net, 4, bias=1.0, alpha=0.001/9.0, beta=0.75, name='norm2') + net = slim.max_pool2d(net, [2, 2], 2, scope='pool2') + end_points['pool2'] = net + net = slim.flatten(net) + end_points['Flatten'] = net + net = slim.fully_connected(net, 384, scope='fc3') + end_points['fc3'] = net + net = slim.dropout(net, dropout_keep_prob, is_training=is_training, + scope='dropout3') + net = slim.fully_connected(net, 192, scope='fc4') + end_points['fc4'] = net + if not num_classes: + return net, end_points + logits = slim.fully_connected( + net, + num_classes, + biases_initializer=tf.compat.v1.zeros_initializer(), + weights_initializer=trunc_normal(1 / 192.0), + weights_regularizer=None, + activation_fn=None, + scope='logits') + + end_points['Logits'] = logits + end_points['Predictions'] = prediction_fn(logits, scope='Predictions') + + return logits, end_points +cifarnet.default_image_size = 32 + + +def cifarnet_arg_scope(weight_decay=0.004): + """Defines the default cifarnet argument scope. + + Args: + weight_decay: The weight decay to use for regularizing the model. + + Returns: + An `arg_scope` to use for the inception v3 model. + """ + with slim.arg_scope( + [slim.conv2d], + weights_initializer=tf.compat.v1.truncated_normal_initializer( + stddev=5e-2), + activation_fn=tf.nn.relu): + with slim.arg_scope( + [slim.fully_connected], + biases_initializer=tf.compat.v1.constant_initializer(0.1), + weights_initializer=trunc_normal(0.04), + weights_regularizer=slim.l2_regularizer(weight_decay), + activation_fn=tf.nn.relu) as sc: + return sc diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/cyclegan.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/cyclegan.py new file mode 100644 index 0000000..7c64237 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/cyclegan.py @@ -0,0 +1,280 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Defines the CycleGAN generator and discriminator networks.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +from six.moves import xrange # pylint: disable=redefined-builtin +import tensorflow as tf +from tensorflow.contrib import framework as contrib_framework +from tensorflow.contrib import layers as contrib_layers +from tensorflow.contrib import util as contrib_util + +layers = contrib_layers + + +def cyclegan_arg_scope(instance_norm_center=True, + instance_norm_scale=True, + instance_norm_epsilon=0.001, + weights_init_stddev=0.02, + weight_decay=0.0): + """Returns a default argument scope for all generators and discriminators. + + Args: + instance_norm_center: Whether instance normalization applies centering. + instance_norm_scale: Whether instance normalization applies scaling. + instance_norm_epsilon: Small float added to the variance in the instance + normalization to avoid dividing by zero. + weights_init_stddev: Standard deviation of the random values to initialize + the convolution kernels with. + weight_decay: Magnitude of weight decay applied to all convolution kernel + variables of the generator. + + Returns: + An arg-scope. + """ + instance_norm_params = { + 'center': instance_norm_center, + 'scale': instance_norm_scale, + 'epsilon': instance_norm_epsilon, + } + + weights_regularizer = None + if weight_decay and weight_decay > 0.0: + weights_regularizer = layers.l2_regularizer(weight_decay) + + with contrib_framework.arg_scope( + [layers.conv2d], + normalizer_fn=layers.instance_norm, + normalizer_params=instance_norm_params, + weights_initializer=tf.compat.v1.random_normal_initializer( + 0, weights_init_stddev), + weights_regularizer=weights_regularizer) as sc: + return sc + + +def cyclegan_upsample(net, num_outputs, stride, method='conv2d_transpose', + pad_mode='REFLECT', align_corners=False): + """Upsamples the given inputs. + + Args: + net: A Tensor of size [batch_size, height, width, filters]. + num_outputs: The number of output filters. + stride: A list of 2 scalars or a 1x2 Tensor indicating the scale, + relative to the inputs, of the output dimensions. For example, if kernel + size is [2, 3], then the output height and width will be twice and three + times the input size. + method: The upsampling method: 'nn_upsample_conv', 'bilinear_upsample_conv', + or 'conv2d_transpose'. + pad_mode: mode for tf.pad, one of "CONSTANT", "REFLECT", or "SYMMETRIC". + align_corners: option for method, 'bilinear_upsample_conv'. If true, the + centers of the 4 corner pixels of the input and output tensors are + aligned, preserving the values at the corner pixels. + + Returns: + A Tensor which was upsampled using the specified method. + + Raises: + ValueError: if `method` is not recognized. + """ + with tf.compat.v1.variable_scope('upconv'): + net_shape = tf.shape(input=net) + height = net_shape[1] + width = net_shape[2] + + # Reflection pad by 1 in spatial dimensions (axes 1, 2 = h, w) to make a 3x3 + # 'valid' convolution produce an output with the same dimension as the + # input. + spatial_pad_1 = np.array([[0, 0], [1, 1], [1, 1], [0, 0]]) + + if method == 'nn_upsample_conv': + net = tf.image.resize( + net, [stride[0] * height, stride[1] * width], + method=tf.image.ResizeMethod.NEAREST_NEIGHBOR) + net = tf.pad(tensor=net, paddings=spatial_pad_1, mode=pad_mode) + net = layers.conv2d(net, num_outputs, kernel_size=[3, 3], padding='valid') + elif method == 'bilinear_upsample_conv': + net = tf.compat.v1.image.resize_bilinear( + net, [stride[0] * height, stride[1] * width], + align_corners=align_corners) + net = tf.pad(tensor=net, paddings=spatial_pad_1, mode=pad_mode) + net = layers.conv2d(net, num_outputs, kernel_size=[3, 3], padding='valid') + elif method == 'conv2d_transpose': + # This corrects 1 pixel offset for images with even width and height. + # conv2d is left aligned and conv2d_transpose is right aligned for even + # sized images (while doing 'SAME' padding). + # Note: This doesn't reflect actual model in paper. + net = layers.conv2d_transpose( + net, num_outputs, kernel_size=[3, 3], stride=stride, padding='valid') + net = net[:, 1:, 1:, :] + else: + raise ValueError('Unknown method: [%s]' % method) + + return net + + +def _dynamic_or_static_shape(tensor): + shape = tf.shape(input=tensor) + static_shape = contrib_util.constant_value(shape) + return static_shape if static_shape is not None else shape + + +def cyclegan_generator_resnet(images, + arg_scope_fn=cyclegan_arg_scope, + num_resnet_blocks=6, + num_filters=64, + upsample_fn=cyclegan_upsample, + kernel_size=3, + tanh_linear_slope=0.0, + is_training=False): + """Defines the cyclegan resnet network architecture. + + As closely as possible following + https://github.com/junyanz/CycleGAN/blob/master/models/architectures.lua#L232 + + FYI: This network requires input height and width to be divisible by 4 in + order to generate an output with shape equal to input shape. Assertions will + catch this if input dimensions are known at graph construction time, but + there's no protection if unknown at graph construction time (you'll see an + error). + + Args: + images: Input image tensor of shape [batch_size, h, w, 3]. + arg_scope_fn: Function to create the global arg_scope for the network. + num_resnet_blocks: Number of ResNet blocks in the middle of the generator. + num_filters: Number of filters of the first hidden layer. + upsample_fn: Upsampling function for the decoder part of the generator. + kernel_size: Size w or list/tuple [h, w] of the filter kernels for all inner + layers. + tanh_linear_slope: Slope of the linear function to add to the tanh over the + logits. + is_training: Whether the network is created in training mode or inference + only mode. Not actually needed, just for compliance with other generator + network functions. + + Returns: + A `Tensor` representing the model output and a dictionary of model end + points. + + Raises: + ValueError: If the input height or width is known at graph construction time + and not a multiple of 4. + """ + # Neither dropout nor batch norm -> dont need is_training + del is_training + + end_points = {} + + input_size = images.shape.as_list() + height, width = input_size[1], input_size[2] + if height and height % 4 != 0: + raise ValueError('The input height must be a multiple of 4.') + if width and width % 4 != 0: + raise ValueError('The input width must be a multiple of 4.') + num_outputs = input_size[3] + + if not isinstance(kernel_size, (list, tuple)): + kernel_size = [kernel_size, kernel_size] + + kernel_height = kernel_size[0] + kernel_width = kernel_size[1] + pad_top = (kernel_height - 1) // 2 + pad_bottom = kernel_height // 2 + pad_left = (kernel_width - 1) // 2 + pad_right = kernel_width // 2 + paddings = np.array( + [[0, 0], [pad_top, pad_bottom], [pad_left, pad_right], [0, 0]], + dtype=np.int32) + spatial_pad_3 = np.array([[0, 0], [3, 3], [3, 3], [0, 0]]) + + with contrib_framework.arg_scope(arg_scope_fn()): + + ########### + # Encoder # + ########### + with tf.compat.v1.variable_scope('input'): + # 7x7 input stage + net = tf.pad(tensor=images, paddings=spatial_pad_3, mode='REFLECT') + net = layers.conv2d(net, num_filters, kernel_size=[7, 7], padding='VALID') + end_points['encoder_0'] = net + + with tf.compat.v1.variable_scope('encoder'): + with contrib_framework.arg_scope([layers.conv2d], + kernel_size=kernel_size, + stride=2, + activation_fn=tf.nn.relu, + padding='VALID'): + + net = tf.pad(tensor=net, paddings=paddings, mode='REFLECT') + net = layers.conv2d(net, num_filters * 2) + end_points['encoder_1'] = net + net = tf.pad(tensor=net, paddings=paddings, mode='REFLECT') + net = layers.conv2d(net, num_filters * 4) + end_points['encoder_2'] = net + + ################### + # Residual Blocks # + ################### + with tf.compat.v1.variable_scope('residual_blocks'): + with contrib_framework.arg_scope([layers.conv2d], + kernel_size=kernel_size, + stride=1, + activation_fn=tf.nn.relu, + padding='VALID'): + for block_id in xrange(num_resnet_blocks): + with tf.compat.v1.variable_scope('block_{}'.format(block_id)): + res_net = tf.pad(tensor=net, paddings=paddings, mode='REFLECT') + res_net = layers.conv2d(res_net, num_filters * 4) + res_net = tf.pad(tensor=res_net, paddings=paddings, mode='REFLECT') + res_net = layers.conv2d(res_net, num_filters * 4, + activation_fn=None) + net += res_net + + end_points['resnet_block_%d' % block_id] = net + + ########### + # Decoder # + ########### + with tf.compat.v1.variable_scope('decoder'): + + with contrib_framework.arg_scope([layers.conv2d], + kernel_size=kernel_size, + stride=1, + activation_fn=tf.nn.relu): + + with tf.compat.v1.variable_scope('decoder1'): + net = upsample_fn(net, num_outputs=num_filters * 2, stride=[2, 2]) + end_points['decoder1'] = net + + with tf.compat.v1.variable_scope('decoder2'): + net = upsample_fn(net, num_outputs=num_filters, stride=[2, 2]) + end_points['decoder2'] = net + + with tf.compat.v1.variable_scope('output'): + net = tf.pad(tensor=net, paddings=spatial_pad_3, mode='REFLECT') + logits = layers.conv2d( + net, + num_outputs, [7, 7], + activation_fn=None, + normalizer_fn=None, + padding='valid') + logits = tf.reshape(logits, _dynamic_or_static_shape(images)) + + end_points['logits'] = logits + end_points['predictions'] = tf.tanh(logits) + logits * tanh_linear_slope + + return end_points['predictions'], end_points diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/cyclegan_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/cyclegan_test.py new file mode 100644 index 0000000..96f0e24 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/cyclegan_test.py @@ -0,0 +1,110 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for tensorflow.contrib.slim.nets.cyclegan.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from nets import cyclegan + + +# TODO(joelshor): Add a test to check generator endpoints. +class CycleganTest(tf.test.TestCase): + + def test_generator_inference(self): + """Check one inference step.""" + img_batch = tf.zeros([2, 32, 32, 3]) + model_output, _ = cyclegan.cyclegan_generator_resnet(img_batch) + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + sess.run(model_output) + + def _test_generator_graph_helper(self, shape): + """Check that generator can take small and non-square inputs.""" + output_imgs, _ = cyclegan.cyclegan_generator_resnet(tf.ones(shape)) + self.assertAllEqual(shape, output_imgs.shape.as_list()) + + def test_generator_graph_small(self): + self._test_generator_graph_helper([4, 32, 32, 3]) + + def test_generator_graph_medium(self): + self._test_generator_graph_helper([3, 128, 128, 3]) + + def test_generator_graph_nonsquare(self): + self._test_generator_graph_helper([2, 80, 400, 3]) + + def test_generator_unknown_batch_dim(self): + """Check that generator can take unknown batch dimension inputs.""" + img = tf.compat.v1.placeholder(tf.float32, shape=[None, 32, None, 3]) + output_imgs, _ = cyclegan.cyclegan_generator_resnet(img) + + self.assertAllEqual([None, 32, None, 3], output_imgs.shape.as_list()) + + def _input_and_output_same_shape_helper(self, kernel_size): + img_batch = tf.compat.v1.placeholder(tf.float32, shape=[None, 32, 32, 3]) + output_img_batch, _ = cyclegan.cyclegan_generator_resnet( + img_batch, kernel_size=kernel_size) + + self.assertAllEqual(img_batch.shape.as_list(), + output_img_batch.shape.as_list()) + + def input_and_output_same_shape_kernel3(self): + self._input_and_output_same_shape_helper(3) + + def input_and_output_same_shape_kernel4(self): + self._input_and_output_same_shape_helper(4) + + def input_and_output_same_shape_kernel5(self): + self._input_and_output_same_shape_helper(5) + + def input_and_output_same_shape_kernel6(self): + self._input_and_output_same_shape_helper(6) + + def _error_if_height_not_multiple_of_four_helper(self, height): + self.assertRaisesRegexp( + ValueError, 'The input height must be a multiple of 4.', + cyclegan.cyclegan_generator_resnet, + tf.compat.v1.placeholder(tf.float32, shape=[None, height, 32, 3])) + + def test_error_if_height_not_multiple_of_four_height29(self): + self._error_if_height_not_multiple_of_four_helper(29) + + def test_error_if_height_not_multiple_of_four_height30(self): + self._error_if_height_not_multiple_of_four_helper(30) + + def test_error_if_height_not_multiple_of_four_height31(self): + self._error_if_height_not_multiple_of_four_helper(31) + + def _error_if_width_not_multiple_of_four_helper(self, width): + self.assertRaisesRegexp( + ValueError, 'The input width must be a multiple of 4.', + cyclegan.cyclegan_generator_resnet, + tf.compat.v1.placeholder(tf.float32, shape=[None, 32, width, 3])) + + def test_error_if_width_not_multiple_of_four_width29(self): + self._error_if_width_not_multiple_of_four_helper(29) + + def test_error_if_width_not_multiple_of_four_width30(self): + self._error_if_width_not_multiple_of_four_helper(30) + + def test_error_if_width_not_multiple_of_four_width31(self): + self._error_if_width_not_multiple_of_four_helper(31) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/dcgan.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/dcgan.py new file mode 100644 index 0000000..598b642 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/dcgan.py @@ -0,0 +1,205 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""DCGAN generator and discriminator from https://arxiv.org/abs/1511.06434.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from math import log + +from six.moves import xrange # pylint: disable=redefined-builtin +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +slim = contrib_slim + + +def _validate_image_inputs(inputs): + inputs.get_shape().assert_has_rank(4) + inputs.get_shape()[1:3].assert_is_fully_defined() + if inputs.get_shape()[1] != inputs.get_shape()[2]: + raise ValueError('Input tensor does not have equal width and height: ', + inputs.get_shape()[1:3]) + width = inputs.get_shape().as_list()[1] + if log(width, 2) != int(log(width, 2)): + raise ValueError('Input tensor `width` is not a power of 2: ', width) + + +# TODO(joelshor): Use fused batch norm by default. Investigate why some GAN +# setups need the gradient of gradient FusedBatchNormGrad. +def discriminator(inputs, + depth=64, + is_training=True, + reuse=None, + scope='Discriminator', + fused_batch_norm=False): + """Discriminator network for DCGAN. + + Construct discriminator network from inputs to the final endpoint. + + Args: + inputs: A tensor of size [batch_size, height, width, channels]. Must be + floating point. + depth: Number of channels in first convolution layer. + is_training: Whether the network is for training or not. + reuse: Whether or not the network variables should be reused. `scope` + must be given to be reused. + scope: Optional variable_scope. + fused_batch_norm: If `True`, use a faster, fused implementation of + batch norm. + + Returns: + logits: The pre-softmax activations, a tensor of size [batch_size, 1] + end_points: a dictionary from components of the network to their activation. + + Raises: + ValueError: If the input image shape is not 4-dimensional, if the spatial + dimensions aren't defined at graph construction time, if the spatial + dimensions aren't square, or if the spatial dimensions aren't a power of + two. + """ + + normalizer_fn = slim.batch_norm + normalizer_fn_args = { + 'is_training': is_training, + 'zero_debias_moving_mean': True, + 'fused': fused_batch_norm, + } + + _validate_image_inputs(inputs) + inp_shape = inputs.get_shape().as_list()[1] + + end_points = {} + with tf.compat.v1.variable_scope( + scope, values=[inputs], reuse=reuse) as scope: + with slim.arg_scope([normalizer_fn], **normalizer_fn_args): + with slim.arg_scope([slim.conv2d], + stride=2, + kernel_size=4, + activation_fn=tf.nn.leaky_relu): + net = inputs + for i in xrange(int(log(inp_shape, 2))): + scope = 'conv%i' % (i + 1) + current_depth = depth * 2**i + normalizer_fn_ = None if i == 0 else normalizer_fn + net = slim.conv2d( + net, current_depth, normalizer_fn=normalizer_fn_, scope=scope) + end_points[scope] = net + + logits = slim.conv2d(net, 1, kernel_size=1, stride=1, padding='VALID', + normalizer_fn=None, activation_fn=None) + logits = tf.reshape(logits, [-1, 1]) + end_points['logits'] = logits + + return logits, end_points + + +# TODO(joelshor): Use fused batch norm by default. Investigate why some GAN +# setups need the gradient of gradient FusedBatchNormGrad. +def generator(inputs, + depth=64, + final_size=32, + num_outputs=3, + is_training=True, + reuse=None, + scope='Generator', + fused_batch_norm=False): + """Generator network for DCGAN. + + Construct generator network from inputs to the final endpoint. + + Args: + inputs: A tensor with any size N. [batch_size, N] + depth: Number of channels in last deconvolution layer. + final_size: The shape of the final output. + num_outputs: Number of output features. For images, this is the number of + channels. + is_training: whether is training or not. + reuse: Whether or not the network has its variables should be reused. scope + must be given to be reused. + scope: Optional variable_scope. + fused_batch_norm: If `True`, use a faster, fused implementation of + batch norm. + + Returns: + logits: the pre-softmax activations, a tensor of size + [batch_size, 32, 32, channels] + end_points: a dictionary from components of the network to their activation. + + Raises: + ValueError: If `inputs` is not 2-dimensional. + ValueError: If `final_size` isn't a power of 2 or is less than 8. + """ + normalizer_fn = slim.batch_norm + normalizer_fn_args = { + 'is_training': is_training, + 'zero_debias_moving_mean': True, + 'fused': fused_batch_norm, + } + + inputs.get_shape().assert_has_rank(2) + if log(final_size, 2) != int(log(final_size, 2)): + raise ValueError('`final_size` (%i) must be a power of 2.' % final_size) + if final_size < 8: + raise ValueError('`final_size` (%i) must be greater than 8.' % final_size) + + end_points = {} + num_layers = int(log(final_size, 2)) - 1 + with tf.compat.v1.variable_scope( + scope, values=[inputs], reuse=reuse) as scope: + with slim.arg_scope([normalizer_fn], **normalizer_fn_args): + with slim.arg_scope([slim.conv2d_transpose], + normalizer_fn=normalizer_fn, + stride=2, + kernel_size=4): + net = tf.expand_dims(tf.expand_dims(inputs, 1), 1) + + # First upscaling is different because it takes the input vector. + current_depth = depth * 2 ** (num_layers - 1) + scope = 'deconv1' + net = slim.conv2d_transpose( + net, current_depth, stride=1, padding='VALID', scope=scope) + end_points[scope] = net + + for i in xrange(2, num_layers): + scope = 'deconv%i' % (i) + current_depth = depth * 2 ** (num_layers - i) + net = slim.conv2d_transpose(net, current_depth, scope=scope) + end_points[scope] = net + + # Last layer has different normalizer and activation. + scope = 'deconv%i' % (num_layers) + net = slim.conv2d_transpose( + net, depth, normalizer_fn=None, activation_fn=None, scope=scope) + end_points[scope] = net + + # Convert to proper channels. + scope = 'logits' + logits = slim.conv2d( + net, + num_outputs, + normalizer_fn=None, + activation_fn=None, + kernel_size=1, + stride=1, + padding='VALID', + scope=scope) + end_points[scope] = logits + + logits.get_shape().assert_has_rank(4) + logits.get_shape().assert_is_compatible_with( + [None, final_size, final_size, num_outputs]) + + return logits, end_points diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/dcgan_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/dcgan_test.py new file mode 100644 index 0000000..53fd9fb --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/dcgan_test.py @@ -0,0 +1,121 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for dcgan.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from six.moves import xrange # pylint: disable=redefined-builtin +import tensorflow as tf + +from nets import dcgan + + +class DCGANTest(tf.test.TestCase): + + def test_generator_run(self): + tf.compat.v1.set_random_seed(1234) + noise = tf.random.normal([100, 64]) + image, _ = dcgan.generator(noise) + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + image.eval() + + def test_generator_graph(self): + tf.compat.v1.set_random_seed(1234) + # Check graph construction for a number of image size/depths and batch + # sizes. + for i, batch_size in zip(xrange(3, 7), xrange(3, 8)): + tf.compat.v1.reset_default_graph() + final_size = 2 ** i + noise = tf.random.normal([batch_size, 64]) + image, end_points = dcgan.generator( + noise, + depth=32, + final_size=final_size) + + self.assertAllEqual([batch_size, final_size, final_size, 3], + image.shape.as_list()) + + expected_names = ['deconv%i' % j for j in xrange(1, i)] + ['logits'] + self.assertSetEqual(set(expected_names), set(end_points.keys())) + + # Check layer depths. + for j in range(1, i): + layer = end_points['deconv%i' % j] + self.assertEqual(32 * 2**(i-j-1), layer.get_shape().as_list()[-1]) + + def test_generator_invalid_input(self): + wrong_dim_input = tf.zeros([5, 32, 32]) + with self.assertRaises(ValueError): + dcgan.generator(wrong_dim_input) + + correct_input = tf.zeros([3, 2]) + with self.assertRaisesRegexp(ValueError, 'must be a power of 2'): + dcgan.generator(correct_input, final_size=30) + + with self.assertRaisesRegexp(ValueError, 'must be greater than 8'): + dcgan.generator(correct_input, final_size=4) + + def test_discriminator_run(self): + image = tf.random.uniform([5, 32, 32, 3], -1, 1) + output, _ = dcgan.discriminator(image) + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output.eval() + + def test_discriminator_graph(self): + # Check graph construction for a number of image size/depths and batch + # sizes. + for i, batch_size in zip(xrange(1, 6), xrange(3, 8)): + tf.compat.v1.reset_default_graph() + img_w = 2 ** i + image = tf.random.uniform([batch_size, img_w, img_w, 3], -1, 1) + output, end_points = dcgan.discriminator( + image, + depth=32) + + self.assertAllEqual([batch_size, 1], output.get_shape().as_list()) + + expected_names = ['conv%i' % j for j in xrange(1, i+1)] + ['logits'] + self.assertSetEqual(set(expected_names), set(end_points.keys())) + + # Check layer depths. + for j in range(1, i+1): + layer = end_points['conv%i' % j] + self.assertEqual(32 * 2**(j-1), layer.get_shape().as_list()[-1]) + + def test_discriminator_invalid_input(self): + wrong_dim_img = tf.zeros([5, 32, 32]) + with self.assertRaises(ValueError): + dcgan.discriminator(wrong_dim_img) + + spatially_undefined_shape = tf.compat.v1.placeholder( + tf.float32, [5, 32, None, 3]) + with self.assertRaises(ValueError): + dcgan.discriminator(spatially_undefined_shape) + + not_square = tf.zeros([5, 32, 16, 3]) + with self.assertRaisesRegexp(ValueError, 'not have equal width and height'): + dcgan.discriminator(not_square) + + not_power_2 = tf.zeros([5, 30, 30, 3]) + with self.assertRaisesRegexp(ValueError, 'not a power of 2'): + dcgan.discriminator(not_power_2) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/i3d.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/i3d.py new file mode 100644 index 0000000..28974ea --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/i3d.py @@ -0,0 +1,181 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains the definition for Inflated 3D Inception V1 (I3D). + +The network architecture is proposed by: + Joao Carreira and Andrew Zisserman, + Quo Vadis, Action Recognition? A New Model and the Kinetics Dataset. + https://arxiv.org/abs/1705.07750 +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets import i3d_utils +from nets import s3dg + +slim = contrib_slim + +# pylint: disable=g-long-lambda +trunc_normal = lambda stddev: tf.compat.v1.truncated_normal_initializer( + 0.0, stddev) +conv3d_spatiotemporal = i3d_utils.conv3d_spatiotemporal + + +def i3d_arg_scope(weight_decay=1e-7, + batch_norm_decay=0.999, + batch_norm_epsilon=0.001, + use_renorm=False, + separable_conv3d=False): + """Defines default arg_scope for I3D. + + Args: + weight_decay: The weight decay to use for regularizing the model. + batch_norm_decay: Decay for batch norm moving average. + batch_norm_epsilon: Small float added to variance to avoid dividing by zero + in batch norm. + use_renorm: Whether to use batch renormalization or not. + separable_conv3d: Whether to use separable 3d Convs. + + Returns: + sc: An arg_scope to use for the models. + """ + batch_norm_params = { + # Decay for the moving averages. + 'decay': batch_norm_decay, + # epsilon to prevent 0s in variance. + 'epsilon': batch_norm_epsilon, + # Turns off fused batch norm. + 'fused': False, + 'renorm': use_renorm, + # collection containing the moving mean and moving variance. + 'variables_collections': { + 'beta': None, + 'gamma': None, + 'moving_mean': ['moving_vars'], + 'moving_variance': ['moving_vars'], + } + } + + with slim.arg_scope( + [slim.conv3d, conv3d_spatiotemporal], + weights_regularizer=slim.l2_regularizer(weight_decay), + activation_fn=tf.nn.relu, + normalizer_fn=slim.batch_norm, + normalizer_params=batch_norm_params): + with slim.arg_scope( + [conv3d_spatiotemporal], separable=separable_conv3d) as sc: + return sc + + +def i3d_base(inputs, final_endpoint='Mixed_5c', + scope='InceptionV1'): + """Defines the I3D base architecture. + + Note that we use the names as defined in Inception V1 to facilitate checkpoint + conversion from an image-trained Inception V1 checkpoint to I3D checkpoint. + + Args: + inputs: A 5-D float tensor of size [batch_size, num_frames, height, width, + channels]. + final_endpoint: Specifies the endpoint to construct the network up to. It + can be one of ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', + 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', + 'MaxPool_4a_3x3', 'Mixed_4b', 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', + 'Mixed_4f', 'MaxPool_5a_2x2', 'Mixed_5b', 'Mixed_5c'] + scope: Optional variable_scope. + + Returns: + A dictionary from components of the network to the corresponding activation. + + Raises: + ValueError: if final_endpoint is not set to one of the predefined values. + """ + + return s3dg.s3dg_base( + inputs, + first_temporal_kernel_size=7, + temporal_conv_startat='Conv2d_2c_3x3', + gating_startat=None, + final_endpoint=final_endpoint, + min_depth=16, + depth_multiplier=1.0, + data_format='NDHWC', + scope=scope) + + +def i3d(inputs, + num_classes=1000, + dropout_keep_prob=0.8, + is_training=True, + prediction_fn=slim.softmax, + spatial_squeeze=True, + reuse=None, + scope='InceptionV1'): + """Defines the I3D architecture. + + The default image size used to train this network is 224x224. + + Args: + inputs: A 5-D float tensor of size [batch_size, num_frames, height, width, + channels]. + num_classes: number of predicted classes. + dropout_keep_prob: the percentage of activation values that are retained. + is_training: whether is training or not. + prediction_fn: a function to get predictions out of logits. + spatial_squeeze: if True, logits is of shape is [B, C], if false logits is + of shape [B, 1, 1, C], where B is batch_size and C is number of classes. + reuse: whether or not the network and its variables should be reused. To be + able to reuse 'scope' must be given. + scope: Optional variable_scope. + + Returns: + logits: the pre-softmax activations, a tensor of size + [batch_size, num_classes] + end_points: a dictionary from components of the network to the corresponding + activation. + """ + # Final pooling and prediction + with tf.compat.v1.variable_scope( + scope, 'InceptionV1', [inputs, num_classes], reuse=reuse) as scope: + with slim.arg_scope( + [slim.batch_norm, slim.dropout], is_training=is_training): + net, end_points = i3d_base(inputs, scope=scope) + with tf.compat.v1.variable_scope('Logits'): + kernel_size = i3d_utils.reduced_kernel_size_3d(net, [2, 7, 7]) + net = slim.avg_pool3d( + net, kernel_size, stride=1, scope='AvgPool_0a_7x7') + net = slim.dropout(net, dropout_keep_prob, scope='Dropout_0b') + logits = slim.conv3d( + net, + num_classes, [1, 1, 1], + activation_fn=None, + normalizer_fn=None, + scope='Conv2d_0c_1x1') + # Temporal average pooling. + logits = tf.reduce_mean(input_tensor=logits, axis=1) + if spatial_squeeze: + logits = tf.squeeze(logits, [1, 2], name='SpatialSqueeze') + + end_points['Logits'] = logits + end_points['Predictions'] = prediction_fn(logits, scope='Predictions') + return logits, end_points + + +i3d.default_image_size = 224 diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/i3d_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/i3d_test.py new file mode 100644 index 0000000..307233c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/i3d_test.py @@ -0,0 +1,149 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for networks.i3d.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from nets import i3d + + +class I3DTest(tf.test.TestCase): + + def testBuildClassificationNetwork(self): + batch_size = 5 + num_frames = 64 + height, width = 224, 224 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, num_frames, height, width, 3)) + logits, end_points = i3d.i3d(inputs, num_classes) + self.assertTrue(logits.op.name.startswith('InceptionV1/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertTrue('Predictions' in end_points) + self.assertListEqual(end_points['Predictions'].get_shape().as_list(), + [batch_size, num_classes]) + + def testBuildBaseNetwork(self): + batch_size = 5 + num_frames = 64 + height, width = 224, 224 + + inputs = tf.random.uniform((batch_size, num_frames, height, width, 3)) + mixed_6c, end_points = i3d.i3d_base(inputs) + self.assertTrue(mixed_6c.op.name.startswith('InceptionV1/Mixed_5c')) + self.assertListEqual(mixed_6c.get_shape().as_list(), + [batch_size, 8, 7, 7, 1024]) + expected_endpoints = ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', + 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', + 'Mixed_3c', 'MaxPool_4a_3x3', 'Mixed_4b', 'Mixed_4c', + 'Mixed_4d', 'Mixed_4e', 'Mixed_4f', 'MaxPool_5a_2x2', + 'Mixed_5b', 'Mixed_5c'] + self.assertItemsEqual(end_points.keys(), expected_endpoints) + + def testBuildOnlyUptoFinalEndpoint(self): + batch_size = 5 + num_frames = 64 + height, width = 224, 224 + endpoints = ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', + 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', + 'MaxPool_4a_3x3', 'Mixed_4b', 'Mixed_4c', 'Mixed_4d', + 'Mixed_4e', 'Mixed_4f', 'MaxPool_5a_2x2', 'Mixed_5b', + 'Mixed_5c'] + for index, endpoint in enumerate(endpoints): + with tf.Graph().as_default(): + inputs = tf.random.uniform((batch_size, num_frames, height, width, 3)) + out_tensor, end_points = i3d.i3d_base( + inputs, final_endpoint=endpoint) + self.assertTrue(out_tensor.op.name.startswith( + 'InceptionV1/' + endpoint)) + self.assertItemsEqual(endpoints[:index+1], end_points) + + def testBuildAndCheckAllEndPointsUptoMixed5c(self): + batch_size = 5 + num_frames = 64 + height, width = 224, 224 + + inputs = tf.random.uniform((batch_size, num_frames, height, width, 3)) + _, end_points = i3d.i3d_base(inputs, + final_endpoint='Mixed_5c') + endpoints_shapes = {'Conv2d_1a_7x7': [5, 32, 112, 112, 64], + 'MaxPool_2a_3x3': [5, 32, 56, 56, 64], + 'Conv2d_2b_1x1': [5, 32, 56, 56, 64], + 'Conv2d_2c_3x3': [5, 32, 56, 56, 192], + 'MaxPool_3a_3x3': [5, 32, 28, 28, 192], + 'Mixed_3b': [5, 32, 28, 28, 256], + 'Mixed_3c': [5, 32, 28, 28, 480], + 'MaxPool_4a_3x3': [5, 16, 14, 14, 480], + 'Mixed_4b': [5, 16, 14, 14, 512], + 'Mixed_4c': [5, 16, 14, 14, 512], + 'Mixed_4d': [5, 16, 14, 14, 512], + 'Mixed_4e': [5, 16, 14, 14, 528], + 'Mixed_4f': [5, 16, 14, 14, 832], + 'MaxPool_5a_2x2': [5, 8, 7, 7, 832], + 'Mixed_5b': [5, 8, 7, 7, 832], + 'Mixed_5c': [5, 8, 7, 7, 1024]} + + self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) + for endpoint_name, expected_shape in endpoints_shapes.iteritems(): + self.assertTrue(endpoint_name in end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testHalfSizeImages(self): + batch_size = 5 + num_frames = 64 + height, width = 112, 112 + + inputs = tf.random.uniform((batch_size, num_frames, height, width, 3)) + mixed_5c, _ = i3d.i3d_base(inputs) + self.assertTrue(mixed_5c.op.name.startswith('InceptionV1/Mixed_5c')) + self.assertListEqual(mixed_5c.get_shape().as_list(), + [batch_size, 8, 4, 4, 1024]) + + def testTenFrames(self): + batch_size = 5 + num_frames = 10 + height, width = 224, 224 + + inputs = tf.random.uniform((batch_size, num_frames, height, width, 3)) + mixed_5c, _ = i3d.i3d_base(inputs) + self.assertTrue(mixed_5c.op.name.startswith('InceptionV1/Mixed_5c')) + self.assertListEqual(mixed_5c.get_shape().as_list(), + [batch_size, 2, 7, 7, 1024]) + + def testEvaluation(self): + batch_size = 2 + num_frames = 64 + height, width = 224, 224 + num_classes = 1000 + + eval_inputs = tf.random.uniform((batch_size, num_frames, height, width, 3)) + logits, _ = i3d.i3d(eval_inputs, num_classes, + is_training=False) + predictions = tf.argmax(input=logits, axis=1) + + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(predictions) + self.assertEquals(output.shape, (batch_size,)) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/i3d_utils.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/i3d_utils.py new file mode 100644 index 0000000..05df301 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/i3d_utils.py @@ -0,0 +1,289 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities for building I3D network models.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf +from tensorflow.contrib import framework as contrib_framework +from tensorflow.contrib import layers as contrib_layers + + +# Orignaly, add_arg_scope = slim.add_arg_scope and layers = slim, now switch to +# more update-to-date tf.contrib.* API. +add_arg_scope = contrib_framework.add_arg_scope +layers = contrib_layers + + +def center_initializer(): + """Centering Initializer for I3D. + + This initializer allows identity mapping for temporal convolution at the + initialization, which is critical for a desired convergence behavior + for training a seprable I3D model. + + The centering behavior of this initializer requires an odd-sized kernel, + typically set to 3. + + Returns: + A weight initializer op used in temporal convolutional layers. + + Raises: + ValueError: Input tensor data type has to be tf.float32. + ValueError: If input tensor is not a 5-D tensor. + ValueError: If input and output channel dimensions are different. + ValueError: If spatial kernel sizes are not 1. + ValueError: If temporal kernel size is even. + """ + + def _initializer(shape, dtype=tf.float32, partition_info=None): # pylint: disable=unused-argument + """Initializer op.""" + + if dtype != tf.float32 and dtype != tf.bfloat16: + raise ValueError( + 'Input tensor data type has to be tf.float32 or tf.bfloat16.') + if len(shape) != 5: + raise ValueError('Input tensor has to be 5-D.') + if shape[3] != shape[4]: + raise ValueError('Input and output channel dimensions must be the same.') + if shape[1] != 1 or shape[2] != 1: + raise ValueError('Spatial kernel sizes must be 1 (pointwise conv).') + if shape[0] % 2 == 0: + raise ValueError('Temporal kernel size has to be odd.') + + center_pos = int(shape[0] / 2) + init_mat = np.zeros( + [shape[0], shape[1], shape[2], shape[3], shape[4]], dtype=np.float32) + for i in range(0, shape[3]): + init_mat[center_pos, 0, 0, i, i] = 1.0 + + init_op = tf.constant(init_mat, dtype=dtype) + return init_op + + return _initializer + + +@add_arg_scope +def conv3d_spatiotemporal(inputs, + num_outputs, + kernel_size, + stride=1, + padding='SAME', + activation_fn=None, + normalizer_fn=None, + normalizer_params=None, + weights_regularizer=None, + separable=False, + data_format='NDHWC', + scope=''): + """A wrapper for conv3d to model spatiotemporal representations. + + This allows switching between original 3D convolution and separable 3D + convolutions for spatial and temporal features respectively. On Kinetics, + seprable 3D convolutions yields better classification performance. + + Args: + inputs: a 5-D tensor `[batch_size, depth, height, width, channels]`. + num_outputs: integer, the number of output filters. + kernel_size: a list of length 3 + `[kernel_depth, kernel_height, kernel_width]` of the filters. Can be an + int if all values are the same. + stride: a list of length 3 `[stride_depth, stride_height, stride_width]`. + Can be an int if all strides are the same. + padding: one of `VALID` or `SAME`. + activation_fn: activation function. + normalizer_fn: normalization function to use instead of `biases`. + normalizer_params: dictionary of normalization function parameters. + weights_regularizer: Optional regularizer for the weights. + separable: If `True`, use separable spatiotemporal convolutions. + data_format: An optional string from: "NDHWC", "NCDHW". Defaults to "NDHWC". + The data format of the input and output data. With the default format + "NDHWC", the data is stored in the order of: [batch, in_depth, in_height, + in_width, in_channels]. Alternatively, the format could be "NCDHW", the + data storage order is: + [batch, in_channels, in_depth, in_height, in_width]. + scope: scope for `variable_scope`. + + Returns: + A tensor representing the output of the (separable) conv3d operation. + + """ + assert len(kernel_size) == 3 + if separable and kernel_size[0] != 1: + spatial_kernel_size = [1, kernel_size[1], kernel_size[2]] + temporal_kernel_size = [kernel_size[0], 1, 1] + if isinstance(stride, list) and len(stride) == 3: + spatial_stride = [1, stride[1], stride[2]] + temporal_stride = [stride[0], 1, 1] + else: + spatial_stride = [1, stride, stride] + temporal_stride = [stride, 1, 1] + net = layers.conv3d( + inputs, + num_outputs, + spatial_kernel_size, + stride=spatial_stride, + padding=padding, + activation_fn=activation_fn, + normalizer_fn=normalizer_fn, + normalizer_params=normalizer_params, + weights_regularizer=weights_regularizer, + data_format=data_format, + scope=scope) + net = layers.conv3d( + net, + num_outputs, + temporal_kernel_size, + stride=temporal_stride, + padding=padding, + scope=scope + '/temporal', + activation_fn=activation_fn, + normalizer_fn=None, + data_format=data_format, + weights_initializer=center_initializer()) + return net + else: + return layers.conv3d( + inputs, + num_outputs, + kernel_size, + stride=stride, + padding=padding, + activation_fn=activation_fn, + normalizer_fn=normalizer_fn, + normalizer_params=normalizer_params, + weights_regularizer=weights_regularizer, + data_format=data_format, + scope=scope) + + +@add_arg_scope +def inception_block_v1_3d(inputs, + num_outputs_0_0a, + num_outputs_1_0a, + num_outputs_1_0b, + num_outputs_2_0a, + num_outputs_2_0b, + num_outputs_3_0b, + temporal_kernel_size=3, + self_gating_fn=None, + data_format='NDHWC', + scope=''): + """A 3D Inception v1 block. + + This allows use of separable 3D convolutions and self-gating, as + described in: + Saining Xie, Chen Sun, Jonathan Huang, Zhuowen Tu and Kevin Murphy, + Rethinking Spatiotemporal Feature Learning For Video Understanding. + https://arxiv.org/abs/1712.04851. + + Args: + inputs: a 5-D tensor `[batch_size, depth, height, width, channels]`. + num_outputs_0_0a: integer, the number of output filters for Branch 0, + operation Conv2d_0a_1x1. + num_outputs_1_0a: integer, the number of output filters for Branch 1, + operation Conv2d_0a_1x1. + num_outputs_1_0b: integer, the number of output filters for Branch 1, + operation Conv2d_0b_3x3. + num_outputs_2_0a: integer, the number of output filters for Branch 2, + operation Conv2d_0a_1x1. + num_outputs_2_0b: integer, the number of output filters for Branch 2, + operation Conv2d_0b_3x3. + num_outputs_3_0b: integer, the number of output filters for Branch 3, + operation Conv2d_0b_1x1. + temporal_kernel_size: integer, the size of the temporal convolutional + filters in the conv3d_spatiotemporal blocks. + self_gating_fn: function which optionally performs self-gating. + Must have two arguments, `inputs` and `scope`, and return one output + tensor the same size as `inputs`. If `None`, no self-gating is + applied. + data_format: An optional string from: "NDHWC", "NCDHW". Defaults to "NDHWC". + The data format of the input and output data. With the default format + "NDHWC", the data is stored in the order of: [batch, in_depth, in_height, + in_width, in_channels]. Alternatively, the format could be "NCDHW", the + data storage order is: + [batch, in_channels, in_depth, in_height, in_width]. + scope: scope for `variable_scope`. + + Returns: + A 5-D tensor `[batch_size, depth, height, width, out_channels]`, where + `out_channels = num_outputs_0_0a + num_outputs_1_0b + num_outputs_2_0b + + num_outputs_3_0b`. + + """ + use_gating = self_gating_fn is not None + + with tf.compat.v1.variable_scope(scope): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = layers.conv3d( + inputs, num_outputs_0_0a, [1, 1, 1], scope='Conv2d_0a_1x1') + if use_gating: + branch_0 = self_gating_fn(branch_0, scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = layers.conv3d( + inputs, num_outputs_1_0a, [1, 1, 1], scope='Conv2d_0a_1x1') + branch_1 = conv3d_spatiotemporal( + branch_1, num_outputs_1_0b, [temporal_kernel_size, 3, 3], + scope='Conv2d_0b_3x3') + if use_gating: + branch_1 = self_gating_fn(branch_1, scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = layers.conv3d( + inputs, num_outputs_2_0a, [1, 1, 1], scope='Conv2d_0a_1x1') + branch_2 = conv3d_spatiotemporal( + branch_2, num_outputs_2_0b, [temporal_kernel_size, 3, 3], + scope='Conv2d_0b_3x3') + if use_gating: + branch_2 = self_gating_fn(branch_2, scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = layers.max_pool3d(inputs, [3, 3, 3], scope='MaxPool_0a_3x3') + branch_3 = layers.conv3d( + branch_3, num_outputs_3_0b, [1, 1, 1], scope='Conv2d_0b_1x1') + if use_gating: + branch_3 = self_gating_fn(branch_3, scope='Conv2d_0b_1x1') + index_c = data_format.index('C') + assert 1 <= index_c <= 4, 'Cannot identify channel dimension.' + output = tf.concat([branch_0, branch_1, branch_2, branch_3], index_c) + return output + + +def reduced_kernel_size_3d(input_tensor, kernel_size): + """Define kernel size which is automatically reduced for small input. + + If the shape of the input images is unknown at graph construction time this + function assumes that the input images are large enough. + + Args: + input_tensor: input tensor of size + [batch_size, time, height, width, channels]. + kernel_size: desired kernel size of length 3, corresponding to time, + height and width. + + Returns: + a tensor with the kernel size. + """ + assert len(kernel_size) == 3 + shape = input_tensor.get_shape().as_list() + assert len(shape) == 5 + if None in shape[1:4]: + kernel_size_out = kernel_size + else: + kernel_size_out = [min(shape[1], kernel_size[0]), + min(shape[2], kernel_size[1]), + min(shape[3], kernel_size[2])] + return kernel_size_out diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception.py new file mode 100644 index 0000000..b69cd2a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception.py @@ -0,0 +1,37 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Brings all inception models under one namespace.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# pylint: disable=unused-import +from nets.inception_resnet_v2 import inception_resnet_v2 +from nets.inception_resnet_v2 import inception_resnet_v2_arg_scope +from nets.inception_resnet_v2 import inception_resnet_v2_base +from nets.inception_v1 import inception_v1 +from nets.inception_v1 import inception_v1_arg_scope +from nets.inception_v1 import inception_v1_base +from nets.inception_v2 import inception_v2 +from nets.inception_v2 import inception_v2_arg_scope +from nets.inception_v2 import inception_v2_base +from nets.inception_v3 import inception_v3 +from nets.inception_v3 import inception_v3_arg_scope +from nets.inception_v3 import inception_v3_base +from nets.inception_v4 import inception_v4 +from nets.inception_v4 import inception_v4_arg_scope +from nets.inception_v4 import inception_v4_base +# pylint: enable=unused-import diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_resnet_v2.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_resnet_v2.py new file mode 100644 index 0000000..cff5895 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_resnet_v2.py @@ -0,0 +1,408 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains the definition of the Inception Resnet V2 architecture. + +As described in http://arxiv.org/abs/1602.07261. + + Inception-v4, Inception-ResNet and the Impact of Residual Connections + on Learning + Christian Szegedy, Sergey Ioffe, Vincent Vanhoucke, Alex Alemi +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +slim = contrib_slim + + +def block35(net, scale=1.0, activation_fn=tf.nn.relu, scope=None, reuse=None): + """Builds the 35x35 resnet block.""" + with tf.compat.v1.variable_scope(scope, 'Block35', [net], reuse=reuse): + with tf.compat.v1.variable_scope('Branch_0'): + tower_conv = slim.conv2d(net, 32, 1, scope='Conv2d_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + tower_conv1_0 = slim.conv2d(net, 32, 1, scope='Conv2d_0a_1x1') + tower_conv1_1 = slim.conv2d(tower_conv1_0, 32, 3, scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + tower_conv2_0 = slim.conv2d(net, 32, 1, scope='Conv2d_0a_1x1') + tower_conv2_1 = slim.conv2d(tower_conv2_0, 48, 3, scope='Conv2d_0b_3x3') + tower_conv2_2 = slim.conv2d(tower_conv2_1, 64, 3, scope='Conv2d_0c_3x3') + mixed = tf.concat(axis=3, values=[tower_conv, tower_conv1_1, tower_conv2_2]) + up = slim.conv2d(mixed, net.get_shape()[3], 1, normalizer_fn=None, + activation_fn=None, scope='Conv2d_1x1') + scaled_up = up * scale + if activation_fn == tf.nn.relu6: + # Use clip_by_value to simulate bandpass activation. + scaled_up = tf.clip_by_value(scaled_up, -6.0, 6.0) + + net += scaled_up + if activation_fn: + net = activation_fn(net) + return net + + +def block17(net, scale=1.0, activation_fn=tf.nn.relu, scope=None, reuse=None): + """Builds the 17x17 resnet block.""" + with tf.compat.v1.variable_scope(scope, 'Block17', [net], reuse=reuse): + with tf.compat.v1.variable_scope('Branch_0'): + tower_conv = slim.conv2d(net, 192, 1, scope='Conv2d_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + tower_conv1_0 = slim.conv2d(net, 128, 1, scope='Conv2d_0a_1x1') + tower_conv1_1 = slim.conv2d(tower_conv1_0, 160, [1, 7], + scope='Conv2d_0b_1x7') + tower_conv1_2 = slim.conv2d(tower_conv1_1, 192, [7, 1], + scope='Conv2d_0c_7x1') + mixed = tf.concat(axis=3, values=[tower_conv, tower_conv1_2]) + up = slim.conv2d(mixed, net.get_shape()[3], 1, normalizer_fn=None, + activation_fn=None, scope='Conv2d_1x1') + + scaled_up = up * scale + if activation_fn == tf.nn.relu6: + # Use clip_by_value to simulate bandpass activation. + scaled_up = tf.clip_by_value(scaled_up, -6.0, 6.0) + + net += scaled_up + if activation_fn: + net = activation_fn(net) + return net + + +def block8(net, scale=1.0, activation_fn=tf.nn.relu, scope=None, reuse=None): + """Builds the 8x8 resnet block.""" + with tf.compat.v1.variable_scope(scope, 'Block8', [net], reuse=reuse): + with tf.compat.v1.variable_scope('Branch_0'): + tower_conv = slim.conv2d(net, 192, 1, scope='Conv2d_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + tower_conv1_0 = slim.conv2d(net, 192, 1, scope='Conv2d_0a_1x1') + tower_conv1_1 = slim.conv2d(tower_conv1_0, 224, [1, 3], + scope='Conv2d_0b_1x3') + tower_conv1_2 = slim.conv2d(tower_conv1_1, 256, [3, 1], + scope='Conv2d_0c_3x1') + mixed = tf.concat(axis=3, values=[tower_conv, tower_conv1_2]) + up = slim.conv2d(mixed, net.get_shape()[3], 1, normalizer_fn=None, + activation_fn=None, scope='Conv2d_1x1') + + scaled_up = up * scale + if activation_fn == tf.nn.relu6: + # Use clip_by_value to simulate bandpass activation. + scaled_up = tf.clip_by_value(scaled_up, -6.0, 6.0) + + net += scaled_up + if activation_fn: + net = activation_fn(net) + return net + + +def inception_resnet_v2_base(inputs, + final_endpoint='Conv2d_7b_1x1', + output_stride=16, + align_feature_maps=False, + scope=None, + activation_fn=tf.nn.relu): + """Inception model from http://arxiv.org/abs/1602.07261. + + Constructs an Inception Resnet v2 network from inputs to the given final + endpoint. This method can construct the network up to the final inception + block Conv2d_7b_1x1. + + Args: + inputs: a tensor of size [batch_size, height, width, channels]. + final_endpoint: specifies the endpoint to construct the network up to. It + can be one of ['Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', + 'MaxPool_3a_3x3', 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', 'MaxPool_5a_3x3', + 'Mixed_5b', 'Mixed_6a', 'PreAuxLogits', 'Mixed_7a', 'Conv2d_7b_1x1'] + output_stride: A scalar that specifies the requested ratio of input to + output spatial resolution. Only supports 8 and 16. + align_feature_maps: When true, changes all the VALID paddings in the network + to SAME padding so that the feature maps are aligned. + scope: Optional variable_scope. + activation_fn: Activation function for block scopes. + + Returns: + tensor_out: output tensor corresponding to the final_endpoint. + end_points: a set of activations for external use, for example summaries or + losses. + + Raises: + ValueError: if final_endpoint is not set to one of the predefined values, + or if the output_stride is not 8 or 16, or if the output_stride is 8 and + we request an end point after 'PreAuxLogits'. + """ + if output_stride != 8 and output_stride != 16: + raise ValueError('output_stride must be 8 or 16.') + + padding = 'SAME' if align_feature_maps else 'VALID' + + end_points = {} + + def add_and_check_final(name, net): + end_points[name] = net + return name == final_endpoint + + with tf.compat.v1.variable_scope(scope, 'InceptionResnetV2', [inputs]): + with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d], + stride=1, padding='SAME'): + # 149 x 149 x 32 + net = slim.conv2d(inputs, 32, 3, stride=2, padding=padding, + scope='Conv2d_1a_3x3') + if add_and_check_final('Conv2d_1a_3x3', net): return net, end_points + + # 147 x 147 x 32 + net = slim.conv2d(net, 32, 3, padding=padding, + scope='Conv2d_2a_3x3') + if add_and_check_final('Conv2d_2a_3x3', net): return net, end_points + # 147 x 147 x 64 + net = slim.conv2d(net, 64, 3, scope='Conv2d_2b_3x3') + if add_and_check_final('Conv2d_2b_3x3', net): return net, end_points + # 73 x 73 x 64 + net = slim.max_pool2d(net, 3, stride=2, padding=padding, + scope='MaxPool_3a_3x3') + if add_and_check_final('MaxPool_3a_3x3', net): return net, end_points + # 73 x 73 x 80 + net = slim.conv2d(net, 80, 1, padding=padding, + scope='Conv2d_3b_1x1') + if add_and_check_final('Conv2d_3b_1x1', net): return net, end_points + # 71 x 71 x 192 + net = slim.conv2d(net, 192, 3, padding=padding, + scope='Conv2d_4a_3x3') + if add_and_check_final('Conv2d_4a_3x3', net): return net, end_points + # 35 x 35 x 192 + net = slim.max_pool2d(net, 3, stride=2, padding=padding, + scope='MaxPool_5a_3x3') + if add_and_check_final('MaxPool_5a_3x3', net): return net, end_points + + # 35 x 35 x 320 + with tf.compat.v1.variable_scope('Mixed_5b'): + with tf.compat.v1.variable_scope('Branch_0'): + tower_conv = slim.conv2d(net, 96, 1, scope='Conv2d_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + tower_conv1_0 = slim.conv2d(net, 48, 1, scope='Conv2d_0a_1x1') + tower_conv1_1 = slim.conv2d(tower_conv1_0, 64, 5, + scope='Conv2d_0b_5x5') + with tf.compat.v1.variable_scope('Branch_2'): + tower_conv2_0 = slim.conv2d(net, 64, 1, scope='Conv2d_0a_1x1') + tower_conv2_1 = slim.conv2d(tower_conv2_0, 96, 3, + scope='Conv2d_0b_3x3') + tower_conv2_2 = slim.conv2d(tower_conv2_1, 96, 3, + scope='Conv2d_0c_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + tower_pool = slim.avg_pool2d(net, 3, stride=1, padding='SAME', + scope='AvgPool_0a_3x3') + tower_pool_1 = slim.conv2d(tower_pool, 64, 1, + scope='Conv2d_0b_1x1') + net = tf.concat( + [tower_conv, tower_conv1_1, tower_conv2_2, tower_pool_1], 3) + + if add_and_check_final('Mixed_5b', net): return net, end_points + # TODO(alemi): Register intermediate endpoints + net = slim.repeat(net, 10, block35, scale=0.17, + activation_fn=activation_fn) + + # 17 x 17 x 1088 if output_stride == 8, + # 33 x 33 x 1088 if output_stride == 16 + use_atrous = output_stride == 8 + + with tf.compat.v1.variable_scope('Mixed_6a'): + with tf.compat.v1.variable_scope('Branch_0'): + tower_conv = slim.conv2d(net, 384, 3, stride=1 if use_atrous else 2, + padding=padding, + scope='Conv2d_1a_3x3') + with tf.compat.v1.variable_scope('Branch_1'): + tower_conv1_0 = slim.conv2d(net, 256, 1, scope='Conv2d_0a_1x1') + tower_conv1_1 = slim.conv2d(tower_conv1_0, 256, 3, + scope='Conv2d_0b_3x3') + tower_conv1_2 = slim.conv2d(tower_conv1_1, 384, 3, + stride=1 if use_atrous else 2, + padding=padding, + scope='Conv2d_1a_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + tower_pool = slim.max_pool2d(net, 3, stride=1 if use_atrous else 2, + padding=padding, + scope='MaxPool_1a_3x3') + net = tf.concat([tower_conv, tower_conv1_2, tower_pool], 3) + + if add_and_check_final('Mixed_6a', net): return net, end_points + + # TODO(alemi): register intermediate endpoints + with slim.arg_scope([slim.conv2d], rate=2 if use_atrous else 1): + net = slim.repeat(net, 20, block17, scale=0.10, + activation_fn=activation_fn) + if add_and_check_final('PreAuxLogits', net): return net, end_points + + if output_stride == 8: + # TODO(gpapan): Properly support output_stride for the rest of the net. + raise ValueError('output_stride==8 is only supported up to the ' + 'PreAuxlogits end_point for now.') + + # 8 x 8 x 2080 + with tf.compat.v1.variable_scope('Mixed_7a'): + with tf.compat.v1.variable_scope('Branch_0'): + tower_conv = slim.conv2d(net, 256, 1, scope='Conv2d_0a_1x1') + tower_conv_1 = slim.conv2d(tower_conv, 384, 3, stride=2, + padding=padding, + scope='Conv2d_1a_3x3') + with tf.compat.v1.variable_scope('Branch_1'): + tower_conv1 = slim.conv2d(net, 256, 1, scope='Conv2d_0a_1x1') + tower_conv1_1 = slim.conv2d(tower_conv1, 288, 3, stride=2, + padding=padding, + scope='Conv2d_1a_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + tower_conv2 = slim.conv2d(net, 256, 1, scope='Conv2d_0a_1x1') + tower_conv2_1 = slim.conv2d(tower_conv2, 288, 3, + scope='Conv2d_0b_3x3') + tower_conv2_2 = slim.conv2d(tower_conv2_1, 320, 3, stride=2, + padding=padding, + scope='Conv2d_1a_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + tower_pool = slim.max_pool2d(net, 3, stride=2, + padding=padding, + scope='MaxPool_1a_3x3') + net = tf.concat( + [tower_conv_1, tower_conv1_1, tower_conv2_2, tower_pool], 3) + + if add_and_check_final('Mixed_7a', net): return net, end_points + + # TODO(alemi): register intermediate endpoints + net = slim.repeat(net, 9, block8, scale=0.20, activation_fn=activation_fn) + net = block8(net, activation_fn=None) + + # 8 x 8 x 1536 + net = slim.conv2d(net, 1536, 1, scope='Conv2d_7b_1x1') + if add_and_check_final('Conv2d_7b_1x1', net): return net, end_points + + raise ValueError('final_endpoint (%s) not recognized', final_endpoint) + + +def inception_resnet_v2(inputs, num_classes=1001, is_training=True, + dropout_keep_prob=0.8, + reuse=None, + scope='InceptionResnetV2', + create_aux_logits=True, + activation_fn=tf.nn.relu): + """Creates the Inception Resnet V2 model. + + Args: + inputs: a 4-D tensor of size [batch_size, height, width, 3]. + Dimension batch_size may be undefined. If create_aux_logits is false, + also height and width may be undefined. + num_classes: number of predicted classes. If 0 or None, the logits layer + is omitted and the input features to the logits layer (before dropout) + are returned instead. + is_training: whether is training or not. + dropout_keep_prob: float, the fraction to keep before final layer. + reuse: whether or not the network and its variables should be reused. To be + able to reuse 'scope' must be given. + scope: Optional variable_scope. + create_aux_logits: Whether to include the auxilliary logits. + activation_fn: Activation function for conv2d. + + Returns: + net: the output of the logits layer (if num_classes is a non-zero integer), + or the non-dropped-out input to the logits layer (if num_classes is 0 or + None). + end_points: the set of end_points from the inception model. + """ + end_points = {} + + with tf.compat.v1.variable_scope( + scope, 'InceptionResnetV2', [inputs], reuse=reuse) as scope: + with slim.arg_scope([slim.batch_norm, slim.dropout], + is_training=is_training): + + net, end_points = inception_resnet_v2_base(inputs, scope=scope, + activation_fn=activation_fn) + + if create_aux_logits and num_classes: + with tf.compat.v1.variable_scope('AuxLogits'): + aux = end_points['PreAuxLogits'] + aux = slim.avg_pool2d(aux, 5, stride=3, padding='VALID', + scope='Conv2d_1a_3x3') + aux = slim.conv2d(aux, 128, 1, scope='Conv2d_1b_1x1') + aux = slim.conv2d(aux, 768, aux.get_shape()[1:3], + padding='VALID', scope='Conv2d_2a_5x5') + aux = slim.flatten(aux) + aux = slim.fully_connected(aux, num_classes, activation_fn=None, + scope='Logits') + end_points['AuxLogits'] = aux + + with tf.compat.v1.variable_scope('Logits'): + # TODO(sguada,arnoegw): Consider adding a parameter global_pool which + # can be set to False to disable pooling here (as in resnet_*()). + kernel_size = net.get_shape()[1:3] + if kernel_size.is_fully_defined(): + net = slim.avg_pool2d(net, kernel_size, padding='VALID', + scope='AvgPool_1a_8x8') + else: + net = tf.reduce_mean( + input_tensor=net, axis=[1, 2], keepdims=True, name='global_pool') + end_points['global_pool'] = net + if not num_classes: + return net, end_points + net = slim.flatten(net) + net = slim.dropout(net, dropout_keep_prob, is_training=is_training, + scope='Dropout') + end_points['PreLogitsFlatten'] = net + logits = slim.fully_connected(net, num_classes, activation_fn=None, + scope='Logits') + end_points['Logits'] = logits + end_points['Predictions'] = tf.nn.softmax(logits, name='Predictions') + + return logits, end_points +inception_resnet_v2.default_image_size = 299 + + +def inception_resnet_v2_arg_scope( + weight_decay=0.00004, + batch_norm_decay=0.9997, + batch_norm_epsilon=0.001, + activation_fn=tf.nn.relu, + batch_norm_updates_collections=tf.compat.v1.GraphKeys.UPDATE_OPS, + batch_norm_scale=False): + """Returns the scope with the default parameters for inception_resnet_v2. + + Args: + weight_decay: the weight decay for weights variables. + batch_norm_decay: decay for the moving average of batch_norm momentums. + batch_norm_epsilon: small float added to variance to avoid dividing by zero. + activation_fn: Activation function for conv2d. + batch_norm_updates_collections: Collection for the update ops for + batch norm. + batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the + activations in the batch normalization layer. + + Returns: + a arg_scope with the parameters needed for inception_resnet_v2. + """ + # Set weight_decay for weights in conv2d and fully_connected layers. + with slim.arg_scope([slim.conv2d, slim.fully_connected], + weights_regularizer=slim.l2_regularizer(weight_decay), + biases_regularizer=slim.l2_regularizer(weight_decay)): + + batch_norm_params = { + 'decay': batch_norm_decay, + 'epsilon': batch_norm_epsilon, + 'updates_collections': batch_norm_updates_collections, + 'fused': None, # Use fused batch norm if possible. + 'scale': batch_norm_scale, + } + # Set activation_fn and parameters for batch_norm. + with slim.arg_scope([slim.conv2d], activation_fn=activation_fn, + normalizer_fn=slim.batch_norm, + normalizer_params=batch_norm_params) as scope: + return scope diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_resnet_v2_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_resnet_v2_test.py new file mode 100644 index 0000000..348c44d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_resnet_v2_test.py @@ -0,0 +1,338 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for slim.inception_resnet_v2.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets import inception + + +class InceptionTest(tf.test.TestCase): + + def testBuildLogits(self): + batch_size = 5 + height, width = 299, 299 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, endpoints = inception.inception_resnet_v2(inputs, num_classes) + self.assertTrue('AuxLogits' in endpoints) + auxlogits = endpoints['AuxLogits'] + self.assertTrue( + auxlogits.op.name.startswith('InceptionResnetV2/AuxLogits')) + self.assertListEqual(auxlogits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertTrue(logits.op.name.startswith('InceptionResnetV2/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + + def testBuildWithoutAuxLogits(self): + batch_size = 5 + height, width = 299, 299 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, endpoints = inception.inception_resnet_v2(inputs, num_classes, + create_aux_logits=False) + self.assertTrue('AuxLogits' not in endpoints) + self.assertTrue(logits.op.name.startswith('InceptionResnetV2/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + + def testBuildNoClasses(self): + batch_size = 5 + height, width = 299, 299 + num_classes = None + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + net, endpoints = inception.inception_resnet_v2(inputs, num_classes) + self.assertTrue('AuxLogits' not in endpoints) + self.assertTrue('Logits' not in endpoints) + self.assertTrue( + net.op.name.startswith('InceptionResnetV2/Logits/AvgPool')) + self.assertListEqual(net.get_shape().as_list(), [batch_size, 1, 1, 1536]) + + def testBuildEndPoints(self): + batch_size = 5 + height, width = 299, 299 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = inception.inception_resnet_v2(inputs, num_classes) + self.assertTrue('Logits' in end_points) + logits = end_points['Logits'] + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertTrue('AuxLogits' in end_points) + aux_logits = end_points['AuxLogits'] + self.assertListEqual(aux_logits.get_shape().as_list(), + [batch_size, num_classes]) + pre_pool = end_points['Conv2d_7b_1x1'] + self.assertListEqual(pre_pool.get_shape().as_list(), + [batch_size, 8, 8, 1536]) + + def testBuildBaseNetwork(self): + batch_size = 5 + height, width = 299, 299 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + net, end_points = inception.inception_resnet_v2_base(inputs) + self.assertTrue(net.op.name.startswith('InceptionResnetV2/Conv2d_7b_1x1')) + self.assertListEqual(net.get_shape().as_list(), + [batch_size, 8, 8, 1536]) + expected_endpoints = ['Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', + 'MaxPool_3a_3x3', 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', + 'MaxPool_5a_3x3', 'Mixed_5b', 'Mixed_6a', + 'PreAuxLogits', 'Mixed_7a', 'Conv2d_7b_1x1'] + self.assertItemsEqual(end_points.keys(), expected_endpoints) + + def testBuildOnlyUptoFinalEndpoint(self): + batch_size = 5 + height, width = 299, 299 + endpoints = ['Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', + 'MaxPool_3a_3x3', 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', + 'MaxPool_5a_3x3', 'Mixed_5b', 'Mixed_6a', + 'PreAuxLogits', 'Mixed_7a', 'Conv2d_7b_1x1'] + for index, endpoint in enumerate(endpoints): + with tf.Graph().as_default(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + out_tensor, end_points = inception.inception_resnet_v2_base( + inputs, final_endpoint=endpoint) + if endpoint != 'PreAuxLogits': + self.assertTrue(out_tensor.op.name.startswith( + 'InceptionResnetV2/' + endpoint)) + self.assertItemsEqual(endpoints[:index+1], end_points.keys()) + + def testBuildAndCheckAllEndPointsUptoPreAuxLogits(self): + batch_size = 5 + height, width = 299, 299 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = inception.inception_resnet_v2_base( + inputs, final_endpoint='PreAuxLogits') + endpoints_shapes = {'Conv2d_1a_3x3': [5, 149, 149, 32], + 'Conv2d_2a_3x3': [5, 147, 147, 32], + 'Conv2d_2b_3x3': [5, 147, 147, 64], + 'MaxPool_3a_3x3': [5, 73, 73, 64], + 'Conv2d_3b_1x1': [5, 73, 73, 80], + 'Conv2d_4a_3x3': [5, 71, 71, 192], + 'MaxPool_5a_3x3': [5, 35, 35, 192], + 'Mixed_5b': [5, 35, 35, 320], + 'Mixed_6a': [5, 17, 17, 1088], + 'PreAuxLogits': [5, 17, 17, 1088] + } + + self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) + for endpoint_name in endpoints_shapes: + expected_shape = endpoints_shapes[endpoint_name] + self.assertTrue(endpoint_name in end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testBuildAndCheckAllEndPointsUptoPreAuxLogitsWithAlignedFeatureMaps(self): + batch_size = 5 + height, width = 299, 299 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = inception.inception_resnet_v2_base( + inputs, final_endpoint='PreAuxLogits', align_feature_maps=True) + endpoints_shapes = {'Conv2d_1a_3x3': [5, 150, 150, 32], + 'Conv2d_2a_3x3': [5, 150, 150, 32], + 'Conv2d_2b_3x3': [5, 150, 150, 64], + 'MaxPool_3a_3x3': [5, 75, 75, 64], + 'Conv2d_3b_1x1': [5, 75, 75, 80], + 'Conv2d_4a_3x3': [5, 75, 75, 192], + 'MaxPool_5a_3x3': [5, 38, 38, 192], + 'Mixed_5b': [5, 38, 38, 320], + 'Mixed_6a': [5, 19, 19, 1088], + 'PreAuxLogits': [5, 19, 19, 1088] + } + + self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) + for endpoint_name in endpoints_shapes: + expected_shape = endpoints_shapes[endpoint_name] + self.assertTrue(endpoint_name in end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testBuildAndCheckAllEndPointsUptoPreAuxLogitsWithOutputStrideEight(self): + batch_size = 5 + height, width = 299, 299 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = inception.inception_resnet_v2_base( + inputs, final_endpoint='PreAuxLogits', output_stride=8) + endpoints_shapes = {'Conv2d_1a_3x3': [5, 149, 149, 32], + 'Conv2d_2a_3x3': [5, 147, 147, 32], + 'Conv2d_2b_3x3': [5, 147, 147, 64], + 'MaxPool_3a_3x3': [5, 73, 73, 64], + 'Conv2d_3b_1x1': [5, 73, 73, 80], + 'Conv2d_4a_3x3': [5, 71, 71, 192], + 'MaxPool_5a_3x3': [5, 35, 35, 192], + 'Mixed_5b': [5, 35, 35, 320], + 'Mixed_6a': [5, 33, 33, 1088], + 'PreAuxLogits': [5, 33, 33, 1088] + } + + self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) + for endpoint_name in endpoints_shapes: + expected_shape = endpoints_shapes[endpoint_name] + self.assertTrue(endpoint_name in end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testVariablesSetDevice(self): + batch_size = 5 + height, width = 299, 299 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + # Force all Variables to reside on the device. + with tf.compat.v1.variable_scope('on_cpu'), tf.device('/cpu:0'): + inception.inception_resnet_v2(inputs, num_classes) + with tf.compat.v1.variable_scope('on_gpu'), tf.device('/gpu:0'): + inception.inception_resnet_v2(inputs, num_classes) + for v in tf.compat.v1.get_collection( + tf.compat.v1.GraphKeys.GLOBAL_VARIABLES, scope='on_cpu'): + self.assertDeviceEqual(v.device, '/cpu:0') + for v in tf.compat.v1.get_collection( + tf.compat.v1.GraphKeys.GLOBAL_VARIABLES, scope='on_gpu'): + self.assertDeviceEqual(v.device, '/gpu:0') + + def testHalfSizeImages(self): + batch_size = 5 + height, width = 150, 150 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, end_points = inception.inception_resnet_v2(inputs, num_classes) + self.assertTrue(logits.op.name.startswith('InceptionResnetV2/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + pre_pool = end_points['Conv2d_7b_1x1'] + self.assertListEqual(pre_pool.get_shape().as_list(), + [batch_size, 3, 3, 1536]) + + def testGlobalPool(self): + batch_size = 1 + height, width = 330, 400 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, end_points = inception.inception_resnet_v2(inputs, num_classes) + self.assertTrue(logits.op.name.startswith('InceptionResnetV2/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + pre_pool = end_points['Conv2d_7b_1x1'] + self.assertListEqual(pre_pool.get_shape().as_list(), + [batch_size, 8, 11, 1536]) + + def testGlobalPoolUnknownImageShape(self): + batch_size = 1 + height, width = 330, 400 + num_classes = 1000 + with self.test_session() as sess: + inputs = tf.compat.v1.placeholder(tf.float32, (batch_size, None, None, 3)) + logits, end_points = inception.inception_resnet_v2( + inputs, num_classes, create_aux_logits=False) + self.assertTrue(logits.op.name.startswith('InceptionResnetV2/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + pre_pool = end_points['Conv2d_7b_1x1'] + images = tf.random.uniform((batch_size, height, width, 3)) + sess.run(tf.compat.v1.global_variables_initializer()) + logits_out, pre_pool_out = sess.run([logits, pre_pool], + {inputs: images.eval()}) + self.assertTupleEqual(logits_out.shape, (batch_size, num_classes)) + self.assertTupleEqual(pre_pool_out.shape, (batch_size, 8, 11, 1536)) + + def testUnknownBatchSize(self): + batch_size = 1 + height, width = 299, 299 + num_classes = 1000 + with self.test_session() as sess: + inputs = tf.compat.v1.placeholder(tf.float32, (None, height, width, 3)) + logits, _ = inception.inception_resnet_v2(inputs, num_classes) + self.assertTrue(logits.op.name.startswith('InceptionResnetV2/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [None, num_classes]) + images = tf.random.uniform((batch_size, height, width, 3)) + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(logits, {inputs: images.eval()}) + self.assertEquals(output.shape, (batch_size, num_classes)) + + def testEvaluation(self): + batch_size = 2 + height, width = 299, 299 + num_classes = 1000 + with self.test_session() as sess: + eval_inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = inception.inception_resnet_v2(eval_inputs, + num_classes, + is_training=False) + predictions = tf.argmax(input=logits, axis=1) + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(predictions) + self.assertEquals(output.shape, (batch_size,)) + + def testTrainEvalWithReuse(self): + train_batch_size = 5 + eval_batch_size = 2 + height, width = 150, 150 + num_classes = 1000 + with self.test_session() as sess: + train_inputs = tf.random.uniform((train_batch_size, height, width, 3)) + inception.inception_resnet_v2(train_inputs, num_classes) + eval_inputs = tf.random.uniform((eval_batch_size, height, width, 3)) + logits, _ = inception.inception_resnet_v2(eval_inputs, + num_classes, + is_training=False, + reuse=True) + predictions = tf.argmax(input=logits, axis=1) + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(predictions) + self.assertEquals(output.shape, (eval_batch_size,)) + + def testNoBatchNormScaleByDefault(self): + height, width = 299, 299 + num_classes = 1000 + inputs = tf.compat.v1.placeholder(tf.float32, (1, height, width, 3)) + with contrib_slim.arg_scope(inception.inception_resnet_v2_arg_scope()): + inception.inception_resnet_v2(inputs, num_classes, is_training=False) + + self.assertEqual(tf.compat.v1.global_variables('.*/BatchNorm/gamma:0$'), []) + + def testBatchNormScale(self): + height, width = 299, 299 + num_classes = 1000 + inputs = tf.compat.v1.placeholder(tf.float32, (1, height, width, 3)) + with contrib_slim.arg_scope( + inception.inception_resnet_v2_arg_scope(batch_norm_scale=True)): + inception.inception_resnet_v2(inputs, num_classes, is_training=False) + + gamma_names = set( + v.op.name + for v in tf.compat.v1.global_variables('.*/BatchNorm/gamma:0$')) + self.assertGreater(len(gamma_names), 0) + for v in tf.compat.v1.global_variables('.*/BatchNorm/moving_mean:0$'): + self.assertIn(v.op.name[:-len('moving_mean')] + 'gamma', gamma_names) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_utils.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_utils.py new file mode 100644 index 0000000..493a684 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_utils.py @@ -0,0 +1,84 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains common code shared by all inception models. + +Usage of arg scope: + with slim.arg_scope(inception_arg_scope()): + logits, end_points = inception.inception_v3(images, num_classes, + is_training=is_training) + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +slim = contrib_slim + + +def inception_arg_scope( + weight_decay=0.00004, + use_batch_norm=True, + batch_norm_decay=0.9997, + batch_norm_epsilon=0.001, + activation_fn=tf.nn.relu, + batch_norm_updates_collections=tf.compat.v1.GraphKeys.UPDATE_OPS, + batch_norm_scale=False): + """Defines the default arg scope for inception models. + + Args: + weight_decay: The weight decay to use for regularizing the model. + use_batch_norm: "If `True`, batch_norm is applied after each convolution. + batch_norm_decay: Decay for batch norm moving average. + batch_norm_epsilon: Small float added to variance to avoid dividing by zero + in batch norm. + activation_fn: Activation function for conv2d. + batch_norm_updates_collections: Collection for the update ops for + batch norm. + batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the + activations in the batch normalization layer. + + Returns: + An `arg_scope` to use for the inception models. + """ + batch_norm_params = { + # Decay for the moving averages. + 'decay': batch_norm_decay, + # epsilon to prevent 0s in variance. + 'epsilon': batch_norm_epsilon, + # collection containing update_ops. + 'updates_collections': batch_norm_updates_collections, + # use fused batch norm if possible. + 'fused': None, + 'scale': batch_norm_scale, + } + if use_batch_norm: + normalizer_fn = slim.batch_norm + normalizer_params = batch_norm_params + else: + normalizer_fn = None + normalizer_params = {} + # Set weight_decay for weights in Conv and FC layers. + with slim.arg_scope([slim.conv2d, slim.fully_connected], + weights_regularizer=slim.l2_regularizer(weight_decay)): + with slim.arg_scope( + [slim.conv2d], + weights_initializer=slim.variance_scaling_initializer(), + activation_fn=activation_fn, + normalizer_fn=normalizer_fn, + normalizer_params=normalizer_params) as sc: + return sc diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v1.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v1.py new file mode 100644 index 0000000..b84104a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v1.py @@ -0,0 +1,347 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains the definition for inception v1 classification network.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets import inception_utils + +slim = contrib_slim + +# pylint: disable=g-long-lambda +trunc_normal = lambda stddev: tf.compat.v1.truncated_normal_initializer( + 0.0, stddev) + + +def inception_v1_base(inputs, + final_endpoint='Mixed_5c', + include_root_block=True, + scope='InceptionV1'): + """Defines the Inception V1 base architecture. + + This architecture is defined in: + Going deeper with convolutions + Christian Szegedy, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed, + Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, Andrew Rabinovich. + http://arxiv.org/pdf/1409.4842v1.pdf. + + Args: + inputs: a tensor of size [batch_size, height, width, channels]. + final_endpoint: specifies the endpoint to construct the network up to. It + can be one of ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', + 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', + 'MaxPool_4a_3x3', 'Mixed_4b', 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', + 'Mixed_4f', 'MaxPool_5a_2x2', 'Mixed_5b', 'Mixed_5c']. If + include_root_block is False, ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', + 'Conv2d_2b_1x1', 'Conv2d_2c_3x3', 'MaxPool_3a_3x3'] will not be available. + include_root_block: If True, include the convolution and max-pooling layers + before the inception modules. If False, excludes those layers. + scope: Optional variable_scope. + + Returns: + A dictionary from components of the network to the corresponding activation. + + Raises: + ValueError: if final_endpoint is not set to one of the predefined values. + """ + end_points = {} + with tf.compat.v1.variable_scope(scope, 'InceptionV1', [inputs]): + with slim.arg_scope( + [slim.conv2d, slim.fully_connected], + weights_initializer=trunc_normal(0.01)): + with slim.arg_scope([slim.conv2d, slim.max_pool2d], + stride=1, padding='SAME'): + net = inputs + if include_root_block: + end_point = 'Conv2d_1a_7x7' + net = slim.conv2d(inputs, 64, [7, 7], stride=2, scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + end_point = 'MaxPool_2a_3x3' + net = slim.max_pool2d(net, [3, 3], stride=2, scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + end_point = 'Conv2d_2b_1x1' + net = slim.conv2d(net, 64, [1, 1], scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + end_point = 'Conv2d_2c_3x3' + net = slim.conv2d(net, 192, [3, 3], scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + end_point = 'MaxPool_3a_3x3' + net = slim.max_pool2d(net, [3, 3], stride=2, scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + + end_point = 'Mixed_3b' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, 64, [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, 96, [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, 128, [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(net, 16, [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, 32, [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, 32, [1, 1], scope='Conv2d_0b_1x1') + net = tf.concat( + axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if final_endpoint == end_point: return net, end_points + + end_point = 'Mixed_3c' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, 128, [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, 128, [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, 192, [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(net, 32, [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, 96, [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, 64, [1, 1], scope='Conv2d_0b_1x1') + net = tf.concat( + axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if final_endpoint == end_point: return net, end_points + + end_point = 'MaxPool_4a_3x3' + net = slim.max_pool2d(net, [3, 3], stride=2, scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: return net, end_points + + end_point = 'Mixed_4b' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, 192, [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, 96, [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, 208, [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(net, 16, [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, 48, [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, 64, [1, 1], scope='Conv2d_0b_1x1') + net = tf.concat( + axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if final_endpoint == end_point: return net, end_points + + end_point = 'Mixed_4c' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, 160, [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, 112, [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, 224, [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(net, 24, [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, 64, [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, 64, [1, 1], scope='Conv2d_0b_1x1') + net = tf.concat( + axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if final_endpoint == end_point: return net, end_points + + end_point = 'Mixed_4d' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, 128, [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, 128, [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, 256, [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(net, 24, [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, 64, [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, 64, [1, 1], scope='Conv2d_0b_1x1') + net = tf.concat( + axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if final_endpoint == end_point: return net, end_points + + end_point = 'Mixed_4e' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, 112, [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, 144, [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, 288, [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(net, 32, [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, 64, [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, 64, [1, 1], scope='Conv2d_0b_1x1') + net = tf.concat( + axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if final_endpoint == end_point: return net, end_points + + end_point = 'Mixed_4f' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, 256, [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, 160, [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, 320, [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(net, 32, [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, 128, [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, 128, [1, 1], scope='Conv2d_0b_1x1') + net = tf.concat( + axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if final_endpoint == end_point: return net, end_points + + end_point = 'MaxPool_5a_2x2' + net = slim.max_pool2d(net, [2, 2], stride=2, scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: return net, end_points + + end_point = 'Mixed_5b' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, 256, [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, 160, [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, 320, [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(net, 32, [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, 128, [3, 3], scope='Conv2d_0a_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, 128, [1, 1], scope='Conv2d_0b_1x1') + net = tf.concat( + axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if final_endpoint == end_point: return net, end_points + + end_point = 'Mixed_5c' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, 384, [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, 192, [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, 384, [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(net, 48, [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, 128, [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, 128, [1, 1], scope='Conv2d_0b_1x1') + net = tf.concat( + axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if final_endpoint == end_point: return net, end_points + raise ValueError('Unknown final endpoint %s' % final_endpoint) + + +def inception_v1(inputs, + num_classes=1000, + is_training=True, + dropout_keep_prob=0.8, + prediction_fn=slim.softmax, + spatial_squeeze=True, + reuse=None, + scope='InceptionV1', + global_pool=False): + """Defines the Inception V1 architecture. + + This architecture is defined in: + + Going deeper with convolutions + Christian Szegedy, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed, + Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, Andrew Rabinovich. + http://arxiv.org/pdf/1409.4842v1.pdf. + + The default image size used to train this network is 224x224. + + Args: + inputs: a tensor of size [batch_size, height, width, channels]. + num_classes: number of predicted classes. If 0 or None, the logits layer + is omitted and the input features to the logits layer (before dropout) + are returned instead. + is_training: whether is training or not. + dropout_keep_prob: the percentage of activation values that are retained. + prediction_fn: a function to get predictions out of logits. + spatial_squeeze: if True, logits is of shape [B, C], if false logits is of + shape [B, 1, 1, C], where B is batch_size and C is number of classes. + reuse: whether or not the network and its variables should be reused. To be + able to reuse 'scope' must be given. + scope: Optional variable_scope. + global_pool: Optional boolean flag to control the avgpooling before the + logits layer. If false or unset, pooling is done with a fixed window + that reduces default-sized inputs to 1x1, while larger inputs lead to + larger outputs. If true, any input size is pooled down to 1x1. + + Returns: + net: a Tensor with the logits (pre-softmax activations) if num_classes + is a non-zero integer, or the non-dropped-out input to the logits layer + if num_classes is 0 or None. + end_points: a dictionary from components of the network to the corresponding + activation. + """ + # Final pooling and prediction + with tf.compat.v1.variable_scope( + scope, 'InceptionV1', [inputs], reuse=reuse) as scope: + with slim.arg_scope([slim.batch_norm, slim.dropout], + is_training=is_training): + net, end_points = inception_v1_base(inputs, scope=scope) + with tf.compat.v1.variable_scope('Logits'): + if global_pool: + # Global average pooling. + net = tf.reduce_mean( + input_tensor=net, axis=[1, 2], keepdims=True, name='global_pool') + end_points['global_pool'] = net + else: + # Pooling with a fixed kernel size. + net = slim.avg_pool2d(net, [7, 7], stride=1, scope='AvgPool_0a_7x7') + end_points['AvgPool_0a_7x7'] = net + if not num_classes: + return net, end_points + net = slim.dropout(net, dropout_keep_prob, scope='Dropout_0b') + logits = slim.conv2d(net, num_classes, [1, 1], activation_fn=None, + normalizer_fn=None, scope='Conv2d_0c_1x1') + if spatial_squeeze: + logits = tf.squeeze(logits, [1, 2], name='SpatialSqueeze') + + end_points['Logits'] = logits + end_points['Predictions'] = prediction_fn(logits, scope='Predictions') + return logits, end_points +inception_v1.default_image_size = 224 + +inception_v1_arg_scope = inception_utils.inception_arg_scope diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v1_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v1_test.py new file mode 100644 index 0000000..ce0fca4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v1_test.py @@ -0,0 +1,300 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for nets.inception_v1.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets import inception + +slim = contrib_slim + + +class InceptionV1Test(tf.test.TestCase): + + def testBuildClassificationNetwork(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, end_points = inception.inception_v1(inputs, num_classes) + self.assertTrue(logits.op.name.startswith( + 'InceptionV1/Logits/SpatialSqueeze')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertTrue('Predictions' in end_points) + self.assertListEqual(end_points['Predictions'].get_shape().as_list(), + [batch_size, num_classes]) + + def testBuildPreLogitsNetwork(self): + batch_size = 5 + height, width = 224, 224 + num_classes = None + + inputs = tf.random.uniform((batch_size, height, width, 3)) + net, end_points = inception.inception_v1(inputs, num_classes) + self.assertTrue(net.op.name.startswith('InceptionV1/Logits/AvgPool')) + self.assertListEqual(net.get_shape().as_list(), [batch_size, 1, 1, 1024]) + self.assertFalse('Logits' in end_points) + self.assertFalse('Predictions' in end_points) + + def testBuildBaseNetwork(self): + batch_size = 5 + height, width = 224, 224 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + mixed_6c, end_points = inception.inception_v1_base(inputs) + self.assertTrue(mixed_6c.op.name.startswith('InceptionV1/Mixed_5c')) + self.assertListEqual(mixed_6c.get_shape().as_list(), + [batch_size, 7, 7, 1024]) + expected_endpoints = ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', + 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', + 'Mixed_3c', 'MaxPool_4a_3x3', 'Mixed_4b', 'Mixed_4c', + 'Mixed_4d', 'Mixed_4e', 'Mixed_4f', 'MaxPool_5a_2x2', + 'Mixed_5b', 'Mixed_5c'] + self.assertItemsEqual(end_points.keys(), expected_endpoints) + + def testBuildOnlyUptoFinalEndpoint(self): + batch_size = 5 + height, width = 224, 224 + endpoints = ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', + 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', + 'MaxPool_4a_3x3', 'Mixed_4b', 'Mixed_4c', 'Mixed_4d', + 'Mixed_4e', 'Mixed_4f', 'MaxPool_5a_2x2', 'Mixed_5b', + 'Mixed_5c'] + for index, endpoint in enumerate(endpoints): + with tf.Graph().as_default(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + out_tensor, end_points = inception.inception_v1_base( + inputs, final_endpoint=endpoint) + self.assertTrue(out_tensor.op.name.startswith( + 'InceptionV1/' + endpoint)) + self.assertItemsEqual(endpoints[:index+1], end_points.keys()) + + def testBuildAndCheckAllEndPointsUptoMixed5c(self): + batch_size = 5 + height, width = 224, 224 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = inception.inception_v1_base(inputs, + final_endpoint='Mixed_5c') + endpoints_shapes = { + 'Conv2d_1a_7x7': [5, 112, 112, 64], + 'MaxPool_2a_3x3': [5, 56, 56, 64], + 'Conv2d_2b_1x1': [5, 56, 56, 64], + 'Conv2d_2c_3x3': [5, 56, 56, 192], + 'MaxPool_3a_3x3': [5, 28, 28, 192], + 'Mixed_3b': [5, 28, 28, 256], + 'Mixed_3c': [5, 28, 28, 480], + 'MaxPool_4a_3x3': [5, 14, 14, 480], + 'Mixed_4b': [5, 14, 14, 512], + 'Mixed_4c': [5, 14, 14, 512], + 'Mixed_4d': [5, 14, 14, 512], + 'Mixed_4e': [5, 14, 14, 528], + 'Mixed_4f': [5, 14, 14, 832], + 'MaxPool_5a_2x2': [5, 7, 7, 832], + 'Mixed_5b': [5, 7, 7, 832], + 'Mixed_5c': [5, 7, 7, 1024] + } + + self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) + for endpoint_name in endpoints_shapes: + expected_shape = endpoints_shapes[endpoint_name] + self.assertTrue(endpoint_name in end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testModelHasExpectedNumberOfParameters(self): + batch_size = 5 + height, width = 224, 224 + inputs = tf.random.uniform((batch_size, height, width, 3)) + with slim.arg_scope(inception.inception_v1_arg_scope()): + inception.inception_v1_base(inputs) + total_params, _ = slim.model_analyzer.analyze_vars( + slim.get_model_variables()) + self.assertAlmostEqual(5607184, total_params) + + def testHalfSizeImages(self): + batch_size = 5 + height, width = 112, 112 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + mixed_5c, _ = inception.inception_v1_base(inputs) + self.assertTrue(mixed_5c.op.name.startswith('InceptionV1/Mixed_5c')) + self.assertListEqual(mixed_5c.get_shape().as_list(), + [batch_size, 4, 4, 1024]) + + def testBuildBaseNetworkWithoutRootBlock(self): + batch_size = 5 + height, width = 28, 28 + channels = 192 + + inputs = tf.random.uniform((batch_size, height, width, channels)) + _, end_points = inception.inception_v1_base( + inputs, include_root_block=False) + endpoints_shapes = { + 'Mixed_3b': [5, 28, 28, 256], + 'Mixed_3c': [5, 28, 28, 480], + 'MaxPool_4a_3x3': [5, 14, 14, 480], + 'Mixed_4b': [5, 14, 14, 512], + 'Mixed_4c': [5, 14, 14, 512], + 'Mixed_4d': [5, 14, 14, 512], + 'Mixed_4e': [5, 14, 14, 528], + 'Mixed_4f': [5, 14, 14, 832], + 'MaxPool_5a_2x2': [5, 7, 7, 832], + 'Mixed_5b': [5, 7, 7, 832], + 'Mixed_5c': [5, 7, 7, 1024] + } + + self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) + for endpoint_name in endpoints_shapes: + expected_shape = endpoints_shapes[endpoint_name] + self.assertTrue(endpoint_name in end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testUnknownImageShape(self): + tf.compat.v1.reset_default_graph() + batch_size = 2 + height, width = 224, 224 + num_classes = 1000 + input_np = np.random.uniform(0, 1, (batch_size, height, width, 3)) + with self.test_session() as sess: + inputs = tf.compat.v1.placeholder( + tf.float32, shape=(batch_size, None, None, 3)) + logits, end_points = inception.inception_v1(inputs, num_classes) + self.assertTrue(logits.op.name.startswith('InceptionV1/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + pre_pool = end_points['Mixed_5c'] + feed_dict = {inputs: input_np} + tf.compat.v1.global_variables_initializer().run() + pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict) + self.assertListEqual(list(pre_pool_out.shape), [batch_size, 7, 7, 1024]) + + def testGlobalPoolUnknownImageShape(self): + tf.compat.v1.reset_default_graph() + batch_size = 1 + height, width = 250, 300 + num_classes = 1000 + input_np = np.random.uniform(0, 1, (batch_size, height, width, 3)) + with self.test_session() as sess: + inputs = tf.compat.v1.placeholder( + tf.float32, shape=(batch_size, None, None, 3)) + logits, end_points = inception.inception_v1(inputs, num_classes, + global_pool=True) + self.assertTrue(logits.op.name.startswith('InceptionV1/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + pre_pool = end_points['Mixed_5c'] + feed_dict = {inputs: input_np} + tf.compat.v1.global_variables_initializer().run() + pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict) + self.assertListEqual(list(pre_pool_out.shape), [batch_size, 8, 10, 1024]) + + def testUnknowBatchSize(self): + batch_size = 1 + height, width = 224, 224 + num_classes = 1000 + + inputs = tf.compat.v1.placeholder(tf.float32, (None, height, width, 3)) + logits, _ = inception.inception_v1(inputs, num_classes) + self.assertTrue(logits.op.name.startswith('InceptionV1/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [None, num_classes]) + images = tf.random.uniform((batch_size, height, width, 3)) + + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(logits, {inputs: images.eval()}) + self.assertEquals(output.shape, (batch_size, num_classes)) + + def testEvaluation(self): + batch_size = 2 + height, width = 224, 224 + num_classes = 1000 + + eval_inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = inception.inception_v1(eval_inputs, num_classes, + is_training=False) + predictions = tf.argmax(input=logits, axis=1) + + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(predictions) + self.assertEquals(output.shape, (batch_size,)) + + def testTrainEvalWithReuse(self): + train_batch_size = 5 + eval_batch_size = 2 + height, width = 224, 224 + num_classes = 1000 + + train_inputs = tf.random.uniform((train_batch_size, height, width, 3)) + inception.inception_v1(train_inputs, num_classes) + eval_inputs = tf.random.uniform((eval_batch_size, height, width, 3)) + logits, _ = inception.inception_v1(eval_inputs, num_classes, reuse=True) + predictions = tf.argmax(input=logits, axis=1) + + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(predictions) + self.assertEquals(output.shape, (eval_batch_size,)) + + def testLogitsNotSqueezed(self): + num_classes = 25 + images = tf.random.uniform([1, 224, 224, 3]) + logits, _ = inception.inception_v1(images, + num_classes=num_classes, + spatial_squeeze=False) + + with self.test_session() as sess: + tf.compat.v1.global_variables_initializer().run() + logits_out = sess.run(logits) + self.assertListEqual(list(logits_out.shape), [1, 1, 1, num_classes]) + + def testNoBatchNormScaleByDefault(self): + height, width = 224, 224 + num_classes = 1000 + inputs = tf.compat.v1.placeholder(tf.float32, (1, height, width, 3)) + with slim.arg_scope(inception.inception_v1_arg_scope()): + inception.inception_v1(inputs, num_classes, is_training=False) + + self.assertEqual(tf.compat.v1.global_variables('.*/BatchNorm/gamma:0$'), []) + + def testBatchNormScale(self): + height, width = 224, 224 + num_classes = 1000 + inputs = tf.compat.v1.placeholder(tf.float32, (1, height, width, 3)) + with slim.arg_scope( + inception.inception_v1_arg_scope(batch_norm_scale=True)): + inception.inception_v1(inputs, num_classes, is_training=False) + + gamma_names = set( + v.op.name + for v in tf.compat.v1.global_variables('.*/BatchNorm/gamma:0$')) + self.assertGreater(len(gamma_names), 0) + for v in tf.compat.v1.global_variables('.*/BatchNorm/moving_mean:0$'): + self.assertIn(v.op.name[:-len('moving_mean')] + 'gamma', gamma_names) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v2.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v2.py new file mode 100644 index 0000000..859c901 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v2.py @@ -0,0 +1,596 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains the definition for inception v2 classification network.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets import inception_utils + +slim = contrib_slim + +# pylint: disable=g-long-lambda +trunc_normal = lambda stddev: tf.compat.v1.truncated_normal_initializer( + 0.0, stddev) + + +def inception_v2_base(inputs, + final_endpoint='Mixed_5c', + min_depth=16, + depth_multiplier=1.0, + use_separable_conv=True, + data_format='NHWC', + include_root_block=True, + scope=None): + """Inception v2 (6a2). + + Constructs an Inception v2 network from inputs to the given final endpoint. + This method can construct the network up to the layer inception(5b) as + described in http://arxiv.org/abs/1502.03167. + + Args: + inputs: a tensor of shape [batch_size, height, width, channels]. + final_endpoint: specifies the endpoint to construct the network up to. It + can be one of ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', + 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', 'Mixed_4a', + 'Mixed_4b', 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', 'Mixed_5a', 'Mixed_5b', + 'Mixed_5c']. If include_root_block is False, ['Conv2d_1a_7x7', + 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', 'Conv2d_2c_3x3', 'MaxPool_3a_3x3'] will + not be available. + min_depth: Minimum depth value (number of channels) for all convolution ops. + Enforced when depth_multiplier < 1, and not an active constraint when + depth_multiplier >= 1. + depth_multiplier: Float multiplier for the depth (number of channels) + for all convolution ops. The value must be greater than zero. Typical + usage will be to set this value in (0, 1) to reduce the number of + parameters or computation cost of the model. + use_separable_conv: Use a separable convolution for the first layer + Conv2d_1a_7x7. If this is False, use a normal convolution instead. + data_format: Data format of the activations ('NHWC' or 'NCHW'). + include_root_block: If True, include the convolution and max-pooling layers + before the inception modules. If False, excludes those layers. + scope: Optional variable_scope. + + Returns: + tensor_out: output tensor corresponding to the final_endpoint. + end_points: a set of activations for external use, for example summaries or + losses. + + Raises: + ValueError: if final_endpoint is not set to one of the predefined values, + or depth_multiplier <= 0 + """ + + # end_points will collect relevant activations for external use, for example + # summaries or losses. + end_points = {} + + # Used to find thinned depths for each layer. + if depth_multiplier <= 0: + raise ValueError('depth_multiplier is not greater than zero.') + depth = lambda d: max(int(d * depth_multiplier), min_depth) + + if data_format != 'NHWC' and data_format != 'NCHW': + raise ValueError('data_format must be either NHWC or NCHW.') + if data_format == 'NCHW' and use_separable_conv: + raise ValueError( + 'separable convolution only supports NHWC layout. NCHW data format can' + ' only be used when use_separable_conv is False.' + ) + + concat_dim = 3 if data_format == 'NHWC' else 1 + with tf.compat.v1.variable_scope(scope, 'InceptionV2', [inputs]): + with slim.arg_scope( + [slim.conv2d, slim.max_pool2d, slim.avg_pool2d], + stride=1, + padding='SAME', + data_format=data_format): + + net = inputs + if include_root_block: + # Note that sizes in the comments below assume an input spatial size of + # 224x224, however, the inputs can be of any size greater 32x32. + + # 224 x 224 x 3 + end_point = 'Conv2d_1a_7x7' + + if use_separable_conv: + # depthwise_multiplier here is different from depth_multiplier. + # depthwise_multiplier determines the output channels of the initial + # depthwise conv (see docs for tf.nn.separable_conv2d), while + # depth_multiplier controls the # channels of the subsequent 1x1 + # convolution. Must have + # in_channels * depthwise_multipler <= out_channels + # so that the separable convolution is not overparameterized. + depthwise_multiplier = min(int(depth(64) / 3), 8) + net = slim.separable_conv2d( + inputs, + depth(64), [7, 7], + depth_multiplier=depthwise_multiplier, + stride=2, + padding='SAME', + weights_initializer=trunc_normal(1.0), + scope=end_point) + else: + # Use a normal convolution instead of a separable convolution. + net = slim.conv2d( + inputs, + depth(64), [7, 7], + stride=2, + weights_initializer=trunc_normal(1.0), + scope=end_point) + end_points[end_point] = net + if end_point == final_endpoint: + return net, end_points + # 112 x 112 x 64 + end_point = 'MaxPool_2a_3x3' + net = slim.max_pool2d(net, [3, 3], scope=end_point, stride=2) + end_points[end_point] = net + if end_point == final_endpoint: + return net, end_points + # 56 x 56 x 64 + end_point = 'Conv2d_2b_1x1' + net = slim.conv2d( + net, + depth(64), [1, 1], + scope=end_point, + weights_initializer=trunc_normal(0.1)) + end_points[end_point] = net + if end_point == final_endpoint: + return net, end_points + # 56 x 56 x 64 + end_point = 'Conv2d_2c_3x3' + net = slim.conv2d(net, depth(192), [3, 3], scope=end_point) + end_points[end_point] = net + if end_point == final_endpoint: + return net, end_points + # 56 x 56 x 192 + end_point = 'MaxPool_3a_3x3' + net = slim.max_pool2d(net, [3, 3], scope=end_point, stride=2) + end_points[end_point] = net + if end_point == final_endpoint: + return net, end_points + + # 28 x 28 x 192 + # Inception module. + end_point = 'Mixed_3b' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d( + net, depth(64), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, depth(64), [3, 3], + scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d( + net, depth(64), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], + scope='Conv2d_0b_3x3') + branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], + scope='Conv2d_0c_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d( + branch_3, depth(32), [1, 1], + weights_initializer=trunc_normal(0.1), + scope='Conv2d_0b_1x1') + net = tf.concat( + axis=concat_dim, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + # 28 x 28 x 256 + end_point = 'Mixed_3c' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d( + net, depth(64), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, depth(96), [3, 3], + scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d( + net, depth(64), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], + scope='Conv2d_0b_3x3') + branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], + scope='Conv2d_0c_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d( + branch_3, depth(64), [1, 1], + weights_initializer=trunc_normal(0.1), + scope='Conv2d_0b_1x1') + net = tf.concat( + axis=concat_dim, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + # 28 x 28 x 320 + end_point = 'Mixed_4a' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d( + net, depth(128), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_0 = slim.conv2d(branch_0, depth(160), [3, 3], stride=2, + scope='Conv2d_1a_3x3') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d( + net, depth(64), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d( + branch_1, depth(96), [3, 3], scope='Conv2d_0b_3x3') + branch_1 = slim.conv2d( + branch_1, depth(96), [3, 3], stride=2, scope='Conv2d_1a_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.max_pool2d( + net, [3, 3], stride=2, scope='MaxPool_1a_3x3') + net = tf.concat(axis=concat_dim, values=[branch_0, branch_1, branch_2]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + # 14 x 14 x 576 + end_point = 'Mixed_4b' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(224), [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d( + net, depth(64), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d( + branch_1, depth(96), [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d( + net, depth(96), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, depth(128), [3, 3], + scope='Conv2d_0b_3x3') + branch_2 = slim.conv2d(branch_2, depth(128), [3, 3], + scope='Conv2d_0c_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d( + branch_3, depth(128), [1, 1], + weights_initializer=trunc_normal(0.1), + scope='Conv2d_0b_1x1') + net = tf.concat( + axis=concat_dim, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + # 14 x 14 x 576 + end_point = 'Mixed_4c' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d( + net, depth(96), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, depth(128), [3, 3], + scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d( + net, depth(96), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, depth(128), [3, 3], + scope='Conv2d_0b_3x3') + branch_2 = slim.conv2d(branch_2, depth(128), [3, 3], + scope='Conv2d_0c_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d( + branch_3, depth(128), [1, 1], + weights_initializer=trunc_normal(0.1), + scope='Conv2d_0b_1x1') + net = tf.concat( + axis=concat_dim, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + # 14 x 14 x 576 + end_point = 'Mixed_4d' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(160), [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d( + net, depth(128), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, depth(160), [3, 3], + scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d( + net, depth(128), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, depth(160), [3, 3], + scope='Conv2d_0b_3x3') + branch_2 = slim.conv2d(branch_2, depth(160), [3, 3], + scope='Conv2d_0c_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d( + branch_3, depth(96), [1, 1], + weights_initializer=trunc_normal(0.1), + scope='Conv2d_0b_1x1') + net = tf.concat( + axis=concat_dim, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + # 14 x 14 x 576 + end_point = 'Mixed_4e' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(96), [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d( + net, depth(128), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, depth(192), [3, 3], + scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d( + net, depth(160), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, depth(192), [3, 3], + scope='Conv2d_0b_3x3') + branch_2 = slim.conv2d(branch_2, depth(192), [3, 3], + scope='Conv2d_0c_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d( + branch_3, depth(96), [1, 1], + weights_initializer=trunc_normal(0.1), + scope='Conv2d_0b_1x1') + net = tf.concat( + axis=concat_dim, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + # 14 x 14 x 576 + end_point = 'Mixed_5a' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d( + net, depth(128), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_0 = slim.conv2d(branch_0, depth(192), [3, 3], stride=2, + scope='Conv2d_1a_3x3') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d( + net, depth(192), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, depth(256), [3, 3], + scope='Conv2d_0b_3x3') + branch_1 = slim.conv2d(branch_1, depth(256), [3, 3], stride=2, + scope='Conv2d_1a_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.max_pool2d(net, [3, 3], stride=2, + scope='MaxPool_1a_3x3') + net = tf.concat( + axis=concat_dim, values=[branch_0, branch_1, branch_2]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + # 7 x 7 x 1024 + end_point = 'Mixed_5b' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(352), [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d( + net, depth(192), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, depth(320), [3, 3], + scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d( + net, depth(160), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, depth(224), [3, 3], + scope='Conv2d_0b_3x3') + branch_2 = slim.conv2d(branch_2, depth(224), [3, 3], + scope='Conv2d_0c_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d( + branch_3, depth(128), [1, 1], + weights_initializer=trunc_normal(0.1), + scope='Conv2d_0b_1x1') + net = tf.concat( + axis=concat_dim, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + # 7 x 7 x 1024 + end_point = 'Mixed_5c' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(352), [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d( + net, depth(192), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, depth(320), [3, 3], + scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d( + net, depth(192), [1, 1], + weights_initializer=trunc_normal(0.09), + scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, depth(224), [3, 3], + scope='Conv2d_0b_3x3') + branch_2 = slim.conv2d(branch_2, depth(224), [3, 3], + scope='Conv2d_0c_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') + branch_3 = slim.conv2d( + branch_3, depth(128), [1, 1], + weights_initializer=trunc_normal(0.1), + scope='Conv2d_0b_1x1') + net = tf.concat( + axis=concat_dim, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + raise ValueError('Unknown final endpoint %s' % final_endpoint) + + +def inception_v2(inputs, + num_classes=1000, + is_training=True, + dropout_keep_prob=0.8, + min_depth=16, + depth_multiplier=1.0, + prediction_fn=slim.softmax, + spatial_squeeze=True, + reuse=None, + scope='InceptionV2', + global_pool=False): + """Inception v2 model for classification. + + Constructs an Inception v2 network for classification as described in + http://arxiv.org/abs/1502.03167. + + The default image size used to train this network is 224x224. + + Args: + inputs: a tensor of shape [batch_size, height, width, channels]. + num_classes: number of predicted classes. If 0 or None, the logits layer + is omitted and the input features to the logits layer (before dropout) + are returned instead. + is_training: whether is training or not. + dropout_keep_prob: the percentage of activation values that are retained. + min_depth: Minimum depth value (number of channels) for all convolution ops. + Enforced when depth_multiplier < 1, and not an active constraint when + depth_multiplier >= 1. + depth_multiplier: Float multiplier for the depth (number of channels) + for all convolution ops. The value must be greater than zero. Typical + usage will be to set this value in (0, 1) to reduce the number of + parameters or computation cost of the model. + prediction_fn: a function to get predictions out of logits. + spatial_squeeze: if True, logits is of shape [B, C], if false logits is of + shape [B, 1, 1, C], where B is batch_size and C is number of classes. + reuse: whether or not the network and its variables should be reused. To be + able to reuse 'scope' must be given. + scope: Optional variable_scope. + global_pool: Optional boolean flag to control the avgpooling before the + logits layer. If false or unset, pooling is done with a fixed window + that reduces default-sized inputs to 1x1, while larger inputs lead to + larger outputs. If true, any input size is pooled down to 1x1. + + Returns: + net: a Tensor with the logits (pre-softmax activations) if num_classes + is a non-zero integer, or the non-dropped-out input to the logits layer + if num_classes is 0 or None. + end_points: a dictionary from components of the network to the corresponding + activation. + + Raises: + ValueError: if final_endpoint is not set to one of the predefined values, + or depth_multiplier <= 0 + """ + if depth_multiplier <= 0: + raise ValueError('depth_multiplier is not greater than zero.') + + # Final pooling and prediction + with tf.compat.v1.variable_scope( + scope, 'InceptionV2', [inputs], reuse=reuse) as scope: + with slim.arg_scope([slim.batch_norm, slim.dropout], + is_training=is_training): + net, end_points = inception_v2_base( + inputs, scope=scope, min_depth=min_depth, + depth_multiplier=depth_multiplier) + with tf.compat.v1.variable_scope('Logits'): + if global_pool: + # Global average pooling. + net = tf.reduce_mean( + input_tensor=net, axis=[1, 2], keepdims=True, name='global_pool') + end_points['global_pool'] = net + else: + # Pooling with a fixed kernel size. + kernel_size = _reduced_kernel_size_for_small_input(net, [7, 7]) + net = slim.avg_pool2d(net, kernel_size, padding='VALID', + scope='AvgPool_1a_{}x{}'.format(*kernel_size)) + end_points['AvgPool_1a'] = net + if not num_classes: + return net, end_points + # 1 x 1 x 1024 + net = slim.dropout(net, keep_prob=dropout_keep_prob, scope='Dropout_1b') + end_points['PreLogits'] = net + logits = slim.conv2d(net, num_classes, [1, 1], activation_fn=None, + normalizer_fn=None, scope='Conv2d_1c_1x1') + if spatial_squeeze: + logits = tf.squeeze(logits, [1, 2], name='SpatialSqueeze') + end_points['Logits'] = logits + end_points['Predictions'] = prediction_fn(logits, scope='Predictions') + return logits, end_points +inception_v2.default_image_size = 224 + + +def _reduced_kernel_size_for_small_input(input_tensor, kernel_size): + """Define kernel size which is automatically reduced for small input. + + If the shape of the input images is unknown at graph construction time this + function assumes that the input images are is large enough. + + Args: + input_tensor: input tensor of size [batch_size, height, width, channels]. + kernel_size: desired kernel size of length 2: [kernel_height, kernel_width] + + Returns: + a tensor with the kernel size. + + TODO(jrru): Make this function work with unknown shapes. Theoretically, this + can be done with the code below. Problems are two-fold: (1) If the shape was + known, it will be lost. (2) inception.slim.ops._two_element_tuple cannot + handle tensors that define the kernel size. + shape = tf.shape(input_tensor) + return = tf.stack([tf.minimum(shape[1], kernel_size[0]), + tf.minimum(shape[2], kernel_size[1])]) + + """ + shape = input_tensor.get_shape().as_list() + if shape[1] is None or shape[2] is None: + kernel_size_out = kernel_size + else: + kernel_size_out = [min(shape[1], kernel_size[0]), + min(shape[2], kernel_size[1])] + return kernel_size_out + + +inception_v2_arg_scope = inception_utils.inception_arg_scope diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v2_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v2_test.py new file mode 100644 index 0000000..089a64d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v2_test.py @@ -0,0 +1,412 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for nets.inception_v2.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets import inception + +slim = contrib_slim + + +class InceptionV2Test(tf.test.TestCase): + + def testBuildClassificationNetwork(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, end_points = inception.inception_v2(inputs, num_classes) + self.assertTrue(logits.op.name.startswith( + 'InceptionV2/Logits/SpatialSqueeze')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertTrue('Predictions' in end_points) + self.assertListEqual(end_points['Predictions'].get_shape().as_list(), + [batch_size, num_classes]) + + def testBuildPreLogitsNetwork(self): + batch_size = 5 + height, width = 224, 224 + num_classes = None + + inputs = tf.random.uniform((batch_size, height, width, 3)) + net, end_points = inception.inception_v2(inputs, num_classes) + self.assertTrue(net.op.name.startswith('InceptionV2/Logits/AvgPool')) + self.assertListEqual(net.get_shape().as_list(), [batch_size, 1, 1, 1024]) + self.assertFalse('Logits' in end_points) + self.assertFalse('Predictions' in end_points) + + def testBuildBaseNetwork(self): + batch_size = 5 + height, width = 224, 224 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + mixed_5c, end_points = inception.inception_v2_base(inputs) + self.assertTrue(mixed_5c.op.name.startswith('InceptionV2/Mixed_5c')) + self.assertListEqual(mixed_5c.get_shape().as_list(), + [batch_size, 7, 7, 1024]) + expected_endpoints = ['Mixed_3b', 'Mixed_3c', 'Mixed_4a', 'Mixed_4b', + 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', 'Mixed_5a', + 'Mixed_5b', 'Mixed_5c', 'Conv2d_1a_7x7', + 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', 'Conv2d_2c_3x3', + 'MaxPool_3a_3x3'] + self.assertItemsEqual(list(end_points.keys()), expected_endpoints) + + def testBuildOnlyUptoFinalEndpoint(self): + batch_size = 5 + height, width = 224, 224 + endpoints = ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', + 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', + 'Mixed_4a', 'Mixed_4b', 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', + 'Mixed_5a', 'Mixed_5b', 'Mixed_5c'] + for index, endpoint in enumerate(endpoints): + with tf.Graph().as_default(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + out_tensor, end_points = inception.inception_v2_base( + inputs, final_endpoint=endpoint) + self.assertTrue(out_tensor.op.name.startswith( + 'InceptionV2/' + endpoint)) + self.assertItemsEqual(endpoints[:index + 1], list(end_points.keys())) + + def testBuildAndCheckAllEndPointsUptoMixed5c(self): + batch_size = 5 + height, width = 224, 224 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = inception.inception_v2_base(inputs, + final_endpoint='Mixed_5c') + endpoints_shapes = {'Mixed_3b': [batch_size, 28, 28, 256], + 'Mixed_3c': [batch_size, 28, 28, 320], + 'Mixed_4a': [batch_size, 14, 14, 576], + 'Mixed_4b': [batch_size, 14, 14, 576], + 'Mixed_4c': [batch_size, 14, 14, 576], + 'Mixed_4d': [batch_size, 14, 14, 576], + 'Mixed_4e': [batch_size, 14, 14, 576], + 'Mixed_5a': [batch_size, 7, 7, 1024], + 'Mixed_5b': [batch_size, 7, 7, 1024], + 'Mixed_5c': [batch_size, 7, 7, 1024], + 'Conv2d_1a_7x7': [batch_size, 112, 112, 64], + 'MaxPool_2a_3x3': [batch_size, 56, 56, 64], + 'Conv2d_2b_1x1': [batch_size, 56, 56, 64], + 'Conv2d_2c_3x3': [batch_size, 56, 56, 192], + 'MaxPool_3a_3x3': [batch_size, 28, 28, 192]} + self.assertItemsEqual( + list(endpoints_shapes.keys()), list(end_points.keys())) + for endpoint_name in endpoints_shapes: + expected_shape = endpoints_shapes[endpoint_name] + self.assertTrue(endpoint_name in end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testModelHasExpectedNumberOfParameters(self): + batch_size = 5 + height, width = 224, 224 + inputs = tf.random.uniform((batch_size, height, width, 3)) + with slim.arg_scope(inception.inception_v2_arg_scope()): + inception.inception_v2_base(inputs) + total_params, _ = slim.model_analyzer.analyze_vars( + slim.get_model_variables()) + self.assertAlmostEqual(10173112, total_params) + + def testBuildEndPointsWithDepthMultiplierLessThanOne(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = inception.inception_v2(inputs, num_classes) + + endpoint_keys = [key for key in end_points.keys() + if key.startswith('Mixed') or key.startswith('Conv')] + + _, end_points_with_multiplier = inception.inception_v2( + inputs, num_classes, scope='depth_multiplied_net', + depth_multiplier=0.5) + + for key in endpoint_keys: + original_depth = end_points[key].get_shape().as_list()[3] + new_depth = end_points_with_multiplier[key].get_shape().as_list()[3] + self.assertEqual(0.5 * original_depth, new_depth) + + def testBuildEndPointsWithDepthMultiplierGreaterThanOne(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = inception.inception_v2(inputs, num_classes) + + endpoint_keys = [key for key in end_points.keys() + if key.startswith('Mixed') or key.startswith('Conv')] + + _, end_points_with_multiplier = inception.inception_v2( + inputs, num_classes, scope='depth_multiplied_net', + depth_multiplier=2.0) + + for key in endpoint_keys: + original_depth = end_points[key].get_shape().as_list()[3] + new_depth = end_points_with_multiplier[key].get_shape().as_list()[3] + self.assertEqual(2.0 * original_depth, new_depth) + + def testRaiseValueErrorWithInvalidDepthMultiplier(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + with self.assertRaises(ValueError): + _ = inception.inception_v2(inputs, num_classes, depth_multiplier=-0.1) + with self.assertRaises(ValueError): + _ = inception.inception_v2(inputs, num_classes, depth_multiplier=0.0) + + def testBuildEndPointsWithUseSeparableConvolutionFalse(self): + batch_size = 5 + height, width = 224, 224 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = inception.inception_v2_base(inputs) + + endpoint_keys = [ + key for key in end_points.keys() + if key.startswith('Mixed') or key.startswith('Conv') + ] + + _, end_points_with_replacement = inception.inception_v2_base( + inputs, use_separable_conv=False) + + # The endpoint shapes must be equal to the original shape even when the + # separable convolution is replaced with a normal convolution. + for key in endpoint_keys: + original_shape = end_points[key].get_shape().as_list() + self.assertTrue(key in end_points_with_replacement) + new_shape = end_points_with_replacement[key].get_shape().as_list() + self.assertListEqual(original_shape, new_shape) + + def testBuildEndPointsNCHWDataFormat(self): + batch_size = 5 + height, width = 224, 224 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = inception.inception_v2_base(inputs) + + endpoint_keys = [ + key for key in end_points.keys() + if key.startswith('Mixed') or key.startswith('Conv') + ] + + inputs_in_nchw = tf.random.uniform((batch_size, 3, height, width)) + _, end_points_with_replacement = inception.inception_v2_base( + inputs_in_nchw, use_separable_conv=False, data_format='NCHW') + + # With the 'NCHW' data format, all endpoint activations have a transposed + # shape from the original shape with the 'NHWC' layout. + for key in endpoint_keys: + transposed_original_shape = tf.transpose( + a=end_points[key], perm=[0, 3, 1, 2]).get_shape().as_list() + self.assertTrue(key in end_points_with_replacement) + new_shape = end_points_with_replacement[key].get_shape().as_list() + self.assertListEqual(transposed_original_shape, new_shape) + + def testBuildErrorsForDataFormats(self): + batch_size = 5 + height, width = 224, 224 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + + # 'NCWH' data format is not supported. + with self.assertRaises(ValueError): + _ = inception.inception_v2_base(inputs, data_format='NCWH') + + # 'NCHW' data format is not supported for separable convolution. + with self.assertRaises(ValueError): + _ = inception.inception_v2_base(inputs, data_format='NCHW') + + def testHalfSizeImages(self): + batch_size = 5 + height, width = 112, 112 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, end_points = inception.inception_v2(inputs, num_classes) + self.assertTrue(logits.op.name.startswith('InceptionV2/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + pre_pool = end_points['Mixed_5c'] + self.assertListEqual(pre_pool.get_shape().as_list(), + [batch_size, 4, 4, 1024]) + + def testBuildBaseNetworkWithoutRootBlock(self): + batch_size = 5 + height, width = 28, 28 + channels = 192 + + inputs = tf.random.uniform((batch_size, height, width, channels)) + _, end_points = inception.inception_v2_base( + inputs, include_root_block=False) + endpoints_shapes = { + 'Mixed_3b': [batch_size, 28, 28, 256], + 'Mixed_3c': [batch_size, 28, 28, 320], + 'Mixed_4a': [batch_size, 14, 14, 576], + 'Mixed_4b': [batch_size, 14, 14, 576], + 'Mixed_4c': [batch_size, 14, 14, 576], + 'Mixed_4d': [batch_size, 14, 14, 576], + 'Mixed_4e': [batch_size, 14, 14, 576], + 'Mixed_5a': [batch_size, 7, 7, 1024], + 'Mixed_5b': [batch_size, 7, 7, 1024], + 'Mixed_5c': [batch_size, 7, 7, 1024] + } + self.assertItemsEqual( + list(endpoints_shapes.keys()), list(end_points.keys())) + for endpoint_name in endpoints_shapes: + expected_shape = endpoints_shapes[endpoint_name] + self.assertTrue(endpoint_name in end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testUnknownImageShape(self): + tf.compat.v1.reset_default_graph() + batch_size = 2 + height, width = 224, 224 + num_classes = 1000 + input_np = np.random.uniform(0, 1, (batch_size, height, width, 3)) + with self.test_session() as sess: + inputs = tf.compat.v1.placeholder( + tf.float32, shape=(batch_size, None, None, 3)) + logits, end_points = inception.inception_v2(inputs, num_classes) + self.assertTrue(logits.op.name.startswith('InceptionV2/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + pre_pool = end_points['Mixed_5c'] + feed_dict = {inputs: input_np} + tf.compat.v1.global_variables_initializer().run() + pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict) + self.assertListEqual(list(pre_pool_out.shape), [batch_size, 7, 7, 1024]) + + def testGlobalPoolUnknownImageShape(self): + tf.compat.v1.reset_default_graph() + batch_size = 1 + height, width = 250, 300 + num_classes = 1000 + input_np = np.random.uniform(0, 1, (batch_size, height, width, 3)) + with self.test_session() as sess: + inputs = tf.compat.v1.placeholder( + tf.float32, shape=(batch_size, None, None, 3)) + logits, end_points = inception.inception_v2(inputs, num_classes, + global_pool=True) + self.assertTrue(logits.op.name.startswith('InceptionV2/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + pre_pool = end_points['Mixed_5c'] + feed_dict = {inputs: input_np} + tf.compat.v1.global_variables_initializer().run() + pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict) + self.assertListEqual(list(pre_pool_out.shape), [batch_size, 8, 10, 1024]) + + def testUnknowBatchSize(self): + batch_size = 1 + height, width = 224, 224 + num_classes = 1000 + + inputs = tf.compat.v1.placeholder(tf.float32, (None, height, width, 3)) + logits, _ = inception.inception_v2(inputs, num_classes) + self.assertTrue(logits.op.name.startswith('InceptionV2/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [None, num_classes]) + images = tf.random.uniform((batch_size, height, width, 3)) + + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(logits, {inputs: images.eval()}) + self.assertEquals(output.shape, (batch_size, num_classes)) + + def testEvaluation(self): + batch_size = 2 + height, width = 224, 224 + num_classes = 1000 + + eval_inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = inception.inception_v2(eval_inputs, num_classes, + is_training=False) + predictions = tf.argmax(input=logits, axis=1) + + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(predictions) + self.assertEquals(output.shape, (batch_size,)) + + def testTrainEvalWithReuse(self): + train_batch_size = 5 + eval_batch_size = 2 + height, width = 150, 150 + num_classes = 1000 + + train_inputs = tf.random.uniform((train_batch_size, height, width, 3)) + inception.inception_v2(train_inputs, num_classes) + eval_inputs = tf.random.uniform((eval_batch_size, height, width, 3)) + logits, _ = inception.inception_v2(eval_inputs, num_classes, reuse=True) + predictions = tf.argmax(input=logits, axis=1) + + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(predictions) + self.assertEquals(output.shape, (eval_batch_size,)) + + def testLogitsNotSqueezed(self): + num_classes = 25 + images = tf.random.uniform([1, 224, 224, 3]) + logits, _ = inception.inception_v2(images, + num_classes=num_classes, + spatial_squeeze=False) + + with self.test_session() as sess: + tf.compat.v1.global_variables_initializer().run() + logits_out = sess.run(logits) + self.assertListEqual(list(logits_out.shape), [1, 1, 1, num_classes]) + + def testNoBatchNormScaleByDefault(self): + height, width = 224, 224 + num_classes = 1000 + inputs = tf.compat.v1.placeholder(tf.float32, (1, height, width, 3)) + with slim.arg_scope(inception.inception_v2_arg_scope()): + inception.inception_v2(inputs, num_classes, is_training=False) + + self.assertEqual(tf.compat.v1.global_variables('.*/BatchNorm/gamma:0$'), []) + + def testBatchNormScale(self): + height, width = 224, 224 + num_classes = 1000 + inputs = tf.compat.v1.placeholder(tf.float32, (1, height, width, 3)) + with slim.arg_scope( + inception.inception_v2_arg_scope(batch_norm_scale=True)): + inception.inception_v2(inputs, num_classes, is_training=False) + + gamma_names = set( + v.op.name + for v in tf.compat.v1.global_variables('.*/BatchNorm/gamma:0$')) + self.assertGreater(len(gamma_names), 0) + for v in tf.compat.v1.global_variables('.*/BatchNorm/moving_mean:0$'): + self.assertIn(v.op.name[:-len('moving_mean')] + 'gamma', gamma_names) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v3.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v3.py new file mode 100644 index 0000000..7c9fd7d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v3.py @@ -0,0 +1,585 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains the definition for inception v3 classification network.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets import inception_utils + +slim = contrib_slim + +# pylint: disable=g-long-lambda +trunc_normal = lambda stddev: tf.compat.v1.truncated_normal_initializer( + 0.0, stddev) + + +def inception_v3_base(inputs, + final_endpoint='Mixed_7c', + min_depth=16, + depth_multiplier=1.0, + scope=None): + """Inception model from http://arxiv.org/abs/1512.00567. + + Constructs an Inception v3 network from inputs to the given final endpoint. + This method can construct the network up to the final inception block + Mixed_7c. + + Note that the names of the layers in the paper do not correspond to the names + of the endpoints registered by this function although they build the same + network. + + Here is a mapping from the old_names to the new names: + Old name | New name + ======================================= + conv0 | Conv2d_1a_3x3 + conv1 | Conv2d_2a_3x3 + conv2 | Conv2d_2b_3x3 + pool1 | MaxPool_3a_3x3 + conv3 | Conv2d_3b_1x1 + conv4 | Conv2d_4a_3x3 + pool2 | MaxPool_5a_3x3 + mixed_35x35x256a | Mixed_5b + mixed_35x35x288a | Mixed_5c + mixed_35x35x288b | Mixed_5d + mixed_17x17x768a | Mixed_6a + mixed_17x17x768b | Mixed_6b + mixed_17x17x768c | Mixed_6c + mixed_17x17x768d | Mixed_6d + mixed_17x17x768e | Mixed_6e + mixed_8x8x1280a | Mixed_7a + mixed_8x8x2048a | Mixed_7b + mixed_8x8x2048b | Mixed_7c + + Args: + inputs: a tensor of size [batch_size, height, width, channels]. + final_endpoint: specifies the endpoint to construct the network up to. It + can be one of ['Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', + 'MaxPool_3a_3x3', 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', 'MaxPool_5a_3x3', + 'Mixed_5b', 'Mixed_5c', 'Mixed_5d', 'Mixed_6a', 'Mixed_6b', 'Mixed_6c', + 'Mixed_6d', 'Mixed_6e', 'Mixed_7a', 'Mixed_7b', 'Mixed_7c']. + min_depth: Minimum depth value (number of channels) for all convolution ops. + Enforced when depth_multiplier < 1, and not an active constraint when + depth_multiplier >= 1. + depth_multiplier: Float multiplier for the depth (number of channels) + for all convolution ops. The value must be greater than zero. Typical + usage will be to set this value in (0, 1) to reduce the number of + parameters or computation cost of the model. + scope: Optional variable_scope. + + Returns: + tensor_out: output tensor corresponding to the final_endpoint. + end_points: a set of activations for external use, for example summaries or + losses. + + Raises: + ValueError: if final_endpoint is not set to one of the predefined values, + or depth_multiplier <= 0 + """ + # end_points will collect relevant activations for external use, for example + # summaries or losses. + end_points = {} + + if depth_multiplier <= 0: + raise ValueError('depth_multiplier is not greater than zero.') + depth = lambda d: max(int(d * depth_multiplier), min_depth) + + with tf.compat.v1.variable_scope(scope, 'InceptionV3', [inputs]): + with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d], + stride=1, padding='VALID'): + # 299 x 299 x 3 + end_point = 'Conv2d_1a_3x3' + net = slim.conv2d(inputs, depth(32), [3, 3], stride=2, scope=end_point) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + # 149 x 149 x 32 + end_point = 'Conv2d_2a_3x3' + net = slim.conv2d(net, depth(32), [3, 3], scope=end_point) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + # 147 x 147 x 32 + end_point = 'Conv2d_2b_3x3' + net = slim.conv2d(net, depth(64), [3, 3], padding='SAME', scope=end_point) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + # 147 x 147 x 64 + end_point = 'MaxPool_3a_3x3' + net = slim.max_pool2d(net, [3, 3], stride=2, scope=end_point) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + # 73 x 73 x 64 + end_point = 'Conv2d_3b_1x1' + net = slim.conv2d(net, depth(80), [1, 1], scope=end_point) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + # 73 x 73 x 80. + end_point = 'Conv2d_4a_3x3' + net = slim.conv2d(net, depth(192), [3, 3], scope=end_point) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + # 71 x 71 x 192. + end_point = 'MaxPool_5a_3x3' + net = slim.max_pool2d(net, [3, 3], stride=2, scope=end_point) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + # 35 x 35 x 192. + + # Inception blocks + with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d], + stride=1, padding='SAME'): + # mixed: 35 x 35 x 256. + end_point = 'Mixed_5b' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, depth(48), [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, depth(64), [5, 5], + scope='Conv2d_0b_5x5') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], + scope='Conv2d_0b_3x3') + branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], + scope='Conv2d_0c_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, depth(32), [1, 1], + scope='Conv2d_0b_1x1') + net = tf.concat(axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + + # mixed_1: 35 x 35 x 288. + end_point = 'Mixed_5c' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, depth(48), [1, 1], scope='Conv2d_0b_1x1') + branch_1 = slim.conv2d(branch_1, depth(64), [5, 5], + scope='Conv_1_0c_5x5') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(net, depth(64), [1, 1], + scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], + scope='Conv2d_0b_3x3') + branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], + scope='Conv2d_0c_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, depth(64), [1, 1], + scope='Conv2d_0b_1x1') + net = tf.concat(axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + + # mixed_2: 35 x 35 x 288. + end_point = 'Mixed_5d' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, depth(48), [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, depth(64), [5, 5], + scope='Conv2d_0b_5x5') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], + scope='Conv2d_0b_3x3') + branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], + scope='Conv2d_0c_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, depth(64), [1, 1], + scope='Conv2d_0b_1x1') + net = tf.concat(axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + + # mixed_3: 17 x 17 x 768. + end_point = 'Mixed_6a' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(384), [3, 3], stride=2, + padding='VALID', scope='Conv2d_1a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, depth(96), [3, 3], + scope='Conv2d_0b_3x3') + branch_1 = slim.conv2d(branch_1, depth(96), [3, 3], stride=2, + padding='VALID', scope='Conv2d_1a_1x1') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.max_pool2d(net, [3, 3], stride=2, padding='VALID', + scope='MaxPool_1a_3x3') + net = tf.concat(axis=3, values=[branch_0, branch_1, branch_2]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + + # mixed4: 17 x 17 x 768. + end_point = 'Mixed_6b' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, depth(128), [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, depth(128), [1, 7], + scope='Conv2d_0b_1x7') + branch_1 = slim.conv2d(branch_1, depth(192), [7, 1], + scope='Conv2d_0c_7x1') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(net, depth(128), [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, depth(128), [7, 1], + scope='Conv2d_0b_7x1') + branch_2 = slim.conv2d(branch_2, depth(128), [1, 7], + scope='Conv2d_0c_1x7') + branch_2 = slim.conv2d(branch_2, depth(128), [7, 1], + scope='Conv2d_0d_7x1') + branch_2 = slim.conv2d(branch_2, depth(192), [1, 7], + scope='Conv2d_0e_1x7') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, depth(192), [1, 1], + scope='Conv2d_0b_1x1') + net = tf.concat(axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + + # mixed_5: 17 x 17 x 768. + end_point = 'Mixed_6c' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, depth(160), [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, depth(160), [1, 7], + scope='Conv2d_0b_1x7') + branch_1 = slim.conv2d(branch_1, depth(192), [7, 1], + scope='Conv2d_0c_7x1') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(net, depth(160), [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, depth(160), [7, 1], + scope='Conv2d_0b_7x1') + branch_2 = slim.conv2d(branch_2, depth(160), [1, 7], + scope='Conv2d_0c_1x7') + branch_2 = slim.conv2d(branch_2, depth(160), [7, 1], + scope='Conv2d_0d_7x1') + branch_2 = slim.conv2d(branch_2, depth(192), [1, 7], + scope='Conv2d_0e_1x7') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, depth(192), [1, 1], + scope='Conv2d_0b_1x1') + net = tf.concat(axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + # mixed_6: 17 x 17 x 768. + end_point = 'Mixed_6d' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, depth(160), [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, depth(160), [1, 7], + scope='Conv2d_0b_1x7') + branch_1 = slim.conv2d(branch_1, depth(192), [7, 1], + scope='Conv2d_0c_7x1') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(net, depth(160), [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, depth(160), [7, 1], + scope='Conv2d_0b_7x1') + branch_2 = slim.conv2d(branch_2, depth(160), [1, 7], + scope='Conv2d_0c_1x7') + branch_2 = slim.conv2d(branch_2, depth(160), [7, 1], + scope='Conv2d_0d_7x1') + branch_2 = slim.conv2d(branch_2, depth(192), [1, 7], + scope='Conv2d_0e_1x7') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, depth(192), [1, 1], + scope='Conv2d_0b_1x1') + net = tf.concat(axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + + # mixed_7: 17 x 17 x 768. + end_point = 'Mixed_6e' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, depth(192), [1, 7], + scope='Conv2d_0b_1x7') + branch_1 = slim.conv2d(branch_1, depth(192), [7, 1], + scope='Conv2d_0c_7x1') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, depth(192), [7, 1], + scope='Conv2d_0b_7x1') + branch_2 = slim.conv2d(branch_2, depth(192), [1, 7], + scope='Conv2d_0c_1x7') + branch_2 = slim.conv2d(branch_2, depth(192), [7, 1], + scope='Conv2d_0d_7x1') + branch_2 = slim.conv2d(branch_2, depth(192), [1, 7], + scope='Conv2d_0e_1x7') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, depth(192), [1, 1], + scope='Conv2d_0b_1x1') + net = tf.concat(axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + + # mixed_8: 8 x 8 x 1280. + end_point = 'Mixed_7a' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') + branch_0 = slim.conv2d(branch_0, depth(320), [3, 3], stride=2, + padding='VALID', scope='Conv2d_1a_3x3') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, depth(192), [1, 7], + scope='Conv2d_0b_1x7') + branch_1 = slim.conv2d(branch_1, depth(192), [7, 1], + scope='Conv2d_0c_7x1') + branch_1 = slim.conv2d(branch_1, depth(192), [3, 3], stride=2, + padding='VALID', scope='Conv2d_1a_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.max_pool2d(net, [3, 3], stride=2, padding='VALID', + scope='MaxPool_1a_3x3') + net = tf.concat(axis=3, values=[branch_0, branch_1, branch_2]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + # mixed_9: 8 x 8 x 2048. + end_point = 'Mixed_7b' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(320), [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, depth(384), [1, 1], scope='Conv2d_0a_1x1') + branch_1 = tf.concat(axis=3, values=[ + slim.conv2d(branch_1, depth(384), [1, 3], scope='Conv2d_0b_1x3'), + slim.conv2d(branch_1, depth(384), [3, 1], scope='Conv2d_0b_3x1')]) + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(net, depth(448), [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d( + branch_2, depth(384), [3, 3], scope='Conv2d_0b_3x3') + branch_2 = tf.concat(axis=3, values=[ + slim.conv2d(branch_2, depth(384), [1, 3], scope='Conv2d_0c_1x3'), + slim.conv2d(branch_2, depth(384), [3, 1], scope='Conv2d_0d_3x1')]) + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d( + branch_3, depth(192), [1, 1], scope='Conv2d_0b_1x1') + net = tf.concat(axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + + # mixed_10: 8 x 8 x 2048. + end_point = 'Mixed_7c' + with tf.compat.v1.variable_scope(end_point): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, depth(320), [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, depth(384), [1, 1], scope='Conv2d_0a_1x1') + branch_1 = tf.concat(axis=3, values=[ + slim.conv2d(branch_1, depth(384), [1, 3], scope='Conv2d_0b_1x3'), + slim.conv2d(branch_1, depth(384), [3, 1], scope='Conv2d_0c_3x1')]) + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(net, depth(448), [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d( + branch_2, depth(384), [3, 3], scope='Conv2d_0b_3x3') + branch_2 = tf.concat(axis=3, values=[ + slim.conv2d(branch_2, depth(384), [1, 3], scope='Conv2d_0c_1x3'), + slim.conv2d(branch_2, depth(384), [3, 1], scope='Conv2d_0d_3x1')]) + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d( + branch_3, depth(192), [1, 1], scope='Conv2d_0b_1x1') + net = tf.concat(axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + end_points[end_point] = net + if end_point == final_endpoint: return net, end_points + raise ValueError('Unknown final endpoint %s' % final_endpoint) + + +def inception_v3(inputs, + num_classes=1000, + is_training=True, + dropout_keep_prob=0.8, + min_depth=16, + depth_multiplier=1.0, + prediction_fn=slim.softmax, + spatial_squeeze=True, + reuse=None, + create_aux_logits=True, + scope='InceptionV3', + global_pool=False): + """Inception model from http://arxiv.org/abs/1512.00567. + + "Rethinking the Inception Architecture for Computer Vision" + + Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens, + Zbigniew Wojna. + + With the default arguments this method constructs the exact model defined in + the paper. However, one can experiment with variations of the inception_v3 + network by changing arguments dropout_keep_prob, min_depth and + depth_multiplier. + + The default image size used to train this network is 299x299. + + Args: + inputs: a tensor of size [batch_size, height, width, channels]. + num_classes: number of predicted classes. If 0 or None, the logits layer + is omitted and the input features to the logits layer (before dropout) + are returned instead. + is_training: whether is training or not. + dropout_keep_prob: the percentage of activation values that are retained. + min_depth: Minimum depth value (number of channels) for all convolution ops. + Enforced when depth_multiplier < 1, and not an active constraint when + depth_multiplier >= 1. + depth_multiplier: Float multiplier for the depth (number of channels) + for all convolution ops. The value must be greater than zero. Typical + usage will be to set this value in (0, 1) to reduce the number of + parameters or computation cost of the model. + prediction_fn: a function to get predictions out of logits. + spatial_squeeze: if True, logits is of shape [B, C], if false logits is of + shape [B, 1, 1, C], where B is batch_size and C is number of classes. + reuse: whether or not the network and its variables should be reused. To be + able to reuse 'scope' must be given. + create_aux_logits: Whether to create the auxiliary logits. + scope: Optional variable_scope. + global_pool: Optional boolean flag to control the avgpooling before the + logits layer. If false or unset, pooling is done with a fixed window + that reduces default-sized inputs to 1x1, while larger inputs lead to + larger outputs. If true, any input size is pooled down to 1x1. + + Returns: + net: a Tensor with the logits (pre-softmax activations) if num_classes + is a non-zero integer, or the non-dropped-out input to the logits layer + if num_classes is 0 or None. + end_points: a dictionary from components of the network to the corresponding + activation. + + Raises: + ValueError: if 'depth_multiplier' is less than or equal to zero. + """ + if depth_multiplier <= 0: + raise ValueError('depth_multiplier is not greater than zero.') + depth = lambda d: max(int(d * depth_multiplier), min_depth) + + with tf.compat.v1.variable_scope( + scope, 'InceptionV3', [inputs], reuse=reuse) as scope: + with slim.arg_scope([slim.batch_norm, slim.dropout], + is_training=is_training): + net, end_points = inception_v3_base( + inputs, scope=scope, min_depth=min_depth, + depth_multiplier=depth_multiplier) + + # Auxiliary Head logits + if create_aux_logits and num_classes: + with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d], + stride=1, padding='SAME'): + aux_logits = end_points['Mixed_6e'] + with tf.compat.v1.variable_scope('AuxLogits'): + aux_logits = slim.avg_pool2d( + aux_logits, [5, 5], stride=3, padding='VALID', + scope='AvgPool_1a_5x5') + aux_logits = slim.conv2d(aux_logits, depth(128), [1, 1], + scope='Conv2d_1b_1x1') + + # Shape of feature map before the final layer. + kernel_size = _reduced_kernel_size_for_small_input( + aux_logits, [5, 5]) + aux_logits = slim.conv2d( + aux_logits, depth(768), kernel_size, + weights_initializer=trunc_normal(0.01), + padding='VALID', scope='Conv2d_2a_{}x{}'.format(*kernel_size)) + aux_logits = slim.conv2d( + aux_logits, num_classes, [1, 1], activation_fn=None, + normalizer_fn=None, weights_initializer=trunc_normal(0.001), + scope='Conv2d_2b_1x1') + if spatial_squeeze: + aux_logits = tf.squeeze(aux_logits, [1, 2], name='SpatialSqueeze') + end_points['AuxLogits'] = aux_logits + + # Final pooling and prediction + with tf.compat.v1.variable_scope('Logits'): + if global_pool: + # Global average pooling. + net = tf.reduce_mean( + input_tensor=net, axis=[1, 2], keepdims=True, name='GlobalPool') + end_points['global_pool'] = net + else: + # Pooling with a fixed kernel size. + kernel_size = _reduced_kernel_size_for_small_input(net, [8, 8]) + net = slim.avg_pool2d(net, kernel_size, padding='VALID', + scope='AvgPool_1a_{}x{}'.format(*kernel_size)) + end_points['AvgPool_1a'] = net + if not num_classes: + return net, end_points + # 1 x 1 x 2048 + net = slim.dropout(net, keep_prob=dropout_keep_prob, scope='Dropout_1b') + end_points['PreLogits'] = net + # 2048 + logits = slim.conv2d(net, num_classes, [1, 1], activation_fn=None, + normalizer_fn=None, scope='Conv2d_1c_1x1') + if spatial_squeeze: + logits = tf.squeeze(logits, [1, 2], name='SpatialSqueeze') + # 1000 + end_points['Logits'] = logits + end_points['Predictions'] = prediction_fn(logits, scope='Predictions') + return logits, end_points +inception_v3.default_image_size = 299 + + +def _reduced_kernel_size_for_small_input(input_tensor, kernel_size): + """Define kernel size which is automatically reduced for small input. + + If the shape of the input images is unknown at graph construction time this + function assumes that the input images are is large enough. + + Args: + input_tensor: input tensor of size [batch_size, height, width, channels]. + kernel_size: desired kernel size of length 2: [kernel_height, kernel_width] + + Returns: + a tensor with the kernel size. + + TODO(jrru): Make this function work with unknown shapes. Theoretically, this + can be done with the code below. Problems are two-fold: (1) If the shape was + known, it will be lost. (2) inception.slim.ops._two_element_tuple cannot + handle tensors that define the kernel size. + shape = tf.shape(input_tensor) + return = tf.stack([tf.minimum(shape[1], kernel_size[0]), + tf.minimum(shape[2], kernel_size[1])]) + + """ + shape = input_tensor.get_shape().as_list() + if shape[1] is None or shape[2] is None: + kernel_size_out = kernel_size + else: + kernel_size_out = [min(shape[1], kernel_size[0]), + min(shape[2], kernel_size[1])] + return kernel_size_out + + +inception_v3_arg_scope = inception_utils.inception_arg_scope diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v3_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v3_test.py new file mode 100644 index 0000000..1bc2c13 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v3_test.py @@ -0,0 +1,350 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for nets.inception_v1.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets import inception + +slim = contrib_slim + + +class InceptionV3Test(tf.test.TestCase): + + def testBuildClassificationNetwork(self): + batch_size = 5 + height, width = 299, 299 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, end_points = inception.inception_v3(inputs, num_classes) + self.assertTrue(logits.op.name.startswith( + 'InceptionV3/Logits/SpatialSqueeze')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertTrue('Predictions' in end_points) + self.assertListEqual(end_points['Predictions'].get_shape().as_list(), + [batch_size, num_classes]) + + def testBuildPreLogitsNetwork(self): + batch_size = 5 + height, width = 299, 299 + num_classes = None + + inputs = tf.random.uniform((batch_size, height, width, 3)) + net, end_points = inception.inception_v3(inputs, num_classes) + self.assertTrue(net.op.name.startswith('InceptionV3/Logits/AvgPool')) + self.assertListEqual(net.get_shape().as_list(), [batch_size, 1, 1, 2048]) + self.assertFalse('Logits' in end_points) + self.assertFalse('Predictions' in end_points) + + def testBuildBaseNetwork(self): + batch_size = 5 + height, width = 299, 299 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + final_endpoint, end_points = inception.inception_v3_base(inputs) + self.assertTrue(final_endpoint.op.name.startswith( + 'InceptionV3/Mixed_7c')) + self.assertListEqual(final_endpoint.get_shape().as_list(), + [batch_size, 8, 8, 2048]) + expected_endpoints = ['Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', + 'MaxPool_3a_3x3', 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', + 'MaxPool_5a_3x3', 'Mixed_5b', 'Mixed_5c', 'Mixed_5d', + 'Mixed_6a', 'Mixed_6b', 'Mixed_6c', 'Mixed_6d', + 'Mixed_6e', 'Mixed_7a', 'Mixed_7b', 'Mixed_7c'] + self.assertItemsEqual(end_points.keys(), expected_endpoints) + + def testBuildOnlyUptoFinalEndpoint(self): + batch_size = 5 + height, width = 299, 299 + endpoints = ['Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', + 'MaxPool_3a_3x3', 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', + 'MaxPool_5a_3x3', 'Mixed_5b', 'Mixed_5c', 'Mixed_5d', + 'Mixed_6a', 'Mixed_6b', 'Mixed_6c', 'Mixed_6d', + 'Mixed_6e', 'Mixed_7a', 'Mixed_7b', 'Mixed_7c'] + + for index, endpoint in enumerate(endpoints): + with tf.Graph().as_default(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + out_tensor, end_points = inception.inception_v3_base( + inputs, final_endpoint=endpoint) + self.assertTrue(out_tensor.op.name.startswith( + 'InceptionV3/' + endpoint)) + self.assertItemsEqual(endpoints[:index+1], end_points.keys()) + + def testBuildAndCheckAllEndPointsUptoMixed7c(self): + batch_size = 5 + height, width = 299, 299 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = inception.inception_v3_base( + inputs, final_endpoint='Mixed_7c') + endpoints_shapes = {'Conv2d_1a_3x3': [batch_size, 149, 149, 32], + 'Conv2d_2a_3x3': [batch_size, 147, 147, 32], + 'Conv2d_2b_3x3': [batch_size, 147, 147, 64], + 'MaxPool_3a_3x3': [batch_size, 73, 73, 64], + 'Conv2d_3b_1x1': [batch_size, 73, 73, 80], + 'Conv2d_4a_3x3': [batch_size, 71, 71, 192], + 'MaxPool_5a_3x3': [batch_size, 35, 35, 192], + 'Mixed_5b': [batch_size, 35, 35, 256], + 'Mixed_5c': [batch_size, 35, 35, 288], + 'Mixed_5d': [batch_size, 35, 35, 288], + 'Mixed_6a': [batch_size, 17, 17, 768], + 'Mixed_6b': [batch_size, 17, 17, 768], + 'Mixed_6c': [batch_size, 17, 17, 768], + 'Mixed_6d': [batch_size, 17, 17, 768], + 'Mixed_6e': [batch_size, 17, 17, 768], + 'Mixed_7a': [batch_size, 8, 8, 1280], + 'Mixed_7b': [batch_size, 8, 8, 2048], + 'Mixed_7c': [batch_size, 8, 8, 2048]} + self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) + for endpoint_name in endpoints_shapes: + expected_shape = endpoints_shapes[endpoint_name] + self.assertTrue(endpoint_name in end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testModelHasExpectedNumberOfParameters(self): + batch_size = 5 + height, width = 299, 299 + inputs = tf.random.uniform((batch_size, height, width, 3)) + with slim.arg_scope(inception.inception_v3_arg_scope()): + inception.inception_v3_base(inputs) + total_params, _ = slim.model_analyzer.analyze_vars( + slim.get_model_variables()) + self.assertAlmostEqual(21802784, total_params) + + def testBuildEndPoints(self): + batch_size = 5 + height, width = 299, 299 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = inception.inception_v3(inputs, num_classes) + self.assertTrue('Logits' in end_points) + logits = end_points['Logits'] + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertTrue('AuxLogits' in end_points) + aux_logits = end_points['AuxLogits'] + self.assertListEqual(aux_logits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertTrue('Mixed_7c' in end_points) + pre_pool = end_points['Mixed_7c'] + self.assertListEqual(pre_pool.get_shape().as_list(), + [batch_size, 8, 8, 2048]) + self.assertTrue('PreLogits' in end_points) + pre_logits = end_points['PreLogits'] + self.assertListEqual(pre_logits.get_shape().as_list(), + [batch_size, 1, 1, 2048]) + + def testBuildEndPointsWithDepthMultiplierLessThanOne(self): + batch_size = 5 + height, width = 299, 299 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = inception.inception_v3(inputs, num_classes) + + endpoint_keys = [key for key in end_points.keys() + if key.startswith('Mixed') or key.startswith('Conv')] + + _, end_points_with_multiplier = inception.inception_v3( + inputs, num_classes, scope='depth_multiplied_net', + depth_multiplier=0.5) + + for key in endpoint_keys: + original_depth = end_points[key].get_shape().as_list()[3] + new_depth = end_points_with_multiplier[key].get_shape().as_list()[3] + self.assertEqual(0.5 * original_depth, new_depth) + + def testBuildEndPointsWithDepthMultiplierGreaterThanOne(self): + batch_size = 5 + height, width = 299, 299 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = inception.inception_v3(inputs, num_classes) + + endpoint_keys = [key for key in end_points.keys() + if key.startswith('Mixed') or key.startswith('Conv')] + + _, end_points_with_multiplier = inception.inception_v3( + inputs, num_classes, scope='depth_multiplied_net', + depth_multiplier=2.0) + + for key in endpoint_keys: + original_depth = end_points[key].get_shape().as_list()[3] + new_depth = end_points_with_multiplier[key].get_shape().as_list()[3] + self.assertEqual(2.0 * original_depth, new_depth) + + def testRaiseValueErrorWithInvalidDepthMultiplier(self): + batch_size = 5 + height, width = 299, 299 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + with self.assertRaises(ValueError): + _ = inception.inception_v3(inputs, num_classes, depth_multiplier=-0.1) + with self.assertRaises(ValueError): + _ = inception.inception_v3(inputs, num_classes, depth_multiplier=0.0) + + def testHalfSizeImages(self): + batch_size = 5 + height, width = 150, 150 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, end_points = inception.inception_v3(inputs, num_classes) + self.assertTrue(logits.op.name.startswith('InceptionV3/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + pre_pool = end_points['Mixed_7c'] + self.assertListEqual(pre_pool.get_shape().as_list(), + [batch_size, 3, 3, 2048]) + + def testUnknownImageShape(self): + tf.compat.v1.reset_default_graph() + batch_size = 2 + height, width = 299, 299 + num_classes = 1000 + input_np = np.random.uniform(0, 1, (batch_size, height, width, 3)) + with self.test_session() as sess: + inputs = tf.compat.v1.placeholder( + tf.float32, shape=(batch_size, None, None, 3)) + logits, end_points = inception.inception_v3(inputs, num_classes) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + pre_pool = end_points['Mixed_7c'] + feed_dict = {inputs: input_np} + tf.compat.v1.global_variables_initializer().run() + pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict) + self.assertListEqual(list(pre_pool_out.shape), [batch_size, 8, 8, 2048]) + + def testGlobalPoolUnknownImageShape(self): + tf.compat.v1.reset_default_graph() + batch_size = 1 + height, width = 330, 400 + num_classes = 1000 + input_np = np.random.uniform(0, 1, (batch_size, height, width, 3)) + with self.test_session() as sess: + inputs = tf.compat.v1.placeholder( + tf.float32, shape=(batch_size, None, None, 3)) + logits, end_points = inception.inception_v3(inputs, num_classes, + global_pool=True) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + pre_pool = end_points['Mixed_7c'] + feed_dict = {inputs: input_np} + tf.compat.v1.global_variables_initializer().run() + pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict) + self.assertListEqual(list(pre_pool_out.shape), [batch_size, 8, 11, 2048]) + + def testUnknowBatchSize(self): + batch_size = 1 + height, width = 299, 299 + num_classes = 1000 + + inputs = tf.compat.v1.placeholder(tf.float32, (None, height, width, 3)) + logits, _ = inception.inception_v3(inputs, num_classes) + self.assertTrue(logits.op.name.startswith('InceptionV3/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [None, num_classes]) + images = tf.random.uniform((batch_size, height, width, 3)) + + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(logits, {inputs: images.eval()}) + self.assertEquals(output.shape, (batch_size, num_classes)) + + def testEvaluation(self): + batch_size = 2 + height, width = 299, 299 + num_classes = 1000 + + eval_inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = inception.inception_v3(eval_inputs, num_classes, + is_training=False) + predictions = tf.argmax(input=logits, axis=1) + + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(predictions) + self.assertEquals(output.shape, (batch_size,)) + + def testTrainEvalWithReuse(self): + train_batch_size = 5 + eval_batch_size = 2 + height, width = 150, 150 + num_classes = 1000 + + train_inputs = tf.random.uniform((train_batch_size, height, width, 3)) + inception.inception_v3(train_inputs, num_classes) + eval_inputs = tf.random.uniform((eval_batch_size, height, width, 3)) + logits, _ = inception.inception_v3(eval_inputs, num_classes, + is_training=False, reuse=True) + predictions = tf.argmax(input=logits, axis=1) + + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(predictions) + self.assertEquals(output.shape, (eval_batch_size,)) + + def testLogitsNotSqueezed(self): + num_classes = 25 + images = tf.random.uniform([1, 299, 299, 3]) + logits, _ = inception.inception_v3(images, + num_classes=num_classes, + spatial_squeeze=False) + + with self.test_session() as sess: + tf.compat.v1.global_variables_initializer().run() + logits_out = sess.run(logits) + self.assertListEqual(list(logits_out.shape), [1, 1, 1, num_classes]) + + def testNoBatchNormScaleByDefault(self): + height, width = 299, 299 + num_classes = 1000 + inputs = tf.compat.v1.placeholder(tf.float32, (1, height, width, 3)) + with slim.arg_scope(inception.inception_v3_arg_scope()): + inception.inception_v3(inputs, num_classes, is_training=False) + + self.assertEqual(tf.compat.v1.global_variables('.*/BatchNorm/gamma:0$'), []) + + def testBatchNormScale(self): + height, width = 299, 299 + num_classes = 1000 + inputs = tf.compat.v1.placeholder(tf.float32, (1, height, width, 3)) + with slim.arg_scope( + inception.inception_v3_arg_scope(batch_norm_scale=True)): + inception.inception_v3(inputs, num_classes, is_training=False) + + gamma_names = set( + v.op.name + for v in tf.compat.v1.global_variables('.*/BatchNorm/gamma:0$')) + self.assertGreater(len(gamma_names), 0) + for v in tf.compat.v1.global_variables('.*/BatchNorm/moving_mean:0$'): + self.assertIn(v.op.name[:-len('moving_mean')] + 'gamma', gamma_names) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v4.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v4.py new file mode 100644 index 0000000..7c6678c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v4.py @@ -0,0 +1,347 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains the definition of the Inception V4 architecture. + +As described in http://arxiv.org/abs/1602.07261. + + Inception-v4, Inception-ResNet and the Impact of Residual Connections + on Learning + Christian Szegedy, Sergey Ioffe, Vincent Vanhoucke, Alex Alemi +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets import inception_utils + +slim = contrib_slim + + +def block_inception_a(inputs, scope=None, reuse=None): + """Builds Inception-A block for Inception v4 network.""" + # By default use stride=1 and SAME padding + with slim.arg_scope([slim.conv2d, slim.avg_pool2d, slim.max_pool2d], + stride=1, padding='SAME'): + with tf.compat.v1.variable_scope( + scope, 'BlockInceptionA', [inputs], reuse=reuse): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(inputs, 96, [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(inputs, 64, [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, 96, [3, 3], scope='Conv2d_0b_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(inputs, 64, [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, 96, [3, 3], scope='Conv2d_0b_3x3') + branch_2 = slim.conv2d(branch_2, 96, [3, 3], scope='Conv2d_0c_3x3') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(inputs, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, 96, [1, 1], scope='Conv2d_0b_1x1') + return tf.concat(axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + + +def block_reduction_a(inputs, scope=None, reuse=None): + """Builds Reduction-A block for Inception v4 network.""" + # By default use stride=1 and SAME padding + with slim.arg_scope([slim.conv2d, slim.avg_pool2d, slim.max_pool2d], + stride=1, padding='SAME'): + with tf.compat.v1.variable_scope( + scope, 'BlockReductionA', [inputs], reuse=reuse): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(inputs, 384, [3, 3], stride=2, padding='VALID', + scope='Conv2d_1a_3x3') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(inputs, 192, [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, 224, [3, 3], scope='Conv2d_0b_3x3') + branch_1 = slim.conv2d(branch_1, 256, [3, 3], stride=2, + padding='VALID', scope='Conv2d_1a_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.max_pool2d(inputs, [3, 3], stride=2, padding='VALID', + scope='MaxPool_1a_3x3') + return tf.concat(axis=3, values=[branch_0, branch_1, branch_2]) + + +def block_inception_b(inputs, scope=None, reuse=None): + """Builds Inception-B block for Inception v4 network.""" + # By default use stride=1 and SAME padding + with slim.arg_scope([slim.conv2d, slim.avg_pool2d, slim.max_pool2d], + stride=1, padding='SAME'): + with tf.compat.v1.variable_scope( + scope, 'BlockInceptionB', [inputs], reuse=reuse): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(inputs, 384, [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(inputs, 192, [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, 224, [1, 7], scope='Conv2d_0b_1x7') + branch_1 = slim.conv2d(branch_1, 256, [7, 1], scope='Conv2d_0c_7x1') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(inputs, 192, [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, 192, [7, 1], scope='Conv2d_0b_7x1') + branch_2 = slim.conv2d(branch_2, 224, [1, 7], scope='Conv2d_0c_1x7') + branch_2 = slim.conv2d(branch_2, 224, [7, 1], scope='Conv2d_0d_7x1') + branch_2 = slim.conv2d(branch_2, 256, [1, 7], scope='Conv2d_0e_1x7') + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(inputs, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, 128, [1, 1], scope='Conv2d_0b_1x1') + return tf.concat(axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + + +def block_reduction_b(inputs, scope=None, reuse=None): + """Builds Reduction-B block for Inception v4 network.""" + # By default use stride=1 and SAME padding + with slim.arg_scope([slim.conv2d, slim.avg_pool2d, slim.max_pool2d], + stride=1, padding='SAME'): + with tf.compat.v1.variable_scope( + scope, 'BlockReductionB', [inputs], reuse=reuse): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(inputs, 192, [1, 1], scope='Conv2d_0a_1x1') + branch_0 = slim.conv2d(branch_0, 192, [3, 3], stride=2, + padding='VALID', scope='Conv2d_1a_3x3') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(inputs, 256, [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, 256, [1, 7], scope='Conv2d_0b_1x7') + branch_1 = slim.conv2d(branch_1, 320, [7, 1], scope='Conv2d_0c_7x1') + branch_1 = slim.conv2d(branch_1, 320, [3, 3], stride=2, + padding='VALID', scope='Conv2d_1a_3x3') + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.max_pool2d(inputs, [3, 3], stride=2, padding='VALID', + scope='MaxPool_1a_3x3') + return tf.concat(axis=3, values=[branch_0, branch_1, branch_2]) + + +def block_inception_c(inputs, scope=None, reuse=None): + """Builds Inception-C block for Inception v4 network.""" + # By default use stride=1 and SAME padding + with slim.arg_scope([slim.conv2d, slim.avg_pool2d, slim.max_pool2d], + stride=1, padding='SAME'): + with tf.compat.v1.variable_scope( + scope, 'BlockInceptionC', [inputs], reuse=reuse): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(inputs, 256, [1, 1], scope='Conv2d_0a_1x1') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(inputs, 384, [1, 1], scope='Conv2d_0a_1x1') + branch_1 = tf.concat(axis=3, values=[ + slim.conv2d(branch_1, 256, [1, 3], scope='Conv2d_0b_1x3'), + slim.conv2d(branch_1, 256, [3, 1], scope='Conv2d_0c_3x1')]) + with tf.compat.v1.variable_scope('Branch_2'): + branch_2 = slim.conv2d(inputs, 384, [1, 1], scope='Conv2d_0a_1x1') + branch_2 = slim.conv2d(branch_2, 448, [3, 1], scope='Conv2d_0b_3x1') + branch_2 = slim.conv2d(branch_2, 512, [1, 3], scope='Conv2d_0c_1x3') + branch_2 = tf.concat(axis=3, values=[ + slim.conv2d(branch_2, 256, [1, 3], scope='Conv2d_0d_1x3'), + slim.conv2d(branch_2, 256, [3, 1], scope='Conv2d_0e_3x1')]) + with tf.compat.v1.variable_scope('Branch_3'): + branch_3 = slim.avg_pool2d(inputs, [3, 3], scope='AvgPool_0a_3x3') + branch_3 = slim.conv2d(branch_3, 256, [1, 1], scope='Conv2d_0b_1x1') + return tf.concat(axis=3, values=[branch_0, branch_1, branch_2, branch_3]) + + +def inception_v4_base(inputs, final_endpoint='Mixed_7d', scope=None): + """Creates the Inception V4 network up to the given final endpoint. + + Args: + inputs: a 4-D tensor of size [batch_size, height, width, 3]. + final_endpoint: specifies the endpoint to construct the network up to. + It can be one of [ 'Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', + 'Mixed_3a', 'Mixed_4a', 'Mixed_5a', 'Mixed_5b', 'Mixed_5c', 'Mixed_5d', + 'Mixed_5e', 'Mixed_6a', 'Mixed_6b', 'Mixed_6c', 'Mixed_6d', 'Mixed_6e', + 'Mixed_6f', 'Mixed_6g', 'Mixed_6h', 'Mixed_7a', 'Mixed_7b', 'Mixed_7c', + 'Mixed_7d'] + scope: Optional variable_scope. + + Returns: + logits: the logits outputs of the model. + end_points: the set of end_points from the inception model. + + Raises: + ValueError: if final_endpoint is not set to one of the predefined values, + """ + end_points = {} + + def add_and_check_final(name, net): + end_points[name] = net + return name == final_endpoint + + with tf.compat.v1.variable_scope(scope, 'InceptionV4', [inputs]): + with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d], + stride=1, padding='SAME'): + # 299 x 299 x 3 + net = slim.conv2d(inputs, 32, [3, 3], stride=2, + padding='VALID', scope='Conv2d_1a_3x3') + if add_and_check_final('Conv2d_1a_3x3', net): return net, end_points + # 149 x 149 x 32 + net = slim.conv2d(net, 32, [3, 3], padding='VALID', + scope='Conv2d_2a_3x3') + if add_and_check_final('Conv2d_2a_3x3', net): return net, end_points + # 147 x 147 x 32 + net = slim.conv2d(net, 64, [3, 3], scope='Conv2d_2b_3x3') + if add_and_check_final('Conv2d_2b_3x3', net): return net, end_points + # 147 x 147 x 64 + with tf.compat.v1.variable_scope('Mixed_3a'): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.max_pool2d(net, [3, 3], stride=2, padding='VALID', + scope='MaxPool_0a_3x3') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, 96, [3, 3], stride=2, padding='VALID', + scope='Conv2d_0a_3x3') + net = tf.concat(axis=3, values=[branch_0, branch_1]) + if add_and_check_final('Mixed_3a', net): return net, end_points + + # 73 x 73 x 160 + with tf.compat.v1.variable_scope('Mixed_4a'): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, 64, [1, 1], scope='Conv2d_0a_1x1') + branch_0 = slim.conv2d(branch_0, 96, [3, 3], padding='VALID', + scope='Conv2d_1a_3x3') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.conv2d(net, 64, [1, 1], scope='Conv2d_0a_1x1') + branch_1 = slim.conv2d(branch_1, 64, [1, 7], scope='Conv2d_0b_1x7') + branch_1 = slim.conv2d(branch_1, 64, [7, 1], scope='Conv2d_0c_7x1') + branch_1 = slim.conv2d(branch_1, 96, [3, 3], padding='VALID', + scope='Conv2d_1a_3x3') + net = tf.concat(axis=3, values=[branch_0, branch_1]) + if add_and_check_final('Mixed_4a', net): return net, end_points + + # 71 x 71 x 192 + with tf.compat.v1.variable_scope('Mixed_5a'): + with tf.compat.v1.variable_scope('Branch_0'): + branch_0 = slim.conv2d(net, 192, [3, 3], stride=2, padding='VALID', + scope='Conv2d_1a_3x3') + with tf.compat.v1.variable_scope('Branch_1'): + branch_1 = slim.max_pool2d(net, [3, 3], stride=2, padding='VALID', + scope='MaxPool_1a_3x3') + net = tf.concat(axis=3, values=[branch_0, branch_1]) + if add_and_check_final('Mixed_5a', net): return net, end_points + + # 35 x 35 x 384 + # 4 x Inception-A blocks + for idx in range(4): + block_scope = 'Mixed_5' + chr(ord('b') + idx) + net = block_inception_a(net, block_scope) + if add_and_check_final(block_scope, net): return net, end_points + + # 35 x 35 x 384 + # Reduction-A block + net = block_reduction_a(net, 'Mixed_6a') + if add_and_check_final('Mixed_6a', net): return net, end_points + + # 17 x 17 x 1024 + # 7 x Inception-B blocks + for idx in range(7): + block_scope = 'Mixed_6' + chr(ord('b') + idx) + net = block_inception_b(net, block_scope) + if add_and_check_final(block_scope, net): return net, end_points + + # 17 x 17 x 1024 + # Reduction-B block + net = block_reduction_b(net, 'Mixed_7a') + if add_and_check_final('Mixed_7a', net): return net, end_points + + # 8 x 8 x 1536 + # 3 x Inception-C blocks + for idx in range(3): + block_scope = 'Mixed_7' + chr(ord('b') + idx) + net = block_inception_c(net, block_scope) + if add_and_check_final(block_scope, net): return net, end_points + raise ValueError('Unknown final endpoint %s' % final_endpoint) + + +def inception_v4(inputs, num_classes=1001, is_training=True, + dropout_keep_prob=0.8, + reuse=None, + scope='InceptionV4', + create_aux_logits=True): + """Creates the Inception V4 model. + + Args: + inputs: a 4-D tensor of size [batch_size, height, width, 3]. + num_classes: number of predicted classes. If 0 or None, the logits layer + is omitted and the input features to the logits layer (before dropout) + are returned instead. + is_training: whether is training or not. + dropout_keep_prob: float, the fraction to keep before final layer. + reuse: whether or not the network and its variables should be reused. To be + able to reuse 'scope' must be given. + scope: Optional variable_scope. + create_aux_logits: Whether to include the auxiliary logits. + + Returns: + net: a Tensor with the logits (pre-softmax activations) if num_classes + is a non-zero integer, or the non-dropped input to the logits layer + if num_classes is 0 or None. + end_points: the set of end_points from the inception model. + """ + end_points = {} + with tf.compat.v1.variable_scope( + scope, 'InceptionV4', [inputs], reuse=reuse) as scope: + with slim.arg_scope([slim.batch_norm, slim.dropout], + is_training=is_training): + net, end_points = inception_v4_base(inputs, scope=scope) + + with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d], + stride=1, padding='SAME'): + # Auxiliary Head logits + if create_aux_logits and num_classes: + with tf.compat.v1.variable_scope('AuxLogits'): + # 17 x 17 x 1024 + aux_logits = end_points['Mixed_6h'] + aux_logits = slim.avg_pool2d(aux_logits, [5, 5], stride=3, + padding='VALID', + scope='AvgPool_1a_5x5') + aux_logits = slim.conv2d(aux_logits, 128, [1, 1], + scope='Conv2d_1b_1x1') + aux_logits = slim.conv2d(aux_logits, 768, + aux_logits.get_shape()[1:3], + padding='VALID', scope='Conv2d_2a') + aux_logits = slim.flatten(aux_logits) + aux_logits = slim.fully_connected(aux_logits, num_classes, + activation_fn=None, + scope='Aux_logits') + end_points['AuxLogits'] = aux_logits + + # Final pooling and prediction + # TODO(sguada,arnoegw): Consider adding a parameter global_pool which + # can be set to False to disable pooling here (as in resnet_*()). + with tf.compat.v1.variable_scope('Logits'): + # 8 x 8 x 1536 + kernel_size = net.get_shape()[1:3] + if kernel_size.is_fully_defined(): + net = slim.avg_pool2d(net, kernel_size, padding='VALID', + scope='AvgPool_1a') + else: + net = tf.reduce_mean( + input_tensor=net, + axis=[1, 2], + keepdims=True, + name='global_pool') + end_points['global_pool'] = net + if not num_classes: + return net, end_points + # 1 x 1 x 1536 + net = slim.dropout(net, dropout_keep_prob, scope='Dropout_1b') + net = slim.flatten(net, scope='PreLogitsFlatten') + end_points['PreLogitsFlatten'] = net + # 1536 + logits = slim.fully_connected(net, num_classes, activation_fn=None, + scope='Logits') + end_points['Logits'] = logits + end_points['Predictions'] = tf.nn.softmax(logits, name='Predictions') + return logits, end_points +inception_v4.default_image_size = 299 + + +inception_v4_arg_scope = inception_utils.inception_arg_scope diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v4_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v4_test.py new file mode 100644 index 0000000..bc8df47 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/inception_v4_test.py @@ -0,0 +1,287 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for slim.inception_v4.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets import inception + + +class InceptionTest(tf.test.TestCase): + + def testBuildLogits(self): + batch_size = 5 + height, width = 299, 299 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, end_points = inception.inception_v4(inputs, num_classes) + auxlogits = end_points['AuxLogits'] + predictions = end_points['Predictions'] + self.assertTrue(auxlogits.op.name.startswith('InceptionV4/AuxLogits')) + self.assertListEqual(auxlogits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertTrue(logits.op.name.startswith('InceptionV4/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertTrue(predictions.op.name.startswith( + 'InceptionV4/Logits/Predictions')) + self.assertListEqual(predictions.get_shape().as_list(), + [batch_size, num_classes]) + + def testBuildPreLogitsNetwork(self): + batch_size = 5 + height, width = 299, 299 + num_classes = None + inputs = tf.random.uniform((batch_size, height, width, 3)) + net, end_points = inception.inception_v4(inputs, num_classes) + self.assertTrue(net.op.name.startswith('InceptionV4/Logits/AvgPool')) + self.assertListEqual(net.get_shape().as_list(), [batch_size, 1, 1, 1536]) + self.assertFalse('Logits' in end_points) + self.assertFalse('Predictions' in end_points) + + def testBuildWithoutAuxLogits(self): + batch_size = 5 + height, width = 299, 299 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, endpoints = inception.inception_v4(inputs, num_classes, + create_aux_logits=False) + self.assertFalse('AuxLogits' in endpoints) + self.assertTrue(logits.op.name.startswith('InceptionV4/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + + def testAllEndPointsShapes(self): + batch_size = 5 + height, width = 299, 299 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = inception.inception_v4(inputs, num_classes) + endpoints_shapes = {'Conv2d_1a_3x3': [batch_size, 149, 149, 32], + 'Conv2d_2a_3x3': [batch_size, 147, 147, 32], + 'Conv2d_2b_3x3': [batch_size, 147, 147, 64], + 'Mixed_3a': [batch_size, 73, 73, 160], + 'Mixed_4a': [batch_size, 71, 71, 192], + 'Mixed_5a': [batch_size, 35, 35, 384], + # 4 x Inception-A blocks + 'Mixed_5b': [batch_size, 35, 35, 384], + 'Mixed_5c': [batch_size, 35, 35, 384], + 'Mixed_5d': [batch_size, 35, 35, 384], + 'Mixed_5e': [batch_size, 35, 35, 384], + # Reduction-A block + 'Mixed_6a': [batch_size, 17, 17, 1024], + # 7 x Inception-B blocks + 'Mixed_6b': [batch_size, 17, 17, 1024], + 'Mixed_6c': [batch_size, 17, 17, 1024], + 'Mixed_6d': [batch_size, 17, 17, 1024], + 'Mixed_6e': [batch_size, 17, 17, 1024], + 'Mixed_6f': [batch_size, 17, 17, 1024], + 'Mixed_6g': [batch_size, 17, 17, 1024], + 'Mixed_6h': [batch_size, 17, 17, 1024], + # Reduction-A block + 'Mixed_7a': [batch_size, 8, 8, 1536], + # 3 x Inception-C blocks + 'Mixed_7b': [batch_size, 8, 8, 1536], + 'Mixed_7c': [batch_size, 8, 8, 1536], + 'Mixed_7d': [batch_size, 8, 8, 1536], + # Logits and predictions + 'AuxLogits': [batch_size, num_classes], + 'global_pool': [batch_size, 1, 1, 1536], + 'PreLogitsFlatten': [batch_size, 1536], + 'Logits': [batch_size, num_classes], + 'Predictions': [batch_size, num_classes]} + self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) + for endpoint_name in endpoints_shapes: + expected_shape = endpoints_shapes[endpoint_name] + self.assertTrue(endpoint_name in end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testBuildBaseNetwork(self): + batch_size = 5 + height, width = 299, 299 + inputs = tf.random.uniform((batch_size, height, width, 3)) + net, end_points = inception.inception_v4_base(inputs) + self.assertTrue(net.op.name.startswith( + 'InceptionV4/Mixed_7d')) + self.assertListEqual(net.get_shape().as_list(), [batch_size, 8, 8, 1536]) + expected_endpoints = [ + 'Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', 'Mixed_3a', + 'Mixed_4a', 'Mixed_5a', 'Mixed_5b', 'Mixed_5c', 'Mixed_5d', + 'Mixed_5e', 'Mixed_6a', 'Mixed_6b', 'Mixed_6c', 'Mixed_6d', + 'Mixed_6e', 'Mixed_6f', 'Mixed_6g', 'Mixed_6h', 'Mixed_7a', + 'Mixed_7b', 'Mixed_7c', 'Mixed_7d'] + self.assertItemsEqual(end_points.keys(), expected_endpoints) + for name, op in end_points.items(): + self.assertTrue(op.name.startswith('InceptionV4/' + name)) + + def testBuildOnlyUpToFinalEndpoint(self): + batch_size = 5 + height, width = 299, 299 + all_endpoints = [ + 'Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', 'Mixed_3a', + 'Mixed_4a', 'Mixed_5a', 'Mixed_5b', 'Mixed_5c', 'Mixed_5d', + 'Mixed_5e', 'Mixed_6a', 'Mixed_6b', 'Mixed_6c', 'Mixed_6d', + 'Mixed_6e', 'Mixed_6f', 'Mixed_6g', 'Mixed_6h', 'Mixed_7a', + 'Mixed_7b', 'Mixed_7c', 'Mixed_7d'] + for index, endpoint in enumerate(all_endpoints): + with tf.Graph().as_default(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + out_tensor, end_points = inception.inception_v4_base( + inputs, final_endpoint=endpoint) + self.assertTrue(out_tensor.op.name.startswith( + 'InceptionV4/' + endpoint)) + self.assertItemsEqual(all_endpoints[:index+1], end_points.keys()) + + def testVariablesSetDevice(self): + batch_size = 5 + height, width = 299, 299 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + # Force all Variables to reside on the device. + with tf.compat.v1.variable_scope('on_cpu'), tf.device('/cpu:0'): + inception.inception_v4(inputs, num_classes) + with tf.compat.v1.variable_scope('on_gpu'), tf.device('/gpu:0'): + inception.inception_v4(inputs, num_classes) + for v in tf.compat.v1.get_collection( + tf.compat.v1.GraphKeys.GLOBAL_VARIABLES, scope='on_cpu'): + self.assertDeviceEqual(v.device, '/cpu:0') + for v in tf.compat.v1.get_collection( + tf.compat.v1.GraphKeys.GLOBAL_VARIABLES, scope='on_gpu'): + self.assertDeviceEqual(v.device, '/gpu:0') + + def testHalfSizeImages(self): + batch_size = 5 + height, width = 150, 150 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, end_points = inception.inception_v4(inputs, num_classes) + self.assertTrue(logits.op.name.startswith('InceptionV4/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + pre_pool = end_points['Mixed_7d'] + self.assertListEqual(pre_pool.get_shape().as_list(), + [batch_size, 3, 3, 1536]) + + def testGlobalPool(self): + batch_size = 1 + height, width = 350, 400 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, end_points = inception.inception_v4(inputs, num_classes) + self.assertTrue(logits.op.name.startswith('InceptionV4/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + pre_pool = end_points['Mixed_7d'] + self.assertListEqual(pre_pool.get_shape().as_list(), + [batch_size, 9, 11, 1536]) + + def testGlobalPoolUnknownImageShape(self): + batch_size = 1 + height, width = 350, 400 + num_classes = 1000 + with self.test_session() as sess: + inputs = tf.compat.v1.placeholder(tf.float32, (batch_size, None, None, 3)) + logits, end_points = inception.inception_v4( + inputs, num_classes, create_aux_logits=False) + self.assertTrue(logits.op.name.startswith('InceptionV4/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + pre_pool = end_points['Mixed_7d'] + images = tf.random.uniform((batch_size, height, width, 3)) + sess.run(tf.compat.v1.global_variables_initializer()) + logits_out, pre_pool_out = sess.run([logits, pre_pool], + {inputs: images.eval()}) + self.assertTupleEqual(logits_out.shape, (batch_size, num_classes)) + self.assertTupleEqual(pre_pool_out.shape, (batch_size, 9, 11, 1536)) + + def testUnknownBatchSize(self): + batch_size = 1 + height, width = 299, 299 + num_classes = 1000 + with self.test_session() as sess: + inputs = tf.compat.v1.placeholder(tf.float32, (None, height, width, 3)) + logits, _ = inception.inception_v4(inputs, num_classes) + self.assertTrue(logits.op.name.startswith('InceptionV4/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [None, num_classes]) + images = tf.random.uniform((batch_size, height, width, 3)) + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(logits, {inputs: images.eval()}) + self.assertEquals(output.shape, (batch_size, num_classes)) + + def testEvaluation(self): + batch_size = 2 + height, width = 299, 299 + num_classes = 1000 + with self.test_session() as sess: + eval_inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = inception.inception_v4(eval_inputs, + num_classes, + is_training=False) + predictions = tf.argmax(input=logits, axis=1) + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(predictions) + self.assertEquals(output.shape, (batch_size,)) + + def testTrainEvalWithReuse(self): + train_batch_size = 5 + eval_batch_size = 2 + height, width = 150, 150 + num_classes = 1000 + with self.test_session() as sess: + train_inputs = tf.random.uniform((train_batch_size, height, width, 3)) + inception.inception_v4(train_inputs, num_classes) + eval_inputs = tf.random.uniform((eval_batch_size, height, width, 3)) + logits, _ = inception.inception_v4(eval_inputs, + num_classes, + is_training=False, + reuse=True) + predictions = tf.argmax(input=logits, axis=1) + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(predictions) + self.assertEquals(output.shape, (eval_batch_size,)) + + def testNoBatchNormScaleByDefault(self): + height, width = 299, 299 + num_classes = 1000 + inputs = tf.compat.v1.placeholder(tf.float32, (1, height, width, 3)) + with contrib_slim.arg_scope(inception.inception_v4_arg_scope()): + inception.inception_v4(inputs, num_classes, is_training=False) + + self.assertEqual(tf.compat.v1.global_variables('.*/BatchNorm/gamma:0$'), []) + + def testBatchNormScale(self): + height, width = 299, 299 + num_classes = 1000 + inputs = tf.compat.v1.placeholder(tf.float32, (1, height, width, 3)) + with contrib_slim.arg_scope( + inception.inception_v4_arg_scope(batch_norm_scale=True)): + inception.inception_v4(inputs, num_classes, is_training=False) + + gamma_names = set( + v.op.name + for v in tf.compat.v1.global_variables('.*/BatchNorm/gamma:0$')) + self.assertGreater(len(gamma_names), 0) + for v in tf.compat.v1.global_variables('.*/BatchNorm/moving_mean:0$'): + self.assertIn(v.op.name[:-len('moving_mean')] + 'gamma', gamma_names) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/lenet.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/lenet.py new file mode 100644 index 0000000..9f269d2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/lenet.py @@ -0,0 +1,98 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains a variant of the LeNet model definition.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +slim = contrib_slim + + +def lenet(images, num_classes=10, is_training=False, + dropout_keep_prob=0.5, + prediction_fn=slim.softmax, + scope='LeNet'): + """Creates a variant of the LeNet model. + + Note that since the output is a set of 'logits', the values fall in the + interval of (-infinity, infinity). Consequently, to convert the outputs to a + probability distribution over the characters, one will need to convert them + using the softmax function: + + logits = lenet.lenet(images, is_training=False) + probabilities = tf.nn.softmax(logits) + predictions = tf.argmax(logits, 1) + + Args: + images: A batch of `Tensors` of size [batch_size, height, width, channels]. + num_classes: the number of classes in the dataset. If 0 or None, the logits + layer is omitted and the input features to the logits layer are returned + instead. + is_training: specifies whether or not we're currently training the model. + This variable will determine the behaviour of the dropout layer. + dropout_keep_prob: the percentage of activation values that are retained. + prediction_fn: a function to get predictions out of logits. + scope: Optional variable_scope. + + Returns: + net: a 2D Tensor with the logits (pre-softmax activations) if num_classes + is a non-zero integer, or the inon-dropped-out nput to the logits layer + if num_classes is 0 or None. + end_points: a dictionary from components of the network to the corresponding + activation. + """ + end_points = {} + + with tf.compat.v1.variable_scope(scope, 'LeNet', [images]): + net = end_points['conv1'] = slim.conv2d(images, 32, [5, 5], scope='conv1') + net = end_points['pool1'] = slim.max_pool2d(net, [2, 2], 2, scope='pool1') + net = end_points['conv2'] = slim.conv2d(net, 64, [5, 5], scope='conv2') + net = end_points['pool2'] = slim.max_pool2d(net, [2, 2], 2, scope='pool2') + net = slim.flatten(net) + end_points['Flatten'] = net + + net = end_points['fc3'] = slim.fully_connected(net, 1024, scope='fc3') + if not num_classes: + return net, end_points + net = end_points['dropout3'] = slim.dropout( + net, dropout_keep_prob, is_training=is_training, scope='dropout3') + logits = end_points['Logits'] = slim.fully_connected( + net, num_classes, activation_fn=None, scope='fc4') + + end_points['Predictions'] = prediction_fn(logits, scope='Predictions') + + return logits, end_points +lenet.default_image_size = 28 + + +def lenet_arg_scope(weight_decay=0.0): + """Defines the default lenet argument scope. + + Args: + weight_decay: The weight decay to use for regularizing the model. + + Returns: + An `arg_scope` to use for the inception v3 model. + """ + with slim.arg_scope( + [slim.conv2d, slim.fully_connected], + weights_regularizer=slim.l2_regularizer(weight_decay), + weights_initializer=tf.compat.v1.truncated_normal_initializer(stddev=0.1), + activation_fn=tf.nn.relu) as sc: + return sc diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/.idea/encodings.xml b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/.idea/encodings.xml new file mode 100644 index 0000000..15a15b2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/.idea/encodings.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/.idea/misc.xml b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/.idea/misc.xml new file mode 100644 index 0000000..65531ca --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/.idea/mobilenet.iml b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/.idea/mobilenet.iml new file mode 100644 index 0000000..d0876a7 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/.idea/mobilenet.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/.idea/modules.xml b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/.idea/modules.xml new file mode 100644 index 0000000..bdad2db --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/README.md b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/README.md new file mode 100644 index 0000000..677ece7 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/README.md @@ -0,0 +1,166 @@ +# MobileNet + +This folder contains building code for +[MobileNetV2](https://arxiv.org/abs/1801.04381) and +[MobilenetV3](https://arxiv.org/abs/1905.02244) networks. The architectural +definition for each model is located in [mobilenet_v2.py](mobilenet_v2.py) and +[mobilenet_v3.py](mobilenet_v3.py) respectively. + +For MobilenetV1 please refer to this [page](../mobilenet_v1.md) + +We have also introduced a family of MobileNets customized for the Edge TPU +accelerator found in +[Google Pixel4](https://blog.google/products/pixel/pixel-4/) devices. The +architectural definition for MobileNetEdgeTPU is located in +[mobilenet_v3.py](mobilenet_v3.py) + +## Performance + +### Mobilenet V3 latency + +This is the timing of [MobileNetV2] vs [MobileNetV3] using TF-Lite on the large +core of Pixel 1 phone. + +![Mobilenet V2 and V3 Latency for Pixel 1.png](g3doc/latency_pixel1.png) + +### MACs + +MACs, also sometimes known as MADDs - the number of multiply-accumulates needed +to compute an inference on a single image is a common metric to measure the +efficiency of the model. Full size Mobilenet V3 on image size 224 uses ~215 +Million MADDs (MMadds) while achieving accuracy 75.1%, while Mobilenet V2 uses +~300MMadds and achieving accuracy 72%. By comparison ResNet-50 uses +approximately 3500 MMAdds while achieving 76% accuracy. + +Below is the graph comparing Mobilenets and a few selected networks. The size of +each blob represents the number of parameters. Note for +[ShuffleNet](https://arxiv.org/abs/1707.01083) there are no published size +numbers. We estimate it to be comparable to MobileNetV2 numbers. + +![madds_top1_accuracy](g3doc/madds_top1_accuracy.png) + +### Mobilenet EdgeTPU latency + +The figure below shows the Pixel 4 Edge TPU latency of int8-quantized Mobilenet +EdgeTPU compared with MobilenetV2 and the minimalistic variants of MobilenetV3 +(see below). + +![Mobilenet Edge TPU latency for Pixel 4 Edge TPU.png](g3doc/edgetpu_latency.png) + +## Pretrained models + +### Mobilenet V3 Imagenet Checkpoints + +All mobilenet V3 checkpoints were trained with image resolution 224x224. All +phone latencies are in milliseconds, measured on large core. In addition to +large and small models this page also contains so-called minimalistic models, +these models have the same per-layer dimensions characteristic as MobilenetV3 +however, they don't utilize any of the advanced blocks (squeeze-and-excite +units, hard-swish, and 5x5 convolutions). While these models are less efficient +on CPU, we find that they are much more performant on GPU/DSP. + +| Imagenet Checkpoint | MACs (M) | Params (M) | Top1 | Pixel 1 | Pixel 2 | Pixel 3 | +| ------------------ | -------- | ---------- | ---- | ------- | ------- | ------- | +| [Large dm=1 (float)] | 217 | 5.4 | 75.2 | 51.2 | 61 | 44 | +| [Large dm=1 (8-bit)] | 217 | 5.4 | 73.9 | 44 | 42.5 | 32 | +| [Large dm=0.75 (float)] | 155 | 4.0 | 73.3 | 39.8 | 48 | 34 | +| [Small dm=1 (float)] | 66 | 2.9 | 67.5 | 15.8 | 19.4 | 14.4 | +| [Small dm=1 (8-bit)] | 66 | 2.9 | 64.9 | 15.5 | 15 | 10.7 | +| [Small dm=0.75 (float)] | 44 | 2.4 | 65.4 | 12.8 | 15.9 | 11.6 | + +#### Minimalistic checkpoints: + +| Imagenet Checkpoint | MACs (M) | Params (M) | Top1 | Pixel 1 | Pixel 2 | Pixel 3 | +| -------------- | -------- | ---------- | ---- | ------- | ------- | ------- | +| [Large minimalistic (float)] | 209 | 3.9 | 72.3 | 44.1 | 51 | 35 | +| [Large minimalistic (8-bit)][lm8] | 209 | 3.9 | 71.3 | 37 | 35 | 27 | +| [Small minimalistic (float)] | 65 | 2.0 | 61.9 | 12.2 | 15.1 | 11 | + +#### Edge TPU checkpoints: + +| Imagenet Checkpoint | MACs (M) | Params (M) | Top1 | Pixel 4 Edge TPU | Pixel 4 CPU | +| ----------------- | -------- | ---------- | ---- | ------- | ----------- | +| [MobilenetEdgeTPU dm=0.75 (8-bit)]| 624 | 2.9 | 73.5 | 3.1 | 13.8 | +| [MobilenetEdgeTPU dm=1 (8-bit)] | 990 | 4.0 | 75.6 | 3.6 | 20.6 | + + +Note: 8-bit quantized versions of the MobilenetEdgeTPU models were obtained +using Tensorflow Lite's +[post training quantization](https://www.tensorflow.org/lite/performance/post_training_quantization) +tool. + +[Small minimalistic (float)]: https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small-minimalistic_224_1.0_float.tgz +[Large minimalistic (float)]: https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_float.tgz +[lm8]: https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_uint8.tgz +[Large dm=1 (float)]: https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_float.tgz +[Small dm=1 (float)]: https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_float.tgz +[Large dm=1 (8-bit)]: https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_uint8.tgz +[Small dm=1 (8-bit)]: https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_uint8.tgz +[Large dm=0.75 (float)]: https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_0.75_float.tgz +[Small dm=0.75 (float)]: https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_0.75_float.tgz +[MobilenetEdgeTPU dm=0.75 (8-bit)]: https://storage.cloud.google.com/mobilenet_edgetpu/checkpoints/mobilenet_edgetpu_224_0.75.tgz +[MobilenetEdgeTPU dm=1 (8-bit)]: https://storage.cloud.google.com/mobilenet_edgetpu/checkpoints/mobilenet_edgetpu_224_1.0.tgz + +### Mobilenet V2 Imagenet Checkpoints + +Classification Checkpoint | MACs (M) | Parameters (M) | Top 1 Accuracy | Top 5 Accuracy | Mobile CPU (ms) Pixel 1 +---------------------------------------------------------------------------------------------------------- | -------- | -------------- | -------------- | -------------- | ----------------------- +[mobilenet_v2_1.4_224](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.4_224.tgz) | 582 | 6.06 | 75.0 | 92.5 | 138.0 +[mobilenet_v2_1.3_224](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.3_224.tgz) | 509 | 5.34 | 74.4 | 92.1 | 123.0 +[mobilenet_v2_1.0_224](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.0_224.tgz) | 300 | 3.47 | 71.8 | 91.0 | 73.8 +[mobilenet_v2_1.0_192](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.0_192.tgz) | 221 | 3.47 | 70.7 | 90.1 | 55.1 +[mobilenet_v2_1.0_160](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.0_160.tgz) | 154 | 3.47 | 68.8 | 89.0 | 40.2 +[mobilenet_v2_1.0_128](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.0_128.tgz) | 99 | 3.47 | 65.3 | 86.9 | 27.6 +[mobilenet_v2_1.0_96](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.0_96.tgz) | 56 | 3.47 | 60.3 | 83.2 | 17.6 +[mobilenet_v2_0.75_224](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_0.75_224.tgz) | 209 | 2.61 | 69.8 | 89.6 | 55.8 +[mobilenet_v2_0.75_192](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_0.75_192.tgz) | 153 | 2.61 | 68.7 | 88.9 | 41.6 +[mobilenet_v2_0.75_160](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_0.75_160.tgz) | 107 | 2.61 | 66.4 | 87.3 | 30.4 +[mobilenet_v2_0.75_128](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_0.75_128.tgz) | 69 | 2.61 | 63.2 | 85.3 | 21.9 +[mobilenet_v2_0.75_96](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_0.75_96.tgz) | 39 | 2.61 | 58.8 | 81.6 | 14.2 +[mobilenet_v2_0.5_224](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_0.5_224.tgz) | 97 | 1.95 | 65.4 | 86.4 | 28.7 +[mobilenet_v2_0.5_192](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_0.5_192.tgz) | 71 | 1.95 | 63.9 | 85.4 | 21.1 +[mobilenet_v2_0.5_160](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_0.5_160.tgz) | 50 | 1.95 | 61.0 | 83.2 | 14.9 +[mobilenet_v2_0.5_128](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_0.5_128.tgz) | 32 | 1.95 | 57.7 | 80.8 | 9.9 +[mobilenet_v2_0.5_96](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_0.5_96.tgz) | 18 | 1.95 | 51.2 | 75.8 | 6.4 +[mobilenet_v2_0.35_224](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_0.35_224.tgz) | 59 | 1.66 | 60.3 | 82.9 | 19.7 +[mobilenet_v2_0.35_192](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_0.35_192.tgz) | 43 | 1.66 | 58.2 | 81.2 | 14.6 +[mobilenet_v2_0.35_160](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_0.35_160.tgz) | 30 | 1.66 | 55.7 | 79.1 | 10.5 +[mobilenet_v2_0.35_128](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_0.35_128.tgz) | 20 | 1.66 | 50.8 | 75.0 | 6.9 +[mobilenet_v2_0.35_96](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_0.35_96.tgz) | 11 | 1.66 | 45.5 | 70.4 | 4.5 + +## Training + +### V3 + +TODO: Add V3 hyperparameters + +### V2 + +The numbers above can be reproduced using slim's +[`train_image_classifier`](https://github.com/tensorflow/models/blob/master/research/slim/README.md#training-a-model-from-scratch). +Below is the set of parameters that achieves 72.0% for full size MobileNetV2, +after about 700K when trained on 8 GPU. If trained on a single GPU the full +convergence is after 5.5M steps. Also note that learning rate and +num_epochs_per_decay both need to be adjusted depending on how many GPUs are +being used due to slim's internal averaging. + +```bash +--model_name="mobilenet_v2" +--learning_rate=0.045 * NUM_GPUS #slim internally averages clones so we compensate +--preprocessing_name="inception_v2" +--label_smoothing=0.1 +--moving_average_decay=0.9999 +--batch_size= 96 +--num_clones = NUM_GPUS # you can use any number here between 1 and 8 depending on your hardware setup. +--learning_rate_decay_factor=0.98 +--num_epochs_per_decay = 2.5 / NUM_GPUS # train_image_classifier does per clone epochs +``` + +# Example + +See this [ipython notebook](mobilenet_example.ipynb) or open and run the network +directly in +[Colaboratory](https://colab.research.google.com/github/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet_example.ipynb). + +[MobilenetV2]: https://arxiv.org/abs/1801.04381 +[MobilenetV3]: https://arxiv.org/abs/1905.02244 diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/__init__.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/conv_blocks.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/conv_blocks.py new file mode 100644 index 0000000..85b0791 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/conv_blocks.py @@ -0,0 +1,475 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Convolution blocks for mobilenet.""" +import contextlib +import functools + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +slim = contrib_slim + + +def _fixed_padding(inputs, kernel_size, rate=1): + """Pads the input along the spatial dimensions independently of input size. + + Pads the input such that if it was used in a convolution with 'VALID' padding, + the output would have the same dimensions as if the unpadded input was used + in a convolution with 'SAME' padding. + + Args: + inputs: A tensor of size [batch, height_in, width_in, channels]. + kernel_size: The kernel to be used in the conv2d or max_pool2d operation. + rate: An integer, rate for atrous convolution. + + Returns: + output: A tensor of size [batch, height_out, width_out, channels] with the + input, either intact (if kernel_size == 1) or padded (if kernel_size > 1). + """ + kernel_size_effective = [kernel_size[0] + (kernel_size[0] - 1) * (rate - 1), + kernel_size[0] + (kernel_size[0] - 1) * (rate - 1)] + pad_total = [kernel_size_effective[0] - 1, kernel_size_effective[1] - 1] + pad_beg = [pad_total[0] // 2, pad_total[1] // 2] + pad_end = [pad_total[0] - pad_beg[0], pad_total[1] - pad_beg[1]] + padded_inputs = tf.pad(inputs, [[0, 0], [pad_beg[0], pad_end[0]], + [pad_beg[1], pad_end[1]], [0, 0]]) + return padded_inputs + + +def _make_divisible(v, divisor, min_value=None): + if min_value is None: + min_value = divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_v < 0.9 * v: + new_v += divisor + return new_v + + +def _split_divisible(num, num_ways, divisible_by=8): + """Evenly splits num, num_ways so each piece is a multiple of divisible_by.""" + assert num % divisible_by == 0 + assert num / num_ways >= divisible_by + # Note: want to round down, we adjust each split to match the total. + base = num // num_ways // divisible_by * divisible_by + result = [] + accumulated = 0 + for i in range(num_ways): + r = base + while accumulated + r < num * (i + 1) / num_ways: + r += divisible_by + result.append(r) + accumulated += r + assert accumulated == num + return result + + +@contextlib.contextmanager +def _v1_compatible_scope_naming(scope): + """v1 compatible scope naming.""" + if scope is None: # Create uniqified separable blocks. + with tf.compat.v1.variable_scope(None, default_name='separable') as s, \ + tf.compat.v1.name_scope(s.original_name_scope): + yield '' + else: + # We use scope_depthwise, scope_pointwise for compatibility with V1 ckpts. + # which provide numbered scopes. + scope += '_' + yield scope + + +@slim.add_arg_scope +def split_separable_conv2d(input_tensor, + num_outputs, + scope=None, + normalizer_fn=None, + stride=1, + rate=1, + endpoints=None, + use_explicit_padding=False): + """Separable mobilenet V1 style convolution. + + Depthwise convolution, with default non-linearity, + followed by 1x1 depthwise convolution. This is similar to + slim.separable_conv2d, but differs in tha it applies batch + normalization and non-linearity to depthwise. This matches + the basic building of Mobilenet Paper + (https://arxiv.org/abs/1704.04861) + + Args: + input_tensor: input + num_outputs: number of outputs + scope: optional name of the scope. Note if provided it will use + scope_depthwise for deptwhise, and scope_pointwise for pointwise. + normalizer_fn: which normalizer function to use for depthwise/pointwise + stride: stride + rate: output rate (also known as dilation rate) + endpoints: optional, if provided, will export additional tensors to it. + use_explicit_padding: Use 'VALID' padding for convolutions, but prepad + inputs so that the output dimensions are the same as if 'SAME' padding + were used. + + Returns: + output tesnor + """ + + with _v1_compatible_scope_naming(scope) as scope: + dw_scope = scope + 'depthwise' + endpoints = endpoints if endpoints is not None else {} + kernel_size = [3, 3] + padding = 'SAME' + if use_explicit_padding: + padding = 'VALID' + input_tensor = _fixed_padding(input_tensor, kernel_size, rate) + net = slim.separable_conv2d( + input_tensor, + None, + kernel_size, + depth_multiplier=1, + stride=stride, + rate=rate, + normalizer_fn=normalizer_fn, + padding=padding, + scope=dw_scope) + + endpoints[dw_scope] = net + + pw_scope = scope + 'pointwise' + net = slim.conv2d( + net, + num_outputs, [1, 1], + stride=1, + normalizer_fn=normalizer_fn, + scope=pw_scope) + endpoints[pw_scope] = net + return net + + +def expand_input_by_factor(n, divisible_by=8): + return lambda num_inputs, **_: _make_divisible(num_inputs * n, divisible_by) + + +def split_conv(input_tensor, + num_outputs, + num_ways, + scope, + divisible_by=8, + **kwargs): + """Creates a split convolution. + + Split convolution splits the input and output into + 'num_blocks' blocks of approximately the same size each, + and only connects $i$-th input to $i$ output. + + Args: + input_tensor: input tensor + num_outputs: number of output filters + num_ways: num blocks to split by. + scope: scope for all the operators. + divisible_by: make sure that every part is divisiable by this. + **kwargs: will be passed directly into conv2d operator + Returns: + tensor + """ + b = input_tensor.get_shape().as_list()[3] + + if num_ways == 1 or min(b // num_ways, + num_outputs // num_ways) < divisible_by: + # Don't do any splitting if we end up with less than 8 filters + # on either side. + return slim.conv2d(input_tensor, num_outputs, [1, 1], scope=scope, **kwargs) + + outs = [] + input_splits = _split_divisible(b, num_ways, divisible_by=divisible_by) + output_splits = _split_divisible( + num_outputs, num_ways, divisible_by=divisible_by) + inputs = tf.split(input_tensor, input_splits, axis=3, name='split_' + scope) + base = scope + for i, (input_tensor, out_size) in enumerate(zip(inputs, output_splits)): + scope = base + '_part_%d' % (i,) + n = slim.conv2d(input_tensor, out_size, [1, 1], scope=scope, **kwargs) + n = tf.identity(n, scope + '_output') + outs.append(n) + return tf.concat(outs, 3, name=scope + '_concat') + + +@slim.add_arg_scope +def expanded_conv(input_tensor, + num_outputs, + expansion_size=expand_input_by_factor(6), + stride=1, + rate=1, + kernel_size=(3, 3), + residual=True, + normalizer_fn=None, + split_projection=1, + split_expansion=1, + split_divisible_by=8, + expansion_transform=None, + depthwise_location='expansion', + depthwise_channel_multiplier=1, + endpoints=None, + use_explicit_padding=False, + padding='SAME', + inner_activation_fn=None, + depthwise_activation_fn=None, + project_activation_fn=tf.identity, + depthwise_fn=slim.separable_conv2d, + expansion_fn=split_conv, + projection_fn=split_conv, + scope=None): + """Depthwise Convolution Block with expansion. + + Builds a composite convolution that has the following structure + expansion (1x1) -> depthwise (kernel_size) -> projection (1x1) + + Args: + input_tensor: input + num_outputs: number of outputs in the final layer. + expansion_size: the size of expansion, could be a constant or a callable. + If latter it will be provided 'num_inputs' as an input. For forward + compatibility it should accept arbitrary keyword arguments. + Default will expand the input by factor of 6. + stride: depthwise stride + rate: depthwise rate + kernel_size: depthwise kernel + residual: whether to include residual connection between input + and output. + normalizer_fn: batchnorm or otherwise + split_projection: how many ways to split projection operator + (that is conv expansion->bottleneck) + split_expansion: how many ways to split expansion op + (that is conv bottleneck->expansion) ops will keep depth divisible + by this value. + split_divisible_by: make sure every split group is divisible by this number. + expansion_transform: Optional function that takes expansion + as a single input and returns output. + depthwise_location: where to put depthwise covnvolutions supported + values None, 'input', 'output', 'expansion' + depthwise_channel_multiplier: depthwise channel multiplier: + each input will replicated (with different filters) + that many times. So if input had c channels, + output will have c x depthwise_channel_multpilier. + endpoints: An optional dictionary into which intermediate endpoints are + placed. The keys "expansion_output", "depthwise_output", + "projection_output" and "expansion_transform" are always populated, even + if the corresponding functions are not invoked. + use_explicit_padding: Use 'VALID' padding for convolutions, but prepad + inputs so that the output dimensions are the same as if 'SAME' padding + were used. + padding: Padding type to use if `use_explicit_padding` is not set. + inner_activation_fn: activation function to use in all inner convolutions. + If none, will rely on slim default scopes. + depthwise_activation_fn: activation function to use for deptwhise only. + If not provided will rely on slim default scopes. If both + inner_activation_fn and depthwise_activation_fn are provided, + depthwise_activation_fn takes precedence over inner_activation_fn. + project_activation_fn: activation function for the project layer. + (note this layer is not affected by inner_activation_fn) + depthwise_fn: Depthwise convolution function. + expansion_fn: Expansion convolution function. If use custom function then + "split_expansion" and "split_divisible_by" will be ignored. + projection_fn: Projection convolution function. If use custom function then + "split_projection" and "split_divisible_by" will be ignored. + + scope: optional scope. + + Returns: + Tensor of depth num_outputs + + Raises: + TypeError: on inval + """ + conv_defaults = {} + dw_defaults = {} + if inner_activation_fn is not None: + conv_defaults['activation_fn'] = inner_activation_fn + dw_defaults['activation_fn'] = inner_activation_fn + if depthwise_activation_fn is not None: + dw_defaults['activation_fn'] = depthwise_activation_fn + # pylint: disable=g-backslash-continuation + with tf.compat.v1.variable_scope(scope, default_name='expanded_conv') as s, \ + tf.compat.v1.name_scope(s.original_name_scope), \ + slim.arg_scope((slim.conv2d,), **conv_defaults), \ + slim.arg_scope((slim.separable_conv2d,), **dw_defaults): + prev_depth = input_tensor.get_shape().as_list()[3] + if depthwise_location not in [None, 'input', 'output', 'expansion']: + raise TypeError('%r is unknown value for depthwise_location' % + depthwise_location) + if use_explicit_padding: + if padding != 'SAME': + raise TypeError('`use_explicit_padding` should only be used with ' + '"SAME" padding.') + padding = 'VALID' + depthwise_func = functools.partial( + depthwise_fn, + num_outputs=None, + kernel_size=kernel_size, + depth_multiplier=depthwise_channel_multiplier, + stride=stride, + rate=rate, + normalizer_fn=normalizer_fn, + padding=padding, + scope='depthwise') + # b1 -> b2 * r -> b2 + # i -> (o * r) (bottleneck) -> o + input_tensor = tf.identity(input_tensor, 'input') + net = input_tensor + + if depthwise_location == 'input': + if use_explicit_padding: + net = _fixed_padding(net, kernel_size, rate) + net = depthwise_func(net, activation_fn=None) + net = tf.identity(net, name='depthwise_output') + if endpoints is not None: + endpoints['depthwise_output'] = net + + if callable(expansion_size): + inner_size = expansion_size(num_inputs=prev_depth) + else: + inner_size = expansion_size + + if inner_size > net.shape[3]: + if expansion_fn == split_conv: + expansion_fn = functools.partial( + expansion_fn, + num_ways=split_expansion, + divisible_by=split_divisible_by, + stride=1) + net = expansion_fn( + net, + inner_size, + scope='expand', + normalizer_fn=normalizer_fn) + net = tf.identity(net, 'expansion_output') + if endpoints is not None: + endpoints['expansion_output'] = net + + if depthwise_location == 'expansion': + if use_explicit_padding: + net = _fixed_padding(net, kernel_size, rate) + net = depthwise_func(net) + net = tf.identity(net, name='depthwise_output') + if endpoints is not None: + endpoints['depthwise_output'] = net + + if expansion_transform: + net = expansion_transform(expansion_tensor=net, input_tensor=input_tensor) + # Note in contrast with expansion, we always have + # projection to produce the desired output size. + if projection_fn == split_conv: + projection_fn = functools.partial( + projection_fn, + num_ways=split_projection, + divisible_by=split_divisible_by, + stride=1) + net = projection_fn( + net, + num_outputs, + scope='project', + normalizer_fn=normalizer_fn, + activation_fn=project_activation_fn) + if endpoints is not None: + endpoints['projection_output'] = net + if depthwise_location == 'output': + if use_explicit_padding: + net = _fixed_padding(net, kernel_size, rate) + net = depthwise_func(net, activation_fn=None) + net = tf.identity(net, name='depthwise_output') + if endpoints is not None: + endpoints['depthwise_output'] = net + + if callable(residual): # custom residual + net = residual(input_tensor=input_tensor, output_tensor=net) + elif (residual and + # stride check enforces that we don't add residuals when spatial + # dimensions are None + stride == 1 and + # Depth matches + net.get_shape().as_list()[3] == + input_tensor.get_shape().as_list()[3]): + net += input_tensor + return tf.identity(net, name='output') + + +@slim.add_arg_scope +def squeeze_excite(input_tensor, + divisible_by=8, + squeeze_factor=3, + inner_activation_fn=tf.nn.relu, + gating_fn=tf.sigmoid, + squeeze_input_tensor=None, + pool=None): + """Squeeze excite block for Mobilenet V3. + + If the squeeze_input_tensor - or the input_tensor if squeeze_input_tensor is + None - contains variable dimensions (Nonetype in tensor shape), perform + average pooling (as the first step in the squeeze operation) by calling + reduce_mean across the H/W of the input tensor. + + Args: + input_tensor: input tensor to apply SE block to. + divisible_by: ensures all inner dimensions are divisible by this number. + squeeze_factor: the factor of squeezing in the inner fully connected layer + inner_activation_fn: non-linearity to be used in inner layer. + gating_fn: non-linearity to be used for final gating function + squeeze_input_tensor: custom tensor to use for computing gating activation. + If provided the result will be input_tensor * SE(squeeze_input_tensor) + instead of input_tensor * SE(input_tensor). + pool: if number is provided will average pool with that kernel size + to compute inner tensor, followed by bilinear upsampling. + + Returns: + Gated input_tensor. (e.g. X * SE(X)) + """ + with tf.compat.v1.variable_scope('squeeze_excite'): + if squeeze_input_tensor is None: + squeeze_input_tensor = input_tensor + input_size = input_tensor.shape.as_list()[1:3] + pool_height, pool_width = squeeze_input_tensor.shape.as_list()[1:3] + stride = 1 + if pool is not None and pool_height >= pool: + pool_height, pool_width, stride = pool, pool, pool + input_channels = squeeze_input_tensor.shape.as_list()[3] + output_channels = input_tensor.shape.as_list()[3] + squeeze_channels = _make_divisible( + input_channels / squeeze_factor, divisor=divisible_by) + + if pool is None: + pooled = tf.reduce_mean(squeeze_input_tensor, axis=[1, 2], keepdims=True) + else: + pooled = tf.nn.avg_pool( + squeeze_input_tensor, (1, pool_height, pool_width, 1), + strides=(1, stride, stride, 1), + padding='VALID') + squeeze = slim.conv2d( + pooled, + kernel_size=(1, 1), + num_outputs=squeeze_channels, + normalizer_fn=None, + activation_fn=inner_activation_fn) + excite_outputs = output_channels + excite = slim.conv2d(squeeze, num_outputs=excite_outputs, + kernel_size=[1, 1], + normalizer_fn=None, + activation_fn=gating_fn) + if pool is not None: + # Note: As of 03/20/2019 only BILINEAR (the default) with + # align_corners=True has gradients implemented in TPU. + excite = tf.image.resize_images( + excite, input_size, + align_corners=True) + result = input_tensor * excite + return result diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/g3doc/edgetpu_latency.png b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/g3doc/edgetpu_latency.png new file mode 100644 index 0000000000000000000000000000000000000000..05ebc50e4bd53aa62c5271c7efd31dad6a741860 GIT binary patch literal 48273 zcmd?RWmuN$)&=?kB1nTG4H7D$geV}bl!PFNAV?$KCEb$Jil}r6h_rw-NJ^u0BM2zn z^#W%;uC@36zV+?%>-;=imzQhl>l61q?>Wa9bIdp7fual{9yJ~cg(8%_C#ix$Va}mY z7}L0C;ZIg3Tvp+?GY)rURdM0R1NUh#{Qo)Idm0WX6tO_QMHkiZ89ooO|N|6DBi|Hqdw?86SeNJ~q=#Z|x*Y4_}zw4R>c>eA9sf#LDtPH<@G z7hUh;fOoen(p(>eE(3|)lKYm1^W?Lg$iws)}J&&Gk|N4G)Z-1=ZX10reN>WYj zBByRG!TIy&l{1kYj`p)9aAo}R5p>N zvfdpI*6})T2G672fW$=Y zRmx4ctKrssm+7~Rkow<9h~ASW@OmBVcjoNbyXk%PU0wHcsc=y4?(XRIM&dVuPVDdo z_T@v_%2??#k%4fH+v@Uv-8B$>JBEdkl@(W7TAEVC&9U~Ff>WzLL-u8Fv61}qprCZK z@9*!)$$c5p^$x!*|9ZOfE$?uplSS%+lA>Y@EMafT9jrHR-^TvTBtg!@*4)?gEG#T3 zCOt`opFUmCj?gibIDsRAJ6`KaGg0Lnl$Vz`So`tgwd$kQD()LMzLZ&w+0&RmdGh3% z)@PECkdWajXD-D=L2MbBR?HsnI6D=m_J7nkpP!$9uxny$ zj2RV_c!7nD?OY5?s{POQfLa+vW*ZxuNO*jOMBKbiXe9>`^8XU7veMVAcL3OTP%$J^BU%4M=|uL>Y(rEkk6NH{W~ zq7|Py9~1Td{rkt&E!Vx=$rm=H~;?5>iihaMd*(Y%SPrFDgHN{J6fM0mHC0;->_@+iFWV zO`1HnX{Ro&G1tW{ilC8~WQB~U>Q&EHqN1W6d^Ai8Jm}i^k<< zgOeZJ?kbpkb59>sS655M>Gl3+{jOftlTbX{JT~^k@!Z+)e|~)bRr%xp{k;Dt@qSG? zbLLDiXG02)x80kXKc7~Ts}D>dkh8S>_|JD>myKcRlcFT!WJXR#ppS^s#s<;B>n$g;{Wo~j zWbu<;`4HZwdc`Oa_+OR|HWrHAe@XC{Q1yd@y8#Pfp`mwc+;$vQ7?41IWEpDS){`g{ z1nG>4krCt9uV015#dqWRtRS483p)L-?!Qe(OFO-?qCH&;>5Y+-lSssUw{t>mxM-+A zw>4k8sy9svpKsP}d3mH{alE2r@dm-40R30!xlKixJbkKnw7atZJGa30XxHi{4o#h<8aBy%01O(>BlHR^O=k4uHNkd~e*A`td zhVQQF-j^ozAV-yw*I3~zXHOoO*+3Mogx3*4N zdjl!xd0?QykEW0}ye34_k@VWX_XzRvCFJDr4)@lGIXH5eB0Uec*`BS8Vn2NNkbs8o zWB=NHBa_*d@PF>gqNhZy(~?^C;qP22`tKh#csr9sXn2h~>*0+YH*0-;eUWDrdbW%o ziQK{5oE^^h8F_j6#A>nFaKGBr^vy=(Xc{1qFT&akp$i+-*+T5huvp-6BiHf%h#7zQ~OMNa~@s|qZ7Iac}VcK zx1Qdmo79Bko`eL25~M0uG_rx3|Ic+`O=A?Zp9>ybY78PaTT?ZD`Uo&s;@Rn)sB{)(GxiWcb z3rC08_!sBAc%zv4_%524Qh)vWRdY;dxx5qC8UcT-bS-;!>Hc)t5K1+$BN>Eu5IUmY&!8yY@j%xC7x)RLK8j`g5(W&0&) zX=%ycc=++>%1S0-VOp!Pvg=MYNP#~LhCIghgUA)O&24u%q(=r{YS!ZqYNlGw_f5{U?S zc<};X6umC-{P}ae=1}te?e9uXhxCk$jBRag_0P|p*E29snJa>Y-T#%9tY>C+j)H>edp=Zrztj5wZUOw*LK{S znwl;_I$0SjM~b74i$}M@HEV0@OfMUDv5wc*&ujbLF+{4*-$_rLhuBFiM+KW%CZefF z>@=*(dCg#~%nIcrAIGV`I$i-->)c?0F4yO86th@F2A8i~G4JBL^Yg`jGK1al)`euI z7EMVRnOO)<)BbGI!`&5pKr-W=L_|agvOymoW2@bhYp8qmSLOTHy-5jObJg++ttSM- zsBa;#fcG80O@66vri%XN+*iL3%m}JD8}|r;opV|NhTv)q4~; z{`Kp*czzpcdwcun*2LQmXJ1}bYI%MANqf!>l{7fR|13}`d&Ev&5WMRf8x5i49KE%k z?nvs&ep1fa+1Y7(dg6%&EDy;4uCeiz_e#k`rJC3O0{m$P-;V;T0S50ME*03#wh%5) zR8g38Cp16+tOJ&WXi0;^j^H1_ynA1MuKvLj+u@V^bGMh5m$@D06#yYVbY#oWgP{9^ zIRhL1SG26PMxFl^6Bifv&+=*6_BF-D-Tj|0^9LZn&i`{=0E{5({2vZ@VIazRZIaH* z%j>iATGFD*y;Mw0%zr|d2dWaH#+{UKTv8ksy59)cF@0%ljE;@HTzO3;3Qla_Wy+g> z0O+4Ul0Ekq|6$EpTr>qRZvfPYn}?@ieWsDeZQJydY2RH(Gp-9XH2CW3>X6p9?wp6f z_)v&MUfOEZ!eVbK798ym_}86pc(8OmcgPR7fBB!H7sQ5Z+{+g?k@_7RdY^~GX+AbF zF&xq=GhqK9Vg_l5ziGgUNZ4=uXogSK1D=(NlSSTn`!1C6*&jbhE?ju7UG7&Q`<^aC{LeLojp~W#Ck8;#Q+^C*yQ$iB3qFhds0zZHd7mCTf3g_j zoZ1B_&&0{8C@%HqFEi7#1IPROFF;@&lwE~G*LF`i?ylc|L1-!2oP%&?IMhDY1GRK0 z7jfIhg$4C1DJiLMzsGi0LgN3tD!|PU&a}_g6O{q_je(3JB6I~cyVR&@ShQ@lJZe8b zKZIH}kk?YiMPG(ZmZ(q4+P`pXB{eVaIuV0t*z~lXR;>q>v9YnOlT&kBG_!Jry;0v! z1a*0=Pm00&C*i_%bq_! z^HH~s9!WtbCr37_1v+%QySqr-o@^zbzWPjrTBs+^fb!GW({)>k}t8Ho!#O{N!H+QviAuUB$7b6Og|_h)O?n~{r#$1753QC z(b1M|J8!bGt`2_GV6G(a|3fSOgATC>}qRk+g2{Z%X|Lh#xrws*umARvJ8@@4Gf zqrITm*hj0oJ4-{%pen!}58m0K5{R{X2&ryJ{+~d-H3!8!Tc<`4Ro~u@x3@YW@%{b% zzHDXY#Dab*N=n>Vxd0&2=2lbXHVl9Ufcna4RlAU8DJ2I=r~CT*VFMcl&zN<{dLB7H_iMY0t_JQ2v(i_#sxt!<-R78k(GsoGVVzV z-d87!zx$-Rw@jmK5bu0y4K0cpqpwrYR1 zYHht+Xl*)(!K+u}dd3qF6QltA{ZGcOsPS}dcGe&Mg}K!jR5~JB0UtO@7*X+S(&5w% zLqim0DuNZjfR?P`~efHhw0aRI=vdAj~*3GceNxYWA! zn>*DO=oac>K(+c*2DJTdo*MJ*+Zq-}pi0avE#;P8=iwm*bPPMAId*(}T;%vmCC>_| z8=zwW_uY?!yS=>+T-@B|b0JTgxw+GgjEww+hNJhcYRJ__RKK#Pb1hNdbehY|SQ-X} zF5okfxT`lkyxg^|B}4%1is$D6F=u=U6fcjQrMAvaL@N96DECu-x@M{QXZxR_lP9j9 zD=HeAo27CK-io+CtDVgziSeu6XrcM`vm>r!TV&F8f$l53^5IMn!{&BRvQ&)}-vPwU;%t#(oVpj%h#xbZ_OaqvFmcY|w>3&o4e%j4+9yb`Lq z8XG0dM?dBCe|W_4wYyu<-pMHzDz&alerKmb4%GQ?pY3KdAT2(0Tc7s*WY|iS190UV zFr<#n8;%hO%AO~>jY6s}hkCxGrl#iG1Tjb!K}2+SyWZWxM@L8HEO;LDvQ+BBoviDhH`mljqb>w&lB(1?JAl!vfmJ9%{a;fqNmy>C z$cfK(%&N!+{VTgi85TXKxY6x#`CD8|&9_n0dsE&)k&(oZrNft( zErH0-f;ij)dzb&xf|U!sjA$ZkR=c1Gy0}k^SDz=xUIPM)x`&F;uJBXV*LT@oBn>2> z+WfM0FXlO@EvX<}2Tao8pg?{duW-Qm3qW^H(4Rk_{#}YhwvP1!qYWJK=p(4Qnu@U` z(W0y!$b*1sIhAa~zg_6-UOO8F3c z?^Ur@0#;Pe*PUPAacKA~+a_LrUpybJAB;yFUfla@bKga%7OTl zvbVpXprEi0LedZ5rczL(lr>PEcV$Lq`-cAkGM7t#th52e2*f(ScEDvL^zO0*sSmu@i)FK0d(_ z_R^YWWRJyzP*)yKe?#We`u@FUmq1^~0}lxh(?aNko=wN9>yT>WvDIHq_F$?!Py5G>?7yGGUtC#eHUJL@tUi=!jue_OyZgvp~#?WIZdL!xy z;H& zp?F?*G8*W#gtn)#*XHV0E-{9N(g%+h(YJ5ko&jyCt-G6Zu)0x_1R1A7#0PNi2avdL)S_;jf%_lyULl>R zai>r(z|_#NB4OZ~0Zs-mw6dFGp%9SKx`1qq-qWW|Jn525MkZKwC+t%rwa*CmmrK$j z*9Hk6fryOH8cJqx_(jhqq?b;bYXXDN-++fe%Hd7ETOit7l+90GqwS)mSk& ztO)Vao~tvGC&|}#z(Q$N@lii1J2Nxg;4i3~kdCZ@a{Ac_2C@9o)LAnt0W-diOPqWN#w$mQt$j@1-St%h5Ah6_}P2cnWV& zH?6gnjnS@_Ejia%e-j-*Wx@B*tOV@$xKPjcM~kD%!K@#rLF~=b%kpZ>H@CSvyO!py z+M@%BeoPh+c=jC5_at{1JC=#MiWhHxs0!B-P zREcEt2zn9CnIwW;EEe-yRS5rQ>KSJ^1AntCi+9pYC z_3wUHz7yE9>Cz#*V|!yIxfH0oYI&Ob$9q$KkXQ3o>W+75%=@z$v)P&0Ic}Nv*Pos& z1l_VI)9-lgOiHnUCHo=ADU4;HK2d0)xB5U>A$}LnxNEcHINIKfVgWU$p_!DN6A&zS zCuVMjIXZfi`<0;6oc`4Skj*FsFJx=(Xq>mUIsGP~7`z<~Jmrr6S$)tFvNcMsAizgl z=j~b}u~?J$$reMy0I9MD(8l%6cYboQfr)2Cm`41w@27Y@k56=V;(G?PL+5gK6 zFnTUKDme62G=`*1N0;+c7TX`sci-61&>W<8rco~^3U$;fGWq^Z^Sw_Fu0oZ= z)ql@Y;&EBm1MZ`1>y&ZkKN6AK2zqZBW4_aj0RM$K_Nsc$hctN1%Y zuWw)TI2AiZ?X6-sEnW0dV54;0&7cf@ccs3wlK{|VtCHC983qQ1kJ?oiTO;WF3knJb z+dU)$Ha0iOV4vOJqMg^yN{69*9zQiM!$zs*d(h`;hRghHU%)_p{c1%@LzCMm$3gG4 z+x(&TKx}VWO0>IswBY)6-v)o*SPtKZSw8*!H!NCw@sdOyQHEAZB&8d5yfz&v&Ro24 z&$qg!29Xrx-ks)a*Dv;My3s%S&_-BMx}zo_C~dehsx&>b!KRq_JJ|cg%>VpVgBfVw zNyl<9K#ICq4JH^HzX1egQyMsNZ<@PHeQwC#x|vpu2f22SLwK*48~&)C)^%aKo>V zeb3tK_O#vdJ{=22q6_%0fNKo#Gl~GHC>Ih>3+gb%j#>DpqoeC5sz#3@8MrDL`T5S4 z*>H_fzerhJ{;CJ%G%_r#37Yd;R%<|&_^d`*Qka<7@r9f*4O@gQS_aJPM=Q@sM>zJm z+43&<_9Tf2{qQ~&VtM~KEp)Y{$ETo{-6c7P%|wXt8-~=aIlx2}#18XD=H{U|#Pqn? z!NHE#S#kH*ypwOuzP$4F#S#^j=Yz!|$Hm?V-j5_TVHqs0f(@7_R$V`r(H|KLbhLFc zu9_S+fChh}$;5t9N~GTmI=_X5g&t#jYg3W$@5ke12fw;<>5@3C$BNVe&KZYBdo#{)+ohosS8eLPB8z^wL~|20 z>8_+h_|R9rcdG;v2DmTCv?XN$lIsH#2OfsUuAUzTs@)e5oZ-e7v^9&G{?qC0eZHtW z<;Agwd%J7DKWDSQ*|H_Xae8OyXMk&xiRHDmcmYxu>*6y^+Il??J)mX~p`bwzoB>NR z?vAA3W*7gaMEvdYvy+|_`#+$|C&}e_DoR3*{rU3+&xDea@hJ5iOve@Wy469##c_>E zX~{2a`HWZ7LV>SfIPaNs?T1lqK6s=!Dz57Dr2Nyu>B)}zfp=0o|4;qP0dIM*-f-Wc zw>RkY-VK3tbkfsrdJ|EZi=5YVSpL?)ydCd|#V|9~eDbWSPP(VaE9!8i(3&pZj_JCr z>}K&6lq2sGjdh-@ifx8X_}{BtNBX5D1=F(`@Cu%k6Il#?yaNz1`qis5{QUd^P`ZD8 z&vIEvbe4>h!Ato|-L&7_+Z(cOnetBVa?yay@-eQv&dq7s`?O;!jD`w>P7qaMTr}f_ z;k7G!?t)X0FmQk1UD6<2%`7Ygfu`Ef-A#m?I^eugaM|hARZaj^^Q)^7Ty?KZ9(?7o zxL9aR7{5{6(o_5RVAFdYCE!W@*;y$`C!72I`+6E1lu38nY1iA+u|84|Vk}?eU)mq} zqjq^Mbkji>6otBW?ON39*Jn`(4jasSa<%xTN-({M8`AwfK5-{jNIYCF&C06Ay##k@ zg}aEn7qLIobeORi^|LtKJZdj;_=?2dr(z2p71ba1BoOkJvudRpp4T4Z|ViL zOHx`Ik@7z~{vr!INWqKN8LOsx!?!YGVPm5=z|J?T+CV3CrNGu{$GqX_<;$N3_0T;) z)9x(A$(l)Kh+PuYHj;ioW6H|Pg(W4808>Cq zY-nfz=E}ZFX<}mXhlq%tq8&U=4i>QK)j?s^y1vYb`zv65-4=Rnp`48=`-kl>&=@{% zH3W>i?3*`#TDEl1h#d)TFGdT$2Z&hTt*G-ji#tyfV`qsO9jiWvPE7n+QExH8UT#lF zCw7#9K(*wB#YJ4`6R9LNf+=HiauRzL>I=h#KW$pUl2r z0Z13-w1gs#G%29qJoB1!qRCi`#A{dgn+yYntCwJW&s$z`Y3V*?_ym{rV zEvcGX$jBuLjY?H^@|Fm?c$;JvSpm<(o88G`!~mmjS_~3QR66;yWo0mZSxD~S&7K44 zBh7}JEH?M5l23_AGNIU8Q98<-+3T}aPT!MG$G*xY`9%fa3mT51xCG4es$zmqX{l4~ z#;lKUAiAVSw8QhdEE$-s;;TYgqf<3&G?VUJZjHHYFQLeyAQxKw$wLwA1aLpeOWNIC zDgHS=zD-O~NlByZ;8(A;RQHn|M=NKsoYOZ|4=p9ta)yuum ziw7>&GjxICilAU+|JqeqGH46{YJc9BF0(KOim`6q5J)^3r2G4~qFw`h`uE#PwGH{8Rey|`1ey_pjde+tdsUS26}_2#$I+;vLxn=c>% z_lG#N2df>Pi@B_$YHMq49UYrMVIV>wWEG5A0ekjW1UWPRm!r$7sLmSv6M(!i($dl# zRd8a%|F}m|zZ`5oT$w-mu2dSoRgK)8=_rg~dc4AOck=eiW=n4>V}?fO*^d%hJGf3V)FU(XVaZq zKp486X~`A9qRLVk{KI^*`Zf{0a4--H3#+^5a==RT!2+rb$g_O=mRbL!xksnqG2@Vssxw#dLi6A=m?gl(DDyRRz!b&7F@19ExA#Qj ztAly>yGO-aWXcstop%#^dbt3!4*Vffr`G=Ak+_}pb8c@t`(o%CG=NrE zx@$F2Ne;~!aZvU)Jtt0&r%qELNI~f8!=FkADCcp%E%#YL~d5BrbgO}UBaImsTI zvN=7;yn8PCjgP<~IQ`>?-;jeB% zFR;j3H^Fv|l=X;y*Ey5T+Nqn6upEEO;QSkH3kVuAuj9 z?5RauT;fpXK(p=z_mJgS86gNQ(4zx6Sq3r}EghY_)9RZyl;EsI=&h-88`fRK^ifqM zl$M^p0j;%}27esbdvOqxcs%xPL4tcf%=;yZ5=8t9+I|MIF~Z^clZ+vd*6Hb+#c;ML zvI>{e)6x)gY-gg7y2X({4soH!fjykvv(WG0+I#Q3s@GFyWqae6)QWMJkKbx>YZEF_ zklc%iA}-F|M&v(_eH-QY>l-%oK>7|`$4V`Npn1*|nFEVci^FnHqqD0^DlX+-3=8o< z*O7m=(~{5L6xWC=B2*bMP*daKKZ)6kzz1(yFG1w=puY=`eV&=>U zoI255eZV5HmJ9d-_m7YUfDbwE}i6O+jv%Q+q9y>dh~+J-o)J= zC~k-eR#dw(rM=Wb{4OR~trU`pst?%#;HC***x~;H8l9g;iLpMwSk00Y$E86+(7Vq+ zreebmq9spe!Ejh;|4uc(zJ+Dn>2lz3!`n`@`-o+=?%CQTA(+kbVhQ$uk3dXgt*)-R z6xuM*Z?C+#%Atmeq@QW{sNq6QOMu>q^H@LFs;}dhPNJZq3PJn@!~HoviQ0X+X4C*$ z{uoF@sf+CoHjSY3EIPFdiEANVI63S9W5B^^LJ2yh5`ByBUpY3Y54j_ zy?(Zn_mh93y}{N225HH-ls%Sn>$|T3g%=s&ua9q^as?d3C4c&Kvs%RV5@_Il&TfLL zC>B&}zE;DSOoX24LN{b4tFKCTql#(%&^$@oFpe~cO@SgTlQBhxQ zIweKWOGd`*Pe%BYE~^@JSFc{Z3D#*y9GYXq;5dM8{Yj02q^bfCIZ^Ku4``L)L)MdU za}$KldVnt$P8!Hk0h~h$lFiJro7C4_<2ShKp7Q3aD$mR;9-vkIPk%>6`!2P#xNvYR zHD5LF$9<54r)792y-z&7Z*B|C+0meZig$=OB{7-IX+fnC}-!1TH-A(h8UIx=*YM}S#&Wpym?fP zQNSj9+v#Z1?ZF1g{t>|*;ZvVzk zzP>Y5-xOlB<<(h-cC>eD=D~k4K9rpCe*WYnI{;g-sxp+dLvN!5oa=wM91)6>lM`vl zfIg63s(mBsXIze>!0`Ytb$GRO$NT%67R=1xG56V+?N}9g>wWN+&xXFtaXFaz;>9S` z@KX~BOxvcv1nhp-Ey+032me|QbAk%6Ym}GsPrA$!y12T|KASGV>$pu`j~zr49&ga=%w&}GA*A(oUaNE)kpD+1knp2Q8e9nafxvo=2@VJa1hGnA#n>0E}WjrdDmQK*I9@-r1P=Xu66VH&pwCR5N;;dXc z#(M?V!7%j?XFN_l4{pr{Cea$FvO23`_+lJwBy=-Gi&rW_b&x!oKO2K3qvmQ>BzxjjBt zXPCPQa+_aR?-7&oI#&Yn>>oU#N|qdWO}(h72;4eSE-nJ2WmX)Ov^PjbMqt!Nom&Oj4Jed`2E z)d!@f*AE{_wnrxtW)IYtPqi0>hx?{n(i`SZHR|hF--}zw@z|8~($0O2)*kdeVJ_nY z8I2q~%3FllWK8hct5fck<>#?rl&VbKBRl{Di=L5D?2&ugw|k0;fti^gQnpK}UF6pH z6IP@}DJaaHtmKt94LrSfu=Gj}d+zw$^!Be!ti)IjilYmE&tM>YCrA(hMDFV7nZ_W~ z@y|RkCG>tL7mz&oO;V@jWY7JX5_rH_Tb+OF?h zSLP++cJBi~@GmxlbgQ9edW?bX8j*uB$fLjgasH)t8X33aqt9S}Ex3Q*SGx-3xv$$q zvdP5@;L!OoSB2a8e{bNnSDTzpz;(;$ckl4PMvYuTbvvut;T=4~ffGnawkgAv_Nv?H z9jbr-n+ttWJzxkxAq2&FRZF#dD}59?w_25s=OA859??TUb-t3hxjcHgE?k3~T`u)1Dp(pdQBtFF%m zPff5GJkL1(J|pbLQb`T_YZCfdBB^A~0ZbpoIlG;?YE1~_$kjA^r74&p|09*Q6B_Pn!WW=?Thy5Y+4?}?IZ7TFj{ z3S})2;UCv}+y>h(BAtaxrFhA zH@#g;*9bXO_7n9asZ4dQqEb@QU%2i`KOQ#0;Rj;<*=>g%MlTTibl*>sb6eYN+#Ujj z-gW5FaoXW>;>C$4u&xAVWo6SYhNlLp<;RQE4qvLm(hYONX+* zy_c0Fj1fpQfc(?pFpkh1HWFHbn-*8ZJeOu0jT3^~>e)V2fhmUvy1}+Eypx#7DQYTe%Nc5_t3gjtoyJSc@cuKqvaVwPZbKqyZs>NGOS!ed5<6SwyzZ6ugM5Mi z1OkvFSJz+Y{}*yI169LZzOxg=&I+j^Urp7 z8!Wj|$#tG)%EojrkZ}?Ex9br9J+lCN0s=tX*gtG{wa4J&L~Ib?Bi7NAP|CoEKC%B< z44H_wg4{_47uvA@fB_xdOZWCNMZoiZb%Qn)$Atq)1RGFM_iBw5);L^xk(_4 zLRYe@Tq%YHid!nwA{Y{>=c+@emF!(^-&l+u4`UEvugg~>6mTK2b-eI|f}D{@S%+_@ z>D+@Z;*1v`Ras@Mo0 z$W;-5C6C!**}8xQVbYuhdJ)XCL>^G03OKpuv$u`4*beGm!2+pIbegr?SMaHFZXSFk z6VR=L@pnQ4S_efKvAcJIsTEq4e_St1D(1^Rr0dALup-v>{cApq)x1p(>X_)!GB3g#K#Y+A5{`HxhDSyiLH~mx zP4*pkaGIo~Ttd2+z{wAGUESQ$fKly3gvuT3sGcBwoyzA-A|9;#v(QjnN-(Z|c(uss zS7_~muE8l92Hy58Ipsi_W*9EQVx1m2AI+|{i}z=?1(Z|hi6tb2UAg}bg#zzbAOKX9 z56n+0CJE=f+(RtP@V+z4%LL$W$@A3pI=F0LU{DB40YYnSn;5zV|QoU@2zUnih59jmw2&1=RAl~+%SKHIfV@3 zzU* zVW0WFy~{;$P0x&Nippk4(Nfz;YHD+c3VL?rd`Fo3`}@-XUo^+ctS3UTaEMaI>N>{W zXJ+byPu8?A4I7q_1eh?;cSdjzuhs17q%N#}&mso0o?8k?(*hY(gQh3`v7kh|?i4Z~ z3E8U$yyX6giHY6aaiBllX?&Dz(3?d*wmOA#xPxWyWp8S9nSZvR=1N(9afZ?X4Yfpj zEin6ovgW3y6!*PV7&X3#s)t!hi@K5$Ub6G&NnxFyN?5~$4#MQEz*lt#2s5{7Z_3@e z?x@Wn{=X!o3INE_NAkQ+K^8p=M9g%mRs<$km46hd=cpdMzsGXS3tg1H%P1fLkT9}Y z51dv;?Rigr=R)PMIE?Ww5^=?Jal<@+h1HA9rP-)J0@Myf^qcQad@^STnp0=_w3Z!A z_u(iiDuPGmS!ye#{Ye_nUvGWq&K=Eyl42uk`mg`;0u&>5*EjFpA$cUzyzOs+DDF@b z8Ypomnm2^m*z9?ewgm33jMhgpqk@Khz0Fnkjb;YD_~+Y)-N|vMh4<{hk^i?bYJLnH zF5zId{(%h5lv=2#&NkvZfNxN%>KP#jpSAhzmzl6KvIB*gOx%i)x!~zvzd}+af}r7> z{WRt!1_i|>K}|31tcdZjax;L}Zrw>Dz}+@4d5b662n%YSD286REF?s8w6`Wx=XI=S zZJl4b1{$vIvu9^7oxXAaNAMeASNo9eIuL&#Go^w#OEC9sFhOM6dy#ARU^0svv||XW zV9tOG(+4?8?GNp5^k(&x7(+YBCtZdxmx|)8chOU0Oow@tL?HCWS5|y^O)gA@EqU%5 zvA{uKHg(5pOQkHbND~p-9`;$qOziEjavjGaFJtC}W+^z^3i9%P8dn>2 z*mFqiB$A$xI2Ni4&qJ5<%dhXVy~UY_0So1&rCsGKV2Zor z*!`IzpwH<<5Ezbd%IO#EO9GUO-=rL+1T%+5;@|<#2}~4CpW<}oziYVBQ9N8?2^RCS z08v1E@|Y>2qN0K{*;?<5g|Aiq!Mt(qGSE=7h=f z;V>o(D7US>sin5R$WShgEz@OVnmg(g(d~9R!ce z7rcjyVWQr2^b;GBGr@t3SZv__k1OmiLt7%%=_8B{NT+{wjsTM&46UYvqHpw!Q1Yn& zV)}W*lRn&fczF0)|8j;ZjO!uZ>GoyxZTu+o@t#&wYsZ+S)lXw#o3)c-{3up!jRmCu z-cQS!?PMqK&=l0i-&m(aA3pqU(DU9Nk%YnWK1X)n{5833j4e!CB1n#y zUbr5FZQIQ`Sz3_eV_3|Nnb<*`y8ku_+!qpzvKNCRUtu(+FH4C5DjW{t2OG@SegVK2nWshEfr|N= z!eEvpt8zR?$%DDS!td(V2o7WJlB^4Hh09LYc4XN-(mi-u&4|LnUu`Kpminq2f;_1G z37^04$_%KQ_VIS&Cw;TfG2nJwe84xSY8^p$M0Mv54!52-7aT1t?_=!kWh#vDE9ZXw zlIY6@kPU+Y49O*^ui&bM5^v_Q(M*Au!*+K$H$5RgzC1dDd5m-zwP_=a2*-}&2W{rE6#QbdCGi(vM% zP)Ogtef!z#*sT`l7d)$R)U2Isg42>82uEXu)`<+7pB8;}YJ9v;M#9@;uLGSVrWJZW zvG((tI3^0>+P9+O_G>QPh>-JFFIv>j(n5B=3z^gorV1u>9!PD^otvJ)n4?=d`kgC& z4>#T}pd!hhj`xf4v^d&#+=@f-V|`WPq?ErPp^q-!>F8M}>Q@p0l9I@vnFnZ%U_9`P zi=z;9TtuOQBO+P=3n0ChhJk1@kUJ>^c;#fr|Ly2pKUyQJbgm1^RnKi4JtPSW4`OCY z`-Xp16T>MU6Bm;l-79s{Y=2xSEO~j&W98))3|$Wl%Eg4HpRa3(jEqfwY&}e#O%INY zN{UUY@_ev$XWyWnf}595qR_jr9`A)x$btHWKM(*$JGc8}-rO`JL$E#ptpEvNnkJCd z0i_nfh&SRty7r{(WB>BvI6*(ievkLzDCSzM2y zOY#|7m+)qLWU()@UZ7tX9d4U+eSS+dy+&*~W?a}M@E~Gjscq-g(qdsI);wKO+*sI{ zYDq)Ba#dlg6*2W7>&U548r${LP~>!Gvjj|9%gf36vB+jW%!*i4kz@HcqU@pjBL4F# z4^NS#(m0n!F?DKckiNAJzw;CG4|wYNmj3ADk7rBG?@NOrZ{u`%!uj!qp`rSXwKNc) zl}WH3KT!79HpF_eTtgE^=s)qeu-P>2B#qKAoZ9{V&7$Y57o)O^F7HY=Nv4}#BcZ`0 z!_MqF!r{t|my$XY=)2`gpVV68u-E-|wm2cN#-X);TbGuWnuNwZQZez}4Xe@5>XSZm zh#9iARYq*p+eW+?T4gZ6Bn1u!HqB40aa_8_V4b|c`oO2YUK|`cH`uRh%-`Qiwl256 zv-4Ze@TIU+rujhUlnGy)QAN)srSnNDt{O&nneKA;z2#2Z#eL8JHX`8duFhUc4SInk z#(n2r*UM+`-oe)Z04JKAnMr*{dDn1q`~Jw7`z83sm)ZS|cjGln*l1@XqVs8KwJnpi zXO)$)f+M-)YTOpqcjLOu*{Fby$XHek zU*G8;n6F=Jl=N6CT42kbR3B(eKD(W~eEOVxMeV^>;yd%8ZM9FYPZK2Vxju=Pn6!1o zarJ;PnN8!{{(It)w;r1P^is-TB_QNbPnv0u)nE;cRc1)!#G{A{fzbu}bZO7$oN?E4 zu1Xrqs>E>jnVCP!B>8bFi@cp@-1o%=+|)aQi4i=8XJITxBKxu3(+{5PYP4U8#R&Ab zO}W@w0@}n?!tfdGv0SsxSG0o<=-cu54I{bq@0p^HB(pdqM~b_D(F>Kk#G`58t1fhO zwK6mUe~S9kR=7T|9XpD7^D6rps|HimPci=Jas2DOmUrl3S|mL^eN-PeJS05a(=#g$ zmx$;J6Vm|)KX1vF>c?Rm_zeZ4rdPmiC1KF{b@Ixon@P8L*Sp><;^onu-PxaHR53>c z>)i8N5~7jRfwXrH5nMS$cu>!DH#hy8P*!#W5u+z0DDFX^|6ro%87|Y0F5vq zC!cd#cligj0Kp`#dP_Q$5Qey5Xf$J~LR55Cw=z#U96Z*kUGUDC{N1N7ul8n>3cEz1 zhrci#x^ZY5#CRe)L-E{R%z}`k>MN_PVHXe5>9j zs!edHe9X(k0OZL6xa0z-*5kT~Z{K1-xG(*3p85uimX}Jj4NMB0V&i{V9{GBjuXX#t zPm!(VyHwtgSAz$(#y8`0$)7#HFf^XD#WoAHxktWZ*}IE>Q#mS*L!Agb`mSblVLwAAw&e}f%%{E^Uet?Swcd)weOpKc176S4_)RNg9`8qB<2@~$h*#|&e@ z;6Z%#UbYN!kk3C<1H(dB${l}1+J)H`kG%f;ec2?gYE zOdQ0qnb}W8W4ZDK=t*dUOIsx%mL}Fx9YoxY3NO zlL78~Ddvw#nMrN*rb;~tb{N`A+bmEYh3be3sCC$7ERCQ>U?U{zyUR1{E4$$yJy zpn!{0!|R5cp-_ z3pLE({o#uOF0-?UKKYr-PGondha?m+UFaD`%^si7CI zzsr9tIscO8UXI0l=UWo0w;s{J=zI8YTH!g&XJ1seUN0*+E>QC58m{S?;>g>2!kcwK zd#Jv~<GU7vx^~TwQ{uF&g?D=6^-}*(xF!`s4Lf06dw~@j@NEY*^_kr1=5^%MHTk(cx zGU#yNR{aLwa~67mEzL$Lj}pfeM)dG0IOkv`-_pmEymLGe?k^zjNxCdnR^%H_lj>yg z{}A>j;8?cZ_poNsAejqMiO4*KD48NMODTmAO@zo)q0IA~nM^5Bh(aY}3L!FP9x`N} z-QT);-skuJzW?zZ-~WByqeJ1oulu^rbDw*!z1G^cebm=ATeJ8rC9}$f1Gpw5TnxwW zxKpt!HprcdGirHxNk2au^c(C3Lb2J{gff4POdF3wsEk_nU8b&w>DLe4tQIYgSzX$QMzke8h?P4bLBP!W`rRbwowDMQ6F;-y;16shoOI3wP+IRdscm6yB*aYPujb2$_ z^g;QgY;MjOz$!C{9fXSI5h!8&cYb}?;dRPn7Yq|%VUlgfP#EI1JuW;vD_(F5$GOK5 zPVI*TD}P^J@jBxrb$_0G-`%HKlt<}<{@s*r*UC@8Z$K8X zj^Ep`UM#pZ0Q_XR5_&*W$TbnD#I=m5840DC#goA2bF9LU{Ao)18jR>qf=zo)d_tgcmulbdy@ny_hc>>- znNhf}GxE&ZMAkFrq($7D#-h+3ymE2SL`4AoNaG_P5)=%^ z8DM7NBx_=c-0Q4wn}(-w>s$+O*QIIR8UMOQ`aW83-GmgKGhqk!%6fJvYHcY*4&Mzu zPwE7LvJBfuzp7hS=>?>AW7_^(c zNme@@De!Earb}{;Wod%x+j!l}$_>Z+NT>Zz`yWi(4_W0lEMBp6OKbqMrgJH3T3Wh5 za4GO=N$;HUgyIBab+4As-1yg+r#V^Uf;DX> z69oZsk&{@UA3dDH(<$p}g%xFov#|*4Hn>T!V#W2~rr>)>z-n)Lo6I_KP!#>hgz4~46 z9@#l{4>-=xcX5~7(*?Ga-2Z!Q+CG+gEpoFh6dLbWP?^_6@ZEm12Fh0gq&dWS1!BjI zn>W4i&e#n=5dDnGy{FI@>Kz+f{q9_qCLd(HI=|29&jyIvru(EE;K_HjLp)GNA~$bu z;t@{66saGBKP;ZGyH|Zo89XDYddx903eXoJ#6TP8Tpq(Vtbdcwo_qE~v<@JG8kQAN zQH;>!RJ6D6gW)Dmp-(zhX8Rj12W4fW>GB6Oe7i2@pLE+gT58B1ZY+)m!NLmvpf{iM<_+}J(q|T=-KzvfTehjFfIrKj z8caEBRF3dOR>0EvNzUw%aZgJ&H4aa9$Ex`)4#3^37D)GIpBt8z)HIN@k#Xu%i{3gL zmp@%jU)`wVo=F`{ecr};kJc`=M5~O7vC%@6TYnC^Q!xdNQ%*xrCEV;v8N`BUMyT`0 z`K=%ScoE`?T1n<^425L*uTHn^Mf)~CAnCZS@~$)#8ksq>Ru>l?BK5^?U{8fB>D>nwiH(B5Pl7I z04YM`BHV_VByT7f5{8woTFJc3W6p4ZGD$jCYlc~=>_y(Hh(zy zgN$ zTWC)&_ck=t7w~nwO|;Wyr@i~MuDqh-u^sRD;@#GG-{&&B8l|kY1lio{*}1CZUlKbS z2_9UZ9x3=;X-FEnd9(e|i<^(fWvAn=kSGhF(ae4qrEe%iP)ptRwW;{n!(uDbLau(`E zU=IX4CT=;NbRfXf^9tgs#S=E+E6Au2#Mhjbu1!yS>}KVV;nksl;O(Oun_Ejx_-dM8 z66cMQw^8-O^*i=yS!eTBm;SR`a%yUjNf{QdJ+8H)y>0^J+m8I_xvAcSLwr$+@9h3` z-UDLF^)7g|v@6fkat9?tMVF=Hh-MC@%-kZ^>58z9=PJ|Le~$%KntrS-*|f&Q6q=nG zfs)_12dTxy-B}eEmn>VHx)OY_M7%NP?Pu{pmZx844>meP|DkzsGa}zga5g0?hTWa& zcnk-HHQIn^8mqxLRjYG-DBkruf)|g!vgdx2rRg2;63bY2=np;|RIH4Ps-Gnd!>R`f`3^H1N@eRJ`Y$chDH$^U*GE^CY9DqFT~YlLqFj*tw4 z!rZnV&yFkO7Zyjk#qZO2)9FYuO|jlvFFxHVVjHFIy79EdPPTwEEP1D?Wj%MWiJWV@ z-n*laik;cW=vQCB)y|^+%3{aUCm$HRiTX-Yz^PW{NmckI6*-8JW*}wI-b=+4okbr# zDC^o|VsQj5rOL2h3^GFQZ7p_Ohr>Dy-36>yCnzIDET#D>C}|$}?Ylan+3YiA5JaMP zuXGo(Sx}!iQIclIS;)Ka)9*hEPBYNQ;2NZcSY{5?)K_XF`-S>Q>m-jyv`0UTG}b?l zD9TwY-5*P$N3K{j^E>^I>Xz{MOW#_HjhlCIwe(A44gFpCCsVUL4axzkj8h2)cx_*u zYniEJVR>aiOLH{k@>x<)@(hW@`^3<+YJzQl$vDJ3Yt3_m1u@wc0=te&E?hY9pV*N~F%34lAh<|y1te6_ zWKcF41N`AJEQx{KE^~h()zQ~E4}Iu$;<@A6bGBe!`UcX zUgb_DYyJ=~Mp~P61vD3k>WJ};>(W330ioi7ZP4_6{sHkt*)ELSi<}M1iK8k0gV|1* zvW_N$@p|IdAEfe*zIR;Ug)xJ|I4h1On4Rdbu_ubCg!5<+Mc|PkSB>ZLZ;rNmCwx4$ zC_GnfS8lKNt(Z>fp9q?e@#sC;urZSNzNN3BOlZo|*CJ{rFoZIhWtX`qcnrulZQ~$a zbQ6vxwcN?IF*=uh;%!@DCbM#;we>{a$^UHe*d+5IzLniFs~=D>!3)yIW!DnYht?8j zNfeP{Z|@|dHDlqiq-?!UKZv)X;n%>Hl9K8Gy=*lXo?PD~&f(mj^0_G6WB+17(Wj-a ze%ee1Gqk1eH_3nqK~QC;Zy;-uNl72vLRWf!&Y}J5)6wz}$}NvK3(nPRZ{!pc$%NC}bjoUuO{_X@n~X31IbBId z#xeHvL$tr8w!bnto19v5#-k3Xs>Z3@D?pFCVr-mj$B<3GkMf`*$2rwAc-E_n2gN0y z)3;pwuDl)+dPp}$|Kn#v4(c_Y3*9a!MkTDgtopws7l++=6TA|m{BHof)U>_&ZUxAr zpP}_RhrE(e%AOw$JzuO9FBOT9q!W`lgL?JudB6##afl_w8yJT%&Kp~nMEiU)%?%gv( zkcN-+ANYH`23ZLy5IH4f+2_yhXcILQJKjUm>*nQ^D&5yF5nWXC&erm+)g~g;1@8#y zoH6|QQt;~+;UoNq0;lC`Ucc|OW{R-LvOCYn{{Om?ElVg?pl$ZkaD>7#QtbBSq6!4^7fPwjyOBLhks z31)yeJx_kIG8no_nK}#<(XzQyMV&)MP+WWhkZB5Z4O_>dL;KLxC4I@bsHmuS)WYdt z0C*kf!l)ok9f4wp%!7iq7hcUNXxsO@US0kG6k_!_~xF>OnX8%Jf*Pg7(csVB7 zk!M!zDy;kZpQNi^`_#n10MUvfKprf_kL~SGaVQFb69U`Y_{0^MWe8ydVrTQKu0bBM zp34UGn33s8)p^u67Ej1_v&y#TgOdw9bakzuZO=7Ekr~qDYuBk=v#G0M177!;o?X~) zQB9YjfbBCRpCWFS6>< z@PY~aXOt|ncfV;JEC~8Kpcdyo(aAQTmdn8b39bG8OeS5mp?g0A_Bbj~F)>qosK0Pp zzs)YAMq@jh+&7MY4oU|oCA_?Aii9q}ACm*ZfZe+TglYRwDE4ESynXksG?#(CFf!?b zST~aHIv=Zx7hB)yc~70SS!uW_adSd$=vXW=1Hj^3SM_BFi1_U}By6Q&rW^NJkH!N+ zA{eNC+J-RPwhqF;5Y~U1oEJ3@S^|{pG|@AYrTlje-1s^D{%a?*=v+GvzdzcFl(5dyo5xOWyEthHdJquJSJ-cQUFO4hG;oT2 zz%K9o3k9MV_4O4BgF&r~FO7{M@bEKR^Dln?p@qI=jSiQ)yk@pnKg?BL!+iOY)~-(R z&%dJGmU4eaoF6EtnO!qA%PgMXf>S)C|MCgVnUq+L)GRUsIKQER9C548Q)n2d;TT*G zxw*L^SepCp!d3njXaY1c#?d}ji6y?5{f>Nv0RYFjt!8Gn^YN-CI_tI)%-n4+4oGpt z+*g~+HHrTjYmx1|0mthh0wjxixu=+NVaa~Lr%!SKRr8cm2@6cp}AZz{n(lNEXmJ+f*@jnu5(BzYXUq>Cz?n!Q(C~ z;%@hLb4*`9>PN}AO4HFrttb}Y#&wi9KZW$KW!E z#9Y&KT1$w(DCSt1;YzRHzxj&SqxV3}z_iI=eq6Od^Qmd*t!=wy=SKhh&aMtvn0z8+ zCU|RgKj#TFKs}Dp7pPmEkq?j6e=IvKX4*Xdyz#Z>XyX_6oE^^NK4KjIN#{7iPt=|< z*`Bcf0OI9WWohm>YlV*L?fJJnifo(xjy)-4afrRTUbeW@R=1r0D0NFzmx;u|QQrgE zUL2;K2rn-WA-q%@3bA#`$!(q;*|cFpUHgT6cfWmXA$l8X{9FpxeNG8<32b?;S2rx03xU>&q{}A?m!4E@d~JHli@JKtVx7#9a~A~~nzDmZBZto}CjQ>N zdVK7hgiclQJ#kv(Rj;pS2IEPHp2^5E+CxuYi8E5iQ_0+~pkSeWr+NxGWCBBVr)LHk z_iFP7$E`-Z;$c(SXZoPPW!$HL{ongE7&zJaPLDKs`pGVhw|B0t^#O^7bFdf$iOuAy z=4T&w_~TzLX|UfeXk+S@rd=w?&b9_e9~l?6dMpLS zmr@C(DFhslnH^O;WSy0EE+m3)tntWFl4?-Tu5R&KDiWK)uF@SJx;0#1|61J0Ar_Mb z@DO`I0;qACxK*$FdN&Tx!>;SAM->iQ$CQ#DPjvQ->3arjL-uFxS-V zeq~ADlpbW@yUCYzcSD0d{rO!5T=siuOENwzJna76J}TbSGnVkW_LlV1M@tl$43{SU zd~YeKqWhY_Tb(NgtueaWB1LG;(H`)9;B!D z=jUGQ`E|;^r}yfV9WLL2VI@%Zl*cRK*Cwbr_RmNt=r|Uh6n)N6JTD*qZ_S1L2Fy^x zY#0m);%r}@2e%~x;A93vRJ}K#u@kO$hIXMffS-B)!ctHCu7a*+^_8-&eF`t`Sv1&~ zXdzpSV#|AcC8@EgJz$XWJzo8PGRC`AAVaF4BJgVq2@ZxoyB-^RP$x*oUw>V2qd#`3 zC=@6Ao&Ksl`L}+4T8fn{jEYnuee#H6*}LiP;D>Y#zT(Ik!lBU75xmn{xS?{Le{}%u z%WL4_YKLuH51Wqc?@=#34PmPM;NTzuSN{hbLTS@cUhaV#$MoWO3HlxY(^E9H*(IYy zEa}kv%s$l)-f`<|v)wzHgvoy4m|1lgp`IIUIWk(chf|vzess%ohR5vyN#cbG5B+W1 zwt)h!4=N8dHwXCn`I~q9^d3i78hJp|^~?hc^Yf-~U;xV_0{*B_bCu;;wtxaW03EfC zq*Tz(&|?>Wq#xtt^h73Obo1s;C?d7y+FhP&Z@-Xt|E9pMD>rX`1@gbM4XXE(02cl% zEDQmEbonxz$93k6QRk9vXdahJvZnQ>U5FsPs@E;3&499>&%eP)=%~{g1wb}@2^l16 z-RrHd65D#bZVbbQ2PRfNyqAi|_??$+%655Be7xusd;Qie|EMVam$8C#LvAwH(YW4Z zxARK<#3?Bi`}vx6g(Z4@h5v;H**@N%03{F6VH)RmSAdhfM#9MrAd_x*;O16#F(8AJ z0{sLjr^P_R^U=si7JTxq`v%9n`~t331Toeg7k#pr(!cwl9?Rvc3t^o$&IP|!;v>NN z8v!ExFLD*lUB%|WdmGGj?<9sdU-)9}HF}d37gv@D-Qb%130CLXhKm7`_>i!i+x!4# z%CHCgP+dLpqO#e$xMg0j$(6I^VD|6!nzqnuSn|SmZ_U>XnaZupFDv~Q`a{KyaYjOR zW9MPGU|j!_lj5HW^Epjal^yS;-aD24?lLZChlFc)yY59E^yXG)G_v_A7;zyrX3f9x zGnku~hvn_v1JCQSf5?c62wY2Ak>e1Flv{sqJ!C*6^d}Ilp#}*-SyHk#e-U2i^~EL* z5 z69)&UjJq5c1J?xv#<+ETUk(oTJM^Sd0rz2CUTzq%fdz^+q$)f12}$nLF{5#mov%+* zW?j}9U#MX97Lp*bk01IUNb;oC^DM+Tsy%x^Jz;-pk2l)Xp8srcb?Jt*W4J9nU+z-H zPK%22hGeBZagmYrBdD}jXI>X!>P1yOLvLT-b_$9lq{566x9qM(SB9=R&-dwn6}r@| ze$8fGzU|EadN|(#(LV8F6fy?|jt(Q-~1XHedO}q^zvW&c$_x z)QXq{RuE1J1sfY-=nAx7TnIyLhmRWgQ88j}eQAMJVVl3HcC5zRusdHuovW0U0FeI| zOr`C|d63-3=6OR_gNEXBc~D#71r_<8`xhd)wxzxL(=Qtw zI@D@GqCiqDs4aH z7{2?D2O~D#0SOS0$%Fo8Q96)P4yNt(jMyh=#SXqB?LaMKG>G^ImueS3ZrpMnWc=qG zYQp#ZWAxn@KigZp%&HE1qBnb!)<@Sw{vE9_=g$LL3&)n~7zS#>k^y=AQ{%gtTl_4; z$PPhB`p|Sg-SxgsMg=NtA9%szFt^~;PN#alx^{eWT(vG*WW=yg@Py@${`Uo9_(v|R zjfT6pVQI1{KYgyVXf11xY&wZLbjqRoNdnucsLQ@?{f5YGl>Ab^1sbArRb=~{DevO> z^Hj{tu`dO*bUW;6@7=q%_Wd3)cLK!He$bFG-i7GvAxH0jq!mW&q8uU9559~Ump^c6 zij}O}-;ybUifk%0FKC}BUYyF}yY{KeS?2}T;VmB%6SJf3T&zUc#J<;vB!UW$DB%L?y#?xtO*iB~nhQKO6; zAqvxB;9qb$t&!M}*s>KkxJDos`-|&fkW~9HaTX53Y2;rz!3xI9fzdX92WSr}NB7{G z3oYy`HOIb`mmjHk&Mcd0t+M{w4M%ZBK)mUza2GqKy9ju&TGm9TS-W5XmQ+YSA-(A~ zH^3LVS@v^JzJjBSe0DoId3^x@t#QAFum;`i%U>ao_iZC42Z1ft5P%V75FMypOBsUo zx#b@*o@68_X-thiGF&Up0xn=#V(}rXukJ}8BU(6p%RDH?;tug0C|LYbO%HV+mqfW+w zvOEv_9~ZDew6ix=lY^!`)|N+UW;S$|QKlFlR2A5r1Bfkt`OPmcJ%tGhL+s|Gh9#@q zm9<&7cNG^GkF*zLuTMSNLP;<|figE#F7|f5*6KZ{q(oQ?JIyug0ZsOYV9Nv0AP5te znSMXd-yWo8o_Sy!F^T}vvR-s{F$blmw^tS(EkaH*#JrNU#wnPAn=ZD>p5S{L7rM8@ zx2U?(QQ8#Q0vQoyLN=)dw;WnojUMV>H4~$xzt!_CRs@6Rgv%(VeOVftnnICP^`Zn; z51hnD!tsme*nOWhgIIys`E{W=LMoW~l%e|~y6x(47^lXkE1Bj&?DC7~CS7Lg^OWp{UCbfC;d|BS{F0?b9Y0fsSqZEZ1&aZUMXB5ysRRYrkemPu_CVK6#2+sf6$?kYedeH>k z49oWm4!)hs*cG`qehT7vRD=y}xg$QT0*k=zTBJ?s1FOJwqUM`KSwW%<5Q}gq1Fb{~_iaI5@ziX!rdc zlOMBQyk2zt+ReF>CY`LDlGaS)x4g}Z*5+SxD`P)}q@<*(LXPYvt#p$xH^2xx1s{1M zmcoT+)&~zKs&eby>RGnk4Qe#$K^S+zpPyz2b2}r+aN`I$AmoG(kdIO^G1UN*i8_9l zgRq)fK*~bwX;=&9m^ZLPn;$88j~6%T8W1254u@A>>c4)Bn&j1;9$5e`<+xriA0K11 zsIL2CB+^BjvDP38+M~p`$q0vKq#n?DeI$y&dyry4`u`Nii8UG>#855d!nl`Qf3X2j z?I9Zfg%JxL9$Qez5##IjpSX|7Gy4cG9#CUQwL}3OZcQ+$4qkRfwq#_Kq!HsV-M=A@ zx!T$0R<3YLSEcS$W7qqR}sKkhqhQG zK7T_)!|l+@h%-x9v@pk@fJFtI9n_h)8e%jFuKI^-QHlK&JARH>CcuhvQOY8*{^U4W z9Pttt^aH(N@H2vLc|67nZHga_wt1i{yK;}_PpX~NT%<80)I zNC9I`Y^CU4Oo?J4K)|l|`i4%3Jw7F# zz|TmT)B}8g#>frzKn$#iq&}=nC~DnEZX{lkq;bU#IQ>?kfj4&(dFkx$mH_OY(y2LZ zE>_Oq;9$c1QCeCWQ=NJzCe{`D$b+=5Thw@?Y{Aa#g^gp@g*&Nv^ujJ4 zVa#$=91o-ddlcXUB^lI@*aD%5gLnD@_W3@KH&_*KvN~}4?B(^&oO@k5AfT43 z2m_dM_6ox&)_$+aBb}(g^AP|H|PeIRH`);!23rKIb>I=Z~+Zu4mz~JrOA^-I$OyWw1bTB+Td^ralE?daN z?w-Y;Y~XMqa~gKN9Amu~`wFV`o!;|^*9@0t9iX4Y!DL$h168+MCCHBlg@r@WQ25=a zdiHDzgz97pmGN{~6v`32st^Wh(0CvQGw?(oc!NnC|AF|eh2DxKm@5Ir2nDSmc^-e1 zV_u5}jr^6Y%-^J65-Eib?1&c%B&Rp_4DU)&M#_wS`#PUrr}O<;_@6v z&g0S6p!1rI_Q*vP9jRypzQ72RQ(Tvd-dIE5&2yaP#emE^ho38onwr6ax4Vvl6!+0v zjXvXn(JaO(9R2abE$q=2J?_9z3)ZsuL(i^!)bY-ndtS+7=<+J}%14XCTwEi5{@2h1 z^!D-bK~D=L?{q?5j(64|J|WUdu-u*@8Y@FbN+Yik3XF!E=#4X(L0YT~r?$BOOw$Yn zE0%GNTPHJ=2#NTW0o-_R4A$$NoXqZC?=8WvDu?rA;JiKxKZva|=%09l;P-JI%BEYd zvJs-V@K4K?n3l)K`~^#@I~z}|#w_01lxD@%SVa{!S)s_9net$f$FS;WZD*&8iswdC zwswYQ4Wbn-t_>&1NT15FL4<0i6Rr^dDGoU3TwAs4 zh_jh>@#0oVhz)Ve4=HtuU#12TXsyV#!&s*HMMmcVdmL74MQ<_oY9wxfnUS#?%~%|` z!P%2{(+L>>vure!%J`(`2;(|2v26!1XE#<&5n0;?elZa5hFMSY8Zqj#|hD$KmX{kaz;iO1b z)_38f{ABVO@^MlJx9=8Xd2Li4gnS$rT>V$vMVcVymne04YzJ>`nWArL+RQX-&G3QTU+1H z%J!h2sD%k~VqIxe7m`5Qzzqc?84jZYjDHlt)&`rv48<;C=kS1-fz;a@{Bp|K*%>)S zGNwapr5__te2jP7%fh0DI?FG=*rK>A!T}>qp$L=HpdotLiLxG`lJgOL;F=n51wTR4 z>r#nve+4%BDMc^N=g97t9IPd|hO4|*3{dVHG(B}Rk6+>vTTS@a$h`L_q@)mZ)G`}& z;7bi+J!#z$j-s<$>=C<>5nb!A3fk%nNpc;mX=h7!gQp!C8A%L15Uy;5!97NI`#yjE z`=Vh-H~vN(u*eZ-Szl@yTUgYc~N7Eb6^>;gpxZ_Yh72RGx;;xIrgrG=c z<6Q$+j^LUql~kFa>5iGt`;|%(IEqzhQ?pKV;Tj_5w{PFxLySE^wSrkS&KGM8G>}%o zW_ZSC+|ZL|PtKmRtETK;re5d6TwUhtF#Shf1HN%Vr%0id;o%kZcT;h`?Il>m7>L#o zzX4QBX%qt(J#?C4djA1&@kme4{%423)_&_7lZyY{Qu$|0-xj7oK)-!4dTZ1;wW7H> z45w_V%?i}SKZ|U+5ywq%Z&4ZWqiqkvW?%0--0y}`S69TCN}Pa8n4)vN9t#F6?O5%IYfe#MwxN-zh_DYY{7)Z0K1AOcK%oQ1Rx|Qn zAq#ZsD@5&M4agpR`#aXw-$^eO6cz7c7D;lBO4v~kOp^upbOn2s$RM`wwaLfX<|BViapm|yYn@qsSQWmqBw;uG=s`oBa3U{V<8>4hV5b|CXeA%DeKPH+7D zNW^2uDYYvnGYxH1<^S6N&3-?lU!ya|g*}9dVtMXzY)nkb0s8FbzGRjTmpkqDZW|k{ z8r&;AjIGPuTdi|;va9vwmW(uZE zgfX*8YtHt${U?}e%btYg6d1}wl()ES$*K?#S3T92XTSK0S}~x+VJe2vI92IO)KIAO zzyyU2kmZksrzp=u zI8=?$fQNlRB)s%~-PMNKammv!ELF_?_P+L5+2s_3K%n)wbz-aG0ozj=LYj@q_2>qJ(lxO(gIX42f_e0kpg zl{Q`6`Se4FIELff-`%_Y45i7|K2a4!@>=vofH&*4XzIBoPO-<;smf$mGU>S_6U#dg$i9zVGs3$!6>nvt?IS}rX999mZM29F zipuSPQb4`EWpnU3lF+r$(}7Nk@8PH$glbm^(@o-}9Apt(qa^R$3h&&kLb=o;$QnPz z*QuMVkZ1GaaWJDXg{{1z;tX!lURbA~3RI024$26ZO+Tx(WeG~aF9c^0@CMQ6#6n0$ zQHsSd6!xWfd*Wg5C|R%3!?X96R8+AvG03-?yArYctwuYM^CG}?i^{RNKX%3>Z;K}f z+!uGNc4i0xp8*eHLv?|I3o53-_xqFLDu;hBsqwkVfg|rt$WS zEL}FaqwDXwURGTQcE!(qicF=j#>1H(S1EZ=g|rrZ8N`Q=P}&cCi)L5te@X)yI$D3^ z#?jy_V#}?`TY&3Oj*}B@BFr1Oi;a_PZ6iyG%naOVGVz}hAER1s)IH2qA%4$uOb#_M zOs_dO>duW4gnp#3=?p3+!kNx{S*}G^`4k&TvvdItfhY^mHUD)tp1$e^lz zgVYeUs*WU0qK0*v67FNU58M$5POA_}cc6LC0uUN1 zg2#w6@C&3tTu8Lr63P8QGz)2C>$eCd)&3Lr|4iSejJ*-twfLyFmO;n^^YtLN_*_>P zfED+<{T`f)zzRfM7-3ql#TlT8%M!Xa=KBMayf7}aihzq>h5*9-g-x>ji0}s>FrNY# z&PBlBq8Fq)bhVMLsD)SXx+_UylcKcV!6fX7psk4_9k*j=K786TXricO(`bU9G-+;;PAdl=d{T6ek>wUFrFa}lv6+DQeUu2ksya*&>L&A^&mPJGj z2+To1;!H?nB%1KMFXrFq!IdEsdyH0_`l~Ca4mi{(qg2<2X$So99j-e8D1>j#9SoYs z{Y}@)X2GgXBk&?=eC)GvQc;kxqHhLSF*{O>%F0TNKfLQ)isk|_9t>+)4#wEcnL5T8 zLWh}tj7q$5WuHC~2KPX>0P_>RQ-p6eW~5@WDgV)`dtot+Y%_IO(F>iJBFlQMW%w9((A#42YTPi)QCB$#BBsNMZ{6G zoY3{eWV&rg^|v~sA5n+Hi?GIn%mrZ36yojg0-5`>o~W{{M!p{g?s^_>^hjG05cHQ@ zTe9@s0L~$t8sp%FM|B8biH+6WJRqnCKp?yp$<=rtaLIH{pSZ*HA+*_mYZW(4QL^QR zqnU#|?J4qI!XkKImLjj*1z;^i4*Q&0;U%;zEpwi~#lN{q@BIAClXBZKvwgYHuleNK z;wxRXwZ!BX;Nipob)d`dKI(VV0B9z-Vn`XTf_^{@Kef7dZwT}Rk+)Y6Ay$cLNU>^Z z-IW*y&wyS7(zB;p}+3UUv)Vhd>KARkxgh#V!lB zqx9OWmh%`?j9|l@Sbp6Kyr(JKFdj&>5HU~T(j@|rN8+i_cpba776drr^qgbWwwvfU zV`S9Jo%RsN1(C#n2Jrz={%L9-ooxDPEuEWAPEzpR3^X!GX-6W6gH{hj6K*!Hhp$}A zW@Q%QIvmIEQipv$8}J;+E0}x%Sfl7n1&!(xt^2;g!BoDA4e&2Zhp{4{AmQP57_2)B z?5c-a6bKhAz?JWw1*pyqk^Bb`ma&=uGjJKa<;O@#US#d>PIaTpz$9g7Wrg;Gk- z*+k%Q0L4ZtzmM49U_YWNpp<5CF%op`opCJiYkANVG$VWr+nK zXx=hnv-NJ-2H-|F{FGo5%x&%)S9kp838G zxMB#O5E%b7Y~)g|v&xdsp?W$8wHi`ySj0u5tuV?*rYry_JKK$Wqo=xMQrIm)$QiiN zS34z8**$z*EnLlgPxzTVhlKgQ7sK(IVU>H@+2{CTC_$M75lBwKXAuqQeSXTJN;5QC8a z&+=7d`G*aXyR%P4RY~{QV8)899E0_u2knVfip34Vv-9s#K9Gs7fb0$0_h1;SvZ|`8 z!~B?jW;+(0J>2535hZa~Va0|Z=f{M%Yw;ojbgaj4C$lBPzcgr6z z`8`{6jT{G2Zh2$|@eTtP`l#1U5JG3ieueXc#ijPz~>i*SlfrAY? z2g!1>Bsr<*N9jY?ZKt-#sq+l)m!%Ce4NQGtbei9&T=rMy`Bz(ZdVak1Uf=-JsdDaU zpGPmxYiw{-8GCd%XR}Lw>z2buIX#7=eonHM#C#DK;{WhL&OzGp>xFNy{&`(jwl)6e z>$>Cd+m*NK`1GDb@~E_Oh2QC_sij3V$`lukRU(zX9vT?3^;xc>YDkgd6i#5-6b)H5CK~;KevY{(|GvdU zu4V*cr~qmZ<=+?zeHF1~b>-2cjl?ikqM-#_q68c^bRVUV!lyw&yI3xNy+7}$;0p1UWqa?h1$vZXNi-W8Oy@E)L5GB0JAVuzpFm!>`nH+S5 zS1_8QjoX<~gH%Ji(_^b&@3|TrOm2O^mBp(%q`~mHv;PUw28)8PR{LGBmX&|Q%koj5 zI1vf?_6Zot`T3`vg}mT17Ua?b0@^E_dgsqab~r=jL4mZJ?4+(P9V+NwzkZRSJ%p0l z7&bi^?s_V$l0Y5NBd~O(j(;#bY)&k@uKDMFD5G8fBfGn_HU3N=-l)0k$^b>8vy~n=f6ugjY%~d8eG2n4YfGe963? zb3KK9=gyt)UDlXDwxS*=EG)#pAcI)m4L2}65Sc!io?eTDq=M>=j<(J>`CwbjfW@69-d6- z*t`5>dS<4K@P)W3b1m~0Yn-&RH$Lbpcy`nSq^9nNkG-j_k0UAAj^BVTzGMu0a}h9p z&+ss39?vs`970(G2f!&rr27-cG@4LI>im6DhNTR~q+wAJVc~iJ0+@fT6v_(M*R5N( z;t}Y!|3n^?ou@Bfj{98eP>36toRYE`#lsddv{A31HA2ihj){#G zK!t;uD{SzCM3Vg?VjZ?1EnloG%07G`Lq0Q?A1|m=u-jK4iuH~3GhF@yYHP;a++1Rg zH3I{KQWmT0;vY6LDfel{apiba@Rt(dlv3 ze){1F3M_sS0EhTtEW53*A7%{u1%^>L>T@f=5otvG2iPpsSXaK8WG$ zm_i3SO+Ylf+%l%?%6;B=hgPVs7q-WNPqYLS+fQvo)p^Itig-PMScG@#_wSk+Glr-GkdqL0%V1yqX?Ix~ zpodRUSx3iy46Y)$CF1@e(LJ4cmh=RO$@f|x)-<1ute&m;cza_g`j+pFjXmSzJF#ov zod99fro)F16CHMJbnGSwSH)^N3Jfk>!0VKu(%C~pE};<3&za%Ys4IG(D3kp ziP6EJu97`KR{XKLdOKROixclA54mjhuF&gph(P=)L$XO&5hI(j!6b5|L5inNZTSBE zJ3jz0GGZj+qSclv5;Gy%0-flpsp(eSQ-U(Audg3A@6~kKtu$SWXg$mF(3Q2c==Ew1 zpf$UJOhQ6JaDkh2UqnWxbVBkBIU6e%C+A5(&yh|86tX_v2gp8<#gy1mJ{HN0fUSHrH6`R`4+dm4g+Yc)(RDf3BOC68D{=`rvq|^+g^@Y-_m;{%QkYG`{&^Pn4^$ED*M#xi%L2ST31D`(iLEfG4 z<_!%(3*jA#ekk0>Ex-}0zDZ98E0_SJ%BVoEB5AcJt)C$7b4Q0k(aoEejO8Ak=P^Z@ zURz%uc*1#E+vp;uqwl5gxB-(Q!G%j;u=k1kKTAoex=F%)F~K#OVe)R8=87s!NJvQX z-!q@omN7b6d5)6qRY-KS?*`56?l*5*Yqn5kI@cDNhX_=lw-bqdXzZHv3I=1yGOo!c1|nj4X3tDbAz7=2(E9xK>s*H6At7aOSo{F42)mUO2o_#5 zloS+3aG_pZ0aWRwf8m1i>aNi6a6vSZ>3}pv9G8@QjW(uoIl-yHd*=4{o0Radu|2ff ze`L;8XKBhqmJN5N7G0GS_J2-heye=s=BAI3HgJoNo7))k4r*IlBWY=Arv_vb)dyXO z>>FEx`Wn)B-#&4edsi_SXC)*&&_93v70LjWrgmVtLSkZO*@mUz?eWw3V40Ui#KO&I zyN9eO82BY;4^ihJ=)D5Apfq+-dY!?TM?@qH-=*8L=VL_J(57z8>kV5$&gskdrLNA* z&}nMRU2AJ248R)tvT4(%D0CS+W`gLu11djPIx|Sx@lX=BIkzkDPAxRc!-)BoqDiXgxo38C@a%oNmU`+<4Qe$Hy+|r3<<~0qhUim9Un4y;|o|@P6 z;-WoaR*{sIo=%R|-dm@oTshd7P{_*4rYwu12jM#AfcE>!Yw>_{b3+Z!f!O6h+&pZO-+a__*8gGx=}sRVH!_;bMrtuHTh=PJnN=|!?>9vGr3UcJh;b=Ur?O8G<7ADf$L<>lpxM(VRQoHuSD&XKwx zlOn4JlgM#dDeuH|c+H*NqW@vw{kG;-=G(vOCQQ>g8K;gGVOBOR0D$ATJlOPYG& zi*_8SBwpBYXwfHEv^YSC$I8je`#3xMw%_~$9BsiKkh^@kWv7l?rp%tzMgrQi+Fv<8 z+0B0j%p+pAzW~S!mRmdKGB8IghXJ|z=@XYigV#38zmtwh#U`buUwjbTshtx<2Z(0|4>7)+0$b3$nT-WsS@vsf=tF~ ztgkr&9aiXZ85tQNkOgvb^1x`@rXN3a-^Nbo15+l9%`@@|mA(2tB+K%k7$eNP19bRP zAeZHxn#uv*W~TGDSNP_~#>TyO2Wxx2^nX3?Dzz1bN#e_w{xLE8y->OE4#*TBdH~E< z%j}${q>@o=_PTwI$T!T=iL(-jvK)$aUvvjt+7GELiJgPM0>dP0m(o$Dvn=n+U;`beT-h_wO-b{P*<`)eE zxryoe)>5fSk!66|S6yb|*oRUG+(R0Y3aFUl99>`4Z?KR3EQGS_Z(WDg-5rJhk z;gv}}06aNNsS6~bgp3R?R3K-{!GKd3b^9@tB03dTSye@jd;Bmijs>tF9~o-}jd4~@ zyGC1kdooUE!ovE+@=1%wrd8aWr3;H_5F9J`29CYe>Y^3wKG;v3AePcwNHS1aY(f5d zHdZ2!#qlgT8%fA^!eDcph$>B@qtGf3Lgoe}g?%z)(c5gx^NmU;?cgii*g&ZW* zhrr*#hps>aPeWJtF~SZqPHJ>vLeej4YCb|5WsEpvkFFeY!mT*KBcDFy?ns*>)3PBW z+qZB=?vysbd$^%c!E#arr4E4&fJC~Hp6_!1Z?G2C>H1!U-9C_S%_2@T_5c7lv_*ZqYxjAP;G(FPwXC`1B>ojs^eu&3I6`O zfOGqS>(h%egQ#?1rG{tf4o<}ZUfx}J?H9VGmZN>Qc}MS;wpA2#0B^f$Y`n?R($f6; z^&aphu|WG;@}6QU0=js3PV_jb5)u>HW~2vEAfWk6M`Y-_x?90^K%h-YO*O*jncZo| zob0|zdWTLI?DZ&tkSepDd5)!l+?E}m5!E>{U68neMAnObj^^$i>?9s^TheDY(Pvzw zAr&D;Q^1gg?BA&96uGAzm8#7xpedgLDR&|fGS_p zX=S_stKJRfZ7A{&325<9krQeyA|Z$3>!iLuW6AO;_bRFq9hOp~sGJ-DA~Mi&>k=bb zSGD*t3FGj@%CSjZd)hCM5TaSO>k2YcV(1HQdqQ@002+QqsH>ccQScH2m53>G_#fmf zegNU7+aW~Av7a)69Fd@fYF54humYhbF7VGprzF+_F*c(WHGW=%D@daU#Ka;>x5VSg z0dhTRyCL`e43a4s1pXXwbHOD~1fPnKQDVUzzQrnq0)^-;62?53EVoMP zXh_RC^j<=T`{Xo`*`0#g3`~B|mGdum6D3HV0kQ z{(*r+^lpjqiKScPsHSB?1NRaT;K-Z2yd3a}b?&(Zp4-?tANUtHq@qo7j}hj~WjUud zy1P6c{}!Y`z=vRjXa67YA=*uPw)9NG-ZDh<+K+eikbB^{vr7kapra2Pz8y6E_hRBb z5gcT-wDw{(soUz2vu*zH>>)7wMq=rYHThywB?+)eU1+@n{ecC0&$Hf0+QX;J#a6N&`?;;#uM~x0d%<}hWGsE z=liuz(%S}al?Q43A8ai1bdKIJ@6u8AvFbD+I)5=mS#8h%udjGVv&_oN%gKhjVlxS1 zeo9)}^S{?fTnJJnJ!bFX(a<4@fx_KE=RPnPXHLkS4u&c60u=`UPbD{EF``>-(A>GG znC$*OZeVb*7ZI9YSXd6#vrZ>7@#cXN8U_Zw;0q_=)Sms5aP&XVPu7#7K|b~`aO$2; zb-4fk|0|JD3{C-FiUN`VJz%Bz>hm@bT+mwHDMBU*ons||jsuYYynrpO_L%kyu{ z_h1y?4oOK#m_-u#HiSH%&{uulOlQ0Uv^WBB#EE|eFckXQz4(r8)5^K{+}?A2hQ~1% zY3Z$wn5po(Wm;-ZT8^O*PEqSVlpInX?TJ=~LN%T;e$GFKKb{l;UHAcT*^d}%4`{c( zrG?-~T7gr(U(#*|(m{f}@U|GRVOe>(eCB%~`ZZ(oW(UQ@hnPjQp>){_e7ubkZLWPZ+E zzD+;_6kAaWKC8hS)FK0qf{6(Cv12cCW1-&QprO#Wv9Y;iGxPc|ollg~q2Qyg3lq%0 zJKu>Rx$Z1>EU2*~>`}3*QTzriIsi8CM?ww&C$w#{&V#t2rFOvlFs2|o5+Us=8eI6z zOU3tfde*M5wiw(rK4)BN3cwNuXq^_#Tm4p|wG>vbk9|>4(Y3%~LkVO+a{zzfwEBpE zOU&e9s7r-8wS*}QLaW2AK=#<4%P)G=%4uCx-Yo_oPt^CZmwfd!D zF}nnrmNDAbfVCBH&RH~V^&>@VD2#n+Xw5D`vQ5nX{Q2`I>NpSZ3owD37MJBb-Xyv{ zU_PdjJ)LfFc{m< zjnWdu6akY01LGGJeQDF3l*CRz)0mM z3^5<)8$L4O+g%2Oc7W-;?#o@c*}J=bF*^foay;pMpe1KbS#M_|i6r~BZW5litU)Ra zu=fdYo@n9gkKWyzBoYgB&Od`7u(T+Q?yg{IMgBjnoqJr) zcN@p=4(mWjQVz{c%jA$0MvdYwmODfav6kYNM-0ofiMmS%Asuyi_H14eEh>`7oMsh~ z=8(h68b$|}*KH`r66<+i+g`8dzvr)~KSjEKzwh@td=A$&%YN>KE6+an+tKE?xw0&R zw)_1qGgnNgLYHk2iIhaK>4TGU?9tMkH`=OsSY zVZ(j|DW%LdQ!14JiZ@RFASiHkb$71j-r{P@f#xOyExR}Y%ivgU;O5pHUFDzGk#Qxp z>TA>Rj+X~KC)x(iXbkMzuk$IB8Y@t2G!EC)SNN6IPHrlDozc7F!qvKBHxNX;GOC zh6V$G_cVVWk{(*6bEp+g_u29BlIJZA9wvtRClBTS%Q7iy-tspWzj%1V;Y5?~K=UB* z8HHRYFyO$|A9B)s7M2uuNqVbW$)2+40?pK>($)6;It#|!EAcT6cbGf9lAE%PR~>&N zX}F40IdS67oN&8s7ptp{cGU%}hUVH#-_rymjjT7l?hCX-rQK(L64vP0BPpsKdC}6W z{JgvdJ}3;l)Xw9R3lBT+D)wBeyXZm4Ve~{71=*_T@W`Z%_X}sfxgXM^2@FmQb_*3G zz}Q$`GxC@D_c#1Pcz#I_Qu1dLcsHsupkOt1qdzM)1*b&Os($pUnF1~Lesi;TX7g{q zIg7_E)W&-zY3cqsv4=P-%_Vg6hr+e}y7TOi=?h!)TZ<=T9?*Q-8hC7l#m0UewO_Km zE-O2G$UH6ToMmXqkyb5)b_<-@wCDs8tY63BYi8lyVD>ba*_Jb3jfgEs>wZ^COY8~y z3RvbKluSZTLJbBsBLLqtX&{3y4Cq2rv$GSB6Eu)W_~>UDhOT?`joV<75lx#hz-4); z&5;rgb6lMCl+Y0B;0bHZX>#BJlx9}3K*b7);$h|tyU<=w?b=? z`gM8kv>Wf751se^WsXTinM-n0AQqMv*RSYa7QP9FVI$+IYadd~KooDI)_b}t)Fy%A z-22@o2+;FLzwf=21%x)|YjqwOr!<{&m{y5<;P}jxR-&NogbAl+rc5?zU$?LDq(NNj z6>koJoN18pM&ycelHpuvD%y~_JET{P6691!LM zD`Ykq=V;q2`dUxWP;)D*%2#uIe0m_TZ|HEeN)+2NuYK{GwpOMqd1f}xzV#?)2l23; zfWeUu+BqO!a3n@SqG!EU=vVAI(Q44WoRno%ri0YAd#Ck!@bKXt$~j?M(McsUvQkgM zg0jKO{COkS$abfqItM8JZiI;tv1=TmV&SUdG27tJabjpF1pNG)377#E!(+0|BuBvB z_}32RCel{&Nn0H;#b`J3>ERVHF?WS@Z^8kdwW5h;Sr{scSllx))B|FsVJ2PULym(L zoa+Oegp8Yx;{!DT6t2}QEO7_axr&DBK|TW^Jv+GtNtC8i9RT5WAK(_T?`r$cH6V*E z2n8!DE2Y?|mC2aeTS8w2l`Ci@1qlYyjq$Q;JodRw0u+gy z#ZlwNCEx?8nchKOb^>c;^P4H(i$l~YPzI4mDrZby>H9jLM0I|DeT|_k4g{OdMMoGx z={bh^0x3Lasr#u)1Q$8G#EA-h@n2*tq~Fxs0}yOnb?nBL0Nx7-%VlVpy2-9VwR&4j z+WT=4)%Jsi#MSIad){JtY|_Bgf39)}tY+N3o?=1Cl6SDe#WLt$p-iu*9L_~r!EW)X z5CTMqh9^%_=$22M=O(#fzOF}@cjQ?c_J094S*xUhSN8Ju_SmrDq~}hcmqy}`ms))$ zs}E?v9oDR8wIY0Nl4>x_2H zn69*5lS%(XkK;kYhC(>ak#J(nLq=xiX_#?PdKoNrH_OXT1@p#40f$!+BNv_80`23K z=?Pj#Hk;N7*DA7kOYM_urAo>xUFLCTVeGgoUzn!)G z86fsqp2Xk;p-W*p3}LZ`t=e@`}#4cRQsBs@1Ioj>HXYU$XQZ{5ZTMD>nzp!d8B& z+mC@r2~o7Qog`ycfKi0fkONa1$jIKIt-I^#s*f|3c=he$dbACSa*}l@h9Mi*btYP^ zwpRD=34kIMurt>fdq6umEN@0QY|h|2j{Zl@?5cWg5}r0AkRv0^A^W0f7-%O(A+cElqtO2;~yw z-il}d#@;j^^pjZ|-6%`J*$>Qh;Y@Jp1iij|nYjuv)Wkuc1_}khxxl0ZUEJC0U;f?~ zvcdosjV-|@#LaB=^KKLanVIvZ#F-B2rY8+}o@;2MAzclelSD1Nib9dCjs)|8W$7!2 zZ`|Do1B5`@TP(2H%9*6oR4vIN27`PrC^%45G$Mj&98yW-vv^XvO2D$Gx4=5>M!t?{sn-+ae5s9sT)w-o@M75&C~&R;7_D7$fUmFZvxtH{Zw%rW zjF=i4>O}J@;{Y8)_7CSy$-O1uw(4q!Z}GIXw6^y9YB{o|90~;^B3ecER!>`r!92By z-#-k~on7;DO^qjtt+rr$aQ`@@0Xf^FK4k{lo1*XVh0A$CP&k=Kx4H~tZ2KDPVa}Mj z>WYz_`JQK;&e*%-z<@J+GoCBeT=eHW4CZerU0y#n#WL z0P)@Q?V)tb*tjVdptiKFvFnM*qHVTegg`t2iD0MdTEBG%zMpVkX-{lMnmRah-3Kdg zn1O-#c2JD|tgE9rUH|s=Sg^z{yJu~IM$}M?zi}QM&JQW=c>E#%-=r1(;m+MB>0&b} z6c>!hDj1ST!OV$IAo4aOI+m7GWR;Mo)UQIW<;GvS@1cx3)YHV!+5LIus{^o{D6nkM zuqPrFUJPmq;A_B+;ftL!ksTNzZ_C);OClYM-LcUS;( zRdMk)J+VxOHxb6zl~ieAy;5JPJ&No!AuVloQp&23kgYU8lk6;he2ZMVIe{1~aw^EJ zL&uI4ADID4j_B`LWn{rV^j<7ijz)^l6E|j@k$47`O4O&Dl9G~qlc$)pA3=a!|I`0lRx&RZj#9~z9zDfB?s8-pUjwtI|#ni+EpvK7o0q?S`%wWve zvF$zd`D}3*Mab~cgvkmPqr&x_2Ok?1R37a*qngQ=F*xH%_q!oeIi!h6{pkW{Otp27j8jtl1@5qA%L%Xda1&pPw}#KSUiXt{&AR zj)FY8eJZ>c+7nS7s+uj~?pe4EuF;zu77bP)9UF+A_E7{C!>&DhVhOX2NbU;{UuLe% zv`|u!PDGL* zz5wg;gv1s(Y$Wd(CnHJ+K=#S@mLjkb0Ewo2wy$q)OGUip&fZZSFCDOCTqu({dC*ka z87;sYaM{~&kCtD#(u3vFzNYK2ldyTU8;!i7kr4uKC7I2_+v0EK7&8hidD@;BJDo?5 z?tRjI+Eb} zA~n;oG!R0*&4=L`7#w27FY!i8*N>;PgUt;hZL_W9v~oI_YqBiyL(RA+hqZ5 z(EP10ANixNKUJM9t-#fECc(~5XAzc2G#}hPF&gP4M$+y`5~y@%)CR0fy;B;QNN+NT z;n)C!pkla#g1E*(+iQQBqxS!9MLbku?X+8<_B}4c32aG624N)UU5Za8p%NPPWcR@9 zcfRp-dT;9Xa-za0l(Jj%ms+T*+`Ii3mky{w@5U$E(&ept`<{DqPpXzWD)^c5%Lwkl>aZU1f#-+sQ$|gXdp?bz`}+&Cdd9` zS->E`vM~O0QMj1@-!2P2j}l=LaIK(Ss?Xf_nfB{MzKY6dh7@z)!yQA6c_6NE+9Ny-!HT$jMR92_9`=2kxPsH*z9m&hW$~u^%5NSD6W%c^I8tu+lb_fZF=5~qo zXcE7?@Vh^-eC7B!7S)s=KZ7O;R8!XnILp6%&+t)pY}iQPGPymdTe2KX;NEr8D%OsV zkF@4_(;rZi7e6kd(`%*d^q2b|2kiJqSB%-%Cll*-_q#ojoYpG`Gx0wiT~}z zFIvxe2Kf*8@w2t|A3R=envHKD+8RDPI^{eH-|FobVmh^)C)}O%(Nk`n=f+5L6c!df zefOtg+f%e(1mmgn9+TJ`$HpBN@6BZEJiY3-vaQ2`r_%m6Y+?wBQMD!i%k&bXChhb| z9TWfB#}6Nsl$Ae;uMyCQE}yiXyJP${y-g+`Fe21=j{2^9p~_)S*rUKLL&RnbP9@@4 z(n{N}Tjg6&-Am46$hOuSjZ^WxKu=H<>IpJGT5M5Xmg~3pO2(zDf#rjaP0}xlVCuQD zFKbW~_dHAk!$c#b?~c2EphjSma{Q>oK!p{krE@&qExFJ0~PHZ2F^tao72 z9|2-cfVM+QYTId#goG6NqPewUlbtW7mL{fc9rC^M4q=ybJdC3@bnXt6zHa>NlVN{>(f#%8F08kjBJ;q0ES^ ztg1-LupWZCRd`#TFeyff+6(0JBjrr;$exveAh%b9@9+?icUsP43jwzWuXCPZoudnC z8%paYkEO`cI3oVg2V60r^1+TRxV`Srr|(<02YK7Rz4*%VLQbwzPwuF8+PIht|E3*7 zL-OHjZD5v6bybc{uWDT{kqeIf#S$HQGcop|wbdCH5wRI1c4~wHzrq;Ic*~9Cf~tX_ zY!(%t&7(KxE3t3WYSUimdTYGfpRH9%Z>niJY}{*hOpB3u&`oz^d)^#9zI20sUIv*i zQ#>X3vJz0G-wb|}wsX{a>xhA<;9$MRU!;pdwJHTeS5gSAC4mK_bdO1L>ZN z>{}VWWp>0|h%xku&elQn^~EkPbZal(~ALyZ$D>wbB^Iu#YDDMRWPOW7D|sk=8m-2D%P63fsoYziIxAcVn&S`aAaSB zP>sWI;Cydys=ZI&3K5ieZfcr@YO6#X7L)$ot_}G`caSyZFWWSzcR9fnBgyD6_j8y+ z_$ct8cKWwZa2Vy#HF( zI%W7CiWR=fRqXxEX!DoHP+qhgfEvaP`&EO#x2QB_9`8<{z8#_v!npOOjX)1R$*Enc zwvO@XOLc0cJku3Lt)JU2udDpCblOdDns`1B+wiag^c{gVp*n(Jj!vq}lpcEfOB;-um9(VgB2TFX`4gsR!<}0W5s89$CS5~Y4ZBb8GulQD)Th&m! z^cOK0ltMQGW($!8gWdN3RPG9d3j0&S5a-+D@&n)4b6@(_M;}Z3Q(9uej~)GCf7Nb` zOLX}?_)OIlzI4kFIS7Z2^_nZp4Lco71W$!s%G2ndC3h{~osQ}x7Hch6(|ZAD7#6kF zDSwkIOX^>VLdp4NMi*IW!!RkGaI)4n7>tfn6%}^r=EaZUAZyNL6D`_*Z59CSU5?k% zfZOD9ccS}S8B^40fqtSy>`Z>=6yVk;T_u7hJ>Kl`3L*C=qD2hXFj(~jPYv7Lh@fQQG#5)(R@$g6XTtMryPj2I63Mz~GEYfqkyn*CYj zxrt~AY5mc~r2yHSUAi+64QGV31^J4rB0+-=HnuFTBr&3d_}d@L zgbp@BmGJ7Nd@gZ8?N&Z2YEqVYrm#e5Q!OU1K>}}%^r9gVs+n9)^Dn>coV8dKTRZbT zqwreHk`2i-@A;&h+(B{8@NQh*Y5O2tI6?RPtfBxSYgf<_!dqfoA698|0>XsySMH zPNV2m@CclKhS-+j`=gjUFa(La7grrG_LGARrGazbvp7L@JF!)QGV$Cn*s1%M3qE7{ z)hhKR$0h0|NLkBi9Sgqvc|dQ3cHk~_5GA`q(l>CS(Zi0H4`VYT0Dj!FOVsnAn2b@b z=h+WR+e+K(SJ$r7Bf*{zoBw#0dHSrc$644}XXiBJ(nT9|_v)tIdW2DRRQk4Lm&tz4 zfy5IOjP*>nSH@=~O$=MIV(e;sTZ5j_CRSGzit`Bv&Wnpr(f*-aJ}HE1nBoIlC~Poi z>S6oA9JWooO|Jpk@pLV8A(VwELRh)XO9*Y|>R*D|Bh9m&O&^5X>n_7;5n*w+PYgr+*x%}&&Ajt4tNg66gp<~XtQ?bp z<5-ySI*;fbv5Y{zy}RMMsN32H1#jgrD`Mpw?V4WTAtJ<W3X7pLV+neh*hPot#<8fbd3nU{lYA0G+jam%hK@pF3aQM zNxc^l-+pS4s_iI|>6D!nCj1Yogp>n(Z>c|`J==lQM|7JIO?DW1Af_uSD#XA?bc{W4 zfezy%b!+g(X}=&tBZWNfA6ao3w$=yP2KPa9qR{c^XXX7VC&Qx&lqTgcF=Y~eq>DoP zcR_k!G#)&fWxsH-Zvl07Qyd1j6cwRx8*;>$X;$|0+w70RQSp&yee{8vvvjJE$Frb^ ztn7*4(Dd&R5(uw0Jio6`qLRW(Tvw{p^LD8wlC-%euEPqa{`470e`rcuw)2g|UXU^o zDi77(EH*1*Ok2QSri8rq`}r@5`UokbZ~Dt8b@WohwddYcXa5GJxbn3i@yc%WTy%9xtt*ksyAMyLOG5W;O+dkwV+G3b9CSL0Sr#Ymj=GNpN9YpURiDFJ5EXVU zsTye64O4qdOBvOu5JL;m=VGH`dxX6qAFc8N`(o%vp*2!q$Tt?q8W7X6i8%37wg50? z5)`0Xj{z1Jyheduyr|@4yIUajTbi87U)U+Rtnu)l@PrfJJ?rRSFh8iSHHkWiQ{fTY zGd^+W`D=Wy2BL+}+u>o8BTN^?SizIpRR)J833=+$*Ut{v=fCdfZEyv8W*cu^NFZ+{ z4{f*>&SEYZ0xJU-ls;oXwL8WJ{L~BSuIE!V_MBF0QVB584q0Zv(3@r1H(h3U*uB;y zvO4q=;~;4LtL2oQ>;kNcc1DbJ`FG|uQ;^`4v9fHeDLuB5fI@9@7C8A^c_;FiI8@PQR&`s?y5@k<{E(rh!{Tzq2hkL~?Wi4?_9(%kc|t6W^fTyC zNgUbe%Pe*#rTIB3c=5;OVD<&_8Kb0qhaCY&d*pRK-qDor;Nr`o@>1b-(W5N4#|VX76bpp2lt%`g&yWV zzkD43$eh-Z**Gl)mhcB>l=9o9xqMekA8;jGJG4LtM|JEPZ*WK3*Y~axz_)#a{$M*< z1oi2zmwFNxc2IsGJG(m&bJ9z4rZzjQ9|UXf&%q%oxH8)2CG!^I)I`{Xh6JTsLY~*Z zzOu&OlB}5-+2r`Y2ZyGQJ{)2%CewnlZwIJ8cs7#Dxo>B-Ow- zGnY;B`>+lb4vh zdZGFffgccJBcFo>gThYgs&C%9W-)v*6`%G)WwXu3$b2sxgY3Vlf@hV2vk+BMw8D#I z-tR$Z79CH1+(&o|Km4eV<0s>KZga@@(f(4vfoNJT)F^kYFtBx^K1U3`Mju5ThVk{i zLt21H0uPGAxR+fgN_Uv$IdDAx z;@JO!IqKh;!5I<>j#DczJ>qSxwCRHlEw#3X6Y~-}`F9>*@L@Tds#Tz^=48j??i(0t z2s$s*)HRasR1eIFSDza9fv!=ZL!5_LNSMb>US=QV2Vagd|Bf`52PDc`<&n${N>;7i zhS*yFVb|>M1HI?ncyN830WU%gKRsBd5LqBdZVBhCZ%%ic)iDX#gvCIs5Yk$=f>}nt zST%mFvR1UVe)kLPGdvsr4a9#_c=V22({)LA)E>cc`36==w#Wsor}F*OR@2LY=Bg8l z0u!XKxeOI)b-^${sm6%gJF3EO_<#zAdP!v&)=tmBKL+P~K5Zw~1Bcd0yd0%=3qKwa zc-8DuCa-Timcg>PXo0>~Q+deSlDVHjcwxPQ-gzA~m!&2BG5=KJORG2){0}AqHnchx ziR%!8FFFDKI2$-Sw>RI4~);=&fJwY5UfgVNzC+ij=@g6x?`g`)?1^aS-}m))QFgCTp&{VRoG;NW0Fnikq0K+L+R-0oTsCqvUi zgJ#6M|B4sc+s)zbT$xKaRwfe_B(H8jS^|>8w07BErvGI%e6#$)V{N!pv0Ma?PVz+d z5w95Z)&hxTPt|%NbudZO-XNi9L+cbIypH5zs(`#kzQ+;F+X zsW|?FfKJm-{yK!HfB07f%0aD=5KHsf&qU}U`)~9si)d$FU1Byb;(5Wbu`TjSDP6v? zzk9V2_D}3lO1bomVMtz-ggSt*1j${7RVN8N&ven{8^77*uaFk3&o@Ch$1(pwr+IKv zU)#0i(vRI7Hh)JSDFf>8^+*Rvk(Jaz)X%>Byp&-VG+-B;DFvzN)NYX<8^?x(^*XeXw+IUpk4 z@h{Y(>XfW^zjsEZh9jp?g1ABksSd5qGT~NS#7JK-4!YlOd)3o4@9`=EB<7y+v2{?E ziTF;i@z#6q;77N_>WhX;_X`D)w4Dq7Zd`=vNyWBq#iHLi2;GLS+3JJYF+j+J-a%B~ zVemY~S5?P+8i-^0b+@NaA&b0}6kf5+KfVjIdrizvNwe|{^%Ah8Y=EoU2 zIO5W7;vx2mFh?sJPxhHOBs!$s_#@Q=>WH9iHn|zC=I0oGeRrpa??F*L55|jD^f(`q z&S@dnF#q09&cUW53iiCcUGvTwV{|#?vzRH~?L+;ZwtW(WFJW*BB$-sExXq}6zD^5< zBrTb?Y%(e_D5SkB`{M=k*lq#R`%LrhWla40_w<_Tk{6yff5j@98(~DFJ?9D<;bfw?pIJG| zXXwqjVWqLIx}keBy$aplv!-tEzd}DBRm)%Th$7=z-$>oBy<@5^Gk{2g?rQkD8=`Pf zC8uAYmLQo&*stC#ux6Bu9Y3#IksR}-M+a{uEZ$}y8#lTAdN&C0_?DpziGz!c<)zjH zOH9Wp4b$xO2d5|d}; zB;NPyQjUNg#%v30i6tk)(mkS1q2~{buw2`!ZKpKgM1tN}dN@u96j7K{rF1 zD5jaeVU}QidyDj?ALJqPLbRP{JUba`>23h)T?V%`jm-tDzHDE0WIm*_|BD*f9NjZ& zfdO}B`*J$HJ7R7nx=l-5&U>oKrS@f~x>^(g3audgjH^0AmAs>><^tk=f(TVvpn945 z<{oR>myiX^72{b}X?GYxl*Di3I=aFUkLQVh)lXPxF+q?t<_2-+TNn>}ko08DpdLQd zS6*18L~Qocw`!@UPBG;8AY%<|uff(WGU;wK1Sn{)ga--YmclYrvk4dFDj^5y;|HG! zUaRPACcF?@i@?Qn=i@EVkq<{*Sq7DTw$T_*i%VTlL!NRXQHp=fhvkF!GDdvD6Nsmo z40p$V81U*EL0=n?M+8nhJLcx`UST9s%EE;mLS$?=5r6pQ4_6RkyfFDL)@grW6vJHum%cx(Rpl0 z(PZXKpoMnd?_R5rCVYn-92O3B2dsqdy8(g}0zrl=4KxLnt?HVdPO1qPY$?b7H6M6KTf~=8(>Is zEO2>dcu%*GaKf+VwdP>7%n_m?Z-9^Iby&*nXZ~&ov-tu^M)$>l=G05AOcg0OfN2IrIvlp57b&wdhLP<@b@6_uNp|?Ci z)@k`??1r8n7egKnndUmt5v% z*WbxX%kunvl4$jOWZa&ZT*NEffXo_v`{{mIx}9iK_!AaLmg?mlv&V9d2ho0z4{~(O z(Irbi#5!Ft$f@}xd>3EV=|t_ye>JebH`2P3Amb%tURtDQjxOU~5JkKft9g?1WE3eV z$Tif>025Up5bco&9?e;#<|9i(hO~-!hRKHMjzPDXem$)qu<#S3B56&h97nCe)^CWH zvfhtbDPrT~wHf6K-m47NO5`Q{CUQQ)4?+hn)zL)>~F<&x=ZPP*~Bu@UV9y<$;KL^37z!Pb>UJGf#=SCvl~ZHSylz13K_X!BlH zaCO8uF5skjKf(AIFNXK_-u!;bHYw!VxTK~>h@fLkUxi)!hE7RFFO^%r<~W59_QhJX z0dK8~R$4tnlJKYi3?;O#yd>Y=ZGNwc5qp;5x@nMmkd6-_cUB(;^+=)m(24=eF~u|1VeSyABCcE zzP2d+Fz8iIcrr2AIK{NQk!YfwpgK>%q>2H5meNJ&(>e)$78>qAXD@0hy8IL$N{_fz z=K@D)d`C2*)Qjxg2xUu;`YrvB8V#GJ;R7+w(hyAErs2a9c#Mh^i*We40v7(y3$Ph# z`-lIUq6qt|^0;xb01GCu-1n2>{HKj=R5%RHjBi_&$w;|gKm0B28$!r!vpGuA2j00D zf+5(#WE zqoD>g*Sdn@R|brHy#ld^F`#EgvSJzDm4KX@pU!;X*i78IuE`U_3X;8XTxFiEb2=K> z`F??h6cXW3RkCUT$f(CP*k%PP0Cy0Mi zHS*cXFH3tPcvf#7$1@rIEFBFVH$t)blb`6xNP`Ld$uK1gwW~vn;Lk~*4Rb(-;zaci3Audd>j)$WU zReZ|k{R*1y)860CP!G3Q8zz~boxN{FHwUQNPt}g+CwOmXaOtHD(S>?^sdVPwSM;9k zOtMm3G*ofdn*89{L&AGZS-8n2>oEkR)g3D9o$1c@X9Rvf7yEnrPqaNdT3U+N)-m)` zit4zUS9;>rJ}^=EhE)V#z4tRMC%3^*cL%Cz2KjJ^hfNN68Q=`ZxmVpmgt_m87K8zy z8SxZzbqd97qF&KEJ9+w_%Oo@&a9)JQr;qz4O@c$&pCmSry<})`H%^jsO}1y84O?&Y zJj&Q`p6KCZzX%)U3d1)Ey!VK~{L>zUs-+$Cfj&T#5!0|)9|rtU1lRWnncpOFk)n}I zHz;qEc>MN(FV3$yYWFK}Z*m53PoDywkRXI;g+(8}_rvwp1|Z}U!KFLqL1=JCKpEmH4wSJm~+xHt($sX>3{p4lFt-j7ePjy$Xr_F zZvf&svu@od5_Td^BmDZzUKGH!LNha_3+oi&2DT(7@s)oZ5F|8{&5Yjy<3pjCO0>${ z_}hRs6-(_{FPop6CguCH#H7vm5YX$Rwj|HSz7?7t z!1_WXL>uJUy8|)@gnT=`;S?bpKMfv->tT_lA5mnE+=j&4HL)%0fUfE8aCi&9hBwJP zT*+Ild{pEw>gM*%cbp=6YSD|EENC0|g%p<&KSV4Wzsow)pCy`FM0@6ipvS?i){mqX z>rpz}*_y&4D5Qsly0@bwEtpT|F$nJFn8zkdBHlj>7i`6R(4`{4NG8%tV~BU;@MIhjTKD?vM4 zO5n&#bOWKnRP~47^F8x)H44U&NRA^wM0vgCoL`QhwFs$*d0l3>BuJ zK_#O2Hm8Hf2+3YkXfxLCApd5Kp%?Ybx;CCvS<3Ham*VH` zfX^n+`*agHWc3oN%)3xZih=2qMQ0UJ;k`E)#VCcp@}DCQtT#0TwSfjkSaB;e&Vbe z6n-8^AQJo2VNSW#@0w;|bl)8xPl~6}v6c?}`St79BLn>jw8mm567}P*hbWBmN|Obu zUk7=bhU)pkL>qwl^|x!Zs)>2@N3-mkUIz=a0=fRxI0S93&gQ3#|4mI^etyGiPD(EN z6;$b>m5VnTn~#%zml&yg*v?GkyOvmDviq5S{$2XU;3QygGx~4;8(x(){rupA z_s?bts05XaeGh8u0k!A;C{PQ7iv-6pkC;`ho=B2U$5EzlKEU(&?!L!>wBlk!!p3Ml zpO_UgWdFmYQBuI;y{Ax)u){2x*wag)=S@D7sv-@@?}B5LT~coWO9V%?dh;Y;uJ|#Z zI3jV>Tv5F`vt<$|L+132IBZX85oJrc!<=;E&%=Q$hmgYhGL`M?>b&OjUe3?p%l&BJ z3QCrN&^$yTIjlGd)L4l%Kwx^Y(ivtXBI&oy-r5gDUL10sZ-BtrfNXKRyE>W3QNStG zt1h7au1<=^Y|KqHV1Np9wZV#28z{AQiut!Skjbl7wxO)2T++^Yk`MAwMY z&DwBOm*Jpp?pw>@&6dAwQOSSzVwRM8P94A8g{chee6@}HX;p>We9F>i)$!EUaLwme zN(VI%xVzafaS_h_-7A6_*%kwC9adb!*mj?7lFh<_05bD7{5od^v4YUSPg9aA zLap98Ty%`K zK5+|3<>Zxl%o_+l1(cIP;?ZV~b33irM_=BBfgO^kWRUMe&#rEOq_nbTRg-DS1s&h? z@>;qC%uIt{)unCCVRDxCLzZSaH=IGCjT*#ZsX$)k5vN0u{`0sAB*E}&kgAC?1=5B*@3-j~>QVY_4sDujvAim~rd4k57ChFuDq<(2 zgvlb(GQg(+|6{ofS{5_qf_9v8l`JKRdlb$4UoKR=9ZrerWC_xTh2zt5{BsBG$Y%QN zvRmaZ>#c(wfB%TpxF)Y-{Y1Jmy5IJTMV5sETlv*p^gNP8YcQEw^vsnw^dC=CS> zcJ>!M^~)$)!s+iwF1CmU$Ln1zHnxTtb8om?Kh!M+-0C^F3^txksDyDzJ&&P_2L}EX zibbs1{NcU<$T@FXlk5E%ps$9k|E~Yy=xtwo4;V8zMsENsh~}=8-RrDt^pcr5z|yAQ z`2K*SSf^Z@U5EYe8IT8F0RG2OyTK%|mr5d!777)*=a2LDrTx(!yJujrr-vH-k@J(5 zKX#F&b9xSiYJ*ARpm;95*UI16vbofz4c&(I9DXsW&kq4Up#ra$s$lGXL;v$BZen8YDb!slf`CI=rKUgrgMAh86xt2K>+8owLE&NWye&WsCURje`zx3pH z^~{QL)r(if+NCJ^slXX~T^8(nvHuG>`y{f@p>AQ*%^|KgPB48D$)eBbHLw`w^g65N zWs<1ECmk7+0`&*|k=@ixMr=klLbY=|D2)m8lk2tVm@e;96YcnuX@=8%QY-xb?;A zE788%tOQzsx&tNo(Pnb%Y#bf*;h1X{YzcaQHaMBq?itF9#vkaQN8ZSt1Z2Gcg_%*89f^}ALBNCCiJcg;_Y?urVq&9}nszwo+KYBmMK1vGeP7ASB4g6*Raq7M zyf{lsFgY|1ONDdIjS2jq!IHD>!UW!~oNXe7nZ06Z$^=8ROO5*sV9pj4(tchEx|mXy zJo$j7QN?!Sx}y{mFM;KIAN(IQ_NWCs%-O>^IIU9FBQy*n=*1e6#IxA&Q^1f!PyVg~ zL#BNG3zx*#m>({QZ2Isba+TOf1MNh7a-f7azM^`<^Cc~$5*t>b=uC$S6I|K7?xl8O zOVohWsn}gFe};fz8fAt7*BfPNf%BVejGsJyJA^W#<$p1V%yXQrd7+o?a_Yil#*Ic} z#^A=Zk`rc*q8_63welYYz#y5a+U&BXxRDXuFsBTpBm`Z5G}RB&?`GG}#gGjXGKrwsndWBFSJx>%j3^E#-DodX0634!c{6k@>w4 zp0M~}*effEd)n^D_hHBiuvxYIWG=%GO0Solmjq{AY@sN-g!S}5j)N6a1>!wt=Hi~9 zK2-30Y?IG=@&cOIVvP)v*8K#iSTrH6`=tor~OwCr6HL8SFbmQ zxF5}M-5ol8y+}%Hf8D$_!DtfJ>wDte@OOYy>M+K+O!b0$FDj3>+r1&0=BSedy9?lC z~wPc(_p_^2RD z4BoQ5X2d|rCPA@;-AUch)nj? zaZ=G1e_F?&o9lu3^2-#yr+}})VxJ+yJSVzrzB2YR5cI&og@Hko?l>%raE~}(r(0rn z?&(`NGeX&Vk0~HDiOSdDX~Ky#f*@ELK?ued5%V>;8c719vXv>lMuBu53@j$#zCTWn zG6V5|FyXj}b%(Oy*_TVj^{l8|qNr(A!r-yI`zTRL=F|BrJ&H&YaVnnH{NT#}M2c%yhYsM_ADDfnYr5P|J|v|6 zVc}5^RZ%ru>@GJ;uRwvcgLX<8N(dqEqZXn8z}$0K0=&6W!t-pNBo1fv4yY{Yq*J~F z_?r|H7(|8WQt5Ayh233#B>8J?40FmW2R2EX2rZoMBz_CcS#7~i?FlTt3KR`=n(2&x zmYa-#8(Xb%3>5d}vXG0VFaIF$3Y^Qey&K-WTJn+2jrXn9P?y>hi=h5AQi~v^t*|%T zP~@`pD#eds(8h~??r6J&Z#XHM8n)Uvq!5$#>PoofVT^7qd|MwXE>wI@^OHQTD;+Hn zMYR`YrQ3kBLYlvhBsb5dnEID4%-mNy8l6${)EYpkE4kjERuFrlSs|G^?_R!ke9@<# zrfKnHnQ~9Ai7Z~rA45KN3tplRlUs?Zgok!Yof`cEm#oh1y0A1poSE&ANp2{bo-A5T zk3=-Bg!YE7r~OI_d@FIE0nK^W@G2;Pm*(JUaHhynyn_AvU!U{NQgBHe?KP`46}jX^ z7fJgCAdLC0-jZZf)u-P*j~0t~f(<6kee?+|-!c*0drTXV_q%><5H|5YkH^Mkg-ts= z;}Ze@k@7ino*Vn^1sP)DOUUvUnE<#mt+7@EoDG)*pu{+M_2Lb{-9;TjzoSC%VG%X% z0HwUmWTDu+b;s%{PO0N=YWrk5Oz6scr}2qiB?J4K$9Zx5mjVxEm8Jx(Jh{To*!}Qn z>#rTb`{F0YhENPV3O=|q<|lx$M6yiyGztw1>3!+nCXP1O9$#!vv)DK3-BM&sg|a)p znA}xlv>@6lGZ>{N@p$qsLJ^M)6=v@KL6jE0?e3XJ66wjvP0x;#t2N(Kc*kOo$IgO9 z7mc>FytT1+*`FiSv@teLyeqV!OqLKXq&%RY%F!Q4l@Qro91_67PB%Uhx5~d*82I=g zm-fYg*IoTIGU1Elk!F+!lUWc;OFng^Vq8Ij4`E({2U9FEfuz1fU4&@oU7_qMvhihi z#h3ODP{f#a`NtqgbRdI&YV!tEta~lpSw{?D6jzPsqL80a#MX@oFoEf4QGd01waPPd z=j5Xre&S0*k`EZ6e1`P5{BP8&c%@g%5>OwW(<*96Il5B_pB=&0QK=rg5UA|5R<|NCnfHMSwRnRk4|vinD}Z{Lh%$rqElL zrys&r$xA~r$;%f?%_-F~)#zQKKPgO>;;M2MEvc~z`juc-%RJNK3IG~9*0ABrozLzD z0laZ^mY-7ku!0eisw7yGHM469G06eZqx>{jAP~#V*2M^X50YhQ`)4D6X`{JSP_Z_p zV^Ru-k=C>0Qa}({O3MJBK+(DT@lI#PmxL+`0|*6pJkOVDfyA8U+26{})Q;UJ6ZiB8 zjilEXDugK#FA>6?ksp$oSgQUS!C)H*?C9#&dSbYc$}ew^!5_P$*MC_=188+6A`&8C z`gM)6Xabamo0iOcu~cbO{^M^fVq*d0`1ZV>BS0hB z)O7!iV$>#JhNKy?upG;hodCc<-{@_HS(oKUQ>rb-tx?AV~qy>Qg=O~6OR zvhD4%O7~*EN+cIYm#jbOcY&=!p>{_)IADCUj(E{}qF&sL>?J)eW@i_;<{pGm`s@y< ze9^nZ6*t%|EGaQ90U-bDh+F)a^=l`KPj7bx-kS;+%ta;(-kD4cz`Rr zx>lhO7}Po9<#U$ZmQ}*Lx4(%mpqK&71rHYxxYNlOv40WXHQT+ZD(b6XbE(UtQGav5 z`HhD$+V0!8)7|*^$druIl~3cP#f=XC=nmr&vN#;8`(!!SY-e8UHhb(}*PD+(n^8Vs zA$J9ODT=~%mns!aXJFLXb@fUhOwWxs>Qebt@F{>*=OQaf;5I1gY!bIeGA9}`T5{a1 zM)E%4=2oTi-RHM5+McQYJ{=T$X1XyD{}+II>-(!X%Lf1t0(yy^^|2D@)O-STY69}n z*uO;KiBpL?=lT8n9k(H0`6rY10gx%%n`VugrnifUs5Af{L1WtJoxcJ{6NSGccT84E z;QG}s{wMDRaiLCm#z6arn*O-B`YQv_&eG_9`AZ+)W4c*Z%D1@`DUT)EA4Nps^Q-cK)AgC2}LVz6}>88X9 zg>;OzjhOJr+=Xzt;@xTr^+jdc(EnR1m{hp-PcWNKZh8sr&-%p(uH=^Rx767Bo% zG9v>P^^B4@N+4ba05-}Nk4C^p#t{)sbrf0VIjgJNE}C7j%nnyCY(N-ZaNR}k6NSYqtdvCOYgD} zvIg0Y100Mc(DV}Iep@08A~k0hueD{u+nf68$E^HASU2#6+YL}ZKom}u4nCcuqQFg~ zVGx?mXl#{;HrU=f4?Z~)#;!S_L-QX?&~NhG0LB**hw&NC5UN5ZsGxEtw;@2r+g4b} zXWEY7Vjl*W&20&iQonA7viOLAbes<)Wms61@pixI-#`M48y#alS+oy0|F!ga70$6S z60azv0@Stn>#Lij%+tx3m7n#<9ZNyGF($~-hP-~YV2Oyb1Kct`P zqtuKEzIn+*6FSOrhyD%)-gbXs5dT^2tC2yDYOhVRZKhFn*s}FRaU44qXV+oN^C`|z z_B3^>DYpC~nYtf3iX8D`eQSJ`^|74<_WI3};ds=)!}QAx8@#N;T$$Tt#4|u>(yM9F zHf9U|xM)pV-UVGYLS~3D>eWtD>`$kqGVB2Da^r=gW40pU_Y>-WXFfPxzr$HxrF&Mk zf}8DVeCcyo}mv!pE^o{8F9B6*ETnWL*y@>&k3c$vE z^Duj{N=G`GRcKsG#PK=E+>v`P5ijD1=5TMiT!&n~nHU8^B|SmgD`DnnJ={x;fK}80 zv|w3Uz`O>~in^i9&3h=CcLwyy)^cv)Z;;G(a=7a#CxzZHIw zV+Uc}9UB9JVwBlepZ$ZnDt_XQ|MLPYX;b1JQ`TSy0H}Zlt|5tqzKGf!x{olPt8UURQgxtr|ELJlA-(1D-9+Jlz2to9~QGqDiTadAZ%T0fsLklwvanIbzM zRQ~7|QdeMlV{JrHC;Unoj0`at-tI_SxS~dlSlcOS5Buo)NEr!vcfb!Ll{xjr`m4xc3LM3jROpdjcL0>UP@Ada!ru z1KVttvm`7qp#W{GmB25e-$x-}ik-M)^97(wLI&v;gC03iU~&v-x-e?2ASIYL7ydh? zRVJt~i0EI>H@UHwY%M1I3p;|umga4@^05fYH$GBxI%~HTg6GC>@00H@fnMf9lmstK zlq7D`tI>g+Ui>^x=g&>w{JZfX z@auT+&-%xI58GYi_>8#NFTXTRuNJOsHlvkiaxfxKV>rXZnXR#%T*Zb%xBKU} zmxpN@!x=ATW0{^8Eqfv3=hy%eztjRKT~;NxKG%KMvnpA7I7@84y*YhdKk_sw}v9(*O4de_`s%Yf4)geYx73P)8xgdX1j82(K#%m1p z$Je{N*rK?gJUN!wh*R9|uA^dz9)p zOx`IWC(Ofs@w49QMC<^B4oOPFm#S45GZkRc7$dkr{UQggl_3v8)#&V0AVw?qBG2kv zOrZD*s0`Sc($AQcy}DM+oCZaq9E5f=UT8O~j;IxtbyoNhwpss=rn8KS@{iW`%+TH4 z-Jpb|bhmVOcgLR^N(7`ENd=_4ySpW%8>CY@-p6y^^O?1V1@pvj?|t3ZW<)2psAU0T z-+$|1i+F$DSA`plUV%q%wr=T+vO$DE`8k@PCpzf35%C3>0ua=v^?gQDx8DMa3+*XV zL9g;WU8=pTrWSHei!Tr%vZ_DqU2Fz4g0Z-X8wZnV)P?q=TOUaZTz_5fk}ff6d)ToI zT$3fbSt_uw@l)c0{+zZT=D|AVH$Y%H0jf`Lk=`9K{j%u;xL3zj=NsbEzcA=?rBe5Y zXVx_LyMKBA8Ia#_BzE`#4SY`bg#hcm=H<=P9 z^sr~K?8;vqkde$$t&rpEkf(+!_4V~x-QsNZYp#@>9Otp3zv)vlEZKxFhR4KXR&|9q*1$#eiX6o-E}RF;ekxWj;;_bH3jnDQ+va?GI?JgVTQxjVPeBtfrw6+mq zkPBxj9EwSN2A9@1NYeff$L&f$tSeqqsQd7KD-hKR91xdGF}6bMNWn)i+j83WhQ zi54hTgzAp|5F*y^q`%Z~SI4_Xw*d1QE>>*B&o;*{`8|msm+pZ>V4$G23d|)Dr*JcW z+MuI0zlkwvG=HggT2dSsq4YO%?8cL~zMfr5y`Stx$c?{h&>JcJ+h(!QoJLD!OvR+v zneTDvn#*0LEjPf$U8O(x?zao&5WflX$lg6}Ixr-*a4|Vu2&i=kNMto`l1X)zO`NO0 zF|kmxj3P+pgu5MS_YIQmZJFPlOCU>ZbMb9y*2-+Z82_=gMVqS2VzarCiFkIlq75sh zu~diSWAe8N5=qnIZjJOET?Ju6eG^&(G~NU#4T<&x>A_9l=lhBI`F%pcBj^#1KXgBqJ~*^1JT+{Lp`;cI!wg~(-!a;@wzJ=6SB zgOG#~U!eQ78xDNg@s1*H;M}9og!+T>D=0SJd89AWhqJg(I3XPIs`37!=~ye3_5+?>21<^ z6lmYScQ1BaE`WiDN81AJTt3C~t*BVs34~Z|@+w4R0-jdA8xN*X zFy{@3;Xo%uk@TNvB66H`fm*GpG9{s~{Z47WD_-Z%-!XuD{9J_n}PeM3(z!?m>Y+C-p3MF;OAXB1Xy?Rr z;inyGw;i;P1*S1-YJ~2P&ywA$*0IRB8Re8{@lz|^G# zBY+^BN4_vYp$HgSxk>XJlmh-blA-XRKI0(EzzQ*#1oDrwYe}VAXn$PuKT!XmfDTo7 zQpz4G7j^#slOC2&1kl^HKn+=qx(dxuD}Qez6auHN?@gZk(}!g%OH>6=L8j&KC033x*5Sa*|Eu(Z4Mb5Xy8rm@H)d zrDT_;9A{jdT?w6)Ty{OeKO~kT76nD!=f{o#qK|FY)7k~sAA6`f{Jb0f?x`L(M+$=8 zR(eE5%6LnyJlDEF>t?pjy0vgi@p0ZJC%7Plp~7ETYch7%C+O*vm#67}Y8pv5dYXF& z;K(w(`c5Ph;-4NG3dU4FaEPxFp&R{N$VF@1^PPIgORf_FN4Mue)lz2iEi9uM)XWZu0TPeC|uk-}Sgy0~_xGwzkk5ZWq>-)CUxbTCNgY>8zdV0Wx3^~5-MjFcjv{}^~n zn`yJk4?@CRGS2b8lGBt;f~uJX6;H-=HM5i28Kn2NgnWN0w`qP?!Iwm2J3nXvN9dvT zQ92XqneX<##JIvjz@%7T`h|_~2ry5z1CE!{`;u%IQ;3OTCbwKLlHjLw{4$uo@D?-F zz77{Aa%=6n>$-LHb+!*zhk5zh90}1J=tcHe&e1412kcTNYqEpkOhR>bAW5J5^opP9 zlt+qoM3*)>qW{p~ar(T@U*tzPkxB$t)4{r?ZmpXhv}I{`nSDqcKr(#v0MxOdYwIzLX0l-ks z${TTWX=C_%#{LCt#n1l@LM|1b+_yMVF_>#;5Em=-wc&(dc2696idQ-V+P2|@@{P5P zeD`!6eK!UOjLYk9Z-@x!xo-fr^Fh~<2{SsC+@8f%xtYlLdi^W7Kom#mWj@&x8vAmXuvGU2Q{<2rIeC)XqbAZz!d*m2FDaik^&F9h# zjQTrQ0VHBhA4>6hp^-5(Pzk)J{--GrlGFUVW*sr>Y@y=Y^UK}FQA=}1EHE0rd;+-I zky`qpI3gN_pyyAohIP_8A}*p=Rr(AV1Z>PHDmls5qp3>H`mOb=E^jYH0dBL){Vxsx zM`U*jx^W$H8(WtA0M%LaJes>~jtw>ZhKr-9sHpLtW|(By^BFBCpK$Mu5&mC$P6t~G za#RMq*lR|(M{UvLgZxguN~SRV=x>4B{jBV{I~r{_yk%aZR$yWjw#wEd>=JRf4r^jdrQVh`s(%4d!&rl+d!y<2i8rr@N2B;f}tD zeRop9*2@DaZsevmq4CFL1xDuY-7H6wKWS;g2E4}TQ$V^J*ml-y%p;Cq9M#p;H8*T? zlamWzAAxp_)ib=yjLf6F$fgJs8T@6L1=lHD=7LC!NEnPrND=a3>PK~1M;8>#Du3@U zaRXkTE7!d)Z-jn)l-#J{O4}Enu)aQft5#o3m(|DG_!x z{N9MNAp55eLqtQI(cGIRyevNJqal0V5kUW__P#$CZF8ivCpLWX29do!B0i1dL0;O< zJHMTx>ge#goVNfk!x!bX;JzWthtI%^J@|LH#ZvVt^l`G5Nk#r;Qbl$Ed9Oh9=z}i* zx&Fh+SXJSF;opsNOSpm##|?otfg0#A!#d}31!E1=Eo-->@6;lNu9QLD@OY<9ad|?g zkjfYh=SZwk$*QtgLL&~knJeLJm&Iz?y?3=uW{7iBl6RPwA0!nj7A0l>f;e6y{*T_Y z8ioB7@xR=AXz`EHuY~?fAt{0u&tDsAy^e+PF5q4MT;wvp=fnPG*%v&`hgpV?2vKy! z%aEEkAldqG4RFV0)cT0Jh?RhaYr-acB$t@dCJqg55PT*S66iVu1!2a=vRvNchMC*9 z8i+>nkkeH+AuQPSHNK1{Dz5fsviVt_2aju%`d-t@PRL8`{hl1CwcR`C3_MvJjZ%F3 z2a&?@)qrfgHG`aK|73Vdu}b8HeVZMR6HW|+L)o<$NxVw;Jgqdy02A&iUUdEEl)`*? ziTdnug!`vVv5jvNkYK(S7y%k{K#|c>U5@|dB_a-`27TkD@e@z;Nr;`B`9Z5%txXT1 zVPvrP3NZ2T1yIq9c3_$xW=P|(ft3DwdCc&nr!uRhH~cR}!0Ryvh?dHro>3Nnf#i+8 zj>Huq7;{6${0zg#$1k-}banXvJ%uPz)OxnKaw(ZLBT_?ZG?}qLt>j9g+WnOIu;sz; z?yQ0pBoNMV{Kbc;wzkG-ctHvc%Q%cfB=8xkv2F95fXC-y(4m)ER5bcC&5-_u+>e!v zJD$h35B_iZ!^71`&vHW{*Iyz4OavxHL2+>AyoQ1^mqk z#NMnTcZ*Uh7Te(ET%q~Vx?T(K#Y);~DgiT1 z#yr&@W}iy5E5f+T)TN{F`31)?0{0MKfzKMfXFs0M<3JMe4VgyL`YR2kG=l>gsh+tz z4Uh=*GHr#rxw)^mxqz4MM$qTo^^;+1fj&{Ir-ToZQ;4FywkwE*@DjcAOM_H77Jd)l za-7%Pv3|#o4bx*Ody87j;ewPfPldhf0m{G51-gw7350=)AFn$wgcHrfkH*f&?n(oV zR~m~}f43N|!il+4Cd`K`l3gk{cVN262gLg@Fj$)Qc31*qgCjHAV{|9#1WNAl`>K_I zXPV2(EFXHYl<~4AyTP7YBo_9iAMFze;A5M!8Fs(2vlzH!R<-6FthtkDwO`6i$Gw>o zewKfWJiKYm7V_qjCEWgdBeq@KBZea;USOpIpaVo3*{Y7)OuLA^ZjuN031nCylll9X z>nXaeIFSMgGc)|nyp1vtY&d#V3w?*M#M$Z*O)ZY;mdQ(7LXl#|_en#))kOWl(MNhe z{HveYCkJT9VS*X=luA&CZN2+bM-H$X(l)~%X_D3 zZ^fyLA#@=s`QbZI4iX{d`AoUL?L4z_-bNJG9(2o<7Qf zM}vJY9WtzX1x&@6v_aXIVR@SMi8c=>2-M4pwVR^NTo#LkRD$MI(Z~qx`tAt`eMwoR zFg^TA3%C$BDt5IHdrwMsT|3MoYO%RsQS(@iPLjxRX#o}uzCjsjO62PVykU|a1t>}l zofkD1-ZQ;XH;&Cpf`_D5X(q}@t<|;gQY)7_S5-Mm$}htNJ1th`e=-gi5J&*RaP^C`%Aju^@f&a$hT~)v1jq=b?U_>HfbB5DDUlk%NOmLax11Y50W~~xd z^)OFl0@^_WRKPo_#KZ?e(1MWu5p@p4p~yE(`j%H2uGz&+j zX3_eR)#CneVp3^v_h{Z+i`IOFqy#@$YQJgSO6|*x2*6k1` zq%X#QK7kUzfDGrtR+lcN)dSZklLB=aMiA6AE(~_Am}VQsq$=nCzzTGM5Zz4GxtUk{ z^FLQ#EgA=v^u^#t8#LOaFnO7GB>!#0qE`jEB!IXv5k0o5T<)e`h1|l;nIbK+Gzx|= zD-URXn|e(#+0zNP+f0LUn^U&7q4ALliUqUk5K2ChfQOcd=q~S1eI!Q^N3l!o(C~st39Q#6!vun z5aA;^gCLt<@p>!Rh3D8sRg2CdfIHPdXU89mxH|;kF;-ZQu4M&z)OX+UXn*_mcZ|Nk z4G)xoqTxJgi^;a>qllr?iaXmqyZ67%<4TynCC~z2LurEe)>Uu}El1y92X0*)>xv{e z7BAVJXI~Am=CjjdM5+G}eXe(t2rIpZdfgcz1AF#78xdt2Kuf}5^C_|sD;PI8Gz^~f zE9rlT#1?KdfY(N}u*AY?8W75b4ccJ#qPeO$dEy{`ju?k=Pf-VaBY0>Z8i+yI<&m=3 z?25QL?%$S-uTjfDkW3M_VqS6_nwuNW_bV-?^wSb$HT`YeGGA{~iJVnc$MeDyOJ+>oDQ z`Rn^>E%G}X@VoqLev6mYLl9&`fYInGEACqYqk5nrM?`Fq{IT?&rkxf$lg-Va89O7G zONE=JV)F<0FyDHIR|sJpmo)bt+pTGmJWDbl*eq0L3bcH@V009nnSW!$5h4-A3whNHXKXb(0RcV1zQJ{u?V@Wd( z$*rp| zMxda*p6jJpPY<~ZO@cT{ewy8u6D_Q7dFwiEY5|qB?&mT!;5?3t()=crQ1x5X0H(xg z@?+Fr;mr48GwDsmAVkQ8OWGzc2@yCv8q9^A2Ak{qi_^4(5)Z#et(i;oKWxN{UHAA} z^y=EcGav~RgXGmKm|DI@voUGJ5RK!lFhl)-=h9rNgvQ{(R{01Ojol{{i|@ELh$El% z8^+vQ=$29l^kn|MEgT-1+A-=~EC$UB9T+!0``c`0gNyJ@-PT|x$93IJ*q--8y&Y!Y z!F`zuolGF?aDnT@s785GfnE6Y1%}TB6TenLq`^vQwSf#91WW9KNx!f&$`JI<#Fp}2 zS3P`Iiu2zMp`MX&wI3Ncp1}*Ks!KqUhqG#?Pq*cY#o8RdUjBR)oLyJ{iCb;i@h1$! zA8An`*}b{f7Y5TN<5TR#iZY|nFLd(cXLiq>$nI3n2^y?>U6oEwh9x#kz}u{f&wHlO ziEtq*hJe@;q9~@+#MJj4LMiIBeDW(_0$~r?R4K@+Dx^f43cTr^OwcSA8WI6PhHUzL{mW!orBCh5~ZEFWqpQMHW2C98+;eu>*^&*673 zI{NyT_2A8Bks7bh3aAV}iY}C|(DSajl_j$p8(!_QQ2xA5{S;x=_J1ru^6J3x=;f(9 z-RYbye|o#RGN}kN4~H_rXO!KkBrv{5O=FZLb<)~I;M}3O8zaJLz55uxZx*#H32#&* z-|rPh(5H$_4N90a9;-)=}DL$V5l%q{6bFL~iK`n)k z>n9d+lxZmZNXuAiiB9N#YOiB!6|(4f&M7~9?+4257QIG?lKS>cP-`=hDcdzXWjvF| zsJp0lU%zG8@R^*{@V3qibeicOSH8frVlR`*y_*$q`}K|D5rNZWyh>Y zg#DXq$+t)%Nr*)Xz;o<2Pv}L%ymgBsbNnQ`F0=FrYu6Uf=^TXso+hlFP$ODU$B>1O zsM0NlAmN0HB^p2ZxAW;ty%HCIL6Uny5m*)_A;u$_{R1Z>8#qm%?VJnyI|5RBp#0RQ z{|SWLJ#NYtiBZC=&w@BjD^dNPJWFP;q(@R%nn;&Blrxjp7|y$%w3L;#Q@x*RMgM%B z!^i+)V7o@uo2-;rj8MwY!F;f+1p;K14InD$7ztFT&ML;uu=VA3Wp6>>SrZ7Hs5XD! zTsVvdfWG$%Z2DwkIlkMG47! zl&v-$Lc~YfHL@^cVhC#l1OdVKv_`pliwYek%;3413IyBI_?``mG&~RHqFZCcG&mZ5 zz+_q=uVxzX=)kC6yh-V7KOoAI3JVRbr-q^?2^X0ZhO?)dF@F;>^wRQ|ojUqblwj-bVha0xO)HJA-!r`L zx<3DVyvWkIH6-H*8jMn5f4%naeEQ9QHNabEIaT9cAlcbh>${a!>okF!4`C)IV^t~W zsJjE3Cf{v2E2R>d=^@hJ&ZW*lzY;AXWz&TOsVnro8tfx>Ub8~*M*f5@5nt>$Rvyv> z&~YJW4Rp3o z@<^hUFi`<$@Z<*DwV-mPeY{h8cbRvco{oAewG_JgSxDuPqKotpL5VPs24vrsxAXs<5e z9B#s#!M_J59uZg6 za)itv`#|60kfz;E!E=;IEFeGL-+2;5gwhY#S8}j+pjx3Owu&(h{@pN8@q@x#A>Uf@ z6vS9!QI~E5fEkk)l`)U1mcS?W_rsDLfenX7?u!Lky~PxQ-i2Pog5*7fdvSzaZX0oL z@{JKA5xHOsur0oHTRJP4GKT#ZX1#p<5&1ChrMk&-yxFD)CJY(Y{PpuDK>AF2)f&2* zK>qU)7N_&CZldeM!-o@v>17iqY!5!KmofoYCrF|Ece`YrGB5TNxGf=IGHrgfHZFB_ zKB708ynfec#ChK4T_|~J7Q#U%1RhB8L_Uu1z-Z0~LpI+-7&pHtz;#gV%Yg@zp^Z39t)f>ET-8$D9U# zvS3sB_&4o^Rj+SzCvzh^%l!x09q?ajtCXU~Os@{yZ()1+@dSmBgui1889N(Z_o2ABppZ@n}3P-4(5LcC2BO>EG3pnh(g~}8! zE~|S_)9tgN$3wQH{543=$H8nFcmc#k(MjgoNGz0|x0>cBmfevY3um{q=7eTz{-HI! z1GC@NrdQ{YL3^hKO?wvkNk%^}O0%^`yeR?+W^qYCSlCasZW5K|y9lYmY+HKSeeXx_!bBum_`BjhHa z3C7BGB0Cm1rEJgExf{x@b$x=v2Fss_rL8~8ox+sYqPA zNPOyUjiC-3XYaY5+cxN?xK`K(c_2LU1wiTWME_*n9XnapTUg$!;8d84~EFA0efp+!}Cu|cbRHrDWn%QtP zZb|jdTgUnV2!p;l`Cj5@sM$nKZ8iL!5r2LBQx&fR>%-o6sBZE?J7Cxyl5g> zVo1$$V{l`)M+RZ6yX`-gYiA%9%E@isp8jxwKyDMUkcp~yY2-=hO~uZz>K9UEImVbs zeRVxWJ&_p+qFruMLBIxT(iXR}rgU zKaoiauph_ZvsC;G;zj=&+8C@K!B{>$NH`?Ci_#ACVG5n+3!9Cwzy->^Ws*&|94N#W z=*xQsM)b;qV3Db^acPpR`7hgaW?fwy^nZJue`Nj^q zpUda$0=D$epXS|>k*l%vmJM$IX}J(~XOCJPdt(Shpc?vmy)xCK&$LqANw}h!qeI`GQBH8FiOFW-qAl~t{LhI8A@9(aD)NpZPQIFyvgO65c zkN&YBY#;H>qQdh|)xkgg=hQcDXM>s13z10-oZ!A&IAm@I!E_^ktWw?fqSd*3`a^@t zy5uo^Wv8G$uMBP5oUF+tql`8Azi%ZR=j-Vi^IUD$z(LWAKdo=WT>#omhj>L$or2@g z&wKMaWR%z^6FZ_!vysd{b~n0fyigWhq1or=X8yC)wBt9>MJ$6fAqmA;hn)9g#8ZVR zO6xe+=PB(O`)}1*;ON%NHm(MB88ifT&>l>(5qz+rtcoK&Tx3J1#QI{Q+(vTNDHh25 zm{gD|q_3`ikSlOCx>sdU^Bivf>S7Ngt83pDcDz(`o3-BIR~FT(3<2WzR3^CEvWAxY zv**#b*Imz73V4*UEBinPB4vJSNaTZp_?f}%@zvw(4~c9WZzhLxng>WEp*R|}2?lYk zCkr0C-wHx4?m2@@(%1J)37Hsal*JeS`71G7fwLS1^jXtS%hKZ^%l}|FCdOGKFW<1i zizw)Tl`i1idn1UUj~*jQ1o@e&gicKgTk`5;iiRz=A^vQlZLIH3nDpub*kY6szD|Kz z{8>QI&_w_`Jef}YK*kBTwHRyC0yys<{M_03|6`Rx9tAJuYH3{U%cqol z>$d|1`3c;(xUrN}1p((n)?Zy&XIzBbqydo!*1Hx|{Ko!qyANa>wgv}mPuj17p5Z6U zej1r`%!S<&p-1_)XKS_EKu`B$=f=Wb(p1`9%2?GY+uD=RQL5=T*ZWOqta&#XQun){ zm1ow9s!G3x)ww_PaKsR*RhI*fYz}v=sf3-9p+e0HEjPJ31&vilzG2W87nhkaF38`6 zDP@9xK%SPY({@40)$LmVH3Q0Ga>zkylK22iG;o-fxpPWM4i(&;3--La`2kaSJfCK1 zE#MzGgqx^fjS*Z6TmHkB!LrH=iJ03a{)&O#a;r64_GNPighCOh&`rYPpG5Y|M+yk25e+Tt(Rw6c@@G(`0N2ikS> z*ZWG|DiMooRetLqBe(?AtBmF={?oh*+7TFAo5~V*BVw^x6nE>i(nYc082pc_rDn${ zRUA)$EGUga^`zy81lr+Ri56vf?HX4nZsN7&bPh<;E_1k7CTIy6G45+C{4Q}BX5gi>=K=u??NK}vg20S zh&;vtr@7B_{d}rjVK>D(p~VVfj);bD7H`2G!4kuk!gk5Xt@LA93rMTkh=DNix?&PT z7%7Vw?d-O`wU1wpUKP%n+1fVN9QmQuX3sUVGPG=%>yyr}`<0?;c>ZygDE-bs!cQ?eCNQia+-_1rxFVrx~i6+OO#`63Gwk#1J0#PDOA5ETkW34I@%@4t2BCbQ`iBSX6BFQ^u*;;UxxKx;H(MiOx!cD( z`Y(^Xt9cZxE>n<_$zJ&1RrVY z-`U*NIi0*g1TLf2>K5JK!HND-wL>!B&Yjs6=-FxvzU=@LW*u@ll(432!DJ9MJ{u-e ziEU}-O;-3f%eU65-QXvR_}4X((woH^NiBWK<%M>_Ptl>ij-)7I4Q-wiJv_|!x-b!c zBNp3?RB4T|4z+%KgP>-Y>g~L2>QI*Jb~SUC=7*&hKPy@qdLQ1q)uO0>St17p7+?KA z;a3W=B3U;@^+y^^8cVe{Fmp@^@pEqqXCnmcfW*c$oawSpJ3IUe#ocP#*Gp4yi3GJg zr#nm~9-f1G?h2#>E_x8cqfQZhVRw=qBw^Gy*8jB36MXB8$D)XfuPAFAU23&n`=EHw zBK8{idd_J+`_;twK`=v}<^W>n#~me#@}^DKvzZINe4J=+SKxE&Y7bBG_Iw7{}M zg{lskuD9E|@!Y!_16L^8qnW?K-6q;^91(C2ACmzs7C?pMv0m1+aC~`MSVZXe1xTYf*vAygNL@ zrG~4m9N+R_$v3%2lCazEVRgYk)7jlELSxhV` z|M&7@i&mu1#jPC|D-UiPc|(1~JMnAjaw9;LiLY))Im$ za{3@QX&|p+j={q?-;Fl}jaDm;;1dMoD~BFSjmPTOTn?|^qZ0l zgG>$YM+5`i2oA8jQA#=v{o#Ux-Z*(iBsP^Gtza>}`5@EJV%L+V(5CV`=fg&lWlttt zt*~Q9mhwk7?II(lzdOfc=66C?NH(s#{GAxL63SI*5NFwqCm?7h1uLr4a9JJy@vPjN8s$P(3=QD_TRTP;N$n(Z{-S&PlJe(|jc%en8Lm~|?Y z20n1sPRQEoGu3&a1w*6Zs${|*Rbhl7k4&|MXf4AM9YJTnO1PAK7CkslGk0>I&0tpW zA{2HzFcUCE!imP0bvP{5GFP~A&ss+}mfn=B9&~9Y8fDwbCxj545pD#%b<{g7angsY z)?XJutpaOgbtXViU{&PTfkPyoQa*2mOovlH4)J-5ZC=06=F20% z5q`1$!fm?d1PCOHsAKbqlVB*t2bM7J?7cxX&c*`N9O^}iU-(Vm&RQJ!zW&e@y-Mo? zMPRH488-s5M<}qyPnT4AXkY}O+Dl$Uj^{3{CUzs?P{iRW@lOp3pnY+ILCLjpmH-La z`T6caFgw2fXFNHy)&j6qJpbF1*YkJgF#traGqz~tH_@Pt+1u)I_&th#yVs^s^r8(| zms1igocVFNhQSnL+V*KX7Pqp8J(&09YgV0v>3xJ`Gm+xL8fJe zcLDFb5HPJMyf&=y{n#Qq7=bmHv2NOj8!dG}bV8=pah>(=8v-Hwz`#gJGapkzW+^_u z2FUELWQ}=pD^!!B_)cj4iiyZ`jZq)29PqQrpGN51VBOuJ1pdj7*ae7-7C!(D44baoVp4-m((xzFz>1$(k{PDL!)ne?-51PL6EtSDwP4oTd6+u}0Q=c_`v1$kRdeqtDJ&2M@n59XQI7<`VsM=QCVA*WwF+HCbdKUHcQD* zIqx$NKn;=J5+X96UR{d8?k<$#wf6D@{~4P+O7EV?9ya@%k4mGnKs2Qu3UN^X z{&pqtV9~g6Z7@cSlTIr2V1#Cky+At<9S-VaI)XBKvwtws!-Ap4K)*h zFaceoC*1Bo0<`qIKwfCTJ0KTN^+MgXC?e5E zoaM=!kNQZj?Pb8lgv=O!=>UjSL+tT%;cC9Fx{a}oy6M%}8D=~V*y*+fCTWyys7*S( z>InNgx3>dcpriAD=AJWsAS1(_dp)2(F7Sjca6!@!&>-y>?IUkfGvNq%L!|aYwmAg(6ArSt#U; zd638FVv`l1djD+hp> zEm55Cm#aZ%fL9)ef_^sz6gN%h7B%>}xeU|V{46b;1Di zR_nw5w~viHXG-^eL)qTL_U9e+DQ#M*8A(>7o$yb)e$rcL!T*yLBfpab*k~esg-g6Y-qKP28%w*$}3};N)e$>LGd*IK40>*s&V3e&X64y?eRX1IGamK%r z0tz_dY!H!4taglxeie4h7aX?g1!`(c5c;?y#IX(V?|K?zmF-s=!_NOxt5wjiS@aIs z&w1z>s!Kn5l(Nx9?|aAKY;ybGy9|`6#-#CcN;3)y->={Yaybo?1jYE@{$x8I5>Ywc zcX}a=bEWWEA$YByi1p6b6KT6?yS>oUzTnD8{6M}`tk$BCDH|C=S75o6I$I{jaj0AO zb7~{I>uDl4fb^*2@Lag4!?mj>w|alGmH#%VcZbsG#~8-C(-5DDB{*gw%mgiZ77^|Jl!{WO(jLb}|3CZ#!L;L6)K2)eQ3t32aq z=)l&DRUCYm4b@o>0M+(v%Hk)Q<2Dm+*P44sh?^^7auf5a6ioTludSCeoEfB38_J#m zD=jf+6Ta_qVs9dxK29386}(-Odj63Hxt1||J z#JZZYla>CFDdKMZ@BeH&g5Y0CDZW*w}Fc1RoQe zO&DQ#qRlMCLsX*!VI>Q8wSv&<|2ma}YWl#A3Ed7N0OJP@DONGyqxum@NaxHRr2c+? ziyotlFTyOc#vQAFGRiMkuHat^MtsJ+axck{N_921(P8Pkfp>d?AP5VX*Z$ibdnV9K zM;NK|h5x!UZAv7APKt#mN&l=}B{%}PaQ3*y<~!M$Wb^PvQYWb!AEq2S=FlbtwJj*o z(k~SlX#&8!>W?yFxuH@rO;U+$mSah`;`i513}2SN7w#r@^Q!-38{q!x%-ZRYPYHZE zLy&};qgXiVY=UN`e_^Idh~hmCLgEPQG{^@e=~1#ErAtPg;4GMaz6>~_TsUU&5v4f+ zOr{0c@;mns^K=NfUs5PK+93HoCo# zAJ=cJ>AWl}hk##K%Av#mI+Q0lQ_j+BhBcVArRDSBJZLAD&unbPj4&5T5=Ee$No%e@8Xcd;P3w}Iz1xt&M@30t5kg5oySOao^IIMNd{u(h->m0ux~_GRAM6-+hR3ih z4ti@=7IO>Ew&5kYg;EGR-Oo`HF0IjV+oZjdnF=*Vm8dLXUme-Lik&+UrVi14AQJGr&!@Q%g^>9ON9?o zRXPKfU_0N(CJLNZG_AYo(u3gkM+^M>0SS~kf)+Ozu@*;k>vumsyQyp|wG<`)TVFyN zKP^smv3l)?P4aDf*OIZ4w3IUJ65af^s!o0B1~NK4WbJAj$%JEg;cAMka|w;83D=!L zDN@nR#j24-?#kdZN=W_jT*;5HOP6WGJ0V7xvrcurCa2T?);NyQhnq`SX!+?hkF;4T zIav#H1oLKb_Glj#E@!YaapV7zobxRq)TNmjFU-J#6Q!we7Cp5QZ(0FS#BQ?z# z-#$~%nE`N%WpjL1&Mir=W&?1}Mt-OIjSAE3l+W8!x6?K<*b@fJg|B%67@N{=x#c3#s1Ep-nq{cH*X1MC0 zVQg-*amRKBEkYffyY;R^sa>EEAA?R4wD$CwHeCYfj+3w?quv1LHx6tqGTwIzRJbt- z?s>o|8^s7kFuz!8x`fROSq*CRU+%KW z?B%OsdDzI{kvSm^eRRV*0{0FAFhX0kcx4NoKoYA$G;>Q;U0yE=Bp*seccm73WkmHW&W3reJ-9qYw)`C}U*7MM{&(f!x`SvyMk!$`29 z;A&_Jm9e-mi*d!Hc#B;4U!98jnDIz^hyOj@&)j;XUByKwJYt?jjkx+!s!pcAmni&C z3g+_yJTE;^{{(a!lm2QI1O;x=rA$U8dZ|Y~x=~jQ2t845Ed;&g3e>>@=O^RO$;V3C+8F zzT2$~yNCNK3O)XCFzZ7(w^n?2-+q2nY0&P}Tp$VIP5Aw7+3aXcxbZ({w+}VajGu}X zFqxoevXnFl{O06O8Z6?ht@H+K@%b2qAv1-;rQQbZS=3E`$te#evl5>p;pnu*o%p}a zX-tyq&@rGpBpLW()4g6HJq#(}6zH$CYIS^27wK0}1@q$@-tgrz{=o-{z#SM=9=WhVQ1JT11yt_B9m`8{?EXJpQWh?fx>_KFDWRboAC4% zJWYwgPk^bH^WJ>$-$Hn5fJg*f+z*mesufHvmxCmRLBAYtR3K5+pR)ibhiwGl@y}Cx z-qN#lQaHT-Xa4HLfkGqy+wPAXjaXX6zbga0B_-u*h||F41I&6S3()%}y7a7;y2}|g z^_-jyG-7T>1^xOEz(A_5MHF@bk4DUh39rL-H}(Yb`lpZ12XRF9#B=I;@rtuoI#PwQm|LA(46ata2JI_rz4HO9 zfJ$?gYmI@2Oc{t0Z%LQJ+wq@uq!Y_P>#JYgQ~X(NUO!~y$=>MDM_Z<(y{7i7pHmiW z9dA|JnE;;gu9I_O6NLgrrgg{vqv23i*KtQ@Ar5mJEI^NCidjDVeG#7JbpL6!U@3q#wmVgWM>PP(ZJDkwZFu>9 z+4{Dx0)GVNskMt=nnV&%(9Gx)K7MvWKtZuitoleeA&wboddVo!UCw=riXf%|34)t0 zywMZ)^M4nZbNs?up6M@WT;41iNf`sv|fi`)mM7e3;FCty}myF?W!mvCE+ zsUwG%fqk(-i=)uU-D9fN$j$amWypfYEs3qpH+^Uo`Y!q55b*UW`N&IxF^lo5ZoQLe zVfQ~`Se)_JNiU5qBB{YUWO|MlYKIi)5x)7~A-E}mh*Xd@Fu!~*=thNRcFCsvlRfjy zU3H~n|6iSUMpw4nqZWcow~%|;Z}ykUi)nd*NS^ZR5h!H2)H%305^gcHJw`Uer6E{y z6uQWZxm{Vw4%-6?tK%LUry&yw-mYn)VolitSKD!22&{r}3F%)UNV5rHW-H69b)m+_ zGaPRWn;ASxmSh|&H1G(TGs`~#PLBR;gqoibNq4U z?s2~(qvC2%rYoiz|b1^j)ah7WTYahZ?RMQgmV zN2b#fswl(qQ96zH)2r5T3XKKX@VaLgeg}kYsIjpLEIU`+oowT1Ti)chIL6)HTt2h9 z8mSYCLTXVv7)dnvcX{95A_#$XFof&<*FN*mqQ$B6I@IOmw(1NZ1SHks_Ac{4K~Eh; zVphEK8$qgWF*J<{4ehZy)r#-$M|p%?FT(8C{wYqoH8U*gY=8S>9mXq_Ew}r(g-z4+HnD`NYKw*J@#+wqXr2L;mV5nuEWgrJAWBBEBV|OmBV8rbt@I` z%g%Mliw^;!p3#XKdq!_hIveFaot-fR<};Wz3#I9?2q8gidRa_b+X5u$BS^J{pzH+x zC_JVe5M7ERy>vPdLnZ3w3wX9D*8w0_t1s?O$n0zbAT3_M+ z8YF3diSh);?{H7LsK%!+|EYPWEt&0{ukHzvjw{=5;%c^r`sC+2=M!MLVn{tSjqDPf zlFqK6fIbFOFiKxkHo9Ym6O+TK`sj~DZ47`6OZ7`YAi)IF3=w{g)&qIB`8F2#lo zrSZPpXX`|iFxY`t+x(+kb<0VG!AG6O)wppeuXYGe|4m|p-7 zn>vAv{M{V`3*rveO(e{;M*X_~!}Ih-oH4$!CX4|+L-%V{%C=c6TmHA|Uf2mbm@piF)Ttd+>;{6*ouA&K$6Gf*J zF08RFfU?V?TUm}t(L)QsaaHD6CRYl3X^-nn5w8#BCqyOJ1icW`G1EXq$KDYdw`WBX zC8!*%T@?*ZNk%8l3pm82?1+cVY~+#1u&+&JEr{(kT+0BYE*^)hRy&G7XcfJp77>u2 zW3hmqEYbDPcQeh^=MHk^#)Y>Jb46@8M-|GWe@~?&UlMuuWydYEpY_$n5`P7O+0)zt ziXUk`cyAF{rB#bUsXxdUms3a1*{Rvz0)V;9?y|yAP8apL$>lCMW0D{gk%tlJGw8Fk z7=O@9iIScbbXwf6{yUw>ic(gXS9|Zp{-%JHr5Vx{J>{6lqtJ$c$i^1+Me(FYo=E@! zw?>a4^B>}ZM(7X%-p_-jYMrk*j>Qk+4qgxh%>Murz4EwWz8^j+cB>4+(KtaP6Oon{ zZ7i%K%Spjbn!wq|-r)4c1;?YRe9yz*=zA*Q+;@q1ebL=%Q-34O#6)Jx0~8)5B`X6N zTKpl}W4bZ~hO{_y(>z;eUDmugn8Nv|vhzli!$EegJ)r#wo@9B08PMh0*PbgCIsoin zMPn*V+Wo&0h9B>XKVGXTEf8O3%+UejV>ZdaV=7R)lACDQWD%)dS@;=xMXw$XJ+aL??}qLVj?O~5bn>0HA43$}RjzoLiE9YcXad)s|~Y2>id zs{jYfPFMAudC}g}i)_;wYKkMx6K&t|3Fq{ubH2+n*D#@n6JV(+3kog>4Up7IIWEO;V zcK&)jjv(Qn1+But*^YfldsV*3jt36Ds9|pZJ8zF2X*UX89#2!3)JEGZq5(sB*>B{` zvG)hu?QIY~uY8wH$>E(1qJ+Ue2wn>=p~{)U{Bj@(`;9zD!m=J4Q9@Tk3~Wu}6RM>- zLvj*`{lePeS;S9{0MJ2;Io&D>33++U?3?ube$J2nF8X8;?A}otuh-Mn?ozr_+m-y^ zKe&?)AHH6CP<O2(M$)~GAzaqmJ{Dj=>=JlC< zPG zVE|vc&(?Sz0{K&v?JvxRfepBr-{bMdj2Ze^09ax(FPl(77OXZML?qsZDRz{~l$8pG ziv7ElHl-h7bwfnkHw`?#yz7x%jd52z0z{ZG3A;f)$lE+LuOtT)`u>kP@C;fp1ixwT zeY!sPj~5V`!7YCwUH_Ydm$qW{&4c5x0c|EH$tCxc{6}J!7QjYdSVi#)ha#ckfIIm0 z0QM$l-l^SqtdQS<;zLE?B0?~*w)&&@+dS}(@B6x1LEridx+FxLNTijYEUsX6GTi^S zM4wPPm(S)iD&%Zql@j9=J~Ni&7~X9oJR^#w-^22mZBW8;pu!5)E{gbfAIJQ3%4(9Y zFT{&rbpupMYXZ0o7=rGFvGHv6FUf4QQ~HLDPst_|1l*~jFcA}L(KJQlLAp^jRdf(O30|ffrqXN&I3XOe&7E}6iRja|z zZzn`<7GL-bZY3rD$6RYMOVBLJ`(uqau}VqIcD|va^d_nx_XBrPT%n)tGI^Rfr867e zgY#hTH>h_9vgE@jTrnmjRP5?j(?oDMH0S4nV*k3Y;g7?`LE|#ZPXvejMVo`vGMrak}dPL-yRjkaj&HT7RkFjh*OGF!a}(E$3N z%~HPV-yLx{dkf8Ou7GkZ(I`-qUnYE(c{JasX6Yp{pE-fSfQ8sZ1wpnJcywOvinavM z^1#;>c9fTg%0R@B2u$Hyj1?iM63ysk!m2SVMr}xJBsKDr$~lO2oFp0Y!vT}5c@}fo z6BcqMR0+um-yY_k6AtvUKrBbC{L+}KN-n@@LNtr;%dH1&)X?!8yh7`*%S z>iZH9@T9piAwdO|TRaaa7NW4#=BZ#g4*zwM>p7CUu^-yZR2nPJr;2zvMt;0#Z@DkJ zWR2_+X>ZS?roL!2D~)gu6wPgnF=}#_aJ~$i!pwOn?*Vq2i5-+>DXHSE(8YiDRX<4H zKmexT#Kej|UGwp<`K2c1m&Tmg2KyOBz~_a)^}P^DD-}mY&!Svod9ohC^fgmX!S>ge zV;{g+KBm^+Y$h`TLPG6V#Pv((PLA*1M}~ZEQ_U8us4vT!qZaj6$0TN5>Qy!6pZJKpA;=g*i+_E zP*9BTW)B}P)8J$_XfJ**zXTne=J-t5tR{a~;2o|M{rN^_PidVexg5G6J~} zSQg<(LSA!IJ07ks-1;|hin9DV|A@rPAr0-4(?jgf7ZfZKEyW+O3&v%-fj6RTf+GJT z2GNZGU6tvnU4+b6f2zBG*#*)DkTskC!erLS57R2s8P4uC zuVuHU8b|?@1+dA;W9vCw&=JjCX6wP?Y|UXK0**_T$qo7-YDuxw3>BBp7et{Z4NibF zxQfFi$jK#bcd?~$@8!=XChC>JLX|I*gh(QE(!u&P4GD~#)n1s=88yD2 z9e+1Z<9{V#H(g#5>;;jGBw-i*Rv=F=`LgpiL�**|QNbZ&PY4FGDb-niv=Tv2CNI2qU3_lN ziZ*?t&l2|BIpVeAH)#74oV>({_cNNf-w>!W7B6R(`KYqcRI*jf*C+2bmnt<#PD!gq z60Bhqzlyy)8x>D^N-BCH&=O_$OP<_m8VTIypRl;*8nc?84mc~JSnk^+nRWFcMHZnq zPr*LMh<_2A5KV{gM?hYHWAi2nF?tLEE<)mc&#b+pk~+6_7%m8(B~ZVU6mv~c>$XWE{A$!HDP##rWBewmZ@G03S9*skbL2RjA2 zAv;5AUP5N~lT5*b@(xVC@EoXUJU$eCI+2qxpFu&5TJTEfr%&o+w)Yyws+1FmIxwIEhos(|{BN-owTC-t*?TU< zX{xtH5G0aN>V^-}xAhq(z36EBc+$hy(bX~2IH#x-QSasSs9wEKC76zqc0P!EA+->M zHYX%88Nb+ipMAussZki#?L#dWKFqK&E0?8xsGtS+>|2F*1rAX<2Nko=;TwrTQ0a2H zC7%b9PbotX`N;I3zAuLU8tG7&{-J^i+8Xid@N3u_>CzZgu;I7~j3fp5V3rD^86^XI z7>XI}5fShSZ~Ps;$|p$>!QT?mittGMXSvtP0{$Y)zGu<3l&SN0XrUK6*;75%=SLZQ zmGQzM(VBGWJtlx)@xnC44J}@$(wBG`hn}-Jp{lpN%mr=GO#&)VA zRTu+1EUlUai%??5V!%xfthxCwX)y6DrVfRtq9M+rkNx}#lg0It)J|0*c z;jD<&leZnxkOPn%l?3(l$^U#I_nl{`-=dAD@s}1x7+vZ!T0)Za9uRzPX&ZTqh@gmE z7ys;26c)1li-pu%lnwk2IdzIOCiH}^XKUZ)9S6rrROpWAY)J6>a)*{WtmhUuOQ>d% zkXeWUY>v`ymT7JT5Not?WrE^xpwBqb=J6cS(>JuJNSrrP?GY7{Rpk_^xcKCs43$G< z@2{7@#+l|;yG6XjMr5BIv@n1ck7^oT&;}-3L_$La13zTV&6zpTQW9WO^estu(8GQY z{7`t6LBVe)?dwDUQ>ZQ~`Z3$+WYR!$zxV~&*|idX(t3e(U+Mzs>}4Bkg2ZG~B7uER zn#_OHK}gZu)+EM+m5xB(J4HGC7Z)PI;CzuHg@ZAW2acW*DkdJT{vQN$ zRa{9?S1ovuuw>5Nih%@kmGWn_GD+Oa})(jm!xp~ME4%2)H_pE)-^$S(UjG|hb|yu*l6G~fH5n6Qc=@s{ z;LfY*k`am=Hz0)H|D*rGiss#Yy{pq{=9$W|!F`Psv=dh;QZ*D|n|s2)W5}=(4}&VCf4slsL7{A$f5m$Q z$hIG;_e|_OG`9OWXfs$1RbP&qXXiQNljqF*tN39Ca~j)x>rC>bAL>DQdrw7)LwKx&}+ z8XndH-3o#oW~go?puw8T@IOy@OTm}+s@kc!AvHF?X`eFJF=OQL>m3(`J^U~K{+42} zK}UJAc@?XY48b%*z%y~WH$VuzHr!G&Ka}2a&52a@_pb$7*cz8iF*l+o)TD5e5$oBk zT^5MlP5KpFE`_`r-yCLYTh@OF2~r{(M_APv6;HpRJ0nc*ff=Q~eR~0NkhVFpKOge9 zu&#{dXnnFtZ?i8)PW;>)QEn=fDSw3RT@2b;RqK!e8<{)Zd^Ws{0t+3wJp$&snlNX7 zYh5(jRFKCbeNnF$ZMMDyloXosye}$5s{y4_QZ}KPz3evp`c%G7&f-wS?cJ-ebY!B8 z+{c@lb|e1Ws9U@g5j#bfo^)P;n_81MA0)TZb+)WMDn0|KkfqT+OxY^q#3KJZB^6iN zQ>rdVK{sUk=*IZr!s&h0ZI|Dq@gn=;v|cpzpisD@Ji?$F5jK7QiFVtITY4T^(FjIY zh;*t>BrfhTe9IE#{|5aRANd!^l(72I0cs=3vqL$#QxxD|3h$bevM+es{{d6*ynKom z&7~vFPIVPd6;v$Lj)0ecWhvS;n~UzMt1=hi;SyAgw7r(4`H{%k+y0lK>%*x}bIFYg zreJ+lMrCT)Ce#kb&RpD2(dfL|$)+s5l!Ho)839}Kx0gs{iHvs!tztls5k)>G z!vi9G$uSyhfQb+jmP|21uz!^{BXjd zKYH4gK%uI8M}1P!B$~hm|D*e`6%A0f*_U|VVt~%_92IwFAKbLgU!i(FqiIUk;5W^v z?G!GcQM}ISkSYl(Ft>~t5v9aw+Jpf)T~o07Ij>0LGuD(*Fy-7k98h)BsC?xSwDL_W zS02SDe}@iJw7b`mo$de^p33c3sQhsGKJ8-Zd2$PdX8u1D!omTObrW@o8=W{*Jo>k` zfM{pLMME}dW8hl@aX5&)lENgfjLJ*eE$H)5%z-qil>O(i6~b($A^x0)8VS0|a-?NQ zH|u1`@9q{+1atx@1hM9APC4KA$TP}mG;g!JZwR7Icrg_Gip1p%QeWeXC1e5(FuMr8 zY=);2R`!E<>z1;tl%j9oyL9Lmf}NeLRP;3I$gh=Y=)G2c$d;jz-&dysNM-?rW2Y;# zCFLy|(PCTECm{}H9p47AbBP8s68^+rp;2AcB5(jyk(VW9qL`0IC}4_Giha~eTSdTO zA80%_n~~8ad{Xqv||Z+ zYnERjK@UG3znOIrjm18gmvwACkzGdJd{-tBE6PyjWt=Sckzc#Bm^dE+nb{TNuGd(B z&{2r!u3KOk8`^gv=(-`F^f#n^-Tc`zYEeCqoeVyAf?z>Mca2uk2vC!B=EBZQ^&=b@ z9-%~TU$fix&H0j;-wjhEbobqko{$eutQwVtfThX7ty5uksUwvIsw`~AVp|7caj8kWU zJa<#}G-*qu_dNBbMr;}s=D3^mBwLWqTai%)wi*C{tJj0(E*xJk@tjg#XbDd%M1d}LHm z2mzR!ThQ{^JGIxG|4KHsc ze5Dlkl+L0N_<7X!55@52R&V@P|xV2cH@s6>Cj)6ra$iLxI!gDXxC33A$T5;LiH90s6)!g$Y|v`7iQKI!W{|fFCK4v zTdZIbu?0U#Ku-fW`xeRmHv9mTIBQjeo36W-%{4dCn1T!Ho2c1pf=(pp8Va!#XQkzO zjdE*|Qz;AA3G^?VBox}bWk)DyS*&r(6NIDZ6lP1G`T1u%rlu#b?fGA8fKV_Nl*;cq zr}S@EcW?J=GBy6LF_MoRk?bt}Xj%5#Lj!kUjPV`*zyQF`w6g*pe6*wRbAPk(X}Y0~ zqI*en!SQa$Uw^3o5Q0Ne7_h!nq=syS829_?77^+Wh0s78W(rUE!{%Hxz#! z;JqSt`4Faq^XDfbzT=V*N!SpjxZh5F>-FFl9+zLzvjHF4`8dwnAv+(IjHMPTzHo9V zgOt0>*0WV>SH_Z%je*ZhMX~SWKhLMgM_V;2ny<2cn#1Y6aESZy#XmX0<2y6NXm&T9$=K^l9F9>!2!MeHPsrg2m)lAj423(^!Eby}cSOyM|ieHmWx8a{}LC`@5sB+-hG zIR3qLZ==FwIROv=FE+7KbbtQVqjLSVjTy$?1_=YoX{Gj`kBPnoU*vbw{0MY&68YGC zwBCL&vw$7(iMb!INbI;hekL#}P0G(&q;t8hO4vpD)^F(4O62}@AFS`hy`Dh=KSF1Kl1o(AV{8!FP*Zi}xQ3by=SD|I%nxciGV_j(rcD z%#)FXr@c2E#hP$)v^&o;CNN$gtt#HuZCUtBe<$E?706mfM}^F7}Dec}^WkHyl{iS)&U z57(`&|Ek_p^5q0xIHBT!2D93z%~v0cdOF$)D1w47P8eM|AG(O2ULu{QT1Os>5qB)5NVK^%3>7C6>Cr8 zg1Wy9l0UEOrJ1^#Cdi5e02s8Uef2Q>>jA=L@ooSI1~tKNT*HC=HMKqHJO@TEWqRZb zqqT-;XOv(opLkW>hR;Opz?X*YYR<4rCl|w1n9TI1^TU>GlUyDRz0|4{pnlLS&CS0) z1*8eP??LNgMiLqWJ{kfPxUY+fI=LV(SutrqdvbMK=_CzZZQ^tW3CeZt3fpo=gyo#j zeguk^1ym418Xczn)w4HX#EFeAw-$;7trjEq>&kPXiTiiBrC4Qq|R}pIra6i$FT?tYFG*f- zs!n+yIg`}+bN)C_!1`+ox4CHU4GNr(|E7F}2<3Br(FPYp;x$8&4Et{Z0sUSSVI}G} zIy@CuQIw5#Da^V!5}{-Kh^LVTc-fS409lLUW?K>!+1Tjb$x|ANW8G`=<#(ufdfL?_ z_$fIGMIoCX`*C2$*f2A|%%mkweo!{xSbbT;dOQa9__H1$5y?Q75R3FokIvzxM4mM+ z>^*P{9AkW=>{cU7xkj$<;udHl=8Y(URa1jwT1-Fl7|c3+2kCO_amd`;>zJp;N%CIlI* zz@~D7xJViyFCu1^H^s0oF%B9sD@&QTl|=VnizlD!t5MrOrfMlx^L4bBS+;_#9&R2&f=OsG@OlpZ`;O7ah(0Xq#lYU@H2+Pdz+2|J z+4LtO3T8B{ixl$nkNd?-7)bqV&`N(3>|Tp(D?s&g~&pNX{k{IlIGIyi1$)oM37 zE^h3WRq5Q*!U$<{Zhs^4gAr2AFG*eyx*Pa0E!3(*8JQL70s*mGc103d4Z*CNCx_@9 z)R=+CxHci2Oz9p!FrR(%19mY!aSE8Mg%`mPb(g}_I%fpeuupxCR6}mgkl@j5FXv~` zLLYV%zH}p%pzaLHKb4kgQuW_f;V%lPB$N&MpsDP+zCpg8=e$BE7Z}VJ@41`H{uqxJ znm_k2b(*dHMGQj&!X#D$pOvWjRk$`~wG>{%^_^@g1Jl&Lx#?8KyBA>;q=ZI^!k)^p zn?a%IEE&;wL83vskyY)*c0zbim5MJPQg*|A=}vo{0*XDGM9h8`6q86=RTE*u(JvNr z{g*Nem+n-9<&ZV%xKwk6*i7I^Y}tA=Z-M)fvSZKpfawb`rJ>31MH9Nc(6!IMRP^c+lLO}XPKiChkiT@2J zBCM=GQyU+C0*IB86?=lbR195i817+oQ`XvlST?4Qw_l;CzAKI>vCwkPZvE#K!QZ*v*umZlyrraN|Zyjd8*jHtIcGK99#uutSg%hzFDBH5x8S5%#Wt`-fUyWk+ z*NMlvlh4U4J-^5rSB6kJYp+UeAFsk>k=B5(&A@B(#x}rGkh87~mVH`~ zb5&ZWi!P9Dp*TH4demuef1JCu)S7=IfrfA+uDFd6iu8LBK|~MQIsC(>5f=4aK(z)N zC4vzW>}i(%!~dJ;<&|5l$;DtnyRoZwTwCdwb|u8fclb9MlI;FkC()tbO6%wfH@aX0|+_Raub%ZM)5H7j`aGi(R9$ zTvv%`Pb_48yQWW3`rYVXvxK|~i4KkR9z+h>2Po#t4SC7WI>=^E&Ttx5fZ?dRxXT_r z7wKxKFiJ)|vZl^NuqiJx;dhL0x5fTXdX%x1&JWj{DM`!BfOW=?CPA;I#mmdtxm|8gK|v#tH^}0P zZ7J>uN90*c#%C$Wcu5;V$tcUByk$X)itH7#&!e+}OdT&1OvYS0xq-a9IPWI23Mer7 zZ27oiZa=zd9-g|mpb4t9cT0Vu!JT2IN@1OE0mrGa!-~PS`4}zH34};csN3$grZ#q0 z{05x{OKu`x=^yzIUV9aozEq-a>L_w&N5IX&^+0=OfU4jq?(?8pDNDaa)+Zo7m{jwv z*n^nY|xXv)<;7N9E@!l#~n* z_gUJz6N+eL^i7>l53I`dLg;9J1T8ClvrTQ}Gd;5OOdCcOCmR-%jBCjPYxXZ%)7lck zUX^_Y@2DX0i!mM#K(#3FB;*nh(;v6HIo~P$gDEX6?1W+~zk~88XNAIm0?i8F9fIdhV5;}VXTuEB)7pw&+KamAG`^S4}P`m8xf?!`IC;M^?_mPs{lBTRE?2Y zRgMC=LU>8A=-2xW?_OP~Wg=O)FV%`S9%nT5VJNfUFJVI6YgC>(cR9_OMFO*&1G-%$!WnyafKNv&e!q2H{Y2Sy2A;LLnhs`?=sztkFr=WQC+`p*yu+|qIIsAhzVGnjBoVILhuYsewI%Vx4T zzRb=&jmOgBR|N6y_^ZZWo7gjTNN70dVzWRH=`4sRigqQd010eJ8WQO_iV;T8V{y5s z*J0Bj2{Wzb;&$9BA{KGRRzil$8|Gav@{Wq;-G%eR$#|`zYUR~US-831WUg=VhSz0v z$P@6HuCB11xyNYoMdUaw7%?xQ7N1O#K&AFo3=GnRkDOEZ8>!BE$UwY&eK$j|f~!Vt zcR`QBU9)$y(OT`se3}}ERx32bpOc>TWo07!-nZh9z1}HY)Q5O?MRhGo?o{Y=-&>TN z@#$Xg1NGYEYn0B#^^pql~0YSoQla3V@sZ&JJjrdcXhu5W= zz$n^T8P1Z7RVwG|Q`eWos_S{L()e3^7K-ejry%0C6-Ce|bVvTFnLRMuNR)C7j>tl@ z=D%6VzMgL~hlGi@H0%51i8Ibg>J1j3n9GbZ^#TKm#+2F%=|D8aYyI@^KdfXIew#6< zl3_4m(B$k29u1@wp*;0Aq!CU#{>mT;k$qZ>$yy8t?QRodJ@y2Kfjg&#iL&n1E7W-{ zB}>gwB99OTtL3s4%V}DD4~)tsOC%XgMr23K!mc%ibdk{62r`D7uJQDgr|i$tq#Ncs zL$FvGuoVOZ2ZtQaAJ;iT8bI~>5wG@j14BH*iBCyM$?oBbD1W2h$GLf)jg%Ka6~S17 zq9LOpv&Pw@ zP+CxX8X<&**HNv!qZT;&^7`X9(bGOU9Qg6&JC8XLN-h_;Q+kklh8z7o^oPg6T*H)C z?{)M-aO;!ix-q6*Kb28AHKzjt1yj7R0bQ!FP!q0MX1)#>E6(YJzS#a@ClS2}-Dwk0 zM&o}I;fyxJxdL^?4s;hRNR7pJ9c1F9H&r$3xAVa*YV3&Fo3?K?UZs40t~~GZwAJ zw=dxcNJL|%KZM%(XJ1FNCa<*PUbV0_nFrF7_V;z<_h#QZJMMgLjP*U9Lvw5-{unaZ zpi^l5&*M=T(y6N&Qk5xw3EU(s@4s9e%oi;@6IwsX5Ay`f6`Af>Hk&ebx&w>jV z)+(+&eXz6g#MPFjbFZFS=OKhP>V4zB_JaZRf>M}$Q<0>0|2}s=49SRQ?XYxjtt}imr9U<7|ExMi0=|+FIER4|Y#7peTn9#6G@cM#p z)ibPrdS2A$<-@MUdM?euW99A25$@~rI93@qrkTZowwkatmT)+#?TfS~dbN^RP}l>7 zi4FDjmk@=SAg)7Py7rTa-h0#;g*wB0Ac7zIq!-?6;$!5`a`oAW#ZO^;Z9)qWWA{D< z4z7ki2Wc~jhUtq;$q2zM16_SM^t9D(epDkO3xijTv=R$KzExW4Xy?xrip(zD)s)kC zWcup=^S$^jw-bUtEWI#n==7t&=p)~1n>>q+-A`#&AqZ^{tK6Kn@zwCI893TEkg>Y^ zbPM~t?uD5H%*$*CqMFAmEv1NUoKVP79QcZQO1I8q<07l;e6LU8$c_P!U=b{2vGt#b z=p1j@0%b8#Va_{bn#>cNFdGJu;kh|wXjj~hG=p>}(P zjQGC%4_4p>J`2PQ4J%kA3u=?v{u$V<)AzJ2J zM`n$;D!4U+Tuss1u1b=y%eNoYPg5Kj7r$R>i7o1k*Pp2MUmF!{PK(@r*i_*<+7Lym z#-S3{^7sT(9YRx(aSor}R^~cF|5fLSfHxF9XV_hC2jG_#21&M;S;kso|EC3LU21Z9 zN0RdTP~{L4A_erl`Es$;EN??MxXPbJv$@OPt2pEepuv$V)_%IP@{=5Mj%dZ~S2-#vcxCsa1TM_2y~*G6048*#~#aw`fk zsO!msxDD0eN#rRgD5UjRqT*SkzW5l$X|=Eq!oVYbDsCVAXcEsw>Ey5WK3S`*u|$J5 ziK36?JJIbZ{lbQlVFhMo)xR-wgr*4@W1<`)!~A zR3mFhPBH}KxE{7#?WD{a^n(7O(=%cI_@JckQ5#c#APk%H*tZjk8-qxqt#`dU zz=>Y#%>4G;i%*m)Dw)T>*GCm9-dS@pg#GI~xb|vQA|`bhgQkldz4_`0LoWRK{ltL+ zAG{aUi@aYu3bZy0Q*ido`-E3Zkl+}%HUGTiXWtDG_I z1(?8ijY4_FHS6Bil`HZ@*jeEz#o1 zZslE~I4dGK7TU*iRWK}D#1&LNLD<$<%i*{00ob}Rn9;N-bV;B8`!Ji5`tkR7seI>r zc3U$3>)lT2N?#rrbkddr&<`uRl?v^;hPPql<5T{&>o_pr7v4oajhLAmIoT!}`b*A) zX)ENn+kd*pQQqY5vOX?&^eM~sM1WYDpLj$!kA&rS=rZk+UESNj6I@3V$?F=y$a}Sd-RsfIM)a9K$}L8$ zhV1fa*(5Ws=?yuC&yza9jUmJZ4XFFxR3~8S$A;u1_2|90v0#gm%AUH5aZ?^m5=uZB z_GbKwETLaWY(@d|1&;2!m$Iy_0uYIBd{+?2y67)p2ey%Z1RZf5>C>YHZ?E)C zh}N7eqLYg)B=A2TJ#DpKu!9G`$JL)%dU7YEPSSnrt;3>SQn}P_{;SIMdf>v<<+5@7 zYyVWjOQ*N9E!IWYKJSAktG2S02>oIMJfe&|wDseg1oi9eLNZkCm`N|$ldWc^;{nz+ z5P1L5VrAv!MtXVyQhNJuNw~9u4q31?dSFIk6KGFONFcdCQP4z)Llhb?&~wFy>MT${ zMR8~tLF(mfWOzTB4){+ovxAMD@)&*92trQ^$NTO~#ZxHoY6MH=)jzE!yn4n#Y-n}9 zx=An=ZguWnLQaG9)%M3}$tF)T9-fRyEExn_S_Ez62s`s72&G+4ocTCat}WfSph12S zQbyyH5U;f?nV$;Y1I@g^8%y_cj+5emj+??}cPs&4L?+^0j||OFkx`R7I%3Ufk@Et;Zk+J7pMD1sXeX7 zL2UPaWf{Z93a;Bq3Ef|bHx^Ee6VlV!wsEUcIM#)GH$$T=d~Q1u1ILH zVekL!9kP3S*YB4AsLEzlr{o9*>9>BC?>n|U^Kzoq!>|a^xJh_}4{*h zSE7TMy53Q{ai`lOrg%JBEY4jbAl0wHu80YdfD)c}ZS;xiE+j~037;!Iy!l$eg@!GC%rtoW-y(Af|o zNjW_ixzSIW_H%q!TD`0@w=z)b$=)H=D-qB6!-GX2xf9c8gt;b!M`%shK*_*d?i*e) zIHHo&)_{Tt$?ffJN=pH;Iix6OXr?>J0>_ABmYyo1MZ(R$0msz!x2}aj`k&Eo$Kke0bp$=xL@$(f&4`3ZBD@lf=FLHl@BySg9?am4TgK#frX)S!(d z#`_Ac?FmM*P)xt}V2l6$uX*sQi?U%#ZdAtpaEP1IAks!OdyL1`%B5y4xggxHQ(Pq% z5zpr3s$mmCa2BaO(xV>Xh?5a3AMDxgr;AgrWwi#osbI*k9{q(-+drayECCcmDB1c+ z-3gDEhuF2dwtt%IB?hw&*P`Mo>yM_KxO3YN99Zy`>j~1ejk13VJZ{De~2Mkm?XRl#CD(azuT+i2*aU z+)|z-As(SW?&fMvhQ6-&IINq=JFV-f??&@yQnio^J<%kdt|D*ELij)od(yne?;Av2GO z!PAi8a@P1e9xuO>)a6J|sz~UT+nw|9`b@fz+TBD4BQTjs%Ovsxu7dGJSsxUe?^=Pm zp8guG=#NmByv>mj0!XT4QxCsVfEpa?MZ6z4$ zruM@6mL11}eR;)GajA(&I+`Rg%l}8#TSrCph5y3D07DPmGNg2e^ia|W64Kor0wOtt zlpr9DqJpA?NH@|U9ZGkHq;%fR_xIj+-Mj8zE(C_N&)IvQ{XCztw?%`YMD2@GBu8HD zsPzrrd@=3L7IXyZPw`V|vZZ`!>i=q1wq{RlG7p#)yl)pzqpPOhK1~sXwHh|i9TJ!% zy*%67B|>aUw9Z_gPz_DKJ~VwNADG_dPJ*OMe;Udeki4O! z_V3;jEP!QO?i#rJ20JlUXI z46SC)gtvxD#Gx7ma``a_v<#5i;@nV~`t_4TtyWOz#HU{U$fliOn9WBK*&Ea>AFyO- zQB2H$a0Mk&Ar2%sI=yqy)(23~cA)yiPau3F`nK-Y-ItmogN|eQ7vhS>Ro*BO-_zRb zTS1Mth)cj=p@<8L^GA(#!AGgDX{B{Ba=cy;(2MDFr1P@8Wwj$<{^vIj(f@hVisyeC z)4Gj#+;=lXamP~NSRi2E6?N;IUeIJXyiVNuJ5I$Q+X0XV-1%W3V0u}<8v#p4@@Yz< z8wG`+LUzlTKQ7t2J+_>o8Rq{P)&JnUiTWmXU}ZS?hj;*)5g!k(x$+zIXIvr0&!?OE zIKW%L5)+0ZIxb3!!uVA7_s?RTZ_@|gSLTO5tGv}}V$8kC&(AM}-zozSiQJG>|8g($ zrRR&o)`^u3pS-nNWQT}QM)Ed=rk*ovlkXMrayLNSJ^B9e8lbeEC4_3|GvtL5hHjjZ zM*jZO?!$;{wT|)tP#0hV)pcXNOvNiAkkG}y!uGp@=+OvLwk!$*Ycat_cJorkB!t`t z^F5RfVBVQ}f~tl_PO_4zdP6Z=F>4rI)khjjYlAYV@}w*W1!__{Ti5@1dFJ5kJ?I9d zTWgQb0~WCb8(UUjAUF>6P0*)^li}187_KXK#nN5fB|ELfVO9l*{C&i_7Q69_lx;w< zabuCaz$0}dSL0{n?{o_xKa{qN$QX2yklSmHE8?3^j&1v-0+`swNs%hXk2Wb}48>5o54tsY=?SeU|V9@z|j zNnpjn7pK6#>Q*-~sTE%;lp7WfeTxgz$oks2TTi$i`$`jQ51bR7yVnYMd8aY0k$jlV zOw1mRxj{1xem`Qdp;)2O l8|D-RPftM1|+m&d!o&+%dp>N4PZe2Yk6b12ue$Sn` zRlq^(pDUp;ChhU{UR^~+hp99>UBFXx#z)Uso=D=7BR2oNVFpVf736W2h(NEqypt=U zntok$8p+j~B)9g^x_2&AY7K=n6h$ra;F4@`T3}GHLxT6ntscG*)psA18hp6^+L(j< zDX}ez%L{rQ9mP0uy);NU(~x|VgJf6hR63%nl4Cgzx3c-6cQ)=V>PT=FR>?6upv)Z33S0QAw= zh&|nP@!}}l$9_=QoY4rjcC;ukn(s*96Z%T?toN!s4moViIPjoE$b=&cHc8awgJb00 z5{;hoJHU=!xAN9zk#FNtbOZj?z5Ja=G_k|q!E`bib1ub<%Q_RVj&2%nX~?m>X~P?W zI$y%x_57R9!N)`l^HP)!a>${I6>EkTdUPUIb#Wo%91*lp#cQDewHT+0b7T`(frTil zC-FBVRSBv+Bd2tRzd)zX3EZE1Ulc0!o|+n$&z?bnT*Uf2%^*{;yQ_dszaOP_wgI4H??-YsK*ieu;t-(R0Uc&)H;|xclFIZs z40Yo|Ms#?&+FG=o>07Z);sIFDVek{OWOQ4c+j0|Plk*&okC@Xn7h)*TD1J1QJD61< z3)&@$_Q9-a%jX|AFPy)2Z*Y5<{&;J~Ot_ftZyvH6A-j~ zFa`9VZ2hQvSV~|J^yIu9H>@I)u(3tz(KPVd8~GXbpbKC}MDZ>l;y^*DBua37b0d@& zrL7F1a3?uRX1RSsbj(ll*x?k!}?fa52GvH6rtlR&uPq7?J{gW@V za-fW~>kpF?7kV1*S(aS*I*d6BpjbT~=!!b5Xjq8JV{6Y>WL5WrOZ*=7Zbu9op>>l+ zd%fEHMlrXRTv)%~@J}26GtdnP?;DixLIam)Ii@#AfJn@Kk5~nMa$3Io^W3Bla(l=w zRu}-k^+-TooUlS8onfpLj0C@E^)-Li{qSo+zeLxv<^Ia2ax)lWKN&C=me)9ru`)Vu zqX2P@_TW3l*88D`?MOH3phNv^m;4)ntW6Hv-?ZTQ`N0xLGap7{E-K>l=g;3fAGby9 zqF>F4c%mqLeCM6x{b(`g9)(k)57Gsa9tt8$!KXWq`AdN5h}u}b29t0^;?MNPyQETS zK6Wsa=Z^0Oy+f83=_f+4Txj(1^5v9f0l#L#59M3jBGhxiyAW zHczCp0Z<2C;fSroiQGSkZ`Rq>WocdM__(f%xNTY-uZNNMUgq?mTg1^@)Y{aoYUTmmSsQ;bA!8?|Mm&ZKt zFmh+#*Uz$?eWr;3#Bhw%It^y*_M@7kM7-_#GssC9WbOUmTobek@M=MYotjXH`5%2I zWq+y=pFsSV)CQFgSh48cxGlCI+t$<)>1;Rd!+IEci~y}4PR?t_RWs`qM+-cZFbv!$ zOJy@V{dT8W0ER>|8x5SP^8#H@0Ak4AWR|Odjq!Bwy~NkB9CjRene}Q3-@jS~SX4v6 zVDNWHMJqH-u;#t6bJ}Pr7>)A1oPNH{wvSv`$=g`C|o(n{AE5sboq*NY%v z)46uMHRcoy-VJ13ip7z2yTG{N3+Q3XJ^of%DJ1;h1v;Bxjjan7;e(W$tFwmLE|+>A zvpGzo@XT{`OQRQT^Dck#K?i!8%12vc0c=9 zPjD}_%a@_EfY_H3Pb1n3oEr3QApNuV8Iyu9mLe%gEr#VFG8b1GqH~`i&%?gU+@4!w zD0tAzDX*VFJ2kR|JpV)Rwn#qZ^z8u)=|akWK!+(Y0mG-@He&YK8P{u*`1l22#W<7C z_vfY5X7-!!%m$FYp&8;n+daU-LeI^^yZN40j7Hs012{o&=$pFinWHPfT4a}{T>1>$ zAKGf6kA-R?m1?z!A_!1f7xCND>bR}d)YWybV>{K}61FZlWOpI&eGPj$|?1pUI}ugiF{DP)K!P1&fYoy?AHkq4Rp2{ zL1*^B_DVAU*iGpgHZ5FWnzyo-|ec(pu zpc3bV>4I3OGj~nKkZ0c0-xGd2f|A4Df%oZ!BTDoI(}JS&FY=46O9}*(HI6u{r0r@V+2mow&y|Zy#%K20<#@n9eNBhNrD1wwei4eWhD_n-f1i zTn-bh*1llXe8*=loRVyROdK&Sr^1+A#d zkAziuFO_pm{VnU!a>%Xq)G^gqo61|};7%Mdt*XLCF?TaW&`x_JSgG5Kze~L8D*(4a z{~J_YDWnBmkM>lSc!}ZDl9IVu~ezcp*l! z$aS{2Hj;6&*t9{vj#t)ZWaat+pnzV60CT{#n54QN2oy+*<7U1IC{XiSQl!ZF&9Vr(WiM-A zGLc4JRU2dw^C1}rIrfUJtsia&Fpd5*%UAap_W~EtTG{GeR#gH{rdQrv^S(5dh~v{? zo$(fit+RzMbiu#b*^f?LYku`m38HNq)33WX9=;Msh*1JJlFyfN!&{SOd3{YCe`94! zWu_kpgrWMzAeX%UP_-S0-d+ra6b2t&bWiG7ZvQDth2c?|O1`FM ztsFBbLm|!rVl2cXBgmW-tm&W?YaLkwZf{_aK1d)WSk6 z+WYWB;M4!a<~)X+c{3jpE)ZkCYEYu(IA7CH91lwDe+!Davgab^VSGmf0$m8Nj)6At z@#3&AcwiyR5nQUO$sL9jzu(j1Q6c3k@M%^r<)dsgawNTcw{^E3Se$a>!=ureeTkjR zStFLg`=hYQ0TXulSKeXWo!N%LcO-DoBX}TFQ0ZjAkd&VO$L!xn6+JkP%!9@zu`v%j zG&29a`8U7W)!lHP`3n$9eR$U2>sh-D^L5bLQ$&@=m}->S0^c%_@m?> zAhnr`pYU?!LYcEKY|4D;#W*T#Y0(QZZ|R4E3=vF8Ng+kugPf|Gqef6Y(tV<-&pVrx z2onnBLywwH=Pmk4*x7B21|L?68JJ6BZ$D=GT)H(HS%d)P1-nmNQh(`7jbocT+9wC+x4|6IEr1@O$D(y3oa= zReyvCHbyN9s#=Hk;$;*(kC%SF^_{T4V7LxTW*_+uoLm+X)YThs?49-Dn9yO7SfRcD zORq_cF`NKf*8BH2ZNuxOOhSG+W*!7h*R5Qu#cB2%Jrtx?Yj7+%N~^dvJF6Qm3^krk zLZ9V%3mFWJi+HR{tfwaOpySfEEsAN%hA)+!(fEeVYDM^>ySbbVqGUTh2utxTa7`)HWZ?Tx@S7r}b8L%$a8 z4YD=~ef-%ZZQ7(#Br|8CzA{7f7=hw&masEvG#@&2k5-c7oB*awMZZdb7$#U_-zYjP zS)%>Zg0w%mvAFgv_qHD`S(>hbm7<^Dd#+g@l2qDgf_*L`{&c4Ul<+Oe(Rf-TazJGP zlER0cphj1_+D^XryOdfp_uWg0l(}V7k?>P(xVch9$C^t@1(fv+c@kLz`c)_bdYpq2 zVW=}!?P|>Om)K)JzSbd)%W&B#Dxa#gH`{;C0okG$6^ANVTTPmQ3PFj8>ozEO&yPy0 z%ws{@(992Grfk{mFxC>IU_eB=Cw~|QmTHBSuCI-|(GLR<6pNv8YW9}i-7-ABs^#W)$Io8{BX8ixHvloe??m>MVY^AE!u2`us zaZtHz0_}6AbMci9AVP-&Odk)M@`6NU^2pF?s}m*B`|w<~l{03DvpfCK*%$&C8o#(f z8%1m0O9$1Lljg!vhBJTBcnrJB@aifPV-chf00%IigbxkRRmk0D*>EYilkEaddrlr| zMq?7xGZg_-D+X1pL1?k*-NJlbsI%}|NZ@S&?{?>J#o5)~nb*AE{>|<9 zzjt2;{m&ZCj25a+-M{beQ%d_<{ckQn>=A?o$2r#jpaO9~C17qtnr3A38JHtEV-h}a zX^6ghFY1Va^|H97*fg3C|Lp0w0>{qxL_8ZQ^4T1zfa#xR;E=v{mB5*8i)WUuH&3RF zzZ2?{?1~3bg^%M?Ci3IIGsteNQQ_Xq7(qL&qeMdzCw>-B)NI63o@@>1zOJZ|<^;Abyq{2LpRWkek{T3e|LS?=?j=MNtg>wZ`Fyg=+^<3muC4YtDWwB2H%9u@vY zv#o9ZOoBw#Sjg?vyE5PJ`+oma;KQt+Kn6S-L(AjWi4?7W?UC@YR7l*YtO&6Pp$J3G zQd-3D1>o}c-n|~~(uidZzH(^g(+CjdEID+8onKmwKUQ9HT4RBg6P+g!S9Z{jmWB-r zu)sPvl6W%O>2FS)VbFt=bLNtzkKu}~YK<@4>nPkbO9JT0NoH@)DJb}<taCEZegLgVczehD!6Ug1K9(f%}r63 zkryO0F4krXFP+C+2dKyF+A$Y?ti45el&)JO2t@lp86bXVi#+e4$pJ9Ia*~(U%ca+H ztR0=CPkS$u)vXMs&+pdK&E0Iv6N&wX=YtnV9}`n*=e*#C^(dZVb4|G2Q^r_2*y`Z? zQ^ZiK#1@rPw0Uc*JGeF*U!~sT&?Z_@;b@L>XeQ)hIcVL~=7W8RA1>2F3?≀yaZH zDvkbB#lNbXN?$7wAGdO(eT%0K#*NoAF8*ma53yQeD?}04SuFhg^no1^w97jhFMB!H zW&W*aT9b1Ngq+Swe1L@6C*P8vVMEZe1!#6XEM2l#w)vv?c?{P_@{yy8U>;8MV6t%wD88<*L>RLG5$=-p$d*G7b z{D)(uWXPbjMLlFsY|LVW(OG4Fr6x$rE#(OsL+d3Q%OgrA`9~4Q7#?E zHyKKfa-rJK5x?PsLX~4ZA+tOzo}o4L%RgfkDIyb}Nn7obP1wsUDpHXsz_s|0NFitq zTS*7?fZ0-)*T)`9yjoGmqSvAId@RrP6z0$v@&gZz-?R&TW`YU4f-@$)p@dcn{z#yA zzx{W)vz}4s-4~l?p$%FHgUod;d6WqhLL;)rY<9-kT0shh$bN9S z^<;zTD>RDwG-hainnWW@JFE)egQio zr81>vO+0E=Vx26zRTE}KTJ}9Z=o@zoD$24J_tuRZ3LG(zC=I&Y(TGKnZP(iQ;lv4p z`S7e~x8#8%kB^b6a7J z*`l}HNnzgqGRE>$sgE_=kq5%)P)tl}h67=Ab2OQYWa0@d3HT70Q%GEu?>TUJqIT6; z3So>^?q|ScBs9$Ejp_KNosXJ%DSbD~I^Cm0&YC1;P_BjP*^XfxB+)5*3ho#Cw8VX6x@+WsQ zB1UK|s3H*^Wr-&BK5zHDxtY@mKK+#$N6jMtR^M4_4{{E)nc;L!q7?2e4&Img z_f?HonBsIN{m*yic<0GQTYqIOyoj^?%>q7j_Y`5=7C3^l*d?^YVCJ98@XCmC7re$P^}V%Jyb9Grv&ayee{Q9 zklVK}O=j+xK0+8j6)}K$lTd2!ggi(f8UtF_KLdL?977i^VLxx&c8$gu;fbPif`wU# zgTMmU&?`bzKiXF+1>1Izq+pnkDvvjm7@6j#I;kt`1qt+yj^e5X=R}1`=3ysc$I%YV zykucaa~G2j5s1ZL1^UTihpkGd*Vq%(R3ubp(cuVV7TGK?2`=GfOo^mn%2ThW_*nEl zeSWC!gs1S18@4sy@Nwu5h+y*CYEI`yCQ7XEKB=aUSyH|0ICus-R<|p0**EQW8iHkhBWx>UxiLYtVWu?T3$8WX}aSof=dShIrclprK zkH2PYxL>83^TK!&L;EAHN`7Jqag`AqM~Dg|5D!aIXo8IhXU>ZS7P;rnI@AY7;>Aru z)%l6mKcaa3fq_%g!o$Oe9z}t{q@`#D3J8yy+Mj1)*Le!tAPb8H`c15ra0g3l?_pl2 zVq(agR9BN>VN=I`{qrhjfy^RilOpnH=KkCv%HL>7cQFIt{@*>=q9a4TV1m6ue8XUz z_ol6yf0&6-z<1t#Qck3WmNa}3FOEb6AegR{O|poiYx`491$m$m46t*}H*^vrzJ?iI zndmsyMEmk?*UZ(}B&-G$l1~*V>e-QOLd%zppBe8bq9BbAnruEz7DTSok~#&^VVPF> z##0WChy1un!NcpRwvyE+nJ5+2ui>fc{r=Yr&u}DMo2p@016$z}(ZmJ|+5Il?e(DZ-R~z{!`#NzmBIozb14W? zYR@3Er~i#56O1LXH^EaiE+1SKZyZ69!~IZJEx#fx3W%U7oXLZ=4kWsYgsjl?P+b<7 z9;9?$pNI_wO%;xV;>G`f2n@qXhHM7uaVmf3 zH}6k2`jXt~Be=ZCO6iANP&TCUCZ5t98j;duDx$-4ymCnUF}T2Cp%U$!*7Fl^pQsOkk)VTjxBcj9-RwVL?^SsG=edH^reQK9bg1+DLG#_s4=Nyxxlx53 zZV39bW^#i3Ih$|dT;q>Lo{jqDyjYG&o!?ULqDyyfhe4~rq!L*f8Dchui~Ig{0Gb{> z+2HyLfq+w&u`g;A#^w=-8+975l+)ijPrh7p93?r|X|Aq(lnuw*3RnPoY;}5Ppy;yB zSI@KpokfF!TE-!{;BxyHrj27#J-nH5JsV7KBu8mMoG!A)<17If-WW-y^P0Q;*pT!+ zUy7Q!1%_G=O+~G)>WHJGHQ&<)eh#2jUe!7Rr-Tf707vx>q?(zt0BTbe-he#b^09)HKY) zx(`td}`K!xPEF zh!JTU(hLZY#z&X)om&;BUQ*Y-0 zzM*GpE~6RV`#v-iQ&M38!1!Ovo-NUy)(BJu4k3--0V}Z@?Hm|RW~5e$k)U}37>R0P zR~!^%Lo7G@z!F!>(m=NS5ne{{6|rz3fy7@v@1tK2Y04unsz4k=j|r%1t6NxbggqAg z7J>%M0(Nc55HLC}o_+Ojj?)pE>T!YUV7N*)=L72f`A6jLx#a{dQ zkh-+?eK4PQ2`9XoU2Ep5NBx3tFYJveqj!Lht<@d<^IhQMU?F3VzGRex^nlj$4y~2S zt{CN14T#K~pCX;9(q~0CjdG$E*Nv#r-W^`fr8@&UJE^A3&2fvKpwG6h_gS7_Hij@o z27PZZ-%R$dMjSV{dU|_9M$TNodgZhkunGQm<{2W2>&zpxkRIf|o$`mDyU!@=tx!ll+`v2U&aX04#6Byt#x}>W z$~JY%@t|EG!DFLv~GXs6@LYZ%)V7<^O%U_;qcw?N;J^I#v2+E5z;Y z9WE9Qa`81>;PGX*wDOYvs_4<9*TRz$)uoU&!nc7iXjdd7?h!lp0wnB;u{~)`mXhw} z<|jL_=ug!VU~kG2TZ$mzz%xFTSSIX3!+uS<<8lWokz7emqef$gz1;F`HA z@x45D2%uc@rp96sF%!y8#1L`VLms@i-1@qEM#N?y_^71^JpB)WM=Co!oqb23MU`p& zMrzz`-NC47Li+Z6C9Up!afMK7Ltp$*tmy2EB}c_4HbTAZwCoSy&VCMT-+P6HN!{eeG}nyxt7`KX zRZnlu7H9&?Z>E81vU|zPBaVuXOK25!>+3k_LPn!KCSENpc8i_p6-J)4xmZ${%B9pIv93c^5(lFMee}#vXt1?=eNC{IG_j& z;Nn+HG9?P1ifw)rk5Dp5fI!d`Qdy zLF0ZzPVg*zu$6}Muc7_>H8uFXWsfc6ruXBf%n*`U9ipj`Y0Qj znp`;AA$U(_Ju~xX`R#pJn!#`V7(?`%E(FcLt5PB>1|&K9nH&K**E>(WstP>#>XI~Eq`;mv=#n}3$5hBNT zF15|04x^xUqw#bsFv)lX_4zwK(l`rBuT|MDze&-kOchb}a@JLK>a-M+A*qq;id&h1 zeAnQrv=_$Nwyyne<#YO{*E*#ZE3oH4@boJ*GofN5AYKpx7jDafhN^9YU!G*K8&)i5 zb#VoN*1iVzvkiDcYD{}OKt8NQi*+=`I%fm+o7N^?#UK$9LLeKqsIy}~9TO@4vFwyV z^2jW@8vm_O3?_5vhoKuNB!Z%R)s17ey$VH+2h0sDQjtBS2mT$jKEJisR$hV%ohzTG zT6PEJ9ee-;>-E_bbnH#@@u=6DB2d%{5@{i5jQ!NgR`NNeU1d1r^MuhaLN7>K%gq{h{Q9Gu4Y)aBrpXZ!Bh zbwyCQ_1tT@#{rPFaWc2{k3ynU(Q0B3Xma{6y5Uc(AXh z^%vZ2S^aWc#A&A(-WsQy?+`<0*GgHVQS9-(3>LtS!$AbScpyi<5bWq8=djg*p;RIt z_l8xSnw+VaY;T?96~e-?6fc`9L5|iO! zL8aNUd|&=1CPx_OzCcupF4n`=-kGRkR*_|M9C0#eQDdMrp{(cHfPT@TGbnN071;9F&IT{%W`fbcd}g+UKe(=25R&T%oAjh~eR7 zv!IhdYxGL*5gwFZt3dn+io^N_Q314LbU!Ui>c{k0vHLM6|BN#ad`k|?kQ1pt2I_O4aJ;; z$VA~L>-K5cY3{V$oZC)++P1O4_RE2NCnMaW{N3yCJ+sB*$wqct3+Y+)62AvF=e^3* z;ExpIjETLDfE{jiiJVzxv}DKJBxklFx%VRZ7s4H)x~`@7C+r%l3~A8fHc$jX7_S_U zJa1aFWZ~7v;UNqJwM5o*Ednu-<5ygsbVq$`Y}V-zGBYldVeMlJ+of4yn;zH|QJU@e zn+dXU2}3V)Qq;dmDxm8?EM5Vs8P5o*_ZYd3clmaTD`ZIdc2$qyoqrl*GQtB6=gltX zowL$=zW;*8_R*W4t;h)X*RFJBR?OUOkES{P!+Cmr{EG(_*-{^7_v;PfBa7k~PwUl@ zN&f=WOr?qA>~1lfkOP9}Ej5ml6E*Y7g}h+2HrX^nD54-MW3Oq2_)d?@-{80r5{4fh z&-CqAt27-Fg;&V-94D`U#m1Fhtox^_B0O0-YFK0he?o@zWu$dgr97L7OoXGzKgS z8=gJMjvONN#nhNLQl}Yu2Iq#740s(6R9#q_y%B!gWXC$I43J2JHa9S;oA{|(0P8#MQ-Qyb<^pE&6i!c2I38!DI{p`?1OqLFQ#O6u` z8$w4_BCQH-bYCkD%icG)YHp!s+P%_@r9CVviU~5IceVBq7MJL&%f+USFT6A1UOX#L z6yegBn@rvZ(cuHzbI-1%b6vbM#ZLRI2PzN81P0-e#+$A31R{NYcUAhK=FT;>@BNo< zDFl4qBnQ+9L^UUIL*oi|%2!iknDtsQ*rO5ACz)3a-5yemuR2{B3B`sh62KyxwL1I= zW$JE%wAJH@y1Ur|m%AK8BqCypepA9rk`4(|q!ggSW%FbyHZDsG*wETj@(`9CS8>0e z%I*I%l}+iQ8-N?LhuahbIBy-)oaW%_LmwK!ao%#aM{v(k-DNjzjro2-$!`f)83g7HMnL%2lErh?N+HHQ3uNX|W z!MKwO5gqCe>gq2P>2tZ9W7ToB7Ot4<-R(CIzN7G-vz@SY4LX?7y*9&@K|`V_B{pOe zMllJ=yQn5V))L`1dBYq!5LB`={Wbw{AXN;dJyN+*Q?+u0)LD|J{|=C)W))I^v@YeY z5fhDcT_llIIukcDWRp6kj0n};wv1Br-#Xke?~7;Ixz7h2MZFcF)TlVWV%257AmGG9 zM(1Im354eBhpAFYme7}Lp3{bHW4@7dVc~pl1TWmAd661yhb^#j8AP#ZSK3tr9&6m^R^Un&qiqAJLh(xB*33HU$3hP1{}In?3^}a-T2Kc;8d9uW#kL435dv`wJwO^eVUW&dCLf^!4skV zMY&Flf%dS>p%#ENFIme~zQiIIshxB@&gH!0%sH@l2`t>3PSWDP!thOoYkLxq1@G5c zaFac;Yl!z0nJS1sPiVyT7);haB7ORMWVfOiBS*GSzJE7ZPeU<}qH?>O{gm{p-~IE+ z#t1VKHXwtSCqlY5^8-?bGL1znR>MT;K7;4uM)s8nCYdP~eh_kSfn_&UXq)fzc6|K} z{_Uo*sq|QmRm|eNck{cM$e^JA%>@9->?STo!cMln?zIi6Bz4s;*bxT(iE#$BllEF} z=fCVOd#LkS(TC)D1Xxp)%kQ@*Kl2tjxkVvziPF}SKT_9NWToMyv!2~_@E*sERPnC)r;EHGvA z&r#_0HEeHdcpYxVT#{JRG`d_rPqxqDxRT+^YK{v`^*oK|)GMXod6=<+{qb!|y9=#b zl6X${I(8jAafpz@Uz$h#xNm0lpKkV(C4JeY52WCP_`6i7lLePV?1*7F@$vYYzOkz< zVNM6OCHIQA39X3RtOLLvB^tUOE!JXD5Lo|?YyHk0gAXYSIvq);qiw8F6 zmp$2!NXAxhVU?B0V!Q9Xmsicc-1%O{0Q+3q;Uu?MS^M1LOZy-6XV8(sSYL_>)~6oc ziA3oS3FH$A2s~($zp3JHJyzqs%jI5&M_{0zk#P70{772x7nwFKQ9LF7NcWuECL>&` zOrtg?qApF^3x(y2_i&Ks6_Z2OXBdn#S>+>pm+Hz5!Gv51+_0GF_Kp55U2`DT^{Ua{ zfM6|P;QBu($S~BQ4Xk2Ei$$D55pekXj;p2RNcG%Uv3=Hkp3iulJ*IwhsWNox%sE)*+p~hkg(hAWHL0FOpCK8ApSam6RWcn4(njEWWcK@cje;oPee13jyA@r zPisdXCyBWT7mqgGm=d4B)nT>evGg)sWtp(=;$rio;BtaoXk zJFBBe>;2)nOP6Z3W2w(6@`0crnp+P1{?s@{S%9LI)G<*C~ zH_}1V?x4~`(^fb*8Hq(GzyOpm?3VPNgNH{Q5oazuj$WQL;$gl#+%IbuSUC4raBsup zNkYKv+UQ!vn;s#%S$wNh5A)8ZqG4`zJcUnJpNDiH%h3SA@AMp(I~Lm}hYINn^>2GR zg2sHGx}(fW7=jYb^A}6{Bxx-G*B#cF<*B@7P0573m(4YjWQ9&`_}YnOF?p<9R=uqt z(Vq@3pLg4ujh&D@!V zE>v=X7`c1kg#0G%ax(ZOGL$$vV$kn5wgNHkFP7)*TcrRBkq6*Pv^7JW3F?@fR~;ke zHP~bHqQ1}9Rl@$m&Jx7$KOEenm14&ad9T4e`a~YhHe_*q2>_^I?|i8QR)8(#it&n_ zactj%BVtd=<`G2EQuoM<5!^~^zwacVWG)Ng>g+vwyiIU_X6tX$)gzGvVX@g?BEvYp%mPC! zJG;Smj$?*tFZ1kB3||<56ZW=NMy;J?ukCVA+i{f4b?f!XkC1^ik))eF&`$c>NX~?- zqZck$&B5=m2GSIR(;r@f^qY5RAE>|6b{|60?Wi-4zOnEEEB zvuVWui}9I-igCk)SwJQT^Bt<2=SW`QwWEtJ#Iyl=5j%kXRSzl>&s?7DxTYQf70E-- z7iqK%esS8XrLfSWJu+3p0!S`QaQh!#pu&acjLs7&y@FC$XY{u(p;R$LUliX1M4#A4 zk2?~iBFy}4zPpVf>#_6@I)wD>bbG{tbxk{!HzVp9a+60pPIZ`em0{WRNy7TIzI?DW+Wzys0X@ zC~z%^XFv_=Mj&&25XQy@pejcUZ2!dG&$4qC-ik15fp`+60SMKjd-|4DBR0b0`2&f2 zEGSno&OmRi1m_qIyBD5J0T`}u`LwKl1AL<&yfmBv20&jF{7liFw{{UNe5&a;RAHqn zegM2UpOGb!-{f?>{luosh(v_nfDn&|`b0;7xS0*0gp4`gneUrB-#Elefs-w#A4mx9 zF02h2*~j)*S~7vY!2&F*tUW%+a?HaVKAo?c!c75ZA^>_ZE`~x_&5VeHECfvjfWRUT zWQb|X1`-SI8Z-P@sP)VfzM<%CH0mj{z2vIQ0KyYC|EJh)fFUUeEV!Iih@d7JS)%S{ zzn0%^iN|iAa6T)kB&)^sjAj?YthGA275K?4GRx^{Dc73R@hXmRBbhnbcs-7iZg~f1 z1OU8fwjb;fa1YmuiRN3|AWPu6=&mVw*m14W`@B|>i&+VS3R-m%+j>q|O0_R^-L zFA&+}1IMm>fCr9j=@Pv>0$k|~`yBtSS&w#n#TPFtABchMh@CdCL9q>mB5TFXfwJxg z&LHH?M#Z;4NSS2Z&_Hv$Q4tzrVQ+Yxu8@Kwc{Eql|R3el&qX4j! zjiFr_{v_`j`U1Q^iN$^bxNx$*o1!hZ2$HL;7LA>TQV(qU|J*k*G5uIQ9T}?_q@w|_ zQ^m~W20`n?{tn6_MY%`fZxk&KpYB(VY5vdkUTl2Gz7Gg4i^^Ar2mLMUE~e-CS@>n1 zKx`Ez=sH~qy3xOtEoOmy%Z>5tT`Xq_-!oGmJ;se=V9d{NkoXFZD!Hfb9>H9)croqx zmBlNt!<=nk@GDsjCJ!~S)h~NdZ~2Oe%E}t%^A-7^ z{O@N!&+WZn6oQT%P4)oUtx z)xb;s5pQlg`SrE{b1oIo(Erql>f-#T&{n_Lz_NM2iF}p{Ynr}X;h_EQSKhv8$YMef zN)~1WM!sPeOlaab>F22ZWzM!yS3)KQR||wTd=EDt0=YLsu4m(y37)VLV0{3I_vW8q zb&jjEXxi|0El=>DXe-tr=r2aSe%<=r!1=9kUp-}W{Tm@8ZX8vnrMej7gmI?Go^Q`u45Tpem z3)>Aa4vL##BpjJ3=lUoz)6hgm04`?H$&u z&kA||S-C-o)W$F+^KcSK9c%53%3I;p{JQ=ZY!Y&FQuY>`uk)UO&_~ntonx7vUe8g1a63*>+Nmx5>uz$2Uu4QNe=r z>m4Dgk)8~{E-L%;-a-)WJZs0th>A$DA&a5(Xa=N*H-D$SyQ8l5fd8K>F(ID?4+sKX z#ksNnPAtRcJkC`CR^sD5Q=s2#^oi<{dh{pw*SYd2^0m*I?)Wy40 zA21M1n0UtnPtVwa{)kdW9%hwz&y}Z9iCr3zK1k6efK)p~u|vYtx!u&3>)~Q(iV!NA z;9SU1{V!rTrHv0mDdF-IU>JD!ao0=^a6881AAAfaDCd{5kWKu475q{1@PSiuIctXf zaxgr0`Bm?A`Ge?(4ImnLw`*`8_B!ZnPLzD?Nq!DMvnPJ8z~I|5uyodczQ47V3^CA9 zyBp6s&i%1}*d{5O#HXgFk59GD~+E;lK5@%whgm^`eGZ8s*(Zg?g? zKIfwxB4z@jT-HwMm>*!=jVb8M@(ZV(BW)R#xxWcC~We#fvE z?QgRy_lh|w^6yY0G`399HRM%>ptQ0X2byOu``5{HtTXCg+rNx#HKXup?AGvcp19sZ z6><>Fj>}=i%#FWQ{_r&X>ppLaqASLw#k-J*Y=`MOAlW#SoagGRG!{6~#MyU{9II}j z)L2Rc${lC55XP9aZ_mBiR#mzNsV&>LK^yyKTam)l27Q9zfr@(%rt;@TQ5bap6Du}l zq86I#F3^U#5}FD;p8Cz8L##ex}GVOQ^^&Bw12qmn9iv4V5i2gGRPy zEK$gk>`5Y9L^PHtR74`inq_1c>3630^ZcIod0zj_XBKD9_cP~v&UIh+b=~*-qIc#9 zXsJ`HRqeDq?1NCiu+x*VMH{`@`yXRywVmh05w)Ue%^M%MU@8#746bs6UYUF%2@A22 zj#Q-S)wejY1t{R%AnI=ZH70)8>_^=pAPpjmfDMw-7GaCB1})HqIgmuheN`_vr!?%F zi!6y^mF?e0)GUfOI_FII)+5gAbUfon*|%{WuZ+B!O7zNA|03X;ii$`VoUoG>TRF#a z*!OqrGXJ;GM&QCb!hE`s*5CKUR;}bbPC9uTH7E!{kh&~Q&$J^MavW+I7cJR=MFu?U*zL;s8F|1Y`qQqU zE-Fp3R4T4JWIsN|`n{E1-ZKldph4x0X>5%YBkn-cxpYzLieu-Z=im}YyP|pS(;gf9 z(e_SZt_|1+3_AI_xzK#OH`2Iz-)Vx1rFWuqDw98Is*gt>?&L(eIRKyGwYj@7wP}^j z7cw>Rj*9YO>Txyv0XxNam^UYq&UJpU*HI2$`+lWyb38V$*|Du9gizXakzLMiLvz^9 z>hNYCs0=+g)r6crZ6~lh!05{*R9&)_GWr95uehd5%iG9}>(NMgBG>zDSEf3v8XYOkDbk&1D^L*av$NauSFdgY zyWQ@GtX-FlBXS3PDHok!(Eh`H)HdH+a8X2{vZ8*w1^pN>&Q_Hzmm1?sFTz;M+=pa< zmNYlKMAS1|jXgaoW5T^Pjw?W@vZN1X0+UrjF9$sBs0sgYk*x1)>KGwjqnMJ8B*g%= zj?M2GF*Ud?8K)8ieST*!qoo3x=3oB0EzPkhPjvvo#j{6DN#~_PQJ;2Ll*1>w&0A!EIpVk|5{&D6VZk?~t3(i@!?qVJ%Z4K` zzDv6Dm(j7YrDyd}n;by`)4vAqVxS%T(QwYxeHFVxPj4D^Wu{b=a)9s{t zftPl>ZL@Ia?e@En_jXApN8#4_XNqFKJgIS{=p+S=Gs(~}G}*98I}*F)N17!_=UbrV zIFyCqU*^rYdqnuGd38}@u@=J{(~5WSF9#H-$)US?ggaFtllwgYYgL_)m=A)+q`fnjnuVLE2;s#buN#G{yN{qI2zgY!7ew}AXd79& zmh#aNP->lHjg6(sz3z0i>QmLwRHAWm_wJy)nDuWc%Szcd&bzS@dGLvH0{s?1|DK(; zeTOR~de+5@lx`%myyVQYo(Q;Lc|g&K_7yDwHP$G*)XEYR`i_=B8f}i>()YOQ`VghB z{HE1FBD+`Y1NR(XbqR=SZ7)b!SXfRewbei=3CeI|Vj;z_)V8!euD6?q`aq(EvqP~7 zVyuAoc=UO=VMu&cG!=a_jVRhl=}FwtknBa*&sRkxN!pQ$KbSVn{G`shFI_{x z9mj>^dS9cvdOiwWZDQV=mh)Om^y`DBH-zq(IjaM}%BluxoXOnOXJ68tN^ep8t!$y{ zaNMqgCN2@apM5!{_4JsGB1%5Gj@5;Uk{wJn?+^dyje`sYJ~gSp-txc3 zVZ6039uP@7hs)j@sz6-4QFGWA%~vK?CG9;x$D9N4NpKU!HxmlcHSY|e#7=r) zcj*B4CO-E7o?9U2pKuC(;wHLLu*MM8wR1MpC|&51j-Ye8>&Tc(D<^i z^)X#;LU8sum>TLRl=xjVug*}q)91S`IVGmDVT3voK0hV-J!)0iRw(#-5fEzUn&TQ_ zG!r0++piyXt2*#P4@)KZ_0n&pCue=Uv-AQu83$FeN6fR`)(r>-Ki`5b@sn}k4p~QQ zi&ja&tQTM$?79d-w#^YdH>faR=P^ls?e3IiaLYiGQAcFxDWiO;Yr5A9r8S_oyvklY zm3TqjK|N|A>rw%TI(l_cLIli|s4ZM$scD%R+To*BE9v0OnXUM>nN`H|JJ6;wa~NzG z1=DWkDM;8$N?S2Ba=bIEtWgIqk9gNF%xeNw0B|V};m$KD35Xw9eh`fL_#xYbdj^rx3c}Y29 z;of$Uh~x`r3{_=i4w@Zh_R@T_IUyXuvR|3e@Tz;kXwo#Sz*AFifpN5Nvq{$i81uHI zn8jFRBTtz5)0rZ(-f}0btx@!p4NAU#UO8Ph{gl~}v~dIT>S0_~=={oKt>3#FueyOB z`iL~hLFQ>W6?@i&P6s!$FYsMQe(pj!I7%NZv#afMW^UdD#yHPQ0n*udhSp$}oaG|` zMOW>Mm3Jq*J}ZnG2qOcGu!)! zGzdgLo|Kcx#`k6JpACqbYk#f{)Z?m!vjG)4s5tfm5K4q*dTbU8`;&bs#m5oP2QkQ= zF0-l!@r3v&?5_{!w7k^N@Fs>N4Ad5>CSKwvU5=bDDtPD)YHM6vgE_-6BPXi zfDRe-##GEQY7X8wnme1H^l7=Ih`B9J zq(2)BVuQZUbbm}{SCt+D#R~98M7%F2xpZCBL)pcr1_)0Cpbxm!gW&y&0e?R?gG?|T zV#q=Sy`Sy7YTW(c*q+JitYIvkcyqlocq70?Mc!D{=j;8Z?!-O6062CDUZ2Nvt^skY zp&{x7NLi$)s_RK~W7GXH-9Vq#-1=p3oT)ZH?`>1ncWEtNqr(c;uWc z@=!F-KADMtQ+C%4Lnh~D%+T3TodNx3#L{5|f&~~kg20`p5MeP(U==QB4Ws8no`qvj zVrY(ZH6MBUNhARU853S=h={6T^c(0WT?eIeTmjF7|KXVAtCkIJBsCd1vA3xT)kDgu zSF?Y-j+2icjWs~cv;lE@X-QMEv{`A8!_N#PPXl8Y=Z?i72pI22O=G0b&7otG!yK(8`wmBvqxzt-DuvtM+8aiaO#^rt=<} z4gbJfO&B$Q%vFqlyScmm#b>*2rF<(Au&V<7Aekb-)6Mdw)cRcdy0K6DZ9qhrSg2-BZBc#o;IT8kMX?K2kwX88nst`vM{S**;dPU^GxOnI{ zdPQ5|PQp$J?;Ko)5WTkgrep=C3a zp!c@$^??bpfw--~)QF#OaRo}o%`X{v_?-SV>hUR@J#J;*0P|0P~? z;!=rGNC`OQwoYSUks%H)$~HlAds^Aa#_-fOh7aR7=_1o(HscAk#bPNF@+fFkUB&7T z0|w*f4az_bTaMj$<6?6l;H}p$wfo}Rm47yGyxf1NyHVSjeNnGV;QoS}=qvC~UMH^2 zIBx4@t8BwBqs|D#Zh@|$^SF6Q6&DStBrjnpY1F$Z$ker=97=3tw|;c4WY#$JP|b@K=nFnr51S5>8>HKSd-75PO;e2=y1$Q>g_S(P^)O%+|fU1M+ym% z@@UV69-`dV(0zFMQ+&jZD~DLcXZMmPF?%1KvgB$%2;z)G*nEdG`yjb%qIY3n=VfAI zZZ*)=HdkFV^!B&U$ZcfK5yB>=c&*LlbrmFF8OItcbFxN_>O#d^d5aZ=BbgdqRiAMe zQ5o*9iejc`HF*g)Q2<(rNPX9Iz7w7na#S-Ax7}bx%*(W3AT1OTh9Y&iO8UZ_>I;`o z%3TP~6eaV6dDQ9rne;zkoeH>W9qeO1nSW584Ca#?X20jZY0y7xD1tFgPEH^Dzfd5+ zglv&z|3rq2XD|lT>oJ{;`k#>)dl!R^{U_HNc^aIJZiTBlf8owQQ1Z{XkQ|S{pZtX= z8Q@b?#XLCYe@2ob^c?&5=u(gzV1nb~muWC%K+JzkHyJa-|L@m$2tDA~f$dMZRQRuX cNjM zVKonf6MbkG^}eOmL;mZKkTu>Qwi0j^vFeW_;&%+6E-24;oV+{s{nmBdq0TZ{ug0>t_eZm;GL%J1ua{c5fT1>4G;v}PlNcMg95*8utJ8!-y?WgY5lL4MFH*)68z5zy#JO> zN)pm*F)7>ge;!2=BDVKG=Jd~B+#`UIf&_vM@UZ@eE#4nR3Sxo&w|)K@tkVJphV&Bm zdivMD&GY{TBmKtxFO&V}1q4KeU?L`RQocg{AB)38jQ*c{C>OYZseL}SvC;jv_5brI zU$Ga&3*}hr^<5 zhjrgOE3;3UxT5Mt;HpxaRP;dKWfugWv`bSsi|Qc&6Wh~)LG}6 zkEKgVOAE&l@t4o{__aNqwO}V16F!4JeX_~EJzci(7YBzAoSjvZ-!vVFZIZvFt!hQ{ ze|`8Q@UVj(hmNLraLz&UlT4T-0Pq+zpcDgl&E1>+QZ97UyeisZ7S;udh$`6|pvz zO%hKXB?qt(e%Sn-W?brCCe)gh~GFMnUULU3(;}#hA ze7%2tI`3223T`^E{2r(vMQ~b`hzxHmZ?P!Mik~#;gn9)wV^GU^`%@mzQk_j$c{$y?Ne>pPvgJj(FAt{+ zM#Qh`4BEW3+Pvz-2~R0N`P(NACR z`SAHZ)~Yu0vTE|U90aVjd8cfP>jOL9!j>+ac_5qE#MVo3DIKQp@%iz%Dt$K`^!H@3 zh9tN3xUyxqZ1bi69&YVNH1F1n{|J49<*5$3Z(BxLg-)FaV6mw}dHk}4OV2*<%saAk zdjrWkk$MYc(ka1~@l4(#edi&H6r+R5T=QkuKC>u|o~n*logc)}X}{SS_JWqUEh<*= zy$Z(0lm(WP8T2E8rIN)LJ&I3EglyLNy-pSsfQkz2u3qINfd_W*hE=y8x4)Yn_I_Nh zd7m5ak7ph%H&hi~1wue~b{fjEc;9|$y`Gj;s^lo#y?0{u?Q#U`*LT!09*roGjnR?P zao$#rf|+6Q;lAqOId8pEaJbWDEN!K+D9P>(=K0Q+hA!z~^rag(ATEK8X8PxsvH7uT zl-+Bi+vyS<dT7 zja)lh-+Q-H4E;(RaIxKCdkiv3&ibX-=|JHc7 z=6#v{_FU8|BMAe`==FR#PzO~&z4tt(-{@RjHP|;m>>rJ9*Vwhi=X;PF#uhZ>Z7_6y zxlMaLb!{@^J-zhjiu2|E>dh-Cud?lCUY8vn+jZYpnk5hWNXn*yTctbn0RoxNXug7t zg#`^5{a2qnWGuy#4NWP3Xg+sTR-K!6_Mx7#Etjm6Mx|cKP2cp@+)86Ax!P?%e_GkZ zQccVcc>>+KZ$>r-gXcj)BtYB-#Y@P#AbLPHqOUc(+Efthu~CzWIQ@M(@34XY$Ura( zi#<#eN`ak3w@|E1p26+>F$|e_UQg}5dm0^$p;!Ia1*WE6F4Xu{`sAe2$$E#OF39nd z;`#PDgGhG@3%2$1uk01DxY*eFK{&(G+60GzI7S!0MiOIUT;$Harx?z);y&=y$b2rj z3@%j0){CI8yx2Q?J82eKU2iYzZ&tiRwOWhhvvGg4$&uysIzx$%{V%$p>#Q+xx3w;L z&oZt=ReC4l1YbR-xR$0RJ4gxyPH6Y>hN#^0lN`V#6S~F8-fSbZj+Sj!8kzrIU5$-v z%EF7hkS(tDv5z={4L``xMxaT;4mGU%!J9+o$8%`_&H#R<2g53h<9_?#h~>WKKaxy8 zlK|~7o{qkIRG3X(i#3TS?D_hzKXOCTKO5?rzVs!%xwM zu7YVc`=w|u2o!sjm|V)a?sJv5H_K<7IU(>=E^+Je_Itgm{jpSIP0s`gO&d*Tct%mM zV%0F?ZslSBD+Ik-Bu*MRs8oME=2Mo3h__o(VxlFFYFu3p6nTn$YRU1JEk z^K`n-5`Nf_&xk-+dgUO?i_1mb$V-V*ZP$_(N{r8NP@Zh{ymDH=A#atZ$4m)YB`cr#i6yOk=HVe8lkCmc_~O?%e-P*SRR?IJ4xyeJX8ArY=`1ke$pF zk=Vwx@MuRQC5njOcw!4vpgO+zL1fz6THYZs z9>oSte>fVKb+X1n@pK)|ogx11GS2^x>zfw7zuAikuCoRU{~8F_8jRa)7{iuX-lx&X zJ&dYc8~faO?c7Vb16@~V+%jzrUjI3f-5FSL%oPF5Kn@E>6T(&tk26WYq->m>;o&e9 z)@$}FLq%ru)2vZHKzd(P`0WS|{(s0dLP4h{nx7 zy{G2({QI+BR^iREc{=qG_9|4t25dq<=Z1utdCJ^S8`=F2)e1$O>soN>q<%<96yzBT z4>^JYCGKXB|IXmIU3IQt2*$lwZg=dagSUJU!&`7i8!Wg>1Y`Aoi%EvU%R?>OD_bQ8s zXo{JC+>T{)FG=E(k&@cuN>1F*@d@lSAH<17@>i(uR>t7-?22q=j!cT#X3FV*^_V{1v8KiS;{)l%XB5qQIlB_oJE zdo{4c8Wm-2W;}(C)<&_Y180NjEfyqBTZ3i+6M=_4zd4vj1suCuJ={Y!QWDo^vBT`Y z?N*z$YRqH$tOlCKxE07WOH?Sli8WOeP%gaRi<%ddh#(2!fUXD^*qqen=?h+lN00ho zZcZGA)Fy|$?l?wvZ!!AtRLAi4soO|i7+46+MMGT?g2s){AFSNm+(NgW*Jf=;a0#?Z zx&`Ry)SJ^ zNjL3F4MAkJI8=9}Z36Q!umd5W#d)7ObOL9d#g+?=pUR4PTG{!)a!OO82pM zQpju?`C^B?g!+tm$mSmB2oca5xm}@GtHnv5Z{3RfN4|m3RjPp8_D4^8GVv~1!CPkX zCx$g@FPqHv=yYN)w4%5QlMe#O*=(Oy?n%I=zVowBRPfhGB`^9%W=uzdFF*T)Lcna zqC-BvZ{Alnyu7GfRHZ^M*|6afY%JlNiQi7X8*gbzQOV!D&zdB!+Uh0Mc+@4wp3X=> z^Up^myx2T0y2jZR5r-!8xCCl1dOp*2k(F|q`mRa0>~B?A1-FnjKaNUQ_&kW9^=lKU z+=(8&XEF!Wu1oCZ?0ch4`=Dx_wtrbu= z$}`65NU3@DQy)OQ5>kMMlfs%WvU1B%Lj#AS7n%9Xzd1)kcB%AJ+L-2K3qEH0yzD`k zg3!>=Xw>qtrXLy84kPb42iB*83CChO+U>lV*faPW7-Vidoa#m%G)EVIj=0S z%G#UFqNDl3S$Lm;&FqH&-hUb|_N^BS10#`hWDJv%j5I?1(%&odgcS7X`FCE|-s&a| zj&+M(>sM#inYG!MOv2CZ;wnlg7wGRBbVw*1uU{g7+V=2KD22`j+VZksw9`OU(BaHy zTJT4jUE^S_q*gN6^t=i8=-f7vh>4A625I~j{)8m z4obdiAmQ`*Jgl#k_gA#oaApxk-ZUGF>E4qoP0n?xSPVJbdonmx!kX+!7oAzv^|d7G zQ50vLGxn>qkLEG)dTxtmI=>;k_saY<^$(A!NJ&XeGnX|jg{mS zlC8|6`Hn^JNRyxJ^sVN0G_M7_f&_z~RQW7a&cUc^xo9nYla4f(NG?Rkr1NqrPF7AX zdhgc4n!Q?FDwX@2Sd5+Je80N3?rHU~lA?{(;;E;(6s~aR`a3#ePfqrTg9he0UDuYG z-vYfYkVv(F;yppL=}PQ#0k-C`jd&V5?t$bjPH)%#u!rwZZARnuW@lsNe@@5w--%V{|aWwa%(XD!!hSleh za*n=UXa<4eDqxQ!JYP z*nYpR?6}s&F_nRWASZE3A{WE{av&cZ8pP)jFwz?s;PvQbN*aO}vc@EXLqh!Wv zm9kzUdF6WQrD{`IsLA6Rze8eZL&OX}NvAe5&Rxk7oA&$$&LddtRc6 zcv?JKiVyBJ2EZXb%<`EYk#Pckb89p)9x$TcU#bS~k7M8CnWVDz-MF@BjCpWx&balA zrqm;&;C1b6_9*MJ8g+QLd#-h4;Z;nE#|@bGBdjhmO41)8MVc@A`F<80)JaY91+;s14NV>tx~@m294pMP zrTl;+bB5vkQbbd%{#r!#!HYJR=c<2+bwq`vNdQUe+{6lA@TR2&CZaTMTV$(Ol=1-~ z4?i8vCPd84ZM$ye`WGc=Lawtj1RTUq79{lRJ%Rosq4#*~3TgSfh`7C}FPYn{G2cK- zON*2|(a+e**0{5rOKx8#*Pnx);5xHGJmlJ|SWcsGUpSS4?ZMvy2G*VLaQd^=Wziim zCk1GHBj@}F9(?`Sss-jqH=%)MNLPDrnrY8Zh-my@Zbn%J^;%(Y1bdaGPR2C_Kk5jb zd`Ck{o|3pl@)o*uBK2nah_N>_cnhz3)1Qbc%s^wqW@8z$QT$O>Sr)nBB+KqD%0-V~ z@RaP~-EUf)8aBC#pSo2}dV!(_m59i$+S)a~NI?A;JwKlrsc}oGeR6mx8D`<3`;C=9 zJ%0^`qY-amT7_E6ZM+yIvXKec47ap9x2xNCdW478T9;1hD)nuXC2ZLcF~Ui!aW6{+ z?3b&h4EUe)FT3u-B$Gh>hju_t_4vdO&PS?8$YHL?BVE|FV^R-5An7I9+S*h!W}vq* zD7Ke`rkZjjiJe%`YRwF_>3=c;adFK+mVc&yP#o{3aOpRk~`+B00oHucPC zrRlis=(mUhw>{6jP#(knX)WhVDH|wLu~bBvhh1moaYFNEM(!h?&o;u=C5(G85l(f? z4N`2HaXRt8i8HE($E55@bH!y~^_RHrW{sVnv$iKZjmG^A6s4}O5*d_TN=S!3i-5Z} zb704=^l=cXQnKVcp$>#E_F$Z4EVN2`_6O%TmI+Yj$(7-U;=?7wKsn|v6PTtIhfROq z(IPaFO5xX$Rrjsm%d5C-2%FU|6(#nlM56kf7XSbWW50`k>!y)^e+))#8Z=jd>%V0h z{IR31m@I#e?b}j2&E$|Z!C)eKCz`AoCgtRJI8@Jyg|^m{A=}DmwW?Mv=F`p+j6*+w z@v_MA+yU$zm3tQM(28J<8?WQ+g;ruN$`jUouuJo3O!{xF?sQGPOX;+KTSzvl-ci_v zZ>Vhu!5<7A^tcvn3O1s`!cvs)MKoI~pF7qHMhJxVu%PNInHz$~&{xOWE*hCOa`lH9 z*KkEd+8z=LpD;qYuA_habbf*W?r?$i=H%D6hxD3 z*uPt^0TtC3A@l>QG;?S2mQ+8^OVpIhLMqrVy*Uj63)-mN5^>vjZ)8jPUPvX~%I*$XWY~~nLke_;BSI6GsRW&l9RaHM z1&e0(A=lA9t#lDjSZ^}J-V&!fuyubT{MrwL=l1(@lXMjawpQ2;MJ`w#OP++$_H;HzS z#z3NJnM>4AmjBQ|^|PE~zoV=Eie05S!4OM0@Z;1uw$vh_-8Ts%Faq(!b#d#aH{vST z(aA?4F)7JvLfU3K&tc}+-tJ7ddqA%vjVt8L{Df6dPIUD3_qxwGy1lQChwfd`GXo4H zH~>+D8z+iM0u+e@J~)kKwgy$9CnoBD^9}1{0B6x~FMAB&{2Sts*Yl{7znMIh#jRxi zi}Zlu0TMC~%R$`M0gOO4#V9R_0I>1iq3c8o|GwzgUKie6`0ov&mHaNYEc`c%`d+{M zK0Z!oj;I@xSQ;2)$!^Q|9*m2w+e(HogN=Uvxt9p%w#R46M5X`vU@hNgL7Q`Vt8vNc z$yj@SAeaHdY@8=3vt5G7xePFGUuB^MK!;EanrMAJ}fTiTquy zG8Bws8)Id_Df&D?8gon4>1b9cXmc1A}S9)&0qsh1&4dl7dHsNt!*G4|hb-YW1jg`+$vIPu3OjftCCcYT$}aEIRl@&4i$Q%zNvh!-s=Tj_LG&zx^9ay!ECKeJs|W*iCp z=Jk~a5Wsz!rat|_q>cJ;><@EMd*{4AKGgWQKItJ(-d{W^Ly50o5WGT*i0i2r8t-8S z2jxb7?QFaFyTFa*7w~!jPCgCp0zoRZ0OSIiia1OYBqH3b9Tt^u?Jjs`h=y zI&TjS4m!`HuX6~%$m7ZY*6@G>HK)U5cQ~o<+6KL(lZMH(AAx=_{j(~F3)N_n?>jS7 zR{-$%4X{F7T-+t&-1pg6d|r|JY}E38I~T}-E`bymzex4%Gy(;!2eIUdw--Tp^iR)s z6Y>8|0J^{XYNjP#Z6yGn$??u?)-T(!S|SJ6=*m5K0k||*MkEK<6pMLP(|5m(PB77r zQh3h2C&zO~b7k!CL-C}dCoMuUxJY1GxLxjds*3z(jqOm*VICZFvn8sRRsJs{4oonl z%q#KD9ik?rj)H|M`Tv7w?1Taift#*OX(x5J0+}twi9lTXXq% zd2C!OvDJlsP<`Fo+skv@?yrv=J_if=P(cnrW;$+S-!{Yoz)tlxTn{Emxxkn#tI{wX z2Vx1o5@?cwK4#XQonRqPZbvKtuw5N*SyHo4^oLbfuF5m6v6TV+Z+n^V|BR53*u$I& zIjP8)k53fwPIA1ci7JoG!0hbr2Nw+@<1ls3$V0sESCn z20IHikTK|j@%!Yp{(EkIF#rYJK}7VXq!6_9VoI=55eiw?fzR;PNm;~W|%{k*`WoE`#oGVoi#)Hmx%?hmQ>+364DTthErhKVJ z54_r+2tMPdrlxi?AdTIg6#XwPKp!|5*xG`R`$D-kE6S{^lq4w#sS7hBv`8*32!H|d zbsOxuU;~7`$LRQ!m6gMzquDHx3v}yINN_jd831-pL-~O}0A-`mNf-|s8(T;xs`N}O zr?wWOHbJdYZ!vRDMqfZ8=jTtz6^CNAl7u*RNB=7ksg<~rP3xZ{Snf(;NQBn^Y|_2a zv|ucHogD2N^DYEYr90o_`3f$tYpRckco?%>I2Bj#aGl!##ctV@#Qf`Pe6-Ki7$*+9 zb>a)Qv~-8$MsEcAJKM(Wv?JBAFsBR^8qXD}g_4|-b4xBGlU_U)k5+8}0=_fMLf_ka1lcn!r`{o0(En0NF%14 z<95DvS}_m!gs#++JulkqS%>L%q&aYf^IB`s2wLI~sne3;gDh90{U3sY57qI5pkEOI z1ksl0c>#1x?q_;z_G@}b5(tA+&T-RDOnKf8*9|%crdgSpF#x;#^uszYE(L|l_mQno z{>N8nB9Bg4dKuO=p^crP?@L|b0Nqr@nSN!B54bp-yAcP+JG{NQRH56@c@ki0iv^ao zgHK2pGdL(EOeKX%MMo#*=*Yaay&a4JX6p6>Y;1obJH?TG?_z1LEFl5y0@tuW2D4>Y zHn&qKV0#$BD~VPU2_cklQUxXg!U)P{>OY>AATVi!P6F`C)z#JE5fRK<6F+%x$>LK| z!t(OSv}^ysYXv0odf%2yNJTeV!p4(cG^G>x>kOqzEBA82I6xc%d-CgW?2v$C`|Hhu zApqARO2iQnUO-SXgNX1%n!l+3=hO((iMRF*>;%~6Jk+uN82qO^PpcVZlt0y`(5a6N zh8=!>IWff>43Gzr1AUg+0=_Hi1+(ZzFq1qY3t0g1%G=3??OH+F`5EeJZ>)#VeFgQG z&{QT9D=P#yH@@SZ98W+^csNymAViVFks4foj5mYPQy~U65Wh(ibKH=@Xy@SmrI$g0 zgkci>^ZGIppV9zUQevar=7mM0PKL`}b>p|pVbm9iio^d{5FjZE6A|`RY5LU*MvGXJ z*2>?P?}OiD>(7^)VBw1ZxP;pUh@6jFNq{GHejHVBgC;kCI_FhII5(a>~2qkx0bD*{-RiSMMYEF)#4nCstrUC-81cDt}p)>Q{{#ngOeQrSX85QEFixFbGW z-V#W#guSU&e}{Ggj_a^9h^ps(TKAuvfZi|~IYA5r#%5`!krX!J$a?urlE~3{OeK>M zb#Q2?tJZ4fVp0_OK?O0&c7K20ba!}x*{Bl~c7ram#q_z$y(^JgwO0JtEJYv7CzI15 z4)+=d3+wB}g#*N$O0zlQdrZp*Ld~~t-@p?CQK%RgK03dRG~!t*5uU&eyaBi`;Ixhx zGSp`iFc+R*5UeGAeE0x} z9AQQFFr6K;F=|*`*2(Gj2Qv~#u1dXH5CsM9|Jz3-)bfgol7Rk#ut{$CksJI~pweYQ zM%ojvq+}6^oI0e}c<}iy2$$7-qbS=Ce5Q$|xERp?SgyD8192^cwz6@HtZ&_OpYRz_ z7Lnu^$YQBJDWo!O=JgHjXZu$cgb}iQ4u5kqU~|R^T=#oIjKSwj#!l`T#N7`9)IC;} z{B2%0EDA+GKk4WwwL^GaACOu?QiJnV1s#Jq)^W^38l~SF`oCcId_}w(iKW+nq~>w; zg1)?jLOTBR&#PSpQOG850BVf|w_Yw~-8yTiei%CaW;Jr0XW8l8kk296@5-OTA9me9 z7#Vqc>%%rwJNOA%_q3lHrSb1{=pG*YYQG~k5JFJGviN1OF7XUdPMKlo&u#+@_Z0Xa zAl`ii_J@HOUuyGR^Ks=&yM+=4vh3E`{oT5MTyL+~sdv`kYd1XO#u(@N>|hL<^waUy z#{JTK_eWU+k(n{*w673!T7}(pJM(_cMv9KIesEqg>~Emcx}f=p_&8f)07vQw z$MM#30LU%oR;mE6(-3YFe7zT~321$}U2X)V26gL&3Yn3>a_EdqcXr5FZrRQhBBrz3 z6sAEY0g2vQzb+fve^<}Ko!Kb&5Br*qbL!b7pr510bs_Ow2x| zYHMqqQs{MrEfLc=?TG-vic+p8vPa^0;O+8!*86K;j4{1HlkT9H9IxenRiUoGdpWuaI*3w!@2c1q&G%5FHHD zpWB+oc-q-9_`SWj>$%RU`z=;e$O?!oRGyPj7ET(!{O9Ey?k|j6ACJoLNDt>Le$)^C zbPWMh0+eHYfOYjM?}L#A!ioLv!E6>P+bo#@x->4R>oVhFv|K!m%_;$l#GZj^O>hdO z;zZc!?ezukaCQJ|(K$1b8428FeFt8FV8F zq!?|kHtB;S;BkRLMMcHrxn8oV+RTp==mwm4#Qi;K@gLA*L;v@>5TlT2(=fCn)9akL zXb}QgtLR=fc5%5jg2bmy%jusH=JZW~E~_pTB>}Dup2wsYW;9!n7|3SDfjB|q63$E! z&1@{A5(4HukrFy4oQS(vahPJ_Iudb+CxNq#C_1@Y{~n2Q*qH zrlunusvXsI+Jdgi>&%$cu$Zvu%b(khICXqwWMq^IWRm`#dlY`GLWYH%a)VY?RUPrq zKL+ox@GV8!v#_{jzKuF$wgHraulR2hJ>a+}Rl`?o$AALR%ImkUL789v2s|>i$=j^^ z5#!ii@4ILY^Rd4+}-pM3zd(bRBmJGp_} z%;-6{f=h#H2Y?)?l!cwXOGw@YM{e|&CVtK)1p`3~s)|{fCQF=IgTt@Y^f$U18p#^T z9w@A5lcnp8=o9+9Gmhp@y+2H~THH?Iwns0g?qbZs3zncb2=^x^C*ii;t`C$e1Tz>< ze)z4Fj~Qo;3H!?LGwoM3Iqnhcu@`QioDfjP7H*ew;9fhYK00P(X7lQ- z`T1i4WnBF}25ACi8JRsNMHLp9**ydnCYy)cU(S3-?qJrR{?TUn)!(+ex*;t;5onz2 zj$?c8GhOaJ&41hf)w0$QJ8s-Qb5FO)8>;1e@}T+FK37FC`!t`%EjRjb=J&9hQs*7* zoO-pk@-%~B7;IZf?gZ|t&0Yb{a9ir;pPE=g?rxNAvXt*hrrQAkND}=AQm~}yhzkk!ZK~SppE<{gj~4U9 zqyC@Z#MxGx8i5NX*W0Z9 zY$zjWocD%s753@oxTtN1Td}0!F(xgvQE0CfaQ;yCf#fT*v2^w&ZJr}eXXjKbiE(!J zu!#Ia0=qxINW;PQTOVVbQyKM}+0)038DzL9?^c}S=D0^+(eW8xxg;Coe{qDRQN+A% zfat2*$_qp}r*Asw-`+<8HN>AR^%Nm^>lai%P=BA(xCTX9|;!_o$C3 z&2~<`qupm(758L3+-Jg81kmUvOQfqOKu~JKQ1r+xr}C##zvb_FF~rC(eRHJkwu<2& zKYF$Y;s$*0R*P<;?hcYP4Lqj^D2+FL?Rml?h_l%Xol_?e1i}E%ols^_JCGEwCIjiY z3GUH-#C@iU+ehwJr(GFa=UgCo?(WV3Zv7dkY%uqjIH&GcHa}(n9zQ`XP<1`AneG3o z-4oNh?t+BQnE-PuKF1D7Ds2h4??%Ur1?(AHPJaLW>x1oVaQW~BIB8;6YDw!${RG0e z!ycYFGka)!J6?T9xJRF2&#`B?M}M;U5MQ?SdQ1W3XHF}jKp1eet<(AlCU9Nhmie!v zS+sw;aZWY6FW(MIv~{dexe2b^STUNOFmB(!KiSpkb3t-mdB|FrFg_Z68eAdRtBL$k zw&KE0XaVev>fb)=-&&p89b@t;SvkWj#?lMg{J916!Y*y?R%CHKc|e>rRltoYth5eWFmkXJr(^Su|ozTGjZ0>szny0Xrx#}xgcQ)6D8 z0bsk24i!DzUEHG&Jr?QHbHEFbtSq$3+L`^6| zS!DZau-2s!xk+sagNulWp!1Kg5HeNJvoX$yOTw;VS#A~m2zp12$GPOM&%C7Ypwl)E zVf8Z(BuGoLW7zjxI(`=Xs3drYm{;u&N(=o{pg*CV9p60OIHO?SQR~#N>?svU#)3@m z&RVAGj@#0gOKC+o&6|m**_X0IC$f`hKjo;GNlQr&Rm-ZAg50`x-a;+22wBaeB?RO` z1>!*1Zp@e)rTp`XWV^7sfg1^VhhqmX^ri(X2Uf)jvPhLDQQi=Nhv3rX4;p z0NlWVO1BndzeNP(C-4=^jVJw;+ZEPVAbk)rSd#^ev#X%qXm3dW*+MXuLH{d{Yg==; zqi(eS=aYyI_|=j5^0a{Q%p!8?Ar5MQP&g+)%JD-$*&gToL?8SO_r5QHMz%Z0(oZW{ z*!J0m23Xn+5~+XTZcLEia#Yn$yDbN+-f(&OQ5k15OLdy$>^BcNzondeXz6Wf(m65% zKO)9z_bI?c%&5|OFaVH;X>1*29lo<3;UQ-5lGUq)IR@%BNoTY(0-c#T`8$AZfbQci zcr&XCl)CudZKjfV-QE2FGU>}Om!6~!<$cQ+d$xM+(oDo~fjj&9j;kN?wEB|P=-DlE zAa8EMPu3pdsg)+$Xh4yI)`-$Rd{LwWfpJ;p{R@R`jP6XJ$j(tMx{HsGKX=lP!d?#@ zVepoaB_{mRJy!MdgT7UpX72UA2G#r_)5djif%&GI$G0x29=5f|32CMA;>7u#oCv38Tbk%RTOZf?6u z866!)(5H@Vmu^_1!ognA(FDBW?klrc;`Q*1jMeb)jjriWsHn;TZQk(l-=^T=`2L#Y zxK-$#=kVi9&5#fD6sMX+h1EykK!Xvz4g?R}qXKl%z0-EC#h%GP5IU;qb^7NJcyKkBD!22>u=se!x)9Ov z@NWaI0JmUdQaHu8#NR0n;8`Ezi`xE{N9x)&^@-9;)wK&50lgd>f$5+h?>@(R;)tyt z1UTX=fKAT;v;ir5adrG%_L_r5_8KCPoLmtXfGeRlx7+@?u)jWgiP5cHZ@y6hQTpW- zthh_0%A~+r5qa*QS}@!l+W-={n~VD}Cjd$%-X_ucSLc;GYO3Iy>-F&+EE$=Mo}I^B zDXHb!vtY4CMf!*VT_naGi0F-p`+{0XlWR^-61VNZ@ES_`(Arh@G3Wgyf!V>@7m$|3s;wq@cq~qae4t z=>PvXVH3z$V!_y8XhWd5bk-9AdKU_*s6F8;bX=WEi1H2SLVAnIkNf=q)Js|xaTy>s zIp=BAAMoCRF@rXqQjLlu8}*;Q#H8^_Nl=ecoL~M-O5#b?WC;b7WWdA0>B$@dz=sq0e+I9To6& zguMLx%+g?NW^&9KuLhfi@k5^H8xcUB$;0CJa{&Ndzon-~0>ay+xLP!IJ#nMcrMhDM zmin28Nqt+x$~%5=Xk;QW1qCFaP~Y%26N6|V?SvRionA~6hj$OYBVLJ(xJh5fUT(2MS$@{yl7QM(xlQR`8e9xgqg~Ri-ta6-{E* z_0hMFGk5X>dsjbf-D+jUuvex}un_RM5T0&Ndp3bqx*ut*$}fozncMpGx?4c|%7M#q zMb2?b_&#cUj@yKhT-4wB3RdNUVQ{uDjhL{o59auJ>N}l*7$>XVggSN2bBAmlg-PB# z#r&L{*iW%UeZ;u9Tsj(G$s-~L!=rlAE*wG6i7zM8At8yID@_2o zHv_)7D6iDwhRkN|LcO)M^<0O)>&47rc6zp|mGLdm?B;0G!;L`yd^PcosS&o#nmk-zyT~oqBRU^13P%oK>h zU#=Af0H2I$d6)>*Ur-C;e5Xn9n%)L>U7>PDcgeWPjn3#WEu4WdwQtrF;X&KIH@HM5XGI(y-dEe|W;il$n>%^&*sZkU-qqSy zQHVIH0nphyVVKQg1_$&=1p>v>N8|IDWixYQSY%}PsPiI6Wfr#yAj)-9P*V2)Aw>MW z#=J|rE%GJC1|T;26^Y|CW7~pM5_v+<_T_idL|R>nInNA!ubtUoM-xT~0KHOt?k5tX zO+ldO(ZSwTulJP0(d?IJ5TZMcx0f8t-`92j?R_LdrJceM^oUOiMt{B{;>;Bys;H=( zDCf6E1Mt}f+;}%o!7YrJ;&6P?asW4)G?>WZ#|J1uF`zy|ToOG@riZNmvKRyuibg=Q z;{Su;I{`+iYeQ3)+R%#B+v1k@p6|&ysx68pO2k-dQ@kUakwIeca0ci546!M;on&nc z4w`z`-(`Eru>I_m4L9V8PdY>Gk4Ln8_Zz`PcG9JHl;AF?K!fV}MJEJu)zT{;W z(3H7Yfgibo5?`tLwVL8hs|5@B_Fxl*6o}f>@gE2J&}NAUw&~j5ooquSp#6Me%maya zVieNdz*IILHBL-TgaSl{3Tp<-h3AF=-Ggi(Czbl~dX{>{^U;Z~=6)t+$;&6br@E`^&ynONKgOfX(9Wvvjmt9)mbAd6PnQ`=Puyb=)g87G#6$WVA z(dz%ILMW{K|KUlNzTc*FRqK6tZZzSj$FLJNk>(rE$FXqyXdyDOrRE{?u>Z(DVB{ciK6Uq%JjRDxY6rr>DK(oy z|5o_99Tn2YrG3j8IsCexu8xNnd8Moj6B8lqA=WFNY0+j3Me1={rO;@a=F1H7KAshR z?TxT}&tih8b?}kE`c+u=BEx)G_J#@ft?pT+Tz0)}R(a%_w%`(9?u=9w5ApqvFo;$f zQB5lwyDKR};{U)7wiNxB763}s0UOXQBqW#gBnvt@LVi zqSrTZnN7o??xT?SrKP2DJ>Ds2j0#3P&K>7V)ZytQr@^qod+zI0pVR2jD`fFS06n}m zJj^+1kANUs`iavNj8?O<9Ez7XiwjjBfUuVFIs0pDmrx6I@bb;^e^U5WGh5ipAMX4$%97s6#E3;c*6zoh#u=X!^!sy964t936!) zCth|xfSG1xQO3hUx`4>3y&ym{$)Cz`qd8irI8Jd?04crWzbOp*8O>1hK%x;wPUi5Z zXCAdc@9pOJ65tN?7uSGefcL}1=K-I`&cd_=@tWR7OAb`5OX;y1OK0x3% z1pur;Ou!BaxLiuB>#_#jY7@rsMEOAM3LzCNwXlmeTeIeQEJwBNWR}7UhF&5Hp|VWii{FXlpdA*tPScfh!tI)#4En+!?tWPixTgf=W;>@ZBSUFe%uK21<%x^f}Xw`!vyWy-buyNf@zd&jgb06!DW)~DP zI3k%8-ZQ0x)s_b3<{c-&)L!GBOI6Wv7q_0TLzfcSsQnDhw3cZ566B)yx1$~ z)558lD$nX;W`<#}PSC^mr|-YTEkmuOx0%l|+7w|+Wrb!}$Pf~fj?h4^ARFLu>s>or zoqAldQLZxJPjPVA*~;W~H@Vyj{!?3m^!iAPQGs7IP{6*BZ@(HU70DlQae*c#9!kna zho-$xv+Xd7l2J8JXv=VV<_sT8{*w6nmQ*Ne^EdmkvJO3F1iNkarm~7&$g?T*H#EZ7 zkrw>zK?tz zWP<5gLJ~b6d2M)7xa-J&x+Aw3!3r_6$OqBDDZ3t#H0kq>(QMH#P?@mXuX&B8V+QVy z=;m3^bvTMvz4ge)SmapWNBs2*4M`+r`OBA=A9!#8-?9_-SLGM&Ba+LJd(ljh{Bj10 z0Sz#}H;D8SCx88V@NYNiF{8(te^cda%|-bq;zOSQ@qV=LOmxxoNz~ z41HIbyvuwc+AtBCVH8aGK`@@nR5TKk%zBB#zmWz_9$*g0Q!=^H(7ao}djuSC= zdKjwRA5?uix!=Yb(b16GZhGdy!o0+h2)F{!&)dR~YRuE->~(v-N^!X%xW*Ao=~a`N zn$&8+4Uc=>{3;eldktTWLcBa2q#c`U|3n%QqbNMFGcv_p&-&2~FPu_7UF$?}b?AVl zkdDa>Y`0LvX2GtztV+V@<#Aq0iYo^HQNBMM3q2miEtjzn4b_q8@)Gn3TT0r?56w*z zH!KUn{xag#E=ag^iu7V8uWwLMN<`0fe~ZEy`Lv&+WbS3$cVtRa0I|Jt?puOhhl^LdVh%>uM;;fdCtaTLOG>pN0N)gla4OMk2(6>Pl<7Z!N5 z?e+Yp%g6lkMC=bj6hIK><#kdz&s6Q@av?7*ETNly)Scq{%63*-`rp7-7be)!^x*KG zlnh*RDaD>tF-&$Wjfk8jn6<7uq__J?`HWuDajsb*v5U-;R1d`~Q^D6mi;~{B5eyb@bB4 z+me^A^K$M0j{ZtWi8g@At&l;t!^2||nMY~|F5TYQ2!oLD@)nH7kcU26KNQ3SU6nq# z*90Hb+~W1`I`WPBk`5MpL~58dG@Nzb9K>MS@P880i1*er@C;ckRfZpq9#}PiD`CIo zX=~sRc&v;;qCi+G)dt>?2zx=UPht^5nOg~aUsg0RsHn-w7?CvWwqh50zPig48)Gpd z!i>%nNTvKPm>Jrl1jqM=0t_e8!?~OasnF0Aj4ijndA45ia;PC8GKd@Rd5MXOTboU% zh#6b0xZpT$>X|phtX9n+!B?mMe7*Z$O4cv*(v3m`shlK?=>_lIR6bdV1^VKoLNpbr>VT$^iL`qPd zE05wEe1FYH@LYehHP}@99%8y?@!?LSso`NqIDGFRKm`g5BWpuvGm8*8W%7FwMYhcX zOiPt^F=S7cjJ+x%CT- zlzaQ|sQCC=SKZAu7mN$%%v6RAC;+JF-~hr+`|0AsmVWCunm{0Aj#hHvp$g~@5vW5$E1R8d|E0C7ey*|G{FO{ zuM`CGs(vL%iIK6=qg#khSqI1$TU)7h&L0vYT$u6B$I_Yf23{EmJcBfC`QvjJ7jvyw zV%IC=ifzpP2T0!3lol$=O4a7_8?C0J;)m+U#PONPg0@dq<25JOV@(K;ewR%j=>i}9 zF9?_GHG?`K#6r%hSR$g|O+RQE5Jj!5${v|h($kFxMyGB7Q=4PW4BgMTBk76%PdxVYIpeQU4K2DQmDz z#5dTHzF6(#`g#-$EQ#Is7Z?m|Np%vcU%!T=Tb^1Woa{~H-9xLG~a zX|^9Qu!#PZkbQF*hEa-)B#( zQr&W`rB(x(#T632{kZe- zTk4b6GUCs;Ks~Qs16{k%p0@NGe39K&F#n5Lq@s)y{2&E|hWlmNP|%BvuJoI1&d7J2 zaF6wtk=3tUt%Fnl1)z&^$mWasyj)wgm4S(pk1Pt*{69Vv5C!ao%poIU&5iGy6<#`qQc?k_V^8k)#O z0ScNDW(pf8_;;g(F_J|NIwqnTb=q1a!h5K;#GkkL{1tCfsXxQE@RTJNu&f zQA|GcSTBg=*0Fw?1qoerZnyuK9W`LG4OwZ)NF)FalfoklF|~tHw}e!#^(@Qr_K{wuGu12 zsCk^fe+B&9|9RbU0*YnO#)jvG-ZBa}8Mc;fxY)tPD#WCpY@Qk2moJ987~fCOdn%9hcId4I}_BVRFt`S@qgNG$G=@HsKG>)O7u@Yu;lw*tkj(VZlNrjX1&dS z{{Ky3?#f;c^&nMaz`w6uf@D$e2B9n{t}zZ{YHDivjUr=%L^I5ZG&*$VQt}s2YfUQ? z9A^cep4v2XI|!`2tY{Fv(SOg=%#5U~G>FnK@0#h+38*Zs@B#B+9P||0E759DdT811 z6|femj0}TPd^ioEBldzo?DCZe>UfvL8?ASyF`%&}@C^O8BVjxp}teS)xFA znPL4ChO`cvQjlg}Rs;wZ_%nfuK?2h33`#y`h}hX+Ir<;EF;VA3(>PTC!7xM`ii0ge zCel(>R77!EJQ|0sG9J_<1Bf)04DXy5TY>i*uG#znSnOmr`O-4zjaD1O#~J0_pYJXh z3VHE3tQN5+Z*Dc#YN0@#>mc&}LyGWG&NRY8eQeff&5my|4JIy*`e!_5Aa#2^-(v8( z49`m7Mo3zkOGai7m5{L5?O4j6=$4?n-2R0*fkKc_=75 zbiT!ZxhE?%|EVq042RL6^|zvki)=VyA$u_5{+&u%8W&*$4QDx zR*SS^hew%<LkR;AeN+VR7}hPXVCXy_3n)z#4+9jYRb z3h;lSsoV^@htKFrf$PGeqW)UR(|~^4zY;2?yMkGfld(>qEc*aMlM8sio5PH>o5R}u zg)LS3pEDIXA-fZjpSS0>=#Z+0M#!3)U)Aw`Bb)DfH9nxn5`+y}zu=AhG#QL1#96J* z60NBKk_$WtQyP!6lEhZs8z1yV5sG7yioUn&)X!%J8tM-e-@J<7=uT#fCAndZ zbR)?~oA*TC@gSg+*IS*)(?2yR$#%_`0j%C{hfN-gJibwIZj>KQv}YbCfsA_XT%C@N zLYTFfhujpWrw;#_j94AovABn;S0F_?xBZ2|9Fz7`F)=ZmQk6OM2$cMx zKzI(1R(kG*-yvZZpDgIp4go}^qKZ0QiPVM2F}UI8?3q>+gBh(T&}w%;yD&7Q?QQI` zKh}yJJzE9R7)h_mg!=K;gGA(PDUY*npovqiC8b{N4MWHii{OdPF#mbmeI)EDfp4K< z&!sH`2x73R^1>pMsxd)VLy%rHe|W+}=R{o3XSV2$p;y#mc&(A&Sz(jR5kVKBl=knnK(?Q`{{jOjf$bGjQ;Vxe- z;2LpjUFOI;Q2tC#E(Q;NaL?npy*o}tJX4&bH~7iW$O|b;qb15F{=m=t`)Bl)^kzMfvyx0gE35h-BPrr!j)R;)A$%vzsFentK z^+CM?P!#wJ?aO{9#et3hb!;At!YZc~Aow-OF?A>WYsc-7aajhjCuOsUL<|A*n>I>i zkJbU-f?oL&0vNBy)k1Kd&(qaq%d?}?$Hj%kjmnndA&*3=c%iwpd{J@{34+ldm`NdZ zPpql%Z1K26JkOrvx08AH)+vMh^UY5}8zpaV>Q8}sLh4&`rBW;E1*=&dSp)pjT~_FK zBd}T8M(ahkh3Yqfvt>bZwc`DeN|W=Yx0$ZD&G3z6Qs$4F5n`uxyOx)MndclNIoRx^ zr2inhj`QdiB7J;qW6i%+>UXtopH^ZN(rvxJ+%fw;JN`WR;}gm3an8c>o0A$kU}S`x zmjla!oD4--J$y`Dyeu^>;$Y9Xu}YsXZK0|>$KnVHc=C}jHZ?~86M0{(4gwP}Gk|U# zuX83&3ki>7j%}<`zhfCCV4st*vU5bfo&+; zjaEbzs#>ALacO`@x=e%_s_lAv8z%x8d89aP>ma$-XGV_x)N+Pgs9)mAzD7I)Er)UI zzT$7F1<>0FX4xZ<9D^zX!~M$qRl(02WWL8069MiYTENtOmBJ~B2RQi43qd)u2PHhF zMZEj5KhYH(7Ljt&Ymk#=>}Jr2munqBQy~oAedIm8;F4@*Vj;-z-;PM0U~waoehryE z5RvS&F&*?sf)I?!4$Ur<&VEGWcNV8At{1!^RtH2NRRi&tHx%L?cv4czKhqA;Jzw^b zYfV1+z`@aFVA*Vz zCXPRD!olQ@J3d;TX2@-o!;T)!Qg==Dae3Haz6sj#@%KQnWco)V<#hOnvRe&|z$7eu zH|XxPm!LD&p7-VwnKJ zB&}d-igP=TR>ytAecyRYtiP;+cP{|8F_F#dQ6q8t&pmhTWIv+{j5G~-*@3uY!)aMq z0>njvx1y8N>MhaITp|ey0*qstqLyg-ro^v=^mLrc%~J85X~F>GfltI0jo-*&lwoDq zI>_=B9KtQln@T{5HVmxo{JCZnD4Qoda$@j9Q#01(IIytLF*!9nISm$2oK#g|r@|N84Z=_s-lYYy(Q8Ib?5Zpt~)o(M%ld3 zdl0tzRz4!AXBrG#8_W!5sD47=`MITZra+V^*SDTS3PC(J4upZ?d)Iy9V=T6a@kJd{ z(H1NzZ2oK^X6R~@oT7LSK&|U-kn(jZWMQEAcNj}!>~opX1d+!rRWozcwY0SK!XhfA zON0l<0&VAGCZyF9)x(6ErA6y!tBel)7vo*5IP|82v0beP|H-Eux%4nUQ@lj$Dx)7) zmH?Lh9bVGjit@eD0mgsbm+|5yZ|8J23w(!^ddq-!NHTl2KON)Fv5|- zCIl5`(hj7S`z-z49r#BPmE@fmU1#ySHD!n&!Z!W0?uS_N*y>qy&0lTs&v)hVLz)PP z&>XUXOxzdMn=V%C0L8T){F*Hr$)MF{nQ_C%fd+}l7RUpFTWZ=sS5RN|uLjc){J99L zs^Z~tI}?$}9cNsRe%g{nXx|Bl5N7X%_YuEy!+7OgoP?C^ksdYYQ?oQ3H}femj%`wm)~Yo zw+NX*T68Lc7G`GL@AU^{Ku7%i{OoKa^_!SWLrDolVBlq=#0O8sMM6Tt+try#;yA|x zX*bK^9_vC6L?JO))$%O}T773XVm)6bg_dYhnilXW$sJF6a2^}xqbHW;dc*xQ&3J74 z9s3wESQ5LHmXz+NPOrDO8mkZ_=7B;ZPfUztrG*k+xVB+hBG4b(_ z2MN#On-Z|r)TCOij-)5SN43s?%31)#ldvg?3zy-0JW9htaiMVW*~}+!_<+0;w(uk&{<-B++q=KaJ4Ve3f z(~~eJ0KW+{PYSQ7n(W(?m5u2LjApKB~1s71`>)zW3j-ydjcW(CY4S3(HS7rzyt(99PKhY>X1gnYu2%^h@FTQE&Ai(D8tnNZy!s(4?d{ETG_@J+ zc5ZF7KWqvJvZS=M#GCDAg|cY`2t*qaIth>=Oyldx-vsxw96pEZOt7cSXJu7YabNGF z1}|G$gtxmfH`iiAfATU=;PtI1grh@>3&AAdphnjaGCoAVv%t0nEYnTnIzvg$nVJTgj*XFVq z=T+qg8cqh4GzR@Iz^om@I4>jjpnec;!TH0(-dleSJRl^r_5ueFHM;QO!wqZ8B0^LDf-|DtrHe-5FO4g(m%> zu-KRWG6+#gA!o;F)c~10hWHl_njkR^@8Z7wdjBGP9it6lswso8uoP(jc>j;`Dzxw= zyl9(teV19%N>}(vAZn~X#b>HG6vp(1Ww1d(7LS6u*^T4w^RfBN1P0l}hOkN>U>Evqfebah=Z*YXV6Gv_!_D*s1YR zk-b%0U9Yegl^rYT0!;A_x5JVaF)2;zW%}5Z{GmMf4}jjRL{5jnxC6{fR2ULoQ+4cT zm$+SxjAJ45yj-=scf+t7S)%hD86?L00OKKBWegcbYyqL5f2aEQ>8K9ijY_A9L$4(z z2Mddiag#CbZDnQj>Q`d1q(i5Po!8+?XS&TT0b~hyfU;84rt@vLJ$?yAztR*i%dK?z zaAJMew?%c%O`Y-wtZ{^0-dX|fZ7lXjHJBoM_G0`>d1MsLQel*WVm6_n6FKTYf)zSd zqt?D2$i3Qdey3*Y+5c)Y$xj224p#8EP$K^<&sTwWpbX3)L4qY4UD;d~WZit9l)#LG z@l1)W&`IrG4QJjX7VDra6y7=!jnVYfrdBGF9Xq&Ja@>!9rNILEs;B3kOoPFN0Rguj zbXq`XoDTO7SB>k#U-6lWw;8t@s>HLWZG&*Dd1pIy6nvdvi`??Cj%)X8ZaxnDM-fme z^__5Xy%u^@wA7bRV7!;9Ow628S-|wf7Cry8zvkcdeJwJbJ%H3WQ&(B|uA!r4=cp1& zcOlDey~X6MJA}*Ym90V@SXh8~Uf@=#uJE*mMoJ3%0}EVT0lO1$btyLJJECs?gCh4E z*W3W?ZzAB_J0Ob8ulm_g-|e{RvZ$t!f}RKf6Vyp+C@3hfTmuSMM5S<`2Mk@>m6=OK zVWTh_4u6r9x1R_|)EA0Xucj|5jRI!eieISqw0FN%v{mugEc3MlJXJ%vciY5crOmgu z{GRkmM+R5n0cJ-`Hg@g4sZLj>e^06zP!W-!C2qp^Ws{PWU?<%OdELc;PVDC6X*=z^ z6s8SFmzI>dr>DM!B9E{Ss*Tkd7}y5nF;Xh0Ul63vG|Auf>z>^x!Q9@Pzx!DDk@PWx zqNB)02TkPcVEtkE(%%~XIzs0xB^aVJ{?Q#pXtrKd=X-agG&Nlr2F!ni&j(33c6~cm zyESZreWsW{(ra|vp#GDr6?boTJCCFdIL?W};dLjmUih7j(mTI@SpJ%!Wd#_+v0!0G z$ELG93Y;(DucVV30xa@fsw*s&T8#OIs|>J6N~}8IDvmC z4!%}-(Yf%)AAf$CGr5}>kVz|>Vrvy96p5)8cF~U}HqPjDZMn-P7d>mNb&AR87be0L z)2~ueMqlunu9D7|IwqGCnUv2y8Eni2g0;rgH zhZnVFPMhT2!KrSLC>Ixnf$aQ$^-~SttH_C18PrjNVCkmW>c(CYFl$i+Q%&{bqwEOy zM;yX^crRV5CeDi(KWXOLWpdbM)+GvA9#qo9CujQ|p+xOG0>`}!=H@Gl!3EFpuTab! zidIbn!_zTXCs1hxOa5LS*7N$qY96fr0E~Bf+8MAYL!mn??F2|rvz3CqXLX_w@|x=_ zDB%8`Y}_-qMs{QxIB>Tt(wfN6aHsTT?=9Jj({eb8^K&R#jS`6pbJe;qYrhO$PYmIG z$-lWd9K6s)6&S)Cc>@yIp<~HWfWhi;0#rW#$|~FX@dUM;L1#)k$M>{OK1&Tx zpwqU*x%-({!qyXK`2tDvp~bJS^Lo?6;%o@J<@`YUYcB_nGkzwW`7+4_DtNW|f_^>y)Yy=H-!xh}dU9YR0;Vx#1a%*5>><(t-e z^W{b{Wx%z@uCm2`isQAlv%i&^!1vKjI;BwrlD>yAY*^@PK1PLZ2w7QmFtm;*ut7%@ zQ%Kn*0Om9Y!MOzjI}P5?rAm<+x`KKeZM1nq)f(yaeBk;9=St7w(*rNXQrTzgDS#6g z8bn=AioObEIg9BFEqwMY%wfC2B!2z3fno)&DE|7}9#oUX0EAl|@)1e_Eed-RM=?V*5Bu<{(dHT9I~O7MS6ACLbHCHsuk4likj`;KLW0T zyW}fWn;Xyc)mbt9Zl^Xonk?@aF&R8KM#Lw!^x7+|A=PN|nBBW$LD0_Yhx)Q=i>6u` zjKpscLOm498bM&`HG*1gC-VdifD^MvMJL6=FMX|U5ZKZ0KKCa^&T%t}VArEY$KL>l zACqC&fcc#CL2_##MJXGxP?r{>l%sf=K~J|6GKeP^q=GdLPG<(oZsV};pK;+0n7z4A zD5eA>*Gkzpe2I`036-d?c8{a3SGs9~Q4b?n?(bK8LlT2$LIt+iLCfyBSOUwqDUh zp~qALy~~;xpI|Nf0EiB(Zmab$^&^|nUmmH4(qaej2{YzM*#)5f`*EC&X5*=>foT40zEX|9xt%exOUQz3%2K6Uel63Z^2LE}2rc?_i0Av+ zk^3&d&?u)bZLLz-VnKsu>DKh~x&Kq!^YwVCuhwvc*Y%)28>pEELlSUaU7X3&)ckl= zom?TELg%p5_4HZob8Y9WK(P;RDi`eZ-RXzPF#W5?+3qO(y0nHF?Q3am?GgqQVfpPLGLAC3I%si^>DM6 z_5iQh;bKCRAbd<{zCc=^pO6$xUc%+FTPQ*h!uue2)RDVF%X46Fg&&ZbGgQVtq{yU8 zLk;u}ptn|_Xu&IW35J(W?Z-MGM=|#>i4Xchae1O&d9EVyN0Dv zdrWE?dv@FVtf1hTV=%t#YU}z8e}wyueGW^;hegItqzB(fH!TC^8jJh=Y6sgA%?+6J z#;1V^;nj3`D`=jEWtuEx@GEBpy%`MwtdlTzE;nHzqW1b8W{H#BZQN2gOj-(q|7s!H z3*m*nF^it$sSs3IpNy5m8zbZsnX;Y`aK0;@0t8+Hiz~`EI-Y%jH-f%+TP9p;2R{>p zoc&lv9O7dxR&3k9-@HpVo9SP}I8FT_Fwgx93hX2K=|*xB zh(yd?Wmnu{WwW6Ui|tK+7eR?Ra;j6IToygtLhLY{&K(=T?lKaeam;mRZ)S?bn&unV zAn1Z~R15CoVR?O&myFUNJ|vSoUxPktz4mI(>TyZXY%)#NJ7u&4ef(y1{C6UQj@{K# zI)_JmVnJORqA-slRs-$xDxh4o{N4K~ z7t2(5-3czXIwh|)+gamy%~)u=aLqK0EfbjA70_=H1}xJWxZwCTEH%^=wWvb^_o!V7 zrWcW8v-o^Nd{*6pWHn7R+#E_F|DB1{xY;Uq;Y_UD3Q~91T5X-vF z*3JJ}rDin=Jc+d3L9w`Pv^dMaWBXmqoeI}xm^k3wh=}57z`p)vImvCoYSnSc6rKF` zW7sXqAydWHSZoPt+Bt_&Czclr#vuyTE8d|HlC7D*z|Q#6Soa0s5JTTS#@0HQMLZ$d za7O&eg~jK;`_D68NZmZ)$--jvB;>)L9q zhGF`83h??KBh=tj91zh!90c4)z?d*TA(3)>;m}IgN2d_L^*~a#tUsU^hs@o#^~h=L zbk%;&2oA$V7)0NzZ!&A~A81a+$K>buHyC(iD(yG?3#=2ud?u?o3UHK5b_J(ZP4L$X z2d08q4IyOj0>a0&XiU0^5<)1pH++GH=i&Nq6rOQjhPAUA9611OfxX0RqO*~>5;unY z5EaJe`U`;fOiCbgpuOSiUMSItu>TPO%W3R_nyCixRG{+Nd`@{?!`X%^!+iBUy8%8! zm>M`JebR^}Ha>COFiJzNb^#`g`LY7Hjc)*IQntI6hmrmf zuTD7-9|-5`Hn`yMuf2JFcQiRL2}R;NX3C;dD7*Lo7DBPu8L5Ez{pW(j8u1LXcW!e{ zO@q2Q69Ht2iqZ!<1MYF%`L0TdC!{%k(EHrm8Kn5V0@i-C@JYMG=cD>*ea1VTg=iYY zR`)=ATlfE7ELkGBclYyyeuA1AO+9f$;M%Z!t=TmGdR>NcWjC65_fGeA5aE|2ight7%qGfxDRTz(-qrfU~+!ds%E$leCMmv_2w#e{1%w zYMn4TGef$wU$^N5_buxAa4!u<{t~~rL6#=V6LJ6-DUwgj7Yk6Coz-(YuU2v0_T$9! zLm`N^31@(RLgwI{;qeIeR;X+P+4d=&N0_B;d?n*XW^<2v&K*wYM1J`WFP*; zKGCLLr#=unkQqUjdQ4#M-xq8-P9lJ^+3=_o0et>s`RO!dw%I(5nm^s?F=e#jH_Lhr zdk%Nr(Y3u!a70*Q&*57&S|wZC6`mz!Lde&ul0uiJ9)Yx+`69oWM6JldP2oAsO|na| zoQgb2YZ%!y_c@&D3{WG}nRo8+?Z#}W^~^b{Pr*fp4$pXZ4aT$Rw4>hdr! z7sc4E-wSxJZ3}Aj9M~cpDIy!p`ea7EPq-s+C6RI+y{NEy+mLvVnfy$ujqObRsHj6- zs*RKWcwc1?GzK+gdi;9YY(rY-o>;R!Kj0nn8R>qy#$bnr;} zpnQua9sNu_h{pq^ASg1DH%4msGU~n~G$QWhy#`C%U|uT0ewK~L=bW-#c~x-FZ&689Q%qVCag5NTjO< zjr=;L?!V;~aMtydxB{g&+6u*0_I%Mm`TD&7!Q;tPYt%WGMEzrS_Qra-CRDqz%&Njtt~$Z-}nRb`{C*X2?H>i&R`f2H;exY_|UZ{<8qIxlQXGxa>)uKepStLWx?4 zCB9q0XKg=oA1kYjvrcxX2Er^%gAyKkG);0YH}F=?I<~&-cNVLvR^iK4frC}$0{n$s z2J0zRN&+NDYeba3%|LWR4ySiPdl5~KDtP=SMy)n6LEBGZgXJAz zEGV{9u2t6Pa;5ttFdB;?zurt1F9z4X7`Ssp>uKQ5qq`y7zs90@TwHJ&Nnr10oBHI^ zdw@5G)Bfv@EK}XSaUflIty~UYOa~MQgi!Ds2tL6{U^zVN_YxzznlOq9>V)-of_V`C z%hsLU=AcRfeA0km2`8W}Yjk1_3P2wogKz~=7MDTYg9nMPH?MCw0s z8hHZIK{0JQPOCr#M6_fx^wcYtz{_f;g5vQ}NA_cg=NvcT3$!SJ&Em3sED z9Ah;Q-f)ipBq>I5GDf#-J(CZ^Zk5P_@5Xh)pt=^Rv%2o^rD!rjGf%Vj5P-hDA!rd z>)UO5D<6&@FBhk!axiP&!!kA9oTn|&Zzro79sskt9KH-iXcO6!m+SEJ4obqL>u;(w z5-ppL4X=;Yxm3?&HP$zKP1>!0Tf0A+vyF}6%1lSmt>?`n>{|;u&lW}rrA4BajRm~M zB~d1V%;O(Vm=$$eB#L$9w>qQn_*z4WJa?8l#0KZt{WT-HAF-AC+XzYa$wG|+GDhrw0x@x0TZ&RPI?T??$>3ix-o=292a>$^ znWV+DD91*sArNh|_|~w!yq>kaG}Xzga$o0YY~)6}Bn`fX$dnJE3<&}w1OXoiN$M{5 zJK^`}Ko>+M%u;+6#`}fS!>y^;=opW!R;$TmMuQAIwuDt*undn^{qNgj!Q!zLkJBX` zUJWhf`0z>RH0_s{gJzGq@>DOifW7G^IMC}c8e-(tUTU5xxUWT zSM8NzZ7OL&8pe{Qin{DDa8b<|g$Ky5re(yJF-mjSlrehyQ%nQ!`VQt7K|V(Kx)X znSFuZNc@KhVH91t$C5yq;?aYut?ijDFn{V214e)k+1oqb@pOhL8a>rIPqd1;Qff82 zOH4&xb9v_O@o~7UYPH?oN_9*ZucR8f!m0Q8BDt1*KWjoa7&cBt(K~9#N6#x>|(n$*3Esa7!--GGFSbX~?PjD$H4bSSunlEc#<>?0P%VfEO z^}*w>Z$RcfXzyl7jj7 zhNrEpa%}w-d(EczU+$NM_j@tR_>KoMOV)wecmM~3(>Gl3wahK2Zla^bZam4-mUC+sq&M5w` z^k-{O-l@8kqZPqn!i#N9I zqre-0O1xw7xf&EZ;kmjZyJt3STwS4gqWEJ{`VzXuemJQ7g zPj+oIy3b{=PX;AH;I-1TejJMf;g_?qxQKIVvdcB&4}<8(7OPOan{cA{P_l1;nO>ml zc`uL0cg06eax~EiXtvJU4^z+Ui>NithJWVOwrt6adFG@h(_vR?=vx8AjYN<#I;~1j z;CgKyPE%9a{}ryK6J{M*W?NQ#&P<*UL?vW(a6NeG-)!fpS@`TsFAbxw_sfcy#Hh7B zx9`P%UXR6R=F@Zhi!FKv1CHWKyc1g8mcA|_@xb!XYG@*u2@&HrL?xfM%NIQU_F~x3 zC(1B5!o{6F!E%e~W-8H_Hb#)mAO8s~`&vSiG4wE7a&z;w*T<%6V6w+5;(8@TT@|3-%B1Gt**)+&SnVQc5Kc zvxX+67_V|XwwU4jfQ`ZURkRz1_qUo@O$&Bl5dY*?6xcNS`rW- zIvd&CG53dQIzS+?PPN7ulQ#QDi#_FHm(O|rFZS&7eZ?;$EfaDA08;+QOyO=aeYEXD zEM{v5kcuV}jwwijNP;IH0W?|~-v>+PVNQC5(A(a^K$TVlf{iszG`y6Eld5kcEXgeZ zri_ARH|z~GHLnJ0$_W5Rl}RC|;B9{dIx&3v05A%@QGPsMr9_-;h~SQq5_r%>?4?DX-I63Rp<=_((}s#>2(-tcYapdW?kLtuWJ4%sDW$ zUKr3F8xGY)kW!wK+z7#+n@P@map=d*ER{Z6eOXfWjPwKi z0OqoGi$b1>U}eX7yRBK1#OAGY4AfGsK`pWWyReV(ljZJcSaf8ZlCEmDcA9;4yV=>9 z+#asoYKu6vi6SkJfc9Z@stVxXnv%vUOZJkY^*$iww!BR!I_?gMS$}=dX|mt+C*(^9 zkl;B#F4u|yTSAI$To-Kd%QzK;5^NmjC}j*Lo%!w0-^>|IyC!jXO$Mb3ULHntsxyH|q$Y6~tUxbxPFQR8$ z{nb06qG!LJ7)bU1UH~N@9Qk~Z%rk)fl~4auhuevK{DqkJbEXQsp^?#9ucE)AvfbNY zO{a-&nS*!`pqYjd3^}_)Hu0tmL^#iv@UOK$B#)*2oCElb9JnXo+)naxa%@>J94)zx zqqxM2JA7>p+XBF<4*B!tF9w`k0NdxQR;|rFkI&8Va=!FY7zWR2eL9D8x#6Wf6eC&g z|4~S*WLP`JTig0?oAK@d_DRh5PD&{01$j=)8Izw)V~aPkeYlG=-xs&@-Fq*#w*zWT z@Hs1XAhFAgE7kS;J`zvo)qvQ%+viqS$z6wR%G|JUi0i}0laCq{zR8x=n}a!` z8rY$V0BE$aZqP20R(9&Uy|pn{1p72$0EIR!PLS1Awc2@8mQTbtTi1~^0|S7M=8$jCZLvo197>X zP3xiM^05*DE06VHBj&wS|zF$Os>Hmz>M?{G-Td? z6xxcFm_5_3V?kpbh>S^$FVOgmfULtJCu@@CZAVM5qd}!xer~N-3QS?ue z9NB<915kdqqi~a%z`}=k3axL2|Lrf9t4MD;%Zlo`YfU1k$#nw2a|7$Mh0WLfp9=`U z_dardl=Yw6)24ir!#3(;m@jlMMuzUS{TwgR^C{w-EeVSKX z0~pgh?H;w!sHH32UDg(4b@i-V1n>}y+7!0KafIW;F4O$Smx+9{xq%RLFo#LM=f2%X4IE(` z_|Co)(2$UZLyf;RObgiLTFm!2?ur2VIL*8ZO(i_=qF1+J`*UbCwtt_AuIY@*X1k)~ zvE^lvVDq5IcOwn`><{Ne$HvYSr>U#P;i+vt*9l<~OFR+td#eI+88b6nHyXLKaZih%kJ@%S zdz;1jcxuX4fI-S<9n=c^HWGfxQf*JJ2>ABHMY;Mkf7%}B*~dfOi0jaC9`#rEeK@|V z$-U*iLOQAw7#chXut&Q1_2E1#j&qyI0SisHIlD^dQOm-x6B04^vff^~)#4WymL8y# z%mpG=F#>Q>&YzxOiT~^re-cOw{Vw)twwEGouwHMdG*y4UlWc*E_Iy6nV*CV5N_tQv zpgd96kSD)2#D3rJc&Fx8YDf%|HB1A8x&}PHk-R)08ZEsOo-+KuZP6+uJQf@EM~&xl zFJR$AK-mrH=QZrZzY-xYja_&S*-s*-PgD|18`Li(ixO|7yvF*-d2(>M>`O#GySe%q zQIva1Tw3(Fe>N3THQ&4L5K#Dw+~fS&>G}pg44(QVih4@rCy^dlMY7=vvk8{Xl_H0 zRky$oo_Xaf+>CX-5^0hVCs6?Cx+>?C(?FQJZCsMbug}PDscN^uAS=( z@tXGxeSyYT_D94)sRjk-X=4oeq~%K67{MDp%Jk$k2n+>}QWFg?1CmtH!0UzIV6`HJ z5r=olQ=AKuZv7hATsqw!rYsUX#%5Duon~OK*_mB`ecj_$?WXS;2k|D9inV;2ZyOa@R(|<@mY~Mt@j(> zzPwL2z0Wvbjy1^o++8g0{-N+TD8-PYDJ!iID1LKzBzn%6ZzqL8Ph%M0&zrv-Wq3fh zVe|Y;Kbc3IW}a5l#ZeoMe2t@I{&KTlYBLzw#p6On7FWND02a=u-r@cN=mLbgPgNT% zZSNP*$@LVd2N>JfBn4%Nz_<5QrA1q36k^QSE+xL+_~Ro$v+TW!cF6Acso&S_xHXeo zg$%Jk)yGhb&7oaO`C$SnZZq0jk|7g078*UKrqnYQ7zzj+M6e5uEhuczeXAfhD4v@U zsnvFOmxfTVh~s}?^oCk>j5;W~LUl+=q$i!0vxO2KN|5Mc4x8hS@B<7u%iljAD3|yr zG~#*Iq=^|U$NzYmKd>^Aj9SM*)$t_=c1I|zwUMYFYs}g!br@JXJHWXh?yP~JklMrN zAMrIEk&h4O2!cgxoz&5HHr`Wy5Ii+Z=NjZbImg*@&LyX9_Lt!NO~x1`xTNZx7zXX# z_FX8`@_yQE+^8rb;*UP78{sJ#6HQQ8G4Q!0k0bgYrmixms%UK=x?l-->?2)sc$et%#WyaO4SRh%}2Zs_-y0r|3 zjjoUus1K4Fv8xl%SbD}~z7W*3aq3eBoQBwd{=@*-#uY{cT;?@cRF%RM&pT7>8T3!- zN1(3!ZyJ5Jr0=dN$y&dfumy@acS}d&E8{k`&+cNm0A3I>qj`U0E7@ys$Ze7A|eh@}RL9ji&V^nd(uHE{Y2>72TVXo17G>unA`;O+goK3grnM*D(<#|EVJnrldAu#vg+lm!vK=yC z2p|+&JL77)2xcQ;$w1jT7jdP3h(6V$RS~fH%-2sgBM@d*mCq0&NF<$NesJMFp8p9H zw7){WLa%u_-Cgje!RVQY+QYQo;5?fQ95%|HL5Els!?cpJ*7qzmcB_P!v!j zRQEA~AoLh9X4K7o>v7)ehwK#I#Mkd5-nUMILYDc6N~mI@&TW<&#p3|bvRjNWvJQe! z#(!LFnLe0wD>bJ`OE8dpNPZ9fY8B%PmgI(t;LLBBKKfU-<)(&JTDkirSrU#8hM>4}bvz{L3gToGiHcygX63mv5MO}?d z0%)(gVP+O9P2+4Afg{GWc-DIg5-I^7g-z{V1ak66BPsF(EkAc)PL`15K*1D)RgU4K zT)@3#yt(%^5xz|a9HlN_d|mD>7(ClslQ^tX(jNo%rp2Xe&A-*`m*WOv_gLB?qY3x` zXPQNXGaLtliab5y>EB}1z`GOSrQ0d>dWWGmF}w~v3Q#bET~3DzTRsR5V$?9py)w); zIzM9>&Mgmn;*Kk=8Z4{nlzxt-@OFm1|JE7OJyEJFHd*ZPjV^y!XQ!2c#hlff?KqUl zb8MKu8_eXHM)cGf+3emWqefd!NLoOugo4Y^n%B;oo;b&=%gRV(=o% z9ZFuDi98|bEws)8yw)X`qr|{JctDJ~yfsosvcM`6ZSg=mB=Uys7X?;%I1V=VnOO$F z0X484O$EVYVPlJ|a9T0?e2lP|s}xH@*F*Bb4OPJu5S50rkJf$%7Si1{FOxUCUqNIU zGd4}WwC1r+Tk315x2LtX{oolbYFyIF8M?kZtm)%&$o+?m+n2hR(iWf(nA`BZ#fCER z-|tpsVvcx=;Wg0wOW`$8#1JutIOkYjrsEcV5OcUUlI{mZtRhZNN=ESxpb=M1vxr9c z>7+(}V^!wUI!f!Y$%BM|HyoB+YfkEkj8syFA~z6ycABi2bG0it6WNl_@{O6O_uOZl zM~=G(y>5dQ?0yVWo#Y}R1{Fr~%Gc-HsrwH!DYA8b2#wX7I%RryhqK0$7uEA0CzPVn zbkWpmZ8G^DPOvOz?My?wACPgMacQwud$^jFur#hY;$7b>%w(78UjvOu zKg$IAb^6H2OUo79s4vkqKH+*KcG1cQV8vfdw5ELsh$3E|nO>LGU3XGJgDW_>Cyq(Gr;_NyN5dl=sbwC`0)6HE zpHT~mXEa;N(a=&8MPE1^&Bkf#8b5r-eD`Y}nGqR+30+CVYO=y1tmNe2IaqE;H=7ea zwA?sv?#L|>H&r~XCEu=81Z{;bDGj*6Afute4EY@7oi-E>lk1~$037Ivi6waNPig+u z_A1JckGJbfUFKZci9>Y@81MjSm+L_p)5VDJp|H3Y)ceghUJz=|g*s$haYAL~Azriy z8YB*Dy=e^ryQ46#+^ZQtv2cCln+u@wAJ>8<70Bp#U-EA($!Wu1GU?N3QhBAt>_?h> z=Y7;D_)mhJ0kpr_8EV;i*`8?`!Zi1)rY(^xN7osMIcOdrn0)oJ+t6Rq*c&&ZB!u$$ ztFcQ}E#LC0No7l(&XsIx*5@v)v_f@Tw_sg$5$yv!O+J}R@q|PcGlx#mQ@+ZK2<$~< zB46=GdZX{z48`3*>eYE}s^nS@RgC!4V~QKzPF{{sdpXYEywv^BPLJ>;E>P4oaOvU5 zy`u?3>G}xbH-BzUz)>F2BjKWfc}U?4xwlh9xc|YZ1p&~&<^qnE#XG0sN=c)-CvOB; zmZqJIzvLZp!#0!ZFUqP|RFL zWpR-%16R9kumB@?E~@5p*1tWZm~Ol7FuONN+vNeWn7I~8QvVo`LR_96Ho^OFUg>Pz zsbEk0@uL87^+O%`XosE*&0fX-IDs#~8(oH3jc4Zoh!jJ-xc$ll_HAPe6?D z?qK?qqOxWTircnRF9{k%9CTJWVsmdpV|lS|En(-v&t$ggxU|CI6I@#tmVu7h=5*XNV_-buyKqU;@i6I@aHv`3dKbL*7GJ>o^p z!F)CrMiEG4lH>v68QOL8Uii)Pl$%)1idiL!M$-e-43m<7;Mnf;bquBQK1Ctaxf;@FF6 zlJ~l@_ZQjl7*}Z&bCAuwA8h*f(=}DKCiVCNADeLjDFt*0|uc`Pj{=Ije5l zq`0VPA3+MGSY`8A(bn@HcFlq~tv+kM;#KmUvmg>Bv(Av@yUn}2X??@DS{`n2#FSTwFIo@@o} zk#FbdU4*5fvE)9M&Ndx4-A5&UZMZ$Ruy-BzjbBaLk#Idc`*#G5wM-Z)x*>tHr3cju zilw9$5L@d*@tN3B)lvEAg{+%yv-6cpQY?8L87OD7_w1W8exyoH%rNLHau%0Me@+_d zawm$BxkY_NOQOA`B1)Wl?rqh;r*EO4nx#Wv5!-YcSp4UM;J30e+K{l&pA#8;9tzFc z&1nshk2mY$Xaury4Nh6r?w>`HC(;C$^R%1Ir3MD?E}Km{g5~dy8eGA}N?vRbkCEB~ zm1yaV7Pl)R$SMwE@%g;yCDZI43ujv$mC*L~%{&u|dw((<;ZdqH^?tY2a=sy>P zDqwHgwX3sp0nD^mm8Dz4i(?Z0I;j1#&HRocOlxVx$x`Vgxf4^v6cUQ41rO>y*?TZb zuoerk7!8P){XqCfI^_y2pufDqU^KSWzsu*f)UniFHI^kobKF`OumN%cropO`2~v$Z znFM@A5Y+DS6$`RUx8}|uluy)MPSt2S8}#E^?Uy8>e6jj8oLJ4a@BH?wIpXy1c??}$ zDZnZf2ZKe;x>XX+clWL?NA?rnX$LmB8+V=9+4gV==xa2d$9e!HN}_=s$3nzsv_0{o zkrtq}Q^=E}VyR9ir}t#R8yv~#5t)lL4uT?oc)@QTW3HKAsQ>U9H=dQ1K7<3a-voxZgc2#zzyz zshVl=@p2(yi^b4zPPxKCYi6?_g^JXt2O~EMUcXIsImg@8epbuVZ1~qus8~o)5J^Fl zr&wS_E2yuLlr4EHKAvMe8G@Unp9QOfiTMM}M&VX;yEkM*T6R5y;oaGJnn{WE{c zgK>b(2IPi-XpG)}!>X9VZCfsrm8Nvn?seo?SPp4|Fq6kq4C?Xx45Va-twAK7xnSj| z3`bvPx{?54x-w9rHko?DGLkU+ltB&NVR2fnVj%yv?(f_nKanN-LVp|+$FtsUDJ|Yh zMcn27^l(J34cbS7zH%iDb$BaPKsG3W7Xm9P^iNf^mZXLSJx*)_aA0#dhgVm;*%sRL zIUe^{xTk6OTHn*jT%agF6}|u6T`?h%gs=>;n_Inw{)TxPei2svlaQ@1$LjmiR!7At z85qJhz5S5}s3ZSTNR!Fm1GL%9Q|}|fUUYmy6>fyT%me7Xi%!s6Fcy}Cp{Gu|_)X{N zL_u^wZCCz<2j)n11g#y~Aey)s=q+o9l0gqedXX@-emKA%x-jkAqT=c@Zz znR&XL>u|6vo=r1o74AU6D9qxAStop)V=?=^S7VuIrJB$P<>7fX%jNOts0y;cB&o!c zz}|AGkRy!?h@C-&PJ6i?gq>;aaFgdJKRRW`Z^{~@AO3ph@pH9aS{}?K5ON%u{Heis zLr@e%yce~0VZGU>2txh5#_|hHzRwtIj6_plZKVx4qJK{foWL0g@<$#9}Mi+h_a5 z_n@L+xe(CIsIH<4L6UBP_Wr}`=4AJPWEdF=rr6zl&`JQMxs&Pj{M1GFV!(Y3jVge% zA=2akgD9Ny%21_C89!pfPT&kchx_1T8nkmp&4vTdo)JC8jkxE-X*Ya2##=5{mSbmu z;d)TQo!8$u8w9U1viJHsDq$&G0T z4{P3q={Blynt32sl?@X)QuNQGb?wZ`3t8Qo@^DerV&dan$OFW$|zAEXyA;2KMXGq!| z#W4Q*%`Z4;e1@~kvV^Tt$g5QVw<2^sb*iGfL;t zIKx;`I-!|UX_z5!pAlas#SwV0abbTsR==$H?>gv*7Te$!@lI4>(^1)|Bn zGkq%6H9NoCr%fViK3r5UdY=6}pJRQm8!i|oSYy2I-0T7zvSEh=P%4x z+X4ccmWW7O4f}f)3{xG|xIZ0ij!erJPU?aR%)RlPa9bB>n;W|EB@CO_0IWA5SynNY zZD8q6GWO`OKA~Sji_jUhL7qza!)E@c-E0m61c>`Uu}KS|TN|pLtijTz)VGHEaM6dA zR9F@8167RHS35M)^w9YEQ)UA_(7qYHrY349cVCxzlyGijw(}=_j--yiP8h#itMCkI z(@}_KB255habl19+y@E2Rs5s%0;|WvpyHhejOu8Xe~)PA=5UF`{^7_<>(|M$b==jl zo z6J^(Q&6DQ4%O$Kfj}z)`P%cqwE-W>~hSb>QdUb#CGOO^1xI;>f?f}SON zJgxpw!%PYMEPb8t4FTP`_3CFcXe4c|Wp(MYluYKDjL?b5B~+#@r0@9qT@A3>lgipf zU~ndiGmsGw8fhq@oMxBqAPR0l-oUv-AQk*n+jZvW=+_KrJ?2@xsBZ#Mu~qn$`@Tv9Fn*JSjc%s7E-v< z?+SS^T`r89_;C{n3a+i)`FAXP`}o#Avbx_`-yI`_#So-+jNnAlgr9vDc^xW!eI#)) zqmQ2}6o9O%Y8e{~r`rXprho1Wf9l(tG-oyFxmK$g5;JtKra>bVZ4a!H1?W|@mn1Z$ z1UR*6HP>qtYZNYh&#e2yhC(9XD}i`EuFME&<~mx!nIdeFI|a441lgaonJH_#if^u) zCmrq_Hp4<$9v?vpx8hsgIg4yw!*lCq(`An3_{-t}Mddv4!9Uc}9kln(N35hdd%xcH z@>_Er^WY|P${W%t_C-kZLLs?f*=g#qkv>891_$`xVDU&NrFC|B{KXZO*7QckQgVZ# ze3A7l;r2N~W8JqMZv9USu#jvMMjlJapk}Mle$;ra#e2IE4YiMw3>hmO`96-)e5+3; zCc(i-D(0mEZg5@b;RKd$*e{$^Mfj8+mj(~$?rbimNQKT+*CrspajFynZvHKDV6<^I zf95Xp_=}p4k=H)&O5{l1{qw)4f%}}^K54wGxWvXnC*^D&OB6#(KsLZH+RH#4E~rk9 z=#Wbj4xA6&%VX0JD=*ep2~`u_=(P_(w+g&NF<%8Sxg(JjGzvv(aBNl@;}BkrJ8GFH zq4AvOBFB9!9HK;6$&3sv#Tu)>y1jc#jhZ1+*o^ADt^<=KUN_U*f7sU9bcNgiipB6- z@pp#h>c?W;%#jZk@653n6NuPLr5D#L5Qw}U73cZxC8^jgf@p?{2c;ee_nglF(=WymUBw8lI@ z`&2sRRwxbpxQ#B^;2@?2h13U*8M!t&cJ;wfuX+{NOzT<)fzBTabDzk&10Rjq5fT^- zlkR~Si}jTFT~gqi5d$${S&<5p=FiSP07^aLk~}us)|^>(Nja z;uUWYqqS6a&`Op1rOG68WigL;nH(L}(P3+MZQya@xV#G>9}ef%K6Z9r9>EF>IT76c z{aRa1fJmtgi%Oi5!+Lkj(e)LTja*te(jvLvPro}Js3>^cRWjQ;Sv2X+s+Q^`1MN{(PYu-C@`NR zjU3`UY|%)@7nJFbW0-Ao`p&ssH<7hJ9q(Q;5lrt=5D60mXmE0{wfKw!1%@Ge%-s$K7?Uy3my>0el2{9hx6&U=1!04g>^|Fz#M_ODTCf=ks5%;+M{5H_4c{p`dK~o zI{qUR{{#PlHtfVaeka*LyoSQnFpLL}KcX*uCgHI^O1- zWx5nCDHucpm@n7d=3__RD-_EQaC@DmEZL?$o+&cl6L5W&(Q|(mI`|P%Wb%)!{`^iF8K{jixe#&XztiRUF?P^u|#nXCb z)6^`kWf9q0ooc0^J2cv#4JW|%``s?*9}cEosv+@6@>4?A+$3W6Qu8xwi~LWuf@3hV z5^+z3IjLIt+MZ(EPzK3dKU{aSjLb$z-SrZ8`0=RDzTEU_z6#Mltfn<*YpD-4$5trb zg^DUH;(QR-KS%S1A^5J&9gGhfTzYgbUQA$80!8%TBJqj9MAejv3AK`b_{B8m#YIr`(Wz(QC>J2RgTQYR6>nW3g6sykR4OOjQ*TOiSMg|ft9WHiMLlF;iHEuC3_o{@Qmlp?;D5aBX1@yb80JBUhlOYki zP~iDEl?(Q+Jomzi2L4rz<+aZK$>wL7xe6M`f89kjThm@ovm%1B;mxiq6q`Ih0Fw~F)9_at$~ED zxJWp-SI~Bnq1@c@nhvI3etC;Y@406xn+L$LT)|&fd7bNIaLl^W=~(TN_`D%xiy>bv zzvl7Mk4R$?=ZP5NrQ=psxn=Km<=}!H8 zP0M+coEgdgOpErqT-4)E!`zel3;cBv8WH-BnX5l+p+xys4`G_UN$`+L;h>?(IPGkQ z^Y?13psOlIF_q6)v-+bN_R~?V{=tl`ETvGPO4)Cys1DS+H8avAwiPweHkS)K<9SVi zDDG~yc~=osB*7ApA2CqPwHV@C0(CQmguKO6;E%?u|QN zkDD=-2_{9@M{M@hzh zc^LSC@yu|oATY~54Odm3il~lR-*dd-_-<>JN$DUh#lxx5s-ehmN`Y`K>yyWh@CE z$lmxRyNx`T^Q~P_gFG6i?U_Jv=~ggN72!W1>#rbeGun2DQh=yxXdk+=2E3Yf?MYRR z`vD=L5_;e1)Gi|CDsMG`jYFY7Ha+p+!-#F#OWJoP;ON8g7`O^&W)N}dey8o1Fy*=5 z{xTo?s{%5#Vx4O2HdzR{Dw~R%pYjC??2et;R?J9=UhdWMZ?jg4o-gJh&pcWOD{?Zu zWG+X^E~y+=pyc_gCkiO3oq@1I3$ZT@l(k1XxPlOhl-{WF@-R+CJdcIiQx1>kN9n^k zMOiqMuHB5#vwYPGIzbuAgk+CEard#m!=U|J#a~WtzpJxzZ!J`Y?&HU1c5CU~K=5v3 zg1`W&U24*Sp{82LIHNIq&v(0lEq}sL zv#U*)_6cte}!`H&< zH|8HJT_Kz>2H4xFUY;gw1C~8k3E`fj(ZadT$Rh2Ur9U^ z9&X>`$*Op-?sN8S6QgJV1i2V^j`u)>9MSFo~0 zB%Vd62((crSw}`IrPR*59XC$zkgIrij0;7*77*foDNX?(AD7F+M`Y3O=z6c;LqdtH z|9!*YIq`-fP8qAU)t@ZU)_%VBdNyW+s=UcQ@8$!hAJv-N&y;6yX!Z#pKIYdOPV$d9 z+8~cC&ow&5l$l`BS+m+Vee&cTcD*tmB&qs){j0#Q7z;|-Sw7GA%`!lMNu11yy?R;g z?++Wp=%FF?=d)>l>_4#k=Kf(XN<)9EmMfbsRBWe-R!fMei)K*OkEc#gpYsq}1)YC= z@CrqA#vNPz9?-PZ%D`|@ljyDnpzjl8Q5}qhddC9B`W^E2H#;~quSkRNToQWfl>U7` z#ksyk5!DfW1X5r{P^_nb;4vixpuqPP-R-W1o%p0aHvU)Z9|u;`vnFZ|S2=SWgC5C% z5DkfkFv)jY3b@SC2v&)6WK$U41Ix*@P}Ey%NxVs2Z652+O$EKr9J#!q@Ejc>VobTo zH_#vX>|NLDO#)_C48Pe^%cNw)Y25+0FWZUgTCy|V)LZ8Xd4N(#5n~O9yW+8RGEk$i zt$oq&4#H6*jcY+qVFiMpbbf&e+J zhePyv8v|pw2fZN6zNu2?y#X|{@*q0DQ_%XwL1Ad^y#WG!rW%f$%pq9NaQElK{3Yfk zjY<&2y>)FG?=))_E}O+aoc7tJ!78sm*`(=OKLRX3ZqNij!r)ONV;) z$C7p%I-dZHai(W1rnWB=F$;N6>eIVE!vhEP4yDEQ7$(`vaPXi%Sc{SxIB? z%}f@+|Iqyr5)uXQSF>vG4Uj8^e_p{kxL{I+uXeBx3d-3}eIvkYlb%2|V%$Ys&DP`r9~HLwcaG3VDDy ze+=LO7vS5uz)`-zlcV-%IINx@;N`*SX-Ostjex_Hj$=U4D9t{3Gpt?*H2YHYm^z*E z0KiC?-F;LAm#9%%>c_u}cv9>7-E7jy6}&`j!keu`>w#Sqlo(vep<0uddOeUz*Z0FH zOO+;+uW7Uvg1XOGGY<#*|kI$Cj&xJuaBfKtZ3&2*${6f6wZf^)a z%$0`|ChVQ~hv3n!qXE!j>^y1ZL6+g8-~9{e7`|!I;&UUCj_%C~?VpcSU%rNKeQWha z0$K_tFO>GJX%yt9N{#rmV$jE>?=aJ@lx>Vq_sJIlzp|p!Qj_eq!Wu zx@Mr%h@Mo}e%L4_>@=i!T3j@S1RAh0k~5q#9k{w zIF&VQ1IT#x#Q;%G^5|SHtE;cqVBr8^|#Qu*BOCwnCYv%qR07)SCI!Izu}1 zZPzvHPrFd;qyle}MmYHTN|_@i;5qy@`)hViR@b?nAI8i^R>;@UF)>$pn<#1K_NR7J z;ZMHN#QoaW+D>hc=8a zG`SCNZ(9J0BQQMVKqw*T3+eO(rJH4B#udK|p5zh7jZvr}anfxf-L?0gBNba7WgsYn}C=`!%c|zpSTYcEz*N z{noBTei5wl&CvdAq^nutGBcE0u%x%$YiAQ6Nm8E|MCt^}26GN9QH<_O^S-u&-3H=0P>1A}>&5NN7(SX$ezO);_+lXL| zg9NsX6zMu%BvBX@IxY885&QTZln5P>`(ZD>u}V{#p9R!^Rj0K{vM?#E*g5lg0;DcD zmLKwOZeXiS<%RLJRH!C-!3F=;{fi{4v|a$0&s@nlhAA~_vu_#Q1sdy$(!IF{W-Rk7 zhWt`1uE1qEu zUg*>jcx()Bpy3%a|46140p^AfSQiL-t|lku3F|qS0-j{rZ0fPGj{?z%ekxKnYEW{{ zmSFIy0r(x4XY4r1K1`@^Y2jJ5oY@|RZA71QAT<+!sRJyx3Ci5l#)`y|h`1D0t zyYZW5!DO5<60t1n2a#ji8=^R5)*ZeMrh`=TNv~Ub!>oc=7Y=*%?iH6mMBbJ7r@F^0M zbS#y!eClj^^Dp<`?+%O;qUYlueVTL28Xq{{XtpH#McaNKv94VWjBxHu{IIGXMli|> zXJYfKhi$rWA{Y}FIwMu1&MqtveJ}?!NZ!3!a5_{n8A)^RiX%`iUU--E*Phg;x$J%2 zR~R5mIeOjy^NRA-oVGD?O=0kBiv8;l+&AA3<-!OzCrej5yT~KI^&*m{Giu}opO%h+ zy9|e39xuY??Ma~9hqZ(P3SNOW7IumHig4IP^%^E?Z-2KR{65G8)^IeWIx=7GlUW-f zn8tSq3#B))`V6aoa^?xFEIZ~kNjdAk7Udl*d zT^g_E%uV6Lb)zW#n4k($!7z6wbo!!47>AW%$pIl`<6^5>vp;zgWhFa663^i1uQJS~ zGSfWi&I85KkNCU3duWaD@Qf-a74T?uCA@C`5azD;uZxIXbHCZyc)BL9DFA|H)Af*T zS)tk>D?B(DoH8oFsge^wc`6L*T6*z~zEnwit3s>PZ%|_c7r{9;NuBVdTPSLt%~yiX z>w`EK%RjfP#47x7D3Gf3I2kGR6VXpOhAwY$_JN^$DX8j4^}5nvtBTbPJJ{GpR9^*N z16ELIP0W6~=)H~0JcLHVyxT+4bq$fRZUakyukt15o}K3CtidZ$pjW! z2HeSf%dv`KVt|+D<)M09{_t0Qyi1;%t6D0eP>J|OA~Uf&?&8>q%fG)fdX9@33FOk# zxiYC=RMEww&YB8GU%+}02cEBx7hGJI-Ng9;v05N=z|_cT0{5V)b~AMn;(3~MHTT!7 z*a|57N)t9WlD#}SBa+^YWE_(781HyW8<5;F_R!YS_Rw=g ze{fR$TxV1HM?^|2ePG=duJS`us+#jrT%QQ?hs)8g>X*&Wii(nwKoQ@$HzwhnD-nmi zH@11^?H&z;l1w{v&WGY2z5)Afr|ZTJYo9&aDeY$siGK5Y2_DYari0{#yzP|v{9kY7 z%lizRzDOosWi7exV>~{&{asr(w1cX&O4bhNnIN4rO^2on;3bX z-6dewGM1JG?(;u%@d>C|u+o^$dadCAqG`eo9;DWRl@JBTRiO-ueI+L4GDnOZ-?j$% zD&xsKB|5iBQJCL60ZkV3)xynBu0UF`p%_MJ$YOxkO>_Npf4(!!FRxAtvHluPy*H5X z*P`QcujreQkl5j@ebenT6im|uux9qY%8cXKq%p*J(m=td&aOq(Re63A0Se9s!%`^)NN`IJy}M(aLu38tY{(ZTHiFUL5f@ z7LC%q2{FAP?NfM}~#ykiN#XE18Pov#{Woy{= zRA$gjD^Da=>J$>Aw~1%+>k3o&E zXP0}pq+6I#$!zI=2Gi~hbBfZOo||J~0D!%O30 zpyG;<8NIeUQg$Q{#X(2|{qE#|=*{|0xKkR7@p3S&j>duYPco0NpFdfXb}QSp)-(MQ zG1u1*i0%YrcEaLG-NKr{gux-QY%||VF4*{cdzM4;X~@>-VTXtjs@5llL7lzM-C?+K z2PS#erM3Ad%bhR{5ucmvw07y2hH%kN6fqZytmL39{*cRW34fB&M1_WbhF4e*!0idg z%U8~G4(%W=ayhoneAJFda7yxeIAv-)5#oHy_mc|#Dz2GQ@taT^-dldHZ!|Jp1$LX- zidAO+*e%kU=;-8t2Fc64!u7JAaq^9{c~WLZhWn82h2wVec%?bMV+M$Q2h?(PJap8M7XL<98b;ON_fr1qt**Vppg zLxpTZO(2r5u}E==BFpjGLscP8 zr%M&t@a5=kiS0YS<#~S0w#AAUVb^_iN?BDefBE~zk;`!7pzaji_WglI5KKaQp?r#Y zj6He4KM z9EW)Ok3g>CoY2>!2Dw{Fh~I7)BRTitx`;w3VIwP$pyFQ<>3KYj6Osx*OR6s|ltQrF z2k0B;v1NGv7r>{OKYj{;BIWa~L5WBU1TOA1v>x}qj3IyWA^&i?Xk?T=KMXz-Jt>u& znY0x=zs@eLOi5^C)r&d|E^fE56*G3)BDE^{Iih^_6y)fA~98ldBq#O-ImJaoQ~orBXQ^ytvvfwGjRoW>I+& zBgJRS@4R69^^EOfAxkxAhGK7yT#WG-eW`s&_j&@_Yt*a)hEPt(yN-`9`3JmU`Rd{g z-&R3=R|4kc-s^Lo#P3m(vmeu#4<-xZR3%kJbNjX0u@E47?LP`gu4m0Z#GmLI6{8c+ z$Tt!_WoUcO=Xc?Y!JTc7-=yZ}3WsB={WFM3kfC&kZdnO5>2FCMiF)7iUZ&x&3TOcd zBDx#&z4TL-K|zI)ewX7~X>HeckgaaOpym*ur}sx6;rB(_UuTEAi$gj!hZs6#t-qQL zcl$S|&vaf-Zj-sA>VzNCd|nedjO56qZn(xDTL_eD&DFntKi9SuBl&NL^XFy=dwM|% zZ|ca|=4DKLt(Kgkn2SQ`3Ow5DF|E@$%{KfdmykUyrW{j|m>GAaD)?WI>Hz$R)P0Hs_XYFa!2e>w+$0oO`v-liDxhoKr6S z(5%($XJT8)#J_BRI5q)@lH8!~1+(qiXF*)EeKHuk1sja^0& zY=KSl%Y08a`W3O|)MfDcW!2Lv&77#M`KDeJ=xHEJ!D!GPJiY!|iMN{?LARbG#9Nd) z(B^dvp^gN~8orC*xq(1if-lkFg$k0-mK0+SQbmBG8mdB``qK3_4Y}x3Wo9*fp)I0i zMe{dmbJ}Ch{a45Nja^)*1iD3)&1KoL)B;08x^7`SD23cUAl;M0;b8bppc-yAg~q%XxMh*(oWG$eMyAL0F4&C)1^9PDVgf?o`|L@1**?3%VJ7qC4!5{Hu_RE z&7dzvJ$ulb);kD9DVBWaa5>%~*)M%x3@;=Ib3nq`@f}3vf`#I_eXhmX|1#vU7_c|G zf8SW@toJ~73Uze(46=|2YU|zpy{(;ppt#avze6N(>NwM_u-&9M!ogFJo{A@BKW8H` z#BiPN5Qg)Hd?g8~gF zlh-z!#zU@;5B9Vm%KvryN(}!qlBUO{m11G7)Yovl(Ys4r#ZsN5z3JtwM5}ya9Cn}( zp4xr9j@xZG{k|O30(sYZ5G-lnqenqpjAktJlA<|r^ z;FQvB{sz?;T^7MmBDF#WH8!q4M6A1i`OVdx|784E>J98Oh8j;cr)`cfHJH-rql%@l z%5jeS?B>@;6#1&>Jeuydip|~3ES5so78<1*wNY-YE5FSDGvoMIq3!b{D1tdZ7jJR9 zTjc1beR8G;UeW$jakkB_ua^io%Tc`(Z&^8Um8+MF*fv6}Y6`%iCQvzh%q z?X`!mIwx)MTZW>n#V(^==cH$eZ(>GMZ>=0g%1<2RB!cH_p_^f4`IY1k{RJ9x0iD2j zA@IM#2skUQC3Maqsj!V-TI)x_Zj_9|#Yu^TD<3w22ZUWo6F(v;(Vi5??GQRj_6;tH zrCPN`0u<5%3Sw)>>RJDaq_)4z|bKPWd@uLw)60XOfJSDd;~^Cj|lBVsSf zTj(ul!zPZh3(%rJCbq zKz%bU;2M)?;x&uUoHL6p_SCQJAFgU!{LQYFdh4K;3Cg8@)_n@N$ zme$@hO%ttnufpTe(Nl3^dLIDh{JY$z{B4SMXkSI>573S#r2*s7aoT(tb@oikdU}0# z$T**h=dHD~Js%k_k&XNo*=yXDVg7r8Pl{hMpUOaz4CXRF0bNnc8-#fRi|AHp%9Q)^ z0SML{8i7p|&MnGM9w?+?OfM6-G@l?8`fLb>j87Q$)>j_g5n3&b-#+{77U?fJWo6M2 zY9Ux(Anis9RPs}xoxuTktWpt!(x6!RIkcMX(T3T9KxtZ-z9q`w3whu z<_Vz_dvZsT)wZVoXcG3A2?!m51`;LUhz|kWj>7rYyu)^~<4^PNrJc}N z=psMLHs=FgJqwKRwS=NoVn;x}%*d2=fjpP5Z(5*6W-ZQ9^}uD})Fs-Ohx>jC%f4G@ zaf-@Z2Z3AII5kU`#!{TWkwy^>#^{ps+5W-1aS1O94{)n9tJ&?mc3&hSiGgP6YmXKr zM)g`NK{a7sr5F`|t?OzP)RyO#ag6+C3$;enoZBFp zVJenEL@KG5DlQ4wb@3C&-zxui1p##VJ?lz%KOSmwg>iHXhxlfHs%(z*Frjy$|nIo zY|)4&NXyxa#1aF8ecl`1x90RSPzLv{VL-t`;FAXa?m}UC%XEg|N|kFAvO(S?a;*Xc zj&)N_z*q54*CcZFf#S8eU&)AipMQ7@S->{|O`Zfhti_2v{~u|ZTjIcOtAF{ z>>=+MxqtG~c+B*iy$$`1EpH>6$8dPxlSrzp<$ z3K%vzkgj2UMM9q#!+FYuTrWe0AHr~))WXTr5Z^)f`U{Yv^)59wOB5)%8Fv3_cE^&4 zxs&Sp#S3aKk`CD?Fe&)rK&&C7s;ZaB^TE7x6DqNL)x5{6^oFB3N{LvFy}QP5na_Rd z){02nvjx!+K-ST9;~Qa&)@>sJo8;fY<-Cq9`fryCKNkUfU ziN8diDnw}Ez(_-uE}h}Ppj!U-rLCg#QKYw?Mz_jp6@moXJZ1PiQDmKKG#CZ=L< zVP}HvGWK}hN+*x*iuwwA>)=}WGTWYQPBfWMdz#KWH6P4;n^ZLHjcV>yw@&^Yh_;B- z`-hFcCm1tmQ~SB?nY={9b$8fq+2i$C?l;StBYN$p#EZQc2Dd8(n@6quiITH$F9XrV zdM}Y~WFyooVj8kl4n(9U{0J;WA1X8_;V`2#kdkCHel$>|J6HxVAA>@4cTB8@cccWF zU&4_8MBJdfmDEh}hqbq8YA$KI%OBN|wRo!{l2!z^i2&vJ++h;3RWepG6NUd{>aC-q z3fsM5K@?D_p(F$b29T2O?rxOsE-5MLZWxB{6hyj3X%Xp`?iA?;sqf}_&N<)vmn>Zi zXZGy5_jUd9>5G>Xn)?P`(sus$p$Lz;)i9m=UX{-kVm-Fb)+j^K%A(!!&nu19G{yT=68Xhd7FtRofv*iTs?l48Cu3aa3ld>;AmA4 z4q3t_cBbu!N3)JxO~VN1=7#uEjDGXm+pkw?6h1fXJ6cq6EH6_ zYirK{LJNJ3dqvyrV3BHX?*5H#D zN}$kAQq_7#dUqP;uHa|vbfk-(+!Qcxs!@&UK;D;yaAoVufl zct7qb4B+-dW5|UUo9DUK^%&=rem7L{zhd$6_x8MzXYcAOrkDR&7J2Nmd2tL5UTSuJ z^?#awm~S2Sa~RNQ6tpOqTRkFnmkhb21@*rV#a|y(l6`P{m+x)A)G9t%A^7_0B%$P1 z0oV9R(&cD@G=PN(`chdnzn7d^*Qw+Mu^F|iHC^ww*?GEfzV_iULBsxsBL;czJQ(gO z!4UZxxd-1xp46khkQNzxEaW+1dR6sZ=bf0A#o#~HhWAYL8~x;on6HMC{{H@fML778 z?c*tO8ejc4Qb8Swk$m@^^A>?ak^WrFwr1J>1h~idU*jdOEZs=Qdw&HKVY@KhP=&t^ z|Bd?7Mux|+Wi6I}zyX^;26=cEEwO{A9@hXoC=_;B>uJ;RdDY5ro4Ca~1F(lS&#R^& zkkVU!g%!IvWjfej&~VIYVKYC>O~{0Xqs6+G&w~}C{TQsB7Txk+ExhG2H-2CVa}eXA zMfogx3hJPafU9pM*y(gz5GTa;Ma+QUrBLNB)7vI&b(< zE8Q9&$JsIH=$7HM4Do< zQLR9#NsrnUGYmeWDT6MHcC`qxjv8saRG&6MM^`Nw`8MK^xq^yCw%gEj0fVLT)>gaO z7+QtFyCjP0o)mk@`!?f1_DOi13t4}!Iu2W(hqR&Me6xeS*$;u`;@6l32qh#V?V+}L zrpK_p8<-7W{L71K=hxiJk-n)bCllxs1z8rNLAPrG(T3geeHJS)8{~Lc<~h9jZv|hy zH0rjyu#e{5(SmTHh#S(?n|pU>fwixtd8NGr&bn!_yUdb?QHeBqlKA}x*Scj*C0QhpOImss^jM$+~z3LcUq1t&Jz`b0Ky z!3%+B!``nJv)jEf>{Z{pT;$S0z5NQ$ZLsAtrsD++dkqrduy_ znpp_5v;ZX4dV1-PLN2H#>b>zBOo>zWWt=;G$$8%)?r(L<7 zylZ~h;qTyZ)arSf~lb31$2pt zC^kUk$!19g#St1V{@C_b(&3znyLQe3Jo0rW;@6L5vu0g{Lu5=OUMls-w0ir6cEz{{ zoK9Bg>kjg*GKKRrR4VN`p$JDRma2Qa(9eLkThXj*o)8SMZ)E@t_!Zi+Scb^vzVIA#;C<1B(avVQ^qZVqA`zcqlWU%@1I3DOUR9ck9SV z>wN*@Gd1Qes!>Ae185?0z*6cm)8geWhYg`bs_B-lK_U5_cwGU$C{uT~1F5XtHtV2o zG{Ve%RV`&uWw>Mg385bLnb2!!1Yg&v?W6i@px}k^d{@pE;QfekRdQ9{xX89*{2+Ds zT9;!N;|mxP=jC>!wRWY3V=Fua{ZMMP6u}!tDt%&-sCTT2rt)x6EeZQMle_r(LH8Lj&`J~H@=JT@G zq6$Z@e{<vSPinnfP90;z$gK#-1&HLO-? zAlL+Q3pXCN@)VX|Z6KgbVbwmo7p!`8Mi|>slh7mx0UYI^VMfjQqaR>IlRQp|)FN2|&x;Ha^~Bo~NE-Xo)j!ZUfC z#DGzVYOcYaoq!tU8J$Y5e<@ihbbGPoyQ5=7NFY-y8u|r%`sJX?Mqkn=0CsQU%#mli z*tk!7iO7IJP)qH`_gQlUot=Iq(i!%{p`7c>9S~j{x_5Rt0=K7e zW1sqynyjErWWbl^*Y{Z!b^X-(!Z*jM+uHMV^qKR;LRNyU$gJ*v&R9x+5f~@zl0hkx zUzK3jkV1P#d#uJxzNvnr+(0pia>TWfo)H@0U8QZ|qPUU~RY^f|0fgnVF>n#75xhkD zvlM)!!j(@c25iGBKolDk_g3_iT+)ElV!|{E+f3l0z;O>%pkWbbTDRRXNT!ePXKtIX zGY8)5rz086HX7?u`ZaC0A?nKLvY)_k%o_Q`?#YC#iPE-#A@16~<$k z9+>y4lrDuSP3=0X_=qB6ML+Z3aEX6=;U+$dXy}8iPQyaiyRRE|zuVC8oQJk*x$5yH ziNxrz1rEzF_U+E}n=QA0sc|q&BOOd)Y*Lqsou@C?(Z{J`V6>P{t3ji6Nn^LndqI3^ zIMXyPxMKD*lGbb>wOg(2E{7b2i5F+O&N@MM+VdxVz*f|1iU6A3bXAAMWo<^@`gYSf zhZ-6wsA1<(Fe}&_4htSU9iV16@2_TZes@<}=+km;z`46-5uHHD-Td0m?8f+;7SH#p zdEFI*ri0DvX1BES+7om6?%o%4dS4wYLc!%BF>lqTl8)u@qCBUu`ZpsZPMj%`4+@e) zs}z8c?l;#gdgf^Lzg#V`2NB;IzvvYx#>R+uN@sdP39I`_hTaCz{dc}++k;`Df+sJ- zQVrstewhJv`ulXu{pw(plN(@x_?BjBym>ajKHLL<1ck4@txMT7x)Y)-kL&1wN;?T0 zbWQ()Hp4t0X*&^o-;zi;1-^~`816-@+1PigVh|>?@xn!LZxeA^^d~con{{e0Pm%HdWy* zMjGI}p(J;)|EbXZAl7wxAThK(nrQ{>~M@UW- zPr+Agslr%2v^UX5ech^CwKq}0TC*_g#SRt^}UH^;~N%-_#~ z9Dj^&DkTsJ55j|_vE)VrB{xE!JHX#hxjvu!*e)yc6pPHxnYIP^S2p2z>}3y@iWvCr zY`r{)6EW=pS~|V8zd6)TtYX7Ow)!=mAv;g$yEkzi+<_OO8Ypd45yG3sgiT#X1Avnh z&=L0y2Qt?NDtE#2WGTCV987v_w|*cDc%kZNX5^7L7XS;${8hj=l(73l6r1Z`4e?0| z-Rv+t0$mPTiOquH2ZnoM@eMUhHHhmNzaU??*`pf!-4Fo*fksl*f`^stgjxUh%r#Ld zJLN?`zmnz?B^>Sgzuvq~`Tpy}ej0x#uC(8O-wXfnS3~$1m%YO_IPvCsX{mR%e(nVn zgqVMsDAkmYBxKHB7M$9rdl+}HU!fqU+3n5N%FilUG4LPnJG$nsZzGQ1?KBpyP-86c zovrdS_c44oh)-}HyUB(UjWuc*dN%)Bdw;RF)MogxbwNEjOK0<6i?4-Gn&Vt>6;@kj zuo(~=Y(9eb6bz|1!NLKcIIQ0WvWen>fKVjU|Sz}Kj4D{K|}JTNYzul!GTa( zJfz|jKPtaUP#5nUv{L_rN)tl?v}nH;~rsC$it(-+2qKV=+0tq+3T z_h6w>Qzy=rY3}5`yRpjO4|GIx7G)e?rv-7?govmJqZ1`j31u71>(A$-wox(XI zhC9~eV%aF8r&RWNI8Ysr`+v0nIibN`qrp|eL@Y-B9f2*MgjdaWP;qbG+rcV~%<*=8 zz{V&^kF@{84gPgSLCVrtO;k-~?dP}8pxSUXk!U>5Y>Tl`=|7S2?7pH)Sq9l}W z6$>yYTqq^~@WhnA+r4r2sopV{nSvVB_oov3)UJ^9@?}wD3^^=w=FwZ^?)_>^VM^20 z^M5P!R$FNQtQuG#_!zMT(b0>n;7`sa;kc0raHF@#B_bKi0`oq=D`O5&m>Q7!hf5tj z$Q}DfKAF+!tY0eK`Cq{$6^!S)_UeMLC9)6vg0>ah&JGHcU=#FSa7-}=U;Ntb(}d2_ zY}#09HB&b{At&7;LM<}nx{#qjGIDbR@#Vc>n8N-uSyjORMStFKDbN)ast?6f;MzjN zMb8k@^2rkmFwMK)G`HjW!ILBbg^I&D>5Jqp&H zTxiE>sJMck(q3@M?d1}A|7K9eC$WHMV|k*e2`t?6fgA0m2d(t#erfcyHOew~(Klmz zW$&F0r>;9Oad%~9n$HHuOR_8}$k*4}<4|0f4kiP5BVlM(YI&<@MTiZN%R^}$^>!XM9ft^fY8d5ZD7ILFs! zVB#w|IcvBI2$oVH@QfT!!C!*b%q4_9??+O07WaFZa>o4=<<%g)(j}beMk8M}5iarg z_v=Y(+BbKUnO#5H#T74cd-qF1*OfV{+tz<>u2bVb8w#_TO?-wJSa|t?Y#p;r!kaiv z?etbjV!NwCIWxObr=ET@ZFaQT?#eaFu78mxsSu&TTNgCRUNuG>k&Nu+li5t9%6%SlF~+x1V%Rd#dq10x(KlD$`je)-Y%hMA0bP*EVq z50b~lp3@<^Nq!#nf7ByFA8o!$-7TC=PqN~nb@t=2T4sqCr=^BPFqWMD&43_a2<|)L z<chrrKzbP!?+S;_n2RuJP(kGOaS1A}OKnbUo+)d{y-*v3p7cQ-1HPMT+!*7luw$ zb1CTWq5ZiYvP*UihN3`e#951{Qj8 z6v%Ry_hb`0#h|q1!Il1L>T8{GB<{c4atVj?$jkqw5E;{8vnX{?`P;)W5f65`9tjW1 zi?Dz2;K=MnawZgT2S1;A{ae1-ZorxX+Eq)FsS89)3PtB0i4TkKYu&Ckr88tAB$rjM z;*T?}KHu>e-VNMX%POO=?C?0Rhet?PQoCaNN5j{0y|0c-*k8Yrl|h4(!r_e)W9nic z2kxT_9*&9_MAFgpEoap8^TnYq?rIya&R}$!CMEii=CjaOI783yUQ=|vR1Oo1B_Tb( z8V832Tr*5~0QUi!(bm?cQE2MpbTuRXD+3q4q63IUX)sLV@bvEU0}^F3BGqOxP$y&X zLl(72{fGi4aYy;V=wg4NKzNy_9{%18CAC;FgND;~lEX|L+=A_)=HT|Gw}7RZd>%aPm8_T&AzPnmyiNHt5AspDD>w zhxMRc>AwLL7FOZvI~osg-UAq8@<0~;B6YTBd4vp5TC>RJ`@8GV@G^rIorMX{1OWTzO@q{r{+0Z}PT0Gn2lXUhlok~LQtHvz^q3EBIIyTs62#SiF` zZGK?m`}flJc^(yb-ya4{?oce)fhd~AQwnq_Gk|S^&8K~aV=)+p&k36zEFc1$!{-Q_ zBFP7U=s&<3HHw;~xENpX_IPmy1qO&I%F779J8x`DyT4x!!lI<%n_a5}sRTIIaV_2Q z%q=i1p-UsZfqkl{3MlkyVh|+k0{tfmeqhg6Yp~N62M9p1yIn~eGivKOF*&Cs;iP09v3nwFik9w zjRt|5|l|afWSHJhpr*g*|lkwISEmrzb$Z{t z+w)6Bvs-M|KHp!+ATX}@5&W8td=9KEMl}V#PO@~Z;UCf+7fGF0$%*L1oR8OK#XqpR z(@<^@S!UwIw?6&S1%dS7`<-kJU&zaB4rSTqN2&5CP#0%d&U|$$kSzz|fu}AdU;*xy z3o+q@fta$=2n~YIg4x<%9rH@xjKP2req!PYQJxkcYvU`rB+zAZXR>prgn>sqjng(6 zO3hOz4B{l6jHgOWY9v#22~a6Re=>-O1S)avCI5#WjTDK5Vwd$2padb}<5zHzFr}u- z(kR}3@unvt(yno}wJp&(yHCULtBazBH7@ynpZDC#ItKTfe99YrJrbY0AL(}5CvV2^ z@R$we3+2H__-U@Af|@k=@|WfXcnBimgzu(Z<5~c!<=nIy$#BykJqN4Y2ew6YAqi1- z*N2G!#P1AbKXzfdb49>q(o-c#?qwz`gJB4(?z}gUX;B59y-3tjr6V3ko} zJ8t7G1Nfpx8g7}atBo>CfW&8ha1cW&627JzWzDiK++J;AeUgZXaXwrIrLkfRP3wqn zyi%ZL#6fOvmRU9ef;w3LBkXx^zT|}rs8Vv7f8No7SEp57;Jhp%x>mX!w~G=u1A%ZI zH7>Yi+}xZDzfVV{g@R0;fI6$u&hkFc?;;EC*68l=zy{0m+FY(t$~8~A$lJgA z^DzKA8W((ZYu2T#C#>o~LyJ^pCQLGF2#!mODSLMlva#HuG*-1@Z%n)53PFI(V`5xJ z=dxVZqfK$=%R^(()qJ1_3lAd1TxL;k2!3qmv6#^80YQKe)KoovWe|%pbY44PWguYkndgb^D$q{pt*@r*hINfdrX^TD^-*(7czB>R2qsHf&Nv(FX zv0@3t3{V04BKz(mdL=QjPpQ)J9K>oXvN9}fAkw@Mtd;!|8#P^#;<7t&-|gqP#3@+n zJH6+MwA+IL8FzS3B^Ax$+p;*jMiSW;+ZZVMT;~-SMMGjC2)Kc1%M32kL^`n6%72T~ ztujNQ6(l$i@H|chr`UL2l~F=ai0}gqdvbSqv=&-r3&?Vw-{!ZqD{JbOW_$=EoOXpP z0P?Y*Vz#39`p$fBOv?1T+yrU^d!R?@w^CnX@_vKbtULcfZF z!4aZ`Ye?P~(H@MBTL@BaoB=&bgCSs7qlGfEoBRIk-K9~bR`olAH}g!w_j-CmJXG=^ zBLE8owWMQHQ~iEk%m;=Dc0Yi$BP}9vy#e(@3d)bvK1>9A0~}DTspzwJ5D@_m^$~TT z%;f`BUD-nwv@~WnrG2%b zAYC}kf8P9V+11N6{}M3IP-xK6w(^|VfH7wTfk7@b@BQm0oeoux!L}P|hGK^84P_~J!AW?PRJ8{Z(9?6Z__ihc69z{T)_)411qY!S6z3|ruqF{ zi(19J#Y|IeWa4X8Ign>e?&bmzB;v3K$M2_wT%_YD$M9!3qSuB1u3q~P+t7nR^Hvbx zBu@3vW@k3sQv~twpzRl@@!Sfx_G?EFDhWBR{6`AXp@H4Y1|Zz%2?+?jWkQ$=<;ns@ zMEvRA9QSS$;x(3q*__dA#8?5|wscMKZAf%3k*GPzpNae0;pBdCtSqMM^CWj!bbN00 z@WWum^fCmYAtN;qP`X9Rux_z%>^k&Ca{UPaL@3ZMeIeo8bx@07pwGhK+I7$%o|3~y zB1iiiDp4#|fejRavaiihSU`N94yd9r;q^FF2nZ+Tcau}f9)OpXz33enNak~$>?^Ib zuq|5C8zlA7`tSTiVSv^pFAk|ZviTkn2S)R*6jiYc$N8^LQtBd`qY{ifmZn8-LBexE zIz~~M)=}2HtESV{MoJ$r45I_CGbdB%R`WwSjUs`37^c@J<(zkPwhfY2Dw%iHwZ9)& zK$~`1o2moFkNJt-1>JV*U@E0MKuz&RMCixy7JA*^UJM3g2$LhG&t!DfP&imkbOw-Y zPyX0fN&IpSZLhsJ901R5`V2061V~mL=c$QzL9ukS41jBtX{qV}6x~tl;ad^N5>it# zTWc8y4i6j8lfkDomdZlCegFDrD!i|j3f=a|*W-ly0CPapeZ+*{55@=EfjNPVtVq+} zte( zi-hG*W0Bx@k{sn~4=?TA!cZuyu`xAoy6=?%I$K^GVN+w?izY!&I(7)c;`@5ycg*+a z`^|>4*$`=-a(yZHk~R~UvXhmSwk$8X>w^4rf8OslhiEssE<;Fdf6tnB`igq+{43_b1w zXeE0dQh7dd;AG9B?}S-^I(7OcHu=jfVc~=l^%MDQfnpdxep&Fa`X8_iQti0zMxt{d0pz`JZuCAPE0>^x&3$M@?1P@khO@!io~v4tzDi2 z1O2wk64q!ITpiEx#qXY~#zeB5^rX;fE$R5?gnE76Wmqwa7q4HG@&}YKIhS6qVx z^dEG{e7^<)N{KglVWi=|W54zTpI(!%I2%wB3bU3Z^!}9vkZp!lhlHpCpc>$%7kwVc z70fda+i^{0tU&K-tBQBDP8BIpF5u{hnGItPObMUk<-8IAwMRHLxEd;HSkz@OvHcAECxsp z%U*qV*JrLTKl;7Frl{%wLPZ4x3sXtZBXnljXpb{cD>sz;0S1MNm`Jcixi}UJ!JwkW z+|oEm0N_R8R{hct+*KH7CWD_JU#8cj1p4|6%`f&l z)f#r8*0Xhpi3oJX=n0PQjua*uiym-yM(&@ zE_VA-63hbh0>y8BhJ!zyM_x!NnMP~EAzol%plZ;>*7xtm0 zfKAMR%!X8orVkrxX=h8?Y?i~Myva@q&nyTN8>nado;+cOhhh**B}Vh0cA=6wBOmo< za62e;6Aw#Ov4+CC*hyMkZ7NmrqDTK&M=*gE)g;QjOeYAQDd4FVVNv!-x!nzDO>$N0 z;g6<2c=~+lIWZ#_xWTN2|?EQD`tE3S^VxGyHTIJ-crPR82(UI2`Y$AXiW4P>n9>t=!$>$w84~&tnU-2Fw8^JE0grx};^o&WVi` zds~&POs@gGI5+S#)oEoNE3J0w(oc-q*~ZRquGzB|JxW1X@RyWVbkq@V^&?zw>SZ-5JJ z#_nLL(IJP3Vr6x(rtbplT0JKxjbfX4Z#h&U)`C`eQDLAuq@}thGx%;27;d9(|`SY z!4CgB@Bp0La;m+5nI?NP!*Nty}UN>y$Wsp8B%Qj3a@eY^;lOE9;- z;T_K49v(;l=#YdRrdL93){h?4G*tGmG|d5f2bXE}p)$IGCoCITXdJ*nxnYYrSKsG? zNo9H14}ML#*ig31OWu?0EjP>{q*OxyCBE0X)GeiC4SQVvcA#Em(in3fbLl06rJ*hU zj#AYQ%fU-mFWoSo^0#G@1A@c#eN%j-ec<<)lv`kWRlHrCToG|ebq@=YnVgt&XcLTa z7p*4Y%dqbQ4<+31QU%ix+gAsqOLB6cJJw}^_@&cpB^hV_Fd&q0*c4bgOk&XN0}q1i z9k*3|Q7JBmTmo(|fQiU~^^6i!mKHa00hgjDh`ibWsZIUye!0?dWSul2r1)IIKObcK z_v}Q~x0YfnmYvf0V`6v}pwJNQO3ovDPzdU(TW_=R?HtUUB)BYGOP7@ow7csb&}6e@ zBm$k9e9rZCk7iOULU{5X>YOYSk1ScZ>u$c`k+oLiupva`y7jt5ts!y-ZJd0LYxF0D zoOe5gE^u$!gV}kLD(<}Jj?44dtbo^KEkd~ugL{7Y+f%yL0#Bg*|LSXcqO{H`WP;rU zZq#oTqKngZtK%j+jrYg-M`EV^_I^|s&s}S+#e#m z{(7~Up+7YeEc#`{V&)ovSr*RShsM``ODh$$EWKi?k2|^(hwq*ufh^`yTh>Q}UV8vr z;>hrDZk}Gc>|srrF2lilly6&!^8)Eah$|CCd|AkA59EMxXY_-}`)|HL5*DYhH1iVs zXs==Jy^Fr-P&EaY)kyo`jn)*owOsTe~Falm&ii^7io85ywmJ8@HDP>%p2n>To3Wy{3J9?jfraH zjgF45-y+=%LY{eS3apu7Pil5Df2@lK4&?c4=+{i8{=gmvucOJ=bb%6wK?1?sV;t^+ zsjdtzjD8!=mu3KfSYWCfhGD+aoAN;Qn96i{d9~p5>(%DM`YTspdz9o4^E}9M@c`L1 zokfehVdY$DoI$QVykSoCQbN%S=Q+I54&=5F+wv_^3ydI{>FbV^vj|R*9{kDq_H63= zpz&sk%PTTr-{$SdC-HYR1k>L-4gTQAA79@-40b?%V;QGd?OC^7j9g>$WDqDJ`JgLq@D}bpd=4HFMbO0|I z_ov>&yENWp**OV(@_c@iMRi1LxMQ-Rm((I#{Czt(6;1@*D&7tl|9cTH>Q-mIV}cp< zcnPwIdcCJSxfjo!@S&9?HHQ{)Ki6`xXp8AJZTR!cq0g57*7w1Hnp(#}O)|d8_mjkv z6b{z7pQ9XJ9t6~);P(Drnw}AM#=dBOjFs9#v4nYUWz;R?|5%Y4R-G=$`aTT(=cXSp zpGh6!T|2KGcK;gD{ySCaQ1^&2d@?giGB%>qf@aoyeU55lL!Y^SJ9@mRBud$Uv2E*0 z-)(&Xnl_VmJfe-&d&|8*zW2UR^7aVxI)i|M_L2fD0fEn75A1`bfF(da@0S|gZskI4 zZd$&h3JY)oPKmzDhdqiRgd*W$k_g@g0%{erilU9w#Ocxbm^^dSpm)^8Lk+eJ*Lp~& zC{Cd;cRGY~MlKH=!+F@VAQDoTgPEnc1y9MB_lYT6Eb!vIr7jl8-O4ulj3miXmkN@s1DFE+_cOtZ8XF-hCB;etORJ$e?X%`eB^k{>we>nl_1oO(-L*H z^#S&nWLkos3Gfk`S)sa_%I4$9YUE|IXX(gwJ66|rVpesDRyw1nCdX0dXbecth>L*U zW9g=mJ%D2pTT6{6sO3ZYHITXci&y(s-mRPTBXtgiX2{06 z)&{N@%5<l-_!d^Ub3c^+7Re!E4?mKVYO#cGJmiL zUQLoNydNm1BcEbrHlZ4RN`h-~E@Ril+7u+B6^L~7U^!mEe9zkZV;=Jb_xt%jYz5uB zQv?H?yzP=}k7N&Fu(}xSS4%p73$Uz+GVSN8MOi*S-lMr@k%v5Nw5F6;5gmGP+f+6- zE?^D014u{uQu)&#fKR-S6r{WsEVG;@Ujz*&PpNcS(pD*E6Fy6r5HLU^P$JO{!rqp6 zy5VN%(PUVr>KWkC@D0psS21h`cW57Ch$@D~*wj5|-tCH`?Gxs{Rp&BuU@6k|tUc*_ zLPXfZw6#QeUwM1a-Y})(9QR1*)pY2b(W9nHRa+S@luI7dlik5ko|*3a+uP?zTo7!8 z$NEb(Uf5G)D|V3cma`EDU$#75$mDhYILFnio)?(8^f`XwY_(aywiAW-eE&P*;L?e` zT|F($`HQJ^)Apj5%Zt<)fjdc0EK<0-WK=rFhxe^>*}=g^cy{SC^J}H!g|fH>bQCHm zZEG2(&xf5^`diL@s@hggtbELmI6LFe#r#VkB+d#tN7lLiPzy&mwk`!N3g5CpBrWtv zs_<~8Ydh=XA@!%F-Ab0ci+4O`-2#K-&5Mh+5ie^tbBW56eqVY}!7i&-+B)($Me!&; zT$1u#1j5O@skw`E8rtM!Af`UyHq5C!=W*)FXwP&olz-=1(mHyqVc|m+ z()GD1H5o_$+mHVB#VF=QhArIHzr|Ypf^6vZ=?FZgbg0jJsQwpnV1T@ zTsrd3P22?huNJ`c$fTF&ws>iCRGv9$5(JGhu-4cc&yM6FK#)n}(>TuejNG0QiGk7H zAc3gH;m*64&q?ffbC{W0w;_?Mago@K$LD)>BZ&zICLD_c`Cb3Yd@4VG#5jLgiINKg z*F4vBl=>Qy=yIGtt(6p7qUm!T{8=z87@?JNLLHHj(15OemPX_!KxGd?NnmT$zsT4v zw6#i>uLFvH&V`aw20zURGI2nCG32zCt7HUK<92+x7X!zpcnpy|eBK9OUQ=ZALq^la zkfs1Ewg9H8^gNG#Tf)Emp86~?-Xr-c zPd^Tii5?>qBjMzJ>(t)MU&?5^P}ce5&u6{Z@zFZ%UWtvgG#7oyLEwu6jB5S;~_tB*l!)-WNhdN!uk=k%mdkii}x$`=X zq&x7U4u^-T+JF>p07tdIL6q;&-wVx=KSrwJsn?ECz>yX|zV!n6GfdWCt3OoLSWXG0 zhe(9f*YZ)&I%&i?pB{?iBkI2B;w7!GXWK3j0gp1CI=gmJV#}IDYk93iJZd$Dadrz2 zq+p?A4jsf1QF*priVg92Ybj+qg_?1{XWZLwTiW1XDI;uy>t}Guo`dJuX7hzm=g z(DgO2r02KY{r&c#O6-X>w}nEi#6X7}6L&C5%+pDNbJZ)#BoFLJY0UQG-l>N{Dgz*-SgG14!)i1z6)cQ{+XS{@Rt_4 za8>WHtl;44RfiLhjJxSi#_q=fD|Giy^zG%!VfzS%8Wmp)D{4uMlXfTOnVn$ zrLA?O4oj4K4AQ@{^9p|q6bdMek{w5wAz3x3Ymd-f7y{#$=R_x%9M))5ka{&smLL!` zf-{k?0zsssvaFGrDAn1iH(Rr+B3)V(ybDgXc7z#|;rL1a<>fOn4(+vpH* zK5!v3t4HHwGPWhueq8dIz6n3|0cS3<>A{s(b`sEeCq)dM?IvY(SrX!XTT?h%s3jcb zIBOCV3(_WCcgit=lhSKuy*3{B@PZ;i% zSEh3^0Fc4`XS-Hvp(kq1n+`vDUHTD;cILj8RUNQ|O;fteZbs82WXlvzex8w8o+f_X zXJQkhBfTc!|Kn+9laoO#Csfv47XAh)%qd?jk8Tt~3{VrXAMyw1A|hF{u5$m!|MiB5 zP!yl96DxXqMX5LaE_(5bPXj27h?X#^t#^PHK*KR&Y|kU)Rx$CY*hUz@)?ehfE}g)H zx?C++#1AG`HdzKf9}JpZhYvbRgF37+ohgBjC>Aq2)m zFKkpUi51e0LvAkDa7y%eb3Pm_Y77DQS8n$23f)NMNHF08B8OQM>_d%!C!b3A zE6m~2DshO)PXJfUmI!664!SI*9CgG$4!2#c+9TMrkcaP)*EE7!{%Mxr#a^Rj|KSP= zpc546v<~Y|x7GDP_Z;1>N^D7x*>P+zJjaabok?0V+v%hWB?HbRD2t>o;beKGID&?4 z9uuxI@R%AV`lE%&R<8O}^*cD%-{f=ZP&Cl^Aok!}0^4zx#T_EB z_htf&W(+WN9KhBV2JCcPbnnWv)EG@l=mIX|BGolIgN_5Hz^w>p_yb})0m~mUL6jH^ zgP|y83(x^C8pZs~Z^#)0zNKrRWEVnV@aFj!PzrR)kb5@>`CJ@mD4;I1 z_y!P!31sI^oU44Vuu z*7911(iyfkfA}*3{VA2a_~SvCp>4;j{h&?6{+EVb4J^iQKO)2P3up>BNJc?|MByJa zSFHeSicIdMeYH<=pI$(#B_8D;f-RA#^AIqya%W3n#TR_pI3tTP-@rKKfJdPWU0r)G*hc8?ON_!>r+OPh5 z4@Zi8p~SW>J_q?F{SMNxJtez#B^qBH2gF*v<_dLJL9q6JnG2duK-Qw6zE5S>cNyda z9p0LaJ65=$U|ZaD$PEyjJpk@vCe;5BKPoc)aa1OE<`P`n2a0&p)6i_?0RuXVx0KgWLog{Ikq+rk`pj^5w-!NlxA8I=IYnn&mV1mrelR)_*qAVcC^$FpG; z2l^zN+mC*$RJq1}xQ0?Irz$`0LW6W{1aal24R6qo_&al0fa}H}fa3D!Jt+4)`SLDl zSHQFIsz}ko?K8U-}#Rc4c38GgRF`4ua>#M}y(o^2R(Vur4= z63wd?MQhBr=z@3$Xj)pDanP2mOeT%4#Iw3qwH&{@CRSw^1Rjgkcu@@N5m5eRj5;?1 zE8K%_xANA>Mh4KW4hp|P1e!X~ohxBtMA#3Hw>?&ut)l==c{clC(;8Rs>g9nl=Y{b@ zJahC*RS|6hkJ;5cX>YLNh+QvVz-O2*QdgbM?$L48%8R<67IE{cS;H zF*)~AQx}3x-OlHwAgf0Ukg!MOs^p0EL_^re7NA}-4IB8S4upgLKNv{FVxm*|*f3mS zshTmb_$T2=hDv)Vbsnd44jTZ#rRJyl!7*yLA?I1|Y`*5T${5FJz<7Up`1{Yz+0^lu zuzRA}WZ-8;SmV?8^hUqvrKQvdTf|51_oQ5ON1K2425J8ze-Pzk#HqS%1A$=})*a2y zI|D*dYH5lsel7xick0``2obMS5MH}Q0R!3WiHru#)2Xh!{r*Ug%>eLL1GHsubo5QL zV4H>~W8xFc3Vmq5l||>o?*%zw;3cxN{rbSD3g%ylBn8B26nd-gEzZ?}pKf1+J-F5B zblr+S?3|a*e;D7Ok&Z}S7dHZ;7b!Q1*#{$Nm9_br8elq3Q79y(TO5{zaLwflia&k! z4|++U+VD{7aRsnyZwSKv`n>lH6|<-E?U%1MjMbKL(Y($Q={YX3x(t=RMoqs>JB%#%Jw(pkmV3ufo1MwtXy=VW@J2*W1-6)y;kgmH*wH~tJh1wV z55BzzVzO>BO4cUP}09e2ioMCNNT$Oi;{_X^@dR#On9@K{W|Z3PDVj)7eu`?{EQAK2`r`*BN3(F%l!&VWe<*AkoQ-%c4fV|&3R9D;Zqn1T8B ze(Ao28Z=)hxgEGE*SxKme@7(Xk?~=_4Cf=liOlh@I2q*bVO(0LUsiS*Elj?7TTwzi z>?jTXe!$(^8VDJ!TWnM8oGt?r=UVk?EX-u~BJmf}HzscHxg3|XG4;=(Q3YqtB z9zwPM%{hoI&$Mjxf6r#FcPWk@97=u~yCg=Gzzrl4Ag_A-mz7|dIzNs4`eyqDRcas= zALijr6ipzk1E3KfAK)se4N-81tc(>FQ_t~6|1H;64BR6<+IVa;3a-_b0vut)d;c-d zfjzi3zFa-wiBso+8PLghybykK0d&XDcT2`Ws(oe8pV3c}>qVqNpyM!J;8ca_Hv*RT zl~y^M5v2Gh%fD%Yp?g*8$D$xl8`h5g0e#!BFXMa=6eKQw(BYC{VWW)g*;POSj{yYy zY}pY|OXwsL3STSGOqJWBNgVlH7mhbB^_iaS$j9GXi)H#qXM9^bH0DjqV%ox^2f8nU zK~|2s3VruoIA`pE++e+}xlPEkrb+N#UgJD@u9!O>ka^!72E()TvdqSI2u8t`Ij%CM z*@VH8UwdvcAmL5E&G>;bE}QRw0V-?DO8k2Xh_iqIF|FAcUIDBUJ+Iv7jTQRrpu{g{ zvn7Tk!7N-xmy4y10^-t{Jdb|LORh(f=GNI=k%Q0Z+p}aW6fMAq=vv#URJMMLMQ8r} zfBMmIE@a^95T{7Co1@vme1+(H@?7pD5T8I@LNK-bdJu$q-RfO3-oQZO(Dk`& zrBqDkWY@oEAyH|#I?p&L&E6H~#!y&O-r+T9;nJWzS${nKhO`UHzP{T*`fR)sun`}B z!&Q{qd~X$oXcRg#yTbobz~H%KpLoCGYlGg0-NlxWkIy$Jc3=eQ9m?@6S=2STQV41~VIkD2DE4%l;18@^~kF%FcZS z$IaJp-5&dyYL>@I>44_@8sr!xf+uiV^Z}Pw%)@>g}oJ; z>wm?NDu6m1eTwkljkB^jZKBcIksgu6_M_&Lp@eKfuaCdo#@=k|0@*ja<8piO1^Qbt z$aW*s;9swk!Rq5#TXXYaGIw&s@kV3c(tWG``>%P%=}^eLqu=F<^!BJ}tlh#tOIXeE zMzy^?qo%?3f)mqMowGOp6~cp50v6;BD;eaeeD~ML@#289UawN)O1p^v6CuS8e$F#jWdC*)&jx3mkP7v$^lnxq$8l&-JUVO7@6YNSR5Xv;03-$57eiYAa!N5Ml`{ zq3Zt{-L(l7U1+ix%<+_Y)A}`drpt;9a!+~nx77Ujx2CYnyLSzGuD?A&pUv9Wo)O)G z&qc$$Tcj9Q=`4^4A88h9|HO&%QeUHLY2lAd&eQ(Av6Y_%M{TujmGDq{eu=7?W{!i= zCQ_r0Tu$Z#=K3?F!HH~!%K>!!PmU{{O{Fq9@BSAl43q%7MIeftLln^iZ~oNPhD$Hn+ky923y-@nmGhmeSD4#%pLmA&^CQQ4C0k&$eL;#k>{ znMATzBAe`t$j)A6@9gI~pYQnnp6CCkzm6Qo`@G%fzOUK2Mc zQHs`th)2wyg*~(e-U-uxI9p60;8OPXo>|B86_K`(fn#hvF687rsD0tr92G&sH7tV_ zTjD~m5Y+Y~;|=r*GF^SNa{v~9Z_yRx>T42WxLtbr+jO6S!i4*d;#inC-USjm?)C=f z7$nzT%^>AD)N{kvjFJu{(bydy{S#YOkUFWSjP|>bpl3e9-OA93{-{aLrWX0x`q}7=8_IMh2nd zpZljRw2p1i-Eac#rh3wl)rXw)JM)RD{Hzd8W6f_v|A`#euB*b~*EMjqx?`%1d;v{t zTX%}+2hikm@?WD@;Xp=mo>VTT@&#ewwWT%I!`ZN}jd7qE$k$l4W-_PG_&0G{1r=!n znjrD$M?vPbZ;ad$fz0xq_12aRaQIcXfCG=krCFF;A$?b?J`(*N+gIdl!zmv>Qff)d z$Te!_7aJ9Y`a6nmu(ZtbTFmU&nbC6zRnhuIp?#IgP8^Qo+rDxWA(jTmXV8q8mn}f^ zVXV`&1H55T)g6gKq}A}`Kr$6n(Es$9FahAOV(qDcuRY8u@>8?4p>C#Ubf3b-DtEDM z*4D!J>%cgO1u=Uk*n1KM+GU*qW1o5^a_qawVhx?-i>YF7?SuY3<7_iL4BR z@85^yra^eq`are@V3_k-V5aZMp~5+AKQcszCFSIS9WwO07l$07L$rKALxybDy>Ijw zPm`4Rdpz_>fq1+#17CV4LAj+CT(zj@P_uiFeB=4@m98U)x)Zne>e4K7X-lD&DV08` z*vL7`27W(ZVBR@*3oo{n4;d#d!02W}FEhA6r^@yB08K&!n-yY%uYT~@|Kd&LA$1MU z=4EV{+Wf|JQ^9wGnf(V|RMv>#l9u25P-rtR$ja<|mh}<{PVSCweEdlnEYVng*%$5_ zxlg~uYFFpE9i1mu1|q>PUY&bA!w>D+Qr%fOy;zq*`z%#Hx6N^{Zq;IZ1b_dI-E+pZ zeFdd@Y!)AnJ`ifQs3aMB_VNUR{7^#UvYYZsaK^D+zhA&#`!H_^O8-LT2dmyKBltd< zjN)EPGa-b*C9=g-(Yua(!s+Z|K!IrN3D@lIYsB0=p%rDK~4)VdoCFenLhKN#D z!m*S0yv_$mYvvT-rPnAzZcI-AO@+q?OWDS19NJ{Hpy&K|lPxsgazoJX^ca_bAoF}3 zt~Uybrp0o4fg#cOTDHZe$B+N0H9`AZasNM#(cC+=wTzF&= z?j#WJ2EdO|N$)uuQ1vRcfehyIHQSoJPLDSrQOALRAPCor2AS=M=>+-*7nX}38Y|S^ z4+&aq(5hY`@%6bJ4MH`!e&(uxGvDRSXOo-4Un+3+VQSSZ1)V}60n%f=chd?}U z!_U0abTxI{u)@Kp5-L9P;i8Ar!QYqp;8x&2_Plq+F(w~A0-bM63gUsZrc`QCPAvT6 z-=i2TAHVi8vc=r?X9(-o(V?*R)&O{5xC*yFTibP(pWPF_mLlm>#@|K5^^*5>qAk0o zWyf(0hzr(UFMQdD zY>X!4bS<1hR)V&JYQw$Yg)u7)f@Vdf$ZW86KPnrc|aTtLVF9wI#sViS;S5t zTs__pIx&aSn^Q8eO|VxCL~>{n6TD@}$n=Q55G(@%rVl`F$)a~;GdS-40e<=Mzw|Mr;M@$JmMXKgR1?8_@vMM?7e|lTaNdHEjcG(pHX77Ap0Y| z{>J^Elk_zz*Npf5gw;H*YI^C=Ec&(PUo>GnN-})ff8C+(o&AB$g01#J@0IqXC|11@ zkH@M}NtxEIwG!pG8p19T6Hngo7P52Yp!;m%K7D@s%dPA^VA{~exCji!O3_4*gIwb% zLArSQ8D%hTbMT({5n+y$>`OKn^>5Hsnr~P-QAQiDjg+<~ygFb(Cs>&H?EY*yU?7uq zcnckoac?xZT#a1ciATtf`USScMk4seb{W!;&39#aExXM;=AzD@D!Rh4ZFu)_3q zVI4iK)Q0;X4*piReVIs`iMvYBBVPrr3V(+6J;qOplw?O{t`kTCrk6E-w8t zrSf;b-Tmcof)>C|jdt~UVRNvRmTywuXt?z~w=yi$7A>9Ujd?i^iv zOcYlVpMlD^>uX(`C&QJSwC$(QZgHACDO^67jEE<*=DydFHpJ96aiNy@Opy8E!RXu= zjXw&Vpo5Lv0AmizTTHhqG~qQ}#-duUp@1q#AxZifid3}}-_-qt% zrTzMlp-VBpjgH{LiPATGCySQEh=24QY_!#nz}n8@UsJtC$}GO(5dYW=ZnoVbn3vAM zP7@8x%cF6+G@w8(_-&L7biK7hHrLL6oBz7f{bS=$QEiR5r?LGkD&R;Stx8DA$@-Fz zqQj}@^yKLA(aKn+#WY2MZSEx#Hj1%YF0i&Ix_GzjX6h)#GNZq;2M9fU-r9MvpyikX z0*85ldqS?^*Y6;+eAr1cTC*eJ|Ejoh-$kp5(#@8r_bHGqeBu`BTZxMm5Z67(0D;N zW!cccF*36FJ&FC|)tmehoI)lp9~U}<_p=@wd`8b+2zQ5Ej- zh-gvdHGsUXk# zlGGH%zE`=Bu7rB`N+=dYpb6wUD{rv6(!V68B^2#TLOB5%o?(IK*h}nU92*%?7)qMJ zl$&qY6X?YBn1-0HRz>6BFuYTilh$cY==T!Xv1{(*EgA_@n?jT5y(`MGs21|`a%iqx z3k!QWh|5TT6z!`TuHMeuW)RqD7?|g}M39Mh!P{GsTXd0jEK8jhMpmIHj@rWzWeT4L z)G3^iPsxc zzUr+d6K#O#KR$2=xI%bG#QVQZ+YJPsa5*qdBr=iUFvW_l#L;A=z*quJOVSftcrHCm zx^Wxqe$1nEtWE|%r#u}O^6Ucno*+eQ{v#DT&;p}Bdy~~Z1E9nGvh@WD>_8MN9TmjR zNXeP|Nbkr*TahW3oQBZ~YFyL(li78h(X_Qob2vl1tkg7W~BTJ^kzMEBsZ7oMJVkPImE68VWgsn3htI9Pz zFVthL@n#^(`XdV#*L%sGQsqQUQQ2I}m_dL-iDUpEg(I^CllQj`txY-N;#y$SCIc_U z6u46bVfN;#?7{yk_@6UJ8X`Zo-*axiPVWh-VfswEZ#>{oait;r(NB;cDsGTxT7w*s zR9fx}Eqwh@Dy9gMI5lP7eUN?=7n%7Y^1ZMeSm%%%S^;hS2K%<{`p^nGb#?!|ZPd9Vkb5!7hg>1p)vA$vaaqXcuuR$e!Ivk~ zh_w&v1rWphfB_`!h1=0`HRkE-V-=3t5#L9ChpoWBOh)-Y{CfO=mGt$R{nYy~7M^~s zR%?!CJ@@J43_gvOyJn!jw&?#8J|Oq;tG+rNsSxbM$xbcQ(YH)$g~|%l+3@h#HIgG^ z*frT!Te&WCt2Il|N%|VXW{D+kV}`|7?-G%k&Q`&=7z^G!jly~J7Hjbc$$i$5^u zisW>Z(<4wWQ5n#z3%bO36GADX#vHzWk0jE>8vA;EaQIc_3>Gh)08SZQ90V|E9(=s` zv#(W*JBQ}J1m(FZF#I#&S~-L79g<+kbtv(Fm1VBeJ&!H%;DU(IL;pyB7WPI;xl)*$ zateviYjY!$MV_j-t&MYHG0p|I8g^yUshWi`pFurKR^`_L>se&cfU<^&f(rH;AZ^ir zOHZijI|iE1G-G(7(*8YEn07EkDZmc9L@ZWo2kPGCL;9F}D7F$jv0FwPRffle$Kr>7 z=@W_^eu|h?Qw8F#5`m zGUk`o*r+17*W^0|UApo{s?ENP^c0pj?ZrWeT^Dbj<0)roVuNmHNW)Eul(a_5C zMG*n08Qt{m09x~y`zAiRAPiu|2AN7{KU<%0{;7LE=Z~QLLI^#+KCd?hE2Fc#u4}l^;m%+xM)R#FH;sf4yTCCvHyhoEHNvZ z;O1mmM9E7(#na=1$Djkh$pcIquzDlY=mj_Al%_n+;yUc;`j_ww3Fi|ep zf<0In@Y9#yBKCq~>w^jb;af1FTLN8*MTXQAhi&01RG>ndSHa=eZDn}6Ds2&Yu9dI* zXwy&nx-5cYG}GAN>gQs)w@f<4C*fZ15`ey1<$@PNfo-5H_ytJg$0t^f>iyka*n0Cj9L8-W>K1b+N-X z^4;sH&K8DYJ>(aGR0Urms~|UrarGB-3k#YHYTRxe)CW(*sf5o!jEc|F z(lXA@eQp~uGBee_3uRf=A3`^he_RqFRXB-zv@H>|6i5pa7i|n1AzkI<5OBD&li~K5 zE>QGs_@^|P_??vZIgnA`>ZP-&)>cF%H0^nb0<6e)U+=x_$;4q&Q}?=kk*#j}(}Z1| z{HwNt&Y!&q4yVW2Jz`wa%dho{bO|z^Q7pt(6u+#xHzFJspL*DP`b0@VB*grdwJM}} z7k|uXaiJ{P)z5bGt$64K!#EDCqk%V|eJz+@dY(H=sOn^_N9J|Lb})zI6sjxHGnZyD zAwc+tbJ0E%xN?K-xU;k2Lc({hBtfrmOjpefvF;;}2rrLUSNn_~@yx$(SqS?U50QjiXQX-TGWI!Ngn839G8oXa+8;fd?uN4CTf)@=DMgJ zhOuIs*HLeqC`0emR+KTW3KQ*7 z-ak4a+Vi6ArXyMOvfU>m6;iSWEVs)zsIqf>W6u|sPU@tAi5J1webeqOOQ)^flb!vV z-EvEFwFFyfiv!A7g?vSSAab!R8*B9;G90sp7i zlV?IeA?W^mJ~Z+PZSy(2AJ~xK+z|U~S^6Kl+`H5G&_g`G8*Ay!0`Bg}UJ}3J|8Zx~ zisgAYXDtG2zpDO_G5i4Emtv5yq4d(VAuCl@&t39BynbG9qayMw(zUVs4PB<-(aF{< z_{nqNta`f^k8Wu^ICaJQXBgLQ37ovrD_!?Ch^t`Q)8!Rrg0a-9Y%$h|ve|?tjn@~w zC*I=v2Y4SlNA|24Y^SMWp*`7kM?5`rf)2@~I6)cf-kIpdI);pgn>R~aGfbo9Tvh!h zXCAau)F};;f9JV+PzmF8gpB{}AWUzW0h0G1QFdS8a+)wzCb`uQwdO78(fK1h1Sj0z zhcSqYY-#_bX_CPS+5|F5l>pcEdwHa%Mbs87p8e{Al|OTxi7ze$dbC-r=IiFhqu%`u zg(C;Fprt@7I=3J-gJDnQK2-tR1m5msxp`~wA-LlbAd3_a4xYGD6xbXsHy2=cz>+75 z9yp}y3%xV%3D|5WureG46uQd|=V`=!d0l2OE*V;l79%EKft ztr`$fqk;))O%dhq{(3Dwb&zu)arX_4I$zRQ*P7(*a(xKv;A1K;t}ra}aDE zKAv8lwX=)bWc8A;`urfst;g#E5tXU$r@Pq^m=2kHZb0F?gi4|~*4U6N{mmLiXB3joknB0{3*dC|$ae12h)m1NcaO)Lh%joZ@>6zd(2_dBZc+_nmlEB<--FL=c+0( z@XmN+oJZ(=#0Zf^cODKlrE%5H*$8r78{$ARhv*)%pyN#qeGlf@`14A!;{2^a1INUo zh&>&N-tj5dl<%+Ohnb&;rARz)Xrm})klEB`VriS)CT*|XzBV-6(O-J|NPSmc3HeQU9fg}2o#Db#Te9=B>X#$R)@y@QsX$}(jb!E4_ZH^HG0}C8sl0V>*991G z7|zfHN1ECy5xLXmcaG4}8Xap^CqADqF*tM&B(0yD1<5OqgzRwE*o=0Nc+mZxc-x{Qcinq#F&M7b46gUdCzTf-UX)0J;^Jb}p=crG@RQ!bpgFK%xw<0n zWL5SJa+MVfz2j_W+W7Bif_dT0*O(hU22+Gz378KUN6A#uE2nM6kpqe~3}SeD^etzd zwXLavyi}DUn1#o0j4YgY+zmQU+Qt4FFQ-&WGm)aEO|1dRHzpvc#Q{zC8WA8ma4Iz8@x# znA+{An9_V@0_B@L#Bxg4?ibkUG^A!R@Lxos-rvTIm!V1U1_k06F2@DQ6S@#tZnEFk zsWUgeSIHKkc5b(s5rx`l258}NM=U4M-}ov%t+0KB61<>U=fwiP#X0nJKbbgUrOM6# zbWQfF2 zxXXn%`!wex}%dBgcBM=9a#{NU)$xQ z=sM?eNv(wYyEElif<_7YY)e>o4Y=nk)ykSla0U3WFj|tjMgwO<+b+OLF|-a}8pi#g zQE3Ony}Dq&8v6ln?>A6=foul>*bV59_o|#$fn3(kiSO~!_0M>4;SHa1hq`Z(iWwiY zdK0=5`C1{7W4)6B2kvNt@`G9NNsonq|L1*d@C2lYzLyzh6LwWQ0^Q(<$I7?MnOP(IFpzp=4Q?JD_q)RvDL?0srJS@^Hwd#DqL$Z&|0FieP zB(6-XP|%;H&WDocOz^qvFyO*fv0mMA`}5Y09TzFe1Nv?-h%w$g{hgNfEnqz z`<7eHen8f3Dd@ZVVNvZ<+r{Sre6$qIfb z>E7EH1PEP{WK9ZUk`d{Z*k*7UC8qX(^Iz@bV~w_`a<0?4kRIIZ!q3BZ`#=fJ*NQ#( zdR(1rn|A8i6sos6uGiGIl9JWd4IalYsU?R!(q2s*{?SdmC|0yy-5tKvLEF!%R+2nv zmK)A?nMk0Vfdd6K3k{7D=s@>M0b`D0i->Fo%U+@CL1Idu6TSKeLVTFp~ zF-yZM1vfnHoZVBe5)ZCkr}d`#1@HOAbIC-rkfs(ywh$Cq3wRp{_W|mLZ>6FWC7;KL zRe+p}2C#5JY8vKj0!cPC zi*3QF&$pjrz>&w}LnNbchEm>z_MLsM&jw|`t3r`E+ zR;&;V*A*97Vdwz0Z>;~ZtG*Z$oKe*uyVN{IuA_^ zR&Er(wqCn`&UX%f#cy`kGJSt|>>=)Uz#@=l>?a=j&B-h(*v&23Jsv7BTA-f*YRbWz zjdVMj$hf-i0UfvZNNKt6J!79njTZN)6mv|9ZGSR=aq*R#`V$GbEhtBDp+IVtKZ=aj+V+D-O^t zp=>(z$$K!B(g`>@IT{*+5!nshBkypU(n5UAtji)F!yi4coVw?*?0|4}#l=NRrS}mq zDbdh#(#zoBky=sV{_Ov|F&%JqJGAh^eR^UFJzbs({`;Drl+taM0^d@<)jdvb(c)?2 z_sRK~YvTq(!EEDD(Cfzm0lv$)&e#6aymr#dQ8InM=p2E-8#!4@4o78lKeI!|KfQ`A zIKJ0++emyAROW?v`7~@WYuuDccxP0@v_Xns!ofwN$)qBSc(9LRsc?mVyc>00abK1= zy`v=&9vpmLIhuv>k5K0C+;{$RE?8q2Vg|8c;8YlMOGm$Y)P+i8YH!SK{(PnX1rCDg z&JP4Bn)SppfeyjcdxZcYjWr1T`fNRwMaXF;&`-9Iupjj)3eCGy$7JTu)VFk=OCx;$M|6t+jq<4$wE2kd7eju`3)56Drzc*vH@mI-ayhi0JYHFv|Dd>0Mt8mb-uB7nY|!uEgh2{5`El4zEz%@>N^sBN4+EPqY382g zchcuLh-4i9z;L)tQt8b|gfz3F@eGU5?~t2qi9PH}S1`!dls{zX&8U1Pr9c^tV}K3S ze=Yz?Xw7v7i8TZ?WO};P+aM_l`T%ji)A92f(OiaKdSNDL26M&I>87AD7~NG%j=&>( zK({|T&#RnyL7jfem-i4xu!9hHpPQFg28Ef2q~3XDnuuNDlG_gBLd)O+r=BM9%j0*# z5ZojmUuhi(neAs&OE3lZ$TFZ!3`e45KSlWFr~^8_3YJr!q4pvhd#QE-kMz0JG?Gb) zu=Q;e_V1s`Oo~tj52IDM2AECqKAW3=P-yg3pp>aEDlgMYrGQD;@3i{Y7pJDhzhJ9+ zRi-1ice|Krb~x@ZZwtP8DBX1s9}0TDTa}5fJvbJe19`ooHt+;2I`dfa^yWsoMng>g545G@2jYC?1!`*mZ`89($ zuuJT9@jii<=FKnYDdQT?`7Umx6nuP`WEvUZufuf583Ajm7%f~LLSk6!J#y2iYA(WI z5UH#$y#+Q}SBVdB#ajcrBzC5tZ4)>JLt$Ny?F#(+RqfRu1cvbrnFe(w)Ty3mBAH zX_fD1;ycMyH-|+Z`<)!lGi$8#$@!`{@bRLsS9+dFDlii#k{SJRdr!3@uPY5mWk@&5n3;hwYB?cd{vAK4 z21d8a8Rla-4&SxCPTmIK$TLYch@{fJkLF5>))nTt@FkbcJ7s8963pn}H=bOkzVQM; zw*%fFLMBYsh%J-_T{)GFXa8`F{`&f?)b8>qb|d1rj)#mc@l3ga505Xv1!`?=gIr|= z_M6(W;l3K@`4Ic@FMW-xnq=h8!7&KTD1*?G`jlhS%05c^IBkglwOJL{C1i9#!}l7X zI)uTjU>In|pXhT!4wk0ftK_;-I)m`6LKzjtUPr3=gUuUN?8G#2L{0!0SDtU!DR8vN{(wA;PuuM+DB{< zl57>RNh8yonUT>5&Bc6$UXD5`T4m;v$ISzTvP8u0W%Qq0U0eemBjB)8cS)}nsq8%# z@2uR+9ibLC>}D>4-Ahh|aS`p3id{JWe8Gh)PX;6TyDXQcVI4|50i%yU$x=$5CRMIE z>}8#s_sQIfGHcQJuB-VW7X$`u3<3`0p7;H%3c1!SPr+*F>ZMD!YrP~En|F4!@3@%L z82pq+XCi#-AsU~RuwOse=J(sj1YAVeWp|hDIg@pVV_LZ^{JsCCNyQT`iO8__WzcYn z{LsD8`OGl)fs2{pKcA~M4IJs{6lzWEXN*G>PY;11r28b$1}wWgg9?Yot^Vhw;XBWz z9>P&Z*qPLa-9fO=#om9M7s0SuuKw@;!9`vP$MMg!TB4@<_jCUBbU1u2{NF#Caak(z z3bFV(>?Qv5V*fl5Nw$Z5;s5y^clcqoX1udB{h!~8{igK4c7y+UVu-?&q)fmaL1w*X RVjTFVEUzI~AY 1). + """ + kernel_size_effective = [kernel_size[0] + (kernel_size[0] - 1) * (rate - 1), + kernel_size[0] + (kernel_size[0] - 1) * (rate - 1)] + pad_total = [kernel_size_effective[0] - 1, kernel_size_effective[1] - 1] + pad_beg = [pad_total[0] // 2, pad_total[1] // 2] + pad_end = [pad_total[0] - pad_beg[0], pad_total[1] - pad_beg[1]] + padded_inputs = tf.pad( + tensor=inputs, + paddings=[[0, 0], [pad_beg[0], pad_end[0]], [pad_beg[1], pad_end[1]], + [0, 0]]) + return padded_inputs + + +def _make_divisible(v, divisor, min_value=None): + if min_value is None: + min_value = divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_v < 0.9 * v: + new_v += divisor + return int(new_v) + + +@contextlib.contextmanager +def _set_arg_scope_defaults(defaults): + """Sets arg scope defaults for all items present in defaults. + + Args: + defaults: dictionary/list of pairs, containing a mapping from + function to a dictionary of default args. + + Yields: + context manager where all defaults are set. + """ + if hasattr(defaults, 'items'): + items = list(defaults.items()) + else: + items = defaults + if not items: + yield + else: + func, default_arg = items[0] + with slim.arg_scope(func, **default_arg): + with _set_arg_scope_defaults(items[1:]): + yield + + +@slim.add_arg_scope +def depth_multiplier(output_params, + multiplier, + divisible_by=8, + min_depth=8, + **unused_kwargs): + if 'num_outputs' not in output_params: + return + d = output_params['num_outputs'] + output_params['num_outputs'] = _make_divisible(d * multiplier, divisible_by, + min_depth) + + +_Op = collections.namedtuple('Op', ['op', 'params', 'multiplier_func']) + + +def op(opfunc, multiplier_func=depth_multiplier, **params): + multiplier = params.pop('multiplier_transform', multiplier_func) + return _Op(opfunc, params=params, multiplier_func=multiplier) + + +class NoOpScope(object): + """No-op context manager.""" + + def __enter__(self): + return None + + def __exit__(self, exc_type, exc_value, traceback): + return False + + +def safe_arg_scope(funcs, **kwargs): + """Returns `slim.arg_scope` with all None arguments removed. + + Arguments: + funcs: Functions to pass to `arg_scope`. + **kwargs: Arguments to pass to `arg_scope`. + + Returns: + arg_scope or No-op context manager. + + Note: can be useful if None value should be interpreted as "do not overwrite + this parameter value". + """ + filtered_args = {name: value for name, value in kwargs.items() + if value is not None} + if filtered_args: + return slim.arg_scope(funcs, **filtered_args) + else: + return NoOpScope() + + +@slim.add_arg_scope +def mobilenet_base( # pylint: disable=invalid-name + inputs, + conv_defs, + multiplier=1.0, + final_endpoint=None, + output_stride=None, + use_explicit_padding=False, + scope=None, + is_training=False): + """Mobilenet base network. + + Constructs a network from inputs to the given final endpoint. By default + the network is constructed in inference mode. To create network + in training mode use: + + with slim.arg_scope(mobilenet.training_scope()): + logits, endpoints = mobilenet_base(...) + + Args: + inputs: a tensor of shape [batch_size, height, width, channels]. + conv_defs: A list of op(...) layers specifying the net architecture. + multiplier: Float multiplier for the depth (number of channels) + for all convolution ops. The value must be greater than zero. Typical + usage will be to set this value in (0, 1) to reduce the number of + parameters or computation cost of the model. + final_endpoint: The name of last layer, for early termination for + for V1-based networks: last layer is "layer_14", for V2: "layer_20" + output_stride: An integer that specifies the requested ratio of input to + output spatial resolution. If not None, then we invoke atrous convolution + if necessary to prevent the network from reducing the spatial resolution + of the activation maps. Allowed values are 1 or any even number, excluding + zero. Typical values are 8 (accurate fully convolutional mode), 16 + (fast fully convolutional mode), and 32 (classification mode). + + NOTE- output_stride relies on all consequent operators to support dilated + operators via "rate" parameter. This might require wrapping non-conv + operators to operate properly. + + use_explicit_padding: Use 'VALID' padding for convolutions, but prepad + inputs so that the output dimensions are the same as if 'SAME' padding + were used. + scope: optional variable scope. + is_training: How to setup batch_norm and other ops. Note: most of the time + this does not need be set directly. Use mobilenet.training_scope() to set + up training instead. This parameter is here for backward compatibility + only. It is safe to set it to the value matching + training_scope(is_training=...). It is also safe to explicitly set + it to False, even if there is outer training_scope set to to training. + (The network will be built in inference mode). If this is set to None, + no arg_scope is added for slim.batch_norm's is_training parameter. + + Returns: + tensor_out: output tensor. + end_points: a set of activations for external use, for example summaries or + losses. + + Raises: + ValueError: depth_multiplier <= 0, or the target output_stride is not + allowed. + """ + if multiplier <= 0: + raise ValueError('multiplier is not greater than zero.') + + # Set conv defs defaults and overrides. + conv_defs_defaults = conv_defs.get('defaults', {}) + conv_defs_overrides = conv_defs.get('overrides', {}) + if use_explicit_padding: + conv_defs_overrides = copy.deepcopy(conv_defs_overrides) + conv_defs_overrides[ + (slim.conv2d, slim.separable_conv2d)] = {'padding': 'VALID'} + + if output_stride is not None: + if output_stride == 0 or (output_stride > 1 and output_stride % 2): + raise ValueError('Output stride must be None, 1 or a multiple of 2.') + + # a) Set the tensorflow scope + # b) set padding to default: note we might consider removing this + # since it is also set by mobilenet_scope + # c) set all defaults + # d) set all extra overrides. + # pylint: disable=g-backslash-continuation + with _scope_all(scope, default_scope='Mobilenet'), \ + safe_arg_scope([slim.batch_norm], is_training=is_training), \ + _set_arg_scope_defaults(conv_defs_defaults), \ + _set_arg_scope_defaults(conv_defs_overrides): + # The current_stride variable keeps track of the output stride of the + # activations, i.e., the running product of convolution strides up to the + # current network layer. This allows us to invoke atrous convolution + # whenever applying the next convolution would result in the activations + # having output stride larger than the target output_stride. + current_stride = 1 + + # The atrous convolution rate parameter. + rate = 1 + + net = inputs + # Insert default parameters before the base scope which includes + # any custom overrides set in mobilenet. + end_points = {} + scopes = {} + for i, opdef in enumerate(conv_defs['spec']): + params = dict(opdef.params) + opdef.multiplier_func(params, multiplier) + stride = params.get('stride', 1) + if output_stride is not None and current_stride == output_stride: + # If we have reached the target output_stride, then we need to employ + # atrous convolution with stride=1 and multiply the atrous rate by the + # current unit's stride for use in subsequent layers. + layer_stride = 1 + layer_rate = rate + rate *= stride + else: + layer_stride = stride + layer_rate = 1 + current_stride *= stride + # Update params. + params['stride'] = layer_stride + # Only insert rate to params if rate > 1 and kernel size is not [1, 1]. + if layer_rate > 1: + if tuple(params.get('kernel_size', [])) != (1, 1): + # We will apply atrous rate in the following cases: + # 1) When kernel_size is not in params, the operation then uses + # default kernel size 3x3. + # 2) When kernel_size is in params, and if the kernel_size is not + # equal to (1, 1) (there is no need to apply atrous convolution to + # any 1x1 convolution). + params['rate'] = layer_rate + # Set padding + if use_explicit_padding: + if 'kernel_size' in params: + net = _fixed_padding(net, params['kernel_size'], layer_rate) + else: + params['use_explicit_padding'] = True + + end_point = 'layer_%d' % (i + 1) + try: + net = opdef.op(net, **params) + except Exception: + print('Failed to create op %i: %r params: %r' % (i, opdef, params)) + raise + end_points[end_point] = net + scope = os.path.dirname(net.name) + scopes[scope] = end_point + if final_endpoint is not None and end_point == final_endpoint: + break + + # Add all tensors that end with 'output' to + # endpoints + for t in net.graph.get_operations(): + scope = os.path.dirname(t.name) + bn = os.path.basename(t.name) + if scope in scopes and t.name.endswith('output'): + end_points[scopes[scope] + '/' + bn] = t.outputs[0] + return net, end_points + + +@contextlib.contextmanager +def _scope_all(scope, default_scope=None): + with tf.compat.v1.variable_scope(scope, default_name=default_scope) as s,\ + tf.compat.v1.name_scope(s.original_name_scope): + yield s + + +@slim.add_arg_scope +def mobilenet(inputs, + num_classes=1001, + prediction_fn=slim.softmax, + reuse=None, + scope='Mobilenet', + base_only=False, + **mobilenet_args): + """Mobilenet model for classification, supports both V1 and V2. + + Note: default mode is inference, use mobilenet.training_scope to create + training network. + + + Args: + inputs: a tensor of shape [batch_size, height, width, channels]. + num_classes: number of predicted classes. If 0 or None, the logits layer + is omitted and the input features to the logits layer (before dropout) + are returned instead. + prediction_fn: a function to get predictions out of logits + (default softmax). + reuse: whether or not the network and its variables should be reused. To be + able to reuse 'scope' must be given. + scope: Optional variable_scope. + base_only: if True will only create the base of the network (no pooling + and no logits). + **mobilenet_args: passed to mobilenet_base verbatim. + - conv_defs: list of conv defs + - multiplier: Float multiplier for the depth (number of channels) + for all convolution ops. The value must be greater than zero. Typical + usage will be to set this value in (0, 1) to reduce the number of + parameters or computation cost of the model. + - output_stride: will ensure that the last layer has at most total stride. + If the architecture calls for more stride than that provided + (e.g. output_stride=16, but the architecture has 5 stride=2 operators), + it will replace output_stride with fractional convolutions using Atrous + Convolutions. + + Returns: + logits: the pre-softmax activations, a tensor of size + [batch_size, num_classes] + end_points: a dictionary from components of the network to the corresponding + activation tensor. + + Raises: + ValueError: Input rank is invalid. + """ + is_training = mobilenet_args.get('is_training', False) + input_shape = inputs.get_shape().as_list() + if len(input_shape) != 4: + raise ValueError('Expected rank 4 input, was: %d' % len(input_shape)) + + with tf.compat.v1.variable_scope(scope, 'Mobilenet', reuse=reuse) as scope: + inputs = tf.identity(inputs, 'input') + net, end_points = mobilenet_base(inputs, scope=scope, **mobilenet_args) + if base_only: + return net, end_points + + net = tf.identity(net, name='embedding') + + with tf.compat.v1.variable_scope('Logits'): + net = global_pool(net) + end_points['global_pool'] = net + if not num_classes: + return net, end_points + # net = slim.dropout(net, scope='Dropout', is_training=is_training) + # 1 x 1 x num_classes + # Note: legacy scope name. + # logits = slim.conv2d( + # net, + # num_classes, [1, 1], + # activation_fn=None, + # normalizer_fn=None, + # biases_initializer=tf.compat.v1.zeros_initializer(), + # scope='Conv2d_1c_1x1') + + # logits = tf.squeeze(logits, [1, 2]) + + # use slim.fully_connected instead + net = tf.squeeze(net) + net = slim.dropout(net, keep_prob=0.8, scope='Dropout', is_training=is_training) + logits = slim.fully_connected( + net, + num_classes, + activation_fn=None, + normalizer_fn=None, + scope='FC' + ) + #logits = tf.expand_dims(logits, axis=[]) + + logits = tf.identity(logits, name='output') + end_points['Logits'] = logits + if prediction_fn: + end_points['Predictions'] = prediction_fn(logits, 'Predictions') + return logits, end_points + + +def global_pool(input_tensor, pool_op=tf.compat.v2.nn.avg_pool2d): + """Applies avg pool to produce 1x1 output. + + NOTE: This function is funcitonally equivalenet to reduce_mean, but it has + baked in average pool which has better support across hardware. + + Args: + input_tensor: input tensor + pool_op: pooling op (avg pool is default) + Returns: + a tensor batch_size x 1 x 1 x depth. + """ + shape = input_tensor.get_shape().as_list() + if shape[1] is None or shape[2] is None: + kernel_size = tf.convert_to_tensor(value=[ + 1, + tf.shape(input=input_tensor)[1], + tf.shape(input=input_tensor)[2], 1 + ]) + else: + kernel_size = [1, shape[1], shape[2], 1] + output = pool_op( + input_tensor, ksize=kernel_size, strides=[1, 1, 1, 1], padding='VALID') + # Recover output shape, for unknown shape. + output.set_shape([None, 1, 1, None]) + return output + + +def training_scope(is_training=True, + weight_decay=0.00004, + stddev=0.09, + dropout_keep_prob=0.8, + bn_decay=0.997): + """Defines Mobilenet training scope. + + Usage: + with tf.contrib.slim.arg_scope(mobilenet.training_scope()): + logits, endpoints = mobilenet_v2.mobilenet(input_tensor) + + # the network created will be trainble with dropout/batch norm + # initialized appropriately. + Args: + is_training: if set to False this will ensure that all customizations are + set to non-training mode. This might be helpful for code that is reused + across both training/evaluation, but most of the time training_scope with + value False is not needed. If this is set to None, the parameters is not + added to the batch_norm arg_scope. + + weight_decay: The weight decay to use for regularizing the model. + stddev: Standard deviation for initialization, if negative uses xavier. + dropout_keep_prob: dropout keep probability (not set if equals to None). + bn_decay: decay for the batch norm moving averages (not set if equals to + None). + + Returns: + An argument scope to use via arg_scope. + """ + # Note: do not introduce parameters that would change the inference + # model here (for example whether to use bias), modify conv_def instead. + batch_norm_params = { + 'decay': bn_decay, + 'is_training': is_training + } + #if stddev < 0: + # weight_intitializer = slim.initializers.xavier_initializer() + #else: + # weight_intitializer = tf.compat.v1.truncated_normal_initializer(stddev=stddev) + + # modified for NPU + weight_2d = tf.initializers.variance_scaling(scale=2., mode="fan_out", distribution="untruncated_normal") + weight_dw = tf.initializers.variance_scaling(scale=2., mode="fan_in", distribution="untruncated_normal") + weight_pw = tf.initializers.variance_scaling(scale=2., mode="fan_out", distribution="untruncated_normal") + weight_fc = tf.initializers.random_normal(stddev=0.01) + + # Set weight_decay for weights in Conv and FC layers. + with slim.arg_scope( + #[slim.conv2d, slim.fully_connected, slim.separable_conv2d], + [slim.conv2d], + #weights_initializer=weight_intitializer, + weights_initializer=weight_2d, + normalizer_fn=slim.batch_norm), \ + slim.arg_scope([slim.fully_connected], weights_initializer=weight_fc, normalizer_fn=slim.batch_norm), \ + slim.arg_scope([slim.separable_conv2d], weights_initializer=weight_dw, pointwise_initializer=weight_pw, normalizer_fn=slim.batch_norm), \ + slim.arg_scope([mobilenet_base, mobilenet], is_training=is_training),\ + safe_arg_scope([slim.batch_norm], **batch_norm_params), \ + safe_arg_scope([slim.dropout], is_training=is_training, + keep_prob=dropout_keep_prob), \ + slim.arg_scope([slim.conv2d], \ + weights_regularizer=slim.l2_regularizer(weight_decay)), \ + slim.arg_scope([slim.separable_conv2d], weights_regularizer=None) as s: + return s diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_example.ipynb b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_example.ipynb new file mode 100644 index 0000000..b0b3689 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_example.ipynb @@ -0,0 +1,445 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "toc", + "id": "aUVxY7xOGD1G" + }, + "source": [ + "\u003e[Prerequisites (downloading tensorflow_models and checkpoints)](#scrollTo=T_cETKXHDTXu)\n", + "\n", + "\u003e[Checkpoint based inference](#scrollTo=fxMe7_pkk_Vo)\n", + "\n", + "\u003e[Frozen inference](#scrollTo=PlwvpK3ElBk6)\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "T_cETKXHDTXu" + }, + "source": [ + "# Prerequisites (downloading tensorflow_models and checkpoints)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "base_uri": "https://localhost:8080/", + "height": 125, + "output_extras": [ + {} + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 31129, + "status": "ok", + "timestamp": 1521483961674, + "user": { + "displayName": "Mark Sandler", + "photoUrl": "//lh5.googleusercontent.com/-CjnV3zpGrlw/AAAAAAAAAAI/AAAAAAAABRU/dfjRy_tzX5M/s50-c-k-no/photo.jpg", + "userId": "108034853522252017283" + }, + "user_tz": 420 + }, + "id": "zo5GyseklSVH", + "outputId": "e12a8a80-c0d2-4ebc-9230-b170f11d236e" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cloning into 'models'...\n", + "remote: Counting objects: 13504, done.\u001b[K\n", + "remote: Total 13504 (delta 2), reused 2 (delta 2), pack-reused 13501\u001b[K\n", + "Receiving objects: 100% (13504/13504), 422.07 MiB | 37.42 MiB/s, done.\n", + "Resolving deltas: 100% (7635/7635), done.\n", + "Checking out files: 100% (1946/1946), done.\n" + ] + } + ], + "source": [ + "!git clone https://github.com/tensorflow/models" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "both", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "base_uri": "https://localhost:8080/", + "height": 35, + "output_extras": [ + {} + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 4504, + "status": "ok", + "timestamp": 1521493157017, + "user": { + "displayName": "Mark Sandler", + "photoUrl": "//lh5.googleusercontent.com/-CjnV3zpGrlw/AAAAAAAAAAI/AAAAAAAABRU/dfjRy_tzX5M/s50-c-k-no/photo.jpg", + "userId": "108034853522252017283" + }, + "user_tz": 420 + }, + "id": "obaW6O8bz3mA", + "outputId": "79b3fb23-caa7-4683-9575-ba7c2f55ebdb" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Successfully downloaded the checkpoint. It is available as mobilenet_v2_1.0_224.ckpt\n" + ] + } + ], + "source": [ + "from __future__ import print_function\n", + "from IPython import display \n", + "checkpoint_name = 'mobilenet_v2_1.0_224' #@param\n", + "url = 'https://storage.googleapis.com/mobilenet_v2/checkpoints/' + checkpoint_name + '.tgz'\n", + "print('Downloading from ', url)\n", + "!wget {url}\n", + "print('Unpacking')\n", + "!tar -xvf {checkpoint_name}.tgz\n", + "checkpoint = checkpoint_name + '.ckpt'\n", + "\n", + "display.clear_output()\n", + "print('Successfully downloaded checkpoint from ', url,\n", + " '. It is available as', checkpoint)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "base_uri": "https://localhost:8080/", + "height": 215, + "output_extras": [ + {} + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 1486, + "status": "ok", + "timestamp": 1521485010457, + "user": { + "displayName": "Mark Sandler", + "photoUrl": "//lh5.googleusercontent.com/-CjnV3zpGrlw/AAAAAAAAAAI/AAAAAAAABRU/dfjRy_tzX5M/s50-c-k-no/photo.jpg", + "userId": "108034853522252017283" + }, + "user_tz": 420 + }, + "id": "qZDfLegf3hpw", + "outputId": "334ed084-b90e-4bd0-bd5e-125434a9a30f" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--2018-03-19 18:43:29-- https://upload.wikimedia.org/wikipedia/commons/f/fe/Giant_Panda_in_Beijing_Zoo_1.JPG\n", + "Resolving upload.wikimedia.org (upload.wikimedia.org)... 208.80.154.240, 2620:0:860:ed1a::2:b\n", + "Connecting to upload.wikimedia.org (upload.wikimedia.org)|208.80.154.240|:443... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 116068 (113K) [image/jpeg]\n", + "Saving to: ‘panda.jpg’\n", + "\n", + "panda.jpg 100%[===================\u003e] 113.35K --.-KB/s in 0.03s \n", + "\n", + "2018-03-19 18:43:30 (3.18 MB/s) - ‘panda.jpg’ saved [116068/116068]\n", + "\n" + ] + } + ], + "source": [ + "!wget https://upload.wikimedia.org/wikipedia/commons/f/fe/Giant_Panda_in_Beijing_Zoo_1.JPG -O panda.jpg" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "g0H2RDadndug" + }, + "outputs": [], + "source": [ + "# setup path\n", + "import sys\n", + "sys.path.append('/content/models/research/slim')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "fxMe7_pkk_Vo" + }, + "source": [ + "# Checkpoint based inference" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "GrQemT66CxXt" + }, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "from nets.mobilenet import mobilenet_v2\n", + "\n", + "tf.reset_default_graph()\n", + "\n", + "# For simplicity we just decode jpeg inside tensorflow.\n", + "# But one can provide any input obviously.\n", + "file_input = tf.placeholder(tf.string, ())\n", + "\n", + "image = tf.image.decode_jpeg(tf.read_file(file_input))\n", + "\n", + "images = tf.expand_dims(image, 0)\n", + "images = tf.cast(images, tf.float32) / 128. - 1\n", + "images.set_shape((None, None, None, 3))\n", + "images = tf.image.resize_images(images, (224, 224))\n", + "\n", + "# Note: arg_scope is optional for inference.\n", + "with tf.contrib.slim.arg_scope(mobilenet_v2.training_scope(is_training=False)):\n", + " logits, endpoints = mobilenet_v2.mobilenet(images)\n", + " \n", + "# Restore using exponential moving average since it produces (1.5-2%) higher \n", + "# accuracy\n", + "ema = tf.train.ExponentialMovingAverage(0.999)\n", + "vars = ema.variables_to_restore()\n", + "\n", + "saver = tf.train.Saver(vars) " + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "base_uri": "https://localhost:8080/", + "height": 666, + "output_extras": [ + {}, + {} + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 1422, + "status": "ok", + "timestamp": 1521493723379, + "user": { + "displayName": "Mark Sandler", + "photoUrl": "//lh5.googleusercontent.com/-CjnV3zpGrlw/AAAAAAAAAAI/AAAAAAAABRU/dfjRy_tzX5M/s50-c-k-no/photo.jpg", + "userId": "108034853522252017283" + }, + "user_tz": 420 + }, + "id": "TJbLYo_FCxXy", + "outputId": "17a4fbaa-dec0-4997-b233-15ba69806083" + }, + "outputs": [ + { + "data": { + "image/jpeg": "/9j/4AAQSkZJRgABAQEAYABgAAD/4SjTRXhpZgAASUkqAAgAAAAKAA8BAgASAAAAhgAAABABAgAK\nAAAAmAAAABIBAwABAAAAAAAAABoBBQABAAAAogAAABsBBQABAAAAqgAAACgBAwABAAAAAgAAADEB\nAgALAAAAsgAAADIBAgAUAAAAvgAAABMCAwABAAAAAgAAAGmHBAABAAAA0gAAAIwDAABOSUtPTiBD\nT1JQT1JBVElPTgBOSUtPTiBEODAALAEAAAEAAAAsAQAAAQAAAFBpY2FzYSAzLjAAADIwMDc6MTE6\nMTggMTM6MTM6MDcAKACaggUAAQAAALgCAACdggUAAQAAAMACAAAiiAMAAQAAAAIAAAAniAMAAQAA\nAEAGAAAAkAcABAAAADAyMjEDkAIAFAAAAMgCAAAEkAIAFAAAANwCAAABkQcABAAAAAECAwACkQUA\nAQAAAPACAAAEkgoAAQAAAPgCAAAFkgUAAQAAAAADAAAHkgMAAQAAAAIAAAAIkgMAAQAAAAAAAAAJ\nkgMAAQAAAAAAAAAKkgUAAQAAAAgDAACGkgcALAAAABADAACQkgIAAwAAADEwAACRkgIAAwAAADEw\nAACSkgIAAwAAADEwAAAAoAcABAAAADAxMDABoAMAAQAAAP//AAACoAMAAQAAALgCAAADoAMAAQAA\nAGUCAAAFoAQAAQAAAG4DAAAXogMAAQAAAAIAAAAAowcAAQAAAAMAAAABowcAAQAAAAEAAAACowcA\nCAAAADwDAAABpAMAAQAAAAAAAAACpAMAAQAAAAAAAAADpAMAAQAAAAAAAAAEpAUAAQAAAEQDAAAF\npAMAAQAAAEUAAAAGpAMAAQAAAAAAAAAHpAMAAQAAAAIAAAAIpAMAAQAAAAAAAAAJpAMAAQAAAAAA\nAAAKpAMAAQAAAAAAAAAMpAMAAQAAAAAAAAAgpAIAIQAAAEwDAAAAAAAACgAAAIgTAABuAAAACgAA\nADIwMDc6MTE6MTggMTM6MTM6MDcAMjAwNzoxMToxOCAxMzoxMzowNwACAAAAAQAAAAAAAAAGAAAA\nMAAAAAoAAADMAQAACgAAAEFTQ0lJAAAAICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg\nICAgAAIAAgECAAEBAAAAAQAAAGQzMzk2MmE2YzhmOWMwZTZmNDY5ZmQ5OWQ3NmE0ZTFhAAACAAEA\nAgAEAAAAUjk4AAIABwAEAAAAMDEwMAAAAAAGAAMBAwABAAAABgAAABoBBQABAAAA2gMAABsBBQAB\nAAAA4gMAACgBAwABAAAAAgAAAAECBAABAAAA6gMAAAICBAABAAAA4SQAAAAAAABIAAAAAQAAAEgA\nAAABAAAA/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAUDBAQEAwUEBAQFBQUGBwwIBwcHBw8LCwkM\nEQ8SEhEPERETFhwXExQaFRERGCEYGh0dHx8fExciJCIeJBweHx7/2wBDAQUFBQcGBw4ICA4eFBEU\nHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh7/wAARCACQ\nAKADASIAAhEBAxEB/8QAHQAAAgMAAwEBAAAAAAAAAAAABQYDBAcBAggACf/EAD0QAAIBAwMCBQIE\nBQEHBAMAAAECAwQFEQASIQYxEyJBUWEUcQcygZEVI0KhsdEIJDNSYnLhksHw8RYlQ//EABoBAAMB\nAQEBAAAAAAAAAAAAAAIDBAEFAAb/xAApEQACAgICAAUEAgMAAAAAAAABAgARAyESMQQTIkFRFCMy\nYULwcYGR/9oADAMBAAIRAxEAPwAv/HnmsH8PpenZ3mgBSLxTujZA39JByRuY8eudJPVjGa4B5LaY\nJaeCTDKpDSysoA4yTjIPro9TVFO3Sgolp8zGbf4u4oY0A4z8k/20Ou00Yhgpp2lZyjxq4G2TKgMM\nZ5B47+owM6+eViGqp1yP1FlqyjaxR0s0UkddTSeSZ4yoYHGUJXJ4xx27507WWlxboaqSfdTFfHUx\nhSGcdgxxyQCMnnPwdIF4mt9LaZKWnhqlnHhsk2MqcM2SfY4IH3B0/dPQ09HY6Jn/AJkDDYVUZ2eQ\ns7c8457D9dMf8OX7iMIJajK9ypadjJJUocUyoPHeXdvU5/5s+bsc9uf2pfQ2uSumWg8GSJlUzRsd\nojwM557j324+dGbhRWyrp2pI6pFkXacYz4YbGSOeOBnGfTXXwqSCljXx4fEmBWDw4ck4ABzkYznA\nGf30CZFGobYtaMD1PT1PPSQrAkQkR1bcBxJx3DAAHv2zqg1pFRC8CW+lQMQ/1KIyyK2ecnP24+dF\naylp5w863ZI44wiSM9RjGD+cqOOTj7DUs3gRJI1SzLHyZJIoyWOAOVAPtjn40wZVvRgDAGlGgtax\n07xxxnZGWy7K380842jPcDPP21VahKKS+6eF5F2gtyFA7Y+MDV1Lt/uCzYlaMkhCQNw5IAwPgHUd\ntraxwalqSYQE/wAtfDJXjjI476MdXGrjUUJbtjR7WViAisieJuwEBJ4+eQNcwUhpquqR6iZfFUIF\nPfOc8ftoUr1clNKqUskshw4VoezKcjP+mo7pdjVQQeI+yQjLNwMP6jk/P9tCFLdQy4B3OOpI/Duo\nZyJR5jk8kew0DWOYXCKRpS8CSlmXsDjlef8AOp75WK8BWZtwVg2Q+DtPGToatY1PLI7SKXh4kjxn\nBJwOPgaoCHjEsRyhmZXmK78SR5Lh93ZvnVK4gwSxNltzjJ5wM6oTVErqkaSMFY5bnC64qjlQrTne\nF/N8/OjRdxbNYkN0UTQNH4oMj8EEcsdDLlSvS0zRDxFjC5ZuwY40St0SO6zNIzsmcBRxwM5OuL9J\nHFaZKiPLvMhVge45xj/309TJ2FxLMbeAW9HHJPxpp6As63TqW1U00ZaNnRZVBx5c5P8AbOlqpnWS\n3eGUaNlPOdO34RT461s6r5jLMirkcfm5/tos/wCOpuD8pvbvFWVIpq+lUUok8KSpgCli6nO7aBxg\n/IJGkPqy32TwJissUtTHKBHUyRyAyLvxgZfAGQe6440h13Wd4heGxwzvPAuCTvJAb1PHcDPr6aI1\nlwmmipKC4MKmJ9oEwYKT3IHpn8x7/wDtrnvhIaxHHMOveHaKjjuUD0cstO8hkSZJDH5SFbAUYIB4\nyc49vnJu7We5VMNTBa0gkowqtHG6jKTH+s+vBwB6e+lEXgSSR0MMjx58rrPjg4wGx6qD84OityvN\nwp6QVLsadxG8CyodqjGOQQSCTg8dhkambGysKP8Aqbjb3MYbXXVVTRlKqbZWQSCNAhA8yjLKzHna\nck4GdD7hcJvElrbpDSxQjP06wEhkGeeAcEd8/OgVkvlbBW1FLcqylnp4nRmnidUd29FODuAO7Hvn\nVPqWpoquetmkZmjjIWEyOFd19Rx39fknRJiYGmEMsK5XLNFdqW6XqhtMTQW6ilqo1aaNMhyWH5j6\nliQO2OfbVl457DR0VJVCrhkiYGqpphukgUs6kqR+ZDt3cYwD66RKKWsW4Ca30TNI00bRwRKXyQwY\nYxkk+XTh+LfVNNeKy2fQwTQ1EdO31MMmY5Fy3l3DGf8An4+fnTnwgOOMiGT7gaCrrfrhFE0EdUTG\n7r4ZiY7WVd2VHr7ao2e63W9STUvjVBqypKHxiMZOMYPpyAP21HIqVMEdQ8YgZSqqFB3AgcH9f9dT\n2qmttHc4ploXWvkmXcpY7QMjLAjscgHVAIVZWTZFyamrI4Z6WkrvFH1EJkLOTkbTjPbPvx8a7x/V\nJcRIJV+mG7xy2SWA4QY9eCOdFuqbMt0ukd2ppJFCyFXdUJSHZw271IPfOOD7g6jaWjhttTDLUqy+\nJl1XPdDjePUZPb0P7aV5gPU1lI76gKFKq4Vs6ttgenhfY8jEI21sct2HPbPtoRQtcvq2p54pS6Sk\nDDDAGcEbux1Zoq3a0yJIyRuxG04Ynkckfv8Avq3JXQwztCpCujMrHIwQOOD+mqEXdGJGSxcnmpmh\npVFQY3U5O5TllPsR+2hDSzCUbsiXO0BRknPrq3Pc/qomEMOY9wy+eQQP7d9c0NDU1CPcoSHMaMAo\nOGU4BBH6HP6HRkBZllpYt6ywGWZlKiRSgA4xlhqG8Qwr0/UyBmeQyBORwuO+NX6eGrmtaTPUCFpW\nI2MM7io5/bGdCLhMaijq9tcjxRIWCbcZyRhs9icnS1azCI1FG4xFbQkxcBpHbK45wMYP2yT+2mro\nSTw620TNGZBFUIxUHGQCDjOprR0fJ1BaKbbWQ0kaGSGd5DyZc5UIuecqR+oPxkx07ZJLPdqGORld\nDIAxJGEKt2PseNb4jMn43uD4ceuoQsvTT092lkm8KilZi0Mm45Cn8pb459B6aOV3T1NR3Z7fWb5l\njiXwphEAry4BIB98559vTVW63+vhpCl2oaMUodGp5EkImcr2D5H5cHsO2NDbjfbhVRRzysTFHtyU\nJXk+vvqUpkc2ZuPJi7WdKpam21qT1gjSSNWRXK7vIQwDZB5bn7Dtq5PTq9sWSvkSpT6XwwJIiwUt\nyAfU4AB9yNL1xgmq7l4yVE4WMIBhd+TnPYcY9865udVNUTbom2vGwdWkbIVh6gDt2A0QQnXvPc0s\nyx03LS2WhmNVa6CtkmAZTUANhjnzYHJwSACe3+KdytddcGiqaVvGHieSnQNlSedq/wCmqtVXQzTt\nJIki1JQeJKqgZ48y4Axg/uNTNcpYaKllpq5qZ4ZTKHBIYED+xHvpgUg8h3Fkg99S34FRQNA1Kf8A\neJV3FIxllYE4z7f/AHoxfrDHUTpdr5c6L6+pLhvDDNMkinnj8rjAHwP00u0Vuv3UNSsdhpqqtq5H\nJPhgg7T3PwM51pNq/CK/vJG3VV+hSMKv8qKQzzoDyRnsrfqdacbkijDHD3EVYWs9ZGVw9K//APVn\njG3j8pBHuM8Z10uNDU/RR1dsgkjaFgwOw/zBtIJ57kHHpjW4dPW3oS0VAmtlhgJgwoasYysWAxnn\njOnSh6upHbw3oaXb6KIVwNaMDgaheatUZ5yt97r46G4R+L4DTK0EpDHawxnAOMZIPbHroBPRfWUy\n0tNIkbTKJJIkkB3jnHf17jGfXXrWooOgb7TOl8tVES4wWjj2EfqPXWDfiA34J9KVk1P0z1XejWsp\njMNJFHVRxH/vfGD/ANpOh+kceoCCcytQuZDNQx0N1qqcMQKcqsjlgxjOM984PfH6auWyl6XnnkNR\nc6szI20FV2jJxg7hn/xzrYei/wAELx1Z019fSVcNN9U6TLU14/nSgDALIpKjOT/pqjW/7N/4h2us\neaCjs9yp2YsTBVHxR3IO11UA5x202no2ICkCIF16L+hmDwtKsbsGaabJU4Hdcfm/NjVxqmntNKVi\ntv1E8GzDbTHHJMAQAFHOAGP3yPTR+7R3yWhFnu9tWlrKRPDWmJKuG4yzA8A9z3AIx7aSrndVmcLP\nVQzCGVd7KSvYEYHp65Ld+PjUYyNkcqRoR5KDqXpaqsqYHpXp4gyylo41AyRuIXkcZCkr9gM6W+pp\nPAtcdvRI4yZTvIUAtjBwPjg6PTT0c8LwWze07pvgjUHBZjnCnse3b1IPuNA+o6eYVVNU1SCVagHf\nkbfDk4BX4wTn5B+NOxgAiY49Op0keotddbYaeVjUQPJWTzKu9Y2Ixg4PIAXIxycnRDpauli6qieq\nBnaedY2iL4DbwM5yewznGh9Hdqqy2lnpaZT4rNC2fzFQMD7D11BSNG12SsiZttM7StIicEkDAA/T\n+2iyjlMwsA0aPBpZ6A0dwiNZvQRlg5JA9MZ7D41ak6ft1tnDQVEdPGacRKkhdvEHocnsc/41SpKG\nZUanjZWkdhuVSGx/f/586IXNJIY6eKWOOdIIgm5s53DJJH6nH6aPIygcYoJ/KAz9DHWSW53qQ28Y\nUZO9j7H8uABq5RLBWRiijpnR8lR4hG/ce3I/xrvDR0dYoLVAoJIMsh8MtknvnH30Ot00lo308c5F\nR4u7fjO49xydJRh8wQBcJT26koK96esgcSCM7Sh8uCMcg9zx3B10ttrihqYYacb97BcOQwHue3Gh\n9ZVV1ZVO0kxnXA2tj8vwf30QolqbdQSXCdNqSeWIyOEDY79+dMJsbMIJc0K33lLHSihsdKGbnxpo\n15c/J74+NcW+/wBZWVDSO0qgNjBJA/zrKoL5V3C4DewhjQ5CQnAz7/J+dPFllTaAO+OTnk6qxgiA\n1e0dbVLvR855cnvorFUeF8fPvpctE6LCWZhgHPfXN0uyUtFUVrHiGNmA+wzqgECJIuIH4/fiPUQh\n+lbNUmN2X/fpUbkA9owfT3P6DWV/htQveOtbXRFd4knXcvwOTpdutdNcLpU107b5Z5WkY+5Jzpv/\nAAPmEX4kW52baFDYPzjRA2ZlcRP0O/D+qUUUMEbBfDULtHbA0+QvuXONYB0jfngq1IZguf31tXT9\ncKqkR94OR76nb0tGrtZj/wDtU9GJUUlL1tRrGk9ERFXMSeYedrAD+oE47cg/GvKPUFkudRW/WU9L\nNMlVgRDcvnI25wc4P5gf/rX6A/iZStX9BXynSnhqH+ikeOOZdyM6ruAI9eRrxx+F1VWVVvrYahqs\nxw1IqIZKdf5Ue9RuAyM9j2HzjnXP8ZaHmOo1fuAIYm2y23K3VlN9NVSRTU8oYSIMGGRs5UevAweO\nMnTzVWm6XLpqf+LVSNOk2XuM8Y3sjEh8gY44G3IzgA5GeDge100zimgpmpHcHNKSRu9gSN2T+320\nN61o5J+mKmoo5qqCSQsm1186xjcfT7EeuCRqE+JTkNgToDCmMRIraZYulI6RVQtTv4NNI0uXnVFz\nxg8nHYf9QGTo5aOh6ux1VDU1ciVdtuIibbA+xlVyOWDAltoLHHrjuO+l+w2eS4dVwU8Eppo7bMoN\nOzbmC7ckg9sZH6Z09dQQ/wAHUk2xq8uNyM7l0Q5ABJ7JjH64x6aod61fcmUcrY9CQ0lDSwVRBWbm\nNshFxiRcAE+xJ7Z1DHY6usiNyk8dIg+dvhncV57A9znjOiFyrblDb2rqeokFZTyEoE4aUq2cAeuc\nHg/pnSh1r1fJW9U19ZAtVTU8jndGkzEJxg4GRjn00OItlJMEZFYQiIZZZY6Wjt0yxysd0wGdh4A3\nE550Dp+mq9Lq1HdqinSUkkEyDc3HDAnhV9yefbV3pnq/w6iWlX6mqWR1TxvDy3yT8fc8euouuY6u\n4LHeKGnlhozHt8WokCAMGxnaDk9u4zoxiKN8XEnIA2xCd6qenoZkt6QL49LEqrJyqyOBnuc5wSQG\n9fbSv1lWPV0KqVXyRZIB4BPoNVZJ4sK1TUq6qvZfOScfPIGdULjVLVosAYbpUCbXxnj1zp3h0Udd\nzebHuBOnqzw3ZXbawbBB1pVpqldEKyd/bWK17vQ14j8wIzkkY5z20y9K3+TxDFI/I7a6I2Ig6M2e\nnqcU+wNznQnrW4eB0tXndgimf9yNUaa5Hwd24dx/fQLrOsaqstXDu/NEdCWhgTHl5UMdNn4Rvjr6\n3D3Y/wCDpScHGPRfXRv8OqpaTrW2zyOFUS4LE+/GmgwCJvP43dTXrpro6FrNK0D1U3hS1CHDRDGQ\nF9s88/GrH+xP+InVtR1zX2a6XqqrbV/D5KhlqpTJ4bqRtKljkZycjtod+JNJHfLDFFIWfw2DRgHg\ntqT8Bemx0tcZqyuqBAapgH2qcmMHIT4ydKyGwQIzGKome5aCvhqAVLDJGcHsRrwp+OV7i/Db8Q7l\naLDcBNGkhkp46eQgxJIS+xm9GUnjGeDr1905do6kGqPkiP5B7Aa8r/7R/wCGP8R6+qOrbWr1FDOq\nmSnU4YSBgCAcHy451KoDin6jXXjtZwOtpp+kLVU3Kgimq/qPq3rtpEkw7GOTYcEEHgkcEaB9S3Ga\nuv8AKIKGuFkqZI5SqtmRsRqdpfgdhnHznQ64QXSokqIpKCWmdYUgigjR0ESrg524/fPvn10Pmgro\n7jHb62okqqKIJI8TT8SqpAKf+ny/bXP+mQm6FxeRjqNNjmHTF6a7NTwyxVsW+ONs++WAPwQPnsff\nQ289ZVqSStDG700x2GJVwQMk5Yj551Q6iBrTNPYIKhqGnqzL5XyIA58qgd/TH6a4tMVZb62ptl0t\n8lJWsu5fqFKhtw7bf6lIwcgjvkHVCIF2e4BcFSo6jhUXKRupZhMkksTSna3JO1QOcEeuBzj19dUL\nr0nVR9K1PUBtYoURs7aqoV3qyzeUoqgbR7bu+qcd8aKSpkXYlTM44Rdngk+gBzj9Mau9N3rbbqij\nkFRdROwVIquXwgpGdoVuRnvjjQoGUdUYvkOjEye6UTVRlRFpJnO1vp5cHJGD5ccc+2isrLUQC0mY\n1FRIi+FFJJkhiCQAPTtgj7as9Ux9NtUvOtO8QO3/AIr7nifngsMblHp20q2CtqoLqk0yBnnJjwpH\nJPYqT8+unn1ixGggye7dO1NrepWadlenyPDXz7z8juAM/wCmlGsrizxxwNyDuZzj8339tNNwvl4r\nLx/CaONXeaNA6xwlpCAASCe54HP20HvNrkhphWR+I2yYDBUeZT2wB/SCDz8gafjJA9U9xMB9cVEk\nt5m8SIQkFTt9iVBY/qSToVZqqOK5QPVSSxwbwJWiXLBfXAPfRu72mqrK2urqZf5W9nbkFuSeT/po\nJVIkMeePFUlDjsfnTkIqhMbuaZa7lZroj0tHcpoKjGKeKpCqJiF4w2eCT6HHfQTqqsq6aKelqI3h\nmA8NlbuP/g/zpFhMksyLuJ8wH21o9xhju1qobfdbrTUVUjhY3lBy8fuzHgn5JH+NAw4kGFdiosNv\np+g/Ep5JdtZXeHUbThfIhKo3qe+7vjjtxpcWVkcMrEEHII00fw2oS0XC2TrBG9PL4oLSMhwpZWkK\ngYYdwCeRnj10pnvpinuCZ6g6CSS59KWaaqO+QQK7k+p0QrbotLUjGBhuPnWafh31hu6PFvWcx1dG\nAh5wShPBHxoDeKjqnqXqNbba46jYrAqwyq/9zN2xrdXUzc9PdLda1dRRJRtwM8vn01J+Kt5q7X0X\nT1ltcCf6+MCTBOzysc/uB341l9LS9Q2rpi7VsQaSWhoZJAyrkFwo5H751PXdW1qfhPaoruGlrauR\nGBBIwq8F2I55Bx+vxpbIOBUiEWNg3KPU3X/VV5kihuHh1dUFJWqMAEgBP/OPbGghhYzRK0paN1JO\n5icHPuf0++njo6opp61qa/3Kmlt8aloZooip2EHjB7+mMj76ltFLba67RUMUUNL4qNmSopSvmJwF\nJ9Nx4/01yEyOpNpX+N3G8C3vEq0yxW2++LJC7U/ixmojV2VCi8bjtxnnn07acbteabqS00UU1kar\nrrJzR1kMxjPgsSfDcMSPtj7eurPU1LXWmtFN4aVdQOJmpysmSB5MYyGGO4x6aHRw5ukNR9VHSE7i\n0daWGST6egBJ+w450vmmQ8wKb9/254ow0DqUeo7HDTV1VUUkbNb9zLG+0qAD2BHtn29hoVSQS2Sg\nrqevWBldPGhlRvEGVONqHsoIzk+vHHOtPoGSahmtlFLJBLTxlA0rrUEsOSc5wCfNweftoWxNupxH\nUQislqYX8WA0++JomUMNozzzk5HbaedDi8U11f8A2AArmiNTKKq80cyfSPRB6IVAeaMOsTSEDg78\nEheTj5z99Are9vWqgZ5qhoFnHkJHiKM/0+nb99ONbB039Y6UltlpqiIENlQrOox2VSeD76WqeWx3\nWWq//VmndnjY4fI54Iz3x27ep101Y31BRK1LEc4o73PWxTPFUQQbBLHwGSQDk47gqxBH+uglVUzy\nVxhmeSWNQzRsXyFG0kAD2BA405Wq3Gw2+lmrohc6CqoKyGONgAVZRgHJ9VyuB9tCqpLXVfw6W20r\nUlQLYkVWARjx1BVmHyVIb0OdMXRjGixvrIY2twm8Ohz4kTOgDMMc8f5zoV1VRLT3doKZ/qI8Bg4G\nCcjJzpthaljip7cYzU0cwIldMeKnP/EXPqPX0I1Xv9oMNwlA+nMRRWWWSTYrKwyGA9c/HOiVwGqC\nVJFxGgTYwl8PDKeR76arnDBebVK9DIgjp8uqyYViABkYznvnjVmk6R/i8sXhXilo4VUh6iRG8BDz\n3YDP7/vq/feg7NT1dPP011DDdk8FfHjicOyzdmCuMZU9xnkduda2RWIF0ZhUgSjdquluVkoIfoK4\nQiiQGsdN5LjhlDe2QfLntjjPOka60i01ZJHHIJYwfJIvZh6HT1TUnUFikr4KO11k9NVwvSxoHLKs\npI86qCfYjt699QdSQzTWuLp+oscdFdKCqmac4O9EYKdhA54Ibv76NGo6mEe8R6CqnoqlZ6dirjg/\nI9tbR+G3U1B/C3mnIieMDxB7D31j89qrY7PBdpIfDo55WiidmAMjL+bA7kDPJxjRTomsSmu0K1IP\n07Ha43YGPc/A0zV3BPU9Mz9U22s/D64ra7pTzVNTEYYUXli7cAYHr8ay9a6pcLHVPLFVQSBcsm0Z\nHpg9jnjjjRC6UkdlejTxqeCObfKjU0PO3HGfQliQAe4HOhVFX0twnEMcU0aRwuJGll3sTjJP9j++\nk5H5bie+oftkbJRGsqqldySn83O19p2qPfnnU1rv18pr3G0VTG8RIDrszxjlj/T6f+NUb5HSUMNP\nQzyGelnRZSwiCsrFRnBz6Zx/fX1JDS0FPXTGoCnasKSFi+Q54GcnsAeMajfkELVuVFgaX4jF+H1X\nBP1bTrNUPHRmY73lUuBxyTg8DOiXW1Zb7l1LLTU+6KzfU7ad5sDYBwwJ7hTjgn3GlmqaWx2xXj2V\nCPl96EKRxnnnnSlX3qP6OC4Q1qzVM25ZElcAoRyOPbHbU58F5xVnGxFL4gp7Xc9CWnpuZ7PPb73b\nYJqoARzTwjDygHyuknqcAAj1zzqo9LcKaKncR3C1sQ6qgcsQobAXcQBkqMkDHB0V6c6wpV6iqbhe\nqSWkeuZ44Itw8BCDnvkBSfY98HtorfqLq+/WVbxVS+DbxVNB9NKmxwvCrIuG5U+Yd/QHsdcpUyE8\n1l+IBSPgzK+rem79d7n9askE7PTNEoLDChP6R282MHkewzpMl6Ar3MUsLvRyMgkYSqFb5JH9K/8A\nUfjW10tTBDc0hDR1FHCsQXwfzBsbWU55J75z/jX3U1H0ZWR1cd/uM1oqkK+HVS8ROnJ2q/uccBsY\nxxnV2PxTWADBZUJsTNOm7DRyVxsdzvH8QieciEeIdkckiAHhcMARj/09jop1T0Jct92razp16QUt\nMrMYqkBZkRApkRSDvG0AnHPGn2u6LsFNHabh0tVxVNNTUQeKbwldw7HxFlbBB/rzg9xgar9Q9UXq\nsn/g1dHTyRgnw6iMOq8AjAG0FT98+um/UG+9xvkrXqmA09EtLbZ62nheopcbiy8iNScc+2W/xqxb\nJKW7WSdqqhq51olLo7oNihiBhWBzjI7HHfR38U6C4w9OQx2W2oaCAL9S1OGMcr4DbyTgtjdjJzg8\nY1Q/D2z3CipaW5RySUsck+0oxwIwpJZsdgP+73Ppql8iPj5kycfbej1L3RdBFDDd6OtZYaWmZZ5m\nZWaOIop3lB2J7Y98Z476p9K2WlWWPqaO2iqpZZHSIOMKGOCGcLx2BOPUgg6LXG4wXKK4yUMcx6bU\nGWp2SukfmOGfYPJkHkY984GmDoCitcPTVNS11PPPRku8VPFGVbbIvEu30bbxtycE7s6HkQC3zHcA\n9D4irtnoxFe2eWnmnaR6BIx5OM4K87iAMjQa2dP1P0dyuE1jWpkqaJzBPGGy0mAScEglvOM+2dav\n11YazqKzwLR1kIggIeKOIbHcNgsScA89v09tALHR9TQ09eKmnpoaWiQxbKlhIDA4ACpj0wOeQRnn\nnjWrmoak2TEQTy6mV3m03y0W6zxRrG1PZ5pK00tVGjlZM5x2yyHaPKeO+lPp2w3i5XSNIaUu0pLD\ngLHjuST2AA16Ou3TVvvFqpL1bKiWgqKaFoXpanzLtyfOjHknv37jWctar5TVMqvSSxlJf5kMgBUs\nRypGePTtp/h84ZdncAqwP6iJX3Ova6RyVz1lRFDkSQsSiCMdxkDtwDnHpp7oOmrlR3a310XgGaeN\nZGgE3BV1/IynBGR9++iNLbaSop5pZILbHMkbTKrt/wAQD8yKTgFgPT11ap6Wa5XW3VwhljMUUcvi\nAbVKhgoHz3AA0x8y0QO4ePFsGUoLfNceqayO4V/0lNSSyJBCrZ3gLtUEn7Zz76ZaLpux/wD45J/E\nKSuTfIsoHj4JIBw2T6d/vqvNSwVtO8luhyY12TRGIhnI9d4/MWGQPnQDqKoqI0WGDqGKnEMcaNBU\npkqwAydvc5XB7+upnZjQU7jHWgWM63cySSxxFUMaDEcRl5c59fUf+ND57DQ3qJo6qop7fNSp4k6p\nEoDsScHIx749dfXK/ULUkEMkkVZNFGUaSniMZ59e3J5+dEYa7pw2SK4U6lqqLAZKpcuG+M+h0as4\nFyO1A6m7XmCghpDLNJOKWXyM5iUhXHJ3Kc/mzzuz21FepKNqKGH+KrS7yIzuQuqN3U/9J5z7aTb5\n1B9U0K04d2aZGeVnHhRgN5uBwy8Abvg6jnaVrdUW+63KhpJJXaY06VTErnG0bVBHPB78c576427q\ndAsQTu400sVJabW0CmnrzJISJQ20qEA49Qc+x7ao9ddC1vVdqkprXMrSQSFlmYkqkZGXXjJI9sDO\nrtpqqaSE0NHIXxTIoSZPJuAzJjJ82O3oR6asW2qrqGn+hp6c0yRSKxVU88isvfcx/pB5GhxLxbl/\nRDxYiVNxb6VtM1usMVtrb9LPVBijvCCHWLaQgGQcgHGMgHjHpo90/so77C/8QqZpWpV8R3AGEfsQ\nBxww5z9vTQ+oenp614qSRlWodZJncouWBOW+2WOcfprrUpDS0yQ1Fc60CVJjWVosxFcBmUEZJY5H\nB0eRATyM0rl48YVul8t9Qn0ayRMzSSBoETCrFypYk9h3JI7a7WFWh6bnjWFJ3h3NBIZ1H1KE7iyk\n9+Vw3uDxq50nUUVrrKjxLVkzRTfVpLCWA3DaV+AQvI0Dul3CuKOjoDDTedU2o4Cj02HGWwAB2Gg8\nmtXFFfmK1NLboamakNgelSfczRSxlUVtwOBznbgep7D407pQRokdVFuz9P8A7ukcn8tUVu2fT29v\nTSPbK67V31MErolJHBLktgMzFfKp9e+OM5010bLQWCa4TU7y0kkC+EDy4bg7eDkLle3+dPDEP8iH\njB5cjJLhDJWRCSkuD0+CFXyZkyucqMnAz8+/rqJYqy6tR0VyoPpKXwuI4XEbscHLjHGMHt86gmvt\nM0q0m1kSdhJAY23GNSOWyVwME8jnAOec6+iueLbV18lLLTSUziKsjnBfY+fKcjAwRjGO457DROxA\n1HuB3IZ666U96Nt6gWlqoXA+mkVQo8MglQ2AAWOD30rdV/WJ1JT0bXWKspxHEmZIlKbTwPKP6vQ9\nsDGNMFfW0NfJQRzU8FXTyTrGrA5ZCeB68c5x8am6ptNlS41FNTxurqgRSGDMvlA3Bie2RjnR4T/I\nxC4ibgKegp6msRZHhgkgiZYFUeXOQOOMjzZ+2dT1MP0MkEcVVHIZMlkPmAbHJ5HOCP7euvkug/js\n0cQcmMFQ0fmCKGweRnOWJ/XReGaJLrDS0kUsgKNG+5QMPxjzDt3/ALaY2T1dbnuFE1APgtGGrZ1r\nssm8PGyhBzwwK+4G3A0udVSR1dxkrWt6EsATs8pAxz5ff5GdGbzd7gakU9bEklOg2ySRuS6qe7MM\nADOQOR7aFWK2dN3quekrY6unmcr4NTT1B2kH3TPbOeAR205Az76MVmBHpEX6+8wUFNE0lHVyxOCh\n8AKCMjjg9x29s6DWeohrqs08PiNTiILHG9KuXAOfMRx6nHOdaRe+maKx0ErTV0FasiMYPDJLSYI7\n5/bIzj50Pi6foqi3+NaauKFiAZcKokXA/Lkdu/cjVSKePEmzJSv6n//Z/+0ALFBob3Rvc2hvcCAz\nLjAAOEJJTQQEAAAAAAAQHAJQAAtQaWNhc2EgMi43AP/bAEMACQYGCAYFCQgHCAoJCQoNFg4NDAwN\nGhMUEBYfHCEgHxweHiMnMiojJS8lHh4rOywvMzU4ODghKj1BPDZBMjc4Nf/bAEMBCQoKDQsNGQ4O\nGTUkHiQ1NTU1NTU1NTU1NTU1NTU1NTU1NTU1NTU1NTU1NTU1NTU1NTU1NTU1NTU1NTU1NTU1Nf/A\nABEIAmUCuAMBIQACEQEDEQH/xAAcAAACAwEBAQEAAAAAAAAAAAAEBQIDBgEABwj/xABFEAACAQMD\nAgQEBAUDAwQBAAsBAgMABBEFEiExQRMiUWEGFHGBIzKRoRVCscHRB1LwJDPhFmJy8UMlNIKikiZT\nVGOywv/EABoBAAMBAQEBAAAAAAAAAAAAAAECAwAEBQb/xAAnEQACAwADAQACAwEBAQEBAQAAAQIR\nIQMSMUEiUQQyYRNxQoEFFP/aAAwDAQACEQMRAD8Aohe9FxDc6cUlSKQP4pOGGP5WXtX0DWtRtte0\nSC6hmMc6tzHzuX1GK83taaR2dGnYssbe/vIwwc+DgjzLikzpqF3BJGiuio+Gk6Lx3xUusktGaTeF\ncV3c29kyWbm5Lkh2VSvOemKXvZzXV4Pl5HDbNzB8Aqw5I96rFCOIq1/TZo9Mtb7cRHK5RoT1Dc5P\n0pfps3/UwxmIyiQhdq9ft71bxCpaFfEUm7V7m6W3McU7HYjDkDhc0Uyx2+nwNZJuuRCiumMqoHOS\nfeggSGF38PWN38LJq2nQ3Zui48SI/lXjnA7ikELL86GyF3HnHFJfwCRJbnwNYd0kDRHAYPlgw9MU\n2D/xdbuWRFSS2XEZRcDA7c+o/pTNUC9CtL02XauLlENxJ4arLjC8ZzTSGe9h/idlGiwyRxLJHIvA\nJbqxHfoakm/p0XYn0a3NxHcWNw/iSt5wDhllwcgg9qLvNXGjWkJtreCOSWRhIBwAAAM8e9P8JT8o\n697FdTmM2/nVAHZuhJ9PbNIHlmcXNkITDIrkCQDKkDrz1FITiiUsTyskdxdymaTaABypGMA/pV1x\npc0cW13LAnakh6H71otjqCBX0d4LUsirJMhJ3Bfy/egrS/1LUZ1sJ7ovbI3KsM/Tn6V1J5YvWmaG\nCwUbS0m/wjhXXAwOOuf0+1J57bUIWmCTRFQ4Vwfyspz1H6VJcj+lJcarDngsNThnbyIh80fXP09q\ntudDSfU90MqRxSeZ+2yiuT8iP/NluqafBDZ79NbxJYRjY3Acd/vSOBZkvNzqUDDpvzj7VRteoEoD\nY7Xi2yJn3A5oGTR5p9pswrAHB56UilRJwoOPwvfW9r4phEqKuW8M520DPYXto6uUMK7d24nHH17U\nbsZIimqfMWq/OoWVZCpcDpkcVYPk5YTJBllB8xHQY9aEnRnjDLdo/nYooycyqJNyjt3o3UNQmuIp\nRv3gkKML/KBXPFSbLrw9bwpbac15I4imKbA2cEEj/ANC/D0gGqPNL5I4H3ZK5/8AvtV0ZO8PG+tr\ni9AkV/DzncOCaIiaK1DmBCW2kgN6UGg0gO4t5ZUEhLeEec+/pVsQZbWVnkKHaASO/Ocf/u0fgY3Y\nwsvEjia8EflIVVPpjqfeoXSN/Fl279zEcL/NkA80jLhz3Ud1aXMuSu5wsaleBt4xVVncvFoc0Msx\neV5sbh1HlPT70V4B+gMd0CSZmYhSE3L161Tc3kdu09vOrOrSCQSZzgY6fvWuhJDnQbmF7YytFkli\niIfLknndx2FUy3dy7R3aY25O9Q/JHoD3pRkEQskNtkxyDlix3cgY70rS8SW2ntokaaG6Hhyv0xzk\n/emTBQrn08fNNG+4qvPI6jFdtY5IrXw422wZ3bexNU+Eq0tDucEAbmBzVfixw2rS5PjKRiMp+YdO\ntMmaj1u5jnaXaF9N3+Kou5UZy0jsw6hR0oSYUg3QmuLgbw34RPhqpHBP/P61bc3kUmpTlECg+VVH\nQEVOPpRrCVo0gX8Rwf5V469T0oK+KzrGYUCSbdr59R3/AEosFYDlPCJCk9VbFHM5lYgAbVY9PSmQ\ngOYwWLZPA4JqDyqzAEE4PBFOhSZnKhwGAJ6A1dHJiDqAMjPFFIVgsh8VgD5sDIquRNrZHXPAxRMR\nlZktsnBbLE0pmj3OWDZyeT6UUIzuw4IyM+tXWqsv5mJwaIAiXUZYT4cbFdw5xx+9BXEzMQ7ZJPUt\n1NFABC+7IqpgSP6U5i2BscHtXpOoUevNYIMwJkwOnSj7dMwD260kh4lM+V6dc1bAm4ZNKEP01DLM\nqqCWJOBXqlJpMvGLas+r201s0Bjey/6V2zLJbxFTn/3Ee9N7e3sLW3Bt/EVic7n8xP1rlgsKSZK6\nnk3eFbCN2UAsC+CPoKV3GpsjT4miMcQBYY8x+n9Ko236JQmT4vmWVJBaOsr+TAXyD3o651nSbm9K\nSQKxwfxx5GjIx7c1kwdfpnfiRXu547WQW7TXL+LBPv2hlA6H0J4qj4WsYrS+F5fLK0lsjziJDhk2\nDq3tn0pn5ZhYl3JNsV1+Z35ysnUd8g9vpQ2l/PNdzRWayHKneipkY6UVSJTlbPoPw58R2EOkwWeo\nt4c8W+HjjjsRxzwcfas/efDj20F3fLIMRyYh6fiqepx24qb9w3gosrm3a4ht/B3SNJtkLDgrx0o3\nUIbrRrdUidHtpJd6ug6kD8p+lUav0Ve2MxYCfTIZZFVS2GDo+5uRxxVUNzNHdxJ4jGUEqXYdPbnt\n7VGbo6Yqy/TNGtEWSSK6azuHbbjaQAPWrtRT5orDIqSLbS+VgMiQ+pPXHFTcpVhOf6QSJlurdBFC\nUK8tg9B1xVc2lNq1ops7iPxVJYhWyWP+0/YVaLtWIsF4igk0+GJlKHO5GjbB78nvjnpUtPsvCWKO\nW5aaGL+WQ9DW0dMNuEsxGcQuWYZYoc7j6fWkM1rbQxSRQb4ujdcOKetwDk16ER2yTgG6mMURwokJ\nINHC1s7MSSWkP8SJVdrySbV3DOOO9GXgsZi63t7+5vJPmreFHb8QHbgD2FNI9LUrlkQll2kBsVNo\n0OTdALi2S3jcxKojA2FQ+X+vNLIo1jvMOqqFAbxWxx7UyeDyCV1Pwrltq7iTyPaikvI4i0pDje2c\nAYAoXRN0EwfFZRzDbWwmkHmILY8tV63eNe6Wsyolv5mEqMAwZe33qjkkgLj+2I4BD8k8cbeIMgsV\nTpjpUg62jrtWMLKQHXZ1FB6zNGkMVmXDBBGyx4DA4yKVyXW6dba3HiO+QAmDjPr9/wCtOkkgK2Xz\n3Eyta262+fDVss/IZs88fahb+0l+Q+aTPiO/Matghe5qS9OisFsMAedjKRGq87t1NrezkumwjDw9\nvJqsiSWlPjRi4Eal325UL1Bou/s1fTmRMxhypz1xgN/mpvCsSsTrDbFVmkKxZVU2evoaM0d2kkgu\nmwqqhG5u5HI/xStFC+aSNLCJdq95SQfXtj171QUZLT5oRna7ZG5OT2z9M0VdCt6ByQYO5VBRyd64\nxS6W0/DZm3nJ48nY/wD1QYHo80OUwaZHhH8RmKsSMckY/pz96Ga1nW8ihlI2oGcgt0DHj9qUZLAm\nSKRYpmkVxEy7cjqKXQJHaFY03kLlvenQAKeZ3MkrlljL4UNwc45qyO4VLcqhPhjB4HrVCb9Jz75I\nGFqyh1G4b+9KjulujFIeV7ZooBXcLLDAX2ttU4oaIvcyKgJyxA49+KSfgV6a62RdMaGFmOYgwlAO\nAGIxx9DS/wAHMzSnON2STQ4/B5E4Lt4btWBUBTnJANRuYlM5IbIY5JH2p6EtnfCDylVAyzbVPuDU\nJrjfIwCqBn+TjNYB5bRnUyNwOnJ5odbVvEJaQBScD1zRTBRWibC288g9+9dvGkZVSM8Hk4qiFOCM\nhYxg5ztyTUblW3kBvynqKIGVZ3RlSdxOW5pbMHeQhSUHXA6UUKW2kMk3UAqOpNEhQinA9zWAwaY+\nI2QxJ9MVCVB4ePSigAUgCynb0zXSPLmnAcTGeamW647VjA+7Egz60dat+A31pGUickiDkfQ1OxG5\n1jI6ilQzNr8E6VDJLG7rucMe3TmvVw8km5M7YJKKDf8A1O8MUq2FzIu7AZSBkexGP61Zp3xVLuke\n9mknAYKscIKke5xjin/59Y4cblcqQ1vviSN9PeeKCVC42ReIc7h0b6Y9ay15DBPcwfLSTWQhH4gm\nlwOeR169felUWWX6DdNuow5sxrFrczEeVTG6njnG7GKr1fXZZ4p9NkhjVR5HdCNzfTFNV4M1WoRX\nMUEkFtAqnfCDufJyxJzzWg0eW3Oj6hHeTJbvJCLeOcjJKseR9sfvTyVKhYv1mYmMMKhDgyI/mKng\nL0/WmtjeLpAt7m2Phi4kZWkJ5CDA/uP0pZaiH0K0X4g0nTbsLqdvHOniljOIxv5/rRmrfK3jSSad\ndW8Vpco22Igg5HoMcUngabM/Nokcdubqe4Alzwu3zbsZH2plp9vLq9zFJOI/CZ2SNW4jVgPT+9Vb\nYYpDe70TUtL0oXjyQyqgDMi8Ag9MY7V3Rrq0N2Fvo8xygYk2qQG/TNRn6MmxtfX9rHbt8rCs0znY\nDEuSKWQQNYzPBeXSSXMw8TaDkc8D+lBf4F21oBPaXMyttGWXLsRJ5cccD3qjQNVWBGVWkV94YrgE\nYHr6UUI/Au60sR3LXFk00UXMjbOFIPUcV1L22W7lDWcY2kMrSKQcEcZ/SqI3g0tV2l5fkwgjAcuW\nIBB6Y9TSue0t5Wd/mZHnYhsIozx7cZrXQX+XoovL2QSlbq3nZW4DSQtgen0plZWs/wDCmltsmRHA\n2DjcKpWWczTTorgkvEmYSqQpkC4HUZHT60SLXWLyLZBG0BH88gNRv4GMHYTeaUk0QEq7LhVxuUYD\nH1PtSUWXgXTfNqGXGNg/K3v60yVFpMudISA0caKRzkcmqjbtcRN4cgbIyB61iTti20svDupfEQJL\nKoRZP9tGwxOqfLzlGSTK5z37EU/+lF4E6VpzJumt3HnBQbuuQe49KtawsjN4szvIQdxGcKPpSth+\nAyGGTUEAUMrNjBftRWmaLPHJcXkcCwu6FId7AHPdwO4AxS9ykUdFpb2bRW8szy3Od2VYgDI5yTXW\njikuRAuRIQTkEEFf80yX0exbIkYy0TLnlcHuavg3G1M0xaNFG0gHqew/52prsTqA28rG7fwwNo8u\n8NjJo95J1hgGzrJuBXDZoNjJUXfLLIGAkRdx6Zy2RniqtLnJsJYEAeJ1PhlRgKRwRj70tDljXU91\nGDsCFG2gr/MOnPtimkW0qS8rFgAoD+b3wPagLSuwObZIGKqSQTkA96Fm85XykBgPzHpisYLs23wB\ngWChz5D1PPWrNT8aNtyqpMmBI4IwncfWlHRK+PiSYeUlkGMp0NL3j8h3u5Oc4I4FOhWIdRjkt7tY\nXUqudw+9GxGNbAOR59vIXniq/Cb9IRSjzNGMnBU/ShpocyeIgUH1C81kBk5Zla2EbbTnOR60Hp1u\n2n30dxEyybTnaRQkrAsGccedrOxZ2fzE88nmpqkskjIqHb0B7UIpIf0FuIGVz2GOuKhbo2QdxyOM\nEdack7C8sWRkYAq+4Dtmq4o8OBJtwp7UBkFyvstU2hTlzx6UslYtyQeua0TMrR8tlsY968AHPP61\nQRlnQHG44rxA3HOP81kKUXFqY3Vm6j0Pag/BzcBQSMnP0omGAKpEdqkKvGe5oF3bxUxxkkn2oisp\nc7ZhtcEMM81yVW29QciigC98iTkVYqgp0pwEXQA4Bya6gOCD2NYxS6/jD0zxRlkQYm+tJIpEsfjk\nVbp0X4oY0i8Hl6fQPgTyoc9Q55x2r1cEn+TO1eIr1Wz0bULS3EDyw6jFGEaRTtEmOMN/mhrP4Ukj\nQzS3QURjJRcHdx0znvTrk6qmT6Xp2y0+TXGM8oMEcTeFECcAnGduM+lMtd+H4JbexMS7rlXSKVC+\nGkQkDIHt/eg+S/BWqZ2T4PstKuvFsmk3lsskrBjxg9R0pdJprgrJPEyvIxO9vM23tg0VO3Yz8oXR\n2aRW7thg27GX6nmpSyx3WnixnlCBSGXwkG5j707dipC9NGmdpfDkDog/KWGXqt9LlgVV3NxnapOe\ncCtZzyR1dKe5lXxGiyRnlsdK86CyURXkYlk3qVyx4TuB9ayeiM91vQ0C7kkO5VPOB6fvT34bvbUX\n0hneGDaGxlcc00tQ8HTNFprXE+mX9pdyB4rsbYWZ87ePT0qh9MgtZAloz+LFthZc4y2Dkj9qg9LJ\nEneOAoHjBlX85ZSCKE06SOLVrze6MfDBV8Y299ufpWTSNtENXurizuZVDePbTAtuxgofY98V7S7F\nJElRUHniw4ZOf3p/oqLppRommxwvMkcfI2hcgZ9aTDVbeG4laRxL4wC+c5AxyMUfAUFQaneW9nKh\nZJFdAYQ4JIXPUfp+1JrLUL2wvBcQQvJgnKSMDn9OQKokvQM04v8A+L6f+PvtTuAaErznHHXtXZfF\nsr6GcuqJCoxGg4PFBMFadX4iiubS6ElvhYcSeIvlCsTxz7UA890dSkaK9EjpHGC7EEcnk+lOqA4u\ny4XDQ6mWlWe4s5otxkiGSjZ/p7UX4tlNCVchstmPK8mpv0ZRF99aJHIroEAJPGKot9qZi4wD5Qvb\n049KwrRHVLVvDjKymJmbnAGCKObSLFBFFBMXuIwHCk4yw75pkY5FHdvcjZasiE5L7l4wf3oG+tr2\n4kkt3jZFZSAexOeOc0rHSI2eiXa3EKCN0jSMs8xPAHUH605lKy7GEjMUJRC3UL2rUPqBbu1FxbCO\nV8kDB4oSOxUyBlkZCoB4H5sVmwBSytK7LCimRfMwC4JHrVc0pktVChcGTOQO9BDoEUeDA3lAJZmy\nF69sfpV1rE0Vt4rYk3IPBEvAwTyawQaO/miuZluIwACAuDkMDx96L0TTBZ3RiMY2vhtuenuD2rGs\nJ1GD5SJnBSPdJhIScAVbbXBfax8MEL/K1A3089mv8OOTtaSYnK9qCYSjCoFbccbWGf1oGLZrtLSL\nwmA8TIy+cbVArkCpekIrFxu3YGelYKDGjLFgqkHJON3qc0DeuyRsOVZR5ge4pkKxRcszqu8xsW8y\nnOTioR+VQAw8wIIFVXhNkrQfiBIc5cHINROJE3JuLegrUYhbxMZF8SMnJOMjrUceZto+3tRsIQ80\ncEeWLFydxC44zU1lmGx13BWHO6l+msuuXa7h2oQPXIxQ624GVMm0gZ5opoRkkdEj8wBX96skgiws\nqhst05rMKZ0pbvp/hiSUXAlyoAG0jHQn1zS2fdHJ4eQT0JrIzKEXzndjAFWnGcDpiqE2QMu1ckZy\nMVEzMV8uDjsawpW9yxjCy9jkYrlvCJ2GWCEnk5zimMFlCA2TkZoeS1V18TdxnBWgBlPyaCVBHIMk\nYAYd6FnlRcjqysQQG7UUwADtu5PWr4E398CqIxySPB681wDCnOaICqPDXKDnrRsS7Y2AH81TkUiT\nTo+aYWkaiRPL5d2CRSLwaXp9C+H7RbASxpkqkzAZ64616vPkvyZ2p4jPRSyXGPCYLK3mIIBA9ad2\nN/CXMUknnSLCoV/Mc9sU8lYfUXXEcNybdGhkk8PMpROpft/allxqRttUUTwPbhANo5JU+p9qn1SJ\nzkl6EyazA9xGba6ZmkUePKfKhPU8fWrJVnlhMeXAkUbHJ4H0NKp09EUk/BbqdvJbIhQrcTZ3MqnG\nAPWhEVbc20kpk+YmY4iUYKAnjk1bjn2V0Flet3MFpfTW9rEoUsH3H8w9RSnxZGVAWO09M8fXmq1a\nOWT0IKNG9vGRIcguoHfJweau1q2UfjFvF3IMNggqR2qV08HUbQDp0STXkYkkMMcuR4g/lOKqOfHO\n1lfYcBsfm5x3rqi8J1TN1YXlvbXKuq/NsbcJ8so/mxyRnvxUPFmlsd0qLBNJdmVHIyQfQVyyZ1x3\n07E9/eXU0twyTIHKbZMgD+9LMRHUWBQgq48w4HHB/rQoekFnVrdtWkEsdzNG8wEWD5VGec13Wjd2\nsiQWKyBp3YIw6D/5H07inQtCu4vJrO7htp7pEKsC4lOWIIJ3Yz/ahDDpsV1IYs3DyEsHdjx7gdMf\nWmFlH9B7ac+o2szIk8ckMeQxHDAcbR+lS0tbexgCWolj1CXLLLJHnYMchewbPela/QuFtta+LZy4\n+YmuRh2JOQxJA4PrV09reS6YrjYm1gjeI20L168U/iNFKzlv/C7NZoJLie/doizxonhxtjtnvUbj\nW7DTHZLTSrRSVCszM0uR9+On9KyGsnB8VwqWtdREdvEoP/aQID+nvQc+oJaxxv4gfupCgE08hF6E\nxyyX06GFRIjoGDse/pQ2pQlLmKS0t5TcIfOEPX/NJ4ZoJGoNasj3cAlbwzuiHb3qTbQySrsy6eVM\n5x9aFhUSr515riGGOYxyf7RwOTxzRNjC11rEts4DSqSRIT5dg7k/U0G2bqXyX7wXklr5pI4kCk9p\nO3FU3NvI4EtpjYEAK4JKt60yQzJxzwBfDnkIkVfMcYyaXCzDXHji8ljQNyi4OB60RWglfCVHJLtt\nJG9uKtlK/LiOPABUNhRnB71mMmDWyJNLtlYrEAzNuXnGScCrLmzjWFplRlQY5PdCOgFJY4DE9ub2\nJEdz4iYDEZ2kD9u1SWzlmuIokZ/FiwpCv1IOefWiBl/xHLmwuk4a4eRGCOvQHt7UJ8MwzrcTvKp8\nIrgLjoa1YTbdj1pNylSCpAzgjHNDBZFSaYkJGighierE4xilKC25naCTdOAzlhxnIriXciMSjFQW\n528YHpTJYBvQmCZnlKr9u1VTSNPLsJwinaST1rJGKL5Ybc7goIbAOfpStJW+ZZ0QMADhRVIiSIxz\nb1HmIORnbV8yQrYeMu9pC2MngL9aLMiVtqZsZ4rm3GSAQ2R5c9D1qILMxYbSWycDtmtVGsJmgMtp\nyFDqACFwM80LzHGxYE4OBk5oI0sLrG4PikMMRleuM0Q1xbosjPkuOE4/rSP0UXSXSSHhTV8c6BW3\nNwBwCe9OgWdM6MpMQJ74zxQTlTLx37U6RrIv+fnGDxXZjsXnk+1MIytZQXGBnjpVLy7ZTyysvXy1\nkKyKp40nGGGcnjBo+1gVd7AAMBnBHWmMi9wBbsCDk87iaBmuFRVVAfU45JoGZ4wNeZJVraP8uG/M\na5rMcLWkSxWyReHHsLoOW+tFIUSIvlGcUTCu36VRGZYV8RvtwPWg5225wfaiAotfNdqeTTKEhi68\njzY/apyHieJCo2OxFNLIb7iEY58RePvSIeR9D+HZHngeRgAWuH4P1r1cEv7M7V4jO24tharMvzDE\noFeQR8Z7j61oNEa2vrg+BBiJQBwPMT/8jVWtJ3hob2yTQlhurVg8qBtwcYxmsRqNtJfXDSyTMznB\n9qjNnLN9mANYzKTmUYp5Za4I9K+Q8QSOFJDFMhR/mkejQWgL2UNtH40V28jSoA6ycce1e1J7bXLC\n2WZN0kQUNMi8gdiaun9R0vcM7eWKpOflyH8xUY9R/miNNspQqXDQeJh9u084JHFOpr6cko6Halpt\n1bfFdnazXAllniDELgBRnpUviDT7mwQ3EYmkiLFSSAcZGBxUo7TKRxGTjZo7lDnwyrk/8FObWCfN\ntcQsk5kfhMc8HHPtXZJ9UTStmhW2s7PVFluJ3RlfcGzjacZPTtnAq1NThXSI9Tz4tq8zKQvLJ6Gu\nK29Oug6K4jlR5YH8eMtkvjPOB1qp3gkRk2qHZshjjIHpT3gFZQ1i23LAAnjy/wBa4Lu6hjCSkAq2\nFPByPpS2F6U3wjmsRceJDLJ+Ur0Ye9JVtomV2Tw2lVTw4IBHv/imbwDHZurv5CNoJhGQoONwOft2\nq46pHe2wt7i1FwSOcvjJ7nHahF0hFG9J28C2tyJofwsoI0t1B25HOc5yD0pfquuP/wDpCONHDRxR\nuydyT+bBx70VJsyWgVhcwAxKtgsRdNzsz7ywPHX1o7WIkjstEsxD4m23M7kjpvO4A59B/WryapUZ\nJ1os/gLXtq8lzdQW+WwA4ySK5fQ5twMhmgwyjttwF/r/AFqUpNgDNNuWVBCG6ANwecD6UVG93NqA\ndWIijcnxEBJz0C+1BP8AYxK0eU7fEUzSAkMqDBYE/pXLh4GPgJGYnjG4Z7kHBo2FADQyNcoLfcy7\nt3PB+1Sm1RxZmxRvljnLu5yX54UDt3rVZm6GEaeCVO0BR0xnpg1HxJIYBNBKzReLjYKZNoT0hKY2\nl/F8wbqO/wCtcjIKNkjaOMdsetaxhZfzznbFCVEKSZ2jv9f2ouCTwtRMKMzv4pUAdCP+CswL0PDP\nHcmMKJFiBlIZuTtPIzQmpXMhvQkTOyOp3buiUo4FHdXM2pwom0rMCXZE4HOB+9Wpcix1d0kKeKH2\nFV6rRAFauHlghlkPiROwVmbnnPFUQ39jZSvbhASr7vKdpopWhJOnYWt7HcTmRG8ndhzXZ7rCKRIF\n8uVXw8gn+9LXwdO1YBf3LTohkWNjjG4LjP2FL3fbgiMAjnIY806QrCFuBJiTYQBwdp71Yi7418yg\nycYNagojPA3yDwHbI6nPvgdSKVKjjb4LZJHOPSnQGTt7aRJGklUbQCQEPPHc1UJDdIyKoIUA5J4+\n9EQ7cTeCzITuHBqy0nBckZzkYwMms/AoMuJSVO1B/LlQckcUH4viL+UYPJxSo0mejOGyOfc0LLIW\nlyeAD9qDROzrDGChye4FQWVt+GUjBzz3p4owY7jwRHGOT3qraVkB2kEL0Ipwg7SjxPbOag84Y7cm\nsKzsbeGOAQTVdyGfcwXzY9KwGTSFlVVG4hhycdKYQyMsSKwDknqeuP8AFGwI6tq88jOG3nYx256A\nCqYY445GKrhsdfSsg0SaZlXJ5OepoTUZH+XQHGx89KYUWbQCewFEwDI/rToDO/lFAXGNzUQFVkP+\npBNGwEiV854bNJIeJNj+Zf8AcRg03sgxYEHGwb8//HmkSGZ9C+FZ459LaSM5Vrhzn6816uCa/Jnb\nF/ihC7PDcGC0mkCHzFgg2+4x60z+FLJrn4hfdKIxGu4dhnA6rVHInQw+JNcmGqzQTSJ4SbRnBINZ\n+a/SUhLRTNM/5R0H3rnldkFAnHpLzFpL6+RWj6QxDGfvU3a3MsMYRYYkfc7L1PHfPaio5ZaMaO68\n8d6q+EmxUHXHLDHQDtVWi6hGixxxTTPpykeJDtAZfXJAzjJqi8oZorlntv4bO0QSMyNtj8c8gnnO\nfTgj70LcsbSeK4tpvCSQAbM7huGMn3zRjFWQkv0S129vdb1C31a2dLa4to+h6MB3+vsaZ/FGqO3w\nfbzxnLyNHmRCODgnp9sVTrVJCKVoxHhCWH5gt+JuJYH0p38N28piYpEGlbzRs8hAVR3/AFq03gY+\njq9hn1S2SeTERjGIlHIb1P6ims8VjplhPHPbh4JiCyltg3bRnH3FcaR2J5SBtO1K28BrKC0jt4pQ\nAXt5/E2DPHBom7sSNRbdFGhAHmPAPvWoDb+npUkupHhtrqEucKjgflPofbtXIVT8GPU4Y2ZDw6rw\n31PqKFE+1AmtadBbP4Vi0bMWUqnAZiT2/wA1S3wzPPIshuIm2csY+QMdR9fWg3+jXZyb5eG0lG9A\n0h2Iyr5fcZ/vXtJga+uhDbrCCg58TIUDvzTLEFYH3UVq3xDb2UUyCZBmViN3BHQftVN+oFvdSyFG\ne2VWiecdu4x3+n0oxf7JWdhuNOls7WVltvEkBZlRAcAe1AandXGqXEjbHeGFPDjjVeg6cCquUWsD\nGT+lL2yQW6STjwUUFQ7dD74qrS7Y37b1YyxbhGSFHOe+cdKmxqCU0mzhmbwJHluADgg7Rx/9V21t\nrq2v2nJCxzDe+x8EetKPRC7gWC5cpJcbSuMluftV9vZxyxlIpMIqgmSVsYz15pro1DS2TTFtI5JS\n7KqnZIGCc/3rPzSadezum2eJg2N+RJz9ulZOw4SeRLOF13OyIu1QeO9ehuCEdvD28bRluPUmmQtH\nGJJ3xDdwML6k0DfX7QNJDkeInXaODTpAfhUlzGyeOwyq+Zlx1wRx+9FAPHewzqrLF5WPODg8j+1C\nWAjpb4civcB8jOVYHgnJ9K5e3ipPCVXy7dj4FL/4OV20cbRPHDcFFcln3Hb9ga5DYW4d59+JAMhm\nbO6oz5JRfgC8zSXGlSQyNEkUKlxtOckHOPvmktzEW08zC4icq+Ni/nyf7VfjnatEuQa6GqC1DFgp\nxzjJ81EXdyVhVo413wkjzjsaPrGjkQJJI7mJZeI8nbt/xXLmBJFj8JkMjAHaD15xTmbJNlI/l4tu\nC/mI7mgzu3MJBjBIIzyMVlpi6GdEk5L7HAAYds9eaXzXYEzeAmyNepbqaZI1kp5fEhVkyMjp3Paq\nbONYWYYJB/MD3pqEsndWyzfiKMZ7DsKkLkpD4UQVc8lx1+maBrIxBopAY2AOM89K6TnC7lJ7hRig\nBsvaRYQPKuCOhGaBmPiEEDHsKFAPIxPlzgelW+L5fDIAGc52/wB6ZBL/AA8Q7zzzkc1V4zRybh19\n+aIS0xwzJjbtY8Bh0FLXTLMgBwO+OaIjJiLJCs3OOKvgjjDGTxGOOADWAXvcHy7FXj96tuGl+XWZ\no0VgduBRMihXwygDAbqQfajJrU2qGWRwWkUFAo496F0EBkMZBDBuT1FKdTlSO5EcTMUHIz9OaaOi\nMBkny2Mcd6JtLuN1dRwccVRALHkBBx2oWUZ+tEBVFxPgcEA0dH/35VPTApGPE5JjxQO1N7GbEbZG\nQUYZ+oxSoZm4+AEA0EpnO24IOPov+a9XDL+zOyPiM/bAkkOshXpuU8/WncU4ht/+ndUfb75P/Paj\nIHoOY7rcHuoEVXwTxndzVvykUSsYnSN5M7FC4C4+vSpfRkqKHeS3g3XUUgG4gNjIY+gNcgu7eSLw\nTCRHK2Q7sDtP1qqSoVnb63jgIkkukmRAUIzg8+nrilvw7cGXWJIbaRXXYxkWUcEcAUVG0Le0V3cE\ns80kjKuEyGC98dKl48F/p4ka2kkmhXhQO/p+hzSNoXwUXMt1HetYv0C+aJjgZA5qnxHX4cuBlhGZ\n0wp9MHp+gq6+EChHfbgE4yTk1rdBvLaCyhUi4DBW8WRfMu3rz6U3J4HjaT0eWtwh1GztHYpGtr42\n7HYsMf2q74vtZLy1jjtxHLN4oYKzgMRjtnH965n/AFOhMVaPBc20RMlmYsAv5vLk9MH/AJir9PuZ\n9X0/5OWQreQEsjMR50z0P0OOfepLUPdht0x0+xW5uAy+UEmFchWHTJHBoSPVdQvRE88atA2SZFHb\nucd6deE5ay25WDVHiWBVijWIqbk8AkV2UXtrpnyMLxpbySDfOJFIUDuT/altIK/0hDrNzGziOCCa\n2jwqksuP0Hr6UzuZLUaMxubZHuGbJEfAA9K33TTS+Gag1u3GtSlYY4mSPaHjzyQOOtX3cJ1aEo4I\nYbXJ6ng8/wBf2orwRRLLxEJeWO2TwwNqwo2No9TirIJ7e3kRZF8PKYHrk9PtQiopgaoNjFpdNEpt\ngADuZgd2D6UJqsr6RcsXkWOzkIyoxwPtQlJqeeBTzSTr8xNHLbKJbWOHIZBggk9D60pvQ0gRYAVe\nJ2bAPU9cU6djRlYbJazX0zXb5it5Ig43469wPvmqILyKfNsqpbwiN9o58x75NM02PYjudZN40kZj\nEYHCgDAXHpVcPjeNAzsyxhxvI5GKaEaFse3VvDeXKRxSr5iGGM8j6Ghbma3tGEMituLFQG4XA/8A\nuigsEuBPbXEgUjEAyrAnDc44oSWXeCzoc9atERsJjsLu408TWkTyRtnLKvfuP70VCGOnRC73q8Si\nNQ/BZc5z/Skm7DHDstyZ9QZk879Ax6stUTXCNLsPld1OFPtSpBbK4NsURJ252lgSDmrCxkk2iN2J\nGFGKSUWwFsNoY7LeYZmLSrlE6kDgjFC3mjTyXUklrbzhGYuBsOMdhRhFrBZK0d06S9jzC8TRquCQ\neMEdsUUxuILW4jeFVDoVVs+v1qyQieAFraBUaPxhuMm3e5wF4q+GC2W7MbXUZx1YA4+tNQScJg2M\nYJjJ5jsfGB7/ANK8fBuJ5Vkk2nDO7N0xSr0a8KNPaO6ZLeA7BjjeRjPqf1oGdY4HfxSFC8EYJqiE\nZdZzRXUZhhYyANuyEPFVSyR27sMuST3XgUfoqIx3rNIqkAAnn2orVPl4yk0AyCW3bBgDHTilfo3w\nWC5Bfr16Zo1AbVgZFDZGRg01C+lck/iNnGMdjUBt5YkZz0NBmCI41KjGA2eOauZsqF44PBPNAY4b\n3xE8N41JAIBAxiqCoC+JK4UY4HrWRmdZwu0o24N2PFVzTovO0k96cmDu/iNjpkce1MLOZVQq0Il8\nuAeRt9+KBgfI8UjcNgPLGiLyRlD7HDrvwMj2rBBgxKBScZHUU2eeOTT5FGGYeHsO7BHHmoMwtMcj\npkkIByOc8UiuVM90x9DTRFYN4e6TAHSvCFo5QeKqKE5J49etVzNxgdu9YxRDJtn+1FwSlpCW6txS\ny8Gj6XSgmUVZC7NKinIQNzz2qaY7R9U+D4jHaTptAHjbuPdENerkkvyZ1ReAMvw9dW/iLZQxO79R\nG+cLXo9LumQNuMM8JAETJgkeuaZxEsItor+8txK4DJG3IjBJODyTSiW+8dFjaYlDISoI5JNS9eFE\n1Qxja202FopZGlLrncTkD1AFDao+kpYzppzIq+CjbJySWbPO2tGLJSmZ2/15pbeGGW2jmEYIjPQj\n7iqbeeOysfGZSsk3AKN5lA9frjFdLjSoTsmw6312K73rEhicgMVbuelSiWSNC/jfLqXxznzZGOBX\nFKLhL0otVlU6rc3JmljW5e1IRplyV+vH1Aq34hhgsYZljIaG7RHQY6MGB8vtXUn4RyhHa7rmVUOB\nHCOmME5rQWL6bb6bc4ubiK52EEAeQn0P2NU5P/RYfsa6Rd+NrEkki+M/gCBFToACD9+lEfEcNvdG\nMTRtuXBJiPmPB+wHI/SoTjhdPBLefGcdnapa21vOJdwRRIxYKOhOOh6dKZWNrAYg0jLE/wDK2Tu2\nnnbjsOaWPHgO1BQ+H7bUtMuI7u++VmQjCqd25e/39KV21l8nHJbWrPLGD5X3lW6YOB9akpbQydjO\nz0+Ce1t4JvFgy5LOp8wzwRjHNeudLhht5I7cXDW+8ZBYHJHc0zh9GUqKLaEW6eGyRrBHM5Dny5JJ\nxnjk0RereafaLLPCWhYbQ5IO04o1YWxHEnh58RNzE7+VweaP04PYXD3CSxvJIwI83t+Wm+UZCjUb\nm8h1CWWa1lRJmLKYx78gYoaCa4vZdh3Pt5HQFR9T9elSULdkZ3Y/kRbWBWE2FwE/D6n6+/vS+9g+\nf1C1tLi58NHyc9dp7Z+5Appz6tISTrABLq/0nUntg8kRJwQc4ODxxT631Cw1uUWmqbLScnyugJVz\nzx7Gg2u1EocnVk7t0TTLd7eR4vDJRY1U8YOcHFCy3F/cB3tgCWYhRMuVA6/WuiKw7Lsjq+nR+FBe\nLbAeKPxQjAAN9KqXRllRGkZlVcHw1kA/XNFWDBhJBbfNiZZWgYbQoYZXgY5bt2qV9p730Xi3TITy\nqyJj8wHP17VlELYFPYQRJIsl0xDH24qy20bS7wsTdGJIlO8o24+3HvVFaEsMt72KK3SNZDHsG0CM\nlR9cVXc6xbzYiuW3sg8vVv61uqYOzItc2GFMcMaOMgsQDQ76hYJgpErOoxkqO9HqbsDnUfBTdEM7\nBnGMgj0r0WrMJQ5/OW6+9CjXZe/xEbZmRmCnPIIPBzUYviFpZlhhIUk4BxgDHqaxgOSKSa9nuBde\nC3hlstht7D+Ue9BWmrFVkiu0Mpddu49VNYm1QS2nl7so/l8u5lkPtkZP0xQ0Gm7LJGVnKy+bGfy8\nkfpxRsZBFgn4R5KqueO2aiyqFKYPhMBncOo7/wB6WxhZ8wsWqboJGWIHy4PbtRF6yzxMRMxeU7id\nucmqIVndLMulC4eORXeaPZh0PT1FUG3MyhxleQvmGQTitZkir5QwuRvy5yCpXoPai4y8kKrJCdoX\nBKvnNawlUtpCI42QHJXJOeakoG7ynoMZPWtdgpF8YQhcZPrkdKHOAW3L0rGZEyHIC+veiYnGSHx5\nTzjuK1AsollAY7Scbv5vSqvEUk55HQZrIDZapVhjI46ZPSvPaybDJ5ZF9UOcUwAZQcZYc5xREErw\nMkg5API96Bg7R7eOXWIN8W6G5bDktwpzk/tXdQjQxAKDxI5z64PFLbsasBIhh8v0zzU0l2yHkY6Y\npmIQlDMzZPkPQUnlXbIw6MKMfQMqUYbng46VXK2XB7VUQsiz4Zc9uKqmwi+Y9e1YING344OMCmFo\ngdSx7mll4NH0LaE71PqK7GmLhF9SBUEy7R9A+AtbVdRutOlKjdl1LdeFUcfpXqhPHhSKw0UGrw+A\nk09tNEZW278cfc1fqKrd2SsCxEbBjgkZXvTsmC6TqlzY6KxtLdFDEgmZ8Y469MUiXTmY2skhgYTN\nktGMkD7cCpKNaa2VanaQpE/hzRyRlyA6jDD2P3oO10DT7yWPxpZYkY+Yg5/rUOX+RLikq8A1owu/\n9OdLUZW+uIz1BYDpSC++GY9F8W7n1S3CqCkUZXez9O33zXRHmbdsZ8aSI2t4sU3yg/h7JHjZemDB\nP1ol9emubc2s00Tt4hPiqoXjsBWlx9naF7JRLLOC0OuQRw+IsMqBZJAchn6kH2/xRPxPoVhYLBeP\ndLfWJBURxYSSMn2oJtSwRV1FOl2dkuoW76fdGVXkAaCaPY3HbJ4zTrVdOuFfJtfBS6l8PyDARtwx\nvHIPBpeRyU+zGilR688T4V1ZLq4aMoMKxC+UZ4wO+cZ59q7efFttc6lDDpFgZVlXc8tydox3I9ut\nG+yspCIPrel/LyXerRTbbaWZfCiZOJON3/DVOlXzarq5S4kSIz8Kqgkg8YyTTt+GeMeNcrbXJglV\npEidlYoACx65+lG2N9prLNerazNmQRBI0AO7GcjHQda511cqDbozE96nzni28cy3QkMm6U52+gp9\no1xG923zs0khaIykKgAd/bjmrJWyXZheqzW9lpcj3EskS3YB8ORMjI54HY+9KS/8dsWNrcoI3/LE\nPLsb3BpeWahpv+q8YI2i3YaKQSF3UbZFJ/MB6GuRQQ6dIpmZnIbmIIVLdc81oSjNYP8A9cPaYl1q\nl4sFpJHEiZYHqR9+9NTYzW9o7XfgkKSzOVGG+pxVOtYZPshYZbSeOVrlimwGSMR9DgZway1zNPdJ\na3xfY8juQTxnpgf89KlOP5Jshy/Av4vmfUP4fe2Ykmd7YJMEBJBXGf6/tQFvMb6ATDdG0ZG8Z831\nxSNVFSOZ/i7NZFqAt9C8SSQeJDIEfZyG3A4Y0vbULm7ULbOeeRsBJb9K6eN4d0dSYU1vO+glb94I\nkjk8UNIPMR34HPp1Heh4tiW0gfUI3cAFEOQGH9qdMYGmuS6P4bBlOckHJBojUL5oNGtFhyG3OWPu\nQv8Az70xhHJfyTMBI5wTz14ovVbmPTbW30+Bt8rr407g/wAzflX7DB+9U/wnvoPby3DgMQI4WHDt\nnGKsaObIbcCp4BAOKLaQFpTJLJFAWIbaWwTQguCXwnTPpisaqDre4uLeXekcm0jg7d2autXcXCSq\nhYo6tsI64OcUjoZWF3K/MahLcyjZ4xJZCMheeooaaykhf8TKB2yg/wBy56/elTGopl2wRyOgbe77\nlAHAwK88KqY2Z2YOod8jGcHkZpvSbRTNfbdVdpQ08TANsD547c04jMRWJ4Zl+XlIPhKPOhHGOe/F\nBo0GgC6kIvgYyywKAcE8k+9ETESyW+WLZO3afX3pEirEj2zpflQFWNZNmR2FF2iSrA8TtGrRuVBZ\nccVVeCA8lzM8rBZGUYOAp9AaDS6eOTzyOfvWSNYetyJ7WZmJbYAFDN5vfFUPEEy0MrLheEzzQeDL\nQi0vU8IKYQ5VMYYnP7VxZEfaCjKfZhRoWw2AN4oQfXHtUp7di8gIVMDd5u4oWEEKKBkEE1DxAilg\nQT06UyEZNJED7vLvHADjgihpnWGVt5CkkYAooBcG8OAtvOSeBXba4zFP+YMMY9D9axiBIB/KR3qK\nuSWz9hQMNdBYC/3MR4ccEjDP8rbTg/rVUuWxGWJ2EjPrS/Q3gMX3EqCc57VUc+Jjvu/xTCjOC0kn\n8ZDgbASPoOv/AD2rP33/AOuNjgVovTNYDPIMA1FFDn1qxMIkfw7XyY5Pel0mWbJ5zWCVI2H5ppaE\nIo96WXg0fRlkeGGOfahmcGVGwR5sVBF2bb4Q0/8A6u6lkUblCMpHuD3r1cvI9LQ8HulRlPEiaZpW\nUGQRt+X3GParX17T49EmvYZ1mAGzwicHd06dhVmSQFD8SWisnjAiNRnAXoa0Vq1nqdgJLQoiscEx\ngD9RRr9marRTrPwnqNtD40VnDeRsM7wcMD7ilb2FpaIyS3USXKKvlbKhc9sZ/eufk47fgUlIvj0f\nVJbeMw3UMpl4XbISPtms58QfDV7bvcXfgNLBAqkyFx+bI3CnjHr8C1aoptvldP1tlntfCtZwAY5P\nNtyB0P1oWSxe41ONMCOOWUIu3sCcZxTtqL7EG249R3BatpC6npoeSWIBWR2X8rBsYz7/AN6vt9Lt\nL7SyLm+Y2ysXRQArbscD1P2rklOUnaCl+NCPRtJmmuVk2hVWTILZA69/ettaW8xmu0W4cNIcY6rn\nAINb+VJtpLRYqvTK/FUeulo4bwfMxvyjooGD+lVabaXEF7ak25nxbkSKWGAATwc0/G4uNIvxujt9\ncXOqaOi3zsEViYxGANoGeAOnrSjSZLcXwlt72QmI42Ou3k9OfWuhLDSabNDd3T2yBsNKeS7Jhst7\n+tNtJS5YxKkhhe6topjIFwE/EKnA+mD9jXE+Kp9jfBne6BaR32IGUo64jLNktIOTk9h7UlivtX0v\nVnhgkQR48RvDAJI9jjiuy1TRByqVCnW9Vdb67hubhZhJJtUtyVAPAFLSWhw2TkncAGxmmUFWk5L8\nrNHouqtqelXUM3h/MxAtEjeXxFxyAfWrUuVto1nvbyS3XJU23/d+3rUJfxkpXErGP7KW1q22Eaar\neORjeyBOvoo5P60olvtVnJjeVpFPBBHFdEY16M3Soqitrq9uBAZEhYjHnOAar0TSJ7z4itIJVJt4\n5zG2TlQducj9P2otohJ6O/ja3TQraK309mgluJPGDI5BAwRj9aYaZa2/xT8FrfxRKmp2aGOcqBmQ\ngDIOPUYrj5IXChLViqytYCJ7K5DobpdnhdCxGGXA9TgY+ppVZXpkvIoUxHEpKAL2B7mn/j2oVI64\nf1H+n29xY2lz8tPHc24hliA2jyN1HHcnJrL6eTcXa2rbSJmC7n7HoP7fpV09HYNLDJp980e4GSKQ\nqwA4JHBBplrlzDPo1gts53eJI0i/7ThRj9qdaD/0W20ghbx5RuC9Af8Ad2/eg5JWmuTLMSzOcsfe\nqfRH4EtqUzrtMjY7DOB+lMtJ1F7q5SyGPxAQpY582Dgc/StIVHroMlqj4bEjMpUc8ilyJukZc9KV\nDsaRakylVty6ScIgHc969d3jw+K0hClTghhgg0tBsnaajIkQ2lWHBywzn0o2QJcW9tdr5RG3y88L\n5yinkEffP7UjVMdMUX+15VhU/iDcWA7Y6VOWP5bS4jMjRyuvlyMdelUTRKViITM0w3HIAwSa0nw7\nazWsBvUHzC4ZgjDPmHAyPqRWkyfGrYM8scl3cmdWiCDeVJwWPXA9uaDN8y7kRzu69c80VErKXwrm\nnk2hjyxGeB3phMss+mxXbFT4jHdjqG7/AOa2ICFgmSNVG4NvBO/0HpS4sTKOnPenQrYfpdsJZpJH\nwyRISATjJ/4auckj5ghVbdtEeemehPt0oN6FMMsr2KWAm4hERCnJUdfvQwIacCAliRwAO9Kg2EeJ\nJjxCDuA4PrzU0uCQySMDnkbuaDCy4Wo8UAHylS+enalrkiXy4cbuxrReiMpeWQueh5q+QfMKPw8s\nMcmqAIFSibTwepOc4q6JlS3YvwX4PHWsLZHBdwyA88AV3ZiQKwbJOMelAYZwvbw2d0J4ZCSgQNG4\nVhk+h4NVIsVxEssUsignBEi9PuKBikpiTGQR6iizZI0BnVXIUbSfU0fhgqZ5LBI/FjKmQb1YfzKc\ncf1rK6gVN++zIHbNCHoZeAMwYMeOKuhGAc8cVYkW3kLx2UO5GVZQXQsMbh/90vdSvBrIzwHwQ4+t\nM4cmJSPStLwaPo1TzQLx2qkjhCezHFc5c2PwhfmPUBCSW8RUGMdAM16uecdKxeGp1h7P52E3rPB4\noZRcRcbD/wC7FJF+CWDG6hkW7hcbjtGC/PXPSryVaiUWESWelKdy2kv4anILbhxVlr8QaXp9rAkt\npPFHM3iRSRLjkdV4/vUIzd6Vabjhprv4sSfSCwUJD4TblkkCupHfA7VipNZ02ezgjmszqV0RtU4z\nk57elPOROKcTyXUsJZIWS32LhVEp/Dyc45+td129W20RZbq3DgDaSeC+T1Ld/pSrRn5Yu0jTodW0\nrdeJIblmPg7Tg8dOprtrHYq63M7bZ1lXIkbDDacHHbtSyi/DnU7A7u4WXXLhZ3lMU8jc7uxOVIPt\nT+wt2t9Ga1k8Ax7ssw53H1/TFQclFUxlIsaSKO0lnl8kcYBJUdunSr7O/PjmKGPc4wXZztIBIwce\n9K+K1aY8VbA7kXE2qxDULme2ijfds2nHB/8AFWS6bpsrz3Ms0g3u5QBWIO7oDj3z1o8GJookkDPY\nW76d8vcW0rMFUYB28g9Rjt1pf/6LgskeYpNboWDHfIDz7GulcmAaVkLKC6tLwzxlVTko7EYYdD16\n+lPQJJZomgjiklt4f+87MuQQTtxjB696SSthRCSCbUSzXFw9pbMAsmGx5wMZA6g+9c1HR5bO28LT\nLotLgCUSlmDYHXJ6E5zjpWTaxk3FXYrXTECCS/WJ5SDtWNCefqcZq+zs90Iig0x4W6rNOrSDOOx6\nD71RzoPSyzULh9Jmtpo44hcOmDtAOxsfTqaWyajtu8XWJjINxXHJJ7Zp1pliKJri2a3BiBjYghge\ngpUBLuwWZkHfJqiIyB5JnjYbstg9K1/wzqljp0RiuZtk0tzGY1PVeQM/oanNEmhj/qLoF1qFrDeW\ncZla2yrr1O09/ekv+mV+1jqd8JCyweEplUc7Tn8xH7ff2qLeCVth1reWWva9Nb3HM9nIz2NwuV3I\nrZ2nnnjoetZ7XbJtJ1+UQKRFL+LDjurc4H0zj7U0Gjq45pqhh8Paq0NwwmUMpXBIbBDYwre/NUXN\nlLOyy28aLcR4WXaepB/N6Drj7U/jLCrV5J4dWnW7/wD1gNmQkbQTjr6Vw+XT4mlDZmZ154wMDnFO\nv8AV3MeEAjOUA8v0pc5IYnB49arEnIlDNCJR48bMuP5Dzn1ogMNN1K2nSRJoldXDA4Bxjg/fj7UW\nhUPpLWQ2F5BBFva3uTIhC8+4H2IOKB1GxuNMuYriW3kjt7iJXQkdW2jI9jnPX0qa/RRrAnTYlvPA\nkIaFjJguWGVHc4pZ8SvMmrPFK29V24cEEPwOaH/1Qv8A82W6VemWLjb4iuAoPA645p7/ABh7GJlk\nbdtdRMgGTggjIPtRktDFgF+iHUYYT/8ArDuS0g5DA/l+nrUr7xrxjC7sVtkzhvMSAeo++aUzBtO0\nUy6+vg5eGAGU7lzkDtR95qLQtD4iFRccuqqFMZ6Dp26H70snbBBdQPXIBHbxR2pL3zrtnXHAI58v\n60mtoJQ0wMZeRUycc47E1WPgsvS621UWM5Ih3uh8u8ZGP6VuHWzv7ZYbWNGae33GONhlGBBJHpxk\n1PkTWopBr6Zi9+C7i28OeG68e2A/FkK8qx/9vcds+tZ+WEbscqqnzHbzVIS7CTjTPNd3MBWO3iKH\nqd6/moy2+ZvLma3kZbWO5KbwwyGPoPSnaXoi10FwWcsF9LbOWfwlKbieDz2pvpltAHNwUZHiHUH/\nADQWoZ4y6eys9rmzeQKSoxKQcetKpbOZJA+5CFbpgg4rKLXpnO/CYuNq/ivl8cEntQIQwnDKPqta\njWVlRvOeBjt61fHIkUGNjFiCdxbj70QAk0jGfbg4zV+1nQAYx70GKWeEYlGCxIBPDcCjprJvBa4O\ndqsoOGBOSPSlY6Oz4/hczDdu8ZFznHGGND2EY3BABuAz06+tEBc5Gc5VcdSattJXnvPBjmxGRuZ2\n4Cgd/wCtM/AVoTf6gLgxSPG0kKACM/lyg4FZjUNr38joMBjkDPShAMgO4yTwe1WwIZ1EfJL+XgVX\n4J9G3xVfm5vltE8tvYJ8tGoHXHU/c5rPzDKjP1rIEgY5yT6UytvyAe1CQ0RnCQYPpVHJKqeAGqP0\nt8HumNLFOJEfay4Ax1r1Sk9KRWGx1i5i1fTnmt5VZIiSy5G73BFS+Hv4hDbG2WUwpJ+Qk/8AbPtV\n1ipnPWhl/Zm0vBJ4wIaLdIM8FvX+9DRXumg7bxl2IB+Gqn8x9Mf2NcslbOhSpC281W01O6n/AOjW\n3uYTtVZVx0GM1zR5LPT7CeV3EFyT4aljjwx/uHqTmhQOyesUx3dzm6nHhyJHj/upyF45JHXGevtQ\nervPIxtxd+PDk8D8p464prrGRnyP4UR6iiJDbKVYjggtjLHvkdKLtNPT52ZbvKeHEXRJEJDtjjn3\nOaevpNagk6bNquhwrFHi4hy0LKMCRcZKn3zmrbZJRoyrdKbeRuVj3ebgdT/io8kU9HihidQit4be\nznhN0JQsX/d2g7xnHTirQfGv/lra2CoYgpLv+U57E9cHFL2bwojjPeWphvpLWR0RggNxJkBlJ6en\nNMLbWbmWQfNYR5XO+NlBA44OewpYrGNZPVoZJ72MwtudT5mUhePTiqtXSNmgmEwVxksrLuwAOR9T\nmkTcU2zMxl9b3SyNNdS7kydjYzgelaQgz6K09z47GO23bFOPE7AfXGKMZKWig2lWggtJrz5Nl2Iz\npC67yRj3ouDT72/0i3uFnDXBAfwGTHXkDB69qpLdGQBb6lNJdwQuTDcRKQzBCOc9+Kuv9edIdisi\npnDjPJI749KZRsdiRviNt7h445967WYjJUk8EY79qG1YWsEkYiuSW2Zl3HkN6VVRaJtic3YVXTna\nwBqcVyvikDd04yatRF2GNBDcWe9Xy+/aV6fel+oo7Mk8bE7T5R7jH+RSsVn2u1u/H0+3uB/+aFSP\nuAcUFLpNte2V6tvbxQXNzE0bsqgEsfXHvXO1Yp880bTo9V1W5trhmtrtVZlK8qrL+YEf4om1fT5d\nPuLfUbgySWsrLCybiVX7Dpn1qcU7NHBd8n4MqtbTC7jXDEquGUe69fvQRnkNyV8UqsjneQTkD39T\n0NdMVZ0Juhld3lmbx7mWWea5RVVInGVTbxlvXp0pJqcl1PJ48swcytxg46+g7CnUdC3gXFYXzacs\nc0BWZMmIkjzL1oW4+WS2imDMRITGy46OBzT3TpAq0CfMQzSqkUaYbu/cVZZ3sVq5+YtY5EZ9oDZw\nCOck/wCKZoRM1eo6iumyabM0EkZmkWWRopcggIF6e/I9eKa/EOlyX+hWcdtcM6eKCnjDAk/NgE9m\n5IrlljOhaBaboradpEssqf8AUeII3JfOwccY/wAetZTV7hLu4IbGWbJZF4NU43bJci6kLJRFBOEc\nOGx+Zff/ADTS5juG0wTRbcZ2SED8x/4aaQkWGf8ATzJYuzFGClJgg7Doc+pzioRLsjujZtsJjCK7\njcRlgMf1qTspGrC47ZwkICud+IZZVJVg3XJ9sYFK71kvNRKbnQFyp4yAfb2rLdGkEai8U1rZyBT4\n8ZaLdu5IXrkde+c0NaXLW9rOLfaZJU8NiBywzzTxIye4S0vRUvpZReTQ2dtHjxJZGORnpgdSabX0\n+mWFo0Wm2cjGKLYly0uDJlgd20cDPNCWuh4Nei221iSHTryG4hMhu48Z3BduTnpQbQgHdLncy5Hb\nNPCNGnLsRGN2W2tjoSc0T5XjPHvkdquvCQZaDw4yXy7Med1eeQiIqOM+lKaz1hKhYx3DHa/G4/yn\nsf1/vVU0zRvsfhlJDcd6BqwDkAmZZcDIHpUxKdilfzE81jWVyxgAv5fckdKEdgrEZGCOeKBrJxf9\nRIqIGZ2YKAo5zTR9MNujLK6iYAEoP5cgEZP0oSYyR6VDZXEbKgYuoddy8H/x0qd/C38UjmAENvIV\nd09R7UjY6RZq9m1qrGNGa3lbekudy49yKEs3aGcSAgAjB3elFO0BqmQvgbefBGQ3mGTxt65oi0hi\neydm4mOCqEdRWk6QUtCJkeTT7eK4bbFEhKAnoM/0yTWfu03XGePqO9GDBIDm/NzyDxRukjZewseQ\nrjirfCX0o1HdJe3DH+aRj+9BSny81kaQLuGG96Z24xED7VmZB1qwOV9qv8AgZAzzmud+nRHwc6IC\n820LkqVbP3avVCT0tHwGa4kjuJX2hQ8fhsegPGP7da1lt8SK1hbN+FLJ4YUqp6EDv7966pq0csWU\nXHxGvgyJeoJiAcQrld2fXNILS/uEMeyVoY5XZow3O1fY1BRGky2XV4rXU4p7XxLho+XMx4YnvTvS\ndZg1jRL5biS3F0Cd0EnkWVeTgN1B9MVpxaFUkgPTLmzs9GvZzGwE34Xhy+bHrz3rPvE6xFYQk64P\nlRsMq/epJPtbFekNLtreCUTXMLSBUO1Gby59T605XUJ5LCdGd3eBAY1MQVef5RVmzBNhOp03+IWk\ngEtqS89s+d2OhxXNQ1gzXRa3Ci3eMbWUeYEjrzn3FT9DFsojige5kS9jkGcMk+7Gw46n+tOraS5t\n5i0N5abXA2tMCeRzx2HUmpyg6uJpTadE7rV5TavDPcW0qSKzCONPE3tn17UjuZZjqitHIqsiASMo\nyVbsAe/HakipespEb3erNDHd3irHFOFyMj8x/wA0rtNb/i8W26nVbhc43vhcdelZxcotDNF9vJDL\ne+Ebu2mSSLeCz+U56jPqKaaRpeqy2/zdrdW89nEzAx+JknHHH7VJcbghU9F15qN1sCB8kPhhEcv7\nj1FV/P6lHDbfLyiDcu9/FO7HmwOMcVVeIZBd7qF3L48nO22IYyAnHpk0jvBY6latHKfx4iZSYjhp\nPr7e1dMGgOwBj8q6FEAZlADbBj78UA1g99cBY0LM2SSaomIwiDTpdQRUgtSzWykHHU85JP70vVts\nzYXzL1xRUrYkngTpzBtVgVwSJH5Ud6t1YxLZx/LBghkZxn32/wBwaEn+VAXh9L0CQz/CmmNk8RKu\nR/7Tj+1Mrd9t9IhGC43qemR0xUX6IYv4tsF0PWptUhcRtdkIqrgAEjDEj3rI2zhZiOG3xmP1Ge37\nmtFbY0T0EnKPMD4qflZDjbim2kwQPcTX91KhMOJBG55lf+UY9MjJq6RbtgsuAHu5ZZCQ7nBOOM56\n8VC4s5ns9xeN1En83WmQpRby3NtcCWOUkq3l3c/1ou9tUudHUFNkrTlpUX3HUftQa+jJ/BE2mvZj\nfchwoI5HA59PeiIrGR5oVUPLD5iB0wQMkfXGKdywTqP10e7d7FJgrwrGFdmb+UsSTjOcjPatHZS3\nl1FJY3UafJykSQu2NquDnOPp1NQk0WiDfEEN9pelR2sHgzWqSeNIyktIDk/t0rK63aQwXEtzbRxm\nC7wbfnzDjL8ex4ocbpgmLbZ5djeVgynJH2xT+0kuv4a6xAyJEVY5GOtXkSCpLedNNS5khLKH8wQ5\nK5qFhc28Ej27WsjuVOD6+n3qPw2pml1GddW0h47G22ySDaznIztGOcfSsiZvBTYjksRtclMYPp/5\noRdYWl4U3jtBMJInLSKAeDk1VYN8rI+xsGUHDAchuwyaqjlkFLeJFr6+Jh0mh2ymQbjyB09PrTC6\nSK0T5fKSRr5SwPela0pxvARZonVE2N4IwrBVwT964s9n4oMizNHjDKMEimVjui/+H6bGX88z4AI8\n35iRnAwPeq7aaGFmCWwz2aRyefTAxTqToWSR1pnmfe/DHkgcCrTllXAA7daZCFEsLAjLY43Z6iir\n8B7aJt6M2MHA5PvWChU7GNsDkAVKPC4PORyKDAim6cgKBwCSSPWhWIlgbaSH9aAQ/TzJaxGcNiQD\nAPof0qz5oSXgaRskKqkEdhx3pXrwdF1yUEUX5mCsyrubop5FEWb5Y+KzMY8hFx1x2/SkYyZRHqjv\nFLZszhAxHtVZlWNSq4f0yKKVCyZJ5BfRNLKMGNVG1RztHTFEs+6zkljAcqQSE/Mn19q0vAxAo7uS\n/ESI6nGU59z70vvYmguTGzBivcHIow9BIClbdj0o3ThtuIeerirEvpVeDbdzJnOHPI+tAz42nH70\nV4Z+gZ/rTiEDwR6YxQbMi+2GJCB2FNLUb3C55K9655enRHweaBiLUstjoFwPvXq55el4+CCFGvJ2\n8yooO0u3TPOPrVt1NFpN0MmObacAB88+tdTlbo5kqVgdxr4aYTXbCVgceGD2q6TXrPUdGe3kgEE8\nbAwGPnC9wT70Hxv4JKYLCpU8cA570xtbNRaKwP4jglc0J4iV6GnVZ0tDHsgYxuEw6jLcdc1Vctbv\ncwkINqY3cfmz2z1qVfSqkW3JjW+VkXamFzHjg8AUxjsdP8BnuISuMuSkhFM9QPpRLYWcFvHcWV1M\nksykKGOdoPv1pZHcT6bOQ8ygDyMiAjPPX963XAxLPiLULnUb3wYz+DCq7YzwucDk1f8ADN89lqgf\nULaaW3YMGRG/LxwaTyJnG5WFQ30N+ZB4HhODhCRjcB0zVDmKECCVYYoJWDtIzElCuT+valY6TR74\nu1K3n02CzgiX5l5F5CYyuOv681nNftZbLUt1qu2FsbQjZwce/wDzmn41RnZ63tRDpLm5d4ZJJQiq\nR0AGSfbqBRum6s9hZPEhSaJuREwO0nHXHrTSjaFbaHuy91a2Sd5YrWN18yrw2O5NaTSbiOTSGtWg\njigSPYqggbcZ7n9a5ejot+hDrGhX2hWs0iCWSyu8Fj1wOwNJbeWG2kjkmcx7m2I23occU8Ua0w8Q\nW1+slx8x4m1N7gDAQ+/Y0viRWhlmt87ID53D9j0p4NvCc1RGf4ga2VY9Mj8EiMh5c5aTPf0HWlKS\nLLHlAN5PJAxmrxRztnNJ3W2rQ3Ew8sLmQr9BVU92fl5lYjyurIvtTyirsyl8PovwJfiT4UhV2I2X\nTxdPXzf3rUxkBvNgFW4rnl6CzCf6nNC15b+M58sJKAfWsyr20VvExc7SB+UDj600dHRz52FJytv+\nIh6sVB/TNME1WcpGgZCmcBcDirLB0rKbic3J3P5TjoMD71WJg0e1VHmOc45oMKRSlszOT4ZbnODR\nVtMsspQyRxFRnz0AhVxse1QmWCbDACMjnHqKIe0NnZi4lhUwuB5w3Q/b7UjdDeldvqUD2a23iFX3\nE7sE8emad6fcyzXEKuoMDEvleBgdRz6ipsZMhr0sWoiR4Vfx7dfCkiGAHVvyt+vH6VmtPT5XSJf4\njbyPACXiKjbt7MQfv09qeLSRpRbegeo6Fc2li91E0lxCxxHOucOPceoPeu6XbyXkcquGgm/lz0JH\nY0/e0JKFMaaPdsEl0+d/Da4ZVDMPygHnHrVywZvbhEkUi2cfiD+YE4JH65NJdBqwmSWfSYIBPdRk\nl/w/DHBByQSO9Uyvb3HiBLLbKw3bg5wMnng1k09Q1OtF17ZTsGmIgVYgAW/KxH070suXaBlWErIC\n25WB7niqRZzzVFl7AzLE6oNwUKWU9SDRMkMkuniMofzbjjtRBAlaxBbORN6xbSHUf7j0/vRFvBDd\n3K+I/gDYd0jc5ajdFURtbJXmba3AB2546Ch7S5TcMqFG45FFGkSDjLBACCT2opG2RHhM4zgnpTEy\nl51DgId30qEk42dTRCDMyySEFsHGagZdi+bPtQZkRyJPOe3Q0JIuydfLhRyc9T9KUI5hv2fRDAso\n8BnyEYD9vSl8xaM5UZIPeska2G27LJYs0qoG6DPP3qxIZbaZ5mAO5VZTnjn1pGxkCyIDG9wqkIDk\nkcj6VFcyQIwygLY4OT+lG8Awi6gjtDB4MskhcFm4xgdAP70u8SdMtBM6qRscr3PvRRjkMLFtqON2\nCQoznNXX0LfKQOsZimwd6N3HqKKNQoMjHrx061rfg21ikjvdRuQPAs4z23ZY8L+9UliEjrM5cufm\n5c4DbyDj1zUY7Ca8V2gjaQoMkLzWUklbBLGVJpszyhCjISPLlDzyKZXNsYLhoI0YZ4VSeak+VNmX\ntF7WL2M4SV0LtGrkLztz2PvRFu5R8+wxU5SvUdMVhofhtoG1Vop2Kuxygx1ODXqjL0qhPaQR20Hz\nQjMat0dgMYx796R3krKzOp4Y5HHOK6ktOaTyhc87yZBzg8VfEijH5mY9cDiqeIgzUWWnJMgEc8cZ\njQMxkXAOewqVxAIpgMh9v+xuOag3boyFV1HJJcvnaMc7mOK5A8kTkl43b+VQ/ejWBG17qiTLaboW\njfw8seoODj+1duNcVt0SAbpBjjpSKNlCBiVdPupGeWN1QBFA3bj/AGHvQpWW5tFleNi7tsyG74/+\nqN/A1Q3sprWPXrr5qKVwWVExF4i5GAd1U61czWtxMlvZrFG8pG8qcgYGAM1F+jXSsv0hkMQkv2RS\nT+GidfvQOtTrdaarQwhpA27eW7+mKWCcmZTwK1BXefx5mX8WGPLoAGBx2PUc+le0GOIakLgzpK9s\nhzHdchh6r6kY71eqDdirWbe8khtwSXBG9sDBLMc/0wPtQUQkt2zKpyDxuUjmqx8Ek9NTpN1Fqdg8\nTkiRiqs+PyKM5Pv2q5keW7YWsU72qJtzNhSxx+bH1qMkk8Kx1aR1vV5P4FNpt1dTSIpHgLG/THXO\naRWA8e3EgtzJGjkFQ24/XFLFNKxU6Y4ntdS0jTUudpW3nfaqMBjJHJK/rzSWaOeO1K+aONvMQMgO\nR7960KJ8jF3LcnPpmoxMIUHUk88mumKwgy+z5mdipIVeV9uKClCzyFdp8y9uvWszId6DrZ0eB7ea\nRktp33q2PyMMD/n0rWwatK23ZIJc483XP6Vy8lozwSfHJh1OK2vRKPwT4UqemeQf+elZCZE5WJnY\nJ2PY96pwu0GLLYoQkUb7sHniiISWTcMKc1Rl0NNFsI5ZxNLh44GUMN/m57gHrioTeHJeSsvl3SHH\noffFIm7GaPGWC7dEG2KTaQWJwuRQNxA/iGREDbF5CjIx/iiCgyzMoi8ddqRxElgeduBn/NModesr\nx5Y76B47d4miXwz0JHB59D/Ws1fgPBBBatLceGiyAqWaSMgZCqMnB9hWu08TRxpC4CpEBHlgMKSc\nfrzSSxFeNfS6SaK3spNQtnjmWCQQTJjOUPQnuMHnNQ+ItQf5WNJYopI4MTSGJRiSNxg59eoqcXtF\nXpmrW8awn8JGmktI2yIWGAFIz0rRDRYbm1N3YTeYg7VYdMrx+h/tTyX0nF/GBa4La0+ELdbmPZdT\nlGYI3UevseelE2ekQzmNQrgpBlpkfKjOduR68YpWn1Nl0cXUvn7cROvgGAeEuUwyenXtihZbqGKZ\nnlmG0vksQfXrQiqB2Eet3JnndYcCInGMZ3e+e3rQMCMsoVxg9QRXUvDll6HL+HJbq5IAyTn+tTnl\neMgoxMR/MQe3ajRosOt0e/0m8mQiJraNXHPB7H+lLFkYbt5by9T60q9KvwviuMSBXYBX9aqLLJdG\nNRjHGQOtUFLZ3Fu2GBAHUe1VyXcbhggx2z3rABg7rO5DAELnnvUd5LjJY55INYyLY0dwzhfKp5Y1\n3xFLbWIb04xSjBMEKysUUEswxtH0om+tIYdNtLyRd8jqYpEby7WA6/cYoNhrCUlkZbK1mt7Rljmj\nLgZzyPrSSUTI/Iy2cECimgNBVn4klwECZOCdqjk4HP7U+t4pHto4oCzWzqHJC5xjgnH/ADrUuRhi\nTS4tX0iDT0/DZmKyFDhSueM57jsaBaxT5y4i3lY45SFOc8Ack0kWyiiVxtbm2DQc4bG5upHpQ9lc\nJBJIjMjKz5OR0xVUKwu4soXnintmPjFiSU6ZPSl/xOsn/qMxSODJFEiNjtxWg9pgfli2RFkYF1z6\nmm3wzq8mhanG0OHimbZPFIPKyHH71aWonHHYZq3w3Yr8ShomZ9OvAZInV8FPUZPBx+9PNI0Cx0f5\naeyvDcSPISk68J0HlcHp3rk5ZPpQOR2yGsRtdXp8HEaBy+044PcZ9KV2lp4moySzFEmD7o1AL7j7\nnPFQ7VESL+lGq+bVJN0gmlBG541AAHoRj1oaLqo5zkZq0P6nbHTTaHEH19edpUDB+oxXqnJ6UM1e\n3/jolv8ALmGNFVUUsTgjqaT3Thx5Aea70jibBkgJPAZs+1MLawwUlOcKOwoSeCsPjZxliSu3BA2k\njj+1NB5ooWVFbcHZwBgMT0xUXJegQJeQ3RsmaNQ0CNhgFyVI7nvilsVxCpVntopSeh3sP6GmhJSQ\nzVMfQTWctqqTW+yUIfD/ABCQeTxil66Q99bSTQwtG0b5Yt+UDvQumOwNbv5VZTLK28jbs9QO30q2\n1vPHVI2UwoQXQKSME8UX+xE3Y0R4Yrq7SW1kuGUlUeOTDR4747jtTG71OO7sQVWQgbN6vzjAwQB1\nJ+9c0k3qK/4TXT7bUnhFoY1WPzujMVfAPI/Tn7VHU49H0/5ayhBkfzeJIjdff7e1CCaGxCG8kKOj\n3Sv4ZiKJg4yA2f70TpOoWK3AbaySBDuZlDhQB19Ont2roStCt0wbVPiVHv4JoZpGc8kDAGc8D9Kt\nX4jF8xhlhMspbjgE579PaqeIFphmt6vDZWlnawxGC8KZLIuAM8j9sVTol9cNI7zu7I4KnccgH1FJ\n8HugcJGsrSXUEk8jkKis2Bju9SliW0cjTjJGCvnDjpzmhFGk1WEm1vUEsmt5ZZGQjlGPAFLbrUby\n9jj8SRpliGxFJztFOoROeTK433PteM4x2qRRTGSAoxwB1p0LRdp8BuJ3VMswhcsq9gB1qWiaK+qr\n40UiRKT4YyDn161OUqDQ3n/0zukgzBqEczSc7GBGSPf9ahEkXwjNJKwl8O+00Og27gkvcfrn9a55\nT7ugNGLtJ5lyjuXRz5lP1pr4YKAqDk10xVeC+Hvlt5AVtre/SiIbRVuEhuA6YPPHNCTLw1E7YwwX\nzDcGjGQWHWr7m1VIY5I1cNIxwW6GkLAUKO7qH8ozzn9qbGJ4Gk8OTZhNrsOQ2ecUzABFQunywK+d\n7AnB9aA+Xbx9pDElsYNNFUJLQqK1mt83EZGFJU55471qrG5tn021t0kDvJIsjYHOME9T6f2qXKW4\nngi02G+s2CRxlRNKQ5c5DjONp/etTDbQOLP8MNCqCJoxnzIW4H/7JH6Gkk/0NGIInw/JqOo3MMMw\njAUBbiVcRnj6fak5hutG18wTnxBHAxVYySkg2nOP3qSkCS0F16NZLe0t4Hdooo1YE98k5zRCajcW\nsUFsWYwsq72R8eblgM/SulJuJPt+Ry2uY9Rt7tpZrqC4D5hQjyuPrS+6muZrRonlXw4/Lh15xWil\nZKbaAlUSwHaThV8v1FVxz7HyCD610E/hebwTIN2OPTriulFdPIzc9Oaxlgy0pwulXNvdRum9wwCj\nl1zzg/alJkLMcMQo8vmHP3pKdlrVHpXV2Vl52CvLcEOZGxnrzTil13ei7lLyRKCQAR9qCeYQyZY8\nN0FYDCtLuIX1aE3CLJAzbX3HbjPfP1oueBopp4sJhHOxwwORQ2wqqKkUlCueTzxQkAEk7h2xgcbq\nBgqVWsrW3uVyHeRtpGegC/3NaVi178MW14FkBW6VX8QAjJBGf2FLIoj0MyalD4dpI4aIN4dtnIIP\nJKe56kfek0kP452si7jkqR0oL0S7I2QX+IeHbnzluZGfGz6fvWjt08G3txHG0lsQQtxE5DJgnkj0\nI7VLlBfwzvi4nKNHv3eROeQc0XqcMmk27xTqYzt2DAz16UUvCsfBIt6suItvbHAx0qNtG/iyZj2K\nAfMw/NVqoV6GfD6t4908gZnjXbCyHBU45yPQA/tQms3ck+omaeQSSuq7mxjOAKEUuxniojbLvhdw\nueOwrtn4fzcRmd40DZ3Iu4j7d+lWfhI+kWmofDMXwxbRTSPNaJL4RaWEcORuPuCMZ4pM/wAT6IdS\nmezhurgz4jlD4AbB4kCrjJ/tXHOLaA036NBDDdSzLGxxFzllwDSGa7W3vFjFvCCW8smCV7n146Vz\nJWqMo28CdRv4LrS7C+lZYI7pfCkZB5UYdMj0PP6UnlspbO8CTjbkgqQcqw9Qe4qvH4dkXlDqC4+S\nu5Z1BLBVKe5Br1Z+lEZedDtDHOWXqTSyYEsFGcAdADzXdZwBemiSCTxG3LEAS3AOf1rQaa7Gbxo7\naO4gaPGJlAI9cY74pJ6qMWay4ivzHAoih25AHUgjvXpNM1Ca2SC1jKySKNrKPyiueupktKpdNv7W\nyi2Sp46dEE3nOepK96AuNJnmxMYlTzBWVFxtH+7Hpxz9aeE1TpUU62H2sELNlJOUG3OcEnsP2q6C\n8v11BIfl5HskkDSgDOR3z7YpX6NJAer6a9trrXAiNz4jb4REu5VUk43e+KFSykutSs5btTZweJvc\nspwp5OCOvIp28JNaajUGMfjvEI0iuGIRxgbx60v0fVkW4eO4W1igB2sznaxb1HGPSp1mGUtGGpyI\ns8Rtni3ON+VO4sn1pFc2balqQaBovmAvlUnAx29qWLwozuq6Jqk+hwSzeGgtWaPGfMxNJI1/hiHx\nZt00yeZQmcr7/erwkK1elNvpkeo7zHcQwyou4Rudu76Z4p9omjHTka8u0CtIPIVkHlA6n7k4ppyN\nFBUfwjfazO800yJlsr4rnOa6/wAI3GmXEV05DLGQZNrkox9DxmoLnjdDsEmuYrS8liuR8wTyDG2F\nU56CvXt1DcOJbOMqGyCvU1WNMRsRTzuPzcdqsspAuc/YetWokwvAySBk56+1VeGSpQd24FKwobaM\nUju2h6tMyRlh5WCk4INcsQ+nwrDENu2U7g/Un0qb9odrDafDuovdy+FIq+QbkUHkY5NZX4luptVM\nazRMjQO/CjIIzU+tPCbRkb+0FpdCMFo2A4DJg16P5yF1ZQsqn0yR96umDq2N7aBp2A2spJxyMYNM\n2tPw1trlBNcyElJQxyRjpj2FJL9lIJrAGPToVn2Rzx/mAwQQR9aY3NjLAgXxw+3zLt6dKRM6KwAW\nOVJcybRkeUdRU4gZ2aPdj+bjuc1QU58rtyAeS3SrpVjt4N9yrbgeQvX6itYGsKNMuuGRyqK0uQ0i\n5wv2/wCda0ltHby3LrZywMjnghcP/wDL7VOaNxyRVFcGE/8A6SMkksU+0MqcccAkVLSL2a11eG2l\ndJUhkLoqZBAb1OPRv2qTWF7Nj/Eo0s1O5FUxgsFP5eOlYz401CW71Kx+V3y+DAys6JnlwR/YVyRv\n/p/grQgvBqGFimtsKgAUDykk881xLZVv18WWRdv4m0Dcu4Bu33r0uyqiDg7sbXlzc3TW3zUawxtj\nYVTauAO9Z67czzyoZUkcHhl6Ghx1YOQWxTyRFsEDByM115mdQdqKf/aKuSRZbJMYxOjqhzs5ohFm\nMn4hDEdSBWCESTsqKCVJYkEEc9KEmdEtX8RDuP5WGAB9aw/wFkBTHuO1eQkrg+n60TBi3aQwgCGN\n3b+ZlzxVU7iZCwhVQDnIFYIA7OxwGHrhhTC6uYmS3kiREcw5kCngnOM1mwI9DM2/zFvMOCBRg0S5\nv4XltSg2pllLcgetL2rTBtxp/wA1bWcTSM3gwE4TjcSx6euOKb6I5+Rv9Hd3VUt2mYtg7SpySPoD\nUJyvwqvRFp7yoRJGSpXDiQLjaR0PtWhutIj1S0tdVu5Ws4zuhnKxndIwJKsgxyG7/ShOXVAhmHtJ\nsNM1m1u9Lt42g1W3QyxAn/vY5/xx7/Wo/CFyJNcurO9l8NLq38MhuPxDjj69aT+ytiMTataPa65L\nDKCPDlZRgdOeP2xXL/8AFjlSTc5V+pOaqvg68Fq6evzIaNyBjpTa0024v4Ggh3Mx5xjOcc08pfsC\nwlFZNbRO6EMwz4ingkn0rLX0z3F3l8ZHFNxq9ElK8DbdjHAq8qp/MaoVsSHafLu71YQfavN4PwJb\nW7xSRNPeGZGYYDLtIOP1A+9Jfh428Gowz3YM6q2BbIxBc+/GMf4qL8Y088N9d3kIuZWu/EhUxiVk\ngPAVh5eD26D7Vj7u+32MCyp4ckqk7k9iRiuKCph4mmiqG6Y6TFbtzEkhOD2yKZaTqUKqLLUwxs2x\ntlHLQHPBX2z1FW69fDoiOprOZL0RBQ7Mhddp4dfUGvVNlkJNVjhjtrF4yAJISWPvvI/tQttMEmjO\nxXB9fQda67w4GG3VxC84aGEJETgIORzRUemNgfjwxptIUuxAzkc1Nv4BC63W5utRS3kKbg5Rj2OD\nkkGmV/fa1YQPqFoWhtZZRCcqDsA6AE+o70kmosoodkUCwiitPm7uaXM5zGzg+fuB9f2ptEjanfwy\nRM6CCM/oR5h+lK3elIw+AGq6QkMv/QoZEUiVcnJHfn9P3oa01BNP0qTx2muZ3P5C+2NPqep+1NH8\ngzSiwaS+kurd7tppYpPy7I3x0749MYrsEzR6ZDczzyyOZJAi7s7sgcnPbginccJr0ZarrFrqscMY\ntzBLCefN5MEjgfals6QSW6QzRvKi5BVWxz6/WjFUib9DreeNdMjt4mDSwrjew8wU/wAtUtATKCqq\nGCYZwecjmkS0oh5qEtxF8P8Az9sd3gMA4OPMDgdKx9/eaffWLrNMbe8Awp8MFH9BwMitHQeFWk6e\n6kXd5GslmW2Ap3bg4/etJ8MGXW9ee31GJfDiBa1Vh5MDPlJ/50pp+My8NcutRwzvbpaxyyxMVKs4\nVRjqcmkmo/GbfxVYrWOFEUjeq8gkfsa4Y8dvTNmb1dVuZDJDC8byOST1z9h0pVcSmIR7HIYLjbjB\n9K9DjjSJtg5kDYLqDnvV0aq+NvbniqgDxsDrtOQ3eiJgBCWCcjnC9ftSNjJASTMzvLg7gBy3WiBf\nz3Uzl2DuTvIVM49zih/oSdhqbW9yswmcrkjAJGf0qx7ozXLyglyx3E9DWqxWV3GqwyFjf6eJctks\nJCr/AGJzR82nCTQRfafJGu+LISQqpJBPQ+vbp2pJJorCmILG7vnu8SOJFQ7mR8DOOvTmtZYmK/gV\nzM9rKzZjEwY+G3oG9KLqhad6RutLiRZJZBsYjLS54oa1gZ3W3jm3iRgAfWpIskUS2rLeNbHopxu9\nTV0WmSWzbkVizkjgbsAY/wA1S8NQZfXtqkPhQWv4yNhnbq3bgUq1Cyub+18S1jO6HzsCeo6VosWS\ntCrTbgz3S+FIkTRjzMw64zRujxvOXmaUCTBYqzE5Ht707RBYN3mjubOLztG0QzkDG4e9XJcW8VxG\nzyRsojwxOPM3aoyR0Rkg1r4NYHcQm/IYlRzk8dKRvCvjh5nkYbjlZBhW+hBpIqmGUtJx7HnzGQIy\nSyo3O3P1qy50hJT4iP59uMAginFuxK66gLkW4aSSFTtGRkLnrj9aEjspUld2jdcOfzDH9TTxpEGX\nnT1D+IQNx7dQfvQksar5sqCXwAF4qqYpOydRHLHkYR8nNEMy7m2kn/4miYCeRzN5hJnPTaTVs9hc\nTzovy58MAEHr15rWVSbLW0bUGIEdq5xx0phZfD13AyzXMaAIreX68UHKgqP7Ov8AB10IEaGSPAHJ\ndsUOPh+6jUIwExPO2Nif7UvcbqDS6JFFbeLMkqKWPmY4XPpRSR6fBbK15GQqq0alBkg5JH1H+azk\n2KkApdRyh7eOTxUfAbK7SMdMVdZXc1mztEWhY5ywbGR/90GvjCl9NNGXcJLdtk4QI7Dcoz0JPaiU\njhttTNyGSeKRJYHMbZyNuC33zU3g1lmm2droi7tRYNA0XiRmJcow4GCPbI+9U2esSapPLbTEGKYG\nKMkDYrj8mB29PvU03Jmj6ZL+K3OmfFXztsPCmhkAKDtjhhz96e/FN7b3EunT2sbQyyO1ywIxwWOM\nevcU0o/om9bK9eum1GZpZY/ClVVZ4xzyAOc/pQhuoWgd4sklzwV4xVF4OsQMrb+V2jB5PNM9FvPl\nNRVfEVGmBQZGckjH260kmKhXdXUkbEb8MrEEZ/WkEvM5IHU9qtxeE5BM0jK4C5wAM1PT7Z7y7it1\nILyMEx75/wDurtoyVj746nOoaxHaWgJtdOiEEahs89z/AM9KzGnTG3v4m3BBvCksudvOM/Xk1NLA\nz3Bxe/Esk2rLIoBRIxbkheHQcCqdQYSwWLBvKI2xhdv8xqModXYOONMpjb8NcdzTG0QD8RhwrA4I\n6mlkdMTc6LNLEUSWMMp5UsvKcjp7c16ossjIXtl83ZWm0lBCGTPZhnd/eg9J0RtQ1cRRM5EYLO54\nRAO5NdClhxqLbHNpHYwXfgyRSyZYENvAA+lNTpcWsaobKNUCmNWRS30z9zipXtlHBfA3TvhmCNmS\n0tpPHeVgCwOcdyP0qXxev/6Fs99uGdXO+NgQJNp4z6c4qU32ejJUZe0mkTV0udWWKYBxmISkhe4x\n6AcVqoZ0MMsIjE28FtyHpzkfYd6dpofsJorjw76VZGjg8RHZBGMqSOcj24P61mJLUDUJl2t5ZMgH\nykqeR+xqnG0ifJo2i8MWzKAgSIFuTyCRjr+lCNHDdww3EgXyBgmw8e39KOsVUFxuDpk9pMS6yyeP\nG4YZ3en0qq3tY5WAMyxgvlWYdMdf7frVEqQlWx1rGn28U1pJax4neAJcAZwp45x9KGt9JS8lhjW4\nG7eA4YEYXGTzjrxUuxWqZK9WeOO5hVYzDMjsiqxOADxms5rfwwbRLB5JlU3KbyUO4AD1INNB0LKi\n+6u4haQ2qyKIbUEIsYB3Me5xirNJvJYoze2UxWa24lBTnaeAQOQR60z8F+UehuHkuWMrMfE4dn6m\nqHGydlCswTuaXpQiL4bSWR8SNtRl5yD0+tXSaJYwp5FcnHGWqidCsW3OjhkLowCjoM80CVe0JeIM\nWA6EcYprMg+xX5kDgZHTmndjbhpNsqM6EEYGc0k3Q6E6vNLcYkUSBRhdq44969JcGzimWOMeI4C7\n0bBA5yvvnit+glFlA9vFHPNE6wlvMSp2/qK0/wAIfDVpr+oSpdq8UJGYWQYz9M9cZFCUqClZbqnw\nFfaRO8UTideHIII47jniqLDSDsW3inRDMv8A3Np2AjnGO5wOtRfJaHUVHQ29ubGGH8Bba6mkUG4d\nQcMQO3pn2rOya6vz0NwdOjiSLh1jyviD39akm16xZy3DR297a30HiGJkt5VJaEENtI6deKASMxRr\ncR7bdQeCy45PQCqq6LR8DIdPM0ZD7YEPDXEjjbn24oB2ktrdxBcymIOq8jDMSM/XFFNtmaKrloYL\n+yjty06MczFhlgcc4PpRTz5j+Y021lGxWV9+Cpzx1FN9Qplzo8ljZtcTupZpATGOOO/P1NTsUaa7\nae1BjRcdDuwelU7XpBxp0TWSdbgq7liZMMpPVTxWhWxhW2CTQqyxAKoYYzznOaWW+DcfulcrQzWK\n27+RA5UA845obU7SPTmitpSXiMe5HZSOpIqY8lgIWm+VitoAGZyFUJycUetsLaFVWV/G3eZXGMD1\noTdYiL0FvLWc6cjKkvjtIDuU8Ff81x1tbiF31WN3YDiSNwGAxwDTpjKOHNMuUbRIlni/DBI3gnkd\niaW6veIIAiKAPEGCTgVSLFaI+AlvMQxASdMFs5Geo5qVsoiuly2RtyzLTi0OrW+sIjKbhZXMiBDj\njj/NVfP2VtIgtbxym78kiDy/vSbZ0J0h1b3cxjVklUluQdy4x+tXLLcTowa6CAcYTDZrNC9r9KJL\naaRkVHuJuDwxwB9quEM8ahUjlXA3EE7e3fB6UjGQDFLot9dNYrCjiSLIcg7WfuBn+tJJtHa+hyoE\nUcUjJvfhQR0Gaa2vQpWEWWiaZZWzpPdySzABc2yhsH0BJyaPg+F3vIUETtEjHhbmMKx9cHPP0qcp\nP02JYFa78Q22mJ8vp3is64tjbyufDAxjOwfm6fvSexuQZHuGkjjMDo3gqNu/nsKCToS9PatfXN9H\nJIiNEkSlG2ngEkH7Un0WX5OZzI0hKNlFUk+cdKpFfjRm2mcvbma51p7m5iVJHcO6Y2gn/wA00zLd\nJb/OwsfA8sUm/GB/t/eswL0vufHkvsInhO6c7umMY/pS6MAJIG5YyEEGsgtsJtbZYbY3V1xCrbY0\nU8ytjp9B61K2u4BP49xA0yDoFYpz25AqMkwL0V3EaGd9+9G3HAalyRmW7VFwSWH9a6uPwm/Qm2g8\na5CswBwevTjmtloz6PJcyapHbzi+tYHmmhAAhAVeCPc8fetNu8Kwr6YWS7ZZnkyd7MWY+prkd/si\nKvDHKDx5k5P3H2/WqfBPujzXNH0+yv7S0WaOK5jtlaRM5BY87c9mwaC1NWhtrFCrIDE2AR/7j/ap\nPUUSohb8hQMfX0oy9MsNoCoIDngg/v7VKXo6Hfwlrd3qV9DazK0i2sZPiKeQu5ck+uK9U5Rd4Uix\nnb6Q99aNPAAIdplwWwQMeYfb+lUrbw2+nFpJtovFBkC/yoDxz7k0W9onFWQhWxt5zJcyOvl8p9+1\nE3NwkUMUdqjfMpKsqy+gHQUqUrM6RrdPn/iNlFm4a2ukBAkVsHn61nviCDULmVbfUN01vbsCZ0OA\n3sT3ozj9QEI00B11B5WWVkdiVVVwMnoPuSBVlldPYXLhraRXTMcgBOfpmh2t0GkHw3VrcqyNbvby\ngsEMhBVMjt3rt3p8SwPJOPEeNQitnO7jik5JdVYX4I7yG3srHzPMHbhjjEf0B9aomCpZwxxPIihS\nd2M5qsNimTTBJL7xFijYKfCUJnHLck/1NNNDtpNR+IYLNfKHyxJ5GF5IqjeGgrYfqE11fao5tY9z\nO4RFTrx2z68VKW22WIjuQ9rdz5XbN3J6Djp0pMSK1oui0rU4YUjUI4DlXEcobIYGlVmVmuFt7gCB\nkO0nHB/WqwmpfCc40rGsOj2sSkR3C7S2cFc11LiytJg6FGwNjAKBuHUj70JKyKbF9zNZxSv4LNHD\nIS8ackj261BJTIokDYwMEEHOKMVgH6Ww3j7Su7OB1o5FNwGlIYrHtywI96Jqs9dlApAA69vSq4dE\n8VX3MxYgFV5wc9Bn1pJSpDccbenF09rC6WN7Oe1DHH4p6k/YcUxfVbeDaluDlYznzDqevI980kW5\nrSk0k8EThHOI5FjOOcck1VHYzTeZLaaSPdt8RUJGT6ce1V7KK0SrNT8LF7DfJeRg6eqZeRoy4DdN\nvPQ1K21p7/XGNjcPGygiJUIXw09z0NQeytFkqWjTXfiq4+VFrJeK8LYEkkYyw/frWTb4gNzdFTI0\ncOQgVQM4HfGetbjg36hZOKxFEpigcNbSNsDEguMEj3FDTmW4A3ruAP8AKOKaXD2abI2VZbKxBHGD\nkKB3+lGW4u4mRbiNyWbK7+AKskkh4tjGbUbyMxW8jF1wQu/zBQfT0oK4knJVnl2452qKFIpYSlnD\neQxLKJIXOR4mSQft6j2oyQrBdW0fhyIpVYC0bjbID/MR696STHQNqFvaw3clkrJKMlGYnp96F06C\n0sHnRS8uxwUdZBt6dxSJvq0K0m7BtecLrcVwyqFmUL+Hjr0zxR3yt1Gq7GkuQEyozkH/AM1VeaRf\nuBEUtxYRxlrcQNNyHljDEY6jHSqNdeW4spbkTzSrG6kkt+UHjGKRay3b8aZdotxaW1lE9wzCTJCt\n1JP9qF1l9jJcx5EZYAD1PelafeyWOOF9vdidSkbFJEHQn830otoIr6KN7uCHxQ+Ny9H46EUZWU46\na0KeMWVvCttbwu/Qoy+Ur6Vg/iWaCW82W8bxbCS6MchTnoKpxsSdfAqO3uLy0tUVdqnkLnrVssYt\niAy4bHIzT/SSB0u5ADGreU+b0xQ2q37XroCqqyDnaMbqpQ9ldtK0YODjPWmbXt0sCzJvjQnZuGdp\nP19aRgCtC1Ob5hprm5kaMOsQUserZ5x9v3raTY06G6kUlpC23eOQvHlH6EVOTplYaKrH4PmN9az7\npZISoaQg8gnOMD06c0O9i93bjTmUJ4kjOW3EZcEjcR06Y4pJSbY1JA8kSQXSxosTTMNqyxtuKt34\n6D/zUphBPcCKy+bkIYZaVtpDHqcishKti2XR7KHVlM7XDMWBbbja5/mBI+1EHTrZry7EsQtWWUBU\nAO0KQOh9c9absvA9K0Dv76eytmsofD8J1Ido0J8Xnqc1ZoWmRtZveS3USGGQl4pMjAHQ/qcUXiFX\noVbi3vbu5siI8zriGbbwJF5Xn0PI+4oKOWHcbd2l3IQXyMbT3H2pY+6M/wBhobxbWTcpLqwCHPVa\n9pWlG71kQyfhQA+ZzznJGAPfrT3QKsq1KFZ7mQopRBnbH12bex9KGtWltozGVGJeSp5DfWtVivGK\nr64V5JH27ck8DpmqLFsXnicDZGzAn1wcfuKrBYIzVfCUOiTaPe/xKcQ3u7ETufKqgdfuc/pQsVxJ\nbfCeov5UN46Q7hnoNxOPbj96FO9GTMsx68HuTjt9aa/DVrbvqguLzAtLP/qJBkZYLyF+5AH61Ri/\nQZbO+1jUpNRmjEMUztKHlbw1wSeBnr9qaXPy8WlabHcMWXwnbyqc/mIBFRnLxDxX0FthGpyhYhBk\n5FRN4pjZBPw/5lZc0iVsq3RrPgu0tVsp2YHxZPw2mVclUOcj9K9UZN2WilQ/+H2bwG8STZDOmEU8\nEDuR6f3oTVdHmICbSYgqhJgwyFJGRjrxgUva2SjhO6+GrGwsrq8vXdt3lgVm2mT04pFbXtzE0qC3\nQtKmwsW/IPanUmTk1Y80CFI7VXvnKQtuWNy2SGA/80VHLPDdz2zyl0VeWXo4rRlY1WBz3MgBaOVo\n0HRMdCOhpPJO0MrSzSIBkMy7Tvf71TqmLQbdfENtq8LRtp6iXnEkb4YfaqEu1toTJLKXDJglV4H1\n96SXHeBE+qa3DOqO+9o1yu09CexxVDX4MZRZAdo27CeneqRg0qJpnLOJb+9FukZc4yWVuK0Fhb3W\nktPq+VjFsvhecAkswx0z6UHipjxWgdpdPBctNHIkcqZKlvU9xUNX16e7ETBl8SM4DoDlsDqTWUEx\nnKhQZz89DK83hyoVlXc3f0rS6jpq6pc2yQgJLM26STaSVZug9Mcg5rSx4BLsKb7SZ9OuTA8xLoMM\nytwT7cUkmgnViGlJHcYANUjJMlKNDnR7e01TR5bE2zzX0GZo2VlBC++cd+wzQ9vKIkI2lmz0YGkT\ndtClsZZm4hGAD7U302a2ldLfeBvC7yRgEjPehyOo2Mhfq083z3mjWNAAoCHO49zWm+GdTsDZStKb\nX5xIlWOJ2wzkZOceuT69qi7nCx4umK/iWeP5pEjd5GQje0j7ixxzz9wKVyXkdzqHipD4MbMPw1PA\n45q/HGkaTti9dySOPN5TgH1ppZa9c2Nq1vDjacsSeoozipKgJ0G6T8Q2EkE1tq10/wAuX8Xwwchm\n9TRdjd/CtzNO7ymFF4CxuU38Z5/X9q59i8OiNNaKdQu7XUZLYq6W+7KSeG2VRV4DfUgZ+9J8NH4s\nkUrSLFJgAgjeD3FXhL4SnH6GG9tvDKGPew8pNVpcQvJtAAJPCg9TVSCRfHK0V45t2ZXQ4GU70wL6\nhe3axOjPJkgeXpStFIj7Tf8AT2+v0d778Nd3kA6mtTa/6f28UShgCyjGaFWMnQxT4MsVUbo1YgYy\nasPwdp5AHgoMdMLQ6h7MHHwLYckxKxznO2hbv/TuxlhKwRCJuu5azjYLFV3/AKaLcWD286RtnBEi\nDDg/X6UsGlXWkWSWfycjXKw+F4yp5SPXPrU5QY1J6KdY0i7i0FJZLkmVGASLYd2CeTmk1/ZyWtoJ\nOGEjbDxjb9aEXWCyTqxWqXCwu0eMIcnHb3oy4lm1OK3jUhViXIGcAknrVmiEZVgyttMguo9wkPjo\nw2qufvTl9OmNulurmZnfhgNuP19KjN0dPGDa7IAkbRnbMgMbLEcfl7/cVidVgCakRKsihhk5PJp+\nPGhZ0PdJ02S2vrYZZllR2ifOVYhTx+1AHxrtVmPBKkEv2Of8U8XciVEH0iQ+bxAQRnpS2aIi4IOc\nAYq1mLo4m7BsfSimuLgacbVnYQBt4Q/7vX1qcqGSL/huA308tuVHhko7uo8ygHsema2Gl+HYx3ln\ncSic3su2Nc5bO3jJ9elTkykfDS2xCWaGZpIUMWwoDtOAOm7t6VlJ0uLTSjLFJIryKyOrEZHP5v0P\nWot2x6ESN/1RkXEayDJHbdjr9+tONEu4ZLlrYqFYQnwSV6vnj+9NJ0hQeLRYY7+OSW8kLHLSADJB\n9qne3lnrVpcWVvPK15Yv40eePGXA3A/Tn9KjxSlyO6DJ0jPwXLKxjQb92cbxn/6o75tprGXTWjjj\nWRjIWUEsSBwv06V2PREL7aGVcuY/KvOc8ind9bi+ij1JISGcgXe3ACvwAfoRk/WhYQSJyUVUDFic\nYK4JFWmae4hilIZ47eYKy54GOc4pWZE9Uhjs9eu8Fysil8KeCrjOetK7q8/EQQLtVVwo65/rTx8F\nkKrkSS+YoABzUI42+XaVMBNwRs+/P9qqnRNkY1DEAgfQ0zed30+K0dyIYiTjHQ0WrAmF6HCNP1uG\nS4gaS3nQrIpj3Agjr+taZdK0VrubT4bSRIZE+ZllcELEApxx1POT171DklTwKszWvab4sh1Se68L\nxEBtrMxsp2AbQeTgA4z96X6u4kstKx//AGpBx672oXdBTsgqNFpBuMgb28PHrwT/AIpfDgv52xim\ngvSjNloME15atFbyEI7LvcD07Z+9erlk3bOqKVDeG8FxL4ckhWIjI8nQ4z+lMv4ppmj28V3dyxs8\nxwp53N749BW614QTQR8QB9QtYbxSpj2g7t35vTFZeO1Mtz4SFVMhxuOMU8XhPrbwK1K8FvAtvJmM\ngYBG059wM+1XzTwyfDNjc2d0ZSGaOQkYJGTjj2rRTrCrpCq4v4bsxLCxjfGGz0+tCZk8QiIeJs/N\nJng/4q3gidlxyEM7sWMYyxBzx0x/T9aqVw9rIQW8qhtuOufWkXphJKY7i4VABjHIFcdI8mMBG3Hz\nsBzirJk2e0qZ9F1UTIdzDgKen3p/Pf3WoEt4xxNy4i6Z+hoSSfplJopMXhAKzbmzgM455qqCNVtp\nBwTuyNwpVgbBpXs2bfJbo7L5e/HeiIdZY5ijEsjngDeTtAHUU/votkRd3cxX5qaSRUBCJnO360Tb\naf8AOuFLJCgx4jsQdoz1pWlFYZMWX6R2F9JDC3ieGTh9m3I9fuKksa+FlVHIyPahHRX6XRzqWVS3\nBXvxzTa3026a0+bhjEkK8ZJHJ9PrWk0kPFWIL+UxXKlpCzSEsMdh6VBTHOOWxng9cnJpksA8YXO8\nZJECsqrgfmz2Gf3qpOCcYOP2pjHbp0gmfk4HQCgVu2ExOOgyAayRip4/mJAy7QWGMAYohYIINuws\n0i9dxyBQaMj0fmAxt2buQB6U1ju/H08RLFGSGO3A5Ge1KNZXp2gXupAeArcvtLkZFfUNF/01tbKP\nddDxmbB2t0X6U1hUQmH4Rgj1KeadU8MEGNQO/c02t7W1ilLBUzu64oemGq3MYHGK6bxe/H0rUAib\n+LnkfeppdBuhomLhMvXNc+YXIwaxjolRhhmH3qTJEy8qrZPpQ9MDXGg2V6hWW3yCc9KQ61/ppaal\nCUillgBO7AGQTWcLN2+GSn/0l1a38SO2uIXRwRlsgmsvffBevaJGGubJwgbAZPPx9qGr0VwvUV6V\nqdzpFzLL4IaRhtUOMBeeuK9L8Q3VzKxmOJQGClSfKfXFTlFSlYY9ooytneXtrqSzDdM44AfOG5rT\n3egnWZhcT6haQyugPgRne4HHTFUm1ChEzQ6fa20M9pbfOSxIk25fGt8bjjbgEHjOcffNY++lKyvF\nuKFHKg498VPidzYzeHhcyDTvDLbwpxuxg/r3oKC1nvJWWKF3xzlQcD610tpAsJTTUjnIu7+C329V\nB8Rv0XP70db6tp0bRwRWvisSA0s2PXqFHXj3qT0a2cvpr6HUnddwtTkxoF8NSh6ECtJ8NvDq2nte\nXcQExlIUDy8AYz9aWSVWUiO9X8DUvhq4hjYSThhG0eeQQc5x78Vm4JHGmruYngKwJztHoa4ufk6e\nDPQa/gjSJgcMxTdnptoPT9UezvYJIAr7Ruyw6EVPik+VWxEiy81TfDOyMIrltrYKnBBJz9O360Rp\nmlxxaRPeSkpfbTcogYAlMYGfqSeK7OOLj4ZoQaVKfnwqq0plBjAAxndwD7c4NPNNt47aGaWZmWdV\nwEU8jscmrPDIouGN3NHCNiqy4LnyhRnv61028VldmOWZZrS9jZMj+Tk4b7EA/TNCwlnhtp6TQTTF\nLqIsqIUz5R3J96TieSz2OsnmBOMDg/UfaitN8sM1LVZ73SLLUrdwk8B+UuBgY4GUOMehI+1LraWX\nVHjjRR4wOQQccdcn2po+UCWlFxdlpXt28xzsLLjk/WjX0yay0ZjJJBslx5PEVmVu3T2U9+9M2RYu\niiaKfEg27exon8S5lCpGGLcAA4prwBo0eLSLGM3DBpFA/IcnGegz+lQ07U7i81efwWkLOrbEnkZA\nFxjHB+tQ62ymJaLn1DUFuZ4WunW2eJmWNn8RVI6L5s8ZI/WluqSNJHalkVT4BbKjgjce3aikkLEv\nuNqaRZQHbuZGmPfknA/ZaTrEzOAF68AevNNHxlH6aSxnSO6SxknNtasRGZF7Huf3r1cso6XjJJBl\n/dzLYRGIbQi5B2DPNZe5mnupQZC7sPKvHaq0cc5fDQfDfxFPboNMu5nW3biJnBIjP+KZT6nBbW8n\nysXiyscB3PAx1IrKN4GE6F00st9dJcTKoKrtyOu2px+EIokBkC8gDPYjH9qqlSHu9B0QeC0Y4I87\nnGWx2FeS+mt7XbHM4GMlQcBu2TRasCdBOm3KXNtEZ1MgDlJHHBK4JA+tE7YJZGVUkjjztO4+Y9et\nI1Twe8EtyUbyxYwvlBUYP/mpWmmRGFXkZw55OOO9ZuibCv4dbSY/EcnOd3rRcdzDaFY4ouoxmt2s\nm2wK81FhOUjQlfUrnBqmebuTlm5o0OgSCznn8XYoIKnrVVtDc21ws6MoeMblyOh+v601gaNBf6rH\nqT+MllFbsP8AubJOGPrigDdL0RYz7HtWAmSJFwjPIwYqAucD7Cq1KqR4sgj7crwKNGOmC3uD5JFl\nKt/tx+lFw/MJA+LrwljO/aznH2HSkkUTFl3ZtdNHIjHcOO/SuxWsqrskdlyMgrRi8FbLo7WCBGEb\nu2RyCe9VZEcpGAR6+1MZM5rKKuJo2BjmUMvtgYP75pazEnYAMY60yAy+2XEbdPX6UQumX05JWNwG\nGRheKVuhkhhpPw3quqRsIIG2g7C54ArW2f8ApyFEXjXLkofNsGKX0e6NtpGn2Wi2gghXCKTjPU89\n6Mn1qMDydaNAsXT6g8x44z6VBJJApx9qdISyYu2H5q584zjG/itRuxzxNpHOfqaLiuQijnmg0FSL\nVvRnDE/QVZ8xGemc+9Cg2WQbWkBJOBTaKWMAYwaZRA5BSTg+1WCQGnRNskMGvFQc5APqMUTC3ULT\nRwu6/itFxzmQAVjtX0//AE6nlZrs2aydzFIVP6ClcExuzWGZk0z/AE9tL9Lqw1KSCRPyhVLjP6Zp\nPNpdlLcyyWmsQXAkO4AwtGwOe3HNSnxtmtDObSLu+aC4itTuWRC4iQ7fLjnB74qOpf6eSXOpu/8A\n1atNO0mTF5Np5HP3rm6zjK0NgPJ8K3unWPiWVjK0xcq4aLc3HcZ45rK6np2tpOVu4L3Y35VZGAH9\nqvDt/wDSNVil4JI2/FUqf/cCP7VZDEGlQHhSQCRTS8Gpo0WuXJ+UWKWSeRLVUSB5EwXQjLD7HvQ9\njqWoGN0tJY48jDZAGSP71JVWm70V6bNqWlyytbxeNLIpOeTg+v1p+kmoQ28cyC3UzRDfG6AcjjP1\n4rm5+KPJ9N3sA1G4uLhDHdeGzN1wP6UKbOS20k3DoQHbw4lHU4/N+gxU/wCPxrjXVDJ9jqWNpJp0\ndxdzSpKMsYgMlxnAUe5xQL6jeX9zcNMhhyoTbjyqBjC/XivQiv2CWHre2NhCJohiZW8QSD+XnNNr\nTUw7maeH5prjO8Hghs5yMCtJGTDbV/A1KWWexaS3Yt4SLg7eCBwwOetC3dgX4fEIIygIAwP1/akH\nWhGoCF7cTyFZJ7eKNHO7JZDwGOP0P1FZm8uY5wDBD4JAORuLZ5poL6aT+DL4Y23k8+lzMVXUYiin\n/bIOVP7H9aq1GEfD9pNAuGvEI8RzkE55wPatf50Bf1sz9pb77Vrln2ktgKO5phb3EItLi1dSPEKy\nBy3QgEf3P606ekZL6e/CW2mkm3EjhSKhBcBEJTgjoe9O3gqC7iSQ2cSSSF17Ar+XJ9aX/MC3imjV\nmWReY3U/rSQdhbs7a6hcakZGn2My4w2MdfX9qMm0+ZDb+N+QqwGTxjJP96E8DH09qN3/ANT4cewJ\nFGEBAyenP7k0Pp0JeaSVy2IlLe5Y9P3NZOolH6OI7K0TQJru9lUTA5iiB5A4ya9XO7Z0JIZrBLIX\nYkFAu0D1q+KIKgG0DaOmK6Inj8jdg15tUFtihj370kmlCyAHgn/bTP0bjv6WElT6YGelElxc28EW\nMGJSWOPzZOc1jpiSsbSOWcu0zRoDuJUZJoXUtkFx4kaGY7AVUgjj7UqDZ2wuXFgWKpEszcqOSMVe\n1vKu3fLywBzxzRobtgHb2SPdlJJtkcSlweOT6URcXcksIE6ggeVSmAfvSTRNyBopJM+WN27Y61e8\nUxXmMpj+Y8UsVoCmVJkTb4iCQHna2R0qyJAypv2kqMZ9aqhjscngMCAuDQ043SySFVQuc4Ws0Zsq\nuHYR4GemcZoMMYpWL4/LnAoL0AWrr4QJG0jmqpZ2nkEb8EDjPQiqIwRZboZA55HTFGu4dSSBj0oN\nBKYb4QTAEEr+Uj60MLtmZgWwRxzSpAKmvMTbgAcjFXxxtJDI/YjA9qIUXy28lzo9oFiLskjRuAOf\nUH+tFad8I31642RtvL7dhHQetNYyVmx0H/TD5e+jlvHyqtll7Hmt8dN0+0U4RFA7YpasbwD8e2tV\nZbaMbTzgDvQk940g4zzTpCNgreIwJ3falz3UkbHcaNAsst7pmk3MfLijvnYwByc0aFK3vAeAtVl9\n3TjFajWeEwHBqazemaNGsuS4fNFR3LHG4j9KyiawpL4IMcVYmpYPFP1EsJXVMDrUl1RieCvpya3U\n1lF98X22kRb7uZE9AW61gPiH/WK6lnaPSyEQf/kPelaHRg9W+JL7VZzJeXUkhPYtwPtS7xstxWow\n20TSJNXn2LNFGeuZO/6V9Y+Gvg21skWUlXcgZBQcfeszG6tLWKMDaqj6CjgoHYUtBslgVxo0b8yg\n/UUQAd9omnajHsvLG3nH/vjBrO6n/pb8NagnFn8o46NC2P2Oc0jgmMptCbWv9LXn0u5it7gXcrxq\nkRnG1kAORg1gLn4cutCuVtr22aBiAAT0Y+oNc0oOHngzakHRiBJ0j8NF7FsnrivJfxpaSmSFZEjd\nQVJIPJ5/SoOSZkgW1+HtQ1nWJbWyACp5/HlOEAPI5+lVyKdHv/l9VvciBS0cKNlZGHOSew6e9bjT\nbHX+mffWJXvPEnZpGZsEHoPp6Ci21YC2MIji8IvvII5z9q7PERk9KNS8GXTIp7a5SNlbEkTdT6Y9\nsV4z23/p5NuRqKSEqRnDKRxzjsQf1pOyHi7GI1i6l0OGe3xbNANk0RJL47SL6gnr6UmXV3lY+Jl2\n/NknkH3rLSjxDSw1eC31pZ9S3fLyxCOQgfynjkdx7+1D6to0um3gCyIySrmCSPpInY/2rJdXYbuJ\nOyX+HxKyAPct+aUZxCM8jHqfWm93BHrfwPfBr3d4MizRFhliOhXPccmlm36PCqox0JEemmBRkRj0\n6nNVwqHk5wPU1WGkJIN1S3WKygEbbnlJ3AN0x04qvTYhLKPGPlTBOMU0vBEhvqKW7wy+CjNKcYyv\nT0IxWWMTtdFCGEgByD696lw39M/S+zRLeCTeQWDKu0n2J/tRl1dRz2VkkRkJhLhy3Tls/wBKeSsy\nBId0jNnLHOeOOKYwJ4dikO4iSabJ9VUDj9Sf2oSeFIrSN/EsUjoCThuT/avUlsdo1Wm39vcx8S7G\n7o3Bqy8ukgtpXQrIyDhA3LHrj9KdKjgnBtiK6vG1GCNkPyysPMsinINJtTuRYIvhypJKePynim9Y\n0Y0Q0/XVcYvWYuDgFe4pst7gEhGCEYBJotUWR7TtfVdWhtguYycFgOSaP1UMtztUtheen3xU26AA\n+G3jIjAAbg2cf89a7c3IeVnjAGxgACc8dKZML8JwLC8sxkdAM5GKLefTo1G/OcenWtLSYOdWgj5g\njJP0wBVTzyTqd7eU8kc0qQUiATxCQqZ5yML+9dVXSMA4B3fzcU45xiSTnBGPWoE4UAY57k5xWMUs\nqqniMxOcgYFRNsi5IQsSeSTQoyL4oFkjAIDEnJPaqHD27uPDO49zzinQGURTPI2TJ0PcUfEx24z1\n6Zos1lMkDtLu2kk8DFdvNIlu38Qq8YI5A9aQxJNLjhjAcMeOpWi47cGNYoznaNxwDQY8aNX8M/Dr\nSRTfMhlhcKwIOCSDW6tjBZeZVw3qaKQW9OT6yADtIFLptSaZuW4qiQjZH5uOMedgaj87Ex4oikXu\nI2X8wpNcP+KcdKJi+GVWA7VeSMZDCigMiHOetTD8Y70QHCue5qcXBxyaxglDjrVqy8UyQGzwfJ5N\nTM+BTCnvGyCT+/asp8TfH0WlMbaw2yzd5CfKPb61jLT5rqOsXN/cGWeQsSc8mgjMc9c96RlER8Rm\nairSGSaXbGAT6E0DH0v4J0aVrcOHhyG8w28j719Q0/8ABjVWPagwocwyKMUTu9KQJMdK7TAPV7FY\nxzFA6to9prFm9veRKynox6qfUUGrwydHxb4l0z+Aav4N2JTGHBJXqy5yMfbFKLnUYvkJ5EVykzYU\nbencf2rznFqTR0Kvgw0vXUn023sZLiW0kthlpiT35w2OR35pXrGlPeJezSXNvcSWe0h4ujZxk5+1\nVh+LC42hHd2L2RRZEAOA/J5Oeaj4vO5MD1GKq3aOeaoqlTPnjPA6iqiDsBJ4FSQiDtL1BrC5imkh\nE8UZ2suAdwI5XH0o3UNPj0y+SW2VJbaVRLbuwyGX0PuP7ULplrtAM0819+D4YeaSRRGwXH/7Ppiv\npPwt8L31vp9vZ6rbQPAgLpvb8VCRztPQD2NGUsofjpq2D6v8E3EXiLpkxnYJ5o2bDhT+zDtxzntS\nezsrqDTJ9PiglXZ+KBIpDIR+dffjt7UnfsqHiktTFM2nrcQM3EYIHTqaBOlGG4hJL+GW87FelWjI\nnNWBteD+IOsTFVd9u70AOK0Vz8Pm2jF5CDPat1ePjBz0YdvqabkdAjGyg2W+9RLe4UeUMrNzg4HB\n9eTSu5too3nFywNzG2FMR4alhL9GlGtK57GOG1L+IhkZl/Czk4xyc1RaDy4Y8ZznHSqCMt0wtHcq\n6jo+AWGRitJfad4MsksLJLFIoljKDGR0I+xzU23ZWHgtvtMu1m+YYKYC6lnHG0H2716k/wC0QtlN\nud1xtZTyPzZ5FFylZbm3ES4UuCyIpYudoFPVi4O30K5e3DWUcgkGMQXbKrN/8eefpSS8tl3TW99E\nI5cbSrDDClUZJkpL6hLD8M+NepHb3CbWxtEmeT9hTy4+HLmC3YrJCfC5Khj27DPeq9gJiWKwUzeL\n5w+fL2rTWcj/ACu24uBx0GMnNSlJDJActvNId6uJJWOFzjgdzS17WWK4yEG3OQOoIpewWG2ttZmO\nSa5MqOR+GIxu59/aoPaTz7mmkREQdMct9BVUxKKPKgEefD48zk5Nde4idwsZkbgAKAck+tNaCh5p\n6tAC8mUO3aFPWvTpA0bK0ZdjyDnoan3t4YCfTSkLbZYyAAcnNDQWyXExjeURljhTiqowVc6W8EX5\n45FQZBB5zS+6tLpNzGORFbnLLkUWZMK0aJvDdeGIOTxXtXiAmSSNgxbr2xQTMLgpKkzHaucAdz9K\nk0ph2sFLoBgAHGKf0yGdrOWhDbdpI74NGQCOUEXEki+hSoTlTozJnSoppR4VyzAj8r1q9A0u3tbc\nm4jXeRtzjtTQfY0bGTXCQx7I+AOlCyXp3fm/eqoLKnukP5qGlucdGpkKCu8kpO1wuP8AdVkbN/8A\n1N1EwSFBXzZoWQBZM7jg1jF8Iz1yauXAGP2NFAZLjPoa6Bz1oiloHvUwwWiCyYf3rwYmmMSJOK4C\nSaIDIfHHxYLK3+TtHBlfhmU9K+aSzs5JJySckmlbGiV7sjmubqUY6p81PdBtWnuV2qTjqRWRj7N8\nMw/K26q20cZworUQShui0JGTDY3IIzRsT5xmp2OEqc1KmTFPV6iY9XCAaxjF/wCqPwvJ8QfD3i2Y\nAurU+IMDlh3FfH4tGZtOtoZHdFd2MhbgJx+tc/Iqdl+PS64mQSQvY2oR4go8wH4hXksee9d1U3E+\npSPHAZzfxbpk28AeoIPJGDUk9KtJIRa9ujv1VgSPBXbn/bgYzj2xQMZp7OOb0sbI5FRRwSQxAB4N\nIIjgUhmT24Ipvo1wb+2/g8rKsnMlqzdCedye2eSPcCtKNqysfRa0kcsjCMkMp78EGnSfFGtwQoIb\n1hsXABAO768f1oVYPGTvP9QtSRGSTlWgMTHqwzjpjGOla34V165utHiutUkaSF28O2kfBlmb0U+n\nXk0Hx0rOnjd4gXUfhuON3mttRgjikBdEuMqUGeme+OlZyC/X52RJHSeKJGXcpyrHBwR+1PDTSWmN\nRC0gYSIS3I7c19C05ru4nkuredGMlohkSXiOQflYHtxjNPy+IXjVthF7pdjp5tbizuEmjuCFG0YA\nI/lU1m9RtY4dTfCBA2W2sPfkZqUBpxSQF8q8peVhtTcAq7T0qvw1g25Yc+tX9RzlthNHDceIUDAE\nnAHfGB+9MLaeWOESLgxqdgyeBnqKw6v4NrKSWfS3guYZeJR4ZdT5lznGT9CK9XickZRmxkhFbLJb\n3izzWvzKj8yM+0H7imsOoyLMr2VqtquDgZyQTx1r3okmekaaWQPLNIzRMCrFjwfWmVzdWOovG93C\nPmmG1pRzn0JFPKNipldrbWpmMbwoJEAPPBI9RXLi7t4pGhO7J7c8ij1QtCWayn+ad13uh5UBTkUV\nZ2ySuFuFnVMcnbiuWfHpaPh6GH8cyqpC5KKCecV7+Dy3pJMWIx1bIXpzUlHQMH1O60+6ulNiot0A\nCbEBwCO+elWWcT4PzEDBSvlcg8mq9RfgStlbMc+EuD5sHipeJY2DRToyxhwVYMM4560Ghds6X+bu\n5I4irOg3gHo6nuDQviKUZVYMwbBII4oQVDpE7mRvARdgx1L5qqC0GPEz3JFVTozQK9vdqGkhhZ1z\nk7un602sdQlQgXS+RxyN2cVm7FoLl8CWQtGFUEcmNcZ+9BXtnHIVmUbyDgKW4I98UPBkUrMGzvt0\nGwYUkV2MwO5MlojBexOKVtjtFMgRpD4SJCD0TPFShBeQAMAT2Nc7tsQf2OlnAZiQOpGacSXYjAUc\nACurij1RkLp77ng0L/ECTzziqmOtemRemKpadpP5gv1ooBOMGQ7WIbFEx4jXg8+lMAuQlhySKqlO\nH471jBlvwvOc/WruCaKAyQGDx+tSB/SmFZ7fUt/HvRQDqtUjLtHHJogPB/XPNZ/4s+J10WzMcI3T\nyjC5P5fetZvT5RPO80jPKxZ2OSTVDHNIUqj249qkCaxicYy1bL4SDpOCM4PbbkH+9GPoH4fU9PY7\nFz+gGK0Nq4CjPFCYIjFPNjFFRLgc5qTKovU7TVqtWTNROvU6FPV6sYiwyCK+Hf6mfD1xo+qiWDeb\nZ3LK27oD2qXL/WynH6ZjTdVmhtJItviEnERJ5TPUD1zTOe3s9MsY49TeWW6uiWIQ7TEAM49ea43K\n/Cq9FGsaRY2cMNxDJMWuovERWIIHPSlCQSSvsi8zkHHuR2p0/wBnPyJXhJ42gt0a7aOGViB4RfzY\n9cdhUUt2ll2xjJPemr6idUwmxsHu0fg/hruOP6V6XZaGCeJts8EhILcYwQRT/C8VWl2oWSJqyXoI\nS1vU8dCTwCeqn6HI+1XQxxX9yIoZ0LspwM4yQOBn3pEaXpzTvhsalm6ut8em25BnfHX2XOMse3an\nNzqQvdUguLWJYLSKBIraBudgHHX1yCc+9C+xWFxRoL60j1fTo7dMGaaPxFycqh7gfXOawNpFImtJ\nYBGEm4RuXXheRmhBejyu0Hv8JaXofhzahqAupCCUtoBncw9WPAH2qU2oHU9Nt23R2KJmN4UHlC5B\nXPr3p9krYrag6G8wkOkPbT+CYonLqy9I8DcpX0yQRj3FZ7Ubu2e0jZ7fxXk800jPghjkYHpU0tBN\n4BWolaMIpZo0KkpnryP813WdIuLS8mMUUj2qyNscAkAD1qt0SirOHT5otOivDtEZbawDcqTyMj0P\n9qf/AA/e2ei7bjV9rr+aKDbksfUg9KZu1gyw1uj61L8fM9jaWqWkULKWkc5P0A9a9XJyQuTB2McZ\nozZsWYRqp5BBNcWKSPJEbMmzduwcYzXfEVlct08jFlG3jBHXih/+7etvkfaq4AIxVLEofaTcvNaC\nIQIWjOY5fTHO01O8HzMUWotCg8NwVVXy3vkemf60jdYMlelj6xcRIVjWIA8HC8n3rq3bNMElTarc\nZ3549aEsRkwG9tbeWTdZTN+Gcsv7UqvRcW8ZKPyp71xvk0ZieL4iv47GSxRkEL5ziJS3XPUint/8\nVX9xpFvFeLbqyKCrRqVYj37V0p4LQLZSyXqTRz7oiUzDIDkE+lAyrJJEySuGbI4pL0yCbDKwLDK2\nwp/223EY9qLRITcBVGHxyVGB9zRQ4YrwkeHt3t2560Xb2bfLB5V2P1CUzAU3hIhK7TgDJA6fpSlp\ng0bHO0gcYxQiKWWUk0gciVSdhIyOuKuS6dm4IwT1A4rNDIu/7keTy6nI96gw87cYz3oUBsquIkVS\nVkDMOgxRujRLJKCy5A6kjpSddAaGS4WNMZxS2e63nlutdJkL3nZuh5zgVBZWZ8MCD9KWxqPSXaRc\nbst6HiqxJLI2RGwH1p0Iw63Mh4wB755opThgMN9aIAoMduKqcfiDFEwTEo+pohB9qZAZYuRXScDq\nPpRFZz+X09KkgYfmogJO4x15rin1rWYHv71LGzkmkcKqDOT0+lfI9b1eTVb+SZ+nRR6CgwpCzOf8\n1xvSgMeUVIUDF0IG4Zz9q+h/BFsJRuERBH8zDp9KeIJeH0O2RUGPSmkEq4A60JAiM7WUkjtTBGz0\naoMqi4dasXrWQWWiu06EPV6iY9is98baJHrOhOrKGaA+IM+3X9qSauLQ0XTPkl9Y2enTpdwQBpWc\nLBEW8pfqffgUnuLGP4jurq4u551l37wduSnGcfTsK87ibxsryVWFM2nxXNnAkcjSSxuFVXbZkHsR\n2wf60FcWk+lXyx3irGynI5yPtTylZCdZQPrnw7cSau7WYBgmxMjM3RWAOD9M4+1TsdGNnckpcmZl\nQ4G04zjn9KtHkXUaKs01pLp+h/Dc8V1MUvpI98SgfnB6Ems5qubjdJCxkEiZJHI3H/hpYN3Z0Sjl\nBmp2McFtp0GrzOxSDmGHG4EknkkYHBFM4LHT9MiinstPgMw2gM+6UqWUEEgnA7jOO1GcnWGUaek/\nibUXGj6fCzfMRTO8kjZONwwAF9MHPFFJoLNFZubhIw7MfxOF28HGOx5Nc/fr6UUfo5jsVBSQQuik\nbA8bkAKR3OaGd1chESGW5/KWIww980OyCvQDX9NGqyxFEhedUKk+IFbPHU9Ox/Ws1Fol2168Pyql\n0cZ3OMFTxgeuavxyE5Ir00djpsUOnXMslmy2rFlWEvuUMoJLFupA479TQEenWuq6R4NmhW7aPxI8\nt5W2nkD6jNartonJpUmIdOuUtNRgebd4BYLLgZJXvx619hshHNpatEiz29woCgYA2+/9KlN4Dift\niTVfg+zS3luzthlgYLBEWGxyecnv5az978OLc2rzG1llLZHz5nULI47Kvp6VSE2qTHasI/02vJNN\nv5rW4t5o1dgykA+Vx3zXqlOnJsk1TEL3TbcIx9SKi95dAhk8SQcDG44x9670AIhiWSMyEOrFsFBi\noB98oXrzjdIwGPrRT0DQxjnWx8zvHtHPlHDUyNxIdJN34ahFYZ2chkYkFcf37YpOR07H41aoT3Za\nCQqjBwwG044IpZcXjFyqy7VB6CnbtCVQTa3rw7ipzvXknviqrrVI7zybMMTyc8GuCUaZmwB9JAmM\n2FCnnwz3+9AStvmdSxwwK8np6V0RlaFTNHakjTLaNCAUGc1XL4MEbFSHlP8AMaVrQfQBpvEjY7VJ\nU4OKZ2tv49uoRMyKvlOePvTrB2X6SjPdnxFy0R/Mw6f5rSNLEkZL87QWz2PtWbsILMyfwe7cRAye\nFnB689/tWTjj8Jc7c7hgZ6ZowFLreUm5C/7wVG3HfiuxSJkMN2Rxg06G+FrXG1htKjHWumZZAA/G\nOpFZoUikBlmHDYJzmnURW3jGMfasjFE93uB9e1VWgaa7XMbugB3bevT/ADWbHigm3sJbeK03gE7z\nLvAyCO4+oq17u1vLW9M6xGVG/BZcK3Xoex4qSbKNIrubW0hjjeKSNxImTkZKt6UvD5YbT3q8WyEl\nQwixsznn6Vcv1Jp0KWhuOuKjnLCiYLhXiiFb0FMhWS9MVL9M0QHc+tRZxisYrXlqszjOegrGPnHx\nzrzXd6LWIkRRdcHhjWQY5P3oBR7P0rvWsE8KkDQMEWSGWYL0yRjnFfXfhZRHZKgjCceu7P3qkRJM\n0yvjgUXBJyOKWQYjO1k54PNNIJOxFQZZBcfJq4DmgZkxUqohD1eomPVB1DAhuQeCPXNYx8N+OLYw\navLpqN4XgybgepwfTNZe036TqccgaTZLxuZRzx35NcEY9LTKT1Gks4bG4SCO6jDvNiUEA5JJ4wQe\nOas+NtAvr62BW2jMtoimVWb8TB44/QVycblGTszjcVQot7fx7GFbzxI5YU8PBwMrn+tPJ9Ds7XwX\ns9k8sjARrJLjJ5z/AI+tW0eEK9M5q2n3epak1xJAts6KqvDkAoM+v+KlFqI0RHtFtDIZdhikc7gT\n0bHHrV1KlRdwt2EXGlyXaSy3ReHxHCkytxj1HpV1loDxmRQ0fhABEbxOWHOcZ79aj2fg7ivRrDDH\nDDBFJHEIYkOxZ2DDPXI7Zq651SGRVKxG4lICxEQs6gnr0HNKwrwYrHqd5bP4VhNDkKU3xYGR04zn\nFZrVrjVBeRvcWs4OWU4jG1/oRzRSt6ButM3d6slrOTCrSTO+PCY9D2GO9HlLicW7STXC3Vw4jCHy\nBcY5xyDirOKRyt9mGfGurXFppsOmWtyhgA8Jwp82FHP6ms1Z3kiRRhWIKnqv/OKaCqLojyP8hrda\nbZCexMZkcXUm2RwdoQnHb6V9K0xVsbKC3s3BsY4ydrt53IJ6H0qLjZ0RiloDqF1YfE2qxW1xK0Vs\ny71BBBJHUg4wc8g+1Z7XNf0yeaSyuUe0ttNbyxwKQD7/AKkfrWcbeFVKgjTPiLQ9ReOz01Znf87M\nUwCPevUJKnpCbtmH/EWQtyADjr2q1XYBlUtxzk9K70ibY1smSGyaafdlvylPTuaFaNLrLRN4+PMB\njBFZehfgL4skAZrmNpkJ/wC2+RitXprRT6eIrWRII5IHUIzg4Y/X/nNS5h+MUxB7mKM3Ug3wMRtQ\nYBX/AIKHfSkLtsCk9fMG5/etFiyw6FSCPE0a5IxnnFVRacXkPhLEwHTGc/QClkr0T1E51kijVZos\nMw568UoukgiuiSm4uMbV9a0RRiJhHaKikKQoHNJ7t8ykGYKe3B/xRoZIJ0eBZbxDvDIwIY//AHT+\n3eKEsqkAA8ZPamoLLDqkCk5ky2ecDoKvn1JXhVgoKk5Ug8UlBQPda5cSR+FEYwvTcv8ASl8N2/gt\nFJKzKDwpHQ/WniqAUQ5S8iZZFGxwxzXBxkK4IY5AHUUbGOq/nIPHvXBMqvkspB4z6URRrbZjTJYE\nHpirS5c8ttTIBY9s0Gxki2GIzFo7eIPIj43OMZrhtNStLt1a5tLOQMB5pByMZ4FTUrHaoSyapdSn\ncbh/Ty8D9qjEwY52k5OefWrRVEm2Go+DjNFxJuGcE04pciY65Gfer09unrRAWqc8E13PP0omCYWy\nPQVaCxpkKy9FI681In0xRMRZvLVBJJ61gFicCg9VvhZWLyZXIB4Y9aAT5FfT+PcyOSDubNCHrWCd\nHJqQ54HagY4OtSJrGCNP/wC+ucgZ6ivr/wAPTAWEZLliVHLdaohJD5JMjNERS8jNKwxDIbnYwxzm\nm1rcZxjvUpIqhik+0DPBomKXIpBi9W4qYNOmIzteogPVyiY+dfH3wnFqfxNDeNM0DNBtDCMspIPc\n59MV8++NNMtdGOnwxeNLMyeM7yAbftXFyv8AOh/Yi3SddOkXlpehI5lRzuiYdV71vtU11L3WEurY\nYjurYxjcCBk8j+1R5Its3E/gs0a4RLpnvbXbKY2LvKuRtz1GeM/pUZbS2+JLaSSyUWi2zsiziTgZ\n55HH149anCDTtnamvQF/g9j/ANvVEmmJJYmNsZ+uc/tTbSHmhHh39vF+BlV3crn2J6euKs2mZ/4W\nahD89B54hID5mCdTj7YPpQEcVw1vJ80ptLdAXBOPKn3xk9eKSjE5bq3gtUjggNxBnJEy5Lr2wMcV\ny91C/uYNlvfGxCJuWBIfCz9x0o+DA/yeq2S6lNHNcRLcWolhd5Msq7hnn9aSXev3mmiGWGcSloiO\nWJweeee9UjrJzxWjOaXZz6pqgVfEZm8zMAT+tNIb17a6jkt5V3wk7XYAEnuT7VWVXRCK/GwLWJD/\nABOZywZmbcSORnrxVVk+2UdftWWLDlm7ZtdPnjvxHatgyRIZYiZfDIOOTnoenT60NFa3et2kxkzE\nEj2W67zuI/Xv71M7o7FFWkTxyhtHu7g2wG50d1IZHwBgn/aRn9KdQfDuzTXaWPexbw50Zt6v3UgZ\n6H+1B18CsYNo+hx6b8Qx/LREwYYu6nCjsAfTnmvUst9JTSbMyzqw4ToeTXlXxJljjRmL9Mdq7rwk\nG3EbINoBCIoUcfr+9COWwVXdnqCvY0FpiKy3TuNzF1x+VlyP1rSwxWFv8N+NIkQnYlRg9QTzjPep\n8nwrD/RHZXsUzssEyjauTuQ1ausRmc5KnIAwAfpms0I0WtIkoLZHHY1yJl3ZI2j+1K0IsOnUYyoh\njCk8jDL2oOfTxJMZEVuDxwc5rJBqwSVCoLLliBnGKCuLNXO64SZS5IyCMcUzQ1B9otta2ce0Mzud\nxUnG0Zo5oG8NUAVMjK7OTj3zWZgYWrWx3zMA3pjrXYrjB7ADjFYIBPF4chYA4Y5JqUNx5igHXtmm\nFLZIjDG8h5O0bRj1quF9gCebjgYFAYshIZf+oDRNyFXHU+9dtwJ7jZt289B0rMUepbN4O2NGYgZY\ngdB6/SqZLyG307ww2ZZXy5AzhR0H170j3wqsBJviA7GKx5dhgueDSp5/E5fzcY9apCFCykzysXb0\nAFG2w8vvVESYbFDlgaPiIC4ogLdwB681dHjqTWMTxn6VwcN3IomC4yDiiEIJ6UUAtJFRZ8DAogKn\neuA56mhZiedq1jfjPUUEPhA7nPYdq1hMCzZPb7VHODWMdz6V6sY5j0ruaxgvThuuVXJGTxX1nQk/\n6WMbRwOuadCSHyNjirBJ+lBmRfC+CMU0tZSSOtTZVDOKU8Dk0xhbnnNTY4VG3FWA80UxWTrtOKQJ\n5qQ5rGMR/qvd3Ol/DsV/aMFeKXYxIB8rfX3Ar4lqOtz6lGq3TtIUGxWYknHpXNyx/Kw9qQDAdtwg\nb14B+9fS/hnR7XV9Itprq6nAiUcJgbWTpg/Sp8rqNm4pfkd1L4dtL+8L2l7PFljuy27yHtmiILYa\nVZT2Omwhiz7xuckSHHJ/btXOuXsjtjH9kJ7zbI8e0IVUNgDYVbqex4oSO1b5SSa8mK24y5LKQrE+\ng7n35oooW23yk0X/AEUEy4AJkdiQO/Tn/ho61EN7drDNbyy5BcIMgcED6YyRWsxTfac0mmm/tZfA\nhaQI0cw2k89QemOfSh001rrXp/HuJhKij8kRkwvUcgUbpWI5UBT2d2LW9drmRbS3Tapk43DIATPv\n1rDandx3lzEIgTGAOCACTVuJNi8kvxxDrRI5LS2upLK5EN3I4h8ARly4I7HP1z34oXWTZ20UDWKx\nyKcq4ySSR1PPvTP+xJf0sR3CtKS+0jcc4q3Srdp76NFxkk9TgfemfhytW0fQ7axaO1bwI4ivhjO9\nc5J6ge/auNaIG5kS2gZgSwJYqc9MCoO2ehFJIJuZbGaUyG1Etyn5bh+MDGP+CqNOuF0y4Mdra5Mm\nHZImO87ec47VNN3QrB9V+Iba8kZbmaSFiAWVPU8/rXqqov6TTRnILGS7fAhmTPCjb5T9T2ohWuLT\nbDFaPgnzTMnb2rq7WSo46BpCZtpBPPXpVMd80crLHC0cZbGQT09axqGUVnfSCOWJGKM+C23I9T+1\nXazvljjitvDUhdyEjgHPX64qEnuFUqQue2aJdrkFwoBZT1NVQo0URR2chiQemcVXsIVufAiDJl9x\nxhjyK4mzGZZpAWOQFPakcqEYbaywg/8AbG1upbkmn+lXNnaN4qBgXGNshDcjkUVNAK1ksJDc7Yo4\no7li7Hw9zc9cZ6fas1qFlppkLRNOuP5T0+3eq9ovwFsHDoreXK7eTuIzV0k29jIijLDGDU2OiovL\nIhEnDE55OasttqjL5PP2rBI3KlmIx5TyDQKwNHud8DH71rMERSOUVckD1auoVxhQuc5yRzTox2Vv\nOrMGwOqnvV9tEAyyREkvyQe1K2arCbiW6h2lWdPEygxwG6cUBc2F2t89q8TLMgJZSegxnNImrKU6\nF8quE3EHPfNUxEsecV0IiwiMDdjkn2phE6qvPB7e9FADIZTiiUc9vvRAWocsMdPeiUzWMWbjXs5a\nijBUI470Qo29eaIDrMAMelVmbJ4omI7uakAOtAxVcT7VPfjNfOvie5W4uCcAY4z60DGc79vtXe3N\nExEHHSvCsYl2rhNYwbpgBu48kAZ5zX1fRpVWBVyvA6A08RGNhNkccV0SkdST9qDCgmCUgc0wgn5A\nXIpGOmMreY5AJNN7eQ8ZOamx0MEbkVchoIzLBXqcQiw5ry1gmd/1DsE1H4Jv4ZN2NgbyjJ4Oa+A6\nNocmoaja2qsXE8m07TyB369wKnyMD8ALm3ktr1onBWSF9jA9iDWr0HWpodMkt1wmGyhU+vXI71y8\n67cYeLJDm/t7+SH5u2jeS2aIOGAztPcH75ppp1rb61FYzrMLe8hBVwv5ZB2zioQhSo6HNqRy5s5o\nrs2st60DgGQiNfMR32nORSP+HvDcCYGW9CycQyjPHY9aZYjpi+x20vdYvbiWMyNapFjCflGD9KIu\ntSvtIuRFbhXbb067h6g/5rdbM8BbnUBrdzD8/bzi0iHhmKIlAz5OMZPPUU5kaeTU7iCB2laCX8OO\nKQFsD2parw55NXpT8VGdngjvJUVJl8SWF9xwOMcAHB/xWY1GTS5opYntjAq+a3ulG45HQMoxgGqc\nbkUtdRXbNI90t1ZHaYgCX2kgMfoPril1zN48u5lkB5DB2ySe5+9dMmjibaVGk0SwtNY+GZd0Je+t\n5sRlDyUPJz+9c0zTIrO4lBi8SbIRlxkJzyc1NuysIp6aYWUskkfjygQb9qK7BcHHUcVK+tbma5Sx\ninwrKCq4I3c5PmxjvU7OgU6pZT2sztvi32cfjPDuOcA4yc9+4HegpPiOcWrS2kkZeaLZNKYxvBPY\nHsMY6VRaTkIoVPzGx2QHcCzueB9TXqLkTodT6jd/IMxupwrkAlnPI5oNdWugqhruZlJwoZzVUkjN\nl4uppJMTouBggsOT96MjS2kANwDgN+VWwT/4rS8MgoRXFnp67biXdL5kRDwB70EsLu535yx43Hj9\nKgnY8mcu1jCk4ZjkDygDFBqSeOretayZTtdJSdoORjnvXJbctIJNjIu3BGOBWfgjOorbvwwdpHQG\niLQN83ET2bvXJNu8AQbxA3U5qxWlCPtVWYr5cimhNxBYmubW4aQGYKHxyVwOK9cOYU2gNkgck8V1\nxl2GTJ2sokgwevWpzM6KyJjJXcM/WqJWE9JLMbmBXUFHhVnwOh6V64jZs5BC4xnp96HgwS1qtjaW\nztNl7nJETdQAcAj6/wBqgIlnViFwQSAfWinhkgrT9LlvJUjMcqow3bmXI9OtaGHTljHg29t1GMfb\n1pHKikYiO81TwF8COPLxEkFiCqnvSm9129nMoknc+Jnd2zRjFPTSk1gALuTwSA5wR0qnxfKp7mul\nEGEQ53hjR6gFPeiKFw42eaiFl48o+9YxfG5Hbr3oqN8qaxie7jmvK/PNEwTG3SiFmxRAVSXAORyT\n7CuK/HmrGPbwCMmpO+1fWgFAV1Ltic98V831h8ztg9+poGFY6100TEe9SFEx6vYx1rAC7Bl8dd3T\nNfTNBZRartGKdCsdK3HWrY+1BmQVHjFExPtYUrGGFvc4YZJ4pzZ3W5c5yfpU5DxGcNwTjIotHB6U\nozLc8V3dTCM8TXqNmBtRtVv9PuLV+BNEyZB9QR/evzLOlxo2tSW0heOW3lKnHDAg4zntSciA/Cu6\n0++e+DsPEWdiyuTyx9PrW10b4WvpdEhdYLCQeYPHI2JA3sw747Gubm/rRThX1jeSyvIpUCK8ws2X\nfbAho9jcHGMc9aTatBeaFqc6RJcFUO6OSNS2E6jntiuJzaOjrFg/8eurtg1xJ4pA2nxFGQPfHIow\nJbXGnNstt87uBsUkcZ/MOaguSUJ1LwpFuqRVrUdvBczRW9vBF4aASEIMyNjjB/5zUdOs4WsYnmhl\nMgX8pbzZyeee1dPLyx4422FyL7zTXa1HhKjuzguWZgFA/wBoz1phNpen22mxX0cUUJjG15lJRgfU\nkc8/1rzP/wDRLli/+b0i2k9I6vexappOnOqEXKK252/MV9/60pvLGxlia41BHOVI8j7f/FXf8iUe\nRRQF4wL4Tj0210e+nvpJV/EHlQ8qgyQx+/FZKYK0rshypYkHGO9enFt6R5ZKkMNEvJrKdzbsFkkX\naCWwPvWv0m2e5uUuZn8O0zm4AGN+OcfUnpTG4JDW1aO4uBeMXkiZ/JbJz4jZ459AP6Va1yb1idhW\nSEkLE44z36UDquwD4ndbj4nhR5NsN3a/LyIuPOzghR6de/ald58GS6PEBcyQyKrksseeD07e1CUq\nRKboBVbe33K1vHsxg5GMj616o9ORkf8AocntU/hEySEt+IhBXkKMMP8AFUWlv4MYUqrFsHLkf07V\n6CY7Qc94bSJm8MMuMbhEr4/XNeke6TT0lV1dnkDAIgjwOmDx70JPAoNvb3TUEYN5IH8qYRThG7jP\nvQUzq0jFCSAeCetc8TNgiTeIGBAIU55rkfmfMYLHsBTUKWs4gO+QDHpQb3CSynb4uSOdxwP0rN4B\noO0/Srm4ZHEMixnguw2DHrk1dcqsbbVaMiPIABySfXNc6N1KmhhtrZVkO+fAJAPlQHnn14oCa+RE\nIU4Y9Aafr28EaFlzOzyZbPTvV7wGezSXykYKkA45HP8ASrRj1CiMVqvhPNbyLKFGGi6Ff816Ccyy\nDMZyq8Z/51qqoJpk0hbz4Te5giMk8OCcDnHf9qV20EkhUPF+EPMiydxXJDk72mVUcsFubSSZ0kkn\nUkHyAfy+wphFCluSWKuNucY6Gqxl8AloxgUQpE0tycg8x4IwDyP1qi9vJ5SrxM0ICbSoPeqxSYJN\niC4UL0GSRyfWllwcZJqgLwBVuSPWrQBvX0qiJsOi5x7UUJMkjpimAGW6s4yQdooqM4NYxaD7/arV\nlxxWMW7813dlx7UDBcbelT3/AKUTEC+AQOBUBJj+asYmjfQ1xmOTQZgO9bELfSvnOsOGvmUdAayM\nADrXSaJiBPPapUTHc17OeDWAX2wBlHXg9q+k6C3/AEiDtgZ55pkBoex8YJPFXq3PFAwRHUbzVLbT\nLUz3T7Ix371jGTvP9UvCYraQYGfKzmuad/rFdQXO6eBGjPULSNWUTPoPwv8A6laZrlyIS4hk4A3d\n8/3reRNzzU2qG9CN2a9mtYtHgamOlFMx7OK+J/60aEbT4httUjQCG7TZJjpvGOv1GP0oy1AFWiyb\n4GD+EyIu78U4K/T0quz1KYz3yRXohV8HyDI/KBke/Fck1pbidId2Ot3UHw8L1CbpIYWy8qYEmGz1\n65HNNdM+Mmu/gnU7+6x8yW8NdvQZyFHvUHELnRVcx6Nr+paRZSo8F9fW6yyTQeTqvTHQkn27UWfg\nbUrCEpC8N7EgLR5BSUHsM1CXHeFITVGI1jWpY53sWhEF1FLtk3r0wevvzTwsTksrZIxmuT+RxOMV\n/wDol1I691J4DFSqMq7gZASAP81mNMlj1OZ21Y3UtvMOSrFVG0+mea38PjUe0ngk3eD7UdujoZNp\nmtY4wcltvB/L5vWsbqOq3WogLOdqjpGDgDPU10fxf46m3N/sE50qQTDMbjRZfCjEKlvDZyd3iEDO\nPalQXgV3RjVkJqqCbCJ576CGEkPJIFXHqcCvol3p189uun2MwVVceK48xbsSfQAcVm6H4HVl8wdI\nxmBhbQgLbpCcHI5yfYnmlKW90wmubyXZHI26OJHw7E9R7ChF2dKbK7mGwuNQsr2/BQQvtZN5KbAe\nvHOc+/NM7vWLR5N9hcfMRHA2FT5f1rRTlNJkueSozt1M0twZbqNFgyVVFGfua9XYcieaXaVzNKYg\nu10O4YOMgZz+x/WoXniTRNIs/l6GJVxg+1TbO4haCS2ETbCfMN2QDuHpT3UEh2D5gMgUAomwZHOe\nvfrUpt/DUKpPDs3ZVCup52tznOD+vvQUk0MjFoozAeBh23UUKyAgk35VWIzjhc0OGntrpY7Yiadi\nRwMBfY03wyGltaX80iKUQSEZYsnlX3JomYwQvFKsUc8keVkkZMKT22juPc1FuxqOTahNNCys0m5j\n0yCAPag7crFdiZlLCLMjK3Ty84+9TbAC3enXlzG98kJYuDJyRjGeeKzrFjqCOWyjIeCOh7VfioSR\ncG2yf7u3PNEzXdpDY7LgNycgIRkH6VVeixLJrKN9KtrnTWknmeQo4C4+gHv/AOKdWFja2cQPxA0c\ncgAAjh80h/8AkAMCpylSLRVM0VlrS2Vu00MUYtUIXbu25B/KMEfqaE1Oyje7ZtPTyy4cBmyTu/sM\nH9a44um2OtAZLBVUBCxBYgZ7VQyrB5rtpBEV4EajOfqapB2CSojHerfXHlDhUUDBOelV3r44713Q\nWEW7E87dckUsu/ymnMLTjxM1YjZIC9R0qiEYdHxt9+pomJt3PTacCiKG20xZSBkBetFxg4z61jFq\nmpBsnnrQCXDP2q5Ov0rIxajljxirD+Xg8miYrY84rg5fFYxdkKMVW7celBhAb1isD7sc8183vm33\nUh9WNZGBxXe3NEB4Ad69RMcrx4rACbMjxlz0zX0jQ/D+VVY+cd6ZAY+U8YJq2IfU+9AB68vYdPtX\nnuGCog6k9/SvlXxN8RT6vdkltsSt5UB4IrMZCAylieTXA9KELsrx7edJEdlZGDAj1Ffoz/TX4tl+\nJvh0zXBUTQSGN/0yD+lTkUgbJJs96IVwRU0wtEl61aDToRniMis58d/D/wD6j+Fbm1UDxkHixE9m\nH+Rmm9AfDLS7Nq0kasjovDxyfzYobUFnki+dgSNrSM8lF5jPo2Occ9ahJUNB2qL7XWFk0ySxjfyg\nkAfynPcfaqmljh01UW5dmIO+IEbQQeO/PFTpgkg34Rv7q3+JLW8Te0kTCNA4J4xjv7V9hsPja2nu\nRZypIHbq+0YT61PtTDxq0YT/AFSudOvru2SxsZPFWXdJemIgNn+UNjnnFXfCl9a37LbrbukyodxY\n7gex6/2p+qpAk7Y0bRDZiXwpvEty28pLyUXHIyfvXzVLwaTDqECSiKcznaoTJZc9Qe1LLjUlXwW6\nZSmr3k9i9nNdMLdmDFT6jpRGiadZ3fzZv5RFtQeCxcKN2ffrTuPSH4mjcpaPL2e3tPh+5t/DtVeV\nFZXi6/mHXtn+1ZW4tjbybCQeAQR7jNT4k6tlOdbgV8P3TWWv2k8Sxl0fOJDx0PP2raaVcR2Wpg3l\n2jfO7nVEzjk8Entk0zRPiWDDWriGWLa3lud+H2nKgVmLy6ihkCqQFbIUt7VGKldHWmlFtiUapLvl\njcIyN0BXp+lW2V1IjK4Yg5y4PpXao07RxTfb0JudQhI4IA7cV6nIMdRfK6fO6bzIzqSNvQZrsOxr\nG6uXMSRRtuLlSeT0GPf2rn7HpgMepxyh5Eh8TYudwQJu9PpT0TtKtr4lufxRtz/MD2wPrgVCPI3j\nBDWIZNUuL2KS2n00btxIkmTw9p9iagNGluG3+LDGAAxbJbA//ZzmmXJ+VDThQdbWdsYJIy04w2J7\nhcbQD+UAdRn1qq20+2idpII5T4efzOOnr71XRG0XXupNbbdqkRsSipjeW9zS1Z7ie7jWSUBXPnfG\nMAdMCl6gshcXsUBkwxZQSA6LwaW/OrdWlw0RJVl27vQ5/wDFTnFpWZPRtpTXB0SCOSVY4/EZBzli\ntJdS0NoZDJCZXgVyu4oR9/pVONpIWSAFeR51hS3kkY90U/0pva/BF1qDNdX0hsLKP80txGQOPQVV\nyo0IjVb/AE/SdFurLRoXlmyCJ5CCJOoJUZ47dPSrNC0b+JTRS3oWdyDJIgbY3Toc9eKjNfiUej34\nm17T7aKLSbaCNUePbMGG1oiOgP8AWs5bXywXEcpnkcqoUDIIIFDj4bVkv+vTAk3wmUszfhjqidSf\nSkV8Ll1MjKAm4nBPaqf8nB2N/wBVPArTngS0Hhk7z+YChb2R93l5+tdC8ESAGR2GZNv2oG74BxRQ\nWKnb8TmrICNwHTJzmqImxijeQ47VdC3nC9utEAbauMsexNGRtlckgCiZFyDdVq4BoBLd3qRipp0o\nGLoyBUi2OlExE5JzUQcMSO9AJMtxVe7mgYX6vJ4dpIwUt5egr5zO34jE9zRRiC81LqKID2PWvE8U\nTHB0rnasAvtG2yjNfSPh5y1qNmAKJjRRKM0SvHSsKZL/AFDmKaVFGRlXbkg4IP8Aevmcjktyf0oD\nIqPWug0Ak0PNfcP9B42l0HUiy8eOAD68f/VTn4UgfTIp9km09uKNRwRxUrKFkc65xmiVbPSnTEaJ\nA8Vw47898U60Q+Jf6ifActlrdxqVkkkkUh8RYo48gf7vp3NYqLVpNK027hiXi7G2VmH8u08f/vVy\n23Khq6+GcvNSbPhwgIinAx6VVZXbpOrhiCD1rqUVQrZtNL1HVG1jTybpPl3nUEyAEqMjPP3NNNfv\nLuLVblGKxoZSYlXjy54I/wDNcHJOKnQI/wCHZZdWvrT5e5uozCcSKJMEnHo1LYJH0i+ULebpJMgi\nM42DtzVFL4HqaK7+OIr3Sp7M2MpunjMJORtyRgkHrn7Vhb+2e3mKtk5AbkY7DrRQs1hRG/OOlPNL\n1mKytHgksLW5ErZZ5lJOPQYNCfgsJdXYROtxrLNDpltH4f5nRVClfp7UTq2kvNNHbpB4U1vAviZO\nd7YHek45JKjoku2izTLC1mLvcXHgtCdzRtGT5e5opZfmrxyZGAZgsQHGAvr9qrESMaRM30tney77\nqNITjw16kn3obUJHu3zEgmUefeG6etbrtjOeUBeH4qxtKwhwvLbaF1TU1t5RHbr5B/N0Jq0SBTb6\nhFckKZCj/wC1+9erNE2jdXttdFS+3MhPOQOB3PFEwT+HosizlSHb8pGOMVyfcPRoSmO3bcsUcjMz\nrug3YBHWml/eXlnawTRrJChPAY7sqcj82e39qDSQYxpgWopf6gbV9OtkuHux+MRGAQ68E5OMA5Bo\nnT/h3WL/AFREF5ZieELiN5lLkY4xjP70UkwTHVqj2VtcpfDZcrj5hDyRnGBx1Hf71RfWsnhvKikJ\nboTlRjdn0NOmTeiO6inM9vLPCYkYZG7jOarhAvL0QbVTgbirjIXv96KYGqJXvw+blQsNxtRWIAz2\n7ULa/C93BG8RZCHO8FTx6c1uTVRNTX0caDp8MVuRexgTRSkqSM4BFNLjTYPknWGR0ZxkPuLDPpg1\nzNMqppiR5G0O1RpbpvFXd4aRR4OT15A9KV3Ot3GpzGO7vHkgBBXfudT9avCmrGb+IfaGdPsX8S1j\nWV2cIm5SSAACSB9Tjp2oy41VbiZzZWjRzozeLJK+zggjABx07VHkklshfTI6uJm1CWW8kV5pDyxw\nGJ98cUALuMSssgZcEAEd66+J2rXhzzQTZ6gviAHIViV3Z6VfLcqy4RlPPcZq4sVpbAEit2Jxk8kg\nUFK+9jnj3pSyZWVwpzS+5QM2Nyj6msYWXlqkMQbx1aRj+Rew9aoiyG4OBVEIwxZfT70RHLtKk/ei\ngBkE/Ug49KOjlDgdKIEEpJt+/pViv5vagxglcY56VIt5eKBiyJsR81zeWbjGKwSzdhaivSsY9I2M\nCqd9AIo124KWcnuMdawkn5jRQp5RUgKJjhrnasY5muZ5ogLYD5h719L+HCFsUGKJjRQkGr+lAAp+\nI7G21LR5ornG4KWjb/aa+OzIUkZSc7TjNZhRXXhQCWLzX6C/0aVbH4DR/wCead3I/b+1T5PCkPTU\nzXS/NIh430zRiCPpUCp0OQ27jPpRMN0SfMMUUwNWFrIMcUPd38dsh3Hk1VMk0fJv9WPjKeHT0srV\n3iadssynGQK+OT3byKctn+lFL9msAZs1dagvKFHc4pxDeaXA9y9pHZgz3ELiQJt64wSP6U91OLT9\nYMiXE09nemRiUMO8R8/l45ry58SlyFIx2zJzWl5bOSZIXjDFFCjJPf7VAzu11vdfCZTgAGrqOhni\nHek3/i3F42oQm4hMP5o0wVYdCcYB++aAlsZbyCJbZPFmyYxGq84Azn9j+lB/izncrB5tCurXTJL2\n4AgjBxGrjzNz6URp+kC8heU3CwQoBmSRTj9ql/07p0GMdCLFJLK8VEuF/EXKmMMNw++K0thpM17M\nqTXOVnDMhVhvDf8AuHTFJqlVFezWGc1MzaZePa3YKuCQSOMjt9vrVUUouHhwu3GFwa7IrDOVg2Ha\nRzJGrLklNw5qmCCOCQskjoSfyg5FOBlk0rTFYsFuoAx7f5pRdWMxnX5lHjB/mZeKKdC0/hFUgskD\nMGlPU8dK9TaxGj67dquouVgZZHllYnD7cL2AxQsukQQzhrqRmDREKsmQA3rnPSvPUzp45YV25025\nglktLV47m2UtmVztZenl5o+zuLDUZILTdGZ1BfZIgAC8AYz1Oa3oZSY7uNAvruzilgE0JYcgvwuA\nRyPQ8Vm5r6805Y21G3tpdjFVuo5AWOM9B9qdxNdgsvxVJaSs6yQTxLjaFTz/AHz96nDr0d3CTcFY\nCX8hMe7j/wB2OgzTqLMgYfFdrczeFqlisgVtiSxjLZHoD2IqF0uk6gpu7Z47K6D7VVxsVV9fL9O/\nrR6tBZCyt5rTWZUnuElDKHBQ+RhjsfWnaKDHuGMGizlmVyHapJ71WWL7o1crlaSgRdMUXt1cxI3h\nyDdjbyBnH1pFJeXXiH8dRk84OCf0oxVYi6dkGv3GFW+khlRsgrIeft1pjb6xrdu8Ud7ItxayDBRl\nEhxj16g0z401pm6KlMM83kjyMkjxfMRQtwiLMN4QgHOMUV+OEnoOtsCxMK85zhiOlcjE0bqsqBFJ\nzgHt6/SrxdmQ1jtmW3DSNkkdjwaAnCBhhiTn1ojo5LzHyePWlFypLnJ6VggEo83tXCE2DaSGp0Kz\n0cmGwe9ELLuxjiiKwqF8t6Cj47jYpwPvRAFRzDbnOOOKviZj5scUGFBKSk8sQPSr85FAJKNqnn04\nrBO7vWuFv1rGK5m96q3DbSsKEevTf9KwGP0rHOfNRj4Bkl6V3NMAiWqOeaJjvWomsYsgY+IAK+kf\nDrf9EuawDRxMPvV+/jFYwo1u58KykO0Pxjae9fK7lN9wx24BPSg2Epa3I5qk4HFAJbbRPNMiINzO\n2FGOp9K+/wDwzHLomg2thMFzHH5mB5BJz/ep8jKQGULm4vouTgtzWoSNt27dwB0qSKE98ZBIZf1q\nxCOueKBgqM8YpB8Tq6KHGcY6jtTxFZ8Q/wBRZDcSROWJMZ281g2bsO3FXIlZ60y0y2kkcOkbMFyT\nitdaA+jfC2jXul2M2qK4WUQO6IRg8D1pReavNuhnaYTXEX5pFOSc9ea82DXJyNo6FcY0w1by1jKS\ntcQDxUy3rz1pKBFPfuqzIq5ypNXSp0xZ11wdfBWsafbT3NnqLxIsrAo0q5VuoI/vVNpeSaD8RMGk\ninjyTG8bAjBBwQevtSShrOcP+K9Vt9Tg099PdSqxh7g7hlXb+Xb1OMHoOhrPaz8RIdLj0iCEqkMm\n95t2PEPbjnihx8XVdQ3tnNEeSzxqMlu7QRINw4zgnGRke9MNB1aWfX7fw7iSONpxtSRs8E0ZzUrp\neDdrCvjC9+Z+KLhrZsxSRoHG3Iz361mxqE1hc4Viygg+b34qvGvxBY3kguJIVdUd4x1YDyj70pYn\nfkcYPamM3gRbwzvaT3ik/wDTMoYhhldx4OO4yP3phaTy3mm7bl1XYx2M4yGz/TBzStBTaKby1j0p\no11RQqXMR2+GAXUHocf2r1PGVoSVpmnOqSXSFdCZbaG0XMsjxhWfJ49f60Xp15OlmovfBureZ2MK\n7PMMcHnHf+1cMWVg7wDmsrRbd5l3RyGbKRHI69ce1ItRWC33TN+dn68nA/4KVv8AJUaTd0fV/hm6\ni1f4fQPM7oPwwVdgVx755rBfGGhWdnqUnyN6hjGGWBpMOnHcE1eN0mwxVMz8FuyRMGKru6ksD79q\nnAouphGZDHvG05XIUdqsOcuYLczusF8JhGFVJGQruY/X06ZqkRSIgZ18RTxx/Kc+/WlTC0OdM1Fk\nJWdi6YAAY5x9KeQ3O+LI/KetBo5pIHe8E10bVFJc/wA2Dirdhix7etCiYr+ILV205pYwCAcNnjFZ\neGykJLtJEm0Z5f8AtTwjY8XhB41jmDGdEDjjeM5+ldt0kMgWKQMM8AEED6VesGsLeN4mx0xUfGjQ\n5lUygdFDYrnkZA/iLywA9gP6UXczxfLoN7IQuODnHtRhKjUz1tG0ceWd3VuRk0PdDafKKsnYUShu\nitjJbuqkOwbPcUtvF55FFBFkxxwOveoqM7QM49aZMzKj5JSD3PWrhJnp2NMhGXiXC88Yq6OcnAzk\nZogGEM3mO48DoKMjmwANwrGL45OveilkOKUZE4Xy+DmrJX2nrQCeWXjmuGTnPTisYrlkyB3qiSTa\nmM80GMhNrH/6sT1zWVkBDUYisl2qJNOAiTXsVjHM13AxWMSi4kBr6H8OSZslPNYxo43A+tWbjjrQ\nswp1mOS5hKR4z71irzSzC3PJz2oNhSA5IQqnIxS5ow8np70EwtGm+FdLZb5L5mUw2p34zyx7celb\n5dYutQl3LIUyOnqe9JNWPHDUfC9s8l8ks825T0Wth4wi4OCDzzU6HsCZofmAIFXBOWNHAZAxSsIV\nH0qu+tlurZo2A5FMmKz4P/qVoktqzv4bsAfzL0X6jvXzNl85GKunZJqiyO0lkcBY2JNbL4e0v5YA\nPIof8xX19qnyyqLFXpqtY1uF7JdNtmMTsgaSQdfpzWHiVpNRNoi5LHCsDkHHc1yfxePpFtl+SVj6\n1+HNMSyJ1GR1nc4Hhfyj3rOXMIhuWWJg4U4Ukdap/wBLlRzshJYRtciSKRlQYLBvze+MVfK48bCj\nHpkc1V6Z+FUjlHC85z9qncW6TWedh8UEAMD17YoeeAWH1P4m0jZ/pyqLsiLRW6g9zgAkD9BXzW20\njUobxJILV2MbbgzHb0PvXPwqrQ/wP1SO6PiXEyCNmO4jd09qQzqXO4jd6mulNeAaL4XvIrVtpYRd\nwG616BxcybFHI5PNFoUbaAgupryyRQZ7iP8AB3D8xXkr7ZHQ+uKLgt7PTkFzK009sbb5hkZdoVt2\nFT65Bz7VKd/C3El6yem3+jzyHVNUkEVyHLxRt5lUnuB6V6jBtIly25YMtQe50+5KRTJJHeSbixj/\nADKOMimOiRq8Re+jXw1JSAuShUc8j161xOLXgsZUwWGxdbOeK+lCeGS6Stkkr04x0pPqFtYiCNpx\nNN4cYZ3WTYJXyQFxg9uuO2aMHuHSkn6Nvg3XWhkubXBSBw00Y6+GFHP14/pSbXtWj1bVZbzzHxFA\n5TBXtXWkZqtFMkqklSxxxjIqSSPE7Ojo23+YdqsgWGLpk1+iwRzgXEmAFYZPtyO1XSLHpdjD81ci\nW48V4Zol4aJs5GPbGOanP/BkwPxVLlQGGGypLU4tdXjRo08RC7HHHrWSwjJ6GaVNG1658Tw97HKv\nyB96ZzPbOuBdQkjsHBNBok9FeszpBp0qyOuGH5SetYqacLLuRSirwBjkU8FhkqITq19HCEJG0kEG\nqtOSSK45QqAfzYxT4hkN5naUcnNVx28kwOFOPUmoTkl6NFNhtpZW6l/mcycDAVsYqm+WCIbYVKkn\n1zxXMpNypHTSSIW8nlILDBPGapnHiE7eecDFd8fCL9B5o5LOXEwUN2XPNUPL4jEA/XPamMByW3n5\nzVF0hgAHODWTAwItk4NWI+KohCwv9816OUq/XNEwwimYMCelMLdsrls/WsAKV8Y60ZG+V5pWMicT\n4k61bI25hmgEirgucjgVGWTnjpQCQZvLVDHiswgGpf8A6sRxzWYmTD1kBlZ4qPaqCnK9isY4eO1c\nrGJx9RW7+FpP+ix3oMxpojmrmfCUAgkh3n2FIdSRdx96WXg0RJNbF9wAzmhI9Dnnl4ZQM96EQyNf\npGmGGJFUnjqfWtXo2nxvLumYjHRVHWiwJmp0/bbTIy54H2o+4u1lBTDAnqRU2OjtueenApnBKpwp\nOKQcMGAvBquSYeuKwDK/FmlR6tZSIQCWXAr4Ld/DtxZ6xJbNG3D9QM1aPhKXofdSHT9i/LszbevA\nqFtfTPeoxDwRq2SVTNZpP0nRLVTEuqK9nNJOhGXZxjn0qW555EjhtmhYEZZep+lK6SHovmuDBKYp\n2IbPAagpY/En2gjceCAa5VHbEaDTC0aBmjCY4J4pZdzMt1vPmBqkZaFrCmCZZLjMikqD0U80wZlt\nlDPhoyfLt9x0p37QtH0L+Of+ovga0ikUxRpGIy45864B+h4zWWt7y4XcJHyR5SHbdyD71GD1oaxZ\nq13deOr/AJ0B6A9T9KUG5n8VwQoU8lfSuhRXprLvGzbkHpjtULRfCcyIcEdqZily6nPp8rMrFSD2\n64rQwfEkfyq/xG1MsLnbh1yM8H6d80vgOr+AmvaRbTxxvBPb2enDEokZSWIY8Dp+1eodjXXppL+4\nlhv3s5FheMHdC4H5Fzng+45qy00xporiQz4CvnBbHGAcZ+g/WuNyrAddDbWfdBtffIHC+RgPynsa\nRavpMl8jS6YVllUu9xbq3nVQOuP8etbjdF4yFXw5clZL+Y7lWK2fA/8AlhMe3X9qBVpZJw0jMykk\nHHtXXGVjPwHnhdycvsBPGa9BZpOWTK5bq7dBinEshZXi29+2xCxAKh4ic59a7I3zCMHQynrljhx6\n/WsGyu0neIOzKHX09qtuNzRK8I6cjA5FChGM9Llmv4NssrKX4ZtuDVN7ax6M8cCKXaVsiU/mPooF\nCUU0bqP4dEutStlS/jCFVyrMR/aql+FZHnASdRz3XP3rmjyNYBoX6hYw6ZebGmL7TlsDvUH1aJk+\nXiiJ3n82BTtOQYo6RG7bVIAX35qmefbxExGDjBpHd0UWFEl48YBklbBxnbQ887SsdufvVlBegs5H\nHjGX+1Ws7W4G0jPXNWRgOaUyytI53MepNMljgurGHwIGEqttJXndRMKrsmJ2UghlOCpGMUtvJnlx\nvyAvStEDAs881JTzVBC4dM1HcAaxgm3n4x1PvTKGXy8HjFEwVBJkjcSaZRsCuBSsJ1W2nPeiHbCj\nNAKKUbANckfjjvQCcdwE+lDCQsCc1gg96R4fJ7elZy44c1kBg7c1GnFPV7PPFYxHNe6DiiY6h5Fa\n/wCGLtVIjx170GY10M2QKt3blxQMVSAKh4pLeeZzjmkkUiDxWoLdOtNLbTVADlTx2NaIJMZQJsXn\nOc0daXxtWJJOO2BTMWxpFrhfasEecf7jR9veSzONwC+wqUisRzacoM559aMQYYUg1hXi4TrQN1cY\nHWikBi55PEOM9azup6TDJeFyp3kZGD1qyWEm9Mj8TWMLTCO3cJcKBhZeVYfXtSmSznht4DcKqsy5\nO1sg8/4pXJBUThCM6hF2gjzHG416L5uO5jmgTBRhtZvX1pZ1RkW6tMdSvpri9CvJFGAdoCg8/wBa\nrE9tDdriE3CEBjtHK+xqUEqBLEOLhLB2QtZou8bl5PSlGoWltthjES+JKCxAJygHembSaoWP5LQP\n4bS0j1ZpJtskKqxYP/LgZ/59aY/LLcWO+82LJICSOAQO3StKP5WbtUaKdO1C60mze3s7uRIpmLSJ\nwQWxjPPtVU92xtASoL43E9M49/WnjEk2E6do0msyIglCqV3P4IMjKD2zjg4pfqWnS6XdSW8+TzhC\n42tjtkUO20ZEtKso7i4PzHiRhRlML+duwoXU9N1DTlVbqNreOdiyOwwGH+famUl9GRQLYuii5fJY\nfnXnIzWg03ULT+KtbShjZThIpImA8rBQu9fQjrQkiqdF68ar/B7hCbOC46nkhQTgn1Awa9UaRCXp\nr7vTFWz3SkpLENsm7kNgev0+1KYLn5ox2MEigSnf91BA5+tQSKv9nNVv3WcNb7R4ChXJbrjv+uar\n06K/0TWYNatlXZL5ZG35JB/4KMrUbQI6yj4gZYrDVdQsLYNHeXKoVX+UqNzEfU4/WskZsSjwmIB5\n+tW4Jdo2XnmBIkYwKchiMjkcCuztEtosjMUVvKzIucH0NXItg9sbbxw0Mm1sHcSpX/NUy2U+4sJU\nkXrjJojF8DqqhZI9zY/lBJxR9g73EoWKBlXuSOlakYdTKltaggqrqc47kUDZwpqGs29xcMzwxvkb\nSAQ3bIPahLwxppNbtBM0bsRMp8ybeQfahNT1b5a1kmhQFVG1nJ27T/muTo0BmOurxHLOXBJ9KHju\nFDhx2HBqsEzWQedppVMcm0fzY61ZFPFvxuZsHkt1qriGzsjQyD8pUY79DUC+9FBAznABOK1As5PO\nU8oYEkdu1V+JIy89PenSDZVkb/xGCrnk46D1qxLx47kJbSER7s55GaILGXxGIrhLLUI8MJ4zHIEO\ncMuOue5BH6UqaOB7d4iGEmfIxYbce9LEZiaaPwnKkg49KpU81VCMtDcV4tzxWAeVvMORTG1n3ALm\niYPjkwwI9KZW8nk9KBi0MN1Tkk8p5FKxkUCbjFT38ZoBIykbcZ4HWqFfcfYUAlF0dyEnoBWeuW3P\nTIVlGciitK099U1SCzjdI2nfYrPnA/Si3SMlbNB8QfBtpp2mPeWGqQ3Py+EuISfOr5AOMe+ayR4P\nHShF2gyVM5Xu9OKdHBpjpl00MwKmszG3028E0C4JPrTmFcjmgYhdeVDjrSspvfpSMeIba2eeTxij\nwoVcdqMVQJM4vX0rxO40QBtgAkgz0p7bqFZSOntUZDpj23YbFwc8VeZQppShXJdYU4oC4uC2aokT\nk2UxN5xmp3cAfa5HIqqRJmd1VdNuS9td2zM2Nwlj4ZT9acfDHwRY31lb3l2GkiQMEgfgHnrxXlPl\nfJyOC+HXH8VZP4l/07hu723l0tLa0CrtkTbgMPXjvWO1nQrzRrhbebwwjAskgOAwFGfbxgwBsZLK\n0uz87ax3kUmNzbT5f1Ao6O30pVkWxCtHJIxMZyrY+poQT+CyiKLmSCOQJGcR5wVY8qR6ULJHDcbi\n0yxvs2bgeg9K6oom1QB/Do7RSkAyzLy2c7qJ8V2AWRuqgc07RNgjRXMlzKkK79i7gAOW+n+KDt5F\nS6C3KnDLtII5X3p0xGjUfBt3rmnzXSaVbRz6eRvnkkGFX3z647Uy11YZdPS5SOFpH4GFyUH3zXD/\nACuXo0kVhG9BYIJtKtoNQkkhuXDD/pl8x2Y65HQivpwi0f4w+HYlniWSCXoMYZD0yPQ1xqb/ALWV\nUVZ8v+Mfg2b4OO9mEtnOdkUwGMd8H3rGNcrb3LTBS6Mw6DgV6vHLurFmjW28LhYdRilEj3ib7d88\nb15Zfr/mvVx92m0yDiPYtUvLWzuBdTQXgkyZl67DnouO2KQp8i7vIG2zgZWNBlTz+xqid6PQVZxr\n8uxeEYYgEuwGO/TvV1wWaGOMg+EV3BEOMjOfTjpVGsAloH8SyXdpFZ2thEqwrAGZWfo7+bn14xSi\n1jFxagXMcS3Cnhojwc88j1puKkhpsl4SQWM0TiMqzZLlCXHtVkfyQsQi+K0bgbgwGM/SqKWkmy3Q\nPkLK9liBMhuBjzLgA+lXarpdq0X4MIjYZBx3NOhotig2VxApCEoMVQXuI2/7rj6HFaXg6PTzStAz\nCU7x0J5NWQtJKiecrgYIPH3qMXY1BKptKlEDMpGXPXFHw+Hc77Wf/sTsVbIBwSeDRlKhG0A3tnAi\nskcZBRdv1pLcRME2Qr5j2zWjIyRGGzlSIySALj+UNzVJmQN5EIYdcnrVkwsks8jnb2HY9qKVA2Dx\ngdSTWABTHxbjapCg8ZJxTKQrb2yx/mcjoen60xhZKGl53KB0yTioJOsUgVuRnkg5/SsFGxsRp118\nKEW8E+xbtfFVmGfytyD26YrJXkyR3G2E7kHGG5xUoN27HfgukRpQz9l6nPShs+lWRM6Gr2eaJjob\nmroZNkg9KIBijnjBHFNLeUvHj0oMxeJuQKk0uRg0rGQPvwasEmBz3pQkZHLLiqdxHlrGKNQn8O32\njknvSKRvNToUh7mtP8EX38GubjVFSOVoU2CNs5boeMd+vNCfgUtEWqX0uo6hcXUnBuJWlZR0BJzQ\nZ5oxVIzOVymAdzXUkKnrWMPdB1R47yOJ2Ijb+tfRrb/tAjuKwCq5GQT+1Conn96RjoapEFRfpzUW\nPPbFMgM9ldvNV7xniswBME+1sH9aZW92fy549alJDod2NxiPk5560RJdjFIrKXgLJdZ4FQzkVVEm\nycf5hR7DMPrxVETZgtSmjh1h3kv7dh+ZQ35R2wcUVY/HclkPBdRJAjHDRH+ma8mXE3yNo6/+iSSD\nbf8A1BN7OscbSxsT0PRvvTHUNf8A4osQkvLezMOcmVAc/encHFUJ3Rm7nVC+nXU0kiXPy4ZFkiHk\nLEcHFZgXDQxM8UpZiPLkcBvf2peKLQZSTFjWnnae5uFnYkEjJBB4q+ZCpkbI8zgBRwMGuxEmeicN\nsUHaTwNtSeEO4fJ3AbW96ICaB0mEsIAljYOpOOoxzVPxAnz18Jre38NWBJ6BmYnJJH/OKVJ2bCnS\nHu5Z/wCHwTPEbny7SSqMfejNLtLvUdZjsyShaTYzDnaB1PvUOeCkwxVKzb2nwxFaX7iW/IgHl3CL\nb4nHTn/nFGfBFtaXv8TsTPIFtbslNsmMqOhrhXC3jLtrGaf4o+El+K9FjsJ7mRI0cOrLjkgEf0Nf\nFdT+GV0bVJrFrgSGJsEjvXbwt8X4snyeWGaSU0+zm8cGWK3dJo9v8pLBSAOxOc16s6bbJemh1fWL\njTNOMsMPhOrYUvHyR9DmsoAwna5m/PKC24+p9qTik5oCYbb6jNJeRzTbJXUBfMoPTpxTHTYzPqEE\nBuJUSWbJVD19R9K6PhlLSOq6zbSavNcCxjlRpMhn547dfQUfb/EWj2wCPNZxPjJUAcf+a0UCcrYo\n1zXNK1a3HydwIplbgsuFb9OtI5GRLUmUhDnpnGfp61qpiOwVpRuRoSdxHBz3pj/6kt4IEjvsyyDj\ndGOB9fWrW0isI5pxfiOybqTt9SKjNcQXIDonhoeAWHDfSk7tqmPVHRsAxHGCcYDYrsNtJK3PT/3f\n4pYgbpBRhWCMFvKAOeKgZlU8MgBGcsPvRatkU7ZHUyzXCTxqDHPFuGOgPQ/vSWALcSyKBkg4zikW\nM6KwjIJ4bgiJY+B1bnFWS6el+0SxPDHKo8yt5d5+vbr+1XixCVr8PXd6rtbIXVOdxON2Oy+ppbKj\nIzJLlHU9DT0KEWdlBKhkuioQHhehqi7wr+XOwcAE5xTGAQfE8mSFJzXlKoxUc89awTQ6NP4fwnq6\n84klgXC+gLMf2BrOXbrJcSNEu1CxKj0GeKRLWO/AcyMqFc4B6iqe9OhCJr2eKJjoqSnB4ogPoOl/\nCump8Pxale3jzIyh5BbkHwsjgEdfajNNX4Xu7ckt8tLKxVQ8x8nHBx39fvXP3lK6LKKoEsvhz5uS\n4V71IngfA3IQrKOd2ee1Rl0f/wDmWHT7aRmWdVYPIm3aCOv0rPkp0bqqwE1jSLnSJE+Y2tHIxWN1\nON2DgnHWgPGyOvFMnaA1R0S7h1qmSTB4oigN9LuwM9qAanQpO1tpr25WC2ieWV+iKMk8VvtCl0zR\n9Mu5rgNDa3tssMjKfxEkAIYD75/Sp8n6Hh+z565G47Mlc8Z64quqrwRnOhrhOTRMerw61jBdgcXk\nZ6eYV9YtD/0kfOfLQAeILt2qccWDzjJoDWFscLQ0hCj60RQeSU+oIqsSc/asEsEvGeeast7gK4wx\nB9DSMZDe31Ro1wWU+1FfPh1OGpaGslFNvbOaOjORVETZclFSvtsZT6ITnpTIVnxnUZ/mLpmjwBuO\nMNmqYLuQFUdfIcjI6iuRx/Jj3hJ8+OrqwAUcAVKW8mePDSvxxgms0IULqPgxSIoJMi7c9uDVURad\n4lIRUz0HGaKw1h8pEyxQBY05wSByaldI7LIfDZBncCf9opeyuhkBGBIYgYJg3IYgZyO1MYmLxEOA\nTngg012EnA7o7C3B3vlQVbGSe1B3Ebzr4VwngTwtuDg5/wDFb/wxdaztbSmYSK0pXCkoMipabrkl\nj8QQ3koWQFsNu9DwTU3H6BydUau78W8vwxJOxQEQcYFJALjQ9TfULGcpJNnchB24PU8VxT5XBh7Y\nab4d+OtR0ye1TVZIbmxnk2LNHndET657Ud8XPpbayZpzEWcK2IwDkY65qsJd14UbUomUl+ILKD5i\nO0tosTgDxHwRx/7TXqp0JJVgXrd5cvpQ8ZY5VlO9iD5h71m4wt1lQwJzzk8img2kIQk/AmXZk4OC\naZaFM1ncT3FwztiJlgK9mbjP2GTVHqDEqulhW1llfiONCf7Ck2i/A2p6/L4wiFrauc+LKO3qF7/0\nrRwaj6VoPwbpehoCkfzE3QyzAH9B0qPxrY6e3w9NLc2aSMo2xtjaVY980L+jeHyaa3urNztVZkC9\nVB/tTDyT28CX8dnabxgq3mlz6gdvuRVYSUlYESt9PgivFSCGMA//AJZV34+g6UNOk1pfyC7uDcDH\nkKtx+lO1aNZNL1WX8NmRx+XPej9MkvJNSit4Y47ia4YKrscBT/7vapdeqbCl2dBGv6bqOkj5y/vL\ne4ifyots+QD6VNbOO602O5jubaUmMu0fiYkXb1GKlHk77RTk4VxuirTbaS+tJobdGdVl8bxHX8uO\no+nSi9A09bnS7i6FpBPglvDjlKsoA449KZ0K3gHJZx3BWJ5JVmYYWNwADzVcPwzZ2d08k+Z/C8/h\neJtXOO5p1KhE0yOuai2oG2vrISW0kHHhIdqqPUYpHcTvdy/MSsWdgC+4D0qkZWZqiguxQ8nGe9Dy\nv5dpPIqgAbLFxRcNtNJGpVOWJwX8ox9elYwyj8bT/hfURt8xuYkzn/2yfrx/Ws+24HnOPXGKVDs4\nTkVUTTCEM17vRMSFeB5omG/w98QTaHfLIMvblgZIsDDj0qM94lxdSyQJ4MbOxVc/lUngUnWnY14X\n6frF3pl6txZzvHKp9eD9QetayD4gttX1KK81G6EVzcR/LuImCqgGBnJ/LSzgno0ZUEa/oUt0trJF\nL81IgMJ838oOEPJ6njkelZrU9JvtIvFtLuLbOyhgoYNnPTkVKM/gWvoH4mCQenpUGYVVCC+c5Y1Q\nadChWjajPpOrQXlu22SJww5wDjsfY9K2OsafZTW2oapcY8KeES2sCMQ29/58emc/vUuTJWV4/DBG\noirIkzh5Nc4omOE13PFYwZpUZkvYxjPmHFfVLfIgQHggAYoACUFWqmZA3asGzshwOe9ATFjnbyaI\nAY/m5OKraUKeQaDCjwmyMZP0rqu27g4pQoKikYtz+tMYn8vNYIfbHvmmcTcCihWEowzV9z5tLuBg\nHMTYz9KZCs+NeJbStsW1WOQEgFGP9DU0tZ0KB4nCdQduT+1c8mkx0sOiJniJERBTqTkf1FK5bjMr\nLjJBxQ9EaZ7w/EiKx4D9duM/errK3kTG8YA7nk0Ww0NLU/KypcvH4mBhVI746/vU9Qlm1ayWSN8+\nDHtaILghT1IPeuPkX5KRpZ4BQhFt1izl5DuJXpijLImKVc+YZ8yqcfWuiP7Ci9EWaeTKukLAiMnG\ndx6Cl3yjQTSLNNtCr4i7+d3qPrmqIL9PK8RXysMntnNDXdvJH5kjLjqD2FZqwUaBtWS/tYrol7aa\nJAs2W4PoQBUJbiSeQbiq4GSTg1yvh7S0DeATvJA+0sCDny9sUXHF4yIyuZAdqsuevtXR1jBYInTA\ntS0q50m4f5SRnceUo0QyoPoSa9QjJNWijVj2aziawjW6uPARJPDkH844znHpQUkFpHMFspRJtGFO\nAp57GoqSWMVgJDPMysrllboo6U5ttOSZki8cL/uVQWOT06dOKbtaCkT/AIZpGn3RN9HLeF2G1N+V\nLdsj9a2cVzH4AJ2xoq529AoopDonaX1vd7mhbcF4zjisZ8d3Zm1CKzk1FLW1VN0kYyzO2fQdePWt\nVgZm/mrS3ieOzSRyw2iVQC556Z6ClEtlK8pBgZZG5LEniqxiltgQC8s8TbVmfynAHpUw8kilpSSS\nOpPWqWFhFjAZUaQSIqRY37n556YHHv3rQJ4OjajDcZkkiIWeHOPxBkcccDoe9Tl4ZY7FVxCl80zw\nTr8rE2SznaQT6KTzQSo0LfhuGUjnv+1CC6qhpS7O2PvhRJUuQ0+Vjc7FUjtjFbPTtStF0GGF8G5w\nVeXZjKg8A+vGBXNyJudoKkq0zWsyxwa3DeNlwvCIfT0FAX9217E7wr4Ebt/OOfpVm2Rbov8AAg+U\nijlmmBHKlV4XHUe/3rP6lEsERKlispLKSO1UgynqFfiNs2BWyTn6/SqS29+nT2xVwUXJFkBwCTUG\nfc3mz9MelYxq4bwR/wCncs1/vnC3MawKwBDMA3BJ7YJ/SsiNXuQcRv4S5ztjGB/5pIjSYK7M7lj1\nJyagVPoaoKcxXQuTWAcNcA5rGLNjd6uiXoAeKxie4L19aqaQhs5rGNF8OfF91pU8MVw/j2iSKwWU\nFvDwc5Xv9ulbPTY3m1XUNa07UYb2WeKTwxjzM56KQeg9DXNyKnaLxdqmY0aVqWp6wYPBf5qctIQ4\nCe+fTFAyQyIH3rtKgg8dKeMr8JtULZPrmqiasvBDg6896+j/AAhfS/Fmk3uiXjRovgBI3VRnGSf2\nwf1qfIsH4/aPn19CttfTwo5kSORkViMZAOM0OadeWK1pHtXKIDlerGGWhnZqkLEE819Qg5ArACF5\nko5I9ibiKxgW6kUHnJNAvJuPTAomBJpAO9AvcLuwW5oMJJJgfQ+4NXLJ6UAh1vzjGTR8RwKBg2Ny\ngBoyG6LYB4ooDDYnz70xGHtHXpuQjr6ijf7FPlN18LatbSgSWcz7iSvhebOf6Gh7dLqwuGgDzxOi\nsSCcYPpXI5Rk2iupWEWcWpXOqfK3E8j+JHuKyMWCj39Ko1L4Tka4EtiwSH/8hkP5G/uD2qD/AJMI\nSoyXZEbLQrq3vEM8ixhjuBXklfSr5IrWG9ljVZklA/M+MDI6jNUXLGbw1FKzR2aHDSuMZLNg+1X2\n7WBsDIJT4jcflpmhWgKIILouC2SNpOOMUQtwyTfhKDgZJJ65o3QyDraze7R0UqoXz5YclgOg96Db\nSZGjiuHkHnDAJuwVPoR2NBTQWvoPb6FHDKrGQo7YwSRtBNM57Ca1u5IbsqsSAkFctu96fsKKgyzs\nfChd3Q8tEjdOwxTFdOuJ1Ajj2tgbgxwRn1FDso6xGtB2sb2C880BlaN8ZByD+tau8urK9+HBLa2S\n20kLAGRVBG4DJIPfpXPzrvVPDRhugg/il1NB8/p8luxUL4yrwR/mvV5fLP8A4y6xZRRf07r3wvLp\n8sSi6a7EhKjDZbgZ5H0rNJmKbKbjj8uRg/pXY12JtGh0tRf4W6aOMkZ3k4yB6mp3zHS7loSNrjBB\nA4PvXTxxqJrEOoarIZS6kZByPYiuf+rbyO2MLGOXAwWbOQKt8GTND8O6lOI8S7FaQZQDP5fej9Q0\ndNRme7ZIHcpggpycDrk/QVOW4Bsz0wltLZbh9NnWCQBg6KGU+/FAXnxHaFNkdu8jv5SNuMUsUzIG\nf4ciKiS2kZlIyS2Mqx7YoK+s2itIVUAvLIyEFeRjuK6Iz+MZoqh0tpHHiyLEy+g9KJaO4uLcQ7/w\nocsGkbCqD2pnoosumQyExkbcfynofaj9EUyX0BmUeBnzn2rNYY0bOh+ILScyZhXAWJOjD1NCyXLO\nFCjamcAlsYFTjH9m9DbqzjuzE8rbXj8y5PFBzQo8gjLodgBYldwH6d6dJAaLfiPVre2s0tdsIdlB\nEiKQy+x7Vnp0XVLDxIZNzWmCUxyVJwePqayVDoBu9ZuYruRLeXZH+VQnTAqMwjvbOO4jjKTI4SU8\nBTnODgfTFUTMBiUdM9OnNSUhlyyg46c0wBxrt0n/AKK0GzRjlxLPKD67iAf2NZmS38HaXbG5QQSO\ntCOBZIOqxkrwQe9cM+RzjH0pxSIKOeVIz3FWCAjcyedQOorGByMmrEUY57VjHd4zg81wS7MgYxWM\nRebcc1ENmsYkjYIpvpN1NaTiSFjuHRex9jQasN6fQvJe6DHJ89HHeSxceCMtg9UYDmsfrANhb+DN\nBLFLIMgyLgGuaH9mhpO9M0Tz3+9QbnmuoQttLY3d1HArojSttBkbC/etDbWV58H30ksqxyM0LGJ4\nnDjPqcf0NLJ/AozsyzSbrh0ba7HLhfLuPahjRXmGapnjUaID1drGDdOnWC6jkZsANzivpdjceLAG\nHfFYAcr7fMOxqqfUnUnLZ9sVjAj3zyflA496Hkum/lI+5rWEX3V6yRlm2hR3rM3utySsRGAo9axj\n2m6pMlwAzbgevNay2uFk5BzQYUHw3ASjYbjcQR3oMIyWVBA0sh2ogyT6Ck+mfFUeoajJBEAoT8pb\n+aihWay0u/KCwp1ZzLINpbaGGD9PUUyFZ821O81v4e166s2vpyGbehL+VlPTH0oaya9kl3NZvIVf\nxDIW/MSTn2rj5Ixi7LwuSwfadqSXEkyXDW8UpAwgI3EepwKI8spdcghkJ/Tn/n1r5r+TFr+S18Ke\nIWI4W9XcCvlYjBGaIktI7uEK2VJbJY8mr8U3xtUJ6BfIWnzbR3OdsS5wVxuz05FX2fwnaTxkyStt\nfLIkbgFSPXPWvZjO1YrR4/DK2lk81xMCrDhUYbl4OCQffirbm3srm0txaWiW43Es7SZbgdMUHJiX\npBZJNOhQS2z+HcMDDJnIUjr0/Srdct7fV3iW1WNLxl3JIchJMdVbHf3qcbq2Vu0RsYfl5oI44od0\niEO9x5o1OepOOP61TFqMVlfNLKssy+Jtd48FAuecZHTg00J2rFUL9GWmazHBpup/IqJvDPjqkqBf\nIPce9ZWHVZ21Oe7uk8ATESEryMEcccdsVZJTVMRppnLzWHazQgOoKMCxPL8/m/TjiibSA3ugwxxx\ns+y6LBVH8u0df0NJKPWODJt+DuH+MeBLumkRbeAyKHXIYgZAwD3r1cj4I8v5SQU5/TNDV54i1wJJ\nPHRcLIBkjtVFpf3E0q3Eh2uudrHy11y4VZKx9LcQy24aQAGWLY3GMk9TQF3DKPxZLlZFxhcuc0Yr\nrgjYI9kt3GyqxSUc8jjFCw6TcWSS3M6lwD+HEq5ZvfHpV4hTNNpFxYxRBjc7GK5dpV2FT6Y608XV\n9P8AB3JqFo+FP/5Bn96m/Ri3Q9Qjl0i2EMqMqxheG5GOPX2r2paLp2pJtuIo43Y+V0ADZ9femozw\nxd+LWwu5IoZWkZSBkLwaLt0hnjik2RzhiVIYgFenSkaob3wW6hDAHbpEp4BU5HFZlr2aU43BgeCK\ntDUZqiSvut2hKLjeH345+lSWWaC4DRthMg7TRAaPRRJc3cSuRu37lAHYdqqaVEjIAU7ecHtS/RgD\nVNXuZlEagkBcEqvTvSiO/ubaRl8SRUkwJApALDOeCe9OkgHJriS6uD4kjOoYkBmzii9KuZLa7VoS\nA0oMZDdwf/sfpRaMiNtoF9eQrNBbNIsxwrDGCc896t063mtpb7T5VdXeFwUHXcvmH9KHb4PQlaPc\ncn8x5wa6xPhHsT2phPpo9Ttkmi0JwgaCLTw8jHgfnfI/eszqtw91cKznjYAqgYAGBihEZgiIT61N\noyKcQjgqM/aiLV2TcU645omLGhEq70BDDlhjr7ihy2OcdaxiljzUaxj39K9WMWRjcwpnbsYwD0+l\nBmGtlqUyjw7fcj53BwM4xTK/t4tY+H5b67uJjexDZHCG8jepH+Kk8djR/RiipJ/93piuzQSQELLG\n6EjIDLtOPXB61W0K0V9B1OKa/D2rR6bfMbmH5iCRdjxlscE9a0vApjX4mv7OHQYtMtFA3zm6IjOV\nUHgDJ5zjBrIkUnGmloZO2RaudqoKertYxJG2uD6Vvvhq/wDGsBubLg9PasYeh8rz3oC7lwxG7ApW\nYCX1yaD1G7FvCTWRjL3N7LKTudsemaHU5OTTGJRko+RWs0lXW3UnOW5oMKGS5ZuhFH2ed4BoMxV8\nY3rWnw4sCkr8zJtJHYDrWHsb2WyuUkgYBk9RmigM+ofD2py3tqGmXDH06VpLaQqck8ds0yFZkfjD\nUrV/iCFpU8SSGMDaTgdc0sfX7y6ghjXESQ8KI8gNn1qHJDsykW0sEcvifNNLEzieA4bB8xA703+H\ntbubiS5jnYXDLAWhjPBZsj79MmuXn4YzVy9NGVy019hLa6feQ3GoWaz20kIJAOSucZ+lPLvQIZWj\nk01ZYbe4cKrTLkc9Mc9M8c1x8X8eKXV6yuJmbmtZRq93C0fiCJhE+AcZH/BVEtrcNbpNZmS3kiky\n5TzLjrzirdXCVDKrBBOUSNpoYzvYO0ksvLn2GOntRc1oxsY7iKPCeLktGfTt9PpTSdC9E2V3twt5\nceM+87RhUDYVM9cCi9NU3DOxBRVIZirYbPTj0rjlzyWFFBRQvvNV/hl9Ms5fKSAoVOWkHfHqfvU0\n+Wu5p2tWcREB3DEgknsf1rrXJUO9CvSCN4F0IUihTJ2gydCM459qayQTX13/AArUI4mk2mS3kZ1K\nHHYcc8dBVovtFMFYJJL+K1gjgvLPMlvlUV4wu3vSqDULqSWTz+HHI+8oOgzmjVsm1Xg9guZ000yI\nERUUEyeMMgfT616s+NMFie2uAkUgI5PoKInm0/UbYpcQv4igbDnHIpmmyDKTIz2qq7gYJAXHShRI\nwThCwXvRaFkXfM/Lbmc4wuQFPU+lTtvia5hBEu58+wOB6VraRkHtqUdzGtyZQoA2E42kUGdMj1KJ\n2jYzxr/Mh6/ekhK9Y0XorfQ4YJ0dpp4oww8UZ5K98e4961i63pFhHBbQTPOMbUYDOD7n/FVUk/B5\nIB1aYTOFiXaCPOzD+lByM1tZxtuxyduT/wA9KDBHD0N0Li3ktykbGbALE4K+4/xWVvbKSwunjOSV\nbg+ozTxHbssi4cFhkmrpCccKeOp9KYBfpN7dWur27IDguARn1rs0GGLSSBNwPQ0roKsuvpTbafEs\njgjZlAOCaz1xMDJnv3owsLJWwaV2bnOepppZwqfCA/7gl8xHpxj+hFNJmXpdq8qR34Kx7BbgR7VO\nMEdcffNettS+auluVVzdROH3k8soGMfYUq8sf6V60lvpNxPbKgkEwDwuDnANZ9s8becgHj1/4KaL\ntWLJUxzqrur2dqzP4cFsiEA8ebzEf/vUuns3ac5V9vTkenH2ooxbJBHGgP5sfmAHT60M8TldwgYL\n13c1kxGcSJZLaTsV8w/aq12qGpjBMMo8QFSVCgk4+n/P1oCQjqOAeaJio81wVjHcV3FYxbbAeICf\nWnIsJZIPEjQso6EUrZgzSYXtny++E55VhyQf/qtHqWnXst4yaJLa/JXIjm8PeAUOAO/Ocg9KlJ0P\nEGuLKI6m1jepaC5XEiy2fDNnuD3YdxVPxJard/DM9zIDJLayKEkkJ8QIeOfbNLfgzWmGxx7DvTvQ\ntJ0/UbKc3l68E4YLEqKDy3AJ9s+lXk6Vk0iv4j0dtM1Hwg/jLgDxFHUjjH7VVqPw5caVplvdXU9u\nGuMFYA+ZAD3IoKXgeonYVGnFPV6sY70pro2rtYuVXGG9axjdWd0JrdWDBsig79wZcUrCgcudnlak\nGsTMXKk0EzMTNz1qQXj29acA90L4bn1UGU/hwKeWI5b6VrRpq20YAAAUYo0Cz0cW7nHAplpNqJ7l\nVGKUYSf6obotQs7XbtjWEv8AUn/6/esXApaUAAlj0A70fAI+nfC8TRWCK2cjsetOr/U7exsHe4D5\nUEgKcHNaxWfM7y/a9l8WQ7tx4PGcVpfgjTbbUr8C5kjVo2DLFK2BLnjr7YqcpFIos121htNak/hq\nxN4qgEJzg9wKEu9Oht4LG4tLiS3vBktuTAyD37VFywzVPDXw3cd5pcdrqGI1CHfLtxnJ4I++K019\nqx0rREtZhF4KRqiqFzJuA/Nj61zxyWD7RgrT4mtTqDLcSiXZJlkA85P8x5q251LShBdHTdQeSaRc\nlJI2wvqOnpTODsohXbarb2YVYpBMW/Nldu0kYAGf619Gm0mC1+HLczwxkKisq92JHbHekmg0K7j4\nXXAkWExAqMDFL/CttNbwXypcE7yDgkds/wBq86cJPwZyQk1+c3MYDQPHZqeGcdW7H2pWt1JDEfl2\n/PwSRXbwcTlx9X4LcUyy1ulWJhch5AG/NjOAaLNxc+Mtu8yJCjeRgM7Vb0PrzXd0SVC2M9QexluR\nJCralLjZ4lwNquRxgYHWu6Z8NLda+YtQh+WsxFkMowu89Bnjpn9qn26rQMI1L4X0fTtHmaWzlu5A\nGQSwOTtOP5lBwK9WjNtaRktMde6Tc2s8qFcFDnIbysPUUCbsl8IAcDkqc1ZIk1oUHc2plbzlR5Yx\n3HrVdjNIZ2DRgxLy7KDwKLM44cvhLdMJYhmDoo2gMTXEiaNgzsqH25NK4moOsbRp2zjAPVnyM1Nd\nXaxmaOxG5UyCSM59amkkZYPJbjRbzT4pTMkMzLl4zzg0huktEkUWijCHIZehpacXg3Ypub47eQST\n1bPJqm9kxb2/IOUJ6+5o9jIWPcsm5RjDcZIzUkufnpljvCFcYVZOuB2z7VSLGLGs9sw2kKA20Ejy\nt7iq72VgZVRVCIMEmqhJaNNu1K3aWQACReQvYGmutx2JnmEEcgkicggYIPPJ5qU7Uh4eaJ9Y2PYx\n3CBlKN4W1jkeopKMSPnoDyKeDwEkNtG0y61a/W1sk3uRu9sD+9aa/wDhiTQbS2uLiVVmlfHhHqOR\nitJ/DRX0Uiym1W7dbW3E0hcswJNGRhdI1K2tpLX8ViN53Yz7DH1Nb4ZvQfUks9W0wXM0ZhezZowI\nudykkqOe/WkAt43uI47beS7AIGHPPQU0cVAbthGrlo7+Z7hlDO5YKp5xnjjt2qd/cTqkEvBS54xE\nwIJGOD74wa10H4NLJLjT74fLzNHkYbI4P2NVX1vPc+I88xYdySahHZWSbEsReJ5IUXIZGByO2M5/\nb9qWsvkAJ611IKLGIjtFXB3ynJP/ALR2/Wh3ywBNEJXiukYrGPZ5qYA21jF9qm6QYrQ2FlOkiBJ0\nGTuww4+9KwF8/iguWkDPn8wP9KRTzTR3HiJIysD1Bx0pV6MWaJrs+h6kt5BHFLKu4AygnBI659fe\ntTYayPi+xu7KdEOo3EOxE3bQ5ByCMnt6GhON6ho+mf07Q5INUul1GLC6chkmRs4YjgDI9Til9veL\nDqi3HggKHJEY6KDngZprsDRpdBbTruVnnZ4hCm6NXYbWODwfvWRu5pZJ2MshkZSQDuzj6e1aK0z8\nByOKgRVBT1eoGO9a8CVOaxjQfD2stFcrDKwKHpntTm8k3SEg5z6UrCgcStjB6UBc6e12SVbntSJh\nBBoV5kAR5HTNaTQvg+FHWa+YyMORGOF+5qyEZsIUjhjCRqqqOgUYxVdxAGTpyacBLTdOBchkzkda\nc2GmLDdJJtHT0pKDYl/1T0f5vQY76Jd01o2GA7oeP64r5/oNkfmC0qHA/KTSthR9I0VVSNS2AqjJ\n4pH8W3MWq3ywbH8JV8rqf1oN4I2Za4tFii/BXGzueQa8mXcKjAHGMA9/rUuwykaS2/6e1hx5ZFBw\nRzgetQLPcGCK4mkdVYbFdiRz7VNjr9jOW9nVpJAqlY/KBkeYj0BrlxqEl2B48cu52C72OSOaRQS0\ndO8KbzRLSO+kkRwCSAJAOpOM1Xe6RcNhbYjdnC44J+tUTsZyolpGjXkd8f4jDsiCElsBmOOgXHc+\n/FN/h7Vb5Pii1fUEu1s4s7YbkE7PKceY9ecf0qUlujxkmjcyfF+kRo0MrSPuOf8Atnp6ZrOa/rFl\ncxK2kR+FMgJHinhm9ealKS+COEl6JITcz6HdwSiOcctI5P5STnj70JaWy7EijUbVGORmmg3RKXpc\nnwybMESzNJHLyAOgqC6Ci3Rm3vkLgJny1wfyP50+KXWisPyVld1b+Bp/ykviSQCTxQCejHjNDrdv\nqV1DbXUswsbVT4kavjew6Vbg/k/9dYs0wzSfiq60i1uobMpbw3AIG5N5J+/evV3rULQmu7KfTb6W\n1nkmkWI7A0meR2P6VF1jisx4EUSEsc4X81UTtCfQUH5eVJpjsGRuB6k+3tRtz4k8eyM7LYZICsM/\nem8GQokTeCvRh0yNtTTU006RGaMM6DvWasVo0OiodeHizMyQeidT9Knd6QNDu1eKbxYZsr5xyp9K\nXrpOToBlt1YcKoPtQrW0gfrx707ipEO4x0H4ZOt32xptkceGc7c5HoPrXfjuDTodYggsEWNYotsi\ng5AbPNc8o7R1xa62ZJ05yeBVTNtbIOCOAaeBkEw6m0cKxY3pncQT6eldvUWa1FxG29CcE91Oehqy\nCD2bhLyJW4DH/BrQyRRXWs3Sjh4pnAYN+fBJ5qc9Y8AGW2Vw9vcAbJGD7vQ4yD+9Irq0WzumWN/F\njXgP0owDNI2H+m+s2GkXt1NfyeGqRZ/LnuOn60RrWqfx7X7e5Ry8GSYgwHAFCS0CeUKk146JevFZ\nhJMf9x/XvVU+ryajqcdw/Dhhj2pvgrQvaeQXDW7PiKZyGPueh+2KjYqbS9e6cgC1Uygf+7ov7nP6\n06ALd0k8rSNudj5mY5JNaDSbSO32/wAQiaO2uR4iSxkMUZOc4HTpjmpzbrCkVYdf3c9zZPqVtJHc\nRKejR4kOOrbR/L2zSiW/aciL5gziSIEBE24c9sGkgvosoIGghuLPU7cXEbIZDt2v6HrQJhZ5SoBO\nTn7V0J2JVFMxMsp54HlA+lUlce1MY8CCo4rhQVjESmDUlU49qxgmDhsrTS2juyFciTwgcZ2kig/D\nB3mmwqpznAC80HqUCRjHAI4x71NBETgbjir7C5ezvIriMkNG2cg1T1GN2msTPN/6g02JXk2hL2Fh\nuWVcdcfbp7UsvvhKTWJBf6NFEltND4qxmTPmB5Htyeh9KivxdjXYjVJtOe4gmjKOqlGRuCp+lKj7\n1ZMRkTxUaYxE9a9WMeHWvUDEo3KNkdRWis9RE0SK58w4zQl4YKJDNxRFsgqVDDWzjG4DH603iXAA\n7elWiIwmOjFVSozg1RADraNQOMCmEfkXFKzCX4zudug+F2nkCn6Dmslp8SPII4ypfOACcZqbCWDV\nX3NAJBFglfN/9UMIpZ/EKjeqcs2RSvwRgsPheMA+FUtk0zlmsbWNQscbEjKhEBzUWgA3jjeDLiPH\nRenWpQzRC/iUEuCcliOOnrRotErvFe8gSRmR4skqD6njmmujwRrZpJMpjjDYBDHy4pZPCsY6Fyok\n/mX8Rgu7y9APU1Ql6tsFVsnPOTyaEfBJ+0GW2rRSckt5T2pva3EZD7UkZWGG44rSJLGIvimVdCuL\ndo44AkyHAlG6k1tqbSWy26hVhAIVV9Sc9etBJUdPZtaGx30Y08rHcqZWO1ogOcA9aN01S5yqltvH\nFSqiEhxJBetbCRYC6r2B5rinjA25AyR3FeL/AP1IfmpF+DFRVPEsqlWUc/7hWYv0tNMHhCdTKSSy\ng5OT61z/AMGck+sSk1aA5bcGCTh/KA4IzivV9LCeac3U1PxHB4RF5fFyCdoTbnH3Has0+2WMB/MO\nvXGD9qjPmcVSDKNHnjE0TMWVnK7fMoJA9qvhs7Gw0aSe5jkuLhXBhWN8EjHfPvWhyyb0yFcyzxWx\neR433DPrk+gNAt8tdMI5InSRuhHI+/eu+LtAZsPh97LRtOS2edPE64Jxn6CuandpfWsaIsviCQkk\nrwB2/tQbojKNi6Lyja3WpblfrxjtVIs56ItqFxYxsbWZotwx5TjNKZGaaUbvNIevcmldFYt+Gp0X\n4G0m7sUbUNQKTzICESQeT/grE6/pw03V5rWGdJo42wsgPUVJXZ09aVlUdptjLOTuxkcVK0kaAsVj\nLq35lI4PtVkANs9NW4vopYTtjY5YOM7D6Y60Xq1jcWWqyGGbInkyrY2jPWlbGimetmj1SN4Xbw7y\nM48NmxvOei/+aSSxQyXRWRXTa+1weoNKsY0mOk0GG206WRWcLIyjf2x1P9BVdjdrLc5jQ+Faxs2R\nwSQDjP3IotsVCWSLCv67+oq63n2ldqKJEYEOTz19KZGZGdCbyXPmw5P96K1CONdLhG/Ml4fFYY6b\nSQP3z+tGzCoApxETjqcVTNMScZyprJAuhomvtZaQlrYmSJpUZLpyciQHOFA+mOaXRwlbXxd4Vg4G\n0HBz1zRSr0zlfhbYtJc61bNPI8hMg5di3fpU7gfJxSBGBd+D7CtdCi134wKrY7voKcx3aEXPWvRD\nfKBx15rGLLpQsmBg/SoDy1jF0LYbtjvWjhmvp7ANbLOtsqHdsY7eOtLLwINDK24FWIPrnmq79i48\nPGT1zUIy/KgiWSMqxyKgn5wMHJ++a6BT6daaS2i/C8EocP4pDeGVA84U8ZPrk0pvrO4isrnbbS2s\n0x3PFuK7T/uTH5h7etcz9GQr1G9GsweLJbst/GAs744dRwG9c8Cs/JbSrGZTE4i3bd5HGfTPrV44\nBkLi1mtsCZDGXAZd3pVBXBpwHCK5iiY5XaBjh61bHKyMCD0rMw2s77y+Y89qd2kofpxSNDDmzPmG\nKaIRmniKy8MCMCrEZgeacQY28p25zzTCJ2btQYTI/wCoN60VzaW/lVVjLkk88nHH6Vh4LyaC5SWN\nxuU8NnGD61Jsw6trm71JlL/iPu8ztjA+pqTafcyySKrwMFPO5+lLrAxcww5zgAHa2OQPcVemy0vo\n5UZsxPlT2Ye9KBA+pGa5uZJ4wTHJllXPK060u3FvZPJKirIiLhicjmiy0SJUzTeHGxJ9Avb1xV2o\nSRxWMFl4skTM27OzO4d81KW4XWA1v8/p2X00i6JfcVb8pHfNCXV9diUtdrHEWOfDTtT1lEXrJQa1\nbxNyXbAyccU70n4kvp9iae8dum7LtKA+729qDQtHL61vZ5t1yqzBmJDKdwA+lL5xFbYwOvYrikqj\nORL5iOPybAO5KL1pvod5IS/g2s0mQCeg/vSy8Ebs32iyy3mI4rG5jIOCGA4HrnNLf9RNIbTNJOrQ\nZjmjIRlVc+IpIH61zz4lyel4PTF22tXbxbUuFaZePCkiwW+hPf61G5ju7nOy1tJGEXiPiJkZfYnu\nfoTUI/w4Ql2jh0N4R0i3mur/AOW2yiN0/EMBz24H3NerrT65Yqimby+8O6sD4iMS0ZBQ8Er34rCH\nTAWDpIEtmYqGPUYHIpOSFslLywALJGQxV1PXzDHHrRc8rSaez5UMGQAY+tTSaJW/BSdJ1Fke6hik\ne1GSSF8gFVCF7e58SGEOyjKnPftn2ruhJJJg36QexmurhZr6YKzMNwHIXJrdXdr/AAz4TuLC1mW4\ndSHJTlh0I4/xU58tvC8IJox8d+iyKs4z67W5qN3q8VoELwyKsq7lzznBxT8fLeHPLg+lNvc/x26W\nGHEKp5nc/wAq+/NCz6gIGYWqPHuOPEdcM3oPaqNgUKIQXnhESHlhxyaNj1WzmnPzlrEydQclcED1\n71v/AAdYUJdS3JLySIoxgDw8ACpnxUY7ZY3AUflHBpzELC4ayvlmJYnOPzY49vfitHql0msXlhdq\n0oVmjSTZjGRwGA9eADU5FE8ozWpnwtYuJjJslWTII45zzipX0j37LdMpeRwEkcH8x6An9KZbooya\ndY9Eg02W4cRPK7SHryMYA9AMUNHBDbaYywSFjPjynuo7frQaCgC5idLVpGjIJORntS3ewfJ6cGmQ\nGRuZJPGdmbBPNNp8y2Gk4Bb8Js8+kjU7AFyzWPyZWJF8T1J746Vlph+Jn3yRRFL7eBpJVAwM85o+\n2t/AuN0kaTBT0Odv3xWMN9DayTUWkuYvO3/bVfyqar+JYbdrXbCCsik7htxxXO3LsPlGTK4R39Ol\nUhvP/auoQnjgk1yKTw3yBWMTVvEkz61ayYOO1YxBT5s44rW/Devww6NfafcFh4kTmIj1x0/pSvww\nst3API5ohwrkMcfWuCbqQwDf28YG7d1HQClQYxuGXqDkV2cb7LQG/wD/AFlpdzZxfOkzEwFZY2XB\nL48rAgYBH9Kx7fEGpu0W++lYQNmPe2dv0plGjWHW2oyeLLfwQqkm0rIONrZ74q2z1RL9LfT7kiG1\njk8QgDhye5/ehJP0AH8QaRJaeHKJJJ0IIZm6IO1IzkgZo8cuysxzvXDVDETXqBj1eHXisYsjcqw5\npxp96VcDNCgmq0+cFRyKaCcYxmigMtjnAwc0daXEbNtJ5NOhGNookxkUwg2BQCeazMYD4xX+I/E0\n6rHv8KFVUt0z7H7mszp1tDDdMl5sVQOd+TXNJhJyJmcLHcmKDbyQ37U1ivYRcCSBjI0UJUkLwfc0\nYt1oJApaOWXCFlYnqw4qE8jt/wB2PgHrjFYCRdawqX2EjGB1NOUtswmAsgQnCgnjPqaSTOiCB7S1\nEN14UDSLIDgu3fHp7VfrkIQ258NPEIwx6nH61K7eFnkQBJLmK0MXikDdgEenvStdHuprwO7oN5ID\nMeBV/wD05W6J31hDpsBjN1HPK/5tqE4o7SF8GELBsldumBg0nI6VjJ9sGBupbXcZJHUgc5PSll9c\n/OvERtVRxnOSTXLxzcvRJRphUULLERjlByR2FaP4XMQu5FlkU70G3PfkVWXgEfRrRV0yzW5J5OBn\nrisj8dfFUc0ZsZY5pPMr4Tgtz+3aoSk1UUdMEjAX0tv4AlVbhppM+RZjuCk483H+aKd7mGKCO/Up\ng7keRuQMcAY9ODinuvSr/wAHVrY6lpuoKyWsogcDw7iEFhISMgsOSD+gr1Z8vH9ZMjf6jPrEyXiy\nkPInmRCQEPQiqI7d5LRIXkVd8mI4kUlnJHJoy9EXpZqGk6rfXK5gZ/DRYhjAP0xSa6lWKwCn87zZ\nKdxtGP6k/pU6sSXo00H4gW30a40u7gLQTKQGz0OOlWQfw2P4dmjVE+aHIbufYU6/rQyVmdlAEh5B\nwcY9/Sr0Yi1uJ3dpLhSmOuFHT+lImUSE0/nkZuc5z1oXVWivLezj2uZ4QyNkcEE5GD9/2q/GqYrL\nbadbS0aGzBRdpDyd5Cf7elcSe5jt23MZomOCrjcP3q3W9J3RJbyz2bbnTkyOjwyFTj75FLpArSMY\nt4UnhX6gUyVCt2WW8/hOCRwpxt9RRVxfRSRkRQNEz8E5zn9aIUCgrMyq5YKCASBmm9gYrHVre0nu\niIZX8MsD+XOAD7c4/elatDL0j8XQvFqBiaMp5mfB68qP75pbpOUMu5jgrtUE/etDwMvRjPEsUdi7\nKduwlsHr5jQQnMsw6rGOi56CiwF8syyWTR+bIOee4pa8RGWXDL3J7VkApuV349xx700ubhYNG0+F\nAC7RsScc/nP/AJpwFGnwG5325Cjechj2NSutHhjJwZWcDkDoaxgRpY4GGEfK/wC40Ut1KtrwxjBO\nQoP5qJi/RbgR61aKwG5p0D9+Cav+Irpf4xfKPOokIVvUdqSvyN8M5IPJsGOBVUNuS/J7ZqlgLflw\nEwWBoeSHa3lHH1rGLIV28CpsQRgVjFRQ5rm4oeKwRnp8yTnYZEjbH85wKeWUembFM02ZB1AfiuWX\nG2zN/oM1OezOntCEjbcvlIHIP1rDToUfn1rogqVC2x5o1pDdfCOrbIElu4tjhtvnRdwBKnr60nNh\ndGza8FvJ8sjlWkCnaG9KKlTpjtfoL0Z4lnMdxHuDKRyxUD3/AFqmRQ2SpKjvzmtdih/8UuL7Tf4f\nuUscBWPVgO1JJonilKSKUYHkEc0UkvAFZFc4pgka9WMcrooGJcVfDKUfPNYw/wBI1He+xjyelPDO\nQnGP1rGFt1rNzZXG4qHhYcDHSr7L4qgMo3Eq2Op4pkxWjU2WvwGEb5kHGc7uKOOvRQ2ctyG3RxDO\n7PBPpWbAkzKT6+uuMLeCOSCe4YLvkwB+tU33wjeafbvPM4faeSGz7VyuWhdISizmExIICdORR0ER\nVTjO3GDz1qi8BJk96BRjgZzRsUhmtsO+Nv5fcUrDE6s/hAjBJx2GTTPRIJr2YXG+Pw1BZ1kGOlSl\nKi8fSWrakITB81H4DSDciBSV2n3zStvCZ2kjUbtucs3bPatBL0fkl8Ju7OvlU5YZHTrQzvJDtV+H\nxyDTy8sgVPYyXSePJbSeGQQJVI25HSmujWvhSh4oyRbr4hB6HHWo8svxo0PQC7lM5eRiRuJbFDeG\nsi+UlCRnjsahDBZ7Kw6KBjbLJ4mSgw25sZphoQvommlsQqMy4RplDKCCDgZ9s/rXR8MjZJrV0+nO\nt9YlpkRRF8vJhWPckEjvisM41S4vTPcw3DTN1ZsqemP6Vyyi+1lFLBleWF3qkEBFrFE8cQQSs/OM\n/vSWXTZQ+JSGhJKvtySpz+YcVdOkOmh7YfEl/plrNbmVbgbsK0hP5Pb07V6uaf8AH7uwOrOtrVu0\n6fJwLOXkJdFOMVoJIBcZltGQMFJDpxtA68+tPJOMgRdo9BqNqtkWe+kWHxFDMRv8xHTOcj1pbHoK\ntrM9xdtFLZrAZyY18MBsnCn7Z596NiyApbPThdxyQBmUjzqXAUn29q7p8FjcamEKLtZWXBcAq3Yj\n2rKVukPBoqaTRZ9Xtl1HfauB4Vwo4CsOA2fQ8dqbat8CWtxYGTRbhgxG7DnIf0GfrVOtaUuz57qP\nw/qtlh7q1Yj+YxnOD9qCtpJUsriK3t3a5k8uWXOxO5H16VeNEWxeZJoZSkqlSCRgjHNMdOldreVG\nUbX75708mqEbChawBOVDE9aGubB2UPbqAqnBVRz9qjGTbAKyxB83XHNWlThMg5HQHiuixkSkcW8e\n7PU460DLI07liecYB9KKQTYXMq/E2i2d3K22e0/6abafzd1b9M/pQ0mmxRwKsL42rnLd8/8Ag1JY\n6Mw630261O1042lq9x4Ub7lVc9GakV5G1vIwkXw5B5SrDGD3p7CejSdk/CRpMEYIXINCtIVIXGex\n4oppg0kFjMC7iS24gcdKbzQxQLbNdjw1WBBGe5yNxx//ABGmAJbycfOBomAQklQvb600gmuJlPjN\n5VhLBgvWs8QUXz/D8g0qDUWdZoiuW8PnbyOG9KVMfDUsxBZjlRnOBSxl2C19O6Un/wCmbbJ5Myn9\n/wDGahcjfcF5CRvXI9630V+ARH4vTqamU/28YNOAqkyp45PfAryqJG2qpJ78VrCWT2wjRhwhGCVP\nWhd3m5NFMzws3jGTVEjjOaICstkf2rqnHSsYLt7ySMjzZA7NzU7x0nUMiBGHUA1qAyvTtTutJuTN\naSGN2XafQj0rWJ/qbdPprWU1jbyRMu1hjANSnxqbKKVIatpv/q3Rm1IrDafw+P8A7caj8TocH7Vm\n9cSx+cVtOARZVDOgbIUnsKTjtOgzWJiaS1kil3xK2EO4n0rRy/DF7dfC0esvHHcJIP5Gw6445/Sq\nyn1VsSMe3hj5BgkcfSqz706d6B5hw9K90omOGuUDHQauhIzg96xgm2cxTr2561oY7ghcE5461jEJ\nwsqEOMqfal50YTSYjYjPQYoGQy034ZHiB7qaRkXqi8Zpnqdws8MUMYCxJwEHT7+tBswTaPf6JbRq\n0kHgqc7Cu5lB54/Wp6vrq31u0BkVN5HGTnApHVE5RtgtjDbs7NcvhSDtD9z2oOZWjmKmIAE5XnqK\nSDf0LVFp0eaWGSXxIEWNN5Bb9h70PDM24MSAPTHatN0GIQ75jwH8ME8lTgn70604SW+lPvm8ONWD\ntnBLAdq5lJydHTDNEetSy6hfNcYKq2dpPGAen7UFLdRxJslDblGBt710RVCcjTeBtpcw3ESb5xCz\nZCFl4J9CaH1G7m8NjKr74iF27MH/AM1m7wWqDtHvmv8ATPBcMio3A9acSqkOkCSPxEdiUbLcMPbF\nc3IqNH9iQI003hoMk9K9Y2xe7Mb58ucikiSY2t7CAbZJVkdx0UHy0aL94xsRdqjoPSrJugkHunY8\nuf1qm51F4IiwOMVhWxNcahdzZK3ci57Z4qyOVriNVkk3Ngg/5p/QOR1LTxGHhXA3sCG3V6h/zf7N\n/wBB38PfDmYjc3DJAG/LHHIHYDsfatDFpU1rZwlJAyKSqbM8jvn964Zcj7UdF1HDsUyI7wRW0aJn\ndhFwQRS34h1hbOOexgtxcRyopaQudoJHTFGDcnTFW+gNpf2MljHttts6KF2q3lP60NKy3kpdSsAB\nIyO1ZQlCbcmFKtQwudNsL6ygu/BMl0gCSgdCOx9zjFN9I1mO2doGkwowFXOcV13SHiyr4onjhZZl\nZUZxjaDjd74pFNFCk/4ar5lU5xjPGf60Eyc/bKLqwtrqILNCHAPY80vn0+KM4hTYo6CnrBUQjsTM\n+0Oq/wDypvFoyfKMBKEIB868nNCKCkZddHiMwz1z1bIJpvqMVlLpzrOFRxH5H25II+lM5O6KVgls\n7aIXVtJdxiWAMPEUnBx/9UF8RaKNN1WRraRZLWf8W3cfzRk8fcf2qkW1L/Bml1/0loqS+NJHGxAf\nAP1HINEXuokXUg6ryRVGtskfRf8ASu5VI+RgMjAZ7c5rJ/6k2nyvxIxbH4i7/KOOTU06kUa/E2H+\nl6W958KOksSNJHIRyPUf+a+ea9oclprlzCCI0SQkAntQUqmzSX4gktrtchctuIPH16UV8Ro0174S\nq2y3j8JRnsOKrZNaJorJ2bDeTPvWiyv8OCRnlYypUdzkf2NZ6OkQ07VZdCdlSTETHEkbAMH6ZyD6\nCjdd0S2vgb7RIwQefDU5DKBktzyPT7VKT6yTGj+Spmf0NS+swMedmWPPTg1bPAZPl40Id1j2/wD7\nx5z6U96TflA7WRD7fFhLD/8AyCvCAKTmRD9DmnQhCVLcdGLN+1cDyJ5o8YU5wBzWYyAdTn+YvGl2\nsu7HX6Cgw56HFGPgW7Ols8ZphafDmqahZi6s7KWeFnKBk5G7HQjqKLdAWgc1rNbSGOeNonXqrjBq\nCHPGayd+AaoviTLgfvR8dhHIB+KAfQ0TA13YGJt2TtPfFDSxxKPJIXOOQVxQW+GPqf8Ap6Vn+Bb2\nAHosgYDqCR/isDYGMyyR7UEgkyrN0xnpUYL8mUn/AFQZqS2XjCNbgvvHndeAGr6H/p/Ml18MtZOQ\n6RsybfY9P60eX+oOO7PnmuWml2Wo3lnKskcyyEpJnP2xWZlChztbcPXFU4/6oWX9iO2vYpgHGqNY\nx4VMHpWMERyAEA9KY6dZSXt0IkkI3d89BWAzVpDBBa4GXkXy4I6+9ca5aCZT8mREB55EGSPeksRM\nJuNZsSFgEyMZBgYHPNBz6X8jG80buI1XcUcZJHtTJBboGsPiK0m0eeylSczSNuhZME/T1qqz0+81\nCYqInQoQTvO3is2L/o2i+H9RaLbLNb49C/NLr+0k0y5SKUiQ4B8pyBUrNYS0Cy6fHJFKrSfzx4xi\nqY9qORLDlCOcHkUstQUeKRyKVRG2gfzUz0/4kj0tFtG0+KUBsF3j3HOODzUkndFYsi1x4kfhN6/m\nFK7vS59QuctJgKuFIGCR71VY9Jt6V3GiWyLHBc6jHE6r5UWNmPPrxRlnElnBIDf/ADAICoXRlZce\nmaEng6aaLrZWaRQC78+mc8UZrLQxmGOBWQqnnUn+auKUmyqVRE28xSiQeXHORRdnMHu2kLckdR3p\nonMNlddgAZgDVTOB3qwxTJMBE7BWfaCSFHNIEvLrUHBS3n8LOCxHAplBsSRZKk8DAOikk8MBjNXW\n4DgM7AMeMA80YxaZO8IB2RkwSOvBNepmhaZ9DtPiK6S2eRra3wV5O0nkdz/iu2mtTT2UwhEb+OrZ\nVJMHJ7D/AG/pXmyf06+3wv8AhvSnliW41G4Mc6gxiMN0X1PvQl3oqzO8cU9uIomyUkYhm98n/NB2\nlaZmsEF/o5hmE8MPgW4PJBB46ZBzzTGOyWGCzknAW3mcIjYBJHc460/b/pX+Aixz8SaXZaRZWEzs\n6eOChRGxuwM59qQ21lDqEksdtgCSEFY2PJPUnP610RRdVR23sNPgi8O+lj3ocgNKenfH9fvRFwdF\ntLtw99brJHlWjknHUcf2qsYkpAkmq6S0WI40kVwcPC4OD+tLOZei/rTNViFRCXZbQO8iFjjCAf7q\nSNrUttfqFZj4eN4B4yayRREtV1fGoRujFgRkg9qd6XqKKRI7eXGTn0pJR2w2LmuEv9TuxcIrqwyj\ndMfpVNxp0lzAsCOQ0Ryn/wAfSrR8NYTpGizvcJJgKqnO3GN2KE1LQIrYs6Slnc52sf1ot7QK+jr4\nL16PSbrEoZoJG8IsD+U4yKV/F2uvqOuteGEeE8W2MH09f1zS1+Q3bB5/pf8AEcNlfyadcgKk53K+\ncc+lVf6kwG0+JQ2/d4qBsY4HOKnJVIzdxEWmOjXMbOucNnn9arZke9YzM/mX+XmrImiw2kCRuzuy\ng8gqu4iu7Y7DTWnjkMsqzKUBGOCDkkfas3Q1ie+f8QHqGy4PrmjvhfW/4Zr8E8+4wnKSKDxhqEti\n0CLqRqNc0ZZtTfULeKGC0SIK4Tyk57j7Gs3qESSxNFGVjToSvU4qPE6wpyKhOieG+B+or0svnAB3\ne/euoiVt/wBzyntk+1H2cMwRCY+JMeYjtQkrQ6QRNaQySNG7xgqCRkZzQTaU8i5it8swOzC8cdc1\nOLaDVlsXwRrlyviW9izIQGDDABBGe5rW/Aum/EXwrqLST2cvysgxJGrD/wDi4PWhLni1QVFpms+L\nPhax+MdMd4YxDfIpMb4AJPoa+PH4X1BJiqxA7TgncP8ANbin8NyL6EjSTpxU3Wzcw4Xd/iopZi4n\njO5UR3C5B96t2JEdSngLywBg3hsVB7HFJpF5OK0WYb/DfxFcaHcSCE5jnUoyE8Hjg0oZm8UnlTmi\nkk7DYQv4oAyckVq/gb4o/wDT88q3MbSRyDCkdjQkrQYOmZbXLptQ1q5uGUoZJC+09QKHisZriJ5E\nQ7EXcWY4FNFUgSduyjIxXD7UQHMZrmKxj2PSuisYmvFMdPu3t5QyPtOetYDNbZ3XzvVQHHX3op7R\npBj8o7mueeMm8eA1poC3E58FBuHTigfiBp7K/NpJcM8jR7CN3Ce31/zR45t4ZaP7XSrHTkhS3gUz\nofOzcNz05pdLrMltfSkwLIc7GEj88elZtr01EbrV3aISpG8ak4yH70uluZJ5SzsSxxyT6UEzUMrO\ndrmWKFIgGc7WZBzg+1W3tuv8Va2tZUuFVtq+XGTQu8HijTXHw7babDbrc7gZEDqMjqevSs1qdikO\noyhM7EIwCeRSw/tpWUUo2UoYIyzSsyICMnqaOgnikkCW8wcgHAcdB96rIjJAWoxXd1cxyNIm1Btw\nFxx9RUbeB7p/B3BRu/MzcAYwaSWqgx8NBomkzWOrRyxvHcxLy+xuQPX+lJNTYvqNzIEYK0jFcjsT\nXHKLLt/iEW+mwXECq4OGHJU9KnJoaWwDQPIv/wAuavGGWct6VT30dn+Hc719JMeWqZ76NYt/iAKe\nhz1+lMojJi+a6kMySW12qSAcbH6+1MNGuJrtJTLIXKnByBzXRFYJIKvbcNF/uYDjI6VmWu0hkzsZ\nGU9iP70WTjoC95LLcF1OAT0zmvUtFEj6veadcXai4S8W1jyQFixGFTsfUk0C1lBYS+JHM1xclQ3G\nMDJ7mvHfhSS0IkupptW099RVrW1Pm8h8rqPYep/rWjvLTTNWCSQwFZJ4yYWVtnA/b60YxT9Crfoh\nit5Y7Np5nt5YxJ4JDsWJOcccdPeva3Bqem20HjBWgjO6KSPrH7HPSljxyTdMNOKsBvGvNUtUF3JJ\nLHGxYNvztJHWhluBaQlkYNJDh0287sghhx7fuK6eOT+jwdi+0gtpZheSSNFKjrGr/lMoPZwe+AeR\nSLWdMSLXLv5kyvmXeJEHUN5gce4IPaumMr8KSWaF2qaPZo3g6sd59YmwDRdvNe/LtLFeWV2oPKoS\nJMZA4XHNO1tkih7sX1ss0zFY3nIEZ/lVcH+9Z4n8eSUnduYk5pqMDXLtLKW9RjrT3Ri0loN2SMY5\n7ii0YkjiK/Kk8saYFyFJ79sUUqCOfhBN3xDG+44WMsAwyM1T8e6ZJb6nNdblXeoVUJHQ9SB7f3qd\n/nRR/wBRR8OW5EWJERRHPDI2T77T/wD7ClOqZWJGLq4Viox0PNV+kn4W2dorWscsefGzncp6Ef0o\ni/nn1GSK5ld7uZhsY54Ur2/Qg0jjbAm6KtMOzUYoXyoeRRknpk4/vR00QjlLEcjgiiZC97iNJ2Lg\n4PBXNEeHHdRXaogwkQZRnpyBQfgRdLarJD4YIDg5TJ/ahksijqjDLNwADyaCNRp9L1Vba0lsr3fc\n+NEYkLHhT2FS1jSYrb4WtL7xAZfEaNl298nvUP6zR0P8oaJbGyN6xVI97FfXA/WjI9G0u1h8S+vF\nEo4McQyc/XpXTKVeEIxs5HN8PwqXaGdgpwAWyW+uOlRfUbC42GO0njAOAvzGQB+lCpP6U/EjdSwP\nOuyF0GQTJkMOnpgYou0lFjqC3QgN9DHyYkY/vjkYpWsoX3w2en/Huj3qbTILfYBkMRx9hR8vxLo6\nKc6jbjPTz81xP+PJMN/BbN8SWOSba6DnPBjPFI7q7tp5pJIssSfOEPQ+tdEIOJOTFV3pl1qMoMRj\nA7ByBXdP0aWOCdbtYUYAlDnJz07V0eIVCef4duyfLC0pY4BjOanB8F6nLIizRpaK5wGuW8MH6Dkn\n7Vu6j6NVvBNd2MljfS28mC8TlSR0OD2qfybySoVRyZDhFAyW+mKp2VWZraHWpfCWoaPpEd/cKE3M\nB4ePMM0qTxI7kJKrKQQSCMYpVKzNUW38cFyzOzMGBwSBipiQT6E1jDuuJA29VReQB1+tOhPBEfzc\n/wD1RmmaVc6rdCC0iMkmNxGe2ep9qz8sZKzVa18D2mg/CyXt3eObxgAI1HlJ9qxVJxz7qxpx64cx\nXsVQQko45omEdBWMab4f1G3t5gLh9ueASK2ipHdW4MZBVxwRUpKyU0WzT2+iabJeOpIixkL1J/4a\n+d+NbX+tXV1es4WRyyg9R6fpSxjTDEbTamjpLKshkTYFDEebPrSZ3MQYOWB3cErmnasJSJQ75Ls/\nsRRlsjO3CnGO9K0EeaI62161w2QtvE0jDHLYHAB+pFURLdXEjzxRmSPliTgZGaFDVSNPJ4t3cWF2\n0axW8UQLKGHkUcZ6eoz96zutNE11JMt0rpKxIbO4kUkNbZTkxIVlDL5lcnHbtVtvdfLXAadgEHmJ\nqpEf+HDc2HlkjVpCdrlsYHvSq+tfA8OLd4ued0bZ/WkaClQ7sFTT9NaaO6aOLw2LopB82epNI5Pi\ne5XdskjdF8qDGcipw/Juy00uuDawmM6ozkZIBNNMBk5qqRxi+7ijdGEihl9KzyaNFFIxTPJzg9qz\ndDoXaj/0l6VB6AYx713TrsrcRvINwzyM4+9OnZmlRqLi5iRT514UNtB5wayt9cw/OsEYOM/Yiiyc\nY0zty8LwL4MKIwGCQTzXqFFUj6Nq9na32mpLpG944eXBOdoxnvWe8ORZjGXKFeWGcemB9ea8hSym\naa2zRfxn5qYaekCyvCnhphcnaOBU0+IV0WGXT7qAzyRklMNwpI6fvQTvSilgugn+e1EMAkaFshB0\nGOnJ6dK1ralFL8PXElxGkvgnw2QODvHt71oNqTK/2iZ59DYbJtMkeS1mB3I+FMZ9DTL4a+HWOowy\nvtdo8gRKM575qii0mg8caekfiv4bsJdSWeTxYCPzmNA25u2M96p1T4OPgwXwhu55Vh2eQIXCnoSM\n9e1U4U4RotyUzCal8N2VpdM1rLcRPjLJNgFW713S7cwubkz5ZfKuByD7ZrtttHNVF081gbaSJ7Ji\nzKcMkm3b7+lIWtU8JmXdsIyM4oJgYMiKegGAOpou2uDZzBi42helU+ACWxNeRyg5LjnFHod8eBnP\nSsYYaPfLpuqwSMWw7BCFPY+taf8A1F08TaRb3u1mZGK70PY9/wClQlakmUjqMV8OyLc600buMyxn\nr3YEP/8A80EvgPYyKYy4SUnketXsR+Bx+HbmH4NOsQThEL58Fkxlc460r+HMyNdxsPM6CRBuz09/\nvSwldmaotEQa6Un8ytkcdeauvrqSO6lbGSWJ2kcDmiATy7TKWK8sc8HgUZpbiSeSFpCiyxMMg85H\nI/cUaAgaDTZPFEzybiDuUdetNmWC3z4YzMyjc5H5D7VhkCSQFl5Y7uME1q1aHWvh23gZYgGnCXO3\n80f/AL/oa5+RfS/G7wzeuWNzol3JZMDH4YHIOQ46g0ouGluCWk5A9BinjKxHHrhWLdmA3AYHIqUa\nEviKMsR15pxTVfDunWN/a/8AUyspjfdIyuMop7gfWm93p1ktmiaO0c+pPIRtuXJygz5kbPHPFc0+\nR3ReMaVmZnVNZmlTUdNEUsRKtNZoQU9cr0YfvSHVfh+405FnBFxaOcLNHnAPoR2PtV+Oa8JziVae\nB44BXoc8Uzi1dLG68qOQxw2B29ado5n6N4L9zIPNlPzL5TTA3yKjK5KFhk8dv7ULGo5D4UEZnTUB\nFMQTEHBwCO5r0l9dX8y3V1cbljXglsk+gFTpN2FYA3En8QuZJLuOORpDyxHPNbzQ9H07SrCKURIG\nVQfFftnnvWm8oaCt2zHf6ifGg1aX+H2sayQx8F27sP8AbWB3vsJyTjvmqQVI0mM7Ig3pJXxPww2w\njO7im8l1FB4VxFHHbz+G21UjKt074waqiLMfMH8Ri6kEknmvon+msEFrpd1fnEk7/hLEPzt3wB70\nnI6RSHot/wBQbL4gMsV5qlr4No65hWPJSP2b3rF7eeAaPG11wM9YbeaPeafDBJdW8kS3Clo9w6ih\nAMgUydiNUdxjtV8Q6GiYvU/ynndnvX0HSLhbbSLdGJJVPWgxJ+Ft3eQ3UDxTf9txhvb3rJxS2MsF\n1ZTSIspnDQS4znnoaAOMGl/6O+kgBBUsOjZBq26Vrk72UBQ2KAzBdrKP9v0romcSruY88cnFYZGr\n0a2WXSb1mdMsqqCP/kKXPbXkNyAgNwUbK45HsKVYPRor95RaQXU92yyyxeE8DryD9KA1Q6LNFD4b\nG3nWIBgo4Zvf0pYL9BmxEbQtIy+KyKTxtNWpZ2e4NcXBYI2FVj+fjufrTkYlpliF2sLudp5YqMgc\nVUtxJLqKQQOqrIdm5gOB70jxWOlbNB8R+DZ2UVrGqlpV2ypt8rgd+O9ZBpIvEXwljKD8yMCCD+tD\nj1WU5MweWO87dvlwKcxO6pljmnRxMDv7gDgfelxmLDjGfQ0JFEK5dNvL1Li7VA0UK5J3Y6f3pciy\nz2EtxHjZCyhucEZp1noyO2RdgSxYkjGTkYB7c1TexrDMvgk4Zemawq9IpLuAPNeolD6X8Lapa6fd\nTR3MhVbhQoYjy7hnr7c1ql0K1ubu5u98RuJ8FGY5VOAMgdPevE5IuMrQ0UmtGegJbtbMwWGS5tWM\nMsiRgEkH29qx2ufCt9/EpZoRHOZpWfYnG0HkcmqZFJAkv0A6XYsbtLW6hYKWIZ3/ACAjscU4uLG1\ntSUgTbE6+dkHiAE/SkfuMvxqogLxQrBDHHeXE4R97qygA4+2ecYr1x8QaxpjR/wubwFnLFYowGwB\nxg8euf0ro+Wii/ZZYXTX0W69vC5WUbtxILccYxTO0+KIYdRSAFGuWiYJjOS/ZSaMPyditnz3XtQG\nvyDx5lW5R8eVMY9c0VbW5hslSSZZGGTurpTyjnlK2QvZhbt4Lxpkc4ABB470sdF8EHohOMA9K1UY\noaCJEJjZDnjGaruY+WIjzGpAJx7U6ZiqS9EThkXbgcDPSjYNb8ODMke49QwFOtM2LZdTlvL/AHL5\nc9FQ9BW0h+KLnVPgiTS2ieeV+ISFydqsOtLNIaMiHw58M3EGt2tzO4j2vkj2PGP0Nem+EHiaVY7t\nsEk7SP0pXyUB1RuY9Njk+EorU5KpEFwDwzV8x/heq6ZflhZSIofIKpu4z7VOMqY8qpFk0EkVwwaK\nRcNuXcpHB5qq+zJudQx3HJAq6kmTYHJBIYzjIJ5qGnSSW2oRyOSpTkZ5zTAQw0tXF5LLPuMQIKge\nuen9KJntt8jSDJLHJGKCYSkBVidmGCwwpqiC6ktZGaEgFwVYE8EEd65uZ/oeOM0Op3lrqF1psxTy\nmAQyB+fN0/xSx9EuHZVFtJnBIBXGQO/0qXHyVg8k5OwRtJZhvYrHHjPuapnsx4AMGYwRg56mumM7\nQrW0V6OJbO8aJxlLpfCOPqOeK2+p6Bp1/wDEzMVurSJYVzJF0LYzgE9OPSo8raeF+JZoPJILKBP4\nMJrVZ0KXInO8Mc9QSOuAaSXNzrVp5YmW9hm4kjeMbSP/AHD6d6bi6tEZzqVCY2kE15mzT5WTo8Jf\ncoI/2n0+tUXVvHb3St4jlmGcKv5T6HNdCZCWvDeaV8IvffCx1CSXbIxBijwOV7/3p3c6BpOo6THf\nxqqPYj8aJP8A8m3rn9P3rncnZeMcsxF3dW13cPNsMbOxzEANq9sVVu8pCALTokyy1haRguNzscKP\nU/8AMU21DUPCtv4e0zyvt8+OQDjOKNWLdCCz021S+R71mdZMq24jAB7/AL0JqWiW9rLiF3kjzgEc\n0JTcZFY1KI70X4cwYZnkW3YDAPViPfmrfibRpYbZJGujcZbChgBtXHarpnM3pgtTglgvNsg5IyuC\nORT3/T3WINH+LbV7vcY38nXO0nvj0rS1FYsZ/wCpXx5Lrt2dPsZl/h0WM7P/AMjetYmzlSK8ikk5\nRJAxx6ZoRjURpO2fTv8AUX4l0u8+FrOzjxJdFVZQOsYx3r5bv9qHGmlpp+4d35FWxNxzkmqiDSwi\nheJnmViVPGDTJ9eMEShY1IHAy2Kwr0UX2r3N8NhbZGf5VNAZKkf2oBiqHFjBJqVg6wwh2tvMSPzE\ndOfbJphZwv8ALFCMn8xz0+1ALF98XRsIhIz1oXwriSQLt4PesZDXTLkppt8hZkYKn0xuAzWt1fTI\n7FLGSyWaMzSKA8chcYxnhev3qM+Tq6L8cewH8RTGxvEKTyySzsciZACAOMj+lJp9SkXi5tYt47Ot\nNxyuKByR0TJf3El2EYjaXGBjpzjFaPUTELkrFEJPCIAVUzz9PrRl6RSBjpN1O5lMDjdz+XFFad8N\nagtxHMbZmQNk4z0/SllLKGS0daha3fiXMi2M0rLAI4gYy3vkfTpWYfQtReY40yfnncYD1pePEUno\nyt7W4i2pNG0TADIIxTAEQRkZJ+tUiziktFlzLuOffmg1MN4/y/zPgSHlS3Q/etelEg6xsf4XIZ9V\nkcxqRtgC/wDdHqKuivVu5QlvZw+H+cR+Fzwc80rbbpD4gf4guJjqsdxLEkMDcCILjB9fpWc1YQ3F\nxuthtx0Hr606EWuwdIgAD055r1MMackKGBIIAOTkAZ+tb74eMkHw5ZWN6Hc3alo2UflUnIBrz+Rp\nR0aCfpbA2p2OsG1szEIJp1d0ERBKHqQ3rjP6Uwj1GS/1OeCzAeG3/wC9I69DnpXHNJx7JlUkdv8A\nVbXQ7aOa5USK53LHGBkn19qz+iaw+rR30cTJbsknjwxKo6HqPeocKcm2xlL4TuLi6U29xew8hZCn\nkGJOOAR26VyXR47m4hD27W8b28eJoJcOhbqCvcA55rrimURG90nT4Sm0z2stoR4TBg4IB6kdepJp\nZd6vpt/LEZIEN7bOWiukOAzbgclftXTBKXoJ4hbrejw2+pSTRg7boCdMejdR9jml+zI5BUAHHNM8\nZHpllclvNO+TgMR/MRzROk21vcySwllkkj/MOop7tEmXS6BAhLRW4Gec54/rSS6trmaR4xNBHGp/\nKZ1z9cCmg/2ZWSXTNMS2Bvb4PI2SPl1LEe3OBRFklqlmkcFvLPLyQW7/AGA/vT2UoJXSribS/Bi0\nsQSeJuWdl2sfUYNM9DhOgW6rfKy3JJZAP5lbGePt61NzwLj+h/FY3T3kW/8ACjbBEjcjPbPpVWrk\nwX0qzo0TknbkYDjrwakmZQwpXXpY9MayUAAnhjS+PWpoWwGPHUGtdCO2HprazWwZiAM7SCOlBX15\nHPauqImW6EAUVPTdRKI1X8wJ47CgJok3YjDZZuOa608AMrktFttkYZg4kIP85HP+PtValzA258Hs\nQc0qCwRrliFjZGwhO3NMLLS9NubRJbjVUtJ9+DHKgIx65zUOSLfg8a+jGW208xlrG7jZIX2tJIfK\nWPQjHbj96t1iS5t/h+L5yYNmTcCJPyjHY+hNQ6tY0dCpK0KtYkX5KwmRm8NotjEnKqw6j60FZ2dx\nqcU/y20pAm5mZuB/5q0HUSM3oPAG3gltjICRhsdq2t6Re/D9nBIu53hLK7Agq23P77SKbkrBoPGV\n6bb63rPw7Da+DttYXLxyynzE/wCOTQevaRdaJLFHclH8QZUJ2pIY6Qk42rEclusrF9pyO9EJ4V/b\nfK3u2Js/g3OOQf8Aa3qDx9K6H4RijRvrq6Z8Gw6bfK6XSkqojPYH82fSl8moG30qO2t32fPBnckE\nHA4x+oP1qFbZ0rFQitFN/d/L26O8n/xwKev8LXMEPiPNAoC5OWxj60ybRFr6FDToNNRWEo+YmiAT\ncc7D1LCs/ewvJO5jMYwerHr6k1SOkG9EguJ9OuJbSTzI35cnpkZ4qyK5ukTwp8oF5G8c4p2kyilX\nhrfh7UElgWIq7SEklz+UCh/jeKW5eFUbdGkeSPXmiRfpm7L5Pf4F9bkxFf8AuDqtaDR9FsJ7hLeQ\nxzTRfixPH5X29efXnFQnJrwfUYC6hCXbxxN4m1iAQMZqdhZtc6jDAVwXcZB44710p4MmQvbhrq8l\nnbrI5bFUAHNFDFqrmiYkVOTjnsaIApXG3agHPbPFWSSGaOOPKkJkDH1rGKntyi9+tDyR4B56UAjX\n4Y+Il+HZp5jB45mTwwpOB1yc1YusxzXRa2hdEk6Rk7u/Y1jMKXUJFDQx26jHJMnNcctfuFaV12jg\nKQFH2FGgFUOlmFLsPcK3iwkDAzg5B/tWo+ENbuoki095YZJYf/1ZyDleuVJ9CDXPzQ7IvwyqWmR+\nKr43/wAUXE3mVY2xtY52Y6/vmrLfxtSs3tYo/Glc70Ynkj296MVUUjS9Y00H4USeFpNQDxurcLuw\nfuK0N9EtvDEYgquvVgOWpJybdInVFdveSy5DSEe9MbGdwx8KYkqOR60jYUO49rxCTad/R89qiJPI\nWDHaOozTpmZlNTkA12ZkYMjwjzFuARSq91JPE8OMhmbpzRUq8IuNsAmuYkO2RmY8cL3NXNapYKry\nwAiQZAeh2HivoPdXs07q1xIzqOBnkADpRXw5cGPUBKoBUKVY5A2g9Tz/AEpo+2LN2X6rYy6jMqwS\nKwXJBkJwKD/9Kl1zLchWHOI16j61RvQQR24+G7eyITxTMWVWIPavVrK9T6R8S/B1i4tYLaxSBw6P\nI5XYCq5zwOppVLczadqstxfucG5KwRSHblCQMg+wry7UlTKQRrY7kiJWgZnRx5XHpjg0Db6dLYaX\ncQpcrIJmZmY/nYnt+mK5nLKEaadmYvdMumRlKuT0256Uy+BdJsrK9e+1RnjuYGBgT+UjnJP0pYzQ\n0U7tg158RG91+4lF3sUDEMJBAxjjtzkH96Njhm+ct7qOZvmJSDz14XlR7VZZR1qn4C6d49nqU88s\nSSpeyBUaVj5M5yMf87UZpOh6dMoaWO0acK6HwlIBPTgk1ZJ/GTe+gGq6cqWFvDK4gNrI0bvKuBtb\nOOfYg/rQWnafbJq8aXJjltw+GZWDBh9qeU7VInTJ/EPwjNbSPNYIJrdm3IF5K+3vWfsrhvm7e2gW\nOBpJBHISm08nmm4eRTj/AKScbY3bRbfXLy6gWZ/FUn5UEkIDnhTnjpVtrHpWiabLpepQuJXJaWWK\nNc5B6At2HH608ZW+o6h9BtPhttVx/CdIgt5mfarSq0uPcdqdabNeaFuW5ZdSEjATTxYCwe20f146\n0vd24sp1QFrFxdNrEEV1KjaXLI0kDKxzjHOMelDC3t7e5jjttRk8BxvVbpS3iqDwQe2BxSfUBBMu\nsS2V7HcWsy3ELt4M0edw4PB+vNbaKCx1jTkGoRtLG3lQMMMpBwfpVVH8Rl+jGfFfwldaNE17Ylrm\n0zzwS0Q9/UUi0rSbvW7lEVJFjPJlK8Y9qk27oEuPbN3D/ptaTac6JdSxyyAAMRlQawWp2VxouqT2\nV1gvEcZHRh6ij10zjSBIbvzNHxsYgEnqtFaPp+3WlupfEaG3V5dp6HYMrn2JxXXG0tIJWxfLuZnd\nwpkZixJU4yeevehrqSaFF2Ic4ySozWQj9A2uS65f7mhZkk5baQp9BToyCtH1BbW8Kz4aC4HhSq3T\nB6HPYg1rdQSK106PEJuobYbZBLyfMODgHp3yKjyxpo6OJ3FiXUmuJPhxQsUIto5wQI84JIPQ/pTP\n4jgj0T4estLsg0Zuvxp2PJJx0z35pa8SNL/RZpzoLqAXEn4RlUPj/bkZ/bNa63C6il5BDK0q20iy\nK5YkYLbdo+go8iND9DvWdcT4f0KSUcMq7IYwO5r5bPqWoa1qCG6nfxS2Ax6D7UsF9Zpbg8sEt7e7\n8HUFklQsFPhggfXOKYfEul2FncWradE8loE3Tu5YhPuOhpnPRVx5YtM660Ba3ciK7bjbP02ngBT7\nGr7+aFdTjthC8ktrGkbBjgZUAHGOxrf4ZP6wZ3uLeWY6ZbP40b+VlyePQj096bWemyXN7p8ty8jL\nIgklSUgqx5PAHbigBsBM8Zv7mbxRLJISf/gPQe1L7+KOS3djhWPGaokccn+QZ8Ow6BqMCQa0ZEni\nBEciHhsdB9aVrYXWpXWy2gkmYrxtGcKPX0rKTRdq1gx0zQNVuYStpGTtfDKHAPTmgvime4n1YrDv\nVYVWML0wR1FMp2I40D6PolzqmoNZ+JDHKActI3BwP7/2plpEUkOorMqhjHbuNyYycE9P0qDlcqKU\n+tgt78PveXZuIgsJLc5UDPuKDGkzWF7dXFzzEls+2VemSNoz/wDxVSMqdElPTNbPOc4HvVrxKBuX\nnPtXREc4iNu5GPtRMkIKgqDjFMY1Gj/B0H/pu61LVmaPaMwgHAP1rLzssEh+WOecqxqcZ9mx3Gki\nL6lM64Kr+lDmdm/Mo+1OKdt2HjoXQMpI3A8ZH1rS6Ntvtegt7eFowXAxE2WUDvk0QG/vdBguJxjG\nCcMx6kYrMX/wo1q0jWiuXXkgnhh/mtZinTdJmEr/ADqlQUyAP6Gm+hWMempPNJFHMWQjBGSv6fau\nTk5fiL8aV6ZrWLZLRXW0iDxKNzlgN+7PP2qzRbY+PHPJuGCPKh6bun7VRbFAkusqNZps0EE3hpwv\nck5Oav1GIOrsMEBetIkB+meWQpJlUmKE/mVcinOnDFxEzxyFCeQ3lpGjIbXt4II/+lwJHPOF5HFJ\n3tJGhaS5upUDrlFDZLH39qRy+AasXXEUEUi+HC7Lt829upxQkWn6e9zveGWAcZxICfsKKj9FteEb\nrSo4dRJ02WSeMYKtIm3B/wDuuXXzlzbC2kKEb92MhiW/r9qZL9jJ/EBizCs0c8pDIfMpQ5FT+Vgh\nmAeVgDyAE9qqgdV9CY9SsrWSKR7ocfmR425H2rQfMJcqghNuu7lceXP3Na39HjGPwhf2914RlC75\nQQPI4bIxxXqRzGcZfDf3Goxys8U7m3LDyk4BzVNvEuuWbNLbw+Gj+HmYg7zjGf2ryIytgOwaf/D0\neIIsSx7QqRyblAwffirRbnad6MOcg44pnxuTwH+lo0KWePcsbYPOSOtR/wDTUuQQmP8A3MKlL+HL\n1B7lWr/CWlzWni3lvEb4LiJ06L9azNvYTR6kviSuETlXTsxHGM+9W2FRseEgi2W4hNu13bzMVbGW\nYFV5xmhr2yIsr14Hx+PviA8xxnBx9+ao5U6KpWN9OvLu/WISWUc9vLAY38UA7pByDg/THpzSSAtb\nXRM2kqomLR+EkYCqe3IqkHhs+Dm6M7aFGtjDb+NuO2MyY2fc/wCKD0j4dfVrcXF/HbWkqPwiJxIR\n1Y/+KzqC7EnhR8QQppRjW5ZvDBO0JIVUnB6etJZtdt2hs7WPTGvZuizXmCE3H0HYe9aL7NMKlg41\nHVjpENpCmoR3EwZvHEK428cbccACkMuu2dllrG2YysSXnlfzEHqBimbfaxbFd/rqX0yK9nuCbirI\nu5/MOf8AgqzTdT0pjClxHcrJBGYUAjDA8dW5B460yg3QOx21niXXTp6SFrUyHDY4bcchsftX0rQD\nLLA63KFWic4B756VV5hRM01ou6KVWA2lcbT0xS64+G7e5ZzaN8rKo/lGF/T/ABQa+jCm2uNY0/UD\nays2/nw9+MMPY0r/ANSrKXVrOzura3JuYsi4CLyRxz/WsmkxZLD5RdSFJPDDEE46Cn41GbT/AINi\n3/ime5aMHqQvBP710Sd0c6+mi+GPhhGtY7vUZPEWRSyxOMBB2yazXxVcacdTlOnjZEnkwB5cjrio\nxTcmwzpRM1czxgAKucHkYxROmXYnkME6k27kbhnkD296u1hKLH+kW+lptEcUOVZjJJIAzIuOCAc1\no73R5rrRzLZS2V2kMe4y2yBZJPYr1HB7VGd/To4/0ZqzsPC01rC73pHLcnOVIYDAw3056Vf8U3vz\nt0tu7uot8eAphwWB6nOe5xSR/smNNHdF+DL3Uo0kdGhi3AneuMr61qrD4YudGEUGnKZo2dzMzcM2\nQMcexFPyTtUjQSTsy/xhq81/MsYh2LDlWXphvpWIF0yXaNIGcIwLKDgkU3EriTm/yPofwm0nxBfG\nAKIEERZi7A554B4z39a18uk6ZPp11pfhhWuI8SInseDUeSNPC0HaMbprWFkf4LPbL80J8CVl/kB3\nZz24oTUHN9HNDCyzaranxGeNf+8nXA9wD+lO7tE0qsf/AAbOyfF8khUlJ7WNQCuMLtBJ/rV3xljT\nnmkgKiZlJiVB+VOh+9ZJJ2BvDC6Vpur6lcLLYWErQtwXbyrj/wCTACmF/pdvFIsV7rFhDgflRzK2\ne4IUY/erb8OT/nbsI0PTdHuHksYL5Lq6ch45FiZNhHTOTzmth8NaYuiXl54sqRzmLeV/LgY5zntm\noSk7po6YwtYZrT76bTL6S8WB0SK8VyTyGRickfY0z+I7K2X4ie4yjRTETBeu4nHXHbPNHxBce2EL\neN7z5uxhTT1F5GyK2zaVbno3WgLPTre2+GXgvrVku0SUC4jJ8g2t1pPo6VIS2OoQppzQ3EhG0lYy\nDz612DdquhXTW4DhisMiyHn68VVxfpwV+TMxPYNbztuhdsAg8HrRtpBHLHGNyLnyk56H6VdFUw9/\nh4HzrcQHaMNzRei6VBbavBJOYmRTnrkE0WrQy9HXxLrNlfWf8OgVgpJLFeB9KyUtjaRR+QAdgWbP\n9qXjj1Q03Ysk0+SUsYmRwD/LURpksaszxHIGfKc1QQu1GyjWOIwKuWQMcdc+lP8A/TqGL5q7lmQi\nWHG0sfXORisA03xVcNBpyNFOsMqt5SzEB/akUOqTfK7riUhX4Vg3GfSlmrQ0UW2msIs6h335IXlu\nMmtNDqElheme1SIMpKbHPkI9TXEk09OmP+GM1W/MvxFJNDb+PGzHxcKRuY9ce1ckuHO5ovwQzKcA\ncDbwP2rp+EPoScw3MjRs5Rju8wov+ISmIxsRtI78UEMxpo0cU1u/jPGioQqquSRR1xHZz3atHCxR\nV8rM2ST/AGFTl6NHyyE8ixxmZZU8h2gDqRiktxdiRtxLA1JLSUmD/NM8cn8qIM571VFOHfEatnOS\nGGQaosEWC64knEhw5Izkgc5qKOBLHtRlAYHI4zz+1Uik0ZS0YvfyWmYZGiljMmWYAbn9snmq72Bp\nYN8ZzvyyBudvbHFNVFPUI0eWK5X5oTQxpyx2Z3c+9aSL4z0vwxHHBK7gYUOgxQd/BGhZLdyXl206\n4iLEeWM4HFeqLTYFN/s+8/EVjYy2266hRnZgoPQn/wChWA1vVfC3x2W6K0Rd0SIu5mI/3eleekoy\novNYG/Dltd31zBfWMcV6PC3/AIy+XPTb9QSf0rXTo1wTIjz2x6PDIcKG9KrBSqmv/wBBVFsTX1md\nxm3hgMq7ZxXrvVwir8xMEUds08+RqNGUdsUSaza3moR2wcBXO0P2BrMJeahdajdLIN1uWMaYHQI3\nOB39c1xL+1lor9C9p7i51QafZTOiuPGiZ+AwHb+vWnXw3aMT4uZVByZJEwygdx98muxawyk14Eme\nCKQ3dteRfLbgsJD8x88qSeOffFDanPcafPPHDFK3iv8AMLk5Ugjse3XFFr9BiqVsNsmtbnS4r5Yg\nssTCN1cZbr2Nc1KBri48ZUdNvHlkYcfrU5SymRctF09pb3MUY1K6leCOTcd0mQO1KbvToEuZmhtk\nTbHkMAfOvQEDt0zR439GrBHPGNw4z+9Dyof+cV00iDZtf9OLG1DXF1IivOMKN4ztHtSPVdIt734q\nuJB4dqplctk4Qe9NH9oovCNt8M7JfmHmDwghUkg82T257Vuvh7UILue4QyNut22EsCGYDvj9a0nb\nopHw0ljJDHa8S7g5/MepqoXoBuHUkYIUEDpRH0R6uj6o8bLqDJHEeCijduPvSC/vda0m8ubW4uVF\nv4YkEjAeZQccd92aTroeyaoyF7Gl5G0Nhp7JEx8txIdxZie5xwPpWxsfg3S4vh61ttRmfxIS0hkQ\n4VS2D37dP0p3L4iXStZmviD4gMd3dadaXSzWirgzIp85PbNZie1aQBQrEKD06VbjVKyM3bAJ7CVu\nQyjI456122BtfzYYn07VQRIYafqHyF6lyih4wu2RCfzKeDToWmo6bKb34bvVngB8UwROC0ffDD0q\nM3ul4eGw0HXh8RxyA2PgSJ53DqDyO6E9fp2pH8TfDtysr61ZXqXBQh38UhWjHY4PBHQdqhFqM6Hb\nKbf4x1K0sbiS+SNdkWYt4OWbcB+mDQr/AOpGpXRjWIRwOHU7o07Z561XqqbFUvlAPxHbXh1me5ij\naUSSBwyAucMARwB6Gk50PUpJzIdNvSTySIGwf2ownHqCUHZrdDsbvSbK3uk069FwH8wBIXGSOR9K\nefGi3CT6ZeaajrdySeHtUbSwPQVKTTZaOI7o3wjcQ3T6pqKg3ziQ21q78k4Iyfbt96H+DZFtNI1j\nXLu0WOaKURc4wT6D060zdieM0mjhLmVtXRooUmUCVpOAijsF7n6VX8VajBa6KurWVvBcPE/DTqSB\n1BIH09aCSRpeHzfX9UvnnaS6vJZkuF3QIGwgUjrtHFJEt2mHlz05q7dHC7Zq9DXT/haC1vZGWe4u\n1ZFDfljPTI9/emT3tpqfw7M000q+HKsUybizopIG7J5Izxz61zO7s60lGNBWlDT7S1ktVlQxqm9d\n5yjDPv06Uz1rR9M1uzt3lgkt5DGRbvH0wcYyR1xQlJx0D1YY+00HX9KkaGKVbZ5gGVi4BcKeME/v\nT1pNQtPh27juozMLuyJaVcbQwU/+KWfLBtXgVJxVMwEBDJtmCnnPB6CntnafK6BI+lKTNLLnw3OT\ngDt966n/AIcr12cTXgoCXlu6noxC+lIpZ43vGliBRXOeeDVIhS0LW62gDccH0FWx3FvK2HuFXA43\nDBphgpdOhEYeG+ZmbkBYD/WqZdPnwNtw7ZPRgKNhPR6EznfJOFYnqP8AxXJdCYOXN2oHspoWGiVv\no6RTI5n8XY24qU600sLJIZZJIpFVpW3E4xRTN1J6vC99YNBcvDJGOmTyOO1Z26trQabbxCadZkBE\nieFlTznINLJspGNg9nf2dqx2WM12VYMNx2/tWt1m7a0e1dwdt0ivgKRt46Y71Ka2xoeNEoLNbuaJ\nZTshzlz08vWqrrSLGOMePcvGByCSoB+pNM/MJxRXPJE1lEVeJgnBYPz19qW3sjXEjEsCCBkhTg9q\nCGYd8OSQzTXtqGESCINKX65B4xXRq1vaaqskTIFiIymf+7jsc+1R5Ho8XSK7u+N5dSzJtSOViyoj\ncL7VQJxxk5Fcj5GmSkyxLy1RSLmR4lJ4Kpv/AF6f3rSQaNa3OlpcaLeLeyA75Y1IUj7HB+1Xhquz\nUjN3do0lw8gWJQxJ2r/LQNu7xiSOSMBVzyDgg10RdE3HQdJHBdckuDkBhViPLKzEE5UDjNNKWBTo\nO8H56MLJGxkVhluDx6YqH/p2zjumkjMZc+o2mpxmwOaYztNPsvKC6o/YZzXqsoqhKN18U/FUUqxP\nHleRtQ4bK/zH+lJksdPu7sNdT+ArqWJyMLzwK8if9jp7KWDz4W//AEfcPBbkzQSOTkdOuM1p5buA\nSeHI4ZmGcdc1Xif46M40Z7VNTksJCiyA7xlA3OKzV5fXF834rBj2CilYJP4LdSlOkRLLdI8buCYl\nZcFj7VZb622myw6kcybyVBK5wvG8cd+Af1rnaakmX4v6hlvCNR1nT5IbaPcoL+IJRyGJ6jHYf1p7\nc6nb/Lfwuyi8NMbfEA2qgH83vk11q16TnroyelPf2ljLY2qxTFCfESQcAZOST3JrRWTLJYJeIhLo\nxiGOiJ15z15J/Smv6P2Xg1spRcpPF5BIfOBt649K9C0dyu0yJnPmYjP2xU5V6Ql7hTc6LbzWj2pT\ndnzrlCoLfen9vo1vFpgVolZmiVCzjOMDA/SmhVUikXh89/8ASlxeapJBBgxqxBlxhRTXSvgCyubm\n5ku53eKF9qKBgE+5p23RNQ3TUQaPFCwESLCgG0MFxn/zXzn4n0K+S+kErJJbPIdrxnHX/cK0HKGs\n6JQT8GfwdNHYfEb6EY1+SFrmSVl/NKcMDn0A4pHpl5LH8S3lzNueCzBgkkUFd5dtoPPpkn7U/ZS+\ng8dIb2OrwvA1ul8VmgfDh3AH1+9GJeTWzhvmFlMjf9rOC2eOKHgxTrOv6Wl1ZWtqF+Z8cMdvBj7Z\nb71QNS1S4uHNu0ZuEuZI5pSBwAAVGT070zYF4D6X8Wa7d38lu0Md1JHlWieMAEfbHPFG6m8GuaY0\nMbNAm0m4jbAPHcH0zxilUllCyjhgbt0tyZxCNjMQIFwcDsazEmpTPcOQ5XPYcYrsg7RyBME6sMkZ\nbHWjbDSLnVZJHg27Y42kIJ5wPbvTN0FF2jWw+dJeLeQjNHuxjcBnB/SmAnvrNpDayCznJ3hYx5Dn\nrgj+lTZrof6Jaahf2t1GbhJXkiDCYHbjIOG+oNNry21TSvh5v4pFba3wBtKlX2D1IwT965nVnRB3\nHRDc/GEFtYraW3w/ZRA9EkBcjP1JoW2+LZ2R5JfCsSD4aCO2QqTj125H71Z8eWFT2hpBealrnw3L\nHa3N183bRkx/Ly5WZSfUdxzx+1Zfx9YkmV21W+FsmDKxmYFfbB7np+tCCV0zSbrDTfCMd18Rpfs1\n7dKyYVAs7bVPbvV3wtbaumrt/GGnf5aTbE8rEgkdxms2lgE2zWyXEqXKyGZJV8YbV25aM49feq9R\n0mCWC0hmyY7mR7gwLwGYDqx749KmhpUL5757i9/hkdzsmlTbEyDCxt/LWdvbjUpLOXTJWeRmbzRg\nDLSZ6ftTX9YjuhRrHw7f2VojaxC8JiH4argl1P8AKPoetVaNDNdoZmtglrGvcYDf3NN2TRDrTJ6u\nkNzoUt28qq9q6hIQOQp4ytS0+aCaHUmDk/MtHtUcNuwzH9xQVlJVJHYJ4YZJRFu8N7d4sMMkZXj9\n6IstcvbewsbOBZZAsjkKOTg45x6ZBppK/ScHSNrpF9d3FuLyWa3lgCeE0bL5o2BHOaYx6TFq3w7J\nGheE+Iytg5UrjGMVxKKc6aKSdnzrW/gW80JJJUkE9snLSKv5ft6UJZK0lnDFFI6yo7OHU+4rvTsh\nKNMGu7qbxpFngSfeArNjnP1qq6j0/wCVI+TkikDfnEnP0weKqkDQe3McxaNJG2gdRztqi80q3SVf\nAkuJQRltwwPtRTCHWGq3kbrbuzGJRgBxyPvTCa5AkVFcMSu7isGyy1uVaM7uG96lK5JznJxmkZWL\nKRc54VfvipLIdwG7rQTGD03RRNcvCZERSOnBPp9aTtBd3FyZruZLdJME8eVc9BgVuwGensZ7C+aO\nJJGjG0PLH5dwYZP0xWltUuZZdPjune509YirIHKmQ5OM49BiklIeKsNigZ5p9K0/Sdu2PxFmnkwc\nE4yD3+lZnUI5Xc2qzSXd0rFWj8MFFGeeWPWnjQkovwkHh0mRLS6tHDS+VDIgCAkHB3D3xx9aloup\n3Eq+PqcJjtLdwI4hgl39gegHJrSdLAqJ3RrK4i1u5vtQtHmRZPP4oBWTPIz7Uq+ItEv57+W6tba3\nEEp42EIFPoBmoqWqx+trBfYaXqHys8rgRC3R2YluCccYxSddXuUAPiHpVFxQn6QkqDYLvVLhQ0ds\n0qnvt6/etJodleS3ELyLDbPu3B5DjbipvhS/qJ2CLi7L3UkkoyzEksB1NSivwscsUqBhKm3D8j7V\nurQVIXXFst1GGt5g2zaojI5PrWot9NsmhHhxICVAcAkkUzk/AT1WjslhHAN0aspPvQFyAPr7UGsO\ndAROG4/Mehr1IyybovvpWtrJLlcqpcq2/OAvajtGge/t7p2ZN0IXGMZwf5h6/T3rnpPSsY/R5oX+\nom3UBa6vaxpHK4jWVPKVPQZ9s03gnlt9Tui6vIYUYlE5b8wAwO4IPWlbdHT/AKAah8V6JIhW+iu4\n5E4KNA27PtikVhrjanrkNtpGnSKwcMGlJJA9SO1CMW5aTavT6l8QfClt8YaHHbXrbLiLlJoxgqe/\n2IrBPYfwnUPkbm0ktltCJIpGGUfGe+OpzV/5PCnUv0NwyX9WG2cTHTvCiTdcs23xBEEJByT29ab6\nJpFtDps99qrxlNrKkZbjp/Wo5J+0CbXphZB8xBdyxRMZBIDG+3youehNNdKR4NKO9NzN51KtgFgc\ngAfrRi7snF27L9P121SfcTMp42YGTn3pnqdkbuwna2EMTy4YSIcNkdRmimjdrYl074qm0547O/ka\n68u5mzu2egzWv0X4qh1y1lG0W7RnaqO3LADk08ZJ+FE14g1444l4AA7Z71To93GJXtTkszFvUZ9D\nTPBvujS6uF8AxouWYfpWY1e0UQ+JPtIJ/m6D60z1DrAAanAmyOIbyowCo68VLSrsTTypIv5088b4\nAIB4P9a4+tSszF2u/BNvbXovbaFb2RVwodtpD/yknuP8Vkrz4XntAb69v3F20hKiI7tue4x0+1W7\ntMSU6CrKS2vPD/iy+JNABsuY02sf/n69Kd3Nq38J1CSRoZElUSbo8E/U4780nJdjwfYt0j4ThaKL\nV7a9ZjHEHXJyDIB1NItRuJ4r46biD/q1bczyjqRwOT06VaKqrBPbSM58SfDeqfDEkLahaPEHbyOH\nDI5HYYrNanOt5eGYIke7qqjvXZB4cTTTPW0eME8YpnpFxcnU7a3tkaXxX2tEvBcHqKdvBkbWy0aG\n0S4W8SNWkjIjz5yp6nOPuKTm3Nzdsvgm1CHOCuAPcCuWPLFsMlRorACP4OnuLSYQXO9o1nKbQ2P5\nf16H1quP4qv/AODmWdfFJUjBGcYGDn71GEnJsrB0jJ3cC6nmSDKXG7JjPUr3Kev0ps9hDd/6fSXS\n7ma2uRuCpg9Mcg9O1dMsRlrKNAEyRC8sZJba6WPLLuwFUMOfp0omf4v0jXJvlddtSkhbHztuQhLD\nuQByPft+9GSuWBTpUwnRdIvdL1GRvh/Wba6eUBngnXwmkGeACeG+oIrX3zzSpbTz28lrKh9QQWI5\nGR1xn9qjKVjwiLNK1lbZbm4vi00hKpHFwWHqxHU1da6k+pTwxW5gMcUmQgk84B6nBrWM4h2taZHN\nqdi0REMjNneOMsB0/apR6XAt5LdnMlzI5ZWJwI89vc55oWK0KNR0e6uFabUbp5m3MsaKPKMjvSBN\nOu5wmn2UIe6dRu3SYA+x4/Sin8FlH6Zz4h0/V9LeWC+sniC8F1GVI+te0sOLa3kCt576NRgdRg5/\nrVF4c9Ybqw+BLSS08T52QTpkFCOD6VRGBpmnNBAdk0jbZpSo3Kg7LQbbFuhelvY6dbXMwuJ54SwB\nMb7SuTnkEc/atR8Na2ZrJprYssEZxMpJYscZUj61F/joVKzQG/s7u3IYna3BV/Q1ktb+Hjp2owXm\nlrF4cTFyjMcdc+lPGV6dHVSRnfjG3ube6h1MPBFBeOQY4j+Qjt96TypFOdysSjYIU5yKvCWEpRpl\nCh4pSqsIkJ2kupx/9VdMz2zjxDCykcMhyDVVpJuiMqzGwW7XmIyeGSB0NG6Z8P6vdn5oWuEPKl2V\nP2Jo2jel38OuI7f5gBSpYggODgjjFaL4b0WLUdIMtyXjcsVOXwfsKhytpWisHop+IbGTRr0Qkl42\n5RsdfvSf5jz8Kc+9QjyvxlHjG8jGXRvld8hklbd4YONp7fUEUlfUVn8K0j3PKrAHd3xj161a7Rmt\nHVzpNzq2q3U8jpaWSKpeffxGAOfuaYSXKvpcVpCsjpFEJFmRMA/UjuetBp0ND12Bw6+un2/jWmqz\noXUxvujGVB7DPbNP7bUdI+ItIiht5kTVoiFBK4E3sT2NZdq1GdXjFFvHY3GnTQalvSbxNshRgVVl\nPrjOf8Upvn+QKODHnzDMg4J7/ftSJ7ozQfHqLiCGTwJGjvAm12fiLsRwMcmib5EntoLfJjKZ8ZAu\nPMfT9qpCKJyk14Lf4XCuA7S4PUFiQaqNpapN4Yt4EGODs5/U1S6OZybLQqglcrjsAa6ybQPy/c1k\n0xHYvv5vBQkMN7cLk5FK7iS5bzO8bE9cGkk1YSyC/EaAxuqyZ5GeaYaT8UzWj+E6qYmOCyrlhyaz\nj9GT+GqiuFu7VJYpGlRv5iMUvvNsbYcgE1miTW4Z++uUjugrygA85yRXqm4sqkbKVptDmSO6dTat\nIImVlOHB6n04/vV0el2qXlvcWdqygSmUSY8uOcfv/SuBXHw6eONLRL8S/D1tc6te3ySNbRudyRIu\nfPgZx98n70VZfGF/b/Dd58y73AgnitotwAcjDEgn6DPNWTfInGiuGus9QtPiP4f8s3kmXEcoY5Vv\n9p9xTn4N+Gk0Ow8aYtLdz8SSnk/Sujhh9fqIzeUjT21xHEArEZc9BWZ+NoPm9SiVo3ljijBZQQAD\nk8mq8z/Bk4rRNdNqZ0hDH4Nud4USTPglKzja1NYX+z5pL23UedI0IXP9zXjzfaaZSaolNdfNxiKI\n+GkzZdDjBAPfFHXd7AssUPhkpbnaGHABPb9O5q/brpFSo5Z2EcV40sg/BBxsVgTj/NNT8vFOY7Ye\nUeYbuDzUW01Y0WKNdsbbwVuEyspcbigwce9LpNNWPTJri8uxbWkTFmljfzEkDyj3PFX4XhSK2xPa\n/wCp+pWY2JDHcIqlUMuc47H60NZ/HWqtqFncOcJFcCR0iGN4PUe/AIrqlH6MtkfW9Y+KbHTGtRcF\nokuQCXYY2Z6Z/pUGubW+JQFWWT8q53A0n/yWeaekso4UDIiLjjOAAPrSqWyW4mkcBidnO3jIHpXK\nm28Bdi6+t7yxvZ5p5naC88oG7/tgqMD25oDVtGvbiaJoIXeMRKGctgE4FdLaXpGUbBJdLksZkSRg\nzMuSVHCcnjPrTSwEccuxifDmjKPgEbsjFJKSbGguoLos91pxu9LcFYZFcRAnJIHUcexzVdr8IJq2\nnCZpZYLyEmJzJyCOxB+mKEuan/8AhTBnq+n3WtWtrpmoyF0hKmKbeASehDf2NZP4s+HjY6Q82mRx\nTQQTPBOqrloWB4LdyKbi5u7wnKC9EFjoyyWqNfStaCQFo2ZCQ/oM05+F0tvh3XrfUL6Nyscnhrt6\nbTkM2fauyU7TSIJaavUoI/GklsZFaGU5RQOOP/silT2cnzTu20K8eQHO5VGOP3FeWm4sPI7ZHXdY\nvtF0VrGKwjms5rfzvt8sbNngY9Cf2rG2F7dwq4jnkR42Ei7zkHsRj0Ndn8dLrYFY9i0mx+IL2S+S\n5FlFEN04AyAw/mHPetfoenqIJ7eHUIL/ADCd6hfz45XcO5+lVk2UjjMR8QNdaXaNDKximmVvFG3b\nwSPKPaqPgbT1nvZrq5jjdI1Cp4q71D+uOmcZqi/rYXH8jZo1xHfLZyot/buGbMNuEkjB7jaOua5a\n2uoaVbTNcyXFxGzq8ZlJ2pzgHHY84qU6oePpZJpCXmvyfhMJVcKecIwbnPuQKrvtKm0sxTWmlyO2\nWIlRCdq58vPrjFS/wp4OrSPUoLO0e5hkd5DuVXHK8Zyc81ZJdyRWUjwxSTzyMNqwYbZjvyazoxO7\n8eeWOGZ1UGQOuTtP3FD30Wnidr6dvlphIsEMikrjPX/7rXQOp6K9vIk+T1KNdQjlG0582/6diaQW\nWirFcwtaBlEd28q7+i4H5SKoiE1aNdBNIwDN4MbZwFzyazetXNkIQxm23LufwmQ4465wOPaiiPXL\nFDW5vYZVs5EnBG54HGHH0p58M2EkGgZtBvL+I0q7sFDkDn7A1Dl1UaMbD9NdbUoyYnEwCb2Ulc9x\n9q7/ABGKGKSO8MROcpgjlT3oRVJI6Y4hbo1pp2rSTW2oEFxKzRqx6/TtSj4j0aXSNSVGjZoTlkdP\nKMdh9q6ON/sWeojbaVdLtkEEmJFzl3GSPpXJrVZZPAnhTp5nIGB7cVRzXiIqD+kERYbZ4YUj8OP8\nQDt+9Ay6heNF40TzSmTIGUDBSOx9BSptsq0kF2+m3wshPLbosKHLgvtDk+g/50oH+J3eng+DcowD\nbo0353fen+CJqzR658U2Wp/DttE9u1xcOMuAf+2R7isiVthIviCWNW//AGjjvjpU1FefR3I1A0Vt\nShtotJuYHEab4xM+2U56gHHm/WlDwI2oESugCSYMqp+Xnmsk1hRu0aL4ms3hs9PtbS3ha2nO7xZp\nML9SB+b70Pf5sjBg+HZwt5hGc7uzZGOmOlMp3SFjGk2xBdafbvdNboUkLN4g38ZXsTzwMUbbWscA\njXTZFgIcvLIRhSw6cjsOfrTt0gJWO5tPsf4Ql5BcNcN82Wco+1SxGTnPUcfvVWqafFqvw5NcRRQt\nLA4dlJ52muRyLJAtjp8kkVnFbsTEkTM9vuwpJyRwOtZS11e6t2k3MSAc4cE4+9dHHLsc/ImsC5fi\nl7eXwjEhcKGPpyM0A+pXVzcg7cqxGeDxTyj2VEUqKUvbw5Lw7VB4OM0NPeXcrHLOAemFpI8Si7M9\nJCK4RQWkJDDIDcn9KnBumVjtJxwSB0qtI2FB095ZTgoh+uKvtbYrIArE87QR3PtTAGei6pcWlo5M\n+wbsImP+4e5Gewq+4vJUAaUsxYbsmpu7BiFhuo5LjEiB/QGvUaDZ9ZvJ21HRY7WKGRpUceJHKmQc\nsRkEjp71yG2fStTjthJI9lAmdhySSevPpzXnJWjtlJJHLi4ULHDdwW17A0+9ASQ0OSP81VqOi6Pe\naRJD4bwLNMZWCyDhwCAf3pofjhl5ZzTNK034YgHgXl2Yr3jwzhgr4/MMdKZWmqanDG1s15fhIxkO\n8MZ3L/WrrlcRUkzkWvXz3ASDUE9QZ7bn6cH05oD4oh+IL/UbZ4r3x4ZAu2SBdqZz3+lT5ORuIHGt\nQJrl4y2kOkmdrzwiWnlc53P6Ae1BwKtvHtgiHzL8Rv0CVwydNJEp6xhpWh7NHub+aQtuQ483J55+\nnPpTBVkt7K2e+gjmWdNrSoOcdgQatoItXTA4NNhj8V4Zt0UhOFGcx88V25aaCVJZZC2GBjPAwvcn\n19K53FJjuNBE929/Es0RWSHGX8KHeXX0A7GsT8WxX+rSKw0+9htkXyQrbELjP5iQev1rp/jtLWVi\nrQo1CTTRp8CpCI5iMkgkk8Y5+9A6bffLXSO77QhwrFcqp9cV2Rt3ZK6PojfEWm/GXwpcW13IFuYP\nKshXnd/KfYHkVPS729i+H1ngQJNaQqSNucqCwJHvwD96jNOmjpi7Qemp3mpSG2uLhGOxs4GNpA7+\n3r9aGTUHv7VoxM6xI3hhmOGYrgnpXnvl/wCadov/AM0/C0XctzbWklwo8KeIqwYY8wJH9MUt1nV0\ntmR7eZ/CzsYHOA4/8c10R5VN0Tnx9dGPwfqHzguZJZFuFGH8OSPnA/mBPB+lahp7WeF/AWLLt5Yz\nx9R7UuKTRzP0WX2iQWsfzSs1q0aEOwcsqDqe/Oavto455IWgTx7UxZ55y1N17azWV3uJZnSGNVdA\nQsZ6AgUNFp0vhyzzSKJLqEi5jX8rAnIPsaXhfVNIZMylpq0mnfNaZcqktgHYxluDGvTqeBRug2qz\n/E9pAUW6091LiZl8q8E4OenP612Qb62xGqZoviD+GPZ2dvbXaQyWrru8JRjHQkj0/wAVZPZ2mmQJ\nM6QyQyHw53U5G31/WudpXrJfRdqdnFqvwpeG0MkwCktAowy4PUeor59YWcdnardXcgbcxTwwASgx\n1I6jrVv47VOikUmNNKsVtrXUobQvLDcbUXK4yRz19Kb2OiTaRbyzR2/gM0WGLNkqT0P0p5zoLfVi\nW4i1O+0Fv4vbNd22WhMyY3DBGGH/ADmmWj6JaaTZacltbS3wkd5HuGfYik8YZRyeAKZcicaQU7/I\nf6pquoC2x4ngKT1hGz9+tJIrub5K9a5lmuYUUO8Zck9e3vnFM6oZPRnNfjTry1mkDGO6EeC/8px/\nWr9Ru5ob6SzlmZlfiMhiBjPSpeFQmC5hNtHGPGa7glJYFuo9MnqKXWLTXdncvCklrIsrMVLYGSez\ndfSgzIpeSVbtJLpnYSqYleTkbgfWjJLi2+JrNreCWCR//wAiZ/7RA6A9eRzRrQSaO2elXulzxM8g\nljRsgKcge49KZT2Nvb23zbyrEm9jy2MnAzmnbsh4hB8Q3LR6rpMsZLQu/wD3IxkHOAKWa3dacNRn\njvbVfFSQ7ZA3OCOD7/espaBqlppvh6HQrq2gS3t1yqArO4ALEdeQadNoMcVhdrpjJHJcrnc6nFSl\nFSYEnHwzlnHqS6bFp21TLDMT+GcZHpnvQTpb3EuyeIGRTtC45yO1ParBIt2VnQoXuPEedwQfyZAA\nPX7U8ubhdUgkgmVQYISVYjOR61ospFtman1R4rSASOAqLhc5zig1jXUF3G+8Fy3kXGQ31NHo27Gt\nIrn+G9TnjRA6g8kMDw31xV1jY3UdzFFeKsSkbSU43iqpUhXoxvbeC7uYIoHc29rGxKHqx2nJNZeK\nwtZw8TIQqnBOcUW6RGUNwNsbWDS2f5VW84GcnP8AWi47ksypNapdqeBHIMivOm5OVlYOlRC2jtoN\nfS5t33rEwfYSMKc/kPemGoss17LPa2scSzygsiLyOOcZ/tXZBv6O0qPPHJqq6eJp1KQOQQT6HoPe\niGsVu5DHBqiiGVyrhzgheenvR+4Mmuov175VHjhj3MoxDJIgxlQOM/ah2uYbiza3iij8KMguqcAe\n5z1NUkKn105aTvJcWUNtn5Qxu78behPYfaj4dYt2tFuIoPLJL4ciBeg7ZqE4Oh1P9h91o5ay8WyL\nQtJnw5ckbVPJ5rH31rZ2t3JAGM0qtlgowD3+9PxOlRLlt6Lr2NEj3YRMgHGBlR9aKFubezSWWUOW\nOV5C8fSupI50ctII2LNJIsaH8rnzc+mBzRqQ2xtidoLjq0jhFo0g2Rf5KVAqPAhznyszn9cUQ0lr\nEmAvmPJIHWh/4TbE2qWzXQU2MLt3ZsYA+9e0HSbu5u2EkJj2AlCTySfT7UG6Q/wq1TTIDdA210VE\nflVCC23HXn60fcPBeQRIsiiULtPHFJ3sS7FculXcEhI8Jh7V6nSNZ9it5tyOZGZgUAUo54Ge3p60\nF8U6bqct9ZTabcJGEG2QSvtVu44755H2rzYulZ2WHxNBcxsTHsZEBPcgZ4I9s0j12wlv1lETSW0S\nPtEuw4kPcL75rSal+UR/64MbeKeLTIbZoVfAGC2SQR2psuHgJ4JPBwBUk3eiRE13atabypLSPwWx\n+VfQUNbavcaZpstrFIoR87dy/lPtVvVRb1GfYTylZTcmedmySVHBotrfW/kBNpssPjrJ4eHI/DTH\nLHOR7Ur4o+sgl+Q90e+uZPguSK6aMxRjYsqg/i5PJH3rUXwt4tCtY5mjYgBSWIGF9aZUxK/IzyWI\nt5hcqxeJsxqVPlAHIzUdS0+S5gilt0Mu08r/ALfcVz8nG8o6JrssFyXV/YXTWttEd9r+MjIAysM8\nqe4Pv61L4msLHXrFdaZ7kGWNljVW5EoH5SO3SrcceqoRpxVI+ZT2cmEADEhcHJ7g1TIjGXBXz4x7\nV3IhdhWj315o18JodhjPDoy7lYc8EYr6voMU0sMGpPAbAPGVe3Vsq4JBDAdhn+lS5f8AC3FKlQk0\nTTVm+J5NMZhJ4pkfUJ1J4UgjYp9c8ml2qfCGpaRBFa2pkmje6cJIpydpA25/Q1zTlCKqX0s56F29\nhdSaNPBPM8NzplwzBHJKurKMcj3GfvV2nQfxt0FzG8KzQxsgePASVTjJPv8A0NZKMmpRYJy/GwzT\nL2FNShtZis4jcxxRxtsCEnknHJ+/FaHU9PR5vl7CIQsyEDeSDn60jS2zlTf0vtmdbKOyvUQzsm7w\nyOHAPXNKjeTTNdWUP4Uloxkd41wGTggD+n2oReUVVMlf3UD2qX9srqJTtdMeYMKBgkubae4vfF3i\nXg2sgB4GMYxUON9JMl3pgOuW1zevBFawDY7FnymCB6Z9av0OaGGWNLZGSPOJlUDDHsc/2rslL8R5\nSs0F7bWpKPPGqM/l8Qr+YdcZH0xV91As6KsCxQwoSMbQd7Y46iuPjlXJT+oVahXpr3vw7BcqZ7ea\nVF8TwT1wPQ0t1uPTdYkt7qXTflZbtcGa3kHX0Ixg12Ql1RVRA0SCCyXTBeXNq6SnZIyhlJPY+nbn\n3o/Qbi8i1MWeqbp5oyw2oclkHRsg4x9aaTUheSkcfVTp128LwNLHLJvAbBBPtij57iQXHCxrGFBC\n5zwRnFKsJxbSBbpIr0BLV5nZ+PDZOAfY0Lb6W0U8tvMMNKgUBWz0OTnHGfrXSmmi0RheWZe2EN2o\nmjGSHJ/IcZA+tekmdrtVtts0dwBJtlfGDgA4btSstZO+gEeTgGdHOAzZzxwP/NLrK/l+V+QukiW4\nm3FIgc5PvQMtFUmoXVhCLO+t5AXDHD52/UehoD4esprYC8iMtsiynGAT4mf606ZNmw0nU/mY5N25\nfCOxgxqGu63b22lqtxbPKpuPDO05wAAd2PrSSMlbF9muoXSGNdyTTSeJbuiBe3Hl9qPX4Vtdb05G\n1rI1EDBdXxn61PRpRTxie9tB8O6TcW9reyRNMwJYdUAPAHPGTmlVhrlybp521W73HjLkkn6c1VRt\nEpv9Gk0DUiNQjnNwZY1zvz3P+aZRRWaXv8SihkBVtxZm3Lz6jrUnHp6IluiL4zs3vcSadKC55aNW\nxuB4PP1oF77+B2cLXcmSUET+GD07++cUYNNDRVNmaVNSF148l54kXIQSbsFfsKIS/tEZWjvVLgZZ\nBEXUn2OBXYvMEa0ZafrEBPnmMH+wgEFz9jWg+eSawMrlJhHj/ufmH3FKm36UoBWKaeeZ4T5ZYZMF\nGztypxSnQtFuS6TSS5jMm2VN2WHofpmi43gl4HSLFa6pPBOpZI1Uq6t1Y9v2qWmxNdtLNHEkyZ2M\nQ2Nh7EVzy4urwMdJfwxbNGKqis75dieSPWqntZ7pWkDFip4x1xVoqvTMtspY4HLPAdo86sW6OPaq\n/EuUWRoZByuFUADb75rL0PbKB9Ot71PGMuHhcZwefN65qjUrW6aNVHhxkjCGNdxz7+tUxaI3ZoNO\n0x7PSLWG8IRj+IJiR6/l/pQGm6dNpt5cTS3CNbscMGTaNx6YqN3Y6Xg2OtXAt2tZwmbfhVfO11I/\nakEpmu4DCsUBYk7ZANzL7Zz0pYR+seaXwrj0t7cDfaNeOp6DlB9fWhjCHlb+JwFiWyOMFR6D2qvJ\nyOMbRz1p69i+UBeyaORGGQVG1h7YNds3juEEdxbNMcFsSqcGmhyqUQUSdBLBshURqOAiAL7mpaVo\nc0zpK8TG3wTwRyfT7mmWCULdSa/uA3jQzxMCQkSDaoA7Y7/WgrL5m0eS5lLxhEYJzyzEYUZ/Wmwa\nwFb1jhG8xHf1q+2l8R0wrsW6KvU1CUa0FDaG4ae4FoIGMnQnfux9a9VE8B1R9oxtIRsb4Vw0AYDb\n7nH64qlbyNp5YXTLIMgf7uO1cKdYzoEtzqUK6ZHe2MEqOkpjlRl/LTk3ED6SJ44WiQJkq3LDPehG\nLi2kwud+i+O8bwf+4sp744PPT9sUVaIkSD8N0J5/NkVObQEy2WE3CqigFznAHORQLaIp/EkRXUHO\nOlNxu1pVSpFraPZTL57eNMHIKjB/agdUtoYbeUwREM6GJCBnBIOatJfiSbdjFLaGz+Ezbj8VEgOE\nH0Pf61nLayi3RHUhNcqyYRhMSBkcD6j+1cHLJxWCVbO3E0ml7msiXtyArxSvll55OK0dhE9ikU8M\nyyW7+Ykn17Cr8HIuWN0dCaaoAEOnXl3Lc2hmDLMTKACPEb/Ht0qK3vi6LqFvctFEJZGit3iG0Hjh\nvrkY+1XSXrH+GLg+BdWvJWEC7WibDtKcAE98UW3+mF1ZQtJf3caFT+RDkkeoNUlypLEc/wDzrRno\nOkWmlAhWVvmCMtI4O3HTy9vrWja2lk/GDiMKu0knCnkHPv0rjUnyPszRpCfTLWEfE5l+YmxllMUa\nlRuOck/XjHvmm2n2t7BpaxPMtxPAD5i+NxyQBn6GjyRU42zTaor+HXnvJL61nhEYx5skHn0oGP4g\nt455o57fa0bFEK4831/52pFx9YpL4HtUUVXlkunmW40xfxr5TIzKMsT/ALR6c88U/spxc6faSXsp\nS5C7XJGPNVGxXov+LILl9P0zUYbtIp7ZSXVzw/PtV9pHaXd+rMrJJcxHxdpwD75oNK0GtPfEl2mh\n6JDHYIGUybZGZS4A9frSVtUg2RN8qniTDh4+dvPGQOlLNJPCbVstkS7/AI0VVXdA6krtA2cct79+\nKUX6rY6lJDGHjZSJFKrt3DrketLK6tFKtUan4e122+IYJLC9UpcJyJAfzj1B7GqNRnm+Glkt7qSW\n9guf+zJj8RCc/Y44/Wn6qVN/AR9ozWq65NqlnZvC7/OIDFMFXb0/3Y6VGPVrKzlBuIvHiVwGUH8h\nNUjFyLppLS3W9bt3nSXTMGBlBbr26qT9/wBqq0m8a6Wa2tMLdXAy8jP/ACD+UfU0WqZzcjthEt7I\nZGsZ51QwkgBMDGPSmtvDcXkFvJDaSRx42MX6sR/N9KSV0FeBdukltsVhk79xYjnHpVlm1vdau8c0\nYt7pTmJlOBKnfI6ZFdHH/XSsUyGpyNdeNDZIzvFOFZQeGwmeP+dqE0edIbCV7+QIxkKxRy9UB5/S\niVDYBFdsWinRlA5ZGyM9eKFtbVNU1aXDiO8VA8bEYOMYz+wpX+gxRM2013brK10biGN9ypKvJ/8A\nbnrQuq3r2lk7ErHIwzEQvT2HajHBJaIdNvx40rykRQOviXMzNgKR0wPUntVkkl/qt/FHaAMs/nLK\ncKi+pPbj1p2zLC+NLmyZjY3Mkwf8JJCPyg/m256ZxT/Sr+RYoGmjLzF9gYDPHq30qUpKrY+VZTrP\nw+mpRPGsjJPOeXJJXmsF/AL6z+I5tJnAVLfzNMy+VU/3Zo8XIpJs53L4M/EtfCaCxLIidJM+Z/c0\n40q1muIZruG5SAp5bhH8wKgcH2/8Uk+XaEvbG9r/AA5r58RxFFC574J65ofWNKMGnGSBd8AcysGP\nTimikWe+GSnv5CIZLVliKg+dhkkHtzSm2s7mG9V4ZbeX5jJcREKcemTjnntXTEm/TTWmj2k91JbT\nJtkCnEm/O0gcVKHS4bSGOTxfOzbcI3ByRz+9TbaZX1ELW4mhlnlt3ETp5FZh1PaibVY3kExZIZ1Y\nEmPo478dqtf0hJBuqWkMtjPLiQPFGWTCjzHIxS7TNGRrc3dy8sK7htwfzevHoKRuxo4Nn020axEj\nXibI13ByQC30zSq8sWa5QWzXOxV3EbcHHrRQH6BufDidztZhwiE+Y89wKvfw7e3RsSySkedDHgMe\nwBor0y8B0+IrQTKtyr26J/KibizehHpTS3+ILe+uooRbRRLs8QyiPHAGT/Q0ZukCKBI/iVprfi2a\nOFpCmQmQW9fvVt1dp4dubiSQxFuVRQ23nvmpqh6Kpr2O/jlNvGxRTg5XJIHTvSae3hciVHwytkBU\nwM1VUsEbBJbm9luC0ssgPsSK4HKdSST1JOahyaqFT0nFOc/mxjpgUUt/J/NM7AdtxriVxeDC+W+k\nS6KtyY1MgGe2DQsHxnd2dqIbeNF5OCedvevS43a0mw1fjye6gWO4tROxXB5yPrVOpa7ps0CRQrLg\njLg/lB9c1egAJtNOeQH5pkZh5T1FPtMubbS7RVEkHiDrKOpGfWklG1QSj5i388ltLKzyufEcHrnt\n7CvUFJLBqPpn8Pl8QytPIMneypkbj9aZ6YkErgLN4dzEfySpnPrXm9bZV+YLNehS1Ez2xklOQZ07\nYPoBSQwSWniXcyyLbY58aQjA6DjOaVRkkznptjPSbkNzbzRyxAElFq641v5eIPHEk+7O9M4al6v6\nVuj1zc6nIbNtPBtZJYy2Sc/YVnH17VbeZ1N3JubO5WA/vVKTwpGn6Pl+IrX+CRSS3IF064K9SD6n\n05H71fdXawy2vy4F665aRUbAzjGM/eqSkkhH7Rdezzn4duJGiSHfHt2q+7aScdfvSWJ57O1XSJYw\nq28vi7/5uRgZ/WuLltK0Dx0SmaNrW31KSIzMwbMKDzMRjr+v7U8jvRNpYQKkUhi3IhxhD3GKf+J+\nMaZWKA7D5lok/BPqdrAY704g0OyaBykCyPKxkYSHJjY9celda/0q3QaLG8FmCqmSVDt2qeSPese1\ntLH8RmW+up5ArYS2ZSo5/bikcfiJOS8IS6dpNtcyTqZbm4xmO1ikBBGecnrjPYVz/wBTTX2iG3WC\nCKbx/BEfG0DnAA+1FQSVkl6VtdPYa5DPHMN/iL5WJKsegz7CuXUc1nJf+NI8brOzo8ZIVT6UrxGa\n9G/whc/MwRur+LLK4MhbP05NZTXJzaz3yyQLFOZzjjjjOSPrTJ5o0lUUPNKuU/hqQXsAjkba6uCQ\nRyOn2o+CwleO4uCr3CtlhskGMDoMY61Pr28DWIV/EE3ylhYSXUO+ORDtRpCjKMnKn1x/mqLq9TSb\nCxulzciRijebGPYj6Uf+f0ZrLAh8YXQZVVkRGbO3G/ip6fDEdas5PC2PcSkyNjaCowRx+uftS9aZ\nz62OZvii0srKQKwa9k3FkxwpOev0qVvosWo/CCSWswlwSyGQbNjZ5A77cmi0qot6E6LZ2+kXRuXt\nWcMn5Ac4x1x606/idjqSR3drjcoKqso6H0xR4lcdAk60zXxMvy2lXMkVqYry4cL4Vsobee+ePTBr\nI6l8L3cPw6NQdWjk8TDxSDkqeh9j/mn459TKf7IHSrzTrQQXKlZs72jJ/L9RUIbVopFeNdzdcBsc\nUZO22TavRvbx29m0PzMKeHL2Ztzx+5NPLa4e3gW1huEuEOXDse2eBmpwldjxHoj8ax3hUEgAP/ik\nmoW4mvYZRJsaFwQ4PB9V9aunhaP7DSI7O1klnZEPi7d5GAPKO9A/IQyOrXtm3y7xsVuEJ4OOPrk0\nV6O0AaITaTGOCNnlc8+IQBkds08KG4uvm4pIQY5DGz4wsPTOe+P70svbGQBbaiNQt5XSUxCB8AEf\nmHtjvXZI4NY0N7a+d4zvLDEfnXHQ/fpWYpVafD2i30EUV1bu02MyBWI+556gY5q+fR447COz0hNu\nnM+64mMmGKA9Cep7+1L2bJyuwf8Ailo2tW+kxwGCEqCk7SbsZ9ffmnEMUUUrWcNyGSHI3Kv5gTye\nKSUWwrcZCCBV1n5dr1XtnXyYOW3fX+xqn4tuiLCWGWHJl8jyN1K0IS9RGZi9OsoxKfCIfHRCeSPW\nidCvEOqPas+Eul2PnnBHPT9qLjZNYh9GkNvchQQjXOVw3cjpRcutWkGn3On3xzL4GFUgnJ9KrH0v\nBujAEXHjrHbQq0Q/30Q8s1vA0klvbMij+VMEV0pJCSDvhDddX73EdsyRqVUFyQCx9j1q+6mvrPUm\ngFrAYVlYqxwS+DwR6Cpy2RaP9SV23jQma4gKhGMuyPqzDAVQffP7VTp+qbry3BWR/GIGJFXC+xxT\n/wDyTl6X3GtXDalJA9t4UcbFc5xnn+lH3ZktLaJjJE4bDqw/lx2+nNSuxkc1Bo75DIVgkXco/DXh\nc9iOo5pdqeqTNCbUK3ikBWkBHC+lZujMF09sPcSMu2KGHIOMmihq1kHjFxskfw96h+Bj1rR10FeB\n9hbRapYXUq29tAVUsNqZZ+vQ0ts7a8vJBDbeFHJGpiRH4JHU/wDM0HJp0ykYponKrwySQS5jidch\nuArH0BpfcRmRhEQAN+CT6Yp4r6Qn+iy2t7rRboTQTNIjnbiN/Ko/9w6mgUu1a5lW4PnclhhKaXtk\nn5RZJahl3JznvQE0DrknFT9EQMmQ3IP61a0R4KZbtgVFxVjHNQt1Mska4E0cAJYZ5B4P9TWaubSS\nCRxtYopIDjpXbx4jMmqNa2bP+WWQY6cqvr9T0qmxsLi/uRHAq7j/ALmCgfUk1ZMAQbZlke2mOXj6\nFORn60ZoUarqaI6JKBztIJGftQl4ZD+XT8ncrwJnO5QdpXn0r1cMoybK2fTmHz9s9vdzEOCVWGJe\npB4yevTBoLTQ9oy3Fs5j3sULvHncR1GTyOh6Yrnhki1HP/Ud1fIGtLONZfGMQMhBBAwCf1IrPzie\n9v5FvbkAKcsWXjA7CqSaQOqGL2k2hafBcQwRGCVQW2kliCOMc/qKpsr621GzK3cbwPkbTgAP2470\nktjYtIfA/JRRKbhtqYaPKggduuKRfEYtRqrm7mV0zhXh7k8kH0xQd4a0hEdQR3js4VRIBJkyYIMg\nJ6tWjstIS4u4hbODKSyBi+0EAd6lzXGqJqpMb28K2jNb30ytazR+ZscBwcdaidE8W6uN8oLTszhg\n+cjtVGuyGa3BddWF3ZeJKZHETREHa35HHOfvj96WW2seDPuv7mNIghIUgsGOOnsTQjCmVTpDiC4m\nuLuzl0+GQWMsWSCo6jrzWmvf/wBGXKS2heWN1xMCMbh2P2qrxjNk/wCPSR2crBZJGbbgcDFJdatL\nzVlkkv3i+USPKW8Ex3Anu3rWUrZKcMsxtvIsV29pA6NgfhSLhir45yfSmmg2cmo3Uj3NugSwdWVo\nx5pWAyM/Sq9REvodrT2rm18IrFKoyQRz0GM/Wua7rsVppr3jCSeS5Xyx48m4ADGaRRCLdNv7yC7t\nJPGjXaAZAeFXjpx1NaHV9B0nVby6uLp5fGdwcK3GSOg+vFJPzAN+C2LU7aG+MF4WHysuzdjBJA5/\nTFA6sZ2Rk05nO90nhw2GKtyGz6f4NBPqrGlLLJaosl9pyWF5di4maMssg42vuyFz9ip/+Qpbps0F\n5pKWlwxkkF40oxyQmAP7mnvLM5WhnL8NKdV0y2it2iYRb2yuWALHzf8AivokOh276dGqxILiMfhy\nsOVJ4/4KSDv00I/s+Sr8OX1l8VNpyyk3Esv5pI9y475/rWqjsZ9KvhNPetdEIE2qCqgcdF6DpQlL\n9Ak+rDblXvg0MMhgRVJbA83PTFB2+nyabZvsuZDuYlSB+XPNGLC5WU3F5cRRQSQRTzv4vRiFU988\nc1ReavrTKYhp9pGrsHGZRu4OQAWbFNGP0VJF0+k6tqWntM2kxiWQESO9yNy47+9C2/w1dSQRyLGl\nvPH5Ad5Jk46jAp3H4hseIq+IolL26zIq3kS4lJBG4Y+lCWWn/OxrEjIkkTAkbT5lJPf1FcaT47sV\nqnQ+X4nitleDwZHWNRyidP8APWhYLVZviDEKtJFcHxFOeI+hO79K6U20qOmGIZanq1vZzQ3OpLvs\nWZkB2ZRhj37570Dbpdz2Cz2uoh7N3/CVieAc8DB4xVdoy9JWSW93Z3M2oOkIiQJndtL5OM/WjrbT\nH025W3jCXFlMpd3kGWb0GenvWa+hsWX8P8Pvk/h1nHJEigpnowP85P14omaC+a6jLwqu9RlVkyBx\nyc/rSsBeZrfR1naMROHXDytJx9AO9L7XV1lN1Z3JWOOdAEkGFZe/GBissQHoRbw2jabJbrMlw+3K\nuT5s/WpWl2bZV/CdJ4xhWRgwxmoyei9fpTasIrgXcZV3E2Qm3GCfUdqLv7pri6eS5tVMkTrsjJzw\neu4UI4wONiO6uLbTmuXljFtLIQY90fldSOQD25rMWTGCPxbbyyqxJZhn2rp41ZFpeDQm7bw2uIZJ\nTCVw+cc5/rRusm3k1hpEcgYVyjNk5xz9qaXpfjj+InuYGe4kKSFEZyR24riwtPbpbS27TM0mFk8Q\nIc/XoadMk1pfbtNY3CWzzvHslBIYg4IP9Kb628P8YnWTLOG3IynaOcGlaTLJYW6zCksFpCXSNpUL\nFRxt70usLeKLzBmLc4XHfPWgnlCSQ/trcX0kniLG3hxbyJMYPrz3NK7iNZ1cZOCPKBxt+lGqQqbJ\nwwBJVmiLsJwEl2fmyOlQ/hZmmJUkRM20hm5HrU290Z+BVtpMFvcv4UpgjVTIxPI2jsR6Vb/CNK1R\nnlEcLuU2ojN4YZSc8Gi8dgi7jpYti2gaXIqW0kk1wcROvnRPYkd6jPdyQ6Z4BdUmK5yUAYEjkVmr\nKQfwVW+vFLIWd3Zi4jQBE3JjYPbvn3oDWvAi1iWC2kLIqoV3dRkdPtTK7wWa+it7iXT7vDIxwu7d\n2Jqu8+JkvLEeJp9vIynYQ+dy+hB96r8IICs7+YsPDY+wPIq1tdi5ju4nVlOMpyDUowb8E+l0V1pt\nyVEd4me4YEV43dvFcqmZHG4fkFZQd6Etj/H1oXLKywmIqQy8nH+arutPZLqRWZPAD525znnj9a6K\npGF9/Yu80gAwzYG309qrjsZ4EKNGQSclSM/tW7ClMelTteIxibBfPTjtTy6m1JXlgtjDZQM+MQIE\nJAPXdjJ/WnTCEgxW7qhAdXG4SO2S3rx9a9S0Gzc3epXY1eBk3sGB4Vefr7jHpQbLcWc09wJSbeNW\nKlMkljkZx25NedVadhfHZ3kulWtxYgTIsTAnoS7MSfL37fpWYN8p1MeKrTDeR4KdyD0+taTX0nKS\nNLc6razfDxEEcd2kJLeGH2SQEeo9unvWdnT+JSQXiSpCGABCoRtI7/StGvSaNFdaU0mmB5rmZlVd\nviRSbh6jiktxol/cWMs8sxitEUS+HLJ+djwfKO9IuTswy8AbXT1a9S1OFD8ZbpzWqg+H9Q8G1e1u\nFY27YZen396Zq9IQT+FeqzR3bx2/jOoRsOpyMdulObLTLXTS1wLhp3EZJUNu49hTR0rVMGTXLHVr\nIvp95Ejo4EizxnDYPTHWg5NCsH8SSLw592GWMPuAOeg/xVGvp0RWaONNvHgkCquyOU4xJ5RGw7D0\nrmo/GLw3E1v8nxEMF85Q5APBpJLB0hfpOpzSSSRMCnjKZAT0BBzjselCXes3/wDGAREhimYhZQNo\n46Z9q3HiJzR6fSLODVBd6c0UbS8OA2VD/wAxwfsap0jULi21u7juItiiI7CrYXsA3vwTxTtv00fC\nOqzw3erraRx+KlydiTr5GUAdfoBzVzLb3mnRwIym3hO2PuuQOpPryaUFJgF+DbWq21paphH8R5FJ\nZm+lErd3iiO7ErJMqkyqOrHA6/oKXrZNo8k/imwuWeKSWWbEscyY42/Trg9feoaOYP4faeIrI8LG\n3KhslVY5AzjkAk1nG1Qm1Q506z0mS4hsJUleNo3TxXPm3Fgw5/50pTpGkW2m/F9zDBbzKNm5mkOR\n5hnCn700YtRGVmplnubC/WWG3jl8ZgJJGkAKJjtWmtbpDtWNlcKNxA+nFHrSLIX6loUGoakLtnaG\n4MYiEisR3zg/561nEhfT9RmOo3EUkKHw1Cksd3uaDgJKFuzmr3mfC+RkCneNzqOFU9+aqa1kkeNT\nf+KY2DEocDBGOneg1+huqYou/kZLNd9089xDIQhhTYTg989aWzarohuJX+WkeQv/APknyMjjpT8a\nf0VwoPk+JZ4rEx2bAEqNgRCQOTnOelLV+IdYmljgSSSNJQq8AE56ZB7UXLRU68GGpafcLBtdTNNn\nmSXgt6AZ71yxtXWc+FcTpJjmGSIhVLDgnn9K5OTWZrSOmNdWUzQ3ccxRX3O65Ix6e1aC01ODx5nS\n1YB+doxgDpVl/VFo4E6lpKXlta6ffNHKir4gXbjqen9K5b2Onm3+SsmjjMJ80YYD9qppnrFWoQeG\n1zFNHG9pbYZVaMktk7cA0VaTapFa/KmI/Lx/9koM8E9Dmi5fCiSom1vLHaML+1k325Yxxq2FfPOC\ne4pE17dRXMcl6kkEMv5sJjC56CkMMZrOyuDGqHxEPmRywyB9qnqWmzXOi5TwtqTZRGXHiD3btSti\nSdFljO73MVs8SMqxkhEwAMc4Jq1JI5NEklRo0uZJCFhBB24z+1I4pit/SiG3kvxFLcmCKQRb3aI4\n2kHuPpXbO7i1CKZppopblcgGM4bYO+e+PSmX6NGVkHubabREt5SbnyvujI5YHp1qgwafPokXyVq7\nRqMOAcGMk9z1rOUovBJR0k/wyt+0kkPiPI4B2mTrj/6FXan8L/K6fbt8p4UjIyyMG3SHAHr7E1W7\n0rHFQpubf/8ASLrvU2sahEmZRtfj9qtitoIdMRrmEbS7smxsq7D/AMCt2dYavpnr1DfXEjQ7VYsC\nSOijNay5S1j1PxZFeTEKNEGPt1IpmZWZ/UbkXWpJlvHbIVTgr9qZW8aq28ReGQhDE4zmg3QGdSEy\n2glkjmKA43FqjtZo5HRdwU4BBzRtUTaCrKExRyW85ZJLiUIqoM7QRndntQDfhTRiAnliG3t0qfpS\nsHVk7SB0aJiBEwAP83HSler2S2lnDcrdhFlGFgKEYOBxVG0JBaT0i7vLWFVtpmiLcqC2V/QjrQ+o\nast8Ykum8dwSV2+Uo3PXGO/rWsZraO6jDJb3Ins8qBxndu2sBznPrS/VTPfJcaoI/EbI3hR0GOD9\nOKylWiziyCKmrNbW74hIj2KexNeHwmtxe3VtG8LGMKJB0POCMVXZeHOnXpVL8H6jZSNm2lQq2VcL\nuVh9RSuT4XmNwZLhlA6kKOv3qsYNIFqzkuiKCBAPCVeueSfpRenI0A85SXngFPMPfPSmQzLJIJbi\nWLwJChD+bd3FG6hbh5wkOwRgL5jgHOKzAhe9tOrsWnjdF5wzAVxJVkO7ejE9lNI4sFl673YbMnkY\nJGMY/tVOoXsvjBY9pYEk7ulFIwhabfc+Iqky7udh616nCfTLTUH1iS1+VkMJZTGN7ZUMrkA5+hH6\nUX8T6lHbQiHTrlGZnw7ooyMcnHqM+tefT+nU3ovb4ultbGOOMq7zKVy0OzHuOcH9KzVncgFy8CS+\nHhtwznr61Dlj2+kJ1eF2rsLe7le2tjcRygkkPnaT0FV2YuFgWF1jRT5cEj+lCP4r00cGZu2+HdPE\nduXL3AKMsxJVV6+UdqJ0y2m1GKG7GGtWYq7ZHkbryM0+N2gvQTUYYjGssMhIlkwHQHIIre6RaTT6\nETGzQsQAkkjc9Acn96rVoEFokOmC4e7kSdLloW8jI2eM85455zRscqae7sHjVJAWG7jB9Cf1qCyd\nHR1sXHSITq8V9Y3ENqjoQ0QYJ4hPXDH3qmXSL60shGbKeFTvVp94kUN1DZU1amx8WFcN38np8cOp\nu0s5J3PEAMjIGf3FMLjTBG48OSYw7NoYD9CTU5FFVFXiXEF9HK9yYYl8oWaQ7pMdAMD602lurrwj\nBZ3FuB4h2tIgOV/28iqxWEnpC7ka2giS4ksWMswQZQLgnn09qT3dxCxuoLaG0tSpDCRXYBs446eo\n7UWgUU6dIY7SRd1uxc+DFJE2SCRk4z7UTo9rJe2EttJLBEGVsusmRjsT75GK1CUSutLWyiupreRb\nd5UWKKSRiBHnrg85qeh2ieCYpmN0AuGmnQIjE9gOv/M1qsLVIH1U/wDVQQ2MhjjjxuNuqnBHY5z/\nAFq7RtJawnuIWkmuPmYWAklH/bbquPvj9aCmniEfG0rYLqV2q6rppuJZZWESmK1hXq2fzMT0HtT+\n/vxDZwT+H4glIUkclT6Zot0BWK9SQwanbzywGdZcKCrcL9QTWpdZvD8GyQmbCgLjG33OO1ZMZMVa\nhdz2Wk3NoySXjEEvMg5Lk9B9MfpSu1XxtOaSFSJmO5RK2FI75pXoZMD1KK7g0qeRUSXdtjEQ82Ac\nEnjpS6PVJobpZJI0t5R/KOhGRzg8dD2oRT+k3LcDNThLXdyG2iIvxkZypHB+tJLr5GG3Ig3JcxlV\nVQAQR6mqBkwU3kn5lmdd3DYNe0m4W81yCyuHfwZCFXYc+bJx++M+1KntAitNT8W2nzepWuntMIYI\nAolnY5TefQf3qPw/DMtjdQ6lOJ4QCsS7vxGweCD6cd6jzOkUoIvbr5iRFt7iQMIuYFyeccnPevaW\n0/yl0sNtJNcTpiNgnAxjr7U/Gm0hovBvqkSSzf8AU30EF3LEqlVbaYyAMY+9J7C9gsbicyWwN6gK\nGZsvuP2OB+lPK0zLGDa78UTJFaSW086LOCCSFClweRjGavj1O/jv4kfUJi5XcUD+UD0Iz174olLR\n291rVJb6GCDUDAMMx8XO1sdie3SjtU1m5tdJspYV+Ya5858WMOoXHQH61r+MHmiWw+IYLuZVvtPt\nvIxAktiYnDf0p5PqMGs2Qt4bqWKSMg/Lzpktj3HBFJJV4K121gU9m1qUlhkJikA3oU59/NUfkYVl\ngNs8ayKd5AbkjvkHg+tSchEz2j289tfXE3zzTQNnC+HtzQGpRTtrcV9bQrAsQ8iFcI3qSR14pU6k\nL4zuozjwo7hTFli25Q3Cj68/2qvT52+aFijz7JgrgRjBz79sfWuqrKmpgJsrpDhpvDAHjKTjPf60\nXrWpLcWFtczvlGl2KyqQwJz19qeKwwjk0yOTxJ3ldY9vCx+dW98UM9wqT20cVxaui4xD+Xdk84J6\nGkHO65FZi9ESRR2ly6l9pzyB+1V6lclViklYSK0IQHoVCkjFFIHgpt445bp5GI4GOO1W2uqouqJt\nUtbkeGQT5vTI+nWl5FgjZaJdRjzabnKl8GNe5HpTiNooLdpmjEcoxmBmyze3tUopisOupbZbWBbN\ndt5O28rM20Kq87R6mspq8TQXswuHSQuxkIhHKkn/AJ0q1bSM3mjRLzwtOaa2fxkjGAJeD96Emnkh\n+GLa7uYZZJFnkWLeMgZxyT6Z4FCvg0KWiG51XWdMntZLi2YZ2yjd9TsyRwPpQV1cTyvJI0QS4kyT\nJu4Y/TtVFEa9JyahNGAbS4c3BIMglUbAfX35rQ6Xdo1qzRAqNm25tyc7wCM7cnp1/Si44DvYuCW8\nlteGC4MDw+YIwwy88YoHTdYuBcyPK26V/wA5zVeN4cso6bbSvjPESI4I4+x4pk8FjrkO91ZC3JeH\nAP8Az61dMRoUXnwXIH8WyuUuCeAkvlYffvSG50G8glCyxNAIzlgwJH1rJBC4IUs4NuQ00jZBb+Vf\naim07Ur2MfINEGbIUyHgYpW6MlZm7HXrdWmh1GL5yXxNrN4YGz1x61o4dMs5YhcQwRvGejJ1Wmi0\nxZJp4US6TDIyvGzKA2X/ABCcj70quvh/5mZzHdZXdwrCjKP6BZ6S4W0URQQhJBxynXHevVPqPZoJ\no9N03TYLGWaP+IsN8jBuUz0UjsSKUxS2jWtwOWuMbY8Hbt5/qRXC5Fwa/tLw6bFLcCc+E5GHB2qO\nMc/rUtIcw4jydtxkcAZfp61Nu9JSVM0dzZreW6WduPAhTIKugIz67hRsHw/oeh2kJ1SA3t0RjbEx\nRVH9Sa0ONS9KKqKdU+JdKudMnij0C4jMJHgm4jd1b6+n60Re/F9tZaDaw6J8vb3EoDziGIBVOOeo\n61Xqk6Ru1eC1td1VbWWR9Wg3RlAkKRgs+fTA44rn8cvree4ZJ28e5iBKSfyA9MCknKhl/oBYPd2j\nzzR6gIodmZJVHJYckY/XpWhsL611myhjvfEDSKFaRACU9HI+9K0pfkWW+Bc2lQrJvv5PmbZLIRlO\no3Lzu9j1pdpVrNLpzR6ZqJNrb7lfc2eTyOtOq+MDu9Fdyk934AuIEu7rPmGCE2+u4H2FaHTWN3pM\n0O6K1xEI/CbLghe+SfT+lZJDiuHQbeeNI1v45hC5OFYuV9v60b8qxvLiUTqVhZmI/wBnfmjVAy6A\nbl7i9sSstlLJEzB0kOcAgnkHFVth9HxczKGj/EhRlGTjIwR1PP8AWlu8YXFeohLoFzc3FqbR40sY\nG3TSysAiZ789O3Ap3YXOn21nM0Ei3MkrbHldAm44ydqjtT3SEopnd7trdfEeMsxffjDBfXnpSzSb\nOW6u57S2vZZtjF49yZxnkHP/ADrSNtKyqSeFUN2sd60sy7ZF/Oj9GOcUbefFdta3bWs7PHlVIliU\nYQkA5I7j6VHir1G5U3hS8zyG2uIrlWiPBmOQgP06ijLu8jt0jtIJN8gwFbd5WdgccduQRXVKvTmS\nfgJp0s10LiK5tiJF84Y8gHPFMxJqk77lhuFjiliJkiO3K7SGOe4BxxU17g3hz+OgePHIZnJOCy4H\nA6g/+KQ3V1JezeDHbSs0nMaKucD6d6EnXhGTBR8zbTiFS8KucSIeDjucYolzbPMMqzwAY3tyf07U\n8W2BFes6lb+JapP4ipdQoUbdjAB2/ttoAaWsshSOXdkFkm4CMPr6g5FCTaQWiCW91HlrdI2jRtn4\nijk090bTY4BDJdW4ju1JcsMbQDjBB7GpxdlIIP1pIIrR7xNkjB/BYuTwwGB5e56frSHRI4odRQym\n4cuAXVgAFUc5PoDTONjUaqw0qO6ikmgkaMrJu3dByPQc1fb2+oW0QaZbciKTrgklfqOppo5gqVGd\nvTBDrEs5jhvGZwzrJKwMbHrjHFLNPSaC5uLk2UhcE+BK5Kqn69aLQ1XhqLOcXukJHqCxTXaSblIT\njnpjI9AKzA+Kri2juc3BjkM2xYgmBjHXpyeKdKw+elV6b3WLaOe3hBFqRuljjOG7HcBWi0V7W+sx\nptzcSPBIgKyqCpgk9PcUslWIKdoyev6DqeiagrbUeBJeJFP5vf7U+0nWzdXKSBIjPEQviAY3qB/i\nlmlKNBbweadFda1p73EexYYpickcYI55rLXNzZwapc310/jRx4t4kzhSTnnPbp+9JCHVkVrCLZri\nK0LSkwFyDGyN5cH3zTTSbi4WV7e/iSeIISk7EZUAetacdDJC97SK9cmxj3lmBUH8vuD9+9Io7i+h\n1GdomZZlYK4Xtnjjt1q8PAo+kfDUTac8XiyExSr+IjjO1yO1L9ajvLi4gtrctIscjPtcYzijZRCu\n4fVNIVGtvJJdA7khcEKvuO1Vm0M1skssbEk45GWU+uP70vgXpyCxTUNQge9ml2qdkcobO0++etd+\nIrd49OTzpKFuHjGCAfXmmQgjd2t7UMOZHYqoUce+TQ0Fybm+igjgZXkwFz0z35/zTTh2WE26NBNd\nXmjgW86YlZi6sF3ZyOzUbpMcd06NdeK0sg27T3PrXM4uIVQtaSS2vixbw5423IgfjHWhNclA1X5s\nBDMCCcKSSCMj271WMn9NJXiGFqWm01ILs+Gs2CISOePWmupXMth8PCFUiMUi7ljcZCqCATj6n9qW\nWvB4KlTFOVjtrdXJ8GW33SgDcrYc7Tz3yO1ATxWUbtd3e0qC+4xjpntj1qsU2aSQp+VjiYOhOxxu\nQ+1FxzO6xhJGDrwpHUd6s4/CKZGF4724C3JQylcB0wCRx1oS6srjTNbljAAjGQkmdwYUkLToX0Y6\nUHuZQkUbMwHCAZJNNtOknhkY2xIVRufuAO2fvVbSA0alLxVsoZMOlyUzIjjAH/ippqWRHHcbAsp2\nEFcgH9KdSy2K1oo1p9MhVTYhZbjftaPcQB2z+tZ68vLho4lR1QMvaUqQw9CKWTsdKhG8UDy75ra3\n3k8zC6Gf2pzp1xDFJ4dnM0gK4YK+7j+1BAlo8t7eO9iZINpJ8pGRQsLtZyPFJGfK+0CrJkmjl4Yr\nseZCo6ZzXqxhBNbGOVJp5Gnkmbc7N35rS6ppNnFqMjwRGNGcExlsjJHUenWvFnNpHS/CttNktBO0\n95NdwxwM6QynyhuMH/xQjfjLHJhFcxswIX8p9qHHPtFkn6M/gtZJrXULyaUyLaoW8Fhw7cEEn70r\n1iE3l+tzNI7SOgJ5wOnanlJxSoeWLBZJBgYSWVRj8u/irrKzzH4niEhAW2tyDg4xTJto0GetLjw7\n1J0UL59+0dOnSmV1M118Vzs4BWaFCRzx5AePvU3uFZIczWFq1wAIsFUZPzccjrj15qi0s/Aubd4n\n2OiCNio/OPf9BU+K1Fh88Gl9IqQISrCSNAVdHK9D3x1+lci1iG3lWFLIbrnG994GcD0AxVomS0Jm\n06KytYo18ymQMM8EA9sjtS25jhSUTRRlJIkcKS2R5TxxWiyyYwt44obyDwYljNwniMR16DAzWe1W\n+kAkjVY1Zk5k2gsaYHrLIdbvrrTYY7iYyR26AKuP+e1B3dtBdiW4eMeJC2M5/Nzjn16/tTLQNJFU\nepXl5LsuJ98KhQkIXCL9AKKihNxKAZGTYM+TjmnaQEw64WSSwtbp5pDcRSkByc5Azx9Of2rTfAMM\nTNdTNGviLtQFRgAY9PtU27jQX4Z/490KG0vp5oWKvJ+KeOORyMUl0r4eh+Irm3aWV4t3gbsDO7OQ\nf/8AUVycD1oabxGy1maNta2xQRwBAVXwwAc4ByfWgpdKtpdXhmEMaGSA79q43EYYHjuCP3r0H4c8\ncYRHBEL6K0EarHKpkkIHLH60TKTbWNyLdmRUYIoJ3AdfWpQQZPTPRmEukTQggDnBxmq5dSl0+9C2\napEzEZcjceaziiMvQu4dr5/mJnczxpsDk5znrkdKz965gvRagDase4EDHfofWjD0KWGgvba3uPhr\nTpxBEkkfiRg7FPGc9x7n9axHxBq8+mp8hHHCYW/EI2Y5P0PFO/aHSRfo138xFNLcxiQhd6bWK7CO\nPWtEw26HNNESrSIuQx3DpStUNEpuLeRrfV5vFBaFxJFuXOw8A4596GuCo0qytsNnUWBkk3HcB6A+\nlKno301b3jRabuhBRoHWHcG5ce+KJspQ9rLMykgk4XPTccdaaPpmtMxq6JY2zm1XZ4khDg85xn+9\nK7aFtUu7eG4lbEhK8HhftWk/yoRNpjD4gE+iW8kOn3DxCzyUJ8xIx0OaDa5GsWtnLfxRuzRhcxqE\nIPXPcZposo91k9LjuNDmaG1uAYZ3G5WTJIPbOaLs7JW+LZrSNjDHJGsg2AeQ7c8Z9xQkqdmihpNc\nvcWUEkwR/G2iQEcsDwRn+9KptAi+HPiKb5WQusi5VXXOzjFJ9CzZ/Hd0Ph74UtLSxhRIbuQROBxx\n1P3NfKruSKS8ESwKqF/KCd22qNEYFrW2yBlDDaDjG30pxFpKwW6TePI28b2XoD7USki9EaxktrmF\nzm4lCOrcjbzxWb2rNq4OCshlOWB4OOnFLdMxuY5wbURsHLbA+4Pjkfal8mqT3t3vmY7nlIJU4Ixg\ncfaih0BQ6Z85f3LRzNELSVT3YuNx4JzUtZlktplmgkdTLuOGOcYNazE1Y3GkSSNhZIAJwy8bj/b7\nUk1hXvdLMjysviy+KQMcMe+aYDBvHJiK45jGM55PvRHwxrT2c92s0EdyGjYgyDkHFXRzTQ7+GoJt\najg8W6kjCg7AACFB69aZfElkmhwhLZnLRxDa5POSck1zcqXobqkJZ7eO5iyyjdIDKGPJUgVeZlie\nzmEMZBgVmRhnJ5HXr2FTXhZIt0o2uoyyLJbssqxl/EEp5I56dvtTjWYEXVrV5AHj8ARmMcAg+b+9\nUSoLMxdmO7F9GsXhR2jsYgGOQAfy59KUoirfzxDd4Uyq2wnO04p0K9HUKW+m6BLeC3E0jt4IVz5Q\nD39c/es0YfAYTRu6luwPHWuhM5n6UfLLbyDwmYZOeecVc5Z9HW4LHdBJ4eM8MD60z0yZK31C4017\nW6tXCOGyQRkHJ2/0p/8AC07R6FqsEh8QIOWJwWydv9s1OUUUg9KrLUpNVMtxdKHaLYmD0I6ftim8\nGsXkmj380cixrbAbIwgKjB/vS/KHpPRB8R38l1bNqkQ8CV2yVXpQltPLNo0TtJlxMeWGRgjpimiq\nROT0Hvnt7ZAz2UErDA6bf6VNrQThZIpGiU//AI+CopgMvtl8GXykhumRxWls18a1LSEtvXkGjYrQ\nsvj4Ejqg8oXv9K9VEKf/2Q==\n", + "text/plain": [ + "\u003cIPython.core.display.Image object\u003e" + ] + }, + "metadata": { + "tags": [] + }, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Restoring parameters from mobilenet_v2_1.0_224.ckpt\n", + "Top 1 prediction: 389 giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca 0.90984344\n" + ] + } + ], + "source": [ + "from IPython import display\n", + "import pylab\n", + "from datasets import imagenet\n", + "import PIL\n", + "display.display(display.Image('panda.jpg'))\n", + "\n", + "with tf.Session() as sess:\n", + " saver.restore(sess, checkpoint)\n", + " x = endpoints['Predictions'].eval(feed_dict={file_input: 'panda.jpg'})\n", + "label_map = imagenet.create_readable_names_for_imagenet_labels() \n", + "print(\"Top 1 prediction: \", x.argmax(),label_map[x.argmax()], x.max())\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "PlwvpK3ElBk6" + }, + "source": [ + "# Frozen inference" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "o0BIbQUUlVrf" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "img = np.array(PIL.Image.open('panda.jpg').resize((224, 224))).astype(np.float) / 128 - 1\n", + "gd = tf.GraphDef.FromString(open(checkpoint_name + '_frozen.pb', 'rb').read())\n", + "inp, predictions = tf.import_graph_def(gd, return_elements = ['input:0', 'MobilenetV2/Predictions/Reshape_1:0'])" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "base_uri": "https://localhost:8080/", + "height": 35, + "output_extras": [ + {} + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 1350, + "status": "ok", + "timestamp": 1521493472822, + "user": { + "displayName": "Mark Sandler", + "photoUrl": "//lh5.googleusercontent.com/-CjnV3zpGrlw/AAAAAAAAAAI/AAAAAAAABRU/dfjRy_tzX5M/s50-c-k-no/photo.jpg", + "userId": "108034853522252017283" + }, + "user_tz": 420 + }, + "id": "qSU2h5NRlN7V", + "outputId": "4fb09105-b729-45c3-b5ef-83c8da30a215" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Top 1 Prediction: 389 giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca 0.9220208\n" + ] + } + ], + "source": [ + "with tf.Session(graph=inp.graph):\n", + " x = predictions.eval(feed_dict={inp: img.reshape(1, 224,224, 3)})\n", + "\n", + "label_map = imagenet.create_readable_names_for_imagenet_labels() \n", + "print(\"Top 1 Prediction: \", x.argmax(),label_map[x.argmax()], x.max())" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "CU8dJF8kCo6X" + }, + "outputs": [], + "source": [ + "" + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [ + "T_cETKXHDTXu" + ], + "default_view": {}, + "name": "Mobilenet Example.ipynb", + "provenance": [ + { + "file_id": "1ylt6hB0JlXmWU9Bm6O1zGKVPgc2csZf5", + "timestamp": 1521507068201 + } + ], + "version": "0.3.2", + "views": {} + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_v2.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_v2.py new file mode 100644 index 0000000..7a06fc2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_v2.py @@ -0,0 +1,249 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Implementation of Mobilenet V2. + +Architecture: https://arxiv.org/abs/1801.04381 + +The base model gives 72.2% accuracy on ImageNet, with 300MMadds, +3.4 M parameters. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import copy +import functools + +import tensorflow as tf +from tensorflow.contrib import layers as contrib_layers +from tensorflow.contrib import slim as contrib_slim + +from nets.mobilenet import conv_blocks as ops +from nets.mobilenet import mobilenet as lib + +slim = contrib_slim +op = lib.op + +expand_input = ops.expand_input_by_factor + +# pyformat: disable +# Architecture: https://arxiv.org/abs/1801.04381 +V2_DEF = dict( + defaults={ + # Note: these parameters of batch norm affect the architecture + # that's why they are here and not in training_scope. + (slim.batch_norm,): {'center': True, 'scale': True}, + (slim.conv2d, slim.fully_connected, slim.separable_conv2d): { + 'normalizer_fn': slim.batch_norm, 'activation_fn': tf.nn.relu6 + }, + (ops.expanded_conv,): { + 'expansion_size': expand_input(6), + 'split_expansion': 1, + 'normalizer_fn': slim.batch_norm, + 'residual': True + }, + (slim.conv2d, slim.separable_conv2d): {'padding': 'SAME'} + }, + spec=[ + op(slim.conv2d, stride=2, num_outputs=32, kernel_size=[3, 3]), + op(ops.expanded_conv, + expansion_size=expand_input(1, divisible_by=1), + num_outputs=16), + op(ops.expanded_conv, stride=2, num_outputs=24), + op(ops.expanded_conv, stride=1, num_outputs=24), + op(ops.expanded_conv, stride=2, num_outputs=32), + op(ops.expanded_conv, stride=1, num_outputs=32), + op(ops.expanded_conv, stride=1, num_outputs=32), + op(ops.expanded_conv, stride=2, num_outputs=64), + op(ops.expanded_conv, stride=1, num_outputs=64), + op(ops.expanded_conv, stride=1, num_outputs=64), + op(ops.expanded_conv, stride=1, num_outputs=64), + op(ops.expanded_conv, stride=1, num_outputs=96), + op(ops.expanded_conv, stride=1, num_outputs=96), + op(ops.expanded_conv, stride=1, num_outputs=96), + op(ops.expanded_conv, stride=2, num_outputs=160), + op(ops.expanded_conv, stride=1, num_outputs=160), + op(ops.expanded_conv, stride=1, num_outputs=160), + op(ops.expanded_conv, stride=1, num_outputs=320), + op(slim.conv2d, stride=1, kernel_size=[1, 1], num_outputs=1280) + ], +) +# pyformat: enable + +# Mobilenet v2 Definition with group normalization. +V2_DEF_GROUP_NORM = copy.deepcopy(V2_DEF) +V2_DEF_GROUP_NORM['defaults'] = { + (contrib_slim.conv2d, contrib_slim.fully_connected, + contrib_slim.separable_conv2d): { + 'normalizer_fn': contrib_layers.group_norm, # pylint: disable=C0330 + 'activation_fn': tf.nn.relu6, # pylint: disable=C0330 + }, # pylint: disable=C0330 + (ops.expanded_conv,): { + 'expansion_size': ops.expand_input_by_factor(6), + 'split_expansion': 1, + 'normalizer_fn': contrib_layers.group_norm, + 'residual': True + }, + (contrib_slim.conv2d, contrib_slim.separable_conv2d): { + 'padding': 'SAME' + } +} + + +@slim.add_arg_scope +def mobilenet(input_tensor, + num_classes=1001, + depth_multiplier=1.0, + scope='MobilenetV2', + conv_defs=None, + finegrain_classification_mode=False, + min_depth=None, + divisible_by=None, + activation_fn=None, + **kwargs): + """Creates mobilenet V2 network. + + Inference mode is created by default. To create training use training_scope + below. + + with tf.contrib.slim.arg_scope(mobilenet_v2.training_scope()): + logits, endpoints = mobilenet_v2.mobilenet(input_tensor) + + Args: + input_tensor: The input tensor + num_classes: number of classes + depth_multiplier: The multiplier applied to scale number of + channels in each layer. + scope: Scope of the operator + conv_defs: Allows to override default conv def. + finegrain_classification_mode: When set to True, the model + will keep the last layer large even for small multipliers. Following + https://arxiv.org/abs/1801.04381 + suggests that it improves performance for ImageNet-type of problems. + *Note* ignored if final_endpoint makes the builder exit earlier. + min_depth: If provided, will ensure that all layers will have that + many channels after application of depth multiplier. + divisible_by: If provided will ensure that all layers # channels + will be divisible by this number. + activation_fn: Activation function to use, defaults to tf.nn.relu6 if not + specified. + **kwargs: passed directly to mobilenet.mobilenet: + prediction_fn- what prediction function to use. + reuse-: whether to reuse variables (if reuse set to true, scope + must be given). + Returns: + logits/endpoints pair + + Raises: + ValueError: On invalid arguments + """ + if conv_defs is None: + conv_defs = V2_DEF + if 'multiplier' in kwargs: + raise ValueError('mobilenetv2 doesn\'t support generic ' + 'multiplier parameter use "depth_multiplier" instead.') + if finegrain_classification_mode: + conv_defs = copy.deepcopy(conv_defs) + if depth_multiplier < 1: + conv_defs['spec'][-1].params['num_outputs'] /= depth_multiplier + if activation_fn: + conv_defs = copy.deepcopy(conv_defs) + defaults = conv_defs['defaults'] + conv_defaults = ( + defaults[(slim.conv2d, slim.fully_connected, slim.separable_conv2d)]) + conv_defaults['activation_fn'] = activation_fn + + depth_args = {} + # NB: do not set depth_args unless they are provided to avoid overriding + # whatever default depth_multiplier might have thanks to arg_scope. + if min_depth is not None: + depth_args['min_depth'] = min_depth + if divisible_by is not None: + depth_args['divisible_by'] = divisible_by + + with slim.arg_scope((lib.depth_multiplier,), **depth_args): + return lib.mobilenet( + input_tensor, + num_classes=num_classes, + conv_defs=conv_defs, + scope=scope, + multiplier=depth_multiplier, + **kwargs) + +mobilenet.default_image_size = 224 + + +def wrapped_partial(func, *args, **kwargs): + partial_func = functools.partial(func, *args, **kwargs) + functools.update_wrapper(partial_func, func) + return partial_func + + +# Wrappers for mobilenet v2 with depth-multipliers. Be noticed that +# 'finegrain_classification_mode' is set to True, which means the embedding +# layer will not be shrinked when given a depth-multiplier < 1.0. +mobilenet_v2_140 = wrapped_partial(mobilenet, depth_multiplier=1.4) +mobilenet_v2_050 = wrapped_partial(mobilenet, depth_multiplier=0.50, + finegrain_classification_mode=True) +mobilenet_v2_035 = wrapped_partial(mobilenet, depth_multiplier=0.35, + finegrain_classification_mode=True) + + +@slim.add_arg_scope +def mobilenet_base(input_tensor, depth_multiplier=1.0, **kwargs): + """Creates base of the mobilenet (no pooling and no logits) .""" + return mobilenet(input_tensor, + depth_multiplier=depth_multiplier, + base_only=True, **kwargs) + + +@slim.add_arg_scope +def mobilenet_base_group_norm(input_tensor, depth_multiplier=1.0, **kwargs): + """Creates base of the mobilenet (no pooling and no logits) .""" + kwargs['conv_defs'] = V2_DEF_GROUP_NORM + kwargs['conv_defs']['defaults'].update({ + (contrib_layers.group_norm,): { + 'groups': kwargs.pop('groups', 8) + } + }) + return mobilenet( + input_tensor, depth_multiplier=depth_multiplier, base_only=True, **kwargs) + + +def training_scope(**kwargs): + """Defines MobilenetV2 training scope. + + Usage: + with tf.contrib.slim.arg_scope(mobilenet_v2.training_scope()): + logits, endpoints = mobilenet_v2.mobilenet(input_tensor) + + with slim. + + Args: + **kwargs: Passed to mobilenet.training_scope. The following parameters + are supported: + weight_decay- The weight decay to use for regularizing the model. + stddev- Standard deviation for initialization, if negative uses xavier. + dropout_keep_prob- dropout keep probability + bn_decay- decay for the batch norm moving averages. + + Returns: + An `arg_scope` to use for the mobilenet v2 model. + """ + return lib.training_scope(**kwargs) + + +__all__ = ['training_scope', 'mobilenet_base', 'mobilenet', 'V2_DEF'] diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_v2_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_v2_test.py new file mode 100644 index 0000000..11ab0eb --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_v2_test.py @@ -0,0 +1,219 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for mobilenet_v2.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import copy +from six.moves import range +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim +from nets.mobilenet import conv_blocks as ops +from nets.mobilenet import mobilenet +from nets.mobilenet import mobilenet_v2 + + +slim = contrib_slim + + +def find_ops(optype): + """Find ops of a given type in graphdef or a graph. + + Args: + optype: operation type (e.g. Conv2D) + Returns: + List of operations. + """ + gd = tf.compat.v1.get_default_graph() + return [var for var in gd.get_operations() if var.type == optype] + + +class MobilenetV2Test(tf.test.TestCase): + + def setUp(self): + tf.compat.v1.reset_default_graph() + + def testCreation(self): + spec = dict(mobilenet_v2.V2_DEF) + _, ep = mobilenet.mobilenet( + tf.compat.v1.placeholder(tf.float32, (10, 224, 224, 16)), + conv_defs=spec) + num_convs = len(find_ops('Conv2D')) + + # This is mostly a sanity test. No deep reason for these particular + # constants. + # + # All but first 2 and last one have two convolutions, and there is one + # extra conv that is not in the spec. (logits) + self.assertEqual(num_convs, len(spec['spec']) * 2 - 2) + # Check that depthwise are exposed. + for i in range(2, 17): + self.assertIn('layer_%d/depthwise_output' % i, ep) + + def testCreationNoClasses(self): + spec = copy.deepcopy(mobilenet_v2.V2_DEF) + net, ep = mobilenet.mobilenet( + tf.compat.v1.placeholder(tf.float32, (10, 224, 224, 16)), + conv_defs=spec, + num_classes=None) + self.assertIs(net, ep['global_pool']) + + def testImageSizes(self): + for input_size, output_size in [(224, 7), (192, 6), (160, 5), + (128, 4), (96, 3)]: + tf.compat.v1.reset_default_graph() + _, ep = mobilenet_v2.mobilenet( + tf.compat.v1.placeholder(tf.float32, (10, input_size, input_size, 3))) + + self.assertEqual(ep['layer_18/output'].get_shape().as_list()[1:3], + [output_size] * 2) + + def testWithSplits(self): + spec = copy.deepcopy(mobilenet_v2.V2_DEF) + spec['overrides'] = { + (ops.expanded_conv,): dict(split_expansion=2), + } + _, _ = mobilenet.mobilenet( + tf.compat.v1.placeholder(tf.float32, (10, 224, 224, 16)), + conv_defs=spec) + num_convs = len(find_ops('Conv2D')) + # All but 3 op has 3 conv operatore, the remainign 3 have one + # and there is one unaccounted. + self.assertEqual(num_convs, len(spec['spec']) * 3 - 5) + + def testWithOutputStride8(self): + out, _ = mobilenet.mobilenet_base( + tf.compat.v1.placeholder(tf.float32, (10, 224, 224, 16)), + conv_defs=mobilenet_v2.V2_DEF, + output_stride=8, + scope='MobilenetV2') + self.assertEqual(out.get_shape().as_list()[1:3], [28, 28]) + + def testDivisibleBy(self): + tf.compat.v1.reset_default_graph() + mobilenet_v2.mobilenet( + tf.compat.v1.placeholder(tf.float32, (10, 224, 224, 16)), + conv_defs=mobilenet_v2.V2_DEF, + divisible_by=16, + min_depth=32) + s = [op.outputs[0].get_shape().as_list()[-1] for op in find_ops('Conv2D')] + s = set(s) + self.assertSameElements([32, 64, 96, 160, 192, 320, 384, 576, 960, 1280, + 1001], s) + + def testDivisibleByWithArgScope(self): + tf.compat.v1.reset_default_graph() + # Verifies that depth_multiplier arg scope actually works + # if no default min_depth is provided. + with slim.arg_scope((mobilenet.depth_multiplier,), min_depth=32): + mobilenet_v2.mobilenet( + tf.compat.v1.placeholder(tf.float32, (10, 224, 224, 2)), + conv_defs=mobilenet_v2.V2_DEF, + depth_multiplier=0.1) + s = [op.outputs[0].get_shape().as_list()[-1] for op in find_ops('Conv2D')] + s = set(s) + self.assertSameElements(s, [32, 192, 128, 1001]) + + def testFineGrained(self): + tf.compat.v1.reset_default_graph() + # Verifies that depth_multiplier arg scope actually works + # if no default min_depth is provided. + + mobilenet_v2.mobilenet( + tf.compat.v1.placeholder(tf.float32, (10, 224, 224, 2)), + conv_defs=mobilenet_v2.V2_DEF, + depth_multiplier=0.01, + finegrain_classification_mode=True) + s = [op.outputs[0].get_shape().as_list()[-1] for op in find_ops('Conv2D')] + s = set(s) + # All convolutions will be 8->48, except for the last one. + self.assertSameElements(s, [8, 48, 1001, 1280]) + + def testMobilenetBase(self): + tf.compat.v1.reset_default_graph() + # Verifies that mobilenet_base returns pre-pooling layer. + with slim.arg_scope((mobilenet.depth_multiplier,), min_depth=32): + net, _ = mobilenet_v2.mobilenet_base( + tf.compat.v1.placeholder(tf.float32, (10, 224, 224, 16)), + conv_defs=mobilenet_v2.V2_DEF, + depth_multiplier=0.1) + self.assertEqual(net.get_shape().as_list(), [10, 7, 7, 128]) + + def testWithOutputStride16(self): + tf.compat.v1.reset_default_graph() + out, _ = mobilenet.mobilenet_base( + tf.compat.v1.placeholder(tf.float32, (10, 224, 224, 16)), + conv_defs=mobilenet_v2.V2_DEF, + output_stride=16) + self.assertEqual(out.get_shape().as_list()[1:3], [14, 14]) + + def testMultiplier(self): + op = mobilenet.op + new_def = copy.deepcopy(mobilenet_v2.V2_DEF) + + def inverse_multiplier(output_params, multiplier): + output_params['num_outputs'] = int( + output_params['num_outputs'] / multiplier) + + new_def['spec'][0] = op( + slim.conv2d, + kernel_size=(3, 3), + multiplier_func=inverse_multiplier, + num_outputs=16) + _ = mobilenet_v2.mobilenet_base( + tf.compat.v1.placeholder(tf.float32, (10, 224, 224, 16)), + conv_defs=new_def, + depth_multiplier=0.1) + s = [op.outputs[0].get_shape().as_list()[-1] for op in find_ops('Conv2D')] + # Expect first layer to be 160 (16 / 0.1), and other layers + # their max(original size * 0.1, 8) + self.assertEqual([160, 8, 48, 8, 48], s[:5]) + + def testWithOutputStride8AndExplicitPadding(self): + tf.compat.v1.reset_default_graph() + out, _ = mobilenet.mobilenet_base( + tf.compat.v1.placeholder(tf.float32, (10, 224, 224, 16)), + conv_defs=mobilenet_v2.V2_DEF, + output_stride=8, + use_explicit_padding=True, + scope='MobilenetV2') + self.assertEqual(out.get_shape().as_list()[1:3], [28, 28]) + + def testWithOutputStride16AndExplicitPadding(self): + tf.compat.v1.reset_default_graph() + out, _ = mobilenet.mobilenet_base( + tf.compat.v1.placeholder(tf.float32, (10, 224, 224, 16)), + conv_defs=mobilenet_v2.V2_DEF, + output_stride=16, + use_explicit_padding=True) + self.assertEqual(out.get_shape().as_list()[1:3], [14, 14]) + + def testBatchNormScopeDoesNotHaveIsTrainingWhenItsSetToNone(self): + sc = mobilenet.training_scope(is_training=None) + self.assertNotIn('is_training', sc[slim.arg_scope_func_key( + slim.batch_norm)]) + + def testBatchNormScopeDoesHasIsTrainingWhenItsNotNone(self): + sc = mobilenet.training_scope(is_training=False) + self.assertIn('is_training', sc[slim.arg_scope_func_key(slim.batch_norm)]) + sc = mobilenet.training_scope(is_training=True) + self.assertIn('is_training', sc[slim.arg_scope_func_key(slim.batch_norm)]) + sc = mobilenet.training_scope() + self.assertIn('is_training', sc[slim.arg_scope_func_key(slim.batch_norm)]) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_v3.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_v3.py new file mode 100644 index 0000000..36dbdaa --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_v3.py @@ -0,0 +1,405 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Mobilenet V3 conv defs and helper functions.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import copy +import functools +import numpy as np + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets.mobilenet import conv_blocks as ops +from nets.mobilenet import mobilenet as lib + +slim = contrib_slim +op = lib.op +expand_input = ops.expand_input_by_factor + +# Squeeze Excite with all parameters filled-in, we use hard-sigmoid +# for gating function and relu for inner activation function. +squeeze_excite = functools.partial( + ops.squeeze_excite, squeeze_factor=4, + inner_activation_fn=tf.nn.relu, + gating_fn=lambda x: tf.nn.relu6(x+3)*0.16667) + +# Wrap squeeze excite op as expansion_transform that takes +# both expansion and input tensor. +_se4 = lambda expansion_tensor, input_tensor: squeeze_excite(expansion_tensor) + + +def hard_swish(x): + with tf.compat.v1.name_scope('hard_swish'): + return x * tf.nn.relu6(x + np.float32(3)) * np.float32(1. / 6.) + + +def reduce_to_1x1(input_tensor, default_size=7, **kwargs): + h, w = input_tensor.shape.as_list()[1:3] + if h is not None and w == h: + k = [h, h] + else: + k = [default_size, default_size] + return slim.avg_pool2d(input_tensor, kernel_size=k, **kwargs) + + +def mbv3_op(ef, n, k, s=1, act=tf.nn.relu, se=None, **kwargs): + """Defines a single Mobilenet V3 convolution block. + + Args: + ef: expansion factor + n: number of output channels + k: stride of depthwise + s: stride + act: activation function in inner layers + se: squeeze excite function. + **kwargs: passed to expanded_conv + + Returns: + An object (lib._Op) for inserting in conv_def, representing this operation. + """ + return op( + ops.expanded_conv, + expansion_size=expand_input(ef), + kernel_size=(k, k), + stride=s, + num_outputs=n, + inner_activation_fn=act, + expansion_transform=se, + **kwargs) + + +def mbv3_fused(ef, n, k, s=1, **kwargs): + """Defines a single Mobilenet V3 convolution block. + + Args: + ef: expansion factor + n: number of output channels + k: stride of depthwise + s: stride + **kwargs: will be passed to mbv3_op + + Returns: + An object (lib._Op) for inserting in conv_def, representing this operation. + """ + expansion_fn = functools.partial(slim.conv2d, kernel_size=k, stride=s) + return mbv3_op( + ef, + n, + k=1, + s=s, + depthwise_location=None, + expansion_fn=expansion_fn, + **kwargs) + + +mbv3_op_se = functools.partial(mbv3_op, se=_se4) + + +DEFAULTS = { + (ops.expanded_conv,): + dict( + normalizer_fn=slim.batch_norm, + residual=True), + (slim.conv2d, slim.fully_connected, slim.separable_conv2d): { + 'normalizer_fn': slim.batch_norm, + 'activation_fn': tf.nn.relu, + }, + (slim.batch_norm,): { + 'center': True, + 'scale': True + }, +} + +# Compatible checkpoint: http://mldash/5511169891790690458#scalars +V3_LARGE = dict( + defaults=dict(DEFAULTS), + spec=([ + # stage 1 + op(slim.conv2d, stride=2, num_outputs=16, kernel_size=(3, 3), + activation_fn=hard_swish), + mbv3_op(ef=1, n=16, k=3), + mbv3_op(ef=4, n=24, k=3, s=2), + mbv3_op(ef=3, n=24, k=3, s=1), + mbv3_op_se(ef=3, n=40, k=5, s=2), + mbv3_op_se(ef=3, n=40, k=5, s=1), + mbv3_op_se(ef=3, n=40, k=5, s=1), + mbv3_op(ef=6, n=80, k=3, s=2, act=hard_swish), + mbv3_op(ef=2.5, n=80, k=3, s=1, act=hard_swish), + mbv3_op(ef=184/80., n=80, k=3, s=1, act=hard_swish), + mbv3_op(ef=184/80., n=80, k=3, s=1, act=hard_swish), + mbv3_op_se(ef=6, n=112, k=3, s=1, act=hard_swish), + mbv3_op_se(ef=6, n=112, k=3, s=1, act=hard_swish), + mbv3_op_se(ef=6, n=160, k=5, s=2, act=hard_swish), + mbv3_op_se(ef=6, n=160, k=5, s=1, act=hard_swish), + mbv3_op_se(ef=6, n=160, k=5, s=1, act=hard_swish), + op(slim.conv2d, stride=1, kernel_size=[1, 1], num_outputs=960, + activation_fn=hard_swish), + op(reduce_to_1x1, default_size=7, stride=1, padding='VALID'), + op(slim.conv2d, stride=1, kernel_size=[1, 1], num_outputs=1280, + normalizer_fn=None, activation_fn=hard_swish) + ])) + +# 72.2% accuracy. +V3_LARGE_MINIMALISTIC = dict( + defaults=dict(DEFAULTS), + spec=([ + # stage 1 + op(slim.conv2d, stride=2, num_outputs=16, kernel_size=(3, 3)), + mbv3_op(ef=1, n=16, k=3), + mbv3_op(ef=4, n=24, k=3, s=2), + mbv3_op(ef=3, n=24, k=3, s=1), + mbv3_op(ef=3, n=40, k=3, s=2), + mbv3_op(ef=3, n=40, k=3, s=1), + mbv3_op(ef=3, n=40, k=3, s=1), + mbv3_op(ef=6, n=80, k=3, s=2), + mbv3_op(ef=2.5, n=80, k=3, s=1), + mbv3_op(ef=184 / 80., n=80, k=3, s=1), + mbv3_op(ef=184 / 80., n=80, k=3, s=1), + mbv3_op(ef=6, n=112, k=3, s=1), + mbv3_op(ef=6, n=112, k=3, s=1), + mbv3_op(ef=6, n=160, k=3, s=2), + mbv3_op(ef=6, n=160, k=3, s=1), + mbv3_op(ef=6, n=160, k=3, s=1), + op(slim.conv2d, stride=1, kernel_size=[1, 1], num_outputs=960), + op(reduce_to_1x1, default_size=7, stride=1, padding='VALID'), + op(slim.conv2d, + stride=1, + kernel_size=[1, 1], + num_outputs=1280, + normalizer_fn=None) + ])) + +# Compatible run: http://mldash/2023283040014348118#scalars +V3_SMALL = dict( + defaults=dict(DEFAULTS), + spec=([ + # stage 1 + op(slim.conv2d, stride=2, num_outputs=16, kernel_size=(3, 3), + activation_fn=hard_swish), + mbv3_op_se(ef=1, n=16, k=3, s=2), + mbv3_op(ef=72./16, n=24, k=3, s=2), + mbv3_op(ef=(88./24), n=24, k=3, s=1), + mbv3_op_se(ef=4, n=40, k=5, s=2, act=hard_swish), + mbv3_op_se(ef=6, n=40, k=5, s=1, act=hard_swish), + mbv3_op_se(ef=6, n=40, k=5, s=1, act=hard_swish), + mbv3_op_se(ef=3, n=48, k=5, s=1, act=hard_swish), + mbv3_op_se(ef=3, n=48, k=5, s=1, act=hard_swish), + mbv3_op_se(ef=6, n=96, k=5, s=2, act=hard_swish), + mbv3_op_se(ef=6, n=96, k=5, s=1, act=hard_swish), + mbv3_op_se(ef=6, n=96, k=5, s=1, act=hard_swish), + op(slim.conv2d, stride=1, kernel_size=[1, 1], num_outputs=576, + activation_fn=hard_swish), + op(reduce_to_1x1, default_size=7, stride=1, padding='VALID'), + op(slim.conv2d, stride=1, kernel_size=[1, 1], num_outputs=1024, + normalizer_fn=None, activation_fn=hard_swish) + ])) + +# 62% accuracy. +V3_SMALL_MINIMALISTIC = dict( + defaults=dict(DEFAULTS), + spec=([ + # stage 1 + op(slim.conv2d, stride=2, num_outputs=16, kernel_size=(3, 3)), + mbv3_op(ef=1, n=16, k=3, s=2), + mbv3_op(ef=72. / 16, n=24, k=3, s=2), + mbv3_op(ef=(88. / 24), n=24, k=3, s=1), + mbv3_op(ef=4, n=40, k=3, s=2), + mbv3_op(ef=6, n=40, k=3, s=1), + mbv3_op(ef=6, n=40, k=3, s=1), + mbv3_op(ef=3, n=48, k=3, s=1), + mbv3_op(ef=3, n=48, k=3, s=1), + mbv3_op(ef=6, n=96, k=3, s=2), + mbv3_op(ef=6, n=96, k=3, s=1), + mbv3_op(ef=6, n=96, k=3, s=1), + op(slim.conv2d, stride=1, kernel_size=[1, 1], num_outputs=576), + op(reduce_to_1x1, default_size=7, stride=1, padding='VALID'), + op(slim.conv2d, + stride=1, + kernel_size=[1, 1], + num_outputs=1024, + normalizer_fn=None) + ])) + + +# EdgeTPU friendly variant of MobilenetV3 that uses fused convolutions +# instead of depthwise in the early layers. +V3_EDGETPU = dict( + defaults=dict(DEFAULTS), + spec=[ + op(slim.conv2d, stride=2, num_outputs=32, kernel_size=(3, 3)), + mbv3_fused(k=3, s=1, ef=1, n=16), + mbv3_fused(k=3, s=2, ef=8, n=32), + mbv3_fused(k=3, s=1, ef=4, n=32), + mbv3_fused(k=3, s=1, ef=4, n=32), + mbv3_fused(k=3, s=1, ef=4, n=32), + mbv3_fused(k=3, s=2, ef=8, n=48), + mbv3_fused(k=3, s=1, ef=4, n=48), + mbv3_fused(k=3, s=1, ef=4, n=48), + mbv3_fused(k=3, s=1, ef=4, n=48), + mbv3_op(k=3, s=2, ef=8, n=96), + mbv3_op(k=3, s=1, ef=4, n=96), + mbv3_op(k=3, s=1, ef=4, n=96), + mbv3_op(k=3, s=1, ef=4, n=96), + mbv3_op(k=3, s=1, ef=8, n=96, residual=False), + mbv3_op(k=3, s=1, ef=4, n=96), + mbv3_op(k=3, s=1, ef=4, n=96), + mbv3_op(k=3, s=1, ef=4, n=96), + mbv3_op(k=5, s=2, ef=8, n=160), + mbv3_op(k=5, s=1, ef=4, n=160), + mbv3_op(k=5, s=1, ef=4, n=160), + mbv3_op(k=5, s=1, ef=4, n=160), + mbv3_op(k=3, s=1, ef=8, n=192), + op(slim.conv2d, stride=1, num_outputs=1280, kernel_size=(1, 1)), + ]) + + +@slim.add_arg_scope +def mobilenet(input_tensor, + num_classes=1001, + depth_multiplier=1.0, + scope='MobilenetV3', + conv_defs=None, + finegrain_classification_mode=False, + **kwargs): + """Creates mobilenet V3 network. + + Inference mode is created by default. To create training use training_scope + below. + + with tf.contrib.slim.arg_scope(mobilenet_v3.training_scope()): + logits, endpoints = mobilenet_v3.mobilenet(input_tensor) + + Args: + input_tensor: The input tensor + num_classes: number of classes + depth_multiplier: The multiplier applied to scale number of + channels in each layer. + scope: Scope of the operator + conv_defs: Which version to create. Could be large/small or + any conv_def (see mobilenet_v3.py for examples). + finegrain_classification_mode: When set to True, the model + will keep the last layer large even for small multipliers. Following + https://arxiv.org/abs/1801.04381 + it improves performance for ImageNet-type of problems. + *Note* ignored if final_endpoint makes the builder exit earlier. + **kwargs: passed directly to mobilenet.mobilenet: + prediction_fn- what prediction function to use. + reuse-: whether to reuse variables (if reuse set to true, scope + must be given). + Returns: + logits/endpoints pair + + Raises: + ValueError: On invalid arguments + """ + if conv_defs is None: + conv_defs = V3_LARGE + if 'multiplier' in kwargs: + raise ValueError('mobilenetv2 doesn\'t support generic ' + 'multiplier parameter use "depth_multiplier" instead.') + if finegrain_classification_mode: + conv_defs = copy.deepcopy(conv_defs) + conv_defs['spec'][-1] = conv_defs['spec'][-1]._replace( + multiplier_func=lambda params, multiplier: params) + depth_args = {} + with slim.arg_scope((lib.depth_multiplier,), **depth_args): + return lib.mobilenet( + input_tensor, + num_classes=num_classes, + conv_defs=conv_defs, + scope=scope, + multiplier=depth_multiplier, + **kwargs) + +mobilenet.default_image_size = 224 +training_scope = lib.training_scope + + +@slim.add_arg_scope +def mobilenet_base(input_tensor, depth_multiplier=1.0, **kwargs): + """Creates base of the mobilenet (no pooling and no logits) .""" + return mobilenet( + input_tensor, depth_multiplier=depth_multiplier, base_only=True, **kwargs) + + +def wrapped_partial(func, new_defaults=None, + **kwargs): + """Partial function with new default parameters and updated docstring.""" + if not new_defaults: + new_defaults = {} + def func_wrapper(*f_args, **f_kwargs): + new_kwargs = dict(new_defaults) + new_kwargs.update(f_kwargs) + return func(*f_args, **new_kwargs) + functools.update_wrapper(func_wrapper, func) + partial_func = functools.partial(func_wrapper, **kwargs) + functools.update_wrapper(partial_func, func) + return partial_func + + +large = wrapped_partial(mobilenet, conv_defs=V3_LARGE) +small = wrapped_partial(mobilenet, conv_defs=V3_SMALL) +edge_tpu = wrapped_partial(mobilenet, + new_defaults={'scope': 'MobilenetEdgeTPU'}, + conv_defs=V3_EDGETPU) +edge_tpu_075 = wrapped_partial( + mobilenet, + new_defaults={'scope': 'MobilenetEdgeTPU'}, + conv_defs=V3_EDGETPU, + depth_multiplier=0.75, + finegrain_classification_mode=True) + +# Minimalistic model that does not have Squeeze Excite blocks, +# Hardswish, or 5x5 depthwise convolution. +# This makes the model very friendly for a wide range of hardware +large_minimalistic = wrapped_partial(mobilenet, conv_defs=V3_LARGE_MINIMALISTIC) +small_minimalistic = wrapped_partial(mobilenet, conv_defs=V3_SMALL_MINIMALISTIC) + + +def _reduce_consecutive_layers(conv_defs, start_id, end_id, multiplier=0.5): + """Reduce the outputs of consecutive layers with multiplier. + + Args: + conv_defs: Mobilenet conv_defs. + start_id: 0-based index of the starting conv_def to be reduced. + end_id: 0-based index of the last conv_def to be reduced. + multiplier: The multiplier by which to reduce the conv_defs. + + Returns: + Mobilenet conv_defs where the output sizes from layers [start_id, end_id], + inclusive, are reduced by multiplier. + + Raises: + ValueError if any layer to be reduced does not have the 'num_outputs' + attribute. + """ + defs = copy.deepcopy(conv_defs) + for d in defs['spec'][start_id:end_id+1]: + d.params.update({ + 'num_outputs': np.int(np.round(d.params['num_outputs'] * multiplier)) + }) + return defs + + +V3_LARGE_DETECTION = _reduce_consecutive_layers(V3_LARGE, 13, 16) +V3_SMALL_DETECTION = _reduce_consecutive_layers(V3_SMALL, 9, 12) + + +__all__ = ['training_scope', 'mobilenet', 'V3_LARGE', 'V3_SMALL', 'large', + 'small', 'V3_LARGE_DETECTION', 'V3_SMALL_DETECTION'] diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_v3_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_v3_test.py new file mode 100644 index 0000000..45f1b10 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet/mobilenet_v3_test.py @@ -0,0 +1,82 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for google3.third_party.tensorflow_models.slim.nets.mobilenet.mobilenet_v3.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl.testing import absltest +import tensorflow as tf + +from nets.mobilenet import mobilenet_v3 + + +class MobilenetV3Test(absltest.TestCase): + + def setUp(self): + super(MobilenetV3Test, self).setUp() + tf.compat.v1.reset_default_graph() + + def testMobilenetV3Large(self): + logits, endpoints = mobilenet_v3.mobilenet( + tf.compat.v1.placeholder(tf.float32, (1, 224, 224, 3))) + self.assertEqual(endpoints['layer_19'].shape, [1, 1, 1, 1280]) + self.assertEqual(logits.shape, [1, 1001]) + + def testMobilenetV3Small(self): + _, endpoints = mobilenet_v3.mobilenet( + tf.compat.v1.placeholder(tf.float32, (1, 224, 224, 3)), + conv_defs=mobilenet_v3.V3_SMALL) + self.assertEqual(endpoints['layer_15'].shape, [1, 1, 1, 1024]) + + def testMobilenetEdgeTpu(self): + _, endpoints = mobilenet_v3.edge_tpu( + tf.compat.v1.placeholder(tf.float32, (1, 224, 224, 3))) + self.assertIn('Inference mode is created by default', + mobilenet_v3.edge_tpu.__doc__) + self.assertEqual(endpoints['layer_24'].shape, [1, 7, 7, 1280]) + self.assertStartsWith( + endpoints['layer_24'].name, 'MobilenetEdgeTPU') + + def testMobilenetEdgeTpuChangeScope(self): + _, endpoints = mobilenet_v3.edge_tpu( + tf.compat.v1.placeholder(tf.float32, (1, 224, 224, 3)), scope='Scope') + self.assertStartsWith( + endpoints['layer_24'].name, 'Scope') + + def testMobilenetV3BaseOnly(self): + result, endpoints = mobilenet_v3.mobilenet( + tf.compat.v1.placeholder(tf.float32, (1, 224, 224, 3)), + conv_defs=mobilenet_v3.V3_LARGE, + base_only=True, + final_endpoint='layer_17') + # Get the latest layer before average pool. + self.assertEqual(endpoints['layer_17'].shape, [1, 7, 7, 960]) + self.assertEqual(result, endpoints['layer_17']) + + def testMobilenetV3BaseOnly_VariableInput(self): + result, endpoints = mobilenet_v3.mobilenet( + tf.placeholder(tf.float32, (None, None, None, 3)), + conv_defs=mobilenet_v3.V3_LARGE, + base_only=True, + final_endpoint='layer_17') + # Get the latest layer before average pool. + self.assertEqual(endpoints['layer_17'].shape.as_list(), + [None, None, None, 960]) + self.assertEqual(result, endpoints['layer_17']) + +if __name__ == '__main__': + absltest.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1.md b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1.md new file mode 100644 index 0000000..ba4cc23 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1.md @@ -0,0 +1,136 @@ +# MobilenetV2 and above +For MobilenetV2+ see this file [mobilenet/README.md](mobilenet/README.md) + +# MobileNetV1 + +[MobileNets](https://arxiv.org/abs/1704.04861) are small, low-latency, low-power models parameterized to meet the resource constraints of a variety of use cases. They can be built upon for classification, detection, embeddings and segmentation similar to how other popular large scale models, such as Inception, are used. MobileNets can be run efficiently on mobile devices with [TensorFlow Mobile](https://www.tensorflow.org/mobile/). + +MobileNets trade off between latency, size and accuracy while comparing favorably with popular models from the literature. + +![alt text](mobilenet_v1.png "MobileNet Graph") + +# Pre-trained Models + +Choose the right MobileNet model to fit your latency and size budget. The size of the network in memory and on disk is proportional to the number of parameters. The latency and power usage of the network scales with the number of Multiply-Accumulates (MACs) which measures the number of fused Multiplication and Addition operations. These MobileNet models have been trained on the +[ILSVRC-2012-CLS](http://www.image-net.org/challenges/LSVRC/2012/) +image classification dataset. Accuracies were computed by evaluating using a single image crop. + +Model | Million MACs | Million Parameters | Top-1 Accuracy| Top-5 Accuracy | +:----:|:------------:|:----------:|:-------:|:-------:| +[MobileNet_v1_1.0_224](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224.tgz)|569|4.24|70.9|89.9| +[MobileNet_v1_1.0_192](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_192.tgz)|418|4.24|70.0|89.2| +[MobileNet_v1_1.0_160](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_160.tgz)|291|4.24|68.0|87.7| +[MobileNet_v1_1.0_128](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_128.tgz)|186|4.24|65.2|85.8| +[MobileNet_v1_0.75_224](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.75_224.tgz)|317|2.59|68.4|88.2| +[MobileNet_v1_0.75_192](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.75_192.tgz)|233|2.59|67.2|87.3| +[MobileNet_v1_0.75_160](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.75_160.tgz)|162|2.59|65.3|86.0| +[MobileNet_v1_0.75_128](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.75_128.tgz)|104|2.59|62.1|83.9| +[MobileNet_v1_0.50_224](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.5_224.tgz)|150|1.34|63.3|84.9| +[MobileNet_v1_0.50_192](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.5_192.tgz)|110|1.34|61.7|83.6| +[MobileNet_v1_0.50_160](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.5_160.tgz)|77|1.34|59.1|81.9| +[MobileNet_v1_0.50_128](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.5_128.tgz)|49|1.34|56.3|79.4| +[MobileNet_v1_0.25_224](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.25_224.tgz)|41|0.47|49.8|74.2| +[MobileNet_v1_0.25_192](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.25_192.tgz)|34|0.47|47.7|72.3| +[MobileNet_v1_0.25_160](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.25_160.tgz)|21|0.47|45.5|70.3| +[MobileNet_v1_0.25_128](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.25_128.tgz)|14|0.47|41.5|66.3| +[MobileNet_v1_1.0_224_quant](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224_quant.tgz)|569|4.24|70.1|88.9| +[MobileNet_v1_1.0_192_quant](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_192_quant.tgz)|418|4.24|69.2|88.3| +[MobileNet_v1_1.0_160_quant](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_160_quant.tgz)|291|4.24|67.2|86.7| +[MobileNet_v1_1.0_128_quant](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_128_quant.tgz)|186|4.24|63.4|84.2| +[MobileNet_v1_0.75_224_quant](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.75_224_quant.tgz)|317|2.59|66.8|87.0| +[MobileNet_v1_0.75_192_quant](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.75_192_quant.tgz)|233|2.59|66.1|86.4| +[MobileNet_v1_0.75_160_quant](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.75_160_quant.tgz)|162|2.59|62.3|83.8| +[MobileNet_v1_0.75_128_quant](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.75_128_quant.tgz)|104|2.59|55.8|78.8| +[MobileNet_v1_0.50_224_quant](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.5_224_quant.tgz)|150|1.34|60.7|83.2| +[MobileNet_v1_0.50_192_quant](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.5_192_quant.tgz)|110|1.34|60.0|82.2| +[MobileNet_v1_0.50_160_quant](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.5_160_quant.tgz)|77|1.34|57.7|80.4| +[MobileNet_v1_0.50_128_quant](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.5_128_quant.tgz)|49|1.34|54.5|77.7| +[MobileNet_v1_0.25_224_quant](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.25_224_quant.tgz)|41|0.47|48.0|72.8| +[MobileNet_v1_0.25_192_quant](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.25_192_quant.tgz)|34|0.47|46.0|71.2| +[MobileNet_v1_0.25_160_quant](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.25_160_quant.tgz)|21|0.47|43.4|68.5| +[MobileNet_v1_0.25_128_quant](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.25_128_quant.tgz)|14|0.47|39.5|64.4| + +Revisions to models: +* July 12, 2018: Update to TFLite models that fixes an accuracy issue resolved by making conversion support weights with narrow_range. We now report validation on the actual TensorFlow Lite model rather than the emulated quantization number of TensorFlow. +* August 2, 2018: Update to TFLite models that fixes an accuracy issue resolved by making sure the numerics of quantization match TF quantized training accurately. + +The linked model tar files contain the following: +* Trained model checkpoints +* Eval graph text protos (to be easily viewed) +* Frozen trained models +* Info file containing input and output information +* Converted [TensorFlow Lite](https://www.tensorflow.org/mobile/tflite/) flatbuffer model + +Note that quantized model GraphDefs are still float models, they just have FakeQuantization +operation embedded to simulate quantization. These are converted by [TensorFlow Lite](https://www.tensorflow.org/mobile/tflite/) +to be fully quantized. The final effect of quantization can be seen by comparing the frozen fake +quantized graph to the size of the TFLite flatbuffer, i.e. The TFLite flatbuffer is about 1/4 +the size. +For more information on the quantization techniques used here, see +[here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/quantize). + +Here is an example of how to download the MobileNet_v1_1.0_224 checkpoint: + +```shell +$ CHECKPOINT_DIR=/tmp/checkpoints +$ mkdir ${CHECKPOINT_DIR} +$ wget http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_1.0_224.tgz +$ tar -xvf mobilenet_v1_1.0_224.tgz +$ mv mobilenet_v1_1.0_224.ckpt.* ${CHECKPOINT_DIR} +``` + +# MobileNet V1 scripts + +This package contains scripts for training floating point and eight-bit fixed +point TensorFlow models. + +Quantization tools used are described in [contrib/quantize](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/quantize). + +Conversion to fully quantized models for mobile can be done through [TensorFlow Lite](https://www.tensorflow.org/mobile/tflite/). + +## Usage + +### Build for GPU + +``` +$ bazel build -c opt --config=cuda mobilenet_v1_{eval,train} +``` + +### Running + +#### Float Training and Eval + +Train: + +``` +$ ./bazel-bin/mobilenet_v1_train --dataset_dir "path/to/dataset" --checkpoint_dir "path/to/checkpoints" +``` + +Eval: + +``` +$ ./bazel-bin/mobilenet_v1_eval --dataset_dir "path/to/dataset" --checkpoint_dir "path/to/checkpoints" +``` + +#### Quantized Training and Eval + +Train from preexisting float checkpoint: + +``` +$ ./bazel-bin/mobilenet_v1_train --dataset_dir "path/to/dataset" --checkpoint_dir "path/to/checkpoints" \ + --quantize=True --fine_tune_checkpoint=float/checkpoint/path +``` + +Train from scratch: + +``` +$ ./bazel-bin/mobilenet_v1_train --dataset_dir "path/to/dataset" --checkpoint_dir "path/to/checkpoints" --quantize=True +``` + +Eval: + +``` +$ ./bazel-bin/mobilenet_v1_eval --dataset_dir "path/to/dataset" --checkpoint_dir "path/to/checkpoints" --quantize=True +``` + +The resulting float and quantized models can be run on-device via [TensorFlow Lite](https://www.tensorflow.org/mobile/tflite/). diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1.py new file mode 100644 index 0000000..107c347 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1.py @@ -0,0 +1,482 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +"""MobileNet v1. + +MobileNet is a general architecture and can be used for multiple use cases. +Depending on the use case, it can use different input layer size and different +head (for example: embeddings, localization and classification). + +As described in https://arxiv.org/abs/1704.04861. + + MobileNets: Efficient Convolutional Neural Networks for + Mobile Vision Applications + Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, + Tobias Weyand, Marco Andreetto, Hartwig Adam + +100% Mobilenet V1 (base) with input size 224x224: + +See mobilenet_v1() + +Layer params macs +-------------------------------------------------------------------------------- +MobilenetV1/Conv2d_0/Conv2D: 864 10,838,016 +MobilenetV1/Conv2d_1_depthwise/depthwise: 288 3,612,672 +MobilenetV1/Conv2d_1_pointwise/Conv2D: 2,048 25,690,112 +MobilenetV1/Conv2d_2_depthwise/depthwise: 576 1,806,336 +MobilenetV1/Conv2d_2_pointwise/Conv2D: 8,192 25,690,112 +MobilenetV1/Conv2d_3_depthwise/depthwise: 1,152 3,612,672 +MobilenetV1/Conv2d_3_pointwise/Conv2D: 16,384 51,380,224 +MobilenetV1/Conv2d_4_depthwise/depthwise: 1,152 903,168 +MobilenetV1/Conv2d_4_pointwise/Conv2D: 32,768 25,690,112 +MobilenetV1/Conv2d_5_depthwise/depthwise: 2,304 1,806,336 +MobilenetV1/Conv2d_5_pointwise/Conv2D: 65,536 51,380,224 +MobilenetV1/Conv2d_6_depthwise/depthwise: 2,304 451,584 +MobilenetV1/Conv2d_6_pointwise/Conv2D: 131,072 25,690,112 +MobilenetV1/Conv2d_7_depthwise/depthwise: 4,608 903,168 +MobilenetV1/Conv2d_7_pointwise/Conv2D: 262,144 51,380,224 +MobilenetV1/Conv2d_8_depthwise/depthwise: 4,608 903,168 +MobilenetV1/Conv2d_8_pointwise/Conv2D: 262,144 51,380,224 +MobilenetV1/Conv2d_9_depthwise/depthwise: 4,608 903,168 +MobilenetV1/Conv2d_9_pointwise/Conv2D: 262,144 51,380,224 +MobilenetV1/Conv2d_10_depthwise/depthwise: 4,608 903,168 +MobilenetV1/Conv2d_10_pointwise/Conv2D: 262,144 51,380,224 +MobilenetV1/Conv2d_11_depthwise/depthwise: 4,608 903,168 +MobilenetV1/Conv2d_11_pointwise/Conv2D: 262,144 51,380,224 +MobilenetV1/Conv2d_12_depthwise/depthwise: 4,608 225,792 +MobilenetV1/Conv2d_12_pointwise/Conv2D: 524,288 25,690,112 +MobilenetV1/Conv2d_13_depthwise/depthwise: 9,216 451,584 +MobilenetV1/Conv2d_13_pointwise/Conv2D: 1,048,576 51,380,224 +-------------------------------------------------------------------------------- +Total: 3,185,088 567,716,352 + + +75% Mobilenet V1 (base) with input size 128x128: + +See mobilenet_v1_075() + +Layer params macs +-------------------------------------------------------------------------------- +MobilenetV1/Conv2d_0/Conv2D: 648 2,654,208 +MobilenetV1/Conv2d_1_depthwise/depthwise: 216 884,736 +MobilenetV1/Conv2d_1_pointwise/Conv2D: 1,152 4,718,592 +MobilenetV1/Conv2d_2_depthwise/depthwise: 432 442,368 +MobilenetV1/Conv2d_2_pointwise/Conv2D: 4,608 4,718,592 +MobilenetV1/Conv2d_3_depthwise/depthwise: 864 884,736 +MobilenetV1/Conv2d_3_pointwise/Conv2D: 9,216 9,437,184 +MobilenetV1/Conv2d_4_depthwise/depthwise: 864 221,184 +MobilenetV1/Conv2d_4_pointwise/Conv2D: 18,432 4,718,592 +MobilenetV1/Conv2d_5_depthwise/depthwise: 1,728 442,368 +MobilenetV1/Conv2d_5_pointwise/Conv2D: 36,864 9,437,184 +MobilenetV1/Conv2d_6_depthwise/depthwise: 1,728 110,592 +MobilenetV1/Conv2d_6_pointwise/Conv2D: 73,728 4,718,592 +MobilenetV1/Conv2d_7_depthwise/depthwise: 3,456 221,184 +MobilenetV1/Conv2d_7_pointwise/Conv2D: 147,456 9,437,184 +MobilenetV1/Conv2d_8_depthwise/depthwise: 3,456 221,184 +MobilenetV1/Conv2d_8_pointwise/Conv2D: 147,456 9,437,184 +MobilenetV1/Conv2d_9_depthwise/depthwise: 3,456 221,184 +MobilenetV1/Conv2d_9_pointwise/Conv2D: 147,456 9,437,184 +MobilenetV1/Conv2d_10_depthwise/depthwise: 3,456 221,184 +MobilenetV1/Conv2d_10_pointwise/Conv2D: 147,456 9,437,184 +MobilenetV1/Conv2d_11_depthwise/depthwise: 3,456 221,184 +MobilenetV1/Conv2d_11_pointwise/Conv2D: 147,456 9,437,184 +MobilenetV1/Conv2d_12_depthwise/depthwise: 3,456 55,296 +MobilenetV1/Conv2d_12_pointwise/Conv2D: 294,912 4,718,592 +MobilenetV1/Conv2d_13_depthwise/depthwise: 6,912 110,592 +MobilenetV1/Conv2d_13_pointwise/Conv2D: 589,824 9,437,184 +-------------------------------------------------------------------------------- +Total: 1,800,144 106,002,432 + +""" + +# Tensorflow mandates these. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from collections import namedtuple +import functools + +import tensorflow as tf +from tensorflow.contrib import layers as contrib_layers +from tensorflow.contrib import slim as contrib_slim + +slim = contrib_slim + +# Conv and DepthSepConv namedtuple define layers of the MobileNet architecture +# Conv defines 3x3 convolution layers +# DepthSepConv defines 3x3 depthwise convolution followed by 1x1 convolution. +# stride is the stride of the convolution +# depth is the number of channels or filters in a layer +Conv = namedtuple('Conv', ['kernel', 'stride', 'depth']) +DepthSepConv = namedtuple('DepthSepConv', ['kernel', 'stride', 'depth']) + +# MOBILENETV1_CONV_DEFS specifies the MobileNet body +MOBILENETV1_CONV_DEFS = [ + Conv(kernel=[3, 3], stride=2, depth=32), + DepthSepConv(kernel=[3, 3], stride=1, depth=64), + DepthSepConv(kernel=[3, 3], stride=2, depth=128), + DepthSepConv(kernel=[3, 3], stride=1, depth=128), + DepthSepConv(kernel=[3, 3], stride=2, depth=256), + DepthSepConv(kernel=[3, 3], stride=1, depth=256), + DepthSepConv(kernel=[3, 3], stride=2, depth=512), + DepthSepConv(kernel=[3, 3], stride=1, depth=512), + DepthSepConv(kernel=[3, 3], stride=1, depth=512), + DepthSepConv(kernel=[3, 3], stride=1, depth=512), + DepthSepConv(kernel=[3, 3], stride=1, depth=512), + DepthSepConv(kernel=[3, 3], stride=1, depth=512), + DepthSepConv(kernel=[3, 3], stride=2, depth=1024), + DepthSepConv(kernel=[3, 3], stride=1, depth=1024) +] + + +def _fixed_padding(inputs, kernel_size, rate=1): + """Pads the input along the spatial dimensions independently of input size. + + Pads the input such that if it was used in a convolution with 'VALID' padding, + the output would have the same dimensions as if the unpadded input was used + in a convolution with 'SAME' padding. + + Args: + inputs: A tensor of size [batch, height_in, width_in, channels]. + kernel_size: The kernel to be used in the conv2d or max_pool2d operation. + rate: An integer, rate for atrous convolution. + + Returns: + output: A tensor of size [batch, height_out, width_out, channels] with the + input, either intact (if kernel_size == 1) or padded (if kernel_size > 1). + """ + kernel_size_effective = [kernel_size[0] + (kernel_size[0] - 1) * (rate - 1), + kernel_size[0] + (kernel_size[0] - 1) * (rate - 1)] + pad_total = [kernel_size_effective[0] - 1, kernel_size_effective[1] - 1] + pad_beg = [pad_total[0] // 2, pad_total[1] // 2] + pad_end = [pad_total[0] - pad_beg[0], pad_total[1] - pad_beg[1]] + padded_inputs = tf.pad( + tensor=inputs, + paddings=[[0, 0], [pad_beg[0], pad_end[0]], [pad_beg[1], pad_end[1]], + [0, 0]]) + return padded_inputs + + +def mobilenet_v1_base(inputs, + final_endpoint='Conv2d_13_pointwise', + min_depth=8, + depth_multiplier=1.0, + conv_defs=None, + output_stride=None, + use_explicit_padding=False, + scope=None): + """Mobilenet v1. + + Constructs a Mobilenet v1 network from inputs to the given final endpoint. + + Args: + inputs: a tensor of shape [batch_size, height, width, channels]. + final_endpoint: specifies the endpoint to construct the network up to. It + can be one of ['Conv2d_0', 'Conv2d_1_pointwise', 'Conv2d_2_pointwise', + 'Conv2d_3_pointwise', 'Conv2d_4_pointwise', 'Conv2d_5'_pointwise, + 'Conv2d_6_pointwise', 'Conv2d_7_pointwise', 'Conv2d_8_pointwise', + 'Conv2d_9_pointwise', 'Conv2d_10_pointwise', 'Conv2d_11_pointwise', + 'Conv2d_12_pointwise', 'Conv2d_13_pointwise']. + min_depth: Minimum depth value (number of channels) for all convolution ops. + Enforced when depth_multiplier < 1, and not an active constraint when + depth_multiplier >= 1. + depth_multiplier: Float multiplier for the depth (number of channels) + for all convolution ops. The value must be greater than zero. Typical + usage will be to set this value in (0, 1) to reduce the number of + parameters or computation cost of the model. + conv_defs: A list of ConvDef namedtuples specifying the net architecture. + output_stride: An integer that specifies the requested ratio of input to + output spatial resolution. If not None, then we invoke atrous convolution + if necessary to prevent the network from reducing the spatial resolution + of the activation maps. Allowed values are 8 (accurate fully convolutional + mode), 16 (fast fully convolutional mode), 32 (classification mode). + use_explicit_padding: Use 'VALID' padding for convolutions, but prepad + inputs so that the output dimensions are the same as if 'SAME' padding + were used. + scope: Optional variable_scope. + + Returns: + tensor_out: output tensor corresponding to the final_endpoint. + end_points: a set of activations for external use, for example summaries or + losses. + + Raises: + ValueError: if final_endpoint is not set to one of the predefined values, + or depth_multiplier <= 0, or the target output_stride is not + allowed. + """ + depth = lambda d: max(int(d * depth_multiplier), min_depth) + end_points = {} + + # Used to find thinned depths for each layer. + if depth_multiplier <= 0: + raise ValueError('depth_multiplier is not greater than zero.') + + if conv_defs is None: + conv_defs = MOBILENETV1_CONV_DEFS + + if output_stride is not None and output_stride not in [8, 16, 32]: + raise ValueError('Only allowed output_stride values are 8, 16, 32.') + + padding = 'SAME' + if use_explicit_padding: + padding = 'VALID' + with tf.compat.v1.variable_scope(scope, 'MobilenetV1', [inputs]): + with slim.arg_scope([slim.conv2d, slim.separable_conv2d], padding=padding): + # The current_stride variable keeps track of the output stride of the + # activations, i.e., the running product of convolution strides up to the + # current network layer. This allows us to invoke atrous convolution + # whenever applying the next convolution would result in the activations + # having output stride larger than the target output_stride. + current_stride = 1 + + # The atrous convolution rate parameter. + rate = 1 + + net = inputs + for i, conv_def in enumerate(conv_defs): + end_point_base = 'Conv2d_%d' % i + + if output_stride is not None and current_stride == output_stride: + # If we have reached the target output_stride, then we need to employ + # atrous convolution with stride=1 and multiply the atrous rate by the + # current unit's stride for use in subsequent layers. + layer_stride = 1 + layer_rate = rate + rate *= conv_def.stride + else: + layer_stride = conv_def.stride + layer_rate = 1 + current_stride *= conv_def.stride + + if isinstance(conv_def, Conv): + end_point = end_point_base + if use_explicit_padding: + net = _fixed_padding(net, conv_def.kernel) + net = slim.conv2d(net, depth(conv_def.depth), conv_def.kernel, + stride=conv_def.stride, + scope=end_point) + end_points[end_point] = net + if end_point == final_endpoint: + return net, end_points + + elif isinstance(conv_def, DepthSepConv): + end_point = end_point_base + '_depthwise' + + # By passing filters=None + # separable_conv2d produces only a depthwise convolution layer + if use_explicit_padding: + net = _fixed_padding(net, conv_def.kernel, layer_rate) + net = slim.separable_conv2d(net, None, conv_def.kernel, + depth_multiplier=1, + stride=layer_stride, + rate=layer_rate, + scope=end_point) + + end_points[end_point] = net + if end_point == final_endpoint: + return net, end_points + + end_point = end_point_base + '_pointwise' + + net = slim.conv2d(net, depth(conv_def.depth), [1, 1], + stride=1, + scope=end_point) + + end_points[end_point] = net + if end_point == final_endpoint: + return net, end_points + else: + raise ValueError('Unknown convolution type %s for layer %d' + % (conv_def.ltype, i)) + raise ValueError('Unknown final endpoint %s' % final_endpoint) + + +def mobilenet_v1(inputs, + num_classes=1000, + dropout_keep_prob=0.999, + is_training=True, + min_depth=8, + depth_multiplier=1.0, + conv_defs=None, + prediction_fn=contrib_layers.softmax, + spatial_squeeze=True, + reuse=None, + scope='MobilenetV1', + global_pool=False): + """Mobilenet v1 model for classification. + + Args: + inputs: a tensor of shape [batch_size, height, width, channels]. + num_classes: number of predicted classes. If 0 or None, the logits layer + is omitted and the input features to the logits layer (before dropout) + are returned instead. + dropout_keep_prob: the percentage of activation values that are retained. + is_training: whether is training or not. + min_depth: Minimum depth value (number of channels) for all convolution ops. + Enforced when depth_multiplier < 1, and not an active constraint when + depth_multiplier >= 1. + depth_multiplier: Float multiplier for the depth (number of channels) + for all convolution ops. The value must be greater than zero. Typical + usage will be to set this value in (0, 1) to reduce the number of + parameters or computation cost of the model. + conv_defs: A list of ConvDef namedtuples specifying the net architecture. + prediction_fn: a function to get predictions out of logits. + spatial_squeeze: if True, logits is of shape is [B, C], if false logits is + of shape [B, 1, 1, C], where B is batch_size and C is number of classes. + reuse: whether or not the network and its variables should be reused. To be + able to reuse 'scope' must be given. + scope: Optional variable_scope. + global_pool: Optional boolean flag to control the avgpooling before the + logits layer. If false or unset, pooling is done with a fixed window + that reduces default-sized inputs to 1x1, while larger inputs lead to + larger outputs. If true, any input size is pooled down to 1x1. + + Returns: + net: a 2D Tensor with the logits (pre-softmax activations) if num_classes + is a non-zero integer, or the non-dropped-out input to the logits layer + if num_classes is 0 or None. + end_points: a dictionary from components of the network to the corresponding + activation. + + Raises: + ValueError: Input rank is invalid. + """ + input_shape = inputs.get_shape().as_list() + if len(input_shape) != 4: + raise ValueError('Invalid input tensor rank, expected 4, was: %d' % + len(input_shape)) + + with tf.compat.v1.variable_scope( + scope, 'MobilenetV1', [inputs], reuse=reuse) as scope: + with slim.arg_scope([slim.batch_norm, slim.dropout], + is_training=is_training): + net, end_points = mobilenet_v1_base(inputs, scope=scope, + min_depth=min_depth, + depth_multiplier=depth_multiplier, + conv_defs=conv_defs) + with tf.compat.v1.variable_scope('Logits'): + if global_pool: + # Global average pooling. + net = tf.reduce_mean( + input_tensor=net, axis=[1, 2], keepdims=True, name='global_pool') + end_points['global_pool'] = net + else: + # Pooling with a fixed kernel size. + kernel_size = _reduced_kernel_size_for_small_input(net, [7, 7]) + net = slim.avg_pool2d(net, kernel_size, padding='VALID', + scope='AvgPool_1a') + end_points['AvgPool_1a'] = net + if not num_classes: + return net, end_points + # 1 x 1 x 1024 + net = slim.dropout(net, keep_prob=dropout_keep_prob, scope='Dropout_1b') + logits = slim.conv2d(net, num_classes, [1, 1], activation_fn=None, + normalizer_fn=None, scope='Conv2d_1c_1x1') + if spatial_squeeze: + logits = tf.squeeze(logits, [1, 2], name='SpatialSqueeze') + end_points['Logits'] = logits + if prediction_fn: + end_points['Predictions'] = prediction_fn(logits, scope='Predictions') + return logits, end_points + +mobilenet_v1.default_image_size = 224 + + +def wrapped_partial(func, *args, **kwargs): + partial_func = functools.partial(func, *args, **kwargs) + functools.update_wrapper(partial_func, func) + return partial_func + + +mobilenet_v1_075 = wrapped_partial(mobilenet_v1, depth_multiplier=0.75) +mobilenet_v1_050 = wrapped_partial(mobilenet_v1, depth_multiplier=0.50) +mobilenet_v1_025 = wrapped_partial(mobilenet_v1, depth_multiplier=0.25) + + +def _reduced_kernel_size_for_small_input(input_tensor, kernel_size): + """Define kernel size which is automatically reduced for small input. + + If the shape of the input images is unknown at graph construction time this + function assumes that the input images are large enough. + + Args: + input_tensor: input tensor of size [batch_size, height, width, channels]. + kernel_size: desired kernel size of length 2: [kernel_height, kernel_width] + + Returns: + a tensor with the kernel size. + """ + shape = input_tensor.get_shape().as_list() + if shape[1] is None or shape[2] is None: + kernel_size_out = kernel_size + else: + kernel_size_out = [min(shape[1], kernel_size[0]), + min(shape[2], kernel_size[1])] + return kernel_size_out + + +def mobilenet_v1_arg_scope( + is_training=True, + weight_decay=0.00004, + stddev=0.09, + regularize_depthwise=False, + batch_norm_decay=0.9997, + batch_norm_epsilon=0.001, + batch_norm_updates_collections=tf.compat.v1.GraphKeys.UPDATE_OPS, + normalizer_fn=slim.batch_norm): + """Defines the default MobilenetV1 arg scope. + + Args: + is_training: Whether or not we're training the model. If this is set to + None, the parameter is not added to the batch_norm arg_scope. + weight_decay: The weight decay to use for regularizing the model. + stddev: The standard deviation of the trunctated normal weight initializer. + regularize_depthwise: Whether or not apply regularization on depthwise. + batch_norm_decay: Decay for batch norm moving average. + batch_norm_epsilon: Small float added to variance to avoid dividing by zero + in batch norm. + batch_norm_updates_collections: Collection for the update ops for + batch norm. + normalizer_fn: Normalization function to apply after convolution. + + Returns: + An `arg_scope` to use for the mobilenet v1 model. + """ + batch_norm_params = { + 'center': True, + 'scale': True, + 'decay': batch_norm_decay, + 'epsilon': batch_norm_epsilon, + 'updates_collections': batch_norm_updates_collections, + } + if is_training is not None: + batch_norm_params['is_training'] = is_training + + # Set weight_decay for weights in Conv and DepthSepConv layers. + weights_init = tf.compat.v1.truncated_normal_initializer(stddev=stddev) + regularizer = contrib_layers.l2_regularizer(weight_decay) + if regularize_depthwise: + depthwise_regularizer = regularizer + else: + depthwise_regularizer = None + with slim.arg_scope([slim.conv2d, slim.separable_conv2d], + weights_initializer=weights_init, + activation_fn=tf.nn.relu6, normalizer_fn=normalizer_fn): + with slim.arg_scope([slim.batch_norm], **batch_norm_params): + with slim.arg_scope([slim.conv2d], weights_regularizer=regularizer): + with slim.arg_scope([slim.separable_conv2d], + weights_regularizer=depthwise_regularizer) as sc: + return sc diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1_eval.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1_eval.py new file mode 100644 index 0000000..c7bd590 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1_eval.py @@ -0,0 +1,157 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Validate mobilenet_v1 with options for quantization.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math +import tensorflow as tf +from tensorflow.contrib import quantize as contrib_quantize +from tensorflow.contrib import slim as contrib_slim + +from datasets import dataset_factory +from nets import mobilenet_v1 +from preprocessing import preprocessing_factory + +slim = contrib_slim + +flags = tf.compat.v1.app.flags + +flags.DEFINE_string('master', '', 'Session master') +flags.DEFINE_integer('batch_size', 250, 'Batch size') +flags.DEFINE_integer('num_classes', 1001, 'Number of classes to distinguish') +flags.DEFINE_integer('num_examples', 50000, 'Number of examples to evaluate') +flags.DEFINE_integer('image_size', 224, 'Input image resolution') +flags.DEFINE_float('depth_multiplier', 1.0, 'Depth multiplier for mobilenet') +flags.DEFINE_bool('quantize', False, 'Quantize training') +flags.DEFINE_string('checkpoint_dir', '', 'The directory for checkpoints') +flags.DEFINE_string('eval_dir', '', 'Directory for writing eval event logs') +flags.DEFINE_string('dataset_dir', '', 'Location of dataset') + +FLAGS = flags.FLAGS + + +def imagenet_input(is_training): + """Data reader for imagenet. + + Reads in imagenet data and performs pre-processing on the images. + + Args: + is_training: bool specifying if train or validation dataset is needed. + Returns: + A batch of images and labels. + """ + if is_training: + dataset = dataset_factory.get_dataset('imagenet', 'train', + FLAGS.dataset_dir) + else: + dataset = dataset_factory.get_dataset('imagenet', 'validation', + FLAGS.dataset_dir) + + provider = slim.dataset_data_provider.DatasetDataProvider( + dataset, + shuffle=is_training, + common_queue_capacity=2 * FLAGS.batch_size, + common_queue_min=FLAGS.batch_size) + [image, label] = provider.get(['image', 'label']) + + image_preprocessing_fn = preprocessing_factory.get_preprocessing( + 'mobilenet_v1', is_training=is_training) + + image = image_preprocessing_fn(image, FLAGS.image_size, FLAGS.image_size) + + images, labels = tf.compat.v1.train.batch( + tensors=[image, label], + batch_size=FLAGS.batch_size, + num_threads=4, + capacity=5 * FLAGS.batch_size) + return images, labels + + +def metrics(logits, labels): + """Specify the metrics for eval. + + Args: + logits: Logits output from the graph. + labels: Ground truth labels for inputs. + + Returns: + Eval Op for the graph. + """ + labels = tf.squeeze(labels) + names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({ + 'Accuracy': + tf.compat.v1.metrics.accuracy( + tf.argmax(input=logits, axis=1), labels), + 'Recall_5': + tf.compat.v1.metrics.recall_at_k(labels, logits, 5), + }) + for name, value in names_to_values.iteritems(): + slim.summaries.add_scalar_summary( + value, name, prefix='eval', print_summary=True) + return names_to_updates.values() + + +def build_model(): + """Build the mobilenet_v1 model for evaluation. + + Returns: + g: graph with rewrites after insertion of quantization ops and batch norm + folding. + eval_ops: eval ops for inference. + variables_to_restore: List of variables to restore from checkpoint. + """ + g = tf.Graph() + with g.as_default(): + inputs, labels = imagenet_input(is_training=False) + + scope = mobilenet_v1.mobilenet_v1_arg_scope( + is_training=False, weight_decay=0.0) + with slim.arg_scope(scope): + logits, _ = mobilenet_v1.mobilenet_v1( + inputs, + is_training=False, + depth_multiplier=FLAGS.depth_multiplier, + num_classes=FLAGS.num_classes) + + if FLAGS.quantize: + contrib_quantize.create_eval_graph() + + eval_ops = metrics(logits, labels) + + return g, eval_ops + + +def eval_model(): + """Evaluates mobilenet_v1.""" + g, eval_ops = build_model() + with g.as_default(): + num_batches = math.ceil(FLAGS.num_examples / float(FLAGS.batch_size)) + slim.evaluation.evaluate_once( + FLAGS.master, + FLAGS.checkpoint_dir, + logdir=FLAGS.eval_dir, + num_evals=num_batches, + eval_op=eval_ops) + + +def main(unused_arg): + eval_model() + + +if __name__ == '__main__': + tf.compat.v1.app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1_test.py new file mode 100644 index 0000000..3f8d9b2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1_test.py @@ -0,0 +1,537 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +"""Tests for MobileNet v1.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets import mobilenet_v1 + +slim = contrib_slim + + +class MobilenetV1Test(tf.test.TestCase): + + def testBuildClassificationNetwork(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, end_points = mobilenet_v1.mobilenet_v1(inputs, num_classes) + self.assertTrue(logits.op.name.startswith( + 'MobilenetV1/Logits/SpatialSqueeze')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertTrue('Predictions' in end_points) + self.assertListEqual(end_points['Predictions'].get_shape().as_list(), + [batch_size, num_classes]) + + def testBuildPreLogitsNetwork(self): + batch_size = 5 + height, width = 224, 224 + num_classes = None + + inputs = tf.random.uniform((batch_size, height, width, 3)) + net, end_points = mobilenet_v1.mobilenet_v1(inputs, num_classes) + self.assertTrue(net.op.name.startswith('MobilenetV1/Logits/AvgPool')) + self.assertListEqual(net.get_shape().as_list(), [batch_size, 1, 1, 1024]) + self.assertFalse('Logits' in end_points) + self.assertFalse('Predictions' in end_points) + + def testBuildBaseNetwork(self): + batch_size = 5 + height, width = 224, 224 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + net, end_points = mobilenet_v1.mobilenet_v1_base(inputs) + self.assertTrue(net.op.name.startswith('MobilenetV1/Conv2d_13')) + self.assertListEqual(net.get_shape().as_list(), + [batch_size, 7, 7, 1024]) + expected_endpoints = ['Conv2d_0', + 'Conv2d_1_depthwise', 'Conv2d_1_pointwise', + 'Conv2d_2_depthwise', 'Conv2d_2_pointwise', + 'Conv2d_3_depthwise', 'Conv2d_3_pointwise', + 'Conv2d_4_depthwise', 'Conv2d_4_pointwise', + 'Conv2d_5_depthwise', 'Conv2d_5_pointwise', + 'Conv2d_6_depthwise', 'Conv2d_6_pointwise', + 'Conv2d_7_depthwise', 'Conv2d_7_pointwise', + 'Conv2d_8_depthwise', 'Conv2d_8_pointwise', + 'Conv2d_9_depthwise', 'Conv2d_9_pointwise', + 'Conv2d_10_depthwise', 'Conv2d_10_pointwise', + 'Conv2d_11_depthwise', 'Conv2d_11_pointwise', + 'Conv2d_12_depthwise', 'Conv2d_12_pointwise', + 'Conv2d_13_depthwise', 'Conv2d_13_pointwise'] + self.assertItemsEqual(end_points.keys(), expected_endpoints) + + def testBuildOnlyUptoFinalEndpoint(self): + batch_size = 5 + height, width = 224, 224 + endpoints = ['Conv2d_0', + 'Conv2d_1_depthwise', 'Conv2d_1_pointwise', + 'Conv2d_2_depthwise', 'Conv2d_2_pointwise', + 'Conv2d_3_depthwise', 'Conv2d_3_pointwise', + 'Conv2d_4_depthwise', 'Conv2d_4_pointwise', + 'Conv2d_5_depthwise', 'Conv2d_5_pointwise', + 'Conv2d_6_depthwise', 'Conv2d_6_pointwise', + 'Conv2d_7_depthwise', 'Conv2d_7_pointwise', + 'Conv2d_8_depthwise', 'Conv2d_8_pointwise', + 'Conv2d_9_depthwise', 'Conv2d_9_pointwise', + 'Conv2d_10_depthwise', 'Conv2d_10_pointwise', + 'Conv2d_11_depthwise', 'Conv2d_11_pointwise', + 'Conv2d_12_depthwise', 'Conv2d_12_pointwise', + 'Conv2d_13_depthwise', 'Conv2d_13_pointwise'] + for index, endpoint in enumerate(endpoints): + with tf.Graph().as_default(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + out_tensor, end_points = mobilenet_v1.mobilenet_v1_base( + inputs, final_endpoint=endpoint) + self.assertTrue(out_tensor.op.name.startswith( + 'MobilenetV1/' + endpoint)) + self.assertItemsEqual(endpoints[:index+1], end_points.keys()) + + def testBuildCustomNetworkUsingConvDefs(self): + batch_size = 5 + height, width = 224, 224 + conv_defs = [ + mobilenet_v1.Conv(kernel=[3, 3], stride=2, depth=32), + mobilenet_v1.DepthSepConv(kernel=[3, 3], stride=1, depth=64), + mobilenet_v1.DepthSepConv(kernel=[3, 3], stride=2, depth=128), + mobilenet_v1.DepthSepConv(kernel=[3, 3], stride=1, depth=512) + ] + + inputs = tf.random.uniform((batch_size, height, width, 3)) + net, end_points = mobilenet_v1.mobilenet_v1_base( + inputs, final_endpoint='Conv2d_3_pointwise', conv_defs=conv_defs) + self.assertTrue(net.op.name.startswith('MobilenetV1/Conv2d_3')) + self.assertListEqual(net.get_shape().as_list(), + [batch_size, 56, 56, 512]) + expected_endpoints = ['Conv2d_0', + 'Conv2d_1_depthwise', 'Conv2d_1_pointwise', + 'Conv2d_2_depthwise', 'Conv2d_2_pointwise', + 'Conv2d_3_depthwise', 'Conv2d_3_pointwise'] + self.assertItemsEqual(end_points.keys(), expected_endpoints) + + def testBuildAndCheckAllEndPointsUptoConv2d_13(self): + batch_size = 5 + height, width = 224, 224 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + with slim.arg_scope([slim.conv2d, slim.separable_conv2d], + normalizer_fn=slim.batch_norm): + _, end_points = mobilenet_v1.mobilenet_v1_base( + inputs, final_endpoint='Conv2d_13_pointwise') + _, explicit_padding_end_points = mobilenet_v1.mobilenet_v1_base( + inputs, final_endpoint='Conv2d_13_pointwise', + use_explicit_padding=True) + endpoints_shapes = {'Conv2d_0': [batch_size, 112, 112, 32], + 'Conv2d_1_depthwise': [batch_size, 112, 112, 32], + 'Conv2d_1_pointwise': [batch_size, 112, 112, 64], + 'Conv2d_2_depthwise': [batch_size, 56, 56, 64], + 'Conv2d_2_pointwise': [batch_size, 56, 56, 128], + 'Conv2d_3_depthwise': [batch_size, 56, 56, 128], + 'Conv2d_3_pointwise': [batch_size, 56, 56, 128], + 'Conv2d_4_depthwise': [batch_size, 28, 28, 128], + 'Conv2d_4_pointwise': [batch_size, 28, 28, 256], + 'Conv2d_5_depthwise': [batch_size, 28, 28, 256], + 'Conv2d_5_pointwise': [batch_size, 28, 28, 256], + 'Conv2d_6_depthwise': [batch_size, 14, 14, 256], + 'Conv2d_6_pointwise': [batch_size, 14, 14, 512], + 'Conv2d_7_depthwise': [batch_size, 14, 14, 512], + 'Conv2d_7_pointwise': [batch_size, 14, 14, 512], + 'Conv2d_8_depthwise': [batch_size, 14, 14, 512], + 'Conv2d_8_pointwise': [batch_size, 14, 14, 512], + 'Conv2d_9_depthwise': [batch_size, 14, 14, 512], + 'Conv2d_9_pointwise': [batch_size, 14, 14, 512], + 'Conv2d_10_depthwise': [batch_size, 14, 14, 512], + 'Conv2d_10_pointwise': [batch_size, 14, 14, 512], + 'Conv2d_11_depthwise': [batch_size, 14, 14, 512], + 'Conv2d_11_pointwise': [batch_size, 14, 14, 512], + 'Conv2d_12_depthwise': [batch_size, 7, 7, 512], + 'Conv2d_12_pointwise': [batch_size, 7, 7, 1024], + 'Conv2d_13_depthwise': [batch_size, 7, 7, 1024], + 'Conv2d_13_pointwise': [batch_size, 7, 7, 1024]} + self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) + for endpoint_name, expected_shape in endpoints_shapes.items(): + self.assertTrue(endpoint_name in end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + self.assertItemsEqual(endpoints_shapes.keys(), + explicit_padding_end_points.keys()) + for endpoint_name, expected_shape in endpoints_shapes.items(): + self.assertTrue(endpoint_name in explicit_padding_end_points) + self.assertListEqual( + explicit_padding_end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testOutputStride16BuildAndCheckAllEndPointsUptoConv2d_13(self): + batch_size = 5 + height, width = 224, 224 + output_stride = 16 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + with slim.arg_scope([slim.conv2d, slim.separable_conv2d], + normalizer_fn=slim.batch_norm): + _, end_points = mobilenet_v1.mobilenet_v1_base( + inputs, output_stride=output_stride, + final_endpoint='Conv2d_13_pointwise') + _, explicit_padding_end_points = mobilenet_v1.mobilenet_v1_base( + inputs, output_stride=output_stride, + final_endpoint='Conv2d_13_pointwise', use_explicit_padding=True) + endpoints_shapes = {'Conv2d_0': [batch_size, 112, 112, 32], + 'Conv2d_1_depthwise': [batch_size, 112, 112, 32], + 'Conv2d_1_pointwise': [batch_size, 112, 112, 64], + 'Conv2d_2_depthwise': [batch_size, 56, 56, 64], + 'Conv2d_2_pointwise': [batch_size, 56, 56, 128], + 'Conv2d_3_depthwise': [batch_size, 56, 56, 128], + 'Conv2d_3_pointwise': [batch_size, 56, 56, 128], + 'Conv2d_4_depthwise': [batch_size, 28, 28, 128], + 'Conv2d_4_pointwise': [batch_size, 28, 28, 256], + 'Conv2d_5_depthwise': [batch_size, 28, 28, 256], + 'Conv2d_5_pointwise': [batch_size, 28, 28, 256], + 'Conv2d_6_depthwise': [batch_size, 14, 14, 256], + 'Conv2d_6_pointwise': [batch_size, 14, 14, 512], + 'Conv2d_7_depthwise': [batch_size, 14, 14, 512], + 'Conv2d_7_pointwise': [batch_size, 14, 14, 512], + 'Conv2d_8_depthwise': [batch_size, 14, 14, 512], + 'Conv2d_8_pointwise': [batch_size, 14, 14, 512], + 'Conv2d_9_depthwise': [batch_size, 14, 14, 512], + 'Conv2d_9_pointwise': [batch_size, 14, 14, 512], + 'Conv2d_10_depthwise': [batch_size, 14, 14, 512], + 'Conv2d_10_pointwise': [batch_size, 14, 14, 512], + 'Conv2d_11_depthwise': [batch_size, 14, 14, 512], + 'Conv2d_11_pointwise': [batch_size, 14, 14, 512], + 'Conv2d_12_depthwise': [batch_size, 14, 14, 512], + 'Conv2d_12_pointwise': [batch_size, 14, 14, 1024], + 'Conv2d_13_depthwise': [batch_size, 14, 14, 1024], + 'Conv2d_13_pointwise': [batch_size, 14, 14, 1024]} + self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) + for endpoint_name, expected_shape in endpoints_shapes.items(): + self.assertTrue(endpoint_name in end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + self.assertItemsEqual(endpoints_shapes.keys(), + explicit_padding_end_points.keys()) + for endpoint_name, expected_shape in endpoints_shapes.items(): + self.assertTrue(endpoint_name in explicit_padding_end_points) + self.assertListEqual( + explicit_padding_end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testOutputStride8BuildAndCheckAllEndPointsUptoConv2d_13(self): + batch_size = 5 + height, width = 224, 224 + output_stride = 8 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + with slim.arg_scope([slim.conv2d, slim.separable_conv2d], + normalizer_fn=slim.batch_norm): + _, end_points = mobilenet_v1.mobilenet_v1_base( + inputs, output_stride=output_stride, + final_endpoint='Conv2d_13_pointwise') + _, explicit_padding_end_points = mobilenet_v1.mobilenet_v1_base( + inputs, output_stride=output_stride, + final_endpoint='Conv2d_13_pointwise', use_explicit_padding=True) + endpoints_shapes = {'Conv2d_0': [batch_size, 112, 112, 32], + 'Conv2d_1_depthwise': [batch_size, 112, 112, 32], + 'Conv2d_1_pointwise': [batch_size, 112, 112, 64], + 'Conv2d_2_depthwise': [batch_size, 56, 56, 64], + 'Conv2d_2_pointwise': [batch_size, 56, 56, 128], + 'Conv2d_3_depthwise': [batch_size, 56, 56, 128], + 'Conv2d_3_pointwise': [batch_size, 56, 56, 128], + 'Conv2d_4_depthwise': [batch_size, 28, 28, 128], + 'Conv2d_4_pointwise': [batch_size, 28, 28, 256], + 'Conv2d_5_depthwise': [batch_size, 28, 28, 256], + 'Conv2d_5_pointwise': [batch_size, 28, 28, 256], + 'Conv2d_6_depthwise': [batch_size, 28, 28, 256], + 'Conv2d_6_pointwise': [batch_size, 28, 28, 512], + 'Conv2d_7_depthwise': [batch_size, 28, 28, 512], + 'Conv2d_7_pointwise': [batch_size, 28, 28, 512], + 'Conv2d_8_depthwise': [batch_size, 28, 28, 512], + 'Conv2d_8_pointwise': [batch_size, 28, 28, 512], + 'Conv2d_9_depthwise': [batch_size, 28, 28, 512], + 'Conv2d_9_pointwise': [batch_size, 28, 28, 512], + 'Conv2d_10_depthwise': [batch_size, 28, 28, 512], + 'Conv2d_10_pointwise': [batch_size, 28, 28, 512], + 'Conv2d_11_depthwise': [batch_size, 28, 28, 512], + 'Conv2d_11_pointwise': [batch_size, 28, 28, 512], + 'Conv2d_12_depthwise': [batch_size, 28, 28, 512], + 'Conv2d_12_pointwise': [batch_size, 28, 28, 1024], + 'Conv2d_13_depthwise': [batch_size, 28, 28, 1024], + 'Conv2d_13_pointwise': [batch_size, 28, 28, 1024]} + self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) + for endpoint_name, expected_shape in endpoints_shapes.items(): + self.assertTrue(endpoint_name in end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + self.assertItemsEqual(endpoints_shapes.keys(), + explicit_padding_end_points.keys()) + for endpoint_name, expected_shape in endpoints_shapes.items(): + self.assertTrue(endpoint_name in explicit_padding_end_points) + self.assertListEqual( + explicit_padding_end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testBuildAndCheckAllEndPointsApproximateFaceNet(self): + batch_size = 5 + height, width = 128, 128 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + with slim.arg_scope([slim.conv2d, slim.separable_conv2d], + normalizer_fn=slim.batch_norm): + _, end_points = mobilenet_v1.mobilenet_v1_base( + inputs, final_endpoint='Conv2d_13_pointwise', depth_multiplier=0.75) + _, explicit_padding_end_points = mobilenet_v1.mobilenet_v1_base( + inputs, final_endpoint='Conv2d_13_pointwise', depth_multiplier=0.75, + use_explicit_padding=True) + # For the Conv2d_0 layer FaceNet has depth=16 + endpoints_shapes = {'Conv2d_0': [batch_size, 64, 64, 24], + 'Conv2d_1_depthwise': [batch_size, 64, 64, 24], + 'Conv2d_1_pointwise': [batch_size, 64, 64, 48], + 'Conv2d_2_depthwise': [batch_size, 32, 32, 48], + 'Conv2d_2_pointwise': [batch_size, 32, 32, 96], + 'Conv2d_3_depthwise': [batch_size, 32, 32, 96], + 'Conv2d_3_pointwise': [batch_size, 32, 32, 96], + 'Conv2d_4_depthwise': [batch_size, 16, 16, 96], + 'Conv2d_4_pointwise': [batch_size, 16, 16, 192], + 'Conv2d_5_depthwise': [batch_size, 16, 16, 192], + 'Conv2d_5_pointwise': [batch_size, 16, 16, 192], + 'Conv2d_6_depthwise': [batch_size, 8, 8, 192], + 'Conv2d_6_pointwise': [batch_size, 8, 8, 384], + 'Conv2d_7_depthwise': [batch_size, 8, 8, 384], + 'Conv2d_7_pointwise': [batch_size, 8, 8, 384], + 'Conv2d_8_depthwise': [batch_size, 8, 8, 384], + 'Conv2d_8_pointwise': [batch_size, 8, 8, 384], + 'Conv2d_9_depthwise': [batch_size, 8, 8, 384], + 'Conv2d_9_pointwise': [batch_size, 8, 8, 384], + 'Conv2d_10_depthwise': [batch_size, 8, 8, 384], + 'Conv2d_10_pointwise': [batch_size, 8, 8, 384], + 'Conv2d_11_depthwise': [batch_size, 8, 8, 384], + 'Conv2d_11_pointwise': [batch_size, 8, 8, 384], + 'Conv2d_12_depthwise': [batch_size, 4, 4, 384], + 'Conv2d_12_pointwise': [batch_size, 4, 4, 768], + 'Conv2d_13_depthwise': [batch_size, 4, 4, 768], + 'Conv2d_13_pointwise': [batch_size, 4, 4, 768]} + self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) + for endpoint_name, expected_shape in endpoints_shapes.items(): + self.assertTrue(endpoint_name in end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + self.assertItemsEqual(endpoints_shapes.keys(), + explicit_padding_end_points.keys()) + for endpoint_name, expected_shape in endpoints_shapes.items(): + self.assertTrue(endpoint_name in explicit_padding_end_points) + self.assertListEqual( + explicit_padding_end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testModelHasExpectedNumberOfParameters(self): + batch_size = 5 + height, width = 224, 224 + inputs = tf.random.uniform((batch_size, height, width, 3)) + with slim.arg_scope([slim.conv2d, slim.separable_conv2d], + normalizer_fn=slim.batch_norm): + mobilenet_v1.mobilenet_v1_base(inputs) + total_params, _ = slim.model_analyzer.analyze_vars( + slim.get_model_variables()) + self.assertAlmostEqual(3217920, total_params) + + def testBuildEndPointsWithDepthMultiplierLessThanOne(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = mobilenet_v1.mobilenet_v1(inputs, num_classes) + + endpoint_keys = [key for key in end_points.keys() if key.startswith('Conv')] + + _, end_points_with_multiplier = mobilenet_v1.mobilenet_v1( + inputs, num_classes, scope='depth_multiplied_net', + depth_multiplier=0.5) + + for key in endpoint_keys: + original_depth = end_points[key].get_shape().as_list()[3] + new_depth = end_points_with_multiplier[key].get_shape().as_list()[3] + self.assertEqual(0.5 * original_depth, new_depth) + + def testBuildEndPointsWithDepthMultiplierGreaterThanOne(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = mobilenet_v1.mobilenet_v1(inputs, num_classes) + + endpoint_keys = [key for key in end_points.keys() + if key.startswith('Mixed') or key.startswith('Conv')] + + _, end_points_with_multiplier = mobilenet_v1.mobilenet_v1( + inputs, num_classes, scope='depth_multiplied_net', + depth_multiplier=2.0) + + for key in endpoint_keys: + original_depth = end_points[key].get_shape().as_list()[3] + new_depth = end_points_with_multiplier[key].get_shape().as_list()[3] + self.assertEqual(2.0 * original_depth, new_depth) + + def testRaiseValueErrorWithInvalidDepthMultiplier(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + with self.assertRaises(ValueError): + _ = mobilenet_v1.mobilenet_v1( + inputs, num_classes, depth_multiplier=-0.1) + with self.assertRaises(ValueError): + _ = mobilenet_v1.mobilenet_v1( + inputs, num_classes, depth_multiplier=0.0) + + def testHalfSizeImages(self): + batch_size = 5 + height, width = 112, 112 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, end_points = mobilenet_v1.mobilenet_v1(inputs, num_classes) + self.assertTrue(logits.op.name.startswith('MobilenetV1/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + pre_pool = end_points['Conv2d_13_pointwise'] + self.assertListEqual(pre_pool.get_shape().as_list(), + [batch_size, 4, 4, 1024]) + + def testUnknownImageShape(self): + tf.compat.v1.reset_default_graph() + batch_size = 2 + height, width = 224, 224 + num_classes = 1000 + input_np = np.random.uniform(0, 1, (batch_size, height, width, 3)) + with self.test_session() as sess: + inputs = tf.compat.v1.placeholder( + tf.float32, shape=(batch_size, None, None, 3)) + logits, end_points = mobilenet_v1.mobilenet_v1(inputs, num_classes) + self.assertTrue(logits.op.name.startswith('MobilenetV1/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + pre_pool = end_points['Conv2d_13_pointwise'] + feed_dict = {inputs: input_np} + tf.compat.v1.global_variables_initializer().run() + pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict) + self.assertListEqual(list(pre_pool_out.shape), [batch_size, 7, 7, 1024]) + + def testGlobalPoolUnknownImageShape(self): + tf.compat.v1.reset_default_graph() + batch_size = 1 + height, width = 250, 300 + num_classes = 1000 + input_np = np.random.uniform(0, 1, (batch_size, height, width, 3)) + with self.test_session() as sess: + inputs = tf.compat.v1.placeholder( + tf.float32, shape=(batch_size, None, None, 3)) + logits, end_points = mobilenet_v1.mobilenet_v1(inputs, num_classes, + global_pool=True) + self.assertTrue(logits.op.name.startswith('MobilenetV1/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + pre_pool = end_points['Conv2d_13_pointwise'] + feed_dict = {inputs: input_np} + tf.compat.v1.global_variables_initializer().run() + pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict) + self.assertListEqual(list(pre_pool_out.shape), [batch_size, 8, 10, 1024]) + + def testUnknowBatchSize(self): + batch_size = 1 + height, width = 224, 224 + num_classes = 1000 + + inputs = tf.compat.v1.placeholder(tf.float32, (None, height, width, 3)) + logits, _ = mobilenet_v1.mobilenet_v1(inputs, num_classes) + self.assertTrue(logits.op.name.startswith('MobilenetV1/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [None, num_classes]) + images = tf.random.uniform((batch_size, height, width, 3)) + + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(logits, {inputs: images.eval()}) + self.assertEquals(output.shape, (batch_size, num_classes)) + + def testEvaluation(self): + batch_size = 2 + height, width = 224, 224 + num_classes = 1000 + + eval_inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = mobilenet_v1.mobilenet_v1(eval_inputs, num_classes, + is_training=False) + predictions = tf.argmax(input=logits, axis=1) + + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(predictions) + self.assertEquals(output.shape, (batch_size,)) + + def testTrainEvalWithReuse(self): + train_batch_size = 5 + eval_batch_size = 2 + height, width = 150, 150 + num_classes = 1000 + + train_inputs = tf.random.uniform((train_batch_size, height, width, 3)) + mobilenet_v1.mobilenet_v1(train_inputs, num_classes) + eval_inputs = tf.random.uniform((eval_batch_size, height, width, 3)) + logits, _ = mobilenet_v1.mobilenet_v1(eval_inputs, num_classes, + reuse=True) + predictions = tf.argmax(input=logits, axis=1) + + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(predictions) + self.assertEquals(output.shape, (eval_batch_size,)) + + def testLogitsNotSqueezed(self): + num_classes = 25 + images = tf.random.uniform([1, 224, 224, 3]) + logits, _ = mobilenet_v1.mobilenet_v1(images, + num_classes=num_classes, + spatial_squeeze=False) + + with self.test_session() as sess: + tf.compat.v1.global_variables_initializer().run() + logits_out = sess.run(logits) + self.assertListEqual(list(logits_out.shape), [1, 1, 1, num_classes]) + + def testBatchNormScopeDoesNotHaveIsTrainingWhenItsSetToNone(self): + sc = mobilenet_v1.mobilenet_v1_arg_scope(is_training=None) + self.assertNotIn('is_training', sc[slim.arg_scope_func_key( + slim.batch_norm)]) + + def testBatchNormScopeDoesHasIsTrainingWhenItsNotNone(self): + sc = mobilenet_v1.mobilenet_v1_arg_scope(is_training=True) + self.assertIn('is_training', sc[slim.arg_scope_func_key(slim.batch_norm)]) + sc = mobilenet_v1.mobilenet_v1_arg_scope(is_training=False) + self.assertIn('is_training', sc[slim.arg_scope_func_key(slim.batch_norm)]) + sc = mobilenet_v1.mobilenet_v1_arg_scope() + self.assertIn('is_training', sc[slim.arg_scope_func_key(slim.batch_norm)]) + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1_train.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1_train.py new file mode 100644 index 0000000..1035ea3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/mobilenet_v1_train.py @@ -0,0 +1,214 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Build and train mobilenet_v1 with options for quantization.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import quantize as contrib_quantize +from tensorflow.contrib import slim as contrib_slim + +from datasets import dataset_factory +from nets import mobilenet_v1 +from preprocessing import preprocessing_factory + +slim = contrib_slim + +flags = tf.compat.v1.app.flags + +flags.DEFINE_string('master', '', 'Session master') +flags.DEFINE_integer('task', 0, 'Task') +flags.DEFINE_integer('ps_tasks', 0, 'Number of ps') +flags.DEFINE_integer('batch_size', 64, 'Batch size') +flags.DEFINE_integer('num_classes', 1001, 'Number of classes to distinguish') +flags.DEFINE_integer('number_of_steps', None, + 'Number of training steps to perform before stopping') +flags.DEFINE_integer('image_size', 224, 'Input image resolution') +flags.DEFINE_float('depth_multiplier', 1.0, 'Depth multiplier for mobilenet') +flags.DEFINE_bool('quantize', False, 'Quantize training') +flags.DEFINE_string('fine_tune_checkpoint', '', + 'Checkpoint from which to start finetuning.') +flags.DEFINE_string('checkpoint_dir', '', + 'Directory for writing training checkpoints and logs') +flags.DEFINE_string('dataset_dir', '', 'Location of dataset') +flags.DEFINE_integer('log_every_n_steps', 100, 'Number of steps per log') +flags.DEFINE_integer('save_summaries_secs', 100, + 'How often to save summaries, secs') +flags.DEFINE_integer('save_interval_secs', 100, + 'How often to save checkpoints, secs') + +FLAGS = flags.FLAGS + +_LEARNING_RATE_DECAY_FACTOR = 0.94 + + +def get_learning_rate(): + if FLAGS.fine_tune_checkpoint: + # If we are fine tuning a checkpoint we need to start at a lower learning + # rate since we are farther along on training. + return 1e-4 + else: + return 0.045 + + +def get_quant_delay(): + if FLAGS.fine_tune_checkpoint: + # We can start quantizing immediately if we are finetuning. + return 0 + else: + # We need to wait for the model to train a bit before we quantize if we are + # training from scratch. + return 250000 + + +def imagenet_input(is_training): + """Data reader for imagenet. + + Reads in imagenet data and performs pre-processing on the images. + + Args: + is_training: bool specifying if train or validation dataset is needed. + Returns: + A batch of images and labels. + """ + if is_training: + dataset = dataset_factory.get_dataset('imagenet', 'train', + FLAGS.dataset_dir) + else: + dataset = dataset_factory.get_dataset('imagenet', 'validation', + FLAGS.dataset_dir) + + provider = slim.dataset_data_provider.DatasetDataProvider( + dataset, + shuffle=is_training, + common_queue_capacity=2 * FLAGS.batch_size, + common_queue_min=FLAGS.batch_size) + [image, label] = provider.get(['image', 'label']) + + image_preprocessing_fn = preprocessing_factory.get_preprocessing( + 'mobilenet_v1', is_training=is_training) + + image = image_preprocessing_fn(image, FLAGS.image_size, FLAGS.image_size) + + images, labels = tf.compat.v1.train.batch([image, label], + batch_size=FLAGS.batch_size, + num_threads=4, + capacity=5 * FLAGS.batch_size) + labels = slim.one_hot_encoding(labels, FLAGS.num_classes) + return images, labels + + +def build_model(): + """Builds graph for model to train with rewrites for quantization. + + Returns: + g: Graph with fake quantization ops and batch norm folding suitable for + training quantized weights. + train_tensor: Train op for execution during training. + """ + g = tf.Graph() + with g.as_default(), tf.device( + tf.compat.v1.train.replica_device_setter(FLAGS.ps_tasks)): + inputs, labels = imagenet_input(is_training=True) + with slim.arg_scope(mobilenet_v1.mobilenet_v1_arg_scope(is_training=True)): + logits, _ = mobilenet_v1.mobilenet_v1( + inputs, + is_training=True, + depth_multiplier=FLAGS.depth_multiplier, + num_classes=FLAGS.num_classes) + + tf.compat.v1.losses.softmax_cross_entropy(labels, logits) + + # Call rewriter to produce graph with fake quant ops and folded batch norms + # quant_delay delays start of quantization till quant_delay steps, allowing + # for better model accuracy. + if FLAGS.quantize: + contrib_quantize.create_training_graph(quant_delay=get_quant_delay()) + + total_loss = tf.compat.v1.losses.get_total_loss(name='total_loss') + # Configure the learning rate using an exponential decay. + num_epochs_per_decay = 2.5 + imagenet_size = 1271167 + decay_steps = int(imagenet_size / FLAGS.batch_size * num_epochs_per_decay) + + learning_rate = tf.compat.v1.train.exponential_decay( + get_learning_rate(), + tf.compat.v1.train.get_or_create_global_step(), + decay_steps, + _LEARNING_RATE_DECAY_FACTOR, + staircase=True) + opt = tf.compat.v1.train.GradientDescentOptimizer(learning_rate) + + train_tensor = slim.learning.create_train_op( + total_loss, + optimizer=opt) + + slim.summaries.add_scalar_summary(total_loss, 'total_loss', 'losses') + slim.summaries.add_scalar_summary(learning_rate, 'learning_rate', 'training') + return g, train_tensor + + +def get_checkpoint_init_fn(): + """Returns the checkpoint init_fn if the checkpoint is provided.""" + if FLAGS.fine_tune_checkpoint: + variables_to_restore = slim.get_variables_to_restore() + global_step_reset = tf.compat.v1.assign( + tf.compat.v1.train.get_or_create_global_step(), 0) + # When restoring from a floating point model, the min/max values for + # quantized weights and activations are not present. + # We instruct slim to ignore variables that are missing during restoration + # by setting ignore_missing_vars=True + slim_init_fn = slim.assign_from_checkpoint_fn( + FLAGS.fine_tune_checkpoint, + variables_to_restore, + ignore_missing_vars=True) + + def init_fn(sess): + slim_init_fn(sess) + # If we are restoring from a floating point model, we need to initialize + # the global step to zero for the exponential decay to result in + # reasonable learning rates. + sess.run(global_step_reset) + return init_fn + else: + return None + + +def train_model(): + """Trains mobilenet_v1.""" + g, train_tensor = build_model() + with g.as_default(): + slim.learning.train( + train_tensor, + FLAGS.checkpoint_dir, + is_chief=(FLAGS.task == 0), + master=FLAGS.master, + log_every_n_steps=FLAGS.log_every_n_steps, + graph=g, + number_of_steps=FLAGS.number_of_steps, + save_summaries_secs=FLAGS.save_summaries_secs, + save_interval_secs=FLAGS.save_interval_secs, + init_fn=get_checkpoint_init_fn(), + global_step=tf.compat.v1.train.get_global_step()) + + +def main(unused_arg): + train_model() + + +if __name__ == '__main__': + tf.compat.v1.app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/README.md b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/README.md new file mode 100644 index 0000000..3955ad1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/README.md @@ -0,0 +1,64 @@ +# TensorFlow-Slim NASNet-A Implementation/Checkpoints +This directory contains the code for the NASNet-A model from the paper +[Learning Transferable Architectures for Scalable Image Recognition](https://arxiv.org/abs/1707.07012) by Zoph et al. +In nasnet.py there are three different configurations of NASNet-A that are implementented. One of the models is the NASNet-A built for CIFAR-10 and the +other two are variants of NASNet-A trained on ImageNet, which are listed below. + +# Pre-Trained Models +Two NASNet-A checkpoints are available that have been trained on the +[ILSVRC-2012-CLS](http://www.image-net.org/challenges/LSVRC/2012/) +image classification dataset. Accuracies were computed by evaluating using a single image crop. + +Model Checkpoint | Million MACs | Million Parameters | Top-1 Accuracy| Top-5 Accuracy | +:----:|:------------:|:----------:|:-------:|:-------:| +[NASNet-A_Mobile_224](https://storage.googleapis.com/download.tensorflow.org/models/nasnet-a_mobile_04_10_2017.tar.gz)|564|5.3|74.0|91.6| +[NASNet-A_Large_331](https://storage.googleapis.com/download.tensorflow.org/models/nasnet-a_large_04_10_2017.tar.gz)|23800|88.9|82.7|96.2| + + +Here is an example of how to download the NASNet-A_Mobile_224 checkpoint. The way to download the NASNet-A_Large_331 is the same. + +```shell +CHECKPOINT_DIR=/tmp/checkpoints +mkdir ${CHECKPOINT_DIR} +cd ${CHECKPOINT_DIR} +wget https://storage.googleapis.com/download.tensorflow.org/models/nasnet-a_mobile_04_10_2017.tar.gz +tar -xvf nasnet-a_mobile_04_10_2017.tar.gz +rm nasnet-a_mobile_04_10_2017.tar.gz +``` +More information on integrating NASNet Models into your project can be found at the [TF-Slim Image Classification Library](https://github.com/tensorflow/models/blob/master/research/slim/README.md). + +To get started running models on-device go to [TensorFlow Mobile](https://www.tensorflow.org/mobile/). + +## Sample Commands for using NASNet-A Mobile and Large Checkpoints for Inference +------- +Run eval with the NASNet-A mobile ImageNet model + +```shell +DATASET_DIR=/tmp/imagenet +EVAL_DIR=/tmp/tfmodel/eval +CHECKPOINT_DIR=/tmp/checkpoints/model.ckpt +python tensorflow_models/research/slim/eval_image_classifier \ +--checkpoint_path=${CHECKPOINT_DIR} \ +--eval_dir=${EVAL_DIR} \ +--dataset_dir=${DATASET_DIR} \ +--dataset_name=imagenet \ +--dataset_split_name=validation \ +--model_name=nasnet_mobile \ +--eval_image_size=224 +``` + +Run eval with the NASNet-A large ImageNet model + +```shell +DATASET_DIR=/tmp/imagenet +EVAL_DIR=/tmp/tfmodel/eval +CHECKPOINT_DIR=/tmp/checkpoints/model.ckpt +python tensorflow_models/research/slim/eval_image_classifier \ +--checkpoint_path=${CHECKPOINT_DIR} \ +--eval_dir=${EVAL_DIR} \ +--dataset_dir=${DATASET_DIR} \ +--dataset_name=imagenet \ +--dataset_split_name=validation \ +--model_name=nasnet_large \ +--eval_image_size=331 +``` diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/__init__.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/__init__.py @@ -0,0 +1 @@ + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/nasnet.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/nasnet.py new file mode 100644 index 0000000..664fa30 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/nasnet.py @@ -0,0 +1,554 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains the definition for the NASNet classification networks. + +Paper: https://arxiv.org/abs/1707.07012 +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import copy +import tensorflow as tf +from tensorflow.contrib import framework as contrib_framework +from tensorflow.contrib import layers as contrib_layers +from tensorflow.contrib import slim as contrib_slim +from tensorflow.contrib import training as contrib_training + +from nets.nasnet import nasnet_utils + +arg_scope = contrib_framework.arg_scope +slim = contrib_slim + + +# Notes for training NASNet Cifar Model +# ------------------------------------- +# batch_size: 32 +# learning rate: 0.025 +# cosine (single period) learning rate decay +# auxiliary head loss weighting: 0.4 +# clip global norm of all gradients by 5 +def cifar_config(): + return contrib_training.HParams( + stem_multiplier=3.0, + drop_path_keep_prob=0.6, + num_cells=18, + use_aux_head=1, + num_conv_filters=32, + dense_dropout_keep_prob=1.0, + filter_scaling_rate=2.0, + num_reduction_layers=2, + data_format='NHWC', + skip_reduction_layer_input=0, + # 600 epochs with a batch size of 32 + # This is used for the drop path probabilities since it needs to increase + # the drop out probability over the course of training. + total_training_steps=937500, + use_bounded_activation=False, + ) + + +# Notes for training large NASNet model on ImageNet +# ------------------------------------- +# batch size (per replica): 16 +# learning rate: 0.015 * 100 +# learning rate decay factor: 0.97 +# num epochs per decay: 2.4 +# sync sgd with 100 replicas +# auxiliary head loss weighting: 0.4 +# label smoothing: 0.1 +# clip global norm of all gradients by 10 +def large_imagenet_config(): + return contrib_training.HParams( + stem_multiplier=3.0, + dense_dropout_keep_prob=0.5, + num_cells=18, + filter_scaling_rate=2.0, + num_conv_filters=168, + drop_path_keep_prob=0.7, + use_aux_head=1, + num_reduction_layers=2, + data_format='NHWC', + skip_reduction_layer_input=1, + total_training_steps=250000, + use_bounded_activation=False, + ) + + +# Notes for training the mobile NASNet ImageNet model +# ------------------------------------- +# batch size (per replica): 32 +# learning rate: 0.04 * 50 +# learning rate scaling factor: 0.97 +# num epochs per decay: 2.4 +# sync sgd with 50 replicas +# auxiliary head weighting: 0.4 +# label smoothing: 0.1 +# clip global norm of all gradients by 10 +def mobile_imagenet_config(): + return contrib_training.HParams( + stem_multiplier=1.0, + dense_dropout_keep_prob=0.5, + num_cells=12, + filter_scaling_rate=2.0, + drop_path_keep_prob=1.0, + num_conv_filters=44, + use_aux_head=1, + num_reduction_layers=2, + data_format='NHWC', + skip_reduction_layer_input=0, + total_training_steps=250000, + use_bounded_activation=False, + ) + + +def _update_hparams(hparams, is_training): + """Update hparams for given is_training option.""" + if not is_training: + hparams.set_hparam('drop_path_keep_prob', 1.0) + + +def nasnet_cifar_arg_scope(weight_decay=5e-4, + batch_norm_decay=0.9, + batch_norm_epsilon=1e-5): + """Defines the default arg scope for the NASNet-A Cifar model. + + Args: + weight_decay: The weight decay to use for regularizing the model. + batch_norm_decay: Decay for batch norm moving average. + batch_norm_epsilon: Small float added to variance to avoid dividing by zero + in batch norm. + + Returns: + An `arg_scope` to use for the NASNet Cifar Model. + """ + batch_norm_params = { + # Decay for the moving averages. + 'decay': batch_norm_decay, + # epsilon to prevent 0s in variance. + 'epsilon': batch_norm_epsilon, + 'scale': True, + 'fused': True, + } + weights_regularizer = contrib_layers.l2_regularizer(weight_decay) + weights_initializer = contrib_layers.variance_scaling_initializer( + mode='FAN_OUT') + with arg_scope([slim.fully_connected, slim.conv2d, slim.separable_conv2d], + weights_regularizer=weights_regularizer, + weights_initializer=weights_initializer): + with arg_scope([slim.fully_connected], + activation_fn=None, scope='FC'): + with arg_scope([slim.conv2d, slim.separable_conv2d], + activation_fn=None, biases_initializer=None): + with arg_scope([slim.batch_norm], **batch_norm_params) as sc: + return sc + + +def nasnet_mobile_arg_scope(weight_decay=4e-5, + batch_norm_decay=0.9997, + batch_norm_epsilon=1e-3): + """Defines the default arg scope for the NASNet-A Mobile ImageNet model. + + Args: + weight_decay: The weight decay to use for regularizing the model. + batch_norm_decay: Decay for batch norm moving average. + batch_norm_epsilon: Small float added to variance to avoid dividing by zero + in batch norm. + + Returns: + An `arg_scope` to use for the NASNet Mobile Model. + """ + batch_norm_params = { + # Decay for the moving averages. + 'decay': batch_norm_decay, + # epsilon to prevent 0s in variance. + 'epsilon': batch_norm_epsilon, + 'scale': True, + 'fused': True, + } + weights_regularizer = contrib_layers.l2_regularizer(weight_decay) + weights_initializer = contrib_layers.variance_scaling_initializer( + mode='FAN_OUT') + with arg_scope([slim.fully_connected, slim.conv2d, slim.separable_conv2d], + weights_regularizer=weights_regularizer, + weights_initializer=weights_initializer): + with arg_scope([slim.fully_connected], + activation_fn=None, scope='FC'): + with arg_scope([slim.conv2d, slim.separable_conv2d], + activation_fn=None, biases_initializer=None): + with arg_scope([slim.batch_norm], **batch_norm_params) as sc: + return sc + + +def nasnet_large_arg_scope(weight_decay=5e-5, + batch_norm_decay=0.9997, + batch_norm_epsilon=1e-3): + """Defines the default arg scope for the NASNet-A Large ImageNet model. + + Args: + weight_decay: The weight decay to use for regularizing the model. + batch_norm_decay: Decay for batch norm moving average. + batch_norm_epsilon: Small float added to variance to avoid dividing by zero + in batch norm. + + Returns: + An `arg_scope` to use for the NASNet Large Model. + """ + batch_norm_params = { + # Decay for the moving averages. + 'decay': batch_norm_decay, + # epsilon to prevent 0s in variance. + 'epsilon': batch_norm_epsilon, + 'scale': True, + 'fused': True, + } + weights_regularizer = contrib_layers.l2_regularizer(weight_decay) + weights_initializer = contrib_layers.variance_scaling_initializer( + mode='FAN_OUT') + with arg_scope([slim.fully_connected, slim.conv2d, slim.separable_conv2d], + weights_regularizer=weights_regularizer, + weights_initializer=weights_initializer): + with arg_scope([slim.fully_connected], + activation_fn=None, scope='FC'): + with arg_scope([slim.conv2d, slim.separable_conv2d], + activation_fn=None, biases_initializer=None): + with arg_scope([slim.batch_norm], **batch_norm_params) as sc: + return sc + + +def _build_aux_head(net, end_points, num_classes, hparams, scope): + """Auxiliary head used for all models across all datasets.""" + activation_fn = tf.nn.relu6 if hparams.use_bounded_activation else tf.nn.relu + with tf.compat.v1.variable_scope(scope): + aux_logits = tf.identity(net) + with tf.compat.v1.variable_scope('aux_logits'): + aux_logits = slim.avg_pool2d( + aux_logits, [5, 5], stride=3, padding='VALID') + aux_logits = slim.conv2d(aux_logits, 128, [1, 1], scope='proj') + aux_logits = slim.batch_norm(aux_logits, scope='aux_bn0') + aux_logits = activation_fn(aux_logits) + # Shape of feature map before the final layer. + shape = aux_logits.shape + if hparams.data_format == 'NHWC': + shape = shape[1:3] + else: + shape = shape[2:4] + aux_logits = slim.conv2d(aux_logits, 768, shape, padding='VALID') + aux_logits = slim.batch_norm(aux_logits, scope='aux_bn1') + aux_logits = activation_fn(aux_logits) + aux_logits = contrib_layers.flatten(aux_logits) + aux_logits = slim.fully_connected(aux_logits, num_classes) + end_points['AuxLogits'] = aux_logits + + +def _imagenet_stem(inputs, hparams, stem_cell, current_step=None): + """Stem used for models trained on ImageNet.""" + num_stem_cells = 2 + + # 149 x 149 x 32 + num_stem_filters = int(32 * hparams.stem_multiplier) + net = slim.conv2d( + inputs, num_stem_filters, [3, 3], stride=2, scope='conv0', + padding='VALID') + net = slim.batch_norm(net, scope='conv0_bn') + + # Run the reduction cells + cell_outputs = [None, net] + filter_scaling = 1.0 / (hparams.filter_scaling_rate**num_stem_cells) + for cell_num in range(num_stem_cells): + net = stem_cell( + net, + scope='cell_stem_{}'.format(cell_num), + filter_scaling=filter_scaling, + stride=2, + prev_layer=cell_outputs[-2], + cell_num=cell_num, + current_step=current_step) + cell_outputs.append(net) + filter_scaling *= hparams.filter_scaling_rate + return net, cell_outputs + + +def _cifar_stem(inputs, hparams): + """Stem used for models trained on Cifar.""" + num_stem_filters = int(hparams.num_conv_filters * hparams.stem_multiplier) + net = slim.conv2d( + inputs, + num_stem_filters, + 3, + scope='l1_stem_3x3') + net = slim.batch_norm(net, scope='l1_stem_bn') + return net, [None, net] + + +def build_nasnet_cifar(images, num_classes, + is_training=True, + config=None, + current_step=None): + """Build NASNet model for the Cifar Dataset.""" + hparams = cifar_config() if config is None else copy.deepcopy(config) + _update_hparams(hparams, is_training) + + if tf.test.is_gpu_available() and hparams.data_format == 'NHWC': + tf.compat.v1.logging.info( + 'A GPU is available on the machine, consider using NCHW ' + 'data format for increased speed on GPU.') + + if hparams.data_format == 'NCHW': + images = tf.transpose(a=images, perm=[0, 3, 1, 2]) + + # Calculate the total number of cells in the network + # Add 2 for the reduction cells + total_num_cells = hparams.num_cells + 2 + + normal_cell = nasnet_utils.NasNetANormalCell( + hparams.num_conv_filters, hparams.drop_path_keep_prob, + total_num_cells, hparams.total_training_steps, + hparams.use_bounded_activation) + reduction_cell = nasnet_utils.NasNetAReductionCell( + hparams.num_conv_filters, hparams.drop_path_keep_prob, + total_num_cells, hparams.total_training_steps, + hparams.use_bounded_activation) + with arg_scope([slim.dropout, nasnet_utils.drop_path, slim.batch_norm], + is_training=is_training): + with arg_scope([slim.avg_pool2d, + slim.max_pool2d, + slim.conv2d, + slim.batch_norm, + slim.separable_conv2d, + nasnet_utils.factorized_reduction, + nasnet_utils.global_avg_pool, + nasnet_utils.get_channel_index, + nasnet_utils.get_channel_dim], + data_format=hparams.data_format): + return _build_nasnet_base(images, + normal_cell=normal_cell, + reduction_cell=reduction_cell, + num_classes=num_classes, + hparams=hparams, + is_training=is_training, + stem_type='cifar', + current_step=current_step) +build_nasnet_cifar.default_image_size = 32 + + +def build_nasnet_mobile(images, num_classes, + is_training=True, + final_endpoint=None, + config=None, + current_step=None): + """Build NASNet Mobile model for the ImageNet Dataset.""" + hparams = (mobile_imagenet_config() if config is None + else copy.deepcopy(config)) + _update_hparams(hparams, is_training) + + if tf.test.is_gpu_available() and hparams.data_format == 'NHWC': + tf.compat.v1.logging.info( + 'A GPU is available on the machine, consider using NCHW ' + 'data format for increased speed on GPU.') + + if hparams.data_format == 'NCHW': + images = tf.transpose(a=images, perm=[0, 3, 1, 2]) + + # Calculate the total number of cells in the network + # Add 2 for the reduction cells + total_num_cells = hparams.num_cells + 2 + # If ImageNet, then add an additional two for the stem cells + total_num_cells += 2 + + normal_cell = nasnet_utils.NasNetANormalCell( + hparams.num_conv_filters, hparams.drop_path_keep_prob, + total_num_cells, hparams.total_training_steps, + hparams.use_bounded_activation) + reduction_cell = nasnet_utils.NasNetAReductionCell( + hparams.num_conv_filters, hparams.drop_path_keep_prob, + total_num_cells, hparams.total_training_steps, + hparams.use_bounded_activation) + with arg_scope([slim.dropout, nasnet_utils.drop_path, slim.batch_norm], + is_training=is_training): + with arg_scope([slim.avg_pool2d, + slim.max_pool2d, + slim.conv2d, + slim.batch_norm, + slim.separable_conv2d, + nasnet_utils.factorized_reduction, + nasnet_utils.global_avg_pool, + nasnet_utils.get_channel_index, + nasnet_utils.get_channel_dim], + data_format=hparams.data_format): + return _build_nasnet_base(images, + normal_cell=normal_cell, + reduction_cell=reduction_cell, + num_classes=num_classes, + hparams=hparams, + is_training=is_training, + stem_type='imagenet', + final_endpoint=final_endpoint, + current_step=current_step) +build_nasnet_mobile.default_image_size = 224 + + +def build_nasnet_large(images, num_classes, + is_training=True, + final_endpoint=None, + config=None, + current_step=None): + """Build NASNet Large model for the ImageNet Dataset.""" + hparams = (large_imagenet_config() if config is None + else copy.deepcopy(config)) + _update_hparams(hparams, is_training) + + if tf.test.is_gpu_available() and hparams.data_format == 'NHWC': + tf.compat.v1.logging.info( + 'A GPU is available on the machine, consider using NCHW ' + 'data format for increased speed on GPU.') + + if hparams.data_format == 'NCHW': + images = tf.transpose(a=images, perm=[0, 3, 1, 2]) + + # Calculate the total number of cells in the network + # Add 2 for the reduction cells + total_num_cells = hparams.num_cells + 2 + # If ImageNet, then add an additional two for the stem cells + total_num_cells += 2 + + normal_cell = nasnet_utils.NasNetANormalCell( + hparams.num_conv_filters, hparams.drop_path_keep_prob, + total_num_cells, hparams.total_training_steps, + hparams.use_bounded_activation) + reduction_cell = nasnet_utils.NasNetAReductionCell( + hparams.num_conv_filters, hparams.drop_path_keep_prob, + total_num_cells, hparams.total_training_steps, + hparams.use_bounded_activation) + with arg_scope([slim.dropout, nasnet_utils.drop_path, slim.batch_norm], + is_training=is_training): + with arg_scope([slim.avg_pool2d, + slim.max_pool2d, + slim.conv2d, + slim.batch_norm, + slim.separable_conv2d, + nasnet_utils.factorized_reduction, + nasnet_utils.global_avg_pool, + nasnet_utils.get_channel_index, + nasnet_utils.get_channel_dim], + data_format=hparams.data_format): + return _build_nasnet_base(images, + normal_cell=normal_cell, + reduction_cell=reduction_cell, + num_classes=num_classes, + hparams=hparams, + is_training=is_training, + stem_type='imagenet', + final_endpoint=final_endpoint, + current_step=current_step) +build_nasnet_large.default_image_size = 331 + + +def _build_nasnet_base(images, + normal_cell, + reduction_cell, + num_classes, + hparams, + is_training, + stem_type, + final_endpoint=None, + current_step=None): + """Constructs a NASNet image model.""" + + end_points = {} + def add_and_check_endpoint(endpoint_name, net): + end_points[endpoint_name] = net + return final_endpoint and (endpoint_name == final_endpoint) + + # Find where to place the reduction cells or stride normal cells + reduction_indices = nasnet_utils.calc_reduction_layers( + hparams.num_cells, hparams.num_reduction_layers) + stem_cell = reduction_cell + + if stem_type == 'imagenet': + stem = lambda: _imagenet_stem(images, hparams, stem_cell) + elif stem_type == 'cifar': + stem = lambda: _cifar_stem(images, hparams) + else: + raise ValueError('Unknown stem_type: ', stem_type) + net, cell_outputs = stem() + if add_and_check_endpoint('Stem', net): return net, end_points + + # Setup for building in the auxiliary head. + aux_head_cell_idxes = [] + if len(reduction_indices) >= 2: + aux_head_cell_idxes.append(reduction_indices[1] - 1) + + # Run the cells + filter_scaling = 1.0 + # true_cell_num accounts for the stem cells + true_cell_num = 2 if stem_type == 'imagenet' else 0 + activation_fn = tf.nn.relu6 if hparams.use_bounded_activation else tf.nn.relu + for cell_num in range(hparams.num_cells): + stride = 1 + if hparams.skip_reduction_layer_input: + prev_layer = cell_outputs[-2] + if cell_num in reduction_indices: + filter_scaling *= hparams.filter_scaling_rate + net = reduction_cell( + net, + scope='reduction_cell_{}'.format(reduction_indices.index(cell_num)), + filter_scaling=filter_scaling, + stride=2, + prev_layer=cell_outputs[-2], + cell_num=true_cell_num, + current_step=current_step) + if add_and_check_endpoint( + 'Reduction_Cell_{}'.format(reduction_indices.index(cell_num)), net): + return net, end_points + true_cell_num += 1 + cell_outputs.append(net) + if not hparams.skip_reduction_layer_input: + prev_layer = cell_outputs[-2] + net = normal_cell( + net, + scope='cell_{}'.format(cell_num), + filter_scaling=filter_scaling, + stride=stride, + prev_layer=prev_layer, + cell_num=true_cell_num, + current_step=current_step) + + if add_and_check_endpoint('Cell_{}'.format(cell_num), net): + return net, end_points + true_cell_num += 1 + if (hparams.use_aux_head and cell_num in aux_head_cell_idxes and + num_classes and is_training): + aux_net = activation_fn(net) + _build_aux_head(aux_net, end_points, num_classes, hparams, + scope='aux_{}'.format(cell_num)) + cell_outputs.append(net) + + # Final softmax layer + with tf.compat.v1.variable_scope('final_layer'): + net = activation_fn(net) + net = nasnet_utils.global_avg_pool(net) + if add_and_check_endpoint('global_pool', net) or not num_classes: + return net, end_points + net = slim.dropout(net, hparams.dense_dropout_keep_prob, scope='dropout') + logits = slim.fully_connected(net, num_classes) + + if add_and_check_endpoint('Logits', logits): + return net, end_points + + predictions = tf.nn.softmax(logits, name='predictions') + if add_and_check_endpoint('Predictions', predictions): + return net, end_points + return logits, end_points diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/nasnet_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/nasnet_test.py new file mode 100644 index 0000000..deb347d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/nasnet_test.py @@ -0,0 +1,413 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for slim.nasnet.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets.nasnet import nasnet + +slim = contrib_slim + + +class NASNetTest(tf.test.TestCase): + + def testBuildLogitsCifarModel(self): + batch_size = 5 + height, width = 32, 32 + num_classes = 10 + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + with slim.arg_scope(nasnet.nasnet_cifar_arg_scope()): + logits, end_points = nasnet.build_nasnet_cifar(inputs, num_classes) + auxlogits = end_points['AuxLogits'] + predictions = end_points['Predictions'] + self.assertListEqual(auxlogits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertListEqual(predictions.get_shape().as_list(), + [batch_size, num_classes]) + + def testBuildLogitsMobileModel(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()): + logits, end_points = nasnet.build_nasnet_mobile(inputs, num_classes) + auxlogits = end_points['AuxLogits'] + predictions = end_points['Predictions'] + self.assertListEqual(auxlogits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertListEqual(predictions.get_shape().as_list(), + [batch_size, num_classes]) + + def testBuildLogitsLargeModel(self): + batch_size = 5 + height, width = 331, 331 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + with slim.arg_scope(nasnet.nasnet_large_arg_scope()): + logits, end_points = nasnet.build_nasnet_large(inputs, num_classes) + auxlogits = end_points['AuxLogits'] + predictions = end_points['Predictions'] + self.assertListEqual(auxlogits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertListEqual(predictions.get_shape().as_list(), + [batch_size, num_classes]) + + def testBuildPreLogitsCifarModel(self): + batch_size = 5 + height, width = 32, 32 + num_classes = None + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + with slim.arg_scope(nasnet.nasnet_cifar_arg_scope()): + net, end_points = nasnet.build_nasnet_cifar(inputs, num_classes) + self.assertFalse('AuxLogits' in end_points) + self.assertFalse('Predictions' in end_points) + self.assertTrue(net.op.name.startswith('final_layer/Mean')) + self.assertListEqual(net.get_shape().as_list(), [batch_size, 768]) + + def testBuildPreLogitsMobileModel(self): + batch_size = 5 + height, width = 224, 224 + num_classes = None + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()): + net, end_points = nasnet.build_nasnet_mobile(inputs, num_classes) + self.assertFalse('AuxLogits' in end_points) + self.assertFalse('Predictions' in end_points) + self.assertTrue(net.op.name.startswith('final_layer/Mean')) + self.assertListEqual(net.get_shape().as_list(), [batch_size, 1056]) + + def testBuildPreLogitsLargeModel(self): + batch_size = 5 + height, width = 331, 331 + num_classes = None + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + with slim.arg_scope(nasnet.nasnet_large_arg_scope()): + net, end_points = nasnet.build_nasnet_large(inputs, num_classes) + self.assertFalse('AuxLogits' in end_points) + self.assertFalse('Predictions' in end_points) + self.assertTrue(net.op.name.startswith('final_layer/Mean')) + self.assertListEqual(net.get_shape().as_list(), [batch_size, 4032]) + + def testAllEndPointsShapesCifarModel(self): + batch_size = 5 + height, width = 32, 32 + num_classes = 10 + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + with slim.arg_scope(nasnet.nasnet_cifar_arg_scope()): + _, end_points = nasnet.build_nasnet_cifar(inputs, num_classes) + endpoints_shapes = {'Stem': [batch_size, 32, 32, 96], + 'Cell_0': [batch_size, 32, 32, 192], + 'Cell_1': [batch_size, 32, 32, 192], + 'Cell_2': [batch_size, 32, 32, 192], + 'Cell_3': [batch_size, 32, 32, 192], + 'Cell_4': [batch_size, 32, 32, 192], + 'Cell_5': [batch_size, 32, 32, 192], + 'Cell_6': [batch_size, 16, 16, 384], + 'Cell_7': [batch_size, 16, 16, 384], + 'Cell_8': [batch_size, 16, 16, 384], + 'Cell_9': [batch_size, 16, 16, 384], + 'Cell_10': [batch_size, 16, 16, 384], + 'Cell_11': [batch_size, 16, 16, 384], + 'Cell_12': [batch_size, 8, 8, 768], + 'Cell_13': [batch_size, 8, 8, 768], + 'Cell_14': [batch_size, 8, 8, 768], + 'Cell_15': [batch_size, 8, 8, 768], + 'Cell_16': [batch_size, 8, 8, 768], + 'Cell_17': [batch_size, 8, 8, 768], + 'Reduction_Cell_0': [batch_size, 16, 16, 256], + 'Reduction_Cell_1': [batch_size, 8, 8, 512], + 'global_pool': [batch_size, 768], + # Logits and predictions + 'AuxLogits': [batch_size, num_classes], + 'Logits': [batch_size, num_classes], + 'Predictions': [batch_size, num_classes]} + self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) + for endpoint_name in endpoints_shapes: + tf.compat.v1.logging.info('Endpoint name: {}'.format(endpoint_name)) + expected_shape = endpoints_shapes[endpoint_name] + self.assertTrue(endpoint_name in end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testNoAuxHeadCifarModel(self): + batch_size = 5 + height, width = 32, 32 + num_classes = 10 + for use_aux_head in (True, False): + tf.compat.v1.reset_default_graph() + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + config = nasnet.cifar_config() + config.set_hparam('use_aux_head', int(use_aux_head)) + with slim.arg_scope(nasnet.nasnet_cifar_arg_scope()): + _, end_points = nasnet.build_nasnet_cifar(inputs, num_classes, + config=config) + self.assertEqual('AuxLogits' in end_points, use_aux_head) + + def testAllEndPointsShapesMobileModel(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()): + _, end_points = nasnet.build_nasnet_mobile(inputs, num_classes) + endpoints_shapes = {'Stem': [batch_size, 28, 28, 88], + 'Cell_0': [batch_size, 28, 28, 264], + 'Cell_1': [batch_size, 28, 28, 264], + 'Cell_2': [batch_size, 28, 28, 264], + 'Cell_3': [batch_size, 28, 28, 264], + 'Cell_4': [batch_size, 14, 14, 528], + 'Cell_5': [batch_size, 14, 14, 528], + 'Cell_6': [batch_size, 14, 14, 528], + 'Cell_7': [batch_size, 14, 14, 528], + 'Cell_8': [batch_size, 7, 7, 1056], + 'Cell_9': [batch_size, 7, 7, 1056], + 'Cell_10': [batch_size, 7, 7, 1056], + 'Cell_11': [batch_size, 7, 7, 1056], + 'Reduction_Cell_0': [batch_size, 14, 14, 352], + 'Reduction_Cell_1': [batch_size, 7, 7, 704], + 'global_pool': [batch_size, 1056], + # Logits and predictions + 'AuxLogits': [batch_size, num_classes], + 'Logits': [batch_size, num_classes], + 'Predictions': [batch_size, num_classes]} + self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) + for endpoint_name in endpoints_shapes: + tf.compat.v1.logging.info('Endpoint name: {}'.format(endpoint_name)) + expected_shape = endpoints_shapes[endpoint_name] + self.assertTrue(endpoint_name in end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testNoAuxHeadMobileModel(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + for use_aux_head in (True, False): + tf.compat.v1.reset_default_graph() + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + config = nasnet.mobile_imagenet_config() + config.set_hparam('use_aux_head', int(use_aux_head)) + with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()): + _, end_points = nasnet.build_nasnet_mobile(inputs, num_classes, + config=config) + self.assertEqual('AuxLogits' in end_points, use_aux_head) + + def testAllEndPointsShapesLargeModel(self): + batch_size = 5 + height, width = 331, 331 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + with slim.arg_scope(nasnet.nasnet_large_arg_scope()): + _, end_points = nasnet.build_nasnet_large(inputs, num_classes) + endpoints_shapes = {'Stem': [batch_size, 42, 42, 336], + 'Cell_0': [batch_size, 42, 42, 1008], + 'Cell_1': [batch_size, 42, 42, 1008], + 'Cell_2': [batch_size, 42, 42, 1008], + 'Cell_3': [batch_size, 42, 42, 1008], + 'Cell_4': [batch_size, 42, 42, 1008], + 'Cell_5': [batch_size, 42, 42, 1008], + 'Cell_6': [batch_size, 21, 21, 2016], + 'Cell_7': [batch_size, 21, 21, 2016], + 'Cell_8': [batch_size, 21, 21, 2016], + 'Cell_9': [batch_size, 21, 21, 2016], + 'Cell_10': [batch_size, 21, 21, 2016], + 'Cell_11': [batch_size, 21, 21, 2016], + 'Cell_12': [batch_size, 11, 11, 4032], + 'Cell_13': [batch_size, 11, 11, 4032], + 'Cell_14': [batch_size, 11, 11, 4032], + 'Cell_15': [batch_size, 11, 11, 4032], + 'Cell_16': [batch_size, 11, 11, 4032], + 'Cell_17': [batch_size, 11, 11, 4032], + 'Reduction_Cell_0': [batch_size, 21, 21, 1344], + 'Reduction_Cell_1': [batch_size, 11, 11, 2688], + 'global_pool': [batch_size, 4032], + # Logits and predictions + 'AuxLogits': [batch_size, num_classes], + 'Logits': [batch_size, num_classes], + 'Predictions': [batch_size, num_classes]} + self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) + for endpoint_name in endpoints_shapes: + tf.compat.v1.logging.info('Endpoint name: {}'.format(endpoint_name)) + expected_shape = endpoints_shapes[endpoint_name] + self.assertTrue(endpoint_name in end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testNoAuxHeadLargeModel(self): + batch_size = 5 + height, width = 331, 331 + num_classes = 1000 + for use_aux_head in (True, False): + tf.compat.v1.reset_default_graph() + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + config = nasnet.large_imagenet_config() + config.set_hparam('use_aux_head', int(use_aux_head)) + with slim.arg_scope(nasnet.nasnet_large_arg_scope()): + _, end_points = nasnet.build_nasnet_large(inputs, num_classes, + config=config) + self.assertEqual('AuxLogits' in end_points, use_aux_head) + + def testVariablesSetDeviceMobileModel(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + # Force all Variables to reside on the device. + with tf.compat.v1.variable_scope('on_cpu'), tf.device('/cpu:0'): + with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()): + nasnet.build_nasnet_mobile(inputs, num_classes) + with tf.compat.v1.variable_scope('on_gpu'), tf.device('/gpu:0'): + with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()): + nasnet.build_nasnet_mobile(inputs, num_classes) + for v in tf.compat.v1.get_collection( + tf.compat.v1.GraphKeys.GLOBAL_VARIABLES, scope='on_cpu'): + self.assertDeviceEqual(v.device, '/cpu:0') + for v in tf.compat.v1.get_collection( + tf.compat.v1.GraphKeys.GLOBAL_VARIABLES, scope='on_gpu'): + self.assertDeviceEqual(v.device, '/gpu:0') + + def testUnknownBatchSizeMobileModel(self): + batch_size = 1 + height, width = 224, 224 + num_classes = 1000 + with self.test_session() as sess: + inputs = tf.compat.v1.placeholder(tf.float32, (None, height, width, 3)) + with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()): + logits, _ = nasnet.build_nasnet_mobile(inputs, num_classes) + self.assertListEqual(logits.get_shape().as_list(), + [None, num_classes]) + images = tf.random.uniform((batch_size, height, width, 3)) + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(logits, {inputs: images.eval()}) + self.assertEquals(output.shape, (batch_size, num_classes)) + + def testEvaluationMobileModel(self): + batch_size = 2 + height, width = 224, 224 + num_classes = 1000 + with self.test_session() as sess: + eval_inputs = tf.random.uniform((batch_size, height, width, 3)) + with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()): + logits, _ = nasnet.build_nasnet_mobile(eval_inputs, + num_classes, + is_training=False) + predictions = tf.argmax(input=logits, axis=1) + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(predictions) + self.assertEquals(output.shape, (batch_size,)) + + def testOverrideHParamsCifarModel(self): + batch_size = 5 + height, width = 32, 32 + num_classes = 10 + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + config = nasnet.cifar_config() + config.set_hparam('data_format', 'NCHW') + with slim.arg_scope(nasnet.nasnet_cifar_arg_scope()): + _, end_points = nasnet.build_nasnet_cifar( + inputs, num_classes, config=config) + self.assertListEqual( + end_points['Stem'].shape.as_list(), [batch_size, 96, 32, 32]) + + def testOverrideHParamsMobileModel(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + config = nasnet.mobile_imagenet_config() + config.set_hparam('data_format', 'NCHW') + with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()): + _, end_points = nasnet.build_nasnet_mobile( + inputs, num_classes, config=config) + self.assertListEqual( + end_points['Stem'].shape.as_list(), [batch_size, 88, 28, 28]) + + def testOverrideHParamsLargeModel(self): + batch_size = 5 + height, width = 331, 331 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + config = nasnet.large_imagenet_config() + config.set_hparam('data_format', 'NCHW') + with slim.arg_scope(nasnet.nasnet_large_arg_scope()): + _, end_points = nasnet.build_nasnet_large( + inputs, num_classes, config=config) + self.assertListEqual( + end_points['Stem'].shape.as_list(), [batch_size, 336, 42, 42]) + + def testCurrentStepCifarModel(self): + batch_size = 5 + height, width = 32, 32 + num_classes = 10 + inputs = tf.random.uniform((batch_size, height, width, 3)) + global_step = tf.compat.v1.train.create_global_step() + with slim.arg_scope(nasnet.nasnet_cifar_arg_scope()): + logits, end_points = nasnet.build_nasnet_cifar(inputs, + num_classes, + current_step=global_step) + auxlogits = end_points['AuxLogits'] + predictions = end_points['Predictions'] + self.assertListEqual(auxlogits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertListEqual(predictions.get_shape().as_list(), + [batch_size, num_classes]) + + def testUseBoundedAcitvationCifarModel(self): + batch_size = 1 + height, width = 32, 32 + num_classes = 10 + for use_bounded_activation in (True, False): + tf.compat.v1.reset_default_graph() + inputs = tf.random.uniform((batch_size, height, width, 3)) + config = nasnet.cifar_config() + config.set_hparam('use_bounded_activation', use_bounded_activation) + with slim.arg_scope(nasnet.nasnet_cifar_arg_scope()): + _, _ = nasnet.build_nasnet_cifar( + inputs, num_classes, config=config) + for node in tf.compat.v1.get_default_graph().as_graph_def().node: + if node.op.startswith('Relu'): + self.assertEqual(node.op == 'Relu6', use_bounded_activation) + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/nasnet_utils.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/nasnet_utils.py new file mode 100644 index 0000000..1d68854 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/nasnet_utils.py @@ -0,0 +1,534 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A custom module for some common operations used by NASNet. + +Functions exposed in this file: +- calc_reduction_layers +- get_channel_index +- get_channel_dim +- global_avg_pool +- factorized_reduction +- drop_path + +Classes exposed in this file: +- NasNetABaseCell +- NasNetANormalCell +- NasNetAReductionCell +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import framework as contrib_framework +from tensorflow.contrib import slim as contrib_slim + +arg_scope = contrib_framework.arg_scope +slim = contrib_slim + +DATA_FORMAT_NCHW = 'NCHW' +DATA_FORMAT_NHWC = 'NHWC' +INVALID = 'null' +# The cap for tf.clip_by_value, it's hinted from the activation distribution +# that the majority of activation values are in the range [-6, 6]. +CLIP_BY_VALUE_CAP = 6 + + +def calc_reduction_layers(num_cells, num_reduction_layers): + """Figure out what layers should have reductions.""" + reduction_layers = [] + for pool_num in range(1, num_reduction_layers + 1): + layer_num = (float(pool_num) / (num_reduction_layers + 1)) * num_cells + layer_num = int(layer_num) + reduction_layers.append(layer_num) + return reduction_layers + + +@contrib_framework.add_arg_scope +def get_channel_index(data_format=INVALID): + assert data_format != INVALID + axis = 3 if data_format == 'NHWC' else 1 + return axis + + +@contrib_framework.add_arg_scope +def get_channel_dim(shape, data_format=INVALID): + assert data_format != INVALID + assert len(shape) == 4 + if data_format == 'NHWC': + return int(shape[3]) + elif data_format == 'NCHW': + return int(shape[1]) + else: + raise ValueError('Not a valid data_format', data_format) + + +@contrib_framework.add_arg_scope +def global_avg_pool(x, data_format=INVALID): + """Average pool away the height and width spatial dimensions of x.""" + assert data_format != INVALID + assert data_format in ['NHWC', 'NCHW'] + assert x.shape.ndims == 4 + if data_format == 'NHWC': + return tf.reduce_mean(input_tensor=x, axis=[1, 2]) + else: + return tf.reduce_mean(input_tensor=x, axis=[2, 3]) + + +@contrib_framework.add_arg_scope +def factorized_reduction(net, output_filters, stride, data_format=INVALID): + """Reduces the shape of net without information loss due to striding.""" + assert data_format != INVALID + if stride == 1: + net = slim.conv2d(net, output_filters, 1, scope='path_conv') + net = slim.batch_norm(net, scope='path_bn') + return net + if data_format == 'NHWC': + stride_spec = [1, stride, stride, 1] + else: + stride_spec = [1, 1, stride, stride] + + # Skip path 1 + path1 = tf.compat.v2.nn.avg_pool2d( + input=net, + ksize=[1, 1, 1, 1], + strides=stride_spec, + padding='VALID', + data_format=data_format) + path1 = slim.conv2d(path1, int(output_filters / 2), 1, scope='path1_conv') + + # Skip path 2 + # First pad with 0's on the right and bottom, then shift the filter to + # include those 0's that were added. + if data_format == 'NHWC': + pad_arr = [[0, 0], [0, 1], [0, 1], [0, 0]] + path2 = tf.pad(tensor=net, paddings=pad_arr)[:, 1:, 1:, :] + concat_axis = 3 + else: + pad_arr = [[0, 0], [0, 0], [0, 1], [0, 1]] + path2 = tf.pad(tensor=net, paddings=pad_arr)[:, :, 1:, 1:] + concat_axis = 1 + + path2 = tf.compat.v2.nn.avg_pool2d( + input=path2, + ksize=[1, 1, 1, 1], + strides=stride_spec, + padding='VALID', + data_format=data_format) + + # If odd number of filters, add an additional one to the second path. + final_filter_size = int(output_filters / 2) + int(output_filters % 2) + path2 = slim.conv2d(path2, final_filter_size, 1, scope='path2_conv') + + # Concat and apply BN + final_path = tf.concat(values=[path1, path2], axis=concat_axis) + final_path = slim.batch_norm(final_path, scope='final_path_bn') + return final_path + + +@contrib_framework.add_arg_scope +def drop_path(net, keep_prob, is_training=True): + """Drops out a whole example hiddenstate with the specified probability.""" + if is_training: + batch_size = tf.shape(input=net)[0] + noise_shape = [batch_size, 1, 1, 1] + random_tensor = keep_prob + random_tensor += tf.random.uniform(noise_shape, dtype=tf.float32) + binary_tensor = tf.cast(tf.floor(random_tensor), net.dtype) + keep_prob_inv = tf.cast(1.0 / keep_prob, net.dtype) + net = net * keep_prob_inv * binary_tensor + + return net + + +def _operation_to_filter_shape(operation): + splitted_operation = operation.split('x') + filter_shape = int(splitted_operation[0][-1]) + assert filter_shape == int( + splitted_operation[1][0]), 'Rectangular filters not supported.' + return filter_shape + + +def _operation_to_num_layers(operation): + splitted_operation = operation.split('_') + if 'x' in splitted_operation[-1]: + return 1 + return int(splitted_operation[-1]) + + +def _operation_to_info(operation): + """Takes in operation name and returns meta information. + + An example would be 'separable_3x3_4' -> (3, 4). + + Args: + operation: String that corresponds to convolution operation. + + Returns: + Tuple of (filter shape, num layers). + """ + num_layers = _operation_to_num_layers(operation) + filter_shape = _operation_to_filter_shape(operation) + return num_layers, filter_shape + + +def _stacked_separable_conv(net, stride, operation, filter_size, + use_bounded_activation): + """Takes in an operations and parses it to the correct sep operation.""" + num_layers, kernel_size = _operation_to_info(operation) + activation_fn = tf.nn.relu6 if use_bounded_activation else tf.nn.relu + for layer_num in range(num_layers - 1): + net = activation_fn(net) + net = slim.separable_conv2d( + net, + filter_size, + kernel_size, + depth_multiplier=1, + scope='separable_{0}x{0}_{1}'.format(kernel_size, layer_num + 1), + stride=stride) + net = slim.batch_norm( + net, scope='bn_sep_{0}x{0}_{1}'.format(kernel_size, layer_num + 1)) + stride = 1 + net = activation_fn(net) + net = slim.separable_conv2d( + net, + filter_size, + kernel_size, + depth_multiplier=1, + scope='separable_{0}x{0}_{1}'.format(kernel_size, num_layers), + stride=stride) + net = slim.batch_norm( + net, scope='bn_sep_{0}x{0}_{1}'.format(kernel_size, num_layers)) + return net + + +def _operation_to_pooling_type(operation): + """Takes in the operation string and returns the pooling type.""" + splitted_operation = operation.split('_') + return splitted_operation[0] + + +def _operation_to_pooling_shape(operation): + """Takes in the operation string and returns the pooling kernel shape.""" + splitted_operation = operation.split('_') + shape = splitted_operation[-1] + assert 'x' in shape + filter_height, filter_width = shape.split('x') + assert filter_height == filter_width + return int(filter_height) + + +def _operation_to_pooling_info(operation): + """Parses the pooling operation string to return its type and shape.""" + pooling_type = _operation_to_pooling_type(operation) + pooling_shape = _operation_to_pooling_shape(operation) + return pooling_type, pooling_shape + + +def _pooling(net, stride, operation, use_bounded_activation): + """Parses operation and performs the correct pooling operation on net.""" + padding = 'SAME' + pooling_type, pooling_shape = _operation_to_pooling_info(operation) + if use_bounded_activation: + net = tf.nn.relu6(net) + if pooling_type == 'avg': + net = slim.avg_pool2d(net, pooling_shape, stride=stride, padding=padding) + elif pooling_type == 'max': + net = slim.max_pool2d(net, pooling_shape, stride=stride, padding=padding) + else: + raise NotImplementedError('Unimplemented pooling type: ', pooling_type) + return net + + +class NasNetABaseCell(object): + """NASNet Cell class that is used as a 'layer' in image architectures. + + Args: + num_conv_filters: The number of filters for each convolution operation. + operations: List of operations that are performed in the NASNet Cell in + order. + used_hiddenstates: Binary array that signals if the hiddenstate was used + within the cell. This is used to determine what outputs of the cell + should be concatenated together. + hiddenstate_indices: Determines what hiddenstates should be combined + together with the specified operations to create the NASNet cell. + use_bounded_activation: Whether or not to use bounded activations. Bounded + activations better lend themselves to quantized inference. + """ + + def __init__(self, num_conv_filters, operations, used_hiddenstates, + hiddenstate_indices, drop_path_keep_prob, total_num_cells, + total_training_steps, use_bounded_activation=False): + self._num_conv_filters = num_conv_filters + self._operations = operations + self._used_hiddenstates = used_hiddenstates + self._hiddenstate_indices = hiddenstate_indices + self._drop_path_keep_prob = drop_path_keep_prob + self._total_num_cells = total_num_cells + self._total_training_steps = total_training_steps + self._use_bounded_activation = use_bounded_activation + + def _reduce_prev_layer(self, prev_layer, curr_layer): + """Matches dimension of prev_layer to the curr_layer.""" + # Set the prev layer to the current layer if it is none + if prev_layer is None: + return curr_layer + curr_num_filters = self._filter_size + prev_num_filters = get_channel_dim(prev_layer.shape) + curr_filter_shape = int(curr_layer.shape[2]) + prev_filter_shape = int(prev_layer.shape[2]) + activation_fn = tf.nn.relu6 if self._use_bounded_activation else tf.nn.relu + if curr_filter_shape != prev_filter_shape: + prev_layer = activation_fn(prev_layer) + prev_layer = factorized_reduction( + prev_layer, curr_num_filters, stride=2) + elif curr_num_filters != prev_num_filters: + prev_layer = activation_fn(prev_layer) + prev_layer = slim.conv2d( + prev_layer, curr_num_filters, 1, scope='prev_1x1') + prev_layer = slim.batch_norm(prev_layer, scope='prev_bn') + return prev_layer + + def _cell_base(self, net, prev_layer): + """Runs the beginning of the conv cell before the predicted ops are run.""" + num_filters = self._filter_size + + # Check to be sure prev layer stuff is setup correctly + prev_layer = self._reduce_prev_layer(prev_layer, net) + + net = tf.nn.relu6(net) if self._use_bounded_activation else tf.nn.relu(net) + net = slim.conv2d(net, num_filters, 1, scope='1x1') + net = slim.batch_norm(net, scope='beginning_bn') + # num_or_size_splits=1 + net = [net] + net.append(prev_layer) + return net + + def __call__(self, net, scope=None, filter_scaling=1, stride=1, + prev_layer=None, cell_num=-1, current_step=None): + """Runs the conv cell.""" + self._cell_num = cell_num + self._filter_scaling = filter_scaling + self._filter_size = int(self._num_conv_filters * filter_scaling) + + i = 0 + with tf.compat.v1.variable_scope(scope): + net = self._cell_base(net, prev_layer) + for iteration in range(5): + with tf.compat.v1.variable_scope('comb_iter_{}'.format(iteration)): + left_hiddenstate_idx, right_hiddenstate_idx = ( + self._hiddenstate_indices[i], + self._hiddenstate_indices[i + 1]) + original_input_left = left_hiddenstate_idx < 2 + original_input_right = right_hiddenstate_idx < 2 + h1 = net[left_hiddenstate_idx] + h2 = net[right_hiddenstate_idx] + + operation_left = self._operations[i] + operation_right = self._operations[i+1] + i += 2 + # Apply conv operations + with tf.compat.v1.variable_scope('left'): + h1 = self._apply_conv_operation(h1, operation_left, + stride, original_input_left, + current_step) + with tf.compat.v1.variable_scope('right'): + h2 = self._apply_conv_operation(h2, operation_right, + stride, original_input_right, + current_step) + + # Combine hidden states using 'add'. + with tf.compat.v1.variable_scope('combine'): + h = h1 + h2 + if self._use_bounded_activation: + h = tf.nn.relu6(h) + + # Add hiddenstate to the list of hiddenstates we can choose from + net.append(h) + + with tf.compat.v1.variable_scope('cell_output'): + net = self._combine_unused_states(net) + + return net + + def _apply_conv_operation(self, net, operation, + stride, is_from_original_input, current_step): + """Applies the predicted conv operation to net.""" + # Dont stride if this is not one of the original hiddenstates + if stride > 1 and not is_from_original_input: + stride = 1 + input_filters = get_channel_dim(net.shape) + filter_size = self._filter_size + if 'separable' in operation: + net = _stacked_separable_conv(net, stride, operation, filter_size, + self._use_bounded_activation) + if self._use_bounded_activation: + net = tf.clip_by_value(net, -CLIP_BY_VALUE_CAP, CLIP_BY_VALUE_CAP) + elif operation in ['none']: + if self._use_bounded_activation: + net = tf.nn.relu6(net) + # Check if a stride is needed, then use a strided 1x1 here + if stride > 1 or (input_filters != filter_size): + if not self._use_bounded_activation: + net = tf.nn.relu(net) + net = slim.conv2d(net, filter_size, 1, stride=stride, scope='1x1') + net = slim.batch_norm(net, scope='bn_1') + if self._use_bounded_activation: + net = tf.clip_by_value(net, -CLIP_BY_VALUE_CAP, CLIP_BY_VALUE_CAP) + elif 'pool' in operation: + net = _pooling(net, stride, operation, self._use_bounded_activation) + if input_filters != filter_size: + net = slim.conv2d(net, filter_size, 1, stride=1, scope='1x1') + net = slim.batch_norm(net, scope='bn_1') + if self._use_bounded_activation: + net = tf.clip_by_value(net, -CLIP_BY_VALUE_CAP, CLIP_BY_VALUE_CAP) + else: + raise ValueError('Unimplemented operation', operation) + + if operation != 'none': + net = self._apply_drop_path(net, current_step=current_step) + return net + + def _combine_unused_states(self, net): + """Concatenate the unused hidden states of the cell.""" + used_hiddenstates = self._used_hiddenstates + + final_height = int(net[-1].shape[2]) + final_num_filters = get_channel_dim(net[-1].shape) + assert len(used_hiddenstates) == len(net) + for idx, used_h in enumerate(used_hiddenstates): + curr_height = int(net[idx].shape[2]) + curr_num_filters = get_channel_dim(net[idx].shape) + + # Determine if a reduction should be applied to make the number of + # filters match. + should_reduce = final_num_filters != curr_num_filters + should_reduce = (final_height != curr_height) or should_reduce + should_reduce = should_reduce and not used_h + if should_reduce: + stride = 2 if final_height != curr_height else 1 + with tf.compat.v1.variable_scope('reduction_{}'.format(idx)): + net[idx] = factorized_reduction( + net[idx], final_num_filters, stride) + + states_to_combine = ( + [h for h, is_used in zip(net, used_hiddenstates) if not is_used]) + + # Return the concat of all the states + concat_axis = get_channel_index() + net = tf.concat(values=states_to_combine, axis=concat_axis) + return net + + @contrib_framework.add_arg_scope # No public API. For internal use only. + def _apply_drop_path(self, net, current_step=None, + use_summaries=False, drop_connect_version='v3'): + """Apply drop_path regularization. + + Args: + net: the Tensor that gets drop_path regularization applied. + current_step: a float32 Tensor with the current global_step value, + to be divided by hparams.total_training_steps. Usually None, which + defaults to tf.train.get_or_create_global_step() properly casted. + use_summaries: a Python boolean. If set to False, no summaries are output. + drop_connect_version: one of 'v1', 'v2', 'v3', controlling whether + the dropout rate is scaled by current_step (v1), layer (v2), or + both (v3, the default). + + Returns: + The dropped-out value of `net`. + """ + drop_path_keep_prob = self._drop_path_keep_prob + if drop_path_keep_prob < 1.0: + assert drop_connect_version in ['v1', 'v2', 'v3'] + if drop_connect_version in ['v2', 'v3']: + # Scale keep prob by layer number + assert self._cell_num != -1 + # The added 2 is for the reduction cells + num_cells = self._total_num_cells + layer_ratio = (self._cell_num + 1)/float(num_cells) + if use_summaries: + with tf.device('/cpu:0'): + tf.compat.v1.summary.scalar('layer_ratio', layer_ratio) + drop_path_keep_prob = 1 - layer_ratio * (1 - drop_path_keep_prob) + if drop_connect_version in ['v1', 'v3']: + # Decrease the keep probability over time + if current_step is None: + current_step = tf.compat.v1.train.get_or_create_global_step() + current_step = tf.cast(current_step, tf.float32) + drop_path_burn_in_steps = self._total_training_steps + current_ratio = current_step / drop_path_burn_in_steps + current_ratio = tf.minimum(1.0, current_ratio) + if use_summaries: + with tf.device('/cpu:0'): + tf.compat.v1.summary.scalar('current_ratio', current_ratio) + drop_path_keep_prob = (1 - current_ratio * (1 - drop_path_keep_prob)) + if use_summaries: + with tf.device('/cpu:0'): + tf.compat.v1.summary.scalar('drop_path_keep_prob', + drop_path_keep_prob) + net = drop_path(net, drop_path_keep_prob) + return net + + +class NasNetANormalCell(NasNetABaseCell): + """NASNetA Normal Cell.""" + + def __init__(self, num_conv_filters, drop_path_keep_prob, total_num_cells, + total_training_steps, use_bounded_activation=False): + operations = ['separable_5x5_2', + 'separable_3x3_2', + 'separable_5x5_2', + 'separable_3x3_2', + 'avg_pool_3x3', + 'none', + 'avg_pool_3x3', + 'avg_pool_3x3', + 'separable_3x3_2', + 'none'] + used_hiddenstates = [1, 0, 0, 0, 0, 0, 0] + hiddenstate_indices = [0, 1, 1, 1, 0, 1, 1, 1, 0, 0] + super(NasNetANormalCell, self).__init__(num_conv_filters, operations, + used_hiddenstates, + hiddenstate_indices, + drop_path_keep_prob, + total_num_cells, + total_training_steps, + use_bounded_activation) + + +class NasNetAReductionCell(NasNetABaseCell): + """NASNetA Reduction Cell.""" + + def __init__(self, num_conv_filters, drop_path_keep_prob, total_num_cells, + total_training_steps, use_bounded_activation=False): + operations = ['separable_5x5_2', + 'separable_7x7_2', + 'max_pool_3x3', + 'separable_7x7_2', + 'avg_pool_3x3', + 'separable_5x5_2', + 'none', + 'avg_pool_3x3', + 'separable_3x3_2', + 'max_pool_3x3'] + used_hiddenstates = [1, 1, 1, 0, 0, 0, 0] + hiddenstate_indices = [0, 1, 0, 1, 0, 1, 3, 2, 2, 0] + super(NasNetAReductionCell, self).__init__(num_conv_filters, operations, + used_hiddenstates, + hiddenstate_indices, + drop_path_keep_prob, + total_num_cells, + total_training_steps, + use_bounded_activation) diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/nasnet_utils_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/nasnet_utils_test.py new file mode 100644 index 0000000..d165418 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/nasnet_utils_test.py @@ -0,0 +1,62 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for slim.nets.nasnet.nasnet_utils.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from nets.nasnet import nasnet_utils + + +class NasnetUtilsTest(tf.test.TestCase): + + def testCalcReductionLayers(self): + num_cells = 18 + num_reduction_layers = 2 + reduction_layers = nasnet_utils.calc_reduction_layers( + num_cells, num_reduction_layers) + self.assertEqual(len(reduction_layers), 2) + self.assertEqual(reduction_layers[0], 6) + self.assertEqual(reduction_layers[1], 12) + + def testGetChannelIndex(self): + data_formats = ['NHWC', 'NCHW'] + for data_format in data_formats: + index = nasnet_utils.get_channel_index(data_format) + correct_index = 3 if data_format == 'NHWC' else 1 + self.assertEqual(index, correct_index) + + def testGetChannelDim(self): + data_formats = ['NHWC', 'NCHW'] + shape = [10, 20, 30, 40] + for data_format in data_formats: + dim = nasnet_utils.get_channel_dim(shape, data_format) + correct_dim = shape[3] if data_format == 'NHWC' else shape[1] + self.assertEqual(dim, correct_dim) + + def testGlobalAvgPool(self): + data_formats = ['NHWC', 'NCHW'] + inputs = tf.compat.v1.placeholder(tf.float32, (5, 10, 20, 10)) + for data_format in data_formats: + output = nasnet_utils.global_avg_pool( + inputs, data_format) + self.assertEqual(output.shape, [5, 10]) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/pnasnet.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/pnasnet.py new file mode 100644 index 0000000..5e612e3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/pnasnet.py @@ -0,0 +1,285 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains the definition for the PNASNet classification networks. + +Paper: https://arxiv.org/abs/1712.00559 +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import copy +import tensorflow as tf +from tensorflow.contrib import framework as contrib_framework +from tensorflow.contrib import slim as contrib_slim +from tensorflow.contrib import training as contrib_training + +from nets.nasnet import nasnet +from nets.nasnet import nasnet_utils + +arg_scope = contrib_framework.arg_scope +slim = contrib_slim + + +def large_imagenet_config(): + """Large ImageNet configuration based on PNASNet-5.""" + return contrib_training.HParams( + stem_multiplier=3.0, + dense_dropout_keep_prob=0.5, + num_cells=12, + filter_scaling_rate=2.0, + num_conv_filters=216, + drop_path_keep_prob=0.6, + use_aux_head=1, + num_reduction_layers=2, + data_format='NHWC', + skip_reduction_layer_input=1, + total_training_steps=250000, + use_bounded_activation=False, + ) + + +def mobile_imagenet_config(): + """Mobile ImageNet configuration based on PNASNet-5.""" + return contrib_training.HParams( + stem_multiplier=1.0, + dense_dropout_keep_prob=0.5, + num_cells=9, + filter_scaling_rate=2.0, + num_conv_filters=54, + drop_path_keep_prob=1.0, + use_aux_head=1, + num_reduction_layers=2, + data_format='NHWC', + skip_reduction_layer_input=1, + total_training_steps=250000, + use_bounded_activation=False, + ) + + +def pnasnet_large_arg_scope(weight_decay=4e-5, batch_norm_decay=0.9997, + batch_norm_epsilon=0.001): + """Default arg scope for the PNASNet Large ImageNet model.""" + return nasnet.nasnet_large_arg_scope( + weight_decay, batch_norm_decay, batch_norm_epsilon) + + +def pnasnet_mobile_arg_scope(weight_decay=4e-5, + batch_norm_decay=0.9997, + batch_norm_epsilon=0.001): + """Default arg scope for the PNASNet Mobile ImageNet model.""" + return nasnet.nasnet_mobile_arg_scope(weight_decay, batch_norm_decay, + batch_norm_epsilon) + + +def _build_pnasnet_base(images, + normal_cell, + num_classes, + hparams, + is_training, + final_endpoint=None): + """Constructs a PNASNet image model.""" + + end_points = {} + + def add_and_check_endpoint(endpoint_name, net): + end_points[endpoint_name] = net + return final_endpoint and (endpoint_name == final_endpoint) + + # Find where to place the reduction cells or stride normal cells + reduction_indices = nasnet_utils.calc_reduction_layers( + hparams.num_cells, hparams.num_reduction_layers) + + # pylint: disable=protected-access + stem = lambda: nasnet._imagenet_stem(images, hparams, normal_cell) + # pylint: enable=protected-access + net, cell_outputs = stem() + if add_and_check_endpoint('Stem', net): + return net, end_points + + # Setup for building in the auxiliary head. + aux_head_cell_idxes = [] + if len(reduction_indices) >= 2: + aux_head_cell_idxes.append(reduction_indices[1] - 1) + + # Run the cells + filter_scaling = 1.0 + # true_cell_num accounts for the stem cells + true_cell_num = 2 + activation_fn = tf.nn.relu6 if hparams.use_bounded_activation else tf.nn.relu + for cell_num in range(hparams.num_cells): + is_reduction = cell_num in reduction_indices + stride = 2 if is_reduction else 1 + if is_reduction: filter_scaling *= hparams.filter_scaling_rate + if hparams.skip_reduction_layer_input or not is_reduction: + prev_layer = cell_outputs[-2] + net = normal_cell( + net, + scope='cell_{}'.format(cell_num), + filter_scaling=filter_scaling, + stride=stride, + prev_layer=prev_layer, + cell_num=true_cell_num) + if add_and_check_endpoint('Cell_{}'.format(cell_num), net): + return net, end_points + true_cell_num += 1 + cell_outputs.append(net) + + if (hparams.use_aux_head and cell_num in aux_head_cell_idxes and + num_classes and is_training): + aux_net = activation_fn(net) + # pylint: disable=protected-access + nasnet._build_aux_head(aux_net, end_points, num_classes, hparams, + scope='aux_{}'.format(cell_num)) + # pylint: enable=protected-access + + # Final softmax layer + with tf.compat.v1.variable_scope('final_layer'): + net = activation_fn(net) + net = nasnet_utils.global_avg_pool(net) + if add_and_check_endpoint('global_pool', net) or not num_classes: + return net, end_points + net = slim.dropout(net, hparams.dense_dropout_keep_prob, scope='dropout') + logits = slim.fully_connected(net, num_classes) + + if add_and_check_endpoint('Logits', logits): + return net, end_points + + predictions = tf.nn.softmax(logits, name='predictions') + if add_and_check_endpoint('Predictions', predictions): + return net, end_points + return logits, end_points + + +def build_pnasnet_large(images, + num_classes, + is_training=True, + final_endpoint=None, + config=None): + """Build PNASNet Large model for the ImageNet Dataset.""" + hparams = copy.deepcopy(config) if config else large_imagenet_config() + # pylint: disable=protected-access + nasnet._update_hparams(hparams, is_training) + # pylint: enable=protected-access + + if tf.test.is_gpu_available() and hparams.data_format == 'NHWC': + tf.compat.v1.logging.info( + 'A GPU is available on the machine, consider using NCHW ' + 'data format for increased speed on GPU.') + + if hparams.data_format == 'NCHW': + images = tf.transpose(a=images, perm=[0, 3, 1, 2]) + + # Calculate the total number of cells in the network. + # There is no distinction between reduction and normal cells in PNAS so the + # total number of cells is equal to the number normal cells plus the number + # of stem cells (two by default). + total_num_cells = hparams.num_cells + 2 + + normal_cell = PNasNetNormalCell(hparams.num_conv_filters, + hparams.drop_path_keep_prob, total_num_cells, + hparams.total_training_steps, + hparams.use_bounded_activation) + with arg_scope( + [slim.dropout, nasnet_utils.drop_path, slim.batch_norm], + is_training=is_training): + with arg_scope([slim.avg_pool2d, slim.max_pool2d, slim.conv2d, + slim.batch_norm, slim.separable_conv2d, + nasnet_utils.factorized_reduction, + nasnet_utils.global_avg_pool, + nasnet_utils.get_channel_index, + nasnet_utils.get_channel_dim], + data_format=hparams.data_format): + return _build_pnasnet_base( + images, + normal_cell=normal_cell, + num_classes=num_classes, + hparams=hparams, + is_training=is_training, + final_endpoint=final_endpoint) +build_pnasnet_large.default_image_size = 331 + + +def build_pnasnet_mobile(images, + num_classes, + is_training=True, + final_endpoint=None, + config=None): + """Build PNASNet Mobile model for the ImageNet Dataset.""" + hparams = copy.deepcopy(config) if config else mobile_imagenet_config() + # pylint: disable=protected-access + nasnet._update_hparams(hparams, is_training) + # pylint: enable=protected-access + + if tf.test.is_gpu_available() and hparams.data_format == 'NHWC': + tf.compat.v1.logging.info( + 'A GPU is available on the machine, consider using NCHW ' + 'data format for increased speed on GPU.') + + if hparams.data_format == 'NCHW': + images = tf.transpose(a=images, perm=[0, 3, 1, 2]) + + # Calculate the total number of cells in the network. + # There is no distinction between reduction and normal cells in PNAS so the + # total number of cells is equal to the number normal cells plus the number + # of stem cells (two by default). + total_num_cells = hparams.num_cells + 2 + + normal_cell = PNasNetNormalCell(hparams.num_conv_filters, + hparams.drop_path_keep_prob, total_num_cells, + hparams.total_training_steps, + hparams.use_bounded_activation) + with arg_scope( + [slim.dropout, nasnet_utils.drop_path, slim.batch_norm], + is_training=is_training): + with arg_scope( + [ + slim.avg_pool2d, slim.max_pool2d, slim.conv2d, slim.batch_norm, + slim.separable_conv2d, nasnet_utils.factorized_reduction, + nasnet_utils.global_avg_pool, nasnet_utils.get_channel_index, + nasnet_utils.get_channel_dim + ], + data_format=hparams.data_format): + return _build_pnasnet_base( + images, + normal_cell=normal_cell, + num_classes=num_classes, + hparams=hparams, + is_training=is_training, + final_endpoint=final_endpoint) + + +build_pnasnet_mobile.default_image_size = 224 + + +class PNasNetNormalCell(nasnet_utils.NasNetABaseCell): + """PNASNet Normal Cell.""" + + def __init__(self, num_conv_filters, drop_path_keep_prob, total_num_cells, + total_training_steps, use_bounded_activation=False): + # Configuration for the PNASNet-5 model. + operations = [ + 'separable_5x5_2', 'max_pool_3x3', 'separable_7x7_2', 'max_pool_3x3', + 'separable_5x5_2', 'separable_3x3_2', 'separable_3x3_2', 'max_pool_3x3', + 'separable_3x3_2', 'none' + ] + used_hiddenstates = [1, 1, 0, 0, 0, 0, 0] + hiddenstate_indices = [1, 1, 0, 0, 0, 0, 4, 0, 1, 0] + + super(PNasNetNormalCell, self).__init__( + num_conv_filters, operations, used_hiddenstates, hiddenstate_indices, + drop_path_keep_prob, total_num_cells, total_training_steps, + use_bounded_activation) diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/pnasnet_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/pnasnet_test.py new file mode 100644 index 0000000..8e1df4d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nasnet/pnasnet_test.py @@ -0,0 +1,257 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for slim.pnasnet.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets.nasnet import pnasnet + +slim = contrib_slim + + +class PNASNetTest(tf.test.TestCase): + + def testBuildLogitsLargeModel(self): + batch_size = 5 + height, width = 331, 331 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + with slim.arg_scope(pnasnet.pnasnet_large_arg_scope()): + logits, end_points = pnasnet.build_pnasnet_large(inputs, num_classes) + auxlogits = end_points['AuxLogits'] + predictions = end_points['Predictions'] + self.assertListEqual(auxlogits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertListEqual(predictions.get_shape().as_list(), + [batch_size, num_classes]) + + def testBuildLogitsMobileModel(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + with slim.arg_scope(pnasnet.pnasnet_mobile_arg_scope()): + logits, end_points = pnasnet.build_pnasnet_mobile(inputs, num_classes) + auxlogits = end_points['AuxLogits'] + predictions = end_points['Predictions'] + self.assertListEqual(auxlogits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertListEqual(predictions.get_shape().as_list(), + [batch_size, num_classes]) + + def testBuildNonExistingLayerLargeModel(self): + """Tests that the model is built correctly without unnecessary layers.""" + inputs = tf.random.uniform((5, 331, 331, 3)) + tf.compat.v1.train.create_global_step() + with slim.arg_scope(pnasnet.pnasnet_large_arg_scope()): + pnasnet.build_pnasnet_large(inputs, 1000) + vars_names = [x.op.name for x in tf.compat.v1.trainable_variables()] + self.assertIn('cell_stem_0/1x1/weights', vars_names) + self.assertNotIn('cell_stem_1/comb_iter_0/right/1x1/weights', vars_names) + + def testBuildNonExistingLayerMobileModel(self): + """Tests that the model is built correctly without unnecessary layers.""" + inputs = tf.random.uniform((5, 224, 224, 3)) + tf.compat.v1.train.create_global_step() + with slim.arg_scope(pnasnet.pnasnet_mobile_arg_scope()): + pnasnet.build_pnasnet_mobile(inputs, 1000) + vars_names = [x.op.name for x in tf.compat.v1.trainable_variables()] + self.assertIn('cell_stem_0/1x1/weights', vars_names) + self.assertNotIn('cell_stem_1/comb_iter_0/right/1x1/weights', vars_names) + + def testBuildPreLogitsLargeModel(self): + batch_size = 5 + height, width = 331, 331 + num_classes = None + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + with slim.arg_scope(pnasnet.pnasnet_large_arg_scope()): + net, end_points = pnasnet.build_pnasnet_large(inputs, num_classes) + self.assertFalse('AuxLogits' in end_points) + self.assertFalse('Predictions' in end_points) + self.assertTrue(net.op.name.startswith('final_layer/Mean')) + self.assertListEqual(net.get_shape().as_list(), [batch_size, 4320]) + + def testBuildPreLogitsMobileModel(self): + batch_size = 5 + height, width = 224, 224 + num_classes = None + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + with slim.arg_scope(pnasnet.pnasnet_mobile_arg_scope()): + net, end_points = pnasnet.build_pnasnet_mobile(inputs, num_classes) + self.assertFalse('AuxLogits' in end_points) + self.assertFalse('Predictions' in end_points) + self.assertTrue(net.op.name.startswith('final_layer/Mean')) + self.assertListEqual(net.get_shape().as_list(), [batch_size, 1080]) + + def testAllEndPointsShapesLargeModel(self): + batch_size = 5 + height, width = 331, 331 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + with slim.arg_scope(pnasnet.pnasnet_large_arg_scope()): + _, end_points = pnasnet.build_pnasnet_large(inputs, num_classes) + + endpoints_shapes = {'Stem': [batch_size, 42, 42, 540], + 'Cell_0': [batch_size, 42, 42, 1080], + 'Cell_1': [batch_size, 42, 42, 1080], + 'Cell_2': [batch_size, 42, 42, 1080], + 'Cell_3': [batch_size, 42, 42, 1080], + 'Cell_4': [batch_size, 21, 21, 2160], + 'Cell_5': [batch_size, 21, 21, 2160], + 'Cell_6': [batch_size, 21, 21, 2160], + 'Cell_7': [batch_size, 21, 21, 2160], + 'Cell_8': [batch_size, 11, 11, 4320], + 'Cell_9': [batch_size, 11, 11, 4320], + 'Cell_10': [batch_size, 11, 11, 4320], + 'Cell_11': [batch_size, 11, 11, 4320], + 'global_pool': [batch_size, 4320], + # Logits and predictions + 'AuxLogits': [batch_size, 1000], + 'Predictions': [batch_size, 1000], + 'Logits': [batch_size, 1000], + } + self.assertEqual(len(end_points), 17) + self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) + for endpoint_name in endpoints_shapes: + tf.compat.v1.logging.info('Endpoint name: {}'.format(endpoint_name)) + expected_shape = endpoints_shapes[endpoint_name] + self.assertIn(endpoint_name, end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testAllEndPointsShapesMobileModel(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + with slim.arg_scope(pnasnet.pnasnet_mobile_arg_scope()): + _, end_points = pnasnet.build_pnasnet_mobile(inputs, num_classes) + + endpoints_shapes = { + 'Stem': [batch_size, 28, 28, 135], + 'Cell_0': [batch_size, 28, 28, 270], + 'Cell_1': [batch_size, 28, 28, 270], + 'Cell_2': [batch_size, 28, 28, 270], + 'Cell_3': [batch_size, 14, 14, 540], + 'Cell_4': [batch_size, 14, 14, 540], + 'Cell_5': [batch_size, 14, 14, 540], + 'Cell_6': [batch_size, 7, 7, 1080], + 'Cell_7': [batch_size, 7, 7, 1080], + 'Cell_8': [batch_size, 7, 7, 1080], + 'global_pool': [batch_size, 1080], + # Logits and predictions + 'AuxLogits': [batch_size, num_classes], + 'Predictions': [batch_size, num_classes], + 'Logits': [batch_size, num_classes], + } + self.assertEqual(len(end_points), 14) + self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) + for endpoint_name in endpoints_shapes: + tf.compat.v1.logging.info('Endpoint name: {}'.format(endpoint_name)) + expected_shape = endpoints_shapes[endpoint_name] + self.assertIn(endpoint_name, end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testNoAuxHeadLargeModel(self): + batch_size = 5 + height, width = 331, 331 + num_classes = 1000 + for use_aux_head in (True, False): + tf.compat.v1.reset_default_graph() + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + config = pnasnet.large_imagenet_config() + config.set_hparam('use_aux_head', int(use_aux_head)) + with slim.arg_scope(pnasnet.pnasnet_large_arg_scope()): + _, end_points = pnasnet.build_pnasnet_large(inputs, num_classes, + config=config) + self.assertEqual('AuxLogits' in end_points, use_aux_head) + + def testNoAuxHeadMobileModel(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + for use_aux_head in (True, False): + tf.compat.v1.reset_default_graph() + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + config = pnasnet.mobile_imagenet_config() + config.set_hparam('use_aux_head', int(use_aux_head)) + with slim.arg_scope(pnasnet.pnasnet_mobile_arg_scope()): + _, end_points = pnasnet.build_pnasnet_mobile( + inputs, num_classes, config=config) + self.assertEqual('AuxLogits' in end_points, use_aux_head) + + def testOverrideHParamsLargeModel(self): + batch_size = 5 + height, width = 331, 331 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + config = pnasnet.large_imagenet_config() + config.set_hparam('data_format', 'NCHW') + with slim.arg_scope(pnasnet.pnasnet_large_arg_scope()): + _, end_points = pnasnet.build_pnasnet_large( + inputs, num_classes, config=config) + self.assertListEqual( + end_points['Stem'].shape.as_list(), [batch_size, 540, 42, 42]) + + def testOverrideHParamsMobileModel(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + inputs = tf.random.uniform((batch_size, height, width, 3)) + tf.compat.v1.train.create_global_step() + config = pnasnet.mobile_imagenet_config() + config.set_hparam('data_format', 'NCHW') + with slim.arg_scope(pnasnet.pnasnet_mobile_arg_scope()): + _, end_points = pnasnet.build_pnasnet_mobile( + inputs, num_classes, config=config) + self.assertListEqual(end_points['Stem'].shape.as_list(), + [batch_size, 135, 28, 28]) + + def testUseBoundedAcitvationMobileModel(self): + batch_size = 1 + height, width = 224, 224 + num_classes = 1000 + for use_bounded_activation in (True, False): + tf.compat.v1.reset_default_graph() + inputs = tf.random.uniform((batch_size, height, width, 3)) + config = pnasnet.mobile_imagenet_config() + config.set_hparam('use_bounded_activation', use_bounded_activation) + with slim.arg_scope(pnasnet.pnasnet_mobile_arg_scope()): + _, _ = pnasnet.build_pnasnet_mobile( + inputs, num_classes, config=config) + for node in tf.compat.v1.get_default_graph().as_graph_def().node: + if node.op.startswith('Relu'): + self.assertEqual(node.op == 'Relu6', use_bounded_activation) + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nets_factory.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nets_factory.py new file mode 100644 index 0000000..1c34f80 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nets_factory.py @@ -0,0 +1,172 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains a factory for building various models.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import functools +from tensorflow.contrib import slim as contrib_slim + +from nets import alexnet +from nets import cifarnet +from nets import i3d +from nets import inception +from nets import lenet +from nets import mobilenet_v1 +from nets import overfeat +from nets import resnet_v1 +from nets import resnet_v2 +from nets import s3dg +from nets import vgg +from nets.mobilenet import mobilenet_v2 +from nets.mobilenet import mobilenet_v3 +from nets.nasnet import nasnet +from nets.nasnet import pnasnet + + +slim = contrib_slim + +networks_map = { + 'alexnet_v2': alexnet.alexnet_v2, + 'cifarnet': cifarnet.cifarnet, + 'overfeat': overfeat.overfeat, + 'vgg_a': vgg.vgg_a, + 'vgg_16': vgg.vgg_16, + 'vgg_19': vgg.vgg_19, + 'inception_v1': inception.inception_v1, + 'inception_v2': inception.inception_v2, + 'inception_v3': inception.inception_v3, + 'inception_v4': inception.inception_v4, + 'inception_resnet_v2': inception.inception_resnet_v2, + 'i3d': i3d.i3d, + 's3dg': s3dg.s3dg, + 'lenet': lenet.lenet, + 'resnet_v1_50': resnet_v1.resnet_v1_50, + 'resnet_v1_101': resnet_v1.resnet_v1_101, + 'resnet_v1_152': resnet_v1.resnet_v1_152, + 'resnet_v1_200': resnet_v1.resnet_v1_200, + 'resnet_v2_50': resnet_v2.resnet_v2_50, + 'resnet_v2_101': resnet_v2.resnet_v2_101, + 'resnet_v2_152': resnet_v2.resnet_v2_152, + 'resnet_v2_200': resnet_v2.resnet_v2_200, + 'mobilenet_v1': mobilenet_v1.mobilenet_v1, + 'mobilenet_v1_075': mobilenet_v1.mobilenet_v1_075, + 'mobilenet_v1_050': mobilenet_v1.mobilenet_v1_050, + 'mobilenet_v1_025': mobilenet_v1.mobilenet_v1_025, + 'mobilenet_v2': mobilenet_v2.mobilenet, + 'mobilenet_v2_140': mobilenet_v2.mobilenet_v2_140, + 'mobilenet_v2_035': mobilenet_v2.mobilenet_v2_035, + 'mobilenet_v3_small': mobilenet_v3.small, + 'mobilenet_v3_large': mobilenet_v3.large, + 'mobilenet_v3_small_minimalistic': mobilenet_v3.small_minimalistic, + 'mobilenet_v3_large_minimalistic': mobilenet_v3.large_minimalistic, + 'mobilenet_edgetpu': mobilenet_v3.edge_tpu, + 'mobilenet_edgetpu_075': mobilenet_v3.edge_tpu_075, + 'nasnet_cifar': nasnet.build_nasnet_cifar, + 'nasnet_mobile': nasnet.build_nasnet_mobile, + 'nasnet_large': nasnet.build_nasnet_large, + 'pnasnet_large': pnasnet.build_pnasnet_large, + 'pnasnet_mobile': pnasnet.build_pnasnet_mobile, +} + +arg_scopes_map = { + 'alexnet_v2': alexnet.alexnet_v2_arg_scope, + 'cifarnet': cifarnet.cifarnet_arg_scope, + 'overfeat': overfeat.overfeat_arg_scope, + 'vgg_a': vgg.vgg_arg_scope, + 'vgg_16': vgg.vgg_arg_scope, + 'vgg_19': vgg.vgg_arg_scope, + 'inception_v1': inception.inception_v3_arg_scope, + 'inception_v2': inception.inception_v3_arg_scope, + 'inception_v3': inception.inception_v3_arg_scope, + 'inception_v4': inception.inception_v4_arg_scope, + 'inception_resnet_v2': inception.inception_resnet_v2_arg_scope, + 'i3d': i3d.i3d_arg_scope, + 's3dg': s3dg.s3dg_arg_scope, + 'lenet': lenet.lenet_arg_scope, + 'resnet_v1_50': resnet_v1.resnet_arg_scope, + 'resnet_v1_101': resnet_v1.resnet_arg_scope, + 'resnet_v1_152': resnet_v1.resnet_arg_scope, + 'resnet_v1_200': resnet_v1.resnet_arg_scope, + 'resnet_v2_50': resnet_v2.resnet_arg_scope, + 'resnet_v2_101': resnet_v2.resnet_arg_scope, + 'resnet_v2_152': resnet_v2.resnet_arg_scope, + 'resnet_v2_200': resnet_v2.resnet_arg_scope, + 'mobilenet_v1': mobilenet_v1.mobilenet_v1_arg_scope, + 'mobilenet_v1_075': mobilenet_v1.mobilenet_v1_arg_scope, + 'mobilenet_v1_050': mobilenet_v1.mobilenet_v1_arg_scope, + 'mobilenet_v1_025': mobilenet_v1.mobilenet_v1_arg_scope, + 'mobilenet_v2': mobilenet_v2.training_scope, + 'mobilenet_v2_035': mobilenet_v2.training_scope, + 'mobilenet_v2_140': mobilenet_v2.training_scope, + 'mobilenet_v3_small': mobilenet_v3.training_scope, + 'mobilenet_v3_large': mobilenet_v3.training_scope, + 'mobilenet_v3_small_minimalistic': mobilenet_v3.training_scope, + 'mobilenet_v3_large_minimalistic': mobilenet_v3.training_scope, + 'mobilenet_edgetpu': mobilenet_v3.training_scope, + 'mobilenet_edgetpu_075': mobilenet_v3.training_scope, + 'nasnet_cifar': nasnet.nasnet_cifar_arg_scope, + 'nasnet_mobile': nasnet.nasnet_mobile_arg_scope, + 'nasnet_large': nasnet.nasnet_large_arg_scope, + 'pnasnet_large': pnasnet.pnasnet_large_arg_scope, + 'pnasnet_mobile': pnasnet.pnasnet_mobile_arg_scope, +} + + +def get_network_fn(name, num_classes, weight_decay=0.0, is_training=False): + """Returns a network_fn such as `logits, end_points = network_fn(images)`. + + Args: + name: The name of the network. + num_classes: The number of classes to use for classification. If 0 or None, + the logits layer is omitted and its input features are returned instead. + weight_decay: The l2 coefficient for the model weights. + is_training: `True` if the model is being used for training and `False` + otherwise. + + Returns: + network_fn: A function that applies the model to a batch of images. It has + the following signature: + net, end_points = network_fn(images) + The `images` input is a tensor of shape [batch_size, height, width, 3 or + 1] with height = width = network_fn.default_image_size. (The + permissibility and treatment of other sizes depends on the network_fn.) + The returned `end_points` are a dictionary of intermediate activations. + The returned `net` is the topmost layer, depending on `num_classes`: + If `num_classes` was a non-zero integer, `net` is a logits tensor + of shape [batch_size, num_classes]. + If `num_classes` was 0 or `None`, `net` is a tensor with the input + to the logits layer of shape [batch_size, 1, 1, num_features] or + [batch_size, num_features]. Dropout has not been applied to this + (even if the network's original classification does); it remains for + the caller to do this or not. + + Raises: + ValueError: If network `name` is not recognized. + """ + if name not in networks_map: + raise ValueError('Name of network unknown %s' % name) + func = networks_map[name] + @functools.wraps(func) + def network_fn(images, **kwargs): + arg_scope = arg_scopes_map[name](weight_decay=weight_decay) + with slim.arg_scope(arg_scope): + return func(images, num_classes=num_classes, is_training=is_training, + **kwargs) + if hasattr(func, 'default_image_size'): + network_fn.default_image_size = func.default_image_size + + return network_fn diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nets_factory_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nets_factory_test.py new file mode 100644 index 0000000..3e16fc1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/nets_factory_test.py @@ -0,0 +1,78 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Tests for slim.inception.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +import tensorflow as tf + +from nets import nets_factory + + +class NetworksTest(tf.test.TestCase): + + def testGetNetworkFnFirstHalf(self): + batch_size = 5 + num_classes = 1000 + for net in list(nets_factory.networks_map.keys())[:10]: + with tf.Graph().as_default() as g, self.test_session(g): + net_fn = nets_factory.get_network_fn(net, num_classes=num_classes) + # Most networks use 224 as their default_image_size + image_size = getattr(net_fn, 'default_image_size', 224) + if net not in ['i3d', 's3dg']: + inputs = tf.random.uniform((batch_size, image_size, image_size, 3)) + logits, end_points = net_fn(inputs) + self.assertTrue(isinstance(logits, tf.Tensor)) + self.assertTrue(isinstance(end_points, dict)) + self.assertEqual(logits.get_shape().as_list()[0], batch_size) + self.assertEqual(logits.get_shape().as_list()[-1], num_classes) + + def testGetNetworkFnSecondHalf(self): + batch_size = 5 + num_classes = 1000 + for net in list(nets_factory.networks_map.keys())[10:]: + with tf.Graph().as_default() as g, self.test_session(g): + net_fn = nets_factory.get_network_fn(net, num_classes=num_classes) + # Most networks use 224 as their default_image_size + image_size = getattr(net_fn, 'default_image_size', 224) + if net not in ['i3d', 's3dg']: + inputs = tf.random.uniform((batch_size, image_size, image_size, 3)) + logits, end_points = net_fn(inputs) + self.assertTrue(isinstance(logits, tf.Tensor)) + self.assertTrue(isinstance(end_points, dict)) + self.assertEqual(logits.get_shape().as_list()[0], batch_size) + self.assertEqual(logits.get_shape().as_list()[-1], num_classes) + + def testGetNetworkFnVideoModels(self): + batch_size = 5 + num_classes = 400 + for net in ['i3d', 's3dg']: + with tf.Graph().as_default() as g, self.test_session(g): + net_fn = nets_factory.get_network_fn(net, num_classes=num_classes) + # Most networks use 224 as their default_image_size + image_size = getattr(net_fn, 'default_image_size', 224) // 2 + inputs = tf.random.uniform((batch_size, 10, image_size, image_size, 3)) + logits, end_points = net_fn(inputs) + self.assertTrue(isinstance(logits, tf.Tensor)) + self.assertTrue(isinstance(end_points, dict)) + self.assertEqual(logits.get_shape().as_list()[0], batch_size) + self.assertEqual(logits.get_shape().as_list()[-1], num_classes) + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/overfeat.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/overfeat.py new file mode 100644 index 0000000..8cd7096 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/overfeat.py @@ -0,0 +1,139 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains the model definition for the OverFeat network. + +The definition for the network was obtained from: + OverFeat: Integrated Recognition, Localization and Detection using + Convolutional Networks + Pierre Sermanet, David Eigen, Xiang Zhang, Michael Mathieu, Rob Fergus and + Yann LeCun, 2014 + http://arxiv.org/abs/1312.6229 + +Usage: + with slim.arg_scope(overfeat.overfeat_arg_scope()): + outputs, end_points = overfeat.overfeat(inputs) + +@@overfeat +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +slim = contrib_slim + +# pylint: disable=g-long-lambda +trunc_normal = lambda stddev: tf.compat.v1.truncated_normal_initializer( + 0.0, stddev) + + +def overfeat_arg_scope(weight_decay=0.0005): + with slim.arg_scope([slim.conv2d, slim.fully_connected], + activation_fn=tf.nn.relu, + weights_regularizer=slim.l2_regularizer(weight_decay), + biases_initializer=tf.compat.v1.zeros_initializer()): + with slim.arg_scope([slim.conv2d], padding='SAME'): + with slim.arg_scope([slim.max_pool2d], padding='VALID') as arg_sc: + return arg_sc + + +def overfeat(inputs, + num_classes=1000, + is_training=True, + dropout_keep_prob=0.5, + spatial_squeeze=True, + scope='overfeat', + global_pool=False): + """Contains the model definition for the OverFeat network. + + The definition for the network was obtained from: + OverFeat: Integrated Recognition, Localization and Detection using + Convolutional Networks + Pierre Sermanet, David Eigen, Xiang Zhang, Michael Mathieu, Rob Fergus and + Yann LeCun, 2014 + http://arxiv.org/abs/1312.6229 + + Note: All the fully_connected layers have been transformed to conv2d layers. + To use in classification mode, resize input to 231x231. To use in fully + convolutional mode, set spatial_squeeze to false. + + Args: + inputs: a tensor of size [batch_size, height, width, channels]. + num_classes: number of predicted classes. If 0 or None, the logits layer is + omitted and the input features to the logits layer are returned instead. + is_training: whether or not the model is being trained. + dropout_keep_prob: the probability that activations are kept in the dropout + layers during training. + spatial_squeeze: whether or not should squeeze the spatial dimensions of the + outputs. Useful to remove unnecessary dimensions for classification. + scope: Optional scope for the variables. + global_pool: Optional boolean flag. If True, the input to the classification + layer is avgpooled to size 1x1, for any input size. (This is not part + of the original OverFeat.) + + Returns: + net: the output of the logits layer (if num_classes is a non-zero integer), + or the non-dropped-out input to the logits layer (if num_classes is 0 or + None). + end_points: a dict of tensors with intermediate activations. + """ + with tf.compat.v1.variable_scope(scope, 'overfeat', [inputs]) as sc: + end_points_collection = sc.original_name_scope + '_end_points' + # Collect outputs for conv2d, fully_connected and max_pool2d + with slim.arg_scope([slim.conv2d, slim.fully_connected, slim.max_pool2d], + outputs_collections=end_points_collection): + net = slim.conv2d(inputs, 64, [11, 11], 4, padding='VALID', + scope='conv1') + net = slim.max_pool2d(net, [2, 2], scope='pool1') + net = slim.conv2d(net, 256, [5, 5], padding='VALID', scope='conv2') + net = slim.max_pool2d(net, [2, 2], scope='pool2') + net = slim.conv2d(net, 512, [3, 3], scope='conv3') + net = slim.conv2d(net, 1024, [3, 3], scope='conv4') + net = slim.conv2d(net, 1024, [3, 3], scope='conv5') + net = slim.max_pool2d(net, [2, 2], scope='pool5') + + # Use conv2d instead of fully_connected layers. + with slim.arg_scope( + [slim.conv2d], + weights_initializer=trunc_normal(0.005), + biases_initializer=tf.compat.v1.constant_initializer(0.1)): + net = slim.conv2d(net, 3072, [6, 6], padding='VALID', scope='fc6') + net = slim.dropout(net, dropout_keep_prob, is_training=is_training, + scope='dropout6') + net = slim.conv2d(net, 4096, [1, 1], scope='fc7') + # Convert end_points_collection into a end_point dict. + end_points = slim.utils.convert_collection_to_dict( + end_points_collection) + if global_pool: + net = tf.reduce_mean( + input_tensor=net, axis=[1, 2], keepdims=True, name='global_pool') + end_points['global_pool'] = net + if num_classes: + net = slim.dropout(net, dropout_keep_prob, is_training=is_training, + scope='dropout7') + net = slim.conv2d( + net, + num_classes, [1, 1], + activation_fn=None, + normalizer_fn=None, + biases_initializer=tf.compat.v1.zeros_initializer(), + scope='fc8') + if spatial_squeeze: + net = tf.squeeze(net, [1, 2], name='fc8/squeezed') + end_points[sc.name + '/fc8'] = net + return net, end_points +overfeat.default_image_size = 231 diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/overfeat_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/overfeat_test.py new file mode 100644 index 0000000..894df8e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/overfeat_test.py @@ -0,0 +1,179 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for slim.nets.overfeat.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets import overfeat + +slim = contrib_slim + + +class OverFeatTest(tf.test.TestCase): + + def testBuild(self): + batch_size = 5 + height, width = 231, 231 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = overfeat.overfeat(inputs, num_classes) + self.assertEquals(logits.op.name, 'overfeat/fc8/squeezed') + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + + def testFullyConvolutional(self): + batch_size = 1 + height, width = 281, 281 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = overfeat.overfeat(inputs, num_classes, spatial_squeeze=False) + self.assertEquals(logits.op.name, 'overfeat/fc8/BiasAdd') + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, 2, 2, num_classes]) + + def testGlobalPool(self): + batch_size = 1 + height, width = 281, 281 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = overfeat.overfeat(inputs, num_classes, spatial_squeeze=False, + global_pool=True) + self.assertEquals(logits.op.name, 'overfeat/fc8/BiasAdd') + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, 1, 1, num_classes]) + + def testEndPoints(self): + batch_size = 5 + height, width = 231, 231 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = overfeat.overfeat(inputs, num_classes) + expected_names = ['overfeat/conv1', + 'overfeat/pool1', + 'overfeat/conv2', + 'overfeat/pool2', + 'overfeat/conv3', + 'overfeat/conv4', + 'overfeat/conv5', + 'overfeat/pool5', + 'overfeat/fc6', + 'overfeat/fc7', + 'overfeat/fc8' + ] + self.assertSetEqual(set(end_points.keys()), set(expected_names)) + + def testNoClasses(self): + batch_size = 5 + height, width = 231, 231 + num_classes = None + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + net, end_points = overfeat.overfeat(inputs, num_classes) + expected_names = ['overfeat/conv1', + 'overfeat/pool1', + 'overfeat/conv2', + 'overfeat/pool2', + 'overfeat/conv3', + 'overfeat/conv4', + 'overfeat/conv5', + 'overfeat/pool5', + 'overfeat/fc6', + 'overfeat/fc7' + ] + self.assertSetEqual(set(end_points.keys()), set(expected_names)) + self.assertTrue(net.op.name.startswith('overfeat/fc7')) + + def testModelVariables(self): + batch_size = 5 + height, width = 231, 231 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + overfeat.overfeat(inputs, num_classes) + expected_names = ['overfeat/conv1/weights', + 'overfeat/conv1/biases', + 'overfeat/conv2/weights', + 'overfeat/conv2/biases', + 'overfeat/conv3/weights', + 'overfeat/conv3/biases', + 'overfeat/conv4/weights', + 'overfeat/conv4/biases', + 'overfeat/conv5/weights', + 'overfeat/conv5/biases', + 'overfeat/fc6/weights', + 'overfeat/fc6/biases', + 'overfeat/fc7/weights', + 'overfeat/fc7/biases', + 'overfeat/fc8/weights', + 'overfeat/fc8/biases', + ] + model_variables = [v.op.name for v in slim.get_model_variables()] + self.assertSetEqual(set(model_variables), set(expected_names)) + + def testEvaluation(self): + batch_size = 2 + height, width = 231, 231 + num_classes = 1000 + with self.test_session(): + eval_inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = overfeat.overfeat(eval_inputs, is_training=False) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + predictions = tf.argmax(input=logits, axis=1) + self.assertListEqual(predictions.get_shape().as_list(), [batch_size]) + + def testTrainEvalWithReuse(self): + train_batch_size = 2 + eval_batch_size = 1 + train_height, train_width = 231, 231 + eval_height, eval_width = 281, 281 + num_classes = 1000 + with self.test_session(): + train_inputs = tf.random.uniform( + (train_batch_size, train_height, train_width, 3)) + logits, _ = overfeat.overfeat(train_inputs) + self.assertListEqual(logits.get_shape().as_list(), + [train_batch_size, num_classes]) + tf.compat.v1.get_variable_scope().reuse_variables() + eval_inputs = tf.random.uniform( + (eval_batch_size, eval_height, eval_width, 3)) + logits, _ = overfeat.overfeat(eval_inputs, is_training=False, + spatial_squeeze=False) + self.assertListEqual(logits.get_shape().as_list(), + [eval_batch_size, 2, 2, num_classes]) + logits = tf.reduce_mean(input_tensor=logits, axis=[1, 2]) + predictions = tf.argmax(input=logits, axis=1) + self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) + + def testForward(self): + batch_size = 1 + height, width = 231, 231 + with self.test_session() as sess: + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = overfeat.overfeat(inputs) + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(logits) + self.assertTrue(output.any()) + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/pix2pix.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/pix2pix.py new file mode 100644 index 0000000..b393d65 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/pix2pix.py @@ -0,0 +1,297 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +"""Implementation of the Image-to-Image Translation model. + +This network represents a port of the following work: + + Image-to-Image Translation with Conditional Adversarial Networks + Phillip Isola, Jun-Yan Zhu, Tinghui Zhou and Alexei A. Efros + Arxiv, 2017 + https://phillipi.github.io/pix2pix/ + +A reference implementation written in Lua can be found at: +https://github.com/phillipi/pix2pix/blob/master/models.lua +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import functools + +import tensorflow as tf +from tensorflow.contrib import framework as contrib_framework +from tensorflow.contrib import layers as contrib_layers + +layers = contrib_layers + + +def pix2pix_arg_scope(): + """Returns a default argument scope for isola_net. + + Returns: + An arg scope. + """ + # These parameters come from the online port, which don't necessarily match + # those in the paper. + # TODO(nsilberman): confirm these values with Philip. + instance_norm_params = { + 'center': True, + 'scale': True, + 'epsilon': 0.00001, + } + + with contrib_framework.arg_scope( + [layers.conv2d, layers.conv2d_transpose], + normalizer_fn=layers.instance_norm, + normalizer_params=instance_norm_params, + weights_initializer=tf.compat.v1.random_normal_initializer(0, + 0.02)) as sc: + return sc + + +def upsample(net, num_outputs, kernel_size, method='nn_upsample_conv'): + """Upsamples the given inputs. + + Args: + net: A `Tensor` of size [batch_size, height, width, filters]. + num_outputs: The number of output filters. + kernel_size: A list of 2 scalars or a 1x2 `Tensor` indicating the scale, + relative to the inputs, of the output dimensions. For example, if kernel + size is [2, 3], then the output height and width will be twice and three + times the input size. + method: The upsampling method. + + Returns: + An `Tensor` which was upsampled using the specified method. + + Raises: + ValueError: if `method` is not recognized. + """ + net_shape = tf.shape(input=net) + height = net_shape[1] + width = net_shape[2] + + if method == 'nn_upsample_conv': + net = tf.image.resize( + net, [kernel_size[0] * height, kernel_size[1] * width], + method=tf.image.ResizeMethod.NEAREST_NEIGHBOR) + net = layers.conv2d(net, num_outputs, [4, 4], activation_fn=None) + elif method == 'conv2d_transpose': + net = layers.conv2d_transpose( + net, num_outputs, [4, 4], stride=kernel_size, activation_fn=None) + else: + raise ValueError('Unknown method: [%s]' % method) + + return net + + +class Block( + collections.namedtuple('Block', ['num_filters', 'decoder_keep_prob'])): + """Represents a single block of encoder and decoder processing. + + The Image-to-Image translation paper works a bit differently than the original + U-Net model. In particular, each block represents a single operation in the + encoder which is concatenated with the corresponding decoder representation. + A dropout layer follows the concatenation and convolution of the concatenated + features. + """ + pass + + +def _default_generator_blocks(): + """Returns the default generator block definitions. + + Returns: + A list of generator blocks. + """ + return [ + Block(64, 0.5), + Block(128, 0.5), + Block(256, 0.5), + Block(512, 0), + Block(512, 0), + Block(512, 0), + Block(512, 0), + ] + + +def pix2pix_generator(net, + num_outputs, + blocks=None, + upsample_method='nn_upsample_conv', + is_training=False): # pylint: disable=unused-argument + """Defines the network architecture. + + Args: + net: A `Tensor` of size [batch, height, width, channels]. Note that the + generator currently requires square inputs (e.g. height=width). + num_outputs: The number of (per-pixel) outputs. + blocks: A list of generator blocks or `None` to use the default generator + definition. + upsample_method: The method of upsampling images, one of 'nn_upsample_conv' + or 'conv2d_transpose' + is_training: Whether or not we're in training or testing mode. + + Returns: + A `Tensor` representing the model output and a dictionary of model end + points. + + Raises: + ValueError: if the input heights do not match their widths. + """ + end_points = {} + + blocks = blocks or _default_generator_blocks() + + input_size = net.get_shape().as_list() + + input_size[3] = num_outputs + + upsample_fn = functools.partial(upsample, method=upsample_method) + + encoder_activations = [] + + ########### + # Encoder # + ########### + with tf.compat.v1.variable_scope('encoder'): + with contrib_framework.arg_scope([layers.conv2d], + kernel_size=[4, 4], + stride=2, + activation_fn=tf.nn.leaky_relu): + + for block_id, block in enumerate(blocks): + # No normalizer for the first encoder layers as per 'Image-to-Image', + # Section 5.1.1 + if block_id == 0: + # First layer doesn't use normalizer_fn + net = layers.conv2d(net, block.num_filters, normalizer_fn=None) + elif block_id < len(blocks) - 1: + net = layers.conv2d(net, block.num_filters) + else: + # Last layer doesn't use activation_fn nor normalizer_fn + net = layers.conv2d( + net, block.num_filters, activation_fn=None, normalizer_fn=None) + + encoder_activations.append(net) + end_points['encoder%d' % block_id] = net + + ########### + # Decoder # + ########### + reversed_blocks = list(blocks) + reversed_blocks.reverse() + + with tf.compat.v1.variable_scope('decoder'): + # Dropout is used at both train and test time as per 'Image-to-Image', + # Section 2.1 (last paragraph). + with contrib_framework.arg_scope([layers.dropout], is_training=True): + + for block_id, block in enumerate(reversed_blocks): + if block_id > 0: + net = tf.concat([net, encoder_activations[-block_id - 1]], axis=3) + + # The Relu comes BEFORE the upsample op: + net = tf.nn.relu(net) + net = upsample_fn(net, block.num_filters, [2, 2]) + if block.decoder_keep_prob > 0: + net = layers.dropout(net, keep_prob=block.decoder_keep_prob) + end_points['decoder%d' % block_id] = net + + with tf.compat.v1.variable_scope('output'): + # Explicitly set the normalizer_fn to None to override any default value + # that may come from an arg_scope, such as pix2pix_arg_scope. + logits = layers.conv2d( + net, num_outputs, [4, 4], activation_fn=None, normalizer_fn=None) + logits = tf.reshape(logits, input_size) + + end_points['logits'] = logits + end_points['predictions'] = tf.tanh(logits) + + return logits, end_points + + +def pix2pix_discriminator(net, num_filters, padding=2, pad_mode='REFLECT', + activation_fn=tf.nn.leaky_relu, is_training=False): + """Creates the Image2Image Translation Discriminator. + + Args: + net: A `Tensor` of size [batch_size, height, width, channels] representing + the input. + num_filters: A list of the filters in the discriminator. The length of the + list determines the number of layers in the discriminator. + padding: Amount of reflection padding applied before each convolution. + pad_mode: mode for tf.pad, one of "CONSTANT", "REFLECT", or "SYMMETRIC". + activation_fn: activation fn for layers.conv2d. + is_training: Whether or not the model is training or testing. + + Returns: + A logits `Tensor` of size [batch_size, N, N, 1] where N is the number of + 'patches' we're attempting to discriminate and a dictionary of model end + points. + """ + del is_training + end_points = {} + + num_layers = len(num_filters) + + def padded(net, scope): + if padding: + with tf.compat.v1.variable_scope(scope): + spatial_pad = tf.constant( + [[0, 0], [padding, padding], [padding, padding], [0, 0]], + dtype=tf.int32) + return tf.pad(tensor=net, paddings=spatial_pad, mode=pad_mode) + else: + return net + + with contrib_framework.arg_scope([layers.conv2d], + kernel_size=[4, 4], + stride=2, + padding='valid', + activation_fn=activation_fn): + + # No normalization on the input layer. + net = layers.conv2d( + padded(net, 'conv0'), num_filters[0], normalizer_fn=None, scope='conv0') + + end_points['conv0'] = net + + for i in range(1, num_layers - 1): + net = layers.conv2d( + padded(net, 'conv%d' % i), num_filters[i], scope='conv%d' % i) + end_points['conv%d' % i] = net + + # Stride 1 on the last layer. + net = layers.conv2d( + padded(net, 'conv%d' % (num_layers - 1)), + num_filters[-1], + stride=1, + scope='conv%d' % (num_layers - 1)) + end_points['conv%d' % (num_layers - 1)] = net + + # 1-dim logits, stride 1, no activation, no normalization. + logits = layers.conv2d( + padded(net, 'conv%d' % num_layers), + 1, + stride=1, + activation_fn=None, + normalizer_fn=None, + scope='conv%d' % num_layers) + end_points['logits'] = logits + end_points['predictions'] = tf.sigmoid(logits) + return logits, end_points diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/pix2pix_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/pix2pix_test.py new file mode 100644 index 0000000..d1bfa1b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/pix2pix_test.py @@ -0,0 +1,157 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +"""Tests for pix2pix.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import framework as contrib_framework +from nets import pix2pix + + +class GeneratorTest(tf.test.TestCase): + + def _reduced_default_blocks(self): + """Returns the default blocks, scaled down to make test run faster.""" + return [pix2pix.Block(b.num_filters // 32, b.decoder_keep_prob) + for b in pix2pix._default_generator_blocks()] + + def test_output_size_nn_upsample_conv(self): + batch_size = 2 + height, width = 256, 256 + num_outputs = 4 + + images = tf.ones((batch_size, height, width, 3)) + with contrib_framework.arg_scope(pix2pix.pix2pix_arg_scope()): + logits, _ = pix2pix.pix2pix_generator( + images, num_outputs, blocks=self._reduced_default_blocks(), + upsample_method='nn_upsample_conv') + + with self.test_session() as session: + session.run(tf.compat.v1.global_variables_initializer()) + np_outputs = session.run(logits) + self.assertListEqual([batch_size, height, width, num_outputs], + list(np_outputs.shape)) + + def test_output_size_conv2d_transpose(self): + batch_size = 2 + height, width = 256, 256 + num_outputs = 4 + + images = tf.ones((batch_size, height, width, 3)) + with contrib_framework.arg_scope(pix2pix.pix2pix_arg_scope()): + logits, _ = pix2pix.pix2pix_generator( + images, num_outputs, blocks=self._reduced_default_blocks(), + upsample_method='conv2d_transpose') + + with self.test_session() as session: + session.run(tf.compat.v1.global_variables_initializer()) + np_outputs = session.run(logits) + self.assertListEqual([batch_size, height, width, num_outputs], + list(np_outputs.shape)) + + def test_block_number_dictates_number_of_layers(self): + batch_size = 2 + height, width = 256, 256 + num_outputs = 4 + + images = tf.ones((batch_size, height, width, 3)) + blocks = [ + pix2pix.Block(64, 0.5), + pix2pix.Block(128, 0), + ] + with contrib_framework.arg_scope(pix2pix.pix2pix_arg_scope()): + _, end_points = pix2pix.pix2pix_generator( + images, num_outputs, blocks) + + num_encoder_layers = 0 + num_decoder_layers = 0 + for end_point in end_points: + if end_point.startswith('encoder'): + num_encoder_layers += 1 + elif end_point.startswith('decoder'): + num_decoder_layers += 1 + + self.assertEqual(num_encoder_layers, len(blocks)) + self.assertEqual(num_decoder_layers, len(blocks)) + + +class DiscriminatorTest(tf.test.TestCase): + + def _layer_output_size(self, input_size, kernel_size=4, stride=2, pad=2): + return (input_size + pad * 2 - kernel_size) // stride + 1 + + def test_four_layers(self): + batch_size = 2 + input_size = 256 + + output_size = self._layer_output_size(input_size) + output_size = self._layer_output_size(output_size) + output_size = self._layer_output_size(output_size) + output_size = self._layer_output_size(output_size, stride=1) + output_size = self._layer_output_size(output_size, stride=1) + + images = tf.ones((batch_size, input_size, input_size, 3)) + with contrib_framework.arg_scope(pix2pix.pix2pix_arg_scope()): + logits, end_points = pix2pix.pix2pix_discriminator( + images, num_filters=[64, 128, 256, 512]) + self.assertListEqual([batch_size, output_size, output_size, 1], + logits.shape.as_list()) + self.assertListEqual([batch_size, output_size, output_size, 1], + end_points['predictions'].shape.as_list()) + + def test_four_layers_no_padding(self): + batch_size = 2 + input_size = 256 + + output_size = self._layer_output_size(input_size, pad=0) + output_size = self._layer_output_size(output_size, pad=0) + output_size = self._layer_output_size(output_size, pad=0) + output_size = self._layer_output_size(output_size, stride=1, pad=0) + output_size = self._layer_output_size(output_size, stride=1, pad=0) + + images = tf.ones((batch_size, input_size, input_size, 3)) + with contrib_framework.arg_scope(pix2pix.pix2pix_arg_scope()): + logits, end_points = pix2pix.pix2pix_discriminator( + images, num_filters=[64, 128, 256, 512], padding=0) + self.assertListEqual([batch_size, output_size, output_size, 1], + logits.shape.as_list()) + self.assertListEqual([batch_size, output_size, output_size, 1], + end_points['predictions'].shape.as_list()) + + def test_four_layers_wrog_paddig(self): + batch_size = 2 + input_size = 256 + + images = tf.ones((batch_size, input_size, input_size, 3)) + with contrib_framework.arg_scope(pix2pix.pix2pix_arg_scope()): + with self.assertRaises(TypeError): + pix2pix.pix2pix_discriminator( + images, num_filters=[64, 128, 256, 512], padding=1.5) + + def test_four_layers_negative_padding(self): + batch_size = 2 + input_size = 256 + + images = tf.ones((batch_size, input_size, input_size, 3)) + with contrib_framework.arg_scope(pix2pix.pix2pix_arg_scope()): + with self.assertRaises(ValueError): + pix2pix.pix2pix_discriminator( + images, num_filters=[64, 128, 256, 512], padding=-1) + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/post_training_quantization.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/post_training_quantization.py new file mode 100644 index 0000000..b2d6e3f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/post_training_quantization.py @@ -0,0 +1,181 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Export quantized tflite model from a trained checkpoint.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools +from absl import app +from absl import flags +import tensorflow as tf +import tensorflow_datasets as tfds +from nets import nets_factory +from preprocessing import preprocessing_factory + +flags.DEFINE_string("model_name", None, + "The name of the architecture to quantize.") +flags.DEFINE_string("checkpoint_path", None, "Path to the training checkpoint.") +flags.DEFINE_string("dataset_name", "imagenet2012", + "Name of the dataset to use for quantization calibration.") +flags.DEFINE_string("dataset_dir", None, "Dataset location.") +flags.DEFINE_string( + "dataset_split", "train", + "The dataset split (train, validation etc.) to use for calibration.") +flags.DEFINE_string("output_tflite", None, "Path to output tflite file.") +flags.DEFINE_boolean( + "use_model_specific_preprocessing", False, + "When true, uses the preprocessing corresponding to the model as specified " + "in preprocessing factory.") +flags.DEFINE_boolean("enable_ema", True, + "Load exponential moving average version of variables.") +flags.DEFINE_integer( + "num_steps", 1000, + "Number of post-training quantization calibration steps to run.") +flags.DEFINE_integer("image_size", 224, "Size of the input image.") +flags.DEFINE_integer("num_classes", 1001, + "Number of output classes for the model.") + +FLAGS = flags.FLAGS + +# Mean and standard deviation used for normalizing the image tensor. +_MEAN_RGB = 127.5 +_STD_RGB = 127.5 + + +def _preprocess_for_quantization(image_data, image_size, crop_padding=32): + """Crops to center of image with padding then scales, normalizes image_size. + + Args: + image_data: A 3D Tensor representing the RGB image data. Image can be of + arbitrary height and width. + image_size: image height/width dimension. + crop_padding: the padding size to use when centering the crop. + + Returns: + A decoded and cropped image Tensor. Image is normalized to [-1,1]. + + """ + + shape = tf.shape(image_data) + image_height = shape[0] + image_width = shape[1] + + padded_center_crop_size = tf.cast( + (image_size * 1.0 / (image_size + crop_padding)) * + tf.cast(tf.minimum(image_height, image_width), tf.float32), tf.int32) + + offset_height = ((image_height - padded_center_crop_size) + 1) // 2 + offset_width = ((image_width - padded_center_crop_size) + 1) // 2 + + image = tf.image.crop_to_bounding_box( + image_data, + offset_height=offset_height, + offset_width=offset_width, + target_height=padded_center_crop_size, + target_width=padded_center_crop_size) + + image = tf.image.resize([image], [image_size, image_size], + method=tf.image.ResizeMethod.BICUBIC)[0] + image = tf.cast(image, tf.float32) + image -= tf.constant(_MEAN_RGB) + image /= tf.constant(_STD_RGB) + return image + + +def restore_model(sess, checkpoint_path, enable_ema=True): + """Restore variables from the checkpoint into the provided session. + + Args: + sess: A tensorflow session where the checkpoint will be loaded. + checkpoint_path: Path to the trained checkpoint. + enable_ema: (optional) Whether to load the exponential moving average (ema) + version of the tensorflow variables. Defaults to True. + """ + if enable_ema: + ema = tf.train.ExponentialMovingAverage(decay=0.0) + ema_vars = tf.trainable_variables() + tf.get_collection("moving_vars") + for v in tf.global_variables(): + if "moving_mean" in v.name or "moving_variance" in v.name: + ema_vars.append(v) + ema_vars = list(set(ema_vars)) + var_dict = ema.variables_to_restore(ema_vars) + else: + var_dict = None + + sess.run(tf.global_variables_initializer()) + saver = tf.train.Saver(var_dict, max_to_keep=1) + saver.restore(sess, checkpoint_path) + + +def _representative_dataset_gen(): + """Gets a python generator of numpy arrays for the given dataset.""" + image_size = FLAGS.image_size + dataset = tfds.builder(FLAGS.dataset_name, data_dir=FLAGS.dataset_dir) + dataset.download_and_prepare() + data = dataset.as_dataset()[FLAGS.dataset_split] + iterator = tf.compat.v1.data.make_one_shot_iterator(data) + if FLAGS.use_model_specific_preprocessing: + preprocess_fn = functools.partial( + preprocessing_factory.get_preprocessing(name=FLAGS.model_name), + output_height=image_size, + output_width=image_size) + else: + preprocess_fn = functools.partial( + _preprocess_for_quantization, image_size=image_size) + features = iterator.get_next() + image = features["image"] + image = preprocess_fn(image) + image = tf.reshape(image, [1, image_size, image_size, 3]) + for _ in range(FLAGS.num_steps): + yield [image.eval()] + + +def main(_): + with tf.Graph().as_default(), tf.Session() as sess: + network_fn = nets_factory.get_network_fn( + FLAGS.model_name, num_classes=FLAGS.num_classes, is_training=False) + image_size = FLAGS.image_size + images = tf.placeholder( + tf.float32, shape=(1, image_size, image_size, 3), name="images") + + logits, _ = network_fn(images) + + output_tensor = tf.nn.softmax(logits) + restore_model(sess, FLAGS.checkpoint_path, enable_ema=FLAGS.enable_ema) + + converter = tf.lite.TFLiteConverter.from_session(sess, [images], + [output_tensor]) + + converter.representative_dataset = tf.lite.RepresentativeDataset( + _representative_dataset_gen) + converter.optimizations = [tf.lite.Optimize.DEFAULT] + converter.inference_input_type = tf.int8 + converter.inference_output_type = tf.int8 + converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8] + + tflite_buffer = converter.convert() + with tf.gfile.GFile(FLAGS.output_tflite, "wb") as output_tflite: + output_tflite.write(tflite_buffer) + print("tflite model written to %s" % FLAGS.output_tflite) + + +if __name__ == "__main__": + flags.mark_flag_as_required("model_name") + flags.mark_flag_as_required("checkpoint_path") + flags.mark_flag_as_required("dataset_dir") + flags.mark_flag_as_required("output_tflite") + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_utils.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_utils.py new file mode 100644 index 0000000..3230e28 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_utils.py @@ -0,0 +1,278 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains building blocks for various versions of Residual Networks. + +Residual networks (ResNets) were proposed in: + Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Deep Residual Learning for Image Recognition. arXiv:1512.03385, 2015 + +More variants were introduced in: + Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Identity Mappings in Deep Residual Networks. arXiv: 1603.05027, 2016 + +We can obtain different ResNet variants by changing the network depth, width, +and form of residual unit. This module implements the infrastructure for +building them. Concrete ResNet units and full ResNet networks are implemented in +the accompanying resnet_v1.py and resnet_v2.py modules. + +Compared to https://github.com/KaimingHe/deep-residual-networks, in the current +implementation we subsample the output activations in the last residual unit of +each block, instead of subsampling the input activations in the first residual +unit of each block. The two implementations give identical results but our +implementation is more memory efficient. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +slim = contrib_slim + + +class Block(collections.namedtuple('Block', ['scope', 'unit_fn', 'args'])): + """A named tuple describing a ResNet block. + + Its parts are: + scope: The scope of the `Block`. + unit_fn: The ResNet unit function which takes as input a `Tensor` and + returns another `Tensor` with the output of the ResNet unit. + args: A list of length equal to the number of units in the `Block`. The list + contains one (depth, depth_bottleneck, stride) tuple for each unit in the + block to serve as argument to unit_fn. + """ + + +def subsample(inputs, factor, scope=None): + """Subsamples the input along the spatial dimensions. + + Args: + inputs: A `Tensor` of size [batch, height_in, width_in, channels]. + factor: The subsampling factor. + scope: Optional variable_scope. + + Returns: + output: A `Tensor` of size [batch, height_out, width_out, channels] with the + input, either intact (if factor == 1) or subsampled (if factor > 1). + """ + if factor == 1: + return inputs + else: + return slim.max_pool2d(inputs, [1, 1], stride=factor, scope=scope) + + +def conv2d_same(inputs, num_outputs, kernel_size, stride, rate=1, scope=None): + """Strided 2-D convolution with 'SAME' padding. + + When stride > 1, then we do explicit zero-padding, followed by conv2d with + 'VALID' padding. + + Note that + + net = conv2d_same(inputs, num_outputs, 3, stride=stride) + + is equivalent to + + net = slim.conv2d(inputs, num_outputs, 3, stride=1, padding='SAME') + net = subsample(net, factor=stride) + + whereas + + net = slim.conv2d(inputs, num_outputs, 3, stride=stride, padding='SAME') + + is different when the input's height or width is even, which is why we add the + current function. For more details, see ResnetUtilsTest.testConv2DSameEven(). + + Args: + inputs: A 4-D tensor of size [batch, height_in, width_in, channels]. + num_outputs: An integer, the number of output filters. + kernel_size: An int with the kernel_size of the filters. + stride: An integer, the output stride. + rate: An integer, rate for atrous convolution. + scope: Scope. + + Returns: + output: A 4-D tensor of size [batch, height_out, width_out, channels] with + the convolution output. + """ + if stride == 1: + return slim.conv2d(inputs, num_outputs, kernel_size, stride=1, rate=rate, + padding='SAME', scope=scope) + else: + kernel_size_effective = kernel_size + (kernel_size - 1) * (rate - 1) + pad_total = kernel_size_effective - 1 + pad_beg = pad_total // 2 + pad_end = pad_total - pad_beg + inputs = tf.pad( + tensor=inputs, + paddings=[[0, 0], [pad_beg, pad_end], [pad_beg, pad_end], [0, 0]]) + return slim.conv2d(inputs, num_outputs, kernel_size, stride=stride, + rate=rate, padding='VALID', scope=scope) + + +@slim.add_arg_scope +def stack_blocks_dense(net, blocks, output_stride=None, + store_non_strided_activations=False, + outputs_collections=None): + """Stacks ResNet `Blocks` and controls output feature density. + + First, this function creates scopes for the ResNet in the form of + 'block_name/unit_1', 'block_name/unit_2', etc. + + Second, this function allows the user to explicitly control the ResNet + output_stride, which is the ratio of the input to output spatial resolution. + This is useful for dense prediction tasks such as semantic segmentation or + object detection. + + Most ResNets consist of 4 ResNet blocks and subsample the activations by a + factor of 2 when transitioning between consecutive ResNet blocks. This results + to a nominal ResNet output_stride equal to 8. If we set the output_stride to + half the nominal network stride (e.g., output_stride=4), then we compute + responses twice. + + Control of the output feature density is implemented by atrous convolution. + + Args: + net: A `Tensor` of size [batch, height, width, channels]. + blocks: A list of length equal to the number of ResNet `Blocks`. Each + element is a ResNet `Block` object describing the units in the `Block`. + output_stride: If `None`, then the output will be computed at the nominal + network stride. If output_stride is not `None`, it specifies the requested + ratio of input to output spatial resolution, which needs to be equal to + the product of unit strides from the start up to some level of the ResNet. + For example, if the ResNet employs units with strides 1, 2, 1, 3, 4, 1, + then valid values for the output_stride are 1, 2, 6, 24 or None (which + is equivalent to output_stride=24). + store_non_strided_activations: If True, we compute non-strided (undecimated) + activations at the last unit of each block and store them in the + `outputs_collections` before subsampling them. This gives us access to + higher resolution intermediate activations which are useful in some + dense prediction problems but increases 4x the computation and memory cost + at the last unit of each block. + outputs_collections: Collection to add the ResNet block outputs. + + Returns: + net: Output tensor with stride equal to the specified output_stride. + + Raises: + ValueError: If the target output_stride is not valid. + """ + # The current_stride variable keeps track of the effective stride of the + # activations. This allows us to invoke atrous convolution whenever applying + # the next residual unit would result in the activations having stride larger + # than the target output_stride. + current_stride = 1 + + # The atrous convolution rate parameter. + rate = 1 + + for block in blocks: + with tf.compat.v1.variable_scope(block.scope, 'block', [net]) as sc: + block_stride = 1 + for i, unit in enumerate(block.args): + if store_non_strided_activations and i == len(block.args) - 1: + # Move stride from the block's last unit to the end of the block. + block_stride = unit.get('stride', 1) + unit = dict(unit, stride=1) + + with tf.compat.v1.variable_scope('unit_%d' % (i + 1), values=[net]): + # If we have reached the target output_stride, then we need to employ + # atrous convolution with stride=1 and multiply the atrous rate by the + # current unit's stride for use in subsequent layers. + if output_stride is not None and current_stride == output_stride: + net = block.unit_fn(net, rate=rate, **dict(unit, stride=1)) + rate *= unit.get('stride', 1) + + else: + net = block.unit_fn(net, rate=1, **unit) + current_stride *= unit.get('stride', 1) + if output_stride is not None and current_stride > output_stride: + raise ValueError('The target output_stride cannot be reached.') + + # Collect activations at the block's end before performing subsampling. + net = slim.utils.collect_named_outputs(outputs_collections, sc.name, net) + + # Subsampling of the block's output activations. + if output_stride is not None and current_stride == output_stride: + rate *= block_stride + else: + net = subsample(net, block_stride) + current_stride *= block_stride + if output_stride is not None and current_stride > output_stride: + raise ValueError('The target output_stride cannot be reached.') + + if output_stride is not None and current_stride != output_stride: + raise ValueError('The target output_stride cannot be reached.') + + return net + + +def resnet_arg_scope( + weight_decay=0.0001, + batch_norm_decay=0.997, + batch_norm_epsilon=1e-5, + batch_norm_scale=True, + activation_fn=tf.nn.relu, + use_batch_norm=True, + batch_norm_updates_collections=tf.compat.v1.GraphKeys.UPDATE_OPS): + """Defines the default ResNet arg scope. + + TODO(gpapan): The batch-normalization related default values above are + appropriate for use in conjunction with the reference ResNet models + released at https://github.com/KaimingHe/deep-residual-networks. When + training ResNets from scratch, they might need to be tuned. + + Args: + weight_decay: The weight decay to use for regularizing the model. + batch_norm_decay: The moving average decay when estimating layer activation + statistics in batch normalization. + batch_norm_epsilon: Small constant to prevent division by zero when + normalizing activations by their variance in batch normalization. + batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the + activations in the batch normalization layer. + activation_fn: The activation function which is used in ResNet. + use_batch_norm: Whether or not to use batch normalization. + batch_norm_updates_collections: Collection for the update ops for + batch norm. + + Returns: + An `arg_scope` to use for the resnet models. + """ + batch_norm_params = { + 'decay': batch_norm_decay, + 'epsilon': batch_norm_epsilon, + 'scale': batch_norm_scale, + 'updates_collections': batch_norm_updates_collections, + 'fused': None, # Use fused batch norm if possible. + } + + with slim.arg_scope( + [slim.conv2d], + weights_regularizer=slim.l2_regularizer(weight_decay), + weights_initializer=slim.variance_scaling_initializer(), + activation_fn=activation_fn, + normalizer_fn=slim.batch_norm if use_batch_norm else None, + normalizer_params=batch_norm_params): + with slim.arg_scope([slim.batch_norm], **batch_norm_params): + # The following implies padding='SAME' for pool1, which makes feature + # alignment easier for dense prediction tasks. This is also used in + # https://github.com/facebook/fb.resnet.torch. However the accompanying + # code of 'Deep Residual Learning for Image Recognition' uses + # padding='VALID' for pool1. You can switch to that choice by setting + # slim.arg_scope([slim.max_pool2d], padding='VALID'). + with slim.arg_scope([slim.max_pool2d], padding='SAME') as arg_sc: + return arg_sc diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_v1.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_v1.py new file mode 100644 index 0000000..8451f46 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_v1.py @@ -0,0 +1,406 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains definitions for the original form of Residual Networks. + +The 'v1' residual networks (ResNets) implemented in this module were proposed +by: +[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Deep Residual Learning for Image Recognition. arXiv:1512.03385 + +Other variants were introduced in: +[2] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Identity Mappings in Deep Residual Networks. arXiv: 1603.05027 + +The networks defined in this module utilize the bottleneck building block of +[1] with projection shortcuts only for increasing depths. They employ batch +normalization *after* every weight layer. This is the architecture used by +MSRA in the Imagenet and MSCOCO 2016 competition models ResNet-101 and +ResNet-152. See [2; Fig. 1a] for a comparison between the current 'v1' +architecture and the alternative 'v2' architecture of [2] which uses batch +normalization *before* every weight layer in the so-called full pre-activation +units. + +Typical use: + + from tensorflow.contrib.slim.nets import resnet_v1 + +ResNet-101 for image classification into 1000 classes: + + # inputs has shape [batch, 224, 224, 3] + with slim.arg_scope(resnet_v1.resnet_arg_scope()): + net, end_points = resnet_v1.resnet_v1_101(inputs, 1000, is_training=False) + +ResNet-101 for semantic segmentation into 21 classes: + + # inputs has shape [batch, 513, 513, 3] + with slim.arg_scope(resnet_v1.resnet_arg_scope()): + net, end_points = resnet_v1.resnet_v1_101(inputs, + 21, + is_training=False, + global_pool=False, + output_stride=16) +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets import resnet_utils + + +resnet_arg_scope = resnet_utils.resnet_arg_scope +slim = contrib_slim + + +class NoOpScope(object): + """No-op context manager.""" + + def __enter__(self): + return None + + def __exit__(self, exc_type, exc_value, traceback): + return False + + +@slim.add_arg_scope +def bottleneck(inputs, + depth, + depth_bottleneck, + stride, + rate=1, + outputs_collections=None, + scope=None, + use_bounded_activations=False): + """Bottleneck residual unit variant with BN after convolutions. + + This is the original residual unit proposed in [1]. See Fig. 1(a) of [2] for + its definition. Note that we use here the bottleneck variant which has an + extra bottleneck layer. + + When putting together two consecutive ResNet blocks that use this unit, one + should use stride = 2 in the last unit of the first block. + + Args: + inputs: A tensor of size [batch, height, width, channels]. + depth: The depth of the ResNet unit output. + depth_bottleneck: The depth of the bottleneck layers. + stride: The ResNet unit's stride. Determines the amount of downsampling of + the units output compared to its input. + rate: An integer, rate for atrous convolution. + outputs_collections: Collection to add the ResNet unit output. + scope: Optional variable_scope. + use_bounded_activations: Whether or not to use bounded activations. Bounded + activations better lend themselves to quantized inference. + + Returns: + The ResNet unit's output. + """ + with tf.compat.v1.variable_scope(scope, 'bottleneck_v1', [inputs]) as sc: + depth_in = slim.utils.last_dimension(inputs.get_shape(), min_rank=4) + if depth == depth_in: + shortcut = resnet_utils.subsample(inputs, stride, 'shortcut') + else: + shortcut = slim.conv2d( + inputs, + depth, [1, 1], + stride=stride, + activation_fn=tf.nn.relu6 if use_bounded_activations else None, + scope='shortcut') + + residual = slim.conv2d(inputs, depth_bottleneck, [1, 1], stride=1, + scope='conv1') + residual = resnet_utils.conv2d_same(residual, depth_bottleneck, 3, stride, + rate=rate, scope='conv2') + residual = slim.conv2d(residual, depth, [1, 1], stride=1, + activation_fn=None, scope='conv3') + + if use_bounded_activations: + # Use clip_by_value to simulate bandpass activation. + residual = tf.clip_by_value(residual, -6.0, 6.0) + output = tf.nn.relu6(shortcut + residual) + else: + output = tf.nn.relu(shortcut + residual) + + return slim.utils.collect_named_outputs(outputs_collections, + sc.name, + output) + + +def resnet_v1(inputs, + blocks, + num_classes=None, + is_training=True, + global_pool=True, + output_stride=None, + include_root_block=True, + spatial_squeeze=True, + store_non_strided_activations=False, + reuse=None, + scope=None): + """Generator for v1 ResNet models. + + This function generates a family of ResNet v1 models. See the resnet_v1_*() + methods for specific model instantiations, obtained by selecting different + block instantiations that produce ResNets of various depths. + + Training for image classification on Imagenet is usually done with [224, 224] + inputs, resulting in [7, 7] feature maps at the output of the last ResNet + block for the ResNets defined in [1] that have nominal stride equal to 32. + However, for dense prediction tasks we advise that one uses inputs with + spatial dimensions that are multiples of 32 plus 1, e.g., [321, 321]. In + this case the feature maps at the ResNet output will have spatial shape + [(height - 1) / output_stride + 1, (width - 1) / output_stride + 1] + and corners exactly aligned with the input image corners, which greatly + facilitates alignment of the features to the image. Using as input [225, 225] + images results in [8, 8] feature maps at the output of the last ResNet block. + + For dense prediction tasks, the ResNet needs to run in fully-convolutional + (FCN) mode and global_pool needs to be set to False. The ResNets in [1, 2] all + have nominal stride equal to 32 and a good choice in FCN mode is to use + output_stride=16 in order to increase the density of the computed features at + small computational and memory overhead, cf. http://arxiv.org/abs/1606.00915. + + Args: + inputs: A tensor of size [batch, height_in, width_in, channels]. + blocks: A list of length equal to the number of ResNet blocks. Each element + is a resnet_utils.Block object describing the units in the block. + num_classes: Number of predicted classes for classification tasks. + If 0 or None, we return the features before the logit layer. + is_training: whether batch_norm layers are in training mode. If this is set + to None, the callers can specify slim.batch_norm's is_training parameter + from an outer slim.arg_scope. + global_pool: If True, we perform global average pooling before computing the + logits. Set to True for image classification, False for dense prediction. + output_stride: If None, then the output will be computed at the nominal + network stride. If output_stride is not None, it specifies the requested + ratio of input to output spatial resolution. + include_root_block: If True, include the initial convolution followed by + max-pooling, if False excludes it. + spatial_squeeze: if True, logits is of shape [B, C], if false logits is + of shape [B, 1, 1, C], where B is batch_size and C is number of classes. + To use this parameter, the input images must be smaller than 300x300 + pixels, in which case the output logit layer does not contain spatial + information and can be removed. + store_non_strided_activations: If True, we compute non-strided (undecimated) + activations at the last unit of each block and store them in the + `outputs_collections` before subsampling them. This gives us access to + higher resolution intermediate activations which are useful in some + dense prediction problems but increases 4x the computation and memory cost + at the last unit of each block. + reuse: whether or not the network and its variables should be reused. To be + able to reuse 'scope' must be given. + scope: Optional variable_scope. + + Returns: + net: A rank-4 tensor of size [batch, height_out, width_out, channels_out]. + If global_pool is False, then height_out and width_out are reduced by a + factor of output_stride compared to the respective height_in and width_in, + else both height_out and width_out equal one. If num_classes is 0 or None, + then net is the output of the last ResNet block, potentially after global + average pooling. If num_classes a non-zero integer, net contains the + pre-softmax activations. + end_points: A dictionary from components of the network to the corresponding + activation. + + Raises: + ValueError: If the target output_stride is not valid. + """ + with tf.compat.v1.variable_scope( + scope, 'resnet_v1', [inputs], reuse=reuse) as sc: + end_points_collection = sc.original_name_scope + '_end_points' + with slim.arg_scope([slim.conv2d, bottleneck, + resnet_utils.stack_blocks_dense], + outputs_collections=end_points_collection): + with (slim.arg_scope([slim.batch_norm], is_training=is_training) + if is_training is not None else NoOpScope()): + net = inputs + if include_root_block: + if output_stride is not None: + if output_stride % 4 != 0: + raise ValueError('The output_stride needs to be a multiple of 4.') + output_stride /= 4 + net = resnet_utils.conv2d_same(net, 64, 7, stride=2, scope='conv1') + net = slim.max_pool2d(net, [3, 3], stride=2, scope='pool1') + net = resnet_utils.stack_blocks_dense(net, blocks, output_stride, + store_non_strided_activations) + # Convert end_points_collection into a dictionary of end_points. + end_points = slim.utils.convert_collection_to_dict( + end_points_collection) + + if global_pool: + # Global average pooling. + net = tf.reduce_mean( + input_tensor=net, axis=[1, 2], name='pool5', keepdims=True) + end_points['global_pool'] = net + if num_classes: + net = slim.conv2d(net, num_classes, [1, 1], activation_fn=None, + normalizer_fn=None, scope='logits') + end_points[sc.name + '/logits'] = net + if spatial_squeeze: + net = tf.squeeze(net, [1, 2], name='SpatialSqueeze') + end_points[sc.name + '/spatial_squeeze'] = net + end_points['predictions'] = slim.softmax(net, scope='predictions') + return net, end_points +resnet_v1.default_image_size = 224 + + +def resnet_v1_block(scope, base_depth, num_units, stride): + """Helper function for creating a resnet_v1 bottleneck block. + + Args: + scope: The scope of the block. + base_depth: The depth of the bottleneck layer for each unit. + num_units: The number of units in the block. + stride: The stride of the block, implemented as a stride in the last unit. + All other units have stride=1. + + Returns: + A resnet_v1 bottleneck block. + """ + return resnet_utils.Block(scope, bottleneck, [{ + 'depth': base_depth * 4, + 'depth_bottleneck': base_depth, + 'stride': 1 + }] * (num_units - 1) + [{ + 'depth': base_depth * 4, + 'depth_bottleneck': base_depth, + 'stride': stride + }]) + + +def resnet_v1_50(inputs, + num_classes=None, + is_training=True, + global_pool=True, + output_stride=None, + spatial_squeeze=True, + store_non_strided_activations=False, + min_base_depth=8, + depth_multiplier=1, + reuse=None, + scope='resnet_v1_50'): + """ResNet-50 model of [1]. See resnet_v1() for arg and return description.""" + depth_func = lambda d: max(int(d * depth_multiplier), min_base_depth) + blocks = [ + resnet_v1_block('block1', base_depth=depth_func(64), num_units=3, + stride=2), + resnet_v1_block('block2', base_depth=depth_func(128), num_units=4, + stride=2), + resnet_v1_block('block3', base_depth=depth_func(256), num_units=6, + stride=2), + resnet_v1_block('block4', base_depth=depth_func(512), num_units=3, + stride=1), + ] + return resnet_v1(inputs, blocks, num_classes, is_training, + global_pool=global_pool, output_stride=output_stride, + include_root_block=True, spatial_squeeze=spatial_squeeze, + store_non_strided_activations=store_non_strided_activations, + reuse=reuse, scope=scope) +resnet_v1_50.default_image_size = resnet_v1.default_image_size + + +def resnet_v1_101(inputs, + num_classes=None, + is_training=True, + global_pool=True, + output_stride=None, + spatial_squeeze=True, + store_non_strided_activations=False, + min_base_depth=8, + depth_multiplier=1, + reuse=None, + scope='resnet_v1_101'): + """ResNet-101 model of [1]. See resnet_v1() for arg and return description.""" + depth_func = lambda d: max(int(d * depth_multiplier), min_base_depth) + blocks = [ + resnet_v1_block('block1', base_depth=depth_func(64), num_units=3, + stride=2), + resnet_v1_block('block2', base_depth=depth_func(128), num_units=4, + stride=2), + resnet_v1_block('block3', base_depth=depth_func(256), num_units=23, + stride=2), + resnet_v1_block('block4', base_depth=depth_func(512), num_units=3, + stride=1), + ] + return resnet_v1(inputs, blocks, num_classes, is_training, + global_pool=global_pool, output_stride=output_stride, + include_root_block=True, spatial_squeeze=spatial_squeeze, + store_non_strided_activations=store_non_strided_activations, + reuse=reuse, scope=scope) +resnet_v1_101.default_image_size = resnet_v1.default_image_size + + +def resnet_v1_152(inputs, + num_classes=None, + is_training=True, + global_pool=True, + output_stride=None, + store_non_strided_activations=False, + spatial_squeeze=True, + min_base_depth=8, + depth_multiplier=1, + reuse=None, + scope='resnet_v1_152'): + """ResNet-152 model of [1]. See resnet_v1() for arg and return description.""" + depth_func = lambda d: max(int(d * depth_multiplier), min_base_depth) + blocks = [ + resnet_v1_block('block1', base_depth=depth_func(64), num_units=3, + stride=2), + resnet_v1_block('block2', base_depth=depth_func(128), num_units=8, + stride=2), + resnet_v1_block('block3', base_depth=depth_func(256), num_units=36, + stride=2), + resnet_v1_block('block4', base_depth=depth_func(512), num_units=3, + stride=1), + ] + return resnet_v1(inputs, blocks, num_classes, is_training, + global_pool=global_pool, output_stride=output_stride, + include_root_block=True, spatial_squeeze=spatial_squeeze, + store_non_strided_activations=store_non_strided_activations, + reuse=reuse, scope=scope) +resnet_v1_152.default_image_size = resnet_v1.default_image_size + + +def resnet_v1_200(inputs, + num_classes=None, + is_training=True, + global_pool=True, + output_stride=None, + store_non_strided_activations=False, + spatial_squeeze=True, + min_base_depth=8, + depth_multiplier=1, + reuse=None, + scope='resnet_v1_200'): + """ResNet-200 model of [2]. See resnet_v1() for arg and return description.""" + depth_func = lambda d: max(int(d * depth_multiplier), min_base_depth) + blocks = [ + resnet_v1_block('block1', base_depth=depth_func(64), num_units=3, + stride=2), + resnet_v1_block('block2', base_depth=depth_func(128), num_units=24, + stride=2), + resnet_v1_block('block3', base_depth=depth_func(256), num_units=36, + stride=2), + resnet_v1_block('block4', base_depth=depth_func(512), num_units=3, + stride=1), + ] + return resnet_v1(inputs, blocks, num_classes, is_training, + global_pool=global_pool, output_stride=output_stride, + include_root_block=True, spatial_squeeze=spatial_squeeze, + store_non_strided_activations=store_non_strided_activations, + reuse=reuse, scope=scope) +resnet_v1_200.default_image_size = resnet_v1.default_image_size diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_v1_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_v1_test.py new file mode 100644 index 0000000..fb70a5a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_v1_test.py @@ -0,0 +1,630 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for slim.nets.resnet_v1.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets import resnet_utils +from nets import resnet_v1 + +slim = contrib_slim + +tf.compat.v1.disable_resource_variables() + + +def create_test_input(batch_size, height, width, channels): + """Create test input tensor. + + Args: + batch_size: The number of images per batch or `None` if unknown. + height: The height of each image or `None` if unknown. + width: The width of each image or `None` if unknown. + channels: The number of channels per image or `None` if unknown. + + Returns: + Either a placeholder `Tensor` of dimension + [batch_size, height, width, channels] if any of the inputs are `None` or a + constant `Tensor` with the mesh grid values along the spatial dimensions. + """ + if None in [batch_size, height, width, channels]: + return tf.compat.v1.placeholder(tf.float32, + (batch_size, height, width, channels)) + else: + return tf.cast( + np.tile( + np.reshape( + np.reshape(np.arange(height), [height, 1]) + + np.reshape(np.arange(width), [1, width]), + [1, height, width, 1]), [batch_size, 1, 1, channels]), + dtype=tf.float32) + + +class ResnetUtilsTest(tf.test.TestCase): + + def testSubsampleThreeByThree(self): + x = tf.reshape(tf.cast(tf.range(9), dtype=tf.float32), [1, 3, 3, 1]) + x = resnet_utils.subsample(x, 2) + expected = tf.reshape(tf.constant([0, 2, 6, 8]), [1, 2, 2, 1]) + with self.test_session(): + self.assertAllClose(x.eval(), expected.eval()) + + def testSubsampleFourByFour(self): + x = tf.reshape(tf.cast(tf.range(16), dtype=tf.float32), [1, 4, 4, 1]) + x = resnet_utils.subsample(x, 2) + expected = tf.reshape(tf.constant([0, 2, 8, 10]), [1, 2, 2, 1]) + with self.test_session(): + self.assertAllClose(x.eval(), expected.eval()) + + def testConv2DSameEven(self): + n, n2 = 4, 2 + + # Input image. + x = create_test_input(1, n, n, 1) + + # Convolution kernel. + w = create_test_input(1, 3, 3, 1) + w = tf.reshape(w, [3, 3, 1, 1]) + + tf.compat.v1.get_variable('Conv/weights', initializer=w) + tf.compat.v1.get_variable('Conv/biases', initializer=tf.zeros([1])) + tf.compat.v1.get_variable_scope().reuse_variables() + + y1 = slim.conv2d(x, 1, [3, 3], stride=1, scope='Conv') + y1_expected = tf.cast([[14, 28, 43, 26], [28, 48, 66, 37], [43, 66, 84, 46], + [26, 37, 46, 22]], + dtype=tf.float32) + y1_expected = tf.reshape(y1_expected, [1, n, n, 1]) + + y2 = resnet_utils.subsample(y1, 2) + y2_expected = tf.cast([[14, 43], [43, 84]], dtype=tf.float32) + y2_expected = tf.reshape(y2_expected, [1, n2, n2, 1]) + + y3 = resnet_utils.conv2d_same(x, 1, 3, stride=2, scope='Conv') + y3_expected = y2_expected + + y4 = slim.conv2d(x, 1, [3, 3], stride=2, scope='Conv') + y4_expected = tf.cast([[48, 37], [37, 22]], dtype=tf.float32) + y4_expected = tf.reshape(y4_expected, [1, n2, n2, 1]) + + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + self.assertAllClose(y1.eval(), y1_expected.eval()) + self.assertAllClose(y2.eval(), y2_expected.eval()) + self.assertAllClose(y3.eval(), y3_expected.eval()) + self.assertAllClose(y4.eval(), y4_expected.eval()) + + def testConv2DSameOdd(self): + n, n2 = 5, 3 + + # Input image. + x = create_test_input(1, n, n, 1) + + # Convolution kernel. + w = create_test_input(1, 3, 3, 1) + w = tf.reshape(w, [3, 3, 1, 1]) + + tf.compat.v1.get_variable('Conv/weights', initializer=w) + tf.compat.v1.get_variable('Conv/biases', initializer=tf.zeros([1])) + tf.compat.v1.get_variable_scope().reuse_variables() + + y1 = slim.conv2d(x, 1, [3, 3], stride=1, scope='Conv') + y1_expected = tf.cast( + [[14, 28, 43, 58, 34], [28, 48, 66, 84, 46], [43, 66, 84, 102, 55], + [58, 84, 102, 120, 64], [34, 46, 55, 64, 30]], + dtype=tf.float32) + y1_expected = tf.reshape(y1_expected, [1, n, n, 1]) + + y2 = resnet_utils.subsample(y1, 2) + y2_expected = tf.cast([[14, 43, 34], [43, 84, 55], [34, 55, 30]], + dtype=tf.float32) + y2_expected = tf.reshape(y2_expected, [1, n2, n2, 1]) + + y3 = resnet_utils.conv2d_same(x, 1, 3, stride=2, scope='Conv') + y3_expected = y2_expected + + y4 = slim.conv2d(x, 1, [3, 3], stride=2, scope='Conv') + y4_expected = y2_expected + + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + self.assertAllClose(y1.eval(), y1_expected.eval()) + self.assertAllClose(y2.eval(), y2_expected.eval()) + self.assertAllClose(y3.eval(), y3_expected.eval()) + self.assertAllClose(y4.eval(), y4_expected.eval()) + + def _resnet_plain(self, inputs, blocks, output_stride=None, scope=None): + """A plain ResNet without extra layers before or after the ResNet blocks.""" + with tf.compat.v1.variable_scope(scope, values=[inputs]): + with slim.arg_scope([slim.conv2d], outputs_collections='end_points'): + net = resnet_utils.stack_blocks_dense(inputs, blocks, output_stride) + end_points = slim.utils.convert_collection_to_dict('end_points') + return net, end_points + + def testEndPointsV1(self): + """Test the end points of a tiny v1 bottleneck network.""" + blocks = [ + resnet_v1.resnet_v1_block( + 'block1', base_depth=1, num_units=2, stride=2), + resnet_v1.resnet_v1_block( + 'block2', base_depth=2, num_units=2, stride=1), + ] + inputs = create_test_input(2, 32, 16, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + _, end_points = self._resnet_plain(inputs, blocks, scope='tiny') + expected = [ + 'tiny/block1/unit_1/bottleneck_v1/shortcut', + 'tiny/block1/unit_1/bottleneck_v1/conv1', + 'tiny/block1/unit_1/bottleneck_v1/conv2', + 'tiny/block1/unit_1/bottleneck_v1/conv3', + 'tiny/block1/unit_2/bottleneck_v1/conv1', + 'tiny/block1/unit_2/bottleneck_v1/conv2', + 'tiny/block1/unit_2/bottleneck_v1/conv3', + 'tiny/block2/unit_1/bottleneck_v1/shortcut', + 'tiny/block2/unit_1/bottleneck_v1/conv1', + 'tiny/block2/unit_1/bottleneck_v1/conv2', + 'tiny/block2/unit_1/bottleneck_v1/conv3', + 'tiny/block2/unit_2/bottleneck_v1/conv1', + 'tiny/block2/unit_2/bottleneck_v1/conv2', + 'tiny/block2/unit_2/bottleneck_v1/conv3'] + self.assertItemsEqual(expected, end_points.keys()) + + def _stack_blocks_nondense(self, net, blocks): + """A simplified ResNet Block stacker without output stride control.""" + for block in blocks: + with tf.compat.v1.variable_scope(block.scope, 'block', [net]): + for i, unit in enumerate(block.args): + with tf.compat.v1.variable_scope('unit_%d' % (i + 1), values=[net]): + net = block.unit_fn(net, rate=1, **unit) + return net + + def testAtrousValuesBottleneck(self): + """Verify the values of dense feature extraction by atrous convolution. + + Make sure that dense feature extraction by stack_blocks_dense() followed by + subsampling gives identical results to feature extraction at the nominal + network output stride using the simple self._stack_blocks_nondense() above. + """ + block = resnet_v1.resnet_v1_block + blocks = [ + block('block1', base_depth=1, num_units=2, stride=2), + block('block2', base_depth=2, num_units=2, stride=2), + block('block3', base_depth=4, num_units=2, stride=2), + block('block4', base_depth=8, num_units=2, stride=1), + ] + nominal_stride = 8 + + # Test both odd and even input dimensions. + height = 30 + width = 31 + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + with slim.arg_scope([slim.batch_norm], is_training=False): + for output_stride in [1, 2, 4, 8, None]: + with tf.Graph().as_default(): + with self.test_session() as sess: + tf.compat.v1.set_random_seed(0) + inputs = create_test_input(1, height, width, 3) + # Dense feature extraction followed by subsampling. + output = resnet_utils.stack_blocks_dense(inputs, + blocks, + output_stride) + if output_stride is None: + factor = 1 + else: + factor = nominal_stride // output_stride + + output = resnet_utils.subsample(output, factor) + # Make the two networks use the same weights. + tf.compat.v1.get_variable_scope().reuse_variables() + # Feature extraction at the nominal network rate. + expected = self._stack_blocks_nondense(inputs, blocks) + sess.run(tf.compat.v1.global_variables_initializer()) + output, expected = sess.run([output, expected]) + self.assertAllClose(output, expected, atol=1e-4, rtol=1e-4) + + def testStridingLastUnitVsSubsampleBlockEnd(self): + """Compares subsampling at the block's last unit or block's end. + + Makes sure that the final output is the same when we use a stride at the + last unit of a block vs. we subsample activations at the end of a block. + """ + block = resnet_v1.resnet_v1_block + + blocks = [ + block('block1', base_depth=1, num_units=2, stride=2), + block('block2', base_depth=2, num_units=2, stride=2), + block('block3', base_depth=4, num_units=2, stride=2), + block('block4', base_depth=8, num_units=2, stride=1), + ] + + # Test both odd and even input dimensions. + height = 30 + width = 31 + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + with slim.arg_scope([slim.batch_norm], is_training=False): + for output_stride in [1, 2, 4, 8, None]: + with tf.Graph().as_default(): + with self.test_session() as sess: + tf.compat.v1.set_random_seed(0) + inputs = create_test_input(1, height, width, 3) + + # Subsampling at the last unit of the block. + output = resnet_utils.stack_blocks_dense( + inputs, blocks, output_stride, + store_non_strided_activations=False, + outputs_collections='output') + output_end_points = slim.utils.convert_collection_to_dict( + 'output') + + # Make the two networks use the same weights. + tf.compat.v1.get_variable_scope().reuse_variables() + + # Subsample activations at the end of the blocks. + expected = resnet_utils.stack_blocks_dense( + inputs, blocks, output_stride, + store_non_strided_activations=True, + outputs_collections='expected') + expected_end_points = slim.utils.convert_collection_to_dict( + 'expected') + + sess.run(tf.compat.v1.global_variables_initializer()) + + # Make sure that the final output is the same. + output, expected = sess.run([output, expected]) + self.assertAllClose(output, expected, atol=1e-4, rtol=1e-4) + + # Make sure that intermediate block activations in + # output_end_points are subsampled versions of the corresponding + # ones in expected_end_points. + for i, block in enumerate(blocks[:-1:]): + output = output_end_points[block.scope] + expected = expected_end_points[block.scope] + atrous_activated = (output_stride is not None and + 2 ** i >= output_stride) + if not atrous_activated: + expected = resnet_utils.subsample(expected, 2) + output, expected = sess.run([output, expected]) + self.assertAllClose(output, expected, atol=1e-4, rtol=1e-4) + + +class ResnetCompleteNetworkTest(tf.test.TestCase): + """Tests with complete small ResNet v1 networks.""" + + def _resnet_small(self, + inputs, + num_classes=None, + is_training=True, + global_pool=True, + output_stride=None, + include_root_block=True, + spatial_squeeze=True, + reuse=None, + scope='resnet_v1_small'): + """A shallow and thin ResNet v1 for faster tests.""" + block = resnet_v1.resnet_v1_block + blocks = [ + block('block1', base_depth=1, num_units=3, stride=2), + block('block2', base_depth=2, num_units=3, stride=2), + block('block3', base_depth=4, num_units=3, stride=2), + block('block4', base_depth=8, num_units=2, stride=1), + ] + return resnet_v1.resnet_v1(inputs, blocks, num_classes, + is_training=is_training, + global_pool=global_pool, + output_stride=output_stride, + include_root_block=include_root_block, + spatial_squeeze=spatial_squeeze, + reuse=reuse, + scope=scope) + + def testClassificationEndPoints(self): + global_pool = True + num_classes = 10 + inputs = create_test_input(2, 224, 224, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + logits, end_points = self._resnet_small(inputs, num_classes, + global_pool=global_pool, + spatial_squeeze=False, + scope='resnet') + self.assertTrue(logits.op.name.startswith('resnet/logits')) + self.assertListEqual(logits.get_shape().as_list(), [2, 1, 1, num_classes]) + self.assertTrue('predictions' in end_points) + self.assertListEqual(end_points['predictions'].get_shape().as_list(), + [2, 1, 1, num_classes]) + self.assertTrue('global_pool' in end_points) + self.assertListEqual(end_points['global_pool'].get_shape().as_list(), + [2, 1, 1, 32]) + + def testClassificationEndPointsWithNoBatchNormArgscope(self): + global_pool = True + num_classes = 10 + inputs = create_test_input(2, 224, 224, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + logits, end_points = self._resnet_small(inputs, num_classes, + global_pool=global_pool, + spatial_squeeze=False, + is_training=None, + scope='resnet') + self.assertTrue(logits.op.name.startswith('resnet/logits')) + self.assertListEqual(logits.get_shape().as_list(), [2, 1, 1, num_classes]) + self.assertTrue('predictions' in end_points) + self.assertListEqual(end_points['predictions'].get_shape().as_list(), + [2, 1, 1, num_classes]) + self.assertTrue('global_pool' in end_points) + self.assertListEqual(end_points['global_pool'].get_shape().as_list(), + [2, 1, 1, 32]) + + def testEndpointNames(self): + # Like ResnetUtilsTest.testEndPointsV1(), but for the public API. + global_pool = True + num_classes = 10 + inputs = create_test_input(2, 224, 224, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + _, end_points = self._resnet_small(inputs, num_classes, + global_pool=global_pool, + scope='resnet') + expected = ['resnet/conv1'] + for block in range(1, 5): + for unit in range(1, 4 if block < 4 else 3): + for conv in range(1, 4): + expected.append('resnet/block%d/unit_%d/bottleneck_v1/conv%d' % + (block, unit, conv)) + expected.append('resnet/block%d/unit_%d/bottleneck_v1' % (block, unit)) + expected.append('resnet/block%d/unit_1/bottleneck_v1/shortcut' % block) + expected.append('resnet/block%d' % block) + expected.extend(['global_pool', 'resnet/logits', 'resnet/spatial_squeeze', + 'predictions']) + self.assertItemsEqual(end_points.keys(), expected) + + def testClassificationShapes(self): + global_pool = True + num_classes = 10 + inputs = create_test_input(2, 224, 224, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + _, end_points = self._resnet_small(inputs, num_classes, + global_pool=global_pool, + scope='resnet') + endpoint_to_shape = { + 'resnet/block1': [2, 28, 28, 4], + 'resnet/block2': [2, 14, 14, 8], + 'resnet/block3': [2, 7, 7, 16], + 'resnet/block4': [2, 7, 7, 32]} + for endpoint in endpoint_to_shape: + shape = endpoint_to_shape[endpoint] + self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) + + def testFullyConvolutionalEndpointShapes(self): + global_pool = False + num_classes = 10 + inputs = create_test_input(2, 321, 321, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + _, end_points = self._resnet_small(inputs, num_classes, + global_pool=global_pool, + spatial_squeeze=False, + scope='resnet') + endpoint_to_shape = { + 'resnet/block1': [2, 41, 41, 4], + 'resnet/block2': [2, 21, 21, 8], + 'resnet/block3': [2, 11, 11, 16], + 'resnet/block4': [2, 11, 11, 32]} + for endpoint in endpoint_to_shape: + shape = endpoint_to_shape[endpoint] + self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) + + def testRootlessFullyConvolutionalEndpointShapes(self): + global_pool = False + num_classes = 10 + inputs = create_test_input(2, 128, 128, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + _, end_points = self._resnet_small(inputs, num_classes, + global_pool=global_pool, + include_root_block=False, + spatial_squeeze=False, + scope='resnet') + endpoint_to_shape = { + 'resnet/block1': [2, 64, 64, 4], + 'resnet/block2': [2, 32, 32, 8], + 'resnet/block3': [2, 16, 16, 16], + 'resnet/block4': [2, 16, 16, 32]} + for endpoint in endpoint_to_shape: + shape = endpoint_to_shape[endpoint] + self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) + + def testAtrousFullyConvolutionalEndpointShapes(self): + global_pool = False + num_classes = 10 + output_stride = 8 + inputs = create_test_input(2, 321, 321, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + _, end_points = self._resnet_small(inputs, + num_classes, + global_pool=global_pool, + output_stride=output_stride, + spatial_squeeze=False, + scope='resnet') + endpoint_to_shape = { + 'resnet/block1': [2, 41, 41, 4], + 'resnet/block2': [2, 41, 41, 8], + 'resnet/block3': [2, 41, 41, 16], + 'resnet/block4': [2, 41, 41, 32]} + for endpoint in endpoint_to_shape: + shape = endpoint_to_shape[endpoint] + self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) + + def testAtrousFullyConvolutionalValues(self): + """Verify dense feature extraction with atrous convolution.""" + nominal_stride = 32 + for output_stride in [4, 8, 16, 32, None]: + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + with tf.Graph().as_default(): + with self.test_session() as sess: + tf.compat.v1.set_random_seed(0) + inputs = create_test_input(2, 81, 81, 3) + # Dense feature extraction followed by subsampling. + output, _ = self._resnet_small(inputs, None, is_training=False, + global_pool=False, + output_stride=output_stride) + if output_stride is None: + factor = 1 + else: + factor = nominal_stride // output_stride + output = resnet_utils.subsample(output, factor) + # Make the two networks use the same weights. + tf.compat.v1.get_variable_scope().reuse_variables() + # Feature extraction at the nominal network rate. + expected, _ = self._resnet_small(inputs, None, is_training=False, + global_pool=False) + sess.run(tf.compat.v1.global_variables_initializer()) + self.assertAllClose(output.eval(), expected.eval(), + atol=1e-4, rtol=1e-4) + + def testUnknownBatchSize(self): + batch = 2 + height, width = 65, 65 + global_pool = True + num_classes = 10 + inputs = create_test_input(None, height, width, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + logits, _ = self._resnet_small(inputs, num_classes, + global_pool=global_pool, + spatial_squeeze=False, + scope='resnet') + self.assertTrue(logits.op.name.startswith('resnet/logits')) + self.assertListEqual(logits.get_shape().as_list(), + [None, 1, 1, num_classes]) + images = create_test_input(batch, height, width, 3) + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(logits, {inputs: images.eval()}) + self.assertEqual(output.shape, (batch, 1, 1, num_classes)) + + def testFullyConvolutionalUnknownHeightWidth(self): + batch = 2 + height, width = 65, 65 + global_pool = False + inputs = create_test_input(batch, None, None, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + output, _ = self._resnet_small(inputs, None, global_pool=global_pool) + self.assertListEqual(output.get_shape().as_list(), + [batch, None, None, 32]) + images = create_test_input(batch, height, width, 3) + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(output, {inputs: images.eval()}) + self.assertEqual(output.shape, (batch, 3, 3, 32)) + + def testAtrousFullyConvolutionalUnknownHeightWidth(self): + batch = 2 + height, width = 65, 65 + global_pool = False + output_stride = 8 + inputs = create_test_input(batch, None, None, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + output, _ = self._resnet_small(inputs, + None, + global_pool=global_pool, + output_stride=output_stride) + self.assertListEqual(output.get_shape().as_list(), + [batch, None, None, 32]) + images = create_test_input(batch, height, width, 3) + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(output, {inputs: images.eval()}) + self.assertEqual(output.shape, (batch, 9, 9, 32)) + + def testDepthMultiplier(self): + resnets = [ + resnet_v1.resnet_v1_50, resnet_v1.resnet_v1_101, + resnet_v1.resnet_v1_152, resnet_v1.resnet_v1_200 + ] + resnet_names = [ + 'resnet_v1_50', 'resnet_v1_101', 'resnet_v1_152', 'resnet_v1_200' + ] + for resnet, resnet_name in zip(resnets, resnet_names): + depth_multiplier = 0.25 + global_pool = True + num_classes = 10 + inputs = create_test_input(2, 224, 224, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + scope_base = resnet_name + '_base' + _, end_points_base = resnet( + inputs, + num_classes, + global_pool=global_pool, + min_base_depth=1, + scope=scope_base) + scope_test = resnet_name + '_test' + _, end_points_test = resnet( + inputs, + num_classes, + global_pool=global_pool, + min_base_depth=1, + depth_multiplier=depth_multiplier, + scope=scope_test) + for block in ['block1', 'block2', 'block3', 'block4']: + block_name_base = scope_base + '/' + block + block_name_test = scope_test + '/' + block + self.assertTrue(block_name_base in end_points_base) + self.assertTrue(block_name_test in end_points_test) + self.assertEqual( + len(end_points_base[block_name_base].get_shape().as_list()), 4) + self.assertEqual( + len(end_points_test[block_name_test].get_shape().as_list()), 4) + self.assertListEqual( + end_points_base[block_name_base].get_shape().as_list()[:3], + end_points_test[block_name_test].get_shape().as_list()[:3]) + self.assertEqual( + int(depth_multiplier * + end_points_base[block_name_base].get_shape().as_list()[3]), + end_points_test[block_name_test].get_shape().as_list()[3]) + + def testMinBaseDepth(self): + resnets = [ + resnet_v1.resnet_v1_50, resnet_v1.resnet_v1_101, + resnet_v1.resnet_v1_152, resnet_v1.resnet_v1_200 + ] + resnet_names = [ + 'resnet_v1_50', 'resnet_v1_101', 'resnet_v1_152', 'resnet_v1_200' + ] + for resnet, resnet_name in zip(resnets, resnet_names): + min_base_depth = 5 + global_pool = True + num_classes = 10 + inputs = create_test_input(2, 224, 224, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + _, end_points = resnet( + inputs, + num_classes, + global_pool=global_pool, + min_base_depth=min_base_depth, + depth_multiplier=0, + scope=resnet_name) + for block in ['block1', 'block2', 'block3', 'block4']: + block_name = resnet_name + '/' + block + self.assertTrue(block_name in end_points) + self.assertEqual( + len(end_points[block_name].get_shape().as_list()), 4) + # The output depth is 4 times base_depth. + depth_expected = min_base_depth * 4 + self.assertEqual( + end_points[block_name].get_shape().as_list()[3], depth_expected) + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_v2.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_v2.py new file mode 100644 index 0000000..b08af07 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_v2.py @@ -0,0 +1,340 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains definitions for the preactivation form of Residual Networks. + +Residual networks (ResNets) were originally proposed in: +[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Deep Residual Learning for Image Recognition. arXiv:1512.03385 + +The full preactivation 'v2' ResNet variant implemented in this module was +introduced by: +[2] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Identity Mappings in Deep Residual Networks. arXiv: 1603.05027 + +The key difference of the full preactivation 'v2' variant compared to the +'v1' variant in [1] is the use of batch normalization before every weight layer. + +Typical use: + + from tensorflow.contrib.slim.nets import resnet_v2 + +ResNet-101 for image classification into 1000 classes: + + # inputs has shape [batch, 224, 224, 3] + with slim.arg_scope(resnet_v2.resnet_arg_scope()): + net, end_points = resnet_v2.resnet_v2_101(inputs, 1000, is_training=False) + +ResNet-101 for semantic segmentation into 21 classes: + + # inputs has shape [batch, 513, 513, 3] + with slim.arg_scope(resnet_v2.resnet_arg_scope()): + net, end_points = resnet_v2.resnet_v2_101(inputs, + 21, + is_training=False, + global_pool=False, + output_stride=16) +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets import resnet_utils + +slim = contrib_slim +resnet_arg_scope = resnet_utils.resnet_arg_scope + + +@slim.add_arg_scope +def bottleneck(inputs, depth, depth_bottleneck, stride, rate=1, + outputs_collections=None, scope=None): + """Bottleneck residual unit variant with BN before convolutions. + + This is the full preactivation residual unit variant proposed in [2]. See + Fig. 1(b) of [2] for its definition. Note that we use here the bottleneck + variant which has an extra bottleneck layer. + + When putting together two consecutive ResNet blocks that use this unit, one + should use stride = 2 in the last unit of the first block. + + Args: + inputs: A tensor of size [batch, height, width, channels]. + depth: The depth of the ResNet unit output. + depth_bottleneck: The depth of the bottleneck layers. + stride: The ResNet unit's stride. Determines the amount of downsampling of + the units output compared to its input. + rate: An integer, rate for atrous convolution. + outputs_collections: Collection to add the ResNet unit output. + scope: Optional variable_scope. + + Returns: + The ResNet unit's output. + """ + with tf.compat.v1.variable_scope(scope, 'bottleneck_v2', [inputs]) as sc: + depth_in = slim.utils.last_dimension(inputs.get_shape(), min_rank=4) + preact = slim.batch_norm(inputs, activation_fn=tf.nn.relu, scope='preact') + if depth == depth_in: + shortcut = resnet_utils.subsample(inputs, stride, 'shortcut') + else: + shortcut = slim.conv2d(preact, depth, [1, 1], stride=stride, + normalizer_fn=None, activation_fn=None, + scope='shortcut') + + residual = slim.conv2d(preact, depth_bottleneck, [1, 1], stride=1, + scope='conv1') + residual = resnet_utils.conv2d_same(residual, depth_bottleneck, 3, stride, + rate=rate, scope='conv2') + residual = slim.conv2d(residual, depth, [1, 1], stride=1, + normalizer_fn=None, activation_fn=None, + scope='conv3') + + output = shortcut + residual + + return slim.utils.collect_named_outputs(outputs_collections, + sc.name, + output) + + +def resnet_v2(inputs, + blocks, + num_classes=None, + is_training=True, + global_pool=True, + output_stride=None, + include_root_block=True, + spatial_squeeze=True, + reuse=None, + scope=None): + """Generator for v2 (preactivation) ResNet models. + + This function generates a family of ResNet v2 models. See the resnet_v2_*() + methods for specific model instantiations, obtained by selecting different + block instantiations that produce ResNets of various depths. + + Training for image classification on Imagenet is usually done with [224, 224] + inputs, resulting in [7, 7] feature maps at the output of the last ResNet + block for the ResNets defined in [1] that have nominal stride equal to 32. + However, for dense prediction tasks we advise that one uses inputs with + spatial dimensions that are multiples of 32 plus 1, e.g., [321, 321]. In + this case the feature maps at the ResNet output will have spatial shape + [(height - 1) / output_stride + 1, (width - 1) / output_stride + 1] + and corners exactly aligned with the input image corners, which greatly + facilitates alignment of the features to the image. Using as input [225, 225] + images results in [8, 8] feature maps at the output of the last ResNet block. + + For dense prediction tasks, the ResNet needs to run in fully-convolutional + (FCN) mode and global_pool needs to be set to False. The ResNets in [1, 2] all + have nominal stride equal to 32 and a good choice in FCN mode is to use + output_stride=16 in order to increase the density of the computed features at + small computational and memory overhead, cf. http://arxiv.org/abs/1606.00915. + + Args: + inputs: A tensor of size [batch, height_in, width_in, channels]. + blocks: A list of length equal to the number of ResNet blocks. Each element + is a resnet_utils.Block object describing the units in the block. + num_classes: Number of predicted classes for classification tasks. + If 0 or None, we return the features before the logit layer. + is_training: whether batch_norm layers are in training mode. + global_pool: If True, we perform global average pooling before computing the + logits. Set to True for image classification, False for dense prediction. + output_stride: If None, then the output will be computed at the nominal + network stride. If output_stride is not None, it specifies the requested + ratio of input to output spatial resolution. + include_root_block: If True, include the initial convolution followed by + max-pooling, if False excludes it. If excluded, `inputs` should be the + results of an activation-less convolution. + spatial_squeeze: if True, logits is of shape [B, C], if false logits is + of shape [B, 1, 1, C], where B is batch_size and C is number of classes. + To use this parameter, the input images must be smaller than 300x300 + pixels, in which case the output logit layer does not contain spatial + information and can be removed. + reuse: whether or not the network and its variables should be reused. To be + able to reuse 'scope' must be given. + scope: Optional variable_scope. + + + Returns: + net: A rank-4 tensor of size [batch, height_out, width_out, channels_out]. + If global_pool is False, then height_out and width_out are reduced by a + factor of output_stride compared to the respective height_in and width_in, + else both height_out and width_out equal one. If num_classes is 0 or None, + then net is the output of the last ResNet block, potentially after global + average pooling. If num_classes is a non-zero integer, net contains the + pre-softmax activations. + end_points: A dictionary from components of the network to the corresponding + activation. + + Raises: + ValueError: If the target output_stride is not valid. + """ + with tf.compat.v1.variable_scope( + scope, 'resnet_v2', [inputs], reuse=reuse) as sc: + end_points_collection = sc.original_name_scope + '_end_points' + with slim.arg_scope([slim.conv2d, bottleneck, + resnet_utils.stack_blocks_dense], + outputs_collections=end_points_collection): + with slim.arg_scope([slim.batch_norm], is_training=is_training): + net = inputs + if include_root_block: + if output_stride is not None: + if output_stride % 4 != 0: + raise ValueError('The output_stride needs to be a multiple of 4.') + output_stride /= 4 + # We do not include batch normalization or activation functions in + # conv1 because the first ResNet unit will perform these. Cf. + # Appendix of [2]. + with slim.arg_scope([slim.conv2d], + activation_fn=None, normalizer_fn=None): + net = resnet_utils.conv2d_same(net, 64, 7, stride=2, scope='conv1') + net = slim.max_pool2d(net, [3, 3], stride=2, scope='pool1') + net = resnet_utils.stack_blocks_dense(net, blocks, output_stride) + # This is needed because the pre-activation variant does not have batch + # normalization or activation functions in the residual unit output. See + # Appendix of [2]. + net = slim.batch_norm(net, activation_fn=tf.nn.relu, scope='postnorm') + # Convert end_points_collection into a dictionary of end_points. + end_points = slim.utils.convert_collection_to_dict( + end_points_collection) + + if global_pool: + # Global average pooling. + net = tf.reduce_mean( + input_tensor=net, axis=[1, 2], name='pool5', keepdims=True) + end_points['global_pool'] = net + if num_classes: + net = slim.conv2d(net, num_classes, [1, 1], activation_fn=None, + normalizer_fn=None, scope='logits') + end_points[sc.name + '/logits'] = net + if spatial_squeeze: + net = tf.squeeze(net, [1, 2], name='SpatialSqueeze') + end_points[sc.name + '/spatial_squeeze'] = net + end_points['predictions'] = slim.softmax(net, scope='predictions') + return net, end_points +resnet_v2.default_image_size = 224 + + +def resnet_v2_block(scope, base_depth, num_units, stride): + """Helper function for creating a resnet_v2 bottleneck block. + + Args: + scope: The scope of the block. + base_depth: The depth of the bottleneck layer for each unit. + num_units: The number of units in the block. + stride: The stride of the block, implemented as a stride in the last unit. + All other units have stride=1. + + Returns: + A resnet_v2 bottleneck block. + """ + return resnet_utils.Block(scope, bottleneck, [{ + 'depth': base_depth * 4, + 'depth_bottleneck': base_depth, + 'stride': 1 + }] * (num_units - 1) + [{ + 'depth': base_depth * 4, + 'depth_bottleneck': base_depth, + 'stride': stride + }]) +resnet_v2.default_image_size = 224 + + +def resnet_v2_50(inputs, + num_classes=None, + is_training=True, + global_pool=True, + output_stride=None, + spatial_squeeze=True, + reuse=None, + scope='resnet_v2_50'): + """ResNet-50 model of [1]. See resnet_v2() for arg and return description.""" + blocks = [ + resnet_v2_block('block1', base_depth=64, num_units=3, stride=2), + resnet_v2_block('block2', base_depth=128, num_units=4, stride=2), + resnet_v2_block('block3', base_depth=256, num_units=6, stride=2), + resnet_v2_block('block4', base_depth=512, num_units=3, stride=1), + ] + return resnet_v2(inputs, blocks, num_classes, is_training=is_training, + global_pool=global_pool, output_stride=output_stride, + include_root_block=True, spatial_squeeze=spatial_squeeze, + reuse=reuse, scope=scope) +resnet_v2_50.default_image_size = resnet_v2.default_image_size + + +def resnet_v2_101(inputs, + num_classes=None, + is_training=True, + global_pool=True, + output_stride=None, + spatial_squeeze=True, + reuse=None, + scope='resnet_v2_101'): + """ResNet-101 model of [1]. See resnet_v2() for arg and return description.""" + blocks = [ + resnet_v2_block('block1', base_depth=64, num_units=3, stride=2), + resnet_v2_block('block2', base_depth=128, num_units=4, stride=2), + resnet_v2_block('block3', base_depth=256, num_units=23, stride=2), + resnet_v2_block('block4', base_depth=512, num_units=3, stride=1), + ] + return resnet_v2(inputs, blocks, num_classes, is_training=is_training, + global_pool=global_pool, output_stride=output_stride, + include_root_block=True, spatial_squeeze=spatial_squeeze, + reuse=reuse, scope=scope) +resnet_v2_101.default_image_size = resnet_v2.default_image_size + + +def resnet_v2_152(inputs, + num_classes=None, + is_training=True, + global_pool=True, + output_stride=None, + spatial_squeeze=True, + reuse=None, + scope='resnet_v2_152'): + """ResNet-152 model of [1]. See resnet_v2() for arg and return description.""" + blocks = [ + resnet_v2_block('block1', base_depth=64, num_units=3, stride=2), + resnet_v2_block('block2', base_depth=128, num_units=8, stride=2), + resnet_v2_block('block3', base_depth=256, num_units=36, stride=2), + resnet_v2_block('block4', base_depth=512, num_units=3, stride=1), + ] + return resnet_v2(inputs, blocks, num_classes, is_training=is_training, + global_pool=global_pool, output_stride=output_stride, + include_root_block=True, spatial_squeeze=spatial_squeeze, + reuse=reuse, scope=scope) +resnet_v2_152.default_image_size = resnet_v2.default_image_size + + +def resnet_v2_200(inputs, + num_classes=None, + is_training=True, + global_pool=True, + output_stride=None, + spatial_squeeze=True, + reuse=None, + scope='resnet_v2_200'): + """ResNet-200 model of [2]. See resnet_v2() for arg and return description.""" + blocks = [ + resnet_v2_block('block1', base_depth=64, num_units=3, stride=2), + resnet_v2_block('block2', base_depth=128, num_units=24, stride=2), + resnet_v2_block('block3', base_depth=256, num_units=36, stride=2), + resnet_v2_block('block4', base_depth=512, num_units=3, stride=1), + ] + return resnet_v2(inputs, blocks, num_classes, is_training=is_training, + global_pool=global_pool, output_stride=output_stride, + include_root_block=True, spatial_squeeze=spatial_squeeze, + reuse=reuse, scope=scope) +resnet_v2_200.default_image_size = resnet_v2.default_image_size diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_v2_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_v2_test.py new file mode 100644 index 0000000..5ca5227 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/resnet_v2_test.py @@ -0,0 +1,474 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for slim.nets.resnet_v2.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets import resnet_utils +from nets import resnet_v2 + +slim = contrib_slim + +tf.compat.v1.disable_resource_variables() + + +def create_test_input(batch_size, height, width, channels): + """Create test input tensor. + + Args: + batch_size: The number of images per batch or `None` if unknown. + height: The height of each image or `None` if unknown. + width: The width of each image or `None` if unknown. + channels: The number of channels per image or `None` if unknown. + + Returns: + Either a placeholder `Tensor` of dimension + [batch_size, height, width, channels] if any of the inputs are `None` or a + constant `Tensor` with the mesh grid values along the spatial dimensions. + """ + if None in [batch_size, height, width, channels]: + return tf.compat.v1.placeholder(tf.float32, + (batch_size, height, width, channels)) + else: + return tf.cast( + np.tile( + np.reshape( + np.reshape(np.arange(height), [height, 1]) + + np.reshape(np.arange(width), [1, width]), + [1, height, width, 1]), [batch_size, 1, 1, channels]), + dtype=tf.float32) + + +class ResnetUtilsTest(tf.test.TestCase): + + def testSubsampleThreeByThree(self): + x = tf.reshape(tf.cast(tf.range(9), dtype=tf.float32), [1, 3, 3, 1]) + x = resnet_utils.subsample(x, 2) + expected = tf.reshape(tf.constant([0, 2, 6, 8]), [1, 2, 2, 1]) + with self.test_session(): + self.assertAllClose(x.eval(), expected.eval()) + + def testSubsampleFourByFour(self): + x = tf.reshape(tf.cast(tf.range(16), dtype=tf.float32), [1, 4, 4, 1]) + x = resnet_utils.subsample(x, 2) + expected = tf.reshape(tf.constant([0, 2, 8, 10]), [1, 2, 2, 1]) + with self.test_session(): + self.assertAllClose(x.eval(), expected.eval()) + + def testConv2DSameEven(self): + n, n2 = 4, 2 + + # Input image. + x = create_test_input(1, n, n, 1) + + # Convolution kernel. + w = create_test_input(1, 3, 3, 1) + w = tf.reshape(w, [3, 3, 1, 1]) + + tf.compat.v1.get_variable('Conv/weights', initializer=w) + tf.compat.v1.get_variable('Conv/biases', initializer=tf.zeros([1])) + tf.compat.v1.get_variable_scope().reuse_variables() + + y1 = slim.conv2d(x, 1, [3, 3], stride=1, scope='Conv') + y1_expected = tf.cast([[14, 28, 43, 26], [28, 48, 66, 37], [43, 66, 84, 46], + [26, 37, 46, 22]], + dtype=tf.float32) + y1_expected = tf.reshape(y1_expected, [1, n, n, 1]) + + y2 = resnet_utils.subsample(y1, 2) + y2_expected = tf.cast([[14, 43], [43, 84]], dtype=tf.float32) + y2_expected = tf.reshape(y2_expected, [1, n2, n2, 1]) + + y3 = resnet_utils.conv2d_same(x, 1, 3, stride=2, scope='Conv') + y3_expected = y2_expected + + y4 = slim.conv2d(x, 1, [3, 3], stride=2, scope='Conv') + y4_expected = tf.cast([[48, 37], [37, 22]], dtype=tf.float32) + y4_expected = tf.reshape(y4_expected, [1, n2, n2, 1]) + + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + self.assertAllClose(y1.eval(), y1_expected.eval()) + self.assertAllClose(y2.eval(), y2_expected.eval()) + self.assertAllClose(y3.eval(), y3_expected.eval()) + self.assertAllClose(y4.eval(), y4_expected.eval()) + + def testConv2DSameOdd(self): + n, n2 = 5, 3 + + # Input image. + x = create_test_input(1, n, n, 1) + + # Convolution kernel. + w = create_test_input(1, 3, 3, 1) + w = tf.reshape(w, [3, 3, 1, 1]) + + tf.compat.v1.get_variable('Conv/weights', initializer=w) + tf.compat.v1.get_variable('Conv/biases', initializer=tf.zeros([1])) + tf.compat.v1.get_variable_scope().reuse_variables() + + y1 = slim.conv2d(x, 1, [3, 3], stride=1, scope='Conv') + y1_expected = tf.cast( + [[14, 28, 43, 58, 34], [28, 48, 66, 84, 46], [43, 66, 84, 102, 55], + [58, 84, 102, 120, 64], [34, 46, 55, 64, 30]], + dtype=tf.float32) + y1_expected = tf.reshape(y1_expected, [1, n, n, 1]) + + y2 = resnet_utils.subsample(y1, 2) + y2_expected = tf.cast([[14, 43, 34], [43, 84, 55], [34, 55, 30]], + dtype=tf.float32) + y2_expected = tf.reshape(y2_expected, [1, n2, n2, 1]) + + y3 = resnet_utils.conv2d_same(x, 1, 3, stride=2, scope='Conv') + y3_expected = y2_expected + + y4 = slim.conv2d(x, 1, [3, 3], stride=2, scope='Conv') + y4_expected = y2_expected + + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + self.assertAllClose(y1.eval(), y1_expected.eval()) + self.assertAllClose(y2.eval(), y2_expected.eval()) + self.assertAllClose(y3.eval(), y3_expected.eval()) + self.assertAllClose(y4.eval(), y4_expected.eval()) + + def _resnet_plain(self, inputs, blocks, output_stride=None, scope=None): + """A plain ResNet without extra layers before or after the ResNet blocks.""" + with tf.compat.v1.variable_scope(scope, values=[inputs]): + with slim.arg_scope([slim.conv2d], outputs_collections='end_points'): + net = resnet_utils.stack_blocks_dense(inputs, blocks, output_stride) + end_points = slim.utils.convert_collection_to_dict('end_points') + return net, end_points + + def testEndPointsV2(self): + """Test the end points of a tiny v2 bottleneck network.""" + blocks = [ + resnet_v2.resnet_v2_block( + 'block1', base_depth=1, num_units=2, stride=2), + resnet_v2.resnet_v2_block( + 'block2', base_depth=2, num_units=2, stride=1), + ] + inputs = create_test_input(2, 32, 16, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + _, end_points = self._resnet_plain(inputs, blocks, scope='tiny') + expected = [ + 'tiny/block1/unit_1/bottleneck_v2/shortcut', + 'tiny/block1/unit_1/bottleneck_v2/conv1', + 'tiny/block1/unit_1/bottleneck_v2/conv2', + 'tiny/block1/unit_1/bottleneck_v2/conv3', + 'tiny/block1/unit_2/bottleneck_v2/conv1', + 'tiny/block1/unit_2/bottleneck_v2/conv2', + 'tiny/block1/unit_2/bottleneck_v2/conv3', + 'tiny/block2/unit_1/bottleneck_v2/shortcut', + 'tiny/block2/unit_1/bottleneck_v2/conv1', + 'tiny/block2/unit_1/bottleneck_v2/conv2', + 'tiny/block2/unit_1/bottleneck_v2/conv3', + 'tiny/block2/unit_2/bottleneck_v2/conv1', + 'tiny/block2/unit_2/bottleneck_v2/conv2', + 'tiny/block2/unit_2/bottleneck_v2/conv3'] + self.assertItemsEqual(expected, end_points.keys()) + + def _stack_blocks_nondense(self, net, blocks): + """A simplified ResNet Block stacker without output stride control.""" + for block in blocks: + with tf.compat.v1.variable_scope(block.scope, 'block', [net]): + for i, unit in enumerate(block.args): + with tf.compat.v1.variable_scope('unit_%d' % (i + 1), values=[net]): + net = block.unit_fn(net, rate=1, **unit) + return net + + def testAtrousValuesBottleneck(self): + """Verify the values of dense feature extraction by atrous convolution. + + Make sure that dense feature extraction by stack_blocks_dense() followed by + subsampling gives identical results to feature extraction at the nominal + network output stride using the simple self._stack_blocks_nondense() above. + """ + block = resnet_v2.resnet_v2_block + blocks = [ + block('block1', base_depth=1, num_units=2, stride=2), + block('block2', base_depth=2, num_units=2, stride=2), + block('block3', base_depth=4, num_units=2, stride=2), + block('block4', base_depth=8, num_units=2, stride=1), + ] + nominal_stride = 8 + + # Test both odd and even input dimensions. + height = 30 + width = 31 + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + with slim.arg_scope([slim.batch_norm], is_training=False): + for output_stride in [1, 2, 4, 8, None]: + with tf.Graph().as_default(): + with self.test_session() as sess: + tf.compat.v1.set_random_seed(0) + inputs = create_test_input(1, height, width, 3) + # Dense feature extraction followed by subsampling. + output = resnet_utils.stack_blocks_dense(inputs, + blocks, + output_stride) + if output_stride is None: + factor = 1 + else: + factor = nominal_stride // output_stride + + output = resnet_utils.subsample(output, factor) + # Make the two networks use the same weights. + tf.compat.v1.get_variable_scope().reuse_variables() + # Feature extraction at the nominal network rate. + expected = self._stack_blocks_nondense(inputs, blocks) + sess.run(tf.compat.v1.global_variables_initializer()) + output, expected = sess.run([output, expected]) + self.assertAllClose(output, expected, atol=1e-4, rtol=1e-4) + + +class ResnetCompleteNetworkTest(tf.test.TestCase): + """Tests with complete small ResNet v2 networks.""" + + def _resnet_small(self, + inputs, + num_classes=None, + is_training=True, + global_pool=True, + output_stride=None, + include_root_block=True, + spatial_squeeze=True, + reuse=None, + scope='resnet_v2_small'): + """A shallow and thin ResNet v2 for faster tests.""" + block = resnet_v2.resnet_v2_block + blocks = [ + block('block1', base_depth=1, num_units=3, stride=2), + block('block2', base_depth=2, num_units=3, stride=2), + block('block3', base_depth=4, num_units=3, stride=2), + block('block4', base_depth=8, num_units=2, stride=1), + ] + return resnet_v2.resnet_v2(inputs, blocks, num_classes, + is_training=is_training, + global_pool=global_pool, + output_stride=output_stride, + include_root_block=include_root_block, + spatial_squeeze=spatial_squeeze, + reuse=reuse, + scope=scope) + + def testClassificationEndPoints(self): + global_pool = True + num_classes = 10 + inputs = create_test_input(2, 224, 224, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + logits, end_points = self._resnet_small(inputs, num_classes, + global_pool=global_pool, + spatial_squeeze=False, + scope='resnet') + self.assertTrue(logits.op.name.startswith('resnet/logits')) + self.assertListEqual(logits.get_shape().as_list(), [2, 1, 1, num_classes]) + self.assertTrue('predictions' in end_points) + self.assertListEqual(end_points['predictions'].get_shape().as_list(), + [2, 1, 1, num_classes]) + self.assertTrue('global_pool' in end_points) + self.assertListEqual(end_points['global_pool'].get_shape().as_list(), + [2, 1, 1, 32]) + + def testEndpointNames(self): + # Like ResnetUtilsTest.testEndPointsV2(), but for the public API. + global_pool = True + num_classes = 10 + inputs = create_test_input(2, 224, 224, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + _, end_points = self._resnet_small(inputs, num_classes, + global_pool=global_pool, + scope='resnet') + expected = ['resnet/conv1'] + for block in range(1, 5): + for unit in range(1, 4 if block < 4 else 3): + for conv in range(1, 4): + expected.append('resnet/block%d/unit_%d/bottleneck_v2/conv%d' % + (block, unit, conv)) + expected.append('resnet/block%d/unit_%d/bottleneck_v2' % (block, unit)) + expected.append('resnet/block%d/unit_1/bottleneck_v2/shortcut' % block) + expected.append('resnet/block%d' % block) + expected.extend(['global_pool', 'resnet/logits', 'resnet/spatial_squeeze', + 'predictions']) + self.assertItemsEqual(end_points.keys(), expected) + + def testClassificationShapes(self): + global_pool = True + num_classes = 10 + inputs = create_test_input(2, 224, 224, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + _, end_points = self._resnet_small(inputs, num_classes, + global_pool=global_pool, + scope='resnet') + endpoint_to_shape = { + 'resnet/block1': [2, 28, 28, 4], + 'resnet/block2': [2, 14, 14, 8], + 'resnet/block3': [2, 7, 7, 16], + 'resnet/block4': [2, 7, 7, 32]} + for endpoint in endpoint_to_shape: + shape = endpoint_to_shape[endpoint] + self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) + + def testFullyConvolutionalEndpointShapes(self): + global_pool = False + num_classes = 10 + inputs = create_test_input(2, 321, 321, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + _, end_points = self._resnet_small(inputs, num_classes, + global_pool=global_pool, + spatial_squeeze=False, + scope='resnet') + endpoint_to_shape = { + 'resnet/block1': [2, 41, 41, 4], + 'resnet/block2': [2, 21, 21, 8], + 'resnet/block3': [2, 11, 11, 16], + 'resnet/block4': [2, 11, 11, 32]} + for endpoint in endpoint_to_shape: + shape = endpoint_to_shape[endpoint] + self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) + + def testRootlessFullyConvolutionalEndpointShapes(self): + global_pool = False + num_classes = 10 + inputs = create_test_input(2, 128, 128, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + _, end_points = self._resnet_small(inputs, num_classes, + global_pool=global_pool, + include_root_block=False, + spatial_squeeze=False, + scope='resnet') + endpoint_to_shape = { + 'resnet/block1': [2, 64, 64, 4], + 'resnet/block2': [2, 32, 32, 8], + 'resnet/block3': [2, 16, 16, 16], + 'resnet/block4': [2, 16, 16, 32]} + for endpoint in endpoint_to_shape: + shape = endpoint_to_shape[endpoint] + self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) + + def testAtrousFullyConvolutionalEndpointShapes(self): + global_pool = False + num_classes = 10 + output_stride = 8 + inputs = create_test_input(2, 321, 321, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + _, end_points = self._resnet_small(inputs, + num_classes, + global_pool=global_pool, + output_stride=output_stride, + spatial_squeeze=False, + scope='resnet') + endpoint_to_shape = { + 'resnet/block1': [2, 41, 41, 4], + 'resnet/block2': [2, 41, 41, 8], + 'resnet/block3': [2, 41, 41, 16], + 'resnet/block4': [2, 41, 41, 32]} + for endpoint in endpoint_to_shape: + shape = endpoint_to_shape[endpoint] + self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) + + def testAtrousFullyConvolutionalValues(self): + """Verify dense feature extraction with atrous convolution.""" + nominal_stride = 32 + for output_stride in [4, 8, 16, 32, None]: + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + with tf.Graph().as_default(): + with self.test_session() as sess: + tf.compat.v1.set_random_seed(0) + inputs = create_test_input(2, 81, 81, 3) + # Dense feature extraction followed by subsampling. + output, _ = self._resnet_small(inputs, None, + is_training=False, + global_pool=False, + output_stride=output_stride) + if output_stride is None: + factor = 1 + else: + factor = nominal_stride // output_stride + output = resnet_utils.subsample(output, factor) + # Make the two networks use the same weights. + tf.compat.v1.get_variable_scope().reuse_variables() + # Feature extraction at the nominal network rate. + expected, _ = self._resnet_small(inputs, None, + is_training=False, + global_pool=False) + sess.run(tf.compat.v1.global_variables_initializer()) + self.assertAllClose(output.eval(), expected.eval(), + atol=1e-4, rtol=1e-4) + + def testUnknownBatchSize(self): + batch = 2 + height, width = 65, 65 + global_pool = True + num_classes = 10 + inputs = create_test_input(None, height, width, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + logits, _ = self._resnet_small(inputs, num_classes, + global_pool=global_pool, + spatial_squeeze=False, + scope='resnet') + self.assertTrue(logits.op.name.startswith('resnet/logits')) + self.assertListEqual(logits.get_shape().as_list(), + [None, 1, 1, num_classes]) + images = create_test_input(batch, height, width, 3) + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(logits, {inputs: images.eval()}) + self.assertEqual(output.shape, (batch, 1, 1, num_classes)) + + def testFullyConvolutionalUnknownHeightWidth(self): + batch = 2 + height, width = 65, 65 + global_pool = False + inputs = create_test_input(batch, None, None, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + output, _ = self._resnet_small(inputs, None, + global_pool=global_pool) + self.assertListEqual(output.get_shape().as_list(), + [batch, None, None, 32]) + images = create_test_input(batch, height, width, 3) + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(output, {inputs: images.eval()}) + self.assertEqual(output.shape, (batch, 3, 3, 32)) + + def testAtrousFullyConvolutionalUnknownHeightWidth(self): + batch = 2 + height, width = 65, 65 + global_pool = False + output_stride = 8 + inputs = create_test_input(batch, None, None, 3) + with slim.arg_scope(resnet_utils.resnet_arg_scope()): + output, _ = self._resnet_small(inputs, + None, + global_pool=global_pool, + output_stride=output_stride) + self.assertListEqual(output.get_shape().as_list(), + [batch, None, None, 32]) + images = create_test_input(batch, height, width, 3) + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(output, {inputs: images.eval()}) + self.assertEqual(output.shape, (batch, 9, 9, 32)) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/s3dg.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/s3dg.py new file mode 100644 index 0000000..3f443ad --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/s3dg.py @@ -0,0 +1,603 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains the definition for Gated Separable 3D network (S3D-G). + +The network architecture is proposed by: + Saining Xie, Chen Sun, Jonathan Huang, Zhuowen Tu and Kevin Murphy, + Rethinking Spatiotemporal Feature Learning For Video Understanding. + https://arxiv.org/abs/1712.04851. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import framework as contrib_framework +from tensorflow.contrib import layers as contrib_layers + +from nets import i3d_utils + +# pylint: disable=g-long-lambda +trunc_normal = lambda stddev: tf.compat.v1.truncated_normal_initializer( + 0.0, stddev) +conv3d_spatiotemporal = i3d_utils.conv3d_spatiotemporal +inception_block_v1_3d = i3d_utils.inception_block_v1_3d + +# Orignaly, arg_scope = slim.arg_scope and layers = slim, now switch to more +# update-to-date tf.contrib.* API. +arg_scope = contrib_framework.arg_scope +layers = contrib_layers + + +def s3dg_arg_scope(weight_decay=1e-7, + batch_norm_decay=0.999, + batch_norm_epsilon=0.001): + """Defines default arg_scope for S3D-G. + + Args: + weight_decay: The weight decay to use for regularizing the model. + batch_norm_decay: Decay for batch norm moving average. + batch_norm_epsilon: Small float added to variance to avoid dividing by zero + in batch norm. + + Returns: + sc: An arg_scope to use for the models. + """ + batch_norm_params = { + # Decay for the moving averages. + 'decay': batch_norm_decay, + # epsilon to prevent 0s in variance. + 'epsilon': batch_norm_epsilon, + # Turns off fused batch norm. + 'fused': False, + # collection containing the moving mean and moving variance. + 'variables_collections': { + 'beta': None, + 'gamma': None, + 'moving_mean': ['moving_vars'], + 'moving_variance': ['moving_vars'], + } + } + + with arg_scope( + [layers.conv3d, conv3d_spatiotemporal], + weights_regularizer=layers.l2_regularizer(weight_decay), + activation_fn=tf.nn.relu, + normalizer_fn=layers.batch_norm, + normalizer_params=batch_norm_params): + with arg_scope([conv3d_spatiotemporal], separable=True) as sc: + return sc + + +def self_gating(input_tensor, scope, data_format='NDHWC'): + """Feature gating as used in S3D-G. + + Transforms the input features by aggregating features from all + spatial and temporal locations, and applying gating conditioned + on the aggregated features. More details can be found at: + https://arxiv.org/abs/1712.04851 + + Args: + input_tensor: A 5-D float tensor of size [batch_size, num_frames, + height, width, channels]. + scope: scope for `variable_scope`. + data_format: An optional string from: "NDHWC", "NCDHW". Defaults to "NDHWC". + The data format of the input and output data. With the default format + "NDHWC", the data is stored in the order of: [batch, in_depth, in_height, + in_width, in_channels]. Alternatively, the format could be "NCDHW", the + data storage order is: + [batch, in_channels, in_depth, in_height, in_width]. + + Returns: + A tensor with the same shape as input_tensor. + """ + + index_c = data_format.index('C') + index_d = data_format.index('D') + index_h = data_format.index('H') + index_w = data_format.index('W') + input_shape = input_tensor.get_shape().as_list() + t = input_shape[index_d] + w = input_shape[index_w] + h = input_shape[index_h] + num_channels = input_shape[index_c] + + spatiotemporal_average = layers.avg_pool3d( + input_tensor, [t, w, h], + stride=1, + data_format=data_format, + scope=scope + '/self_gating/avg_pool3d') + + weights = layers.conv3d( + spatiotemporal_average, + num_channels, [1, 1, 1], + activation_fn=None, + normalizer_fn=None, + biases_initializer=None, + data_format=data_format, + weights_initializer=trunc_normal(0.01), + scope=scope + '/self_gating/transformer_W') + + tile_multiples = [1, t, w, h] + tile_multiples.insert(index_c, 1) + weights = tf.tile(weights, tile_multiples) + weights = tf.nn.sigmoid(weights) + + return tf.multiply(weights, input_tensor) + + +def s3dg_base(inputs, + first_temporal_kernel_size=3, + temporal_conv_startat='Conv2d_2c_3x3', + gating_startat='Conv2d_2c_3x3', + final_endpoint='Mixed_5c', + min_depth=16, + depth_multiplier=1.0, + data_format='NDHWC', + scope='InceptionV1'): + """Defines the I3D/S3DG base architecture. + + Note that we use the names as defined in Inception V1 to facilitate checkpoint + conversion from an image-trained Inception V1 checkpoint to I3D checkpoint. + + Args: + inputs: A 5-D float tensor of size [batch_size, num_frames, height, width, + channels]. + first_temporal_kernel_size: Specifies the temporal kernel size for the first + conv3d filter. A larger value slows down the model but provides little + accuracy improvement. The default is 7 in the original I3D and S3D-G but 3 + gives better performance. Must be set to one of 1, 3, 5 or 7. + temporal_conv_startat: Specifies the first conv block to use 3D or separable + 3D convs rather than 2D convs (implemented as [1, k, k] 3D conv). This is + used to construct the inverted pyramid models. 'Conv2d_2c_3x3' is the + first valid block to use separable 3D convs. If provided block name is + not present, all valid blocks will use separable 3D convs. Note that + 'Conv2d_1a_7x7' cannot be made into a separable 3D conv, but can be made + into a 2D or 3D conv using the `first_temporal_kernel_size` option. + gating_startat: Specifies the first conv block to use self gating. + 'Conv2d_2c_3x3' is the first valid block to use self gating. If provided + block name is not present, all valid blocks will use separable 3D convs. + final_endpoint: Specifies the endpoint to construct the network up to. It + can be one of ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', + 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', + 'MaxPool_4a_3x3', 'Mixed_4b', 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', + 'Mixed_4f', 'MaxPool_5a_2x2', 'Mixed_5b', 'Mixed_5c'] + min_depth: Minimum depth value (number of channels) for all convolution ops. + Enforced when depth_multiplier < 1, and not an active constraint when + depth_multiplier >= 1. + depth_multiplier: Float multiplier for the depth (number of channels) + for all convolution ops. The value must be greater than zero. Typical + usage will be to set this value in (0, 1) to reduce the number of + parameters or computation cost of the model. + data_format: An optional string from: "NDHWC", "NCDHW". Defaults to "NDHWC". + The data format of the input and output data. With the default format + "NDHWC", the data is stored in the order of: [batch, in_depth, in_height, + in_width, in_channels]. Alternatively, the format could be "NCDHW", the + data storage order is: + [batch, in_channels, in_depth, in_height, in_width]. + scope: Optional variable_scope. + + Returns: + A dictionary from components of the network to the corresponding activation. + + Raises: + ValueError: if final_endpoint is not set to one of the predefined values, or + if depth_multiplier <= 0. + """ + + assert data_format in ['NDHWC', 'NCDHW'] + end_points = {} + t = 1 + # For inverted pyramid models, we start with gating switched off. + use_gating = False + self_gating_fn = None + def gating_fn(inputs, scope): + return self_gating(inputs, scope, data_format=data_format) + + if depth_multiplier <= 0: + raise ValueError('depth_multiplier is not greater than zero.') + depth = lambda d: max(int(d * depth_multiplier), min_depth) + + with tf.compat.v1.variable_scope(scope, 'InceptionV1', [inputs]): + with arg_scope([layers.conv3d], weights_initializer=trunc_normal(0.01)): + with arg_scope( + [layers.conv3d, layers.max_pool3d, conv3d_spatiotemporal], + stride=1, + data_format=data_format, + padding='SAME'): + # batch_size x 32 x 112 x 112 x 64 + end_point = 'Conv2d_1a_7x7' + if first_temporal_kernel_size not in [1, 3, 5, 7]: + raise ValueError( + 'first_temporal_kernel_size can only be 1, 3, 5 or 7.') + # Separable conv is slow when used at first conv layer. + net = conv3d_spatiotemporal( + inputs, + depth(64), [first_temporal_kernel_size, 7, 7], + stride=2, + separable=False, + scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + # batch_size x 32 x 56 x 56 x 64 + end_point = 'MaxPool_2a_3x3' + net = layers.max_pool3d( + net, [1, 3, 3], stride=[1, 2, 2], scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + # batch_size x 32 x 56 x 56 x 64 + end_point = 'Conv2d_2b_1x1' + net = layers.conv3d(net, depth(64), [1, 1, 1], scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + # batch_size x 32 x 56 x 56 x 192 + end_point = 'Conv2d_2c_3x3' + if temporal_conv_startat == end_point: + t = 3 + if gating_startat == end_point: + use_gating = True + self_gating_fn = gating_fn + net = conv3d_spatiotemporal(net, depth(192), [t, 3, 3], scope=end_point) + if use_gating: + net = self_gating(net, scope=end_point, data_format=data_format) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + # batch_size x 32 x 28 x 28 x 192 + end_point = 'MaxPool_3a_3x3' + net = layers.max_pool3d( + net, [1, 3, 3], stride=[1, 2, 2], scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + + # batch_size x 32 x 28 x 28 x 256 + end_point = 'Mixed_3b' + if temporal_conv_startat == end_point: + t = 3 + if gating_startat == end_point: + use_gating = True + self_gating_fn = gating_fn + net = inception_block_v1_3d( + net, + num_outputs_0_0a=depth(64), + num_outputs_1_0a=depth(96), + num_outputs_1_0b=depth(128), + num_outputs_2_0a=depth(16), + num_outputs_2_0b=depth(32), + num_outputs_3_0b=depth(32), + temporal_kernel_size=t, + self_gating_fn=self_gating_fn, + data_format=data_format, + scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + + end_point = 'Mixed_3c' + if temporal_conv_startat == end_point: + t = 3 + if gating_startat == end_point: + use_gating = True + self_gating_fn = gating_fn + net = inception_block_v1_3d( + net, + num_outputs_0_0a=depth(128), + num_outputs_1_0a=depth(128), + num_outputs_1_0b=depth(192), + num_outputs_2_0a=depth(32), + num_outputs_2_0b=depth(96), + num_outputs_3_0b=depth(64), + temporal_kernel_size=t, + self_gating_fn=self_gating_fn, + data_format=data_format, + scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + + end_point = 'MaxPool_4a_3x3' + net = layers.max_pool3d( + net, [3, 3, 3], stride=[2, 2, 2], scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + + # batch_size x 16 x 14 x 14 x 512 + end_point = 'Mixed_4b' + if temporal_conv_startat == end_point: + t = 3 + if gating_startat == end_point: + use_gating = True + self_gating_fn = gating_fn + net = inception_block_v1_3d( + net, + num_outputs_0_0a=depth(192), + num_outputs_1_0a=depth(96), + num_outputs_1_0b=depth(208), + num_outputs_2_0a=depth(16), + num_outputs_2_0b=depth(48), + num_outputs_3_0b=depth(64), + temporal_kernel_size=t, + self_gating_fn=self_gating_fn, + data_format=data_format, + scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + + # batch_size x 16 x 14 x 14 x 512 + end_point = 'Mixed_4c' + if temporal_conv_startat == end_point: + t = 3 + if gating_startat == end_point: + use_gating = True + self_gating_fn = gating_fn + net = inception_block_v1_3d( + net, + num_outputs_0_0a=depth(160), + num_outputs_1_0a=depth(112), + num_outputs_1_0b=depth(224), + num_outputs_2_0a=depth(24), + num_outputs_2_0b=depth(64), + num_outputs_3_0b=depth(64), + temporal_kernel_size=t, + self_gating_fn=self_gating_fn, + data_format=data_format, + scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + + # batch_size x 16 x 14 x 14 x 512 + end_point = 'Mixed_4d' + if temporal_conv_startat == end_point: + t = 3 + if gating_startat == end_point: + use_gating = True + self_gating_fn = gating_fn + net = inception_block_v1_3d( + net, + num_outputs_0_0a=depth(128), + num_outputs_1_0a=depth(128), + num_outputs_1_0b=depth(256), + num_outputs_2_0a=depth(24), + num_outputs_2_0b=depth(64), + num_outputs_3_0b=depth(64), + temporal_kernel_size=t, + self_gating_fn=self_gating_fn, + data_format=data_format, + scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + + # batch_size x 16 x 14 x 14 x 528 + end_point = 'Mixed_4e' + if temporal_conv_startat == end_point: + t = 3 + if gating_startat == end_point: + use_gating = True + self_gating_fn = gating_fn + net = inception_block_v1_3d( + net, + num_outputs_0_0a=depth(112), + num_outputs_1_0a=depth(144), + num_outputs_1_0b=depth(288), + num_outputs_2_0a=depth(32), + num_outputs_2_0b=depth(64), + num_outputs_3_0b=depth(64), + temporal_kernel_size=t, + self_gating_fn=self_gating_fn, + data_format=data_format, + scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + + # batch_size x 16 x 14 x 14 x 832 + end_point = 'Mixed_4f' + if temporal_conv_startat == end_point: + t = 3 + if gating_startat == end_point: + use_gating = True + self_gating_fn = gating_fn + net = inception_block_v1_3d( + net, + num_outputs_0_0a=depth(256), + num_outputs_1_0a=depth(160), + num_outputs_1_0b=depth(320), + num_outputs_2_0a=depth(32), + num_outputs_2_0b=depth(128), + num_outputs_3_0b=depth(128), + temporal_kernel_size=t, + self_gating_fn=self_gating_fn, + data_format=data_format, + scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + + end_point = 'MaxPool_5a_2x2' + net = layers.max_pool3d( + net, [2, 2, 2], stride=[2, 2, 2], scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + + # batch_size x 8 x 7 x 7 x 832 + end_point = 'Mixed_5b' + if temporal_conv_startat == end_point: + t = 3 + if gating_startat == end_point: + use_gating = True + self_gating_fn = gating_fn + net = inception_block_v1_3d( + net, + num_outputs_0_0a=depth(256), + num_outputs_1_0a=depth(160), + num_outputs_1_0b=depth(320), + num_outputs_2_0a=depth(32), + num_outputs_2_0b=depth(128), + num_outputs_3_0b=depth(128), + temporal_kernel_size=t, + self_gating_fn=self_gating_fn, + data_format=data_format, + scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + + # batch_size x 8 x 7 x 7 x 1024 + end_point = 'Mixed_5c' + if temporal_conv_startat == end_point: + t = 3 + if gating_startat == end_point: + use_gating = True + self_gating_fn = gating_fn + net = inception_block_v1_3d( + net, + num_outputs_0_0a=depth(384), + num_outputs_1_0a=depth(192), + num_outputs_1_0b=depth(384), + num_outputs_2_0a=depth(48), + num_outputs_2_0b=depth(128), + num_outputs_3_0b=depth(128), + temporal_kernel_size=t, + self_gating_fn=self_gating_fn, + data_format=data_format, + scope=end_point) + end_points[end_point] = net + if final_endpoint == end_point: + return net, end_points + raise ValueError('Unknown final endpoint %s' % final_endpoint) + + +def s3dg(inputs, + num_classes=1000, + first_temporal_kernel_size=3, + temporal_conv_startat='Conv2d_2c_3x3', + gating_startat='Conv2d_2c_3x3', + final_endpoint='Mixed_5c', + min_depth=16, + depth_multiplier=1.0, + dropout_keep_prob=0.8, + is_training=True, + prediction_fn=layers.softmax, + spatial_squeeze=True, + reuse=None, + data_format='NDHWC', + scope='InceptionV1'): + """Defines the S3D-G architecture. + + The default image size used to train this network is 224x224. + + Args: + inputs: A 5-D float tensor of size [batch_size, num_frames, height, width, + channels]. + num_classes: number of predicted classes. + first_temporal_kernel_size: Specifies the temporal kernel size for the first + conv3d filter. A larger value slows down the model but provides little + accuracy improvement. Must be set to one of 1, 3, 5 or 7. + temporal_conv_startat: Specifies the first conv block to use separable 3D + convs rather than 2D convs (implemented as [1, k, k] 3D conv). This is + used to construct the inverted pyramid models. 'Conv2d_2c_3x3' is the + first valid block to use separable 3D convs. If provided block name is + not present, all valid blocks will use separable 3D convs. + gating_startat: Specifies the first conv block to use self gating. + 'Conv2d_2c_3x3' is the first valid block to use self gating. If provided + block name is not present, all valid blocks will use separable 3D convs. + final_endpoint: Specifies the endpoint to construct the network up to. It + can be one of ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', + 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', + 'MaxPool_4a_3x3', 'Mixed_4b', 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', + 'Mixed_4f', 'MaxPool_5a_2x2', 'Mixed_5b', 'Mixed_5c'] + min_depth: Minimum depth value (number of channels) for all convolution ops. + Enforced when depth_multiplier < 1, and not an active constraint when + depth_multiplier >= 1. + depth_multiplier: Float multiplier for the depth (number of channels) + for all convolution ops. The value must be greater than zero. Typical + usage will be to set this value in (0, 1) to reduce the number of + parameters or computation cost of the model. + dropout_keep_prob: the percentage of activation values that are retained. + is_training: whether is training or not. + prediction_fn: a function to get predictions out of logits. + spatial_squeeze: if True, logits is of shape is [B, C], if false logits is + of shape [B, 1, 1, C], where B is batch_size and C is number of classes. + reuse: whether or not the network and its variables should be reused. To be + able to reuse 'scope' must be given. + data_format: An optional string from: "NDHWC", "NCDHW". Defaults to "NDHWC". + The data format of the input and output data. With the default format + "NDHWC", the data is stored in the order of: [batch, in_depth, in_height, + in_width, in_channels]. Alternatively, the format could be "NCDHW", the + data storage order is: + [batch, in_channels, in_depth, in_height, in_width]. + scope: Optional variable_scope. + + Returns: + logits: the pre-softmax activations, a tensor of size + [batch_size, num_classes] + end_points: a dictionary from components of the network to the corresponding + activation. + """ + assert data_format in ['NDHWC', 'NCDHW'] + # Final pooling and prediction + with tf.compat.v1.variable_scope( + scope, 'InceptionV1', [inputs, num_classes], reuse=reuse) as scope: + with arg_scope( + [layers.batch_norm, layers.dropout], is_training=is_training): + net, end_points = s3dg_base( + inputs, + first_temporal_kernel_size=first_temporal_kernel_size, + temporal_conv_startat=temporal_conv_startat, + gating_startat=gating_startat, + final_endpoint=final_endpoint, + min_depth=min_depth, + depth_multiplier=depth_multiplier, + data_format=data_format, + scope=scope) + with tf.compat.v1.variable_scope('Logits'): + if data_format.startswith('NC'): + net = tf.transpose(a=net, perm=[0, 2, 3, 4, 1]) + kernel_size = i3d_utils.reduced_kernel_size_3d(net, [2, 7, 7]) + net = layers.avg_pool3d( + net, + kernel_size, + stride=1, + data_format='NDHWC', + scope='AvgPool_0a_7x7') + net = layers.dropout(net, dropout_keep_prob, scope='Dropout_0b') + logits = layers.conv3d( + net, + num_classes, [1, 1, 1], + activation_fn=None, + normalizer_fn=None, + data_format='NDHWC', + scope='Conv2d_0c_1x1') + # Temporal average pooling. + logits = tf.reduce_mean(input_tensor=logits, axis=1) + if spatial_squeeze: + logits = tf.squeeze(logits, [1, 2], name='SpatialSqueeze') + + end_points['Logits'] = logits + end_points['Predictions'] = prediction_fn(logits, scope='Predictions') + return logits, end_points + + +s3dg.default_image_size = 224 diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/s3dg_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/s3dg_test.py new file mode 100644 index 0000000..c3dc57c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/s3dg_test.py @@ -0,0 +1,150 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for networks.s3dg.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from nets import s3dg + + +class S3DGTest(tf.test.TestCase): + + def testBuildClassificationNetwork(self): + batch_size = 5 + num_frames = 64 + height, width = 224, 224 + num_classes = 1000 + + inputs = tf.random.uniform((batch_size, num_frames, height, width, 3)) + logits, end_points = s3dg.s3dg(inputs, num_classes) + self.assertTrue(logits.op.name.startswith('InceptionV1/Logits')) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + self.assertTrue('Predictions' in end_points) + self.assertListEqual(end_points['Predictions'].get_shape().as_list(), + [batch_size, num_classes]) + + def testBuildBaseNetwork(self): + batch_size = 5 + num_frames = 64 + height, width = 224, 224 + + inputs = tf.random.uniform((batch_size, num_frames, height, width, 3)) + mixed_6c, end_points = s3dg.s3dg_base(inputs) + self.assertTrue(mixed_6c.op.name.startswith('InceptionV1/Mixed_5c')) + self.assertListEqual(mixed_6c.get_shape().as_list(), + [batch_size, 8, 7, 7, 1024]) + expected_endpoints = ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', + 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', + 'Mixed_3c', 'MaxPool_4a_3x3', 'Mixed_4b', 'Mixed_4c', + 'Mixed_4d', 'Mixed_4e', 'Mixed_4f', 'MaxPool_5a_2x2', + 'Mixed_5b', 'Mixed_5c'] + self.assertItemsEqual(end_points.keys(), expected_endpoints) + + def testBuildOnlyUptoFinalEndpointNoGating(self): + batch_size = 5 + num_frames = 64 + height, width = 224, 224 + endpoints = ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', + 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', + 'MaxPool_4a_3x3', 'Mixed_4b', 'Mixed_4c', 'Mixed_4d', + 'Mixed_4e', 'Mixed_4f', 'MaxPool_5a_2x2', 'Mixed_5b', + 'Mixed_5c'] + for index, endpoint in enumerate(endpoints): + with tf.Graph().as_default(): + inputs = tf.random.uniform((batch_size, num_frames, height, width, 3)) + out_tensor, end_points = s3dg.s3dg_base( + inputs, final_endpoint=endpoint, gating_startat=None) + print(endpoint, out_tensor.op.name) + self.assertTrue(out_tensor.op.name.startswith( + 'InceptionV1/' + endpoint)) + self.assertItemsEqual(endpoints[:index+1], end_points) + + def testBuildAndCheckAllEndPointsUptoMixed5c(self): + batch_size = 5 + num_frames = 64 + height, width = 224, 224 + + inputs = tf.random.uniform((batch_size, num_frames, height, width, 3)) + _, end_points = s3dg.s3dg_base(inputs, + final_endpoint='Mixed_5c') + endpoints_shapes = {'Conv2d_1a_7x7': [5, 32, 112, 112, 64], + 'MaxPool_2a_3x3': [5, 32, 56, 56, 64], + 'Conv2d_2b_1x1': [5, 32, 56, 56, 64], + 'Conv2d_2c_3x3': [5, 32, 56, 56, 192], + 'MaxPool_3a_3x3': [5, 32, 28, 28, 192], + 'Mixed_3b': [5, 32, 28, 28, 256], + 'Mixed_3c': [5, 32, 28, 28, 480], + 'MaxPool_4a_3x3': [5, 16, 14, 14, 480], + 'Mixed_4b': [5, 16, 14, 14, 512], + 'Mixed_4c': [5, 16, 14, 14, 512], + 'Mixed_4d': [5, 16, 14, 14, 512], + 'Mixed_4e': [5, 16, 14, 14, 528], + 'Mixed_4f': [5, 16, 14, 14, 832], + 'MaxPool_5a_2x2': [5, 8, 7, 7, 832], + 'Mixed_5b': [5, 8, 7, 7, 832], + 'Mixed_5c': [5, 8, 7, 7, 1024]} + + self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) + for endpoint_name, expected_shape in endpoints_shapes.iteritems(): + self.assertTrue(endpoint_name in end_points) + self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), + expected_shape) + + def testHalfSizeImages(self): + batch_size = 5 + num_frames = 64 + height, width = 112, 112 + + inputs = tf.random.uniform((batch_size, num_frames, height, width, 3)) + mixed_5c, _ = s3dg.s3dg_base(inputs) + self.assertTrue(mixed_5c.op.name.startswith('InceptionV1/Mixed_5c')) + self.assertListEqual(mixed_5c.get_shape().as_list(), + [batch_size, 8, 4, 4, 1024]) + + def testTenFrames(self): + batch_size = 5 + num_frames = 10 + height, width = 224, 224 + + inputs = tf.random.uniform((batch_size, num_frames, height, width, 3)) + mixed_5c, _ = s3dg.s3dg_base(inputs) + self.assertTrue(mixed_5c.op.name.startswith('InceptionV1/Mixed_5c')) + self.assertListEqual(mixed_5c.get_shape().as_list(), + [batch_size, 2, 7, 7, 1024]) + + def testEvaluation(self): + batch_size = 2 + num_frames = 64 + height, width = 224, 224 + num_classes = 1000 + + eval_inputs = tf.random.uniform((batch_size, num_frames, height, width, 3)) + logits, _ = s3dg.s3dg(eval_inputs, num_classes, + is_training=False) + predictions = tf.argmax(input=logits, axis=1) + + with self.test_session() as sess: + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(predictions) + self.assertEquals(output.shape, (batch_size,)) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/vgg.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/vgg.py new file mode 100644 index 0000000..2f30a74 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/vgg.py @@ -0,0 +1,317 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains model definitions for versions of the Oxford VGG network. + +These model definitions were introduced in the following technical report: + + Very Deep Convolutional Networks For Large-Scale Image Recognition + Karen Simonyan and Andrew Zisserman + arXiv technical report, 2015 + PDF: http://arxiv.org/pdf/1409.1556.pdf + ILSVRC 2014 Slides: http://www.robots.ox.ac.uk/~karen/pdf/ILSVRC_2014.pdf + CC-BY-4.0 + +More information can be obtained from the VGG website: +www.robots.ox.ac.uk/~vgg/research/very_deep/ + +Usage: + with slim.arg_scope(vgg.vgg_arg_scope()): + outputs, end_points = vgg.vgg_a(inputs) + + with slim.arg_scope(vgg.vgg_arg_scope()): + outputs, end_points = vgg.vgg_16(inputs) + +@@vgg_a +@@vgg_16 +@@vgg_19 +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +slim = contrib_slim + + +def vgg_arg_scope(weight_decay=0.0005): + """Defines the VGG arg scope. + + Args: + weight_decay: The l2 regularization coefficient. + + Returns: + An arg_scope. + """ + with slim.arg_scope([slim.conv2d, slim.fully_connected], + activation_fn=tf.nn.relu, + weights_regularizer=slim.l2_regularizer(weight_decay), + biases_initializer=tf.compat.v1.zeros_initializer()): + with slim.arg_scope([slim.conv2d], padding='SAME') as arg_sc: + return arg_sc + + +def vgg_a(inputs, + num_classes=1000, + is_training=True, + dropout_keep_prob=0.5, + spatial_squeeze=True, + reuse=None, + scope='vgg_a', + fc_conv_padding='VALID', + global_pool=False): + """Oxford Net VGG 11-Layers version A Example. + + Note: All the fully_connected layers have been transformed to conv2d layers. + To use in classification mode, resize input to 224x224. + + Args: + inputs: a tensor of size [batch_size, height, width, channels]. + num_classes: number of predicted classes. If 0 or None, the logits layer is + omitted and the input features to the logits layer are returned instead. + is_training: whether or not the model is being trained. + dropout_keep_prob: the probability that activations are kept in the dropout + layers during training. + spatial_squeeze: whether or not should squeeze the spatial dimensions of the + outputs. Useful to remove unnecessary dimensions for classification. + reuse: whether or not the network and its variables should be reused. To be + able to reuse 'scope' must be given. + scope: Optional scope for the variables. + fc_conv_padding: the type of padding to use for the fully connected layer + that is implemented as a convolutional layer. Use 'SAME' padding if you + are applying the network in a fully convolutional manner and want to + get a prediction map downsampled by a factor of 32 as an output. + Otherwise, the output prediction map will be (input / 32) - 6 in case of + 'VALID' padding. + global_pool: Optional boolean flag. If True, the input to the classification + layer is avgpooled to size 1x1, for any input size. (This is not part + of the original VGG architecture.) + + Returns: + net: the output of the logits layer (if num_classes is a non-zero integer), + or the input to the logits layer (if num_classes is 0 or None). + end_points: a dict of tensors with intermediate activations. + """ + with tf.compat.v1.variable_scope(scope, 'vgg_a', [inputs], reuse=reuse) as sc: + end_points_collection = sc.original_name_scope + '_end_points' + # Collect outputs for conv2d, fully_connected and max_pool2d. + with slim.arg_scope([slim.conv2d, slim.max_pool2d], + outputs_collections=end_points_collection): + net = slim.repeat(inputs, 1, slim.conv2d, 64, [3, 3], scope='conv1') + net = slim.max_pool2d(net, [2, 2], scope='pool1') + net = slim.repeat(net, 1, slim.conv2d, 128, [3, 3], scope='conv2') + net = slim.max_pool2d(net, [2, 2], scope='pool2') + net = slim.repeat(net, 2, slim.conv2d, 256, [3, 3], scope='conv3') + net = slim.max_pool2d(net, [2, 2], scope='pool3') + net = slim.repeat(net, 2, slim.conv2d, 512, [3, 3], scope='conv4') + net = slim.max_pool2d(net, [2, 2], scope='pool4') + net = slim.repeat(net, 2, slim.conv2d, 512, [3, 3], scope='conv5') + net = slim.max_pool2d(net, [2, 2], scope='pool5') + + # Use conv2d instead of fully_connected layers. + net = slim.conv2d(net, 4096, [7, 7], padding=fc_conv_padding, scope='fc6') + net = slim.dropout(net, dropout_keep_prob, is_training=is_training, + scope='dropout6') + net = slim.conv2d(net, 4096, [1, 1], scope='fc7') + # Convert end_points_collection into a end_point dict. + end_points = slim.utils.convert_collection_to_dict(end_points_collection) + if global_pool: + net = tf.reduce_mean( + input_tensor=net, axis=[1, 2], keepdims=True, name='global_pool') + end_points['global_pool'] = net + if num_classes: + net = slim.dropout(net, dropout_keep_prob, is_training=is_training, + scope='dropout7') + net = slim.conv2d(net, num_classes, [1, 1], + activation_fn=None, + normalizer_fn=None, + scope='fc8') + if spatial_squeeze: + net = tf.squeeze(net, [1, 2], name='fc8/squeezed') + end_points[sc.name + '/fc8'] = net + return net, end_points +vgg_a.default_image_size = 224 + + +def vgg_16(inputs, + num_classes=1000, + is_training=True, + dropout_keep_prob=0.5, + spatial_squeeze=True, + reuse=None, + scope='vgg_16', + fc_conv_padding='VALID', + global_pool=False): + """Oxford Net VGG 16-Layers version D Example. + + Note: All the fully_connected layers have been transformed to conv2d layers. + To use in classification mode, resize input to 224x224. + + Args: + inputs: a tensor of size [batch_size, height, width, channels]. + num_classes: number of predicted classes. If 0 or None, the logits layer is + omitted and the input features to the logits layer are returned instead. + is_training: whether or not the model is being trained. + dropout_keep_prob: the probability that activations are kept in the dropout + layers during training. + spatial_squeeze: whether or not should squeeze the spatial dimensions of the + outputs. Useful to remove unnecessary dimensions for classification. + reuse: whether or not the network and its variables should be reused. To be + able to reuse 'scope' must be given. + scope: Optional scope for the variables. + fc_conv_padding: the type of padding to use for the fully connected layer + that is implemented as a convolutional layer. Use 'SAME' padding if you + are applying the network in a fully convolutional manner and want to + get a prediction map downsampled by a factor of 32 as an output. + Otherwise, the output prediction map will be (input / 32) - 6 in case of + 'VALID' padding. + global_pool: Optional boolean flag. If True, the input to the classification + layer is avgpooled to size 1x1, for any input size. (This is not part + of the original VGG architecture.) + + Returns: + net: the output of the logits layer (if num_classes is a non-zero integer), + or the input to the logits layer (if num_classes is 0 or None). + end_points: a dict of tensors with intermediate activations. + """ + with tf.compat.v1.variable_scope( + scope, 'vgg_16', [inputs], reuse=reuse) as sc: + end_points_collection = sc.original_name_scope + '_end_points' + # Collect outputs for conv2d, fully_connected and max_pool2d. + with slim.arg_scope([slim.conv2d, slim.fully_connected, slim.max_pool2d], + outputs_collections=end_points_collection): + net = slim.repeat(inputs, 2, slim.conv2d, 64, [3, 3], scope='conv1') + net = slim.max_pool2d(net, [2, 2], scope='pool1') + net = slim.repeat(net, 2, slim.conv2d, 128, [3, 3], scope='conv2') + net = slim.max_pool2d(net, [2, 2], scope='pool2') + net = slim.repeat(net, 3, slim.conv2d, 256, [3, 3], scope='conv3') + net = slim.max_pool2d(net, [2, 2], scope='pool3') + net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv4') + net = slim.max_pool2d(net, [2, 2], scope='pool4') + net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv5') + net = slim.max_pool2d(net, [2, 2], scope='pool5') + + # Use conv2d instead of fully_connected layers. + net = slim.conv2d(net, 4096, [7, 7], padding=fc_conv_padding, scope='fc6') + net = slim.dropout(net, dropout_keep_prob, is_training=is_training, + scope='dropout6') + net = slim.conv2d(net, 4096, [1, 1], scope='fc7') + # Convert end_points_collection into a end_point dict. + end_points = slim.utils.convert_collection_to_dict(end_points_collection) + if global_pool: + net = tf.reduce_mean( + input_tensor=net, axis=[1, 2], keepdims=True, name='global_pool') + end_points['global_pool'] = net + if num_classes: + net = slim.dropout(net, dropout_keep_prob, is_training=is_training, + scope='dropout7') + net = slim.conv2d(net, num_classes, [1, 1], + activation_fn=None, + normalizer_fn=None, + scope='fc8') + if spatial_squeeze: + net = tf.squeeze(net, [1, 2], name='fc8/squeezed') + end_points[sc.name + '/fc8'] = net + return net, end_points +vgg_16.default_image_size = 224 + + +def vgg_19(inputs, + num_classes=1000, + is_training=True, + dropout_keep_prob=0.5, + spatial_squeeze=True, + reuse=None, + scope='vgg_19', + fc_conv_padding='VALID', + global_pool=False): + """Oxford Net VGG 19-Layers version E Example. + + Note: All the fully_connected layers have been transformed to conv2d layers. + To use in classification mode, resize input to 224x224. + + Args: + inputs: a tensor of size [batch_size, height, width, channels]. + num_classes: number of predicted classes. If 0 or None, the logits layer is + omitted and the input features to the logits layer are returned instead. + is_training: whether or not the model is being trained. + dropout_keep_prob: the probability that activations are kept in the dropout + layers during training. + spatial_squeeze: whether or not should squeeze the spatial dimensions of the + outputs. Useful to remove unnecessary dimensions for classification. + reuse: whether or not the network and its variables should be reused. To be + able to reuse 'scope' must be given. + scope: Optional scope for the variables. + fc_conv_padding: the type of padding to use for the fully connected layer + that is implemented as a convolutional layer. Use 'SAME' padding if you + are applying the network in a fully convolutional manner and want to + get a prediction map downsampled by a factor of 32 as an output. + Otherwise, the output prediction map will be (input / 32) - 6 in case of + 'VALID' padding. + global_pool: Optional boolean flag. If True, the input to the classification + layer is avgpooled to size 1x1, for any input size. (This is not part + of the original VGG architecture.) + + Returns: + net: the output of the logits layer (if num_classes is a non-zero integer), + or the non-dropped-out input to the logits layer (if num_classes is 0 or + None). + end_points: a dict of tensors with intermediate activations. + """ + with tf.compat.v1.variable_scope( + scope, 'vgg_19', [inputs], reuse=reuse) as sc: + end_points_collection = sc.original_name_scope + '_end_points' + # Collect outputs for conv2d, fully_connected and max_pool2d. + with slim.arg_scope([slim.conv2d, slim.fully_connected, slim.max_pool2d], + outputs_collections=end_points_collection): + net = slim.repeat(inputs, 2, slim.conv2d, 64, [3, 3], scope='conv1') + net = slim.max_pool2d(net, [2, 2], scope='pool1') + net = slim.repeat(net, 2, slim.conv2d, 128, [3, 3], scope='conv2') + net = slim.max_pool2d(net, [2, 2], scope='pool2') + net = slim.repeat(net, 4, slim.conv2d, 256, [3, 3], scope='conv3') + net = slim.max_pool2d(net, [2, 2], scope='pool3') + net = slim.repeat(net, 4, slim.conv2d, 512, [3, 3], scope='conv4') + net = slim.max_pool2d(net, [2, 2], scope='pool4') + net = slim.repeat(net, 4, slim.conv2d, 512, [3, 3], scope='conv5') + net = slim.max_pool2d(net, [2, 2], scope='pool5') + + # Use conv2d instead of fully_connected layers. + net = slim.conv2d(net, 4096, [7, 7], padding=fc_conv_padding, scope='fc6') + net = slim.dropout(net, dropout_keep_prob, is_training=is_training, + scope='dropout6') + net = slim.conv2d(net, 4096, [1, 1], scope='fc7') + # Convert end_points_collection into a end_point dict. + end_points = slim.utils.convert_collection_to_dict(end_points_collection) + if global_pool: + net = tf.reduce_mean( + input_tensor=net, axis=[1, 2], keepdims=True, name='global_pool') + end_points['global_pool'] = net + if num_classes: + net = slim.dropout(net, dropout_keep_prob, is_training=is_training, + scope='dropout7') + net = slim.conv2d(net, num_classes, [1, 1], + activation_fn=None, + normalizer_fn=None, + scope='fc8') + if spatial_squeeze: + net = tf.squeeze(net, [1, 2], name='fc8/squeezed') + end_points[sc.name + '/fc8'] = net + return net, end_points +vgg_19.default_image_size = 224 + +# Alias +vgg_d = vgg_16 +vgg_e = vgg_19 diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/vgg_test.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/vgg_test.py new file mode 100644 index 0000000..988c3db --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/nets/vgg_test.py @@ -0,0 +1,584 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for slim.nets.vgg.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +from nets import vgg + +slim = contrib_slim + + +class VGGATest(tf.test.TestCase): + + def testBuild(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = vgg.vgg_a(inputs, num_classes) + self.assertEquals(logits.op.name, 'vgg_a/fc8/squeezed') + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + + def testFullyConvolutional(self): + batch_size = 1 + height, width = 256, 256 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = vgg.vgg_a(inputs, num_classes, spatial_squeeze=False) + self.assertEquals(logits.op.name, 'vgg_a/fc8/BiasAdd') + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, 2, 2, num_classes]) + + def testGlobalPool(self): + batch_size = 1 + height, width = 256, 256 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = vgg.vgg_a(inputs, num_classes, spatial_squeeze=False, + global_pool=True) + self.assertEquals(logits.op.name, 'vgg_a/fc8/BiasAdd') + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, 1, 1, num_classes]) + + def testEndPoints(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = vgg.vgg_a(inputs, num_classes) + expected_names = ['vgg_a/conv1/conv1_1', + 'vgg_a/pool1', + 'vgg_a/conv2/conv2_1', + 'vgg_a/pool2', + 'vgg_a/conv3/conv3_1', + 'vgg_a/conv3/conv3_2', + 'vgg_a/pool3', + 'vgg_a/conv4/conv4_1', + 'vgg_a/conv4/conv4_2', + 'vgg_a/pool4', + 'vgg_a/conv5/conv5_1', + 'vgg_a/conv5/conv5_2', + 'vgg_a/pool5', + 'vgg_a/fc6', + 'vgg_a/fc7', + 'vgg_a/fc8' + ] + self.assertSetEqual(set(end_points.keys()), set(expected_names)) + + def testNoClasses(self): + batch_size = 5 + height, width = 224, 224 + num_classes = None + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + net, end_points = vgg.vgg_a(inputs, num_classes) + expected_names = ['vgg_a/conv1/conv1_1', + 'vgg_a/pool1', + 'vgg_a/conv2/conv2_1', + 'vgg_a/pool2', + 'vgg_a/conv3/conv3_1', + 'vgg_a/conv3/conv3_2', + 'vgg_a/pool3', + 'vgg_a/conv4/conv4_1', + 'vgg_a/conv4/conv4_2', + 'vgg_a/pool4', + 'vgg_a/conv5/conv5_1', + 'vgg_a/conv5/conv5_2', + 'vgg_a/pool5', + 'vgg_a/fc6', + 'vgg_a/fc7', + ] + self.assertSetEqual(set(end_points.keys()), set(expected_names)) + self.assertTrue(net.op.name.startswith('vgg_a/fc7')) + + def testModelVariables(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + vgg.vgg_a(inputs, num_classes) + expected_names = ['vgg_a/conv1/conv1_1/weights', + 'vgg_a/conv1/conv1_1/biases', + 'vgg_a/conv2/conv2_1/weights', + 'vgg_a/conv2/conv2_1/biases', + 'vgg_a/conv3/conv3_1/weights', + 'vgg_a/conv3/conv3_1/biases', + 'vgg_a/conv3/conv3_2/weights', + 'vgg_a/conv3/conv3_2/biases', + 'vgg_a/conv4/conv4_1/weights', + 'vgg_a/conv4/conv4_1/biases', + 'vgg_a/conv4/conv4_2/weights', + 'vgg_a/conv4/conv4_2/biases', + 'vgg_a/conv5/conv5_1/weights', + 'vgg_a/conv5/conv5_1/biases', + 'vgg_a/conv5/conv5_2/weights', + 'vgg_a/conv5/conv5_2/biases', + 'vgg_a/fc6/weights', + 'vgg_a/fc6/biases', + 'vgg_a/fc7/weights', + 'vgg_a/fc7/biases', + 'vgg_a/fc8/weights', + 'vgg_a/fc8/biases', + ] + model_variables = [v.op.name for v in slim.get_model_variables()] + self.assertSetEqual(set(model_variables), set(expected_names)) + + def testEvaluation(self): + batch_size = 2 + height, width = 224, 224 + num_classes = 1000 + with self.test_session(): + eval_inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = vgg.vgg_a(eval_inputs, is_training=False) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + predictions = tf.argmax(input=logits, axis=1) + self.assertListEqual(predictions.get_shape().as_list(), [batch_size]) + + def testTrainEvalWithReuse(self): + train_batch_size = 2 + eval_batch_size = 1 + train_height, train_width = 224, 224 + eval_height, eval_width = 256, 256 + num_classes = 1000 + with self.test_session(): + train_inputs = tf.random.uniform( + (train_batch_size, train_height, train_width, 3)) + logits, _ = vgg.vgg_a(train_inputs) + self.assertListEqual(logits.get_shape().as_list(), + [train_batch_size, num_classes]) + tf.compat.v1.get_variable_scope().reuse_variables() + eval_inputs = tf.random.uniform( + (eval_batch_size, eval_height, eval_width, 3)) + logits, _ = vgg.vgg_a(eval_inputs, is_training=False, + spatial_squeeze=False) + self.assertListEqual(logits.get_shape().as_list(), + [eval_batch_size, 2, 2, num_classes]) + logits = tf.reduce_mean(input_tensor=logits, axis=[1, 2]) + predictions = tf.argmax(input=logits, axis=1) + self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) + + def testForward(self): + batch_size = 1 + height, width = 224, 224 + with self.test_session() as sess: + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = vgg.vgg_a(inputs) + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(logits) + self.assertTrue(output.any()) + + +class VGG16Test(tf.test.TestCase): + + def testBuild(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = vgg.vgg_16(inputs, num_classes) + self.assertEquals(logits.op.name, 'vgg_16/fc8/squeezed') + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + + def testFullyConvolutional(self): + batch_size = 1 + height, width = 256, 256 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = vgg.vgg_16(inputs, num_classes, spatial_squeeze=False) + self.assertEquals(logits.op.name, 'vgg_16/fc8/BiasAdd') + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, 2, 2, num_classes]) + + def testGlobalPool(self): + batch_size = 1 + height, width = 256, 256 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = vgg.vgg_16(inputs, num_classes, spatial_squeeze=False, + global_pool=True) + self.assertEquals(logits.op.name, 'vgg_16/fc8/BiasAdd') + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, 1, 1, num_classes]) + + def testEndPoints(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = vgg.vgg_16(inputs, num_classes) + expected_names = ['vgg_16/conv1/conv1_1', + 'vgg_16/conv1/conv1_2', + 'vgg_16/pool1', + 'vgg_16/conv2/conv2_1', + 'vgg_16/conv2/conv2_2', + 'vgg_16/pool2', + 'vgg_16/conv3/conv3_1', + 'vgg_16/conv3/conv3_2', + 'vgg_16/conv3/conv3_3', + 'vgg_16/pool3', + 'vgg_16/conv4/conv4_1', + 'vgg_16/conv4/conv4_2', + 'vgg_16/conv4/conv4_3', + 'vgg_16/pool4', + 'vgg_16/conv5/conv5_1', + 'vgg_16/conv5/conv5_2', + 'vgg_16/conv5/conv5_3', + 'vgg_16/pool5', + 'vgg_16/fc6', + 'vgg_16/fc7', + 'vgg_16/fc8' + ] + self.assertSetEqual(set(end_points.keys()), set(expected_names)) + + def testNoClasses(self): + batch_size = 5 + height, width = 224, 224 + num_classes = None + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + net, end_points = vgg.vgg_16(inputs, num_classes) + expected_names = ['vgg_16/conv1/conv1_1', + 'vgg_16/conv1/conv1_2', + 'vgg_16/pool1', + 'vgg_16/conv2/conv2_1', + 'vgg_16/conv2/conv2_2', + 'vgg_16/pool2', + 'vgg_16/conv3/conv3_1', + 'vgg_16/conv3/conv3_2', + 'vgg_16/conv3/conv3_3', + 'vgg_16/pool3', + 'vgg_16/conv4/conv4_1', + 'vgg_16/conv4/conv4_2', + 'vgg_16/conv4/conv4_3', + 'vgg_16/pool4', + 'vgg_16/conv5/conv5_1', + 'vgg_16/conv5/conv5_2', + 'vgg_16/conv5/conv5_3', + 'vgg_16/pool5', + 'vgg_16/fc6', + 'vgg_16/fc7', + ] + self.assertSetEqual(set(end_points.keys()), set(expected_names)) + self.assertTrue(net.op.name.startswith('vgg_16/fc7')) + + def testModelVariables(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + vgg.vgg_16(inputs, num_classes) + expected_names = ['vgg_16/conv1/conv1_1/weights', + 'vgg_16/conv1/conv1_1/biases', + 'vgg_16/conv1/conv1_2/weights', + 'vgg_16/conv1/conv1_2/biases', + 'vgg_16/conv2/conv2_1/weights', + 'vgg_16/conv2/conv2_1/biases', + 'vgg_16/conv2/conv2_2/weights', + 'vgg_16/conv2/conv2_2/biases', + 'vgg_16/conv3/conv3_1/weights', + 'vgg_16/conv3/conv3_1/biases', + 'vgg_16/conv3/conv3_2/weights', + 'vgg_16/conv3/conv3_2/biases', + 'vgg_16/conv3/conv3_3/weights', + 'vgg_16/conv3/conv3_3/biases', + 'vgg_16/conv4/conv4_1/weights', + 'vgg_16/conv4/conv4_1/biases', + 'vgg_16/conv4/conv4_2/weights', + 'vgg_16/conv4/conv4_2/biases', + 'vgg_16/conv4/conv4_3/weights', + 'vgg_16/conv4/conv4_3/biases', + 'vgg_16/conv5/conv5_1/weights', + 'vgg_16/conv5/conv5_1/biases', + 'vgg_16/conv5/conv5_2/weights', + 'vgg_16/conv5/conv5_2/biases', + 'vgg_16/conv5/conv5_3/weights', + 'vgg_16/conv5/conv5_3/biases', + 'vgg_16/fc6/weights', + 'vgg_16/fc6/biases', + 'vgg_16/fc7/weights', + 'vgg_16/fc7/biases', + 'vgg_16/fc8/weights', + 'vgg_16/fc8/biases', + ] + model_variables = [v.op.name for v in slim.get_model_variables()] + self.assertSetEqual(set(model_variables), set(expected_names)) + + def testEvaluation(self): + batch_size = 2 + height, width = 224, 224 + num_classes = 1000 + with self.test_session(): + eval_inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = vgg.vgg_16(eval_inputs, is_training=False) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + predictions = tf.argmax(input=logits, axis=1) + self.assertListEqual(predictions.get_shape().as_list(), [batch_size]) + + def testTrainEvalWithReuse(self): + train_batch_size = 2 + eval_batch_size = 1 + train_height, train_width = 224, 224 + eval_height, eval_width = 256, 256 + num_classes = 1000 + with self.test_session(): + train_inputs = tf.random.uniform( + (train_batch_size, train_height, train_width, 3)) + logits, _ = vgg.vgg_16(train_inputs) + self.assertListEqual(logits.get_shape().as_list(), + [train_batch_size, num_classes]) + tf.compat.v1.get_variable_scope().reuse_variables() + eval_inputs = tf.random.uniform( + (eval_batch_size, eval_height, eval_width, 3)) + logits, _ = vgg.vgg_16(eval_inputs, is_training=False, + spatial_squeeze=False) + self.assertListEqual(logits.get_shape().as_list(), + [eval_batch_size, 2, 2, num_classes]) + logits = tf.reduce_mean(input_tensor=logits, axis=[1, 2]) + predictions = tf.argmax(input=logits, axis=1) + self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) + + def testForward(self): + batch_size = 1 + height, width = 224, 224 + with self.test_session() as sess: + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = vgg.vgg_16(inputs) + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(logits) + self.assertTrue(output.any()) + + +class VGG19Test(tf.test.TestCase): + + def testBuild(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = vgg.vgg_19(inputs, num_classes) + self.assertEquals(logits.op.name, 'vgg_19/fc8/squeezed') + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + + def testFullyConvolutional(self): + batch_size = 1 + height, width = 256, 256 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = vgg.vgg_19(inputs, num_classes, spatial_squeeze=False) + self.assertEquals(logits.op.name, 'vgg_19/fc8/BiasAdd') + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, 2, 2, num_classes]) + + def testGlobalPool(self): + batch_size = 1 + height, width = 256, 256 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = vgg.vgg_19(inputs, num_classes, spatial_squeeze=False, + global_pool=True) + self.assertEquals(logits.op.name, 'vgg_19/fc8/BiasAdd') + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, 1, 1, num_classes]) + + def testEndPoints(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + _, end_points = vgg.vgg_19(inputs, num_classes) + expected_names = [ + 'vgg_19/conv1/conv1_1', + 'vgg_19/conv1/conv1_2', + 'vgg_19/pool1', + 'vgg_19/conv2/conv2_1', + 'vgg_19/conv2/conv2_2', + 'vgg_19/pool2', + 'vgg_19/conv3/conv3_1', + 'vgg_19/conv3/conv3_2', + 'vgg_19/conv3/conv3_3', + 'vgg_19/conv3/conv3_4', + 'vgg_19/pool3', + 'vgg_19/conv4/conv4_1', + 'vgg_19/conv4/conv4_2', + 'vgg_19/conv4/conv4_3', + 'vgg_19/conv4/conv4_4', + 'vgg_19/pool4', + 'vgg_19/conv5/conv5_1', + 'vgg_19/conv5/conv5_2', + 'vgg_19/conv5/conv5_3', + 'vgg_19/conv5/conv5_4', + 'vgg_19/pool5', + 'vgg_19/fc6', + 'vgg_19/fc7', + 'vgg_19/fc8' + ] + self.assertSetEqual(set(end_points.keys()), set(expected_names)) + + def testNoClasses(self): + batch_size = 5 + height, width = 224, 224 + num_classes = None + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + net, end_points = vgg.vgg_19(inputs, num_classes) + expected_names = [ + 'vgg_19/conv1/conv1_1', + 'vgg_19/conv1/conv1_2', + 'vgg_19/pool1', + 'vgg_19/conv2/conv2_1', + 'vgg_19/conv2/conv2_2', + 'vgg_19/pool2', + 'vgg_19/conv3/conv3_1', + 'vgg_19/conv3/conv3_2', + 'vgg_19/conv3/conv3_3', + 'vgg_19/conv3/conv3_4', + 'vgg_19/pool3', + 'vgg_19/conv4/conv4_1', + 'vgg_19/conv4/conv4_2', + 'vgg_19/conv4/conv4_3', + 'vgg_19/conv4/conv4_4', + 'vgg_19/pool4', + 'vgg_19/conv5/conv5_1', + 'vgg_19/conv5/conv5_2', + 'vgg_19/conv5/conv5_3', + 'vgg_19/conv5/conv5_4', + 'vgg_19/pool5', + 'vgg_19/fc6', + 'vgg_19/fc7', + ] + self.assertSetEqual(set(end_points.keys()), set(expected_names)) + self.assertTrue(net.op.name.startswith('vgg_19/fc7')) + + def testModelVariables(self): + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + with self.test_session(): + inputs = tf.random.uniform((batch_size, height, width, 3)) + vgg.vgg_19(inputs, num_classes) + expected_names = [ + 'vgg_19/conv1/conv1_1/weights', + 'vgg_19/conv1/conv1_1/biases', + 'vgg_19/conv1/conv1_2/weights', + 'vgg_19/conv1/conv1_2/biases', + 'vgg_19/conv2/conv2_1/weights', + 'vgg_19/conv2/conv2_1/biases', + 'vgg_19/conv2/conv2_2/weights', + 'vgg_19/conv2/conv2_2/biases', + 'vgg_19/conv3/conv3_1/weights', + 'vgg_19/conv3/conv3_1/biases', + 'vgg_19/conv3/conv3_2/weights', + 'vgg_19/conv3/conv3_2/biases', + 'vgg_19/conv3/conv3_3/weights', + 'vgg_19/conv3/conv3_3/biases', + 'vgg_19/conv3/conv3_4/weights', + 'vgg_19/conv3/conv3_4/biases', + 'vgg_19/conv4/conv4_1/weights', + 'vgg_19/conv4/conv4_1/biases', + 'vgg_19/conv4/conv4_2/weights', + 'vgg_19/conv4/conv4_2/biases', + 'vgg_19/conv4/conv4_3/weights', + 'vgg_19/conv4/conv4_3/biases', + 'vgg_19/conv4/conv4_4/weights', + 'vgg_19/conv4/conv4_4/biases', + 'vgg_19/conv5/conv5_1/weights', + 'vgg_19/conv5/conv5_1/biases', + 'vgg_19/conv5/conv5_2/weights', + 'vgg_19/conv5/conv5_2/biases', + 'vgg_19/conv5/conv5_3/weights', + 'vgg_19/conv5/conv5_3/biases', + 'vgg_19/conv5/conv5_4/weights', + 'vgg_19/conv5/conv5_4/biases', + 'vgg_19/fc6/weights', + 'vgg_19/fc6/biases', + 'vgg_19/fc7/weights', + 'vgg_19/fc7/biases', + 'vgg_19/fc8/weights', + 'vgg_19/fc8/biases', + ] + model_variables = [v.op.name for v in slim.get_model_variables()] + self.assertSetEqual(set(model_variables), set(expected_names)) + + def testEvaluation(self): + batch_size = 2 + height, width = 224, 224 + num_classes = 1000 + with self.test_session(): + eval_inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = vgg.vgg_19(eval_inputs, is_training=False) + self.assertListEqual(logits.get_shape().as_list(), + [batch_size, num_classes]) + predictions = tf.argmax(input=logits, axis=1) + self.assertListEqual(predictions.get_shape().as_list(), [batch_size]) + + def testTrainEvalWithReuse(self): + train_batch_size = 2 + eval_batch_size = 1 + train_height, train_width = 224, 224 + eval_height, eval_width = 256, 256 + num_classes = 1000 + with self.test_session(): + train_inputs = tf.random.uniform( + (train_batch_size, train_height, train_width, 3)) + logits, _ = vgg.vgg_19(train_inputs) + self.assertListEqual(logits.get_shape().as_list(), + [train_batch_size, num_classes]) + tf.compat.v1.get_variable_scope().reuse_variables() + eval_inputs = tf.random.uniform( + (eval_batch_size, eval_height, eval_width, 3)) + logits, _ = vgg.vgg_19(eval_inputs, is_training=False, + spatial_squeeze=False) + self.assertListEqual(logits.get_shape().as_list(), + [eval_batch_size, 2, 2, num_classes]) + logits = tf.reduce_mean(input_tensor=logits, axis=[1, 2]) + predictions = tf.argmax(input=logits, axis=1) + self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) + + def testForward(self): + batch_size = 1 + height, width = 224, 224 + with self.test_session() as sess: + inputs = tf.random.uniform((batch_size, height, width, 3)) + logits, _ = vgg.vgg_19(inputs) + sess.run(tf.compat.v1.global_variables_initializer()) + output = sess.run(logits) + self.assertTrue(output.any()) + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/__init__.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/__init__.py @@ -0,0 +1 @@ + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/cifarnet_preprocessing.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/cifarnet_preprocessing.py new file mode 100644 index 0000000..2f66e77 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/cifarnet_preprocessing.py @@ -0,0 +1,150 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Provides utilities to preprocess images in CIFAR-10. + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +_PADDING = 4 + +slim = contrib_slim + + +def preprocess_for_train(image, + output_height, + output_width, + padding=_PADDING, + add_image_summaries=True, + use_grayscale=False): + """Preprocesses the given image for training. + + Note that the actual resizing scale is sampled from + [`resize_size_min`, `resize_size_max`]. + + Args: + image: A `Tensor` representing an image of arbitrary size. + output_height: The height of the image after preprocessing. + output_width: The width of the image after preprocessing. + padding: The amound of padding before and after each dimension of the image. + add_image_summaries: Enable image summaries. + use_grayscale: Whether to convert the image from RGB to grayscale. + + Returns: + A preprocessed image. + """ + if add_image_summaries: + tf.summary.image('image', tf.expand_dims(image, 0)) + + # Transform the image to floats. + image = tf.to_float(image) + if use_grayscale: + image = tf.image.rgb_to_grayscale(image) + if padding > 0: + image = tf.pad(image, [[padding, padding], [padding, padding], [0, 0]]) + # Randomly crop a [height, width] section of the image. + distorted_image = tf.random_crop(image, + [output_height, output_width, 3]) + + # Randomly flip the image horizontally. + distorted_image = tf.image.random_flip_left_right(distorted_image) + + if add_image_summaries: + tf.summary.image('distorted_image', tf.expand_dims(distorted_image, 0)) + + # Because these operations are not commutative, consider randomizing + # the order their operation. + distorted_image = tf.image.random_brightness(distorted_image, + max_delta=63) + distorted_image = tf.image.random_contrast(distorted_image, + lower=0.2, upper=1.8) + # Subtract off the mean and divide by the variance of the pixels. + return tf.image.per_image_standardization(distorted_image) + + +def preprocess_for_eval(image, + output_height, + output_width, + add_image_summaries=True, + use_grayscale=False): + """Preprocesses the given image for evaluation. + + Args: + image: A `Tensor` representing an image of arbitrary size. + output_height: The height of the image after preprocessing. + output_width: The width of the image after preprocessing. + add_image_summaries: Enable image summaries. + use_grayscale: Whether to convert the image from RGB to grayscale. + + Returns: + A preprocessed image. + """ + if add_image_summaries: + tf.summary.image('image', tf.expand_dims(image, 0)) + # Transform the image to floats. + image = tf.to_float(image) + if use_grayscale: + image = tf.image.rgb_to_grayscale(image) + + # Resize and crop if needed. + resized_image = tf.image.resize_image_with_crop_or_pad(image, + output_width, + output_height) + if add_image_summaries: + tf.summary.image('resized_image', tf.expand_dims(resized_image, 0)) + + # Subtract off the mean and divide by the variance of the pixels. + return tf.image.per_image_standardization(resized_image) + + +def preprocess_image(image, + output_height, + output_width, + is_training=False, + add_image_summaries=True, + use_grayscale=False): + """Preprocesses the given image. + + Args: + image: A `Tensor` representing an image of arbitrary size. + output_height: The height of the image after preprocessing. + output_width: The width of the image after preprocessing. + is_training: `True` if we're preprocessing the image for training and + `False` otherwise. + add_image_summaries: Enable image summaries. + use_grayscale: Whether to convert the image from RGB to grayscale. + + Returns: + A preprocessed image. + """ + if is_training: + return preprocess_for_train( + image, + output_height, + output_width, + add_image_summaries=add_image_summaries, + use_grayscale=use_grayscale) + else: + return preprocess_for_eval( + image, + output_height, + output_width, + add_image_summaries=add_image_summaries, + use_grayscale=use_grayscale) diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/inception_preprocessing.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/inception_preprocessing.py new file mode 100644 index 0000000..1b58e66 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/inception_preprocessing.py @@ -0,0 +1,349 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Provides utilities to preprocess images for the Inception networks.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from tensorflow.python.ops import control_flow_ops + + +def apply_with_random_selector(x, func, num_cases): + """Computes func(x, sel), with sel sampled from [0...num_cases-1]. + + Args: + x: input Tensor. + func: Python function to apply. + num_cases: Python int32, number of cases to sample sel from. + + Returns: + The result of func(x, sel), where func receives the value of the + selector as a python integer, but sel is sampled dynamically. + """ + sel = tf.random_uniform([], maxval=num_cases, dtype=tf.int32) + # Pass the real x only to one of the func calls. + return control_flow_ops.merge([ + func(control_flow_ops.switch(x, tf.equal(sel, case))[1], case) + for case in range(num_cases)])[0] + + +def distort_color(image, color_ordering=0, fast_mode=True, scope=None): + """Distort the color of a Tensor image. + + Each color distortion is non-commutative and thus ordering of the color ops + matters. Ideally we would randomly permute the ordering of the color ops. + Rather then adding that level of complication, we select a distinct ordering + of color ops for each preprocessing thread. + + Args: + image: 3-D Tensor containing single image in [0, 1]. + color_ordering: Python int, a type of distortion (valid values: 0-3). + fast_mode: Avoids slower ops (random_hue and random_contrast) + scope: Optional scope for name_scope. + Returns: + 3-D Tensor color-distorted image on range [0, 1] + Raises: + ValueError: if color_ordering not in [0, 3] + """ + with tf.name_scope(scope, 'distort_color', [image]): + if fast_mode: + if color_ordering == 0: + image = tf.image.random_brightness(image, max_delta=32. / 255.) + image = tf.image.random_saturation(image, lower=0.5, upper=1.5) + else: + image = tf.image.random_saturation(image, lower=0.5, upper=1.5) + image = tf.image.random_brightness(image, max_delta=32. / 255.) + else: + if color_ordering == 0: + image = tf.image.random_brightness(image, max_delta=32. / 255.) + image = tf.image.random_saturation(image, lower=0.5, upper=1.5) + image = tf.image.random_hue(image, max_delta=0.2) + image = tf.image.random_contrast(image, lower=0.5, upper=1.5) + elif color_ordering == 1: + image = tf.image.random_saturation(image, lower=0.5, upper=1.5) + image = tf.image.random_brightness(image, max_delta=32. / 255.) + image = tf.image.random_contrast(image, lower=0.5, upper=1.5) + image = tf.image.random_hue(image, max_delta=0.2) + elif color_ordering == 2: + image = tf.image.random_contrast(image, lower=0.5, upper=1.5) + image = tf.image.random_hue(image, max_delta=0.2) + image = tf.image.random_brightness(image, max_delta=32. / 255.) + image = tf.image.random_saturation(image, lower=0.5, upper=1.5) + elif color_ordering == 3: + image = tf.image.random_hue(image, max_delta=0.2) + image = tf.image.random_saturation(image, lower=0.5, upper=1.5) + image = tf.image.random_contrast(image, lower=0.5, upper=1.5) + image = tf.image.random_brightness(image, max_delta=32. / 255.) + else: + raise ValueError('color_ordering must be in [0, 3]') + + # The random_* ops do not necessarily clamp. + return tf.clip_by_value(image, 0.0, 1.0) + + +def distorted_bounding_box_crop(image, + bbox, + min_object_covered=0.1, + aspect_ratio_range=(0.75, 1.33), + area_range=(0.05, 1.0), + max_attempts=100, + scope=None): + """Generates cropped_image using a one of the bboxes randomly distorted. + + See `tf.image.sample_distorted_bounding_box` for more documentation. + + Args: + image: 3-D Tensor of image (it will be converted to floats in [0, 1]). + bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords] + where each coordinate is [0, 1) and the coordinates are arranged + as [ymin, xmin, ymax, xmax]. If num_boxes is 0 then it would use the whole + image. + min_object_covered: An optional `float`. Defaults to `0.1`. The cropped + area of the image must contain at least this fraction of any bounding box + supplied. + aspect_ratio_range: An optional list of `floats`. The cropped area of the + image must have an aspect ratio = width / height within this range. + area_range: An optional list of `floats`. The cropped area of the image + must contain a fraction of the supplied image within in this range. + max_attempts: An optional `int`. Number of attempts at generating a cropped + region of the image of the specified constraints. After `max_attempts` + failures, return the entire image. + scope: Optional scope for name_scope. + Returns: + A tuple, a 3-D Tensor cropped_image and the distorted bbox + """ + with tf.name_scope(scope, 'distorted_bounding_box_crop', [image, bbox]): + # Each bounding box has shape [1, num_boxes, box coords] and + # the coordinates are ordered [ymin, xmin, ymax, xmax]. + + # A large fraction of image datasets contain a human-annotated bounding + # box delineating the region of the image containing the object of interest. + # We choose to create a new bounding box for the object which is a randomly + # distorted version of the human-annotated bounding box that obeys an + # allowed range of aspect ratios, sizes and overlap with the human-annotated + # bounding box. If no box is supplied, then we assume the bounding box is + # the entire image. + sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box( + tf.shape(image), + bounding_boxes=bbox, + min_object_covered=min_object_covered, + aspect_ratio_range=aspect_ratio_range, + area_range=area_range, + max_attempts=max_attempts, + use_image_if_no_bounding_boxes=True) + bbox_begin, bbox_size, distort_bbox = sample_distorted_bounding_box + + # Crop the image to the specified bounding box. + cropped_image = tf.slice(image, bbox_begin, bbox_size) + return cropped_image, distort_bbox + + +def preprocess_for_train(image, height, width, bbox, fast_mode=True, scope=None, + add_image_summaries=True, random_crop=True, use_grayscale=False): + """Distort one image for training a network. + + Distorting images provides a useful technique for augmenting the data + set during training in order to make the network invariant to aspects + of the image that do not effect the label. + + Additionally it would create image_summaries to display the different + transformations applied to the image. + + Args: + image: 3-D Tensor of image. If dtype is tf.float32 then the range should be + [0, 1], otherwise it would converted to tf.float32 assuming that the range + is [0, MAX], where MAX is largest positive representable number for + int(8/16/32) data type (see `tf.image.convert_image_dtype` for details). + height: integer + width: integer + bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords] + where each coordinate is [0, 1) and the coordinates are arranged + as [ymin, xmin, ymax, xmax]. + fast_mode: Optional boolean, if True avoids slower transformations (i.e. + bi-cubic resizing, random_hue or random_contrast). + scope: Optional scope for name_scope. + add_image_summaries: Enable image summaries. + random_crop: Enable random cropping of images during preprocessing for + training. + use_grayscale: Whether to convert the image from RGB to grayscale. + Returns: + 3-D float Tensor of distorted image used for training with range [-1, 1]. + """ + with tf.name_scope(scope, 'distort_image', [image, height, width, bbox]): + if bbox is None: + bbox = tf.constant([0.0, 0.0, 1.0, 1.0], dtype=tf.float32, shape=[1, 1, 4]) + if image.dtype != tf.float32: + image = tf.image.convert_image_dtype(image, dtype=tf.float32) + # Each bounding box has shape [1, num_boxes, box coords] and + # the coordinates are ordered [ymin, xmin, ymax, xmax]. + image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0), # xling --> (0.08, 1.0) + bbox) + if add_image_summaries: + tf.summary.image('image_with_bounding_boxes', image_with_box) + + if not random_crop: + distorted_image = image + else: + distorted_image, distorted_bbox = distorted_bounding_box_crop(image, bbox) + # Restore the shape since the dynamic slice based upon the bbox_size loses + # the third dimension. + distorted_image.set_shape([None, None, 3]) + image_with_distorted_box = tf.image.draw_bounding_boxes( + tf.expand_dims(image, 0), distorted_bbox) + if add_image_summaries: + tf.summary.image('images_with_distorted_bounding_box', + image_with_distorted_box) + + # This resizing operation may distort the images because the aspect + # ratio is not respected. We select a resize method in a round robin + # fashion based on the thread number. + # Note that ResizeMethod contains 4 enumerated resizing methods. + + # We select only 1 case for fast_mode bilinear. + num_resize_cases = 1 if fast_mode else 4 + distorted_image = apply_with_random_selector( + distorted_image, + lambda x, method: tf.image.resize_images(x, [height, width], method), + num_cases=num_resize_cases) + + if add_image_summaries: + tf.summary.image(('cropped_' if random_crop else '') + 'resized_image', + tf.expand_dims(distorted_image, 0)) + + # Randomly flip the image horizontally. + distorted_image = tf.image.random_flip_left_right(distorted_image) # xling -> 0.5? + + # Randomly distort the colors. There are 1 or 4 ways to do it. + num_distort_cases = 1 if fast_mode else 4 + distorted_image = apply_with_random_selector( + distorted_image, + lambda x, ordering: distort_color(x, ordering, fast_mode), # xling -> 0.4, 0.4, 0.4 + num_cases=num_distort_cases) + + if use_grayscale: + distorted_image = tf.image.rgb_to_grayscale(distorted_image) + + if add_image_summaries: + tf.summary.image('final_distorted_image', + tf.expand_dims(distorted_image, 0)) + + distorted_image = tf.subtract(distorted_image, 0.5) # xling: [485, 456, 406] + distorted_image = tf.multiply(distorted_image, 2.0) # xling: [229, 224, 225] + return distorted_image # xling: format? # xling: calling order? + + +def preprocess_for_eval(image, + height, # xling: 256 + width, # xling: 256 + central_fraction=0.875, + scope=None, + central_crop=True, + use_grayscale=False): + """Prepare one image for evaluation. + + If height and width are specified it would output an image with that size by + applying resize_bilinear. + + If central_fraction is specified it would crop the central fraction of the + input image. + + Args: + image: 3-D Tensor of image. If dtype is tf.float32 then the range should be + [0, 1], otherwise it would converted to tf.float32 assuming that the range + is [0, MAX], where MAX is largest positive representable number for + int(8/16/32) data type (see `tf.image.convert_image_dtype` for details). + height: integer + width: integer + central_fraction: Optional Float, fraction of the image to crop. + scope: Optional scope for name_scope. + central_crop: Enable central cropping of images during preprocessing for + evaluation. + use_grayscale: Whether to convert the image from RGB to grayscale. + Returns: + 3-D float Tensor of prepared image. + """ + with tf.name_scope(scope, 'eval_image', [image, height, width]): + if image.dtype != tf.float32: + image = tf.image.convert_image_dtype(image, dtype=tf.float32) + if use_grayscale: + image = tf.image.rgb_to_grayscale(image) + # Crop the central region of the image with an area containing 87.5% of + # the original image. + if central_crop and central_fraction: + image = tf.image.central_crop(image, central_fraction=central_fraction) # xling: image_size? + + if height and width: + # Resize the image to the specified height and width. + image = tf.expand_dims(image, 0) + image = tf.image.resize_bilinear(image, [height, width], + align_corners=False) # xling: match calling order with ME + image = tf.squeeze(image, [0]) + image = tf.subtract(image, 0.5) # xling: + image = tf.multiply(image, 2.0) # xling: + return image + + +def preprocess_image(image, + height, + width, + is_training=False, + bbox=None, + fast_mode=True, + add_image_summaries=True, + crop_image=True, + use_grayscale=False): + """Pre-process one image for training or evaluation. + + Args: + image: 3-D Tensor [height, width, channels] with the image. If dtype is + tf.float32 then the range should be [0, 1], otherwise it would converted + to tf.float32 assuming that the range is [0, MAX], where MAX is largest + positive representable number for int(8/16/32) data type (see + `tf.image.convert_image_dtype` for details). + height: integer, image expected height. + width: integer, image expected width. + is_training: Boolean. If true it would transform an image for train, + otherwise it would transform it for evaluation. + bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords] + where each coordinate is [0, 1) and the coordinates are arranged as + [ymin, xmin, ymax, xmax]. + fast_mode: Optional boolean, if True avoids slower transformations. + add_image_summaries: Enable image summaries. + crop_image: Whether to enable cropping of images during preprocessing for + both training and evaluation. + use_grayscale: Whether to convert the image from RGB to grayscale. + + Returns: + 3-D float Tensor containing an appropriately scaled image + + Raises: + ValueError: if user does not provide bounding box + """ + if is_training: + return preprocess_for_train(image, height, width, bbox, fast_mode, + add_image_summaries=add_image_summaries, random_crop=crop_image, + use_grayscale=use_grayscale) + else: + return preprocess_for_eval( + image, + height, + width, + central_crop=crop_image, + use_grayscale=use_grayscale) + + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/lenet_preprocessing.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/lenet_preprocessing.py new file mode 100644 index 0000000..d5cdec9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/lenet_preprocessing.py @@ -0,0 +1,53 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Provides utilities for preprocessing.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +slim = contrib_slim + + +def preprocess_image(image, + output_height, + output_width, + is_training, + use_grayscale=False): + """Preprocesses the given image. + + Args: + image: A `Tensor` representing an image of arbitrary size. + output_height: The height of the image after preprocessing. + output_width: The width of the image after preprocessing. + is_training: `True` if we're preprocessing the image for training and + `False` otherwise. + use_grayscale: Whether to convert the image from RGB to grayscale. + + Returns: + A preprocessed image. + """ + del is_training # Unused argument + image = tf.to_float(image) + if use_grayscale: + image = tf.image.rgb_to_grayscale(image) + image = tf.image.resize_image_with_crop_or_pad( + image, output_width, output_height) + image = tf.subtract(image, 128.0) + image = tf.div(image, 128.0) + return image diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/preprocessing_factory.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/preprocessing_factory.py new file mode 100644 index 0000000..7d5b9d2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/preprocessing_factory.py @@ -0,0 +1,98 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains a factory for building various models.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from tensorflow.contrib import slim as contrib_slim + +from preprocessing import cifarnet_preprocessing +from preprocessing import inception_preprocessing +from preprocessing import lenet_preprocessing +from preprocessing import vgg_preprocessing + +slim = contrib_slim + + +def get_preprocessing(name, is_training=False, use_grayscale=False, add_image_summaries=False): + """Returns preprocessing_fn(image, height, width, **kwargs). + + Args: + name: The name of the preprocessing function. + is_training: `True` if the model is being used for training and `False` + otherwise. + use_grayscale: Whether to convert the image from RGB to grayscale. + + Returns: + preprocessing_fn: A function that preprocessing a single image (pre-batch). + It has the following signature: + image = preprocessing_fn(image, output_height, output_width, ...). + + Raises: + ValueError: If Preprocessing `name` is not recognized. + """ + preprocessing_fn_map = { + 'cifarnet': cifarnet_preprocessing, + 'inception': inception_preprocessing, + 'inception_v1': inception_preprocessing, + 'inception_v2': inception_preprocessing, + 'inception_v3': inception_preprocessing, + 'inception_v4': inception_preprocessing, + 'inception_resnet_v2': inception_preprocessing, + 'lenet': lenet_preprocessing, + 'mobilenet_v1': inception_preprocessing, + 'mobilenet_v2': inception_preprocessing, + 'mobilenet_v2_035': inception_preprocessing, + 'mobilenet_v3_small': inception_preprocessing, + 'mobilenet_v3_large': inception_preprocessing, + 'mobilenet_v3_small_minimalistic': inception_preprocessing, + 'mobilenet_v3_large_minimalistic': inception_preprocessing, + 'mobilenet_edgetpu': inception_preprocessing, + 'mobilenet_edgetpu_075': inception_preprocessing, + 'mobilenet_v2_140': inception_preprocessing, + 'nasnet_mobile': inception_preprocessing, + 'nasnet_large': inception_preprocessing, + 'pnasnet_mobile': inception_preprocessing, + 'pnasnet_large': inception_preprocessing, + 'resnet_v1_50': vgg_preprocessing, + 'resnet_v1_101': vgg_preprocessing, + 'resnet_v1_152': vgg_preprocessing, + 'resnet_v1_200': vgg_preprocessing, + 'resnet_v2_50': vgg_preprocessing, + 'resnet_v2_101': vgg_preprocessing, + 'resnet_v2_152': vgg_preprocessing, + 'resnet_v2_200': vgg_preprocessing, + 'vgg': vgg_preprocessing, + 'vgg_a': vgg_preprocessing, + 'vgg_16': vgg_preprocessing, + 'vgg_19': vgg_preprocessing, + } + + if name not in preprocessing_fn_map: + raise ValueError('Preprocessing name [%s] was not recognized' % name) + + def preprocessing_fn(image, output_height, output_width, **kwargs): + return preprocessing_fn_map[name].preprocess_image( + image, + output_height, + output_width, + is_training=is_training, + use_grayscale=use_grayscale, + add_image_summaries=add_image_summaries, + **kwargs) + + return preprocessing_fn + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/vgg_preprocessing.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/vgg_preprocessing.py new file mode 100644 index 0000000..ae1db14 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/preprocessing/vgg_preprocessing.py @@ -0,0 +1,383 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Provides utilities to preprocess images. + +The preprocessing steps for VGG were introduced in the following technical +report: + + Very Deep Convolutional Networks For Large-Scale Image Recognition + Karen Simonyan and Andrew Zisserman + arXiv technical report, 2015 + PDF: http://arxiv.org/pdf/1409.1556.pdf + ILSVRC 2014 Slides: http://www.robots.ox.ac.uk/~karen/pdf/ILSVRC_2014.pdf + CC-BY-4.0 + +More information can be obtained from the VGG website: +www.robots.ox.ac.uk/~vgg/research/very_deep/ +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.contrib import slim as contrib_slim + +slim = contrib_slim + +_R_MEAN = 123.68 +_G_MEAN = 116.78 +_B_MEAN = 103.94 + +_RESIZE_SIDE_MIN = 256 +_RESIZE_SIDE_MAX = 512 + + +def _crop(image, offset_height, offset_width, crop_height, crop_width): + """Crops the given image using the provided offsets and sizes. + + Note that the method doesn't assume we know the input image size but it does + assume we know the input image rank. + + Args: + image: an image of shape [height, width, channels]. + offset_height: a scalar tensor indicating the height offset. + offset_width: a scalar tensor indicating the width offset. + crop_height: the height of the cropped image. + crop_width: the width of the cropped image. + + Returns: + the cropped (and resized) image. + + Raises: + InvalidArgumentError: if the rank is not 3 or if the image dimensions are + less than the crop size. + """ + original_shape = tf.shape(image) + + rank_assertion = tf.Assert( + tf.equal(tf.rank(image), 3), + ['Rank of image must be equal to 3.']) + with tf.control_dependencies([rank_assertion]): + cropped_shape = tf.stack([crop_height, crop_width, original_shape[2]]) + + size_assertion = tf.Assert( + tf.logical_and( + tf.greater_equal(original_shape[0], crop_height), + tf.greater_equal(original_shape[1], crop_width)), + ['Crop size greater than the image size.']) + + offsets = tf.to_int32(tf.stack([offset_height, offset_width, 0])) + + # Use tf.slice instead of crop_to_bounding box as it accepts tensors to + # define the crop size. + with tf.control_dependencies([size_assertion]): + image = tf.slice(image, offsets, cropped_shape) + return tf.reshape(image, cropped_shape) + + +def _random_crop(image_list, crop_height, crop_width): + """Crops the given list of images. + + The function applies the same crop to each image in the list. This can be + effectively applied when there are multiple image inputs of the same + dimension such as: + + image, depths, normals = _random_crop([image, depths, normals], 120, 150) + + Args: + image_list: a list of image tensors of the same dimension but possibly + varying channel. + crop_height: the new height. + crop_width: the new width. + + Returns: + the image_list with cropped images. + + Raises: + ValueError: if there are multiple image inputs provided with different size + or the images are smaller than the crop dimensions. + """ + if not image_list: + raise ValueError('Empty image_list.') + + # Compute the rank assertions. + rank_assertions = [] + for i in range(len(image_list)): + image_rank = tf.rank(image_list[i]) + rank_assert = tf.Assert( + tf.equal(image_rank, 3), + ['Wrong rank for tensor %s [expected] [actual]', + image_list[i].name, 3, image_rank]) + rank_assertions.append(rank_assert) + + with tf.control_dependencies([rank_assertions[0]]): + image_shape = tf.shape(image_list[0]) + image_height = image_shape[0] + image_width = image_shape[1] + crop_size_assert = tf.Assert( + tf.logical_and( + tf.greater_equal(image_height, crop_height), + tf.greater_equal(image_width, crop_width)), + ['Crop size greater than the image size.']) + + asserts = [rank_assertions[0], crop_size_assert] + + for i in range(1, len(image_list)): + image = image_list[i] + asserts.append(rank_assertions[i]) + with tf.control_dependencies([rank_assertions[i]]): + shape = tf.shape(image) + height = shape[0] + width = shape[1] + + height_assert = tf.Assert( + tf.equal(height, image_height), + ['Wrong height for tensor %s [expected][actual]', + image.name, height, image_height]) + width_assert = tf.Assert( + tf.equal(width, image_width), + ['Wrong width for tensor %s [expected][actual]', + image.name, width, image_width]) + asserts.extend([height_assert, width_assert]) + + # Create a random bounding box. + # + # Use tf.random_uniform and not numpy.random.rand as doing the former would + # generate random numbers at graph eval time, unlike the latter which + # generates random numbers at graph definition time. + with tf.control_dependencies(asserts): + max_offset_height = tf.reshape(image_height - crop_height + 1, []) + with tf.control_dependencies(asserts): + max_offset_width = tf.reshape(image_width - crop_width + 1, []) + offset_height = tf.random_uniform( + [], maxval=max_offset_height, dtype=tf.int32) + offset_width = tf.random_uniform( + [], maxval=max_offset_width, dtype=tf.int32) + + return [_crop(image, offset_height, offset_width, + crop_height, crop_width) for image in image_list] + + +def _central_crop(image_list, crop_height, crop_width): + """Performs central crops of the given image list. + + Args: + image_list: a list of image tensors of the same dimension but possibly + varying channel. + crop_height: the height of the image following the crop. + crop_width: the width of the image following the crop. + + Returns: + the list of cropped images. + """ + outputs = [] + for image in image_list: + image_height = tf.shape(image)[0] + image_width = tf.shape(image)[1] + + offset_height = (image_height - crop_height) / 2 + offset_width = (image_width - crop_width) / 2 + + outputs.append(_crop(image, offset_height, offset_width, + crop_height, crop_width)) + return outputs + + +def _mean_image_subtraction(image, means): + """Subtracts the given means from each image channel. + + For example: + means = [123.68, 116.779, 103.939] + image = _mean_image_subtraction(image, means) + + Note that the rank of `image` must be known. + + Args: + image: a tensor of size [height, width, C]. + means: a C-vector of values to subtract from each channel. + + Returns: + the centered image. + + Raises: + ValueError: If the rank of `image` is unknown, if `image` has a rank other + than three or if the number of channels in `image` doesn't match the + number of values in `means`. + """ + if image.get_shape().ndims != 3: + raise ValueError('Input must be of size [height, width, C>0]') + num_channels = image.get_shape().as_list()[-1] + if len(means) != num_channels: + raise ValueError('len(means) must match the number of channels') + + channels = tf.split(axis=2, num_or_size_splits=num_channels, value=image) + for i in range(num_channels): + channels[i] -= means[i] + return tf.concat(axis=2, values=channels) + + +def _smallest_size_at_least(height, width, smallest_side): + """Computes new shape with the smallest side equal to `smallest_side`. + + Computes new shape with the smallest side equal to `smallest_side` while + preserving the original aspect ratio. + + Args: + height: an int32 scalar tensor indicating the current height. + width: an int32 scalar tensor indicating the current width. + smallest_side: A python integer or scalar `Tensor` indicating the size of + the smallest side after resize. + + Returns: + new_height: an int32 scalar tensor indicating the new height. + new_width: and int32 scalar tensor indicating the new width. + """ + smallest_side = tf.convert_to_tensor(smallest_side, dtype=tf.int32) + + height = tf.to_float(height) + width = tf.to_float(width) + smallest_side = tf.to_float(smallest_side) + + scale = tf.cond(tf.greater(height, width), + lambda: smallest_side / width, + lambda: smallest_side / height) + new_height = tf.to_int32(tf.rint(height * scale)) + new_width = tf.to_int32(tf.rint(width * scale)) + return new_height, new_width + + +def _aspect_preserving_resize(image, smallest_side): + """Resize images preserving the original aspect ratio. + + Args: + image: A 3-D image `Tensor`. + smallest_side: A python integer or scalar `Tensor` indicating the size of + the smallest side after resize. + + Returns: + resized_image: A 3-D tensor containing the resized image. + """ + smallest_side = tf.convert_to_tensor(smallest_side, dtype=tf.int32) + + shape = tf.shape(image) + height = shape[0] + width = shape[1] + new_height, new_width = _smallest_size_at_least(height, width, smallest_side) + image = tf.expand_dims(image, 0) + resized_image = tf.image.resize_bilinear(image, [new_height, new_width], + align_corners=False) + resized_image = tf.squeeze(resized_image) + resized_image.set_shape([None, None, 3]) + return resized_image + + +def preprocess_for_train(image, + output_height, + output_width, + resize_side_min=_RESIZE_SIDE_MIN, + resize_side_max=_RESIZE_SIDE_MAX, + use_grayscale=False): + """Preprocesses the given image for training. + + Note that the actual resizing scale is sampled from + [`resize_size_min`, `resize_size_max`]. + + Args: + image: A `Tensor` representing an image of arbitrary size. + output_height: The height of the image after preprocessing. + output_width: The width of the image after preprocessing. + resize_side_min: The lower bound for the smallest side of the image for + aspect-preserving resizing. + resize_side_max: The upper bound for the smallest side of the image for + aspect-preserving resizing. + use_grayscale: Whether to convert the image from RGB to grayscale. + + Returns: + A preprocessed image. + """ + resize_side = tf.random_uniform( + [], minval=resize_side_min, maxval=resize_side_max+1, dtype=tf.int32) + + image = _aspect_preserving_resize(image, resize_side) + image = _random_crop([image], output_height, output_width)[0] + image.set_shape([output_height, output_width, 3]) + image = tf.to_float(image) + if use_grayscale: + image = tf.image.rgb_to_grayscale(image) + image = tf.image.random_flip_left_right(image) + return _mean_image_subtraction(image, [_R_MEAN, _G_MEAN, _B_MEAN]) + + +def preprocess_for_eval(image, + output_height, + output_width, + resize_side, + use_grayscale=False): + """Preprocesses the given image for evaluation. + + Args: + image: A `Tensor` representing an image of arbitrary size. + output_height: The height of the image after preprocessing. + output_width: The width of the image after preprocessing. + resize_side: The smallest side of the image for aspect-preserving resizing. + use_grayscale: Whether to convert the image from RGB to grayscale. + + Returns: + A preprocessed image. + """ + image = _aspect_preserving_resize(image, resize_side) + image = _central_crop([image], output_height, output_width)[0] + image.set_shape([output_height, output_width, 3]) + image = tf.to_float(image) + if use_grayscale: + image = tf.image.rgb_to_grayscale(image) + return _mean_image_subtraction(image, [_R_MEAN, _G_MEAN, _B_MEAN]) + + +def preprocess_image(image, + output_height, + output_width, + is_training=False, + resize_side_min=_RESIZE_SIDE_MIN, + resize_side_max=_RESIZE_SIDE_MAX, + use_grayscale=False): + """Preprocesses the given image. + + Args: + image: A `Tensor` representing an image of arbitrary size. + output_height: The height of the image after preprocessing. + output_width: The width of the image after preprocessing. + is_training: `True` if we're preprocessing the image for training and + `False` otherwise. + resize_side_min: The lower bound for the smallest side of the image for + aspect-preserving resizing. If `is_training` is `False`, then this value + is used for rescaling. + resize_side_max: The upper bound for the smallest side of the image for + aspect-preserving resizing. If `is_training` is `False`, this value is + ignored. Otherwise, the resize side is sampled from + [resize_size_min, resize_size_max]. + use_grayscale: Whether to convert the image from RGB to grayscale. + + Returns: + A preprocessed image. + """ + if is_training: + return preprocess_for_train(image, output_height, output_width, + resize_side_min, resize_side_max, + use_grayscale) + else: + return preprocess_for_eval(image, output_height, output_width, + resize_side_min, use_grayscale) diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/setup.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/setup.py new file mode 100644 index 0000000..3ec7ecd --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/setup.py @@ -0,0 +1,27 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Setup script for slim.""" + +from setuptools import find_packages +from setuptools import setup + + +setup( + name='slim', + version='0.1', + include_package_data=True, + packages=find_packages(), + description='tf-slim', +) diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/train.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/train.py new file mode 100644 index 0000000..fa9e97e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/code/train.py @@ -0,0 +1,300 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Generic training script that trains a model using a given dataset.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import datetime +import os +import ssl +import sys + +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '../'))) +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '../../../../utils/atlasboost'))) + +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter +import tensorflow as tf +from env import Env +from estimator_impl import EstimatorImpl + + +ssl._create_default_https_context = ssl._create_unverified_context + + +def parse_args(): + tf.app.flags.DEFINE_string('master', '', 'The address of the TensorFlow master to use.') + tf.app.flags.DEFINE_string('train_dir', './mobilenet_v2_result', + 'Directory where checkpoints and event logs are written to.') + tf.app.flags.DEFINE_string('msg', '', 'extra message for creating log folder') + tf.app.flags.DEFINE_string('gpu_ids', '0', 'the gpu to use') + + # cosine learning rate. [DECOUPLED WEIGHT DECAY REGULARIZATION] + tf.app.flags.DEFINE_string('checkpoint_path', '', '') + tf.app.flags.DEFINE_integer('max_epoch', '200', 'max epochs to train') + tf.app.flags.DEFINE_integer('max_train_steps', '500', 'max steps to train') + tf.app.flags.DEFINE_float('eta_min', '0.0', 'eta_min in cosine_annealing scheduler') + tf.app.flags.DEFINE_integer('T_max', '200', 'T-max in cosine_annealing scheduler') + tf.app.flags.DEFINE_integer('ckp_freq', '5000', 'Frequency (in steps) to save checkpoint') + tf.app.flags.DEFINE_integer('iterations_per_loop', None, 'Iterations per loop when running on Ascend') + + tf.app.flags.DEFINE_float('warmup_epochs', 5, + 'Linearly warmup learning rate from 0 to learning_rate over this many epochs.') + tf.app.flags.DEFINE_boolean('enable_summary', False, '') + + tf.app.flags.DEFINE_integer('num_clones', 1, + 'Number of model clones to deploy. Note For ' + 'historical reasons loss from all clones averaged ' + 'out and learning rate decay happen per clone ' + 'epochs') + + tf.app.flags.DEFINE_boolean('clone_on_cpu', False, + 'Use CPUs to deploy clones.') + + tf.app.flags.DEFINE_integer('worker_replicas', 1, 'Number of worker replicas.') + + tf.app.flags.DEFINE_integer( + 'num_ps_tasks', 0, + 'The number of parameter servers. If the value is 0, then the parameters ' + 'are handled locally by the worker.') + + tf.app.flags.DEFINE_integer( + 'num_readers', 4, + 'The number of parallel readers that read data from the dataset.') + + tf.app.flags.DEFINE_integer( + 'num_preprocessing_threads', 4, + 'The number of threads used to create the batches.') + + tf.app.flags.DEFINE_integer( + 'log_every_n_steps', 10, + 'The frequency with which logs are print.') + + tf.app.flags.DEFINE_integer( + 'save_summaries_secs', 60, + 'The frequency with which summaries are saved, in seconds.') + + tf.app.flags.DEFINE_integer( + 'save_interval_secs', 60, + 'The frequency with which the model is saved, in seconds.') + + tf.app.flags.DEFINE_integer( + 'task', 0, 'Task id of the replica running the training.') + + ###################### + # Optimization Flags # + ###################### + + tf.app.flags.DEFINE_float( + # 'weight_decay', 0.00004, 'The weight decay on the model weights.') + 'weight_decay', 0, 'The weight decay on the model weights.') + + tf.app.flags.DEFINE_string( + 'optimizer', 'sgd', + 'The name of the optimizer, one of "adadelta", "adagrad", "adam",' + '"ftrl", "momentum", "sgd" or "rmsprop".') + + tf.app.flags.DEFINE_float( + 'adadelta_rho', 0.95, + 'The decay rate for adadelta.') + + tf.app.flags.DEFINE_float( + 'adagrad_initial_accumulator_value', 0.1, + 'Starting value for the AdaGrad accumulators.') + + tf.app.flags.DEFINE_float( + 'adam_beta1', 0.9, + 'The exponential decay rate for the 1st moment estimates.') + + tf.app.flags.DEFINE_float( + 'adam_beta2', 0.999, + 'The exponential decay rate for the 2nd moment estimates.') + + tf.app.flags.DEFINE_float('opt_epsilon', 1.0, 'Epsilon term for the optimizer.') + + tf.app.flags.DEFINE_float('ftrl_learning_rate_power', -0.5, + 'The learning rate power.') + + tf.app.flags.DEFINE_float( + 'ftrl_initial_accumulator_value', 0.1, + 'Starting value for the FTRL accumulators.') + + tf.app.flags.DEFINE_float( + 'ftrl_l1', 0.0, 'The FTRL l1 regularization strength.') + + tf.app.flags.DEFINE_float( + 'ftrl_l2', 0.0, 'The FTRL l2 regularization strength.') + + tf.app.flags.DEFINE_float( + 'momentum', 0.9, + 'The momentum for the MomentumOptimizer and RMSPropOptimizer.') + + tf.app.flags.DEFINE_float('rmsprop_momentum', 0.9, 'Momentum.') + + tf.app.flags.DEFINE_float('rmsprop_decay', 0.9, 'Decay term for RMSProp.') + + tf.app.flags.DEFINE_integer( + 'quantize_delay', -1, + 'Number of steps to start quantized training. Set to -1 would disable ' + 'quantized training.') + + ####################### + # Learning Rate Flags # + ####################### + + tf.app.flags.DEFINE_string( + 'learning_rate_decay_type', + 'fixed', + 'Specifies how the learning rate is decayed. One of "fixed", "exponential",' + ' or "polynomial"') + + # tf.app.flags.DEFINE_float('learning_rate', 0.4, 'Initial learning rate.') + tf.app.flags.DEFINE_float('learning_rate', 0.1, 'Initial learning rate.') + + tf.app.flags.DEFINE_float( + 'end_learning_rate', 0.0001, + 'The minimal end learning rate used by a polynomial decay learning rate.') + + tf.app.flags.DEFINE_float( + 'label_smoothing', 0.1, 'The amount of label smoothing.') + + tf.app.flags.DEFINE_float( + 'learning_rate_decay_factor', 0.94, 'Learning rate decay factor.') + + tf.app.flags.DEFINE_float( + 'num_epochs_per_decay', 2.0, + 'Number of epochs after which learning rate decays. Note: this flag counts ' + 'epochs per clone but aggregates per sync replicas. So 1.0 means that ' + 'each clone will go over full epoch individually, but replicas will go ' + 'once across all replicas.') + + tf.app.flags.DEFINE_bool( + 'sync_replicas', False, + 'Whether or not to synchronize the replicas during training.') + + tf.app.flags.DEFINE_integer( + 'replicas_to_aggregate', 1, + 'The Number of gradients to collect before updating params.') + + tf.app.flags.DEFINE_float( + 'moving_average_decay', None, + 'The decay to use for the moving average.' + 'If left as None, then moving averages are not used.') + + ####################### + # Dataset Flags # + ####################### + + tf.app.flags.DEFINE_string( + 'dataset_name', 'imagenet', 'The name of the dataset to load.') + + tf.app.flags.DEFINE_string( + 'dataset_split_name', 'train', 'The name of the train/test split.') + + tf.app.flags.DEFINE_string( + 'dataset_dir', '/opt/npu/models/slimImagenet', 'The directory where the dataset files are stored.') + + tf.app.flags.DEFINE_integer( + 'labels_offset', 0, + 'An offset for the labels in the dataset. This flag is primarily used to ' + 'evaluate the VGG and ResNet architectures which do not use a background ' + 'class for the ImageNet dataset.') + + tf.app.flags.DEFINE_string( + 'model_name', 'mobilenet_v2_140', 'The name of the architecture to train.') + + tf.app.flags.DEFINE_string( + 'preprocessing_name', 'inception_v2', 'The name of the preprocessing to use. If left ' + 'as `None`, then the model_name flag is used.') + + tf.app.flags.DEFINE_integer( + 'batch_size', 96, 'The number of samples in each batch.') + + tf.app.flags.DEFINE_integer( + 'train_image_size', None, 'Train image size') + + tf.app.flags.DEFINE_integer('max_number_of_steps', 20000, + 'The maximum number of training steps.') + + tf.app.flags.DEFINE_bool('use_grayscale', False, + 'Whether to convert input images to grayscale.') + + ##################### + # Fine-Tuning Flags # + ##################### + # + # tf.app.flags.DEFINE_string( + # 'checkpoint_path', None, + # 'The path to a checkpoint from which to fine-tune.') + # + tf.app.flags.DEFINE_string( + 'checkpoint_exclude_scopes', None, + 'Comma-separated list of scopes of variables to exclude when restoring ' + 'from a checkpoint.') + + tf.app.flags.DEFINE_string( + 'trainable_scopes', None, + 'Comma-separated list of scopes to filter the set of variables to train.' + 'By default, None would train all the variables.') + + tf.app.flags.DEFINE_boolean( + 'ignore_missing_vars', False, + 'When restoring a checkpoint would ignore missing variables.') + + FLAGS = tf.app.flags.FLAGS + + return FLAGS + + +FLAGS = parse_args() + + +def main(_): + num_samples = 1281167 + + # import pdb;pdb.set_trace() + if int(os.getenv('RANK_SIZE')) == 1: + FLAGS.max_number_of_steps = FLAGS.max_train_steps + else: + FLAGS.max_number_of_steps = num_samples // (FLAGS.batch_size * int(os.getenv('RANK_SIZE'))) * FLAGS.max_epoch + + env = Env(FLAGS) + + estimator_impl = EstimatorImpl(env) + try: + estimator_impl.main() + except: + pass + +if __name__ == '__main__': + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("tensorflow") + config_info = get_model_parameter("tensorflow_config") + initinal_data = {"base_lr": 0.128, "dataset": "imagenet1024", "optimizer": "SGD", "loss_scale": 512, "batchsize": 32} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + hwlog.remark_print(key=hwlog.INPUT_BATCH_SIZE, value=initinal_data.get("batchsize")) + tf.app.run() diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/config/__init__.py b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/config/hccl_sample.json b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/config/hccl_sample.json new file mode 100644 index 0000000..96ec094 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/config/hccl_sample.json @@ -0,0 +1,9 @@ +{ + "server_count": "1", + "server_list": [{ + "device": [{devices}], + "server_id": "127.0.0.1" + }], + "status": "completed", + "version": "1.0" +} diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/config/npu_set_env.sh b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/config/npu_set_env.sh new file mode 100644 index 0000000..7381628 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/config/npu_set_env.sh @@ -0,0 +1,30 @@ +# main env + +if [ -d /usr/local/Ascend/nnae/latest ];then + + export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/Ascend/driver/tools/hccn_tool/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages + export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp +else + export LD_LIBRARY_PATH=/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/ascend-toolkit/latest//fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/tfplugin/python/site-packages:$projectDir + export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + +fi + +export SOC_VERSION=Ascend910 +export HCCL_CONNECT_TIMEOUT=600 + +# profiling env +export PROFILING_MODE={PROFILING_MODE} +export AICPU_PROFILING_MODE={AICPU_PROFILING_MODE} +export PROFILING_OPTIONS={PROFILING_OPTIONS} +export FP_POINT={FP_POINT} +export BP_POINT={BP_POINT} + + +# system env +ulimit -c unlimited + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/scripts/eval.sh b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/scripts/eval.sh new file mode 100644 index 0000000..b900df6 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/scripts/eval.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +yamlPath=$1 +toolsPath=$2 + + +currentDir=$(cd "$(dirname "$0")/.."; pwd) +export REMARK_LOG_FILE=hw_mobilenet.log +benchmark_log_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils +export PYTHONPATH=$PYTHONPATH:${benchmark_log_path} + +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "common_config") + +cd ${ckpt_path%results*}/results +rm -rf ./hw_mobilenet.log +rm -rf ./eval.out + +python3.7 ${currentDir}/code/eval_image_classifier_mobilenet.py --dataset_dir=${data_url} \ + --checkpoint_path=${ckpt_path} > ./eval.out 2>&1 + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/scripts/run.sh b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/scripts/run.sh new file mode 100644 index 0000000..0221abb --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/scripts/run.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +rank_size=$1 +yamlPath=$2 +toolsPath=$3 +currentDir=$(cd "$(dirname "$0")/.."; pwd) + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "tensorflow_config") +if [ -f /.dockerenv ];then + CLUSTER=$4 + MPIRUN_ALL_IP="$5" + export CLUSTER=${CLUSTER} +fi + +if [ x"$mode" != x"evaluate" ];then + currtime=`date +%Y%m%d%H%M%S` + mkdir -p ${currentDir%train*}/train/result/tf_mobilenet/training_job_${currtime}/ + train_job_dir=${currentDir%train*}/train/result/tf_mobilenet/training_job_${currtime}/ + echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir} &" +fi + +# device 列表, 若无指定 device 根据 rank_size 顺序选择 +eval device_group=\$device_group_${rank_size}p +if [ x"${device_group}" == x"" ] || [ ${rank_size} -ge 8 ];then + device_group="$(seq 0 "$(expr $rank_size - 1)")" +fi + +# get last device id in device_group, hw log in performance from the dir named last_device_id +device_group_str=`echo ${device_group} | sed 's/ //g'` +first_device_id=`echo ${device_group_str: 0:1}` + +if [ x"${CLUSTER}" == x"True" ];then + # ln hw log + ln -snf ${currentDir%train*}/train/result/tf_mobilenet/training_job_${currtime}/0/hw_mobilenet.log ${currentDir%train*}/train/result/tf_mobilenet/training_job_${currtime}/ + + this_ip=$(hostname -I |awk '{print $1}') + for ip in $MPIRUN_ALL_IP;do + if [ x"$ip" != x"$this_ip" ];then + scp $yamlPath root@$ip:$yamlPath + fi + done + export PATH=$PATH:/usr/local/mpirun4.0/bin + mpirun -H ${mpirun_ip} \ + --bind-to none -map-by slot\ + --allow-run-as-root \ + --mca btl_tcp_if_exclude lo,docker0,endvnic,virbr0,vethf40501b,docker_gwbridge,br-f42ac38052b4\ + --prefix /usr/local/mpirun4.0/ \ + ${currentDir}/scripts/train.sh 0 $rank_size $yamlPath $currtime ${toolsPath} ${CLUSTER} +elif [ x"$mode" == x"train" ];then + # ln hw log + ln -snf ${currentDir%train*}/train/result/tf_mobilenet/training_job_${currtime}/${first_device_id}/hw_mobilenet.log ${currentDir%train*}/train/result/tf_mobilenet/training_job_${currtime}/ + rank_id=0 + for device_id in $device_group;do + ${currentDir}/scripts/train.sh $device_id $rank_size $yamlPath $currtime ${toolsPath} $rank_id & + let rank_id++ + done +else + echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${ckpt_path%results*}/results &" + ln -snf ${ckpt_path%results*}/results/hw_mobilenet.log ${ckpt_path%results*}/.. + bash ${currentDir}/scripts/eval.sh ${yamlPath} ${toolsPath} + +fi +wait + + diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/scripts/train.sh b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/scripts/train.sh new file mode 100644 index 0000000..69f69c9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/tensorflow/scripts/train.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash + +device_id=$1 +rank_size=$2 +yamlPath=$3 +currentDir=$(cd "$(dirname "$0")/.."; pwd) +currtime=$4 +toolsPath=$5 + +export YAML_PATH=$3 +mkdir -p ${currentDir%train*}/train/result/tf_mobilenet/training_job_${currtime}/ +export train_job_dir=${currentDir%train*}/train/result/tf_mobilenet/training_job_${currtime}/ + + + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "tensorflow_config") +export REMARK_LOG_FILE=hw_mobilenet.log +benchmark_log_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils +export PYTHONPATH=$PYTHONPATH:${benchmark_log_path} + +source ${currentDir}/config/npu_set_env.sh + +# user env +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 +export RANK_TABLE_FILE=${currentDir}/config/${rank_size}p.json +export RANK_SIZE=${rank_size} +export SLOG_PRINT_TO_STDOUT=0 +export DEVICE_ID=${device_id} +DEVICE_INDEX=$(( DEVICE_ID + RANK_INDEX * 8 )) +export DEVICE_INDEX=${DEVICE_INDEX} + +if [ ${profiling_mode} == True ]; +then + export PROFILING_MODE=true +else + export PROFILING_MODE=false +fi + +if [ ${aicpu_profiling_mode} == True ]; +then + export AICPU_PROFILING_MODE=true +else + export AICPU_PROFILING_MODE=false +fi + +export PROFILING_OPTIONS=${profiling_options} +export FP_POINT=${fp_point} +export BP_POINT=${bp_point} + + + +cd ${train_job_dir} +curd_dir=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +export PYTHONPATH=$PYTHONPATH:${curd_dir} + +if [ x"$6" != x"True" ];then + rank_id=$6 + export RANK_ID=$6 +else + device_id_mo=$(python3.7 -c "import src.tensorflow.mpi_ops as atlasboost;atlasboost.init(); \ + device_id = atlasboost.local_rank();cluster_device_id = str(device_id); \ + atlasboost.set_device_id(device_id);print(atlasboost.rank())") + device_id_mo=`echo $device_id_mo` + rank_id=${device_id_mo##* } + export RANK_ID=${rank_id} + device=${device_id_mo##*deviceid = } + device_id=${device%% phyid=*} + export DEVICE_ID=${device_id} + hccljson=${train_job_dir}/*.json + cp ${hccljson} ${currentDir}/config/${rank_size}p.json +fi + +#mkdir exec path +mkdir -p ${train_job_dir}/${device_id} +cd ${train_job_dir}/${device_id} + +startTime=`date +%Y%m%d-%H:%M:%S` +startTime_s=`date +%s` + +if [ x"${mode}" == x"evaluate" ];then + # 评测 + python3.7 ${currentDir}/code/eval_image_classifier_mobilenet.py \ + --checkpoint_path="${ckpt_path}" \ + --dataset_dir=${data_url} > ./train.log 2>&1 +else + # 根据单卡/多卡区分调用参数 + if [ x"$6" == x"True" ];then + export CLUSTER=True + python3.7 ${currentDir}/code/train.py \ + --dataset_dir=${data_url} \ + --max_epoch=${epoches} \ + --model_name="mobilenet_v2" \ + --moving_average_decay=0.9999 \ + --label_smoothing=0.1 \ + --preprocessing_name="inception_v2" \ + --weight_decay='0.00004' \ + --batch_size=${batch_size} \ + --learning_rate_decay_type='cosine_annealing' \ + --learning_rate=0.8 \ + --optimizer='momentum' \ + --momentum='0.9' \ + --warmup_epochs=5 > ${train_job_dir}/train_${device_id}.log 2>&1 + elif [ x"${rank_size}" == x"1" ];then + # 单卡 + python3.7 ${currentDir}/code/train.py \ + --dataset_dir=${data_url} \ + --max_train_steps=${max_steps} \ + --iterations_per_loop=50 \ + --model_name="mobilenet_v2" \ + --moving_average_decay=0.9999 \ + --label_smoothing=0.1 \ + --preprocessing_name="inception_v2" \ + --weight_decay='0.00004' \ + --batch_size=${batch_size} \ + --learning_rate_decay_type='cosine_annealing' \ + --learning_rate=0.4 \ + --optimizer='momentum' \ + --momentum='0.9' \ + --warmup_epochs=5 > ${train_job_dir}/train_${device_id}.log 2>&1 + elif [ ${rank_size} -le 8 ];then + # 多卡单机 + python3.7 ${currentDir}/code/train.py \ + --dataset_dir=${data_url} \ + --max_epoch=${epoches} \ + --model_name="mobilenet_v2" \ + --moving_average_decay=0.9999 \ + --label_smoothing=0.1 \ + --preprocessing_name="inception_v2" \ + --weight_decay='0.00004' \ + --batch_size=${batch_size} \ + --learning_rate_decay_type='cosine_annealing' \ + --learning_rate=0.8 \ + --optimizer='momentum' \ + --momentum='0.9' \ + --warmup_epochs=5 > ${train_job_dir}/train_${device_id}.log 2>&1 + fi +fi + +if [ $? -eq 0 ];then + echo ":::ABK 1.0.0 hw_mobilenet train success" + echo ":::ABK 1.0.0 hw_mobilenet train success" >> ${train_job_dir}/train_${device_id}.log 2 + echo ":::ABK 1.0.0 hw_mobilenet train success" >> ./hw_mobilenet.log +else + echo ":::ABK 1.0.0 hw_mobilenet train failed" + echo ":::ABK 1.0.0 hw_mobilenet train failed" >> ${train_job_dir}/train_${device_id}.log 2 + echo ":::ABK 1.0.0 hw_mobilenet train failed" >> ./hw_mobilenet.log +fi + +endTime=`date +%Y%m%d-%H:%M:%S` +endTime_s=`date +%s` + +sumTime=$[ $endTime_s - $startTime_s ] + +hour=$(( $sumTime/3600 )) +min=$(( ($sumTime-${hour}*3600)/60 )) +sec=$(( $sumTime-${hour}*3600-${min}*60 )) +echo ":::ABK 1.0.0 mobilenet train total time：${hour}:${min}:${sec}" + +echo ":::ABK 1.0.0 mobilenet train total time： ${hour}:${min}:${sec}" >> ./hw_mobilenet.log diff --git a/train/atlas_benchmark-master/image_classification/MobileNet/verify_dataset.sh b/train/atlas_benchmark-master/image_classification/MobileNet/verify_dataset.sh new file mode 100644 index 0000000..a9bf588 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/MobileNet/verify_dataset.sh @@ -0,0 +1 @@ +#!/bin/bash diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/README.md new file mode 100644 index 0000000..87e3965 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/README.md @@ -0,0 +1,37 @@ +# ResNet101_tensorflow训练说明 + +### 1. 模型训练参数配置 + +在train/yaml/ResNet101.yaml中修改相应配置， 配置项含义: + +``` +tensorflow_config: + # 基本参数 + data_url: /home/imagenet_TF/ + # 1p/8p,epoches设为150 + epoches: 1 + epochs_between_evals: 1 + max_train_steps: 1000 + batch_size: 128 + + # 仅多机执行需要配置: ip1:卡数量1,ip2:卡数量2 + mpirun_ip: 90.90.176.152:8,90.90.176.154:8 + + # docker 镜像名称:版本号 + docker_image: c73:b02 + + # 指定 device id, 多个 id 使用空格分隔, 数量需与 rank_size 相同 + device_group_1p: 0 + device_group_2p: 0 1 + device_group_4p: 0 1 2 3 +``` + +------ + + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/LICENSE b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/LICENSE new file mode 100644 index 0000000..d3da228 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/LICENSE @@ -0,0 +1,203 @@ +Copyright 2015 The TensorFlow Authors. All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015, The TensorFlow Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/README-GPU.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/README-GPU.md new file mode 100644 index 0000000..ead609e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/README-GPU.md @@ -0,0 +1,151 @@ +![Logo](https://storage.googleapis.com/model_garden_artifacts/TF_Model_Garden.png) + +# TensorFlow Official Models + +The TensorFlow official models are a collection of models +that use TensorFlow’s high-level APIs. +They are intended to be well-maintained, tested, and kept up to date +with the latest TensorFlow API. +They should also be reasonably optimized for fast performance while still +being easy to read. +These models are used as end-to-end tests, ensuring that the models run +with the same or improved speed and performance with each new TensorFlow build. + +## Model Implementations + +### Natural Language Processing + +| Model | Description | Reference | +| ----- | ----------- | --------- | +| [ALBERT](nlp/albert) | A Lite BERT for Self-supervised Learning of Language Representations | [arXiv:1909.11942](https://arxiv.org/abs/1909.11942) | +| [BERT](nlp/bert) | A powerful pre-trained language representation model: BERT (Bidirectional Encoder Representations from Transformers) | [arXiv:1810.04805](https://arxiv.org/abs/1810.04805) | +| [NHNet](nlp/nhnet) | A transformer-based multi-sequence to sequence model: Generating Representative Headlines for News Stories | [arXiv:2001.09386](https://arxiv.org/abs/2001.09386) | +| [Transformer](nlp/transformer) | A transformer model to translate the WMT English to German dataset | [arXiv:1706.03762](https://arxiv.org/abs/1706.03762) | +| [XLNet](nlp/xlnet) | XLNet: Generalized Autoregressive Pretraining for Language Understanding | [arXiv:1906.08237](https://arxiv.org/abs/1906.08237) | + +### Computer Vision + +| Model | Description | Reference | +| ----- | ----------- | --------- | +| [MNIST](vision/image_classification) | A basic model to classify digits from the MNIST dataset | [Link](http://yann.lecun.com/exdb/mnist/) | +| [ResNet](vision/image_classification) | A deep residual network for image recognition | [arXiv:1512.03385](https://arxiv.org/abs/1512.03385) | +| [RetinaNet](vision/detection) | A fast and powerful object detector | [arXiv:1708.02002](https://arxiv.org/abs/1708.02002) | +| [Mask R-CNN](vision/detection) | An object detection and instance segmentation model | [arXiv:1703.06870](https://arxiv.org/abs/1703.06870) | + +### Other models + +| Model | Description | Reference | +| ----- | ----------- | --------- | +| [NCF](recommendation) | Neural Collaborative Filtering model for recommendation tasks | [arXiv:1708.05031](https://arxiv.org/abs/1708.05031) | + +--- + +## How to get started with the Model Garden official models + +* The models in the master branch are developed using TensorFlow 2, +and they target the TensorFlow [nightly binaries](https://github.com/tensorflow/tensorflow#installation) +built from the +[master branch of TensorFlow](https://github.com/tensorflow/tensorflow/tree/master). +* The stable versions targeting releases of TensorFlow are available +as tagged branches or [downloadable releases](https://github.com/tensorflow/models/releases). +* Model repository version numbers match the target TensorFlow release, +such that +[release v2.1.0](https://github.com/tensorflow/models/releases/tag/v2.1.0) +are compatible with +[TensorFlow v2.1.0](https://github.com/tensorflow/tensorflow/releases/tag/v2.1.0). + +Please follow the below steps before running models in this repository. + +### Requirements + +* The latest TensorFlow Model Garden release and TensorFlow 2 + * If you are on a version of TensorFlow earlier than 2.1, please +upgrade your TensorFlow to [the latest TensorFlow 2](https://www.tensorflow.org/install/). + +```shell +pip3 install tf-nightly +``` + +### Installation + +#### Method 1: Install the TensorFlow Model Garden pip package + +**tf-models-nightly** is the nightly Model Garden package +created daily automatically. pip will install all models +and dependencies automatically. + +```shell +pip install tf-models-nightly +``` + +Please check out our [example](colab/bert.ipynb) +to learn how to use a PIP package. + +#### Method 2: Clone the source + +1. Clone the GitHub repository: + +```shell +git clone https://github.com/tensorflow/models.git +``` + +2. Add the top-level ***/models*** folder to the Python path. + +```shell +export PYTHONPATH=$PYTHONPATH:/path/to/models +``` + +If you are using a Colab notebook, please set the Python path with os.environ. + +```python +import os +os.environ['PYTHONPATH'] += ":/path/to/models" +``` + +3. Install other dependencies + +```shell +pip3 install --user -r official/requirements.txt +``` + +--- + +## More models to come! + +The team is actively developing new models. +In the near future, we will add: + +- State-of-the-art language understanding models: + More members in Transformer family +- Start-of-the-art image classification models: + EfficientNet, MnasNet and variants. +- A set of excellent objection detection models. + +If you would like to make any fixes or improvements to the models, please +[submit a pull request](https://github.com/tensorflow/models/compare). + +--- + +## Contributions + +Every model should follow our guidelines to uphold our objectives of readable, +usable, and maintainable code. + +### General Guidelines + +- Code should be well documented and tested. +- Runnable from a blank environment with ease. +- Trainable on: single GPU/CPU (baseline), multiple GPUs & TPUs +- Compatible with Python 3 (using [six](https://pythonhosted.org/six/) +when being compatible with Python 2 is necessary) +- Conform to + [Google Python Style Guide](https://github.com/google/styleguide/blob/gh-pages/pyguide.md) + +### Implementation Guidelines + +These guidelines are to ensure consistent model implementations for +better readability and maintainability. + +- Use [common utility functions](utils) +- Export SavedModel at the end of the training. +- Consistent flags and flag-parsing library ([read more here](utils/flags/guidelines.md)) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/README-TPU.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/README-TPU.md new file mode 100644 index 0000000..8a54f95 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/README-TPU.md @@ -0,0 +1,25 @@ +# Offically Supported TensorFlow 2.1+ Models on Cloud TPU + +## Natural Language Processing + +* [bert](nlp/bert): A powerful pre-trained language representation model: + BERT, which stands for Bidirectional Encoder Representations from + Transformers. + [BERT FineTuning with Cloud TPU](https://cloud.google.com/tpu/docs/tutorials/bert-2.x) provides step by step instructions on Cloud TPU training. You can look [Bert MNLI Tensorboard.dev metrics](https://tensorboard.dev/experiment/LijZ1IrERxKALQfr76gndA) for MNLI fine tuning task. +* [transformer](nlp/transformer): A transformer model to translate the WMT + English to German dataset. + [Training transformer on Cloud TPU](https://cloud.google.com/tpu/docs/tutorials/transformer-2.x) for step by step instructions on Cloud TPU training. + +## Computer Vision + +* [efficientnet](vision/image_classification): A family of convolutional + neural networks that scale by balancing network depth, width, and + resolution and can be used to classify ImageNet's dataset of 1000 classes. + See [Tensorboard.dev training metrics](https://tensorboard.dev/experiment/KnaWjrq5TXGfv0NW5m7rpg/#scalars). +* [mnist](vision/image_classification): A basic model to classify digits + from the MNIST dataset. See [Running MNIST on Cloud TPU](https://cloud.google.com/tpu/docs/tutorials/mnist-2.x) tutorial and [Tensorboard.dev metrics](https://tensorboard.dev/experiment/mIah5lppTASvrHqWrdr6NA). +* [mask-rcnn](vision/detection): An object detection and instance segmentation model. See [Tensorboard.dev training metrics](https://tensorboard.dev/experiment/LH7k0fMsRwqUAcE09o9kPA). +* [resnet](vision/image_classification): A deep residual network that can + be used to classify ImageNet's dataset of 1000 classes. + See [Training ResNet on Cloud TPU](https://cloud.google.com/tpu/docs/tutorials/resnet-2.x) tutorial and [Tensorboard.dev metrics](https://tensorboard.dev/experiment/CxlDK8YMRrSpYEGtBRpOhg). +* [retinanet](vision/detection): A fast and powerful object detector. See [Tensorboard.dev training metrics](https://tensorboard.dev/experiment/b8NRnWU3TqG6Rw0UxueU6Q). diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/README.md new file mode 100644 index 0000000..8a5bd10 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/README.md @@ -0,0 +1,79 @@ +# ResNet in TensorFlow On NPU +--- + +# Classification Model +## Overview +1. This is an implementation of the ResNet101 model as described in the [Deep Residual Learning for Image Recognition](https://arxiv.org/pdf/1512.03385.pdf) paper. +2. Current implementation is based on the code from the TensorFlow Official implementation in the [tensorflow/models Repo](https://github.com/tensorflow/models). + +## Introduction +1. ResNet is a relatively good network for classification problems in the ImageNet competition. It introduces the concept of residual learning. It protects the integrity of information by adding direct connection channels, solves problems such as information loss, gradient disappearance, and gradient explosion. The network can also be trained. ResNet has different network layers, commonly used are 18-layer, 34-layer, 50-layer, 101-layer, 152-layer. +2. Ascend provides the V1.5 version of the 50-layer ResNet network this time. The difference between the V1.5 version of the ResNet network and the V1 version is that in the bottleneck module, the V1 version is set stride=2 in the first 1x1 convolutional layer, and V1.5 sets stride=2 in the 3x3 convolutional layer. + +## Dataset +We have used the [ImageNet](http://www.image-net.org/)dataset as an example here, you can use mnist or your own dataset to modify and adapt. +We use [build_imagenet_data](https://github.com/tensorflow/models/blob/1af55e018eebce03fb61bba9959a04672536107d/research/slim/datasets/build_imagenet_data.py) to build record for training. + +## Running Code +### Config the env paramater +check if path '/usr/local/HiAI' or ''/usr/local/Ascend' is existed or not. +modify '/usr/local/HiAI' to the actual path in scripts/run.sh + +### Train and evaluate model +[imagenet_main.py](official/r1/resnet/imagenet_main.py) is the Entry Python script. +[resnet_run_loop.py](official/r1/resnet/resnet_run_loop.py) is the Main Python script. + +### Check your rank_table +default rank_table setting in [configs](official/r1/resnet/configs) is usrd for X86. +if you use aach64, please modify board_id from "0x0000" --> + +To train and evaluate the model, issue the following command: +``` +# for single training +bash ./scripts/train_1p.sh +# for multi training +bash ./scripts/train_8p.sh +``` + +Default Args: +- Batch size: 128 +- Momentum: 0.9 +- LR scheduler: cosine +- Learning rate(LR): 0.064 +- loss scale: 512 +- Weight decay: 0.0001 +- Label smoothing: 0.1 +- train epoch: 90 + +There are other arguments about models and training process. Use the `--help` or `-h` flag to get a full list of possible arguments with detailed descriptions. + +### Train and evaluate result +- 1 NPU + - Train performance：109ms/step，1170images/sec. +- 8 NPU + - Train performance：109ms/step，9390images/sec. +- best result + - Accuracy(Top1): 79.03 + - Accuracy(Top5): 94.53 + +### More + +#### modify file +- The npu modify file list as follows: +- DaVinci npu platform adaptation code,including + 1.official/r1/resnet/imagenet_main.py + 2.official/r1/resnet/resnet_model.py + 3.official/r1/resnet/resnet_run_loop.py + 4.official/utils/flags/_base.py + +#### FileTree Intro +- Main Dir + - ./official/r1/resnet +- Single NPU Training Shell + - npu_train_1p_test.sh +- Multi NPU(8p) Training Shell + - npu_train_8p_test.sh +- Log Info + - STDOUT nohup.out + - Performance perf.log + \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/benchmark_uploader.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/benchmark_uploader.py new file mode 100644 index 0000000..5946d96 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/benchmark_uploader.py @@ -0,0 +1,157 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Library to upload benchmark generated by BenchmarkLogger to remote repo. + +This library require google cloud bigquery lib as dependency, which can be +installed with: + > pip install --upgrade google-cloud-bigquery +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json + +from google.cloud import bigquery +from google.cloud import exceptions + +import tensorflow as tf + + +class BigQueryUploader(object): + """Upload the benchmark and metric info from JSON input to BigQuery. """ + + def __init__(self, gcp_project=None, credentials=None): + """Initialized BigQueryUploader with proper setting. + + Args: + gcp_project: string, the name of the GCP project that the log will be + uploaded to. The default project name will be detected from local + environment if no value is provided. + credentials: google.auth.credentials. The credential to access the + BigQuery service. The default service account credential will be + detected from local environment if no value is provided. Please use + google.oauth2.service_account.Credentials to load credential from local + file for the case that the test is run out side of GCP. + """ + self._bq_client = bigquery.Client( + project=gcp_project, credentials=credentials) + + def upload_benchmark_run_json( + self, dataset_name, table_name, run_id, run_json): + """Upload benchmark run information to Bigquery. + + Args: + dataset_name: string, the name of bigquery dataset where the data will be + uploaded. + table_name: string, the name of bigquery table under the dataset where + the data will be uploaded. + run_id: string, a unique ID that will be attached to the data, usually + this is a UUID4 format. + run_json: dict, the JSON data that contains the benchmark run info. + """ + run_json["model_id"] = run_id + self._upload_json(dataset_name, table_name, [run_json]) + + def upload_benchmark_metric_json( + self, dataset_name, table_name, run_id, metric_json_list): + """Upload metric information to Bigquery. + + Args: + dataset_name: string, the name of bigquery dataset where the data will be + uploaded. + table_name: string, the name of bigquery table under the dataset where + the metric data will be uploaded. This is different from the + benchmark_run table. + run_id: string, a unique ID that will be attached to the data, usually + this is a UUID4 format. This should be the same as the benchmark run_id. + metric_json_list: list, a list of JSON object that record the metric info. + """ + for m in metric_json_list: + m["run_id"] = run_id + self._upload_json(dataset_name, table_name, metric_json_list) + + def upload_benchmark_run_file( + self, dataset_name, table_name, run_id, run_json_file): + """Upload benchmark run information to Bigquery from input json file. + + Args: + dataset_name: string, the name of bigquery dataset where the data will be + uploaded. + table_name: string, the name of bigquery table under the dataset where + the data will be uploaded. + run_id: string, a unique ID that will be attached to the data, usually + this is a UUID4 format. + run_json_file: string, the file path that contains the run JSON data. + """ + with tf.io.gfile.GFile(run_json_file) as f: + benchmark_json = json.load(f) + self.upload_benchmark_run_json( + dataset_name, table_name, run_id, benchmark_json) + + def upload_metric_file( + self, dataset_name, table_name, run_id, metric_json_file): + """Upload metric information to Bigquery from input json file. + + Args: + dataset_name: string, the name of bigquery dataset where the data will be + uploaded. + table_name: string, the name of bigquery table under the dataset where + the metric data will be uploaded. This is different from the + benchmark_run table. + run_id: string, a unique ID that will be attached to the data, usually + this is a UUID4 format. This should be the same as the benchmark run_id. + metric_json_file: string, the file path that contains the metric JSON + data. + """ + with tf.io.gfile.GFile(metric_json_file) as f: + metrics = [] + for line in f: + metrics.append(json.loads(line.strip())) + self.upload_benchmark_metric_json( + dataset_name, table_name, run_id, metrics) + + def _upload_json(self, dataset_name, table_name, json_list): + # Find the unique table reference based on dataset and table name, so that + # the data can be inserted to it. + table_ref = self._bq_client.dataset(dataset_name).table(table_name) + errors = self._bq_client.insert_rows_json(table_ref, json_list) + if errors: + tf.logging.error( + "Failed to upload benchmark info to bigquery: {}".format(errors)) + + def insert_run_status(self, dataset_name, table_name, run_id, run_status): + """Insert the run status in to Bigquery run status table.""" + query = ("INSERT {ds}.{tb} " + "(run_id, status) " + "VALUES('{rid}', '{status}')").format( + ds=dataset_name, tb=table_name, rid=run_id, status=run_status) + try: + self._bq_client.query(query=query).result() + except exceptions.GoogleCloudError as e: + tf.logging.error("Failed to insert run status: %s", e) + + def update_run_status(self, dataset_name, table_name, run_id, run_status): + """Update the run status in in Bigquery run status table.""" + query = ("UPDATE {ds}.{tb} " + "SET status = '{status}' " + "WHERE run_id = '{rid}'").format( + ds=dataset_name, tb=table_name, status=run_status, rid=run_id) + try: + self._bq_client.query(query=query).result() + except exceptions.GoogleCloudError as e: + tf.logging.error("Failed to update run status: %s", e) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/benchmark_uploader_main.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/benchmark_uploader_main.py new file mode 100644 index 0000000..e015051 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/benchmark_uploader_main.py @@ -0,0 +1,66 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Binary to upload benchmark generated by BenchmarkLogger to remote repo. + +This library require google cloud bigquery lib as dependency, which can be +installed with: + > pip install --upgrade google-cloud-bigquery +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys +import uuid + +from absl import app as absl_app +from absl import flags + +from official.benchmark import benchmark_uploader +from official.utils.flags import core as flags_core +from official.utils.logs import logger + +def main(_): + if not flags.FLAGS.benchmark_log_dir: + print("Usage: benchmark_uploader.py --benchmark_log_dir=/some/dir") + sys.exit(1) + + uploader = benchmark_uploader.BigQueryUploader( + gcp_project=flags.FLAGS.gcp_project) + run_id = str(uuid.uuid4()) + run_json_file = os.path.join( + flags.FLAGS.benchmark_log_dir, logger.BENCHMARK_RUN_LOG_FILE_NAME) + metric_json_file = os.path.join( + flags.FLAGS.benchmark_log_dir, logger.METRIC_LOG_FILE_NAME) + + uploader.upload_benchmark_run_file( + flags.FLAGS.bigquery_data_set, flags.FLAGS.bigquery_run_table, run_id, + run_json_file) + uploader.upload_metric_file( + flags.FLAGS.bigquery_data_set, flags.FLAGS.bigquery_metric_table, run_id, + metric_json_file) + # Assume the run finished successfully before user invoke the upload script. + uploader.insert_run_status( + flags.FLAGS.bigquery_data_set, flags.FLAGS.bigquery_run_status_table, + run_id, logger.RUN_STATUS_SUCCESS) + + +if __name__ == "__main__": + flags_core.define_benchmark() + flags.adopt_module_key_flags(flags_core) + absl_app.run(main=main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/benchmark_uploader_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/benchmark_uploader_test.py new file mode 100644 index 0000000..c0ab282 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/benchmark_uploader_test.py @@ -0,0 +1,123 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Tests for benchmark_uploader.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import os +import tempfile +import unittest +from mock import MagicMock +from mock import patch + +import tensorflow as tf # pylint: disable=g-bad-import-order + +try: + from google.cloud import bigquery + from official.benchmark import benchmark_uploader +except ImportError: + bigquery = None + benchmark_uploader = None + + +@unittest.skipIf(bigquery is None, "Bigquery dependency is not installed.") +class BigQueryUploaderTest(tf.test.TestCase): + + @patch.object(bigquery, "Client") + def setUp(self, mock_bigquery): + self.mock_client = mock_bigquery.return_value + self.mock_dataset = MagicMock(name="dataset") + self.mock_table = MagicMock(name="table") + self.mock_client.dataset.return_value = self.mock_dataset + self.mock_dataset.table.return_value = self.mock_table + self.mock_client.insert_rows_json.return_value = [] + + self.benchmark_uploader = benchmark_uploader.BigQueryUploader() + self.benchmark_uploader._bq_client = self.mock_client + + self.log_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + with open(os.path.join(self.log_dir, "metric.log"), "a") as f: + json.dump({"name": "accuracy", "value": 1.0}, f) + f.write("\n") + json.dump({"name": "loss", "value": 0.5}, f) + f.write("\n") + with open(os.path.join(self.log_dir, "run.log"), "w") as f: + json.dump({"model_name": "value"}, f) + + def tearDown(self): + tf.io.gfile.rmtree(self.get_temp_dir()) + + def test_upload_benchmark_run_json(self): + self.benchmark_uploader.upload_benchmark_run_json( + "dataset", "table", "run_id", {"model_name": "value"}) + + self.mock_client.insert_rows_json.assert_called_once_with( + self.mock_table, [{"model_name": "value", "model_id": "run_id"}]) + + def test_upload_benchmark_metric_json(self): + metric_json_list = [ + {"name": "accuracy", "value": 1.0}, + {"name": "loss", "value": 0.5} + ] + expected_params = [ + {"run_id": "run_id", "name": "accuracy", "value": 1.0}, + {"run_id": "run_id", "name": "loss", "value": 0.5} + ] + self.benchmark_uploader.upload_benchmark_metric_json( + "dataset", "table", "run_id", metric_json_list) + self.mock_client.insert_rows_json.assert_called_once_with( + self.mock_table, expected_params) + + def test_upload_benchmark_run_file(self): + self.benchmark_uploader.upload_benchmark_run_file( + "dataset", "table", "run_id", os.path.join(self.log_dir, "run.log")) + + self.mock_client.insert_rows_json.assert_called_once_with( + self.mock_table, [{"model_name": "value", "model_id": "run_id"}]) + + def test_upload_metric_file(self): + self.benchmark_uploader.upload_metric_file( + "dataset", "table", "run_id", + os.path.join(self.log_dir, "metric.log")) + expected_params = [ + {"run_id": "run_id", "name": "accuracy", "value": 1.0}, + {"run_id": "run_id", "name": "loss", "value": 0.5} + ] + self.mock_client.insert_rows_json.assert_called_once_with( + self.mock_table, expected_params) + + def test_insert_run_status(self): + self.benchmark_uploader.insert_run_status( + "dataset", "table", "run_id", "status") + expected_query = ("INSERT dataset.table " + "(run_id, status) " + "VALUES('run_id', 'status')") + self.mock_client.query.assert_called_once_with(query=expected_query) + + def test_update_run_status(self): + self.benchmark_uploader.update_run_status( + "dataset", "table", "run_id", "status") + expected_query = ("UPDATE dataset.table " + "SET status = 'status' " + "WHERE run_id = 'run_id'") + self.mock_client.query.assert_called_once_with(query=expected_query) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/benchmark_wrappers.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/benchmark_wrappers.py new file mode 100644 index 0000000..3d38b69 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/benchmark_wrappers.py @@ -0,0 +1,97 @@ +# Lint as: python3 +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utils to annotate and trace benchmarks.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import flags +from absl import logging +from absl.testing import flagsaver + +FLAGS = flags.FLAGS + +flags.DEFINE_multi_string( + 'benchmark_method_flags', None, + 'Optional list of runtime flags of the form key=value. Specify ' + 'multiple times to specify different flags. These will override the FLAGS ' + 'object directly after hardcoded settings in individual benchmark methods ' + 'before they call _run_and_report benchmark. Example if we set ' + '--benchmark_method_flags=train_steps=10 and a benchmark method hardcodes ' + 'FLAGS.train_steps=10000 and later calls _run_and_report_benchmark, ' + 'it\'ll only run for 10 steps. This is useful for ' + 'debugging/profiling workflows.') + + +def enable_runtime_flags(decorated_func): + """Sets attributes from --benchmark_method_flags for method execution. + + @enable_runtime_flags decorator temporarily adds flags passed in via + --benchmark_method_flags and runs the decorated function in that context. + + A user can set --benchmark_method_flags=train_steps=5 to run the benchmark + method in the snippet below with FLAGS.train_steps=5 for debugging (without + modifying the benchmark code). + + class ModelBenchmark(): + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self): + # run benchmark ... + # report benchmark results ... + + def benchmark_method(self): + FLAGS.train_steps = 1000 + ... + self._run_and_report_benchmark() + + Args: + decorated_func: The method that runs the benchmark after previous setup + execution that set some flags. + + Returns: + new_func: The same method which executes in a temporary context where flag + overrides from --benchmark_method_flags are active. + """ + + def runner(*args, **kwargs): + """Creates a temporary context to activate --benchmark_method_flags.""" + if FLAGS.benchmark_method_flags: + saved_flag_values = flagsaver.save_flag_values() + for key_value in FLAGS.benchmark_method_flags: + key, value = key_value.split('=', 1) + try: + numeric_float = float(value) + numeric_int = int(numeric_float) + if abs(numeric_int) == abs(numeric_float): + flag_value = numeric_int + else: + flag_value = numeric_float + except ValueError: + flag_value = value + logging.info('Setting --%s=%s', key, flag_value) + setattr(FLAGS, key, flag_value) + else: + saved_flag_values = None + try: + result = decorated_func(*args, **kwargs) + return result + finally: + if saved_flag_values: + flagsaver.restore_flag_values(saved_flag_values) + + return runner diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/bert_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/bert_benchmark.py new file mode 100644 index 0000000..79d9dd7 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/bert_benchmark.py @@ -0,0 +1,354 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Executes BERT benchmarks and accuracy tests.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools +import json +import math +import os +import time + +# pylint: disable=g-bad-import-order +from absl import flags +from absl.testing import flagsaver +import tensorflow as tf +# pylint: enable=g-bad-import-order + +from official.benchmark import bert_benchmark_utils as benchmark_utils +from official.nlp.bert import configs +from official.nlp.bert import run_classifier +from official.utils.misc import distribution_utils +from official.benchmark import benchmark_wrappers + +# pylint: disable=line-too-long +PRETRAINED_CHECKPOINT_PATH = 'gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16/bert_model.ckpt' +CLASSIFIER_TRAIN_DATA_PATH = 'gs://tf-perfzero-data/bert/classification/mrpc_train.tf_record' +CLASSIFIER_EVAL_DATA_PATH = 'gs://tf-perfzero-data/bert/classification/mrpc_eval.tf_record' +CLASSIFIER_INPUT_META_DATA_PATH = 'gs://tf-perfzero-data/bert/classification/mrpc_meta_data' +MODEL_CONFIG_FILE_PATH = 'gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16/bert_config.json' +# pylint: enable=line-too-long + +TMP_DIR = os.getenv('TMPDIR') +FLAGS = flags.FLAGS + + +class BertClassifyBenchmarkBase(benchmark_utils.BertBenchmarkBase): + """Base class to hold methods common to test classes in the module.""" + + def __init__(self, output_dir=None, tpu=None): + super(BertClassifyBenchmarkBase, self).__init__(output_dir) + self.num_epochs = None + self.num_steps_per_epoch = None + self.tpu = tpu + FLAGS.steps_per_loop = 50 + + @flagsaver.flagsaver + def _run_bert_classifier(self, callbacks=None, use_ds=True): + """Starts BERT classification task.""" + with tf.io.gfile.GFile(FLAGS.input_meta_data_path, 'rb') as reader: + input_meta_data = json.loads(reader.read().decode('utf-8')) + + bert_config = configs.BertConfig.from_json_file(FLAGS.bert_config_file) + epochs = self.num_epochs if self.num_epochs else FLAGS.num_train_epochs + if self.num_steps_per_epoch: + steps_per_epoch = self.num_steps_per_epoch + else: + train_data_size = input_meta_data['train_data_size'] + steps_per_epoch = int(train_data_size / FLAGS.train_batch_size) + warmup_steps = int(epochs * steps_per_epoch * 0.1) + eval_steps = int( + math.ceil(input_meta_data['eval_data_size'] / FLAGS.eval_batch_size)) + if self.tpu: + strategy = distribution_utils.get_distribution_strategy( + distribution_strategy='tpu', tpu_address=self.tpu) + else: + strategy = distribution_utils.get_distribution_strategy( + distribution_strategy='mirrored' if use_ds else 'off', + num_gpus=self.num_gpus) + + max_seq_length = input_meta_data['max_seq_length'] + train_input_fn = run_classifier.get_dataset_fn( + FLAGS.train_data_path, + max_seq_length, + FLAGS.train_batch_size, + is_training=True) + eval_input_fn = run_classifier.get_dataset_fn( + FLAGS.eval_data_path, + max_seq_length, + FLAGS.eval_batch_size, + is_training=False) + run_classifier.run_bert_classifier( + strategy, + bert_config, + input_meta_data, + FLAGS.model_dir, + epochs, + steps_per_epoch, + FLAGS.steps_per_loop, + eval_steps, + warmup_steps, + FLAGS.learning_rate, + FLAGS.init_checkpoint, + train_input_fn, + eval_input_fn, + custom_callbacks=callbacks) + + +class BertClassifyBenchmarkReal(BertClassifyBenchmarkBase): + """Short benchmark performance tests for BERT model. + + Tests BERT classification performance in different GPU, TPU configurations. + The naming convention of below test cases follow + `benchmark_(number of gpus)_gpu_(dataset type)` for GPUs and + `benchmark_(topology)_tpu_(dataset type)` for TPUs. + """ + + def __init__(self, output_dir=TMP_DIR, tpu=None, **kwargs): + super(BertClassifyBenchmarkReal, self).__init__( + output_dir=output_dir, tpu=tpu) + + self.train_data_path = CLASSIFIER_TRAIN_DATA_PATH + self.eval_data_path = CLASSIFIER_EVAL_DATA_PATH + self.bert_config_file = MODEL_CONFIG_FILE_PATH + self.input_meta_data_path = CLASSIFIER_INPUT_META_DATA_PATH + + # Since we only care about performance metrics, we limit + # the number of training steps and epochs to prevent unnecessarily + # long tests. + self.num_steps_per_epoch = 100 + self.num_epochs = 1 + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self, + training_summary_path, + min_accuracy=0, + max_accuracy=1, + use_ds=True): + """Starts BERT performance benchmark test.""" + start_time_sec = time.time() + self._run_bert_classifier(callbacks=[self.timer_callback], use_ds=use_ds) + wall_time_sec = time.time() - start_time_sec + + with tf.io.gfile.GFile(training_summary_path, 'rb') as reader: + summary = json.loads(reader.read().decode('utf-8')) + + # Since we do not load from any pretrained checkpoints, we ignore all + # accuracy metrics. + summary.pop('eval_metrics', None) + summary['start_time_sec'] = start_time_sec + + super(BertClassifyBenchmarkReal, self)._report_benchmark( + stats=summary, + wall_time_sec=wall_time_sec, + min_accuracy=min_accuracy, + max_accuracy=max_accuracy) + + def benchmark_1_gpu_mrpc(self): + """Test BERT model performance with 1 GPU.""" + + self._setup() + self.num_gpus = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_mrpc') + FLAGS.train_data_path = self.train_data_path + FLAGS.eval_data_path = self.eval_data_path + FLAGS.input_meta_data_path = self.input_meta_data_path + FLAGS.bert_config_file = self.bert_config_file + FLAGS.train_batch_size = 4 + FLAGS.eval_batch_size = 4 + + summary_path = os.path.join(FLAGS.model_dir, + 'summaries/training_summary.txt') + self._run_and_report_benchmark(summary_path) + + def benchmark_1_gpu_mrpc_xla(self): + """Test BERT model performance with 1 GPU.""" + + self._setup() + self.num_gpus = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_mrpc_xla') + FLAGS.train_data_path = self.train_data_path + FLAGS.eval_data_path = self.eval_data_path + FLAGS.input_meta_data_path = self.input_meta_data_path + FLAGS.bert_config_file = self.bert_config_file + FLAGS.train_batch_size = 4 + FLAGS.eval_batch_size = 4 + FLAGS.enable_xla = True + + summary_path = os.path.join(FLAGS.model_dir, + 'summaries/training_summary.txt') + self._run_and_report_benchmark(summary_path) + + def benchmark_1_gpu_mrpc_no_dist_strat(self): + """Test BERT model performance with 1 GPU, no distribution strategy.""" + + self._setup() + self.num_gpus = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_mrpc_no_dist_strat') + FLAGS.train_data_path = self.train_data_path + FLAGS.eval_data_path = self.eval_data_path + FLAGS.input_meta_data_path = self.input_meta_data_path + FLAGS.bert_config_file = self.bert_config_file + FLAGS.train_batch_size = 4 + FLAGS.eval_batch_size = 4 + + summary_path = os.path.join(FLAGS.model_dir, + 'summaries/training_summary.txt') + self._run_and_report_benchmark(summary_path, use_ds=False) + + def benchmark_8_gpu_mrpc(self): + """Test BERT model performance with 8 GPUs.""" + + self._setup() + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_mrpc') + FLAGS.train_data_path = self.train_data_path + FLAGS.eval_data_path = self.eval_data_path + FLAGS.input_meta_data_path = self.input_meta_data_path + FLAGS.bert_config_file = self.bert_config_file + + summary_path = os.path.join(FLAGS.model_dir, + 'summaries/training_summary.txt') + self._run_and_report_benchmark(summary_path) + + def benchmark_1_gpu_amp_mrpc_no_dist_strat(self): + """Performance for 1 GPU no DS with automatic mixed precision.""" + self._setup() + self.num_gpus = 1 + FLAGS.model_dir = self._get_model_dir( + 'benchmark_1_gpu_amp_mrpc_no_dist_strat') + FLAGS.train_data_path = self.train_data_path + FLAGS.eval_data_path = self.eval_data_path + FLAGS.input_meta_data_path = self.input_meta_data_path + FLAGS.bert_config_file = self.bert_config_file + FLAGS.train_batch_size = 4 + FLAGS.eval_batch_size = 4 + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + + summary_path = os.path.join(FLAGS.model_dir, + 'summaries/training_summary.txt') + self._run_and_report_benchmark(summary_path, use_ds=False) + + def benchmark_8_gpu_amp_mrpc(self): + """Test BERT model performance with 8 GPUs with automatic mixed precision. + """ + + self._setup() + self.num_gpus = 8 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_amp_mrpc') + FLAGS.train_data_path = self.train_data_path + FLAGS.eval_data_path = self.eval_data_path + FLAGS.input_meta_data_path = self.input_meta_data_path + FLAGS.bert_config_file = self.bert_config_file + FLAGS.train_batch_size = 32 + FLAGS.eval_batch_size = 32 + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + + summary_path = os.path.join(FLAGS.model_dir, + 'summaries/training_summary.txt') + self._run_and_report_benchmark(summary_path, use_ds=False) + + def benchmark_2x2_tpu_mrpc(self): + """Test BERT model performance with 2x2 TPU.""" + + self._setup() + FLAGS.model_dir = self._get_model_dir('benchmark_2x2_tpu_mrpc') + FLAGS.train_data_path = self.train_data_path + FLAGS.eval_data_path = self.eval_data_path + FLAGS.input_meta_data_path = self.input_meta_data_path + FLAGS.bert_config_file = self.bert_config_file + FLAGS.train_batch_size = 32 + FLAGS.eval_batch_size = 32 + + summary_path = os.path.join(FLAGS.model_dir, + 'summaries/training_summary.txt') + self._run_and_report_benchmark(summary_path, use_ds=False) + + +class BertClassifyAccuracy(BertClassifyBenchmarkBase): + """Short accuracy test for BERT model. + + Tests BERT classification task model accuracy. The naming + convention of below test cases follow + `benchmark_(number of gpus)_gpu_(dataset type)` format. + """ + + def __init__(self, output_dir=TMP_DIR, **kwargs): + self.train_data_path = CLASSIFIER_TRAIN_DATA_PATH + self.eval_data_path = CLASSIFIER_EVAL_DATA_PATH + self.bert_config_file = MODEL_CONFIG_FILE_PATH + self.input_meta_data_path = CLASSIFIER_INPUT_META_DATA_PATH + self.pretrained_checkpoint_path = PRETRAINED_CHECKPOINT_PATH + + super(BertClassifyAccuracy, self).__init__(output_dir=output_dir) + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self, + training_summary_path, + min_accuracy=0.84, + max_accuracy=0.88): + """Starts BERT accuracy benchmark test.""" + + start_time_sec = time.time() + self._run_bert_classifier(callbacks=[self.timer_callback]) + wall_time_sec = time.time() - start_time_sec + + with tf.io.gfile.GFile(training_summary_path, 'rb') as reader: + summary = json.loads(reader.read().decode('utf-8')) + + super(BertClassifyAccuracy, self)._report_benchmark( + stats=summary, + wall_time_sec=wall_time_sec, + min_accuracy=min_accuracy, + max_accuracy=max_accuracy) + + def _setup(self): + super(BertClassifyAccuracy, self)._setup() + FLAGS.train_data_path = self.train_data_path + FLAGS.eval_data_path = self.eval_data_path + FLAGS.input_meta_data_path = self.input_meta_data_path + FLAGS.bert_config_file = self.bert_config_file + FLAGS.init_checkpoint = self.pretrained_checkpoint_path + + def benchmark_8_gpu_mrpc(self): + """Run BERT model accuracy test with 8 GPUs. + + Due to comparatively small cardinality of MRPC dataset, training + accuracy metric has high variance between trainings. As so, we + set the wide range of allowed accuracy (84% to 88%). + """ + self._setup() + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_mrpc') + + summary_path = os.path.join(FLAGS.model_dir, + 'summaries/training_summary.txt') + self._run_and_report_benchmark(summary_path) + + def benchmark_8_gpu_mrpc_xla(self): + """Run BERT model accuracy test with 8 GPUs with XLA.""" + self._setup() + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_mrpc_xla') + FLAGS.enable_xla = True + summary_path = os.path.join(FLAGS.model_dir, + 'summaries/training_summary.txt') + self._run_and_report_benchmark(summary_path) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/bert_benchmark_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/bert_benchmark_utils.py new file mode 100644 index 0000000..11bdd7e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/bert_benchmark_utils.py @@ -0,0 +1,126 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utility functions or classes shared between BERT benchmarks.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import time + +# pylint: disable=g-bad-import-order +import numpy as np +from absl import flags +import tensorflow as tf +# pylint: enable=g-bad-import-order + +from official.utils.flags import core as flags_core +from official.benchmark.perfzero_benchmark import PerfZeroBenchmark + +FLAGS = flags.FLAGS + + +class BenchmarkTimerCallback(tf.keras.callbacks.Callback): + """Callback that records time it takes to run each batch.""" + + def __init__(self, num_batches_to_skip=10): + super(BenchmarkTimerCallback, self).__init__() + self.batch_start_times = {} + self.batch_stop_times = {} + + def on_batch_begin(self, batch, logs=None): + self.batch_start_times[batch] = time.time() + + def on_batch_end(self, batch, logs=None): + # If there are multiple steps_per_loop, the end batch index will not be the + # same as the starting index. Use the last starting index instead. + if batch not in self.batch_start_times: + batch = max(self.batch_start_times.keys()) + + self.batch_stop_times[batch] = time.time() + + def get_examples_per_sec(self, batch_size, num_batches_to_skip=1): + batch_durations = [] + for batch in self.batch_start_times: + if batch in self.batch_stop_times and batch >= num_batches_to_skip: + batch_durations.append(self.batch_stop_times[batch] - + self.batch_start_times[batch]) + return batch_size / np.mean(batch_durations) + + def get_startup_time(self, program_start_time): + return self.batch_start_times[0] - program_start_time + + +class BertBenchmarkBase(PerfZeroBenchmark): + """Base class to hold methods common to test classes.""" + local_flags = None + + def __init__(self, output_dir=None): + super(BertBenchmarkBase, self).__init__(output_dir=output_dir) + self.num_gpus = 8 + self.timer_callback = None + + def _setup(self): + """Sets up and resets flags before each test.""" + super(BertBenchmarkBase, self)._setup() + self.timer_callback = BenchmarkTimerCallback() + + def _report_benchmark(self, stats, wall_time_sec, min_accuracy, max_accuracy): + """Report benchmark results by writing to local protobuf file. + + Args: + stats: dict returned from BERT models with known entries. + wall_time_sec: the during of the benchmark execution in seconds + min_accuracy: Minimum classification accuracy constraint to verify + correctness of the model. + max_accuracy: Maximum classification accuracy constraint to verify + correctness of the model. + """ + metrics = [{ + 'name': 'training_loss', + 'value': stats['train_loss'], + }] + if self.timer_callback: + metrics.append({ + 'name': + 'exp_per_second', + 'value': + self.timer_callback.get_examples_per_sec(FLAGS.train_batch_size * + FLAGS.steps_per_loop) + }) + else: + metrics.append({ + 'name': 'exp_per_second', + 'value': 0.0, + }) + if self.timer_callback and 'start_time_sec' in stats: + metrics.append({ + 'name': 'startup_time', + 'value': self.timer_callback.get_startup_time(stats['start_time_sec']) + }) + + if 'eval_metrics' in stats: + metrics.append({ + 'name': 'eval_accuracy', + 'value': stats['eval_metrics'], + 'min_value': min_accuracy, + 'max_value': max_accuracy, + }) + flags_str = flags_core.get_nondefault_flags_as_str() + self.report_benchmark( + iters=stats['total_training_steps'], + wall_time=wall_time_sec, + metrics=metrics, + extras={'flags': flags_str}) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/bert_squad_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/bert_squad_benchmark.py new file mode 100644 index 0000000..d794316 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/bert_squad_benchmark.py @@ -0,0 +1,654 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Executes BERT SQuAD benchmarks and accuracy tests.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import os +import time + +# pylint: disable=g-bad-import-order +from absl import flags +from absl import logging +from absl.testing import flagsaver +import tensorflow as tf +# pylint: enable=g-bad-import-order + +from official.benchmark import bert_benchmark_utils as benchmark_utils +from official.nlp.bert import run_squad +from official.utils.misc import distribution_utils +from official.utils.misc import keras_utils +from official.benchmark import benchmark_wrappers + + +# pylint: disable=line-too-long +PRETRAINED_CHECKPOINT_PATH = 'gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16/bert_model.ckpt' +SQUAD_TRAIN_DATA_PATH = 'gs://tf-perfzero-data/bert/squad/squad_train.tf_record' +SQUAD_PREDICT_FILE = 'gs://tf-perfzero-data/bert/squad/dev-v1.1.json' +SQUAD_VOCAB_FILE = 'gs://tf-perfzero-data/bert/squad/vocab.txt' +SQUAD_MEDIUM_INPUT_META_DATA_PATH = 'gs://tf-perfzero-data/bert/squad/squad_medium_meta_data' +SQUAD_LONG_INPUT_META_DATA_PATH = 'gs://tf-perfzero-data/bert/squad/squad_long_meta_data' +SQUAD_FULL_INPUT_META_DATA_PATH = 'gs://tf-perfzero-data/bert/squad/squad_full_meta_data' +MODEL_CONFIG_FILE_PATH = 'gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16/bert_config.json' +# pylint: enable=line-too-long + +TMP_DIR = os.getenv('TMPDIR') +FLAGS = flags.FLAGS + + +class BertSquadBenchmarkBase(benchmark_utils.BertBenchmarkBase): + """Base class to hold methods common to test classes in the module.""" + + def __init__(self, output_dir=None, tpu=None): + super(BertSquadBenchmarkBase, self).__init__(output_dir=output_dir) + self.tpu = tpu + + def _read_training_summary_from_file(self): + """Reads the training summary from a file.""" + summary_path = os.path.join(FLAGS.model_dir, + 'summaries/training_summary.txt') + with tf.io.gfile.GFile(summary_path, 'rb') as reader: + return json.loads(reader.read().decode('utf-8')) + + def _read_input_meta_data_from_file(self): + """Reads the input metadata from a file.""" + with tf.io.gfile.GFile(FLAGS.input_meta_data_path, 'rb') as reader: + return json.loads(reader.read().decode('utf-8')) + + def _get_distribution_strategy(self, ds_type='mirrored'): + """Gets the distribution strategy. + + Args: + ds_type: String, the distribution strategy type to be used. Can be + 'mirrored', 'multi_worker_mirrored', 'tpu' and 'off'. + + Returns: + A `tf.distribute.DistibutionStrategy` object. + """ + if self.tpu or ds_type == 'tpu': + return distribution_utils.get_distribution_strategy( + distribution_strategy='tpu', tpu_address=self.tpu) + elif ds_type == 'multi_worker_mirrored': + # Configures cluster spec for multi-worker distribution strategy. + _ = distribution_utils.configure_cluster(FLAGS.worker_hosts, + FLAGS.task_index) + return distribution_utils.get_distribution_strategy( + distribution_strategy=ds_type, + num_gpus=self.num_gpus, + all_reduce_alg=FLAGS.all_reduce_alg) + + def _init_gpu_and_data_threads(self): + """Set env variables before any TF calls.""" + if FLAGS.tf_gpu_thread_mode: + keras_utils.set_gpu_thread_mode_and_count( + per_gpu_thread_count=FLAGS.per_gpu_thread_count, + gpu_thread_mode=FLAGS.tf_gpu_thread_mode, + num_gpus=self.num_gpus, + datasets_num_private_threads=FLAGS.datasets_num_private_threads) + + @flagsaver.flagsaver + def _train_squad(self, run_eagerly=False, ds_type='mirrored'): + """Runs BERT SQuAD training. Uses mirrored strategy by default.""" + self._init_gpu_and_data_threads() + input_meta_data = self._read_input_meta_data_from_file() + strategy = self._get_distribution_strategy(ds_type) + + run_squad.train_squad( + strategy=strategy, + input_meta_data=input_meta_data, + run_eagerly=run_eagerly, + custom_callbacks=[self.timer_callback]) + + @flagsaver.flagsaver + def _evaluate_squad(self, ds_type='mirrored'): + """Runs BERT SQuAD evaluation. Uses mirrored strategy by default.""" + self._init_gpu_and_data_threads() + input_meta_data = self._read_input_meta_data_from_file() + strategy = self._get_distribution_strategy(ds_type) + + if input_meta_data.get('version_2_with_negative', False): + logging.error('In memory evaluation result for SQuAD v2 is not accurate') + eval_metrics = run_squad.eval_squad(strategy=strategy, + input_meta_data=input_meta_data) + # Use F1 score as reported evaluation metric. + self.eval_metrics = eval_metrics['final_f1'] + + +class BertSquadBenchmarkReal(BertSquadBenchmarkBase): + """Short benchmark performance tests for BERT SQuAD model. + + Tests BERT SQuAD performance in different GPU configurations. + The naming convention of below test cases follow + `benchmark_(number of gpus)_gpu` format for GPUs and + `benchmark_(topology)_tpu` format for TPUs. + """ + + def __init__(self, output_dir=TMP_DIR, tpu=None, **kwargs): + super(BertSquadBenchmarkReal, self).__init__(output_dir=output_dir, tpu=tpu) + + def _setup(self): + """Sets up the benchmark and SQuAD flags.""" + super(BertSquadBenchmarkReal, self)._setup() + FLAGS.train_data_path = SQUAD_TRAIN_DATA_PATH + FLAGS.predict_file = SQUAD_PREDICT_FILE + FLAGS.vocab_file = SQUAD_VOCAB_FILE + FLAGS.bert_config_file = MODEL_CONFIG_FILE_PATH + FLAGS.num_train_epochs = 1 + FLAGS.steps_per_loop = 100 + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self, + run_eagerly=False, + ds_type='mirrored'): + """Runs the benchmark and reports various metrics.""" + if FLAGS.train_batch_size <= 4 or run_eagerly: + FLAGS.input_meta_data_path = SQUAD_MEDIUM_INPUT_META_DATA_PATH + else: + FLAGS.input_meta_data_path = SQUAD_LONG_INPUT_META_DATA_PATH + start_time_sec = time.time() + self._train_squad(run_eagerly=run_eagerly, ds_type=ds_type) + wall_time_sec = time.time() - start_time_sec + + summary = self._read_training_summary_from_file() + summary['start_time_sec'] = start_time_sec + + super(BertSquadBenchmarkReal, self)._report_benchmark( + stats=summary, + wall_time_sec=wall_time_sec, + min_accuracy=0, + max_accuracy=1) + + def benchmark_1_gpu(self): + """Tests BERT SQuAD model performance with 1 GPU.""" + + self._setup() + self.num_gpus = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_squad') + FLAGS.train_batch_size = 4 + + self._run_and_report_benchmark() + + def benchmark_1_gpu_eager(self): + """Tests BERT SQuAD model performance with 1 GPU.""" + + self._setup() + self.num_gpus = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_squad_eager') + FLAGS.train_batch_size = 2 + + self._run_and_report_benchmark(run_eagerly=True) + + def benchmark_1_gpu_xla(self): + """Tests BERT SQuAD model performance with 1 GPU with XLA.""" + + self._setup() + self.num_gpus = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_xla_squad') + # XLA runs out of memory when running with batch size 4. + FLAGS.train_batch_size = 3 + FLAGS.enable_xla = True + + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_dist_strat(self): + """Tests BERT SQuAD model performance with 1 GPU without DS.""" + + self._setup() + self.num_gpus = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_no_dist_strat_squad') + FLAGS.train_batch_size = 4 + + self._run_and_report_benchmark(ds_type='off') + + def benchmark_1_gpu_eager_no_dist_strat(self): + """Tests BERT SQuAD model performance with 1 GPU with eager execution.""" + + self._setup() + self.num_gpus = 1 + FLAGS.model_dir = self._get_model_dir( + 'benchmark_1_gpu_eager_no_dist_strat_squad') + FLAGS.train_batch_size = 4 + + self._run_and_report_benchmark(ds_type='off', run_eagerly=True) + + def benchmark_2_gpu(self): + """Tests BERT SQuAD model performance with 2 GPUs.""" + + self._setup() + self.num_gpus = 2 + FLAGS.model_dir = self._get_model_dir('benchmark_2_gpu_squad') + FLAGS.train_batch_size = 8 + + self._run_and_report_benchmark() + + def benchmark_4_gpu(self): + """Tests BERT SQuAD model performance with 4 GPUs.""" + + self._setup() + self.num_gpus = 4 + FLAGS.model_dir = self._get_model_dir('benchmark_4_gpu_squad') + FLAGS.train_batch_size = 16 + + self._run_and_report_benchmark() + + def benchmark_8_gpu(self): + """Tests BERT SQuAD model performance with 8 GPUs.""" + + self._setup() + self.num_gpus = 8 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_squad') + FLAGS.train_batch_size = 24 + FLAGS.tf_gpu_thread_mode = 'gpu_private' + + self._run_and_report_benchmark() + + def benchmark_1_gpu_fp16_eager(self): + """Tests BERT SQuAD model performance with 1 GPU and FP16.""" + + self._setup() + self.num_gpus = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_squad_fp16_eager') + FLAGS.train_batch_size = 4 + FLAGS.dtype = 'fp16' + FLAGS.loss_scale = 'dynamic' + + self._run_and_report_benchmark(run_eagerly=True) + + def benchmark_1_gpu_fp16(self): + """Tests BERT SQuAD model performance with 1 GPU and FP16.""" + + self._setup() + self.num_gpus = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_squad_fp16') + FLAGS.train_batch_size = 4 + FLAGS.dtype = 'fp16' + FLAGS.loss_scale = 'dynamic' + + self._run_and_report_benchmark() + + def benchmark_1_gpu_xla_fp16(self): + """Tests BERT SQuAD model performance with 1 GPU with XLA and FP16.""" + + self._setup() + self.num_gpus = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_xla_squad_fp16') + FLAGS.train_batch_size = 4 + FLAGS.enable_xla = True + FLAGS.dtype = 'fp16' + FLAGS.loss_scale = 'dynamic' + + self._run_and_report_benchmark() + + def benchmark_2_gpu_fp16(self): + """Tests BERT SQuAD model performance with 2 GPUs and FP16.""" + + self._setup() + self.num_gpus = 2 + FLAGS.model_dir = self._get_model_dir('benchmark_2_gpu_squad_fp16') + FLAGS.train_batch_size = 8 + FLAGS.dtype = 'fp16' + FLAGS.loss_scale = 'dynamic' + + self._run_and_report_benchmark() + + def benchmark_4_gpu_fp16(self): + """Tests BERT SQuAD model performance with 4 GPUs and FP16.""" + + self._setup() + self.num_gpus = 4 + FLAGS.model_dir = self._get_model_dir('benchmark_4_gpu_squad_fp16') + FLAGS.train_batch_size = 16 + FLAGS.dtype = 'fp16' + FLAGS.loss_scale = 'dynamic' + + self._run_and_report_benchmark() + + def benchmark_8_gpu_fp16(self): + """Tests BERT SQuAD model performance with 8 GPUs.""" + + self._setup() + self.num_gpus = 8 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_squad_fp16') + FLAGS.train_batch_size = 32 + FLAGS.dtype = 'fp16' + FLAGS.loss_scale = 'dynamic' + FLAGS.tf_gpu_thread_mode = 'gpu_private' + + self._run_and_report_benchmark() + + def benchmark_8_gpu_xla_fp16(self): + """Tests BERT SQuAD model performance with 8 GPUs with XLA.""" + + self._setup() + self.num_gpus = 8 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_squad_fp16') + FLAGS.train_batch_size = 32 + FLAGS.enable_xla = True + FLAGS.dtype = 'fp16' + FLAGS.loss_scale = 'dynamic' + + self._run_and_report_benchmark() + + def benchmark_1_gpu_amp(self): + """Tests BERT SQuAD model performance with 1 GPU with automatic mixed precision.""" + + self._setup() + self.num_gpus = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_amp_squad') + FLAGS.train_batch_size = 4 + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + + self._run_and_report_benchmark() + + def benchmark_4_gpu_amp(self): + """Tests BERT SQuAD model performance with 1 GPU with automatic mixed precision.""" + + self._setup() + self.num_gpus = 4 + FLAGS.model_dir = self._get_model_dir('benchmark_4_gpu_amp_squad') + FLAGS.train_batch_size = 16 + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + + self._run_and_report_benchmark() + + def benchmark_8_gpu_amp(self): + """Tests BERT SQuAD model performance with 1 GPU with automatic mixed precision.""" + + self._setup() + self.num_gpus = 8 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_amp_squad') + FLAGS.train_batch_size = 32 + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.tf_gpu_thread_mode = 'gpu_private' + + self._run_and_report_benchmark() + + def benchmark_2x2_tpu(self): + """Tests BERT SQuAD model performance with 2x2 TPU.""" + + self._setup() + FLAGS.model_dir = self._get_model_dir('benchmark_2x2_tpu') + FLAGS.train_batch_size = 48 + + self._run_and_report_benchmark() + + +class BertSquadAccuracy(BertSquadBenchmarkBase): + """Short accuracy test for BERT SQuAD model. + + Tests BERT SQuAD accuracy. The naming convention of below test cases follow + `benchmark_(number of gpus)_gpu` format for GPUs and + `benchmark_(topology)_tpu` format for TPUs. + """ + + def __init__(self, output_dir=None, tpu=None, **kwargs): + super(BertSquadAccuracy, self).__init__(output_dir=output_dir, tpu=tpu) + + def _setup(self): + """Sets up the benchmark and SQuAD flags.""" + super(BertSquadAccuracy, self)._setup() + FLAGS.train_data_path = SQUAD_TRAIN_DATA_PATH + FLAGS.predict_file = SQUAD_PREDICT_FILE + FLAGS.vocab_file = SQUAD_VOCAB_FILE + FLAGS.input_meta_data_path = SQUAD_FULL_INPUT_META_DATA_PATH + FLAGS.bert_config_file = MODEL_CONFIG_FILE_PATH + FLAGS.init_checkpoint = PRETRAINED_CHECKPOINT_PATH + FLAGS.num_train_epochs = 2 + FLAGS.steps_per_loop = 100 + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self, + run_eagerly=False, + ds_type='mirrored'): + """Runs the benchmark and reports various metrics.""" + start_time_sec = time.time() + self._train_squad(run_eagerly=run_eagerly, ds_type=ds_type) + self._evaluate_squad(ds_type=ds_type) + wall_time_sec = time.time() - start_time_sec + + summary = self._read_training_summary_from_file() + summary['eval_metrics'] = self.eval_metrics + summary['start_time_sec'] = start_time_sec + + super(BertSquadAccuracy, self)._report_benchmark( + stats=summary, + wall_time_sec=wall_time_sec, + min_accuracy=0.900, + max_accuracy=0.920) + + def benchmark_1_gpu_eager(self): + """Tests BERT SQuAD model accuracy with 1 GPU with eager execution.""" + + self._setup() + self.num_gpus = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_squad_eager') + FLAGS.train_batch_size = 4 + + self._run_and_report_benchmark(ds_type='off', run_eagerly=True) + + def benchmark_8_gpu(self): + """Tests BERT SQuAD model accuracy with 8 GPUs.""" + + self._setup() + self.num_gpus = 8 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_squad') + FLAGS.train_batch_size = 24 + FLAGS.tf_gpu_thread_mode = 'gpu_private' + + self._run_and_report_benchmark() + + def benchmark_8_gpu_fp16(self): + """Tests BERT SQuAD model accuracy with 8 GPUs and FP16.""" + + self._setup() + self.num_gpus = 8 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_squad_fp16') + FLAGS.train_batch_size = 32 + FLAGS.dtype = 'fp16' + FLAGS.loss_scale = 'dynamic' + FLAGS.tf_gpu_thread_mode = 'gpu_private' + + self._run_and_report_benchmark() + + def benchmark_8_gpu_xla(self): + """Tests BERT SQuAD model accuracy with 8 GPUs.""" + + self._setup() + self.num_gpus = 8 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_squad_xla') + FLAGS.train_batch_size = 32 + FLAGS.enable_xla = True + FLAGS.tf_gpu_thread_mode = 'gpu_private' + + self._run_and_report_benchmark() + + def benchmark_2x2_tpu(self): + """Tests BERT SQuAD model accuracy with 2x2 TPU.""" + + self._setup() + FLAGS.model_dir = self._get_model_dir('benchmark_2x2_tpu') + FLAGS.train_batch_size = 48 + + self._run_and_report_benchmark() + + +class BertSquadMultiWorkerAccuracy(BertSquadBenchmarkBase): + """BERT SQuAD distributed accuracy tests with multiple workers.""" + + def __init__(self, output_dir=None, tpu=None, **kwargs): + super(BertSquadMultiWorkerAccuracy, self).__init__( + output_dir=output_dir, tpu=tpu) + + def _setup(self): + """Sets up the benchmark and SQuAD flags.""" + super(BertSquadMultiWorkerAccuracy, self)._setup() + FLAGS.train_data_path = SQUAD_TRAIN_DATA_PATH + FLAGS.predict_file = SQUAD_PREDICT_FILE + FLAGS.vocab_file = SQUAD_VOCAB_FILE + FLAGS.input_meta_data_path = SQUAD_FULL_INPUT_META_DATA_PATH + FLAGS.bert_config_file = MODEL_CONFIG_FILE_PATH + FLAGS.init_checkpoint = PRETRAINED_CHECKPOINT_PATH + FLAGS.num_train_epochs = 2 + FLAGS.steps_per_loop = 100 + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self, + use_ds=True, + run_eagerly=False): + """Runs the benchmark and reports various metrics.""" + start_time_sec = time.time() + self._train_squad(run_eagerly=run_eagerly, + ds_type='multi_worker_mirrored') + self._evaluate_squad(ds_type='multi_worker_mirrored') + wall_time_sec = time.time() - start_time_sec + + summary = self._read_training_summary_from_file() + summary['eval_metrics'] = self.eval_metrics + + super(BertSquadMultiWorkerAccuracy, self)._report_benchmark( + stats=summary, + wall_time_sec=wall_time_sec, + min_accuracy=0.900, + max_accuracy=0.920) + + def _benchmark_common(self, num_workers, all_reduce_alg): + """Common to all benchmarks in this class.""" + self._setup() + + num_gpus = 8 + FLAGS.num_gpus = num_gpus + FLAGS.dtype = 'fp16' + FLAGS.enable_xla = False + FLAGS.distribution_strategy = 'multi_worker_mirrored' + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.datasets_num_private_threads = 32 + FLAGS.model_dir = self._get_model_dir( + 'benchmark_8_gpu_{}_worker_fp16_{}_tweaked'.format( + num_workers, all_reduce_alg)) + FLAGS.train_batch_size = 4 * num_gpus * num_workers + FLAGS.all_reduce_alg = all_reduce_alg + + self._run_and_report_benchmark() + + def benchmark_eager_8_gpu_2_workers_fp16_ring_tweaked(self): + """8 GPUs per worker, 2 workers, fp16, ring all-reduce.""" + self._benchmark_common(num_workers=2, all_reduce_alg='ring') + + def benchmark_eager_8_gpu_2_workers_fp16_nccl_tweaked(self): + """8 GPUs per worker, 2 workers, fp16, nccl all-reduce.""" + self._benchmark_common(num_workers=2, all_reduce_alg='nccl') + + def benchmark_8_gpu_8_workers_fp16_ring_tweaked(self): + """8 GPUs per worker, 8 workers, fp16, ring all-reduce.""" + self._benchmark_common(num_workers=8, all_reduce_alg='ring') + + def benchmark_8_gpu_8_workers_fp16_nccl_tweaked(self): + """8 GPUs per worker, 8 workers, fp16, nccl all-reduce.""" + self._benchmark_common(num_workers=8, all_reduce_alg='nccl') + + +class BertSquadMultiWorkerBenchmark(BertSquadBenchmarkBase): + """BERT SQuAD distributed benchmark tests with multiple workers.""" + + def __init__(self, output_dir=TMP_DIR, tpu=None, **kwargs): + super(BertSquadMultiWorkerBenchmark, self).__init__( + output_dir=output_dir, tpu=tpu) + + def _setup(self): + """Sets up the benchmark and SQuAD flags.""" + super(BertSquadMultiWorkerBenchmark, self)._setup() + FLAGS.train_data_path = SQUAD_TRAIN_DATA_PATH + FLAGS.predict_file = SQUAD_PREDICT_FILE + FLAGS.vocab_file = SQUAD_VOCAB_FILE + FLAGS.input_meta_data_path = SQUAD_FULL_INPUT_META_DATA_PATH + FLAGS.bert_config_file = MODEL_CONFIG_FILE_PATH + FLAGS.num_train_epochs = 1 + FLAGS.steps_per_loop = 100 + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self, + use_ds=True, + run_eagerly=False): + """Runs the benchmark and reports various metrics.""" + if FLAGS.train_batch_size <= 4 * 8: + FLAGS.input_meta_data_path = SQUAD_LONG_INPUT_META_DATA_PATH + else: + FLAGS.input_meta_data_path = SQUAD_FULL_INPUT_META_DATA_PATH + start_time_sec = time.time() + self._train_squad(run_eagerly=run_eagerly, + ds_type='multi_worker_mirrored') + wall_time_sec = time.time() - start_time_sec + + summary = self._read_training_summary_from_file() + summary['start_time_sec'] = start_time_sec + + super(BertSquadMultiWorkerBenchmark, self)._report_benchmark( + stats=summary, + wall_time_sec=wall_time_sec, + min_accuracy=0, + max_accuracy=1) + + def _benchmark_common(self, num_workers, all_reduce_alg): + """Common to all benchmarks in this class.""" + self._setup() + + num_gpus = 8 + FLAGS.num_gpus = num_gpus + FLAGS.dtype = 'fp16' + FLAGS.enable_xla = False + FLAGS.distribution_strategy = 'multi_worker_mirrored' + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.datasets_num_private_threads = 32 + FLAGS.model_dir = self._get_model_dir( + 'benchmark_8_gpu_{}_worker_fp16_{}_tweaked'.format( + num_workers, all_reduce_alg)) + FLAGS.train_batch_size = 4 * num_gpus * num_workers + FLAGS.all_reduce_alg = all_reduce_alg + + self._run_and_report_benchmark() + + def benchmark_8_gpu_1_worker_fp16_ring_tweaked(self): + """8 GPUs per worker, 1 worker, fp16, ring all-reduce.""" + self._benchmark_common(num_workers=1, all_reduce_alg='ring') + + def benchmark_8_gpu_1_worker_fp16_nccl_tweaked(self): + """8 GPUs per worker, 1 worker, fp16, nccl all-reduce.""" + self._benchmark_common(num_workers=1, all_reduce_alg='nccl') + + def benchmark_8_gpu_2_workers_fp16_ring_tweaked(self): + """8 GPUs per worker, 2 workers, fp16, ring all-reduce.""" + self._benchmark_common(num_workers=2, all_reduce_alg='ring') + + def benchmark_8_gpu_2_workers_fp16_nccl_tweaked(self): + """8 GPUs per worker, 2 workers, fp16, nccl all-reduce.""" + self._benchmark_common(num_workers=2, all_reduce_alg='nccl') + + def benchmark_8_gpu_8_workers_fp16_ring_tweaked(self): + """8 GPUs per worker, 8 workers, fp16, ring all-reduce.""" + self._benchmark_common(num_workers=8, all_reduce_alg='ring') + + def benchmark_8_gpu_8_workers_fp16_nccl_tweaked(self): + """8 GPUs per worker, 8 workers, fp16, nccl all-reduce.""" + self._benchmark_common(num_workers=8, all_reduce_alg='nccl') + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/datastore/schema/benchmark_metric.json b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/datastore/schema/benchmark_metric.json new file mode 100644 index 0000000..cc571d4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/datastore/schema/benchmark_metric.json @@ -0,0 +1,56 @@ +[ + { + "description": "The ID of the benchmark run, where this metric should tie to.", + "mode": "REQUIRED", + "name": "run_id", + "type": "STRING" + }, + { + "description": "The name of the metric, which should be descriptive. E.g. training_loss, accuracy.", + "mode": "REQUIRED", + "name": "name", + "type": "STRING" + }, + { + "description": "The unit of the metric. E.g. MB per sec.", + "mode": "NULLABLE", + "name": "unit", + "type": "STRING" + }, + { + "description": "The value of the metric.", + "mode": "NULLABLE", + "name": "value", + "type": "FLOAT" + }, + { + "description": "The timestamp when the metric is recorded.", + "mode": "REQUIRED", + "name": "timestamp", + "type": "TIMESTAMP" + }, + { + "description": "The global step when this metric is recorded.", + "mode": "NULLABLE", + "name": "global_step", + "type": "INTEGER" + }, + { + "description": "Free format metadata for the extra information about the metric.", + "mode": "REPEATED", + "name": "extras", + "type": "RECORD", + "fields": [ + { + "mode": "NULLABLE", + "name": "name", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "STRING" + } + ] + } +] diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/datastore/schema/benchmark_run.json b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/datastore/schema/benchmark_run.json new file mode 100644 index 0000000..58e5ddc --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/datastore/schema/benchmark_run.json @@ -0,0 +1,368 @@ +[ + { + "description": "The UUID of the run for the benchmark.", + "mode": "REQUIRED", + "name": "model_id", + "type": "STRING" + }, + { + "description": "The name of the model, E.g ResNet50, LeNet-5 etc.", + "mode": "REQUIRED", + "name": "model_name", + "type": "STRING" + }, + { + "description": "The date when the test of the model is started", + "mode": "REQUIRED", + "name": "run_date", + "type": "TIMESTAMP" + }, + { + "description": "The unique name for a test by the combination of key parameters, eg batch size, num of GPU, etc. It is hardware independent.", + "mode": "NULLABLE", + "name": "test_id", + "type": "STRING" + }, + { + "description": "The tensorflow version information.", + "fields": [ + { + "description": "Version of the tensorflow. E.g. 1.7.0-rc0", + "mode": "REQUIRED", + "name": "version", + "type": "STRING" + }, + { + "description": "Git Hash of the tensorflow", + "mode": "NULLABLE", + "name": "git_hash", + "type": "STRING" + }, + { + "description": "The channel of the tensorflow binary, eg, nightly, RC, final, custom.", + "mode": "NULLABLE", + "name": "channel", + "type": "STRING" + }, + { + "description": "Identify anything special about the build, eg CUDA 10, NCCL, MKL, etc.", + "mode": "NULLABLE", + "name": "build_type", + "type": "STRING" + } + ], + "mode": "REQUIRED", + "name": "tensorflow_version", + "type": "RECORD" + }, + { + "description": "The arbitrary attribute of the model.", + "fields": [ + { + "description": "The name of the attribute.", + "mode": "REQUIRED", + "name": "name", + "type": "STRING" + }, + { + "description": "The value of the attribute.", + "mode": "NULLABLE", + "name": "value", + "type": "STRING" + } + ], + "mode": "REPEATED", + "name": "attribute", + "type": "RECORD" + }, + { + "description": "Environment variables when the benchmark run is executed.", + "fields": [ + { + "description": "The name of the variable.", + "mode": "REQUIRED", + "name": "name", + "type": "STRING" + }, + { + "description": "The value of the variable.", + "mode": "NULLABLE", + "name": "value", + "type": "STRING" + } + ], + "mode": "REPEATED", + "name": "environment_variable", + "type": "RECORD" + }, + { + "description": "TF Environment variables when the benchmark run is executed.", + "fields": [ + { + "description": "The name of the variable.", + "mode": "REQUIRED", + "name": "name", + "type": "STRING" + }, + { + "description": "The value of the variable.", + "mode": "NULLABLE", + "name": "value", + "type": "STRING" + } + ], + "mode": "REPEATED", + "name": "tensorflow_environment_variables", + "type": "RECORD" + }, + { + "description": "The list of parameters run with the model. It could contain hyperparameters or others.", + "fields": [ + { + "description": "The name of the parameter.", + "mode": "REQUIRED", + "name": "name", + "type": "STRING" + }, + { + "description": "The string value of the parameter.", + "mode": "NULLABLE", + "name": "string_value", + "type": "STRING" + }, + { + "description": "The bool value of the parameter.", + "mode": "NULLABLE", + "name": "bool_value", + "type": "STRING" + }, + { + "description": "The int/long value of the parameter.", + "mode": "NULLABLE", + "name": "long_value", + "type": "INTEGER" + }, + { + "description": "The double/float value of parameter.", + "mode": "NULLABLE", + "name": "float_value", + "type": "FLOAT" + } + ], + "mode": "REPEATED", + "name": "run_parameters", + "type": "RECORD" + }, + { + "description": "The dataset that run with the benchmark.", + "mode": "NULLABLE", + "name": "dataset", + "type": "RECORD", + "fields": [ + { + "description": "The name of the dataset that the model is trained/validated with. E.g ImageNet, mnist.", + "mode": "REQUIRED", + "name": "name", + "type": "STRING" + }, + { + "description": "The arbitrary attribute of the dataset.", + "fields": [ + { + "description": "The name of the attribute.", + "mode": "REQUIRED", + "name": "name", + "type": "STRING" + }, + { + "description": "The value of the attribute.", + "mode": "NULLABLE", + "name": "value", + "type": "STRING" + } + ], + "mode": "REPEATED", + "name": "attribute", + "type": "RECORD" + } + ] + }, + { + "description": "Used to differentiate from AWS, GCE or DGX-1 at a high level", + "mode": "NULLABLE", + "name": "test_environment", + "type": "STRING" + }, + { + "description": "The machine configuration of the benchmark run.", + "mode": "NULLABLE", + "name": "machine_config", + "type": "RECORD", + "fields": [ + { + "description": "The platform information of the benchmark run.", + "mode": "NULLABLE", + "name": "platform_info", + "type": "RECORD", + "fields": [ + { + "description": "Eg: 64bit.", + "mode": "NULLABLE", + "name": "bits", + "type": "STRING" + }, + { + "description": "Eg: ELF.", + "mode": "NULLABLE", + "name": "linkage", + "type": "STRING" + }, + { + "description": "Eg: i386.", + "mode": "NULLABLE", + "name": "machine", + "type": "STRING" + }, + { + "description": "Eg: 3.13.0-76-generic.", + "mode": "NULLABLE", + "name": "release", + "type": "STRING" + }, + { + "description": "Eg: Linux.", + "mode": "NULLABLE", + "name": "system", + "type": "STRING" + }, + { + "description": "Eg: #120-Ubuntu SMP Mon Jan 18 15:59:10 UTC 2016.", + "mode": "NULLABLE", + "name": "version", + "type": "STRING" + } + ] + }, + { + "description": "The CPU information of the benchmark run.", + "mode": "NULLABLE", + "name": "cpu_info", + "type": "RECORD", + "fields": [ + { + "mode": "NULLABLE", + "name": "num_cores", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "num_cores_allowed", + "type": "INTEGER" + }, + { + "description" : "How fast are those CPUs.", + "mode": "NULLABLE", + "name": "mhz_per_cpu", + "type": "FLOAT" + }, + { + "description" : "Additional CPU info, Eg: Intel Ivybridge with HyperThreading (24 cores).", + "mode": "NULLABLE", + "name": "cpu_info", + "type": "STRING" + }, + { + "description" : "What kind of cpu scaling is enabled on the host. Eg performance, ondemand, conservative, mixed.", + "mode": "NULLABLE", + "name": "cpu_governor", + "type": "STRING" + }, + { + "description": "Cache size of the CPUs.", + "mode": "NULLABLE", + "name": "cache_size", + "type": "RECORD", + "fields": [ + { + "mode": "NULLABLE", + "name": "level", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "size", + "type": "INTEGER" + } + ] + } + ] + }, + { + "mode": "NULLABLE", + "name": "gpu_info", + "type": "RECORD", + "fields": [ + { + "mode": "NULLABLE", + "name": "count", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "model", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "cuda_version", + "type": "STRING" + } + ] + }, + { + "description": "The cloud instance inforation if the benchmark run is executed on cloud", + "mode": "NULLABLE", + "name": "cloud_info", + "type": "RECORD", + "fields": [ + { + "description": "The instance type, E.g. n1-standard-4.", + "mode": "NULLABLE", + "name": "instance_type", + "type": "STRING" + }, + { + "description": "The arbitrary attribute of the cloud info.", + "fields": [ + { + "description": "The name of the attribute.", + "mode": "REQUIRED", + "name": "name", + "type": "STRING" + }, + { + "description": "The value of the attribute.", + "mode": "NULLABLE", + "name": "value", + "type": "STRING" + } + ], + "mode": "REPEATED", + "name": "attribute", + "type": "RECORD" + } + ] + }, + { + "mode": "NULLABLE", + "name": "memory_total", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "memory_available", + "type": "STRING" + } + ] + } +] diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/datastore/schema/benchmark_run_status.json b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/datastore/schema/benchmark_run_status.json new file mode 100644 index 0000000..f7ac59e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/datastore/schema/benchmark_run_status.json @@ -0,0 +1,14 @@ +[ + { + "description": "The UUID of the run for the benchmark.", + "mode": "REQUIRED", + "name": "run_id", + "type": "STRING" + }, + { + "description": "The status of the run for the benchmark. Eg, running, failed, success", + "mode": "REQUIRED", + "name": "status", + "type": "STRING" + } +] \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/keras_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/keras_benchmark.py new file mode 100644 index 0000000..770674a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/keras_benchmark.py @@ -0,0 +1,98 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Executes Keras benchmarks and accuracy tests.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from official.benchmark.perfzero_benchmark import PerfZeroBenchmark +from official.utils.flags import core as flags_core + + +class KerasBenchmark(PerfZeroBenchmark): + """Base benchmark class with methods to simplify testing.""" + + def __init__(self, + output_dir=None, + default_flags=None, + flag_methods=None, + tpu=None): + super(KerasBenchmark, self).__init__( + output_dir=output_dir, + default_flags=default_flags, + flag_methods=flag_methods, + tpu=tpu) + + def _report_benchmark(self, + stats, + wall_time_sec, + top_1_max=None, + top_1_min=None, + log_steps=None, + total_batch_size=None, + warmup=1, + start_time_sec=None): + """Report benchmark results by writing to local protobuf file. + + Args: + stats: dict returned from keras models with known entries. + wall_time_sec: the during of the benchmark execution in seconds + top_1_max: highest passing level for top_1 accuracy. + top_1_min: lowest passing level for top_1 accuracy. + log_steps: How often the log was created for stats['step_timestamp_log']. + total_batch_size: Global batch-size. + warmup: number of entries in stats['step_timestamp_log'] to ignore. + start_time_sec: the start time of the program in seconds since epoch + """ + + metrics = [] + if 'accuracy_top_1' in stats: + metrics.append({'name': 'accuracy_top_1', + 'value': stats['accuracy_top_1'], + 'min_value': top_1_min, + 'max_value': top_1_max}) + metrics.append({'name': 'top_1_train_accuracy', + 'value': stats['training_accuracy_top_1']}) + + if (warmup and 'step_timestamp_log' in stats and + len(stats['step_timestamp_log']) > warmup): + # first entry in the time_log is start of step 1. The rest of the + # entries are the end of each step recorded + time_log = stats['step_timestamp_log'] + elapsed = time_log[-1].timestamp - time_log[warmup].timestamp + num_examples = ( + total_batch_size * log_steps * (len(time_log) - warmup - 1)) + examples_per_sec = num_examples / elapsed + metrics.append({'name': 'exp_per_second', + 'value': examples_per_sec}) + + if 'avg_exp_per_second' in stats: + metrics.append({'name': 'avg_exp_per_second', + 'value': stats['avg_exp_per_second']}) + + if start_time_sec and 'step_timestamp_log' in stats: + time_log = stats['step_timestamp_log'] + # time_log[0] is recorded at the beginning of the first step. + startup_time = time_log[0].timestamp - start_time_sec + metrics.append({'name': 'startup_time', 'value': startup_time}) + + flags_str = flags_core.get_nondefault_flags_as_str() + self.report_benchmark( + iters=-1, + wall_time=wall_time_sec, + metrics=metrics, + extras={'flags': flags_str}) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/keras_cifar_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/keras_cifar_benchmark.py new file mode 100644 index 0000000..694200f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/keras_cifar_benchmark.py @@ -0,0 +1,402 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Executes Keras benchmarks and accuracy tests.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import time +from absl import flags +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.benchmark import keras_benchmark +from official.benchmark import benchmark_wrappers +from official.benchmark.models import resnet_cifar_main + +MIN_TOP_1_ACCURACY = 0.929 +MAX_TOP_1_ACCURACY = 0.938 + +FLAGS = flags.FLAGS +CIFAR_DATA_DIR_NAME = 'cifar-10-batches-bin' + + +class Resnet56KerasAccuracy(keras_benchmark.KerasBenchmark): + """Accuracy tests for ResNet56 Keras CIFAR-10.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + """A benchmark class. + + Args: + output_dir: directory where to output e.g. log files + root_data_dir: directory under which to look for dataset + **kwargs: arbitrary named arguments. This is needed to make the + constructor forward compatible in case PerfZero provides more + named arguments before updating the constructor. + """ + + self.data_dir = os.path.join(root_data_dir, CIFAR_DATA_DIR_NAME) + flag_methods = [resnet_cifar_main.define_cifar_flags] + + super(Resnet56KerasAccuracy, self).__init__( + output_dir=output_dir, flag_methods=flag_methods) + + def _setup(self): + super(Resnet56KerasAccuracy, self)._setup() + FLAGS.use_tensor_lr = False + + def benchmark_graph_1_gpu(self): + """Test keras based model with Keras fit and distribution strategies.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 128 + FLAGS.train_epochs = 182 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_1_gpu') + FLAGS.dtype = 'fp32' + self._run_and_report_benchmark() + + def benchmark_1_gpu(self): + """Test keras based model with eager and distribution strategies.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 128 + FLAGS.train_epochs = 182 + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu') + FLAGS.dtype = 'fp32' + FLAGS.enable_eager = True + self._run_and_report_benchmark() + + def benchmark_cpu(self): + """Test keras based model on CPU.""" + self._setup() + FLAGS.num_gpus = 0 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 128 + FLAGS.train_epochs = 182 + FLAGS.model_dir = self._get_model_dir('benchmark_cpu') + FLAGS.dtype = 'fp32' + FLAGS.enable_eager = True + FLAGS.data_format = 'channels_last' + self._run_and_report_benchmark() + + def benchmark_cpu_no_dist_strat(self): + """Test keras based model on CPU without distribution strategies.""" + self._setup() + FLAGS.num_gpus = 0 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 128 + FLAGS.train_epochs = 182 + FLAGS.model_dir = self._get_model_dir('benchmark_cpu_no_dist_strat') + FLAGS.dtype = 'fp32' + FLAGS.enable_eager = True + FLAGS.distribution_strategy = 'off' + FLAGS.data_format = 'channels_last' + self._run_and_report_benchmark() + + def benchmark_cpu_no_dist_strat_run_eagerly(self): + """Test keras based model on CPU w/forced eager and no dist_strat.""" + self._setup() + FLAGS.num_gpus = 0 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 128 + FLAGS.train_epochs = 182 + FLAGS.model_dir = self._get_model_dir( + 'benchmark_cpu_no_dist_strat_run_eagerly') + FLAGS.dtype = 'fp32' + FLAGS.enable_eager = True + FLAGS.run_eagerly = True + FLAGS.distribution_strategy = 'off' + FLAGS.data_format = 'channels_last' + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_dist_strat(self): + """Test keras based model with eager and no dist strat.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 128 + FLAGS.train_epochs = 182 + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_no_dist_strat') + FLAGS.dtype = 'fp32' + FLAGS.enable_eager = True + FLAGS.distribution_strategy = 'off' + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_dist_strat_run_eagerly(self): + """Test keras based model w/forced eager and no dist_strat.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 128 + FLAGS.train_epochs = 182 + FLAGS.model_dir = self._get_model_dir( + 'benchmark_1_gpu_no_dist_strat_run_eagerly') + FLAGS.dtype = 'fp32' + FLAGS.enable_eager = True + FLAGS.run_eagerly = True + FLAGS.distribution_strategy = 'off' + self._run_and_report_benchmark() + + def benchmark_graph_1_gpu_no_dist_strat(self): + """Test keras based model with Keras fit but not distribution strategies.""" + self._setup() + FLAGS.distribution_strategy = 'off' + FLAGS.num_gpus = 1 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 128 + FLAGS.train_epochs = 182 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_1_gpu_no_dist_strat') + FLAGS.dtype = 'fp32' + self._run_and_report_benchmark() + + def benchmark_2_gpu(self): + """Test keras based model with eager and distribution strategies.""" + self._setup() + FLAGS.num_gpus = 2 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 128 + FLAGS.train_epochs = 182 + FLAGS.model_dir = self._get_model_dir('benchmark_2_gpu') + FLAGS.dtype = 'fp32' + FLAGS.enable_eager = True + self._run_and_report_benchmark() + + def benchmark_graph_2_gpu(self): + """Test keras based model with Keras fit and distribution strategies.""" + self._setup() + FLAGS.num_gpus = 2 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 128 + FLAGS.train_epochs = 182 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_2_gpu') + FLAGS.dtype = 'fp32' + self._run_and_report_benchmark() + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self): + start_time_sec = time.time() + stats = resnet_cifar_main.run(FLAGS) + wall_time_sec = time.time() - start_time_sec + + super(Resnet56KerasAccuracy, self)._report_benchmark( + stats, + wall_time_sec, + top_1_min=MIN_TOP_1_ACCURACY, + top_1_max=MAX_TOP_1_ACCURACY, + total_batch_size=FLAGS.batch_size, + log_steps=100) + + +class Resnet56KerasBenchmarkBase(keras_benchmark.KerasBenchmark): + """Short performance tests for ResNet56 via Keras and CIFAR-10.""" + + def __init__(self, output_dir=None, default_flags=None): + flag_methods = [resnet_cifar_main.define_cifar_flags] + + super(Resnet56KerasBenchmarkBase, self).__init__( + output_dir=output_dir, + flag_methods=flag_methods, + default_flags=default_flags) + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self): + start_time_sec = time.time() + stats = resnet_cifar_main.run(FLAGS) + wall_time_sec = time.time() - start_time_sec + + super(Resnet56KerasBenchmarkBase, self)._report_benchmark( + stats, + wall_time_sec, + total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_1_gpu(self): + """Test 1 gpu.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu') + FLAGS.batch_size = 128 + self._run_and_report_benchmark() + + def benchmark_1_gpu_xla(self): + """Test 1 gpu with xla enabled.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.run_eagerly = False + FLAGS.enable_xla = True + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_xla') + FLAGS.batch_size = 128 + self._run_and_report_benchmark() + + def benchmark_graph_1_gpu(self): + """Test 1 gpu graph.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.enable_eager = False + FLAGS.run_eagerly = False + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_graph_1_gpu') + FLAGS.batch_size = 128 + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_dist_strat(self): + """Test 1 gpu without distribution strategies.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.distribution_strategy = 'off' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_no_dist_strat') + FLAGS.batch_size = 128 + self._run_and_report_benchmark() + + def benchmark_graph_1_gpu_no_dist_strat(self): + """Test 1 gpu graph mode without distribution strategies.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.enable_eager = False + FLAGS.distribution_strategy = 'off' + FLAGS.model_dir = self._get_model_dir('benchmark_graph_1_gpu_no_dist_strat') + FLAGS.batch_size = 128 + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_dist_strat_run_eagerly(self): + """Test 1 gpu without distribution strategy and forced eager.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.batch_size = 128 + FLAGS.model_dir = self._get_model_dir( + 'benchmark_1_gpu_no_dist_strat_run_eagerly') + FLAGS.dtype = 'fp32' + FLAGS.enable_eager = True + FLAGS.run_eagerly = True + FLAGS.distribution_strategy = 'off' + self._run_and_report_benchmark() + + def benchmark_2_gpu(self): + """Test 2 gpu.""" + self._setup() + FLAGS.num_gpus = 2 + FLAGS.enable_eager = True + FLAGS.run_eagerly = False + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir('benchmark_2_gpu') + FLAGS.batch_size = 128 * 2 # 2 GPUs + self._run_and_report_benchmark() + + def benchmark_graph_2_gpu(self): + """Test 2 gpu graph mode.""" + self._setup() + FLAGS.num_gpus = 2 + FLAGS.enable_eager = False + FLAGS.run_eagerly = False + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir('benchmark_graph_2_gpu') + FLAGS.batch_size = 128 * 2 # 2 GPUs + self._run_and_report_benchmark() + + def benchmark_cpu(self): + """Test cpu.""" + self._setup() + FLAGS.num_gpus = 0 + FLAGS.enable_eager = True + FLAGS.model_dir = self._get_model_dir('benchmark_cpu') + FLAGS.batch_size = 128 + FLAGS.data_format = 'channels_last' + self._run_and_report_benchmark() + + def benchmark_graph_cpu(self): + """Test cpu graph mode.""" + self._setup() + FLAGS.num_gpus = 0 + FLAGS.enable_eager = False + FLAGS.model_dir = self._get_model_dir('benchmark_graph_cpu') + FLAGS.batch_size = 128 + FLAGS.data_format = 'channels_last' + self._run_and_report_benchmark() + + def benchmark_cpu_no_dist_strat_run_eagerly(self): + """Test cpu without distribution strategy and forced eager.""" + self._setup() + FLAGS.num_gpus = 0 + FLAGS.distribution_strategy = 'off' + FLAGS.enable_eager = True + FLAGS.run_eagerly = True + FLAGS.model_dir = self._get_model_dir( + 'benchmark_cpu_no_dist_strat_run_eagerly') + FLAGS.batch_size = 128 + FLAGS.data_format = 'channels_last' + self._run_and_report_benchmark() + + def benchmark_cpu_no_dist_strat(self): + """Test cpu without distribution strategies.""" + self._setup() + FLAGS.num_gpus = 0 + FLAGS.enable_eager = True + FLAGS.distribution_strategy = 'off' + FLAGS.model_dir = self._get_model_dir('benchmark_cpu_no_dist_strat') + FLAGS.batch_size = 128 + FLAGS.data_format = 'channels_last' + self._run_and_report_benchmark() + + def benchmark_graph_cpu_no_dist_strat(self): + """Test cpu graph mode without distribution strategies.""" + self._setup() + FLAGS.num_gpus = 0 + FLAGS.enable_eager = False + FLAGS.distribution_strategy = 'off' + FLAGS.model_dir = self._get_model_dir('benchmark_graph_cpu_no_dist_strat') + FLAGS.batch_size = 128 + FLAGS.data_format = 'channels_last' + self._run_and_report_benchmark() + + +class Resnet56KerasBenchmarkSynth(Resnet56KerasBenchmarkBase): + """Synthetic benchmarks for ResNet56 and Keras.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + default_flags = {} + default_flags['skip_eval'] = True + default_flags['use_synthetic_data'] = True + default_flags['train_steps'] = 110 + default_flags['log_steps'] = 10 + default_flags['use_tensor_lr'] = False + + super(Resnet56KerasBenchmarkSynth, self).__init__( + output_dir=output_dir, default_flags=default_flags) + + +class Resnet56KerasBenchmarkReal(Resnet56KerasBenchmarkBase): + """Real data benchmarks for ResNet56 and Keras.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + default_flags = {} + default_flags['skip_eval'] = True + default_flags['data_dir'] = os.path.join(root_data_dir, CIFAR_DATA_DIR_NAME) + default_flags['train_steps'] = 110 + default_flags['log_steps'] = 10 + default_flags['use_tensor_lr'] = False + + super(Resnet56KerasBenchmarkReal, self).__init__( + output_dir=output_dir, default_flags=default_flags) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/keras_imagenet_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/keras_imagenet_benchmark.py new file mode 100644 index 0000000..87b278d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/keras_imagenet_benchmark.py @@ -0,0 +1,1685 @@ +# Lint as: python3 +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Executes Keras benchmarks and accuracy tests.""" +# pylint: disable=line-too-long +from __future__ import print_function + +import json +import os +import time + +from typing import Any, MutableMapping, Optional + +from absl import flags +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.benchmark import benchmark_wrappers +from official.benchmark import keras_benchmark +from official.vision.image_classification import classifier_trainer +from official.vision.image_classification.resnet import resnet_imagenet_main + +MIN_TOP_1_ACCURACY = 0.76 +MAX_TOP_1_ACCURACY = 0.77 + +MOBILENET_V1_MIN_TOP_1_ACCURACY = 0.65 +MOBILENET_V1_MAX_TOP_1_ACCURACY = 0.68 + +# Range of top-1 accracies for model optimization techniques. +# Each item indicates (MIN_TOP_1_ACCURACY, MAX_TOP_1_ACCURACY). +MODEL_OPTIMIZATION_TOP_1_ACCURACY = { + 'RESNET50_FINETUNE_PRUNING': (0.76, 0.77), + 'MOBILENET_V1_FINETUNE_PRUNING': (0.67, 0.68), +} + +FLAGS = flags.FLAGS + + +def _get_classifier_parameters( + num_gpus: int = 0, + builder: str = 'records', + skip_eval: bool = False, + distribution_strategy: str = 'mirrored', + per_replica_batch_size: int = 128, + epochs: int = 90, + steps: int = 0, + epochs_between_evals: int = 1, + dtype: str = 'float32', + enable_xla: bool = False, + run_eagerly: bool = False, + gpu_thread_mode: Optional[str] = None, + dataset_num_private_threads: Optional[int] = None, + loss_scale: Optional[str] = None) -> MutableMapping[str, Any]: + """Gets classifier trainer's ResNet parameters.""" + return { + 'runtime': { + 'num_gpus': num_gpus, + 'distribution_strategy': distribution_strategy, + 'run_eagerly': run_eagerly, + 'enable_xla': enable_xla, + 'dataset_num_private_threads': dataset_num_private_threads, + 'gpu_thread_mode': gpu_thread_mode, + 'loss_scale': loss_scale, + }, + 'train_dataset': { + 'builder': builder, + 'use_per_replica_batch_size': True, + 'batch_size': per_replica_batch_size, + 'image_size': 224, + 'dtype': dtype, + }, + 'validation_dataset': { + 'builder': builder, + 'batch_size': per_replica_batch_size, + 'use_per_replica_batch_size': True, + 'image_size': 224, + 'dtype': dtype, + }, + 'train': { + 'epochs': epochs, + 'steps': steps, + 'callbacks': { + 'enable_tensorboard': False, + 'enable_checkpoint_and_export': False, + 'enable_time_history': True, + }, + }, + 'evaluation': { + 'epochs_between_evals': epochs_between_evals, + 'skip_eval': skip_eval, + }, + } + + +class Resnet50KerasAccuracy(keras_benchmark.KerasBenchmark): + """Benchmark accuracy tests for ResNet50 in Keras.""" + + def __init__(self, + output_dir: Optional[str] = None, + root_data_dir: Optional[str] = None, + **kwargs): + """A benchmark class. + + Args: + output_dir: directory where to output e.g. log files + root_data_dir: directory under which to look for dataset + **kwargs: arbitrary named arguments. This is needed to make the + constructor forward compatible in case PerfZero provides more + named arguments before updating the constructor. + """ + + flag_methods = [classifier_trainer.define_classifier_flags] + + self.data_dir = os.path.join(root_data_dir, 'imagenet') + super(Resnet50KerasAccuracy, self).__init__( + output_dir=output_dir, flag_methods=flag_methods) + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark( + self, + experiment_name: str, + top_1_min: float = MIN_TOP_1_ACCURACY, + top_1_max: float = MAX_TOP_1_ACCURACY, + num_gpus: int = 0, + distribution_strategy: str = 'mirrored', + per_replica_batch_size: int = 128, + epochs: int = 90, + steps: int = 0, + epochs_between_evals: int = 1, + dtype: str = 'float32', + enable_xla: bool = False, + run_eagerly: bool = False, + gpu_thread_mode: Optional[str] = None, + dataset_num_private_threads: Optional[int] = None, + loss_scale: Optional[str] = None): + """Runs and reports the benchmark given the provided configuration.""" + FLAGS.model_type = 'resnet' + FLAGS.dataset = 'imagenet' + FLAGS.mode = 'train_and_eval' + FLAGS.data_dir = self.data_dir + FLAGS.model_dir = self._get_model_dir(experiment_name) + parameters = _get_classifier_parameters( + num_gpus=num_gpus, + distribution_strategy=distribution_strategy, + per_replica_batch_size=per_replica_batch_size, + epochs=epochs, + steps=steps, + epochs_between_evals=epochs_between_evals, + dtype=dtype, + enable_xla=enable_xla, + run_eagerly=run_eagerly, + gpu_thread_mode=gpu_thread_mode, + dataset_num_private_threads=dataset_num_private_threads, + loss_scale=loss_scale) + FLAGS.params_override = json.dumps(parameters) + total_batch_size = num_gpus * per_replica_batch_size + + start_time_sec = time.time() + stats = classifier_trainer.run(flags.FLAGS) + wall_time_sec = time.time() - start_time_sec + + super(Resnet50KerasAccuracy, self)._report_benchmark( + stats, + wall_time_sec, + top_1_min=top_1_min, + top_1_max=top_1_max, + total_batch_size=total_batch_size, + log_steps=100) + + def benchmark_8_gpu(self): + """Tests Keras model with eager, dist_strat and 8 GPUs.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_8_gpu', + num_gpus=8, + per_replica_batch_size=128, + epochs=90, + epochs_between_evals=10, + dtype='float32') + + def benchmark_8_gpu_fp16(self): + """Tests Keras model with eager, dist_strat, 8 GPUs, and fp16.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_8_gpu_fp16', + num_gpus=8, + per_replica_batch_size=256, + epochs=90, + epochs_between_evals=10, + dtype='float16') + + def benchmark_xla_8_gpu_fp16(self): + """Tests Keras model with XLA, eager, dist_strat, 8 GPUs and fp16.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_xla_8_gpu_fp16', + num_gpus=8, + per_replica_batch_size=256, + epochs=90, + epochs_between_evals=10, + dtype='float16', + enable_xla=True) + + def benchmark_xla_8_gpu_fp16_dynamic(self): + """Tests Keras model with XLA, eager, dist_strat, 8 GPUs, dynamic fp16.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_xla_8_gpu_fp16_dynamic', + top_1_min=0.736, + num_gpus=8, + per_replica_batch_size=256, + epochs=90, + epochs_between_evals=10, + dtype='float16', + loss_scale='dynamic') + + def _get_model_dir(self, folder_name): + return os.path.join(self.output_dir, folder_name) + + +class MobilenetV1KerasAccuracy(keras_benchmark.KerasBenchmark): + """Benchmark accuracy tests for MobilenetV1 in Keras.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + """A benchmark class. + + Args: + output_dir: directory where to output e.g. log files + root_data_dir: directory under which to look for dataset + **kwargs: arbitrary named arguments. This is needed to make the + constructor forward compatible in case PerfZero provides more + named arguments before updating the constructor. + """ + + flag_methods = [resnet_imagenet_main.define_imagenet_keras_flags] + + self.data_dir = os.path.join(root_data_dir, 'imagenet') + super(MobilenetV1KerasAccuracy, self).__init__( + output_dir=output_dir, + flag_methods=flag_methods, + default_flags={ + 'model': 'mobilenet', + 'optimizer': 'mobilenet_default', + 'initial_learning_rate_per_sample': 0.00039, + }) + + def benchmark_8_gpu(self): + """Test Keras model with eager, dist_strat and 8 GPUs.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 128 * 8 + FLAGS.train_epochs = 90 + FLAGS.epochs_between_evals = 10 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu') + FLAGS.dtype = 'fp32' + FLAGS.enable_eager = True + self._run_and_report_benchmark() + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self, + top_1_min=MOBILENET_V1_MIN_TOP_1_ACCURACY, + top_1_max=MOBILENET_V1_MAX_TOP_1_ACCURACY): + start_time_sec = time.time() + stats = resnet_imagenet_main.run(flags.FLAGS) + wall_time_sec = time.time() - start_time_sec + + super(MobilenetV1KerasAccuracy, self)._report_benchmark( + stats, + wall_time_sec, + top_1_min=top_1_min, + top_1_max=top_1_max, + total_batch_size=FLAGS.batch_size, + log_steps=100) + + def _get_model_dir(self, folder_name): + return os.path.join(self.output_dir, folder_name) + + +class Resnet50KerasClassifierBenchmarkBase(keras_benchmark.KerasBenchmark): + """Resnet50 (classifier_trainer) benchmarks.""" + + def __init__(self, output_dir=None, default_flags=None, + tpu=None, dataset_builder='records', train_epochs=1, + train_steps=110, data_dir=None): + flag_methods = [classifier_trainer.define_classifier_flags] + + self.dataset_builder = dataset_builder + self.train_epochs = train_epochs + self.train_steps = train_steps + self.data_dir = data_dir + + super(Resnet50KerasClassifierBenchmarkBase, self).__init__( + output_dir=output_dir, + flag_methods=flag_methods, + default_flags=default_flags, + tpu=tpu) + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark( + self, + experiment_name: str, + skip_steps: Optional[int] = None, + top_1_min: float = MIN_TOP_1_ACCURACY, + top_1_max: float = MAX_TOP_1_ACCURACY, + num_gpus: int = 0, + distribution_strategy: str = 'mirrored', + per_replica_batch_size: int = 128, + epochs_between_evals: int = 1, + dtype: str = 'float32', + enable_xla: bool = False, + run_eagerly: bool = False, + gpu_thread_mode: Optional[str] = None, + dataset_num_private_threads: Optional[int] = None, + loss_scale: Optional[str] = None): + """Runs and reports the benchmark given the provided configuration.""" + FLAGS.model_type = 'resnet' + FLAGS.dataset = 'imagenet' + FLAGS.mode = 'train_and_eval' + FLAGS.data_dir = self.data_dir + FLAGS.model_dir = self._get_model_dir(experiment_name) + parameters = _get_classifier_parameters( + builder=self.dataset_builder, + skip_eval=True, + num_gpus=num_gpus, + distribution_strategy=distribution_strategy, + per_replica_batch_size=per_replica_batch_size, + epochs=self.train_epochs, + steps=self.train_steps, + epochs_between_evals=epochs_between_evals, + dtype=dtype, + enable_xla=enable_xla, + gpu_thread_mode=gpu_thread_mode, + dataset_num_private_threads=dataset_num_private_threads, + loss_scale=loss_scale) + FLAGS.params_override = json.dumps(parameters) + total_batch_size = num_gpus * per_replica_batch_size + + start_time_sec = time.time() + stats = classifier_trainer.run(flags.FLAGS) + wall_time_sec = time.time() - start_time_sec + # Number of logged step time entries that are excluded in performance + # report. We keep results from last 100 batches, or skip the steps based on + # input skip_steps. + warmup = (skip_steps or (self.train_steps - 100)) // FLAGS.log_steps + + super(Resnet50KerasClassifierBenchmarkBase, self)._report_benchmark( + stats, + wall_time_sec, + total_batch_size=total_batch_size, + log_steps=FLAGS.log_steps, + warmup=warmup, + start_time_sec=start_time_sec) + + def benchmark_1_gpu_no_dist_strat(self): + """Tests Keras model with 1 GPU, no distribution strategy.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_1_gpu_no_dist_strat', + num_gpus=1, + distribution_strategy='off', + per_replica_batch_size=128) + + def benchmark_1_gpu_no_dist_strat_run_eagerly(self): + """Tests Keras model with 1 GPU, no distribution strategy, run eagerly.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_1_gpu_no_dist_strat_run_eagerly', + num_gpus=1, + run_eagerly=True, + distribution_strategy='off', + per_replica_batch_size=64) + + def benchmark_1_gpu_no_dist_strat_run_eagerly_fp16(self): + """Tests with 1 GPU, no distribution strategy, fp16, run eagerly.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_1_gpu_no_dist_strat_run_eagerly_fp16', + num_gpus=1, + run_eagerly=True, + distribution_strategy='off', + dtype='float16', + per_replica_batch_size=128) + + def benchmark_1_gpu(self): + """Tests Keras model with 1 GPU.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_1_gpu', + num_gpus=1, + distribution_strategy='one_device', + per_replica_batch_size=128) + + def benchmark_xla_1_gpu(self): + """Tests Keras model with XLA and 1 GPU.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_xla_1_gpu', + num_gpus=1, + enable_xla=True, + distribution_strategy='one_device', + per_replica_batch_size=128) + + def benchmark_1_gpu_fp16(self): + """Tests Keras model with 1 GPU and fp16.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_1_gpu_fp16', + num_gpus=1, + distribution_strategy='one_device', + dtype='float16', + per_replica_batch_size=256) + + def benchmark_1_gpu_fp16_dynamic(self): + """Tests Keras model with 1 GPU, fp16, and dynamic loss scaling.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_1_gpu_fp16_dynamic', + num_gpus=1, + distribution_strategy='one_device', + dtype='float16', + per_replica_batch_size=256, + loss_scale='dynamic') + + def benchmark_xla_1_gpu_fp16(self): + """Tests Keras model with XLA, 1 GPU and fp16.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_xla_1_gpu_fp16', + num_gpus=1, + enable_xla=True, + distribution_strategy='one_device', + dtype='float16', + per_replica_batch_size=256) + + def benchmark_xla_1_gpu_fp16_tweaked(self): + """Tests Keras model with XLA, 1 GPU, fp16, and manual config tuning.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_xla_1_gpu_fp16_tweaked', + num_gpus=1, + enable_xla=True, + distribution_strategy='one_device', + dtype='float16', + per_replica_batch_size=256, + gpu_thread_mode='gpu_private') + + def benchmark_xla_1_gpu_fp16_dynamic(self): + """Tests Keras model with XLA, 1 GPU, fp16, and dynamic loss scaling.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_xla_1_gpu_fp16_dynamic', + num_gpus=1, + enable_xla=True, + distribution_strategy='one_device', + dtype='float16', + per_replica_batch_size=256, + loss_scale='dynamic') + + def benchmark_8_gpu(self): + """Tests Keras model with 8 GPUs.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_8_gpu', + num_gpus=8, + distribution_strategy='mirrored', + per_replica_batch_size=128) + + def benchmark_8_gpu_tweaked(self): + """Tests Keras model with manual config tuning and 8 GPUs.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_8_gpu_tweaked', + num_gpus=8, + distribution_strategy='mirrored', + per_replica_batch_size=128, + dataset_num_private_threads=14) + + def benchmark_xla_8_gpu(self): + """Tests Keras model with XLA and 8 GPUs.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_xla_8_gpu', + num_gpus=8, + enable_xla=True, + distribution_strategy='mirrored', + per_replica_batch_size=128) + + def benchmark_xla_8_gpu_tweaked(self): + """Tests Keras model with manual config tuning, 8 GPUs, and XLA.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_xla_8_gpu_tweaked', + num_gpus=8, + enable_xla=True, + distribution_strategy='mirrored', + per_replica_batch_size=128, + gpu_thread_mode='gpu_private', + dataset_num_private_threads=24) + + def benchmark_8_gpu_fp16(self): + """Tests Keras model with 8 GPUs and fp16.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_8_gpu_fp16', + num_gpus=8, + dtype='float16', + distribution_strategy='mirrored', + per_replica_batch_size=256) + + def benchmark_8_gpu_fp16_tweaked(self): + """Tests Keras model with 8 GPUs, fp16, and manual config tuning.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_8_gpu_fp16_tweaked', + num_gpus=8, + dtype='float16', + distribution_strategy='mirrored', + per_replica_batch_size=256, + gpu_thread_mode='gpu_private', + dataset_num_private_threads=40) + + def benchmark_8_gpu_fp16_dynamic_tweaked(self): + """Tests Keras model with 8 GPUs, fp16, dynamic loss scaling, and tuned.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_8_gpu_fp16_dynamic_tweaked', + num_gpus=8, + dtype='float16', + distribution_strategy='mirrored', + per_replica_batch_size=256, + loss_scale='dynamic', + gpu_thread_mode='gpu_private', + dataset_num_private_threads=40) + + def benchmark_xla_8_gpu_fp16(self): + """Tests Keras model with XLA, 8 GPUs and fp16.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_xla_8_gpu_fp16', + dtype='float16', + num_gpus=8, + enable_xla=True, + distribution_strategy='mirrored', + per_replica_batch_size=256) + + def benchmark_xla_8_gpu_fp16_tweaked(self): + """Test Keras model with manual config tuning, XLA, 8 GPUs and fp16.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_xla_8_gpu_fp16_tweaked', + dtype='float16', + num_gpus=8, + enable_xla=True, + distribution_strategy='mirrored', + per_replica_batch_size=256, + gpu_thread_mode='gpu_private', + dataset_num_private_threads=48) + + def benchmark_xla_8_gpu_fp16_tweaked_delay_measure(self): + """Tests with manual config tuning, XLA, 8 GPUs and fp16. + + Delay performance measurement for stable performance on 96 vCPU platforms. + """ + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_xla_8_gpu_fp16_tweaked_delay_measure', + dtype='float16', + num_gpus=8, + enable_xla=True, + distribution_strategy='mirrored', + per_replica_batch_size=256, + gpu_thread_mode='gpu_private', + dataset_num_private_threads=48, + steps=310) + + def benchmark_xla_8_gpu_fp16_dynamic_tweaked(self): + """Tests Keras model with config tuning, XLA, 8 GPUs and dynamic fp16.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_xla_8_gpu_fp16_dynamic_tweaked', + dtype='float16', + num_gpus=8, + enable_xla=True, + distribution_strategy='mirrored', + per_replica_batch_size=256, + gpu_thread_mode='gpu_private', + loss_scale='dynamic', + dataset_num_private_threads=48) + + def benchmark_2x2_tpu_fp16(self): + """Test Keras model with 2x2 TPU, fp16.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_2x2_tpu_fp16', + dtype='bfloat16', + distribution_strategy='tpu', + per_replica_batch_size=128) + + def benchmark_4x4_tpu_fp16(self): + """Test Keras model with 4x4 TPU, fp16.""" + self._setup() + self._run_and_report_benchmark( + experiment_name='benchmark_4x4_tpu_fp16', + dtype='bfloat16', + distribution_strategy='tpu', + per_replica_batch_size=128) + + def fill_report_object(self, stats): + super(Resnet50KerasClassifierBenchmarkBase, self).fill_report_object( + stats, + total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + +class Resnet50KerasBenchmarkBase(keras_benchmark.KerasBenchmark): + """Resnet50 benchmarks.""" + + def __init__(self, output_dir=None, default_flags=None, tpu=None): + flag_methods = [resnet_imagenet_main.define_imagenet_keras_flags] + + super(Resnet50KerasBenchmarkBase, self).__init__( + output_dir=output_dir, + flag_methods=flag_methods, + default_flags=default_flags, + tpu=tpu) + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self, skip_steps=None): + start_time_sec = time.time() + stats = resnet_imagenet_main.run(FLAGS) + wall_time_sec = time.time() - start_time_sec + # Number of logged step time entries that are excluded in performance + # report. We keep results from last 100 batches, or skip the steps based on + # input skip_steps. + warmup = (skip_steps or (FLAGS.train_steps - 100)) // FLAGS.log_steps + + super(Resnet50KerasBenchmarkBase, self)._report_benchmark( + stats, + wall_time_sec, + total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps, + warmup=warmup, + start_time_sec=start_time_sec) + + def benchmark_1_gpu_no_dist_strat(self): + """Test Keras model with 1 GPU, no distribution strategy.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.distribution_strategy = 'off' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_no_dist_strat') + FLAGS.batch_size = 128 + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_dist_strat_run_eagerly(self): + """Test Keras model with 1 GPU, no distribution strategy, run eagerly.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.run_eagerly = True + FLAGS.distribution_strategy = 'off' + FLAGS.model_dir = self._get_model_dir( + 'benchmark_1_gpu_no_dist_strat_run_eagerly') + FLAGS.batch_size = 64 + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_dist_strat_run_eagerly_tweaked(self): + """Test Keras model with 1 GPU, no distribution strategy, run eagerly.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.run_eagerly = True + FLAGS.explicit_gpu_placement = True + FLAGS.distribution_strategy = 'off' + FLAGS.model_dir = self._get_model_dir( + 'benchmark_1_gpu_no_dist_strat_run_eagerly_tweaked') + FLAGS.batch_size = 64 + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_dist_strat_run_eagerly_fp16(self): + """Test with 1 GPU, no distribution strategy, fp16, run eagerly.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.run_eagerly = True + FLAGS.distribution_strategy = 'off' + FLAGS.model_dir = self._get_model_dir( + 'benchmark_1_gpu_no_dist_strat_run_eagerly_fp16') + FLAGS.dtype = 'fp16' + FLAGS.batch_size = 128 + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_dist_strat_run_eagerly_fp16_tweaked(self): + """Test with 1 GPU, no distribution strategy, fp16, run eagerly.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.run_eagerly = True + FLAGS.explicit_gpu_placement = True + FLAGS.distribution_strategy = 'off' + FLAGS.model_dir = self._get_model_dir( + 'benchmark_1_gpu_no_dist_strat_run_eagerly_fp16_tweaked') + FLAGS.dtype = 'fp16' + FLAGS.batch_size = 128 + self._run_and_report_benchmark() + + def benchmark_1_gpu(self): + """Test Keras model with 1 GPU.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu') + FLAGS.batch_size = 128 + self._run_and_report_benchmark() + + def benchmark_1_gpu_amp(self): + """Test Keras model with 1 GPU with automatic mixed precision.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_amp') + FLAGS.batch_size = 256 + self._run_and_report_benchmark() + + def benchmark_xla_1_gpu(self): + """Test Keras model with XLA and 1 GPU.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.enable_xla = True + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu') + FLAGS.batch_size = 128 + self._run_and_report_benchmark() + + def benchmark_xla_1_gpu_amp(self): + """Test Keras model with XLA and 1 GPU with automatic mixed precision.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.enable_xla = True + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu_amp') + FLAGS.batch_size = 256 + self._run_and_report_benchmark() + + def benchmark_1_gpu_fp16(self): + """Test Keras model with 1 GPU and fp16.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_fp16') + FLAGS.dtype = 'fp16' + FLAGS.batch_size = 256 + self._run_and_report_benchmark() + + def benchmark_1_gpu_fp16_dynamic(self): + """Test Keras model with 1 GPU, fp16, and dynamic loss scaling.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_fp16_dynamic') + FLAGS.dtype = 'fp16' + FLAGS.batch_size = 256 + FLAGS.loss_scale = 'dynamic' + self._run_and_report_benchmark() + + def benchmark_xla_1_gpu_fp16(self): + """Test Keras model with XLA, 1 GPU and fp16.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.enable_xla = True + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu_fp16') + FLAGS.dtype = 'fp16' + FLAGS.batch_size = 256 + self._run_and_report_benchmark() + + def benchmark_xla_1_gpu_fp16_tweaked(self): + """Test Keras model with XLA, 1 GPU, fp16, and manual config tuning.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.enable_xla = True + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu_fp16_tweaked') + FLAGS.dtype = 'fp16' + FLAGS.batch_size = 256 + FLAGS.tf_gpu_thread_mode = 'gpu_private' + self._run_and_report_benchmark() + + def benchmark_xla_1_gpu_fp16_dynamic(self): + """Test Keras model with XLA, 1 GPU, fp16, and dynamic loss scaling.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.enable_xla = True + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu_fp16_dynamic') + FLAGS.dtype = 'fp16' + FLAGS.batch_size = 256 + FLAGS.loss_scale = 'dynamic' + self._run_and_report_benchmark() + + def benchmark_8_gpu(self): + """Test Keras model with 8 GPUs.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.enable_eager = True + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu') + FLAGS.batch_size = 128 * 8 # 8 GPUs + self._run_and_report_benchmark() + + def benchmark_8_gpu_amp(self): + """Test Keras model with 8 GPUs with automatic mixed precision.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.enable_eager = True + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_amp') + FLAGS.batch_size = 256 * 8 # 8 GPUs + self._run_and_report_benchmark() + + def benchmark_8_gpu_tweaked(self): + """Test Keras model with manual config tuning and 8 GPUs.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.enable_eager = True + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_tweaked') + FLAGS.batch_size = 128 * 8 # 8 GPUs + FLAGS.datasets_num_private_threads = 14 + self._run_and_report_benchmark() + + def benchmark_xla_8_gpu(self): + """Test Keras model with XLA and 8 GPUs.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.enable_eager = True + FLAGS.enable_xla = True + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir('benchmark_xla_8_gpu') + FLAGS.batch_size = 128 * 8 # 8 GPUs + self._run_and_report_benchmark() + + def benchmark_xla_8_gpu_amp(self): + """Test Keras model with XLA and 8 GPUs with automatic mixed precision.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.enable_eager = True + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.enable_xla = True + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir('benchmark_xla_8_gpu_amp') + FLAGS.batch_size = 256 * 8 # 8 GPUs + self._run_and_report_benchmark() + + def benchmark_xla_8_gpu_tweaked(self): + """Test Keras model with manual config tuning, 8 GPUs, and XLA.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.enable_eager = True + FLAGS.enable_xla = True + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir('benchmark_xla_8_gpu_tweaked') + FLAGS.batch_size = 128 * 8 + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.datasets_num_private_threads = 24 + self._run_and_report_benchmark() + + def benchmark_8_gpu_fp16(self): + """Test Keras model with 8 GPUs and fp16.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.dtype = 'fp16' + FLAGS.enable_eager = True + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_fp16') + FLAGS.batch_size = 256 * 8 # 8 GPUs + self._run_and_report_benchmark() + + def benchmark_8_gpu_fp16_tweaked(self): + """Test Keras model with 8 GPUs, fp16, and manual config tuning.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.dtype = 'fp16' + FLAGS.enable_eager = True + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_fp16_tweaked') + FLAGS.batch_size = 256 * 8 # 8 GPUs + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.dataset_num_private_threads = 40 + self._run_and_report_benchmark() + + def benchmark_8_gpu_fp16_dynamic_tweaked(self): + """Test Keras model with 8 GPUs, fp16, dynamic loss scaling, and tuned.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.dtype = 'fp16' + FLAGS.enable_eager = True + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir( + 'benchmark_8_gpu_fp16_dynamic_tweaked') + FLAGS.batch_size = 256 * 8 # 8 GPUs + FLAGS.loss_scale = 'dynamic' + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.dataset_num_private_threads = 40 + self._run_and_report_benchmark() + + def benchmark_xla_8_gpu_fp16(self): + """Test Keras model with XLA, 8 GPUs and fp16.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.dtype = 'fp16' + FLAGS.enable_eager = True + FLAGS.enable_xla = True + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir('benchmark_xla_8_gpu_fp16') + FLAGS.batch_size = 256 * 8 # 8 GPUs + self._run_and_report_benchmark() + + def benchmark_xla_8_gpu_fp16_tweaked(self): + """Test Keras model with manual config tuning, XLA, 8 GPUs and fp16.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.dtype = 'fp16' + FLAGS.enable_eager = True + FLAGS.enable_xla = True + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir('benchmark_xla_8_gpu_fp16_tweaked') + FLAGS.batch_size = 256 * 8 # 8 GPUs + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.datasets_num_private_threads = 48 + self._run_and_report_benchmark() + + def benchmark_xla_8_gpu_fp16_tweaked_delay_measure(self): + """Test with manual config tuning, XLA, 8 GPUs and fp16. + + Delay performance measurement for stable performance on 96 vCPU platforms. + """ + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.dtype = 'fp16' + FLAGS.enable_eager = True + FLAGS.enable_xla = True + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir( + 'benchmark_xla_8_gpu_fp16_tweaked_delay_measure') + FLAGS.batch_size = 256 * 8 + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.datasets_num_private_threads = 48 + FLAGS.train_steps = 310 + self._run_and_report_benchmark() + + def benchmark_xla_8_gpu_fp16_dynamic_tweaked(self): + """Test Keras model with config tuning, XLA, 8 GPUs and dynamic fp16.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.dtype = 'fp16' + FLAGS.enable_eager = True + FLAGS.enable_xla = True + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir( + 'benchmark_xla_8_gpu_fp16_dynamic_tweaked') + FLAGS.batch_size = 256 * 8 # 8 GPUs + FLAGS.loss_scale = 'dynamic' + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.datasets_num_private_threads = 48 + self._run_and_report_benchmark() + + def benchmark_2x2_tpu_fp16(self): + """Test Keras model with 2x2 TPU, fp16.""" + self._setup() + + FLAGS.dtype = 'bf16' + FLAGS.distribution_strategy = 'tpu' + FLAGS.model_dir = self._get_model_dir('benchmark_2x2_tpu_fp16') + FLAGS.batch_size = 1024 + self._run_and_report_benchmark() + + def benchmark_4x4_tpu_fp16(self): + """Test Keras model with 4x4 TPU, fp16.""" + self._setup() + + FLAGS.dtype = 'bf16' + FLAGS.distribution_strategy = 'tpu' + FLAGS.model_dir = self._get_model_dir('benchmark_4x4_tpu_fp16') + FLAGS.batch_size = 4096 + self._run_and_report_benchmark() + + def fill_report_object(self, stats): + super(Resnet50KerasBenchmarkBase, self).fill_report_object( + stats, + total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + +class Resnet50KerasBenchmarkSynth(Resnet50KerasClassifierBenchmarkBase): + """Resnet50 synthetic benchmark tests.""" + + def __init__(self, output_dir=None, root_data_dir=None, tpu=None, **kwargs): + def_flags = {} + def_flags['log_steps'] = 10 + + super(Resnet50KerasBenchmarkSynth, self).__init__( + output_dir=output_dir, default_flags=def_flags, tpu=tpu, + dataset_builder='synthetic', train_epochs=1, train_steps=110) + + +class Resnet50KerasBenchmarkReal(Resnet50KerasClassifierBenchmarkBase): + """Resnet50 real data benchmark tests.""" + + def __init__(self, output_dir=None, root_data_dir=None, tpu=None, **kwargs): + data_dir = os.path.join(root_data_dir, 'imagenet') + def_flags = {} + def_flags['log_steps'] = 10 + + super(Resnet50KerasBenchmarkReal, self).__init__( + output_dir=output_dir, default_flags=def_flags, tpu=tpu, + dataset_builder='records', train_epochs=1, train_steps=110, + data_dir=data_dir) + + +class Resnet50KerasBenchmarkRemoteData(Resnet50KerasBenchmarkBase): + """Resnet50 real data (stored in remote storage) benchmark tests.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + def_flags = {} + def_flags['skip_eval'] = True + def_flags['report_accuracy_metrics'] = False + def_flags['data_dir'] = os.path.join(root_data_dir, 'imagenet') + # Defining multiple epochs overrides the train_steps setting in benchmarks. + def_flags['train_epochs'] = 2 + # Cache dataset so performance is stable after the first epoch. + def_flags['training_dataset_cache'] = True + def_flags['log_steps'] = 100 + # Note that for single GPU and pure eager tests which are less likely to be + # input bound and more stable, these tests will run for shorter time by + # overriding FLAGS.train_epochs, train_seteps, log_steps in benchmark + # methods, and skip_steps in _run_and_report_benchmark(). + + super(Resnet50KerasBenchmarkRemoteData, self).__init__( + output_dir=output_dir, default_flags=def_flags) + + def _override_flags_to_run_test_shorter(self): + FLAGS.train_epochs = 1 + FLAGS.train_steps = 300 + FLAGS.log_steps = 10 + + def benchmark_1_gpu_no_dist_strat(self): + """Test Keras model with 1 GPU, no distribution strategy.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.distribution_strategy = 'off' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_no_dist_strat') + FLAGS.batch_size = 128 + self._override_flags_to_run_test_shorter() + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_dist_strat_run_eagerly(self): + """Test Keras model with 1 GPU, no distribution strategy, run eagerly.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.run_eagerly = True + FLAGS.distribution_strategy = 'off' + FLAGS.model_dir = self._get_model_dir( + 'benchmark_1_gpu_no_dist_strat_run_eagerly') + FLAGS.batch_size = 64 + self._override_flags_to_run_test_shorter() + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_dist_strat_run_eagerly_tweaked(self): + """Test Keras model with 1 GPU, no distribution strategy, run eagerly.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.run_eagerly = True + FLAGS.explicit_gpu_placement = True + FLAGS.distribution_strategy = 'off' + FLAGS.model_dir = self._get_model_dir( + 'benchmark_1_gpu_no_dist_strat_run_eagerly_tweaked') + FLAGS.batch_size = 64 + self._override_flags_to_run_test_shorter() + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_dist_strat_run_eagerly_fp16(self): + """Test with 1 GPU, no distribution strategy, fp16, run eagerly.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.run_eagerly = True + FLAGS.distribution_strategy = 'off' + FLAGS.model_dir = self._get_model_dir( + 'benchmark_1_gpu_no_dist_strat_run_eagerly_fp16') + FLAGS.dtype = 'fp16' + FLAGS.batch_size = 128 + self._override_flags_to_run_test_shorter() + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_dist_strat_run_eagerly_fp16_tweaked(self): + """Test with 1 GPU, no distribution strategy, fp16, run eagerly.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.run_eagerly = True + FLAGS.explicit_gpu_placement = True + FLAGS.distribution_strategy = 'off' + FLAGS.model_dir = self._get_model_dir( + 'benchmark_1_gpu_no_dist_strat_run_eagerly_fp16_tweaked') + FLAGS.dtype = 'fp16' + FLAGS.batch_size = 128 + self._override_flags_to_run_test_shorter() + self._run_and_report_benchmark() + + def benchmark_1_gpu(self): + """Test Keras model with 1 GPU.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu') + FLAGS.batch_size = 128 + self._override_flags_to_run_test_shorter() + self._run_and_report_benchmark() + + def benchmark_1_gpu_amp(self): + """Test Keras model with 1 GPU with automatic mixed precision.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_amp') + FLAGS.batch_size = 256 + self._override_flags_to_run_test_shorter() + self._run_and_report_benchmark() + + def benchmark_xla_1_gpu(self): + """Test Keras model with XLA and 1 GPU.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.enable_xla = True + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu') + FLAGS.batch_size = 128 + self._override_flags_to_run_test_shorter() + self._run_and_report_benchmark() + + def benchmark_xla_1_gpu_amp(self): + """Test Keras model with XLA and 1 GPU with automatic mixed precision.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.enable_xla = True + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu_amp') + FLAGS.batch_size = 256 + self._override_flags_to_run_test_shorter() + self._run_and_report_benchmark() + + def benchmark_1_gpu_fp16(self): + """Test Keras model with 1 GPU and fp16.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_fp16') + FLAGS.dtype = 'fp16' + FLAGS.batch_size = 256 + self._override_flags_to_run_test_shorter() + self._run_and_report_benchmark() + + def benchmark_1_gpu_fp16_dynamic(self): + """Test Keras model with 1 GPU, fp16, and dynamic loss scaling.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_fp16_dynamic') + FLAGS.dtype = 'fp16' + FLAGS.batch_size = 256 + FLAGS.loss_scale = 'dynamic' + self._override_flags_to_run_test_shorter() + self._run_and_report_benchmark() + + def benchmark_xla_1_gpu_fp16(self): + """Test Keras model with XLA, 1 GPU and fp16.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.enable_xla = True + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu_fp16') + FLAGS.dtype = 'fp16' + FLAGS.batch_size = 256 + self._override_flags_to_run_test_shorter() + self._run_and_report_benchmark() + + def benchmark_xla_1_gpu_fp16_tweaked(self): + """Test Keras model with XLA, 1 GPU, fp16, and manual config tuning.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.enable_xla = True + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu_fp16_tweaked') + FLAGS.dtype = 'fp16' + FLAGS.batch_size = 256 + FLAGS.tf_gpu_thread_mode = 'gpu_private' + self._override_flags_to_run_test_shorter() + self._run_and_report_benchmark() + + def benchmark_xla_1_gpu_fp16_dynamic(self): + """Test Keras model with XLA, 1 GPU, fp16, and dynamic loss scaling.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.enable_eager = True + FLAGS.enable_xla = True + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu_fp16_dynamic') + FLAGS.dtype = 'fp16' + FLAGS.batch_size = 256 + FLAGS.loss_scale = 'dynamic' + self._override_flags_to_run_test_shorter() + self._run_and_report_benchmark() + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self): + if FLAGS.num_gpus == 1 or FLAGS.run_eagerly: + # For single GPU and pure eager tests which are less likely to be input + # bound and more stable, run for shorter time and use the default + # skip_steps. + skip_steps = None + else: + # skip the first epoch for performance measurement. + skip_steps = 600 + super(Resnet50KerasBenchmarkRemoteData, + self)._run_and_report_benchmark(skip_steps=skip_steps) + + +class TrivialKerasBenchmarkReal(keras_benchmark.KerasBenchmark): + """Trivial model with real data benchmark tests.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + flag_methods = [resnet_imagenet_main.define_imagenet_keras_flags] + + def_flags = {} + def_flags['use_trivial_model'] = True + def_flags['skip_eval'] = True + def_flags['report_accuracy_metrics'] = False + def_flags['dtype'] = 'fp16' + def_flags['data_dir'] = os.path.join(root_data_dir, 'imagenet') + def_flags['train_steps'] = 600 + def_flags['log_steps'] = 100 + def_flags['distribution_strategy'] = 'mirrored' + + super(TrivialKerasBenchmarkReal, self).__init__( + output_dir=output_dir, + flag_methods=flag_methods, + default_flags=def_flags) + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self): + start_time_sec = time.time() + stats = resnet_imagenet_main.run(FLAGS) + wall_time_sec = time.time() - start_time_sec + + super(TrivialKerasBenchmarkReal, self)._report_benchmark( + stats, + wall_time_sec, + total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_8_gpu_warmup(self): + """Dummy test that runs over an epoch to warmup the machine.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.enable_eager = True + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_warmup') + FLAGS.batch_size = 256 * 8 + FLAGS.train_steps = 700 + self._run_and_report_benchmark() + + def fill_report_object(self, stats): + super(TrivialKerasBenchmarkReal, self).fill_report_object( + stats, + total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + +class Resnet50MultiWorkerKerasAccuracy(keras_benchmark.KerasBenchmark): + """Resnet50 distributed accuracy tests with multiple workers.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + flag_methods = [classifier_trainer.define_imagenet_keras_flags] + self.data_dir = os.path.join(root_data_dir, 'imagenet') + super(Resnet50MultiWorkerKerasAccuracy, self).__init__( + output_dir=output_dir, flag_methods=flag_methods) + + def _benchmark_common(self, eager, num_workers, all_reduce_alg): + """Common to all benchmarks in this class.""" + self._setup() + + num_gpus = 8 + FLAGS.num_gpus = num_gpus + FLAGS.data_dir = self.data_dir + FLAGS.train_epochs = 90 + FLAGS.epochs_between_evals = 10 + FLAGS.dtype = 'fp16' + FLAGS.enable_eager = eager + FLAGS.enable_xla = False + FLAGS.distribution_strategy = 'multi_worker_mirrored' + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.datasets_num_private_threads = 32 + FLAGS.model_dir = self._get_model_dir( + 'benchmark_{}_8_gpu_{}_worker_fp16_{}_tweaked'.format( + 'eager' if eager else 'graph', num_workers, all_reduce_alg)) + FLAGS.batch_size = 256 * num_gpus * num_workers + FLAGS.all_reduce_alg = all_reduce_alg + + self._run_and_report_benchmark() + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self, + top_1_min=MIN_TOP_1_ACCURACY, + top_1_max=MAX_TOP_1_ACCURACY): + start_time_sec = time.time() + stats = classifier_trainer.run(flags.FLAGS) + wall_time_sec = time.time() - start_time_sec + + super(Resnet50MultiWorkerKerasAccuracy, self)._report_benchmark( + stats, + wall_time_sec, + top_1_min=top_1_min, + top_1_max=top_1_max, + total_batch_size=FLAGS.batch_size, + log_steps=100) + + def _get_model_dir(self, folder_name): + return os.path.join(self.output_dir, folder_name) + + def benchmark_eager_8_gpu_2_workers_fp16_ring_tweaked(self): + """Eager, 8 GPUs per worker, 2 workers, fp16, ring all-reduce.""" + self._benchmark_common(eager=True, num_workers=2, all_reduce_alg='ring') + + def benchmark_eager_8_gpu_2_workers_fp16_nccl_tweaked(self): + """Eager, 8 GPUs per worker, 2 workers, fp16, nccl all-reduce.""" + self._benchmark_common(eager=True, num_workers=2, all_reduce_alg='nccl') + + def benchmark_eager_8_gpu_8_workers_fp16_ring_tweaked(self): + """Eager, 8 GPUs per worker, 8 workers, fp16, ring all-reduce.""" + self._benchmark_common(eager=True, num_workers=8, all_reduce_alg='ring') + + def benchmark_eager_8_gpu_8_workers_fp16_nccl_tweaked(self): + """Eager, 8 GPUs per worker, 8 workers, fp16, nccl all-reduce.""" + self._benchmark_common(eager=True, num_workers=8, all_reduce_alg='nccl') + + +class Resnet50MultiWorkerKerasBenchmark(Resnet50KerasBenchmarkBase): + """Resnet50 distributed benchmark tests with multiple workers.""" + + def __init__(self, output_dir=None, default_flags=None): + super(Resnet50MultiWorkerKerasBenchmark, self).__init__( + output_dir=output_dir, default_flags=default_flags) + + def _benchmark_common(self, eager, num_workers, all_reduce_alg): + """Common to all benchmarks in this class.""" + self._setup() + + num_gpus = 8 + FLAGS.num_gpus = num_gpus + FLAGS.dtype = 'fp16' + FLAGS.enable_eager = eager + FLAGS.enable_xla = False + FLAGS.distribution_strategy = 'multi_worker_mirrored' + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.datasets_num_private_threads = 32 + FLAGS.model_dir = self._get_model_dir( + 'benchmark_{}_8_gpu_{}_worker_fp16_{}_tweaked'.format( + 'eager' if eager else 'graph', num_workers, all_reduce_alg)) + FLAGS.batch_size = 256 * num_gpus * num_workers + FLAGS.all_reduce_alg = all_reduce_alg + + self._run_and_report_benchmark() + + def benchmark_eager_8_gpu_1_worker_fp16_ring_tweaked(self): + """Eager, 8 GPUs per worker, 1 worker, fp16, ring all-reduce.""" + self._benchmark_common(eager=True, num_workers=1, all_reduce_alg='ring') + + def benchmark_eager_8_gpu_1_worker_fp16_nccl_tweaked(self): + """Eager, 8 GPUs per worker, 1 worker, fp16, nccl all-reduce.""" + self._benchmark_common(eager=True, num_workers=1, all_reduce_alg='nccl') + + def benchmark_eager_8_gpu_2_workers_fp16_ring_tweaked(self): + """Eager, 8 GPUs per worker, 2 workers, fp16, ring all-reduce.""" + self._benchmark_common(eager=True, num_workers=2, all_reduce_alg='ring') + + def benchmark_eager_8_gpu_2_workers_fp16_nccl_tweaked(self): + """Eager, 8 GPUs per worker, 2 workers, fp16, nccl all-reduce.""" + self._benchmark_common(eager=True, num_workers=2, all_reduce_alg='nccl') + + def benchmark_eager_8_gpu_8_workers_fp16_ring_tweaked(self): + """Eager, 8 GPUs per worker, 8 workers, fp16, ring all-reduce.""" + self._benchmark_common(eager=True, num_workers=8, all_reduce_alg='ring') + + def benchmark_eager_8_gpu_8_workers_fp16_nccl_tweaked(self): + """Eager, 8 GPUs per worker, 8 workers, fp16, nccl all-reduce.""" + self._benchmark_common(eager=True, num_workers=8, all_reduce_alg='nccl') + + +class Resnet50MultiWorkerKerasBenchmarkSynth(Resnet50MultiWorkerKerasBenchmark): + """Resnet50 multi-worker synthetic data benchmark tests.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + def_flags = {} + def_flags['skip_eval'] = True + def_flags['report_accuracy_metrics'] = False + def_flags['use_synthetic_data'] = True + def_flags['train_steps'] = 110 + def_flags['log_steps'] = 10 + + super(Resnet50MultiWorkerKerasBenchmarkSynth, self).__init__( + output_dir=output_dir, default_flags=def_flags) + + +class Resnet50MultiWorkerKerasBenchmarkReal(Resnet50MultiWorkerKerasBenchmark): + """Resnet50 multi-worker real data benchmark tests.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + def_flags = {} + def_flags['skip_eval'] = True + def_flags['report_accuracy_metrics'] = False + def_flags['data_dir'] = os.path.join(root_data_dir, 'imagenet') + def_flags['train_steps'] = 110 + def_flags['log_steps'] = 10 + + super(Resnet50MultiWorkerKerasBenchmarkReal, self).__init__( + output_dir=output_dir, default_flags=def_flags) + + +# TODO(kimjaehong): It also should be also cover other metheods of model +# optimization techniques. In that time, this class will change to something +# like 'KerasModelOptimizationAccuracyBase'. +class KerasPruningAccuracyBase(keras_benchmark.KerasBenchmark): + """Benchmark accuracy tests for pruning method.""" + + def __init__(self, + output_dir=None, + root_data_dir=None, + default_flags=None, + **kwargs): + """A accuracy benchmark class for pruning method. + + Args: + output_dir: directory where to output e.g. log files + root_data_dir: directory under which to look for dataset + default_flags: default flags + **kwargs: arbitrary named arguments. This is needed to make the + constructor forward compatible in case PerfZero provides more + named arguments before updating the constructor. + """ + if default_flags is None: + default_flags = {} + default_flags['pruning_method'] = 'polynomial_decay' + default_flags['data_dir'] = os.path.join(root_data_dir, 'imagenet') + + flag_methods = [resnet_imagenet_main.define_imagenet_keras_flags] + + super(KerasPruningAccuracyBase, self).__init__( + output_dir=output_dir, + flag_methods=flag_methods, + default_flags=default_flags, + **kwargs) + + def benchmark_8_gpu(self): + """Test Keras model with eager, dist_strat and 8 GPUs.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.batch_size = 32 * 8 + FLAGS.train_epochs = 90 + FLAGS.epochs_between_evals = 10 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu') + FLAGS.dtype = 'fp32' + FLAGS.enable_eager = True + self._run_and_report_benchmark() + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self, + top_1_min=MODEL_OPTIMIZATION_TOP_1_ACCURACY[ + 'RESNET50_FINETUNE_PRUNING'][0], + top_1_max=MODEL_OPTIMIZATION_TOP_1_ACCURACY[ + 'RESNET50_FINETUNE_PRUNING'][1]): + start_time_sec = time.time() + stats = resnet_imagenet_main.run(flags.FLAGS) + wall_time_sec = time.time() - start_time_sec + + super(KerasPruningAccuracyBase, self)._report_benchmark( + stats, + wall_time_sec, + top_1_min=top_1_min, + top_1_max=top_1_max, + total_batch_size=FLAGS.batch_size, + log_steps=100) + + +class MobilenetV1KerasPruningAccuracy(KerasPruningAccuracyBase): + """Benchmark accuracy tests for MobilenetV1 with pruning method.""" + + def __init__(self, root_data_dir=None, **kwargs): + default_flags = { + 'model': 'mobilenet', + 'optimizer': 'mobilenet_default', + 'initial_learning_rate_per_sample': 0.00007, + 'pretrained_filepath': tf.train.latest_checkpoint( + os.path.join(root_data_dir, 'mobilenet_v1')), + 'pruning_begin_step': 0, + 'pruning_end_step': 100000, + 'pruning_initial_sparsity': 0.0, + 'pruning_final_sparsity': 0.5, + 'pruning_frequency': 100, + } + super(MobilenetV1KerasPruningAccuracy, self).__init__( + root_data_dir=root_data_dir, + default_flags=default_flags, + **kwargs) + + def _run_and_report_benchmark(self): + super(MobilenetV1KerasPruningAccuracy, self)._run_and_report_benchmark( + top_1_min=\ + MODEL_OPTIMIZATION_TOP_1_ACCURACY['MOBILENET_V1_FINETUNE_PRUNING'][0], + top_1_max=\ + MODEL_OPTIMIZATION_TOP_1_ACCURACY['MOBILENET_V1_FINETUNE_PRUNING'][1]) + + +class Resnet50KerasPruningAccuracy(KerasPruningAccuracyBase): + """Benchmark accuracy tests for resnet50 with pruning method.""" + + def __init__(self, root_data_dir=None, **kwargs): + default_flags = { + 'model': 'resnet50_v1.5', + 'optimizer': 'mobilenet_default', + 'initial_learning_rate_per_sample': 0.0000039, + 'pretrained_filepath': tf.train.latest_checkpoint( + os.path.join(root_data_dir, 'resnet50')), + 'pruning_begin_step': 0, + 'pruning_end_step': 50000, + 'pruning_initial_sparsity': 0.0, + 'pruning_final_sparsity': 0.5, + 'pruning_frequency': 100, + } + super(Resnet50KerasPruningAccuracy, self).__init__( + root_data_dir=root_data_dir, + default_flags=default_flags, + **kwargs) + + def _run_and_report_benchmark(self): + super(Resnet50KerasPruningAccuracy, self)._run_and_report_benchmark( + top_1_min=\ + MODEL_OPTIMIZATION_TOP_1_ACCURACY['RESNET50_FINETUNE_PRUNING'][0], + top_1_max=\ + MODEL_OPTIMIZATION_TOP_1_ACCURACY['RESNET50_FINETUNE_PRUNING'][1]) + + +class KerasPruningBenchmarkRealBase(Resnet50KerasBenchmarkBase): + """Pruning method benchmarks.""" + + def __init__(self, root_data_dir=None, default_flags=None, **kwargs): + if default_flags is None: + default_flags = {} + default_flags.update({ + 'skip_eval': True, + 'report_accuracy_metrics': False, + 'data_dir': os.path.join(root_data_dir, 'imagenet'), + 'train_steps': 110, + 'log_steps': 10, + 'pruning_method': 'polynomial_decay', + 'pruning_begin_step': 0, + 'pruning_end_step': 50000, + 'pruning_initial_sparsity': 0, + 'pruning_final_sparsity': 0.5, + 'pruning_frequency': 100, + }) + super(KerasPruningBenchmarkRealBase, self).__init__( + default_flags=default_flags, **kwargs) + + +class MobilenetV1KerasPruningBenchmarkReal(KerasPruningBenchmarkRealBase): + """Pruning method benchmarks for MobilenetV1.""" + + def __init__(self, **kwargs): + default_flags = { + 'model': 'mobilenet', + 'optimizer': 'mobilenet_default', + } + super(MobilenetV1KerasPruningBenchmarkReal, self).__init__( + default_flags=default_flags, **kwargs) + + +class Resnet50KerasPruningBenchmarkReal(KerasPruningBenchmarkRealBase): + """Pruning method benchmarks for resnet50.""" + + def __init__(self, **kwargs): + default_flags = { + 'model': 'resnet50_v1.5', + 'optimizer': 'mobilenet_default', + } + super(Resnet50KerasPruningBenchmarkReal, self).__init__( + default_flags=default_flags, **kwargs) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/resnet_cifar_main.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/resnet_cifar_main.py new file mode 100644 index 0000000..455d3a9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/resnet_cifar_main.py @@ -0,0 +1,287 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Runs a ResNet model on the Cifar-10 dataset.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import app +from absl import flags +from absl import logging +import numpy as np +import tensorflow as tf +from official.benchmark.models import resnet_cifar_model +from official.benchmark.models import synthetic_util +from official.utils.flags import core as flags_core +from official.utils.logs import logger +from official.utils.misc import distribution_utils +from official.utils.misc import keras_utils +from official.vision.image_classification.resnet import cifar_preprocessing +from official.vision.image_classification.resnet import common + + +LR_SCHEDULE = [ # (multiplier, epoch to start) tuples + (0.1, 91), (0.01, 136), (0.001, 182) +] + + +def learning_rate_schedule(current_epoch, + current_batch, + batches_per_epoch, + batch_size): + """Handles linear scaling rule and LR decay. + + Scale learning rate at epoch boundaries provided in LR_SCHEDULE by the + provided scaling factor. + + Args: + current_epoch: integer, current epoch indexed from 0. + current_batch: integer, current batch in the current epoch, indexed from 0. + batches_per_epoch: integer, number of steps in an epoch. + batch_size: integer, total batch sized. + + Returns: + Adjusted learning rate. + """ + del current_batch, batches_per_epoch # not used + initial_learning_rate = common.BASE_LEARNING_RATE * batch_size / 128 + learning_rate = initial_learning_rate + for mult, start_epoch in LR_SCHEDULE: + if current_epoch >= start_epoch: + learning_rate = initial_learning_rate * mult + else: + break + return learning_rate + + +class LearningRateBatchScheduler(tf.keras.callbacks.Callback): + """Callback to update learning rate on every batch (not epoch boundaries). + + N.B. Only support Keras optimizers, not TF optimizers. + + Attributes: + schedule: a function that takes an epoch index and a batch index as input + (both integer, indexed from 0) and returns a new learning rate as + output (float). + """ + + def __init__(self, schedule, batch_size, steps_per_epoch): + super(LearningRateBatchScheduler, self).__init__() + self.schedule = schedule + self.steps_per_epoch = steps_per_epoch + self.batch_size = batch_size + self.epochs = -1 + self.prev_lr = -1 + + def on_epoch_begin(self, epoch, logs=None): + if not hasattr(self.model.optimizer, 'learning_rate'): + raise ValueError('Optimizer must have a "learning_rate" attribute.') + self.epochs += 1 + + def on_batch_begin(self, batch, logs=None): + """Executes before step begins.""" + lr = self.schedule(self.epochs, + batch, + self.steps_per_epoch, + self.batch_size) + if not isinstance(lr, (float, np.float32, np.float64)): + raise ValueError('The output of the "schedule" function should be float.') + if lr != self.prev_lr: + self.model.optimizer.learning_rate = lr # lr should be a float here + self.prev_lr = lr + logging.debug( + 'Epoch %05d Batch %05d: LearningRateBatchScheduler ' + 'change learning rate to %s.', self.epochs, batch, lr) + + +def run(flags_obj): + """Run ResNet Cifar-10 training and eval loop using native Keras APIs. + + Args: + flags_obj: An object containing parsed flag values. + + Raises: + ValueError: If fp16 is passed as it is not currently supported. + + Returns: + Dictionary of training and eval stats. + """ + keras_utils.set_session_config( + enable_eager=flags_obj.enable_eager, + enable_xla=flags_obj.enable_xla) + + # Execute flag override logic for better model performance + if flags_obj.tf_gpu_thread_mode: + keras_utils.set_gpu_thread_mode_and_count( + per_gpu_thread_count=flags_obj.per_gpu_thread_count, + gpu_thread_mode=flags_obj.tf_gpu_thread_mode, + num_gpus=flags_obj.num_gpus, + datasets_num_private_threads=flags_obj.datasets_num_private_threads) + common.set_cudnn_batchnorm_mode() + + dtype = flags_core.get_tf_dtype(flags_obj) + if dtype == 'fp16': + raise ValueError('dtype fp16 is not supported in Keras. Use the default ' + 'value(fp32).') + + data_format = flags_obj.data_format + if data_format is None: + data_format = ('channels_first' if tf.config.list_physical_devices('GPU') + else 'channels_last') + tf.keras.backend.set_image_data_format(data_format) + + strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=flags_obj.distribution_strategy, + num_gpus=flags_obj.num_gpus, + all_reduce_alg=flags_obj.all_reduce_alg, + num_packs=flags_obj.num_packs) + + if strategy: + # flags_obj.enable_get_next_as_optional controls whether enabling + # get_next_as_optional behavior in DistributedIterator. If true, last + # partial batch can be supported. + strategy.extended.experimental_enable_get_next_as_optional = ( + flags_obj.enable_get_next_as_optional + ) + + strategy_scope = distribution_utils.get_strategy_scope(strategy) + + if flags_obj.use_synthetic_data: + synthetic_util.set_up_synthetic_data() + input_fn = common.get_synth_input_fn( + height=cifar_preprocessing.HEIGHT, + width=cifar_preprocessing.WIDTH, + num_channels=cifar_preprocessing.NUM_CHANNELS, + num_classes=cifar_preprocessing.NUM_CLASSES, + dtype=flags_core.get_tf_dtype(flags_obj), + drop_remainder=True) + else: + synthetic_util.undo_set_up_synthetic_data() + input_fn = cifar_preprocessing.input_fn + + train_input_dataset = input_fn( + is_training=True, + data_dir=flags_obj.data_dir, + batch_size=flags_obj.batch_size, + parse_record_fn=cifar_preprocessing.parse_record, + datasets_num_private_threads=flags_obj.datasets_num_private_threads, + dtype=dtype, + # Setting drop_remainder to avoid the partial batch logic in normalization + # layer, which triggers tf.where and leads to extra memory copy of input + # sizes between host and GPU. + drop_remainder=(not flags_obj.enable_get_next_as_optional)) + + eval_input_dataset = None + if not flags_obj.skip_eval: + eval_input_dataset = input_fn( + is_training=False, + data_dir=flags_obj.data_dir, + batch_size=flags_obj.batch_size, + parse_record_fn=cifar_preprocessing.parse_record) + + steps_per_epoch = ( + cifar_preprocessing.NUM_IMAGES['train'] // flags_obj.batch_size) + lr_schedule = 0.1 + if flags_obj.use_tensor_lr: + initial_learning_rate = common.BASE_LEARNING_RATE * flags_obj.batch_size / 128 + lr_schedule = tf.keras.optimizers.schedules.PiecewiseConstantDecay( + boundaries=list(p[1] * steps_per_epoch for p in LR_SCHEDULE), + values=[initial_learning_rate] + + list(p[0] * initial_learning_rate for p in LR_SCHEDULE)) + + with strategy_scope: + optimizer = common.get_optimizer(lr_schedule) + model = resnet_cifar_model.resnet56(classes=cifar_preprocessing.NUM_CLASSES) + model.compile( + loss='sparse_categorical_crossentropy', + optimizer=optimizer, + metrics=(['sparse_categorical_accuracy'] + if flags_obj.report_accuracy_metrics else None), + run_eagerly=flags_obj.run_eagerly) + + train_epochs = flags_obj.train_epochs + + callbacks = common.get_callbacks(steps_per_epoch) + + if not flags_obj.use_tensor_lr: + lr_callback = LearningRateBatchScheduler( + schedule=learning_rate_schedule, + batch_size=flags_obj.batch_size, + steps_per_epoch=steps_per_epoch) + callbacks.append(lr_callback) + + # if mutliple epochs, ignore the train_steps flag. + if train_epochs <= 1 and flags_obj.train_steps: + steps_per_epoch = min(flags_obj.train_steps, steps_per_epoch) + train_epochs = 1 + + num_eval_steps = (cifar_preprocessing.NUM_IMAGES['validation'] // + flags_obj.batch_size) + + validation_data = eval_input_dataset + if flags_obj.skip_eval: + if flags_obj.set_learning_phase_to_train: + # TODO(haoyuzhang): Understand slowdown of setting learning phase when + # not using distribution strategy. + tf.keras.backend.set_learning_phase(1) + num_eval_steps = None + validation_data = None + + if not strategy and flags_obj.explicit_gpu_placement: + # TODO(b/135607227): Add device scope automatically in Keras training loop + # when not using distribition strategy. + no_dist_strat_device = tf.device('/device:GPU:0') + no_dist_strat_device.__enter__() + + history = model.fit(train_input_dataset, + epochs=train_epochs, + steps_per_epoch=steps_per_epoch, + callbacks=callbacks, + validation_steps=num_eval_steps, + validation_data=validation_data, + validation_freq=flags_obj.epochs_between_evals, + verbose=2) + eval_output = None + if not flags_obj.skip_eval: + eval_output = model.evaluate(eval_input_dataset, + steps=num_eval_steps, + verbose=2) + + if not strategy and flags_obj.explicit_gpu_placement: + no_dist_strat_device.__exit__() + + stats = common.build_stats(history, eval_output, callbacks) + return stats + + +def define_cifar_flags(): + common.define_keras_flags(dynamic_loss_scale=False) + + flags_core.set_defaults(data_dir='/tmp/cifar10_data/cifar-10-batches-bin', + model_dir='/tmp/cifar10_model', + epochs_between_evals=10, + batch_size=128) + + +def main(_): + with logger.benchmark_context(flags.FLAGS): + return run(flags.FLAGS) + + +if __name__ == '__main__': + logging.set_verbosity(logging.INFO) + define_cifar_flags() + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/resnet_cifar_model.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/resnet_cifar_model.py new file mode 100644 index 0000000..1b50738 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/resnet_cifar_model.py @@ -0,0 +1,262 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""ResNet56 model for Keras adapted from tf.keras.applications.ResNet50. + +# Reference: +- [Deep Residual Learning for Image Recognition]( + https://arxiv.org/abs/1512.03385) +Adapted from code contributed by BigMoyan. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools +import tensorflow as tf +from tensorflow.python.keras import backend +from tensorflow.python.keras import initializers +from tensorflow.python.keras import layers +from tensorflow.python.keras import regularizers + + +BATCH_NORM_DECAY = 0.997 +BATCH_NORM_EPSILON = 1e-5 +L2_WEIGHT_DECAY = 2e-4 + + +def identity_building_block(input_tensor, + kernel_size, + filters, + stage, + block, + training=None): + """The identity block is the block that has no conv layer at shortcut. + + Arguments: + input_tensor: input tensor + kernel_size: default 3, the kernel size of + middle conv layer at main path + filters: list of integers, the filters of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + block: current block label, used for generating layer names + training: Only used if training keras model with Estimator. In other + scenarios it is handled automatically. + + Returns: + Output tensor for the block. + """ + filters1, filters2 = filters + if backend.image_data_format() == 'channels_last': + bn_axis = 3 + else: + bn_axis = 1 + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + + x = layers.Conv2D(filters1, kernel_size, + padding='same', use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), + name=conv_name_base + '2a')(input_tensor) + x = layers.BatchNormalization( + axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, + name=bn_name_base + '2a')(x, training=training) + x = layers.Activation('relu')(x) + + x = layers.Conv2D(filters2, kernel_size, + padding='same', use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), + name=conv_name_base + '2b')(x) + x = layers.BatchNormalization( + axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, + name=bn_name_base + '2b')(x, training=training) + + x = layers.add([x, input_tensor]) + x = layers.Activation('relu')(x) + return x + + +def conv_building_block(input_tensor, + kernel_size, + filters, + stage, + block, + strides=(2, 2), + training=None): + """A block that has a conv layer at shortcut. + + Arguments: + input_tensor: input tensor + kernel_size: default 3, the kernel size of + middle conv layer at main path + filters: list of integers, the filters of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + block: current block label, used for generating layer names + strides: Strides for the first conv layer in the block. + training: Only used if training keras model with Estimator. In other + scenarios it is handled automatically. + + Returns: + Output tensor for the block. + + Note that from stage 3, + the first conv layer at main path is with strides=(2, 2) + And the shortcut should have strides=(2, 2) as well + """ + filters1, filters2 = filters + if tf.keras.backend.image_data_format() == 'channels_last': + bn_axis = 3 + else: + bn_axis = 1 + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + + x = layers.Conv2D(filters1, kernel_size, strides=strides, + padding='same', use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), + name=conv_name_base + '2a')(input_tensor) + x = layers.BatchNormalization( + axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, + name=bn_name_base + '2a')(x, training=training) + x = layers.Activation('relu')(x) + + x = layers.Conv2D(filters2, kernel_size, padding='same', use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), + name=conv_name_base + '2b')(x) + x = layers.BatchNormalization( + axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, + name=bn_name_base + '2b')(x, training=training) + + shortcut = layers.Conv2D(filters2, (1, 1), strides=strides, use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), + name=conv_name_base + '1')(input_tensor) + shortcut = layers.BatchNormalization( + axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, + name=bn_name_base + '1')(shortcut, training=training) + + x = layers.add([x, shortcut]) + x = layers.Activation('relu')(x) + return x + + +def resnet_block(input_tensor, + size, + kernel_size, + filters, + stage, + conv_strides=(2, 2), + training=None): + """A block which applies conv followed by multiple identity blocks. + + Arguments: + input_tensor: input tensor + size: integer, number of constituent conv/identity building blocks. + A conv block is applied once, followed by (size - 1) identity blocks. + kernel_size: default 3, the kernel size of + middle conv layer at main path + filters: list of integers, the filters of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + conv_strides: Strides for the first conv layer in the block. + training: Only used if training keras model with Estimator. In other + scenarios it is handled automatically. + + Returns: + Output tensor after applying conv and identity blocks. + """ + + x = conv_building_block(input_tensor, kernel_size, filters, stage=stage, + strides=conv_strides, block='block_0', + training=training) + for i in range(size - 1): + x = identity_building_block(x, kernel_size, filters, stage=stage, + block='block_%d' % (i + 1), training=training) + return x + + +def resnet(num_blocks, classes=10, training=None): + """Instantiates the ResNet architecture. + + Arguments: + num_blocks: integer, the number of conv/identity blocks in each block. + The ResNet contains 3 blocks with each block containing one conv block + followed by (layers_per_block - 1) number of idenity blocks. Each + conv/idenity block has 2 convolutional layers. With the input + convolutional layer and the pooling layer towards the end, this brings + the total size of the network to (6*num_blocks + 2) + classes: optional number of classes to classify images into + training: Only used if training keras model with Estimator. In other + scenarios it is handled automatically. + + Returns: + A Keras model instance. + """ + + input_shape = (32, 32, 3) + img_input = layers.Input(shape=input_shape) + + if backend.image_data_format() == 'channels_first': + x = layers.Lambda(lambda x: backend.permute_dimensions(x, (0, 3, 1, 2)), + name='transpose')(img_input) + bn_axis = 1 + else: # channel_last + x = img_input + bn_axis = 3 + + x = layers.ZeroPadding2D(padding=(1, 1), name='conv1_pad')(x) + x = layers.Conv2D(16, (3, 3), + strides=(1, 1), + padding='valid', use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), + name='conv1')(x) + x = layers.BatchNormalization(axis=bn_axis, + momentum=BATCH_NORM_DECAY, + epsilon=BATCH_NORM_EPSILON, + name='bn_conv1',)(x, training=training) + x = layers.Activation('relu')(x) + + x = resnet_block(x, size=num_blocks, kernel_size=3, filters=[16, 16], + stage=2, conv_strides=(1, 1), training=training) + + x = resnet_block(x, size=num_blocks, kernel_size=3, filters=[32, 32], + stage=3, conv_strides=(2, 2), training=training) + + x = resnet_block(x, size=num_blocks, kernel_size=3, filters=[64, 64], + stage=4, conv_strides=(2, 2), training=training) + + rm_axes = [1, 2] if backend.image_data_format() == 'channels_last' else [2, 3] + x = layers.Lambda(lambda x: backend.mean(x, rm_axes), name='reduce_mean')(x) + x = layers.Dense(classes, + activation='softmax', + kernel_initializer=initializers.RandomNormal(stddev=0.01), + kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), + bias_regularizer=regularizers.l2(L2_WEIGHT_DECAY), + name='fc10')(x) + + inputs = img_input + # Create model. + model = tf.keras.models.Model(inputs, x, name='resnet56') + + return model + + +resnet20 = functools.partial(resnet, num_blocks=3) +resnet32 = functools.partial(resnet, num_blocks=5) +resnet56 = functools.partial(resnet, num_blocks=9) +resnet10 = functools.partial(resnet, num_blocks=110) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/resnet_cifar_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/resnet_cifar_test.py new file mode 100644 index 0000000..6dbb2fa --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/resnet_cifar_test.py @@ -0,0 +1,187 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Test the keras ResNet model with Cifar data.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tempfile + +import tensorflow as tf + +from tensorflow.python.eager import context +from tensorflow.python.platform import googletest +from official.benchmark.models import resnet_cifar_main +from official.utils.misc import keras_utils +from official.utils.testing import integration +from official.vision.image_classification.resnet import cifar_preprocessing + + +class KerasCifarTest(googletest.TestCase): + """Unit tests for Keras ResNet with Cifar.""" + + _extra_flags = [ + "-batch_size", "4", + "-train_steps", "1", + "-use_synthetic_data", "true" + ] + _tempdir = None + + def get_temp_dir(self): + if not self._tempdir: + self._tempdir = tempfile.mkdtemp(dir=googletest.GetTempDir()) + return self._tempdir + + @classmethod + def setUpClass(cls): # pylint: disable=invalid-name + super(KerasCifarTest, cls).setUpClass() + resnet_cifar_main.define_cifar_flags() + + def setUp(self): + super(KerasCifarTest, self).setUp() + cifar_preprocessing.NUM_IMAGES["validation"] = 4 + + def tearDown(self): + super(KerasCifarTest, self).tearDown() + tf.io.gfile.rmtree(self.get_temp_dir()) + + def test_end_to_end_no_dist_strat(self): + """Test Keras model with 1 GPU, no distribution strategy.""" + config = keras_utils.get_config_proto_v1() + tf.compat.v1.enable_eager_execution(config=config) + + extra_flags = [ + "-distribution_strategy", "off", + "-model_dir", "keras_cifar_no_dist_strat", + "-data_format", "channels_last", + ] + extra_flags = extra_flags + self._extra_flags + + integration.run_synthetic( + main=resnet_cifar_main.run, + tmp_root=self.get_temp_dir(), + extra_flags=extra_flags + ) + + def test_end_to_end_graph_no_dist_strat(self): + """Test Keras model in legacy graph mode with 1 GPU, no dist strat.""" + extra_flags = [ + "-enable_eager", "false", + "-distribution_strategy", "off", + "-model_dir", "keras_cifar_graph_no_dist_strat", + "-data_format", "channels_last", + ] + extra_flags = extra_flags + self._extra_flags + + integration.run_synthetic( + main=resnet_cifar_main.run, + tmp_root=self.get_temp_dir(), + extra_flags=extra_flags + ) + + def test_end_to_end_1_gpu(self): + """Test Keras model with 1 GPU.""" + config = keras_utils.get_config_proto_v1() + tf.compat.v1.enable_eager_execution(config=config) + + if context.num_gpus() < 1: + self.skipTest( + "{} GPUs are not available for this test. {} GPUs are available". + format(1, context.num_gpus())) + + extra_flags = [ + "-num_gpus", "1", + "-distribution_strategy", "mirrored", + "-model_dir", "keras_cifar_1_gpu", + "-data_format", "channels_last", + ] + extra_flags = extra_flags + self._extra_flags + + integration.run_synthetic( + main=resnet_cifar_main.run, + tmp_root=self.get_temp_dir(), + extra_flags=extra_flags + ) + + def test_end_to_end_graph_1_gpu(self): + """Test Keras model in legacy graph mode with 1 GPU.""" + if context.num_gpus() < 1: + self.skipTest( + "{} GPUs are not available for this test. {} GPUs are available". + format(1, context.num_gpus())) + + extra_flags = [ + "-num_gpus", "1", + "-noenable_eager", + "-distribution_strategy", "mirrored", + "-model_dir", "keras_cifar_graph_1_gpu", + "-data_format", "channels_last", + ] + extra_flags = extra_flags + self._extra_flags + + integration.run_synthetic( + main=resnet_cifar_main.run, + tmp_root=self.get_temp_dir(), + extra_flags=extra_flags + ) + + def test_end_to_end_2_gpu(self): + """Test Keras model with 2 GPUs.""" + config = keras_utils.get_config_proto_v1() + tf.compat.v1.enable_eager_execution(config=config) + + if context.num_gpus() < 2: + self.skipTest( + "{} GPUs are not available for this test. {} GPUs are available". + format(2, context.num_gpus())) + + extra_flags = [ + "-num_gpus", "2", + "-distribution_strategy", "mirrored", + "-model_dir", "keras_cifar_2_gpu", + ] + extra_flags = extra_flags + self._extra_flags + + integration.run_synthetic( + main=resnet_cifar_main.run, + tmp_root=self.get_temp_dir(), + extra_flags=extra_flags + ) + + def test_end_to_end_graph_2_gpu(self): + """Test Keras model in legacy graph mode with 2 GPUs.""" + if context.num_gpus() < 2: + self.skipTest( + "{} GPUs are not available for this test. {} GPUs are available". + format(2, context.num_gpus())) + + extra_flags = [ + "-num_gpus", "2", + "-enable_eager", "false", + "-distribution_strategy", "mirrored", + "-model_dir", "keras_cifar_graph_2_gpu", + ] + extra_flags = extra_flags + self._extra_flags + + integration.run_synthetic( + main=resnet_cifar_main.run, + tmp_root=self.get_temp_dir(), + extra_flags=extra_flags + ) + + +if __name__ == "__main__": + googletest.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/shakespeare/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/shakespeare/README.md new file mode 100644 index 0000000..5395cc9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/shakespeare/README.md @@ -0,0 +1,31 @@ +# Shakespeare character LSTM model + +This is an implemention of a simple character LSTM used to generate text. + +## Instructions + +First download the source data: + +``` +wget https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt +``` + +Note that files other than shakepeare.txt can also be used to train the model to generater other text. + +Then train the model: + +```python +python3 shakespeare_main.py --training_data shakespeare.txt \ + --model_dir /tmp/shakespeare +``` + +This will place model checkpoints in `/tmp/shakespeare`, so that we can use them to make predictions. + +Then generate predictions: + +```python +python3 shakespeare_main.py --training_data shakespeare.txt \ + --model_dir /tmp/shakespeare --notrain --predict_context=ROMEO: +``` + +Change `--predict_context` and `--predict_length` to suit your needs. diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/shakespeare/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/shakespeare/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/shakespeare/__init__.py @@ -0,0 +1 @@ + diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/shakespeare/shakespeare_main.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/shakespeare/shakespeare_main.py new file mode 100644 index 0000000..fd490b7 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/shakespeare/shakespeare_main.py @@ -0,0 +1,316 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Runs a character LSTM model trained on Shakespeare.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools +import os + +# pylint: disable=wrong-import-order +from absl import app +from absl import flags +import numpy as np +import tensorflow as tf +# pylint: enable=wrong-import-order + +from official.utils.flags import core as flags_core +from official.utils.misc import distribution_utils +from official.utils.misc import keras_utils + +EMBEDDING_DIM = 256 +RNN_UNITS = 1024 +SEQ_LENGTH = 100 +# Calculated by running batch_size=1 +BATCHES_PER_EPOCH = 11043 + + +def define_flags(): + """Define the flags for the Shakespeare character LSTM.""" + flags_core.define_base(data_dir=False, + clean=False, + train_epochs=True, + epochs_between_evals=False, + stop_threshold=False, + num_gpu=True, + hooks=False, + export_dir=False, + run_eagerly=True, + distribution_strategy=True) + + flags_core.define_performance(num_parallel_calls=False, + inter_op=False, + intra_op=False, + synthetic_data=False, + max_train_steps=False, + dtype=True, + loss_scale=True, + enable_xla=True) + + flags_core.set_defaults(train_epochs=43, + batch_size=64) + + flags.DEFINE_boolean(name='enable_eager', default=True, help='Enable eager?') + flags.DEFINE_boolean( + name='train', default=True, + help='If true trains the model.') + flags.DEFINE_string( + name='predict_context', default=None, + help='If set, makes a prediction with the given context.') + flags.DEFINE_integer( + name='predict_length', default=1000, + help='Length of the predicted text including the context.') + flags.DEFINE_integer(name='train_steps', default=None, + help='Overrides train_steps per epoch if not None.') + flags.DEFINE_integer( + name='log_steps', default=100, + help='For every log_steps, we log the timing information such as ' + 'examples per second.') + flags.DEFINE_string( + name='training_data', default=None, + help='Path to file containing the training data.') + flags.DEFINE_boolean(name='cudnn', default=True, help='Use CuDNN LSTM.') + + +def get_dataset(path_to_file, batch_size=None, seq_length=SEQ_LENGTH): + """Creates a dataset from a given text file. + + Args: + path_to_file: The path to the training data. + batch_size: Batch size to use. + seq_length: The length of the LSTM sequence. + + Returns: + A tuple, consisting of the Dataset and the class to character mapping + and character to class mapping. + """ + with tf.io.gfile.GFile(path_to_file, 'rb') as train_data: + text = train_data.read().decode(encoding='utf-8') + + # Create vocab + vocab = sorted(set(text)) + char2idx = {u: i for i, u in enumerate(vocab)} + idx2char = np.array(vocab) + + # Split text into sequence length + 1 chucks to create examples + text_as_int = np.array([char2idx[c] for c in text]) + char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int) + sequences = char_dataset.batch(seq_length+1, drop_remainder=True) + + def split_input_target(chunk): + input_text = chunk[:-1] + target_text = chunk[1:] + return input_text, tf.one_hot(target_text, len(vocab)) + dataset = sequences.map(split_input_target) + dataset = dataset.shuffle(10000).repeat() + dataset = dataset.batch(batch_size, drop_remainder=True) + + return dataset, idx2char, char2idx + + +def build_model(vocab_size, + embedding_dim=EMBEDDING_DIM, + rnn_units=RNN_UNITS, + batch_size=None, + stateful=False, + use_cudnn=True): + """Builds the Shakespeare model. + + Args: + vocab_size: The number of character classes in the input. + embedding_dim: The dimension of the embedding space for each class. + rnn_units: The number of RNN units in the layer. + batch_size: When predicting, the batch size of the predictions. + stateful: If true, the LSTM is stateful. + + Returns: + A Keras Model. + """ + assert keras_utils.is_v2_0() + LSTM = functools.partial(tf.keras.layers.LSTM, implementation=2) + + # By indirecting the activation through a lambda layer, the logic to dispatch + # to CuDNN in V2 doesn't trigger and we force the LSTM to run in non-CuDNN + # mode. + lstm_activation = ('tanh' if use_cudnn else + lambda x: tf.math.tanh(x)) + + batch_shape = [batch_size if stateful else None, None] + return tf.keras.Sequential([ + tf.keras.layers.Embedding(vocab_size, embedding_dim, + batch_input_shape=batch_shape), + LSTM(rnn_units, + activation=lstm_activation, + return_sequences=True, + stateful=stateful, + recurrent_initializer='glorot_uniform'), + tf.keras.layers.Dense(vocab_size), + tf.keras.layers.Softmax(dtype=tf.float32)]) + + +def train_model(flags_obj, dataset, vocab_size, strategy, checkpoint_dir=None): + """Trains a Shakespeare model. + + Args: + flags_obj: An object containing parsed flag values.s + dataset: the training data set. + vocab_size: the number of unique character classes. + strategy: distribution strategy to use. + checkpoint_dir: if not None, the directory in which to make checkpoints. + + Returns: + The training history and callbacks. + """ + if flags_obj.train_steps: + train_steps = flags_obj.train_steps + else: + train_steps = BATCHES_PER_EPOCH // flags_obj.batch_size + strategy_scope = distribution_utils.get_strategy_scope(strategy) + + with strategy_scope: + model = build_model(vocab_size=vocab_size, batch_size=flags_obj.batch_size, + use_cudnn=flags_obj.cudnn) + + # When keras_use_ctl is False, Model.fit() automatically applies + # loss scaling so we don't need to create a LossScaleOptimizer. + model.compile( + optimizer=tf.keras.optimizers.Adam(), + loss=tf.keras.losses.CategoricalCrossentropy(), + metrics=[tf.keras.metrics.Recall(top_k=1, name='RecallAt1'), + tf.keras.metrics.Recall(top_k=5, name='RecallAt5')], + run_eagerly=flags_obj.run_eagerly) + + callbacks = [] + if checkpoint_dir: + checkpoint_prefix = os.path.join(checkpoint_dir, 'ckpt_{epoch}') + checkpoint_callback = tf.keras.callbacks.ModelCheckpoint( + filepath=checkpoint_prefix, + save_weights_only=True) + callbacks.append(checkpoint_callback) + time_callback = keras_utils.TimeHistory(flags_obj.batch_size, + flags_obj.log_steps) + callbacks.append(time_callback) + history = model.fit(dataset, + epochs=flags_obj.train_epochs, + steps_per_epoch=train_steps, + callbacks=callbacks, + verbose=2) + return history, callbacks + + +def make_prediction(checkpoint_dir, length, context, idx2char, char2idx): + """Make predictions from a Shakespeare model. + + Args: + checkpoint_dir: the directory from which to load checkpoints + length: the total length of the generated text (including the context). + context: the initial text with which the LSTM is primed. + idx2char: the character class to character mapping. + char2idx: the character to character class mapping. + + Returns: + A generated string of text of the given length. + """ + prediction_model = build_model( + vocab_size=len(idx2char), batch_size=1, stateful=True) + prediction_model.load_weights(tf.train.latest_checkpoint(checkpoint_dir)) + prediction_model.build(tf.TensorShape([1, None])) + + input_eval = [char2idx[s] for s in context] + input_eval = tf.expand_dims(input_eval, 0) + + text_generated = [] + + prediction_model.reset_states() + for _ in range(length - len(context)): + predictions = prediction_model(input_eval) + predictions = tf.squeeze(predictions, 0) + + # We applied a softmax to the output of the model so that + # tf.keras.metrics.Recall would work. We need logits for + # tf.random.categorical, so we convert the probabilities back to log odds + predictions = tf.math.log(predictions / (1 - predictions)) + + random_output = tf.random.categorical(predictions, num_samples=1) + selected_id = random_output[-1, 0].numpy() + input_eval = tf.expand_dims([selected_id], 0) + text_generated.append(idx2char[selected_id]) + + return context + ''.join(text_generated) + + +def run(flags_obj): + """Run Shakespeare training and predict. + + Args: + flags_obj: An object containing parsed flag values. + + Returns: + Dictionary with status from the run. + """ + if not flags_obj.training_data: + raise ValueError( + 'Must set the path to a training data file. e.g download the following ' + 'https://storage.googleapis.com/download.tensorflow.org/data/' + 'shakespeare.txt') + + if flags_obj.dtype == 'fp16': + policy = tf.keras.mixed_precision.experimental.Policy( + 'mixed_float16', + loss_scale=flags_core.get_loss_scale(flags_obj, + default_for_fp16='dynamic')) + tf.keras.mixed_precision.experimental.set_policy(policy) + + keras_utils.set_session_config( + enable_eager=flags_obj.enable_eager, + enable_xla=flags_obj.enable_xla) + + strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=flags_obj.distribution_strategy, + num_gpus=flags_obj.num_gpus) + + dataset, idx2char, char2idx = get_dataset(flags_obj.training_data, + batch_size=flags_obj.batch_size) + stats = {} + if flags_obj.train: + history, callbacks = train_model(flags_obj, dataset, + len(idx2char), strategy, + checkpoint_dir=flags_obj.model_dir) + + stats['history'] = history.history + stats['callbacks'] = callbacks + + if flags_obj.predict_context: + if not flags_obj.model_dir: + raise ValueError('Must set model_dir to get predictions.') + print(make_prediction(flags_obj.model_dir, + flags_obj.predict_length, + flags_obj.predict_context, + idx2char, + char2idx)) + + return stats + + +def main(_): + flags_obj = flags.FLAGS + run(flags_obj) + + +if __name__ == '__main__': + define_flags() + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/synthetic_util.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/synthetic_util.py new file mode 100644 index 0000000..c14d022 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/models/synthetic_util.py @@ -0,0 +1,129 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Helper functions to generate data directly on devices.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import random +import string + +from absl import logging +import tensorflow as tf + + +# The `SyntheticDataset` is a temporary solution for generating synthetic data +# directly on devices. It is only useful for Keras with Distribution +# Strategies. We will have better support in `tf.data` or Distribution Strategy +# later. +class SyntheticDataset(object): + """A dataset that generates synthetic data on each device.""" + + def __init__(self, dataset, split_by=1): + # dataset.take(1) doesn't have GPU kernel. + with tf.device('device:CPU:0'): + tensor = tf.data.experimental.get_single_element(dataset.take(1)) + flat_tensor = tf.nest.flatten(tensor) + variable_data = [] + initializers = [] + for t in flat_tensor: + rebatched_t = tf.split(t, num_or_size_splits=split_by, axis=0)[0] + assert rebatched_t.shape.is_fully_defined(), rebatched_t.shape + v = tf.compat.v1.get_local_variable(self._random_name(), + initializer=rebatched_t) + variable_data.append(v) + initializers.append(v.initializer) + input_data = tf.nest.pack_sequence_as(tensor, variable_data) + self._iterator = SyntheticIterator(input_data, initializers) + + def _random_name(self, size=10, chars=string.ascii_uppercase + string.digits): + return ''.join(random.choice(chars) for _ in range(size)) + + def __iter__(self): + return self._iterator + + def make_one_shot_iterator(self): + return self._iterator + + def make_initializable_iterator(self): + return self._iterator + + +class SyntheticIterator(object): + """A dataset that generates synthetic data on each device.""" + + def __init__(self, input_data, initializers): + self._input_data = input_data + self._initializers = initializers + + def get_next(self): + return self._input_data + + def next(self): + return self.__next__() + + def __next__(self): + try: + return self.get_next() + except tf.errors.OutOfRangeError: + raise StopIteration + + def initialize(self): + if tf.executing_eagerly(): + return tf.no_op() + else: + return self._initializers + + +def _monkey_patch_dataset_method(strategy): + """Monkey-patch `strategy`'s `make_dataset_iterator` method.""" + def make_dataset(self, dataset): + logging.info('Using pure synthetic data.') + with self.scope(): + if self.extended._global_batch_size: # pylint: disable=protected-access + return SyntheticDataset(dataset, self.num_replicas_in_sync) + else: + return SyntheticDataset(dataset) + + def make_iterator(self, dataset): + dist_dataset = make_dataset(self, dataset) + return iter(dist_dataset) + + strategy.orig_make_dataset_iterator = strategy.make_dataset_iterator + strategy.make_dataset_iterator = make_iterator + strategy.orig_distribute_dataset = strategy.experimental_distribute_dataset + strategy.experimental_distribute_dataset = make_dataset + + +def _undo_monkey_patch_dataset_method(strategy): + if hasattr(strategy, 'orig_make_dataset_iterator'): + strategy.make_dataset_iterator = strategy.orig_make_dataset_iterator + if hasattr(strategy, 'orig_distribute_dataset'): + strategy.make_dataset_iterator = strategy.orig_distribute_dataset + + +def set_up_synthetic_data(): + _monkey_patch_dataset_method(tf.distribute.OneDeviceStrategy) + _monkey_patch_dataset_method(tf.distribute.MirroredStrategy) + _monkey_patch_dataset_method( + tf.distribute.experimental.MultiWorkerMirroredStrategy) + + +def undo_set_up_synthetic_data(): + _undo_monkey_patch_dataset_method(tf.distribute.OneDeviceStrategy) + _undo_monkey_patch_dataset_method(tf.distribute.MirroredStrategy) + _undo_monkey_patch_dataset_method( + tf.distribute.experimental.MultiWorkerMirroredStrategy) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/ncf_keras_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/ncf_keras_benchmark.py new file mode 100644 index 0000000..d8c9f3f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/ncf_keras_benchmark.py @@ -0,0 +1,457 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Executes Keras benchmarks and accuracy tests.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import time + +from absl import flags +from absl import logging +from absl.testing import flagsaver +import tensorflow as tf +from official.benchmark import benchmark_wrappers +from official.recommendation import ncf_common +from official.recommendation import ncf_keras_main +from official.utils.flags import core + +FLAGS = flags.FLAGS +NCF_DATA_DIR_NAME = 'movielens_data' +NCF_TF_DATA_1M_BATCH_DIR_NAME = 'gs://tf-perfzero-data/movielens_data/ncf_8gpu_1M_batch' + + +class NCFKerasBenchmarkBase(tf.test.Benchmark): + """Base class for NCF model benchmark.""" + local_flags = None + + def __init__(self, + output_dir=None, + default_flags=None, + **kwargs): + self.output_dir = output_dir + self.default_flags = default_flags or {} + # Run all benchmarks with ml_perf flag. + self.default_flags['ml_perf'] = True + + def _setup(self): + """Sets up and resets flags before each test.""" + logging.set_verbosity(logging.INFO) + if NCFKerasBenchmarkBase.local_flags is None: + ncf_common.define_ncf_flags() + # Loads flags to get defaults to then override. List cannot be empty. + flags.FLAGS(['foo']) + core.set_defaults(**self.default_flags) + saved_flag_values = flagsaver.save_flag_values() + NCFKerasBenchmarkBase.local_flags = saved_flag_values + else: + flagsaver.restore_flag_values(NCFKerasBenchmarkBase.local_flags) + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self, hr_at_10_min=0, hr_at_10_max=0): + start_time_sec = time.time() + stats = ncf_keras_main.run_ncf(FLAGS) + wall_time_sec = time.time() - start_time_sec + + metrics = [] + metrics.append({'name': 'exp_per_second', + 'value': stats['avg_exp_per_second']}) + + if hr_at_10_min > 0: + metrics.append({'name': 'hr_at_10', + 'value': stats['eval_hit_rate'], + 'min_value': hr_at_10_min, + 'max_value': hr_at_10_max}) + + metrics.append({'name': 'train_loss', + 'value': stats['loss']}) + + self.report_benchmark(iters=-1, wall_time=wall_time_sec, metrics=metrics) + + +class NCFKerasAccuracy(NCFKerasBenchmarkBase): + """Benchmark NCF model using real data.""" + + def __init__(self, + output_dir=None, + root_data_dir=None, + default_flags=None, + **kwargs): + root_data_dir = root_data_dir if root_data_dir else '' + default_flags = {} + default_flags['dataset'] = 'ml-20m' + default_flags['num_gpus'] = 1 + default_flags['train_epochs'] = 10 + default_flags['clean'] = True + default_flags['batch_size'] = 99000 + default_flags['learning_rate'] = 0.00382059 + default_flags['beta1'] = 0.783529 + default_flags['beta2'] = 0.909003 + default_flags['epsilon'] = 1.45439e-07 + default_flags['layers'] = [256, 256, 128, 64] + default_flags['num_factors'] = 64 + default_flags['hr_threshold'] = 0.635 + default_flags['ml_perf'] = True + default_flags['use_synthetic_data'] = False + default_flags['data_dir'] = os.path.join(root_data_dir, NCF_DATA_DIR_NAME) + + super(NCFKerasAccuracy, self).__init__( + output_dir=output_dir, + default_flags=default_flags, + **kwargs) + + def _run_and_report_benchmark_mlperf_like(self): + """Run test and report results. + + Note: MLPerf like tests are not tuned to hit a specific hr@10 value, but + we want it recorded. + """ + self._run_and_report_benchmark(hr_at_10_min=0.61) + + def _run_and_report_benchmark(self, hr_at_10_min=0.630, hr_at_10_max=0.645): + """Run test and report results. + + Note: Target is 0.635, but some runs are below that level. Until we have + multi-run tests, we have to accept a lower target. + + Args: + hr_at_10_min: Minimum acceptable hr@10 value. + hr_at_10_max: Maximum acceptable hr@10 value. + """ + super(NCFKerasAccuracy, self)._run_and_report_benchmark( + hr_at_10_min=hr_at_10_min, + hr_at_10_max=hr_at_10_max) + + def benchmark_1_gpu_early_stop(self): + self._setup() + FLAGS.early_stopping = True + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_dist_strat_early_stop(self): + self._setup() + FLAGS.distribution_strategy = 'off' + FLAGS.early_stopping = True + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_dist_strat_run_eagerly_early_stop(self): + self._setup() + FLAGS.distribution_strategy = 'off' + FLAGS.early_stopping = True + FLAGS.run_eagerly = True + self._run_and_report_benchmark() + + def benchmark_xla_1_gpu_early_stop(self): + self._setup() + FLAGS.early_stopping = True + FLAGS.enable_xla = True + self._run_and_report_benchmark() + + def benchmark_1_gpu_ctl_early_stop(self): + self._setup() + FLAGS.keras_use_ctl = True + FLAGS.early_stopping = True + self._run_and_report_benchmark() + + def benchmark_1_gpu_ctl_run_eagerly_early_stop(self): + self._setup() + FLAGS.keras_use_ctl = True + FLAGS.early_stopping = True + FLAGS.run_eagerly = True + self._run_and_report_benchmark() + + def benchmark_xla_1_gpu_ctl_early_stop(self): + self._setup() + FLAGS.keras_use_ctl = True + FLAGS.early_stopping = True + FLAGS.enable_xla = True + self._run_and_report_benchmark() + + def benchmark_2_gpus_early_stop(self): + self._setup() + FLAGS.early_stopping = True + FLAGS.num_gpus = 2 + FLAGS.eval_batch_size = 160000 + self._run_and_report_benchmark() + + def benchmark_2_gpus_ctl_early_stop(self): + """NCF with custom training loop. Works only in TF 2.0.""" + self._setup() + FLAGS.keras_use_ctl = True + FLAGS.early_stopping = True + FLAGS.num_gpus = 2 + FLAGS.eval_batch_size = 160000 + self._run_and_report_benchmark() + +############################################# +# Tests below with mlperf in the test name are of two types: +# 1) 1 GPU tests are based on MLPerf 0.5 and the TensorFlow pulled submission. +# 2) 8 GPU tests are based on MLPerf 0.5 and use NVIDIA's hyper parameters. +# +# The purpose of both is to get a number to compare to existing results. To do +# this the number of epochs is held constant rather than a race to a given +# accuracy. The accuracy validation is done by the "early_stop" tests. +############################################# + + def benchmark_1_gpu_mlperf_like(self): + """1 GPU using keras fit/compile.""" + self._setup() + FLAGS.train_epochs = 7 + self._run_and_report_benchmark_mlperf_like() + + def benchmark_1_gpu_no_dist_strat_mlperf_like(self): + """1 GPU using compile/fit without dist_strat.""" + self._setup() + FLAGS.train_epochs = 7 + FLAGS.distribution_strategy = 'off' + self._run_and_report_benchmark_mlperf_like() + + def benchmark_1_gpu_no_dist_strat_run_eagerly_mlperf_like(self): + self._setup() + FLAGS.train_epochs = 7 + FLAGS.distribution_strategy = 'off' + FLAGS.run_eagerly = True + self._run_and_report_benchmark_mlperf_like() + + def benchmark_xla_1_gpu_mlperf_like(self): + """1 GPU using compile/fit with XLA.""" + self._setup() + FLAGS.train_epochs = 7 + FLAGS.enable_xla = True + self._run_and_report_benchmark_mlperf_like() + + def benchmark_1_gpu_ctl_mlperf_like(self): + """1 GPU using CTL.""" + self._setup() + FLAGS.keras_use_ctl = True + FLAGS.train_epochs = 7 + self._run_and_report_benchmark_mlperf_like() + + def benchmark_1_gpu_ctl_fp16_mlperf_like(self): + """1 GPU using CTL and FP16.""" + self._setup() + FLAGS.keras_use_ctl = True + FLAGS.train_epochs = 7 + FLAGS.dtype = 'fp16' + FLAGS.loss_scale = 8192 + self._run_and_report_benchmark_mlperf_like() + + def benchmark_1_gpu_fp16_mlperf_like(self): + """1 GPU using FP16.""" + self._setup() + FLAGS.train_epochs = 7 + FLAGS.dtype = 'fp16' + FLAGS.loss_scale = 8192 + self._run_and_report_benchmark_mlperf_like() + + def benchmark_1_gpu_ctl_fp16_graph_rewrite_mlperf_like(self): + """1 GPU using CTL and FP16 graph rewrite.""" + self._setup() + FLAGS.keras_use_ctl = True + FLAGS.train_epochs = 7 + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.loss_scale = 8192 + self._run_and_report_benchmark_mlperf_like() + + def benchmark_1_gpu_fp16_graph_rewrite_mlperf_like(self): + """1 GPU using FP16 graph rewrite.""" + self._setup() + FLAGS.train_epochs = 7 + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.loss_scale = 8192 + self._run_and_report_benchmark_mlperf_like() + + def benchmark_1_gpu_ctl_run_eagerly_mlperf_like(self): + """1 GPU using CTL with eager and distribution strategy.""" + self._setup() + FLAGS.keras_use_ctl = True + FLAGS.run_eagerly = True + FLAGS.train_epochs = 7 + self._run_and_report_benchmark() + + def benchmark_xla_1_gpu_ctl_mlperf_like(self): + """1 GPU using CTL with XLA.""" + self._setup() + FLAGS.keras_use_ctl = True + FLAGS.enable_xla = True + FLAGS.train_epochs = 7 + self._run_and_report_benchmark_mlperf_like() + + def benchmark_xla_1_gpu_fp16_mlperf_like(self): + """1 GPU using with XLA and FP16.""" + self._setup() + FLAGS.enable_xla = True + FLAGS.train_epochs = 7 + FLAGS.dtype = 'fp16' + FLAGS.loss_scale = 8192 + self._run_and_report_benchmark_mlperf_like() + + def benchmark_xla_1_gpu_ctl_fp16_mlperf_like(self): + """1 GPU using CTL with XLA and FP16.""" + self._setup() + FLAGS.keras_use_ctl = True + FLAGS.enable_xla = True + FLAGS.train_epochs = 7 + FLAGS.dtype = 'fp16' + FLAGS.loss_scale = 8192 + self._run_and_report_benchmark_mlperf_like() + + def benchmark_8_gpu_mlperf_like(self): + """8 GPU using keras fit/compile.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.train_epochs = 17 + FLAGS.batch_size = 1048576 + FLAGS.eval_batch_size = 160000 + FLAGS.learning_rate = 0.0045 + FLAGS.beta1 = 0.25 + FLAGS.beta2 = 0.5 + FLAGS.epsilon = 1e-8 + self._run_and_report_benchmark_mlperf_like() + + def benchmark_8_gpu_ctl_mlperf_like(self): + """8 GPU using CTL.""" + self._setup() + FLAGS.keras_use_ctl = True + FLAGS.num_gpus = 8 + FLAGS.train_epochs = 17 + FLAGS.batch_size = 1048576 + FLAGS.eval_batch_size = 160000 + FLAGS.learning_rate = 0.0045 + FLAGS.beta1 = 0.25 + FLAGS.beta2 = 0.5 + FLAGS.epsilon = 1e-8 + self._run_and_report_benchmark_mlperf_like() + + def benchmark_8_gpu_tf_data_ctl_mlperf_like(self): + """8 GPU using CTL.""" + self._setup() + FLAGS.keras_use_ctl = True + FLAGS.num_gpus = 8 + FLAGS.train_epochs = 17 + FLAGS.batch_size = 1048576 + FLAGS.eval_batch_size = 1048000 + FLAGS.learning_rate = 0.0045 + FLAGS.beta1 = 0.25 + FLAGS.beta2 = 0.5 + FLAGS.epsilon = 1e-8 + FLAGS.train_dataset_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, "training_cycle_*/*") + FLAGS.eval_dataset_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, "eval_data/*") + FLAGS.input_meta_data_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, "meta_data.json") + self._run_and_report_benchmark_mlperf_like() + + def benchmark_8_gpu_tf_data_fp16_mlperf_like(self): + """8 GPU FP16""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.train_epochs = 17 + FLAGS.batch_size = 1048576 + FLAGS.eval_batch_size = 1048000 + FLAGS.learning_rate = 0.0045 + FLAGS.beta1 = 0.25 + FLAGS.beta2 = 0.5 + FLAGS.epsilon = 1e-8 + FLAGS.dtype = 'fp16' + FLAGS.loss_scale = 8192 + FLAGS.train_dataset_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, "training_cycle_*/*") + FLAGS.eval_dataset_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, "eval_data/*") + FLAGS.input_meta_data_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, "meta_data.json") + self._run_and_report_benchmark_mlperf_like() + + def benchmark_8_gpu_tf_data_ctl_fp16_mlperf_like(self): + """8 GPU FP16 using CTL""" + self._setup() + FLAGS.keras_use_ctl = True + FLAGS.num_gpus = 8 + FLAGS.train_epochs = 17 + FLAGS.batch_size = 1048576 + FLAGS.eval_batch_size = 1048000 + FLAGS.learning_rate = 0.0045 + FLAGS.beta1 = 0.25 + FLAGS.beta2 = 0.5 + FLAGS.epsilon = 1e-8 + FLAGS.dtype = 'fp16' + FLAGS.loss_scale = 8192 + FLAGS.train_dataset_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, "training_cycle_*/*") + FLAGS.eval_dataset_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, "eval_data/*") + FLAGS.input_meta_data_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, "meta_data.json") + self._run_and_report_benchmark_mlperf_like() + + def benchmark_8_gpu_tf_data_ctl_fp16_graph_rewrite_mlperf_like(self): + """8 GPU FP16 graph rewrite using CTL.""" + self._setup() + FLAGS.keras_use_ctl = True + FLAGS.num_gpus = 8 + FLAGS.train_epochs = 17 + FLAGS.batch_size = 1048576 + FLAGS.eval_batch_size = 1048000 + FLAGS.learning_rate = 0.0045 + FLAGS.beta1 = 0.25 + FLAGS.beta2 = 0.5 + FLAGS.epsilon = 1e-8 + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.loss_scale = 8192 + FLAGS.train_dataset_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, + 'training_cycle_*/*') + FLAGS.eval_dataset_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, + 'eval_data/*') + FLAGS.input_meta_data_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, + 'meta_data.json') + self._run_and_report_benchmark_mlperf_like() + + +class NCFKerasSynth(NCFKerasBenchmarkBase): + """Benchmark NCF model using synthetic data.""" + + def __init__(self, + output_dir=None, + default_flags=None, + **kwargs): + + default_flags = {} + default_flags['dataset'] = 'ml-20m' + default_flags['num_gpus'] = 1 + default_flags['train_epochs'] = 8 + default_flags['batch_size'] = 99000 + default_flags['eval_batch_size'] = 160000 + default_flags['learning_rate'] = 0.00382059 + default_flags['beta1'] = 0.783529 + default_flags['beta2'] = 0.909003 + default_flags['epsilon'] = 1.45439e-07 + default_flags['layers'] = [256, 256, 128, 64] + default_flags['num_factors'] = 64 + default_flags['hr_threshold'] = 0.635 + default_flags['use_synthetic_data'] = True + + super(NCFKerasSynth, self).__init__( + output_dir=output_dir, + default_flags=default_flags, + **kwargs) + + def benchmark_1_gpu(self): + self._setup() + self._run_and_report_benchmark() + + def benchmark_2_gpus(self): + self._setup() + FLAGS.num_gpus = 2 + self._run_and_report_benchmark() + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/perfzero_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/perfzero_benchmark.py new file mode 100644 index 0000000..895f823 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/perfzero_benchmark.py @@ -0,0 +1,91 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utils for creating PerfZero benchmarks.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from absl import flags +from absl import logging +from absl.testing import flagsaver +import tensorflow as tf + +FLAGS = flags.FLAGS + + +class PerfZeroBenchmark(tf.test.Benchmark): + """Common methods used in PerfZero Benchmarks. + + Handles the resetting of flags between tests, loading of default_flags, + overriding of defaults. PerfZero (OSS) runs each test in a separate + process reducing some need to reset the flags. + """ + local_flags = None + + def __init__(self, + output_dir=None, + default_flags=None, + flag_methods=None, + tpu=None): + """Initialize class. + + Args: + output_dir: Base directory to store all output for the test. + default_flags: Set of flags to pass to model. + flag_methods: Set of flag methods to run during setup. + tpu: (optional) TPU name to use in a TPU benchmark. + """ + if os.getenv('BENCHMARK_OUTPUT_DIR'): + self.output_dir = os.getenv('BENCHMARK_OUTPUT_DIR') + elif output_dir: + self.output_dir = output_dir + else: + self.output_dir = '/tmp' + self.default_flags = default_flags or {} + self.flag_methods = flag_methods or {} + + if os.getenv('BENCHMARK_TPU'): + resolved_tpu = os.getenv('BENCHMARK_TPU') + elif tpu: + resolved_tpu = tpu + else: + resolved_tpu = None + + if resolved_tpu: + # TPU models are expected to accept a --tpu=name flag. PerfZero creates + # the TPU at runtime and passes the TPU's name to this flag. + self.default_flags['tpu'] = resolved_tpu + + def _get_model_dir(self, folder_name): + """Returns directory to store info, e.g. saved model and event log.""" + return os.path.join(self.output_dir, folder_name) + + def _setup(self): + """Sets up and resets flags before each test.""" + logging.set_verbosity(logging.INFO) + if PerfZeroBenchmark.local_flags is None: + for flag_method in self.flag_methods: + flag_method() + # Loads flags to get defaults to then override. List cannot be empty. + flags.FLAGS(['foo']) + # Overrides flag values with defaults for the class of tests. + for k, v in self.default_flags.items(): + setattr(FLAGS, k, v) + saved_flag_values = flagsaver.save_flag_values() + PerfZeroBenchmark.local_flags = saved_flag_values + else: + flagsaver.restore_flag_values(PerfZeroBenchmark.local_flags) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/resnet_ctl_imagenet_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/resnet_ctl_imagenet_benchmark.py new file mode 100644 index 0000000..c62bcb9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/resnet_ctl_imagenet_benchmark.py @@ -0,0 +1,412 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Executes CTL benchmarks and accuracy tests.""" +# pylint: disable=line-too-long,g-bad-import-order +from __future__ import print_function + +import os +import time + +from absl import flags +import tensorflow as tf + +from official.vision.image_classification.resnet import common +from official.vision.image_classification.resnet import resnet_ctl_imagenet_main +from official.benchmark.perfzero_benchmark import PerfZeroBenchmark +from official.benchmark import benchmark_wrappers +from official.utils.flags import core as flags_core + +MIN_TOP_1_ACCURACY = 0.76 +MAX_TOP_1_ACCURACY = 0.77 + +FLAGS = flags.FLAGS + + +class CtlBenchmark(PerfZeroBenchmark): + """Base benchmark class with methods to simplify testing.""" + + def __init__(self, output_dir=None, default_flags=None, flag_methods=None): + self.output_dir = output_dir + self.default_flags = default_flags or {} + self.flag_methods = flag_methods or {} + super(CtlBenchmark, self).__init__( + output_dir=self.output_dir, + default_flags=self.default_flags, + flag_methods=self.flag_methods) + + def _report_benchmark(self, + stats, + wall_time_sec, + top_1_max=None, + top_1_min=None, + total_batch_size=None, + log_steps=None, + warmup=1, + start_time_sec=None): + """Report benchmark results by writing to local protobuf file. + + Args: + stats: dict returned from keras models with known entries. + wall_time_sec: the during of the benchmark execution in seconds + top_1_max: highest passing level for top_1 accuracy. + top_1_min: lowest passing level for top_1 accuracy. + total_batch_size: Global batch-size. + log_steps: How often the log was created for stats['step_timestamp_log']. + warmup: number of entries in stats['step_timestamp_log'] to ignore. + start_time_sec: the start time of the program in seconds since epoch. + """ + + metrics = [] + if 'eval_acc' in stats: + metrics.append({ + 'name': 'accuracy_top_1', + 'value': stats['eval_acc'], + 'min_value': top_1_min, + 'max_value': top_1_max + }) + metrics.append({'name': 'eval_loss', 'value': stats['eval_loss']}) + + metrics.append({ + 'name': 'top_1_train_accuracy', + 'value': stats['train_acc'] + }) + metrics.append({'name': 'train_loss', 'value': stats['train_loss']}) + + if (warmup and 'step_timestamp_log' in stats and + len(stats['step_timestamp_log']) > warmup + 1): + # first entry in the time_log is start of step 0. The rest of the + # entries are the end of each step recorded + time_log = stats['step_timestamp_log'] + steps_elapsed = time_log[-1].batch_index - time_log[warmup].batch_index + time_elapsed = time_log[-1].timestamp - time_log[warmup].timestamp + examples_per_sec = total_batch_size * (steps_elapsed / time_elapsed) + metrics.append({'name': 'exp_per_second', 'value': examples_per_sec}) + + if 'avg_exp_per_second' in stats: + metrics.append({ + 'name': 'avg_exp_per_second', + 'value': stats['avg_exp_per_second'] + }) + + if start_time_sec and 'step_timestamp_log' in stats: + time_log = stats['step_timestamp_log'] + # time_log[0] is recorded at the beginning of the first step. + startup_time = time_log[0].timestamp - start_time_sec + metrics.append({'name': 'startup_time', 'value': startup_time}) + + flags_str = flags_core.get_nondefault_flags_as_str() + self.report_benchmark( + iters=-1, + wall_time=wall_time_sec, + metrics=metrics, + extras={'flags': flags_str}) + + +class Resnet50CtlAccuracy(CtlBenchmark): + """Benchmark accuracy tests for ResNet50 in CTL.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + """A benchmark class. + + Args: + output_dir: directory where to output e.g. log files + root_data_dir: directory under which to look for dataset + **kwargs: arbitrary named arguments. This is needed to make the + constructor forward compatible in case PerfZero provides more named + arguments before updating the constructor. + """ + + flag_methods = [common.define_keras_flags] + + self.data_dir = os.path.join(root_data_dir, 'imagenet') + super(Resnet50CtlAccuracy, self).__init__( + output_dir=output_dir, flag_methods=flag_methods) + + def benchmark_8_gpu(self): + """Test Keras model with eager, dist_strat and 8 GPUs.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 128 * 8 + FLAGS.train_epochs = 90 + FLAGS.epochs_between_evals = 10 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu') + FLAGS.dtype = 'fp32' + self._run_and_report_benchmark() + + def benchmark_8_gpu_fp16(self): + """Test Keras model with eager, 8 GPUs with tf.keras mixed precision.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 256 * 8 + FLAGS.train_epochs = 90 + FLAGS.epochs_between_evals = 10 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_fp16') + FLAGS.dtype = 'fp16' + self._run_and_report_benchmark() + + def benchmark_8_gpu_amp(self): + """Test Keras model with 8 GPUs and mixed precision via graph rewrite.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 256 * 8 + FLAGS.train_epochs = 90 + FLAGS.epochs_between_evals = 10 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_amp') + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + self._run_and_report_benchmark() + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self): + start_time_sec = time.time() + stats = resnet_ctl_imagenet_main.run(flags.FLAGS) + wall_time_sec = time.time() - start_time_sec + + super(Resnet50CtlAccuracy, self)._report_benchmark( + stats, + wall_time_sec, + top_1_min=MIN_TOP_1_ACCURACY, + top_1_max=MAX_TOP_1_ACCURACY, + total_batch_size=FLAGS.batch_size, + log_steps=100, + start_time_sec=start_time_sec) + + def _get_model_dir(self, folder_name): + return os.path.join(self.output_dir, folder_name) + + +class Resnet50CtlBenchmarkBase(CtlBenchmark): + """Resnet50 benchmarks.""" + + def __init__(self, output_dir=None, default_flags=None): + flag_methods = [common.define_keras_flags] + + super(Resnet50CtlBenchmarkBase, self).__init__( + output_dir=output_dir, + flag_methods=flag_methods, + default_flags=default_flags) + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self): + start_time_sec = time.time() + stats = resnet_ctl_imagenet_main.run(FLAGS) + wall_time_sec = time.time() - start_time_sec + + # Number of logged step time entries that are excluded in performance + # report. We keep results from last 100 batches in this case. + warmup = (FLAGS.train_steps - 100) // FLAGS.log_steps + + super(Resnet50CtlBenchmarkBase, self)._report_benchmark( + stats, + wall_time_sec, + total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps, + warmup=warmup, + start_time_sec=start_time_sec) + + def benchmark_1_gpu_no_dist_strat(self): + """Test Keras model with 1 GPU, no distribution strategy.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.distribution_strategy = 'off' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_no_dist_strat') + FLAGS.batch_size = 128 + self._run_and_report_benchmark() + + def benchmark_1_gpu(self): + """Test Keras model with 1 GPU.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu') + FLAGS.batch_size = 128 + self._run_and_report_benchmark() + + def benchmark_1_gpu_fp16(self): + """Test Keras model with 1 GPU with tf.keras mixed precision.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_fp16') + FLAGS.batch_size = 256 + FLAGS.dtype = 'fp16' + self._run_and_report_benchmark() + + def benchmark_1_gpu_amp(self): + """Test Keras model with 1 GPU with automatic mixed precision.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_amp') + FLAGS.batch_size = 256 + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + self._run_and_report_benchmark() + + def benchmark_xla_1_gpu_amp(self): + """Test Keras model with XLA and 1 GPU with automatic mixed precision.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu_amp') + FLAGS.batch_size = 256 + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.enable_xla = True + self._run_and_report_benchmark() + + def benchmark_1_gpu_eager(self): + """Test Keras model with 1 GPU in pure eager mode.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_eager') + FLAGS.batch_size = 120 + FLAGS.use_tf_function = False + FLAGS.use_tf_while_loop = False + FLAGS.single_l2_loss_op = True + self._run_and_report_benchmark() + + def benchmark_1_gpu_fp16_eager(self): + """Test Keras model with 1 GPU with fp16 and pure eager mode.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.distribution_strategy = 'one_device' + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_fp16_eager') + FLAGS.batch_size = 240 + FLAGS.dtype = 'fp16' + FLAGS.use_tf_function = False + FLAGS.use_tf_while_loop = False + FLAGS.single_l2_loss_op = True + self._run_and_report_benchmark() + + def benchmark_8_gpu(self): + """Test Keras model with 8 GPUs.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu') + FLAGS.batch_size = 128 * 8 # 8 GPUs + self._run_and_report_benchmark() + + def benchmark_8_gpu_fp16(self): + """Test Keras model with 8 GPUs with tf.keras mixed precision.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_fp16') + FLAGS.batch_size = 256 * 8 # 8 GPUs + FLAGS.dtype = 'fp16' + self._run_and_report_benchmark() + + def benchmark_8_gpu_eager(self): + """Test Keras model with 8 GPUs, eager, fp32.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.use_tf_function = False + FLAGS.use_tf_while_loop = False + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_eager') + FLAGS.batch_size = 128 + self._run_and_report_benchmark() + + def benchmark_8_gpu_eager_fp16(self): + """Test Keras model with 8 GPUs, eager, fp16.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.dtype = 'fp16' + FLAGS.use_tf_function = False + FLAGS.use_tf_while_loop = False + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_eager_fp16') + FLAGS.batch_size = 128 + self._run_and_report_benchmark() + + def benchmark_8_gpu_amp(self): + """Test Keras model with 8 GPUs with automatic mixed precision.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_amp') + FLAGS.batch_size = 256 * 8 # 8 GPUs + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + self._run_and_report_benchmark() + + def benchmark_xla_8_gpu_amp(self): + """Test Keras model with XLA and 8 GPUs with automatic mixed precision.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.distribution_strategy = 'mirrored' + FLAGS.model_dir = self._get_model_dir('benchmark_xla_8_gpu_amp') + FLAGS.batch_size = 256 * 8 # 8 GPUs + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.enable_xla = True + self._run_and_report_benchmark() + + def fill_report_object(self, stats): + super(Resnet50CtlBenchmarkBase, self).fill_report_object( + stats, total_batch_size=FLAGS.batch_size, log_steps=FLAGS.log_steps) + + +class Resnet50CtlBenchmarkSynth(Resnet50CtlBenchmarkBase): + """Resnet50 synthetic benchmark tests.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + def_flags = {} + def_flags['skip_eval'] = True + def_flags['use_synthetic_data'] = True + def_flags['train_steps'] = 110 + def_flags['steps_per_loop'] = 20 + def_flags['log_steps'] = 10 + + super(Resnet50CtlBenchmarkSynth, self).__init__( + output_dir=output_dir, default_flags=def_flags) + + +class Resnet50CtlBenchmarkReal(Resnet50CtlBenchmarkBase): + """Resnet50 real data benchmark tests.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + def_flags = {} + def_flags['skip_eval'] = True + def_flags['data_dir'] = os.path.join(root_data_dir, 'imagenet') + def_flags['train_steps'] = 110 + def_flags['steps_per_loop'] = 20 + def_flags['log_steps'] = 10 + + super(Resnet50CtlBenchmarkReal, self).__init__( + output_dir=output_dir, default_flags=def_flags) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/retinanet_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/retinanet_benchmark.py new file mode 100644 index 0000000..43ef66a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/retinanet_benchmark.py @@ -0,0 +1,294 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Executes RetinaNet benchmarks and accuracy tests.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# pylint: disable=g-bad-import-order +import copy +import json +import os +import time + +from absl import flags +from absl import logging +from absl.testing import flagsaver +import tensorflow as tf +# pylint: enable=g-bad-import-order + +from official.benchmark import bert_benchmark_utils as benchmark_utils +from official.utils.flags import core as flags_core +from official.benchmark import benchmark_wrappers +from official.vision.detection import main as detection + +TMP_DIR = os.getenv('TMPDIR') +FLAGS = flags.FLAGS + +# pylint: disable=line-too-long +COCO_TRAIN_DATA = 'gs://tf-perfzero-data/coco/train*' +COCO_EVAL_DATA = 'gs://tf-perfzero-data/coco/val*' +COCO_EVAL_JSON = 'gs://tf-perfzero-data/coco/instances_val2017.json' +RESNET_CHECKPOINT_PATH = 'gs://cloud-tpu-checkpoints/retinanet/resnet50-checkpoint-2018-02-07' +# pylint: enable=line-too-long + + +class DetectionBenchmarkBase(tf.test.Benchmark): + """Base class to hold methods common to test classes.""" + local_flags = None + + def __init__(self, output_dir=None): + self.num_gpus = 8 + + if not output_dir: + output_dir = '/tmp' + self.output_dir = output_dir + self.timer_callback = None + + def _get_model_dir(self, folder_name): + """Returns directory to store info, e.g. saved model and event log.""" + return os.path.join(self.output_dir, folder_name) + + def _setup(self): + """Sets up and resets flags before each test.""" + self.timer_callback = benchmark_utils.BenchmarkTimerCallback() + + if DetectionBenchmarkBase.local_flags is None: + # Loads flags to get defaults to then override. List cannot be empty. + flags.FLAGS(['foo']) + saved_flag_values = flagsaver.save_flag_values() + DetectionBenchmarkBase.local_flags = saved_flag_values + else: + flagsaver.restore_flag_values(DetectionBenchmarkBase.local_flags) + + def _report_benchmark(self, + stats, + wall_time_sec, + min_ap, + max_ap, + train_batch_size=None): + """Report benchmark results by writing to local protobuf file. + + Args: + stats: dict returned from Detection models with known entries. + wall_time_sec: the during of the benchmark execution in seconds + min_ap: Minimum detection AP constraint to verify correctness of the + model. + max_ap: Maximum detection AP accuracy constraint to verify correctness of + the model. + train_batch_size: Train batch size. It is needed for computing + exp_per_second. + """ + metrics = [{ + 'name': 'total_loss', + 'value': stats['total_loss'], + }] + if self.timer_callback: + metrics.append({ + 'name': 'exp_per_second', + 'value': self.timer_callback.get_examples_per_sec(train_batch_size) + }) + else: + metrics.append({ + 'name': 'exp_per_second', + 'value': 0.0, + }) + + if 'eval_metrics' in stats: + metrics.append({ + 'name': 'AP', + 'value': stats['AP'], + 'min_value': min_ap, + 'max_value': max_ap, + }) + flags_str = flags_core.get_nondefault_flags_as_str() + self.report_benchmark( + iters=stats['total_steps'], + wall_time=wall_time_sec, + metrics=metrics, + extras={'flags': flags_str}) + + +class RetinanetBenchmarkBase(DetectionBenchmarkBase): + """Base class to hold methods common to test classes in the module.""" + + def __init__(self, output_dir=None, **kwargs): + self.train_data_path = COCO_TRAIN_DATA + self.eval_data_path = COCO_EVAL_DATA + self.eval_json_path = COCO_EVAL_JSON + self.resnet_checkpoint_path = RESNET_CHECKPOINT_PATH + + super(RetinanetBenchmarkBase, self).__init__(output_dir=output_dir) + + def _run_detection_main(self): + """Starts detection job.""" + if self.timer_callback: + return detection.run(callbacks=[self.timer_callback]) + else: + return detection.run() + + +class RetinanetAccuracy(RetinanetBenchmarkBase): + """Accuracy test for RetinaNet model. + + Tests RetinaNet detection task model accuracy. The naming + convention of below test cases follow + `benchmark_(number of gpus)_gpu_(dataset type)` format. + """ + + def __init__(self, output_dir=TMP_DIR, **kwargs): + super(RetinanetAccuracy, self).__init__(output_dir=output_dir) + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self, min_ap=0.325, max_ap=0.35): + """Starts RetinaNet accuracy benchmark test.""" + + start_time_sec = time.time() + FLAGS.mode = 'train' + summary, _ = self._run_detection_main() + wall_time_sec = time.time() - start_time_sec + + FLAGS.mode = 'eval' + eval_metrics = self._run_detection_main() + summary.update(eval_metrics) + + summary['train_batch_size'] = self.params_override['train']['batch_size'] + summary['total_steps'] = self.params_override['train']['total_steps'] + super(RetinanetAccuracy, self)._report_benchmark( + stats=summary, + wall_time_sec=wall_time_sec, + min_ap=min_ap, + max_ap=max_ap, + train_batch_size=self.params_override['train']['batch_size']) + + def _setup(self): + super(RetinanetAccuracy, self)._setup() + FLAGS.strategy_type = 'mirrored' + FLAGS.model = 'retinanet' + + self.params_override = { + 'train': { + 'batch_size': 64, + 'iterations_per_loop': 100, + 'total_steps': 22500, + 'train_file_pattern': self.train_data_path, + 'checkpoint': { + 'path': self.resnet_checkpoint_path, + 'prefix': 'resnet50/' + }, + }, + 'eval': { + 'batch_size': 8, + 'eval_samples': 5000, + 'val_json_file': self.eval_json_path, + 'eval_file_pattern': self.eval_data_path, + }, + } + + @flagsaver.flagsaver + def benchmark_8_gpu_coco(self): + """Run RetinaNet model accuracy test with 8 GPUs.""" + self._setup() + params = copy.deepcopy(self.params_override) + FLAGS.params_override = json.dumps(params) + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_coco') + # Sets timer_callback to None as we do not use it now. + self.timer_callback = None + + self._run_and_report_benchmark() + + +class RetinanetBenchmarkReal(RetinanetAccuracy): + """Short benchmark performance tests for RetinaNet model. + + Tests RetinaNet performance in different GPU configurations. + The naming convention of below test cases follow + `benchmark_(number of gpus)_gpu` format. + """ + + def __init__(self, output_dir=TMP_DIR, **kwargs): + super(RetinanetBenchmarkReal, self).__init__(output_dir=output_dir) + + @flagsaver.flagsaver + def benchmark_8_gpu_coco(self): + """Run RetinaNet model accuracy test with 8 GPUs.""" + self.num_gpus = 8 + self._setup() + params = copy.deepcopy(self.params_override) + params['train']['total_steps'] = 1875 # One epoch. + # The iterations_per_loop must be one, otherwise the number of examples per + # second would be wrong. Currently only support calling callback per batch + # when each loop only runs on one batch, i.e. host loop for one step. The + # performance of this situation might be lower than the case of + # iterations_per_loop > 1. + # Related bug: b/135933080 + params['train']['iterations_per_loop'] = 1 + params['eval']['eval_samples'] = 8 + FLAGS.num_gpus = self.num_gpus + FLAGS.params_override = json.dumps(params) + FLAGS.model_dir = self._get_model_dir('real_benchmark_8_gpu_coco') + # Use negative value to avoid saving checkpoints. + FLAGS.save_checkpoint_freq = -1 + if self.timer_callback is None: + logging.error('Cannot measure performance without timer callback') + else: + self._run_and_report_benchmark() + + @flagsaver.flagsaver + def benchmark_1_gpu_coco(self): + """Run RetinaNet model accuracy test with 1 GPU.""" + self.num_gpus = 1 + self._setup() + params = copy.deepcopy(self.params_override) + params['train']['batch_size'] = 8 + params['train']['total_steps'] = 200 + params['train']['iterations_per_loop'] = 1 + params['eval']['eval_samples'] = 8 + FLAGS.num_gpus = self.num_gpus + FLAGS.params_override = json.dumps(params) + FLAGS.model_dir = self._get_model_dir('real_benchmark_1_gpu_coco') + FLAGS.strategy_type = 'one_device' + # Use negative value to avoid saving checkpoints. + FLAGS.save_checkpoint_freq = -1 + if self.timer_callback is None: + logging.error('Cannot measure performance without timer callback') + else: + self._run_and_report_benchmark() + + @flagsaver.flagsaver + def benchmark_xla_1_gpu_coco(self): + """Run RetinaNet model accuracy test with 1 GPU and XLA enabled.""" + self.num_gpus = 1 + self._setup() + params = copy.deepcopy(self.params_override) + params['train']['batch_size'] = 8 + params['train']['total_steps'] = 200 + params['train']['iterations_per_loop'] = 1 + params['eval']['eval_samples'] = 8 + FLAGS.num_gpus = self.num_gpus + FLAGS.params_override = json.dumps(params) + FLAGS.model_dir = self._get_model_dir('real_benchmark_1_gpu_coco') + FLAGS.strategy_type = 'one_device' + FLAGS.enable_xla = True + # Use negative value to avoid saving checkpoints. + FLAGS.save_checkpoint_freq = -1 + if self.timer_callback is None: + logging.error('Cannot measure performance without timer callback') + else: + self._run_and_report_benchmark() + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/shakespeare_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/shakespeare_benchmark.py new file mode 100644 index 0000000..ee42ffe --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/shakespeare_benchmark.py @@ -0,0 +1,359 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Executes Shakespeare (LSTM) benchmark and accuracy tests.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import time + +from absl import flags +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.benchmark.models.shakespeare import shakespeare_main +from official.utils.flags import core as flags_core +from official.utils.misc import keras_utils +from official.benchmark import benchmark_wrappers +from official.benchmark.perfzero_benchmark import PerfZeroBenchmark + +SHAKESPEARE_TRAIN_DATA = 'shakespeare/shakespeare.txt' +TMP_DIR = os.getenv('TMPDIR') +FLAGS = flags.FLAGS + + +class ShakespeareBenchmarkBase(PerfZeroBenchmark): + """Base class for Shakespeare (LSTM) benchmark and accuracy tests.""" + + def __init__(self, output_dir=None, default_flags=None, root_data_dir=None): + super(ShakespeareBenchmarkBase, self).__init__( + output_dir=output_dir, + default_flags=default_flags, + flag_methods=[shakespeare_main.define_flags]) + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self, + top_1_train_min=0.91, + top_1_train_max=0.94, + warmup=1, + log_steps=100): + """Report benchmark results by writing to local protobuf file. + + Average epoch time is calculated by skipping the first epoch. This average + ignores time spent between epoch and is recorded by begin and end epoch. To + skip accuracy check set `top_1_train_min=None`. + + Args: + top_1_train_min: lowest passing value. + top_1_train_max: highest passing value. + warmup: number of entries in `timestamp_log` to ignore. + log_steps: How often the log was created for `timestamp_log`. + """ + total_batch_size = FLAGS.batch_size + metrics = [] + start_time_sec = time.time() + stats = shakespeare_main.run(FLAGS) + wall_time_sec = time.time() - start_time_sec + + if top_1_train_min: + metrics.append({'name': 'accuracy_top_1_train', + 'value': stats['history']['RecallAt1'][-1], + 'min_value': top_1_train_min, + 'max_value': top_1_train_max}) + + # Look for the time history callback which was used during keras.fit + for callback in stats['callbacks']: + if isinstance(callback, keras_utils.TimeHistory): + epoch_timings = callback.epoch_runtime_log + if len(epoch_timings) > 1: + average_time = sum(epoch_timings[1:]) / len(epoch_timings[1:]) + metrics.append({'name': 'avg_epoch_time', + 'value': average_time}) + + # First entry in timestamp_log is the start of step 1. The rest of the + # entries are the end of each step recorded. + time_log = callback.timestamp_log + elapsed = time_log[-1].timestamp - time_log[warmup].timestamp + num_examples = ( + total_batch_size * log_steps * (len(time_log) - warmup - 1)) + if elapsed > 0: + examples_per_sec = num_examples / elapsed + metrics.append({'name': 'exp_per_second', + 'value': examples_per_sec}) + + flags_str = flags_core.get_nondefault_flags_as_str() + self.report_benchmark(iters=-1, wall_time=wall_time_sec, + metrics=metrics, + extras={'flags': flags_str}) + + +class ShakespeareAccuracy(ShakespeareBenchmarkBase): + """Shakespeare accuracy tests. + + This is not an ideal test. The best we can use for the accuracy check is to + validate top_1 of the training set. At batch size 64 the top_1 training + stabilizes to ~0.92 around 40-45 epochs. + """ + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + """Shakespeare accuracy tests. + + Args: + output_dir: directory where to output e.g. log files + root_data_dir: directory under which to look for dataset + **kwargs: arbitrary named arguments. This is needed to make the + constructor forward compatible in case PerfZero provides more + named arguments before updating the constructor. + """ + self.train_data = os.path.join(root_data_dir, SHAKESPEARE_TRAIN_DATA) + super(ShakespeareAccuracy, self).__init__( + output_dir=output_dir, root_data_dir=root_data_dir) + + def benchmark_cpu(self): + """Benchmark cpu.""" + self._setup() + FLAGS.num_gpus = 0 + FLAGS.training_data = self.train_data + FLAGS.batch_size = 64 + FLAGS.train_epochs = 43 + FLAGS.model_dir = '' + self._run_and_report_benchmark() + + def benchmark_cpu_no_ds_run_eagerly(self): + """Benchmark cpu without distribution strategies and run eagerly.""" + self._setup() + FLAGS.num_gpus = 0 + FLAGS.training_data = self.train_data + FLAGS.batch_size = 64 + FLAGS.train_epochs = 43 + FLAGS.model_dir = '' + FLAGS.run_eagerly = True + FLAGS.distribution_strategy = 'off' + self._run_and_report_benchmark() + + def benchmark_1_gpu(self): + """Benchmark 1 gpu.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.training_data = self.train_data + FLAGS.batch_size = 64 + FLAGS.train_epochs = 43 + FLAGS.model_dir = '' + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_ds(self): + """Benchmark 1 gpu without distribution strategies.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.training_data = self.train_data + FLAGS.batch_size = 64 + FLAGS.train_epochs = 43 + FLAGS.model_dir = '' + FLAGS.distribution_strategy = 'off' + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_ds_run_eagerly(self): + """Benchmark 1 gpu without distribution strategies and run eagerly.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.training_data = self.train_data + FLAGS.batch_size = 64 + FLAGS.train_epochs = 43 + FLAGS.model_dir = '' + FLAGS.run_eagerly = True + FLAGS.distribution_strategy = 'off' + self._run_and_report_benchmark() + + def benchmark_xla_1_gpu(self): + """Benchmark 1 gpu w/xla.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.training_data = self.train_data + FLAGS.batch_size = 64 + FLAGS.train_epochs = 43 + FLAGS.model_dir = '' + FLAGS.enable_xla = True + self._run_and_report_benchmark() + + def benchmark_8_gpu(self): + """Benchmark 8 gpu. + + This is test is for accuracy not scaling. The batch-size is not scaled to + the number of gpus. + """ + self._setup() + FLAGS.num_gpus = 8 + FLAGS.training_data = self.train_data + FLAGS.batch_size = 64 + FLAGS.train_epochs = 43 + FLAGS.model_dir = '' + self._run_and_report_benchmark() + + +class ShakespeareKerasBenchmarkReal(ShakespeareBenchmarkBase): + """Benchmark accuracy tests.""" + + def __init__(self, output_dir=None, root_data_dir=TMP_DIR, **kwargs): + """Benchmark tests w/Keras. + + Args: + output_dir: directory where to output e.g. log files + root_data_dir: directory under which to look for dataset + **kwargs: arbitrary named arguments. This is needed to make the + constructor forward compatible in case PerfZero provides more + named arguments before updating the constructor. + """ + self.train_data = os.path.join(root_data_dir, SHAKESPEARE_TRAIN_DATA) + + def_flags = {} + def_flags['training_data'] = self.train_data + def_flags['model_dir'] = '' + def_flags['train_epochs'] = 4 + def_flags['log_steps'] = 50 + + super(ShakespeareKerasBenchmarkReal, self).__init__( + output_dir=output_dir, + root_data_dir=root_data_dir, + default_flags=def_flags) + + def benchmark_cpu(self): + """Benchmark cpu.""" + self._setup() + FLAGS.num_gpus = 0 + FLAGS.batch_size = 64 + self._run_and_report_benchmark() + + def benchmark_cpu_no_ds_run_eagerly(self): + """Benchmark cpu without distribution strategy and run eagerly.""" + self._setup() + FLAGS.num_gpus = 0 + FLAGS.batch_size = 64 + FLAGS.distribution_strategy = 'off' + FLAGS.run_eagerly = True + self._run_and_report_benchmark() + + def benchmark_cpu_no_ds(self): + """Benchmark cpu without distribution strategy.""" + self._setup() + FLAGS.num_gpus = 0 + FLAGS.batch_size = 64 + FLAGS.distribution_strategy = 'off' + self._run_and_report_benchmark() + + def benchmark_cpu_no_ds_force_v2(self): + """Benchmark cpu no ds, and force v2.""" + self._setup() + FLAGS.num_gpus = 0 + FLAGS.batch_size = 64 + FLAGS.distribution_strategy = 'off' + self._run_and_report_benchmark() + + def benchmark_1_gpu(self): + """Benchmark 1 gpu.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.batch_size = 64 + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_cudnn(self): + """Benchmark 1 gpu with CuDNN disabled.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.batch_size = 64 + FLAGS.cudnn = False + FLAGS.enable_eager = keras_utils.is_v2_0() + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_ds(self): + """Benchmark 1 gpu without distribution strategies.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.batch_size = 64 + FLAGS.distribution_strategy = 'off' + self._run_and_report_benchmark() + + def benchmark_1_gpu_no_ds_run_eagerly(self): + """Benchmark 1 gpu.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.batch_size = 64 + FLAGS.run_eagerly = True + FLAGS.distribution_strategy = 'off' + self._run_and_report_benchmark() + + def benchmark_xla_1_gpu(self): + """Benchmark 1 gpu.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.batch_size = 64 + FLAGS.enable_xla = True + self._run_and_report_benchmark() + + def benchmark_xla_1_gpu_no_cudnn(self): + """Benchmark 1 gpu w/xla and CuDNN disabled.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.batch_size = 64 + FLAGS.cudnn = False + FLAGS.enable_eager = keras_utils.is_v2_0() + FLAGS.enable_xla = True + self._run_and_report_benchmark() + + def benchmark_8_gpu(self): + """Benchmark 8 gpu.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.batch_size = 64 * 8 + FLAGS.log_steps = 10 + self._run_and_report_benchmark() + + def benchmark_8_gpu_no_cudnn(self): + """Benchmark 8 gpu with CuDNN disabled.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.batch_size = 64 * 8 + FLAGS.log_steps = 10 + FLAGS.cudnn = False + FLAGS.enable_eager = keras_utils.is_v2_0() + self._run_and_report_benchmark() + + def benchmark_xla_8_gpu(self): + """Benchmark 8 gpu w/xla.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.batch_size = 64 * 8 + FLAGS.log_steps = 10 + FLAGS.enable_xla = True + self._run_and_report_benchmark() + + def benchmark_xla_8_gpu_no_cudnn(self): + """Benchmark 8 gpu w/xla and CuDNN disabled.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.batch_size = 64 * 8 + FLAGS.log_steps = 10 + FLAGS.cudnn = False + FLAGS.enable_eager = keras_utils.is_v2_0() + FLAGS.enable_xla = True + self._run_and_report_benchmark() + + def _run_and_report_benchmark(self): + """Run and report benchmark.""" + super(ShakespeareKerasBenchmarkReal, self)._run_and_report_benchmark( + top_1_train_min=None, log_steps=FLAGS.log_steps) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/tfhub_memory_usage_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/tfhub_memory_usage_benchmark.py new file mode 100644 index 0000000..7f50ecf --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/tfhub_memory_usage_benchmark.py @@ -0,0 +1,69 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Runs a memory usage benchmark for a Tensorflow Hub model. + +Loads a SavedModel and records memory usage. +""" +import functools +import time + +from absl import flags +import tensorflow as tf +import tensorflow_hub as hub + +from official.benchmark.perfzero_benchmark import PerfZeroBenchmark + +FLAGS = flags.FLAGS + + +class TfHubMemoryUsageBenchmark(PerfZeroBenchmark): + """A benchmark measuring memory usage for a given TF Hub SavedModel.""" + + def __init__(self, + hub_model_handle_list=None, + output_dir=None, + default_flags=None, + root_data_dir=None, + **kwargs): + super(TfHubMemoryUsageBenchmark, self).__init__( + output_dir=output_dir, default_flags=default_flags, **kwargs) + if hub_model_handle_list: + for hub_model_handle in hub_model_handle_list.split(';'): + # Converts a model handle of the form + # https://tfhub.dev/google/nnlm-en-dim128/1 to valid python method name + # like google_nnlm_en_dim128_1. + hub_model_method_name = hub_model_handle.replace( + 'https://tfhub.dev', + '').replace('/', '_').replace('-', '_').strip('_') + setattr( + self, 'benchmark_' + hub_model_method_name, + functools.partial(self.benchmark_memory_usage, hub_model_handle)) + + def benchmark_memory_usage( + self, hub_model_handle='https://tfhub.dev/google/nnlm-en-dim128/1'): + start_time_sec = time.time() + self.load_model(hub_model_handle) + wall_time_sec = time.time() - start_time_sec + + metrics = [] + self.report_benchmark(iters=-1, wall_time=wall_time_sec, metrics=metrics) + + def load_model(self, hub_model_handle): + """Loads a TF Hub module.""" + hub.load(hub_model_handle) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/transformer_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/transformer_benchmark.py new file mode 100644 index 0000000..0b3bc8a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/transformer_benchmark.py @@ -0,0 +1,681 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Executes Transformer w/Keras benchmark and accuracy tests.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import time + +from absl import flags +import tensorflow as tf +from official.benchmark import benchmark_wrappers +from official.benchmark.perfzero_benchmark import PerfZeroBenchmark +from official.nlp.transformer import misc +from official.nlp.transformer import transformer_main as transformer_main +from official.utils.flags import core as flags_core + +TRANSFORMER_EN2DE_DATA_DIR_NAME = 'wmt32k-en2de-official' +EN2DE_2014_BLEU_DATA_DIR_NAME = 'newstest2014' +FLAGS = flags.FLAGS +TMP_DIR = os.getenv('TMPDIR') + + +class TransformerBenchmark(PerfZeroBenchmark): + """Methods common to executing transformer w/keras tests. + + Code under test for the Transformer Keras models report the same data and + require the same FLAG setup. + """ + + def __init__(self, output_dir=None, default_flags=None, root_data_dir=None, + flag_methods=None): + root_data_dir = root_data_dir if root_data_dir else '' + + self.train_data_dir = os.path.join(root_data_dir, + TRANSFORMER_EN2DE_DATA_DIR_NAME) + + self.vocab_file = os.path.join(root_data_dir, + TRANSFORMER_EN2DE_DATA_DIR_NAME, + 'vocab.ende.32768') + + self.bleu_source = os.path.join(root_data_dir, + EN2DE_2014_BLEU_DATA_DIR_NAME, + 'newstest2014.en') + + self.bleu_ref = os.path.join(root_data_dir, + EN2DE_2014_BLEU_DATA_DIR_NAME, + 'newstest2014.de') + + if default_flags is None: + default_flags = {} + default_flags['data_dir'] = self.train_data_dir + default_flags['vocab_file'] = self.vocab_file + + super(TransformerBenchmark, self).__init__( + output_dir=output_dir, + default_flags=default_flags, + flag_methods=flag_methods) + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self, + bleu_max=None, + bleu_min=None, + log_steps=None, + total_batch_size=None, + warmup=1): + """Report benchmark results by writing to local protobuf file. + + Args: + bleu_max: highest passing level for bleu score. + bleu_min: lowest passing level for bleu score. + log_steps: How often the log was created for stats['step_timestamp_log']. + total_batch_size: Global batch-size. + warmup: number of entries in stats['step_timestamp_log'] to ignore. + """ + start_time_sec = time.time() + task = transformer_main.TransformerTask(FLAGS) + stats = task.train() + wall_time_sec = time.time() - start_time_sec + + metrics = [] + if 'bleu_uncased' in stats: + if 'bleu_uncased_history' in stats: + bleu_uncased_best = max(stats['bleu_uncased_history'], + key=lambda x: x[1]) + metrics.append({'name': 'bleu_uncased', + 'value': bleu_uncased_best[1], + 'min_value': bleu_min, + 'max_value': bleu_max}) + metrics.append({'name': 'bleu_best_score_iteration', + 'value': bleu_uncased_best[0]}) + metrics.append({'name': 'bleu_uncased_last', + 'value': stats['bleu_uncased']}) + else: + metrics.append({'name': 'bleu_uncased', + 'value': stats['bleu_uncased'], + 'min_value': bleu_min, + 'max_value': bleu_max}) + + if (warmup and 'step_timestamp_log' in stats and + len(stats['step_timestamp_log']) > warmup): + # first entry in the time_log is start of step 1. The rest of the + # entries are the end of each step recorded + time_log = stats['step_timestamp_log'] + elapsed = time_log[-1].timestamp - time_log[warmup].timestamp + num_examples = ( + total_batch_size * log_steps * (len(time_log) - warmup - 1)) + examples_per_sec = num_examples / elapsed + metrics.append({'name': 'exp_per_second', + 'value': examples_per_sec}) + + if 'avg_exp_per_second' in stats: + metrics.append({'name': 'avg_exp_per_second', + 'value': stats['avg_exp_per_second']}) + + flags_str = flags_core.get_nondefault_flags_as_str() + self.report_benchmark(iters=-1, wall_time=wall_time_sec, metrics=metrics, + extras={'flags': flags_str}) + + +class TransformerBaseKerasAccuracy(TransformerBenchmark): + """Benchmark accuracy tests for Transformer Base model w/ Keras.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + """Benchmark accuracy tests for Transformer Base model w/ Keras. + + Args: + output_dir: directory where to output e.g. log files + root_data_dir: directory under which to look for dataset + **kwargs: arbitrary named arguments. This is needed to make the + constructor forward compatible in case PerfZero provides more + named arguments before updating the constructor. + """ + flag_methods = [misc.define_transformer_flags] + + super(TransformerBaseKerasAccuracy, self).__init__( + output_dir=output_dir, root_data_dir=root_data_dir, + flag_methods=flag_methods) + + def benchmark_1_gpu(self): + """Benchmark 1 gpu. + + The paper uses 8 GPUs and a much larger effective batch size, this is will + not converge to the 27.3 BLEU (uncased) SOTA. + """ + self._setup() + FLAGS.num_gpus = 1 + FLAGS.data_dir = self.train_data_dir + FLAGS.vocab_file = self.vocab_file + # Sets values directly to avoid validation check. + FLAGS['bleu_source'].value = self.bleu_source + FLAGS['bleu_ref'].value = self.bleu_ref + FLAGS.param_set = 'base' + FLAGS.batch_size = 2048 + FLAGS.train_steps = 1000 + FLAGS.steps_between_evals = 500 + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu') + # These bleu scores are based on test runs after at this limited + # number of steps and batch size after verifying SOTA at 8xV100s. + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps, + bleu_min=25.3, + bleu_max=26) + + def benchmark_1_gpu_static_batch(self): + """Benchmark 1 gpu with static_batch. + + The paper uses 8 GPUs and a much larger effective batch size, this is will + not converge to the 27.3 BLEU (uncased) SOTA. + """ + self._setup() + FLAGS.num_gpus = 1 + FLAGS.data_dir = self.train_data_dir + FLAGS.vocab_file = self.vocab_file + # Sets values directly to avoid validation check. + FLAGS['bleu_source'].value = self.bleu_source + FLAGS['bleu_ref'].value = self.bleu_ref + FLAGS.param_set = 'base' + FLAGS.batch_size = 4096 + FLAGS.train_steps = 100000 + FLAGS.steps_between_evals = 5000 + FLAGS.static_batch = True + FLAGS.max_length = 64 + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_static_batch') + # These bleu scores are based on test runs after at this limited + # number of steps and batch size after verifying SOTA at 8xV100s. + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps, + bleu_min=25.3, + bleu_max=26) + + def benchmark_8_gpu(self): + """Benchmark 8 gpu. + + Should converge to 27.3 BLEU (uncased). This has not been confirmed yet. + """ + self._setup() + FLAGS.num_gpus = 8 + FLAGS.data_dir = self.train_data_dir + FLAGS.vocab_file = self.vocab_file + # Sets values directly to avoid validation check. + FLAGS['bleu_source'].value = self.bleu_source + FLAGS['bleu_ref'].value = self.bleu_ref + FLAGS.param_set = 'base' + FLAGS.batch_size = 4096*8 + FLAGS.train_steps = 100000 + FLAGS.steps_between_evals = 20000 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu') + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps, + bleu_min=27, + bleu_max=28) + + def benchmark_8_gpu_static_batch(self): + """Benchmark 8 gpu. + + Should converge to 27.3 BLEU (uncased). This has not been confirmed yet. + """ + self._setup() + FLAGS.num_gpus = 8 + FLAGS.data_dir = self.train_data_dir + FLAGS.vocab_file = self.vocab_file + # Sets values directly to avoid validation check. + FLAGS['bleu_source'].value = self.bleu_source + FLAGS['bleu_ref'].value = self.bleu_ref + FLAGS.param_set = 'base' + FLAGS.batch_size = 4096*8 + FLAGS.train_steps = 100000 + FLAGS.static_batch = True + FLAGS.max_length = 64 + FLAGS.steps_between_evals = 5000 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_static_batch') + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps, + bleu_min=27, + bleu_max=28) + + +class TransformerBigKerasAccuracy(TransformerBenchmark): + """Benchmark accuracy tests for Transformer Big model w/ Keras.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + """Benchmark accuracy tests for Transformer Big model w/ Keras. + + Args: + output_dir: directory where to output e.g. log files + root_data_dir: directory under which to look for dataset + **kwargs: arbitrary named arguments. This is needed to make the + constructor forward compatible in case PerfZero provides more + named arguments before updating the constructor. + """ + flag_methods = [misc.define_transformer_flags] + + super(TransformerBigKerasAccuracy, self).__init__( + output_dir=output_dir, root_data_dir=root_data_dir, + flag_methods=flag_methods) + + def benchmark_8_gpu(self): + """Benchmark 8 gpu. + + Over 6 runs with eval every 20K steps the average highest value was 28.195 + (bleu uncased). 28.424 was the highest and 27.96 the lowest. The values are + the highest value seen during a run and occurred at a median of iteration 9. + Iterations are not epochs, an iteration is a number of steps between evals. + """ + self._setup() + FLAGS.num_gpus = 8 + FLAGS.data_dir = self.train_data_dir + FLAGS.vocab_file = self.vocab_file + # Sets values directly to avoid validation check. + FLAGS['bleu_source'].value = self.bleu_source + FLAGS['bleu_ref'].value = self.bleu_ref + FLAGS.param_set = 'big' + FLAGS.batch_size = 3072*8 + FLAGS.train_steps = 20000 * 12 + FLAGS.steps_between_evals = 20000 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu') + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps, + bleu_min=27.9, + bleu_max=29.2) + + def benchmark_8_gpu_static_batch(self): + """Benchmark 8 gpu. + + Should converge to 28.4 BLEU (uncased). This has not be verified yet." + """ + self._setup() + FLAGS.num_gpus = 8 + FLAGS.data_dir = self.train_data_dir + FLAGS.vocab_file = self.vocab_file + # Sets values directly to avoid validation check. + FLAGS['bleu_source'].value = self.bleu_source + FLAGS['bleu_ref'].value = self.bleu_ref + FLAGS.param_set = 'big' + FLAGS.batch_size = 3072*8 + FLAGS.static_batch = True + FLAGS.max_length = 64 + FLAGS.train_steps = 20000 * 12 + FLAGS.steps_between_evals = 20000 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_static_batch') + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps, + bleu_min=28, + bleu_max=29.2) + + def benchmark_8_gpu_fp16(self): + """Benchmark 8 gpu with dynamic batch and fp16. + + Over 6 runs with eval every 20K steps the average highest value was 28.247 + (bleu uncased). 28.424 was the highest and 28.09 the lowest. The values are + the highest value seen during a run and occurred at a median of iteration + 11. While this could be interpreted as worse than FP32, if looking at the + first iteration at which 28 is passed FP16 performs equal and possibly + better. Although not part of the initial test runs, the highest value + recorded with the arguments below was 28.9 at iteration 12. Iterations are + not epochs, an iteration is a number of steps between evals. + """ + self._setup() + FLAGS.num_gpus = 8 + FLAGS.dtype = 'fp16' + FLAGS.data_dir = self.train_data_dir + FLAGS.vocab_file = self.vocab_file + # Sets values directly to avoid validation check. + FLAGS['bleu_source'].value = self.bleu_source + FLAGS['bleu_ref'].value = self.bleu_ref + FLAGS.param_set = 'big' + FLAGS.batch_size = 3072*8 + FLAGS.train_steps = 20000 * 12 + FLAGS.steps_between_evals = 20000 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_fp16') + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps, + bleu_min=28, + bleu_max=29.2) + + def benchmark_8_gpu_fp16_amp(self): + """Benchmark 8 gpu with dynamic batch and fp16 with automatic mixed precision. + + Should converge to 28.4 BLEU (uncased). This has not be verified yet." + """ + self._setup() + FLAGS.num_gpus = 8 + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.data_dir = self.train_data_dir + FLAGS.vocab_file = self.vocab_file + # Sets values directly to avoid validation check. + FLAGS['bleu_source'].value = self.bleu_source + FLAGS['bleu_ref'].value = self.bleu_ref + FLAGS.param_set = 'big' + FLAGS.batch_size = 3072*8 + FLAGS.train_steps = 20000 * 12 + FLAGS.steps_between_evals = 20000 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_fp16_amp') + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps, + bleu_min=28, + bleu_max=29) + + def benchmark_8_gpu_static_batch_fp16(self): + """Benchmark 8 gpu with static batch and fp16. + + Should converge to 28.4 BLEU (uncased). This has not be verified yet." + """ + self._setup() + FLAGS.num_gpus = 8 + FLAGS.dtype = 'fp16' + FLAGS.data_dir = self.train_data_dir + FLAGS.vocab_file = self.vocab_file + # Sets values directly to avoid validation check. + FLAGS['bleu_source'].value = self.bleu_source + FLAGS['bleu_ref'].value = self.bleu_ref + FLAGS.param_set = 'big' + FLAGS.batch_size = 3072*8 + FLAGS.static_batch = True + FLAGS.max_length = 64 + FLAGS.train_steps = 400000 + FLAGS.steps_between_evals = 20000 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_static_batch_fp16') + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps, + bleu_min=28, + bleu_max=29.2) + + def benchmark_xla_8_gpu_static_batch_fp16(self): + """Benchmark 8 gpu with static batch, XLA, and FP16. + + Should converge to 28.4 BLEU (uncased). This has not be verified yet." + """ + self._setup() + FLAGS.num_gpus = 8 + FLAGS.dtype = 'fp16' + FLAGS.enable_xla = True + FLAGS.data_dir = self.train_data_dir + FLAGS.vocab_file = self.vocab_file + # Sets values directly to avoid validation check. + FLAGS['bleu_source'].value = self.bleu_source + FLAGS['bleu_ref'].value = self.bleu_ref + FLAGS.param_set = 'big' + FLAGS.batch_size = 3072*8 + FLAGS.static_batch = True + FLAGS.max_length = 64 + FLAGS.train_steps = 400000 + FLAGS.steps_between_evals = 20000 + FLAGS.model_dir = self._get_model_dir( + 'benchmark_xla_8_gpu_static_batch_fp16') + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps, + bleu_min=28, + bleu_max=29.2) + + +class TransformerKerasBenchmark(TransformerBenchmark): + """Benchmarks for Transformer (Base and Big) using Keras.""" + + def __init__(self, output_dir=None, default_flags=None, + root_data_dir=None, batch_per_gpu=4096): + """Initialize. + + Args: + output_dir: Based directory for saving artifacts, e.g. checkpoints. + default_flags: default flags to use for all tests. + root_data_dir: root directory for data, e.g. training. + batch_per_gpu: batch size to use per gpu. + """ + flag_methods = [misc.define_transformer_flags] + self.batch_per_gpu = batch_per_gpu + + super(TransformerKerasBenchmark, self).__init__( + output_dir=output_dir, + default_flags=default_flags, + root_data_dir=root_data_dir, + flag_methods=flag_methods) + + def benchmark_1_gpu_no_dist_strat(self): + """Benchmark 1 gpu without distribution strategy.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.distribution_strategy = 'off' + FLAGS.batch_size = self.batch_per_gpu + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_no_dist_strat') + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_1_gpu_no_dist_strat_static_batch(self): + """Benchmark 1 gpu without distribution strategy with static batch.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.distribution_strategy = 'off' + FLAGS.batch_size = self.batch_per_gpu + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_no_ds_sb') + FLAGS.static_batch = True + FLAGS.max_length = 64 + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_1_gpu(self): + """Benchmark 1 gpu.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.batch_size = self.batch_per_gpu + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu') + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_1_gpu_fp16(self): + """Benchmark 1 gpu FP16.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.batch_size = self.batch_per_gpu + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_fp16') + FLAGS.dtype = 'fp16' + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_xla_1_gpu(self): + """Benchmark 1 gpu w/xla.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.batch_size = self.batch_per_gpu + FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu') + FLAGS.enable_xla = True + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_xla_1_gpu_fp16(self): + """Benchmark 1 gpu w/xla and FP16.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.batch_size = self.batch_per_gpu + FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu_fp16') + FLAGS.enable_xla = True + FLAGS.dtype = 'fp16' + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_1_gpu_static_batch(self): + """Benchmark 1 gpu with static batch.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.batch_size = self.batch_per_gpu + FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_static_batch') + FLAGS.static_batch = True + FLAGS.max_length = 64 + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_xla_1_gpu_static_batch(self): + """Benchmark 1 gpu with static batch w/xla.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.batch_size = self.batch_per_gpu + FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu_static_batch') + FLAGS.static_batch = True + FLAGS.max_length = 64 + FLAGS.enable_xla = True + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_1_gpu_static_batch_fp16(self): + """Benchmark 1 gpu with static batch FP16.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.batch_size = self.batch_per_gpu + FLAGS.model_dir = self._get_model_dir( + 'benchmark_1_gpu_static_batch_fp16') + FLAGS.static_batch = True + FLAGS.max_length = 64 + FLAGS.dtype = 'fp16' + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_xla_1_gpu_static_batch_fp16(self): + """Benchmark 1 gpu with static batch w/xla and FP16.""" + self._setup() + FLAGS.num_gpus = 1 + FLAGS.batch_size = self.batch_per_gpu + FLAGS.model_dir = self._get_model_dir( + 'benchmark_xla_1_gpu_static_batch_fp16') + FLAGS.static_batch = True + FLAGS.max_length = 64 + FLAGS.enable_xla = True + FLAGS.dtype = 'fp16' + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_8_gpu(self): + """Benchmark 8 gpu.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.batch_size = self.batch_per_gpu * 8 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu') + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_8_gpu_fp16(self): + """Benchmark 8 gpu FP16.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.dtype = 'fp16' + FLAGS.batch_size = self.batch_per_gpu * 8 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_fp16') + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_xla_8_gpu(self): + """Benchmark 8 gpu w/xla.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.enable_xla = True + FLAGS.batch_size = self.batch_per_gpu * 8 + FLAGS.model_dir = self._get_model_dir('benchmark_xla_8_gpu') + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_xla_8_gpu_fp16(self): + """Benchmark 8 gpu w/xla and FP16.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.enable_xla = True + FLAGS.dtype = 'fp16' + FLAGS.batch_size = self.batch_per_gpu * 8 + FLAGS.model_dir = self._get_model_dir('benchmark_xla_8_gpu_fp16') + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_8_gpu_static_batch(self): + """Benchmark 8 gpu with static batch.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.batch_size = self.batch_per_gpu * 8 + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_static_batch') + FLAGS.static_batch = True + FLAGS.max_length = 64 + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_8_gpu_static_batch_fp16(self): + """Benchmark 8 gpu with static batch FP16.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.dtype = 'fp16' + FLAGS.batch_size = self.batch_per_gpu * 8 + FLAGS.model_dir = self._get_model_dir( + 'benchmark_8_gpu_static_batch_fp16') + FLAGS.static_batch = True + FLAGS.max_length = 64 + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_xla_8_gpu_static_batch(self): + """Benchmark 8 gpu with static batch w/xla.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.enable_xla = True + FLAGS.batch_size = self.batch_per_gpu * 8 + FLAGS.model_dir = self._get_model_dir('benchmark_xla_8_gpu_static_batch') + FLAGS.static_batch = True + FLAGS.max_length = 64 + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + def benchmark_xla_8_gpu_static_batch_fp16(self): + """Benchmark 8 gpu with static batch w/xla and FP16.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.enable_xla = True + FLAGS.dtype = 'fp16' + FLAGS.batch_size = self.batch_per_gpu * 8 + FLAGS.model_dir = self._get_model_dir( + 'benchmark_xla_8_gpu_static_batch_fp16') + FLAGS.static_batch = True + FLAGS.max_length = 64 + self._run_and_report_benchmark(total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps) + + +class TransformerBaseKerasBenchmarkReal(TransformerKerasBenchmark): + """Transformer based version real data benchmark tests.""" + + def __init__(self, output_dir=TMP_DIR, root_data_dir=TMP_DIR, **kwargs): + def_flags = {} + def_flags['param_set'] = 'base' + def_flags['train_steps'] = 50 + def_flags['log_steps'] = 10 + + super(TransformerBaseKerasBenchmarkReal, self).__init__( + output_dir=output_dir, default_flags=def_flags, + root_data_dir=root_data_dir, batch_per_gpu=4096) + + +class TransformerBigKerasBenchmarkReal(TransformerKerasBenchmark): + """Transformer based version real data benchmark tests.""" + + def __init__(self, output_dir=TMP_DIR, root_data_dir=TMP_DIR, **kwargs): + def_flags = {} + def_flags['param_set'] = 'big' + def_flags['train_steps'] = 50 + def_flags['log_steps'] = 10 + + super(TransformerBigKerasBenchmarkReal, self).__init__( + output_dir=output_dir, default_flags=def_flags, + root_data_dir=root_data_dir, batch_per_gpu=3072) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/xlnet_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/xlnet_benchmark.py new file mode 100644 index 0000000..aff617e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/benchmark/xlnet_benchmark.py @@ -0,0 +1,216 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Executes XLNet benchmarks and accuracy tests.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import os +import time + +# pylint: disable=g-bad-import-order +from absl import flags +from absl.testing import flagsaver +import tensorflow as tf +# pylint: enable=g-bad-import-order + +from official.benchmark import bert_benchmark_utils as benchmark_utils +from official.nlp.xlnet import run_classifier +from official.nlp.xlnet import run_squad +from official.benchmark import benchmark_wrappers + + +# pylint: disable=line-too-long +PRETRAINED_CHECKPOINT_PATH = 'gs://cloud-tpu-checkpoints/xlnet/large/xlnet_model-1' +CLASSIFIER_TRAIN_DATA_PATH = 'gs://tf-perfzero-data/xlnet/imdb/spiece.model.len-512.train.tf_record' +CLASSIFIER_EVAL_DATA_PATH = 'gs://tf-perfzero-data/xlnet/imdb/spiece.model.len-512.dev.eval.tf_record' +SQUAD_DATA_PATH = 'gs://tf-perfzero-data/xlnet/squadv2_cased/' +# pylint: enable=line-too-long + +FLAGS = flags.FLAGS + + +class XLNetBenchmarkBase(benchmark_utils.BertBenchmarkBase): + """Base class to hold methods common to test classes in the module.""" + + def __init__(self, output_dir=None): + super(XLNetBenchmarkBase, self).__init__(output_dir) + self.num_epochs = None + self.num_steps_per_epoch = None + + @flagsaver.flagsaver + def _run_xlnet_classifier(self): + """Starts XLNet classification task.""" + run_classifier.main(unused_argv=None) + + @flagsaver.flagsaver + def _run_xlnet_squad(self): + """Starts XLNet classification task.""" + run_squad.main(unused_argv=None) + + +class XLNetClassifyAccuracy(XLNetBenchmarkBase): + """Short accuracy test for XLNet classifier model. + + Tests XLNet classification task model accuracy. The naming + convention of below test cases follow + `benchmark_(number of gpus)_gpu_(dataset type)` format. + """ + + def __init__(self, output_dir=None, **kwargs): + self.train_data_path = CLASSIFIER_TRAIN_DATA_PATH + self.eval_data_path = CLASSIFIER_EVAL_DATA_PATH + self.pretrained_checkpoint_path = PRETRAINED_CHECKPOINT_PATH + + super(XLNetClassifyAccuracy, self).__init__(output_dir=output_dir) + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self, + training_summary_path, + min_accuracy=0.95, + max_accuracy=0.97): + """Starts XLNet accuracy benchmark test.""" + + start_time_sec = time.time() + self._run_xlnet_classifier() + wall_time_sec = time.time() - start_time_sec + + with tf.io.gfile.GFile(training_summary_path, 'rb') as reader: + summary = json.loads(reader.read().decode('utf-8')) + + super(XLNetClassifyAccuracy, self)._report_benchmark( + stats=summary, + wall_time_sec=wall_time_sec, + min_accuracy=min_accuracy, + max_accuracy=max_accuracy) + + def _setup(self): + super(XLNetClassifyAccuracy, self)._setup() + FLAGS.test_data_size = 25024 + FLAGS.train_batch_size = 16 + FLAGS.seq_len = 512 + FLAGS.mem_len = 0 + FLAGS.n_layer = 24 + FLAGS.d_model = 1024 + FLAGS.d_embed = 1024 + FLAGS.n_head = 16 + FLAGS.d_head = 64 + FLAGS.d_inner = 4096 + FLAGS.untie_r = True + FLAGS.n_class = 2 + FLAGS.ff_activation = 'gelu' + FLAGS.strategy_type = 'mirror' + FLAGS.learning_rate = 2e-5 + FLAGS.train_steps = 4000 + FLAGS.warmup_steps = 500 + FLAGS.iterations = 200 + FLAGS.bi_data = False + FLAGS.init_checkpoint = self.pretrained_checkpoint_path + FLAGS.train_tfrecord_path = self.train_data_path + FLAGS.test_tfrecord_path = self.eval_data_path + + def benchmark_8_gpu_imdb(self): + """Run XLNet model accuracy test with 8 GPUs.""" + self._setup() + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_imdb') + # Sets timer_callback to None as we do not use it now. + self.timer_callback = None + + summary_path = os.path.join(FLAGS.model_dir, + 'summaries/training_summary.txt') + self._run_and_report_benchmark(summary_path) + + +class XLNetSquadAccuracy(XLNetBenchmarkBase): + """Short accuracy test for XLNet squad model. + + Tests XLNet squad task model accuracy. The naming + convention of below test cases follow + `benchmark_(number of gpus)_gpu_(dataset type)` format. + """ + + def __init__(self, output_dir=None, **kwargs): + self.train_data_path = SQUAD_DATA_PATH + self.predict_file = os.path.join(SQUAD_DATA_PATH, "dev-v2.0.json") + self.test_data_path = os.path.join(SQUAD_DATA_PATH, "12048.eval.tf_record") + self.spiece_model_file = os.path.join(SQUAD_DATA_PATH, "spiece.cased.model") + self.pretrained_checkpoint_path = PRETRAINED_CHECKPOINT_PATH + + super(XLNetSquadAccuracy, self).__init__(output_dir=output_dir) + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self, + training_summary_path, + min_accuracy=87.0, + max_accuracy=89.0): + """Starts XLNet accuracy benchmark test.""" + + start_time_sec = time.time() + self._run_xlnet_squad() + wall_time_sec = time.time() - start_time_sec + + with tf.io.gfile.GFile(training_summary_path, 'rb') as reader: + summary = json.loads(reader.read().decode('utf-8')) + + super(XLNetSquadAccuracy, self)._report_benchmark( + stats=summary, + wall_time_sec=wall_time_sec, + min_accuracy=min_accuracy, + max_accuracy=max_accuracy) + + def _setup(self): + super(XLNetSquadAccuracy, self)._setup() + FLAGS.train_batch_size = 16 + FLAGS.seq_len = 512 + FLAGS.mem_len = 0 + FLAGS.n_layer = 24 + FLAGS.d_model = 1024 + FLAGS.d_embed = 1024 + FLAGS.n_head = 16 + FLAGS.d_head = 64 + FLAGS.d_inner = 4096 + FLAGS.untie_r = True + FLAGS.ff_activation = 'gelu' + FLAGS.strategy_type = 'mirror' + FLAGS.learning_rate = 3e-5 + FLAGS.train_steps = 8000 + FLAGS.warmup_steps = 1000 + FLAGS.iterations = 1000 + FLAGS.bi_data = False + FLAGS.init_checkpoint = self.pretrained_checkpoint_path + FLAGS.train_tfrecord_path = self.train_data_path + FLAGS.test_tfrecord_path = self.test_data_path + FLAGS.spiece_model_file = self.spiece_model_file + FLAGS.predict_file = self.predict_file + FLAGS.adam_epsilon=1e-6 + FLAGS.lr_layer_decay_rate=0.75 + + def benchmark_8_gpu_squadv2(self): + """Run XLNet model squad v2 accuracy test with 8 GPUs.""" + self._setup() + FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_squadv2') + FLAGS.predict_dir = FLAGS.model_dir + # Sets timer_callback to None as we do not use it now. + self.timer_callback = None + + summary_path = os.path.join(FLAGS.model_dir, + 'summaries/training_summary.txt') + self._run_and_report_benchmark(summary_path) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/colab/bert.ipynb b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/colab/bert.ipynb new file mode 100644 index 0000000..88c92dd --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/colab/bert.ipynb @@ -0,0 +1,383 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "How-to Guide: Using a PIP package for fine-tuning a BERT model", + "provenance": [], + "collapsed_sections": [], + "toc_visible": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "5T_-iFRIqliG", + "colab_type": "text" + }, + "source": [ + "## How-to Guide: Using a PIP package for fine-tuning a BERT model\n", + "\n", + "Author: [Chen Chen](https://github.com/chenGitHuber)\n", + "\n", + "In this example, we will work through fine-tuning a BERT model using the tensorflow-models PIP package." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mY1vX5VAq4SS", + "colab_type": "text" + }, + "source": [ + "## License\n", + "\n", + "Copyright 2020 The TensorFlow Authors. All Rights Reserved.\n", + "\n", + "Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "you may not use this file except in compliance with the License.\n", + "You may obtain a copy of the License at\n", + "\n", + " http://www.apache.org/licenses/LICENSE-2.0\n", + "\n", + "Unless required by applicable law or agreed to in writing, software\n", + "distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "See the License for the specific language governing permissions and\n", + "limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XV3k63bt0ihl", + "colab_type": "text" + }, + "source": [ + "## Learning objectives\n", + "\n", + "In this Colab notebook, you will learn how to fine-tune a BERT model using the TensorFlow Model Garden PIP package." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MHA-RWherfG4", + "colab_type": "text" + }, + "source": [ + "## Enable the GPU acceleration\n", + "Please enable GPU for better performance.\n", + "* Navigate to Edit 🡒 Notebook settings\n", + "* Select GPU from the \"Hardware Accelerator\" drop-down list\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "l2B4N5Djrs2l", + "colab_type": "text" + }, + "source": [ + "## Install the Model Garden PIP package\n", + "\n", + "Install the Model Garden PIP package (tf-models-nightly) and other necessary PIP packages." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "VMYZDly6rx97", + "colab_type": "code", + "outputId": "146956ab-4568-4de6-c78e-cf25f115a5a8", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + } + }, + "source": [ + "pip install tf-models-nightly" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Collecting tf-models-nightly\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/bd/7c/1390d4e05d4d370e91d32dd9700d3a462dbc560c7f4e95a6477592b17def/tf_models_nightly-2.2.0.dev20200326-py2.py3-none-any.whl (710kB)\n", + "\u001b[K |████████████████████████████████| 716kB 2.8MB/s \n", + "\u001b[?25hRequirement already satisfied: scipy>=0.19.1 in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (1.4.1)\n", + "Collecting opencv-python-headless\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/0b/23/5f10b30a48b218a4884bc84188c14381ac71288b210f6f8079a54f7a05e8/opencv_python_headless-4.2.0.32-cp36-cp36m-manylinux1_x86_64.whl (21.6MB)\n", + "\u001b[K |████████████████████████████████| 21.6MB 1.3MB/s \n", + "\u001b[?25hRequirement already satisfied: tensorflow-hub>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (0.7.0)\n", + "Requirement already satisfied: typing in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (3.6.6)\n", + "Collecting tensorflow-model-optimization>=0.2.1\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/8f/c4/4c3d011e432bd9c19f0323f7da7d3f783402615e4c3b5a98416c7da9cb05/tensorflow_model_optimization-0.2.1-py2.py3-none-any.whl (93kB)\n", + "\u001b[K |████████████████████████████████| 102kB 10.2MB/s \n", + "\u001b[?25hRequirement already satisfied: numpy>=1.15.4 in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (1.18.2)\n", + "Requirement already satisfied: pandas>=0.22.0 in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (0.25.3)\n", + "Requirement already satisfied: gin-config in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (0.3.0)\n", + "Requirement already satisfied: google-cloud-bigquery>=0.31.0 in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (1.21.0)\n", + "Collecting mlperf-compliance==0.0.10\n", + " Downloading https://files.pythonhosted.org/packages/f4/08/f2febd8cbd5c9371f7dab311e90400d83238447ba7609b3bf0145b4cb2a2/mlperf_compliance-0.0.10-py3-none-any.whl\n", + "Requirement already satisfied: kaggle>=1.3.9 in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (1.5.6)\n", + "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (1.12.0)\n", + "Requirement already satisfied: google-api-python-client>=1.6.7 in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (1.7.12)\n", + "Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (0.8.3)\n", + "Requirement already satisfied: psutil>=5.4.3 in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (5.4.8)\n", + "Requirement already satisfied: tensorflow-datasets in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (2.1.0)\n", + "Collecting sentencepiece\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n", + "\u001b[K |████████████████████████████████| 1.0MB 58.3MB/s \n", + "\u001b[?25hCollecting tf-nightly\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/39/1c/4408b4c4b0d8008a7de62162e35089d59d19cc7543cfd1b23a70121f3086/tf_nightly-2.2.0.dev20200325-cp36-cp36m-manylinux2010_x86_64.whl (516.1MB)\n", + "\u001b[K |████████████████████████████████| 516.1MB 21kB/s \n", + "\u001b[?25hCollecting py-cpuinfo>=3.3.0\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/42/60/63f28a5401da733043abe7053e7d9591491b4784c4f87c339bf51215aa0a/py-cpuinfo-5.0.0.tar.gz (82kB)\n", + "\u001b[K |████████████████████████████████| 92kB 13.7MB/s \n", + "\u001b[?25hRequirement already satisfied: Pillow in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (7.0.0)\n", + "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (3.13)\n", + "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (3.2.1)\n", + "Requirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (0.7)\n", + "Requirement already satisfied: Cython in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (0.29.15)\n", + "Requirement already satisfied: oauth2client>=4.1.2 in /usr/local/lib/python3.6/dist-packages (from tf-models-nightly) (4.1.3)\n", + "Requirement already satisfied: protobuf>=3.4.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-hub>=0.6.0->tf-models-nightly) (3.10.0)\n", + "Collecting enum34~=1.1\n", + " Downloading https://files.pythonhosted.org/packages/63/f6/ccb1c83687756aeabbf3ca0f213508fcfb03883ff200d201b3a4c60cedcc/enum34-1.1.10-py3-none-any.whl\n", + "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.22.0->tf-models-nightly) (2.8.1)\n", + "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.22.0->tf-models-nightly) (2018.9)\n", + "Requirement already satisfied: google-resumable-media!=0.4.0,<0.5.0dev,>=0.3.1 in /usr/local/lib/python3.6/dist-packages (from google-cloud-bigquery>=0.31.0->tf-models-nightly) (0.4.1)\n", + "Requirement already satisfied: google-cloud-core<2.0dev,>=1.0.3 in /usr/local/lib/python3.6/dist-packages (from google-cloud-bigquery>=0.31.0->tf-models-nightly) (1.0.3)\n", + "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from kaggle>=1.3.9->tf-models-nightly) (1.24.3)\n", + "Requirement already satisfied: certifi in /usr/local/lib/python3.6/dist-packages (from kaggle>=1.3.9->tf-models-nightly) (2019.11.28)\n", + "Requirement already satisfied: python-slugify in /usr/local/lib/python3.6/dist-packages (from kaggle>=1.3.9->tf-models-nightly) (4.0.0)\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from kaggle>=1.3.9->tf-models-nightly) (4.38.0)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from kaggle>=1.3.9->tf-models-nightly) (2.21.0)\n", + "Requirement already satisfied: google-auth-httplib2>=0.0.3 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.6.7->tf-models-nightly) (0.0.3)\n", + "Requirement already satisfied: httplib2<1dev,>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.6.7->tf-models-nightly) (0.17.0)\n", + "Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.6.7->tf-models-nightly) (3.0.1)\n", + "Requirement already satisfied: google-auth>=1.4.1 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.6.7->tf-models-nightly) (1.7.2)\n", + "Requirement already satisfied: typeguard in /usr/local/lib/python3.6/dist-packages (from tensorflow-addons->tf-models-nightly) (2.7.1)\n", + "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->tf-models-nightly) (0.16.0)\n", + "Requirement already satisfied: absl-py in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->tf-models-nightly) (0.9.0)\n", + "Requirement already satisfied: wrapt in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->tf-models-nightly) (1.12.1)\n", + "Requirement already satisfied: promise in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->tf-models-nightly) (2.3)\n", + "Requirement already satisfied: attrs>=18.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->tf-models-nightly) (19.3.0)\n", + "Requirement already satisfied: termcolor in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->tf-models-nightly) (1.1.0)\n", + "Requirement already satisfied: tensorflow-metadata in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->tf-models-nightly) (0.21.1)\n", + "Requirement already satisfied: dill in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->tf-models-nightly) (0.3.1.1)\n", + "Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.6/dist-packages (from tf-nightly->tf-models-nightly) (0.3.3)\n", + "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tf-nightly->tf-models-nightly) (0.34.2)\n", + "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tf-nightly->tf-models-nightly) (1.27.2)\n", + "Collecting tb-nightly<2.3.0a0,>=2.2.0a0\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/52/b6/aa559ea9edbc6129a64ff752dbf6567bcd62fba34566defca00fdff4345e/tb_nightly-2.2.0a20200324-py3-none-any.whl (2.8MB)\n", + "\u001b[K |████████████████████████████████| 2.8MB 51.9MB/s \n", + "\u001b[?25hRequirement already satisfied: h5py<2.11.0,>=2.10.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly->tf-models-nightly) (2.10.0)\n", + "Collecting tf-estimator-nightly\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/bf/eb/0d6c06d1181cd9b52d7c82535073887d68d224f3dbeeb00adefd04762a9c/tf_estimator_nightly-2.3.0.dev2020032501-py2.py3-none-any.whl (455kB)\n", + "\u001b[K |████████████████████████████████| 460kB 53.5MB/s \n", + "\u001b[?25hRequirement already satisfied: google-pasta>=0.1.8 in /usr/local/lib/python3.6/dist-packages (from tf-nightly->tf-models-nightly) (0.2.0)\n", + "Requirement already satisfied: astunparse==1.6.3 in /usr/local/lib/python3.6/dist-packages (from tf-nightly->tf-models-nightly) (1.6.3)\n", + "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tf-nightly->tf-models-nightly) (3.2.0)\n", + "Requirement already satisfied: keras-preprocessing>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly->tf-models-nightly) (1.1.0)\n", + "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->tf-models-nightly) (2.4.6)\n", + "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->tf-models-nightly) (1.1.0)\n", + "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->tf-models-nightly) (0.10.0)\n", + "Requirement already satisfied: pyasn1-modules>=0.0.5 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.1.2->tf-models-nightly) (0.2.8)\n", + "Requirement already satisfied: rsa>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.1.2->tf-models-nightly) (4.0)\n", + "Requirement already satisfied: pyasn1>=0.1.7 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.1.2->tf-models-nightly) (0.4.8)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.4.0->tensorflow-hub>=0.6.0->tf-models-nightly) (46.0.0)\n", + "Requirement already satisfied: google-api-core<2.0.0dev,>=1.14.0 in /usr/local/lib/python3.6/dist-packages (from google-cloud-core<2.0dev,>=1.0.3->google-cloud-bigquery>=0.31.0->tf-models-nightly) (1.16.0)\n", + "Requirement already satisfied: text-unidecode>=1.3 in /usr/local/lib/python3.6/dist-packages (from python-slugify->kaggle>=1.3.9->tf-models-nightly) (1.3)\n", + "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->kaggle>=1.3.9->tf-models-nightly) (2.8)\n", + "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->kaggle>=1.3.9->tf-models-nightly) (3.0.4)\n", + "Requirement already satisfied: cachetools<3.2,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.4.1->google-api-python-client>=1.6.7->tf-models-nightly) (3.1.1)\n", + "Requirement already satisfied: googleapis-common-protos in /usr/local/lib/python3.6/dist-packages (from tensorflow-metadata->tensorflow-datasets->tf-models-nightly) (1.51.0)\n", + "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly->tf-models-nightly) (3.2.1)\n", + "Collecting tensorboard-plugin-wit>=1.6.0\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/41/ec/3da49289b93963bd8b32d29ed108f1809436ff3d9cd4e29c90bac4a7292f/tensorboard_plugin_wit-1.6.0.post2-py3-none-any.whl (775kB)\n", + "\u001b[K |████████████████████████████████| 778kB 43.8MB/s \n", + "\u001b[?25hRequirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly->tf-models-nightly) (0.4.1)\n", + "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly->tf-models-nightly) (1.0.0)\n", + "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly->tf-models-nightly) (1.3.0)\n", + "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly->tf-models-nightly) (3.1.0)\n", + "Building wheels for collected packages: py-cpuinfo\n", + " Building wheel for py-cpuinfo (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for py-cpuinfo: filename=py_cpuinfo-5.0.0-cp36-none-any.whl size=18684 sha256=093853bc49757be8f9facd8d38eab95b3cc2de13514dafa221ad7898725491ff\n", + " Stored in directory: /root/.cache/pip/wheels/01/7e/a9/b982d0fea22b7e4ae5619de949570cde5ad55420cec16e86a5\n", + "Successfully built py-cpuinfo\n", + "Installing collected packages: opencv-python-headless, enum34, tensorflow-model-optimization, mlperf-compliance, sentencepiece, tensorboard-plugin-wit, tb-nightly, tf-estimator-nightly, tf-nightly, py-cpuinfo, tf-models-nightly\n", + "Successfully installed enum34-1.1.10 mlperf-compliance-0.0.10 opencv-python-headless-4.2.0.32 py-cpuinfo-5.0.0 sentencepiece-0.1.85 tb-nightly-2.2.0a20200324 tensorboard-plugin-wit-1.6.0.post2 tensorflow-model-optimization-0.2.1 tf-estimator-nightly-2.3.0.dev2020032501 tf-models-nightly-2.2.0.dev20200326 tf-nightly-2.2.0.dev20200325\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.colab-display-data+json": { + "pip_warning": { + "packages": [ + "enum" + ] + } + } + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "w4oyRCTji-aa", + "colab_type": "text" + }, + "source": [ + "## BERT Fine-tuning\n", + "\n", + "The following code import necessary modules for fine-tuning a BERT model on a classification task.\n", + "\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "rV8MIX7g078-", + "colab_type": "code", + "colab": {} + }, + "source": [ + "%tensorflow_version 2.x\n", + "import tensorflow as tf\n", + "\n", + "import json\n", + "import math\n", + "\n", + "from official.utils.misc import distribution_utils\n", + "from official.nlp import optimization\n", + "from official.nlp.bert import bert_models\n", + "from official.nlp.bert import configs as bert_configs\n", + "from official.nlp.bert import run_classifier\n", + "from official.modeling import activations\n", + "from official.nlp.modeling import networks\n", + "from official.nlp.modeling.models import bert_classifier" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DIuS8nYD08n3", + "colab_type": "text" + }, + "source": [ + "This section of code performs the following tasks:\n", + "* Load data for fine-tuning\n", + "* Fine-tune a BERT model\n", + "* Save the fine-tuned model to a TensorFlow SavedModel file\n", + "\n", + "Please check [create_finetuning_data.py](https://github.com/tensorflow/models/blob/master/official/nlp/data/create_finetuning_data.py) if you want to know how the train/eval data are created." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "PAby1RTCi_1e", + "colab_type": "code", + "outputId": "e663e830-cc9b-4b5d-99db-4504fd66d5f3", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 258 + } + }, + "source": [ + "\n", + "train_data_path = \"gs://cloud-tpu-checkpoints/bert/classification/mrpc_train.tf_record\"\n", + "eval_data_path = \"gs://cloud-tpu-checkpoints/bert/classification/mrpc_eval.tf_record\"\n", + "input_meta_path = \"gs://cloud-tpu-checkpoints/bert/classification/mrpc_meta_data\"\n", + "\n", + "bert_config_file = \"gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-12_H-768_A-12/bert_config.json\"\n", + "ckpt_path = 'gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-12_H-768_A-12/bert_model.ckpt'\n", + "\n", + "with tf.io.gfile.GFile(input_meta_path, 'rb') as reader:\n", + " input_meta_data = json.loads(reader.read().decode('utf-8'))\n", + "\n", + "max_seq_length = input_meta_data['max_seq_length']\n", + "num_classes = input_meta_data['num_labels']\n", + "batch_size = 32\n", + "eval_batch_size = 32\n", + "train_input_fn = run_classifier.get_dataset_fn(train_data_path, max_seq_length, batch_size, is_training=True)\n", + "eval_input_fn = run_classifier.get_dataset_fn(eval_data_path, max_seq_length, eval_batch_size, is_training=False)\n", + "\n", + "strategy = distribution_utils.get_distribution_strategy(\n", + " distribution_strategy='one_device', num_gpus=1)\n", + "\n", + "with strategy.scope():\n", + " training_dataset = train_input_fn()\n", + " evaluation_dataset = eval_input_fn()\n", + " bert_config = bert_configs.BertConfig.from_json_file(bert_config_file)\n", + " classifier_model, encoder = bert_models.classifier_model(\n", + " bert_config, num_classes, max_seq_length)\n", + "\n", + " checkpoint = tf.train.Checkpoint(model=encoder)\n", + " checkpoint.restore(ckpt_path).assert_consumed()\n", + "\n", + " epochs = 3\n", + " train_data_size = input_meta_data['train_data_size']\n", + " eval_data_size = input_meta_data['eval_data_size']\n", + " steps_per_epoch = int(train_data_size / batch_size)\n", + " warmup_steps = int(epochs * train_data_size * 0.1 / batch_size)\n", + " optimizer = optimization.create_optimizer(\n", + " 2e-5, num_train_steps=steps_per_epoch * epochs, num_warmup_steps=warmup_steps)\n", + "\n", + " def metric_fn():\n", + " return tf.keras.metrics.SparseCategoricalAccuracy(\n", + " 'test_accuracy', dtype=tf.float32)\n", + "\n", + " classifier_model.compile(optimizer=optimizer,\n", + " loss=run_classifier.get_loss_fn(num_classes=2),\n", + " metrics=[metric_fn()])\n", + " classifier_model.fit(\n", + " x=training_dataset,\n", + " validation_data=evaluation_dataset,\n", + " steps_per_epoch=steps_per_epoch,\n", + " epochs=epochs,\n", + " validation_steps=int(eval_data_size / eval_batch_size))\n", + "\n", + " classifier_model.save('/tmp/saved_model', include_optimizer=False, save_format='tf')" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "WARNING:tensorflow:BertClassifier inputs must come from `tf.keras.Input` (thus holding past layer metadata), they cannot be the output of a previous non-Input layer. Here, a tensor specified as input to \"bert_classifier\" was not an Input tensor, it was generated by layer input_mask.\n", + "Note that input tensors are instantiated via `tensor = tf.keras.Input(shape)`.\n", + "The tensor that caused the issue was: input_mask:0\n", + "WARNING:tensorflow:BertClassifier inputs must come from `tf.keras.Input` (thus holding past layer metadata), they cannot be the output of a previous non-Input layer. Here, a tensor specified as input to \"bert_classifier\" was not an Input tensor, it was generated by layer input_type_ids.\n", + "Note that input tensors are instantiated via `tensor = tf.keras.Input(shape)`.\n", + "The tensor that caused the issue was: input_type_ids:0\n", + "Epoch 1/3\n", + "114/114 [==============================] - 96s 840ms/step - loss: 0.5932 - test_accuracy: 0.6960 - val_loss: 0.5083 - val_test_accuracy: 0.7604\n", + "Epoch 2/3\n", + "114/114 [==============================] - 100s 878ms/step - loss: 0.4225 - test_accuracy: 0.8183 - val_loss: 0.4020 - val_test_accuracy: 0.8438\n", + "Epoch 3/3\n", + "114/114 [==============================] - 100s 880ms/step - loss: 0.2482 - test_accuracy: 0.9134 - val_loss: 0.4065 - val_test_accuracy: 0.8151\n", + "INFO:tensorflow:Assets written to: /tmp/saved_model/assets\n" + ], + "name": "stdout" + } + ] + } + ] +} \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/__init__.py new file mode 100644 index 0000000..2b558fe --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Activations package definition.""" +from official.modeling.activations.gelu import gelu +from official.modeling.activations.swish import hard_swish +from official.modeling.activations.swish import identity +from official.modeling.activations.swish import simple_swish diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/gelu.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/gelu.py new file mode 100644 index 0000000..c045bff --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/gelu.py @@ -0,0 +1,40 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Gaussian error linear unit.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math + +import tensorflow as tf + + +@tf.keras.utils.register_keras_serializable(package='Text') +def gelu(x): + """Gaussian Error Linear Unit. + + This is a smoother version of the RELU. + Original paper: https://arxiv.org/abs/1606.08415 + Args: + x: float Tensor to perform activation. + + Returns: + `x` with the GELU activation applied. + """ + cdf = 0.5 * (1.0 + tf.tanh( + (math.sqrt(2 / math.pi) * (x + 0.044715 * tf.pow(x, 3))))) + return x * cdf diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/gelu_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/gelu_test.py new file mode 100644 index 0000000..dc3b95c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/gelu_test.py @@ -0,0 +1,38 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for the Gaussian error linear unit.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.modeling import activations + + +@keras_parameterized.run_all_keras_modes +class GeluTest(keras_parameterized.TestCase): + + def test_gelu(self): + expected_data = [[0.14967535, 0., -0.10032465], + [-0.15880796, -0.04540223, 2.9963627]] + gelu_data = activations.gelu([[.25, 0, -.25], [-1, -2, 3]]) + self.assertAllClose(expected_data, gelu_data) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/swish.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/swish.py new file mode 100644 index 0000000..1d79961 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/swish.py @@ -0,0 +1,75 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Customized Swish activation.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + + +@tf.keras.utils.register_keras_serializable(package='Text') +def simple_swish(features): + """Computes the Swish activation function. + + The tf.nn.swish operation uses a custom gradient to reduce memory usage. + Since saving custom gradients in SavedModel is currently not supported, and + one would not be able to use an exported TF-Hub module for fine-tuning, we + provide this wrapper that can allow to select whether to use the native + TensorFlow swish operation, or whether to use a customized operation that + has uses default TensorFlow gradient computation. + + Args: + features: A `Tensor` representing preactivation values. + + Returns: + The activation value. + """ + features = tf.convert_to_tensor(features) + return features * tf.nn.sigmoid(features) + + +@tf.keras.utils.register_keras_serializable(package='Text') +def hard_swish(features): + """Computes a hard version of the swish function. + + This operation can be used to reduce computational cost and improve + quantization for edge devices. + + Args: + features: A `Tensor` representing preactivation values. + + Returns: + The activation value. + """ + features = tf.convert_to_tensor(features) + return features * tf.nn.relu6(features + tf.constant(3.)) * (1. / 6.) + + +@tf.keras.utils.register_keras_serializable(package='Text') +def identity(features): + """Computes the identity function. + + Useful for helping in quantization. + + Args: + features: A `Tensor` representing preactivation values. + + Returns: + The activation value. + """ + features = tf.convert_to_tensor(features) + return tf.identity(features) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/swish_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/swish_test.py new file mode 100644 index 0000000..22042e9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/activations/swish_test.py @@ -0,0 +1,49 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for the customized Swish activation.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.modeling import activations + + +@keras_parameterized.run_all_keras_modes +class CustomizedSwishTest(keras_parameterized.TestCase): + + def _hard_swish_np(self, x): + x = np.float32(x) + return x * np.clip(x + 3, 0, 6) / 6 + + def test_simple_swish(self): + features = [[.25, 0, -.25], [-1, -2, 3]] + customized_swish_data = activations.simple_swish(features) + swish_data = tf.nn.swish(features) + self.assertAllClose(customized_swish_data, swish_data) + + def test_hard_swish(self): + features = [[.25, 0, -.25], [-1, -2, 3]] + customized_swish_data = activations.hard_swish(features) + swish_data = self._hard_swish_np(features) + self.assertAllClose(customized_swish_data, swish_data) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/base_config.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/base_config.py new file mode 100644 index 0000000..23a6790 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/base_config.py @@ -0,0 +1,323 @@ +# Lint as: python3 +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Base configurations to standardize experiments.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import copy +import functools +from typing import Any, List, Mapping, Optional, Type + +import dataclasses +import tensorflow as tf +import yaml + +from official.modeling.hyperparams import params_dict + + +@dataclasses.dataclass +class Config(params_dict.ParamsDict): + """The base configuration class that supports YAML/JSON based overrides. + + * It recursively enforces a whitelist of basic types and container types, so + it avoids surprises with copy and reuse caused by unanticipated types. + * It converts dict to Config even within sequences, + e.g. for config = Config({'key': [([{'a': 42}],)]), + type(config.key[0][0][0]) is Config rather than dict. + """ + + # It's safe to add bytes and other immutable types here. + IMMUTABLE_TYPES = (str, int, float, bool, type(None)) + # It's safe to add set, frozenset and other collections here. + SEQUENCE_TYPES = (list, tuple) + + default_params: dataclasses.InitVar[Optional[Mapping[str, Any]]] = None + restrictions: dataclasses.InitVar[Optional[List[str]]] = None + + @classmethod + def _isvalidsequence(cls, v): + """Check if the input values are valid sequences. + + Args: + v: Input sequence. + + Returns: + True if the sequence is valid. Valid sequence includes the sequence + type in cls.SEQUENCE_TYPES and element type is in cls.IMMUTABLE_TYPES or + is dict or ParamsDict. + """ + if not isinstance(v, cls.SEQUENCE_TYPES): + return False + return (all(isinstance(e, cls.IMMUTABLE_TYPES) for e in v) or + all(isinstance(e, dict) for e in v) or + all(isinstance(e, params_dict.ParamsDict) for e in v)) + + @classmethod + def _import_config(cls, v, subconfig_type): + """Returns v with dicts converted to Configs, recursively.""" + if not issubclass(subconfig_type, params_dict.ParamsDict): + raise TypeError( + 'Subconfig_type should be subclass of ParamsDict, found {!r}'.format( + subconfig_type)) + if isinstance(v, cls.IMMUTABLE_TYPES): + return v + elif isinstance(v, cls.SEQUENCE_TYPES): + # Only support one layer of sequence. + if not cls._isvalidsequence(v): + raise TypeError( + 'Invalid sequence: only supports single level {!r} of {!r} or ' + 'dict or ParamsDict found: {!r}'.format(cls.SEQUENCE_TYPES, + cls.IMMUTABLE_TYPES, v)) + import_fn = functools.partial( + cls._import_config, subconfig_type=subconfig_type) + return type(v)(map(import_fn, v)) + elif isinstance(v, params_dict.ParamsDict): + # Deepcopy here is a temporary solution for preserving type in nested + # Config object. + return copy.deepcopy(v) + elif isinstance(v, dict): + return subconfig_type(v) + else: + raise TypeError('Unknown type: {!r}'.format(type(v))) + + @classmethod + def _export_config(cls, v): + """Returns v with Configs converted to dicts, recursively.""" + if isinstance(v, cls.IMMUTABLE_TYPES): + return v + elif isinstance(v, cls.SEQUENCE_TYPES): + return type(v)(map(cls._export_config, v)) + elif isinstance(v, params_dict.ParamsDict): + return v.as_dict() + elif isinstance(v, dict): + raise TypeError('dict value not supported in converting.') + else: + raise TypeError('Unknown type: {!r}'.format(type(v))) + + @classmethod + def _get_subconfig_type(cls, k) -> Type[params_dict.ParamsDict]: + """Get element type by the field name. + + Args: + k: the key/name of the field. + + Returns: + Config as default. If a type annotation is found for `k`, + 1) returns the type of the annotation if it is subtype of ParamsDict; + 2) returns the element type if the annotation of `k` is List[SubType] + or Tuple[SubType]. + """ + subconfig_type = Config + if k in cls.__annotations__: + # Directly Config subtype. + type_annotation = cls.__annotations__[k] + if (isinstance(type_annotation, type) and + issubclass(type_annotation, Config)): + subconfig_type = cls.__annotations__[k] + else: + # Check if the field is a sequence of subtypes. + field_type = getattr(type_annotation, '__origin__', type(None)) + if (isinstance(field_type, type) and + issubclass(field_type, cls.SEQUENCE_TYPES)): + element_type = getattr(type_annotation, '__args__', [type(None)])[0] + subconfig_type = ( + element_type if issubclass(element_type, params_dict.ParamsDict) + else subconfig_type) + return subconfig_type + + def __post_init__(self, default_params, restrictions, *args, **kwargs): + super().__init__(default_params=default_params, + restrictions=restrictions, + *args, + **kwargs) + + def _set(self, k, v): + """Overrides same method in ParamsDict. + + Also called by ParamsDict methods. + + Args: + k: key to set. + v: value. + + Raises: + RuntimeError + """ + subconfig_type = self._get_subconfig_type(k) + if isinstance(v, dict): + if k not in self.__dict__ or not self.__dict__[k]: + # If the key not exist or the value is None, a new Config-family object + # sould be created for the key. + self.__dict__[k] = subconfig_type(v) + else: + self.__dict__[k].override(v) + else: + self.__dict__[k] = self._import_config(v, subconfig_type) + + def __setattr__(self, k, v): + if k not in self.RESERVED_ATTR: + if getattr(self, '_locked', False): + raise ValueError('The Config has been locked. ' 'No change is allowed.') + self._set(k, v) + + def _override(self, override_dict, is_strict=True): + """Overrides same method in ParamsDict. + + Also called by ParamsDict methods. + + Args: + override_dict: dictionary to write to . + is_strict: If True, not allows to add new keys. + + Raises: + KeyError: overriding reserved keys or keys not exist (is_strict=True). + """ + for k, v in sorted(override_dict.items()): + if k in self.RESERVED_ATTR: + raise KeyError('The key {!r} is internally reserved. ' + 'Can not be overridden.'.format(k)) + if k not in self.__dict__: + if is_strict: + raise KeyError('The key {!r} does not exist in {!r}. ' + 'To extend the existing keys, use ' + '`override` with `is_strict` = False.'.format( + k, type(self))) + else: + self._set(k, v) + else: + if isinstance(v, dict) and self.__dict__[k]: + self.__dict__[k]._override(v, is_strict) # pylint: disable=protected-access + elif isinstance(v, params_dict.ParamsDict) and self.__dict__[k]: + self.__dict__[k]._override(v.as_dict(), is_strict) # pylint: disable=protected-access + else: + self._set(k, v) + + def as_dict(self): + """Returns a dict representation of params_dict.ParamsDict. + + For the nested params_dict.ParamsDict, a nested dict will be returned. + """ + return { + k: self._export_config(v) + for k, v in self.__dict__.items() + if k not in self.RESERVED_ATTR + } + + def replace(self, **kwargs): + """Like `override`, but returns a copy with the current config unchanged.""" + params = self.__class__(self) + params.override(kwargs, is_strict=True) + return params + + @classmethod + def from_yaml(cls, file_path: str): + # Note: This only works if the Config has all default values. + with tf.io.gfile.GFile(file_path, 'r') as f: + loaded = yaml.load(f) + config = cls() + config.override(loaded) + return config + + @classmethod + def from_json(cls, file_path: str): + """Wrapper for `from_yaml`.""" + return cls.from_yaml(file_path) + + @classmethod + def from_args(cls, *args, **kwargs): + """Builds a config from the given list of arguments.""" + attributes = list(cls.__annotations__.keys()) + default_params = {a: p for a, p in zip(attributes, args)} + default_params.update(kwargs) + return cls(default_params) + + +@dataclasses.dataclass +class RuntimeConfig(Config): + """High-level configurations for Runtime. + + These include parameters that are not directly related to the experiment, + e.g. directories, accelerator type, etc. + + Attributes: + distribution_strategy: e.g. 'mirrored', 'tpu', etc. + enable_xla: Whether or not to enable XLA. + per_gpu_thread_count: thread count per GPU. + gpu_thread_mode: Whether and how the GPU device uses its own threadpool. + dataset_num_private_threads: Number of threads for a private threadpool + created for all datasets computation. + tpu: The address of the TPU to use, if any. + num_gpus: The number of GPUs to use, if any. + worker_hosts: comma-separated list of worker ip:port pairs for running + multi-worker models with DistributionStrategy. + task_index: If multi-worker training, the task index of this worker. + all_reduce_alg: Defines the algorithm for performing all-reduce. + num_packs: Sets `num_packs` in the cross device ops used in + MirroredStrategy. For details, see tf.distribute.NcclAllReduce. + loss_scale: The type of loss scale. This is used when setting the mixed + precision policy. + run_eagerly: Whether or not to run the experiment eagerly. + + """ + distribution_strategy: str = 'mirrored' + enable_xla: bool = False + gpu_thread_mode: Optional[str] = None + dataset_num_private_threads: Optional[int] = None + per_gpu_thread_count: int = 0 + tpu: Optional[str] = None + num_gpus: int = 0 + worker_hosts: Optional[str] = None + task_index: int = -1 + all_reduce_alg: Optional[str] = None + num_packs: int = 1 + loss_scale: Optional[str] = None + run_eagerly: bool = False + + +@dataclasses.dataclass +class TensorboardConfig(Config): + """Configuration for Tensorboard. + + Attributes: + track_lr: Whether or not to track the learning rate in Tensorboard. Defaults + to True. + write_model_weights: Whether or not to write the model weights as + images in Tensorboard. Defaults to False. + + """ + track_lr: bool = True + write_model_weights: bool = False + + +@dataclasses.dataclass +class CallbacksConfig(Config): + """Configuration for Callbacks. + + Attributes: + enable_checkpoint_and_export: Whether or not to enable checkpoints as a + Callback. Defaults to True. + enable_tensorboard: Whether or not to enable Tensorboard as a Callback. + Defaults to True. + enable_time_history: Whether or not to enable TimeHistory Callbacks. + Defaults to True. + + """ + enable_checkpoint_and_export: bool = True + enable_tensorboard: bool = True + enable_time_history: bool = True diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/base_config_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/base_config_test.py new file mode 100644 index 0000000..501f958 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/base_config_test.py @@ -0,0 +1,299 @@ +# Lint as: python3 +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import pprint +from typing import List, Tuple + +from absl.testing import parameterized +import dataclasses +import tensorflow as tf +from official.modeling.hyperparams import base_config + + +@dataclasses.dataclass +class DumpConfig1(base_config.Config): + a: int = 1 + b: str = 'text' + + +@dataclasses.dataclass +class DumpConfig2(base_config.Config): + c: int = 2 + d: str = 'text' + e: DumpConfig1 = DumpConfig1() + + +@dataclasses.dataclass +class DumpConfig3(DumpConfig2): + f: int = 2 + g: str = 'text' + h: List[DumpConfig1] = dataclasses.field( + default_factory=lambda: [DumpConfig1(), DumpConfig1()]) + g: Tuple[DumpConfig1, ...] = (DumpConfig1(),) + + +class BaseConfigTest(parameterized.TestCase, tf.test.TestCase): + + def assertHasSameTypes(self, c, d, msg=''): + """Checks if a Config has the same structure as a given dict. + + Args: + c: the Config object to be check. + d: the reference dict object. + msg: The error message to show when type mismatched. + """ + # Make sure d is not a Config. Assume d is either + # dictionary or primitive type and c is the Config or primitive types. + self.assertNotIsInstance(d, base_config.Config) + if isinstance(d, base_config.Config.IMMUTABLE_TYPES): + self.assertEqual(pprint.pformat(c), pprint.pformat(d), msg=msg) + elif isinstance(d, base_config.Config.SEQUENCE_TYPES): + self.assertEqual(type(c), type(d), msg=msg) + for i, v in enumerate(d): + self.assertHasSameTypes(c[i], v, msg='{}[{!r}]'.format(msg, i)) + elif isinstance(d, dict): + self.assertIsInstance(c, base_config.Config, msg=msg) + for k, v in sorted(d.items()): + self.assertHasSameTypes(getattr(c, k), v, msg='{}[{!r}]'.format(msg, k)) + else: + raise TypeError('Unknown type: %r' % type(d)) + + def assertImportExport(self, v): + config = base_config.Config({'key': v}) + back = config.as_dict()['key'] + self.assertEqual(pprint.pformat(back), pprint.pformat(v)) + self.assertHasSameTypes(config.key, v, msg='=%s v' % pprint.pformat(v)) + + def test_invalid_keys(self): + params = base_config.Config() + with self.assertRaises(AttributeError): + _ = params.a + + def test_nested_config_types(self): + config = DumpConfig3() + self.assertIsInstance(config.e, DumpConfig1) + self.assertIsInstance(config.h[0], DumpConfig1) + self.assertIsInstance(config.h[1], DumpConfig1) + self.assertIsInstance(config.g[0], DumpConfig1) + + config.override({'e': {'a': 2, 'b': 'new text'}}) + self.assertIsInstance(config.e, DumpConfig1) + self.assertEqual(config.e.a, 2) + self.assertEqual(config.e.b, 'new text') + + config.override({'h': [{'a': 3, 'b': 'new text 2'}]}) + self.assertIsInstance(config.h[0], DumpConfig1) + self.assertLen(config.h, 1) + self.assertEqual(config.h[0].a, 3) + self.assertEqual(config.h[0].b, 'new text 2') + + config.override({'g': [{'a': 4, 'b': 'new text 3'}]}) + self.assertIsInstance(config.g[0], DumpConfig1) + self.assertLen(config.g, 1) + self.assertEqual(config.g[0].a, 4) + self.assertEqual(config.g[0].b, 'new text 3') + + @parameterized.parameters( + ('_locked', "The key '_locked' is internally reserved."), + ('_restrictions', "The key '_restrictions' is internally reserved."), + ('aa', "The key 'aa' does not exist."), + ) + def test_key_error(self, key, msg): + params = base_config.Config() + with self.assertRaisesRegex(KeyError, msg): + params.override({key: True}) + + @parameterized.parameters( + ('str data',), + (123,), + (1.23,), + (None,), + (['str', 1, 2.3, None],), + (('str', 1, 2.3, None),), + ) + def test_import_export_immutable_types(self, v): + self.assertImportExport(v) + out = base_config.Config({'key': v}) + self.assertEqual(pprint.pformat(v), pprint.pformat(out.key)) + + def test_override_is_strict_true(self): + params = base_config.Config({ + 'a': 'aa', + 'b': 2, + 'c': { + 'c1': 'cc', + 'c2': 20 + } + }) + params.override({'a': 2, 'c': {'c1': 'ccc'}}, is_strict=True) + self.assertEqual(params.a, 2) + self.assertEqual(params.c.c1, 'ccc') + with self.assertRaises(KeyError): + params.override({'d': 'ddd'}, is_strict=True) + with self.assertRaises(KeyError): + params.override({'c': {'c3': 30}}, is_strict=True) + + config = base_config.Config({'key': [{'a': 42}]}) + config.override({'key': [{'b': 43}]}) + self.assertEqual(config.key[0].b, 43) + with self.assertRaisesRegex(AttributeError, 'The key `a` does not exist'): + _ = config.key[0].a + + @parameterized.parameters( + (lambda x: x, 'Unknown type'), + (object(), 'Unknown type'), + (set(), 'Unknown type'), + (frozenset(), 'Unknown type'), + ) + def test_import_unsupport_types(self, v, msg): + with self.assertRaisesRegex(TypeError, msg): + _ = base_config.Config({'key': v}) + + @parameterized.parameters( + ({ + 'a': [{ + 'b': 2, + }, { + 'c': 3, + }] + },), + ({ + 'c': [{ + 'f': 1.1, + }, { + 'h': [1, 2], + }] + },), + (({ + 'a': 'aa', + 'b': 2, + 'c': { + 'c1': 10, + 'c2': 20, + } + },),), + ) + def test_import_export_nested_structure(self, d): + self.assertImportExport(d) + + @parameterized.parameters( + ([{ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + }],), + (({ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + },),), + ) + def test_import_export_nested_sequences(self, v): + self.assertImportExport(v) + + @parameterized.parameters( + ([([{}],)],), + ([['str', 1, 2.3, None]],), + ((('str', 1, 2.3, None),),), + ([ + ('str', 1, 2.3, None), + ],), + ([ + ('str', 1, 2.3, None), + ],), + ([[{ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + }]],), + ([[[{ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + }]]],), + ((({ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + },),),), + (((({ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + },),),),), + ([({ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + },)],), + (([{ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + }],),), + ) + def test_import_export_unsupport_sequence(self, v): + with self.assertRaisesRegex(TypeError, + 'Invalid sequence: only supports single level'): + _ = base_config.Config({'key': v}) + + def test_construct_subtype(self): + pass + + def test_import_config(self): + params = base_config.Config({'a': [{'b': 2}, {'c': {'d': 3}}]}) + self.assertLen(params.a, 2) + self.assertEqual(params.a[0].b, 2) + self.assertEqual(type(params.a[0]), base_config.Config) + self.assertEqual(pprint.pformat(params.a[0].b), '2') + self.assertEqual(type(params.a[1]), base_config.Config) + self.assertEqual(type(params.a[1].c), base_config.Config) + self.assertEqual(pprint.pformat(params.a[1].c.d), '3') + + def test_override(self): + params = base_config.Config({'a': [{'b': 2}, {'c': {'d': 3}}]}) + params.override({'a': [{'b': 4}, {'c': {'d': 5}}]}, is_strict=False) + self.assertEqual(type(params.a), list) + self.assertEqual(type(params.a[0]), base_config.Config) + self.assertEqual(pprint.pformat(params.a[0].b), '4') + self.assertEqual(type(params.a[1]), base_config.Config) + self.assertEqual(type(params.a[1].c), base_config.Config) + self.assertEqual(pprint.pformat(params.a[1].c.d), '5') + + @parameterized.parameters( + ([{}],), + (({},),), + ) + def test_config_vs_params_dict(self, v): + d = {'key': v} + self.assertEqual(type(base_config.Config(d).key[0]), base_config.Config) + self.assertEqual(type(base_config.params_dict.ParamsDict(d).key[0]), dict) + + def test_ppformat(self): + self.assertEqual( + pprint.pformat([ + 's', 1, 1.0, True, None, {}, [], (), { + (2,): (3, [4], { + 6: 7, + }), + 8: 9, + } + ]), + "['s', 1, 1.0, True, None, {}, [], (), {8: 9, (2,): (3, [4], {6: 7})}]") + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/params_dict.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/params_dict.py new file mode 100644 index 0000000..9806a2b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/params_dict.py @@ -0,0 +1,410 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A parameter dictionary class which supports the nest structure.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import copy +import re + +import six +import tensorflow as tf +import yaml + +# regex pattern that matches on key-value pairs in a comma-separated +# key-value pair string. It splits each k-v pair on the = sign, and +# matches on values that are within single quotes, double quotes, single +# values (e.g. floats, ints, etc.), and a lists within brackets. +_PARAM_RE = re.compile(r""" + (?P[a-zA-Z][\w\.]*) # variable name: "var" or "x" + \s*=\s* + ((?P\'(.*?)\' # single quote + | + \"(.*?)\" # double quote + | + [^,\[]* # single value + | + \[[^\]]*\])) # list of values + ($|,\s*)""", re.VERBOSE) + + +class ParamsDict(object): + """A hyperparameter container class.""" + + RESERVED_ATTR = ['_locked', '_restrictions'] + + def __init__(self, default_params=None, restrictions=None): + """Instantiate a ParamsDict. + + Instantiate a ParamsDict given a set of default parameters and a list of + restrictions. Upon initialization, it validates itself by checking all the + defined restrictions, and raise error if it finds inconsistency. + + Args: + default_params: a Python dict or another ParamsDict object including the + default parameters to initialize. + restrictions: a list of strings, which define a list of restrictions to + ensure the consistency of different parameters internally. Each + restriction string is defined as a binary relation with a set of + operators, including {'==', '!=', '<', '<=', '>', '>='}. + """ + self._locked = False + self._restrictions = [] + if restrictions: + self._restrictions = restrictions + if default_params is None: + default_params = {} + self.override(default_params, is_strict=False) + self.validate() + + def _set(self, k, v): + if isinstance(v, dict): + self.__dict__[k] = ParamsDict(v) + else: + self.__dict__[k] = copy.deepcopy(v) + + def __setattr__(self, k, v): + """Sets the value of the existing key. + + Note that this does not allow directly defining a new key. Use the + `override` method with `is_strict=False` instead. + + Args: + k: the key string. + v: the value to be used to set the key `k`. + + Raises: + KeyError: if k is not defined in the ParamsDict. + """ + if k not in ParamsDict.RESERVED_ATTR: + if k not in self.__dict__.keys(): + raise KeyError('The key `%{}` does not exist. ' + 'To extend the existing keys, use ' + '`override` with `is_strict` = True.'.format(k)) + if self._locked: + raise ValueError('The ParamsDict has been locked. ' + 'No change is allowed.') + self._set(k, v) + + def __getattr__(self, k): + """Gets the value of the existing key. + + Args: + k: the key string. + + Returns: + the value of the key. + + Raises: + AttributeError: if k is not defined in the ParamsDict. + """ + if k not in self.__dict__.keys(): + raise AttributeError('The key `{}` does not exist. '.format(k)) + return self.__dict__[k] + + def __contains__(self, key): + """Implements the membership test operator.""" + return key in self.__dict__ + + def get(self, key, value=None): + """Accesses through built-in dictionary get method.""" + return self.__dict__.get(key, value) + + def override(self, override_params, is_strict=True): + """Override the ParamsDict with a set of given params. + + Args: + override_params: a dict or a ParamsDict specifying the parameters to + be overridden. + is_strict: a boolean specifying whether override is strict or not. If + True, keys in `override_params` must be present in the ParamsDict. + If False, keys in `override_params` can be different from what is + currently defined in the ParamsDict. In this case, the ParamsDict will + be extended to include the new keys. + """ + if self._locked: + raise ValueError('The ParamsDict has been locked. No change is allowed.') + if isinstance(override_params, ParamsDict): + override_params = override_params.as_dict() + self._override(override_params, is_strict) # pylint: disable=protected-access + + def _override(self, override_dict, is_strict=True): + """The implementation of `override`.""" + for k, v in six.iteritems(override_dict): + if k in ParamsDict.RESERVED_ATTR: + raise KeyError('The key `%{}` is internally reserved. ' + 'Can not be overridden.') + if k not in self.__dict__.keys(): + if is_strict: + raise KeyError('The key `{}` does not exist. ' + 'To extend the existing keys, use ' + '`override` with `is_strict` = False.'.format(k)) + else: + self._set(k, v) + else: + if isinstance(v, dict): + self.__dict__[k]._override(v, is_strict) # pylint: disable=protected-access + elif isinstance(v, ParamsDict): + self.__dict__[k]._override(v.as_dict(), is_strict) # pylint: disable=protected-access + else: + self.__dict__[k] = copy.deepcopy(v) + + def lock(self): + """Makes the ParamsDict immutable.""" + self._locked = True + + def as_dict(self): + """Returns a dict representation of ParamsDict. + + For the nested ParamsDict, a nested dict will be returned. + """ + params_dict = {} + for k, v in six.iteritems(self.__dict__): + if k not in ParamsDict.RESERVED_ATTR: + if isinstance(v, ParamsDict): + params_dict[k] = v.as_dict() + else: + params_dict[k] = copy.deepcopy(v) + return params_dict + + def validate(self): + """Validate the parameters consistency based on the restrictions. + + This method validates the internal consistency using the pre-defined list of + restrictions. A restriction is defined as a string which specfiies a binary + operation. The supported binary operations are {'==', '!=', '<', '<=', '>', + '>='}. Note that the meaning of these operators are consistent with the + underlying Python immplementation. Users should make sure the define + restrictions on their type make sense. + + For example, for a ParamsDict like the following + ``` + a: + a1: 1 + a2: 2 + b: + bb: + bb1: 10 + bb2: 20 + ccc: + a1: 1 + a3: 3 + ``` + one can define two restrictions like this + ['a.a1 == b.ccc.a1', 'a.a2 <= b.bb.bb2'] + + What it enforces are: + - a.a1 = 1 == b.ccc.a1 = 2 + - a.a2 = 2 <= b.bb.bb2 = 20 + + Raises: + KeyError: if any of the following happens + (1) any of parameters in any of restrictions is not defined in + ParamsDict, + (2) any inconsistency violating the restriction is found. + ValueError: if the restriction defined in the string is not supported. + """ + def _get_kv(dotted_string, params_dict): + tokenized_params = dotted_string.split('.') + v = params_dict + for t in tokenized_params: + v = v[t] + return tokenized_params[-1], v + + def _get_kvs(tokens, params_dict): + if len(tokens) != 2: + raise ValueError('Only support binary relation in restriction.') + stripped_tokens = [t.strip() for t in tokens] + left_k, left_v = _get_kv(stripped_tokens[0], params_dict) + right_k, right_v = _get_kv(stripped_tokens[1], params_dict) + return left_k, left_v, right_k, right_v + + params_dict = self.as_dict() + for restriction in self._restrictions: + if '==' in restriction: + tokens = restriction.split('==') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v != right_v: + raise KeyError('Found inconsistncy between key `{}` and key `{}`.' + .format(tokens[0], tokens[1])) + elif '!=' in restriction: + tokens = restriction.split('!=') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v == right_v: + raise KeyError('Found inconsistncy between key `{}` and key `{}`.' + .format(tokens[0], tokens[1])) + elif '<' in restriction: + tokens = restriction.split('<') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v >= right_v: + raise KeyError('Found inconsistncy between key `{}` and key `{}`.' + .format(tokens[0], tokens[1])) + elif '<=' in restriction: + tokens = restriction.split('<=') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v > right_v: + raise KeyError('Found inconsistncy between key `{}` and key `{}`.' + .format(tokens[0], tokens[1])) + elif '>' in restriction: + tokens = restriction.split('>') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v <= right_v: + raise KeyError('Found inconsistncy between key `{}` and key `{}`.' + .format(tokens[0], tokens[1])) + elif '>=' in restriction: + tokens = restriction.split('>=') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v < right_v: + raise KeyError('Found inconsistncy between key `{}` and key `{}`.' + .format(tokens[0], tokens[1])) + else: + raise ValueError('Unsupported relation in restriction.') + + +def read_yaml_to_params_dict(file_path): + """Reads a YAML file to a ParamsDict.""" + with tf.io.gfile.GFile(file_path, 'r') as f: + params_dict = yaml.load(f) + return ParamsDict(params_dict) + + +def save_params_dict_to_yaml(params, file_path): + """Saves the input ParamsDict to a YAML file.""" + with tf.io.gfile.GFile(file_path, 'w') as f: + + def _my_list_rep(dumper, data): + # u'tag:yaml.org,2002:seq' is the YAML internal tag for sequence. + return dumper.represent_sequence( + u'tag:yaml.org,2002:seq', data, flow_style=True) + yaml.add_representer(list, _my_list_rep) + yaml.dump(params.as_dict(), f, default_flow_style=False) + + +def nested_csv_str_to_json_str(csv_str): + """Converts a nested (using '.') comma-separated k=v string to a JSON string. + + Converts a comma-separated string of key/value pairs that supports + nesting of keys to a JSON string. Nesting is implemented using + '.' between levels for a given key. + + Spacing between commas and = is supported (e.g. there is no difference between + "a=1,b=2", "a = 1, b = 2", or "a=1, b=2") but there should be no spaces before + keys or after values (e.g. " a=1,b=2" and "a=1,b=2 " are not supported). + + Note that this will only support values supported by CSV, meaning + values such as nested lists (e.g. "a=[[1,2,3],[4,5,6]]") are not + supported. Strings are supported as well, e.g. "a='hello'". + + An example conversion would be: + + "a=1, b=2, c.a=2, c.b=3, d.a.a=5" + + to + + "{ a: 1, b : 2, c: {a : 2, b : 3}, d: {a: {a : 5}}}" + + Args: + csv_str: the comma separated string. + + Returns: + the converted JSON string. + + Raises: + ValueError: If csv_str is not in a comma separated string or + if the string is formatted incorrectly. + """ + if not csv_str: + return '' + + formatted_entries = [] + nested_map = collections.defaultdict(list) + pos = 0 + while pos < len(csv_str): + m = _PARAM_RE.match(csv_str, pos) + if not m: + raise ValueError('Malformed hyperparameter value while parsing ' + 'CSV string: %s' % csv_str[pos:]) + pos = m.end() + # Parse the values. + m_dict = m.groupdict() + name = m_dict['name'] + v = m_dict['val'] + + # If a GCS path (e.g. gs://...) is provided, wrap this in quotes + # as yaml.load would otherwise throw an exception + if re.match(r'(?=[^\"\'])(?=[gs://])', v): + v = '\'{}\''.format(v) + + name_nested = name.split('.') + if len(name_nested) > 1: + grouping = name_nested[0] + value = '.'.join(name_nested[1:]) + '=' + v + nested_map[grouping].append(value) + else: + formatted_entries.append('%s : %s' % (name, v)) + + for grouping, value in nested_map.items(): + value = ','.join(value) + value = nested_csv_str_to_json_str(value) + formatted_entries.append('%s : %s' % (grouping, value)) + return '{' + ', '.join(formatted_entries) + '}' + + +def override_params_dict(params, dict_or_string_or_yaml_file, is_strict): + """Override a given ParamsDict using a dict, JSON/YAML/CSV string or YAML file. + + The logic of the function is outlined below: + 1. Test that the input is a dict. If not, proceed to 2. + 2. Tests that the input is a string. If not, raise unknown ValueError + 2.1. Test if the string is in a CSV format. If so, parse. + If not, proceed to 2.2. + 2.2. Try loading the string as a YAML/JSON. If successful, parse to + dict and use it to override. If not, proceed to 2.3. + 2.3. Try using the string as a file path and load the YAML file. + + Args: + params: a ParamsDict object to be overridden. + dict_or_string_or_yaml_file: a Python dict, JSON/YAML/CSV string or + path to a YAML file specifying the parameters to be overridden. + is_strict: a boolean specifying whether override is strict or not. + + Returns: + params: the overridden ParamsDict object. + + Raises: + ValueError: if failed to override the parameters. + """ + if not dict_or_string_or_yaml_file: + return params + if isinstance(dict_or_string_or_yaml_file, dict): + params.override(dict_or_string_or_yaml_file, is_strict) + elif isinstance(dict_or_string_or_yaml_file, six.string_types): + try: + dict_or_string_or_yaml_file = ( + nested_csv_str_to_json_str(dict_or_string_or_yaml_file)) + except ValueError: + pass + params_dict = yaml.load(dict_or_string_or_yaml_file) + if isinstance(params_dict, dict): + params.override(params_dict, is_strict) + else: + with tf.io.gfile.GFile(dict_or_string_or_yaml_file) as f: + params.override(yaml.load(f), is_strict) + else: + raise ValueError('Unknown input type to parse.') + return params diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/params_dict_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/params_dict_test.py new file mode 100644 index 0000000..3d53ea1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/hyperparams/params_dict_test.py @@ -0,0 +1,322 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Tests for official.modeling.hyperparams.params_dict.py.""" + +import os + +import tensorflow as tf +import yaml + +from official.modeling.hyperparams import params_dict + + +class ParamsDictTest(tf.test.TestCase): + + def test_init_from_an_empty_dict(self): + params = params_dict.ParamsDict() + with self.assertRaises(AttributeError): + _ = params.a + + with self.assertRaises(KeyError): + params.a = 'aa' + + def test_init_from_a_dict(self): + params = params_dict.ParamsDict({'a': 'aa', 'b': 2}) + self.assertEqual(params.a, 'aa') + self.assertEqual(params.b, 2) + + def test_init_from_a_param_dict(self): + params_init = params_dict.ParamsDict({'a': 'aa', 'b': 2}) + params = params_dict.ParamsDict(params_init) + self.assertEqual(params.a, 'aa') + self.assertEqual(params.b, 2) + + def test_lock(self): + params = params_dict.ParamsDict({'a': 1, 'b': 2}) + params.lock() + with self.assertRaises(ValueError): + params.a = 10 + with self.assertRaises(ValueError): + params.override({'b': 20}) + + def test_setattr(self): + params = params_dict.ParamsDict() + params.override( + {'a': 'aa', 'b': 2, 'c': None}, is_strict=False) + params.c = 'ccc' + self.assertEqual(params.a, 'aa') + self.assertEqual(params.b, 2) + self.assertEqual(params.c, 'ccc') + + def test_getattr(self): + params = params_dict.ParamsDict() + params.override( + {'a': 'aa', 'b': 2, 'c': None}, is_strict=False) + self.assertEqual(params.a, 'aa') + self.assertEqual(params.b, 2) + self.assertEqual(params.c, None) + + def test_contains(self): + params = params_dict.ParamsDict() + params.override( + {'a': 'aa'}, is_strict=False) + self.assertIn('a', params) + self.assertNotIn('b', params) + + def test_get(self): + params = params_dict.ParamsDict() + params.override( + {'a': 'aa'}, is_strict=False) + self.assertEqual(params.get('a'), 'aa') + self.assertEqual(params.get('b', 2), 2) + self.assertEqual(params.get('b'), None) + + def test_override_is_strict_true(self): + params = params_dict.ParamsDict( + {'a': 'aa', 'b': 2, 'c': {'c1': 'cc', 'c2': 20}}) + params.override({'a': 2, 'c': {'c1': 'ccc'}}, is_strict=True) + self.assertEqual(params.a, 2) + self.assertEqual(params.c.c1, 'ccc') + with self.assertRaises(KeyError): + params.override({'d': 'ddd'}, is_strict=True) + with self.assertRaises(KeyError): + params.override({'c': {'c3': 30}}, is_strict=True) + + def test_override_is_strict_false(self): + params = params_dict.ParamsDict( + {'a': 'aa', 'b': 2, 'c': {'c1': 10, 'c2': 20}}) + params.override({'a': 2, 'c': {'c3': 3000}}, is_strict=False) + self.assertEqual(params.a, 2) + self.assertEqual(params.c.c3, 3000) + params.override({'d': 'ddd'}, is_strict=False) + self.assertEqual(params.d, 'ddd') + params.override({'c': {'c4': 4444}}, is_strict=False) + self.assertEqual(params.c.c4, 4444) + + def test_as_dict(self): + params = params_dict.ParamsDict( + {'a': 'aa', 'b': 2, 'c': {'c1': 10, 'c2': 20}}) + params_d = params.as_dict() + self.assertEqual(params_d['a'], 'aa') + self.assertEqual(params_d['b'], 2) + self.assertEqual(params_d['c']['c1'], 10) + self.assertEqual(params_d['c']['c2'], 20) + + def test_validate(self): + # Raise error due to the unknown parameter. + with self.assertRaises(KeyError): + params = params_dict.ParamsDict( + {'a': 1, 'b': {'a': 11}}, ['a == c']) + + # OK to check equality of two nested dicts. + params = params_dict.ParamsDict( + {'a': 1, 'b': {'a': 10}, 'c': {'a': 10}}, ['b == c']) + + # Raise error due to inconsistency + with self.assertRaises(KeyError): + params = params_dict.ParamsDict( + {'a': 1, 'c': {'a': 10}}, ['a == c.a']) + + # Valid rule. + params = params_dict.ParamsDict( + {'a': 1, 'c': {'a': 1}}, ['a == c.a']) + + # Overridding violates the existing rule, raise error upon validate. + params.override({'a': 11}) + with self.assertRaises(KeyError): + params.validate() + + +class ParamsDictIOTest(tf.test.TestCase): + + def write_temp_file(self, filename, text): + temp_file = os.path.join(self.get_temp_dir(), filename) + with tf.io.gfile.GFile(temp_file, 'w') as writer: + writer.write(text) + return temp_file + + def test_save_params_dict_to_yaml(self): + params = params_dict.ParamsDict( + {'a': 'aa', 'b': 2, 'c': {'c1': 10, 'c2': 20}}) + output_yaml_file = os.path.join(self.get_temp_dir(), 'params.yaml') + params_dict.save_params_dict_to_yaml(params, output_yaml_file) + + with tf.io.gfile.GFile(output_yaml_file, 'r') as f: + params_d = yaml.load(f) + self.assertEqual(params.a, params_d['a']) + self.assertEqual(params.b, params_d['b']) + self.assertEqual(params.c.c1, params_d['c']['c1']) + self.assertEqual(params.c.c2, params_d['c']['c2']) + + def test_read_yaml_to_params_dict(self): + input_yaml_file = self.write_temp_file( + 'params.yaml', r""" + a: 'aa' + b: 2 + c: + c1: 10 + c2: 20 + """) + params = params_dict.read_yaml_to_params_dict(input_yaml_file) + + self.assertEqual(params.a, 'aa') + self.assertEqual(params.b, 2) + self.assertEqual(params.c.c1, 10) + self.assertEqual(params.c.c2, 20) + + def test_override_params_dict_using_dict(self): + params = params_dict.ParamsDict({ + 'a': 1, 'b': 2.5, 'c': [3, 4], 'd': 'hello', 'e': False}) + override_dict = {'b': 5.2, 'c': [30, 40]} + params = params_dict.override_params_dict( + params, override_dict, is_strict=True) + self.assertEqual(1, params.a) + self.assertEqual(5.2, params.b) + self.assertEqual([30, 40], params.c) + self.assertEqual('hello', params.d) + self.assertEqual(False, params.e) + + def test_override_params_dict_using_yaml_string(self): + params = params_dict.ParamsDict({ + 'a': 1, 'b': 2.5, 'c': [3, 4], 'd': 'hello', 'e': False}) + override_yaml_string = "'b': 5.2\n'c': [30, 40]" + params = params_dict.override_params_dict( + params, override_yaml_string, is_strict=True) + self.assertEqual(1, params.a) + self.assertEqual(5.2, params.b) + self.assertEqual([30, 40], params.c) + self.assertEqual('hello', params.d) + self.assertEqual(False, params.e) + + def test_override_params_dict_using_json_string(self): + params = params_dict.ParamsDict({ + 'a': 1, 'b': {'b1': 2, 'b2': [2, 3],}, + 'd': {'d1': {'d2': 'hello'}}, 'e': False}) + override_json_string = "{ b: { b2: [3, 4] }, d: { d1: { d2: 'hi' } } }" + params = params_dict.override_params_dict( + params, override_json_string, is_strict=True) + self.assertEqual(1, params.a) + self.assertEqual(2, params.b.b1) + self.assertEqual([3, 4], params.b.b2) + self.assertEqual('hi', params.d.d1.d2) + self.assertEqual(False, params.e) + + def test_override_params_dict_using_csv_string(self): + params = params_dict.ParamsDict({ + 'a': 1, 'b': {'b1': 2, 'b2': [2, 3],}, + 'd': {'d1': {'d2': 'hello'}}, 'e': False}) + override_csv_string = "b.b2=[3,4], d.d1.d2='hi, world', e=gs://test" + params = params_dict.override_params_dict( + params, override_csv_string, is_strict=True) + self.assertEqual(1, params.a) + self.assertEqual(2, params.b.b1) + self.assertEqual([3, 4], params.b.b2) + self.assertEqual('hi, world', params.d.d1.d2) + self.assertEqual('gs://test', params.e) + + def test_override_params_dict_using_yaml_file(self): + params = params_dict.ParamsDict({ + 'a': 1, 'b': 2.5, 'c': [3, 4], 'd': 'hello', 'e': False}) + override_yaml_file = self.write_temp_file( + 'params.yaml', r""" + b: 5.2 + c: [30, 40] + """) + params = params_dict.override_params_dict( + params, override_yaml_file, is_strict=True) + self.assertEqual(1, params.a) + self.assertEqual(5.2, params.b) + self.assertEqual([30, 40], params.c) + self.assertEqual('hello', params.d) + self.assertEqual(False, params.e) + + +class IOTest(tf.test.TestCase): + + def test_basic_csv_str_to_json_str(self): + csv_str = 'a=1,b=2,c=3' + json_str = '{a : 1, b : 2, c : 3}' + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + self.assertEqual(converted_csv_str, json_str) + + def test_basic_csv_str_load(self): + csv_str = 'a=1,b=2,c=3' + expected_output = {'a': 1, 'b': 2, 'c': 3} + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + converted_dict = yaml.load(converted_csv_str) + self.assertDictEqual(converted_dict, expected_output) + + def test_basic_nested_csv_str_to_json_str(self): + csv_str = 'a=1,b.b1=2' + json_str = '{a : 1, b : {b1 : 2}}' + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + self.assertEqual(converted_csv_str, json_str) + + def test_basic_nested_csv_str_load(self): + csv_str = 'a=1,b.b1=2,c.c1=3' + expected_output = {'a': 1, 'b': {'b1': 2}, 'c': {'c1': 3}} + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + converted_dict = yaml.load(converted_csv_str) + self.assertDictEqual(converted_dict, expected_output) + + def test_complex_nested_csv_str_to_json_str(self): + csv_str = 'a.aa.aaa.aaaaa.a=1' + json_str = '{a : {aa : {aaa : {aaaaa : {a : 1}}}}}' + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + self.assertEqual(converted_csv_str, json_str) + + def test_complex_nested_csv_str_load(self): + csv_str = 'a.aa.aaa.aaaaa.a=1,a.a=2' + expected_output = {'a': {'aa': {'aaa': {'aaaaa': {'a': 1}}}, 'a': 2}} + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + converted_dict = yaml.load(converted_csv_str) + self.assertDictEqual(converted_dict, expected_output) + + def test_csv_str_load_supported_datatypes(self): + csv_str = 'a=1,b=2.,c=[1,2,3],d=\'hello, there\',e=\"Hi.\"' + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + converted_dict = yaml.load(converted_csv_str) + self.assertEqual(converted_dict['a'], 1) + self.assertEqual(converted_dict['b'], 2.) + self.assertEqual(converted_dict['c'], [1, 2, 3]) + self.assertEqual(converted_dict['d'], 'hello, there') + self.assertEqual(converted_dict['e'], 'Hi.') + + def test_csv_str_load_unsupported_datatypes(self): + csv_str = 'a=[[1,2,3],[4,5,6]]' + self.assertRaises(ValueError, + params_dict.nested_csv_str_to_json_str, + csv_str) + + def test_csv_str_to_json_str_spacing(self): + csv_str1 = 'a=1,b=2,c=3' + csv_str2 = 'a = 1, b = 2, c = 3' + json_str = '{a : 1, b : 2, c : 3}' + converted_csv_str1 = params_dict.nested_csv_str_to_json_str(csv_str1) + converted_csv_str2 = params_dict.nested_csv_str_to_json_str(csv_str2) + self.assertEqual(converted_csv_str1, converted_csv_str2) + self.assertEqual(converted_csv_str1, json_str) + self.assertEqual(converted_csv_str2, json_str) + + def test_gcs_added_quotes(self): + csv_str = 'a=gs://abc, b=gs://def' + expected_output = '{a : \'gs://abc\', b : \'gs://def\'}' + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + self.assertEqual(converted_csv_str, expected_output) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/performance.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/performance.py new file mode 100644 index 0000000..4b264f5 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/performance.py @@ -0,0 +1,56 @@ +# Lint as: python3 +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functions and classes related to training performance.""" + +import tensorflow as tf + + +def configure_optimizer(optimizer, + use_float16=False, + use_graph_rewrite=False, + loss_scale="dynamic"): + """Configures optimizer object with performance options.""" + if use_float16: + # Wraps optimizer with a LossScaleOptimizer. This is done automatically + # in compile() with the "mixed_float16" policy, but since we do not call + # compile(), we must wrap the optimizer manually. + optimizer = ( + tf.keras.mixed_precision.experimental.LossScaleOptimizer( + optimizer, loss_scale=loss_scale)) + if use_graph_rewrite: + # Note: the model dtype must be 'float32', which will ensure + # tf.ckeras.mixed_precision and + # tf.train.experimental.enable_mixed_precision_graph_rewrite do not double + # up. + optimizer = tf.train.experimental.enable_mixed_precision_graph_rewrite( + optimizer) + return optimizer + + +def set_mixed_precision_policy(dtype, loss_scale=None): + """Sets mix precision policy.""" + if dtype == tf.float16: + policy = tf.keras.mixed_precision.experimental.Policy( + 'mixed_float16', loss_scale=loss_scale) + tf.keras.mixed_precision.experimental.set_policy(policy) + elif dtype == tf.bfloat16: + policy = tf.keras.mixed_precision.experimental.Policy( + 'mixed_bfloat16') + tf.keras.mixed_precision.experimental.set_policy(policy) + elif dtype == tf.float32: + tf.keras.mixed_precision.experimental.set_policy('float32') + else: + raise ValueError("Unexpected dtype: %s" % dtype) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/tf_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/tf_utils.py new file mode 100644 index 0000000..34f8f66 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/tf_utils.py @@ -0,0 +1,175 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Common TF utilities.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import six +import tensorflow as tf + +from tensorflow.python.util import deprecation +from official.modeling import activations + + +@deprecation.deprecated( + None, + "tf.keras.layers.Layer supports multiple positional args and kwargs as " + "input tensors. pack/unpack inputs to override __call__ is no longer " + "needed." +) +def pack_inputs(inputs): + """Pack a list of `inputs` tensors to a tuple. + + Args: + inputs: a list of tensors. + + Returns: + a tuple of tensors. if any input is None, replace it with a special constant + tensor. + """ + inputs = tf.nest.flatten(inputs) + outputs = [] + for x in inputs: + if x is None: + outputs.append(tf.constant(0, shape=[], dtype=tf.int32)) + else: + outputs.append(x) + return tuple(outputs) + + +@deprecation.deprecated( + None, + "tf.keras.layers.Layer supports multiple positional args and kwargs as " + "input tensors. pack/unpack inputs to override __call__ is no longer " + "needed." +) +def unpack_inputs(inputs): + """unpack a tuple of `inputs` tensors to a tuple. + + Args: + inputs: a list of tensors. + + Returns: + a tuple of tensors. if any input is a special constant tensor, replace it + with None. + """ + inputs = tf.nest.flatten(inputs) + outputs = [] + for x in inputs: + if is_special_none_tensor(x): + outputs.append(None) + else: + outputs.append(x) + x = tuple(outputs) + + # To trick the very pointless 'unbalanced-tuple-unpacking' pylint check + # from triggering. + if len(x) == 1: + return x[0] + return tuple(outputs) + + +def is_special_none_tensor(tensor): + """Checks if a tensor is a special None Tensor.""" + return tensor.shape.ndims == 0 and tensor.dtype == tf.int32 + + +# TODO(hongkuny): consider moving custom string-map lookup to keras api. +def get_activation(identifier): + """Maps a identifier to a Python function, e.g., "relu" => `tf.nn.relu`. + + It checks string first and if it is one of customized activation not in TF, + the corresponding activation will be returned. For non-customized activation + names and callable identifiers, always fallback to tf.keras.activations.get. + + Args: + identifier: String name of the activation function or callable. + + Returns: + A Python function corresponding to the activation function. + """ + if isinstance(identifier, six.string_types): + name_to_fn = { + "gelu": activations.gelu, + "simple_swish": activations.simple_swish, + "hard_swish": activations.hard_swish, + "identity": activations.identity, + } + identifier = str(identifier).lower() + if identifier in name_to_fn: + return tf.keras.activations.get(name_to_fn[identifier]) + return tf.keras.activations.get(identifier) + + +def get_shape_list(tensor, expected_rank=None, name=None): + """Returns a list of the shape of tensor, preferring static dimensions. + + Args: + tensor: A tf.Tensor object to find the shape of. + expected_rank: (optional) int. The expected rank of `tensor`. If this is + specified and the `tensor` has a different rank, and exception will be + thrown. + name: Optional name of the tensor for the error message. + + Returns: + A list of dimensions of the shape of tensor. All static dimensions will + be returned as python integers, and dynamic dimensions will be returned + as tf.Tensor scalars. + """ + if expected_rank is not None: + assert_rank(tensor, expected_rank, name) + + shape = tensor.shape.as_list() + + non_static_indexes = [] + for (index, dim) in enumerate(shape): + if dim is None: + non_static_indexes.append(index) + + if not non_static_indexes: + return shape + + dyn_shape = tf.shape(tensor) + for index in non_static_indexes: + shape[index] = dyn_shape[index] + return shape + + +def assert_rank(tensor, expected_rank, name=None): + """Raises an exception if the tensor rank is not of the expected rank. + + Args: + tensor: A tf.Tensor to check the rank of. + expected_rank: Python integer or list of integers, expected rank. + name: Optional name of the tensor for the error message. + + Raises: + ValueError: If the expected shape doesn't match the actual shape. + """ + expected_rank_dict = {} + if isinstance(expected_rank, six.integer_types): + expected_rank_dict[expected_rank] = True + else: + for x in expected_rank: + expected_rank_dict[x] = True + + actual_rank = tensor.shape.ndims + if actual_rank not in expected_rank_dict: + raise ValueError( + "For the tensor `%s`, the actual tensor rank `%d` (shape = %s) is not " + "equal to the expected tensor rank `%s`" % + (name, actual_rank, str(tensor.shape), str(expected_rank))) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/training/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/training/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/training/distributed_executor.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/training/distributed_executor.py new file mode 100644 index 0000000..e44178c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/modeling/training/distributed_executor.py @@ -0,0 +1,759 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Custom training loop for running TensorFlow 2.0 models.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import os + +from absl import flags +from absl import logging + +import numpy as np +import tensorflow as tf + +# pylint: disable=unused-import,g-import-not-at-top,redefined-outer-name,reimported +from typing import Optional, Dict, List, Text, Callable, Union, Iterator, Any +from official.modeling.hyperparams import params_dict +from official.utils.misc import distribution_utils +from official.utils.misc import keras_utils +from official.utils import hyperparams_flags + +FLAGS = flags.FLAGS + +strategy_flags_dict = hyperparams_flags.strategy_flags_dict +hparam_flags_dict = hyperparams_flags.hparam_flags_dict + + +def _save_checkpoint(checkpoint, model_dir, checkpoint_prefix): + """Saves model to model_dir with provided checkpoint prefix.""" + + checkpoint_path = os.path.join(model_dir, checkpoint_prefix) + saved_path = checkpoint.save(checkpoint_path) + logging.info('Saving model as TF checkpoint: %s', saved_path) + + +def _steps_to_run(current_step, total_steps, steps_per_loop): + """Calculates steps to run on device.""" + if steps_per_loop <= 0: + raise ValueError('steps_per_loop should be positive integer.') + return min(total_steps - current_step, steps_per_loop) + + +def _no_metric(): + return None + + +class SummaryWriter(object): + """Simple SummaryWriter for writing dictionary of metrics. + + Attributes: + writer: The tf.SummaryWriter. + """ + + def __init__(self, model_dir: Text, name: Text): + """Inits SummaryWriter with paths. + + Arguments: + model_dir: the model folder path. + name: the summary subfolder name. + """ + self.writer = tf.summary.create_file_writer(os.path.join(model_dir, name)) + + def __call__(self, metrics: Union[Dict[Text, float], float], step: int): + """Write metrics to summary with the given writer. + + Args: + metrics: a dictionary of metrics values. Prefer dictionary. + step: integer. The training step. + """ + if not isinstance(metrics, dict): + # Support scalar metric without name. + logging.warning('Warning: summary writer prefer metrics as dictionary.') + metrics = {'metric': metrics} + + with self.writer.as_default(): + for k, v in metrics.items(): + tf.summary.scalar(k, v, step=step) + self.writer.flush() + + +class DistributedExecutor(object): + """Interface to train and eval models with tf.distribute.Strategy. + + Arguments: + strategy: an instance of tf.distribute.Strategy. + params: Model configuration needed to run distribution strategy. + model_fn: Keras model function. Signature: + (params: ParamsDict) -> tf.keras.models.Model. + loss_fn: loss function. Signature: + (y_true: Tensor, y_pred: Tensor) -> Tensor + metric_fn: metric function. Signature: () -> tf.keras.metrics.Metric. + is_multi_host: Set to True when using multi hosts for training, like multi + worker GPU or TPU pod (slice). Otherwise, False. + """ + + def __init__(self, + strategy, + params, + model_fn, + loss_fn, + is_multi_host=False): + + self._params = params + self._model_fn = model_fn + self._loss_fn = loss_fn + self._strategy = strategy + self._checkpoint_name = 'ctl_step_{step}.ckpt' + self._is_multi_host = is_multi_host + self.train_summary_writer = None + self.eval_summary_writer = None + self.global_train_step = None + + @property + def checkpoint_name(self): + """Returns default checkpoint name.""" + return self._checkpoint_name + + @checkpoint_name.setter + def checkpoint_name(self, name): + """Sets default summary writer for the current thread.""" + self._checkpoint_name = name + + def loss_fn(self): + return self._loss_fn() + + def model_fn(self, params): + return self._model_fn(params) + + def _save_config(self, model_dir): + """Save parameters to config files if model_dir is defined.""" + + logging.info('Save config to model_dir %s.', model_dir) + if model_dir: + if not tf.io.gfile.exists(model_dir): + tf.io.gfile.makedirs(model_dir) + self._params.lock() + params_dict.save_params_dict_to_yaml(self._params, + model_dir + '/params.yaml') + else: + logging.warning('model_dir is empty, so skip the save config.') + + def _get_input_iterator( + self, input_fn: Callable[..., tf.data.Dataset], + strategy: tf.distribute.Strategy) -> Optional[Iterator[Any]]: + """Returns distributed dataset iterator. + + Args: + input_fn: (params: dict) -> tf.data.Dataset. + strategy: an instance of tf.distribute.Strategy. + + Returns: + An iterator that yields input tensors. + """ + + if input_fn is None: + return None + # When training with multiple TPU workers, datasets needs to be cloned + # across workers. Since Dataset instance cannot be cloned in eager mode, + # we instead pass callable that returns a dataset. + if self._is_multi_host: + return iter( + strategy.experimental_distribute_datasets_from_function(input_fn)) + else: + input_data = input_fn() + return iter(strategy.experimental_distribute_dataset(input_data)) + + def _create_replicated_step(self, + strategy, + model, + loss_fn, + optimizer, + metric=None): + + def _replicated_step(inputs): + """Replicated training step.""" + inputs, labels = inputs + + with tf.GradientTape() as tape: + outputs = model(inputs, training=True) + prediction_loss = loss_fn(labels, outputs) + loss = tf.reduce_mean(prediction_loss) + loss = loss / strategy.num_replicas_in_sync + if isinstance(metric, tf.keras.metrics.Metric): + metric.update_state(labels, outputs) + else: + logging.error('train metric is not an instance of ' + 'tf.keras.metrics.Metric.') + + grads = tape.gradient(loss, model.trainable_variables) + optimizer.apply_gradients(zip(grads, model.trainable_variables)) + return loss + + return _replicated_step + + def _create_train_step(self, + strategy, + model, + loss_fn, + optimizer, + metric=None): + """Creates a distributed training step. + + Args: + strategy: an instance of tf.distribute.Strategy. + model: (Tensor, bool) -> Tensor. model function. + loss_fn: (y_true: Tensor, y_pred: Tensor) -> Tensor. + optimizer: tf.keras.optimizers.Optimizer. + iterator: an iterator that yields input tensors. + metric: tf.keras.metrics.Metric subclass. + + Returns: + The training step callable. + """ + _replicated_step = self._create_replicated_step(strategy, model, loss_fn, + optimizer, metric) + + @tf.function + def train_step(iterator, num_steps): + """Performs a distributed training step. + + Args: + iterator: an iterator that yields input tensors. + + Returns: + The loss tensor. + """ + if not isinstance(num_steps, tf.Tensor): + raise ValueError('steps should be an Tensor. Python object may cause ' + 'retracing.') + + per_replica_losses = strategy.run( + _replicated_step, args=(next(iterator),)) + for _ in tf.range(num_steps - 1): + per_replica_losses = strategy.run( + _replicated_step, args=(next(iterator),)) + + # For reporting, we returns the mean of losses. + losses = tf.nest.map_structure( + lambda x: strategy.reduce(tf.distribute.ReduceOp.MEAN, x, axis=None), + per_replica_losses) + return losses + + return train_step + + def _create_test_step(self, strategy, model, metric): + """Creates a distributed test step.""" + + @tf.function + def test_step(iterator): + """Calculates evaluation metrics on distributed devices.""" + if not metric: + logging.info('Skip test_step because metric is None (%s)', metric) + return None, None + if not isinstance(metric, tf.keras.metrics.Metric): + raise ValueError( + 'Metric must be an instance of tf.keras.metrics.Metric ' + 'for running in test_step. Actual {}'.format(metric)) + + def _test_step_fn(inputs): + """Replicated accuracy calculation.""" + inputs, labels = inputs + model_outputs = model(inputs, training=False) + metric.update_state(labels, model_outputs) + return labels, model_outputs + + return strategy.run(_test_step_fn, args=(next(iterator),)) + + return test_step + + def train(self, + train_input_fn: Callable[[params_dict.ParamsDict], tf.data.Dataset], + eval_input_fn: Callable[[params_dict.ParamsDict], + tf.data.Dataset] = None, + model_dir: Text = None, + total_steps: int = 1, + iterations_per_loop: int = 1, + train_metric_fn: Callable[[], Any] = None, + eval_metric_fn: Callable[[], Any] = None, + summary_writer_fn: Callable[[Text, Text], + SummaryWriter] = SummaryWriter, + init_checkpoint: Callable[[tf.keras.Model], Any] = None, + custom_callbacks: List[tf.keras.callbacks.Callback] = None, + save_config: bool = True): + """Runs distributed training. + + Args: + train_input_fn: (params: dict) -> tf.data.Dataset training data input + function. + eval_input_fn: (Optional) same type as train_input_fn. If not None, will + trigger evaluting metric on eval data. If None, will not run eval step. + model_dir: the folder path for model checkpoints. + total_steps: total training steps. + iterations_per_loop: train steps per loop. After each loop, this job will + update metrics like loss and save checkpoint. + train_metric_fn: metric_fn for evaluation in train_step. + eval_metric_fn: metric_fn for evaluation in test_step. + summary_writer_fn: function to create summary writer. + init_checkpoint: function to load checkpoint. + custom_callbacks: A list of Keras Callbacks objects to run during + training. More specifically, `on_batch_begin()`, `on_batch_end()`, + methods are invoked during training. + save_config: bool. Whether to save params to model_dir. + + Returns: + The training loss and eval metrics. + """ + assert train_input_fn is not None + if train_metric_fn and not callable(train_metric_fn): + raise ValueError('if `train_metric_fn` is specified, ' + 'train_metric_fn must be a callable.') + if eval_metric_fn and not callable(eval_metric_fn): + raise ValueError('if `eval_metric_fn` is specified, ' + 'eval_metric_fn must be a callable.') + train_metric_fn = train_metric_fn or _no_metric + eval_metric_fn = eval_metric_fn or _no_metric + + if custom_callbacks and iterations_per_loop != 1: + logging.warning( + 'It is sematically wrong to run callbacks when ' + 'iterations_per_loop is not one (%s)', iterations_per_loop) + + custom_callbacks = custom_callbacks or [] + + def _run_callbacks_on_batch_begin(batch): + """Runs custom callbacks at the start of every step.""" + if not custom_callbacks: + return + for callback in custom_callbacks: + if callback: + callback.on_batch_begin(batch) + + def _run_callbacks_on_batch_end(batch): + """Runs custom callbacks at the end of every step.""" + if not custom_callbacks: + return + for callback in custom_callbacks: + if callback: + callback.on_batch_end(batch) + + if save_config: + self._save_config(model_dir) + + if FLAGS.save_checkpoint_freq: + save_freq = FLAGS.save_checkpoint_freq + else: + save_freq = iterations_per_loop + + params = self._params + strategy = self._strategy + # To reduce unnecessary send/receive input pipeline operation, we place + # input pipeline ops in worker task. + train_iterator = self._get_input_iterator(train_input_fn, strategy) + train_loss = None + eval_metric_result = None + with strategy.scope(): + # To correctly place the model weights on accelerators, + # model and optimizer should be created in scope. + model = self.model_fn(params.as_dict()) + if not hasattr(model, 'optimizer'): + raise ValueError('User should set optimizer attribute to model ' + 'inside `model_fn`.') + optimizer = model.optimizer + + # Training loop starts here. + checkpoint = tf.train.Checkpoint(model=model, optimizer=optimizer) + latest_checkpoint_file = tf.train.latest_checkpoint(model_dir) + initial_step = 0 + if latest_checkpoint_file: + logging.info( + 'Checkpoint file %s found and restoring from ' + 'checkpoint', latest_checkpoint_file) + checkpoint.restore(latest_checkpoint_file) + initial_step = optimizer.iterations.numpy() + logging.info('Loading from checkpoint file completed. Init step %d', + initial_step) + elif init_checkpoint: + logging.info('Restoring from init checkpoint function') + init_checkpoint(model) + logging.info('Loading from init checkpoint file completed') + + current_step = optimizer.iterations.numpy() + checkpoint_name = self.checkpoint_name + + eval_metric = eval_metric_fn() + train_metric = train_metric_fn() + train_summary_writer = summary_writer_fn(model_dir, 'eval_train') + self.train_summary_writer = train_summary_writer.writer + + test_summary_writer = summary_writer_fn(model_dir, 'eval_test') + self.eval_summary_writer = test_summary_writer.writer + + # Use training summary writer in TimeHistory if it's in use + for cb in custom_callbacks: + if isinstance(cb, keras_utils.TimeHistory): + cb.summary_writer = self.train_summary_writer + + # Continue training loop. + train_step = self._create_train_step( + strategy=strategy, + model=model, + loss_fn=self.loss_fn(), + optimizer=optimizer, + metric=train_metric) + test_step = None + if eval_input_fn and eval_metric: + self.global_train_step = model.optimizer.iterations + test_step = self._create_test_step(strategy, model, metric=eval_metric) + + # Step-0 operations + _save_checkpoint( + checkpoint, model_dir, checkpoint_name.format(step=current_step)) + if test_step: + eval_iterator = self._get_input_iterator(eval_input_fn, strategy) + eval_metric_result = self._run_evaluation( + test_step, current_step, eval_metric, eval_iterator) + logging.info( + 'Step: %s evalation metric = %s.', current_step, eval_metric_result) + test_summary_writer( + metrics=eval_metric_result, step=optimizer.iterations) + eval_metric.reset_states() + + logging.info('Training started') + last_save_checkpoint_step = current_step + while current_step < total_steps: + + num_steps = _steps_to_run(current_step, total_steps, iterations_per_loop) + _run_callbacks_on_batch_begin(current_step) + train_loss = train_step(train_iterator, + tf.convert_to_tensor(num_steps, dtype=tf.int32)) + current_step += num_steps + + train_loss = tf.nest.map_structure(lambda x: x.numpy().astype(float), + train_loss) + + _run_callbacks_on_batch_end(current_step - 1) + if not isinstance(train_loss, dict): + train_loss = {'total_loss': train_loss} + if np.isnan(train_loss['total_loss']): + raise ValueError('total loss is NaN.') + + if train_metric: + train_metric_result = train_metric.result() + if isinstance(train_metric, tf.keras.metrics.Metric): + train_metric_result = tf.nest.map_structure( + lambda x: x.numpy().astype(float), train_metric_result) + if not isinstance(train_metric_result, dict): + train_metric_result = {'metric': train_metric_result} + train_metric_result.update(train_loss) + else: + train_metric_result = train_loss + if callable(optimizer.lr): + train_metric_result.update( + {'learning_rate': optimizer.lr(current_step).numpy()}) + else: + train_metric_result.update({'learning_rate': optimizer.lr.numpy()}) + logging.info('Train Step: %d/%d / loss = %s / training metric = %s', + current_step, total_steps, train_loss, + train_metric_result) + + train_summary_writer( + metrics=train_metric_result, step=optimizer.iterations) + + # Saves model checkpoints and run validation steps at every + # iterations_per_loop steps. + # To avoid repeated model saving, we do not save after the last + # step of training. + if save_freq > 0 and current_step < total_steps and ( + current_step - last_save_checkpoint_step) >= save_freq: + _save_checkpoint(checkpoint, model_dir, + checkpoint_name.format(step=current_step)) + last_save_checkpoint_step = current_step + + if test_step: + eval_iterator = self._get_input_iterator(eval_input_fn, strategy) + eval_metric_result = self._run_evaluation(test_step, current_step, + eval_metric, eval_iterator) + logging.info('Step: %s evalation metric = %s.', current_step, + eval_metric_result) + test_summary_writer( + metrics=eval_metric_result, step=optimizer.iterations) + + # Re-initialize evaluation metric, except the last step. + if eval_metric and current_step < total_steps: + eval_metric.reset_states() + if train_metric and current_step < total_steps: + train_metric.reset_states() + + # Reaches the end of training and saves the last checkpoint. + if last_save_checkpoint_step < total_steps: + _save_checkpoint(checkpoint, model_dir, + checkpoint_name.format(step=current_step)) + + if test_step: + logging.info('Running final evaluation after training is complete.') + eval_iterator = self._get_input_iterator(eval_input_fn, strategy) + eval_metric_result = self._run_evaluation(test_step, current_step, + eval_metric, eval_iterator) + logging.info('Final evaluation metric = %s.', eval_metric_result) + test_summary_writer( + metrics=eval_metric_result, step=optimizer.iterations) + + self.train_summary_writer.close() + self.eval_summary_writer.close() + + return train_loss, eval_metric_result + + def _run_evaluation(self, test_step, current_training_step, metric, + test_iterator): + """Runs validation steps and aggregate metrics.""" + if not test_iterator or not metric: + logging.warning( + 'Both test_iterator (%s) and metrics (%s) must not be None.', + test_iterator, metric) + return None + logging.info('Running evaluation after step: %s.', current_training_step) + while True: + try: + test_step(test_iterator) + except (StopIteration, tf.errors.OutOfRangeError): + break + + metric_result = metric.result() + if isinstance(metric, tf.keras.metrics.Metric): + metric_result = metric_result.numpy().astype(float) + logging.info('Step: [%d] Validation metric = %f', current_training_step, + metric_result) + return metric_result + + def evaluate_from_model_dir( + self, + model_dir: Text, + eval_input_fn: Callable[[params_dict.ParamsDict], tf.data.Dataset], + eval_metric_fn: Callable[[], Any], + total_steps: int = -1, + eval_timeout: int = None, + min_eval_interval: int = 180, + summary_writer_fn: Callable[[Text, Text], SummaryWriter] = SummaryWriter): + """Runs distributed evaluation on model folder. + + Args: + eval_input_fn: (Optional) same type as train_input_fn. If not None, will + trigger evaluting metric on eval data. If None, will not run eval step. + eval_metric_fn: metric_fn for evaluation in test_step. + model_dir: the folder for storing model checkpoints. + total_steps: total training steps. If the current step reaches the + total_steps, the evaluation loop will stop. + eval_timeout: The maximum number of seconds to wait between checkpoints. + If left as None, then the process will wait indefinitely. Used by + tf.train.checkpoints_iterator. + min_eval_interval: The minimum number of seconds between yielding + checkpoints. Used by tf.train.checkpoints_iterator. + summary_writer_fn: function to create summary writer. + + Returns: + Eval metrics dictionary of the last checkpoint. + """ + + if not model_dir: + raise ValueError('model_dir must be set.') + + def terminate_eval(): + tf.logging.info('Terminating eval after %d seconds of no checkpoints' % + eval_timeout) + return True + + summary_writer = summary_writer_fn(model_dir, 'eval') + self.eval_summary_writer = summary_writer.writer + + # Read checkpoints from the given model directory + # until `eval_timeout` seconds elapses. + for checkpoint_path in tf.train.checkpoints_iterator( + model_dir, + min_interval_secs=min_eval_interval, + timeout=eval_timeout, + timeout_fn=terminate_eval): + eval_metric_result, current_step = self.evaluate_checkpoint( + checkpoint_path=checkpoint_path, + eval_input_fn=eval_input_fn, + eval_metric_fn=eval_metric_fn, + summary_writer=summary_writer) + if total_steps > 0 and current_step >= total_steps: + logging.info('Evaluation finished after training step %d', current_step) + break + return eval_metric_result + + def evaluate_checkpoint(self, + checkpoint_path: Text, + eval_input_fn: Callable[[params_dict.ParamsDict], + tf.data.Dataset], + eval_metric_fn: Callable[[], Any], + summary_writer: SummaryWriter = None): + """Runs distributed evaluation on the one checkpoint. + + Args: + eval_input_fn: (Optional) same type as train_input_fn. If not None, will + trigger evaluting metric on eval data. If None, will not run eval step. + eval_metric_fn: metric_fn for evaluation in test_step. + checkpoint_path: the checkpoint to evaluate. + summary_writer_fn: function to create summary writer. + + Returns: + Eval metrics dictionary of the last checkpoint. + """ + if not callable(eval_metric_fn): + raise ValueError('if `eval_metric_fn` is specified, ' + 'eval_metric_fn must be a callable.') + + params = self._params + strategy = self._strategy + # To reduce unnecessary send/receive input pipeline operation, we place + # input pipeline ops in worker task. + with strategy.scope(): + + # To correctly place the model weights on accelerators, + # model and optimizer should be created in scope. + model = self.model_fn(params.as_dict()) + checkpoint = tf.train.Checkpoint(model=model) + + eval_metric = eval_metric_fn() + assert eval_metric, 'eval_metric does not exist' + test_step = self._create_test_step(strategy, model, metric=eval_metric) + + logging.info('Starting to evaluate.') + if not checkpoint_path: + raise ValueError('checkpoint path is empty') + reader = tf.compat.v1.train.NewCheckpointReader(checkpoint_path) + current_step = reader.get_tensor( + 'optimizer/iter/.ATTRIBUTES/VARIABLE_VALUE') + logging.info( + 'Checkpoint file %s found and restoring from ' + 'checkpoint', checkpoint_path) + checkpoint.restore(checkpoint_path) + + self.global_train_step = model.optimizer.iterations + eval_iterator = self._get_input_iterator(eval_input_fn, strategy) + eval_metric_result = self._run_evaluation(test_step, current_step, + eval_metric, eval_iterator) + logging.info('Step: %s evalation metric = %s.', current_step, + eval_metric_result) + summary_writer(metrics=eval_metric_result, step=current_step) + eval_metric.reset_states() + + return eval_metric_result, current_step + + def predict(self): + return NotImplementedError('Unimplmented function.') + + +class ExecutorBuilder(object): + """Builder of DistributedExecutor. + + Example 1: Builds an executor with supported Strategy. + builder = ExecutorBuilder( + strategy_type='tpu', + strategy_config={'tpu': '/bns/xxx'}) + dist_executor = builder.build_executor( + params=params, + model_fn=my_model_fn, + loss_fn=my_loss_fn, + metric_fn=my_metric_fn) + + Example 2: Builds an executor with customized Strategy. + builder = ExecutorBuilder() + builder.strategy = + dist_executor = builder.build_executor( + params=params, + model_fn=my_model_fn, + loss_fn=my_loss_fn, + metric_fn=my_metric_fn) + + Example 3: Builds a customized executor with customized Strategy. + class MyDistributedExecutor(DistributedExecutor): + # implementation ... + + builder = ExecutorBuilder() + builder.strategy = + dist_executor = builder.build_executor( + class_ctor=MyDistributedExecutor, + params=params, + model_fn=my_model_fn, + loss_fn=my_loss_fn, + metric_fn=my_metric_fn) + + Args: + strategy_type: string. One of 'tpu', 'mirrored', 'multi_worker_mirrored'. If + None. User is responsible to set the strategy before calling + build_executor(...). + strategy_config: necessary config for constructing the proper Strategy. + Check strategy_flags_dict() for examples of the structure. + """ + + def __init__(self, strategy_type=None, strategy_config=None): + _ = distribution_utils.configure_cluster( + strategy_config.worker_hosts, strategy_config.task_index) + self._strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=strategy_type, + num_gpus=strategy_config.num_gpus, + all_reduce_alg=strategy_config.all_reduce_alg, + num_packs=strategy_config.num_packs, + tpu_address=strategy_config.tpu) + + @property + def strategy(self): + """Returns default checkpoint name.""" + return self._strategy + + @strategy.setter + def strategy(self, new_strategy): + """Sets default summary writer for the current thread.""" + self._strategy = new_strategy + + + def build_executor(self, + class_ctor=DistributedExecutor, + params=None, + model_fn=None, + loss_fn=None, + **kwargs): + """Creates an executor according to strategy type. + + See doc string of the DistributedExecutor.__init__ for more information of + the + input arguments. + + Args: + class_ctor: A constructor of executor (default: DistributedExecutor). + params: ParamsDict, all the model parameters and runtime parameters. + model_fn: Keras model function. + loss_fn: loss function. + **kwargs: other arguments to the executor constructor. + + Returns: + An instance of DistributedExecutor or its subclass. + """ + if self._strategy is None: + raise ValueError('`strategy` should not be None. You need to specify ' + '`strategy_type` in the builder contructor or directly ' + 'set the `strategy` property of the builder.') + return class_ctor( + strategy=self._strategy, + params=params, + model_fn=model_fn, + loss_fn=loss_fn, + **kwargs) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/README.md new file mode 100644 index 0000000..2c78b4e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/README.md @@ -0,0 +1,19 @@ +# TensorFlow Natural Language Processing Modelling Toolkit + +tensorflow/models/official/nlp provides a [modeling library](modeling) for constructing +NLP model achitectures, as well as TF2 reference implementations for +state-of-the-art models. + +The repository contains the following models, with implementations, pre-trained +model weights, usage scripts and conversion utilities: + +* [Albert](albert) +* [Bert](bert) +* [NHNet](nhnet) +* [XLNet](xlnet) +* [Transformer for translation](transformer) + +Addtional features: + +* Distributed trainable on both multi-GPU and TPU +* e2e training for custom models, including both pretraining and finetuning. diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/README.md new file mode 100644 index 0000000..cfb726c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/README.md @@ -0,0 +1,332 @@ +# ALBERT (ALBERT: A Lite BERT for Self-supervised Learning of Language Representations) + +The academic paper which describes ALBERT in detail and provides full results on +a number of tasks can be found here: https://arxiv.org/abs/1909.11942. + +This repository contains TensorFlow 2.x implementation for ALBERT. + +## Contents + * [Contents](#contents) + * [Pre-trained Models](#pre-trained-models) + * [Restoring from Checkpoints](#restoring-from-checkpoints) + * [Set Up](#set-up) + * [Process Datasets](#process-datasets) + * [Fine-tuning with BERT](#fine-tuning-with-bert) + * [Cloud GPUs and TPUs](#cloud-gpus-and-tpus) + * [Sentence and Sentence-pair Classification Tasks](#sentence-and-sentence-pair-classification-tasks) + * [SQuAD 1.1](#squad-1.1) + + +## Pre-trained Models + +We released both checkpoints and tf.hub modules as the pretrained models for +fine-tuning. They are TF 2.x compatible and are converted from the ALBERT v2 +checkpoints released in TF 1.x official ALBERT repository +[google-research/albert](https://github.com/google-research/albert) +in order to keep consistent with ALBERT paper. + +Our current released checkpoints are exactly the same as TF 1.x official ALBERT +repository. + +### Access to Pretrained Checkpoints + +Pretrained checkpoints can be found in the following links: + +**Note: We implemented ALBERT using Keras functional-style networks in [nlp/modeling](../modeling). +ALBERT V2 models compatible with TF 2.x checkpoints are:** + +* **[`ALBERT V2 Base`](https://storage.googleapis.com/cloud-tpu-checkpoints/albert/checkpoints/albert_v2_base.tar.gz)**: + 12-layer, 768-hidden, 12-heads, 12M parameters +* **[`ALBERT V2 Large`](https://storage.googleapis.com/cloud-tpu-checkpoints/albert/checkpoints/albert_v2_large.tar.gz)**: + 24-layer, 1024-hidden, 16-heads, 18M parameters +* **[`ALBERT V2 XLarge`](https://storage.googleapis.com/cloud-tpu-checkpoints/albert/checkpoints/albert_v2_xlarge.tar.gz)**: + 24-layer, 2048-hidden, 32-heads, 60M parameters +* **[`ALBERT V2 XXLarge`](https://storage.googleapis.com/cloud-tpu-checkpoints/albert/checkpoints/albert_v2_xxlarge.tar.gz)**: + 12-layer, 4096-hidden, 64-heads, 235M parameters + +We recommend to host checkpoints on Google Cloud storage buckets when you use +Cloud GPU/TPU. + +### Restoring from Checkpoints + +`tf.train.Checkpoint` is used to manage model checkpoints in TF 2. To restore +weights from provided pre-trained checkpoints, you can use the following code: + +```python +init_checkpoint='the pretrained model checkpoint path.' +model=tf.keras.Model() # Bert pre-trained model as feature extractor. +checkpoint = tf.train.Checkpoint(model=model) +checkpoint.restore(init_checkpoint) +``` + +Checkpoints featuring native serialized Keras models +(i.e. model.load()/load_weights()) will be available soon. + +### Access to Pretrained hub modules. + +Pretrained tf.hub modules in TF 2.x SavedModel format can be found in the +following links: + +* **[`ALBERT V2 Base`](https://tfhub.dev/tensorflow/albert_en_base/1)**: + 12-layer, 768-hidden, 12-heads, 12M parameters +* **[`ALBERT V2 Large`](https://tfhub.dev/tensorflow/albert_en_large/1)**: + 24-layer, 1024-hidden, 16-heads, 18M parameters +* **[`ALBERT V2 XLarge`](https://tfhub.dev/tensorflow/albert_en_xlarge/1)**: + 24-layer, 2048-hidden, 32-heads, 60M parameters +* **[`ALBERT V2 XXLarge`](https://tfhub.dev/tensorflow/albert_en_xxlarge/1)**: + 12-layer, 4096-hidden, 64-heads, 235M parameters + +## Set Up + +```shell +export PYTHONPATH="$PYTHONPATH:/path/to/models" +``` + +Install `tf-nightly` to get latest updates: + +```shell +pip install tf-nightly-gpu +``` + +With TPU, GPU support is not necessary. First, you need to create a `tf-nightly` +TPU with [ctpu tool](https://github.com/tensorflow/tpu/tree/master/tools/ctpu): + +```shell +ctpu up -name --tf-version=”nightly” +``` + +Second, you need to install TF 2 `tf-nightly` on your VM: + +```shell +pip install tf-nightly +``` + +Warning: More details TPU-specific set-up instructions and tutorial should come +along with official TF 2.x release for TPU. Note that this repo is not +officially supported by Google Cloud TPU team yet until TF 2.1 released. + +## Process Datasets + +### Pre-training + +Pre-train ALBERT using TF2.x will come soon. +For now, please use [ALBERT research repo](https://github.com/google-research/ALBERT) +to pretrain the model and convert the checkpoint to TF2.x compatible ones using +[tf2_albert_encoder_checkpoint_converter.py](tf2_albert_encoder_checkpoint_converter.py). + + + +### Fine-tuning + +To prepare the fine-tuning data for final model training, use the +[`../data/create_finetuning_data.py`](../data/create_finetuning_data.py) script. +Note that different from BERT models that use word piece tokenzer, +ALBERT models employ sentence piece tokenizer. So the FLAG tokenizer_impl has +to be set to 'sentence_piece'. +Resulting datasets in `tf_record` format and training meta data should be later +passed to training or evaluation scripts. The task-specific arguments are +described in following sections: + +* GLUE + +Users can download the +[GLUE data](https://gluebenchmark.com/tasks) by running +[this script](https://gist.github.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e) +and unpack it to some directory `$GLUE_DIR`. + +```shell +export GLUE_DIR=~/glue +export ALBERT_DIR=gs://cloud-tpu-checkpoints/albert/checkpoints/albert_v2_base + +export TASK_NAME=MNLI +export OUTPUT_DIR=gs://some_bucket/datasets +python ../data/create_finetuning_data.py \ + --input_data_dir=${GLUE_DIR}/${TASK_NAME}/ \ + --sp_model_file=${ALBERT_DIR}/30k-clean.model \ + --train_data_output_path=${OUTPUT_DIR}/${TASK_NAME}_train.tf_record \ + --eval_data_output_path=${OUTPUT_DIR}/${TASK_NAME}_eval.tf_record \ + --meta_data_file_path=${OUTPUT_DIR}/${TASK_NAME}_meta_data \ + --fine_tuning_task_type=classification --max_seq_length=128 \ + --classification_task_name=${TASK_NAME} \ + --tokenizer_impl=sentence_piece +``` + +* SQUAD + +The [SQuAD website](https://rajpurkar.github.io/SQuAD-explorer/) contains +detailed information about the SQuAD datasets and evaluation. + +The necessary files can be found here: + +* [train-v1.1.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json) +* [dev-v1.1.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json) +* [evaluate-v1.1.py](https://github.com/allenai/bi-att-flow/blob/master/squad/evaluate-v1.1.py) +* [train-v2.0.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json) +* [dev-v2.0.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json) +* [evaluate-v2.0.py](https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/) + +```shell +export SQUAD_DIR=~/squad +export SQUAD_VERSION=v1.1 +export ALBERT_DIR=gs://cloud-tpu-checkpoints/albert/checkpoints/albert_v2_base +export OUTPUT_DIR=gs://some_bucket/datasets + +python ../data/create_finetuning_data.py \ + --squad_data_file=${SQUAD_DIR}/train-${SQUAD_VERSION}.json \ + --sp_model_file=${ALBERT_DIR}/30k-clean.model \ + --train_data_output_path=${OUTPUT_DIR}/squad_${SQUAD_VERSION}_train.tf_record \ + --meta_data_file_path=${OUTPUT_DIR}/squad_${SQUAD_VERSION}_meta_data \ + --fine_tuning_task_type=squad --max_seq_length=384 \ + --tokenizer_impl=sentence_piece +``` + +## Fine-tuning with ALBERT + +### Cloud GPUs and TPUs + +* Cloud Storage + +The unzipped pre-trained model files can also be found in the Google Cloud +Storage folder `gs://cloud-tpu-checkpoints/albert/checkpoints`. For example: + +```shell +export ALBERT_DIR=gs://cloud-tpu-checkpoints/albert/checkpoints/albert_v2_base +export MODEL_DIR=gs://some_bucket/my_output_dir +``` + +Currently, users are able to access to `tf-nightly` TPUs and the following TPU +script should run with `tf-nightly`. + +* GPU -> TPU + +Just add the following flags to `run_classifier.py` or `run_squad.py`: + +```shell + --distribution_strategy=tpu + --tpu=grpc://${TPU_IP_ADDRESS}:8470 +``` + +### Sentence and Sentence-pair Classification Tasks + +This example code fine-tunes `albert_v2_base` on the Microsoft Research +Paraphrase Corpus (MRPC) corpus, which only contains 3,600 examples and can +fine-tune in a few minutes on most GPUs. + +We use the `albert_v2_base` as an example throughout the +workflow. + + +```shell +export ALBERT_DIR=gs://cloud-tpu-checkpoints/albert/checkpoints/albert_v2_base +export MODEL_DIR=gs://some_bucket/my_output_dir +export GLUE_DIR=gs://some_bucket/datasets +export TASK=MRPC + +python run_classifier.py \ + --mode='train_and_eval' \ + --input_meta_data_path=${GLUE_DIR}/${TASK}_meta_data \ + --train_data_path=${GLUE_DIR}/${TASK}_train.tf_record \ + --eval_data_path=${GLUE_DIR}/${TASK}_eval.tf_record \ + --bert_config_file=${ALBERT_DIR}/albert_config.json \ + --init_checkpoint=${ALBERT_DIR}/bert_model.ckpt \ + --train_batch_size=4 \ + --eval_batch_size=4 \ + --steps_per_loop=1 \ + --learning_rate=2e-5 \ + --num_train_epochs=3 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=mirrored +``` + +Alternatively, instead of specifying `init_checkpoint`, you can specify +`hub_module_url` to employ a pretraind BERT hub module, e.g., +` --hub_module_url=https://tfhub.dev/tensorflow/albert_en_base/1`. + +To use TPU, you only need to switch distribution strategy type to `tpu` with TPU +information and use remote storage for model checkpoints. + +```shell +export ALBERT_DIR=gs://cloud-tpu-checkpoints/albert/checkpoints/albert_v2_base +export TPU_IP_ADDRESS='???' +export MODEL_DIR=gs://some_bucket/my_output_dir +export GLUE_DIR=gs://some_bucket/datasets + +python run_classifier.py \ + --mode='train_and_eval' \ + --input_meta_data_path=${GLUE_DIR}/${TASK}_meta_data \ + --train_data_path=${GLUE_DIR}/${TASK}_train.tf_record \ + --eval_data_path=${GLUE_DIR}/${TASK}_eval.tf_record \ + --bert_config_file=$ALBERT_DIR/albert_config.json \ + --init_checkpoint=$ALBERT_DIR/bert_model.ckpt \ + --train_batch_size=32 \ + --eval_batch_size=32 \ + --learning_rate=2e-5 \ + --num_train_epochs=3 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=tpu \ + --tpu=grpc://${TPU_IP_ADDRESS}:8470 +``` + +### SQuAD 1.1 + +The Stanford Question Answering Dataset (SQuAD) is a popular question answering +benchmark dataset. See more in [SQuAD website](https://rajpurkar.github.io/SQuAD-explorer/). + +We use the `albert_v2_base` as an example throughout the +workflow. + +```shell +export ALBERT_DIR=gs://cloud-tpu-checkpoints/albert/checkpoints/albert_v2_base +export SQUAD_DIR=gs://some_bucket/datasets +export MODEL_DIR=gs://some_bucket/my_output_dir +export SQUAD_VERSION=v1.1 + +python run_squad.py \ + --input_meta_data_path=${SQUAD_DIR}/squad_${SQUAD_VERSION}_meta_data \ + --train_data_path=${SQUAD_DIR}/squad_${SQUAD_VERSION}_train.tf_record \ + --predict_file=${SQUAD_DIR}/dev-v1.1.json \ + --sp_model_file=${ALBERT_DIR}/30k-clean.model \ + --bert_config_file=$ALBERT_DIR/albert_config.json \ + --init_checkpoint=$ALBERT_DIR/bert_model.ckpt \ + --train_batch_size=4 \ + --predict_batch_size=4 \ + --learning_rate=8e-5 \ + --num_train_epochs=2 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=mirrored +``` + +Similarily, you can replace `init_checkpoint` FLAGS with `hub_module_url` to +specify a hub module path. + +To use TPU, you need switch distribution strategy type to `tpu` with TPU +information. + +```shell +export ALBERT_DIR=gs://cloud-tpu-checkpoints/albert/checkpoints/albert_v2_base +export TPU_IP_ADDRESS='???' +export MODEL_DIR=gs://some_bucket/my_output_dir +export SQUAD_DIR=gs://some_bucket/datasets +export SQUAD_VERSION=v1.1 + +python run_squad.py \ + --input_meta_data_path=${SQUAD_DIR}/squad_${SQUAD_VERSION}_meta_data \ + --train_data_path=${SQUAD_DIR}/squad_${SQUAD_VERSION}_train.tf_record \ + --predict_file=${SQUAD_DIR}/dev-v1.1.json \ + --sp_model_file=${ALBERT_DIR}/30k-clean.model \ + --bert_config_file=$ALBERT_DIR/albert_config.json \ + --init_checkpoint=$ALBERT_DIR/bert_model.ckpt \ + --train_batch_size=32 \ + --learning_rate=8e-5 \ + --num_train_epochs=2 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=tpu \ + --tpu=grpc://${TPU_IP_ADDRESS}:8470 +``` + +The dev set predictions will be saved into a file called predictions.json in the +model_dir: + +```shell +python $SQUAD_DIR/evaluate-v1.1.py $SQUAD_DIR/dev-v1.1.json ./squad/predictions.json +``` diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/configs.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/configs.py new file mode 100644 index 0000000..010789d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/configs.py @@ -0,0 +1,61 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The ALBERT configurations.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import six + +from official.nlp.bert import configs + + +class AlbertConfig(configs.BertConfig): + """Configuration for `ALBERT`.""" + + def __init__(self, + embedding_size, + num_hidden_groups=1, + inner_group_num=1, + **kwargs): + """Constructs AlbertConfig. + + Args: + embedding_size: Size of the factorized word embeddings. + num_hidden_groups: Number of group for the hidden layers, parameters in + the same group are shared. Note that this value and also the following + 'inner_group_num' has to be 1 for now, because all released ALBERT + models set them to 1. We may support arbitary valid values in future. + inner_group_num: Number of inner repetition of attention and ffn. + **kwargs: The remaining arguments are the same as above 'BertConfig'. + """ + super(AlbertConfig, self).__init__(**kwargs) + self.embedding_size = embedding_size + + # TODO(chendouble): 'inner_group_num' and 'num_hidden_groups' are always 1 + # in the released ALBERT. Support other values in AlbertTransformerEncoder + # if needed. + if inner_group_num != 1 or num_hidden_groups != 1: + raise ValueError("We only support 'inner_group_num' and " + "'num_hidden_groups' as 1.") + + @classmethod + def from_dict(cls, json_object): + """Constructs a `AlbertConfig` from a Python dictionary of parameters.""" + config = AlbertConfig(embedding_size=None, vocab_size=None) + for (key, value) in six.iteritems(json_object): + config.__dict__[key] = value + return config diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/export_albert_tfhub.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/export_albert_tfhub.py new file mode 100644 index 0000000..9a1af1a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/export_albert_tfhub.py @@ -0,0 +1,88 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A script to export the ALBERT core model as a TF-Hub SavedModel.""" +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +from absl import app +from absl import flags +import tensorflow as tf +from typing import Text + +from official.nlp.albert import configs +from official.nlp.bert import bert_models + +FLAGS = flags.FLAGS + +flags.DEFINE_string("albert_config_file", None, + "Albert configuration file to define core albert layers.") +flags.DEFINE_string("model_checkpoint_path", None, + "File path to TF model checkpoint.") +flags.DEFINE_string("export_path", None, "TF-Hub SavedModel destination path.") +flags.DEFINE_string( + "sp_model_file", None, + "The sentence piece model file that the ALBERT model was trained on.") + + +def create_albert_model( + albert_config: configs.AlbertConfig) -> tf.keras.Model: + """Creates an ALBERT keras core model from ALBERT configuration. + + Args: + albert_config: An `AlbertConfig` to create the core model. + + Returns: + A keras model. + """ + # Adds input layers just as placeholders. + input_word_ids = tf.keras.layers.Input( + shape=(None,), dtype=tf.int32, name="input_word_ids") + input_mask = tf.keras.layers.Input( + shape=(None,), dtype=tf.int32, name="input_mask") + input_type_ids = tf.keras.layers.Input( + shape=(None,), dtype=tf.int32, name="input_type_ids") + transformer_encoder = bert_models.get_transformer_encoder( + albert_config, sequence_length=None) + sequence_output, pooled_output = transformer_encoder( + [input_word_ids, input_mask, input_type_ids]) + # To keep consistent with legacy hub modules, the outputs are + # "pooled_output" and "sequence_output". + return tf.keras.Model( + inputs=[input_word_ids, input_mask, input_type_ids], + outputs=[pooled_output, sequence_output]), transformer_encoder + + +def export_albert_tfhub(albert_config: configs.AlbertConfig, + model_checkpoint_path: Text, hub_destination: Text, + sp_model_file: Text): + """Restores a tf.keras.Model and saves for TF-Hub.""" + core_model, encoder = create_albert_model(albert_config) + checkpoint = tf.train.Checkpoint(model=encoder) + checkpoint.restore(model_checkpoint_path).assert_consumed() + core_model.sp_model_file = tf.saved_model.Asset(sp_model_file) + core_model.save(hub_destination, include_optimizer=False, save_format="tf") + + +def main(_): + albert_config = configs.AlbertConfig.from_json_file( + FLAGS.albert_config_file) + export_albert_tfhub(albert_config, FLAGS.model_checkpoint_path, + FLAGS.export_path, FLAGS.sp_model_file) + + +if __name__ == "__main__": + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/export_albert_tfhub_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/export_albert_tfhub_test.py new file mode 100644 index 0000000..4973090 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/export_albert_tfhub_test.py @@ -0,0 +1,89 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests official.nlp.albert.export_albert_tfhub.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +import numpy as np + +import tensorflow as tf +import tensorflow_hub as hub + +from official.nlp.albert import configs +from official.nlp.albert import export_albert_tfhub + + +class ExportAlbertTfhubTest(tf.test.TestCase): + + def test_export_albert_tfhub(self): + # Exports a savedmodel for TF-Hub + albert_config = configs.AlbertConfig( + vocab_size=100, + embedding_size=8, + hidden_size=16, + intermediate_size=32, + max_position_embeddings=128, + num_attention_heads=2, + num_hidden_layers=1) + bert_model, encoder = export_albert_tfhub.create_albert_model(albert_config) + model_checkpoint_dir = os.path.join(self.get_temp_dir(), "checkpoint") + checkpoint = tf.train.Checkpoint(model=encoder) + checkpoint.save(os.path.join(model_checkpoint_dir, "test")) + model_checkpoint_path = tf.train.latest_checkpoint(model_checkpoint_dir) + + sp_model_file = os.path.join(self.get_temp_dir(), "sp_tokenizer.model") + with tf.io.gfile.GFile(sp_model_file, "w") as f: + f.write("dummy content") + + hub_destination = os.path.join(self.get_temp_dir(), "hub") + export_albert_tfhub.export_albert_tfhub( + albert_config, + model_checkpoint_path, + hub_destination, + sp_model_file=sp_model_file) + + # Restores a hub KerasLayer. + hub_layer = hub.KerasLayer(hub_destination, trainable=True) + + if hasattr(hub_layer, "resolved_object"): + with tf.io.gfile.GFile( + hub_layer.resolved_object.sp_model_file.asset_path.numpy()) as f: + self.assertEqual("dummy content", f.read()) + # Checks the hub KerasLayer. + for source_weight, hub_weight in zip(bert_model.trainable_weights, + hub_layer.trainable_weights): + self.assertAllClose(source_weight.numpy(), hub_weight.numpy()) + + dummy_ids = np.zeros((2, 10), dtype=np.int32) + hub_outputs = hub_layer([dummy_ids, dummy_ids, dummy_ids]) + source_outputs = bert_model([dummy_ids, dummy_ids, dummy_ids]) + + # The outputs of hub module are "pooled_output" and "sequence_output", + # while the outputs of encoder is in reversed order, i.e., + # "sequence_output" and "pooled_output". + encoder_outputs = reversed(encoder([dummy_ids, dummy_ids, dummy_ids])) + self.assertEqual(hub_outputs[0].shape, (2, 16)) + self.assertEqual(hub_outputs[1].shape, (2, 10, 16)) + for source_output, hub_output, encoder_output in zip( + source_outputs, hub_outputs, encoder_outputs): + self.assertAllClose(source_output.numpy(), hub_output.numpy()) + self.assertAllClose(source_output.numpy(), encoder_output.numpy()) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/run_classifier.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/run_classifier.py new file mode 100644 index 0000000..51a6d42 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/run_classifier.py @@ -0,0 +1,69 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""ALBERT classification finetuning runner in tf2.x.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json + +from absl import app +from absl import flags +import tensorflow as tf + +from official.nlp.albert import configs as albert_configs +from official.nlp.bert import run_classifier as run_classifier_bert +from official.utils.misc import distribution_utils + +FLAGS = flags.FLAGS + + +def main(_): + # Users should always run this script under TF 2.x + + with tf.io.gfile.GFile(FLAGS.input_meta_data_path, 'rb') as reader: + input_meta_data = json.loads(reader.read().decode('utf-8')) + + if not FLAGS.model_dir: + FLAGS.model_dir = '/tmp/bert20/' + + strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=FLAGS.distribution_strategy, + num_gpus=FLAGS.num_gpus, + tpu_address=FLAGS.tpu) + max_seq_length = input_meta_data['max_seq_length'] + train_input_fn = run_classifier_bert.get_dataset_fn( + FLAGS.train_data_path, + max_seq_length, + FLAGS.train_batch_size, + is_training=True) + eval_input_fn = run_classifier_bert.get_dataset_fn( + FLAGS.eval_data_path, + max_seq_length, + FLAGS.eval_batch_size, + is_training=False) + + albert_config = albert_configs.AlbertConfig.from_json_file( + FLAGS.bert_config_file) + run_classifier_bert.run_bert(strategy, input_meta_data, albert_config, + train_input_fn, eval_input_fn) + + +if __name__ == '__main__': + flags.mark_flag_as_required('bert_config_file') + flags.mark_flag_as_required('input_meta_data_path') + flags.mark_flag_as_required('model_dir') + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/run_squad.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/run_squad.py new file mode 100644 index 0000000..ed3c2da --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/run_squad.py @@ -0,0 +1,139 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Run ALBERT on SQuAD 1.1 and SQuAD 2.0 in TF 2.x.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import os +import time + +from absl import app +from absl import flags +from absl import logging +import tensorflow as tf + +from official.nlp.albert import configs as albert_configs +from official.nlp.bert import run_squad_helper +from official.nlp.bert import tokenization +from official.nlp.data import squad_lib_sp +from official.utils.misc import distribution_utils + +flags.DEFINE_string( + 'sp_model_file', None, + 'The path to the sentence piece model. Used by sentence piece tokenizer ' + 'employed by ALBERT.') + +# More flags can be found in run_squad_helper. +run_squad_helper.define_common_squad_flags() + +FLAGS = flags.FLAGS + + +def train_squad(strategy, + input_meta_data, + custom_callbacks=None, + run_eagerly=False): + """Runs bert squad training.""" + bert_config = albert_configs.AlbertConfig.from_json_file( + FLAGS.bert_config_file) + run_squad_helper.train_squad(strategy, input_meta_data, bert_config, + custom_callbacks, run_eagerly) + + +def predict_squad(strategy, input_meta_data): + """Makes predictions for the squad dataset.""" + bert_config = albert_configs.AlbertConfig.from_json_file( + FLAGS.bert_config_file) + tokenizer = tokenization.FullSentencePieceTokenizer( + sp_model_file=FLAGS.sp_model_file) + + run_squad_helper.predict_squad(strategy, input_meta_data, tokenizer, + bert_config, squad_lib_sp) + + +def eval_squad(strategy, input_meta_data): + """Evaluate on the squad dataset.""" + bert_config = albert_configs.AlbertConfig.from_json_file( + FLAGS.bert_config_file) + tokenizer = tokenization.FullSentencePieceTokenizer( + sp_model_file=FLAGS.sp_model_file) + + eval_metrics = run_squad_helper.eval_squad( + strategy, input_meta_data, tokenizer, bert_config, squad_lib_sp) + return eval_metrics + + +def export_squad(model_export_path, input_meta_data): + """Exports a trained model as a `SavedModel` for inference. + + Args: + model_export_path: a string specifying the path to the SavedModel directory. + input_meta_data: dictionary containing meta data about input and model. + + Raises: + Export path is not specified, got an empty string or None. + """ + bert_config = albert_configs.AlbertConfig.from_json_file( + FLAGS.bert_config_file) + run_squad_helper.export_squad(model_export_path, input_meta_data, bert_config) + + +def main(_): + # Users should always run this script under TF 2.x + + with tf.io.gfile.GFile(FLAGS.input_meta_data_path, 'rb') as reader: + input_meta_data = json.loads(reader.read().decode('utf-8')) + + if FLAGS.mode == 'export_only': + export_squad(FLAGS.model_export_path, input_meta_data) + return + + # Configures cluster spec for multi-worker distribution strategy. + if FLAGS.num_gpus > 0: + _ = distribution_utils.configure_cluster(FLAGS.worker_hosts, + FLAGS.task_index) + strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=FLAGS.distribution_strategy, + num_gpus=FLAGS.num_gpus, + all_reduce_alg=FLAGS.all_reduce_alg, + tpu_address=FLAGS.tpu) + + if 'train' in FLAGS.mode: + train_squad(strategy, input_meta_data, run_eagerly=FLAGS.run_eagerly) + if 'predict' in FLAGS.mode: + predict_squad(strategy, input_meta_data) + if 'eval' in FLAGS.mode: + eval_metrics = eval_squad(strategy, input_meta_data) + f1_score = eval_metrics['final_f1'] + logging.info('SQuAD eval F1-score: %f', f1_score) + summary_dir = os.path.join(FLAGS.model_dir, 'summaries', 'eval') + summary_writer = tf.summary.create_file_writer(summary_dir) + with summary_writer.as_default(): + # TODO(lehou): write to the correct step number. + tf.summary.scalar('F1-score', f1_score, step=0) + summary_writer.flush() + # Also write eval_metrics to json file. + squad_lib_sp.write_to_json_files( + eval_metrics, os.path.join(summary_dir, 'eval_metrics.json')) + time.sleep(60) + + +if __name__ == '__main__': + flags.mark_flag_as_required('bert_config_file') + flags.mark_flag_as_required('model_dir') + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/tf2_albert_encoder_checkpoint_converter.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/tf2_albert_encoder_checkpoint_converter.py new file mode 100644 index 0000000..682bed8 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/albert/tf2_albert_encoder_checkpoint_converter.py @@ -0,0 +1,132 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A converter from a tf1 ALBERT encoder checkpoint to a tf2 encoder checkpoint. + +The conversion will yield an object-oriented checkpoint that can be used +to restore a AlbertTransformerEncoder object. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from absl import app +from absl import flags + +import tensorflow as tf +from official.modeling import activations +from official.nlp.albert import configs +from official.nlp.bert import tf1_checkpoint_converter_lib +from official.nlp.modeling import networks + +FLAGS = flags.FLAGS + +flags.DEFINE_string("albert_config_file", None, + "Albert configuration file to define core bert layers.") +flags.DEFINE_string( + "checkpoint_to_convert", None, + "Initial checkpoint from a pretrained BERT model core (that is, only the " + "BertModel, with no task heads.)") +flags.DEFINE_string("converted_checkpoint_path", None, + "Name for the created object-based V2 checkpoint.") + + +ALBERT_NAME_REPLACEMENTS = ( + ("bert/encoder/", ""), + ("bert/", ""), + ("embeddings/word_embeddings", "word_embeddings/embeddings"), + ("embeddings/position_embeddings", "position_embedding/embeddings"), + ("embeddings/token_type_embeddings", "type_embeddings/embeddings"), + ("embeddings/LayerNorm", "embeddings/layer_norm"), + ("embedding_hidden_mapping_in", "embedding_projection"), + ("group_0/inner_group_0/", ""), + ("attention_1/self", "self_attention"), + ("attention_1/output/dense", "self_attention_output"), + ("LayerNorm/", "self_attention_layer_norm/"), + ("ffn_1/intermediate/dense", "intermediate"), + ("ffn_1/intermediate/output/dense", "output"), + ("LayerNorm_1/", "output_layer_norm/"), + ("pooler/dense", "pooler_transform"), + ("cls/predictions/output_bias", "cls/predictions/output_bias/bias"), + ("cls/seq_relationship/output_bias", "predictions/transform/logits/bias"), + ("cls/seq_relationship/output_weights", + "predictions/transform/logits/kernel"), +) + + +def _create_albert_model(cfg): + """Creates a BERT keras core model from BERT configuration. + + Args: + cfg: A `BertConfig` to create the core model. + + Returns: + A keras model. + """ + albert_encoder = networks.AlbertTransformerEncoder( + vocab_size=cfg.vocab_size, + hidden_size=cfg.hidden_size, + embedding_width=cfg.embedding_size, + num_layers=cfg.num_hidden_layers, + num_attention_heads=cfg.num_attention_heads, + intermediate_size=cfg.intermediate_size, + activation=activations.gelu, + dropout_rate=cfg.hidden_dropout_prob, + attention_dropout_rate=cfg.attention_probs_dropout_prob, + sequence_length=cfg.max_position_embeddings, + type_vocab_size=cfg.type_vocab_size, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=cfg.initializer_range)) + return albert_encoder + + +def convert_checkpoint(bert_config, output_path, v1_checkpoint): + """Converts a V1 checkpoint into an OO V2 checkpoint.""" + output_dir, _ = os.path.split(output_path) + + # Create a temporary V1 name-converted checkpoint in the output directory. + temporary_checkpoint_dir = os.path.join(output_dir, "temp_v1") + temporary_checkpoint = os.path.join(temporary_checkpoint_dir, "ckpt") + tf1_checkpoint_converter_lib.convert( + checkpoint_from_path=v1_checkpoint, + checkpoint_to_path=temporary_checkpoint, + num_heads=bert_config.num_attention_heads, + name_replacements=ALBERT_NAME_REPLACEMENTS, + permutations=tf1_checkpoint_converter_lib.BERT_V2_PERMUTATIONS, + exclude_patterns=["adam", "Adam"]) + + # Create a V2 checkpoint from the temporary checkpoint. + model = _create_albert_model(bert_config) + tf1_checkpoint_converter_lib.create_v2_checkpoint(model, temporary_checkpoint, + output_path) + + # Clean up the temporary checkpoint, if it exists. + try: + tf.io.gfile.rmtree(temporary_checkpoint_dir) + except tf.errors.OpError: + # If it doesn't exist, we don't need to clean it up; continue. + pass + + +def main(_): + output_path = FLAGS.converted_checkpoint_path + v1_checkpoint = FLAGS.checkpoint_to_convert + albert_config = configs.AlbertConfig.from_json_file(FLAGS.albert_config_file) + convert_checkpoint(albert_config, output_path, v1_checkpoint) + + +if __name__ == "__main__": + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/README.md new file mode 100644 index 0000000..c3c8297 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/README.md @@ -0,0 +1,350 @@ +# BERT (Bidirectional Encoder Representations from Transformers) + +The academic paper which describes BERT in detail and provides full results on a +number of tasks can be found here: https://arxiv.org/abs/1810.04805. + +This repository contains TensorFlow 2.x implementation for BERT. + +## Contents + * [Contents](#contents) + * [Pre-trained Models](#pre-trained-models) + * [Restoring from Checkpoints](#restoring-from-checkpoints) + * [Set Up](#set-up) + * [Process Datasets](#process-datasets) + * [Fine-tuning with BERT](#fine-tuning-with-bert) + * [Cloud GPUs and TPUs](#cloud-gpus-and-tpus) + * [Sentence and Sentence-pair Classification Tasks](#sentence-and-sentence-pair-classification-tasks) + * [SQuAD 1.1](#squad-1.1) + + +## Pre-trained Models + +We released both checkpoints and tf.hub modules as the pretrained models for +fine-tuning. They are TF 2.x compatible and are converted from the checkpoints +released in TF 1.x official BERT repository +[google-research/bert](https://github.com/google-research/bert) +in order to keep consistent with BERT paper. + + +### Access to Pretrained Checkpoints + +Pretrained checkpoints can be found in the following links: + +**Note: We have switched BERT implementation +to use Keras functional-style networks in [nlp/modeling](../modeling). +The new checkpoints are:** + +* **[`BERT-Large, Uncased (Whole Word Masking)`](https://storage.googleapis.com/cloud-tpu-checkpoints/bert/keras_bert/wwm_uncased_L-24_H-1024_A-16.tar.gz)**: + 24-layer, 1024-hidden, 16-heads, 340M parameters +* **[`BERT-Large, Cased (Whole Word Masking)`](https://storage.googleapis.com/cloud-tpu-checkpoints/bert/keras_bert/wwm_cased_L-24_H-1024_A-16.tar.gz)**: + 24-layer, 1024-hidden, 16-heads, 340M parameters +* **[`BERT-Base, Uncased`](https://storage.googleapis.com/cloud-tpu-checkpoints/bert/keras_bert/uncased_L-12_H-768_A-12.tar.gz)**: + 12-layer, 768-hidden, 12-heads, 110M parameters +* **[`BERT-Large, Uncased`](https://storage.googleapis.com/cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16.tar.gz)**: + 24-layer, 1024-hidden, 16-heads, 340M parameters +* **[`BERT-Base, Cased`](https://storage.googleapis.com/cloud-tpu-checkpoints/bert/keras_bert/cased_L-12_H-768_A-12.tar.gz)**: + 12-layer, 768-hidden, 12-heads , 110M parameters +* **[`BERT-Large, Cased`](https://storage.googleapis.com/cloud-tpu-checkpoints/bert/keras_bert/cased_L-24_H-1024_A-16.tar.gz)**: + 24-layer, 1024-hidden, 16-heads, 340M parameters + +We recommend to host checkpoints on Google Cloud storage buckets when you use +Cloud GPU/TPU. + +### Restoring from Checkpoints + +`tf.train.Checkpoint` is used to manage model checkpoints in TF 2. To restore +weights from provided pre-trained checkpoints, you can use the following code: + +```python +init_checkpoint='the pretrained model checkpoint path.' +model=tf.keras.Model() # Bert pre-trained model as feature extractor. +checkpoint = tf.train.Checkpoint(model=model) +checkpoint.restore(init_checkpoint) +``` + +Checkpoints featuring native serialized Keras models +(i.e. model.load()/load_weights()) will be available soon. + +### Access to Pretrained hub modules. + +Pretrained tf.hub modules in TF 2.x SavedModel format can be found in the +following links: + +* **[`BERT-Large, Uncased (Whole Word Masking)`](https://tfhub.dev/tensorflow/bert_en_wwm_uncased_L-24_H-1024_A-16/1)**: + 24-layer, 1024-hidden, 16-heads, 340M parameters +* **[`BERT-Large, Cased (Whole Word Masking)`](https://tfhub.dev/tensorflow/bert_en_wwm_cased_L-24_H-1024_A-16/1)**: + 24-layer, 1024-hidden, 16-heads, 340M parameters +* **[`BERT-Base, Uncased`](https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1)**: + 12-layer, 768-hidden, 12-heads, 110M parameters +* **[`BERT-Large, Uncased`](https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1)**: + 24-layer, 1024-hidden, 16-heads, 340M parameters +* **[`BERT-Base, Cased`](https://tfhub.dev/tensorflow/bert_en_cased_L-12_H-768_A-12/1)**: + 12-layer, 768-hidden, 12-heads , 110M parameters +* **[`BERT-Large, Cased`](https://tfhub.dev/tensorflow/bert_en_cased_L-24_H-1024_A-16/1)**: + 24-layer, 1024-hidden, 16-heads, 340M parameters +* **[`BERT-Base, Multilingual Cased`](https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/1)**: + 104 languages, 12-layer, 768-hidden, 12-heads, 110M parameters +* **[`BERT-Base, Chinese`](https://tfhub.dev/tensorflow/bert_zh_L-12_H-768_A-12/1)**: + Chinese Simplified and Traditional, 12-layer, 768-hidden, 12-heads, + 110M parameters + +## Set Up + +```shell +export PYTHONPATH="$PYTHONPATH:/path/to/models" +``` + +Install `tf-nightly` to get latest updates: + +```shell +pip install tf-nightly-gpu +``` + +With TPU, GPU support is not necessary. First, you need to create a `tf-nightly` +TPU with [ctpu tool](https://github.com/tensorflow/tpu/tree/master/tools/ctpu): + +```shell +ctpu up -name --tf-version=”nightly” +``` + +Second, you need to install TF 2 `tf-nightly` on your VM: + +```shell +pip install tf-nightly +``` + +Warning: More details TPU-specific set-up instructions and tutorial should come +along with official TF 2.x release for TPU. Note that this repo is not +officially supported by Google Cloud TPU team yet until TF 2.1 released. + +## Process Datasets + +### Pre-training + +There is no change to generate pre-training data. Please use the script +[`../data/create_pretraining_data.py`](../data/create_pretraining_data.py) +which is essentially branched from [BERT research repo](https://github.com/google-research/bert) +to get processed pre-training data and it adapts to TF2 symbols and python3 +compatibility. + + +### Fine-tuning + +To prepare the fine-tuning data for final model training, use the +[`../data/create_finetuning_data.py`](../data/create_finetuning_data.py) script. +Resulting datasets in `tf_record` format and training meta data should be later +passed to training or evaluation scripts. The task-specific arguments are +described in following sections: + +* GLUE + +Users can download the +[GLUE data](https://gluebenchmark.com/tasks) by running +[this script](https://gist.github.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e) +and unpack it to some directory `$GLUE_DIR`. + +```shell +export GLUE_DIR=~/glue +export BERT_DIR=gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16 + +export TASK_NAME=MNLI +export OUTPUT_DIR=gs://some_bucket/datasets +python ../data/create_finetuning_data.py \ + --input_data_dir=${GLUE_DIR}/${TASK_NAME}/ \ + --vocab_file=${BERT_DIR}/vocab.txt \ + --train_data_output_path=${OUTPUT_DIR}/${TASK_NAME}_train.tf_record \ + --eval_data_output_path=${OUTPUT_DIR}/${TASK_NAME}_eval.tf_record \ + --meta_data_file_path=${OUTPUT_DIR}/${TASK_NAME}_meta_data \ + --fine_tuning_task_type=classification --max_seq_length=128 \ + --classification_task_name=${TASK_NAME} +``` + +* SQUAD + +The [SQuAD website](https://rajpurkar.github.io/SQuAD-explorer/) contains +detailed information about the SQuAD datasets and evaluation. + +The necessary files can be found here: + +* [train-v1.1.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json) +* [dev-v1.1.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json) +* [evaluate-v1.1.py](https://github.com/allenai/bi-att-flow/blob/master/squad/evaluate-v1.1.py) +* [train-v2.0.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json) +* [dev-v2.0.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json) +* [evaluate-v2.0.py](https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/) + +```shell +export SQUAD_DIR=~/squad +export SQUAD_VERSION=v1.1 +export BERT_DIR=gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16 +export OUTPUT_DIR=gs://some_bucket/datasets + +python ../data/create_finetuning_data.py \ + --squad_data_file=${SQUAD_DIR}/train-${SQUAD_VERSION}.json \ + --vocab_file=${BERT_DIR}/vocab.txt \ + --train_data_output_path=${OUTPUT_DIR}/squad_${SQUAD_VERSION}_train.tf_record \ + --meta_data_file_path=${OUTPUT_DIR}/squad_${SQUAD_VERSION}_meta_data \ + --fine_tuning_task_type=squad --max_seq_length=384 +``` + +## Fine-tuning with BERT + +### Cloud GPUs and TPUs + +* Cloud Storage + +The unzipped pre-trained model files can also be found in the Google Cloud +Storage folder `gs://cloud-tpu-checkpoints/bert/keras_bert`. For example: + +```shell +export BERT_DIR=gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16 +export MODEL_DIR=gs://some_bucket/my_output_dir +``` + +Currently, users are able to access to `tf-nightly` TPUs and the following TPU +script should run with `tf-nightly`. + +* GPU -> TPU + +Just add the following flags to `run_classifier.py` or `run_squad.py`: + +```shell + --distribution_strategy=tpu + --tpu=grpc://${TPU_IP_ADDRESS}:8470 +``` + +### Sentence and Sentence-pair Classification Tasks + +This example code fine-tunes `BERT-Large` on the Microsoft Research Paraphrase +Corpus (MRPC) corpus, which only contains 3,600 examples and can fine-tune in a +few minutes on most GPUs. + +We use the `BERT-Large` (uncased_L-24_H-1024_A-16) as an example throughout the +workflow. +For GPU memory of 16GB or smaller, you may try to use `BERT-Base` +(uncased_L-12_H-768_A-12). + +```shell +export BERT_DIR=gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16 +export MODEL_DIR=gs://some_bucket/my_output_dir +export GLUE_DIR=gs://some_bucket/datasets +export TASK=MRPC + +python run_classifier.py \ + --mode='train_and_eval' \ + --input_meta_data_path=${GLUE_DIR}/${TASK}_meta_data \ + --train_data_path=${GLUE_DIR}/${TASK}_train.tf_record \ + --eval_data_path=${GLUE_DIR}/${TASK}_eval.tf_record \ + --bert_config_file=${BERT_DIR}/bert_config.json \ + --init_checkpoint=${BERT_DIR}/bert_model.ckpt \ + --train_batch_size=4 \ + --eval_batch_size=4 \ + --steps_per_loop=1 \ + --learning_rate=2e-5 \ + --num_train_epochs=3 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=mirrored +``` + +Alternatively, instead of specifying `init_checkpoint`, you can specify +`hub_module_url` to employ a pretraind BERT hub module, e.g., +` --hub_module_url=https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1`. + +To use TPU, you only need to switch distribution strategy type to `tpu` with TPU +information and use remote storage for model checkpoints. + +```shell +export BERT_DIR=gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16 +export TPU_IP_ADDRESS='???' +export MODEL_DIR=gs://some_bucket/my_output_dir +export GLUE_DIR=gs://some_bucket/datasets +export TASK=MRPC + +python run_classifier.py \ + --mode='train_and_eval' \ + --input_meta_data_path=${GLUE_DIR}/${TASK}_meta_data \ + --train_data_path=${GLUE_DIR}/${TASK}_train.tf_record \ + --eval_data_path=${GLUE_DIR}/${TASK}_eval.tf_record \ + --bert_config_file=${BERT_DIR}/bert_config.json \ + --init_checkpoint=${BERT_DIR}/bert_model.ckpt \ + --train_batch_size=32 \ + --eval_batch_size=32 \ + --steps_per_loop=1000 \ + --learning_rate=2e-5 \ + --num_train_epochs=3 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=tpu \ + --tpu=grpc://${TPU_IP_ADDRESS}:8470 +``` + +Note that, we specify `steps_per_loop=1000` for TPU, because running a loop of +training steps inside a `tf.function` can significantly increase TPU utilization +and callbacks will not be called inside the loop. + +### SQuAD 1.1 + +The Stanford Question Answering Dataset (SQuAD) is a popular question answering +benchmark dataset. See more in [SQuAD website](https://rajpurkar.github.io/SQuAD-explorer/). + +We use the `BERT-Large` (uncased_L-24_H-1024_A-16) as an example throughout the +workflow. +For GPU memory of 16GB or smaller, you may try to use `BERT-Base` +(uncased_L-12_H-768_A-12). + +```shell +export BERT_DIR=gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16 +export SQUAD_DIR=gs://some_bucket/datasets +export MODEL_DIR=gs://some_bucket/my_output_dir +export SQUAD_VERSION=v1.1 + +python run_squad.py \ + --input_meta_data_path=${SQUAD_DIR}/squad_${SQUAD_VERSION}_meta_data \ + --train_data_path=${SQUAD_DIR}/squad_${SQUAD_VERSION}_train.tf_record \ + --predict_file=${SQUAD_DIR}/dev-v1.1.json \ + --vocab_file=${BERT_DIR}/vocab.txt \ + --bert_config_file=${BERT_DIR}/bert_config.json \ + --init_checkpoint=${BERT_DIR}/bert_model.ckpt \ + --train_batch_size=4 \ + --predict_batch_size=4 \ + --learning_rate=8e-5 \ + --num_train_epochs=2 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=mirrored +``` + +Similarily, you can replace `init_checkpoint` FLAG with `hub_module_url` to +specify a hub module path. + +To use TPU, you need switch distribution strategy type to `tpu` with TPU +information. + +```shell +export BERT_DIR=gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16 +export TPU_IP_ADDRESS='???' +export MODEL_DIR=gs://some_bucket/my_output_dir +export SQUAD_DIR=gs://some_bucket/datasets +export SQUAD_VERSION=v1.1 + +python run_squad.py \ + --input_meta_data_path=${SQUAD_DIR}/squad_${SQUAD_VERSION}_meta_data \ + --train_data_path=${SQUAD_DIR}/squad_${SQUAD_VERSION}_train.tf_record \ + --predict_file=${SQUAD_DIR}/dev-v1.1.json \ + --vocab_file=${BERT_DIR}/vocab.txt \ + --bert_config_file=${BERT_DIR}/bert_config.json \ + --init_checkpoint=${BERT_DIR}/bert_model.ckpt \ + --train_batch_size=32 \ + --learning_rate=8e-5 \ + --num_train_epochs=2 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=tpu \ + --tpu=grpc://${TPU_IP_ADDRESS}:8470 +``` + +The dev set predictions will be saved into a file called predictions.json in the +model_dir: + +```shell +python $SQUAD_DIR/evaluate-v1.1.py $SQUAD_DIR/dev-v1.1.json ./squad/predictions.json +``` + + diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/__init__.py @@ -0,0 +1 @@ + diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/bert_cloud_tpu.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/bert_cloud_tpu.md new file mode 100644 index 0000000..e5e6758 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/bert_cloud_tpu.md @@ -0,0 +1,110 @@ +# BERT FineTuning with Cloud TPU: Sentence and Sentence-Pair Classification Tasks (TF 2.1) +This tutorial shows you how to train the Bidirectional Encoder Representations from Transformers (BERT) model on Cloud TPU. + + +## Set up Cloud Storage and Compute Engine VM +1. [Open a cloud shell window](https://console.cloud.google.com/?cloudshell=true&_ga=2.11844148.-1612541229.1552429951) +2. Create a variable for the project's name: +``` +export PROJECT_NAME=your-project_name +``` +3. Configure `gcloud` command-line tool to use the project where you want to create Cloud TPU. +``` +gcloud config set project ${PROJECT_NAME} +``` +4. Create a Cloud Storage bucket using the following command: +``` +gsutil mb -p ${PROJECT_NAME} -c standard -l europe-west4 -b on gs://your-bucket-name +``` +This Cloud Storage bucket stores the data you use to train your model and the training results. +5. Launch a Compute Engine VM and Cloud TPU using the ctpu up command. +``` +ctpu up --tpu-size=v3-8 \ + --machine-type=n1-standard-8 \ + --zone=europe-west4-a \ + --tf-version=2.1 [optional flags: --project, --name] +``` +6. The configuration you specified appears. Enter y to approve or n to cancel. +7. When the ctpu up command has finished executing, verify that your shell prompt has changed from username@project to username@tpuname. This change shows that you are now logged into your Compute Engine VM. +``` +gcloud compute ssh vm-name --zone=europe-west4-a +(vm)$ export TPU_NAME=vm-name +``` +As you continue these instructions, run each command that begins with `(vm)$` in your VM session window. + +## Prepare the Dataset +1. From your Compute Engine virtual machine (VM), install requirements.txt. +``` +(vm)$ cd /usr/share/models +(vm)$ sudo pip3 install -r official/requirements.txt +``` +2. Optional: download download_glue_data.py + +This tutorial uses the General Language Understanding Evaluation (GLUE) benchmark to evaluate and analyze the performance of the model. The GLUE data is provided for this tutorial at gs://cloud-tpu-checkpoints/bert/classification. + +## Define parameter values +Next, define several parameter values that are required when you train and evaluate your model: + +``` +(vm)$ export PYTHONPATH="$PYTHONPATH:/usr/share/tpu/models" +(vm)$ export STORAGE_BUCKET=gs://your-bucket-name +(vm)$ export BERT_BASE_DIR=gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16 +(vm)$ export MODEL_DIR=${STORAGE_BUCKET}/bert-output +(vm)$ export GLUE_DIR=gs://cloud-tpu-checkpoints/bert/classification +(vm)$ export TASK=mnli +``` + +## Train the model +From your Compute Engine VM, run the following command. + +``` +(vm)$ python3 official/nlp/bert/run_classifier.py \ + --mode='train_and_eval' \ + --input_meta_data_path=${GLUE_DIR}/${TASK}_meta_data \ + --train_data_path=${GLUE_DIR}/${TASK}_train.tf_record \ + --eval_data_path=${GLUE_DIR}/${TASK}_eval.tf_record \ + --bert_config_file=$BERT_BASE_DIR/bert_config.json \ + --init_checkpoint=$BERT_BASE_DIR/bert_model.ckpt \ + --train_batch_size=32 \ + --eval_batch_size=32 \ + --learning_rate=2e-5 \ + --num_train_epochs=3 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=tpu \ + --tpu=${TPU_NAME} +``` + +## Verify your results +The training takes approximately 1 hour on a v3-8 TPU. When script completes, you should see results similar to the following: +``` +Training Summary: +{'train_loss': 0.28142181038856506, +'last_train_metrics': 0.9467429518699646, +'eval_metrics': 0.8599063158035278, +'total_training_steps': 36813} +``` + +## Clean up +To avoid incurring charges to your GCP account for the resources used in this topic: +1. Disconnect from the Compute Engine VM: +``` +(vm)$ exit +``` +2. In your Cloud Shell, run ctpu delete with the --zone flag you used when you set up the Cloud TPU to delete your Compute Engine VM and your Cloud TPU: +``` +$ ctpu delete --zone=your-zone +``` +3. Run ctpu status specifying your zone to make sure you have no instances allocated to avoid unnecessary charges for TPU usage. The deletion might take several minutes. A response like the one below indicates there are no more allocated instances: +``` +$ ctpu status --zone=your-zone +``` +4. Run gsutil as shown, replacing your-bucket with the name of the Cloud Storage bucket you created for this tutorial: +``` +$ gsutil rm -r gs://your-bucket +``` + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/bert_models.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/bert_models.py new file mode 100644 index 0000000..04745fd --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/bert_models.py @@ -0,0 +1,349 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""BERT models that are compatible with TF 2.0.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gin +import tensorflow as tf +import tensorflow_hub as hub + +from official.modeling import tf_utils +from official.nlp.albert import configs as albert_configs +from official.nlp.bert import configs +from official.nlp.modeling import losses +from official.nlp.modeling import models +from official.nlp.modeling import networks + + +class BertPretrainLossAndMetricLayer(tf.keras.layers.Layer): + """Returns layer that computes custom loss and metrics for pretraining.""" + + def __init__(self, vocab_size, **kwargs): + super(BertPretrainLossAndMetricLayer, self).__init__(**kwargs) + self._vocab_size = vocab_size + self.config = { + 'vocab_size': vocab_size, + } + + def _add_metrics(self, lm_output, lm_labels, lm_label_weights, + lm_example_loss, sentence_output, sentence_labels, + next_sentence_loss): + """Adds metrics.""" + masked_lm_accuracy = tf.keras.metrics.sparse_categorical_accuracy( + lm_labels, lm_output) + numerator = tf.reduce_sum(masked_lm_accuracy * lm_label_weights) + denominator = tf.reduce_sum(lm_label_weights) + 1e-5 + masked_lm_accuracy = numerator / denominator + self.add_metric( + masked_lm_accuracy, name='masked_lm_accuracy', aggregation='mean') + + self.add_metric(lm_example_loss, name='lm_example_loss', aggregation='mean') + + if sentence_labels is not None: + next_sentence_accuracy = tf.keras.metrics.sparse_categorical_accuracy( + sentence_labels, sentence_output) + self.add_metric( + next_sentence_accuracy, + name='next_sentence_accuracy', + aggregation='mean') + + if next_sentence_loss is not None: + self.add_metric( + next_sentence_loss, name='next_sentence_loss', aggregation='mean') + + def call(self, + lm_output, + sentence_output, + lm_label_ids, + lm_label_weights, + sentence_labels=None): + """Implements call() for the layer.""" + lm_label_weights = tf.cast(lm_label_weights, tf.float32) + lm_output = tf.cast(lm_output, tf.float32) + + mask_label_loss = losses.weighted_sparse_categorical_crossentropy_loss( + labels=lm_label_ids, predictions=lm_output, weights=lm_label_weights) + + if sentence_labels is not None: + sentence_output = tf.cast(sentence_output, tf.float32) + sentence_loss = losses.weighted_sparse_categorical_crossentropy_loss( + labels=sentence_labels, predictions=sentence_output) + loss = mask_label_loss + sentence_loss + else: + sentence_loss = None + loss = mask_label_loss + + batch_shape = tf.slice(tf.shape(lm_label_ids), [0], [1]) + # TODO(hongkuny): Avoids the hack and switches add_loss. + final_loss = tf.fill(batch_shape, loss) + + self._add_metrics(lm_output, lm_label_ids, lm_label_weights, + mask_label_loss, sentence_output, sentence_labels, + sentence_loss) + return final_loss + + +@gin.configurable +def get_transformer_encoder(bert_config, + sequence_length, + transformer_encoder_cls=None): + """Gets a 'TransformerEncoder' object. + + Args: + bert_config: A 'modeling.BertConfig' or 'modeling.AlbertConfig' object. + sequence_length: Maximum sequence length of the training data. + transformer_encoder_cls: A EncoderScaffold class. If it is None, uses the + default BERT encoder implementation. + + Returns: + A networks.TransformerEncoder object. + """ + if transformer_encoder_cls is not None: + # TODO(hongkuny): evaluate if it is better to put cfg definition in gin. + embedding_cfg = dict( + vocab_size=bert_config.vocab_size, + type_vocab_size=bert_config.type_vocab_size, + hidden_size=bert_config.hidden_size, + seq_length=sequence_length, + max_seq_length=bert_config.max_position_embeddings, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=bert_config.initializer_range), + dropout_rate=bert_config.hidden_dropout_prob, + ) + hidden_cfg = dict( + num_attention_heads=bert_config.num_attention_heads, + intermediate_size=bert_config.intermediate_size, + intermediate_activation=tf_utils.get_activation(bert_config.hidden_act), + dropout_rate=bert_config.hidden_dropout_prob, + attention_dropout_rate=bert_config.attention_probs_dropout_prob, + ) + kwargs = dict( + embedding_cfg=embedding_cfg, + hidden_cfg=hidden_cfg, + num_hidden_instances=bert_config.num_hidden_layers, + pooled_output_dim=bert_config.hidden_size, + ) + + # Relies on gin configuration to define the Transformer encoder arguments. + return transformer_encoder_cls(**kwargs) + + kwargs = dict( + vocab_size=bert_config.vocab_size, + hidden_size=bert_config.hidden_size, + num_layers=bert_config.num_hidden_layers, + num_attention_heads=bert_config.num_attention_heads, + intermediate_size=bert_config.intermediate_size, + activation=tf_utils.get_activation(bert_config.hidden_act), + dropout_rate=bert_config.hidden_dropout_prob, + attention_dropout_rate=bert_config.attention_probs_dropout_prob, + sequence_length=sequence_length, + max_sequence_length=bert_config.max_position_embeddings, + type_vocab_size=bert_config.type_vocab_size, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=bert_config.initializer_range)) + if isinstance(bert_config, albert_configs.AlbertConfig): + kwargs['embedding_width'] = bert_config.embedding_size + return networks.AlbertTransformerEncoder(**kwargs) + else: + assert isinstance(bert_config, configs.BertConfig) + return networks.TransformerEncoder(**kwargs) + + +def pretrain_model(bert_config, + seq_length, + max_predictions_per_seq, + initializer=None, + use_next_sentence_label=True): + """Returns model to be used for pre-training. + + Args: + bert_config: Configuration that defines the core BERT model. + seq_length: Maximum sequence length of the training data. + max_predictions_per_seq: Maximum number of tokens in sequence to mask out + and use for pretraining. + initializer: Initializer for weights in BertPretrainer. + use_next_sentence_label: Whether to use the next sentence label. + + Returns: + Pretraining model as well as core BERT submodel from which to save + weights after pretraining. + """ + input_word_ids = tf.keras.layers.Input( + shape=(seq_length,), name='input_word_ids', dtype=tf.int32) + input_mask = tf.keras.layers.Input( + shape=(seq_length,), name='input_mask', dtype=tf.int32) + input_type_ids = tf.keras.layers.Input( + shape=(seq_length,), name='input_type_ids', dtype=tf.int32) + masked_lm_positions = tf.keras.layers.Input( + shape=(max_predictions_per_seq,), + name='masked_lm_positions', + dtype=tf.int32) + masked_lm_ids = tf.keras.layers.Input( + shape=(max_predictions_per_seq,), name='masked_lm_ids', dtype=tf.int32) + masked_lm_weights = tf.keras.layers.Input( + shape=(max_predictions_per_seq,), + name='masked_lm_weights', + dtype=tf.int32) + + if use_next_sentence_label: + next_sentence_labels = tf.keras.layers.Input( + shape=(1,), name='next_sentence_labels', dtype=tf.int32) + else: + next_sentence_labels = None + + transformer_encoder = get_transformer_encoder(bert_config, seq_length) + if initializer is None: + initializer = tf.keras.initializers.TruncatedNormal( + stddev=bert_config.initializer_range) + pretrainer_model = models.BertPretrainer( + network=transformer_encoder, + num_classes=2, # The next sentence prediction label has two classes. + num_token_predictions=max_predictions_per_seq, + initializer=initializer, + output='predictions') + + lm_output, sentence_output = pretrainer_model( + [input_word_ids, input_mask, input_type_ids, masked_lm_positions]) + + pretrain_loss_layer = BertPretrainLossAndMetricLayer( + vocab_size=bert_config.vocab_size) + output_loss = pretrain_loss_layer(lm_output, sentence_output, masked_lm_ids, + masked_lm_weights, next_sentence_labels) + inputs = { + 'input_word_ids': input_word_ids, + 'input_mask': input_mask, + 'input_type_ids': input_type_ids, + 'masked_lm_positions': masked_lm_positions, + 'masked_lm_ids': masked_lm_ids, + 'masked_lm_weights': masked_lm_weights, + } + if use_next_sentence_label: + inputs['next_sentence_labels'] = next_sentence_labels + + keras_model = tf.keras.Model(inputs=inputs, outputs=output_loss) + return keras_model, transformer_encoder + + +def squad_model(bert_config, + max_seq_length, + initializer=None, + hub_module_url=None, + hub_module_trainable=True): + """Returns BERT Squad model along with core BERT model to import weights. + + Args: + bert_config: BertConfig, the config defines the core Bert model. + max_seq_length: integer, the maximum input sequence length. + initializer: Initializer for the final dense layer in the span labeler. + Defaulted to TruncatedNormal initializer. + hub_module_url: TF-Hub path/url to Bert module. + hub_module_trainable: True to finetune layers in the hub module. + + Returns: + A tuple of (1) keras model that outputs start logits and end logits and + (2) the core BERT transformer encoder. + """ + if initializer is None: + initializer = tf.keras.initializers.TruncatedNormal( + stddev=bert_config.initializer_range) + if not hub_module_url: + bert_encoder = get_transformer_encoder(bert_config, max_seq_length) + return models.BertSpanLabeler( + network=bert_encoder, initializer=initializer), bert_encoder + + input_word_ids = tf.keras.layers.Input( + shape=(max_seq_length,), dtype=tf.int32, name='input_word_ids') + input_mask = tf.keras.layers.Input( + shape=(max_seq_length,), dtype=tf.int32, name='input_mask') + input_type_ids = tf.keras.layers.Input( + shape=(max_seq_length,), dtype=tf.int32, name='input_type_ids') + core_model = hub.KerasLayer(hub_module_url, trainable=hub_module_trainable) + pooled_output, sequence_output = core_model( + [input_word_ids, input_mask, input_type_ids]) + bert_encoder = tf.keras.Model( + inputs={ + 'input_word_ids': input_word_ids, + 'input_mask': input_mask, + 'input_type_ids': input_type_ids, + }, + outputs=[sequence_output, pooled_output], + name='core_model') + return models.BertSpanLabeler( + network=bert_encoder, initializer=initializer), bert_encoder + + +def classifier_model(bert_config, + num_labels, + max_seq_length, + final_layer_initializer=None, + hub_module_url=None, + hub_module_trainable=True): + """BERT classifier model in functional API style. + + Construct a Keras model for predicting `num_labels` outputs from an input with + maximum sequence length `max_seq_length`. + + Args: + bert_config: BertConfig or AlbertConfig, the config defines the core BERT or + ALBERT model. + num_labels: integer, the number of classes. + max_seq_length: integer, the maximum input sequence length. + final_layer_initializer: Initializer for final dense layer. Defaulted + TruncatedNormal initializer. + hub_module_url: TF-Hub path/url to Bert module. + hub_module_trainable: True to finetune layers in the hub module. + + Returns: + Combined prediction model (words, mask, type) -> (one-hot labels) + BERT sub-model (words, mask, type) -> (bert_outputs) + """ + if final_layer_initializer is not None: + initializer = final_layer_initializer + else: + initializer = tf.keras.initializers.TruncatedNormal( + stddev=bert_config.initializer_range) + + if not hub_module_url: + bert_encoder = get_transformer_encoder(bert_config, max_seq_length) + return models.BertClassifier( + bert_encoder, + num_classes=num_labels, + dropout_rate=bert_config.hidden_dropout_prob, + initializer=initializer), bert_encoder + + input_word_ids = tf.keras.layers.Input( + shape=(max_seq_length,), dtype=tf.int32, name='input_word_ids') + input_mask = tf.keras.layers.Input( + shape=(max_seq_length,), dtype=tf.int32, name='input_mask') + input_type_ids = tf.keras.layers.Input( + shape=(max_seq_length,), dtype=tf.int32, name='input_type_ids') + bert_model = hub.KerasLayer(hub_module_url, trainable=hub_module_trainable) + pooled_output, _ = bert_model([input_word_ids, input_mask, input_type_ids]) + output = tf.keras.layers.Dropout(rate=bert_config.hidden_dropout_prob)( + pooled_output) + + output = tf.keras.layers.Dense( + num_labels, kernel_initializer=initializer, name='output')( + output) + return tf.keras.Model( + inputs={ + 'input_word_ids': input_word_ids, + 'input_mask': input_mask, + 'input_type_ids': input_type_ids + }, + outputs=output), bert_model diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/common_flags.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/common_flags.py new file mode 100644 index 0000000..67fdade --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/common_flags.py @@ -0,0 +1,118 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Defining common flags used across all BERT models/applications.""" + +from absl import flags +import tensorflow as tf + +from official.utils.flags import core as flags_core + + +def define_gin_flags(): + """Define common gin configurable flags.""" + flags.DEFINE_multi_string('gin_file', None, + 'List of paths to the config files.') + flags.DEFINE_multi_string( + 'gin_param', None, 'Newline separated list of Gin parameter bindings.') + + +def define_common_bert_flags(): + """Define common flags for BERT tasks.""" + flags_core.define_base( + data_dir=False, + model_dir=True, + clean=False, + train_epochs=False, + epochs_between_evals=False, + stop_threshold=False, + batch_size=False, + num_gpu=True, + hooks=False, + export_dir=False, + distribution_strategy=True, + run_eagerly=True) + flags_core.define_distribution() + flags.DEFINE_string('bert_config_file', None, + 'Bert configuration file to define core bert layers.') + flags.DEFINE_string( + 'model_export_path', None, + 'Path to the directory, where trainined model will be ' + 'exported.') + flags.DEFINE_string('tpu', '', 'TPU address to connect to.') + flags.DEFINE_string( + 'init_checkpoint', None, + 'Initial checkpoint (usually from a pre-trained BERT model).') + flags.DEFINE_integer('num_train_epochs', 3, + 'Total number of training epochs to perform.') + flags.DEFINE_integer( + 'steps_per_loop', 1, + 'Number of steps per graph-mode loop. Only training step ' + 'happens inside the loop. Callbacks will not be called ' + 'inside.') + flags.DEFINE_float('learning_rate', 5e-5, + 'The initial learning rate for Adam.') + flags.DEFINE_float('end_lr', 0.0, + 'The end learning rate for learning rate decay.') + flags.DEFINE_string('optimizer_type', 'adamw', + 'The type of optimizer to use for training (adamw|lamb)') + flags.DEFINE_boolean( + 'scale_loss', False, + 'Whether to divide the loss by number of replica inside the per-replica ' + 'loss function.') + flags.DEFINE_boolean( + 'use_keras_compile_fit', False, + 'If True, uses Keras compile/fit() API for training logic. Otherwise ' + 'use custom training loop.') + flags.DEFINE_string( + 'hub_module_url', None, 'TF-Hub path/url to Bert module. ' + 'If specified, init_checkpoint flag should not be used.') + flags.DEFINE_bool('hub_module_trainable', True, + 'True to make keras layers in the hub module trainable.') + + flags_core.define_log_steps() + + # Adds flags for mixed precision and multi-worker training. + flags_core.define_performance( + num_parallel_calls=False, + inter_op=False, + intra_op=False, + synthetic_data=False, + max_train_steps=False, + dtype=True, + dynamic_loss_scale=True, + loss_scale=True, + all_reduce_alg=True, + num_packs=False, + tf_gpu_thread_mode=True, + datasets_num_private_threads=True, + enable_xla=True, + fp16_implementation=True, + ) + + +def dtype(): + return flags_core.get_tf_dtype(flags.FLAGS) + + +def use_float16(): + return flags_core.get_tf_dtype(flags.FLAGS) == tf.float16 + + +def use_graph_rewrite(): + return flags.FLAGS.fp16_implementation == 'graph_rewrite' + + +def get_loss_scale(): + return flags_core.get_loss_scale(flags.FLAGS, default_for_fp16='dynamic') diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/configs.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/configs.py new file mode 100644 index 0000000..ae22cfb --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/configs.py @@ -0,0 +1,105 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The main BERT model and related functions.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import copy +import json +import six +import tensorflow as tf + + +class BertConfig(object): + """Configuration for `BertModel`.""" + + def __init__(self, + vocab_size, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + backward_compatible=True): + """Constructs BertConfig. + + Args: + vocab_size: Vocabulary size of `inputs_ids` in `BertModel`. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. + hidden_dropout_prob: The dropout probability for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `BertModel`. + initializer_range: The stdev of the truncated_normal_initializer for + initializing all weight matrices. + backward_compatible: Boolean, whether the variables shape are compatible + with checkpoints converted from TF 1.x BERT. + """ + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.backward_compatible = backward_compatible + + @classmethod + def from_dict(cls, json_object): + """Constructs a `BertConfig` from a Python dictionary of parameters.""" + config = BertConfig(vocab_size=None) + for (key, value) in six.iteritems(json_object): + config.__dict__[key] = value + return config + + @classmethod + def from_json_file(cls, json_file): + """Constructs a `BertConfig` from a json file of parameters.""" + with tf.io.gfile.GFile(json_file, "r") as reader: + text = reader.read() + return cls.from_dict(json.loads(text)) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" + diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/export_tfhub.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/export_tfhub.py new file mode 100644 index 0000000..b718907 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/export_tfhub.py @@ -0,0 +1,86 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A script to export the BERT core model as a TF-Hub SavedModel.""" +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +from absl import app +from absl import flags +import tensorflow as tf +from typing import Text +from official.nlp.bert import bert_models +from official.nlp.bert import configs + +FLAGS = flags.FLAGS + +flags.DEFINE_string("bert_config_file", None, + "Bert configuration file to define core bert layers.") +flags.DEFINE_string("model_checkpoint_path", None, + "File path to TF model checkpoint.") +flags.DEFINE_string("export_path", None, "TF-Hub SavedModel destination path.") +flags.DEFINE_string("vocab_file", None, + "The vocabulary file that the BERT model was trained on.") + + +def create_bert_model(bert_config: configs.BertConfig) -> tf.keras.Model: + """Creates a BERT keras core model from BERT configuration. + + Args: + bert_config: A `BertConfig` to create the core model. + + Returns: + A keras model. + """ + # Adds input layers just as placeholders. + input_word_ids = tf.keras.layers.Input( + shape=(None,), dtype=tf.int32, name="input_word_ids") + input_mask = tf.keras.layers.Input( + shape=(None,), dtype=tf.int32, name="input_mask") + input_type_ids = tf.keras.layers.Input( + shape=(None,), dtype=tf.int32, name="input_type_ids") + transformer_encoder = bert_models.get_transformer_encoder( + bert_config, sequence_length=None) + sequence_output, pooled_output = transformer_encoder( + [input_word_ids, input_mask, input_type_ids]) + # To keep consistent with legacy hub modules, the outputs are + # "pooled_output" and "sequence_output". + return tf.keras.Model( + inputs=[input_word_ids, input_mask, input_type_ids], + outputs=[pooled_output, sequence_output]), transformer_encoder + + +def export_bert_tfhub(bert_config: configs.BertConfig, + model_checkpoint_path: Text, hub_destination: Text, + vocab_file: Text): + """Restores a tf.keras.Model and saves for TF-Hub.""" + core_model, encoder = create_bert_model(bert_config) + checkpoint = tf.train.Checkpoint(model=encoder) + checkpoint.restore(model_checkpoint_path).assert_consumed() + core_model.vocab_file = tf.saved_model.Asset(vocab_file) + core_model.do_lower_case = tf.Variable( + "uncased" in vocab_file, trainable=False) + core_model.save(hub_destination, include_optimizer=False, save_format="tf") + + +def main(_): + bert_config = configs.BertConfig.from_json_file(FLAGS.bert_config_file) + export_bert_tfhub(bert_config, FLAGS.model_checkpoint_path, FLAGS.export_path, + FLAGS.vocab_file) + + +if __name__ == "__main__": + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/export_tfhub_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/export_tfhub_test.py new file mode 100644 index 0000000..b2d6e8a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/export_tfhub_test.py @@ -0,0 +1,87 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests official.nlp.bert.export_tfhub.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +import numpy as np + +import tensorflow as tf +import tensorflow_hub as hub +from official.nlp.bert import configs +from official.nlp.bert import export_tfhub + + +class ExportTfhubTest(tf.test.TestCase): + + def test_export_tfhub(self): + # Exports a savedmodel for TF-Hub + bert_config = configs.BertConfig( + vocab_size=100, + hidden_size=16, + intermediate_size=32, + max_position_embeddings=128, + num_attention_heads=2, + num_hidden_layers=1) + bert_model, encoder = export_tfhub.create_bert_model(bert_config) + model_checkpoint_dir = os.path.join(self.get_temp_dir(), "checkpoint") + checkpoint = tf.train.Checkpoint(model=encoder) + checkpoint.save(os.path.join(model_checkpoint_dir, "test")) + model_checkpoint_path = tf.train.latest_checkpoint(model_checkpoint_dir) + + vocab_file = os.path.join(self.get_temp_dir(), "uncased_vocab.txt") + with tf.io.gfile.GFile(vocab_file, "w") as f: + f.write("dummy content") + + hub_destination = os.path.join(self.get_temp_dir(), "hub") + export_tfhub.export_bert_tfhub(bert_config, model_checkpoint_path, + hub_destination, vocab_file) + + # Restores a hub KerasLayer. + hub_layer = hub.KerasLayer(hub_destination, trainable=True) + + if hasattr(hub_layer, "resolved_object"): + # Checks meta attributes. + self.assertTrue(hub_layer.resolved_object.do_lower_case.numpy()) + with tf.io.gfile.GFile( + hub_layer.resolved_object.vocab_file.asset_path.numpy()) as f: + self.assertEqual("dummy content", f.read()) + # Checks the hub KerasLayer. + for source_weight, hub_weight in zip(bert_model.trainable_weights, + hub_layer.trainable_weights): + self.assertAllClose(source_weight.numpy(), hub_weight.numpy()) + + dummy_ids = np.zeros((2, 10), dtype=np.int32) + hub_outputs = hub_layer([dummy_ids, dummy_ids, dummy_ids]) + source_outputs = bert_model([dummy_ids, dummy_ids, dummy_ids]) + + # The outputs of hub module are "pooled_output" and "sequence_output", + # while the outputs of encoder is in reversed order, i.e., + # "sequence_output" and "pooled_output". + encoder_outputs = reversed(encoder([dummy_ids, dummy_ids, dummy_ids])) + self.assertEqual(hub_outputs[0].shape, (2, 16)) + self.assertEqual(hub_outputs[1].shape, (2, 10, 16)) + for source_output, hub_output, encoder_output in zip( + source_outputs, hub_outputs, encoder_outputs): + self.assertAllClose(source_output.numpy(), hub_output.numpy()) + self.assertAllClose(source_output.numpy(), encoder_output.numpy()) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/input_pipeline.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/input_pipeline.py new file mode 100644 index 0000000..f8efb92 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/input_pipeline.py @@ -0,0 +1,231 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""BERT model input pipelines.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + + +def decode_record(record, name_to_features): + """Decodes a record to a TensorFlow example.""" + example = tf.io.parse_single_example(record, name_to_features) + + # tf.Example only supports tf.int64, but the TPU only supports tf.int32. + # So cast all int64 to int32. + for name in list(example.keys()): + t = example[name] + if t.dtype == tf.int64: + t = tf.cast(t, tf.int32) + example[name] = t + + return example + + +def single_file_dataset(input_file, name_to_features): + """Creates a single-file dataset to be passed for BERT custom training.""" + # For training, we want a lot of parallel reading and shuffling. + # For eval, we want no shuffling and parallel reading doesn't matter. + d = tf.data.TFRecordDataset(input_file) + d = d.map(lambda record: decode_record(record, name_to_features)) + + # When `input_file` is a path to a single file or a list + # containing a single path, disable auto sharding so that + # same input file is sent to all workers. + if isinstance(input_file, str) or len(input_file) == 1: + options = tf.data.Options() + options.experimental_distribute.auto_shard_policy = ( + tf.data.experimental.AutoShardPolicy.OFF) + d = d.with_options(options) + return d + + +def create_pretrain_dataset(input_patterns, + seq_length, + max_predictions_per_seq, + batch_size, + is_training=True, + input_pipeline_context=None, + use_next_sentence_label=True): + """Creates input dataset from (tf)records files for pretraining.""" + name_to_features = { + 'input_ids': + tf.io.FixedLenFeature([seq_length], tf.int64), + 'input_mask': + tf.io.FixedLenFeature([seq_length], tf.int64), + 'segment_ids': + tf.io.FixedLenFeature([seq_length], tf.int64), + 'masked_lm_positions': + tf.io.FixedLenFeature([max_predictions_per_seq], tf.int64), + 'masked_lm_ids': + tf.io.FixedLenFeature([max_predictions_per_seq], tf.int64), + 'masked_lm_weights': + tf.io.FixedLenFeature([max_predictions_per_seq], tf.float32), + } + if use_next_sentence_label: + name_to_features['next_sentence_labels'] = tf.io.FixedLenFeature([1], + tf.int64) + + for input_pattern in input_patterns: + if not tf.io.gfile.glob(input_pattern): + raise ValueError('%s does not match any files.' % input_pattern) + + dataset = tf.data.Dataset.list_files(input_patterns, shuffle=is_training) + + if input_pipeline_context and input_pipeline_context.num_input_pipelines > 1: + dataset = dataset.shard(input_pipeline_context.num_input_pipelines, + input_pipeline_context.input_pipeline_id) + if is_training: + dataset = dataset.repeat() + + # We set shuffle buffer to exactly match total number of + # training files to ensure that training data is well shuffled. + input_files = [] + for input_pattern in input_patterns: + input_files.extend(tf.io.gfile.glob(input_pattern)) + dataset = dataset.shuffle(len(input_files)) + + # In parallel, create tf record dataset for each train files. + # cycle_length = 8 means that up to 8 files will be read and deserialized in + # parallel. You may want to increase this number if you have a large number of + # CPU cores. + dataset = dataset.interleave( + tf.data.TFRecordDataset, cycle_length=8, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + + decode_fn = lambda record: decode_record(record, name_to_features) + dataset = dataset.map( + decode_fn, num_parallel_calls=tf.data.experimental.AUTOTUNE) + + def _select_data_from_record(record): + """Filter out features to use for pretraining.""" + x = { + 'input_word_ids': record['input_ids'], + 'input_mask': record['input_mask'], + 'input_type_ids': record['segment_ids'], + 'masked_lm_positions': record['masked_lm_positions'], + 'masked_lm_ids': record['masked_lm_ids'], + 'masked_lm_weights': record['masked_lm_weights'], + } + if use_next_sentence_label: + x['next_sentence_labels'] = record['next_sentence_labels'] + + y = record['masked_lm_weights'] + + return (x, y) + + dataset = dataset.map( + _select_data_from_record, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + + if is_training: + dataset = dataset.shuffle(100) + + dataset = dataset.batch(batch_size, drop_remainder=is_training) + dataset = dataset.prefetch(1024) + return dataset + + +def create_classifier_dataset(file_path, + seq_length, + batch_size, + is_training=True, + input_pipeline_context=None): + """Creates input dataset from (tf)records files for train/eval.""" + name_to_features = { + 'input_ids': tf.io.FixedLenFeature([seq_length], tf.int64), + 'input_mask': tf.io.FixedLenFeature([seq_length], tf.int64), + 'segment_ids': tf.io.FixedLenFeature([seq_length], tf.int64), + 'label_ids': tf.io.FixedLenFeature([], tf.int64), + 'is_real_example': tf.io.FixedLenFeature([], tf.int64), + } + dataset = single_file_dataset(file_path, name_to_features) + + # The dataset is always sharded by number of hosts. + # num_input_pipelines is the number of hosts rather than number of cores. + if input_pipeline_context and input_pipeline_context.num_input_pipelines > 1: + dataset = dataset.shard(input_pipeline_context.num_input_pipelines, + input_pipeline_context.input_pipeline_id) + + def _select_data_from_record(record): + x = { + 'input_word_ids': record['input_ids'], + 'input_mask': record['input_mask'], + 'input_type_ids': record['segment_ids'] + } + y = record['label_ids'] + return (x, y) + + dataset = dataset.map(_select_data_from_record) + + if is_training: + dataset = dataset.shuffle(100) + dataset = dataset.repeat() + + dataset = dataset.batch(batch_size, drop_remainder=is_training) + dataset = dataset.prefetch(1024) + return dataset + + +def create_squad_dataset(file_path, + seq_length, + batch_size, + is_training=True, + input_pipeline_context=None): + """Creates input dataset from (tf)records files for train/eval.""" + name_to_features = { + 'input_ids': tf.io.FixedLenFeature([seq_length], tf.int64), + 'input_mask': tf.io.FixedLenFeature([seq_length], tf.int64), + 'segment_ids': tf.io.FixedLenFeature([seq_length], tf.int64), + } + if is_training: + name_to_features['start_positions'] = tf.io.FixedLenFeature([], tf.int64) + name_to_features['end_positions'] = tf.io.FixedLenFeature([], tf.int64) + else: + name_to_features['unique_ids'] = tf.io.FixedLenFeature([], tf.int64) + + dataset = single_file_dataset(file_path, name_to_features) + + # The dataset is always sharded by number of hosts. + # num_input_pipelines is the number of hosts rather than number of cores. + if input_pipeline_context and input_pipeline_context.num_input_pipelines > 1: + dataset = dataset.shard(input_pipeline_context.num_input_pipelines, + input_pipeline_context.input_pipeline_id) + + def _select_data_from_record(record): + """Dispatches record to features and labels.""" + x, y = {}, {} + for name, tensor in record.items(): + if name in ('start_positions', 'end_positions'): + y[name] = tensor + elif name == 'input_ids': + x['input_word_ids'] = tensor + elif name == 'segment_ids': + x['input_type_ids'] = tensor + else: + x[name] = tensor + return (x, y) + + dataset = dataset.map(_select_data_from_record) + + if is_training: + dataset = dataset.shuffle(100) + dataset = dataset.repeat() + + dataset = dataset.batch(batch_size, drop_remainder=True) + dataset = dataset.prefetch(1024) + return dataset diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/model_saving_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/model_saving_utils.py new file mode 100644 index 0000000..e8e8fa8 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/model_saving_utils.py @@ -0,0 +1,101 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities to save models.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import os + +from absl import logging +import tensorflow as tf +import typing + + +def export_bert_model(model_export_path: typing.Text, + model: tf.keras.Model, + checkpoint_dir: typing.Optional[typing.Text] = None, + restore_model_using_load_weights: bool = False) -> None: + """Export BERT model for serving which does not include the optimizer. + + Arguments: + model_export_path: Path to which exported model will be saved. + model: Keras model object to export. + checkpoint_dir: Path from which model weights will be loaded, if + specified. + restore_model_using_load_weights: Whether to use checkpoint.restore() API + for custom checkpoint or to use model.load_weights() API. + There are 2 different ways to save checkpoints. One is using + tf.train.Checkpoint and another is using Keras model.save_weights(). + Custom training loop implementation uses tf.train.Checkpoint API + and Keras ModelCheckpoint callback internally uses model.save_weights() + API. Since these two API's cannot be used toghether, model loading logic + must be take into account how model checkpoint was saved. + + Raises: + ValueError when either model_export_path or model is not specified. + """ + if not model_export_path: + raise ValueError('model_export_path must be specified.') + if not isinstance(model, tf.keras.Model): + raise ValueError('model must be a tf.keras.Model object.') + + if checkpoint_dir: + # Keras compile/fit() was used to save checkpoint using + # model.save_weights(). + if restore_model_using_load_weights: + model_weight_path = os.path.join(checkpoint_dir, 'checkpoint') + assert tf.io.gfile.exists(model_weight_path) + model.load_weights(model_weight_path) + + # tf.train.Checkpoint API was used via custom training loop logic. + else: + checkpoint = tf.train.Checkpoint(model=model) + + # Restores the model from latest checkpoint. + latest_checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir) + assert latest_checkpoint_file + logging.info('Checkpoint file %s found and restoring from ' + 'checkpoint', latest_checkpoint_file) + checkpoint.restore( + latest_checkpoint_file).assert_existing_objects_matched() + + model.save(model_export_path, include_optimizer=False, save_format='tf') + + +class BertModelCheckpoint(tf.keras.callbacks.Callback): + """Keras callback that saves model at the end of every epoch.""" + + def __init__(self, checkpoint_dir, checkpoint): + """Initializes BertModelCheckpoint. + + Arguments: + checkpoint_dir: Directory of the to be saved checkpoint file. + checkpoint: tf.train.Checkpoint object. + """ + super(BertModelCheckpoint, self).__init__() + self.checkpoint_file_name = os.path.join( + checkpoint_dir, 'bert_training_checkpoint_step_{global_step}.ckpt') + assert isinstance(checkpoint, tf.train.Checkpoint) + self.checkpoint = checkpoint + + def on_epoch_end(self, epoch, logs=None): + global_step = tf.keras.backend.get_value(self.model.optimizer.iterations) + formatted_file_name = self.checkpoint_file_name.format( + global_step=global_step) + saved_path = self.checkpoint.save(formatted_file_name) + logging.info('Saving model TF checkpoint to : %s', saved_path) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/model_training_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/model_training_utils.py new file mode 100644 index 0000000..a8c4adb --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/model_training_utils.py @@ -0,0 +1,491 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A light weight utilities to train NLP models.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import os +import tempfile + +from absl import logging +import tensorflow as tf +from official.staging.training import grad_utils +from official.utils.misc import distribution_utils + +_SUMMARY_TXT = 'training_summary.txt' +_MIN_SUMMARY_STEPS = 10 + + +def _should_export_checkpoint(strategy): + return (not strategy) or strategy.extended.should_checkpoint + + +def _should_export_summary(strategy): + return (not strategy) or strategy.extended.should_save_summary + + +def _save_checkpoint(strategy, checkpoint, model_dir, checkpoint_prefix): + """Saves model to with provided checkpoint prefix.""" + + if _should_export_checkpoint(strategy): + checkpoint_path = os.path.join(model_dir, checkpoint_prefix) + saved_path = checkpoint.save(checkpoint_path) + logging.info('Saving model as TF checkpoint: %s', saved_path) + else: + # In multi worker training we need every worker to save checkpoint, because + # variables can trigger synchronization on read and synchronization needs + # all workers to participate. To avoid workers overriding each other we save + # to a temporary directory on non-chief workers. + tmp_dir = tempfile.mkdtemp() + checkpoint.save(os.path.join(tmp_dir, 'ckpt')) + tf.io.gfile.rmtree(tmp_dir) + return + + +def _get_input_iterator(input_fn, strategy): + """Returns distributed dataset iterator.""" + # When training with TPU pods, datasets needs to be cloned across + # workers. Since Dataset instance cannot be cloned in eager mode, we instead + # pass callable that returns a dataset. + if not callable(input_fn): + raise ValueError('`input_fn` should be a closure that returns a dataset.') + iterator = iter( + strategy.experimental_distribute_datasets_from_function(input_fn)) + return iterator + + +def _float_metric_value(metric): + """Gets the value of a float-value keras metric.""" + return metric.result().numpy().astype(float) + + +def steps_to_run(current_step, steps_per_epoch, steps_per_loop): + """Calculates steps to run on device.""" + if steps_per_loop <= 0: + raise ValueError('steps_per_loop should be positive integer.') + if steps_per_loop == 1: + return steps_per_loop + remainder_in_epoch = current_step % steps_per_epoch + if remainder_in_epoch != 0: + return min(steps_per_epoch - remainder_in_epoch, steps_per_loop) + else: + return steps_per_loop + + +def write_txt_summary(training_summary, summary_dir): + """Writes a summary text file to record stats.""" + summary_path = os.path.join(summary_dir, _SUMMARY_TXT) + with tf.io.gfile.GFile(summary_path, 'wb') as f: + logging.info('Training Summary: \n%s', str(training_summary)) + f.write(json.dumps(training_summary, indent=4)) + + +def run_customized_training_loop( + # pylint: disable=invalid-name + _sentinel=None, + # pylint: enable=invalid-name + strategy=None, + model_fn=None, + loss_fn=None, + scale_loss=True, + model_dir=None, + train_input_fn=None, + steps_per_epoch=None, + steps_per_loop=1, + epochs=1, + eval_input_fn=None, + eval_steps=None, + metric_fn=None, + init_checkpoint=None, + custom_callbacks=None, + run_eagerly=False, + sub_model_export_name=None, + explicit_allreduce=False, + pre_allreduce_callbacks=None, + post_allreduce_callbacks=None): + """Run BERT pretrain model training using low-level API. + + Arguments: + _sentinel: Used to prevent positional parameters. Internal, do not use. + strategy: Distribution strategy on which to run low level training loop. + model_fn: Function that returns a tuple (model, sub_model). Caller of this + function should add optimizer to the `model` via calling + `model.compile()` API or manually setting `model.optimizer` attribute. + Second element of the returned tuple(sub_model) is an optional sub model + to be used for initial checkpoint -- if provided. + loss_fn: Function with signature func(labels, logits) and returns a loss + tensor. + scale_loss: Whether to divide the raw loss by number of replicas before + gradients calculation. + model_dir: Model directory used during training for restoring/saving model + weights. + train_input_fn: Function that returns a tf.data.Dataset used for training. + steps_per_epoch: Number of steps to run per epoch. At the end of each + epoch, model checkpoint will be saved and evaluation will be conducted + if evaluation dataset is provided. + steps_per_loop: Number of steps per graph-mode loop. In order to reduce + communication in eager context, training logs are printed every + steps_per_loop. + epochs: Number of epochs to train. + eval_input_fn: Function that returns evaluation dataset. If none, + evaluation is skipped. + eval_steps: Number of steps to run evaluation. Required if `eval_input_fn` + is not none. + metric_fn: A metrics function that returns a Keras Metric object to record + evaluation result using evaluation dataset or with training dataset + after every epoch. + init_checkpoint: Optional checkpoint to load to `sub_model` returned by + `model_fn`. + custom_callbacks: A list of Keras Callbacks objects to run during + training. More specifically, `on_batch_begin()`, `on_batch_end()`, + methods are invoked during training. + run_eagerly: Whether to run model training in pure eager execution. This + should be disable for TPUStrategy. + sub_model_export_name: If not None, will export `sub_model` returned by + `model_fn` into checkpoint files. The name of intermediate checkpoint + file is {sub_model_export_name}_step_{step}.ckpt and the last + checkpint's name is {sub_model_export_name}.ckpt; + if None, `sub_model` will not be exported as checkpoint. + explicit_allreduce: Whether to explicitly perform gradient allreduce, + instead of relying on implicit allreduce in optimizer.apply_gradients(). + default is False. For now, if training using FP16 mixed precision, + explicit allreduce will aggregate gradients in FP16 format. For TPU and + GPU training using FP32, explicit allreduce will aggregate gradients in + FP32 format. + pre_allreduce_callbacks: A list of callback functions that takes gradients + and model variables pairs as input, manipulate them, and returns a new + gradients and model variables paris. The callback functions will be + invoked in the list order and before gradients are allreduced. + With mixed precision training, the pre_allreduce_allbacks will be + applied on scaled_gradients. Default is no callbacks. + Only used when explicit_allreduce=True. + post_allreduce_callbacks: A list of callback functions that takes + gradients and model variables pairs as input, manipulate them, and + returns a new gradients and model variables paris. The callback + functions will be invoked in the list order and right before gradients + are applied to variables for updates. Default is no callbacks. Only used + when explicit_allreduce=True. + + Returns: + Trained model. + + Raises: + ValueError: (1) When model returned by `model_fn` does not have optimizer + attribute or when required parameters are set to none. (2) eval args are + not specified correctly. (3) metric_fn must be a callable if specified. + (4) sub_model_checkpoint_name is specified, but `sub_model` returned + by `model_fn` is None. + """ + + if _sentinel is not None: + raise ValueError('only call `run_customized_training_loop()` ' + 'with named arguments.') + + required_arguments = [ + strategy, model_fn, loss_fn, model_dir, steps_per_epoch, train_input_fn + ] + if [arg for arg in required_arguments if arg is None]: + raise ValueError('`strategy`, `model_fn`, `loss_fn`, `model_dir`, ' + '`steps_per_loop` and `steps_per_epoch` are required ' + 'parameters.') + if steps_per_loop > steps_per_epoch: + logging.error( + 'steps_per_loop: %d is specified to be greater than ' + ' steps_per_epoch: %d, we will use steps_per_epoch as' + ' steps_per_loop.', steps_per_loop, steps_per_epoch) + steps_per_loop = steps_per_epoch + assert tf.executing_eagerly() + + if run_eagerly: + if isinstance(strategy, tf.distribute.experimental.TPUStrategy): + raise ValueError( + 'TPUStrategy should not run eagerly as it heavily relies on graph' + ' optimization for the distributed system.') + + if eval_input_fn and (eval_steps is None or metric_fn is None): + raise ValueError( + '`eval_step` and `metric_fn` are required when `eval_input_fn ` ' + 'is not none.') + if metric_fn and not callable(metric_fn): + raise ValueError( + 'if `metric_fn` is specified, metric_fn must be a callable.') + + total_training_steps = steps_per_epoch * epochs + train_iterator = _get_input_iterator(train_input_fn, strategy) + + with distribution_utils.get_strategy_scope(strategy): + # To correctly place the model weights on accelerators, + # model and optimizer should be created in scope. + model, sub_model = model_fn() + if not hasattr(model, 'optimizer'): + raise ValueError('User should set optimizer attribute to model ' + 'inside `model_fn`.') + if sub_model_export_name and sub_model is None: + raise ValueError('sub_model_export_name is specified as %s, but ' + 'sub_model is None.' % sub_model_export_name) + + optimizer = model.optimizer + + if init_checkpoint: + logging.info( + 'Checkpoint file %s found and restoring from ' + 'initial checkpoint for core model.', init_checkpoint) + checkpoint = tf.train.Checkpoint(model=sub_model) + checkpoint.restore(init_checkpoint).assert_existing_objects_matched() + logging.info('Loading from checkpoint file completed') + + train_loss_metric = tf.keras.metrics.Mean( + 'training_loss', dtype=tf.float32) + eval_metrics = [metric_fn()] if metric_fn else [] + # If evaluation is required, make a copy of metric as it will be used by + # both train and evaluation. + train_metrics = [ + metric.__class__.from_config(metric.get_config()) + for metric in eval_metrics + ] + + # Create summary writers + if _should_export_summary(strategy): + summary_dir = os.path.join(model_dir, 'summaries') + else: + # In multi worker training we need every worker to write summary, because + # variables can trigger synchronization on read and synchronization needs + # all workers to participate. + summary_dir = tempfile.mkdtemp() + eval_summary_writer = tf.summary.create_file_writer( + os.path.join(summary_dir, 'eval')) + if steps_per_loop >= _MIN_SUMMARY_STEPS: + # Only writes summary when the stats are collected sufficiently over + # enough steps. + train_summary_writer = tf.summary.create_file_writer( + os.path.join(summary_dir, 'train')) + else: + train_summary_writer = None + + # Collects training variables. + training_vars = model.trainable_variables + + def _replicated_step(inputs): + """Replicated training step.""" + + inputs, labels = inputs + with tf.GradientTape() as tape: + model_outputs = model(inputs, training=True) + loss = loss_fn(labels, model_outputs) + # Raw loss is used for reporting in metrics/logs. + raw_loss = loss + if scale_loss: + # Scales down the loss for gradients to be invariant from replicas. + loss = loss / strategy.num_replicas_in_sync + + if explicit_allreduce: + grad_utils.minimize_using_explicit_allreduce(tape, optimizer, loss, + training_vars, + pre_allreduce_callbacks, + post_allreduce_callbacks) + else: + if isinstance(optimizer, + tf.keras.mixed_precision.experimental.LossScaleOptimizer): + with tape: + scaled_loss = optimizer.get_scaled_loss(loss) + scaled_grads = tape.gradient(scaled_loss, training_vars) + grads = optimizer.get_unscaled_gradients(scaled_grads) + else: + grads = tape.gradient(loss, training_vars) + optimizer.apply_gradients(zip(grads, training_vars)) + # For reporting, the metric takes the mean of losses. + train_loss_metric.update_state(raw_loss) + for metric in train_metrics: + metric.update_state(labels, model_outputs) + + @tf.function + def train_steps(iterator, steps): + """Performs distributed training steps in a loop. + + Args: + iterator: the distributed iterator of training datasets. + steps: an tf.int32 integer tensor to specify number of steps to run + inside host training loop. + + Raises: + ValueError: Any of the arguments or tensor shapes are invalid. + """ + if not isinstance(steps, tf.Tensor): + raise ValueError('steps should be an Tensor. Python object may cause ' + 'retracing.') + + for _ in tf.range(steps): + strategy.run(_replicated_step, args=(next(iterator),)) + + def train_single_step(iterator): + """Performs a distributed training step. + + Args: + iterator: the distributed iterator of training datasets. + + Raises: + ValueError: Any of the arguments or tensor shapes are invalid. + """ + strategy.run(_replicated_step, args=(next(iterator),)) + + def test_step(iterator): + """Calculates evaluation metrics on distributed devices.""" + + def _test_step_fn(inputs): + """Replicated accuracy calculation.""" + + inputs, labels = inputs + model_outputs = model(inputs, training=False) + for metric in eval_metrics: + metric.update_state(labels, model_outputs) + + strategy.run(_test_step_fn, args=(next(iterator),)) + + if not run_eagerly: + train_single_step = tf.function(train_single_step) + test_step = tf.function(test_step) + + def _run_evaluation(current_training_step, test_iterator): + """Runs validation steps and aggregate metrics.""" + for _ in range(eval_steps): + test_step(test_iterator) + + with eval_summary_writer.as_default(): + for metric in eval_metrics + model.metrics: + metric_value = _float_metric_value(metric) + logging.info('Step: [%d] Validation %s = %f', current_training_step, + metric.name, metric_value) + tf.summary.scalar( + metric.name, metric_value, step=current_training_step) + eval_summary_writer.flush() + + def _run_callbacks_on_batch_begin(batch): + """Runs custom callbacks at the start of every step.""" + if not custom_callbacks: + return + for callback in custom_callbacks: + callback.on_batch_begin(batch) + + def _run_callbacks_on_batch_end(batch, logs): + """Runs custom callbacks at the end of every step.""" + if not custom_callbacks: + return + for callback in custom_callbacks: + callback.on_batch_end(batch, logs) + + # Training loop starts here. + checkpoint = tf.train.Checkpoint(model=model, optimizer=optimizer) + sub_model_checkpoint = tf.train.Checkpoint( + model=sub_model) if sub_model_export_name else None + + latest_checkpoint_file = tf.train.latest_checkpoint(model_dir) + if latest_checkpoint_file: + logging.info( + 'Checkpoint file %s found and restoring from ' + 'checkpoint', latest_checkpoint_file) + checkpoint.restore(latest_checkpoint_file) + logging.info('Loading from checkpoint file completed') + + current_step = optimizer.iterations.numpy() + checkpoint_name = 'ctl_step_{step}.ckpt' + + while current_step < total_training_steps: + # Training loss/metric are taking average over steps inside micro + # training loop. We reset the their values before each round. + train_loss_metric.reset_states() + for metric in train_metrics + model.metrics: + metric.reset_states() + + _run_callbacks_on_batch_begin(current_step) + # Runs several steps in the host while loop. + steps = steps_to_run(current_step, steps_per_epoch, steps_per_loop) + + if tf.config.list_physical_devices('GPU'): + # TODO(zongweiz): merge with train_steps once tf.while_loop + # GPU performance bugs are fixed. + for _ in range(steps): + train_single_step(train_iterator) + else: + # Converts steps to a Tensor to avoid tf.function retracing. + train_steps(train_iterator, + tf.convert_to_tensor(steps, dtype=tf.int32)) + train_loss = _float_metric_value(train_loss_metric) + current_step += steps + _run_callbacks_on_batch_end(current_step - 1, {'loss': train_loss}) + + # Updates training logging. + training_status = 'Train Step: %d/%d / loss = %s' % ( + current_step, total_training_steps, train_loss) + + if train_summary_writer: + with train_summary_writer.as_default(): + tf.summary.scalar( + train_loss_metric.name, train_loss, step=current_step) + for metric in train_metrics + model.metrics: + metric_value = _float_metric_value(metric) + training_status += ' %s = %f' % (metric.name, metric_value) + tf.summary.scalar(metric.name, metric_value, step=current_step) + train_summary_writer.flush() + logging.info(training_status) + + # Saves model checkpoints and run validation steps at every epoch end. + if current_step % steps_per_epoch == 0: + # To avoid repeated model saving, we do not save after the last + # step of training. + if current_step < total_training_steps: + _save_checkpoint(strategy, checkpoint, model_dir, + checkpoint_name.format(step=current_step)) + if sub_model_export_name: + _save_checkpoint( + strategy, sub_model_checkpoint, model_dir, + '%s_step_%d.ckpt' % (sub_model_export_name, current_step)) + if eval_input_fn: + logging.info('Running evaluation after step: %s.', current_step) + _run_evaluation(current_step, + _get_input_iterator(eval_input_fn, strategy)) + # Re-initialize evaluation metric. + for metric in eval_metrics + model.metrics: + metric.reset_states() + + _save_checkpoint(strategy, checkpoint, model_dir, + checkpoint_name.format(step=current_step)) + if sub_model_export_name: + _save_checkpoint(strategy, sub_model_checkpoint, model_dir, + '%s.ckpt' % sub_model_export_name) + + if eval_input_fn: + logging.info('Running final evaluation after training is complete.') + _run_evaluation(current_step, + _get_input_iterator(eval_input_fn, strategy)) + + training_summary = { + 'total_training_steps': total_training_steps, + 'train_loss': _float_metric_value(train_loss_metric), + } + if eval_metrics: + # TODO(hongkuny): Cleans up summary reporting in text. + training_summary['last_train_metrics'] = _float_metric_value( + train_metrics[0]) + training_summary['eval_metrics'] = _float_metric_value(eval_metrics[0]) + + write_txt_summary(training_summary, summary_dir) + + if not _should_export_summary(strategy): + tf.io.gfile.rmtree(summary_dir) + + return model diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/model_training_utils_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/model_training_utils_test.py new file mode 100644 index 0000000..87ef7b6 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/model_training_utils_test.py @@ -0,0 +1,236 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for official.modeling.training.model_training_utils.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from absl import logging +from absl.testing import parameterized +from absl.testing.absltest import mock +import numpy as np +import tensorflow as tf + +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +from official.nlp.bert import model_training_utils + + +def eager_strategy_combinations(): + return combinations.combine( + distribution=[ + strategy_combinations.default_strategy, + strategy_combinations.tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + strategy_combinations.mirrored_strategy_with_gpu_and_cpu, + strategy_combinations.mirrored_strategy_with_two_gpus, + ], + mode='eager', + ) + + +def eager_gpu_strategy_combinations(): + return combinations.combine( + distribution=[ + strategy_combinations.default_strategy, + strategy_combinations.one_device_strategy_gpu, + strategy_combinations.mirrored_strategy_with_gpu_and_cpu, + strategy_combinations.mirrored_strategy_with_two_gpus, + ], + mode='eager', + ) + + +def create_fake_data_input_fn(batch_size, features_shape, num_classes): + """Creates a dummy input function with the given feature and label shapes. + + Args: + batch_size: integer. + features_shape: list[int]. Feature shape for an individual example. + num_classes: integer. Number of labels. + + Returns: + An input function that is usable in the executor. + """ + + def _dataset_fn(input_context=None): + """An input function for generating fake data.""" + local_batch_size = input_context.get_per_replica_batch_size(batch_size) + features = np.random.rand(64, *features_shape) + labels = np.random.randint(2, size=[64, num_classes]) + # Convert the inputs to a Dataset. + dataset = tf.data.Dataset.from_tensor_slices((features, labels)) + dataset = dataset.shard(input_context.num_input_pipelines, + input_context.input_pipeline_id) + + def _assign_dtype(features, labels): + features = tf.cast(features, tf.float32) + labels = tf.cast(labels, tf.float32) + return features, labels + + # Shuffle, repeat, and batch the examples. + dataset = dataset.map(_assign_dtype) + dataset = dataset.shuffle(64).repeat() + dataset = dataset.batch(local_batch_size, drop_remainder=True) + dataset = dataset.prefetch(buffer_size=64) + return dataset + + return _dataset_fn + + +def create_model_fn(input_shape, num_classes, use_float16=False): + + def _model_fn(): + """A one-layer softmax model suitable for testing.""" + input_layer = tf.keras.layers.Input(shape=input_shape) + x = tf.keras.layers.Dense(num_classes, activation='relu')(input_layer) + output_layer = tf.keras.layers.Dense(num_classes, activation='softmax')(x) + sub_model = tf.keras.models.Model(input_layer, x, name='sub_model') + model = tf.keras.models.Model(input_layer, output_layer, name='model') + model.add_metric( + tf.reduce_mean(input_layer), name='mean_input', aggregation='mean') + model.optimizer = tf.keras.optimizers.SGD(learning_rate=0.1, momentum=0.9) + if use_float16: + model.optimizer = ( + tf.keras.mixed_precision.experimental.LossScaleOptimizer( + model.optimizer, loss_scale='dynamic')) + return model, sub_model + + return _model_fn + + +def metric_fn(): + """Gets a tf.keras metric object.""" + return tf.keras.metrics.CategoricalAccuracy(name='accuracy', dtype=tf.float32) + + +def summaries_with_matching_keyword(keyword, summary_dir): + """Yields summary protos matching given keyword from event file.""" + event_paths = tf.io.gfile.glob(os.path.join(summary_dir, 'events*')) + for event in tf.compat.v1.train.summary_iterator(event_paths[-1]): + if event.summary is not None: + for value in event.summary.value: + if keyword in value.tag: + logging.error(event) + yield event.summary + + +def check_eventfile_for_keyword(keyword, summary_dir): + """Checks event files for the keyword.""" + return any(summaries_with_matching_keyword(keyword, summary_dir)) + + +class ModelTrainingUtilsTest(tf.test.TestCase, parameterized.TestCase): + + def setUp(self): + super(ModelTrainingUtilsTest, self).setUp() + self._model_fn = create_model_fn(input_shape=[128], num_classes=3) + + def run_training(self, strategy, model_dir, steps_per_loop, run_eagerly): + input_fn = create_fake_data_input_fn( + batch_size=8, features_shape=[128], num_classes=3) + model_training_utils.run_customized_training_loop( + strategy=strategy, + model_fn=self._model_fn, + loss_fn=tf.keras.losses.categorical_crossentropy, + model_dir=model_dir, + steps_per_epoch=20, + steps_per_loop=steps_per_loop, + epochs=2, + train_input_fn=input_fn, + eval_input_fn=input_fn, + eval_steps=10, + init_checkpoint=None, + metric_fn=metric_fn, + custom_callbacks=None, + run_eagerly=run_eagerly) + + @combinations.generate(eager_strategy_combinations()) + def test_train_eager_single_step(self, distribution): + model_dir = self.get_temp_dir() + if isinstance(distribution, tf.distribute.experimental.TPUStrategy): + with self.assertRaises(ValueError): + self.run_training( + distribution, model_dir, steps_per_loop=1, run_eagerly=True) + else: + self.run_training( + distribution, model_dir, steps_per_loop=1, run_eagerly=True) + + @combinations.generate(eager_gpu_strategy_combinations()) + def test_train_eager_mixed_precision(self, distribution): + model_dir = self.get_temp_dir() + policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16') + tf.keras.mixed_precision.experimental.set_policy(policy) + self._model_fn = create_model_fn( + input_shape=[128], num_classes=3, use_float16=True) + self.run_training( + distribution, model_dir, steps_per_loop=1, run_eagerly=True) + + @combinations.generate(eager_strategy_combinations()) + def test_train_check_artifacts(self, distribution): + model_dir = self.get_temp_dir() + self.run_training( + distribution, model_dir, steps_per_loop=10, run_eagerly=False) + + # Two checkpoints should be saved after two epochs. + self.assertNotEmpty(tf.io.gfile.glob(os.path.join(model_dir, 'ctl_step_*'))) + self.assertNotEmpty( + tf.io.gfile.glob( + os.path.join(model_dir, 'summaries/training_summary*'))) + + # Loss and accuracy values should be written into summaries. + self.assertTrue( + check_eventfile_for_keyword('loss', + os.path.join(model_dir, 'summaries/train'))) + self.assertTrue( + check_eventfile_for_keyword('accuracy', + os.path.join(model_dir, 'summaries/train'))) + self.assertTrue( + check_eventfile_for_keyword('mean_input', + os.path.join(model_dir, 'summaries/train'))) + self.assertTrue( + check_eventfile_for_keyword('accuracy', + os.path.join(model_dir, 'summaries/eval'))) + self.assertTrue( + check_eventfile_for_keyword('mean_input', + os.path.join(model_dir, 'summaries/eval'))) + + @combinations.generate( + combinations.combine( + distribution=[ + strategy_combinations.one_device_strategy_gpu, + ], + mode='eager', + )) + def test_train_check_artifacts_non_chief(self, distribution): + # We shouldn't export artifacts on non-chief workers. Since there's no easy + # way to test with real MultiWorkerMirroredStrategy, we patch the strategy + # to make it as if it's MultiWorkerMirroredStrategy on non-chief workers. + extended = distribution.extended + with mock.patch.object(extended.__class__, 'should_checkpoint', + new_callable=mock.PropertyMock, return_value=False), \ + mock.patch.object(extended.__class__, 'should_save_summary', + new_callable=mock.PropertyMock, return_value=False): + model_dir = self.get_temp_dir() + self.run_training( + distribution, model_dir, steps_per_loop=10, run_eagerly=False) + self.assertEmpty(tf.io.gfile.listdir(model_dir)) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/run_classifier.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/run_classifier.py new file mode 100644 index 0000000..d737deb --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/run_classifier.py @@ -0,0 +1,432 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""BERT classification finetuning runner in TF 2.x.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import math +import os + +from absl import app +from absl import flags +from absl import logging +import tensorflow as tf +from official.modeling import performance +from official.nlp import optimization +from official.nlp.bert import bert_models +from official.nlp.bert import common_flags +from official.nlp.bert import configs as bert_configs +from official.nlp.bert import input_pipeline +from official.nlp.bert import model_saving_utils +from official.nlp.bert import model_training_utils +from official.utils.misc import distribution_utils +from official.utils.misc import keras_utils + + +flags.DEFINE_enum( + 'mode', 'train_and_eval', ['train_and_eval', 'export_only'], + 'One of {"train_and_eval", "export_only"}. `train_and_eval`: ' + 'trains the model and evaluates in the meantime. ' + '`export_only`: will take the latest checkpoint inside ' + 'model_dir and export a `SavedModel`.') +flags.DEFINE_string('train_data_path', None, + 'Path to training data for BERT classifier.') +flags.DEFINE_string('eval_data_path', None, + 'Path to evaluation data for BERT classifier.') +# Model training specific flags. +flags.DEFINE_string( + 'input_meta_data_path', None, + 'Path to file that contains meta data about input ' + 'to be used for training and evaluation.') +flags.DEFINE_integer('train_batch_size', 32, 'Batch size for training.') +flags.DEFINE_integer('eval_batch_size', 32, 'Batch size for evaluation.') + +common_flags.define_common_bert_flags() + +FLAGS = flags.FLAGS + + +def get_loss_fn(num_classes): + """Gets the classification loss function.""" + + def classification_loss_fn(labels, logits): + """Classification loss.""" + labels = tf.squeeze(labels) + log_probs = tf.nn.log_softmax(logits, axis=-1) + one_hot_labels = tf.one_hot( + tf.cast(labels, dtype=tf.int32), depth=num_classes, dtype=tf.float32) + per_example_loss = -tf.reduce_sum( + tf.cast(one_hot_labels, dtype=tf.float32) * log_probs, axis=-1) + return tf.reduce_mean(per_example_loss) + + return classification_loss_fn + + +def get_dataset_fn(input_file_pattern, max_seq_length, global_batch_size, + is_training): + """Gets a closure to create a dataset.""" + + def _dataset_fn(ctx=None): + """Returns tf.data.Dataset for distributed BERT pretraining.""" + batch_size = ctx.get_per_replica_batch_size( + global_batch_size) if ctx else global_batch_size + dataset = input_pipeline.create_classifier_dataset( + input_file_pattern, + max_seq_length, + batch_size, + is_training=is_training, + input_pipeline_context=ctx) + return dataset + + return _dataset_fn + + +def run_bert_classifier(strategy, + bert_config, + input_meta_data, + model_dir, + epochs, + steps_per_epoch, + steps_per_loop, + eval_steps, + warmup_steps, + initial_lr, + init_checkpoint, + train_input_fn, + eval_input_fn, + custom_callbacks=None, + run_eagerly=False, + use_keras_compile_fit=False): + """Run BERT classifier training using low-level API.""" + max_seq_length = input_meta_data['max_seq_length'] + num_classes = input_meta_data['num_labels'] + + def _get_classifier_model(): + """Gets a classifier model.""" + classifier_model, core_model = ( + bert_models.classifier_model( + bert_config, + num_classes, + max_seq_length, + hub_module_url=FLAGS.hub_module_url, + hub_module_trainable=FLAGS.hub_module_trainable)) + optimizer = optimization.create_optimizer( + initial_lr, steps_per_epoch * epochs, warmup_steps, + FLAGS.end_lr, FLAGS.optimizer_type) + classifier_model.optimizer = performance.configure_optimizer( + optimizer, + use_float16=common_flags.use_float16(), + use_graph_rewrite=common_flags.use_graph_rewrite()) + return classifier_model, core_model + + loss_fn = get_loss_fn(num_classes) + + # Defines evaluation metrics function, which will create metrics in the + # correct device and strategy scope. + def metric_fn(): + return tf.keras.metrics.SparseCategoricalAccuracy( + 'test_accuracy', dtype=tf.float32) + + if use_keras_compile_fit: + # Start training using Keras compile/fit API. + logging.info('Training using TF 2.0 Keras compile/fit API with ' + 'distribution strategy.') + return run_keras_compile_fit( + model_dir, + strategy, + _get_classifier_model, + train_input_fn, + eval_input_fn, + loss_fn, + metric_fn, + init_checkpoint, + epochs, + steps_per_epoch, + steps_per_loop, + eval_steps, + custom_callbacks=custom_callbacks) + + # Use user-defined loop to start training. + logging.info('Training using customized training loop TF 2.0 with ' + 'distribution strategy.') + return model_training_utils.run_customized_training_loop( + strategy=strategy, + model_fn=_get_classifier_model, + loss_fn=loss_fn, + model_dir=model_dir, + steps_per_epoch=steps_per_epoch, + steps_per_loop=steps_per_loop, + epochs=epochs, + train_input_fn=train_input_fn, + eval_input_fn=eval_input_fn, + eval_steps=eval_steps, + init_checkpoint=init_checkpoint, + metric_fn=metric_fn, + custom_callbacks=custom_callbacks, + run_eagerly=run_eagerly) + + +def run_keras_compile_fit(model_dir, + strategy, + model_fn, + train_input_fn, + eval_input_fn, + loss_fn, + metric_fn, + init_checkpoint, + epochs, + steps_per_epoch, + steps_per_loop, + eval_steps, + custom_callbacks=None): + """Runs BERT classifier model using Keras compile/fit API.""" + + with strategy.scope(): + training_dataset = train_input_fn() + evaluation_dataset = eval_input_fn() + bert_model, sub_model = model_fn() + optimizer = bert_model.optimizer + + if init_checkpoint: + checkpoint = tf.train.Checkpoint(model=sub_model) + checkpoint.restore(init_checkpoint).assert_existing_objects_matched() + + bert_model.compile( + optimizer=optimizer, + loss=loss_fn, + metrics=[metric_fn()], + experimental_steps_per_execution=steps_per_loop) + + summary_dir = os.path.join(model_dir, 'summaries') + summary_callback = tf.keras.callbacks.TensorBoard(summary_dir) + checkpoint_path = os.path.join(model_dir, 'checkpoint') + checkpoint_callback = tf.keras.callbacks.ModelCheckpoint( + checkpoint_path, save_weights_only=True) + + if custom_callbacks is not None: + custom_callbacks += [summary_callback, checkpoint_callback] + else: + custom_callbacks = [summary_callback, checkpoint_callback] + + bert_model.fit( + x=training_dataset, + validation_data=evaluation_dataset, + steps_per_epoch=steps_per_epoch, + epochs=epochs, + validation_steps=eval_steps, + callbacks=custom_callbacks) + + return bert_model + + +def get_predictions_and_labels(strategy, trained_model, eval_input_fn, + eval_steps): + """Obtains predictions of trained model on evaluation data. + + Note that list of labels is returned along with the predictions because the + order changes on distributing dataset over TPU pods. + + Args: + strategy: Distribution strategy. + trained_model: Trained model with preloaded weights. + eval_input_fn: Input function for evaluation data. + eval_steps: Number of evaluation steps. + + Returns: + predictions: List of predictions. + labels: List of gold labels corresponding to predictions. + """ + + @tf.function + def test_step(iterator): + """Computes predictions on distributed devices.""" + + def _test_step_fn(inputs): + """Replicated predictions.""" + inputs, labels = inputs + model_outputs = trained_model(inputs, training=False) + return model_outputs, labels + + outputs, labels = strategy.run( + _test_step_fn, args=(next(iterator),)) + # outputs: current batch logits as a tuple of shard logits + outputs = tf.nest.map_structure(strategy.experimental_local_results, + outputs) + labels = tf.nest.map_structure(strategy.experimental_local_results, labels) + return outputs, labels + + def _run_evaluation(test_iterator): + """Runs evaluation steps.""" + preds, golds = list(), list() + for _ in range(eval_steps): + logits, labels = test_step(test_iterator) + for cur_logits, cur_labels in zip(logits, labels): + preds.extend(tf.math.argmax(cur_logits, axis=1).numpy()) + golds.extend(cur_labels.numpy().tolist()) + return preds, golds + + test_iter = iter( + strategy.experimental_distribute_datasets_from_function(eval_input_fn)) + predictions, labels = _run_evaluation(test_iter) + + return predictions, labels + + +def export_classifier(model_export_path, input_meta_data, + restore_model_using_load_weights, bert_config, model_dir): + """Exports a trained model as a `SavedModel` for inference. + + Args: + model_export_path: a string specifying the path to the SavedModel directory. + input_meta_data: dictionary containing meta data about input and model. + restore_model_using_load_weights: Whether to use checkpoint.restore() API + for custom checkpoint or to use model.load_weights() API. There are 2 + different ways to save checkpoints. One is using tf.train.Checkpoint and + another is using Keras model.save_weights(). Custom training loop + implementation uses tf.train.Checkpoint API and Keras ModelCheckpoint + callback internally uses model.save_weights() API. Since these two API's + cannot be used together, model loading logic must be take into account how + model checkpoint was saved. + bert_config: Bert configuration file to define core bert layers. + model_dir: The directory where the model weights and training/evaluation + summaries are stored. + + Raises: + Export path is not specified, got an empty string or None. + """ + if not model_export_path: + raise ValueError('Export path is not specified: %s' % model_export_path) + if not model_dir: + raise ValueError('Export path is not specified: %s' % model_dir) + + # Export uses float32 for now, even if training uses mixed precision. + tf.keras.mixed_precision.experimental.set_policy('float32') + classifier_model = bert_models.classifier_model( + bert_config, input_meta_data['num_labels'], + input_meta_data['max_seq_length'])[0] + + model_saving_utils.export_bert_model( + model_export_path, + model=classifier_model, + checkpoint_dir=model_dir, + restore_model_using_load_weights=restore_model_using_load_weights) + + +def run_bert(strategy, + input_meta_data, + model_config, + train_input_fn=None, + eval_input_fn=None): + """Run BERT training.""" + if FLAGS.mode == 'export_only': + # As Keras ModelCheckpoint callback used with Keras compile/fit() API + # internally uses model.save_weights() to save checkpoints, we must + # use model.load_weights() when Keras compile/fit() is used. + export_classifier(FLAGS.model_export_path, input_meta_data, + FLAGS.use_keras_compile_fit, + model_config, FLAGS.model_dir) + return + + if FLAGS.mode != 'train_and_eval': + raise ValueError('Unsupported mode is specified: %s' % FLAGS.mode) + # Enables XLA in Session Config. Should not be set for TPU. + keras_utils.set_config_v2(FLAGS.enable_xla) + performance.set_mixed_precision_policy(common_flags.dtype()) + + epochs = FLAGS.num_train_epochs + train_data_size = input_meta_data['train_data_size'] + steps_per_epoch = int(train_data_size / FLAGS.train_batch_size) + warmup_steps = int(epochs * train_data_size * 0.1 / FLAGS.train_batch_size) + eval_steps = int( + math.ceil(input_meta_data['eval_data_size'] / FLAGS.eval_batch_size)) + + if not strategy: + raise ValueError('Distribution strategy has not been specified.') + + if FLAGS.log_steps: + custom_callbacks = [keras_utils.TimeHistory( + batch_size=FLAGS.train_batch_size, + log_steps=FLAGS.log_steps, + logdir=FLAGS.model_dir, + )] + else: + custom_callbacks = None + + trained_model = run_bert_classifier( + strategy, + model_config, + input_meta_data, + FLAGS.model_dir, + epochs, + steps_per_epoch, + FLAGS.steps_per_loop, + eval_steps, + warmup_steps, + FLAGS.learning_rate, + FLAGS.init_checkpoint, + train_input_fn, + eval_input_fn, + run_eagerly=FLAGS.run_eagerly, + use_keras_compile_fit=FLAGS.use_keras_compile_fit, + custom_callbacks=custom_callbacks) + + if FLAGS.model_export_path: + # As Keras ModelCheckpoint callback used with Keras compile/fit() API + # internally uses model.save_weights() to save checkpoints, we must + # use model.load_weights() when Keras compile/fit() is used. + model_saving_utils.export_bert_model( + FLAGS.model_export_path, + model=trained_model, + restore_model_using_load_weights=FLAGS.use_keras_compile_fit) + return trained_model + + +def main(_): + # Users should always run this script under TF 2.x + + with tf.io.gfile.GFile(FLAGS.input_meta_data_path, 'rb') as reader: + input_meta_data = json.loads(reader.read().decode('utf-8')) + + if not FLAGS.model_dir: + FLAGS.model_dir = '/tmp/bert20/' + + strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=FLAGS.distribution_strategy, + num_gpus=FLAGS.num_gpus, + tpu_address=FLAGS.tpu) + max_seq_length = input_meta_data['max_seq_length'] + train_input_fn = get_dataset_fn( + FLAGS.train_data_path, + max_seq_length, + FLAGS.train_batch_size, + is_training=True) + eval_input_fn = get_dataset_fn( + FLAGS.eval_data_path, + max_seq_length, + FLAGS.eval_batch_size, + is_training=False) + + bert_config = bert_configs.BertConfig.from_json_file(FLAGS.bert_config_file) + run_bert(strategy, input_meta_data, bert_config, train_input_fn, + eval_input_fn) + + +if __name__ == '__main__': + flags.mark_flag_as_required('bert_config_file') + flags.mark_flag_as_required('input_meta_data_path') + flags.mark_flag_as_required('model_dir') + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/run_pretraining.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/run_pretraining.py new file mode 100644 index 0000000..25d412e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/run_pretraining.py @@ -0,0 +1,187 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Run masked LM/next sentence pre-training for BERT in TF 2.x.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import app +from absl import flags +from absl import logging +import gin +import tensorflow as tf +from official.modeling import performance +from official.nlp import optimization +from official.nlp.bert import bert_models +from official.nlp.bert import common_flags +from official.nlp.bert import configs +from official.nlp.bert import input_pipeline +from official.nlp.bert import model_training_utils +from official.utils.misc import distribution_utils + + +flags.DEFINE_string('input_files', None, + 'File path to retrieve training data for pre-training.') +# Model training specific flags. +flags.DEFINE_integer( + 'max_seq_length', 128, + 'The maximum total input sequence length after WordPiece tokenization. ' + 'Sequences longer than this will be truncated, and sequences shorter ' + 'than this will be padded.') +flags.DEFINE_integer('max_predictions_per_seq', 20, + 'Maximum predictions per sequence_output.') +flags.DEFINE_integer('train_batch_size', 32, 'Total batch size for training.') +flags.DEFINE_integer('num_steps_per_epoch', 1000, + 'Total number of training steps to run per epoch.') +flags.DEFINE_float('warmup_steps', 10000, + 'Warmup steps for Adam weight decay optimizer.') +flags.DEFINE_bool('use_next_sentence_label', True, + 'Whether to use next sentence label to compute final loss.') + +common_flags.define_common_bert_flags() +common_flags.define_gin_flags() + +FLAGS = flags.FLAGS + + +def get_pretrain_dataset_fn(input_file_pattern, seq_length, + max_predictions_per_seq, global_batch_size, + use_next_sentence_label=True): + """Returns input dataset from input file string.""" + def _dataset_fn(ctx=None): + """Returns tf.data.Dataset for distributed BERT pretraining.""" + input_patterns = input_file_pattern.split(',') + batch_size = ctx.get_per_replica_batch_size(global_batch_size) + train_dataset = input_pipeline.create_pretrain_dataset( + input_patterns, + seq_length, + max_predictions_per_seq, + batch_size, + is_training=True, + input_pipeline_context=ctx, + use_next_sentence_label=use_next_sentence_label) + return train_dataset + + return _dataset_fn + + +def get_loss_fn(): + """Returns loss function for BERT pretraining.""" + + def _bert_pretrain_loss_fn(unused_labels, losses, **unused_args): + return tf.reduce_mean(losses) + + return _bert_pretrain_loss_fn + + +def run_customized_training(strategy, + bert_config, + max_seq_length, + max_predictions_per_seq, + model_dir, + steps_per_epoch, + steps_per_loop, + epochs, + initial_lr, + warmup_steps, + end_lr, + optimizer_type, + input_files, + train_batch_size, + use_next_sentence_label=True): + """Run BERT pretrain model training using low-level API.""" + + train_input_fn = get_pretrain_dataset_fn(input_files, max_seq_length, + max_predictions_per_seq, + train_batch_size, + use_next_sentence_label) + + def _get_pretrain_model(): + """Gets a pretraining model.""" + pretrain_model, core_model = bert_models.pretrain_model( + bert_config, max_seq_length, max_predictions_per_seq, + use_next_sentence_label=use_next_sentence_label) + optimizer = optimization.create_optimizer( + initial_lr, steps_per_epoch * epochs, warmup_steps, + end_lr, optimizer_type) + pretrain_model.optimizer = performance.configure_optimizer( + optimizer, + use_float16=common_flags.use_float16(), + use_graph_rewrite=common_flags.use_graph_rewrite()) + return pretrain_model, core_model + + trained_model = model_training_utils.run_customized_training_loop( + strategy=strategy, + model_fn=_get_pretrain_model, + loss_fn=get_loss_fn(), + scale_loss=FLAGS.scale_loss, + model_dir=model_dir, + train_input_fn=train_input_fn, + steps_per_epoch=steps_per_epoch, + steps_per_loop=steps_per_loop, + epochs=epochs, + sub_model_export_name='pretrained/bert_model') + + return trained_model + + +def run_bert_pretrain(strategy): + """Runs BERT pre-training.""" + + bert_config = configs.BertConfig.from_json_file(FLAGS.bert_config_file) + if not strategy: + raise ValueError('Distribution strategy is not specified.') + + # Runs customized training loop. + logging.info('Training using customized training loop TF 2.0 with distrubuted' + 'strategy.') + + performance.set_mixed_precision_policy(common_flags.dtype()) + + return run_customized_training( + strategy, + bert_config, + FLAGS.max_seq_length, + FLAGS.max_predictions_per_seq, + FLAGS.model_dir, + FLAGS.num_steps_per_epoch, + FLAGS.steps_per_loop, + FLAGS.num_train_epochs, + FLAGS.learning_rate, + FLAGS.warmup_steps, + FLAGS.end_lr, + FLAGS.optimizer_type, + FLAGS.input_files, + FLAGS.train_batch_size, + FLAGS.use_next_sentence_label) + + +def main(_): + # Users should always run this script under TF 2.x + gin.parse_config_files_and_bindings(FLAGS.gin_file, FLAGS.gin_param) + if not FLAGS.model_dir: + FLAGS.model_dir = '/tmp/bert20/' + strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=FLAGS.distribution_strategy, + num_gpus=FLAGS.num_gpus, + tpu_address=FLAGS.tpu) + if strategy: + print('***** Number of cores used : ', strategy.num_replicas_in_sync) + + run_bert_pretrain(strategy) + + +if __name__ == '__main__': + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/run_squad.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/run_squad.py new file mode 100644 index 0000000..2a5856e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/run_squad.py @@ -0,0 +1,149 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Run BERT on SQuAD 1.1 and SQuAD 2.0 in TF 2.x.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import os +import time + +from absl import app +from absl import flags +from absl import logging +import tensorflow as tf + +from official.nlp.bert import configs as bert_configs +from official.nlp.bert import run_squad_helper +from official.nlp.bert import tokenization +from official.nlp.data import squad_lib as squad_lib_wp +from official.utils.misc import distribution_utils +from official.utils.misc import keras_utils + + +flags.DEFINE_string('vocab_file', None, + 'The vocabulary file that the BERT model was trained on.') + +# More flags can be found in run_squad_helper. +run_squad_helper.define_common_squad_flags() + +FLAGS = flags.FLAGS + + +def train_squad(strategy, + input_meta_data, + custom_callbacks=None, + run_eagerly=False, + init_checkpoint=None): + """Run bert squad training.""" + bert_config = bert_configs.BertConfig.from_json_file(FLAGS.bert_config_file) + init_checkpoint = init_checkpoint or FLAGS.init_checkpoint + run_squad_helper.train_squad(strategy, input_meta_data, bert_config, + custom_callbacks, run_eagerly, init_checkpoint) + + +def predict_squad(strategy, input_meta_data): + """Makes predictions for the squad dataset.""" + bert_config = bert_configs.BertConfig.from_json_file(FLAGS.bert_config_file) + tokenizer = tokenization.FullTokenizer( + vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) + run_squad_helper.predict_squad( + strategy, input_meta_data, tokenizer, bert_config, squad_lib_wp) + + +def eval_squad(strategy, input_meta_data): + """Evaluate on the squad dataset.""" + bert_config = bert_configs.BertConfig.from_json_file(FLAGS.bert_config_file) + tokenizer = tokenization.FullTokenizer( + vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) + eval_metrics = run_squad_helper.eval_squad( + strategy, input_meta_data, tokenizer, bert_config, squad_lib_wp) + return eval_metrics + + +def export_squad(model_export_path, input_meta_data): + """Exports a trained model as a `SavedModel` for inference. + + Args: + model_export_path: a string specifying the path to the SavedModel directory. + input_meta_data: dictionary containing meta data about input and model. + + Raises: + Export path is not specified, got an empty string or None. + """ + bert_config = bert_configs.BertConfig.from_json_file(FLAGS.bert_config_file) + run_squad_helper.export_squad(model_export_path, input_meta_data, bert_config) + + +def main(_): + # Users should always run this script under TF 2.x + + with tf.io.gfile.GFile(FLAGS.input_meta_data_path, 'rb') as reader: + input_meta_data = json.loads(reader.read().decode('utf-8')) + + if FLAGS.mode == 'export_only': + export_squad(FLAGS.model_export_path, input_meta_data) + return + + # Configures cluster spec for multi-worker distribution strategy. + if FLAGS.num_gpus > 0: + _ = distribution_utils.configure_cluster(FLAGS.worker_hosts, + FLAGS.task_index) + strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=FLAGS.distribution_strategy, + num_gpus=FLAGS.num_gpus, + all_reduce_alg=FLAGS.all_reduce_alg, + tpu_address=FLAGS.tpu) + + if 'train' in FLAGS.mode: + if FLAGS.log_steps: + custom_callbacks = [keras_utils.TimeHistory( + batch_size=FLAGS.train_batch_size, + log_steps=FLAGS.log_steps, + logdir=FLAGS.model_dir, + )] + else: + custom_callbacks = None + + train_squad( + strategy, + input_meta_data, + custom_callbacks=custom_callbacks, + run_eagerly=FLAGS.run_eagerly, + ) + if 'predict' in FLAGS.mode: + predict_squad(strategy, input_meta_data) + if 'eval' in FLAGS.mode: + eval_metrics = eval_squad(strategy, input_meta_data) + f1_score = eval_metrics['final_f1'] + logging.info('SQuAD eval F1-score: %f', f1_score) + summary_dir = os.path.join(FLAGS.model_dir, 'summaries', 'eval') + summary_writer = tf.summary.create_file_writer(summary_dir) + with summary_writer.as_default(): + # TODO(lehou): write to the correct step number. + tf.summary.scalar('F1-score', f1_score, step=0) + summary_writer.flush() + # Also write eval_metrics to json file. + squad_lib_wp.write_to_json_files( + eval_metrics, os.path.join(summary_dir, 'eval_metrics.json')) + time.sleep(60) + + +if __name__ == '__main__': + flags.mark_flag_as_required('bert_config_file') + flags.mark_flag_as_required('model_dir') + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/run_squad_helper.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/run_squad_helper.py new file mode 100644 index 0000000..07c22ed --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/run_squad_helper.py @@ -0,0 +1,432 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Library for running BERT family models on SQuAD 1.1/2.0 in TF 2.x.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import json +import os +from absl import flags +from absl import logging +import tensorflow as tf +from official.modeling import performance +from official.nlp import optimization +from official.nlp.bert import bert_models +from official.nlp.bert import common_flags +from official.nlp.bert import input_pipeline +from official.nlp.bert import model_saving_utils +from official.nlp.bert import model_training_utils +from official.nlp.bert import squad_evaluate_v1_1 +from official.nlp.bert import squad_evaluate_v2_0 +from official.nlp.data import squad_lib_sp +from official.utils.misc import keras_utils + + +def define_common_squad_flags(): + """Defines common flags used by SQuAD tasks.""" + flags.DEFINE_enum( + 'mode', 'train_and_eval', + ['train_and_eval', 'train_and_predict', + 'train', 'eval', 'predict', 'export_only'], + 'One of {"train_and_eval", "train_and_predict", ' + '"train", "eval", "predict", "export_only"}. ' + '`train_and_eval`: train & predict to json files & compute eval metrics. ' + '`train_and_predict`: train & predict to json files. ' + '`train`: only trains the model. ' + '`eval`: predict answers from squad json file & compute eval metrics. ' + '`predict`: predict answers from the squad json file. ' + '`export_only`: will take the latest checkpoint inside ' + 'model_dir and export a `SavedModel`.') + flags.DEFINE_string('train_data_path', '', + 'Training data path with train tfrecords.') + flags.DEFINE_string( + 'input_meta_data_path', None, + 'Path to file that contains meta data about input ' + 'to be used for training and evaluation.') + # Model training specific flags. + flags.DEFINE_integer('train_batch_size', 32, 'Total batch size for training.') + # Predict processing related. + flags.DEFINE_string('predict_file', None, + 'Prediction data path with train tfrecords.') + flags.DEFINE_bool( + 'do_lower_case', True, + 'Whether to lower case the input text. Should be True for uncased ' + 'models and False for cased models.') + flags.DEFINE_float( + 'null_score_diff_threshold', 0.0, + 'If null_score - best_non_null is greater than the threshold, ' + 'predict null. This is only used for SQuAD v2.') + flags.DEFINE_bool( + 'verbose_logging', False, + 'If true, all of the warnings related to data processing will be ' + 'printed. A number of warnings are expected for a normal SQuAD ' + 'evaluation.') + flags.DEFINE_integer('predict_batch_size', 8, + 'Total batch size for prediction.') + flags.DEFINE_integer( + 'n_best_size', 20, + 'The total number of n-best predictions to generate in the ' + 'nbest_predictions.json output file.') + flags.DEFINE_integer( + 'max_answer_length', 30, + 'The maximum length of an answer that can be generated. This is needed ' + 'because the start and end predictions are not conditioned on one ' + 'another.') + + common_flags.define_common_bert_flags() + + +FLAGS = flags.FLAGS + + +def squad_loss_fn(start_positions, + end_positions, + start_logits, + end_logits): + """Returns sparse categorical crossentropy for start/end logits.""" + start_loss = tf.keras.losses.sparse_categorical_crossentropy( + start_positions, start_logits, from_logits=True) + end_loss = tf.keras.losses.sparse_categorical_crossentropy( + end_positions, end_logits, from_logits=True) + + total_loss = (tf.reduce_mean(start_loss) + tf.reduce_mean(end_loss)) / 2 + return total_loss + + +def get_loss_fn(): + """Gets a loss function for squad task.""" + + def _loss_fn(labels, model_outputs): + start_positions = labels['start_positions'] + end_positions = labels['end_positions'] + start_logits, end_logits = model_outputs + return squad_loss_fn( + start_positions, + end_positions, + start_logits, + end_logits) + + return _loss_fn + + +RawResult = collections.namedtuple('RawResult', + ['unique_id', 'start_logits', 'end_logits']) + + +def get_raw_results(predictions): + """Converts multi-replica predictions to RawResult.""" + for unique_ids, start_logits, end_logits in zip(predictions['unique_ids'], + predictions['start_logits'], + predictions['end_logits']): + for values in zip(unique_ids.numpy(), start_logits.numpy(), + end_logits.numpy()): + yield RawResult( + unique_id=values[0], + start_logits=values[1].tolist(), + end_logits=values[2].tolist()) + + +def get_dataset_fn(input_file_pattern, max_seq_length, global_batch_size, + is_training): + """Gets a closure to create a dataset..""" + + def _dataset_fn(ctx=None): + """Returns tf.data.Dataset for distributed BERT pretraining.""" + batch_size = ctx.get_per_replica_batch_size( + global_batch_size) if ctx else global_batch_size + dataset = input_pipeline.create_squad_dataset( + input_file_pattern, + max_seq_length, + batch_size, + is_training=is_training, + input_pipeline_context=ctx) + return dataset + + return _dataset_fn + + +def predict_squad_customized(strategy, + input_meta_data, + bert_config, + checkpoint_path, + predict_tfrecord_path, + num_steps): + """Make predictions using a Bert-based squad model.""" + predict_dataset_fn = get_dataset_fn( + predict_tfrecord_path, + input_meta_data['max_seq_length'], + FLAGS.predict_batch_size, + is_training=False) + predict_iterator = iter( + strategy.experimental_distribute_datasets_from_function( + predict_dataset_fn)) + + with strategy.scope(): + # Prediction always uses float32, even if training uses mixed precision. + tf.keras.mixed_precision.experimental.set_policy('float32') + squad_model, _ = bert_models.squad_model( + bert_config, + input_meta_data['max_seq_length'], + hub_module_url=FLAGS.hub_module_url) + + if checkpoint_path is None: + checkpoint_path = tf.train.latest_checkpoint(FLAGS.model_dir) + logging.info('Restoring checkpoints from %s', checkpoint_path) + checkpoint = tf.train.Checkpoint(model=squad_model) + checkpoint.restore(checkpoint_path).expect_partial() + + @tf.function + def predict_step(iterator): + """Predicts on distributed devices.""" + + def _replicated_step(inputs): + """Replicated prediction calculation.""" + x, _ = inputs + unique_ids = x.pop('unique_ids') + start_logits, end_logits = squad_model(x, training=False) + return dict( + unique_ids=unique_ids, + start_logits=start_logits, + end_logits=end_logits) + + outputs = strategy.run(_replicated_step, args=(next(iterator),)) + return tf.nest.map_structure(strategy.experimental_local_results, outputs) + + all_results = [] + for _ in range(num_steps): + predictions = predict_step(predict_iterator) + for result in get_raw_results(predictions): + all_results.append(result) + if len(all_results) % 100 == 0: + logging.info('Made predictions for %d records.', len(all_results)) + return all_results + + +def train_squad(strategy, + input_meta_data, + bert_config, + custom_callbacks=None, + run_eagerly=False, + init_checkpoint=None): + """Run bert squad training.""" + if strategy: + logging.info('Training using customized training loop with distribution' + ' strategy.') + # Enables XLA in Session Config. Should not be set for TPU. + keras_utils.set_config_v2(FLAGS.enable_xla) + performance.set_mixed_precision_policy(common_flags.dtype()) + + epochs = FLAGS.num_train_epochs + num_train_examples = input_meta_data['train_data_size'] + max_seq_length = input_meta_data['max_seq_length'] + steps_per_epoch = int(num_train_examples / FLAGS.train_batch_size) + warmup_steps = int(epochs * num_train_examples * 0.1 / FLAGS.train_batch_size) + train_input_fn = get_dataset_fn( + FLAGS.train_data_path, + max_seq_length, + FLAGS.train_batch_size, + is_training=True) + + def _get_squad_model(): + """Get Squad model and optimizer.""" + squad_model, core_model = bert_models.squad_model( + bert_config, + max_seq_length, + hub_module_url=FLAGS.hub_module_url, + hub_module_trainable=FLAGS.hub_module_trainable) + optimizer = optimization.create_optimizer(FLAGS.learning_rate, + steps_per_epoch * epochs, + warmup_steps, + FLAGS.end_lr, + FLAGS.optimizer_type) + + squad_model.optimizer = performance.configure_optimizer( + optimizer, + use_float16=common_flags.use_float16(), + use_graph_rewrite=common_flags.use_graph_rewrite()) + return squad_model, core_model + + # If explicit_allreduce = True, apply_gradients() no longer implicitly + # allreduce gradients, users manually allreduce gradient and pass the + # allreduced grads_and_vars to apply_gradients(). clip_by_global_norm will be + # applied to allreduced gradients. + def clip_by_global_norm_callback(grads_and_vars): + grads, variables = zip(*grads_and_vars) + (clipped_grads, _) = tf.clip_by_global_norm(grads, clip_norm=1.0) + return zip(clipped_grads, variables) + + model_training_utils.run_customized_training_loop( + strategy=strategy, + model_fn=_get_squad_model, + loss_fn=get_loss_fn(), + model_dir=FLAGS.model_dir, + steps_per_epoch=steps_per_epoch, + steps_per_loop=FLAGS.steps_per_loop, + epochs=epochs, + train_input_fn=train_input_fn, + init_checkpoint=init_checkpoint or FLAGS.init_checkpoint, + run_eagerly=run_eagerly, + custom_callbacks=custom_callbacks, + explicit_allreduce=False, + post_allreduce_callbacks=[clip_by_global_norm_callback]) + + +def prediction_output_squad( + strategy, input_meta_data, tokenizer, bert_config, squad_lib, checkpoint): + """Makes predictions for a squad dataset.""" + doc_stride = input_meta_data['doc_stride'] + max_query_length = input_meta_data['max_query_length'] + # Whether data should be in Ver 2.0 format. + version_2_with_negative = input_meta_data.get('version_2_with_negative', + False) + eval_examples = squad_lib.read_squad_examples( + input_file=FLAGS.predict_file, + is_training=False, + version_2_with_negative=version_2_with_negative) + + eval_writer = squad_lib.FeatureWriter( + filename=os.path.join(FLAGS.model_dir, 'eval.tf_record'), + is_training=False) + eval_features = [] + + def _append_feature(feature, is_padding): + if not is_padding: + eval_features.append(feature) + eval_writer.process_feature(feature) + + # TPU requires a fixed batch size for all batches, therefore the number + # of examples must be a multiple of the batch size, or else examples + # will get dropped. So we pad with fake examples which are ignored + # later on. + kwargs = dict( + examples=eval_examples, + tokenizer=tokenizer, + max_seq_length=input_meta_data['max_seq_length'], + doc_stride=doc_stride, + max_query_length=max_query_length, + is_training=False, + output_fn=_append_feature, + batch_size=FLAGS.predict_batch_size) + + # squad_lib_sp requires one more argument 'do_lower_case'. + if squad_lib == squad_lib_sp: + kwargs['do_lower_case'] = FLAGS.do_lower_case + dataset_size = squad_lib.convert_examples_to_features(**kwargs) + eval_writer.close() + + logging.info('***** Running predictions *****') + logging.info(' Num orig examples = %d', len(eval_examples)) + logging.info(' Num split examples = %d', len(eval_features)) + logging.info(' Batch size = %d', FLAGS.predict_batch_size) + + num_steps = int(dataset_size / FLAGS.predict_batch_size) + all_results = predict_squad_customized( + strategy, input_meta_data, bert_config, + checkpoint, eval_writer.filename, num_steps) + + all_predictions, all_nbest_json, scores_diff_json = ( + squad_lib.postprocess_output( + eval_examples, + eval_features, + all_results, + FLAGS.n_best_size, + FLAGS.max_answer_length, + FLAGS.do_lower_case, + version_2_with_negative=version_2_with_negative, + null_score_diff_threshold=FLAGS.null_score_diff_threshold, + verbose=FLAGS.verbose_logging)) + + return all_predictions, all_nbest_json, scores_diff_json + + +def dump_to_files(all_predictions, all_nbest_json, scores_diff_json, + squad_lib, version_2_with_negative): + """Save output to json files.""" + output_prediction_file = os.path.join(FLAGS.model_dir, 'predictions.json') + output_nbest_file = os.path.join(FLAGS.model_dir, 'nbest_predictions.json') + output_null_log_odds_file = os.path.join(FLAGS.model_dir, 'null_odds.json') + logging.info('Writing predictions to: %s', (output_prediction_file)) + logging.info('Writing nbest to: %s', (output_nbest_file)) + + squad_lib.write_to_json_files(all_predictions, output_prediction_file) + squad_lib.write_to_json_files(all_nbest_json, output_nbest_file) + if version_2_with_negative: + squad_lib.write_to_json_files(scores_diff_json, output_null_log_odds_file) + + +def predict_squad(strategy, + input_meta_data, + tokenizer, + bert_config, + squad_lib, + init_checkpoint=None): + """Get prediction results and evaluate them to hard drive.""" + if init_checkpoint is None: + init_checkpoint = tf.train.latest_checkpoint(FLAGS.model_dir) + all_predictions, all_nbest_json, scores_diff_json = prediction_output_squad( + strategy, input_meta_data, tokenizer, + bert_config, squad_lib, init_checkpoint) + dump_to_files(all_predictions, all_nbest_json, scores_diff_json, squad_lib, + input_meta_data.get('version_2_with_negative', False)) + + +def eval_squad(strategy, + input_meta_data, + tokenizer, + bert_config, + squad_lib, + init_checkpoint=None): + """Get prediction results and evaluate them against ground truth.""" + if init_checkpoint is None: + init_checkpoint = tf.train.latest_checkpoint(FLAGS.model_dir) + all_predictions, all_nbest_json, scores_diff_json = prediction_output_squad( + strategy, input_meta_data, tokenizer, + bert_config, squad_lib, init_checkpoint) + dump_to_files(all_predictions, all_nbest_json, scores_diff_json, squad_lib, + input_meta_data.get('version_2_with_negative', False)) + + with tf.io.gfile.GFile(FLAGS.predict_file, 'r') as reader: + dataset_json = json.load(reader) + pred_dataset = dataset_json['data'] + if input_meta_data.get('version_2_with_negative', False): + eval_metrics = squad_evaluate_v2_0.evaluate(pred_dataset, + all_predictions, + scores_diff_json) + else: + eval_metrics = squad_evaluate_v1_1.evaluate(pred_dataset, all_predictions) + return eval_metrics + + +def export_squad(model_export_path, input_meta_data, bert_config): + """Exports a trained model as a `SavedModel` for inference. + + Args: + model_export_path: a string specifying the path to the SavedModel directory. + input_meta_data: dictionary containing meta data about input and model. + bert_config: Bert configuration file to define core bert layers. + + Raises: + Export path is not specified, got an empty string or None. + """ + if not model_export_path: + raise ValueError('Export path is not specified: %s' % model_export_path) + # Export uses float32 for now, even if training uses mixed precision. + tf.keras.mixed_precision.experimental.set_policy('float32') + squad_model, _ = bert_models.squad_model(bert_config, + input_meta_data['max_seq_length']) + model_saving_utils.export_bert_model( + model_export_path, model=squad_model, checkpoint_dir=FLAGS.model_dir) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/squad_evaluate_v1_1.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/squad_evaluate_v1_1.py new file mode 100644 index 0000000..c7f4f4d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/squad_evaluate_v1_1.py @@ -0,0 +1,108 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Evaluation of SQuAD predictions (version 1.1). + +The functions are copied from +https://worksheets.codalab.org/rest/bundles/0xbcd57bee090b421c982906709c8c27e1/contents/blob/. + +The SQuAD dataset is described in this paper: +SQuAD: 100,000+ Questions for Machine Comprehension of Text +Pranav Rajpurkar, Jian Zhang, Konstantin Lopyrev, Percy Liang +https://nlp.stanford.edu/pubs/rajpurkar2016squad.pdf +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import re +import string + +# pylint: disable=g-bad-import-order +from absl import logging +# pylint: enable=g-bad-import-order + + +def _normalize_answer(s): + """Lowers text and remove punctuation, articles and extra whitespace.""" + + def remove_articles(text): + return re.sub(r"\b(a|an|the)\b", " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def _f1_score(prediction, ground_truth): + """Computes F1 score by comparing prediction to ground truth.""" + prediction_tokens = _normalize_answer(prediction).split() + ground_truth_tokens = _normalize_answer(ground_truth).split() + prediction_counter = collections.Counter(prediction_tokens) + ground_truth_counter = collections.Counter(ground_truth_tokens) + common = prediction_counter & ground_truth_counter + num_same = sum(common.values()) + if num_same == 0: + return 0 + precision = 1.0 * num_same / len(prediction_tokens) + recall = 1.0 * num_same / len(ground_truth_tokens) + f1 = (2 * precision * recall) / (precision + recall) + return f1 + + +def _exact_match_score(prediction, ground_truth): + """Checks if predicted answer exactly matches ground truth answer.""" + return _normalize_answer(prediction) == _normalize_answer(ground_truth) + + +def _metric_max_over_ground_truths(metric_fn, prediction, ground_truths): + """Computes the max over all metric scores.""" + scores_for_ground_truths = [] + for ground_truth in ground_truths: + score = metric_fn(prediction, ground_truth) + scores_for_ground_truths.append(score) + return max(scores_for_ground_truths) + + +def evaluate(dataset, predictions): + """Evaluates predictions for a dataset.""" + f1 = exact_match = total = 0 + for article in dataset: + for paragraph in article["paragraphs"]: + for qa in paragraph["qas"]: + total += 1 + if qa["id"] not in predictions: + message = "Unanswered question " + qa["id"] + " will receive score 0." + logging.error(message) + continue + ground_truths = [entry["text"] for entry in qa["answers"]] + prediction = predictions[qa["id"]] + exact_match += _metric_max_over_ground_truths(_exact_match_score, + prediction, ground_truths) + f1 += _metric_max_over_ground_truths(_f1_score, prediction, + ground_truths) + + exact_match = exact_match / total + f1 = f1 / total + + return {"exact_match": exact_match, "final_f1": f1} diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/squad_evaluate_v2_0.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/squad_evaluate_v2_0.py new file mode 100644 index 0000000..54fb84e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/squad_evaluate_v2_0.py @@ -0,0 +1,252 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Evaluation script for SQuAD version 2.0. + +The functions are copied and modified from +https://raw.githubusercontent.com/white127/SQUAD-2.0-bidaf/master/evaluate-v2.0.py + +In addition to basic functionality, we also compute additional statistics and +plot precision-recall curves if an additional na_prob.json file is provided. +This file is expected to map question ID's to the model's predicted probability +that a question is unanswerable. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import re +import string + +from absl import logging + + +def _make_qid_to_has_ans(dataset): + qid_to_has_ans = {} + for article in dataset: + for p in article['paragraphs']: + for qa in p['qas']: + qid_to_has_ans[qa['id']] = bool(qa['answers']) + return qid_to_has_ans + + +def _normalize_answer(s): + """Lower text and remove punctuation, articles and extra whitespace.""" + def remove_articles(text): + regex = re.compile(r'\b(a|an|the)\b', re.UNICODE) + return re.sub(regex, ' ', text) + def white_space_fix(text): + return ' '.join(text.split()) + def remove_punc(text): + exclude = set(string.punctuation) + return ''.join(ch for ch in text if ch not in exclude) + def lower(text): + return text.lower() + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def _get_tokens(s): + if not s: return [] + return _normalize_answer(s).split() + + +def _compute_exact(a_gold, a_pred): + return int(_normalize_answer(a_gold) == _normalize_answer(a_pred)) + + +def _compute_f1(a_gold, a_pred): + """Compute F1-score.""" + gold_toks = _get_tokens(a_gold) + pred_toks = _get_tokens(a_pred) + common = collections.Counter(gold_toks) & collections.Counter(pred_toks) + num_same = sum(common.values()) + if not gold_toks or not pred_toks: + # If either is no-answer, then F1 is 1 if they agree, 0 otherwise + return int(gold_toks == pred_toks) + if num_same == 0: + return 0 + precision = 1.0 * num_same / len(pred_toks) + recall = 1.0 * num_same / len(gold_toks) + f1 = (2 * precision * recall) / (precision + recall) + return f1 + + +def _get_raw_scores(dataset, predictions): + """Compute raw scores.""" + exact_scores = {} + f1_scores = {} + for article in dataset: + for p in article['paragraphs']: + for qa in p['qas']: + qid = qa['id'] + gold_answers = [a['text'] for a in qa['answers'] + if _normalize_answer(a['text'])] + if not gold_answers: + # For unanswerable questions, only correct answer is empty string + gold_answers = [''] + if qid not in predictions: + logging.error('Missing prediction for %s', qid) + continue + a_pred = predictions[qid] + # Take max over all gold answers + exact_scores[qid] = max(_compute_exact(a, a_pred) for a in gold_answers) + f1_scores[qid] = max(_compute_f1(a, a_pred) for a in gold_answers) + return exact_scores, f1_scores + + +def _apply_no_ans_threshold( + scores, na_probs, qid_to_has_ans, na_prob_thresh=1.0): + new_scores = {} + for qid, s in scores.items(): + pred_na = na_probs[qid] > na_prob_thresh + if pred_na: + new_scores[qid] = float(not qid_to_has_ans[qid]) + else: + new_scores[qid] = s + return new_scores + + +def _make_eval_dict(exact_scores, f1_scores, qid_list=None): + """Make evaluation result dictionary.""" + if not qid_list: + total = len(exact_scores) + return collections.OrderedDict([ + ('exact', 100.0 * sum(exact_scores.values()) / total), + ('f1', 100.0 * sum(f1_scores.values()) / total), + ('total', total), + ]) + else: + total = len(qid_list) + return collections.OrderedDict([ + ('exact', 100.0 * sum(exact_scores[k] for k in qid_list) / total), + ('f1', 100.0 * sum(f1_scores[k] for k in qid_list) / total), + ('total', total), + ]) + + +def _merge_eval(main_eval, new_eval, prefix): + for k in new_eval: + main_eval['%s_%s' % (prefix, k)] = new_eval[k] + + +def _make_precision_recall_eval(scores, na_probs, num_true_pos, qid_to_has_ans): + """Make evaluation dictionary containing average recision recall.""" + qid_list = sorted(na_probs, key=lambda k: na_probs[k]) + true_pos = 0.0 + cur_p = 1.0 + cur_r = 0.0 + precisions = [1.0] + recalls = [0.0] + avg_prec = 0.0 + for i, qid in enumerate(qid_list): + if qid_to_has_ans[qid]: + true_pos += scores[qid] + cur_p = true_pos / float(i+1) + cur_r = true_pos / float(num_true_pos) + if i == len(qid_list) - 1 or na_probs[qid] != na_probs[qid_list[i+1]]: + # i.e., if we can put a threshold after this point + avg_prec += cur_p * (cur_r - recalls[-1]) + precisions.append(cur_p) + recalls.append(cur_r) + return {'ap': 100.0 * avg_prec} + + +def _run_precision_recall_analysis( + main_eval, exact_raw, f1_raw, na_probs, qid_to_has_ans): + """Run precision recall analysis and return result dictionary.""" + num_true_pos = sum(1 for v in qid_to_has_ans.values() if v) + if num_true_pos == 0: + return + pr_exact = _make_precision_recall_eval( + exact_raw, na_probs, num_true_pos, qid_to_has_ans) + pr_f1 = _make_precision_recall_eval( + f1_raw, na_probs, num_true_pos, qid_to_has_ans) + oracle_scores = {k: float(v) for k, v in qid_to_has_ans.items()} + pr_oracle = _make_precision_recall_eval( + oracle_scores, na_probs, num_true_pos, qid_to_has_ans) + _merge_eval(main_eval, pr_exact, 'pr_exact') + _merge_eval(main_eval, pr_f1, 'pr_f1') + _merge_eval(main_eval, pr_oracle, 'pr_oracle') + + +def _find_best_thresh(predictions, scores, na_probs, qid_to_has_ans): + """Find the best threshold for no answer probability.""" + num_no_ans = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]) + cur_score = num_no_ans + best_score = cur_score + best_thresh = 0.0 + qid_list = sorted(na_probs, key=lambda k: na_probs[k]) + for qid in qid_list: + if qid not in scores: continue + if qid_to_has_ans[qid]: + diff = scores[qid] + else: + if predictions[qid]: + diff = -1 + else: + diff = 0 + cur_score += diff + if cur_score > best_score: + best_score = cur_score + best_thresh = na_probs[qid] + return 100.0 * best_score / len(scores), best_thresh + + +def _find_all_best_thresh( + main_eval, predictions, exact_raw, f1_raw, na_probs, qid_to_has_ans): + best_exact, exact_thresh = _find_best_thresh( + predictions, exact_raw, na_probs, qid_to_has_ans) + best_f1, f1_thresh = _find_best_thresh( + predictions, f1_raw, na_probs, qid_to_has_ans) + main_eval['final_exact'] = best_exact + main_eval['final_exact_thresh'] = exact_thresh + main_eval['final_f1'] = best_f1 + main_eval['final_f1_thresh'] = f1_thresh + + +def evaluate(dataset, predictions, na_probs=None): + """Evaluate prediction results.""" + new_orig_data = [] + for article in dataset: + for p in article['paragraphs']: + for qa in p['qas']: + if qa['id'] in predictions: + new_para = {'qas': [qa]} + new_article = {'paragraphs': [new_para]} + new_orig_data.append(new_article) + dataset = new_orig_data + + if na_probs is None: + na_probs = {k: 0.0 for k in predictions} + qid_to_has_ans = _make_qid_to_has_ans(dataset) # maps qid to True/False + has_ans_qids = [k for k, v in qid_to_has_ans.items() if v] + no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v] + exact_raw, f1_raw = _get_raw_scores(dataset, predictions) + exact_thresh = _apply_no_ans_threshold(exact_raw, na_probs, qid_to_has_ans) + f1_thresh = _apply_no_ans_threshold(f1_raw, na_probs, qid_to_has_ans) + out_eval = _make_eval_dict(exact_thresh, f1_thresh) + if has_ans_qids: + has_ans_eval = _make_eval_dict( + exact_thresh, f1_thresh, qid_list=has_ans_qids) + _merge_eval(out_eval, has_ans_eval, 'HasAns') + if no_ans_qids: + no_ans_eval = _make_eval_dict(exact_thresh, f1_thresh, qid_list=no_ans_qids) + _merge_eval(out_eval, no_ans_eval, 'NoAns') + + _find_all_best_thresh( + out_eval, predictions, exact_raw, f1_raw, na_probs, qid_to_has_ans) + _run_precision_recall_analysis( + out_eval, exact_raw, f1_raw, na_probs, qid_to_has_ans) + return out_eval diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/tf1_checkpoint_converter_lib.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/tf1_checkpoint_converter_lib.py new file mode 100644 index 0000000..daec829 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/tf1_checkpoint_converter_lib.py @@ -0,0 +1,195 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""Convert checkpoints created by Estimator (tf1) to be Keras compatible.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow.compat.v1 as tf # TF 1.x + +# Mapping between old <=> new names. The source pattern in original variable +# name will be replaced by destination pattern. +BERT_NAME_REPLACEMENTS = ( + ("bert", "bert_model"), + ("embeddings/word_embeddings", "word_embeddings/embeddings"), + ("embeddings/token_type_embeddings", + "embedding_postprocessor/type_embeddings"), + ("embeddings/position_embeddings", + "embedding_postprocessor/position_embeddings"), + ("embeddings/LayerNorm", "embedding_postprocessor/layer_norm"), + ("attention/self", "self_attention"), + ("attention/output/dense", "self_attention_output"), + ("attention/output/LayerNorm", "self_attention_layer_norm"), + ("intermediate/dense", "intermediate"), + ("output/dense", "output"), + ("output/LayerNorm", "output_layer_norm"), + ("pooler/dense", "pooler_transform"), +) + +BERT_V2_NAME_REPLACEMENTS = ( + ("bert/", ""), + ("encoder", "transformer"), + ("embeddings/word_embeddings", "word_embeddings/embeddings"), + ("embeddings/token_type_embeddings", "type_embeddings/embeddings"), + ("embeddings/position_embeddings", "position_embedding/embeddings"), + ("embeddings/LayerNorm", "embeddings/layer_norm"), + ("attention/self", "self_attention"), + ("attention/output/dense", "self_attention_output"), + ("attention/output/LayerNorm", "self_attention_layer_norm"), + ("intermediate/dense", "intermediate"), + ("output/dense", "output"), + ("output/LayerNorm", "output_layer_norm"), + ("pooler/dense", "pooler_transform"), + ("cls/predictions/output_bias", "cls/predictions/output_bias/bias"), + ("cls/seq_relationship/output_bias", "predictions/transform/logits/bias"), + ("cls/seq_relationship/output_weights", + "predictions/transform/logits/kernel"), +) + +BERT_PERMUTATIONS = () + +BERT_V2_PERMUTATIONS = (("cls/seq_relationship/output_weights", (1, 0)),) + + +def _bert_name_replacement(var_name, name_replacements): + """Gets the variable name replacement.""" + for src_pattern, tgt_pattern in name_replacements: + if src_pattern in var_name: + old_var_name = var_name + var_name = var_name.replace(src_pattern, tgt_pattern) + tf.logging.info("Converted: %s --> %s", old_var_name, var_name) + return var_name + + +def _has_exclude_patterns(name, exclude_patterns): + """Checks if a string contains substrings that match patterns to exclude.""" + for p in exclude_patterns: + if p in name: + return True + return False + + +def _get_permutation(name, permutations): + """Checks whether a variable requires transposition by pattern matching.""" + for src_pattern, permutation in permutations: + if src_pattern in name: + tf.logging.info("Permuted: %s --> %s", name, permutation) + return permutation + + return None + + +def _get_new_shape(name, shape, num_heads): + """Checks whether a variable requires reshape by pattern matching.""" + if "self_attention_output/kernel" in name: + return tuple([num_heads, shape[0] // num_heads, shape[1]]) + if "self_attention_output/bias" in name: + return shape + + patterns = [ + "self_attention/query", "self_attention/value", "self_attention/key" + ] + for pattern in patterns: + if pattern in name: + if "kernel" in name: + return tuple([shape[0], num_heads, shape[1] // num_heads]) + if "bias" in name: + return tuple([num_heads, shape[0] // num_heads]) + return None + + +def create_v2_checkpoint(model, src_checkpoint, output_path): + """Converts a name-based matched TF V1 checkpoint to TF V2 checkpoint.""" + # Uses streaming-restore in eager model to read V1 name-based checkpoints. + model.load_weights(src_checkpoint).assert_existing_objects_matched() + checkpoint = tf.train.Checkpoint(model=model) + checkpoint.save(output_path) + + +def convert(checkpoint_from_path, + checkpoint_to_path, + num_heads, + name_replacements, + permutations, + exclude_patterns=None): + """Migrates the names of variables within a checkpoint. + + Args: + checkpoint_from_path: Path to source checkpoint to be read in. + checkpoint_to_path: Path to checkpoint to be written out. + num_heads: The number of heads of the model. + name_replacements: A list of tuples of the form (match_str, replace_str) + describing variable names to adjust. + permutations: A list of tuples of the form (match_str, permutation) + describing permutations to apply to given variables. Note that match_str + should match the original variable name, not the replaced one. + exclude_patterns: A list of string patterns to exclude variables from + checkpoint conversion. + + Returns: + A dictionary that maps the new variable names to the Variable objects. + A dictionary that maps the old variable names to the new variable names. + """ + with tf.Graph().as_default(): + tf.logging.info("Reading checkpoint_from_path %s", checkpoint_from_path) + reader = tf.train.NewCheckpointReader(checkpoint_from_path) + name_shape_map = reader.get_variable_to_shape_map() + new_variable_map = {} + conversion_map = {} + for var_name in name_shape_map: + if exclude_patterns and _has_exclude_patterns(var_name, exclude_patterns): + continue + # Get the original tensor data. + tensor = reader.get_tensor(var_name) + + # Look up the new variable name, if any. + new_var_name = _bert_name_replacement(var_name, name_replacements) + + # See if we need to reshape the underlying tensor. + new_shape = None + if num_heads > 0: + new_shape = _get_new_shape(new_var_name, tensor.shape, num_heads) + if new_shape: + tf.logging.info("Veriable %s has a shape change from %s to %s", + + var_name, tensor.shape, new_shape) + tensor = np.reshape(tensor, new_shape) + + # See if we need to permute the underlying tensor. + permutation = _get_permutation(var_name, permutations) + if permutation: + tensor = np.transpose(tensor, permutation) + + # Create a new variable with the possibly-reshaped or transposed tensor. + var = tf.Variable(tensor, name=var_name) + + # Save the variable into the new variable map. + new_variable_map[new_var_name] = var + + # Keep a list of converter variables for sanity checking. + if new_var_name != var_name: + conversion_map[var_name] = new_var_name + + saver = tf.train.Saver(new_variable_map) + + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + tf.logging.info("Writing checkpoint_to_path %s", checkpoint_to_path) + saver.save(sess, checkpoint_to_path) + + tf.logging.info("Summary:") + tf.logging.info(" Converted %d variable name(s).", len(new_variable_map)) + tf.logging.info(" Converted: %s", str(conversion_map)) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/tf2_encoder_checkpoint_converter.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/tf2_encoder_checkpoint_converter.py new file mode 100644 index 0000000..203b238 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/tf2_encoder_checkpoint_converter.py @@ -0,0 +1,108 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A converter from a V1 BERT encoder checkpoint to a V2 encoder checkpoint. + +The conversion will yield an object-oriented checkpoint that can be used +to restore a TransformerEncoder object. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from absl import app +from absl import flags + +import tensorflow as tf +from official.modeling import activations +from official.nlp.bert import configs +from official.nlp.bert import tf1_checkpoint_converter_lib +from official.nlp.modeling import networks + +FLAGS = flags.FLAGS + +flags.DEFINE_string("bert_config_file", None, + "Bert configuration file to define core bert layers.") +flags.DEFINE_string( + "checkpoint_to_convert", None, + "Initial checkpoint from a pretrained BERT model core (that is, only the " + "BertModel, with no task heads.)") +flags.DEFINE_string("converted_checkpoint_path", None, + "Name for the created object-based V2 checkpoint.") + + +def _create_bert_model(cfg): + """Creates a BERT keras core model from BERT configuration. + + Args: + cfg: A `BertConfig` to create the core model. + Returns: + A TransformerEncoder netowork. + """ + bert_encoder = networks.TransformerEncoder( + vocab_size=cfg.vocab_size, + hidden_size=cfg.hidden_size, + num_layers=cfg.num_hidden_layers, + num_attention_heads=cfg.num_attention_heads, + intermediate_size=cfg.intermediate_size, + activation=activations.gelu, + dropout_rate=cfg.hidden_dropout_prob, + attention_dropout_rate=cfg.attention_probs_dropout_prob, + sequence_length=cfg.max_position_embeddings, + type_vocab_size=cfg.type_vocab_size, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=cfg.initializer_range)) + + return bert_encoder + + +def convert_checkpoint(bert_config, output_path, v1_checkpoint): + """Converts a V1 checkpoint into an OO V2 checkpoint.""" + output_dir, _ = os.path.split(output_path) + + # Create a temporary V1 name-converted checkpoint in the output directory. + temporary_checkpoint_dir = os.path.join(output_dir, "temp_v1") + temporary_checkpoint = os.path.join(temporary_checkpoint_dir, "ckpt") + tf1_checkpoint_converter_lib.convert( + checkpoint_from_path=v1_checkpoint, + checkpoint_to_path=temporary_checkpoint, + num_heads=bert_config.num_attention_heads, + name_replacements=tf1_checkpoint_converter_lib.BERT_V2_NAME_REPLACEMENTS, + permutations=tf1_checkpoint_converter_lib.BERT_V2_PERMUTATIONS, + exclude_patterns=["adam", "Adam"]) + + # Create a V2 checkpoint from the temporary checkpoint. + model = _create_bert_model(bert_config) + tf1_checkpoint_converter_lib.create_v2_checkpoint(model, temporary_checkpoint, + output_path) + + # Clean up the temporary checkpoint, if it exists. + try: + tf.io.gfile.rmtree(temporary_checkpoint_dir) + except tf.errors.OpError: + # If it doesn't exist, we don't need to clean it up; continue. + pass + + +def main(_): + output_path = FLAGS.converted_checkpoint_path + v1_checkpoint = FLAGS.checkpoint_to_convert + bert_config = configs.BertConfig.from_json_file(FLAGS.bert_config_file) + convert_checkpoint(bert_config, output_path, v1_checkpoint) + + +if __name__ == "__main__": + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/tokenization.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/tokenization.py new file mode 100644 index 0000000..4c08efb --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/tokenization.py @@ -0,0 +1,545 @@ +# coding=utf-8 +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tokenization classes implementation. + +The file is forked from: +https://github.com/google-research/bert/blob/master/tokenization.py. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import re +import unicodedata + +import six +import tensorflow as tf + +import sentencepiece as spm + +SPIECE_UNDERLINE = "▁" + + +def validate_case_matches_checkpoint(do_lower_case, init_checkpoint): + """Checks whether the casing config is consistent with the checkpoint name.""" + + # The casing has to be passed in by the user and there is no explicit check + # as to whether it matches the checkpoint. The casing information probably + # should have been stored in the bert_config.json file, but it's not, so + # we have to heuristically detect it to validate. + + if not init_checkpoint: + return + + m = re.match("^.*?([A-Za-z0-9_-]+)/bert_model.ckpt", init_checkpoint) + if m is None: + return + + model_name = m.group(1) + + lower_models = [ + "uncased_L-24_H-1024_A-16", "uncased_L-12_H-768_A-12", + "multilingual_L-12_H-768_A-12", "chinese_L-12_H-768_A-12" + ] + + cased_models = [ + "cased_L-12_H-768_A-12", "cased_L-24_H-1024_A-16", + "multi_cased_L-12_H-768_A-12" + ] + + is_bad_config = False + if model_name in lower_models and not do_lower_case: + is_bad_config = True + actual_flag = "False" + case_name = "lowercased" + opposite_flag = "True" + + if model_name in cased_models and do_lower_case: + is_bad_config = True + actual_flag = "True" + case_name = "cased" + opposite_flag = "False" + + if is_bad_config: + raise ValueError( + "You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`. " + "However, `%s` seems to be a %s model, so you " + "should pass in `--do_lower_case=%s` so that the fine-tuning matches " + "how the model was pre-training. If this error is wrong, please " + "just comment out this check." % + (actual_flag, init_checkpoint, model_name, case_name, opposite_flag)) + + +def convert_to_unicode(text): + """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text.decode("utf-8", "ignore") + elif isinstance(text, unicode): + return text + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def printable_text(text): + """Returns text encoded in a way suitable for print or `tf.logging`.""" + + # These functions want `str` for both Python2 and Python3, but in one case + # it's a Unicode string and in the other it's a byte string. + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text + elif isinstance(text, unicode): + return text.encode("utf-8") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + index = 0 + with tf.io.gfile.GFile(vocab_file, "r") as reader: + while True: + token = convert_to_unicode(reader.readline()) + if not token: + break + token = token.strip() + vocab[token] = index + index += 1 + return vocab + + +def convert_by_vocab(vocab, items): + """Converts a sequence of [tokens|ids] using the vocab.""" + output = [] + for item in items: + output.append(vocab[item]) + return output + + +def convert_tokens_to_ids(vocab, tokens): + return convert_by_vocab(vocab, tokens) + + +def convert_ids_to_tokens(inv_vocab, ids): + return convert_by_vocab(inv_vocab, ids) + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a piece of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class FullTokenizer(object): + """Runs end-to-end tokenziation.""" + + def __init__(self, vocab_file, do_lower_case=True, split_on_punc=True): + self.vocab = load_vocab(vocab_file) + self.inv_vocab = {v: k for k, v in self.vocab.items()} + self.basic_tokenizer = BasicTokenizer( + do_lower_case=do_lower_case, split_on_punc=split_on_punc) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + + def tokenize(self, text): + split_tokens = [] + for token in self.basic_tokenizer.tokenize(text): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + + return split_tokens + + def convert_tokens_to_ids(self, tokens): + return convert_by_vocab(self.vocab, tokens) + + def convert_ids_to_tokens(self, ids): + return convert_by_vocab(self.inv_vocab, ids) + + +class BasicTokenizer(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" + + def __init__(self, do_lower_case=True, split_on_punc=True): + """Constructs a BasicTokenizer. + + Args: + do_lower_case: Whether to lower case the input. + split_on_punc: Whether to apply split on punctuations. By default BERT + starts a new token for punctuations. This makes detokenization difficult + for tasks like seq2seq decoding. + """ + self.do_lower_case = do_lower_case + self.split_on_punc = split_on_punc + + def tokenize(self, text): + """Tokenizes a piece of text.""" + text = convert_to_unicode(text) + text = self._clean_text(text) + + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + text = self._tokenize_chinese_chars(text) + + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case: + token = token.lower() + token = self._run_strip_accents(token) + if self.split_on_punc: + split_tokens.extend(self._run_split_on_punc(token)) + else: + split_tokens.append(token) + + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(" ") + output.append(char) + output.append(" ") + else: + output.append(char) + return "".join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ((cp >= 0x4E00 and cp <= 0x9FFF) or # + (cp >= 0x3400 and cp <= 0x4DBF) or # + (cp >= 0x20000 and cp <= 0x2A6DF) or # + (cp >= 0x2A700 and cp <= 0x2B73F) or # + (cp >= 0x2B740 and cp <= 0x2B81F) or # + (cp >= 0x2B820 and cp <= 0x2CEAF) or + (cp >= 0xF900 and cp <= 0xFAFF) or # + (cp >= 0x2F800 and cp <= 0x2FA1F)): # + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xfffd or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenziation.""" + + def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=200): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + + For example: + input = "unaffable" + output = ["un", "##aff", "##able"] + + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer. + + Returns: + A list of wordpiece tokens. + """ + + text = convert_to_unicode(text) + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = "".join(chars[start:end]) + if start > 0: + substr = "##" + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically control characters but we treat them + # as whitespace since they are generally considered as such. + if char == " " or char == "\t" or char == "\n" or char == "\r": + return True + cat = unicodedata.category(char) + if cat == "Zs": + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == "\t" or char == "\n" or char == "\r": + return False + cat = unicodedata.category(char) + if cat in ("Cc", "Cf"): + return True + return False + + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or + (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + return True + cat = unicodedata.category(char) + if cat.startswith("P"): + return True + return False + + +def preprocess_text(inputs, remove_space=True, lower=False): + """Preprocesses data by removing extra space and normalize data. + + This method is used together with sentence piece tokenizer and is forked from: + https://github.com/google-research/google-research/blob/master/albert/tokenization.py + + Args: + inputs: The input text. + remove_space: Whether to remove the extra space. + lower: Whether to lowercase the text. + + Returns: + The preprocessed text. + + """ + outputs = inputs + if remove_space: + outputs = " ".join(inputs.strip().split()) + + if six.PY2 and isinstance(outputs, str): + try: + outputs = six.ensure_text(outputs, "utf-8") + except UnicodeDecodeError: + outputs = six.ensure_text(outputs, "latin-1") + + outputs = unicodedata.normalize("NFKD", outputs) + outputs = "".join([c for c in outputs if not unicodedata.combining(c)]) + if lower: + outputs = outputs.lower() + + return outputs + + +def encode_pieces(sp_model, text, sample=False): + """Segements text into pieces. + + This method is used together with sentence piece tokenizer and is forked from: + https://github.com/google-research/google-research/blob/master/albert/tokenization.py + + + Args: + sp_model: A spm.SentencePieceProcessor object. + text: The input text to be segemented. + sample: Whether to randomly sample a segmentation output or return a + deterministic one. + + Returns: + A list of token pieces. + """ + if six.PY2 and isinstance(text, six.text_type): + text = six.ensure_binary(text, "utf-8") + + if not sample: + pieces = sp_model.EncodeAsPieces(text) + else: + pieces = sp_model.SampleEncodeAsPieces(text, 64, 0.1) + new_pieces = [] + for piece in pieces: + piece = printable_text(piece) + if len(piece) > 1 and piece[-1] == "," and piece[-2].isdigit(): + cur_pieces = sp_model.EncodeAsPieces(piece[:-1].replace( + SPIECE_UNDERLINE, "")) + if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE: + if len(cur_pieces[0]) == 1: + cur_pieces = cur_pieces[1:] + else: + cur_pieces[0] = cur_pieces[0][1:] + cur_pieces.append(piece[-1]) + new_pieces.extend(cur_pieces) + else: + new_pieces.append(piece) + + return new_pieces + + +def encode_ids(sp_model, text, sample=False): + """Segments text and return token ids. + + This method is used together with sentence piece tokenizer and is forked from: + https://github.com/google-research/google-research/blob/master/albert/tokenization.py + + Args: + sp_model: A spm.SentencePieceProcessor object. + text: The input text to be segemented. + sample: Whether to randomly sample a segmentation output or return a + deterministic one. + + Returns: + A list of token ids. + """ + pieces = encode_pieces(sp_model, text, sample=sample) + ids = [sp_model.PieceToId(piece) for piece in pieces] + return ids + + +class FullSentencePieceTokenizer(object): + """Runs end-to-end sentence piece tokenization. + + The interface of this class is intended to keep the same as above + `FullTokenizer` class for easier usage. + """ + + def __init__(self, sp_model_file): + """Inits FullSentencePieceTokenizer. + + Args: + sp_model_file: The path to the sentence piece model file. + """ + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(sp_model_file) + self.vocab = { + self.sp_model.IdToPiece(i): i + for i in six.moves.range(self.sp_model.GetPieceSize()) + } + + def tokenize(self, text): + """Tokenizes text into pieces.""" + return encode_pieces(self.sp_model, text) + + def convert_tokens_to_ids(self, tokens): + """Converts a list of tokens to a list of ids.""" + return [self.sp_model.PieceToId(printable_text(token)) for token in tokens] + + def convert_ids_to_tokens(self, ids): + """Converts a list of ids ot a list of tokens.""" + return [self.sp_model.IdToPiece(id_) for id_ in ids] diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/tokenization_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/tokenization_test.py new file mode 100644 index 0000000..4a0503c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/bert/tokenization_test.py @@ -0,0 +1,160 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import tempfile + +import six +import tensorflow as tf + +from official.nlp.bert import tokenization + + +class TokenizationTest(tf.test.TestCase): + """Tokenization test. + + The implementation is forked from + https://github.com/google-research/bert/blob/master/tokenization_test.py." + """ + + def test_full_tokenizer(self): + vocab_tokens = [ + "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", + "##ing", "," + ] + with tempfile.NamedTemporaryFile(delete=False) as vocab_writer: + if six.PY2: + vocab_writer.write("".join([x + "\n" for x in vocab_tokens])) + else: + vocab_writer.write("".join([x + "\n" for x in vocab_tokens + ]).encode("utf-8")) + + vocab_file = vocab_writer.name + + tokenizer = tokenization.FullTokenizer(vocab_file) + os.unlink(vocab_file) + + tokens = tokenizer.tokenize(u"UNwant\u00E9d,running") + self.assertAllEqual(tokens, ["un", "##want", "##ed", ",", "runn", "##ing"]) + + self.assertAllEqual( + tokenizer.convert_tokens_to_ids(tokens), [7, 4, 5, 10, 8, 9]) + + def test_chinese(self): + tokenizer = tokenization.BasicTokenizer() + + self.assertAllEqual( + tokenizer.tokenize(u"ah\u535A\u63A8zz"), + [u"ah", u"\u535A", u"\u63A8", u"zz"]) + + def test_basic_tokenizer_lower(self): + tokenizer = tokenization.BasicTokenizer(do_lower_case=True) + + self.assertAllEqual( + tokenizer.tokenize(u" \tHeLLo!how \n Are yoU? "), + ["hello", "!", "how", "are", "you", "?"]) + self.assertAllEqual(tokenizer.tokenize(u"H\u00E9llo"), ["hello"]) + + def test_basic_tokenizer_no_lower(self): + tokenizer = tokenization.BasicTokenizer(do_lower_case=False) + + self.assertAllEqual( + tokenizer.tokenize(u" \tHeLLo!how \n Are yoU? "), + ["HeLLo", "!", "how", "Are", "yoU", "?"]) + + def test_basic_tokenizer_no_split_on_punc(self): + tokenizer = tokenization.BasicTokenizer( + do_lower_case=True, split_on_punc=False) + + self.assertAllEqual( + tokenizer.tokenize(u" \tHeLLo!how \n Are yoU? "), + ["hello!how", "are", "you?"]) + + def test_wordpiece_tokenizer(self): + vocab_tokens = [ + "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", + "##ing", "##!", "!" + ] + + vocab = {} + for (i, token) in enumerate(vocab_tokens): + vocab[token] = i + tokenizer = tokenization.WordpieceTokenizer(vocab=vocab) + + self.assertAllEqual(tokenizer.tokenize(""), []) + + self.assertAllEqual( + tokenizer.tokenize("unwanted running"), + ["un", "##want", "##ed", "runn", "##ing"]) + + self.assertAllEqual( + tokenizer.tokenize("unwanted running !"), + ["un", "##want", "##ed", "runn", "##ing", "!"]) + + self.assertAllEqual( + tokenizer.tokenize("unwanted running!"), + ["un", "##want", "##ed", "runn", "##ing", "##!"]) + + self.assertAllEqual( + tokenizer.tokenize("unwantedX running"), ["[UNK]", "runn", "##ing"]) + + def test_convert_tokens_to_ids(self): + vocab_tokens = [ + "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", + "##ing" + ] + + vocab = {} + for (i, token) in enumerate(vocab_tokens): + vocab[token] = i + + self.assertAllEqual( + tokenization.convert_tokens_to_ids( + vocab, ["un", "##want", "##ed", "runn", "##ing"]), [7, 4, 5, 8, 9]) + + def test_is_whitespace(self): + self.assertTrue(tokenization._is_whitespace(u" ")) + self.assertTrue(tokenization._is_whitespace(u"\t")) + self.assertTrue(tokenization._is_whitespace(u"\r")) + self.assertTrue(tokenization._is_whitespace(u"\n")) + self.assertTrue(tokenization._is_whitespace(u"\u00A0")) + + self.assertFalse(tokenization._is_whitespace(u"A")) + self.assertFalse(tokenization._is_whitespace(u"-")) + + def test_is_control(self): + self.assertTrue(tokenization._is_control(u"\u0005")) + + self.assertFalse(tokenization._is_control(u"A")) + self.assertFalse(tokenization._is_control(u" ")) + self.assertFalse(tokenization._is_control(u"\t")) + self.assertFalse(tokenization._is_control(u"\r")) + self.assertFalse(tokenization._is_control(u"\U0001F4A9")) + + def test_is_punctuation(self): + self.assertTrue(tokenization._is_punctuation(u"-")) + self.assertTrue(tokenization._is_punctuation(u"$")) + self.assertTrue(tokenization._is_punctuation(u"`")) + self.assertTrue(tokenization._is_punctuation(u".")) + + self.assertFalse(tokenization._is_punctuation(u"A")) + self.assertFalse(tokenization._is_punctuation(u" ")) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/classifier_data_lib.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/classifier_data_lib.py new file mode 100644 index 0000000..b143b5d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/classifier_data_lib.py @@ -0,0 +1,676 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""BERT library to process data for classification task.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import csv +import os + +from absl import logging +import tensorflow as tf +import tensorflow_datasets as tfds + +from official.nlp.bert import tokenization + + +class InputExample(object): + """A single training/test example for simple sequence classification.""" + + def __init__(self, guid, text_a, text_b=None, label=None): + """Constructs a InputExample. + + Args: + guid: Unique id for the example. + text_a: string. The untokenized text of the first sequence. For single + sequence tasks, only this sequence must be specified. + text_b: (Optional) string. The untokenized text of the second sequence. + Only must be specified for sequence pair tasks. + label: (Optional) string. The label of the example. This should be + specified for train and dev examples, but not for test examples. + """ + self.guid = guid + self.text_a = text_a + self.text_b = text_b + self.label = label + + +class InputFeatures(object): + """A single set of features of data.""" + + def __init__(self, + input_ids, + input_mask, + segment_ids, + label_id, + is_real_example=True): + self.input_ids = input_ids + self.input_mask = input_mask + self.segment_ids = segment_ids + self.label_id = label_id + self.is_real_example = is_real_example + + +class DataProcessor(object): + """Base class for data converters for sequence classification data sets.""" + + def __init__(self, process_text_fn=tokenization.convert_to_unicode): + self.process_text_fn = process_text_fn + + def get_train_examples(self, data_dir): + """Gets a collection of `InputExample`s for the train set.""" + raise NotImplementedError() + + def get_dev_examples(self, data_dir): + """Gets a collection of `InputExample`s for the dev set.""" + raise NotImplementedError() + + def get_test_examples(self, data_dir): + """Gets a collection of `InputExample`s for prediction.""" + raise NotImplementedError() + + def get_labels(self): + """Gets the list of labels for this data set.""" + raise NotImplementedError() + + @staticmethod + def get_processor_name(): + """Gets the string identifier of the processor.""" + raise NotImplementedError() + + @classmethod + def _read_tsv(cls, input_file, quotechar=None): + """Reads a tab separated value file.""" + with tf.io.gfile.GFile(input_file, "r") as f: + reader = csv.reader(f, delimiter="\t", quotechar=quotechar) + lines = [] + for line in reader: + lines.append(line) + return lines + + +class XnliProcessor(DataProcessor): + """Processor for the XNLI data set.""" + + def __init__(self, process_text_fn=tokenization.convert_to_unicode): + super(XnliProcessor, self).__init__(process_text_fn) + self.language = "zh" + + def get_train_examples(self, data_dir): + """See base class.""" + lines = self._read_tsv( + os.path.join(data_dir, "multinli", + "multinli.train.%s.tsv" % self.language)) + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "train-%d" % (i) + text_a = self.process_text_fn(line[0]) + text_b = self.process_text_fn(line[1]) + label = self.process_text_fn(line[2]) + if label == self.process_text_fn("contradictory"): + label = self.process_text_fn("contradiction") + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_dev_examples(self, data_dir): + """See base class.""" + lines = self._read_tsv(os.path.join(data_dir, "xnli.dev.tsv")) + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "dev-%d" % (i) + language = self.process_text_fn(line[0]) + if language != self.process_text_fn(self.language): + continue + text_a = self.process_text_fn(line[6]) + text_b = self.process_text_fn(line[7]) + label = self.process_text_fn(line[1]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "XNLI" + + +class MnliProcessor(DataProcessor): + """Processor for the MultiNLI data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev_matched.tsv")), + "dev_matched") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "test_matched.tsv")), "test") + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "MNLI" + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, self.process_text_fn(line[0])) + text_a = self.process_text_fn(line[8]) + text_b = self.process_text_fn(line[9]) + if set_type == "test": + label = "contradiction" + else: + label = self.process_text_fn(line[-1]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class MrpcProcessor(DataProcessor): + """Processor for the MRPC data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "test.tsv")), "test") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "MRPC" + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, i) + text_a = self.process_text_fn(line[3]) + text_b = self.process_text_fn(line[4]) + if set_type == "test": + label = "0" + else: + label = self.process_text_fn(line[0]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class ColaProcessor(DataProcessor): + """Processor for the CoLA data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "test.tsv")), "test") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "COLA" + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + # Only the test set has a header + if set_type == "test" and i == 0: + continue + guid = "%s-%s" % (set_type, i) + if set_type == "test": + text_a = self.process_text_fn(line[1]) + label = "0" + else: + text_a = self.process_text_fn(line[3]) + label = self.process_text_fn(line[1]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + return examples + + +class SstProcessor(DataProcessor): + """Processor for the SST-2 data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "test.tsv")), "test") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "SST-2" + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, i) + if set_type == "test": + text_a = tokenization.convert_to_unicode(line[1]) + label = "0" + else: + text_a = tokenization.convert_to_unicode(line[0]) + label = tokenization.convert_to_unicode(line[1]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + return examples + + +class QnliProcessor(DataProcessor): + """Processor for the QNLI data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev_matched") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "test.tsv")), "test") + + def get_labels(self): + """See base class.""" + return ["entailment", "not_entailment"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "QNLI" + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, 1) + if set_type == "test": + text_a = tokenization.convert_to_unicode(line[1]) + text_b = tokenization.convert_to_unicode(line[2]) + label = "entailment" + else: + text_a = tokenization.convert_to_unicode(line[1]) + text_b = tokenization.convert_to_unicode(line[2]) + label = tokenization.convert_to_unicode(line[-1]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class TfdsProcessor(DataProcessor): + """Processor for generic text classification TFDS data set. + + The TFDS parameters are expected to be provided in the tfds_params string, in + a comma-separated list of parameter assignments. + Examples: + tfds_params="dataset=scicite,text_key=string" + tfds_params="dataset=imdb_reviews,test_split=,dev_split=test" + tfds_params="dataset=glue/cola,text_key=sentence" + tfds_params="dataset=glue/sst2,text_key=sentence" + tfds_params="dataset=glue/qnli,text_key=question,text_b_key=sentence" + tfds_params="dataset=glue/mrpc,text_key=sentence1,text_b_key=sentence2" + Possible parameters (please refer to the documentation of Tensorflow Datasets + (TFDS) for the meaning of individual parameters): + dataset: Required dataset name (potentially with subset and version number). + data_dir: Optional TFDS source root directory. + train_split: Name of the train split (defaults to `train`). + dev_split: Name of the dev split (defaults to `validation`). + test_split: Name of the test split (defaults to `test`). + text_key: Key of the text_a feature (defaults to `text`). + text_b_key: Key of the second text feature if available. + label_key: Key of the label feature (defaults to `label`). + test_text_key: Key of the text feature to use in test set. + test_text_b_key: Key of the second text feature to use in test set. + test_label: String to be used as the label for all test examples. + """ + + def __init__(self, tfds_params, + process_text_fn=tokenization.convert_to_unicode): + super(TfdsProcessor, self).__init__(process_text_fn) + self._process_tfds_params_str(tfds_params) + self.dataset, info = tfds.load(self.dataset_name, data_dir=self.data_dir, + with_info=True) + self._labels = list(range(info.features[self.label_key].num_classes)) + + def _process_tfds_params_str(self, params_str): + """Extracts TFDS parameters from a comma-separated assignements string.""" + tuples = [x.split("=") for x in params_str.split(",")] + d = {k.strip(): v.strip() for k, v in tuples} + self.dataset_name = d["dataset"] # Required. + self.data_dir = d.get("data_dir", None) + self.train_split = d.get("train_split", "train") + self.dev_split = d.get("dev_split", "validation") + self.test_split = d.get("test_split", "test") + self.text_key = d.get("text_key", "text") + self.text_b_key = d.get("text_b_key", None) + self.label_key = d.get("label_key", "label") + self.test_text_key = d.get("test_text_key", self.text_key) + self.test_text_b_key = d.get("test_text_b_key", self.text_b_key) + self.test_label = d.get("test_label", "test_example") + + def get_train_examples(self, data_dir): + assert data_dir is None + return self._create_examples(self.train_split, "train") + + def get_dev_examples(self, data_dir): + assert data_dir is None + return self._create_examples(self.dev_split, "dev") + + def get_test_examples(self, data_dir): + assert data_dir is None + return self._create_examples(self.test_split, "test") + + def get_labels(self): + return self._labels + + def get_processor_name(self): + return "TFDS_" + self.dataset_name + + def _create_examples(self, split_name, set_type): + """Creates examples for the training and dev sets.""" + if split_name not in self.dataset: + raise ValueError("Split {} not available.".format(split_name)) + dataset = self.dataset[split_name].as_numpy_iterator() + examples = [] + text_b = None + for i, example in enumerate(dataset): + guid = "%s-%s" % (set_type, i) + if set_type == "test": + text_a = self.process_text_fn(example[self.test_text_key]) + if self.test_text_b_key: + text_b = self.process_text_fn(example[self.test_text_b_key]) + label = self.test_label + else: + text_a = self.process_text_fn(example[self.text_key]) + if self.text_b_key: + text_b = self.process_text_fn(example[self.text_b_key]) + label = int(example[self.label_key]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +def convert_single_example(ex_index, example, label_list, max_seq_length, + tokenizer): + """Converts a single `InputExample` into a single `InputFeatures`.""" + label_map = {} + for (i, label) in enumerate(label_list): + label_map[label] = i + + tokens_a = tokenizer.tokenize(example.text_a) + tokens_b = None + if example.text_b: + tokens_b = tokenizer.tokenize(example.text_b) + + if tokens_b: + # Modifies `tokens_a` and `tokens_b` in place so that the total + # length is less than the specified length. + # Account for [CLS], [SEP], [SEP] with "- 3" + _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3) + else: + # Account for [CLS] and [SEP] with "- 2" + if len(tokens_a) > max_seq_length - 2: + tokens_a = tokens_a[0:(max_seq_length - 2)] + + # The convention in BERT is: + # (a) For sequence pairs: + # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] + # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 + # (b) For single sequences: + # tokens: [CLS] the dog is hairy . [SEP] + # type_ids: 0 0 0 0 0 0 0 + # + # Where "type_ids" are used to indicate whether this is the first + # sequence or the second sequence. The embedding vectors for `type=0` and + # `type=1` were learned during pre-training and are added to the wordpiece + # embedding vector (and position vector). This is not *strictly* necessary + # since the [SEP] token unambiguously separates the sequences, but it makes + # it easier for the model to learn the concept of sequences. + # + # For classification tasks, the first vector (corresponding to [CLS]) is + # used as the "sentence vector". Note that this only makes sense because + # the entire model is fine-tuned. + tokens = [] + segment_ids = [] + tokens.append("[CLS]") + segment_ids.append(0) + for token in tokens_a: + tokens.append(token) + segment_ids.append(0) + tokens.append("[SEP]") + segment_ids.append(0) + + if tokens_b: + for token in tokens_b: + tokens.append(token) + segment_ids.append(1) + tokens.append("[SEP]") + segment_ids.append(1) + + input_ids = tokenizer.convert_tokens_to_ids(tokens) + + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + input_mask = [1] * len(input_ids) + + # Zero-pad up to the sequence length. + while len(input_ids) < max_seq_length: + input_ids.append(0) + input_mask.append(0) + segment_ids.append(0) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + + label_id = label_map[example.label] + if ex_index < 5: + logging.info("*** Example ***") + logging.info("guid: %s", (example.guid)) + logging.info("tokens: %s", + " ".join([tokenization.printable_text(x) for x in tokens])) + logging.info("input_ids: %s", " ".join([str(x) for x in input_ids])) + logging.info("input_mask: %s", " ".join([str(x) for x in input_mask])) + logging.info("segment_ids: %s", " ".join([str(x) for x in segment_ids])) + logging.info("label: %s (id = %d)", example.label, label_id) + + feature = InputFeatures( + input_ids=input_ids, + input_mask=input_mask, + segment_ids=segment_ids, + label_id=label_id, + is_real_example=True) + return feature + + +def file_based_convert_examples_to_features(examples, label_list, + max_seq_length, tokenizer, + output_file): + """Convert a set of `InputExample`s to a TFRecord file.""" + + tf.io.gfile.makedirs(os.path.dirname(output_file)) + writer = tf.io.TFRecordWriter(output_file) + + for (ex_index, example) in enumerate(examples): + if ex_index % 10000 == 0: + logging.info("Writing example %d of %d", ex_index, len(examples)) + + feature = convert_single_example(ex_index, example, label_list, + max_seq_length, tokenizer) + + def create_int_feature(values): + f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) + return f + + features = collections.OrderedDict() + features["input_ids"] = create_int_feature(feature.input_ids) + features["input_mask"] = create_int_feature(feature.input_mask) + features["segment_ids"] = create_int_feature(feature.segment_ids) + features["label_ids"] = create_int_feature([feature.label_id]) + features["is_real_example"] = create_int_feature( + [int(feature.is_real_example)]) + + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + writer.write(tf_example.SerializeToString()) + writer.close() + + +def _truncate_seq_pair(tokens_a, tokens_b, max_length): + """Truncates a sequence pair in place to the maximum length.""" + + # This is a simple heuristic which will always truncate the longer sequence + # one token at a time. This makes more sense than truncating an equal percent + # of tokens from each, since if one sequence is very short then each token + # that's truncated likely contains more information than a longer sequence. + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_length: + break + if len(tokens_a) > len(tokens_b): + tokens_a.pop() + else: + tokens_b.pop() + + +def generate_tf_record_from_data_file(processor, + data_dir, + tokenizer, + train_data_output_path=None, + eval_data_output_path=None, + max_seq_length=128): + """Generates and saves training data into a tf record file. + + Arguments: + processor: Input processor object to be used for generating data. Subclass + of `DataProcessor`. + data_dir: Directory that contains train/eval data to process. Data files + should be in from "dev.tsv", "test.tsv", or "train.tsv". + tokenizer: The tokenizer to be applied on the data. + train_data_output_path: Output to which processed tf record for training + will be saved. + eval_data_output_path: Output to which processed tf record for evaluation + will be saved. + max_seq_length: Maximum sequence length of the to be generated + training/eval data. + + Returns: + A dictionary containing input meta data. + """ + assert train_data_output_path or eval_data_output_path + + label_list = processor.get_labels() + assert train_data_output_path + train_input_data_examples = processor.get_train_examples(data_dir) + file_based_convert_examples_to_features(train_input_data_examples, label_list, + max_seq_length, tokenizer, + train_data_output_path) + num_training_data = len(train_input_data_examples) + + if eval_data_output_path: + eval_input_data_examples = processor.get_dev_examples(data_dir) + file_based_convert_examples_to_features(eval_input_data_examples, + label_list, max_seq_length, + tokenizer, eval_data_output_path) + + meta_data = { + "task_type": "bert_classification", + "processor_type": processor.get_processor_name(), + "num_labels": len(processor.get_labels()), + "train_data_size": num_training_data, + "max_seq_length": max_seq_length, + } + + if eval_data_output_path: + meta_data["eval_data_size"] = len(eval_input_data_examples) + + return meta_data diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/create_finetuning_data.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/create_finetuning_data.py new file mode 100644 index 0000000..92b8abe --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/create_finetuning_data.py @@ -0,0 +1,203 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""BERT finetuning task dataset generator.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools +import json +import os + +from absl import app +from absl import flags +import tensorflow as tf +from official.nlp.bert import tokenization +from official.nlp.data import classifier_data_lib +# word-piece tokenizer based squad_lib +from official.nlp.data import squad_lib as squad_lib_wp +# sentence-piece tokenizer based squad_lib +from official.nlp.data import squad_lib_sp + +FLAGS = flags.FLAGS + +flags.DEFINE_enum( + "fine_tuning_task_type", "classification", ["classification", "squad"], + "The name of the BERT fine tuning task for which data " + "will be generated..") + +# BERT classification specific flags. +flags.DEFINE_string( + "input_data_dir", None, + "The input data dir. Should contain the .tsv files (or other data files) " + "for the task.") + +flags.DEFINE_enum("classification_task_name", "MNLI", + ["COLA", "MNLI", "MRPC", "QNLI", "SST-2", "XNLI"], + "The name of the task to train BERT classifier.") + +# BERT Squad task specific flags. +flags.DEFINE_string( + "squad_data_file", None, + "The input data file in for generating training data for BERT squad task.") + +flags.DEFINE_integer( + "doc_stride", 128, + "When splitting up a long document into chunks, how much stride to " + "take between chunks.") + +flags.DEFINE_integer( + "max_query_length", 64, + "The maximum number of tokens for the question. Questions longer than " + "this will be truncated to this length.") + +flags.DEFINE_bool( + "version_2_with_negative", False, + "If true, the SQuAD examples contain some that do not have an answer.") + +# Shared flags across BERT fine-tuning tasks. +flags.DEFINE_string("vocab_file", None, + "The vocabulary file that the BERT model was trained on.") + +flags.DEFINE_string( + "train_data_output_path", None, + "The path in which generated training input data will be written as tf" + " records.") + +flags.DEFINE_string( + "eval_data_output_path", None, + "The path in which generated training input data will be written as tf" + " records.") + +flags.DEFINE_string("meta_data_file_path", None, + "The path in which input meta data will be written.") + +flags.DEFINE_bool( + "do_lower_case", True, + "Whether to lower case the input text. Should be True for uncased " + "models and False for cased models.") + +flags.DEFINE_integer( + "max_seq_length", 128, + "The maximum total input sequence length after WordPiece tokenization. " + "Sequences longer than this will be truncated, and sequences shorter " + "than this will be padded.") + +flags.DEFINE_string("sp_model_file", "", + "The path to the model used by sentence piece tokenizer.") + +flags.DEFINE_enum( + "tokenizer_impl", "word_piece", ["word_piece", "sentence_piece"], + "Specifies the tokenizer implementation, i.e., whehter to use word_piece " + "or sentence_piece tokenizer. Canonical BERT uses word_piece tokenizer, " + "while ALBERT uses sentence_piece tokenizer.") + +flags.DEFINE_string("tfds_params", "", + "Comma-separated list of TFDS parameter assigments for " + "generic classfication data import (for more details " + "see the TfdsProcessor class documentation).") + + +def generate_classifier_dataset(): + """Generates classifier dataset and returns input meta data.""" + assert (FLAGS.input_data_dir and FLAGS.classification_task_name + or FLAGS.tfds_params) + + if FLAGS.tokenizer_impl == "word_piece": + tokenizer = tokenization.FullTokenizer( + vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) + processor_text_fn = tokenization.convert_to_unicode + else: + assert FLAGS.tokenizer_impl == "sentence_piece" + tokenizer = tokenization.FullSentencePieceTokenizer(FLAGS.sp_model_file) + processor_text_fn = functools.partial( + tokenization.preprocess_text, lower=FLAGS.do_lower_case) + + if FLAGS.tfds_params: + processor = classifier_data_lib.TfdsProcessor( + tfds_params=FLAGS.tfds_params, + process_text_fn=processor_text_fn) + return classifier_data_lib.generate_tf_record_from_data_file( + processor, + None, + tokenizer, + train_data_output_path=FLAGS.train_data_output_path, + eval_data_output_path=FLAGS.eval_data_output_path, + max_seq_length=FLAGS.max_seq_length) + else: + processors = { + "cola": classifier_data_lib.ColaProcessor, + "mnli": classifier_data_lib.MnliProcessor, + "mrpc": classifier_data_lib.MrpcProcessor, + "qnli": classifier_data_lib.QnliProcessor, + "sst-2": classifier_data_lib.SstProcessor, + "xnli": classifier_data_lib.XnliProcessor, + } + task_name = FLAGS.classification_task_name.lower() + if task_name not in processors: + raise ValueError("Task not found: %s" % (task_name)) + + processor = processors[task_name](processor_text_fn) + return classifier_data_lib.generate_tf_record_from_data_file( + processor, + FLAGS.input_data_dir, + tokenizer, + train_data_output_path=FLAGS.train_data_output_path, + eval_data_output_path=FLAGS.eval_data_output_path, + max_seq_length=FLAGS.max_seq_length) + + +def generate_squad_dataset(): + """Generates squad training dataset and returns input meta data.""" + assert FLAGS.squad_data_file + if FLAGS.tokenizer_impl == "word_piece": + return squad_lib_wp.generate_tf_record_from_json_file( + FLAGS.squad_data_file, FLAGS.vocab_file, FLAGS.train_data_output_path, + FLAGS.max_seq_length, FLAGS.do_lower_case, FLAGS.max_query_length, + FLAGS.doc_stride, FLAGS.version_2_with_negative) + else: + assert FLAGS.tokenizer_impl == "sentence_piece" + return squad_lib_sp.generate_tf_record_from_json_file( + FLAGS.squad_data_file, FLAGS.sp_model_file, + FLAGS.train_data_output_path, FLAGS.max_seq_length, FLAGS.do_lower_case, + FLAGS.max_query_length, FLAGS.doc_stride, FLAGS.version_2_with_negative) + + +def main(_): + if FLAGS.tokenizer_impl == "word_piece": + if not FLAGS.vocab_file: + raise ValueError( + "FLAG vocab_file for word-piece tokenizer is not specified.") + else: + assert FLAGS.tokenizer_impl == "sentence_piece" + if not FLAGS.sp_model_file: + raise ValueError( + "FLAG sp_model_file for sentence-piece tokenizer is not specified.") + + if FLAGS.fine_tuning_task_type == "classification": + input_meta_data = generate_classifier_dataset() + else: + input_meta_data = generate_squad_dataset() + + tf.io.gfile.makedirs(os.path.dirname(FLAGS.meta_data_file_path)) + with tf.io.gfile.GFile(FLAGS.meta_data_file_path, "w") as writer: + writer.write(json.dumps(input_meta_data, indent=4) + "\n") + + +if __name__ == "__main__": + flags.mark_flag_as_required("train_data_output_path") + flags.mark_flag_as_required("meta_data_file_path") + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/create_pretraining_data.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/create_pretraining_data.py new file mode 100644 index 0000000..79dac57 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/create_pretraining_data.py @@ -0,0 +1,486 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Create masked LM/next sentence masked_lm TF examples for BERT.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import random + +from absl import app +from absl import flags +from absl import logging +import tensorflow as tf + +from official.nlp.bert import tokenization + +FLAGS = flags.FLAGS + +flags.DEFINE_string("input_file", None, + "Input raw text file (or comma-separated list of files).") + +flags.DEFINE_string( + "output_file", None, + "Output TF example file (or comma-separated list of files).") + +flags.DEFINE_string("vocab_file", None, + "The vocabulary file that the BERT model was trained on.") + +flags.DEFINE_bool( + "do_lower_case", True, + "Whether to lower case the input text. Should be True for uncased " + "models and False for cased models.") + +flags.DEFINE_bool( + "do_whole_word_mask", False, + "Whether to use whole word masking rather than per-WordPiece masking.") + +flags.DEFINE_bool( + "gzip_compress", False, + "Whether to use `GZIP` compress option to get compressed TFRecord files.") + +flags.DEFINE_integer("max_seq_length", 128, "Maximum sequence length.") + +flags.DEFINE_integer("max_predictions_per_seq", 20, + "Maximum number of masked LM predictions per sequence.") + +flags.DEFINE_integer("random_seed", 12345, "Random seed for data generation.") + +flags.DEFINE_integer( + "dupe_factor", 10, + "Number of times to duplicate the input data (with different masks).") + +flags.DEFINE_float("masked_lm_prob", 0.15, "Masked LM probability.") + +flags.DEFINE_float( + "short_seq_prob", 0.1, + "Probability of creating sequences which are shorter than the " + "maximum length.") + + +class TrainingInstance(object): + """A single training instance (sentence pair).""" + + def __init__(self, tokens, segment_ids, masked_lm_positions, masked_lm_labels, + is_random_next): + self.tokens = tokens + self.segment_ids = segment_ids + self.is_random_next = is_random_next + self.masked_lm_positions = masked_lm_positions + self.masked_lm_labels = masked_lm_labels + + def __str__(self): + s = "" + s += "tokens: %s\n" % (" ".join( + [tokenization.printable_text(x) for x in self.tokens])) + s += "segment_ids: %s\n" % (" ".join([str(x) for x in self.segment_ids])) + s += "is_random_next: %s\n" % self.is_random_next + s += "masked_lm_positions: %s\n" % (" ".join( + [str(x) for x in self.masked_lm_positions])) + s += "masked_lm_labels: %s\n" % (" ".join( + [tokenization.printable_text(x) for x in self.masked_lm_labels])) + s += "\n" + return s + + def __repr__(self): + return self.__str__() + + +def write_instance_to_example_files(instances, tokenizer, max_seq_length, + max_predictions_per_seq, output_files, + gzip_compress): + """Create TF example files from `TrainingInstance`s.""" + writers = [] + for output_file in output_files: + writers.append( + tf.io.TFRecordWriter( + output_file, options="GZIP" if gzip_compress else "")) + + writer_index = 0 + + total_written = 0 + for (inst_index, instance) in enumerate(instances): + input_ids = tokenizer.convert_tokens_to_ids(instance.tokens) + input_mask = [1] * len(input_ids) + segment_ids = list(instance.segment_ids) + assert len(input_ids) <= max_seq_length + + while len(input_ids) < max_seq_length: + input_ids.append(0) + input_mask.append(0) + segment_ids.append(0) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + + masked_lm_positions = list(instance.masked_lm_positions) + masked_lm_ids = tokenizer.convert_tokens_to_ids(instance.masked_lm_labels) + masked_lm_weights = [1.0] * len(masked_lm_ids) + + while len(masked_lm_positions) < max_predictions_per_seq: + masked_lm_positions.append(0) + masked_lm_ids.append(0) + masked_lm_weights.append(0.0) + + next_sentence_label = 1 if instance.is_random_next else 0 + + features = collections.OrderedDict() + features["input_ids"] = create_int_feature(input_ids) + features["input_mask"] = create_int_feature(input_mask) + features["segment_ids"] = create_int_feature(segment_ids) + features["masked_lm_positions"] = create_int_feature(masked_lm_positions) + features["masked_lm_ids"] = create_int_feature(masked_lm_ids) + features["masked_lm_weights"] = create_float_feature(masked_lm_weights) + features["next_sentence_labels"] = create_int_feature([next_sentence_label]) + + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + + writers[writer_index].write(tf_example.SerializeToString()) + writer_index = (writer_index + 1) % len(writers) + + total_written += 1 + + if inst_index < 20: + logging.info("*** Example ***") + logging.info("tokens: %s", " ".join( + [tokenization.printable_text(x) for x in instance.tokens])) + + for feature_name in features.keys(): + feature = features[feature_name] + values = [] + if feature.int64_list.value: + values = feature.int64_list.value + elif feature.float_list.value: + values = feature.float_list.value + logging.info("%s: %s", feature_name, " ".join([str(x) for x in values])) + + for writer in writers: + writer.close() + + logging.info("Wrote %d total instances", total_written) + + +def create_int_feature(values): + feature = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) + return feature + + +def create_float_feature(values): + feature = tf.train.Feature(float_list=tf.train.FloatList(value=list(values))) + return feature + + +def create_training_instances(input_files, + tokenizer, + max_seq_length, + dupe_factor, + short_seq_prob, + masked_lm_prob, + max_predictions_per_seq, + rng, + do_whole_word_mask=False): + """Create `TrainingInstance`s from raw text.""" + all_documents = [[]] + + # Input file format: + # (1) One sentence per line. These should ideally be actual sentences, not + # entire paragraphs or arbitrary spans of text. (Because we use the + # sentence boundaries for the "next sentence prediction" task). + # (2) Blank lines between documents. Document boundaries are needed so + # that the "next sentence prediction" task doesn't span between documents. + for input_file in input_files: + with tf.io.gfile.GFile(input_file, "rb") as reader: + while True: + line = tokenization.convert_to_unicode(reader.readline()) + if not line: + break + line = line.strip() + + # Empty lines are used as document delimiters + if not line: + all_documents.append([]) + tokens = tokenizer.tokenize(line) + if tokens: + all_documents[-1].append(tokens) + + # Remove empty documents + all_documents = [x for x in all_documents if x] + rng.shuffle(all_documents) + + vocab_words = list(tokenizer.vocab.keys()) + instances = [] + for _ in range(dupe_factor): + for document_index in range(len(all_documents)): + instances.extend( + create_instances_from_document( + all_documents, document_index, max_seq_length, short_seq_prob, + masked_lm_prob, max_predictions_per_seq, vocab_words, rng, + do_whole_word_mask)) + + rng.shuffle(instances) + return instances + + +def create_instances_from_document( + all_documents, document_index, max_seq_length, short_seq_prob, + masked_lm_prob, max_predictions_per_seq, vocab_words, rng, + do_whole_word_mask=False): + """Creates `TrainingInstance`s for a single document.""" + document = all_documents[document_index] + + # Account for [CLS], [SEP], [SEP] + max_num_tokens = max_seq_length - 3 + + # We *usually* want to fill up the entire sequence since we are padding + # to `max_seq_length` anyways, so short sequences are generally wasted + # computation. However, we *sometimes* + # (i.e., short_seq_prob == 0.1 == 10% of the time) want to use shorter + # sequences to minimize the mismatch between pre-training and fine-tuning. + # The `target_seq_length` is just a rough target however, whereas + # `max_seq_length` is a hard limit. + target_seq_length = max_num_tokens + if rng.random() < short_seq_prob: + target_seq_length = rng.randint(2, max_num_tokens) + + # We DON'T just concatenate all of the tokens from a document into a long + # sequence and choose an arbitrary split point because this would make the + # next sentence prediction task too easy. Instead, we split the input into + # segments "A" and "B" based on the actual "sentences" provided by the user + # input. + instances = [] + current_chunk = [] + current_length = 0 + i = 0 + while i < len(document): + segment = document[i] + current_chunk.append(segment) + current_length += len(segment) + if i == len(document) - 1 or current_length >= target_seq_length: + if current_chunk: + # `a_end` is how many segments from `current_chunk` go into the `A` + # (first) sentence. + a_end = 1 + if len(current_chunk) >= 2: + a_end = rng.randint(1, len(current_chunk) - 1) + + tokens_a = [] + for j in range(a_end): + tokens_a.extend(current_chunk[j]) + + tokens_b = [] + # Random next + is_random_next = False + if len(current_chunk) == 1 or rng.random() < 0.5: + is_random_next = True + target_b_length = target_seq_length - len(tokens_a) + + # This should rarely go for more than one iteration for large + # corpora. However, just to be careful, we try to make sure that + # the random document is not the same as the document + # we're processing. + for _ in range(10): + random_document_index = rng.randint(0, len(all_documents) - 1) + if random_document_index != document_index: + break + + random_document = all_documents[random_document_index] + random_start = rng.randint(0, len(random_document) - 1) + for j in range(random_start, len(random_document)): + tokens_b.extend(random_document[j]) + if len(tokens_b) >= target_b_length: + break + # We didn't actually use these segments so we "put them back" so + # they don't go to waste. + num_unused_segments = len(current_chunk) - a_end + i -= num_unused_segments + # Actual next + else: + is_random_next = False + for j in range(a_end, len(current_chunk)): + tokens_b.extend(current_chunk[j]) + truncate_seq_pair(tokens_a, tokens_b, max_num_tokens, rng) + + assert len(tokens_a) >= 1 + assert len(tokens_b) >= 1 + + tokens = [] + segment_ids = [] + tokens.append("[CLS]") + segment_ids.append(0) + for token in tokens_a: + tokens.append(token) + segment_ids.append(0) + + tokens.append("[SEP]") + segment_ids.append(0) + + for token in tokens_b: + tokens.append(token) + segment_ids.append(1) + tokens.append("[SEP]") + segment_ids.append(1) + + (tokens, masked_lm_positions, + masked_lm_labels) = create_masked_lm_predictions( + tokens, masked_lm_prob, max_predictions_per_seq, vocab_words, rng, + do_whole_word_mask) + instance = TrainingInstance( + tokens=tokens, + segment_ids=segment_ids, + is_random_next=is_random_next, + masked_lm_positions=masked_lm_positions, + masked_lm_labels=masked_lm_labels) + instances.append(instance) + current_chunk = [] + current_length = 0 + i += 1 + + return instances + + +MaskedLmInstance = collections.namedtuple("MaskedLmInstance", + ["index", "label"]) + + +def create_masked_lm_predictions(tokens, masked_lm_prob, + max_predictions_per_seq, vocab_words, rng, + do_whole_word_mask): + """Creates the predictions for the masked LM objective.""" + + cand_indexes = [] + for (i, token) in enumerate(tokens): + if token == "[CLS]" or token == "[SEP]": + continue + # Whole Word Masking means that if we mask all of the wordpieces + # corresponding to an original word. When a word has been split into + # WordPieces, the first token does not have any marker and any subsequence + # tokens are prefixed with ##. So whenever we see the ## token, we + # append it to the previous set of word indexes. + # + # Note that Whole Word Masking does *not* change the training code + # at all -- we still predict each WordPiece independently, softmaxed + # over the entire vocabulary. + if (do_whole_word_mask and len(cand_indexes) >= 1 and + token.startswith("##")): + cand_indexes[-1].append(i) + else: + cand_indexes.append([i]) + + rng.shuffle(cand_indexes) + + output_tokens = list(tokens) + + num_to_predict = min(max_predictions_per_seq, + max(1, int(round(len(tokens) * masked_lm_prob)))) + + masked_lms = [] + covered_indexes = set() + for index_set in cand_indexes: + if len(masked_lms) >= num_to_predict: + break + # If adding a whole-word mask would exceed the maximum number of + # predictions, then just skip this candidate. + if len(masked_lms) + len(index_set) > num_to_predict: + continue + is_any_index_covered = False + for index in index_set: + if index in covered_indexes: + is_any_index_covered = True + break + if is_any_index_covered: + continue + for index in index_set: + covered_indexes.add(index) + + masked_token = None + # 80% of the time, replace with [MASK] + if rng.random() < 0.8: + masked_token = "[MASK]" + else: + # 10% of the time, keep original + if rng.random() < 0.5: + masked_token = tokens[index] + # 10% of the time, replace with random word + else: + masked_token = vocab_words[rng.randint(0, len(vocab_words) - 1)] + + output_tokens[index] = masked_token + + masked_lms.append(MaskedLmInstance(index=index, label=tokens[index])) + assert len(masked_lms) <= num_to_predict + masked_lms = sorted(masked_lms, key=lambda x: x.index) + + masked_lm_positions = [] + masked_lm_labels = [] + for p in masked_lms: + masked_lm_positions.append(p.index) + masked_lm_labels.append(p.label) + + return (output_tokens, masked_lm_positions, masked_lm_labels) + + +def truncate_seq_pair(tokens_a, tokens_b, max_num_tokens, rng): + """Truncates a pair of sequences to a maximum sequence length.""" + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_num_tokens: + break + + trunc_tokens = tokens_a if len(tokens_a) > len(tokens_b) else tokens_b + assert len(trunc_tokens) >= 1 + + # We want to sometimes truncate from the front and sometimes from the + # back to add more randomness and avoid biases. + if rng.random() < 0.5: + del trunc_tokens[0] + else: + trunc_tokens.pop() + + +def main(_): + tokenizer = tokenization.FullTokenizer( + vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) + + input_files = [] + for input_pattern in FLAGS.input_file.split(","): + input_files.extend(tf.io.gfile.glob(input_pattern)) + + logging.info("*** Reading from input files ***") + for input_file in input_files: + logging.info(" %s", input_file) + + rng = random.Random(FLAGS.random_seed) + instances = create_training_instances( + input_files, tokenizer, FLAGS.max_seq_length, FLAGS.dupe_factor, + FLAGS.short_seq_prob, FLAGS.masked_lm_prob, FLAGS.max_predictions_per_seq, + rng, FLAGS.do_whole_word_mask) + + output_files = FLAGS.output_file.split(",") + logging.info("*** Writing to output files ***") + for output_file in output_files: + logging.info(" %s", output_file) + + write_instance_to_example_files(instances, tokenizer, FLAGS.max_seq_length, + FLAGS.max_predictions_per_seq, output_files, + FLAGS.gzip_compress) + + +if __name__ == "__main__": + flags.mark_flag_as_required("input_file") + flags.mark_flag_as_required("output_file") + flags.mark_flag_as_required("vocab_file") + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/squad_lib.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/squad_lib.py new file mode 100644 index 0000000..54f75a6 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/squad_lib.py @@ -0,0 +1,880 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Library to process data for SQuAD 1.1 and SQuAD 2.0.""" + +# pylint: disable=g-bad-import-order +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import copy +import json +import math +import six + +from absl import logging +import tensorflow as tf + +from official.nlp.bert import tokenization + + +class SquadExample(object): + """A single training/test example for simple sequence classification. + + For examples without an answer, the start and end position are -1. + """ + + def __init__(self, + qas_id, + question_text, + doc_tokens, + orig_answer_text=None, + start_position=None, + end_position=None, + is_impossible=False): + self.qas_id = qas_id + self.question_text = question_text + self.doc_tokens = doc_tokens + self.orig_answer_text = orig_answer_text + self.start_position = start_position + self.end_position = end_position + self.is_impossible = is_impossible + + def __str__(self): + return self.__repr__() + + def __repr__(self): + s = "" + s += "qas_id: %s" % (tokenization.printable_text(self.qas_id)) + s += ", question_text: %s" % ( + tokenization.printable_text(self.question_text)) + s += ", doc_tokens: [%s]" % (" ".join(self.doc_tokens)) + if self.start_position: + s += ", start_position: %d" % (self.start_position) + if self.start_position: + s += ", end_position: %d" % (self.end_position) + if self.start_position: + s += ", is_impossible: %r" % (self.is_impossible) + return s + + +class InputFeatures(object): + """A single set of features of data.""" + + def __init__(self, + unique_id, + example_index, + doc_span_index, + tokens, + token_to_orig_map, + token_is_max_context, + input_ids, + input_mask, + segment_ids, + start_position=None, + end_position=None, + is_impossible=None): + self.unique_id = unique_id + self.example_index = example_index + self.doc_span_index = doc_span_index + self.tokens = tokens + self.token_to_orig_map = token_to_orig_map + self.token_is_max_context = token_is_max_context + self.input_ids = input_ids + self.input_mask = input_mask + self.segment_ids = segment_ids + self.start_position = start_position + self.end_position = end_position + self.is_impossible = is_impossible + + +class FeatureWriter(object): + """Writes InputFeature to TF example file.""" + + def __init__(self, filename, is_training): + self.filename = filename + self.is_training = is_training + self.num_features = 0 + self._writer = tf.io.TFRecordWriter(filename) + + def process_feature(self, feature): + """Write a InputFeature to the TFRecordWriter as a tf.train.Example.""" + self.num_features += 1 + + def create_int_feature(values): + feature = tf.train.Feature( + int64_list=tf.train.Int64List(value=list(values))) + return feature + + features = collections.OrderedDict() + features["unique_ids"] = create_int_feature([feature.unique_id]) + features["input_ids"] = create_int_feature(feature.input_ids) + features["input_mask"] = create_int_feature(feature.input_mask) + features["segment_ids"] = create_int_feature(feature.segment_ids) + + if self.is_training: + features["start_positions"] = create_int_feature([feature.start_position]) + features["end_positions"] = create_int_feature([feature.end_position]) + impossible = 0 + if feature.is_impossible: + impossible = 1 + features["is_impossible"] = create_int_feature([impossible]) + + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + self._writer.write(tf_example.SerializeToString()) + + def close(self): + self._writer.close() + + +def read_squad_examples(input_file, is_training, version_2_with_negative): + """Read a SQuAD json file into a list of SquadExample.""" + with tf.io.gfile.GFile(input_file, "r") as reader: + input_data = json.load(reader)["data"] + + def is_whitespace(c): + if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F: + return True + return False + + examples = [] + for entry in input_data: + for paragraph in entry["paragraphs"]: + paragraph_text = paragraph["context"] + doc_tokens = [] + char_to_word_offset = [] + prev_is_whitespace = True + for c in paragraph_text: + if is_whitespace(c): + prev_is_whitespace = True + else: + if prev_is_whitespace: + doc_tokens.append(c) + else: + doc_tokens[-1] += c + prev_is_whitespace = False + char_to_word_offset.append(len(doc_tokens) - 1) + + for qa in paragraph["qas"]: + qas_id = qa["id"] + question_text = qa["question"] + start_position = None + end_position = None + orig_answer_text = None + is_impossible = False + if is_training: + + if version_2_with_negative: + is_impossible = qa["is_impossible"] + if (len(qa["answers"]) != 1) and (not is_impossible): + raise ValueError( + "For training, each question should have exactly 1 answer.") + if not is_impossible: + answer = qa["answers"][0] + orig_answer_text = answer["text"] + answer_offset = answer["answer_start"] + answer_length = len(orig_answer_text) + start_position = char_to_word_offset[answer_offset] + end_position = char_to_word_offset[answer_offset + answer_length - + 1] + # Only add answers where the text can be exactly recovered from the + # document. If this CAN'T happen it's likely due to weird Unicode + # stuff so we will just skip the example. + # + # Note that this means for training mode, every example is NOT + # guaranteed to be preserved. + actual_text = " ".join( + doc_tokens[start_position:(end_position + 1)]) + cleaned_answer_text = " ".join( + tokenization.whitespace_tokenize(orig_answer_text)) + if actual_text.find(cleaned_answer_text) == -1: + logging.warning("Could not find answer: '%s' vs. '%s'", + actual_text, cleaned_answer_text) + continue + else: + start_position = -1 + end_position = -1 + orig_answer_text = "" + + example = SquadExample( + qas_id=qas_id, + question_text=question_text, + doc_tokens=doc_tokens, + orig_answer_text=orig_answer_text, + start_position=start_position, + end_position=end_position, + is_impossible=is_impossible) + examples.append(example) + + return examples + + +def convert_examples_to_features(examples, + tokenizer, + max_seq_length, + doc_stride, + max_query_length, + is_training, + output_fn, + batch_size=None): + """Loads a data file into a list of `InputBatch`s.""" + + base_id = 1000000000 + unique_id = base_id + feature = None + for (example_index, example) in enumerate(examples): + query_tokens = tokenizer.tokenize(example.question_text) + + if len(query_tokens) > max_query_length: + query_tokens = query_tokens[0:max_query_length] + + tok_to_orig_index = [] + orig_to_tok_index = [] + all_doc_tokens = [] + for (i, token) in enumerate(example.doc_tokens): + orig_to_tok_index.append(len(all_doc_tokens)) + sub_tokens = tokenizer.tokenize(token) + for sub_token in sub_tokens: + tok_to_orig_index.append(i) + all_doc_tokens.append(sub_token) + + tok_start_position = None + tok_end_position = None + if is_training and example.is_impossible: + tok_start_position = -1 + tok_end_position = -1 + if is_training and not example.is_impossible: + tok_start_position = orig_to_tok_index[example.start_position] + if example.end_position < len(example.doc_tokens) - 1: + tok_end_position = orig_to_tok_index[example.end_position + 1] - 1 + else: + tok_end_position = len(all_doc_tokens) - 1 + (tok_start_position, tok_end_position) = _improve_answer_span( + all_doc_tokens, tok_start_position, tok_end_position, tokenizer, + example.orig_answer_text) + + # The -3 accounts for [CLS], [SEP] and [SEP] + max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 + + # We can have documents that are longer than the maximum sequence length. + # To deal with this we do a sliding window approach, where we take chunks + # of the up to our max length with a stride of `doc_stride`. + _DocSpan = collections.namedtuple( # pylint: disable=invalid-name + "DocSpan", ["start", "length"]) + doc_spans = [] + start_offset = 0 + while start_offset < len(all_doc_tokens): + length = len(all_doc_tokens) - start_offset + if length > max_tokens_for_doc: + length = max_tokens_for_doc + doc_spans.append(_DocSpan(start=start_offset, length=length)) + if start_offset + length == len(all_doc_tokens): + break + start_offset += min(length, doc_stride) + + for (doc_span_index, doc_span) in enumerate(doc_spans): + tokens = [] + token_to_orig_map = {} + token_is_max_context = {} + segment_ids = [] + tokens.append("[CLS]") + segment_ids.append(0) + for token in query_tokens: + tokens.append(token) + segment_ids.append(0) + tokens.append("[SEP]") + segment_ids.append(0) + + for i in range(doc_span.length): + split_token_index = doc_span.start + i + token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index] + + is_max_context = _check_is_max_context(doc_spans, doc_span_index, + split_token_index) + token_is_max_context[len(tokens)] = is_max_context + tokens.append(all_doc_tokens[split_token_index]) + segment_ids.append(1) + tokens.append("[SEP]") + segment_ids.append(1) + + input_ids = tokenizer.convert_tokens_to_ids(tokens) + + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + input_mask = [1] * len(input_ids) + + # Zero-pad up to the sequence length. + while len(input_ids) < max_seq_length: + input_ids.append(0) + input_mask.append(0) + segment_ids.append(0) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + + start_position = None + end_position = None + if is_training and not example.is_impossible: + # For training, if our document chunk does not contain an annotation + # we throw it out, since there is nothing to predict. + doc_start = doc_span.start + doc_end = doc_span.start + doc_span.length - 1 + out_of_span = False + if not (tok_start_position >= doc_start and + tok_end_position <= doc_end): + out_of_span = True + if out_of_span: + start_position = 0 + end_position = 0 + else: + doc_offset = len(query_tokens) + 2 + start_position = tok_start_position - doc_start + doc_offset + end_position = tok_end_position - doc_start + doc_offset + + if is_training and example.is_impossible: + start_position = 0 + end_position = 0 + + if example_index < 20: + logging.info("*** Example ***") + logging.info("unique_id: %s", (unique_id)) + logging.info("example_index: %s", (example_index)) + logging.info("doc_span_index: %s", (doc_span_index)) + logging.info("tokens: %s", + " ".join([tokenization.printable_text(x) for x in tokens])) + logging.info( + "token_to_orig_map: %s", " ".join([ + "%d:%d" % (x, y) for (x, y) in six.iteritems(token_to_orig_map) + ])) + logging.info( + "token_is_max_context: %s", " ".join([ + "%d:%s" % (x, y) + for (x, y) in six.iteritems(token_is_max_context) + ])) + logging.info("input_ids: %s", " ".join([str(x) for x in input_ids])) + logging.info("input_mask: %s", " ".join([str(x) for x in input_mask])) + logging.info("segment_ids: %s", " ".join([str(x) for x in segment_ids])) + if is_training and example.is_impossible: + logging.info("impossible example") + if is_training and not example.is_impossible: + answer_text = " ".join(tokens[start_position:(end_position + 1)]) + logging.info("start_position: %d", (start_position)) + logging.info("end_position: %d", (end_position)) + logging.info("answer: %s", tokenization.printable_text(answer_text)) + + feature = InputFeatures( + unique_id=unique_id, + example_index=example_index, + doc_span_index=doc_span_index, + tokens=tokens, + token_to_orig_map=token_to_orig_map, + token_is_max_context=token_is_max_context, + input_ids=input_ids, + input_mask=input_mask, + segment_ids=segment_ids, + start_position=start_position, + end_position=end_position, + is_impossible=example.is_impossible) + + # Run callback + if is_training: + output_fn(feature) + else: + output_fn(feature, is_padding=False) + + unique_id += 1 + + if not is_training and feature: + assert batch_size + num_padding = 0 + num_examples = unique_id - base_id + if unique_id % batch_size != 0: + num_padding = batch_size - (num_examples % batch_size) + logging.info("Adding padding examples to make sure no partial batch.") + logging.info("Adds %d padding examples for inference.", num_padding) + dummy_feature = copy.deepcopy(feature) + for _ in range(num_padding): + dummy_feature.unique_id = unique_id + + # Run callback + output_fn(feature, is_padding=True) + unique_id += 1 + return unique_id - base_id + + +def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer, + orig_answer_text): + """Returns tokenized answer spans that better match the annotated answer.""" + + # The SQuAD annotations are character based. We first project them to + # whitespace-tokenized words. But then after WordPiece tokenization, we can + # often find a "better match". For example: + # + # Question: What year was John Smith born? + # Context: The leader was John Smith (1895-1943). + # Answer: 1895 + # + # The original whitespace-tokenized answer will be "(1895-1943).". However + # after tokenization, our tokens will be "( 1895 - 1943 ) .". So we can match + # the exact answer, 1895. + # + # However, this is not always possible. Consider the following: + # + # Question: What country is the top exporter of electornics? + # Context: The Japanese electronics industry is the lagest in the world. + # Answer: Japan + # + # In this case, the annotator chose "Japan" as a character sub-span of + # the word "Japanese". Since our WordPiece tokenizer does not split + # "Japanese", we just use "Japanese" as the annotation. This is fairly rare + # in SQuAD, but does happen. + tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text)) + + for new_start in range(input_start, input_end + 1): + for new_end in range(input_end, new_start - 1, -1): + text_span = " ".join(doc_tokens[new_start:(new_end + 1)]) + if text_span == tok_answer_text: + return (new_start, new_end) + + return (input_start, input_end) + + +def _check_is_max_context(doc_spans, cur_span_index, position): + """Check if this is the 'max context' doc span for the token.""" + + # Because of the sliding window approach taken to scoring documents, a single + # token can appear in multiple documents. E.g. + # Doc: the man went to the store and bought a gallon of milk + # Span A: the man went to the + # Span B: to the store and bought + # Span C: and bought a gallon of + # ... + # + # Now the word 'bought' will have two scores from spans B and C. We only + # want to consider the score with "maximum context", which we define as + # the *minimum* of its left and right context (the *sum* of left and + # right context will always be the same, of course). + # + # In the example the maximum context for 'bought' would be span C since + # it has 1 left context and 3 right context, while span B has 4 left context + # and 0 right context. + best_score = None + best_span_index = None + for (span_index, doc_span) in enumerate(doc_spans): + end = doc_span.start + doc_span.length - 1 + if position < doc_span.start: + continue + if position > end: + continue + num_left_context = position - doc_span.start + num_right_context = end - position + score = min(num_left_context, num_right_context) + 0.01 * doc_span.length + if best_score is None or score > best_score: + best_score = score + best_span_index = span_index + + return cur_span_index == best_span_index + + +def write_predictions(all_examples, + all_features, + all_results, + n_best_size, + max_answer_length, + do_lower_case, + output_prediction_file, + output_nbest_file, + output_null_log_odds_file, + version_2_with_negative=False, + null_score_diff_threshold=0.0, + verbose=False): + """Write final predictions to the json file and log-odds of null if needed.""" + logging.info("Writing predictions to: %s", (output_prediction_file)) + logging.info("Writing nbest to: %s", (output_nbest_file)) + + all_predictions, all_nbest_json, scores_diff_json = ( + postprocess_output(all_examples=all_examples, + all_features=all_features, + all_results=all_results, + n_best_size=n_best_size, + max_answer_length=max_answer_length, + do_lower_case=do_lower_case, + version_2_with_negative=version_2_with_negative, + null_score_diff_threshold=null_score_diff_threshold, + verbose=verbose)) + + write_to_json_files(all_predictions, output_prediction_file) + write_to_json_files(all_nbest_json, output_nbest_file) + if version_2_with_negative: + write_to_json_files(scores_diff_json, output_null_log_odds_file) + + +def postprocess_output(all_examples, + all_features, + all_results, + n_best_size, + max_answer_length, + do_lower_case, + version_2_with_negative=False, + null_score_diff_threshold=0.0, + verbose=False): + """Postprocess model output, to form predicton results.""" + + example_index_to_features = collections.defaultdict(list) + for feature in all_features: + example_index_to_features[feature.example_index].append(feature) + unique_id_to_result = {} + for result in all_results: + unique_id_to_result[result.unique_id] = result + + _PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name + "PrelimPrediction", + ["feature_index", "start_index", "end_index", "start_logit", "end_logit"]) + + all_predictions = collections.OrderedDict() + all_nbest_json = collections.OrderedDict() + scores_diff_json = collections.OrderedDict() + + for (example_index, example) in enumerate(all_examples): + features = example_index_to_features[example_index] + + prelim_predictions = [] + # keep track of the minimum score of null start+end of position 0 + score_null = 1000000 # large and positive + min_null_feature_index = 0 # the paragraph slice with min mull score + null_start_logit = 0 # the start logit at the slice with min null score + null_end_logit = 0 # the end logit at the slice with min null score + for (feature_index, feature) in enumerate(features): + result = unique_id_to_result[feature.unique_id] + start_indexes = _get_best_indexes(result.start_logits, n_best_size) + end_indexes = _get_best_indexes(result.end_logits, n_best_size) + # if we could have irrelevant answers, get the min score of irrelevant + if version_2_with_negative: + feature_null_score = result.start_logits[0] + result.end_logits[0] + if feature_null_score < score_null: + score_null = feature_null_score + min_null_feature_index = feature_index + null_start_logit = result.start_logits[0] + null_end_logit = result.end_logits[0] + for start_index in start_indexes: + for end_index in end_indexes: + # We could hypothetically create invalid predictions, e.g., predict + # that the start of the span is in the question. We throw out all + # invalid predictions. + if start_index >= len(feature.tokens): + continue + if end_index >= len(feature.tokens): + continue + if start_index not in feature.token_to_orig_map: + continue + if end_index not in feature.token_to_orig_map: + continue + if not feature.token_is_max_context.get(start_index, False): + continue + if end_index < start_index: + continue + length = end_index - start_index + 1 + if length > max_answer_length: + continue + prelim_predictions.append( + _PrelimPrediction( + feature_index=feature_index, + start_index=start_index, + end_index=end_index, + start_logit=result.start_logits[start_index], + end_logit=result.end_logits[end_index])) + + if version_2_with_negative: + prelim_predictions.append( + _PrelimPrediction( + feature_index=min_null_feature_index, + start_index=0, + end_index=0, + start_logit=null_start_logit, + end_logit=null_end_logit)) + prelim_predictions = sorted( + prelim_predictions, + key=lambda x: (x.start_logit + x.end_logit), + reverse=True) + + _NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name + "NbestPrediction", ["text", "start_logit", "end_logit"]) + + seen_predictions = {} + nbest = [] + for pred in prelim_predictions: + if len(nbest) >= n_best_size: + break + feature = features[pred.feature_index] + if pred.start_index > 0: # this is a non-null prediction + tok_tokens = feature.tokens[pred.start_index:(pred.end_index + 1)] + orig_doc_start = feature.token_to_orig_map[pred.start_index] + orig_doc_end = feature.token_to_orig_map[pred.end_index] + orig_tokens = example.doc_tokens[orig_doc_start:(orig_doc_end + 1)] + tok_text = " ".join(tok_tokens) + + # De-tokenize WordPieces that have been split off. + tok_text = tok_text.replace(" ##", "") + tok_text = tok_text.replace("##", "") + + # Clean whitespace + tok_text = tok_text.strip() + tok_text = " ".join(tok_text.split()) + orig_text = " ".join(orig_tokens) + + final_text = get_final_text( + tok_text, orig_text, do_lower_case, verbose=verbose) + if final_text in seen_predictions: + continue + + seen_predictions[final_text] = True + else: + final_text = "" + seen_predictions[final_text] = True + + nbest.append( + _NbestPrediction( + text=final_text, + start_logit=pred.start_logit, + end_logit=pred.end_logit)) + + # if we didn't inlude the empty option in the n-best, inlcude it + if version_2_with_negative: + if "" not in seen_predictions: + nbest.append( + _NbestPrediction( + text="", start_logit=null_start_logit, + end_logit=null_end_logit)) + # In very rare edge cases we could have no valid predictions. So we + # just create a nonce prediction in this case to avoid failure. + if not nbest: + nbest.append( + _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) + + assert len(nbest) >= 1 + + total_scores = [] + best_non_null_entry = None + for entry in nbest: + total_scores.append(entry.start_logit + entry.end_logit) + if not best_non_null_entry: + if entry.text: + best_non_null_entry = entry + + probs = _compute_softmax(total_scores) + + nbest_json = [] + for (i, entry) in enumerate(nbest): + output = collections.OrderedDict() + output["text"] = entry.text + output["probability"] = probs[i] + output["start_logit"] = entry.start_logit + output["end_logit"] = entry.end_logit + nbest_json.append(output) + + assert len(nbest_json) >= 1 + + if not version_2_with_negative: + all_predictions[example.qas_id] = nbest_json[0]["text"] + else: + # pytype: disable=attribute-error + # predict "" iff the null score - the score of best non-null > threshold + score_diff = score_null - best_non_null_entry.start_logit - ( + best_non_null_entry.end_logit) + scores_diff_json[example.qas_id] = score_diff + if score_diff > null_score_diff_threshold: + all_predictions[example.qas_id] = "" + else: + all_predictions[example.qas_id] = best_non_null_entry.text + # pytype: enable=attribute-error + + all_nbest_json[example.qas_id] = nbest_json + + return all_predictions, all_nbest_json, scores_diff_json + + +def write_to_json_files(json_records, json_file): + with tf.io.gfile.GFile(json_file, "w") as writer: + writer.write(json.dumps(json_records, indent=4) + "\n") + + +def get_final_text(pred_text, orig_text, do_lower_case, verbose=False): + """Project the tokenized prediction back to the original text.""" + + # When we created the data, we kept track of the alignment between original + # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So + # now `orig_text` contains the span of our original text corresponding to the + # span that we predicted. + # + # However, `orig_text` may contain extra characters that we don't want in + # our prediction. + # + # For example, let's say: + # pred_text = steve smith + # orig_text = Steve Smith's + # + # We don't want to return `orig_text` because it contains the extra "'s". + # + # We don't want to return `pred_text` because it's already been normalized + # (the SQuAD eval script also does punctuation stripping/lower casing but + # our tokenizer does additional normalization like stripping accent + # characters). + # + # What we really want to return is "Steve Smith". + # + # Therefore, we have to apply a semi-complicated alignment heruistic between + # `pred_text` and `orig_text` to get a character-to-charcter alignment. This + # can fail in certain cases in which case we just return `orig_text`. + + def _strip_spaces(text): + ns_chars = [] + ns_to_s_map = collections.OrderedDict() + for (i, c) in enumerate(text): + if c == " ": + continue + ns_to_s_map[len(ns_chars)] = i + ns_chars.append(c) + ns_text = "".join(ns_chars) + return (ns_text, ns_to_s_map) + + # We first tokenize `orig_text`, strip whitespace from the result + # and `pred_text`, and check if they are the same length. If they are + # NOT the same length, the heuristic has failed. If they are the same + # length, we assume the characters are one-to-one aligned. + tokenizer = tokenization.BasicTokenizer(do_lower_case=do_lower_case) + + tok_text = " ".join(tokenizer.tokenize(orig_text)) + + start_position = tok_text.find(pred_text) + if start_position == -1: + if verbose: + logging.info("Unable to find text: '%s' in '%s'", pred_text, orig_text) + return orig_text + end_position = start_position + len(pred_text) - 1 + + (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text) + (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text) + + if len(orig_ns_text) != len(tok_ns_text): + if verbose: + logging.info("Length not equal after stripping spaces: '%s' vs '%s'", + orig_ns_text, tok_ns_text) + return orig_text + + # We then project the characters in `pred_text` back to `orig_text` using + # the character-to-character alignment. + tok_s_to_ns_map = {} + for (i, tok_index) in six.iteritems(tok_ns_to_s_map): + tok_s_to_ns_map[tok_index] = i + + orig_start_position = None + if start_position in tok_s_to_ns_map: + ns_start_position = tok_s_to_ns_map[start_position] + if ns_start_position in orig_ns_to_s_map: + orig_start_position = orig_ns_to_s_map[ns_start_position] + + if orig_start_position is None: + if verbose: + logging.info("Couldn't map start position") + return orig_text + + orig_end_position = None + if end_position in tok_s_to_ns_map: + ns_end_position = tok_s_to_ns_map[end_position] + if ns_end_position in orig_ns_to_s_map: + orig_end_position = orig_ns_to_s_map[ns_end_position] + + if orig_end_position is None: + if verbose: + logging.info("Couldn't map end position") + return orig_text + + output_text = orig_text[orig_start_position:(orig_end_position + 1)] + return output_text + + +def _get_best_indexes(logits, n_best_size): + """Get the n-best logits from a list.""" + index_and_score = sorted(enumerate(logits), key=lambda x: x[1], reverse=True) + + best_indexes = [] + for i in range(len(index_and_score)): # pylint: disable=consider-using-enumerate + if i >= n_best_size: + break + best_indexes.append(index_and_score[i][0]) + return best_indexes + + +def _compute_softmax(scores): + """Compute softmax probability over raw logits.""" + if not scores: + return [] + + max_score = None + for score in scores: + if max_score is None or score > max_score: + max_score = score + + exp_scores = [] + total_sum = 0.0 + for score in scores: + x = math.exp(score - max_score) + exp_scores.append(x) + total_sum += x + + probs = [] + for score in exp_scores: + probs.append(score / total_sum) + return probs + + +def generate_tf_record_from_json_file(input_file_path, + vocab_file_path, + output_path, + max_seq_length=384, + do_lower_case=True, + max_query_length=64, + doc_stride=128, + version_2_with_negative=False): + """Generates and saves training data into a tf record file.""" + train_examples = read_squad_examples( + input_file=input_file_path, + is_training=True, + version_2_with_negative=version_2_with_negative) + tokenizer = tokenization.FullTokenizer( + vocab_file=vocab_file_path, do_lower_case=do_lower_case) + train_writer = FeatureWriter(filename=output_path, is_training=True) + number_of_examples = convert_examples_to_features( + examples=train_examples, + tokenizer=tokenizer, + max_seq_length=max_seq_length, + doc_stride=doc_stride, + max_query_length=max_query_length, + is_training=True, + output_fn=train_writer.process_feature) + train_writer.close() + + meta_data = { + "task_type": "bert_squad", + "train_data_size": number_of_examples, + "max_seq_length": max_seq_length, + "max_query_length": max_query_length, + "doc_stride": doc_stride, + "version_2_with_negative": version_2_with_negative, + } + + return meta_data diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/squad_lib_sp.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/squad_lib_sp.py new file mode 100644 index 0000000..6caa6e1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/data/squad_lib_sp.py @@ -0,0 +1,890 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Run ALBERT on SQuAD 1.1 and SQuAD 2.0 using sentence piece tokenization. + +The file is forked from: + +https://github.com/google-research/ALBERT/blob/master/run_squad_sp.py +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import copy +import json +import math +from absl import logging +import numpy as np +import tensorflow as tf + +from official.nlp.bert import tokenization + + +class SquadExample(object): + """A single training/test example for simple sequence classification. + + For examples without an answer, the start and end position are -1. + """ + + def __init__(self, + qas_id, + question_text, + paragraph_text, + orig_answer_text=None, + start_position=None, + end_position=None, + is_impossible=False): + self.qas_id = qas_id + self.question_text = question_text + self.paragraph_text = paragraph_text + self.orig_answer_text = orig_answer_text + self.start_position = start_position + self.end_position = end_position + self.is_impossible = is_impossible + + def __str__(self): + return self.__repr__() + + def __repr__(self): + s = "" + s += "qas_id: %s" % (tokenization.printable_text(self.qas_id)) + s += ", question_text: %s" % ( + tokenization.printable_text(self.question_text)) + s += ", paragraph_text: [%s]" % (" ".join(self.paragraph_text)) + if self.start_position: + s += ", start_position: %d" % (self.start_position) + if self.start_position: + s += ", end_position: %d" % (self.end_position) + if self.start_position: + s += ", is_impossible: %r" % (self.is_impossible) + return s + + +class InputFeatures(object): + """A single set of features of data.""" + + def __init__(self, + unique_id, + example_index, + doc_span_index, + tok_start_to_orig_index, + tok_end_to_orig_index, + token_is_max_context, + tokens, + input_ids, + input_mask, + segment_ids, + paragraph_len, + start_position=None, + end_position=None, + is_impossible=None): + self.unique_id = unique_id + self.example_index = example_index + self.doc_span_index = doc_span_index + self.tok_start_to_orig_index = tok_start_to_orig_index + self.tok_end_to_orig_index = tok_end_to_orig_index + self.token_is_max_context = token_is_max_context + self.tokens = tokens + self.input_ids = input_ids + self.input_mask = input_mask + self.segment_ids = segment_ids + self.paragraph_len = paragraph_len + self.start_position = start_position + self.end_position = end_position + self.is_impossible = is_impossible + + +def read_squad_examples(input_file, is_training, version_2_with_negative): + """Read a SQuAD json file into a list of SquadExample.""" + del version_2_with_negative + with tf.io.gfile.GFile(input_file, "r") as reader: + input_data = json.load(reader)["data"] + + examples = [] + for entry in input_data: + for paragraph in entry["paragraphs"]: + paragraph_text = paragraph["context"] + + for qa in paragraph["qas"]: + qas_id = qa["id"] + question_text = qa["question"] + start_position = None + orig_answer_text = None + is_impossible = False + + if is_training: + is_impossible = qa.get("is_impossible", False) + if (len(qa["answers"]) != 1) and (not is_impossible): + raise ValueError( + "For training, each question should have exactly 1 answer.") + if not is_impossible: + answer = qa["answers"][0] + orig_answer_text = answer["text"] + start_position = answer["answer_start"] + else: + start_position = -1 + orig_answer_text = "" + + example = SquadExample( + qas_id=qas_id, + question_text=question_text, + paragraph_text=paragraph_text, + orig_answer_text=orig_answer_text, + start_position=start_position, + is_impossible=is_impossible) + examples.append(example) + + return examples + + +def _convert_index(index, pos, m=None, is_start=True): + """Converts index.""" + if index[pos] is not None: + return index[pos] + n = len(index) + rear = pos + while rear < n - 1 and index[rear] is None: + rear += 1 + front = pos + while front > 0 and index[front] is None: + front -= 1 + assert index[front] is not None or index[rear] is not None + if index[front] is None: + if index[rear] >= 1: + if is_start: + return 0 + else: + return index[rear] - 1 + return index[rear] + if index[rear] is None: + if m is not None and index[front] < m - 1: + if is_start: + return index[front] + 1 + else: + return m - 1 + return index[front] + if is_start: + if index[rear] > index[front] + 1: + return index[front] + 1 + else: + return index[rear] + else: + if index[rear] > index[front] + 1: + return index[rear] - 1 + else: + return index[front] + + +def convert_examples_to_features(examples, + tokenizer, + max_seq_length, + doc_stride, + max_query_length, + is_training, + output_fn, + do_lower_case, + batch_size=None): + """Loads a data file into a list of `InputBatch`s.""" + cnt_pos, cnt_neg = 0, 0 + base_id = 1000000000 + unique_id = base_id + max_n, max_m = 1024, 1024 + f = np.zeros((max_n, max_m), dtype=np.float32) + + for (example_index, example) in enumerate(examples): + + if example_index % 100 == 0: + logging.info("Converting %d/%d pos %d neg %d", example_index, + len(examples), cnt_pos, cnt_neg) + + query_tokens = tokenization.encode_ids( + tokenizer.sp_model, + tokenization.preprocess_text( + example.question_text, lower=do_lower_case)) + + if len(query_tokens) > max_query_length: + query_tokens = query_tokens[0:max_query_length] + + paragraph_text = example.paragraph_text + para_tokens = tokenization.encode_pieces( + tokenizer.sp_model, + tokenization.preprocess_text( + example.paragraph_text, lower=do_lower_case)) + + chartok_to_tok_index = [] + tok_start_to_chartok_index = [] + tok_end_to_chartok_index = [] + char_cnt = 0 + for i, token in enumerate(para_tokens): + new_token = token.replace(tokenization.SPIECE_UNDERLINE, " ") + chartok_to_tok_index.extend([i] * len(new_token)) + tok_start_to_chartok_index.append(char_cnt) + char_cnt += len(new_token) + tok_end_to_chartok_index.append(char_cnt - 1) + + tok_cat_text = "".join(para_tokens).replace(tokenization.SPIECE_UNDERLINE, + " ") + n, m = len(paragraph_text), len(tok_cat_text) + + if n > max_n or m > max_m: + max_n = max(n, max_n) + max_m = max(m, max_m) + f = np.zeros((max_n, max_m), dtype=np.float32) + + g = {} + # pylint: disable=cell-var-from-loop + def _lcs_match(max_dist, n=n, m=m): + """Longest-common-substring algorithm.""" + f.fill(0) + g.clear() + + ### longest common sub sequence + # f[i, j] = max(f[i - 1, j], f[i, j - 1], f[i - 1, j - 1] + match(i, j)) + for i in range(n): + + # unlike standard LCS, this is specifically optimized for the setting + # because the mismatch between sentence pieces and original text will + # be small + for j in range(i - max_dist, i + max_dist): + if j >= m or j < 0: + continue + + if i > 0: + g[(i, j)] = 0 + f[i, j] = f[i - 1, j] + + if j > 0 and f[i, j - 1] > f[i, j]: + g[(i, j)] = 1 + f[i, j] = f[i, j - 1] + + f_prev = f[i - 1, j - 1] if i > 0 and j > 0 else 0 + if (tokenization.preprocess_text( + paragraph_text[i], lower=do_lower_case, + remove_space=False) == tok_cat_text[j] and f_prev + 1 > f[i, j]): + g[(i, j)] = 2 + f[i, j] = f_prev + 1 + # pylint: enable=cell-var-from-loop + + max_dist = abs(n - m) + 5 + for _ in range(2): + _lcs_match(max_dist) + if f[n - 1, m - 1] > 0.8 * n: + break + max_dist *= 2 + + orig_to_chartok_index = [None] * n + chartok_to_orig_index = [None] * m + i, j = n - 1, m - 1 + while i >= 0 and j >= 0: + if (i, j) not in g: + break + if g[(i, j)] == 2: + orig_to_chartok_index[i] = j + chartok_to_orig_index[j] = i + i, j = i - 1, j - 1 + elif g[(i, j)] == 1: + j = j - 1 + else: + i = i - 1 + + if (all(v is None for v in orig_to_chartok_index) or + f[n - 1, m - 1] < 0.8 * n): + logging.info("MISMATCH DETECTED!") + continue + + tok_start_to_orig_index = [] + tok_end_to_orig_index = [] + for i in range(len(para_tokens)): + start_chartok_pos = tok_start_to_chartok_index[i] + end_chartok_pos = tok_end_to_chartok_index[i] + start_orig_pos = _convert_index( + chartok_to_orig_index, start_chartok_pos, n, is_start=True) + end_orig_pos = _convert_index( + chartok_to_orig_index, end_chartok_pos, n, is_start=False) + + tok_start_to_orig_index.append(start_orig_pos) + tok_end_to_orig_index.append(end_orig_pos) + + if not is_training: + tok_start_position = tok_end_position = None + + if is_training and example.is_impossible: + tok_start_position = 0 + tok_end_position = 0 + + if is_training and not example.is_impossible: + start_position = example.start_position + end_position = start_position + len(example.orig_answer_text) - 1 + + start_chartok_pos = _convert_index( + orig_to_chartok_index, start_position, is_start=True) + tok_start_position = chartok_to_tok_index[start_chartok_pos] + + end_chartok_pos = _convert_index( + orig_to_chartok_index, end_position, is_start=False) + tok_end_position = chartok_to_tok_index[end_chartok_pos] + assert tok_start_position <= tok_end_position + + def _piece_to_id(x): + return tokenizer.sp_model.PieceToId(x) + + all_doc_tokens = list(map(_piece_to_id, para_tokens)) + + # The -3 accounts for [CLS], [SEP] and [SEP] + max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 + + # We can have documents that are longer than the maximum sequence length. + # To deal with this we do a sliding window approach, where we take chunks + # of the up to our max length with a stride of `doc_stride`. + _DocSpan = collections.namedtuple( # pylint: disable=invalid-name + "DocSpan", ["start", "length"]) + doc_spans = [] + start_offset = 0 + while start_offset < len(all_doc_tokens): + length = len(all_doc_tokens) - start_offset + if length > max_tokens_for_doc: + length = max_tokens_for_doc + doc_spans.append(_DocSpan(start=start_offset, length=length)) + if start_offset + length == len(all_doc_tokens): + break + start_offset += min(length, doc_stride) + + for (doc_span_index, doc_span) in enumerate(doc_spans): + tokens = [] + token_is_max_context = {} + segment_ids = [] + + cur_tok_start_to_orig_index = [] + cur_tok_end_to_orig_index = [] + + tokens.append(tokenizer.sp_model.PieceToId("[CLS]")) + segment_ids.append(0) + for token in query_tokens: + tokens.append(token) + segment_ids.append(0) + tokens.append(tokenizer.sp_model.PieceToId("[SEP]")) + segment_ids.append(0) + + for i in range(doc_span.length): + split_token_index = doc_span.start + i + + cur_tok_start_to_orig_index.append( + tok_start_to_orig_index[split_token_index]) + cur_tok_end_to_orig_index.append( + tok_end_to_orig_index[split_token_index]) + + is_max_context = _check_is_max_context(doc_spans, doc_span_index, + split_token_index) + token_is_max_context[len(tokens)] = is_max_context + tokens.append(all_doc_tokens[split_token_index]) + segment_ids.append(1) + tokens.append(tokenizer.sp_model.PieceToId("[SEP]")) + segment_ids.append(1) + + paragraph_len = len(tokens) + input_ids = tokens + + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + input_mask = [1] * len(input_ids) + + # Zero-pad up to the sequence length. + while len(input_ids) < max_seq_length: + input_ids.append(0) + input_mask.append(0) + segment_ids.append(0) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + + span_is_impossible = example.is_impossible + start_position = None + end_position = None + if is_training and not span_is_impossible: + # For training, if our document chunk does not contain an annotation + # we throw it out, since there is nothing to predict. + doc_start = doc_span.start + doc_end = doc_span.start + doc_span.length - 1 + out_of_span = False + if not (tok_start_position >= doc_start and + tok_end_position <= doc_end): + out_of_span = True + if out_of_span: + # continue + start_position = 0 + end_position = 0 + span_is_impossible = True + else: + doc_offset = len(query_tokens) + 2 + start_position = tok_start_position - doc_start + doc_offset + end_position = tok_end_position - doc_start + doc_offset + + if is_training and span_is_impossible: + start_position = 0 + end_position = 0 + + if example_index < 20: + logging.info("*** Example ***") + logging.info("unique_id: %s", (unique_id)) + logging.info("example_index: %s", (example_index)) + logging.info("doc_span_index: %s", (doc_span_index)) + logging.info("tok_start_to_orig_index: %s", + " ".join([str(x) for x in cur_tok_start_to_orig_index])) + logging.info("tok_end_to_orig_index: %s", + " ".join([str(x) for x in cur_tok_end_to_orig_index])) + logging.info( + "token_is_max_context: %s", " ".join( + ["%d:%s" % (x, y) for (x, y) in token_is_max_context.items()])) + logging.info( + "input_pieces: %s", + " ".join([tokenizer.sp_model.IdToPiece(x) for x in tokens])) + logging.info("input_ids: %s", " ".join([str(x) for x in input_ids])) + logging.info("input_mask: %s", " ".join([str(x) for x in input_mask])) + logging.info("segment_ids: %s", " ".join([str(x) for x in segment_ids])) + + if is_training and span_is_impossible: + logging.info("impossible example span") + + if is_training and not span_is_impossible: + pieces = [ + tokenizer.sp_model.IdToPiece(token) + for token in tokens[start_position:(end_position + 1)] + ] + answer_text = tokenizer.sp_model.DecodePieces(pieces) + logging.info("start_position: %d", (start_position)) + logging.info("end_position: %d", (end_position)) + logging.info("answer: %s", (tokenization.printable_text(answer_text))) + + # With multi processing, the example_index is actually the index + # within the current process therefore we use example_index=None + # to avoid being used in the future. + # The current code does not use example_index of training data. + if is_training: + feat_example_index = None + else: + feat_example_index = example_index + + feature = InputFeatures( + unique_id=unique_id, + example_index=feat_example_index, + doc_span_index=doc_span_index, + tok_start_to_orig_index=cur_tok_start_to_orig_index, + tok_end_to_orig_index=cur_tok_end_to_orig_index, + token_is_max_context=token_is_max_context, + tokens=[tokenizer.sp_model.IdToPiece(x) for x in tokens], + input_ids=input_ids, + input_mask=input_mask, + segment_ids=segment_ids, + paragraph_len=paragraph_len, + start_position=start_position, + end_position=end_position, + is_impossible=span_is_impossible) + + # Run callback + if is_training: + output_fn(feature) + else: + output_fn(feature, is_padding=False) + + unique_id += 1 + if span_is_impossible: + cnt_neg += 1 + else: + cnt_pos += 1 + + if not is_training and feature: + assert batch_size + num_padding = 0 + num_examples = unique_id - base_id + if unique_id % batch_size != 0: + num_padding = batch_size - (num_examples % batch_size) + dummy_feature = copy.deepcopy(feature) + for _ in range(num_padding): + dummy_feature.unique_id = unique_id + + # Run callback + output_fn(feature, is_padding=True) + unique_id += 1 + + logging.info("Total number of instances: %d = pos %d neg %d", + cnt_pos + cnt_neg, cnt_pos, cnt_neg) + return unique_id - base_id + + +def _check_is_max_context(doc_spans, cur_span_index, position): + """Check if this is the 'max context' doc span for the token.""" + + # Because of the sliding window approach taken to scoring documents, a single + # token can appear in multiple documents. E.g. + # Doc: the man went to the store and bought a gallon of milk + # Span A: the man went to the + # Span B: to the store and bought + # Span C: and bought a gallon of + # ... + # + # Now the word 'bought' will have two scores from spans B and C. We only + # want to consider the score with "maximum context", which we define as + # the *minimum* of its left and right context (the *sum* of left and + # right context will always be the same, of course). + # + # In the example the maximum context for 'bought' would be span C since + # it has 1 left context and 3 right context, while span B has 4 left context + # and 0 right context. + best_score = None + best_span_index = None + for (span_index, doc_span) in enumerate(doc_spans): + end = doc_span.start + doc_span.length - 1 + if position < doc_span.start: + continue + if position > end: + continue + num_left_context = position - doc_span.start + num_right_context = end - position + score = min(num_left_context, num_right_context) + 0.01 * doc_span.length + if best_score is None or score > best_score: + best_score = score + best_span_index = span_index + + return cur_span_index == best_span_index + + +def write_predictions(all_examples, + all_features, + all_results, + n_best_size, + max_answer_length, + do_lower_case, + output_prediction_file, + output_nbest_file, + output_null_log_odds_file, + version_2_with_negative=False, + null_score_diff_threshold=0.0, + verbose=False): + """Write final predictions to the json file and log-odds of null if needed.""" + logging.info("Writing predictions to: %s", (output_prediction_file)) + logging.info("Writing nbest to: %s", (output_nbest_file)) + + all_predictions, all_nbest_json, scores_diff_json = ( + postprocess_output(all_examples=all_examples, + all_features=all_features, + all_results=all_results, + n_best_size=n_best_size, + max_answer_length=max_answer_length, + do_lower_case=do_lower_case, + version_2_with_negative=version_2_with_negative, + null_score_diff_threshold=null_score_diff_threshold, + verbose=verbose)) + + write_to_json_files(all_predictions, output_prediction_file) + write_to_json_files(all_nbest_json, output_nbest_file) + if version_2_with_negative: + write_to_json_files(scores_diff_json, output_null_log_odds_file) + + +def postprocess_output(all_examples, + all_features, + all_results, + n_best_size, + max_answer_length, + do_lower_case, + version_2_with_negative=False, + null_score_diff_threshold=0.0, + verbose=False): + """Postprocess model output, to form predicton results.""" + + del do_lower_case, verbose + + example_index_to_features = collections.defaultdict(list) + for feature in all_features: + example_index_to_features[feature.example_index].append(feature) + + unique_id_to_result = {} + for result in all_results: + unique_id_to_result[result.unique_id] = result + + _PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name + "PrelimPrediction", + ["feature_index", "start_index", "end_index", "start_logit", "end_logit"]) + + all_predictions = collections.OrderedDict() + all_nbest_json = collections.OrderedDict() + scores_diff_json = collections.OrderedDict() + + for (example_index, example) in enumerate(all_examples): + features = example_index_to_features[example_index] + + prelim_predictions = [] + # keep track of the minimum score of null start+end of position 0 + score_null = 1000000 # large and positive + min_null_feature_index = 0 # the paragraph slice with min mull score + null_start_logit = 0 # the start logit at the slice with min null score + null_end_logit = 0 # the end logit at the slice with min null score + for (feature_index, feature) in enumerate(features): + result = unique_id_to_result[feature.unique_id] + start_indexes = _get_best_indexes(result.start_logits, n_best_size) + end_indexes = _get_best_indexes(result.end_logits, n_best_size) + # if we could have irrelevant answers, get the min score of irrelevant + if version_2_with_negative: + feature_null_score = result.start_logits[0] + result.end_logits[0] + if feature_null_score < score_null: + score_null = feature_null_score + min_null_feature_index = feature_index + null_start_logit = result.start_logits[0] + null_end_logit = result.end_logits[0] + for start_index in start_indexes: + for end_index in end_indexes: + doc_offset = feature.tokens.index("[SEP]") + 1 + # We could hypothetically create invalid predictions, e.g., predict + # that the start of the span is in the question. We throw out all + # invalid predictions. + if start_index - doc_offset >= len(feature.tok_start_to_orig_index): + continue + if end_index - doc_offset >= len(feature.tok_end_to_orig_index): + continue + # if start_index not in feature.tok_start_to_orig_index: + # continue + # if end_index not in feature.tok_end_to_orig_index: + # continue + if not feature.token_is_max_context.get(start_index, False): + continue + if end_index < start_index: + continue + length = end_index - start_index + 1 + if length > max_answer_length: + continue + prelim_predictions.append( + _PrelimPrediction( + feature_index=feature_index, + start_index=start_index - doc_offset, + end_index=end_index - doc_offset, + start_logit=result.start_logits[start_index], + end_logit=result.end_logits[end_index])) + + if version_2_with_negative: + prelim_predictions.append( + _PrelimPrediction( + feature_index=min_null_feature_index, + start_index=-1, + end_index=-1, + start_logit=null_start_logit, + end_logit=null_end_logit)) + prelim_predictions = sorted( + prelim_predictions, + key=lambda x: (x.start_logit + x.end_logit), + reverse=True) + + _NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name + "NbestPrediction", ["text", "start_logit", "end_logit"]) + + seen_predictions = {} + nbest = [] + for pred in prelim_predictions: + if len(nbest) >= n_best_size: + break + feature = features[pred.feature_index] + if pred.start_index >= 0: # this is a non-null prediction + tok_start_to_orig_index = feature.tok_start_to_orig_index + tok_end_to_orig_index = feature.tok_end_to_orig_index + start_orig_pos = tok_start_to_orig_index[pred.start_index] + end_orig_pos = tok_end_to_orig_index[pred.end_index] + + paragraph_text = example.paragraph_text + final_text = paragraph_text[start_orig_pos:end_orig_pos + 1].strip() + if final_text in seen_predictions: + continue + + seen_predictions[final_text] = True + else: + final_text = "" + seen_predictions[final_text] = True + + nbest.append( + _NbestPrediction( + text=final_text, + start_logit=pred.start_logit, + end_logit=pred.end_logit)) + + # if we didn't inlude the empty option in the n-best, inlcude it + if version_2_with_negative: + if "" not in seen_predictions: + nbest.append( + _NbestPrediction( + text="", start_logit=null_start_logit, + end_logit=null_end_logit)) + # In very rare edge cases we could have no valid predictions. So we + # just create a nonce prediction in this case to avoid failure. + if not nbest: + nbest.append( + _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) + + assert len(nbest) >= 1 + + total_scores = [] + best_non_null_entry = None + for entry in nbest: + total_scores.append(entry.start_logit + entry.end_logit) + if not best_non_null_entry: + if entry.text: + best_non_null_entry = entry + + probs = _compute_softmax(total_scores) + + nbest_json = [] + for (i, entry) in enumerate(nbest): + output = collections.OrderedDict() + output["text"] = entry.text + output["probability"] = probs[i] + output["start_logit"] = entry.start_logit + output["end_logit"] = entry.end_logit + nbest_json.append(output) + + assert len(nbest_json) >= 1 + + if not version_2_with_negative: + all_predictions[example.qas_id] = nbest_json[0]["text"] + else: + assert best_non_null_entry is not None + # predict "" iff the null score - the score of best non-null > threshold + score_diff = score_null - best_non_null_entry.start_logit - ( + best_non_null_entry.end_logit) + scores_diff_json[example.qas_id] = score_diff + if score_diff > null_score_diff_threshold: + all_predictions[example.qas_id] = "" + else: + all_predictions[example.qas_id] = best_non_null_entry.text + + all_nbest_json[example.qas_id] = nbest_json + + return all_predictions, all_nbest_json, scores_diff_json + + +def write_to_json_files(json_records, json_file): + with tf.io.gfile.GFile(json_file, "w") as writer: + writer.write(json.dumps(json_records, indent=4) + "\n") + + +def _get_best_indexes(logits, n_best_size): + """Get the n-best logits from a list.""" + index_and_score = sorted(enumerate(logits), key=lambda x: x[1], reverse=True) + + best_indexes = [] + for i in range(len(index_and_score)): + if i >= n_best_size: + break + best_indexes.append(index_and_score[i][0]) + return best_indexes + + +def _compute_softmax(scores): + """Compute softmax probability over raw logits.""" + if not scores: + return [] + + max_score = None + for score in scores: + if max_score is None or score > max_score: + max_score = score + + exp_scores = [] + total_sum = 0.0 + for score in scores: + x = math.exp(score - max_score) + exp_scores.append(x) + total_sum += x + + probs = [] + for score in exp_scores: + probs.append(score / total_sum) + return probs + + +class FeatureWriter(object): + """Writes InputFeature to TF example file.""" + + def __init__(self, filename, is_training): + self.filename = filename + self.is_training = is_training + self.num_features = 0 + self._writer = tf.io.TFRecordWriter(filename) + + def process_feature(self, feature): + """Write a InputFeature to the TFRecordWriter as a tf.train.Example.""" + self.num_features += 1 + + def create_int_feature(values): + feature = tf.train.Feature( + int64_list=tf.train.Int64List(value=list(values))) + return feature + + features = collections.OrderedDict() + features["unique_ids"] = create_int_feature([feature.unique_id]) + features["input_ids"] = create_int_feature(feature.input_ids) + features["input_mask"] = create_int_feature(feature.input_mask) + features["segment_ids"] = create_int_feature(feature.segment_ids) + + if self.is_training: + features["start_positions"] = create_int_feature([feature.start_position]) + features["end_positions"] = create_int_feature([feature.end_position]) + impossible = 0 + if feature.is_impossible: + impossible = 1 + features["is_impossible"] = create_int_feature([impossible]) + + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + self._writer.write(tf_example.SerializeToString()) + + def close(self): + self._writer.close() + + +def generate_tf_record_from_json_file(input_file_path, + sp_model_file, + output_path, + max_seq_length=384, + do_lower_case=True, + max_query_length=64, + doc_stride=128, + version_2_with_negative=False): + """Generates and saves training data into a tf record file.""" + train_examples = read_squad_examples( + input_file=input_file_path, + is_training=True, + version_2_with_negative=version_2_with_negative) + tokenizer = tokenization.FullSentencePieceTokenizer( + sp_model_file=sp_model_file) + train_writer = FeatureWriter(filename=output_path, is_training=True) + number_of_examples = convert_examples_to_features( + examples=train_examples, + tokenizer=tokenizer, + max_seq_length=max_seq_length, + doc_stride=doc_stride, + max_query_length=max_query_length, + is_training=True, + output_fn=train_writer.process_feature, + do_lower_case=do_lower_case) + train_writer.close() + + meta_data = { + "task_type": "bert_squad", + "train_data_size": number_of_examples, + "max_seq_length": max_seq_length, + "max_query_length": max_query_length, + "doc_stride": doc_stride, + "version_2_with_negative": version_2_with_negative, + } + + return meta_data diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/README.md new file mode 100644 index 0000000..0e74b46 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/README.md @@ -0,0 +1,43 @@ +# NLP Modeling Library + +This libary provides a set of Keras primitives (Layers, Networks, and Models) +that can be assembled into transformer-based models. They are +flexible, validated, interoperable, and both TF1 and TF2 compatible. + +* [`layers`](layers) are the fundamental building blocks for NLP models. +They can be used to assemble new layers, networks, or models. + +* [`networks`](networks) are combinations of layers (and possibly other networks). They are sub-units of models that would not be trained alone. They +encapsulate common network structures like a classification head +or a transformer encoder into an easily handled object with a +standardized configuration. + +* [`models`](models) are combinations of layers and networks that would be trained. Pre-built canned models are provided as both convenience functions and canonical examples. + +* [`losses`](losses) contains common loss computation used in NLP tasks. + +Besides the pre-defined primitives, it also provides scaffold classes to allow +easy experimentation with noval achitectures, e.g., you don’t need to fork a whole Transformer object to try a different kind of attention primitive, for instance. + +* [`TransformerScaffold`](layers/transformer_scaffold.py) implements the +Transformer from ["Attention Is All You Need"] +(https://arxiv.org/abs/1706.03762), with a customizable attention layer +option. Users can pass a class to `attention_cls` and associated config to +`attention_cfg`, in which case the scaffold will instantiate the class with +the config, or pass a class instance to `attention_cls`. + +* [`EncoderScaffold`](networks/encoder_scaffold.py) implements the transformer +encoder from ["BERT: Pre-training of Deep Bidirectional Transformers for +Language Understanding"](https://arxiv.org/abs/1810.04805), with customizable +embedding subnetwork (which will replace the standard embedding logic) and/or a +custom hidden layer (which will replace the Transformer instantiation in the +encoder). + +BERT and ALBERT models in this repo are implemented using this library. Code examples can be found in the corresponding model folder. + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/__init__.py @@ -0,0 +1 @@ + diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/README.md new file mode 100644 index 0000000..211b3ab --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/README.md @@ -0,0 +1,29 @@ +# Layers +Layers are the fundamental building blocks for NLP models. They can be used to +assemble new layers, networks, or models. + +* [DenseEinsum](dense_einsum.py) implements a feedforward network using tf.einsum. This layer contains the einsum op, the associated weight, and the +logic required to generate the einsum expression for the given initialization +parameters. + +* [MultiHeadAttention](attention.py) implements an optionally masked attention +between two tensors, from_tensor and to_tensor, as described in +["Attention Is All You Need"](https://arxiv.org/abs/1706.03762). +If `from_tensor` and `to_tensor` are the same, then this is self-attention. + +* [CachedAttention](attention.py) implements an attention layer with cache used +for auto-agressive decoding. + +* [Transformer](transformer.py) implements an optionally masked transformer as +described in ["Attention Is All You Need"](https://arxiv.org/abs/1706.03762). + +* [OnDeviceEmbedding](on_device_embedding.py) implements efficient embedding lookups designed for TPU-based models. + +* [PositionalEmbedding](position_embedding.py) creates a positional embedding + as described in ["BERT: Pre-training + of Deep Bidirectional Transformers for Language Understanding"] + (https://arxiv.org/abs/1810.04805). + +* [SelfAttentionMask](self_attention_mask.py) creates a 3D attention mask from a 2D tensor mask. + +* [MaskedSoftmax](masked_softmax.py) implements a softmax with an optional masking input. If no mask is provided to this layer, it performs a standard softmax; however, if a mask tensor is applied (which should be 1 in positions where the data should be allowed through, and 0 where the data should be masked), the output will have masked positions set to approximately zero. diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/__init__.py new file mode 100644 index 0000000..2de8f6a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/__init__.py @@ -0,0 +1,23 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Layers package definition.""" +from official.nlp.modeling.layers.attention import * # pylint: disable=wildcard-import +from official.nlp.modeling.layers.dense_einsum import DenseEinsum +from official.nlp.modeling.layers.masked_softmax import MaskedSoftmax +from official.nlp.modeling.layers.on_device_embedding import OnDeviceEmbedding +from official.nlp.modeling.layers.position_embedding import PositionEmbedding +from official.nlp.modeling.layers.self_attention_mask import SelfAttentionMask +from official.nlp.modeling.layers.transformer import Transformer +from official.nlp.modeling.layers.transformer_scaffold import TransformerScaffold diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/attention.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/attention.py new file mode 100644 index 0000000..aa01daa --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/attention.py @@ -0,0 +1,264 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Keras-based attention layer.""" +# pylint: disable=g-classes-have-attributes +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import math +import tensorflow as tf + +from official.nlp.modeling.layers import dense_einsum +from official.nlp.modeling.layers import masked_softmax + + +@tf.keras.utils.register_keras_serializable(package="Text") +class MultiHeadAttention(tf.keras.layers.Layer): + """MultiHeadAttention layer. + + This is an implementation of multi-headed attention based on "Attention + is all you Need". If `from_tensor` and `to_tensor` are the same, then + this is self-attention. Each timestep in `from_tensor` attends to the + corresponding sequence in `to_tensor`, and returns a fixed-width vector. + + This function first projects `from_tensor` into a "query" tensor and + `to_tensor` into "key" and "value" tensors. These are (effectively) a list + of tensors of length `num_attention_heads`, where each tensor is of shape + [batch_size, seq_length, size_per_head]. + + Then, the query and key tensors are dot-producted and scaled. These are + softmaxed to obtain attention probabilities. The value tensors are then + interpolated by these probabilities, then concatenated back to a single + tensor and returned. + + Arguments: + num_heads: Number of attention heads. + head_size: Size of each attention head. + dropout: Dropout probability. + kernel_initializer: Initializer for dense layer kernels. + bias_initializer: Initializer for dense layer biases. + kernel_regularizer: Regularizer for dense layer kernels. + bias_regularizer: Regularizer for dense layer biases. + activity_regularizer: Regularizer for dense layer activity. + kernel_constraint: Constraint for dense layer kernels. + bias_constraint: Constraint for dense layer kernels. + """ + + def __init__(self, + num_heads, + head_size, + dropout_rate=0.0, + kernel_initializer="glorot_uniform", + bias_initializer="zeros", + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + **kwargs): + super(MultiHeadAttention, self).__init__(**kwargs) + self._num_heads = num_heads + self._head_size = head_size + self._dropout_rate = dropout_rate + self._kernel_initializer = tf.keras.initializers.get(kernel_initializer) + self._bias_initializer = tf.keras.initializers.get(bias_initializer) + self._kernel_regularizer = tf.keras.regularizers.get(kernel_regularizer) + self._bias_regularizer = tf.keras.regularizers.get(bias_regularizer) + self._kernel_constraint = tf.keras.constraints.get(kernel_constraint) + self._bias_constraint = tf.keras.constraints.get(bias_constraint) + + self._query_dense = dense_einsum.DenseEinsum( + output_shape=(self._num_heads, self._head_size), + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + activity_regularizer=self._activity_regularizer, + kernel_constraint=self._kernel_constraint, + bias_constraint=self._bias_constraint, + name="query") + + self._key_dense = dense_einsum.DenseEinsum( + output_shape=(self._num_heads, self._head_size), + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + activity_regularizer=self._activity_regularizer, + kernel_constraint=self._kernel_constraint, + bias_constraint=self._bias_constraint, + name="key") + + self._value_dense = dense_einsum.DenseEinsum( + output_shape=(self._num_heads, self._head_size), + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + activity_regularizer=self._activity_regularizer, + kernel_constraint=self._kernel_constraint, + bias_constraint=self._bias_constraint, + name="value") + + self._masked_softmax = masked_softmax.MaskedSoftmax(mask_expansion_axes=[1]) + + self._dropout = tf.keras.layers.Dropout(rate=self._dropout_rate) + + def get_config(self): + config = { + "num_heads": + self._num_heads, + "head_size": + self._head_size, + "dropout_rate": + self._dropout_rate, + "kernel_initializer": + tf.keras.initializers.serialize(self._kernel_initializer), + "bias_initializer": + tf.keras.initializers.serialize(self._bias_initializer), + "kernel_regularizer": + tf.keras.regularizers.serialize(self._kernel_regularizer), + "bias_regularizer": + tf.keras.regularizers.serialize(self._bias_regularizer), + "activity_regularizer": + tf.keras.regularizers.serialize(self._activity_regularizer), + "kernel_constraint": + tf.keras.constraints.serialize(self._kernel_constraint), + "bias_constraint": + tf.keras.constraints.serialize(self._bias_constraint) + } + base_config = super(MultiHeadAttention, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + def call(self, inputs): + from_tensor = inputs[0] + to_tensor = inputs[1] + attention_mask = inputs[2] if len(inputs) == 3 else None + + # Scalar dimensions referenced here: + # B = batch size (number of sequences) + # F = `from_tensor` sequence length + # T = `to_tensor` sequence length + # N = `num_attention_heads` + # H = `size_per_head` + # `query_tensor` = [B, F, N ,H] + query_tensor = self._query_dense(from_tensor) + + # `key_tensor` = [B, T, N, H] + key_tensor = self._key_dense(to_tensor) + + # `value_tensor` = [B, T, N, H] + value_tensor = self._value_dense(to_tensor) + + # Take the dot product between "query" and "key" to get the raw + # attention scores. + attention_scores = tf.einsum("BTNH,BFNH->BNFT", key_tensor, query_tensor) + attention_scores = tf.multiply(attention_scores, + 1.0 / math.sqrt(float(self._head_size))) + + # Normalize the attention scores to probabilities. + # `attention_probs` = [B, N, F, T] + attention_probs = self._masked_softmax([attention_scores, attention_mask]) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self._dropout(attention_probs) + + # `context_layer` = [B, F, N, H] + return tf.einsum("BNFT,BTNH->BFNH", attention_probs, value_tensor) + + +@tf.keras.utils.register_keras_serializable(package="Text") +class CachedAttention(MultiHeadAttention): + """Attention layer with cache used for auto-agressive decoding. + + Arguments: + num_heads: Number of attention heads. + head_size: Size of each attention head. + **kwargs: Other keyword arguments inherit from `Attention` class. + """ + + def __init__(self, num_heads, head_size, **kwargs): + super(CachedAttention, self).__init__(num_heads, head_size, **kwargs) + + def _update_cache(self, key_tensor, value_tensor, cache, decode_loop_step): + """Updates cache states and gets full-length key/value tensors.""" + # Combines cached keys and values with new keys and values. + if decode_loop_step is not None: + # TPU special case. + key_seq_dim = cache["key"].shape.as_list()[1] + indices = tf.reshape( + tf.one_hot(decode_loop_step, key_seq_dim, dtype=key_tensor.dtype), + [1, key_seq_dim, 1, 1]) + key_tensor = cache["key"] + key_tensor * indices + value_seq_dim = cache["value"].shape.as_list()[1] + indices = tf.reshape( + tf.one_hot(decode_loop_step, value_seq_dim, dtype=value_tensor.dtype), + [1, value_seq_dim, 1, 1]) + value_tensor = cache["value"] + value_tensor * indices + else: + key_tensor = tf.concat( + [tf.cast(cache["key"], key_tensor.dtype), key_tensor], axis=1) + value_tensor = tf.concat( + [tf.cast(cache["value"], value_tensor.dtype), value_tensor], axis=1) + + # Update cache + cache["key"] = key_tensor + cache["value"] = value_tensor + + return key_tensor, value_tensor + + def call(self, inputs, decode_loop_step=None): + from_tensor = inputs[0] + to_tensor = inputs[1] + attention_mask = inputs[2] if len(inputs) >= 3 else None + cache = inputs[3] if len(inputs) >= 4 else None + # Scalar dimensions referenced here: + # B = batch size (number of sequences) + # F = `from_tensor` sequence length + # T = `to_tensor` sequence length + # N = `num_attention_heads` + # H = `size_per_head` + # `query_tensor` = [B, F, N ,H] + query_tensor = self._query_dense(from_tensor) + + # `key_tensor` = [B, T, N, H] + key_tensor = self._key_dense(to_tensor) + + # `value_tensor` = [B, T, N, H] + value_tensor = self._value_dense(to_tensor) + + if cache: + key_tensor, value_tensor = self._update_cache(key_tensor, value_tensor, + cache, decode_loop_step) + + # Take the dot product between "query" and "key" to get the raw + # attention scores. + attention_scores = tf.einsum("BTNH,BFNH->BNFT", key_tensor, query_tensor) + attention_scores = tf.multiply(attention_scores, + 1.0 / math.sqrt(float(self._head_size))) + + # Normalize the attention scores to probabilities. + # `attention_probs` = [B, N, F, T] + attention_probs = self._masked_softmax([attention_scores, attention_mask]) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self._dropout(attention_probs) + + # `context_layer` = [B, F, N, H] + return tf.einsum("BNFT,BTNH->BFNH", attention_probs, value_tensor), cache diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/attention_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/attention_test.py new file mode 100644 index 0000000..8c577bb --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/attention_test.py @@ -0,0 +1,157 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for the attention layer.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.nlp.modeling.layers import attention + + +# This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It +# guarantees forward compatibility of this code for the V2 switchover. +@keras_parameterized.run_all_keras_modes +class MultiHeadAttentionTest(keras_parameterized.TestCase): + + def test_non_masked_attention(self): + """Test that the attention layer can be created without a mask tensor.""" + test_layer = attention.MultiHeadAttention(num_heads=12, head_size=64) + # Create a 3-dimensional input (the first dimension is implicit). + from_tensor = tf.keras.Input(shape=(40, 80)) + to_tensor = tf.keras.Input(shape=(20, 80)) + output = test_layer([from_tensor, to_tensor]) + self.assertEqual(output.shape.as_list(), [None, 40, 12, 64]) + + def test_non_masked_self_attention(self): + """Test with one input (self-attenntion) and no mask tensor.""" + test_layer = attention.MultiHeadAttention(num_heads=12, head_size=64) + # Create a 3-dimensional input (the first dimension is implicit). + from_tensor = tf.keras.Input(shape=(40, 80)) + output = test_layer([from_tensor, from_tensor]) + self.assertEqual(output.shape.as_list(), [None, 40, 12, 64]) + + def test_masked_attention(self): + """Test with a mask tensor.""" + test_layer = attention.MultiHeadAttention(num_heads=2, head_size=2) + # Create a 3-dimensional input (the first dimension is implicit). + from_tensor = tf.keras.Input(shape=(4, 8)) + to_tensor = tf.keras.Input(shape=(2, 8)) + mask_tensor = tf.keras.Input(shape=(4, 2)) + output = test_layer([from_tensor, to_tensor, mask_tensor]) + + # Create a model containing the test layer. + model = tf.keras.Model([from_tensor, to_tensor, mask_tensor], output) + + # Generate data for the input (non-mask) tensors. + from_data = 10 * np.random.random_sample((3, 4, 8)) + to_data = 10 * np.random.random_sample((3, 2, 8)) + + # Invoke the data with a random set of mask data. This should mask at least + # one element. + mask_data = np.random.randint(2, size=(3, 4, 2)) + masked_output_data = model.predict([from_data, to_data, mask_data]) + + # Invoke the same data, but with a null mask (where no elements are masked). + null_mask_data = np.ones((3, 4, 2)) + unmasked_output_data = model.predict([from_data, to_data, null_mask_data]) + + # Because one data is masked and one is not, the outputs should not be the + # same. + self.assertNotAllClose(masked_output_data, unmasked_output_data) + + def test_initializer(self): + """Test with a specified initializer.""" + test_layer = attention.MultiHeadAttention( + num_heads=12, + head_size=64, + kernel_initializer=tf.keras.initializers.TruncatedNormal(stddev=0.02)) + # Create a 3-dimensional input (the first dimension is implicit). + from_tensor = tf.keras.Input(shape=(40, 80)) + output = test_layer([from_tensor, from_tensor]) + self.assertEqual(output.shape.as_list(), [None, 40, 12, 64]) + + +def _create_cache(batch_size, init_decode_length, num_heads, head_size): + return { + "key": + tf.zeros([batch_size, init_decode_length, num_heads, head_size], + dtype=tf.float32), + "value": + tf.zeros([batch_size, init_decode_length, num_heads, head_size], + dtype=tf.float32) + } + + +@keras_parameterized.run_all_keras_modes +class CachedAttentionTest(keras_parameterized.TestCase): + + def test_masked_attention(self): + """Test with a mask tensor.""" + num_heads, head_size = 2, 2 + # Create a 3-dimensional input (the first dimension is implicit). + from_seq_length = 4 + batch_size = 3 + # GPU/CPU case. + init_decode_length = 0 + # Directly tests the keras layer. + cache = _create_cache(batch_size, init_decode_length, num_heads, head_size) + layer = attention.CachedAttention(num_heads=num_heads, head_size=head_size) + + # Generate data for the input (non-mask) tensors. + from_data = tf.zeros((batch_size, from_seq_length, 8), dtype=np.float32) + # Invoke the data with a random set of mask data. This should mask at least + # one element. + mask_data = np.random.randint( + 2, size=(batch_size, from_seq_length, from_seq_length)) + masked_output_data, cache = layer([from_data, from_data, mask_data, cache]) + self.assertEqual(masked_output_data.shape, (3, 4, 2, 2)) + self.assertEqual(cache["value"].shape, (3, 4, 2, 2)) + + # Tests inputs without cache. + masked_output_data, cache = layer([from_data, from_data, mask_data]) + self.assertEqual(masked_output_data.shape, (3, 4, 2, 2)) + self.assertIsNone(cache) + + def test_padded_decode(self): + """Test with a mask tensor.""" + num_heads, head_size = 2, 2 + from_seq_length = 4 + # TPU decoding should pre-allocate the entire sequence. + batch_size = 3 + init_decode_length = from_seq_length + + # Directly tests the keras layer. + cache = _create_cache(batch_size, init_decode_length, num_heads, head_size) + layer = attention.CachedAttention(num_heads=num_heads, head_size=head_size) + + # Generate data for the input (non-mask) tensors. + from_data = tf.zeros((batch_size, from_seq_length, 8), dtype=np.float32) + decode_loop_step = 2 + mask_data = np.random.randint( + 2, size=(batch_size, from_seq_length, from_seq_length), dtype=np.int32) + # Testing the invocation directly as Keras cannot consume inputs correctly. + masked_output_data, cache = layer([from_data, from_data, mask_data, cache], + decode_loop_step=decode_loop_step) + self.assertEqual(masked_output_data.shape, (3, 4, 2, 2)) + self.assertEqual(cache["value"].shape, (3, 4, 2, 2)) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/dense_einsum.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/dense_einsum.py new file mode 100644 index 0000000..ba2383e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/dense_einsum.py @@ -0,0 +1,180 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Keras-based einsum layer.""" +# pylint: disable=g-classes-have-attributes +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow as tf + +_CHR_IDX = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m"] + + +@tf.keras.utils.register_keras_serializable(package="Text") +class DenseEinsum(tf.keras.layers.Layer): + """A densely connected layer that uses tf.einsum as the backing computation. + + This layer can perform einsum calculations of arbitrary dimensionality. + + Arguments: + output_shape: Positive integer or tuple, dimensionality of the output space. + num_summed_dimensions: The number of dimensions to sum over. Standard 2D + matmul should use 1, 3D matmul should use 2, and so forth. + activation: Activation function to use. If you don't specify anything, no + activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + kernel_initializer: Initializer for the `kernel` weights matrix. + bias_initializer: Initializer for the bias vector. + kernel_regularizer: Regularizer function applied to the `kernel` weights + matrix. + bias_regularizer: Regularizer function applied to the bias vector. + activity_regularizer: Regularizer function applied to the output of the + layer (its "activation").. + kernel_constraint: Constraint function applied to the `kernel` weights + matrix. + bias_constraint: Constraint function applied to the bias vector. + Input shape: + N-D tensor with shape: `(batch_size, ..., input_dim)`. The most common + situation would be a 2D input with shape `(batch_size, input_dim)`. + Output shape: + N-D tensor with shape: `(batch_size, ..., units)`. For instance, for a 2D + input with shape `(batch_size, input_dim)`, the output would have shape + `(batch_size, units)`. + """ + + def __init__(self, + output_shape, + num_summed_dimensions=1, + activation=None, + use_bias=True, + kernel_initializer="glorot_uniform", + bias_initializer="zeros", + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + **kwargs): + super(DenseEinsum, self).__init__(**kwargs) + self._output_shape = output_shape if isinstance( + output_shape, (list, tuple)) else (output_shape,) + self._activation = tf.keras.activations.get(activation) + self._use_bias = use_bias + self._kernel_initializer = tf.keras.initializers.get(kernel_initializer) + self._bias_initializer = tf.keras.initializers.get(bias_initializer) + self._kernel_regularizer = tf.keras.regularizers.get(kernel_regularizer) + self._bias_regularizer = tf.keras.regularizers.get(bias_regularizer) + self._kernel_constraint = tf.keras.constraints.get(kernel_constraint) + self._bias_constraint = tf.keras.constraints.get(bias_constraint) + self._num_summed_dimensions = num_summed_dimensions + self._einsum_string = None + + def _build_einsum_string(self, free_input_dims, bound_dims, output_dims): + input_str = "" + kernel_str = "" + output_str = "" + letter_offset = 0 + for i in range(free_input_dims): + char = _CHR_IDX[i + letter_offset] + input_str += char + output_str += char + + letter_offset += free_input_dims + for i in range(bound_dims): + char = _CHR_IDX[i + letter_offset] + input_str += char + kernel_str += char + + letter_offset += bound_dims + for i in range(output_dims): + char = _CHR_IDX[i + letter_offset] + kernel_str += char + output_str += char + + return input_str + "," + kernel_str + "->" + output_str + + def build(self, input_shape): + input_shape = tf.TensorShape(input_shape) + input_rank = input_shape.rank + free_input_dims = input_rank - self._num_summed_dimensions + output_dims = len(self._output_shape) + + self._einsum_string = self._build_einsum_string(free_input_dims, + self._num_summed_dimensions, + output_dims) + + # This is only saved for testing purposes. + self._kernel_shape = ( + input_shape[free_input_dims:].concatenate(self._output_shape)) + + self._kernel = self.add_weight( + "kernel", + shape=self._kernel_shape, + initializer=self._kernel_initializer, + regularizer=self._kernel_regularizer, + constraint=self._kernel_constraint, + dtype=self.dtype, + trainable=True) + if self._use_bias: + self._bias = self.add_weight( + "bias", + shape=self._output_shape, + initializer=self._bias_initializer, + regularizer=self._bias_regularizer, + constraint=self._bias_constraint, + dtype=self.dtype, + trainable=True) + else: + self._bias = None + super(DenseEinsum, self).build(input_shape) + + def get_config(self): + config = { + "output_shape": + self._output_shape, + "num_summed_dimensions": + self._num_summed_dimensions, + "activation": + tf.keras.activations.serialize(self._activation), + "use_bias": + self._use_bias, + "kernel_initializer": + tf.keras.initializers.serialize(self._kernel_initializer), + "bias_initializer": + tf.keras.initializers.serialize(self._bias_initializer), + "kernel_regularizer": + tf.keras.regularizers.serialize(self._kernel_regularizer), + "bias_regularizer": + tf.keras.regularizers.serialize(self._bias_regularizer), + "activity_regularizer": + tf.keras.regularizers.serialize(self._activity_regularizer), + "kernel_constraint": + tf.keras.constraints.serialize(self._kernel_constraint), + "bias_constraint": + tf.keras.constraints.serialize(self._bias_constraint) + } + base_config = super(DenseEinsum, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + def call(self, inputs): + ret = tf.einsum(self._einsum_string, inputs, self._kernel) + if self._use_bias: + ret += self._bias + if self._activation is not None: + ret = self._activation(ret) + return ret diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/dense_einsum_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/dense_einsum_test.py new file mode 100644 index 0000000..57a60fe --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/dense_einsum_test.py @@ -0,0 +1,123 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Keras-based einsum layer.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.nlp.modeling.layers import dense_einsum + + +# This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It +# guarantees forward compatibility of this code for the V2 switchover. +@keras_parameterized.run_all_keras_modes +class DenseEinsumLayer(keras_parameterized.TestCase): + + def test_3D_einsum_with_two_bound_dimensions(self): + test_layer = dense_einsum.DenseEinsum( + output_shape=(64,), num_summed_dimensions=2) + # Create a 4-dimensional input (the first dimension is implicit). + input_tensor = tf.keras.Input(shape=(None, 40, 80)) + _ = test_layer(input_tensor) + self.assertEqual(test_layer._einsum_string, "abcd,cde->abe") + self.assertEqual(test_layer._kernel_shape, (40, 80, 64)) + + def test_3D_einsum_with_one_bound_dimensions(self): + test_layer = dense_einsum.DenseEinsum( + output_shape=(64, 32), num_summed_dimensions=1) + # Create a 3-dimensional input (the first dimension is implicit). + input_tensor = tf.keras.Input(shape=(None, 80)) + _ = test_layer(input_tensor) + self.assertEqual(test_layer._einsum_string, "abc,cde->abde") + self.assertEqual(test_layer._kernel_shape, (80, 64, 32)) + + def test_2D_einsum_with_one_bound_dimensions(self): + test_layer = dense_einsum.DenseEinsum( + output_shape=(64,), num_summed_dimensions=1) + # Create a 3-dimensional input (the first dimension is implicit). + input_tensor = tf.keras.Input(shape=(None, 80)) + _ = test_layer(input_tensor) + self.assertEqual(test_layer._einsum_string, "abc,cd->abd") + self.assertEqual(test_layer._kernel_shape, (80, 64)) + + def test_bias_term_can_be_disabled(self): + # A layer created using the bias should have two weights. + test_layer = dense_einsum.DenseEinsum( + output_shape=64, num_summed_dimensions=1, use_bias=True) + input_tensor = tf.keras.Input(shape=(None, 80)) + _ = test_layer(input_tensor) + self.assertEqual(2, len(test_layer.get_weights())) + + # A layer created without the bias should have only one weight. + test_layer = dense_einsum.DenseEinsum( + output_shape=64, num_summed_dimensions=1, use_bias=False) + input_tensor = tf.keras.Input(shape=(None, 80)) + _ = test_layer(input_tensor) + self.assertEqual(1, len(test_layer.get_weights())) + + def test_activation(self): + # Create a model that does not use an activation. + no_activation_layer = dense_einsum.DenseEinsum( + output_shape=64, num_summed_dimensions=1, activation=None) + input_tensor = tf.keras.Input(shape=(None, 80)) + output_tensor = no_activation_layer(input_tensor) + no_activation_model = tf.keras.Model(input_tensor, output_tensor) + + # Create a model that uses a softmax activation. + activation_layer = dense_einsum.DenseEinsum( + output_shape=64, num_summed_dimensions=1, activation="softmax") + input_tensor = tf.keras.Input(shape=(None, 80)) + output_tensor = activation_layer(input_tensor) + activation_model = tf.keras.Model(input_tensor, output_tensor) + + # Make sure the models' weights are identical. + activation_model.set_weights(no_activation_model.get_weights()) + + # Predict using each model on the same input data. The output should be + # different, since one is using a softmax - even though the models' weights + # are the same. + input_values = 10 * np.random.random_sample((10, 4, 80)) + non_activated_data = no_activation_model.predict(input_values) + activated_data = activation_model.predict(input_values) + self.assertNotAllClose(activated_data, non_activated_data) + + def test_non_iterable_output_shape(self): + test_layer = dense_einsum.DenseEinsum( + output_shape=64, num_summed_dimensions=1) + # Create a 3-dimensional input (the first dimension is implicit). + input_tensor = tf.keras.Input(shape=(None, 80)) + _ = test_layer(input_tensor) + self.assertEqual(test_layer._einsum_string, "abc,cd->abd") + self.assertEqual(test_layer._kernel_shape, (80, 64)) + + def test_with_explicit_initializer(self): + test_layer = dense_einsum.DenseEinsum( + output_shape=(64,), + num_summed_dimensions=2, + kernel_initializer=tf.keras.initializers.TruncatedNormal(stddev=0.02)) + # Create a 4-dimensional input (the first dimension is implicit). + input_tensor = tf.keras.Input(shape=(None, 40, 80)) + _ = test_layer(input_tensor) + self.assertEqual(test_layer._einsum_string, "abcd,cde->abe") + self.assertEqual(test_layer._kernel_shape, (40, 80, 64)) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/masked_softmax.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/masked_softmax.py new file mode 100644 index 0000000..f4a04e4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/masked_softmax.py @@ -0,0 +1,61 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Keras-based softmax layer with optional masking.""" +# pylint: disable=g-classes-have-attributes +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow as tf + + +@tf.keras.utils.register_keras_serializable(package='Text') +class MaskedSoftmax(tf.keras.layers.Layer): + """Performs a softmax with optional masking on a tensor. + + Arguments: + mask_expansion_axes: Any axes that should be padded on the mask tensor. + """ + + def __init__(self, mask_expansion_axes=None, **kwargs): + self._mask_expansion_axes = mask_expansion_axes + super(MaskedSoftmax, self).__init__(**kwargs) + + def call(self, inputs): + if isinstance(inputs, list) and len(inputs) == 2: + scores, mask = inputs + else: + scores, mask = (inputs, None) + + if mask is not None: + if self._mask_expansion_axes is not None: + mask = tf.expand_dims(mask, axis=self._mask_expansion_axes) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + adder = (1.0 - tf.cast(mask, scores.dtype)) * -10000.0 + + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + scores += adder + + return tf.nn.softmax(scores) + + def get_config(self): + config = {'mask_expansion_axes': self._mask_expansion_axes} + base_config = super(MaskedSoftmax, self).get_config() + return dict(list(base_config.items()) + list(config.items())) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/masked_softmax_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/masked_softmax_test.py new file mode 100644 index 0000000..148f60c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/masked_softmax_test.py @@ -0,0 +1,88 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Keras-based masked softmax layer.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.nlp.modeling.layers import masked_softmax + + +# This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It +# guarantees forward compatibility of this code for the V2 switchover. +@keras_parameterized.run_all_keras_modes +class MaskedSoftmaxLayerTest(keras_parameterized.TestCase): + + def test_non_masked_softmax(self): + test_layer = masked_softmax.MaskedSoftmax() + input_tensor = tf.keras.Input(shape=(4, 8)) + output = test_layer(input_tensor) + model = tf.keras.Model(input_tensor, output) + + input_data = 10 * np.random.random_sample((3, 4, 8)) + output_data = model.predict(input_data) + expected_data = tf.nn.softmax(input_data) + self.assertAllClose(expected_data, output_data) + + def test_masked_softmax(self): + test_layer = masked_softmax.MaskedSoftmax() + input_tensor = tf.keras.Input(shape=(4, 8)) + mask_tensor = tf.keras.Input(shape=(4, 8)) + output = test_layer([input_tensor, mask_tensor]) + model = tf.keras.Model([input_tensor, mask_tensor], output) + + input_data = 10 * np.random.random_sample((3, 4, 8)) + mask_data = np.random.randint(2, size=(3, 4, 8)) + + output_data = model.predict([input_data, mask_data]) + expected_zeros = np.greater(mask_data, 0) + is_zeros = np.greater(output_data, 0) + self.assertAllEqual(expected_zeros, is_zeros) + + def test_masked_softmax_with_none_mask(self): + test_layer = masked_softmax.MaskedSoftmax() + input_tensor = tf.keras.Input(shape=(4, 8)) + output = test_layer([input_tensor, None]) + model = tf.keras.Model(input_tensor, output) + + input_data = 10 * np.random.random_sample((3, 4, 8)) + output_data = model.predict(input_data) + expected_data = tf.nn.softmax(input_data) + self.assertAllClose(expected_data, output_data) + + def test_softmax_with_axes_expansion(self): + test_layer = masked_softmax.MaskedSoftmax(mask_expansion_axes=[1]) + input_tensor = tf.keras.Input(shape=(4, 8)) + mask_tensor = tf.keras.Input(shape=(8)) + output = test_layer([input_tensor, mask_tensor]) + model = tf.keras.Model([input_tensor, mask_tensor], output) + + input_data = 10 * np.random.random_sample((3, 4, 8)) + mask_data = np.random.randint(2, size=(3, 8)) + + output_data = model.predict([input_data, mask_data]) + expanded_mask = np.expand_dims(mask_data, axis=1) * np.ones_like(input_data) + expected_zeros = np.greater(expanded_mask, 0) + is_zeros = np.greater(output_data, 0) + self.assertAllEqual(expected_zeros, is_zeros) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/on_device_embedding.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/on_device_embedding.py new file mode 100644 index 0000000..a3afad9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/on_device_embedding.py @@ -0,0 +1,92 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Keras-based one-hot embedding layer.""" +# pylint: disable=g-classes-have-attributes +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow as tf + +from official.modeling import tf_utils + + +@tf.keras.utils.register_keras_serializable(package="Text") +class OnDeviceEmbedding(tf.keras.layers.Layer): + """Performs an embedding lookup suitable for accelerator devices. + + This layer uses either tf.gather or tf.one_hot to translate integer indices to + float embeddings. + + Arguments: + vocab_size: Number of elements in the vocabulary. + embedding_width: Output size of the embedding layer. + initializer: The initializer to use for the embedding weights. Defaults to + "glorot_uniform". + use_one_hot: Whether to use tf.one_hot over tf.gather for the embedding + lookup. Defaults to False (that is, using tf.gather). Setting this option + to True may improve performance, especially on small vocabulary sizes, + but will generally require more memory. + """ + + def __init__(self, + vocab_size, + embedding_width, + initializer="glorot_uniform", + use_one_hot=False, + **kwargs): + # We need to have a default dtype of float32, since the inputs (which Keras + # usually uses to infer the dtype) will always be int32. + if "dtype" not in kwargs: + kwargs["dtype"] = "float32" + + super(OnDeviceEmbedding, self).__init__(**kwargs) + self._vocab_size = vocab_size + self._embedding_width = embedding_width + self._initializer = initializer + self._use_one_hot = use_one_hot + + def get_config(self): + config = { + "vocab_size": self._vocab_size, + "embedding_width": self._embedding_width, + "initializer": self._initializer, + "use_one_hot": self._use_one_hot, + } + base_config = super(OnDeviceEmbedding, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + def build(self, input_shape): + self.embeddings = self.add_weight( + "embeddings", + shape=[self._vocab_size, self._embedding_width], + initializer=self._initializer) + + super(OnDeviceEmbedding, self).build(input_shape) + + def call(self, inputs): + input_shape = tf_utils.get_shape_list(inputs, expected_rank=2) + input_shape.append(self._embedding_width) + flat_inputs = tf.reshape(inputs, [-1]) + if self._use_one_hot: + one_hot_data = tf.one_hot( + flat_inputs, depth=self._vocab_size, dtype=self._dtype) + embeddings = tf.matmul(one_hot_data, self.embeddings) + else: + embeddings = tf.gather(self.embeddings, flat_inputs) + embeddings = tf.reshape(embeddings, input_shape) + + return embeddings diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/on_device_embedding_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/on_device_embedding_test.py new file mode 100644 index 0000000..e2346f9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/on_device_embedding_test.py @@ -0,0 +1,193 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Keras-based one-hot embedding layer.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.nlp.modeling.layers import on_device_embedding + + +# This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It +# guarantees forward compatibility of this code for the V2 switchover. +@keras_parameterized.run_all_keras_modes +class OnDeviceEmbeddingTest(keras_parameterized.TestCase): + + def test_layer_creation(self): + vocab_size = 31 + embedding_width = 27 + test_layer = on_device_embedding.OnDeviceEmbedding( + vocab_size=vocab_size, embedding_width=embedding_width) + # Create a 2-dimensional input (the first dimension is implicit). + sequence_length = 23 + input_tensor = tf.keras.Input(shape=(sequence_length), dtype=tf.int32) + output_tensor = test_layer(input_tensor) + + # The output should be the same as the input, save that it has an extra + # embedding_width dimension on the end. + expected_output_shape = [None, sequence_length, embedding_width] + self.assertEqual(expected_output_shape, output_tensor.shape.as_list()) + self.assertEqual(output_tensor.dtype, tf.float32) + + def test_layer_creation_with_float16_dtype(self): + vocab_size = 31 + embedding_width = 27 + test_layer = on_device_embedding.OnDeviceEmbedding( + vocab_size=vocab_size, embedding_width=embedding_width, dtype="float16") + # Create a 2-dimensional input (the first dimension is implicit). + sequence_length = 23 + input_tensor = tf.keras.Input(shape=(sequence_length), dtype=tf.int32) + output_tensor = test_layer(input_tensor) + + # The output should be the same as the input, save that it has an extra + # embedding_width dimension on the end. + expected_output_shape = [None, sequence_length, embedding_width] + self.assertEqual(expected_output_shape, output_tensor.shape.as_list()) + self.assertEqual(output_tensor.dtype, tf.float16) + + def test_layer_invocation(self): + vocab_size = 31 + embedding_width = 27 + test_layer = on_device_embedding.OnDeviceEmbedding( + vocab_size=vocab_size, embedding_width=embedding_width) + # Create a 2-dimensional input (the first dimension is implicit). + sequence_length = 23 + input_tensor = tf.keras.Input(shape=(sequence_length), dtype=tf.int32) + output_tensor = test_layer(input_tensor) + + # Create a model from the test layer. + model = tf.keras.Model(input_tensor, output_tensor) + + # Invoke the model on test data. We can't validate the output data itself + # (the NN is too complex) but this will rule out structural runtime errors. + batch_size = 3 + input_data = np.random.randint( + vocab_size, size=(batch_size, sequence_length)) + output = model.predict(input_data) + self.assertEqual(tf.float32, output.dtype) + + def test_layer_invocation_with_float16_dtype(self): + vocab_size = 31 + embedding_width = 27 + test_layer = on_device_embedding.OnDeviceEmbedding( + vocab_size=vocab_size, embedding_width=embedding_width, dtype="float16") + # Create a 2-dimensional input (the first dimension is implicit). + sequence_length = 23 + input_tensor = tf.keras.Input(shape=(sequence_length), dtype=tf.int32) + output_tensor = test_layer(input_tensor) + + # Create a model from the test layer. + model = tf.keras.Model(input_tensor, output_tensor) + + # Invoke the model on test data. We can't validate the output data itself + # (the NN is too complex) but this will rule out structural runtime errors. + batch_size = 3 + input_data = np.random.randint( + vocab_size, size=(batch_size, sequence_length)) + output = model.predict(input_data) + self.assertEqual(tf.float16, output.dtype) + + def test_one_hot_layer_creation(self): + vocab_size = 31 + embedding_width = 27 + test_layer = on_device_embedding.OnDeviceEmbedding( + vocab_size=vocab_size, + embedding_width=embedding_width, + use_one_hot=True) + # Create a 2-dimensional input (the first dimension is implicit). + sequence_length = 23 + input_tensor = tf.keras.Input(shape=(sequence_length), dtype=tf.int32) + output_tensor = test_layer(input_tensor) + + # The output should be the same as the input, save that it has an extra + # embedding_width dimension on the end. + expected_output_shape = [None, sequence_length, embedding_width] + self.assertEqual(expected_output_shape, output_tensor.shape.as_list()) + self.assertEqual(output_tensor.dtype, tf.float32) + + def test_one_hot_layer_creation_with_float16_dtype(self): + vocab_size = 31 + embedding_width = 27 + test_layer = on_device_embedding.OnDeviceEmbedding( + vocab_size=vocab_size, + embedding_width=embedding_width, + dtype="float16", + use_one_hot=True) + # Create a 2-dimensional input (the first dimension is implicit). + sequence_length = 23 + input_tensor = tf.keras.Input(shape=(sequence_length), dtype=tf.int32) + output_tensor = test_layer(input_tensor) + + # The output should be the same as the input, save that it has an extra + # embedding_width dimension on the end. + expected_output_shape = [None, sequence_length, embedding_width] + self.assertEqual(expected_output_shape, output_tensor.shape.as_list()) + self.assertEqual(output_tensor.dtype, tf.float16) + + def test_one_hot_layer_invocation(self): + vocab_size = 31 + embedding_width = 27 + test_layer = on_device_embedding.OnDeviceEmbedding( + vocab_size=vocab_size, + embedding_width=embedding_width, + use_one_hot=True) + # Create a 2-dimensional input (the first dimension is implicit). + sequence_length = 23 + input_tensor = tf.keras.Input(shape=(sequence_length), dtype=tf.int32) + output_tensor = test_layer(input_tensor) + + # Create a model from the test layer. + model = tf.keras.Model(input_tensor, output_tensor) + + # Invoke the model on test data. We can't validate the output data itself + # (the NN is too complex) but this will rule out structural runtime errors. + batch_size = 3 + input_data = np.random.randint( + vocab_size, size=(batch_size, sequence_length)) + output = model.predict(input_data) + self.assertEqual(tf.float32, output.dtype) + + def test_one_hot_layer_invocation_with_float16_dtype(self): + vocab_size = 31 + embedding_width = 27 + test_layer = on_device_embedding.OnDeviceEmbedding( + vocab_size=vocab_size, + embedding_width=embedding_width, + dtype="float16", + use_one_hot=True) + # Create a 2-dimensional input (the first dimension is implicit). + sequence_length = 23 + input_tensor = tf.keras.Input(shape=(sequence_length), dtype=tf.int32) + output_tensor = test_layer(input_tensor) + + # Create a model from the test layer. + model = tf.keras.Model(input_tensor, output_tensor) + + # Invoke the model on test data. We can't validate the output data itself + # (the NN is too complex) but this will rule out structural runtime errors. + batch_size = 3 + input_data = np.random.randint( + vocab_size, size=(batch_size, sequence_length)) + output = model.predict(input_data) + self.assertEqual(tf.float16, output.dtype) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/position_embedding.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/position_embedding.py new file mode 100644 index 0000000..f49e491 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/position_embedding.py @@ -0,0 +1,120 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Keras-based positional embedding layer.""" +# pylint: disable=g-classes-have-attributes +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow as tf + +from official.modeling import tf_utils + + +@tf.keras.utils.register_keras_serializable(package="Text") +class PositionEmbedding(tf.keras.layers.Layer): + """Creates a positional embedding. + + This layer creates a positional embedding as described in "BERT: Pre-training + of Deep Bidirectional Transformers for Language Understanding" + (https://arxiv.org/abs/1810.04805). + + This layer can be set up to either create a statically shaped slice or a + dynamically shaped slice. If `use_dynamic_slicing` is True, the input tensor + can have a dynamic 1st dimension, while if `use_dynamic_slicing` is False the + input size must be fixed. + + Arguments: + use_dynamic_slicing: Whether to use the dynamic slicing path. + max_sequence_length: The maximum size of the dynamic sequence. Only + applicable if `use_dynamic_slicing` is True. + initializer: The initializer to use for the embedding weights. Defaults to + "glorot_uniform". + """ + + def __init__(self, + initializer="glorot_uniform", + use_dynamic_slicing=False, + max_sequence_length=None, + **kwargs): + # We need to have a default dtype of float32, since the inputs (which Keras + # usually uses to infer the dtype) will always be int32. + if "dtype" not in kwargs: + kwargs["dtype"] = "float32" + + super(PositionEmbedding, self).__init__(**kwargs) + if use_dynamic_slicing and max_sequence_length is None: + raise ValueError( + "If `use_dynamic_slicing` is True, `max_sequence_length` must be set." + ) + self._max_sequence_length = max_sequence_length + self._initializer = tf.keras.initializers.get(initializer) + self._use_dynamic_slicing = use_dynamic_slicing + + def get_config(self): + config = { + "max_sequence_length": self._max_sequence_length, + "initializer": tf.keras.initializers.serialize(self._initializer), + "use_dynamic_slicing": self._use_dynamic_slicing, + } + base_config = super(PositionEmbedding, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + def build(self, input_shape): + """Implements build() for the layer.""" + dimension_list = input_shape.as_list() + + if len(dimension_list) != 3: + raise ValueError("PositionEmbedding expects a 3-dimensional input tensor " + "of shape [batch, sequence, width]") + seq_length = dimension_list[1] + width = dimension_list[2] + + # If we are not using dynamic slicing, we must assume that the sequence + # length is fixed and max_sequence_length should not be specified. + if not self._use_dynamic_slicing: + if seq_length is None: + raise ValueError( + "PositionEmbedding must have `use_dynamic_slicing` set " + "to True (and max_sequence_length set) when the " + "sequence (1st) dimension of the input is None.") + if self._max_sequence_length is not None: + raise ValueError( + "When `use_dynamic_slicing` is False, max_sequence_length should " + "not be specified and we ought to use seq_length to get the " + "variable shape.") + + if self._max_sequence_length is not None: + weight_sequence_length = self._max_sequence_length + else: + weight_sequence_length = seq_length + + self._position_embeddings = self.add_weight( + "embeddings", + shape=[weight_sequence_length, width], + initializer=self._initializer) + + super(PositionEmbedding, self).build(input_shape) + + def call(self, inputs): + """Implements call() for the layer.""" + input_shape = tf_utils.get_shape_list(inputs, expected_rank=3) + if self._use_dynamic_slicing: + position_embeddings = self._position_embeddings[:input_shape[1], :] + else: + position_embeddings = self._position_embeddings + + return tf.broadcast_to(position_embeddings, input_shape) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/position_embedding_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/position_embedding_test.py new file mode 100644 index 0000000..bc446c9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/position_embedding_test.py @@ -0,0 +1,103 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Keras-based positional embedding layer.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.nlp.modeling.layers import position_embedding + + +# This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It +# guarantees forward compatibility of this code for the V2 switchover. +@keras_parameterized.run_all_keras_modes +class PositionEmbeddingLayerTest(keras_parameterized.TestCase): + + def test_static_layer_output_shape(self): + test_layer = position_embedding.PositionEmbedding() + # Create a 3-dimensional input (the first dimension is implicit). + sequence_length = 21 + width = 30 + input_tensor = tf.keras.Input(shape=(sequence_length, width)) + output_tensor = test_layer(input_tensor) + + # When using static positional embedding shapes, the output is expected + # to be the same as the input shape in all dimensions save batch. + expected_output_shape = [None, sequence_length, width] + self.assertEqual(expected_output_shape, output_tensor.shape.as_list()) + # The default output dtype for this layer should be tf.float32. + self.assertEqual(tf.float32, output_tensor.dtype) + + def test_float16_dtype(self): + test_layer = position_embedding.PositionEmbedding(dtype="float16") + # Create a 3-dimensional input (the first dimension is implicit). + sequence_length = 21 + width = 30 + input_tensor = tf.keras.Input(shape=(sequence_length, width)) + output_tensor = test_layer(input_tensor) + + # When using static positional embedding shapes, the output is expected + # to be the same as the input shape in all dimensions save batch. + expected_output_shape = [None, sequence_length, width] + self.assertEqual(expected_output_shape, output_tensor.shape.as_list()) + # The default output dtype for this layer should be tf.float32. + self.assertEqual(tf.float16, output_tensor.dtype) + + def test_dynamic_layer_output_shape(self): + max_sequence_length = 40 + test_layer = position_embedding.PositionEmbedding( + use_dynamic_slicing=True, max_sequence_length=max_sequence_length) + # Create a 3-dimensional input (the first dimension is implicit). + width = 30 + input_tensor = tf.keras.Input(shape=(None, width)) + output_tensor = test_layer(input_tensor) + + # When using dynamic positional embedding shapes, the output is expected + # to be the same as the input shape in all dimensions - but may be None if + # the input shape is None there. + expected_output_shape = [None, None, width] + self.assertEqual(expected_output_shape, output_tensor.shape.as_list()) + + def test_dynamic_layer_slicing(self): + max_sequence_length = 40 + test_layer = position_embedding.PositionEmbedding( + use_dynamic_slicing=True, max_sequence_length=max_sequence_length) + # Create a 3-dimensional input (the first dimension is implicit). + width = 30 + input_tensor = tf.keras.Input(shape=(None, width)) + output_tensor = test_layer(input_tensor) + + model = tf.keras.Model(input_tensor, output_tensor) + + # Create input data that is shorter than max_sequence_length, which should + # trigger a down-slice. + input_length = 17 + # Note: This test explicitly uses a batch size of 1. This is to get around + # Keras' restriction on Model invocations: inputs are expected to have the + # same batch cardinality as outputs. In practice, this layer should be used + # inside a model, where it can be projected when added to another tensor. + input_data = np.ones((1, input_length, width)) + output_data = model.predict(input_data) + + self.assertAllEqual([1, input_length, width], output_data.shape) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/self_attention_mask.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/self_attention_mask.py new file mode 100644 index 0000000..933b496 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/self_attention_mask.py @@ -0,0 +1,63 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Keras layer that creates a self-attention mask.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow as tf +from official.modeling import tf_utils + + +@tf.keras.utils.register_keras_serializable(package='Text') +class SelfAttentionMask(tf.keras.layers.Layer): + """Create 3D attention mask from a 2D tensor mask. + + inputs[0]: from_tensor: 2D or 3D Tensor of shape + [batch_size, from_seq_length, ...]. + inputs[1]: to_mask: int32 Tensor of shape [batch_size, to_seq_length]. + + Returns: + float Tensor of shape [batch_size, from_seq_length, to_seq_length]. + """ + + def call(self, inputs): + from_tensor = inputs[0] + to_mask = inputs[1] + from_shape = tf_utils.get_shape_list(from_tensor, expected_rank=[2, 3]) + batch_size = from_shape[0] + from_seq_length = from_shape[1] + + to_shape = tf_utils.get_shape_list(to_mask, expected_rank=2) + to_seq_length = to_shape[1] + + to_mask = tf.cast( + tf.reshape(to_mask, [batch_size, 1, to_seq_length]), + dtype=from_tensor.dtype) + + # We don't assume that `from_tensor` is a mask (although it could be). We + # don't actually care if we attend *from* padding tokens (only *to* padding) + # tokens so we create a tensor of all ones. + # + # `broadcast_ones` = [batch_size, from_seq_length, 1] + broadcast_ones = tf.ones( + shape=[batch_size, from_seq_length, 1], dtype=from_tensor.dtype) + + # Here we broadcast along two dimensions to create the mask. + mask = broadcast_ones * to_mask + + return mask diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/transformer.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/transformer.py new file mode 100644 index 0000000..9bd71e8 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/transformer.py @@ -0,0 +1,229 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Keras-based transformer block layer.""" +# pylint: disable=g-classes-have-attributes +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow as tf + +from official.nlp.modeling.layers import attention +from official.nlp.modeling.layers import dense_einsum +from official.nlp.modeling.layers.util import tf_function_if_eager + + +@tf.keras.utils.register_keras_serializable(package="Text") +class Transformer(tf.keras.layers.Layer): + """Transformer layer. + + This layer implements the Transformer from "Attention Is All You Need". + (https://arxiv.org/abs/1706.03762). + + Arguments: + num_attention_heads: Number of attention heads. + intermediate_size: Size of the intermediate layer. + intermediate_activation: Activation for the intermediate layer. + dropout_rate: Dropout probability for the post-attention and output dropout. + attention_dropout_rate: Dropout probability for within the attention layer. + kernel_initializer: Initializer for dense layer kernels. + bias_initializer: Initializer for dense layer biases. + kernel_regularizer: Regularizer for dense layer kernels. + bias_regularizer: Regularizer for dense layer biases. + activity_regularizer: Regularizer for dense layer activity. + kernel_constraint: Constraint for dense layer kernels. + bias_constraint: Constraint for dense layer kernels. + """ + + def __init__(self, + num_attention_heads, + intermediate_size, + intermediate_activation, + dropout_rate=0.0, + attention_dropout_rate=0.0, + kernel_initializer="glorot_uniform", + bias_initializer="zeros", + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + **kwargs): + super(Transformer, self).__init__(**kwargs) + + self._num_heads = num_attention_heads + self._intermediate_size = intermediate_size + self._intermediate_activation = intermediate_activation + self._attention_dropout_rate = attention_dropout_rate + self._dropout_rate = dropout_rate + self._kernel_initializer = tf.keras.initializers.get(kernel_initializer) + self._bias_initializer = tf.keras.initializers.get(bias_initializer) + self._kernel_regularizer = tf.keras.regularizers.get(kernel_regularizer) + self._bias_regularizer = tf.keras.regularizers.get(bias_regularizer) + self._kernel_constraint = tf.keras.constraints.get(kernel_constraint) + self._bias_constraint = tf.keras.constraints.get(bias_constraint) + + def build(self, input_shape): + input_tensor = input_shape[0] if len(input_shape) == 2 else input_shape + input_tensor_shape = tf.TensorShape(input_tensor) + if len(input_tensor_shape) != 3: + raise ValueError("TransformerLayer expects a three-dimensional input of " + "shape [batch, sequence, width].") + batch_size, sequence_length, hidden_size = input_tensor_shape + + if len(input_shape) == 2: + mask_tensor_shape = tf.TensorShape(input_shape[1]) + expected_mask_tensor_shape = tf.TensorShape( + [batch_size, sequence_length, sequence_length]) + if not expected_mask_tensor_shape.is_compatible_with(mask_tensor_shape): + raise ValueError("When passing a mask tensor to TransformerLayer, the " + "mask tensor must be of shape [batch, " + "sequence_length, sequence_length] (here %s). Got a " + "mask tensor of shape %s." % + (expected_mask_tensor_shape, mask_tensor_shape)) + if hidden_size % self._num_heads != 0: + raise ValueError( + "The input size (%d) is not a multiple of the number of attention " + "heads (%d)" % (hidden_size, self._num_heads)) + self._attention_head_size = int(hidden_size // self._num_heads) + + self._attention_layer = attention.MultiHeadAttention( + num_heads=self._num_heads, + head_size=self._attention_head_size, + dropout_rate=self._attention_dropout_rate, + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + activity_regularizer=self._activity_regularizer, + kernel_constraint=self._kernel_constraint, + bias_constraint=self._bias_constraint, + name="self_attention") + self._attention_output_dense = dense_einsum.DenseEinsum( + output_shape=hidden_size, + num_summed_dimensions=2, + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + activity_regularizer=self._activity_regularizer, + kernel_constraint=self._kernel_constraint, + bias_constraint=self._bias_constraint, + name="self_attention_output") + self._attention_dropout = tf.keras.layers.Dropout(rate=self._dropout_rate) + # Use float32 in layernorm for numeric stability. + # It is probably safe in mixed_float16, but we haven't validated this yet. + self._attention_layer_norm = ( + tf.keras.layers.LayerNormalization( + name="self_attention_layer_norm", + axis=-1, + epsilon=1e-12, + dtype=tf.float32)) + self._intermediate_dense = dense_einsum.DenseEinsum( + output_shape=self._intermediate_size, + activation=None, + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + activity_regularizer=self._activity_regularizer, + kernel_constraint=self._kernel_constraint, + bias_constraint=self._bias_constraint, + name="intermediate") + self._intermediate_activation_layer = tf.keras.layers.Activation( + self._intermediate_activation) + self._output_dense = dense_einsum.DenseEinsum( + output_shape=hidden_size, + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + activity_regularizer=self._activity_regularizer, + kernel_constraint=self._kernel_constraint, + bias_constraint=self._bias_constraint, + name="output") + self._output_dropout = tf.keras.layers.Dropout(rate=self._dropout_rate) + # Use float32 in layernorm for numeric stability. + self._output_layer_norm = tf.keras.layers.LayerNormalization( + name="output_layer_norm", axis=-1, epsilon=1e-12, dtype=tf.float32) + + super(Transformer, self).build(input_shape) + + def get_config(self): + config = { + "num_attention_heads": + self._num_heads, + "intermediate_size": + self._intermediate_size, + "intermediate_activation": + self._intermediate_activation, + "dropout_rate": + self._dropout_rate, + "attention_dropout_rate": + self._attention_dropout_rate, + "kernel_initializer": + tf.keras.initializers.serialize(self._kernel_initializer), + "bias_initializer": + tf.keras.initializers.serialize(self._bias_initializer), + "kernel_regularizer": + tf.keras.regularizers.serialize(self._kernel_regularizer), + "bias_regularizer": + tf.keras.regularizers.serialize(self._bias_regularizer), + "activity_regularizer": + tf.keras.regularizers.serialize(self._activity_regularizer), + "kernel_constraint": + tf.keras.constraints.serialize(self._kernel_constraint), + "bias_constraint": + tf.keras.constraints.serialize(self._bias_constraint) + } + base_config = super(Transformer, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + def call(self, inputs): + if isinstance(inputs, (list, tuple)) and len(inputs) == 2: + input_tensor, attention_mask = inputs + else: + input_tensor, attention_mask = (inputs, None) + + attention_inputs = [input_tensor, input_tensor] + + if attention_mask is not None: + attention_inputs.append(attention_mask) + + attention_output = self._attention_layer(attention_inputs) + attention_output = self._attention_output_dense(attention_output) + attention_output = self._attention_dropout(attention_output) + attention_output = self._attention_layer_norm(input_tensor + + attention_output) + intermediate_output = self._intermediate_dense(attention_output) + intermediate_output = self._intermediate_activation_layer( + intermediate_output) + layer_output = self._output_dense(intermediate_output) + layer_output = self._output_dropout(layer_output) + # During mixed precision training, attention_output is from layer norm and + # is always fp32 for now. Cast layer_output to fp32 for the subsequent + # add. + layer_output = tf.cast(layer_output, tf.float32) + layer_output = self._output_layer_norm(layer_output + attention_output) + + return layer_output + + +class CompiledTransformer(Transformer): + + @tf_function_if_eager(experimental_compile=True) + def call(self, inputs): + return super(CompiledTransformer, self).call(inputs) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/transformer_scaffold.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/transformer_scaffold.py new file mode 100644 index 0000000..c45bd2d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/transformer_scaffold.py @@ -0,0 +1,238 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Keras-based transformer scaffold layer.""" +# pylint: disable=g-classes-have-attributes +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import gin +import tensorflow as tf + +from official.nlp.modeling.layers import attention +from official.nlp.modeling.layers import dense_einsum + + +@tf.keras.utils.register_keras_serializable(package="Text") +@gin.configurable +class TransformerScaffold(tf.keras.layers.Layer): + """Transformer scaffold layer. + + This layer implements the Transformer from "Attention Is All You Need". + (https://arxiv.org/abs/1706.03762), with a customizable attention layer + option. Users can pass a class to `attention_cls` and associated config to + `attention_cfg`, in which case the scaffold will instantiate the class with + the config, or pass a class instance to `attention_cls`. + + Arguments: + num_attention_heads: Number of attention heads. + intermediate_size: Size of the intermediate layer. + intermediate_activation: Activation for the intermediate layer. + attention_cls: A class to instantate, or a layer instance. + attention_cfg: The config with which to instantiate `attention_cls`. Ignored + if attention_cls is a layer instance. + dropout_rate: Dropout probability for the post-attention and output dropout. + attention_dropout_rate: Dropout probability for within the attention layer. + kernel_initializer: Initializer for dense layer kernels. + bias_initializer: Initializer for dense layer biases. + kernel_regularizer: Regularizer for dense layer kernels. + bias_regularizer: Regularizer for dense layer biases. + activity_regularizer: Regularizer for dense layer activity. + kernel_constraint: Constraint for dense layer kernels. + bias_constraint: Constraint for dense layer kernels. + """ + + def __init__(self, + num_attention_heads, + intermediate_size, + intermediate_activation, + attention_cls=attention.MultiHeadAttention, + attention_cfg=None, + dropout_rate=0.0, + attention_dropout_rate=0.0, + kernel_initializer="glorot_uniform", + bias_initializer="zeros", + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + **kwargs): + super(TransformerScaffold, self).__init__(**kwargs) + + self._attention_cfg = attention_cfg + self._attention_cls = attention_cls + self._num_heads = num_attention_heads + self._intermediate_size = intermediate_size + self._intermediate_activation = intermediate_activation + self._attention_dropout_rate = attention_dropout_rate + self._dropout_rate = dropout_rate + self._kernel_initializer = tf.keras.initializers.get(kernel_initializer) + self._bias_initializer = tf.keras.initializers.get(bias_initializer) + self._kernel_regularizer = tf.keras.regularizers.get(kernel_regularizer) + self._bias_regularizer = tf.keras.regularizers.get(bias_regularizer) + self._kernel_constraint = tf.keras.constraints.get(kernel_constraint) + self._bias_constraint = tf.keras.constraints.get(bias_constraint) + + def build(self, input_shape): + input_tensor = input_shape[0] if len(input_shape) == 2 else input_shape + input_tensor_shape = tf.TensorShape(input_tensor) + if len(input_tensor_shape) != 3: + raise ValueError( + "TransformerScaffold expects a three-dimensional input of " + "shape [batch, sequence, width].") + batch_size, sequence_length, hidden_size = input_tensor_shape + + if len(input_shape) == 2: + mask_tensor_shape = tf.TensorShape(input_shape[1]) + expected_mask_tensor_shape = tf.TensorShape( + [batch_size, sequence_length, sequence_length]) + if not expected_mask_tensor_shape.is_compatible_with(mask_tensor_shape): + raise ValueError("When passing a mask tensor to TransformerLayer, the " + "mask tensor must be of shape [batch, " + "sequence_length, sequence_length] (here %s). Got a " + "mask tensor of shape %s." % + (expected_mask_tensor_shape, mask_tensor_shape)) + if hidden_size % self._num_heads != 0: + raise ValueError( + "The input size (%d) is not a multiple of the number of attention " + "heads (%d)" % (hidden_size, self._num_heads)) + self._attention_head_size = int(hidden_size // self._num_heads) + + if isinstance(self._attention_cls, tf.keras.layers.Layer): + self._attention_layer = self._attention_cls + else: + if self._attention_cfg is None: + attention_cfg = { + "num_heads": self._num_heads, + "head_size": self._attention_head_size, + "dropout_rate": self._attention_dropout_rate, + "kernel_initializer": self._kernel_initializer, + "bias_initializer": self._bias_initializer, + "kernel_regularizer": self._kernel_regularizer, + "bias_regularizer": self._bias_regularizer, + "activity_regularizer": self._activity_regularizer, + "kernel_constraint": self._kernel_constraint, + "bias_constraint": self._bias_constraint, + "name": "self_attention" + } + else: + attention_cfg = self._attention_cfg + self._attention_layer = self._attention_cls(**attention_cfg) + + self._attention_output_dense = dense_einsum.DenseEinsum( + output_shape=hidden_size, + num_summed_dimensions=2, + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + activity_regularizer=self._activity_regularizer, + kernel_constraint=self._kernel_constraint, + bias_constraint=self._bias_constraint, + name="self_attention_output") + self._attention_dropout = tf.keras.layers.Dropout(rate=self._dropout_rate) + # Use float32 in layernorm for numeric stability. + # It is probably safe in mixed_float16, but we haven't validated this yet. + self._attention_layer_norm = ( + tf.keras.layers.LayerNormalization( + name="self_attention_layer_norm", axis=-1, epsilon=1e-12, + dtype=tf.float32)) + self._intermediate_dense = dense_einsum.DenseEinsum( + output_shape=self._intermediate_size, + activation=self._intermediate_activation, + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + activity_regularizer=self._activity_regularizer, + kernel_constraint=self._kernel_constraint, + bias_constraint=self._bias_constraint, + name="intermediate") + self._output_dense = dense_einsum.DenseEinsum( + output_shape=hidden_size, + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + activity_regularizer=self._activity_regularizer, + kernel_constraint=self._kernel_constraint, + bias_constraint=self._bias_constraint, + name="output") + self._output_dropout = tf.keras.layers.Dropout(rate=self._dropout_rate) + # Use float32 in layernorm for numeric stability. + self._output_layer_norm = tf.keras.layers.LayerNormalization( + name="output_layer_norm", axis=-1, epsilon=1e-12, dtype=tf.float32) + + super(TransformerScaffold, self).build(input_shape) + + def get_config(self): + config = { + "attention_cls": + self._attention_layer, + "num_attention_heads": + self._num_heads, + "intermediate_size": + self._intermediate_size, + "intermediate_activation": + self._intermediate_activation, + "dropout_rate": + self._dropout_rate, + "attention_dropout_rate": + self._attention_dropout_rate, + "kernel_initializer": + tf.keras.initializers.serialize(self._kernel_initializer), + "bias_initializer": + tf.keras.initializers.serialize(self._bias_initializer), + "kernel_regularizer": + tf.keras.regularizers.serialize(self._kernel_regularizer), + "bias_regularizer": + tf.keras.regularizers.serialize(self._bias_regularizer), + "activity_regularizer": + tf.keras.regularizers.serialize(self._activity_regularizer), + "kernel_constraint": + tf.keras.constraints.serialize(self._kernel_constraint), + "bias_constraint": + tf.keras.constraints.serialize(self._bias_constraint) + } + base_config = super(TransformerScaffold, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + def call(self, inputs): + if isinstance(inputs, (list, tuple)) and len(inputs) == 2: + input_tensor, attention_mask = inputs + else: + input_tensor, attention_mask = (inputs, None) + + attention_inputs = [input_tensor, input_tensor] + + if attention_mask is not None: + attention_inputs.append(attention_mask) + + attention_output = self._attention_layer(attention_inputs) + attention_output = self._attention_output_dense(attention_output) + attention_output = self._attention_dropout(attention_output) + attention_output = self._attention_layer_norm(input_tensor + + attention_output) + intermediate_output = self._intermediate_dense(attention_output) + layer_output = self._output_dense(intermediate_output) + layer_output = self._output_dropout(layer_output) + # During mixed precision training, attention_output is from layer norm and + # is always fp32 for now. Cast layer_output to fp32 for the subsequent add. + layer_output = tf.cast(layer_output, tf.float32) + layer_output = self._output_layer_norm(layer_output + attention_output) + + return layer_output diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/transformer_scaffold_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/transformer_scaffold_test.py new file mode 100644 index 0000000..243514e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/transformer_scaffold_test.py @@ -0,0 +1,350 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Keras-based transformer block layer.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json + +import numpy as np +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.nlp.modeling.layers import attention +from official.nlp.modeling.layers import transformer_scaffold + + +# Test class that wraps a standard attention layer. If this layer is called +# at any point, the list passed to the config object will be filled with a +# boolean 'True'. We register this class as a Keras serializable so we can +# test serialization below. +@tf.keras.utils.register_keras_serializable(package='TestOnly') +class ValidatedAttentionLayer(attention.MultiHeadAttention): + + def __init__(self, call_list, **kwargs): + super(ValidatedAttentionLayer, self).__init__(**kwargs) + self.list = call_list + + def call(self, inputs): + self.list.append(True) + return super(ValidatedAttentionLayer, self).call(inputs) + + def get_config(self): + config = super(ValidatedAttentionLayer, self).get_config() + config['call_list'] = [] + return config + + +# This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It +# guarantees forward compatibility of this code for the V2 switchover. +@keras_parameterized.run_all_keras_modes +class TransformerLayerTest(keras_parameterized.TestCase): + + def tearDown(self): + super(TransformerLayerTest, self).tearDown() + tf.keras.mixed_precision.experimental.set_policy('float32') + + def test_layer_creation(self): + sequence_length = 21 + width = 80 + + call_list = [] + attention_layer_cfg = { + 'num_heads': 10, + 'head_size': 8, + 'call_list': call_list, + } + test_layer = transformer_scaffold.TransformerScaffold( + attention_cls=ValidatedAttentionLayer, + attention_cfg=attention_layer_cfg, + num_attention_heads=10, + intermediate_size=2048, + intermediate_activation='relu') + + # Create a 3-dimensional input (the first dimension is implicit). + data_tensor = tf.keras.Input(shape=(sequence_length, width)) + output_tensor = test_layer(data_tensor) + # The default output of a transformer layer should be the same as the input. + self.assertEqual(data_tensor.shape.as_list(), output_tensor.shape.as_list()) + + # If call_list[0] exists and is True, the passed layer class was + # instantiated from the given config properly. + self.assertNotEmpty(call_list) + self.assertTrue(call_list[0], "The passed layer class wasn't instantiated.") + + def test_layer_creation_with_mask(self): + sequence_length = 21 + width = 80 + + call_list = [] + attention_layer_cfg = { + 'num_heads': 10, + 'head_size': 8, + 'call_list': call_list, + } + test_layer = transformer_scaffold.TransformerScaffold( + attention_cls=ValidatedAttentionLayer, + attention_cfg=attention_layer_cfg, + num_attention_heads=10, + intermediate_size=2048, + intermediate_activation='relu') + + # Create a 3-dimensional input (the first dimension is implicit). + data_tensor = tf.keras.Input(shape=(sequence_length, width)) + # Create a 2-dimensional input (the first dimension is implicit). + mask_tensor = tf.keras.Input(shape=(sequence_length, sequence_length)) + output_tensor = test_layer([data_tensor, mask_tensor]) + # The default output of a transformer layer should be the same as the input. + self.assertEqual(data_tensor.shape.as_list(), output_tensor.shape.as_list()) + # If call_list[0] exists and is True, the passed layer class was + # instantiated from the given config properly. + self.assertNotEmpty(call_list) + self.assertTrue(call_list[0], "The passed layer class wasn't instantiated.") + + def test_layer_creation_with_incorrect_mask_fails(self): + sequence_length = 21 + width = 80 + + call_list = [] + attention_layer_cfg = { + 'num_heads': 10, + 'head_size': 8, + 'call_list': call_list, + } + test_layer = transformer_scaffold.TransformerScaffold( + attention_cls=ValidatedAttentionLayer, + attention_cfg=attention_layer_cfg, + num_attention_heads=10, + intermediate_size=2048, + intermediate_activation='relu') + + # Create a 3-dimensional input (the first dimension is implicit). + data_tensor = tf.keras.Input(shape=(sequence_length, width)) + # Create a 2-dimensional input (the first dimension is implicit). + mask_tensor = tf.keras.Input(shape=(sequence_length, sequence_length - 3)) + with self.assertRaisesRegex(ValueError, 'When passing a mask tensor.*'): + _ = test_layer([data_tensor, mask_tensor]) + + def test_layer_invocation(self): + sequence_length = 21 + width = 80 + + call_list = [] + attention_layer_cfg = { + 'num_heads': 10, + 'head_size': 8, + 'call_list': call_list, + } + test_layer = transformer_scaffold.TransformerScaffold( + attention_cls=ValidatedAttentionLayer, + attention_cfg=attention_layer_cfg, + num_attention_heads=10, + intermediate_size=2048, + intermediate_activation='relu') + + # Create a 3-dimensional input (the first dimension is implicit). + data_tensor = tf.keras.Input(shape=(sequence_length, width)) + output_tensor = test_layer(data_tensor) + + # Create a model from the test layer. + model = tf.keras.Model(data_tensor, output_tensor) + + # Invoke the model on test data. We can't validate the output data itself + # (the NN is too complex) but this will rule out structural runtime errors. + batch_size = 6 + input_data = 10 * np.random.random_sample( + (batch_size, sequence_length, width)) + _ = model.predict(input_data) + # If call_list[0] exists and is True, the passed layer class was + # instantiated from the given config properly. + self.assertNotEmpty(call_list) + self.assertTrue(call_list[0], "The passed layer class wasn't instantiated.") + + def test_layer_invocation_with_mask(self): + sequence_length = 21 + width = 80 + + call_list = [] + attention_layer_cfg = { + 'num_heads': 10, + 'head_size': 8, + 'call_list': call_list, + } + test_layer = transformer_scaffold.TransformerScaffold( + attention_cls=ValidatedAttentionLayer, + attention_cfg=attention_layer_cfg, + num_attention_heads=10, + intermediate_size=2048, + intermediate_activation='relu') + + # Create a 3-dimensional input (the first dimension is implicit). + data_tensor = tf.keras.Input(shape=(sequence_length, width)) + # Create a 2-dimensional input (the first dimension is implicit). + mask_tensor = tf.keras.Input(shape=(sequence_length, sequence_length)) + output_tensor = test_layer([data_tensor, mask_tensor]) + + # Create a model from the test layer. + model = tf.keras.Model([data_tensor, mask_tensor], output_tensor) + + # Invoke the model on test data. We can't validate the output data itself + # (the NN is too complex) but this will rule out structural runtime errors. + batch_size = 6 + input_data = 10 * np.random.random_sample( + (batch_size, sequence_length, width)) + # The attention mask should be of shape (batch, from_seq_len, to_seq_len), + # which here is (batch, sequence_length, sequence_length) + mask_data = np.random.randint( + 2, size=(batch_size, sequence_length, sequence_length)) + _ = model.predict([input_data, mask_data]) + # If call_list[0] exists and is True, the passed layer class was + # instantiated from the given config properly. + self.assertNotEmpty(call_list) + self.assertTrue(call_list[0], "The passed layer class wasn't instantiated.") + + def test_layer_invocation_with_float16_dtype(self): + tf.keras.mixed_precision.experimental.set_policy('mixed_float16') + sequence_length = 21 + width = 80 + + call_list = [] + attention_layer_cfg = { + 'num_heads': 10, + 'head_size': 8, + 'call_list': call_list, + } + test_layer = transformer_scaffold.TransformerScaffold( + attention_cls=ValidatedAttentionLayer, + attention_cfg=attention_layer_cfg, + num_attention_heads=10, + intermediate_size=2048, + intermediate_activation='relu') + + # Create a 3-dimensional input (the first dimension is implicit). + data_tensor = tf.keras.Input(shape=(sequence_length, width)) + # Create a 2-dimensional input (the first dimension is implicit). + mask_tensor = tf.keras.Input(shape=(sequence_length, sequence_length)) + output_tensor = test_layer([data_tensor, mask_tensor]) + + # Create a model from the test layer. + model = tf.keras.Model([data_tensor, mask_tensor], output_tensor) + + # Invoke the model on test data. We can't validate the output data itself + # (the NN is too complex) but this will rule out structural runtime errors. + batch_size = 6 + input_data = (10 * np.random.random_sample( + (batch_size, sequence_length, width))) + # The attention mask should be of shape (batch, from_seq_len, to_seq_len), + # which here is (batch, sequence_length, sequence_length) + mask_data = np.random.randint( + 2, size=(batch_size, sequence_length, sequence_length)) + _ = model.predict([input_data, mask_data]) + # If call_list[0] exists and is True, the passed layer class was + # instantiated from the given config properly. + self.assertNotEmpty(call_list) + self.assertTrue(call_list[0], "The passed layer class wasn't instantiated.") + + def test_transform_with_initializer(self): + sequence_length = 21 + width = 80 + + call_list = [] + attention_layer_cfg = { + 'num_heads': 10, + 'head_size': 8, + 'call_list': call_list, + } + test_layer = transformer_scaffold.TransformerScaffold( + attention_cls=ValidatedAttentionLayer, + attention_cfg=attention_layer_cfg, + num_attention_heads=10, + intermediate_size=2048, + intermediate_activation='relu', + kernel_initializer=tf.keras.initializers.TruncatedNormal(stddev=0.02)) + + # Create a 3-dimensional input (the first dimension is implicit). + data_tensor = tf.keras.Input(shape=(sequence_length, width)) + output = test_layer(data_tensor) + # The default output of a transformer layer should be the same as the input. + self.assertEqual(data_tensor.shape.as_list(), output.shape.as_list()) + # If call_list[0] exists and is True, the passed layer class was + # instantiated from the given config properly. + self.assertNotEmpty(call_list) + self.assertTrue(call_list[0]) + + def test_layer_restoration_from_config(self): + sequence_length = 21 + width = 80 + + call_list = [] + attention_layer_cfg = { + 'num_heads': 10, + 'head_size': 8, + 'call_list': call_list, + 'name': 'test_layer', + } + test_layer = transformer_scaffold.TransformerScaffold( + attention_cls=ValidatedAttentionLayer, + attention_cfg=attention_layer_cfg, + num_attention_heads=10, + intermediate_size=2048, + intermediate_activation='relu') + + # Create a 3-dimensional input (the first dimension is implicit). + data_tensor = tf.keras.Input(shape=(sequence_length, width)) + # Create a 2-dimensional input (the first dimension is implicit). + mask_tensor = tf.keras.Input(shape=(sequence_length, sequence_length)) + output_tensor = test_layer([data_tensor, mask_tensor]) + + # Create a model from the test layer. + model = tf.keras.Model([data_tensor, mask_tensor], output_tensor) + + # Invoke the model on test data. We can't validate the output data itself + # (the NN is too complex) but this will rule out structural runtime errors. + batch_size = 6 + input_data = 10 * np.random.random_sample( + (batch_size, sequence_length, width)) + # The attention mask should be of shape (batch, from_seq_len, to_seq_len), + # which here is (batch, sequence_length, sequence_length) + mask_data = np.random.randint( + 2, size=(batch_size, sequence_length, sequence_length)) + pre_serialization_output = model.predict([input_data, mask_data]) + + # Serialize the model config. Pass the serialized data through json to + # ensure that we can serialize this layer to disk. + serialized_data = json.dumps(model.get_config()) + post_string_serialized_data = json.loads(serialized_data) + + # Create a new model from the old config, and copy the weights. These models + # should have identical outputs. + new_model = tf.keras.Model.from_config(post_string_serialized_data) + new_model.set_weights(model.get_weights()) + output = new_model.predict([input_data, mask_data]) + + self.assertAllClose(pre_serialization_output, output) + + # If the layer was configured correctly, it should have a list attribute + # (since it should have the custom class and config passed to it). + new_model.summary() + new_call_list = new_model.get_layer( + name='transformer_scaffold')._attention_layer.list + self.assertNotEmpty(new_call_list) + self.assertTrue(new_call_list[0], + "The passed layer class wasn't instantiated.") + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/transformer_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/transformer_test.py new file mode 100644 index 0000000..120504d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/transformer_test.py @@ -0,0 +1,192 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Keras-based transformer block layer.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl.testing import parameterized +import numpy as np +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.nlp.modeling.layers import transformer + + +# This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It +# guarantees forward compatibility of this code for the V2 switchover. +@keras_parameterized.run_all_keras_modes +@parameterized.parameters(transformer.Transformer, + transformer.CompiledTransformer) +class TransformerLayerTest(keras_parameterized.TestCase): + + def tearDown(self): + super(TransformerLayerTest, self).tearDown() + tf.keras.mixed_precision.experimental.set_policy('float32') + + def test_layer_creation(self, transformer_cls): + test_layer = transformer_cls( + num_attention_heads=10, + intermediate_size=2048, + intermediate_activation='relu') + sequence_length = 21 + width = 80 + # Create a 3-dimensional input (the first dimension is implicit). + data_tensor = tf.keras.Input(shape=(sequence_length, width)) + output_tensor = test_layer(data_tensor) + # The default output of a transformer layer should be the same as the input. + self.assertEqual(data_tensor.shape.as_list(), output_tensor.shape.as_list()) + + def test_layer_creation_with_mask(self, transformer_cls): + test_layer = transformer_cls( + num_attention_heads=10, + intermediate_size=2048, + intermediate_activation='relu') + sequence_length = 21 + width = 80 + # Create a 3-dimensional input (the first dimension is implicit). + data_tensor = tf.keras.Input(shape=(sequence_length, width)) + # Create a 2-dimensional input (the first dimension is implicit). + mask_tensor = tf.keras.Input(shape=(sequence_length, sequence_length)) + output_tensor = test_layer([data_tensor, mask_tensor]) + # The default output of a transformer layer should be the same as the input. + self.assertEqual(data_tensor.shape.as_list(), output_tensor.shape.as_list()) + + def test_layer_creation_with_incorrect_mask_fails(self, transformer_cls): + test_layer = transformer_cls( + num_attention_heads=10, + intermediate_size=2048, + intermediate_activation='relu') + sequence_length = 21 + width = 80 + # Create a 3-dimensional input (the first dimension is implicit). + data_tensor = tf.keras.Input(shape=(sequence_length, width)) + # Create a 2-dimensional input (the first dimension is implicit). + mask_tensor = tf.keras.Input(shape=(sequence_length, sequence_length - 3)) + with self.assertRaisesRegex(ValueError, 'When passing a mask tensor.*'): + _ = test_layer([data_tensor, mask_tensor]) + + def test_layer_invocation(self, transformer_cls): + test_layer = transformer_cls( + num_attention_heads=10, + intermediate_size=2048, + intermediate_activation='relu') + sequence_length = 21 + width = 80 + # Create a 3-dimensional input (the first dimension is implicit). + data_tensor = tf.keras.Input(shape=(sequence_length, width)) + output_tensor = test_layer(data_tensor) + + # Create a model from the test layer. + model = tf.keras.Model(data_tensor, output_tensor) + + # Invoke the model on test data. We can't validate the output data itself + # (the NN is too complex) but this will rule out structural runtime errors. + batch_size = 6 + input_data = 10 * np.random.random_sample( + (batch_size, sequence_length, width)) + _ = model.predict(input_data) + + def test_layer_invocation_with_mask(self, transformer_cls): + test_layer = transformer_cls( + num_attention_heads=10, + intermediate_size=2048, + intermediate_activation='relu') + sequence_length = 21 + width = 80 + # Create a 3-dimensional input (the first dimension is implicit). + data_tensor = tf.keras.Input(shape=(sequence_length, width)) + # Create a 2-dimensional input (the first dimension is implicit). + mask_tensor = tf.keras.Input(shape=(sequence_length, sequence_length)) + output_tensor = test_layer([data_tensor, mask_tensor]) + + # Create a model from the test layer. + model = tf.keras.Model([data_tensor, mask_tensor], output_tensor) + + # Invoke the model on test data. We can't validate the output data itself + # (the NN is too complex) but this will rule out structural runtime errors. + batch_size = 6 + input_data = 10 * np.random.random_sample( + (batch_size, sequence_length, width)) + # The attention mask should be of shape (batch, from_seq_len, to_seq_len), + # which here is (batch, sequence_length, sequence_length) + mask_data = np.random.randint( + 2, size=(batch_size, sequence_length, sequence_length)) + _ = model.predict([input_data, mask_data]) + + def test_layer_invocation_with_float16_dtype(self, transformer_cls): + tf.keras.mixed_precision.experimental.set_policy('mixed_float16') + test_layer = transformer_cls( + num_attention_heads=10, + intermediate_size=2048, + intermediate_activation='relu') + sequence_length = 21 + width = 80 + # Create a 3-dimensional input (the first dimension is implicit). + data_tensor = tf.keras.Input(shape=(sequence_length, width)) + # Create a 2-dimensional input (the first dimension is implicit). + mask_tensor = tf.keras.Input(shape=(sequence_length, sequence_length)) + output_tensor = test_layer([data_tensor, mask_tensor]) + + # Create a model from the test layer. + model = tf.keras.Model([data_tensor, mask_tensor], output_tensor) + + # Invoke the model on test data. We can't validate the output data itself + # (the NN is too complex) but this will rule out structural runtime errors. + batch_size = 6 + input_data = (10 * np.random.random_sample( + (batch_size, sequence_length, width))) + # The attention mask should be of shape (batch, from_seq_len, to_seq_len), + # which here is (batch, sequence_length, sequence_length) + mask_data = np.random.randint( + 2, size=(batch_size, sequence_length, sequence_length)) + _ = model.predict([input_data, mask_data]) + + def test_transform_with_initializer(self, transformer_cls): + test_layer = transformer_cls( + num_attention_heads=10, + intermediate_size=2048, + intermediate_activation='relu', + kernel_initializer=tf.keras.initializers.TruncatedNormal(stddev=0.02)) + sequence_length = 21 + width = 80 + # Create a 3-dimensional input (the first dimension is implicit). + data_tensor = tf.keras.Input(shape=(sequence_length, width)) + output = test_layer(data_tensor) + # The default output of a transformer layer should be the same as the input. + self.assertEqual(data_tensor.shape.as_list(), output.shape.as_list()) + + def test_dynamic_layer_sequence(self, transformer_cls): + test_layer = transformer_cls( + num_attention_heads=10, + intermediate_size=2048, + intermediate_activation='relu', + kernel_initializer=tf.keras.initializers.TruncatedNormal(stddev=0.02)) + # Create a 3-dimensional input (the first dimension is implicit). + width = 30 + input_tensor = tf.keras.Input(shape=(None, width)) + output_tensor = test_layer(input_tensor) + model = tf.keras.Model(input_tensor, output_tensor) + + input_length = 17 + input_data = np.ones((1, input_length, width)) + output_data = model.predict(input_data) + + self.assertAllEqual([1, input_length, width], output_data.shape) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/util.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/util.py new file mode 100644 index 0000000..354f216 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/layers/util.py @@ -0,0 +1,51 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Keras-based transformer block layer.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import functools + +import tensorflow as tf + + +class TfFunctionIfEagerDecorator(object): + """Helper decorator function to optionally apply the @tf.function annotation.""" + + def __init__(self, **kwargs): + self.func_kwargs = kwargs + + def __call__(self, func): + + @functools.wraps(func) + def wrapped_func(*args): + # TODO(b/150147476, b/150024785): Fix tf.function in TF1 crash. + if not hasattr(tf.compat.v1, "executing_eagerly_outside_functions" + ) or tf.compat.v1.executing_eagerly_outside_functions(): + return tf.function(func=func, **self.func_kwargs)(*args) + return func(*args) + + # Cache the created function in self._call_impl. + if not hasattr(self, "_call_impl"): + self._call_impl = wrapped_func + return self._call_impl + + +def tf_function_if_eager(**kwargs): + """Applies the @tf.function decorator only if running in eager mode.""" + return TfFunctionIfEagerDecorator(**kwargs) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/losses/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/losses/README.md new file mode 100644 index 0000000..522150c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/losses/README.md @@ -0,0 +1,9 @@ +# Losses + +Losses contains common loss computation used in NLP tasks. + +* `weighted_sparse_categorical_crossentropy_loss` computes per-batch sparse +categorical crossentropy loss. + +* `weighted_sparse_categorical_crossentropy_per_example_loss` computes +per-example sparse categorical crossentropy loss. diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/losses/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/losses/__init__.py new file mode 100644 index 0000000..919bad3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/losses/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Activations package definition. Subject to change.""" +from official.nlp.modeling.losses.weighted_sparse_categorical_crossentropy import loss as weighted_sparse_categorical_crossentropy_loss +from official.nlp.modeling.losses.weighted_sparse_categorical_crossentropy import per_example_loss as weighted_sparse_categorical_crossentropy_per_example_loss diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/losses/weighted_sparse_categorical_crossentropy.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/losses/weighted_sparse_categorical_crossentropy.py new file mode 100644 index 0000000..b88d8e3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/losses/weighted_sparse_categorical_crossentropy.py @@ -0,0 +1,106 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Sparse categorical cross-entropy losses.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow as tf + + +def _adjust_labels(labels, predictions): + """Adjust the 'labels' tensor by squeezing it if needed.""" + labels = tf.cast(labels, tf.int32) + if len(predictions.shape) == len(labels.shape): + labels = tf.squeeze(labels, [-1]) + return labels, predictions + + +def _validate_rank(labels, predictions, weights): + if weights is not None and len(weights.shape) != len(labels.shape): + raise RuntimeError( + ("Weight and label tensors were not of the same rank. weights.shape " + "was %s, and labels.shape was %s.") % + (predictions.shape, labels.shape)) + if (len(predictions.shape) - 1) != len(labels.shape): + raise RuntimeError( + ("Weighted sparse categorical crossentropy expects `labels` to have a " + "rank of one less than `predictions`. labels.shape was %s, and " + "predictions.shape was %s.") % (labels.shape, predictions.shape)) + + +def per_example_loss(labels, predictions, weights=None): + """Calculate a per-example sparse categorical crossentropy loss. + + This loss function assumes that the predictions are post-softmax. + Args: + labels: The labels to evaluate against. Should be a set of integer indices + ranging from 0 to (vocab_size-1). + predictions: The network predictions. Should have softmax already applied. + weights: An optional weight array of the same shape as the 'labels' array. + If None, all examples will be used. + + Returns: + A tensor of shape predictions.shape[:-1] containing the per-example + loss. + """ + # When using these functions with the Keras core API, we will need to squeeze + # the labels tensor - Keras adds a spurious inner dimension. + labels, predictions = _adjust_labels(labels, predictions) + _validate_rank(labels, predictions, weights) + + labels_one_hot = tf.one_hot(labels, predictions.shape[-1]) + labels_one_hot = tf.cast(labels_one_hot, predictions.dtype) + per_example_loss_data = -tf.reduce_sum( + predictions * labels_one_hot, axis=[-1]) + if weights is not None: + weights = tf.cast(weights, per_example_loss_data.dtype) + per_example_loss_data = weights * per_example_loss_data + return per_example_loss_data + + +def loss(labels, predictions, weights=None): + """Calculate a per-batch sparse categorical crossentropy loss. + + This loss function assumes that the predictions are post-softmax. + Args: + labels: The labels to evaluate against. Should be a set of integer indices + ranging from 0 to (vocab_size-1). + predictions: The network predictions. Should have softmax already applied. + weights: An optional weight array of the same shape as the 'labels' array. + If None, all examples will be used. + + Returns: + A loss scalar. + + Raises: + RuntimeError if the passed tensors do not have the same rank. + """ + # When using these functions with the Keras core API, we will need to squeeze + # the labels tensor - Keras adds a spurious inner dimension. + labels, predictions = _adjust_labels(labels, predictions) + _validate_rank(labels, predictions, weights) + + per_example_loss_data = per_example_loss(labels, predictions, weights) + + if weights is None: + return tf.reduce_mean(per_example_loss_data) + else: + numerator = tf.reduce_sum(per_example_loss_data) + weights = tf.cast(weights, predictions.dtype) + denominator = tf.reduce_sum(weights) + 1e-5 + return numerator / denominator diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/losses/weighted_sparse_categorical_crossentropy_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/losses/weighted_sparse_categorical_crossentropy_test.py new file mode 100644 index 0000000..deb4d12 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/losses/weighted_sparse_categorical_crossentropy_test.py @@ -0,0 +1,381 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for masked LM loss.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.nlp.modeling import networks +from official.nlp.modeling.losses import weighted_sparse_categorical_crossentropy + + +@keras_parameterized.run_all_keras_modes +class ClassificationLossTest(keras_parameterized.TestCase): + + def create_lm_model(self, + vocab_size, + sequence_length, + hidden_size, + num_predictions, + output="predictions"): + # First, create a transformer stack that we can use to get the LM's + # vocabulary weight. + xformer_stack = networks.TransformerEncoder( + vocab_size=vocab_size, + num_layers=1, + sequence_length=sequence_length, + hidden_size=hidden_size, + num_attention_heads=4, + ) + word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + lm_outputs, _ = xformer_stack([word_ids, mask, type_ids]) + + # Create a maskedLM from the transformer stack. + test_network = networks.MaskedLM( + num_predictions=num_predictions, + input_width=lm_outputs.shape[-1], + source_network=xformer_stack, + output=output) + + # Create a model from the masked LM layer. + lm_input_tensor = tf.keras.Input(shape=(sequence_length, hidden_size)) + masked_lm_positions = tf.keras.Input( + shape=(num_predictions,), dtype=tf.int32) + output = test_network([lm_input_tensor, masked_lm_positions]) + return tf.keras.Model([lm_input_tensor, masked_lm_positions], output) + + def create_classification_model(self, input_width, num_classes): + test_object = networks.Classification( + input_width=input_width, num_classes=num_classes) + # Create a 2-dimensional input (the first dimension is implicit). + pooled_data = tf.keras.Input(shape=(input_width,), dtype=tf.float32) + output = test_object(pooled_data) + return tf.keras.Model(pooled_data, output) + + def test_per_example_loss_3d_input(self): + """Test per-example loss with a 3-dimensional input, from a masked LM.""" + vocab_size = 100 + sequence_length = 32 + hidden_size = 64 + num_predictions = 21 + model = self.create_lm_model( + vocab_size=vocab_size, + sequence_length=sequence_length, + hidden_size=hidden_size, + num_predictions=num_predictions) + + # Get the output of the masked LM. + batch_size = 3 + lm_input_data = 10 * np.random.random_sample( + (batch_size, sequence_length, hidden_size)) + masked_position_data = np.random.randint( + 2, size=(batch_size, num_predictions)) + output_data = model.predict([lm_input_data, masked_position_data]) + + # Calculate per-example loss. + labels = np.random.randint(vocab_size, size=(batch_size, num_predictions)) + per_example_loss_data = weighted_sparse_categorical_crossentropy.per_example_loss( + predictions=output_data, labels=labels) + + # Per-example loss data should have one value per prediction, and those + # values shouldn't be zero in this case (as we're using random data). + expected_shape = [batch_size, num_predictions] + self.assertEqual(expected_shape, per_example_loss_data.shape.as_list()) + self.assertNotAllClose( + tf.zeros_like(per_example_loss_data), per_example_loss_data) + + def test_per_example_loss_2d_input(self): + """Test per-example loss with a 2-d input, from a classifier.""" + input_width = 512 + num_classes = 10 + model = self.create_classification_model(input_width, num_classes) + + # Invoke the network as part of a Model. + batch_size = 3 + input_data = 10 * np.random.random_sample((batch_size, input_width)) + output_data = model.predict(input_data) + + # Calculate per example loss. + labels = np.random.randint(num_classes, size=(batch_size)) + per_example_loss_data = weighted_sparse_categorical_crossentropy.per_example_loss( + predictions=output_data, labels=labels) + + # Per-example loss data should have one value per batch item, and those + # values shouldn't be zero in this case (as we're using random data). + self.assertEqual([batch_size], per_example_loss_data.shape.as_list()) + self.assertNotAllClose( + tf.zeros_like(per_example_loss_data), per_example_loss_data) + + def test_per_example_loss_weights_3d_input(self): + """Test weighted per-example loss with a 3-d input, from a masked LM.""" + vocab_size = 100 + sequence_length = 32 + hidden_size = 64 + num_predictions = 21 + model = self.create_lm_model( + vocab_size=vocab_size, + sequence_length=sequence_length, + hidden_size=hidden_size, + num_predictions=num_predictions) + + # Get the output of the masked LM. + batch_size = 3 + lm_input_data = 10 * np.random.random_sample( + (batch_size, sequence_length, hidden_size)) + masked_position_data = np.random.randint( + 2, size=(batch_size, num_predictions)) + output_data = model.predict([lm_input_data, masked_position_data]) + + # Calculate per-example loss with weights. + labels = np.random.randint(vocab_size, size=(batch_size, num_predictions)) + weights = np.random.randint(2, size=(batch_size, num_predictions)) + + per_example_loss_data = weighted_sparse_categorical_crossentropy.per_example_loss( + predictions=output_data, labels=labels, weights=weights) + + # Weighted per-example loss data should be equivalent to multiplying the + # loss tensor by the weights tensor. + expected_weighted_loss = per_example_loss_data * weights + self.assertAllClose(expected_weighted_loss, per_example_loss_data) + + def test_per_example_loss_weights_2d_input(self): + """Test weighted per-example loss with a 2-d input, from a classifier.""" + input_width = 512 + num_classes = 10 + model = self.create_classification_model(input_width, num_classes) + + # Invoke the network as part of a Model. + batch_size = 3 + input_data = 10 * np.random.random_sample((batch_size, input_width)) + output_data = model.predict(input_data) + + # Calculate per-example loss with weights. + labels = np.random.randint(num_classes, size=(batch_size)) + weights = np.random.randint(2, size=(batch_size)) + + per_example_loss_data = weighted_sparse_categorical_crossentropy.per_example_loss( + predictions=output_data, labels=labels, weights=weights) + + # Weighted per-example loss data should be equivalent to multiplying the + # loss tensor by the weights tensor. + expected_weighted_loss = per_example_loss_data * weights + self.assertAllClose(expected_weighted_loss, per_example_loss_data) + + def test_loss_3d_input(self): + """Test overall loss with a 3-dimensional input, from a masked LM.""" + vocab_size = 100 + sequence_length = 32 + hidden_size = 64 + num_predictions = 21 + model = self.create_lm_model( + vocab_size=vocab_size, + sequence_length=sequence_length, + hidden_size=hidden_size, + num_predictions=num_predictions) + + # Get the output of the masked LM. + batch_size = 3 + lm_input_data = 10 * np.random.random_sample( + (batch_size, sequence_length, hidden_size)) + masked_position_data = np.random.randint( + 2, size=(batch_size, num_predictions)) + output_data = model.predict([lm_input_data, masked_position_data]) + + # Calculate loss. + labels = np.random.randint(vocab_size, size=(batch_size, num_predictions)) + weights = np.random.randint(2, size=(batch_size, num_predictions)) + per_example_loss_data = weighted_sparse_categorical_crossentropy.loss( + predictions=output_data, labels=labels, weights=weights) + + # Total loss data should have one value, and that value shouldn't be zero + # in this case (as we're using random data). + expected_shape = [] # Scalar + self.assertEqual(expected_shape, per_example_loss_data.shape.as_list()) + self.assertNotAllClose( + tf.zeros_like(per_example_loss_data), per_example_loss_data) + + def test_loss_2d_input(self): + """Test overall loss with a 2-d input, from a classifier.""" + input_width = 512 + num_classes = 10 + model = self.create_classification_model(input_width, num_classes) + + # Invoke the network as part of a Model. + batch_size = 3 + input_data = 10 * np.random.random_sample((batch_size, input_width)) + output_data = model.predict(input_data) + + # Calculate per example loss. + labels = np.random.randint(num_classes, size=(batch_size)) + loss_data = weighted_sparse_categorical_crossentropy.loss( + predictions=output_data, labels=labels) + + # Loss data should have one value only, and that value shouldn't be zero in + # this case (as we're using random data). + self.assertNotAllClose(0, loss_data) + + def test_loss_weights_3d_input(self): + """Test masked loss with a 3-dimensional input, from a masked LM.""" + vocab_size = 100 + sequence_length = 32 + hidden_size = 64 + num_predictions = 21 + model = self.create_lm_model( + vocab_size=vocab_size, + sequence_length=sequence_length, + hidden_size=hidden_size, + num_predictions=num_predictions) + + # Get the output of the masked LM. + batch_size = 3 + lm_input_data = 10 * np.random.random_sample( + (batch_size, sequence_length, hidden_size)) + masked_position_data = np.random.randint( + 2, size=(batch_size, num_predictions)) + output_data = model.predict([lm_input_data, masked_position_data]) + + # Calculate a fully masked weight tensor. This should give a loss of zero. + labels = np.random.randint(vocab_size, size=(batch_size, num_predictions)) + null_weights = np.zeros((batch_size, num_predictions)) + weighted_loss_data = weighted_sparse_categorical_crossentropy.loss( + predictions=output_data, labels=labels, weights=null_weights) + + # Because the tensor is fully masked, the loss should be 0. + self.assertAllClose(0, weighted_loss_data) + + def test_loss_weights_2d_input(self): + """Test masked loss with a 2-d input, from a classifier.""" + input_width = 512 + num_classes = 10 + model = self.create_classification_model(input_width, num_classes) + + # Invoke the network as part of a Model. + batch_size = 3 + input_data = 10 * np.random.random_sample((batch_size, input_width)) + output_data = model.predict(input_data) + + # Calculate a fully masked weight tensor. This should give a loss of zero. + labels = np.random.randint(num_classes, size=(batch_size)) + null_weights = np.zeros((batch_size)) + weighted_loss_data = weighted_sparse_categorical_crossentropy.loss( + predictions=output_data, labels=labels, weights=null_weights) + + # Because the tensor is fully masked, the loss should be 0. + self.assertAllClose(0, weighted_loss_data) + + def test_mismatched_predictions_and_labels_ranks_squeezes(self): + """Test that the loss asserts when rank(predictions)-1 != rank(labels).""" + batch_size = 3 + output_data = np.random.random_sample((batch_size, 10)) + labels = np.random.randint(10, size=(batch_size, 1)) + + # All that this test tests is that the squeeze is successful. + _ = weighted_sparse_categorical_crossentropy.per_example_loss( + predictions=output_data, labels=labels) + + def test_mismatched_weights_and_labels_ranks_fail(self): + """Test that the loss asserts when rank(predictions) != rank(labels).""" + batch_size = 3 + output_data = np.random.random_sample((batch_size, 10, 15)) + labels = np.random.randint(10, size=(batch_size, 10)) + weights = np.random.randint(2, size=(batch_size)) + + with self.assertRaisesRegex(RuntimeError, ".*of the same rank.*"): + _ = weighted_sparse_categorical_crossentropy.per_example_loss( + predictions=output_data, labels=labels, weights=weights) + with self.assertRaisesRegex(RuntimeError, ".*of the same rank.*"): + _ = weighted_sparse_categorical_crossentropy.loss( + predictions=output_data, labels=labels, weights=weights) + + def test_tf_tensor_inputs(self): + """Test that tf.Tensors can be used as inputs to the loss function.""" + batch_size = 3 + output_data = tf.convert_to_tensor( + np.random.random_sample((batch_size, 10, 15))) + labels = tf.convert_to_tensor(np.random.randint(10, size=(batch_size, 10))) + weights = tf.convert_to_tensor(np.random.randint(2, size=(batch_size, 10))) + + # We're not trying to validate numerical correctness, just ensure that + # we can in fact pass tensors to these functions without causing runtime + # errors from the shape checking code. + _ = weighted_sparse_categorical_crossentropy.per_example_loss( + predictions=output_data, labels=labels, weights=weights) + _ = weighted_sparse_categorical_crossentropy.loss( + predictions=output_data, labels=labels, weights=weights) + + def test_legacy_lm_loss_compatibility(self): + """Test to validate computational correctness during refactors.""" + # This is the empirical output of a masked LM with the following parameters: + # batch_size = 3 + # vocab_size = 5 + # sequence_length = 4 + # num_predictions = 2 + output_data = np.array( + [[[-2.5286622, -1.0963473, -1.4925185, -2.4451098, -1.2923571], + [-2.7117882, -1.1205841, -4.02187, -0.9966936, -1.5119683]], + [[-2.5379114, -0.82479054, -2.287932, -1.3747153, -2.053741], + [-2.5379114, -0.82479054, -2.287932, -1.3747153, -2.053741]], + [[-2.7760355, -1.8219438, -3.0924666, -1.0779881, -0.9407509], + [-2.7760355, -1.8219438, -3.0924666, -1.0779881, -0.9407509]]]) + labels = np.array([[4, 0], [2, 2], [2, 1]]) + + # Validate that per_example loss calculations are the same. + per_example_loss_data = weighted_sparse_categorical_crossentropy.per_example_loss( + predictions=output_data, labels=labels) + expected_per_example_loss_data = [[1.2923571, 2.7117882], + [2.287932, 2.287932], + [3.0924666, 1.8219438]] + self.assertAllClose(expected_per_example_loss_data, per_example_loss_data) + + # Validate that overall loss calculations are the same. + weights = np.array([[1, 0], [0, 0], [0, 0]]) + loss_data = weighted_sparse_categorical_crossentropy.loss( + predictions=output_data, labels=labels, weights=weights) + expected_loss_data = 1.2923441 + self.assertAllClose(expected_loss_data, loss_data) + + def test_legacy_classification_loss_compatibility(self): + """Test to validate computational correctness during refactors.""" + # This is the empirical output of a classifier with the following params: + # batch_size = 2 + # num_classes = 3 + output_data = np.array([[-1.6094601e-03, -1.0966038e+01, -6.4434357e+00], + [-1.6975292e-03, -6.4009643e+00, -1.0226612e+01]]) + labels = np.array([2, 1]) + + # Validate that per_example loss calculations are the same. + per_example_loss_data = weighted_sparse_categorical_crossentropy.per_example_loss( + predictions=output_data, labels=labels) + expected_per_example_loss_data = [6.4434357, 6.4009643] + self.assertAllClose(expected_per_example_loss_data, per_example_loss_data) + + # Validate that overall loss calculations are the same. + weights = None + loss_data = weighted_sparse_categorical_crossentropy.loss( + predictions=output_data, labels=labels, weights=weights) + expected_loss_data = 6.4222 + self.assertAllClose(expected_loss_data, loss_data) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/README.md new file mode 100644 index 0000000..99c3307 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/README.md @@ -0,0 +1,17 @@ +# Models + +Models are combinations of layers and networks that would be trained. + +Several pre-built canned models are provided to train encoder networks. These +models are intended as both convenience functions and canonical examples. + +* [`BertClassifier`](bert_classifier.py) implements a simple classification +model containing a single classification head using the Classification network. + +* [`BertSpanLabeler`](bert_span_labeler.py) implementats a simple single-span +start-end predictor (that is, a model that predicts two values: a start token +index and an end token index), suitable for SQuAD-style tasks. + +* [`BertPretrainer`](bert_pretrainer.py) implements a masked LM and a +classification head using the Masked LM and Classification networks, +respectively. diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/__init__.py new file mode 100644 index 0000000..dfcbc1d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Models package definition.""" +from official.nlp.modeling.models.bert_classifier import BertClassifier +from official.nlp.modeling.models.bert_pretrainer import BertPretrainer +from official.nlp.modeling.models.bert_span_labeler import BertSpanLabeler diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_classifier.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_classifier.py new file mode 100644 index 0000000..de8158c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_classifier.py @@ -0,0 +1,91 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Trainer network for BERT-style models.""" +# pylint: disable=g-classes-have-attributes +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow as tf + +from official.nlp.modeling import networks + + +@tf.keras.utils.register_keras_serializable(package='Text') +class BertClassifier(tf.keras.Model): + """Classifier model based on a BERT-style transformer-based encoder. + + This is an implementation of the network structure surrounding a transformer + encoder as described in "BERT: Pre-training of Deep Bidirectional Transformers + for Language Understanding" (https://arxiv.org/abs/1810.04805). + + The BertClassifier allows a user to pass in a transformer stack, and + instantiates a classification network based on the passed `num_classes` + argument. + + Arguments: + network: A transformer network. This network should output a sequence output + and a classification output. Furthermore, it should expose its embedding + table via a "get_embedding_table" method. + num_classes: Number of classes to predict from the classification network. + initializer: The initializer (if any) to use in the classification networks. + Defaults to a Glorot uniform initializer. + output: The output style for this network. Can be either 'logits' or + 'predictions'. + """ + + def __init__(self, + network, + num_classes, + initializer='glorot_uniform', + output='logits', + dropout_rate=0.1, + **kwargs): + self._self_setattr_tracking = False + self._config = { + 'network': network, + 'num_classes': num_classes, + 'initializer': initializer, + 'output': output, + } + + # We want to use the inputs of the passed network as the inputs to this + # Model. To do this, we need to keep a handle to the network inputs for use + # when we construct the Model object at the end of init. + inputs = network.inputs + + # Because we have a copy of inputs to create this Model object, we can + # invoke the Network object with its own input tensors to start the Model. + _, cls_output = network(inputs) + cls_output = tf.keras.layers.Dropout(rate=dropout_rate)(cls_output) + + self.classifier = networks.Classification( + input_width=cls_output.shape[-1], + num_classes=num_classes, + initializer=initializer, + output=output, + name='classification') + predictions = self.classifier(cls_output) + + super(BertClassifier, self).__init__( + inputs=inputs, outputs=predictions, **kwargs) + + def get_config(self): + return self._config + + @classmethod + def from_config(cls, config, custom_objects=None): + return cls(**config) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_classifier_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_classifier_test.py new file mode 100644 index 0000000..085c2f7 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_classifier_test.py @@ -0,0 +1,105 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for BERT trainer network.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.nlp.modeling import networks +from official.nlp.modeling.models import bert_classifier + + +# This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It +# guarantees forward compatibility of this code for the V2 switchover. +@keras_parameterized.run_all_keras_modes +class BertClassifierTest(keras_parameterized.TestCase): + + def test_bert_trainer(self): + """Validate that the Keras object can be created.""" + # Build a transformer network to use within the BERT trainer. + vocab_size = 100 + sequence_length = 512 + test_network = networks.TransformerEncoder( + vocab_size=vocab_size, num_layers=2, sequence_length=sequence_length) + + # Create a BERT trainer with the created network. + num_classes = 3 + bert_trainer_model = bert_classifier.BertClassifier( + test_network, + num_classes=num_classes) + + # Create a set of 2-dimensional inputs (the first dimension is implicit). + word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + + # Invoke the trainer model on the inputs. This causes the layer to be built. + cls_outs = bert_trainer_model([word_ids, mask, type_ids]) + + # Validate that the outputs are of the expected shape. + expected_classification_shape = [None, num_classes] + self.assertAllEqual(expected_classification_shape, cls_outs.shape.as_list()) + + def test_bert_trainer_tensor_call(self): + """Validate that the Keras object can be invoked.""" + # Build a transformer network to use within the BERT trainer. (Here, we use + # a short sequence_length for convenience.) + test_network = networks.TransformerEncoder( + vocab_size=100, num_layers=2, sequence_length=2) + + # Create a BERT trainer with the created network. + bert_trainer_model = bert_classifier.BertClassifier( + test_network, num_classes=2) + + # Create a set of 2-dimensional data tensors to feed into the model. + word_ids = tf.constant([[1, 1], [2, 2]], dtype=tf.int32) + mask = tf.constant([[1, 1], [1, 0]], dtype=tf.int32) + type_ids = tf.constant([[1, 1], [2, 2]], dtype=tf.int32) + + # Invoke the trainer model on the tensors. In Eager mode, this does the + # actual calculation. (We can't validate the outputs, since the network is + # too complex: this simply ensures we're not hitting runtime errors.) + _ = bert_trainer_model([word_ids, mask, type_ids]) + + def test_serialize_deserialize(self): + """Validate that the BERT trainer can be serialized and deserialized.""" + # Build a transformer network to use within the BERT trainer. (Here, we use + # a short sequence_length for convenience.) + test_network = networks.TransformerEncoder( + vocab_size=100, num_layers=2, sequence_length=5) + + # Create a BERT trainer with the created network. (Note that all the args + # are different, so we can catch any serialization mismatches.) + bert_trainer_model = bert_classifier.BertClassifier( + test_network, num_classes=4, initializer='zeros', output='predictions') + + # Create another BERT trainer via serialization and deserialization. + config = bert_trainer_model.get_config() + new_bert_trainer_model = bert_classifier.BertClassifier.from_config(config) + + # Validate that the config can be forced to JSON. + _ = new_bert_trainer_model.to_json() + + # If the serialization was successful, the new config should match the old. + self.assertAllEqual(bert_trainer_model.get_config(), + new_bert_trainer_model.get_config()) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_pretrainer.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_pretrainer.py new file mode 100644 index 0000000..1db8609 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_pretrainer.py @@ -0,0 +1,125 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Trainer network for BERT-style models.""" +# pylint: disable=g-classes-have-attributes +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import copy +import tensorflow as tf + +from official.nlp.modeling import networks + + +@tf.keras.utils.register_keras_serializable(package='Text') +class BertPretrainer(tf.keras.Model): + """BERT network training model. + + This is an implementation of the network structure surrounding a transformer + encoder as described in "BERT: Pre-training of Deep Bidirectional Transformers + for Language Understanding" (https://arxiv.org/abs/1810.04805). + + The BertPretrainer allows a user to pass in a transformer stack, and + instantiates the masked language model and classification networks that are + used to create the training objectives. + + Arguments: + network: A transformer network. This network should output a sequence output + and a classification output. Furthermore, it should expose its embedding + table via a "get_embedding_table" method. + num_classes: Number of classes to predict from the classification network. + num_token_predictions: Number of tokens to predict from the masked LM. + activation: The activation (if any) to use in the masked LM and + classification networks. If None, no activation will be used. + initializer: The initializer (if any) to use in the masked LM and + classification networks. Defaults to a Glorot uniform initializer. + output: The output style for this network. Can be either 'logits' or + 'predictions'. + """ + + def __init__(self, + network, + num_classes, + num_token_predictions, + activation=None, + initializer='glorot_uniform', + output='logits', + **kwargs): + self._self_setattr_tracking = False + self._config = { + 'network': network, + 'num_classes': num_classes, + 'num_token_predictions': num_token_predictions, + 'activation': activation, + 'initializer': initializer, + 'output': output, + } + + # We want to use the inputs of the passed network as the inputs to this + # Model. To do this, we need to keep a copy of the network inputs for use + # when we construct the Model object at the end of init. (We keep a copy + # because we'll be adding another tensor to the copy later.) + network_inputs = network.inputs + inputs = copy.copy(network_inputs) + + # Because we have a copy of inputs to create this Model object, we can + # invoke the Network object with its own input tensors to start the Model. + # Note that, because of how deferred construction happens, we can't use + # the copy of the list here - by the time the network is invoked, the list + # object contains the additional input added below. + sequence_output, cls_output = network(network_inputs) + + sequence_output_length = sequence_output.shape.as_list()[1] + if sequence_output_length < num_token_predictions: + raise ValueError( + "The passed network's output length is %s, which is less than the " + 'requested num_token_predictions %s.' % + (sequence_output_length, num_token_predictions)) + + masked_lm_positions = tf.keras.layers.Input( + shape=(num_token_predictions,), + name='masked_lm_positions', + dtype=tf.int32) + inputs.append(masked_lm_positions) + + self.masked_lm = networks.MaskedLM( + num_predictions=num_token_predictions, + input_width=sequence_output.shape[-1], + source_network=network, + activation=activation, + initializer=initializer, + output=output, + name='masked_lm') + lm_outputs = self.masked_lm([sequence_output, masked_lm_positions]) + + self.classification = networks.Classification( + input_width=cls_output.shape[-1], + num_classes=num_classes, + initializer=initializer, + output=output, + name='classification') + sentence_outputs = self.classification(cls_output) + + super(BertPretrainer, self).__init__( + inputs=inputs, outputs=[lm_outputs, sentence_outputs], **kwargs) + + def get_config(self): + return self._config + + @classmethod + def from_config(cls, config, custom_objects=None): + return cls(**config) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_pretrainer_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_pretrainer_test.py new file mode 100644 index 0000000..587c2b0 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_pretrainer_test.py @@ -0,0 +1,111 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for BERT trainer network.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.nlp.modeling import networks +from official.nlp.modeling.models import bert_pretrainer + + +# This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It +# guarantees forward compatibility of this code for the V2 switchover. +@keras_parameterized.run_all_keras_modes +class BertPretrainerTest(keras_parameterized.TestCase): + + def test_bert_trainer(self): + """Validate that the Keras object can be created.""" + # Build a transformer network to use within the BERT trainer. + vocab_size = 100 + sequence_length = 512 + test_network = networks.TransformerEncoder( + vocab_size=vocab_size, num_layers=2, sequence_length=sequence_length) + + # Create a BERT trainer with the created network. + num_classes = 3 + num_token_predictions = 2 + bert_trainer_model = bert_pretrainer.BertPretrainer( + test_network, + num_classes=num_classes, + num_token_predictions=num_token_predictions) + + # Create a set of 2-dimensional inputs (the first dimension is implicit). + word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + lm_mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + + # Invoke the trainer model on the inputs. This causes the layer to be built. + lm_outs, cls_outs = bert_trainer_model([word_ids, mask, type_ids, lm_mask]) + + # Validate that the outputs are of the expected shape. + expected_lm_shape = [None, num_token_predictions, vocab_size] + expected_classification_shape = [None, num_classes] + self.assertAllEqual(expected_lm_shape, lm_outs.shape.as_list()) + self.assertAllEqual(expected_classification_shape, cls_outs.shape.as_list()) + + def test_bert_trainer_tensor_call(self): + """Validate that the Keras object can be invoked.""" + # Build a transformer network to use within the BERT trainer. (Here, we use + # a short sequence_length for convenience.) + test_network = networks.TransformerEncoder( + vocab_size=100, num_layers=2, sequence_length=2) + + # Create a BERT trainer with the created network. + bert_trainer_model = bert_pretrainer.BertPretrainer( + test_network, num_classes=2, num_token_predictions=2) + + # Create a set of 2-dimensional data tensors to feed into the model. + word_ids = tf.constant([[1, 1], [2, 2]], dtype=tf.int32) + mask = tf.constant([[1, 1], [1, 0]], dtype=tf.int32) + type_ids = tf.constant([[1, 1], [2, 2]], dtype=tf.int32) + lm_mask = tf.constant([[1, 1], [1, 0]], dtype=tf.int32) + + # Invoke the trainer model on the tensors. In Eager mode, this does the + # actual calculation. (We can't validate the outputs, since the network is + # too complex: this simply ensures we're not hitting runtime errors.) + _, _ = bert_trainer_model([word_ids, mask, type_ids, lm_mask]) + + def test_serialize_deserialize(self): + """Validate that the BERT trainer can be serialized and deserialized.""" + # Build a transformer network to use within the BERT trainer. (Here, we use + # a short sequence_length for convenience.) + test_network = networks.TransformerEncoder( + vocab_size=100, num_layers=2, sequence_length=5) + + # Create a BERT trainer with the created network. (Note that all the args + # are different, so we can catch any serialization mismatches.) + bert_trainer_model = bert_pretrainer.BertPretrainer( + test_network, num_classes=4, num_token_predictions=3) + + # Create another BERT trainer via serialization and deserialization. + config = bert_trainer_model.get_config() + new_bert_trainer_model = bert_pretrainer.BertPretrainer.from_config(config) + + # Validate that the config can be forced to JSON. + _ = new_bert_trainer_model.to_json() + + # If the serialization was successful, the new config should match the old. + self.assertAllEqual(bert_trainer_model.get_config(), + new_bert_trainer_model.get_config()) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_span_labeler.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_span_labeler.py new file mode 100644 index 0000000..9cc8d62 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_span_labeler.py @@ -0,0 +1,97 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Trainer network for BERT-style models.""" +# pylint: disable=g-classes-have-attributes +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow as tf + +from official.nlp.modeling import networks + + +@tf.keras.utils.register_keras_serializable(package='Text') +class BertSpanLabeler(tf.keras.Model): + """Span labeler model based on a BERT-style transformer-based encoder. + + This is an implementation of the network structure surrounding a transformer + encoder as described in "BERT: Pre-training of Deep Bidirectional Transformers + for Language Understanding" (https://arxiv.org/abs/1810.04805). + + The BertSpanLabeler allows a user to pass in a transformer stack, and + instantiates a span labeling network based on a single dense layer. + + Arguments: + network: A transformer network. This network should output a sequence output + and a classification output. Furthermore, it should expose its embedding + table via a "get_embedding_table" method. + initializer: The initializer (if any) to use in the span labeling network. + Defaults to a Glorot uniform initializer. + output: The output style for this network. Can be either 'logits' or + 'predictions'. + """ + + def __init__(self, + network, + initializer='glorot_uniform', + output='logits', + **kwargs): + self._self_setattr_tracking = False + self._config = { + 'network': network, + 'initializer': initializer, + 'output': output, + } + # We want to use the inputs of the passed network as the inputs to this + # Model. To do this, we need to keep a handle to the network inputs for use + # when we construct the Model object at the end of init. + inputs = network.inputs + + # Because we have a copy of inputs to create this Model object, we can + # invoke the Network object with its own input tensors to start the Model. + sequence_output, _ = network(inputs) + + # This is an instance variable for ease of access to the underlying task + # network. + self.span_labeling = networks.SpanLabeling( + input_width=sequence_output.shape[-1], + initializer=initializer, + output=output, + name='span_labeling') + start_logits, end_logits = self.span_labeling(sequence_output) + + # Use identity layers wrapped in lambdas to explicitly name the output + # tensors. This allows us to use string-keyed dicts in Keras fit/predict/ + # evaluate calls. + start_logits = tf.keras.layers.Lambda( + tf.identity, name='start_positions')( + start_logits) + end_logits = tf.keras.layers.Lambda( + tf.identity, name='end_positions')( + end_logits) + + logits = [start_logits, end_logits] + + super(BertSpanLabeler, self).__init__( + inputs=inputs, outputs=logits, **kwargs) + + def get_config(self): + return self._config + + @classmethod + def from_config(cls, config, custom_objects=None): + return cls(**config) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_span_labeler_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_span_labeler_test.py new file mode 100644 index 0000000..d05e91b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/models/bert_span_labeler_test.py @@ -0,0 +1,124 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for BERT trainer network.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.nlp.modeling import networks +from official.nlp.modeling.models import bert_span_labeler + + +# This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It +# guarantees forward compatibility of this code for the V2 switchover. +@keras_parameterized.run_all_keras_modes +class BertSpanLabelerTest(keras_parameterized.TestCase): + + def test_bert_trainer(self): + """Validate that the Keras object can be created.""" + # Build a transformer network to use within the BERT trainer. + vocab_size = 100 + sequence_length = 512 + test_network = networks.TransformerEncoder( + vocab_size=vocab_size, num_layers=2, sequence_length=sequence_length) + + # Create a BERT trainer with the created network. + bert_trainer_model = bert_span_labeler.BertSpanLabeler(test_network) + + # Create a set of 2-dimensional inputs (the first dimension is implicit). + word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + + # Invoke the trainer model on the inputs. This causes the layer to be built. + cls_outs = bert_trainer_model([word_ids, mask, type_ids]) + + # Validate that there are 2 outputs are of the expected shape. + self.assertEqual(2, len(cls_outs)) + expected_shape = [None, sequence_length] + for out in cls_outs: + self.assertAllEqual(expected_shape, out.shape.as_list()) + + def test_bert_trainer_named_compilation(self): + """Validate compilation using explicit output names.""" + # Build a transformer network to use within the BERT trainer. + vocab_size = 100 + sequence_length = 512 + test_network = networks.TransformerEncoder( + vocab_size=vocab_size, num_layers=2, sequence_length=sequence_length) + + # Create a BERT trainer with the created network. + bert_trainer_model = bert_span_labeler.BertSpanLabeler(test_network) + + # Attempt to compile the model using a string-keyed dict of output names to + # loss functions. This will validate that the outputs are named as we + # expect. + bert_trainer_model.compile( + optimizer='sgd', + loss={ + 'start_positions': 'mse', + 'end_positions': 'mse' + }) + + def test_bert_trainer_tensor_call(self): + """Validate that the Keras object can be invoked.""" + # Build a transformer network to use within the BERT trainer. (Here, we use + # a short sequence_length for convenience.) + test_network = networks.TransformerEncoder( + vocab_size=100, num_layers=2, sequence_length=2) + + # Create a BERT trainer with the created network. + bert_trainer_model = bert_span_labeler.BertSpanLabeler(test_network) + + # Create a set of 2-dimensional data tensors to feed into the model. + word_ids = tf.constant([[1, 1], [2, 2]], dtype=tf.int32) + mask = tf.constant([[1, 1], [1, 0]], dtype=tf.int32) + type_ids = tf.constant([[1, 1], [2, 2]], dtype=tf.int32) + + # Invoke the trainer model on the tensors. In Eager mode, this does the + # actual calculation. (We can't validate the outputs, since the network is + # too complex: this simply ensures we're not hitting runtime errors.) + _ = bert_trainer_model([word_ids, mask, type_ids]) + + def test_serialize_deserialize(self): + """Validate that the BERT trainer can be serialized and deserialized.""" + # Build a transformer network to use within the BERT trainer. (Here, we use + # a short sequence_length for convenience.) + test_network = networks.TransformerEncoder( + vocab_size=100, num_layers=2, sequence_length=5) + + # Create a BERT trainer with the created network. (Note that all the args + # are different, so we can catch any serialization mismatches.) + bert_trainer_model = bert_span_labeler.BertSpanLabeler(test_network) + + # Create another BERT trainer via serialization and deserialization. + config = bert_trainer_model.get_config() + new_bert_trainer_model = bert_span_labeler.BertSpanLabeler.from_config( + config) + + # Validate that the config can be forced to JSON. + _ = new_bert_trainer_model.to_json() + + # If the serialization was successful, the new config should match the old. + self.assertAllEqual(bert_trainer_model.get_config(), + new_bert_trainer_model.get_config()) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/README.md new file mode 100644 index 0000000..be91b79 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/README.md @@ -0,0 +1,24 @@ +# Networks + +Networks are combinations of layers (and possibly other networks). They are sub-units of models that would not be trained alone. It +encapsulates common network structures like a classification head +or a transformer encoder into an easily handled object with a +standardized configuration. + +* [`TransformerEncoder`](transformer_encoder.py) implements a bi-directional +Transformer-based encoder as described in ["BERT: Pre-training of Deep +Bidirectional Transformers for Language Understanding"](https://arxiv.org/abs/1810.04805). It includes the embedding lookups, +transformer layers and pooling layer. + +* [`AlbertTransformerEncoder`](albert_transformer_encoder.py) implements a +Transformer-encoder described in the paper ["ALBERT: A Lite BERT for +Self-supervised Learning of Language Representations] +(https://arxiv.org/abs/1909.11942). Compared with [BERT](https://arxiv.org/abs/1810.04805), ALBERT refactorizes embedding parameters +into two smaller matrices and shares parameters across layers. + +* [`MaskedLM`](masked_lm.py) implements a masked language model for BERT pretraining. It assumes that the network being passed has a `get_embedding_table()` method. + +* [`Classification`](classification.py) contains a single hidden layer, and is intended for use as a classification head. + +* [`SpanLabeling`](span_labeling.py) implements a single-span labeler (that is, a prediction head that can predict one start and end index per batch item) based on a single dense hidden layer. It can be used in the SQuAD task. + diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/__init__.py new file mode 100644 index 0000000..c4a22b5 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Networks package definition.""" +from official.nlp.modeling.networks.albert_transformer_encoder import AlbertTransformerEncoder +from official.nlp.modeling.networks.classification import Classification +from official.nlp.modeling.networks.encoder_scaffold import EncoderScaffold +from official.nlp.modeling.networks.masked_lm import MaskedLM +from official.nlp.modeling.networks.span_labeling import SpanLabeling +from official.nlp.modeling.networks.transformer_encoder import TransformerEncoder diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/albert_transformer_encoder.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/albert_transformer_encoder.py new file mode 100644 index 0000000..437f562 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/albert_transformer_encoder.py @@ -0,0 +1,191 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""ALBERT (https://arxiv.org/abs/1810.04805) text encoder network.""" +# pylint: disable=g-classes-have-attributes +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow as tf + +from tensorflow.python.keras.engine import network # pylint: disable=g-direct-tensorflow-import +from official.modeling import activations +from official.nlp.modeling import layers + + +@tf.keras.utils.register_keras_serializable(package='Text') +class AlbertTransformerEncoder(network.Network): + """ALBERT (https://arxiv.org/abs/1810.04805) text encoder network. + + This network implements the encoder described in the paper "ALBERT: A Lite + BERT for Self-supervised Learning of Language Representations" + (https://arxiv.org/abs/1909.11942). + + Compared with BERT (https://arxiv.org/abs/1810.04805), ALBERT refactorizes + embedding parameters into two smaller matrices and shares parameters + across layers. + + The default values for this object are taken from the ALBERT-Base + implementation described in the paper. + + Arguments: + vocab_size: The size of the token vocabulary. + embedding_width: The width of the word embeddings. If the embedding width + is not equal to hidden size, embedding parameters will be factorized into + two matrices in the shape of ['vocab_size', 'embedding_width'] and + ['embedding_width', 'hidden_size'] ('embedding_width' is usually much + smaller than 'hidden_size'). + hidden_size: The size of the transformer hidden layers. + num_layers: The number of transformer layers. + num_attention_heads: The number of attention heads for each transformer. The + hidden size must be divisible by the number of attention heads. + sequence_length: The sequence length that this encoder expects. If None, the + sequence length is dynamic; if an integer, the encoder will require + sequences padded to this length. + max_sequence_length: The maximum sequence length that this encoder can + consume. If None, max_sequence_length uses the value from sequence length. + This determines the variable shape for positional embeddings. + type_vocab_size: The number of types that the 'type_ids' input can take. + intermediate_size: The intermediate size for the transformer layers. + activation: The activation to use for the transformer layers. + dropout_rate: The dropout rate to use for the transformer layers. + attention_dropout_rate: The dropout rate to use for the attention layers + within the transformer layers. + initializer: The initialzer to use for all weights in this encoder. + """ + + def __init__(self, + vocab_size, + embedding_width=128, + hidden_size=768, + num_layers=12, + num_attention_heads=12, + sequence_length=512, + max_sequence_length=None, + type_vocab_size=16, + intermediate_size=3072, + activation=activations.gelu, + dropout_rate=0.1, + attention_dropout_rate=0.1, + initializer=tf.keras.initializers.TruncatedNormal(stddev=0.02), + **kwargs): + activation = tf.keras.activations.get(activation) + initializer = tf.keras.initializers.get(initializer) + + if not max_sequence_length: + max_sequence_length = sequence_length + self._self_setattr_tracking = False + self._config_dict = { + 'vocab_size': vocab_size, + 'embedding_width': embedding_width, + 'hidden_size': hidden_size, + 'num_layers': num_layers, + 'num_attention_heads': num_attention_heads, + 'sequence_length': sequence_length, + 'max_sequence_length': max_sequence_length, + 'type_vocab_size': type_vocab_size, + 'intermediate_size': intermediate_size, + 'activation': tf.keras.activations.serialize(activation), + 'dropout_rate': dropout_rate, + 'attention_dropout_rate': attention_dropout_rate, + 'initializer': tf.keras.initializers.serialize(initializer), + } + + word_ids = tf.keras.layers.Input( + shape=(sequence_length,), dtype=tf.int32, name='input_word_ids') + mask = tf.keras.layers.Input( + shape=(sequence_length,), dtype=tf.int32, name='input_mask') + type_ids = tf.keras.layers.Input( + shape=(sequence_length,), dtype=tf.int32, name='input_type_ids') + + self._embedding_layer = layers.OnDeviceEmbedding( + vocab_size=vocab_size, + embedding_width=embedding_width, + initializer=initializer, + name='word_embeddings') + word_embeddings = self._embedding_layer(word_ids) + + # Always uses dynamic slicing for simplicity. + self._position_embedding_layer = layers.PositionEmbedding( + initializer=initializer, + use_dynamic_slicing=True, + max_sequence_length=max_sequence_length) + position_embeddings = self._position_embedding_layer(word_embeddings) + + type_embeddings = ( + layers.OnDeviceEmbedding( + vocab_size=type_vocab_size, + embedding_width=embedding_width, + initializer=initializer, + use_one_hot=True, + name='type_embeddings')(type_ids)) + + embeddings = tf.keras.layers.Add()( + [word_embeddings, position_embeddings, type_embeddings]) + embeddings = ( + tf.keras.layers.LayerNormalization( + name='embeddings/layer_norm', + axis=-1, + epsilon=1e-12, + dtype=tf.float32)(embeddings)) + embeddings = ( + tf.keras.layers.Dropout(rate=dropout_rate)(embeddings)) + # We project the 'embedding' output to 'hidden_size' if it is not already + # 'hidden_size'. + if embedding_width != hidden_size: + embeddings = layers.DenseEinsum( + output_shape=hidden_size, + kernel_initializer=initializer, + name='embedding_projection')( + embeddings) + + data = embeddings + attention_mask = layers.SelfAttentionMask()([data, mask]) + shared_layer = layers.Transformer( + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + intermediate_activation=activation, + dropout_rate=dropout_rate, + attention_dropout_rate=attention_dropout_rate, + kernel_initializer=initializer, + name='transformer') + for _ in range(num_layers): + data = shared_layer([data, attention_mask]) + + first_token_tensor = ( + tf.keras.layers.Lambda(lambda x: tf.squeeze(x[:, 0:1, :], axis=1))(data) + ) + cls_output = tf.keras.layers.Dense( + units=hidden_size, + activation='tanh', + kernel_initializer=initializer, + name='pooler_transform')( + first_token_tensor) + + super(AlbertTransformerEncoder, self).__init__( + inputs=[word_ids, mask, type_ids], + outputs=[data, cls_output], + **kwargs) + + def get_embedding_table(self): + return self._embedding_layer.embeddings + + def get_config(self): + return self._config_dict + + @classmethod + def from_config(cls, config): + return cls(**config) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/albert_transformer_encoder_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/albert_transformer_encoder_test.py new file mode 100644 index 0000000..44368e4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/albert_transformer_encoder_test.py @@ -0,0 +1,174 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for ALBERT transformer-based text encoder network.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl.testing import parameterized +import numpy as np +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.nlp.modeling.networks import albert_transformer_encoder + + +# This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It +# guarantees forward compatibility of this code for the V2 switchover. +@keras_parameterized.run_all_keras_modes +class AlbertTransformerEncoderTest(keras_parameterized.TestCase): + + def tearDown(self): + super(AlbertTransformerEncoderTest, self).tearDown() + tf.keras.mixed_precision.experimental.set_policy("float32") + + @parameterized.named_parameters( + dict(testcase_name="default", expected_dtype=tf.float32), + dict( + testcase_name="with_float16_dtype", + expected_dtype=tf.float16), + ) + def test_network_creation(self, expected_dtype): + hidden_size = 32 + sequence_length = 21 + + kwargs = dict( + vocab_size=100, + hidden_size=hidden_size, + sequence_length=sequence_length, + num_attention_heads=2, + num_layers=3) + if expected_dtype == tf.float16: + tf.keras.mixed_precision.experimental.set_policy("mixed_float16") + + # Create a small TransformerEncoder for testing. + test_network = albert_transformer_encoder.AlbertTransformerEncoder(**kwargs) + + # Create the inputs (note that the first dimension is implicit). + word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + data, pooled = test_network([word_ids, mask, type_ids]) + + expected_data_shape = [None, sequence_length, hidden_size] + expected_pooled_shape = [None, hidden_size] + self.assertAllEqual(expected_data_shape, data.shape.as_list()) + self.assertAllEqual(expected_pooled_shape, pooled.shape.as_list()) + + # If float_dtype is set to float16, the data output is float32 (from a layer + # norm) and pool output should be float16. + self.assertEqual(tf.float32, data.dtype) + self.assertEqual(expected_dtype, pooled.dtype) + + # ALBERT has additonal 'embedding_hidden_mapping_in' weights and + # it shares transformer weights. + self.assertNotEmpty( + [x for x in test_network.weights if "embedding_projection/" in x.name]) + self.assertNotEmpty( + [x for x in test_network.weights if "transformer/" in x.name]) + self.assertEmpty( + [x for x in test_network.weights if "transformer/layer" in x.name]) + + def test_network_invocation(self): + hidden_size = 32 + sequence_length = 21 + vocab_size = 57 + num_types = 7 + # Create a small TransformerEncoder for testing. + test_network = albert_transformer_encoder.AlbertTransformerEncoder( + vocab_size=vocab_size, + embedding_width=8, + hidden_size=hidden_size, + sequence_length=sequence_length, + num_attention_heads=2, + num_layers=3, + type_vocab_size=num_types) + self.assertTrue( + test_network._position_embedding_layer._use_dynamic_slicing) + # Create the inputs (note that the first dimension is implicit). + word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + data, pooled = test_network([word_ids, mask, type_ids]) + + # Create a model based off of this network: + model = tf.keras.Model([word_ids, mask, type_ids], [data, pooled]) + + # Invoke the model. We can't validate the output data here (the model is too + # complex) but this will catch structural runtime errors. + batch_size = 3 + word_id_data = np.random.randint( + vocab_size, size=(batch_size, sequence_length)) + mask_data = np.random.randint(2, size=(batch_size, sequence_length)) + type_id_data = np.random.randint( + num_types, size=(batch_size, sequence_length)) + _ = model.predict([word_id_data, mask_data, type_id_data]) + + # Creates a TransformerEncoder with max_sequence_length != sequence_length + max_sequence_length = 128 + test_network = albert_transformer_encoder.AlbertTransformerEncoder( + vocab_size=vocab_size, + embedding_width=8, + hidden_size=hidden_size, + sequence_length=sequence_length, + max_sequence_length=max_sequence_length, + num_attention_heads=2, + num_layers=3, + type_vocab_size=num_types) + self.assertTrue(test_network._position_embedding_layer._use_dynamic_slicing) + model = tf.keras.Model([word_ids, mask, type_ids], [data, pooled]) + _ = model.predict([word_id_data, mask_data, type_id_data]) + + def test_serialize_deserialize(self): + tf.keras.mixed_precision.experimental.set_policy("mixed_float16") + # Create a network object that sets all of its config options. + kwargs = dict( + vocab_size=100, + embedding_width=8, + hidden_size=32, + num_layers=3, + num_attention_heads=2, + sequence_length=21, + max_sequence_length=21, + type_vocab_size=12, + intermediate_size=1223, + activation="relu", + dropout_rate=0.05, + attention_dropout_rate=0.22, + initializer="glorot_uniform") + network = albert_transformer_encoder.AlbertTransformerEncoder(**kwargs) + + expected_config = dict(kwargs) + expected_config["activation"] = tf.keras.activations.serialize( + tf.keras.activations.get(expected_config["activation"])) + expected_config["initializer"] = tf.keras.initializers.serialize( + tf.keras.initializers.get(expected_config["initializer"])) + self.assertEqual(network.get_config(), expected_config) + + # Create another network object from the first object's config. + new_network = ( + albert_transformer_encoder.AlbertTransformerEncoder.from_config( + network.get_config())) + + # Validate that the config can be forced to JSON. + _ = new_network.to_json() + + # If the serialization was successful, the new config should match the old. + self.assertAllEqual(network.get_config(), new_network.get_config()) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/classification.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/classification.py new file mode 100644 index 0000000..d3263e8 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/classification.py @@ -0,0 +1,86 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Classification network.""" +# pylint: disable=g-classes-have-attributes +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow as tf + +# pylint: disable=g-direct-tensorflow-import +from tensorflow.python.keras.engine import network + + +@tf.keras.utils.register_keras_serializable(package='Text') +class Classification(network.Network): + """Classification network head for BERT modeling. + + This network implements a simple classifier head based on a dense layer. + + Arguments: + input_width: The innermost dimension of the input tensor to this network. + num_classes: The number of classes that this network should classify to. + activation: The activation, if any, for the dense layer in this network. + initializer: The intializer for the dense layer in this network. Defaults to + a Glorot uniform initializer. + output: The output style for this network. Can be either 'logits' or + 'predictions'. + """ + + def __init__(self, + input_width, + num_classes, + initializer='glorot_uniform', + output='logits', + **kwargs): + self._self_setattr_tracking = False + self._config_dict = { + 'input_width': input_width, + 'num_classes': num_classes, + 'initializer': initializer, + 'output': output, + } + + cls_output = tf.keras.layers.Input( + shape=(input_width,), name='cls_output', dtype=tf.float32) + + self.logits = tf.keras.layers.Dense( + num_classes, + activation=None, + kernel_initializer=initializer, + name='predictions/transform/logits')( + cls_output) + predictions = tf.keras.layers.Activation(tf.nn.log_softmax)(self.logits) + + if output == 'logits': + output_tensors = self.logits + elif output == 'predictions': + output_tensors = predictions + else: + raise ValueError( + ('Unknown `output` value "%s". `output` can be either "logits" or ' + '"predictions"') % output) + + super(Classification, self).__init__( + inputs=[cls_output], outputs=output_tensors, **kwargs) + + def get_config(self): + return self._config_dict + + @classmethod + def from_config(cls, config, custom_objects=None): + return cls(**config) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/classification_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/classification_test.py new file mode 100644 index 0000000..6f71074 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/classification_test.py @@ -0,0 +1,179 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for classification network.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.nlp.modeling.networks import classification + + +# This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It +# guarantees forward compatibility of this code for the V2 switchover. +@keras_parameterized.run_all_keras_modes +class ClassificationTest(keras_parameterized.TestCase): + + def test_network_creation(self): + """Validate that the Keras object can be created.""" + input_width = 512 + num_classes = 10 + test_object = classification.Classification( + input_width=input_width, num_classes=num_classes) + # Create a 2-dimensional input (the first dimension is implicit). + cls_data = tf.keras.Input(shape=(input_width,), dtype=tf.float32) + output = test_object(cls_data) + + # Validate that the outputs are of the expected shape. + expected_output_shape = [None, num_classes] + self.assertEqual(expected_output_shape, output.shape.as_list()) + + def test_network_invocation(self): + """Validate that the Keras object can be invoked.""" + input_width = 512 + num_classes = 10 + test_object = classification.Classification( + input_width=input_width, num_classes=num_classes, output='predictions') + # Create a 2-dimensional input (the first dimension is implicit). + cls_data = tf.keras.Input(shape=(input_width,), dtype=tf.float32) + output = test_object(cls_data) + + # Invoke the network as part of a Model. + model = tf.keras.Model(cls_data, output) + input_data = 10 * np.random.random_sample((3, input_width)) + _ = model.predict(input_data) + + def test_network_invocation_with_internal_logits(self): + """Validate that the logit outputs are correct.""" + input_width = 512 + num_classes = 10 + test_object = classification.Classification( + input_width=input_width, num_classes=num_classes, output='predictions') + + # Create a 2-dimensional input (the first dimension is implicit). + cls_data = tf.keras.Input(shape=(input_width,), dtype=tf.float32) + output = test_object(cls_data) + model = tf.keras.Model(cls_data, output) + logits_model = tf.keras.Model(test_object.inputs, test_object.logits) + + batch_size = 3 + input_data = 10 * np.random.random_sample((batch_size, input_width)) + outputs = model.predict(input_data) + logits = logits_model.predict(input_data) + + # Ensure that the tensor shapes are correct. + expected_output_shape = (batch_size, num_classes) + self.assertEqual(expected_output_shape, outputs.shape) + self.assertEqual(expected_output_shape, logits.shape) + + # Ensure that the logits, when softmaxed, create the outputs. + input_tensor = tf.keras.Input(expected_output_shape[1:]) + output_tensor = tf.keras.layers.Activation(tf.nn.log_softmax)(input_tensor) + softmax_model = tf.keras.Model(input_tensor, output_tensor) + + calculated_softmax = softmax_model.predict(logits) + self.assertAllClose(outputs, calculated_softmax) + + def test_network_invocation_with_internal_and_external_logits(self): + """Validate that the logit outputs are correct.""" + input_width = 512 + num_classes = 10 + test_object = classification.Classification( + input_width=input_width, num_classes=num_classes, output='logits') + + # Create a 2-dimensional input (the first dimension is implicit). + cls_data = tf.keras.Input(shape=(input_width,), dtype=tf.float32) + output = test_object(cls_data) + model = tf.keras.Model(cls_data, output) + logits_model = tf.keras.Model(test_object.inputs, test_object.logits) + + batch_size = 3 + input_data = 10 * np.random.random_sample((batch_size, input_width)) + outputs = model.predict(input_data) + logits = logits_model.predict(input_data) + + # Ensure that the tensor shapes are correct. + expected_output_shape = (batch_size, num_classes) + self.assertEqual(expected_output_shape, outputs.shape) + self.assertEqual(expected_output_shape, logits.shape) + + self.assertAllClose(outputs, logits) + + def test_network_invocation_with_logit_output(self): + """Validate that the logit outputs are correct.""" + input_width = 512 + num_classes = 10 + test_object = classification.Classification( + input_width=input_width, num_classes=num_classes, output='predictions') + logit_object = classification.Classification( + input_width=input_width, num_classes=num_classes, output='logits') + logit_object.set_weights(test_object.get_weights()) + + # Create a 2-dimensional input (the first dimension is implicit). + cls_data = tf.keras.Input(shape=(input_width,), dtype=tf.float32) + output = test_object(cls_data) + logit_output = logit_object(cls_data) + + model = tf.keras.Model(cls_data, output) + logits_model = tf.keras.Model(cls_data, logit_output) + + batch_size = 3 + input_data = 10 * np.random.random_sample((batch_size, input_width)) + outputs = model.predict(input_data) + logits = logits_model.predict(input_data) + + # Ensure that the tensor shapes are correct. + expected_output_shape = (batch_size, num_classes) + self.assertEqual(expected_output_shape, outputs.shape) + self.assertEqual(expected_output_shape, logits.shape) + + # Ensure that the logits, when softmaxed, create the outputs. + input_tensor = tf.keras.Input(expected_output_shape[1:]) + output_tensor = tf.keras.layers.Activation(tf.nn.log_softmax)(input_tensor) + softmax_model = tf.keras.Model(input_tensor, output_tensor) + + calculated_softmax = softmax_model.predict(logits) + self.assertAllClose(outputs, calculated_softmax) + + def test_serialize_deserialize(self): + # Create a network object that sets all of its config options. + network = classification.Classification( + input_width=128, + num_classes=10, + initializer='zeros', + output='predictions') + + # Create another network object from the first object's config. + new_network = classification.Classification.from_config( + network.get_config()) + + # Validate that the config can be forced to JSON. + _ = new_network.to_json() + + # If the serialization was successful, the new config should match the old. + self.assertAllEqual(network.get_config(), new_network.get_config()) + + def test_unknown_output_type_fails(self): + with self.assertRaisesRegex(ValueError, 'Unknown `output` value "bad".*'): + _ = classification.Classification( + input_width=128, num_classes=10, output='bad') + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/encoder_scaffold.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/encoder_scaffold.py new file mode 100644 index 0000000..12dd61e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/encoder_scaffold.py @@ -0,0 +1,245 @@ +# Lint as: python3 +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Transformer-based text encoder network.""" +# pylint: disable=g-classes-have-attributes +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import inspect + +import gin +import tensorflow as tf + +from tensorflow.python.keras.engine import network # pylint: disable=g-direct-tensorflow-import +from official.nlp.modeling import layers + + +@tf.keras.utils.register_keras_serializable(package='Text') +@gin.configurable +class EncoderScaffold(network.Network): + """Bi-directional Transformer-based encoder network scaffold. + + This network allows users to flexibly implement an encoder similar to the one + described in "BERT: Pre-training of Deep Bidirectional Transformers for + Language Understanding" (https://arxiv.org/abs/1810.04805). + + In this network, users can choose to provide a custom embedding subnetwork + (which will replace the standard embedding logic) and/or a custom hidden layer + class (which will replace the Transformer instantiation in the encoder). For + each of these custom injection points, users can pass either a class or a + class instance. If a class is passed, that class will be instantiated using + the 'embedding_cfg' or 'hidden_cfg' argument, respectively; if an instance + is passed, that instance will be invoked. (In the case of hidden_cls, the + instance will be invoked 'num_hidden_instances' times. + + If the hidden_cls is not overridden, a default transformer layer will be + instantiated. + + Arguments: + pooled_output_dim: The dimension of pooled output. + pooler_layer_initializer: The initializer for the classification + layer. + embedding_cls: The class or instance to use to embed the input data. This + class or instance defines the inputs to this encoder. If embedding_cls is + not set, a default embedding network (from the original BERT paper) will + be created. + embedding_cfg: A dict of kwargs to pass to the embedding_cls, if it needs to + be instantiated. If embedding_cls is not set, a config dict must be + passed to 'embedding_cfg' with the following values: + "vocab_size": The size of the token vocabulary. + "type_vocab_size": The size of the type vocabulary. + "hidden_size": The hidden size for this encoder. + "max_seq_length": The maximum sequence length for this encoder. + "seq_length": The sequence length for this encoder. + "initializer": The initializer for the embedding portion of this encoder. + "dropout_rate": The dropout rate to apply before the encoding layers. + embedding_data: A reference to the embedding weights that will be used to + train the masked language model, if necessary. This is optional, and only + needed if (1) you are overriding embedding_cls and (2) are doing standard + pretraining. + num_hidden_instances: The number of times to instantiate and/or invoke the + hidden_cls. + hidden_cls: The class or instance to encode the input data. If hidden_cls is + not set, a KerasBERT transformer layer will be used as the encoder class. + hidden_cfg: A dict of kwargs to pass to the hidden_cls, if it needs to be + instantiated. If hidden_cls is not set, a config dict must be passed to + 'hidden_cfg' with the following values: + "num_attention_heads": The number of attention heads. The hidden size + must be divisible by num_attention_heads. + "intermediate_size": The intermediate size of the transformer. + "intermediate_activation": The activation to apply in the transfomer. + "dropout_rate": The overall dropout rate for the transformer layers. + "attention_dropout_rate": The dropout rate for the attention layers. + "kernel_initializer": The initializer for the transformer layers. + """ + + def __init__( + self, + pooled_output_dim, + pooler_layer_initializer=tf.keras.initializers.TruncatedNormal( + stddev=0.02), + embedding_cls=None, + embedding_cfg=None, + embedding_data=None, + num_hidden_instances=1, + hidden_cls=layers.Transformer, + hidden_cfg=None, + **kwargs): + self._self_setattr_tracking = False + self._hidden_cls = hidden_cls + self._hidden_cfg = hidden_cfg + self._num_hidden_instances = num_hidden_instances + self._pooled_output_dim = pooled_output_dim + self._pooler_layer_initializer = pooler_layer_initializer + self._embedding_cls = embedding_cls + self._embedding_cfg = embedding_cfg + self._embedding_data = embedding_data + self._kwargs = kwargs + + if embedding_cls: + if inspect.isclass(embedding_cls): + self._embedding_network = embedding_cls(embedding_cfg) + else: + self._embedding_network = embedding_cls + inputs = self._embedding_network.inputs + embeddings, mask = self._embedding_network(inputs) + else: + self._embedding_network = None + word_ids = tf.keras.layers.Input( + shape=(embedding_cfg['seq_length'],), + dtype=tf.int32, + name='input_word_ids') + mask = tf.keras.layers.Input( + shape=(embedding_cfg['seq_length'],), + dtype=tf.int32, + name='input_mask') + type_ids = tf.keras.layers.Input( + shape=(embedding_cfg['seq_length'],), + dtype=tf.int32, + name='input_type_ids') + inputs = [word_ids, mask, type_ids] + + self._embedding_layer = layers.OnDeviceEmbedding( + vocab_size=embedding_cfg['vocab_size'], + embedding_width=embedding_cfg['hidden_size'], + initializer=embedding_cfg['initializer'], + name='word_embeddings') + + word_embeddings = self._embedding_layer(word_ids) + + # Always uses dynamic slicing for simplicity. + self._position_embedding_layer = layers.PositionEmbedding( + initializer=embedding_cfg['initializer'], + use_dynamic_slicing=True, + max_sequence_length=embedding_cfg['max_seq_length']) + position_embeddings = self._position_embedding_layer(word_embeddings) + + type_embeddings = ( + layers.OnDeviceEmbedding( + vocab_size=embedding_cfg['type_vocab_size'], + embedding_width=embedding_cfg['hidden_size'], + initializer=embedding_cfg['initializer'], + use_one_hot=True, + name='type_embeddings')(type_ids)) + + embeddings = tf.keras.layers.Add()( + [word_embeddings, position_embeddings, type_embeddings]) + embeddings = ( + tf.keras.layers.LayerNormalization( + name='embeddings/layer_norm', + axis=-1, + epsilon=1e-12, + dtype=tf.float32)(embeddings)) + embeddings = ( + tf.keras.layers.Dropout( + rate=embedding_cfg['dropout_rate'])(embeddings)) + + attention_mask = layers.SelfAttentionMask()([embeddings, mask]) + data = embeddings + + for _ in range(num_hidden_instances): + if inspect.isclass(hidden_cls): + layer = self._hidden_cls( + **hidden_cfg) if hidden_cfg else self._hidden_cls() + else: + layer = self._hidden_cls + data = layer([data, attention_mask]) + + first_token_tensor = ( + tf.keras.layers.Lambda(lambda x: tf.squeeze(x[:, 0:1, :], axis=1))(data) + ) + cls_output = tf.keras.layers.Dense( + units=pooled_output_dim, + activation='tanh', + kernel_initializer=pooler_layer_initializer, + name='cls_transform')( + first_token_tensor) + + super(EncoderScaffold, self).__init__( + inputs=inputs, outputs=[data, cls_output], **kwargs) + + def get_config(self): + config_dict = { + 'num_hidden_instances': + self._num_hidden_instances, + 'pooled_output_dim': + self._pooled_output_dim, + 'pooler_layer_initializer': + self._pooler_layer_initializer, + 'embedding_cls': + self._embedding_network, + 'embedding_cfg': + self._embedding_cfg, + 'hidden_cfg': + self._hidden_cfg, + } + if inspect.isclass(self._hidden_cls): + config_dict['hidden_cls_string'] = tf.keras.utils.get_registered_name( + self._hidden_cls) + else: + config_dict['hidden_cls'] = self._hidden_cls + + config_dict.update(self._kwargs) + return config_dict + + @classmethod + def from_config(cls, config, custom_objects=None): + if 'hidden_cls_string' in config: + config['hidden_cls'] = tf.keras.utils.get_registered_object( + config['hidden_cls_string'], custom_objects=custom_objects) + del config['hidden_cls_string'] + return cls(**config) + + def get_embedding_table(self): + if self._embedding_network is None: + # In this case, we don't have a custom embedding network and can return + # the standard embedding data. + return self._embedding_layer.embeddings + + if self._embedding_data is None: + raise RuntimeError(('The EncoderScaffold %s does not have a reference ' + 'to the embedding data. This is required when you ' + 'pass a custom embedding network to the scaffold. ' + 'It is also possible that you are trying to get ' + 'embedding data from an embedding scaffold with a ' + 'custom embedding network where the scaffold has ' + 'been serialized and deserialized. Unfortunately, ' + 'accessing custom embedding references after ' + 'serialization is not yet supported.') % self.name) + else: + return self._embedding_data diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/encoder_scaffold_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/encoder_scaffold_test.py new file mode 100644 index 0000000..3042397 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/encoder_scaffold_test.py @@ -0,0 +1,629 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for transformer-based text encoder network.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.modeling import activations +from official.nlp.modeling import layers +from official.nlp.modeling.networks import encoder_scaffold + + +# Test class that wraps a standard transformer layer. If this layer is called +# at any point, the list passed to the config object will be filled with a +# boolean 'True'. We register this class as a Keras serializable so we can +# test serialization below. +@tf.keras.utils.register_keras_serializable(package="TestOnly") +class ValidatedTransformerLayer(layers.Transformer): + + def __init__(self, call_list, **kwargs): + super(ValidatedTransformerLayer, self).__init__(**kwargs) + self.list = call_list + + def call(self, inputs): + self.list.append(True) + return super(ValidatedTransformerLayer, self).call(inputs) + + def get_config(self): + config = super(ValidatedTransformerLayer, self).get_config() + config["call_list"] = [] + return config + + +# This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It +# guarantees forward compatibility of this code for the V2 switchover. +@keras_parameterized.run_all_keras_modes +class EncoderScaffoldLayerClassTest(keras_parameterized.TestCase): + + def tearDown(self): + super(EncoderScaffoldLayerClassTest, self).tearDown() + tf.keras.mixed_precision.experimental.set_policy("float32") + + def test_network_creation(self): + hidden_size = 32 + sequence_length = 21 + num_hidden_instances = 3 + embedding_cfg = { + "vocab_size": 100, + "type_vocab_size": 16, + "hidden_size": hidden_size, + "seq_length": sequence_length, + "max_seq_length": sequence_length, + "initializer": tf.keras.initializers.TruncatedNormal(stddev=0.02), + "dropout_rate": 0.1, + } + + call_list = [] + hidden_cfg = { + "num_attention_heads": + 2, + "intermediate_size": + 3072, + "intermediate_activation": + activations.gelu, + "dropout_rate": + 0.1, + "attention_dropout_rate": + 0.1, + "kernel_initializer": + tf.keras.initializers.TruncatedNormal(stddev=0.02), + "call_list": + call_list + } + # Create a small EncoderScaffold for testing. + test_network = encoder_scaffold.EncoderScaffold( + num_hidden_instances=num_hidden_instances, + pooled_output_dim=hidden_size, + pooler_layer_initializer=tf.keras.initializers.TruncatedNormal( + stddev=0.02), + hidden_cls=ValidatedTransformerLayer, + hidden_cfg=hidden_cfg, + embedding_cfg=embedding_cfg) + # Create the inputs (note that the first dimension is implicit). + word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + data, pooled = test_network([word_ids, mask, type_ids]) + + expected_data_shape = [None, sequence_length, hidden_size] + expected_pooled_shape = [None, hidden_size] + self.assertAllEqual(expected_data_shape, data.shape.as_list()) + self.assertAllEqual(expected_pooled_shape, pooled.shape.as_list()) + + # The default output dtype is float32. + self.assertAllEqual(tf.float32, data.dtype) + self.assertAllEqual(tf.float32, pooled.dtype) + + # If call_list[0] exists and is True, the passed layer class was + # instantiated from the given config properly. + self.assertNotEmpty(call_list) + self.assertTrue(call_list[0], "The passed layer class wasn't instantiated.") + + def test_network_creation_with_float16_dtype(self): + tf.keras.mixed_precision.experimental.set_policy("mixed_float16") + hidden_size = 32 + sequence_length = 21 + embedding_cfg = { + "vocab_size": 100, + "type_vocab_size": 16, + "hidden_size": hidden_size, + "seq_length": sequence_length, + "max_seq_length": sequence_length, + "initializer": tf.keras.initializers.TruncatedNormal(stddev=0.02), + "dropout_rate": 0.1, + } + hidden_cfg = { + "num_attention_heads": + 2, + "intermediate_size": + 3072, + "intermediate_activation": + activations.gelu, + "dropout_rate": + 0.1, + "attention_dropout_rate": + 0.1, + "kernel_initializer": + tf.keras.initializers.TruncatedNormal(stddev=0.02), + } + # Create a small EncoderScaffold for testing. + test_network = encoder_scaffold.EncoderScaffold( + num_hidden_instances=3, + pooled_output_dim=hidden_size, + pooler_layer_initializer=tf.keras.initializers.TruncatedNormal( + stddev=0.02), + hidden_cfg=hidden_cfg, + embedding_cfg=embedding_cfg) + # Create the inputs (note that the first dimension is implicit). + word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + data, pooled = test_network([word_ids, mask, type_ids]) + + expected_data_shape = [None, sequence_length, hidden_size] + expected_pooled_shape = [None, hidden_size] + self.assertAllEqual(expected_data_shape, data.shape.as_list()) + self.assertAllEqual(expected_pooled_shape, pooled.shape.as_list()) + + # If float_dtype is set to float16, the data output is float32 (from a layer + # norm) and pool output should be float16. + self.assertAllEqual(tf.float32, data.dtype) + self.assertAllEqual(tf.float16, pooled.dtype) + + def test_network_invocation(self): + hidden_size = 32 + sequence_length = 21 + vocab_size = 57 + num_types = 7 + embedding_cfg = { + "vocab_size": vocab_size, + "type_vocab_size": num_types, + "hidden_size": hidden_size, + "seq_length": sequence_length, + "max_seq_length": sequence_length, + "initializer": tf.keras.initializers.TruncatedNormal(stddev=0.02), + "dropout_rate": 0.1, + } + hidden_cfg = { + "num_attention_heads": + 2, + "intermediate_size": + 3072, + "intermediate_activation": + activations.gelu, + "dropout_rate": + 0.1, + "attention_dropout_rate": + 0.1, + "kernel_initializer": + tf.keras.initializers.TruncatedNormal(stddev=0.02), + } + print(hidden_cfg) + print(embedding_cfg) + # Create a small EncoderScaffold for testing. + test_network = encoder_scaffold.EncoderScaffold( + num_hidden_instances=3, + pooled_output_dim=hidden_size, + pooler_layer_initializer=tf.keras.initializers.TruncatedNormal( + stddev=0.02), + hidden_cfg=hidden_cfg, + embedding_cfg=embedding_cfg) + + # Create the inputs (note that the first dimension is implicit). + word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + data, pooled = test_network([word_ids, mask, type_ids]) + + # Create a model based off of this network: + model = tf.keras.Model([word_ids, mask, type_ids], [data, pooled]) + + # Invoke the model. We can't validate the output data here (the model is too + # complex) but this will catch structural runtime errors. + batch_size = 3 + word_id_data = np.random.randint( + vocab_size, size=(batch_size, sequence_length)) + mask_data = np.random.randint(2, size=(batch_size, sequence_length)) + type_id_data = np.random.randint( + num_types, size=(batch_size, sequence_length)) + _ = model.predict([word_id_data, mask_data, type_id_data]) + + # Creates a EncoderScaffold with max_sequence_length != sequence_length + num_types = 7 + embedding_cfg = { + "vocab_size": vocab_size, + "type_vocab_size": num_types, + "hidden_size": hidden_size, + "seq_length": sequence_length, + "max_seq_length": sequence_length * 2, + "initializer": tf.keras.initializers.TruncatedNormal(stddev=0.02), + "dropout_rate": 0.1, + } + hidden_cfg = { + "num_attention_heads": + 2, + "intermediate_size": + 3072, + "intermediate_activation": + activations.gelu, + "dropout_rate": + 0.1, + "attention_dropout_rate": + 0.1, + "kernel_initializer": + tf.keras.initializers.TruncatedNormal(stddev=0.02), + } + # Create a small EncoderScaffold for testing. + test_network = encoder_scaffold.EncoderScaffold( + num_hidden_instances=3, + pooled_output_dim=hidden_size, + pooler_layer_initializer=tf.keras.initializers.TruncatedNormal( + stddev=0.02), + hidden_cfg=hidden_cfg, + embedding_cfg=embedding_cfg) + + model = tf.keras.Model([word_ids, mask, type_ids], [data, pooled]) + _ = model.predict([word_id_data, mask_data, type_id_data]) + + def test_serialize_deserialize(self): + # Create a network object that sets all of its config options. + hidden_size = 32 + sequence_length = 21 + embedding_cfg = { + "vocab_size": 100, + "type_vocab_size": 16, + "hidden_size": hidden_size, + "seq_length": sequence_length, + "max_seq_length": sequence_length, + "initializer": tf.keras.initializers.TruncatedNormal(stddev=0.02), + "dropout_rate": 0.1, + } + hidden_cfg = { + "num_attention_heads": + 2, + "intermediate_size": + 3072, + "intermediate_activation": + activations.gelu, + "dropout_rate": + 0.1, + "attention_dropout_rate": + 0.1, + "kernel_initializer": + tf.keras.initializers.TruncatedNormal(stddev=0.02), + } + # Create a small EncoderScaffold for testing. + network = encoder_scaffold.EncoderScaffold( + num_hidden_instances=3, + pooled_output_dim=hidden_size, + pooler_layer_initializer=tf.keras.initializers.TruncatedNormal( + stddev=0.02), + hidden_cfg=hidden_cfg, + embedding_cfg=embedding_cfg) + + # Create another network object from the first object's config. + new_network = encoder_scaffold.EncoderScaffold.from_config( + network.get_config()) + + # Validate that the config can be forced to JSON. + _ = new_network.to_json() + + # If the serialization was successful, the new config should match the old. + self.assertAllEqual(network.get_config(), new_network.get_config()) + + +@keras_parameterized.run_all_keras_modes +class EncoderScaffoldEmbeddingNetworkTest(keras_parameterized.TestCase): + + def test_network_invocation(self): + hidden_size = 32 + sequence_length = 21 + vocab_size = 57 + + # Build an embedding network to swap in for the default network. This one + # will have 2 inputs (mask and word_ids) instead of 3, and won't use + # positional embeddings. + + word_ids = tf.keras.layers.Input( + shape=(sequence_length,), dtype=tf.int32, name="input_word_ids") + mask = tf.keras.layers.Input( + shape=(sequence_length,), dtype=tf.int32, name="input_mask") + embedding_layer = layers.OnDeviceEmbedding( + vocab_size=vocab_size, + embedding_width=hidden_size, + initializer=tf.keras.initializers.TruncatedNormal(stddev=0.02), + name="word_embeddings") + word_embeddings = embedding_layer(word_ids) + network = tf.keras.Model([word_ids, mask], [word_embeddings, mask]) + + hidden_cfg = { + "num_attention_heads": + 2, + "intermediate_size": + 3072, + "intermediate_activation": + activations.gelu, + "dropout_rate": + 0.1, + "attention_dropout_rate": + 0.1, + "kernel_initializer": + tf.keras.initializers.TruncatedNormal(stddev=0.02), + } + + # Create a small EncoderScaffold for testing. + test_network = encoder_scaffold.EncoderScaffold( + num_hidden_instances=3, + pooled_output_dim=hidden_size, + pooler_layer_initializer=tf.keras.initializers.TruncatedNormal( + stddev=0.02), + hidden_cfg=hidden_cfg, + embedding_cls=network, + embedding_data=embedding_layer.embeddings) + + # Create the inputs (note that the first dimension is implicit). + word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + data, pooled = test_network([word_ids, mask]) + + # Create a model based off of this network: + model = tf.keras.Model([word_ids, mask], [data, pooled]) + + # Invoke the model. We can't validate the output data here (the model is too + # complex) but this will catch structural runtime errors. + batch_size = 3 + word_id_data = np.random.randint( + vocab_size, size=(batch_size, sequence_length)) + mask_data = np.random.randint(2, size=(batch_size, sequence_length)) + _ = model.predict([word_id_data, mask_data]) + + # Test that we can get the embedding data that we passed to the object. This + # is necessary to support standard language model training. + self.assertIs(embedding_layer.embeddings, + test_network.get_embedding_table()) + + def test_serialize_deserialize(self): + hidden_size = 32 + sequence_length = 21 + vocab_size = 57 + + # Build an embedding network to swap in for the default network. This one + # will have 2 inputs (mask and word_ids) instead of 3, and won't use + # positional embeddings. + + word_ids = tf.keras.layers.Input( + shape=(sequence_length,), dtype=tf.int32, name="input_word_ids") + mask = tf.keras.layers.Input( + shape=(sequence_length,), dtype=tf.int32, name="input_mask") + embedding_layer = layers.OnDeviceEmbedding( + vocab_size=vocab_size, + embedding_width=hidden_size, + initializer=tf.keras.initializers.TruncatedNormal(stddev=0.02), + name="word_embeddings") + word_embeddings = embedding_layer(word_ids) + network = tf.keras.Model([word_ids, mask], [word_embeddings, mask]) + + hidden_cfg = { + "num_attention_heads": + 2, + "intermediate_size": + 3072, + "intermediate_activation": + activations.gelu, + "dropout_rate": + 0.1, + "attention_dropout_rate": + 0.1, + "kernel_initializer": + tf.keras.initializers.TruncatedNormal(stddev=0.02), + } + + # Create a small EncoderScaffold for testing. + test_network = encoder_scaffold.EncoderScaffold( + num_hidden_instances=3, + pooled_output_dim=hidden_size, + pooler_layer_initializer=tf.keras.initializers.TruncatedNormal( + stddev=0.02), + hidden_cfg=hidden_cfg, + embedding_cls=network, + embedding_data=embedding_layer.embeddings) + + # Create another network object from the first object's config. + new_network = encoder_scaffold.EncoderScaffold.from_config( + test_network.get_config()) + + # Validate that the config can be forced to JSON. + _ = new_network.to_json() + + # If the serialization was successful, the new config should match the old. + self.assertAllEqual(test_network.get_config(), new_network.get_config()) + + # Create a model based off of the old and new networks: + word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + + data, pooled = new_network([word_ids, mask]) + new_model = tf.keras.Model([word_ids, mask], [data, pooled]) + + data, pooled = test_network([word_ids, mask]) + model = tf.keras.Model([word_ids, mask], [data, pooled]) + + # Copy the weights between models. + new_model.set_weights(model.get_weights()) + + # Invoke the models. + batch_size = 3 + word_id_data = np.random.randint( + vocab_size, size=(batch_size, sequence_length)) + mask_data = np.random.randint(2, size=(batch_size, sequence_length)) + data, cls = model.predict([word_id_data, mask_data]) + new_data, new_cls = new_model.predict([word_id_data, mask_data]) + + # The output should be equal. + self.assertAllEqual(data, new_data) + self.assertAllEqual(cls, new_cls) + + # We should not be able to get a reference to the embedding data. + with self.assertRaisesRegex(RuntimeError, ".*does not have a reference.*"): + new_network.get_embedding_table() + + +@keras_parameterized.run_all_keras_modes +class EncoderScaffoldHiddenInstanceTest(keras_parameterized.TestCase): + + def test_network_invocation(self): + hidden_size = 32 + sequence_length = 21 + vocab_size = 57 + num_types = 7 + + embedding_cfg = { + "vocab_size": vocab_size, + "type_vocab_size": num_types, + "hidden_size": hidden_size, + "seq_length": sequence_length, + "max_seq_length": sequence_length, + "initializer": tf.keras.initializers.TruncatedNormal(stddev=0.02), + "dropout_rate": 0.1, + } + + call_list = [] + hidden_cfg = { + "num_attention_heads": + 2, + "intermediate_size": + 3072, + "intermediate_activation": + activations.gelu, + "dropout_rate": + 0.1, + "attention_dropout_rate": + 0.1, + "kernel_initializer": + tf.keras.initializers.TruncatedNormal(stddev=0.02), + "call_list": + call_list + } + # Create a small EncoderScaffold for testing. This time, we pass an already- + # instantiated layer object. + + xformer = ValidatedTransformerLayer(**hidden_cfg) + + test_network = encoder_scaffold.EncoderScaffold( + num_hidden_instances=3, + pooled_output_dim=hidden_size, + pooler_layer_initializer=tf.keras.initializers.TruncatedNormal( + stddev=0.02), + hidden_cls=xformer, + embedding_cfg=embedding_cfg) + + # Create the inputs (note that the first dimension is implicit). + word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + data, pooled = test_network([word_ids, mask, type_ids]) + + # Create a model based off of this network: + model = tf.keras.Model([word_ids, mask, type_ids], [data, pooled]) + + # Invoke the model. We can't validate the output data here (the model is too + # complex) but this will catch structural runtime errors. + batch_size = 3 + word_id_data = np.random.randint( + vocab_size, size=(batch_size, sequence_length)) + mask_data = np.random.randint(2, size=(batch_size, sequence_length)) + type_id_data = np.random.randint( + num_types, size=(batch_size, sequence_length)) + _ = model.predict([word_id_data, mask_data, type_id_data]) + + # If call_list[0] exists and is True, the passed layer class was + # called as part of the graph creation. + self.assertNotEmpty(call_list) + self.assertTrue(call_list[0], "The passed layer class wasn't instantiated.") + + def test_serialize_deserialize(self): + hidden_size = 32 + sequence_length = 21 + vocab_size = 57 + num_types = 7 + + embedding_cfg = { + "vocab_size": vocab_size, + "type_vocab_size": num_types, + "hidden_size": hidden_size, + "seq_length": sequence_length, + "max_seq_length": sequence_length, + "initializer": tf.keras.initializers.TruncatedNormal(stddev=0.02), + "dropout_rate": 0.1, + } + + call_list = [] + hidden_cfg = { + "num_attention_heads": + 2, + "intermediate_size": + 3072, + "intermediate_activation": + activations.gelu, + "dropout_rate": + 0.1, + "attention_dropout_rate": + 0.1, + "kernel_initializer": + tf.keras.initializers.TruncatedNormal(stddev=0.02), + "call_list": + call_list + } + # Create a small EncoderScaffold for testing. This time, we pass an already- + # instantiated layer object. + + xformer = ValidatedTransformerLayer(**hidden_cfg) + + test_network = encoder_scaffold.EncoderScaffold( + num_hidden_instances=3, + pooled_output_dim=hidden_size, + pooler_layer_initializer=tf.keras.initializers.TruncatedNormal( + stddev=0.02), + hidden_cls=xformer, + embedding_cfg=embedding_cfg) + + # Create another network object from the first object's config. + new_network = encoder_scaffold.EncoderScaffold.from_config( + test_network.get_config()) + + # Validate that the config can be forced to JSON. + _ = new_network.to_json() + + # If the serialization was successful, the new config should match the old. + self.assertAllEqual(test_network.get_config(), new_network.get_config()) + + # Create a model based off of the old and new networks: + word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + + data, pooled = new_network([word_ids, mask, type_ids]) + new_model = tf.keras.Model([word_ids, mask, type_ids], [data, pooled]) + + data, pooled = test_network([word_ids, mask, type_ids]) + model = tf.keras.Model([word_ids, mask, type_ids], [data, pooled]) + + # Copy the weights between models. + new_model.set_weights(model.get_weights()) + + # Invoke the models. + batch_size = 3 + word_id_data = np.random.randint( + vocab_size, size=(batch_size, sequence_length)) + mask_data = np.random.randint(2, size=(batch_size, sequence_length)) + type_id_data = np.random.randint( + num_types, size=(batch_size, sequence_length)) + data, cls = model.predict([word_id_data, mask_data, type_id_data]) + new_data, new_cls = new_model.predict( + [word_id_data, mask_data, type_id_data]) + + # The output should be equal. + self.assertAllEqual(data, new_data) + self.assertAllEqual(cls, new_cls) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/masked_lm.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/masked_lm.py new file mode 100644 index 0000000..5843882 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/masked_lm.py @@ -0,0 +1,186 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Masked language model network.""" +# pylint: disable=g-classes-have-attributes +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow as tf + +from tensorflow.python.keras.engine import network # pylint: disable=g-direct-tensorflow-import +from official.modeling import tf_utils + + +@tf.keras.utils.register_keras_serializable(package='Text') +class MaskedLM(network.Network): + """Masked language model network head for BERT modeling. + + This network implements a masked language model based on the provided network. + It assumes that the network being passed has a "get_embedding_table()" method. + + Arguments: + input_width: The innermost dimension of the input tensor to this network. + num_predictions: The number of predictions to make per sequence. + source_network: The network with the embedding layer to use for the + embedding layer. + activation: The activation, if any, for the dense layer in this network. + initializer: The intializer for the dense layer in this network. Defaults to + a Glorot uniform initializer. + output: The output style for this network. Can be either 'logits' or + 'predictions'. + """ + + def __init__(self, + input_width, + num_predictions, + source_network, + activation=None, + initializer='glorot_uniform', + output='logits', + **kwargs): + + embedding_table = source_network.get_embedding_table() + vocab_size, hidden_size = embedding_table.shape + + sequence_data = tf.keras.layers.Input( + shape=(None, input_width), name='sequence_data', dtype=tf.float32) + masked_lm_positions = tf.keras.layers.Input( + shape=(num_predictions,), name='masked_lm_positions', dtype=tf.int32) + + masked_lm_input = tf.keras.layers.Lambda( + lambda x: self._gather_indexes(x[0], x[1]))( + [sequence_data, masked_lm_positions]) + lm_data = ( + tf.keras.layers.Dense( + hidden_size, + activation=activation, + kernel_initializer=initializer, + name='cls/predictions/transform/dense')(masked_lm_input)) + lm_data = tf.keras.layers.LayerNormalization( + axis=-1, epsilon=1e-12, name='cls/predictions/transform/LayerNorm')( + lm_data) + lm_data = tf.keras.layers.Lambda( + lambda x: tf.matmul(x, embedding_table, transpose_b=True))( + lm_data) + logits = Bias( + initializer=tf.keras.initializers.Zeros(), + name='cls/predictions/output_bias')( + lm_data) + + # We can't use the standard Keras reshape layer here, since it expects + # the input and output batch size to be the same. + reshape_layer = tf.keras.layers.Lambda( + lambda x: tf.reshape(x, [-1, num_predictions, vocab_size])) + + self.logits = reshape_layer(logits) + predictions = tf.keras.layers.Activation(tf.nn.log_softmax)(self.logits) + + if output == 'logits': + output_tensors = self.logits + elif output == 'predictions': + output_tensors = predictions + else: + raise ValueError( + ('Unknown `output` value "%s". `output` can be either "logits" or ' + '"predictions"') % output) + + super(MaskedLM, self).__init__( + inputs=[sequence_data, masked_lm_positions], + outputs=output_tensors, + **kwargs) + + def get_config(self): + raise NotImplementedError('MaskedLM cannot be directly serialized at this ' + 'time. Please use it only in Layers or ' + 'functionally subclassed Models/Networks.') + + def _gather_indexes(self, sequence_tensor, positions): + """Gathers the vectors at the specific positions. + + Args: + sequence_tensor: Sequence output of `BertModel` layer of shape + (`batch_size`, `seq_length`, num_hidden) where num_hidden is number of + hidden units of `BertModel` layer. + positions: Positions ids of tokens in sequence to mask for pretraining + of with dimension (batch_size, num_predictions) where + `num_predictions` is maximum number of tokens to mask out and predict + per each sequence. + + Returns: + Masked out sequence tensor of shape (batch_size * num_predictions, + num_hidden). + """ + sequence_shape = tf_utils.get_shape_list( + sequence_tensor, name='sequence_output_tensor') + batch_size, seq_length, width = sequence_shape + + flat_offsets = tf.reshape( + tf.range(0, batch_size, dtype=tf.int32) * seq_length, [-1, 1]) + flat_positions = tf.reshape(positions + flat_offsets, [-1]) + flat_sequence_tensor = tf.reshape(sequence_tensor, + [batch_size * seq_length, width]) + output_tensor = tf.gather(flat_sequence_tensor, flat_positions) + + return output_tensor + + +@tf.keras.utils.register_keras_serializable(package='Text') +# Temporary until we can create a Dense layer that ties the embedding. +class Bias(tf.keras.layers.Layer): + """Adds a bias term to an input.""" + + def __init__(self, + initializer='zeros', + regularizer=None, + constraint=None, + activation=None, + **kwargs): + super(Bias, self).__init__(**kwargs) + self._initializer = tf.keras.initializers.get(initializer) + self._regularizer = tf.keras.regularizers.get(regularizer) + self._constraint = tf.keras.constraints.get(constraint) + self._activation = tf.keras.activations.get(activation) + + def build(self, input_shape): + input_shape = tf.TensorShape(input_shape) + self._bias = self.add_weight( + 'bias', + shape=input_shape[1:], + initializer=self._initializer, + regularizer=self._regularizer, + constraint=self._constraint, + dtype=self._dtype, + trainable=True) + + super(Bias, self).build(input_shape) + + def get_config(self): + config = { + 'activation': tf.keras.activations.serialize(self._activation), + 'initializer': tf.keras.initializers.serialize(self._initializer), + 'regularizer': tf.keras.regularizers.serialize(self._regularizer), + 'constraint': tf.keras.constraints.serialize(self._constraint) + } + base_config = super(Bias, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + def call(self, inputs): + outputs = tf.nn.bias_add(inputs, self._bias) + if self._activation is not None: + return self._activation(outputs) # pylint: disable=not-callable + else: + return outputs diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/masked_lm_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/masked_lm_test.py new file mode 100644 index 0000000..2b7b382 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/masked_lm_test.py @@ -0,0 +1,227 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for masked language model network.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import + +from official.nlp.modeling.networks import masked_lm +from official.nlp.modeling.networks import transformer_encoder + + +# This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It +# guarantees forward compatibility of this code for the V2 switchover. +@keras_parameterized.run_all_keras_modes +class MaskedLMTest(keras_parameterized.TestCase): + + def create_network(self, + vocab_size, + sequence_length, + hidden_size, + num_predictions, + output='predictions', + xformer_stack=None): + # First, create a transformer stack that we can use to get the LM's + # vocabulary weight. + if xformer_stack is None: + xformer_stack = transformer_encoder.TransformerEncoder( + vocab_size=vocab_size, + num_layers=1, + sequence_length=sequence_length, + hidden_size=hidden_size, + num_attention_heads=4, + ) + word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + lm_outputs, _ = xformer_stack([word_ids, mask, type_ids]) + + # Create a maskedLM from the transformer stack. + test_network = masked_lm.MaskedLM( + num_predictions=num_predictions, + input_width=lm_outputs.shape[-1], + source_network=xformer_stack, + output=output) + return test_network + + def test_network_creation(self): + vocab_size = 100 + sequence_length = 32 + hidden_size = 64 + num_predictions = 21 + test_network = self.create_network( + vocab_size=vocab_size, + sequence_length=sequence_length, + hidden_size=hidden_size, + num_predictions=num_predictions) + + # Make sure that the output tensor of the masked LM is the right shape. + lm_input_tensor = tf.keras.Input(shape=(sequence_length, hidden_size)) + masked_lm_positions = tf.keras.Input( + shape=(num_predictions,), dtype=tf.int32) + output = test_network([lm_input_tensor, masked_lm_positions]) + + expected_output_shape = [None, num_predictions, vocab_size] + self.assertEqual(expected_output_shape, output.shape.as_list()) + + def test_network_invocation_with_internal_logits(self): + vocab_size = 100 + sequence_length = 32 + hidden_size = 64 + num_predictions = 21 + test_network = self.create_network( + vocab_size=vocab_size, + sequence_length=sequence_length, + hidden_size=hidden_size, + num_predictions=num_predictions) + + # Create a model from the masked LM layer. + lm_input_tensor = tf.keras.Input(shape=(sequence_length, hidden_size)) + masked_lm_positions = tf.keras.Input( + shape=(num_predictions,), dtype=tf.int32) + output = test_network([lm_input_tensor, masked_lm_positions]) + model = tf.keras.Model([lm_input_tensor, masked_lm_positions], output) + logits_model = tf.keras.Model(test_network.inputs, test_network.logits) + + # Invoke the masked LM on some fake data to make sure there are no runtime + # errors in the code. + batch_size = 3 + lm_input_data = 10 * np.random.random_sample( + (batch_size, sequence_length, hidden_size)) + masked_position_data = np.random.randint( + 2, size=(batch_size, num_predictions)) + outputs = model.predict([lm_input_data, masked_position_data]) + logits = logits_model.predict([lm_input_data, masked_position_data]) + + # Ensure that the tensor shapes are correct. + expected_output_shape = (batch_size, num_predictions, vocab_size) + self.assertEqual(expected_output_shape, outputs.shape) + self.assertEqual(expected_output_shape, logits.shape) + + # Ensure that the logits, when softmaxed, create the outputs. + input_tensor = tf.keras.Input(expected_output_shape[1:]) + output_tensor = tf.keras.layers.Activation(tf.nn.log_softmax)(input_tensor) + softmax_model = tf.keras.Model(input_tensor, output_tensor) + + calculated_softmax = softmax_model.predict(logits) + self.assertAllClose(outputs, calculated_softmax) + + def test_network_invocation_with_external_logits(self): + vocab_size = 100 + sequence_length = 32 + hidden_size = 64 + num_predictions = 21 + xformer_stack = transformer_encoder.TransformerEncoder( + vocab_size=vocab_size, + num_layers=1, + sequence_length=sequence_length, + hidden_size=hidden_size, + num_attention_heads=4, + ) + test_network = self.create_network( + vocab_size=vocab_size, + sequence_length=sequence_length, + hidden_size=hidden_size, + num_predictions=num_predictions, + xformer_stack=xformer_stack, + output='predictions') + logit_network = self.create_network( + vocab_size=vocab_size, + sequence_length=sequence_length, + hidden_size=hidden_size, + num_predictions=num_predictions, + xformer_stack=xformer_stack, + output='logits') + logit_network.set_weights(test_network.get_weights()) + + # Create a model from the masked LM layer. + lm_input_tensor = tf.keras.Input(shape=(sequence_length, hidden_size)) + masked_lm_positions = tf.keras.Input( + shape=(num_predictions,), dtype=tf.int32) + output = test_network([lm_input_tensor, masked_lm_positions]) + logit_output = logit_network([lm_input_tensor, masked_lm_positions]) + + model = tf.keras.Model([lm_input_tensor, masked_lm_positions], output) + logits_model = tf.keras.Model(([lm_input_tensor, masked_lm_positions]), + logit_output) + + # Invoke the masked LM on some fake data to make sure there are no runtime + # errors in the code. + batch_size = 3 + lm_input_data = 10 * np.random.random_sample( + (batch_size, sequence_length, hidden_size)) + masked_position_data = np.random.randint( + 2, size=(batch_size, num_predictions)) + outputs = model.predict([lm_input_data, masked_position_data]) + logits = logits_model.predict([lm_input_data, masked_position_data]) + + # Ensure that the tensor shapes are correct. + expected_output_shape = (batch_size, num_predictions, vocab_size) + self.assertEqual(expected_output_shape, outputs.shape) + self.assertEqual(expected_output_shape, logits.shape) + + # Ensure that the logits, when softmaxed, create the outputs. + input_tensor = tf.keras.Input(expected_output_shape[1:]) + output_tensor = tf.keras.layers.Activation(tf.nn.log_softmax)(input_tensor) + softmax_model = tf.keras.Model(input_tensor, output_tensor) + + calculated_softmax = softmax_model.predict(logits) + self.assertAllClose(outputs, calculated_softmax) + + def test_network_invocation(self): + vocab_size = 100 + sequence_length = 32 + hidden_size = 64 + num_predictions = 21 + test_network = self.create_network( + vocab_size=vocab_size, + sequence_length=sequence_length, + hidden_size=hidden_size, + num_predictions=num_predictions) + + # Create a model from the masked LM layer. + lm_input_tensor = tf.keras.Input(shape=(sequence_length, hidden_size)) + masked_lm_positions = tf.keras.Input( + shape=(num_predictions,), dtype=tf.int32) + output = test_network([lm_input_tensor, masked_lm_positions]) + model = tf.keras.Model([lm_input_tensor, masked_lm_positions], output) + + # Invoke the masked LM on some fake data to make sure there are no runtime + # errors in the code. + batch_size = 3 + lm_input_data = 10 * np.random.random_sample( + (batch_size, sequence_length, hidden_size)) + masked_position_data = np.random.randint( + 2, size=(batch_size, num_predictions)) + _ = model.predict([lm_input_data, masked_position_data]) + + def test_unknown_output_type_fails(self): + with self.assertRaisesRegex(ValueError, 'Unknown `output` value "bad".*'): + _ = self.create_network( + vocab_size=8, + sequence_length=8, + hidden_size=8, + num_predictions=8, + output='bad') + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/span_labeling.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/span_labeling.py new file mode 100644 index 0000000..e2fc400 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/span_labeling.py @@ -0,0 +1,95 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Span labeling network.""" +# pylint: disable=g-classes-have-attributes +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow as tf + +# pylint: disable=g-direct-tensorflow-import +from tensorflow.python.keras.engine import network + + +@tf.keras.utils.register_keras_serializable(package='Text') +class SpanLabeling(network.Network): + """Span labeling network head for BERT modeling. + + This network implements a simple single-span labeler based on a dense layer. + + Arguments: + input_width: The innermost dimension of the input tensor to this network. + activation: The activation, if any, for the dense layer in this network. + initializer: The intializer for the dense layer in this network. Defaults to + a Glorot uniform initializer. + output: The output style for this network. Can be either 'logits' or + 'predictions'. + """ + + def __init__(self, + input_width, + activation=None, + initializer='glorot_uniform', + output='logits', + **kwargs): + self._self_setattr_tracking = False + self._config = { + 'input_width': input_width, + 'activation': activation, + 'initializer': initializer, + 'output': output, + } + + sequence_data = tf.keras.layers.Input( + shape=(None, input_width), name='sequence_data', dtype=tf.float32) + + intermediate_logits = tf.keras.layers.Dense( + 2, # This layer predicts start location and end location. + activation=activation, + kernel_initializer=initializer, + name='predictions/transform/logits')( + sequence_data) + self.start_logits, self.end_logits = ( + tf.keras.layers.Lambda(self._split_output_tensor)(intermediate_logits)) + + start_predictions = tf.keras.layers.Activation(tf.nn.log_softmax)( + self.start_logits) + end_predictions = tf.keras.layers.Activation(tf.nn.log_softmax)( + self.end_logits) + + if output == 'logits': + output_tensors = [self.start_logits, self.end_logits] + elif output == 'predictions': + output_tensors = [start_predictions, end_predictions] + else: + raise ValueError( + ('Unknown `output` value "%s". `output` can be either "logits" or ' + '"predictions"') % output) + + super(SpanLabeling, self).__init__( + inputs=[sequence_data], outputs=output_tensors, **kwargs) + + def _split_output_tensor(self, tensor): + transposed_tensor = tf.transpose(tensor, [2, 0, 1]) + return tf.unstack(transposed_tensor) + + def get_config(self): + return self._config + + @classmethod + def from_config(cls, config, custom_objects=None): + return cls(**config) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/span_labeling_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/span_labeling_test.py new file mode 100644 index 0000000..8533a77 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/span_labeling_test.py @@ -0,0 +1,174 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for span_labeling network.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.nlp.modeling.networks import span_labeling + + +# This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It +# guarantees forward compatibility of this code for the V2 switchover. +@keras_parameterized.run_all_keras_modes +class SpanLabelingTest(keras_parameterized.TestCase): + + def test_network_creation(self): + """Validate that the Keras object can be created.""" + sequence_length = 15 + input_width = 512 + test_network = span_labeling.SpanLabeling( + input_width=input_width, output='predictions') + # Create a 3-dimensional input (the first dimension is implicit). + sequence_data = tf.keras.Input( + shape=(sequence_length, input_width), dtype=tf.float32) + start_outputs, end_outputs = test_network(sequence_data) + + # Validate that the outputs are of the expected shape. + expected_output_shape = [None, sequence_length] + self.assertEqual(expected_output_shape, start_outputs.shape.as_list()) + self.assertEqual(expected_output_shape, end_outputs.shape.as_list()) + + def test_network_invocation(self): + """Validate that the Keras object can be invoked.""" + sequence_length = 15 + input_width = 512 + test_network = span_labeling.SpanLabeling(input_width=input_width) + + # Create a 3-dimensional input (the first dimension is implicit). + sequence_data = tf.keras.Input( + shape=(sequence_length, input_width), dtype=tf.float32) + outputs = test_network(sequence_data) + model = tf.keras.Model(sequence_data, outputs) + + # Invoke the network as part of a Model. + batch_size = 3 + input_data = 10 * np.random.random_sample( + (batch_size, sequence_length, input_width)) + start_outputs, end_outputs = model.predict(input_data) + + # Validate that the outputs are of the expected shape. + expected_output_shape = (batch_size, sequence_length) + self.assertEqual(expected_output_shape, start_outputs.shape) + self.assertEqual(expected_output_shape, end_outputs.shape) + + def test_network_invocation_with_internal_logit_output(self): + """Validate that the logit outputs are correct.""" + sequence_length = 15 + input_width = 512 + test_network = span_labeling.SpanLabeling( + input_width=input_width, output='predictions') + # Create a 3-dimensional input (the first dimension is implicit). + sequence_data = tf.keras.Input( + shape=(sequence_length, input_width), dtype=tf.float32) + output = test_network(sequence_data) + model = tf.keras.Model(sequence_data, output) + logit_model = tf.keras.Model( + test_network.inputs, + [test_network.start_logits, test_network.end_logits]) + + batch_size = 3 + input_data = 10 * np.random.random_sample( + (batch_size, sequence_length, input_width)) + start_outputs, end_outputs = model.predict(input_data) + start_logits, end_logits = logit_model.predict(input_data) + + # Ensure that the tensor shapes are correct. + expected_output_shape = (batch_size, sequence_length) + self.assertEqual(expected_output_shape, start_outputs.shape) + self.assertEqual(expected_output_shape, end_outputs.shape) + self.assertEqual(expected_output_shape, start_logits.shape) + self.assertEqual(expected_output_shape, end_logits.shape) + + # Ensure that the logits, when softmaxed, create the outputs. + input_tensor = tf.keras.Input(expected_output_shape[1:]) + output_tensor = tf.keras.layers.Activation(tf.nn.log_softmax)(input_tensor) + softmax_model = tf.keras.Model(input_tensor, output_tensor) + + start_softmax = softmax_model.predict(start_logits) + self.assertAllClose(start_outputs, start_softmax) + end_softmax = softmax_model.predict(end_logits) + self.assertAllClose(end_outputs, end_softmax) + + def test_network_invocation_with_external_logit_output(self): + """Validate that the logit outputs are correct.""" + sequence_length = 15 + input_width = 512 + test_network = span_labeling.SpanLabeling( + input_width=input_width, output='predictions') + logit_network = span_labeling.SpanLabeling( + input_width=input_width, output='logits') + logit_network.set_weights(test_network.get_weights()) + + # Create a 3-dimensional input (the first dimension is implicit). + sequence_data = tf.keras.Input( + shape=(sequence_length, input_width), dtype=tf.float32) + output = test_network(sequence_data) + logit_output = logit_network(sequence_data) + model = tf.keras.Model(sequence_data, output) + logit_model = tf.keras.Model(sequence_data, logit_output) + + batch_size = 3 + input_data = 10 * np.random.random_sample( + (batch_size, sequence_length, input_width)) + start_outputs, end_outputs = model.predict(input_data) + start_logits, end_logits = logit_model.predict(input_data) + + # Ensure that the tensor shapes are correct. + expected_output_shape = (batch_size, sequence_length) + self.assertEqual(expected_output_shape, start_outputs.shape) + self.assertEqual(expected_output_shape, end_outputs.shape) + self.assertEqual(expected_output_shape, start_logits.shape) + self.assertEqual(expected_output_shape, end_logits.shape) + + # Ensure that the logits, when softmaxed, create the outputs. + input_tensor = tf.keras.Input(expected_output_shape[1:]) + output_tensor = tf.keras.layers.Activation(tf.nn.log_softmax)(input_tensor) + softmax_model = tf.keras.Model(input_tensor, output_tensor) + + start_softmax = softmax_model.predict(start_logits) + self.assertAllClose(start_outputs, start_softmax) + end_softmax = softmax_model.predict(end_logits) + self.assertAllClose(end_outputs, end_softmax) + + def test_serialize_deserialize(self): + # Create a network object that sets all of its config options. + network = span_labeling.SpanLabeling( + input_width=128, + activation='relu', + initializer='zeros', + output='predictions') + + # Create another network object from the first object's config. + new_network = span_labeling.SpanLabeling.from_config(network.get_config()) + + # Validate that the config can be forced to JSON. + _ = new_network.to_json() + + # If the serialization was successful, the new config should match the old. + self.assertAllEqual(network.get_config(), new_network.get_config()) + + def test_unknown_output_type_fails(self): + with self.assertRaisesRegex(ValueError, 'Unknown `output` value "bad".*'): + _ = span_labeling.SpanLabeling(input_width=10, output='bad') + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/transformer_encoder.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/transformer_encoder.py new file mode 100644 index 0000000..8b0bd40 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/transformer_encoder.py @@ -0,0 +1,191 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Transformer-based text encoder network.""" +# pylint: disable=g-classes-have-attributes +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow as tf + +from tensorflow.python.keras.engine import network # pylint: disable=g-direct-tensorflow-import +from official.modeling import activations +from official.nlp.modeling import layers + + +@tf.keras.utils.register_keras_serializable(package='Text') +class TransformerEncoder(network.Network): + """Bi-directional Transformer-based encoder network. + + This network implements a bi-directional Transformer-based encoder as + described in "BERT: Pre-training of Deep Bidirectional Transformers for + Language Understanding" (https://arxiv.org/abs/1810.04805). It includes the + embedding lookups and transformer layers, but not the masked language model + or classification task networks. + + The default values for this object are taken from the BERT-Base implementation + in "BERT: Pre-training of Deep Bidirectional Transformers for Language + Understanding". + + Arguments: + vocab_size: The size of the token vocabulary. + hidden_size: The size of the transformer hidden layers. + num_layers: The number of transformer layers. + num_attention_heads: The number of attention heads for each transformer. The + hidden size must be divisible by the number of attention heads. + sequence_length: The sequence length that this encoder expects. If None, the + sequence length is dynamic; if an integer, the encoder will require + sequences padded to this length. + max_sequence_length: The maximum sequence length that this encoder can + consume. If None, max_sequence_length uses the value from sequence length. + This determines the variable shape for positional embeddings. + type_vocab_size: The number of types that the 'type_ids' input can take. + intermediate_size: The intermediate size for the transformer layers. + activation: The activation to use for the transformer layers. + dropout_rate: The dropout rate to use for the transformer layers. + attention_dropout_rate: The dropout rate to use for the attention layers + within the transformer layers. + initializer: The initialzer to use for all weights in this encoder. + return_all_encoder_outputs: Whether to output sequence embedding outputs of + all encoder transformer layers. + """ + + def __init__(self, + vocab_size, + hidden_size=768, + num_layers=12, + num_attention_heads=12, + sequence_length=512, + max_sequence_length=None, + type_vocab_size=16, + intermediate_size=3072, + activation=activations.gelu, + dropout_rate=0.1, + attention_dropout_rate=0.1, + initializer=tf.keras.initializers.TruncatedNormal(stddev=0.02), + return_all_encoder_outputs=False, + **kwargs): + activation = tf.keras.activations.get(activation) + initializer = tf.keras.initializers.get(initializer) + + if not max_sequence_length: + max_sequence_length = sequence_length + self._self_setattr_tracking = False + self._config_dict = { + 'vocab_size': vocab_size, + 'hidden_size': hidden_size, + 'num_layers': num_layers, + 'num_attention_heads': num_attention_heads, + 'sequence_length': sequence_length, + 'max_sequence_length': max_sequence_length, + 'type_vocab_size': type_vocab_size, + 'intermediate_size': intermediate_size, + 'activation': tf.keras.activations.serialize(activation), + 'dropout_rate': dropout_rate, + 'attention_dropout_rate': attention_dropout_rate, + 'initializer': tf.keras.initializers.serialize(initializer), + 'return_all_encoder_outputs': return_all_encoder_outputs, + } + + word_ids = tf.keras.layers.Input( + shape=(sequence_length,), dtype=tf.int32, name='input_word_ids') + mask = tf.keras.layers.Input( + shape=(sequence_length,), dtype=tf.int32, name='input_mask') + type_ids = tf.keras.layers.Input( + shape=(sequence_length,), dtype=tf.int32, name='input_type_ids') + + self._embedding_layer = layers.OnDeviceEmbedding( + vocab_size=vocab_size, + embedding_width=hidden_size, + initializer=initializer, + name='word_embeddings') + word_embeddings = self._embedding_layer(word_ids) + + # Always uses dynamic slicing for simplicity. + self._position_embedding_layer = layers.PositionEmbedding( + initializer=initializer, + use_dynamic_slicing=True, + max_sequence_length=max_sequence_length) + position_embeddings = self._position_embedding_layer(word_embeddings) + + type_embeddings = ( + layers.OnDeviceEmbedding( + vocab_size=type_vocab_size, + embedding_width=hidden_size, + initializer=initializer, + use_one_hot=True, + name='type_embeddings')(type_ids)) + + embeddings = tf.keras.layers.Add()( + [word_embeddings, position_embeddings, type_embeddings]) + embeddings = ( + tf.keras.layers.LayerNormalization( + name='embeddings/layer_norm', + axis=-1, + epsilon=1e-12, + dtype=tf.float32)(embeddings)) + embeddings = ( + tf.keras.layers.Dropout(rate=dropout_rate)(embeddings)) + + self._transformer_layers = [] + data = embeddings + attention_mask = layers.SelfAttentionMask()([data, mask]) + encoder_outputs = [] + for i in range(num_layers): + layer = layers.Transformer( + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + intermediate_activation=activation, + dropout_rate=dropout_rate, + attention_dropout_rate=attention_dropout_rate, + kernel_initializer=initializer, + name='transformer/layer_%d' % i) + self._transformer_layers.append(layer) + data = layer([data, attention_mask]) + encoder_outputs.append(data) + + first_token_tensor = ( + tf.keras.layers.Lambda(lambda x: tf.squeeze(x[:, 0:1, :], axis=1))( + encoder_outputs[-1])) + cls_output = tf.keras.layers.Dense( + units=hidden_size, + activation='tanh', + kernel_initializer=initializer, + name='pooler_transform')( + first_token_tensor) + + if return_all_encoder_outputs: + outputs = [encoder_outputs, cls_output] + else: + outputs = [encoder_outputs[-1], cls_output] + + super(TransformerEncoder, self).__init__( + inputs=[word_ids, mask, type_ids], outputs=outputs, **kwargs) + + def get_embedding_table(self): + return self._embedding_layer.embeddings + + def get_config(self): + return self._config_dict + + @property + def transformer_layers(self): + """List of Transformer layers in the encoder.""" + return self._transformer_layers + + @classmethod + def from_config(cls, config, custom_objects=None): + return cls(**config) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/transformer_encoder_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/transformer_encoder_test.py new file mode 100644 index 0000000..506ea1e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/modeling/networks/transformer_encoder_test.py @@ -0,0 +1,203 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for transformer-based text encoder network.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.nlp.modeling.networks import transformer_encoder + + +# This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It +# guarantees forward compatibility of this code for the V2 switchover. +@keras_parameterized.run_all_keras_modes +class TransformerEncoderTest(keras_parameterized.TestCase): + + def tearDown(self): + super(TransformerEncoderTest, self).tearDown() + tf.keras.mixed_precision.experimental.set_policy("float32") + + def test_network_creation(self): + hidden_size = 32 + sequence_length = 21 + # Create a small TransformerEncoder for testing. + test_network = transformer_encoder.TransformerEncoder( + vocab_size=100, + hidden_size=hidden_size, + sequence_length=sequence_length, + num_attention_heads=2, + num_layers=3) + # Create the inputs (note that the first dimension is implicit). + word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + data, pooled = test_network([word_ids, mask, type_ids]) + + expected_data_shape = [None, sequence_length, hidden_size] + expected_pooled_shape = [None, hidden_size] + self.assertAllEqual(expected_data_shape, data.shape.as_list()) + self.assertAllEqual(expected_pooled_shape, pooled.shape.as_list()) + + # The default output dtype is float32. + self.assertAllEqual(tf.float32, data.dtype) + self.assertAllEqual(tf.float32, pooled.dtype) + + def test_all_encoder_outputs_network_creation(self): + hidden_size = 32 + sequence_length = 21 + # Create a small TransformerEncoder for testing. + test_network = transformer_encoder.TransformerEncoder( + vocab_size=100, + hidden_size=hidden_size, + sequence_length=sequence_length, + num_attention_heads=2, + num_layers=3, + return_all_encoder_outputs=True) + # Create the inputs (note that the first dimension is implicit). + word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + all_encoder_outputs, pooled = test_network([word_ids, mask, type_ids]) + + expected_data_shape = [None, sequence_length, hidden_size] + expected_pooled_shape = [None, hidden_size] + self.assertLen(all_encoder_outputs, 3) + for data in all_encoder_outputs: + self.assertAllEqual(expected_data_shape, data.shape.as_list()) + self.assertAllEqual(expected_pooled_shape, pooled.shape.as_list()) + + # The default output dtype is float32. + self.assertAllEqual(tf.float32, all_encoder_outputs[-1].dtype) + self.assertAllEqual(tf.float32, pooled.dtype) + + def test_network_creation_with_float16_dtype(self): + hidden_size = 32 + sequence_length = 21 + tf.keras.mixed_precision.experimental.set_policy("mixed_float16") + # Create a small TransformerEncoder for testing. + test_network = transformer_encoder.TransformerEncoder( + vocab_size=100, + hidden_size=hidden_size, + sequence_length=sequence_length, + num_attention_heads=2, + num_layers=3) + # Create the inputs (note that the first dimension is implicit). + word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + data, pooled = test_network([word_ids, mask, type_ids]) + + expected_data_shape = [None, sequence_length, hidden_size] + expected_pooled_shape = [None, hidden_size] + self.assertAllEqual(expected_data_shape, data.shape.as_list()) + self.assertAllEqual(expected_pooled_shape, pooled.shape.as_list()) + + # If float_dtype is set to float16, the data output is float32 (from a layer + # norm) and pool output should be float16. + self.assertAllEqual(tf.float32, data.dtype) + self.assertAllEqual(tf.float16, pooled.dtype) + + def test_network_invocation(self): + hidden_size = 32 + sequence_length = 21 + vocab_size = 57 + num_types = 7 + # Create a small TransformerEncoder for testing. + test_network = transformer_encoder.TransformerEncoder( + vocab_size=vocab_size, + hidden_size=hidden_size, + sequence_length=sequence_length, + num_attention_heads=2, + num_layers=3, + type_vocab_size=num_types) + self.assertTrue( + test_network._position_embedding_layer._use_dynamic_slicing) + # Create the inputs (note that the first dimension is implicit). + word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32) + data, pooled = test_network([word_ids, mask, type_ids]) + + # Create a model based off of this network: + model = tf.keras.Model([word_ids, mask, type_ids], [data, pooled]) + + # Invoke the model. We can't validate the output data here (the model is too + # complex) but this will catch structural runtime errors. + batch_size = 3 + word_id_data = np.random.randint( + vocab_size, size=(batch_size, sequence_length)) + mask_data = np.random.randint(2, size=(batch_size, sequence_length)) + type_id_data = np.random.randint( + num_types, size=(batch_size, sequence_length)) + _ = model.predict([word_id_data, mask_data, type_id_data]) + + # Creates a TransformerEncoder with max_sequence_length != sequence_length + max_sequence_length = 128 + test_network = transformer_encoder.TransformerEncoder( + vocab_size=vocab_size, + hidden_size=hidden_size, + sequence_length=sequence_length, + max_sequence_length=max_sequence_length, + num_attention_heads=2, + num_layers=3, + type_vocab_size=num_types) + self.assertTrue(test_network._position_embedding_layer._use_dynamic_slicing) + model = tf.keras.Model([word_ids, mask, type_ids], [data, pooled]) + _ = model.predict([word_id_data, mask_data, type_id_data]) + + def test_serialize_deserialize(self): + tf.keras.mixed_precision.experimental.set_policy("mixed_float16") + # Create a network object that sets all of its config options. + kwargs = dict( + vocab_size=100, + hidden_size=32, + num_layers=3, + num_attention_heads=2, + sequence_length=21, + max_sequence_length=21, + type_vocab_size=12, + intermediate_size=1223, + activation="relu", + dropout_rate=0.05, + attention_dropout_rate=0.22, + initializer="glorot_uniform", + return_all_encoder_outputs=False) + network = transformer_encoder.TransformerEncoder(**kwargs) + + expected_config = dict(kwargs) + expected_config["activation"] = tf.keras.activations.serialize( + tf.keras.activations.get(expected_config["activation"])) + expected_config["initializer"] = tf.keras.initializers.serialize( + tf.keras.initializers.get(expected_config["initializer"])) + self.assertEqual(network.get_config(), expected_config) + + # Create another network object from the first object's config. + new_network = transformer_encoder.TransformerEncoder.from_config( + network.get_config()) + + # Validate that the config can be forced to JSON. + _ = new_network.to_json() + + # If the serialization was successful, the new config should match the old. + self.assertAllEqual(network.get_config(), new_network.get_config()) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/README.md new file mode 100644 index 0000000..1664bd3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/README.md @@ -0,0 +1,167 @@ +# Multi-doc News Headline Generation Model: NHNet + +This repository contains TensorFlow 2.x implementation for NHNet [[1]](#1) as +well as instructions for producing the data we described in the paper. + +## Introduction + +NHNet is a multi-doc news headline generation model. It extends a standard +Transformer-based encoder-decoder model to multi-doc setting and relies on an +article-level attention layer to capture information common to most (if not all) +input news articles in a news cluster or story, and provide robustness against +potential outliers in the input due to clustering quality. + +Our academic paper [[1]](#1) which describes NHNet in detail can be found here: +https://arxiv.org/abs/2001.09386. + +## Dataset + +**Raw Data:** One can [download](https://github.com/google-research-datasets/NewSHead) +our multi-doc headline dataset which +contains 369,940 news stories and 932,571 unique URLs. We split these stories +into train (359,940 stories), validation (5,000 stories) and test set (5,000 +stories) by timestamp. + +More information, please checkout: +https://github.com/google-research-datasets/NewSHead + +### Crawling + +Unfortunately, we will not be able to release the pre-processed dataset that is +exactly used in the paper. Users need to crawl the URLs and the recommended +pre-processing is using an open-sourced library to download and parse the news +content including title and leading paragraphs. For ease of this process, we +provide a config of [news-please](https://github.com/fhamborg/news-please) that +will crawl and extract news articles on a local machine. + +First, install the `news-please` CLI (requires python 3.x) +```shell +$ pip3 install news-please +``` + +Next, run the crawler with our provided config and URL list + +```shell +# Sets to path of the downloaded data folder +$ DATA_FOLDER=/path/to/downloaded_dataset + +# Uses CLI interface to crawl +$ news-please -c $DATA_FOLDER/news_please +``` +By default, it will store crawled +articles under `/tmp/nhnet/`. To terminate the process press `CTRL+C`. + +The crawling may take some days (48 hours in our test) and it depends on the +network environment and #threads set in the config. As the crawling tool won't +stop automatically, it is not straightforward to check the progress. We suggest +to terminate the job if there are no new articles crawled in a short time period +(e.g., 10 minutes) by running +```shell +$ find /tmp/nhnet -type f | wc -l +``` +Please note that it is expected that some URLs are no longer available on the +web as time goes by. + +### Data Processing + +Given the crawled articles under `/tmp/nhnet/`, we would like to transform these +textual articles into a set of `TFRecord` files containing serialized +tensorflow.Example protocol buffers, with feature keys following the BERT +[[2]](#2) tradition but is extended for multiple text segments. We will later +use these processed TFRecords for training and evaluation. + +To do this, please first download a [BERT pretrained checkpoint](https://github.com/tensorflow/models/tree/master/official/nlp/bert#access-to-pretrained-checkpoints) +(`BERT-Base,Uncased` preferred for efficiency) and decompress the `tar.gz` file. +We need the vocabulary file and later use the checkpoint for NHNet +initialization. + +Next, we can run the following data preprocess script which may take a few hours + to read files and tokenize article content. + + +```shell +# Recall that we use DATA_FOLDER=/path/to/downloaded_dataset +$ python3 raw_data_preprocess.py \ + -crawled_articles=/tmp/nhnet \ + -vocab=/path/to/bert_checkpoint/vocab.txt \ + -do_lower_case=True \ + -len_title=15 \ + -len_passage=200 \ + -max_num_articles=5 \ + -data_folder=$DATA_FOLDER +``` + +This python script will export processed train/valid/eval files under +`$DATA_FOLDER/processed/`. + +## Training + +Please first install TensorFlow 2 and Tensorflow Model Garden following the +[requirments section](https://github.com/tensorflow/models/tree/master/official#requirements). + +### CPU/GPU +```shell +$ python3 trainer.py \ + --mode=train_and_eval \ + --vocab=/path/to/bert_checkpoint/vocab.txt \ + --init_checkpoint=/path/to/bert_checkpoint/bert_model.ckpt \ + --params_override='init_from_bert2bert=false' \ + --train_file_pattern=$DATA_FOLDER/processed/train.tfrecord* \ + --model_dir=/path/to/output/model \ + --len_title=15 \ + --len_passage=200 \ + --max_num_articles=5 \ + --model_type=nhnet \ + --train_batch_size=16 \ + --train_steps=10000 \ + --steps_per_loop=1 \ + --checkpoint_interval=100 +``` + +### TPU +```shell +$ python3 trainer.py \ + --mode=train_and_eval \ + --vocab=/path/to/bert_checkpoint/vocab.txt \ + --init_checkpoint=/path/to/bert_checkpoint/bert_model.ckpt \ + --params_override='init_from_bert2bert=false' \ + --train_file_pattern=$DATA_FOLDER/processed/train.tfrecord* \ + --model_dir=/path/to/output/model \ + --len_title=15 \ + --len_passage=200 \ + --max_num_articles=5 \ + --model_type=nhnet \ + --train_batch_size=1024 \ + --train_steps=10000 \ + --steps_per_loop=1000 \ + --checkpoint_interval=1000 \ + --distribution_strategy=tpu \ + --tpu=grpc://${TPU_IP_ADDRESS}:8470 +``` +In the paper, we train more than 10k steps with batch size set as 1024 with +TPU-v3-64. + +Note that, `trainer.py` also supports `train` mode and continuous `eval` mode. +For large scale TPU training, we recommend the have a process running the +`train` mode and another process running the continuous `eval` mode which can +runs on GPUs. +This is the setting we commonly used for large-scale experiments, because `eval` +will be non-blocking to the expensive training load. + +### Metrics +**Note: the metrics reported by `evaluation.py` are approximated on +word-piece level rather than the real string tokens. Some metrics like BLEU +scores can be off.** + +We will release a colab to evaluate results on string-level soon. + +## References + +[1] Xiaotao Gu, Yuning Mao, Jiawei Han, Jialu Liu, You Wu, Cong +Yu, Daniel Finnie, Hongkun Yu, Jiaqi Zhai and Nicholas Zukoski "Generating +Representative Headlines for News Stories": https://arxiv.org/abs/2001.09386. +World Wide Web Conf. (WWW’2020). + +[2] Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina +Toutanova "BERT: Pre-training of Deep Bidirectional Transformers for Language +Understanding": https://arxiv.org/abs/1810.04805. diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/configs.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/configs.py new file mode 100644 index 0000000..41cfa61 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/configs.py @@ -0,0 +1,107 @@ +# Lint as: python3 +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Common NHNet/Bert2Bert configuration.""" + +from typing import List, Text + +import dataclasses + +from official.modeling.hyperparams import base_config + + +@dataclasses.dataclass +class BERT2BERTConfig(base_config.Config): + """High-level configurations for BERT2BERT model. + + These include parameters that are not directly related to the experiment, + e.g. encoder, decoder, prediction, training, etc. + """ + vocab_size: int = 30522 + hidden_size: int = 768 + num_hidden_layers: int = 12 + num_attention_heads: int = 12 + intermediate_size: int = 3072 + hidden_act: str = "gelu" + hidden_dropout_prob: float = 0.1 + attention_probs_dropout_prob: float = 0.1 + max_position_embeddings: int = 512 + type_vocab_size: int = 2 + initializer_range: float = 0.02 + decoder_intermediate_size: int = 3072 + num_decoder_attn_heads: int = 12 + num_decoder_layers: int = 12 + + label_smoothing: float = 0.1 + learning_rate: float = 0.05 + learning_rate_warmup_steps: int = 20000 + optimizer: str = "Adam" + adam_beta1: float = 0.9 + adam_beta2: float = 0.997 + adam_epsilon: float = 1e-09 + + # predict params + beam_size: int = 5 + alpha: float = 0.6 + initializer_gain: float = 1.0 + use_cache: bool = True + + # input params + input_sharding: bool = False + input_data_not_padded: bool = False + pad_token_id: int = 0 + end_token_id: int = 102 + start_token_id: int = 101 + + +@dataclasses.dataclass +class NHNetConfig(BERT2BERTConfig): + """High-level configurations for NHNet model. + + These include parameters that are not directly related to the experiment, + e.g. encoder, decoder, prediction, training, etc. + """ + multi_channel_cross_attention: bool = True + passage_list: List[Text] = dataclasses.field( + default_factory=lambda: [chr(ord("b") + i) for i in range(5)]) + + # Initialization method. + # If init_from_bert2bert is false, we assume the checkpoint is from BERT + # pretraining and only encoder and self-attention variables are initialized. + init_from_bert2bert: bool = True + + +UNITTEST_CONFIG = { + "attention_probs_dropout_prob": 0.0, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.0, + "hidden_size": 16, + "initializer_range": 0.02, + "intermediate_size": 32, + "max_position_embeddings": 128, + "num_attention_heads": 2, + "num_hidden_layers": 1, + "type_vocab_size": 2, + "vocab_size": 30522, + "initializer_gain": 1.0, + "decoder_intermediate_size": 32, + "num_decoder_attn_heads": 2, + "num_decoder_layers": 1, + "use_cache": True, + "input_data_not_padded": False, + "pad_token_id": 0, + "end_token_id": 102, + "start_token_id": 101, +} diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/configs_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/configs_test.py new file mode 100644 index 0000000..2b855ec --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/configs_test.py @@ -0,0 +1,121 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for configs.""" + +import tensorflow as tf +from official.nlp.nhnet import configs + +BERT2BERT_CONFIG = { + "vocab_size": 30522, + "hidden_size": 768, + "num_hidden_layers": 12, + "num_attention_heads": 12, + "intermediate_size": 3072, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "attention_probs_dropout_prob": 0.1, + "max_position_embeddings": 512, + "type_vocab_size": 2, + "initializer_range": 0.02, + + # model params + "decoder_intermediate_size": 3072, + "num_decoder_attn_heads": 12, + "num_decoder_layers": 12, + + # training params + "label_smoothing": 0.1, + "learning_rate": 0.05, + "learning_rate_warmup_steps": 20000, + "optimizer": "Adam", + "adam_beta1": 0.9, + "adam_beta2": 0.997, + "adam_epsilon": 1e-09, + + # predict params + "beam_size": 5, + "alpha": 0.6, + "initializer_gain": 1.0, + "use_cache": True, + + # input params + "input_sharding": False, + "input_data_not_padded": False, + "pad_token_id": 0, + "end_token_id": 102, + "start_token_id": 101, +} + +NHNET_CONFIG = { + "vocab_size": 30522, + "hidden_size": 768, + "num_hidden_layers": 12, + "num_attention_heads": 12, + "intermediate_size": 3072, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "attention_probs_dropout_prob": 0.1, + "max_position_embeddings": 512, + "type_vocab_size": 2, + "initializer_range": 0.02, + + # model params + "decoder_intermediate_size": 3072, + "num_decoder_attn_heads": 12, + "num_decoder_layers": 12, + "multi_channel_cross_attention": True, + + # training params + "label_smoothing": 0.1, + "learning_rate": 0.05, + "learning_rate_warmup_steps": 20000, + "optimizer": "Adam", + "adam_beta1": 0.9, + "adam_beta2": 0.997, + "adam_epsilon": 1e-09, + + # predict params + "beam_size": 5, + "alpha": 0.6, + "initializer_gain": 1.0, + "use_cache": True, + + # input params + "passage_list": ["b", "c", "d", "e", "f"], + "input_sharding": False, + "input_data_not_padded": False, + "pad_token_id": 0, + "end_token_id": 102, + "start_token_id": 101, + + "init_from_bert2bert": True, +} + + +class ConfigsTest(tf.test.TestCase): + + def test_configs(self): + cfg = configs.BERT2BERTConfig() + cfg.validate() + self.assertEqual(cfg.as_dict(), BERT2BERT_CONFIG) + + def test_nhnet_config(self): + cfg = configs.NHNetConfig() + cfg.validate() + self.assertEqual(cfg.as_dict(), NHNET_CONFIG) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/decoder.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/decoder.py new file mode 100644 index 0000000..18960ed --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/decoder.py @@ -0,0 +1,526 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Transformer decoder that mimics a BERT encoder, to load BERT checkpoints.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow as tf +from official.modeling import tf_utils +from official.nlp.modeling import layers +from official.nlp.nhnet import multi_channel_attention +from official.nlp.transformer import model_utils as transformer_utils + + +class TransformerDecoderBlock(tf.keras.layers.Layer): + """Single transformer layer for decoder. + + It has three sub-layers: + (1) a multi-head self-attention mechanism. + (2) a encoder-decoder attention. + (3) a positionwise fully connected feed-forward network. + """ + + def __init__(self, + hidden_size=768, + num_attention_heads=12, + intermediate_size=3072, + intermediate_activation="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + initializer_range=0.02, + multi_channel_cross_attention=False, + **kwargs): + super(TransformerDecoderBlock, self).__init__(**kwargs) + self.hidden_size = hidden_size + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.intermediate_activation = tf_utils.get_activation( + intermediate_activation) + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.multi_channel_cross_attention = multi_channel_cross_attention + self._kernel_initializer = tf.keras.initializers.TruncatedNormal( + stddev=initializer_range) + self._bias_initializer = tf.keras.initializers.get("zeros") + if self.multi_channel_cross_attention: + self._cross_attention_cls = multi_channel_attention.MultiChannelAttention + else: + self._cross_attention_cls = layers.MultiHeadAttention + + if self.hidden_size % self.num_attention_heads != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (self.hidden_size, self.num_attention_heads)) + self.attention_head_size = int(self.hidden_size / self.num_attention_heads) + + def build(self, unused_input_shapes): + # Self attention. + self.self_attention = layers.CachedAttention( + num_heads=self.num_attention_heads, + head_size=self.attention_head_size, + dropout_rate=self.attention_probs_dropout_prob, + kernel_initializer=self._kernel_initializer, + name="self_attention") + self.self_attention_output_dense = layers.DenseEinsum( + output_shape=self.hidden_size, + num_summed_dimensions=2, + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + name="self_attention_output") + self.self_attention_dropout = tf.keras.layers.Dropout( + rate=self.hidden_dropout_prob) + self.self_attention_layer_norm = ( + tf.keras.layers.LayerNormalization( + name="self_attention_layer_norm", axis=-1, epsilon=1e-12)) + # Encoder-decoder attention. + self.encdec_attention = self._cross_attention_cls( + num_heads=self.num_attention_heads, + head_size=self.attention_head_size, + dropout_rate=self.attention_probs_dropout_prob, + kernel_initializer=self._kernel_initializer, + name="attention/encdec") + self.encdec_attention_output_dense = layers.DenseEinsum( + output_shape=self.hidden_size, + num_summed_dimensions=2, + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + name="attention/encdec_output") + self.encdec_attention_dropout = tf.keras.layers.Dropout( + rate=self.hidden_dropout_prob) + self.encdec_attention_layer_norm = ( + tf.keras.layers.LayerNormalization( + name="attention/encdec_output_layer_norm", axis=-1, epsilon=1e-12)) + + # Feed-forward projection. + self.intermediate_dense = layers.DenseEinsum( + output_shape=self.intermediate_size, + activation=None, + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + name="intermediate") + self.intermediate_activation_layer = tf.keras.layers.Activation( + self.intermediate_activation) + self.output_dense = layers.DenseEinsum( + output_shape=self.hidden_size, + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + name="output") + self.output_dropout = tf.keras.layers.Dropout(rate=self.hidden_dropout_prob) + self.output_layer_norm = tf.keras.layers.LayerNormalization( + name="output_layer_norm", axis=-1, epsilon=1e-12) + super(TransformerDecoderBlock, self).build(unused_input_shapes) + + def common_layers_with_encoder(self): + """Gets layer objects that can make a Transformer encoder block.""" + return [ + self.self_attention, self.self_attention_output_dense, + self.self_attention_layer_norm, self.intermediate_dense, + self.output_dense, self.output_layer_norm + ] + + def call(self, inputs, cache=None, decode_loop_step=None): + if self.multi_channel_cross_attention: + if len(inputs) != 5: + raise ValueError( + "TransformerDecoderBlock must have 5 inputs, when it uses " + "multi_channel_cross_attention. But it got: %d" % len(inputs)) + elif len(inputs) != 4: + raise ValueError( + "TransformerDecoderBlock must have 4 inputs, but it got: %d" % + len(inputs)) + input_tensor, memory, attention_mask, self_attention_mask = inputs[:4] + if cache is None: + self_attention_inputs = [input_tensor, input_tensor, self_attention_mask] + else: + self_attention_inputs = [ + input_tensor, input_tensor, self_attention_mask, cache + ] + self_attention_output, cache = self.self_attention( + self_attention_inputs, decode_loop_step=decode_loop_step) + self_attention_output = self.self_attention_output_dense( + self_attention_output) + self_attention_output = self.self_attention_dropout(self_attention_output) + self_attention_output = self.self_attention_layer_norm( + input_tensor + self_attention_output) + + cross_attn_inputs = [self_attention_output, memory, attention_mask] + if self.multi_channel_cross_attention: + # Accesses the 5-th input tensor for the doc-attention probabilities. + cross_attn_inputs.append(inputs[-1]) + attention_output = self.encdec_attention(cross_attn_inputs) + attention_output = self.encdec_attention_output_dense(attention_output) + attention_output = self.encdec_attention_dropout(attention_output) + attention_output = self.encdec_attention_layer_norm(self_attention_output + + attention_output) + + intermediate_output = self.intermediate_dense(attention_output) + intermediate_output = self.intermediate_activation_layer( + intermediate_output) + layer_output = self.output_dense(intermediate_output) + layer_output = self.output_dropout(layer_output) + layer_output = self.output_layer_norm(layer_output + attention_output) + return layer_output, cache + + +class TransformerDecoder(tf.keras.layers.Layer): + """Transformer decoder stack.""" + + def __init__(self, + num_hidden_layers=12, + hidden_size=768, + num_attention_heads=12, + intermediate_size=3072, + intermediate_activation="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + initializer_range=0.02, + attend_to_last_layer=True, + multi_channel_cross_attention=False, + **kwargs): + super(TransformerDecoder, self).__init__(**kwargs) + self.num_hidden_layers = num_hidden_layers + self.hidden_size = hidden_size + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.intermediate_activation = tf_utils.get_activation( + intermediate_activation) + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.initializer_range = initializer_range + self.attend_to_last_layer = attend_to_last_layer + self.multi_channel_cross_attention = multi_channel_cross_attention + + def build(self, unused_input_shapes): + """Implements build() for the layer.""" + self.layers = [] + for i in range(self.num_hidden_layers): + self.layers.append( + TransformerDecoderBlock( + hidden_size=self.hidden_size, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + intermediate_activation=self.intermediate_activation, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + initializer_range=self.initializer_range, + multi_channel_cross_attention=self.multi_channel_cross_attention, + name=("layer_%d" % i))) + super(TransformerDecoder, self).build(unused_input_shapes) + + def call(self, inputs, cache=None, decode_loop_step=None): + """Return the output of the decoder layer stacks. + + Args: + inputs: A dictionary of inputs. `decoder_inputs` is a tf.int32 tensor for + input ids. `encoder_outputs` is a list of tensors with shape + [batch_size, input_length, hidden_size]. `self_attention_mask` is the + bias for decoder self-attention layer. [1, 1, target_length, + target_length]. `attention_mask` is the bias for encoder-decoder + attention layer, [batch_size, 1, 1, input_length]. + cache: A dictionary of cache tensors, including key & value attentions. + decode_loop_step: an integer to indicate the step inside a decoding loop. + + Returns: + Output of decoder layer stack. + float32 tensor with shape [batch_size, target_length, hidden_size] + """ + decoder_inputs = inputs["decoder_inputs"] + encoder_outputs = inputs["encoder_outputs"] + self_attention_mask = inputs["self_attention_mask"] + attention_mask = inputs["attention_mask"] + decoder_shape = tf_utils.get_shape_list(decoder_inputs, expected_rank=3) + batch_size = decoder_shape[0] + decoder_length = decoder_shape[1] + + def _to_bert_self_attention_mask(matrix): + """[1, 1, target_len, target_len] -> [bs, target_len, target_len].""" + matrix = tf.squeeze(matrix, axis=[1]) + matrix = tf.tile(matrix, [batch_size, 1, 1]) + return matrix + + def _to_bert_encdec_attention_mask(matrix): + """[bs, 1, 1, input_len] -> [bs, target_len, input_len].""" + if self.multi_channel_cross_attention: + matrix = tf.expand_dims(matrix, axis=2) + matrix = tf.tile(matrix, [1, 1, decoder_length, 1]) + else: + matrix = tf.squeeze(matrix, axis=[1]) + matrix = tf.tile(matrix, [1, decoder_length, 1]) + return matrix + + attention_mask = _to_bert_encdec_attention_mask(attention_mask) + self_attention_mask = _to_bert_self_attention_mask(self_attention_mask) + + output_tensor = decoder_inputs + for layer_idx in range(self.num_hidden_layers): + if self.attend_to_last_layer: + memory = encoder_outputs[-1] + else: + memory = encoder_outputs[layer_idx] + if self.multi_channel_cross_attention: + transformer_inputs = [ + output_tensor, memory, attention_mask, self_attention_mask, + inputs["doc_attention_probs"] + ] + else: + transformer_inputs = [ + output_tensor, memory, attention_mask, self_attention_mask + ] + # Gets the cache for decoding. + if cache is None: + output_tensor, _ = self.layers[layer_idx](transformer_inputs) + else: + cache_layer_idx = str(layer_idx) + output_tensor, cache[cache_layer_idx] = self.layers[layer_idx]( + transformer_inputs, + cache=cache[cache_layer_idx], + decode_loop_step=decode_loop_step) + return output_tensor, cache + + +def get_attention_bias(input_tensor, + bias_type, + padding_value=0, + max_length=None): + """A helper function to get various attention bias tensors.""" + if bias_type not in ("single_cross", "multi_cross", "decoder_self"): + raise ValueError("Invalid attention bias type: %s" % bias_type) + if bias_type == "single_cross": + length = tf_utils.get_shape_list(input_tensor, expected_rank=2)[1] + bias = transformer_utils.get_padding_bias( + input_tensor, padding_value=padding_value) + elif bias_type == "multi_cross": + length = tf_utils.get_shape_list(input_tensor, expected_rank=3)[2] + padding = transformer_utils.get_padding( + input_tensor, padding_value=padding_value) + bias = padding * -1e9 + else: + if max_length is not None: + length = max_length + else: + length = tf_utils.get_shape_list(input_tensor, expected_rank=2)[1] + bias = transformer_utils.get_decoder_self_attention_bias(length) + + return tf.where(bias < 0, tf.zeros_like(bias), tf.ones_like(bias)) + + +class AttentionBias(tf.keras.layers.Layer): + + def __init__(self, bias_type, **kwargs): + super(AttentionBias, self).__init__(**kwargs) + self.bias_type = bias_type + + def call(self, inputs): + return get_attention_bias(inputs, self.bias_type) + + +class EmbeddingPostprocessor(tf.keras.layers.Layer): + """Performs various post-processing on a word embedding tensor.""" + + def __init__(self, + use_type_embeddings=False, + token_type_vocab_size=None, + use_position_embeddings=True, + max_position_embeddings=512, + dropout_prob=0.0, + initializer_range=0.02, + initializer=None, + **kwargs): + super(EmbeddingPostprocessor, self).__init__(**kwargs) + self.use_type_embeddings = use_type_embeddings + self.token_type_vocab_size = token_type_vocab_size + self.use_position_embeddings = use_position_embeddings + self.max_position_embeddings = max_position_embeddings + self.dropout_prob = dropout_prob + self.initializer_range = initializer_range + + if not initializer: + self.initializer = tf.keras.initializers.TruncatedNormal( + stddev=initializer_range) + else: + self.initializer = initializer + + if self.use_type_embeddings and not self.token_type_vocab_size: + raise ValueError("If `use_type_embeddings` is True, then " + "`token_type_vocab_size` must be specified.") + + def build(self, input_shapes): + """Implements build() for the layer.""" + (word_embeddings_shape, _) = input_shapes + width = word_embeddings_shape.as_list()[-1] + self.type_embeddings = None + if self.use_type_embeddings: + self.type_embeddings = self.add_weight( + "type_embeddings", + shape=[self.token_type_vocab_size, width], + initializer=tf.keras.initializers.TruncatedNormal( + stddev=self.initializer_range), + dtype=self.dtype) + + self.position_embeddings = None + if self.use_position_embeddings: + self.position_embeddings = self.add_weight( + "position_embeddings", + shape=[self.max_position_embeddings, width], + initializer=tf.keras.initializers.TruncatedNormal( + stddev=self.initializer_range), + dtype=self.dtype) + + self.output_layer_norm = tf.keras.layers.LayerNormalization( + name="layer_norm", axis=-1, epsilon=1e-12, dtype=tf.float32) + self.output_dropout = tf.keras.layers.Dropout( + rate=self.dropout_prob, dtype=tf.float32) + super(EmbeddingPostprocessor, self).build(input_shapes) + + def __call__(self, word_embeddings, token_type_ids=None, **kwargs): + inputs = tf_utils.pack_inputs([word_embeddings, token_type_ids]) + return super(EmbeddingPostprocessor, self).__call__(inputs, **kwargs) + + def call(self, inputs): + """Implements call() for the layer.""" + unpacked_inputs = tf_utils.unpack_inputs(inputs) + word_embeddings = unpacked_inputs[0] + token_type_ids = unpacked_inputs[1] + input_shape = tf_utils.get_shape_list(word_embeddings, expected_rank=3) + batch_size = input_shape[0] + seq_length = input_shape[1] + width = input_shape[2] + + output = word_embeddings + if self.use_type_embeddings: + flat_token_type_ids = tf.reshape(token_type_ids, [-1]) + token_type_embeddings = tf.gather(self.type_embeddings, + flat_token_type_ids) + token_type_embeddings = tf.reshape(token_type_embeddings, + [batch_size, seq_length, width]) + output += token_type_embeddings + + if self.use_position_embeddings: + position_embeddings = tf.expand_dims( + tf.slice(self.position_embeddings, [0, 0], [seq_length, width]), + axis=0) + + output += position_embeddings + + output = self.output_layer_norm(output) + output = self.output_dropout(output) + + return output + + +class Decoder(tf.keras.layers.Layer): + """The decoder network which can reuse encoder embeddings for target.""" + + def __init__(self, config, embedding_lookup=None, **kwargs): + super(Decoder, self).__init__(**kwargs) + self.config = config + # Shares vocabulary embedding. + self.embedding_lookup = None + if embedding_lookup: + self.embedding_lookup = embedding_lookup + + def build(self, unused_input_shapes): + """Implements build() for the layer.""" + if self.embedding_lookup is None: + self.embedding_lookup = layers.OnDeviceEmbedding( + vocab_size=self.config.vocab_size, + embedding_width=self.config.hidden_size, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=self.config.initializer_range), + name="target_embeddings") + self.embedding_postprocessor = EmbeddingPostprocessor( + use_type_embeddings=False, + use_position_embeddings=True, + max_position_embeddings=self.config.max_position_embeddings, + dropout_prob=self.config.hidden_dropout_prob, + initializer=tf.keras.initializers.VarianceScaling( + scale=self.config.initializer_gain, + mode="fan_avg", + distribution="uniform"), + name="embedding_postprocessor") + # Decoder can use a different intermediate size. + self.multi_channel_cross_attention = self.config.get( + "multi_channel_cross_attention", False) + self.decoder = TransformerDecoder( + num_hidden_layers=self.config.num_decoder_layers, + hidden_size=self.config.hidden_size, + num_attention_heads=self.config.num_decoder_attn_heads, + intermediate_size=self.config.decoder_intermediate_size, + intermediate_activation=self.config.hidden_act, + hidden_dropout_prob=self.config.hidden_dropout_prob, + attention_probs_dropout_prob=self.config.attention_probs_dropout_prob, + initializer_range=self.config.initializer_range, + multi_channel_cross_attention=self.multi_channel_cross_attention, + name="decoder") + super(Decoder, self).build(unused_input_shapes) + + def _decoding_step_time_signal(self, target_embeds, decode_loop_step): + """Applies time signal (positional embeddings) for decoded embeddings.""" + # TODO(hongkuny): migrate to keras bert and design a module to handle this. + output = target_embeds + if self.embedding_postprocessor.use_position_embeddings: + position_embeddings = tf.gather( + self.embedding_postprocessor.position_embeddings, [decode_loop_step]) + # Broadcasts to all sequences inside a batch. + output += position_embeddings + + output = self.embedding_postprocessor.output_layer_norm(output) + output = self.embedding_postprocessor.output_dropout(output) + return output + + def call(self, + inputs, + cache=None, + decode_loop_step=None, + padded_decode=False): + """Implements call() for the layer. + + Args: + inputs: a list of input tensors. + cache: A dictionary of cache tensors, including key & value attentions. + Due to the limit of keras, we uses the side effect to update cache and + states of tensors will be mutated. + decode_loop_step: an integer to indicate the step inside a decoding loop. + padded_decode: a boolean indicates if the pass is for padded decoding. + + Returns: + Decoder output tensors. + """ + attention_bias = inputs["attention_bias"] + target_ids = inputs["target_ids"] + all_encoder_outputs = inputs["all_encoder_outputs"] + self_attention_bias = inputs["self_attention_bias"] + if not isinstance(all_encoder_outputs, list): + all_encoder_outputs = [all_encoder_outputs] + + target_embeds = self.embedding_lookup(target_ids) + if decode_loop_step is None: + target_embeds = self.embedding_postprocessor(target_embeds) + else: + target_embeds = self._decoding_step_time_signal(target_embeds, + decode_loop_step) + decoder_inputs = dict( + decoder_inputs=target_embeds, + encoder_outputs=all_encoder_outputs, + self_attention_mask=self_attention_bias, + attention_mask=attention_bias) + if self.multi_channel_cross_attention: + decoder_inputs["doc_attention_probs"] = inputs["doc_attention_probs"] + decode_outputs, cache = self.decoder( + decoder_inputs, cache, decode_loop_step if padded_decode else None) + return decode_outputs diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/decoder_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/decoder_test.py new file mode 100644 index 0000000..1a58c7e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/decoder_test.py @@ -0,0 +1,182 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for nlp.nhnet.decoder.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf +from official.nlp.modeling import layers +from official.nlp.nhnet import configs +from official.nlp.nhnet import decoder +from official.nlp.nhnet import utils + + +def _create_cache(batch_size, init_decode_length, num_heads, head_size): + return { + "key": + tf.zeros([batch_size, init_decode_length, num_heads, head_size], + dtype=tf.float32), + "value": + tf.zeros([batch_size, init_decode_length, num_heads, head_size], + dtype=tf.float32) + } + + +class DecoderTest(tf.test.TestCase): + + def setUp(self): + super(DecoderTest, self).setUp() + self._config = utils.get_test_params() + + def test_transformer_decoder(self): + decoder_block = decoder.TransformerDecoder( + num_hidden_layers=self._config.num_hidden_layers, + hidden_size=self._config.hidden_size, + num_attention_heads=self._config.num_attention_heads, + intermediate_size=self._config.intermediate_size, + intermediate_activation=self._config.hidden_act, + hidden_dropout_prob=self._config.hidden_dropout_prob, + attention_probs_dropout_prob=self._config.attention_probs_dropout_prob, + initializer_range=self._config.initializer_range) + decoder_block.build(None) + self.assertEqual(len(decoder_block.layers), self._config.num_hidden_layers) + + def test_decoder_block_with_cache(self): + decoder_block = decoder.TransformerDecoderBlock( + hidden_size=self._config.hidden_size, + num_attention_heads=self._config.num_attention_heads, + intermediate_size=self._config.intermediate_size, + intermediate_activation=self._config.hidden_act, + hidden_dropout_prob=self._config.hidden_dropout_prob, + attention_probs_dropout_prob=self._config.attention_probs_dropout_prob, + initializer_range=self._config.initializer_range) + # Forward path. + dummy_tensor = tf.zeros([2, 4, self._config.hidden_size], dtype=tf.float32) + dummy_mask = tf.zeros([2, 4, 4], dtype=tf.float32) + inputs = [dummy_tensor, dummy_tensor, dummy_mask, dummy_mask] + cache = _create_cache( + 2, 0, self._config.num_attention_heads, + self._config.hidden_size // self._config.num_attention_heads) + output, cache = decoder_block(inputs, cache) + self.assertEqual(output.shape, (2, 4, self._config.hidden_size)) + self.assertEqual(cache["value"].shape, (2, 4, 2, 8)) + + def test_bert_decoder(self): + seq_length = 10 + encoder_input_ids = tf.keras.layers.Input( + shape=(seq_length,), name="encoder_input_ids", dtype=tf.int32) + target_ids = tf.keras.layers.Input( + shape=(seq_length,), name="target_ids", dtype=tf.int32) + encoder_outputs = tf.keras.layers.Input( + shape=(seq_length, self._config.hidden_size), + name="all_encoder_outputs", + dtype=tf.float32) + embedding_lookup = layers.OnDeviceEmbedding( + vocab_size=self._config.vocab_size, + embedding_width=self._config.hidden_size, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=self._config.initializer_range), + name="word_embeddings") + cross_attention_bias = decoder.AttentionBias(bias_type="single_cross")( + encoder_input_ids) + self_attention_bias = decoder.AttentionBias(bias_type="decoder_self")( + target_ids) + inputs = dict( + attention_bias=cross_attention_bias, + self_attention_bias=self_attention_bias, + target_ids=target_ids, + all_encoder_outputs=encoder_outputs) + decoder_layer = decoder.Decoder(self._config, embedding_lookup) + outputs = decoder_layer(inputs) + model_inputs = dict( + encoder_input_ids=encoder_input_ids, + target_ids=target_ids, + all_encoder_outputs=encoder_outputs) + model = tf.keras.Model(inputs=model_inputs, outputs=outputs, name="test") + self.assertLen(decoder_layer.trainable_weights, 30) + # Forward path. + fake_inputs = { + "encoder_input_ids": np.zeros((2, 10), dtype=np.int32), + "target_ids": np.zeros((2, 10), dtype=np.int32), + "all_encoder_outputs": np.zeros((2, 10, 16), dtype=np.float32), + } + output_tensor = model(fake_inputs) + self.assertEqual(output_tensor.shape, (2, 10, 16)) + + def test_multi_doc_decoder(self): + self._config = utils.get_test_params(cls=configs.NHNetConfig) + seq_length = 10 + num_docs = 5 + encoder_input_ids = tf.keras.layers.Input( + shape=(num_docs, seq_length), name="encoder_input_ids", dtype=tf.int32) + target_ids = tf.keras.layers.Input( + shape=(seq_length,), name="target_ids", dtype=tf.int32) + encoder_outputs = tf.keras.layers.Input( + shape=(num_docs, seq_length, self._config.hidden_size), + name="all_encoder_outputs", + dtype=tf.float32) + embedding_lookup = layers.OnDeviceEmbedding( + vocab_size=self._config.vocab_size, + embedding_width=self._config.hidden_size, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=self._config.initializer_range), + name="word_embeddings") + doc_attention_probs = tf.keras.layers.Input( + shape=(self._config.num_decoder_attn_heads, seq_length, num_docs), + name="doc_attention_probs", + dtype=tf.float32) + cross_attention_bias = decoder.AttentionBias(bias_type="multi_cross")( + encoder_input_ids) + self_attention_bias = decoder.AttentionBias(bias_type="decoder_self")( + target_ids) + + inputs = dict( + attention_bias=cross_attention_bias, + self_attention_bias=self_attention_bias, + target_ids=target_ids, + all_encoder_outputs=encoder_outputs, + doc_attention_probs=doc_attention_probs) + + decoder_layer = decoder.Decoder(self._config, embedding_lookup) + outputs = decoder_layer(inputs) + model_inputs = dict( + encoder_input_ids=encoder_input_ids, + target_ids=target_ids, + all_encoder_outputs=encoder_outputs, + doc_attention_probs=doc_attention_probs) + model = tf.keras.Model(inputs=model_inputs, outputs=outputs, name="test") + self.assertLen(decoder_layer.trainable_weights, 30) + # Forward path. + fake_inputs = { + "encoder_input_ids": + np.zeros((2, num_docs, seq_length), dtype=np.int32), + "target_ids": + np.zeros((2, seq_length), dtype=np.int32), + "all_encoder_outputs": + np.zeros((2, num_docs, seq_length, 16), dtype=np.float32), + "doc_attention_probs": + np.zeros( + (2, self._config.num_decoder_attn_heads, seq_length, num_docs), + dtype=np.float32) + } + output_tensor = model(fake_inputs) + self.assertEqual(output_tensor.shape, (2, seq_length, 16)) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/evaluation.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/evaluation.py new file mode 100644 index 0000000..b9c94dc --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/evaluation.py @@ -0,0 +1,185 @@ +# Lint as: python3 +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Evaluation for Bert2Bert.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import os +from absl import logging +import numpy as np +import tensorflow as tf + +from official.nlp.nhnet import input_pipeline +from official.nlp.nhnet import models +from official.nlp.transformer import metrics as metrics_v2 +from official.nlp.transformer.utils import metrics + + +def rouge_l_fscore(logits, labels): + """ROUGE scores computation between labels and predictions. + + This is an approximate ROUGE scoring method since we do not glue word pieces + or decode the ids and tokenize the output. + + Args: + logits: tensor, model predictions + labels: tensor, gold output. + + Returns: + rouge_l_fscore: approx rouge-l f1 score. + """ + predictions = np.argmax(logits, axis=-1) + rouge_l_f_score = metrics.rouge_l_sentence_level(predictions, labels) + return rouge_l_f_score + + +def rouge_2_fscore(logits, labels): + """ROUGE-2 F1 score computation between labels and predictions. + + This is an approximate ROUGE scoring method since we do not glue word pieces + or decode the ids and tokenize the output. + + Args: + logits: tensor, model predictions + labels: tensor, gold output. + + Returns: + rouge2_fscore: approx rouge-2 f1 score. + """ + predictions = np.argmax(logits, axis=-1) + rouge_2_f_score = metrics.rouge_n(predictions, labels) + return rouge_2_f_score + + +def bleu_score(logits, labels): + """Approximate BLEU score computation between labels and predictions. + + An approximate BLEU scoring method since we do not glue word pieces or + decode the ids and tokenize the output. By default, we use ngram order of 4 + and use brevity penalty. Also, this does not have beam search. + + Args: + logits: Tensor of size [batch_size, length_logits, vocab_size] + labels: Tensor of size [batch-size, length_labels] + + Returns: + bleu: int, approx bleu score + """ + predictions = np.argmax(logits, axis=-1) + bleu = metrics.compute_bleu(labels, predictions) + return bleu + + +def continuous_eval(strategy, + params, + model_type, + eval_file_pattern=None, + batch_size=4, + eval_steps=None, + model_dir=None, + timeout=3000): + """Continuously evaluate checkpoints on testing data.""" + test_dataset = input_pipeline.get_input_dataset( + eval_file_pattern, + batch_size=batch_size, + params=params, + is_training=False, + strategy=strategy) + + with strategy.scope(): + model = models.create_model(model_type, params) + metric_layer = metrics_v2.MetricLayer(params.vocab_size) + eval_summary_writer = tf.summary.create_file_writer( + os.path.join(model_dir, "summaries/eval")) + global_step = tf.Variable( + 0, + trainable=False, + dtype=tf.int64, + aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA, + shape=[]) + model.global_step = global_step + + @tf.function + def test_step(inputs): + """Calculates evaluation metrics on distributed devices.""" + + def _test_step_fn(inputs): + """Replicated accuracy calculation.""" + targets = models.remove_sos_from_seq(inputs["target_ids"], + params.pad_token_id) + + # Using ground truth sequences as targets to calculate logits for accuracy + # and perplexity metrics. + logits, _, _ = model(inputs, training=False, mode="train") + metric_layer([logits, targets]) + + # Get logits from top beam search results for bleu and rouge metrics. + logits = model(inputs, training=False, mode="eval") + + return targets, logits + + outputs = strategy.run(_test_step_fn, args=(inputs,)) + + return tf.nest.map_structure(strategy.experimental_local_results, outputs) + + metrics_and_funcs = [ + (tf.keras.metrics.Mean("bleu", dtype=tf.float32), bleu_score), + (tf.keras.metrics.Mean("rouge_2_fscore", + dtype=tf.float32), rouge_2_fscore), + (tf.keras.metrics.Mean("rouge_l_fscore", + dtype=tf.float32), rouge_l_fscore), + ] + eval_results = {} + for latest_checkpoint in tf.train.checkpoints_iterator( + model_dir, timeout=timeout): + checkpoint = tf.train.Checkpoint(model=model) + checkpoint.restore(latest_checkpoint).expect_partial() + logging.info("Loaded checkpoint %s", latest_checkpoint) + + for i, inputs in enumerate(test_dataset): + if eval_steps and i >= eval_steps: + break + outputs = test_step(inputs) + for metric, func in metrics_and_funcs: + for targets, logits in zip(outputs[0], outputs[1]): + metric.update_state(func(logits.numpy(), targets.numpy())) + + with eval_summary_writer.as_default(): + step = model.global_step.numpy() + for metric, _ in metrics_and_funcs: + eval_results[metric.name] = metric.result().numpy().astype(float) + tf.summary.scalar( + metric.name, + eval_results[metric.name], + step=step) + for metric in metric_layer.metrics: + eval_results[metric.name] = metric.result().numpy().astype(float) + tf.summary.scalar( + metric.name, + eval_results[metric.name], + step=step) + logging.info("Step %d Metrics= %s", step, str(eval_results)) + eval_summary_writer.flush() + + # Resets metrics. + for metric, _ in metrics_and_funcs: + metric.reset_states() + for metric in metric_layer.metrics: + metric.reset_states() + return eval_results diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/input_pipeline.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/input_pipeline.py new file mode 100644 index 0000000..04f7890 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/input_pipeline.py @@ -0,0 +1,254 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Input pipelines.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow.compat.v2 as tf + + +def decode_record(record, name_to_features): + """Decodes a record to a TensorFlow example.""" + example = tf.io.parse_single_example(record, name_to_features) + + # tf.Example only supports tf.int64, but the TPU only supports tf.int32. + # So cast all int64 to int32. + for name in list(example.keys()): + t = example[name] + if t.dtype == tf.int64: + t = tf.cast(t, tf.int32) + example[name] = t + + return example + + +def process_singledoc_dataset(dataset, batch_size, params): + """Parses and batches single-doc dataset.""" + name_to_features = { + "input_ids_a": tf.io.FixedLenFeature([params.len_title], tf.int64), + "input_ids_b": tf.io.FixedLenFeature([params.len_passage], tf.int64), + "input_mask_b": tf.io.FixedLenFeature([params.len_passage], tf.int64), + "segment_ids_b": tf.io.FixedLenFeature([params.len_passage], tf.int64), + } + decode_fn = lambda record: decode_record(record, name_to_features) + dataset = dataset.map( + decode_fn, num_parallel_calls=tf.data.experimental.AUTOTUNE) + + def _select_data_from_record(record): + """Filter out features to use for pretraining.""" + return { + "input_ids": record["input_ids_b"], + "input_mask": record["input_mask_b"], + "segment_ids": record["segment_ids_b"], + "target_ids": record["input_ids_a"], + } + + dataset = dataset.map( + _select_data_from_record, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + dataset = dataset.batch(batch_size, drop_remainder=True) + return dataset + + +def decode_sparse_record(record, name_to_features): + """Decodes a sparse record to a TensorFlow example.""" + example = tf.io.parse_single_example(record, name_to_features) + + # tf.Example only supports tf.int64, but the TPU only supports tf.int32. + # So cast all int64 to int32. + for name in list(example.keys()): + t = example[name] + if t.dtype == tf.int64: + t = tf.cast(t, tf.int32) + example[name] = tf.sparse.to_dense(t) + + return example + + +def _filter_max_length(example, max_title_length=256): + """Indicates whether the example's length is lower than the maximum length.""" + return tf.size(example["targets"]) <= max_title_length + + +def process_singledoc_transformer_dataset(dataset, batch_size, params): + """Parses, batches and pads single-doc dataset.""" + name_to_features = { + "inputs": tf.io.VarLenFeature(tf.int64), + "targets": tf.io.VarLenFeature(tf.int64), + } + decode_fn = lambda record: decode_sparse_record(record, name_to_features) + dataset = dataset.map( + decode_fn, num_parallel_calls=tf.data.experimental.AUTOTUNE) + + def _select_data_from_record(record): + """Filter out features to use for pretraining.""" + input_ids = record["inputs"][:params.len_passage] + target_ids = record["targets"] + input_mask = tf.ones_like(input_ids) + segment_ids = tf.zeros_like(input_ids) + return { + "input_ids": input_ids, + "input_mask": input_mask, + "segment_ids": segment_ids, + "target_ids": target_ids, + } + + dataset = dataset.filter(lambda x: _filter_max_length(x, params.len_title)) + + dataset = dataset.map( + _select_data_from_record, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + + dataset = dataset.padded_batch( + batch_size, { + "input_ids": [params.len_passage], + "input_mask": [params.len_passage], + "segment_ids": [params.len_passage], + "target_ids": [params.len_title], + }, + padding_values={ + "input_ids": params.pad_token_id, + "input_mask": 0, + "segment_ids": 0, + "target_ids": params.pad_token_id, + }, + drop_remainder=True) + + return dataset + + +def multidoc_parse_spec(params, training=True): + """Gets the mutli-doc tf.Example parsing spec.""" + len_p = params.len_passage + name_to_features = {} + feature_list = ["input_ids", "input_mask", "segment_ids"] + for idx in params.passage_list: + for feature in feature_list: + name_to_features["%s_%s" % (feature, idx)] = tf.io.FixedLenFeature( + [len_p], tf.int64) + if training: + # Cluster title. + name_to_features["input_ids_a"] = tf.io.FixedLenFeature([params.len_title], + tf.int64) + return name_to_features, feature_list + + +def process_multidoc_dataset(dataset, batch_size, params): + """Parses, organizes and batches multi-doc dataset.""" + name_to_features, feature_list = multidoc_parse_spec(params) + decode_fn = lambda record: decode_record(record, name_to_features) + dataset = dataset.map( + decode_fn, num_parallel_calls=tf.data.experimental.AUTOTUNE) + + def _select_data_from_record(record): + """Filter out features to use for pretraining.""" + features = {"target_ids": record["input_ids_a"]} + for feature in feature_list: + tensors = [record["%s_%s" % (feature, i)] for i in params.passage_list] + features[feature] = tf.stack(tensors) + return features + + dataset = dataset.map( + _select_data_from_record, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + dataset = dataset.batch(batch_size, drop_remainder=True) + return dataset + + +def create_dataset(file_paths, + batch_size, + params, + is_training=True, + input_pipeline_context=None): + """Creates input dataset from (tf)records files for pretraining.""" + dataset = tf.data.Dataset.list_files(file_paths, shuffle=is_training) + + if input_pipeline_context and input_pipeline_context.num_input_pipelines > 1: + if not is_training or params.input_sharding: + dataset = dataset.shard(input_pipeline_context.num_input_pipelines, + input_pipeline_context.input_pipeline_id) + + if is_training: + dataset = dataset.repeat() + # We set shuffle buffer to exactly match total number of + # training files to ensure that training data is well shuffled. + dataset = dataset.shuffle(len(file_paths)) + + # In parallel, create tf record dataset for each train files. + # cycle_length = 8 means that up to 8 files will be read and deserialized in + # parallel. You may want to increase this number if you have a large number of + # CPU cores. + dataset = dataset.interleave( + tf.data.TFRecordDataset, + cycle_length=8, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + + if is_training: + dataset = dataset.shuffle(100) + + if params.get("multi_channel_cross_attention", value=False): + dataset = process_multidoc_dataset(dataset, batch_size, params) + else: + if not params.input_data_not_padded: + dataset = process_singledoc_dataset(dataset, batch_size, params) + else: + dataset = process_singledoc_transformer_dataset(dataset, batch_size, + params) + dataset = dataset.prefetch(1024) + return dataset + + +def get_input_dataset(input_file_pattern, + batch_size, + params, + is_training, + strategy=None): + """Returns input dataset from input file string.""" + + # When using TPU pods, we need to clone dataset across + # workers and need to pass in function that returns the dataset rather + # than passing dataset instance itself. + use_dataset_fn = isinstance(strategy, tf.distribute.experimental.TPUStrategy) + if use_dataset_fn: + if batch_size % strategy.num_replicas_in_sync != 0: + raise ValueError( + "Batch size must be divisible by number of replicas : {}".format( + strategy.num_replicas_in_sync)) + + # As auto rebatching is not supported in + # `experimental_distribute_datasets_from_function()` API, which is + # required when cloning dataset to multiple workers in eager mode, + # we use per-replica batch size. + batch_size = int(batch_size / strategy.num_replicas_in_sync) + + def _dataset_fn(ctx=None): + """Returns tf.data.Dataset for distributed BERT pretraining.""" + input_files = [] + for input_pattern in input_file_pattern.split(","): + input_files.extend(tf.io.gfile.glob(input_pattern)) + + return create_dataset( + input_files, + batch_size, + params, + is_training=is_training, + input_pipeline_context=ctx) + + if use_dataset_fn: + return strategy.experimental_distribute_datasets_from_function(_dataset_fn) + else: + return strategy.experimental_distribute_dataset(_dataset_fn()) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/models.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/models.py new file mode 100644 index 0000000..52b6c43 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/models.py @@ -0,0 +1,590 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""tf.keras Models for NHNet.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +from absl import logging +import gin +import tensorflow as tf +from typing import Optional, Text + +from official.modeling import tf_utils +from official.modeling.hyperparams import params_dict +from official.nlp.modeling import networks +from official.nlp.nhnet import configs +from official.nlp.nhnet import decoder +from official.nlp.nhnet import multi_channel_attention +from official.nlp.nhnet import utils +from official.nlp.transformer import beam_search + + +def embedding_linear(embedding_matrix, x): + """Uses embeddings as linear transformation weights.""" + with tf.name_scope("presoftmax_linear"): + batch_size = tf.shape(x)[0] + length = tf.shape(x)[1] + hidden_size = tf.shape(x)[2] + vocab_size = tf.shape(embedding_matrix)[0] + + x = tf.reshape(x, [-1, hidden_size]) + logits = tf.matmul(x, embedding_matrix, transpose_b=True) + + return tf.reshape(logits, [batch_size, length, vocab_size]) + + +def _add_sos_to_seq(seq, start_token_id): + """Add a start sequence token while keeping seq length.""" + batch_size = tf.shape(seq)[0] + seq_len = tf.shape(seq)[1] + sos_ids = tf.ones([batch_size], tf.int32) * start_token_id + targets = tf.concat([tf.expand_dims(sos_ids, axis=1), seq], axis=1) + targets = targets[:, :-1] + tf.assert_equal(tf.shape(targets), (batch_size, seq_len)) + return targets + + +def remove_sos_from_seq(seq, pad_token_id): + """Remove the start sequence token while keeping seq length.""" + batch_size, seq_len = tf_utils.get_shape_list(seq, expected_rank=2) + # remove + targets = seq[:, 1:] + # pad + pad_ids = tf.ones([batch_size], tf.int32) * pad_token_id + targets = tf.concat([targets, tf.expand_dims(pad_ids, axis=1)], axis=1) + tf.assert_equal(tf.shape(targets), (batch_size, seq_len)) + return targets + + +class Bert2Bert(tf.keras.Model): + """Bert2Bert encoder decoder model for training.""" + + def __init__(self, params, bert_layer, decoder_layer, name=None): + super(Bert2Bert, self).__init__(name=name) + self.params = params + if not bert_layer.built: + raise ValueError("bert_layer should be built.") + if not decoder_layer.built: + raise ValueError("decoder_layer should be built.") + self.bert_layer = bert_layer + self.decoder_layer = decoder_layer + + def get_config(self): + return {"params": self.params.as_dict()} + + def get_decode_logits(self, + decoder_inputs, + ids, + decoder_self_attention_bias, + step, + cache=None): + if cache: + if self.params.get("padded_decode", False): + bias_shape = decoder_self_attention_bias.shape.as_list() + self_attention_bias = tf.slice( + decoder_self_attention_bias, [0, 0, step, 0], + [bias_shape[0], bias_shape[1], 1, bias_shape[3]]) + else: + self_attention_bias = decoder_self_attention_bias[:, :, step:step + + 1, :step + 1] + # Sets decoder input to the last generated IDs. + decoder_input = ids[:, -1:] + else: + self_attention_bias = decoder_self_attention_bias[:, :, :step + 1, :step + + 1] + decoder_input = ids + decoder_inputs["target_ids"] = decoder_input + decoder_inputs["self_attention_bias"] = self_attention_bias + if cache: + decoder_outputs = self.decoder_layer( + decoder_inputs, + cache, + decode_loop_step=step, + padded_decode=self.params.get("padded_decode", False)) + else: + decoder_outputs = self.decoder_layer(decoder_inputs) + logits = embedding_linear(self.decoder_layer.embedding_lookup.embeddings, + decoder_outputs[:, -1:, :]) + logits = tf.squeeze(logits, axis=[1]) + return logits + + def _get_symbols_to_logits_fn(self, max_decode_length): + """Returns a decoding function that calculates logits of the next tokens.""" + # Max decode length should be smaller than the positional embedding max + # sequence length. + decoder_self_attention_bias = decoder.get_attention_bias( + input_tensor=None, + bias_type="decoder_self", + max_length=max_decode_length) + + def _symbols_to_logits_fn(ids, i, cache): + """Generate logits for next candidate IDs. + + Args: + ids: Current decoded sequences. int tensor with shape [batch_size * + beam_size, i + 1] + i: Loop index + cache: dictionary of values storing the encoder output, encoder-decoder + attention bias, and previous decoder attention values. + + Returns: + Tuple of + (logits with shape [batch_size * beam_size, vocab_size], + updated cache values) + """ + decoder_inputs = dict( + all_encoder_outputs=cache["all_encoder_outputs"], + attention_bias=cache["attention_bias"]) + logits = self.get_decode_logits( + decoder_inputs, + ids, + decoder_self_attention_bias, + step=i, + cache=cache if self.params.use_cache else None) + return logits, cache + + return _symbols_to_logits_fn + + def train_decode(self, decode_outputs): + logits = embedding_linear(self.decoder_layer.embedding_lookup.embeddings, + decode_outputs) + decode_output_ids = tf.cast(tf.argmax(logits, axis=-1), tf.int32) + output_log_probs = tf.nn.log_softmax(logits, axis=-1) + return logits, decode_output_ids, output_log_probs + + def predict_decode(self, start_token_ids, cache): + symbols_to_logits_fn = self._get_symbols_to_logits_fn(self.params.len_title) + # Use beam search to find the top beam_size sequences and scores. + decoded_ids, scores = beam_search.sequence_beam_search( + symbols_to_logits_fn=symbols_to_logits_fn, + initial_ids=start_token_ids, + initial_cache=cache, + vocab_size=self.params.vocab_size, + beam_size=self.params.beam_size, + alpha=self.params.alpha, + max_decode_length=self.params.len_title, + padded_decode=self.params.get("padded_decode", False), + eos_id=self.params.end_token_id) + return decoded_ids, scores + + def _get_logits_for_decode_ids(self, decoder_inputs, top_decoded_ids): + """Returns the log probabilities for ids.""" + target_ids = _add_sos_to_seq(top_decoded_ids, self.params.start_token_id) + decoder_inputs["self_attention_bias"] = decoder.get_attention_bias( + target_ids, bias_type="decoder_self") + decoder_inputs["target_ids"] = target_ids + decoder_outputs = self.decoder_layer(decoder_inputs) + logits = embedding_linear(self.decoder_layer.embedding_lookup.embeddings, + decoder_outputs) + return logits + + def _init_cache(self, batch_size): + num_heads = self.params.num_decoder_attn_heads + dim_per_head = self.params.hidden_size // num_heads + init_decode_length = ( + self.params.len_title if self.params.get("padded_decode", False) else 0) + cache = {} + for layer in range(self.params.num_decoder_layers): + cache[str(layer)] = { + "key": + tf.zeros( + [batch_size, init_decode_length, num_heads, dim_per_head], + dtype=tf.float32), + "value": + tf.zeros( + [batch_size, init_decode_length, num_heads, dim_per_head], + dtype=tf.float32) + } + return cache + + def call(self, inputs, mode="train"): + """Implements call(). + + Args: + inputs: a dictionary of tensors. + mode: string, an enum for mode, train/eval. + + Returns: + logits, decode_output_ids, output_log_probs for training. top_decoded_ids + for eval. + """ + input_ids = inputs["input_ids"] + input_mask = inputs["input_mask"] + segment_ids = inputs["segment_ids"] + all_encoder_outputs, _ = self.bert_layer( + [input_ids, input_mask, segment_ids]) + + if mode not in ("train", "eval", "predict"): + raise ValueError("Invalid call mode: %s" % mode) + encoder_decoder_attention_bias = decoder.get_attention_bias( + input_ids, + bias_type="single_cross", + padding_value=self.params.pad_token_id) + if mode == "train": + self_attention_bias = decoder.get_attention_bias( + inputs["target_ids"], bias_type="decoder_self") + decoder_inputs = dict( + attention_bias=encoder_decoder_attention_bias, + all_encoder_outputs=all_encoder_outputs, + target_ids=inputs["target_ids"], + self_attention_bias=self_attention_bias) + decoder_outputs = self.decoder_layer(decoder_inputs) + return self.train_decode(decoder_outputs) + + batch_size = tf.shape(input_ids)[0] + start_token_ids = tf.ones([batch_size], + tf.int32) * self.params.start_token_id + # Add encoder output and attention bias to the cache. + if self.params.use_cache: + cache = self._init_cache(batch_size) + else: + cache = {} + cache["all_encoder_outputs"] = all_encoder_outputs + cache["attention_bias"] = encoder_decoder_attention_bias + decoded_ids, scores = self.predict_decode(start_token_ids, cache) + if mode == "predict": + return decoded_ids[:, :self.params.beam_size, + 1:], scores[:, :self.params.beam_size] + + decoder_inputs = dict( + attention_bias=encoder_decoder_attention_bias, + all_encoder_outputs=all_encoder_outputs) + top_decoded_ids = decoded_ids[:, 0, 1:] + return self._get_logits_for_decode_ids(decoder_inputs, top_decoded_ids) + + +class NHNet(Bert2Bert): + """NHNet model which performs multi-doc decoding.""" + + def __init__(self, params, bert_layer, decoder_layer, name=None): + super(NHNet, self).__init__(params, bert_layer, decoder_layer, name=name) + self.doc_attention = multi_channel_attention.DocAttention( + num_heads=params.num_decoder_attn_heads, + head_size=params.hidden_size // params.num_decoder_attn_heads) + + def _expand_doc_attention_probs(self, doc_attention_probs, target_length): + """Expands doc attention probs to fit the decoding sequence length.""" + doc_attention_probs = tf.expand_dims( + doc_attention_probs, axis=[1]) # [B, 1, A] + doc_attention_probs = tf.expand_dims( + doc_attention_probs, axis=[2]) # [B, 1, 1, A] + return tf.tile(doc_attention_probs, + [1, self.params.num_decoder_attn_heads, target_length, 1]) + + def _get_symbols_to_logits_fn(self, max_decode_length): + """Returns a decoding function that calculates logits of the next tokens.""" + # Max decode length should be smaller than the positional embedding max + # sequence length. + decoder_self_attention_bias = decoder.get_attention_bias( + input_tensor=None, + bias_type="decoder_self", + max_length=max_decode_length) + + def _symbols_to_logits_fn(ids, i, cache): + """Generate logits for next candidate IDs.""" + if self.params.use_cache: + target_length = 1 + else: + target_length = i + 1 + decoder_inputs = dict( + doc_attention_probs=self._expand_doc_attention_probs( + cache["doc_attention_probs"], target_length), + all_encoder_outputs=cache["all_encoder_outputs"], + attention_bias=cache["attention_bias"]) + logits = self.get_decode_logits( + decoder_inputs, + ids, + decoder_self_attention_bias, + step=i, + cache=cache if self.params.use_cache else None) + return logits, cache + + return _symbols_to_logits_fn + + def call(self, inputs, mode="training"): + input_shape = tf_utils.get_shape_list(inputs["input_ids"], expected_rank=3) + batch_size, num_docs, len_passage = (input_shape[0], input_shape[1], + input_shape[2]) + input_ids = tf.reshape(inputs["input_ids"], [-1, len_passage]) + input_mask = tf.reshape(inputs["input_mask"], [-1, len_passage]) + segment_ids = tf.reshape(inputs["segment_ids"], [-1, len_passage]) + all_encoder_outputs, _ = self.bert_layer( + [input_ids, input_mask, segment_ids]) + encoder_outputs = tf.reshape( + all_encoder_outputs[-1], + [batch_size, num_docs, len_passage, self.params.hidden_size]) + doc_attention_mask = tf.reshape( + tf.cast( + tf.math.count_nonzero(input_mask, axis=1, dtype=tf.int32) > 2, + tf.int32), [batch_size, num_docs]) + + doc_attention_probs = self.doc_attention(encoder_outputs, + doc_attention_mask) + encoder_decoder_attention_bias = decoder.get_attention_bias( + inputs["input_ids"], + bias_type="multi_cross", + padding_value=self.params.pad_token_id) + + if mode == "train": + target_length = tf_utils.get_shape_list( + inputs["target_ids"], expected_rank=2)[1] + doc_attention_probs = self._expand_doc_attention_probs( + doc_attention_probs, target_length) + self_attention_bias = decoder.get_attention_bias( + inputs["target_ids"], bias_type="decoder_self") + decoder_inputs = dict( + attention_bias=encoder_decoder_attention_bias, + self_attention_bias=self_attention_bias, + target_ids=inputs["target_ids"], + all_encoder_outputs=encoder_outputs, + doc_attention_probs=doc_attention_probs) + decoder_outputs = self.decoder_layer(decoder_inputs) + return self.train_decode(decoder_outputs) + + # Adds encoder output and attention bias to the cache. + if self.params.use_cache: + cache = self._init_cache(batch_size) + else: + cache = {} + cache["all_encoder_outputs"] = [encoder_outputs] + cache["attention_bias"] = encoder_decoder_attention_bias + cache["doc_attention_probs"] = doc_attention_probs + + start_token_ids = tf.ones([batch_size], + tf.int32) * self.params.start_token_id + decoded_ids, scores = self.predict_decode(start_token_ids, cache) + if mode == "predict": + return decoded_ids[:, :self.params.beam_size, + 1:], scores[:, :self.params.beam_size] + + top_decoded_ids = decoded_ids[:, 0, 1:] + target_length = tf_utils.get_shape_list(top_decoded_ids)[-1] + decoder_inputs = dict( + attention_bias=encoder_decoder_attention_bias, + all_encoder_outputs=[encoder_outputs], + doc_attention_probs=self._expand_doc_attention_probs( + doc_attention_probs, target_length)) + return self._get_logits_for_decode_ids(decoder_inputs, top_decoded_ids) + + +def get_bert2bert_layers(params: configs.BERT2BERTConfig): + """Creates a Bert2Bert stem model and returns Bert encoder/decoder. + + We use funtional-style to create stem model because we need to make all layers + built to restore variables in a customized way. The layers are called with + placeholder inputs to make them fully built. + + Args: + params: ParamsDict. + + Returns: + two keras Layers, bert_model_layer and decoder_layer + """ + input_ids = tf.keras.layers.Input( + shape=(None,), name="input_ids", dtype=tf.int32) + input_mask = tf.keras.layers.Input( + shape=(None,), name="input_mask", dtype=tf.int32) + segment_ids = tf.keras.layers.Input( + shape=(None,), name="segment_ids", dtype=tf.int32) + target_ids = tf.keras.layers.Input( + shape=(None,), name="target_ids", dtype=tf.int32) + bert_config = utils.get_bert_config_from_params(params) + bert_model_layer = networks.TransformerEncoder( + vocab_size=bert_config.vocab_size, + hidden_size=bert_config.hidden_size, + num_layers=bert_config.num_hidden_layers, + num_attention_heads=bert_config.num_attention_heads, + intermediate_size=bert_config.intermediate_size, + activation=tf_utils.get_activation(bert_config.hidden_act), + dropout_rate=bert_config.hidden_dropout_prob, + attention_dropout_rate=bert_config.attention_probs_dropout_prob, + sequence_length=None, + max_sequence_length=bert_config.max_position_embeddings, + type_vocab_size=bert_config.type_vocab_size, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=bert_config.initializer_range), + return_all_encoder_outputs=True, + name="bert_encoder") + all_encoder_outputs, _ = bert_model_layer( + [input_ids, input_mask, segment_ids]) + # pylint: disable=protected-access + decoder_layer = decoder.Decoder(params, bert_model_layer._embedding_layer) + # pylint: enable=protected-access + cross_attention_bias = decoder.AttentionBias(bias_type="single_cross")( + input_ids) + self_attention_bias = decoder.AttentionBias(bias_type="decoder_self")( + target_ids) + decoder_inputs = dict( + attention_bias=cross_attention_bias, + self_attention_bias=self_attention_bias, + target_ids=target_ids, + all_encoder_outputs=all_encoder_outputs) + _ = decoder_layer(decoder_inputs) + + return bert_model_layer, decoder_layer + + +def get_nhnet_layers(params: configs.NHNetConfig): + """Creates a Mult-doc encoder/decoder. + + Args: + params: ParamsDict. + + Returns: + two keras Layers, bert_model_layer and decoder_layer + """ + input_ids = tf.keras.layers.Input( + shape=(None,), name="input_ids", dtype=tf.int32) + input_mask = tf.keras.layers.Input( + shape=(None,), name="input_mask", dtype=tf.int32) + segment_ids = tf.keras.layers.Input( + shape=(None,), name="segment_ids", dtype=tf.int32) + bert_config = utils.get_bert_config_from_params(params) + bert_model_layer = networks.TransformerEncoder( + vocab_size=bert_config.vocab_size, + hidden_size=bert_config.hidden_size, + num_layers=bert_config.num_hidden_layers, + num_attention_heads=bert_config.num_attention_heads, + intermediate_size=bert_config.intermediate_size, + activation=tf_utils.get_activation(bert_config.hidden_act), + dropout_rate=bert_config.hidden_dropout_prob, + attention_dropout_rate=bert_config.attention_probs_dropout_prob, + sequence_length=None, + max_sequence_length=bert_config.max_position_embeddings, + type_vocab_size=bert_config.type_vocab_size, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=bert_config.initializer_range), + return_all_encoder_outputs=True, + name="bert_encoder") + bert_model_layer([input_ids, input_mask, segment_ids]) + + input_ids = tf.keras.layers.Input( + shape=(None, None), name="input_ids", dtype=tf.int32) + all_encoder_outputs = tf.keras.layers.Input((None, None, params.hidden_size), + dtype=tf.float32) + target_ids = tf.keras.layers.Input( + shape=(None,), name="target_ids", dtype=tf.int32) + doc_attention_probs = tf.keras.layers.Input( + (params.num_decoder_attn_heads, None, None), dtype=tf.float32) + # pylint: disable=protected-access + decoder_layer = decoder.Decoder(params, bert_model_layer._embedding_layer) + # pylint: enable=protected-access + cross_attention_bias = decoder.AttentionBias(bias_type="multi_cross")( + input_ids) + self_attention_bias = decoder.AttentionBias(bias_type="decoder_self")( + target_ids) + decoder_inputs = dict( + attention_bias=cross_attention_bias, + self_attention_bias=self_attention_bias, + target_ids=target_ids, + all_encoder_outputs=all_encoder_outputs, + doc_attention_probs=doc_attention_probs) + _ = decoder_layer(decoder_inputs) + + return bert_model_layer, decoder_layer + + +def create_transformer_model(params, + init_checkpoint: Optional[Text] = None + ) -> tf.keras.Model: + """A helper to create Transformer model.""" + bert_layer, decoder_layer = get_bert2bert_layers(params=params) + model = Bert2Bert( + params=params, + bert_layer=bert_layer, + decoder_layer=decoder_layer, + name="transformer") + + if init_checkpoint: + logging.info( + "Checkpoint file %s found and restoring from " + "initial checkpoint.", init_checkpoint) + ckpt = tf.train.Checkpoint(model=model) + ckpt.restore(init_checkpoint).expect_partial() + + return model + + +def create_bert2bert_model( + params: configs.BERT2BERTConfig, + cls=Bert2Bert, + init_checkpoint: Optional[Text] = None) -> tf.keras.Model: + """A helper to create Bert2Bert model.""" + bert_layer, decoder_layer = get_bert2bert_layers(params=params) + if init_checkpoint: + utils.initialize_bert2bert_from_pretrained_bert(bert_layer, decoder_layer, + init_checkpoint) + return cls( + params=params, + bert_layer=bert_layer, + decoder_layer=decoder_layer, + name="bert2bert") + + +def create_nhnet_model( + params: configs.NHNetConfig, + cls=NHNet, + init_checkpoint: Optional[Text] = None) -> tf.keras.Model: + """A helper to create NHNet model.""" + bert_layer, decoder_layer = get_nhnet_layers(params=params) + model = cls( + params=params, + bert_layer=bert_layer, + decoder_layer=decoder_layer, + name="nhnet") + if init_checkpoint: + logging.info( + "Checkpoint file %s found and restoring from " + "initial checkpoint.", init_checkpoint) + if params.init_from_bert2bert: + ckpt = tf.train.Checkpoint(model=model) + ckpt.restore(init_checkpoint).assert_existing_objects_matched() + else: + utils.initialize_bert2bert_from_pretrained_bert(bert_layer, decoder_layer, + init_checkpoint) + return model + + +@gin.configurable +def get_model_params(model: Optional[Text] = "bert2bert", + config_class=None) -> params_dict.ParamsDict: + """Helper function to convert config file to ParamsDict.""" + if model == "bert2bert": + return configs.BERT2BERTConfig() + elif model == "nhnet": + return configs.NHNetConfig() + elif config_class: + return config_class() + else: + raise KeyError("The model type is not defined: %s" % model) + + +@gin.configurable +def create_model(model_type: Text, + params, + init_checkpoint: Optional[Text] = None): + """A factory function to create different types of models.""" + if model_type == "bert2bert": + return create_bert2bert_model(params, init_checkpoint=init_checkpoint) + elif model_type == "nhnet": + return create_nhnet_model(params, init_checkpoint=init_checkpoint) + elif "transformer" in model_type: + return create_transformer_model( + params, init_checkpoint=init_checkpoint) + else: + raise KeyError("The model type is not defined: %s" % model_type) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/multi_channel_attention.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/multi_channel_attention.py new file mode 100644 index 0000000..66b2a5e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/multi_channel_attention.py @@ -0,0 +1,153 @@ +# Lint as: python3 +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Multi-channel decoder.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import math + +import tensorflow as tf +from official.modeling import tf_utils +from official.nlp.modeling import layers + + +class DocAttention(tf.keras.layers.Layer): + """Documents Attention layer.""" + + def __init__(self, + num_heads, + head_size, + kernel_initializer="glorot_uniform", + bias_initializer="zeros", + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + **kwargs): + super(DocAttention, self).__init__(**kwargs) + self._num_heads = num_heads + self._head_size = head_size + self._kernel_initializer = tf.keras.initializers.get(kernel_initializer) + self._bias_initializer = tf.keras.initializers.get(bias_initializer) + self._kernel_regularizer = tf.keras.regularizers.get(kernel_regularizer) + self._bias_regularizer = tf.keras.regularizers.get(bias_regularizer) + self._kernel_constraint = tf.keras.constraints.get(kernel_constraint) + self._bias_constraint = tf.keras.constraints.get(bias_constraint) + + def build(self, unused_input_shapes): + self._query_dense = layers.DenseEinsum( + output_shape=(self._num_heads, self._head_size), + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + activity_regularizer=self._activity_regularizer, + kernel_constraint=self._kernel_constraint, + bias_constraint=self._bias_constraint, + dtype=self.dtype, + name="encdocatt_query") + self._key_dense = layers.DenseEinsum( + output_shape=(self._num_heads, self._head_size), + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + activity_regularizer=self._activity_regularizer, + kernel_constraint=self._kernel_constraint, + bias_constraint=self._bias_constraint, + dtype=self.dtype, + name="encdocatt_key") + super(DocAttention, self).build(unused_input_shapes) + + def call(self, encoder_outputs, doc_attention_mask): + num_docs = tf_utils.get_shape_list(encoder_outputs, expected_rank=[4])[1] + cls_embeddings = encoder_outputs[:, :, 0, :] + key = self._key_dense(cls_embeddings) + query = self._query_dense(cls_embeddings) + doc_attention_mask = tf.cast(doc_attention_mask, tf.float32) + + key = tf.einsum("BANH,BA->BANH", key, doc_attention_mask) + query = tf.einsum("BANH,BA->BANH", query, doc_attention_mask) + attention_matrix = tf.einsum("BXNH,BYNH->BNXY", query, key) + mask = tf.ones([num_docs, num_docs]) + mask = tf.linalg.set_diag(mask, tf.zeros(num_docs)) + attention_matrix = tf.einsum("BNXY,XY->BNXY", attention_matrix, mask) + doc_attention_probs = tf.einsum("BNAY->BNA", attention_matrix) + doc_attention_probs = tf.einsum("BNA->BA", doc_attention_probs) + infadder = (1.0 - doc_attention_mask) * -100000.0 + return tf.nn.softmax(doc_attention_probs + infadder) + + +class MultiChannelAttention(layers.MultiHeadAttention): + """Multi-channel Attention layer.""" + + def __init__(self, num_heads, head_size, **kwargs): + super(MultiChannelAttention, self).__init__(num_heads, head_size, **kwargs) + self._masked_softmax = layers.MaskedSoftmax(mask_expansion_axes=[2]) + + def compute_output_shape(self, input_shape): + if len(input_shape) != 4: + raise ValueError("Layer %s must have 4 input tensors." % self.name) + from_tensor_shape = tf.TensorShape(input_shape[0]) + batch = from_tensor_shape[0] + from_tensor_length = from_tensor_shape[1] + return tf.TensorShape( + (batch, from_tensor_length, self._num_heads, self._head_size)) + + def call(self, inputs): + from_tensor = inputs[0] + to_tensor = inputs[1] + attention_mask = inputs[2] + doc_attention_probs = inputs[3] + + # Scalar dimensions referenced here: + # B = batch size (number of stories) + # A = num_docs (number of docs) + # F = `from_tensor` sequence length + # T = `to_tensor` sequence length + # N = `num_attention_heads` + # H = `size_per_head` + # `query_tensor` = [B, F, N ,H] + query_tensor = self._query_dense(from_tensor) + + # `key_tensor` = [B, A, T, N, H] + key_tensor = self._key_dense(to_tensor) + + # `value_tensor` = [B, A, T, N, H] + value_tensor = self._value_dense(to_tensor) + + # Take the dot product between "query" and "key" to get the raw + # attention scores. + attention_scores = tf.einsum("BATNH,BFNH->BANFT", key_tensor, query_tensor) + attention_scores = tf.multiply(attention_scores, + 1.0 / math.sqrt(float(self._head_size))) + + # Normalize the attention scores to probabilities. + # `attention_probs` = [B, A, N, F, T] + attention_probs = self._masked_softmax([attention_scores, attention_mask]) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self._dropout(attention_probs) + + # `context_layer` = [B, F, N, H] + context_layer = tf.einsum("BANFT,BATNH->BAFNH", attention_probs, + value_tensor) + return tf.einsum("BNFA,BAFNH->BFNH", doc_attention_probs, context_layer) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/multi_channel_attention_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/multi_channel_attention_test.py new file mode 100644 index 0000000..2a2e5d3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/multi_channel_attention_test.py @@ -0,0 +1,55 @@ +# Lint as: python3 +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for nlp.nhnet.multi_channel_attention.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf + +from official.nlp.nhnet import multi_channel_attention + + +class MultiChannelAttentionTest(tf.test.TestCase): + + def test_doc_attention(self): + num_heads = 2 + doc_attention = multi_channel_attention.DocAttention(num_heads, head_size=8) + num_docs = 3 + inputs = np.zeros((2, num_docs, 10, 16), dtype=np.float32) + doc_mask = np.zeros((2, num_docs), dtype=np.float32) + outputs = doc_attention(inputs, doc_mask) + self.assertEqual(outputs.shape, (2, num_docs)) + + def test_multi_channel_attention(self): + num_heads = 2 + num_docs = 5 + attention_layer = multi_channel_attention.MultiChannelAttention( + num_heads, head_size=2) + + from_data = 10 * np.random.random_sample((3, 4, 8)) + to_data = 10 * np.random.random_sample((3, num_docs, 2, 8)) + mask_data = np.random.randint(2, size=(3, num_docs, 4, 2)) + doc_probs = np.random.randint( + 2, size=(3, num_heads, 4, num_docs)).astype(float) + outputs = attention_layer([from_data, to_data, mask_data, doc_probs]) + self.assertEqual(outputs.shape, (3, 4, num_heads, 2)) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/optimizer.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/optimizer.py new file mode 100644 index 0000000..15c7e24 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/optimizer.py @@ -0,0 +1,82 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Optimizer and learning rate scheduler.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow as tf + +from official.modeling.hyperparams import params_dict + + +class LearningRateSchedule(tf.keras.optimizers.schedules.LearningRateSchedule): + """Learning rate schedule.""" + + def __init__(self, initial_learning_rate, hidden_size, warmup_steps): + """Initialize configuration of the learning rate schedule. + + Args: + initial_learning_rate: A float, the initial learning rate. + hidden_size: An integer, the model dimension in the hidden layers. + warmup_steps: An integer, the number of steps required for linear warmup. + """ + super(LearningRateSchedule, self).__init__() + self.initial_learning_rate = initial_learning_rate + self.hidden_size = hidden_size + self.warmup_steps = tf.cast(warmup_steps, tf.float32) + + def __call__(self, global_step): + """Calculate learning rate with linear warmup and rsqrt decay. + + Args: + global_step: An integer, the current global step used for learning rate + calculation. + + Returns: + A float, the learning rate needs to be used for current global step. + """ + with tf.name_scope('learning_rate_schedule'): + global_step = tf.cast(global_step, tf.float32) + learning_rate = self.initial_learning_rate + learning_rate *= (self.hidden_size**-0.5) + # Apply linear warmup + learning_rate *= tf.minimum(1.0, global_step / self.warmup_steps) + # Apply rsqrt decay + learning_rate /= tf.sqrt(tf.maximum(global_step, self.warmup_steps)) + return learning_rate + + def get_config(self): + """Get the configuration of the learning rate schedule.""" + return { + 'initial_learning_rate': self.initial_learning_rate, + 'hidden_size': self.hidden_size, + 'warmup_steps': self.warmup_steps, + } + + +def create_optimizer(params: params_dict.ParamsDict): + """Creates optimizer.""" + lr_schedule = LearningRateSchedule( + params.learning_rate, + params.hidden_size, + params.learning_rate_warmup_steps) + return tf.keras.optimizers.Adam( + learning_rate=lr_schedule, + beta_1=params.adam_beta1, + beta_2=params.adam_beta2, + epsilon=params.adam_epsilon) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/raw_data_process.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/raw_data_process.py new file mode 100644 index 0000000..9597043 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/raw_data_process.py @@ -0,0 +1,91 @@ +# Lint as: python3 +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Processes crawled content from news URLs by generating tfrecords.""" + +import os +from absl import app +from absl import flags +from official.nlp.nhnet import raw_data_processor + +FLAGS = flags.FLAGS + +flags.DEFINE_string("crawled_articles", "/tmp/nhnet/", + "Folder path to the crawled articles using news-please.") +flags.DEFINE_string("vocab", None, "Filepath of the BERT vocabulary.") +flags.DEFINE_bool("do_lower_case", True, + "Whether the vocabulary is uncased or not.") +flags.DEFINE_integer("len_title", 15, + "Maximum number of tokens in story headline.") +flags.DEFINE_integer("len_passage", 200, + "Maximum number of tokens in article passage.") +flags.DEFINE_integer("max_num_articles", 5, + "Maximum number of articles in a story.") +flags.DEFINE_bool("include_article_title_in_passage", False, + "Whether to include article title in article passage.") +flags.DEFINE_string("data_folder", None, + "Folder path to the downloaded data folder (output).") +flags.DEFINE_integer("num_tfrecords_shards", 20, + "Number of shards for train/valid/test.") + + +def transform_as_tfrecords(data_processor, filename): + """Transforms story from json to tfrecord (sharded). + + Args: + data_processor: Instance of RawDataProcessor. + filename: 'train', 'valid', or 'test'. + """ + print("Transforming json to tfrecord for %s..." % filename) + story_filepath = os.path.join(FLAGS.data_folder, filename + ".json") + output_folder = os.path.join(FLAGS.data_folder, "processed") + os.makedirs(output_folder, exist_ok=True) + output_filepaths = [] + for i in range(FLAGS.num_tfrecords_shards): + output_filepaths.append( + os.path.join( + output_folder, "%s.tfrecord-%.5d-of-%.5d" % + (filename, i, FLAGS.num_tfrecords_shards))) + (total_num_examples, + generated_num_examples) = data_processor.generate_examples( + story_filepath, output_filepaths) + print("For %s, %d examples have been generated from %d stories in json." % + (filename, generated_num_examples, total_num_examples)) + + +def main(_): + if not FLAGS.data_folder: + raise ValueError("data_folder must be set as the downloaded folder path.") + if not FLAGS.vocab: + raise ValueError("vocab must be set as the filepath of BERT vocabulary.") + data_processor = raw_data_processor.RawDataProcessor( + vocab=FLAGS.vocab, + do_lower_case=FLAGS.do_lower_case, + len_title=FLAGS.len_title, + len_passage=FLAGS.len_passage, + max_num_articles=FLAGS.max_num_articles, + include_article_title_in_passage=FLAGS.include_article_title_in_passage, + include_text_snippet_in_example=True) + print("Loading crawled articles...") + num_articles = data_processor.read_crawled_articles(FLAGS.crawled_articles) + print("Total number of articles loaded: %d" % num_articles) + print() + transform_as_tfrecords(data_processor, "train") + transform_as_tfrecords(data_processor, "valid") + transform_as_tfrecords(data_processor, "test") + + +if __name__ == "__main__": + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/raw_data_processor.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/raw_data_processor.py new file mode 100644 index 0000000..0a30532 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/raw_data_processor.py @@ -0,0 +1,228 @@ +# Lint as: python3 +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Library for processing crawled content and generating tfrecords.""" + +import collections +import json +import multiprocessing +import os +import urllib.parse +import tensorflow as tf + +from official.nlp.bert import tokenization +from official.nlp.data import classifier_data_lib + + +class RawDataProcessor(object): + """Data converter for story examples.""" + + def __init__(self, + vocab: str, + do_lower_case: bool, + len_title: int = 15, + len_passage: int = 200, + max_num_articles: int = 5, + include_article_title_in_passage: bool = False, + include_text_snippet_in_example: bool = False): + """Constructs a RawDataProcessor. + + Args: + vocab: Filepath of the BERT vocabulary. + do_lower_case: Whether the vocabulary is uncased or not. + len_title: Maximum number of tokens in story headline. + len_passage: Maximum number of tokens in article passage. + max_num_articles: Maximum number of articles in a story. + include_article_title_in_passage: Whether to include article title in + article passage. + include_text_snippet_in_example: Whether to include text snippet + (headline and article content) in generated tensorflow Examples, for + debug usage. If include_article_title_in_passage=True, title and body + will be separated by [SEP]. + """ + self.articles = dict() + self.tokenizer = tokenization.FullTokenizer( + vocab, do_lower_case=do_lower_case, split_on_punc=False) + self.len_title = len_title + self.len_passage = len_passage + self.max_num_articles = max_num_articles + self.include_article_title_in_passage = include_article_title_in_passage + self.include_text_snippet_in_example = include_text_snippet_in_example + # ex_index=5 deactivates printing inside convert_single_example. + self.ex_index = 5 + # Parameters used in InputExample, not used in NHNet. + self.label = 0 + self.guid = 0 + self.num_generated_examples = 0 + + def read_crawled_articles(self, folder_path): + """Reads crawled articles under folder_path.""" + for path, _, files in os.walk(folder_path): + for name in files: + if not name.endswith(".json"): + continue + url, article = self._get_article_content_from_json( + os.path.join(path, name)) + if not article.text_a: + continue + self.articles[RawDataProcessor.normalize_url(url)] = article + if len(self.articles) % 5000 == 0: + print("Number of articles loaded: %d\r" % len(self.articles), end="") + print() + return len(self.articles) + + def generate_examples(self, input_file, output_files): + """Loads story from input json file and exports examples in output_files.""" + writers = [] + story_partition = [] + for output_file in output_files: + writers.append(tf.io.TFRecordWriter(output_file)) + story_partition.append(list()) + with tf.io.gfile.GFile(input_file, "r") as story_json_file: + stories = json.load(story_json_file) + writer_index = 0 + for story in stories: + articles = [] + for url in story["urls"]: + normalized_url = RawDataProcessor.normalize_url(url) + if normalized_url in self.articles: + articles.append(self.articles[normalized_url]) + if not articles: + continue + story_partition[writer_index].append((story["label"], articles)) + writer_index = (writer_index + 1) % len(writers) + lock = multiprocessing.Lock() + pool = multiprocessing.pool.ThreadPool(len(writers)) + data = [(story_partition[i], writers[i], lock) for i in range(len(writers))] + pool.map(self._write_story_partition, data) + return len(stories), self.num_generated_examples + + @classmethod + def normalize_url(cls, url): + """Normalize url for better matching.""" + url = urllib.parse.unquote( + urllib.parse.urlsplit(url)._replace(query=None).geturl()) + output, part = [], None + for part in url.split("//"): + if part == "http:" or part == "https:": + continue + else: + output.append(part) + return "//".join(output) + + def _get_article_content_from_json(self, file_path): + """Returns (url, InputExample) keeping content extracted from file_path.""" + with tf.io.gfile.GFile(file_path, "r") as article_json_file: + article = json.load(article_json_file) + if self.include_article_title_in_passage: + return article["url"], classifier_data_lib.InputExample( + guid=self.guid, + text_a=article["title"], + text_b=article["maintext"], + label=self.label) + else: + return article["url"], classifier_data_lib.InputExample( + guid=self.guid, text_a=article["maintext"], label=self.label) + + def _write_story_partition(self, data): + """Writes stories in a partition into file.""" + for (story_headline, articles) in data[0]: + story_example = tf.train.Example( + features=tf.train.Features( + feature=self._get_single_story_features(story_headline, + articles))) + data[1].write(story_example.SerializeToString()) + data[2].acquire() + try: + self.num_generated_examples += 1 + if self.num_generated_examples % 1000 == 0: + print( + "Number of stories written: %d\r" % self.num_generated_examples, + end="") + finally: + data[2].release() + + def _get_single_story_features(self, story_headline, articles): + """Converts a list of articles to a tensorflow Example.""" + def get_text_snippet(article): + if article.text_b: + return " [SEP] ".join([article.text_a, article.text_b]) + else: + return article.text_a + + story_features = collections.OrderedDict() + story_headline_feature = classifier_data_lib.convert_single_example( + ex_index=self.ex_index, + example=classifier_data_lib.InputExample( + guid=self.guid, text_a=story_headline, label=self.label), + label_list=[self.label], + max_seq_length=self.len_title, + tokenizer=self.tokenizer) + if self.include_text_snippet_in_example: + story_headline_feature.label_id = story_headline + self._add_feature_with_suffix( + feature=story_headline_feature, + suffix="a", + story_features=story_features) + for (article_index, article) in enumerate(articles): + if article_index == self.max_num_articles: + break + article_feature = classifier_data_lib.convert_single_example( + ex_index=self.ex_index, + example=article, + label_list=[self.label], + max_seq_length=self.len_passage, + tokenizer=self.tokenizer) + if self.include_text_snippet_in_example: + article_feature.label_id = get_text_snippet(article) + suffix = chr(ord("b") + article_index) + self._add_feature_with_suffix( + feature=article_feature, suffix=suffix, story_features=story_features) + + # Adds empty features as placeholder. + for article_index in range(len(articles), self.max_num_articles): + suffix = chr(ord("b") + article_index) + empty_article = classifier_data_lib.InputExample( + guid=self.guid, text_a="", label=self.label) + empty_feature = classifier_data_lib.convert_single_example( + ex_index=self.ex_index, + example=empty_article, + label_list=[self.label], + max_seq_length=self.len_passage, + tokenizer=self.tokenizer) + if self.include_text_snippet_in_example: + empty_feature.label_id = "" + self._add_feature_with_suffix( + feature=empty_feature, suffix=suffix, story_features=story_features) + return story_features + + def _add_feature_with_suffix(self, feature, suffix, story_features): + """Appends suffix to feature names and fills in the corresponding values.""" + + def _create_int_feature(values): + return tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) + + def _create_string_feature(value): + return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) + + story_features["input_ids_%c" % suffix] = _create_int_feature( + feature.input_ids) + story_features["input_mask_%c" % suffix] = _create_int_feature( + feature.input_mask) + story_features["segment_ids_%c" % suffix] = _create_int_feature( + feature.segment_ids) + if self.include_text_snippet_in_example: + story_features["text_snippet_%c" % suffix] = _create_string_feature( + bytes(feature.label_id.encode())) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.html b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.html new file mode 100644 index 0000000..0a8549c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.html @@ -0,0 +1,3 @@ + + +Page Title 0 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.json b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.json new file mode 100644 index 0000000..b730859 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.json @@ -0,0 +1,5 @@ +{ + "title": "title for 0", + "maintext": "text snippet for 0", + "url": "http://url_000.html" +} diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.html b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.html new file mode 100644 index 0000000..7c8bb8d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.html @@ -0,0 +1,3 @@ + + +Page Title 1 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.json b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.json new file mode 100644 index 0000000..dbc2322 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.json @@ -0,0 +1,5 @@ +{ + "title": "title for 1", + "maintext": "text snippet for 1", + "url": "url_001.html" +} diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/stories.json b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/stories.json new file mode 100644 index 0000000..0618f3d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/stories.json @@ -0,0 +1,29 @@ +[ + { + "urls": [ + "http://url_000.html", + "http://url_001.html" + ], + "label": "headline 0" + }, + { + "urls": [ + "http://url_000.html", + "http://url_001.html" + ], + "label": "headline 1" + }, + { + "urls": [ + "http://url_002.html", + "http://url_001.html" + ], + "label": "headline 2" + }, + { + "urls": [ + "http://url_003.html" + ], + "label": "headline 3" + } +] diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/vocab.txt b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/vocab.txt new file mode 100644 index 0000000..dd708d7 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/testdata/vocab.txt @@ -0,0 +1,23 @@ +[UNK] +[CLS] +[SEP] +[MASK] +0 +1 +this +is +a +title +snippet +for +url +main +text +http +www +html +: +// +. +_ +headline diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/trainer.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/trainer.py new file mode 100644 index 0000000..e14c05e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/trainer.py @@ -0,0 +1,238 @@ +# Lint as: python3 +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Run NHNet model training and eval.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from absl import app +from absl import flags +from absl import logging +from six.moves import zip +import tensorflow as tf +from official.modeling.hyperparams import params_dict +from official.nlp.nhnet import evaluation +from official.nlp.nhnet import input_pipeline +from official.nlp.nhnet import models +from official.nlp.nhnet import optimizer +from official.nlp.transformer import metrics as transformer_metrics +from official.utils.misc import distribution_utils + +FLAGS = flags.FLAGS + + +def define_flags(): + """Defines command line flags used by NHNet trainer.""" + ## Required parameters + flags.DEFINE_enum("mode", "train", ["train", "eval", "train_and_eval"], + "Execution mode.") + flags.DEFINE_string("train_file_pattern", "", "Train file pattern.") + flags.DEFINE_string("eval_file_pattern", "", "Eval file pattern.") + flags.DEFINE_string( + "model_dir", None, + "The output directory where the model checkpoints will be written.") + + # Model training specific flags. + flags.DEFINE_enum( + "distribution_strategy", "mirrored", ["tpu", "mirrored"], + "Distribution Strategy type to use for training. `tpu` uses TPUStrategy " + "for running on TPUs, `mirrored` uses GPUs with single host.") + flags.DEFINE_string("tpu", "", "TPU address to connect to.") + flags.DEFINE_string( + "init_checkpoint", None, + "Initial checkpoint (usually from a pre-trained BERT model).") + flags.DEFINE_integer("train_steps", 100000, "Max train steps") + flags.DEFINE_integer("eval_steps", 32, "Number of eval steps per run.") + flags.DEFINE_integer("train_batch_size", 32, "Total batch size for training.") + flags.DEFINE_integer("eval_batch_size", 4, "Total batch size for evaluation.") + flags.DEFINE_integer( + "steps_per_loop", 1000, + "Number of steps per graph-mode loop. Only training step " + "happens inside the loop.") + flags.DEFINE_integer("checkpoint_interval", 2000, "Checkpointing interval.") + flags.DEFINE_integer("len_title", 15, "Title length.") + flags.DEFINE_integer("len_passage", 200, "Passage length.") + flags.DEFINE_integer("num_encoder_layers", 12, + "Number of hidden layers of encoder.") + flags.DEFINE_integer("num_decoder_layers", 12, + "Number of hidden layers of decoder.") + flags.DEFINE_string("model_type", "nhnet", + "Model type to choose a model configuration.") + flags.DEFINE_integer( + "num_nhnet_articles", 5, + "Maximum number of articles in NHNet, only used when model_type=nhnet") + flags.DEFINE_string( + "params_override", + default=None, + help=("a YAML/JSON string or a YAML file which specifies additional " + "overrides over the default parameters")) + + +# pylint: disable=protected-access + + +class Trainer(tf.keras.Model): + """A training only model.""" + + def __init__(self, model, params): + super(Trainer, self).__init__() + self.model = model + self.params = params + self._num_replicas_in_sync = tf.distribute.get_strategy( + ).num_replicas_in_sync + + def call(self, inputs, mode="train"): + return self.model(inputs, mode) + + def train_step(self, inputs): + """The logic for one training step.""" + with tf.GradientTape() as tape: + logits, _, _ = self(inputs, mode="train", training=True) + targets = models.remove_sos_from_seq(inputs["target_ids"], + self.params.pad_token_id) + loss = transformer_metrics.transformer_loss(logits, targets, + self.params.label_smoothing, + self.params.vocab_size) + # Scales the loss, which results in using the average loss across all + # of the replicas for backprop. + scaled_loss = loss / self._num_replicas_in_sync + + tvars = self.trainable_variables + grads = tape.gradient(scaled_loss, tvars) + self.optimizer.apply_gradients(list(zip(grads, tvars))) + return { + "training_loss": loss, + "learning_rate": self.optimizer._decayed_lr(var_dtype=tf.float32) + } + + +class SimpleCheckpoint(tf.keras.callbacks.Callback): + """Keras callback to save tf.train.Checkpoints.""" + + def __init__(self, checkpoint_manager): + super(SimpleCheckpoint, self).__init__() + self.checkpoint_manager = checkpoint_manager + + def on_epoch_end(self, epoch, logs=None): + step_counter = self.checkpoint_manager._step_counter.numpy() + self.checkpoint_manager.save(checkpoint_number=step_counter) + + +def train(params, strategy, dataset=None): + """Runs training.""" + + if not dataset: + dataset = input_pipeline.get_input_dataset( + FLAGS.train_file_pattern, + FLAGS.train_batch_size, + params, + is_training=True, + strategy=strategy) + + with strategy.scope(): + model = models.create_model( + FLAGS.model_type, params, init_checkpoint=FLAGS.init_checkpoint) + opt = optimizer.create_optimizer(params) + trainer = Trainer(model, params) + model.global_step = opt.iterations + + trainer.compile( + optimizer=opt, + experimental_steps_per_execution=FLAGS.steps_per_loop) + summary_dir = os.path.join(FLAGS.model_dir, "summaries") + summary_callback = tf.keras.callbacks.TensorBoard( + summary_dir, update_freq=max(100, FLAGS.steps_per_loop)) + checkpoint = tf.train.Checkpoint(model=model, optimizer=opt) + checkpoint_manager = tf.train.CheckpointManager( + checkpoint, + directory=FLAGS.model_dir, + max_to_keep=10, + step_counter=model.global_step, + checkpoint_interval=FLAGS.checkpoint_interval) + if checkpoint_manager.restore_or_initialize(): + logging.info("Training restored from the checkpoints in: %s", + FLAGS.model_dir) + checkpoint_callback = SimpleCheckpoint(checkpoint_manager) + + # Trains the model. + steps_per_epoch = min(FLAGS.train_steps, FLAGS.checkpoint_interval) + epochs = FLAGS.train_steps // steps_per_epoch + trainer.fit( + x=dataset, + steps_per_epoch=steps_per_epoch, + epochs=epochs, + callbacks=[summary_callback, checkpoint_callback], + verbose=2) + + +def run(): + """Runs NHNet using Keras APIs.""" + strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=FLAGS.distribution_strategy, tpu_address=FLAGS.tpu) + if strategy: + logging.info("***** Number of cores used : %d", + strategy.num_replicas_in_sync) + + params = models.get_model_params(FLAGS.model_type) + params = params_dict.override_params_dict( + params, FLAGS.params_override, is_strict=True) + params.override( + { + "len_title": + FLAGS.len_title, + "len_passage": + FLAGS.len_passage, + "num_hidden_layers": + FLAGS.num_encoder_layers, + "num_decoder_layers": + FLAGS.num_decoder_layers, + "passage_list": + [chr(ord("b") + i) for i in range(FLAGS.num_nhnet_articles)], + }, + is_strict=False) + stats = {} + if "train" in FLAGS.mode: + train(params, strategy) + if "eval" in FLAGS.mode: + timeout = 0 if FLAGS.mode == "train_and_eval" else 3000 + # Uses padded decoding for TPU. Always uses cache. + padded_decode = isinstance(strategy, tf.distribute.experimental.TPUStrategy) + params.override({ + "padded_decode": padded_decode, + }, is_strict=False) + stats = evaluation.continuous_eval( + strategy, + params, + model_type=FLAGS.model_type, + eval_file_pattern=FLAGS.eval_file_pattern, + batch_size=FLAGS.eval_batch_size, + eval_steps=FLAGS.eval_steps, + model_dir=FLAGS.model_dir, + timeout=timeout) + return stats + + +def main(_): + stats = run() + if stats: + logging.info("Stats:\n%s", stats) + +if __name__ == "__main__": + define_flags() + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/trainer_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/trainer_test.py new file mode 100644 index 0000000..36ba0f4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/trainer_test.py @@ -0,0 +1,100 @@ +# Lint as: python3 +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for official.nlp.nhnet.trainer.""" + +import os + +from absl import flags +from absl.testing import parameterized +import tensorflow as tf + +# pylint: disable=g-direct-tensorflow-import +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +# pylint: enable=g-direct-tensorflow-import +from official.nlp.nhnet import trainer +from official.nlp.nhnet import utils + +FLAGS = flags.FLAGS +trainer.define_flags() + + +def all_strategy_combinations(): + return combinations.combine( + distribution=[ + strategy_combinations.one_device_strategy, + strategy_combinations.one_device_strategy_gpu, + strategy_combinations.tpu_strategy, + ], + mode="eager", + ) + + +def get_trivial_data(config) -> tf.data.Dataset: + """Gets trivial data in the ImageNet size.""" + batch_size, num_docs = 2, len(config.passage_list), + len_passage = config.len_passage + len_title = config.len_title + def generate_data(_) -> tf.data.Dataset: + fake_ids = tf.zeros((num_docs, len_passage), dtype=tf.int32) + title = tf.zeros((len_title), dtype=tf.int32) + return dict( + input_ids=fake_ids, + input_mask=fake_ids, + segment_ids=fake_ids, + target_ids=title) + + dataset = tf.data.Dataset.range(1) + dataset = dataset.repeat() + dataset = dataset.map(generate_data, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + dataset = dataset.prefetch(buffer_size=1).batch(batch_size) + return dataset + + +class TrainerTest(tf.test.TestCase, parameterized.TestCase): + + def setUp(self): + super(TrainerTest, self).setUp() + self._config = utils.get_test_params() + self._config.override( + { + "vocab_size": 49911, + "max_position_embeddings": 200, + "len_title": 15, + "len_passage": 20, + "beam_size": 5, + "alpha": 0.6, + "learning_rate": 0.0, + "learning_rate_warmup_steps": 0, + "multi_channel_cross_attention": True, + "passage_list": ["a", "b"], + }, + is_strict=False) + + @combinations.generate(all_strategy_combinations()) + def test_train(self, distribution): + FLAGS.train_steps = 10 + FLAGS.checkpoint_interval = 5 + FLAGS.model_dir = self.get_temp_dir() + FLAGS.model_type = "nhnet" + trainer.train(self._config, distribution, get_trivial_data(self._config)) + self.assertLen( + tf.io.gfile.glob(os.path.join(FLAGS.model_dir, "ckpt*.index")), 2) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/utils.py new file mode 100644 index 0000000..fe055f3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/nhnet/utils.py @@ -0,0 +1,91 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utility helpers for Bert2Bert.""" +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +from absl import logging +import tensorflow as tf +from typing import Optional, Text +from official.modeling.hyperparams import params_dict +from official.nlp.bert import configs +from official.nlp.nhnet import configs as nhnet_configs + + +def get_bert_config_from_params( + params: params_dict.ParamsDict) -> configs.BertConfig: + """Converts a BertConfig to ParamsDict.""" + return configs.BertConfig.from_dict(params.as_dict()) + + +def get_test_params(cls=nhnet_configs.BERT2BERTConfig): + return cls.from_args(**nhnet_configs.UNITTEST_CONFIG) + + +# pylint: disable=protected-access +def encoder_common_layers(transformer_block): + return [ + transformer_block._attention_layer, + transformer_block._attention_output_dense, + transformer_block._attention_layer_norm, + transformer_block._intermediate_dense, transformer_block._output_dense, + transformer_block._output_layer_norm + ] +# pylint: enable=protected-access + + +def initialize_bert2bert_from_pretrained_bert( + bert_encoder: tf.keras.layers.Layer, + bert_decoder: tf.keras.layers.Layer, + init_checkpoint: Optional[Text] = None) -> None: + """Helper function to initialze Bert2Bert from Bert pretrained checkpoint.""" + ckpt = tf.train.Checkpoint(model=bert_encoder) + logging.info( + "Checkpoint file %s found and restoring from " + "initial checkpoint for core model.", init_checkpoint) + status = ckpt.restore(init_checkpoint) + + # Expects the bert model is a subset of checkpoint as pooling layer is + # not used. + status.assert_existing_objects_matched() + logging.info("Loading from checkpoint file completed.") + + # Saves a checkpoint with transformer layers. + encoder_layers = [] + for transformer_block in bert_encoder.transformer_layers: + encoder_layers.extend(encoder_common_layers(transformer_block)) + + # Restores from the checkpoint with encoder layers. + decoder_layers_to_initialize = [] + for decoder_block in bert_decoder.decoder.layers: + decoder_layers_to_initialize.extend( + decoder_block.common_layers_with_encoder()) + + if len(decoder_layers_to_initialize) != len(encoder_layers): + raise ValueError( + "Source encoder layers with %d objects does not match destination " + "decoder layers with %d objects." % + (len(decoder_layers_to_initialize), len(encoder_layers))) + + for dest_layer, source_layer in zip(decoder_layers_to_initialize, + encoder_layers): + try: + dest_layer.set_weights(source_layer.get_weights()) + except ValueError as e: + logging.error( + "dest_layer: %s failed to set weights from " + "source_layer: %s as %s", dest_layer.name, source_layer.name, str(e)) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/optimization.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/optimization.py new file mode 100644 index 0000000..bdb0819 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/optimization.py @@ -0,0 +1,227 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functions and classes related to optimization (weight updates).""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import re + +from absl import logging +import tensorflow as tf +import tensorflow_addons.optimizers as tfa_optimizers + + +class WarmUp(tf.keras.optimizers.schedules.LearningRateSchedule): + """Applies a warmup schedule on a given learning rate decay schedule.""" + + def __init__(self, + initial_learning_rate, + decay_schedule_fn, + warmup_steps, + power=1.0, + name=None): + super(WarmUp, self).__init__() + self.initial_learning_rate = initial_learning_rate + self.warmup_steps = warmup_steps + self.power = power + self.decay_schedule_fn = decay_schedule_fn + self.name = name + + def __call__(self, step): + with tf.name_scope(self.name or 'WarmUp') as name: + # Implements polynomial warmup. i.e., if global_step < warmup_steps, the + # learning rate will be `global_step/num_warmup_steps * init_lr`. + global_step_float = tf.cast(step, tf.float32) + warmup_steps_float = tf.cast(self.warmup_steps, tf.float32) + warmup_percent_done = global_step_float / warmup_steps_float + warmup_learning_rate = ( + self.initial_learning_rate * + tf.math.pow(warmup_percent_done, self.power)) + return tf.cond( + global_step_float < warmup_steps_float, + lambda: warmup_learning_rate, + lambda: self.decay_schedule_fn(step), + name=name) + + def get_config(self): + return { + 'initial_learning_rate': self.initial_learning_rate, + 'decay_schedule_fn': self.decay_schedule_fn, + 'warmup_steps': self.warmup_steps, + 'power': self.power, + 'name': self.name + } + + +def create_optimizer(init_lr, + num_train_steps, + num_warmup_steps, + end_lr=0.0, + optimizer_type='adamw'): + """Creates an optimizer with learning rate schedule.""" + # Implements linear decay of the learning rate. + lr_schedule = tf.keras.optimizers.schedules.PolynomialDecay( + initial_learning_rate=init_lr, + decay_steps=num_train_steps, + end_learning_rate=end_lr) + if num_warmup_steps: + lr_schedule = WarmUp( + initial_learning_rate=init_lr, + decay_schedule_fn=lr_schedule, + warmup_steps=num_warmup_steps) + + if optimizer_type == 'adamw': + logging.info('using Adamw optimizer') + optimizer = AdamWeightDecay( + learning_rate=lr_schedule, + weight_decay_rate=0.01, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-6, + exclude_from_weight_decay=['layer_norm', 'bias']) + elif optimizer_type == 'lamb': + logging.info('using Lamb optimizer') + optimizer = tfa_optimizers.LAMB( + learning_rate=lr_schedule, + weight_decay_rate=0.01, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-6, + exclude_from_weight_decay=['layer_norm', 'bias']) + else: + raise ValueError('Unsupported optimizer type: ', optimizer_type) + + return optimizer + + +class AdamWeightDecay(tf.keras.optimizers.Adam): + """Adam enables L2 weight decay and clip_by_global_norm on gradients. + + Just adding the square of the weights to the loss function is *not* the + correct way of using L2 regularization/weight decay with Adam, since that will + interact with the m and v parameters in strange ways. + + Instead we want ot decay the weights in a manner that doesn't interact with + the m/v parameters. This is equivalent to adding the square of the weights to + the loss with plain (non-momentum) SGD. + """ + + def __init__(self, + learning_rate=0.001, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-7, + amsgrad=False, + weight_decay_rate=0.0, + include_in_weight_decay=None, + exclude_from_weight_decay=None, + name='AdamWeightDecay', + **kwargs): + super(AdamWeightDecay, self).__init__(learning_rate, beta_1, beta_2, + epsilon, amsgrad, name, **kwargs) + self.weight_decay_rate = weight_decay_rate + self._include_in_weight_decay = include_in_weight_decay + self._exclude_from_weight_decay = exclude_from_weight_decay + + @classmethod + def from_config(cls, config): + """Creates an optimizer from its config with WarmUp custom object.""" + custom_objects = {'WarmUp': WarmUp} + return super(AdamWeightDecay, cls).from_config( + config, custom_objects=custom_objects) + + def _prepare_local(self, var_device, var_dtype, apply_state): + super(AdamWeightDecay, self)._prepare_local(var_device, var_dtype, + apply_state) + apply_state[(var_device, var_dtype)]['weight_decay_rate'] = tf.constant( + self.weight_decay_rate, name='adam_weight_decay_rate') + + def _decay_weights_op(self, var, learning_rate, apply_state): + do_decay = self._do_use_weight_decay(var.name) + if do_decay: + return var.assign_sub( + learning_rate * var * + apply_state[(var.device, var.dtype.base_dtype)]['weight_decay_rate'], + use_locking=self._use_locking) + return tf.no_op() + + def apply_gradients(self, + grads_and_vars, + name=None, + experimental_aggregate_gradients=True): + grads, tvars = list(zip(*grads_and_vars)) + if experimental_aggregate_gradients: + # when experimental_aggregate_gradients = False, apply_gradients() no + # longer implicitly allreduce gradients, users manually allreduce gradient + # and passed the allreduced grads_and_vars. For now, the + # clip_by_global_norm will be moved to before the explicit allreduce to + # keep the math the same as TF 1 and pre TF 2.2 implementation. + (grads, _) = tf.clip_by_global_norm(grads, clip_norm=1.0) + return super(AdamWeightDecay, self).apply_gradients( + zip(grads, tvars), + name=name, + experimental_aggregate_gradients=experimental_aggregate_gradients) + + def _get_lr(self, var_device, var_dtype, apply_state): + """Retrieves the learning rate with the given state.""" + if apply_state is None: + return self._decayed_lr_t[var_dtype], {} + + apply_state = apply_state or {} + coefficients = apply_state.get((var_device, var_dtype)) + if coefficients is None: + coefficients = self._fallback_apply_state(var_device, var_dtype) + apply_state[(var_device, var_dtype)] = coefficients + + return coefficients['lr_t'], dict(apply_state=apply_state) + + def _resource_apply_dense(self, grad, var, apply_state=None): + lr_t, kwargs = self._get_lr(var.device, var.dtype.base_dtype, apply_state) + decay = self._decay_weights_op(var, lr_t, apply_state) + with tf.control_dependencies([decay]): + return super(AdamWeightDecay, + self)._resource_apply_dense(grad, var, **kwargs) + + def _resource_apply_sparse(self, grad, var, indices, apply_state=None): + lr_t, kwargs = self._get_lr(var.device, var.dtype.base_dtype, apply_state) + decay = self._decay_weights_op(var, lr_t, apply_state) + with tf.control_dependencies([decay]): + return super(AdamWeightDecay, + self)._resource_apply_sparse(grad, var, indices, **kwargs) + + def get_config(self): + config = super(AdamWeightDecay, self).get_config() + config.update({ + 'weight_decay_rate': self.weight_decay_rate, + }) + return config + + def _do_use_weight_decay(self, param_name): + """Whether to use L2 weight decay for `param_name`.""" + if self.weight_decay_rate == 0: + return False + + if self._include_in_weight_decay: + for r in self._include_in_weight_decay: + if re.search(r, param_name) is not None: + return True + + if self._exclude_from_weight_decay: + for r in self._exclude_from_weight_decay: + if re.search(r, param_name) is not None: + return False + return True diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/README.md new file mode 100644 index 0000000..1215ed5 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/README.md @@ -0,0 +1,218 @@ +# Transformer Translation Model +This is an implementation of the Transformer translation model as described in +the [Attention is All You Need](https://arxiv.org/abs/1706.03762) paper. The +implementation leverages tf.keras and makes sure it is compatible with TF 2.x. + +**Note: this transformer folder is subject to be integrated into official/nlp +folder. Due to its dependencies, we will finish the refactoring after the model +garden 2.1 release.** + +## Contents + * [Contents](#contents) + * [Walkthrough](#walkthrough) + * [Detailed instructions](#detailed-instructions) + * [Environment preparation](#environment-preparation) + * [Download and preprocess datasets](#download-and-preprocess-datasets) + * [Model training and evaluation](#model-training-and-evaluation) + * [Implementation overview](#implementation-overview) + * [Model Definition](#model-definition) + * [Model Trainer](#model-trainer) + * [Test dataset](#test-dataset) + +## Walkthrough + +Below are the commands for running the Transformer model. See the +[Detailed instructions](#detailed-instructions) for more details on running the +model. + +``` +# Ensure that PYTHONPATH is correctly defined as described in +# https://github.com/tensorflow/models/tree/master/official#requirements +export PYTHONPATH="$PYTHONPATH:/path/to/models" + +cd /path/to/models/official/nlp/transformer + +# Export variables +PARAM_SET=big +DATA_DIR=$HOME/transformer/data +MODEL_DIR=$HOME/transformer/model_$PARAM_SET +VOCAB_FILE=$DATA_DIR/vocab.ende.32768 + +# Download training/evaluation/test datasets +python3 data_download.py --data_dir=$DATA_DIR + +# Train the model for 100000 steps and evaluate every 5000 steps on a single GPU. +# Each train step, takes 4096 tokens as a batch budget with 64 as sequence +# maximal length. +python3 transformer_main.py --data_dir=$DATA_DIR --model_dir=$MODEL_DIR \ + --vocab_file=$VOCAB_FILE --param_set=$PARAM_SET \ + --train_steps=100000 --steps_between_evals=5000 \ + --batch_size=4096 --max_length=64 \ + --bleu_source=$DATA_DIR/newstest2014.en \ + --bleu_ref=$DATA_DIR/newstest2014.de \ + --num_gpus=1 \ + --enable_time_history=false + +# Run during training in a separate process to get continuous updates, +# or after training is complete. +tensorboard --logdir=$MODEL_DIR +``` + +## Detailed instructions + + +0. ### Environment preparation + + #### Add models repo to PYTHONPATH + Follow the instructions described in the [Requirements](https://github.com/tensorflow/models/tree/master/official#requirements) section to add the models folder to the python path. + + #### Export variables (optional) + + Export the following variables, or modify the values in each of the snippets below: + + ```shell + PARAM_SET=big + DATA_DIR=$HOME/transformer/data + MODEL_DIR=$HOME/transformer/model_$PARAM_SET + VOCAB_FILE=$DATA_DIR/vocab.ende.32768 + ``` + +1. ### Download and preprocess datasets + + [data_download.py](data_download.py) downloads and preprocesses the training and evaluation WMT datasets. After the data is downloaded and extracted, the training data is used to generate a vocabulary of subtokens. The evaluation and training strings are tokenized, and the resulting data is sharded, shuffled, and saved as TFRecords. + + 1.75GB of compressed data will be downloaded. In total, the raw files (compressed, extracted, and combined files) take up 8.4GB of disk space. The resulting TFRecord and vocabulary files are 722MB. The script takes around 40 minutes to run, with the bulk of the time spent downloading and ~15 minutes spent on preprocessing. + + Command to run: + ``` + python3 data_download.py --data_dir=$DATA_DIR + ``` + + Arguments: + * `--data_dir`: Path where the preprocessed TFRecord data, and vocab file will be saved. + * Use the `--help` or `-h` flag to get a full list of possible arguments. + +2. ### Model training and evaluation + + [transformer_main.py](transformer_main.py) creates a Transformer keras model, + and trains it uses keras model.fit(). + + Users need to adjust `batch_size` and `num_gpus` to get good performance + running multiple GPUs. + + **Note that:** + when using multiple GPUs or TPUs, this is the global batch size for all + devices. For example, if the batch size is `4096*4` and there are 4 devices, + each device will take 4096 tokens as a batch budget. + + Command to run: + ``` + python3 transformer_main.py --data_dir=$DATA_DIR --model_dir=$MODEL_DIR \ + --vocab_file=$VOCAB_FILE --param_set=$PARAM_SET + ``` + + Arguments: + * `--data_dir`: This should be set to the same directory given to the `data_download`'s `data_dir` argument. + * `--model_dir`: Directory to save Transformer model training checkpoints. + * `--vocab_file`: Path to subtoken vocabulary file. If data_download was used, you may find the file in `data_dir`. + * `--param_set`: Parameter set to use when creating and training the model. Options are `base` and `big` (default). + * `--enable_time_history`: Whether add TimeHistory call. If so, --log_steps must be specified. + * `--batch_size`: The number of tokens to consider in a batch. Combining with + `--max_length`, they decide how many sequences are used per batch. + * Use the `--help` or `-h` flag to get a full list of possible arguments. + + #### Using multiple GPUs + You can train these models on multiple GPUs using `tf.distribute.Strategy` API. + You can read more about them in this + [guide](https://www.tensorflow.org/guide/distribute_strategy). + + In this example, we have made it easier to use is with just a command line flag + `--num_gpus`. By default this flag is 1 if TensorFlow is compiled with CUDA, + and 0 otherwise. + + - --num_gpus=0: Uses tf.distribute.OneDeviceStrategy with CPU as the device. + - --num_gpus=1: Uses tf.distribute.OneDeviceStrategy with GPU as the device. + - --num_gpus=2+: Uses tf.distribute.MirroredStrategy to run synchronous + distributed training across the GPUs. + + #### Using Cloud TPUs + + You can train the Transformer model on Cloud TPUs using + `tf.distribute.TPUStrategy`. If you are not familiar with Cloud TPUs, it is + strongly recommended that you go through the + [quickstart](https://cloud.google.com/tpu/docs/quickstart) to learn how to + create a TPU and GCE VM. + + To run the Transformer model on a TPU, you must set + `--distribution_strategy=tpu`, `--tpu=$TPU_NAME`, and `--use_ctl=True` where + `$TPU_NAME` the name of your TPU in the Cloud Console. + + An example command to run Transformer on a v2-8 or v3-8 TPU would be: + + ```bash + python transformer_main.py \ + --tpu=$TPU_NAME \ + --model_dir=$MODEL_DIR \ + --data_dir=$DATA_DIR \ + --vocab_file=$DATA_DIR/vocab.ende.32768 \ + --bleu_source=$DATA_DIR/newstest2014.en \ + --bleu_ref=$DATA_DIR/newstest2014.end \ + --batch_size=6144 \ + --train_steps=2000 \ + --static_batch=true \ + --use_ctl=true \ + --param_set=big \ + --max_length=64 \ + --decode_batch_size=32 \ + --decode_max_length=97 \ + --padded_decode=true \ + --distribution_strategy=tpu + ``` + Note: `$MODEL_DIR` and `$DATA_DIR` must be GCS paths. + + #### Customizing training schedule + + By default, the model will train for 10 epochs, and evaluate after every epoch. The training schedule may be defined through the flags: + + * Training with steps: + * `--train_steps`: sets the total number of training steps to run. + * `--steps_between_evals`: Number of training steps to run between evaluations. + + #### Compute BLEU score during model evaluation + + Use these flags to compute the BLEU when the model evaluates: + + * `--bleu_source`: Path to file containing text to translate. + * `--bleu_ref`: Path to file containing the reference translation. + + When running `transformer_main.py`, use the flags: `--bleu_source=$DATA_DIR/newstest2014.en --bleu_ref=$DATA_DIR/newstest2014.de` + + #### Tensorboard + Training and evaluation metrics (loss, accuracy, approximate BLEU score, etc.) are logged, and can be displayed in the browser using Tensorboard. + ``` + tensorboard --logdir=$MODEL_DIR + ``` + The values are displayed at [localhost:6006](localhost:6006). + +## Implementation overview + +A brief look at each component in the code: + +### Model Definition +* [transformer.py](transformer.py): Defines a tf.keras.Model: `Transformer`. +* [embedding_layer.py](embedding_layer.py): Contains the layer that calculates the embeddings. The embedding weights are also used to calculate the pre-softmax probabilities from the decoder output. +* [attention_layer.py](attention_layer.py): Defines the multi-headed and self attention layers that are used in the encoder/decoder stacks. +* [ffn_layer.py](ffn_layer.py): Defines the feedforward network that is used in the encoder/decoder stacks. The network is composed of 2 fully connected layers. + +Other files: +* [beam_search.py](beam_search.py) contains the beam search implementation, which is used during model inference to find high scoring translations. + +### Model Trainer +[transformer_main.py](transformer_main.py) creates an `TransformerTask` to train and evaluate the model using tf.keras. + +### Test dataset +The [newstest2014 files](https://storage.googleapis.com/tf-perf-public/official_transformer/test_data/newstest2014.tgz) +are extracted from the [NMT Seq2Seq tutorial](https://google.github.io/seq2seq/nmt/#download-data). +The raw text files are converted from the SGM format of the +[WMT 2016](http://www.statmt.org/wmt16/translation-task.html) test sets. The +newstest2014 files are put into the `$DATA_DIR` when executing `data_download.py` diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/attention_layer.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/attention_layer.py new file mode 100644 index 0000000..c798e9f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/attention_layer.py @@ -0,0 +1,159 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Implementation of multiheaded attention and self-attention layers.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from official.nlp.modeling import layers + + +class Attention(tf.keras.layers.Layer): + """Multi-headed attention layer.""" + + def __init__(self, hidden_size, num_heads, attention_dropout): + """Initialize Attention. + + Args: + hidden_size: int, output dim of hidden layer. + num_heads: int, number of heads to repeat the same attention structure. + attention_dropout: float, dropout rate inside attention for training. + """ + if hidden_size % num_heads: + raise ValueError( + "Hidden size ({}) must be divisible by the number of heads ({})." + .format(hidden_size, num_heads)) + + super(Attention, self).__init__() + self.hidden_size = hidden_size + self.num_heads = num_heads + self.attention_dropout = attention_dropout + + def build(self, input_shape): + """Builds the layer.""" + # Layers for linearly projecting the queries, keys, and values. + size_per_head = self.hidden_size // self.num_heads + self.query_dense_layer = layers.DenseEinsum( + output_shape=(self.num_heads, size_per_head), + kernel_initializer="glorot_uniform", + use_bias=False, + name="query") + self.key_dense_layer = layers.DenseEinsum( + output_shape=(self.num_heads, size_per_head), + kernel_initializer="glorot_uniform", + use_bias=False, + name="key") + self.value_dense_layer = layers.DenseEinsum( + output_shape=(self.num_heads, size_per_head), + kernel_initializer="glorot_uniform", + use_bias=False, + name="value") + self.output_dense_layer = layers.DenseEinsum( + output_shape=self.hidden_size, + num_summed_dimensions=2, + kernel_initializer="glorot_uniform", + use_bias=False, + name="output_transform") + super(Attention, self).build(input_shape) + + def get_config(self): + return { + "hidden_size": self.hidden_size, + "num_heads": self.num_heads, + "attention_dropout": self.attention_dropout, + } + + def call(self, query_input, source_input, bias, training, cache=None, + decode_loop_step=None): + """Apply attention mechanism to query_input and source_input. + + Args: + query_input: A tensor with shape [batch_size, length_query, hidden_size]. + source_input: A tensor with shape [batch_size, length_source, + hidden_size]. + bias: A tensor with shape [batch_size, 1, length_query, length_source], + the attention bias that will be added to the result of the dot product. + training: A bool, whether in training mode or not. + cache: (Used during prediction) A dictionary with tensors containing + results of previous attentions. The dictionary must have the items: + {"k": tensor with shape [batch_size, i, heads, dim_per_head], + "v": tensor with shape [batch_size, i, heads, dim_per_head]} + where i is the current decoded length for non-padded decode, or max + sequence length for padded decode. + decode_loop_step: An integer, step number of the decoding loop. Used only + for autoregressive inference on TPU. + + Returns: + Attention layer output with shape [batch_size, length_query, hidden_size] + """ + # Linearly project the query, key and value using different learned + # projections. Splitting heads is automatically done during the linear + # projections --> [batch_size, length, num_heads, dim_per_head]. + query = self.query_dense_layer(query_input) + key = self.key_dense_layer(source_input) + value = self.value_dense_layer(source_input) + + if cache is not None: + # Combine cached keys and values with new keys and values. + if decode_loop_step is not None: + cache_k_shape = cache["k"].shape.as_list() + indices = tf.reshape( + tf.one_hot(decode_loop_step, cache_k_shape[1], dtype=key.dtype), + [1, cache_k_shape[1], 1, 1]) + key = cache["k"] + key * indices + cache_v_shape = cache["v"].shape.as_list() + indices = tf.reshape( + tf.one_hot(decode_loop_step, cache_v_shape[1], dtype=value.dtype), + [1, cache_v_shape[1], 1, 1]) + value = cache["v"] + value * indices + else: + key = tf.concat([tf.cast(cache["k"], key.dtype), key], axis=1) + value = tf.concat([tf.cast(cache["v"], value.dtype), value], axis=1) + + # Update cache + cache["k"] = key + cache["v"] = value + + # Scale query to prevent the dot product between query and key from growing + # too large. + depth = (self.hidden_size // self.num_heads) + query *= depth ** -0.5 + + # Calculate dot product attention + logits = tf.einsum("BTNH,BFNH->BNFT", key, query) + logits += bias + # Note that softmax internally performs math operations using float32 + # for numeric stability. When training with float16, we keep the input + # and output in float16 for better performance. + weights = tf.nn.softmax(logits, name="attention_weights") + if training: + weights = tf.nn.dropout(weights, rate=self.attention_dropout) + attention_output = tf.einsum("BNFT,BTNH->BFNH", weights, value) + + # Run the outputs through another linear projection layer. Recombining heads + # is automatically done --> [batch_size, length, hidden_size] + attention_output = self.output_dense_layer(attention_output) + return attention_output + + +class SelfAttention(Attention): + """Multiheaded self-attention layer.""" + + def call(self, query_input, bias, training, cache=None, + decode_loop_step=None): + return super(SelfAttention, self).call( + query_input, query_input, bias, training, cache, decode_loop_step) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/beam_search.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/beam_search.py new file mode 100644 index 0000000..fa1ae52 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/beam_search.py @@ -0,0 +1,138 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Beam search in TF v2.""" + +import tensorflow as tf + +from official.nlp.transformer import beam_search_v1 as v1 +from official.nlp.transformer import misc + +_StateKeys = v1._StateKeys # pylint: disable=protected-access + + +class SequenceBeamSearchV2(v1.SequenceBeamSearch): + """Implementation of beam search loop in v2.""" + + def search(self, initial_ids, initial_cache): + """Beam search for sequences with highest scores.""" + state, state_shapes = self._create_initial_state(initial_ids, initial_cache) + + finished_state = tf.nest.map_structure( + tf.stop_gradient, + tf.while_loop(self._continue_search, + self._search_step, + loop_vars=[state], + shape_invariants=[state_shapes], + parallel_iterations=1)) + finished_state = finished_state[0] + + alive_seq = finished_state[_StateKeys.ALIVE_SEQ] + alive_log_probs = finished_state[_StateKeys.ALIVE_LOG_PROBS] + finished_seq = finished_state[_StateKeys.FINISHED_SEQ] + finished_scores = finished_state[_StateKeys.FINISHED_SCORES] + finished_flags = finished_state[_StateKeys.FINISHED_FLAGS] + + # 2.0 changes tf.where behavior. Should make parameters broadcastable. + finished_cond = tf.reduce_any(finished_flags, 1, name="finished_cond") + seq_cond = _expand_to_same_rank(finished_cond, finished_seq) + score_cond = _expand_to_same_rank(finished_cond, finished_scores) + + # Account for corner case where there are no finished sequences for a + # particular batch item. In that case, return alive sequences for that batch + # item. + finished_seq = tf.compat.v2.where(seq_cond, finished_seq, alive_seq) + finished_scores = tf.compat.v2.where( + score_cond, finished_scores, alive_log_probs) + return finished_seq, finished_scores + + +def sequence_beam_search(symbols_to_logits_fn, + initial_ids, + initial_cache, + vocab_size, + beam_size, + alpha, + max_decode_length, + eos_id, + padded_decode=False, + dtype="float32"): + """Search for sequence of subtoken ids with the largest probability. + + Args: + symbols_to_logits_fn: A function that takes in ids, index, and cache as + arguments. The passed in arguments will have shape: + ids -> A tensor with shape [batch_size * beam_size, index]. + index -> A scalar. + cache -> A nested dictionary of tensors [batch_size * beam_size, ...]. + The function must return a tuple of logits and new cache: + logits -> A tensor with shape [batch * beam_size, vocab_size]. + new cache -> A nested dictionary with the same shape/structure as the + inputted cache. + initial_ids: An int32 tensor with shape [batch_size]. Starting ids for + each batch item. + initial_cache: A dictionary, containing starting decoder variables + information. + vocab_size: An integer, the size of tokens. + beam_size: An integer, the number of beams. + alpha: A float, defining the strength of length normalization. + max_decode_length: An integer, the maximum length to decoded a sequence. + eos_id: An integer, ID of eos token, used to determine when a sequence has + finished. + padded_decode: A bool, indicating if max_sequence_length padding is used + for beam search. + dtype: A tensorflow data type used for score computation. The default is + tf.float32. + + Returns: + Top decoded sequences [batch_size, beam_size, max_decode_length] + sequence scores [batch_size, beam_size] + """ + batch_size = ( + initial_ids.shape.as_list()[0] if padded_decode else + tf.shape(initial_ids)[0]) + if misc.is_v2(): + sbs = SequenceBeamSearchV2(symbols_to_logits_fn, vocab_size, batch_size, + beam_size, alpha, max_decode_length, eos_id, + padded_decode, dtype) + else: + sbs = v1.SequenceBeamSearch(symbols_to_logits_fn, vocab_size, batch_size, + beam_size, alpha, max_decode_length, eos_id, + padded_decode, dtype) + return sbs.search(initial_ids, initial_cache) + + +def _expand_to_same_rank(tensor, target): + """Expands a given tensor to target's rank to be broadcastable. + + Args: + tensor: input tensor to tile. Shape: [b, d1, ..., da] + target: target tensor. Shape: [b, d1, ..., da, ..., dn] + + Returns: + Tiled tensor of shape [b, d1, ..., da, 1, ..., 1] with same rank of target. + + Raises: + ValueError, if the shape rank of rank tensor/target is None. + """ + if tensor.shape.rank is None: + raise ValueError("Expect rank for tensor shape, but got None.") + if target.shape.rank is None: + raise ValueError("Expect rank for target shape, but got None.") + + with tf.name_scope("expand_rank"): + diff_rank = target.shape.rank - tensor.shape.rank + for _ in range(diff_rank): + tensor = tf.expand_dims(tensor, -1) + return tensor diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/beam_search_v1.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/beam_search_v1.py new file mode 100644 index 0000000..8b143b1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/beam_search_v1.py @@ -0,0 +1,675 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Beam search to find the translated sequence with the highest probability. + +Source implementation from Tensor2Tensor: +https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/beam_search.py +""" + +import numpy as np +import tensorflow.compat.v1 as tf +from tensorflow.python.util import nest + + +def inf(dtype): + """Returns a value close to infinity, but is still finite in `dtype`. + + This is useful to get a very large value that is still zero when multiplied by + zero. The floating-point "Inf" value is NaN when multiplied by zero. + + Args: + dtype: A dtype. The returned value will be finite when casted to this dtype. + + Returns: + A very large value. + """ + if dtype == "float32" or dtype == "bfloat16": + return 1e7 + elif dtype == "float16": + # Disable no-member lint error, as the linter thinks np.float16 does not + # exist for some reason. + return np.finfo(np.float16).max # pylint: disable=no-member + else: + raise AssertionError('Invalid dtype: %s' % dtype) + + +class _StateKeys(object): + """Keys to dictionary storing the state of the beam search loop.""" + + # Variable storing the loop index. + CUR_INDEX = "CUR_INDEX" + + # Top sequences that are alive for each batch item. Alive sequences are ones + # that have not generated an EOS token. Sequences that reach EOS are marked as + # finished and moved to the FINISHED_SEQ tensor. + # Has shape [batch_size, beam_size, CUR_INDEX + 1] + ALIVE_SEQ = "ALIVE_SEQ" + # Log probabilities of each alive sequence. Shape [batch_size, beam_size] + ALIVE_LOG_PROBS = "ALIVE_LOG_PROBS" + # Dictionary of cached values for each alive sequence. The cache stores + # the encoder output, attention bias, and the decoder attention output from + # the previous iteration. + ALIVE_CACHE = "ALIVE_CACHE" + + # Top finished sequences for each batch item. + # Has shape [batch_size, beam_size, CUR_INDEX + 1]. Sequences that are + # shorter than CUR_INDEX + 1 are padded with 0s. + FINISHED_SEQ = "FINISHED_SEQ" + # Scores for each finished sequence. Score = log probability / length norm + # Shape [batch_size, beam_size] + FINISHED_SCORES = "FINISHED_SCORES" + # Flags indicating which sequences in the finished sequences are finished. + # At the beginning, all of the sequences in FINISHED_SEQ are filler values. + # True -> finished sequence, False -> filler. Shape [batch_size, beam_size] + FINISHED_FLAGS = "FINISHED_FLAGS" + + +class SequenceBeamSearch(object): + """Implementation of beam search loop.""" + + def __init__(self, + symbols_to_logits_fn, + vocab_size, + batch_size, + beam_size, + alpha, + max_decode_length, + eos_id, + padded_decode, + dtype=tf.float32): + """Initialize sequence beam search. + + Args: + symbols_to_logits_fn: A function to provide logits, which is the + interface to the Transformer model. The passed in arguments are: + ids -> A tensor with shape [batch_size * beam_size, index]. + index -> A scalar. + cache -> A nested dictionary of tensors [batch_size * beam_size, ...]. + The function must return a tuple of logits and the updated cache: + logits -> A tensor with shape [batch * beam_size, vocab_size]. + updated cache -> A nested dictionary with the same structure as the + input cache. + vocab_size: An integer, the size of the vocabulary, used for topk + computation. + batch_size: An integer, the decode batch size. + beam_size: An integer, number of beams for beam search. + alpha: A float, defining the strength of length normalization. + max_decode_length: An integer, the maximum number of steps to decode + a sequence. + eos_id: An integer. ID of end of sentence token. + padded_decode: A bool, indicating if max_sequence_length padding is used + for beam search. + dtype: A tensorflow data type used for score computation. The default is + tf.float32. + """ + self.symbols_to_logits_fn = symbols_to_logits_fn + self.vocab_size = vocab_size + self.batch_size = batch_size + self.beam_size = beam_size + self.alpha = alpha + self.max_decode_length = max_decode_length + self.eos_id = eos_id + self.padded_decode = padded_decode + self.dtype = tf.as_dtype(dtype) + + def search(self, initial_ids, initial_cache): + """Beam search for sequences with highest scores.""" + state, state_shapes = self._create_initial_state(initial_ids, initial_cache) + + finished_state = tf.while_loop( + self._continue_search, self._search_step, loop_vars=[state], + shape_invariants=[state_shapes], parallel_iterations=1, back_prop=False) + finished_state = finished_state[0] + + alive_seq = finished_state[_StateKeys.ALIVE_SEQ] + alive_log_probs = finished_state[_StateKeys.ALIVE_LOG_PROBS] + finished_seq = finished_state[_StateKeys.FINISHED_SEQ] + finished_scores = finished_state[_StateKeys.FINISHED_SCORES] + finished_flags = finished_state[_StateKeys.FINISHED_FLAGS] + + # Account for corner case where there are no finished sequences for a + # particular batch item. In that case, return alive sequences for that batch + # item. + finished_seq = tf.where( + tf.reduce_any(finished_flags, 1), finished_seq, alive_seq) + finished_scores = tf.where( + tf.reduce_any(finished_flags, 1), finished_scores, alive_log_probs) + return finished_seq, finished_scores + + def _create_initial_state(self, initial_ids, initial_cache): + """Return initial state dictionary and its shape invariants. + + Args: + initial_ids: initial ids to pass into the symbols_to_logits_fn. + int tensor with shape [batch_size, 1] + initial_cache: dictionary storing values to be passed into the + symbols_to_logits_fn. + + Returns: + state and shape invariant dictionaries with keys from _StateKeys + """ + for key, value in initial_cache.items(): + for inner_value in nest.flatten(value): + if inner_value.dtype != self.dtype: + raise TypeError( + "initial_cache element for key '%s' has dtype %s that does not " + "match SequenceBeamSearch's dtype of %s. Value: %s" % + (key, value.dtype.name, self.dtype.name, inner_value)) + + # Current loop index (starts at 0) + cur_index = tf.constant(0) + + # Create alive sequence with shape [batch_size, beam_size, 1] + alive_seq = _expand_to_beam_size(initial_ids, self.beam_size) + alive_seq = tf.expand_dims(alive_seq, axis=2) + if self.padded_decode: + alive_seq = tf.tile(alive_seq, [1, 1, self.max_decode_length + 1]) + + # Create tensor for storing initial log probabilities. + # Assume initial_ids are prob 1.0 + initial_log_probs = tf.constant( + [[0.] + [-float("inf")] * (self.beam_size - 1)], dtype=self.dtype) + alive_log_probs = tf.tile(initial_log_probs, [self.batch_size, 1]) + + # Expand all values stored in the dictionary to the beam size, so that each + # beam has a separate cache. + alive_cache = nest.map_structure( + lambda t: _expand_to_beam_size(t, self.beam_size), initial_cache) + + # Initialize tensor storing finished sequences with filler values. + finished_seq = tf.zeros(tf.shape(alive_seq), tf.int32) + + # Set scores of the initial finished seqs to negative infinity. + finished_scores = tf.ones([self.batch_size, self.beam_size], + dtype=self.dtype) * -inf(self.dtype) + + # Initialize finished flags with all False values. + finished_flags = tf.zeros([self.batch_size, self.beam_size], tf.bool) + + # Create state dictionary + state = { + _StateKeys.CUR_INDEX: cur_index, + _StateKeys.ALIVE_SEQ: alive_seq, + _StateKeys.ALIVE_LOG_PROBS: alive_log_probs, + _StateKeys.ALIVE_CACHE: alive_cache, + _StateKeys.FINISHED_SEQ: finished_seq, + _StateKeys.FINISHED_SCORES: finished_scores, + _StateKeys.FINISHED_FLAGS: finished_flags + } + + # Create state invariants for each value in the state dictionary. Each + # dimension must be a constant or None. A None dimension means either: + # 1) the dimension's value is a tensor that remains the same but may + # depend on the input sequence to the model (e.g. batch size). + # 2) the dimension may have different values on different iterations. + if self.padded_decode: + state_shape_invariants = { + _StateKeys.CUR_INDEX: + tf.TensorShape([]), + _StateKeys.ALIVE_SEQ: + tf.TensorShape( + [self.batch_size, self.beam_size, + self.max_decode_length + 1]), + _StateKeys.ALIVE_LOG_PROBS: + tf.TensorShape([self.batch_size, self.beam_size]), + _StateKeys.ALIVE_CACHE: + nest.map_structure(_get_shape, alive_cache), + _StateKeys.FINISHED_SEQ: + tf.TensorShape( + [self.batch_size, self.beam_size, + self.max_decode_length + 1]), + _StateKeys.FINISHED_SCORES: + tf.TensorShape([self.batch_size, self.beam_size]), + _StateKeys.FINISHED_FLAGS: + tf.TensorShape([self.batch_size, self.beam_size]) + } + else: + state_shape_invariants = { + _StateKeys.CUR_INDEX: + tf.TensorShape([]), + _StateKeys.ALIVE_SEQ: + tf.TensorShape([None, self.beam_size, None]), + _StateKeys.ALIVE_LOG_PROBS: + tf.TensorShape([None, self.beam_size]), + _StateKeys.ALIVE_CACHE: + nest.map_structure(_get_shape_keep_last_dim, alive_cache), + _StateKeys.FINISHED_SEQ: + tf.TensorShape([None, self.beam_size, None]), + _StateKeys.FINISHED_SCORES: + tf.TensorShape([None, self.beam_size]), + _StateKeys.FINISHED_FLAGS: + tf.TensorShape([None, self.beam_size]) + } + + return state, state_shape_invariants + + def _continue_search(self, state): + """Return whether to continue the search loop. + + The loops should terminate when + 1) when decode length has been reached, or + 2) when the worst score in the finished sequences is better than the best + score in the alive sequences (i.e. the finished sequences are provably + unchanging) + + Args: + state: A dictionary with the current loop state. + + Returns: + Bool tensor with value True if loop should continue, False if loop should + terminate. + """ + i = state[_StateKeys.CUR_INDEX] + alive_log_probs = state[_StateKeys.ALIVE_LOG_PROBS] + finished_scores = state[_StateKeys.FINISHED_SCORES] + finished_flags = state[_StateKeys.FINISHED_FLAGS] + + not_at_max_decode_length = tf.less(i, self.max_decode_length) + + # Calculate largest length penalty (the larger penalty, the better score). + max_length_norm = _length_normalization(self.alpha, self.max_decode_length, + dtype=self.dtype) + # Get the best possible scores from alive sequences. + best_alive_scores = alive_log_probs[:, 0] / max_length_norm + + # Compute worst score in finished sequences for each batch element + finished_scores *= tf.cast(finished_flags, + self.dtype) # set filler scores to zero + lowest_finished_scores = tf.reduce_min(finished_scores, axis=1) + + # If there are no finished sequences in a batch element, then set the lowest + # finished score to -INF for that element. + finished_batches = tf.reduce_any(finished_flags, 1) + lowest_finished_scores += ((1.0 - + tf.cast(finished_batches, self.dtype)) * + -inf(self.dtype)) + + worst_finished_score_better_than_best_alive_score = tf.reduce_all( + tf.greater(lowest_finished_scores, best_alive_scores) + ) + + return tf.logical_and( + not_at_max_decode_length, + tf.logical_not(worst_finished_score_better_than_best_alive_score) + ) + + def _search_step(self, state): + """Beam search loop body. + + Grow alive sequences by a single ID. Sequences that have reached the EOS + token are marked as finished. The alive and finished sequences with the + highest log probabilities and scores are returned. + + A sequence's finished score is calculating by dividing the log probability + by the length normalization factor. Without length normalization, the + search is more likely to return shorter sequences. + + Args: + state: A dictionary with the current loop state. + + Returns: + new state dictionary. + """ + # Grow alive sequences by one token. + new_seq, new_log_probs, topk_ids, new_cache = self._grow_alive_seq(state) + new_finished_flags = tf.equal(topk_ids, self.eos_id) + # Collect top beam_size alive sequences + alive_state = self._get_new_alive_state(new_seq, new_log_probs, + new_finished_flags, new_cache) + + # Combine newly finished sequences with existing finished sequences, and + # collect the top k scoring sequences. + finished_state = self._get_new_finished_state(state, new_seq, new_log_probs, + new_finished_flags) + + # Increment loop index and create new state dictionary + new_state = {_StateKeys.CUR_INDEX: state[_StateKeys.CUR_INDEX] + 1} + new_state.update(alive_state) + new_state.update(finished_state) + return [new_state] + + def _grow_alive_seq(self, state): + """Grow alive sequences by one token, and collect top 2*beam_size sequences. + + 2*beam_size sequences are collected because some sequences may have reached + the EOS token. 2*beam_size ensures that at least beam_size sequences are + still alive. + + Args: + state: A dictionary with the current loop state. + Returns: + Tuple of + (Top 2*beam_size sequences [batch_size, 2 * beam_size, cur_index + 1], + Scores of returned sequences [batch_size, 2 * beam_size], + New alive cache, for each of the 2 * beam_size sequences) + """ + i = state[_StateKeys.CUR_INDEX] + alive_seq = state[_StateKeys.ALIVE_SEQ] + alive_log_probs = state[_StateKeys.ALIVE_LOG_PROBS] + alive_cache = state[_StateKeys.ALIVE_CACHE] + + beams_to_keep = 2 * self.beam_size + + # Get logits for the next candidate IDs for the alive sequences. Get the new + # cache values at the same time. + if self.padded_decode: + flat_ids = tf.reshape( + tf.slice(alive_seq, [0, 0, i], [self.batch_size, self.beam_size, 1]), + [self.batch_size * self.beam_size, -1]) + else: + flat_ids = _flatten_beam_dim(alive_seq) # [batch_size * beam_size] + flat_cache = nest.map_structure(_flatten_beam_dim, alive_cache) + + flat_logits, flat_cache = self.symbols_to_logits_fn(flat_ids, i, flat_cache) + + # Unflatten logits to shape [batch_size, beam_size, vocab_size] + logits = _unflatten_beam_dim(flat_logits, self.batch_size, self.beam_size) + new_cache = nest.map_structure( + lambda t: _unflatten_beam_dim(t, self.batch_size, self.beam_size), + flat_cache) + + # Convert logits to normalized log probs + candidate_log_probs = _log_prob_from_logits(logits) + + # Calculate new log probabilities if each of the alive sequences were + # extended # by the the candidate IDs. + # Shape [batch_size, beam_size, vocab_size] + log_probs = candidate_log_probs + tf.expand_dims(alive_log_probs, axis=2) + + # Each batch item has beam_size * vocab_size candidate sequences. For each + # batch item, get the k candidates with the highest log probabilities. + flat_log_probs = tf.reshape(log_probs, + [-1, self.beam_size * self.vocab_size]) + topk_log_probs, topk_indices = tf.nn.top_k(flat_log_probs, k=beams_to_keep) + + # Extract the alive sequences that generate the highest log probabilities + # after being extended. + topk_beam_indices = topk_indices // self.vocab_size + topk_seq, new_cache = _gather_beams( + [alive_seq, new_cache], topk_beam_indices, self.batch_size, + beams_to_keep) + + # Append the most probable IDs to the topk sequences + topk_ids = topk_indices % self.vocab_size + if self.padded_decode: + topk_seq = tf.transpose(topk_seq, perm=[2, 0, 1]) + # TODO(b/145533236, hongkuny): Reverts once TF fix the validation. + topk_seq = tf.tensor_scatter_nd_update(topk_seq, [[i + 1]], + tf.expand_dims(topk_ids, axis=0)) + topk_seq = tf.transpose(topk_seq, perm=[1, 2, 0]) + else: + topk_seq = tf.concat([topk_seq, tf.expand_dims(topk_ids, axis=2)], axis=2) + return topk_seq, topk_log_probs, topk_ids, new_cache + + def _get_new_alive_state(self, new_seq, new_log_probs, new_finished_flags, + new_cache): + """Gather the top k sequences that are still alive. + + Args: + new_seq: New sequences generated by growing the current alive sequences + int32 tensor with shape [batch_size, 2 * beam_size, cur_index + 1] + new_log_probs: Log probabilities of new sequences float32 tensor with + shape [batch_size, beam_size] + new_finished_flags: A boolean Tensor indicates which sequences are live + inside the beam. + new_cache: Dict of cached values for each sequence. + + Returns: + Dictionary with alive keys from _StateKeys: + {Top beam_size sequences that are still alive (don't end with eos_id) + Log probabilities of top alive sequences + Dict cache storing decoder states for top alive sequences} + """ + # To prevent finished sequences from being considered, set log probs to -inf + new_log_probs += tf.cast(new_finished_flags, self.dtype) * -inf(self.dtype) + + top_alive_seq, top_alive_log_probs, top_alive_cache = _gather_topk_beams( + [new_seq, new_log_probs, new_cache], new_log_probs, self.batch_size, + self.beam_size) + + return { + _StateKeys.ALIVE_SEQ: top_alive_seq, + _StateKeys.ALIVE_LOG_PROBS: top_alive_log_probs, + _StateKeys.ALIVE_CACHE: top_alive_cache + } + + def _get_new_finished_state(self, state, new_seq, new_log_probs, + new_finished_flags): + """Combine new and old finished sequences, and gather the top k sequences. + + Args: + state: A dictionary with the current loop state. + new_seq: New sequences generated by growing the current alive sequences + int32 tensor with shape [batch_size, beam_size, i + 1] + new_log_probs: Log probabilities of new sequences float32 tensor with + shape [batch_size, beam_size] + new_finished_flags: A boolean Tensor indicates which sequences are live + inside the beam. + + Returns: + Dictionary with finished keys from _StateKeys: + {Top beam_size finished sequences based on score, + Scores of finished sequences, + Finished flags of finished sequences} + """ + i = state[_StateKeys.CUR_INDEX] + finished_seq = state[_StateKeys.FINISHED_SEQ] + finished_scores = state[_StateKeys.FINISHED_SCORES] + finished_flags = state[_StateKeys.FINISHED_FLAGS] + + # First append a column of 0-ids to finished_seq to increment the length. + # New shape of finished_seq: [batch_size, beam_size, i + 1] + if not self.padded_decode: + finished_seq = tf.concat([ + finished_seq, + tf.zeros([self.batch_size, self.beam_size, 1], tf.int32) + ], + axis=2) + + # Calculate new seq scores from log probabilities. + length_norm = _length_normalization(self.alpha, i + 1, dtype=self.dtype) + new_scores = new_log_probs / length_norm + + # Set the scores of the still-alive seq in new_seq to large negative values. + new_scores += ((1. - tf.cast(new_finished_flags, self.dtype)) * + -inf(self.dtype)) + + # Combine sequences, scores, and flags. + finished_seq = tf.concat([finished_seq, new_seq], axis=1) + finished_scores = tf.concat([finished_scores, new_scores], axis=1) + finished_flags = tf.concat([finished_flags, new_finished_flags], axis=1) + + # Return the finished sequences with the best scores. + top_finished_seq, top_finished_scores, top_finished_flags = ( + _gather_topk_beams([finished_seq, finished_scores, finished_flags], + finished_scores, self.batch_size, self.beam_size)) + + return { + _StateKeys.FINISHED_SEQ: top_finished_seq, + _StateKeys.FINISHED_SCORES: top_finished_scores, + _StateKeys.FINISHED_FLAGS: top_finished_flags + } + + +def sequence_beam_search( + symbols_to_logits_fn, initial_ids, initial_cache, vocab_size, beam_size, + alpha, max_decode_length, eos_id, padded_decode=False): + """Search for sequence of subtoken ids with the largest probability. + + Args: + symbols_to_logits_fn: A function that takes in ids, index, and cache as + arguments. The passed in arguments will have shape: + ids -> A tensor with shape [batch_size * beam_size, index]. + index -> A scalar. + cache -> A nested dictionary of tensors [batch_size * beam_size, ...]. + The function must return a tuple of logits and new cache: + logits -> A tensor with shape [batch * beam_size, vocab_size]. + new cache -> A nested dictionary with the same shape/structure as the + inputted cache. + initial_ids: An int32 tensor with shape [batch_size]. Starting ids for + each batch item. + initial_cache: A dictionary, containing starting decoder variables + information. + vocab_size: An integer, the size of the vocabulary, used for topk + computation. + beam_size: An integer, the number of beams. + alpha: A float, defining the strength of length normalization. + max_decode_length: An integer, the maximum length to decoded a sequence. + eos_id: An integer, ID of eos token, used to determine when a sequence has + finished. + padded_decode: A bool, indicating if max_sequence_length padding is used + for beam search. + + Returns: + Top decoded sequences [batch_size, beam_size, max_decode_length] + sequence scores [batch_size, beam_size] + """ + batch_size = ( + initial_ids.shape.as_list()[0] if padded_decode else + tf.shape(initial_ids)[0]) + sbs = SequenceBeamSearch(symbols_to_logits_fn, vocab_size, batch_size, + beam_size, alpha, max_decode_length, eos_id, + padded_decode) + return sbs.search(initial_ids, initial_cache) + + +def _log_prob_from_logits(logits): + return logits - tf.reduce_logsumexp(logits, axis=2, keepdims=True) + + +def _length_normalization(alpha, length, dtype=tf.float32): + """Return length normalization factor.""" + return tf.pow(((5. + tf.cast(length, dtype)) / 6.), alpha) + + +def _expand_to_beam_size(tensor, beam_size): + """Tiles a given tensor by beam_size. + + Args: + tensor: tensor to tile [batch_size, ...] + beam_size: How much to tile the tensor by. + + Returns: + Tiled tensor [batch_size, beam_size, ...] + """ + tensor = tf.expand_dims(tensor, axis=1) + tile_dims = [1] * tensor.shape.ndims + tile_dims[1] = beam_size + + return tf.tile(tensor, tile_dims) + + +def _shape_list(tensor): + """Return a list of the tensor's shape, and ensure no None values in list.""" + # Get statically known shape (may contain None's for unknown dimensions) + shape = tensor.get_shape().as_list() + + # Ensure that the shape values are not None + dynamic_shape = tf.shape(tensor) + for i in range(len(shape)): # pylint: disable=consider-using-enumerate + if shape[i] is None: + shape[i] = dynamic_shape[i] + return shape + + +def _get_shape_keep_last_dim(tensor): + shape_list = _shape_list(tensor) + + # Only the last + for i in range(len(shape_list) - 1): + shape_list[i] = None + + if isinstance(shape_list[-1], tf.Tensor): + shape_list[-1] = None + return tf.TensorShape(shape_list) + + +def _get_shape(tensor): + """Return the shape of the input tensor.""" + return tf.TensorShape(_shape_list(tensor)) + + +def _flatten_beam_dim(tensor): + """Reshapes first two dimensions in to single dimension. + + Args: + tensor: Tensor to reshape of shape [A, B, ...] + + Returns: + Reshaped tensor of shape [A*B, ...] + """ + shape = _shape_list(tensor) + shape[0] *= shape[1] + shape.pop(1) # Remove beam dim + return tf.reshape(tensor, shape) + + +def _unflatten_beam_dim(tensor, batch_size, beam_size): + """Reshapes first dimension back to [batch_size, beam_size]. + + Args: + tensor: Tensor to reshape of shape [batch_size*beam_size, ...] + batch_size: Tensor, original batch size. + beam_size: int, original beam size. + + Returns: + Reshaped tensor of shape [batch_size, beam_size, ...] + """ + shape = _shape_list(tensor) + new_shape = [batch_size, beam_size] + shape[1:] + return tf.reshape(tensor, new_shape) + + +def _gather_beams(nested, beam_indices, batch_size, new_beam_size): + """Gather beams from nested structure of tensors. + + Each tensor in nested represents a batch of beams, where beam refers to a + single search state (beam search involves searching through multiple states + in parallel). + + This function is used to gather the top beams, specified by + beam_indices, from the nested tensors. + + Args: + nested: Nested structure (tensor, list, tuple or dict) containing tensors + with shape [batch_size, beam_size, ...]. + beam_indices: int32 tensor with shape [batch_size, new_beam_size]. Each + value in beam_indices must be between [0, beam_size), and are not + necessarily unique. + batch_size: int size of batch + new_beam_size: int number of beams to be pulled from the nested tensors. + + Returns: + Nested structure containing tensors with shape + [batch_size, new_beam_size, ...] + """ + # Computes the i'th coodinate that contains the batch index for gather_nd. + # Batch pos is a tensor like [[0,0,0,0,],[1,1,1,1],..]. + batch_pos = tf.range(batch_size * new_beam_size) // new_beam_size + batch_pos = tf.reshape(batch_pos, [batch_size, new_beam_size]) + + # Create coordinates to be passed to tf.gather_nd. Stacking creates a tensor + # with shape [batch_size, beam_size, 2], where the last dimension contains + # the (i, j) gathering coordinates. + coordinates = tf.stack([batch_pos, beam_indices], axis=2) + + return nest.map_structure( + lambda state: tf.gather_nd(state, coordinates), nested) + + +def _gather_topk_beams(nested, score_or_log_prob, batch_size, beam_size): + """Gather top beams from nested structure.""" + _, topk_indexes = tf.nn.top_k(score_or_log_prob, k=beam_size) + return _gather_beams(nested, topk_indexes, batch_size, beam_size) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/beam_search_v1_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/beam_search_v1_test.py new file mode 100644 index 0000000..53cf921 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/beam_search_v1_test.py @@ -0,0 +1,101 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Test beam search helper methods.""" + +import tensorflow.compat.v1 as tf + +from official.nlp.transformer import beam_search_v1 as beam_search + + +class BeamSearchHelperTests(tf.test.TestCase): + + def setUp(self): + super(BeamSearchHelperTests, self).setUp() + tf.compat.v1.disable_eager_execution() + + def test_expand_to_beam_size(self): + x = tf.ones([7, 4, 2, 5]) + x = beam_search._expand_to_beam_size(x, 3) + with self.session() as sess: + shape = sess.run(tf.shape(x)) + self.assertAllEqual([7, 3, 4, 2, 5], shape) + + def test_shape_list(self): + y = tf.compat.v1.placeholder(dtype=tf.int32, shape=[]) + x = tf.ones([7, y, 2, 5]) + shape = beam_search._shape_list(x) + self.assertIsInstance(shape[0], int) + self.assertIsInstance(shape[1], tf.Tensor) + self.assertIsInstance(shape[2], int) + self.assertIsInstance(shape[3], int) + + def test_get_shape_keep_last_dim(self): + y = tf.constant(4.0) + x = tf.ones([7, tf.cast(tf.sqrt(y), tf.int32), 2, 5]) + shape = beam_search._get_shape_keep_last_dim(x) + self.assertAllEqual([None, None, None, 5], + shape.as_list()) + + def test_flatten_beam_dim(self): + x = tf.ones([7, 4, 2, 5]) + x = beam_search._flatten_beam_dim(x) + with self.session() as sess: + shape = sess.run(tf.shape(x)) + self.assertAllEqual([28, 2, 5], shape) + + def test_unflatten_beam_dim(self): + x = tf.ones([28, 2, 5]) + x = beam_search._unflatten_beam_dim(x, 7, 4) + with self.session() as sess: + shape = sess.run(tf.shape(x)) + self.assertAllEqual([7, 4, 2, 5], shape) + + def test_gather_beams(self): + x = tf.reshape(tf.range(24), [2, 3, 4]) + # x looks like: [[[ 0 1 2 3] + # [ 4 5 6 7] + # [ 8 9 10 11]] + # + # [[12 13 14 15] + # [16 17 18 19] + # [20 21 22 23]]] + + y = beam_search._gather_beams(x, [[1, 2], [0, 2]], 2, 2) + with self.session() as sess: + y = sess.run(y) + + self.assertAllEqual([[[4, 5, 6, 7], + [8, 9, 10, 11]], + [[12, 13, 14, 15], + [20, 21, 22, 23]]], + y) + + def test_gather_topk_beams(self): + x = tf.reshape(tf.range(24), [2, 3, 4]) + x_scores = [[0, 1, 1], [1, 0, 1]] + + y = beam_search._gather_topk_beams(x, x_scores, 2, 2) + with self.session() as sess: + y = sess.run(y) + + self.assertAllEqual([[[4, 5, 6, 7], + [8, 9, 10, 11]], + [[12, 13, 14, 15], + [20, 21, 22, 23]]], + y) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/compute_bleu.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/compute_bleu.py new file mode 100644 index 0000000..92d54c3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/compute_bleu.py @@ -0,0 +1,148 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Script to compute official BLEU score. + +Source: +https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/bleu_hook.py +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import re +import sys +import unicodedata + +from absl import app as absl_app +from absl import flags +import six +from six.moves import range +import tensorflow as tf + +from official.nlp.transformer.utils import metrics +from official.nlp.transformer.utils import tokenizer +from official.utils.flags import core as flags_core + + +class UnicodeRegex(object): + """Ad-hoc hack to recognize all punctuation and symbols.""" + + def __init__(self): + punctuation = self.property_chars("P") + self.nondigit_punct_re = re.compile(r"([^\d])([" + punctuation + r"])") + self.punct_nondigit_re = re.compile(r"([" + punctuation + r"])([^\d])") + self.symbol_re = re.compile("([" + self.property_chars("S") + "])") + + def property_chars(self, prefix): + return "".join( + six.unichr(x) + for x in range(sys.maxunicode) + if unicodedata.category(six.unichr(x)).startswith(prefix)) + + +uregex = UnicodeRegex() + + +def bleu_tokenize(string): + r"""Tokenize a string following the official BLEU implementation. + + See https://github.com/moses-smt/mosesdecoder/' + 'blob/master/scripts/generic/mteval-v14.pl#L954-L983 + In our case, the input string is expected to be just one line + and no HTML entities de-escaping is needed. + So we just tokenize on punctuation and symbols, + except when a punctuation is preceded and followed by a digit + (e.g. a comma/dot as a thousand/decimal separator). + + Note that a numer (e.g. a year) followed by a dot at the end of sentence + is NOT tokenized, + i.e. the dot stays with the number because `s/(\p{P})(\P{N})/ $1 $2/g` + does not match this case (unless we add a space after each sentence). + However, this error is already in the original mteval-v14.pl + and we want to be consistent with it. + + Args: + string: the input string + + Returns: + a list of tokens + """ + string = uregex.nondigit_punct_re.sub(r"\1 \2 ", string) + string = uregex.punct_nondigit_re.sub(r" \1 \2", string) + string = uregex.symbol_re.sub(r" \1 ", string) + return string.split() + + +def bleu_wrapper(ref_filename, hyp_filename, case_sensitive=False): + """Compute BLEU for two files (reference and hypothesis translation).""" + ref_lines = tokenizer.native_to_unicode( + tf.io.gfile.GFile(ref_filename).read()).strip().splitlines() + hyp_lines = tokenizer.native_to_unicode( + tf.io.gfile.GFile(hyp_filename).read()).strip().splitlines() + + if len(ref_lines) != len(hyp_lines): + raise ValueError( + "Reference and translation files have different number of " + "lines (%d VS %d). If training only a few steps (100-200), the " + "translation may be empty." % (len(ref_lines), len(hyp_lines))) + if not case_sensitive: + ref_lines = [x.lower() for x in ref_lines] + hyp_lines = [x.lower() for x in hyp_lines] + ref_tokens = [bleu_tokenize(x) for x in ref_lines] + hyp_tokens = [bleu_tokenize(x) for x in hyp_lines] + return metrics.compute_bleu(ref_tokens, hyp_tokens) * 100 + + +def main(unused_argv): + if FLAGS.bleu_variant in ("both", "uncased"): + score = bleu_wrapper(FLAGS.reference, FLAGS.translation, False) + tf.logging.info("Case-insensitive results: %f" % score) + + if FLAGS.bleu_variant in ("both", "cased"): + score = bleu_wrapper(FLAGS.reference, FLAGS.translation, True) + tf.logging.info("Case-sensitive results: %f" % score) + + +def define_compute_bleu_flags(): + """Add flags for computing BLEU score.""" + flags.DEFINE_string( + name="translation", + default=None, + help=flags_core.help_wrap("File containing translated text.")) + flags.mark_flag_as_required("translation") + + flags.DEFINE_string( + name="reference", + default=None, + help=flags_core.help_wrap("File containing reference translation.")) + flags.mark_flag_as_required("reference") + + flags.DEFINE_enum( + name="bleu_variant", + short_name="bv", + default="both", + enum_values=["both", "uncased", "cased"], + case_sensitive=False, + help=flags_core.help_wrap( + "Specify one or more BLEU variants to calculate. Variants: \"cased\"" + ", \"uncased\", or \"both\".")) + + +if __name__ == "__main__": + tf.logging.set_verbosity(tf.logging.INFO) + define_compute_bleu_flags() + FLAGS = flags.FLAGS + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/compute_bleu_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/compute_bleu_test.py new file mode 100644 index 0000000..6c578e3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/compute_bleu_test.py @@ -0,0 +1,64 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Test functions in compute_blue.py.""" + +import tempfile + +import tensorflow as tf + +from official.nlp.transformer import compute_bleu + + +class ComputeBleuTest(tf.test.TestCase): + + def _create_temp_file(self, text): + temp_file = tempfile.NamedTemporaryFile(delete=False) + with tf.io.gfile.GFile(temp_file.name, "w") as w: + w.write(text) + return temp_file.name + + def test_bleu_same(self): + ref = self._create_temp_file("test 1 two 3\nmore tests!") + hyp = self._create_temp_file("test 1 two 3\nmore tests!") + + uncased_score = compute_bleu.bleu_wrapper(ref, hyp, False) + cased_score = compute_bleu.bleu_wrapper(ref, hyp, True) + self.assertEqual(100, uncased_score) + self.assertEqual(100, cased_score) + + def test_bleu_same_different_case(self): + ref = self._create_temp_file("Test 1 two 3\nmore tests!") + hyp = self._create_temp_file("test 1 two 3\nMore tests!") + uncased_score = compute_bleu.bleu_wrapper(ref, hyp, False) + cased_score = compute_bleu.bleu_wrapper(ref, hyp, True) + self.assertEqual(100, uncased_score) + self.assertLess(cased_score, 100) + + def test_bleu_different(self): + ref = self._create_temp_file("Testing\nmore tests!") + hyp = self._create_temp_file("Dog\nCat") + uncased_score = compute_bleu.bleu_wrapper(ref, hyp, False) + cased_score = compute_bleu.bleu_wrapper(ref, hyp, True) + self.assertLess(uncased_score, 100) + self.assertLess(cased_score, 100) + + def test_bleu_tokenize(self): + s = "Test0, 1 two, 3" + tokenized = compute_bleu.bleu_tokenize(s) + self.assertEqual(["Test0", ",", "1", "two", ",", "3"], tokenized) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/data_download.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/data_download.py new file mode 100644 index 0000000..54c5434 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/data_download.py @@ -0,0 +1,439 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Download and preprocess WMT17 ende training and evaluation datasets.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import random +import tarfile + +# pylint: disable=g-bad-import-order +from absl import app as absl_app +from absl import flags +from absl import logging +import six +from six.moves import range +from six.moves import urllib +from six.moves import zip +import tensorflow.compat.v1 as tf + +from official.nlp.transformer.utils import tokenizer +from official.utils.flags import core as flags_core +# pylint: enable=g-bad-import-order + +# Data sources for training/evaluating the transformer translation model. +# If any of the training sources are changed, then either: +# 1) use the flag `--search` to find the best min count or +# 2) update the _TRAIN_DATA_MIN_COUNT constant. +# min_count is the minimum number of times a token must appear in the data +# before it is added to the vocabulary. "Best min count" refers to the value +# that generates a vocabulary set that is closest in size to _TARGET_VOCAB_SIZE. +_TRAIN_DATA_SOURCES = [ + { + "url": "http://data.statmt.org/wmt17/translation-task/" + "training-parallel-nc-v12.tgz", + "input": "news-commentary-v12.de-en.en", + "target": "news-commentary-v12.de-en.de", + }, + { + "url": "http://www.statmt.org/wmt13/training-parallel-commoncrawl.tgz", + "input": "commoncrawl.de-en.en", + "target": "commoncrawl.de-en.de", + }, + { + "url": "http://www.statmt.org/wmt13/training-parallel-europarl-v7.tgz", + "input": "europarl-v7.de-en.en", + "target": "europarl-v7.de-en.de", + }, +] +# Use pre-defined minimum count to generate subtoken vocabulary. +_TRAIN_DATA_MIN_COUNT = 6 + +_EVAL_DATA_SOURCES = [ + { + "url": "http://data.statmt.org/wmt17/translation-task/dev.tgz", + "input": "newstest2013.en", + "target": "newstest2013.de", + } +] + +_TEST_DATA_SOURCES = [ + { + "url": ("https://storage.googleapis.com/tf-perf-public/" + "official_transformer/test_data/newstest2014.tgz"), + "input": "newstest2014.en", + "target": "newstest2014.de", + } +] + +# Vocabulary constants +_TARGET_VOCAB_SIZE = 32768 # Number of subtokens in the vocabulary list. +_TARGET_THRESHOLD = 327 # Accept vocabulary if size is within this threshold +VOCAB_FILE = "vocab.ende.%d" % _TARGET_VOCAB_SIZE + +# Strings to inclue in the generated files. +_PREFIX = "wmt32k" +_TRAIN_TAG = "train" +_EVAL_TAG = "dev" # Following WMT and Tensor2Tensor conventions, in which the +# evaluation datasets are tagged as "dev" for development. + +# Number of files to split train and evaluation data +_TRAIN_SHARDS = 100 +_EVAL_SHARDS = 1 + + +def find_file(path, filename, max_depth=5): + """Returns full filepath if the file is in path or a subdirectory.""" + for root, dirs, files in os.walk(path): + if filename in files: + return os.path.join(root, filename) + + # Don't search past max_depth + depth = root[len(path) + 1:].count(os.sep) + if depth > max_depth: + del dirs[:] # Clear dirs + return None + + +############################################################################### +# Download and extraction functions +############################################################################### +def get_raw_files(raw_dir, data_source): + """Return raw files from source. Downloads/extracts if needed. + + Args: + raw_dir: string directory to store raw files + data_source: dictionary with + {"url": url of compressed dataset containing input and target files + "input": file with data in input language + "target": file with data in target language} + + Returns: + dictionary with + {"inputs": list of files containing data in input language + "targets": list of files containing corresponding data in target language + } + """ + raw_files = { + "inputs": [], + "targets": [], + } # keys + for d in data_source: + input_file, target_file = download_and_extract( + raw_dir, d["url"], d["input"], d["target"]) + raw_files["inputs"].append(input_file) + raw_files["targets"].append(target_file) + return raw_files + + +def download_report_hook(count, block_size, total_size): + """Report hook for download progress. + + Args: + count: current block number + block_size: block size + total_size: total size + """ + percent = int(count * block_size * 100 / total_size) + print(six.ensure_str("\r%d%%" % percent) + " completed", end="\r") + + +def download_from_url(path, url): + """Download content from a url. + + Args: + path: string directory where file will be downloaded + url: string url + + Returns: + Full path to downloaded file + """ + filename = six.ensure_str(url).split("/")[-1] + found_file = find_file(path, filename, max_depth=0) + if found_file is None: + filename = os.path.join(path, filename) + logging.info("Downloading from %s to %s." % (url, filename)) + inprogress_filepath = six.ensure_str(filename) + ".incomplete" + inprogress_filepath, _ = urllib.request.urlretrieve( + url, inprogress_filepath, reporthook=download_report_hook) + # Print newline to clear the carriage return from the download progress. + print() + tf.gfile.Rename(inprogress_filepath, filename) + return filename + else: + logging.info("Already downloaded: %s (at %s)." % (url, found_file)) + return found_file + + +def download_and_extract(path, url, input_filename, target_filename): + """Extract files from downloaded compressed archive file. + + Args: + path: string directory where the files will be downloaded + url: url containing the compressed input and target files + input_filename: name of file containing data in source language + target_filename: name of file containing data in target language + + Returns: + Full paths to extracted input and target files. + + Raises: + OSError: if the the download/extraction fails. + """ + # Check if extracted files already exist in path + input_file = find_file(path, input_filename) + target_file = find_file(path, target_filename) + if input_file and target_file: + logging.info("Already downloaded and extracted %s." % url) + return input_file, target_file + + # Download archive file if it doesn't already exist. + compressed_file = download_from_url(path, url) + + # Extract compressed files + logging.info("Extracting %s." % compressed_file) + with tarfile.open(compressed_file, "r:gz") as corpus_tar: + corpus_tar.extractall(path) + + # Return file paths of the requested files. + input_file = find_file(path, input_filename) + target_file = find_file(path, target_filename) + + if input_file and target_file: + return input_file, target_file + + raise OSError("Download/extraction failed for url %s to path %s" % + (url, path)) + + +def txt_line_iterator(path): + """Iterate through lines of file.""" + with tf.io.gfile.GFile(path) as f: + for line in f: + yield line.strip() + + +def compile_files(raw_dir, raw_files, tag): + """Compile raw files into a single file for each language. + + Args: + raw_dir: Directory containing downloaded raw files. + raw_files: Dict containing filenames of input and target data. + {"inputs": list of files containing data in input language + "targets": list of files containing corresponding data in target language + } + tag: String to append to the compiled filename. + + Returns: + Full path of compiled input and target files. + """ + logging.info("Compiling files with tag %s." % tag) + filename = "%s-%s" % (_PREFIX, tag) + input_compiled_file = os.path.join(raw_dir, + six.ensure_str(filename) + ".lang1") + target_compiled_file = os.path.join(raw_dir, + six.ensure_str(filename) + ".lang2") + + with tf.io.gfile.GFile(input_compiled_file, mode="w") as input_writer: + with tf.io.gfile.GFile(target_compiled_file, mode="w") as target_writer: + for i in range(len(raw_files["inputs"])): + input_file = raw_files["inputs"][i] + target_file = raw_files["targets"][i] + + logging.info("Reading files %s and %s." % (input_file, target_file)) + write_file(input_writer, input_file) + write_file(target_writer, target_file) + return input_compiled_file, target_compiled_file + + +def write_file(writer, filename): + """Write all of lines from file using the writer.""" + for line in txt_line_iterator(filename): + writer.write(line) + writer.write("\n") + + +############################################################################### +# Data preprocessing +############################################################################### +def encode_and_save_files( + subtokenizer, data_dir, raw_files, tag, total_shards): + """Save data from files as encoded Examples in TFrecord format. + + Args: + subtokenizer: Subtokenizer object that will be used to encode the strings. + data_dir: The directory in which to write the examples + raw_files: A tuple of (input, target) data files. Each line in the input and + the corresponding line in target file will be saved in a tf.Example. + tag: String that will be added onto the file names. + total_shards: Number of files to divide the data into. + + Returns: + List of all files produced. + """ + # Create a file for each shard. + filepaths = [shard_filename(data_dir, tag, n + 1, total_shards) + for n in range(total_shards)] + + if all_exist(filepaths): + logging.info("Files with tag %s already exist." % tag) + return filepaths + + logging.info("Saving files with tag %s." % tag) + input_file = raw_files[0] + target_file = raw_files[1] + + # Write examples to each shard in round robin order. + tmp_filepaths = [six.ensure_str(fname) + ".incomplete" for fname in filepaths] + writers = [tf.python_io.TFRecordWriter(fname) for fname in tmp_filepaths] + counter, shard = 0, 0 + for counter, (input_line, target_line) in enumerate(zip( + txt_line_iterator(input_file), txt_line_iterator(target_file))): + if counter > 0 and counter % 100000 == 0: + logging.info("\tSaving case %d." % counter) + example = dict_to_example( + {"inputs": subtokenizer.encode(input_line, add_eos=True), + "targets": subtokenizer.encode(target_line, add_eos=True)}) + writers[shard].write(example.SerializeToString()) + shard = (shard + 1) % total_shards + for writer in writers: + writer.close() + + for tmp_name, final_name in zip(tmp_filepaths, filepaths): + tf.gfile.Rename(tmp_name, final_name) + + logging.info("Saved %d Examples", counter + 1) + return filepaths + + +def shard_filename(path, tag, shard_num, total_shards): + """Create filename for data shard.""" + return os.path.join( + path, "%s-%s-%.5d-of-%.5d" % (_PREFIX, tag, shard_num, total_shards)) + + +def shuffle_records(fname): + """Shuffle records in a single file.""" + logging.info("Shuffling records in file %s" % fname) + + # Rename file prior to shuffling + tmp_fname = six.ensure_str(fname) + ".unshuffled" + tf.gfile.Rename(fname, tmp_fname) + + reader = tf.io.tf_record_iterator(tmp_fname) + records = [] + for record in reader: + records.append(record) + if len(records) % 100000 == 0: + logging.info("\tRead: %d", len(records)) + + random.shuffle(records) + + # Write shuffled records to original file name + with tf.python_io.TFRecordWriter(fname) as w: + for count, record in enumerate(records): + w.write(record) + if count > 0 and count % 100000 == 0: + logging.info("\tWriting record: %d" % count) + + tf.gfile.Remove(tmp_fname) + + +def dict_to_example(dictionary): + """Converts a dictionary of string->int to a tf.Example.""" + features = {} + for k, v in six.iteritems(dictionary): + features[k] = tf.train.Feature(int64_list=tf.train.Int64List(value=v)) + return tf.train.Example(features=tf.train.Features(feature=features)) + + +def all_exist(filepaths): + """Returns true if all files in the list exist.""" + for fname in filepaths: + if not tf.gfile.Exists(fname): + return False + return True + + +def make_dir(path): + if not tf.gfile.Exists(path): + logging.info("Creating directory %s" % path) + tf.gfile.MakeDirs(path) + + +def main(unused_argv): + """Obtain training and evaluation data for the Transformer model.""" + make_dir(FLAGS.raw_dir) + make_dir(FLAGS.data_dir) + + # Download test_data + logging.info("Step 1/5: Downloading test data") + train_files = get_raw_files(FLAGS.data_dir, _TEST_DATA_SOURCES) + + # Get paths of download/extracted training and evaluation files. + logging.info("Step 2/5: Downloading data from source") + train_files = get_raw_files(FLAGS.raw_dir, _TRAIN_DATA_SOURCES) + eval_files = get_raw_files(FLAGS.raw_dir, _EVAL_DATA_SOURCES) + + # Create subtokenizer based on the training files. + logging.info("Step 3/5: Creating subtokenizer and building vocabulary") + train_files_flat = train_files["inputs"] + train_files["targets"] + vocab_file = os.path.join(FLAGS.data_dir, VOCAB_FILE) + subtokenizer = tokenizer.Subtokenizer.init_from_files( + vocab_file, train_files_flat, _TARGET_VOCAB_SIZE, _TARGET_THRESHOLD, + min_count=None if FLAGS.search else _TRAIN_DATA_MIN_COUNT) + + logging.info("Step 4/5: Compiling training and evaluation data") + compiled_train_files = compile_files(FLAGS.raw_dir, train_files, _TRAIN_TAG) + compiled_eval_files = compile_files(FLAGS.raw_dir, eval_files, _EVAL_TAG) + + # Tokenize and save data as Examples in the TFRecord format. + logging.info("Step 5/5: Preprocessing and saving data") + train_tfrecord_files = encode_and_save_files( + subtokenizer, FLAGS.data_dir, compiled_train_files, _TRAIN_TAG, + _TRAIN_SHARDS) + encode_and_save_files( + subtokenizer, FLAGS.data_dir, compiled_eval_files, _EVAL_TAG, + _EVAL_SHARDS) + + for fname in train_tfrecord_files: + shuffle_records(fname) + + +def define_data_download_flags(): + """Add flags specifying data download arguments.""" + flags.DEFINE_string( + name="data_dir", short_name="dd", default="/tmp/translate_ende", + help=flags_core.help_wrap( + "Directory for where the translate_ende_wmt32k dataset is saved.")) + flags.DEFINE_string( + name="raw_dir", short_name="rd", default="/tmp/translate_ende_raw", + help=flags_core.help_wrap( + "Path where the raw data will be downloaded and extracted.")) + flags.DEFINE_bool( + name="search", default=False, + help=flags_core.help_wrap( + "If set, use binary search to find the vocabulary set with size" + "closest to the target size (%d)." % _TARGET_VOCAB_SIZE)) + + +if __name__ == "__main__": + logging.set_verbosity(logging.INFO) + define_data_download_flags() + FLAGS = flags.FLAGS + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/data_pipeline.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/data_pipeline.py new file mode 100644 index 0000000..a9a1621 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/data_pipeline.py @@ -0,0 +1,317 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Input pipeline for the transformer model to read, filter, and batch examples. + +Two things to note in the pipeline: + +1. Batching scheme + + The examples encoded in the TFRecord files contain data in the format: + {"inputs": [variable length array of integers], + "targets": [variable length array of integers]} + Where integers in the arrays refer to tokens in the English and German vocab + file (named `vocab.ende.32768`). + + Prior to batching, elements in the dataset are grouped by length (max between + "inputs" and "targets" length). Each group is then batched such that: + group_batch_size * length <= batch_size. + + Another way to view batch_size is the maximum number of tokens in each batch. + + Once batched, each element in the dataset will have the shape: + {"inputs": [group_batch_size, padded_input_length], + "targets": [group_batch_size, padded_target_length]} + Lengths are padded to the longest "inputs" or "targets" sequence in the batch + (padded_input_length and padded_target_length can be different). + + This batching scheme decreases the fraction of padding tokens per training + batch, thus improving the training speed significantly. + +2. Shuffling + + While training, the dataset is shuffled in two places in the code. The first + is the list of training files. Second, while reading records using + `parallel_interleave`, the `sloppy` argument is used to generate randomness + in the order of the examples. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math +import os + +from absl import logging +import tensorflow as tf + +from official.nlp.transformer import misc +from official.utils.misc import model_helpers + +# Buffer size for reading records from a TFRecord file. Each training file is +# 7.2 MB, so 8 MB allows an entire file to be kept in memory. +_READ_RECORD_BUFFER = 8 * 1000 * 1000 + +# Example grouping constants. Defines length boundaries for each group. +# These values are the defaults used in Tensor2Tensor. +_MIN_BOUNDARY = 8 +_BOUNDARY_SCALE = 1.1 + + +def _load_records(filename): + """Read file and return a dataset of tf.Examples.""" + return tf.data.TFRecordDataset(filename, buffer_size=_READ_RECORD_BUFFER) + + +def _parse_example(serialized_example): + """Return inputs and targets Tensors from a serialized tf.Example.""" + data_fields = { + "inputs": tf.io.VarLenFeature(tf.int64), + "targets": tf.io.VarLenFeature(tf.int64) + } + parsed = tf.io.parse_single_example(serialized_example, data_fields) + inputs = tf.sparse.to_dense(parsed["inputs"]) + targets = tf.sparse.to_dense(parsed["targets"]) + return inputs, targets + + +def _filter_max_length(example, max_length=256): + """Indicates whether the example's length is lower than the maximum length.""" + return tf.logical_and(tf.size(example[0]) <= max_length, + tf.size(example[1]) <= max_length) + + +def _get_example_length(example): + """Returns the maximum length between the example inputs and targets.""" + length = tf.maximum(tf.shape(example[0])[0], tf.shape(example[1])[0]) + return length + + +def _create_min_max_boundaries( + max_length, min_boundary=_MIN_BOUNDARY, boundary_scale=_BOUNDARY_SCALE): + """Create min and max boundary lists up to max_length. + + For example, when max_length=24, min_boundary=4 and boundary_scale=2, the + returned values will be: + buckets_min = [0, 4, 8, 16, 24] + buckets_max = [4, 8, 16, 24, 25] + + Args: + max_length: The maximum length of example in dataset. + min_boundary: Minimum length in boundary. + boundary_scale: Amount to scale consecutive boundaries in the list. + + Returns: + min and max boundary lists + + """ + # Create bucket boundaries list by scaling the previous boundary or adding 1 + # (to ensure increasing boundary sizes). + bucket_boundaries = [] + x = min_boundary + while x < max_length: + bucket_boundaries.append(x) + x = max(x + 1, int(x * boundary_scale)) + + # Create min and max boundary lists from the initial list. + buckets_min = [0] + bucket_boundaries + buckets_max = bucket_boundaries + [max_length + 1] + return buckets_min, buckets_max + + +def _batch_examples(dataset, batch_size, max_length): + """Group examples by similar lengths, and return batched dataset. + + Each batch of similar-length examples are padded to the same length, and may + have different number of elements in each batch, such that: + group_batch_size * padded_length <= batch_size. + + This decreases the number of padding tokens per batch, which improves the + training speed. + + Args: + dataset: Dataset of unbatched examples. + batch_size: Max number of tokens per batch of examples. + max_length: Max number of tokens in an example input or target sequence. + + Returns: + Dataset of batched examples with similar lengths. + """ + # Get min and max boundary lists for each example. These are used to calculate + # the `bucket_id`, which is the index at which: + # buckets_min[bucket_id] <= len(example) < buckets_max[bucket_id] + # Note that using both min and max lists improves the performance. + buckets_min, buckets_max = _create_min_max_boundaries(max_length) + + # Create list of batch sizes for each bucket_id, so that + # bucket_batch_size[bucket_id] * buckets_max[bucket_id] <= batch_size + bucket_batch_sizes = [batch_size // x for x in buckets_max] + # bucket_id will be a tensor, so convert this list to a tensor as well. + bucket_batch_sizes = tf.constant(bucket_batch_sizes, dtype=tf.int64) + + def example_to_bucket_id(example_input, example_target): + """Return int64 bucket id for this example, calculated based on length.""" + seq_length = _get_example_length((example_input, example_target)) + + # TODO(xunkai): investigate if removing code branching improves performance. + conditions_c = tf.logical_and( + tf.less_equal(buckets_min, seq_length), + tf.less(seq_length, buckets_max)) + bucket_id = tf.reduce_min(tf.where(conditions_c)) + return bucket_id + + def window_size_fn(bucket_id): + """Return number of examples to be grouped when given a bucket id.""" + return bucket_batch_sizes[bucket_id] + + def batching_fn(bucket_id, grouped_dataset): + """Batch and add padding to a dataset of elements with similar lengths.""" + bucket_batch_size = window_size_fn(bucket_id) + + # Batch the dataset and add padding so that all input sequences in the + # examples have the same length, and all target sequences have the same + # lengths as well. Resulting lengths of inputs and targets can differ. + return grouped_dataset.padded_batch(bucket_batch_size, ([None], [None])) + + return dataset.apply(tf.data.experimental.group_by_window( + key_func=example_to_bucket_id, + reduce_func=batching_fn, + window_size=None, + window_size_func=window_size_fn)) + + +def _read_and_batch_from_files( + file_pattern, batch_size, max_length, num_parallel_calls, shuffle, repeat, + static_batch=False, num_replicas=1, ctx=None): + """Create dataset where each item is a dict of "inputs" and "targets". + + Args: + file_pattern: String used to match the input TFRecord files. + batch_size: Maximum number of tokens per global batch of examples. + max_length: Maximum number of tokens per example + num_parallel_calls: Number of cpu cores for parallel input processing. + shuffle: If true, randomizes order of elements. + repeat: Number of times to repeat the dataset. If None, the dataset is + repeated forever. + static_batch: Whether the batches in the dataset should have static shapes. + If True, the input is batched so that every batch has the + shape [batch_size // max_length, max_length]. If False, the input is + grouped by length, and batched so that batches may have different + shapes [N, M], where: + N * M <= batch_size + M <= max_length + In general, this setting should be False. Dynamic shapes allow the inputs + to be grouped so that the number of padding tokens is minimized, and helps + model training. In cases where the input shape must be static + (e.g. running on TPU), this setting should be set to True. + num_replicas: Number of GPUs or other workers. We will generate global + batches, and each global batch is equally divisible by number of replicas. + Currently it is only effective when static_batch==True. TODO: make it + effective when static_batch=False. + ctx: Input context. + + Returns: + tf.data.Dataset object containing examples loaded from the files. + """ + dataset = tf.data.Dataset.list_files(file_pattern, shuffle=shuffle) + + if ctx and ctx.num_input_pipelines > 1: + logging.info("Shard %d of the dataset.", ctx.input_pipeline_id) + dataset = dataset.shard(ctx.num_input_pipelines, ctx.input_pipeline_id) + + # Read files and interleave results. When training, the order of the examples + # will be non-deterministic. + options = tf.data.Options() + options.experimental_deterministic = False + dataset = dataset.interleave( + _load_records, + cycle_length=num_parallel_calls, + num_parallel_calls=tf.data.experimental.AUTOTUNE).with_options(options) + + # Parse each tf.Example into a dictionary + # TODO: Look into prefetch_input_elements for performance optimization. + dataset = dataset.map(_parse_example, + num_parallel_calls=num_parallel_calls) + + # Remove examples where the input or target length exceeds the maximum length, + dataset = dataset.filter(lambda x, y: _filter_max_length((x, y), max_length)) + + if static_batch: + dataset = dataset.padded_batch( + # First calculate batch size (token number) per worker, then divide it + # into sentences, and finally expand to a global batch. It could prove + # the global batch divisble for distribution strategy. + int(batch_size // num_replicas // max_length * num_replicas), + ([max_length], [max_length]), drop_remainder=True) + else: + # Group and batch such that each batch has examples of similar length. + # TODO(xunkai): _batch_examples might need to do something special for + # num_replicas. + dataset = _batch_examples(dataset, batch_size, max_length) + + dataset = dataset.repeat(repeat) + + # Prefetch the next element to improve speed of input pipeline. + dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE) + return dataset + + +def _generate_synthetic_data(params): + """Create synthetic data based on the parameter batch size.""" + batch = length = int(math.sqrt(params["batch_size"])) + dataset = model_helpers.generate_synthetic_data( + input_shape=tf.TensorShape([length]), + input_value=1, + input_dtype=tf.int64, + label_shape=tf.TensorShape([length]), + label_value=1, + label_dtype=tf.int64, + ) + return dataset.batch(batch, drop_remainder=True) + + +def train_input_fn(params, ctx=None): + """Load and return dataset of batched examples for use during training.""" + file_pattern = os.path.join(params["data_dir"] or "", "*train*") + if params["use_synthetic_data"]: + return _generate_synthetic_data(params) + return _read_and_batch_from_files( + file_pattern, params["batch_size"], params["max_length"], + params["num_parallel_calls"], shuffle=True, + repeat=params["repeat_dataset"], static_batch=params["static_batch"], + num_replicas=params["num_gpus"], ctx=ctx) + + +def eval_input_fn(params, ctx=None): + """Load and return dataset of batched examples for use during evaluation.""" + file_pattern = os.path.join(params["data_dir"] or "", "*dev*") + if params["use_synthetic_data"]: + return _generate_synthetic_data(params) + return _read_and_batch_from_files( + file_pattern, params["batch_size"], params["max_length"], + params["num_parallel_calls"], shuffle=False, repeat=1, + static_batch=params["static_batch"], num_replicas=params["num_gpus"], + ctx=ctx) + + +def map_data_for_transformer_fn(x, y): + """Maps data for training, and handles weried behaviors for different vers.""" + # Will transform input x and targets y into tuple(x, y) as new model inputs. + if misc.is_v2(): + # For TF v2, the 2nd parameter is omitted to make Keras training work. + return ((x, y),) + else: + # For TF v1, Keras requires a dummy placeholder as the 2nd parameter. + return ((x, y), tf.constant(0.0)) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/embedding_layer.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/embedding_layer.py new file mode 100644 index 0000000..6694e2b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/embedding_layer.py @@ -0,0 +1,103 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Implementation of embedding layer with shared weights.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + + +class EmbeddingSharedWeights(tf.keras.layers.Layer): + """Calculates input embeddings and pre-softmax linear with shared weights.""" + + def __init__(self, vocab_size, hidden_size): + """Specify characteristic parameters of embedding layer. + + Args: + vocab_size: Number of tokens in the embedding. (Typically ~32,000) + hidden_size: Dimensionality of the embedding. (Typically 512 or 1024) + """ + super(EmbeddingSharedWeights, self).__init__() + self.vocab_size = vocab_size + self.hidden_size = hidden_size + + def build(self, input_shape): + """Build embedding layer.""" + with tf.name_scope("embedding_and_softmax"): + # Create and initialize weights. The random normal initializer was chosen + # arbitrarily, and works well. + self.shared_weights = self.add_weight( + "weights", + shape=[self.vocab_size, self.hidden_size], + initializer=tf.random_normal_initializer( + mean=0., stddev=self.hidden_size**-0.5)) + super(EmbeddingSharedWeights, self).build(input_shape) + + def get_config(self): + return { + "vocab_size": self.vocab_size, + "hidden_size": self.hidden_size, + } + + def call(self, inputs, mode="embedding"): + """Get token embeddings of inputs. + + Args: + inputs: An int64 tensor with shape [batch_size, length] + mode: string, a valid value is one of "embedding" and "linear". + Returns: + outputs: (1) If mode == "embedding", output embedding tensor, float32 with + shape [batch_size, length, embedding_size]; (2) mode == "linear", output + linear tensor, float32 with shape [batch_size, length, vocab_size]. + Raises: + ValueError: if mode is not valid. + """ + if mode == "embedding": + return self._embedding(inputs) + elif mode == "linear": + return self._linear(inputs) + else: + raise ValueError("mode {} is not valid.".format(mode)) + + def _embedding(self, inputs): + """Applies embedding based on inputs tensor.""" + with tf.name_scope("embedding"): + # Create binary mask of size [batch_size, length] + embeddings = tf.gather(self.shared_weights, inputs) + mask = tf.cast(tf.not_equal(inputs, 0), embeddings.dtype) + embeddings *= tf.expand_dims(mask, -1) + # Scale embedding by the sqrt of the hidden size + embeddings *= self.hidden_size ** 0.5 + + return embeddings + + def _linear(self, inputs): + """Computes logits by running inputs through a linear layer. + + Args: + inputs: A float32 tensor with shape [batch_size, length, hidden_size] + Returns: + float32 tensor with shape [batch_size, length, vocab_size]. + """ + with tf.name_scope("presoftmax_linear"): + batch_size = tf.shape(inputs)[0] + length = tf.shape(inputs)[1] + + x = tf.reshape(inputs, [-1, self.hidden_size]) + logits = tf.matmul(x, self.shared_weights, transpose_b=True) + + return tf.reshape(logits, [batch_size, length, self.vocab_size]) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/ffn_layer.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/ffn_layer.py new file mode 100644 index 0000000..a7785f2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/ffn_layer.py @@ -0,0 +1,77 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Implementation of fully connected network.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + + +class FeedForwardNetwork(tf.keras.layers.Layer): + """Fully connected feedforward network.""" + + def __init__(self, hidden_size, filter_size, relu_dropout): + """Initialize FeedForwardNetwork. + + Args: + hidden_size: int, output dim of hidden layer. + filter_size: int, filter size for the inner (first) dense layer. + relu_dropout: float, dropout rate for training. + """ + super(FeedForwardNetwork, self).__init__() + self.hidden_size = hidden_size + self.filter_size = filter_size + self.relu_dropout = relu_dropout + + def build(self, input_shape): + self.filter_dense_layer = tf.keras.layers.Dense( + self.filter_size, + use_bias=True, + activation=tf.nn.relu, + name="filter_layer") + self.output_dense_layer = tf.keras.layers.Dense( + self.hidden_size, use_bias=True, name="output_layer") + super(FeedForwardNetwork, self).build(input_shape) + + def get_config(self): + return { + "hidden_size": self.hidden_size, + "filter_size": self.filter_size, + "relu_dropout": self.relu_dropout, + } + + def call(self, x, training): + """Return outputs of the feedforward network. + + Args: + x: tensor with shape [batch_size, length, hidden_size] + training: boolean, whether in training mode or not. + + Returns: + Output of the feedforward network. + tensor with shape [batch_size, length, hidden_size] + """ + # Retrieve dynamically known shapes + batch_size = tf.shape(x)[0] + length = tf.shape(x)[1] + + output = self.filter_dense_layer(x) + if training: + output = tf.nn.dropout(output, rate=self.relu_dropout) + output = self.output_dense_layer(output) + + return output diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/metrics.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/metrics.py new file mode 100644 index 0000000..4bd6bba --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/metrics.py @@ -0,0 +1,183 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functions for calculating loss, accuracy, and other model metrics. + +Metrics: + - Padded loss, accuracy, and negative log perplexity. Source: + https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/metrics.py + - BLEU approximation. Source: + https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/bleu_hook.py + - ROUGE score. Source: + https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/rouge.py +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools + +import tensorflow as tf + + +def _pad_tensors_to_same_length(x, y): + """Pad x and y so that the results have the same length (second dimension).""" + with tf.name_scope("pad_to_same_length"): + x_length = tf.shape(x)[1] + y_length = tf.shape(y)[1] + + max_length = tf.maximum(x_length, y_length) + + x = tf.pad(x, [[0, 0], [0, max_length - x_length], [0, 0]]) + y = tf.pad(y, [[0, 0], [0, max_length - y_length]]) + return x, y + + +def padded_cross_entropy_loss(logits, labels, smoothing, vocab_size): + """Calculate cross entropy loss while ignoring padding. + + Args: + logits: Tensor of size [batch_size, length_logits, vocab_size] + labels: Tensor of size [batch_size, length_labels] + smoothing: Label smoothing constant, used to determine the on and off values + vocab_size: int size of the vocabulary + + Returns: + Returns the cross entropy loss and weight tensors: float32 tensors with + shape [batch_size, max(length_logits, length_labels)] + """ + with tf.name_scope("loss"): + logits, labels = _pad_tensors_to_same_length(logits, labels) + + # Calculate smoothing cross entropy + with tf.name_scope("smoothing_cross_entropy"): + confidence = 1.0 - smoothing + low_confidence = (1.0 - confidence) / tf.cast(vocab_size - 1, tf.float32) + soft_targets = tf.one_hot( + tf.cast(labels, tf.int32), + depth=vocab_size, + on_value=confidence, + off_value=low_confidence) + xentropy = tf.nn.softmax_cross_entropy_with_logits( + logits=logits, labels=soft_targets) + + # Calculate the best (lowest) possible value of cross entropy, and + # subtract from the cross entropy loss. + normalizing_constant = -( + confidence * tf.math.log(confidence) + + tf.cast(vocab_size - 1, tf.float32) * low_confidence * + tf.math.log(low_confidence + 1e-20)) + xentropy -= normalizing_constant + + weights = tf.cast(tf.not_equal(labels, 0), tf.float32) + return xentropy * weights, weights + + +def padded_accuracy(logits, labels): + """Percentage of times that predictions matches labels on non-0s.""" + with tf.name_scope("padded_accuracy"): + logits, labels = _pad_tensors_to_same_length(logits, labels) + weights = tf.cast(tf.not_equal(labels, 0), tf.float32) + outputs = tf.cast(tf.argmax(logits, axis=-1), tf.int32) + padded_labels = tf.cast(labels, tf.int32) + return tf.cast(tf.equal(outputs, padded_labels), tf.float32), weights + + +def padded_accuracy_topk(logits, labels, k): + """Percentage of times that top-k predictions matches labels on non-0s.""" + with tf.name_scope("padded_accuracy_topk"): + logits, labels = _pad_tensors_to_same_length(logits, labels) + weights = tf.cast(tf.not_equal(labels, 0), tf.float32) + effective_k = tf.minimum(k, tf.shape(logits)[-1]) + _, outputs = tf.nn.top_k(logits, k=effective_k) + outputs = tf.cast(outputs, tf.int32) + padded_labels = tf.cast(labels, tf.int32) + padded_labels = tf.expand_dims(padded_labels, axis=-1) + padded_labels += tf.zeros_like(outputs) # Pad to same shape. + same = tf.cast(tf.equal(outputs, padded_labels), tf.float32) + same_topk = tf.reduce_sum(same, axis=-1) + return same_topk, weights + + +def padded_accuracy_top5(logits, labels): + return padded_accuracy_topk(logits, labels, 5) + + +def padded_sequence_accuracy(logits, labels): + """Percentage of times that predictions matches labels everywhere (non-0).""" + with tf.name_scope("padded_sequence_accuracy"): + logits, labels = _pad_tensors_to_same_length(logits, labels) + weights = tf.cast(tf.not_equal(labels, 0), tf.float32) + outputs = tf.cast(tf.argmax(logits, axis=-1), tf.int32) + padded_labels = tf.cast(labels, tf.int32) + not_correct = tf.cast(tf.not_equal(outputs, padded_labels), + tf.float32) * weights + axis = list(range(1, len(outputs.get_shape()))) + correct_seq = 1.0 - tf.minimum(1.0, tf.reduce_sum(not_correct, axis=axis)) + return correct_seq, tf.constant(1.0) + + +def padded_neg_log_perplexity(logits, labels, vocab_size): + """Average log-perplexity excluding padding 0s. No smoothing.""" + num, den = padded_cross_entropy_loss(logits, labels, 0, vocab_size) + return -num, den + + +class MetricLayer(tf.keras.layers.Layer): + """Custom a layer of metrics for Transformer model.""" + + def __init__(self, vocab_size): + super(MetricLayer, self).__init__() + self.vocab_size = vocab_size + self.metric_mean_fns = [] + + def build(self, input_shape): + """"Builds metric layer.""" + neg_log_perplexity = functools.partial( + padded_neg_log_perplexity, vocab_size=self.vocab_size) + self.metric_mean_fns = [ + (tf.keras.metrics.Mean("accuracy"), padded_accuracy), + (tf.keras.metrics.Mean("accuracy_top5"), padded_accuracy_top5), + (tf.keras.metrics.Mean("accuracy_per_sequence"), + padded_sequence_accuracy), + (tf.keras.metrics.Mean("neg_log_perplexity"), neg_log_perplexity), + ] + super(MetricLayer, self).build(input_shape) + + def get_config(self): + return {"vocab_size": self.vocab_size} + + def call(self, inputs): + logits, targets = inputs[0], inputs[1] + for mean, fn in self.metric_mean_fns: + m = mean(*fn(logits, targets)) + self.add_metric(m) + return logits + + +def transformer_loss(logits, labels, smoothing, vocab_size): + """Calculates total loss containing cross entropy with padding ignored. + + Args: + logits: Tensor of size [batch_size, length_logits, vocab_size] + labels: Tensor of size [batch_size, length_labels] + smoothing: Label smoothing constant, used to determine the on and off values + vocab_size: int size of the vocabulary + + Returns: + A scalar float tensor for loss. + """ + xentropy, weights = padded_cross_entropy_loss(logits, labels, smoothing, + vocab_size) + return tf.reduce_sum(xentropy) / tf.reduce_sum(weights) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/misc.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/misc.py new file mode 100644 index 0000000..3eb430f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/misc.py @@ -0,0 +1,296 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Misc for Transformer.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# pylint: disable=g-bad-import-order +from absl import flags +import tensorflow as tf + +# TODO(tianlin) Import internal library. Remove this when some functions for +# different TF versions are fixed. +from tensorflow.python import tf2 as tf2_internal + +from official.nlp.transformer import model_params +from official.utils.flags import core as flags_core +from official.utils.misc import keras_utils + +FLAGS = flags.FLAGS + +PARAMS_MAP = { + 'tiny': model_params.TINY_PARAMS, + 'base': model_params.BASE_PARAMS, + 'big': model_params.BIG_PARAMS, +} + + +def is_v2(): + """Returns whether it is v2.""" + return tf2_internal.enabled() + + +def get_model_params(param_set, num_gpus): + """Gets predefined model params.""" + if num_gpus > 1: + if param_set == 'big': + return model_params.BIG_MULTI_GPU_PARAMS.copy() + elif param_set == 'base': + return model_params.BASE_MULTI_GPU_PARAMS.copy() + else: + raise ValueError('Not valid params: param_set={} num_gpus={}'.format( + param_set, num_gpus)) + + return PARAMS_MAP[param_set].copy() + + +def define_transformer_flags(): + """Add flags and flag validators for running transformer_main.""" + # Add common flags (data_dir, model_dir, etc.). + flags_core.define_base(num_gpu=True, distribution_strategy=True) + flags_core.define_performance( + num_parallel_calls=True, + inter_op=False, + intra_op=False, + synthetic_data=True, + max_train_steps=False, + dtype=True, + loss_scale=True, + all_reduce_alg=True, + num_packs=True, + tf_gpu_thread_mode=True, + datasets_num_private_threads=True, + enable_xla=True, + fp16_implementation=True + ) + + # Additional performance flags + # TODO(b/76028325): Remove when generic layout optimizer is ready. + flags.DEFINE_boolean( + name='enable_grappler_layout_optimizer', + default=True, + help='Enable Grappler layout optimizer. Currently Grappler can ' + 'de-optimize fp16 graphs by forcing NCHW layout for all ' + 'convolutions and batch normalizations, and this flag allows to ' + 'disable it.' + ) + + flags_core.define_benchmark() + flags_core.define_device(tpu=True) + + flags.DEFINE_integer( + name='train_steps', short_name='ts', default=300000, + help=flags_core.help_wrap('The number of steps used to train.')) + flags.DEFINE_integer( + name='steps_between_evals', short_name='sbe', default=1000, + help=flags_core.help_wrap( + 'The Number of training steps to run between evaluations. This is ' + 'used if --train_steps is defined.')) + flags.DEFINE_boolean( + name='enable_time_history', default=True, + help='Whether to enable TimeHistory callback.') + flags.DEFINE_boolean( + name='enable_tensorboard', default=False, + help='Whether to enable Tensorboard callback.') + flags.DEFINE_boolean( + name='enable_metrics_in_training', default=False, + help='Whether to enable metrics during training.') + flags.DEFINE_string( + name='profile_steps', default=None, + help='Save profiling data to model dir at given range of steps. The ' + 'value must be a comma separated pair of positive integers, specifying ' + 'the first and last step to profile. For example, "--profile_steps=2,4" ' + 'triggers the profiler to process 3 steps, starting from the 2nd step. ' + 'Note that profiler has a non-trivial performance overhead, and the ' + 'output file can be gigantic if profiling many steps.') + # Set flags from the flags_core module as 'key flags' so they're listed when + # the '-h' flag is used. Without this line, the flags defined above are + # only shown in the full `--helpful` help text. + flags.adopt_module_key_flags(flags_core) + + # Add transformer-specific flags + flags.DEFINE_enum( + name='param_set', short_name='mp', default='big', + enum_values=PARAMS_MAP.keys(), + help=flags_core.help_wrap( + 'Parameter set to use when creating and training the model. The ' + 'parameters define the input shape (batch size and max length), ' + 'model configuration (size of embedding, # of hidden layers, etc.), ' + 'and various other settings. The big parameter set increases the ' + 'default batch size, embedding/hidden size, and filter size. For a ' + 'complete list of parameters, please see model/model_params.py.')) + + flags.DEFINE_bool( + name='static_batch', short_name='sb', default=False, + help=flags_core.help_wrap( + 'Whether the batches in the dataset should have static shapes. In ' + 'general, this setting should be False. Dynamic shapes allow the ' + 'inputs to be grouped so that the number of padding tokens is ' + 'minimized, and helps model training. In cases where the input shape ' + 'must be static (e.g. running on TPU), this setting will be ignored ' + 'and static batching will always be used.')) + flags.DEFINE_integer( + name='max_length', short_name='ml', default=256, + help=flags_core.help_wrap( + 'Max sentence length for Transformer. Default is 256. Note: Usually ' + 'it is more effective to use a smaller max length if static_batch is ' + 'enabled, e.g. 64.')) + + # Flags for training with steps (may be used for debugging) + flags.DEFINE_integer( + name='validation_steps', short_name='vs', default=64, + help=flags_core.help_wrap('The number of steps used in validation.')) + + # BLEU score computation + flags.DEFINE_string( + name='bleu_source', short_name='bls', default=None, + help=flags_core.help_wrap( + 'Path to source file containing text translate when calculating the ' + 'official BLEU score. Both --bleu_source and --bleu_ref must be set. ' + )) + flags.DEFINE_string( + name='bleu_ref', short_name='blr', default=None, + help=flags_core.help_wrap( + 'Path to source file containing text translate when calculating the ' + 'official BLEU score. Both --bleu_source and --bleu_ref must be set. ' + )) + flags.DEFINE_string( + name='vocab_file', short_name='vf', default=None, + help=flags_core.help_wrap( + 'Path to subtoken vocabulary file. If data_download.py was used to ' + 'download and encode the training data, look in the data_dir to find ' + 'the vocab file.')) + flags.DEFINE_string( + name='mode', default='train', + help=flags_core.help_wrap('mode: train, eval, or predict')) + flags.DEFINE_bool( + name='use_ctl', + default=False, + help=flags_core.help_wrap( + 'Whether the model runs with custom training loop.')) + flags.DEFINE_integer( + name='decode_batch_size', + default=32, + help=flags_core.help_wrap( + 'Global batch size used for Transformer autoregressive decoding on ' + 'TPU.')) + flags.DEFINE_integer( + name='decode_max_length', + default=97, + help=flags_core.help_wrap( + 'Max sequence length of the decode/eval data. This is used by ' + 'Transformer autoregressive decoding on TPU to have minimum ' + 'paddings.')) + flags.DEFINE_bool( + name='padded_decode', + default=False, + help=flags_core.help_wrap( + 'Whether the autoregressive decoding runs with input data padded to ' + 'the decode_max_length. For TPU/XLA-GPU runs, this flag has to be ' + 'set due the static shape requirement. Although CPU/GPU could also ' + 'use padded_decode, it has not been tested. In addition, this method ' + 'will introduce unnecessary overheads which grow quadratically with ' + 'the max sequence length.')) + flags.DEFINE_bool( + name='enable_checkpointing', + default=True, + help=flags_core.help_wrap( + 'Whether to do checkpointing during training. When running under ' + 'benchmark harness, we will avoid checkpointing.')) + + flags_core.set_defaults(data_dir='/tmp/translate_ende', + model_dir='/tmp/transformer_model', + batch_size=None) + + # pylint: disable=unused-variable + @flags.multi_flags_validator( + ['bleu_source', 'bleu_ref'], + message='Both or neither --bleu_source and --bleu_ref must be defined.') + def _check_bleu_files(flags_dict): + return (flags_dict['bleu_source'] is None) == ( + flags_dict['bleu_ref'] is None) + + @flags.multi_flags_validator( + ['bleu_source', 'bleu_ref', 'vocab_file'], + message='--vocab_file must be defined if --bleu_source and --bleu_ref ' + 'are defined.') + def _check_bleu_vocab_file(flags_dict): + if flags_dict['bleu_source'] and flags_dict['bleu_ref']: + return flags_dict['vocab_file'] is not None + return True + # pylint: enable=unused-variable + + +def get_callbacks(steps_per_epoch): + """Returns common callbacks.""" + callbacks = [] + if FLAGS.enable_time_history: + time_callback = keras_utils.TimeHistory( + FLAGS.batch_size, + FLAGS.log_steps, + FLAGS.model_dir if FLAGS.enable_tensorboard else None) + callbacks.append(time_callback) + + if FLAGS.enable_tensorboard: + tensorboard_callback = tf.keras.callbacks.TensorBoard( + log_dir=FLAGS.model_dir) + callbacks.append(tensorboard_callback) + + if FLAGS.profile_steps: + profiler_callback = keras_utils.get_profiler_callback( + FLAGS.model_dir, + FLAGS.profile_steps, + FLAGS.enable_tensorboard, + steps_per_epoch) + callbacks.append(profiler_callback) + + return callbacks + + +def build_stats(history, callbacks): + """Normalizes and returns dictionary of stats. + + Args: + history: Results of the training step. + callbacks: a list of callbacks which might include a time history callback + used during keras.fit. + + Returns: + Dictionary of normalized results. + """ + stats = {} + + if history and history.history: + train_hist = history.history + # Gets final loss from training. + stats['loss'] = float(train_hist['loss'][-1]) + + if not callbacks: + return stats + + # Look for the time history callback which was used during keras.fit + for callback in callbacks: + if isinstance(callback, keras_utils.TimeHistory): + timestamp_log = callback.timestamp_log + stats['step_timestamp_log'] = timestamp_log + stats['train_finish_time'] = callback.train_finish_time + if len(timestamp_log) > 1: + stats['avg_exp_per_second'] = ( + callback.batch_size * callback.log_steps * + (len(callback.timestamp_log)-1) / + (timestamp_log[-1].timestamp - timestamp_log[0].timestamp)) + return stats diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/model_params.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/model_params.py new file mode 100644 index 0000000..e978abe --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/model_params.py @@ -0,0 +1,96 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Defines Transformer model parameters.""" + +from collections import defaultdict + + +BASE_PARAMS = defaultdict( + lambda: None, # Set default value to None. + + # Input params + default_batch_size=2048, # Maximum number of tokens per batch of examples. + default_batch_size_tpu=32768, + max_length=256, # Maximum number of tokens per example. + + # Model params + initializer_gain=1.0, # Used in trainable variable initialization. + vocab_size=33708, # Number of tokens defined in the vocabulary file. + hidden_size=512, # Model dimension in the hidden layers. + num_hidden_layers=6, # Number of layers in the encoder and decoder stacks. + num_heads=8, # Number of heads to use in multi-headed attention. + filter_size=2048, # Inner layer dimension in the feedforward network. + + # Dropout values (only used when training) + layer_postprocess_dropout=0.1, + attention_dropout=0.1, + relu_dropout=0.1, + + # Training params + label_smoothing=0.1, + learning_rate=2.0, + learning_rate_decay_rate=1.0, + learning_rate_warmup_steps=16000, + + # Optimizer params + optimizer_adam_beta1=0.9, + optimizer_adam_beta2=0.997, + optimizer_adam_epsilon=1e-09, + + # Default prediction params + extra_decode_length=50, + beam_size=4, + alpha=0.6, # used to calculate length normalization in beam search + + # TPU specific parameters + use_tpu=False, + static_batch=False, + allow_ffn_pad=True, +) + +BIG_PARAMS = BASE_PARAMS.copy() +BIG_PARAMS.update( + default_batch_size=4096, + + # default batch size is smaller than for BASE_PARAMS due to memory limits. + default_batch_size_tpu=16384, + + hidden_size=1024, + filter_size=4096, + num_heads=16, +) + +# Parameters for running the model in multi gpu. These should not change the +# params that modify the model shape (such as the hidden_size or num_heads). +BASE_MULTI_GPU_PARAMS = BASE_PARAMS.copy() +BASE_MULTI_GPU_PARAMS.update( + learning_rate_warmup_steps=8000 +) + +BIG_MULTI_GPU_PARAMS = BIG_PARAMS.copy() +BIG_MULTI_GPU_PARAMS.update( + layer_postprocess_dropout=0.3, + learning_rate_warmup_steps=8000 +) + +# Parameters for testing the model +TINY_PARAMS = BASE_PARAMS.copy() +TINY_PARAMS.update( + default_batch_size=1024, + default_batch_size_tpu=1024, + hidden_size=32, + num_heads=4, + filter_size=256, +) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/model_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/model_utils.py new file mode 100644 index 0000000..3f860f0 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/model_utils.py @@ -0,0 +1,123 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Transformer model helper methods.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math + +import numpy as np +import tensorflow as tf + +# Very low numbers to represent -infinity. We do not actually use -Inf, since we +# want to be able to multiply these values by zero to get zero. (-Inf * 0 = NaN) +_NEG_INF_FP32 = -1e9 +_NEG_INF_FP16 = np.finfo(np.float16).min + + +def get_position_encoding( + length, hidden_size, min_timescale=1.0, max_timescale=1.0e4): + """Return positional encoding. + + Calculates the position encoding as a mix of sine and cosine functions with + geometrically increasing wavelengths. + Defined and formulized in Attention is All You Need, section 3.5. + + Args: + length: Sequence length. + hidden_size: Size of the + min_timescale: Minimum scale that will be applied at each position + max_timescale: Maximum scale that will be applied at each position + + Returns: + Tensor with shape [length, hidden_size] + """ + # We compute the positional encoding in float32 even if the model uses + # float16, as many of the ops used, like log and exp, are numerically unstable + # in float16. + position = tf.cast(tf.range(length), tf.float32) + num_timescales = hidden_size // 2 + log_timescale_increment = ( + math.log(float(max_timescale) / float(min_timescale)) / + (tf.cast(num_timescales, tf.float32) - 1)) + inv_timescales = min_timescale * tf.exp( + tf.cast(tf.range(num_timescales), tf.float32) * -log_timescale_increment) + scaled_time = tf.expand_dims(position, 1) * tf.expand_dims(inv_timescales, 0) + signal = tf.concat([tf.sin(scaled_time), tf.cos(scaled_time)], axis=1) + return signal + + +def get_decoder_self_attention_bias(length, dtype=tf.float32): + """Calculate bias for decoder that maintains model's autoregressive property. + + Creates a tensor that masks out locations that correspond to illegal + connections, so prediction at position i cannot draw information from future + positions. + + Args: + length: int length of sequences in batch. + dtype: The dtype of the return value. + + Returns: + float tensor of shape [1, 1, length, length] + """ + neg_inf = _NEG_INF_FP16 if dtype == tf.float16 else _NEG_INF_FP32 + with tf.name_scope("decoder_self_attention_bias"): + valid_locs = tf.linalg.band_part(tf.ones([length, length], dtype=dtype), + -1, 0) + valid_locs = tf.reshape(valid_locs, [1, 1, length, length]) + decoder_bias = neg_inf * (1.0 - valid_locs) + return decoder_bias + + +def get_padding(x, padding_value=0, dtype=tf.float32): + """Return float tensor representing the padding values in x. + + Args: + x: int tensor with any shape + padding_value: int which represents padded values in input + dtype: The dtype of the return value. + + Returns: + float tensor with same shape as x containing values 0 or 1. + 0 -> non-padding, 1 -> padding + """ + with tf.name_scope("padding"): + return tf.cast(tf.equal(x, padding_value), dtype) + + +def get_padding_bias(x, padding_value=0, dtype=tf.float32): + """Calculate bias tensor from padding values in tensor. + + Bias tensor that is added to the pre-softmax multi-headed attention logits, + which has shape [batch_size, num_heads, length, length]. The tensor is zero at + non-padding locations, and -1e9 (negative infinity) at padding locations. + + Args: + x: int tensor with shape [batch_size, length] + padding_value: int which represents padded values in input + dtype: The dtype of the return value + + Returns: + Attention bias tensor of shape [batch_size, 1, 1, length]. + """ + with tf.name_scope("attention_bias"): + padding = get_padding(x, padding_value, dtype) + attention_bias = padding * _NEG_INF_FP32 + attention_bias = tf.expand_dims( + tf.expand_dims(attention_bias, axis=1), axis=1) + return attention_bias diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/model_utils_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/model_utils_test.py new file mode 100644 index 0000000..a8c4a15 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/model_utils_test.py @@ -0,0 +1,62 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Test Transformer model helper methods.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from official.nlp.transformer import model_utils + +NEG_INF = -1e9 + + +class ModelUtilsTest(tf.test.TestCase): + + def test_get_padding(self): + x = tf.constant([[1, 0, 0, 0, 2], [3, 4, 0, 0, 0], [0, 5, 6, 0, 7]]) + padding = model_utils.get_padding(x, padding_value=0) + + self.assertAllEqual([[0, 1, 1, 1, 0], [0, 0, 1, 1, 1], [1, 0, 0, 1, 0]], + padding) + + def test_get_padding_bias(self): + x = tf.constant([[1, 0, 0, 0, 2], [3, 4, 0, 0, 0], [0, 5, 6, 0, 7]]) + bias = model_utils.get_padding_bias(x) + bias_shape = tf.shape(bias) + flattened_bias = tf.reshape(bias, [3, 5]) + + self.assertAllEqual([[0, NEG_INF, NEG_INF, NEG_INF, 0], + [0, 0, NEG_INF, NEG_INF, NEG_INF], + [NEG_INF, 0, 0, NEG_INF, 0]], + flattened_bias) + self.assertAllEqual([3, 1, 1, 5], bias_shape) + + def test_get_decoder_self_attention_bias(self): + length = 5 + bias = model_utils.get_decoder_self_attention_bias(length) + + self.assertAllEqual([[[[0, NEG_INF, NEG_INF, NEG_INF, NEG_INF], + [0, 0, NEG_INF, NEG_INF, NEG_INF], + [0, 0, 0, NEG_INF, NEG_INF], + [0, 0, 0, 0, NEG_INF], + [0, 0, 0, 0, 0]]]], + bias) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/optimizer.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/optimizer.py new file mode 100644 index 0000000..176b5eb --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/optimizer.py @@ -0,0 +1,137 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Optimizer from addons and learning rate scheduler.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf +K = tf.keras.backend + + +class LearningRateSchedule(tf.keras.optimizers.schedules.LearningRateSchedule): + """Learning rate schedule.""" + + def __init__(self, initial_learning_rate, hidden_size, warmup_steps): + """Initialize configuration of the learning rate schedule. + + Args: + initial_learning_rate: A float, the initial learning rate. + hidden_size: An integer, the model dimension in the hidden layers. + warmup_steps: An integer, the number of steps required for linear warmup. + """ + super(LearningRateSchedule, self).__init__() + self.initial_learning_rate = initial_learning_rate + self.hidden_size = hidden_size + self.warmup_steps = tf.cast(warmup_steps, tf.float32) + + def __call__(self, global_step): + """Calculate learning rate with linear warmup and rsqrt decay. + + Args: + global_step: An integer, the current global step used for learning rate + calculation. + + Returns: + A float, the learning rate needs to be used for current global step. + """ + with tf.name_scope('learning_rate_schedule'): + global_step = tf.cast(global_step, tf.float32) + learning_rate = self.initial_learning_rate + learning_rate *= (self.hidden_size**-0.5) + # Apply linear warmup + learning_rate *= tf.minimum(1.0, global_step / self.warmup_steps) + # Apply rsqrt decay + learning_rate /= tf.sqrt(tf.maximum(global_step, self.warmup_steps)) + return learning_rate + + def get_config(self): + """Get the configuration of the learning rate schedule.""" + return { + 'initial_learning_rate': self.initial_learning_rate, + 'hidden_size': self.hidden_size, + 'warmup_steps': self.warmup_steps, + } + + +class LearningRateFn(object): + """Creates learning rate function.""" + + def __init__(self, learning_rate, hidden_size, warmup_steps): + self.learning_rate = learning_rate + self.hidden_size = hidden_size + self.warmup_steps = float(warmup_steps) + + def __call__(self, global_step): + """Calculate learning rate with linear warmup and rsqrt decay.""" + step = float(global_step) + learning_rate = self.learning_rate + learning_rate *= (self.hidden_size ** -0.5) + # Apply linear warmup + learning_rate *= np.minimum(1.0, step / self.warmup_steps) + # Apply rsqrt decay + learning_rate /= np.sqrt(np.maximum(step, self.warmup_steps)) + return learning_rate + + +class LearningRateScheduler(tf.keras.callbacks.Callback): + """Keras callback to schedule learning rate. + + TODO(tianlin): Refactor this scheduler and LearningRateBatchScheduler in + official/resnet/keras/keras_common.py. + """ + + def __init__(self, schedule, init_steps=None, verbose=False): + super(LearningRateScheduler, self).__init__() + self.schedule = schedule + self.verbose = verbose + if init_steps is None: + init_steps = 0.0 + self.steps = float(init_steps) # Total steps during training. + + def on_epoch_begin(self, epoch, logs=None): + if not hasattr(self.model.optimizer, 'lr'): + raise ValueError('Optimizer must have a "lr" attribute.') + if not hasattr(self.model.optimizer, 'iterations'): + raise ValueError('Optimizer must have a "iterations" attribute.') + + def on_train_batch_begin(self, batch, logs=None): + """Adjusts learning rate for each train batch.""" + if self.verbose > 0: + iterations = K.get_value(self.model.optimizer.iterations) + print('Original iteration %d' % iterations) + + self.steps += 1.0 + try: # new API + lr = float(K.get_value(self.model.optimizer.lr)) + lr = self.schedule(self.steps, lr) + except TypeError: # Support for old API for backward compatibility + lr = self.schedule(self.steps) + if not isinstance(lr, (float, np.float32, np.float64)): + raise ValueError('The output of the "schedule" function ' + 'should be float.') + K.set_value(self.model.optimizer.lr, lr) + K.set_value(self.model.optimizer.iterations, self.steps) + + if self.verbose > 0: + print('Batch %05d Step %05d: LearningRateScheduler setting learning ' + 'rate to %s.' % (batch + 1, self.steps, lr)) + + def on_epoch_end(self, epoch, logs=None): + logs = logs or {} + logs['lr'] = K.get_value(self.model.optimizer.lr) + logs['steps'] = self.steps diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer.py new file mode 100644 index 0000000..34d0d09 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer.py @@ -0,0 +1,566 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Defines the Transformer model in TF 2.0. + +Model paper: https://arxiv.org/pdf/1706.03762.pdf +Transformer model code source: https://github.com/tensorflow/tensor2tensor +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from official.nlp.transformer import attention_layer +from official.nlp.transformer import beam_search +from official.nlp.transformer import embedding_layer +from official.nlp.transformer import ffn_layer +from official.nlp.transformer import metrics +from official.nlp.transformer import model_utils +from official.nlp.transformer.utils.tokenizer import EOS_ID + + +# Disable the not-callable lint error, since it claims many objects are not +# callable when they actually are. +# pylint: disable=not-callable + + +def create_model(params, is_train): + """Creates transformer model.""" + with tf.name_scope("model"): + if is_train: + inputs = tf.keras.layers.Input((None,), dtype="int64", name="inputs") + targets = tf.keras.layers.Input((None,), dtype="int64", name="targets") + internal_model = Transformer(params, name="transformer_v2") + logits = internal_model([inputs, targets], training=is_train) + vocab_size = params["vocab_size"] + label_smoothing = params["label_smoothing"] + if params["enable_metrics_in_training"]: + logits = metrics.MetricLayer(vocab_size)([logits, targets]) + logits = tf.keras.layers.Lambda(lambda x: x, name="logits", + dtype=tf.float32)(logits) + model = tf.keras.Model([inputs, targets], logits) + # TODO(reedwm): Can we do this loss in float16 instead of float32? + loss = metrics.transformer_loss( + logits, targets, label_smoothing, vocab_size) + model.add_loss(loss) + return model + + else: + inputs = tf.keras.layers.Input((None,), dtype="int64", name="inputs") + internal_model = Transformer(params, name="transformer_v2") + ret = internal_model([inputs], training=is_train) + outputs, scores = ret["outputs"], ret["scores"] + return tf.keras.Model(inputs, [outputs, scores]) + + +class Transformer(tf.keras.Model): + """Transformer model with Keras. + + Implemented as described in: https://arxiv.org/pdf/1706.03762.pdf + + The Transformer model consists of an encoder and decoder. The input is an int + sequence (or a batch of sequences). The encoder produces a continuous + representation, and the decoder uses the encoder output to generate + probabilities for the output sequence. + """ + + def __init__(self, params, name=None): + """Initialize layers to build Transformer model. + + Args: + params: hyperparameter object defining layer sizes, dropout values, etc. + name: name of the model. + """ + super(Transformer, self).__init__(name=name) + self.params = params + self.embedding_softmax_layer = embedding_layer.EmbeddingSharedWeights( + params["vocab_size"], params["hidden_size"]) + self.encoder_stack = EncoderStack(params) + self.decoder_stack = DecoderStack(params) + + def get_config(self): + return { + "params": self.params, + } + + def call(self, inputs, training): + """Calculate target logits or inferred target sequences. + + Args: + inputs: input tensor list of size 1 or 2. + First item, inputs: int tensor with shape [batch_size, input_length]. + Second item (optional), targets: None or int tensor with shape + [batch_size, target_length]. + training: boolean, whether in training mode or not. + + Returns: + If targets is defined, then return logits for each word in the target + sequence. float tensor with shape [batch_size, target_length, vocab_size] + If target is none, then generate output sequence one token at a time. + returns a dictionary { + outputs: [batch_size, decoded length] + scores: [batch_size, float]} + Even when float16 is used, the output tensor(s) are always float32. + + Raises: + NotImplementedError: If try to use padded decode method on CPU/GPUs. + """ + if len(inputs) == 2: + inputs, targets = inputs[0], inputs[1] + else: + # Decoding path. + inputs, targets = inputs[0], None + if self.params["padded_decode"]: + if not self.params["num_replicas"]: + raise NotImplementedError( + "Padded decoding on CPU/GPUs is not supported.") + decode_batch_size = int(self.params["decode_batch_size"] / + self.params["num_replicas"]) + inputs.set_shape([ + decode_batch_size, self.params["decode_max_length"] + ]) + + # Variance scaling is used here because it seems to work in many problems. + # Other reasonable initializers may also work just as well. + with tf.name_scope("Transformer"): + # Calculate attention bias for encoder self-attention and decoder + # multi-headed attention layers. + attention_bias = model_utils.get_padding_bias(inputs) + + # Run the inputs through the encoder layer to map the symbol + # representations to continuous representations. + encoder_outputs = self.encode(inputs, attention_bias, training) + # Generate output sequence if targets is None, or return logits if target + # sequence is known. + if targets is None: + return self.predict(encoder_outputs, attention_bias, training) + else: + logits = self.decode(targets, encoder_outputs, attention_bias, training) + return logits + + def encode(self, inputs, attention_bias, training): + """Generate continuous representation for inputs. + + Args: + inputs: int tensor with shape [batch_size, input_length]. + attention_bias: float tensor with shape [batch_size, 1, 1, input_length]. + training: boolean, whether in training mode or not. + + Returns: + float tensor with shape [batch_size, input_length, hidden_size] + """ + with tf.name_scope("encode"): + # Prepare inputs to the layer stack by adding positional encodings and + # applying dropout. + embedded_inputs = self.embedding_softmax_layer(inputs) + embedded_inputs = tf.cast(embedded_inputs, self.params["dtype"]) + inputs_padding = model_utils.get_padding(inputs) + attention_bias = tf.cast(attention_bias, self.params["dtype"]) + + with tf.name_scope("add_pos_encoding"): + length = tf.shape(embedded_inputs)[1] + pos_encoding = model_utils.get_position_encoding( + length, self.params["hidden_size"]) + pos_encoding = tf.cast(pos_encoding, self.params["dtype"]) + encoder_inputs = embedded_inputs + pos_encoding + + if training: + encoder_inputs = tf.nn.dropout( + encoder_inputs, rate=self.params["layer_postprocess_dropout"]) + + return self.encoder_stack( + encoder_inputs, attention_bias, inputs_padding, training=training) + + def decode(self, targets, encoder_outputs, attention_bias, training): + """Generate logits for each value in the target sequence. + + Args: + targets: target values for the output sequence. int tensor with shape + [batch_size, target_length] + encoder_outputs: continuous representation of input sequence. float tensor + with shape [batch_size, input_length, hidden_size] + attention_bias: float tensor with shape [batch_size, 1, 1, input_length] + training: boolean, whether in training mode or not. + + Returns: + float32 tensor with shape [batch_size, target_length, vocab_size] + """ + with tf.name_scope("decode"): + # Prepare inputs to decoder layers by shifting targets, adding positional + # encoding and applying dropout. + decoder_inputs = self.embedding_softmax_layer(targets) + decoder_inputs = tf.cast(decoder_inputs, self.params["dtype"]) + attention_bias = tf.cast(attention_bias, self.params["dtype"]) + with tf.name_scope("shift_targets"): + # Shift targets to the right, and remove the last element + decoder_inputs = tf.pad(decoder_inputs, + [[0, 0], [1, 0], [0, 0]])[:, :-1, :] + with tf.name_scope("add_pos_encoding"): + length = tf.shape(decoder_inputs)[1] + pos_encoding = model_utils.get_position_encoding( + length, self.params["hidden_size"]) + pos_encoding = tf.cast(pos_encoding, self.params["dtype"]) + decoder_inputs += pos_encoding + if training: + decoder_inputs = tf.nn.dropout( + decoder_inputs, rate=self.params["layer_postprocess_dropout"]) + + # Run values + decoder_self_attention_bias = model_utils.get_decoder_self_attention_bias( + length, dtype=self.params["dtype"]) + outputs = self.decoder_stack( + decoder_inputs, + encoder_outputs, + decoder_self_attention_bias, + attention_bias, + training=training) + logits = self.embedding_softmax_layer(outputs, mode="linear") + logits = tf.cast(logits, tf.float32) + return logits + + def _get_symbols_to_logits_fn(self, max_decode_length, training): + """Returns a decoding function that calculates logits of the next tokens.""" + + timing_signal = model_utils.get_position_encoding( + max_decode_length + 1, self.params["hidden_size"]) + timing_signal = tf.cast(timing_signal, self.params["dtype"]) + decoder_self_attention_bias = model_utils.get_decoder_self_attention_bias( + max_decode_length, dtype=self.params["dtype"]) + + # TODO(b/139770046): Refactor code with better naming of i. + def symbols_to_logits_fn(ids, i, cache): + """Generate logits for next potential IDs. + + Args: + ids: Current decoded sequences. int tensor with shape [batch_size * + beam_size, i + 1]. + i: Loop index. + cache: dictionary of values storing the encoder output, encoder-decoder + attention bias, and previous decoder attention values. + + Returns: + Tuple of + (logits with shape [batch_size * beam_size, vocab_size], + updated cache values) + """ + # Set decoder input to the last generated IDs + decoder_input = ids[:, -1:] + + # Preprocess decoder input by getting embeddings and adding timing signal. + decoder_input = self.embedding_softmax_layer(decoder_input) + + if self.params["padded_decode"]: + timing_signal_shape = timing_signal.shape.as_list() + decoder_input += tf.slice(timing_signal, [i, 0], + [1, timing_signal_shape[1]]) + + bias_shape = decoder_self_attention_bias.shape.as_list() + self_attention_bias = tf.slice( + decoder_self_attention_bias, [0, 0, i, 0], + [bias_shape[0], bias_shape[1], 1, bias_shape[3]]) + else: + decoder_input += timing_signal[i:i + 1] + + self_attention_bias = decoder_self_attention_bias[:, :, i:i + 1, :i + 1] + + decoder_outputs = self.decoder_stack( + decoder_input, + cache.get("encoder_outputs"), + self_attention_bias, + cache.get("encoder_decoder_attention_bias"), + training=training, + cache=cache, + decode_loop_step=i if self.params["padded_decode"] else None) + logits = self.embedding_softmax_layer(decoder_outputs, mode="linear") + logits = tf.squeeze(logits, axis=[1]) + return logits, cache + + return symbols_to_logits_fn + + def predict(self, encoder_outputs, encoder_decoder_attention_bias, training): + """Return predicted sequence.""" + encoder_outputs = tf.cast(encoder_outputs, self.params["dtype"]) + if self.params["padded_decode"]: + batch_size = encoder_outputs.shape.as_list()[0] + input_length = encoder_outputs.shape.as_list()[1] + else: + batch_size = tf.shape(encoder_outputs)[0] + input_length = tf.shape(encoder_outputs)[1] + max_decode_length = input_length + self.params["extra_decode_length"] + encoder_decoder_attention_bias = tf.cast(encoder_decoder_attention_bias, + self.params["dtype"]) + + symbols_to_logits_fn = self._get_symbols_to_logits_fn( + max_decode_length, training) + + # Create initial set of IDs that will be passed into symbols_to_logits_fn. + initial_ids = tf.zeros([batch_size], dtype=tf.int32) + + # Create cache storing decoder attention values for each layer. + # pylint: disable=g-complex-comprehension + init_decode_length = ( + max_decode_length if self.params["padded_decode"] else 0) + num_heads = self.params["num_heads"] + dim_per_head = self.params["hidden_size"] // num_heads + cache = { + "layer_%d" % layer: { + "k": + tf.zeros([ + batch_size, init_decode_length, num_heads, dim_per_head + ], + dtype=self.params["dtype"]), + "v": + tf.zeros([ + batch_size, init_decode_length, num_heads, dim_per_head + ], + dtype=self.params["dtype"]) + } for layer in range(self.params["num_hidden_layers"]) + } + # pylint: enable=g-complex-comprehension + + # Add encoder output and attention bias to the cache. + cache["encoder_outputs"] = encoder_outputs + cache["encoder_decoder_attention_bias"] = encoder_decoder_attention_bias + + # Use beam search to find the top beam_size sequences and scores. + decoded_ids, scores = beam_search.sequence_beam_search( + symbols_to_logits_fn=symbols_to_logits_fn, + initial_ids=initial_ids, + initial_cache=cache, + vocab_size=self.params["vocab_size"], + beam_size=self.params["beam_size"], + alpha=self.params["alpha"], + max_decode_length=max_decode_length, + eos_id=EOS_ID, + padded_decode=self.params["padded_decode"], + dtype=self.params["dtype"]) + + # Get the top sequence for each batch element + top_decoded_ids = decoded_ids[:, 0, 1:] + top_scores = scores[:, 0] + + return {"outputs": top_decoded_ids, "scores": top_scores} + + +class PrePostProcessingWrapper(tf.keras.layers.Layer): + """Wrapper class that applies layer pre-processing and post-processing.""" + + def __init__(self, layer, params): + super(PrePostProcessingWrapper, self).__init__() + self.layer = layer + self.params = params + self.postprocess_dropout = params["layer_postprocess_dropout"] + + def build(self, input_shape): + # Create normalization layer + self.layer_norm = tf.keras.layers.LayerNormalization( + epsilon=1e-6, dtype="float32") + super(PrePostProcessingWrapper, self).build(input_shape) + + def get_config(self): + return { + "params": self.params, + } + + def call(self, x, *args, **kwargs): + """Calls wrapped layer with same parameters.""" + # Preprocessing: apply layer normalization + training = kwargs["training"] + + y = self.layer_norm(x) + + # Get layer output + y = self.layer(y, *args, **kwargs) + + # Postprocessing: apply dropout and residual connection + if training: + y = tf.nn.dropout(y, rate=self.postprocess_dropout) + return x + y + + +class EncoderStack(tf.keras.layers.Layer): + """Transformer encoder stack. + + The encoder stack is made up of N identical layers. Each layer is composed + of the sublayers: + 1. Self-attention layer + 2. Feedforward network (which is 2 fully-connected layers) + """ + + def __init__(self, params): + super(EncoderStack, self).__init__() + self.params = params + self.layers = [] + + def build(self, input_shape): + """Builds the encoder stack.""" + params = self.params + for _ in range(params["num_hidden_layers"]): + # Create sublayers for each layer. + self_attention_layer = attention_layer.SelfAttention( + params["hidden_size"], params["num_heads"], + params["attention_dropout"]) + feed_forward_network = ffn_layer.FeedForwardNetwork( + params["hidden_size"], params["filter_size"], params["relu_dropout"]) + + self.layers.append([ + PrePostProcessingWrapper(self_attention_layer, params), + PrePostProcessingWrapper(feed_forward_network, params) + ]) + + # Create final layer normalization layer. + self.output_normalization = tf.keras.layers.LayerNormalization( + epsilon=1e-6, dtype="float32") + super(EncoderStack, self).build(input_shape) + + def get_config(self): + return { + "params": self.params, + } + + def call(self, encoder_inputs, attention_bias, inputs_padding, training): + """Return the output of the encoder layer stacks. + + Args: + encoder_inputs: tensor with shape [batch_size, input_length, hidden_size] + attention_bias: bias for the encoder self-attention layer. [batch_size, 1, + 1, input_length] + inputs_padding: tensor with shape [batch_size, input_length], inputs with + zero paddings. + training: boolean, whether in training mode or not. + + Returns: + Output of encoder layer stack. + float32 tensor with shape [batch_size, input_length, hidden_size] + """ + for n, layer in enumerate(self.layers): + # Run inputs through the sublayers. + self_attention_layer = layer[0] + feed_forward_network = layer[1] + + with tf.name_scope("layer_%d" % n): + with tf.name_scope("self_attention"): + encoder_inputs = self_attention_layer( + encoder_inputs, attention_bias, training=training) + with tf.name_scope("ffn"): + encoder_inputs = feed_forward_network( + encoder_inputs, training=training) + + return self.output_normalization(encoder_inputs) + + +class DecoderStack(tf.keras.layers.Layer): + """Transformer decoder stack. + + Like the encoder stack, the decoder stack is made up of N identical layers. + Each layer is composed of the sublayers: + 1. Self-attention layer + 2. Multi-headed attention layer combining encoder outputs with results from + the previous self-attention layer. + 3. Feedforward network (2 fully-connected layers) + """ + + def __init__(self, params): + super(DecoderStack, self).__init__() + self.params = params + self.layers = [] + + def build(self, input_shape): + """Builds the decoder stack.""" + params = self.params + for _ in range(params["num_hidden_layers"]): + self_attention_layer = attention_layer.SelfAttention( + params["hidden_size"], params["num_heads"], + params["attention_dropout"]) + enc_dec_attention_layer = attention_layer.Attention( + params["hidden_size"], params["num_heads"], + params["attention_dropout"]) + feed_forward_network = ffn_layer.FeedForwardNetwork( + params["hidden_size"], params["filter_size"], params["relu_dropout"]) + + self.layers.append([ + PrePostProcessingWrapper(self_attention_layer, params), + PrePostProcessingWrapper(enc_dec_attention_layer, params), + PrePostProcessingWrapper(feed_forward_network, params) + ]) + self.output_normalization = tf.keras.layers.LayerNormalization( + epsilon=1e-6, dtype="float32") + super(DecoderStack, self).build(input_shape) + + def get_config(self): + return { + "params": self.params, + } + + def call(self, + decoder_inputs, + encoder_outputs, + decoder_self_attention_bias, + attention_bias, + training, + cache=None, + decode_loop_step=None): + """Return the output of the decoder layer stacks. + + Args: + decoder_inputs: A tensor with shape + [batch_size, target_length, hidden_size]. + encoder_outputs: A tensor with shape + [batch_size, input_length, hidden_size] + decoder_self_attention_bias: A tensor with shape + [1, 1, target_len, target_length], the bias for decoder self-attention + layer. + attention_bias: A tensor with shape [batch_size, 1, 1, input_length], + the bias for encoder-decoder attention layer. + training: A bool, whether in training mode or not. + cache: (Used for fast decoding) A nested dictionary storing previous + decoder self-attention values. The items are: + {layer_n: {"k": A tensor with shape [batch_size, i, key_channels], + "v": A tensor with shape [batch_size, i, value_channels]}, + ...} + decode_loop_step: An integer, the step number of the decoding loop. Used + only for autoregressive inference on TPU. + + Returns: + Output of decoder layer stack. + float32 tensor with shape [batch_size, target_length, hidden_size] + """ + for n, layer in enumerate(self.layers): + self_attention_layer = layer[0] + enc_dec_attention_layer = layer[1] + feed_forward_network = layer[2] + + # Run inputs through the sublayers. + layer_name = "layer_%d" % n + layer_cache = cache[layer_name] if cache is not None else None + with tf.name_scope(layer_name): + with tf.name_scope("self_attention"): + decoder_inputs = self_attention_layer( + decoder_inputs, + decoder_self_attention_bias, + training=training, + cache=layer_cache, + decode_loop_step=decode_loop_step) + with tf.name_scope("encdec_attention"): + decoder_inputs = enc_dec_attention_layer( + decoder_inputs, + encoder_outputs, + attention_bias, + training=training) + with tf.name_scope("ffn"): + decoder_inputs = feed_forward_network( + decoder_inputs, training=training) + + return self.output_normalization(decoder_inputs) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer_layers_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer_layers_test.py new file mode 100644 index 0000000..82d3725 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer_layers_test.py @@ -0,0 +1,97 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for layers in Transformer.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from official.nlp.transformer import attention_layer +from official.nlp.transformer import embedding_layer +from official.nlp.transformer import ffn_layer +from official.nlp.transformer import metrics + + +class TransformerLayersTest(tf.test.TestCase): + + def test_attention_layer(self): + hidden_size = 64 + num_heads = 4 + dropout = 0.5 + dim_per_head = hidden_size // num_heads + layer = attention_layer.SelfAttention(hidden_size, num_heads, dropout) + self.assertDictEqual(layer.get_config(), { + "hidden_size": hidden_size, + "num_heads": num_heads, + "attention_dropout": dropout, + }) + length = 2 + x = tf.ones([1, length, hidden_size]) + bias = tf.ones([1]) + cache = { + "k": tf.zeros([1, 0, num_heads, dim_per_head]), + "v": tf.zeros([1, 0, num_heads, dim_per_head]), + } + y = layer(x, bias, training=True, cache=cache) + self.assertEqual(y.shape, (1, length, 64,)) + self.assertEqual(cache["k"].shape, (1, length, num_heads, dim_per_head,)) + self.assertEqual(cache["v"].shape, (1, length, num_heads, dim_per_head,)) + + def test_embedding_shared_weights(self): + vocab_size = 50 + hidden_size = 64 + length = 2 + layer = embedding_layer.EmbeddingSharedWeights(vocab_size, hidden_size) + self.assertDictEqual(layer.get_config(), { + "vocab_size": 50, + "hidden_size": 64, + }) + + idx = tf.ones([1, length], dtype="int32") + y = layer(idx) + self.assertEqual(y.shape, (1, length, hidden_size,)) + x = tf.ones([1, length, hidden_size]) + output = layer(x, "linear") + self.assertEqual(output.shape, (1, length, vocab_size,)) + + def test_feed_forward_network(self): + hidden_size = 64 + filter_size = 32 + relu_dropout = 0.5 + layer = ffn_layer.FeedForwardNetwork(hidden_size, filter_size, relu_dropout) + self.assertDictEqual(layer.get_config(), { + "hidden_size": hidden_size, + "filter_size": filter_size, + "relu_dropout": relu_dropout, + }) + length = 2 + x = tf.ones([1, length, hidden_size]) + y = layer(x, training=True) + self.assertEqual(y.shape, (1, length, hidden_size,)) + + def test_metric_layer(self): + vocab_size = 50 + logits = tf.keras.layers.Input((None, vocab_size), + dtype="float32", + name="logits") + targets = tf.keras.layers.Input((None,), dtype="int64", name="targets") + output_logits = metrics.MetricLayer(vocab_size)([logits, targets]) + self.assertEqual(output_logits.shape.as_list(), [None, None, vocab_size,]) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer_main.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer_main.py new file mode 100644 index 0000000..f72cbe7 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer_main.py @@ -0,0 +1,497 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Train and evaluate the Transformer model. + +See README for description of setting the training schedule and evaluating the +BLEU score. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import tempfile + +from absl import app +from absl import flags +from absl import logging +import tensorflow as tf + +from official.modeling import performance +from official.nlp.transformer import compute_bleu +from official.nlp.transformer import data_pipeline +from official.nlp.transformer import metrics +from official.nlp.transformer import misc +from official.nlp.transformer import optimizer +from official.nlp.transformer import transformer +from official.nlp.transformer import translate +from official.nlp.transformer.utils import tokenizer +from official.utils.flags import core as flags_core +from official.utils.logs import logger +from official.utils.misc import distribution_utils +from official.utils.misc import keras_utils + +INF = int(1e9) +BLEU_DIR = "bleu" +_SINGLE_SAMPLE = 1 + + +def translate_and_compute_bleu(model, + params, + subtokenizer, + bleu_source, + bleu_ref, + distribution_strategy=None): + """Translate file and report the cased and uncased bleu scores. + + Args: + model: A Keras model, used to generate the translations. + params: A dictionary, containing the translation related parameters. + subtokenizer: A subtokenizer object, used for encoding and decoding source + and translated lines. + bleu_source: A file containing source sentences for translation. + bleu_ref: A file containing the reference for the translated sentences. + distribution_strategy: A platform distribution strategy, used for TPU based + translation. + + Returns: + uncased_score: A float, the case insensitive BLEU score. + cased_score: A float, the case sensitive BLEU score. + """ + # Create temporary file to store translation. + tmp = tempfile.NamedTemporaryFile(delete=False) + tmp_filename = tmp.name + + translate.translate_file( + model, + params, + subtokenizer, + bleu_source, + output_file=tmp_filename, + print_all_translations=False, + distribution_strategy=distribution_strategy) + + # Compute uncased and cased bleu scores. + uncased_score = compute_bleu.bleu_wrapper(bleu_ref, tmp_filename, False) + cased_score = compute_bleu.bleu_wrapper(bleu_ref, tmp_filename, True) + os.remove(tmp_filename) + return uncased_score, cased_score + + +def evaluate_and_log_bleu(model, + params, + bleu_source, + bleu_ref, + vocab_file, + distribution_strategy=None): + """Calculate and record the BLEU score. + + Args: + model: A Keras model, used to generate the translations. + params: A dictionary, containing the translation related parameters. + bleu_source: A file containing source sentences for translation. + bleu_ref: A file containing the reference for the translated sentences. + vocab_file: A file containing the vocabulary for translation. + distribution_strategy: A platform distribution strategy, used for TPU based + translation. + + Returns: + uncased_score: A float, the case insensitive BLEU score. + cased_score: A float, the case sensitive BLEU score. + """ + subtokenizer = tokenizer.Subtokenizer(vocab_file) + + uncased_score, cased_score = translate_and_compute_bleu( + model, params, subtokenizer, bleu_source, bleu_ref, distribution_strategy) + + logging.info("Bleu score (uncased): %s", uncased_score) + logging.info("Bleu score (cased): %s", cased_score) + return uncased_score, cased_score + + +class TransformerTask(object): + """Main entry of Transformer model.""" + + def __init__(self, flags_obj): + """Init function of TransformerMain. + + Args: + flags_obj: Object containing parsed flag values, i.e., FLAGS. + + Raises: + ValueError: if not using static batch for input data on TPU. + """ + self.flags_obj = flags_obj + self.predict_model = None + + # Add flag-defined parameters to params object + num_gpus = flags_core.get_num_gpus(flags_obj) + self.params = params = misc.get_model_params(flags_obj.param_set, num_gpus) + + params["num_gpus"] = num_gpus + params["use_ctl"] = flags_obj.use_ctl + params["data_dir"] = flags_obj.data_dir + params["model_dir"] = flags_obj.model_dir + params["static_batch"] = flags_obj.static_batch + params["max_length"] = flags_obj.max_length + params["decode_batch_size"] = flags_obj.decode_batch_size + params["decode_max_length"] = flags_obj.decode_max_length + params["padded_decode"] = flags_obj.padded_decode + params["num_parallel_calls"] = ( + flags_obj.num_parallel_calls or tf.data.experimental.AUTOTUNE) + + params["use_synthetic_data"] = flags_obj.use_synthetic_data + params["batch_size"] = flags_obj.batch_size or params["default_batch_size"] + params["repeat_dataset"] = None + params["dtype"] = flags_core.get_tf_dtype(flags_obj) + params["enable_tensorboard"] = flags_obj.enable_tensorboard + params["enable_metrics_in_training"] = flags_obj.enable_metrics_in_training + params["steps_between_evals"] = flags_obj.steps_between_evals + params["enable_checkpointing"] = flags_obj.enable_checkpointing + + self.distribution_strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=flags_obj.distribution_strategy, + num_gpus=num_gpus, + all_reduce_alg=flags_obj.all_reduce_alg, + num_packs=flags_obj.num_packs, + tpu_address=flags_obj.tpu or "") + if self.use_tpu: + params["num_replicas"] = self.distribution_strategy.num_replicas_in_sync + if not params["static_batch"]: + raise ValueError("TPU requires static batch for input data.") + else: + logging.info("Running transformer with num_gpus = %d", num_gpus) + + if self.distribution_strategy: + logging.info("For training, using distribution strategy: %s", + self.distribution_strategy) + else: + logging.info("Not using any distribution strategy.") + + performance.set_mixed_precision_policy( + params["dtype"], + flags_core.get_loss_scale(flags_obj, default_for_fp16="dynamic")) + + @property + def use_tpu(self): + if self.distribution_strategy: + return isinstance(self.distribution_strategy, + tf.distribute.experimental.TPUStrategy) + return False + + def train(self): + """Trains the model.""" + params = self.params + flags_obj = self.flags_obj + # Sets config options. + keras_utils.set_session_config(enable_xla=flags_obj.enable_xla) + + _ensure_dir(flags_obj.model_dir) + with distribution_utils.get_strategy_scope(self.distribution_strategy): + model = transformer.create_model(params, is_train=True) + opt = self._create_optimizer() + + current_step = 0 + checkpoint = tf.train.Checkpoint(model=model, optimizer=opt) + latest_checkpoint = tf.train.latest_checkpoint(flags_obj.model_dir) + if latest_checkpoint: + checkpoint.restore(latest_checkpoint) + logging.info("Loaded checkpoint %s", latest_checkpoint) + current_step = opt.iterations.numpy() + + if params["use_ctl"]: + train_loss_metric = tf.keras.metrics.Mean( + "training_loss", dtype=tf.float32) + if params["enable_tensorboard"]: + summary_writer = tf.compat.v2.summary.create_file_writer( + flags_obj.model_dir) + else: + summary_writer = tf.compat.v2.summary.create_noop_writer() + train_metrics = [train_loss_metric] + if params["enable_metrics_in_training"]: + train_metrics = train_metrics + model.metrics + else: + model.compile(opt) + + model.summary() + + if self.use_tpu: + # Different from experimental_distribute_dataset, + # experimental_distribute_datasets_from_function requires + # per-replica/local batch size. + params["batch_size"] /= self.distribution_strategy.num_replicas_in_sync + train_ds = ( + self.distribution_strategy + .experimental_distribute_datasets_from_function( + lambda ctx: data_pipeline.train_input_fn(params, ctx))) + else: + train_ds = data_pipeline.train_input_fn(params) + map_data_fn = data_pipeline.map_data_for_transformer_fn + train_ds = train_ds.map( + map_data_fn, num_parallel_calls=params["num_parallel_calls"]) + if params["use_ctl"]: + train_ds_iterator = iter(train_ds) + + callbacks = self._create_callbacks(flags_obj.model_dir, 0, params) + + # Only TimeHistory callback is supported for CTL + if params["use_ctl"]: + callbacks = [cb for cb in callbacks + if isinstance(cb, keras_utils.TimeHistory)] + + # TODO(b/139418525): Refactor the custom training loop logic. + @tf.function + def train_steps(iterator, steps): + """Training steps function for TPU runs. + + Args: + iterator: The input iterator of the training dataset. + steps: An integer, the number of training steps. + + Returns: + A float, the loss value. + """ + + def _step_fn(inputs): + """Per-replica step function.""" + inputs, targets = inputs + with tf.GradientTape() as tape: + logits = model([inputs, targets], training=True) + loss = metrics.transformer_loss(logits, targets, + params["label_smoothing"], + params["vocab_size"]) + # Scales the loss, which results in using the average loss across all + # of the replicas for backprop. + scaled_loss = loss / self.distribution_strategy.num_replicas_in_sync + + # De-dupes variables due to keras tracking issues. + tvars = list({id(v): v for v in model.trainable_variables}.values()) + grads = tape.gradient(scaled_loss, tvars) + opt.apply_gradients(zip(grads, tvars)) + # For reporting, the metric takes the mean of losses. + train_loss_metric.update_state(loss) + + for _ in tf.range(steps): + train_loss_metric.reset_states() + self.distribution_strategy.run( + _step_fn, args=(next(iterator),)) + + cased_score, uncased_score = None, None + cased_score_history, uncased_score_history = [], [] + while current_step < flags_obj.train_steps: + remaining_steps = flags_obj.train_steps - current_step + train_steps_per_eval = ( + remaining_steps if remaining_steps < flags_obj.steps_between_evals + else flags_obj.steps_between_evals) + current_iteration = current_step // flags_obj.steps_between_evals + + logging.info( + "Start train iteration at global step:{}".format(current_step)) + history = None + if params["use_ctl"]: + if not self.use_tpu: + raise NotImplementedError( + "Custom training loop on GPUs is not implemented.") + + # Runs training steps. + with summary_writer.as_default(): + for cb in callbacks: + cb.on_epoch_begin(current_iteration) + cb.on_batch_begin(0) + + train_steps( + train_ds_iterator, + tf.convert_to_tensor(train_steps_per_eval, dtype=tf.int32)) + current_step += train_steps_per_eval + train_loss = train_loss_metric.result().numpy().astype(float) + logging.info("Train Step: %d/%d / loss = %s", current_step, + flags_obj.train_steps, train_loss) + + for cb in callbacks: + cb.on_batch_end(train_steps_per_eval - 1) + cb.on_epoch_end(current_iteration) + + if params["enable_tensorboard"]: + for metric_obj in train_metrics: + tf.compat.v2.summary.scalar(metric_obj.name, metric_obj.result(), + current_step) + summary_writer.flush() + + for cb in callbacks: + cb.on_train_end() + + if flags_obj.enable_checkpointing: + # avoid check-pointing when running for benchmarking. + checkpoint_name = checkpoint.save( + os.path.join(flags_obj.model_dir, + "ctl_step_{}.ckpt".format(current_step))) + logging.info("Saved checkpoint to %s", checkpoint_name) + else: + if self.use_tpu: + raise NotImplementedError( + "Keras model.fit on TPUs is not implemented.") + history = model.fit( + train_ds, + initial_epoch=current_iteration, + epochs=current_iteration + 1, + steps_per_epoch=train_steps_per_eval, + callbacks=callbacks, + # If TimeHistory is enabled, progress bar would be messy. Increase + # the verbose level to get rid of it. + verbose=(2 if flags_obj.enable_time_history else 1)) + current_step += train_steps_per_eval + logging.info("Train history: {}".format(history.history)) + + logging.info("End train iteration at global step:{}".format(current_step)) + + if (flags_obj.bleu_source and flags_obj.bleu_ref): + uncased_score, cased_score = self.eval() + cased_score_history.append([current_iteration + 1, cased_score]) + uncased_score_history.append([current_iteration + 1, uncased_score]) + + stats = ({ + "loss": train_loss + } if history is None else misc.build_stats(history, callbacks)) + if uncased_score and cased_score: + stats["bleu_uncased"] = uncased_score + stats["bleu_cased"] = cased_score + stats["bleu_uncased_history"] = uncased_score_history + stats["bleu_cased_history"] = cased_score_history + return stats + + def eval(self): + """Evaluates the model.""" + distribution_strategy = self.distribution_strategy if self.use_tpu else None + + # We only want to create the model under DS scope for TPU case. + # When 'distribution_strategy' is None, a no-op DummyContextManager will + # be used. + with distribution_utils.get_strategy_scope(distribution_strategy): + if not self.predict_model: + self.predict_model = transformer.create_model(self.params, False) + self._load_weights_if_possible( + self.predict_model, + tf.train.latest_checkpoint(self.flags_obj.model_dir)) + self.predict_model.summary() + return evaluate_and_log_bleu( + self.predict_model, self.params, self.flags_obj.bleu_source, + self.flags_obj.bleu_ref, self.flags_obj.vocab_file, + distribution_strategy) + + def predict(self): + """Predicts result from the model.""" + params = self.params + flags_obj = self.flags_obj + + with tf.name_scope("model"): + model = transformer.create_model(params, is_train=False) + self._load_weights_if_possible( + model, tf.train.latest_checkpoint(self.flags_obj.model_dir)) + model.summary() + subtokenizer = tokenizer.Subtokenizer(flags_obj.vocab_file) + + ds = data_pipeline.eval_input_fn(params) + ds = ds.map(lambda x, y: x).take(_SINGLE_SAMPLE) + ret = model.predict(ds) + val_outputs, _ = ret + length = len(val_outputs) + for i in range(length): + translate.translate_from_input(val_outputs[i], subtokenizer) + + def _create_callbacks(self, cur_log_dir, init_steps, params): + """Creates a list of callbacks.""" + sfunc = optimizer.LearningRateFn(params["learning_rate"], + params["hidden_size"], + params["learning_rate_warmup_steps"]) + scheduler_callback = optimizer.LearningRateScheduler(sfunc, init_steps) + callbacks = misc.get_callbacks(params["steps_between_evals"]) + callbacks.append(scheduler_callback) + if params["enable_checkpointing"]: + ckpt_full_path = os.path.join(cur_log_dir, "cp-{epoch:04d}.ckpt") + callbacks.append( + tf.keras.callbacks.ModelCheckpoint( + ckpt_full_path, save_weights_only=True)) + return callbacks + + def _load_weights_if_possible(self, model, init_weight_path=None): + """Loads model weights when it is provided.""" + if init_weight_path: + logging.info("Load weights: {}".format(init_weight_path)) + # TODO(b/139414977): Having the same variable restoring method for both + # TPU and GPU. + if self.use_tpu: + checkpoint = tf.train.Checkpoint( + model=model, optimizer=self._create_optimizer()) + checkpoint.restore(init_weight_path) + else: + model.load_weights(init_weight_path) + else: + logging.info("Weights not loaded from path:{}".format(init_weight_path)) + + def _create_optimizer(self): + """Creates optimizer.""" + params = self.params + lr_schedule = optimizer.LearningRateSchedule( + params["learning_rate"], params["hidden_size"], + params["learning_rate_warmup_steps"]) + opt = tf.keras.optimizers.Adam( + lr_schedule if self.use_tpu else params["learning_rate"], + params["optimizer_adam_beta1"], + params["optimizer_adam_beta2"], + epsilon=params["optimizer_adam_epsilon"]) + + opt = performance.configure_optimizer( + opt, + use_float16=params["dtype"] == tf.float16, + use_graph_rewrite=self.flags_obj.fp16_implementation == "graph_rewrite", + loss_scale=flags_core.get_loss_scale( + self.flags_obj, default_for_fp16="dynamic")) + + return opt + + +def _ensure_dir(log_dir): + """Makes log dir if not existed.""" + if not tf.io.gfile.exists(log_dir): + tf.io.gfile.makedirs(log_dir) + + +def main(_): + flags_obj = flags.FLAGS + with logger.benchmark_context(flags_obj): + task = TransformerTask(flags_obj) + + # Execute flag override logic for better model performance + if flags_obj.tf_gpu_thread_mode: + keras_utils.set_gpu_thread_mode_and_count( + per_gpu_thread_count=flags_obj.per_gpu_thread_count, + gpu_thread_mode=flags_obj.tf_gpu_thread_mode, + num_gpus=flags_obj.num_gpus, + datasets_num_private_threads=flags_obj.datasets_num_private_threads) + + if flags_obj.mode == "train": + task.train() + elif flags_obj.mode == "predict": + task.predict() + elif flags_obj.mode == "eval": + task.eval() + else: + raise ValueError("Invalid mode {}".format(flags_obj.mode)) + + +if __name__ == "__main__": + logging.set_verbosity(logging.INFO) + misc.define_transformer_flags() + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer_main_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer_main_test.py new file mode 100644 index 0000000..2880217 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer_main_test.py @@ -0,0 +1,190 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Test Transformer model.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import re +import sys +import unittest + +from absl import flags +from absl.testing import flagsaver +import tensorflow as tf +from tensorflow.python.eager import context # pylint: disable=ungrouped-imports +from official.nlp.transformer import misc +from official.nlp.transformer import transformer_main +from official.utils.misc import keras_utils + +FLAGS = flags.FLAGS +FIXED_TIMESTAMP = 'my_time_stamp' +WEIGHT_PATTERN = re.compile(r'weights-epoch-.+\.hdf5') + + +def _generate_file(filepath, lines): + with open(filepath, 'w') as f: + for l in lines: + f.write('{}\n'.format(l)) + + +class TransformerTaskTest(tf.test.TestCase): + local_flags = None + + def setUp(self): + temp_dir = self.get_temp_dir() + if TransformerTaskTest.local_flags is None: + misc.define_transformer_flags() + # Loads flags, array cannot be blank. + flags.FLAGS(['foo']) + TransformerTaskTest.local_flags = flagsaver.save_flag_values() + else: + flagsaver.restore_flag_values(TransformerTaskTest.local_flags) + FLAGS.model_dir = os.path.join(temp_dir, FIXED_TIMESTAMP) + FLAGS.param_set = 'tiny' + FLAGS.use_synthetic_data = True + FLAGS.steps_between_evals = 1 + FLAGS.train_steps = 2 + FLAGS.validation_steps = 1 + FLAGS.batch_size = 8 + FLAGS.num_gpus = 1 + FLAGS.distribution_strategy = 'off' + FLAGS.dtype = 'fp32' + self.model_dir = FLAGS.model_dir + self.temp_dir = temp_dir + self.vocab_file = os.path.join(temp_dir, 'vocab') + self.vocab_size = misc.get_model_params(FLAGS.param_set, 0)['vocab_size'] + self.bleu_source = os.path.join(temp_dir, 'bleu_source') + self.bleu_ref = os.path.join(temp_dir, 'bleu_ref') + self.orig_policy = ( + tf.compat.v2.keras.mixed_precision.experimental.global_policy()) + + def tearDown(self): + tf.compat.v2.keras.mixed_precision.experimental.set_policy(self.orig_policy) + + def _assert_exists(self, filepath): + self.assertTrue(os.path.exists(filepath)) + + def test_train_no_dist_strat(self): + if context.num_gpus() >= 2: + self.skipTest('No need to test 2+ GPUs without a distribution strategy.') + t = transformer_main.TransformerTask(FLAGS) + t.train() + + def test_train_static_batch(self): + if context.num_gpus() >= 2: + self.skipTest('No need to test 2+ GPUs without a distribution strategy.') + FLAGS.distribution_strategy = 'one_device' + if tf.test.is_built_with_cuda(): + FLAGS.num_gpus = 1 + else: + FLAGS.num_gpus = 0 + FLAGS.static_batch = True + t = transformer_main.TransformerTask(FLAGS) + t.train() + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_train_1_gpu_with_dist_strat(self): + FLAGS.distribution_strategy = 'one_device' + t = transformer_main.TransformerTask(FLAGS) + t.train() + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_train_fp16(self): + FLAGS.distribution_strategy = 'one_device' + FLAGS.dtype = 'fp16' + t = transformer_main.TransformerTask(FLAGS) + t.train() + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_train_2_gpu(self): + if context.num_gpus() < 2: + self.skipTest( + '{} GPUs are not available for this test. {} GPUs are available' + .format(2, context.num_gpus())) + FLAGS.distribution_strategy = 'mirrored' + FLAGS.num_gpus = 2 + FLAGS.param_set = 'base' + t = transformer_main.TransformerTask(FLAGS) + t.train() + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_train_2_gpu_fp16(self): + if context.num_gpus() < 2: + self.skipTest( + '{} GPUs are not available for this test. {} GPUs are available' + .format(2, context.num_gpus())) + FLAGS.distribution_strategy = 'mirrored' + FLAGS.num_gpus = 2 + FLAGS.param_set = 'base' + FLAGS.dtype = 'fp16' + t = transformer_main.TransformerTask(FLAGS) + t.train() + + def _prepare_files_and_flags(self, *extra_flags): + # Make log dir. + if not os.path.exists(self.temp_dir): + os.makedirs(self.temp_dir) + + # Fake vocab, bleu_source and bleu_ref. + tokens = [ + "''", "''", "'_'", "'a'", "'b'", "'c'", "'d'", "'a_'", "'b_'", + "'c_'", "'d_'" + ] + tokens += ["'{}'".format(i) for i in range(self.vocab_size - len(tokens))] + _generate_file(self.vocab_file, tokens) + _generate_file(self.bleu_source, ['a b', 'c d']) + _generate_file(self.bleu_ref, ['a b', 'd c']) + + # Update flags. + update_flags = [ + 'ignored_program_name', + '--vocab_file={}'.format(self.vocab_file), + '--bleu_source={}'.format(self.bleu_source), + '--bleu_ref={}'.format(self.bleu_ref), + ] + if extra_flags: + update_flags.extend(extra_flags) + FLAGS(update_flags) + + def test_predict(self): + if context.num_gpus() >= 2: + self.skipTest('No need to test 2+ GPUs without a distribution strategy.') + self._prepare_files_and_flags() + t = transformer_main.TransformerTask(FLAGS) + t.predict() + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_predict_fp16(self): + if context.num_gpus() >= 2: + self.skipTest('No need to test 2+ GPUs without a distribution strategy.') + self._prepare_files_and_flags('--dtype=fp16') + t = transformer_main.TransformerTask(FLAGS) + t.predict() + + def test_eval(self): + if context.num_gpus() >= 2: + self.skipTest('No need to test 2+ GPUs without a distribution strategy.') + if 'test_xla' in sys.argv[0]: + self.skipTest('TODO(xla): Make this test faster under XLA.') + self._prepare_files_and_flags() + t = transformer_main.TransformerTask(FLAGS) + t.eval() + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer_test.py new file mode 100644 index 0000000..227b43d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/transformer_test.py @@ -0,0 +1,68 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Test Transformer model.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from official.nlp.transformer import model_params +from official.nlp.transformer import transformer + + +class TransformerV2Test(tf.test.TestCase): + + def setUp(self): + self.params = params = model_params.TINY_PARAMS + params["batch_size"] = params["default_batch_size"] = 16 + params["use_synthetic_data"] = True + params["hidden_size"] = 12 + params["num_hidden_layers"] = 2 + params["filter_size"] = 14 + params["num_heads"] = 2 + params["vocab_size"] = 41 + params["extra_decode_length"] = 2 + params["beam_size"] = 3 + params["dtype"] = tf.float32 + + def test_create_model_train(self): + model = transformer.create_model(self.params, True) + inputs, outputs = model.inputs, model.outputs + self.assertEqual(len(inputs), 2) + self.assertEqual(len(outputs), 1) + self.assertEqual(inputs[0].shape.as_list(), [None, None]) + self.assertEqual(inputs[0].dtype, tf.int64) + self.assertEqual(inputs[1].shape.as_list(), [None, None]) + self.assertEqual(inputs[1].dtype, tf.int64) + self.assertEqual(outputs[0].shape.as_list(), [None, None, 41]) + self.assertEqual(outputs[0].dtype, tf.float32) + + def test_create_model_not_train(self): + model = transformer.create_model(self.params, False) + inputs, outputs = model.inputs, model.outputs + self.assertEqual(len(inputs), 1) + self.assertEqual(len(outputs), 2) + self.assertEqual(inputs[0].shape.as_list(), [None, None]) + self.assertEqual(inputs[0].dtype, tf.int64) + self.assertEqual(outputs[0].shape.as_list(), [None, None]) + self.assertEqual(outputs[0].dtype, tf.int32) + self.assertEqual(outputs[1].shape.as_list(), [None]) + self.assertEqual(outputs[1].dtype, tf.float32) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/translate.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/translate.py new file mode 100644 index 0000000..1f92504 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/translate.py @@ -0,0 +1,199 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Translate text or files using trained transformer model.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import logging +import numpy as np +import tensorflow as tf + +from official.nlp.transformer.utils import tokenizer + +_EXTRA_DECODE_LENGTH = 100 +_BEAM_SIZE = 4 +_ALPHA = 0.6 + + +def _get_sorted_inputs(filename): + """Read and sort lines from the file sorted by decreasing length. + + Args: + filename: String name of file to read inputs from. + Returns: + Sorted list of inputs, and dictionary mapping original index->sorted index + of each element. + """ + with tf.io.gfile.GFile(filename) as f: + records = f.read().split("\n") + inputs = [record.strip() for record in records] + if not inputs[-1]: + inputs.pop() + + input_lens = [(i, len(line.split())) for i, line in enumerate(inputs)] + sorted_input_lens = sorted(input_lens, key=lambda x: x[1], reverse=True) + + sorted_inputs = [None] * len(sorted_input_lens) + sorted_keys = [0] * len(sorted_input_lens) + for i, (index, _) in enumerate(sorted_input_lens): + sorted_inputs[i] = inputs[index] + sorted_keys[index] = i + return sorted_inputs, sorted_keys + + +def _encode_and_add_eos(line, subtokenizer): + """Encode line with subtokenizer, and add EOS id to the end.""" + return subtokenizer.encode(line) + [tokenizer.EOS_ID] + + +def _trim_and_decode(ids, subtokenizer): + """Trim EOS and PAD tokens from ids, and decode to return a string.""" + try: + index = list(ids).index(tokenizer.EOS_ID) + return subtokenizer.decode(ids[:index]) + except ValueError: # No EOS found in sequence + return subtokenizer.decode(ids) + + +def translate_file(model, + params, + subtokenizer, + input_file, + output_file=None, + print_all_translations=True, + distribution_strategy=None): + """Translate lines in file, and save to output file if specified. + + Args: + model: A Keras model, used to generate the translations. + params: A dictionary, containing the translation related parameters. + subtokenizer: A subtokenizer object, used for encoding and decoding source + and translated lines. + input_file: A file containing lines to translate. + output_file: A file that stores the generated translations. + print_all_translations: A bool. If true, all translations are printed to + stdout. + distribution_strategy: A distribution strategy, used to perform inference + directly with tf.function instead of Keras model.predict(). + + Raises: + ValueError: if output file is invalid. + """ + batch_size = params["decode_batch_size"] + + # Read and sort inputs by length. Keep dictionary (original index-->new index + # in sorted list) to write translations in the original order. + sorted_inputs, sorted_keys = _get_sorted_inputs(input_file) + total_samples = len(sorted_inputs) + num_decode_batches = (total_samples - 1) // batch_size + 1 + + def input_generator(): + """Yield encoded strings from sorted_inputs.""" + for i in range(num_decode_batches): + lines = [ + sorted_inputs[j + i * batch_size] + for j in range(batch_size) + if j + i * batch_size < total_samples + ] + lines = [_encode_and_add_eos(l, subtokenizer) for l in lines] + if distribution_strategy: + for j in range(batch_size - len(lines)): + lines.append([tokenizer.EOS_ID]) + batch = tf.keras.preprocessing.sequence.pad_sequences( + lines, + maxlen=params["decode_max_length"], + dtype="int32", + padding="post") + logging.info("Decoding batch %d out of %d.", i, num_decode_batches) + yield batch + + @tf.function + def predict_step(inputs): + """Decoding step function for TPU runs.""" + + def _step_fn(inputs): + """Per replica step function.""" + tag = inputs[0] + val_inputs = inputs[1] + val_outputs, _ = model([val_inputs], training=False) + return tag, val_outputs + + return distribution_strategy.run(_step_fn, args=(inputs,)) + + translations = [] + if distribution_strategy: + num_replicas = distribution_strategy.num_replicas_in_sync + local_batch_size = params["decode_batch_size"] // num_replicas + for i, text in enumerate(input_generator()): + if distribution_strategy: + text = np.reshape(text, [num_replicas, local_batch_size, -1]) + # Add tag to the input of each replica with the reordering logic after + # outputs, to ensure the output order matches the input order. + text = tf.constant(text) + + @tf.function + def text_as_per_replica(): + replica_context = tf.distribute.get_replica_context() + replica_id = replica_context.replica_id_in_sync_group + return replica_id, text[replica_id] + + text = distribution_strategy.run(text_as_per_replica) + outputs = distribution_strategy.experimental_local_results( + predict_step(text)) + tags, unordered_val_outputs = outputs[0] + tags = [tag.numpy() for tag in tags._values] + unordered_val_outputs = [ + val_output.numpy() for val_output in unordered_val_outputs._values] + # pylint: enable=protected-access + val_outputs = [None] * len(tags) + for k in range(len(tags)): + val_outputs[tags[k]] = unordered_val_outputs[k] + val_outputs = np.reshape(val_outputs, [params["decode_batch_size"], -1]) + else: + val_outputs, _ = model.predict(text) + + length = len(val_outputs) + for j in range(length): + if j + i * batch_size < total_samples: + translation = _trim_and_decode(val_outputs[j], subtokenizer) + translations.append(translation) + if print_all_translations: + logging.info("Translating:\n\tInput: %s\n\tOutput: %s", + sorted_inputs[j + i * batch_size], translation) + + # Write translations in the order they appeared in the original file. + if output_file is not None: + if tf.io.gfile.isdir(output_file): + raise ValueError("File output is a directory, will not save outputs to " + "file.") + logging.info("Writing to file %s", output_file) + with tf.compat.v1.gfile.Open(output_file, "w") as f: + for i in sorted_keys: + f.write("%s\n" % translations[i]) + + +def translate_from_text(model, subtokenizer, txt): + encoded_txt = _encode_and_add_eos(txt, subtokenizer) + result = model.predict(encoded_txt) + outputs = result["outputs"] + logging.info("Original: \"%s\"", txt) + translate_from_input(outputs, subtokenizer) + + +def translate_from_input(outputs, subtokenizer): + translation = _trim_and_decode(outputs, subtokenizer) + logging.info("Translation: \"%s\"", translation) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/utils/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/utils/metrics.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/utils/metrics.py new file mode 100644 index 0000000..7900cf8 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/utils/metrics.py @@ -0,0 +1,490 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functions for calculating loss, accuracy, and other model metrics. + +Metrics: + - Padded loss, accuracy, and negative log perplexity. Source: + https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/metrics.py + - BLEU approximation. Source: + https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/bleu_hook.py + - ROUGE score. Source: + https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/rouge.py +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import math + +import numpy as np +import six +from six.moves import xrange # pylint: disable=redefined-builtin +import tensorflow.compat.v1 as tf + + +def _pad_tensors_to_same_length(x, y): + """Pad x and y so that the results have the same length (second dimension).""" + with tf.name_scope("pad_to_same_length"): + x_length = tf.shape(x)[1] + y_length = tf.shape(y)[1] + + max_length = tf.maximum(x_length, y_length) + + x = tf.pad(x, [[0, 0], [0, max_length - x_length], [0, 0]]) + y = tf.pad(y, [[0, 0], [0, max_length - y_length]]) + return x, y + + +def padded_cross_entropy_loss(logits, labels, smoothing, vocab_size): + """Calculate cross entropy loss while ignoring padding. + + Args: + logits: Tensor of size [batch_size, length_logits, vocab_size] + labels: Tensor of size [batch_size, length_labels] + smoothing: Label smoothing constant, used to determine the on and off values + vocab_size: int size of the vocabulary + Returns: + Returns the cross entropy loss and weight tensors: float32 tensors with + shape [batch_size, max(length_logits, length_labels)] + """ + with tf.name_scope("loss", values=[logits, labels]): + logits, labels = _pad_tensors_to_same_length(logits, labels) + + # Calculate smoothing cross entropy + with tf.name_scope("smoothing_cross_entropy", values=[logits, labels]): + confidence = 1.0 - smoothing + low_confidence = (1.0 - confidence) / tf.to_float(vocab_size - 1) + soft_targets = tf.one_hot( + tf.cast(labels, tf.int32), + depth=vocab_size, + on_value=confidence, + off_value=low_confidence) + xentropy = tf.nn.softmax_cross_entropy_with_logits_v2( + logits=logits, labels=soft_targets) + + # Calculate the best (lowest) possible value of cross entropy, and + # subtract from the cross entropy loss. + normalizing_constant = -( + confidence * tf.log(confidence) + tf.to_float(vocab_size - 1) * + low_confidence * tf.log(low_confidence + 1e-20)) + xentropy -= normalizing_constant + + weights = tf.to_float(tf.not_equal(labels, 0)) + return xentropy * weights, weights + + +def _convert_to_eval_metric(metric_fn): + """Wrap a metric fn that returns scores and weights as an eval metric fn. + + The input metric_fn returns values for the current batch. The wrapper + aggregates the return values collected over all of the batches evaluated. + + Args: + metric_fn: function that returns scores and weights for the current batch's + logits and predicted labels. + + Returns: + function that aggregates the scores and weights from metric_fn. + """ + def problem_metric_fn(*args): + """Returns an aggregation of the metric_fn's returned values.""" + (scores, weights) = metric_fn(*args) + + # The tf.metrics.mean function assures correct aggregation. + return tf.metrics.mean(scores, weights) + return problem_metric_fn + + +def get_eval_metrics(logits, labels, params): + """Return dictionary of model evaluation metrics.""" + metrics = { + "accuracy": _convert_to_eval_metric(padded_accuracy)(logits, labels), + "accuracy_top5": _convert_to_eval_metric(padded_accuracy_top5)( + logits, labels), + "accuracy_per_sequence": _convert_to_eval_metric( + padded_sequence_accuracy)(logits, labels), + "neg_log_perplexity": _convert_to_eval_metric(padded_neg_log_perplexity)( + logits, labels, params["vocab_size"]), + } + + if not params["use_tpu"]: + # TPU does not support tf.py_func + metrics.update({ + "approx_bleu_score": _convert_to_eval_metric( + bleu_score)(logits, labels), + "rouge_2_fscore": _convert_to_eval_metric( + rouge_2_fscore)(logits, labels), + "rouge_L_fscore": _convert_to_eval_metric( + rouge_l_fscore)(logits, labels), + }) + + # Prefix each of the metric names with "metrics/". This allows the metric + # graphs to display under the "metrics" category in TensorBoard. + metrics = {"metrics/%s" % k: v for k, v in six.iteritems(metrics)} + return metrics + + +def padded_accuracy(logits, labels): + """Percentage of times that predictions matches labels on non-0s.""" + with tf.variable_scope("padded_accuracy", values=[logits, labels]): + logits, labels = _pad_tensors_to_same_length(logits, labels) + weights = tf.to_float(tf.not_equal(labels, 0)) + outputs = tf.to_int32(tf.argmax(logits, axis=-1)) + padded_labels = tf.to_int32(labels) + return tf.to_float(tf.equal(outputs, padded_labels)), weights + + +def padded_accuracy_topk(logits, labels, k): + """Percentage of times that top-k predictions matches labels on non-0s.""" + with tf.variable_scope("padded_accuracy_topk", values=[logits, labels]): + logits, labels = _pad_tensors_to_same_length(logits, labels) + weights = tf.to_float(tf.not_equal(labels, 0)) + effective_k = tf.minimum(k, tf.shape(logits)[-1]) + _, outputs = tf.nn.top_k(logits, k=effective_k) + outputs = tf.to_int32(outputs) + padded_labels = tf.to_int32(labels) + padded_labels = tf.expand_dims(padded_labels, axis=-1) + padded_labels += tf.zeros_like(outputs) # Pad to same shape. + same = tf.to_float(tf.equal(outputs, padded_labels)) + same_topk = tf.reduce_sum(same, axis=-1) + return same_topk, weights + + +def padded_accuracy_top5(logits, labels): + return padded_accuracy_topk(logits, labels, 5) + + +def padded_sequence_accuracy(logits, labels): + """Percentage of times that predictions matches labels everywhere (non-0).""" + with tf.variable_scope("padded_sequence_accuracy", values=[logits, labels]): + logits, labels = _pad_tensors_to_same_length(logits, labels) + weights = tf.to_float(tf.not_equal(labels, 0)) + outputs = tf.to_int32(tf.argmax(logits, axis=-1)) + padded_labels = tf.to_int32(labels) + not_correct = tf.to_float(tf.not_equal(outputs, padded_labels)) * weights + axis = list(range(1, len(outputs.get_shape()))) + correct_seq = 1.0 - tf.minimum(1.0, tf.reduce_sum(not_correct, axis=axis)) + return correct_seq, tf.constant(1.0) + + +def padded_neg_log_perplexity(logits, labels, vocab_size): + """Average log-perplexity excluding padding 0s. No smoothing.""" + num, den = padded_cross_entropy_loss(logits, labels, 0, vocab_size) + return -num, den + + +def bleu_score(logits, labels): + """Approximate BLEU score computation between labels and predictions. + + An approximate BLEU scoring method since we do not glue word pieces or + decode the ids and tokenize the output. By default, we use ngram order of 4 + and use brevity penalty. Also, this does not have beam search. + + Args: + logits: Tensor of size [batch_size, length_logits, vocab_size] + labels: Tensor of size [batch-size, length_labels] + + Returns: + bleu: int, approx bleu score + """ + predictions = tf.to_int32(tf.argmax(logits, axis=-1)) + # TODO: Look into removing use of py_func + bleu = tf.py_func(compute_bleu, (labels, predictions), tf.float32) + return bleu, tf.constant(1.0) + + +def _get_ngrams_with_counter(segment, max_order): + """Extracts all n-grams up to a given maximum order from an input segment. + + Args: + segment: text segment from which n-grams will be extracted. + max_order: maximum length in tokens of the n-grams returned by this + methods. + + Returns: + The Counter containing all n-grams upto max_order in segment + with a count of how many times each n-gram occurred. + """ + ngram_counts = collections.Counter() + for order in xrange(1, max_order + 1): + for i in xrange(0, len(segment) - order + 1): + ngram = tuple(segment[i:i + order]) + ngram_counts[ngram] += 1 + return ngram_counts + + +def compute_bleu(reference_corpus, translation_corpus, max_order=4, + use_bp=True): + """Computes BLEU score of translated segments against one or more references. + + Args: + reference_corpus: list of references for each translation. Each + reference should be tokenized into a list of tokens. + translation_corpus: list of translations to score. Each translation + should be tokenized into a list of tokens. + max_order: Maximum n-gram order to use when computing BLEU score. + use_bp: boolean, whether to apply brevity penalty. + + Returns: + BLEU score. + """ + reference_length = 0 + translation_length = 0 + bp = 1.0 + geo_mean = 0 + + matches_by_order = [0] * max_order + possible_matches_by_order = [0] * max_order + precisions = [] + + for (references, translations) in zip(reference_corpus, translation_corpus): + reference_length += len(references) + translation_length += len(translations) + ref_ngram_counts = _get_ngrams_with_counter(references, max_order) + translation_ngram_counts = _get_ngrams_with_counter(translations, max_order) + + overlap = dict((ngram, + min(count, translation_ngram_counts[ngram])) + for ngram, count in ref_ngram_counts.items()) + + for ngram in overlap: + matches_by_order[len(ngram) - 1] += overlap[ngram] + for ngram in translation_ngram_counts: + possible_matches_by_order[len(ngram) - 1] += translation_ngram_counts[ + ngram] + + precisions = [0] * max_order + smooth = 1.0 + + for i in xrange(0, max_order): + if possible_matches_by_order[i] > 0: + precisions[i] = float(matches_by_order[i]) / possible_matches_by_order[i] + if matches_by_order[i] > 0: + precisions[i] = float(matches_by_order[i]) / possible_matches_by_order[ + i] + else: + smooth *= 2 + precisions[i] = 1.0 / (smooth * possible_matches_by_order[i]) + else: + precisions[i] = 0.0 + + if max(precisions) > 0: + p_log_sum = sum(math.log(p) for p in precisions if p) + geo_mean = math.exp(p_log_sum / max_order) + + if use_bp: + ratio = translation_length / reference_length + bp = math.exp(1 - 1. / ratio) if ratio < 1.0 else 1.0 + bleu = geo_mean * bp + return np.float32(bleu) + + +def rouge_2_fscore(logits, labels): + """ROUGE-2 F1 score computation between labels and predictions. + + This is an approximate ROUGE scoring method since we do not glue word pieces + or decode the ids and tokenize the output. + + Args: + logits: tensor, model predictions + labels: tensor, gold output. + + Returns: + rouge2_fscore: approx rouge-2 f1 score. + """ + predictions = tf.to_int32(tf.argmax(logits, axis=-1)) + # TODO: Look into removing use of py_func + rouge_2_f_score = tf.py_func(rouge_n, (predictions, labels), tf.float32) + return rouge_2_f_score, tf.constant(1.0) + + +def _get_ngrams(n, text): + """Calculates n-grams. + + Args: + n: which n-grams to calculate + text: An array of tokens + + Returns: + A set of n-grams + """ + ngram_set = set() + text_length = len(text) + max_index_ngram_start = text_length - n + for i in range(max_index_ngram_start + 1): + ngram_set.add(tuple(text[i:i + n])) + return ngram_set + + +def rouge_n(eval_sentences, ref_sentences, n=2): + """Computes ROUGE-N f1 score of two text collections of sentences. + + Source: https://www.microsoft.com/en-us/research/publication/ + rouge-a-package-for-automatic-evaluation-of-summaries/ + + Args: + eval_sentences: Predicted sentences. + ref_sentences: Sentences from the reference set + n: Size of ngram. Defaults to 2. + + Returns: + f1 score for ROUGE-N + """ + f1_scores = [] + for eval_sentence, ref_sentence in zip(eval_sentences, ref_sentences): + eval_ngrams = _get_ngrams(n, eval_sentence) + ref_ngrams = _get_ngrams(n, ref_sentence) + ref_count = len(ref_ngrams) + eval_count = len(eval_ngrams) + + # Count the overlapping ngrams between evaluated and reference + overlapping_ngrams = eval_ngrams.intersection(ref_ngrams) + overlapping_count = len(overlapping_ngrams) + + # Handle edge case. This isn't mathematically correct, but it's good enough + if eval_count == 0: + precision = 0.0 + else: + precision = float(overlapping_count) / eval_count + if ref_count == 0: + recall = 0.0 + else: + recall = float(overlapping_count) / ref_count + f1_scores.append(2.0 * ((precision * recall) / (precision + recall + 1e-8))) + + # return overlapping_count / reference_count + return np.mean(f1_scores, dtype=np.float32) + + +def rouge_l_fscore(predictions, labels): + """ROUGE scores computation between labels and predictions. + + This is an approximate ROUGE scoring method since we do not glue word pieces + or decode the ids and tokenize the output. + + Args: + predictions: tensor, model predictions + labels: tensor, gold output. + + Returns: + rouge_l_fscore: approx rouge-l f1 score. + """ + outputs = tf.to_int32(tf.argmax(predictions, axis=-1)) + rouge_l_f_score = tf.py_func(rouge_l_sentence_level, (outputs, labels), + tf.float32) + return rouge_l_f_score, tf.constant(1.0) + + +def rouge_l_sentence_level(eval_sentences, ref_sentences): + """Computes ROUGE-L (sentence level) of two collections of sentences. + + Source: https://www.microsoft.com/en-us/research/publication/ + rouge-a-package-for-automatic-evaluation-of-summaries/ + + Calculated according to: + R_lcs = LCS(X,Y)/m + P_lcs = LCS(X,Y)/n + F_lcs = ((1 + beta^2)*R_lcs*P_lcs) / (R_lcs + (beta^2) * P_lcs) + + where: + X = reference summary + Y = Candidate summary + m = length of reference summary + n = length of candidate summary + + Args: + eval_sentences: The sentences that have been picked by the summarizer + ref_sentences: The sentences from the reference set + + Returns: + A float: F_lcs + """ + + f1_scores = [] + for eval_sentence, ref_sentence in zip(eval_sentences, ref_sentences): + m = float(len(ref_sentence)) + n = float(len(eval_sentence)) + lcs = _len_lcs(eval_sentence, ref_sentence) + f1_scores.append(_f_lcs(lcs, m, n)) + return np.mean(f1_scores, dtype=np.float32) + + +def _len_lcs(x, y): + """Returns the length of the Longest Common Subsequence between two seqs. + + Source: http://www.algorithmist.com/index.php/Longest_Common_Subsequence + + Args: + x: sequence of words + y: sequence of words + + Returns + integer: Length of LCS between x and y + """ + table = _lcs(x, y) + n, m = len(x), len(y) + return table[n, m] + + +def _lcs(x, y): + """Computes the length of the LCS between two seqs. + + The implementation below uses a DP programming algorithm and runs + in O(nm) time where n = len(x) and m = len(y). + Source: http://www.algorithmist.com/index.php/Longest_Common_Subsequence + + Args: + x: collection of words + y: collection of words + + Returns: + Table of dictionary of coord and len lcs + """ + n, m = len(x), len(y) + table = dict() + for i in range(n + 1): + for j in range(m + 1): + if i == 0 or j == 0: + table[i, j] = 0 + elif x[i - 1] == y[j - 1]: + table[i, j] = table[i - 1, j - 1] + 1 + else: + table[i, j] = max(table[i - 1, j], table[i, j - 1]) + return table + + +def _f_lcs(llcs, m, n): + """Computes the LCS-based F-measure score. + + Source: http://research.microsoft.com/en-us/um/people/cyl/download/papers/ + rouge-working-note-v1.3.1.pdf + + Args: + llcs: Length of LCS + m: number of words in reference summary + n: number of words in candidate summary + + Returns: + Float. LCS-based F-measure score + """ + r_lcs = llcs / m + p_lcs = llcs / n + beta = p_lcs / (r_lcs + 1e-12) + num = (1 + (beta ** 2)) * r_lcs * p_lcs + denom = r_lcs + ((beta ** 2) * p_lcs) + f_lcs = num / (denom + 1e-12) + return f_lcs diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/utils/tokenizer.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/utils/tokenizer.py new file mode 100644 index 0000000..3749dfe --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/utils/tokenizer.py @@ -0,0 +1,660 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Defines Subtokenizer class to encode and decode strings.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import re +import sys +import unicodedata +from absl import logging + +import numpy as np +import six +from six.moves import xrange # pylint: disable=redefined-builtin +import tensorflow as tf + + +# pylint: disable=g-complex-comprehension +PAD = "" +PAD_ID = 0 +EOS = "" +EOS_ID = 1 +RESERVED_TOKENS = [PAD, EOS] + +# Set of characters that will be used in the function _escape_token() (see func +# docstring for more details). +# This set is added to the alphabet list to ensure that all escaped tokens can +# be encoded. +_ESCAPE_CHARS = set(u"\\_u;0123456789") +# Regex for the function _unescape_token(), the inverse of _escape_token(). +# This is used to find "\u", "\\", and "\###;" substrings in the token. +_UNESCAPE_REGEX = re.compile(r"\\u|\\\\|\\([0-9]+);") + +_UNDEFINED_UNICODE = u"\u3013" + + +def alphanumeric_char_set(): + return set( + six.unichr(i) + for i in xrange(sys.maxunicode) + if (unicodedata.category(six.unichr(i)).startswith("L") or + unicodedata.category(six.unichr(i)).startswith("N"))) + + +# Set contains all letter and number characters. +_ALPHANUMERIC_CHAR_SET = alphanumeric_char_set() + +# min_count is the minimum number of times a subtoken must appear in the data +# before before it is added to the vocabulary. The value is found using binary +# search to obtain the target vocabulary size. +_MIN_MIN_COUNT = 1 # min value to use when binary searching for min_count +_MAX_MIN_COUNT = 1000 # max value to use when binary searching for min_count + + +class Subtokenizer(object): + """Encodes and decodes strings to/from integer IDs.""" + + def __init__(self, vocab_file, reserved_tokens=None, master_char_set=None): + """Initializes class, creating a vocab file if data_files is provided.""" + logging.info("Initializing Subtokenizer from file %s.", vocab_file) + + if master_char_set is None: + master_char_set = _ALPHANUMERIC_CHAR_SET + + if reserved_tokens is None: + reserved_tokens = RESERVED_TOKENS + + self.subtoken_list = _load_vocab_file(vocab_file, reserved_tokens) + self.alphabet = _generate_alphabet_dict(self.subtoken_list) + self.subtoken_to_id_dict = _list_to_index_dict(self.subtoken_list) + + self.max_subtoken_length = 0 + for subtoken in self.subtoken_list: + self.max_subtoken_length = max(self.max_subtoken_length, len(subtoken)) + + # Create cache to speed up subtokenization + self._cache_size = 2**20 + self._cache = [(None, None)] * self._cache_size + self._master_char_set = master_char_set + + @staticmethod + def init_from_files(vocab_file, + files, + target_vocab_size, + threshold, + min_count=None, + file_byte_limit=1e6, + reserved_tokens=None, + correct_strip=True, + master_char_set=None): + """Create subtoken vocabulary based on files, and save vocab to file. + + Args: + vocab_file: String name of vocab file to store subtoken vocabulary. + files: List of file paths that will be used to generate vocabulary. + target_vocab_size: target vocabulary size to generate. + threshold: int threshold of vocabulary size to accept. + min_count: int minimum count to use for generating the vocabulary. The min + count is the minimum number of times a subtoken should appear in the + files before it is added to the vocabulary. If set to none, this value + is found using binary search. + file_byte_limit: (Default 1e6) Maximum number of bytes of sample text that + will be drawn from the files. + reserved_tokens: List of string tokens that are guaranteed to be at the + beginning of the subtoken vocabulary list. + correct_strip: Whether to convert text to unicode before strip. + master_char_set: the char set. + + Returns: + Subtokenizer object + """ + if master_char_set is None: + master_char_set = _ALPHANUMERIC_CHAR_SET + if reserved_tokens is None: + reserved_tokens = RESERVED_TOKENS + + if tf.io.gfile.exists(vocab_file): + logging.info("Vocab file already exists (%s)", vocab_file) + else: + logging.info("Begin steps to create subtoken vocabulary...") + token_counts = _count_tokens(files, file_byte_limit, correct_strip, + master_char_set) + alphabet = _generate_alphabet_dict(token_counts) + subtoken_list = _generate_subtokens_with_target_vocab_size( + token_counts, alphabet, target_vocab_size, threshold, min_count, + reserved_tokens) + logging.info("Generated vocabulary with %d subtokens.", + len(subtoken_list)) + _save_vocab_file(vocab_file, subtoken_list) + return Subtokenizer(vocab_file, master_char_set=master_char_set) + + def encode(self, raw_string, add_eos=False): + """Encodes a string into a list of int subtoken ids.""" + ret = [] + tokens = _split_string_to_tokens( + native_to_unicode(raw_string), self._master_char_set) + for token in tokens: + ret.extend(self._token_to_subtoken_ids(token)) + if add_eos: + assert EOS in self.subtoken_list, \ + "Can't append 'EOS' because it is not in list of known subtokens." + ret.append(EOS_ID) + return ret + + def _token_to_subtoken_ids(self, token): + """Encode a single token into a list of subtoken ids.""" + cache_location = hash(token) % self._cache_size + cache_key, cache_value = self._cache[cache_location] + if cache_key == token: + return cache_value + + ret = _split_token_to_subtokens( + _escape_token(token, self.alphabet), self.subtoken_to_id_dict, + self.max_subtoken_length) + ret = [self.subtoken_to_id_dict[subtoken_id] for subtoken_id in ret] + + self._cache[cache_location] = (token, ret) + return ret + + def decode(self, subtokens): + """Converts list of int subtokens ids into a string.""" + if isinstance(subtokens, np.ndarray): + # Note that list(subtokens) converts subtokens to a python list, but the + # items remain as np.int32. This converts both the array and its items. + subtokens = subtokens.tolist() + + if not subtokens: + return "" + + assert isinstance(subtokens, list) and isinstance(subtokens[0], int), ( + "Subtokens argument passed into decode() must be a list of integers.") + + return _unicode_to_native( + _join_tokens_to_string( + self._subtoken_ids_to_tokens(subtokens), self._master_char_set)) + + def _subtoken_ids_to_tokens(self, subtokens): + """Convert list of int subtoken ids to a list of string tokens.""" + escaped_tokens = "".join([ + self.subtoken_list[s] for s in subtokens if s < len(self.subtoken_list) + ]) + escaped_tokens = escaped_tokens.split("_") + + # All tokens in the vocabulary list have been escaped (see _escape_token()) + # so each token must be unescaped when decoding. + ret = [] + for token in escaped_tokens: + if token: + ret.append(_unescape_token(token)) + return ret + + +def _save_vocab_file(vocab_file, subtoken_list): + """Save subtokens to file.""" + with tf.io.gfile.GFile(vocab_file, mode="w") as f: + for subtoken in subtoken_list: + f.write("'%s'\n" % _unicode_to_native(subtoken)) + + +def _load_vocab_file(vocab_file, reserved_tokens=None): + """Load vocabulary while ensuring reserved tokens are at the top.""" + if reserved_tokens is None: + reserved_tokens = RESERVED_TOKENS + + subtoken_list = [] + with tf.io.gfile.GFile(vocab_file, mode="r") as f: + for line in f: + subtoken = native_to_unicode(line.strip()) + subtoken = subtoken[1:-1] # Remove surrounding single-quotes + if subtoken in reserved_tokens: + continue + subtoken_list.append(native_to_unicode(subtoken)) + return reserved_tokens + subtoken_list + + +def native_to_unicode(s): + """Convert string to unicode (required in Python 2).""" + try: # Python 2 + return s if isinstance(s, unicode) else s.decode("utf-8") + except NameError: # Python 3 + return s + + +def _unicode_to_native(s): + """Convert string from unicode to native format (required in Python 2).""" + try: # Python 2 + return s.encode("utf-8") if isinstance(s, unicode) else s + except NameError: # Python 3 + return s + + +def _split_string_to_tokens(text, master_char_set): + """Splits text to a list of string tokens.""" + if not text: + return [] + ret = [] + token_start = 0 + # Classify each character in the input string + is_master = [c in master_char_set for c in text] + for pos in xrange(1, len(text)): + if is_master[pos] != is_master[pos - 1]: + token = text[token_start:pos] + if token != u" " or token_start == 0: + ret.append(token) + token_start = pos + final_token = text[token_start:] + ret.append(final_token) + return ret + + +def _join_tokens_to_string(tokens, master_char_set): + """Join a list of string tokens into a single string.""" + token_is_master = [t[0] in master_char_set for t in tokens] + ret = [] + for i, token in enumerate(tokens): + if i > 0 and token_is_master[i - 1] and token_is_master[i]: + ret.append(u" ") + ret.append(token) + return "".join(ret) + + +def _escape_token(token, alphabet): + r"""Replace characters that aren't in the alphabet and append "_" to token. + + Apply three transformations to the token: + 1. Replace underline character "_" with "\u", and backslash "\" with "\\". + 2. Replace characters outside of the alphabet with "\###;", where ### is the + character's Unicode code point. + 3. Appends "_" to mark the end of a token. + + Args: + token: unicode string to be escaped + alphabet: list of all known characters + + Returns: + escaped string + """ + token = token.replace(u"\\", u"\\\\").replace(u"_", u"\\u") + ret = [c if c in alphabet and c != u"\n" else r"\%d;" % ord(c) for c in token] + return u"".join(ret) + "_" + + +def _unescape_token(token): + r"""Replaces escaped characters in the token with their unescaped versions. + + Applies inverse transformations as _escape_token(): + 1. Replace "\u" with "_", and "\\" with "\". + 2. Replace "\###;" with the unicode character the ### refers to. + + Args: + token: escaped string + + Returns: + unescaped string + """ + + def match(m): + r"""Returns replacement string for matched object. + + Matched objects contain one of the strings that matches the regex pattern: + r"\\u|\\\\|\\([0-9]+);" + The strings can be '\u', '\\', or '\###;' (### is any digit number). + + m.group(0) refers to the entire matched string ('\u', '\\', or '\###;'). + m.group(1) refers to the first parenthesized subgroup ('###'). + + m.group(0) exists for all match objects, while m.group(1) exists only for + the string '\###;'. + + This function looks to see if m.group(1) exists. If it doesn't, then the + matched string must be '\u' or '\\' . In this case, the corresponding + replacement ('_' and '\') are returned. Note that in python, a single + backslash is written as '\\', and double backslash as '\\\\'. + + If m.goup(1) exists, then use the integer in m.group(1) to return a + unicode character. + + Args: + m: match object + + Returns: + String to replace matched object with. + """ + # Check if the matched strings are '\u' or '\\'. + if m.group(1) is None: + return u"_" if m.group(0) == u"\\u" else u"\\" + + # If m.group(1) exists, try and return unicode character. + try: + return six.unichr(int(m.group(1))) + except (ValueError, OverflowError) as _: + return _UNDEFINED_UNICODE + + # Use match function to replace escaped substrings in the token. + return _UNESCAPE_REGEX.sub(match, token) + + +def _count_tokens(files, + file_byte_limit=1e6, + correct_strip=True, + master_char_set=None): + """Return token counts of words in the files. + + Samples file_byte_limit bytes from each file, and counts the words that appear + in the samples. The samples are semi-evenly distributed across the file. + + Args: + files: List of filepaths + file_byte_limit: Max number of bytes that will be read from each file. + correct_strip: Whether to convert text to unicode before strip. This affects + vocabulary generation for PY2. Sets correct_strip to False in PY2 to + reproduce previous common public result. Sets correct_strip to True will + let PY2 and PY3 get a consistent vocabulary. + master_char_set: the char set. + + Returns: + Dictionary mapping tokens to the number of times they appear in the sampled + lines from the files. + """ + if master_char_set is None: + master_char_set = _ALPHANUMERIC_CHAR_SET + + token_counts = collections.defaultdict(int) + + for filepath in files: + with tf.io.gfile.GFile(filepath, mode="r") as reader: + file_byte_budget = file_byte_limit + counter = 0 + lines_to_skip = int(reader.size() / (file_byte_budget * 2)) + for line in reader: + if counter < lines_to_skip: + counter += 1 + else: + if file_byte_budget < 0: + break + if correct_strip: + line = native_to_unicode(line) + line = line.strip() + file_byte_budget -= len(line) + counter = 0 + + # Add words to token counts + for token in _split_string_to_tokens( + native_to_unicode(line), master_char_set): + token_counts[token] += 1 + return token_counts + + +def _list_to_index_dict(lst): + """Create dictionary mapping list items to their indices in the list.""" + return {item: n for n, item in enumerate(lst)} + + +def _split_token_to_subtokens(token, subtoken_dict, max_subtoken_length): + """Splits a token into subtokens defined in the subtoken dict.""" + ret = [] + start = 0 + token_len = len(token) + while start < token_len: + # Find the longest subtoken, so iterate backwards. + for end in xrange(min(token_len, start + max_subtoken_length), start, -1): + subtoken = token[start:end] + if subtoken in subtoken_dict: + ret.append(subtoken) + start = end + break + else: # Did not break + # If there is no possible encoding of the escaped token then one of the + # characters in the token is not in the alphabet. This should be + # impossible and would be indicative of a bug. + raise ValueError("Was unable to split token \"%s\" into subtokens." % + token) + return ret + + +def _generate_subtokens_with_target_vocab_size(token_counts, + alphabet, + target_size, + threshold, + min_count=None, + reserved_tokens=None): + """Generate subtoken vocabulary close to the target size.""" + if reserved_tokens is None: + reserved_tokens = RESERVED_TOKENS + + if min_count is not None: + logging.info("Using min_count=%d to generate vocab with target size %d", + min_count, target_size) + return _generate_subtokens( + token_counts, alphabet, min_count, reserved_tokens=reserved_tokens) + + def bisect(min_val, max_val): + """Recursive function to binary search for subtoken vocabulary.""" + cur_count = (min_val + max_val) // 2 + logging.info("Binary search: trying min_count=%d (%d %d)", cur_count, + min_val, max_val) + subtoken_list = _generate_subtokens( + token_counts, alphabet, cur_count, reserved_tokens=reserved_tokens) + + val = len(subtoken_list) + logging.info("Binary search: min_count=%d resulted in %d tokens", cur_count, + val) + + within_threshold = abs(val - target_size) < threshold + if within_threshold or min_val >= max_val or cur_count < 2: + return subtoken_list + if val > target_size: + other_subtoken_list = bisect(cur_count + 1, max_val) + else: + other_subtoken_list = bisect(min_val, cur_count - 1) + + # Return vocabulary dictionary with the closest number of tokens. + other_val = len(other_subtoken_list) + if abs(other_val - target_size) < abs(val - target_size): + return other_subtoken_list + return subtoken_list + + logging.info("Finding best min_count to get target size of %d", target_size) + return bisect(_MIN_MIN_COUNT, _MAX_MIN_COUNT) + + +def _generate_alphabet_dict(iterable, reserved_tokens=None): + """Create set of characters that appear in any element in the iterable.""" + if reserved_tokens is None: + reserved_tokens = RESERVED_TOKENS + alphabet = {c for token in iterable for c in token} + alphabet |= {c for token in reserved_tokens for c in token} + alphabet |= _ESCAPE_CHARS # Add escape characters to alphabet set. + return alphabet + + +def _count_and_gen_subtokens(token_counts, alphabet, subtoken_dict, + max_subtoken_length): + """Count number of times subtokens appear, and generate new subtokens. + + Args: + token_counts: dict mapping tokens to the number of times they appear in the + original files. + alphabet: list of allowed characters. Used to escape the tokens, which + guarantees that all tokens can be split into subtokens. + subtoken_dict: dict mapping subtokens to ids. + max_subtoken_length: maximum length of subtoken in subtoken_dict. + + Returns: + A defaultdict mapping subtokens to the number of times they appear in the + tokens. The dict may contain new subtokens. + """ + subtoken_counts = collections.defaultdict(int) + for token, count in six.iteritems(token_counts): + token = _escape_token(token, alphabet) + subtokens = _split_token_to_subtokens(token, subtoken_dict, + max_subtoken_length) + + # Generate new subtokens by taking substrings from token. + start = 0 + for subtoken in subtokens: + for end in xrange(start + 1, len(token) + 1): + new_subtoken = token[start:end] + subtoken_counts[new_subtoken] += count + start += len(subtoken) + + return subtoken_counts + + +def _filter_and_bucket_subtokens(subtoken_counts, min_count): + """Return a bucketed list of subtokens that are filtered by count. + + Args: + subtoken_counts: defaultdict mapping subtokens to their counts + min_count: int count used to filter subtokens + + Returns: + List of subtoken sets, where subtokens in set i have the same length=i. + """ + # Create list of buckets, where subtokens in bucket i have length i. + subtoken_buckets = [] + for subtoken, count in six.iteritems(subtoken_counts): + if count < min_count: # Filter out subtokens that don't appear enough + continue + while len(subtoken_buckets) <= len(subtoken): + subtoken_buckets.append(set()) + subtoken_buckets[len(subtoken)].add(subtoken) + return subtoken_buckets + + +def _gen_new_subtoken_list(subtoken_counts, + min_count, + alphabet, + reserved_tokens=None): + """Generate candidate subtokens ordered by count, and new max subtoken length. + + Add subtokens to the candiate list in order of length (longest subtokens + first). When a subtoken is added, the counts of each of its prefixes are + decreased. Prefixes that don't appear much outside the subtoken are not added + to the candidate list. + + For example: + subtoken being added to candidate list: 'translate' + subtoken_counts: {'translate':10, 't':40, 'tr':16, 'tra':12, ...} + min_count: 5 + + When 'translate' is added, subtoken_counts is updated to: + {'translate':0, 't':30, 'tr':6, 'tra': 2, ...} + + The subtoken 'tra' will not be added to the candidate list, because it appears + twice (less than min_count) outside of 'translate'. + + Args: + subtoken_counts: defaultdict mapping str subtokens to int counts + min_count: int minumum count requirement for subtokens + alphabet: set of characters. Each character is added to the subtoken list to + guarantee that all tokens can be encoded. + reserved_tokens: list of tokens that will be added to the beginning of the + returned subtoken list. + + Returns: + List of candidate subtokens in decreasing count order, and maximum subtoken + length + """ + if reserved_tokens is None: + reserved_tokens = RESERVED_TOKENS + + # Create a list of (count, subtoken) for each candidate subtoken. + subtoken_candidates = [] + + # Use bucketted list to iterate through subtokens in order of length. + # subtoken_buckets[i] = set(subtokens), where each subtoken has length i. + subtoken_buckets = _filter_and_bucket_subtokens(subtoken_counts, min_count) + max_subtoken_length = len(subtoken_buckets) - 1 + + # Go through the list in reverse order to consider longer subtokens first. + for subtoken_len in xrange(max_subtoken_length, 0, -1): + for subtoken in subtoken_buckets[subtoken_len]: + count = subtoken_counts[subtoken] + + # Possible if this subtoken is a prefix of another token. + if count < min_count: + continue + + # Ignore alphabet/reserved tokens, which will be added manually later. + if subtoken not in alphabet and subtoken not in reserved_tokens: + subtoken_candidates.append((count, subtoken)) + + # Decrement count of the subtoken's prefixes (if a longer subtoken is + # added, its prefixes lose priority to be added). + for end in xrange(1, subtoken_len): + subtoken_counts[subtoken[:end]] -= count + + # Add alphabet subtokens (guarantees that all strings are encodable). + subtoken_candidates.extend((subtoken_counts.get(a, 0), a) for a in alphabet) + + # Order subtoken candidates by decreasing count. + subtoken_list = [t for _, t in sorted(subtoken_candidates, reverse=True)] + + # Add reserved tokens to beginning of the list. + subtoken_list = reserved_tokens + subtoken_list + return subtoken_list, max_subtoken_length + + +def _generate_subtokens(token_counts, + alphabet, + min_count, + num_iterations=4, + reserved_tokens=None): + """Create a list of subtokens in decreasing order of frequency. + + Args: + token_counts: dict mapping str tokens -> int count + alphabet: set of characters + min_count: int minimum number of times a subtoken must appear before it is + added to the vocabulary. + num_iterations: int number of iterations to generate new tokens. + reserved_tokens: list of tokens that will be added to the beginning to the + returned subtoken list. + + Returns: + Sorted list of subtokens (most frequent first) + """ + if reserved_tokens is None: + reserved_tokens = RESERVED_TOKENS + + # Use alphabet set to create initial list of subtokens + subtoken_list = reserved_tokens + list(alphabet) + max_subtoken_length = 1 + + # On each iteration, segment all words using the subtokens defined in + # subtoken_dict, count how often the resulting subtokens appear, and update + # the dictionary with subtokens w/ high enough counts. + for i in xrange(num_iterations): + logging.info("\tGenerating subtokens: iteration %d", i) + # Generate new subtoken->id dictionary using the new subtoken list. + subtoken_dict = _list_to_index_dict(subtoken_list) + + # Create dict mapping subtoken->count, with additional subtokens created + # from substrings taken from the tokens. + subtoken_counts = _count_and_gen_subtokens(token_counts, alphabet, + subtoken_dict, + max_subtoken_length) + + # Generate new list of subtokens sorted by subtoken count. + subtoken_list, max_subtoken_length = _gen_new_subtoken_list( + subtoken_counts, min_count, alphabet, reserved_tokens) + + logging.info("\tVocab size: %d", len(subtoken_list)) + return subtoken_list diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/utils/tokenizer_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/utils/tokenizer_test.py new file mode 100644 index 0000000..307398f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/transformer/utils/tokenizer_test.py @@ -0,0 +1,204 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Test Subtokenizer and string helper methods.""" + +import collections +import tempfile + +import tensorflow as tf + +from official.nlp.transformer.utils import tokenizer + + +class SubtokenizerTest(tf.test.TestCase): + + def _init_subtokenizer(self, vocab_list): + temp_file = tempfile.NamedTemporaryFile(delete=False) + with tf.io.gfile.GFile(temp_file.name, "w") as w: + for subtoken in vocab_list: + w.write("'%s'" % subtoken) + w.write("\n") + return tokenizer.Subtokenizer(temp_file.name, reserved_tokens=[]) + + def test_encode(self): + vocab_list = ["123_", "test", "ing_"] + subtokenizer = self._init_subtokenizer(vocab_list) + s = "testing 123" + encoded_list = subtokenizer.encode(s) + self.assertEqual([1, 2, 0], encoded_list) + + def test_decode(self): + vocab_list = ["123_", "test", "ing_"] + subtokenizer = self._init_subtokenizer(vocab_list) + encoded_list = [1, 2, 0] # testing 123 + decoded_str = subtokenizer.decode(encoded_list) + self.assertEqual("testing 123", decoded_str) + + def test_subtoken_ids_to_tokens(self): + vocab_list = ["123_", "test", "ing_"] + subtokenizer = self._init_subtokenizer(vocab_list) + encoded_list = [1, 2, 0] # testing 123 + token_list = subtokenizer._subtoken_ids_to_tokens(encoded_list) + self.assertEqual([u"testing", u"123"], token_list) + + +class StringHelperTest(tf.test.TestCase): + + def test_split_string_to_tokens(self): + text = "test? testing 123." + + tokens = tokenizer._split_string_to_tokens(text, + tokenizer._ALPHANUMERIC_CHAR_SET) + self.assertEqual(["test", "? ", "testing", "123", "."], tokens) + + def test_join_tokens_to_string(self): + tokens = ["test", "? ", "testing", "123", "."] + + s = tokenizer._join_tokens_to_string(tokens, + tokenizer._ALPHANUMERIC_CHAR_SET) + self.assertEqual("test? testing 123.", s) + + def test_escape_token(self): + token = u"abc_\\4" + alphabet = set("abc_\\u;") + + escaped_token = tokenizer._escape_token(token, alphabet) + self.assertEqual("abc\\u\\\\\\52;_", escaped_token) + + def test_unescape_token(self): + escaped_token = u"Underline: \\u, Backslash: \\\\, Unicode: \\52;" + + unescaped_token = tokenizer._unescape_token(escaped_token) + self.assertEqual("Underline: _, Backslash: \\, Unicode: 4", unescaped_token) + + def test_list_to_index_dict(self): + lst = ["test", "strings"] + + d = tokenizer._list_to_index_dict(lst) + self.assertDictEqual({"test": 0, "strings": 1}, d) + + def test_split_token_to_subtokens(self): + token = "abc" + subtoken_dict = {"a": 0, "b": 1, "c": 2, "ab": 3} + max_subtoken_length = 2 + + subtokens = tokenizer._split_token_to_subtokens(token, subtoken_dict, + max_subtoken_length) + self.assertEqual(["ab", "c"], subtokens) + + def test_generate_alphabet_dict(self): + s = ["testing", "123"] + reserved_tokens = ["???"] + + alphabet = tokenizer._generate_alphabet_dict(s, reserved_tokens) + self.assertIn("?", alphabet) + self.assertIn("t", alphabet) + self.assertIn("e", alphabet) + self.assertIn("s", alphabet) + self.assertIn("i", alphabet) + self.assertIn("n", alphabet) + self.assertIn("g", alphabet) + self.assertIn("1", alphabet) + self.assertIn("2", alphabet) + self.assertIn("3", alphabet) + + def test_count_and_gen_subtokens(self): + token_counts = {"abc": 5} + alphabet = set("abc_") + subtoken_dict = {"a": 0, "b": 1, "c": 2, "_": 3} + max_subtoken_length = 2 + + subtoken_counts = tokenizer._count_and_gen_subtokens( + token_counts, alphabet, subtoken_dict, max_subtoken_length) + + self.assertIsInstance(subtoken_counts, collections.defaultdict) + self.assertDictEqual( + { + "a": 5, + "b": 5, + "c": 5, + "_": 5, + "ab": 5, + "bc": 5, + "c_": 5, + "abc": 5, + "bc_": 5, + "abc_": 5 + }, subtoken_counts) + + def test_filter_and_bucket_subtokens(self): + subtoken_counts = collections.defaultdict(int, { + "a": 2, + "b": 4, + "c": 1, + "ab": 6, + "ac": 3, + "abbc": 5 + }) + min_count = 3 + + subtoken_buckets = tokenizer._filter_and_bucket_subtokens( + subtoken_counts, min_count) + + self.assertEqual(len(subtoken_buckets[0]), 0) + self.assertEqual(set("b"), subtoken_buckets[1]) + self.assertEqual(set(["ab", "ac"]), subtoken_buckets[2]) + self.assertEqual(len(subtoken_buckets[3]), 0) + self.assertEqual(set(["abbc"]), subtoken_buckets[4]) + + def test_gen_new_subtoken_list(self): + subtoken_counts = collections.defaultdict(int, { + "translate": 10, + "t": 40, + "tr": 16, + "tra": 12 + }) + min_count = 5 + alphabet = set("translate") + reserved_tokens = ["reserved", "tokens"] + + subtoken_list, max_token_length = tokenizer._gen_new_subtoken_list( + subtoken_counts, min_count, alphabet, reserved_tokens) + + # Check that "tra" isn"t in the list (its count should be decremented to 2, + # so it should not be added to the canddiate list). + self.assertNotIn("tra", subtoken_list) + + self.assertIn("tr", subtoken_list) + self.assertIn("t", subtoken_list) + + self.assertEqual(len("translate"), max_token_length) + + def test_generate_subtokens(self): + token_counts = {"ab": 1, "bc": 3, "abc": 5} + alphabet = set("abc_") + min_count = 100 + num_iterations = 1 + reserved_tokens = ["reserved", "tokens"] + + vocab_list = tokenizer._generate_subtokens(token_counts, alphabet, + min_count, num_iterations, + reserved_tokens) + + # Check that reserved tokens are at the front of the list + self.assertEqual(vocab_list[:2], reserved_tokens) + + # Check that each character in alphabet is in the vocab list + for c in alphabet: + self.assertIn(c, vocab_list) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/README.md new file mode 100644 index 0000000..9675f01 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/README.md @@ -0,0 +1,16 @@ +# XLNet: Generalized Autoregressive Pretraining for Language Understanding + +The academic paper which describes XLNet in detail and provides full results on +a number of tasks can be found here: https://arxiv.org/abs/1906.08237. + +**Instructions and user guide will be added soon.** + +XLNet is a generalized autoregressive BERT-like pretraining language model that +enables learning bidirectional contexts by maximizing the expected likelihood +over all permutations of the factorization order. It can learn dependency beyond +a fixed length without disrupting temporal coherence by using segment-level +recurrence mechanism and relative positional encoding scheme introduced in +[Transformer-XL](https://arxiv.org/pdf/1901.02860.pdf). XLNet outperforms BERT +on 20 NLP benchmark tasks and achieves state-of-the-art results on 18 tasks +including question answering, natural language inference, sentiment analysis, +and document ranking. diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/__init__.py @@ -0,0 +1 @@ + diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/classifier_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/classifier_utils.py new file mode 100644 index 0000000..64363e3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/classifier_utils.py @@ -0,0 +1,162 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities for pre-processing classification data.""" +from absl import logging + +from official.nlp.xlnet import data_utils + +SEG_ID_A = 0 +SEG_ID_B = 1 + + +class PaddingInputExample(object): + """Fake example so the num input examples is a multiple of the batch size. + + When running eval/predict on the TPU, we need to pad the number of examples + to be a multiple of the batch size, because the TPU requires a fixed batch + size. The alternative is to drop the last batch, which is bad because it means + the entire output data won't be generated. + We use this class instead of `None` because treating `None` as padding + battches could cause silent errors. + """ + + +class InputFeatures(object): + """A single set of features of data.""" + + def __init__(self, + input_ids, + input_mask, + segment_ids, + label_id, + is_real_example=True): + self.input_ids = input_ids + self.input_mask = input_mask + self.segment_ids = segment_ids + self.label_id = label_id + self.is_real_example = is_real_example + + +def _truncate_seq_pair(tokens_a, tokens_b, max_length): + """Truncates a sequence pair in place to the maximum length.""" + + # This is a simple heuristic which will always truncate the longer sequence + # one token at a time. This makes more sense than truncating an equal percent + # of tokens from each, since if one sequence is very short then each token + # that's truncated likely contains more information than a longer sequence. + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_length: + break + if len(tokens_a) > len(tokens_b): + tokens_a.pop() + else: + tokens_b.pop() + + +def convert_single_example(example_index, example, label_list, max_seq_length, + tokenize_fn, use_bert_format): + """Converts a single `InputExample` into a single `InputFeatures`.""" + + if isinstance(example, PaddingInputExample): + return InputFeatures( + input_ids=[0] * max_seq_length, + input_mask=[1] * max_seq_length, + segment_ids=[0] * max_seq_length, + label_id=0, + is_real_example=False) + + if label_list is not None: + label_map = {} + for (i, label) in enumerate(label_list): + label_map[label] = i + + tokens_a = tokenize_fn(example.text_a) + tokens_b = None + if example.text_b: + tokens_b = tokenize_fn(example.text_b) + + if tokens_b: + # Modifies `tokens_a` and `tokens_b` in place so that the total + # length is less than the specified length. + # Account for two [SEP] & one [CLS] with "- 3" + _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3) + else: + # Account for one [SEP] & one [CLS] with "- 2" + if len(tokens_a) > max_seq_length - 2: + tokens_a = tokens_a[:max_seq_length - 2] + + tokens = [] + segment_ids = [] + for token in tokens_a: + tokens.append(token) + segment_ids.append(SEG_ID_A) + tokens.append(data_utils.SEP_ID) + segment_ids.append(SEG_ID_A) + + if tokens_b: + for token in tokens_b: + tokens.append(token) + segment_ids.append(SEG_ID_B) + tokens.append(data_utils.SEP_ID) + segment_ids.append(SEG_ID_B) + + if use_bert_format: + tokens.insert(0, data_utils.CLS_ID) + segment_ids.insert(0, data_utils.SEG_ID_CLS) + else: + tokens.append(data_utils.CLS_ID) + segment_ids.append(data_utils.SEG_ID_CLS) + + input_ids = tokens + + # The mask has 0 for real tokens and 1 for padding tokens. Only real + # tokens are attended to. + input_mask = [0] * len(input_ids) + + # Zero-pad up to the sequence length. + if len(input_ids) < max_seq_length: + delta_len = max_seq_length - len(input_ids) + if use_bert_format: + input_ids = input_ids + [0] * delta_len + input_mask = input_mask + [1] * delta_len + segment_ids = segment_ids + [data_utils.SEG_ID_PAD] * delta_len + else: + input_ids = [0] * delta_len + input_ids + input_mask = [1] * delta_len + input_mask + segment_ids = [data_utils.SEG_ID_PAD] * delta_len + segment_ids + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + + if label_list is not None: + label_id = label_map[example.label] + else: + label_id = example.label + if example_index < 5: + logging.info("*** Example ***") + logging.info("guid: %s", (example.guid)) + logging.info("input_ids: %s", " ".join([str(x) for x in input_ids])) + logging.info("input_mask: %s", " ".join([str(x) for x in input_mask])) + logging.info("segment_ids: %s", " ".join([str(x) for x in segment_ids])) + logging.info("label: %d (id = %d)", example.label, label_id) + + feature = InputFeatures( + input_ids=input_ids, + input_mask=input_mask, + segment_ids=segment_ids, + label_id=label_id) + return feature diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/common_flags.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/common_flags.py new file mode 100644 index 0000000..93d9499 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/common_flags.py @@ -0,0 +1,146 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Common flags used in XLNet model.""" +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +from absl import flags + +flags.DEFINE_string("master", default=None, help="master") +flags.DEFINE_string( + "tpu", + default=None, + help="The Cloud TPU to use for training. This should be " + "either the name used when creating the Cloud TPU, or a " + "url like grpc://ip.address.of.tpu:8470.") +flags.DEFINE_bool( + "use_tpu", default=True, help="Use TPUs rather than plain CPUs.") +flags.DEFINE_string("tpu_topology", "2x2", help="TPU topology.") +flags.DEFINE_integer( + "num_core_per_host", default=8, help="number of cores per host") + +flags.DEFINE_string("model_dir", default=None, help="Estimator model_dir.") +flags.DEFINE_string( + "init_checkpoint", + default=None, + help="Checkpoint path for initializing the model.") +flags.DEFINE_bool( + "init_from_transformerxl", + default=False, + help="Init from a transformerxl model checkpoint. Otherwise, init from the " + "entire model checkpoint.") + +# Optimization config +flags.DEFINE_float("learning_rate", default=1e-4, help="Maximum learning rate.") +flags.DEFINE_float("clip", default=1.0, help="Gradient clipping value.") +flags.DEFINE_float("weight_decay_rate", default=0.0, help="Weight decay rate.") + +# lr decay +flags.DEFINE_integer( + "warmup_steps", default=0, help="Number of steps for linear lr warmup.") +flags.DEFINE_float("adam_epsilon", default=1e-8, help="Adam epsilon.") +flags.DEFINE_float( + "lr_layer_decay_rate", + default=1.0, + help="Top layer: lr[L] = FLAGS.learning_rate." + "Lower layers: lr[l-1] = lr[l] * lr_layer_decay_rate.") +flags.DEFINE_float( + "min_lr_ratio", default=0.0, help="Minimum ratio learning rate.") + +# Training config +flags.DEFINE_integer( + "train_batch_size", + default=16, + help="Size of the train batch across all hosts.") +flags.DEFINE_integer( + "train_steps", default=100000, help="Total number of training steps.") +flags.DEFINE_integer( + "iterations", default=1000, help="Number of iterations per repeat loop.") + +# Data config +flags.DEFINE_integer( + "seq_len", default=0, help="Sequence length for pretraining.") +flags.DEFINE_integer( + "reuse_len", + default=0, + help="How many tokens to be reused in the next batch. " + "Could be half of `seq_len`.") +flags.DEFINE_bool("uncased", False, help="Use uncased inputs or not.") +flags.DEFINE_bool( + "bi_data", + default=False, + help="Use bidirectional data streams, " + "i.e., forward & backward.") +flags.DEFINE_integer("n_token", 32000, help="Vocab size") + +# Model config +flags.DEFINE_integer("mem_len", default=0, help="Number of steps to cache") +flags.DEFINE_bool("same_length", default=False, help="Same length attention") +flags.DEFINE_integer("clamp_len", default=-1, help="Clamp length") + +flags.DEFINE_integer("n_layer", default=6, help="Number of layers.") +flags.DEFINE_integer("d_model", default=32, help="Dimension of the model.") +flags.DEFINE_integer("d_embed", default=32, help="Dimension of the embeddings.") +flags.DEFINE_integer("n_head", default=4, help="Number of attention heads.") +flags.DEFINE_integer( + "d_head", default=8, help="Dimension of each attention head.") +flags.DEFINE_integer( + "d_inner", + default=32, + help="Dimension of inner hidden size in positionwise " + "feed-forward.") +flags.DEFINE_float("dropout", default=0.1, help="Dropout rate.") +flags.DEFINE_float("dropout_att", default=0.1, help="Attention dropout rate.") +flags.DEFINE_bool("untie_r", default=False, help="Untie r_w_bias and r_r_bias") +flags.DEFINE_string( + "ff_activation", + default="relu", + help="Activation type used in position-wise feed-forward.") +flags.DEFINE_string( + "strategy_type", + default="tpu", + help="Activation type used in position-wise feed-forward.") +flags.DEFINE_bool("use_bfloat16", False, help="Whether to use bfloat16.") + +# Parameter initialization +flags.DEFINE_enum( + "init_method", + default="normal", + enum_values=["normal", "uniform"], + help="Initialization method.") +flags.DEFINE_float( + "init_std", default=0.02, help="Initialization std when init is normal.") +flags.DEFINE_float( + "init_range", default=0.1, help="Initialization std when init is uniform.") + +flags.DEFINE_integer( + "test_data_size", default=12048, help="Number of test data samples.") +flags.DEFINE_string( + "train_tfrecord_path", + default=None, + help="Path to preprocessed training set tfrecord.") +flags.DEFINE_string( + "test_tfrecord_path", + default=None, + help="Path to preprocessed test set tfrecord.") +flags.DEFINE_integer( + "test_batch_size", + default=16, + help="Size of the test batch across all hosts.") +flags.DEFINE_integer( + "save_steps", default=1000, help="Number of steps for saving checkpoint.") +FLAGS = flags.FLAGS diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/data_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/data_utils.py new file mode 100644 index 0000000..91326e0 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/data_utils.py @@ -0,0 +1,828 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities used for data preparation.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import collections +import json +import os +from absl import logging + +import numpy as np +import tensorflow as tf + + +special_symbols = { + "": 0, + "": 1, + "": 2, + "": 3, + "": 4, + "": 5, + "": 6, + "": 7, + "": 8, +} + +VOCAB_SIZE = 32000 +UNK_ID = special_symbols[""] +CLS_ID = special_symbols[""] +SEP_ID = special_symbols[""] +MASK_ID = special_symbols[""] +EOD_ID = special_symbols[""] +SEG_ID_P = 0 +SEG_ID_Q = 1 +SEG_ID_CLS = 2 +SEG_ID_PAD = 3 + + +OnlineMaskingConfig = collections.namedtuple("OnlineMaskingConfig", [ + "sample_strategy", "max_num_tokens", "min_num_tokens", "max_num_words", + "min_num_words"]) + + +def file_based_input_fn_builder(input_file, name_to_features, batch_size, + is_training): + """Creates an `input_fn` closure.""" + + logging.info("Input tfrecord file %s", input_file) + + def _decode_record(record, name_to_features): + """Decodes a record to a TensorFlow example.""" + example = tf.io.parse_single_example(record, name_to_features) + + # tf.Example only supports tf.int64, but the TPU only supports tf.int32. + # So cast all int64 to int32. + for name in list(example.keys()): + t = example[name] + if t.dtype == tf.int64: + t = tf.cast(t, tf.int32) + example[name] = t + + return example + + def input_fn(): + """Returns dataset for training/evaluation.""" + num_threads = 8 + if isinstance(input_file, str): + d = tf.data.TFRecordDataset(input_file) + # For training, we want a lot of parallel reading and shuffling. + # For eval, we want no shuffling and parallel reading doesn't matter. + if is_training: + d = d.shuffle(2048) + d = d.repeat() + else: + cycle_length = min(num_threads, len(input_file)) + d = tf.data.Dataset.from_tensor_slices(input_file) + # file level shuffle + d = d.shuffle(len(input_file)).repeat() + + d = d.apply( + tf.data.experimental.parallel_interleave( + tf.data.TFRecordDataset, + sloppy=is_training, + cycle_length=cycle_length)) + + if is_training: + # sample level shuffle + d = d.shuffle(buffer_size=2048) + + # TODO(b/138223458): Hard-code drop_remainder=True to get around the bug + # that under TPU strategy, setting drop_remainder=False in + # tf.data.Dataset.batch() while data_size can be divided by global + # batch_size will trigger dynamic_dimension related TPU compilation error. + d = d.apply( + tf.data.experimental.map_and_batch( + lambda record: _decode_record(record, name_to_features), + batch_size=batch_size, + num_parallel_batches=num_threads, + drop_remainder=True)) + + # When `input_file` is a path to a single file or a list + # containing a single path, disable auto sharding so that + # same input file is sent to all workers. + if isinstance(input_file, str) or len(input_file) == 1: + options = tf.data.Options() + options.experimental_distribute.auto_shard_policy = ( + tf.data.experimental.AutoShardPolicy.OFF) + d = d.with_options(options) + + d = d.prefetch(tf.data.experimental.AUTOTUNE) + return d + + return input_fn + + +def create_classification_dataset(file_path, seq_length, batch_size, + is_training): + """Creates input dataset from (tf)records files for pretraining.""" + name_to_features = { + "input_ids": tf.io.FixedLenFeature([seq_length], tf.int64), + "input_mask": tf.io.FixedLenFeature([seq_length], tf.float32), + "segment_ids": tf.io.FixedLenFeature([seq_length], tf.int64), + "label_ids": tf.io.FixedLenFeature([], tf.int64), + "is_real_example": tf.io.FixedLenFeature([], tf.int64), + } + + input_fn = file_based_input_fn_builder(file_path, name_to_features, + batch_size, is_training) + dataset = input_fn() + return dataset + + +def create_squad_dataset(file_path, seq_length, batch_size, is_training): + """Creates input dataset from (tf)records files for pretraining.""" + name_to_features = { + "unique_ids": tf.io.FixedLenFeature([], tf.int64), + "input_ids": tf.io.FixedLenFeature([seq_length], tf.int64), + "input_mask": tf.io.FixedLenFeature([seq_length], tf.float32), + "segment_ids": tf.io.FixedLenFeature([seq_length], tf.int64), + "cls_index": tf.io.FixedLenFeature([], tf.int64), + "p_mask": tf.io.FixedLenFeature([seq_length], tf.float32) + } + + if is_training: + name_to_features["start_positions"] = tf.io.FixedLenFeature([], tf.int64) + name_to_features["end_positions"] = tf.io.FixedLenFeature([], tf.int64) + name_to_features["is_impossible"] = tf.io.FixedLenFeature([], tf.float32) + + input_fn = file_based_input_fn_builder(file_path, name_to_features, + batch_size, is_training) + dataset = input_fn() + return dataset + + +def get_input_iterator(input_fn, strategy): + """Returns distributed dataset iterator.""" + + # When training with TPU pods, datasets needs to be cloned across + # workers. Since Dataset instance cannot be cloned in eager mode, we instead + # pass callable that returns a dataset. + input_data = input_fn() + if callable(input_data): + iterator = iter( + strategy.experimental_distribute_datasets_from_function(input_data)) + else: + iterator = iter(strategy.experimental_distribute_dataset(input_data)) + return iterator + + +def get_classification_input_data(batch_size, seq_len, strategy, is_training, + file_path): + """Returns input dataset from input file string.""" + + # When using TPU pods, we need to clone dataset across + # workers and need to pass in function that returns the dataset rather + # than passing dataset instance itself. + use_dataset_fn = isinstance(strategy, tf.distribute.experimental.TPUStrategy) + if use_dataset_fn: + if batch_size % strategy.num_replicas_in_sync != 0: + raise ValueError( + "Batch size must be divisible by number of replicas : {}".format( + strategy.num_replicas_in_sync)) + + # As auto rebatching is not supported in + # `experimental_distribute_datasets_from_function()` API, which is + # required when cloning dataset to multiple workers in eager mode, + # we use per-replica batch size. + batch_size = int(batch_size / strategy.num_replicas_in_sync) + + def _dataset_fn(ctx=None): + del ctx + + train_dataset = create_classification_dataset( + file_path=file_path, + seq_length=seq_len, + batch_size=batch_size, + is_training=is_training) + return train_dataset + + return _dataset_fn if use_dataset_fn else _dataset_fn() + + +def get_squad_input_data(batch_size, seq_len, q_len, strategy, is_training, + file_path): + """Returns input dataset from input file string.""" + + # When using TPU pods, we need to clone dataset across + # workers and need to pass in function that returns the dataset rather + # than passing dataset instance itself. + use_dataset_fn = isinstance(strategy, tf.distribute.experimental.TPUStrategy) + if use_dataset_fn: + if batch_size % strategy.num_replicas_in_sync != 0: + raise ValueError( + "Batch size must be divisible by number of replicas : {}".format( + strategy.num_replicas_in_sync)) + + # As auto rebatching is not supported in + # `experimental_distribute_datasets_from_function()` API, which is + # required when cloning dataset to multiple workers in eager mode, + # we use per-replica batch size. + batch_size = int(batch_size / strategy.num_replicas_in_sync) + + if is_training: + input_glob = os.path.join( + file_path, + "spiece.model.*.slen-{}.qlen-{}.train.tf_record".format(seq_len, q_len)) + + global_input_paths = tf.io.gfile.glob(input_glob) + else: + global_input_paths = file_path + + def _dataset_fn(ctx=None): + del ctx + + train_dataset = create_squad_dataset( + file_path=global_input_paths, + seq_length=seq_len, + batch_size=batch_size, + is_training=is_training) + return train_dataset + + return _dataset_fn if use_dataset_fn else _dataset_fn() + + +def _idx_pair_to_mask(beg_indices, end_indices, inputs, tgt_len, num_predict): + """Turn beg and end indices into actual mask.""" + non_func_mask = tf.logical_and( + tf.not_equal(inputs, SEP_ID), + tf.not_equal(inputs, CLS_ID)) + all_indices = tf.where( + non_func_mask, + tf.range(tgt_len, dtype=tf.int64), + tf.constant(-1, shape=[tgt_len], dtype=tf.int64)) + candidate_matrix = tf.cast( + tf.logical_and( + all_indices[None, :] >= beg_indices[:, None], + all_indices[None, :] < end_indices[:, None]), + tf.float32) + cumsum_matrix = tf.reshape( + tf.cumsum(tf.reshape(candidate_matrix, [-1])), + [-1, tgt_len]) + masked_matrix = tf.cast(cumsum_matrix <= num_predict, tf.float32) + target_mask = tf.reduce_sum(candidate_matrix * masked_matrix, axis=0) + is_masked = tf.cast(target_mask, tf.bool) + + return is_masked, target_mask + + +def _word_span_mask(inputs, tgt_len, num_predict, min_num_words, + max_num_words, boundary): + """Sample whole word spans as prediction targets.""" + # Note: 1.2 is the token-to-word ratio + mask_alpha = tgt_len / num_predict / 1.2 + round_to_int = lambda x: tf.cast(tf.round(x), tf.int64) + + # Sample span lengths from a zipf distribution + span_len_seq = np.arange(min_num_words, max_num_words + 1) + probs = np.array([1.0 / (i + 1) for i in span_len_seq]) + probs /= np.sum(probs) + logits = tf.constant(np.log(probs), dtype=tf.float32) + + # Sample `num_predict` words here: note that this is over sampling + span_lens = tf.random.categorical( + logits=logits[None], + num_samples=num_predict, + dtype=tf.int64, + )[0] + min_num_words + + # Sample the ratio [0.0, 1.0) of left context lengths + span_lens_float = tf.cast(span_lens, tf.float32) + left_ratio = tf.random.uniform(shape=[num_predict], minval=0.0, maxval=1.0) + left_ctx_len = left_ratio * span_lens_float * (mask_alpha - 1) + + left_ctx_len = round_to_int(left_ctx_len) + right_offset = round_to_int(span_lens_float * mask_alpha) - left_ctx_len + + beg_indices = (tf.cumsum(left_ctx_len) + + tf.cumsum(right_offset, exclusive=True)) + end_indices = beg_indices + span_lens + + # Remove out of range indices + max_boundary_index = tf.cast(tf.shape(boundary)[0] - 1, tf.int64) + valid_idx_mask = end_indices < max_boundary_index + beg_indices = tf.boolean_mask(beg_indices, valid_idx_mask) + end_indices = tf.boolean_mask(end_indices, valid_idx_mask) + + beg_indices = tf.gather(boundary, beg_indices) + end_indices = tf.gather(boundary, end_indices) + + # Shuffle valid indices + num_valid = tf.cast(tf.shape(beg_indices)[0], tf.int64) + order = tf.random.shuffle(tf.range(num_valid, dtype=tf.int64)) + beg_indices = tf.gather(beg_indices, order) + end_indices = tf.gather(end_indices, order) + + return _idx_pair_to_mask(beg_indices, end_indices, inputs, tgt_len, + num_predict) + + +def _token_span_mask(inputs, tgt_len, num_predict, min_num_tokens, + max_num_tokens): + """Sample token spans as prediction targets.""" + mask_alpha = tgt_len / num_predict + round_to_int = lambda x: tf.cast(tf.round(x), tf.int64) + + # Sample span lengths from a zipf distribution + span_len_seq = np.arange(min_num_tokens, max_num_tokens + 1) + probs = np.array([1.0 / (i + 1) for i in span_len_seq]) + + probs /= np.sum(probs) + logits = tf.constant(np.log(probs), dtype=tf.float32) + span_lens = tf.random.categorical( + logits=logits[None], + num_samples=num_predict, + dtype=tf.int64, + )[0] + min_num_tokens + + # Sample the ratio [0.0, 1.0) of left context lengths + span_lens_float = tf.cast(span_lens, tf.float32) + left_ratio = tf.random.uniform(shape=[num_predict], minval=0.0, maxval=1.0) + left_ctx_len = left_ratio * span_lens_float * (mask_alpha - 1) + left_ctx_len = round_to_int(left_ctx_len) + + # Compute the offset from left start to the right end + right_offset = round_to_int(span_lens_float * mask_alpha) - left_ctx_len + + # Get the actual begin and end indices + beg_indices = (tf.cumsum(left_ctx_len) + + tf.cumsum(right_offset, exclusive=True)) + end_indices = beg_indices + span_lens + + # Remove out of range indices + valid_idx_mask = end_indices < tgt_len + beg_indices = tf.boolean_mask(beg_indices, valid_idx_mask) + end_indices = tf.boolean_mask(end_indices, valid_idx_mask) + + # Shuffle valid indices + num_valid = tf.cast(tf.shape(beg_indices)[0], tf.int64) + order = tf.random.shuffle(tf.range(num_valid, dtype=tf.int64)) + beg_indices = tf.gather(beg_indices, order) + end_indices = tf.gather(end_indices, order) + + return _idx_pair_to_mask(beg_indices, end_indices, inputs, tgt_len, + num_predict) + + +def _whole_word_mask(inputs, tgt_len, num_predict, boundary): + """Sample whole words as prediction targets.""" + pair_indices = tf.concat([boundary[:-1, None], boundary[1:, None]], axis=1) + cand_pair_indices = tf.random.shuffle(pair_indices)[:num_predict] + beg_indices = cand_pair_indices[:, 0] + end_indices = cand_pair_indices[:, 1] + + return _idx_pair_to_mask(beg_indices, end_indices, inputs, tgt_len, + num_predict) + + +def _single_token_mask(inputs, tgt_len, num_predict): + """Sample individual tokens as prediction targets.""" + all_indices = tf.range(tgt_len, dtype=tf.int64) + non_func_mask = tf.logical_and( + tf.not_equal(inputs, SEP_ID), + tf.not_equal(inputs, CLS_ID)) + non_func_indices = tf.boolean_mask(all_indices, non_func_mask) + + masked_pos = tf.random.shuffle(non_func_indices) + masked_pos = tf.sort(masked_pos[:num_predict]) + target_mask = tf.sparse_to_dense( + sparse_indices=masked_pos, + output_shape=[tgt_len], + sparse_values=1.0, + default_value=0.0) + + is_masked = tf.cast(target_mask, tf.bool) + + return is_masked, target_mask + + +def _online_sample_masks(inputs, tgt_len, num_predict, online_masking_config, + boundary=None): + """Sample target positions to predict.""" + logging.info("Online sample with strategy: `%s`.", + online_masking_config.sample_strategy) + if online_masking_config.sample_strategy == "single_token": + return _single_token_mask(inputs, tgt_len, num_predict) + elif online_masking_config.sample_strategy == "whole_word": + assert boundary is not None, "whole word sampling requires `boundary`" + return _whole_word_mask(inputs, tgt_len, num_predict, boundary) + elif online_masking_config.sample_strategy == "token_span": + return _token_span_mask(inputs, tgt_len, num_predict, + online_masking_config.min_num_tokens, + online_masking_config.max_num_tokens) + elif online_masking_config.sample_strategy == "word_span": + assert boundary is not None, "word span sampling requires `boundary`" + return _word_span_mask(inputs, tgt_len, num_predict, + online_masking_config.min_num_words, + online_masking_config.max_num_words, + boundary) + else: + raise NotImplementedError + + +def create_pretrain_dataset(file_names, + bsz_per_core, + seq_len, + reuse_len, + perm_size, + leak_ratio, + online_masking_config, + num_predict=None, + input_pipeline_context=None): + """Creates pretrain dataset.""" + + def parser(record): + """Function used to parse tfrecord.""" + + record_spec = { + "input": tf.io.FixedLenFeature([seq_len], tf.int64), + "seg_id": tf.io.FixedLenFeature([seq_len], tf.int64), + "label": tf.io.FixedLenFeature([1], tf.int64), + } + + if online_masking_config.sample_strategy in ["whole_word", "word_span"]: + logging.info("Add `boundary` spec for %s", + online_masking_config.sample_strategy) + record_spec["boundary"] = tf.io.VarLenFeature(tf.int64) + + # retrieve serialized example + example = tf.io.parse_single_example( + serialized=record, features=record_spec) + + inputs = example.pop("input") + if online_masking_config.sample_strategy in ["whole_word", "word_span"]: + boundary = tf.sparse.to_dense(example.pop("boundary")) + else: + boundary = None + is_masked, _ = _online_sample_masks( + inputs, seq_len, num_predict, online_masking_config, boundary=boundary) + + if reuse_len > 0: + ##### Use memory + # permutate the reuse and non-reuse parts separately + non_reuse_len = seq_len - reuse_len + assert reuse_len % perm_size == 0 and non_reuse_len % perm_size == 0 + + # Creates permutation mask and target mask for the first reuse_len tokens. + # The tokens in this part are reused from the last sequence. + perm_mask_0, target_mask_0, input_k_0, input_q_0 = _local_perm( + inputs[:reuse_len], is_masked[:reuse_len], perm_size, reuse_len, + leak_ratio) + + # Creates permutation mask and target mask for the rest of tokens in + # current example, which are concatentation of two new segments. + perm_mask_1, target_mask_1, input_k_1, input_q_1 = _local_perm( + inputs[reuse_len:], is_masked[reuse_len:], perm_size, non_reuse_len, + leak_ratio) + + perm_mask_0 = tf.concat( + [perm_mask_0, tf.ones([reuse_len, non_reuse_len])], axis=1) + perm_mask_1 = tf.concat( + [tf.zeros([non_reuse_len, reuse_len]), perm_mask_1], axis=1) + perm_mask = tf.concat([perm_mask_0, perm_mask_1], axis=0) + target_mask = tf.concat([target_mask_0, target_mask_1], axis=0) + input_k = tf.concat([input_k_0, input_k_1], axis=0) + input_q = tf.concat([input_q_0, input_q_1], axis=0) + else: + ##### Do not use memory + assert seq_len % perm_size == 0 + # permutate the entire sequence together + perm_mask, target_mask, input_k, input_q = _local_perm( + inputs, is_masked, perm_size, seq_len, leak_ratio) + + # reshape back to fixed shape + example["perm_mask"] = tf.reshape(perm_mask, [seq_len, seq_len]) + example["input_k"] = tf.reshape(input_k, [seq_len]) + example["input_q"] = tf.reshape(input_q, [seq_len]) + + # Directly use raw inputs as the target + target = inputs + + if num_predict is not None: + indices = tf.range(seq_len, dtype=tf.int64) + bool_target_mask = tf.cast(target_mask, tf.bool) + indices = tf.boolean_mask(indices, bool_target_mask) + + ##### extra padding due to CLS/SEP introduced after prepro + actual_num_predict = tf.shape(indices)[0] + pad_len = num_predict - actual_num_predict + + ##### target_mapping + target_mapping = tf.one_hot(indices, seq_len, dtype=tf.float32) + paddings = tf.zeros([pad_len, seq_len], dtype=target_mapping.dtype) + target_mapping = tf.concat([target_mapping, paddings], axis=0) + example["target_mapping"] = tf.reshape(target_mapping, + [num_predict, seq_len]) + + ##### target + target = tf.boolean_mask(target, bool_target_mask) + paddings = tf.zeros([pad_len], dtype=target.dtype) + target = tf.concat([target, paddings], axis=0) + example["target"] = tf.reshape(target, [num_predict]) + + ##### target mask + target_mask = tf.concat( + [tf.ones([actual_num_predict], dtype=tf.float32), + tf.zeros([pad_len], dtype=tf.float32)], + axis=0) + example["target_mask"] = tf.reshape(target_mask, [num_predict]) + else: + example["target"] = tf.reshape(target, [seq_len]) + example["target_mask"] = tf.reshape(target_mask, [seq_len]) + + for key in list(example.keys()): + val = example[key] + if tf.keras.backend.is_sparse(val): + val = tf.sparse.to_dense(val) + if val.dtype == tf.int64: + val = tf.cast(val, tf.int32) + + example[key] = val + + for k, v in example.items(): + logging.info("%s: %s", k, v) + + return example + + dataset = parse_files_to_dataset( + parser=parser, + file_paths=file_names, + bsz_per_core=bsz_per_core, + sequential=reuse_len > 0, + input_pipeline_context=input_pipeline_context) + + return dataset + + +def format_filename(prefix, suffix, bsz_per_host, seq_len, reuse_len=None, + uncased=False): + """Generates input file name pattern.""" + if reuse_len is not None and reuse_len > 0: + reuse_str = "reuse-{}.".format(reuse_len) + bsz_str = "hostbsz-{}.".format(bsz_per_host) + else: + reuse_str = "" + bsz_str = "" + + if not uncased: + case_str = "" + else: + case_str = "uncased." + + file_name = "{}.seq-{}.{}{}{}{}".format( + prefix, seq_len, reuse_str, bsz_str, case_str, suffix) + + return file_name + + +def get_pretrain_input_data(batch_size, + seq_len, + strategy, + file_path, + reuse_len, + perm_size, + leak_ratio, + num_predict, + uncased, + online_masking_config, + num_hosts=1): + """Returns input dataset from input file string.""" + + # When using TPU pods, we need to clone dataset across + # workers and need to pass in function that returns the dataset rather + # than passing dataset instance itself. + use_dataset_fn = isinstance(strategy, tf.distribute.experimental.TPUStrategy) + split = "train" + bsz_per_host = int(batch_size / num_hosts) + record_glob_base = format_filename( + prefix="meta.{}.pass-*".format(split), + suffix="json*", + bsz_per_host=bsz_per_host, + seq_len=seq_len, + reuse_len=reuse_len, + uncased=uncased) + + def _get_num_batch(info): + if "num_batch" in info: + return info["num_batch"] + elif "num_example" in info: + return info["num_example"] / bsz_per_host + else: + raise ValueError("Do not have sample info.") + + if use_dataset_fn: + if batch_size % strategy.num_replicas_in_sync != 0: + raise ValueError( + "Batch size must be divisible by number of replicas : {}".format( + strategy.num_replicas_in_sync)) + + # As auto rebatching is not supported in + # `experimental_distribute_datasets_from_function()` API, which is + # required when cloning dataset to multiple workers in eager mode, + # we use per-replica batch size. + batch_size = int(batch_size / strategy.num_replicas_in_sync) + + record_info = {"num_batch": 0, "filenames": []} + + tfrecord_dirs = file_path.split(",") + logging.info("Use the following tfrecord dirs: %s", tfrecord_dirs) + + for idx, record_dir in enumerate(tfrecord_dirs): + record_glob = os.path.join(record_dir, record_glob_base) + logging.info("[%d] Record glob: %s", idx, record_glob) + + record_paths = sorted(tf.io.gfile.glob(record_glob)) + logging.info("[%d] Num of record info path: %d", idx, len(record_paths)) + + cur_record_info = {"num_batch": 0, "filenames": []} + + for record_info_path in record_paths: + with tf.io.gfile.GFile(record_info_path, "r") as fp: + info = json.load(fp) + cur_record_info["num_batch"] += int(_get_num_batch(info)) + cur_record_info["filenames"] += info["filenames"] + + # overwrite directory for `cur_record_info` + new_filenames = [] + for filename in cur_record_info["filenames"]: + basename = os.path.basename(filename) + new_filename = os.path.join(record_dir, basename) + new_filenames.append(new_filename) + cur_record_info["filenames"] = new_filenames + + logging.info("[Dir %d] Number of chosen batches: %s", idx, + cur_record_info["num_batch"]) + logging.info("[Dir %d] Number of chosen files: %s", idx, + len(cur_record_info["filenames"])) + logging.info(cur_record_info["filenames"]) + + # add `cur_record_info` to global `record_info` + record_info["num_batch"] += cur_record_info["num_batch"] + record_info["filenames"] += cur_record_info["filenames"] + + logging.info("Total number of batches: %d", record_info["num_batch"]) + logging.info("Total number of files: %d", len(record_info["filenames"])) + logging.info(record_info["filenames"]) + + def _dataset_fn(ctx=None): + """Function that can create a pretrain dataset.""" + + train_dataset = create_pretrain_dataset( + file_names=record_info["filenames"], + bsz_per_core=batch_size, + seq_len=seq_len, + reuse_len=reuse_len, + perm_size=perm_size, + leak_ratio=leak_ratio, + online_masking_config=online_masking_config, + num_predict=num_predict, + input_pipeline_context=ctx) + return train_dataset + + return _dataset_fn if use_dataset_fn else _dataset_fn() + + +def parse_files_to_dataset(parser, + file_paths, + bsz_per_core, + sequential, + input_pipeline_context=None): + """Creates the dataset given file paths.""" + + dataset = tf.data.Dataset.from_tensor_slices(file_paths) + + # Note: we cannot perform sample-level shuffle here because this will violate + # the consecutive requirement of data stream. + + if input_pipeline_context and input_pipeline_context.num_input_pipelines > 1: + dataset = dataset.shard(input_pipeline_context.num_input_pipelines, + input_pipeline_context.input_pipeline_id) + # file-level shuffle + if len(file_paths) > 1: + dataset = dataset.shuffle(len(file_paths)) + + if sequential: + # Note: cannot perform sample-level shuffle here because this will violate + # the consecutive requirement of data stream. + dataset = tf.data.TFRecordDataset(dataset) + else: + # `cycle_length` is the number of parallel files that get read. + cycle_length = min(8, len(file_paths)) + logging.info("Interleave %d files", cycle_length) + + # `sloppy` mode means that the interleaving is not exact. This adds + # even more randomness to the training pipeline. + dataset = dataset.apply( + tf.data.experimental.parallel_interleave( + tf.data.TFRecordDataset, + sloppy=True, + cycle_length=cycle_length)) + buffer_size = 2048 + logging.info("Perform sample-level shuffle with size %d", buffer_size) + dataset = dataset.shuffle(buffer_size=buffer_size) + + # (zihang): since we are doing online preprocessing, the parsed result of + # the same input at each time will be different. Thus, cache processed data + # is not helpful. It will use a lot of memory and lead to contrainer OOM. + # So, change to cache non-parsed raw data instead. + dataset = dataset.cache().map(parser).repeat() + dataset = dataset.batch(bsz_per_core, drop_remainder=True) + dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE) + + return dataset + + +def _local_perm(inputs, is_masked, perm_size, seq_len, leak_ratio): + """Samples a permutation of the factorization order. + + Creates perm_mask and target_mask accordingly. + + Args: + inputs: int64 Tensor in shape [seq_len], input ids. + is_masked: bool Tensor in shape [seq_len]. True means being selected for + partial prediction. + perm_size: the length of longest permutation. Could be set to be reuse_len. + Should not be larger than reuse_len or there will be data leaks. + seq_len: int, sequence length. + leak_ratio: float, percent of masked tokens that are leaked. + + Returns: + perm_mask: float32 Tensor in shape [seq_len, seq_len] consisted of 0 and 1. + If perm_mask[i][j] == 1, it means the ith token (in original order) cannot + attend to the jth token + (in original order). This case will happen only when the ith token's + permutated position <= the jth token's permutated position, + and the jth token is masked or is func token. If perm_mask[i][j] == 0, it + means the ith token (in original order) can attend to the jth token + (in original order). Note that non-masked tokens can be attended by all + other tokens, which is different from the description in original paper. + target_mask: float32 Tensor in shape [seq_len] consisted of 0 and 1. If + target_mask[i] == 1, + the ith token needs to be predicted and mask will be used as input. This + token will count for loss. + If target_mask[i] == 0, token (or [SEP], [CLS]) will be used as input. This + token will not count for loss. + inputs_k: int64 Tensor in shape [seq_len], input ids. + inputs_q: float32 Tensor in shape [seq_len], the same as target_mask. + + """ + + # Generate permutation indices + index = tf.range(seq_len, dtype=tf.int64) + index = tf.transpose(tf.reshape(index, [-1, perm_size])) + index = tf.random.shuffle(index) + index = tf.reshape(tf.transpose(index), [-1]) + + # non-functional tokens + non_func_tokens = tf.logical_not(tf.logical_or( + tf.equal(inputs, SEP_ID), + tf.equal(inputs, CLS_ID))) + masked_tokens = tf.logical_and(is_masked, non_func_tokens) + non_masked_or_func_tokens = tf.logical_not(masked_tokens) + + smallest_index = -2 * tf.ones([seq_len], dtype=tf.int64) + + # Similar to BERT, randomly leak some masked tokens + if leak_ratio > 0: + leak_tokens = tf.logical_and( + masked_tokens, + tf.random.uniform([seq_len], maxval=1.0) < leak_ratio) + can_attend_self = tf.logical_or(non_masked_or_func_tokens, leak_tokens) + else: + can_attend_self = non_masked_or_func_tokens + to_index = tf.where(can_attend_self, smallest_index, index) + from_index = tf.where(can_attend_self, to_index + 1, to_index) + + # For masked tokens, can attend if i > j + # For context tokens, always can attend each other + can_attend = from_index[:, None] > to_index[None, :] + + # In modeling, 1 indicates cannot attend. Hence, reverse the value here. + perm_mask = 1.0 - tf.cast(can_attend, tf.float32) + + # Only masked tokens are included in the loss + target_mask = tf.cast(masked_tokens, tf.float32) + + # construct inputs_k + inputs_k = inputs + + # construct inputs_q + inputs_q = masked_tokens + + return perm_mask, target_mask, inputs_k, inputs_q diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/optimization.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/optimization.py new file mode 100644 index 0000000..0d90316 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/optimization.py @@ -0,0 +1,102 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functions and classes related to optimization (weight updates).""" +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +from absl import logging +import tensorflow as tf +from official.nlp import optimization + + +class WarmUp(tf.keras.optimizers.schedules.LearningRateSchedule): + """Applys a warmup schedule on a given learning rate decay schedule.""" + + def __init__(self, + initial_learning_rate, + decay_schedule_fn, + warmup_steps, + power=1.0, + name=None): + super(WarmUp, self).__init__() + self.initial_learning_rate = initial_learning_rate + self.warmup_steps = warmup_steps + self.power = power + self.decay_schedule_fn = decay_schedule_fn + self.name = name + + def __call__(self, step): + with tf.name_scope(self.name or "WarmUp") as name: + # Implements polynomial warmup. i.e., if global_step < warmup_steps, the + # learning rate will be `global_step/num_warmup_steps * init_lr`. + global_step_float = tf.cast(step, tf.float32) + warmup_steps_float = tf.cast(self.warmup_steps, tf.float32) + warmup_percent_done = global_step_float / warmup_steps_float + warmup_learning_rate = ( + self.initial_learning_rate * + tf.math.pow(warmup_percent_done, self.power)) + return tf.cond( + global_step_float < warmup_steps_float, + lambda: warmup_learning_rate, + lambda: self.decay_schedule_fn(step - self.warmup_steps), + name=name) + + def get_config(self): + return { + "initial_learning_rate": self.initial_learning_rate, + "decay_schedule_fn": self.decay_schedule_fn, + "warmup_steps": self.warmup_steps, + "power": self.power, + "name": self.name + } + + +def create_optimizer(init_lr, + num_train_steps, + num_warmup_steps, + min_lr_ratio=0.0, + adam_epsilon=1e-8, + weight_decay_rate=0.0): + """Creates an optimizer with learning rate schedule.""" + # Implements linear decay of the learning rate. + learning_rate_fn = tf.keras.optimizers.schedules.PolynomialDecay( + initial_learning_rate=init_lr, + decay_steps=num_train_steps - num_warmup_steps, + end_learning_rate=init_lr * min_lr_ratio) + if num_warmup_steps: + learning_rate_fn = WarmUp( + initial_learning_rate=init_lr, + decay_schedule_fn=learning_rate_fn, + warmup_steps=num_warmup_steps) + if weight_decay_rate > 0.0: + logging.info( + "Using AdamWeightDecay with adam_epsilon=%.9f weight_decay_rate=%.3f", + adam_epsilon, weight_decay_rate) + optimizer = optimization.AdamWeightDecay( + learning_rate=learning_rate_fn, + weight_decay_rate=weight_decay_rate, + beta_1=0.9, + beta_2=0.999, + epsilon=adam_epsilon, + exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"], + include_in_weight_decay=["r_s_bias", "r_r_bias", "r_w_bias"]) + else: + logging.info("Using Adam with adam_epsilon=%.9f", (adam_epsilon)) + optimizer = tf.keras.optimizers.Adam( + learning_rate=learning_rate_fn, epsilon=adam_epsilon) + + return optimizer, learning_rate_fn diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/preprocess_classification_data.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/preprocess_classification_data.py new file mode 100644 index 0000000..9b34ffe --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/preprocess_classification_data.py @@ -0,0 +1,457 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Script to pre-process classification data into tfrecords.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import csv +import os + +from absl import app +from absl import flags +from absl import logging +import numpy as np +import tensorflow as tf + +import sentencepiece as spm +from official.nlp.xlnet import classifier_utils +from official.nlp.xlnet import preprocess_utils + + +flags.DEFINE_bool( + "overwrite_data", + default=False, + help="If False, will use cached data if available.") +flags.DEFINE_string("output_dir", default="", help="Output dir for TF records.") +flags.DEFINE_string( + "spiece_model_file", default="", help="Sentence Piece model path.") +flags.DEFINE_string("data_dir", default="", help="Directory for input data.") + +# task specific +flags.DEFINE_string("eval_split", default="dev", help="could be dev or test") +flags.DEFINE_string("task_name", default=None, help="Task name") +flags.DEFINE_integer( + "eval_batch_size", default=64, help="batch size for evaluation") +flags.DEFINE_integer("max_seq_length", default=128, help="Max sequence length") +flags.DEFINE_integer( + "num_passes", + default=1, + help="Num passes for processing training data. " + "This is use to batch data without loss for TPUs.") +flags.DEFINE_bool("uncased", default=False, help="Use uncased.") +flags.DEFINE_bool( + "is_regression", default=False, help="Whether it's a regression task.") +flags.DEFINE_bool( + "use_bert_format", + default=False, + help="Whether to use BERT format to arrange input data.") + +FLAGS = flags.FLAGS + + +class InputExample(object): + """A single training/test example for simple sequence classification.""" + + def __init__(self, guid, text_a, text_b=None, label=None): + """Constructs a InputExample. + + Args: + guid: Unique id for the example. + text_a: string. The untokenized text of the first sequence. For single + sequence tasks, only this sequence must be specified. + text_b: (Optional) string. The untokenized text of the second sequence. + Only must be specified for sequence pair tasks. + label: (Optional) string. The label of the example. This should be + specified for train and dev examples, but not for test examples. + """ + self.guid = guid + self.text_a = text_a + self.text_b = text_b + self.label = label + + +class DataProcessor(object): + """Base class for data converters for sequence classification data sets.""" + + def get_train_examples(self, data_dir): + """Gets a collection of `InputExample`s for the train set.""" + raise NotImplementedError() + + def get_dev_examples(self, data_dir): + """Gets a collection of `InputExample`s for the dev set.""" + raise NotImplementedError() + + def get_test_examples(self, data_dir): + """Gets a collection of `InputExample`s for prediction.""" + raise NotImplementedError() + + def get_labels(self): + """Gets the list of labels for this data set.""" + raise NotImplementedError() + + @classmethod + def _read_tsv(cls, input_file, quotechar=None): + """Reads a tab separated value file.""" + with tf.io.gfile.GFile(input_file, "r") as f: + reader = csv.reader(f, delimiter="\t", quotechar=quotechar) + lines = [] + for line in reader: + # pylint: disable=g-explicit-length-test + if len(line) == 0: + continue + lines.append(line) + return lines + + +class GLUEProcessor(DataProcessor): + """GLUEProcessor.""" + + def __init__(self): + self.train_file = "train.tsv" + self.dev_file = "dev.tsv" + self.test_file = "test.tsv" + self.label_column = None + self.text_a_column = None + self.text_b_column = None + self.contains_header = True + self.test_text_a_column = None + self.test_text_b_column = None + self.test_contains_header = True + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, self.train_file)), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, self.dev_file)), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + if self.test_text_a_column is None: + self.test_text_a_column = self.text_a_column + if self.test_text_b_column is None: + self.test_text_b_column = self.text_b_column + + return self._create_examples( + self._read_tsv(os.path.join(data_dir, self.test_file)), "test") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0 and self.contains_header and set_type != "test": + continue + if i == 0 and self.test_contains_header and set_type == "test": + continue + guid = "%s-%s" % (set_type, i) + + a_column = ( + self.text_a_column if set_type != "test" else self.test_text_a_column) + b_column = ( + self.text_b_column if set_type != "test" else self.test_text_b_column) + + # there are some incomplete lines in QNLI + if len(line) <= a_column: + logging.warning("Incomplete line, ignored.") + continue + text_a = line[a_column] + + if b_column is not None: + if len(line) <= b_column: + logging.warning("Incomplete line, ignored.") + continue + text_b = line[b_column] + else: + text_b = None + + if set_type == "test": + label = self.get_labels()[0] + else: + if len(line) <= self.label_column: + logging.warning("Incomplete line, ignored.") + continue + label = line[self.label_column] + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class Yelp5Processor(DataProcessor): + """Yelp5Processor.""" + + def get_train_examples(self, data_dir): + return self._create_examples(os.path.join(data_dir, "train.csv")) + + def get_dev_examples(self, data_dir): + return self._create_examples(os.path.join(data_dir, "test.csv")) + + def get_labels(self): + """See base class.""" + return ["1", "2", "3", "4", "5"] + + def _create_examples(self, input_file): + """Creates examples for the training and dev sets.""" + examples = [] + with tf.io.gfile.GFile(input_file) as f: + reader = csv.reader(f) + for i, line in enumerate(reader): + + label = line[0] + text_a = line[1].replace('""', '"').replace('\\"', '"') + examples.append( + InputExample(guid=str(i), text_a=text_a, text_b=None, label=label)) + return examples + + +class ImdbProcessor(DataProcessor): + """ImdbProcessor.""" + + def get_labels(self): + return ["neg", "pos"] + + def get_train_examples(self, data_dir): + return self._create_examples(os.path.join(data_dir, "train")) + + def get_dev_examples(self, data_dir): + return self._create_examples(os.path.join(data_dir, "test")) + + def _create_examples(self, data_dir): + """Creates examples.""" + examples = [] + for label in ["neg", "pos"]: + cur_dir = os.path.join(data_dir, label) + for filename in tf.io.gfile.listdir(cur_dir): + if not filename.endswith("txt"): + continue + + if len(examples) % 1000 == 0: + logging.info("Loading dev example %d", len(examples)) + + path = os.path.join(cur_dir, filename) + with tf.io.gfile.GFile(path) as f: + text = f.read().strip().replace("
", " ") + examples.append( + InputExample( + guid="unused_id", text_a=text, text_b=None, label=label)) + return examples + + +class MnliMatchedProcessor(GLUEProcessor): + """MnliMatchedProcessor.""" + + def __init__(self): + super(MnliMatchedProcessor, self).__init__() + self.dev_file = "dev_matched.tsv" + self.test_file = "test_matched.tsv" + self.label_column = -1 + self.text_a_column = 8 + self.text_b_column = 9 + + def get_labels(self): + return ["contradiction", "entailment", "neutral"] + + +class MnliMismatchedProcessor(MnliMatchedProcessor): + + def __init__(self): + super(MnliMismatchedProcessor, self).__init__() + self.dev_file = "dev_mismatched.tsv" + self.test_file = "test_mismatched.tsv" + + +class StsbProcessor(GLUEProcessor): + """StsbProcessor.""" + + def __init__(self): + super(StsbProcessor, self).__init__() + self.label_column = 9 + self.text_a_column = 7 + self.text_b_column = 8 + + def get_labels(self): + return [0.0] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0 and self.contains_header and set_type != "test": + continue + if i == 0 and self.test_contains_header and set_type == "test": + continue + guid = "%s-%s" % (set_type, i) + + a_column = ( + self.text_a_column if set_type != "test" else self.test_text_a_column) + b_column = ( + self.text_b_column if set_type != "test" else self.test_text_b_column) + + # there are some incomplete lines in QNLI + if len(line) <= a_column: + logging.warning("Incomplete line, ignored.") + continue + text_a = line[a_column] + + if b_column is not None: + if len(line) <= b_column: + logging.warning("Incomplete line, ignored.") + continue + text_b = line[b_column] + else: + text_b = None + + if set_type == "test": + label = self.get_labels()[0] + else: + if len(line) <= self.label_column: + logging.warning("Incomplete line, ignored.") + continue + label = float(line[self.label_column]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + + return examples + + +def file_based_convert_examples_to_features(examples, + label_list, + max_seq_length, + tokenize_fn, + output_file, + num_passes=1): + """Convert a set of `InputExample`s to a TFRecord file.""" + + # do not create duplicated records + if tf.io.gfile.exists(output_file) and not FLAGS.overwrite_data: + logging.info("Do not overwrite tfrecord %s exists.", output_file) + return + + logging.info("Create new tfrecord %s.", output_file) + + writer = tf.io.TFRecordWriter(output_file) + + examples *= num_passes + + for (ex_index, example) in enumerate(examples): + if ex_index % 10000 == 0: + logging.info("Writing example %d of %d", ex_index, len(examples)) + + feature = classifier_utils.convert_single_example(ex_index, example, + label_list, + max_seq_length, + tokenize_fn, + FLAGS.use_bert_format) + + def create_int_feature(values): + f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) + return f + + def create_float_feature(values): + f = tf.train.Feature(float_list=tf.train.FloatList(value=list(values))) + return f + + features = collections.OrderedDict() + features["input_ids"] = create_int_feature(feature.input_ids) + features["input_mask"] = create_float_feature(feature.input_mask) + features["segment_ids"] = create_int_feature(feature.segment_ids) + if label_list is not None: + features["label_ids"] = create_int_feature([feature.label_id]) + else: + features["label_ids"] = create_float_feature([float(feature.label_id)]) + features["is_real_example"] = create_int_feature( + [int(feature.is_real_example)]) + + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + writer.write(tf_example.SerializeToString()) + writer.close() + + +def main(_): + logging.set_verbosity(logging.INFO) + processors = { + "mnli_matched": MnliMatchedProcessor, + "mnli_mismatched": MnliMismatchedProcessor, + "sts-b": StsbProcessor, + "imdb": ImdbProcessor, + "yelp5": Yelp5Processor + } + + task_name = FLAGS.task_name.lower() + + if task_name not in processors: + raise ValueError("Task not found: %s" % (task_name)) + + processor = processors[task_name]() + label_list = processor.get_labels() if not FLAGS.is_regression else None + + sp = spm.SentencePieceProcessor() + sp.Load(FLAGS.spiece_model_file) + + def tokenize_fn(text): + text = preprocess_utils.preprocess_text(text, lower=FLAGS.uncased) + return preprocess_utils.encode_ids(sp, text) + + spm_basename = os.path.basename(FLAGS.spiece_model_file) + + train_file_base = "{}.len-{}.train.tf_record".format(spm_basename, + FLAGS.max_seq_length) + train_file = os.path.join(FLAGS.output_dir, train_file_base) + logging.info("Use tfrecord file %s", train_file) + + train_examples = processor.get_train_examples(FLAGS.data_dir) + np.random.shuffle(train_examples) + logging.info("Num of train samples: %d", len(train_examples)) + + file_based_convert_examples_to_features(train_examples, label_list, + FLAGS.max_seq_length, tokenize_fn, + train_file, FLAGS.num_passes) + if FLAGS.eval_split == "dev": + eval_examples = processor.get_dev_examples(FLAGS.data_dir) + else: + eval_examples = processor.get_test_examples(FLAGS.data_dir) + + logging.info("Num of eval samples: %d", len(eval_examples)) + + # TPU requires a fixed batch size for all batches, therefore the number + # of examples must be a multiple of the batch size, or else examples + # will get dropped. So we pad with fake examples which are ignored + # later on. These do NOT count towards the metric (all tf.metrics + # support a per-instance weight, and these get a weight of 0.0). + # + # Modified in XL: We also adopt the same mechanism for GPUs. + while len(eval_examples) % FLAGS.eval_batch_size != 0: + eval_examples.append(classifier_utils.PaddingInputExample()) + + eval_file_base = "{}.len-{}.{}.eval.tf_record".format(spm_basename, + FLAGS.max_seq_length, + FLAGS.eval_split) + eval_file = os.path.join(FLAGS.output_dir, eval_file_base) + + file_based_convert_examples_to_features(eval_examples, label_list, + FLAGS.max_seq_length, tokenize_fn, + eval_file) + + +if __name__ == "__main__": + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/preprocess_pretrain_data.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/preprocess_pretrain_data.py new file mode 100644 index 0000000..9bf5367 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/preprocess_pretrain_data.py @@ -0,0 +1,998 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Script to pre-process pre-training data into tfrecords.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import os +import random + +from absl import app +from absl import flags +import absl.logging as _logging # pylint: disable=unused-import + +import numpy as np + + +import tensorflow.google as tf +from official.nlp.xlnet import preprocess_utils +import sentencepiece as spm + + +special_symbols = { + "" : 0, + "" : 1, + "" : 2, + "" : 3, + "" : 4, + "" : 5, + "" : 6, + "" : 7, + "" : 8, +} + +VOCAB_SIZE = 32000 +UNK_ID = special_symbols[""] +CLS_ID = special_symbols[""] +SEP_ID = special_symbols[""] +MASK_ID = special_symbols[""] +EOD_ID = special_symbols[""] + + +def _int64_feature(values): + return tf.train.Feature(int64_list=tf.train.Int64List(value=values)) + + +def _float_feature(values): + return tf.train.Feature(float_list=tf.train.FloatList(value=values)) + + +def format_filename(prefix, bsz_per_host, seq_len, bi_data, suffix, + mask_alpha=5, mask_beta=1, reuse_len=None, uncased=False, + fixed_num_predict=None): + """docs.""" + if reuse_len is None: + reuse_len_str = "" + else: + reuse_len_str = "reuse-{}.".format(reuse_len) + if not uncased: + uncased_str = "" + else: + uncased_str = "uncased." + if bi_data: + bi_data_str = "bi" + else: + bi_data_str = "uni" + if fixed_num_predict is not None: + fnp_str = "fnp-{}.".format(fixed_num_predict) + else: + fnp_str = "" + + file_name = "{}.bsz-{}.seqlen-{}.{}{}{}.alpha-{}.beta-{}.{}{}".format( + prefix, bsz_per_host, seq_len, reuse_len_str, uncased_str, bi_data_str, + mask_alpha, mask_beta, fnp_str, suffix) + + return file_name + + +def _create_data(idx, input_paths): + # Load sentence-piece model + sp = spm.SentencePieceProcessor() + sp.Load(FLAGS.sp_path) + + input_shards = [] + total_line_cnt = 0 + for input_path in input_paths: + input_data, sent_ids = [], [] + sent_id, line_cnt = True, 0 + tf.logging.info("Processing %s", input_path) + for line in tf.gfile.Open(input_path): + if line_cnt % 100000 == 0: + tf.logging.info("Loading line %d", line_cnt) + line_cnt += 1 + + if not line.strip(): + if FLAGS.use_eod: + sent_id = not sent_id + cur_sent = [EOD_ID] + else: + continue + else: + if FLAGS.from_raw_text: + cur_sent = preprocess_utils.preprocess_text( + line.strip(), lower=FLAGS.uncased) + cur_sent = preprocess_utils.encode_ids(sp, cur_sent) + else: + cur_sent = list(map(int, line.strip().split())) + + input_data.extend(cur_sent) + sent_ids.extend([sent_id] * len(cur_sent)) + sent_id = not sent_id + + tf.logging.info("Finish with line %d", line_cnt) + if line_cnt == 0: + continue + + input_data = np.array(input_data, dtype=np.int64) + sent_ids = np.array(sent_ids, dtype=np.bool) + + total_line_cnt += line_cnt + input_shards.append((input_data, sent_ids)) + + tf.logging.info("[Task %d] Total number line: %d", idx, total_line_cnt) + + tfrecord_dir = os.path.join(FLAGS.save_dir, "tfrecords") + + filenames, num_batch = [], 0 + + # Randomly shuffle input shards (with a fixed but distinct random seed) + np.random.seed(100 * FLAGS.task + FLAGS.pass_id) + + perm_indices = np.random.permutation(len(input_shards)) + tf.logging.info("Using perm indices %s for pass %d", + perm_indices.tolist(), FLAGS.pass_id) + + input_data_list, sent_ids_list = [], [] + prev_sent_id = None + for perm_idx in perm_indices: + input_data, sent_ids = input_shards[perm_idx] + # make sure the `send_ids[0] == not prev_sent_id` + if prev_sent_id is not None and sent_ids[0] == prev_sent_id: + sent_ids = np.logical_not(sent_ids) + + # append to temporary list + input_data_list.append(input_data) + sent_ids_list.append(sent_ids) + + # update `prev_sent_id` + prev_sent_id = sent_ids[-1] + + input_data = np.concatenate(input_data_list) + sent_ids = np.concatenate(sent_ids_list) + + file_name, cur_num_batch = create_tfrecords( + save_dir=tfrecord_dir, + basename="{}-{}-{}".format(FLAGS.split, idx, FLAGS.pass_id), + data=[input_data, sent_ids], + bsz_per_host=FLAGS.bsz_per_host, + seq_len=FLAGS.seq_len, + bi_data=FLAGS.bi_data, + sp=sp, + ) + + filenames.append(file_name) + num_batch += cur_num_batch + + record_info = { + "filenames": filenames, + "num_batch": num_batch + } + + return record_info + + +def create_data(_): + # Validate FLAGS + assert FLAGS.bsz_per_host % FLAGS.num_core_per_host == 0 + if not FLAGS.use_tpu: + FLAGS.num_core_per_host = 1 # forced to be one + + # Make workdirs + if not tf.gfile.Exists(FLAGS.save_dir): + tf.gfile.MakeDirs(FLAGS.save_dir) + + tfrecord_dir = os.path.join(FLAGS.save_dir, "tfrecords") + if not tf.gfile.Exists(tfrecord_dir): + tf.gfile.MakeDirs(tfrecord_dir) + + # Create and dump corpus_info from task 0 + if FLAGS.task == 0 and FLAGS.pass_id == 0: + corpus_info = { + "vocab_size": VOCAB_SIZE, + "bsz_per_host": FLAGS.bsz_per_host, + "num_core_per_host": FLAGS.num_core_per_host, + "seq_len": FLAGS.seq_len, + "reuse_len": FLAGS.reuse_len, + "uncased": FLAGS.uncased, + "bi_data": FLAGS.bi_data, + "mask_alpha": FLAGS.mask_alpha, + "mask_beta": FLAGS.mask_beta, + "num_predict": FLAGS.num_predict, + "use_eod": FLAGS.use_eod, + "sp_path": FLAGS.sp_path, + "input_glob": FLAGS.input_glob, + } + corpus_info_path = os.path.join(FLAGS.save_dir, "corpus_info.json") + with tf.gfile.Open(corpus_info_path, "w") as fp: + json.dump(corpus_info, fp) + + # Interleavely split the work into FLAGS.num_task splits + file_paths = sorted(tf.gfile.Glob(FLAGS.input_glob)) + tf.logging.info("Use glob: %s", FLAGS.input_glob) + tf.logging.info("Find %d files: %s", len(file_paths), file_paths) + + task_file_paths = file_paths[FLAGS.task::FLAGS.num_task] + if not task_file_paths: + tf.logging.info("Exit: task %d has no file to process.", FLAGS.task) + return + + tf.logging.info("Task %d process %d files: %s", + FLAGS.task, len(task_file_paths), task_file_paths) + record_info = _create_data(FLAGS.task, task_file_paths) + + record_prefix = "record_info-{}-{}-{}".format( + FLAGS.split, FLAGS.task, FLAGS.pass_id) + record_name = format_filename( + prefix=record_prefix, + bsz_per_host=FLAGS.bsz_per_host, + seq_len=FLAGS.seq_len, + mask_alpha=FLAGS.mask_alpha, + mask_beta=FLAGS.mask_beta, + reuse_len=FLAGS.reuse_len, + bi_data=FLAGS.bi_data, + suffix="json", + uncased=FLAGS.uncased, + fixed_num_predict=FLAGS.num_predict) + record_info_path = os.path.join(tfrecord_dir, record_name) + + with tf.gfile.Open(record_info_path, "w") as fp: + json.dump(record_info, fp) + + +def batchify(data, bsz_per_host, sent_ids=None): + num_step = len(data) // bsz_per_host + data = data[:bsz_per_host * num_step] + data = data.reshape(bsz_per_host, num_step) + if sent_ids is not None: + sent_ids = sent_ids[:bsz_per_host * num_step] + sent_ids = sent_ids.reshape(bsz_per_host, num_step) + + if sent_ids is not None: + return data, sent_ids + return data + + +def _split_a_and_b(data, sent_ids, begin_idx, tot_len, extend_target=False): + """Split two segments from `data` starting from the index `begin_idx`.""" + + data_len = data.shape[0] + if begin_idx + tot_len >= data_len: + tf.logging.info("[_split_a_and_b] returns None: " + "begin_idx %d + tot_len %d >= data_len %d", + begin_idx, tot_len, data_len) + return None + + end_idx = begin_idx + 1 + cut_points = [] + while end_idx < data_len: + if sent_ids[end_idx] != sent_ids[end_idx - 1]: + if end_idx - begin_idx >= tot_len: break + cut_points.append(end_idx) + end_idx += 1 + + a_begin = begin_idx + if len(cut_points) == 0 or random.random() < 0.5: + label = 0 + if len(cut_points) == 0: + a_end = end_idx + else: + a_end = random.choice(cut_points) + + b_len = max(1, tot_len - (a_end - a_begin)) + # (zihangd): `data_len - 1` to account for extend_target + b_begin = random.randint(0, data_len - 1 - b_len) + b_end = b_begin + b_len + while b_begin > 0 and sent_ids[b_begin - 1] == sent_ids[b_begin]: + b_begin -= 1 + # (zihangd): `data_len - 1` to account for extend_target + while b_end < data_len - 1 and sent_ids[b_end - 1] == sent_ids[b_end]: + b_end += 1 + + new_begin = a_end + else: + label = 1 + a_end = random.choice(cut_points) + b_begin = a_end + b_end = end_idx + + new_begin = b_end + + while a_end - a_begin + b_end - b_begin > tot_len: + if a_end - a_begin > b_end - b_begin: + # delete the right side only for the LM objective + a_end -= 1 + else: + b_end -= 1 + + ret = [data[a_begin: a_end], data[b_begin: b_end], label, new_begin] + + if extend_target: + if a_end >= data_len or b_end >= data_len: + tf.logging.info("[_split_a_and_b] returns None: " + "a_end %d or b_end %d >= data_len %d", + a_end, b_end, data_len) + return None + a_target = data[a_begin + 1: a_end + 1] + b_target = data[b_begin: b_end + 1] + ret.extend([a_target, b_target]) + + return ret + + +def _is_start_piece(piece): + special_pieces = set(list('!"#$%&\"()*+,-./:;?@[\\]^_`{|}~')) + if (piece.startswith("▁") or piece.startswith("<") + or piece in special_pieces): + return True + else: + return False + + +def _sample_mask(sp, seg, reverse=False, max_gram=5, goal_num_predict=None): + """Sample `goal_num_predict` tokens for partial prediction. + About `mask_beta` tokens are chosen in a context of `mask_alpha` tokens.""" + + seg_len = len(seg) + mask = np.array([False] * seg_len, dtype=np.bool) + + num_predict = 0 + + ngrams = np.arange(1, max_gram + 1, dtype=np.int64) + pvals = 1. / np.arange(1, max_gram + 1) + pvals /= pvals.sum(keepdims=True) + + if reverse: + seg = np.flip(seg, 0) + + cur_len = 0 + while cur_len < seg_len: + if goal_num_predict is not None and num_predict >= goal_num_predict: break + + n = np.random.choice(ngrams, p=pvals) + if goal_num_predict is not None: + n = min(n, goal_num_predict - num_predict) + ctx_size = (n * FLAGS.mask_alpha) // FLAGS.mask_beta + l_ctx = np.random.choice(ctx_size) + r_ctx = ctx_size - l_ctx + + # Find the start position of a complete token + beg = cur_len + l_ctx + while beg < seg_len and not _is_start_piece(sp.IdToPiece(seg[beg].item())): + beg += 1 + if beg >= seg_len: + break + + # Find the end position of the n-gram (start pos of the n+1-th gram) + end = beg + 1 + cnt_ngram = 1 + while end < seg_len: + cnt_ngram += 1 + if cnt_ngram > n: + break + end += 1 + if end >= seg_len: + break + + # Update + mask[beg:end] = True + num_predict += end - beg + + cur_len = end + r_ctx + + while goal_num_predict is not None and num_predict < goal_num_predict: + i = np.random.randint(seg_len) + if not mask[i]: + mask[i] = True + num_predict += 1 + + if reverse: + mask = np.flip(mask, 0) + + return mask + + +def _sample_mask_ngram(sp, seg, reverse=False, max_gram=5, + goal_num_predict=None): + """Sample `goal_num_predict` tokens for partial prediction. + About `mask_beta` tokens are chosen in a context of `mask_alpha` tokens.""" + + seg_len = len(seg) + mask = np.array([False] * seg_len, dtype=np.bool) + + num_predict = 0 + + ngrams = np.arange(1, max_gram + 1, dtype=np.int64) + pvals = 1. / np.arange(1, max_gram + 1) + pvals /= pvals.sum(keepdims=True) + + if reverse: + seg = np.flip(seg, 0) + + cur_len = 0 + while cur_len < seg_len: + if goal_num_predict is not None and num_predict >= goal_num_predict: break + + n = np.random.choice(ngrams, p=pvals) + if goal_num_predict is not None: + n = min(n, goal_num_predict - num_predict) + ctx_size = (n * FLAGS.mask_alpha) // FLAGS.mask_beta + l_ctx = np.random.choice(ctx_size) + r_ctx = ctx_size - l_ctx + + # Find the start position of a complete token + beg = cur_len + l_ctx + while beg < seg_len and not _is_start_piece(sp.IdToPiece(seg[beg].item())): + beg += 1 + if beg >= seg_len: + break + + # Find the end position of the n-gram (start pos of the n+1-th gram) + end = beg + cnt_ngram = 0 + while end < seg_len: + if _is_start_piece(sp.IdToPiece(seg[end].item())): + cnt_ngram += 1 + if cnt_ngram > n: + break + + # select current piece + mask[end] = True + + # update the end pointer and increment num_predict + end += 1 + num_predict += 1 + + if goal_num_predict is not None and num_predict >= goal_num_predict: + break + + cur_len = end + r_ctx + + while goal_num_predict is not None and num_predict < goal_num_predict: + i = np.random.randint(seg_len) + if not mask[i]: + mask[i] = True + num_predict += 1 + + if reverse: + mask = np.flip(mask, 0) + + return mask + + +def create_tfrecords(save_dir, basename, data, bsz_per_host, seq_len, + bi_data, sp): + data, sent_ids = data[0], data[1] + + num_core = FLAGS.num_core_per_host + bsz_per_core = bsz_per_host // num_core + + if bi_data: + assert bsz_per_host % (2 * FLAGS.num_core_per_host) == 0 + fwd_data, fwd_sent_ids = batchify(data, bsz_per_host // 2, sent_ids) + + fwd_data = fwd_data.reshape(num_core, 1, bsz_per_core // 2, -1) + fwd_sent_ids = fwd_sent_ids.reshape(num_core, 1, bsz_per_core // 2, -1) + + bwd_data = fwd_data[:, :, :, ::-1] + bwd_sent_ids = fwd_sent_ids[:, :, :, ::-1] + + data = np.concatenate( + [fwd_data, bwd_data], 1).reshape(bsz_per_host, -1) + sent_ids = np.concatenate( + [fwd_sent_ids, bwd_sent_ids], 1).reshape(bsz_per_host, -1) + else: + data, sent_ids = batchify(data, bsz_per_host, sent_ids) + + tf.logging.info("Raw data shape %s.", data.shape) + + file_name = format_filename( + prefix=basename, + bsz_per_host=bsz_per_host, + seq_len=seq_len, + bi_data=bi_data, + suffix="tfrecords", + mask_alpha=FLAGS.mask_alpha, + mask_beta=FLAGS.mask_beta, + reuse_len=FLAGS.reuse_len, + uncased=FLAGS.uncased, + fixed_num_predict=FLAGS.num_predict + ) + save_path = os.path.join(save_dir, file_name) + record_writer = tf.python_io.TFRecordWriter(save_path) + tf.logging.info("Start writing %s.", save_path) + + num_batch = 0 + reuse_len = FLAGS.reuse_len + + # [sep] x 2 + [cls] + assert reuse_len < seq_len - 3 + + data_len = data.shape[1] + sep_array = np.array([SEP_ID], dtype=np.int64) + cls_array = np.array([CLS_ID], dtype=np.int64) + + i = 0 + while i + seq_len <= data_len: + if num_batch % 500 == 0: + tf.logging.info("Processing batch %d", num_batch) + + all_ok = True + features = [] + for idx in range(bsz_per_host): + inp = data[idx, i: i + reuse_len] + tgt = data[idx, i + 1: i + reuse_len + 1] + + results = _split_a_and_b( + data[idx], + sent_ids[idx], + begin_idx=i + reuse_len, + tot_len=seq_len - reuse_len - 3, + extend_target=True) + if results is None: + tf.logging.info("Break out with seq idx %d", i) + all_ok = False + break + + # unpack the results + (a_data, b_data, label, _, a_target, b_target) = tuple(results) + + # sample ngram spans to predict + reverse = bi_data and (idx // (bsz_per_core // 2)) % 2 == 1 + if FLAGS.num_predict is None: + num_predict_0 = num_predict_1 = None + else: + num_predict_1 = FLAGS.num_predict // 2 + num_predict_0 = FLAGS.num_predict - num_predict_1 + mask_0 = _sample_mask(sp, inp, reverse=reverse, + goal_num_predict=num_predict_0) + mask_1 = _sample_mask(sp, np.concatenate([a_data, sep_array, b_data, + sep_array, cls_array]), + reverse=reverse, goal_num_predict=num_predict_1) + + # concatenate data + cat_data = np.concatenate([inp, a_data, sep_array, b_data, + sep_array, cls_array]) + seg_id = ([0] * (reuse_len + a_data.shape[0]) + [0] + + [1] * b_data.shape[0] + [1] + [2]) + assert cat_data.shape[0] == seq_len + assert mask_0.shape[0] == seq_len // 2 + assert mask_1.shape[0] == seq_len // 2 + + # the last two CLS's are not used, just for padding purposes + tgt = np.concatenate([tgt, a_target, b_target, cls_array, cls_array]) + assert tgt.shape[0] == seq_len + + is_masked = np.concatenate([mask_0, mask_1], 0) + if FLAGS.num_predict is not None: + assert np.sum(is_masked) == FLAGS.num_predict + + feature = { + "input": _int64_feature(cat_data), + "is_masked": _int64_feature(is_masked), + "target": _int64_feature(tgt), + "seg_id": _int64_feature(seg_id), + "label": _int64_feature([label]), + } + features.append(feature) + + if all_ok: + assert len(features) == bsz_per_host + for feature in features: + example = tf.train.Example(features=tf.train.Features(feature=feature)) + record_writer.write(example.SerializeToString()) + num_batch += 1 + else: + break + + i += reuse_len + + record_writer.close() + tf.logging.info("Done writing %s. Num of batches: %d", save_path, num_batch) + + return save_path, num_batch + + +################ +# get_input_fn # +################ +def _convert_example(example, use_bfloat16): + """Cast int64 into int32 and float32 to bfloat16 if use_bfloat16.""" + for key in list(example.keys()): + val = example[key] + if tf.keras.backend.is_sparse(val): + val = tf.sparse.to_dense(val) + if val.dtype == tf.int64: + val = tf.cast(val, tf.int32) + if use_bfloat16 and val.dtype == tf.float32: + val = tf.cast(val, tf.bfloat16) + + example[key] = val + + +def parse_files_to_dataset(parser, file_names, split, num_batch, num_hosts, + host_id, num_core_per_host, bsz_per_core): + # list of file pathes + num_files = len(file_names) + num_files_per_host = num_files // num_hosts + my_start_file_id = host_id * num_files_per_host + my_end_file_id = (host_id + 1) * num_files_per_host + if host_id == num_hosts - 1: + my_end_file_id = num_files + file_paths = file_names[my_start_file_id: my_end_file_id] + tf.logging.info("Host %d handles %d files", host_id, len(file_paths)) + + assert split == "train" + dataset = tf.data.Dataset.from_tensor_slices(file_paths) + + # file-level shuffle + if len(file_paths) > 1: + dataset = dataset.shuffle(len(file_paths)) + + # Note: we cannot perform sample-level shuffle here because this will violate + # the consecutive requirement of data stream. + dataset = tf.data.TFRecordDataset(dataset) + + # Note: since we are doing online preprocessing, the parsed result of + # the same input at each time will be different. Thus, cache processed data + # is not helpful. It will use a lot of memory and lead to contrainer OOM. + # So, change to cache non-parsed raw data instead. + dataset = dataset.cache().map(parser).repeat() + dataset = dataset.batch(bsz_per_core, drop_remainder=True) + dataset = dataset.prefetch(num_core_per_host * bsz_per_core) + + return dataset + + +def _local_perm(inputs, targets, is_masked, perm_size, seq_len): + """ + Sample a permutation of the factorization order, and create an + attention mask accordingly. + + Args: + inputs: int64 Tensor in shape [seq_len], input ids. + targets: int64 Tensor in shape [seq_len], target ids. + is_masked: bool Tensor in shape [seq_len]. True means being selected + for partial prediction. + perm_size: the length of longest permutation. Could be set to be reuse_len. + Should not be larger than reuse_len or there will be data leaks. + seq_len: int, sequence length. + """ + + # Generate permutation indices + index = tf.range(seq_len, dtype=tf.int64) + index = tf.transpose(tf.reshape(index, [-1, perm_size])) + index = tf.random_shuffle(index) + index = tf.reshape(tf.transpose(index), [-1]) + + # `perm_mask` and `target_mask` + # non-functional tokens + non_func_tokens = tf.logical_not(tf.logical_or( + tf.equal(inputs, SEP_ID), + tf.equal(inputs, CLS_ID))) + + non_mask_tokens = tf.logical_and(tf.logical_not(is_masked), non_func_tokens) + masked_or_func_tokens = tf.logical_not(non_mask_tokens) + + # Set the permutation indices of non-masked (& non-funcional) tokens to the + # smallest index (-1): + # (1) they can be seen by all other positions + # (2) they cannot see masked positions, so there won"t be information leak + smallest_index = -tf.ones([seq_len], dtype=tf.int64) + rev_index = tf.where(non_mask_tokens, smallest_index, index) + + # Create `target_mask`: non-funcional and maksed tokens + # 1: use mask as input and have loss + # 0: use token (or [SEP], [CLS]) as input and do not have loss + target_tokens = tf.logical_and(masked_or_func_tokens, non_func_tokens) + target_mask = tf.cast(target_tokens, tf.float32) + + # Create `perm_mask` + # `target_tokens` cannot see themselves + self_rev_index = tf.where(target_tokens, rev_index, rev_index + 1) + + # 1: cannot attend if i <= j and j is not non-masked (masked_or_func_tokens) + # 0: can attend if i > j or j is non-masked + perm_mask = tf.logical_and( + self_rev_index[:, None] <= rev_index[None, :], + masked_or_func_tokens) + perm_mask = tf.cast(perm_mask, tf.float32) + + # new target: [next token] for LM and [curr token] (self) for PLM + new_targets = tf.concat([inputs[0: 1], targets[: -1]], + axis=0) + + # construct inputs_k + inputs_k = inputs + + # construct inputs_q + inputs_q = target_mask + + return perm_mask, new_targets, target_mask, inputs_k, inputs_q + + +def get_dataset(params, num_hosts, num_core_per_host, split, file_names, + num_batch, seq_len, reuse_len, perm_size, mask_alpha, + mask_beta, use_bfloat16=False, num_predict=None): + + bsz_per_core = params["batch_size"] + if num_hosts > 1: + host_id = params["context"].current_host + else: + host_id = 0 + + #### Function used to parse tfrecord + def parser(record): + """function used to parse tfrecord.""" + + record_spec = { + "input": tf.FixedLenFeature([seq_len], tf.int64), + "target": tf.FixedLenFeature([seq_len], tf.int64), + "seg_id": tf.FixedLenFeature([seq_len], tf.int64), + "label": tf.FixedLenFeature([1], tf.int64), + "is_masked": tf.FixedLenFeature([seq_len], tf.int64), + } + + # retrieve serialized example + example = tf.parse_single_example( + serialized=record, + features=record_spec) + + inputs = example.pop("input") + target = example.pop("target") + is_masked = tf.cast(example.pop("is_masked"), tf.bool) + + non_reuse_len = seq_len - reuse_len + assert perm_size <= reuse_len and perm_size <= non_reuse_len + + perm_mask_0, target_0, target_mask_0, input_k_0, input_q_0 = _local_perm( + inputs[:reuse_len], + target[:reuse_len], + is_masked[:reuse_len], + perm_size, + reuse_len) + + perm_mask_1, target_1, target_mask_1, input_k_1, input_q_1 = _local_perm( + inputs[reuse_len:], + target[reuse_len:], + is_masked[reuse_len:], + perm_size, + non_reuse_len) + + perm_mask_0 = tf.concat([perm_mask_0, tf.ones([reuse_len, non_reuse_len])], + axis=1) + perm_mask_1 = tf.concat([tf.zeros([non_reuse_len, reuse_len]), perm_mask_1], + axis=1) + perm_mask = tf.concat([perm_mask_0, perm_mask_1], axis=0) + target = tf.concat([target_0, target_1], axis=0) + target_mask = tf.concat([target_mask_0, target_mask_1], axis=0) + input_k = tf.concat([input_k_0, input_k_1], axis=0) + input_q = tf.concat([input_q_0, input_q_1], axis=0) + + if num_predict is not None: + indices = tf.range(seq_len, dtype=tf.int64) + bool_target_mask = tf.cast(target_mask, tf.bool) + indices = tf.boolean_mask(indices, bool_target_mask) + + ##### extra padding due to CLS/SEP introduced after prepro + actual_num_predict = tf.shape(indices)[0] + pad_len = num_predict - actual_num_predict + + ##### target_mapping + target_mapping = tf.one_hot(indices, seq_len, dtype=tf.float32) + paddings = tf.zeros([pad_len, seq_len], dtype=target_mapping.dtype) + target_mapping = tf.concat([target_mapping, paddings], axis=0) + example["target_mapping"] = tf.reshape(target_mapping, + [num_predict, seq_len]) + + ##### target + target = tf.boolean_mask(target, bool_target_mask) + paddings = tf.zeros([pad_len], dtype=target.dtype) + target = tf.concat([target, paddings], axis=0) + example["target"] = tf.reshape(target, [num_predict]) + + ##### target mask + target_mask = tf.concat( + [tf.ones([actual_num_predict], dtype=tf.float32), + tf.zeros([pad_len], dtype=tf.float32)], + axis=0) + example["target_mask"] = tf.reshape(target_mask, [num_predict]) + else: + example["target"] = tf.reshape(target, [seq_len]) + example["target_mask"] = tf.reshape(target_mask, [seq_len]) + + # reshape back to fixed shape + example["perm_mask"] = tf.reshape(perm_mask, [seq_len, seq_len]) + example["input_k"] = tf.reshape(input_k, [seq_len]) + example["input_q"] = tf.reshape(input_q, [seq_len]) + + _convert_example(example, use_bfloat16) + + for k, v in example.items(): + tf.logging.info("%s: %s", k, v) + + return example + + # Get dataset + dataset = parse_files_to_dataset( + parser=parser, + file_names=file_names, + split=split, + num_batch=num_batch, + num_hosts=num_hosts, + host_id=host_id, + num_core_per_host=num_core_per_host, + bsz_per_core=bsz_per_core) + + return dataset + + +def get_input_fn( + tfrecord_dir, + split, + bsz_per_host, + seq_len, + reuse_len, + bi_data, + num_hosts=1, + num_core_per_host=1, + perm_size=None, + mask_alpha=None, + mask_beta=None, + uncased=False, + num_passes=None, + use_bfloat16=False, + num_predict=None): + + # Merge all record infos into a single one + record_glob_base = format_filename( + prefix="record_info-{}-*".format(split), + bsz_per_host=bsz_per_host, + seq_len=seq_len, + bi_data=bi_data, + suffix="json", + mask_alpha=mask_alpha, + mask_beta=mask_beta, + reuse_len=reuse_len, + uncased=uncased, + fixed_num_predict=num_predict) + + record_info = {"num_batch": 0, "filenames": []} + + tfrecord_dirs = tfrecord_dir.split(",") + tf.logging.info("Use the following tfrecord dirs: %s", tfrecord_dirs) + + for idx, record_dir in enumerate(tfrecord_dirs): + record_glob = os.path.join(record_dir, record_glob_base) + tf.logging.info("[%d] Record glob: %s", idx, record_glob) + + record_paths = sorted(tf.gfile.Glob(record_glob)) + tf.logging.info("[%d] Num of record info path: %d", + idx, len(record_paths)) + + cur_record_info = {"num_batch": 0, "filenames": []} + + for record_info_path in record_paths: + if num_passes is not None: + record_info_name = os.path.basename(record_info_path) + fields = record_info_name.split(".")[0].split("-") + pass_id = int(fields[-1]) + if len(fields) == 5 and pass_id >= num_passes: + tf.logging.info("Skip pass %d: %s", pass_id, record_info_name) + continue + + with tf.gfile.Open(record_info_path, "r") as fp: + info = json.load(fp) + if num_passes is not None: + eff_num_passes = min(num_passes, len(info["filenames"])) + ratio = eff_num_passes / len(info["filenames"]) + cur_record_info["num_batch"] += int(info["num_batch"] * ratio) + cur_record_info["filenames"] += info["filenames"][:eff_num_passes] + else: + cur_record_info["num_batch"] += info["num_batch"] + cur_record_info["filenames"] += info["filenames"] + + # overwrite directory for `cur_record_info` + new_filenames = [] + for filename in cur_record_info["filenames"]: + basename = os.path.basename(filename) + new_filename = os.path.join(record_dir, basename) + new_filenames.append(new_filename) + cur_record_info["filenames"] = new_filenames + + tf.logging.info("[Dir %d] Number of chosen batches: %s", + idx, cur_record_info["num_batch"]) + tf.logging.info("[Dir %d] Number of chosen files: %s", + idx, len(cur_record_info["filenames"])) + tf.logging.info(cur_record_info["filenames"]) + + # add `cur_record_info` to global `record_info` + record_info["num_batch"] += cur_record_info["num_batch"] + record_info["filenames"] += cur_record_info["filenames"] + + tf.logging.info("Total number of batches: %d", + record_info["num_batch"]) + tf.logging.info("Total number of files: %d", + len(record_info["filenames"])) + tf.logging.info(record_info["filenames"]) + + def input_fn(params): + """docs.""" + assert params["batch_size"] * num_core_per_host == bsz_per_host + + dataset = get_dataset( + params=params, + num_hosts=num_hosts, + num_core_per_host=num_core_per_host, + split=split, + file_names=record_info["filenames"], + num_batch=record_info["num_batch"], + seq_len=seq_len, + reuse_len=reuse_len, + perm_size=perm_size, + mask_alpha=mask_alpha, + mask_beta=mask_beta, + use_bfloat16=use_bfloat16, + num_predict=num_predict) + + return dataset + + return input_fn, record_info + + +if __name__ == "__main__": + FLAGS = flags.FLAGS + flags.DEFINE_bool("use_tpu", True, help="whether to use TPUs") + flags.DEFINE_integer("bsz_per_host", 32, help="batch size per host.") + flags.DEFINE_integer("num_core_per_host", 8, help="num TPU cores per host.") + + flags.DEFINE_integer("seq_len", 512, + help="Sequence length.") + flags.DEFINE_integer("reuse_len", 256, + help="Number of token that can be reused as memory. " + "Could be half of `seq_len`.") + flags.DEFINE_bool("uncased", False, help="Use uncased inputs or not.") + flags.DEFINE_bool("bi_data", True, + help="whether to create bidirectional data") + flags.DEFINE_integer("mask_alpha", default=6, + help="How many tokens to form a group.") + flags.DEFINE_integer("mask_beta", default=1, + help="How many tokens to mask within each group.") + flags.DEFINE_bool("use_eod", True, + help="whether to append EOD at the end of a doc.") + flags.DEFINE_bool("from_raw_text", True, + help="Whether the input is raw text or encoded ids.") + flags.DEFINE_integer("num_predict", default=85, + help="Num of tokens to predict.") + + flags.DEFINE_string("input_glob", "data/example/*.txt", + help="Input file glob.") + flags.DEFINE_string("sp_path", "", help="Path to the sentence piece model.") + flags.DEFINE_string("save_dir", "proc_data/example", + help="Directory for saving the processed data.") + flags.DEFINE_enum("split", "train", ["train", "dev", "test"], + help="Save the data as which split.") + + flags.DEFINE_integer("pass_id", 0, help="ID of the current pass." + "Different passes sample different negative segment.") + flags.DEFINE_integer("num_task", 1, help="Number of total tasks.") + flags.DEFINE_integer("task", 0, help="The Task ID. This value is used when " + "using multiple workers to identify each worker.") + + tf.logging.set_verbosity(tf.logging.INFO) + app.run(create_data) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/preprocess_squad_data.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/preprocess_squad_data.py new file mode 100644 index 0000000..59c8944 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/preprocess_squad_data.py @@ -0,0 +1,110 @@ +# coding=utf-8 +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Script to pre-process SQUAD data into tfrecords.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import random + +from absl import app +from absl import flags +from absl import logging +import tensorflow as tf + +import sentencepiece as spm +from official.nlp.xlnet import squad_utils + +flags.DEFINE_integer( + "num_proc", default=1, help="Number of preprocessing processes.") +flags.DEFINE_integer("proc_id", default=0, help="Process id for preprocessing.") + +# I/O paths +flags.DEFINE_string("output_dir", default="", help="Output dir for TF records.") +flags.DEFINE_string( + "spiece_model_file", default="", help="Sentence Piece model path.") +flags.DEFINE_string("train_file", default="", help="Path of train file.") +flags.DEFINE_string("predict_file", default="", help="Path of prediction file.") + +# Data preprocessing config +flags.DEFINE_integer("max_seq_length", default=512, help="Max sequence length") +flags.DEFINE_integer("max_query_length", default=64, help="Max query length") +flags.DEFINE_integer("doc_stride", default=128, help="Doc stride") +flags.DEFINE_bool("uncased", default=False, help="Use uncased data.") +flags.DEFINE_bool( + "create_train_data", default=True, help="Whether to create training data.") +flags.DEFINE_bool( + "create_eval_data", default=False, help="Whether to create eval data.") + +FLAGS = flags.FLAGS + + +def preprocess(): + """Preprocesses SQUAD data.""" + sp_model = spm.SentencePieceProcessor() + sp_model.Load(FLAGS.spiece_model_file) + spm_basename = os.path.basename(FLAGS.spiece_model_file) + if FLAGS.create_train_data: + train_rec_file = os.path.join( + FLAGS.output_dir, + "{}.{}.slen-{}.qlen-{}.train.tf_record".format(spm_basename, + FLAGS.proc_id, + FLAGS.max_seq_length, + FLAGS.max_query_length)) + + logging.info("Read examples from %s", FLAGS.train_file) + train_examples = squad_utils.read_squad_examples( + FLAGS.train_file, is_training=True) + train_examples = train_examples[FLAGS.proc_id::FLAGS.num_proc] + + # Pre-shuffle the input to avoid having to make a very large shuffle + # buffer in the `input_fn`. + random.shuffle(train_examples) + write_to_logging = "Write to " + train_rec_file + logging.info(write_to_logging) + train_writer = squad_utils.FeatureWriter( + filename=train_rec_file, is_training=True) + squad_utils.convert_examples_to_features( + examples=train_examples, + sp_model=sp_model, + max_seq_length=FLAGS.max_seq_length, + doc_stride=FLAGS.doc_stride, + max_query_length=FLAGS.max_query_length, + is_training=True, + output_fn=train_writer.process_feature, + uncased=FLAGS.uncased) + train_writer.close() + if FLAGS.create_eval_data: + eval_examples = squad_utils.read_squad_examples( + FLAGS.predict_file, is_training=False) + squad_utils.create_eval_data(spm_basename, sp_model, eval_examples, + FLAGS.max_seq_length, FLAGS.max_query_length, + FLAGS.doc_stride, FLAGS.uncased, + FLAGS.output_dir) + + +def main(_): + logging.set_verbosity(logging.INFO) + + if not tf.io.gfile.exists(FLAGS.output_dir): + tf.io.gfile.mkdir(FLAGS.output_dir) + + preprocess() + + +if __name__ == "__main__": + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/preprocess_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/preprocess_utils.py new file mode 100644 index 0000000..d0e8ae8 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/preprocess_utils.py @@ -0,0 +1,125 @@ +# coding=utf-8 +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities for pre-processing.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import unicodedata + +import six + + +SPIECE_UNDERLINE = '▁' + + +def printable_text(text): + """Returns text encoded in a way suitable for print or `tf.logging`.""" + + # These functions want `str` for both Python2 and Python3, but in one case + # it's a Unicode string and in the other it's a byte string. + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode('utf-8', 'ignore') + else: + raise ValueError('Unsupported string type: %s' % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text + elif isinstance(text, unicode): + return text.encode('utf-8') + else: + raise ValueError('Unsupported string type: %s' % (type(text))) + else: + raise ValueError('Not running on Python2 or Python 3?') + + +def print_(*args): + new_args = [] + for arg in args: + if isinstance(arg, list): + s = [printable_text(i) for i in arg] + s = ' '.join(s) + new_args.append(s) + else: + new_args.append(printable_text(arg)) + print(*new_args) + + +def preprocess_text(inputs, lower=False, remove_space=True, keep_accents=False): + """Preprocesses texts.""" + if remove_space: + outputs = ' '.join(inputs.strip().split()) + else: + outputs = inputs + + outputs = outputs.replace('``', '"').replace("''", '"') + + if six.PY2 and isinstance(outputs, str): + outputs = outputs.decode('utf-8') + + if not keep_accents: + outputs = unicodedata.normalize('NFKD', outputs) + outputs = ''.join([c for c in outputs if not unicodedata.combining(c)]) + if lower: + outputs = outputs.lower() + + return outputs + + +def encode_pieces(sp_model, text, return_unicode=True, sample=False): + """Encodes pieces.""" + # return_unicode is used only for py2 + + if six.PY2 and isinstance(text, unicode): + text = text.encode('utf-8') + + if not sample: + pieces = sp_model.EncodeAsPieces(text) + else: + pieces = sp_model.SampleEncodeAsPieces(text, 64, 0.1) + new_pieces = [] + for piece in pieces: + if len(piece) > 1 and piece[-1] == ',' and piece[-2].isdigit(): + cur_pieces = sp_model.EncodeAsPieces( + piece[:-1].replace(SPIECE_UNDERLINE, '')) + if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE: + if len(cur_pieces[0]) == 1: + cur_pieces = cur_pieces[1:] + else: + cur_pieces[0] = cur_pieces[0][1:] + cur_pieces.append(piece[-1]) + new_pieces.extend(cur_pieces) + else: + new_pieces.append(piece) + + # note(zhiliny): convert back to unicode for py2 + if six.PY2 and return_unicode: + ret_pieces = [] + for piece in new_pieces: + if isinstance(piece, str): + piece = piece.decode('utf-8') + ret_pieces.append(piece) + new_pieces = ret_pieces + + return new_pieces + + +def encode_ids(sp_model, text, sample=False): + pieces = encode_pieces(sp_model, text, return_unicode=False, sample=sample) + ids = [sp_model.PieceToId(piece) for piece in pieces] + return ids diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/run_classifier.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/run_classifier.py new file mode 100644 index 0000000..79a27f2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/run_classifier.py @@ -0,0 +1,196 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""XLNet classification finetuning runner in tf2.0.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import functools +from absl import app +from absl import flags +from absl import logging + +import numpy as np +import tensorflow as tf +# pylint: disable=unused-import +from official.nlp.xlnet import common_flags +from official.nlp.xlnet import data_utils +from official.nlp.xlnet import optimization +from official.nlp.xlnet import training_utils +from official.nlp.xlnet import xlnet_config +from official.nlp.xlnet import xlnet_modeling as modeling +from official.utils.misc import tpu_lib + +flags.DEFINE_integer("n_class", default=2, help="Number of classes.") +flags.DEFINE_string( + "summary_type", + default="last", + help="Method used to summarize a sequence into a vector.") + +FLAGS = flags.FLAGS + + +def get_classificationxlnet_model(model_config, + run_config, + n_class, + summary_type="last"): + model = modeling.ClassificationXLNetModel( + model_config, run_config, n_class, summary_type, name="model") + return model + + +def run_evaluation(strategy, + test_input_fn, + eval_steps, + model, + step, + eval_summary_writer=None): + """Run evaluation for classification task. + + Args: + strategy: distribution strategy. + test_input_fn: input function for evaluation data. + eval_steps: total number of evaluation steps. + model: keras model object. + step: current train step. + eval_summary_writer: summary writer used to record evaluation metrics. As + there are fake data samples in validation set, we use mask to get rid of + them when calculating the accuracy. For the reason that there will be + dynamic-shape tensor, we first collect logits, labels and masks from TPU + and calculate the accuracy via numpy locally. + + Returns: + A float metric, accuracy. + """ + + def _test_step_fn(inputs): + """Replicated validation step.""" + + inputs["mems"] = None + _, logits = model(inputs, training=False) + return logits, inputs["label_ids"], inputs["is_real_example"] + + @tf.function + def _run_evaluation(test_iterator): + """Runs validation steps.""" + logits, labels, masks = strategy.run( + _test_step_fn, args=(next(test_iterator),)) + return logits, labels, masks + + test_iterator = data_utils.get_input_iterator(test_input_fn, strategy) + correct = 0 + total = 0 + for _ in range(eval_steps): + logits, labels, masks = _run_evaluation(test_iterator) + logits = strategy.experimental_local_results(logits) + labels = strategy.experimental_local_results(labels) + masks = strategy.experimental_local_results(masks) + merged_logits = [] + merged_labels = [] + merged_masks = [] + + for i in range(strategy.num_replicas_in_sync): + merged_logits.append(logits[i].numpy()) + merged_labels.append(labels[i].numpy()) + merged_masks.append(masks[i].numpy()) + merged_logits = np.vstack(np.array(merged_logits)) + merged_labels = np.hstack(np.array(merged_labels)) + merged_masks = np.hstack(np.array(merged_masks)) + real_index = np.where(np.equal(merged_masks, 1)) + correct += np.sum( + np.equal( + np.argmax(merged_logits[real_index], axis=-1), + merged_labels[real_index])) + total += np.shape(real_index)[-1] + accuracy = float(correct) / float(total) + logging.info("Train step: %d / acc = %d/%d = %f", step, correct, total, + accuracy) + if eval_summary_writer: + with eval_summary_writer.as_default(): + tf.summary.scalar("eval_acc", float(correct) / float(total), step=step) + eval_summary_writer.flush() + return accuracy + + +def get_metric_fn(): + train_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy( + "acc", dtype=tf.float32) + return train_acc_metric + + +def main(unused_argv): + del unused_argv + if FLAGS.strategy_type == "mirror": + strategy = tf.distribute.MirroredStrategy() + elif FLAGS.strategy_type == "tpu": + cluster_resolver = tpu_lib.tpu_initialize(FLAGS.tpu) + strategy = tf.distribute.experimental.TPUStrategy(cluster_resolver) + else: + raise ValueError("The distribution strategy type is not supported: %s" % + FLAGS.strategy_type) + if strategy: + logging.info("***** Number of cores used : %d", + strategy.num_replicas_in_sync) + train_input_fn = functools.partial(data_utils.get_classification_input_data, + FLAGS.train_batch_size, FLAGS.seq_len, + strategy, True, FLAGS.train_tfrecord_path) + test_input_fn = functools.partial(data_utils.get_classification_input_data, + FLAGS.test_batch_size, FLAGS.seq_len, + strategy, False, FLAGS.test_tfrecord_path) + + total_training_steps = FLAGS.train_steps + steps_per_loop = FLAGS.iterations + eval_steps = int(FLAGS.test_data_size / FLAGS.test_batch_size) + eval_fn = functools.partial(run_evaluation, strategy, test_input_fn, + eval_steps) + optimizer, learning_rate_fn = optimization.create_optimizer( + FLAGS.learning_rate, + total_training_steps, + FLAGS.warmup_steps, + adam_epsilon=FLAGS.adam_epsilon) + model_config = xlnet_config.XLNetConfig(FLAGS) + run_config = xlnet_config.create_run_config(True, False, FLAGS) + model_fn = functools.partial(get_classificationxlnet_model, model_config, + run_config, FLAGS.n_class, FLAGS.summary_type) + input_meta_data = {} + input_meta_data["d_model"] = FLAGS.d_model + input_meta_data["mem_len"] = FLAGS.mem_len + input_meta_data["batch_size_per_core"] = int(FLAGS.train_batch_size / + strategy.num_replicas_in_sync) + input_meta_data["n_layer"] = FLAGS.n_layer + input_meta_data["lr_layer_decay_rate"] = FLAGS.lr_layer_decay_rate + input_meta_data["n_class"] = FLAGS.n_class + + training_utils.train( + strategy=strategy, + model_fn=model_fn, + input_meta_data=input_meta_data, + eval_fn=eval_fn, + metric_fn=get_metric_fn, + train_input_fn=train_input_fn, + init_checkpoint=FLAGS.init_checkpoint, + init_from_transformerxl=FLAGS.init_from_transformerxl, + total_training_steps=total_training_steps, + steps_per_loop=steps_per_loop, + optimizer=optimizer, + learning_rate_fn=learning_rate_fn, + model_dir=FLAGS.model_dir, + save_steps=FLAGS.save_steps) + + +if __name__ == "__main__": + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/run_pretrain.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/run_pretrain.py new file mode 100644 index 0000000..e136f4d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/run_pretrain.py @@ -0,0 +1,156 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""XLNet classification finetuning runner in tf2.0.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import functools +import os + +from absl import app +from absl import flags +from absl import logging +import tensorflow as tf +# pylint: disable=unused-import +from official.nlp.xlnet import common_flags +from official.nlp.xlnet import data_utils +from official.nlp.xlnet import optimization +from official.nlp.xlnet import training_utils +from official.nlp.xlnet import xlnet_config +from official.nlp.xlnet import xlnet_modeling as modeling +from official.utils.misc import tpu_lib + +flags.DEFINE_integer( + "num_predict", + default=None, + help="Number of tokens to predict in partial prediction.") + +# FLAGS for pretrain input preprocessing +flags.DEFINE_integer("perm_size", 0, help="Window size of permutation.") +flags.DEFINE_float("leak_ratio", default=0.1, + help="Percent of masked tokens that are leaked.") + +flags.DEFINE_enum("sample_strategy", default="token_span", + enum_values=["single_token", "whole_word", "token_span", + "word_span"], + help="Stragey used to sample prediction targets.") +flags.DEFINE_integer("max_num_tokens", default=5, + help="Maximum number of tokens to sample in a span." + "Effective when token_span strategy is used.") +flags.DEFINE_integer("min_num_tokens", default=1, + help="Minimum number of tokens to sample in a span." + "Effective when token_span strategy is used.") + +flags.DEFINE_integer("max_num_words", default=5, + help="Maximum number of whole words to sample in a span." + "Effective when word_span strategy is used.") +flags.DEFINE_integer("min_num_words", default=1, + help="Minimum number of whole words to sample in a span." + "Effective when word_span strategy is used.") +FLAGS = flags.FLAGS + + +def get_pretrainxlnet_model(model_config, run_config): + return modeling.PretrainingXLNetModel( + use_proj=True, + xlnet_config=model_config, + run_config=run_config, + name="model") + + +def main(unused_argv): + del unused_argv + num_hosts = 1 + if FLAGS.strategy_type == "mirror": + strategy = tf.distribute.MirroredStrategy() + elif FLAGS.strategy_type == "tpu": + cluster_resolver = tpu_lib.tpu_initialize(FLAGS.tpu) + strategy = tf.distribute.experimental.TPUStrategy(cluster_resolver) + topology = FLAGS.tpu_topology.split("x") + total_num_core = 2 * int(topology[0]) * int(topology[1]) + num_hosts = total_num_core // FLAGS.num_core_per_host + else: + raise ValueError("The distribution strategy type is not supported: %s" % + FLAGS.strategy_type) + if strategy: + logging.info("***** Number of cores used : %d", + strategy.num_replicas_in_sync) + logging.info("***** Number of hosts used : %d", num_hosts) + online_masking_config = data_utils.OnlineMaskingConfig( + sample_strategy=FLAGS.sample_strategy, + max_num_tokens=FLAGS.max_num_tokens, + min_num_tokens=FLAGS.min_num_tokens, + max_num_words=FLAGS.max_num_words, + min_num_words=FLAGS.min_num_words) + + train_input_fn = functools.partial( + data_utils.get_pretrain_input_data, FLAGS.train_batch_size, FLAGS.seq_len, + strategy, FLAGS.train_tfrecord_path, FLAGS.reuse_len, FLAGS.perm_size, + FLAGS.leak_ratio, FLAGS.num_predict, FLAGS.uncased, online_masking_config, + num_hosts) + + total_training_steps = FLAGS.train_steps + + steps_per_loop = FLAGS.iterations + + optimizer, learning_rate_fn = optimization.create_optimizer( + init_lr=FLAGS.learning_rate, + num_train_steps=total_training_steps, + num_warmup_steps=FLAGS.warmup_steps, + min_lr_ratio=FLAGS.min_lr_ratio, + adam_epsilon=FLAGS.adam_epsilon, + weight_decay_rate=FLAGS.weight_decay_rate) + + model_config = xlnet_config.XLNetConfig(FLAGS) + run_config = xlnet_config.create_run_config(True, False, FLAGS) + input_meta_data = {} + input_meta_data["d_model"] = FLAGS.d_model + input_meta_data["mem_len"] = FLAGS.mem_len + input_meta_data["batch_size_per_core"] = int(FLAGS.train_batch_size / + strategy.num_replicas_in_sync) + input_meta_data["n_layer"] = FLAGS.n_layer + input_meta_data["lr_layer_decay_rate"] = FLAGS.lr_layer_decay_rate + model_fn = functools.partial(get_pretrainxlnet_model, model_config, + run_config) + + model = training_utils.train( + strategy=strategy, + model_fn=model_fn, + input_meta_data=input_meta_data, + eval_fn=None, + metric_fn=None, + train_input_fn=train_input_fn, + init_checkpoint=FLAGS.init_checkpoint, + init_from_transformerxl=FLAGS.init_from_transformerxl, + total_training_steps=total_training_steps, + steps_per_loop=steps_per_loop, + optimizer=optimizer, + learning_rate_fn=learning_rate_fn, + model_dir=FLAGS.model_dir, + save_steps=FLAGS.save_steps) + + # Export transformer-xl model checkpoint to be used in finetuning. + checkpoint = tf.train.Checkpoint(transformer_xl=model.transformerxl_model) + saved_path = checkpoint.save( + os.path.join(FLAGS.model_dir, "pretrained/transformer_xl.ckpt")) + logging.info("Exporting the transformer-xl model as a new TF checkpoint: %s", + saved_path) + + +if __name__ == "__main__": + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/run_squad.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/run_squad.py new file mode 100644 index 0000000..013893f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/run_squad.py @@ -0,0 +1,304 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""XLNet SQUAD finetuning runner in tf2.0.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import functools +import json +import os +import pickle + +from absl import app +from absl import flags +from absl import logging + +import tensorflow as tf +# pylint: disable=unused-import +import sentencepiece as spm +from official.nlp.xlnet import common_flags +from official.nlp.xlnet import data_utils +from official.nlp.xlnet import optimization +from official.nlp.xlnet import squad_utils +from official.nlp.xlnet import training_utils +from official.nlp.xlnet import xlnet_config +from official.nlp.xlnet import xlnet_modeling as modeling +from official.utils.misc import tpu_lib + +flags.DEFINE_string( + "test_feature_path", default=None, help="Path to feature of test set.") +flags.DEFINE_integer("query_len", default=64, help="Max query length.") +flags.DEFINE_integer("start_n_top", default=5, help="Beam size for span start.") +flags.DEFINE_integer("end_n_top", default=5, help="Beam size for span end.") +flags.DEFINE_string( + "predict_dir", default=None, help="Path to write predictions.") +flags.DEFINE_string( + "predict_file", default=None, help="Path to json file of test set.") +flags.DEFINE_integer( + "n_best_size", default=5, help="n best size for predictions.") +flags.DEFINE_integer("max_answer_length", default=64, help="Max answer length.") +# Data preprocessing config +flags.DEFINE_string( + "spiece_model_file", default=None, help="Sentence Piece model path.") +flags.DEFINE_integer("max_seq_length", default=512, help="Max sequence length.") +flags.DEFINE_integer("max_query_length", default=64, help="Max query length.") +flags.DEFINE_integer("doc_stride", default=128, help="Doc stride.") + +FLAGS = flags.FLAGS + + +class InputFeatures(object): + """A single set of features of data.""" + + def __init__(self, + unique_id, + example_index, + doc_span_index, + tok_start_to_orig_index, + tok_end_to_orig_index, + token_is_max_context, + input_ids, + input_mask, + p_mask, + segment_ids, + paragraph_len, + cls_index, + start_position=None, + end_position=None, + is_impossible=None): + self.unique_id = unique_id + self.example_index = example_index + self.doc_span_index = doc_span_index + self.tok_start_to_orig_index = tok_start_to_orig_index + self.tok_end_to_orig_index = tok_end_to_orig_index + self.token_is_max_context = token_is_max_context + self.input_ids = input_ids + self.input_mask = input_mask + self.p_mask = p_mask + self.segment_ids = segment_ids + self.paragraph_len = paragraph_len + self.cls_index = cls_index + self.start_position = start_position + self.end_position = end_position + self.is_impossible = is_impossible + + +# pylint: disable=unused-argument +def run_evaluation(strategy, test_input_fn, eval_examples, eval_features, + original_data, eval_steps, input_meta_data, model, + current_step, eval_summary_writer): + """Run evaluation for SQUAD task. + + Args: + strategy: distribution strategy. + test_input_fn: input function for evaluation data. + eval_examples: tf.Examples of the evaluation set. + eval_features: Feature objects of the evaluation set. + original_data: The original json data for the evaluation set. + eval_steps: total number of evaluation steps. + input_meta_data: input meta data. + model: keras model object. + current_step: current training step. + eval_summary_writer: summary writer used to record evaluation metrics. + + Returns: + A float metric, F1 score. + """ + + def _test_step_fn(inputs): + """Replicated validation step.""" + + inputs["mems"] = None + res = model(inputs, training=False) + return res, inputs["unique_ids"] + + @tf.function + def _run_evaluation(test_iterator): + """Runs validation steps.""" + res, unique_ids = strategy.run( + _test_step_fn, args=(next(test_iterator),)) + return res, unique_ids + + test_iterator = data_utils.get_input_iterator(test_input_fn, strategy) + cur_results = [] + for _ in range(eval_steps): + results, unique_ids = _run_evaluation(test_iterator) + unique_ids = strategy.experimental_local_results(unique_ids) + + for result_key in results: + results[result_key] = ( + strategy.experimental_local_results(results[result_key])) + for core_i in range(strategy.num_replicas_in_sync): + bsz = int(input_meta_data["test_batch_size"] / + strategy.num_replicas_in_sync) + for j in range(bsz): + result = {} + for result_key in results: + result[result_key] = results[result_key][core_i].numpy()[j] + result["unique_ids"] = unique_ids[core_i].numpy()[j] + # We appended a fake example into dev set to make data size can be + # divided by test_batch_size. Ignores this fake example during + # evaluation. + if result["unique_ids"] == 1000012047: + continue + unique_id = int(result["unique_ids"]) + + start_top_log_probs = ([ + float(x) for x in result["start_top_log_probs"].flat + ]) + start_top_index = [int(x) for x in result["start_top_index"].flat] + end_top_log_probs = ([ + float(x) for x in result["end_top_log_probs"].flat + ]) + end_top_index = [int(x) for x in result["end_top_index"].flat] + + cls_logits = float(result["cls_logits"].flat[0]) + cur_results.append( + squad_utils.RawResult( + unique_id=unique_id, + start_top_log_probs=start_top_log_probs, + start_top_index=start_top_index, + end_top_log_probs=end_top_log_probs, + end_top_index=end_top_index, + cls_logits=cls_logits)) + if len(cur_results) % 1000 == 0: + logging.info("Processing example: %d", len(cur_results)) + + output_prediction_file = os.path.join(input_meta_data["predict_dir"], + "predictions.json") + output_nbest_file = os.path.join(input_meta_data["predict_dir"], + "nbest_predictions.json") + output_null_log_odds_file = os.path.join(input_meta_data["predict_dir"], + "null_odds.json") + + results = squad_utils.write_predictions( + eval_examples, eval_features, cur_results, input_meta_data["n_best_size"], + input_meta_data["max_answer_length"], output_prediction_file, + output_nbest_file, output_null_log_odds_file, original_data, + input_meta_data["start_n_top"], input_meta_data["end_n_top"]) + + # Log current results. + log_str = "Result | " + for key, val in results.items(): + log_str += "{} {} | ".format(key, val) + logging.info(log_str) + with eval_summary_writer.as_default(): + tf.summary.scalar("best_f1", results["best_f1"], step=current_step) + tf.summary.scalar("best_exact", results["best_exact"], step=current_step) + eval_summary_writer.flush() + return results["best_f1"] + + +def get_qaxlnet_model(model_config, run_config, start_n_top, end_n_top): + model = modeling.QAXLNetModel( + model_config, + run_config, + start_n_top=start_n_top, + end_n_top=end_n_top, + name="model") + return model + + +def main(unused_argv): + del unused_argv + if FLAGS.strategy_type == "mirror": + strategy = tf.distribute.MirroredStrategy() + elif FLAGS.strategy_type == "tpu": + cluster_resolver = tpu_lib.tpu_initialize(FLAGS.tpu) + strategy = tf.distribute.experimental.TPUStrategy(cluster_resolver) + else: + raise ValueError("The distribution strategy type is not supported: %s" % + FLAGS.strategy_type) + if strategy: + logging.info("***** Number of cores used : %d", + strategy.num_replicas_in_sync) + train_input_fn = functools.partial(data_utils.get_squad_input_data, + FLAGS.train_batch_size, FLAGS.seq_len, + FLAGS.query_len, strategy, True, + FLAGS.train_tfrecord_path) + + test_input_fn = functools.partial(data_utils.get_squad_input_data, + FLAGS.test_batch_size, FLAGS.seq_len, + FLAGS.query_len, strategy, False, + FLAGS.test_tfrecord_path) + + total_training_steps = FLAGS.train_steps + steps_per_loop = FLAGS.iterations + eval_steps = int(FLAGS.test_data_size / FLAGS.test_batch_size) + + optimizer, learning_rate_fn = optimization.create_optimizer( + FLAGS.learning_rate, + total_training_steps, + FLAGS.warmup_steps, + adam_epsilon=FLAGS.adam_epsilon) + model_config = xlnet_config.XLNetConfig(FLAGS) + run_config = xlnet_config.create_run_config(True, False, FLAGS) + input_meta_data = {} + input_meta_data["start_n_top"] = FLAGS.start_n_top + input_meta_data["end_n_top"] = FLAGS.end_n_top + input_meta_data["lr_layer_decay_rate"] = FLAGS.lr_layer_decay_rate + input_meta_data["predict_dir"] = FLAGS.predict_dir + input_meta_data["n_best_size"] = FLAGS.n_best_size + input_meta_data["max_answer_length"] = FLAGS.max_answer_length + input_meta_data["test_batch_size"] = FLAGS.test_batch_size + input_meta_data["batch_size_per_core"] = int(FLAGS.train_batch_size / + strategy.num_replicas_in_sync) + input_meta_data["mem_len"] = FLAGS.mem_len + model_fn = functools.partial(get_qaxlnet_model, model_config, run_config, + FLAGS.start_n_top, FLAGS.end_n_top) + eval_examples = squad_utils.read_squad_examples( + FLAGS.predict_file, is_training=False) + if FLAGS.test_feature_path: + logging.info("start reading pickle file...") + with tf.io.gfile.GFile(FLAGS.test_feature_path, "rb") as f: + eval_features = pickle.load(f) + logging.info("finishing reading pickle file...") + else: + sp_model = spm.SentencePieceProcessor() + sp_model.LoadFromSerializedProto( + tf.io.gfile.GFile(FLAGS.spiece_model_file, "rb").read()) + spm_basename = os.path.basename(FLAGS.spiece_model_file) + eval_features = squad_utils.create_eval_data( + spm_basename, sp_model, eval_examples, FLAGS.max_seq_length, + FLAGS.max_query_length, FLAGS.doc_stride, FLAGS.uncased) + + with tf.io.gfile.GFile(FLAGS.predict_file) as f: + original_data = json.load(f)["data"] + eval_fn = functools.partial(run_evaluation, strategy, test_input_fn, + eval_examples, eval_features, original_data, + eval_steps, input_meta_data) + + training_utils.train( + strategy=strategy, + model_fn=model_fn, + input_meta_data=input_meta_data, + eval_fn=eval_fn, + metric_fn=None, + train_input_fn=train_input_fn, + init_checkpoint=FLAGS.init_checkpoint, + init_from_transformerxl=FLAGS.init_from_transformerxl, + total_training_steps=total_training_steps, + steps_per_loop=steps_per_loop, + optimizer=optimizer, + learning_rate_fn=learning_rate_fn, + model_dir=FLAGS.model_dir, + save_steps=FLAGS.save_steps) + + +if __name__ == "__main__": + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/squad_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/squad_utils.py new file mode 100644 index 0000000..efab6da --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/squad_utils.py @@ -0,0 +1,973 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# coding=utf-8 +"""Utilities used in SQUAD task.""" +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import collections +import gc +import json +import math +import os +import pickle +import re +import string + +from absl import logging +import numpy as np +import six +import tensorflow as tf + +from official.nlp.xlnet import data_utils +from official.nlp.xlnet import preprocess_utils + +SPIECE_UNDERLINE = u"▁" + + +class InputFeatures(object): + """A single set of features of data.""" + + def __init__(self, + unique_id, + example_index, + doc_span_index, + tok_start_to_orig_index, + tok_end_to_orig_index, + token_is_max_context, + input_ids, + input_mask, + p_mask, + segment_ids, + paragraph_len, + cls_index, + start_position=None, + end_position=None, + is_impossible=None): + self.unique_id = unique_id + self.example_index = example_index + self.doc_span_index = doc_span_index + self.tok_start_to_orig_index = tok_start_to_orig_index + self.tok_end_to_orig_index = tok_end_to_orig_index + self.token_is_max_context = token_is_max_context + self.input_ids = input_ids + self.input_mask = input_mask + self.p_mask = p_mask + self.segment_ids = segment_ids + self.paragraph_len = paragraph_len + self.cls_index = cls_index + self.start_position = start_position + self.end_position = end_position + self.is_impossible = is_impossible + + +def make_qid_to_has_ans(dataset): + qid_to_has_ans = {} + for article in dataset: + for p in article["paragraphs"]: + for qa in p["qas"]: + qid_to_has_ans[qa["id"]] = bool(qa["answers"]) + return qid_to_has_ans + + +def get_raw_scores(dataset, preds): + """Gets exact scores and f1 scores.""" + exact_scores = {} + f1_scores = {} + for article in dataset: + for p in article["paragraphs"]: + for qa in p["qas"]: + qid = qa["id"] + gold_answers = [ + a["text"] for a in qa["answers"] if normalize_answer(a["text"]) + ] + if not gold_answers: + # For unanswerable questions, only correct answer is empty string + gold_answers = [""] + if qid not in preds: + print("Missing prediction for %s" % qid) + continue + a_pred = preds[qid] + # Take max over all gold answers + exact_scores[qid] = max(compute_exact(a, a_pred) for a in gold_answers) + f1_scores[qid] = max(compute_f1(a, a_pred) for a in gold_answers) + return exact_scores, f1_scores + + +def normalize_answer(s): + """Lower text and remove punctuation, articles and extra whitespace.""" + + def remove_articles(text): + regex = re.compile(r"\b(a|an|the)\b", re.UNICODE) + return re.sub(regex, " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def compute_exact(a_gold, a_pred): + return int(normalize_answer(a_gold) == normalize_answer(a_pred)) + + +def get_tokens(s): + if not s: + return [] + return normalize_answer(s).split() + + +def compute_f1(a_gold, a_pred): + """Computes f1 score.""" + gold_toks = get_tokens(a_gold) + pred_toks = get_tokens(a_pred) + common = collections.Counter(gold_toks) & collections.Counter(pred_toks) + num_same = sum(common.values()) + # pylint: disable=g-explicit-length-test + if len(gold_toks) == 0 or len(pred_toks) == 0: + # If either is no-answer, then F1 is 1 if they agree, 0 otherwise + return int(gold_toks == pred_toks) + if num_same == 0: + return 0 + precision = 1.0 * num_same / len(pred_toks) + recall = 1.0 * num_same / len(gold_toks) + f1 = (2 * precision * recall) / (precision + recall) + return f1 + + +def find_best_thresh(preds, scores, na_probs, qid_to_has_ans): + """Finds best threshold.""" + num_no_ans = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]) + cur_score = num_no_ans + best_score = cur_score + best_thresh = 0.0 + qid_list = sorted(na_probs, key=lambda k: na_probs[k]) + for qid in qid_list: + if qid not in scores: + continue + if qid_to_has_ans[qid]: + diff = scores[qid] + else: + if preds[qid]: + diff = -1 + else: + diff = 0 + cur_score += diff + if cur_score > best_score: + best_score = cur_score + best_thresh = na_probs[qid] + + has_ans_score, has_ans_cnt = 0, 0 + for qid in qid_list: + if not qid_to_has_ans[qid]: + continue + has_ans_cnt += 1 + + if qid not in scores: + continue + has_ans_score += scores[qid] + + return 100.0 * best_score / len( + scores), best_thresh, 1.0 * has_ans_score / has_ans_cnt + + +def find_all_best_thresh(main_eval, preds, exact_raw, f1_raw, na_probs, + qid_to_has_ans): + """Finds all best threshold.""" + best_exact, exact_thresh, has_ans_exact = find_best_thresh( + preds, exact_raw, na_probs, qid_to_has_ans) + best_f1, f1_thresh, has_ans_f1 = find_best_thresh(preds, f1_raw, na_probs, + qid_to_has_ans) + main_eval["best_exact"] = best_exact + main_eval["best_exact_thresh"] = exact_thresh + main_eval["best_f1"] = best_f1 + main_eval["best_f1_thresh"] = f1_thresh + main_eval["has_ans_exact"] = has_ans_exact + main_eval["has_ans_f1"] = has_ans_f1 + + +_PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name + "PrelimPrediction", [ + "feature_index", "start_index", "end_index", "start_log_prob", + "end_log_prob" + ]) + +_NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name + "NbestPrediction", ["text", "start_log_prob", "end_log_prob"]) +RawResult = collections.namedtuple("RawResult", [ + "unique_id", "start_top_log_probs", "start_top_index", "end_top_log_probs", + "end_top_index", "cls_logits" +]) + + +def _compute_softmax(scores): + """Computes softmax probability over raw logits.""" + if not scores: + return [] + + max_score = None + for score in scores: + if max_score is None or score > max_score: + max_score = score + + exp_scores = [] + total_sum = 0.0 + for score in scores: + x = math.exp(score - max_score) + exp_scores.append(x) + total_sum += x + + probs = [] + for score in exp_scores: + probs.append(score / total_sum) + return probs + + +class SquadExample(object): + """A single training/test example for simple sequence classification. + + For examples without an answer, the start and end position are -1. + """ + + def __init__(self, + qas_id, + question_text, + paragraph_text, + orig_answer_text=None, + start_position=None, + is_impossible=False): + self.qas_id = qas_id + self.question_text = question_text + self.paragraph_text = paragraph_text + self.orig_answer_text = orig_answer_text + self.start_position = start_position + self.is_impossible = is_impossible + + def __str__(self): + return self.__repr__() + + def __repr__(self): + s = "" + s += "qas_id: %s" % (preprocess_utils.printable_text(self.qas_id)) + s += ", question_text: %s" % ( + preprocess_utils.printable_text(self.question_text)) + s += ", paragraph_text: [%s]" % (" ".join(self.paragraph_text)) + if self.start_position: + s += ", start_position: %d" % (self.start_position) + if self.start_position: + s += ", is_impossible: %r" % (self.is_impossible) + return s + + +def write_predictions(all_examples, all_features, all_results, n_best_size, + max_answer_length, output_prediction_file, + output_nbest_file, output_null_log_odds_file, orig_data, + start_n_top, end_n_top): + """Writes final predictions to the json file and log-odds of null if needed.""" + logging.info("Writing predictions to: %s", (output_prediction_file)) + + example_index_to_features = collections.defaultdict(list) + for feature in all_features: + example_index_to_features[feature.example_index].append(feature) + + unique_id_to_result = {} + for result in all_results: + unique_id_to_result[result.unique_id] = result + + all_predictions = collections.OrderedDict() + all_nbest_json = collections.OrderedDict() + scores_diff_json = collections.OrderedDict() + + for (example_index, example) in enumerate(all_examples): + features = example_index_to_features[example_index] + + prelim_predictions = [] + # keep track of the minimum score of null start+end of position 0 + score_null = 1000000 # large and positive + + for (feature_index, feature) in enumerate(features): + result = unique_id_to_result[feature.unique_id] + + cur_null_score = result.cls_logits + + # if we could have irrelevant answers, get the min score of irrelevant + score_null = min(score_null, cur_null_score) + + for i in range(start_n_top): + for j in range(end_n_top): + start_log_prob = result.start_top_log_probs[i] + start_index = result.start_top_index[i] + + j_index = i * end_n_top + j + + end_log_prob = result.end_top_log_probs[j_index] + end_index = result.end_top_index[j_index] + + # We could hypothetically create invalid predictions, e.g., predict + # that the start of the span is in the question. We throw out all + # invalid predictions. + if start_index >= feature.paragraph_len - 1: + continue + if end_index >= feature.paragraph_len - 1: + continue + + if not feature.token_is_max_context.get(start_index, False): + continue + if end_index < start_index: + continue + length = end_index - start_index + 1 + if length > max_answer_length: + continue + + prelim_predictions.append( + _PrelimPrediction( + feature_index=feature_index, + start_index=start_index, + end_index=end_index, + start_log_prob=start_log_prob, + end_log_prob=end_log_prob)) + + prelim_predictions = sorted( + prelim_predictions, + key=lambda x: (x.start_log_prob + x.end_log_prob), + reverse=True) + + seen_predictions = {} + nbest = [] + for pred in prelim_predictions: + if len(nbest) >= n_best_size: + break + feature = features[pred.feature_index] + + tok_start_to_orig_index = feature.tok_start_to_orig_index + tok_end_to_orig_index = feature.tok_end_to_orig_index + start_orig_pos = tok_start_to_orig_index[pred.start_index] + end_orig_pos = tok_end_to_orig_index[pred.end_index] + + paragraph_text = example.paragraph_text + final_text = paragraph_text[start_orig_pos:end_orig_pos + 1].strip() + + if final_text in seen_predictions: + continue + + seen_predictions[final_text] = True + + nbest.append( + _NbestPrediction( + text=final_text, + start_log_prob=pred.start_log_prob, + end_log_prob=pred.end_log_prob)) + + # In very rare edge cases we could have no valid predictions. So we + # just create a nonce prediction in this case to avoid failure. + if not nbest: + nbest.append( + _NbestPrediction(text="", start_log_prob=-1e6, end_log_prob=-1e6)) + + total_scores = [] + best_non_null_entry = None + for entry in nbest: + total_scores.append(entry.start_log_prob + entry.end_log_prob) + if not best_non_null_entry: + best_non_null_entry = entry + + probs = _compute_softmax(total_scores) + + nbest_json = [] + for (i, entry) in enumerate(nbest): + output = collections.OrderedDict() + output["text"] = entry.text + output["probability"] = probs[i] + output["start_log_prob"] = entry.start_log_prob + output["end_log_prob"] = entry.end_log_prob + nbest_json.append(output) + + assert len(nbest_json) >= 1 + assert best_non_null_entry is not None + + score_diff = score_null + scores_diff_json[example.qas_id] = score_diff + + all_predictions[example.qas_id] = best_non_null_entry.text + + all_nbest_json[example.qas_id] = nbest_json + + with tf.io.gfile.GFile(output_prediction_file, "w") as writer: + writer.write(json.dumps(all_predictions, indent=4) + "\n") + + with tf.io.gfile.GFile(output_nbest_file, "w") as writer: + writer.write(json.dumps(all_nbest_json, indent=4) + "\n") + + with tf.io.gfile.GFile(output_null_log_odds_file, "w") as writer: + writer.write(json.dumps(scores_diff_json, indent=4) + "\n") + + qid_to_has_ans = make_qid_to_has_ans(orig_data) + exact_raw, f1_raw = get_raw_scores(orig_data, all_predictions) + out_eval = {} + + find_all_best_thresh(out_eval, all_predictions, exact_raw, f1_raw, + scores_diff_json, qid_to_has_ans) + + return out_eval + + +def read_squad_examples(input_file, is_training): + """Reads a SQuAD json file into a list of SquadExample.""" + with tf.io.gfile.GFile(input_file, "r") as reader: + input_data = json.load(reader)["data"] + + examples = [] + for entry in input_data: + for paragraph in entry["paragraphs"]: + paragraph_text = paragraph["context"] + + for qa in paragraph["qas"]: + qas_id = qa["id"] + question_text = qa["question"] + start_position = None + orig_answer_text = None + is_impossible = False + + if is_training: + is_impossible = qa["is_impossible"] + if (len(qa["answers"]) != 1) and (not is_impossible): + raise ValueError( + "For training, each question should have exactly 1 answer.") + if not is_impossible: + answer = qa["answers"][0] + orig_answer_text = answer["text"] + start_position = answer["answer_start"] + else: + start_position = -1 + orig_answer_text = "" + + example = SquadExample( + qas_id=qas_id, + question_text=question_text, + paragraph_text=paragraph_text, + orig_answer_text=orig_answer_text, + start_position=start_position, + is_impossible=is_impossible) + examples.append(example) + + return examples + + +# pylint: disable=invalid-name +def _convert_index(index, pos, M=None, is_start=True): + """Converts index.""" + if index[pos] is not None: + return index[pos] + N = len(index) + rear = pos + while rear < N - 1 and index[rear] is None: + rear += 1 + front = pos + while front > 0 and index[front] is None: + front -= 1 + assert index[front] is not None or index[rear] is not None + if index[front] is None: + if index[rear] >= 1: + if is_start: + return 0 + else: + return index[rear] - 1 + return index[rear] + if index[rear] is None: + if M is not None and index[front] < M - 1: + if is_start: + return index[front] + 1 + else: + return M - 1 + return index[front] + if is_start: + if index[rear] > index[front] + 1: + return index[front] + 1 + else: + return index[rear] + else: + if index[rear] > index[front] + 1: + return index[rear] - 1 + else: + return index[front] + + +def convert_examples_to_features(examples, sp_model, max_seq_length, doc_stride, + max_query_length, is_training, output_fn, + uncased): + """Loads a data file into a list of `InputBatch`s.""" + + cnt_pos, cnt_neg = 0, 0 + unique_id = 1000000000 + max_N, max_M = 1024, 1024 + f = np.zeros((max_N, max_M), dtype=np.float32) + + for (example_index, example) in enumerate(examples): + # pylint: disable=logging-format-interpolation + if example_index % 100 == 0: + logging.info("Converting {}/{} pos {} neg {}".format( + example_index, len(examples), cnt_pos, cnt_neg)) + + query_tokens = preprocess_utils.encode_ids( + sp_model, + preprocess_utils.preprocess_text(example.question_text, lower=uncased)) + + if len(query_tokens) > max_query_length: + query_tokens = query_tokens[0:max_query_length] + + paragraph_text = example.paragraph_text + para_tokens = preprocess_utils.encode_pieces( + sp_model, + preprocess_utils.preprocess_text(example.paragraph_text, lower=uncased)) + + chartok_to_tok_index = [] + tok_start_to_chartok_index = [] + tok_end_to_chartok_index = [] + char_cnt = 0 + for i, token in enumerate(para_tokens): + chartok_to_tok_index.extend([i] * len(token)) + tok_start_to_chartok_index.append(char_cnt) + char_cnt += len(token) + tok_end_to_chartok_index.append(char_cnt - 1) + + tok_cat_text = "".join(para_tokens).replace(SPIECE_UNDERLINE, " ") + N, M = len(paragraph_text), len(tok_cat_text) + + if N > max_N or M > max_M: + max_N = max(N, max_N) + max_M = max(M, max_M) + f = np.zeros((max_N, max_M), dtype=np.float32) + gc.collect() + + g = {} + + # pylint: disable=cell-var-from-loop + def _lcs_match(max_dist): + """LCS match.""" + f.fill(0) + g.clear() + + ### longest common sub sequence + # f[i, j] = max(f[i - 1, j], f[i, j - 1], f[i - 1, j - 1] + match(i, j)) + for i in range(N): + + # note(zhiliny): + # unlike standard LCS, this is specifically optimized for the setting + # because the mismatch between sentence pieces and original text will + # be small + for j in range(i - max_dist, i + max_dist): + if j >= M or j < 0: + continue + + if i > 0: + g[(i, j)] = 0 + f[i, j] = f[i - 1, j] + + if j > 0 and f[i, j - 1] > f[i, j]: + g[(i, j)] = 1 + f[i, j] = f[i, j - 1] + + f_prev = f[i - 1, j - 1] if i > 0 and j > 0 else 0 + if (preprocess_utils.preprocess_text( + paragraph_text[i], lower=uncased, + remove_space=False) == tok_cat_text[j] and f_prev + 1 > f[i, j]): + g[(i, j)] = 2 + f[i, j] = f_prev + 1 + + max_dist = abs(N - M) + 5 + for _ in range(2): + _lcs_match(max_dist) + if f[N - 1, M - 1] > 0.8 * N: + break + max_dist *= 2 + + orig_to_chartok_index = [None] * N + chartok_to_orig_index = [None] * M + i, j = N - 1, M - 1 + while i >= 0 and j >= 0: + if (i, j) not in g: + break + if g[(i, j)] == 2: + orig_to_chartok_index[i] = j + chartok_to_orig_index[j] = i + i, j = i - 1, j - 1 + elif g[(i, j)] == 1: + j = j - 1 + else: + i = i - 1 + + if all( + v is None for v in orig_to_chartok_index) or f[N - 1, M - 1] < 0.8 * N: + print("MISMATCH DETECTED!") + continue + + tok_start_to_orig_index = [] + tok_end_to_orig_index = [] + for i in range(len(para_tokens)): + start_chartok_pos = tok_start_to_chartok_index[i] + end_chartok_pos = tok_end_to_chartok_index[i] + start_orig_pos = _convert_index( + chartok_to_orig_index, start_chartok_pos, N, is_start=True) + end_orig_pos = _convert_index( + chartok_to_orig_index, end_chartok_pos, N, is_start=False) + + tok_start_to_orig_index.append(start_orig_pos) + tok_end_to_orig_index.append(end_orig_pos) + + if not is_training: + tok_start_position = tok_end_position = None + + if is_training and example.is_impossible: + tok_start_position = -1 + tok_end_position = -1 + + if is_training and not example.is_impossible: + start_position = example.start_position + end_position = start_position + len(example.orig_answer_text) - 1 + + start_chartok_pos = _convert_index( + orig_to_chartok_index, start_position, is_start=True) + tok_start_position = chartok_to_tok_index[start_chartok_pos] + + end_chartok_pos = _convert_index( + orig_to_chartok_index, end_position, is_start=False) + tok_end_position = chartok_to_tok_index[end_chartok_pos] + assert tok_start_position <= tok_end_position + + def _piece_to_id(x): + if six.PY2 and isinstance(x, unicode): + x = x.encode("utf-8") + return sp_model.PieceToId(x) + + all_doc_tokens = list(map(_piece_to_id, para_tokens)) + + # The -3 accounts for [CLS], [SEP] and [SEP] + max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 + + # We can have documents that are longer than the maximum sequence length. + # To deal with this we do a sliding window approach, where we take chunks + # of the up to our max length with a stride of `doc_stride`. + _DocSpan = collections.namedtuple( # pylint: disable=invalid-name + "DocSpan", ["start", "length"]) + doc_spans = [] + start_offset = 0 + while start_offset < len(all_doc_tokens): + length = len(all_doc_tokens) - start_offset + if length > max_tokens_for_doc: + length = max_tokens_for_doc + doc_spans.append(_DocSpan(start=start_offset, length=length)) + if start_offset + length == len(all_doc_tokens): + break + start_offset += min(length, doc_stride) + + for (doc_span_index, doc_span) in enumerate(doc_spans): + tokens = [] + token_is_max_context = {} + segment_ids = [] + p_mask = [] + + cur_tok_start_to_orig_index = [] + cur_tok_end_to_orig_index = [] + + for i in range(doc_span.length): + split_token_index = doc_span.start + i + + cur_tok_start_to_orig_index.append( + tok_start_to_orig_index[split_token_index]) + cur_tok_end_to_orig_index.append( + tok_end_to_orig_index[split_token_index]) + + is_max_context = _check_is_max_context(doc_spans, doc_span_index, + split_token_index) + token_is_max_context[len(tokens)] = is_max_context + tokens.append(all_doc_tokens[split_token_index]) + segment_ids.append(data_utils.SEG_ID_P) + p_mask.append(0) + + paragraph_len = len(tokens) + + tokens.append(data_utils.SEP_ID) + segment_ids.append(data_utils.SEG_ID_P) + p_mask.append(1) + + # note(zhiliny): we put P before Q + # because during pretraining, B is always shorter than A + for token in query_tokens: + tokens.append(token) + segment_ids.append(data_utils.SEG_ID_Q) + p_mask.append(1) + tokens.append(data_utils.SEP_ID) + segment_ids.append(data_utils.SEG_ID_Q) + p_mask.append(1) + + cls_index = len(segment_ids) + tokens.append(data_utils.CLS_ID) + segment_ids.append(data_utils.SEG_ID_CLS) + p_mask.append(0) + + input_ids = tokens + + # The mask has 0 for real tokens and 1 for padding tokens. Only real + # tokens are attended to. + input_mask = [0] * len(input_ids) + + # Zero-pad up to the sequence length. + while len(input_ids) < max_seq_length: + input_ids.append(0) + input_mask.append(1) + segment_ids.append(data_utils.SEG_ID_PAD) + p_mask.append(1) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + assert len(p_mask) == max_seq_length + + span_is_impossible = example.is_impossible + start_position = None + end_position = None + if is_training and not span_is_impossible: + # For training, if our document chunk does not contain an annotation + # we throw it out, since there is nothing to predict. + doc_start = doc_span.start + doc_end = doc_span.start + doc_span.length - 1 + out_of_span = False + if not (tok_start_position >= doc_start and + tok_end_position <= doc_end): + out_of_span = True + if out_of_span: + # continue + start_position = 0 + end_position = 0 + span_is_impossible = True + else: + # note: we put P before Q, so doc_offset should be zero. + # doc_offset = len(query_tokens) + 2 + doc_offset = 0 + start_position = tok_start_position - doc_start + doc_offset + end_position = tok_end_position - doc_start + doc_offset + + if is_training and span_is_impossible: + start_position = cls_index + end_position = cls_index + + if example_index < 20: + logging.info("*** Example ***") + logging.info("unique_id: %s", unique_id) + logging.info("example_index: %s", example_index) + logging.info("doc_span_index: %s", doc_span_index) + logging.info("tok_start_to_orig_index: %s", + " ".join([str(x) for x in cur_tok_start_to_orig_index])) + logging.info("tok_end_to_orig_index: %s", + " ".join([str(x) for x in cur_tok_end_to_orig_index])) + logging.info( + "token_is_max_context: %s", " ".join([ + "%d:%s" % (x, y) + for (x, y) in six.iteritems(token_is_max_context) + ])) + logging.info("input_ids: %s", " ".join([str(x) for x in input_ids])) + logging.info("input_mask: %s", " ".join([str(x) for x in input_mask])) + logging.info("segment_ids: %s", " ".join([str(x) for x in segment_ids])) + + if is_training and span_is_impossible: + logging.info("impossible example span") + + if is_training and not span_is_impossible: + pieces = [ + sp_model.IdToPiece(token) + for token in tokens[start_position:(end_position + 1)] + ] + answer_text = sp_model.DecodePieces(pieces) + logging.info("start_position: %d", start_position) + logging.info("end_position: %d", end_position) + logging.info("answer: %s", + preprocess_utils.printable_text(answer_text)) + + # With multi processing, the example_index is actually the index + # within the current process therefore we use example_index=None to + # avoid being used in the future. # The current code does not use + # example_index of training data. + if is_training: + feat_example_index = None + else: + feat_example_index = example_index + + feature = InputFeatures( + unique_id=unique_id, + example_index=feat_example_index, + doc_span_index=doc_span_index, + tok_start_to_orig_index=cur_tok_start_to_orig_index, + tok_end_to_orig_index=cur_tok_end_to_orig_index, + token_is_max_context=token_is_max_context, + input_ids=input_ids, + input_mask=input_mask, + p_mask=p_mask, + segment_ids=segment_ids, + paragraph_len=paragraph_len, + cls_index=cls_index, + start_position=start_position, + end_position=end_position, + is_impossible=span_is_impossible) + + # Run callback + output_fn(feature) + + unique_id += 1 + if span_is_impossible: + cnt_neg += 1 + else: + cnt_pos += 1 + + logging.info("Total number of instances: %d = pos %d + neg %d", + cnt_pos + cnt_neg, cnt_pos, cnt_neg) + + +def _check_is_max_context(doc_spans, cur_span_index, position): + """Check if this is the "max context" doc span for the token.""" + + # Because of the sliding window approach taken to scoring documents, a single + # token can appear in multiple documents. E.g. + # Doc: the man went to the store and bought a gallon of milk + # Span A: the man went to the + # Span B: to the store and bought + # Span C: and bought a gallon of + # ... + # + # Now the word "bought" will have two scores from spans B and C. We only + # want to consider the score with "maximum context", which we define as + # the *minimum* of its left and right context (the *sum* of left and + # right context will always be the same, of course). + # + # In the example the maximum context for "bought" would be span C since + # it has 1 left context and 3 right context, while span B has 4 left context + # and 0 right context. + best_score = None + best_span_index = None + for (span_index, doc_span) in enumerate(doc_spans): + end = doc_span.start + doc_span.length - 1 + if position < doc_span.start: + continue + if position > end: + continue + num_left_context = position - doc_span.start + num_right_context = end - position + score = min(num_left_context, num_right_context) + 0.01 * doc_span.length + if best_score is None or score > best_score: + best_score = score + best_span_index = span_index + + return cur_span_index == best_span_index + + +class FeatureWriter(object): + """Writes InputFeature to TF example file.""" + + def __init__(self, filename, is_training): + self.filename = filename + self.is_training = is_training + self.num_features = 0 + self._writer = tf.io.TFRecordWriter(filename) + + def process_feature(self, feature): + """Write a InputFeature to the TFRecordWriter as a tf.train.Example.""" + self.num_features += 1 + + def create_int_feature(values): + feature = tf.train.Feature( + int64_list=tf.train.Int64List(value=list(values))) + return feature + + def create_float_feature(values): + f = tf.train.Feature(float_list=tf.train.FloatList(value=list(values))) + return f + + features = collections.OrderedDict() + features["unique_ids"] = create_int_feature([feature.unique_id]) + features["input_ids"] = create_int_feature(feature.input_ids) + features["input_mask"] = create_float_feature(feature.input_mask) + features["p_mask"] = create_float_feature(feature.p_mask) + features["segment_ids"] = create_int_feature(feature.segment_ids) + + features["cls_index"] = create_int_feature([feature.cls_index]) + + if self.is_training: + features["start_positions"] = create_int_feature([feature.start_position]) + features["end_positions"] = create_int_feature([feature.end_position]) + impossible = 0 + if feature.is_impossible: + impossible = 1 + features["is_impossible"] = create_float_feature([impossible]) + + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + self._writer.write(tf_example.SerializeToString()) + + def close(self): + self._writer.close() + + +def create_eval_data(spm_basename, + sp_model, + eval_examples, + max_seq_length, + max_query_length, + doc_stride, + uncased, + output_dir=None): + """Creates evaluation tfrecords.""" + eval_features = [] + eval_writer = None + if output_dir: + eval_rec_file = os.path.join( + output_dir, + "{}.slen-{}.qlen-{}.eval.tf_record".format(spm_basename, max_seq_length, + max_query_length)) + eval_feature_file = os.path.join( + output_dir, + "{}.slen-{}.qlen-{}.eval.features.pkl".format(spm_basename, + max_seq_length, + max_query_length)) + + eval_writer = FeatureWriter(filename=eval_rec_file, is_training=False) + + def append_feature(feature): + eval_features.append(feature) + if eval_writer: + eval_writer.process_feature(feature) + + convert_examples_to_features( + examples=eval_examples, + sp_model=sp_model, + max_seq_length=max_seq_length, + doc_stride=doc_stride, + max_query_length=max_query_length, + is_training=False, + output_fn=append_feature, + uncased=uncased) + + if eval_writer: + eval_writer.close() + with tf.io.gfile.GFile(eval_feature_file, "wb") as fout: + pickle.dump(eval_features, fout) + + return eval_features diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/training_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/training_utils.py new file mode 100644 index 0000000..293e463 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/training_utils.py @@ -0,0 +1,310 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""XLNet training utils.""" +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import os +import re + +from absl import logging + +# pytype: disable=attribute-error +# pylint: disable=g-bare-generic,unused-import +import tensorflow as tf +from typing import Any, Callable, Dict, Text, Optional + +from official.nlp.bert import model_training_utils +from official.nlp.xlnet import data_utils +from official.nlp.xlnet import xlnet_modeling as modeling + +_MIN_SUMMARY_STEPS = 10 + + +def _save_checkpoint(checkpoint, model_dir, checkpoint_prefix): + """Saves model to with provided checkpoint prefix.""" + + checkpoint_path = os.path.join(model_dir, checkpoint_prefix) + saved_path = checkpoint.save(checkpoint_path) + logging.info("Saving model as TF checkpoint: %s", saved_path) + return + + +def _float_metric_value(metric): + """Gets the value of a float-value keras metric.""" + return metric.result().numpy().astype(float) + + +def train( + strategy: tf.distribute.Strategy, + model_fn: Callable, + input_meta_data: Dict, + train_input_fn: Callable, + total_training_steps: int, + steps_per_loop: int, + optimizer: tf.keras.optimizers.Optimizer, + learning_rate_fn: tf.keras.optimizers.schedules.LearningRateSchedule, + eval_fn: Optional[Callable[[tf.keras.Model, int, tf.summary.SummaryWriter], + Any]] = None, + metric_fn: Optional[Callable[[], tf.keras.metrics.Metric]] = None, + init_checkpoint: Optional[Text] = None, + init_from_transformerxl: Optional[bool] = False, + model_dir: Optional[Text] = None, + save_steps: Optional[int] = None, + run_eagerly: Optional[bool] = False): + """Runs customized training. + + Args: + strategy: Distribution strategy on which to run low level training loop. + model_fn: The function returns a keras.Model. + input_meta_data: A dictionary of params: `mem_len`, `lr_layer_decay_rate`, + `n_layer`, `batch_size_per_core` and `d_model`. + train_input_fn: Function returns a tf.data.Dataset used for training. + total_training_steps: Number of steps to train in total. + steps_per_loop: Number of steps per graph-mode loop. In order to reduce + communication in eager context, training logs are printed every + steps_per_loop. + optimizer: The optimizer for model. + learning_rate_fn: the learning rate schedule. + eval_fn: A callback of evaluation function, that takes a keras.Model, + current step and evaluation summary writer. + metric_fn: A metrics function returns a Keras Metric object to record + evaluation result using evaluation dataset or with training dataset + after every epoch. + init_checkpoint: Optional checkpoint to load to `sub_model` returned by + `model_fn`. + init_from_transformerxl: Whether to load to `transformerxl_model` of + `model_fn`. + model_dir: The directory of model (checkpoints, summaries). + save_steps: The frequency to save checkpoints. Every save_steps, we save a + model checkpoint. Model checkpoint will be saved and evaluation will be + conducted if evaluation dataset is provided. + run_eagerly: Whether to run training eagerly. + + Returns: + Last training step logits if training happens, otherwise returns None. + Raises: + TypeError: if model directory is not specified. + """ + required_arguments = [ + train_input_fn, total_training_steps, steps_per_loop, optimizer, + learning_rate_fn, save_steps + ] + if [arg for arg in required_arguments if arg is None]: + raise ValueError("`train_input_fn`, `total_training_steps`, " + "`steps_per_loop`, `optimizer`, `save_steps` and " + "`learning_rate_fn` are required parameters.") + if not model_dir: + raise TypeError("Model directory must be specified.") + train_iterator = data_utils.get_input_iterator(train_input_fn, strategy) + if not tf.io.gfile.exists(model_dir): + tf.io.gfile.mkdir(model_dir) + # Create summary writers + summary_dir = os.path.join(model_dir, "summaries") + if not tf.io.gfile.exists(summary_dir): + tf.io.gfile.mkdir(summary_dir) + train_summary_writer = None + eval_summary_writer = None + if eval_fn: + eval_summary_writer = tf.summary.create_file_writer( + os.path.join(summary_dir, "eval")) + if steps_per_loop >= _MIN_SUMMARY_STEPS: + # Only writes summary when the stats are collected sufficiently over + # enough steps. + train_summary_writer = tf.summary.create_file_writer( + os.path.join(summary_dir, "train")) + + with strategy.scope(): + model = model_fn() + + if init_checkpoint: + logging.info("restore from %s", init_checkpoint) + if init_from_transformerxl: + checkpoint = tf.train.Checkpoint( + transformer_xl=model.transformerxl_model) + else: + checkpoint = tf.train.Checkpoint(model=model) + checkpoint.restore(init_checkpoint) + + model.optimizer = optimizer + + if not hasattr(model, "optimizer"): + raise ValueError("User should set optimizer attribute to model.") + + train_loss_metric = tf.keras.metrics.Mean("training_loss", dtype=tf.float32) + train_metric = None + if metric_fn: + train_metric = metric_fn() + + def _replicated_step(inputs, mem=None): + """Replicated training step.""" + + inputs["mems"] = mem + with tf.GradientTape() as tape: + mem, logits = model(inputs, training=True) + loss = model.losses + train_loss_metric.update_state(loss) + if train_metric: + train_metric.update_state(inputs["label_ids"], logits) + scaled_loss = loss[0] * 1.0 / float(strategy.num_replicas_in_sync) + + # Collects training variables. + tvars = model.trainable_variables + grads = tape.gradient(scaled_loss, tvars) + clipped, _ = tf.clip_by_global_norm(grads, clip_norm=1.0) + + if input_meta_data["lr_layer_decay_rate"] != 1.0: + n_layer = 0 + for i in range(len(clipped)): + m = re.search(r"model/transformer/layer_(\d+?)/", tvars[i].name) + if not m: + continue + n_layer = max(n_layer, int(m.group(1)) + 1) + + for i in range(len(clipped)): + for l in range(n_layer): + if "model/transformer/layer_{}/".format(l) in tvars[i].name: + abs_rate = input_meta_data["lr_layer_decay_rate"]**( + n_layer - 1 - l) + clipped[i] *= abs_rate + logging.info("Apply mult {:.4f} to layer-{} grad of {}".format( + abs_rate, l, tvars[i].name)) + break + + optimizer.apply_gradients(zip(clipped, tvars)) + if input_meta_data["mem_len"] > 0: + return mem + + def train_steps(iterator, steps): + """Performs distributed training steps in a loop. + + Args: + iterator: the distributed iterator of training datasets. + steps: an tf.int32 integer tensor to specify number of steps to run + inside host training loop. + + Raises: + ValueError: Any of the arguments or tensor shapes are invalid. + + Returns: + logits: logits computed. + """ + if not isinstance(steps, tf.Tensor): + raise ValueError("steps should be an Tensor. Python object may cause " + "retracing.") + + def cache_fn(): + """Initializes memory tensor used in XLNet pretraining.""" + mems = [] + if input_meta_data["mem_len"] > 0: + for _ in range(input_meta_data["n_layer"]): + zeros = tf.zeros([ + input_meta_data["mem_len"], + input_meta_data["batch_size_per_core"], + input_meta_data["d_model"] + ], + dtype=tf.float32) + mems.append(zeros) + return mems + + if input_meta_data["mem_len"] > 0: + mem = strategy.run(cache_fn) + for _ in tf.range(steps): + mem = strategy.run( + _replicated_step, args=( + next(iterator), + mem, + )) + else: + for _ in tf.range(steps): + strategy.run(_replicated_step, args=(next(iterator),)) + + if not run_eagerly: + train_steps = tf.function(train_steps) + + logging.info("Start training...") + checkpoint = tf.train.Checkpoint(model=model, optimizer=optimizer) + latest_checkpoint_file = tf.train.latest_checkpoint(model_dir) + if latest_checkpoint_file: + logging.info("Checkpoint file %s found and restoring from checkpoint", + latest_checkpoint_file) + checkpoint.restore(latest_checkpoint_file) + logging.info("Loading from checkpoint file completed") + + current_step = optimizer.iterations.numpy() + checkpoint_name = "xlnet_step_{step}.ckpt" + + while current_step < total_training_steps: + train_loss_metric.reset_states() + if train_metric: + train_metric.reset_states() + + steps = model_training_utils.steps_to_run(current_step, save_steps, + steps_per_loop) + train_steps(train_iterator, tf.convert_to_tensor(steps, dtype=tf.int32)) + current_step += steps + train_loss = _float_metric_value(train_loss_metric) + log_stream = "Train step: %d/%d / lr = %.9f / loss = %.7f" % ( + current_step, total_training_steps, learning_rate_fn(current_step), + train_loss) + if train_metric: + log_stream += " / %s = %f" % (train_metric.name, + _float_metric_value(train_metric)) + logging.info(log_stream) + if train_summary_writer: + with train_summary_writer.as_default(): + tf.summary.scalar( + "learning_rate", + learning_rate_fn(current_step), + step=current_step) + tf.summary.scalar( + train_loss_metric.name, train_loss, step=current_step) + if train_metric: + tf.summary.scalar( + train_metric.name, + _float_metric_value(train_metric), + step=current_step) + train_summary_writer.flush() + if model_dir and current_step % save_steps == 0: + _save_checkpoint(checkpoint, model_dir, + checkpoint_name.format(step=current_step)) + + if eval_fn and current_step % save_steps == 0: + + logging.info("Running evaluation after step: %s.", current_step) + + eval_fn(model, current_step, eval_summary_writer) + if model_dir: + _save_checkpoint(checkpoint, model_dir, + checkpoint_name.format(step=current_step)) + if eval_fn: + logging.info("Running final evaluation after training is complete.") + eval_metric = eval_fn(model, current_step, eval_summary_writer) + + training_summary = { + "total_training_steps": total_training_steps, + "train_loss": _float_metric_value(train_loss_metric), + } + if train_metric: + training_summary["last_train_metrics"] = _float_metric_value(train_metric) + if eval_fn: + # eval_metric is supposed to be a float. + training_summary["eval_metrics"] = eval_metric + + model_training_utils.write_txt_summary(training_summary, summary_dir) + + return model diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/xlnet_config.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/xlnet_config.py new file mode 100644 index 0000000..7852ead --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/xlnet_config.py @@ -0,0 +1,181 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utility functions used in XLNet model.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import json +import os + +import tensorflow as tf + + +def create_run_config(is_training, is_finetune, flags): + """Helper function for creating RunConfig.""" + kwargs = dict( + is_training=is_training, + use_tpu=flags.use_tpu, + dropout=flags.dropout, + dropout_att=flags.dropout_att, + init_method=flags.init_method, + init_range=flags.init_range, + init_std=flags.init_std, + clamp_len=flags.clamp_len) + + if not is_finetune: + kwargs.update(dict( + mem_len=flags.mem_len, + reuse_len=flags.reuse_len, + bi_data=flags.bi_data, + clamp_len=flags.clamp_len, + same_length=flags.same_length)) + + return RunConfig(**kwargs) + + +# TODO(hongkuny): refactor XLNetConfig and RunConfig. +class XLNetConfig(object): + """Configs for XLNet model. + + XLNetConfig contains hyperparameters that are specific to a model checkpoint; + i.e., these hyperparameters should be the same between + pretraining and finetuning. + + The following hyperparameters are defined: + n_layer: int, the number of layers. + d_model: int, the hidden size. + n_head: int, the number of attention heads. + d_head: int, the dimension size of each attention head. + d_inner: int, the hidden size in feed-forward layers. + ff_activation: str, "relu" or "gelu". + untie_r: bool, whether to untie the biases in attention. + n_token: int, the vocab size. + """ + + def __init__(self, FLAGS=None, json_path=None, args_dict=None): + """Constructing an XLNetConfig. + + One of FLAGS or json_path should be provided. + + Args: + FLAGS: An FLAGS instance. + json_path: A path to a json config file. + args_dict: A dict for args. + """ + + assert FLAGS is not None or json_path is not None or args_dict is not None + + self.keys = ['n_layer', 'd_model', 'n_head', 'd_head', 'd_inner', + 'ff_activation', 'untie_r', 'n_token'] + + if FLAGS is not None: + self.init_from_flags(FLAGS) + + if json_path is not None: + self.init_from_json(json_path) + + if args_dict is not None: + self.init_from_dict(args_dict) + + def init_from_dict(self, args_dict): + """Constructs a `BertConfig` from a Python dictionary of parameters.""" + for key in self.keys: + setattr(self, key, args_dict[key]) + + def init_from_flags(self, flags): + for key in self.keys: + setattr(self, key, getattr(flags, key)) + + def init_from_json(self, json_path): + with tf.io.gfile.GFile(json_path) as f: + json_data = json.load(f) + self.init_from_dict(json_data) + + def to_json(self, json_path): + """Save XLNetConfig to a json file.""" + json_data = {} + for key in self.keys: + json_data[key] = getattr(self, key) + + json_dir = os.path.dirname(json_path) + if not tf.io.gfile.exists(json_dir): + tf.io.gfile.makedirs(json_dir) + with tf.io.gfile.GFile(json_path, 'w') as f: + json.dump(json_data, f, indent=4, sort_keys=True) + + +class RunConfig(object): + """Class of RunConfig. + + RunConfig contains hyperparameters that could be different + between pretraining and finetuning. + These hyperparameters can also be changed from run to run. + We store them separately from XLNetConfig for flexibility. + """ + + def __init__(self, + is_training, + use_tpu, + dropout, + dropout_att, + init_method='normal', + init_range=0.1, + init_std=0.02, + mem_len=None, + reuse_len=None, + bi_data=False, + clamp_len=-1, + same_length=False, + use_cls_mask=True): + """Initializes RunConfig. + + Args: + is_training: bool, whether in training mode. + use_tpu: bool, whether TPUs are used. + dropout: float, dropout rate. + dropout_att: float, dropout rate on attention probabilities. + init_method: str, the initialization scheme, either "normal" or "uniform". + init_range: float, initialize the parameters with a uniform distribution + in [-init_range, init_range]. Only effective when init="uniform". + init_std: float, initialize the parameters with a normal distribution + with mean 0 and stddev init_std. Only effective when init="normal". + mem_len: int, the number of tokens to cache. + reuse_len: int, the number of tokens in the currect batch to be cached + and reused in the future. + bi_data: bool, whether to use bidirectional input pipeline. + Usually set to True during pretraining and False during finetuning. + clamp_len: int, clamp all relative distances larger than clamp_len. + -1 means no clamping. + same_length: bool, whether to use the same attention length + for each token. + use_cls_mask: bool, whether to introduce cls mask. + """ + + self.init_method = init_method + self.init_range = init_range + self.init_std = init_std + self.is_training = is_training + self.dropout = dropout + self.dropout_att = dropout_att + self.use_tpu = use_tpu + self.mem_len = mem_len + self.reuse_len = reuse_len + self.bi_data = bi_data + self.clamp_len = clamp_len + self.same_length = same_length + self.use_cls_mask = use_cls_mask diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/xlnet_modeling.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/xlnet_modeling.py new file mode 100644 index 0000000..3e16af8 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/xlnet_modeling.py @@ -0,0 +1,1290 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Keras layers of XLNet model in TF 2.0.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import copy +import numpy as np + +import tensorflow as tf +from official.nlp.xlnet import data_utils + + +def gelu(x): + """Gaussian Error Linear Unit. + + This is a smoother version of the RELU. + Original paper: https://arxiv.org/abs/1606.08415 + Args: + x: float Tensor to perform activation. + + Returns: + `x` with the GELU activation applied. + """ + cdf = 0.5 * (1.0 + tf.tanh( + (np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))) + return x * cdf + + +def rel_shift(x, klen=-1): + """Performs relative shift to form the relative attention score.""" + x_size = tf.shape(x) + + x = tf.reshape(x, [x_size[1], x_size[0], x_size[2], x_size[3]]) + x = tf.slice(x, [1, 0, 0, 0], [-1, -1, -1, -1]) + x = tf.reshape(x, [x_size[0], x_size[1] - 1, x_size[2], x_size[3]]) + x = tf.slice(x, [0, 0, 0, 0], [-1, klen, -1, -1]) + + return x + + +def _get_initializer(flags): + """Get variable intializer.""" + if flags.init_method == 'uniform': + initializer = tf.keras.initializers.RandomUniform( + minval=-flags.init_range, maxval=flags.init_range) + elif flags.init_method == 'normal': + initializer = tf.keras.initializers.RandomNormal(stddev=flags.init_std) + else: + raise ValueError('Initializer {} not supported'.format(flags.init_method)) + return initializer + + +def _create_mask(qlen, mlen, dtype=tf.float32, same_length=False): + """Creates attention mask when single-side context allowed only.""" + attn_mask = tf.ones([qlen, qlen], dtype=dtype) + mask_u = tf.linalg.band_part(attn_mask, 0, -1) + mask_dia = tf.linalg.band_part(attn_mask, 0, 0) + attn_mask_pad = tf.zeros([qlen, mlen], dtype=dtype) + ret = tf.concat([attn_mask_pad, mask_u - mask_dia], 1) + if same_length: + mask_l = tf.linalg.band_part(attn_mask, -1, 0) + ret = tf.concat([ret[:, :qlen] + mask_l - mask_dia, ret[:, qlen:]], 1) + + return ret + + +def _cache_mem(curr_out, prev_mem, mem_len, reuse_len=None): + """cache hidden states into memory.""" + + if mem_len is None or mem_len == 0: + return None + else: + if reuse_len is not None and reuse_len > 0: + curr_out = curr_out[:reuse_len] + + if prev_mem is None: + new_mem = curr_out[-mem_len:] + else: + new_mem = tf.concat([prev_mem, curr_out], 0)[-mem_len:] + + return tf.keras.backend.stop_gradient(new_mem) + + +def is_special_none_tensor(tensor): + """Checks if a tensor is a special None Tensor.""" + return tensor.shape.ndims == 0 and tensor.dtype == tf.int32 + + +class PositionalEmbedding(tf.keras.layers.Layer): + """Generates relative positional embeddings used in Transformer-XL and XLNet.""" + + def __init__(self, dim, **kwargs): + super(PositionalEmbedding, self).__init__(**kwargs) + self.dim = dim + + def build(self, unused_input_shapes): + """Constructs inversed frequency vector for positional embedding layer.""" + self.inv_freq = 1.0 / (10000.0**(tf.range(0, self.dim, 2.0) / self.dim)) + super(PositionalEmbedding, self).build(unused_input_shapes) + + def call(self, pos_seq, batch_size): + """Implements call() for the layer.""" + sinusoid_inp = tf.einsum('i,d->id', pos_seq, self.inv_freq) + pos_emb = tf.concat([tf.sin(sinusoid_inp), tf.cos(sinusoid_inp)], -1) + pos_emb = pos_emb[:, None, :] + + if batch_size is not None: + pos_emb = tf.tile(pos_emb, [1, batch_size, 1]) + + return pos_emb + + +class RelativeAttention(tf.keras.layers.Layer): + """Core calculations for relative attention.""" + + def __init__(self, dropout_att, scale): + super(RelativeAttention, self).__init__() + self.scale = scale + self.dropout_att = dropout_att + + def build(self, unused_input_shapes): + """Implements build() for the layer.""" + + self.attention_probs_dropout = tf.keras.layers.Dropout( + rate=self.dropout_att) + + super(RelativeAttention, self).build(unused_input_shapes) + + def call(self, q_head, k_head_h, v_head_h, k_head_r, seg_embed, seg_mat, + r_w_bias, r_r_bias, r_s_bias, attn_mask): + """Implements call() for the layer.""" + + # content based attention score + ac = tf.einsum('ibnd,jbnd->ijbn', q_head + r_w_bias, k_head_h) + + # position based attention score + bd = tf.einsum('ibnd,jbnd->ijbn', q_head + r_r_bias, k_head_r) + bd = rel_shift(bd, klen=tf.shape(ac)[1]) + + # segment-based attention score + if seg_mat is None: + ef = 0 + else: + ef = tf.einsum('ibnd,snd->isbn', q_head + r_s_bias, seg_embed) + tgt_shape = tf.shape(bd) + ef = tf.where( + tf.broadcast_to(tf.expand_dims(seg_mat, 3), tgt_shape), + tf.broadcast_to(ef[:, 1:, :, :], tgt_shape), + tf.broadcast_to(ef[:, :1, :, :], tgt_shape)) + + # merges attention scores and performs masking + attn_score = (ac + bd + ef) * self.scale + if attn_mask is not None: + attn_score = attn_score - 1e30 * attn_mask + + # attention probability + attn_prob = tf.nn.softmax(attn_score, 1) + attn_prob = self.attention_probs_dropout(attn_prob) + + # attention output + attn_vec = tf.einsum('ijbn,jbnd->ibnd', attn_prob, v_head_h) + + return attn_vec + + +class PositionwiseFF(tf.keras.layers.Layer): + """Positionwise feed-forward layer.""" + + def __init__(self, d_model, d_inner, dropout, kernel_initializer, + activation_type, **kwargs): + super(PositionwiseFF, self).__init__(**kwargs) + self.d_model = d_model + self.d_inner = d_inner + self.dropout = dropout + self.activation_type = activation_type + self.kernel_initializer = kernel_initializer + + def build(self, unused_input_shapes): + """Implements build() for the layer.""" + if self.activation_type == 'relu': + activation = tf.nn.relu + elif self.activation_type == 'gelu': + activation = gelu + else: + raise (ValueError('Unsupported activation type {}'.format( + self.activation_type))) + self.inner_projection_layer = ( + tf.keras.layers.Dense( + units=self.d_inner, + activation=activation, + kernel_initializer=self.kernel_initializer, + name='layer_1')) + self.output_projection_layer = ( + tf.keras.layers.Dense( + units=self.d_model, + kernel_initializer=self.kernel_initializer, + name='layer_2')) + self.output_dropout = tf.keras.layers.Dropout( + rate=self.dropout, name='drop_2') + self.output_layer_norm = ( + tf.keras.layers.LayerNormalization( + name='LayerNorm', axis=-1, epsilon=1e-12)) + super(PositionwiseFF, self).build(unused_input_shapes) + + def call(self, inp): + """Implements call() for the layer.""" + + output = self.inner_projection_layer(inp) + output = self.output_projection_layer(output) + output = self.output_dropout(output) + output = self.output_layer_norm(output + inp) + return output + + +class EmbeddingLookup(tf.keras.layers.Layer): + """Looks up words embeddings for id tensor.""" + + def __init__(self, n_token, d_embed, initializer, **kwargs): + super(EmbeddingLookup, self).__init__(**kwargs) + self.n_token = n_token + self.d_embed = d_embed + self.initializer = initializer + + def build(self, unused_input_shapes): + """Implements build() for the layer.""" + self.lookup_table = self.add_weight( + 'lookup_table', + shape=[self.n_token, self.d_embed], + initializer=self.initializer, + dtype=self.dtype) + + super(EmbeddingLookup, self).build(unused_input_shapes) + + def call(self, inputs): + return tf.nn.embedding_lookup(self.lookup_table, inputs) + + +class RelativeMultiheadAttention(tf.keras.layers.Layer): + """Multi-head attention with relative embedding.""" + + def __init__(self, d_model, n_head, d_head, dropout, dropout_att, + kernel_initializer, **kwargs): + super(RelativeMultiheadAttention, self).__init__(**kwargs) + self.d_model = d_model + self.n_head = n_head + self.d_head = d_head + self.dropout = dropout + self.dropout_att = dropout_att + self.initializer = kernel_initializer + + def build(self, unused_input_shapes): + """Implements build() for the layer.""" + self.scale = 1.0 / (self.d_head**0.5) + + self.output_layer_norm = tf.keras.layers.LayerNormalization( + name='LayerNorm', axis=-1, epsilon=1e-12) + + self.kh_projection_layer = self.add_weight( + 'k/kernel', + shape=[self.d_model, self.n_head, self.d_head], + initializer=self.initializer) + self.vh_projection_layer = self.add_weight( + 'v/kernel', + shape=[self.d_model, self.n_head, self.d_head], + initializer=self.initializer) + self.kr_projection_layer = self.add_weight( + 'r/kernel', + shape=[self.d_model, self.n_head, self.d_head], + initializer=self.initializer) + self.qh_projection_layer = self.add_weight( + 'q/kernel', + shape=[self.d_model, self.n_head, self.d_head], + initializer=self.initializer) + + self.relative_attention_layer = RelativeAttention( + dropout_att=self.dropout_att, scale=self.scale) + + self.proj_o = self.add_weight( + 'o/kernel', + shape=[self.d_model, self.n_head, self.d_head], + initializer=self.initializer) + + self.attention_dropout = tf.keras.layers.Dropout(rate=self.dropout) + + super(RelativeMultiheadAttention, self).build(unused_input_shapes) + + def call(self, h, g, r, r_w_bias, r_r_bias, seg_mat, r_s_bias, seg_embed, + attn_mask_h, attn_mask_g, mems, target_mapping): + """Implements call() for the layer.""" + + if mems is not None and mems.shape.ndims > 1: + cat = tf.concat([mems, h], 0) + else: + cat = h + + # content heads + q_head_h = tf.einsum('ibh,hnd->ibnd', h, self.qh_projection_layer) + k_head_h = tf.einsum('ibh,hnd->ibnd', cat, self.kh_projection_layer) + v_head_h = tf.einsum('ibh,hnd->ibnd', cat, self.vh_projection_layer) + + # positional heads + k_head_r = tf.einsum('ibh,hnd->ibnd', r, self.kr_projection_layer) + + # core attention ops + attn_vec_h = self.relative_attention_layer(q_head_h, k_head_h, v_head_h, + k_head_r, seg_embed, seg_mat, + r_w_bias, r_r_bias, r_s_bias, + attn_mask_h) + + # post processing + output_h = tf.einsum('ibnd,hnd->ibh', attn_vec_h, self.proj_o) + output_h = self.attention_dropout(output_h) + output_h = self.output_layer_norm(output_h + h) + + output_g = None + if g is not None: # enable two-stream attention + # g-stream + q_head_g = tf.einsum('ibh,hnd->ibnd', g, self.qh_projection_layer) + if target_mapping is not None: + q_head_g = tf.einsum('mbnd,mlb->lbnd', q_head_g, target_mapping) + attn_vec_g = self.relative_attention_layer(q_head_g, k_head_h, v_head_h, + k_head_r, seg_embed, seg_mat, + r_w_bias, r_r_bias, r_s_bias, + attn_mask_g) + attn_vec_g = tf.einsum('lbnd,mlb->mbnd', attn_vec_g, target_mapping) + + else: + attn_vec_g = self.relative_attention_layer(q_head_g, k_head_h, v_head_h, + k_head_r, seg_embed, seg_mat, + r_w_bias, r_r_bias, r_s_bias, + attn_mask_g) + + # post processing + output_g = tf.einsum('ibnd,hnd->ibh', attn_vec_g, self.proj_o) + output_g = self.attention_dropout(output_g) + output_g = self.output_layer_norm(output_g + g) + + return (output_h, output_g) + + +class TransformerXLModel(tf.keras.layers.Layer): + """Defines a Transformer-XL computation graph with additional support for XLNet.""" + + def __init__(self, + n_token, + n_layer, + d_model, + n_head, + d_head, + d_inner, + dropout, + dropout_att, + attn_type, + bi_data, + is_training, + initializer, + mem_len=None, + same_length=False, + clamp_len=-1, + untie_r=False, + use_tpu=True, + reuse_len=None, + ff_activation='relu', + use_cls_mask=False, + **kwargs): + """Initializes TransformerXLModel. + + Args: + n_token: int, the number of tokens in vocabulary. + n_layer: int, the number of layers. + d_model: int, the hidden size. + n_head: int, the number of attention heads. + d_head: int, the dimension size of each attention head. + d_inner: int, the hidden size in feed-forward layers. + dropout: float, dropout rate. + dropout_att: float, dropout rate on attention probabilities. + attn_type: str, "uni" or "bi". + bi_data: bool, whether to use bidirectional input pipeline. Usually set to + True during pretraining and False during finetuning. + is_training: bool, whether in training mode. + initializer: A tf initializer. + mem_len: int, the number of tokens to cache. + same_length: bool, whether to use the same attention length for each + token. + clamp_len: int, clamp all relative distances larger than clamp_len. -1 + means no clamping. + untie_r: bool, whether to untie the biases in attention. + use_tpu: bool, whether TPUs are used. + reuse_len: int, the number of tokens in the currect batch to be cached and + reused in the future. + ff_activation: str, "relu" or "gelu". + use_cls_mask: bool, whether to introduce cls mask. + **kwargs: Other parameters. + """ + + super(TransformerXLModel, self).__init__(**kwargs) + + self.n_token = n_token + self.initializer = initializer + self.attn_type = attn_type + self.n_layer = n_layer + self.d_model = d_model + self.n_head = n_head + self.d_head = d_head + self.d_inner = d_inner + self.ff_activation = ff_activation + self.untie_r = untie_r + self.use_tpu = use_tpu + self.dropout = dropout + self.dropout_att = dropout_att + + self.mem_len = mem_len + self.reuse_len = reuse_len + self.bi_data = bi_data + self.clamp_len = clamp_len + self.same_length = same_length + self.use_cls_mask = use_cls_mask + + def build(self, unused_input_shapes): + """Implements build() for the layer.""" + self.tf_float = tf.float32 + + self.embedding_lookup = EmbeddingLookup( + n_token=self.n_token, + d_embed=self.d_model, + initializer=self.initializer, + dtype=self.tf_float, + name='word_embedding') + + self.h_dropout = tf.keras.layers.Dropout(rate=self.dropout) + self.g_dropout = tf.keras.layers.Dropout(rate=self.dropout) + + if self.untie_r: + self.r_w_bias = ( + self.add_weight( + 'r_w_bias', + shape=[self.n_layer, self.n_head, self.d_head], + dtype=self.tf_float, + initializer=self.initializer)) + self.r_r_bias = ( + self.add_weight( + 'r_r_bias', + shape=[self.n_layer, self.n_head, self.d_head], + dtype=self.tf_float, + initializer=self.initializer)) + self.r_s_bias = ( + self.add_weight( + 'r_s_bias', + shape=[self.n_layer, self.n_head, self.d_head], + dtype=self.tf_float, + initializer=self.initializer)) + else: + self.r_w_bias = ( + self.add_weight( + 'r_w_bias', + shape=[self.n_head, self.d_head], + dtype=self.tf_float, + initializer=self.initializer)) + self.r_r_bias = ( + self.add_weight( + 'r_r_bias', + shape=[self.n_head, self.d_head], + dtype=self.tf_float, + initializer=self.initializer)) + self.r_s_bias = ( + self.add_weight( + 'r_s_bias', [self.n_head, self.d_head], + dtype=self.tf_float, + initializer=self.initializer)) + + self.seg_embed = self.add_weight( + 'seg_embed', [self.n_layer, 2, self.n_head, self.d_head], + dtype=self.tf_float, + initializer=self.initializer) + + self.mask_emb = self.add_weight( + 'mask_emb/mask_emb', shape=[1, 1, self.d_model], dtype=self.tf_float) + + self.emb_dropout = tf.keras.layers.Dropout(rate=self.dropout) + self.fwd_position_embedding = PositionalEmbedding(self.d_model) + self.bwd_position_embedding = PositionalEmbedding(self.d_model) + + self.rel_multihead_layers = [] + self.h_positionwise_ffn_layers = [] + for i in range(self.n_layer): + self.rel_multihead_layers.append( + RelativeMultiheadAttention( + d_model=self.d_model, + dropout=self.dropout, + n_head=self.n_head, + d_head=self.d_head, + dropout_att=self.dropout_att, + kernel_initializer=self.initializer, + name='layer_%d/rel_attn' % (i))) + self.h_positionwise_ffn_layers.append( + PositionwiseFF( + d_model=self.d_model, + d_inner=self.d_inner, + dropout=self.dropout, + kernel_initializer=self.initializer, + activation_type=self.ff_activation, + name='layer_%d/ff' % (i))) + + self.output_dropout = tf.keras.layers.Dropout(rate=self.dropout) + + super(TransformerXLModel, self).build(unused_input_shapes) + + def __call__(self, + inp_k, + seg_id=None, + input_mask=None, + mems=None, + perm_mask=None, + target_mapping=None, + inp_q=None, + **kwargs): + # Uses dict to feed inputs into call() in order to keep mems as a python + # list. + inputs = { + 'inp_k': inp_k, + 'seg_id': seg_id, + 'input_mask': input_mask, + 'mems': mems, + 'perm_mask': perm_mask, + 'target_mapping': target_mapping, + 'inp_q': inp_q + } + return super(TransformerXLModel, self).__call__(inputs, **kwargs) + + def call(self, inputs): + """Implements call() for the layer.""" + inp_k = inputs['inp_k'] + seg_id = inputs['seg_id'] + input_mask = inputs['input_mask'] + mems = inputs['mems'] + perm_mask = inputs['perm_mask'] + target_mapping = inputs['target_mapping'] + inp_q = inputs['inp_q'] + + new_mems = [] + + bsz = tf.shape(inp_k)[1] + + qlen = inp_k.shape.as_list()[0] + + mlen = mems[0].shape.as_list()[0] if mems is not None else 0 + klen = mlen + qlen + + ##### Attention mask + # causal attention mask + if self.attn_type == 'uni': + attn_mask = _create_mask(qlen, mlen, self.tf_float, self.same_length) + # pylint: enable=protected-access + attn_mask = attn_mask[:, :, None, None] + elif self.attn_type == 'bi': + attn_mask = None + else: + raise ValueError('Unsupported attention type: {}'.format(self.attn_type)) + + # data mask: input mask & perm mask + if input_mask is not None and perm_mask is not None: + data_mask = input_mask[None] + perm_mask + + elif input_mask is not None and perm_mask is None: + data_mask = input_mask[None] + elif input_mask is None and perm_mask is not None: + data_mask = perm_mask + else: + data_mask = None + + if data_mask is not None: + # all mems can be attended to + mems_mask = tf.zeros([tf.shape(data_mask)[0], mlen, bsz], + dtype=self.tf_float) + data_mask = tf.concat([mems_mask, data_mask], 1) + if attn_mask is None: + attn_mask = data_mask[:, :, :, None] + else: + attn_mask += data_mask[:, :, :, None] + + if attn_mask is not None: + attn_mask = tf.cast(attn_mask > 0, dtype=self.tf_float) + + if attn_mask is not None: + non_tgt_mask = -tf.eye(qlen, dtype=self.tf_float) + non_tgt_mask = tf.concat( + [tf.zeros([qlen, mlen], dtype=self.tf_float), non_tgt_mask], axis=-1) + non_tgt_mask = tf.cast( + (attn_mask + non_tgt_mask[:, :, None, None]) > 0, dtype=self.tf_float) + else: + non_tgt_mask = None + + word_emb_k = self.embedding_lookup(inp_k) + + if inp_q is not None: + if target_mapping is not None: + word_emb_q = tf.tile(self.mask_emb, + [tf.shape(target_mapping)[0], bsz, 1]) + else: + inp_q_ext = inp_q[:, :, None] + word_emb_q = inp_q_ext * self.mask_emb + (1 - inp_q_ext) * word_emb_k + + output_h = self.h_dropout(word_emb_k) + output_g = None + if inp_q is not None: + output_g = self.g_dropout(word_emb_q) + + ##### Segment embedding + if seg_id is not None: + + # Convert `seg_id` to one-hot `seg_mat` + + mem_pad = tf.zeros([mlen, bsz], dtype=tf.int32) + + cat_id = tf.concat([mem_pad, seg_id], 0) + + if self.use_cls_mask: + # `1` indicates not in the same segment [qlen x klen x bsz] + # seg_id: [qlen x bsz] & cat_id: [klen x bsz] + cls_mat = tf.logical_or( + tf.equal(seg_id, tf.constant([data_utils.SEG_ID_CLS]))[:, None], + tf.equal(cat_id, tf.constant([data_utils.SEG_ID_CLS]))[None, :]) + seg_mat = tf.equal(seg_id[:, None], cat_id[None, :]) + seg_mat = tf.logical_or(cls_mat, seg_mat) + else: + seg_mat = tf.logical_not(tf.equal(seg_id[:, None], cat_id[None, :])) + else: + seg_mat = None + + dtype = self.tf_float + freq_seq = tf.range(0, self.d_model, 2.0) + if dtype is not None and dtype != tf.float32: + freq_seq = tf.cast(freq_seq, dtype=self.dtype) + + if self.attn_type == 'bi': + beg, end = klen, -qlen + elif self.attn_type == 'uni': + beg, end = klen, -1 + else: + raise ValueError('Unknown `attn_type` {}.'.format(self.attn_type)) + + if self.bi_data: + fwd_pos_seq = tf.range(beg, end, -1.0) + bwd_pos_seq = tf.range(-beg, -end, 1.0) + + if dtype is not None and dtype != tf.float32: + fwd_pos_seq = tf.cast(fwd_pos_seq, dtype=dtype) + bwd_pos_seq = tf.cast(bwd_pos_seq, dtype=dtype) + + if self.clamp_len > 0: + fwd_pos_seq = tf.clip_by_value(fwd_pos_seq, -self.clamp_len, + self.clamp_len) + bwd_pos_seq = tf.clip_by_value(bwd_pos_seq, -self.clamp_len, + self.clamp_len) + + if bsz is not None: + fwd_pos_emb = self.fwd_position_embedding(fwd_pos_seq, bsz // 2) + bwd_pos_emb = self.bwd_position_embedding(bwd_pos_seq, bsz // 2) + else: + fwd_pos_emb = self.fwd_position_embedding(fwd_pos_seq, None) + bwd_pos_emb = self.bwd_position_embedding(bwd_pos_seq, None) + + pos_emb = tf.concat([fwd_pos_emb, bwd_pos_emb], axis=1) + else: + fwd_pos_seq = tf.range(beg, end, -1.0) + if dtype is not None and dtype != tf.float32: + fwd_pos_seq = tf.cast(fwd_pos_seq, dtype=dtype) + if self.clamp_len > 0: + fwd_pos_seq = tf.clip_by_value(fwd_pos_seq, -self.clamp_len, + self.lamp_len) + + pos_emb = self.fwd_position_embedding(fwd_pos_seq, bsz) + + pos_emb = self.emb_dropout(pos_emb) + + if mems is None: + mems = [None] * self.n_layer + for i in range(self.n_layer): + # cache new mems + new_mems.append( + _cache_mem(output_h, mems[i], self.mem_len, self.reuse_len)) + # pylint: enable=protected-access + + # segment bias + if seg_id is None: + r_s_bias_i = None + seg_embed_i = None + else: + r_s_bias_i = self.r_s_bias if not self.untie_r else self.r_s_bias[i] + seg_embed_i = self.seg_embed[i] + + ffn_layer = self.h_positionwise_ffn_layers[i] + attention_layer = self.rel_multihead_layers[i] + output_h, output_g = attention_layer( + h=output_h, + g=output_g, + r=pos_emb, + r_w_bias=self.r_w_bias if not self.untie_r else self.r_w_bias[i], + r_r_bias=self.r_r_bias if not self.untie_r else self.r_r_bias[i], + seg_mat=seg_mat, + r_s_bias=r_s_bias_i, + seg_embed=seg_embed_i, + attn_mask_h=non_tgt_mask, + attn_mask_g=attn_mask, + mems=mems[i], + target_mapping=target_mapping) + output_h = ffn_layer(output_h) + if output_g is not None: + output_g = ffn_layer(output_g) + + if inp_q is not None: + output = output_g + else: + output = output_h + + return output, new_mems, None + + +class PretrainingXLNetModel(tf.keras.Model): + """XLNet keras model combined with pretraining LM loss layer. + + See the original paper: https://arxiv.org/pdf/1906.08237.pdf + + """ + + def __init__(self, use_proj, xlnet_config, run_config, **kwargs): + super(PretrainingXLNetModel, self).__init__(**kwargs) + self.run_config = run_config + self.initializer = _get_initializer(run_config) + self.xlnet_config = copy.deepcopy(xlnet_config) + + self.transformerxl_model = TransformerXLModel( + n_token=self.xlnet_config.n_token, + initializer=self.initializer, + attn_type='bi', + n_layer=self.xlnet_config.n_layer, + d_model=self.xlnet_config.d_model, + n_head=self.xlnet_config.n_head, + d_head=self.xlnet_config.d_head, + d_inner=self.xlnet_config.d_inner, + ff_activation=self.xlnet_config.ff_activation, + untie_r=self.xlnet_config.untie_r, + is_training=self.run_config.is_training, + use_tpu=self.run_config.use_tpu, + dropout=self.run_config.dropout, + dropout_att=self.run_config.dropout_att, + mem_len=self.run_config.mem_len, + reuse_len=self.run_config.reuse_len, + bi_data=self.run_config.bi_data, + clamp_len=self.run_config.clamp_len, + same_length=self.run_config.same_length, + use_cls_mask=self.run_config.use_cls_mask, + name='transformer') + self.lmloss_layer = LMLossLayer( + n_token=self.xlnet_config.n_token, + d_model=self.xlnet_config.d_model, + initializer=self.initializer, + tie_weight=True, + bi_data=self.run_config.bi_data, + use_tpu=self.run_config.use_tpu, + use_proj=use_proj, + name='lm_loss') + + def call(self, features): + """Implements call() for the layer.""" + + input_ids = tf.transpose(features['input_k'], [1, 0]) + inp_q = tf.transpose(features['input_q'], [1, 0]) + + seg_ids = tf.transpose(features['seg_id'], [1, 0]) + + perm_mask = tf.transpose(features['perm_mask'], [1, 2, 0]) + + target_mapping = tf.transpose(features['target_mapping'], [1, 2, 0]) + + # target for LM loss + target = tf.transpose(features['target'], [1, 0]) + + # target mask for LM loss + tgt_mask = tf.transpose(features['target_mask'], [1, 0]) + + mems = features.get('mems', None) + + transformerxl_output, self.new_mems, self.lookup_table = self.transformerxl_model( + input_ids, + seg_id=seg_ids, + input_mask=None, + mems=mems, + perm_mask=perm_mask, + target_mapping=target_mapping, + inp_q=inp_q) + lm_loss, _ = self.lmloss_layer( + hidden=transformerxl_output, + target=target, + lookup_table=self.transformerxl_model.embedding_lookup.lookup_table, + target_mask=tgt_mask) + self.add_loss(lm_loss) + return self.new_mems, transformerxl_output + + +class ClassificationXLNetModel(tf.keras.Model): + """XLNet keras model combined with classification loss layer. + + See the original paper: https://arxiv.org/pdf/1906.08237.pdf + + """ + + def __init__(self, xlnet_config, run_config, n_class, summary_type, **kwargs): + super(ClassificationXLNetModel, self).__init__(**kwargs) + self.run_config = run_config + self.initializer = _get_initializer(run_config) + self.xlnet_config = copy.deepcopy(xlnet_config) + + self.transformerxl_model = TransformerXLModel( + n_token=self.xlnet_config.n_token, + initializer=self.initializer, + attn_type='bi', + n_layer=self.xlnet_config.n_layer, + d_model=self.xlnet_config.d_model, + n_head=self.xlnet_config.n_head, + d_head=self.xlnet_config.d_head, + d_inner=self.xlnet_config.d_inner, + ff_activation=self.xlnet_config.ff_activation, + untie_r=self.xlnet_config.untie_r, + is_training=self.run_config.is_training, + use_tpu=self.run_config.use_tpu, + dropout=self.run_config.dropout, + dropout_att=self.run_config.dropout_att, + mem_len=self.run_config.mem_len, + reuse_len=self.run_config.reuse_len, + bi_data=self.run_config.bi_data, + clamp_len=self.run_config.clamp_len, + same_length=self.run_config.same_length, + name='transformer') + + self.summarization_layer = Summarization( + d_model=self.xlnet_config.d_model, + n_head=self.xlnet_config.n_head, + d_head=self.xlnet_config.d_head, + dropout=self.run_config.dropout, + dropout_att=self.run_config.dropout_att, + initializer=self.initializer, + use_proj=True, + summary_type=summary_type, + name='sequence_summary') + + self.cl_loss_layer = ClassificationLossLayer( + n_class=n_class, initializer=self.initializer, name='classification') + + def call(self, features): + """Implements call() for the layer.""" + bsz_per_core = tf.shape(features['input_ids'])[0] + + input_ids = tf.transpose(features['input_ids'], [1, 0]) + seg_ids = tf.transpose(features['segment_ids'], [1, 0]) + input_mask = tf.transpose(features['input_mask'], [1, 0]) + + label = tf.reshape(features['label_ids'], [bsz_per_core]) + + mems = features.get('mems', None) + + transformerxl_output, new_mems, self.lookup_table = ( + self.transformerxl_model(input_ids, seg_ids, input_mask, mems)) + + summary = self.summarization_layer(transformerxl_output) + per_example_loss, logits = self.cl_loss_layer(hidden=summary, labels=label) + self.add_loss(tf.keras.backend.mean(per_example_loss)) + return new_mems, logits + + +class LMLossLayer(tf.keras.layers.Layer): + """Layer computing cross entropy loss for language modeling.""" + + def __init__(self, + n_token, + d_model, + initializer, + tie_weight=False, + bi_data=True, + use_tpu=False, + use_proj=False, + **kwargs): + """Constructs LMLoss layer. + + Args: + n_token: Number of tokens in vocabulary. + d_model: The dimension of model hidden state. + initializer: Initializer used for parameters. + tie_weight: Whether to share weights between embedding lookup layer and + next-token prediction layer. + bi_data: Whether to use bidirectional input pipeline. Usually set to True + during pretraining and False during finetuning. + use_tpu: bool, whether to use TPU. + use_proj: bool, whether to add a projection layer before LM prediction. + **kwargs: Other parameters. + """ + super(LMLossLayer, self).__init__(**kwargs) + self.n_token = n_token + self.d_model = d_model + self.initializer = initializer + + self.tie_weight = tie_weight + self.bi_data = bi_data + self.use_tpu = use_tpu + self.use_proj = use_proj + + def build(self, unused_input_shapes): + """Implements build() for the layer.""" + if self.use_proj: + self.proj_layer = tf.keras.layers.Dense( + units=self.d_model, + kernel_initializer=self.initializer, + activation=gelu, + name='lm_projection/dense') + self.proj_layer_norm = tf.keras.layers.LayerNormalization( + axis=-1, epsilon=1e-12, name='lm_projection/LayerNorm') + if not self.tie_weight: + self.softmax_w = self.add_weight( + 'weight', + shape=[self.n_token, self.d_model], + initializer=self.initializer) + + self.softmax_b = self.add_weight( + 'bias', shape=[self.n_token], initializer=tf.zeros_initializer()) + + super(LMLossLayer, self).build(unused_input_shapes) + + def call(self, hidden, target, lookup_table, target_mask): + """Implements call() for the layer.""" + if self.use_proj: + hidden = self.proj_layer_norm(self.proj_layer(hidden)) + if self.tie_weight: + logits = tf.einsum('ibd,nd->ibn', hidden, lookup_table) + self.softmax_b + else: + logits = tf.einsum('ibd,nd->ibn', hidden, self.softmax_w) + self.softmax_b + + if self.use_tpu: + one_hot_target = tf.one_hot(target, self.n_token, dtype=logits.dtype) + loss = -tf.reduce_sum(tf.nn.log_softmax(logits) * one_hot_target, -1) + else: + loss = tf.nn.sparse_softmax_cross_entropy_with_logits( + labels=target, logits=logits) + + total_loss = tf.reduce_sum(loss * target_mask) / tf.reduce_sum(target_mask) + + return total_loss, logits + + +class Summarization(tf.keras.layers.Layer): + """The layer to pool the output from XLNet model into a vector.""" + + def __init__(self, + d_model, + n_head, + d_head, + dropout, + dropout_att, + initializer, + use_proj=True, + summary_type='last', + **kwargs): + """Constructs Summarization layer. + + Args: + d_model: int, the dimension of model hidden state. + n_head: int, the number of attention heads. + d_head: int, the dimension size of each attention head. + dropout: float, dropout rate. + dropout_att: float, dropout rate on attention probabilities. + initializer: Initializer used for parameters. + use_proj: bool, whether to use projection layer for summarization. + summary_type: Method used to summarize a sequence into a compact vector. + **kwargs: Other parameters. + """ + super(Summarization, self).__init__(**kwargs) + self.d_model = d_model + self.n_head = n_head + self.d_head = d_head + self.initializer = initializer + + self.dropout = dropout + self.dropout_att = dropout_att + self.use_proj = use_proj + self.summary_type = summary_type + + def build(self, unused_input_shapes): + """Implements build() for the layer.""" + if self.use_proj: + self.proj_layer = tf.keras.layers.Dense( + units=self.d_model, + kernel_initializer=self.initializer, + activation=tf.nn.tanh, + name='summary') + self.dropout_layer = tf.keras.layers.Dropout(rate=self.dropout) + + super(Summarization, self).build(unused_input_shapes) + + def call(self, inputs): + """Implements call() for the layer.""" + if self.summary_type == 'last': + summary = inputs[-1] + elif self.summary_type == 'first': + summary = inputs[0] + else: + raise ValueError('Invalid summary type provided: %s' % self.summary_type) + if self.use_proj: + summary = self.proj_layer(summary) + summary = self.dropout_layer(summary) + return summary + + +class ClassificationLossLayer(tf.keras.layers.Layer): + """Layer computing cross entropy loss for classification task.""" + + def __init__(self, n_class, initializer, **kwargs): + """Constructs Summarization layer. + + Args: + n_class: Number of tokens in vocabulary. + initializer: Initializer used for parameters. + **kwargs: Other parameters. + """ + super(ClassificationLossLayer, self).__init__(**kwargs) + + self.n_class = n_class + self.initializer = initializer + + def build(self, unused_input_shapes): + """Implements build() for the layer.""" + self.proj_layer = tf.keras.layers.Dense( + units=self.n_class, kernel_initializer=self.initializer, name='logit') + + super(ClassificationLossLayer, self).build(unused_input_shapes) + + def call(self, hidden, labels): + """Implements call() for the layer.""" + + logits = self.proj_layer(hidden) + one_hot_target = tf.one_hot(labels, self.n_class, dtype=hidden.dtype) # pytype: disable=attribute-error + loss = -tf.reduce_sum(tf.nn.log_softmax(logits) * one_hot_target, -1) + + return loss, logits + + +class QAXLNetModel(tf.keras.Model): + """XLNet keras model combined with question answering loss layer. + + See the original paper: https://arxiv.org/pdf/1906.08237.pdf + + """ + + def __init__(self, xlnet_config, run_config, start_n_top, end_n_top, + **kwargs): + super(QAXLNetModel, self).__init__(**kwargs) + self.run_config = run_config + self.initializer = _get_initializer(run_config) + self.xlnet_config = copy.deepcopy(xlnet_config) + + self.transformerxl_model = TransformerXLModel( + n_token=self.xlnet_config.n_token, + initializer=self.initializer, + attn_type='bi', + n_layer=self.xlnet_config.n_layer, + d_model=self.xlnet_config.d_model, + n_head=self.xlnet_config.n_head, + d_head=self.xlnet_config.d_head, + d_inner=self.xlnet_config.d_inner, + ff_activation=self.xlnet_config.ff_activation, + untie_r=self.xlnet_config.untie_r, + is_training=self.run_config.is_training, + use_tpu=self.run_config.use_tpu, + dropout=self.run_config.dropout, + dropout_att=self.run_config.dropout_att, + mem_len=self.run_config.mem_len, + reuse_len=self.run_config.reuse_len, + bi_data=self.run_config.bi_data, + clamp_len=self.run_config.clamp_len, + same_length=self.run_config.same_length, + name='transformer') + + self.qa_loss_layer = QALossLayer( + d_model=self.xlnet_config.d_model, + start_n_top=start_n_top, + end_n_top=end_n_top, + initializer=self.initializer, + dropout=self.run_config.dropout) + + def call(self, features, training=False): + """Implements call() for the layer.""" + + input_ids = tf.transpose(features['input_ids'], [1, 0]) + seg_ids = tf.transpose(features['segment_ids'], [1, 0]) + input_mask = tf.transpose(features['input_mask'], [1, 0]) + + cls_index = tf.reshape(features['cls_index'], [-1]) + p_mask = features['p_mask'] + + transformerxl_output, new_mems, self.lookup_table = ( + self.transformerxl_model(input_ids, seg_ids, input_mask)) + + if training: + loss, logits = self.qa_loss_layer( + hidden=transformerxl_output, + p_mask=p_mask, + cls_index=cls_index, + start_positions=features['start_positions'], + end_positions=features['end_positions'], + is_impossible=features['is_impossible']) + self.add_loss(loss) + return new_mems, logits + else: + results = self.qa_loss_layer( + hidden=transformerxl_output, p_mask=p_mask, cls_index=cls_index) + return results + + +class QALossLayer(tf.keras.layers.Layer): + """Layer computing position and regression loss for question answering task.""" + + def __init__(self, d_model, start_n_top, end_n_top, initializer, dropout, + **kwargs): + """Constructs Summarization layer. + + Args: + d_model: Int, the hidden size. + start_n_top: Beam size for span start. + end_n_top: Beam size for span end. + initializer: Initializer used for parameters. + dropout: float, dropout rate. + **kwargs: Other parameters. + """ + super(QALossLayer, self).__init__(**kwargs) + self.d_model = d_model + self.start_n_top = start_n_top + self.end_n_top = end_n_top + self.initializer = initializer + self.dropout = dropout + + def build(self, unused_input_shapes): + """Implements build() for the layer.""" + self.start_logits_proj_layer = tf.keras.layers.Dense( + units=1, kernel_initializer=self.initializer, name='start_logits/dense') + self.end_logits_proj_layer0 = tf.keras.layers.Dense( + units=self.d_model, + kernel_initializer=self.initializer, + activation=tf.nn.tanh, + name='end_logits/dense_0') + self.end_logits_proj_layer1 = tf.keras.layers.Dense( + units=1, kernel_initializer=self.initializer, name='end_logits/dense_1') + self.end_logits_layer_norm = tf.keras.layers.LayerNormalization( + axis=-1, epsilon=1e-12, name='end_logits/LayerNorm') + self.answer_class_proj_layer0 = tf.keras.layers.Dense( + units=self.d_model, + kernel_initializer=self.initializer, + activation=tf.nn.tanh, + name='answer_class/dense_0') + self.answer_class_proj_layer1 = tf.keras.layers.Dense( + units=1, + kernel_initializer=self.initializer, + use_bias=False, + name='answer_class/dense_1') + self.ans_feature_dropout = tf.keras.layers.Dropout(rate=self.dropout) + super(QALossLayer, self).build(unused_input_shapes) + + def __call__(self, hidden, p_mask, cls_index, **kwargs): + return super(QALossLayer, self).__call__( + (hidden, p_mask, cls_index, kwargs)) + + def call(self, inputs, training=False): + """Implements call() for the layer.""" + hidden, p_mask, cls_index, kwargs = inputs + return_dict = {} + seq_len = tf.shape(hidden)[0] + + start_logits = self.start_logits_proj_layer(hidden) + start_logits = tf.transpose(tf.squeeze(start_logits, -1), [1, 0]) + start_logits_masked = start_logits * (1 - p_mask) - 1e30 * p_mask + start_log_probs = tf.nn.log_softmax(start_logits_masked, -1) + if training: + start_positions = kwargs['start_positions'] + end_positions = kwargs['end_positions'] + is_impossible = kwargs['is_impossible'] + start_positions = tf.reshape(start_positions, [-1]) + start_index = tf.one_hot( + start_positions, depth=seq_len, axis=-1, dtype=tf.float32) + start_features = tf.einsum('lbh,bl->bh', hidden, start_index) + start_features = tf.tile(start_features[None], [seq_len, 1, 1]) + end_logits = self.end_logits_proj_layer0( + tf.concat([hidden, start_features], axis=-1)) + + end_logits = self.end_logits_layer_norm(end_logits) + + end_logits = self.end_logits_proj_layer1(end_logits) + end_logits = tf.transpose(tf.squeeze(end_logits, -1), [1, 0]) + end_logits_masked = end_logits * (1 - p_mask) - 1e30 * p_mask + end_log_probs = tf.nn.log_softmax(end_logits_masked, -1) + else: + # during inference, compute the end logits based on beam search + + start_top_log_probs, start_top_index = tf.nn.top_k( + start_log_probs, k=self.start_n_top) + start_index = tf.one_hot( + start_top_index, depth=seq_len, axis=-1, dtype=tf.float32) + start_features = tf.einsum('lbh,bkl->bkh', hidden, start_index) + end_input = tf.tile(hidden[:, :, None], [1, 1, self.start_n_top, 1]) + start_features = tf.tile(start_features[None], [seq_len, 1, 1, 1]) + end_input = tf.concat([end_input, start_features], axis=-1) + end_logits = self.end_logits_proj_layer0(end_input) + end_logits = tf.reshape(end_logits, [seq_len, -1, self.d_model]) + end_logits = self.end_logits_layer_norm(end_logits) + + end_logits = tf.reshape(end_logits, + [seq_len, -1, self.start_n_top, self.d_model]) + + end_logits = self.end_logits_proj_layer1(end_logits) + end_logits = tf.reshape(end_logits, [seq_len, -1, self.start_n_top]) + end_logits = tf.transpose(end_logits, [1, 2, 0]) + end_logits_masked = end_logits * ( + 1 - p_mask[:, None]) - 1e30 * p_mask[:, None] + end_log_probs = tf.nn.log_softmax(end_logits_masked, -1) + end_top_log_probs, end_top_index = tf.nn.top_k( + end_log_probs, k=self.end_n_top) + end_top_log_probs = tf.reshape(end_top_log_probs, + [-1, self.start_n_top * self.end_n_top]) + end_top_index = tf.reshape(end_top_index, + [-1, self.start_n_top * self.end_n_top]) + + if training: + return_dict['start_log_probs'] = start_log_probs + return_dict['end_log_probs'] = end_log_probs + else: + return_dict['start_top_log_probs'] = start_top_log_probs + return_dict['start_top_index'] = start_top_index + return_dict['end_top_log_probs'] = end_top_log_probs + return_dict['end_top_index'] = end_top_index + # an additional layer to predict answerability + + # get the representation of CLS + cls_index = tf.one_hot(cls_index, seq_len, axis=-1, dtype=tf.float32) + cls_feature = tf.einsum('lbh,bl->bh', hidden, cls_index) + + # get the representation of START + start_p = tf.nn.softmax(start_logits_masked, axis=-1, name='softmax_start') + start_feature = tf.einsum('lbh,bl->bh', hidden, start_p) + + ans_feature = tf.concat([start_feature, cls_feature], -1) + ans_feature = self.answer_class_proj_layer0(ans_feature) + ans_feature = self.ans_feature_dropout(ans_feature) + cls_logits = self.answer_class_proj_layer1(ans_feature) + cls_logits = tf.squeeze(cls_logits, -1) + return_dict['cls_logits'] = cls_logits + + if not training: + return return_dict + + def compute_loss(log_probs, positions): + one_hot_positions = tf.one_hot(positions, depth=seq_len, dtype=tf.float32) + + loss = -tf.reduce_sum(one_hot_positions * log_probs, axis=-1) + loss = tf.reduce_mean(loss) + return loss + + start_loss = compute_loss(start_log_probs, start_positions) + end_loss = compute_loss(end_log_probs, end_positions) + + total_loss = (start_loss + end_loss) * 0.5 + + is_impossible = tf.reshape(is_impossible, [-1]) + regression_loss = tf.nn.sigmoid_cross_entropy_with_logits( + labels=is_impossible, logits=cls_logits) + regression_loss = tf.reduce_mean(regression_loss) + + total_loss += regression_loss * 0.5 + return total_loss, cls_logits diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/xlnet_modeling_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/xlnet_modeling_test.py new file mode 100644 index 0000000..dce887a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/nlp/xlnet/xlnet_modeling_test.py @@ -0,0 +1,52 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import logging +import numpy as np +import tensorflow as tf + +from official.nlp.xlnet import xlnet_modeling + + +class PositionalEmbeddingLayerTest(tf.test.TestCase): + + def test_positional_embedding(self): + """A low-dimensional example is tested. + + With len(pos_seq)=2 and d_model=4: + + pos_seq = [[1.], [0.]] + inv_freq = [1., 0.01] + pos_seq x inv_freq = [[1, 0.01], [0., 0.]] + pos_emb = [[sin(1.), sin(0.01), cos(1.), cos(0.01)], + [sin(0.), sin(0.), cos(0.), cos(0.)]] + = [[0.84147096, 0.00999983, 0.54030228, 0.99994999], + [0., 0., 1., 1.]] + """ + target = np.array([[[0.84147096, 0.00999983, 0.54030228, 0.99994999]], + [[0., 0., 1., 1.]]]) + d_model = 4 + pos_seq = tf.range(1, -1, -1.0) # [1., 0.] + pos_emb_layer = xlnet_modeling.PositionalEmbedding(d_model) + pos_emb = pos_emb_layer(pos_seq, batch_size=None).numpy().astype(float) + + logging.info(pos_emb) + self.assertAllClose(pos_emb, target) + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/pip_package/setup.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/pip_package/setup.py new file mode 100644 index 0000000..e708bc4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/pip_package/setup.py @@ -0,0 +1,89 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Sets up TensorFlow Official Models.""" +import datetime +import os +import sys + +from setuptools import find_packages +from setuptools import setup + +version = '2.2.0' + +project_name = 'tf-models-official' + +long_description = """The TensorFlow official models are a collection of +models that use TensorFlow's high-level APIs. +They are intended to be well-maintained, tested, and kept up to date with the +latest TensorFlow API. They should also be reasonably optimized for fast +performance while still being easy to read.""" + +if '--project_name' in sys.argv: + project_name_idx = sys.argv.index('--project_name') + project_name = sys.argv[project_name_idx + 1] + sys.argv.remove('--project_name') + sys.argv.pop(project_name_idx) + + +def _get_requirements(): + """Parses requirements.txt file.""" + install_requires_tmp = [] + dependency_links_tmp = [] + with open( + os.path.join(os.path.dirname(__file__), '../requirements.txt'), 'r') as f: + for line in f: + package_name = line.strip() + if package_name.startswith('-e '): + dependency_links_tmp.append(package_name[3:].strip()) + else: + install_requires_tmp.append(package_name) + return install_requires_tmp, dependency_links_tmp + +install_requires, dependency_links = _get_requirements() + +if project_name == 'tf-models-nightly': + version += '.dev' + datetime.datetime.now().strftime('%Y%m%d') + install_requires.append('tf-nightly') +else: + install_requires.append('tensorflow>=2.1.0') + +print('install_requires: ', install_requires) +print('dependency_links: ', dependency_links) + +setup( + name=project_name, + version=version, + description='TensorFlow Official Models', + long_description=long_description, + author='Google Inc.', + author_email='no-reply@google.com', + url='https://github.com/tensorflow/models', + license='Apache 2.0', + packages=find_packages(exclude=[ + 'research*', + 'tutorials*', + 'samples*', + 'official.r1*', + 'official.pip_package*', + 'official.benchmark*', + 'official.colab*', + ]), + exclude_package_data={ + '': ['*_test.py',], + }, + install_requires=install_requires, + dependency_links=dependency_links, + python_requires='>=3.6', +) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/README.md new file mode 100644 index 0000000..7251417 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/README.md @@ -0,0 +1,23 @@ +![No Maintenance Intended](https://img.shields.io/badge/No%20Maintenance%20Intended-%E2%9C%95-red.svg) +![TensorFlow Requirement: 1.x](https://img.shields.io/badge/TensorFlow%20Requirement-1.x-brightgreen) +![TensorFlow 2 Not Supported](https://img.shields.io/badge/TensorFlow%202%20Not%20Supported-%E2%9C%95-red.svg) + +# Legacy Models + +The **r1** folder contains legacy model implementations developed +using TensorFlow 1.x. + +**Note: We will remove this r1 folder from the master branch in June, 2020.** + +After removal, you will still be able to access legacy models +in the previous releases. +(e.g., [v2.1.0](https://github.com/tensorflow/models/releases/tag/v2.1.0)) + +| Model | Description | Reference | +| ----- | ----------- | --------- | +| [Gradient Boosted Trees](boosted_trees) | A gradient boosted trees model to classify higgs boson process from HIGGS dataset | [Link](https://en.wikipedia.org/wiki/Gradient_boosting) | +| [MNIST](mnist) | A basic model to classify digits from the MNIST dataset | [Link](http://yann.lecun.com/exdb/mnist/) | +| [NCF](ncf) | NCF Estimator implementation | [arXiv:1708.05031](https://arxiv.org/abs/1708.05031) | +| [ResNet](resnet) | A deep residual network for image recognition | [arXiv:1512.03385](https://arxiv.org/abs/1512.03385) | +| [Transformer](transformer) | A transformer model to translate the WMT English to German dataset | [arXiv:1706.03762](https://arxiv.org/abs/1706.03762) | +| [Wide & Deep Learning](wide_deep) | A model that combines a wide linear model and deep neural network for recommender systems | [arXiv:1606.07792](https://arxiv.org/abs/1606.07792) | diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/boosted_trees/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/boosted_trees/README.md new file mode 100644 index 0000000..56c40aa --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/boosted_trees/README.md @@ -0,0 +1,117 @@ +![No Maintenance Intended](https://img.shields.io/badge/No%20Maintenance%20Intended-%E2%9C%95-red.svg) +![TensorFlow Requirement: 1.x](https://img.shields.io/badge/TensorFlow%20Requirement-1.x-brightgreen) +![TensorFlow 2 Not Supported](https://img.shields.io/badge/TensorFlow%202%20Not%20Supported-%E2%9C%95-red.svg) + +# Classifying Higgs boson processes in the HIGGS Data Set + +## Overview +The [HIGGS Data Set](https://archive.ics.uci.edu/ml/datasets/HIGGS) contains 11 million samples with 28 features, and is for the classification problem to distinguish between a signal process which produces Higgs bosons and a background process which does not. + +We use Gradient Boosted Trees algorithm to distinguish the two classes. + +--- + +The code sample uses the high level `tf.estimator.Estimator` and `tf.data.Dataset`. These APIs are great for fast iteration and quickly adapting models to your own datasets without major code overhauls. It allows you to move from single-worker training to distributed training, and makes it easy to export model binaries for prediction. Here, for further simplicity and faster execution, we use a utility function `tf.contrib.estimator.boosted_trees_classifier_train_in_memory`. This utility function is especially effective when the input is provided as in-memory data sets like numpy arrays. + +An input function for the `Estimator` typically uses `tf.data.Dataset` API, which can handle various data control like streaming, batching, transform and shuffling. However `boosted_trees_classifier_train_in_memory()` utility function requires that the entire data is provided as a single batch (i.e. without using `batch()` API). Thus in this practice, simply `Dataset.from_tensors()` is used to convert numpy arrays into structured tensors, and `Dataset.zip()` is used to put features and label together. +For further references of `Dataset`, [Read more here](https://www.tensorflow.org/guide/datasets). + +## Running the code +First make sure you've [added the models folder to your Python path](/official/#running-the-models); otherwise you may encounter an error like `ImportError: No module named official.boosted_trees`. + +### Setup +The [HIGGS Data Set](https://archive.ics.uci.edu/ml/datasets/HIGGS) that this sample uses for training is hosted by the [UC Irvine Machine Learning Repository](https://archive.ics.uci.edu/ml/datasets/). We have provided a script that downloads and cleans the necessary files. + +``` +python data_download.py +``` + +This will download a file and store the processed file under the directory designated by `--data_dir` (defaults to `/tmp/higgs_data/`). To change the target directory, set the `--data_dir` flag. The directory could be network storages that Tensorflow supports (like Google Cloud Storage, `gs:////`). +The file downloaded to the local temporary folder is about 2.8 GB, and the processed file is about 0.8 GB, so there should be enough storage to handle them. + + +### Training + +This example uses about 3 GB of RAM during training. +You can run the code locally as follows: + +``` +python train_higgs.py +``` + +The model is by default saved to `/tmp/higgs_model`, which can be changed using the `--model_dir` flag. +Note that the model_dir is cleaned up before every time training starts. + +Model parameters can be adjusted by flags, like `--n_trees`, `--max_depth`, `--learning_rate` and so on. Check out the code for details. + +The final accuracy will be around 74% and loss will be around 0.516 over the eval set, when trained with the default parameters. + +By default, the first 1 million examples among 11 millions are used for training, and the last 1 million examples are used for evaluation. +The training/evaluation data can be selected as index ranges by flags `--train_start`, `--train_count`, `--eval_start`, `--eval_count`, etc. + +### TensorBoard + +Run TensorBoard to inspect the details about the graph and training progression. + +``` +tensorboard --logdir=/tmp/higgs_model # set logdir as --model_dir set during training. +``` + +## Inference with SavedModel +You can export the model into Tensorflow [SavedModel](https://www.tensorflow.org/guide/saved_model) format by using the argument `--export_dir`: + +``` +python train_higgs.py --export_dir /tmp/higgs_boosted_trees_saved_model +``` + +After the model finishes training, use [`saved_model_cli`](https://www.tensorflow.org/guide/saved_model#cli_to_inspect_and_execute_savedmodel) to inspect and execute the SavedModel. + +Try the following commands to inspect the SavedModel: + +**Replace `${TIMESTAMP}` with the folder produced (e.g. 1524249124)** +``` +# List possible tag_sets. Only one metagraph is saved, so there will be one option. +saved_model_cli show --dir /tmp/higgs_boosted_trees_saved_model/${TIMESTAMP}/ + +# Show SignatureDefs for tag_set=serve. SignatureDefs define the outputs to show. +saved_model_cli show --dir /tmp/higgs_boosted_trees_saved_model/${TIMESTAMP}/ \ + --tag_set serve --all +``` + +### Inference +Let's use the model to predict the income group of two examples. +Note that this model exports SavedModel with the custom parsing module that accepts csv lines as features. (Each line is an example with 28 columns; be careful to not add a label column, unlike in the training data.) + +``` +saved_model_cli run --dir /tmp/boosted_trees_higgs_saved_model/${TIMESTAMP}/ \ + --tag_set serve --signature_def="predict" \ + --input_exprs='inputs=["0.869293,-0.635082,0.225690,0.327470,-0.689993,0.754202,-0.248573,-1.092064,0.0,1.374992,-0.653674,0.930349,1.107436,1.138904,-1.578198,-1.046985,0.0,0.657930,-0.010455,-0.045767,3.101961,1.353760,0.979563,0.978076,0.920005,0.721657,0.988751,0.876678", "1.595839,-0.607811,0.007075,1.818450,-0.111906,0.847550,-0.566437,1.581239,2.173076,0.755421,0.643110,1.426367,0.0,0.921661,-1.190432,-1.615589,0.0,0.651114,-0.654227,-1.274345,3.101961,0.823761,0.938191,0.971758,0.789176,0.430553,0.961357,0.957818"]' +``` + +This will print out the predicted classes and class probabilities. Something like: + +``` +Result for output key class_ids: +[[1] + [0]] +Result for output key classes: +[['1'] + ['0']] +Result for output key logistic: +[[0.6440273 ] + [0.10902369]] +Result for output key logits: +[[ 0.59288704] + [-2.1007526 ]] +Result for output key probabilities: +[[0.3559727 0.6440273] + [0.8909763 0.1090237]] +``` + +Please note that "predict" signature_def gives out different (more detailed) results than "classification" or "serving_default". + +## Additional Links + +If you are interested in distributed training, take a look at [Distributed TensorFlow](https://www.tensorflow.org/deploy/distributed). + +You can also [train models on Cloud ML Engine](https://cloud.google.com/ml-engine/docs/getting-started-training-prediction), which provides [hyperparameter tuning](https://cloud.google.com/ml-engine/docs/getting-started-training-prediction#hyperparameter_tuning) to maximize your model's results and enables [deploying your model for prediction](https://cloud.google.com/ml-engine/docs/getting-started-training-prediction#deploy_a_model_to_support_prediction). diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/boosted_trees/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/boosted_trees/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/boosted_trees/data_download.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/boosted_trees/data_download.py new file mode 100644 index 0000000..1b6fc05 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/boosted_trees/data_download.py @@ -0,0 +1,97 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Downloads the UCI HIGGS Dataset and prepares train data. + +The details on the dataset are in https://archive.ics.uci.edu/ml/datasets/HIGGS + +It takes a while as it needs to download 2.8 GB over the network, process, then +store it into the specified location as a compressed numpy file. + +Usage: +$ python data_download.py --data_dir=/tmp/higgs_data +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gzip +import os +import tempfile + +# pylint: disable=g-bad-import-order +import numpy as np +import pandas as pd +from six.moves import urllib +from absl import app as absl_app +from absl import flags +import tensorflow as tf + +from official.utils.flags import core as flags_core + +URL_ROOT = "https://archive.ics.uci.edu/ml/machine-learning-databases/00280" +INPUT_FILE = "HIGGS.csv.gz" +NPZ_FILE = "HIGGS.csv.gz.npz" # numpy compressed file to contain "data" array. + + +def _download_higgs_data_and_save_npz(data_dir): + """Download higgs data and store as a numpy compressed file.""" + input_url = URL_ROOT + "/" + INPUT_FILE + np_filename = os.path.join(data_dir, NPZ_FILE) + if tf.gfile.Exists(np_filename): + raise ValueError("data_dir already has the processed data file: {}".format( + np_filename)) + if not tf.gfile.Exists(data_dir): + tf.gfile.MkDir(data_dir) + # 2.8 GB to download. + try: + tf.logging.info("Data downloading...") + temp_filename, _ = urllib.request.urlretrieve(input_url) + # Reading and parsing 11 million csv lines takes 2~3 minutes. + tf.logging.info("Data processing... taking multiple minutes...") + with gzip.open(temp_filename, "rb") as csv_file: + data = pd.read_csv( + csv_file, + dtype=np.float32, + names=["c%02d" % i for i in range(29)] # label + 28 features. + ).as_matrix() + finally: + tf.gfile.Remove(temp_filename) + + # Writing to temporary location then copy to the data_dir (0.8 GB). + f = tempfile.NamedTemporaryFile() + np.savez_compressed(f, data=data) + tf.gfile.Copy(f.name, np_filename) + tf.logging.info("Data saved to: {}".format(np_filename)) + + +def main(unused_argv): + if not tf.gfile.Exists(FLAGS.data_dir): + tf.gfile.MkDir(FLAGS.data_dir) + _download_higgs_data_and_save_npz(FLAGS.data_dir) + + +def define_data_download_flags(): + """Add flags specifying data download arguments.""" + flags.DEFINE_string( + name="data_dir", default="/tmp/higgs_data", + help=flags_core.help_wrap( + "Directory to download higgs dataset and store training/eval data.")) + + +if __name__ == "__main__": + tf.logging.set_verbosity(tf.logging.INFO) + define_data_download_flags() + FLAGS = flags.FLAGS + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/boosted_trees/train_higgs.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/boosted_trees/train_higgs.py new file mode 100644 index 0000000..d496a39 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/boosted_trees/train_higgs.py @@ -0,0 +1,297 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""A script that builds boosted trees over higgs data. + +If you haven't, please run data_download.py beforehand to prepare the data. + +For some more details on this example, please refer to README.md as well. + +Note that the model_dir is cleaned up before starting the training. + +Usage: +$ python train_higgs.py --n_trees=100 --max_depth=6 --learning_rate=0.1 \ + --model_dir=/tmp/higgs_model + +Note that BoostedTreesClassifier is available since Tensorflow 1.8.0. +So you need to install recent enough version of Tensorflow to use this example. + +The training data is by default the first million examples out of 11M examples, +and eval data is by default the last million examples. +They are controlled by --train_start, --train_count, --eval_start, --eval_count. +e.g. to train over the first 10 million examples instead of 1 million: +$ python train_higgs.py --n_trees=100 --max_depth=6 --learning_rate=0.1 \ + --model_dir=/tmp/higgs_model --train_count=10000000 + +Training history and metrics can be inspected using tensorboard. +Set --logdir as the --model_dir set by flag when training +(or the default /tmp/higgs_model). +$ tensorboard --logdir=/tmp/higgs_model +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +# pylint: disable=g-bad-import-order +import numpy as np +from absl import app as absl_app +from absl import flags +import tensorflow as tf +# pylint: enable=g-bad-import-order + +from official.utils.flags import core as flags_core +from official.utils.flags._conventions import help_wrap +from official.utils.logs import logger + +NPZ_FILE = "HIGGS.csv.gz.npz" # numpy compressed file containing "data" array + + +def read_higgs_data(data_dir, train_start, train_count, eval_start, eval_count): + """Reads higgs data from csv and returns train and eval data. + + Args: + data_dir: A string, the directory of higgs dataset. + train_start: An integer, the start index of train examples within the data. + train_count: An integer, the number of train examples within the data. + eval_start: An integer, the start index of eval examples within the data. + eval_count: An integer, the number of eval examples within the data. + + Returns: + Numpy array of train data and eval data. + """ + npz_filename = os.path.join(data_dir, NPZ_FILE) + try: + # gfile allows numpy to read data from network data sources as well. + with tf.gfile.Open(npz_filename, "rb") as npz_file: + with np.load(npz_file) as npz: + data = npz["data"] + except tf.errors.NotFoundError as e: + raise RuntimeError( + "Error loading data; use data_download.py to prepare the data.\n{}: {}" + .format(type(e).__name__, e)) + return (data[train_start:train_start+train_count], + data[eval_start:eval_start+eval_count]) + + +# This showcases how to make input_fn when the input data is available in the +# form of numpy arrays. +def make_inputs_from_np_arrays(features_np, label_np): + """Makes and returns input_fn and feature_columns from numpy arrays. + + The generated input_fn will return tf.data.Dataset of feature dictionary and a + label, and feature_columns will consist of the list of + tf.feature_column.BucketizedColumn. + + Note, for in-memory training, tf.data.Dataset should contain the whole data + as a single tensor. Don't use batch. + + Args: + features_np: A numpy ndarray (shape=[batch_size, num_features]) for + float32 features. + label_np: A numpy ndarray (shape=[batch_size, 1]) for labels. + + Returns: + input_fn: A function returning a Dataset of feature dict and label. + feature_names: A list of feature names. + feature_column: A list of tf.feature_column.BucketizedColumn. + """ + num_features = features_np.shape[1] + features_np_list = np.split(features_np, num_features, axis=1) + # 1-based feature names. + feature_names = ["feature_%02d" % (i + 1) for i in range(num_features)] + + # Create source feature_columns and bucketized_columns. + def get_bucket_boundaries(feature): + """Returns bucket boundaries for feature by percentiles.""" + return np.unique(np.percentile(feature, range(0, 100))).tolist() + source_columns = [ + tf.feature_column.numeric_column( + feature_name, dtype=tf.float32, + # Although higgs data have no missing values, in general, default + # could be set as 0 or some reasonable value for missing values. + default_value=0.0) + for feature_name in feature_names + ] + bucketized_columns = [ + tf.feature_column.bucketized_column( + source_columns[i], + boundaries=get_bucket_boundaries(features_np_list[i])) + for i in range(num_features) + ] + + # Make an input_fn that extracts source features. + def input_fn(): + """Returns features as a dictionary of numpy arrays, and a label.""" + features = { + feature_name: tf.constant(features_np_list[i]) + for i, feature_name in enumerate(feature_names) + } + return tf.data.Dataset.zip((tf.data.Dataset.from_tensors(features), + tf.data.Dataset.from_tensors(label_np),)) + + return input_fn, feature_names, bucketized_columns + + +def make_eval_inputs_from_np_arrays(features_np, label_np): + """Makes eval input as streaming batches.""" + num_features = features_np.shape[1] + features_np_list = np.split(features_np, num_features, axis=1) + # 1-based feature names. + feature_names = ["feature_%02d" % (i + 1) for i in range(num_features)] + + def input_fn(): + features = { + feature_name: tf.constant(features_np_list[i]) + for i, feature_name in enumerate(feature_names) + } + return tf.data.Dataset.zip(( + tf.data.Dataset.from_tensor_slices(features), + tf.data.Dataset.from_tensor_slices(label_np),)).batch(1000) + + return input_fn + + +def _make_csv_serving_input_receiver_fn(column_names, column_defaults): + """Returns serving_input_receiver_fn for csv. + + The input arguments are relevant to `tf.decode_csv()`. + + Args: + column_names: a list of column names in the order within input csv. + column_defaults: a list of default values with the same size of + column_names. Each entity must be either a list of one scalar, or an + empty list to denote the corresponding column is required. + e.g. [[""], [2.5], []] indicates the third column is required while + the first column must be string and the second must be float/double. + + Returns: + a serving_input_receiver_fn that handles csv for serving. + """ + def serving_input_receiver_fn(): + csv = tf.placeholder(dtype=tf.string, shape=[None], name="csv") + features = dict(zip(column_names, tf.decode_csv(csv, column_defaults))) + receiver_tensors = {"inputs": csv} + return tf.estimator.export.ServingInputReceiver(features, receiver_tensors) + + return serving_input_receiver_fn + + +def train_boosted_trees(flags_obj): + """Train boosted_trees estimator on HIGGS data. + + Args: + flags_obj: An object containing parsed flag values. + """ + # Clean up the model directory if present. + if tf.gfile.Exists(flags_obj.model_dir): + tf.gfile.DeleteRecursively(flags_obj.model_dir) + tf.logging.info("## Data loading...") + train_data, eval_data = read_higgs_data( + flags_obj.data_dir, flags_obj.train_start, flags_obj.train_count, + flags_obj.eval_start, flags_obj.eval_count) + tf.logging.info("## Data loaded; train: {}{}, eval: {}{}".format( + train_data.dtype, train_data.shape, eval_data.dtype, eval_data.shape)) + # Data consists of one label column followed by 28 feature columns. + train_input_fn, feature_names, feature_columns = make_inputs_from_np_arrays( + features_np=train_data[:, 1:], label_np=train_data[:, 0:1]) + eval_input_fn = make_eval_inputs_from_np_arrays( + features_np=eval_data[:, 1:], label_np=eval_data[:, 0:1]) + tf.logging.info("## Features prepared. Training starts...") + + # Create benchmark logger to log info about the training and metric values + run_params = { + "train_start": flags_obj.train_start, + "train_count": flags_obj.train_count, + "eval_start": flags_obj.eval_start, + "eval_count": flags_obj.eval_count, + "n_trees": flags_obj.n_trees, + "max_depth": flags_obj.max_depth, + } + benchmark_logger = logger.config_benchmark_logger(flags_obj) + benchmark_logger.log_run_info( + model_name="boosted_trees", + dataset_name="higgs", + run_params=run_params, + test_id=flags_obj.benchmark_test_id) + + # Though BoostedTreesClassifier is under tf.estimator, faster in-memory + # training is yet provided as a contrib library. + from tensorflow.contrib import estimator as contrib_estimator # pylint: disable=g-import-not-at-top + classifier = contrib_estimator.boosted_trees_classifier_train_in_memory( + train_input_fn, + feature_columns, + model_dir=flags_obj.model_dir or None, + n_trees=flags_obj.n_trees, + max_depth=flags_obj.max_depth, + learning_rate=flags_obj.learning_rate) + + # Evaluation. + eval_results = classifier.evaluate(eval_input_fn) + # Benchmark the evaluation results + benchmark_logger.log_evaluation_result(eval_results) + + # Exporting the savedmodel with csv parsing. + if flags_obj.export_dir is not None: + classifier.export_savedmodel( + flags_obj.export_dir, + _make_csv_serving_input_receiver_fn( + column_names=feature_names, + # columns are all floats. + column_defaults=[[0.0]] * len(feature_names)), + strip_default_attrs=True) + + +def main(_): + train_boosted_trees(flags.FLAGS) + + +def define_train_higgs_flags(): + """Add tree related flags as well as training/eval configuration.""" + flags_core.define_base(clean=False, stop_threshold=False, batch_size=False, + num_gpu=False, export_dir=True) + flags_core.define_benchmark() + flags.adopt_module_key_flags(flags_core) + + flags.DEFINE_integer( + name="train_start", default=0, + help=help_wrap("Start index of train examples within the data.")) + flags.DEFINE_integer( + name="train_count", default=1000000, + help=help_wrap("Number of train examples within the data.")) + flags.DEFINE_integer( + name="eval_start", default=10000000, + help=help_wrap("Start index of eval examples within the data.")) + flags.DEFINE_integer( + name="eval_count", default=1000000, + help=help_wrap("Number of eval examples within the data.")) + + flags.DEFINE_integer( + "n_trees", default=100, help=help_wrap("Number of trees to build.")) + flags.DEFINE_integer( + "max_depth", default=6, help=help_wrap("Maximum depths of each tree.")) + flags.DEFINE_float( + "learning_rate", default=0.1, + help=help_wrap("The learning rate.")) + + flags_core.set_defaults(data_dir="/tmp/higgs_data", + model_dir="/tmp/higgs_model") + + +if __name__ == "__main__": + # Training progress and eval results are shown as logging.INFO; so enables it. + tf.logging.set_verbosity(tf.logging.INFO) + define_train_higgs_flags() + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/README.md new file mode 100644 index 0000000..55f3523 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/README.md @@ -0,0 +1,91 @@ +![No Maintenance Intended](https://img.shields.io/badge/No%20Maintenance%20Intended-%E2%9C%95-red.svg) +![TensorFlow Requirement: 1.x](https://img.shields.io/badge/TensorFlow%20Requirement-1.x-brightgreen) +![TensorFlow 2 Not Supported](https://img.shields.io/badge/TensorFlow%202%20Not%20Supported-%E2%9C%95-red.svg) + +# MNIST in TensorFlow + +This directory builds a convolutional neural net to classify the [MNIST +dataset](http://yann.lecun.com/exdb/mnist/) using the +[tf.data](https://www.tensorflow.org/api_docs/python/tf/data), +[tf.estimator.Estimator](https://www.tensorflow.org/api_docs/python/tf/estimator/Estimator), +and +[tf.layers](https://www.tensorflow.org/api_docs/python/tf/layers) +APIs. + + +## Setup + +To begin, you'll simply need the latest version of TensorFlow installed. +First make sure you've [added the models folder to your Python path]: + +```shell +export PYTHONPATH="$PYTHONPATH:/path/to/models" +``` + +Otherwise you may encounter an error like `ImportError: No module named official.mnist`. + +Then to train the model, run the following: + +``` +python mnist.py +``` + +The model will begin training and will automatically evaluate itself on the +validation data. + +Illustrative unit tests and benchmarks can be run with: + +``` +python mnist_test.py +python mnist_test.py --benchmarks=. +``` + +## Exporting the model + +You can export the model into Tensorflow [SavedModel](https://www.tensorflow.org/guide/saved_model) format by using the argument `--export_dir`: + +``` +python mnist.py --export_dir /tmp/mnist_saved_model +``` + +The SavedModel will be saved in a timestamped directory under `/tmp/mnist_saved_model/` (e.g. `/tmp/mnist_saved_model/1513630966/`). + +**Getting predictions with SavedModel** +Use [`saved_model_cli`](https://www.tensorflow.org/guide/saved_model#cli_to_inspect_and_execute_savedmodel) to inspect and execute the SavedModel. + +``` +saved_model_cli run --dir /tmp/mnist_saved_model/TIMESTAMP --tag_set serve --signature_def classify --inputs image=examples.npy +``` + +`examples.npy` contains the data from `example5.png` and `example3.png` in a numpy array, in that order. The array values are normalized to values between 0 and 1. + +The output should look similar to below: +``` +Result for output key classes: +[5 3] +Result for output key probabilities: +[[ 1.53558474e-07 1.95694142e-13 1.31193523e-09 5.47467265e-03 + 5.85711526e-22 9.94520664e-01 3.48423509e-06 2.65365645e-17 + 9.78631419e-07 3.15522470e-08] + [ 1.22413359e-04 5.87615965e-08 1.72251271e-06 9.39960718e-01 + 3.30306928e-11 2.87386645e-02 2.82353517e-02 8.21146413e-18 + 2.52568233e-03 4.15460236e-04]] +``` + +## Experimental: Eager Execution + +[Eager execution](https://research.googleblog.com/2017/10/eager-execution-imperative-define-by.html) +(an preview feature in TensorFlow 1.5) is an imperative interface to TensorFlow. +The exact same model defined in `mnist.py` can be trained without creating a +TensorFlow graph using: + +``` +python mnist_eager.py +``` + +## Experimental: TPU Acceleration + +`mnist.py` (and `mnist_eager.py`) demonstrate training a neural network to +classify digits on CPUs and GPUs. `mnist_tpu.py` can be used to train the +same model using TPUs for hardware acceleration. More information in +the [tensorflow/tpu](https://github.com/tensorflow/tpu) repository. diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/dataset.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/dataset.py new file mode 100644 index 0000000..2bdd155 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/dataset.py @@ -0,0 +1,117 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""tf.data.Dataset interface to the MNIST dataset.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gzip +import os +import shutil +import tempfile + +import numpy as np +from six.moves import urllib +import tensorflow as tf + + +def read32(bytestream): + """Read 4 bytes from bytestream as an unsigned 32-bit integer.""" + dt = np.dtype(np.uint32).newbyteorder('>') + return np.frombuffer(bytestream.read(4), dtype=dt)[0] + + +def check_image_file_header(filename): + """Validate that filename corresponds to images for the MNIST dataset.""" + with tf.io.gfile.GFile(filename, 'rb') as f: + magic = read32(f) + read32(f) # num_images, unused + rows = read32(f) + cols = read32(f) + if magic != 2051: + raise ValueError('Invalid magic number %d in MNIST file %s' % (magic, + f.name)) + if rows != 28 or cols != 28: + raise ValueError( + 'Invalid MNIST file %s: Expected 28x28 images, found %dx%d' % + (f.name, rows, cols)) + + +def check_labels_file_header(filename): + """Validate that filename corresponds to labels for the MNIST dataset.""" + with tf.io.gfile.GFile(filename, 'rb') as f: + magic = read32(f) + read32(f) # num_items, unused + if magic != 2049: + raise ValueError('Invalid magic number %d in MNIST file %s' % (magic, + f.name)) + + +def download(directory, filename): + """Download (and unzip) a file from the MNIST dataset if not already done.""" + filepath = os.path.join(directory, filename) + if tf.io.gfile.exists(filepath): + return filepath + if not tf.io.gfile.exists(directory): + tf.io.gfile.makedirs(directory) + # CVDF mirror of http://yann.lecun.com/exdb/mnist/ + url = 'https://storage.googleapis.com/cvdf-datasets/mnist/' + filename + '.gz' + _, zipped_filepath = tempfile.mkstemp(suffix='.gz') + print('Downloading %s to %s' % (url, zipped_filepath)) + urllib.request.urlretrieve(url, zipped_filepath) + with gzip.open(zipped_filepath, 'rb') as f_in, \ + tf.io.gfile.GFile(filepath, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + os.remove(zipped_filepath) + return filepath + + +def dataset(directory, images_file, labels_file): + """Download and parse MNIST dataset.""" + + images_file = download(directory, images_file) + labels_file = download(directory, labels_file) + + check_image_file_header(images_file) + check_labels_file_header(labels_file) + + def decode_image(image): + # Normalize from [0, 255] to [0.0, 1.0] + image = tf.io.decode_raw(image, tf.uint8) + image = tf.cast(image, tf.float32) + image = tf.reshape(image, [784]) + return image / 255.0 + + def decode_label(label): + label = tf.io.decode_raw(label, tf.uint8) # tf.string -> [tf.uint8] + label = tf.reshape(label, []) # label is a scalar + return tf.cast(label, tf.int32) + + images = tf.data.FixedLengthRecordDataset( + images_file, 28 * 28, header_bytes=16).map(decode_image) + labels = tf.data.FixedLengthRecordDataset( + labels_file, 1, header_bytes=8).map(decode_label) + return tf.data.Dataset.zip((images, labels)) + + +def train(directory): + """tf.data.Dataset object for MNIST training data.""" + return dataset(directory, 'train-images-idx3-ubyte', + 'train-labels-idx1-ubyte') + + +def test(directory): + """tf.data.Dataset object for MNIST test data.""" + return dataset(directory, 't10k-images-idx3-ubyte', 't10k-labels-idx1-ubyte') diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist.py new file mode 100644 index 0000000..e1eeb2c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist.py @@ -0,0 +1,247 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Convolutional Neural Network Estimator for MNIST, built with tf.layers.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import app as absl_app +from absl import flags +from absl import logging +from six.moves import range +import tensorflow as tf + +from official.r1.mnist import dataset +from official.utils.flags import core as flags_core +from official.utils.logs import hooks_helper +from official.utils.misc import distribution_utils +from official.utils.misc import model_helpers + + +LEARNING_RATE = 1e-4 + + +def create_model(data_format): + """Model to recognize digits in the MNIST dataset. + + Network structure is equivalent to: + https://github.com/tensorflow/tensorflow/blob/r1.5/tensorflow/examples/tutorials/mnist/mnist_deep.py + + But uses the tf.keras API. + + Args: + data_format: Either 'channels_first' or 'channels_last'. 'channels_first' is + typically faster on GPUs while 'channels_last' is typically faster on + CPUs. See + https://www.tensorflow.org/performance/performance_guide#data_formats + + Returns: + A tf.keras.Model. + """ + if data_format == 'channels_first': + input_shape = [1, 28, 28] + else: + assert data_format == 'channels_last' + input_shape = [28, 28, 1] + + l = tf.keras.layers + max_pool = l.MaxPooling2D( + (2, 2), (2, 2), padding='same', data_format=data_format) + # The model consists of a sequential chain of layers, so tf.keras.Sequential + # (a subclass of tf.keras.Model) makes for a compact description. + return tf.keras.Sequential( + [ + l.Reshape( + target_shape=input_shape, + input_shape=(28 * 28,)), + l.Conv2D( + 32, + 5, + padding='same', + data_format=data_format, + activation=tf.nn.relu), + max_pool, + l.Conv2D( + 64, + 5, + padding='same', + data_format=data_format, + activation=tf.nn.relu), + max_pool, + l.Flatten(), + l.Dense(1024, activation=tf.nn.relu), + l.Dropout(0.4), + l.Dense(10) + ]) + + +def define_mnist_flags(): + """Defines flags for mnist.""" + flags_core.define_base(clean=True, train_epochs=True, + epochs_between_evals=True, stop_threshold=True, + num_gpu=True, hooks=True, export_dir=True, + distribution_strategy=True) + flags_core.define_performance(inter_op=True, intra_op=True, + num_parallel_calls=False, + all_reduce_alg=True) + flags_core.define_image() + flags.adopt_module_key_flags(flags_core) + flags_core.set_defaults(data_dir='/tmp/mnist_data', + model_dir='/tmp/mnist_model', + batch_size=100, + train_epochs=40) + + +def model_fn(features, labels, mode, params): + """The model_fn argument for creating an Estimator.""" + model = create_model(params['data_format']) + image = features + if isinstance(image, dict): + image = features['image'] + + if mode == tf.estimator.ModeKeys.PREDICT: + logits = model(image, training=False) + predictions = { + 'classes': tf.argmax(logits, axis=1), + 'probabilities': tf.nn.softmax(logits), + } + return tf.estimator.EstimatorSpec( + mode=tf.estimator.ModeKeys.PREDICT, + predictions=predictions, + export_outputs={ + 'classify': tf.estimator.export.PredictOutput(predictions) + }) + if mode == tf.estimator.ModeKeys.TRAIN: + optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=LEARNING_RATE) + + logits = model(image, training=True) + loss = tf.compat.v1.losses.sparse_softmax_cross_entropy(labels=labels, + logits=logits) + accuracy = tf.compat.v1.metrics.accuracy( + labels=labels, predictions=tf.argmax(logits, axis=1)) + + # Name tensors to be logged with LoggingTensorHook. + tf.identity(LEARNING_RATE, 'learning_rate') + tf.identity(loss, 'cross_entropy') + tf.identity(accuracy[1], name='train_accuracy') + + # Save accuracy scalar to Tensorboard output. + tf.summary.scalar('train_accuracy', accuracy[1]) + + return tf.estimator.EstimatorSpec( + mode=tf.estimator.ModeKeys.TRAIN, + loss=loss, + train_op=optimizer.minimize( + loss, + tf.compat.v1.train.get_or_create_global_step())) + if mode == tf.estimator.ModeKeys.EVAL: + logits = model(image, training=False) + loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) + return tf.estimator.EstimatorSpec( + mode=tf.estimator.ModeKeys.EVAL, + loss=loss, + eval_metric_ops={ + 'accuracy': + tf.metrics.accuracy( + labels=labels, predictions=tf.argmax(logits, axis=1)), + }) + + +def run_mnist(flags_obj): + """Run MNIST training and eval loop. + + Args: + flags_obj: An object containing parsed flag values. + """ + model_helpers.apply_clean(flags_obj) + model_function = model_fn + + session_config = tf.compat.v1.ConfigProto( + inter_op_parallelism_threads=flags_obj.inter_op_parallelism_threads, + intra_op_parallelism_threads=flags_obj.intra_op_parallelism_threads, + allow_soft_placement=True) + + distribution_strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=flags_obj.distribution_strategy, + num_gpus=flags_core.get_num_gpus(flags_obj), + all_reduce_alg=flags_obj.all_reduce_alg) + + run_config = tf.estimator.RunConfig( + train_distribute=distribution_strategy, session_config=session_config) + + data_format = flags_obj.data_format + if data_format is None: + data_format = ('channels_first' if tf.config.list_physical_devices('GPU') + else 'channels_last') + mnist_classifier = tf.estimator.Estimator( + model_fn=model_function, + model_dir=flags_obj.model_dir, + config=run_config, + params={ + 'data_format': data_format, + }) + + # Set up training and evaluation input functions. + def train_input_fn(): + """Prepare data for training.""" + + # When choosing shuffle buffer sizes, larger sizes result in better + # randomness, while smaller sizes use less memory. MNIST is a small + # enough dataset that we can easily shuffle the full epoch. + ds = dataset.train(flags_obj.data_dir) + ds = ds.cache().shuffle(buffer_size=50000).batch(flags_obj.batch_size) + + # Iterate through the dataset a set number (`epochs_between_evals`) of times + # during each training session. + ds = ds.repeat(flags_obj.epochs_between_evals) + return ds + + def eval_input_fn(): + return dataset.test(flags_obj.data_dir).batch( + flags_obj.batch_size).make_one_shot_iterator().get_next() + + # Set up hook that outputs training logs every 100 steps. + train_hooks = hooks_helper.get_train_hooks( + flags_obj.hooks, model_dir=flags_obj.model_dir, + batch_size=flags_obj.batch_size) + + # Train and evaluate model. + for _ in range(flags_obj.train_epochs // flags_obj.epochs_between_evals): + mnist_classifier.train(input_fn=train_input_fn, hooks=train_hooks) + eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn) + print('\nEvaluation results:\n\t%s\n' % eval_results) + + if model_helpers.past_stop_threshold(flags_obj.stop_threshold, + eval_results['accuracy']): + break + + # Export the model + if flags_obj.export_dir is not None: + image = tf.compat.v1.placeholder(tf.float32, [None, 28, 28]) + input_fn = tf.estimator.export.build_raw_serving_input_receiver_fn({ + 'image': image, + }) + mnist_classifier.export_savedmodel(flags_obj.export_dir, input_fn, + strip_default_attrs=True) + + +def main(_): + run_mnist(flags.FLAGS) + + +if __name__ == '__main__': + logging.set_verbosity(logging.INFO) + define_mnist_flags() + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist_eager.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist_eager.py new file mode 100644 index 0000000..0a3a3b3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist_eager.py @@ -0,0 +1,212 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""MNIST model training with TensorFlow eager execution. + +See: +https://research.googleblog.com/2017/10/eager-execution-imperative-define-by.html + +This program demonstrates training of the convolutional neural network model +defined in mnist.py with eager execution enabled. + +If you are not interested in eager execution, you should ignore this file. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import time + +# pylint: disable=g-bad-import-order +from absl import app as absl_app +from absl import flags +import tensorflow as tf +from tensorflow.python import eager as tfe +# pylint: enable=g-bad-import-order + +from official.r1.mnist import dataset as mnist_dataset +from official.r1.mnist import mnist +from official.utils.flags import core as flags_core +from official.utils.misc import model_helpers + + +def loss(logits, labels): + return tf.reduce_mean( + tf.nn.sparse_softmax_cross_entropy_with_logits( + logits=logits, labels=labels)) + + +def compute_accuracy(logits, labels): + predictions = tf.argmax(logits, axis=1, output_type=tf.int64) + labels = tf.cast(labels, tf.int64) + batch_size = int(logits.shape[0]) + return tf.reduce_sum( + tf.cast(tf.equal(predictions, labels), dtype=tf.float32)) / batch_size + + +def train(model, optimizer, dataset, step_counter, log_interval=None): + """Trains model on `dataset` using `optimizer`.""" + from tensorflow.contrib import summary as contrib_summary # pylint: disable=g-import-not-at-top + + start = time.time() + for (batch, (images, labels)) in enumerate(dataset): + with contrib_summary.record_summaries_every_n_global_steps( + 10, global_step=step_counter): + # Record the operations used to compute the loss given the input, + # so that the gradient of the loss with respect to the variables + # can be computed. + with tf.GradientTape() as tape: + logits = model(images, training=True) + loss_value = loss(logits, labels) + contrib_summary.scalar('loss', loss_value) + contrib_summary.scalar('accuracy', + compute_accuracy(logits, labels)) + grads = tape.gradient(loss_value, model.variables) + optimizer.apply_gradients( + zip(grads, model.variables), global_step=step_counter) + if log_interval and batch % log_interval == 0: + rate = log_interval / (time.time() - start) + print('Step #%d\tLoss: %.6f (%d steps/sec)' % (batch, loss_value, rate)) + start = time.time() + + +def test(model, dataset): + """Perform an evaluation of `model` on the examples from `dataset`.""" + from tensorflow.contrib import summary as contrib_summary # pylint: disable=g-import-not-at-top + avg_loss = tf.keras.metrics.Mean('loss', dtype=tf.float32) + accuracy = tf.keras.metrics.Accuracy('accuracy', dtype=tf.float32) + + for (images, labels) in dataset: + logits = model(images, training=False) + avg_loss.update_state(loss(logits, labels)) + accuracy.update_state( + tf.argmax(logits, axis=1, output_type=tf.int64), + tf.cast(labels, tf.int64)) + print('Test set: Average loss: %.4f, Accuracy: %4f%%\n' % + (avg_loss.result(), 100 * accuracy.result())) + with contrib_summary.always_record_summaries(): + contrib_summary.scalar('loss', avg_loss.result()) + contrib_summary.scalar('accuracy', accuracy.result()) + + +def run_mnist_eager(flags_obj): + """Run MNIST training and eval loop in eager mode. + + Args: + flags_obj: An object containing parsed flag values. + """ + tf.enable_eager_execution() + model_helpers.apply_clean(flags.FLAGS) + + # Automatically determine device and data_format + (device, data_format) = ('/gpu:0', 'channels_first') + if flags_obj.no_gpu or not tf.test.is_gpu_available(): + (device, data_format) = ('/cpu:0', 'channels_last') + # If data_format is defined in FLAGS, overwrite automatically set value. + if flags_obj.data_format is not None: + data_format = flags_obj.data_format + print('Using device %s, and data format %s.' % (device, data_format)) + + # Load the datasets + train_ds = mnist_dataset.train(flags_obj.data_dir).shuffle(60000).batch( + flags_obj.batch_size) + test_ds = mnist_dataset.test(flags_obj.data_dir).batch( + flags_obj.batch_size) + + # Create the model and optimizer + model = mnist.create_model(data_format) + optimizer = tf.train.MomentumOptimizer(flags_obj.lr, flags_obj.momentum) + + # Create file writers for writing TensorBoard summaries. + if flags_obj.output_dir: + # Create directories to which summaries will be written + # tensorboard --logdir= + # can then be used to see the recorded summaries. + train_dir = os.path.join(flags_obj.output_dir, 'train') + test_dir = os.path.join(flags_obj.output_dir, 'eval') + tf.gfile.MakeDirs(flags_obj.output_dir) + else: + train_dir = None + test_dir = None + summary_writer = tf.compat.v2.summary.create_file_writer( + train_dir, flush_millis=10000) + test_summary_writer = tf.compat.v2.summary.create_file_writer( + test_dir, flush_millis=10000, name='test') + + # Create and restore checkpoint (if one exists on the path) + checkpoint_prefix = os.path.join(flags_obj.model_dir, 'ckpt') + step_counter = tf.train.get_or_create_global_step() + checkpoint = tf.train.Checkpoint( + model=model, optimizer=optimizer, step_counter=step_counter) + # Restore variables on creation if a checkpoint exists. + checkpoint.restore(tf.train.latest_checkpoint(flags_obj.model_dir)) + + # Train and evaluate for a set number of epochs. + with tf.device(device): + for _ in range(flags_obj.train_epochs): + start = time.time() + with summary_writer.as_default(): + train(model, optimizer, train_ds, step_counter, + flags_obj.log_interval) + end = time.time() + print('\nTrain time for epoch #%d (%d total steps): %f' % + (checkpoint.save_counter.numpy() + 1, + step_counter.numpy(), + end - start)) + with test_summary_writer.as_default(): + test(model, test_ds) + checkpoint.save(checkpoint_prefix) + + +def define_mnist_eager_flags(): + """Defined flags and defaults for MNIST in eager mode.""" + flags_core.define_base(clean=True, train_epochs=True, export_dir=True, + distribution_strategy=True) + flags_core.define_image() + flags.adopt_module_key_flags(flags_core) + + flags.DEFINE_integer( + name='log_interval', short_name='li', default=10, + help=flags_core.help_wrap('batches between logging training status')) + + flags.DEFINE_string( + name='output_dir', short_name='od', default=None, + help=flags_core.help_wrap('Directory to write TensorBoard summaries')) + + flags.DEFINE_float(name='learning_rate', short_name='lr', default=0.01, + help=flags_core.help_wrap('Learning rate.')) + + flags.DEFINE_float(name='momentum', short_name='m', default=0.5, + help=flags_core.help_wrap('SGD momentum.')) + + flags.DEFINE_bool(name='no_gpu', short_name='nogpu', default=False, + help=flags_core.help_wrap( + 'disables GPU usage even if a GPU is available')) + + flags_core.set_defaults( + data_dir='/tmp/tensorflow/mnist/input_data', + model_dir='/tmp/tensorflow/mnist/checkpoints/', + batch_size=100, + train_epochs=10, + ) + + +def main(_): + run_mnist_eager(flags.FLAGS) + + +if __name__ == '__main__': + define_mnist_eager_flags() + absl_app.run(main=main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist_eager_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist_eager_test.py new file mode 100644 index 0000000..2fe7e66 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist_eager_test.py @@ -0,0 +1,95 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest + +import tensorflow as tf # pylint: disable=g-bad-import-order +from tensorflow.python import eager as tfe # pylint: disable=g-bad-import-order + +from official.r1.mnist import mnist +from official.r1.mnist import mnist_eager +from official.utils.misc import keras_utils + + +def device(): + return '/device:GPU:0' if tfe.context.num_gpus() else '/device:CPU:0' + + +def data_format(): + return 'channels_first' if tfe.context.num_gpus() else 'channels_last' + + +def random_dataset(): + batch_size = 64 + images = tf.random_normal([batch_size, 784]) + labels = tf.random_uniform([batch_size], minval=0, maxval=10, dtype=tf.int32) + return tf.data.Dataset.from_tensors((images, labels)) + + +def train(defun=False): + model = mnist.create_model(data_format()) + if defun: + model.call = tf.function(model.call) + optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01) + dataset = random_dataset() + with tf.device(device()): + mnist_eager.train(model, optimizer, dataset, + step_counter=tf.train.get_or_create_global_step()) + + +def evaluate(defun=False): + model = mnist.create_model(data_format()) + dataset = random_dataset() + if defun: + model.call = tf.function(model.call) + with tf.device(device()): + mnist_eager.test(model, dataset) + + +class MNISTTest(tf.test.TestCase): + """Run tests for MNIST eager loop. + + MNIST eager uses contrib and will not work with TF 2.0. All tests are + disabled if using TF 2.0. + """ + + def setUp(self): + if not keras_utils.is_v2_0(): + tf.compat.v1.enable_v2_behavior() + super(MNISTTest, self).setUp() + + @unittest.skipIf(keras_utils.is_v2_0(), 'TF 1.0 only test.') + def test_train(self): + train(defun=False) + + @unittest.skipIf(keras_utils.is_v2_0(), 'TF 1.0 only test.') + def test_evaluate(self): + evaluate(defun=False) + + @unittest.skipIf(keras_utils.is_v2_0(), 'TF 1.0 only test.') + def test_train_with_defun(self): + train(defun=True) + + @unittest.skipIf(keras_utils.is_v2_0(), 'TF 1.0 only test.') + def test_evaluate_with_defun(self): + evaluate(defun=True) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist_test.py new file mode 100644 index 0000000..207c0d0 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist_test.py @@ -0,0 +1,147 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import time +import unittest + +import tensorflow as tf # pylint: disable=g-bad-import-order +from absl import logging +from official.r1.mnist import mnist +from official.utils.misc import keras_utils + +BATCH_SIZE = 100 + + +def dummy_input_fn(): + image = tf.random.uniform([BATCH_SIZE, 784]) + labels = tf.random.uniform([BATCH_SIZE, 1], maxval=9, dtype=tf.int32) + return image, labels + + +def make_estimator(): + data_format = 'channels_last' + if tf.test.is_built_with_cuda(): + data_format = 'channels_first' + return tf.estimator.Estimator( + model_fn=mnist.model_fn, params={ + 'data_format': data_format + }) + + +class Tests(tf.test.TestCase): + """Run tests for MNIST model. + + MNIST uses contrib and will not work with TF 2.0. All tests are disabled if + using TF 2.0. + """ + + @unittest.skipIf(keras_utils.is_v2_0(), 'TF 1.0 only test.') + def test_mnist(self): + classifier = make_estimator() + classifier.train(input_fn=dummy_input_fn, steps=2) + eval_results = classifier.evaluate(input_fn=dummy_input_fn, steps=1) + + loss = eval_results['loss'] + global_step = eval_results['global_step'] + accuracy = eval_results['accuracy'] + self.assertEqual(loss.shape, ()) + self.assertEqual(2, global_step) + self.assertEqual(accuracy.shape, ()) + + input_fn = lambda: tf.random.uniform([3, 784]) + predictions_generator = classifier.predict(input_fn) + for _ in range(3): + predictions = next(predictions_generator) + self.assertEqual(predictions['probabilities'].shape, (10,)) + self.assertEqual(predictions['classes'].shape, ()) + + @unittest.skipIf(keras_utils.is_v2_0(), 'TF 1.0 only test.') + def mnist_model_fn_helper(self, mode, multi_gpu=False): + features, labels = dummy_input_fn() + image_count = features.shape[0] + spec = mnist.model_fn(features, labels, mode, { + 'data_format': 'channels_last', + 'multi_gpu': multi_gpu + }) + + if mode == tf.estimator.ModeKeys.PREDICT: + predictions = spec.predictions + self.assertAllEqual(predictions['probabilities'].shape, (image_count, 10)) + self.assertEqual(predictions['probabilities'].dtype, tf.float32) + self.assertAllEqual(predictions['classes'].shape, (image_count,)) + self.assertEqual(predictions['classes'].dtype, tf.int64) + + if mode != tf.estimator.ModeKeys.PREDICT: + loss = spec.loss + self.assertAllEqual(loss.shape, ()) + self.assertEqual(loss.dtype, tf.float32) + + if mode == tf.estimator.ModeKeys.EVAL: + eval_metric_ops = spec.eval_metric_ops + self.assertAllEqual(eval_metric_ops['accuracy'][0].shape, ()) + self.assertAllEqual(eval_metric_ops['accuracy'][1].shape, ()) + self.assertEqual(eval_metric_ops['accuracy'][0].dtype, tf.float32) + self.assertEqual(eval_metric_ops['accuracy'][1].dtype, tf.float32) + + @unittest.skipIf(keras_utils.is_v2_0(), 'TF 1.0 only test.') + def test_mnist_model_fn_train_mode(self): + self.mnist_model_fn_helper(tf.estimator.ModeKeys.TRAIN) + + @unittest.skipIf(keras_utils.is_v2_0(), 'TF 1.0 only test.') + def test_mnist_model_fn_train_mode_multi_gpu(self): + self.mnist_model_fn_helper(tf.estimator.ModeKeys.TRAIN, multi_gpu=True) + + @unittest.skipIf(keras_utils.is_v2_0(), 'TF 1.0 only test.') + def test_mnist_model_fn_eval_mode(self): + self.mnist_model_fn_helper(tf.estimator.ModeKeys.EVAL) + + @unittest.skipIf(keras_utils.is_v2_0(), 'TF 1.0 only test.') + def test_mnist_model_fn_predict_mode(self): + self.mnist_model_fn_helper(tf.estimator.ModeKeys.PREDICT) + + +class Benchmarks(tf.test.Benchmark): + """Simple speed benchmarking for MNIST.""" + + def benchmark_train_step_time(self): + classifier = make_estimator() + # Run one step to warmup any use of the GPU. + classifier.train(input_fn=dummy_input_fn, steps=1) + + have_gpu = tf.test.is_gpu_available() + num_steps = 1000 if have_gpu else 100 + name = 'train_step_time_%s' % ('gpu' if have_gpu else 'cpu') + + start = time.time() + classifier.train(input_fn=dummy_input_fn, steps=num_steps) + end = time.time() + + wall_time = (end - start) / num_steps + self.report_benchmark( + iters=num_steps, + wall_time=wall_time, + name=name, + extras={ + 'examples_per_sec': BATCH_SIZE / wall_time + }) + + +if __name__ == '__main__': + logging.set_verbosity(logging.ERROR) + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist_tpu.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist_tpu.py new file mode 100644 index 0000000..4ca62ef --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/mnist/mnist_tpu.py @@ -0,0 +1,202 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""MNIST model training using TPUs. + +This program demonstrates training of the convolutional neural network model +defined in mnist.py on Google Cloud TPUs (https://cloud.google.com/tpu/). + +If you are not interested in TPUs, you should ignore this file. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys + +# pylint: disable=g-bad-import-order +from absl import app as absl_app # pylint: disable=unused-import +import tensorflow.compat.v1 as tf +# pylint: enable=g-bad-import-order + +# For open source environment, add grandparent directory for import +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(sys.path[0])))) + +from official.r1.mnist import dataset # pylint: disable=wrong-import-position +from official.r1.mnist import mnist # pylint: disable=wrong-import-position + +# Cloud TPU Cluster Resolver flags +tf.flags.DEFINE_string( + "tpu", default=None, + help="The Cloud TPU to use for training. This should be either the name " + "used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 " + "url.") +tf.flags.DEFINE_string( + "tpu_zone", default=None, + help="[Optional] GCE zone where the Cloud TPU is located in. If not " + "specified, we will attempt to automatically detect the GCE project from " + "metadata.") +tf.flags.DEFINE_string( + "gcp_project", default=None, + help="[Optional] Project name for the Cloud TPU-enabled project. If not " + "specified, we will attempt to automatically detect the GCE project from " + "metadata.") + +# Model specific parameters +tf.flags.DEFINE_string("data_dir", "", + "Path to directory containing the MNIST dataset") +tf.flags.DEFINE_string("model_dir", None, "Estimator model_dir") +tf.flags.DEFINE_integer("batch_size", 1024, + "Mini-batch size for the training. Note that this " + "is the global batch size and not the per-shard batch.") +tf.flags.DEFINE_integer("train_steps", 1000, "Total number of training steps.") +tf.flags.DEFINE_integer("eval_steps", 0, + "Total number of evaluation steps. If `0`, evaluation " + "after training is skipped.") +tf.flags.DEFINE_float("learning_rate", 0.05, "Learning rate.") + +tf.flags.DEFINE_bool("use_tpu", True, "Use TPUs rather than plain CPUs") +tf.flags.DEFINE_bool("enable_predict", True, "Do some predictions at the end") +tf.flags.DEFINE_integer("iterations", 50, + "Number of iterations per TPU training loop.") +tf.flags.DEFINE_integer("num_shards", 8, "Number of shards (TPU chips).") + +FLAGS = tf.flags.FLAGS + + +def metric_fn(labels, logits): + accuracy = tf.metrics.accuracy( + labels=labels, predictions=tf.argmax(logits, axis=1)) + return {"accuracy": accuracy} + + +def model_fn(features, labels, mode, params): + """model_fn constructs the ML model used to predict handwritten digits.""" + + del params + image = features + if isinstance(image, dict): + image = features["image"] + + model = mnist.create_model("channels_last") + + if mode == tf.estimator.ModeKeys.PREDICT: + logits = model(image, training=False) + predictions = { + 'class_ids': tf.argmax(logits, axis=1), + 'probabilities': tf.nn.softmax(logits), + } + return tf.estimator.tpu.TPUEstimatorSpec(mode, predictions=predictions) + + logits = model(image, training=(mode == tf.estimator.ModeKeys.TRAIN)) + loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) + + if mode == tf.estimator.ModeKeys.TRAIN: + learning_rate = tf.train.exponential_decay( + FLAGS.learning_rate, + tf.train.get_global_step(), + decay_steps=100000, + decay_rate=0.96) + optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate) + if FLAGS.use_tpu: + optimizer = tf.tpu.CrossShardOptimizer(optimizer) + return tf.estimator.tpu.TPUEstimatorSpec( + mode=mode, + loss=loss, + train_op=optimizer.minimize(loss, tf.train.get_global_step())) + + if mode == tf.estimator.ModeKeys.EVAL: + return tf.estimator.tpu.TPUEstimatorSpec( + mode=mode, loss=loss, eval_metrics=(metric_fn, [labels, logits])) + + +def train_input_fn(params): + """train_input_fn defines the input pipeline used for training.""" + batch_size = params["batch_size"] + data_dir = params["data_dir"] + # Retrieves the batch size for the current shard. The # of shards is + # computed according to the input pipeline deployment. See + # `tf.estimator.tpu.RunConfig` for details. + ds = dataset.train(data_dir).cache().repeat().shuffle( + buffer_size=50000).batch(batch_size, drop_remainder=True) + return ds + + +def eval_input_fn(params): + batch_size = params["batch_size"] + data_dir = params["data_dir"] + ds = dataset.test(data_dir).batch(batch_size, drop_remainder=True) + return ds + + +def predict_input_fn(params): + batch_size = params["batch_size"] + data_dir = params["data_dir"] + # Take out top 10 samples from test data to make the predictions. + ds = dataset.test(data_dir).take(10).batch(batch_size) + return ds + + +def main(argv): + del argv # Unused. + tf.logging.set_verbosity(tf.logging.INFO) + + tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver( + FLAGS.tpu, + zone=FLAGS.tpu_zone, + project=FLAGS.gcp_project + ) + + run_config = tf.estimator.tpu.RunConfig( + cluster=tpu_cluster_resolver, + model_dir=FLAGS.model_dir, + session_config=tf.ConfigProto( + allow_soft_placement=True, log_device_placement=True), + tpu_config=tf.estimator.tpu.TPUConfig(FLAGS.iterations, FLAGS.num_shards), + ) + + estimator = tf.estimator.tpu.TPUEstimator( + model_fn=model_fn, + use_tpu=FLAGS.use_tpu, + train_batch_size=FLAGS.batch_size, + eval_batch_size=FLAGS.batch_size, + predict_batch_size=FLAGS.batch_size, + params={"data_dir": FLAGS.data_dir}, + config=run_config) + # TPUEstimator.train *requires* a max_steps argument. + estimator.train(input_fn=train_input_fn, max_steps=FLAGS.train_steps) + # TPUEstimator.evaluate *requires* a steps argument. + # Note that the number of examples used during evaluation is + # --eval_steps * --batch_size. + # So if you change --batch_size then change --eval_steps too. + if FLAGS.eval_steps: + estimator.evaluate(input_fn=eval_input_fn, steps=FLAGS.eval_steps) + + # Run prediction on top few samples of test data. + if FLAGS.enable_predict: + predictions = estimator.predict(input_fn=predict_input_fn) + + for pred_dict in predictions: + template = ('Prediction is "{}" ({:.1f}%).') + + class_id = pred_dict['class_ids'] + probability = pred_dict['probabilities'][class_id] + + print(template.format(class_id, 100 * probability)) + + +if __name__ == "__main__": + tf.disable_v2_behavior() + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/ncf/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/ncf/README.md new file mode 100644 index 0000000..8156d39 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/ncf/README.md @@ -0,0 +1,7 @@ +![No Maintenance Intended](https://img.shields.io/badge/No%20Maintenance%20Intended-%E2%9C%95-red.svg) +![TensorFlow Requirement: 1.x](https://img.shields.io/badge/TensorFlow%20Requirement-1.x-brightgreen) +![TensorFlow 2 Not Supported](https://img.shields.io/badge/TensorFlow%202%20Not%20Supported-%E2%9C%95-red.svg) + +# NCF Estimator implementation + +NCF framework to train and evaluate the NeuMF model diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/ncf/ncf_estimator_main.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/ncf/ncf_estimator_main.py new file mode 100644 index 0000000..76b9c5a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/ncf/ncf_estimator_main.py @@ -0,0 +1,189 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""NCF framework to train and evaluate the NeuMF model. + +The NeuMF model assembles both MF and MLP models under the NCF framework. Check +`neumf_model.py` for more details about the models. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import contextlib +import heapq +import json +import math +import multiprocessing +import os +import signal +import typing + +# pylint: disable=g-bad-import-order +import numpy as np +from absl import app as absl_app +from absl import flags +from absl import logging +import tensorflow as tf +# pylint: enable=g-bad-import-order + +from official.recommendation import constants as rconst +from official.recommendation import data_pipeline +from official.recommendation import data_preprocessing +from official.recommendation import movielens +from official.recommendation import ncf_common +from official.recommendation import neumf_model +from official.utils.flags import core as flags_core +from official.utils.logs import hooks_helper +from official.utils.logs import logger +from official.utils.logs import mlperf_helper +from official.utils.misc import distribution_utils +from official.utils.misc import model_helpers + + +FLAGS = flags.FLAGS + + +def construct_estimator(model_dir, params): + """Construct either an Estimator for NCF. + + Args: + model_dir: The model directory for the estimator + params: The params dict for the estimator + + Returns: + An Estimator. + """ + distribution = ncf_common.get_v1_distribution_strategy(params) + run_config = tf.estimator.RunConfig(train_distribute=distribution, + eval_distribute=distribution) + model_fn = neumf_model.neumf_model_fn + estimator = tf.estimator.Estimator(model_fn=model_fn, model_dir=model_dir, + config=run_config, params=params) + return estimator + + +def log_and_get_hooks(eval_batch_size): + """Convenience function for hook and logger creation.""" + # Create hooks that log information about the training and metric values + train_hooks = hooks_helper.get_train_hooks( + FLAGS.hooks, + model_dir=FLAGS.model_dir, + batch_size=FLAGS.batch_size, # for ExamplesPerSecondHook + tensors_to_log={"cross_entropy": "cross_entropy"} + ) + run_params = { + "batch_size": FLAGS.batch_size, + "eval_batch_size": eval_batch_size, + "number_factors": FLAGS.num_factors, + "hr_threshold": FLAGS.hr_threshold, + "train_epochs": FLAGS.train_epochs, + } + benchmark_logger = logger.get_benchmark_logger() + benchmark_logger.log_run_info( + model_name="recommendation", + dataset_name=FLAGS.dataset, + run_params=run_params, + test_id=FLAGS.benchmark_test_id) + + return benchmark_logger, train_hooks + + +def main(_): + with logger.benchmark_context(FLAGS), \ + mlperf_helper.LOGGER(FLAGS.output_ml_perf_compliance_logging): + mlperf_helper.set_ncf_root(os.path.split(os.path.abspath(__file__))[0]) + run_ncf(FLAGS) + + +def run_ncf(_): + """Run NCF training and eval loop.""" + params = ncf_common.parse_flags(FLAGS) + + num_users, num_items, num_train_steps, num_eval_steps, producer = ( + ncf_common.get_inputs(params)) + + params["num_users"], params["num_items"] = num_users, num_items + producer.start() + model_helpers.apply_clean(flags.FLAGS) + + estimator = construct_estimator(model_dir=FLAGS.model_dir, params=params) + + benchmark_logger, train_hooks = log_and_get_hooks(params["eval_batch_size"]) + total_training_cycle = FLAGS.train_epochs // FLAGS.epochs_between_evals + + target_reached = False + mlperf_helper.ncf_print(key=mlperf_helper.TAGS.TRAIN_LOOP) + for cycle_index in range(total_training_cycle): + assert FLAGS.epochs_between_evals == 1 or not mlperf_helper.LOGGER.enabled + logging.info("Starting a training cycle: {}/{}".format( + cycle_index + 1, total_training_cycle)) + + mlperf_helper.ncf_print(key=mlperf_helper.TAGS.TRAIN_EPOCH, + value=cycle_index) + + train_input_fn = producer.make_input_fn(is_training=True) + estimator.train(input_fn=train_input_fn, hooks=train_hooks, + steps=num_train_steps) + + logging.info("Beginning evaluation.") + eval_input_fn = producer.make_input_fn(is_training=False) + + mlperf_helper.ncf_print(key=mlperf_helper.TAGS.EVAL_START, + value=cycle_index) + eval_results = estimator.evaluate(eval_input_fn, steps=num_eval_steps) + logging.info("Evaluation complete.") + + hr = float(eval_results[rconst.HR_KEY]) + ndcg = float(eval_results[rconst.NDCG_KEY]) + loss = float(eval_results["loss"]) + + mlperf_helper.ncf_print( + key=mlperf_helper.TAGS.EVAL_TARGET, + value={"epoch": cycle_index, "value": FLAGS.hr_threshold}) + mlperf_helper.ncf_print(key=mlperf_helper.TAGS.EVAL_ACCURACY, + value={"epoch": cycle_index, "value": hr}) + mlperf_helper.ncf_print( + key=mlperf_helper.TAGS.EVAL_HP_NUM_NEG, + value={"epoch": cycle_index, "value": rconst.NUM_EVAL_NEGATIVES}) + + mlperf_helper.ncf_print(key=mlperf_helper.TAGS.EVAL_STOP, value=cycle_index) + + # Benchmark the evaluation results + benchmark_logger.log_evaluation_result(eval_results) + # Log the HR and NDCG results. + logging.info( + "Iteration {}: HR = {:.4f}, NDCG = {:.4f}, Loss = {:.4f}".format( + cycle_index + 1, hr, ndcg, loss)) + + # If some evaluation threshold is met + if model_helpers.past_stop_threshold(FLAGS.hr_threshold, hr): + target_reached = True + break + + mlperf_helper.ncf_print(key=mlperf_helper.TAGS.RUN_STOP, + value={"success": target_reached}) + producer.stop_loop() + producer.join() + + # Clear the session explicitly to avoid session delete error + tf.keras.backend.clear_session() + mlperf_helper.ncf_print(key=mlperf_helper.TAGS.RUN_FINAL) + + +if __name__ == "__main__": + logging.set_verbosity(logging.INFO) + ncf_common.define_ncf_flags() + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/README.md new file mode 100644 index 0000000..7f70b50 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/README.md @@ -0,0 +1,156 @@ +![No Maintenance Intended](https://img.shields.io/badge/No%20Maintenance%20Intended-%E2%9C%95-red.svg) +![TensorFlow Requirement: 1.x](https://img.shields.io/badge/TensorFlow%20Requirement-1.x-brightgreen) +![TensorFlow 2 Not Supported](https://img.shields.io/badge/TensorFlow%202%20Not%20Supported-%E2%9C%95-red.svg) + +# ResNet in TensorFlow + +Deep residual networks, or ResNets for short, provided the breakthrough idea of +identity mappings in order to enable training of very deep convolutional neural +networks. This folder contains an implementation of ResNet for the ImageNet +dataset written in TensorFlow. + +See the following papers for more background: + +[1] [Deep Residual Learning for Image Recognition](https://arxiv.org/pdf/1512.03385.pdf) by Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun, Dec 2015. + +[2] [Identity Mappings in Deep Residual Networks](https://arxiv.org/pdf/1603.05027.pdf) by Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun, Jul 2016. + +In code, v1 refers to the ResNet defined in [1] but where a stride 2 is used on +the 3x3 conv rather than the first 1x1 in the bottleneck. This change results +in higher and more stable accuracy with less epochs than the original v1 and has +shown to scale to higher batch sizes with minimal degradation in accuracy. +There is no originating paper. The first mention we are aware of was in the +torch version of [ResNetv1](https://github.com/facebook/fb.resnet.torch). Most +popular v1 implementations are this implementation which we call ResNetv1.5. + +In testing we found v1.5 requires ~12% more compute to train and has 6% reduced +throughput for inference compared to ResNetv1. CIFAR-10 ResNet does not use the +bottleneck and is thus the same for v1 as v1.5. + +v2 refers to [2]. The principle difference between the two versions is that v1 +applies batch normalization and activation after convolution, while v2 applies +batch normalization, then activation, and finally convolution. A schematic +comparison is presented in Figure 1 (left) of [2]. + +Please proceed according to which dataset you would like to train/evaluate on: + + +## CIFAR-10 + +### Setup + +You need to have the latest version of TensorFlow installed. +First, make sure [the models folder is in your Python path](/official/#running-the-models); otherwise you may encounter `ImportError: No module named official.resnet`. + +Then, download and extract the CIFAR-10 data from Alex's website, specifying the location with the `--data_dir` flag. Run the following: + +```bash +python cifar10_download_and_extract.py --data_dir +``` + +Then, to train the model: + +```bash +python cifar10_main.py --data_dir /cifar-10-batches-bin --model_dir +``` + +Use `--data_dir` to specify the location of the CIFAR-10 data used in the previous step. There are more flag options as described in `cifar10_main.py`. + +To export a `SavedModel` from the trained checkpoint: + +```bash +python cifar10_main.py --data_dir /cifar-10-batches-bin --model_dir --eval_only --export_dir +``` + +Note: The `` must be present. You might want to run `mkdir ` beforehand. + +The `SavedModel` can then be [loaded](https://www.tensorflow.org/guide/saved_model#loading_a_savedmodel_in_python) in order to use the ResNet for prediction. + + +## ImageNet + +### Setup +To begin, you will need to download the ImageNet dataset and convert it to +TFRecord format. The following [script](https://github.com/tensorflow/tpu/blob/master/tools/datasets/imagenet_to_gcs.py) +and [README](https://github.com/tensorflow/tpu/tree/master/tools/datasets#imagenet_to_gcspy) +provide a few options. + +Once your dataset is ready, you can begin training the model as follows: + +```bash +python imagenet_main.py --data_dir=/path/to/imagenet +``` + +The model will begin training and will automatically evaluate itself on the +validation data roughly once per epoch. + +Note that there are a number of other options you can specify, including +`--model_dir` to choose where to store the model and `--resnet_size` to choose +the model size (options include ResNet-18 through ResNet-200). See +[`resnet_run_loop.py`](resnet_run_loop.py) for the full list of options. + + +## Compute Devices +Training is accomplished using the DistributionStrategies API. (https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/distribute/README.md) + +The appropriate distribution strategy is chosen based on the `--num_gpus` flag. +By default this flag is one if TensorFlow is compiled with CUDA, and zero +otherwise. + +num_gpus: ++ 0: Use OneDeviceStrategy and train on CPU. ++ 1: Use OneDeviceStrategy and train on GPU. ++ 2+: Use MirroredStrategy (data parallelism) to distribute a batch between devices. + +### Pre-trained model +You can download pre-trained versions of ResNet-50. Reported accuracies are top-1 single-crop accuracy for the ImageNet validation set. +Models are reported as both checkpoints produced by Estimator during training, and as SavedModels which are more portable. Checkpoints are fragile, +and these are not guaranteed to work with future versions of the code. Both ResNet v1 +and ResNet v2 have been trained in both fp16 and fp32 precision. (Here v1 refers to "v1.5". See the note above.) Furthermore, SavedModels +are generated to accept either tensor or JPG inputs, and with channels_first (NCHW) and channels_last (NHWC) convolutions. NCHW is generally +better for GPUs, while NHWC is generally better for CPUs. See the TensorFlow [performance guide](https://www.tensorflow.org/performance/performance_guide#data_formats) +for more details. + +ResNet-50 v2 (fp32, Accuracy 76.47%): +* [Checkpoint](http://download.tensorflow.org/models/official/20181001_resnet/checkpoints/resnet_imagenet_v2_fp32_20181001.tar.gz) +* SavedModel [(NCHW)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp32_savedmodel_NCHW.tar.gz), +[(NCHW, JPG)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp32_savedmodel_NCHW_jpg.tar.gz), +[(NHWC)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp32_savedmodel_NHWC.tar.gz), +[(NHWC, JPG)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp32_savedmodel_NHWC_jpg.tar.gz) + +ResNet-50 v2 (fp16, Accuracy 76.56%): +* [Checkpoint](http://download.tensorflow.org/models/official/20181001_resnet/checkpoints/resnet_imagenet_v2_fp16_20180928.tar.gz) +* SavedModel [(NCHW)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp16_savedmodel_NCHW.tar.gz), +[(NCHW, JPG)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp16_savedmodel_NCHW_jpg.tar.gz), +[(NHWC)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp16_savedmodel_NHWC.tar.gz), +[(NHWC, JPG)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp16_savedmodel_NHWC_jpg.tar.gz) + +ResNet-50 v1 (fp32, Accuracy 76.53%): +* [Checkpoint](http://download.tensorflow.org/models/official/20181001_resnet/checkpoints/resnet_imagenet_v1_fp32_20181001.tar.gz) +* SavedModel [(NCHW)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v1_fp32_savedmodel_NCHW.tar.gz), +[(NCHW, JPG)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v1_fp32_savedmodel_NCHW_jpg.tar.gz), +[(NHWC)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v1_fp32_savedmodel_NHWC.tar.gz), +[(NHWC, JPG)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v1_fp32_savedmodel_NHWC_jpg.tar.gz) + +ResNet-50 v1 (fp16, Accuracy 76.18%): +* [Checkpoint](http://download.tensorflow.org/models/official/20181001_resnet/checkpoints/resnet_imagenet_v1_fp16_20181001.tar.gz) +* SavedModel [(NCHW)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v1_fp16_savedmodel_NCHW.tar.gz), +[(NCHW, JPG)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v1_fp16_savedmodel_NCHW_jpg.tar.gz), +[(NHWC)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v1_fp16_savedmodel_NHWC.tar.gz), +[(NHWC, JPG)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v1_fp16_savedmodel_NHWC_jpg.tar.gz) + +### Transfer Learning +You can use a pretrained model to initialize a training process. In addition you are able to freeze all but the final fully connected layers to fine tune your model. Transfer Learning is useful when training on your own small datasets. For a brief look at transfer learning in the context of convolutional neural networks, we recommend reading these [short notes](http://cs231n.github.io/transfer-learning/). + + +To fine tune a pretrained resnet you must make three changes to your training procedure: + +1) Build the exact same model as previously except we change the number of labels in the final classification layer. + +2) Restore all weights from the pre-trained resnet except for the final classification layer; this will get randomly initialized instead. + +3) Freeze earlier layers of the network + +We can perform these three operations by specifying two flags: ```--pretrained_model_checkpoint_path``` and ```--fine_tune```. The first flag is a string that points to the path of a pre-trained resnet model. If this flag is specified, it will load all but the final classification layer. A key thing to note: if both ```--pretrained_model_checkpoint_path``` and a non empty ```model_dir``` directory are passed, the tensorflow estimator will load only the ```model_dir```. For more on this please see [WarmStartSettings](https://www.tensorflow.org/versions/master/api_docs/python/tf/estimator/WarmStartSettings) and [Estimators](https://www.tensorflow.org/guide/estimators). + +The second flag ```--fine_tune``` is a boolean that indicates whether earlier layers of the network should be frozen. You may set this flag to false if you wish to continue training a pre-trained model from a checkpoint. If you set this flag to true, you can train a new classification layer from scratch. diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/cifar10_download_and_extract.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/cifar10_download_and_extract.py new file mode 100644 index 0000000..a44d042 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/cifar10_download_and_extract.py @@ -0,0 +1,63 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Downloads and extracts the binary version of the CIFAR-10 dataset.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import os +import sys +import tarfile + +from six.moves import urllib +import tensorflow as tf + +DATA_URL = 'https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz' + +parser = argparse.ArgumentParser() + +parser.add_argument( + '--data_dir', type=str, default='/tmp/cifar10_data', + help='Directory to download data and extract the tarball') + + +def main(_): + """Download and extract the tarball from Alex's website.""" + if not os.path.exists(FLAGS.data_dir): + os.makedirs(FLAGS.data_dir) + + filename = DATA_URL.split('/')[-1] + filepath = os.path.join(FLAGS.data_dir, filename) + + if not os.path.exists(filepath): + def _progress(count, block_size, total_size): + sys.stdout.write('\r>> Downloading %s %.1f%%' % ( + filename, 100.0 * count * block_size / total_size)) + sys.stdout.flush() + + filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath, _progress) + print() + statinfo = os.stat(filepath) + print('Successfully downloaded', filename, statinfo.st_size, 'bytes.') + + tarfile.open(filepath, 'r:gz').extractall(FLAGS.data_dir) + + +if __name__ == '__main__': + FLAGS, unparsed = parser.parse_known_args() + tf.compat.v1.app.run(argv=[sys.argv[0]] + unparsed) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/cifar10_main.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/cifar10_main.py new file mode 100644 index 0000000..63b2364 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/cifar10_main.py @@ -0,0 +1,297 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Runs a ResNet model on the CIFAR-10 dataset.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from absl import app as absl_app +from absl import flags +from absl import logging +from six.moves import range +import tensorflow as tf + +from official.r1.resnet import resnet_model +from official.r1.resnet import resnet_run_loop +from official.utils.flags import core as flags_core +from official.utils.logs import logger + +HEIGHT = 32 +WIDTH = 32 +NUM_CHANNELS = 3 +_DEFAULT_IMAGE_BYTES = HEIGHT * WIDTH * NUM_CHANNELS +# The record is the image plus a one-byte label +_RECORD_BYTES = _DEFAULT_IMAGE_BYTES + 1 +NUM_CLASSES = 10 +_NUM_DATA_FILES = 5 + +# TODO(tobyboyd): Change to best practice 45K(train)/5K(val)/10K(test) splits. +NUM_IMAGES = { + 'train': 50000, + 'validation': 10000, +} + +DATASET_NAME = 'CIFAR-10' + + +############################################################################### +# Data processing +############################################################################### +def get_filenames(is_training, data_dir): + """Returns a list of filenames.""" + assert tf.io.gfile.exists(data_dir), ( + 'Run cifar10_download_and_extract.py first to download and extract the ' + 'CIFAR-10 data.') + + if is_training: + return [ + os.path.join(data_dir, 'data_batch_%d.bin' % i) + for i in range(1, _NUM_DATA_FILES + 1) + ] + else: + return [os.path.join(data_dir, 'test_batch.bin')] + + +def parse_record(raw_record, is_training, dtype): + """Parse CIFAR-10 image and label from a raw record.""" + # Convert bytes to a vector of uint8 that is record_bytes long. + record_vector = tf.io.decode_raw(raw_record, tf.uint8) + + # The first byte represents the label, which we convert from uint8 to int32 + # and then to one-hot. + label = tf.cast(record_vector[0], tf.int32) + + # The remaining bytes after the label represent the image, which we reshape + # from [depth * height * width] to [depth, height, width]. + depth_major = tf.reshape(record_vector[1:_RECORD_BYTES], + [NUM_CHANNELS, HEIGHT, WIDTH]) + + # Convert from [depth, height, width] to [height, width, depth], and cast as + # float32. + image = tf.cast(tf.transpose(a=depth_major, perm=[1, 2, 0]), tf.float32) + + image = preprocess_image(image, is_training) + image = tf.cast(image, dtype) + + return image, label + + +def preprocess_image(image, is_training): + """Preprocess a single image of layout [height, width, depth].""" + if is_training: + # Resize the image to add four extra pixels on each side. + image = tf.image.resize_with_crop_or_pad( + image, HEIGHT + 8, WIDTH + 8) + + # Randomly crop a [HEIGHT, WIDTH] section of the image. + image = tf.image.random_crop(image, [HEIGHT, WIDTH, NUM_CHANNELS]) + + # Randomly flip the image horizontally. + image = tf.image.random_flip_left_right(image) + + # Subtract off the mean and divide by the variance of the pixels. + image = tf.image.per_image_standardization(image) + return image + + +def input_fn(is_training, + data_dir, + batch_size, + num_epochs=1, + dtype=tf.float32, + datasets_num_private_threads=None, + parse_record_fn=parse_record, + input_context=None, + drop_remainder=False): + """Input function which provides batches for train or eval. + + Args: + is_training: A boolean denoting whether the input is for training. + data_dir: The directory containing the input data. + batch_size: The number of samples per batch. + num_epochs: The number of epochs to repeat the dataset. + dtype: Data type to use for images/features + datasets_num_private_threads: Number of private threads for tf.data. + parse_record_fn: Function to use for parsing the records. + input_context: A `tf.distribute.InputContext` object passed in by + `tf.distribute.Strategy`. + drop_remainder: A boolean indicates whether to drop the remainder of the + batches. If True, the batch dimension will be static. + + Returns: + A dataset that can be used for iteration. + """ + filenames = get_filenames(is_training, data_dir) + dataset = tf.data.FixedLengthRecordDataset(filenames, _RECORD_BYTES) + + if input_context: + logging.info( + 'Sharding the dataset: input_pipeline_id=%d num_input_pipelines=%d', + input_context.input_pipeline_id, input_context.num_input_pipelines) + dataset = dataset.shard(input_context.num_input_pipelines, + input_context.input_pipeline_id) + + return resnet_run_loop.process_record_dataset( + dataset=dataset, + is_training=is_training, + batch_size=batch_size, + shuffle_buffer=NUM_IMAGES['train'], + parse_record_fn=parse_record_fn, + num_epochs=num_epochs, + dtype=dtype, + datasets_num_private_threads=datasets_num_private_threads, + drop_remainder=drop_remainder + ) + + +def get_synth_input_fn(dtype): + return resnet_run_loop.get_synth_input_fn( + HEIGHT, WIDTH, NUM_CHANNELS, NUM_CLASSES, dtype=dtype) + + +############################################################################### +# Running the model +############################################################################### +class Cifar10Model(resnet_model.Model): + """Model class with appropriate defaults for CIFAR-10 data.""" + + def __init__(self, resnet_size, data_format=None, num_classes=NUM_CLASSES, + resnet_version=resnet_model.DEFAULT_VERSION, + dtype=resnet_model.DEFAULT_DTYPE): + """These are the parameters that work for CIFAR-10 data. + + Args: + resnet_size: The number of convolutional layers needed in the model. + data_format: Either 'channels_first' or 'channels_last', specifying which + data format to use when setting up the model. + num_classes: The number of output classes needed from the model. This + enables users to extend the same model to their own datasets. + resnet_version: Integer representing which version of the ResNet network + to use. See README for details. Valid values: [1, 2] + dtype: The TensorFlow dtype to use for calculations. + + Raises: + ValueError: if invalid resnet_size is chosen + """ + if resnet_size % 6 != 2: + raise ValueError('resnet_size must be 6n + 2:', resnet_size) + + num_blocks = (resnet_size - 2) // 6 + + super(Cifar10Model, self).__init__( + resnet_size=resnet_size, + bottleneck=False, + num_classes=num_classes, + num_filters=16, + kernel_size=3, + conv_stride=1, + first_pool_size=None, + first_pool_stride=None, + block_sizes=[num_blocks] * 3, + block_strides=[1, 2, 2], + resnet_version=resnet_version, + data_format=data_format, + dtype=dtype + ) + + +def cifar10_model_fn(features, labels, mode, params): + """Model function for CIFAR-10.""" + features = tf.reshape(features, [-1, HEIGHT, WIDTH, NUM_CHANNELS]) + # Learning rate schedule follows arXiv:1512.03385 for ResNet-56 and under. + learning_rate_fn = resnet_run_loop.learning_rate_with_decay( + batch_size=params['batch_size'] * params.get('num_workers', 1), + batch_denom=128, num_images=NUM_IMAGES['train'], + boundary_epochs=[91, 136, 182], decay_rates=[1, 0.1, 0.01, 0.001]) + + # Weight decay of 2e-4 diverges from 1e-4 decay used in the ResNet paper + # and seems more stable in testing. The difference was nominal for ResNet-56. + weight_decay = 2e-4 + + # Empirical testing showed that including batch_normalization variables + # in the calculation of regularized loss helped validation accuracy + # for the CIFAR-10 dataset, perhaps because the regularization prevents + # overfitting on the small data set. We therefore include all vars when + # regularizing and computing loss during training. + def loss_filter_fn(_): + return True + + return resnet_run_loop.resnet_model_fn( + features=features, + labels=labels, + mode=mode, + model_class=Cifar10Model, + resnet_size=params['resnet_size'], + weight_decay=weight_decay, + learning_rate_fn=learning_rate_fn, + momentum=0.9, + data_format=params['data_format'], + resnet_version=params['resnet_version'], + loss_scale=params['loss_scale'], + loss_filter_fn=loss_filter_fn, + dtype=params['dtype'], + fine_tune=params['fine_tune'] + ) + + +def define_cifar_flags(): + resnet_run_loop.define_resnet_flags() + flags.adopt_module_key_flags(resnet_run_loop) + flags_core.set_defaults(data_dir='/tmp/cifar10_data/cifar-10-batches-bin', + model_dir='/tmp/cifar10_model', + resnet_size='56', + train_epochs=182, + epochs_between_evals=10, + batch_size=128, + image_bytes_as_serving_input=False) + + +def run_cifar(flags_obj): + """Run ResNet CIFAR-10 training and eval loop. + + Args: + flags_obj: An object containing parsed flag values. + + Returns: + Dictionary of results. Including final accuracy. + """ + if flags_obj.image_bytes_as_serving_input: + logging.fatal( + '--image_bytes_as_serving_input cannot be set to True for CIFAR. ' + 'This flag is only applicable to ImageNet.') + return + + input_function = (flags_obj.use_synthetic_data and + get_synth_input_fn(flags_core.get_tf_dtype(flags_obj)) or + input_fn) + result = resnet_run_loop.resnet_main( + flags_obj, cifar10_model_fn, input_function, DATASET_NAME, + shape=[HEIGHT, WIDTH, NUM_CHANNELS]) + + return result + + +def main(_): + with logger.benchmark_context(flags.FLAGS): + run_cifar(flags.FLAGS) + + +if __name__ == '__main__': + logging.set_verbosity(logging.INFO) + define_cifar_flags() + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/cifar10_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/cifar10_test.py new file mode 100644 index 0000000..827cb0c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/cifar10_test.py @@ -0,0 +1,185 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tempfile import mkstemp + +from absl import logging +import numpy as np +import tensorflow as tf + +from official.r1.resnet import cifar10_main +from official.utils.misc import keras_utils +from official.utils.testing import integration + +logging.set_verbosity(logging.ERROR) + +_BATCH_SIZE = 128 +_HEIGHT = 32 +_WIDTH = 32 +_NUM_CHANNELS = 3 + + +class BaseTest(tf.test.TestCase): + """Tests for the Cifar10 version of Resnet. + """ + + _num_validation_images = None + + @classmethod + def setUpClass(cls): # pylint: disable=invalid-name + super(BaseTest, cls).setUpClass() + if keras_utils.is_v2_0: + tf.compat.v1.disable_eager_execution() + cifar10_main.define_cifar_flags() + + def setUp(self): + super(BaseTest, self).setUp() + self._num_validation_images = cifar10_main.NUM_IMAGES['validation'] + cifar10_main.NUM_IMAGES['validation'] = 4 + + def tearDown(self): + super(BaseTest, self).tearDown() + tf.io.gfile.rmtree(self.get_temp_dir()) + cifar10_main.NUM_IMAGES['validation'] = self._num_validation_images + + def test_dataset_input_fn(self): + fake_data = bytearray() + fake_data.append(7) + for i in range(_NUM_CHANNELS): + for _ in range(_HEIGHT * _WIDTH): + fake_data.append(i) + + _, filename = mkstemp(dir=self.get_temp_dir()) + data_file = open(filename, 'wb') + data_file.write(fake_data) + data_file.close() + + fake_dataset = tf.data.FixedLengthRecordDataset( + filename, cifar10_main._RECORD_BYTES) # pylint: disable=protected-access + fake_dataset = fake_dataset.map( + lambda val: cifar10_main.parse_record(val, False, tf.float32)) + image, label = tf.compat.v1.data.make_one_shot_iterator( + fake_dataset).get_next() + + self.assertAllEqual(label.shape, ()) + self.assertAllEqual(image.shape, (_HEIGHT, _WIDTH, _NUM_CHANNELS)) + + with self.session() as sess: + image, label = sess.run([image, label]) + + self.assertEqual(label, 7) + + for row in image: + for pixel in row: + self.assertAllClose(pixel, np.array([-1.225, 0., 1.225]), rtol=1e-3) + + def cifar10_model_fn_helper(self, mode, resnet_version, dtype): + input_fn = cifar10_main.get_synth_input_fn(dtype) + dataset = input_fn(True, '', _BATCH_SIZE) + iterator = tf.compat.v1.data.make_initializable_iterator(dataset) + features, labels = iterator.get_next() + spec = cifar10_main.cifar10_model_fn( + features, labels, mode, { + 'dtype': dtype, + 'resnet_size': 32, + 'data_format': 'channels_last', + 'batch_size': _BATCH_SIZE, + 'resnet_version': resnet_version, + 'loss_scale': 128 if dtype == tf.float16 else 1, + 'fine_tune': False, + }) + + predictions = spec.predictions + self.assertAllEqual(predictions['probabilities'].shape, + (_BATCH_SIZE, 10)) + self.assertEqual(predictions['probabilities'].dtype, tf.float32) + self.assertAllEqual(predictions['classes'].shape, (_BATCH_SIZE,)) + self.assertEqual(predictions['classes'].dtype, tf.int64) + + if mode != tf.estimator.ModeKeys.PREDICT: + loss = spec.loss + self.assertAllEqual(loss.shape, ()) + self.assertEqual(loss.dtype, tf.float32) + + if mode == tf.estimator.ModeKeys.EVAL: + eval_metric_ops = spec.eval_metric_ops + self.assertAllEqual(eval_metric_ops['accuracy'][0].shape, ()) + self.assertAllEqual(eval_metric_ops['accuracy'][1].shape, ()) + self.assertEqual(eval_metric_ops['accuracy'][0].dtype, tf.float32) + self.assertEqual(eval_metric_ops['accuracy'][1].dtype, tf.float32) + + def test_cifar10_model_fn_train_mode_v1(self): + self.cifar10_model_fn_helper(tf.estimator.ModeKeys.TRAIN, resnet_version=1, + dtype=tf.float32) + + def test_cifar10_model_fn_trainmode__v2(self): + self.cifar10_model_fn_helper(tf.estimator.ModeKeys.TRAIN, resnet_version=2, + dtype=tf.float32) + + def test_cifar10_model_fn_eval_mode_v1(self): + self.cifar10_model_fn_helper(tf.estimator.ModeKeys.EVAL, resnet_version=1, + dtype=tf.float32) + + def test_cifar10_model_fn_eval_mode_v2(self): + self.cifar10_model_fn_helper(tf.estimator.ModeKeys.EVAL, resnet_version=2, + dtype=tf.float32) + + def test_cifar10_model_fn_predict_mode_v1(self): + self.cifar10_model_fn_helper(tf.estimator.ModeKeys.PREDICT, + resnet_version=1, dtype=tf.float32) + + def test_cifar10_model_fn_predict_mode_v2(self): + self.cifar10_model_fn_helper(tf.estimator.ModeKeys.PREDICT, + resnet_version=2, dtype=tf.float32) + + def _test_cifar10model_shape(self, resnet_version): + batch_size = 135 + num_classes = 246 + + model = cifar10_main.Cifar10Model(32, data_format='channels_last', + num_classes=num_classes, + resnet_version=resnet_version) + fake_input = tf.random.uniform([batch_size, _HEIGHT, _WIDTH, _NUM_CHANNELS]) + output = model(fake_input, training=True) + + self.assertAllEqual(output.shape, (batch_size, num_classes)) + + def test_cifar10model_shape_v1(self): + self._test_cifar10model_shape(resnet_version=1) + + def test_cifar10model_shape_v2(self): + self._test_cifar10model_shape(resnet_version=2) + + def test_cifar10_end_to_end_synthetic_v1(self): + integration.run_synthetic( + main=cifar10_main.run_cifar, tmp_root=self.get_temp_dir(), + extra_flags=['-resnet_version', '1', '-batch_size', '4', + '--max_train_steps', '1'] + ) + + def test_cifar10_end_to_end_synthetic_v2(self): + integration.run_synthetic( + main=cifar10_main.run_cifar, tmp_root=self.get_temp_dir(), + extra_flags=['-resnet_version', '2', '-batch_size', '4', + '--max_train_steps', '1'] + ) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/estimator_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/estimator_benchmark.py new file mode 100644 index 0000000..eef3ea4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/estimator_benchmark.py @@ -0,0 +1,500 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Executes Estimator benchmarks and accuracy tests.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import time + +from absl import flags +from absl import logging +from absl.testing import flagsaver +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.r1.resnet import cifar10_main as cifar_main +from official.r1.resnet import imagenet_main +from official.utils.flags import core as flags_core +from official.utils.logs import hooks + +IMAGENET_DATA_DIR_NAME = 'imagenet' +CIFAR_DATA_DIR_NAME = 'cifar-10-batches-bin' +FLAGS = flags.FLAGS + + +class EstimatorBenchmark(tf.test.Benchmark): + """Base class to hold methods common to test classes in the module. + + Code under test for Estimator models (ResNet50 and 56) report mostly the + same data and require the same FLAG setup. + """ + + local_flags = None + + def __init__(self, output_dir=None, default_flags=None, flag_methods=None): + if not output_dir: + output_dir = '/tmp' + self.output_dir = output_dir + self.default_flags = default_flags or {} + self.flag_methods = flag_methods or {} + + def _get_model_dir(self, folder_name): + """Returns directory to store info, e.g. saved model and event log.""" + return os.path.join(self.output_dir, folder_name) + + def _setup(self): + """Sets up and resets flags before each test.""" + logging.set_verbosity(logging.INFO) + if EstimatorBenchmark.local_flags is None: + for flag_method in self.flag_methods: + flag_method() + # Loads flags to get defaults to then override. List cannot be empty. + flags.FLAGS(['foo']) + # Overrides flag values with defaults for the class of tests. + for k, v in self.default_flags.items(): + setattr(FLAGS, k, v) + saved_flag_values = flagsaver.save_flag_values() + EstimatorBenchmark.local_flags = saved_flag_values + else: + flagsaver.restore_flag_values(EstimatorBenchmark.local_flags) + + def _report_benchmark(self, + stats, + wall_time_sec, + top_1_max=None, + top_1_min=None): + """Report benchmark results by writing to local protobuf file. + + Args: + stats: dict returned from estimator models with known entries. + wall_time_sec: the during of the benchmark execution in seconds + top_1_max: highest passing level for top_1 accuracy. + top_1_min: lowest passing level for top_1 accuracy. + """ + + examples_per_sec_hook = None + for hook in stats['train_hooks']: + if isinstance(hook, hooks.ExamplesPerSecondHook): + examples_per_sec_hook = hook + break + + eval_results = stats['eval_results'] + metrics = [] + if 'accuracy' in eval_results: + metrics.append({'name': 'accuracy_top_1', + 'value': float(eval_results['accuracy']), + 'min_value': top_1_min, + 'max_value': top_1_max}) + if 'accuracy_top_5' in eval_results: + metrics.append({'name': 'accuracy_top_5', + 'value': float(eval_results['accuracy_top_5'])}) + + if examples_per_sec_hook: + exp_per_second_list = examples_per_sec_hook.current_examples_per_sec_list + # ExamplesPerSecondHook skips the first 10 steps. + exp_per_sec = sum(exp_per_second_list) / (len(exp_per_second_list)) + metrics.append({'name': 'exp_per_second', + 'value': exp_per_sec}) + flags_str = flags_core.get_nondefault_flags_as_str() + self.report_benchmark( + iters=eval_results.get('global_step', None), + wall_time=wall_time_sec, + metrics=metrics, + extras={'flags': flags_str}) + + +class Resnet50EstimatorAccuracy(EstimatorBenchmark): + """Benchmark accuracy tests for ResNet50 w/ Estimator.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + """Benchmark accuracy tests for ResNet50 w/ Estimator. + + Args: + output_dir: directory where to output e.g. log files + root_data_dir: directory under which to look for dataset + **kwargs: arbitrary named arguments. This is needed to make the + constructor forward compatible in case PerfZero provides more + named arguments before updating the constructor. + """ + flag_methods = [imagenet_main.define_imagenet_flags] + + self.data_dir = os.path.join(root_data_dir, IMAGENET_DATA_DIR_NAME) + super(Resnet50EstimatorAccuracy, self).__init__( + output_dir=output_dir, flag_methods=flag_methods) + + def benchmark_graph_8_gpu(self): + """Test 8 GPUs graph mode.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 128 * 8 + FLAGS.train_epochs = 90 + FLAGS.epochs_between_evals = 10 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_8_gpu') + FLAGS.dtype = 'fp32' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_8_gpu(self): + """Test FP16 8 GPUs graph mode.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 256 * 8 + FLAGS.train_epochs = 90 + FLAGS.epochs_between_evals = 10 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_fp16_8_gpu') + FLAGS.dtype = 'fp16' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_graph_rewrite_8_gpu(self): + """Test FP16 graph rewrite 8 GPUs graph mode.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 256 * 8 + FLAGS.train_epochs = 90 + FLAGS.epochs_between_evals = 10 + FLAGS.model_dir = self._get_model_dir( + 'benchmark_graph_fp16_graph_rewrite_8_gpu') + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def _run_and_report_benchmark(self): + start_time_sec = time.time() + stats = imagenet_main.run_imagenet(flags.FLAGS) + wall_time_sec = time.time() - start_time_sec + self._report_benchmark(stats, + wall_time_sec, + top_1_min=0.762, + top_1_max=0.766) + + +class Resnet50EstimatorBenchmarkBase(EstimatorBenchmark): + """Base class for benchmarks for ResNet50 using Estimator.""" + local_flags = None + + def __init__(self, output_dir=None, default_flags=None): + flag_methods = [imagenet_main.define_imagenet_flags] + + super(Resnet50EstimatorBenchmarkBase, self).__init__( + output_dir=output_dir, + default_flags=default_flags, + flag_methods=flag_methods) + + def _run_and_report_benchmark(self): + start_time_sec = time.time() + stats = imagenet_main.run_imagenet(FLAGS) + wall_time_sec = time.time() - start_time_sec + print(stats) + # Remove values to skip triggering accuracy check. + stats['eval_results'].pop('accuracy', None) + stats['eval_results'].pop('accuracy_top_5', None) + + self._report_benchmark(stats, wall_time_sec) + + +class Resnet50EstimatorBenchmark(Resnet50EstimatorBenchmarkBase): + """Benchmarks for ResNet50 using Estimator with 1 worker.""" + + def __init__(self, output_dir=None, default_flags=None): + super(Resnet50EstimatorBenchmark, self).__init__( + output_dir=output_dir, + default_flags=default_flags) + + def benchmark_graph_fp16_1_gpu(self): + """Benchmarks graph fp16 1 gpu.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_fp16_1_gpu') + FLAGS.batch_size = 128 + FLAGS.dtype = 'fp16' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_1_gpu_tweaked(self): + """Benchmarks graph fp16 1 gpu tweaked.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.intra_op_parallelism_threads = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_fp16_1_gpu_tweaked') + FLAGS.batch_size = 256 + FLAGS.dtype = 'fp16' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_graph_rewrite_1_gpu_tweaked(self): + """Benchmarks graph fp16 graph rewrite 1 gpu tweaked.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.intra_op_parallelism_threads = 1 + FLAGS.model_dir = self._get_model_dir( + 'benchmark_graph_fp16_graph_rewrite_1_gpu_tweaked') + FLAGS.batch_size = 256 + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_1_gpu(self): + """Benchmarks graph 1 gpu.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_1_gpu') + FLAGS.batch_size = 128 + FLAGS.dtype = 'fp32' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_8_gpu(self): + """Benchmarks graph 8 gpus.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_8_gpu') + FLAGS.batch_size = 128*8 + FLAGS.dtype = 'fp32' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_8_gpu(self): + """Benchmarks graph fp16 8 gpus.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_fp16_8_gpu') + FLAGS.batch_size = 256*8 + FLAGS.dtype = 'fp16' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_8_gpu_tweaked(self): + """Benchmarks graph fp16 8 gpus tweaked.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.intra_op_parallelism_threads = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_fp16_8_gpu_tweaked') + FLAGS.batch_size = 256*8 + FLAGS.dtype = 'fp16' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_graph_rewrite_8_gpu_tweaked(self): + """Benchmarks graph fp16 graph rewrite 8 gpus tweaked.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.intra_op_parallelism_threads = 1 + FLAGS.model_dir = self._get_model_dir( + 'benchmark_graph_fp16_graph_rewrite_8_gpu_tweaked') + FLAGS.batch_size = 256*8 + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + +class Resnet50EstimatorBenchmarkSynth(Resnet50EstimatorBenchmark): + """Resnet50 synthetic benchmark tests.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + def_flags = {} + def_flags['use_synthetic_data'] = True + def_flags['max_train_steps'] = 110 + def_flags['train_epochs'] = 1 + + super(Resnet50EstimatorBenchmarkSynth, self).__init__( + output_dir=output_dir, default_flags=def_flags) + + +class Resnet50EstimatorBenchmarkReal(Resnet50EstimatorBenchmark): + """Resnet50 real data benchmark tests.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + def_flags = {} + def_flags['data_dir'] = os.path.join(root_data_dir, IMAGENET_DATA_DIR_NAME) + def_flags['max_train_steps'] = 110 + def_flags['train_epochs'] = 1 + + super(Resnet50EstimatorBenchmarkReal, self).__init__( + output_dir=output_dir, default_flags=def_flags) + + +class Resnet50MultiWorkerEstimatorBenchmark(Resnet50EstimatorBenchmarkBase): + """Benchmarks for ResNet50 using Estimator with multiple workers.""" + + def __init__(self, output_dir=None, default_flags=None): + super(Resnet50MultiWorkerEstimatorBenchmark, self).__init__( + output_dir=output_dir, + default_flags=default_flags) + + def benchmark_graph_fp16_8_gpu_ring_tweaked(self): + """Benchmarks graph fp16 8 gpus with ring collective tweaked.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.distribution_strategy = 'multi_worker_mirrored' + FLAGS.all_reduce_alg = 'ring' + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.intra_op_parallelism_threads = 1 + FLAGS.datasets_num_private_threads = 32 + FLAGS.model_dir = self._get_model_dir( + folder_name='benchmark_graph_fp16_8_gpu_ring_tweaked') + FLAGS.batch_size = 256*8 + FLAGS.dtype = 'fp16' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_8_gpu_nccl_tweaked(self): + """Benchmarks graph fp16 8 gpus with nccl collective tweaked.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.distribution_strategy = 'multi_worker_mirrored' + FLAGS.all_reduce_alg = 'nccl' + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.intra_op_parallelism_threads = 1 + FLAGS.datasets_num_private_threads = 32 + FLAGS.model_dir = self._get_model_dir( + folder_name='benchmark_graph_fp16_8_gpu_nccl_tweaked') + FLAGS.batch_size = 256*8 + FLAGS.dtype = 'fp16' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + +class Resnet50MultiWorkerEstimatorBenchmarkSynth( + Resnet50MultiWorkerEstimatorBenchmark): + """ResNet50, multi-worker, Estimator, synthetic data.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + def_flags = {} + def_flags['use_synthetic_data'] = True + def_flags['max_train_steps'] = 110 + def_flags['train_epochs'] = 1 + + super(Resnet50MultiWorkerEstimatorBenchmarkSynth, self).__init__( + output_dir=output_dir, default_flags=def_flags) + + +class Resnet56EstimatorAccuracy(EstimatorBenchmark): + """Accuracy tests for Estimator ResNet56.""" + + local_flags = None + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + """A benchmark class. + + Args: + output_dir: directory where to output e.g. log files + root_data_dir: directory under which to look for dataset + **kwargs: arbitrary named arguments. This is needed to make the + constructor forward compatible in case PerfZero provides more + named arguments before updating the constructor. + """ + flag_methods = [cifar_main.define_cifar_flags] + + self.data_dir = os.path.join(root_data_dir, CIFAR_DATA_DIR_NAME) + super(Resnet56EstimatorAccuracy, self).__init__( + output_dir=output_dir, flag_methods=flag_methods) + + def benchmark_graph_1_gpu(self): + """Test layers model with Estimator and distribution strategies.""" + self._setup() + flags.FLAGS.num_gpus = 1 + flags.FLAGS.data_dir = self.data_dir + flags.FLAGS.batch_size = 128 + flags.FLAGS.train_epochs = 182 + flags.FLAGS.model_dir = self._get_model_dir('benchmark_graph_1_gpu') + flags.FLAGS.resnet_size = 56 + flags.FLAGS.dtype = 'fp32' + flags.FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_1_gpu(self): + """Test layers FP16 model with Estimator and distribution strategies.""" + self._setup() + flags.FLAGS.num_gpus = 1 + flags.FLAGS.data_dir = self.data_dir + flags.FLAGS.batch_size = 128 + flags.FLAGS.train_epochs = 182 + flags.FLAGS.model_dir = self._get_model_dir('benchmark_graph_fp16_1_gpu') + flags.FLAGS.resnet_size = 56 + flags.FLAGS.dtype = 'fp16' + flags.FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_2_gpu(self): + """Test layers model with Estimator and dist_strat. 2 GPUs.""" + self._setup() + flags.FLAGS.num_gpus = 2 + flags.FLAGS.data_dir = self.data_dir + flags.FLAGS.batch_size = 128 + flags.FLAGS.train_epochs = 182 + flags.FLAGS.model_dir = self._get_model_dir('benchmark_graph_2_gpu') + flags.FLAGS.resnet_size = 56 + flags.FLAGS.dtype = 'fp32' + flags.FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_2_gpu(self): + """Test layers FP16 model with Estimator and dist_strat. 2 GPUs.""" + self._setup() + flags.FLAGS.num_gpus = 2 + flags.FLAGS.data_dir = self.data_dir + flags.FLAGS.batch_size = 128 + flags.FLAGS.train_epochs = 182 + flags.FLAGS.model_dir = self._get_model_dir('benchmark_graph_fp16_2_gpu') + flags.FLAGS.resnet_size = 56 + flags.FLAGS.dtype = 'fp16' + flags.FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def unit_test(self): + """A lightweight test that can finish quickly.""" + self._setup() + flags.FLAGS.num_gpus = 1 + flags.FLAGS.data_dir = self.data_dir + flags.FLAGS.batch_size = 128 + flags.FLAGS.train_epochs = 1 + flags.FLAGS.model_dir = self._get_model_dir('unit_test') + flags.FLAGS.resnet_size = 8 + flags.FLAGS.dtype = 'fp32' + flags.FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def _run_and_report_benchmark(self): + """Executes benchmark and reports result.""" + start_time_sec = time.time() + stats = cifar_main.run_cifar(flags.FLAGS) + wall_time_sec = time.time() - start_time_sec + + self._report_benchmark(stats, + wall_time_sec, + top_1_min=0.926, + top_1_max=0.938) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/gpu_train.sh b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/gpu_train.sh new file mode 100644 index 0000000..7fe2e53 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/gpu_train.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + +nohup python imagenet_main.py \ +--resnet_size=101 \ +--batch_size=1024 \ +--num_gpus=8 \ +--dtype=fp32 \ +--train_epochs=90 \ +--epochs_between_evals=5 \ +--hooks=ExamplesPerSecondHook \ +--data_dir=/home/hiscv/dataset/imagenet_TF_record \ +--model_dir=./model_dir > train.log & diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/imagenet_main.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/imagenet_main.py new file mode 100644 index 0000000..c665f3f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/imagenet_main.py @@ -0,0 +1,471 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Runs a ResNet model on the ImageNet dataset.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys +import datetime +import numpy as np + +from absl import app as absl_app +from absl import flags +from absl import logging +from six.moves import range +import tensorflow as tf +import shutil +import random + +sys.path.append(os.path.dirname(os.path.realpath(__file__)) + '/../../../') +sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '../../../../../../../utils/atlasboost')) +flags.DEFINE_integer('device_count', 1, 'number of device') + +from official.r1.resnet import imagenet_preprocessing +from official.r1.resnet import resnet_model +from official.r1.resnet import resnet_run_loop +from official.utils.flags import core as flags_core +from official.utils.logs import logger +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter + +############## npu modify begin ############# +from npu_bridge.estimator import npu_ops +from hccl.manage.api import get_local_rank_id +from hccl.manage.api import get_rank_size +from hccl.manage.api import get_rank_id +from tensorflow.core.protobuf import rewriter_config_pb2 + +############## npu modify end ############### + +DEFAULT_IMAGE_SIZE = 224 +NUM_CHANNELS = 3 +NUM_CLASSES = 1001 + +NUM_IMAGES = { + 'train': 1281167, + 'validation': 50000, +} + +_NUM_TRAIN_FILES = 1024 +_SHUFFLE_BUFFER = 10000 + +DATASET_NAME = 'ImageNet' + + +############################################################################### +# Data processing +############################################################################### +def get_filenames(is_training, data_dir, test_mode=False): + """Return filenames for dataset.""" + if test_mode == True: + tf.compat.v1.logging.info('test mode, data_dir is : %s', data_dir) + return [ + os.path.join(data_dir, i) + for i in os.listdir(data_dir)] + elif is_training: + return [ + os.path.join(data_dir, 'train-%05d-of-01024' % i) + for i in range(_NUM_TRAIN_FILES)] + else: + return [ + os.path.join(data_dir, 'validation-%05d-of-00128' % i) + for i in range(128)] + + +def _parse_example_proto(example_serialized): + """Parses an Example proto containing a training example of an image. + + The output of the build_image_data.py image preprocessing script is a dataset + containing serialized Example protocol buffers. Each Example proto contains + the following fields (values are included as examples): + + image/height: 462 + image/width: 581 + image/colorspace: 'RGB' + image/channels: 3 + image/class/label: 615 + image/class/synset: 'n03623198' + image/class/text: 'knee pad' + image/object/bbox/xmin: 0.1 + image/object/bbox/xmax: 0.9 + image/object/bbox/ymin: 0.2 + image/object/bbox/ymax: 0.6 + image/object/bbox/label: 615 + image/format: 'JPEG' + image/filename: 'ILSVRC2012_val_00041207.JPEG' + image/encoded: + + Args: + example_serialized: scalar Tensor tf.string containing a serialized + Example protocol buffer. + + Returns: + image_buffer: Tensor tf.string containing the contents of a JPEG file. + label: Tensor tf.int32 containing the label. + bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords] + where each coordinate is [0, 1) and the coordinates are arranged as + [ymin, xmin, ymax, xmax]. + """ + # Dense features in Example proto. + feature_map = { + 'image/encoded': tf.io.FixedLenFeature([], dtype=tf.string, + default_value=''), + 'image/class/label': tf.io.FixedLenFeature([], dtype=tf.int64, + default_value=-1), + 'image/class/text': tf.io.FixedLenFeature([], dtype=tf.string, + default_value=''), + } + sparse_float32 = tf.io.VarLenFeature(dtype=tf.float32) + # Sparse features in Example proto. + feature_map.update( + {k: sparse_float32 for k in ['image/object/bbox/xmin', + 'image/object/bbox/ymin', + 'image/object/bbox/xmax', + 'image/object/bbox/ymax']}) + + features = tf.io.parse_single_example(serialized=example_serialized, + features=feature_map) + label = tf.cast(features['image/class/label'], dtype=tf.int32) + + xmin = tf.expand_dims(features['image/object/bbox/xmin'].values, 0) + ymin = tf.expand_dims(features['image/object/bbox/ymin'].values, 0) + xmax = tf.expand_dims(features['image/object/bbox/xmax'].values, 0) + ymax = tf.expand_dims(features['image/object/bbox/ymax'].values, 0) + + # Note that we impose an ordering of (y, x) just to make life difficult. + bbox = tf.concat([ymin, xmin, ymax, xmax], 0) + + # Force the variable number of bounding boxes into the shape + # [1, num_boxes, coords]. + bbox = tf.expand_dims(bbox, 0) + bbox = tf.transpose(a=bbox, perm=[0, 2, 1]) + + return features['image/encoded'], label, bbox + + +def parse_record(raw_record, is_training, dtype): + """Parses a record containing a training example of an image. + + The input record is parsed into a label and image, and the image is passed + through preprocessing steps (cropping, flipping, and so on). + + Args: + raw_record: scalar Tensor tf.string containing a serialized + Example protocol buffer. + is_training: A boolean denoting whether the input is for training. + dtype: data type to use for images/features. + + Returns: + Tuple with processed image tensor and one-hot-encoded label tensor. + """ + image_buffer, label, bbox = _parse_example_proto(raw_record) + # add by wx933141 + # resnet_dedicated_log.logger1.info("namespace:%s,time_ms:%d, event_type:pre_process_event,root_dir:%s" % ()) + # resnet_dedicated_log.logger1.info("namespace:%s,time_ms:%d,event_type:init_start, root_dir:%s" % ()) + image = imagenet_preprocessing.preprocess_image( + image_buffer=image_buffer, + bbox=bbox, + output_height=DEFAULT_IMAGE_SIZE, + output_width=DEFAULT_IMAGE_SIZE, + num_channels=NUM_CHANNELS, + is_training=is_training) + image = tf.cast(image, dtype) + # resnet_dedicated_log.logger1.info("namespace:%s,time_ms:%d,event_type:init_stop, root_dir:%s" % (resnet_dedicated_log.work_num, resnet_dedicated_log.datatime, resnet_dedicated_log.root_dir)) + return image, label + + +def input_fn(is_training, + data_dir, + batch_size, + num_epochs=1, + dtype=tf.float32, + datasets_num_private_threads=None, + parse_record_fn=parse_record, + input_context=None, + drop_remainder=False, + tf_data_experimental_slack=False, + + rank_size=None, + rank_id=None, + + test_mode=False, + + ): + """Input function which provides batches for train or eval. + + Args: + is_training: A boolean denoting whether the input is for training. + data_dir: The directory containing the input data. + batch_size: The number of samples per batch. + num_epochs: The number of epochs to repeat the dataset. + dtype: Data type to use for images/features + datasets_num_private_threads: Number of private threads for tf.data. + parse_record_fn: Function to use for parsing the records. + input_context: A `tf.distribute.InputContext` object passed in by + `tf.distribute.Strategy`. + drop_remainder: A boolean indicates whether to drop the remainder of the + batches. If True, the batch dimension will be static. + tf_data_experimental_slack: Whether to enable tf.data's + `experimental_slack` option. + + Returns: + A dataset that can be used for iteration. + """ + filenames = get_filenames(is_training, data_dir, test_mode=test_mode) + dataset = tf.data.Dataset.from_tensor_slices(filenames) + + if input_context and not test_mode: + ############## npu modify begin ############# + dataset = dataset.shard(rank_size, rank_id) + ############## npu modify end ############### + + if is_training and not test_mode: + # Shuffle the input files + dataset = dataset.shuffle(buffer_size=_NUM_TRAIN_FILES) + + # Convert to individual records. + # cycle_length = 10 means that up to 10 files will be read and deserialized in + # parallel. You may want to increase this number if you have a large number of + # CPU cores. + dataset = dataset.interleave( + tf.data.TFRecordDataset, + cycle_length=10, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + + return resnet_run_loop.process_record_dataset( + dataset=dataset, + is_training=is_training, + batch_size=batch_size, + shuffle_buffer=_SHUFFLE_BUFFER, + parse_record_fn=parse_record_fn, + num_epochs=num_epochs, + dtype=dtype, + datasets_num_private_threads=datasets_num_private_threads, + drop_remainder=drop_remainder, + tf_data_experimental_slack=tf_data_experimental_slack, + test_mode=test_mode, + ) + + +def get_synth_input_fn(dtype): + return resnet_run_loop.get_synth_input_fn( + DEFAULT_IMAGE_SIZE, DEFAULT_IMAGE_SIZE, NUM_CHANNELS, NUM_CLASSES, + dtype=dtype) + + +############################################################################### +# Running the model +############################################################################### +class ImagenetModel(resnet_model.Model): + """Model class with appropriate defaults for Imagenet data.""" + + def __init__(self, resnet_size, data_format=None, num_classes=NUM_CLASSES, + resnet_version=resnet_model.DEFAULT_VERSION, + dtype=resnet_model.DEFAULT_DTYPE): + """These are the parameters that work for Imagenet data. + + Args: + resnet_size: The number of convolutional layers needed in the model. + data_format: Either 'channels_first' or 'channels_last', specifying which + data format to use when setting up the model. + num_classes: The number of output classes needed from the model. This + enables users to extend the same model to their own datasets. + resnet_version: Integer representing which version of the ResNet network + to use. See README for details. Valid values: [1, 2] + dtype: The TensorFlow dtype to use for calculations. + """ + + # For bigger models, we want to use "bottleneck" layers + if resnet_size < 50: + bottleneck = False + else: + bottleneck = True + + super(ImagenetModel, self).__init__( + resnet_size=resnet_size, + bottleneck=bottleneck, + num_classes=num_classes, + num_filters=64, + kernel_size=7, + conv_stride=2, + first_pool_size=3, + first_pool_stride=2, + block_sizes=_get_block_sizes(resnet_size), + block_strides=[1, 2, 2, 2], + resnet_version=resnet_version, + data_format=data_format, + dtype=dtype + ) + + +def _get_block_sizes(resnet_size): + """Retrieve the size of each block_layer in the ResNet model. + + The number of block layers used for the Resnet model varies according + to the size of the model. This helper grabs the layer set we want, throwing + an error if a non-standard size has been selected. + + Args: + resnet_size: The number of convolutional layers needed in the model. + + Returns: + A list of block sizes to use in building the model. + + Raises: + KeyError: if invalid resnet_size is received. + """ + choices = { + 18: [2, 2, 2, 2], + 34: [3, 4, 6, 3], + 50: [3, 4, 6, 3], + 101: [3, 4, 23, 3], + 152: [3, 8, 36, 3], + 200: [3, 24, 36, 3] + } + + try: + return choices[resnet_size] + except KeyError: + err = ('Could not find layers for selected Resnet size.\n' + 'Size received: {}; sizes allowed: {}.'.format( + resnet_size, list(choices.keys()))) + raise ValueError(err) + + +def imagenet_model_fn(features, labels, mode, params): + """Our model_fn for ResNet to be used with our Estimator.""" + + # Warmup and higher lr may not be valid for fine tuning with small batches + # and smaller numbers of training images. + if params['fine_tune']: + warmup = False + base_lr = .1 + else: + warmup = True + base_lr = .128 + + ############## npu modify begin ############# + # For NPU lr setting + if params['base_lr'] != 0: + base_lr = params['base_lr'] + ############## npu modify end ############### + + learning_rate_fn = resnet_run_loop.learning_rate_with_decay( + batch_size=params['num_gpus'] * params['batch_size'], + batch_denom=256, num_images=NUM_IMAGES['train'], + boundary_epochs=[30, 60, 80, 90], decay_rates=[1, 0.1, 0.01, 0.001, 1e-4], + warmup=warmup, base_lr=base_lr, + cosine_lr=params['cosine_lr'], train_epochs=params['train_epochs'], + ) + + return resnet_run_loop.resnet_model_fn( + features=features, + labels=labels, + mode=mode, + model_class=ImagenetModel, + resnet_size=params['resnet_size'], + weight_decay=flags.FLAGS.weight_decay, + learning_rate_fn=learning_rate_fn, + momentum=0.9, + data_format=params['data_format'], + resnet_version=params['resnet_version'], + loss_scale=params['loss_scale'], + loss_filter_fn=None, + dtype=params['dtype'], + fine_tune=params['fine_tune'], + label_smoothing=flags.FLAGS.label_smoothing + ) + + +def define_imagenet_flags(): + resnet_run_loop.define_resnet_flags( + resnet_size_choices=['18', '34', '50', '101', '152', '200'], + dynamic_loss_scale=True, + fp16_implementation=True) + flags.adopt_module_key_flags(resnet_run_loop) + flags_core.set_defaults(train_epochs=90) + + +def run_imagenet(flags_obj): + """Run ResNet ImageNet training and eval loop. + + Args: + flags_obj: An object containing parsed flag values. + + Returns: + Dict of results of the run. Contains the keys `eval_results` and + `train_hooks`. `eval_results` contains accuracy (top_1) and + accuracy_top_5. `train_hooks` is a list the instances of hooks used during + training. + """ + if flags_obj.random_seed >= 0: + SEED = flags_obj.random_seed + else: + SEED = 0 + os.environ['PYTHONHASHSEED'] = str(SEED) + random.seed(SEED) + tf.compat.v1.set_random_seed(SEED) + np.random.seed(SEED) + tf.compat.v1.logging.info('setting random_seed: %d', SEED) + + if flags_obj.use_synthetic_data == True: + tf.compat.v1.logging.info('using synthetic data.') + input_function = get_synth_input_fn(flags_core.get_tf_dtype(flags_obj)) + else: + input_function = input_fn + result = resnet_run_loop.resnet_main( + flags_obj, imagenet_model_fn, input_function, DATASET_NAME, NUM_IMAGES, + shape=[DEFAULT_IMAGE_SIZE, DEFAULT_IMAGE_SIZE, NUM_CHANNELS]) + + return result + + + +def main(_): + ############## npu modify begin ############# + # Init NPU ,then can call HCCL Interface + init_sess, npu_init = resnet_run_loop.init_npu() + init_sess.run(npu_init) + ############## npu modify end ############### + + with logger.benchmark_context(flags.FLAGS): + flags.FLAGS.model_dir = os.getenv('MODEL_CKPT_PATH') + run_imagenet(flags.FLAGS) + + +if __name__ == '__main__': + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("tensorflow") + config_info = get_model_parameter("tensorflow_config") + initinal_data = {"base_lr": 0.128, "dataset": "imagenet1024", "optimizer": "SGD", "loss_scale": 512} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + hwlog.remark_print(key=hwlog.INPUT_BATCH_SIZE, value=initinal_data.get("batchsize")) + cluster_device_id = None + logging.set_verbosity(logging.INFO) + define_imagenet_flags() + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/imagenet_preprocessing.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/imagenet_preprocessing.py new file mode 100644 index 0000000..891b58a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/imagenet_preprocessing.py @@ -0,0 +1,262 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Provides utilities to preprocess images. + +Training images are sampled using the provided bounding boxes, and subsequently +cropped to the sampled bounding box. Images are additionally flipped randomly, +then resized to the target output size (without aspect-ratio preservation). + +Images used during evaluation are resized (with aspect-ratio preservation) and +centrally cropped. + +All images undergo mean color subtraction. + +Note that these steps are colloquially referred to as "ResNet preprocessing," +and they differ from "VGG preprocessing," which does not use bounding boxes +and instead does an aspect-preserving resize followed by random crop during +training. (These both differ from "Inception preprocessing," which introduces +color distortion steps.) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +_R_MEAN = 123.68 +_G_MEAN = 116.78 +_B_MEAN = 103.94 +_CHANNEL_MEANS = [_R_MEAN, _G_MEAN, _B_MEAN] + +# The lower bound for the smallest side of the image for aspect-preserving +# resizing. For example, if an image is 500 x 1000, it will be resized to +# _RESIZE_MIN x (_RESIZE_MIN * 2). +_RESIZE_MIN = 256 + + +def _decode_crop_and_flip(image_buffer, bbox, num_channels): + """Crops the given image to a random part of the image, and randomly flips. + + We use the fused decode_and_crop op, which performs better than the two ops + used separately in series, but note that this requires that the image be + passed in as an un-decoded string Tensor. + + Args: + image_buffer: scalar string Tensor representing the raw JPEG image buffer. + bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords] + where each coordinate is [0, 1) and the coordinates are arranged as + [ymin, xmin, ymax, xmax]. + num_channels: Integer depth of the image buffer for decoding. + + Returns: + 3-D tensor with cropped image. + + """ + # A large fraction of image datasets contain a human-annotated bounding box + # delineating the region of the image containing the object of interest. We + # choose to create a new bounding box for the object which is a randomly + # distorted version of the human-annotated bounding box that obeys an + # allowed range of aspect ratios, sizes and overlap with the human-annotated + # bounding box. If no box is supplied, then we assume the bounding box is + # the entire image. + sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box( + tf.image.extract_jpeg_shape(image_buffer), + bounding_boxes=bbox, + min_object_covered=0.1, + aspect_ratio_range=[0.75, 1.33], + area_range=[0.05, 1.0], + max_attempts=100, + use_image_if_no_bounding_boxes=True) + bbox_begin, bbox_size, _ = sample_distorted_bounding_box + + # Reassemble the bounding box in the format the crop op requires. + offset_y, offset_x, _ = tf.unstack(bbox_begin) + target_height, target_width, _ = tf.unstack(bbox_size) + crop_window = tf.stack([offset_y, offset_x, target_height, target_width]) + + # Use the fused decode and crop op here, which is faster than each in series. + cropped = tf.image.decode_and_crop_jpeg( + image_buffer, crop_window, channels=num_channels) + + # Flip to add a little more random distortion in. + cropped = tf.image.random_flip_left_right(cropped) + return cropped + + +def _central_crop(image, crop_height, crop_width): + """Performs central crops of the given image list. + + Args: + image: a 3-D image tensor + crop_height: the height of the image following the crop. + crop_width: the width of the image following the crop. + + Returns: + 3-D tensor with cropped image. + """ + shape = tf.shape(input=image) + height, width = shape[0], shape[1] + + amount_to_be_cropped_h = (height - crop_height) + crop_top = amount_to_be_cropped_h // 2 + amount_to_be_cropped_w = (width - crop_width) + crop_left = amount_to_be_cropped_w // 2 + return tf.slice( + image, [crop_top, crop_left, 0], [crop_height, crop_width, -1]) + + +def _mean_image_subtraction(image, means, num_channels): + """Subtracts the given means from each image channel. + + For example: + means = [123.68, 116.779, 103.939] + image = _mean_image_subtraction(image, means) + + Note that the rank of `image` must be known. + + Args: + image: a tensor of size [height, width, C]. + means: a C-vector of values to subtract from each channel. + num_channels: number of color channels in the image that will be distorted. + + Returns: + the centered image. + + Raises: + ValueError: If the rank of `image` is unknown, if `image` has a rank other + than three or if the number of channels in `image` doesn't match the + number of values in `means`. + """ + if image.get_shape().ndims != 3: + raise ValueError('Input must be of size [height, width, C>0]') + + if len(means) != num_channels: + raise ValueError('len(means) must match the number of channels') + + # We have a 1-D tensor of means; convert to 3-D. + # Note(b/130245863): we explicitly call `broadcast` instead of simply + # expanding dimensions for better performance. + means = tf.broadcast_to(means, tf.shape(image)) + + return image - means + + +def _smallest_size_at_least(height, width, resize_min): + """Computes new shape with the smallest side equal to `smallest_side`. + + Computes new shape with the smallest side equal to `smallest_side` while + preserving the original aspect ratio. + + Args: + height: an int32 scalar tensor indicating the current height. + width: an int32 scalar tensor indicating the current width. + resize_min: A python integer or scalar `Tensor` indicating the size of + the smallest side after resize. + + Returns: + new_height: an int32 scalar tensor indicating the new height. + new_width: an int32 scalar tensor indicating the new width. + """ + resize_min = tf.cast(resize_min, tf.float32) + + # Convert to floats to make subsequent calculations go smoothly. + height, width = tf.cast(height, tf.float32), tf.cast(width, tf.float32) + + smaller_dim = tf.minimum(height, width) + scale_ratio = resize_min / smaller_dim + + # Convert back to ints to make heights and widths that TF ops will accept. + new_height = tf.cast(height * scale_ratio, tf.int32) + new_width = tf.cast(width * scale_ratio, tf.int32) + + return new_height, new_width + + +def _aspect_preserving_resize(image, resize_min): + """Resize images preserving the original aspect ratio. + + Args: + image: A 3-D image `Tensor`. + resize_min: A python integer or scalar `Tensor` indicating the size of + the smallest side after resize. + + Returns: + resized_image: A 3-D tensor containing the resized image. + """ + shape = tf.shape(input=image) + height, width = shape[0], shape[1] + + new_height, new_width = _smallest_size_at_least(height, width, resize_min) + + return _resize_image(image, new_height, new_width) + + +def _resize_image(image, height, width): + """Simple wrapper around tf.resize_images. + + This is primarily to make sure we use the same `ResizeMethod` and other + details each time. + + Args: + image: A 3-D image `Tensor`. + height: The target height for the resized image. + width: The target width for the resized image. + + Returns: + resized_image: A 3-D tensor containing the resized image. The first two + dimensions have the shape [height, width]. + """ + return tf.compat.v1.image.resize( + image, [height, width], method=tf.image.ResizeMethod.BILINEAR, + align_corners=False) + + +def preprocess_image(image_buffer, bbox, output_height, output_width, + num_channels, is_training=False): + """Preprocesses the given image. + + Preprocessing includes decoding, cropping, and resizing for both training + and eval images. Training preprocessing, however, introduces some random + distortion of the image to improve accuracy. + + Args: + image_buffer: scalar string Tensor representing the raw JPEG image buffer. + bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords] + where each coordinate is [0, 1) and the coordinates are arranged as + [ymin, xmin, ymax, xmax]. + output_height: The height of the image after preprocessing. + output_width: The width of the image after preprocessing. + num_channels: Integer depth of the image buffer for decoding. + is_training: `True` if we're preprocessing the image for training and + `False` otherwise. + + Returns: + A preprocessed image. + """ + if is_training: + # For training, we want to randomize some of the distortions. + image = _decode_crop_and_flip(image_buffer, bbox, num_channels) + image = _resize_image(image, output_height, output_width) + else: + # For validation, we want to decode, resize, then just crop the middle. + image = tf.image.decode_jpeg(image_buffer, channels=num_channels) + image = _aspect_preserving_resize(image, _RESIZE_MIN) + image = _central_crop(image, output_height, output_width) + + image.set_shape([output_height, output_width, num_channels]) + + return _mean_image_subtraction(image, _CHANNEL_MEANS, num_channels) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/imagenet_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/imagenet_test.py new file mode 100644 index 0000000..7d33ef0 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/imagenet_test.py @@ -0,0 +1,327 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest + +import tensorflow as tf # pylint: disable=g-bad-import-order +from absl import logging + +from official.r1.resnet import imagenet_main +from official.utils.misc import keras_utils +from official.utils.testing import integration + +logging.set_verbosity(logging.ERROR) + +_BATCH_SIZE = 32 +_LABEL_CLASSES = 1001 + + +class BaseTest(tf.test.TestCase): + + _num_validation_images = None + + @classmethod + def setUpClass(cls): # pylint: disable=invalid-name + super(BaseTest, cls).setUpClass() + imagenet_main.define_imagenet_flags() + + def setUp(self): + super(BaseTest, self).setUp() + if keras_utils.is_v2_0: + tf.compat.v1.disable_eager_execution() + self._num_validation_images = imagenet_main.NUM_IMAGES['validation'] + imagenet_main.NUM_IMAGES['validation'] = 4 + + def tearDown(self): + super(BaseTest, self).tearDown() + tf.io.gfile.rmtree(self.get_temp_dir()) + imagenet_main.NUM_IMAGES['validation'] = self._num_validation_images + + def _tensor_shapes_helper(self, resnet_size, resnet_version, dtype, with_gpu): + """Checks the tensor shapes after each phase of the ResNet model.""" + def reshape(shape): + """Returns the expected dimensions depending on if a GPU is being used.""" + + # If a GPU is used for the test, the shape is returned (already in NCHW + # form). When GPU is not used, the shape is converted to NHWC. + if with_gpu: + return shape + return shape[0], shape[2], shape[3], shape[1] + + graph = tf.Graph() + + with graph.as_default(), self.test_session( + graph=graph, use_gpu=with_gpu, force_gpu=with_gpu): + model = imagenet_main.ImagenetModel( + resnet_size=resnet_size, + data_format='channels_first' if with_gpu else 'channels_last', + resnet_version=resnet_version, + dtype=dtype + ) + inputs = tf.random.uniform([1, 224, 224, 3]) + output = model(inputs, training=True) + + initial_conv = graph.get_tensor_by_name('resnet_model/initial_conv:0') + max_pool = graph.get_tensor_by_name('resnet_model/initial_max_pool:0') + block_layer1 = graph.get_tensor_by_name('resnet_model/block_layer1:0') + block_layer2 = graph.get_tensor_by_name('resnet_model/block_layer2:0') + block_layer3 = graph.get_tensor_by_name('resnet_model/block_layer3:0') + block_layer4 = graph.get_tensor_by_name('resnet_model/block_layer4:0') + reduce_mean = graph.get_tensor_by_name('resnet_model/final_reduce_mean:0') + dense = graph.get_tensor_by_name('resnet_model/final_dense:0') + + self.assertAllEqual(initial_conv.shape, reshape((1, 64, 112, 112))) + self.assertAllEqual(max_pool.shape, reshape((1, 64, 56, 56))) + + # The number of channels after each block depends on whether we're + # using the building_block or the bottleneck_block. + if resnet_size < 50: + self.assertAllEqual(block_layer1.shape, reshape((1, 64, 56, 56))) + self.assertAllEqual(block_layer2.shape, reshape((1, 128, 28, 28))) + self.assertAllEqual(block_layer3.shape, reshape((1, 256, 14, 14))) + self.assertAllEqual(block_layer4.shape, reshape((1, 512, 7, 7))) + self.assertAllEqual(reduce_mean.shape, reshape((1, 512, 1, 1))) + else: + self.assertAllEqual(block_layer1.shape, reshape((1, 256, 56, 56))) + self.assertAllEqual(block_layer2.shape, reshape((1, 512, 28, 28))) + self.assertAllEqual(block_layer3.shape, reshape((1, 1024, 14, 14))) + self.assertAllEqual(block_layer4.shape, reshape((1, 2048, 7, 7))) + self.assertAllEqual(reduce_mean.shape, reshape((1, 2048, 1, 1))) + + self.assertAllEqual(dense.shape, (1, _LABEL_CLASSES)) + self.assertAllEqual(output.shape, (1, _LABEL_CLASSES)) + + def tensor_shapes_helper(self, resnet_size, resnet_version, with_gpu=False): + self._tensor_shapes_helper(resnet_size=resnet_size, + resnet_version=resnet_version, + dtype=tf.float32, with_gpu=with_gpu) + self._tensor_shapes_helper(resnet_size=resnet_size, + resnet_version=resnet_version, + dtype=tf.float16, with_gpu=with_gpu) + + def test_tensor_shapes_resnet_18_v1(self): + self.tensor_shapes_helper(18, resnet_version=1) + + def test_tensor_shapes_resnet_18_v2(self): + self.tensor_shapes_helper(18, resnet_version=2) + + def test_tensor_shapes_resnet_34_v1(self): + self.tensor_shapes_helper(34, resnet_version=1) + + def test_tensor_shapes_resnet_34_v2(self): + self.tensor_shapes_helper(34, resnet_version=2) + + def test_tensor_shapes_resnet_50_v1(self): + self.tensor_shapes_helper(50, resnet_version=1) + + def test_tensor_shapes_resnet_50_v2(self): + self.tensor_shapes_helper(50, resnet_version=2) + + def test_tensor_shapes_resnet_101_v1(self): + self.tensor_shapes_helper(101, resnet_version=1) + + def test_tensor_shapes_resnet_101_v2(self): + self.tensor_shapes_helper(101, resnet_version=2) + + def test_tensor_shapes_resnet_152_v1(self): + self.tensor_shapes_helper(152, resnet_version=1) + + def test_tensor_shapes_resnet_152_v2(self): + self.tensor_shapes_helper(152, resnet_version=2) + + def test_tensor_shapes_resnet_200_v1(self): + self.tensor_shapes_helper(200, resnet_version=1) + + def test_tensor_shapes_resnet_200_v2(self): + self.tensor_shapes_helper(200, resnet_version=2) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_18_with_gpu_v1(self): + self.tensor_shapes_helper(18, resnet_version=1, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_18_with_gpu_v2(self): + self.tensor_shapes_helper(18, resnet_version=2, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_34_with_gpu_v1(self): + self.tensor_shapes_helper(34, resnet_version=1, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_34_with_gpu_v2(self): + self.tensor_shapes_helper(34, resnet_version=2, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_50_with_gpu_v1(self): + self.tensor_shapes_helper(50, resnet_version=1, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_50_with_gpu_v2(self): + self.tensor_shapes_helper(50, resnet_version=2, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_101_with_gpu_v1(self): + self.tensor_shapes_helper(101, resnet_version=1, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_101_with_gpu_v2(self): + self.tensor_shapes_helper(101, resnet_version=2, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_152_with_gpu_v1(self): + self.tensor_shapes_helper(152, resnet_version=1, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_152_with_gpu_v2(self): + self.tensor_shapes_helper(152, resnet_version=2, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_200_with_gpu_v1(self): + self.tensor_shapes_helper(200, resnet_version=1, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_200_with_gpu_v2(self): + self.tensor_shapes_helper(200, resnet_version=2, with_gpu=True) + + def resnet_model_fn_helper(self, mode, resnet_version, dtype): + """Tests that the EstimatorSpec is given the appropriate arguments.""" + tf.compat.v1.train.create_global_step() + + input_fn = imagenet_main.get_synth_input_fn(dtype) + dataset = input_fn(True, '', _BATCH_SIZE) + iterator = tf.compat.v1.data.make_initializable_iterator(dataset) + features, labels = iterator.get_next() + spec = imagenet_main.imagenet_model_fn( + features, labels, mode, { + 'dtype': dtype, + 'resnet_size': 50, + 'data_format': 'channels_last', + 'batch_size': _BATCH_SIZE, + 'resnet_version': resnet_version, + 'loss_scale': 128 if dtype == tf.float16 else 1, + 'fine_tune': False, + }) + + predictions = spec.predictions + self.assertAllEqual(predictions['probabilities'].shape, + (_BATCH_SIZE, _LABEL_CLASSES)) + self.assertEqual(predictions['probabilities'].dtype, tf.float32) + self.assertAllEqual(predictions['classes'].shape, (_BATCH_SIZE,)) + self.assertEqual(predictions['classes'].dtype, tf.int64) + + if mode != tf.estimator.ModeKeys.PREDICT: + loss = spec.loss + self.assertAllEqual(loss.shape, ()) + self.assertEqual(loss.dtype, tf.float32) + + if mode == tf.estimator.ModeKeys.EVAL: + eval_metric_ops = spec.eval_metric_ops + self.assertAllEqual(eval_metric_ops['accuracy'][0].shape, ()) + self.assertAllEqual(eval_metric_ops['accuracy'][1].shape, ()) + self.assertEqual(eval_metric_ops['accuracy'][0].dtype, tf.float32) + self.assertEqual(eval_metric_ops['accuracy'][1].dtype, tf.float32) + + def test_resnet_model_fn_train_mode_v1(self): + self.resnet_model_fn_helper(tf.estimator.ModeKeys.TRAIN, resnet_version=1, + dtype=tf.float32) + + def test_resnet_model_fn_train_mode_v2(self): + self.resnet_model_fn_helper(tf.estimator.ModeKeys.TRAIN, resnet_version=2, + dtype=tf.float32) + + def test_resnet_model_fn_eval_mode_v1(self): + self.resnet_model_fn_helper(tf.estimator.ModeKeys.EVAL, resnet_version=1, + dtype=tf.float32) + + def test_resnet_model_fn_eval_mode_v2(self): + self.resnet_model_fn_helper(tf.estimator.ModeKeys.EVAL, resnet_version=2, + dtype=tf.float32) + + def test_resnet_model_fn_predict_mode_v1(self): + self.resnet_model_fn_helper(tf.estimator.ModeKeys.PREDICT, resnet_version=1, + dtype=tf.float32) + + def test_resnet_model_fn_predict_mode_v2(self): + self.resnet_model_fn_helper(tf.estimator.ModeKeys.PREDICT, resnet_version=2, + dtype=tf.float32) + + def _test_imagenetmodel_shape(self, resnet_version): + batch_size = 135 + num_classes = 246 + + model = imagenet_main.ImagenetModel( + 50, data_format='channels_last', num_classes=num_classes, + resnet_version=resnet_version) + + fake_input = tf.random.uniform([batch_size, 224, 224, 3]) + output = model(fake_input, training=True) + + self.assertAllEqual(output.shape, (batch_size, num_classes)) + + def test_imagenetmodel_shape_v1(self): + self._test_imagenetmodel_shape(resnet_version=1) + + def test_imagenetmodel_shape_v2(self): + self._test_imagenetmodel_shape(resnet_version=2) + + def test_imagenet_end_to_end_synthetic_v1(self): + integration.run_synthetic( + main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(), + extra_flags=['-resnet_version', '1', '-batch_size', '4', + '--max_train_steps', '1'] + ) + + def test_imagenet_end_to_end_synthetic_v2(self): + integration.run_synthetic( + main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(), + extra_flags=['-resnet_version', '2', '-batch_size', '4', + '--max_train_steps', '1'] + ) + + def test_imagenet_end_to_end_synthetic_v1_tiny(self): + integration.run_synthetic( + main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(), + extra_flags=['-resnet_version', '1', '-batch_size', '4', + '-resnet_size', '18', '--max_train_steps', '1'] + ) + + def test_imagenet_end_to_end_synthetic_v2_tiny(self): + integration.run_synthetic( + main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(), + extra_flags=['-resnet_version', '2', '-batch_size', '4', + '-resnet_size', '18', '--max_train_steps', '1'] + ) + + def test_imagenet_end_to_end_synthetic_v1_huge(self): + integration.run_synthetic( + main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(), + extra_flags=['-resnet_version', '1', '-batch_size', '4', + '-resnet_size', '200', '--max_train_steps', '1'] + ) + + def test_imagenet_end_to_end_synthetic_v2_huge(self): + integration.run_synthetic( + main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(), + extra_flags=['-resnet_version', '2', '-batch_size', '4', + '-resnet_size', '200', '--max_train_steps', '1'] + ) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/resnet_model.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/resnet_model.py new file mode 100644 index 0000000..611c880 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/resnet_model.py @@ -0,0 +1,552 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains definitions for Residual Networks. + +Residual networks ('v1' ResNets) were originally proposed in: +[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Deep Residual Learning for Image Recognition. arXiv:1512.03385 + +The full preactivation 'v2' ResNet variant was introduced by: +[2] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Identity Mappings in Deep Residual Networks. arXiv: 1603.05027 + +The key difference of the full preactivation 'v2' variant compared to the +'v1' variant in [1] is the use of batch normalization before every weight layer +rather than after. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +_BATCH_NORM_DECAY = 0.997 +_BATCH_NORM_EPSILON = 1e-5 +DEFAULT_VERSION = 2 +DEFAULT_DTYPE = tf.float32 +CASTABLE_TYPES = (tf.float16,) +ALLOWED_TYPES = (DEFAULT_DTYPE,) + CASTABLE_TYPES + + +################################################################################ +# Convenience functions for building the ResNet model. +################################################################################ +def batch_norm(inputs, training, data_format): + """Performs a batch normalization using a standard set of parameters.""" + # We set fused=True for a significant performance boost. See + # https://www.tensorflow.org/performance/performance_guide#common_fused_ops + return tf.compat.v1.layers.batch_normalization( + inputs=inputs, axis=1 if data_format == 'channels_first' else 3, + momentum=_BATCH_NORM_DECAY, epsilon=_BATCH_NORM_EPSILON, center=True, + scale=True, training=training, fused=True) + + +def fixed_padding(inputs, kernel_size, data_format): + """Pads the input along the spatial dimensions independently of input size. + + Args: + inputs: A tensor of size [batch, channels, height_in, width_in] or + [batch, height_in, width_in, channels] depending on data_format. + kernel_size: The kernel to be used in the conv2d or max_pool2d operation. + Should be a positive integer. + data_format: The input format ('channels_last' or 'channels_first'). + + Returns: + A tensor with the same format as the input with the data either intact + (if kernel_size == 1) or padded (if kernel_size > 1). + """ + pad_total = kernel_size - 1 + pad_beg = pad_total // 2 + pad_end = pad_total - pad_beg + + if data_format == 'channels_first': + padded_inputs = tf.pad(tensor=inputs, + paddings=[[0, 0], [0, 0], [pad_beg, pad_end], + [pad_beg, pad_end]]) + else: + padded_inputs = tf.pad(tensor=inputs, + paddings=[[0, 0], [pad_beg, pad_end], + [pad_beg, pad_end], [0, 0]]) + return padded_inputs + + +def conv2d_fixed_padding(inputs, filters, kernel_size, strides, data_format): + """Strided 2-D convolution with explicit padding.""" + # The padding is consistent and is based only on `kernel_size`, not on the + # dimensions of `inputs` (as opposed to using `tf.layers.conv2d` alone). + if strides > 1: + inputs = fixed_padding(inputs, kernel_size, data_format) + + return tf.compat.v1.layers.conv2d( + inputs=inputs, filters=filters, kernel_size=kernel_size, strides=strides, + padding=('SAME' if strides == 1 else 'VALID'), use_bias=False, + kernel_initializer=tf.compat.v1.variance_scaling_initializer(), + data_format=data_format) + + +################################################################################ +# ResNet block definitions. +################################################################################ +def _building_block_v1(inputs, filters, training, projection_shortcut, strides, + data_format): + """A single block for ResNet v1, without a bottleneck. + + Convolution then batch normalization then ReLU as described by: + Deep Residual Learning for Image Recognition + https://arxiv.org/pdf/1512.03385.pdf + by Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun, Dec 2015. + + Args: + inputs: A tensor of size [batch, channels, height_in, width_in] or + [batch, height_in, width_in, channels] depending on data_format. + filters: The number of filters for the convolutions. + training: A Boolean for whether the model is in training or inference + mode. Needed for batch normalization. + projection_shortcut: The function to use for projection shortcuts + (typically a 1x1 convolution when downsampling the input). + strides: The block's stride. If greater than 1, this block will ultimately + downsample the input. + data_format: The input format ('channels_last' or 'channels_first'). + + Returns: + The output tensor of the block; shape should match inputs. + """ + shortcut = inputs + + if projection_shortcut is not None: + shortcut = projection_shortcut(inputs) + shortcut = batch_norm(inputs=shortcut, training=training, + data_format=data_format) + + inputs = conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=3, strides=strides, + data_format=data_format) + inputs = batch_norm(inputs, training, data_format) + inputs = tf.nn.relu(inputs) + + inputs = conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=3, strides=1, + data_format=data_format) + inputs = batch_norm(inputs, training, data_format) + inputs += shortcut + inputs = tf.nn.relu(inputs) + + return inputs + + +def _building_block_v2(inputs, filters, training, projection_shortcut, strides, + data_format): + """A single block for ResNet v2, without a bottleneck. + + Batch normalization then ReLu then convolution as described by: + Identity Mappings in Deep Residual Networks + https://arxiv.org/pdf/1603.05027.pdf + by Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun, Jul 2016. + + Args: + inputs: A tensor of size [batch, channels, height_in, width_in] or + [batch, height_in, width_in, channels] depending on data_format. + filters: The number of filters for the convolutions. + training: A Boolean for whether the model is in training or inference + mode. Needed for batch normalization. + projection_shortcut: The function to use for projection shortcuts + (typically a 1x1 convolution when downsampling the input). + strides: The block's stride. If greater than 1, this block will ultimately + downsample the input. + data_format: The input format ('channels_last' or 'channels_first'). + + Returns: + The output tensor of the block; shape should match inputs. + """ + shortcut = inputs + inputs = batch_norm(inputs, training, data_format) + inputs = tf.nn.relu(inputs) + + # The projection shortcut should come after the first batch norm and ReLU + # since it performs a 1x1 convolution. + if projection_shortcut is not None: + shortcut = projection_shortcut(inputs) + + inputs = conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=3, strides=strides, + data_format=data_format) + + inputs = batch_norm(inputs, training, data_format) + inputs = tf.nn.relu(inputs) + inputs = conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=3, strides=1, + data_format=data_format) + + return inputs + shortcut + + +def _bottleneck_block_v1(inputs, filters, training, projection_shortcut, + strides, data_format): + """A single block for ResNet v1, with a bottleneck. + + Similar to _building_block_v1(), except using the "bottleneck" blocks + described in: + Convolution then batch normalization then ReLU as described by: + Deep Residual Learning for Image Recognition + https://arxiv.org/pdf/1512.03385.pdf + by Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun, Dec 2015. + + Args: + inputs: A tensor of size [batch, channels, height_in, width_in] or + [batch, height_in, width_in, channels] depending on data_format. + filters: The number of filters for the convolutions. + training: A Boolean for whether the model is in training or inference + mode. Needed for batch normalization. + projection_shortcut: The function to use for projection shortcuts + (typically a 1x1 convolution when downsampling the input). + strides: The block's stride. If greater than 1, this block will ultimately + downsample the input. + data_format: The input format ('channels_last' or 'channels_first'). + + Returns: + The output tensor of the block; shape should match inputs. + """ + shortcut = inputs + + if projection_shortcut is not None: + shortcut = projection_shortcut(inputs) + shortcut = batch_norm(inputs=shortcut, training=training, + data_format=data_format) + + inputs = conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=1, strides=1, + data_format=data_format) + inputs = batch_norm(inputs, training, data_format) + inputs = tf.nn.relu(inputs) + + inputs = conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=3, strides=strides, + data_format=data_format) + inputs = batch_norm(inputs, training, data_format) + inputs = tf.nn.relu(inputs) + + inputs = conv2d_fixed_padding( + inputs=inputs, filters=4 * filters, kernel_size=1, strides=1, + data_format=data_format) + inputs = batch_norm(inputs, training, data_format) + inputs += shortcut + inputs = tf.nn.relu(inputs) + + return inputs + + +def _bottleneck_block_v2(inputs, filters, training, projection_shortcut, + strides, data_format): + """A single block for ResNet v2, with a bottleneck. + + Similar to _building_block_v2(), except using the "bottleneck" blocks + described in: + Convolution then batch normalization then ReLU as described by: + Deep Residual Learning for Image Recognition + https://arxiv.org/pdf/1512.03385.pdf + by Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun, Dec 2015. + + Adapted to the ordering conventions of: + Batch normalization then ReLu then convolution as described by: + Identity Mappings in Deep Residual Networks + https://arxiv.org/pdf/1603.05027.pdf + by Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun, Jul 2016. + + Args: + inputs: A tensor of size [batch, channels, height_in, width_in] or + [batch, height_in, width_in, channels] depending on data_format. + filters: The number of filters for the convolutions. + training: A Boolean for whether the model is in training or inference + mode. Needed for batch normalization. + projection_shortcut: The function to use for projection shortcuts + (typically a 1x1 convolution when downsampling the input). + strides: The block's stride. If greater than 1, this block will ultimately + downsample the input. + data_format: The input format ('channels_last' or 'channels_first'). + + Returns: + The output tensor of the block; shape should match inputs. + """ + shortcut = inputs + inputs = batch_norm(inputs, training, data_format) + inputs = tf.nn.relu(inputs) + + # The projection shortcut should come after the first batch norm and ReLU + # since it performs a 1x1 convolution. + if projection_shortcut is not None: + shortcut = projection_shortcut(inputs) + + inputs = conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=1, strides=1, + data_format=data_format) + + inputs = batch_norm(inputs, training, data_format) + inputs = tf.nn.relu(inputs) + inputs = conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=3, strides=strides, + data_format=data_format) + + inputs = batch_norm(inputs, training, data_format) + inputs = tf.nn.relu(inputs) + inputs = conv2d_fixed_padding( + inputs=inputs, filters=4 * filters, kernel_size=1, strides=1, + data_format=data_format) + + return inputs + shortcut + + +def block_layer(inputs, filters, bottleneck, block_fn, blocks, strides, + training, name, data_format): + """Creates one layer of blocks for the ResNet model. + + Args: + inputs: A tensor of size [batch, channels, height_in, width_in] or + [batch, height_in, width_in, channels] depending on data_format. + filters: The number of filters for the first convolution of the layer. + bottleneck: Is the block created a bottleneck block. + block_fn: The block to use within the model, either `building_block` or + `bottleneck_block`. + blocks: The number of blocks contained in the layer. + strides: The stride to use for the first convolution of the layer. If + greater than 1, this layer will ultimately downsample the input. + training: Either True or False, whether we are currently training the + model. Needed for batch norm. + name: A string name for the tensor output of the block layer. + data_format: The input format ('channels_last' or 'channels_first'). + + Returns: + The output tensor of the block layer. + """ + + # Bottleneck blocks end with 4x the number of filters as they start with + filters_out = filters * 4 if bottleneck else filters + + def projection_shortcut(inputs): + return conv2d_fixed_padding( + inputs=inputs, filters=filters_out, kernel_size=1, strides=strides, + data_format=data_format) + + # Only the first block per block_layer uses projection_shortcut and strides + inputs = block_fn(inputs, filters, training, projection_shortcut, strides, + data_format) + + for _ in range(1, blocks): + inputs = block_fn(inputs, filters, training, None, 1, data_format) + + return tf.identity(inputs, name) + + +class Model(object): + """Base class for building the Resnet Model.""" + + def __init__(self, resnet_size, bottleneck, num_classes, num_filters, + kernel_size, + conv_stride, first_pool_size, first_pool_stride, + block_sizes, block_strides, + resnet_version=DEFAULT_VERSION, data_format=None, + dtype=DEFAULT_DTYPE): + """Creates a model for classifying an image. + + Args: + resnet_size: A single integer for the size of the ResNet model. + bottleneck: Use regular blocks or bottleneck blocks. + num_classes: The number of classes used as labels. + num_filters: The number of filters to use for the first block layer + of the model. This number is then doubled for each subsequent block + layer. + kernel_size: The kernel size to use for convolution. + conv_stride: stride size for the initial convolutional layer + first_pool_size: Pool size to be used for the first pooling layer. + If none, the first pooling layer is skipped. + first_pool_stride: stride size for the first pooling layer. Not used + if first_pool_size is None. + block_sizes: A list containing n values, where n is the number of sets of + block layers desired. Each value should be the number of blocks in the + i-th set. + block_strides: List of integers representing the desired stride size for + each of the sets of block layers. Should be same length as block_sizes. + resnet_version: Integer representing which version of the ResNet network + to use. See README for details. Valid values: [1, 2] + data_format: Input format ('channels_last', 'channels_first', or None). + If set to None, the format is dependent on whether a GPU is available. + dtype: The TensorFlow dtype to use for calculations. If not specified + tf.float32 is used. + + Raises: + ValueError: if invalid version is selected. + """ + self.resnet_size = resnet_size + + if not data_format: + data_format = ( + 'channels_first' if tf.test.is_built_with_cuda() else 'channels_last') + + self.resnet_version = resnet_version + if resnet_version not in (1, 2): + raise ValueError( + 'Resnet version should be 1 or 2. See README for citations.') + + self.bottleneck = bottleneck + if bottleneck: + if resnet_version == 1: + self.block_fn = _bottleneck_block_v1 + else: + self.block_fn = _bottleneck_block_v2 + else: + if resnet_version == 1: + self.block_fn = _building_block_v1 + else: + self.block_fn = _building_block_v2 + + if dtype not in ALLOWED_TYPES: + raise ValueError('dtype must be one of: {}'.format(ALLOWED_TYPES)) + + self.data_format = data_format + self.num_classes = num_classes + self.num_filters = num_filters + self.kernel_size = kernel_size + self.conv_stride = conv_stride + self.first_pool_size = first_pool_size + self.first_pool_stride = first_pool_stride + self.block_sizes = block_sizes + self.block_strides = block_strides + self.dtype = dtype + self.pre_activation = resnet_version == 2 + + def _custom_dtype_getter(self, getter, name, shape=None, dtype=DEFAULT_DTYPE, + *args, **kwargs): + """Creates variables in fp32, then casts to fp16 if necessary. + + This function is a custom getter. A custom getter is a function with the + same signature as tf.get_variable, except it has an additional getter + parameter. Custom getters can be passed as the `custom_getter` parameter of + tf.variable_scope. Then, tf.get_variable will call the custom getter, + instead of directly getting a variable itself. This can be used to change + the types of variables that are retrieved with tf.get_variable. + The `getter` parameter is the underlying variable getter, that would have + been called if no custom getter was used. Custom getters typically get a + variable with `getter`, then modify it in some way. + + This custom getter will create an fp32 variable. If a low precision + (e.g. float16) variable was requested it will then cast the variable to the + requested dtype. The reason we do not directly create variables in low + precision dtypes is that applying small gradients to such variables may + cause the variable not to change. + + Args: + getter: The underlying variable getter, that has the same signature as + tf.get_variable and returns a variable. + name: The name of the variable to get. + shape: The shape of the variable to get. + dtype: The dtype of the variable to get. Note that if this is a low + precision dtype, the variable will be created as a tf.float32 variable, + then cast to the appropriate dtype + *args: Additional arguments to pass unmodified to getter. + **kwargs: Additional keyword arguments to pass unmodified to getter. + + Returns: + A variable which is cast to fp16 if necessary. + """ + + if dtype in CASTABLE_TYPES: + var = getter(name, shape, tf.float32, *args, **kwargs) + return tf.cast(var, dtype=dtype, name=name + '_cast') + else: + return getter(name, shape, dtype, *args, **kwargs) + + def _model_variable_scope(self): + """Returns a variable scope that the model should be created under. + + If self.dtype is a castable type, model variable will be created in fp32 + then cast to self.dtype before being used. + + Returns: + A variable scope for the model. + """ + + return tf.compat.v1.variable_scope('resnet_model', + custom_getter=self._custom_dtype_getter) + + def __call__(self, inputs, training): + """Add operations to classify a batch of input images. + + Args: + inputs: A Tensor representing a batch of input images. + training: A boolean. Set to True to add operations required only when + training the classifier. + + Returns: + A logits Tensor with shape [, self.num_classes]. + """ + + with self._model_variable_scope(): + if self.data_format == 'channels_first': + # Convert the inputs from channels_last (NHWC) to channels_first (NCHW). + # This provides a large performance boost on GPU. See + # https://www.tensorflow.org/performance/performance_guide#data_formats + inputs = tf.transpose(a=inputs, perm=[0, 3, 1, 2]) + + inputs = conv2d_fixed_padding( + inputs=inputs, filters=self.num_filters, kernel_size=self.kernel_size, + strides=self.conv_stride, data_format=self.data_format) + inputs = tf.identity(inputs, 'initial_conv') + + # We do not include batch normalization or activation functions in V2 + # for the initial conv1 because the first ResNet unit will perform these + # for both the shortcut and non-shortcut paths as part of the first + # block's projection. Cf. Appendix of [2]. + if self.resnet_version == 1: + inputs = batch_norm(inputs, training, self.data_format) + inputs = tf.nn.relu(inputs) + + if self.first_pool_size: + ############## npu modify begin ############# + #max_pooling2d is replaced by max_pool_with_argmax for better performance + inputs,argmax = tf.compat.v1.nn.max_pool_with_argmax( + input=inputs, ksize=(1,self.first_pool_size,self.first_pool_size,1), + strides=(1,self.first_pool_stride,self.first_pool_stride,1), padding='SAME', + data_format='NCHW' if self.data_format == 'channels_first' else 'NHWC') + ############## npu modify end ############### + + inputs = tf.identity(inputs, 'initial_max_pool') + + for i, num_blocks in enumerate(self.block_sizes): + num_filters = self.num_filters * (2**i) + inputs = block_layer( + inputs=inputs, filters=num_filters, bottleneck=self.bottleneck, + block_fn=self.block_fn, blocks=num_blocks, + strides=self.block_strides[i], training=training, + name='block_layer{}'.format(i + 1), data_format=self.data_format) + + # Only apply the BN and ReLU for model that does pre_activation in each + # building/bottleneck block, eg resnet V2. + if self.pre_activation: + inputs = batch_norm(inputs, training, self.data_format) + inputs = tf.nn.relu(inputs) + + # The current top layer has shape + # `batch_size x pool_size x pool_size x final_size`. + # ResNet does an Average Pooling layer over pool_size, + # but that is the same as doing a reduce_mean. We do a reduce_mean + # here because it performs better than AveragePooling2D. + axes = [2, 3] if self.data_format == 'channels_first' else [1, 2] + inputs = tf.reduce_mean(input_tensor=inputs, axis=axes, keepdims=True) + inputs = tf.identity(inputs, 'final_reduce_mean') + + inputs = tf.squeeze(inputs, axes) + inputs = tf.compat.v1.layers.dense(inputs=inputs, units=self.num_classes) + inputs = tf.identity(inputs, 'final_dense') + return inputs diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/resnet_run_loop.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/resnet_run_loop.py new file mode 100644 index 0000000..80d0b98 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/resnet_run_loop.py @@ -0,0 +1,989 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains utility and supporting functions for ResNet. + + This module contains ResNet code which does not directly build layers. This +includes dataset management, hyperparameter and optimizer code, and argument +parsing. Code for defining the ResNet layers can be found in resnet_model.py. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools +import math +import multiprocessing +import os +import sys +import time +import datetime +from absl import flags +from absl import logging +import tensorflow as tf + +############## npu modify begin ############# +from npu_bridge.estimator.npu.npu_config import NPURunConfig +from npu_bridge.estimator.npu.npu_estimator import NPUEstimator +from npu_bridge.estimator.npu.npu_optimizer import NPUDistributedOptimizer +from npu_bridge.estimator import npu_ops +from npu_bridge.hccl import hccl_ops +from hccl.manage.api import get_rank_size +from hccl.manage.api import get_rank_id +from tensorflow.core.protobuf import rewriter_config_pb2 + +import numpy as np +import random +SEED=123 +os.environ['PYTHONHASHSEED'] = str(SEED) +random.seed(SEED) +np.random.seed(SEED) +tf.compat.v1.set_random_seed(SEED) +tf.compat.v1.logging.info('setting random_seed: %d', SEED) +from benchmark_log import hwlog +############## npu modify end ############### + +from official.r1.resnet import imagenet_preprocessing +from official.r1.resnet import resnet_model +from official.r1.utils import export +from official.utils.flags import core as flags_core +from official.utils.logs import hooks_helper +from official.utils.logs import logger +from official.utils.misc import distribution_utils +from official.utils.misc import model_helpers + +################################################################################ +# Functions for input processing. +################################################################################ +def process_record_dataset(dataset, + is_training, + batch_size, + shuffle_buffer, + parse_record_fn, + num_epochs=1, + dtype=tf.float32, + datasets_num_private_threads=None, + drop_remainder=False, + tf_data_experimental_slack=False, + test_mode=False, + ): + """Given a Dataset with raw records, return an iterator over the records. + + Args: + dataset: A Dataset representing raw records + is_training: A boolean denoting whether the input is for training. + batch_size: The number of samples per batch. + shuffle_buffer: The buffer size to use when shuffling records. A larger + value results in better randomness, but smaller values reduce startup + time and use less memory. + parse_record_fn: A function that takes a raw record and returns the + corresponding (image, label) pair. + num_epochs: The number of epochs to repeat the dataset. + dtype: Data type to use for images/features. + datasets_num_private_threads: Number of threads for a private + threadpool created for all datasets computation. + drop_remainder: A boolean indicates whether to drop the remainder of the + batches. If True, the batch dimension will be static. + tf_data_experimental_slack: Whether to enable tf.data's + `experimental_slack` option. + + Returns: + Dataset of (image, label) pairs ready for iteration. + """ + # Defines a specific size thread pool for tf.data operations. + if datasets_num_private_threads: + options = tf.data.Options() + options.experimental_threading.private_threadpool_size = ( + datasets_num_private_threads) + dataset = dataset.with_options(options) + tf.compat.v1.logging.info('datasets_num_private_threads: %s', + datasets_num_private_threads) + + # Disable intra-op parallelism to optimize for throughput instead of latency. + options = tf.data.Options() + options.experimental_threading.max_intra_op_parallelism = 1 + dataset = dataset.with_options(options) + + if test_mode == True: + is_training = False + + # Prefetches a batch at a time to smooth out the time taken to load input + # files for shuffling and processing. + dataset = dataset.prefetch(buffer_size=batch_size) + if is_training: + # Shuffles records before repeating to respect epoch boundaries. + dataset = dataset.shuffle(buffer_size=shuffle_buffer) + + # Repeats the dataset for the number of epochs to train. + dataset = dataset.repeat() + + # Parses the raw records into images and labels. + dataset = dataset.map( + lambda value: parse_record_fn(value, is_training, dtype), + num_parallel_calls=tf.data.experimental.AUTOTUNE) + dataset = dataset.batch(batch_size, drop_remainder=drop_remainder) + + # Operations between the final prefetch and the get_next call to the iterator + # will happen synchronously during run time. We prefetch here again to + # background all of the above processing work and keep it out of the + # critical training path. Setting buffer_size to tf.data.experimental.AUTOTUNE + # allows DistributionStrategies to adjust how many batches to fetch based + # on how many devices are present. + dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE) + + if tf_data_experimental_slack: + options = tf.data.Options() + options.experimental_slack = True + dataset = dataset.with_options(options) + + return dataset + + +def get_synth_input_fn(height, width, num_channels, num_classes, + dtype=tf.float32): + """Returns an input function that returns a dataset with random data. + + This input_fn returns a data set that iterates over a set of random data and + bypasses all preprocessing, e.g. jpeg decode and copy. The host to device + copy is still included. This used to find the upper throughput bound when + tunning the full input pipeline. + + Args: + height: Integer height that will be used to create a fake image tensor. + width: Integer width that will be used to create a fake image tensor. + num_channels: Integer depth that will be used to create a fake image tensor. + num_classes: Number of classes that should be represented in the fake labels + tensor + dtype: Data type for features/images. + + Returns: + An input_fn that can be used in place of a real one to return a dataset + that can be used for iteration. + """ + # pylint: disable=unused-argument + def input_fn(is_training, data_dir, batch_size, *args, **kwargs): + """Returns dataset filled with random data.""" + # Synthetic input should be within [0, 255]. + inputs = tf.random.truncated_normal( + [batch_size] + [height, width, num_channels], + dtype=dtype, + mean=127, + stddev=60, + seed=123, + name='synthetic_inputs') + + labels = tf.random.uniform( + [batch_size], + minval=0, + maxval=num_classes - 1, + dtype=tf.int32, + seed=123, + name='synthetic_labels') + + data = tf.data.Dataset.from_tensors((inputs, labels)).repeat() + data = data.prefetch(buffer_size=tf.data.experimental.AUTOTUNE) + return data + + return input_fn + + +def image_bytes_serving_input_fn(image_shape, dtype=tf.float32): + """Serving input fn for raw jpeg images.""" + + def _preprocess_image(image_bytes): + """Preprocess a single raw image.""" + # Bounding box around the whole image. + bbox = tf.constant([0.0, 0.0, 1.0, 1.0], dtype=dtype, shape=[1, 1, 4]) + height, width, num_channels = image_shape + image = imagenet_preprocessing.preprocess_image( + image_bytes, bbox, height, width, num_channels, is_training=False) + return image + + image_bytes_list = tf.compat.v1.placeholder( + shape=[None], dtype=tf.string, name='input_tensor') + images = tf.map_fn( + _preprocess_image, image_bytes_list, back_prop=False, dtype=dtype) + return tf.estimator.export.TensorServingInputReceiver( + images, {'image_bytes': image_bytes_list}) + + +def override_flags_and_set_envars_for_gpu_thread_pool(flags_obj): + """Override flags and set env_vars for performance. + + These settings exist to test the difference between using stock settings + and manual tuning. It also shows some of the ENV_VARS that can be tweaked to + squeeze a few extra examples per second. These settings are defaulted to the + current platform of interest, which changes over time. + + On systems with small numbers of cpu cores, e.g. under 8 logical cores, + setting up a gpu thread pool with `tf_gpu_thread_mode=gpu_private` may perform + poorly. + + Args: + flags_obj: Current flags, which will be adjusted possibly overriding + what has been set by the user on the command-line. + """ + cpu_count = multiprocessing.cpu_count() + tf.compat.v1.logging.info('Logical CPU cores: %s', cpu_count) + + # Sets up thread pool for each GPU for op scheduling. + per_gpu_thread_count = 1 + total_gpu_thread_count = per_gpu_thread_count * flags_obj.num_gpus + os.environ['TF_GPU_THREAD_MODE'] = flags_obj.tf_gpu_thread_mode + os.environ['TF_GPU_THREAD_COUNT'] = str(per_gpu_thread_count) + tf.compat.v1.logging.info('TF_GPU_THREAD_COUNT: %s', os.environ['TF_GPU_THREAD_COUNT']) + tf.compat.v1.logging.info('TF_GPU_THREAD_MODE: %s', os.environ['TF_GPU_THREAD_MODE']) + + # Reduces general thread pool by number of threads used for GPU pool. + main_thread_count = cpu_count - total_gpu_thread_count + flags_obj.inter_op_parallelism_threads = main_thread_count + + # Sets thread count for tf.data. Logical cores minus threads assign to the + # private GPU pool along with 2 thread per GPU for event monitoring and + # sending / receiving tensors. + num_monitoring_threads = 2 * flags_obj.num_gpus + flags_obj.datasets_num_private_threads = (cpu_count - total_gpu_thread_count + - num_monitoring_threads) + + +################################################################################ +# Functions for running training/eval/validation loops for the model. +################################################################################ +def learning_rate_with_decay( + batch_size, batch_denom, num_images, boundary_epochs, decay_rates, + base_lr=0.1, warmup=False, cosine_lr=False, train_epochs=90): + """Get a learning rate that decays step-wise as training progresses. + + Args: + batch_size: the number of examples processed in each training batch. + batch_denom: this value will be used to scale the base learning rate. + `0.1 * batch size` is divided by this number, such that when + batch_denom == batch_size, the initial learning rate will be 0.1. + num_images: total number of images that will be used for training. + boundary_epochs: list of ints representing the epochs at which we + decay the learning rate. + decay_rates: list of floats representing the decay rates to be used + for scaling the learning rate. It should have one more element + than `boundary_epochs`, and all elements should have the same type. + base_lr: Initial learning rate scaled based on batch_denom. + warmup: Run a 5 epoch warmup to the initial lr. + Returns: + Returns a function that takes a single argument - the number of batches + trained so far (global_step)- and returns the learning rate to be used + for training the next batch. + """ + + initial_learning_rate = base_lr * batch_size / batch_denom + batches_per_epoch = num_images / batch_size + + # Reduce the learning rate at certain epochs. + # CIFAR-10: divide by 10 at epoch 100, 150, and 200 + # ImageNet: divide by 10 at epoch 30, 60, 80, and 90 + boundaries = [int(batches_per_epoch * epoch) for epoch in boundary_epochs] + vals = [initial_learning_rate * decay for decay in decay_rates] + + tf.compat.v1.logging.info('base_lr: %.4f', float(base_lr)) + tf.compat.v1.logging.info('initial_learning_rate: %.4f', float(initial_learning_rate)) + tf.compat.v1.logging.info('batches_per_epoch: %d', int(batches_per_epoch)) + tf.compat.v1.logging.info('lr boundaries: %s', ','.join(str(i) for i in boundaries)) + tf.compat.v1.logging.info('lr vals: %s', ','.join(str(i) for i in vals)) + + tf.compat.v1.logging.info('warmup: %d', int(warmup)) + tf.compat.v1.logging.info('cosine_lr: %d', int(cosine_lr)) + tf.compat.v1.logging.info('train_epochs: %d', int(train_epochs)) + + def learning_rate_fn(global_step): + """Builds scaled learning rate function with 5 epoch warm up.""" + + ############## npu modify begin ############# + #Using int32 for better computing performance + global_step=tf.cast(global_step,tf.int32) + ############## npu modify end ############### + + ############## npu modify begin ############# + if cosine_lr: + total_steps = int(train_epochs * batches_per_epoch) + warmup_steps = int(batches_per_epoch * 5 * int(warmup)) + + tf.compat.v1.logging.info('total_steps: %d', int(total_steps)) + tf.compat.v1.logging.info('warmup_steps: %d', int(warmup_steps)) + + lr = tf.maximum( + tf.compat.v1.train.cosine_decay( + learning_rate=initial_learning_rate, + global_step=global_step - warmup_steps, + decay_steps=total_steps - warmup_steps, + ), + 0, + ) + ############## npu modify end ############### + else: + lr = tf.compat.v1.train.piecewise_constant(global_step, boundaries, vals) + + if warmup: + warmup_steps = int(batches_per_epoch * 5) + warmup_lr = ( + initial_learning_rate * tf.cast(global_step, tf.float32) / tf.cast( + warmup_steps, tf.float32)) + return tf.cond(pred=global_step < warmup_steps, + true_fn=lambda: warmup_lr, + false_fn=lambda: lr) + return lr + + def poly_rate_fn(global_step): + """Handles linear scaling rule, gradual warmup, and LR decay. + + The learning rate starts at 0, then it increases linearly per step. After + FLAGS.poly_warmup_epochs, we reach the base learning rate (scaled to account + for batch size). The learning rate is then decayed using a polynomial rate + decay schedule with power 2.0. + + Args: + global_step: the current global_step + + Returns: + returns the current learning rate + """ + + # Learning rate schedule for LARS polynomial schedule + if flags.FLAGS.batch_size < 8192: + plr = 5.0 + w_epochs = 5 + elif flags.FLAGS.batch_size < 16384: + plr = 10.0 + w_epochs = 5 + elif flags.FLAGS.batch_size < 32768: + plr = 25.0 + w_epochs = 5 + else: + plr = 32.0 + w_epochs = 14 + + w_steps = int(w_epochs * batches_per_epoch) + wrate = (plr * tf.cast(global_step, tf.float32) / tf.cast( + w_steps, tf.float32)) + + # TODO(pkanwar): use a flag to help calc num_epochs. + num_epochs = 90 + train_steps = batches_per_epoch * num_epochs + + min_step = tf.constant(1, dtype=tf.int64) + decay_steps = tf.maximum(min_step, tf.subtract(global_step, w_steps)) + poly_rate = tf.train.polynomial_decay( + plr, + decay_steps, + train_steps - w_steps + 1, + power=2.0) + return tf.where(global_step <= w_steps, wrate, poly_rate) + + # For LARS we have a new learning rate schedule + if flags.FLAGS.enable_lars: + return poly_rate_fn + + return learning_rate_fn + + +def per_replica_batch_size(batch_size, num_gpus): + """For multi-gpu, batch-size must be a multiple of the number of GPUs. + + + Note that distribution strategy handles this automatically when used with + Keras. For using with Estimator, we need to get per GPU batch. + + Args: + batch_size: Global batch size to be divided among devices. This should be + equal to num_gpus times the single-GPU batch_size for multi-gpu training. + num_gpus: How many GPUs are used with DistributionStrategies. + + Returns: + Batch size per device. + + Raises: + ValueError: if batch_size is not divisible by number of devices + """ + if num_gpus <= 1: + return batch_size + + remainder = batch_size % num_gpus + if remainder: + err = ('When running with multiple GPUs, batch size ' + 'must be a multiple of the number of available GPUs. Found {} ' + 'GPUs with a batch size of {}; try --batch_size={} instead.' + ).format(num_gpus, batch_size, batch_size - remainder) + raise ValueError(err) + return int(batch_size / num_gpus) + + +def resnet_model_fn(features, labels, mode, model_class, + resnet_size, weight_decay, learning_rate_fn, momentum, + data_format, resnet_version, loss_scale, + loss_filter_fn=None, dtype=resnet_model.DEFAULT_DTYPE, + fine_tune=False, label_smoothing=0.0): + """Shared functionality for different resnet model_fns. + + Initializes the ResnetModel representing the model layers + and uses that model to build the necessary EstimatorSpecs for + the `mode` in question. For training, this means building losses, + the optimizer, and the train op that get passed into the EstimatorSpec. + For evaluation and prediction, the EstimatorSpec is returned without + a train op, but with the necessary parameters for the given mode. + + Args: + features: tensor representing input images + labels: tensor representing class labels for all input images + mode: current estimator mode; should be one of + `tf.estimator.ModeKeys.TRAIN`, `EVALUATE`, `PREDICT` + model_class: a class representing a TensorFlow model that has a __call__ + function. We assume here that this is a subclass of ResnetModel. + resnet_size: A single integer for the size of the ResNet model. + weight_decay: weight decay loss rate used to regularize learned variables. + learning_rate_fn: function that returns the current learning rate given + the current global_step + momentum: momentum term used for optimization + data_format: Input format ('channels_last', 'channels_first', or None). + If set to None, the format is dependent on whether a GPU is available. + resnet_version: Integer representing which version of the ResNet network to + use. See README for details. Valid values: [1, 2] + loss_scale: The factor to scale the loss for numerical stability. A detailed + summary is present in the arg parser help text. + loss_filter_fn: function that takes a string variable name and returns + True if the var should be included in loss calculation, and False + otherwise. If None, batch_normalization variables will be excluded + from the loss. + dtype: the TensorFlow dtype to use for calculations. + fine_tune: If True only train the dense layers(final layers). + label_smoothing: If greater than 0 then smooth the labels. + + Returns: + EstimatorSpec parameterized according to the input params and the + current mode. + """ + + # Generate a summary node for the images + tf.compat.v1.summary.image('images', features, max_outputs=6) + + ############## npu modify begin ############# + # Checks that features/images have same data type being used for calculations. + if features.dtype != dtype: + features=tf.cast(features,dtype) + ############## npu modify end ############### + + model = model_class(resnet_size, data_format, resnet_version=resnet_version, + dtype=dtype) + + logits = model(features, mode == tf.estimator.ModeKeys.TRAIN) + + # This acts as a no-op if the logits are already in fp32 (provided logits are + # not a SparseTensor). If dtype is is low precision, logits must be cast to + # fp32 for numerical stability. + logits = tf.cast(logits, tf.float32) + + predictions = { + 'classes': tf.argmax(input=logits, axis=1), + 'probabilities': tf.nn.softmax(logits, name='softmax_tensor') + } + + if mode == tf.estimator.ModeKeys.PREDICT: + # Return the predictions and the specification for serving a SavedModel + return tf.estimator.EstimatorSpec( + mode=mode, + predictions=predictions, + export_outputs={ + 'predict': tf.estimator.export.PredictOutput(predictions) + }) + + # Calculate loss, which includes softmax cross entropy and L2 regularization. + if label_smoothing != 0.0: + one_hot_labels = tf.one_hot(labels, 1001) + cross_entropy = tf.losses.softmax_cross_entropy( + logits=logits, onehot_labels=one_hot_labels, + label_smoothing=label_smoothing) + else: + cross_entropy = tf.compat.v1.losses.sparse_softmax_cross_entropy( + logits=logits, labels=labels) + + # Create a tensor named cross_entropy for logging purposes. + tf.identity(cross_entropy, name='cross_entropy') + tf.compat.v1.summary.scalar('cross_entropy', cross_entropy) + + # If no loss_filter_fn is passed, assume we want the default behavior, + # which is that batch_normalization variables are excluded from loss. + def exclude_batch_norm(name): + return 'batch_normalization' not in name + loss_filter_fn = loss_filter_fn or exclude_batch_norm + + # Add weight decay to the loss. + l2_loss = weight_decay * tf.add_n( + # loss is computed using fp32 for numerical stability. + [ + tf.nn.l2_loss(tf.cast(v, tf.float32)) + for v in tf.compat.v1.trainable_variables() + if loss_filter_fn(v.name) + ]) + tf.compat.v1.summary.scalar('l2_loss', l2_loss) + loss = cross_entropy + l2_loss + + if mode == tf.estimator.ModeKeys.TRAIN: + global_step = tf.compat.v1.train.get_or_create_global_step() + + learning_rate = learning_rate_fn(global_step) + + # Create a tensor named learning_rate for logging purposes + tf.identity(learning_rate, name='learning_rate') + tf.compat.v1.summary.scalar('learning_rate', learning_rate) + + if flags.FLAGS.enable_lars: + from tensorflow.contrib import opt as contrib_opt # pylint: disable=g-import-not-at-top + optimizer = contrib_opt.LARSOptimizer( + learning_rate, + momentum=momentum, + weight_decay=weight_decay, + skip_list=['batch_normalization', 'bias']) + else: + optimizer = tf.compat.v1.train.MomentumOptimizer( + learning_rate=learning_rate, + momentum=momentum + ) + + ############## npu modify begin ############# + optimizer = NPUDistributedOptimizer(optimizer) + ############## npu modify end ############### + + fp16_implementation = getattr(flags.FLAGS, 'fp16_implementation', None) + if fp16_implementation == 'graph_rewrite': + optimizer = ( + tf.compat.v1.train.experimental.enable_mixed_precision_graph_rewrite( + optimizer, loss_scale=loss_scale)) + + def _dense_grad_filter(gvs): + """Only apply gradient updates to the final layer. + + This function is used for fine tuning. + + Args: + gvs: list of tuples with gradients and variable info + Returns: + filtered gradients so that only the dense layer remains + """ + return [(g, v) for g, v in gvs if 'dense' in v.name] + + if loss_scale != 1 and fp16_implementation != 'graph_rewrite': + # When computing fp16 gradients, often intermediate tensor values are + # so small, they underflow to 0. To avoid this, we multiply the loss by + # loss_scale to make these tensor values loss_scale times bigger. + scaled_grad_vars = optimizer.compute_gradients(loss * loss_scale) + + if fine_tune: + scaled_grad_vars = _dense_grad_filter(scaled_grad_vars) + + # Once the gradient computation is complete we can scale the gradients + # back to the correct scale before passing them to the optimizer. + unscaled_grad_vars = [(grad / loss_scale, var) + for grad, var in scaled_grad_vars] + minimize_op = optimizer.apply_gradients(unscaled_grad_vars, global_step) + else: + grad_vars = optimizer.compute_gradients(loss) + if fine_tune: + grad_vars = _dense_grad_filter(grad_vars) + minimize_op = optimizer.apply_gradients(grad_vars, global_step) + + update_ops = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.UPDATE_OPS) + train_op = tf.group(minimize_op, update_ops) + else: + train_op = None + + ############## npu modify begin ############# + #Using float32 for better performance + accuracy = tf.compat.v1.metrics.accuracy(tf.cast(labels, tf.float32), predictions['classes']) + ############## npu modify end ############### + + accuracy_top_5 = tf.compat.v1.metrics.mean( + tf.nn.in_top_k(predictions=logits, targets=labels, k=5, name='top_5_op')) + + ############## npu modify begin ############# + #Using for 8P + rank_size = int(os.getenv('RANK_SIZE')) + newaccuracy = (hccl_ops.allreduce(accuracy[0], "sum") / rank_size, accuracy[1]) + newaccuracy_top_5 = (hccl_ops.allreduce(accuracy_top_5[0], "sum") / rank_size, accuracy_top_5[1]) + ############## npu modify begin ############# + + metrics = {'accuracy': newaccuracy, + 'accuracy_top_5': newaccuracy_top_5} + + # Create a tensor named train_accuracy for logging purposes + tf.identity(accuracy[1], name='train_accuracy') + tf.identity(accuracy_top_5[1], name='train_accuracy_top_5') + tf.compat.v1.summary.scalar('train_accuracy', accuracy[1]) + tf.compat.v1.summary.scalar('train_accuracy_top_5', accuracy_top_5[1]) + + return tf.estimator.EstimatorSpec( + mode=mode, + predictions=predictions, + loss=loss, + train_op=train_op, + eval_metric_ops=metrics) + +############## npu modify begin ############# +def init_npu(): + """Initialize npu manually. + Returns: + `init_sess` npu init session config. + `npu_init` npu init ops. + """ + npu_init = npu_ops.initialize_system() + config = tf.ConfigProto() + + #npu mix precision attribute set to true when using mix precision + config.graph_options.rewrite_options.remapping = rewriter_config_pb2.RewriterConfig.OFF + custom_op = config.graph_options.rewrite_options.custom_optimizers.add() + custom_op.name = "NpuOptimizer" + custom_op.parameter_map["precision_mode"].s = tf.compat.as_bytes("allow_mix_precision") + custom_op.parameter_map["use_off_line"].b = True + + init_sess = tf.Session(config=config) + return init_sess,npu_init +############## npu modify end ############### + + +def resnet_main( + flags_obj, model_function, input_function, dataset_name, num_images, shape=None): + """Shared main loop for ResNet Models. + + Args: + flags_obj: An object containing parsed flags. See define_resnet_flags() + for details. + model_function: the function that instantiates the Model and builds the + ops for train/eval. This will be passed directly into the estimator. + input_function: the function that processes the dataset and returns a + dataset that the estimator can train on. This will be wrapped with + all the relevant flags for running and passed to estimator. + dataset_name: the name of the dataset for training and evaluation. This is + used for logging purpose. + shape: list of ints representing the shape of the images used for training. + This is only used if flags_obj.export_dir is passed. + + Returns: + Dict of results of the run. Contains the keys `eval_results` and + `train_hooks`. `eval_results` contains accuracy (top_1) and accuracy_top_5. + `train_hooks` is a list the instances of hooks used during training. + """ + + model_helpers.apply_clean(flags.FLAGS) + + rank_size = get_rank_size() + rank_id = get_rank_id() + + # Ensures flag override logic is only executed if explicitly triggered. + if flags_obj.tf_gpu_thread_mode: + override_flags_and_set_envars_for_gpu_thread_pool(flags_obj) + + # Configures cluster spec for distribution strategy. + num_workers = distribution_utils.configure_cluster(flags_obj.worker_hosts, + flags_obj.task_index) + + # Creates session config. allow_soft_placement = True, is required for + # multi-GPU and is not harmful for other modes. + session_config = tf.compat.v1.ConfigProto( + inter_op_parallelism_threads=flags_obj.inter_op_parallelism_threads, + intra_op_parallelism_threads=flags_obj.intra_op_parallelism_threads, + allow_soft_placement=True) + + session_config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1 + + ############## npu modify begin ############# + # Creates a `NPURunConfig` that checkpoints every 115G00 steps + iterations_per_loop=flags_obj.iterations_per_loop + run_config = NPURunConfig( + model_dir=flags_obj.model_dir, + tf_random_seed=flags_obj.random_seed if flags_obj.random_seed>=0 else 0, + session_config=session_config, + keep_checkpoint_max=5, + save_checkpoints_steps=5000, + enable_data_pre_proc=True, + iterations_per_loop=iterations_per_loop, + log_step_count_steps=iterations_per_loop, + precision_mode='allow_mix_precision', + hcom_parallel=True + ) + ############## npu modify end ############### + + ############## npu modify begin ############# + # Creates a `NPUEstimator` instead of using tf.estimator.Estimator + classifier = NPUEstimator( + model_fn=model_function, model_dir=flags_obj.model_dir, config=run_config, + params={ + 'resnet_size': int(flags_obj.resnet_size), + 'data_format': flags_obj.data_format, + 'batch_size': flags_obj.batch_size, + 'resnet_version': int(flags_obj.resnet_version), + 'loss_scale': flags_core.get_loss_scale(flags_obj, + default_for_fp16=512), + 'dtype': flags_core.get_tf_dtype(flags_obj), + 'fine_tune': flags_obj.fine_tune, + 'num_workers': num_workers, + 'num_gpus' : flags_core.get_num_gpus(flags_obj), + 'base_lr': flags_obj.base_lr, + 'cosine_lr': flags_obj.cosine_lr, + 'train_epochs': flags_obj.train_epochs, + }) + ############## npu modify end ############### + + run_params = { + 'batch_size': flags_obj.batch_size, + 'dtype': flags_core.get_tf_dtype(flags_obj), + 'resnet_size': flags_obj.resnet_size, + 'resnet_version': flags_obj.resnet_version, + 'synthetic_data': flags_obj.use_synthetic_data, + 'train_epochs': flags_obj.train_epochs, + 'num_workers': num_workers, + } + if flags_obj.use_synthetic_data: + dataset_name = dataset_name + '-synthetic' + + benchmark_logger = logger.get_benchmark_logger() + benchmark_logger.log_run_info('resnet', dataset_name, run_params, + test_id=flags_obj.benchmark_test_id) + + train_hooks = hooks_helper.get_train_hooks( + flags_obj.hooks, + model_dir=flags_obj.model_dir, + batch_size=flags_obj.batch_size) + + def input_fn_train(num_epochs, input_context=None, rank_size=None, rank_id=None): + ############## npu modify begin ############# + # Using dtype=tf.float16 for higher data transmission performance + # drop_remainder currently only support true + # batch_size means single card batch instead of global batch size + return input_function( + is_training=True, + data_dir=flags_obj.data_dir, + batch_size=flags_obj.batch_size, + num_epochs=num_epochs, + dtype=tf.float16, + input_context=input_context, + drop_remainder=True, + rank_size=rank_size, + rank_id=rank_id, + test_mode=flags_obj.test_mode, + ) + + def input_fn_eval(): + # batch_size means single card batch instead of global batch size + # Using dtype=tf.float16 for higher data transmission performance + # drop_remainder currently only support true + return input_function( + is_training=False, + data_dir=flags_obj.data_dir, + batch_size=flags_obj.batch_size, + num_epochs=1, + dtype=tf.float16, + drop_remainder=True) + ############## npu modify end ############### + + train_epochs = (0 if flags_obj.eval_only or not flags_obj.train_epochs else + flags_obj.train_epochs) + + use_train_and_evaluate = flags_obj.use_train_and_evaluate or num_workers > 1 + if use_train_and_evaluate: + train_spec = tf.estimator.TrainSpec( + input_fn=lambda input_context=None: input_fn_train( + train_epochs, input_context=input_context, rank_size=rank_size, rank_id=rank_id), + hooks=train_hooks, + max_steps=flags_obj.max_train_steps) + eval_spec = tf.estimator.EvalSpec(input_fn=input_fn_eval) + tf.compat.v1.logging.info('Starting to train and evaluate.') + tf.estimator.train_and_evaluate(classifier, train_spec, eval_spec) + # tf.estimator.train_and_evalute doesn't return anything in multi-worker + # case. + eval_results = {} + else: + if train_epochs == 0: + # If --eval_only is set, perform a single loop with zero train epochs. + schedule, n_loops = [0], 1 + else: + # Compute the number of times to loop while training. All but the last + # pass will train for `epochs_between_evals` epochs, while the last will + # train for the number needed to reach `training_epochs`. For instance if + # train_epochs = 25 and epochs_between_evals = 10 + # schedule will be set to [10, 10, 5]. That is to say, the loop will: + # Train for 10 epochs and then evaluate. + # Train for another 10 epochs and then evaluate. + # Train for a final 5 epochs (to reach 25 epochs) and then evaluate. + n_loops = math.ceil(train_epochs / flags_obj.epochs_between_evals) + schedule = [flags_obj.epochs_between_evals for _ in range(int(n_loops))] + schedule[-1] = train_epochs - sum(schedule[:-1]) # over counting. + + ############## npu modify begin ############# + tf.compat.v1.logging.info('schedule: %s', ','.join([str(i) for i in schedule])) + ############## npu modify end ############# + + for cycle_index, num_train_epochs in enumerate(schedule): + tf.compat.v1.logging.info('Starting cycle: %d/%d Num_train_epochs: %d', cycle_index, int(n_loops), int(num_train_epochs)) + hwlog.remark_print(key=hwlog.CURRENT_EPOCH, value=num_train_epochs) + if num_train_epochs: + # Since we are calling classifier.train immediately in each loop, the + # value of num_train_epochs in the lambda function will not be changed + # before it is used. So it is safe to ignore the pylint error here + # pylint: disable=cell-var-from-loop + ############## npu modify begin ############# + if flags_obj.max_train_steps is None: + classifier.train( + input_fn=lambda input_context=True: input_fn_train( + num_train_epochs, input_context=input_context, rank_size=rank_size, rank_id=rank_id), + hooks=train_hooks, + steps=(num_images['train']/flags_obj.batch_size/rank_size)*num_train_epochs // iterations_per_loop * iterations_per_loop ) # step control func.2 + else: + classifier.train( + input_fn=lambda input_context=True: input_fn_train( + num_train_epochs, input_context=input_context, rank_size=rank_size, rank_id=rank_id), + hooks=train_hooks, + max_steps=flags_obj.max_train_steps) + ############## npu modify end ############### + date_time_end = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') + if cycle_index == len(schedule) - 1 and flags_obj.test_mode == False: + # flags_obj.max_train_steps is generally associated with testing and + # profiling. As a result it is frequently called with synthetic data, + # which will iterate forever. Passing steps=flags_obj.max_train_steps + # allows the eval (which is generally unimportant in those circumstances) + # to terminate. Note that eval will run for max_train_steps each loop, + # regardless of the global_step count. + tf.compat.v1.logging.info('Starting to evaluate.') + eval_results = classifier.evaluate(input_fn=input_fn_eval, + steps=num_images['validation']//flags_obj.batch_size) # step control func.1 + + benchmark_logger.log_evaluation_result(eval_results) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value=float(eval_results.get("accuracy"))) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value=float(eval_results.get("accuracy_top_5"))) + if model_helpers.past_stop_threshold( + flags_obj.stop_threshold, eval_results['accuracy']): + break + + if flags_obj.export_dir is not None: + # Exports a saved model for the given classifier. + export_dtype = flags_core.get_tf_dtype(flags_obj) + if flags_obj.image_bytes_as_serving_input: + input_receiver_fn = functools.partial( + image_bytes_serving_input_fn, shape, dtype=export_dtype) + else: + input_receiver_fn = export.build_tensor_serving_input_receiver_fn( + shape, batch_size=flags_obj.batch_size, dtype=export_dtype) + classifier.export_savedmodel(flags_obj.export_dir, input_receiver_fn, + strip_default_attrs=True) + + stats = {} + stats['eval_results'] = eval_results if flags_obj.test_mode == False else [] + stats['train_hooks'] = train_hooks + + return stats + + +def define_resnet_flags(resnet_size_choices=None, dynamic_loss_scale=False, + fp16_implementation=False): + """Add flags and validators for ResNet.""" + flags_core.define_base(clean=True, train_epochs=True, + epochs_between_evals=True, stop_threshold=True, + num_gpu=True, hooks=True, export_dir=True, + distribution_strategy=True) + flags_core.define_performance(num_parallel_calls=False, + inter_op=True, + intra_op=True, + synthetic_data=True, + dtype=True, + all_reduce_alg=True, + num_packs=True, + tf_gpu_thread_mode=True, + datasets_num_private_threads=True, + dynamic_loss_scale=dynamic_loss_scale, + fp16_implementation=fp16_implementation, + loss_scale=True, + tf_data_experimental_slack=True, + max_train_steps=True) + flags_core.define_image() + flags_core.define_benchmark() + flags_core.define_distribution() + flags.adopt_module_key_flags(flags_core) + + flags.DEFINE_enum( + name='resnet_version', short_name='rv', default='1', + enum_values=['1', '2'], + help=flags_core.help_wrap( + 'Version of ResNet. (1 or 2) See README.md for details.')) + flags.DEFINE_bool( + name='fine_tune', short_name='ft', default=False, + help=flags_core.help_wrap( + 'If True do not train any parameters except for the final layer.')) + flags.DEFINE_boolean( + name='eval_only', default=False, + help=flags_core.help_wrap('Skip training and only perform evaluation on ' + 'the latest checkpoint.')) + flags.DEFINE_boolean( + name='image_bytes_as_serving_input', default=False, + help=flags_core.help_wrap( + 'If True exports savedmodel with serving signature that accepts ' + 'JPEG image bytes instead of a fixed size [HxWxC] tensor that ' + 'represents the image. The former is easier to use for serving at ' + 'the expense of image resize/cropping being done as part of model ' + 'inference. Note, this flag only applies to ImageNet and cannot ' + 'be used for CIFAR.')) + flags.DEFINE_boolean( + name='use_train_and_evaluate', default=False, + help=flags_core.help_wrap( + 'If True, uses `tf.estimator.train_and_evaluate` for the training ' + 'and evaluation loop, instead of separate calls to `classifier.train ' + 'and `classifier.evaluate`, which is the default behavior.')) + flags.DEFINE_bool( + name='enable_lars', default=False, + help=flags_core.help_wrap( + 'Enable LARS optimizer for large batch training.')) + flags.DEFINE_float( + name='label_smoothing', default=0.0, + help=flags_core.help_wrap( + 'Label smoothing parameter used in the softmax_cross_entropy')) + flags.DEFINE_float( + name='weight_decay', default=1e-4, + help=flags_core.help_wrap( + 'Weight decay coefficiant for l2 regularization.')) + + ##############NPU modify####################### + flags.DEFINE_float( + name='base_lr', default=0.0, + help=flags_core.help_wrap( + 'base learning rate in training.')) + flags.DEFINE_bool( + name='cosine_lr', default=False, + help=flags_core.help_wrap( + 'Enable Cosine Learnin_rate for training.')) + flags.DEFINE_bool( + name='test_mode', default=False, + help=flags_core.help_wrap( + 'turn to test mode.')) + flags.DEFINE_integer( + name='iterations_per_loop', default=10, + help=flags_core.help_wrap( + 'setting iterations_per_loop, this config will affect performance.')) + flags.DEFINE_integer( + name='random_seed', default=-1, + help=flags_core.help_wrap( + 'setting global random_seed to make result stable.')) + ##############NPU modify end####################### + + choice_kwargs = dict( + name='resnet_size', short_name='rs', default='50', + help=flags_core.help_wrap('The size of the ResNet model to use.')) + + if resnet_size_choices is None: + flags.DEFINE_string(**choice_kwargs) + else: + flags.DEFINE_enum(enum_values=resnet_size_choices, **choice_kwargs) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/run_imagenet.sh b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/run_imagenet.sh new file mode 100644 index 0000000..0d913a1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/run_imagenet.sh @@ -0,0 +1,27 @@ +export RANK_ID=$1 +export RANK_SIZE=$2 + +export DEVICE_ID=$RANK_ID +export DEVICE_INDEX=$RANK_ID +export RANK_TABLE_FILE=rank_table.json + +export JOB_ID=10087 +export FUSION_TENSOR_SIZE=1000000000 +#sleep 5 +python3 $3/imagenet_main.py \ +--resnet_size=101 \ +--batch_size=128 \ +--num_gpus=1 \ +--cosine_lr=True \ +--dtype=fp16 \ +--label_smoothing=0.1 \ +--loss_scale=512 \ +--train_epochs=3 \ +--epochs_between_evals=1 \ +--hooks=ExamplesPerSecondHook,loggingtensorhook,loggingmetrichook \ +--data_dir=/home/imagenet_TF \ +--model_dir=./model_dir + +mkdir graph +mv *.txt graph +mv *.pbtxt graph diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/train_accuracy_rewrite.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/train_accuracy_rewrite.py new file mode 100644 index 0000000..40ca4a4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/resnet/train_accuracy_rewrite.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2020/4/24 15:10 +# @Author : w00558981 +# @Site : +# @File : train_accuracy_rewrite.py + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.eager import context +from tensorflow.python.framework import dtypes,ops +from tensorflow.python.ops import array_ops,math_ops,state_ops,variable_scope,weights_broadcast_ops +from tensorflow.python.ops.metrics_impl import _remove_squeezable_dimensions,metric_variable,_aggregate_across_replicas + + +def accuracy(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): + """Calculates how often `predictions` matches `labels`. + + The `accuracy` function creates two local variables, `total` and + `count` that are used to compute the frequency with which `predictions` + matches `labels`. This frequency is ultimately returned as `accuracy`: an + idempotent operation that simply divides `total` by `count`. + + For estimation of the metric over a stream of data, the function creates an + `update_op` operation that updates these variables and returns the `accuracy`. + Internally, an `is_correct` operation computes a `Tensor` with elements 1.0 + where the corresponding elements of `predictions` and `labels` match and 0.0 + otherwise. Then `update_op` increments `total` with the reduced sum of the + product of `weights` and `is_correct`, and it increments `count` with the + reduced sum of `weights`. + + If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. + + Args: + labels: The ground truth values, a `Tensor` whose shape matches + `predictions`. + predictions: The predicted values, a `Tensor` of any shape. + weights: Optional `Tensor` whose rank is either 0, or the same rank as + `labels`, and must be broadcastable to `labels` (i.e., all dimensions must + be either `1`, or the same as the corresponding `labels` dimension). + metrics_collections: An optional list of collections that `accuracy` should + be added to. + updates_collections: An optional list of collections that `update_op` should + be added to. + name: An optional variable_scope name. + + Returns: + accuracy: A `Tensor` representing the accuracy, the value of `total` divided + by `count`. + update_op: An operation that increments the `total` and `count` variables + appropriately and whose value matches `accuracy`. + + Raises: + ValueError: If `predictions` and `labels` have mismatched shapes, or if + `weights` is not `None` and its shape doesn't match `predictions`, or if + either `metrics_collections` or `updates_collections` are not a list or + tuple. + RuntimeError: If eager execution is enabled. + """ + if context.executing_eagerly(): + raise RuntimeError('tf.metrics.accuracy is not supported when eager ' + 'execution is enabled.') + + predictions, labels, weights = _remove_squeezable_dimensions( + predictions=predictions, labels=labels, weights=weights) + predictions.get_shape().assert_is_compatible_with(labels.get_shape()) + if labels.dtype != predictions.dtype: + predictions = math_ops.cast(predictions, labels.dtype) + is_correct = math_ops.cast( + math_ops.equal(predictions, labels), dtypes.float32) + return mean(is_correct, weights, metrics_collections, updates_collections, + name or 'accuracy') + + + +def mean(values, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): + """Computes the (weighted) mean of the given values. + + The `mean` function creates two local variables, `total` and `count` + that are used to compute the average of `values`. This average is ultimately + returned as `mean` which is an idempotent operation that simply divides + `total` by `count`. + + For estimation of the metric over a stream of data, the function creates an + `update_op` operation that updates these variables and returns the `mean`. + `update_op` increments `total` with the reduced sum of the product of `values` + and `weights`, and it increments `count` with the reduced sum of `weights`. + + If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. + + Args: + values: A `Tensor` of arbitrary dimensions. + weights: Optional `Tensor` whose rank is either 0, or the same rank as + `values`, and must be broadcastable to `values` (i.e., all dimensions must + be either `1`, or the same as the corresponding `values` dimension). + metrics_collections: An optional list of collections that `mean` + should be added to. + updates_collections: An optional list of collections that `update_op` + should be added to. + name: An optional variable_scope name. + + Returns: + mean: A `Tensor` representing the current mean, the value of `total` divided + by `count`. + update_op: An operation that increments the `total` and `count` variables + appropriately and whose value matches `mean_value`. + + Raises: + ValueError: If `weights` is not `None` and its shape doesn't match `values`, + or if either `metrics_collections` or `updates_collections` are not a list + or tuple. + RuntimeError: If eager execution is enabled. + """ + if context.executing_eagerly(): + raise RuntimeError('tf.metrics.mean is not supported when eager execution ' + 'is enabled.') + + with variable_scope.variable_scope(name, 'mean', (values, weights)): + values = math_ops.cast(values, dtypes.float32) + + total = metric_variable([], dtypes.float32, name='total') + count = metric_variable([], dtypes.float32, name='count') + + if weights is None: + num_values = math_ops.cast(array_ops.size(values), dtypes.float32) + else: + values, _, weights = _remove_squeezable_dimensions( + predictions=values, labels=None, weights=weights) + weights = weights_broadcast_ops.broadcast_weights( + math_ops.cast(weights, dtypes.float32), values) + values = math_ops.multiply(values, weights) + num_values = math_ops.reduce_sum(weights) + + update_total_op = state_ops.assign_add(total, math_ops.reduce_sum(values)) + with ops.control_dependencies([values]): + update_count_op = state_ops.assign_add(count, num_values) + + def compute_mean(_, t, c): + return math_ops.div_no_nan(t, math_ops.maximum(c, 0), name='value') + + mean_t = _aggregate_across_replicas( + metrics_collections, compute_mean, total, count) + + #### modified to current value + update_op = math_ops.reduce_sum(values) + + if updates_collections: + ops.add_to_collections(updates_collections, update_op) + + return mean_t, update_op + + + diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/README.md new file mode 100644 index 0000000..c680f8b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/README.md @@ -0,0 +1,380 @@ +![No Maintenance Intended](https://img.shields.io/badge/No%20Maintenance%20Intended-%E2%9C%95-red.svg) +![TensorFlow Requirement: 1.x](https://img.shields.io/badge/TensorFlow%20Requirement-1.x-brightgreen) +![TensorFlow 2 Not Supported](https://img.shields.io/badge/TensorFlow%202%20Not%20Supported-%E2%9C%95-red.svg) + +# Transformer Translation Model +This is an implementation of the Transformer translation model as described in the [Attention is All You Need](https://arxiv.org/abs/1706.03762) paper. Based on the code provided by the authors: [Transformer code](https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/models/transformer.py) from [Tensor2Tensor](https://github.com/tensorflow/tensor2tensor). Also, check out the [tutorial](https://www.tensorflow.org/beta/tutorials/text/transformer) on Transformer in TF 2.0. + +**Please follow the [README](https://github.com/tensorflow/models/official/transformer/README.md), the new Keras-based TF 2 implementation, to walk through the new Transformer.** + +Transformer is a neural network architecture that solves sequence to sequence problems using attention mechanisms. Unlike traditional neural seq2seq models, Transformer does not involve recurrent connections. The attention mechanism learns dependencies between tokens in two sequences. Since attention weights apply to all tokens in the sequences, the Transformer model is able to easily capture long-distance dependencies. + +Transformer's overall structure follows the standard encoder-decoder pattern. The encoder uses self-attention to compute a representation of the input sequence. The decoder generates the output sequence one token at a time, taking the encoder output and previous decoder-outputted tokens as inputs. + +The model also applies embeddings on the input and output tokens, and adds a constant positional encoding. The positional encoding adds information about the position of each token. + +## Contents + * [Contents](#contents) + * [Walkthrough](#walkthrough) + * [Benchmarks](#benchmarks) + * [Training times](#training-times) + * [Evaluation results](#evaluation-results) + * [Detailed instructions](#detailed-instructions) + * [Environment preparation](#environment-preparation) + * [Download and preprocess datasets](#download-and-preprocess-datasets) + * [Model training and evaluation](#model-training-and-evaluation) + * [Translate using the model](#translate-using-the-model) + * [Compute official BLEU score](#compute-official-bleu-score) + * [TPU](#tpu) + * [Export trained model](#export-trained-model) + * [Example translation](#example-translation) + * [Implementation overview](#implementation-overview) + * [Model Definition](#model-definition) + * [Model Estimator](#model-estimator) + * [Other scripts](#other-scripts) + * [Test dataset](#test-dataset) + * [Term definitions](#term-definitions) + +## Walkthrough + +Below are the commands for running the Transformer model. See the +[Detailed instructions](#detailed-instructions) for more details on running the +model. + +``` +cd /path/to/models/official/transformer + +# Ensure that PYTHONPATH is correctly defined as described in +# https://github.com/tensorflow/models/tree/master/official#requirements +# export PYTHONPATH="$PYTHONPATH:/path/to/models" + +# Export variables +PARAM_SET=big +DATA_DIR=$HOME/transformer/data +MODEL_DIR=$HOME/transformer/model_$PARAM_SET +VOCAB_FILE=$DATA_DIR/vocab.ende.32768 + +# Download training/evaluation/test datasets +python data_download.py --data_dir=$DATA_DIR + +# Train the model for 10 epochs, and evaluate after every epoch. +python transformer_main.py --data_dir=$DATA_DIR --model_dir=$MODEL_DIR \ + --vocab_file=$VOCAB_FILE --param_set=$PARAM_SET \ + --bleu_source=$DATA_DIR/newstest2014.en --bleu_ref=$DATA_DIR/newstest2014.de + +# Run during training in a separate process to get continuous updates, +# or after training is complete. +tensorboard --logdir=$MODEL_DIR + +# Translate some text using the trained model +python translate.py --model_dir=$MODEL_DIR --vocab_file=$VOCAB_FILE \ + --param_set=$PARAM_SET --text="hello world" + +# Compute model's BLEU score using the newstest2014 dataset. +python translate.py --model_dir=$MODEL_DIR --vocab_file=$VOCAB_FILE \ + --param_set=$PARAM_SET --file=$DATA_DIR/newstest2014.en --file_out=translation.en +python compute_bleu.py --translation=translation.en --reference=$DATA_DIR/newstest2014.de +``` + +## Benchmarks +### Training times + +Currently, both big and base parameter sets run on a single GPU. The measurements below +are reported from running the model on a P100 GPU. + +Param Set | batches/sec | batches per epoch | time per epoch +--- | --- | --- | --- +base | 4.8 | 83244 | 4 hr +big | 1.1 | 41365 | 10 hr + +### Evaluation results +Below are the case-insensitive BLEU scores after 10 epochs. + +Param Set | Score +--- | --- | +base | 27.7 +big | 28.9 + + +## Detailed instructions + + +0. ### Environment preparation + + #### Add models repo to PYTHONPATH + Follow the instructions described in the [Requirements](https://github.com/tensorflow/models/tree/master/official#requirements) section to add the models folder to the python path. + + #### Export variables (optional) + + Export the following variables, or modify the values in each of the snippets below: + ``` + PARAM_SET=big + DATA_DIR=$HOME/transformer/data + MODEL_DIR=$HOME/transformer/model_$PARAM_SET + VOCAB_FILE=$DATA_DIR/vocab.ende.32768 + ``` + +1. ### Download and preprocess datasets + + [data_download.py](data_download.py) downloads and preprocesses the training and evaluation WMT datasets. After the data is downloaded and extracted, the training data is used to generate a vocabulary of subtokens. The evaluation and training strings are tokenized, and the resulting data is sharded, shuffled, and saved as TFRecords. + + 1.75GB of compressed data will be downloaded. In total, the raw files (compressed, extracted, and combined files) take up 8.4GB of disk space. The resulting TFRecord and vocabulary files are 722MB. The script takes around 40 minutes to run, with the bulk of the time spent downloading and ~15 minutes spent on preprocessing. + + Command to run: + ``` + python data_download.py --data_dir=$DATA_DIR + ``` + + Arguments: + * `--data_dir`: Path where the preprocessed TFRecord data, and vocab file will be saved. + * Use the `--help` or `-h` flag to get a full list of possible arguments. + +2. ### Model training and evaluation + + [transformer_main.py](transformer_main.py) creates a Transformer model, and trains it using Tensorflow Estimator. + + Command to run: + ``` + python transformer_main.py --data_dir=$DATA_DIR --model_dir=$MODEL_DIR \ + --vocab_file=$VOCAB_FILE --param_set=$PARAM_SET + ``` + + Arguments: + * `--data_dir`: This should be set to the same directory given to the `data_download`'s `data_dir` argument. + * `--model_dir`: Directory to save Transformer model training checkpoints. + * `--vocab_file`: Path to subtoken vocabulary file. If data_download was used, you may find the file in `data_dir`. + * `--param_set`: Parameter set to use when creating and training the model. Options are `base` and `big` (default). + * Use the `--help` or `-h` flag to get a full list of possible arguments. + + #### Customizing training schedule + + By default, the model will train for 10 epochs, and evaluate after every epoch. The training schedule may be defined through the flags: + * Training with epochs (default): + * `--train_epochs`: The total number of complete passes to make through the dataset + * `--epochs_between_evals`: The number of epochs to train between evaluations. + * Training with steps: + * `--train_steps`: sets the total number of training steps to run. + * `--steps_between_evals`: Number of training steps to run between evaluations. + + Only one of `train_epochs` or `train_steps` may be set. Since the default option is to evaluate the model after training for an epoch, it may take 4 or more hours between model evaluations. To get more frequent evaluations, use the flags `--train_steps=250000 --steps_between_evals=1000`. + + Note: At the beginning of each training session, the training dataset is reloaded and shuffled. Stopping the training before completing an epoch may result in worse model quality, due to the chance that some examples may be seen more than others. Therefore, it is recommended to use epochs when the model quality is important. + + #### Compute BLEU score during model evaluation + + Use these flags to compute the BLEU when the model evaluates: + * `--bleu_source`: Path to file containing text to translate. + * `--bleu_ref`: Path to file containing the reference translation. + * `--stop_threshold`: Train until the BLEU score reaches this lower bound. This setting overrides the `--train_steps` and `--train_epochs` flags. + + When running `transformer_main.py`, use the flags: `--bleu_source=$DATA_DIR/newstest2014.en --bleu_ref=$DATA_DIR/newstest2014.de` + + #### Tensorboard + Training and evaluation metrics (loss, accuracy, approximate BLEU score, etc.) are logged, and can be displayed in the browser using Tensorboard. + ``` + tensorboard --logdir=$MODEL_DIR + ``` + The values are displayed at [localhost:6006](localhost:6006). + +3. ### Translate using the model + [translate.py](translate.py) contains the script to use the trained model to translate input text or file. Each line in the file is translated separately. + + Command to run: + ``` + python translate.py --model_dir=$MODEL_DIR --vocab_file=$VOCAB_FILE \ + --param_set=$PARAM_SET --text="hello world" + ``` + + Arguments for initializing the Subtokenizer and trained model: + * `--model_dir` and `--param_set`: These parameters are used to rebuild the trained model + * `--vocab_file`: Path to subtoken vocabulary file. If data_download was used, you may find the file in `data_dir`. + + Arguments for specifying what to translate: + * `--text`: Text to translate + * `--file`: Path to file containing text to translate + * `--file_out`: If `--file` is set, then this file will store the input file's translations. + + To translate the newstest2014 data, run: + ``` + python translate.py --model_dir=$MODEL_DIR --vocab_file=$VOCAB_FILE \ + --param_set=$PARAM_SET --file=$DATA_DIR/newstest2014.en --file_out=translation.en + ``` + + Translating the file takes around 15 minutes on a GTX1080, or 5 minutes on a P100. + +4. ### Compute official BLEU score + Use [compute_bleu.py](compute_bleu.py) to compute the BLEU by comparing generated translations to the reference translation. + + Command to run: + ``` + python compute_bleu.py --translation=translation.en --reference=$DATA_DIR/newstest2014.de + ``` + + Arguments: + * `--translation`: Path to file containing generated translations. + * `--reference`: Path to file containing reference translations. + * Use the `--help` or `-h` flag to get a full list of possible arguments. + +5. ### TPU + TPU support for this version of Transformer is experimental. Currently it is present for + demonstration purposes only, but will be optimized in the coming weeks. + +## Export trained model +To export the model as a Tensorflow [SavedModel](https://www.tensorflow.org/guide/saved_model) format, use the argument `--export_dir` when running `transformer_main.py`. A folder will be created in the directory with the name as the timestamp (e.g. $EXPORT_DIR/1526427396). + +``` +EXPORT_DIR=$HOME/transformer/saved_model +python transformer_main.py --data_dir=$DATA_DIR --model_dir=$MODEL_DIR \ + --vocab_file=$VOCAB_FILE --param_set=$PARAM_SET --export_model=$EXPORT_DIR +``` + +To inspect the SavedModel, use saved_model_cli: +``` +SAVED_MODEL_DIR=$EXPORT_DIR/{TIMESTAMP} # replace {TIMESTAMP} with the name of the folder created +saved_model_cli show --dir=$SAVED_MODEL_DIR --all +``` + +### Example translation +Let's translate **"hello world!"**, **"goodbye world."**, and **"Would you like some pie?"**. + +The SignatureDef for "translate" is: + + signature_def['translate']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_INT64 + shape: (-1, -1) + name: Placeholder:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['outputs'] tensor_info: + dtype: DT_INT32 + shape: (-1, -1) + name: model/Transformer/strided_slice_19:0 + outputs['scores'] tensor_info: + dtype: DT_FLOAT + shape: (-1) + name: model/Transformer/strided_slice_20:0 + +Follow the steps below to use the translate signature def: + +1. #### Encode the inputs to integer arrays. + This can be done using `utils.tokenizer.Subtokenizer`, and the vocab file in the SavedModel assets (`$SAVED_MODEL_DIR/assets.extra/vocab.txt`). + + ``` + from official.transformer.utils.tokenizer import Subtokenizer + s = Subtokenizer(PATH_TO_VOCAB_FILE) + print(s.encode("hello world!", add_eos=True)) + ``` + + The encoded inputs are: + * `"hello world!" = [6170, 3731, 178, 207, 1]` + * `"goodbye world." = [15431, 13966, 36, 178, 3, 1]` + * `"Would you like some pie?" = [9092, 72, 155, 202, 19851, 102, 1]` + +2. #### Run `saved_model_cli` to obtain the predicted translations + The encoded inputs should be padded so that they are the same length. The padding token is `0`. + ``` + ENCODED_INPUTS="[[26228, 145, 178, 1, 0, 0, 0], \ + [15431, 13966, 36, 178, 3, 1, 0], \ + [9092, 72, 155, 202, 19851, 102, 1]]" + ``` + + Now, use the `run` command with `saved_model_cli` to get the outputs. + + ``` + saved_model_cli run --dir=$SAVED_MODEL_DIR --tag_set=serve --signature_def=translate \ + --input_expr="input=$ENCODED_INPUTS" + ``` + + The outputs will look similar to: + ``` + Result for output key outputs: + [[18744 145 297 1 0 0 0 0 0 0 0 0 + 0 0] + [ 5450 4642 21 11 297 3 1 0 0 0 0 0 + 0 0] + [25940 22 66 103 21713 31 102 1 0 0 0 0 + 0 0]] + Result for output key scores: + [-1.5493642 -1.4032784 -3.252089 ] + ``` + +3. #### Decode the outputs to strings. + Use the `Subtokenizer` and vocab file as described in step 1 to decode the output integer arrays. + ``` + from official.transformer.utils.tokenizer import Subtokenizer + s = Subtokenizer(PATH_TO_VOCAB_FILE) + print(s.decode([18744, 145, 297, 1])) + ``` + The decoded outputs from above are: + * `[18744, 145, 297, 1] = "Hallo Welt"` + * `[5450, 4642, 21, 11, 297, 3, 1] = "Abschied von der Welt."` + * `[25940, 22, 66, 103, 21713, 31, 102, 1] = "Möchten Sie einen Kuchen?"` + +## Implementation overview + +A brief look at each component in the code: + +### Model Definition +The [model](model) subdirectory contains the implementation of the Transformer model. The following files define the Transformer model and its layers: +* [transformer.py](model/transformer.py): Defines the transformer model and its encoder/decoder layer stacks. +* [embedding_layer.py](model/embedding_layer.py): Contains the layer that calculates the embeddings. The embedding weights are also used to calculate the pre-softmax probabilities from the decoder output. +* [attention_layer.py](model/attention_layer.py): Defines the multi-headed and self attention layers that are used in the encoder/decoder stacks. +* [ffn_layer.py](model/ffn_layer.py): Defines the feedforward network that is used in the encoder/decoder stacks. The network is composed of 2 fully connected layers. + +Other files: +* [beam_search.py](model/beam_search.py) contains the beam search implementation, which is used during model inference to find high scoring translations. +* [model_params.py](model/model_params.py) contains the parameters used for the big and base models. +* [model_utils.py](model/model_utils.py) defines some helper functions used in the model (calculating padding, bias, etc.). + + +### Model Estimator +[transformer_main.py](model/transformer.py) creates an `Estimator` to train and evaluate the model. + +Helper functions: +* [utils/dataset.py](utils/dataset.py): contains functions for creating a `dataset` that is passed to the `Estimator`. +* [utils/metrics.py](utils/metrics.py): defines metrics functions used by the `Estimator` to evaluate the + +### Other scripts + +Aside from the main file to train the Transformer model, we provide other scripts for using the model or downloading the data: + +#### Data download and preprocessing + +[data_download.py](data_download.py) downloads and extracts data, then uses `Subtokenizer` to tokenize strings into arrays of int IDs. The int arrays are converted to `tf.Examples` and saved in the `tf.RecordDataset` format. + + The data is downloaded from the Workshop of Machine Translation (WMT) [news translation task](http://www.statmt.org/wmt17/translation-task.html). The following datasets are used: + + * Europarl v7 + * Common Crawl corpus + * News Commentary v12 + + See the [download section](http://www.statmt.org/wmt17/translation-task.html#download) to explore the raw datasets. The parameters in this model are tuned to fit the English-German translation data, so the EN-DE texts are extracted from the downloaded compressed files. + +The text is transformed into arrays of integer IDs using the `Subtokenizer` defined in [`utils/tokenizer.py`](util/tokenizer.py). During initialization of the `Subtokenizer`, the raw training data is used to generate a vocabulary list containing common subtokens. + +The target vocabulary size of the WMT dataset is 32,768. The set of subtokens is found through binary search on the minimum number of times a subtoken appears in the data. The actual vocabulary size is 33,708, and is stored in a 324kB file. + +#### Translation +Translation is defined in [translate.py](translate.py). First, `Subtokenizer` tokenizes the input. The vocabulary file is the same used to tokenize the training/eval files. Next, beam search is used to find the combination of tokens that maximizes the probability outputted by the model decoder. The tokens are then converted back to strings with `Subtokenizer`. + +#### BLEU computation +[compute_bleu.py](compute_bleu.py): Implementation from [https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/bleu_hook.py](https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/bleu_hook.py). + +### Test dataset +The [newstest2014 files](https://storage.googleapis.com/tf-perf-public/official_transformer/test_data/newstest2014.tgz) +are extracted from the [NMT Seq2Seq tutorial](https://google.github.io/seq2seq/nmt/#download-data). +The raw text files are converted from the SGM format of the +[WMT 2016](http://www.statmt.org/wmt16/translation-task.html) test sets. The +newstest2014 files are put into the `$DATA_DIR` when executing +`data_download.py` + +## Term definitions + +**Steps / Epochs**: +* Step: unit for processing a single batch of data +* Epoch: a complete run through the dataset + +Example: Consider a training a dataset with 100 examples that is divided into 20 batches with 5 examples per batch. A single training step trains the model on one batch. After 20 training steps, the model will have trained on every batch in the dataset, or one epoch. + +**Subtoken**: Words are referred to as tokens, and parts of words are referred to as 'subtokens'. For example, the word 'inclined' may be split into `['incline', 'd_']`. The '\_' indicates the end of the token. The subtoken vocabulary list is guaranteed to contain the alphabet (including numbers and special characters), so all words can be tokenized. diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/attention_layer.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/attention_layer.py new file mode 100644 index 0000000..e353793 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/attention_layer.py @@ -0,0 +1,148 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Implementation of multiheaded attention and self-attention layers.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow.compat.v1 as tf + + +class Attention(tf.layers.Layer): + """Multi-headed attention layer.""" + + def __init__(self, hidden_size, num_heads, attention_dropout, train): + if hidden_size % num_heads != 0: + raise ValueError("Hidden size must be evenly divisible by the number of " + "heads.") + + super(Attention, self).__init__() + self.hidden_size = hidden_size + self.num_heads = num_heads + self.attention_dropout = attention_dropout + self.train = train + + # Layers for linearly projecting the queries, keys, and values. + self.q_dense_layer = tf.layers.Dense(hidden_size, use_bias=False, name="q") + self.k_dense_layer = tf.layers.Dense(hidden_size, use_bias=False, name="k") + self.v_dense_layer = tf.layers.Dense(hidden_size, use_bias=False, name="v") + + self.output_dense_layer = tf.layers.Dense(hidden_size, use_bias=False, + name="output_transform") + + def split_heads(self, x): + """Split x into different heads, and transpose the resulting value. + + The tensor is transposed to insure the inner dimensions hold the correct + values during the matrix multiplication. + + Args: + x: A tensor with shape [batch_size, length, hidden_size] + + Returns: + A tensor with shape [batch_size, num_heads, length, hidden_size/num_heads] + """ + with tf.name_scope("split_heads"): + batch_size = tf.shape(x)[0] + length = tf.shape(x)[1] + + # Calculate depth of last dimension after it has been split. + depth = (self.hidden_size // self.num_heads) + + # Split the last dimension + x = tf.reshape(x, [batch_size, length, self.num_heads, depth]) + + # Transpose the result + return tf.transpose(x, [0, 2, 1, 3]) + + def combine_heads(self, x): + """Combine tensor that has been split. + + Args: + x: A tensor [batch_size, num_heads, length, hidden_size/num_heads] + + Returns: + A tensor with shape [batch_size, length, hidden_size] + """ + with tf.name_scope("combine_heads"): + batch_size = tf.shape(x)[0] + length = tf.shape(x)[2] + x = tf.transpose(x, [0, 2, 1, 3]) # --> [batch, length, num_heads, depth] + return tf.reshape(x, [batch_size, length, self.hidden_size]) + + def call(self, x, y, bias, cache=None): + """Apply attention mechanism to x and y. + + Args: + x: a tensor with shape [batch_size, length_x, hidden_size] + y: a tensor with shape [batch_size, length_y, hidden_size] + bias: attention bias that will be added to the result of the dot product. + cache: (Used during prediction) dictionary with tensors containing results + of previous attentions. The dictionary must have the items: + {"k": tensor with shape [batch_size, i, key_channels], + "v": tensor with shape [batch_size, i, value_channels]} + where i is the current decoded length. + + Returns: + Attention layer output with shape [batch_size, length_x, hidden_size] + """ + # Linearly project the query (q), key (k) and value (v) using different + # learned projections. This is in preparation of splitting them into + # multiple heads. Multi-head attention uses multiple queries, keys, and + # values rather than regular attention (which uses a single q, k, v). + q = self.q_dense_layer(x) + k = self.k_dense_layer(y) + v = self.v_dense_layer(y) + + if cache is not None: + # Combine cached keys and values with new keys and values. + k = tf.concat([cache["k"], k], axis=1) + v = tf.concat([cache["v"], v], axis=1) + + # Update cache + cache["k"] = k + cache["v"] = v + + # Split q, k, v into heads. + q = self.split_heads(q) + k = self.split_heads(k) + v = self.split_heads(v) + + # Scale q to prevent the dot product between q and k from growing too large. + depth = (self.hidden_size // self.num_heads) + q *= depth ** -0.5 + + # Calculate dot product attention + logits = tf.matmul(q, k, transpose_b=True) + logits += bias + weights = tf.nn.softmax(logits, name="attention_weights") + if self.train: + weights = tf.nn.dropout(weights, 1.0 - self.attention_dropout) + attention_output = tf.matmul(weights, v) + + # Recombine heads --> [batch_size, length, hidden_size] + attention_output = self.combine_heads(attention_output) + + # Run the combined outputs through another linear projection layer. + attention_output = self.output_dense_layer(attention_output) + return attention_output + + +class SelfAttention(Attention): + """Multiheaded self-attention layer.""" + + def call(self, x, bias, cache=None): + return super(SelfAttention, self).call(x, x, bias, cache) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/dataset.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/dataset.py new file mode 100644 index 0000000..4798774 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/dataset.py @@ -0,0 +1,284 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Input pipeline for the transformer model to read, filter, and batch examples. + +Two things to note in the pipeline: + +1. Batching scheme + + The examples encoded in the TFRecord files contain data in the format: + {"inputs": [variable length array of integers], + "targets": [variable length array of integers]} + Where integers in the arrays refer to tokens in the English and German vocab + file (named `vocab.ende.32768`). + + Prior to batching, elements in the dataset are grouped by length (max between + "inputs" and "targets" length). Each group is then batched such that: + group_batch_size * length <= batch_size. + + Another way to view batch_size is the maximum number of tokens in each batch. + + Once batched, each element in the dataset will have the shape: + {"inputs": [group_batch_size, padded_input_length], + "targets": [group_batch_size, padded_target_length]} + Lengths are padded to the longest "inputs" or "targets" sequence in the batch + (padded_input_length and padded_target_length can be different). + + This batching scheme decreases the fraction of padding tokens per training + batch, thus improving the training speed significantly. + +2. Shuffling + + While training, the dataset is shuffled in two places in the code. The first + is the list of training files. Second, while reading records using + `parallel_interleave`, the `sloppy` argument is used to generate randomness + in the order of the examples. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math +import os + +import tensorflow.compat.v1 as tf + +from official.utils.misc import model_helpers + +# Buffer size for reading records from a TFRecord file. Each training file is +# 7.2 MB, so 8 MB allows an entire file to be kept in memory. +_READ_RECORD_BUFFER = 8 * 1000 * 1000 + +# Example grouping constants. Defines length boundaries for each group. +# These values are the defaults used in Tensor2Tensor. +_MIN_BOUNDARY = 8 +_BOUNDARY_SCALE = 1.1 + + +def _load_records(filename): + """Read file and return a dataset of tf.Examples.""" + return tf.data.TFRecordDataset(filename, buffer_size=_READ_RECORD_BUFFER) + + +def _parse_example(serialized_example): + """Return inputs and targets Tensors from a serialized tf.Example.""" + data_fields = { + "inputs": tf.VarLenFeature(tf.int64), + "targets": tf.VarLenFeature(tf.int64) + } + parsed = tf.parse_single_example(serialized_example, data_fields) + inputs = tf.sparse_tensor_to_dense(parsed["inputs"]) + targets = tf.sparse_tensor_to_dense(parsed["targets"]) + return inputs, targets + + +def _filter_max_length(example, max_length=256): + """Indicates whether the example's length is lower than the maximum length.""" + return tf.logical_and(tf.size(example[0]) <= max_length, + tf.size(example[1]) <= max_length) + + +def _get_example_length(example): + """Returns the maximum length between the example inputs and targets.""" + length = tf.maximum(tf.shape(example[0])[0], tf.shape(example[1])[0]) + return length + + +def _create_min_max_boundaries( + max_length, min_boundary=_MIN_BOUNDARY, boundary_scale=_BOUNDARY_SCALE): + """Create min and max boundary lists up to max_length. + + For example, when max_length=24, min_boundary=4 and boundary_scale=2, the + returned values will be: + buckets_min = [0, 4, 8, 16, 24] + buckets_max = [4, 8, 16, 24, 25] + + Args: + max_length: The maximum length of example in dataset. + min_boundary: Minimum length in boundary. + boundary_scale: Amount to scale consecutive boundaries in the list. + + Returns: + min and max boundary lists + + """ + # Create bucket boundaries list by scaling the previous boundary or adding 1 + # (to ensure increasing boundary sizes). + bucket_boundaries = [] + x = min_boundary + while x < max_length: + bucket_boundaries.append(x) + x = max(x + 1, int(x * boundary_scale)) + + # Create min and max boundary lists from the initial list. + buckets_min = [0] + bucket_boundaries + buckets_max = bucket_boundaries + [max_length + 1] + return buckets_min, buckets_max + + +def _batch_examples(dataset, batch_size, max_length): + """Group examples by similar lengths, and return batched dataset. + + Each batch of similar-length examples are padded to the same length, and may + have different number of elements in each batch, such that: + group_batch_size * padded_length <= batch_size. + + This decreases the number of padding tokens per batch, which improves the + training speed. + + Args: + dataset: Dataset of unbatched examples. + batch_size: Max number of tokens per batch of examples. + max_length: Max number of tokens in an example input or target sequence. + + Returns: + Dataset of batched examples with similar lengths. + """ + # Get min and max boundary lists for each example. These are used to calculate + # the `bucket_id`, which is the index at which: + # buckets_min[bucket_id] <= len(example) < buckets_max[bucket_id] + # Note that using both min and max lists improves the performance. + buckets_min, buckets_max = _create_min_max_boundaries(max_length) + + # Create list of batch sizes for each bucket_id, so that + # bucket_batch_size[bucket_id] * buckets_max[bucket_id] <= batch_size + bucket_batch_sizes = [batch_size // x for x in buckets_max] + # bucket_id will be a tensor, so convert this list to a tensor as well. + bucket_batch_sizes = tf.constant(bucket_batch_sizes, dtype=tf.int64) + + def example_to_bucket_id(example_input, example_target): + """Return int64 bucket id for this example, calculated based on length.""" + seq_length = _get_example_length((example_input, example_target)) + + # TODO: investigate whether removing code branching improves performance. + conditions_c = tf.logical_and( + tf.less_equal(buckets_min, seq_length), + tf.less(seq_length, buckets_max)) + bucket_id = tf.reduce_min(tf.where(conditions_c)) + return bucket_id + + def window_size_fn(bucket_id): + """Return number of examples to be grouped when given a bucket id.""" + return bucket_batch_sizes[bucket_id] + + def batching_fn(bucket_id, grouped_dataset): + """Batch and add padding to a dataset of elements with similar lengths.""" + bucket_batch_size = window_size_fn(bucket_id) + + # Batch the dataset and add padding so that all input sequences in the + # examples have the same length, and all target sequences have the same + # lengths as well. Resulting lengths of inputs and targets can differ. + return grouped_dataset.padded_batch(bucket_batch_size, ([None], [None])) + + return dataset.apply(tf.data.experimental.group_by_window( + key_func=example_to_bucket_id, + reduce_func=batching_fn, + window_size=None, + window_size_func=window_size_fn)) + + +def _read_and_batch_from_files( + file_pattern, batch_size, max_length, num_parallel_calls, shuffle, repeat, + static_batch=False): + """Create dataset where each item is a dict of "inputs" and "targets". + + Args: + file_pattern: String used to match the input TFRecord files. + batch_size: Maximum number of tokens per batch of examples + max_length: Maximum number of tokens per example + num_parallel_calls: Number of cpu cores for parallel input processing. + shuffle: If true, randomizes order of elements. + repeat: Number of times to repeat the dataset. If None, the dataset is + repeated forever. + static_batch: Whether the batches in the dataset should have static shapes. + If True, the input is batched so that every batch has the + shape [batch_size // max_length, max_length]. If False, the input is + grouped by length, and batched so that batches may have different + shapes [N, M], where: + N * M <= batch_size + M <= max_length + In general, this setting should be False. Dynamic shapes allow the inputs + to be grouped so that the number of padding tokens is minimized, and helps + model training. In cases where the input shape must be static + (e.g. running on TPU), this setting should be set to True. + + Returns: + tf.data.Dataset object containing examples loaded from the files. + """ + dataset = tf.data.Dataset.list_files(file_pattern, shuffle=shuffle) + + # Read files and interleave results. When training, the order of the examples + # will be non-deterministic. + dataset = dataset.apply( + tf.data.experimental.parallel_interleave( + _load_records, sloppy=shuffle, cycle_length=num_parallel_calls)) + + # Parse each tf.Example into a dictionary + # TODO: Look into prefetch_input_elements for performance optimization. + dataset = dataset.map(_parse_example, + num_parallel_calls=num_parallel_calls) + + # Remove examples where the input or target length exceeds the maximum length, + dataset = dataset.filter(lambda x, y: _filter_max_length((x, y), max_length)) + + if static_batch: + dataset = dataset.padded_batch( + batch_size // max_length, ([max_length], [max_length]), + drop_remainder=True) + else: + # Group and batch such that each batch has examples of similar length. + dataset = _batch_examples(dataset, batch_size, max_length) + + dataset = dataset.repeat(repeat) + + # Prefetch the next element to improve speed of input pipeline. + dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE) + return dataset + + +def _generate_synthetic_data(params): + """Create synthetic data based on the parameter batch size.""" + batch = length = int(math.sqrt(params["batch_size"])) + return model_helpers.generate_synthetic_data( + input_shape=tf.TensorShape([batch, length]), + input_value=1, + input_dtype=tf.int32, + label_shape=tf.TensorShape([batch, length]), + label_value=1, + label_dtype=tf.int32, + ) + + +def train_input_fn(params): + """Load and return dataset of batched examples for use during training.""" + file_pattern = os.path.join(params["data_dir"] or "", "*train*") + if params["use_synthetic_data"]: + return _generate_synthetic_data(params) + return _read_and_batch_from_files( + file_pattern, params["batch_size"], params["max_length"], + params["num_parallel_calls"], shuffle=True, + repeat=params["repeat_dataset"], static_batch=params["static_batch"]) + + +def eval_input_fn(params): + """Load and return dataset of batched examples for use during evaluation.""" + file_pattern = os.path.join(params["data_dir"] or "", "*dev*") + if params["use_synthetic_data"]: + return _generate_synthetic_data(params) + return _read_and_batch_from_files( + file_pattern, params["batch_size"], params["max_length"], + params["num_parallel_calls"], shuffle=False, repeat=1, + static_batch=params["static_batch"]) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/embedding_layer.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/embedding_layer.py new file mode 100644 index 0000000..3ebedea --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/embedding_layer.py @@ -0,0 +1,108 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Implementation of embedding layer with shared weights.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow.compat.v1 as tf # pylint: disable=g-bad-import-order + +from official.r1.utils import tpu as tpu_utils + + +class EmbeddingSharedWeights(tf.layers.Layer): + """Calculates input embeddings and pre-softmax linear with shared weights.""" + + def __init__(self, vocab_size, hidden_size, method="gather"): + """Specify characteristic parameters of embedding layer. + + Args: + vocab_size: Number of tokens in the embedding. (Typically ~32,000) + hidden_size: Dimensionality of the embedding. (Typically 512 or 1024) + method: Strategy for performing embedding lookup. "gather" uses tf.gather + which performs well on CPUs and GPUs, but very poorly on TPUs. "matmul" + one-hot encodes the indicies and formulates the embedding as a sparse + matrix multiplication. The matmul formulation is wasteful as it does + extra work, however matrix multiplication is very fast on TPUs which + makes "matmul" considerably faster than "gather" on TPUs. + """ + super(EmbeddingSharedWeights, self).__init__() + self.vocab_size = vocab_size + self.hidden_size = hidden_size + if method not in ("gather", "matmul"): + raise ValueError("method {} must be 'gather' or 'matmul'".format(method)) + self.method = method + + def build(self, _): + with tf.variable_scope("embedding_and_softmax", reuse=tf.AUTO_REUSE): + # Create and initialize weights. The random normal initializer was chosen + # randomly, and works well. + self.shared_weights = tf.get_variable( + "weights", [self.vocab_size, self.hidden_size], + initializer=tf.random_normal_initializer( + 0., self.hidden_size ** -0.5)) + + self.built = True + + def call(self, x): + """Get token embeddings of x. + + Args: + x: An int64 tensor with shape [batch_size, length] + Returns: + embeddings: float32 tensor with shape [batch_size, length, embedding_size] + padding: float32 tensor with shape [batch_size, length] indicating the + locations of the padding tokens in x. + """ + with tf.name_scope("embedding"): + # Create binary mask of size [batch_size, length] + mask = tf.to_float(tf.not_equal(x, 0)) + + if self.method == "gather": + embeddings = tf.gather(self.shared_weights, x) + embeddings *= tf.expand_dims(mask, -1) + else: # matmul + embeddings = tpu_utils.embedding_matmul( + embedding_table=self.shared_weights, + values=tf.cast(x, dtype=tf.int32), + mask=mask + ) + # embedding_matmul already zeros out masked positions, so + # `embeddings *= tf.expand_dims(mask, -1)` is unnecessary. + + + # Scale embedding by the sqrt of the hidden size + embeddings *= self.hidden_size ** 0.5 + + return embeddings + + + def linear(self, x): + """Computes logits by running x through a linear layer. + + Args: + x: A float32 tensor with shape [batch_size, length, hidden_size] + Returns: + float32 tensor with shape [batch_size, length, vocab_size]. + """ + with tf.name_scope("presoftmax_linear"): + batch_size = tf.shape(x)[0] + length = tf.shape(x)[1] + + x = tf.reshape(x, [-1, self.hidden_size]) + logits = tf.matmul(x, self.shared_weights, transpose_b=True) + + return tf.reshape(logits, [batch_size, length, self.vocab_size]) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/ffn_layer.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/ffn_layer.py new file mode 100644 index 0000000..fc47503 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/ffn_layer.py @@ -0,0 +1,89 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Implementation of fully connected network.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow.compat.v1 as tf + + +class FeedFowardNetwork(tf.layers.Layer): + """Fully connected feedforward network.""" + + def __init__(self, hidden_size, filter_size, relu_dropout, train, allow_pad): + super(FeedFowardNetwork, self).__init__() + self.hidden_size = hidden_size + self.filter_size = filter_size + self.relu_dropout = relu_dropout + self.train = train + self.allow_pad = allow_pad + + self.filter_dense_layer = tf.layers.Dense( + filter_size, use_bias=True, activation=tf.nn.relu, name="filter_layer") + self.output_dense_layer = tf.layers.Dense( + hidden_size, use_bias=True, name="output_layer") + + def call(self, x, padding=None): + """Return outputs of the feedforward network. + + Args: + x: tensor with shape [batch_size, length, hidden_size] + padding: (optional) If set, the padding values are temporarily removed + from x (provided self.allow_pad is set). The padding values are placed + back in the output tensor in the same locations. + shape [batch_size, length] + + Returns: + Output of the feedforward network. + tensor with shape [batch_size, length, hidden_size] + """ + padding = None if not self.allow_pad else padding + + # Retrieve dynamically known shapes + batch_size = tf.shape(x)[0] + length = tf.shape(x)[1] + + if padding is not None: + with tf.name_scope("remove_padding"): + # Flatten padding to [batch_size*length] + pad_mask = tf.reshape(padding, [-1]) + + nonpad_ids = tf.to_int32(tf.where(pad_mask < 1e-9)) + + # Reshape x to [batch_size*length, hidden_size] to remove padding + x = tf.reshape(x, [-1, self.hidden_size]) + x = tf.gather_nd(x, indices=nonpad_ids) + + # Reshape x from 2 dimensions to 3 dimensions. + x.set_shape([None, self.hidden_size]) + x = tf.expand_dims(x, axis=0) + + output = self.filter_dense_layer(x) + if self.train: + output = tf.nn.dropout(output, 1.0 - self.relu_dropout) + output = self.output_dense_layer(output) + + if padding is not None: + with tf.name_scope("re_add_padding"): + output = tf.squeeze(output, axis=0) + output = tf.scatter_nd( + indices=nonpad_ids, + updates=output, + shape=[batch_size * length, self.hidden_size] + ) + output = tf.reshape(output, [batch_size, length, self.hidden_size]) + return output diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/schedule.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/schedule.py new file mode 100644 index 0000000..60aedff --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/schedule.py @@ -0,0 +1,130 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Abstract training on a step or epoch basis.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math + +import tensorflow.compat.v1 as tf + + +_TRAIN, _EVAL = tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL + + +NUM_EXAMPLES = { + tf.estimator.ModeKeys.TRAIN: 4572160, + # # Examples that are too long are filtered out, thus the total is less + # # than the total number of lines. + # 2399123 + # news-commentary-v12.de-en + # 1920209 + # commoncrawl.de-en + # 270769, # europarl-v7.de-en + tf.estimator.ModeKeys.EVAL: 3000, # newstest2013 +} + + +class Manager(object): + """Container for convenience functions to abstract step or epoch basis. + Transformer allows users to specify an epoch basis (generally recommended for + full training) or a number of steps basis (convenient since epochs are rather + large). TPUs furthermore require a step basis; however epochs are the norm in + the machine learning community and it is desirable to allow users to specify + epochs even when running with TPUS which requires behind the scenes + conversions. + This container simply groups what are largely mundane checks and conversions + rather than interspersing them throughout the run loop code. + """ + + def __init__(self, train_steps, steps_between_evals, train_epochs, + epochs_between_evals, default_train_epochs, batch_size, + max_length, use_tpu=False, num_tpu_shards=8): + if train_steps and train_epochs: + raise ValueError("Both train_steps or train_epochs were be defined.") + + # Determine training schedule based on flags. + if train_steps: + self.train_eval_iterations = train_steps // steps_between_evals + self._single_iteration_train_steps = steps_between_evals + self._single_iteration_train_epochs = None + else: + train_epochs = train_epochs or default_train_epochs + self.train_eval_iterations = train_epochs // epochs_between_evals + self._single_iteration_train_steps = None + self._single_iteration_train_epochs = epochs_between_evals + + self.max_length = max_length + self.batch_size = batch_size + self.use_tpu = use_tpu + self.num_tpu_shards = num_tpu_shards + + if self.use_tpu: + assert (self.batch_size // self.max_length) % self.num_tpu_shards == 0 + + @property + def single_iteration_train_steps(self): + if self._single_iteration_train_steps or not self.use_tpu: + return self._single_iteration_train_steps + + return self.epochs_to_steps( + num_epochs=self._single_iteration_train_epochs, mode=_TRAIN) + + @property + def single_iteration_eval_steps(self): + if not self.use_tpu: + return None + + return self.epochs_to_steps(num_epochs=1, mode=_EVAL) + + @property + def train_increment_str(self): + if self._single_iteration_train_steps: + return "{} steps.".format(self._single_iteration_train_steps) + + if not self.use_tpu: + return "{} epochs.".format(self._single_iteration_train_epochs) + + return "~{} epochs. ({} steps)".format( + self._single_iteration_train_epochs, + self.single_iteration_train_steps) + + @property + def repeat_dataset(self): + if (self._single_iteration_train_epochs is None and + self._single_iteration_train_steps > NUM_EXAMPLES[_TRAIN]): + return math.ceil(self._single_iteration_train_steps / + NUM_EXAMPLES[_TRAIN]) + return self._single_iteration_train_epochs + + def epochs_to_steps(self, num_epochs, mode): + """Converts a number of epochs to a number of training steps. + + TPU only: This function assumes that static_batch is True. + + TPU can not tolerate an OutOfRange error from a dataset. As a result the + number of examples to be processed must be known ahead of time. TPUs also + do not allow partial batches, so this function rounds down. + + Args: + num_epochs: An integer of the number of epochs to convert to steps. + mode: The estimator ModeKey of the computation + + Returns: + An integer of the number of equivalent steps rounded down. + """ + assert self.use_tpu, "epochs_to_steps should only be reached when using TPU" + total_num_tokens = NUM_EXAMPLES[mode] * self.max_length * num_epochs + return total_num_tokens // self.batch_size diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/schedule_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/schedule_test.py new file mode 100644 index 0000000..29b4d5f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/schedule_test.py @@ -0,0 +1,84 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Test Transformer's schedule manager.""" + +import tensorflow.compat.v1 as tf + +from official.r1.transformer import schedule + + +class ScheduleBaseTester(tf.test.TestCase): + def test_mutual_exclusivity(self): + with self.assertRaises(ValueError): + schedule.Manager( + train_steps=100, steps_between_evals=100, train_epochs=2, + epochs_between_evals=1, default_train_epochs=None, batch_size=2048, + max_length=256) + + def test_step_basis(self): + manager = schedule.Manager( + train_steps=1000, steps_between_evals=100, train_epochs=None, + epochs_between_evals=None, default_train_epochs=None, batch_size=2048, + max_length=256) + + self.assertEqual(manager.single_iteration_train_steps, 100) + + # Evaluation uses the full set + self.assertIsNone(manager.single_iteration_eval_steps) + + self.assertIsNone(manager.repeat_dataset) + + def test_epoch_basis(self): + manager = schedule.Manager( + train_steps=None, steps_between_evals=None, train_epochs=10, + epochs_between_evals=2, default_train_epochs=None, batch_size=2048, + max_length=256) + + # For non-TPU, estimator relies on dataset exhausion + self.assertIsNone(manager.single_iteration_train_steps) + self.assertIsNone(manager.single_iteration_eval_steps) + + self.assertEqual(manager.repeat_dataset, 2) + + def test_step_basis_tpu(self): + manager = schedule.Manager( + train_steps=1000, steps_between_evals=100, train_epochs=None, + epochs_between_evals=None, default_train_epochs=None, batch_size=2048, + max_length=256, use_tpu=True) + + self.assertEqual(manager.single_iteration_train_steps, 100) + # num_eval_examples / (batch_size / max_length) == 3000 / (2048 / 256) + self.assertEqual(manager.single_iteration_eval_steps, 375) + self.assertIsNone(manager.repeat_dataset) + + def test_epoch_basis_tpu(self): + manager = schedule.Manager( + train_steps=None, steps_between_evals=None, train_epochs=10, + epochs_between_evals=2, default_train_epochs=None, batch_size=2048, + max_length=256, use_tpu=True) + + self.assertEqual( + manager.single_iteration_train_steps, + schedule.NUM_EXAMPLES[tf.estimator.ModeKeys.TRAIN] * 2 // (2048 / 256) + ) + + # num_eval_examples / (batch_size / max_length) == 3000 / (2048 / 256) + self.assertEqual(manager.single_iteration_eval_steps, 375) + + self.assertEqual(manager.repeat_dataset, 2) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/transformer.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/transformer.py new file mode 100644 index 0000000..708c3dd --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/transformer.py @@ -0,0 +1,417 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Defines the Transformer model, and its encoder and decoder stacks. + +Model paper: https://arxiv.org/pdf/1706.03762.pdf +Transformer model code source: https://github.com/tensorflow/tensor2tensor +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow.compat.v1 as tf + +from official.nlp.transformer import beam_search_v1 as beam_search +from official.nlp.transformer import model_utils +from official.nlp.transformer.utils.tokenizer import EOS_ID +from official.r1.transformer import attention_layer +from official.r1.transformer import embedding_layer +from official.r1.transformer import ffn_layer + +_NEG_INF = -1e9 + + +class Transformer(object): + """Transformer model for sequence to sequence data. + + Implemented as described in: https://arxiv.org/pdf/1706.03762.pdf + + The Transformer model consists of an encoder and decoder. The input is an int + sequence (or a batch of sequences). The encoder produces a continous + representation, and the decoder uses the encoder output to generate + probabilities for the output sequence. + """ + + def __init__(self, params, train): + """Initialize layers to build Transformer model. + + Args: + params: hyperparameter object defining layer sizes, dropout values, etc. + train: boolean indicating whether the model is in training mode. Used to + determine if dropout layers should be added. + """ + self.train = train + self.params = params + + self.embedding_softmax_layer = embedding_layer.EmbeddingSharedWeights( + params["vocab_size"], params["hidden_size"], + method="matmul" if params["tpu"] else "gather") + self.encoder_stack = EncoderStack(params, train) + self.decoder_stack = DecoderStack(params, train) + + def __call__(self, inputs, targets=None): + """Calculate target logits or inferred target sequences. + + Args: + inputs: int tensor with shape [batch_size, input_length]. + targets: None or int tensor with shape [batch_size, target_length]. + + Returns: + If targets is defined, then return logits for each word in the target + sequence. float tensor with shape [batch_size, target_length, vocab_size] + If target is none, then generate output sequence one token at a time. + returns a dictionary { + output: [batch_size, decoded length] + score: [batch_size, float]} + """ + # Variance scaling is used here because it seems to work in many problems. + # Other reasonable initializers may also work just as well. + initializer = tf.variance_scaling_initializer( + self.params["initializer_gain"], mode="fan_avg", distribution="uniform") + with tf.variable_scope("Transformer", initializer=initializer): + # Calculate attention bias for encoder self-attention and decoder + # multi-headed attention layers. + attention_bias = model_utils.get_padding_bias(inputs) + + # Run the inputs through the encoder layer to map the symbol + # representations to continuous representations. + encoder_outputs = self.encode(inputs, attention_bias) + + # Generate output sequence if targets is None, or return logits if target + # sequence is known. + if targets is None: + return self.predict(encoder_outputs, attention_bias) + else: + logits = self.decode(targets, encoder_outputs, attention_bias) + return logits + + def encode(self, inputs, attention_bias): + """Generate continuous representation for inputs. + + Args: + inputs: int tensor with shape [batch_size, input_length]. + attention_bias: float tensor with shape [batch_size, 1, 1, input_length] + + Returns: + float tensor with shape [batch_size, input_length, hidden_size] + """ + with tf.name_scope("encode"): + # Prepare inputs to the layer stack by adding positional encodings and + # applying dropout. + embedded_inputs = self.embedding_softmax_layer(inputs) + inputs_padding = model_utils.get_padding(inputs) + + with tf.name_scope("add_pos_encoding"): + length = tf.shape(embedded_inputs)[1] + pos_encoding = model_utils.get_position_encoding( + length, self.params["hidden_size"]) + encoder_inputs = embedded_inputs + pos_encoding + + if self.train: + encoder_inputs = tf.nn.dropout( + encoder_inputs, 1 - self.params["layer_postprocess_dropout"]) + + return self.encoder_stack(encoder_inputs, attention_bias, inputs_padding) + + def decode(self, targets, encoder_outputs, attention_bias): + """Generate logits for each value in the target sequence. + + Args: + targets: target values for the output sequence. + int tensor with shape [batch_size, target_length] + encoder_outputs: continuous representation of input sequence. + float tensor with shape [batch_size, input_length, hidden_size] + attention_bias: float tensor with shape [batch_size, 1, 1, input_length] + + Returns: + float32 tensor with shape [batch_size, target_length, vocab_size] + """ + with tf.name_scope("decode"): + # Prepare inputs to decoder layers by shifting targets, adding positional + # encoding and applying dropout. + decoder_inputs = self.embedding_softmax_layer(targets) + with tf.name_scope("shift_targets"): + # Shift targets to the right, and remove the last element + decoder_inputs = tf.pad( + decoder_inputs, [[0, 0], [1, 0], [0, 0]])[:, :-1, :] + with tf.name_scope("add_pos_encoding"): + length = tf.shape(decoder_inputs)[1] + decoder_inputs += model_utils.get_position_encoding( + length, self.params["hidden_size"]) + if self.train: + decoder_inputs = tf.nn.dropout( + decoder_inputs, 1 - self.params["layer_postprocess_dropout"]) + + # Run values + decoder_self_attention_bias = model_utils.get_decoder_self_attention_bias( + length) + outputs = self.decoder_stack( + decoder_inputs, encoder_outputs, decoder_self_attention_bias, + attention_bias) + logits = self.embedding_softmax_layer.linear(outputs) + return logits + + def _get_symbols_to_logits_fn(self, max_decode_length): + """Returns a decoding function that calculates logits of the next tokens.""" + + timing_signal = model_utils.get_position_encoding( + max_decode_length + 1, self.params["hidden_size"]) + decoder_self_attention_bias = model_utils.get_decoder_self_attention_bias( + max_decode_length) + + def symbols_to_logits_fn(ids, i, cache): + """Generate logits for next potential IDs. + + Args: + ids: Current decoded sequences. + int tensor with shape [batch_size * beam_size, i + 1] + i: Loop index + cache: dictionary of values storing the encoder output, encoder-decoder + attention bias, and previous decoder attention values. + + Returns: + Tuple of + (logits with shape [batch_size * beam_size, vocab_size], + updated cache values) + """ + # Set decoder input to the last generated IDs + decoder_input = ids[:, -1:] + + # Preprocess decoder input by getting embeddings and adding timing signal. + decoder_input = self.embedding_softmax_layer(decoder_input) + decoder_input += timing_signal[i:i + 1] + + self_attention_bias = decoder_self_attention_bias[:, :, i:i + 1, :i + 1] + decoder_outputs = self.decoder_stack( + decoder_input, cache.get("encoder_outputs"), self_attention_bias, + cache.get("encoder_decoder_attention_bias"), cache) + logits = self.embedding_softmax_layer.linear(decoder_outputs) + logits = tf.squeeze(logits, axis=[1]) + return logits, cache + return symbols_to_logits_fn + + def predict(self, encoder_outputs, encoder_decoder_attention_bias): + """Return predicted sequence.""" + batch_size = tf.shape(encoder_outputs)[0] + input_length = tf.shape(encoder_outputs)[1] + max_decode_length = input_length + self.params["extra_decode_length"] + + symbols_to_logits_fn = self._get_symbols_to_logits_fn(max_decode_length) + + # Create initial set of IDs that will be passed into symbols_to_logits_fn. + initial_ids = tf.zeros([batch_size], dtype=tf.int32) + + # Create cache storing decoder attention values for each layer. + cache = { + "layer_%d" % layer: { + "k": tf.zeros([batch_size, 0, self.params["hidden_size"]]), + "v": tf.zeros([batch_size, 0, self.params["hidden_size"]]), + } for layer in range(self.params["num_hidden_layers"])} + + # Add encoder output and attention bias to the cache. + cache["encoder_outputs"] = encoder_outputs + cache["encoder_decoder_attention_bias"] = encoder_decoder_attention_bias + + # Use beam search to find the top beam_size sequences and scores. + decoded_ids, scores = beam_search.sequence_beam_search( + symbols_to_logits_fn=symbols_to_logits_fn, + initial_ids=initial_ids, + initial_cache=cache, + vocab_size=self.params["vocab_size"], + beam_size=self.params["beam_size"], + alpha=self.params["alpha"], + max_decode_length=max_decode_length, + eos_id=EOS_ID) + + # Get the top sequence for each batch element + top_decoded_ids = decoded_ids[:, 0, 1:] + top_scores = scores[:, 0] + + return {"outputs": top_decoded_ids, "scores": top_scores} + + +class LayerNormalization(tf.layers.Layer): + """Applies layer normalization.""" + + def __init__(self, hidden_size): + super(LayerNormalization, self).__init__() + self.hidden_size = hidden_size + + def build(self, _): + self.scale = tf.get_variable("layer_norm_scale", [self.hidden_size], + initializer=tf.ones_initializer()) + self.bias = tf.get_variable("layer_norm_bias", [self.hidden_size], + initializer=tf.zeros_initializer()) + self.built = True + + def call(self, x, epsilon=1e-6): + mean = tf.reduce_mean(x, axis=[-1], keepdims=True) + variance = tf.reduce_mean(tf.square(x - mean), axis=[-1], keepdims=True) + norm_x = (x - mean) * tf.rsqrt(variance + epsilon) + return norm_x * self.scale + self.bias + + +class PrePostProcessingWrapper(object): + """Wrapper class that applies layer pre-processing and post-processing.""" + + def __init__(self, layer, params, train): + self.layer = layer + self.postprocess_dropout = params["layer_postprocess_dropout"] + self.train = train + + # Create normalization layer + self.layer_norm = LayerNormalization(params["hidden_size"]) + + def __call__(self, x, *args, **kwargs): + # Preprocessing: apply layer normalization + y = self.layer_norm(x) + + # Get layer output + y = self.layer(y, *args, **kwargs) + + # Postprocessing: apply dropout and residual connection + if self.train: + y = tf.nn.dropout(y, 1 - self.postprocess_dropout) + return x + y + + +class EncoderStack(tf.layers.Layer): + """Transformer encoder stack. + + The encoder stack is made up of N identical layers. Each layer is composed + of the sublayers: + 1. Self-attention layer + 2. Feedforward network (which is 2 fully-connected layers) + """ + + def __init__(self, params, train): + super(EncoderStack, self).__init__() + self.layers = [] + for _ in range(params["num_hidden_layers"]): + # Create sublayers for each layer. + self_attention_layer = attention_layer.SelfAttention( + params["hidden_size"], params["num_heads"], + params["attention_dropout"], train) + feed_forward_network = ffn_layer.FeedFowardNetwork( + params["hidden_size"], params["filter_size"], + params["relu_dropout"], train, params["allow_ffn_pad"]) + + self.layers.append([ + PrePostProcessingWrapper(self_attention_layer, params, train), + PrePostProcessingWrapper(feed_forward_network, params, train)]) + + # Create final layer normalization layer. + self.output_normalization = LayerNormalization(params["hidden_size"]) + + def call(self, encoder_inputs, attention_bias, inputs_padding): + """Return the output of the encoder layer stacks. + + Args: + encoder_inputs: tensor with shape [batch_size, input_length, hidden_size] + attention_bias: bias for the encoder self-attention layer. + [batch_size, 1, 1, input_length] + inputs_padding: P + + Returns: + Output of encoder layer stack. + float32 tensor with shape [batch_size, input_length, hidden_size] + """ + for n, layer in enumerate(self.layers): + # Run inputs through the sublayers. + self_attention_layer = layer[0] + feed_forward_network = layer[1] + + with tf.variable_scope("layer_%d" % n): + with tf.variable_scope("self_attention"): + encoder_inputs = self_attention_layer(encoder_inputs, attention_bias) + with tf.variable_scope("ffn"): + encoder_inputs = feed_forward_network(encoder_inputs, inputs_padding) + + return self.output_normalization(encoder_inputs) + + +class DecoderStack(tf.layers.Layer): + """Transformer decoder stack. + + Like the encoder stack, the decoder stack is made up of N identical layers. + Each layer is composed of the sublayers: + 1. Self-attention layer + 2. Multi-headed attention layer combining encoder outputs with results from + the previous self-attention layer. + 3. Feedforward network (2 fully-connected layers) + """ + + def __init__(self, params, train): + super(DecoderStack, self).__init__() + self.layers = [] + for _ in range(params["num_hidden_layers"]): + self_attention_layer = attention_layer.SelfAttention( + params["hidden_size"], params["num_heads"], + params["attention_dropout"], train) + enc_dec_attention_layer = attention_layer.Attention( + params["hidden_size"], params["num_heads"], + params["attention_dropout"], train) + feed_forward_network = ffn_layer.FeedFowardNetwork( + params["hidden_size"], params["filter_size"], + params["relu_dropout"], train, params["allow_ffn_pad"]) + + self.layers.append([ + PrePostProcessingWrapper(self_attention_layer, params, train), + PrePostProcessingWrapper(enc_dec_attention_layer, params, train), + PrePostProcessingWrapper(feed_forward_network, params, train)]) + + self.output_normalization = LayerNormalization(params["hidden_size"]) + + def call(self, decoder_inputs, encoder_outputs, decoder_self_attention_bias, + attention_bias, cache=None): + """Return the output of the decoder layer stacks. + + Args: + decoder_inputs: tensor with shape [batch_size, target_length, hidden_size] + encoder_outputs: tensor with shape [batch_size, input_length, hidden_size] + decoder_self_attention_bias: bias for decoder self-attention layer. + [1, 1, target_len, target_length] + attention_bias: bias for encoder-decoder attention layer. + [batch_size, 1, 1, input_length] + cache: (Used for fast decoding) A nested dictionary storing previous + decoder self-attention values. The items are: + {layer_n: {"k": tensor with shape [batch_size, i, key_channels], + "v": tensor with shape [batch_size, i, value_channels]}, + ...} + + Returns: + Output of decoder layer stack. + float32 tensor with shape [batch_size, target_length, hidden_size] + """ + for n, layer in enumerate(self.layers): + self_attention_layer = layer[0] + enc_dec_attention_layer = layer[1] + feed_forward_network = layer[2] + + # Run inputs through the sublayers. + layer_name = "layer_%d" % n + layer_cache = cache[layer_name] if cache is not None else None + with tf.variable_scope(layer_name): + with tf.variable_scope("self_attention"): + decoder_inputs = self_attention_layer( + decoder_inputs, decoder_self_attention_bias, cache=layer_cache) + with tf.variable_scope("encdec_attention"): + decoder_inputs = enc_dec_attention_layer( + decoder_inputs, encoder_outputs, attention_bias) + with tf.variable_scope("ffn"): + decoder_inputs = feed_forward_network(decoder_inputs) + + return self.output_normalization(decoder_inputs) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/transformer_main.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/transformer_main.py new file mode 100644 index 0000000..f9ba2ca --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/transformer_main.py @@ -0,0 +1,710 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Train and evaluate the Transformer model. + +See README for description of setting the training schedule and evaluating the +BLEU score. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import tempfile + +# pylint: disable=g-bad-import-order +from six.moves import xrange # pylint: disable=redefined-builtin +from absl import app as absl_app +from absl import flags +import tensorflow.compat.v1 as tf +# pylint: enable=g-bad-import-order + +from official.nlp.transformer import model_params +from official.r1.utils import export +from official.r1.utils import tpu as tpu_util +from official.r1.transformer import translate +from official.r1.transformer import transformer +from official.r1.transformer import dataset +from official.r1.transformer import schedule +from official.nlp.transformer import compute_bleu +from official.nlp.transformer.utils import metrics +from official.nlp.transformer.utils import tokenizer +from official.utils.flags import core as flags_core +from official.utils.logs import hooks_helper +from official.utils.logs import logger +from official.utils.misc import distribution_utils +from official.utils.misc import model_helpers + +PARAMS_MAP = { + "tiny": model_params.TINY_PARAMS, + "base": model_params.BASE_PARAMS, + "big": model_params.BIG_PARAMS, +} + + +DEFAULT_TRAIN_EPOCHS = 10 +INF = 1000000000 # 1e9 +BLEU_DIR = "bleu" + +# Dictionary containing tensors that are logged by the logging hooks. Each item +# maps a string to the tensor name. +TENSORS_TO_LOG = { + "learning_rate": "model/get_train_op/learning_rate/learning_rate", + "cross_entropy_loss": "model/cross_entropy"} + + +def model_fn(features, labels, mode, params): + """Defines how to train, evaluate and predict from the transformer model.""" + with tf.variable_scope("model"): + inputs, targets = features, labels + + # Create model and get output logits. + model = transformer.Transformer(params, mode == tf.estimator.ModeKeys.TRAIN) + + logits = model(inputs, targets) + + # When in prediction mode, the labels/targets is None. The model output + # is the prediction + if mode == tf.estimator.ModeKeys.PREDICT: + if params["use_tpu"]: + raise NotImplementedError("Prediction is not yet supported on TPUs.") + return tf.estimator.EstimatorSpec( + tf.estimator.ModeKeys.PREDICT, + predictions=logits, + export_outputs={ + "translate": tf.estimator.export.PredictOutput(logits) + }) + + # Explicitly set the shape of the logits for XLA (TPU). This is needed + # because the logits are passed back to the host VM CPU for metric + # evaluation, and the shape of [?, ?, vocab_size] is too vague. However + # it is known from Transformer that the first two dimensions of logits + # are the dimensions of targets. Note that the ambiguous shape of logits is + # not a problem when computing xentropy, because padded_cross_entropy_loss + # resolves the shape on the TPU. + logits.set_shape(targets.shape.as_list() + logits.shape.as_list()[2:]) + + # Calculate model loss. + # xentropy contains the cross entropy loss of every nonpadding token in the + # targets. + xentropy, weights = metrics.padded_cross_entropy_loss( + logits, targets, params["label_smoothing"], params["vocab_size"]) + loss = tf.reduce_sum(xentropy) / tf.reduce_sum(weights) + + # Save loss as named tensor that will be logged with the logging hook. + tf.identity(loss, "cross_entropy") + + if mode == tf.estimator.ModeKeys.EVAL: + if params["use_tpu"]: + # host call functions should only have tensors as arguments. + # This lambda pre-populates params so that metric_fn is + # TPUEstimator compliant. + metric_fn = lambda logits, labels: ( + metrics.get_eval_metrics(logits, labels, params=params)) + eval_metrics = (metric_fn, [logits, labels]) + return tf.estimator.tpu.TPUEstimatorSpec( + mode=mode, + loss=loss, + predictions={"predictions": logits}, + eval_metrics=eval_metrics) + return tf.estimator.EstimatorSpec( + mode=mode, loss=loss, predictions={"predictions": logits}, + eval_metric_ops=metrics.get_eval_metrics(logits, labels, params)) + else: + train_op, metric_dict = get_train_op_and_metrics(loss, params) + + # Epochs can be quite long. This gives some intermediate information + # in TensorBoard. + metric_dict["minibatch_loss"] = loss + if params["use_tpu"]: + return tf.estimator.tpu.TPUEstimatorSpec( + mode=mode, + loss=loss, + train_op=train_op, + host_call=tpu_util.construct_scalar_host_call( + metric_dict=metric_dict, + model_dir=params["model_dir"], + prefix="training/")) + record_scalars(metric_dict) + return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op) + + +def record_scalars(metric_dict): + for key, value in metric_dict.items(): + tf.summary.scalar(name=key, tensor=value) + + +def get_learning_rate(learning_rate, hidden_size, learning_rate_warmup_steps): + """Calculate learning rate with linear warmup and rsqrt decay.""" + with tf.name_scope("learning_rate"): + warmup_steps = tf.to_float(learning_rate_warmup_steps) + step = tf.to_float(tf.train.get_or_create_global_step()) + + learning_rate *= (hidden_size ** -0.5) + # Apply linear warmup + learning_rate *= tf.minimum(1.0, step / warmup_steps) + # Apply rsqrt decay + learning_rate *= tf.rsqrt(tf.maximum(step, warmup_steps)) + + # Create a named tensor that will be logged using the logging hook. + # The full name includes variable and names scope. In this case, the name + # is model/get_train_op/learning_rate/learning_rate + tf.identity(learning_rate, "learning_rate") + + return learning_rate + + +def get_train_op_and_metrics(loss, params): + """Generate training op and metrics to save in TensorBoard.""" + with tf.variable_scope("get_train_op"): + learning_rate = get_learning_rate( + learning_rate=params["learning_rate"], + hidden_size=params["hidden_size"], + learning_rate_warmup_steps=params["learning_rate_warmup_steps"]) + + # Create optimizer. Use LazyAdamOptimizer from TF contrib, which is faster + # than the TF core Adam optimizer. + from tensorflow.contrib import opt as contrib_opt # pylint: disable=g-import-not-at-top + optimizer = contrib_opt.LazyAdamOptimizer( + learning_rate, + beta1=params["optimizer_adam_beta1"], + beta2=params["optimizer_adam_beta2"], + epsilon=params["optimizer_adam_epsilon"]) + + if params["use_tpu"] and params["tpu"] != tpu_util.LOCAL: + optimizer = tf.compat.v1.tpu.CrossShardOptimizer(optimizer) + + # Uses automatic mixed precision FP16 training if on GPU. + if params["dtype"] == "fp16": + optimizer = tf.train.experimental.enable_mixed_precision_graph_rewrite( + optimizer) + + # Calculate and apply gradients using LazyAdamOptimizer. + global_step = tf.train.get_global_step() + tvars = tf.trainable_variables() + gradients = optimizer.compute_gradients( + loss, tvars, colocate_gradients_with_ops=True) + minimize_op = optimizer.apply_gradients( + gradients, global_step=global_step, name="train") + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) + train_op = tf.group(minimize_op, update_ops) + + train_metrics = {"learning_rate": learning_rate} + + if not params["use_tpu"]: + # gradient norm is not included as a summary when running on TPU, as + # it can cause instability between the TPU and the host controller. + gradient_norm = tf.global_norm(list(zip(*gradients))[0]) + train_metrics["global_norm/gradient_norm"] = gradient_norm + + return train_op, train_metrics + + +def translate_and_compute_bleu(estimator, subtokenizer, bleu_source, bleu_ref): + """Translate file and report the cased and uncased bleu scores.""" + # Create temporary file to store translation. + tmp = tempfile.NamedTemporaryFile(delete=False) + tmp_filename = tmp.name + + translate.translate_file( + estimator, subtokenizer, bleu_source, output_file=tmp_filename, + print_all_translations=False) + + # Compute uncased and cased bleu scores. + uncased_score = compute_bleu.bleu_wrapper(bleu_ref, tmp_filename, False) + cased_score = compute_bleu.bleu_wrapper(bleu_ref, tmp_filename, True) + os.remove(tmp_filename) + return uncased_score, cased_score + + +def get_global_step(estimator): + """Return estimator's last checkpoint.""" + return int(estimator.latest_checkpoint().split("-")[-1]) + + +def evaluate_and_log_bleu(estimator, bleu_source, bleu_ref, vocab_file): + """Calculate and record the BLEU score.""" + subtokenizer = tokenizer.Subtokenizer(vocab_file) + + uncased_score, cased_score = translate_and_compute_bleu( + estimator, subtokenizer, bleu_source, bleu_ref) + + tf.logging.info("Bleu score (uncased): %f", uncased_score) + tf.logging.info("Bleu score (cased): %f", cased_score) + return uncased_score, cased_score + + +def _validate_file(filepath): + """Make sure that file exists.""" + if not tf.io.gfile.exists(filepath): + raise tf.errors.NotFoundError(None, None, "File %s not found." % filepath) + + +def run_loop( + estimator, schedule_manager, train_hooks=None, benchmark_logger=None, + bleu_source=None, bleu_ref=None, bleu_threshold=None, vocab_file=None): + """Train and evaluate model, and optionally compute model's BLEU score. + + **Step vs. Epoch vs. Iteration** + + Steps and epochs are canonical terms used in TensorFlow and general machine + learning. They are used to describe running a single process (train/eval): + - Step refers to running the process through a single or batch of examples. + - Epoch refers to running the process through an entire dataset. + + E.g. training a dataset with 100 examples. The dataset is + divided into 20 batches with 5 examples per batch. A single training step + trains the model on one batch. After 20 training steps, the model will have + trained on every batch in the dataset, or, in other words, one epoch. + + Meanwhile, iteration is used in this implementation to describe running + multiple processes (training and eval). + - A single iteration: + 1. trains the model for a specific number of steps or epochs. + 2. evaluates the model. + 3. (if source and ref files are provided) compute BLEU score. + + This function runs through multiple train+eval+bleu iterations. + + Args: + estimator: tf.Estimator containing model to train. + schedule_manager: A schedule.Manager object to guide the run loop. + train_hooks: List of hooks to pass to the estimator during training. + benchmark_logger: a BenchmarkLogger object that logs evaluation data + bleu_source: File containing text to be translated for BLEU calculation. + bleu_ref: File containing reference translations for BLEU calculation. + bleu_threshold: minimum BLEU score before training is stopped. + vocab_file: Path to vocab file that will be used to subtokenize bleu_source. + + Returns: + Dict of results of the run. Contains the keys `eval_results`, + `train_hooks`, `bleu_cased`, and `bleu_uncased`. `train_hooks` is a list the + instances of hooks used during training. + + Raises: + ValueError: if both or none of single_iteration_train_steps and + single_iteration_train_epochs were defined. + NotFoundError: if the vocab file or bleu files don't exist. + """ + if bleu_source: + _validate_file(bleu_source) + if bleu_ref: + _validate_file(bleu_ref) + if vocab_file: + _validate_file(vocab_file) + + evaluate_bleu = bleu_source is not None and bleu_ref is not None + if evaluate_bleu and schedule_manager.use_tpu: + raise ValueError("BLEU score can not be computed when training with a TPU, " + "as it requires estimator.predict which is not yet " + "supported.") + + # Print details of training schedule. + tf.logging.info("Training schedule:") + tf.logging.info( + "\t1. Train for {}".format(schedule_manager.train_increment_str)) + tf.logging.info("\t2. Evaluate model.") + if evaluate_bleu: + tf.logging.info("\t3. Compute BLEU score.") + if bleu_threshold is not None: + tf.logging.info("Repeat above steps until the BLEU score reaches %f" % + bleu_threshold) + if not evaluate_bleu or bleu_threshold is None: + tf.logging.info("Repeat above steps %d times." % + schedule_manager.train_eval_iterations) + + if evaluate_bleu: + # Create summary writer to log bleu score (values can be displayed in + # Tensorboard). + bleu_writer = tf.summary.FileWriter( + os.path.join(estimator.model_dir, BLEU_DIR)) + if bleu_threshold is not None: + # Change loop stopping condition if bleu_threshold is defined. + schedule_manager.train_eval_iterations = INF + + # Loop training/evaluation/bleu cycles + stats = {} + for i in xrange(schedule_manager.train_eval_iterations): + tf.logging.info("Starting iteration %d" % (i + 1)) + + # Train the model for single_iteration_train_steps or until the input fn + # runs out of examples (if single_iteration_train_steps is None). + estimator.train( + dataset.train_input_fn, + steps=schedule_manager.single_iteration_train_steps, + hooks=train_hooks) + + eval_results = None + eval_results = estimator.evaluate( + input_fn=dataset.eval_input_fn, + steps=schedule_manager.single_iteration_eval_steps) + + tf.logging.info("Evaluation results (iter %d/%d):" % + (i + 1, schedule_manager.train_eval_iterations)) + tf.logging.info(eval_results) + benchmark_logger.log_evaluation_result(eval_results) + + # The results from estimator.evaluate() are measured on an approximate + # translation, which utilize the target golden values provided. The actual + # bleu score must be computed using the estimator.predict() path, which + # outputs translations that are not based on golden values. The translations + # are compared to reference file to get the actual bleu score. + if evaluate_bleu: + uncased_score, cased_score = evaluate_and_log_bleu( + estimator, bleu_source, bleu_ref, vocab_file) + + stats["bleu_uncased"] = uncased_score + stats["bleu_cased"] = cased_score + + # Write actual bleu scores using summary writer and benchmark logger + global_step = get_global_step(estimator) + summary = tf.Summary(value=[ + tf.Summary.Value(tag="bleu/uncased", simple_value=uncased_score), + tf.Summary.Value(tag="bleu/cased", simple_value=cased_score), + ]) + bleu_writer.add_summary(summary, global_step) + bleu_writer.flush() + benchmark_logger.log_metric( + "bleu_uncased", uncased_score, global_step=global_step) + benchmark_logger.log_metric( + "bleu_cased", cased_score, global_step=global_step) + + # Stop training if bleu stopping threshold is met. + if model_helpers.past_stop_threshold(bleu_threshold, uncased_score): + bleu_writer.close() + break + + stats["eval_results"] = eval_results + stats["train_hooks"] = train_hooks + + return stats + + +def define_transformer_flags(): + """Add flags and flag validators for running transformer_main.""" + # Add common flags (data_dir, model_dir, train_epochs, etc.). + flags.DEFINE_integer( + name="max_length", short_name="ml", default=None, + help=flags_core.help_wrap("Max length.")) + + flags_core.define_base(clean=True, train_epochs=True, + epochs_between_evals=True, stop_threshold=True, + num_gpu=True, hooks=True, export_dir=True, + distribution_strategy=True) + flags_core.define_performance( + num_parallel_calls=True, + inter_op=False, + intra_op=False, + synthetic_data=True, + max_train_steps=False, + dtype=True, + all_reduce_alg=True + ) + flags_core.define_benchmark() + flags_core.define_device(tpu=True) + + # Set flags from the flags_core module as "key flags" so they're listed when + # the '-h' flag is used. Without this line, the flags defined above are + # only shown in the full `--helpful` help text. + flags.adopt_module_key_flags(flags_core) + + # Add transformer-specific flags + flags.DEFINE_enum( + name="param_set", short_name="mp", default="big", + enum_values=PARAMS_MAP.keys(), + help=flags_core.help_wrap( + "Parameter set to use when creating and training the model. The " + "parameters define the input shape (batch size and max length), " + "model configuration (size of embedding, # of hidden layers, etc.), " + "and various other settings. The big parameter set increases the " + "default batch size, embedding/hidden size, and filter size. For a " + "complete list of parameters, please see model/model_params.py.")) + + flags.DEFINE_bool( + name="static_batch", default=False, + help=flags_core.help_wrap( + "Whether the batches in the dataset should have static shapes. In " + "general, this setting should be False. Dynamic shapes allow the " + "inputs to be grouped so that the number of padding tokens is " + "minimized, and helps model training. In cases where the input shape " + "must be static (e.g. running on TPU), this setting will be ignored " + "and static batching will always be used.")) + + # Flags for training with steps (may be used for debugging) + flags.DEFINE_integer( + name="train_steps", short_name="ts", default=None, + help=flags_core.help_wrap("The number of steps used to train.")) + flags.DEFINE_integer( + name="steps_between_evals", short_name="sbe", default=1000, + help=flags_core.help_wrap( + "The Number of training steps to run between evaluations. This is " + "used if --train_steps is defined.")) + + # BLEU score computation + flags.DEFINE_string( + name="bleu_source", short_name="bls", default=None, + help=flags_core.help_wrap( + "Path to source file containing text translate when calculating the " + "official BLEU score. Both --bleu_source and --bleu_ref must be set. " + "Use the flag --stop_threshold to stop the script based on the " + "uncased BLEU score.")) + flags.DEFINE_string( + name="bleu_ref", short_name="blr", default=None, + help=flags_core.help_wrap( + "Path to source file containing text translate when calculating the " + "official BLEU score. Both --bleu_source and --bleu_ref must be set. " + "Use the flag --stop_threshold to stop the script based on the " + "uncased BLEU score.")) + flags.DEFINE_string( + name="vocab_file", short_name="vf", default=None, + help=flags_core.help_wrap( + "Path to subtoken vocabulary file. If data_download.py was used to " + "download and encode the training data, look in the data_dir to find " + "the vocab file.")) + + flags_core.set_defaults(data_dir="/tmp/translate_ende", + model_dir="/tmp/transformer_model", + batch_size=None, + train_epochs=None) + + @flags.multi_flags_validator( + ["train_epochs", "train_steps"], + message="Both --train_steps and --train_epochs were set. Only one may be " + "defined.") + def _check_train_limits(flag_dict): + return flag_dict["train_epochs"] is None or flag_dict["train_steps"] is None + + @flags.multi_flags_validator( + ["bleu_source", "bleu_ref"], + message="Both or neither --bleu_source and --bleu_ref must be defined.") + def _check_bleu_files(flags_dict): + return (flags_dict["bleu_source"] is None) == ( + flags_dict["bleu_ref"] is None) + + @flags.multi_flags_validator( + ["bleu_source", "bleu_ref", "vocab_file"], + message="--vocab_file must be defined if --bleu_source and --bleu_ref " + "are defined.") + def _check_bleu_vocab_file(flags_dict): + if flags_dict["bleu_source"] and flags_dict["bleu_ref"]: + return flags_dict["vocab_file"] is not None + return True + + @flags.multi_flags_validator( + ["export_dir", "vocab_file"], + message="--vocab_file must be defined if --export_dir is set.") + def _check_export_vocab_file(flags_dict): + if flags_dict["export_dir"]: + return flags_dict["vocab_file"] is not None + return True + + flags_core.require_cloud_storage(["data_dir", "model_dir", "export_dir"]) + + +def construct_estimator(flags_obj, params, schedule_manager): + """Construct an estimator from either Estimator or TPUEstimator. + + Args: + flags_obj: The FLAGS object parsed from command line. + params: A dict of run specific parameters. + schedule_manager: A schedule.Manager object containing the run schedule. + + Returns: + An estimator object to be used for training and eval. + """ + if not params["use_tpu"]: + distribution_strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=flags_obj.distribution_strategy, + num_gpus=flags_core.get_num_gpus(flags_obj), + all_reduce_alg=flags_obj.all_reduce_alg) + return tf.estimator.Estimator( + model_fn=model_fn, model_dir=flags_obj.model_dir, params=params, + config=tf.estimator.RunConfig(train_distribute=distribution_strategy)) + + tpu_cluster_resolver = tf.compat.v1.cluster_resolver.TPUClusterResolver( + tpu=flags_obj.tpu, + zone=flags_obj.tpu_zone, + project=flags_obj.tpu_gcp_project + ) + + tpu_config = tf.estimator.tpu.TPUConfig( + iterations_per_loop=schedule_manager.single_iteration_train_steps, + num_shards=flags_obj.num_tpu_shards) + + run_config = tf.estimator.tpu.RunConfig( + cluster=tpu_cluster_resolver, + model_dir=flags_obj.model_dir, + session_config=tf.ConfigProto( + allow_soft_placement=True, log_device_placement=True), + tpu_config=tpu_config) + + return tf.estimator.tpu.TPUEstimator( + model_fn=model_fn, + use_tpu=params["use_tpu"] and flags_obj.tpu != tpu_util.LOCAL, + train_batch_size=schedule_manager.batch_size, + eval_batch_size=schedule_manager.batch_size, + params={ + # TPUEstimator needs to populate batch_size itself due to sharding. + key: value for key, value in params.items() if key != "batch_size" + }, + config=run_config) + +def per_replica_batch_size(batch_size, num_gpus): + """For multi-gpu, batch-size must be a multiple of the number of GPUs. + + + Note that distribution strategy handles this automatically when used with + Keras. For using with Estimator, we need to get per GPU batch. + + Args: + batch_size: Global batch size to be divided among devices. This should be + equal to num_gpus times the single-GPU batch_size for multi-gpu training. + num_gpus: How many GPUs are used with DistributionStrategies. + + Returns: + Batch size per device. + + Raises: + ValueError: if batch_size is not divisible by number of devices + """ + if num_gpus <= 1: + return batch_size + + remainder = batch_size % num_gpus + if remainder: + err = ('When running with multiple GPUs, batch size ' + 'must be a multiple of the number of available GPUs. Found {} ' + 'GPUs with a batch size of {}; try --batch_size={} instead.' + ).format(num_gpus, batch_size, batch_size - remainder) + raise ValueError(err) + return int(batch_size / num_gpus) + + +def run_transformer(flags_obj): + """Create tf.Estimator to train and evaluate transformer model. + + Args: + flags_obj: Object containing parsed flag values. + + Returns: + Dict of results of the run. Contains the keys `eval_results`, + `train_hooks`, `bleu_cased`, and `bleu_uncased`. `train_hooks` is a list the + instances of hooks used during training. + """ + num_gpus = flags_core.get_num_gpus(flags_obj) + + # Add flag-defined parameters to params object + params = PARAMS_MAP[flags_obj.param_set] + if num_gpus > 1: + if flags_obj.param_set == "big": + params = model_params.BIG_MULTI_GPU_PARAMS + elif flags_obj.param_set == "base": + params = model_params.BASE_MULTI_GPU_PARAMS + + params["data_dir"] = flags_obj.data_dir + params["model_dir"] = flags_obj.model_dir + params["num_parallel_calls"] = flags_obj.num_parallel_calls + + params["tpu"] = flags_obj.tpu + params["use_tpu"] = bool(flags_obj.tpu) # was a tpu specified. + params["static_batch"] = flags_obj.static_batch or params["use_tpu"] + params["allow_ffn_pad"] = not params["use_tpu"] + + params["max_length"] = flags_obj.max_length or params["max_length"] + + params["use_synthetic_data"] = flags_obj.use_synthetic_data + + # Set batch size parameter, which depends on the availability of + # TPU and GPU, and distribution settings. + params["batch_size"] = (flags_obj.batch_size or ( + params["default_batch_size_tpu"] if params["use_tpu"] + else params["default_batch_size"])) + + total_batch_size = params["batch_size"] + if not params["use_tpu"]: + params["batch_size"] = per_replica_batch_size(params["batch_size"], + num_gpus) + + schedule_manager = schedule.Manager( + train_steps=flags_obj.train_steps, + steps_between_evals=flags_obj.steps_between_evals, + train_epochs=flags_obj.train_epochs, + epochs_between_evals=flags_obj.epochs_between_evals, + default_train_epochs=DEFAULT_TRAIN_EPOCHS, + batch_size=params["batch_size"], + max_length=params["max_length"], + use_tpu=params["use_tpu"], + num_tpu_shards=flags_obj.num_tpu_shards + ) + + params["repeat_dataset"] = schedule_manager.repeat_dataset + + model_helpers.apply_clean(flags.FLAGS) + + # Create hooks that log information about the training and metric values + train_hooks = hooks_helper.get_train_hooks( + flags_obj.hooks, + model_dir=flags_obj.model_dir, + tensors_to_log=TENSORS_TO_LOG, # used for logging hooks + batch_size=total_batch_size, # for ExamplesPerSecondHook + use_tpu=params["use_tpu"] # Not all hooks can run with TPUs + ) + benchmark_logger = logger.get_benchmark_logger() + benchmark_logger.log_run_info( + model_name="transformer", + dataset_name="wmt_translate_ende", + run_params=params, + test_id=flags_obj.benchmark_test_id) + + # Train and evaluate transformer model + estimator = construct_estimator(flags_obj, params, schedule_manager) + stats = run_loop( + estimator=estimator, + # Training arguments + schedule_manager=schedule_manager, + train_hooks=train_hooks, + benchmark_logger=benchmark_logger, + # BLEU calculation arguments + bleu_source=flags_obj.bleu_source, + bleu_ref=flags_obj.bleu_ref, + bleu_threshold=flags_obj.stop_threshold, + vocab_file=flags_obj.vocab_file) + + if flags_obj.export_dir and not params["use_tpu"]: + serving_input_fn = export.build_tensor_serving_input_receiver_fn( + shape=[None], dtype=tf.int64, batch_size=None) + # Export saved model, and save the vocab file as an extra asset. The vocab + # file is saved to allow consistent input encoding and output decoding. + # (See the "Export trained model" section in the README for an example of + # how to use the vocab file.) + # Since the model itself does not use the vocab file, this file is saved as + # an extra asset rather than a core asset. + estimator.export_savedmodel( + flags_obj.export_dir, serving_input_fn, + assets_extra={"vocab.txt": flags_obj.vocab_file}, + strip_default_attrs=True) + return stats + + +def main(_): + with logger.benchmark_context(flags.FLAGS): + run_transformer(flags.FLAGS) + + +if __name__ == "__main__": + tf.logging.set_verbosity(tf.logging.INFO) + define_transformer_flags() + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/translate.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/translate.py new file mode 100644 index 0000000..9912ee3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/transformer/translate.py @@ -0,0 +1,237 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Translate text or files using trained transformer model.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +# pylint: disable=g-bad-import-order +from absl import app as absl_app +from absl import flags +import tensorflow.compat.v1 as tf +# pylint: enable=g-bad-import-order + +from official.nlp.transformer.utils import tokenizer +from official.utils.flags import core as flags_core + +_DECODE_BATCH_SIZE = 32 +_EXTRA_DECODE_LENGTH = 100 +_BEAM_SIZE = 4 +_ALPHA = 0.6 + + +def _get_sorted_inputs(filename): + """Read and sort lines from the file sorted by decreasing length. + + Args: + filename: String name of file to read inputs from. + Returns: + Sorted list of inputs, and dictionary mapping original index->sorted index + of each element. + """ + with tf.io.gfile.GFile(filename) as f: + records = f.read().split("\n") + inputs = [record.strip() for record in records] + if not inputs[-1]: + inputs.pop() + + input_lens = [(i, len(line.split())) for i, line in enumerate(inputs)] + sorted_input_lens = sorted(input_lens, key=lambda x: x[1], reverse=True) + + sorted_inputs = [None] * len(sorted_input_lens) + sorted_keys = [0] * len(sorted_input_lens) + for i, (index, _) in enumerate(sorted_input_lens): + sorted_inputs[i] = inputs[index] + sorted_keys[index] = i + + return sorted_inputs, sorted_keys + + +def _encode_and_add_eos(line, subtokenizer): + """Encode line with subtokenizer, and add EOS id to the end.""" + return subtokenizer.encode(line) + [tokenizer.EOS_ID] + + +def _trim_and_decode(ids, subtokenizer): + """Trim EOS and PAD tokens from ids, and decode to return a string.""" + try: + index = list(ids).index(tokenizer.EOS_ID) + return subtokenizer.decode(ids[:index]) + except ValueError: # No EOS found in sequence + return subtokenizer.decode(ids) + + +def translate_file( + estimator, subtokenizer, input_file, output_file=None, + print_all_translations=True): + """Translate lines in file, and save to output file if specified. + + Args: + estimator: tf.Estimator used to generate the translations. + subtokenizer: Subtokenizer object for encoding and decoding source and + translated lines. + input_file: file containing lines to translate + output_file: file that stores the generated translations. + print_all_translations: If true, all translations are printed to stdout. + + Raises: + ValueError: if output file is invalid. + """ + batch_size = _DECODE_BATCH_SIZE + + # Read and sort inputs by length. Keep dictionary (original index-->new index + # in sorted list) to write translations in the original order. + sorted_inputs, sorted_keys = _get_sorted_inputs(input_file) + num_decode_batches = (len(sorted_inputs) - 1) // batch_size + 1 + + def input_generator(): + """Yield encoded strings from sorted_inputs.""" + for i, line in enumerate(sorted_inputs): + if i % batch_size == 0: + batch_num = (i // batch_size) + 1 + + tf.logging.info("Decoding batch %d out of %d." % + (batch_num, num_decode_batches)) + yield _encode_and_add_eos(line, subtokenizer) + + def input_fn(): + """Created batched dataset of encoded inputs.""" + ds = tf.data.Dataset.from_generator( + input_generator, tf.int64, tf.TensorShape([None])) + ds = ds.padded_batch(batch_size, [None]) + return ds + + translations = [] + for i, prediction in enumerate(estimator.predict(input_fn)): + translation = _trim_and_decode(prediction["outputs"], subtokenizer) + translations.append(translation) + + if print_all_translations: + tf.logging.info("Translating:\n\tInput: %s\n\tOutput: %s" % + (sorted_inputs[i], translation)) + + # Write translations in the order they appeared in the original file. + if output_file is not None: + if tf.io.gfile.isdir(output_file): + raise ValueError("File output is a directory, will not save outputs to " + "file.") + tf.logging.info("Writing to file %s" % output_file) + with tf.io.gfile.GFile(output_file, "w") as f: + for i in sorted_keys: + f.write("%s\n" % translations[i]) + + +def translate_text(estimator, subtokenizer, txt): + """Translate a single string.""" + encoded_txt = _encode_and_add_eos(txt, subtokenizer) + + def input_fn(): + ds = tf.data.Dataset.from_tensors(encoded_txt) + ds = ds.batch(_DECODE_BATCH_SIZE) + return ds + + predictions = estimator.predict(input_fn) + translation = next(predictions)["outputs"] + translation = _trim_and_decode(translation, subtokenizer) + tf.logging.info("Translation of \"%s\": \"%s\"" % (txt, translation)) + + +def main(unused_argv): + from official.transformer import transformer_main + + tf.logging.set_verbosity(tf.logging.INFO) + + if FLAGS.text is None and FLAGS.file is None: + tf.logging.warn("Nothing to translate. Make sure to call this script using " + "flags --text or --file.") + return + + subtokenizer = tokenizer.Subtokenizer(FLAGS.vocab_file) + + # Set up estimator and params + params = transformer_main.PARAMS_MAP[FLAGS.param_set] + params["beam_size"] = _BEAM_SIZE + params["alpha"] = _ALPHA + params["extra_decode_length"] = _EXTRA_DECODE_LENGTH + params["batch_size"] = _DECODE_BATCH_SIZE + estimator = tf.estimator.Estimator( + model_fn=transformer_main.model_fn, model_dir=FLAGS.model_dir, + params=params) + + if FLAGS.text is not None: + tf.logging.info("Translating text: %s" % FLAGS.text) + translate_text(estimator, subtokenizer, FLAGS.text) + + if FLAGS.file is not None: + input_file = os.path.abspath(FLAGS.file) + tf.logging.info("Translating file: %s" % input_file) + if not tf.gfile.Exists(FLAGS.file): + raise ValueError("File does not exist: %s" % input_file) + + output_file = None + if FLAGS.file_out is not None: + output_file = os.path.abspath(FLAGS.file_out) + tf.logging.info("File output specified: %s" % output_file) + + translate_file(estimator, subtokenizer, input_file, output_file) + + +def define_translate_flags(): + """Define flags used for translation script.""" + # Model flags + flags.DEFINE_string( + name="model_dir", short_name="md", default="/tmp/transformer_model", + help=flags_core.help_wrap( + "Directory containing Transformer model checkpoints.")) + flags.DEFINE_enum( + name="param_set", short_name="mp", default="big", + enum_values=["base", "big"], + help=flags_core.help_wrap( + "Parameter set to use when creating and training the model. The " + "parameters define the input shape (batch size and max length), " + "model configuration (size of embedding, # of hidden layers, etc.), " + "and various other settings. The big parameter set increases the " + "default batch size, embedding/hidden size, and filter size. For a " + "complete list of parameters, please see model/model_params.py.")) + flags.DEFINE_string( + name="vocab_file", short_name="vf", default=None, + help=flags_core.help_wrap( + "Path to subtoken vocabulary file. If data_download.py was used to " + "download and encode the training data, look in the data_dir to find " + "the vocab file.")) + flags.mark_flag_as_required("vocab_file") + + flags.DEFINE_string( + name="text", default=None, + help=flags_core.help_wrap( + "Text to translate. Output will be printed to console.")) + flags.DEFINE_string( + name="file", default=None, + help=flags_core.help_wrap( + "File containing text to translate. Translation will be printed to " + "console and, if --file_out is provided, saved to an output file.")) + flags.DEFINE_string( + name="file_out", default=None, + help=flags_core.help_wrap( + "If --file flag is specified, save translation to this file.")) + + +if __name__ == "__main__": + define_translate_flags() + FLAGS = flags.FLAGS + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/data/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/data/file_io.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/data/file_io.py new file mode 100644 index 0000000..b7776fc --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/data/file_io.py @@ -0,0 +1,207 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Convenience functions for managing dataset file buffers.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import atexit +import multiprocessing +import multiprocessing.dummy +import os +import tempfile +import uuid + +from absl import logging +import numpy as np +import six +import tensorflow as tf +# pylint:disable=logging-format-interpolation + + +class _GarbageCollector(object): + """Deletes temporary buffer files at exit. + + Certain tasks (such as NCF Recommendation) require writing buffers to + temporary files. (Which may be local or distributed.) It is not generally safe + to delete these files during operation, but they should be cleaned up. This + class keeps track of temporary files created, and deletes them at exit. + """ + def __init__(self): + self.temp_buffers = [] + + def register(self, filepath): + self.temp_buffers.append(filepath) + + def purge(self): + try: + for i in self.temp_buffers: + if tf.io.gfile.exists(i): + tf.io.gfile.remove(i) + logging.info("Buffer file {} removed".format(i)) + except Exception as e: + logging.error("Failed to cleanup buffer files: {}".format(e)) + + +_GARBAGE_COLLECTOR = _GarbageCollector() +atexit.register(_GARBAGE_COLLECTOR.purge) + +_ROWS_PER_CORE = 50000 + + +def write_to_temp_buffer(dataframe, buffer_folder, columns): + if buffer_folder is None: + _, buffer_path = tempfile.mkstemp() + else: + tf.io.gfile.makedirs(buffer_folder) + buffer_path = os.path.join(buffer_folder, str(uuid.uuid4())) + _GARBAGE_COLLECTOR.register(buffer_path) + + return write_to_buffer(dataframe, buffer_path, columns) + + +def iter_shard_dataframe(df, rows_per_core=1000): + """Two way shard of a dataframe. + + This function evenly shards a dataframe so that it can be mapped efficiently. + It yields a list of dataframes with length equal to the number of CPU cores, + with each dataframe having rows_per_core rows. (Except for the last batch + which may have fewer rows in the dataframes.) Passing vectorized inputs to + a pool is more effecient than iterating through a dataframe in serial and + passing a list of inputs to the pool. + + Args: + df: Pandas dataframe to be sharded. + rows_per_core: Number of rows in each shard. + + Returns: + A list of dataframe shards. + """ + n = len(df) + num_cores = min([multiprocessing.cpu_count(), n]) + + num_blocks = int(np.ceil(n / num_cores / rows_per_core)) + max_batch_size = num_cores * rows_per_core + for i in range(num_blocks): + min_index = i * max_batch_size + max_index = min([(i + 1) * max_batch_size, n]) + df_shard = df[min_index:max_index] + n_shard = len(df_shard) + boundaries = np.linspace(0, n_shard, num_cores + 1, dtype=np.int64) + yield [df_shard[boundaries[j]:boundaries[j+1]] for j in range(num_cores)] + + +def _shard_dict_to_examples(shard_dict): + """Converts a dict of arrays into a list of example bytes.""" + n = [i for i in shard_dict.values()][0].shape[0] + feature_list = [{} for _ in range(n)] + for column, values in shard_dict.items(): + if len(values.shape) == 1: + values = np.reshape(values, values.shape + (1,)) + + if values.dtype.kind == "i": + feature_map = lambda x: tf.train.Feature( + int64_list=tf.train.Int64List(value=x)) + elif values.dtype.kind == "f": + feature_map = lambda x: tf.train.Feature( + float_list=tf.train.FloatList(value=x)) + else: + raise ValueError("Invalid dtype") + for i in range(n): + feature_list[i][column] = feature_map(values[i]) + examples = [ + tf.train.Example(features=tf.train.Features(feature=example_features)) + for example_features in feature_list + ] + + return [e.SerializeToString() for e in examples] + + +def _serialize_shards(df_shards, columns, pool, writer): + """Map sharded dataframes to bytes, and write them to a buffer. + + Args: + df_shards: A list of pandas dataframes. (Should be of similar size) + columns: The dataframe columns to be serialized. + pool: A pool to serialize in parallel. + writer: A TFRecordWriter to write the serialized shards. + """ + # Pandas does not store columns of arrays as nd arrays. stack remedies this. + map_inputs = [{c: np.stack(shard[c].values, axis=0) for c in columns} + for shard in df_shards] + + # Failure within pools is very irksome. Thus, it is better to thoroughly check + # inputs in the main process. + for inp in map_inputs: + # Check that all fields have the same number of rows. + assert len(set([v.shape[0] for v in inp.values()])) == 1 + for val in inp.values(): + assert hasattr(val, "dtype") + assert hasattr(val.dtype, "kind") + assert val.dtype.kind in ("i", "f") + assert len(val.shape) in (1, 2) + shard_bytes = pool.map(_shard_dict_to_examples, map_inputs) + for s in shard_bytes: + for example in s: + writer.write(example) + + +def write_to_buffer(dataframe, buffer_path, columns, expected_size=None): + """Write a dataframe to a binary file for a dataset to consume. + + Args: + dataframe: The pandas dataframe to be serialized. + buffer_path: The path where the serialized results will be written. + columns: The dataframe columns to be serialized. + expected_size: The size in bytes of the serialized results. This is used to + lazily construct the buffer. + + Returns: + The path of the buffer. + """ + if (tf.io.gfile.exists(buffer_path) and + tf.io.gfile.stat(buffer_path).length > 0): + actual_size = tf.io.gfile.stat(buffer_path).length + if expected_size == actual_size: + return buffer_path + logging.warning( + "Existing buffer {} has size {}. Expected size {}. Deleting and " + "rebuilding buffer.".format(buffer_path, actual_size, expected_size)) + tf.io.gfile.remove(buffer_path) + + if dataframe is None: + raise ValueError( + "dataframe was None but a valid existing buffer was not found.") + + tf.io.gfile.makedirs(os.path.split(buffer_path)[0]) + + logging.info("Constructing TFRecordDataset buffer: {}".format(buffer_path)) + + count = 0 + pool = multiprocessing.dummy.Pool(multiprocessing.cpu_count()) + try: + with tf.io.TFRecordWriter(buffer_path) as writer: + for df_shards in iter_shard_dataframe(df=dataframe, + rows_per_core=_ROWS_PER_CORE): + _serialize_shards(df_shards, columns, pool, writer) + count += sum([len(s) for s in df_shards]) + logging.info("{}/{} examples written.".format( + str(count).ljust(8), len(dataframe))) + finally: + pool.terminate() + + logging.info("Buffer write complete.") + return buffer_path diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/data/file_io_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/data/file_io_test.py new file mode 100644 index 0000000..ba90d94 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/data/file_io_test.py @@ -0,0 +1,199 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for binary data file utilities.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import contextlib +import multiprocessing + +# pylint: disable=wrong-import-order +import numpy as np +import pandas as pd +import tensorflow as tf +# pylint: enable=wrong-import-order + +from official.r1.utils.data import file_io +from official.utils.misc import keras_utils + + +_RAW_ROW = "raw_row" +_DUMMY_COL = "column_0" +_DUMMY_VEC_COL = "column_1" +_DUMMY_VEC_LEN = 4 + +_ROWS_PER_CORE = 4 +_TEST_CASES = [ + # One batch of one + dict(row_count=1, cpu_count=1, expected=[ + [[0]] + ]), + + dict(row_count=10, cpu_count=1, expected=[ + [[0, 1, 2, 3]], [[4, 5, 6, 7]], [[8, 9]] + ]), + + dict(row_count=21, cpu_count=1, expected=[ + [[0, 1, 2, 3]], [[4, 5, 6, 7]], [[8, 9, 10, 11]], + [[12, 13, 14, 15]], [[16, 17, 18, 19]], [[20]] + ]), + + dict(row_count=1, cpu_count=4, expected=[ + [[0]] + ]), + + dict(row_count=10, cpu_count=4, expected=[ + [[0, 1], [2, 3, 4], [5, 6], [7, 8, 9]] + ]), + + dict(row_count=21, cpu_count=4, expected=[ + [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11], [12, 13, 14, 15]], + [[16], [17], [18], [19, 20]] + ]), + + dict(row_count=10, cpu_count=8, expected=[ + [[0], [1], [2], [3, 4], [5], [6], [7], [8, 9]] + ]), + + dict(row_count=40, cpu_count=8, expected=[ + [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11], [12, 13, 14, 15], + [16, 17, 18, 19], [20, 21, 22, 23], [24, 25, 26, 27], + [28, 29, 30, 31]], + [[32], [33], [34], [35], [36], [37], [38], [39]] + ]), +] + +_FEATURE_MAP = { + _RAW_ROW: tf.io.FixedLenFeature([1], dtype=tf.int64), + _DUMMY_COL: tf.io.FixedLenFeature([1], dtype=tf.int64), + _DUMMY_VEC_COL: tf.io.FixedLenFeature([_DUMMY_VEC_LEN], dtype=tf.float32) +} + + +@contextlib.contextmanager +def fixed_core_count(cpu_count): + """Override CPU count. + + file_io.py uses the cpu_count function to scale to the size of the instance. + However, this is not desirable for testing because it can make the test flaky. + Instead, this context manager fixes the count for more robust testing. + + Args: + cpu_count: How many cores multiprocessing claims to have. + + Yields: + Nothing. (for context manager only) + """ + old_count_fn = multiprocessing.cpu_count + multiprocessing.cpu_count = lambda: cpu_count + yield + multiprocessing.cpu_count = old_count_fn + + +class BaseTest(tf.test.TestCase): + + def setUp(self): + super(BaseTest, self).setUp() + if keras_utils.is_v2_0: + tf.compat.v1.disable_eager_execution() + + def _test_sharding(self, row_count, cpu_count, expected): + df = pd.DataFrame({_DUMMY_COL: list(range(row_count))}) + with fixed_core_count(cpu_count): + shards = list(file_io.iter_shard_dataframe(df, _ROWS_PER_CORE)) + result = [[j[_DUMMY_COL].tolist() for j in i] for i in shards] + self.assertAllEqual(expected, result) + + def test_tiny_rows_low_core(self): + self._test_sharding(**_TEST_CASES[0]) + + def test_small_rows_low_core(self): + self._test_sharding(**_TEST_CASES[1]) + + def test_large_rows_low_core(self): + self._test_sharding(**_TEST_CASES[2]) + + def test_tiny_rows_medium_core(self): + self._test_sharding(**_TEST_CASES[3]) + + def test_small_rows_medium_core(self): + self._test_sharding(**_TEST_CASES[4]) + + def test_large_rows_medium_core(self): + self._test_sharding(**_TEST_CASES[5]) + + def test_small_rows_large_core(self): + self._test_sharding(**_TEST_CASES[6]) + + def test_large_rows_large_core(self): + self._test_sharding(**_TEST_CASES[7]) + + def _serialize_deserialize(self, num_cores=1, num_rows=20): + np.random.seed(1) + df = pd.DataFrame({ + # Serialization order is only deterministic for num_cores=1. raw_row is + # used in validation after the deserialization. + _RAW_ROW: np.array(range(num_rows), dtype=np.int64), + _DUMMY_COL: np.random.randint(0, 35, size=(num_rows,)), + _DUMMY_VEC_COL: [ + np.array([np.random.random() for _ in range(_DUMMY_VEC_LEN)]) + for i in range(num_rows) # pylint: disable=unused-variable + ] + }) + + with fixed_core_count(num_cores): + buffer_path = file_io.write_to_temp_buffer( + df, self.get_temp_dir(), [_RAW_ROW, _DUMMY_COL, _DUMMY_VEC_COL]) + + with self.session(graph=tf.Graph()) as sess: + dataset = tf.data.TFRecordDataset(buffer_path) + dataset = dataset.batch(1).map( + lambda x: tf.io.parse_example(serialized=x, features=_FEATURE_MAP)) + + data_iter = tf.compat.v1.data.make_one_shot_iterator(dataset) + seen_rows = set() + for i in range(num_rows+5): + row = data_iter.get_next() + try: + row_id, val_0, val_1 = sess.run( + [row[_RAW_ROW], row[_DUMMY_COL], row[_DUMMY_VEC_COL]]) + row_id, val_0, val_1 = row_id[0][0], val_0[0][0], val_1[0] + assert row_id not in seen_rows + seen_rows.add(row_id) + + self.assertEqual(val_0, df[_DUMMY_COL][row_id]) + self.assertAllClose(val_1, df[_DUMMY_VEC_COL][row_id]) + + self.assertLess(i, num_rows, msg="Too many rows.") + except tf.errors.OutOfRangeError: + self.assertGreaterEqual(i, num_rows, msg="Too few rows.") + + file_io._GARBAGE_COLLECTOR.purge() + assert not tf.io.gfile.exists(buffer_path) + + def test_serialize_deserialize_0(self): + self._serialize_deserialize(num_cores=1) + + def test_serialize_deserialize_1(self): + self._serialize_deserialize(num_cores=2) + + def test_serialize_deserialize_2(self): + self._serialize_deserialize(num_cores=8) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/export.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/export.py new file mode 100644 index 0000000..8061c28 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/export.py @@ -0,0 +1,49 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Convenience functions for exporting models as SavedModels or other types.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + + +def build_tensor_serving_input_receiver_fn(shape, dtype=tf.float32, + batch_size=1): + """Returns a input_receiver_fn that can be used during serving. + + This expects examples to come through as float tensors, and simply + wraps them as TensorServingInputReceivers. + + Arguably, this should live in tf.estimator.export. Testing here first. + + Args: + shape: list representing target size of a single example. + dtype: the expected datatype for the input example + batch_size: number of input tensors that will be passed for prediction + + Returns: + A function that itself returns a TensorServingInputReceiver. + """ + def serving_input_receiver_fn(): + # Prep a placeholder where the input example will be fed in + features = tf.compat.v1.placeholder( + dtype=dtype, shape=[batch_size] + shape, name='input_tensor') + + return tf.estimator.export.TensorServingInputReceiver( + features=features, receiver_tensors=features) + + return serving_input_receiver_fn diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/export_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/export_test.py new file mode 100644 index 0000000..3785edd --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/export_test.py @@ -0,0 +1,63 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for exporting utils.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.r1.utils import export + + +class ExportUtilsTest(tf.test.TestCase): + """Tests for the ExportUtils.""" + + def test_build_tensor_serving_input_receiver_fn(self): + receiver_fn = export.build_tensor_serving_input_receiver_fn(shape=[4, 5]) + with tf.Graph().as_default(): + receiver = receiver_fn() + self.assertIsInstance( + receiver, tf.estimator.export.TensorServingInputReceiver) + + self.assertIsInstance(receiver.features, tf.Tensor) + self.assertEqual(receiver.features.shape, tf.TensorShape([1, 4, 5])) + self.assertEqual(receiver.features.dtype, tf.float32) + self.assertIsInstance(receiver.receiver_tensors, dict) + # Note that Python 3 can no longer index .values() directly; cast to list. + self.assertEqual(list(receiver.receiver_tensors.values())[0].shape, + tf.TensorShape([1, 4, 5])) + + def test_build_tensor_serving_input_receiver_fn_batch_dtype(self): + receiver_fn = export.build_tensor_serving_input_receiver_fn( + shape=[4, 5], dtype=tf.int8, batch_size=10) + + with tf.Graph().as_default(): + receiver = receiver_fn() + self.assertIsInstance( + receiver, tf.estimator.export.TensorServingInputReceiver) + + self.assertIsInstance(receiver.features, tf.Tensor) + self.assertEqual(receiver.features.shape, tf.TensorShape([10, 4, 5])) + self.assertEqual(receiver.features.dtype, tf.int8) + self.assertIsInstance(receiver.receiver_tensors, dict) + # Note that Python 3 can no longer index .values() directly; cast to list. + self.assertEqual(list(receiver.receiver_tensors.values())[0].shape, + tf.TensorShape([10, 4, 5])) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/tpu.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/tpu.py new file mode 100644 index 0000000..737a794 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/tpu.py @@ -0,0 +1,116 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functions specific to running TensorFlow on TPUs.""" + +import tensorflow as tf + + +# "local" is a magic word in the TPU cluster resolver; it informs the resolver +# to use the local CPU as the compute device. This is useful for testing and +# debugging; the code flow is ostensibly identical, but without the need to +# actually have a TPU on the other end. +LOCAL = "local" + + +def construct_scalar_host_call(metric_dict, model_dir, prefix=""): + """Construct a host call to log scalars when training on TPU. + + Args: + metric_dict: A dict of the tensors to be logged. + model_dir: The location to write the summary. + prefix: The prefix (if any) to prepend to the metric names. + + Returns: + A tuple of (function, args_to_be_passed_to_said_function) + """ + # type: (dict, str) -> (function, list) + metric_names = list(metric_dict.keys()) + + def host_call_fn(global_step, *args): + """Training host call. Creates scalar summaries for training metrics. + + This function is executed on the CPU and should not directly reference + any Tensors in the rest of the `model_fn`. To pass Tensors from the + model to the `metric_fn`, provide as part of the `host_call`. See + https://www.tensorflow.org/api_docs/python/tf/contrib/tpu/TPUEstimatorSpec + for more information. + + Arguments should match the list of `Tensor` objects passed as the second + element in the tuple passed to `host_call`. + + Args: + global_step: `Tensor with shape `[batch]` for the global_step + *args: Remaining tensors to log. + + Returns: + List of summary ops to run on the CPU host. + """ + step = global_step[0] + with tf.compat.v1.summary.create_file_writer( + logdir=model_dir, filename_suffix=".host_call").as_default(): + with tf.compat.v1.summary.always_record_summaries(): + for i, name in enumerate(metric_names): + tf.compat.v1.summary.scalar(prefix + name, args[i][0], step=step) + + return tf.compat.v1.summary.all_summary_ops() + + # To log the current learning rate, and gradient norm for Tensorboard, the + # summary op needs to be run on the host CPU via host_call. host_call + # expects [batch_size, ...] Tensors, thus reshape to introduce a batch + # dimension. These Tensors are implicitly concatenated to + # [params['batch_size']]. + global_step_tensor = tf.reshape( + tf.compat.v1.train.get_or_create_global_step(), [1]) + other_tensors = [tf.reshape(metric_dict[key], [1]) for key in metric_names] + + return host_call_fn, [global_step_tensor] + other_tensors + + +def embedding_matmul(embedding_table, values, mask, name="embedding_matmul"): + """Performs embedding lookup via a matmul. + + The matrix to be multiplied by the embedding table Tensor is constructed + via an implementation of scatter based on broadcasting embedding indices + and performing an equality comparison against a broadcasted + range(num_embedding_table_rows). All masked positions will produce an + embedding vector of zeros. + + Args: + embedding_table: Tensor of embedding table. + Rank 2 (table_size x embedding dim) + values: Tensor of embedding indices. Rank 2 (batch x n_indices) + mask: Tensor of mask / weights. Rank 2 (batch x n_indices) + name: Optional name scope for created ops + + Returns: + Rank 3 tensor of embedding vectors. + """ + + with tf.name_scope(name): + n_embeddings = embedding_table.get_shape().as_list()[0] + batch_size, padded_size = values.shape.as_list() + + emb_idcs = tf.tile( + tf.reshape(values, (batch_size, padded_size, 1)), (1, 1, n_embeddings)) + emb_weights = tf.tile( + tf.reshape(mask, (batch_size, padded_size, 1)), (1, 1, n_embeddings)) + col_idcs = tf.tile( + tf.reshape(tf.range(n_embeddings), (1, 1, n_embeddings)), + (batch_size, padded_size, 1)) + one_hot = tf.where( + tf.equal(emb_idcs, col_idcs), emb_weights, + tf.zeros((batch_size, padded_size, n_embeddings))) + + return tf.tensordot(one_hot, embedding_table, 1) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/tpu_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/tpu_test.py new file mode 100644 index 0000000..ba5b868 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/utils/tpu_test.py @@ -0,0 +1,108 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Test TPU optimized matmul embedding.""" + +import numpy as np +import tensorflow as tf + +from official.r1.utils import tpu as tpu_utils + + +TEST_CASES = [ + dict(embedding_dim=256, vocab_size=1000, sequence_length=64, + batch_size=32, seed=54131), + dict(embedding_dim=8, vocab_size=15, sequence_length=12, + batch_size=256, seed=536413), + dict(embedding_dim=2048, vocab_size=512, sequence_length=50, + batch_size=8, seed=35124) +] + + +class TPUBaseTester(tf.test.TestCase): + def construct_embedding_and_values(self, embedding_dim, vocab_size, + sequence_length, batch_size, seed): + np.random.seed(seed) + + embeddings = np.random.random(size=(vocab_size, embedding_dim)) + embedding_table = tf.convert_to_tensor(value=embeddings, dtype=tf.float32) + + tokens = np.random.randint(low=1, high=vocab_size-1, + size=(batch_size, sequence_length)) + for i in range(batch_size): + tokens[i, np.random.randint(low=0, high=sequence_length-1):] = 0 + values = tf.convert_to_tensor(value=tokens, dtype=tf.int32) + mask = tf.cast(tf.not_equal(values, 0), dtype=tf.float32) + return embedding_table, values, mask + + def _test_embedding(self, embedding_dim, vocab_size, + sequence_length, batch_size, seed): + """Test that matmul embedding matches embedding lookup (gather).""" + + with self.test_session(): + embedding_table, values, mask = self.construct_embedding_and_values( + embedding_dim=embedding_dim, + vocab_size=vocab_size, + sequence_length=sequence_length, + batch_size=batch_size, + seed=seed + ) + + embedding = (tf.nn.embedding_lookup(params=embedding_table, ids=values) * + tf.expand_dims(mask, -1)) + + matmul_embedding = tpu_utils.embedding_matmul( + embedding_table=embedding_table, values=values, mask=mask) + + self.assertAllClose(embedding, matmul_embedding) + + def _test_masking(self, embedding_dim, vocab_size, + sequence_length, batch_size, seed): + """Test that matmul embedding properly zeros masked positions.""" + with self.test_session(): + embedding_table, values, mask = self.construct_embedding_and_values( + embedding_dim=embedding_dim, + vocab_size=vocab_size, + sequence_length=sequence_length, + batch_size=batch_size, + seed=seed + ) + + matmul_embedding = tpu_utils.embedding_matmul( + embedding_table=embedding_table, values=values, mask=mask) + + self.assertAllClose(matmul_embedding, + matmul_embedding * tf.expand_dims(mask, -1)) + + def test_embedding_0(self): + self._test_embedding(**TEST_CASES[0]) + + def test_embedding_1(self): + self._test_embedding(**TEST_CASES[1]) + + def test_embedding_2(self): + self._test_embedding(**TEST_CASES[2]) + + def test_masking_0(self): + self._test_masking(**TEST_CASES[0]) + + def test_masking_1(self): + self._test_masking(**TEST_CASES[1]) + + def test_masking_2(self): + self._test_masking(**TEST_CASES[2]) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/README.md new file mode 100644 index 0000000..6598d89 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/README.md @@ -0,0 +1,102 @@ +![No Maintenance Intended](https://img.shields.io/badge/No%20Maintenance%20Intended-%E2%9C%95-red.svg) +![TensorFlow Requirement: 1.x](https://img.shields.io/badge/TensorFlow%20Requirement-1.x-brightgreen) +![TensorFlow 2 Not Supported](https://img.shields.io/badge/TensorFlow%202%20Not%20Supported-%E2%9C%95-red.svg) + +# Predicting Income with the Census Income Dataset + +The implementation is based on TensorFlow 1.x. + +## Overview +The [Census Income Data Set](https://archive.ics.uci.edu/ml/datasets/Census+Income) contains over 48,000 samples with attributes including age, occupation, education, and income (a binary label, either `>50K` or `<=50K`). The dataset is split into roughly 32,000 training and 16,000 testing samples. + +Here, we use the [wide and deep model](https://research.googleblog.com/2016/06/wide-deep-learning-better-together-with.html) to predict the income labels. The **wide model** is able to memorize interactions with data with a large number of features but not able to generalize these learned interactions on new data. The **deep model** generalizes well but is unable to learn exceptions within the data. The **wide and deep model** combines the two models and is able to generalize while learning exceptions. + +For the purposes of this example code, the Census Income Data Set was chosen to allow the model to train in a reasonable amount of time. You'll notice that the deep model performs almost as well as the wide and deep model on this dataset. The wide and deep model truly shines on larger data sets with high-cardinality features, where each feature has millions/billions of unique possible values (which is the specialty of the wide model). + +Finally, a key point. As a modeler and developer, think about how this dataset is used and the potential benefits and harm a model's predictions can cause. A model like this could reinforce societal biases and disparities. Is a feature relevant to the problem you want to solve, or will it introduce bias? For more information, read about [ML fairness](https://developers.google.com/machine-learning/fairness-overview/). + +--- + +The code sample in this directory uses the high level `tf.estimator.Estimator` API. This API is great for fast iteration and quickly adapting models to your own datasets without major code overhauls. It allows you to move from single-worker training to distributed training, and it makes it easy to export model binaries for prediction. + +The input function for the `Estimator` uses `tf.contrib.data.TextLineDataset`, which creates a `Dataset` object. The `Dataset` API makes it easy to apply transformations (map, batch, shuffle, etc.) to the data. [Read more here](https://www.tensorflow.org/guide/datasets). + +The `Estimator` and `Dataset` APIs are both highly encouraged for fast development and efficient training. + +## Running the code +First make sure you've [added the models folder to your Python path](/official/#running-the-models); otherwise you may encounter an error like `ImportError: No module named official.wide_deep`. + +### Setup +The [Census Income Data Set](https://archive.ics.uci.edu/ml/datasets/Census+Income) that this sample uses for training is hosted by the [UC Irvine Machine Learning Repository](https://archive.ics.uci.edu/ml/datasets/). We have provided a script that downloads and cleans the necessary files. + +``` +python census_dataset.py +``` + +This will download the files to `/tmp/census_data`. To change the directory, set the `--data_dir` flag. + +### Training +You can run the code locally as follows: + +``` +python census_main.py +``` + +The model is saved to `/tmp/census_model` by default, which can be changed using the `--model_dir` flag. + +To run the *wide* or *deep*-only models, set the `--model_type` flag to `wide` or `deep`. Other flags are configurable as well; see `census_main.py` for details. + +The final accuracy should be over 83% with any of the three model types. + +You can also experiment with `-inter` and `-intra` flag to explore inter/intra op parallelism for potential better performance as follows: + +``` +python census_main.py --inter= --intra= +``` +Please note the above optional inter/intra op does not affect model accuracy. These are TensorFlow framework configurations that only affect execution time. +For more details regarding the above inter/intra flags, please refer to [Optimizing_for_CPU](https://www.tensorflow.org/performance/performance_guide#optimizing_for_cpu) or [TensorFlow config.proto source code](https://github.com/tensorflow/tensorflow/blob/26b4dfa65d360f2793ad75083c797d57f8661b93/tensorflow/core/protobuf/config.proto#L165). + +### TensorBoard + +Run TensorBoard to inspect the details about the graph and training progression. + +``` +tensorboard --logdir=/tmp/census_model +``` + +## Inference with SavedModel +You can export the model into Tensorflow [SavedModel](https://www.tensorflow.org/guide/saved_model) format by using the argument `--export_dir`: + +``` +python census_main.py --export_dir /tmp/wide_deep_saved_model +``` + +After the model finishes training, use [`saved_model_cli`](https://www.tensorflow.org/guide/saved_model#cli_to_inspect_and_execute_savedmodel) to inspect and execute the SavedModel. + +Try the following commands to inspect the SavedModel: + +**Replace `${TIMESTAMP}` with the folder produced (e.g. 1524249124)** +``` +# List possible tag_sets. Only one metagraph is saved, so there will be one option. +saved_model_cli show --dir /tmp/wide_deep_saved_model/${TIMESTAMP}/ + +# Show SignatureDefs for tag_set=serve. SignatureDefs define the outputs to show. +saved_model_cli show --dir /tmp/wide_deep_saved_model/${TIMESTAMP}/ \ + --tag_set serve --all +``` + +### Inference +Let's use the model to predict the income group of two examples: +``` +saved_model_cli run --dir /tmp/wide_deep_saved_model/${TIMESTAMP}/ \ +--tag_set serve --signature_def="predict" \ +--input_examples='examples=[{"age":[46.], "education_num":[10.], "capital_gain":[7688.], "capital_loss":[0.], "hours_per_week":[38.]}, {"age":[24.], "education_num":[13.], "capital_gain":[0.], "capital_loss":[0.], "hours_per_week":[50.]}]' +``` + +This will print out the predicted classes and class probabilities. Class 0 is the <=50k group and 1 is the >50k group. + +## Additional Links + +If you are interested in distributed training, take a look at [Distributed TensorFlow](https://www.tensorflow.org/deploy/distributed). + +You can also [run this model on Cloud ML Engine](https://cloud.google.com/ml-engine/docs/getting-started-training-prediction), which provides [hyperparameter tuning](https://cloud.google.com/ml-engine/docs/getting-started-training-prediction#hyperparameter_tuning) to maximize your model's results and enables [deploying your model for prediction](https://cloud.google.com/ml-engine/docs/getting-started-training-prediction#deploy_a_model_to_support_prediction). diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/census_dataset.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/census_dataset.py new file mode 100644 index 0000000..7aac80e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/census_dataset.py @@ -0,0 +1,205 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Download and clean the Census Income Dataset.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys + +# pylint: disable=wrong-import-order +from absl import app as absl_app +from absl import flags +from six.moves import urllib +from six.moves import zip +import tensorflow as tf +# pylint: enable=wrong-import-order + +from official.utils.flags import core as flags_core + + +DATA_URL = 'https://archive.ics.uci.edu/ml/machine-learning-databases/adult' +TRAINING_FILE = 'adult.data' +TRAINING_URL = '%s/%s' % (DATA_URL, TRAINING_FILE) +EVAL_FILE = 'adult.test' +EVAL_URL = '%s/%s' % (DATA_URL, EVAL_FILE) + + +_CSV_COLUMNS = [ + 'age', 'workclass', 'fnlwgt', 'education', 'education_num', + 'marital_status', 'occupation', 'relationship', 'race', 'gender', + 'capital_gain', 'capital_loss', 'hours_per_week', 'native_country', + 'income_bracket' +] + +_CSV_COLUMN_DEFAULTS = [[0], [''], [0], [''], [0], [''], [''], [''], [''], [''], + [0], [0], [0], [''], ['']] + +_HASH_BUCKET_SIZE = 1000 + +_NUM_EXAMPLES = { + 'train': 32561, + 'validation': 16281, +} + + +def _download_and_clean_file(filename, url): + """Downloads data from url, and makes changes to match the CSV format.""" + temp_file, _ = urllib.request.urlretrieve(url) + with tf.gfile.Open(temp_file, 'r') as temp_eval_file: + with tf.gfile.Open(filename, 'w') as eval_file: + for line in temp_eval_file: + line = line.strip() + line = line.replace(', ', ',') + if not line or ',' not in line: + continue + if line[-1] == '.': + line = line[:-1] + line += '\n' + eval_file.write(line) + tf.gfile.Remove(temp_file) + + +def download(data_dir): + """Download census data if it is not already present.""" + tf.gfile.MakeDirs(data_dir) + + training_file_path = os.path.join(data_dir, TRAINING_FILE) + if not tf.gfile.Exists(training_file_path): + _download_and_clean_file(training_file_path, TRAINING_URL) + + eval_file_path = os.path.join(data_dir, EVAL_FILE) + if not tf.gfile.Exists(eval_file_path): + _download_and_clean_file(eval_file_path, EVAL_URL) + + +def build_model_columns(): + """Builds a set of wide and deep feature columns.""" + # Continuous variable columns + age = tf.feature_column.numeric_column('age') + education_num = tf.feature_column.numeric_column('education_num') + capital_gain = tf.feature_column.numeric_column('capital_gain') + capital_loss = tf.feature_column.numeric_column('capital_loss') + hours_per_week = tf.feature_column.numeric_column('hours_per_week') + + education = tf.feature_column.categorical_column_with_vocabulary_list( + 'education', [ + 'Bachelors', 'HS-grad', '11th', 'Masters', '9th', 'Some-college', + 'Assoc-acdm', 'Assoc-voc', '7th-8th', 'Doctorate', 'Prof-school', + '5th-6th', '10th', '1st-4th', 'Preschool', '12th']) + + marital_status = tf.feature_column.categorical_column_with_vocabulary_list( + 'marital_status', [ + 'Married-civ-spouse', 'Divorced', 'Married-spouse-absent', + 'Never-married', 'Separated', 'Married-AF-spouse', 'Widowed']) + + relationship = tf.feature_column.categorical_column_with_vocabulary_list( + 'relationship', [ + 'Husband', 'Not-in-family', 'Wife', 'Own-child', 'Unmarried', + 'Other-relative']) + + workclass = tf.feature_column.categorical_column_with_vocabulary_list( + 'workclass', [ + 'Self-emp-not-inc', 'Private', 'State-gov', 'Federal-gov', + 'Local-gov', '?', 'Self-emp-inc', 'Without-pay', 'Never-worked']) + + # To show an example of hashing: + occupation = tf.feature_column.categorical_column_with_hash_bucket( + 'occupation', hash_bucket_size=_HASH_BUCKET_SIZE) + + # Transformations. + age_buckets = tf.feature_column.bucketized_column( + age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65]) + + # Wide columns and deep columns. + base_columns = [ + education, marital_status, relationship, workclass, occupation, + age_buckets, + ] + + crossed_columns = [ + tf.feature_column.crossed_column( + ['education', 'occupation'], hash_bucket_size=_HASH_BUCKET_SIZE), + tf.feature_column.crossed_column( + [age_buckets, 'education', 'occupation'], + hash_bucket_size=_HASH_BUCKET_SIZE), + ] + + wide_columns = base_columns + crossed_columns + + deep_columns = [ + age, + education_num, + capital_gain, + capital_loss, + hours_per_week, + tf.feature_column.indicator_column(workclass), + tf.feature_column.indicator_column(education), + tf.feature_column.indicator_column(marital_status), + tf.feature_column.indicator_column(relationship), + # To show an example of embedding + tf.feature_column.embedding_column(occupation, dimension=8), + ] + + return wide_columns, deep_columns + + +def input_fn(data_file, num_epochs, shuffle, batch_size): + """Generate an input function for the Estimator.""" + assert tf.gfile.Exists(data_file), ( + '%s not found. Please make sure you have run census_dataset.py and ' + 'set the --data_dir argument to the correct path.' % data_file) + + def parse_csv(value): + tf.logging.info('Parsing {}'.format(data_file)) + columns = tf.decode_csv(value, record_defaults=_CSV_COLUMN_DEFAULTS) + features = dict(list(zip(_CSV_COLUMNS, columns))) + labels = features.pop('income_bracket') + classes = tf.equal(labels, '>50K') # binary classification + return features, classes + + # Extract lines from input files using the Dataset API. + dataset = tf.data.TextLineDataset(data_file) + + if shuffle: + dataset = dataset.shuffle(buffer_size=_NUM_EXAMPLES['train']) + + dataset = dataset.map(parse_csv, num_parallel_calls=5) + + # We call repeat after shuffling, rather than before, to prevent separate + # epochs from blending together. + dataset = dataset.repeat(num_epochs) + dataset = dataset.batch(batch_size) + return dataset + + +def define_data_download_flags(): + """Add flags specifying data download arguments.""" + flags.DEFINE_string( + name="data_dir", default="/tmp/census_data/", + help=flags_core.help_wrap( + "Directory to download and extract data.")) + + +def main(_): + download(flags.FLAGS.data_dir) + + +if __name__ == '__main__': + tf.logging.set_verbosity(tf.logging.INFO) + define_data_download_flags() + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/census_main.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/census_main.py new file mode 100644 index 0000000..bcd9cd6 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/census_main.py @@ -0,0 +1,116 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Train DNN on census income dataset.""" + +import os + +from absl import app as absl_app +from absl import flags +import tensorflow as tf + +from official.utils.flags import core as flags_core +from official.utils.logs import logger +from official.r1.wide_deep import census_dataset +from official.r1.wide_deep import wide_deep_run_loop + + +def define_census_flags(): + wide_deep_run_loop.define_wide_deep_flags() + flags.adopt_module_key_flags(wide_deep_run_loop) + flags_core.set_defaults(data_dir='/tmp/census_data', + model_dir='/tmp/census_model', + train_epochs=40, + epochs_between_evals=2, + inter_op_parallelism_threads=0, + intra_op_parallelism_threads=0, + batch_size=40) + + +def build_estimator(model_dir, model_type, model_column_fn, inter_op, intra_op): + """Build an estimator appropriate for the given model type.""" + wide_columns, deep_columns = model_column_fn() + hidden_units = [100, 75, 50, 25] + + # Create a tf.estimator.RunConfig to ensure the model is run on CPU, which + # trains faster than GPU for this model. + run_config = tf.estimator.RunConfig().replace( + session_config=tf.ConfigProto(device_count={'GPU': 0}, + inter_op_parallelism_threads=inter_op, + intra_op_parallelism_threads=intra_op)) + + if model_type == 'wide': + return tf.estimator.LinearClassifier( + model_dir=model_dir, + feature_columns=wide_columns, + config=run_config) + elif model_type == 'deep': + return tf.estimator.DNNClassifier( + model_dir=model_dir, + feature_columns=deep_columns, + hidden_units=hidden_units, + config=run_config) + else: + return tf.estimator.DNNLinearCombinedClassifier( + model_dir=model_dir, + linear_feature_columns=wide_columns, + dnn_feature_columns=deep_columns, + dnn_hidden_units=hidden_units, + config=run_config) + + +def run_census(flags_obj): + """Construct all necessary functions and call run_loop. + + Args: + flags_obj: Object containing user specified flags. + """ + if flags_obj.download_if_missing: + census_dataset.download(flags_obj.data_dir) + + train_file = os.path.join(flags_obj.data_dir, census_dataset.TRAINING_FILE) + test_file = os.path.join(flags_obj.data_dir, census_dataset.EVAL_FILE) + + # Train and evaluate the model every `flags.epochs_between_evals` epochs. + def train_input_fn(): + return census_dataset.input_fn( + train_file, flags_obj.epochs_between_evals, True, flags_obj.batch_size) + + def eval_input_fn(): + return census_dataset.input_fn(test_file, 1, False, flags_obj.batch_size) + + tensors_to_log = { + 'average_loss': '{loss_prefix}head/truediv', + 'loss': '{loss_prefix}head/weighted_loss/Sum' + } + + wide_deep_run_loop.run_loop( + name="Census Income", train_input_fn=train_input_fn, + eval_input_fn=eval_input_fn, + model_column_fn=census_dataset.build_model_columns, + build_estimator_fn=build_estimator, + flags_obj=flags_obj, + tensors_to_log=tensors_to_log, + early_stop=True) + + +def main(_): + with logger.benchmark_context(flags.FLAGS): + run_census(flags.FLAGS) + + +if __name__ == '__main__': + tf.logging.set_verbosity(tf.logging.INFO) + define_census_flags() + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/census_test.csv b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/census_test.csv new file mode 100644 index 0000000..374397d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/census_test.csv @@ -0,0 +1,30 @@ +39,State-gov,77516,Bachelors,13,Never-married,Adm-clerical,Not-in-family,,,2174,0,40,,<=50K +50,Self-emp-not-inc,83311,Bachelors,13,Married-civ-spouse,Exec-managerial,Husband,,,0,0,13,,<=50K +38,Private,215646,HS-grad,9,Divorced,Handlers-cleaners,Not-in-family,,,0,0,40,,<=50K +53,Private,234721,11th,7,Married-civ-spouse,Handlers-cleaners,Husband,,,0,0,40,,<=50K +28,Private,338409,Bachelors,13,Married-civ-spouse,Prof-specialty,Wife,,,0,0,40,,<=50K +37,Private,284582,Masters,14,Married-civ-spouse,Exec-managerial,Wife,,,0,0,40,,<=50K +49,Private,160187,9th,5,Married-spouse-absent,Other-service,Not-in-family,,,0,0,16,,<=50K +52,Self-emp-not-inc,209642,HS-grad,9,Married-civ-spouse,Exec-managerial,Husband,,,0,0,45,,>50K +31,Private,45781,Masters,14,Never-married,Prof-specialty,Not-in-family,,,14084,0,50,,>50K +42,Private,159449,Bachelors,13,Married-civ-spouse,Exec-managerial,Husband,,,5178,0,40,,>50K +37,Private,280464,Some-college,10,Married-civ-spouse,Exec-managerial,Husband,,,0,0,80,,>50K +30,State-gov,141297,Bachelors,13,Married-civ-spouse,Prof-specialty,Husband,,,0,0,40,,>50K +23,Private,122272,Bachelors,13,Never-married,Adm-clerical,Own-child,,,0,0,30,,<=50K +32,Private,205019,Assoc-acdm,12,Never-married,Sales,Not-in-family,,,0,0,50,,<=50K +40,Private,121772,Assoc-voc,11,Married-civ-spouse,Craft-repair,Husband,,,0,0,40,,>50K +34,Private,245487,7th-8th,4,Married-civ-spouse,Transport-moving,Husband,,,0,0,45,,<=50K +25,Self-emp-not-inc,176756,HS-grad,9,Never-married,Farming-fishing,Own-child,,,0,0,35,,<=50K +32,Private,186824,HS-grad,9,Never-married,Machine-op-inspct,Unmarried,,,0,0,40,,<=50K +38,Private,28887,11th,7,Married-civ-spouse,Sales,Husband,,,0,0,50,,<=50K +43,Self-emp-not-inc,292175,Masters,14,Divorced,Exec-managerial,Unmarried,,,0,0,45,,>50K +40,Private,193524,Doctorate,16,Married-civ-spouse,Prof-specialty,Husband,,,0,0,60,,>50K +56,Local-gov,216851,Bachelors,13,Married-civ-spouse,Tech-support,Husband,,,0,0,40,,>50K +54,?,180211,Some-college,10,Married-civ-spouse,?,Husband,,,0,0,60,,>50K +22,State-gov,311512,Some-college,10,Married-civ-spouse,Other-service,Husband,,,0,0,15,,<=50K +31,Private,84154,Some-college,10,Married-civ-spouse,Sales,Husband,,,0,0,38,,>50K +57,Federal-gov,337895,Bachelors,13,Married-civ-spouse,Prof-specialty,Husband,,,0,0,40,,>50K +47,Private,51835,Prof-school,15,Married-civ-spouse,Prof-specialty,Wife,,,0,1902,60,,>50K +50,Federal-gov,251585,Bachelors,13,Divorced,Exec-managerial,Not-in-family,,,0,0,55,,>50K +25,Private,289980,HS-grad,9,Never-married,Handlers-cleaners,Not-in-family,,,0,0,35,,<=50K +42,Private,116632,Doctorate,16,Married-civ-spouse,Prof-specialty,Husband,,,0,0,45,,>50K diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/census_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/census_test.py new file mode 100644 index 0000000..8e1c865 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/census_test.py @@ -0,0 +1,169 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import unittest + +import tensorflow as tf # pylint: disable=g-bad-import-order +from absl import logging + +from official.utils.misc import keras_utils +from official.utils.testing import integration +from official.r1.wide_deep import census_dataset +from official.r1.wide_deep import census_main + +logging.set_verbosity(logging.ERROR) + +TEST_INPUT = ('18,Self-emp-not-inc,987,Bachelors,12,Married-civ-spouse,abc,' + 'Husband,zyx,wvu,34,56,78,tsr,<=50K') + +TEST_INPUT_VALUES = { + 'age': 18, + 'education_num': 12, + 'capital_gain': 34, + 'capital_loss': 56, + 'hours_per_week': 78, + 'education': 'Bachelors', + 'marital_status': 'Married-civ-spouse', + 'relationship': 'Husband', + 'workclass': 'Self-emp-not-inc', + 'occupation': 'abc', +} + +TEST_CSV = os.path.join(os.path.dirname(__file__), 'census_test.csv') + + +class BaseTest(tf.test.TestCase): + """Tests for Wide Deep model.""" + + @classmethod + def setUpClass(cls): # pylint: disable=invalid-name + super(BaseTest, cls).setUpClass() + census_main.define_census_flags() + + def setUp(self): + # Create temporary CSV file + self.temp_dir = self.get_temp_dir() + self.input_csv = os.path.join(self.temp_dir, 'test.csv') + with tf.io.gfile.GFile(self.input_csv, 'w') as temp_csv: + temp_csv.write(TEST_INPUT) + + with tf.io.gfile.GFile(TEST_CSV, 'r') as temp_csv: + test_csv_contents = temp_csv.read() + + # Used for end-to-end tests. + for fname in [census_dataset.TRAINING_FILE, census_dataset.EVAL_FILE]: + with tf.io.gfile.GFile( + os.path.join(self.temp_dir, fname), 'w') as test_csv: + test_csv.write(test_csv_contents) + + @unittest.skipIf(keras_utils.is_v2_0(), 'TF 1.0 only test.') + def test_input_fn(self): + dataset = census_dataset.input_fn(self.input_csv, 1, False, 1) + features, labels = dataset.make_one_shot_iterator().get_next() + + with self.test_session() as sess: + features, labels = sess.run((features, labels)) + + # Compare the two features dictionaries. + for key in TEST_INPUT_VALUES: + self.assertTrue(key in features) + self.assertEqual(len(features[key]), 1) + feature_value = features[key][0] + + # Convert from bytes to string for Python 3. + if isinstance(feature_value, bytes): + feature_value = feature_value.decode() + + self.assertEqual(TEST_INPUT_VALUES[key], feature_value) + + self.assertFalse(labels) + + def build_and_test_estimator(self, model_type): + """Ensure that model trains and minimizes loss.""" + model = census_main.build_estimator( + self.temp_dir, model_type, + model_column_fn=census_dataset.build_model_columns, + inter_op=0, intra_op=0) + + # Train for 1 step to initialize model and evaluate initial loss + def get_input_fn(num_epochs, shuffle, batch_size): + def input_fn(): + return census_dataset.input_fn( + TEST_CSV, num_epochs=num_epochs, shuffle=shuffle, + batch_size=batch_size) + return input_fn + + model.train(input_fn=get_input_fn(1, True, 1), steps=1) + initial_results = model.evaluate(input_fn=get_input_fn(1, False, 1)) + + # Train for 100 epochs at batch size 3 and evaluate final loss + model.train(input_fn=get_input_fn(100, True, 3)) + final_results = model.evaluate(input_fn=get_input_fn(1, False, 1)) + + print('%s initial results:' % model_type, initial_results) + print('%s final results:' % model_type, final_results) + + # Ensure loss has decreased, while accuracy and both AUCs have increased. + self.assertLess(final_results['loss'], initial_results['loss']) + self.assertGreater(final_results['auc'], initial_results['auc']) + self.assertGreater(final_results['auc_precision_recall'], + initial_results['auc_precision_recall']) + self.assertGreater(final_results['accuracy'], initial_results['accuracy']) + + @unittest.skipIf(keras_utils.is_v2_0(), 'TF 1.0 only test.') + def test_wide_deep_estimator_training(self): + self.build_and_test_estimator('wide_deep') + + @unittest.skipIf(keras_utils.is_v2_0(), 'TF 1.0 only test.') + def test_end_to_end_wide(self): + integration.run_synthetic( + main=census_main.main, tmp_root=self.get_temp_dir(), + extra_flags=[ + '--data_dir', self.get_temp_dir(), + '--model_type', 'wide', + '--download_if_missing=false' + ], + synth=False) + + @unittest.skipIf(keras_utils.is_v2_0(), 'TF 1.0 only test.') + def test_end_to_end_deep(self): + integration.run_synthetic( + main=census_main.main, tmp_root=self.get_temp_dir(), + extra_flags=[ + '--data_dir', self.get_temp_dir(), + '--model_type', 'deep', + '--download_if_missing=false' + ], + synth=False) + + @unittest.skipIf(keras_utils.is_v2_0(), 'TF 1.0 only test.') + def test_end_to_end_wide_deep(self): + integration.run_synthetic( + main=census_main.main, tmp_root=self.get_temp_dir(), + extra_flags=[ + '--data_dir', self.get_temp_dir(), + '--model_type', 'wide_deep', + '--download_if_missing=false' + ], + synth=False) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/movielens_dataset.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/movielens_dataset.py new file mode 100644 index 0000000..311eb7f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/movielens_dataset.py @@ -0,0 +1,165 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Prepare MovieLens dataset for wide-deep.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools +import os + +# pylint: disable=wrong-import-order +from absl import app as absl_app +from absl import flags +import numpy as np +import tensorflow as tf +# pylint: enable=wrong-import-order + +from official.recommendation import movielens +from official.r1.utils.data import file_io +from official.utils.flags import core as flags_core + + +_BUFFER_SUBDIR = "wide_deep_buffer" +_FEATURE_MAP = { + movielens.USER_COLUMN: tf.compat.v1.FixedLenFeature([1], dtype=tf.int64), + movielens.ITEM_COLUMN: tf.compat.v1.FixedLenFeature([1], dtype=tf.int64), + movielens.TIMESTAMP_COLUMN: tf.compat.v1.FixedLenFeature([1], + dtype=tf.int64), + movielens.GENRE_COLUMN: tf.compat.v1.FixedLenFeature( + [movielens.N_GENRE], dtype=tf.int64), + movielens.RATING_COLUMN: tf.compat.v1.FixedLenFeature([1], + dtype=tf.float32), +} + +_BUFFER_SIZE = { + movielens.ML_1M: {"train": 107978119, "eval": 26994538}, + movielens.ML_20M: {"train": 2175203810, "eval": 543802008} +} + +_USER_EMBEDDING_DIM = 16 +_ITEM_EMBEDDING_DIM = 64 + +def build_model_columns(dataset): + """Builds a set of wide and deep feature columns.""" + user_id = tf.feature_column.categorical_column_with_vocabulary_list( + movielens.USER_COLUMN, range(1, movielens.NUM_USER_IDS[dataset])) + user_embedding = tf.feature_column.embedding_column( + user_id, _USER_EMBEDDING_DIM, max_norm=np.sqrt(_USER_EMBEDDING_DIM)) + + item_id = tf.feature_column.categorical_column_with_vocabulary_list( + movielens.ITEM_COLUMN, range(1, movielens.NUM_ITEM_IDS)) + item_embedding = tf.feature_column.embedding_column( + item_id, _ITEM_EMBEDDING_DIM, max_norm=np.sqrt(_ITEM_EMBEDDING_DIM)) + + time = tf.feature_column.numeric_column(movielens.TIMESTAMP_COLUMN) + genres = tf.feature_column.numeric_column( + movielens.GENRE_COLUMN, shape=(movielens.N_GENRE,), dtype=tf.uint8) + + deep_columns = [user_embedding, item_embedding, time, genres] + wide_columns = [] + + return wide_columns, deep_columns + + +def _deserialize(examples_serialized): + features = tf.parse_example(examples_serialized, _FEATURE_MAP) + return features, features[movielens.RATING_COLUMN] / movielens.MAX_RATING + + +def _buffer_path(data_dir, dataset, name): + return os.path.join(data_dir, _BUFFER_SUBDIR, + "{}_{}_buffer".format(dataset, name)) + + +def _df_to_input_fn(df, name, dataset, data_dir, batch_size, repeat, shuffle): + """Serialize a dataframe and write it to a buffer file.""" + buffer_path = _buffer_path(data_dir, dataset, name) + expected_size = _BUFFER_SIZE[dataset].get(name) + + file_io.write_to_buffer( + dataframe=df, buffer_path=buffer_path, + columns=list(_FEATURE_MAP.keys()), expected_size=expected_size) + + def input_fn(): + dataset = tf.data.TFRecordDataset(buffer_path) + # batch comes before map because map can deserialize multiple examples. + dataset = dataset.batch(batch_size) + dataset = dataset.map(_deserialize, num_parallel_calls=16) + if shuffle: + dataset = dataset.shuffle(shuffle) + + dataset = dataset.repeat(repeat) + return dataset.prefetch(1) + + return input_fn + + +def _check_buffers(data_dir, dataset): + train_path = os.path.join(data_dir, _BUFFER_SUBDIR, + "{}_{}_buffer".format(dataset, "train")) + eval_path = os.path.join(data_dir, _BUFFER_SUBDIR, + "{}_{}_buffer".format(dataset, "eval")) + + if not tf.gfile.Exists(train_path) or not tf.gfile.Exists(eval_path): + return False + + return all([ + tf.gfile.Stat(_buffer_path(data_dir, dataset, "train")).length == + _BUFFER_SIZE[dataset]["train"], + tf.gfile.Stat(_buffer_path(data_dir, dataset, "eval")).length == + _BUFFER_SIZE[dataset]["eval"], + ]) + + +def construct_input_fns(dataset, data_dir, batch_size=16, repeat=1): + """Construct train and test input functions, as well as the column fn.""" + if _check_buffers(data_dir, dataset): + train_df, eval_df = None, None + else: + df = movielens.csv_to_joint_dataframe(dataset=dataset, data_dir=data_dir) + df = movielens.integerize_genres(dataframe=df) + df = df.drop(columns=[movielens.TITLE_COLUMN]) + + train_df = df.sample(frac=0.8, random_state=0) + eval_df = df.drop(train_df.index) + + train_df = train_df.reset_index(drop=True) + eval_df = eval_df.reset_index(drop=True) + + train_input_fn = _df_to_input_fn( + df=train_df, name="train", dataset=dataset, data_dir=data_dir, + batch_size=batch_size, repeat=repeat, + shuffle=movielens.NUM_RATINGS[dataset]) + eval_input_fn = _df_to_input_fn( + df=eval_df, name="eval", dataset=dataset, data_dir=data_dir, + batch_size=batch_size, repeat=repeat, shuffle=None) + model_column_fn = functools.partial(build_model_columns, dataset=dataset) + + train_input_fn() + return train_input_fn, eval_input_fn, model_column_fn + + +def main(_): + movielens.download(dataset=flags.FLAGS.dataset, data_dir=flags.FLAGS.data_dir) + construct_input_fns(flags.FLAGS.dataset, flags.FLAGS.data_dir) + +if __name__ == "__main__": + tf.logging.set_verbosity(tf.logging.INFO) + movielens.define_data_download_flags() + flags.adopt_module_key_flags(movielens) + flags_core.set_defaults(dataset="ml-1m") + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/movielens_main.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/movielens_main.py new file mode 100644 index 0000000..39e98ce --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/movielens_main.py @@ -0,0 +1,115 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Train DNN on Kaggle movie dataset.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from absl import app as absl_app +from absl import flags +import tensorflow as tf + +from official.recommendation import movielens +from official.utils.flags import core as flags_core +from official.utils.logs import logger +from official.r1.wide_deep import movielens_dataset +from official.r1.wide_deep import wide_deep_run_loop + + +def define_movie_flags(): + """Define flags for movie dataset training.""" + wide_deep_run_loop.define_wide_deep_flags() + flags.DEFINE_enum( + name="dataset", default=movielens.ML_1M, + enum_values=movielens.DATASETS, case_sensitive=False, + help=flags_core.help_wrap("Dataset to be trained and evaluated.")) + flags.adopt_module_key_flags(wide_deep_run_loop) + flags_core.set_defaults(data_dir="/tmp/movielens-data/", + model_dir='/tmp/movie_model', + model_type="deep", + train_epochs=50, + epochs_between_evals=5, + inter_op_parallelism_threads=0, + intra_op_parallelism_threads=0, + batch_size=256) + + @flags.validator("stop_threshold", + message="stop_threshold not supported for movielens model") + def _no_stop(stop_threshold): + return stop_threshold is None + + +def build_estimator(model_dir, model_type, model_column_fn, inter_op, intra_op): + """Build an estimator appropriate for the given model type.""" + if model_type != "deep": + raise NotImplementedError("movie dataset only supports `deep` model_type") + _, deep_columns = model_column_fn() + hidden_units = [256, 256, 256, 128] + + run_config = tf.estimator.RunConfig().replace( + session_config=tf.ConfigProto(device_count={'GPU': 0}, + inter_op_parallelism_threads=inter_op, + intra_op_parallelism_threads=intra_op)) + return tf.estimator.DNNRegressor( + model_dir=model_dir, + feature_columns=deep_columns, + hidden_units=hidden_units, + optimizer=tf.compat.v1.train.AdamOptimizer(), + activation_fn=tf.nn.sigmoid, + dropout=0.3, + loss_reduction=tf.losses.Reduction.MEAN) + + +def run_movie(flags_obj): + """Construct all necessary functions and call run_loop. + + Args: + flags_obj: Object containing user specified flags. + """ + + if flags_obj.download_if_missing: + movielens.download(dataset=flags_obj.dataset, data_dir=flags_obj.data_dir) + + train_input_fn, eval_input_fn, model_column_fn = \ + movielens_dataset.construct_input_fns( + dataset=flags_obj.dataset, data_dir=flags_obj.data_dir, + batch_size=flags_obj.batch_size, repeat=flags_obj.epochs_between_evals) + + tensors_to_log = { + 'loss': '{loss_prefix}head/weighted_loss/value' + } + + wide_deep_run_loop.run_loop( + name="MovieLens", train_input_fn=train_input_fn, + eval_input_fn=eval_input_fn, + model_column_fn=model_column_fn, + build_estimator_fn=build_estimator, + flags_obj=flags_obj, + tensors_to_log=tensors_to_log, + early_stop=False) + + +def main(_): + with logger.benchmark_context(flags.FLAGS): + run_movie(flags.FLAGS) + + +if __name__ == '__main__': + tf.logging.set_verbosity(tf.logging.INFO) + define_movie_flags() + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/movielens_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/movielens_test.py new file mode 100644 index 0000000..5117f62 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/movielens_test.py @@ -0,0 +1,120 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import unittest + +import numpy as np +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.recommendation import movielens +from official.utils.misc import keras_utils +from official.utils.testing import integration +from official.r1.wide_deep import movielens_dataset +from official.r1.wide_deep import movielens_main +from absl import logging + +logging.set_verbosity(logging.ERROR) + + +TEST_INPUT_VALUES = { + "genres": np.array( + [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + "user_id": [3], + "item_id": [4], +} + +TEST_ITEM_DATA = """item_id,titles,genres +1,Movie_1,Comedy|Romance +2,Movie_2,Adventure|Children's +3,Movie_3,Comedy|Drama +4,Movie_4,Comedy +5,Movie_5,Action|Crime|Thriller +6,Movie_6,Action +7,Movie_7,Action|Adventure|Thriller""" + +TEST_RATING_DATA = """user_id,item_id,rating,timestamp +1,2,5,978300760 +1,3,3,978302109 +1,6,3,978301968 +2,1,4,978300275 +2,7,5,978824291 +3,1,3,978302268 +3,4,5,978302039 +3,5,5,978300719 +""" + + +class BaseTest(tf.test.TestCase): + """Tests for Wide Deep model.""" + + @classmethod + def setUpClass(cls): # pylint: disable=invalid-name + super(BaseTest, cls).setUpClass() + movielens_main.define_movie_flags() + + def setUp(self): + # Create temporary CSV file + self.temp_dir = self.get_temp_dir() + tf.io.gfile.makedirs(os.path.join(self.temp_dir, movielens.ML_1M)) + + self.ratings_csv = os.path.join( + self.temp_dir, movielens.ML_1M, movielens.RATINGS_FILE) + self.item_csv = os.path.join( + self.temp_dir, movielens.ML_1M, movielens.MOVIES_FILE) + + with tf.io.gfile.GFile(self.ratings_csv, "w") as f: + f.write(TEST_RATING_DATA) + + with tf.io.gfile.GFile(self.item_csv, "w") as f: + f.write(TEST_ITEM_DATA) + + @unittest.skipIf(keras_utils.is_v2_0(), "TF 1.0 only test.") + def test_input_fn(self): + train_input_fn, _, _ = movielens_dataset.construct_input_fns( + dataset=movielens.ML_1M, data_dir=self.temp_dir, batch_size=8, repeat=1) + + dataset = train_input_fn() + features, labels = dataset.make_one_shot_iterator().get_next() + + with self.session() as sess: + features, labels = sess.run((features, labels)) + + # Compare the two features dictionaries. + for key in TEST_INPUT_VALUES: + self.assertTrue(key in features) + self.assertAllClose(TEST_INPUT_VALUES[key], features[key][0]) + + self.assertAllClose(labels[0], [1.0]) + + @unittest.skipIf(keras_utils.is_v2_0(), "TF 1.0 only test.") + def test_end_to_end_deep(self): + integration.run_synthetic( + main=movielens_main.main, tmp_root=self.temp_dir, + extra_flags=[ + "--data_dir", self.temp_dir, + "--download_if_missing=false", + "--train_epochs", "1", + "--epochs_between_evals", "1" + ], + synth=False) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/wide_deep_run_loop.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/wide_deep_run_loop.py new file mode 100644 index 0000000..a8e75e1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/r1/wide_deep/wide_deep_run_loop.py @@ -0,0 +1,133 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Core run logic for TensorFlow Wide & Deep Tutorial using tf.estimator API.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import shutil + +from absl import app as absl_app +from absl import flags +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.utils.flags import core as flags_core +from official.utils.logs import hooks_helper +from official.utils.logs import logger +from official.utils.misc import model_helpers + + +LOSS_PREFIX = {'wide': 'linear/', 'deep': 'dnn/'} + + +def define_wide_deep_flags(): + """Add supervised learning flags, as well as wide-deep model type.""" + flags_core.define_base(clean=True, train_epochs=True, + epochs_between_evals=True, stop_threshold=True, + hooks=True, export_dir=True) + flags_core.define_benchmark() + flags_core.define_performance( + num_parallel_calls=False, inter_op=True, intra_op=True, + synthetic_data=False, max_train_steps=False, dtype=False, + all_reduce_alg=False) + + flags.adopt_module_key_flags(flags_core) + + flags.DEFINE_enum( + name="model_type", short_name="mt", default="wide_deep", + enum_values=['wide', 'deep', 'wide_deep'], + help="Select model topology.") + flags.DEFINE_boolean( + name="download_if_missing", default=True, help=flags_core.help_wrap( + "Download data to data_dir if it is not already present.")) + + +def export_model(model, model_type, export_dir, model_column_fn): + """Export to SavedModel format. + + Args: + model: Estimator object + model_type: string indicating model type. "wide", "deep" or "wide_deep" + export_dir: directory to export the model. + model_column_fn: Function to generate model feature columns. + """ + wide_columns, deep_columns = model_column_fn() + if model_type == 'wide': + columns = wide_columns + elif model_type == 'deep': + columns = deep_columns + else: + columns = wide_columns + deep_columns + feature_spec = tf.feature_column.make_parse_example_spec(columns) + example_input_fn = ( + tf.estimator.export.build_parsing_serving_input_receiver_fn(feature_spec)) + model.export_savedmodel(export_dir, example_input_fn, + strip_default_attrs=True) + + +def run_loop(name, train_input_fn, eval_input_fn, model_column_fn, + build_estimator_fn, flags_obj, tensors_to_log, early_stop=False): + """Define training loop.""" + model_helpers.apply_clean(flags.FLAGS) + model = build_estimator_fn( + model_dir=flags_obj.model_dir, model_type=flags_obj.model_type, + model_column_fn=model_column_fn, + inter_op=flags_obj.inter_op_parallelism_threads, + intra_op=flags_obj.intra_op_parallelism_threads) + + run_params = { + 'batch_size': flags_obj.batch_size, + 'train_epochs': flags_obj.train_epochs, + 'model_type': flags_obj.model_type, + } + + benchmark_logger = logger.get_benchmark_logger() + benchmark_logger.log_run_info('wide_deep', name, run_params, + test_id=flags_obj.benchmark_test_id) + + loss_prefix = LOSS_PREFIX.get(flags_obj.model_type, '') + tensors_to_log = {k: v.format(loss_prefix=loss_prefix) + for k, v in tensors_to_log.items()} + train_hooks = hooks_helper.get_train_hooks( + flags_obj.hooks, model_dir=flags_obj.model_dir, + batch_size=flags_obj.batch_size, tensors_to_log=tensors_to_log) + + # Train and evaluate the model every `flags.epochs_between_evals` epochs. + for n in range(flags_obj.train_epochs // flags_obj.epochs_between_evals): + model.train(input_fn=train_input_fn, hooks=train_hooks) + + results = model.evaluate(input_fn=eval_input_fn) + + # Display evaluation metrics + tf.logging.info('Results at epoch %d / %d', + (n + 1) * flags_obj.epochs_between_evals, + flags_obj.train_epochs) + tf.logging.info('-' * 60) + + for key in sorted(results): + tf.logging.info('%s: %s' % (key, results[key])) + + benchmark_logger.log_evaluation_result(results) + + if early_stop and model_helpers.past_stop_threshold( + flags_obj.stop_threshold, results['accuracy']): + break + + # Export the model + if flags_obj.export_dir is not None: + export_model(model, flags_obj.model_type, flags_obj.export_dir, + model_column_fn) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/README.md new file mode 100644 index 0000000..be916c9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/README.md @@ -0,0 +1,71 @@ +# Recommendation Model +## Overview +This is an implementation of the Neural Collaborative Filtering (NCF) framework with Neural Matrix Factorization (NeuMF) model as described in the [Neural Collaborative Filtering](https://arxiv.org/abs/1708.05031) paper. Current implementation is based on the code from the authors' [NCF code](https://github.com/hexiangnan/neural_collaborative_filtering) and the Stanford implementation in the [MLPerf Repo](https://github.com/mlperf/reference/tree/master/recommendation/pytorch). + +NCF is a general framework for collaborative filtering of recommendations in which a neural network architecture is used to model user-item interactions. Unlike traditional models, NCF does not resort to Matrix Factorization (MF) with an inner product on latent features of users and items. It replaces the inner product with a multi-layer perceptron that can learn an arbitrary function from data. + +Two instantiations of NCF are Generalized Matrix Factorization (GMF) and Multi-Layer Perceptron (MLP). GMF applies a linear kernel to model the latent feature interactions, and and MLP uses a nonlinear kernel to learn the interaction function from data. NeuMF is a fused model of GMF and MLP to better model the complex user-item interactions, and unifies the strengths of linearity of MF and non-linearity of MLP for modeling the user-item latent structures. NeuMF allows GMF and MLP to learn separate embeddings, and combines the two models by concatenating their last hidden layer. [neumf_model.py](neumf_model.py) defines the architecture details. + +Some abbreviations used the code base include: + - NCF: Neural Collaborative Filtering + - NeuMF: Neural Matrix Factorization + - GMF: Generalized Matrix Factorization + - MLP: Multi-Layer Perceptron + - HR: Hit Ratio (HR) + - NDCG: Normalized Discounted Cumulative Gain + - ml-1m: MovieLens 1 million dataset + - ml-20m: MovieLens 20 million dataset + +## Dataset +The [MovieLens datasets](http://files.grouplens.org/datasets/movielens/) are used for model training and evaluation. Specifically, we use two datasets: **ml-1m** (short for MovieLens 1 million) and **ml-20m** (short for MovieLens 20 million). + +### ml-1m +ml-1m dataset contains 1,000,209 anonymous ratings of approximately 3,706 movies made by 6,040 users who joined MovieLens in 2000. All ratings are contained in the file "ratings.dat" without header row, and are in the following format: +``` + UserID::MovieID::Rating::Timestamp +``` + - UserIDs range between 1 and 6040. + - MovieIDs range between 1 and 3952. + - Ratings are made on a 5-star scale (whole-star ratings only). + +### ml-20m +ml-20m dataset contains 20,000,263 ratings of 26,744 movies by 138493 users. All ratings are contained in the file "ratings.csv". Each line of this file after the header row represents one rating of one movie by one user, and has the following format: +``` +userId,movieId,rating,timestamp +``` + - The lines within this file are ordered first by userId, then, within user, by movieId. + - Ratings are made on a 5-star scale, with half-star increments (0.5 stars - 5.0 stars). + +In both datasets, the timestamp is represented in seconds since midnight Coordinated Universal Time (UTC) of January 1, 1970. Each user has at least 20 ratings. + +## Running Code + +### Download and preprocess dataset +To download the dataset, please install Pandas package first. Then issue the following command: +``` +python movielens.py +``` +Arguments: + * `--data_dir`: Directory where to download and save the preprocessed data. By default, it is `/tmp/movielens-data/`. + * `--dataset`: The dataset name to be downloaded and preprocessed. By default, it is `ml-1m`. + +Use the `--help` or `-h` flag to get a full list of possible arguments. + +Note the ml-20m dataset is large (the rating file is ~500 MB), and it may take several minutes (~2 mins) for data preprocessing. +Both the ml-1m and ml-20m datasets will be coerced into a common format when downloaded. + +### Train and evaluate model + +[ncf_keras_main.py](ncf_keras_main.py) is the Keras trainer that supports +features in TF 2.x. Users can train the model on both GPU and TPU. + +To train and evaluate the model, issue the following command: +``` +python ncf_keras_main.py +``` +Arguments: + * `--model_dir`: Directory to save model training checkpoints. By default, it is `/tmp/ncf/`. + * `--data_dir`: This should be set to the same directory given to the `data_download`'s `data_dir` argument. + * `--dataset`: The dataset name to be downloaded and preprocessed. By default, it is `ml-1m`. + +There are other arguments about models and training process. Refer to the [Flags package](https://abseil.io/docs/python/guides/flags) documentation or use the `--helpfull` flag to get a full list of possible arguments with detailed descriptions. diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/constants.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/constants.py new file mode 100644 index 0000000..8e313bf --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/constants.py @@ -0,0 +1,79 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Central location for NCF specific values.""" + +import sys + +import numpy as np + +from official.recommendation import movielens + +# ============================================================================== +# == Main Thread Data Processing =============================================== +# ============================================================================== + +# Keys for data shards +TRAIN_USER_KEY = "train_{}".format(movielens.USER_COLUMN) +TRAIN_ITEM_KEY = "train_{}".format(movielens.ITEM_COLUMN) +TRAIN_LABEL_KEY = "train_labels" +MASK_START_INDEX = "mask_start_index" +VALID_POINT_MASK = "valid_point_mask" +EVAL_USER_KEY = "eval_{}".format(movielens.USER_COLUMN) +EVAL_ITEM_KEY = "eval_{}".format(movielens.ITEM_COLUMN) + +USER_MAP = "user_map" +ITEM_MAP = "item_map" + +USER_DTYPE = np.int32 +ITEM_DTYPE = np.int32 + +# In both datasets, each user has at least 20 ratings. +MIN_NUM_RATINGS = 20 + +# The number of negative examples attached with a positive example +# when performing evaluation. +NUM_EVAL_NEGATIVES = 999 + +# keys for evaluation metrics +TOP_K = 10 # Top-k list for evaluation +HR_KEY = "HR" +NDCG_KEY = "NDCG" +DUPLICATE_MASK = "duplicate_mask" + +# Metric names +HR_METRIC_NAME = "HR_METRIC" +NDCG_METRIC_NAME = "NDCG_METRIC" + +# Trying to load a cache created in py2 when running in py3 will cause an +# error due to differences in unicode handling. +RAW_CACHE_FILE = "raw_data_cache_py{}.pickle".format(sys.version_info[0]) +CACHE_INVALIDATION_SEC = 3600 * 24 + +# ============================================================================== +# == Data Generation =========================================================== +# ============================================================================== +CYCLES_TO_BUFFER = 3 # The number of train cycles worth of data to "run ahead" + # of the main training loop. + +# Number of batches to run per epoch when using synthetic data. At high batch +# sizes, we run for more batches than with real data, which is good since +# running more batches reduces noise when measuring the average batches/second. +SYNTHETIC_BATCHES_PER_EPOCH = 2000 + +# Only used when StreamingFilesDataset is used. +NUM_FILE_SHARDS = 16 +TRAIN_FOLDER_TEMPLATE = "training_cycle_{}" +EVAL_FOLDER = "eval_data" +SHARD_TEMPLATE = "shard_{}.tfrecords" diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/create_ncf_data.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/create_ncf_data.py new file mode 100644 index 0000000..60267bc --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/create_ncf_data.py @@ -0,0 +1,117 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Binary to generate training/evaluation dataset for NCF model.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json + +# pylint: disable=g-bad-import-order +from absl import app +from absl import flags +import tensorflow.compat.v2 as tf +# pylint: enable=g-bad-import-order + +from official.recommendation import movielens +from official.recommendation import data_preprocessing + +flags.DEFINE_string( + "data_dir", None, + "The input data dir at which training and evaluation tf record files " + "will be saved.") +flags.DEFINE_string("meta_data_file_path", None, + "The path in which input meta data will be written.") +flags.DEFINE_enum("dataset", "ml-20m", ["ml-1m", "ml-20m"], + "Dataset to be trained/evaluated.") +flags.DEFINE_enum( + "constructor_type", "bisection", ["bisection", "materialized"], + "Strategy to use for generating false negatives. materialized has a " + "precompute that scales badly, but a faster per-epoch construction " + "time and can be faster on very large systems.") +flags.DEFINE_integer("num_train_epochs", 14, + "Total number of training epochs to generate.") +flags.DEFINE_integer( + "num_negative_samples", 4, + "Number of negative instances to pair with positive instance.") +flags.DEFINE_integer( + "train_prebatch_size", 99000, + "Batch size to be used for prebatching the dataset " + "for training.") +flags.DEFINE_integer( + "eval_prebatch_size", 99000, + "Batch size to be used for prebatching the dataset " + "for training.") + +FLAGS = flags.FLAGS + + +def prepare_raw_data(flag_obj): + """Downloads and prepares raw data for data generation.""" + movielens.download(flag_obj.dataset, flag_obj.data_dir) + + data_processing_params = { + "train_epochs": flag_obj.num_train_epochs, + "batch_size": flag_obj.train_prebatch_size, + "eval_batch_size": flag_obj.eval_prebatch_size, + "batches_per_step": 1, + "stream_files": True, + "num_neg": flag_obj.num_negative_samples, + } + + num_users, num_items, producer = data_preprocessing.instantiate_pipeline( + dataset=flag_obj.dataset, + data_dir=flag_obj.data_dir, + params=data_processing_params, + constructor_type=flag_obj.constructor_type, + epoch_dir=flag_obj.data_dir, + generate_data_offline=True) + + # pylint: disable=protected-access + input_metadata = { + "num_users": num_users, + "num_items": num_items, + "constructor_type": flag_obj.constructor_type, + "num_train_elements": producer._elements_in_epoch, + "num_eval_elements": producer._eval_elements_in_epoch, + "num_train_epochs": flag_obj.num_train_epochs, + "train_prebatch_size": flag_obj.train_prebatch_size, + "eval_prebatch_size": flag_obj.eval_prebatch_size, + "num_train_steps": producer.train_batches_per_epoch, + "num_eval_steps": producer.eval_batches_per_epoch, + } + # pylint: enable=protected-access + + return producer, input_metadata + + +def generate_data(): + """Creates NCF train/eval dataset and writes input metadata as a file.""" + producer, input_metadata = prepare_raw_data(FLAGS) + producer.run() + + with tf.io.gfile.GFile(FLAGS.meta_data_file_path, "w") as writer: + writer.write(json.dumps(input_metadata, indent=4) + "\n") + + +def main(_): + generate_data() + + +if __name__ == "__main__": + flags.mark_flag_as_required("data_dir") + flags.mark_flag_as_required("meta_data_file_path") + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/data_pipeline.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/data_pipeline.py new file mode 100644 index 0000000..1b4dd33 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/data_pipeline.py @@ -0,0 +1,959 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Asynchronous data producer for the NCF pipeline.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import atexit +import functools +import os +import sys +import tempfile +import threading +import time +import timeit +import traceback +import typing + +import numpy as np +import six +from six.moves import queue +import tensorflow as tf +from absl import logging + +from official.recommendation import constants as rconst +from official.recommendation import movielens +from official.recommendation import popen_helper +from official.recommendation import stat_utils +from tensorflow.python.tpu.datasets import StreamingFilesDataset + + +SUMMARY_TEMPLATE = """General: +{spacer}Num users: {num_users} +{spacer}Num items: {num_items} + +Training: +{spacer}Positive count: {train_pos_ct} +{spacer}Batch size: {train_batch_size} {multiplier} +{spacer}Batch count per epoch: {train_batch_ct} + +Eval: +{spacer}Positive count: {eval_pos_ct} +{spacer}Batch size: {eval_batch_size} {multiplier} +{spacer}Batch count per epoch: {eval_batch_ct}""" + + +class DatasetManager(object): + """Helper class for handling TensorFlow specific data tasks. + + This class takes the (relatively) framework agnostic work done by the data + constructor classes and handles the TensorFlow specific portions (TFRecord + management, tf.Dataset creation, etc.). + """ + + def __init__(self, + is_training, + stream_files, + batches_per_epoch, + shard_root=None, + deterministic=False, + num_train_epochs=None): + # type: (bool, bool, int, typing.Optional[str], bool, int) -> None + """Constructs a `DatasetManager` instance. + Args: + is_training: Boolean of whether the data provided is training or + evaluation data. This determines whether to reuse the data + (if is_training=False) and the exact structure to use when storing and + yielding data. + stream_files: Boolean indicating whether data should be serialized and + written to file shards. + batches_per_epoch: The number of batches in a single epoch. + shard_root: The base directory to be used when stream_files=True. + deterministic: Forgo non-deterministic speedups. (i.e. sloppy=True) + num_train_epochs: Number of epochs to generate. If None, then each + call to `get_dataset()` increments the number of epochs requested. + """ + self._is_training = is_training + self._deterministic = deterministic + self._stream_files = stream_files + self._writers = [] + self._write_locks = [threading.RLock() for _ in + range(rconst.NUM_FILE_SHARDS)] if stream_files else [] + self._batches_per_epoch = batches_per_epoch + self._epochs_completed = 0 + self._epochs_requested = num_train_epochs if num_train_epochs else 0 + self._shard_root = shard_root + + self._result_queue = queue.Queue() + self._result_reuse = [] + + @property + def current_data_root(self): + subdir = (rconst.TRAIN_FOLDER_TEMPLATE.format(self._epochs_completed) + if self._is_training else rconst.EVAL_FOLDER) + return os.path.join(self._shard_root, subdir) + + def buffer_reached(self): + # Only applicable for training. + return (self._epochs_completed - self._epochs_requested >= + rconst.CYCLES_TO_BUFFER and self._is_training) + + @staticmethod + def serialize(data): + """Convert NumPy arrays into a TFRecords entry.""" + + def create_int_feature(values): + return tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) + + feature_dict = { + k: create_int_feature(v.astype(np.int64)) for k, v in data.items() + } + + return tf.train.Example( + features=tf.train.Features(feature=feature_dict)).SerializeToString() + + @staticmethod + def deserialize(serialized_data, batch_size=None, is_training=True): + """Convert serialized TFRecords into tensors. + + Args: + serialized_data: A tensor containing serialized records. + batch_size: The data arrives pre-batched, so batch size is needed to + deserialize the data. + is_training: Boolean, whether data to deserialize to training data + or evaluation data. + """ + + def _get_feature_map(batch_size, is_training=True): + """Returns data format of the serialized tf record file.""" + + if is_training: + return { + movielens.USER_COLUMN: + tf.io.FixedLenFeature([batch_size, 1], dtype=tf.int64), + movielens.ITEM_COLUMN: + tf.io.FixedLenFeature([batch_size, 1], dtype=tf.int64), + rconst.VALID_POINT_MASK: + tf.io.FixedLenFeature([batch_size, 1], dtype=tf.int64), + "labels": + tf.io.FixedLenFeature([batch_size, 1], dtype=tf.int64) + } + else: + return { + movielens.USER_COLUMN: + tf.io.FixedLenFeature([batch_size, 1], dtype=tf.int64), + movielens.ITEM_COLUMN: + tf.io.FixedLenFeature([batch_size, 1], dtype=tf.int64), + rconst.DUPLICATE_MASK: + tf.io.FixedLenFeature([batch_size, 1], dtype=tf.int64) + } + + features = tf.io.parse_single_example( + serialized_data, _get_feature_map(batch_size, is_training=is_training)) + users = tf.cast(features[movielens.USER_COLUMN], rconst.USER_DTYPE) + items = tf.cast(features[movielens.ITEM_COLUMN], rconst.ITEM_DTYPE) + + if is_training: + valid_point_mask = tf.cast(features[rconst.VALID_POINT_MASK], tf.bool) + fake_dup_mask = tf.zeros_like(users) + return { + movielens.USER_COLUMN: users, + movielens.ITEM_COLUMN: items, + rconst.VALID_POINT_MASK: valid_point_mask, + rconst.TRAIN_LABEL_KEY: + tf.reshape(tf.cast(features["labels"], tf.bool), + (batch_size, 1)), + rconst.DUPLICATE_MASK: fake_dup_mask + } + else: + labels = tf.cast(tf.zeros_like(users), tf.bool) + fake_valid_pt_mask = tf.cast(tf.zeros_like(users), tf.bool) + return { + movielens.USER_COLUMN: + users, + movielens.ITEM_COLUMN: + items, + rconst.DUPLICATE_MASK: + tf.cast(features[rconst.DUPLICATE_MASK], tf.bool), + rconst.VALID_POINT_MASK: + fake_valid_pt_mask, + rconst.TRAIN_LABEL_KEY: + labels + } + + def put(self, index, data): + # type: (int, dict) -> None + """Store data for later consumption. + + Because there are several paths for storing and yielding data (queues, + lists, files) the data producer simply provides the data in a standard + format at which point the dataset manager handles storing it in the correct + form. + + Args: + index: Used to select shards when writing to files. + data: A dict of the data to be stored. This method mutates data, and + therefore expects to be the only consumer. + """ + if self._is_training: + mask_start_index = data.pop(rconst.MASK_START_INDEX) + batch_size = data[movielens.ITEM_COLUMN].shape[0] + data[rconst.VALID_POINT_MASK] = np.expand_dims( + np.less(np.arange(batch_size), mask_start_index), -1) + + if self._stream_files: + example_bytes = self.serialize(data) + with self._write_locks[index % rconst.NUM_FILE_SHARDS]: + self._writers[index % rconst.NUM_FILE_SHARDS].write(example_bytes) + + else: + self._result_queue.put(( + data, data.pop("labels")) if self._is_training else data) + + def start_construction(self): + if self._stream_files: + tf.io.gfile.makedirs(self.current_data_root) + template = os.path.join(self.current_data_root, rconst.SHARD_TEMPLATE) + self._writers = [tf.io.TFRecordWriter(template.format(i)) + for i in range(rconst.NUM_FILE_SHARDS)] + + def end_construction(self): + if self._stream_files: + [writer.close() for writer in self._writers] + self._writers = [] + self._result_queue.put(self.current_data_root) + + self._epochs_completed += 1 + + def data_generator(self, epochs_between_evals): + """Yields examples during local training.""" + assert not self._stream_files + assert self._is_training or epochs_between_evals == 1 + + if self._is_training: + for _ in range(self._batches_per_epoch * epochs_between_evals): + yield self._result_queue.get(timeout=300) + + else: + if self._result_reuse: + assert len(self._result_reuse) == self._batches_per_epoch + + for i in self._result_reuse: + yield i + else: + # First epoch. + for _ in range(self._batches_per_epoch * epochs_between_evals): + result = self._result_queue.get(timeout=300) + self._result_reuse.append(result) + yield result + + def increment_request_epoch(self): + self._epochs_requested += 1 + + def get_dataset(self, batch_size, epochs_between_evals): + """Construct the dataset to be used for training and eval. + + For local training, data is provided through Dataset.from_generator. For + remote training (TPUs) the data is first serialized to files and then sent + to the TPU through a StreamingFilesDataset. + + Args: + batch_size: The per-replica batch size of the dataset. + epochs_between_evals: How many epochs worth of data to yield. + (Generator mode only.) + """ + self.increment_request_epoch() + if self._stream_files: + if epochs_between_evals > 1: + raise ValueError("epochs_between_evals > 1 not supported for file " + "based dataset.") + epoch_data_dir = self._result_queue.get(timeout=300) + if not self._is_training: + self._result_queue.put(epoch_data_dir) # Eval data is reused. + + file_pattern = os.path.join( + epoch_data_dir, rconst.SHARD_TEMPLATE.format("*")) + dataset = StreamingFilesDataset( + files=file_pattern, worker_job=popen_helper.worker_job(), + num_parallel_reads=rconst.NUM_FILE_SHARDS, num_epochs=1, + sloppy=not self._deterministic) + map_fn = functools.partial( + self.deserialize, + batch_size=batch_size, + is_training=self._is_training) + dataset = dataset.map(map_fn, num_parallel_calls=16) + + else: + types = {movielens.USER_COLUMN: rconst.USER_DTYPE, + movielens.ITEM_COLUMN: rconst.ITEM_DTYPE} + shapes = { + movielens.USER_COLUMN: tf.TensorShape([batch_size, 1]), + movielens.ITEM_COLUMN: tf.TensorShape([batch_size, 1]) + } + + if self._is_training: + types[rconst.VALID_POINT_MASK] = np.bool + shapes[rconst.VALID_POINT_MASK] = tf.TensorShape([batch_size, 1]) + + types = (types, np.bool) + shapes = (shapes, tf.TensorShape([batch_size, 1])) + + else: + types[rconst.DUPLICATE_MASK] = np.bool + shapes[rconst.DUPLICATE_MASK] = tf.TensorShape([batch_size, 1]) + + data_generator = functools.partial( + self.data_generator, epochs_between_evals=epochs_between_evals) + dataset = tf.data.Dataset.from_generator( + generator=data_generator, output_types=types, + output_shapes=shapes) + + return dataset.prefetch(16) + + def make_input_fn(self, batch_size): + """Create an input_fn which checks for batch size consistency.""" + + def input_fn(params): + """Returns batches for training.""" + + # Estimator passes batch_size during training and eval_batch_size during + # eval. + param_batch_size = (params["batch_size"] if self._is_training else + params.get("eval_batch_size") or params["batch_size"]) + if batch_size != param_batch_size: + raise ValueError("producer batch size ({}) differs from params batch " + "size ({})".format(batch_size, param_batch_size)) + + epochs_between_evals = (params.get("epochs_between_evals", 1) + if self._is_training else 1) + return self.get_dataset(batch_size=batch_size, + epochs_between_evals=epochs_between_evals) + + return input_fn + + +class BaseDataConstructor(threading.Thread): + """Data constructor base class. + + This class manages the control flow for constructing data. It is not meant + to be used directly, but instead subclasses should implement the following + two methods: + + self.construct_lookup_variables + self.lookup_negative_items + + """ + + def __init__( + self, + maximum_number_epochs, # type: int + num_users, # type: int + num_items, # type: int + user_map, # type: dict + item_map, # type: dict + train_pos_users, # type: np.ndarray + train_pos_items, # type: np.ndarray + train_batch_size, # type: int + batches_per_train_step, # type: int + num_train_negatives, # type: int + eval_pos_users, # type: np.ndarray + eval_pos_items, # type: np.ndarray + eval_batch_size, # type: int + batches_per_eval_step, # type: int + stream_files, # type: bool + deterministic=False, # type: bool + epoch_dir=None, # type: str + num_train_epochs=None, # type: int + create_data_offline=False # type: bool + ): + # General constants + self._maximum_number_epochs = maximum_number_epochs + self._num_users = num_users + self._num_items = num_items + self.user_map = user_map + self.item_map = item_map + self._train_pos_users = train_pos_users + self._train_pos_items = train_pos_items + self.train_batch_size = train_batch_size + self._num_train_negatives = num_train_negatives + self._batches_per_train_step = batches_per_train_step + self._eval_pos_users = eval_pos_users + self._eval_pos_items = eval_pos_items + self.eval_batch_size = eval_batch_size + self.num_train_epochs = num_train_epochs + self.create_data_offline = create_data_offline + + # Training + if self._train_pos_users.shape != self._train_pos_items.shape: + raise ValueError( + "User positives ({}) is different from item positives ({})".format( + self._train_pos_users.shape, self._train_pos_items.shape)) + + (self._train_pos_count,) = self._train_pos_users.shape + self._elements_in_epoch = (1 + num_train_negatives) * self._train_pos_count + self.train_batches_per_epoch = self._count_batches( + self._elements_in_epoch, train_batch_size, batches_per_train_step) + + # Evaluation + if eval_batch_size % (1 + rconst.NUM_EVAL_NEGATIVES): + raise ValueError("Eval batch size {} is not divisible by {}".format( + eval_batch_size, 1 + rconst.NUM_EVAL_NEGATIVES)) + self._eval_users_per_batch = int( + eval_batch_size // (1 + rconst.NUM_EVAL_NEGATIVES)) + self._eval_elements_in_epoch = num_users * (1 + rconst.NUM_EVAL_NEGATIVES) + self.eval_batches_per_epoch = self._count_batches( + self._eval_elements_in_epoch, eval_batch_size, batches_per_eval_step) + + # Intermediate artifacts + self._current_epoch_order = np.empty(shape=(0,)) + self._shuffle_iterator = None + + self._shuffle_with_forkpool = not stream_files + if stream_files: + self._shard_root = epoch_dir or tempfile.mkdtemp(prefix="ncf_") + if not create_data_offline: + atexit.register(tf.io.gfile.rmtree, self._shard_root) + else: + self._shard_root = None + + self._train_dataset = DatasetManager(True, stream_files, + self.train_batches_per_epoch, + self._shard_root, deterministic, + num_train_epochs) + self._eval_dataset = DatasetManager(False, stream_files, + self.eval_batches_per_epoch, + self._shard_root, deterministic, + num_train_epochs) + + # Threading details + super(BaseDataConstructor, self).__init__() + self.daemon = True + self._stop_loop = False + self._fatal_exception = None + self.deterministic = deterministic + + def __str__(self): + multiplier = ("(x{} devices)".format(self._batches_per_train_step) + if self._batches_per_train_step > 1 else "") + summary = SUMMARY_TEMPLATE.format( + spacer=" ", num_users=self._num_users, num_items=self._num_items, + train_pos_ct=self._train_pos_count, + train_batch_size=self.train_batch_size, + train_batch_ct=self.train_batches_per_epoch, + eval_pos_ct=self._num_users, eval_batch_size=self.eval_batch_size, + eval_batch_ct=self.eval_batches_per_epoch, multiplier=multiplier) + return super(BaseDataConstructor, self).__str__() + "\n" + summary + + @staticmethod + def _count_batches(example_count, batch_size, batches_per_step): + """Determine the number of batches, rounding up to fill all devices.""" + x = (example_count + batch_size - 1) // batch_size + return (x + batches_per_step - 1) // batches_per_step * batches_per_step + + def stop_loop(self): + self._stop_loop = True + + def construct_lookup_variables(self): + """Perform any one time pre-compute work.""" + raise NotImplementedError + + def lookup_negative_items(self, **kwargs): + """Randomly sample negative items for given users.""" + raise NotImplementedError + + def _run(self): + atexit.register(self.stop_loop) + self._start_shuffle_iterator() + self.construct_lookup_variables() + self._construct_training_epoch() + self._construct_eval_epoch() + for _ in range(self._maximum_number_epochs - 1): + self._construct_training_epoch() + self.stop_loop() + + def run(self): + try: + self._run() + except Exception as e: + # The Thread base class swallows stack traces, so unfortunately it is + # necessary to catch and re-raise to get debug output + traceback.print_exc() + self._fatal_exception = e + sys.stderr.flush() + raise + + def _start_shuffle_iterator(self): + if self._shuffle_with_forkpool: + pool = popen_helper.get_forkpool(3, closing=False) + else: + pool = popen_helper.get_threadpool(1, closing=False) + atexit.register(pool.close) + args = [(self._elements_in_epoch, stat_utils.random_int32()) + for _ in range(self._maximum_number_epochs)] + imap = pool.imap if self.deterministic else pool.imap_unordered + self._shuffle_iterator = imap(stat_utils.permutation, args) + + def _get_training_batch(self, i): + """Construct a single batch of training data. + + Args: + i: The index of the batch. This is used when stream_files=True to assign + data to file shards. + """ + batch_indices = self._current_epoch_order[i * self.train_batch_size: + (i + 1) * self.train_batch_size] + (mask_start_index,) = batch_indices.shape + + batch_ind_mod = np.mod(batch_indices, self._train_pos_count) + users = self._train_pos_users[batch_ind_mod] + + negative_indices = np.greater_equal(batch_indices, self._train_pos_count) + negative_users = users[negative_indices] + + negative_items = self.lookup_negative_items(negative_users=negative_users) + + items = self._train_pos_items[batch_ind_mod] + items[negative_indices] = negative_items + + labels = np.logical_not(negative_indices) + + # Pad last partial batch + pad_length = self.train_batch_size - mask_start_index + if pad_length: + # We pad with arange rather than zeros because the network will still + # compute logits for padded examples, and padding with zeros would create + # a very "hot" embedding key which can have performance implications. + user_pad = np.arange(pad_length, dtype=users.dtype) % self._num_users + item_pad = np.arange(pad_length, dtype=items.dtype) % self._num_items + label_pad = np.zeros(shape=(pad_length,), dtype=labels.dtype) + users = np.concatenate([users, user_pad]) + items = np.concatenate([items, item_pad]) + labels = np.concatenate([labels, label_pad]) + + self._train_dataset.put( + i, { + movielens.USER_COLUMN: + np.reshape(users, (self.train_batch_size, 1)), + movielens.ITEM_COLUMN: + np.reshape(items, (self.train_batch_size, 1)), + rconst.MASK_START_INDEX: + np.array(mask_start_index, dtype=np.int32), + "labels": + np.reshape(labels, (self.train_batch_size, 1)), + }) + + def _wait_to_construct_train_epoch(self): + count = 0 + while self._train_dataset.buffer_reached() and not self._stop_loop: + time.sleep(0.01) + count += 1 + if count >= 100 and np.log10(count) == np.round(np.log10(count)): + logging.info( + "Waited {} times for training data to be consumed".format(count)) + + def _construct_training_epoch(self): + """Loop to construct a batch of training data.""" + if not self.create_data_offline: + self._wait_to_construct_train_epoch() + + start_time = timeit.default_timer() + if self._stop_loop: + return + + self._train_dataset.start_construction() + map_args = list(range(self.train_batches_per_epoch)) + self._current_epoch_order = next(self._shuffle_iterator) + + get_pool = (popen_helper.get_fauxpool if self.deterministic else + popen_helper.get_threadpool) + with get_pool(6) as pool: + pool.map(self._get_training_batch, map_args) + self._train_dataset.end_construction() + + logging.info("Epoch construction complete. Time: {:.1f} seconds".format( + timeit.default_timer() - start_time)) + + @staticmethod + def _assemble_eval_batch(users, positive_items, negative_items, + users_per_batch): + """Construct duplicate_mask and structure data accordingly. + + The positive items should be last so that they lose ties. However, they + should not be masked out if the true eval positive happens to be + selected as a negative. So instead, the positive is placed in the first + position, and then switched with the last element after the duplicate + mask has been computed. + + Args: + users: An array of users in a batch. (should be identical along axis 1) + positive_items: An array (batch_size x 1) of positive item indices. + negative_items: An array of negative item indices. + users_per_batch: How many users should be in the batch. This is passed + as an argument so that ncf_test.py can use this method. + + Returns: + User, item, and duplicate_mask arrays. + """ + items = np.concatenate([positive_items, negative_items], axis=1) + + # We pad the users and items here so that the duplicate mask calculation + # will include padding. The metric function relies on all padded elements + # except the positive being marked as duplicate to mask out padded points. + if users.shape[0] < users_per_batch: + pad_rows = users_per_batch - users.shape[0] + padding = np.zeros(shape=(pad_rows, users.shape[1]), dtype=np.int32) + users = np.concatenate([users, padding.astype(users.dtype)], axis=0) + items = np.concatenate([items, padding.astype(items.dtype)], axis=0) + + duplicate_mask = stat_utils.mask_duplicates(items, axis=1).astype(np.bool) + + items[:, (0, -1)] = items[:, (-1, 0)] + duplicate_mask[:, (0, -1)] = duplicate_mask[:, (-1, 0)] + + assert users.shape == items.shape == duplicate_mask.shape + return users, items, duplicate_mask + + def _get_eval_batch(self, i): + """Construct a single batch of evaluation data. + + Args: + i: The index of the batch. + """ + low_index = i * self._eval_users_per_batch + high_index = (i + 1) * self._eval_users_per_batch + users = np.repeat(self._eval_pos_users[low_index:high_index, np.newaxis], + 1 + rconst.NUM_EVAL_NEGATIVES, axis=1) + positive_items = self._eval_pos_items[low_index:high_index, np.newaxis] + negative_items = (self.lookup_negative_items(negative_users=users[:, :-1]) + .reshape(-1, rconst.NUM_EVAL_NEGATIVES)) + + users, items, duplicate_mask = self._assemble_eval_batch( + users, positive_items, negative_items, self._eval_users_per_batch) + + self._eval_dataset.put( + i, { + movielens.USER_COLUMN: + np.reshape(users.flatten(), (self.eval_batch_size, 1)), + movielens.ITEM_COLUMN: + np.reshape(items.flatten(), (self.eval_batch_size, 1)), + rconst.DUPLICATE_MASK: + np.reshape(duplicate_mask.flatten(), (self.eval_batch_size, 1)), + }) + + def _construct_eval_epoch(self): + """Loop to construct data for evaluation.""" + if self._stop_loop: + return + + start_time = timeit.default_timer() + + self._eval_dataset.start_construction() + map_args = [i for i in range(self.eval_batches_per_epoch)] + + get_pool = (popen_helper.get_fauxpool if self.deterministic else + popen_helper.get_threadpool) + with get_pool(6) as pool: + pool.map(self._get_eval_batch, map_args) + self._eval_dataset.end_construction() + + logging.info("Eval construction complete. Time: {:.1f} seconds".format( + timeit.default_timer() - start_time)) + + def make_input_fn(self, is_training): + # It isn't feasible to provide a foolproof check, so this is designed to + # catch most failures rather than provide an exhaustive guard. + if self._fatal_exception is not None: + raise ValueError("Fatal exception in the data production loop: {}" + .format(self._fatal_exception)) + + return ( + self._train_dataset.make_input_fn(self.train_batch_size) if is_training + else self._eval_dataset.make_input_fn(self.eval_batch_size)) + + def increment_request_epoch(self): + self._train_dataset.increment_request_epoch() + + +class DummyConstructor(threading.Thread): + """Class for running with synthetic data.""" + + def __init__(self, *args, **kwargs): + super(DummyConstructor, self).__init__(*args, **kwargs) + self.train_batches_per_epoch = rconst.SYNTHETIC_BATCHES_PER_EPOCH + self.eval_batches_per_epoch = rconst.SYNTHETIC_BATCHES_PER_EPOCH + + def run(self): + pass + + def stop_loop(self): + pass + + def increment_request_epoch(self): + pass + + @staticmethod + def make_input_fn(is_training): + """Construct training input_fn that uses synthetic data.""" + + def input_fn(params): + """Returns dummy input batches for training.""" + + # Estimator passes batch_size during training and eval_batch_size during + # eval. + batch_size = (params["batch_size"] if is_training else + params.get("eval_batch_size") or params["batch_size"]) + num_users = params["num_users"] + num_items = params["num_items"] + + users = tf.random.uniform([batch_size, 1], + dtype=tf.int32, + minval=0, + maxval=num_users) + items = tf.random.uniform([batch_size, 1], + dtype=tf.int32, + minval=0, + maxval=num_items) + + if is_training: + valid_point_mask = tf.cast( + tf.random.uniform([batch_size, 1], + dtype=tf.int32, + minval=0, + maxval=2), tf.bool) + labels = tf.cast( + tf.random.uniform([batch_size, 1], + dtype=tf.int32, + minval=0, + maxval=2), tf.bool) + data = { + movielens.USER_COLUMN: users, + movielens.ITEM_COLUMN: items, + rconst.VALID_POINT_MASK: valid_point_mask, + }, labels + else: + dupe_mask = tf.cast( + tf.random.uniform([batch_size, 1], + dtype=tf.int32, + minval=0, + maxval=2), tf.bool) + data = { + movielens.USER_COLUMN: users, + movielens.ITEM_COLUMN: items, + rconst.DUPLICATE_MASK: dupe_mask, + } + + dataset = tf.data.Dataset.from_tensors(data).repeat( + rconst.SYNTHETIC_BATCHES_PER_EPOCH * params["batches_per_step"]) + dataset = dataset.prefetch(32) + return dataset + + return input_fn + + +class MaterializedDataConstructor(BaseDataConstructor): + """Materialize a table of negative examples for fast negative generation. + + This class creates a table (num_users x num_items) containing all of the + negative examples for each user. This table is conceptually ragged; that is to + say the items dimension will have a number of unused elements at the end equal + to the number of positive elements for a given user. For instance: + + num_users = 3 + num_items = 5 + positives = [[1, 3], [0], [1, 2, 3, 4]] + + will generate a negative table: + [ + [0 2 4 int32max int32max], + [1 2 3 4 int32max], + [0 int32max int32max int32max int32max], + ] + + and a vector of per-user negative counts, which in this case would be: + [3, 4, 1] + + When sampling negatives, integers are (nearly) uniformly selected from the + range [0, per_user_neg_count[user]) which gives a column_index, at which + point the negative can be selected as: + negative_table[user, column_index] + + This technique will not scale; however MovieLens is small enough that even + a pre-compute which is quadratic in problem size will still fit in memory. A + more scalable lookup method is in the works. + """ + def __init__(self, *args, **kwargs): + super(MaterializedDataConstructor, self).__init__(*args, **kwargs) + self._negative_table = None + self._per_user_neg_count = None + + def construct_lookup_variables(self): + # Materialize negatives for fast lookup sampling. + start_time = timeit.default_timer() + inner_bounds = np.argwhere(self._train_pos_users[1:] - + self._train_pos_users[:-1])[:, 0] + 1 + (upper_bound,) = self._train_pos_users.shape + index_bounds = [0] + inner_bounds.tolist() + [upper_bound] + self._negative_table = np.zeros(shape=(self._num_users, self._num_items), + dtype=rconst.ITEM_DTYPE) + + # Set the table to the max value to make sure the embedding lookup will fail + # if we go out of bounds, rather than just overloading item zero. + self._negative_table += np.iinfo(rconst.ITEM_DTYPE).max + assert self._num_items < np.iinfo(rconst.ITEM_DTYPE).max + + # Reuse arange during generation. np.delete will make a copy. + full_set = np.arange(self._num_items, dtype=rconst.ITEM_DTYPE) + + self._per_user_neg_count = np.zeros( + shape=(self._num_users,), dtype=np.int32) + + # Threading does not improve this loop. For some reason, the np.delete + # call does not parallelize well. Multiprocessing incurs too much + # serialization overhead to be worthwhile. + for i in range(self._num_users): + positives = self._train_pos_items[index_bounds[i]:index_bounds[i+1]] + negatives = np.delete(full_set, positives) + self._per_user_neg_count[i] = self._num_items - positives.shape[0] + self._negative_table[i, :self._per_user_neg_count[i]] = negatives + + logging.info("Negative sample table built. Time: {:.1f} seconds".format( + timeit.default_timer() - start_time)) + + def lookup_negative_items(self, negative_users, **kwargs): + negative_item_choice = stat_utils.very_slightly_biased_randint( + self._per_user_neg_count[negative_users]) + return self._negative_table[negative_users, negative_item_choice] + + +class BisectionDataConstructor(BaseDataConstructor): + """Use bisection to index within positive examples. + + This class tallies the number of negative items which appear before each + positive item for a user. This means that in order to select the ith negative + item for a user, it only needs to determine which two positive items bound + it at which point the item id for the ith negative is a simply algebraic + expression. + """ + def __init__(self, *args, **kwargs): + super(BisectionDataConstructor, self).__init__(*args, **kwargs) + self.index_bounds = None + self._sorted_train_pos_items = None + self._total_negatives = None + + def _index_segment(self, user): + lower, upper = self.index_bounds[user:user+2] + items = self._sorted_train_pos_items[lower:upper] + + negatives_since_last_positive = np.concatenate( + [items[0][np.newaxis], items[1:] - items[:-1] - 1]) + + return np.cumsum(negatives_since_last_positive) + + def construct_lookup_variables(self): + start_time = timeit.default_timer() + inner_bounds = np.argwhere(self._train_pos_users[1:] - + self._train_pos_users[:-1])[:, 0] + 1 + (upper_bound,) = self._train_pos_users.shape + self.index_bounds = np.array([0] + inner_bounds.tolist() + [upper_bound]) + + # Later logic will assume that the users are in sequential ascending order. + assert np.array_equal(self._train_pos_users[self.index_bounds[:-1]], + np.arange(self._num_users)) + + self._sorted_train_pos_items = self._train_pos_items.copy() + + for i in range(self._num_users): + lower, upper = self.index_bounds[i:i+2] + self._sorted_train_pos_items[lower:upper].sort() + + self._total_negatives = np.concatenate([ + self._index_segment(i) for i in range(self._num_users)]) + + logging.info("Negative total vector built. Time: {:.1f} seconds".format( + timeit.default_timer() - start_time)) + + def lookup_negative_items(self, negative_users, **kwargs): + output = np.zeros(shape=negative_users.shape, dtype=rconst.ITEM_DTYPE) - 1 + + left_index = self.index_bounds[negative_users] + right_index = self.index_bounds[negative_users + 1] - 1 + + num_positives = right_index - left_index + 1 + num_negatives = self._num_items - num_positives + neg_item_choice = stat_utils.very_slightly_biased_randint(num_negatives) + + # Shortcuts: + # For points where the negative is greater than or equal to the tally before + # the last positive point there is no need to bisect. Instead the item id + # corresponding to the negative item choice is simply: + # last_postive_index + 1 + (neg_choice - last_negative_tally) + # Similarly, if the selection is less than the tally at the first positive + # then the item_id is simply the selection. + # + # Because MovieLens organizes popular movies into low integers (which is + # preserved through the preprocessing), the first shortcut is very + # efficient, allowing ~60% of samples to bypass the bisection. For the same + # reason, the second shortcut is rarely triggered (<0.02%) and is therefore + # not worth implementing. + use_shortcut = neg_item_choice >= self._total_negatives[right_index] + output[use_shortcut] = ( + self._sorted_train_pos_items[right_index] + 1 + + (neg_item_choice - self._total_negatives[right_index]) + )[use_shortcut] + + if np.all(use_shortcut): + # The bisection code is ill-posed when there are no elements. + return output + + not_use_shortcut = np.logical_not(use_shortcut) + left_index = left_index[not_use_shortcut] + right_index = right_index[not_use_shortcut] + neg_item_choice = neg_item_choice[not_use_shortcut] + + num_loops = np.max( + np.ceil(np.log2(num_positives[not_use_shortcut])).astype(np.int32)) + + for i in range(num_loops): + mid_index = (left_index + right_index) // 2 + right_criteria = self._total_negatives[mid_index] > neg_item_choice + left_criteria = np.logical_not(right_criteria) + + right_index[right_criteria] = mid_index[right_criteria] + left_index[left_criteria] = mid_index[left_criteria] + + # Expected state after bisection pass: + # The right index is the smallest index whose tally is greater than the + # negative item choice index. + + assert np.all((right_index - left_index) <= 1) + + output[not_use_shortcut] = ( + self._sorted_train_pos_items[right_index] - + (self._total_negatives[right_index] - neg_item_choice) + ) + + assert np.all(output >= 0) + + return output + + +def get_constructor(name): + if name == "bisection": + return BisectionDataConstructor + if name == "materialized": + return MaterializedDataConstructor + raise ValueError("Unrecognized constructor: {}".format(name)) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/data_preprocessing.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/data_preprocessing.py new file mode 100644 index 0000000..8dd9a9e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/data_preprocessing.py @@ -0,0 +1,243 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Preprocess dataset and construct any necessary artifacts.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import pickle +import time +import timeit +import typing + +# pylint: disable=wrong-import-order +import numpy as np +import pandas as pd +import tensorflow as tf +from absl import logging +# pylint: enable=wrong-import-order + +from official.recommendation import constants as rconst +from official.recommendation import data_pipeline +from official.recommendation import movielens +from official.utils.logs import mlperf_helper + + +DATASET_TO_NUM_USERS_AND_ITEMS = { + "ml-1m": (6040, 3706), + "ml-20m": (138493, 26744) +} + + +_EXPECTED_CACHE_KEYS = ( + rconst.TRAIN_USER_KEY, rconst.TRAIN_ITEM_KEY, rconst.EVAL_USER_KEY, + rconst.EVAL_ITEM_KEY, rconst.USER_MAP, rconst.ITEM_MAP) + + +def _filter_index_sort(raw_rating_path, cache_path): + # type: (str, str, bool) -> (dict, bool) + """Read in data CSV, and output structured data. + + This function reads in the raw CSV of positive items, and performs three + preprocessing transformations: + + 1) Filter out all users who have not rated at least a certain number + of items. (Typically 20 items) + + 2) Zero index the users and items such that the largest user_id is + `num_users - 1` and the largest item_id is `num_items - 1` + + 3) Sort the dataframe by user_id, with timestamp as a secondary sort key. + This allows the dataframe to be sliced by user in-place, and for the last + item to be selected simply by calling the `-1` index of a user's slice. + + While all of these transformations are performed by Pandas (and are therefore + single-threaded), they only take ~2 minutes, and the overhead to apply a + MapReduce pattern to parallel process the dataset adds significant complexity + for no computational gain. For a larger dataset parallelizing this + preprocessing could yield speedups. (Also, this preprocessing step is only + performed once for an entire run. + + Args: + raw_rating_path: The path to the CSV which contains the raw dataset. + cache_path: The path to the file where results of this function are saved. + + Returns: + A filtered, zero-index remapped, sorted dataframe, a dict mapping raw user + IDs to regularized user IDs, and a dict mapping raw item IDs to regularized + item IDs. + """ + valid_cache = tf.io.gfile.exists(cache_path) + if valid_cache: + with tf.io.gfile.GFile(cache_path, "rb") as f: + cached_data = pickle.load(f) + + # (nnigania)disabled this check as the dataset is not expected to change + # cache_age = time.time() - cached_data.get("create_time", 0) + # if cache_age > rconst.CACHE_INVALIDATION_SEC: + # valid_cache = False + + for key in _EXPECTED_CACHE_KEYS: + if key not in cached_data: + valid_cache = False + + if not valid_cache: + logging.info("Removing stale raw data cache file.") + tf.io.gfile.remove(cache_path) + + if valid_cache: + data = cached_data + else: + with tf.io.gfile.GFile(raw_rating_path) as f: + df = pd.read_csv(f) + + # Get the info of users who have more than 20 ratings on items + grouped = df.groupby(movielens.USER_COLUMN) + df = grouped.filter( + lambda x: len(x) >= rconst.MIN_NUM_RATINGS) # type: pd.DataFrame + + original_users = df[movielens.USER_COLUMN].unique() + original_items = df[movielens.ITEM_COLUMN].unique() + + # Map the ids of user and item to 0 based index for following processing + logging.info("Generating user_map and item_map...") + user_map = {user: index for index, user in enumerate(original_users)} + item_map = {item: index for index, item in enumerate(original_items)} + + df[movielens.USER_COLUMN] = df[movielens.USER_COLUMN].apply( + lambda user: user_map[user]) + df[movielens.ITEM_COLUMN] = df[movielens.ITEM_COLUMN].apply( + lambda item: item_map[item]) + + num_users = len(original_users) + num_items = len(original_items) + + mlperf_helper.ncf_print(key=mlperf_helper.TAGS.PREPROC_HP_NUM_EVAL, + value=rconst.NUM_EVAL_NEGATIVES) + + assert num_users <= np.iinfo(rconst.USER_DTYPE).max + assert num_items <= np.iinfo(rconst.ITEM_DTYPE).max + assert df[movielens.USER_COLUMN].max() == num_users - 1 + assert df[movielens.ITEM_COLUMN].max() == num_items - 1 + + # This sort is used to shard the dataframe by user, and later to select + # the last item for a user to be used in validation. + logging.info("Sorting by user, timestamp...") + + # This sort is equivalent to + # df.sort_values([movielens.USER_COLUMN, movielens.TIMESTAMP_COLUMN], + # inplace=True) + # except that the order of items with the same user and timestamp are + # sometimes different. For some reason, this sort results in a better + # hit-rate during evaluation, matching the performance of the MLPerf + # reference implementation. + df.sort_values(by=movielens.TIMESTAMP_COLUMN, inplace=True) + df.sort_values([movielens.USER_COLUMN, movielens.TIMESTAMP_COLUMN], + inplace=True, kind="mergesort") + + # The dataframe does not reconstruct indices in the sort or filter steps. + df = df.reset_index() + + grouped = df.groupby(movielens.USER_COLUMN, group_keys=False) + eval_df, train_df = grouped.tail(1), grouped.apply(lambda x: x.iloc[:-1]) + + data = { + rconst.TRAIN_USER_KEY: train_df[movielens.USER_COLUMN] + .values.astype(rconst.USER_DTYPE), + rconst.TRAIN_ITEM_KEY: train_df[movielens.ITEM_COLUMN] + .values.astype(rconst.ITEM_DTYPE), + rconst.EVAL_USER_KEY: eval_df[movielens.USER_COLUMN] + .values.astype(rconst.USER_DTYPE), + rconst.EVAL_ITEM_KEY: eval_df[movielens.ITEM_COLUMN] + .values.astype(rconst.ITEM_DTYPE), + rconst.USER_MAP: user_map, + rconst.ITEM_MAP: item_map, + "create_time": time.time(), + } + + logging.info("Writing raw data cache.") + with tf.io.gfile.GFile(cache_path, "wb") as f: + pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL) + + # TODO(robieta): MLPerf cache clear. + return data, valid_cache + + +def instantiate_pipeline(dataset, + data_dir, + params, + constructor_type=None, + deterministic=False, + epoch_dir=None, + generate_data_offline=False): + # type: (str, str, dict, typing.Optional[str], bool, typing.Optional[str], bool) -> (int, int, data_pipeline.BaseDataConstructor) + """Load and digest data CSV into a usable form. + + Args: + dataset: The name of the dataset to be used. + data_dir: The root directory of the dataset. + params: dict of parameters for the run. + constructor_type: The name of the constructor subclass that should be used + for the input pipeline. + deterministic: Tell the data constructor to produce deterministically. + epoch_dir: Directory in which to store the training epochs. + generate_data_offline: Boolean, whether current pipeline is done offline + or while training. + """ + logging.info("Beginning data preprocessing.") + + st = timeit.default_timer() + raw_rating_path = os.path.join(data_dir, dataset, movielens.RATINGS_FILE) + cache_path = os.path.join(data_dir, dataset, rconst.RAW_CACHE_FILE) + + raw_data, _ = _filter_index_sort(raw_rating_path, cache_path) + user_map, item_map = raw_data["user_map"], raw_data["item_map"] + num_users, num_items = DATASET_TO_NUM_USERS_AND_ITEMS[dataset] + + if num_users != len(user_map): + raise ValueError("Expected to find {} users, but found {}".format( + num_users, len(user_map))) + if num_items != len(item_map): + raise ValueError("Expected to find {} items, but found {}".format( + num_items, len(item_map))) + + producer = data_pipeline.get_constructor(constructor_type or "materialized")( + maximum_number_epochs=params["train_epochs"], + num_users=num_users, + num_items=num_items, + user_map=user_map, + item_map=item_map, + train_pos_users=raw_data[rconst.TRAIN_USER_KEY], + train_pos_items=raw_data[rconst.TRAIN_ITEM_KEY], + train_batch_size=params["batch_size"], + batches_per_train_step=params["batches_per_step"], + num_train_negatives=params["num_neg"], + eval_pos_users=raw_data[rconst.EVAL_USER_KEY], + eval_pos_items=raw_data[rconst.EVAL_ITEM_KEY], + eval_batch_size=params["eval_batch_size"], + batches_per_eval_step=params["batches_per_step"], + stream_files=params["stream_files"], + deterministic=deterministic, + epoch_dir=epoch_dir, + create_data_offline=generate_data_offline) + + run_time = timeit.default_timer() - st + logging.info("Data preprocessing complete. Time: {:.1f} sec." + .format(run_time)) + + print(producer) + return num_users, num_items, producer diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/data_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/data_test.py new file mode 100644 index 0000000..11cd4cf --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/data_test.py @@ -0,0 +1,358 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Test NCF data pipeline.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from collections import defaultdict +import hashlib +import os + +import mock +import numpy as np +import scipy.stats +import tensorflow as tf + +from official.recommendation import constants as rconst +from official.recommendation import data_preprocessing +from official.recommendation import movielens +from official.recommendation import popen_helper +from official.utils.misc import keras_utils + + +DATASET = "ml-test" +NUM_USERS = 1000 +NUM_ITEMS = 2000 +NUM_PTS = 50000 +BATCH_SIZE = 2048 +EVAL_BATCH_SIZE = 4000 +NUM_NEG = 4 + + +END_TO_END_TRAIN_MD5 = "b218738e915e825d03939c5e305a2698" +END_TO_END_EVAL_MD5 = "d753d0f3186831466d6e218163a9501e" +FRESH_RANDOMNESS_MD5 = "63d0dff73c0e5f1048fbdc8c65021e22" + + +def mock_download(*args, **kwargs): + return + + +# The forkpool used by data producers interacts badly with the threading +# used by TestCase. Without this patch tests will hang, and no amount +# of diligent closing and joining within the producer will prevent it. +@mock.patch.object(popen_helper, "get_forkpool", popen_helper.get_fauxpool) +class BaseTest(tf.test.TestCase): + + def setUp(self): + if keras_utils.is_v2_0: + tf.compat.v1.disable_eager_execution() + self.temp_data_dir = self.get_temp_dir() + ratings_folder = os.path.join(self.temp_data_dir, DATASET) + tf.io.gfile.makedirs(ratings_folder) + np.random.seed(0) + raw_user_ids = np.arange(NUM_USERS * 3) + np.random.shuffle(raw_user_ids) + raw_user_ids = raw_user_ids[:NUM_USERS] + + raw_item_ids = np.arange(NUM_ITEMS * 3) + np.random.shuffle(raw_item_ids) + raw_item_ids = raw_item_ids[:NUM_ITEMS] + + users = np.random.choice(raw_user_ids, NUM_PTS) + items = np.random.choice(raw_item_ids, NUM_PTS) + scores = np.random.randint(low=0, high=5, size=NUM_PTS) + times = np.random.randint(low=1000000000, high=1200000000, size=NUM_PTS) + + self.rating_file = os.path.join(ratings_folder, movielens.RATINGS_FILE) + self.seen_pairs = set() + self.holdout = {} + with tf.io.gfile.GFile(self.rating_file, "w") as f: + f.write("user_id,item_id,rating,timestamp\n") + for usr, itm, scr, ts in zip(users, items, scores, times): + pair = (usr, itm) + if pair in self.seen_pairs: + continue + self.seen_pairs.add(pair) + if usr not in self.holdout or (ts, itm) > self.holdout[usr]: + self.holdout[usr] = (ts, itm) + + f.write("{},{},{},{}\n".format(usr, itm, scr, ts)) + + movielens.download = mock_download + movielens.NUM_RATINGS[DATASET] = NUM_PTS + data_preprocessing.DATASET_TO_NUM_USERS_AND_ITEMS[DATASET] = (NUM_USERS, + NUM_ITEMS) + + def make_params(self, train_epochs=1): + return { + "train_epochs": train_epochs, + "batches_per_step": 1, + "use_seed": False, + "batch_size": BATCH_SIZE, + "eval_batch_size": EVAL_BATCH_SIZE, + "num_neg": NUM_NEG, + "match_mlperf": True, + "use_tpu": False, + "use_xla_for_gpu": False, + "stream_files": False, + } + + def test_preprocessing(self): + # For the most part the necessary checks are performed within + # _filter_index_sort() + + cache_path = os.path.join(self.temp_data_dir, "test_cache.pickle") + data, valid_cache = data_preprocessing._filter_index_sort( + self.rating_file, cache_path=cache_path) + + assert len(data[rconst.USER_MAP]) == NUM_USERS + assert len(data[rconst.ITEM_MAP]) == NUM_ITEMS + + def drain_dataset(self, dataset, g): + # type: (tf.data.Dataset, tf.Graph) -> list + with self.session(graph=g) as sess: + with g.as_default(): + batch = tf.compat.v1.data.make_one_shot_iterator(dataset).get_next() + output = [] + while True: + try: + output.append(sess.run(batch)) + except tf.errors.OutOfRangeError: + break + return output + + def _test_end_to_end(self, constructor_type): + params = self.make_params(train_epochs=1) + _, _, producer = data_preprocessing.instantiate_pipeline( + dataset=DATASET, data_dir=self.temp_data_dir, params=params, + constructor_type=constructor_type, deterministic=True) + + producer.start() + producer.join() + assert producer._fatal_exception is None + + user_inv_map = {v: k for k, v in producer.user_map.items()} + item_inv_map = {v: k for k, v in producer.item_map.items()} + + # ========================================================================== + # == Training Data ========================================================= + # ========================================================================== + g = tf.Graph() + with g.as_default(): + input_fn = producer.make_input_fn(is_training=True) + dataset = input_fn(params) + + first_epoch = self.drain_dataset(dataset=dataset, g=g) + + counts = defaultdict(int) + train_examples = { + True: set(), + False: set(), + } + + md5 = hashlib.md5() + for features, labels in first_epoch: + data_list = [ + features[movielens.USER_COLUMN].flatten(), + features[movielens.ITEM_COLUMN].flatten(), + features[rconst.VALID_POINT_MASK].flatten(), + labels.flatten() + ] + for i in data_list: + md5.update(i.tobytes()) + + for u, i, v, l in zip(*data_list): + if not v: + continue # ignore padding + + u_raw = user_inv_map[u] + i_raw = item_inv_map[i] + if ((u_raw, i_raw) in self.seen_pairs) != l: + # The evaluation item is not considered during false negative + # generation, so it will occasionally appear as a negative example + # during training. + assert not l + self.assertEqual(i_raw, self.holdout[u_raw][1]) + train_examples[l].add((u_raw, i_raw)) + counts[(u_raw, i_raw)] += 1 + + self.assertRegexpMatches(md5.hexdigest(), END_TO_END_TRAIN_MD5) + + num_positives_seen = len(train_examples[True]) + self.assertEqual(producer._train_pos_users.shape[0], num_positives_seen) + + # This check is more heuristic because negatives are sampled with + # replacement. It only checks that negative generation is reasonably random. + self.assertGreater( + len(train_examples[False]) / NUM_NEG / num_positives_seen, 0.9) + + # This checks that the samples produced are independent by checking the + # number of duplicate entries. If workers are not properly independent there + # will be lots of repeated pairs. + self.assertLess(np.mean(list(counts.values())), 1.1) + + # ========================================================================== + # == Eval Data ============================================================= + # ========================================================================== + with g.as_default(): + input_fn = producer.make_input_fn(is_training=False) + dataset = input_fn(params) + + eval_data = self.drain_dataset(dataset=dataset, g=g) + + current_user = None + md5 = hashlib.md5() + for features in eval_data: + data_list = [ + features[movielens.USER_COLUMN].flatten(), + features[movielens.ITEM_COLUMN].flatten(), + features[rconst.DUPLICATE_MASK].flatten() + ] + for i in data_list: + md5.update(i.tobytes()) + + for idx, (u, i, d) in enumerate(zip(*data_list)): + u_raw = user_inv_map[u] + i_raw = item_inv_map[i] + if current_user is None: + current_user = u + + # Ensure that users appear in blocks, as the evaluation logic expects + # this structure. + self.assertEqual(u, current_user) + + # The structure of evaluation data is 999 negative examples followed + # by the holdout positive. + if not (idx + 1) % (rconst.NUM_EVAL_NEGATIVES + 1): + # Check that the last element in each chunk is the holdout item. + self.assertEqual(i_raw, self.holdout[u_raw][1]) + current_user = None + + elif i_raw == self.holdout[u_raw][1]: + # Because the holdout item is not given to the negative generation + # process, it can appear as a negative. In that case, it should be + # masked out as a duplicate. (Since the true positive is placed at + # the end and would therefore lose the tie.) + assert d + + else: + # Otherwise check that the other 999 points for a user are selected + # from the negatives. + assert (u_raw, i_raw) not in self.seen_pairs + + self.assertRegexpMatches(md5.hexdigest(), END_TO_END_EVAL_MD5) + + def _test_fresh_randomness(self, constructor_type): + train_epochs = 5 + params = self.make_params(train_epochs=train_epochs) + _, _, producer = data_preprocessing.instantiate_pipeline( + dataset=DATASET, data_dir=self.temp_data_dir, params=params, + constructor_type=constructor_type, deterministic=True) + + producer.start() + + results = [] + g = tf.Graph() + with g.as_default(): + for _ in range(train_epochs): + input_fn = producer.make_input_fn(is_training=True) + dataset = input_fn(params) + results.extend(self.drain_dataset(dataset=dataset, g=g)) + + producer.join() + assert producer._fatal_exception is None + + positive_counts, negative_counts = defaultdict(int), defaultdict(int) + md5 = hashlib.md5() + for features, labels in results: + data_list = [ + features[movielens.USER_COLUMN].flatten(), + features[movielens.ITEM_COLUMN].flatten(), + features[rconst.VALID_POINT_MASK].flatten(), + labels.flatten() + ] + for i in data_list: + md5.update(i.tobytes()) + + for u, i, v, l in zip(*data_list): + if not v: + continue # ignore padding + + if l: + positive_counts[(u, i)] += 1 + else: + negative_counts[(u, i)] += 1 + + self.assertRegexpMatches(md5.hexdigest(), FRESH_RANDOMNESS_MD5) + + # The positive examples should appear exactly once each epoch + self.assertAllEqual(list(positive_counts.values()), + [train_epochs for _ in positive_counts]) + + # The threshold for the negatives is heuristic, but in general repeats are + # expected, but should not appear too frequently. + + pair_cardinality = NUM_USERS * NUM_ITEMS + neg_pair_cardinality = pair_cardinality - len(self.seen_pairs) + + # Approximation for the expectation number of times that a particular + # negative will appear in a given epoch. Implicit in this calculation is the + # treatment of all negative pairs as equally likely. Normally is not + # necessarily reasonable; however the generation in self.setUp() will + # approximate this behavior sufficiently for heuristic testing. + e_sample = len(self.seen_pairs) * NUM_NEG / neg_pair_cardinality + + # The frequency of occurance of a given negative pair should follow an + # approximately binomial distribution in the limit that the cardinality of + # the negative pair set >> number of samples per epoch. + approx_pdf = scipy.stats.binom.pmf(k=np.arange(train_epochs+1), + n=train_epochs, p=e_sample) + + # Tally the actual observed counts. + count_distribution = [0 for _ in range(train_epochs + 1)] + for i in negative_counts.values(): + i = min([i, train_epochs]) # round down tail for simplicity. + count_distribution[i] += 1 + count_distribution[0] = neg_pair_cardinality - sum(count_distribution[1:]) + + # Check that the frequency of negative pairs is approximately binomial. + for i in range(train_epochs + 1): + if approx_pdf[i] < 0.05: + continue # Variance will be high at the tails. + + observed_fraction = count_distribution[i] / neg_pair_cardinality + deviation = (2 * abs(observed_fraction - approx_pdf[i]) / + (observed_fraction + approx_pdf[i])) + + self.assertLess(deviation, 0.2) + + def test_end_to_end_materialized(self): + self._test_end_to_end("materialized") + + def test_end_to_end_bisection(self): + self._test_end_to_end("bisection") + + def test_fresh_randomness_materialized(self): + self._test_fresh_randomness("materialized") + + def test_fresh_randomness_bisection(self): + self._test_fresh_randomness("bisection") + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/movielens.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/movielens.py new file mode 100644 index 0000000..acfa8d8 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/movielens.py @@ -0,0 +1,309 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Download and extract the MovieLens dataset from GroupLens website. + +Download the dataset, and perform basic preprocessing. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys +import tempfile +import zipfile + +# pylint: disable=g-bad-import-order +import numpy as np +import pandas as pd +import six +from six.moves import urllib # pylint: disable=redefined-builtin +from absl import app as absl_app +from absl import flags +from absl import logging +import tensorflow as tf +# pylint: enable=g-bad-import-order + +from official.utils.flags import core as flags_core + + + +ML_1M = "ml-1m" +ML_20M = "ml-20m" +DATASETS = [ML_1M, ML_20M] + +RATINGS_FILE = "ratings.csv" +MOVIES_FILE = "movies.csv" + +# URL to download dataset +_DATA_URL = "http://files.grouplens.org/datasets/movielens/" + +GENRE_COLUMN = "genres" +ITEM_COLUMN = "item_id" # movies +RATING_COLUMN = "rating" +TIMESTAMP_COLUMN = "timestamp" +TITLE_COLUMN = "titles" +USER_COLUMN = "user_id" + +GENRES = [ + 'Action', 'Adventure', 'Animation', "Children", 'Comedy', 'Crime', + 'Documentary', 'Drama', 'Fantasy', 'Film-Noir', 'Horror', "IMAX", 'Musical', + 'Mystery', 'Romance', 'Sci-Fi', 'Thriller', 'War', 'Western' +] +N_GENRE = len(GENRES) + +RATING_COLUMNS = [USER_COLUMN, ITEM_COLUMN, RATING_COLUMN, TIMESTAMP_COLUMN] +MOVIE_COLUMNS = [ITEM_COLUMN, TITLE_COLUMN, GENRE_COLUMN] + +# Note: Users are indexed [1, k], not [0, k-1] +NUM_USER_IDS = { + ML_1M: 6040, + ML_20M: 138493, +} + +# Note: Movies are indexed [1, k], not [0, k-1] +# Both the 1m and 20m datasets use the same movie set. +NUM_ITEM_IDS = 3952 + +MAX_RATING = 5 + +NUM_RATINGS = { + ML_1M: 1000209, + ML_20M: 20000263 +} + + +def _download_and_clean(dataset, data_dir): + """Download MovieLens dataset in a standard format. + + This function downloads the specified MovieLens format and coerces it into a + standard format. The only difference between the ml-1m and ml-20m datasets + after this point (other than size, of course) is that the 1m dataset uses + whole number ratings while the 20m dataset allows half integer ratings. + """ + if dataset not in DATASETS: + raise ValueError("dataset {} is not in {{{}}}".format( + dataset, ",".join(DATASETS))) + + data_subdir = os.path.join(data_dir, dataset) + + expected_files = ["{}.zip".format(dataset), RATINGS_FILE, MOVIES_FILE] + + tf.io.gfile.makedirs(data_subdir) + if set(expected_files).intersection( + tf.io.gfile.listdir(data_subdir)) == set(expected_files): + logging.info("Dataset {} has already been downloaded".format(dataset)) + return + + url = "{}{}.zip".format(_DATA_URL, dataset) + + temp_dir = tempfile.mkdtemp() + try: + zip_path = os.path.join(temp_dir, "{}.zip".format(dataset)) + zip_path, _ = urllib.request.urlretrieve(url, zip_path) + statinfo = os.stat(zip_path) + # A new line to clear the carriage return from download progress + # logging.info is not applicable here + print() + logging.info( + "Successfully downloaded {} {} bytes".format( + zip_path, statinfo.st_size)) + + zipfile.ZipFile(zip_path, "r").extractall(temp_dir) + + if dataset == ML_1M: + _regularize_1m_dataset(temp_dir) + else: + _regularize_20m_dataset(temp_dir) + + for fname in tf.io.gfile.listdir(temp_dir): + if not tf.io.gfile.exists(os.path.join(data_subdir, fname)): + tf.io.gfile.copy(os.path.join(temp_dir, fname), + os.path.join(data_subdir, fname)) + else: + logging.info("Skipping copy of {}, as it already exists in the " + "destination folder.".format(fname)) + + finally: + tf.io.gfile.rmtree(temp_dir) + + +def _transform_csv(input_path, output_path, names, skip_first, separator=","): + """Transform csv to a regularized format. + + Args: + input_path: The path of the raw csv. + output_path: The path of the cleaned csv. + names: The csv column names. + skip_first: Boolean of whether to skip the first line of the raw csv. + separator: Character used to separate fields in the raw csv. + """ + if six.PY2: + names = [six.ensure_text(n, "utf-8") for n in names] + + with tf.io.gfile.GFile(output_path, "wb") as f_out, \ + tf.io.gfile.GFile(input_path, "rb") as f_in: + + # Write column names to the csv. + f_out.write(",".join(names).encode("utf-8")) + f_out.write(b"\n") + for i, line in enumerate(f_in): + if i == 0 and skip_first: + continue # ignore existing labels in the csv + + line = six.ensure_text(line, "utf-8", errors="ignore") + fields = line.split(separator) + if separator != ",": + fields = ['"{}"'.format(field) if "," in field else field + for field in fields] + f_out.write(",".join(fields).encode("utf-8")) + + +def _regularize_1m_dataset(temp_dir): + """ + ratings.dat + The file has no header row, and each line is in the following format: + UserID::MovieID::Rating::Timestamp + - UserIDs range from 1 and 6040 + - MovieIDs range from 1 and 3952 + - Ratings are made on a 5-star scale (whole-star ratings only) + - Timestamp is represented in seconds since midnight Coordinated Universal + Time (UTC) of January 1, 1970. + - Each user has at least 20 ratings + + movies.dat + Each line has the following format: + MovieID::Title::Genres + - MovieIDs range from 1 and 3952 + """ + working_dir = os.path.join(temp_dir, ML_1M) + + _transform_csv( + input_path=os.path.join(working_dir, "ratings.dat"), + output_path=os.path.join(temp_dir, RATINGS_FILE), + names=RATING_COLUMNS, skip_first=False, separator="::") + + _transform_csv( + input_path=os.path.join(working_dir, "movies.dat"), + output_path=os.path.join(temp_dir, MOVIES_FILE), + names=MOVIE_COLUMNS, skip_first=False, separator="::") + + tf.io.gfile.rmtree(working_dir) + + +def _regularize_20m_dataset(temp_dir): + """ + ratings.csv + Each line of this file after the header row represents one rating of one + movie by one user, and has the following format: + userId,movieId,rating,timestamp + - The lines within this file are ordered first by userId, then, within user, + by movieId. + - Ratings are made on a 5-star scale, with half-star increments + (0.5 stars - 5.0 stars). + - Timestamps represent seconds since midnight Coordinated Universal Time + (UTC) of January 1, 1970. + - All the users had rated at least 20 movies. + + movies.csv + Each line has the following format: + MovieID,Title,Genres + - MovieIDs range from 1 and 3952 + """ + working_dir = os.path.join(temp_dir, ML_20M) + + _transform_csv( + input_path=os.path.join(working_dir, "ratings.csv"), + output_path=os.path.join(temp_dir, RATINGS_FILE), + names=RATING_COLUMNS, skip_first=True, separator=",") + + _transform_csv( + input_path=os.path.join(working_dir, "movies.csv"), + output_path=os.path.join(temp_dir, MOVIES_FILE), + names=MOVIE_COLUMNS, skip_first=True, separator=",") + + tf.io.gfile.rmtree(working_dir) + + +def download(dataset, data_dir): + if dataset: + _download_and_clean(dataset, data_dir) + else: + _ = [_download_and_clean(d, data_dir) for d in DATASETS] + + +def ratings_csv_to_dataframe(data_dir, dataset): + with tf.io.gfile.GFile(os.path.join(data_dir, dataset, RATINGS_FILE)) as f: + return pd.read_csv(f, encoding="utf-8") + + +def csv_to_joint_dataframe(data_dir, dataset): + ratings = ratings_csv_to_dataframe(data_dir, dataset) + + with tf.io.gfile.GFile(os.path.join(data_dir, dataset, MOVIES_FILE)) as f: + movies = pd.read_csv(f, encoding="utf-8") + + df = ratings.merge(movies, on=ITEM_COLUMN) + df[RATING_COLUMN] = df[RATING_COLUMN].astype(np.float32) + + return df + + +def integerize_genres(dataframe): + """Replace genre string with a binary vector. + + Args: + dataframe: a pandas dataframe of movie data. + + Returns: + The transformed dataframe. + """ + def _map_fn(entry): + entry.replace("Children's", "Children") # naming difference. + movie_genres = entry.split("|") + output = np.zeros((len(GENRES),), dtype=np.int64) + for i, genre in enumerate(GENRES): + if genre in movie_genres: + output[i] = 1 + return output + + dataframe[GENRE_COLUMN] = dataframe[GENRE_COLUMN].apply(_map_fn) + + return dataframe + + +def define_data_download_flags(): + """Add flags specifying data download arguments.""" + flags.DEFINE_string( + name="data_dir", default="/tmp/movielens-data/", + help=flags_core.help_wrap( + "Directory to download and extract data.")) + + flags.DEFINE_enum( + name="dataset", default=None, + enum_values=DATASETS, case_sensitive=False, + help=flags_core.help_wrap("Dataset to be trained and evaluated.")) + + +def main(_): + """Download and extract the data from GroupLens website.""" + download(flags.FLAGS.dataset, flags.FLAGS.data_dir) + + +if __name__ == "__main__": + define_data_download_flags() + FLAGS = flags.FLAGS + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/ncf_common.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/ncf_common.py new file mode 100644 index 0000000..0819a51 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/ncf_common.py @@ -0,0 +1,331 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Common functionalities used by both Keras and Estimator implementations. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import os + +# pylint: disable=g-bad-import-order +import numpy as np +from absl import flags +from absl import logging +import tensorflow as tf +# pylint: enable=g-bad-import-order + +from official.recommendation import constants as rconst +from official.recommendation import data_pipeline +from official.recommendation import data_preprocessing +from official.recommendation import movielens +from official.utils.flags import core as flags_core +from official.utils.misc import distribution_utils +from official.utils.misc import keras_utils + +FLAGS = flags.FLAGS + + +def get_inputs(params): + """Returns some parameters used by the model.""" + if FLAGS.download_if_missing and not FLAGS.use_synthetic_data: + movielens.download(FLAGS.dataset, FLAGS.data_dir) + + if FLAGS.seed is not None: + np.random.seed(FLAGS.seed) + + if FLAGS.use_synthetic_data: + producer = data_pipeline.DummyConstructor() + num_users, num_items = data_preprocessing.DATASET_TO_NUM_USERS_AND_ITEMS[ + FLAGS.dataset] + num_train_steps = rconst.SYNTHETIC_BATCHES_PER_EPOCH + num_eval_steps = rconst.SYNTHETIC_BATCHES_PER_EPOCH + else: + num_users, num_items, producer = data_preprocessing.instantiate_pipeline( + dataset=FLAGS.dataset, data_dir=FLAGS.data_dir, params=params, + constructor_type=FLAGS.constructor_type, + deterministic=FLAGS.seed is not None) + num_train_steps = producer.train_batches_per_epoch + num_eval_steps = producer.eval_batches_per_epoch + + return num_users, num_items, num_train_steps, num_eval_steps, producer + + +def parse_flags(flags_obj): + """Convenience function to turn flags into params.""" + num_gpus = flags_core.get_num_gpus(flags_obj) + + batch_size = flags_obj.batch_size + eval_batch_size = flags_obj.eval_batch_size or flags_obj.batch_size + + return { + "train_epochs": flags_obj.train_epochs, + "batches_per_step": 1, + "use_seed": flags_obj.seed is not None, + "batch_size": batch_size, + "eval_batch_size": eval_batch_size, + "learning_rate": flags_obj.learning_rate, + "mf_dim": flags_obj.num_factors, + "model_layers": [int(layer) for layer in flags_obj.layers], + "mf_regularization": flags_obj.mf_regularization, + "mlp_reg_layers": [float(reg) for reg in flags_obj.mlp_regularization], + "num_neg": flags_obj.num_neg, + "distribution_strategy": flags_obj.distribution_strategy, + "num_gpus": num_gpus, + "use_tpu": flags_obj.tpu is not None, + "tpu": flags_obj.tpu, + "tpu_zone": flags_obj.tpu_zone, + "tpu_gcp_project": flags_obj.tpu_gcp_project, + "beta1": flags_obj.beta1, + "beta2": flags_obj.beta2, + "epsilon": flags_obj.epsilon, + "match_mlperf": flags_obj.ml_perf, + "epochs_between_evals": FLAGS.epochs_between_evals, + "keras_use_ctl": flags_obj.keras_use_ctl, + "hr_threshold": flags_obj.hr_threshold, + "stream_files": flags_obj.tpu is not None, + "train_dataset_path": flags_obj.train_dataset_path, + "eval_dataset_path": flags_obj.eval_dataset_path, + "input_meta_data_path": flags_obj.input_meta_data_path, + } + + +def get_v1_distribution_strategy(params): + """Returns the distribution strategy to use.""" + if params["use_tpu"]: + # Some of the networking libraries are quite chatty. + for name in ["googleapiclient.discovery", "googleapiclient.discovery_cache", + "oauth2client.transport"]: + logging.getLogger(name).setLevel(logging.ERROR) + + tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver( + tpu=params["tpu"], + zone=params["tpu_zone"], + project=params["tpu_gcp_project"], + coordinator_name="coordinator" + ) + + logging.info("Issuing reset command to TPU to ensure a clean state.") + tf.Session.reset(tpu_cluster_resolver.get_master()) + + # Estimator looks at the master it connects to for MonitoredTrainingSession + # by reading the `TF_CONFIG` environment variable, and the coordinator + # is used by StreamingFilesDataset. + tf_config_env = { + "session_master": tpu_cluster_resolver.get_master(), + "eval_session_master": tpu_cluster_resolver.get_master(), + "coordinator": tpu_cluster_resolver.cluster_spec() + .as_dict()["coordinator"] + } + os.environ["TF_CONFIG"] = json.dumps(tf_config_env) + + distribution = tf.distribute.experimental.TPUStrategy( + tpu_cluster_resolver, steps_per_run=100) + + else: + distribution = distribution_utils.get_distribution_strategy( + num_gpus=params["num_gpus"]) + + return distribution + + +def define_ncf_flags(): + """Add flags for running ncf_main.""" + # Add common flags + flags_core.define_base(model_dir=True, clean=True, train_epochs=True, + epochs_between_evals=True, export_dir=False, + run_eagerly=True, stop_threshold=True, num_gpu=True, + hooks=True, distribution_strategy=True) + flags_core.define_performance( + synthetic_data=True, + dtype=True, + fp16_implementation=True, + loss_scale=True, + dynamic_loss_scale=True, + enable_xla=True, + ) + flags_core.define_device(tpu=True) + flags_core.define_benchmark() + + flags.adopt_module_key_flags(flags_core) + + flags_core.set_defaults( + model_dir="/tmp/ncf/", + data_dir="/tmp/movielens-data/", + train_epochs=2, + batch_size=99000, + hooks="ProfilerHook", + tpu=None + ) + + # Add ncf-specific flags + flags.DEFINE_enum( + name="dataset", default="ml-1m", + enum_values=["ml-1m", "ml-20m"], case_sensitive=False, + help=flags_core.help_wrap( + "Dataset to be trained and evaluated.")) + + flags.DEFINE_boolean( + name="download_if_missing", default=True, help=flags_core.help_wrap( + "Download data to data_dir if it is not already present.")) + + flags.DEFINE_integer( + name="eval_batch_size", default=None, help=flags_core.help_wrap( + "The batch size used for evaluation. This should generally be larger" + "than the training batch size as the lack of back propagation during" + "evaluation can allow for larger batch sizes to fit in memory. If not" + "specified, the training batch size (--batch_size) will be used.")) + + flags.DEFINE_integer( + name="num_factors", default=8, + help=flags_core.help_wrap("The Embedding size of MF model.")) + + # Set the default as a list of strings to be consistent with input arguments + flags.DEFINE_list( + name="layers", default=["64", "32", "16", "8"], + help=flags_core.help_wrap( + "The sizes of hidden layers for MLP. Example " + "to specify different sizes of MLP layers: --layers=32,16,8,4")) + + flags.DEFINE_float( + name="mf_regularization", default=0., + help=flags_core.help_wrap( + "The regularization factor for MF embeddings. The factor is used by " + "regularizer which allows to apply penalties on layer parameters or " + "layer activity during optimization.")) + + flags.DEFINE_list( + name="mlp_regularization", default=["0.", "0.", "0.", "0."], + help=flags_core.help_wrap( + "The regularization factor for each MLP layer. See mf_regularization " + "help for more info about regularization factor.")) + + flags.DEFINE_integer( + name="num_neg", default=4, + help=flags_core.help_wrap( + "The Number of negative instances to pair with a positive instance.")) + + flags.DEFINE_float( + name="learning_rate", default=0.001, + help=flags_core.help_wrap("The learning rate.")) + + flags.DEFINE_float( + name="beta1", default=0.9, + help=flags_core.help_wrap("beta1 hyperparameter for the Adam optimizer.")) + + flags.DEFINE_float( + name="beta2", default=0.999, + help=flags_core.help_wrap("beta2 hyperparameter for the Adam optimizer.")) + + flags.DEFINE_float( + name="epsilon", default=1e-8, + help=flags_core.help_wrap("epsilon hyperparameter for the Adam " + "optimizer.")) + + flags.DEFINE_float( + name="hr_threshold", default=1.0, + help=flags_core.help_wrap( + "If passed, training will stop when the evaluation metric HR is " + "greater than or equal to hr_threshold. For dataset ml-1m, the " + "desired hr_threshold is 0.68 which is the result from the paper; " + "For dataset ml-20m, the threshold can be set as 0.95 which is " + "achieved by MLPerf implementation.")) + + flags.DEFINE_enum( + name="constructor_type", default="bisection", + enum_values=["bisection", "materialized"], case_sensitive=False, + help=flags_core.help_wrap( + "Strategy to use for generating false negatives. materialized has a" + "precompute that scales badly, but a faster per-epoch construction" + "time and can be faster on very large systems.")) + + flags.DEFINE_string( + name="train_dataset_path", + default=None, + help=flags_core.help_wrap("Path to training data.")) + + flags.DEFINE_string( + name="eval_dataset_path", + default=None, + help=flags_core.help_wrap("Path to evaluation data.")) + + flags.DEFINE_string( + name="input_meta_data_path", + default=None, + help=flags_core.help_wrap("Path to input meta data file.")) + + flags.DEFINE_bool( + name="ml_perf", default=False, + help=flags_core.help_wrap( + "If set, changes the behavior of the model slightly to match the " + "MLPerf reference implementations here: \n" + "https://github.com/mlperf/reference/tree/master/recommendation/" + "pytorch\n" + "The two changes are:\n" + "1. When computing the HR and NDCG during evaluation, remove " + "duplicate user-item pairs before the computation. This results in " + "better HRs and NDCGs.\n" + "2. Use a different soring algorithm when sorting the input data, " + "which performs better due to the fact the sorting algorithms are " + "not stable.")) + + flags.DEFINE_bool( + name="output_ml_perf_compliance_logging", default=False, + help=flags_core.help_wrap( + "If set, output the MLPerf compliance logging. This is only useful " + "if one is running the model for MLPerf. See " + "https://github.com/mlperf/policies/blob/master/training_rules.adoc" + "#submission-compliance-logs for details. This uses sudo and so may " + "ask for your password, as root access is needed to clear the system " + "caches, which is required for MLPerf compliance." + ) + ) + + flags.DEFINE_integer( + name="seed", default=None, help=flags_core.help_wrap( + "This value will be used to seed both NumPy and TensorFlow.")) + + @flags.validator("eval_batch_size", "eval_batch_size must be at least {}" + .format(rconst.NUM_EVAL_NEGATIVES + 1)) + def eval_size_check(eval_batch_size): + return (eval_batch_size is None or + int(eval_batch_size) > rconst.NUM_EVAL_NEGATIVES) + + flags.DEFINE_bool( + name="early_stopping", + default=False, + help=flags_core.help_wrap( + "If True, we stop the training when it reaches hr_threshold")) + + flags.DEFINE_bool( + name="keras_use_ctl", + default=False, + help=flags_core.help_wrap( + "If True, we use a custom training loop for keras.")) + + +def convert_to_softmax_logits(logits): + """Convert the logits returned by the base model to softmax logits. + + Args: + logits: used to create softmax. + + Returns: + Softmax with the first column of zeros is equivalent to sigmoid. + """ + softmax_logits = tf.concat([logits * 0, logits], axis=1) + return softmax_logits diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/ncf_input_pipeline.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/ncf_input_pipeline.py new file mode 100644 index 0000000..09dbb14 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/ncf_input_pipeline.py @@ -0,0 +1,184 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""NCF model input pipeline.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools + +# pylint: disable=g-bad-import-order +import tensorflow.compat.v2 as tf +# pylint: enable=g-bad-import-order + +from official.recommendation import constants as rconst +from official.recommendation import movielens +from official.recommendation import data_pipeline + +NUM_SHARDS = 16 + + +def create_dataset_from_tf_record_files(input_file_pattern, + pre_batch_size, + batch_size, + is_training=True): + """Creates dataset from (tf)records files for training/evaluation.""" + + files = tf.data.Dataset.list_files(input_file_pattern, shuffle=is_training) + + def make_dataset(files_dataset, shard_index): + """Returns dataset for sharded tf record files.""" + if pre_batch_size != batch_size: + raise ValueError("Pre-batch ({}) size is not equal to batch " + "size ({})".format(pre_batch_size, batch_size)) + files_dataset = files_dataset.shard(NUM_SHARDS, shard_index) + dataset = files_dataset.interleave(tf.data.TFRecordDataset) + decode_fn = functools.partial( + data_pipeline.DatasetManager.deserialize, + batch_size=pre_batch_size, + is_training=is_training) + dataset = dataset.map( + decode_fn, num_parallel_calls=tf.data.experimental.AUTOTUNE) + return dataset + + dataset = tf.data.Dataset.range(NUM_SHARDS) + map_fn = functools.partial(make_dataset, files) + dataset = dataset.interleave( + map_fn, + cycle_length=NUM_SHARDS, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE) + return dataset + + +def create_dataset_from_data_producer(producer, params): + """Return dataset online-generating data.""" + + def preprocess_train_input(features, labels): + """Pre-process the training data. + + This is needed because + - The label needs to be extended to be used in the loss fn + - We need the same inputs for training and eval so adding fake inputs + for DUPLICATE_MASK in training data. + + Args: + features: Dictionary of features for training. + labels: Training labels. + + Returns: + Processed training features. + """ + fake_dup_mask = tf.zeros_like(features[movielens.USER_COLUMN]) + features[rconst.DUPLICATE_MASK] = fake_dup_mask + features[rconst.TRAIN_LABEL_KEY] = labels + return features + + train_input_fn = producer.make_input_fn(is_training=True) + train_input_dataset = train_input_fn(params).map(preprocess_train_input) + + def preprocess_eval_input(features): + """Pre-process the eval data. + + This is needed because: + - The label needs to be extended to be used in the loss fn + - We need the same inputs for training and eval so adding fake inputs + for VALID_PT_MASK in eval data. + + Args: + features: Dictionary of features for evaluation. + + Returns: + Processed evaluation features. + """ + labels = tf.cast(tf.zeros_like(features[movielens.USER_COLUMN]), tf.bool) + fake_valid_pt_mask = tf.cast( + tf.zeros_like(features[movielens.USER_COLUMN]), tf.bool) + features[rconst.VALID_POINT_MASK] = fake_valid_pt_mask + features[rconst.TRAIN_LABEL_KEY] = labels + return features + + eval_input_fn = producer.make_input_fn(is_training=False) + eval_input_dataset = eval_input_fn(params).map(preprocess_eval_input) + + return train_input_dataset, eval_input_dataset + + +def create_ncf_input_data(params, + producer=None, + input_meta_data=None, + strategy=None): + """Creates NCF training/evaluation dataset. + + Args: + params: Dictionary containing parameters for train/evaluation data. + producer: Instance of BaseDataConstructor that generates data online. Must + not be None when params['train_dataset_path'] or + params['eval_dataset_path'] is not specified. + input_meta_data: A dictionary of input metadata to be used when reading data + from tf record files. Must be specified when params["train_input_dataset"] + is specified. + strategy: Distribution strategy used for distributed training. If specified, + used to assert that evaluation batch size is correctly a multiple of + total number of devices used. + + Returns: + (training dataset, evaluation dataset, train steps per epoch, + eval steps per epoch) + + Raises: + ValueError: If data is being generated online for when using TPU's. + """ + # NCF evaluation metric calculation logic assumes that evaluation data + # sample size are in multiples of (1 + number of negative samples in + # evaluation) for each device. As so, evaluation batch size must be a + # multiple of (number of replicas * (1 + number of negative samples)). + num_devices = strategy.num_replicas_in_sync if strategy else 1 + if (params["eval_batch_size"] % (num_devices * + (1 + rconst.NUM_EVAL_NEGATIVES))): + raise ValueError("Evaluation batch size must be divisible by {} " + "times {}".format(num_devices, + (1 + rconst.NUM_EVAL_NEGATIVES))) + + if params["train_dataset_path"]: + assert params["eval_dataset_path"] + + train_dataset = create_dataset_from_tf_record_files( + params["train_dataset_path"], + input_meta_data["train_prebatch_size"], + params["batch_size"], + is_training=True) + eval_dataset = create_dataset_from_tf_record_files( + params["eval_dataset_path"], + input_meta_data["eval_prebatch_size"], + params["eval_batch_size"], + is_training=False) + + num_train_steps = int(input_meta_data["num_train_steps"]) + num_eval_steps = int(input_meta_data["num_eval_steps"]) + else: + if params["use_tpu"]: + raise ValueError("TPU training does not support data producer yet. " + "Use pre-processed data.") + + assert producer + # Start retrieving data from producer. + train_dataset, eval_dataset = create_dataset_from_data_producer( + producer, params) + num_train_steps = producer.train_batches_per_epoch + num_eval_steps = producer.eval_batches_per_epoch + + return train_dataset, eval_dataset, num_train_steps, num_eval_steps diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/ncf_keras_main.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/ncf_keras_main.py new file mode 100644 index 0000000..a35bb9c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/ncf_keras_main.py @@ -0,0 +1,562 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""NCF framework to train and evaluate the NeuMF model. + +The NeuMF model assembles both MF and MLP models under the NCF framework. Check +`neumf_model.py` for more details about the models. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import os + +# pylint: disable=g-bad-import-order +from absl import app +from absl import flags +from absl import logging +import tensorflow.compat.v2 as tf +# pylint: enable=g-bad-import-order + +from official.recommendation import constants as rconst +from official.recommendation import movielens +from official.recommendation import ncf_common +from official.recommendation import ncf_input_pipeline +from official.recommendation import neumf_model +from official.utils.logs import logger +from official.utils.logs import mlperf_helper +from official.utils.misc import distribution_utils +from official.utils.misc import keras_utils +from official.utils.misc import model_helpers +from official.utils.flags import core as flags_core + +FLAGS = flags.FLAGS + + +def metric_fn(logits, dup_mask, params): + dup_mask = tf.cast(dup_mask, tf.float32) + logits = tf.slice(logits, [0, 1], [-1, -1]) + in_top_k, _, metric_weights, _ = neumf_model.compute_top_k_and_ndcg( + logits, + dup_mask, + params["match_mlperf"]) + metric_weights = tf.cast(metric_weights, tf.float32) + return in_top_k, metric_weights + + +class MetricLayer(tf.keras.layers.Layer): + """Custom layer of metrics for NCF model.""" + + def __init__(self, params): + super(MetricLayer, self).__init__() + self.params = params + + def call(self, inputs, training=False): + logits, dup_mask = inputs + + if training: + hr_sum = 0.0 + hr_count = 0.0 + else: + metric, metric_weights = metric_fn(logits, dup_mask, self.params) + hr_sum = tf.reduce_sum(metric * metric_weights) + hr_count = tf.reduce_sum(metric_weights) + + self.add_metric(hr_sum, name="hr_sum", aggregation="mean") + self.add_metric(hr_count, name="hr_count", aggregation="mean") + return logits + + +class LossLayer(tf.keras.layers.Layer): + """Pass-through loss layer for NCF model.""" + + def __init__(self, loss_normalization_factor): + # The loss may overflow in float16, so we use float32 instead. + super(LossLayer, self).__init__(dtype="float32") + self.loss_normalization_factor = loss_normalization_factor + self.loss = tf.keras.losses.SparseCategoricalCrossentropy( + from_logits=True, reduction="sum") + + def call(self, inputs): + logits, labels, valid_pt_mask_input = inputs + loss = self.loss( + y_true=labels, y_pred=logits, sample_weight=valid_pt_mask_input) + loss = loss * (1.0 / self.loss_normalization_factor) + self.add_loss(loss) + return logits + + +class IncrementEpochCallback(tf.keras.callbacks.Callback): + """A callback to increase the requested epoch for the data producer. + + The reason why we need this is because we can only buffer a limited amount of + data. So we keep a moving window to represent the buffer. This is to move the + one of the window's boundaries for each epoch. + """ + + def __init__(self, producer): + self._producer = producer + + def on_epoch_begin(self, epoch, logs=None): + self._producer.increment_request_epoch() + + +class CustomEarlyStopping(tf.keras.callbacks.Callback): + """Stop training has reached a desired hit rate.""" + + def __init__(self, monitor, desired_value): + super(CustomEarlyStopping, self).__init__() + + self.monitor = monitor + self.desired = desired_value + self.stopped_epoch = 0 + + def on_epoch_end(self, epoch, logs=None): + current = self.get_monitor_value(logs) + if current and current >= self.desired: + self.stopped_epoch = epoch + self.model.stop_training = True + + def on_train_end(self, logs=None): + if self.stopped_epoch > 0: + print("Epoch %05d: early stopping" % (self.stopped_epoch + 1)) + + def get_monitor_value(self, logs): + logs = logs or {} + monitor_value = logs.get(self.monitor) + if monitor_value is None: + logging.warning("Early stopping conditioned on metric `%s` " + "which is not available. Available metrics are: %s", + self.monitor, ",".join(list(logs.keys()))) + return monitor_value + + +def _get_keras_model(params): + """Constructs and returns the model.""" + batch_size = params["batch_size"] + + user_input = tf.keras.layers.Input( + shape=(1,), name=movielens.USER_COLUMN, dtype=tf.int32) + + item_input = tf.keras.layers.Input( + shape=(1,), name=movielens.ITEM_COLUMN, dtype=tf.int32) + + valid_pt_mask_input = tf.keras.layers.Input( + shape=(1,), name=rconst.VALID_POINT_MASK, dtype=tf.bool) + + dup_mask_input = tf.keras.layers.Input( + shape=(1,), name=rconst.DUPLICATE_MASK, dtype=tf.int32) + + label_input = tf.keras.layers.Input( + shape=(1,), name=rconst.TRAIN_LABEL_KEY, dtype=tf.bool) + + base_model = neumf_model.construct_model(user_input, item_input, params) + + logits = base_model.output + + zeros = tf.keras.layers.Lambda( + lambda x: x * 0)(logits) + + softmax_logits = tf.keras.layers.concatenate( + [zeros, logits], + axis=-1) + + # Custom training loop calculates loss and metric as a part of + # training/evaluation step function. + if not params["keras_use_ctl"]: + softmax_logits = MetricLayer(params)([softmax_logits, dup_mask_input]) + # TODO(b/134744680): Use model.add_loss() instead once the API is well + # supported. + softmax_logits = LossLayer(batch_size)( + [softmax_logits, label_input, valid_pt_mask_input]) + + keras_model = tf.keras.Model( + inputs={ + movielens.USER_COLUMN: user_input, + movielens.ITEM_COLUMN: item_input, + rconst.VALID_POINT_MASK: valid_pt_mask_input, + rconst.DUPLICATE_MASK: dup_mask_input, + rconst.TRAIN_LABEL_KEY: label_input}, + outputs=softmax_logits) + + keras_model.summary() + return keras_model + + +def run_ncf(_): + """Run NCF training and eval with Keras.""" + + keras_utils.set_session_config(enable_xla=FLAGS.enable_xla) + + if FLAGS.seed is not None: + print("Setting tf seed") + tf.random.set_seed(FLAGS.seed) + + model_helpers.apply_clean(FLAGS) + + if FLAGS.dtype == "fp16" and FLAGS.fp16_implementation == "keras": + policy = tf.keras.mixed_precision.experimental.Policy( + "mixed_float16", + loss_scale=flags_core.get_loss_scale(FLAGS, default_for_fp16="dynamic")) + tf.keras.mixed_precision.experimental.set_policy(policy) + + strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=FLAGS.distribution_strategy, + num_gpus=FLAGS.num_gpus, + tpu_address=FLAGS.tpu) + + params = ncf_common.parse_flags(FLAGS) + params["distribute_strategy"] = strategy + + if not keras_utils.is_v2_0() and strategy is not None: + logging.error("NCF Keras only works with distribution strategy in TF 2.0") + return + if (params["keras_use_ctl"] and ( + not keras_utils.is_v2_0() or strategy is None)): + logging.error( + "Custom training loop only works with tensorflow 2.0 and dist strat.") + return + if params["use_tpu"] and not params["keras_use_ctl"]: + logging.error("Custom training loop must be used when using TPUStrategy.") + return + + batch_size = params["batch_size"] + time_callback = keras_utils.TimeHistory(batch_size, FLAGS.log_steps) + callbacks = [time_callback] + + producer, input_meta_data = None, None + generate_input_online = params["train_dataset_path"] is None + + if generate_input_online: + # Start data producing thread. + num_users, num_items, _, _, producer = ncf_common.get_inputs(params) + producer.start() + per_epoch_callback = IncrementEpochCallback(producer) + callbacks.append(per_epoch_callback) + else: + assert params["eval_dataset_path"] and params["input_meta_data_path"] + with tf.io.gfile.GFile(params["input_meta_data_path"], "rb") as reader: + input_meta_data = json.loads(reader.read().decode("utf-8")) + num_users = input_meta_data["num_users"] + num_items = input_meta_data["num_items"] + + params["num_users"], params["num_items"] = num_users, num_items + + if FLAGS.early_stopping: + early_stopping_callback = CustomEarlyStopping( + "val_HR_METRIC", desired_value=FLAGS.hr_threshold) + callbacks.append(early_stopping_callback) + + (train_input_dataset, eval_input_dataset, + num_train_steps, num_eval_steps) = \ + (ncf_input_pipeline.create_ncf_input_data( + params, producer, input_meta_data, strategy)) + steps_per_epoch = None if generate_input_online else num_train_steps + + with distribution_utils.get_strategy_scope(strategy): + keras_model = _get_keras_model(params) + optimizer = tf.keras.optimizers.Adam( + learning_rate=params["learning_rate"], + beta_1=params["beta1"], + beta_2=params["beta2"], + epsilon=params["epsilon"]) + if FLAGS.fp16_implementation == "graph_rewrite": + optimizer = \ + tf.compat.v1.train.experimental.enable_mixed_precision_graph_rewrite( + optimizer, + loss_scale=flags_core.get_loss_scale(FLAGS, + default_for_fp16="dynamic")) + elif FLAGS.dtype == "fp16" and params["keras_use_ctl"]: + # When keras_use_ctl is False, instead Model.fit() automatically applies + # loss scaling so we don't need to create a LossScaleOptimizer. + optimizer = tf.keras.mixed_precision.experimental.LossScaleOptimizer( + optimizer, + tf.keras.mixed_precision.experimental.global_policy().loss_scale) + + if params["keras_use_ctl"]: + train_loss, eval_results = run_ncf_custom_training( + params, + strategy, + keras_model, + optimizer, + callbacks, + train_input_dataset, + eval_input_dataset, + num_train_steps, + num_eval_steps, + generate_input_online=generate_input_online) + else: + keras_model.compile(optimizer=optimizer, run_eagerly=FLAGS.run_eagerly) + + if not FLAGS.ml_perf: + # Create Tensorboard summary and checkpoint callbacks. + summary_dir = os.path.join(FLAGS.model_dir, "summaries") + summary_callback = tf.keras.callbacks.TensorBoard(summary_dir) + checkpoint_path = os.path.join(FLAGS.model_dir, "checkpoint") + checkpoint_callback = tf.keras.callbacks.ModelCheckpoint( + checkpoint_path, save_weights_only=True) + + callbacks += [summary_callback, checkpoint_callback] + + history = keras_model.fit( + train_input_dataset, + epochs=FLAGS.train_epochs, + steps_per_epoch=steps_per_epoch, + callbacks=callbacks, + validation_data=eval_input_dataset, + validation_steps=num_eval_steps, + verbose=2) + + logging.info("Training done. Start evaluating") + + eval_loss_and_metrics = keras_model.evaluate( + eval_input_dataset, steps=num_eval_steps, verbose=2) + + logging.info("Keras evaluation is done.") + + # Keras evaluate() API returns scalar loss and metric values from + # evaluation as a list. Here, the returned list would contain + # [evaluation loss, hr sum, hr count]. + eval_hit_rate = eval_loss_and_metrics[1] / eval_loss_and_metrics[2] + + # Format evaluation result into [eval loss, eval hit accuracy]. + eval_results = [eval_loss_and_metrics[0], eval_hit_rate] + + if history and history.history: + train_history = history.history + train_loss = train_history["loss"][-1] + + stats = build_stats(train_loss, eval_results, time_callback) + return stats + + +def run_ncf_custom_training(params, + strategy, + keras_model, + optimizer, + callbacks, + train_input_dataset, + eval_input_dataset, + num_train_steps, + num_eval_steps, + generate_input_online=True): + """Runs custom training loop. + + Args: + params: Dictionary containing training parameters. + strategy: Distribution strategy to be used for distributed training. + keras_model: Model used for training. + optimizer: Optimizer used for training. + callbacks: Callbacks to be invoked between batches/epochs. + train_input_dataset: tf.data.Dataset used for training. + eval_input_dataset: tf.data.Dataset used for evaluation. + num_train_steps: Total number of steps to run for training. + num_eval_steps: Total number of steps to run for evaluation. + generate_input_online: Whether input data was generated by data producer. + When data is generated by data producer, then train dataset must be + re-initialized after every epoch. + + Returns: + A tuple of train loss and a list of training and evaluation results. + """ + loss_object = tf.keras.losses.SparseCategoricalCrossentropy( + reduction="sum", from_logits=True) + train_input_iterator = iter( + strategy.experimental_distribute_dataset(train_input_dataset)) + + def train_step(train_iterator): + """Called once per step to train the model.""" + + def step_fn(features): + """Computes loss and applied gradient per replica.""" + with tf.GradientTape() as tape: + softmax_logits = keras_model(features) + # The loss can overflow in float16, so we cast to float32. + softmax_logits = tf.cast(softmax_logits, "float32") + labels = features[rconst.TRAIN_LABEL_KEY] + loss = loss_object( + labels, + softmax_logits, + sample_weight=features[rconst.VALID_POINT_MASK]) + loss *= (1.0 / params["batch_size"]) + if FLAGS.dtype == "fp16": + loss = optimizer.get_scaled_loss(loss) + + grads = tape.gradient(loss, keras_model.trainable_variables) + if FLAGS.dtype == "fp16": + grads = optimizer.get_unscaled_gradients(grads) + # Converting gradients to dense form helps in perf on GPU for NCF + grads = neumf_model.sparse_to_dense_grads( + list(zip(grads, keras_model.trainable_variables))) + optimizer.apply_gradients(grads) + return loss + + per_replica_losses = strategy.run( + step_fn, args=(next(train_iterator),)) + mean_loss = strategy.reduce( + tf.distribute.ReduceOp.SUM, per_replica_losses, axis=None) + return mean_loss + + def eval_step(eval_iterator): + """Called once per eval step to compute eval metrics.""" + + def step_fn(features): + """Computes eval metrics per replica.""" + softmax_logits = keras_model(features) + in_top_k, metric_weights = metric_fn(softmax_logits, + features[rconst.DUPLICATE_MASK], + params) + hr_sum = tf.reduce_sum(in_top_k * metric_weights) + hr_count = tf.reduce_sum(metric_weights) + return hr_sum, hr_count + + per_replica_hr_sum, per_replica_hr_count = ( + strategy.run( + step_fn, args=(next(eval_iterator),))) + hr_sum = strategy.reduce( + tf.distribute.ReduceOp.SUM, per_replica_hr_sum, axis=None) + hr_count = strategy.reduce( + tf.distribute.ReduceOp.SUM, per_replica_hr_count, axis=None) + return hr_sum, hr_count + + if not FLAGS.run_eagerly: + train_step = tf.function(train_step) + eval_step = tf.function(eval_step) + + for callback in callbacks: + callback.on_train_begin() + + # Not writing tensorboard summaries if running in MLPerf. + if FLAGS.ml_perf: + eval_summary_writer, train_summary_writer = None, None + else: + summary_dir = os.path.join(FLAGS.model_dir, "summaries") + eval_summary_writer = tf.summary.create_file_writer( + os.path.join(summary_dir, "eval")) + train_summary_writer = tf.summary.create_file_writer( + os.path.join(summary_dir, "train")) + + train_loss = 0 + for epoch in range(FLAGS.train_epochs): + for cb in callbacks: + cb.on_epoch_begin(epoch) + + # As NCF dataset is sampled with randomness, not repeating + # data elements in each epoch has significant impact on + # convergence. As so, offline-generated TF record files + # contains all epoch worth of data. Thus we do not need + # to initialize dataset when reading from tf record files. + if generate_input_online: + train_input_iterator = iter( + strategy.experimental_distribute_dataset(train_input_dataset)) + + train_loss = 0 + for step in range(num_train_steps): + current_step = step + epoch * num_train_steps + for c in callbacks: + c.on_batch_begin(current_step) + + train_loss += train_step(train_input_iterator) + + # Write train loss once in every 1000 steps. + if train_summary_writer and step % 1000 == 0: + with train_summary_writer.as_default(): + tf.summary.scalar("training_loss", train_loss/(step + 1), + step=current_step) + + for c in callbacks: + c.on_batch_end(current_step) + + train_loss /= num_train_steps + logging.info("Done training epoch %s, epoch loss=%s.", epoch + 1, + train_loss) + + eval_input_iterator = iter( + strategy.experimental_distribute_dataset(eval_input_dataset)) + hr_sum = 0 + hr_count = 0 + for _ in range(num_eval_steps): + step_hr_sum, step_hr_count = eval_step(eval_input_iterator) + hr_sum += step_hr_sum + hr_count += step_hr_count + + logging.info("Done eval epoch %s, hit_rate=%s.", epoch + 1, + hr_sum / hr_count) + if eval_summary_writer: + with eval_summary_writer.as_default(): + tf.summary.scalar("hit_rate", hr_sum / hr_count, step=current_step) + + if (FLAGS.early_stopping and + float(hr_sum / hr_count) > params["hr_threshold"]): + break + + for c in callbacks: + c.on_train_end() + + # Saving the model at the end of training. + if not FLAGS.ml_perf: + checkpoint = tf.train.Checkpoint(model=keras_model, optimizer=optimizer) + checkpoint_path = os.path.join(FLAGS.model_dir, "ctl_checkpoint") + checkpoint.save(checkpoint_path) + logging.info("Saving model as TF checkpoint: %s", checkpoint_path) + + return train_loss, [None, hr_sum / hr_count] + + +def build_stats(loss, eval_result, time_callback): + """Normalizes and returns dictionary of stats. + + Args: + loss: The final loss at training time. + eval_result: Output of the eval step. Assumes first value is eval_loss and + second value is accuracy_top_1. + time_callback: Time tracking callback likely used during keras.fit. + + Returns: + Dictionary of normalized results. + """ + stats = {} + if loss: + stats["loss"] = loss + + if eval_result: + stats["eval_loss"] = eval_result[0] + stats["eval_hit_rate"] = eval_result[1] + + if time_callback: + timestamp_log = time_callback.timestamp_log + stats["step_timestamp_log"] = timestamp_log + stats["train_finish_time"] = time_callback.train_finish_time + if len(timestamp_log) > 1: + stats["avg_exp_per_second"] = ( + time_callback.batch_size * time_callback.log_steps * + (len(time_callback.timestamp_log)-1) / + (timestamp_log[-1].timestamp - timestamp_log[0].timestamp)) + + return stats + + +def main(_): + with logger.benchmark_context(FLAGS), \ + mlperf_helper.LOGGER(FLAGS.output_ml_perf_compliance_logging): + mlperf_helper.set_ncf_root(os.path.split(os.path.abspath(__file__))[0]) + run_ncf(FLAGS) + + +if __name__ == "__main__": + ncf_common.define_ncf_flags() + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/ncf_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/ncf_test.py new file mode 100644 index 0000000..03e0ee1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/ncf_test.py @@ -0,0 +1,255 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests NCF.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math +import unittest + +import numpy as np +import tensorflow as tf +from tensorflow.python.eager import context # pylint: disable=ungrouped-imports +from official.recommendation import constants as rconst +from official.recommendation import data_pipeline +from official.recommendation import ncf_common +from official.recommendation import ncf_keras_main +from official.recommendation import neumf_model +from official.utils.misc import keras_utils +from official.utils.testing import integration + + +NUM_TRAIN_NEG = 4 + + +class NcfTest(tf.test.TestCase): + + @classmethod + def setUpClass(cls): # pylint: disable=invalid-name + super(NcfTest, cls).setUpClass() + ncf_common.define_ncf_flags() + + def setUp(self): + self.top_k_old = rconst.TOP_K + self.num_eval_negatives_old = rconst.NUM_EVAL_NEGATIVES + rconst.NUM_EVAL_NEGATIVES = 2 + + def tearDown(self): + rconst.NUM_EVAL_NEGATIVES = self.num_eval_negatives_old + rconst.TOP_K = self.top_k_old + + @unittest.skipIf(keras_utils.is_v2_0(), "TODO(b/136018594)") + def get_hit_rate_and_ndcg(self, predicted_scores_by_user, items_by_user, + top_k=rconst.TOP_K, match_mlperf=False): + rconst.TOP_K = top_k + rconst.NUM_EVAL_NEGATIVES = predicted_scores_by_user.shape[1] - 1 + batch_size = items_by_user.shape[0] + + users = np.repeat(np.arange(batch_size)[:, np.newaxis], + rconst.NUM_EVAL_NEGATIVES + 1, axis=1) + users, items, duplicate_mask = \ + data_pipeline.BaseDataConstructor._assemble_eval_batch( + users, items_by_user[:, -1:], items_by_user[:, :-1], batch_size) + + g = tf.Graph() + with g.as_default(): + logits = tf.convert_to_tensor( + predicted_scores_by_user.reshape((-1, 1)), tf.float32) + softmax_logits = tf.concat([tf.zeros(logits.shape, dtype=logits.dtype), + logits], axis=1) + duplicate_mask = tf.convert_to_tensor(duplicate_mask, tf.float32) + + metric_ops = neumf_model._get_estimator_spec_with_metrics( + logits=logits, softmax_logits=softmax_logits, + duplicate_mask=duplicate_mask, num_training_neg=NUM_TRAIN_NEG, + match_mlperf=match_mlperf).eval_metric_ops + + hr = metric_ops[rconst.HR_KEY] + ndcg = metric_ops[rconst.NDCG_KEY] + + init = [tf.compat.v1.global_variables_initializer(), + tf.compat.v1.local_variables_initializer()] + + with self.session(graph=g) as sess: + sess.run(init) + return sess.run([hr[1], ndcg[1]]) + + def test_hit_rate_and_ndcg(self): + # Test with no duplicate items + predictions = np.array([ + [2., 0., 1.], # In top 2 + [1., 0., 2.], # In top 1 + [2., 1., 0.], # In top 3 + [3., 4., 2.] # In top 3 + ]) + items = np.array([ + [2, 3, 1], + [3, 1, 2], + [2, 1, 3], + [1, 3, 2], + ]) + + hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 1) + self.assertAlmostEqual(hr, 1 / 4) + self.assertAlmostEqual(ndcg, 1 / 4) + + hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 2) + self.assertAlmostEqual(hr, 2 / 4) + self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3)) / 4) + + hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 3) + self.assertAlmostEqual(hr, 4 / 4) + self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3) + + 2 * math.log(2) / math.log(4)) / 4) + + hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 1, + match_mlperf=True) + self.assertAlmostEqual(hr, 1 / 4) + self.assertAlmostEqual(ndcg, 1 / 4) + + hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 2, + match_mlperf=True) + self.assertAlmostEqual(hr, 2 / 4) + self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3)) / 4) + + hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 3, + match_mlperf=True) + self.assertAlmostEqual(hr, 4 / 4) + self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3) + + 2 * math.log(2) / math.log(4)) / 4) + + # Test with duplicate items. In the MLPerf case, we treat the duplicates as + # a single item. Otherwise, we treat the duplicates as separate items. + predictions = np.array([ + [2., 2., 3., 1.], # In top 4. MLPerf: In top 3 + [1., 0., 2., 3.], # In top 1. MLPerf: In top 1 + [2., 3., 2., 0.], # In top 4. MLPerf: In top 3 + [2., 4., 2., 3.] # In top 2. MLPerf: In top 2 + ]) + items = np.array([ + [2, 2, 3, 1], + [2, 3, 4, 1], + [2, 3, 2, 1], + [3, 2, 1, 4], + ]) + hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 1) + self.assertAlmostEqual(hr, 1 / 4) + self.assertAlmostEqual(ndcg, 1 / 4) + + hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 2) + self.assertAlmostEqual(hr, 2 / 4) + self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3)) / 4) + + hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 3) + self.assertAlmostEqual(hr, 2 / 4) + self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3)) / 4) + + hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 4) + self.assertAlmostEqual(hr, 4 / 4) + self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3) + + 2 * math.log(2) / math.log(5)) / 4) + + hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 1, + match_mlperf=True) + self.assertAlmostEqual(hr, 1 / 4) + self.assertAlmostEqual(ndcg, 1 / 4) + + hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 2, + match_mlperf=True) + self.assertAlmostEqual(hr, 2 / 4) + self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3)) / 4) + + hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 3, + match_mlperf=True) + self.assertAlmostEqual(hr, 4 / 4) + self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3) + + 2 * math.log(2) / math.log(4)) / 4) + + hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 4, + match_mlperf=True) + self.assertAlmostEqual(hr, 4 / 4) + self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3) + + 2 * math.log(2) / math.log(4)) / 4) + + _BASE_END_TO_END_FLAGS = ['-batch_size', '1044', '-train_epochs', '1'] + + @unittest.mock.patch.object(rconst, "SYNTHETIC_BATCHES_PER_EPOCH", 100) + def test_end_to_end_keras_no_dist_strat(self): + integration.run_synthetic( + ncf_keras_main.main, tmp_root=self.get_temp_dir(), + extra_flags=self._BASE_END_TO_END_FLAGS + + ['-distribution_strategy', 'off']) + + @unittest.mock.patch.object(rconst, "SYNTHETIC_BATCHES_PER_EPOCH", 100) + @unittest.skipUnless(keras_utils.is_v2_0(), 'TF 2.0 only test.') + def test_end_to_end_keras_dist_strat(self): + integration.run_synthetic( + ncf_keras_main.main, tmp_root=self.get_temp_dir(), + extra_flags=self._BASE_END_TO_END_FLAGS + ['-num_gpus', '0']) + + @unittest.mock.patch.object(rconst, "SYNTHETIC_BATCHES_PER_EPOCH", 100) + @unittest.skipUnless(keras_utils.is_v2_0(), 'TF 2.0 only test.') + def test_end_to_end_keras_dist_strat_ctl(self): + flags = (self._BASE_END_TO_END_FLAGS + + ['-num_gpus', '0'] + + ['-keras_use_ctl', 'True']) + integration.run_synthetic( + ncf_keras_main.main, tmp_root=self.get_temp_dir(), + extra_flags=flags) + + @unittest.mock.patch.object(rconst, "SYNTHETIC_BATCHES_PER_EPOCH", 100) + @unittest.skipUnless(keras_utils.is_v2_0(), 'TF 2.0 only test.') + def test_end_to_end_keras_1_gpu_dist_strat_fp16(self): + if context.num_gpus() < 1: + self.skipTest( + "{} GPUs are not available for this test. {} GPUs are available". + format(1, context.num_gpus())) + + integration.run_synthetic( + ncf_keras_main.main, tmp_root=self.get_temp_dir(), + extra_flags=self._BASE_END_TO_END_FLAGS + ['-num_gpus', '1', + '--dtype', 'fp16']) + + @unittest.mock.patch.object(rconst, "SYNTHETIC_BATCHES_PER_EPOCH", 100) + @unittest.skipUnless(keras_utils.is_v2_0(), 'TF 2.0 only test.') + def test_end_to_end_keras_1_gpu_dist_strat_ctl_fp16(self): + if context.num_gpus() < 1: + self.skipTest( + '{} GPUs are not available for this test. {} GPUs are available'. + format(1, context.num_gpus())) + + integration.run_synthetic( + ncf_keras_main.main, tmp_root=self.get_temp_dir(), + extra_flags=self._BASE_END_TO_END_FLAGS + ['-num_gpus', '1', + '--dtype', 'fp16', + '--keras_use_ctl']) + + @unittest.mock.patch.object(rconst, 'SYNTHETIC_BATCHES_PER_EPOCH', 100) + @unittest.skipUnless(keras_utils.is_v2_0(), 'TF 2.0 only test.') + def test_end_to_end_keras_2_gpu_fp16(self): + if context.num_gpus() < 2: + self.skipTest( + "{} GPUs are not available for this test. {} GPUs are available". + format(2, context.num_gpus())) + + integration.run_synthetic( + ncf_keras_main.main, tmp_root=self.get_temp_dir(), + extra_flags=self._BASE_END_TO_END_FLAGS + ['-num_gpus', '2', + '--dtype', 'fp16']) + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/neumf_model.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/neumf_model.py new file mode 100644 index 0000000..7c561b2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/neumf_model.py @@ -0,0 +1,457 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Defines NeuMF model for NCF framework. + +Some abbreviations used in the code base: +NeuMF: Neural Matrix Factorization +NCF: Neural Collaborative Filtering +GMF: Generalized Matrix Factorization +MLP: Multi-Layer Perceptron + +GMF applies a linear kernel to model the latent feature interactions, and MLP +uses a nonlinear kernel to learn the interaction function from data. NeuMF model +is a fused model of GMF and MLP to better model the complex user-item +interactions, and unifies the strengths of linearity of MF and non-linearity of +MLP for modeling the user-item latent structures. + +In NeuMF model, it allows GMF and MLP to learn separate embeddings, and combine +the two models by concatenating their last hidden layer. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys + +from six.moves import xrange # pylint: disable=redefined-builtin +import tensorflow as tf + +from official.recommendation import constants as rconst +from official.recommendation import movielens +from official.recommendation import ncf_common +from official.recommendation import stat_utils +from official.utils.logs import mlperf_helper + + +def sparse_to_dense_grads(grads_and_vars): + """Convert sparse gradients to dense gradients. + + All sparse gradients, which are represented as instances of tf.IndexedSlices, + are converted to dense Tensors. Dense gradients, which are represents as + Tensors, are unchanged. + + The purpose of this conversion is that for small embeddings, which are used by + this model, applying dense gradients with the AdamOptimizer is faster than + applying sparse gradients. + + Args + grads_and_vars: A list of (gradient, variable) tuples. Each gradient can + be a Tensor or an IndexedSlices. Tensors are unchanged, and IndexedSlices + are converted to dense Tensors. + Returns: + The same list of (gradient, variable) as `grads_and_vars`, except each + IndexedSlices gradient is converted to a Tensor. + """ + + # Calling convert_to_tensor changes IndexedSlices into Tensors, and leaves + # Tensors unchanged. + return [(tf.convert_to_tensor(g), v) for g, v in grads_and_vars] + + +def neumf_model_fn(features, labels, mode, params): + """Model Function for NeuMF estimator.""" + if params.get("use_seed"): + tf.set_random_seed(stat_utils.random_int32()) + + users = features[movielens.USER_COLUMN] + items = features[movielens.ITEM_COLUMN] + + user_input = tf.keras.layers.Input(tensor=users) + item_input = tf.keras.layers.Input(tensor=items) + logits = construct_model(user_input, item_input, params).output + + # Softmax with the first column of zeros is equivalent to sigmoid. + softmax_logits = ncf_common.convert_to_softmax_logits(logits) + + if mode == tf.estimator.ModeKeys.EVAL: + duplicate_mask = tf.cast(features[rconst.DUPLICATE_MASK], tf.float32) + return _get_estimator_spec_with_metrics( + logits, + softmax_logits, + duplicate_mask, + params["num_neg"], + params["match_mlperf"], + use_tpu_spec=params["use_tpu"]) + + elif mode == tf.estimator.ModeKeys.TRAIN: + labels = tf.cast(labels, tf.int32) + valid_pt_mask = features[rconst.VALID_POINT_MASK] + + mlperf_helper.ncf_print(key=mlperf_helper.TAGS.OPT_NAME, value="adam") + mlperf_helper.ncf_print(key=mlperf_helper.TAGS.OPT_LR, + value=params["learning_rate"]) + mlperf_helper.ncf_print(key=mlperf_helper.TAGS.OPT_HP_ADAM_BETA1, + value=params["beta1"]) + mlperf_helper.ncf_print(key=mlperf_helper.TAGS.OPT_HP_ADAM_BETA2, + value=params["beta2"]) + mlperf_helper.ncf_print(key=mlperf_helper.TAGS.OPT_HP_ADAM_EPSILON, + value=params["epsilon"]) + + optimizer = tf.compat.v1.train.AdamOptimizer( + learning_rate=params["learning_rate"], + beta1=params["beta1"], + beta2=params["beta2"], + epsilon=params["epsilon"]) + if params["use_tpu"]: + optimizer = tf.compat.v1.tpu.CrossShardOptimizer(optimizer) + + mlperf_helper.ncf_print(key=mlperf_helper.TAGS.MODEL_HP_LOSS_FN, + value=mlperf_helper.TAGS.BCE) + + loss = tf.compat.v1.losses.sparse_softmax_cross_entropy( + labels=labels, + logits=softmax_logits, + weights=tf.cast(valid_pt_mask, tf.float32) + ) + + # This tensor is used by logging hooks. + tf.identity(loss, name="cross_entropy") + + global_step = tf.compat.v1.train.get_global_step() + tvars = tf.compat.v1.trainable_variables() + gradients = optimizer.compute_gradients( + loss, tvars, colocate_gradients_with_ops=True) + gradients = sparse_to_dense_grads(gradients) + minimize_op = optimizer.apply_gradients( + gradients, global_step=global_step, name="train") + update_ops = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.UPDATE_OPS) + train_op = tf.group(minimize_op, update_ops) + + return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op) + + else: + raise NotImplementedError + + +def _strip_first_and_last_dimension(x, batch_size): + return tf.reshape(x[0, :], (batch_size,)) + + +def construct_model(user_input, item_input, params): + # type: (tf.Tensor, tf.Tensor, dict) -> tf.keras.Model + """Initialize NeuMF model. + + Args: + user_input: keras input layer for users + item_input: keras input layer for items + params: Dict of hyperparameters. + Raises: + ValueError: if the first model layer is not even. + Returns: + model: a keras Model for computing the logits + """ + num_users = params["num_users"] + num_items = params["num_items"] + + model_layers = params["model_layers"] + + mf_regularization = params["mf_regularization"] + mlp_reg_layers = params["mlp_reg_layers"] + + mf_dim = params["mf_dim"] + + mlperf_helper.ncf_print(key=mlperf_helper.TAGS.MODEL_HP_MF_DIM, value=mf_dim) + mlperf_helper.ncf_print(key=mlperf_helper.TAGS.MODEL_HP_MLP_LAYER_SIZES, + value=model_layers) + + if model_layers[0] % 2 != 0: + raise ValueError("The first layer size should be multiple of 2!") + + # Initializer for embedding layers + embedding_initializer = "glorot_uniform" + + def mf_slice_fn(x): + x = tf.squeeze(x, [1]) + return x[:, :mf_dim] + + def mlp_slice_fn(x): + x = tf.squeeze(x, [1]) + return x[:, mf_dim:] + + # It turns out to be significantly more effecient to store the MF and MLP + # embedding portions in the same table, and then slice as needed. + embedding_user = tf.keras.layers.Embedding( + num_users, + mf_dim + model_layers[0] // 2, + embeddings_initializer=embedding_initializer, + embeddings_regularizer=tf.keras.regularizers.l2(mf_regularization), + input_length=1, + name="embedding_user")( + user_input) + + embedding_item = tf.keras.layers.Embedding( + num_items, + mf_dim + model_layers[0] // 2, + embeddings_initializer=embedding_initializer, + embeddings_regularizer=tf.keras.regularizers.l2(mf_regularization), + input_length=1, + name="embedding_item")( + item_input) + + # GMF part + mf_user_latent = tf.keras.layers.Lambda( + mf_slice_fn, name="embedding_user_mf")(embedding_user) + mf_item_latent = tf.keras.layers.Lambda( + mf_slice_fn, name="embedding_item_mf")(embedding_item) + + # MLP part + mlp_user_latent = tf.keras.layers.Lambda( + mlp_slice_fn, name="embedding_user_mlp")(embedding_user) + mlp_item_latent = tf.keras.layers.Lambda( + mlp_slice_fn, name="embedding_item_mlp")(embedding_item) + + # Element-wise multiply + mf_vector = tf.keras.layers.multiply([mf_user_latent, mf_item_latent]) + + # Concatenation of two latent features + mlp_vector = tf.keras.layers.concatenate([mlp_user_latent, mlp_item_latent]) + + num_layer = len(model_layers) # Number of layers in the MLP + for layer in xrange(1, num_layer): + model_layer = tf.keras.layers.Dense( + model_layers[layer], + kernel_regularizer=tf.keras.regularizers.l2(mlp_reg_layers[layer]), + activation="relu") + mlp_vector = model_layer(mlp_vector) + + # Concatenate GMF and MLP parts + predict_vector = tf.keras.layers.concatenate([mf_vector, mlp_vector]) + + # Final prediction layer + logits = tf.keras.layers.Dense( + 1, activation=None, kernel_initializer="lecun_uniform", + name=movielens.RATING_COLUMN)(predict_vector) + + # Print model topology. + model = tf.keras.models.Model([user_input, item_input], logits) + model.summary() + sys.stdout.flush() + + return model + + +def _get_estimator_spec_with_metrics(logits, # type: tf.Tensor + softmax_logits, # type: tf.Tensor + duplicate_mask, # type: tf.Tensor + num_training_neg, # type: int + match_mlperf=False, # type: bool + use_tpu_spec=False # type: bool + ): + """Returns a EstimatorSpec that includes the metrics.""" + cross_entropy, \ + metric_fn, \ + in_top_k, \ + ndcg, \ + metric_weights = compute_eval_loss_and_metrics_helper( + logits, + softmax_logits, + duplicate_mask, + num_training_neg, + match_mlperf) + + if use_tpu_spec: + return tf.estimator.tpu.TPUEstimatorSpec( + mode=tf.estimator.ModeKeys.EVAL, + loss=cross_entropy, + eval_metrics=(metric_fn, [in_top_k, ndcg, metric_weights])) + + return tf.estimator.EstimatorSpec( + mode=tf.estimator.ModeKeys.EVAL, + loss=cross_entropy, + eval_metric_ops=metric_fn(in_top_k, ndcg, metric_weights) + ) + + +def compute_eval_loss_and_metrics_helper( + logits, # type: tf.Tensor + softmax_logits, # type: tf.Tensor + duplicate_mask, # type: tf.Tensor + num_training_neg, # type: int + match_mlperf=False # type: bool +): + """Model evaluation with HR and NDCG metrics. + + The evaluation protocol is to rank the test interacted item (truth items) + among the randomly chosen 999 items that are not interacted by the user. + The performance of the ranked list is judged by Hit Ratio (HR) and Normalized + Discounted Cumulative Gain (NDCG). + + For evaluation, the ranked list is truncated at 10 for both metrics. As such, + the HR intuitively measures whether the test item is present on the top-10 + list, and the NDCG accounts for the position of the hit by assigning higher + scores to hits at top ranks. Both metrics are calculated for each test user, + and the average scores are reported. + + If `match_mlperf` is True, then the HR and NDCG computations are done in a + slightly unusual way to match the MLPerf reference implementation. + Specifically, if the evaluation negatives contain duplicate items, it will be + treated as if the item only appeared once. Effectively, for duplicate items in + a row, the predicted score for all but one of the items will be set to + -infinity + + For example, suppose we have that following inputs: + logits_by_user: [[ 2, 3, 3], + [ 5, 4, 4]] + + items_by_user: [[10, 20, 20], + [30, 40, 40]] + + # Note: items_by_user is not explicitly present. Instead the relevant \ + information is contained within `duplicate_mask` + + top_k: 2 + + Then with match_mlperf=True, the HR would be 2/2 = 1.0. With + match_mlperf=False, the HR would be 1/2 = 0.5. This is because each user has + predicted scores for only 2 unique items: 10 and 20 for the first user, and 30 + and 40 for the second. Therefore, with match_mlperf=True, it's guaranteed the + first item's score is in the top 2. With match_mlperf=False, this function + would compute the first user's first item is not in the top 2, because item 20 + has a higher score, and item 20 occurs twice. + + Args: + logits: A tensor containing the predicted logits for each user. The shape + of logits is (num_users_per_batch * (1 + NUM_EVAL_NEGATIVES),) Logits + for a user are grouped, and the last element of the group is the true + element. + + softmax_logits: The same tensor, but with zeros left-appended. + + duplicate_mask: A vector with the same shape as logits, with a value of 1 + if the item corresponding to the logit at that position has already + appeared for that user. + + num_training_neg: The number of negatives per positive during training. + + match_mlperf: Use the MLPerf reference convention for computing rank. + + Returns: + cross_entropy: the loss + metric_fn: the metrics function + in_top_k: hit rate metric + ndcg: ndcg metric + metric_weights: metric weights + """ + in_top_k, ndcg, metric_weights, logits_by_user = compute_top_k_and_ndcg( + logits, duplicate_mask, match_mlperf) + + # Examples are provided by the eval Dataset in a structured format, so eval + # labels can be reconstructed on the fly. + eval_labels = tf.reshape(shape=(-1,), tensor=tf.one_hot( + tf.zeros(shape=(logits_by_user.shape[0],), dtype=tf.int32) + + rconst.NUM_EVAL_NEGATIVES, logits_by_user.shape[1], dtype=tf.int32)) + + eval_labels_float = tf.cast(eval_labels, tf.float32) + + # During evaluation, the ratio of negatives to positives is much higher + # than during training. (Typically 999 to 1 vs. 4 to 1) By adjusting the + # weights for the negative examples we compute a loss which is consistent with + # the training data. (And provides apples-to-apples comparison) + negative_scale_factor = num_training_neg / rconst.NUM_EVAL_NEGATIVES + example_weights = ( + (eval_labels_float + (1 - eval_labels_float) * negative_scale_factor) * + (1 + rconst.NUM_EVAL_NEGATIVES) / (1 + num_training_neg)) + + # Tile metric weights back to logit dimensions + expanded_metric_weights = tf.reshape(tf.tile( + metric_weights[:, tf.newaxis], (1, rconst.NUM_EVAL_NEGATIVES + 1)), (-1,)) + + # ignore padded examples + example_weights *= tf.cast(expanded_metric_weights, tf.float32) + + cross_entropy = tf.compat.v1.losses.sparse_softmax_cross_entropy( + logits=softmax_logits, labels=eval_labels, weights=example_weights) + + def metric_fn(top_k_tensor, ndcg_tensor, weight_tensor): + return { + rconst.HR_KEY: tf.compat.v1.metrics.mean(top_k_tensor, + weights=weight_tensor, + name=rconst.HR_METRIC_NAME), + rconst.NDCG_KEY: tf.compat.v1.metrics.mean(ndcg_tensor, + weights=weight_tensor, + name=rconst.NDCG_METRIC_NAME) + } + + return cross_entropy, metric_fn, in_top_k, ndcg, metric_weights + + +def compute_top_k_and_ndcg(logits, # type: tf.Tensor + duplicate_mask, # type: tf.Tensor + match_mlperf=False # type: bool + ): + """Compute inputs of metric calculation. + + Args: + logits: A tensor containing the predicted logits for each user. The shape + of logits is (num_users_per_batch * (1 + NUM_EVAL_NEGATIVES),) Logits + for a user are grouped, and the first element of the group is the true + element. + duplicate_mask: A vector with the same shape as logits, with a value of 1 + if the item corresponding to the logit at that position has already + appeared for that user. + match_mlperf: Use the MLPerf reference convention for computing rank. + + Returns: + is_top_k, ndcg and weights, all of which has size (num_users_in_batch,), and + logits_by_user which has size + (num_users_in_batch, (rconst.NUM_EVAL_NEGATIVES + 1)). + """ + logits_by_user = tf.reshape(logits, (-1, rconst.NUM_EVAL_NEGATIVES + 1)) + duplicate_mask_by_user = tf.cast( + tf.reshape(duplicate_mask, (-1, rconst.NUM_EVAL_NEGATIVES + 1)), + logits_by_user.dtype) + + if match_mlperf: + # Set duplicate logits to the min value for that dtype. The MLPerf + # reference dedupes during evaluation. + logits_by_user *= (1 - duplicate_mask_by_user) + logits_by_user += duplicate_mask_by_user * logits_by_user.dtype.min + + # Determine the location of the first element in each row after the elements + # are sorted. + sort_indices = tf.argsort( + logits_by_user, axis=1, direction="DESCENDING") + + # Use matrix multiplication to extract the position of the true item from the + # tensor of sorted indices. This approach is chosen because both GPUs and TPUs + # perform matrix multiplications very quickly. This is similar to np.argwhere. + # However this is a special case because the target will only appear in + # sort_indices once. + one_hot_position = tf.cast(tf.equal(sort_indices, rconst.NUM_EVAL_NEGATIVES), + tf.int32) + sparse_positions = tf.multiply( + one_hot_position, tf.range(logits_by_user.shape[1])[tf.newaxis, :]) + position_vector = tf.reduce_sum(sparse_positions, axis=1) + + in_top_k = tf.cast(tf.less(position_vector, rconst.TOP_K), tf.float32) + ndcg = tf.math.log(2.) / tf.math.log( + tf.cast(position_vector, tf.float32) + 2) + ndcg *= in_top_k + + # If a row is a padded row, all but the first element will be a duplicate. + metric_weights = tf.not_equal(tf.reduce_sum(duplicate_mask_by_user, axis=1), + rconst.NUM_EVAL_NEGATIVES) + + return in_top_k, ndcg, metric_weights, logits_by_user diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/popen_helper.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/popen_helper.py new file mode 100644 index 0000000..dcdca4c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/popen_helper.py @@ -0,0 +1,64 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Helper file for running the async data generation process in OSS.""" + +import contextlib +import multiprocessing +import multiprocessing.pool + + +def get_forkpool(num_workers, init_worker=None, closing=True): + pool = multiprocessing.Pool(processes=num_workers, initializer=init_worker) + return contextlib.closing(pool) if closing else pool + + +def get_threadpool(num_workers, init_worker=None, closing=True): + pool = multiprocessing.pool.ThreadPool(processes=num_workers, + initializer=init_worker) + return contextlib.closing(pool) if closing else pool + + +class FauxPool(object): + """Mimic a pool using for loops. + + This class is used in place of proper pools when true determinism is desired + for testing or debugging. + """ + def __init__(self, *args, **kwargs): + pass + + def map(self, func, iterable, chunksize=None): + return [func(i) for i in iterable] + + def imap(self, func, iterable, chunksize=1): + for i in iterable: + yield func(i) + + def close(self): + pass + + def terminate(self): + pass + + def join(self): + pass + +def get_fauxpool(num_workers, init_worker=None, closing=True): + pool = FauxPool(processes=num_workers, initializer=init_worker) + return contextlib.closing(pool) if closing else pool + + +def worker_job(): + return "worker" diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/stat_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/stat_utils.py new file mode 100644 index 0000000..658a272 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/recommendation/stat_utils.py @@ -0,0 +1,92 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Statistics utility functions of NCF.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +import numpy as np + + +def random_int32(): + return np.random.randint(low=0, high=np.iinfo(np.int32).max, dtype=np.int32) + + +def permutation(args): + """Fork safe permutation function. + + This function can be called within a multiprocessing worker and give + appropriately random results. + + Args: + args: A size two tuple that will unpacked into the size of the permutation + and the random seed. This form is used because starmap is not universally + available. + + returns: + A NumPy array containing a random permutation. + """ + x, seed = args + + # If seed is None NumPy will seed randomly. + state = np.random.RandomState(seed=seed) # pylint: disable=no-member + output = np.arange(x, dtype=np.int32) + state.shuffle(output) + return output + + +def very_slightly_biased_randint(max_val_vector): + sample_dtype = np.uint64 + out_dtype = max_val_vector.dtype + samples = np.random.randint(low=0, high=np.iinfo(sample_dtype).max, + size=max_val_vector.shape, dtype=sample_dtype) + return np.mod(samples, max_val_vector.astype(sample_dtype)).astype(out_dtype) + + +def mask_duplicates(x, axis=1): # type: (np.ndarray, int) -> np.ndarray + """Identify duplicates from sampling with replacement. + + Args: + x: A 2D NumPy array of samples + axis: The axis along which to de-dupe. + + Returns: + A NumPy array with the same shape as x with one if an element appeared + previously along axis 1, else zero. + """ + if axis != 1: + raise NotImplementedError + + x_sort_ind = np.argsort(x, axis=1, kind="mergesort") + sorted_x = x[np.arange(x.shape[0])[:, np.newaxis], x_sort_ind] + + # compute the indices needed to map values back to their original position. + inv_x_sort_ind = np.argsort(x_sort_ind, axis=1, kind="mergesort") + + # Compute the difference of adjacent sorted elements. + diffs = sorted_x[:, :-1] - sorted_x[:, 1:] + + # We are only interested in whether an element is zero. Therefore left padding + # with ones to restore the original shape is sufficient. + diffs = np.concatenate( + [np.ones((diffs.shape[0], 1), dtype=diffs.dtype), diffs], axis=1) + + # Duplicate values will have a difference of zero. By definition the first + # element is never a duplicate. + return np.where(diffs[np.arange(x.shape[0])[:, np.newaxis], + inv_x_sort_ind], 0, 1) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/requirements.txt b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/requirements.txt new file mode 100644 index 0000000..4b0c214 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/requirements.txt @@ -0,0 +1,25 @@ +six +google-api-python-client>=1.6.7 +google-cloud-bigquery>=0.31.0 +kaggle>=1.3.9 +mlperf_compliance==0.0.10 +numpy>=1.15.4 +oauth2client>=4.1.2 +pandas>=0.22.0 +psutil>=5.4.3 +py-cpuinfo>=3.3.0 +scipy>=0.19.1 +tensorflow-hub>=0.6.0 +tensorflow-model-optimization>=0.2.1 +tensorflow-datasets +tensorflow-addons +dataclasses +gin-config +typing +sentencepiece +Cython +matplotlib +opencv-python-headless +pyyaml +Pillow +-e git+https://github.com/cocodataset/cocoapi#egg=pycocotools&subdirectory=PythonAPI diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/__init__.py new file mode 100644 index 0000000..931c2ef --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/controller.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/controller.py new file mode 100644 index 0000000..a07be66 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/controller.py @@ -0,0 +1,337 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A light weight utilities to train TF2 models.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import time + +from absl import logging + +import tensorflow.compat.v2 as tf +from typing import Callable, Dict, Optional, Text + +from official.staging.training import utils + + +class Controller(object): + """Class that facilitates training and evaluation of models.""" + + def __init__( + self, + strategy: Optional[tf.distribute.Strategy] = None, + train_fn: Optional[Callable[[tf.Tensor], + Optional[Dict[Text, tf.Tensor]]]] = None, + eval_fn: Optional[Callable[[tf.Tensor], + Optional[Dict[Text, tf.Tensor]]]] = None, + global_step: Optional[tf.Variable] = None, + # Train related + train_steps: Optional[int] = None, + steps_per_loop: Optional[int] = None, + summary_dir: Optional[Text] = None, + checkpoint_manager: Optional[tf.train.CheckpointManager] = None, + # summary related + summary_interval: Optional[int] = None, + # Evaluation related + eval_summary_dir: Optional[Text] = None, + eval_steps: Optional[int] = None, + eval_interval: Optional[int] = None): + """Constructs a `Controller` instance. + + Args: + strategy: An instance of `tf.distribute.Strategy`. + train_fn: A callable defined as `def train_fn(num_steps)`, which + `num_steps` indicates the number of steps to run for each loop. + eval_fn: A callable defined as `def eval_fn(num_steps)`, which `num_steps` + indicates the number of steps for one evaluation. + global_step: An integer `tf.Variable` indicating the global training step + number. Usually this can be obtained from `iterations` property of the + model's optimizer (e.g. `self.optimizer.iterations`), or users can + create their own global step variable as well. If the users create their + own global step variable, it is recommended to create the `tf.Variable` + inside strategy scope, and with + `aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA`. + train_steps: The total (maximum) number of training steps to perform. + steps_per_loop: The number of steps to run in each "inner loop" of + training (passed to the `num_steps` parameter of `train_fn`). + summary_dir: The directory to restore and write checkpoints and summaries. + If None, it will be set to `checkpoint_manager.directory`. + checkpoint_manager: An instance of `tf.train.CheckpointManager`. + summary_interval: Step interval for training summaries. Note that this + argument only applies to the summaries outside the training loop. If the + value is None, then training summaries are not enabled. + eval_summary_dir: The directory to write eval summaries. If None, it will + be set to `summary_dir`. + eval_steps: Number of steps to run evaluation. + eval_interval: Step interval for evaluation. If None, will skip evaluation + in the middle of training. Note that evaluation only happens outside the + training loop, which the loop iteration is specify by `steps_per_loop` + parameter. + + Raises: + ValueError: If both `train_fn` and `eval_fn` are None. + ValueError: If `train_fn` is not None and `train_steps` is None. + ValueError: If `steps_per_loop` is None when `train_fn` is provided. + ValueError: If `steps_per_loop` is not a positive integer. + """ + if train_fn is None and eval_fn is None: + raise ValueError("`train_fn` and `eval_fn` should not both be None") + + # TODO(rxsang): Support training until exhaustion by passing + # `train_steps=-1`. Currently it cannot be supported with a host training + # loop because break statements are not supported with distributed dataset. + if train_fn is not None: + if train_steps is None: + raise ValueError("`train_steps` is required when `train_fn` is " + "provided.") + if steps_per_loop is None: + raise ValueError("`steps_per_loop` is required when `train_fn is " + "provided.") + if not isinstance(steps_per_loop, int) or steps_per_loop < 1: + raise ValueError("`steps_per_loop` should be a positive integer") + if summary_interval is not None and summary_interval <= 0: + raise ValueError("`summary_interval` should be larger than 0") + + self.strategy = strategy or tf.distribute.get_strategy() + + self.train_fn = train_fn + self.eval_fn = eval_fn + self.global_step = global_step + self.checkpoint_manager = checkpoint_manager + + if self.train_fn is not None: + self.train_steps = train_steps + self.steps_per_loop = steps_per_loop + if summary_dir: + self.summary_dir = summary_dir + elif checkpoint_manager: + self.summary_dir = checkpoint_manager.directory + else: + self.summary_dir = None + + self.summary_interval = summary_interval + if self.summary_dir and self.summary_interval: + summary_writer = tf.summary.create_file_writer(self.summary_dir) + else: + summary_writer = None + # TODO(rxsang): Consider pass SummaryManager directly into Controller for + # maximum customizability. + self.summary_manager = utils.SummaryManager( + summary_writer, + tf.summary.scalar, + global_step=self.global_step, + summary_interval=self.summary_interval) + + if self.eval_fn is not None: + eval_summary_dir = eval_summary_dir or self.summary_dir + eval_summary_writer = tf.summary.create_file_writer( + eval_summary_dir) if eval_summary_dir else None + self.eval_summary_manager = utils.SummaryManager( + eval_summary_writer, tf.summary.scalar, global_step=self.global_step) + + self.eval_steps = eval_steps + self.eval_interval = eval_interval + + # Creates and initializes the interval triggers. + self.eval_trigger = utils.IntervalTrigger(self.eval_interval, + self.global_step.numpy()) # pytype: disable=attribute-error + + if self.global_step: + tf.summary.experimental.set_step(self.global_step) + + # Restores the model if needed. + if self.checkpoint_manager is not None: + model_restored = self._restore_model() + if not model_restored and self.checkpoint_manager.checkpoint_interval: + # If the model is not restored from a checkpoint, save an initial + # checkpoint. + ckpt_path = self.checkpoint_manager.save( + checkpoint_number=self.global_step) + logging.info("Saved checkpoins in %s", ckpt_path) + + def _restore_model(self, checkpoint_path=None): + """Restore or initialize the model. + + Args: + checkpoint_path: An optional string indicates the checkpoint path to + restore. If None, will restore from `self.checkpoint_manager`. + + Returns: + True if the latest checkpoint is found or restored. Otherwise False. + """ + with self.strategy.scope(): + # Checkpoint restoring should be inside scope. b/139450638 + if checkpoint_path is not None: + self.checkpoint_manager.checkpoint.restore(checkpoint_path) + return True + return self.checkpoint_manager.restore_or_initialize() + + def _evaluate_once(self, current_step): + """Runs the evaluation once.""" + logging.info("Start evaluation at step: %s", current_step) + + with self.eval_summary_manager.summary_writer.as_default(): + eval_outputs = self.eval_fn(self.eval_steps) + + if eval_outputs: + eval_outputs = tf.nest.map_structure(lambda x: x.numpy(), eval_outputs) + + info = "step: {} evaluation metric: {}".format( + current_step, eval_outputs) + self._log_info(info) + + self.eval_summary_manager.write_summaries(eval_outputs) + self.eval_summary_manager.flush() + + def _maybe_save_checkpoints(self, current_step, force_trigger=False): + if self.checkpoint_manager and self.checkpoint_manager.checkpoint_interval: + ckpt_path = self.checkpoint_manager.save( + checkpoint_number=current_step, check_interval=not force_trigger) + if ckpt_path is not None: + logging.info("Saved checkpoins in %s", ckpt_path) + + def _maybe_evaluate(self, current_step, force_trigger=False): + if self.eval_trigger(current_step, force_trigger): + self._evaluate_once(current_step) + + def _log_info(self, message): + """Logs `message` to the `info` log, and also prints to stdout.""" + logging.info(message) + print(message) + + def train(self, evaluate=True): + """Runs the training, with optional evaluation. + + This handles evaluation, gathering summaries, and saving checkpoints. + + Args: + evaluate: A boolean indicates whether to perform evaluation during + training. + + Raises: + RuntimeError: If `global_step` is not updated correctly in `train_fn`. + """ + if self.train_fn is None: + raise ValueError("`self.train_fn` is required when calling `train` " + "method.") + if self.global_step is None: + raise ValueError("`self.global_step` is required when calling `train` " + "method.") + if evaluate and self.eval_fn is None: + raise ValueError("`self.eval_fn` is required when calling `train` method " + "with `evaluate=True`") + + step_timer = _StepTimer(self.global_step) + current_step = self.global_step.numpy() + logging.info("Train at step %s of %s", current_step, self.train_steps) + while current_step < self.train_steps: + # Calculates steps to run for the next train loop. + steps_per_loop = min(self.train_steps - current_step, self.steps_per_loop) + logging.info("Entering training loop with %s steps, at step %s of %s", + steps_per_loop, current_step, self.train_steps) + current_step += steps_per_loop + steps_per_loop = tf.convert_to_tensor(steps_per_loop, dtype=tf.int32) + + with self.summary_manager.summary_writer.as_default(): + train_outputs = self.train_fn(steps_per_loop) + + # Updates and verifies the current step after a training loop finishes. + if current_step != self.global_step.numpy(): + raise RuntimeError("`self.train_fn` is not updating `global_step` " + "correctly, expected: %s, actual: %s" % + (current_step, self.global_step.numpy())) + + # Print information like metrics and steps_per_second after a training + # loop. + if train_outputs: + train_outputs = tf.nest.map_structure( + lambda x: x.numpy(), train_outputs) + steps_per_second = step_timer.steps_per_second() + info = "step: {} steps_per_second: {:.2f} {}".format( + current_step, steps_per_second, train_outputs) + self._log_info(info) + + train_outputs = train_outputs or {} + train_outputs["steps_per_second"] = steps_per_second + self.summary_manager.write_summaries(train_outputs) + + self._maybe_save_checkpoints(current_step) + + if evaluate: + self._maybe_evaluate(current_step) + + self.summary_manager.write_summaries(train_outputs, always_write=True) + self.summary_manager.flush() + self._maybe_save_checkpoints(current_step, force_trigger=True) + if evaluate: + self._maybe_evaluate(current_step, force_trigger=True) + + def evaluate(self, continuous=False, timeout_fn=None): + """Runs the evaluation. + + Args: + continuous: If `True`, will continously monitor the checkpoint directory + to evaluate on the latest checkpoint. If `False`, will do the evaluation + once. + timeout_fn: Optional callable to call after a timeout. If the function + returns True, then it means that no new checkpoints will be generated + and the iterator will exit. + + Raises: + ValueError: If no checkpoint found in `self.checkpoint_manager.directory`. + """ + if self.eval_fn is None: + raise ValueError("`self.eval_fn` should not be None to call " + "`evaluate()` method.") + + if not continuous and timeout_fn is not None: + raise ValueError("`timeout_fn` can be only passed when `continuous` is " + "True") + + if continuous: + for checkpoint_path in tf.train.checkpoints_iterator( + self.checkpoint_manager.directory, timeout_fn=timeout_fn): + self._restore_model(checkpoint_path) + self._evaluate_once(self.global_step.numpy()) + return + + latest_checkpoint = self.checkpoint_manager.latest_checkpoint + if not latest_checkpoint: + raise ValueError("no checkpoint found in dir %s" % + self.checkpoint_manager.directory) + self._restore_model() + self._evaluate_once(self.global_step.numpy()) + + +class _StepTimer(object): + """Utility class for measuring steps/second.""" + + def __init__(self, step): + self.step = step + self.start() + + def start(self): + self.last_iteration = self.step.numpy() + self.last_time = time.time() + + def steps_per_second(self, restart=True): + value = ((self.step.numpy() - self.last_iteration) / + (time.time() - self.last_time)) + if restart: + self.start() + return value diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/controller_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/controller_test.py new file mode 100644 index 0000000..eeaa191 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/controller_test.py @@ -0,0 +1,308 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for official.staging.training.controller.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from absl.testing import parameterized +import numpy as np +import tensorflow as tf + +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +from official.staging.training import controller +from official.staging.training import standard_runnable + + +def all_strategy_combinations(): + """Gets combinations of distribution strategies.""" + return combinations.combine( + strategy=[ + strategy_combinations.one_device_strategy, + strategy_combinations.tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + strategy_combinations.mirrored_strategy_with_gpu_and_cpu, + ], + mode="eager", + ) + + +def create_model(): + x = tf.keras.layers.Input(shape=(3,), name="input") + y = tf.keras.layers.Dense(4, name="dense")(x) + model = tf.keras.Model(x, y) + return model + + +def summaries_with_matching_keyword(keyword, summary_dir): + """Yields summary protos matching given keyword from event file.""" + event_paths = tf.io.gfile.glob(os.path.join(summary_dir, "events*")) + for event in tf.compat.v1.train.summary_iterator(event_paths[-1]): + if event.summary is not None: + for value in event.summary.value: + if keyword in value.tag: + tf.compat.v1.logging.error(event) + yield event.summary + + +def check_eventfile_for_keyword(keyword, summary_dir): + """Checks event files for the keyword.""" + return any(summaries_with_matching_keyword(keyword, summary_dir)) + + +def dataset_fn(ctx): + del ctx + inputs = np.zeros((10, 3), dtype=np.float32) + targets = np.zeros((10, 4), dtype=np.float32) + dataset = tf.data.Dataset.from_tensor_slices((inputs, targets)) + dataset = dataset.repeat(100) + dataset = dataset.batch(10, drop_remainder=True) + return dataset + + +class TestRunnable(standard_runnable.StandardTrainable, + standard_runnable.StandardEvaluable): + """Implements the training and evaluation APIs for the test model.""" + + def __init__(self): + standard_runnable.StandardTrainable.__init__(self) + standard_runnable.StandardEvaluable.__init__(self) + self.strategy = tf.distribute.get_strategy() + self.model = create_model() + self.optimizer = tf.keras.optimizers.RMSprop() + self.global_step = self.optimizer.iterations + self.train_loss = tf.keras.metrics.Mean("train_loss", dtype=tf.float32) + self.eval_loss = tf.keras.metrics.Mean("eval_loss", dtype=tf.float32) + + def build_train_dataset(self): + return self.strategy.experimental_distribute_datasets_from_function( + dataset_fn) + + def train_step(self, iterator): + + def _replicated_step(inputs): + """Replicated training step.""" + inputs, targets = inputs + with tf.GradientTape() as tape: + outputs = self.model(inputs) + loss = tf.math.reduce_sum(outputs - targets) + grads = tape.gradient(loss, self.model.variables) + self.optimizer.apply_gradients(zip(grads, self.model.variables)) + self.train_loss.update_state(loss) + + self.strategy.run(_replicated_step, args=(next(iterator),)) + + def train_loop_end(self): + return { + "loss": self.train_loss.result(), + } + + def build_eval_dataset(self): + return self.strategy.experimental_distribute_datasets_from_function( + dataset_fn) + + def eval_begin(self): + self.eval_loss.reset_states() + + def eval_step(self, iterator): + + def _replicated_step(inputs): + """Replicated evaluation step.""" + inputs, targets = inputs + outputs = self.model(inputs) + loss = tf.math.reduce_sum(outputs - targets) + self.eval_loss.update_state(loss) + + self.strategy.run(_replicated_step, args=(next(iterator),)) + + def eval_end(self): + return { + "eval_loss": self.eval_loss.result(), + } + + +class ControllerTest(tf.test.TestCase, parameterized.TestCase): + + def setUp(self): + super(ControllerTest, self).setUp() + self.model_dir = self.get_temp_dir() + + def test_no_checkpoint(self): + test_runnable = TestRunnable() + # No checkpoint manager and no strategy. + test_controller = controller.Controller( + train_fn=test_runnable.train, + eval_fn=test_runnable.evaluate, + global_step=test_runnable.global_step, + train_steps=10, + steps_per_loop=2, + summary_dir=os.path.join(self.model_dir, "summaries/train"), + summary_interval=2, + eval_summary_dir=os.path.join(self.model_dir, "summaries/eval"), + eval_steps=2, + eval_interval=5) + test_controller.train(evaluate=True) + self.assertEqual(test_runnable.global_step.numpy(), 10) + # Loss and accuracy values should be written into summaries. + self.assertNotEmpty( + tf.io.gfile.listdir(os.path.join(self.model_dir, "summaries/train"))) + self.assertTrue( + check_eventfile_for_keyword( + "loss", os.path.join(self.model_dir, "summaries/train"))) + self.assertNotEmpty( + tf.io.gfile.listdir(os.path.join(self.model_dir, "summaries/eval"))) + self.assertTrue( + check_eventfile_for_keyword( + "eval_loss", os.path.join(self.model_dir, "summaries/eval"))) + # No checkpoint, so global step starts from 0. + test_runnable.global_step.assign(0) + test_controller.train(evaluate=True) + self.assertEqual(test_runnable.global_step.numpy(), 10) + + def test_no_checkpoint_and_summaries(self): + test_runnable = TestRunnable() + # No checkpoint + summary directories. + test_controller = controller.Controller( + train_fn=test_runnable.train, + eval_fn=test_runnable.evaluate, + global_step=test_runnable.global_step, + train_steps=10, + steps_per_loop=2, + eval_steps=2, + eval_interval=5) + test_controller.train(evaluate=True) + self.assertEqual(test_runnable.global_step.numpy(), 10) + + @combinations.generate(all_strategy_combinations()) + def test_train_and_evaluate(self, strategy): + with strategy.scope(): + test_runnable = TestRunnable() + + checkpoint = tf.train.Checkpoint( + model=test_runnable.model, optimizer=test_runnable.optimizer) + checkpoint_manager = tf.train.CheckpointManager( + checkpoint, + self.model_dir, + max_to_keep=None, + step_counter=test_runnable.global_step, + checkpoint_interval=10) + test_controller = controller.Controller( + strategy=strategy, + train_fn=test_runnable.train, + eval_fn=test_runnable.evaluate, + global_step=test_runnable.global_step, + train_steps=10, + steps_per_loop=2, + summary_dir=os.path.join(self.model_dir, "summaries/train"), + summary_interval=2, + checkpoint_manager=checkpoint_manager, + eval_summary_dir=os.path.join(self.model_dir, "summaries/eval"), + eval_steps=2, + eval_interval=5) + test_controller.train(evaluate=True) + + # Checkpoints are saved. + self.assertNotEmpty(tf.io.gfile.glob(os.path.join(self.model_dir, "ckpt*"))) + + # Loss and accuracy values should be written into summaries. + self.assertNotEmpty( + tf.io.gfile.listdir(os.path.join(self.model_dir, "summaries/train"))) + self.assertTrue( + check_eventfile_for_keyword( + "loss", os.path.join(self.model_dir, "summaries/train"))) + self.assertNotEmpty( + tf.io.gfile.listdir(os.path.join(self.model_dir, "summaries/eval"))) + self.assertTrue( + check_eventfile_for_keyword( + "eval_loss", os.path.join(self.model_dir, "summaries/eval"))) + + @combinations.generate(all_strategy_combinations()) + def test_train_only(self, strategy): + with strategy.scope(): + test_runnable = TestRunnable() + + checkpoint = tf.train.Checkpoint( + model=test_runnable.model, optimizer=test_runnable.optimizer) + checkpoint_manager = tf.train.CheckpointManager( + checkpoint, + self.model_dir, + max_to_keep=None, + step_counter=test_runnable.global_step, + checkpoint_interval=10) + test_controller = controller.Controller( + strategy=strategy, + train_fn=test_runnable.train, + global_step=test_runnable.global_step, + train_steps=10, + steps_per_loop=2, + summary_dir=os.path.join(self.model_dir, "summaries/train"), + summary_interval=2, + checkpoint_manager=checkpoint_manager, + eval_summary_dir=os.path.join(self.model_dir, "summaries/eval"), + ) + test_controller.train(evaluate=False) + + # Checkpoints are saved. + self.assertNotEmpty(tf.io.gfile.glob(os.path.join(self.model_dir, "ckpt*"))) + + # Only train summaries are written. + self.assertNotEmpty( + tf.io.gfile.listdir(os.path.join(self.model_dir, "summaries/train"))) + self.assertTrue( + check_eventfile_for_keyword( + "loss", os.path.join(self.model_dir, "summaries/train"))) + self.assertFalse( + tf.io.gfile.exists(os.path.join(self.model_dir, "summaries/eval"))) + + @combinations.generate(all_strategy_combinations()) + def test_evaluate_only(self, strategy): + with strategy.scope(): + test_runnable = TestRunnable() + + checkpoint = tf.train.Checkpoint(model=test_runnable.model) + checkpoint.save(os.path.join(self.model_dir, "ckpt")) + + checkpoint_manager = tf.train.CheckpointManager( + checkpoint, + self.model_dir, + max_to_keep=None, + step_counter=test_runnable.global_step) + test_controller = controller.Controller( + strategy=strategy, + eval_fn=test_runnable.evaluate, + global_step=test_runnable.global_step, + checkpoint_manager=checkpoint_manager, + summary_dir=os.path.join(self.model_dir, "summaries/train"), + eval_summary_dir=os.path.join(self.model_dir, "summaries/eval"), + eval_steps=2, + eval_interval=5) + test_controller.evaluate() + + # Only eval summaries are written + self.assertFalse( + tf.io.gfile.exists(os.path.join(self.model_dir, "summaries/train"))) + self.assertNotEmpty( + tf.io.gfile.listdir(os.path.join(self.model_dir, "summaries/eval"))) + self.assertTrue( + check_eventfile_for_keyword( + "eval_loss", os.path.join(self.model_dir, "summaries/eval"))) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/grad_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/grad_utils.py new file mode 100644 index 0000000..efda2e7 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/grad_utils.py @@ -0,0 +1,143 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Some gradient util functions to help users writing custom training loop.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +from absl import logging + +import tensorflow.compat.v2 as tf + + +def _filter_grads(grads_and_vars): + """Filter out iterable with grad equal to None.""" + grads_and_vars = tuple(grads_and_vars) + if not grads_and_vars: + return grads_and_vars + filtered = [] + vars_with_empty_grads = [] + for grad, var in grads_and_vars: + if grad is None: + vars_with_empty_grads.append(var) + else: + filtered.append((grad, var)) + filtered = tuple(filtered) + if not filtered: + raise ValueError("No gradients provided for any variable: %s." % + ([v.name for _, v in grads_and_vars],)) + if vars_with_empty_grads: + logging.warning( + ("Gradients do not exist for variables %s when minimizing the loss."), + ([v.name for v in vars_with_empty_grads])) + return filtered + + +def _filter_and_allreduce_gradients(grads_and_vars, + allreduce_precision="float32"): + """Filter None grads and then allreduce gradients in specified precision. + + This utils function is used when users intent to explicitly allreduce + gradients and customize gradients operations before and after allreduce. + The allreduced gradients are then passed to optimizer.apply_gradients( + experimental_aggregate_gradients=False). + + Arguments: + grads_and_vars: gradients and variables pairs. + allreduce_precision: Whether to allreduce gradients in float32 or float16. + + Returns: + pairs of allreduced non-None gradients and variables. + """ + filtered_grads_and_vars = _filter_grads(grads_and_vars) + (grads, variables) = zip(*filtered_grads_and_vars) + if allreduce_precision == "float16": + grads = [tf.cast(grad, "float16") for grad in grads] + allreduced_grads = tf.distribute.get_replica_context().all_reduce( + tf.distribute.ReduceOp.SUM, grads) + if allreduce_precision == "float16": + allreduced_grads = [tf.cast(grad, "float32") for grad in allreduced_grads] + return allreduced_grads, variables + + +def _run_callbacks(callbacks, grads_and_vars): + for callback in callbacks: + grads_and_vars = callback(grads_and_vars) + return grads_and_vars + + +def minimize_using_explicit_allreduce(tape, + optimizer, + loss, + trainable_variables, + pre_allreduce_callbacks=None, + post_allreduce_callbacks=None): + """Minimizes loss for one step by updating `trainable_variables`. + + Minimizes loss for one step by updating `trainable_variables`. + This explicitly performs gradient allreduce, instead of relying on implicit + allreduce in optimizer.apply_gradients(). If training using FP16 mixed + precision, explicit allreduce will aggregate gradients in FP16 format. + For TPU and GPU training using FP32, explicit allreduce will aggregate + gradients in FP32 format. + + Arguments: + tape: An instance of `tf.GradientTape`. + optimizer: An instance of `tf.keras.optimizers.Optimizer`. + loss: the loss tensor. + trainable_variables: A list of model Variables. + pre_allreduce_callbacks: A list of callback functions that takes gradients + and model variables pairs as input, manipulate them, and returns a new + gradients and model variables pairs. The callback functions will be + invoked in the list order and before gradients are allreduced. + With mixed precision training, the pre_allreduce_allbacks will be + applied on scaled_gradients. Default is no callbacks. + post_allreduce_callbacks: A list of callback functions that takes + gradients and model variables pairs as input, manipulate them, and + returns a new gradients and model variables paris. The callback + functions will be invoked in the list order and right before gradients + are applied to variables for updates. Default is no callbacks. + """ + if isinstance(optimizer, + tf.keras.mixed_precision.experimental.LossScaleOptimizer): + # FP16 GPU code path + with tape: + scaled_loss = optimizer.get_scaled_loss(loss) + scaled_grads = tape.gradient(scaled_loss, trainable_variables) + grads_and_vars = zip(scaled_grads, trainable_variables) + if pre_allreduce_callbacks: + grads_and_vars = _run_callbacks(pre_allreduce_callbacks, grads_and_vars) + (allreduced_scaled_grads, + filtered_training_vars) = _filter_and_allreduce_gradients( + grads_and_vars, allreduce_precision="float16") + allreduced_unscaled_grads = optimizer.get_unscaled_gradients( + allreduced_scaled_grads) + grads_and_vars = zip(allreduced_unscaled_grads, filtered_training_vars) + else: + # TPU or FP32 GPU code path + grads = tape.gradient(loss, trainable_variables) + grads_and_vars = zip(grads, trainable_variables) + if pre_allreduce_callbacks: + grads_and_vars = _run_callbacks(pre_allreduce_callbacks, grads_and_vars) + (allreduced_grads, + filtered_training_vars) = _filter_and_allreduce_gradients( + grads_and_vars, allreduce_precision="float32") + grads_and_vars = zip(allreduced_grads, filtered_training_vars) + if post_allreduce_callbacks: + grads_and_vars = _run_callbacks(post_allreduce_callbacks, grads_and_vars) + optimizer.apply_gradients( + grads_and_vars, experimental_aggregate_gradients=False) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/runnable.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/runnable.py new file mode 100644 index 0000000..1af6eca --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/runnable.py @@ -0,0 +1,79 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""An abstraction that users can easily handle their custom training loops.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import abc +import six +import tensorflow.compat.v2 as tf +from typing import Dict, Optional, Text + + +@six.add_metaclass(abc.ABCMeta) +class AbstractTrainable(tf.Module): + """An abstract class defining the APIs required for training.""" + + @abc.abstractmethod + def train(self, + num_steps: Optional[tf.Tensor]) -> Optional[Dict[Text, tf.Tensor]]: + """Implements model training with multiple steps. + + In training, it is common to break the total training steps into several + training loops, so users can do checkpointing, write summaries and run some + python callbacks. This is necessary for getting good performance in TPU + training, as the overhead for launching a multi worker tf.function may be + large in Eager mode. It is usually encouraged to create a host training loop + (e.g. using a `tf.range` wrapping `strategy.run` inside a + `tf.function`) in the TPU case. For the cases that don't require host + training loop to acheive peak performance, users can just implement a simple + python loop to drive each step. + + Args: + num_steps: A guideline for how many training steps to run. Note that it is + up to the model what constitutes a "step" (this may involve more than + one update to model parameters, e.g. if training a GAN). + + Returns: + The function may return a dictionary of `Tensors`, which will be + written to logs and as TensorBoard summaries. + """ + pass + + +@six.add_metaclass(abc.ABCMeta) +class AbstractEvaluable(tf.Module): + """An abstract class defining the APIs required for evaluation.""" + + @abc.abstractmethod + def evaluate( + self, num_steps: Optional[tf.Tensor]) -> Optional[Dict[Text, tf.Tensor]]: + """Implements model evaluation. + + Args: + num_steps: A guideline for how many evaluation steps to run. Note that it + is up to the model what constitutes a "step". Generally, it may be + desirable to support both a limited number of eval steps and iterating + over a full dataset (however many steps are required) when `num_steps` + is `None`. + + Returns: + The function may return a dictionary of `Tensors`, which will be + written to logs and as TensorBoard summaries. + """ + pass diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/standard_runnable.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/standard_runnable.py new file mode 100644 index 0000000..20dd66f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/standard_runnable.py @@ -0,0 +1,181 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""An abstraction that users can easily handle their custom training loops.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import abc +import six +import tensorflow.compat.v2 as tf +from typing import Dict, Optional, Text + +from official.staging.training import runnable +from official.staging.training import utils + + +@six.add_metaclass(abc.ABCMeta) +class StandardTrainable(runnable.AbstractTrainable): + """Implements the standard functionality of AbstractTrainable APIs.""" + + def __init__(self, use_tf_while_loop=True, use_tf_function=True): + if use_tf_while_loop and not use_tf_function: + raise ValueError("`use_tf_while_loop=True` and `use_tf_function=False` " + "is not supported") + self.use_tf_while_loop = use_tf_while_loop + self.use_tf_function = use_tf_function + self.train_dataset = None + self.train_iter = None + self.train_loop_fn = None + + @abc.abstractmethod + def build_train_dataset(self): + """Builds the training datasets. + + Returns: + A tf.nest-compatible structure of tf.data.Dataset or DistributedDataset. + """ + pass + + def train(self, + num_steps: Optional[tf.Tensor]) -> Optional[Dict[Text, tf.Tensor]]: + """See base class.""" + if self.train_dataset is None: + # Build train input dataset + self.train_dataset = self.build_train_dataset() + self.train_iter = tf.nest.map_structure(iter, self.train_dataset) + + if self.train_loop_fn is None: + train_fn = self.train_step + if self.use_tf_while_loop: + self.train_loop_fn = utils.create_tf_while_loop_fn(train_fn) + else: + if self.use_tf_function: + train_fn = tf.function(train_fn) + self.train_loop_fn = utils.create_loop_fn(train_fn) + + self.train_loop_begin() + self.train_loop_fn(self.train_iter, num_steps) + return self.train_loop_end() + + def train_loop_begin(self): + """Called once at the beginning of the training loop. + + This is a good place to reset metrics that accumulate values over multiple + steps of training. + """ + pass + + @abc.abstractmethod + def train_step(self, iterator): + """Implements one step of training. + + What a "step" consists of is up to the implementer. If using distribution + strategies, the call to this method should take place in the "cross-replica + context" for generality, to allow e.g. multiple iterator dequeues and calls + to `strategy.run`. + + Args: + iterator: A tf.nest-compatible structure of tf.data Iterator or + DistributedIterator. + """ + pass + + def train_loop_end(self) -> Optional[Dict[Text, tf.Tensor]]: + """Called at the end of the training loop. + + This is a good place to get metric results. The value returned from this + function will be returned as-is from the train() method. + + Returns: + The function may return a dictionary of `Tensors`, which will be + written to logs and as TensorBoard summaries. + """ + pass + + +@six.add_metaclass(abc.ABCMeta) +class StandardEvaluable(runnable.AbstractEvaluable): + """Implements the standard functionality of AbstractEvaluable APIs.""" + + def __init__(self, use_tf_function=True): + self.eval_use_tf_function = use_tf_function + self.eval_dataset = None + self.eval_loop_fn = None + + @abc.abstractmethod + def build_eval_dataset(self): + """Builds the evaluation datasets. + + Returns: + A tf.nest-compatible structure of tf.data.Dataset or DistributedDataset. + """ + pass + + def evaluate( + self, num_steps: Optional[tf.Tensor]) -> Optional[Dict[Text, tf.Tensor]]: + """See base class.""" + if self.eval_dataset is None: + # Build train input dataset + self.eval_dataset = self.build_eval_dataset() + + if self.eval_loop_fn is None: + eval_fn = self.eval_step + if self.eval_use_tf_function: + eval_fn = tf.function(eval_fn) + self.eval_loop_fn = utils.create_loop_fn(eval_fn) + + eval_iter = tf.nest.map_structure(iter, self.eval_dataset) + + self.eval_begin() + self.eval_loop_fn(eval_iter, num_steps) + return self.eval_end() + + def eval_begin(self): + """Called once at the beginning of the evaluation. + + This is a good place to reset metrics that accumulate values over the entire + evaluation. + """ + pass + + @abc.abstractmethod + def eval_step(self, iterator): + """Implements one step of evaluation. + + What a "step" consists of is up to the implementer. If using distribution + strategies, the call to this method should take place in the "cross-replica + context" for generality, to allow e.g. multiple iterator dequeues and calls + to `strategy.run`. + + Args: + iterator: A tf.nest-compatible structure of tf.data Iterator or + DistributedIterator. + """ + pass + + def eval_end(self) -> Optional[Dict[Text, tf.Tensor]]: + """Called at the end of the evaluation. + + This is a good place to get metric results. The value returned from this + function will be returned as-is from the evaluate() method. + + Returns: + The function may return a dictionary of `Tensors`, which will be + written to logs and as TensorBoard summaries. + """ + pass diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/utils.py new file mode 100644 index 0000000..33fa368 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/staging/training/utils.py @@ -0,0 +1,342 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Some layered modules/functions to help users writing custom training loop.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import abc +import inspect +import six + +import tensorflow.compat.v2 as tf + + +def create_loop_fn(step_fn): + """Creates a multiple steps function driven by the python while loop. + + Args: + step_fn: A function which takes `iterator` as input. + + Returns: + A callable defined as the `loop_fn` defination below. + """ + + def loop_fn(iterator, num_steps, state=None, reduce_fn=None): + """A loop function with multiple steps. + + Args: + iterator: A nested structure of tf.data `Iterator` or + `DistributedIterator`. + num_steps: The number of steps in the loop. If `num_steps==-1`, will + iterate until exausting the iterator. + state: An optional initial state before running the loop. + reduce_fn: a callable defined as `def reduce_fn(state, value)`, where + `value` is the outputs from `step_fn`. + + Returns: + The updated state. + """ + try: + step = 0 + # To make sure the OutOfRangeError exception can be handled well with + # async remote eager, we need to wrap the loop body in a `async_scope`. + with tf.experimental.async_scope(): + while (num_steps == -1 or step < num_steps): + outputs = step_fn(iterator) + if reduce_fn is not None: + state = reduce_fn(state, outputs) + step += 1 + return state + except (StopIteration, tf.errors.OutOfRangeError): + tf.experimental.async_clear_error() + return state + + return loop_fn + + +def create_tf_while_loop_fn(step_fn): + """Create a multiple steps function driven by tf.while_loop on the host. + + Args: + step_fn: A function which takes `iterator` as input. + + Returns: + A callable defined as the `loop_fn` defination below. + """ + + @tf.function + def loop_fn(iterator, num_steps): + """A loop function with multiple steps. + + Args: + iterator: A nested structure of tf.data `Iterator` or + `DistributedIterator`. + num_steps: The number of steps in the loop. Must be a tf.Tensor. + """ + if not isinstance(num_steps, tf.Tensor): + raise ValueError("`num_steps` should be an `tf.Tensor`. Python object " + "may cause retracing.") + + for _ in tf.range(num_steps): + step_fn(iterator) + + return loop_fn + + +def make_distributed_dataset(strategy, dataset_or_fn, *args, **kwargs): + """A helper function to create distributed dataset. + + Args: + strategy: An instance of `tf.distribute.Strategy`. + dataset_or_fn: A instance of `tf.data.Dataset` or a function which takes an + `tf.distribute.InputContext` as input and returns a `tf.data.Dataset`. If + it is a function, it could optionally have an argument named + `input_context` which is `tf.distribute.InputContext` argument type. + *args: The list of arguments to be passed to dataset_or_fn. + **kwargs: Any keyword arguments to be passed. + + Returns: + A distributed Dataset. + """ + if strategy is None: + strategy = tf.distribute.get_strategy() + + if isinstance(dataset_or_fn, tf.data.Dataset): + return strategy.experimental_distribute_dataset(dataset_or_fn) + + if not callable(dataset_or_fn): + raise ValueError("`dataset_or_fn` should be either callable or an instance " + "of `tf.data.Dataset`") + + def dataset_fn(ctx): + """Wrapped dataset function for creating distributed dataset..""" + + # If `dataset_or_fn` is a function and has `input_context` as argument + # names, pass `ctx` as the value of `input_context` when calling + # `dataset_or_fn`. Otherwise `ctx` will not be used when calling + # `dataset_or_fn`. + if six.PY3: + argspec = inspect.getfullargspec(dataset_or_fn) + else: + argspec = inspect.getargspec(dataset_or_fn) + args_names = argspec.args + + if "input_context" in args_names: + kwargs["input_context"] = ctx + ds = dataset_or_fn(*args, **kwargs) + return ds + + return strategy.experimental_distribute_datasets_from_function(dataset_fn) + + +class SummaryManager(object): + """A class manages writing summaries.""" + + def __init__(self, + summary_writer, + summary_fn, + global_step=None, + summary_interval=None): + """Construct a summary manager object. + + Args: + summary_writer: A `tf.summary.SummaryWriter` instance for writing + summaries. + summary_fn: A callable defined as `def summary_fn(name, tensor, + step=None)`, which describes the summary operation. + global_step: A `tf.Variable` instance for checking the current global step + value, in case users want to save summaries every N steps. + summary_interval: An integer, indicates the minimum step interval between + two summaries. + """ + if summary_writer is not None: + self._summary_writer = summary_writer + self._enabled = True + else: + self._summary_writer = tf.summary.create_noop_writer() + self._enabled = False + self._summary_fn = summary_fn + + if global_step is None: + self._global_step = tf.summary.experimental.get_step() + else: + self._global_step = global_step + + if summary_interval is not None: + if self._global_step is None: + raise ValueError("`summary_interval` is not None, but no `global_step` " + "can be obtained ") + self._last_summary_step = self._global_step.numpy() + self._summary_interval = summary_interval + + @property + def summary_interval(self): + return self._summary_interval + + @property + def summary_writer(self): + """Returns the underlying summary writer.""" + return self._summary_writer + + def flush(self): + """Flush the underlying summary writer.""" + if self._enabled: + tf.summary.flush(self._summary_writer) + + def write_summaries(self, items, always_write=True): + """Write a bulk of summaries. + + Args: + items: a dictionary of `Tensors` for writing summaries. + always_write: An optional boolean. If `True`, the manager will always + write summaries unless the summaries have been written for the same + step. Otherwise the manager will only write the summaries if the + interval between summaries are larger than `summary_interval`. + + Returns: + A boolean indicates whether the summaries are written or not. + """ + # TODO(rxsang): Support writing summaries with nested structure, so users + # can split the summaries into different directories for nicer visualization + # in Tensorboard, like train and eval metrics. + if not self._enabled: + return False + + if self._summary_interval is not None: + current_step = self._global_step.numpy() + if current_step == self._last_summary_step: + return False + if not always_write and current_step < (self._last_summary_step + + self._summary_interval): + return False + self._last_summary_step = current_step + + with self._summary_writer.as_default(): + for name, tensor in items.items(): + self._summary_fn(name, tensor, step=self._global_step) + return True + + +@six.add_metaclass(abc.ABCMeta) +class Trigger(object): + """An abstract class representing a "trigger" for some event.""" + + @abc.abstractmethod + def __call__(self, value: float, force_trigger=False): + """Maybe trigger the event based on the given value. + + Args: + value: the value for triggering. + force_trigger: Whether the trigger is forced triggered. + + Returns: + `True` if the trigger is triggered on the given `value`, and + `False` otherwise. + """ + + @abc.abstractmethod + def reset(self): + """Reset states in the trigger.""" + + +class IntervalTrigger(Trigger): + """Triggers on every fixed interval.""" + + def __init__(self, interval, start=0): + """Constructs the IntervalTrigger. + + Args: + interval: The triggering interval. + start: An initial value for the trigger. + """ + self._interval = interval + self._last_trigger_value = start + + def __call__(self, value, force_trigger=False): + """Maybe trigger the event based on the given value. + + Args: + value: the value for triggering. + force_trigger: If True, the trigger will be forced triggered unless the + last trigger value is equal to `value`. + + Returns: + `True` if the trigger is triggered on the given `value`, and + `False` otherwise. + """ + if force_trigger and value != self._last_trigger_value: + self._last_trigger_value = value + return True + + if self._interval and self._interval > 0: + if value >= self._last_trigger_value + self._interval: + self._last_trigger_value = value + return True + return False + + def reset(self): + """See base class.""" + self._last_trigger_value = 0 + + +class EpochHelper(object): + """A Helper class to handle epochs in Customized Training Loop.""" + + def __init__(self, epoch_steps, global_step): + """Constructs the EpochHelper. + + Args: + epoch_steps: An integer indicates how many steps in an epoch. + global_step: A `tf.Variable` instance indicates the current global step. + """ + self._epoch_steps = epoch_steps + self._global_step = global_step + self._current_epoch = None + self._epoch_start_step = None + self._in_epoch = False + + def epoch_begin(self): + """Returns whether a new epoch should begin.""" + if self._in_epoch: + return False + current_step = self._global_step.numpy() + self._epoch_start_step = current_step + self._current_epoch = current_step // self._epoch_steps + self._in_epoch = True + return True + + def epoch_end(self): + """Returns whether the current epoch should end.""" + if not self._in_epoch: + raise ValueError("`epoch_end` can only be called inside an epoch") + current_step = self._global_step.numpy() + epoch = current_step // self._epoch_steps + + if epoch > self._current_epoch: + self._in_epoch = False + return True + return False + + @property + def batch_index(self): + """Index of the next batch within the current epoch.""" + return self._global_step.numpy() - self._epoch_start_step + + @property + def current_epoch(self): + return self._current_epoch diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/README.md new file mode 100644 index 0000000..18160f7 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/README.md @@ -0,0 +1,97 @@ +# Adding Abseil (absl) flags quickstart +## Defining a flag +absl flag definitions are similar to argparse, although they are defined on a global namespace. + +For instance defining a string flag looks like: +```$xslt +from absl import flags +flags.DEFINE_string( + name="my_flag", + default="a_sensible_default", + help="Here is what this flag does." +) +``` + +All three arguments are required, but default may be `None`. A common optional argument is +short_name for defining abreviations. Certain `DEFINE_*` methods will have other required arguments. +For instance `DEFINE_enum` requires the `enum_values` argument to be specified. + +## Key Flags +absl has the concept of a key flag. Any flag defined in `__main__` is considered a key flag by +default. Key flags are displayed in `--help`, others only appear in `--helpfull`. In order to +handle key flags that are defined outside the module in question, absl provides the +`flags.adopt_module_key_flags()` method. This adds the key flags of a different module to one's own +key flags. For example: +```$xslt +File: flag_source.py +--------------------------------------- + +from absl import flags +flags.DEFINE_string(name="my_flag", default="abc", help="a flag.") +``` + +```$xslt +File: my_module.py +--------------------------------------- + +from absl import app as absl_app +from absl import flags + +import flag_source + +flags.adopt_module_key_flags(flag_source) + +def main(_): + pass + +absl_app.run(main, [__file__, "-h"] +``` + +when `my_module.py` is run it will show the help text for `my_flag`. Because not all flags defined +in a file are equally important, `official/utils/flags/core.py` (generally imported as flags_core) +provides an abstraction for handling key flag declaration in an easy way through the +`register_key_flags_in_core()` function, which allows a module to make a single +`adopt_key_flags(flags_core)` call when using the util flag declaration functions. + +## Validators +Often the constraints on a flag are complicated. absl provides the validator decorator to allow +one to mark a function as a flag validation function. Suppose we want users to provide a flag +which is a palindrome. + +```$xslt +from absl import flags + +flags.DEFINE_string(name="pal_flag", short_name="pf", default="", help="Give me a palindrome") + +@flags.validator("pal_flag") +def _check_pal(provided_pal_flag): + return provided_pal_flag == provided_pal_flag[::-1] + +``` + +Validators take the form that returning True (truthy) passes, and all others +(False, None, exception) fail. + +## Testing +To test using absl, simply declare flags in the setupClass method of TensorFlow's TestCase. + +```$xslt +from absl import flags +import tensorflow as tf + +def define_flags(): + flags.DEFINE_string(name="test_flag", default="abc", help="an example flag") + + +class BaseTester(unittest.TestCase): + + @classmethod + def setUpClass(cls): + super(BaseTester, cls).setUpClass() + define_flags() + + def test_trivial(self): + flags_core.parse_flags([__file__, "test_flag", "def"]) + self.AssertEqual(flags.FLAGS.test_flag, "def") + +``` diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_base.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_base.py new file mode 100644 index 0000000..8d0f675 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_base.py @@ -0,0 +1,173 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Flags which will be nearly universal across models.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import flags +import tensorflow as tf + +from official.utils.flags._conventions import help_wrap +from official.utils.logs import hooks_helper + +############## npu modify begin ############# +from hccl.manage.api import get_rank_size +from hccl.manage.api import get_rank_id +############## npu modify end ############### + + +def define_base(data_dir=True, model_dir=True, clean=False, train_epochs=False, + epochs_between_evals=False, stop_threshold=False, + batch_size=True, num_gpu=False, hooks=False, export_dir=False, + distribution_strategy=False, run_eagerly=False): + """Register base flags. + + Args: + data_dir: Create a flag for specifying the input data directory. + model_dir: Create a flag for specifying the model file directory. + clean: Create a flag for removing the model_dir. + train_epochs: Create a flag to specify the number of training epochs. + epochs_between_evals: Create a flag to specify the frequency of testing. + stop_threshold: Create a flag to specify a threshold accuracy or other + eval metric which should trigger the end of training. + batch_size: Create a flag to specify the batch size. + num_gpu: Create a flag to specify the number of GPUs used. + hooks: Create a flag to specify hooks for logging. + export_dir: Create a flag to specify where a SavedModel should be exported. + distribution_strategy: Create a flag to specify which Distribution Strategy + to use. + run_eagerly: Create a flag to specify to run eagerly op by op. + Returns: + A list of flags for core.py to marks as key flags. + """ + key_flags = [] + + if data_dir: + flags.DEFINE_string( + name="data_dir", short_name="dd", default="/tmp", + help=help_wrap("The location of the input data.")) + key_flags.append("data_dir") + + if model_dir: + flags.DEFINE_string( + name="model_dir", short_name="md", default="/tmp", + help=help_wrap("The location of the model checkpoint files.")) + key_flags.append("model_dir") + + if clean: + flags.DEFINE_boolean( + name="clean", default=False, + help=help_wrap("If set, model_dir will be removed if it exists.")) + key_flags.append("clean") + + if train_epochs: + flags.DEFINE_integer( + name="train_epochs", short_name="te", default=1, + help=help_wrap("The number of epochs used to train.")) + key_flags.append("train_epochs") + + if epochs_between_evals: + flags.DEFINE_integer( + name="epochs_between_evals", short_name="ebe", default=1, + help=help_wrap("The number of training epochs to run between " + "evaluations.")) + key_flags.append("epochs_between_evals") + + if stop_threshold: + flags.DEFINE_float( + name="stop_threshold", short_name="st", + default=None, + help=help_wrap("If passed, training will stop at the earlier of " + "train_epochs and when the evaluation metric is " + "greater than or equal to stop_threshold.")) + + if batch_size: + flags.DEFINE_integer( + name="batch_size", short_name="bs", default=32, + help=help_wrap("Batch size for training and evaluation. When using " + "multiple gpus, this is the global batch size for " + "all devices. For example, if the batch size is 32 " + "and there are 4 GPUs, each GPU will get 8 examples on " + "each step.")) + key_flags.append("batch_size") + + if num_gpu: + flags.DEFINE_integer( + name="num_gpus", short_name="ng", + default=1, + help=help_wrap( + "How many GPUs to use at each worker with the " + "DistributionStrategies API. The default is 1.")) + + if run_eagerly: + flags.DEFINE_boolean( + name="run_eagerly", default=False, + help="Run the model op by op without building a model function.") + + if hooks: + # Construct a pretty summary of hooks. + hook_list_str = ( + u"\ufeff Hook:\n" + u"\n".join([u"\ufeff {}".format(key) for key + in hooks_helper.HOOKS])) + flags.DEFINE_list( + name="hooks", short_name="hk", default="LoggingTensorHook", + help=help_wrap( + u"A list of (case insensitive) strings to specify the names of " + u"training hooks.\n{}\n\ufeff Example: `--hooks ProfilerHook," + u"ExamplesPerSecondHook`\n See official.utils.logs.hooks_helper " + u"for details.".format(hook_list_str)) + ) + key_flags.append("hooks") + + if export_dir: + flags.DEFINE_string( + name="export_dir", short_name="ed", default=None, + help=help_wrap("If set, a SavedModel serialization of the model will " + "be exported to this directory at the end of training. " + "See the README for more details and relevant links.") + ) + key_flags.append("export_dir") + + if distribution_strategy: + flags.DEFINE_string( + name="distribution_strategy", short_name="ds", default="mirrored", + help=help_wrap("The Distribution Strategy to use for training. " + "Accepted values are 'off', 'one_device', " + "'mirrored', 'parameter_server', 'collective', " + "case insensitive. 'off' means not to use " + "Distribution Strategy; 'default' means to choose " + "from `MirroredStrategy` or `OneDeviceStrategy` " + "according to the number of GPUs.") + ) + + + return key_flags + +def get_num_gpus(flags_obj): + """get the num npus using hccl api""" + ############## npu modify begin ############# + return get_rank_size() + ############## npu modify end ############### + +# def get_num_gpus(flags_obj): +# """Treat num_gpus=-1 as 'use all'.""" +# if flags_obj.num_gpus != -1: +# return flags_obj.num_gpus + +# from tensorflow.python.client import device_lib # pylint: disable=g-import-not-at-top +# local_device_protos = device_lib.list_local_devices() +# return sum([1 for d in local_device_protos if d.device_type == "GPU"]) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_benchmark.py new file mode 100644 index 0000000..eddae80 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_benchmark.py @@ -0,0 +1,109 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Flags for benchmarking models.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import flags + +from official.utils.flags._conventions import help_wrap + + +def define_log_steps(): + flags.DEFINE_integer( + name="log_steps", default=100, + help="Frequency with which to log timing information with TimeHistory.") + + return [] + + +def define_benchmark(benchmark_log_dir=True, bigquery_uploader=True): + """Register benchmarking flags. + + Args: + benchmark_log_dir: Create a flag to specify location for benchmark logging. + bigquery_uploader: Create flags for uploading results to BigQuery. + + Returns: + A list of flags for core.py to marks as key flags. + """ + + key_flags = [] + + flags.DEFINE_enum( + name="benchmark_logger_type", default="BaseBenchmarkLogger", + enum_values=["BaseBenchmarkLogger", "BenchmarkFileLogger", + "BenchmarkBigQueryLogger"], + help=help_wrap("The type of benchmark logger to use. Defaults to using " + "BaseBenchmarkLogger which logs to STDOUT. Different " + "loggers will require other flags to be able to work.")) + flags.DEFINE_string( + name="benchmark_test_id", short_name="bti", default=None, + help=help_wrap("The unique test ID of the benchmark run. It could be the " + "combination of key parameters. It is hardware " + "independent and could be used compare the performance " + "between different test runs. This flag is designed for " + "human consumption, and does not have any impact within " + "the system.")) + + define_log_steps() + + if benchmark_log_dir: + flags.DEFINE_string( + name="benchmark_log_dir", short_name="bld", default=None, + help=help_wrap("The location of the benchmark logging.") + ) + + if bigquery_uploader: + flags.DEFINE_string( + name="gcp_project", short_name="gp", default=None, + help=help_wrap( + "The GCP project name where the benchmark will be uploaded.")) + + flags.DEFINE_string( + name="bigquery_data_set", short_name="bds", default="test_benchmark", + help=help_wrap( + "The Bigquery dataset name where the benchmark will be uploaded.")) + + flags.DEFINE_string( + name="bigquery_run_table", short_name="brt", default="benchmark_run", + help=help_wrap("The Bigquery table name where the benchmark run " + "information will be uploaded.")) + + flags.DEFINE_string( + name="bigquery_run_status_table", short_name="brst", + default="benchmark_run_status", + help=help_wrap("The Bigquery table name where the benchmark run " + "status information will be uploaded.")) + + flags.DEFINE_string( + name="bigquery_metric_table", short_name="bmt", + default="benchmark_metric", + help=help_wrap("The Bigquery table name where the benchmark metric " + "information will be uploaded.")) + + @flags.multi_flags_validator( + ["benchmark_logger_type", "benchmark_log_dir"], + message="--benchmark_logger_type=BenchmarkFileLogger will require " + "--benchmark_log_dir being set") + def _check_benchmark_log_dir(flags_dict): + benchmark_logger_type = flags_dict["benchmark_logger_type"] + if benchmark_logger_type == "BenchmarkFileLogger": + return flags_dict["benchmark_log_dir"] + return True + + return key_flags diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_conventions.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_conventions.py new file mode 100644 index 0000000..81ad21b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_conventions.py @@ -0,0 +1,54 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Central location for shared argparse convention definitions.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys +import codecs +import functools + +from absl import app as absl_app +from absl import flags + + +# This codifies help string conventions and makes it easy to update them if +# necessary. Currently the only major effect is that help bodies start on the +# line after flags are listed. All flag definitions should wrap the text bodies +# with help wrap when calling DEFINE_*. +_help_wrap = functools.partial(flags.text_wrap, length=80, indent="", + firstline_indent="\n") + + +# Pretty formatting causes issues when utf-8 is not installed on a system. +def _stdout_utf8(): + try: + codecs.lookup("utf-8") + except LookupError: + return False + return sys.stdout.encoding == "UTF-8" + + +if _stdout_utf8(): + help_wrap = _help_wrap +else: + def help_wrap(text, *args, **kwargs): + return _help_wrap(text, *args, **kwargs).replace(u"\ufeff", u"") + + +# Replace None with h to also allow -h +absl_app.HelpshortFlag.SHORT_NAME = "h" diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_device.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_device.py new file mode 100644 index 0000000..d8974fc --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_device.py @@ -0,0 +1,85 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Flags for managing compute devices. Currently only contains TPU flags.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import flags +from absl import logging + +from official.utils.flags._conventions import help_wrap + + +def require_cloud_storage(flag_names): + """Register a validator to check directory flags. + Args: + flag_names: An iterable of strings containing the names of flags to be + checked. + """ + msg = "TPU requires GCS path for {}".format(", ".join(flag_names)) + @flags.multi_flags_validator(["tpu"] + flag_names, message=msg) + def _path_check(flag_values): # pylint: disable=missing-docstring + if flag_values["tpu"] is None: + return True + + valid_flags = True + for key in flag_names: + if not flag_values[key].startswith("gs://"): + logging.error("%s must be a GCS path.", key) + valid_flags = False + + return valid_flags + + +def define_device(tpu=True): + """Register device specific flags. + Args: + tpu: Create flags to specify TPU operation. + Returns: + A list of flags for core.py to marks as key flags. + """ + + key_flags = [] + + if tpu: + flags.DEFINE_string( + name="tpu", default=None, + help=help_wrap( + "The Cloud TPU to use for training. This should be either the name " + "used when creating the Cloud TPU, or a " + "grpc://ip.address.of.tpu:8470 url. Passing `local` will use the" + "CPU of the local instance instead. (Good for debugging.)")) + key_flags.append("tpu") + + flags.DEFINE_string( + name="tpu_zone", default=None, + help=help_wrap( + "[Optional] GCE zone where the Cloud TPU is located in. If not " + "specified, we will attempt to automatically detect the GCE " + "project from metadata.")) + + flags.DEFINE_string( + name="tpu_gcp_project", default=None, + help=help_wrap( + "[Optional] Project name for the Cloud TPU-enabled project. If not " + "specified, we will attempt to automatically detect the GCE " + "project from metadata.")) + + flags.DEFINE_integer(name="num_tpu_shards", default=8, + help=help_wrap("Number of shards (TPU chips).")) + + return key_flags diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_distribution.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_distribution.py new file mode 100644 index 0000000..ca331bf --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_distribution.py @@ -0,0 +1,54 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Flags related to distributed execution.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import flags +import tensorflow as tf + +from official.utils.flags._conventions import help_wrap + + +def define_distribution(worker_hosts=True, task_index=True): + """Register distributed execution flags. + + Args: + worker_hosts: Create a flag for specifying comma-separated list of workers. + task_index: Create a flag for specifying index of task. + + Returns: + A list of flags for core.py to marks as key flags. + """ + key_flags = [] + + if worker_hosts: + flags.DEFINE_string( + name='worker_hosts', default=None, + help=help_wrap( + 'Comma-separated list of worker ip:port pairs for running ' + 'multi-worker models with DistributionStrategy. The user would ' + 'start the program on each host with identical value for this ' + 'flag.')) + + if task_index: + flags.DEFINE_integer( + name='task_index', default=-1, + help=help_wrap('If multi-worker training, the task_index of this ' + 'worker.')) + + return key_flags diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_misc.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_misc.py new file mode 100644 index 0000000..c6fa24b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_misc.py @@ -0,0 +1,50 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Misc flags.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import flags + +from official.utils.flags._conventions import help_wrap + + +def define_image(data_format=True): + """Register image specific flags. + + Args: + data_format: Create a flag to specify image axis convention. + + Returns: + A list of flags for core.py to marks as key flags. + """ + + key_flags = [] + + if data_format: + flags.DEFINE_enum( + name="data_format", short_name="df", default=None, + enum_values=["channels_first", "channels_last"], + help=help_wrap( + "A flag to override the data format used in the model. " + "channels_first provides a performance boost on GPU but is not " + "always compatible with CPU. If left unspecified, the data format " + "will be chosen automatically based on whether TensorFlow was " + "built for CPU or GPU.")) + key_flags.append("data_format") + + return key_flags diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_performance.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_performance.py new file mode 100644 index 0000000..cc5840f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/_performance.py @@ -0,0 +1,289 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Register flags for optimizing performance.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import multiprocessing + +from absl import flags # pylint: disable=g-bad-import-order +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.utils.flags._conventions import help_wrap + + +# Map string to TensorFlow dtype +DTYPE_MAP = { + "fp16": tf.float16, + "bf16": tf.bfloat16, + "fp32": tf.float32, +} + + +def get_tf_dtype(flags_obj): + if getattr(flags_obj, "fp16_implementation", None) == "graph_rewrite": + # If the graph_rewrite is used, we build the graph with fp32, and let the + # graph rewrite change ops to fp16. + return tf.float32 + return DTYPE_MAP[flags_obj.dtype] + + +def get_loss_scale(flags_obj, default_for_fp16): + dtype = get_tf_dtype(flags_obj) + if flags_obj.loss_scale == "dynamic": + return flags_obj.loss_scale + elif flags_obj.loss_scale is not None: + return float(flags_obj.loss_scale) + elif dtype == tf.float32 or dtype == tf.bfloat16: + return 1 # No loss scaling is needed for fp32 + else: + assert dtype == tf.float16 + return default_for_fp16 + + +def define_performance(num_parallel_calls=False, inter_op=False, intra_op=False, + synthetic_data=False, max_train_steps=False, dtype=False, + all_reduce_alg=False, num_packs=False, + tf_gpu_thread_mode=False, + datasets_num_private_threads=False, + datasets_num_parallel_batches=False, + dynamic_loss_scale=False, fp16_implementation=False, + loss_scale=False, + tf_data_experimental_slack=False, enable_xla=False, + training_dataset_cache=False): + """Register flags for specifying performance tuning arguments. + + Args: + num_parallel_calls: Create a flag to specify parallelism of data loading. + inter_op: Create a flag to allow specification of inter op threads. + intra_op: Create a flag to allow specification of intra op threads. + synthetic_data: Create a flag to allow the use of synthetic data. + max_train_steps: Create a flags to allow specification of maximum number + of training steps + dtype: Create flags for specifying dtype. + all_reduce_alg: If set forces a specific algorithm for multi-gpu. + num_packs: If set provides number of packs for MirroredStrategy's cross + device ops. + tf_gpu_thread_mode: gpu_private triggers us of private thread pool. + datasets_num_private_threads: Number of private threads for datasets. + datasets_num_parallel_batches: Determines how many batches to process in + parallel when using map and batch from tf.data. + dynamic_loss_scale: Allow the "loss_scale" flag to take on the value + "dynamic". Only valid if `dtype` is True. + fp16_implementation: Create fp16_implementation flag. + loss_scale: Controls the loss scaling, normally for mixed-precision + training. Can only be turned on if dtype is also True. + tf_data_experimental_slack: Determines whether to enable tf.data's + `experimental_slack` option. + enable_xla: Determines if XLA (auto clustering) is turned on. + training_dataset_cache: Whether to cache the training dataset on workers. + Typically used to improve training performance when training data is in + remote storage and can fit into worker memory. + + Returns: + A list of flags for core.py to marks as key flags. + """ + + key_flags = [] + if num_parallel_calls: + flags.DEFINE_integer( + name="num_parallel_calls", short_name="npc", + default=multiprocessing.cpu_count(), + help=help_wrap("The number of records that are processed in parallel " + "during input processing. This can be optimized per " + "data set but for generally homogeneous data sets, " + "should be approximately the number of available CPU " + "cores. (default behavior)")) + + if inter_op: + flags.DEFINE_integer( + name="inter_op_parallelism_threads", short_name="inter", default=0, + help=help_wrap("Number of inter_op_parallelism_threads to use for CPU. " + "See TensorFlow config.proto for details.") + ) + + if intra_op: + flags.DEFINE_integer( + name="intra_op_parallelism_threads", short_name="intra", default=0, + help=help_wrap("Number of intra_op_parallelism_threads to use for CPU. " + "See TensorFlow config.proto for details.")) + + if synthetic_data: + flags.DEFINE_bool( + name="use_synthetic_data", short_name="synth", default=False, + help=help_wrap( + "If set, use fake data (zeroes) instead of a real dataset. " + "This mode is useful for performance debugging, as it removes " + "input processing steps, but will not learn anything.")) + + if max_train_steps: + flags.DEFINE_integer( + name="max_train_steps", short_name="mts", default=None, help=help_wrap( + "The model will stop training if the global_step reaches this " + "value. If not set, training will run until the specified number " + "of epochs have run as usual. It is generally recommended to set " + "--train_epochs=1 when using this flag." + )) + + if dtype: + flags.DEFINE_enum( + name="dtype", short_name="dt", default="fp32", + enum_values=DTYPE_MAP.keys(), + help=help_wrap("The TensorFlow datatype used for calculations. " + "Variables may be cast to a higher precision on a " + "case-by-case basis for numerical stability.")) + + loss_scale_help_text = ( + "The amount to scale the loss by when the model is run. {}. Before " + "gradients are computed, the loss is multiplied by the loss scale, " + "making all gradients loss_scale times larger. To adjust for this, " + "gradients are divided by the loss scale before being applied to " + "variables. This is mathematically equivalent to training without " + "a loss scale, but the loss scale helps avoid some intermediate " + "gradients from underflowing to zero. If not provided the default " + "for fp16 is 128 and 1 for all other dtypes.{}" + ) + if dynamic_loss_scale: + loss_scale_help_text = loss_scale_help_text.format( + "This can be an int/float or the string 'dynamic'", + " The string 'dynamic' can be used to dynamically determine the " + "optimal loss scale during training, but currently this " + "significantly slows down performance") + loss_scale_validation_msg = ("loss_scale should be a positive int/float " + "or the string 'dynamic'.") + else: + loss_scale_help_text = loss_scale_help_text.format( + "This must be an int/float", "") + loss_scale_validation_msg = "loss_scale should be a positive int/float." + if loss_scale: + flags.DEFINE_string( + name="loss_scale", short_name="ls", default=None, + help=help_wrap(loss_scale_help_text)) + + @flags.validator(flag_name="loss_scale", + message=loss_scale_validation_msg) + def _check_loss_scale(loss_scale): # pylint: disable=unused-variable + """Validator to check the loss scale flag is valid.""" + if loss_scale is None: + return True # null case is handled in get_loss_scale() + + if loss_scale == "dynamic" and dynamic_loss_scale: + return True + + try: + loss_scale = float(loss_scale) + except ValueError: + return False + + return loss_scale > 0 + + if fp16_implementation: + flags.DEFINE_enum( + name="fp16_implementation", default="keras", + enum_values=("keras', 'graph_rewrite"), + help=help_wrap( + "When --dtype=fp16, how fp16 should be implemented. This has no " + "impact on correctness. 'keras' uses the " + "tf.keras.mixed_precision API. 'graph_rewrite' uses the " + "tf.train.experimental.enable_mixed_precision_graph_rewrite " + "API.")) + + @flags.multi_flags_validator(["fp16_implementation", "dtype", + "loss_scale"]) + def _check_fp16_implementation(flags_dict): + """Validator to check fp16_implementation flag is valid.""" + if (flags_dict["fp16_implementation"] == "graph_rewrite" and + flags_dict["dtype"] != "fp16"): + raise flags.ValidationError("--fp16_implementation should not be " + "specified unless --dtype=fp16") + return True + + if all_reduce_alg: + flags.DEFINE_string( + name="all_reduce_alg", short_name="ara", default=None, + help=help_wrap("Defines the algorithm to use for performing all-reduce." + "When specified with MirroredStrategy for single " + "worker, this controls " + "tf.contrib.distribute.AllReduceCrossTowerOps. When " + "specified with MultiWorkerMirroredStrategy, this " + "controls " + "tf.distribute.experimental.CollectiveCommunication; " + "valid options are `ring` and `nccl`.")) + + if num_packs: + flags.DEFINE_integer( + name="num_packs", default=1, + help=help_wrap("Sets `num_packs` in the cross device ops used in " + "MirroredStrategy. For details, see " + "tf.distribute.NcclAllReduce.")) + + if tf_gpu_thread_mode: + flags.DEFINE_string( + name="tf_gpu_thread_mode", short_name="gt_mode", default=None, + help=help_wrap( + "Whether and how the GPU device uses its own threadpool.") + ) + + flags.DEFINE_integer( + name="per_gpu_thread_count", short_name="pgtc", default=0, + help=help_wrap( + "The number of threads to use for GPU. Only valid when " + "tf_gpu_thread_mode is not global.") + ) + + if datasets_num_private_threads: + flags.DEFINE_integer( + name="datasets_num_private_threads", + default=None, + help=help_wrap( + "Number of threads for a private threadpool created for all" + "datasets computation..") + ) + + if datasets_num_parallel_batches: + flags.DEFINE_integer( + name="datasets_num_parallel_batches", + default=None, + help=help_wrap( + "Determines how many batches to process in parallel when using " + "map and batch from tf.data.") + ) + + if training_dataset_cache: + flags.DEFINE_boolean( + name="training_dataset_cache", + default=False, + help=help_wrap( + "Determines whether to cache the training dataset on workers. " + "Typically used to improve training performance when training " + "data is in remote storage and can fit into worker memory.") + ) + + if tf_data_experimental_slack: + flags.DEFINE_boolean( + name="tf_data_experimental_slack", + default=False, + help=help_wrap( + "Whether to enable tf.data's `experimental_slack` option.") + ) + + if enable_xla: + flags.DEFINE_boolean( + name="enable_xla", default=False, + help="Whether to enable XLA auto jit compilation") + + return key_flags diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/core.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/core.py new file mode 100644 index 0000000..fa36944 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/core.py @@ -0,0 +1,133 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Public interface for flag definition. + +See _example.py for detailed instructions on defining flags. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys +from six.moves import shlex_quote + +from absl import app as absl_app +from absl import flags + +from official.utils.flags import _base +from official.utils.flags import _benchmark +from official.utils.flags import _conventions +from official.utils.flags import _device +from official.utils.flags import _distribution +from official.utils.flags import _misc +from official.utils.flags import _performance + + +def set_defaults(**kwargs): + for key, value in kwargs.items(): + flags.FLAGS.set_default(name=key, value=value) + + +def parse_flags(argv=None): + """Reset flags and reparse. Currently only used in testing.""" + flags.FLAGS.unparse_flags() + absl_app.parse_flags_with_usage(argv or sys.argv) + + +def register_key_flags_in_core(f): + """Defines a function in core.py, and registers its key flags. + + absl uses the location of a flags.declare_key_flag() to determine the context + in which a flag is key. By making all declares in core, this allows model + main functions to call flags.adopt_module_key_flags() on core and correctly + chain key flags. + + Args: + f: The function to be wrapped + + Returns: + The "core-defined" version of the input function. + """ + + def core_fn(*args, **kwargs): + key_flags = f(*args, **kwargs) + [flags.declare_key_flag(fl) for fl in key_flags] # pylint: disable=expression-not-assigned + return core_fn + + +define_base = register_key_flags_in_core(_base.define_base) +# We have define_base_eager for compatibility, since it used to be a separate +# function from define_base. +define_base_eager = define_base +define_log_steps = register_key_flags_in_core(_benchmark.define_log_steps) +define_benchmark = register_key_flags_in_core(_benchmark.define_benchmark) +define_device = register_key_flags_in_core(_device.define_device) +define_image = register_key_flags_in_core(_misc.define_image) +define_performance = register_key_flags_in_core(_performance.define_performance) +define_distribution = register_key_flags_in_core( + _distribution.define_distribution) + + +help_wrap = _conventions.help_wrap + + +get_num_gpus = _base.get_num_gpus +get_tf_dtype = _performance.get_tf_dtype +get_loss_scale = _performance.get_loss_scale +DTYPE_MAP = _performance.DTYPE_MAP +require_cloud_storage = _device.require_cloud_storage + +def _get_nondefault_flags_as_dict(): + """Returns the nondefault flags as a dict from flag name to value.""" + nondefault_flags = {} + for flag_name in flags.FLAGS: + flag_value = getattr(flags.FLAGS, flag_name) + if (flag_name != flags.FLAGS[flag_name].short_name and + flag_value != flags.FLAGS[flag_name].default): + nondefault_flags[flag_name] = flag_value + return nondefault_flags + + +def get_nondefault_flags_as_str(): + """Returns flags as a string that can be passed as command line arguments. + + E.g., returns: "--batch_size=256 --use_synthetic_data" for the following code + block: + + ``` + flags.FLAGS.batch_size = 256 + flags.FLAGS.use_synthetic_data = True + print(get_nondefault_flags_as_str()) + ``` + + Only flags with nondefault values are returned, as passing default flags as + command line arguments has no effect. + + Returns: + A string with the flags, that can be passed as command line arguments to a + program to use the flags. + """ + nondefault_flags = _get_nondefault_flags_as_dict() + flag_strings = [] + for name, value in sorted(nondefault_flags.items()): + if isinstance(value, bool): + flag_str = '--{}'.format(name) if value else '--no{}'.format(name) + elif isinstance(value, list): + flag_str = '--{}={}'.format(name, ','.join(value)) + else: + flag_str = '--{}={}'.format(name, value) + flag_strings.append(flag_str) + return ' '.join(shlex_quote(flag_str) for flag_str in flag_strings) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/flags_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/flags_test.py new file mode 100644 index 0000000..e11a164 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/flags_test.py @@ -0,0 +1,162 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import unittest + +from absl import flags +import tensorflow as tf + +from official.utils.flags import core as flags_core # pylint: disable=g-bad-import-order + + +def define_flags(): + flags_core.define_base(clean=True, num_gpu=False, stop_threshold=True, + hooks=True, train_epochs=True, + epochs_between_evals=True) + flags_core.define_performance( + num_parallel_calls=True, inter_op=True, intra_op=True, + dynamic_loss_scale=True, loss_scale=True, synthetic_data=True, + dtype=True) + flags_core.define_image() + flags_core.define_benchmark() + + +class BaseTester(unittest.TestCase): + + @classmethod + def setUpClass(cls): + super(BaseTester, cls).setUpClass() + define_flags() + + def test_default_setting(self): + """Test to ensure fields exist and defaults can be set. + """ + + defaults = dict( + data_dir="dfgasf", + model_dir="dfsdkjgbs", + train_epochs=534, + epochs_between_evals=15, + batch_size=256, + hooks=["LoggingTensorHook"], + num_parallel_calls=18, + inter_op_parallelism_threads=5, + intra_op_parallelism_threads=10, + data_format="channels_first" + ) + + flags_core.set_defaults(**defaults) + flags_core.parse_flags() + + for key, value in defaults.items(): + assert flags.FLAGS.get_flag_value(name=key, default=None) == value + + def test_benchmark_setting(self): + defaults = dict( + hooks=["LoggingMetricHook"], + benchmark_log_dir="/tmp/12345", + gcp_project="project_abc", + ) + + flags_core.set_defaults(**defaults) + flags_core.parse_flags() + + for key, value in defaults.items(): + assert flags.FLAGS.get_flag_value(name=key, default=None) == value + + def test_booleans(self): + """Test to ensure boolean flags trigger as expected. + """ + + flags_core.parse_flags([__file__, "--use_synthetic_data"]) + + assert flags.FLAGS.use_synthetic_data + + def test_parse_dtype_info(self): + flags_core.parse_flags([__file__, "--dtype", "fp16"]) + self.assertEqual(flags_core.get_tf_dtype(flags.FLAGS), tf.float16) + self.assertEqual(flags_core.get_loss_scale(flags.FLAGS, + default_for_fp16=2), 2) + + flags_core.parse_flags( + [__file__, "--dtype", "fp16", "--loss_scale", "5"]) + self.assertEqual(flags_core.get_loss_scale(flags.FLAGS, + default_for_fp16=2), 5) + + flags_core.parse_flags( + [__file__, "--dtype", "fp16", "--loss_scale", "dynamic"]) + self.assertEqual(flags_core.get_loss_scale(flags.FLAGS, + default_for_fp16=2), "dynamic") + + flags_core.parse_flags([__file__, "--dtype", "fp32"]) + self.assertEqual(flags_core.get_tf_dtype(flags.FLAGS), tf.float32) + self.assertEqual(flags_core.get_loss_scale(flags.FLAGS, + default_for_fp16=2), 1) + + flags_core.parse_flags([__file__, "--dtype", "fp32", "--loss_scale", "5"]) + self.assertEqual(flags_core.get_loss_scale(flags.FLAGS, + default_for_fp16=2), 5) + + + with self.assertRaises(SystemExit): + flags_core.parse_flags([__file__, "--dtype", "int8"]) + + with self.assertRaises(SystemExit): + flags_core.parse_flags([__file__, "--dtype", "fp16", + "--loss_scale", "abc"]) + + def test_get_nondefault_flags_as_str(self): + defaults = dict( + clean=True, + data_dir="abc", + hooks=["LoggingTensorHook"], + stop_threshold=1.5, + use_synthetic_data=False + ) + flags_core.set_defaults(**defaults) + flags_core.parse_flags() + + expected_flags = "" + self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags) + + flags.FLAGS.clean = False + expected_flags += "--noclean" + self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags) + + flags.FLAGS.data_dir = "xyz" + expected_flags += " --data_dir=xyz" + self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags) + + flags.FLAGS.hooks = ["aaa", "bbb", "ccc"] + expected_flags += " --hooks=aaa,bbb,ccc" + self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags) + + flags.FLAGS.stop_threshold = 3. + expected_flags += " --stop_threshold=3.0" + self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags) + + flags.FLAGS.use_synthetic_data = True + expected_flags += " --use_synthetic_data" + self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags) + + # Assert that explicit setting a flag to its default value does not cause it + # to appear in the string + flags.FLAGS.use_synthetic_data = False + expected_flags = expected_flags[:-len(" --use_synthetic_data")] + self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags) + + +if __name__ == "__main__": + unittest.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/guidelines.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/guidelines.md new file mode 100644 index 0000000..db963aa --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/flags/guidelines.md @@ -0,0 +1,65 @@ +# Using flags in official models + +1. **All common flags must be incorporated in the models.** + + Common flags (i.e. batch_size, model_dir, etc.) are provided by various flag definition functions, + and channeled through `official.utils.flags.core`. For instance to define common supervised + learning parameters one could use the following code: + + ```$xslt + from absl import app as absl_app + from absl import flags + + from official.utils.flags import core as flags_core + + + def define_flags(): + flags_core.define_base() + flags.adopt_key_flags(flags_core) + + + def main(_): + flags_obj = flags.FLAGS + print(flags_obj) + + + if __name__ == "__main__" + absl_app.run(main) + ``` +2. **Validate flag values.** + + See the [Validators](#validators) section for implementation details. + + Validators in the official model repo should not access the file system, such as verifying + that files exist, due to the strict ordering requirements. + +3. **Flag values should not be mutated.** + + Instead of mutating flag values, use getter functions to return the desired values. An example + getter function is `get_tf_dtype` function below: + + ``` + # Map string to TensorFlow dtype + DTYPE_MAP = { + "fp16": tf.float16, + "fp32": tf.float32, + } + + def get_tf_dtype(flags_obj): + if getattr(flags_obj, "fp16_implementation", None) == "graph_rewrite": + # If the graph_rewrite is used, we build the graph with fp32, and let the + # graph rewrite change ops to fp16. + return tf.float32 + return DTYPE_MAP[flags_obj.dtype] + + + def main(_): + flags_obj = flags.FLAGS() + + # Do not mutate flags_obj + # if flags_obj.fp16_implementation == "graph_rewrite": + # flags_obj.dtype = "float32" # Don't do this + + print(get_tf_dtype(flags_obj)) + ... + ``` \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/hyperparams_flags.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/hyperparams_flags.py new file mode 100644 index 0000000..3f51b08 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/hyperparams_flags.py @@ -0,0 +1,120 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Common flags for importing hyperparameters.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +from absl import flags +from official.utils.flags import core as flags_core + +FLAGS = flags.FLAGS + + +def define_common_hparams_flags(): + """Define the common flags across models.""" + + flags.DEFINE_string( + 'model_dir', + default=None, + help=('The directory where the model and training/evaluation summaries' + 'are stored.')) + + flags.DEFINE_integer( + 'train_batch_size', default=None, help='Batch size for training.') + + flags.DEFINE_integer( + 'eval_batch_size', default=None, help='Batch size for evaluation.') + + flags.DEFINE_string( + 'precision', + default=None, + help=('Precision to use; one of: {bfloat16, float32}')) + + flags.DEFINE_string( + 'config_file', + default=None, + help=('A YAML file which specifies overrides. Note that this file can be ' + 'used as an override template to override the default parameters ' + 'specified in Python. If the same parameter is specified in both ' + '`--config_file` and `--params_override`, the one in ' + '`--params_override` will be used finally.')) + + flags.DEFINE_string( + 'params_override', + default=None, + help=('a YAML/JSON string or a YAML file which specifies additional ' + 'overrides over the default parameters and those specified in ' + '`--config_file`. Note that this is supposed to be used only to ' + 'override the model parameters, but not the parameters like TPU ' + 'specific flags. One canonical use case of `--config_file` and ' + '`--params_override` is users first define a template config file ' + 'using `--config_file`, then use `--params_override` to adjust the ' + 'minimal set of tuning parameters, for example setting up different' + ' `train_batch_size`. ' + 'The final override order of parameters: default_model_params --> ' + 'params from config_file --> params in params_override.' + 'See also the help message of `--config_file`.')) + flags.DEFINE_integer('save_checkpoint_freq', None, + 'Number of steps to save checkpoint.') + + +def initialize_common_flags(): + """Define the common flags across models.""" + define_common_hparams_flags() + + flags_core.define_device(tpu=True) + flags_core.define_base( + num_gpu=True, model_dir=False, data_dir=False, batch_size=False) + flags_core.define_distribution(worker_hosts=True, task_index=True) + flags_core.define_performance(all_reduce_alg=True, num_packs=True) + + # Reset the default value of num_gpus to zero. + FLAGS.num_gpus = 0 + + flags.DEFINE_string( + 'strategy_type', 'mirrored', 'Type of distribute strategy.' + 'One of mirrored, tpu and multiworker.') + + +def strategy_flags_dict(): + """Returns TPU and/or GPU related flags in a dictionary.""" + return { + 'distribution_strategy': FLAGS.strategy_type, + # TPUStrategy related flags. + 'tpu': FLAGS.tpu, + # MultiWorkerMirroredStrategy related flags. + 'all_reduce_alg': FLAGS.all_reduce_alg, + 'worker_hosts': FLAGS.worker_hosts, + 'task_index': FLAGS.task_index, + # MirroredStrategy and OneDeviceStrategy + 'num_gpus': FLAGS.num_gpus, + 'num_packs': FLAGS.num_packs, + } + + +def hparam_flags_dict(): + """Returns model params related flags in a dictionary.""" + return { + 'data_dir': FLAGS.data_dir, + 'model_dir': FLAGS.model_dir, + 'train_batch_size': FLAGS.train_batch_size, + 'eval_batch_size': FLAGS.eval_batch_size, + 'precision': FLAGS.precision, + 'config_file': FLAGS.config_file, + 'params_override': FLAGS.params_override, + } diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/cloud_lib.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/cloud_lib.py new file mode 100644 index 0000000..a2d9bd3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/cloud_lib.py @@ -0,0 +1,34 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Utilities that interact with cloud service. +""" + +import requests + +GCP_METADATA_URL = "http://metadata/computeMetadata/v1/instance/hostname" +GCP_METADATA_HEADER = {"Metadata-Flavor": "Google"} + + +def on_gcp(): + """Detect whether the current running environment is on GCP.""" + try: + # Timeout in 5 seconds, in case the test environment has connectivity issue. + # There is not default timeout, which means it might block forever. + response = requests.get( + GCP_METADATA_URL, headers=GCP_METADATA_HEADER, timeout=5) + return response.status_code == 200 + except requests.exceptions.RequestException: + return False diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/cloud_lib_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/cloud_lib_test.py new file mode 100644 index 0000000..901576d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/cloud_lib_test.py @@ -0,0 +1,48 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Tests for cloud_lib.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest + +import mock +import requests + +from official.utils.logs import cloud_lib + + +class CloudLibTest(unittest.TestCase): + + @mock.patch("requests.get") + def test_on_gcp(self, mock_requests_get): + mock_response = mock.MagicMock() + mock_requests_get.return_value = mock_response + mock_response.status_code = 200 + + self.assertEqual(cloud_lib.on_gcp(), True) + + @mock.patch("requests.get") + def test_not_on_gcp(self, mock_requests_get): + mock_requests_get.side_effect = requests.exceptions.ConnectionError() + + self.assertEqual(cloud_lib.on_gcp(), False) + + +if __name__ == "__main__": + unittest.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/guidelines.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/guidelines.md new file mode 100644 index 0000000..408c3cd --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/guidelines.md @@ -0,0 +1,58 @@ +# Logging in official models + +This library adds logging functions that print or save tensor values. Official models should define all common hooks +(using hooks helper) and a benchmark logger. + +1. **Training Hooks** + + Hooks are a TensorFlow concept that define specific actions at certain points of the execution. We use them to obtain and log + tensor values during training. + + hooks_helper.py provides an easy way to create common hooks. The following hooks are currently defined: + * LoggingTensorHook: Logs tensor values + * ProfilerHook: Writes a timeline json that can be loaded into chrome://tracing. + * ExamplesPerSecondHook: Logs the number of examples processed per second. + * LoggingMetricHook: Similar to LoggingTensorHook, except that the tensors are logged in a format defined by our data + anaylsis pipeline. + + +2. **Benchmarks** + + The benchmark logger provides useful functions for logging environment information, and evaluation results. + The module also contains a context which is used to update the status of the run. + +Example usage: + +``` +from absl import app as absl_app + +from official.utils.logs import hooks_helper +from official.utils.logs import logger + +def model_main(flags_obj): + estimator = ... + + benchmark_logger = logger.get_benchmark_logger() + benchmark_logger.log_run_info(...) + + train_hooks = hooks_helper.get_train_hooks(...) + + for epoch in range(10): + estimator.train(..., hooks=train_hooks) + eval_results = estimator.evaluate(...) + + # Log a dictionary of metrics + benchmark_logger.log_evaluation_result(eval_results) + + # Log an individual metric + benchmark_logger.log_metric(...) + + +def main(_): + with logger.benchmark_context(flags.FLAGS): + model_main(flags.FLAGS) + +if __name__ == "__main__": + # define flags + absl_app.run(main) +``` diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks.py new file mode 100644 index 0000000..e0a0a37 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks.py @@ -0,0 +1,148 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Hook that counts examples per second every N steps or seconds.""" + + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from hccl.manage.api import get_rank_size + +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.utils.logs import logger +from benchmark_log import hwlog +import datetime +import time +import os +import sys + + +class ExamplesPerSecondHook(tf.estimator.SessionRunHook): + """Hook to print out examples per second. + + Total time is tracked and then divided by the total number of steps + to get the average step time and then batch_size is used to determine + the running average of examples per second. The examples per second for the + most recent interval is also logged. + """ + + def __init__(self, + batch_size, + every_n_steps=None, + every_n_secs=None, + warm_steps=0, + metric_logger=None): + """Initializer for ExamplesPerSecondHook. + + Args: + batch_size: Total batch size across all workers used to calculate + examples/second from global time. + every_n_steps: Log stats every n steps. + every_n_secs: Log stats every n seconds. Exactly one of the + `every_n_steps` or `every_n_secs` should be set. + warm_steps: The number of steps to be skipped before logging and running + average calculation. warm_steps steps refers to global steps across all + workers, not on each worker + metric_logger: instance of `BenchmarkLogger`, the benchmark logger that + hook should use to write the log. If None, BaseBenchmarkLogger will + be used. + + Raises: + ValueError: if neither `every_n_steps` or `every_n_secs` is set, or + both are set. + """ + if (every_n_steps is None) == (every_n_secs is None): + raise ValueError("exactly one of every_n_steps" + " and every_n_secs should be provided.") + + self._logger = metric_logger or logger.BaseBenchmarkLogger() + + self._timer = tf.estimator.SecondOrStepTimer( + every_steps=every_n_steps, every_secs=every_n_secs) + + self._step_train_time = 0 + self._total_steps = 0 + self._batch_size = batch_size + self._warm_steps = warm_steps + # List of examples per second logged every_n_steps. + self.current_examples_per_sec_list = [] + + def begin(self): + """Called once before using the session to check global step.""" + tf.compat.v1.logging.warning("##########ExamplesPerSecondHook begin") + self._global_step_tensor = tf.compat.v1.train.get_global_step() + if self._global_step_tensor is None: + raise RuntimeError( + "Global step should be created to use StepCounterHook.") + + def before_run(self, run_context): # pylint: disable=unused-argument + """Called before each call to run(). + + Args: + run_context: A SessionRunContext object. + + Returns: + A SessionRunArgs object or None if never triggered. + """ + self.t0 = time.time() + tf.compat.v1.logging.warning("##########ExamplesPerSecondHook before") + return tf.estimator.SessionRunArgs(self._global_step_tensor) + + def after_run(self, run_context, run_values): # pylint: disable=unused-argument + """Called after each call to run(). + + Args: + run_context: A SessionRunContext object. + run_values: A SessionRunValues object. + """ + tf.compat.v1.logging.warning("##########ExamplesPerSecondHook after_run") + global_step = run_values.results + + #if self._timer.should_trigger_for_step( + #global_step) and global_step > self._warm_steps: + elapsed_time, elapsed_steps = self._timer.update_last_triggered_step( + global_step) + batch_time = time.time() - self.t0 + ips = self._batch_size/batch_time #有问题，应该与FPS一个意思，ips还要乘以iterations_per_loop在乘以rank_size: ips=ips*iterations_per_loop*rank_size + if elapsed_time is not None: + self._step_train_time += elapsed_time + self._total_steps += elapsed_steps + + # average examples per second is based on the total (accumulative) + # training steps and training time so far + average_examples_per_sec = self._batch_size * ( + self._total_steps / self._step_train_time) + # current examples per second is based on the elapsed training steps + # and training time per batch + current_examples_per_sec = self._batch_size * get_rank_size() * ( + elapsed_steps / elapsed_time) + # Logs entries to be read from hook during or after run. + self.current_examples_per_sec_list.append(current_examples_per_sec) + self._logger.log_metric( + "average_examples_per_sec", average_examples_per_sec, + global_step=global_step) + + self._logger.log_metric( + "current_examples_per_sec", current_examples_per_sec, + global_step=global_step) + tf.compat.v1.logging.warning( + "steps: %s,elapsed_steps:%d,batch:%d,FPS:%f,ips:%f,batch_time:%f", int(self._total_steps), + int(elapsed_steps),int(self._batch_size),float(current_examples_per_sec),float(ips), + float(batch_time)) + hwlog.remark_print(key=hwlog.FPS, value=float(current_examples_per_sec)) + + diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks_bak.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks_bak.py new file mode 100644 index 0000000..64743b7 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks_bak.py @@ -0,0 +1,130 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Hook that counts examples per second every N steps or seconds.""" + + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.utils.logs import logger + + +class ExamplesPerSecondHook(tf.estimator.SessionRunHook): + """Hook to print out examples per second. + + Total time is tracked and then divided by the total number of steps + to get the average step time and then batch_size is used to determine + the running average of examples per second. The examples per second for the + most recent interval is also logged. + """ + + def __init__(self, + batch_size, + every_n_steps=None, + every_n_secs=None, + warm_steps=0, + metric_logger=None): + """Initializer for ExamplesPerSecondHook. + + Args: + batch_size: Total batch size across all workers used to calculate + examples/second from global time. + every_n_steps: Log stats every n steps. + every_n_secs: Log stats every n seconds. Exactly one of the + `every_n_steps` or `every_n_secs` should be set. + warm_steps: The number of steps to be skipped before logging and running + average calculation. warm_steps steps refers to global steps across all + workers, not on each worker + metric_logger: instance of `BenchmarkLogger`, the benchmark logger that + hook should use to write the log. If None, BaseBenchmarkLogger will + be used. + + Raises: + ValueError: if neither `every_n_steps` or `every_n_secs` is set, or + both are set. + """ + + if (every_n_steps is None) == (every_n_secs is None): + raise ValueError("exactly one of every_n_steps" + " and every_n_secs should be provided.") + + self._logger = metric_logger or logger.BaseBenchmarkLogger() + + self._timer = tf.estimator.SecondOrStepTimer( + every_steps=every_n_steps, every_secs=every_n_secs) + + self._step_train_time = 0 + self._total_steps = 0 + self._batch_size = batch_size + self._warm_steps = warm_steps + # List of examples per second logged every_n_steps. + self.current_examples_per_sec_list = [] + + def begin(self): + """Called once before using the session to check global step.""" + self._global_step_tensor = tf.compat.v1.train.get_global_step() + if self._global_step_tensor is None: + raise RuntimeError( + "Global step should be created to use StepCounterHook.") + + def before_run(self, run_context): # pylint: disable=unused-argument + """Called before each call to run(). + + Args: + run_context: A SessionRunContext object. + + Returns: + A SessionRunArgs object or None if never triggered. + """ + return tf.estimator.SessionRunArgs(self._global_step_tensor) + + def after_run(self, run_context, run_values): # pylint: disable=unused-argument + """Called after each call to run(). + + Args: + run_context: A SessionRunContext object. + run_values: A SessionRunValues object. + """ + global_step = run_values.results + + if self._timer.should_trigger_for_step( + global_step) and global_step > self._warm_steps: + elapsed_time, elapsed_steps = self._timer.update_last_triggered_step( + global_step) + if elapsed_time is not None: + self._step_train_time += elapsed_time + self._total_steps += elapsed_steps + + # average examples per second is based on the total (accumulative) + # training steps and training time so far + average_examples_per_sec = self._batch_size * ( + self._total_steps / self._step_train_time) + # current examples per second is based on the elapsed training steps + # and training time per batch + current_examples_per_sec = self._batch_size * ( + elapsed_steps / elapsed_time) + # Logs entries to be read from hook during or after run. + self.current_examples_per_sec_list.append(current_examples_per_sec) + self._logger.log_metric( + "average_examples_per_sec", average_examples_per_sec, + global_step=global_step) + + self._logger.log_metric( + "current_examples_per_sec", current_examples_per_sec, + global_step=global_step) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks_helper.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks_helper.py new file mode 100644 index 0000000..aa0e100 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks_helper.py @@ -0,0 +1,172 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Hooks helper to return a list of TensorFlow hooks for training by name. + +More hooks can be added to this set. To add a new hook, 1) add the new hook to +the registry in HOOKS, 2) add a corresponding function that parses out necessary +parameters. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.utils.logs import hooks +from official.utils.logs import logger +from official.utils.logs import metric_hook + +_TENSORS_TO_LOG = dict((x, x) for x in ['learning_rate', + 'cross_entropy', + 'train_accuracy']) + + +def get_train_hooks(name_list, use_tpu=False, **kwargs): + """Factory for getting a list of TensorFlow hooks for training by name. + + Args: + name_list: a list of strings to name desired hook classes. Allowed: + LoggingTensorHook, ProfilerHook, ExamplesPerSecondHook, which are defined + as keys in HOOKS + use_tpu: Boolean of whether computation occurs on a TPU. This will disable + hooks altogether. + **kwargs: a dictionary of arguments to the hooks. + + Returns: + list of instantiated hooks, ready to be used in a classifier.train call. + + Raises: + ValueError: if an unrecognized name is passed. + """ + + if not name_list: + return [] + + if use_tpu: + tf.compat.v1.logging.warning('hooks_helper received name_list `{}`, but a ' + 'TPU is specified. No hooks will be used.' + .format(name_list)) + return [] + + train_hooks = [] + for name in name_list: + hook_name = HOOKS.get(name.strip().lower()) + if hook_name is None: + raise ValueError('Unrecognized training hook requested: {}'.format(name)) + else: + train_hooks.append(hook_name(**kwargs)) + + return train_hooks + + +def get_logging_tensor_hook(every_n_iter=100, tensors_to_log=None, **kwargs): # pylint: disable=unused-argument + """Function to get LoggingTensorHook. + + Args: + every_n_iter: `int`, print the values of `tensors` once every N local + steps taken on the current worker. + tensors_to_log: List of tensor names or dictionary mapping labels to tensor + names. If not set, log _TENSORS_TO_LOG by default. + **kwargs: a dictionary of arguments to LoggingTensorHook. + + Returns: + Returns a LoggingTensorHook with a standard set of tensors that will be + printed to stdout. + """ + if tensors_to_log is None: + tensors_to_log = _TENSORS_TO_LOG + + return tf.estimator.LoggingTensorHook( + tensors=tensors_to_log, + every_n_iter=every_n_iter) + + +def get_profiler_hook(model_dir, save_steps=1000, **kwargs): # pylint: disable=unused-argument + """Function to get ProfilerHook. + + Args: + model_dir: The directory to save the profile traces to. + save_steps: `int`, print profile traces every N steps. + **kwargs: a dictionary of arguments to ProfilerHook. + + Returns: + Returns a ProfilerHook that writes out timelines that can be loaded into + profiling tools like chrome://tracing. + """ + return tf.estimator.ProfilerHook(save_steps=save_steps, output_dir=model_dir) + + +def get_examples_per_second_hook(every_n_steps=100, + batch_size=128, + warm_steps=5, + **kwargs): # pylint: disable=unused-argument + """Function to get ExamplesPerSecondHook. + + Args: + every_n_steps: `int`, print current and average examples per second every + N steps. + batch_size: `int`, total batch size used to calculate examples/second from + global time. + warm_steps: skip this number of steps before logging and running average. + **kwargs: a dictionary of arguments to ExamplesPerSecondHook. + + Returns: + Returns a ProfilerHook that writes out timelines that can be loaded into + profiling tools like chrome://tracing. + """ + return hooks.ExamplesPerSecondHook( + batch_size=batch_size, every_n_steps=every_n_steps, + warm_steps=warm_steps, metric_logger=logger.get_benchmark_logger()) + + +def get_logging_metric_hook(tensors_to_log=None, + every_n_secs=60, + **kwargs): # pylint: disable=unused-argument + """Function to get LoggingMetricHook. + + Args: + tensors_to_log: List of tensor names or dictionary mapping labels to tensor + names. If not set, log _TENSORS_TO_LOG by default. + every_n_secs: `int`, the frequency for logging the metric. Default to every + 10 mins. + **kwargs: a dictionary of arguments. + + Returns: + Returns a LoggingMetricHook that saves tensor values in a JSON format. + """ + if tensors_to_log is None: + tensors_to_log = _TENSORS_TO_LOG + return metric_hook.LoggingMetricHook( + tensors=tensors_to_log, + metric_logger=logger.get_benchmark_logger(), + every_n_secs=every_n_secs) + + +def get_step_counter_hook(**kwargs): + """Function to get StepCounterHook.""" + del kwargs + return tf.estimator.StepCounterHook() + + +# A dictionary to map one hook name and its corresponding function +HOOKS = { + 'loggingtensorhook': get_logging_tensor_hook, + 'profilerhook': get_profiler_hook, + 'examplespersecondhook': get_examples_per_second_hook, + 'loggingmetrichook': get_logging_metric_hook, + 'stepcounterhook': get_step_counter_hook +} diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks_helper_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks_helper_test.py new file mode 100644 index 0000000..693311b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks_helper_test.py @@ -0,0 +1,73 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Tests for hooks_helper.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest + +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.utils.logs import hooks_helper +from official.utils.misc import keras_utils + + +class BaseTest(unittest.TestCase): + + def setUp(self): + super(BaseTest, self).setUp() + if keras_utils.is_v2_0: + tf.compat.v1.disable_eager_execution() + + def test_raise_in_non_list_names(self): + with self.assertRaises(ValueError): + hooks_helper.get_train_hooks( + 'LoggingTensorHook, ProfilerHook', model_dir="", batch_size=256) + + def test_raise_in_invalid_names(self): + invalid_names = ['StepCounterHook', 'StopAtStepHook'] + with self.assertRaises(ValueError): + hooks_helper.get_train_hooks(invalid_names, model_dir="", batch_size=256) + + def validate_train_hook_name(self, + test_hook_name, + expected_hook_name, + **kwargs): + returned_hook = hooks_helper.get_train_hooks( + [test_hook_name], model_dir="", **kwargs) + self.assertEqual(len(returned_hook), 1) + self.assertIsInstance(returned_hook[0], tf.estimator.SessionRunHook) + self.assertEqual(returned_hook[0].__class__.__name__.lower(), + expected_hook_name) + + def test_get_train_hooks_logging_tensor_hook(self): + self.validate_train_hook_name('LoggingTensorHook', 'loggingtensorhook') + + def test_get_train_hooks_profiler_hook(self): + self.validate_train_hook_name('ProfilerHook', 'profilerhook') + + def test_get_train_hooks_examples_per_second_hook(self): + self.validate_train_hook_name('ExamplesPerSecondHook', + 'examplespersecondhook') + + def test_get_logging_metric_hook(self): + test_hook_name = 'LoggingMetricHook' + self.validate_train_hook_name(test_hook_name, 'loggingmetrichook') + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks_test.py new file mode 100644 index 0000000..adbc0a2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/hooks_test.py @@ -0,0 +1,159 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Tests for hooks.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import time + +from absl import logging +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.utils.logs import hooks +from official.utils.testing import mock_lib + +logging.set_verbosity(logging.DEBUG) + + +class ExamplesPerSecondHookTest(tf.test.TestCase): + """Tests for the ExamplesPerSecondHook. + + In the test, we explicitly run global_step tensor after train_op in order to + keep the global_step value and the train_op (which increase the glboal_step + by 1) consistent. This is to correct the discrepancies in reported global_step + value when running on GPUs. + """ + + def setUp(self): + """Mock out logging calls to verify if correct info is being monitored.""" + self._logger = mock_lib.MockBenchmarkLogger() + + self.graph = tf.Graph() + with self.graph.as_default(): + tf.compat.v1.train.create_global_step() + self.train_op = tf.compat.v1.assign_add( + tf.compat.v1.train.get_global_step(), 1) + self.global_step = tf.compat.v1.train.get_global_step() + + def test_raise_in_both_secs_and_steps(self): + with self.assertRaises(ValueError): + hooks.ExamplesPerSecondHook( + batch_size=256, + every_n_steps=10, + every_n_secs=20, + metric_logger=self._logger) + + def test_raise_in_none_secs_and_steps(self): + with self.assertRaises(ValueError): + hooks.ExamplesPerSecondHook( + batch_size=256, + every_n_steps=None, + every_n_secs=None, + metric_logger=self._logger) + + def _validate_log_every_n_steps(self, every_n_steps, warm_steps): + hook = hooks.ExamplesPerSecondHook( + batch_size=256, + every_n_steps=every_n_steps, + warm_steps=warm_steps, + metric_logger=self._logger) + + with tf.compat.v1.train.MonitoredSession( + tf.compat.v1.train.ChiefSessionCreator(), [hook]) as mon_sess: + for _ in range(every_n_steps): + # Explicitly run global_step after train_op to get the accurate + # global_step value + mon_sess.run(self.train_op) + mon_sess.run(self.global_step) + # Nothing should be in the list yet + self.assertFalse(self._logger.logged_metric) + + mon_sess.run(self.train_op) + global_step_val = mon_sess.run(self.global_step) + + if global_step_val > warm_steps: + self._assert_metrics() + else: + # Nothing should be in the list yet + self.assertFalse(self._logger.logged_metric) + + # Add additional run to verify proper reset when called multiple times. + prev_log_len = len(self._logger.logged_metric) + mon_sess.run(self.train_op) + global_step_val = mon_sess.run(self.global_step) + + if every_n_steps == 1 and global_step_val > warm_steps: + # Each time, we log two additional metrics. Did exactly 2 get added? + self.assertEqual(len(self._logger.logged_metric), prev_log_len + 2) + else: + # No change in the size of the metric list. + self.assertEqual(len(self._logger.logged_metric), prev_log_len) + + def test_examples_per_sec_every_1_steps(self): + with self.graph.as_default(): + self._validate_log_every_n_steps(1, 0) + + def test_examples_per_sec_every_5_steps(self): + with self.graph.as_default(): + self._validate_log_every_n_steps(5, 0) + + def test_examples_per_sec_every_1_steps_with_warm_steps(self): + with self.graph.as_default(): + self._validate_log_every_n_steps(1, 10) + + def test_examples_per_sec_every_5_steps_with_warm_steps(self): + with self.graph.as_default(): + self._validate_log_every_n_steps(5, 10) + + def _validate_log_every_n_secs(self, every_n_secs): + hook = hooks.ExamplesPerSecondHook( + batch_size=256, + every_n_steps=None, + every_n_secs=every_n_secs, + metric_logger=self._logger) + + with tf.compat.v1.train.MonitoredSession( + tf.compat.v1.train.ChiefSessionCreator(), [hook]) as mon_sess: + # Explicitly run global_step after train_op to get the accurate + # global_step value + mon_sess.run(self.train_op) + mon_sess.run(self.global_step) + # Nothing should be in the list yet + self.assertFalse(self._logger.logged_metric) + time.sleep(every_n_secs) + + mon_sess.run(self.train_op) + mon_sess.run(self.global_step) + self._assert_metrics() + + def test_examples_per_sec_every_1_secs(self): + with self.graph.as_default(): + self._validate_log_every_n_secs(1) + + def test_examples_per_sec_every_5_secs(self): + with self.graph.as_default(): + self._validate_log_every_n_secs(5) + + def _assert_metrics(self): + metrics = self._logger.logged_metric + self.assertEqual(metrics[-2]["name"], "average_examples_per_sec") + self.assertEqual(metrics[-1]["name"], "current_examples_per_sec") + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/logger.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/logger.py new file mode 100644 index 0000000..398aa8a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/logger.py @@ -0,0 +1,423 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Logging utilities for benchmark. + +For collecting local environment metrics like CPU and memory, certain python +packages need be installed. See README for details. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import contextlib +import datetime +import json +import multiprocessing +import numbers +import os +import threading +import uuid + +from six.moves import _thread as thread +from absl import flags +import tensorflow as tf +from tensorflow.python.client import device_lib + +from official.utils.logs import cloud_lib + +METRIC_LOG_FILE_NAME = "metric.log" +BENCHMARK_RUN_LOG_FILE_NAME = "benchmark_run.log" +_DATE_TIME_FORMAT_PATTERN = "%Y-%m-%dT%H:%M:%S.%fZ" +GCP_TEST_ENV = "GCP" +RUN_STATUS_SUCCESS = "success" +RUN_STATUS_FAILURE = "failure" +RUN_STATUS_RUNNING = "running" + + +FLAGS = flags.FLAGS + +# Don't use it directly. Use get_benchmark_logger to access a logger. +_benchmark_logger = None +_logger_lock = threading.Lock() + + +def config_benchmark_logger(flag_obj=None): + """Config the global benchmark logger.""" + _logger_lock.acquire() + try: + global _benchmark_logger + if not flag_obj: + flag_obj = FLAGS + + if (not hasattr(flag_obj, "benchmark_logger_type") or + flag_obj.benchmark_logger_type == "BaseBenchmarkLogger"): + _benchmark_logger = BaseBenchmarkLogger() + elif flag_obj.benchmark_logger_type == "BenchmarkFileLogger": + _benchmark_logger = BenchmarkFileLogger(flag_obj.benchmark_log_dir) + elif flag_obj.benchmark_logger_type == "BenchmarkBigQueryLogger": + from official.benchmark import benchmark_uploader as bu # pylint: disable=g-import-not-at-top + bq_uploader = bu.BigQueryUploader(gcp_project=flag_obj.gcp_project) + _benchmark_logger = BenchmarkBigQueryLogger( + bigquery_uploader=bq_uploader, + bigquery_data_set=flag_obj.bigquery_data_set, + bigquery_run_table=flag_obj.bigquery_run_table, + bigquery_run_status_table=flag_obj.bigquery_run_status_table, + bigquery_metric_table=flag_obj.bigquery_metric_table, + run_id=str(uuid.uuid4())) + else: + raise ValueError("Unrecognized benchmark_logger_type: %s" + % flag_obj.benchmark_logger_type) + + finally: + _logger_lock.release() + return _benchmark_logger + + +def get_benchmark_logger(): + if not _benchmark_logger: + config_benchmark_logger() + return _benchmark_logger + + +@contextlib.contextmanager +def benchmark_context(flag_obj): + """Context of benchmark, which will update status of the run accordingly.""" + benchmark_logger = config_benchmark_logger(flag_obj) + try: + yield + benchmark_logger.on_finish(RUN_STATUS_SUCCESS) + except Exception: # pylint: disable=broad-except + # Catch all the exception, update the run status to be failure, and re-raise + benchmark_logger.on_finish(RUN_STATUS_FAILURE) + raise + + +class BaseBenchmarkLogger(object): + """Class to log the benchmark information to STDOUT.""" + + def log_evaluation_result(self, eval_results): + """Log the evaluation result. + + The evaluate result is a dictionary that contains metrics defined in + model_fn. It also contains a entry for global_step which contains the value + of the global step when evaluation was performed. + + Args: + eval_results: dict, the result of evaluate. + """ + if not isinstance(eval_results, dict): + tf.compat.v1.logging.warning( + "eval_results should be dictionary for logging. Got %s", + type(eval_results)) + return + global_step = eval_results[tf.compat.v1.GraphKeys.GLOBAL_STEP] + for key in sorted(eval_results): + if key != tf.compat.v1.GraphKeys.GLOBAL_STEP: + self.log_metric(key, eval_results[key], global_step=global_step) + + def log_metric(self, name, value, unit=None, global_step=None, extras=None): + """Log the benchmark metric information to local file. + + Currently the logging is done in a synchronized way. This should be updated + to log asynchronously. + + Args: + name: string, the name of the metric to log. + value: number, the value of the metric. The value will not be logged if it + is not a number type. + unit: string, the unit of the metric, E.g "image per second". + global_step: int, the global_step when the metric is logged. + extras: map of string:string, the extra information about the metric. + """ + metric = _process_metric_to_json(name, value, unit, global_step, extras) + if metric: + tf.compat.v1.logging.info("Benchmark metric: %s", metric) + + def log_run_info(self, model_name, dataset_name, run_params, test_id=None): + tf.compat.v1.logging.info( + "Benchmark run: %s", _gather_run_info(model_name, dataset_name, + run_params, test_id)) + + def on_finish(self, status): + pass + + +class BenchmarkFileLogger(BaseBenchmarkLogger): + """Class to log the benchmark information to local disk.""" + + def __init__(self, logging_dir): + super(BenchmarkFileLogger, self).__init__() + self._logging_dir = logging_dir + if not tf.io.gfile.isdir(self._logging_dir): + tf.io.gfile.makedirs(self._logging_dir) + self._metric_file_handler = tf.io.gfile.GFile( + os.path.join(self._logging_dir, METRIC_LOG_FILE_NAME), "a") + + def log_metric(self, name, value, unit=None, global_step=None, extras=None): + """Log the benchmark metric information to local file. + + Currently the logging is done in a synchronized way. This should be updated + to log asynchronously. + + Args: + name: string, the name of the metric to log. + value: number, the value of the metric. The value will not be logged if it + is not a number type. + unit: string, the unit of the metric, E.g "image per second". + global_step: int, the global_step when the metric is logged. + extras: map of string:string, the extra information about the metric. + """ + metric = _process_metric_to_json(name, value, unit, global_step, extras) + if metric: + try: + json.dump(metric, self._metric_file_handler) + self._metric_file_handler.write("\n") + self._metric_file_handler.flush() + except (TypeError, ValueError) as e: + tf.compat.v1.logging.warning( + "Failed to dump metric to log file: name %s, value %s, error %s", + name, value, e) + + def log_run_info(self, model_name, dataset_name, run_params, test_id=None): + """Collect most of the TF runtime information for the local env. + + The schema of the run info follows official/benchmark/datastore/schema. + + Args: + model_name: string, the name of the model. + dataset_name: string, the name of dataset for training and evaluation. + run_params: dict, the dictionary of parameters for the run, it could + include hyperparameters or other params that are important for the run. + test_id: string, the unique name of the test run by the combination of key + parameters, eg batch size, num of GPU. It is hardware independent. + """ + run_info = _gather_run_info(model_name, dataset_name, run_params, test_id) + + with tf.io.gfile.GFile(os.path.join( + self._logging_dir, BENCHMARK_RUN_LOG_FILE_NAME), "w") as f: + try: + json.dump(run_info, f) + f.write("\n") + except (TypeError, ValueError) as e: + tf.compat.v1.logging.warning( + "Failed to dump benchmark run info to log file: %s", e) + + def on_finish(self, status): + self._metric_file_handler.flush() + self._metric_file_handler.close() + + +class BenchmarkBigQueryLogger(BaseBenchmarkLogger): + """Class to log the benchmark information to BigQuery data store.""" + + def __init__(self, + bigquery_uploader, + bigquery_data_set, + bigquery_run_table, + bigquery_run_status_table, + bigquery_metric_table, + run_id): + super(BenchmarkBigQueryLogger, self).__init__() + self._bigquery_uploader = bigquery_uploader + self._bigquery_data_set = bigquery_data_set + self._bigquery_run_table = bigquery_run_table + self._bigquery_run_status_table = bigquery_run_status_table + self._bigquery_metric_table = bigquery_metric_table + self._run_id = run_id + + def log_metric(self, name, value, unit=None, global_step=None, extras=None): + """Log the benchmark metric information to bigquery. + + Args: + name: string, the name of the metric to log. + value: number, the value of the metric. The value will not be logged if it + is not a number type. + unit: string, the unit of the metric, E.g "image per second". + global_step: int, the global_step when the metric is logged. + extras: map of string:string, the extra information about the metric. + """ + metric = _process_metric_to_json(name, value, unit, global_step, extras) + if metric: + # Starting new thread for bigquery upload in case it might take long time + # and impact the benchmark and performance measurement. Starting a new + # thread might have potential performance impact for model that run on + # CPU. + thread.start_new_thread( + self._bigquery_uploader.upload_benchmark_metric_json, + (self._bigquery_data_set, + self._bigquery_metric_table, + self._run_id, + [metric])) + + def log_run_info(self, model_name, dataset_name, run_params, test_id=None): + """Collect most of the TF runtime information for the local env. + + The schema of the run info follows official/benchmark/datastore/schema. + + Args: + model_name: string, the name of the model. + dataset_name: string, the name of dataset for training and evaluation. + run_params: dict, the dictionary of parameters for the run, it could + include hyperparameters or other params that are important for the run. + test_id: string, the unique name of the test run by the combination of key + parameters, eg batch size, num of GPU. It is hardware independent. + """ + run_info = _gather_run_info(model_name, dataset_name, run_params, test_id) + # Starting new thread for bigquery upload in case it might take long time + # and impact the benchmark and performance measurement. Starting a new + # thread might have potential performance impact for model that run on CPU. + thread.start_new_thread( + self._bigquery_uploader.upload_benchmark_run_json, + (self._bigquery_data_set, + self._bigquery_run_table, + self._run_id, + run_info)) + thread.start_new_thread( + self._bigquery_uploader.insert_run_status, + (self._bigquery_data_set, + self._bigquery_run_status_table, + self._run_id, + RUN_STATUS_RUNNING)) + + def on_finish(self, status): + self._bigquery_uploader.update_run_status( + self._bigquery_data_set, + self._bigquery_run_status_table, + self._run_id, + status) + + +def _gather_run_info(model_name, dataset_name, run_params, test_id): + """Collect the benchmark run information for the local environment.""" + run_info = { + "model_name": model_name, + "dataset": {"name": dataset_name}, + "machine_config": {}, + "test_id": test_id, + "run_date": datetime.datetime.utcnow().strftime( + _DATE_TIME_FORMAT_PATTERN)} + _collect_tensorflow_info(run_info) + _collect_tensorflow_environment_variables(run_info) + _collect_run_params(run_info, run_params) + _collect_cpu_info(run_info) + _collect_memory_info(run_info) + _collect_test_environment(run_info) + return run_info + + +def _process_metric_to_json( + name, value, unit=None, global_step=None, extras=None): + """Validate the metric data and generate JSON for insert.""" + if not isinstance(value, numbers.Number): + tf.compat.v1.logging.warning( + "Metric value to log should be a number. Got %s", type(value)) + return None + + extras = _convert_to_json_dict(extras) + return { + "name": name, + "value": float(value), + "unit": unit, + "global_step": global_step, + "timestamp": datetime.datetime.utcnow().strftime( + _DATE_TIME_FORMAT_PATTERN), + "extras": extras} + + +def _collect_tensorflow_info(run_info): + run_info["tensorflow_version"] = { + "version": tf.version.VERSION, "git_hash": tf.version.GIT_VERSION} + + +def _collect_run_params(run_info, run_params): + """Log the parameter information for the benchmark run.""" + def process_param(name, value): + type_check = { + str: {"name": name, "string_value": value}, + int: {"name": name, "long_value": value}, + bool: {"name": name, "bool_value": str(value)}, + float: {"name": name, "float_value": value}, + } + return type_check.get(type(value), + {"name": name, "string_value": str(value)}) + if run_params: + run_info["run_parameters"] = [ + process_param(k, v) for k, v in sorted(run_params.items())] + + +def _collect_tensorflow_environment_variables(run_info): + run_info["tensorflow_environment_variables"] = [ + {"name": k, "value": v} + for k, v in sorted(os.environ.items()) if k.startswith("TF_")] + + +# The following code is mirrored from tensorflow/tools/test/system_info_lib +# which is not exposed for import. +def _collect_cpu_info(run_info): + """Collect the CPU information for the local environment.""" + cpu_info = {} + + cpu_info["num_cores"] = multiprocessing.cpu_count() + + try: + # Note: cpuinfo is not installed in the TensorFlow OSS tree. + # It is installable via pip. + import cpuinfo # pylint: disable=g-import-not-at-top + + info = cpuinfo.get_cpu_info() + cpu_info["cpu_info"] = info["brand"] + cpu_info["mhz_per_cpu"] = info["hz_advertised_raw"][0] / 1.0e6 + + run_info["machine_config"]["cpu_info"] = cpu_info + except ImportError: + tf.compat.v1.logging.warn( + "'cpuinfo' not imported. CPU info will not be logged.") + + +def _collect_memory_info(run_info): + try: + # Note: psutil is not installed in the TensorFlow OSS tree. + # It is installable via pip. + import psutil # pylint: disable=g-import-not-at-top + vmem = psutil.virtual_memory() + run_info["machine_config"]["memory_total"] = vmem.total + run_info["machine_config"]["memory_available"] = vmem.available + except ImportError: + tf.compat.v1.logging.warn( + "'psutil' not imported. Memory info will not be logged.") + + +def _collect_test_environment(run_info): + """Detect the local environment, eg GCE, AWS or DGX, etc.""" + if cloud_lib.on_gcp(): + run_info["test_environment"] = GCP_TEST_ENV + # TODO(scottzhu): Add more testing env detection for other platform + + +def _parse_gpu_model(physical_device_desc): + # Assume all the GPU connected are same model + for kv in physical_device_desc.split(","): + k, _, v = kv.partition(":") + if k.strip() == "name": + return v.strip() + return None + + +def _convert_to_json_dict(input_dict): + if input_dict: + return [{"name": k, "value": v} for k, v in sorted(input_dict.items())] + else: + return [] diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/logger_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/logger_test.py new file mode 100644 index 0000000..770ad24 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/logger_test.py @@ -0,0 +1,366 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Tests for benchmark logger.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import os +import tempfile +import time +import unittest + +import mock +from absl.testing import flagsaver +import tensorflow as tf # pylint: disable=g-bad-import-order +from absl import logging + +try: + from google.cloud import bigquery +except ImportError: + bigquery = None + +from official.utils.misc import keras_utils +from official.utils.flags import core as flags_core +from official.utils.logs import logger + + +class BenchmarkLoggerTest(tf.test.TestCase): + + @classmethod + def setUpClass(cls): # pylint: disable=invalid-name + super(BenchmarkLoggerTest, cls).setUpClass() + flags_core.define_benchmark() + + def test_get_default_benchmark_logger(self): + with flagsaver.flagsaver(benchmark_logger_type="foo"): + self.assertIsInstance(logger.get_benchmark_logger(), + logger.BaseBenchmarkLogger) + + def test_config_base_benchmark_logger(self): + with flagsaver.flagsaver(benchmark_logger_type="BaseBenchmarkLogger"): + logger.config_benchmark_logger() + self.assertIsInstance(logger.get_benchmark_logger(), + logger.BaseBenchmarkLogger) + + def test_config_benchmark_file_logger(self): + # Set the benchmark_log_dir first since the benchmark_logger_type will need + # the value to be set when it does the validation. + with flagsaver.flagsaver(benchmark_log_dir="/tmp"): + with flagsaver.flagsaver(benchmark_logger_type="BenchmarkFileLogger"): + logger.config_benchmark_logger() + self.assertIsInstance(logger.get_benchmark_logger(), + logger.BenchmarkFileLogger) + + @unittest.skipIf(bigquery is None, "Bigquery dependency is not installed.") + @mock.patch.object(bigquery, "Client") + def test_config_benchmark_bigquery_logger(self, mock_bigquery_client): + with flagsaver.flagsaver(benchmark_logger_type="BenchmarkBigQueryLogger"): + logger.config_benchmark_logger() + self.assertIsInstance(logger.get_benchmark_logger(), + logger.BenchmarkBigQueryLogger) + + @mock.patch("official.utils.logs.logger.config_benchmark_logger") + def test_benchmark_context(self, mock_config_benchmark_logger): + mock_logger = mock.MagicMock() + mock_config_benchmark_logger.return_value = mock_logger + with logger.benchmark_context(None): + logging.info("start benchmarking") + mock_logger.on_finish.assert_called_once_with(logger.RUN_STATUS_SUCCESS) + + @mock.patch("official.utils.logs.logger.config_benchmark_logger") + def test_benchmark_context_failure(self, mock_config_benchmark_logger): + mock_logger = mock.MagicMock() + mock_config_benchmark_logger.return_value = mock_logger + with self.assertRaises(RuntimeError): + with logger.benchmark_context(None): + raise RuntimeError("training error") + mock_logger.on_finish.assert_called_once_with(logger.RUN_STATUS_FAILURE) + + +class BaseBenchmarkLoggerTest(tf.test.TestCase): + + def setUp(self): + super(BaseBenchmarkLoggerTest, self).setUp() + self._actual_log = logging.info + self.logged_message = None + + def mock_log(*args, **kwargs): + self.logged_message = args + self._actual_log(*args, **kwargs) + + logging.info = mock_log + + def tearDown(self): + super(BaseBenchmarkLoggerTest, self).tearDown() + logging.info = self._actual_log + + def test_log_metric(self): + log = logger.BaseBenchmarkLogger() + log.log_metric("accuracy", 0.999, global_step=1e4, extras={"name": "value"}) + + expected_log_prefix = "Benchmark metric:" + self.assertRegexpMatches(str(self.logged_message), expected_log_prefix) + + +class BenchmarkFileLoggerTest(tf.test.TestCase): + + def setUp(self): + super(BenchmarkFileLoggerTest, self).setUp() + # Avoid pulling extra env vars from test environment which affects the test + # result, eg. Kokoro test has a TF_PKG env which affect the test case + # test_collect_tensorflow_environment_variables() + self.original_environ = dict(os.environ) + os.environ.clear() + + def tearDown(self): + super(BenchmarkFileLoggerTest, self).tearDown() + tf.io.gfile.rmtree(self.get_temp_dir()) + os.environ.clear() + os.environ.update(self.original_environ) + + def test_create_logging_dir(self): + non_exist_temp_dir = os.path.join(self.get_temp_dir(), "unknown_dir") + self.assertFalse(tf.io.gfile.isdir(non_exist_temp_dir)) + + logger.BenchmarkFileLogger(non_exist_temp_dir) + self.assertTrue(tf.io.gfile.isdir(non_exist_temp_dir)) + + def test_log_metric(self): + log_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + log = logger.BenchmarkFileLogger(log_dir) + log.log_metric("accuracy", 0.999, global_step=1e4, extras={"name": "value"}) + + metric_log = os.path.join(log_dir, "metric.log") + self.assertTrue(tf.io.gfile.exists(metric_log)) + with tf.io.gfile.GFile(metric_log) as f: + metric = json.loads(f.readline()) + self.assertEqual(metric["name"], "accuracy") + self.assertEqual(metric["value"], 0.999) + self.assertEqual(metric["unit"], None) + self.assertEqual(metric["global_step"], 1e4) + self.assertEqual(metric["extras"], [{"name": "name", "value": "value"}]) + + def test_log_multiple_metrics(self): + log_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + log = logger.BenchmarkFileLogger(log_dir) + log.log_metric("accuracy", 0.999, global_step=1e4, extras={"name": "value"}) + log.log_metric("loss", 0.02, global_step=1e4) + + metric_log = os.path.join(log_dir, "metric.log") + self.assertTrue(tf.io.gfile.exists(metric_log)) + with tf.io.gfile.GFile(metric_log) as f: + accuracy = json.loads(f.readline()) + self.assertEqual(accuracy["name"], "accuracy") + self.assertEqual(accuracy["value"], 0.999) + self.assertEqual(accuracy["unit"], None) + self.assertEqual(accuracy["global_step"], 1e4) + self.assertEqual(accuracy["extras"], [{"name": "name", "value": "value"}]) + + loss = json.loads(f.readline()) + self.assertEqual(loss["name"], "loss") + self.assertEqual(loss["value"], 0.02) + self.assertEqual(loss["unit"], None) + self.assertEqual(loss["global_step"], 1e4) + self.assertEqual(loss["extras"], []) + + def test_log_non_number_value(self): + log_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + log = logger.BenchmarkFileLogger(log_dir) + const = tf.constant(1) + log.log_metric("accuracy", const) + + metric_log = os.path.join(log_dir, "metric.log") + self.assertFalse(tf.io.gfile.exists(metric_log)) + + def test_log_evaluation_result(self): + eval_result = {"loss": 0.46237424, + "global_step": 207082, + "accuracy": 0.9285} + log_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + log = logger.BenchmarkFileLogger(log_dir) + log.log_evaluation_result(eval_result) + + metric_log = os.path.join(log_dir, "metric.log") + self.assertTrue(tf.io.gfile.exists(metric_log)) + with tf.io.gfile.GFile(metric_log) as f: + accuracy = json.loads(f.readline()) + self.assertEqual(accuracy["name"], "accuracy") + self.assertEqual(accuracy["value"], 0.9285) + self.assertEqual(accuracy["unit"], None) + self.assertEqual(accuracy["global_step"], 207082) + + loss = json.loads(f.readline()) + self.assertEqual(loss["name"], "loss") + self.assertEqual(loss["value"], 0.46237424) + self.assertEqual(loss["unit"], None) + self.assertEqual(loss["global_step"], 207082) + + def test_log_evaluation_result_with_invalid_type(self): + eval_result = "{'loss': 0.46237424, 'global_step': 207082}" + log_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + log = logger.BenchmarkFileLogger(log_dir) + log.log_evaluation_result(eval_result) + + metric_log = os.path.join(log_dir, "metric.log") + self.assertFalse(tf.io.gfile.exists(metric_log)) + + @mock.patch("official.utils.logs.logger._gather_run_info") + def test_log_run_info(self, mock_gather_run_info): + log_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + log = logger.BenchmarkFileLogger(log_dir) + run_info = {"model_name": "model_name", + "dataset": "dataset_name", + "run_info": "run_value"} + mock_gather_run_info.return_value = run_info + log.log_run_info("model_name", "dataset_name", {}) + + run_log = os.path.join(log_dir, "benchmark_run.log") + self.assertTrue(tf.io.gfile.exists(run_log)) + with tf.io.gfile.GFile(run_log) as f: + run_info = json.loads(f.readline()) + self.assertEqual(run_info["model_name"], "model_name") + self.assertEqual(run_info["dataset"], "dataset_name") + self.assertEqual(run_info["run_info"], "run_value") + + def test_collect_tensorflow_info(self): + run_info = {} + logger._collect_tensorflow_info(run_info) + self.assertNotEqual(run_info["tensorflow_version"], {}) + self.assertEqual(run_info["tensorflow_version"]["version"], + tf.version.VERSION) + self.assertEqual(run_info["tensorflow_version"]["git_hash"], + tf.version.GIT_VERSION) + + def test_collect_run_params(self): + run_info = {} + run_parameters = { + "batch_size": 32, + "synthetic_data": True, + "train_epochs": 100.00, + "dtype": "fp16", + "resnet_size": 50, + "random_tensor": tf.constant(2.0) + } + logger._collect_run_params(run_info, run_parameters) + self.assertEqual(len(run_info["run_parameters"]), 6) + self.assertEqual(run_info["run_parameters"][0], + {"name": "batch_size", "long_value": 32}) + self.assertEqual(run_info["run_parameters"][1], + {"name": "dtype", "string_value": "fp16"}) + v1_tensor = {"name": "random_tensor", "string_value": + "Tensor(\"Const:0\", shape=(), dtype=float32)"} + v2_tensor = {"name": "random_tensor", "string_value": + "tf.Tensor(2.0, shape=(), dtype=float32)"} + self.assertIn(run_info["run_parameters"][2], [v1_tensor, v2_tensor]) + + + self.assertEqual(run_info["run_parameters"][3], + {"name": "resnet_size", "long_value": 50}) + self.assertEqual(run_info["run_parameters"][4], + {"name": "synthetic_data", "bool_value": "True"}) + self.assertEqual(run_info["run_parameters"][5], + {"name": "train_epochs", "float_value": 100.00}) + + def test_collect_tensorflow_environment_variables(self): + os.environ["TF_ENABLE_WINOGRAD_NONFUSED"] = "1" + os.environ["TF_OTHER"] = "2" + os.environ["OTHER"] = "3" + + run_info = {} + logger._collect_tensorflow_environment_variables(run_info) + self.assertIsNotNone(run_info["tensorflow_environment_variables"]) + expected_tf_envs = [ + {"name": "TF_ENABLE_WINOGRAD_NONFUSED", "value": "1"}, + {"name": "TF_OTHER", "value": "2"}, + ] + self.assertEqual(run_info["tensorflow_environment_variables"], + expected_tf_envs) + + def test_collect_memory_info(self): + run_info = {"machine_config": {}} + logger._collect_memory_info(run_info) + self.assertIsNotNone(run_info["machine_config"]["memory_total"]) + self.assertIsNotNone(run_info["machine_config"]["memory_available"]) + + +@unittest.skipIf(bigquery is None, "Bigquery dependency is not installed.") +class BenchmarkBigQueryLoggerTest(tf.test.TestCase): + + def setUp(self): + super(BenchmarkBigQueryLoggerTest, self).setUp() + # Avoid pulling extra env vars from test environment which affects the test + # result, eg. Kokoro test has a TF_PKG env which affect the test case + # test_collect_tensorflow_environment_variables() + self.original_environ = dict(os.environ) + os.environ.clear() + + self.mock_bq_uploader = mock.MagicMock() + self.logger = logger.BenchmarkBigQueryLogger( + self.mock_bq_uploader, "dataset", "run_table", "run_status_table", + "metric_table", "run_id") + + def tearDown(self): + super(BenchmarkBigQueryLoggerTest, self).tearDown() + tf.io.gfile.rmtree(self.get_temp_dir()) + os.environ.clear() + os.environ.update(self.original_environ) + + def test_log_metric(self): + self.logger.log_metric( + "accuracy", 0.999, global_step=1e4, extras={"name": "value"}) + expected_metric_json = [{ + "name": "accuracy", + "value": 0.999, + "unit": None, + "global_step": 1e4, + "timestamp": mock.ANY, + "extras": [{"name": "name", "value": "value"}] + }] + # log_metric will call upload_benchmark_metric_json in a separate thread. + # Give it some grace period for the new thread before assert. + time.sleep(1) + self.mock_bq_uploader.upload_benchmark_metric_json.assert_called_once_with( + "dataset", "metric_table", "run_id", expected_metric_json) + + @mock.patch("official.utils.logs.logger._gather_run_info") + def test_log_run_info(self, mock_gather_run_info): + run_info = {"model_name": "model_name", + "dataset": "dataset_name", + "run_info": "run_value"} + mock_gather_run_info.return_value = run_info + self.logger.log_run_info("model_name", "dataset_name", {}) + # log_metric will call upload_benchmark_metric_json in a separate thread. + # Give it some grace period for the new thread before assert. + time.sleep(1) + self.mock_bq_uploader.upload_benchmark_run_json.assert_called_once_with( + "dataset", "run_table", "run_id", run_info) + self.mock_bq_uploader.insert_run_status.assert_called_once_with( + "dataset", "run_status_table", "run_id", "running") + + def test_on_finish(self): + self.logger.on_finish(logger.RUN_STATUS_SUCCESS) + # log_metric will call upload_benchmark_metric_json in a separate thread. + # Give it some grace period for the new thread before assert. + time.sleep(1) + self.mock_bq_uploader.update_run_status.assert_called_once_with( + "dataset", "run_status_table", "run_id", logger.RUN_STATUS_SUCCESS) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/metric_hook.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/metric_hook.py new file mode 100644 index 0000000..f408e3e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/metric_hook.py @@ -0,0 +1,97 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Session hook for logging benchmark metric.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf # pylint: disable=g-bad-import-order + + +class LoggingMetricHook(tf.estimator.LoggingTensorHook): + """Hook to log benchmark metric information. + + This hook is very similar as tf.train.LoggingTensorHook, which logs given + tensors every N local steps, every N seconds, or at the end. The metric + information will be logged to given log_dir or via metric_logger in JSON + format, which can be consumed by data analysis pipeline later. + + Note that if `at_end` is True, `tensors` should not include any tensor + whose evaluation produces a side effect such as consuming additional inputs. + """ + + def __init__(self, tensors, metric_logger=None, + every_n_iter=None, every_n_secs=None, at_end=False): + """Initializer for LoggingMetricHook. + + Args: + tensors: `dict` that maps string-valued tags to tensors/tensor names, + or `iterable` of tensors/tensor names. + metric_logger: instance of `BenchmarkLogger`, the benchmark logger that + hook should use to write the log. + every_n_iter: `int`, print the values of `tensors` once every N local + steps taken on the current worker. + every_n_secs: `int` or `float`, print the values of `tensors` once every N + seconds. Exactly one of `every_n_iter` and `every_n_secs` should be + provided. + at_end: `bool` specifying whether to print the values of `tensors` at the + end of the run. + + Raises: + ValueError: + 1. `every_n_iter` is non-positive, or + 2. Exactly one of every_n_iter and every_n_secs should be provided. + 3. Exactly one of log_dir and metric_logger should be provided. + """ + super(LoggingMetricHook, self).__init__( + tensors=tensors, + every_n_iter=every_n_iter, + every_n_secs=every_n_secs, + at_end=at_end) + + if metric_logger is None: + raise ValueError("metric_logger should be provided.") + self._logger = metric_logger + + def begin(self): + super(LoggingMetricHook, self).begin() + self._global_step_tensor = tf.compat.v1.train.get_global_step() + if self._global_step_tensor is None: + raise RuntimeError( + "Global step should be created to use LoggingMetricHook.") + if self._global_step_tensor.name not in self._current_tensors: + self._current_tensors[self._global_step_tensor.name] = ( + self._global_step_tensor) + + def after_run(self, unused_run_context, run_values): + # should_trigger is a internal state that populated at before_run, and it is + # using self_timer to determine whether it should trigger. + if self._should_trigger: + self._log_metric(run_values.results) + + self._iter_count += 1 + + def end(self, session): + if self._log_at_end: + values = session.run(self._current_tensors) + self._log_metric(values) + + def _log_metric(self, tensor_values): + self._timer.update_last_triggered_step(self._iter_count) + global_step = tensor_values[self._global_step_tensor.name] + # self._tag_order is populated during the init of LoggingTensorHook + for tag in self._tag_order: + self._logger.log_metric(tag, tensor_values[tag], global_step=global_step) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/metric_hook_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/metric_hook_test.py new file mode 100644 index 0000000..870ed6e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/metric_hook_test.py @@ -0,0 +1,217 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for metric_hook.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tempfile +import time + +import tensorflow as tf # pylint: disable=g-bad-import-order +from tensorflow.python.training import monitored_session # pylint: disable=g-bad-import-order + +from official.utils.logs import metric_hook +from official.utils.testing import mock_lib + + +class LoggingMetricHookTest(tf.test.TestCase): + """Tests for LoggingMetricHook.""" + + def setUp(self): + super(LoggingMetricHookTest, self).setUp() + + self._log_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + self._logger = mock_lib.MockBenchmarkLogger() + + def tearDown(self): + super(LoggingMetricHookTest, self).tearDown() + tf.io.gfile.rmtree(self.get_temp_dir()) + + def test_illegal_args(self): + with self.assertRaisesRegexp(ValueError, "nvalid every_n_iter"): + metric_hook.LoggingMetricHook(tensors=["t"], every_n_iter=0) + with self.assertRaisesRegexp(ValueError, "nvalid every_n_iter"): + metric_hook.LoggingMetricHook(tensors=["t"], every_n_iter=-10) + with self.assertRaisesRegexp(ValueError, "xactly one of"): + metric_hook.LoggingMetricHook( + tensors=["t"], every_n_iter=5, every_n_secs=5) + with self.assertRaisesRegexp(ValueError, "xactly one of"): + metric_hook.LoggingMetricHook(tensors=["t"]) + with self.assertRaisesRegexp(ValueError, "metric_logger"): + metric_hook.LoggingMetricHook(tensors=["t"], every_n_iter=5) + + def test_print_at_end_only(self): + with tf.Graph().as_default(), tf.compat.v1.Session() as sess: + tf.compat.v1.train.get_or_create_global_step() + t = tf.constant(42.0, name="foo") + train_op = tf.constant(3) + hook = metric_hook.LoggingMetricHook( + tensors=[t.name], at_end=True, metric_logger=self._logger) + hook.begin() + mon_sess = monitored_session._HookedSession(sess, [hook]) # pylint: disable=protected-access + sess.run(tf.compat.v1.global_variables_initializer()) + + for _ in range(3): + mon_sess.run(train_op) + self.assertEqual(self._logger.logged_metric, []) + + hook.end(sess) + self.assertEqual(len(self._logger.logged_metric), 1) + metric = self._logger.logged_metric[0] + self.assertRegexpMatches(metric["name"], "foo") + self.assertEqual(metric["value"], 42.0) + self.assertEqual(metric["unit"], None) + self.assertEqual(metric["global_step"], 0) + + def test_global_step_not_found(self): + with tf.Graph().as_default(): + t = tf.constant(42.0, name="foo") + hook = metric_hook.LoggingMetricHook( + tensors=[t.name], at_end=True, metric_logger=self._logger) + + with self.assertRaisesRegexp( + RuntimeError, "should be created to use LoggingMetricHook."): + hook.begin() + + def test_log_tensors(self): + with tf.Graph().as_default(), tf.compat.v1.Session() as sess: + tf.compat.v1.train.get_or_create_global_step() + t1 = tf.constant(42.0, name="foo") + t2 = tf.constant(43.0, name="bar") + train_op = tf.constant(3) + hook = metric_hook.LoggingMetricHook( + tensors=[t1, t2], at_end=True, metric_logger=self._logger) + hook.begin() + mon_sess = monitored_session._HookedSession(sess, [hook]) # pylint: disable=protected-access + sess.run(tf.compat.v1.global_variables_initializer()) + + for _ in range(3): + mon_sess.run(train_op) + self.assertEqual(self._logger.logged_metric, []) + + hook.end(sess) + self.assertEqual(len(self._logger.logged_metric), 2) + metric1 = self._logger.logged_metric[0] + self.assertRegexpMatches(str(metric1["name"]), "foo") + self.assertEqual(metric1["value"], 42.0) + self.assertEqual(metric1["unit"], None) + self.assertEqual(metric1["global_step"], 0) + + metric2 = self._logger.logged_metric[1] + self.assertRegexpMatches(str(metric2["name"]), "bar") + self.assertEqual(metric2["value"], 43.0) + self.assertEqual(metric2["unit"], None) + self.assertEqual(metric2["global_step"], 0) + + def _validate_print_every_n_steps(self, sess, at_end): + t = tf.constant(42.0, name="foo") + + train_op = tf.constant(3) + hook = metric_hook.LoggingMetricHook( + tensors=[t.name], every_n_iter=10, at_end=at_end, + metric_logger=self._logger) + hook.begin() + mon_sess = monitored_session._HookedSession(sess, [hook]) # pylint: disable=protected-access + sess.run(tf.compat.v1.global_variables_initializer()) + mon_sess.run(train_op) + self.assertRegexpMatches(str(self._logger.logged_metric), t.name) + for _ in range(3): + self._logger.logged_metric = [] + for _ in range(9): + mon_sess.run(train_op) + # assertNotRegexpMatches is not supported by python 3.1 and later + self.assertEqual(str(self._logger.logged_metric).find(t.name), -1) + mon_sess.run(train_op) + self.assertRegexpMatches(str(self._logger.logged_metric), t.name) + + # Add additional run to verify proper reset when called multiple times. + self._logger.logged_metric = [] + mon_sess.run(train_op) + # assertNotRegexpMatches is not supported by python 3.1 and later + self.assertEqual(str(self._logger.logged_metric).find(t.name), -1) + + self._logger.logged_metric = [] + hook.end(sess) + if at_end: + self.assertRegexpMatches(str(self._logger.logged_metric), t.name) + else: + # assertNotRegexpMatches is not supported by python 3.1 and later + self.assertEqual(str(self._logger.logged_metric).find(t.name), -1) + + def test_print_every_n_steps(self): + with tf.Graph().as_default(), tf.compat.v1.Session() as sess: + tf.compat.v1.train.get_or_create_global_step() + self._validate_print_every_n_steps(sess, at_end=False) + # Verify proper reset. + self._validate_print_every_n_steps(sess, at_end=False) + + def test_print_every_n_steps_and_end(self): + with tf.Graph().as_default(), tf.compat.v1.Session() as sess: + tf.compat.v1.train.get_or_create_global_step() + self._validate_print_every_n_steps(sess, at_end=True) + # Verify proper reset. + self._validate_print_every_n_steps(sess, at_end=True) + + def _validate_print_every_n_secs(self, sess, at_end): + t = tf.constant(42.0, name="foo") + train_op = tf.constant(3) + + hook = metric_hook.LoggingMetricHook( + tensors=[t.name], every_n_secs=1.0, at_end=at_end, + metric_logger=self._logger) + hook.begin() + mon_sess = monitored_session._HookedSession(sess, [hook]) # pylint: disable=protected-access + sess.run(tf.compat.v1.global_variables_initializer()) + + mon_sess.run(train_op) + self.assertRegexpMatches(str(self._logger.logged_metric), t.name) + + # assertNotRegexpMatches is not supported by python 3.1 and later + self._logger.logged_metric = [] + mon_sess.run(train_op) + self.assertEqual(str(self._logger.logged_metric).find(t.name), -1) + time.sleep(1.0) + + self._logger.logged_metric = [] + mon_sess.run(train_op) + self.assertRegexpMatches(str(self._logger.logged_metric), t.name) + + self._logger.logged_metric = [] + hook.end(sess) + if at_end: + self.assertRegexpMatches(str(self._logger.logged_metric), t.name) + else: + # assertNotRegexpMatches is not supported by python 3.1 and later + self.assertEqual(str(self._logger.logged_metric).find(t.name), -1) + + def test_print_every_n_secs(self): + with tf.Graph().as_default(), tf.compat.v1.Session() as sess: + tf.compat.v1.train.get_or_create_global_step() + self._validate_print_every_n_secs(sess, at_end=False) + # Verify proper reset. + self._validate_print_every_n_secs(sess, at_end=False) + + def test_print_every_n_secs_and_end(self): + with tf.Graph().as_default(), tf.compat.v1.Session() as sess: + tf.compat.v1.train.get_or_create_global_step() + self._validate_print_every_n_secs(sess, at_end=True) + # Verify proper reset. + self._validate_print_every_n_secs(sess, at_end=True) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/mlperf_helper.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/mlperf_helper.py new file mode 100644 index 0000000..f0b0374 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/logs/mlperf_helper.py @@ -0,0 +1,194 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Wrapper for the mlperf logging utils. + +MLPerf compliance logging is only desired under a limited set of circumstances. +This module is intended to keep users from needing to consider logging (or +install the module) unless they are performing mlperf runs. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from collections import namedtuple +import json +import os +import re +import subprocess +import sys +import typing + + + +import tensorflow as tf + +_MIN_VERSION = (0, 0, 10) +_STACK_OFFSET = 2 + +SUDO = "sudo" if os.geteuid() else "" + +# This indirection is used in docker. +DROP_CACHE_LOC = os.getenv("DROP_CACHE_LOC", "/proc/sys/vm/drop_caches") + +_NCF_PREFIX = "NCF_RAW_" + +# TODO(robieta): move line parsing to mlperf util +_PREFIX = r"(?:{})?:::MLPv([0-9]+).([0-9]+).([0-9]+)".format(_NCF_PREFIX) +_BENCHMARK = r"([a-zA-Z0-9_]+)" +_TIMESTAMP = r"([0-9]+\.[0-9]+)" +_CALLSITE = r"\((.+):([0-9]+)\)" +_TAG = r"([a-zA-Z0-9_]+)" +_VALUE = r"(.*)" + +ParsedLine = namedtuple("ParsedLine", ["version", "benchmark", "timestamp", + "callsite", "tag", "value"]) + +LINE_PATTERN = re.compile( + "^{prefix} {benchmark} {timestamp} {callsite} {tag}(: |$){value}?$".format( + prefix=_PREFIX, benchmark=_BENCHMARK, timestamp=_TIMESTAMP, + callsite=_CALLSITE, tag=_TAG, value=_VALUE)) + + +def parse_line(line): # type: (str) -> typing.Optional[ParsedLine] + match = LINE_PATTERN.match(line.strip()) + if not match: + return + + major, minor, micro, benchmark, timestamp = match.groups()[:5] + call_file, call_line, tag, _, value = match.groups()[5:] + + return ParsedLine(version=(int(major), int(minor), int(micro)), + benchmark=benchmark, timestamp=timestamp, + callsite=(call_file, call_line), tag=tag, value=value) + + +def unparse_line(parsed_line): # type: (ParsedLine) -> str + version_str = "{}.{}.{}".format(*parsed_line.version) + callsite_str = "({}:{})".format(*parsed_line.callsite) + value_str = ": {}".format(parsed_line.value) if parsed_line.value else "" + return ":::MLPv{} {} {} {} {} {}".format( + version_str, parsed_line.benchmark, parsed_line.timestamp, callsite_str, + parsed_line.tag, value_str) + + +def get_mlperf_log(): + """Shielded import of mlperf_log module.""" + try: + import mlperf_compliance + + def test_mlperf_log_pip_version(): + """Check that mlperf_compliance is up to date.""" + import pkg_resources + version = pkg_resources.get_distribution("mlperf_compliance") + version = tuple(int(i) for i in version.version.split(".")) + if version < _MIN_VERSION: + tf.compat.v1.logging.warning( + "mlperf_compliance is version {}, must be >= {}".format( + ".".join([str(i) for i in version]), + ".".join([str(i) for i in _MIN_VERSION]))) + raise ImportError + return mlperf_compliance.mlperf_log + + mlperf_log = test_mlperf_log_pip_version() + + except ImportError: + mlperf_log = None + + return mlperf_log + + +class Logger(object): + """MLPerf logger indirection class. + + This logger only logs for MLPerf runs, and prevents various errors associated + with not having the mlperf_compliance package installed. + """ + class Tags(object): + def __init__(self, mlperf_log): + self._enabled = False + self._mlperf_log = mlperf_log + + def __getattr__(self, item): + if self._mlperf_log is None or not self._enabled: + return + return getattr(self._mlperf_log, item) + + def __init__(self): + self._enabled = False + self._mlperf_log = get_mlperf_log() + self.tags = self.Tags(self._mlperf_log) + + def __call__(self, enable=False): + if enable and self._mlperf_log is None: + raise ImportError("MLPerf logging was requested, but mlperf_compliance " + "module could not be loaded.") + + self._enabled = enable + self.tags._enabled = enable + return self + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + self._enabled = False + self.tags._enabled = False + + @property + def log_file(self): + if self._mlperf_log is None: + return + return self._mlperf_log.LOG_FILE + + @property + def enabled(self): + return self._enabled + + def ncf_print(self, key, value=None, stack_offset=_STACK_OFFSET, + deferred=False, extra_print=False, prefix=_NCF_PREFIX): + if self._mlperf_log is None or not self.enabled: + return + self._mlperf_log.ncf_print(key=key, value=value, stack_offset=stack_offset, + deferred=deferred, extra_print=extra_print, + prefix=prefix) + + def set_ncf_root(self, path): + if self._mlperf_log is None: + return + self._mlperf_log.ROOT_DIR_NCF = path + + +LOGGER = Logger() +ncf_print, set_ncf_root = LOGGER.ncf_print, LOGGER.set_ncf_root +TAGS = LOGGER.tags + + +def clear_system_caches(): + if not LOGGER.enabled: + return + ret_code = subprocess.call( + ["sync && echo 3 | {} tee {}".format(SUDO, DROP_CACHE_LOC)], + shell=True) + + if ret_code: + raise ValueError("Failed to clear caches") + + +if __name__ == "__main__": + tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO) + with LOGGER(True): + ncf_print(key=TAGS.RUN_START) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/callstack_sampler.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/callstack_sampler.py new file mode 100644 index 0000000..984f133 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/callstack_sampler.py @@ -0,0 +1,62 @@ +"""A simple Python callstack sampler.""" + +import contextlib +import datetime +import signal +import traceback + + +class CallstackSampler(object): + """A simple signal-based Python callstack sampler. + """ + + def __init__(self, interval=None): + self.stacks = [] + self.interval = 0.001 if interval is None else interval + + def _sample(self, signum, frame): + """Samples the current stack.""" + del signum + stack = traceback.extract_stack(frame) + formatted_stack = [] + formatted_stack.append(datetime.datetime.utcnow()) + for filename, lineno, function_name, text in stack: + formatted_frame = '{}:{}({})({})'.format(filename, lineno, function_name, + text) + formatted_stack.append(formatted_frame) + self.stacks.append(formatted_stack) + signal.setitimer(signal.ITIMER_VIRTUAL, self.interval, 0) + + @contextlib.contextmanager + def profile(self): + signal.signal(signal.SIGVTALRM, self._sample) + signal.setitimer(signal.ITIMER_VIRTUAL, self.interval, 0) + try: + yield + finally: + signal.setitimer(signal.ITIMER_VIRTUAL, 0) + + def save(self, fname): + with open(fname, 'w') as f: + for s in self.stacks: + for l in s: + f.write('%s\n' % l) + f.write('\n') + + +@contextlib.contextmanager +def callstack_sampling(filename, interval=None): + """Periodically samples the Python callstack. + + Args: + filename: the filename + interval: the sampling interval, in seconds. Defaults to 0.001. + + Yields: + nothing + """ + sampler = CallstackSampler(interval=interval) + with sampler.profile(): + yield + sampler.save(filename) + diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/distribution_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/distribution_utils.py new file mode 100644 index 0000000..e4823a9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/distribution_utils.py @@ -0,0 +1,205 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Helper functions for running models in a distributed setting.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import os +import random +import string + +from absl import logging +import tensorflow.compat.v2 as tf + +from official.utils.misc import tpu_lib + + +def _collective_communication(all_reduce_alg): + """Return a CollectiveCommunication based on all_reduce_alg. + + Args: + all_reduce_alg: a string specifying which collective communication to pick, + or None. + + Returns: + tf.distribute.experimental.CollectiveCommunication object + + Raises: + ValueError: if `all_reduce_alg` not in [None, "ring", "nccl"] + """ + collective_communication_options = { + None: tf.distribute.experimental.CollectiveCommunication.AUTO, + "ring": tf.distribute.experimental.CollectiveCommunication.RING, + "nccl": tf.distribute.experimental.CollectiveCommunication.NCCL + } + if all_reduce_alg not in collective_communication_options: + raise ValueError( + "When used with `multi_worker_mirrored`, valid values for " + "all_reduce_alg are [`ring`, `nccl`]. Supplied value: {}".format( + all_reduce_alg)) + return collective_communication_options[all_reduce_alg] + + +def _mirrored_cross_device_ops(all_reduce_alg, num_packs): + """Return a CrossDeviceOps based on all_reduce_alg and num_packs. + + Args: + all_reduce_alg: a string specifying which cross device op to pick, or None. + num_packs: an integer specifying number of packs for the cross device op. + + Returns: + tf.distribute.CrossDeviceOps object or None. + + Raises: + ValueError: if `all_reduce_alg` not in [None, "nccl", "hierarchical_copy"]. + """ + if all_reduce_alg is None: + return None + mirrored_all_reduce_options = { + "nccl": tf.distribute.NcclAllReduce, + "hierarchical_copy": tf.distribute.HierarchicalCopyAllReduce + } + if all_reduce_alg not in mirrored_all_reduce_options: + raise ValueError( + "When used with `mirrored`, valid values for all_reduce_alg are " + "[`nccl`, `hierarchical_copy`]. Supplied value: {}".format( + all_reduce_alg)) + cross_device_ops_class = mirrored_all_reduce_options[all_reduce_alg] + return cross_device_ops_class(num_packs=num_packs) + + +def get_distribution_strategy(distribution_strategy="mirrored", + num_gpus=0, + all_reduce_alg=None, + num_packs=1, + tpu_address=None): + """Return a DistributionStrategy for running the model. + + Args: + distribution_strategy: a string specifying which distribution strategy to + use. Accepted values are "off", "one_device", "mirrored", + "parameter_server", "multi_worker_mirrored", and "tpu" -- case insensitive. + "off" means not to use Distribution Strategy; "tpu" means to use + TPUStrategy using `tpu_address`. + num_gpus: Number of GPUs to run this model. + all_reduce_alg: Optional. Specifies which algorithm to use when performing + all-reduce. For `MirroredStrategy`, valid values are "nccl" and + "hierarchical_copy". For `MultiWorkerMirroredStrategy`, valid values are + "ring" and "nccl". If None, DistributionStrategy will choose based on + device topology. + num_packs: Optional. Sets the `num_packs` in `tf.distribute.NcclAllReduce` + or `tf.distribute.HierarchicalCopyAllReduce` for `MirroredStrategy`. + tpu_address: Optional. String that represents TPU to connect to. Must not + be None if `distribution_strategy` is set to `tpu`. + Returns: + tf.distribute.DistibutionStrategy object. + Raises: + ValueError: if `distribution_strategy` is "off" or "one_device" and + `num_gpus` is larger than 1; or `num_gpus` is negative or if + `distribution_strategy` is `tpu` but `tpu_address` is not specified. + """ + if num_gpus < 0: + raise ValueError("`num_gpus` can not be negative.") + + distribution_strategy = distribution_strategy.lower() + if distribution_strategy == "off": + if num_gpus > 1: + raise ValueError( + "When {} GPUs are specified, distribution_strategy " + "flag cannot be set to `off`.".format(num_gpus)) + return None + + if distribution_strategy == "tpu": + # When tpu_address is an empty string, we communicate with local TPUs. + cluster_resolver = tpu_lib.tpu_initialize(tpu_address) + return tf.distribute.experimental.TPUStrategy(cluster_resolver) + + if distribution_strategy == "multi_worker_mirrored": + return tf.distribute.experimental.MultiWorkerMirroredStrategy( + communication=_collective_communication(all_reduce_alg)) + + if distribution_strategy == "one_device": + if num_gpus == 0: + return tf.distribute.OneDeviceStrategy("device:CPU:0") + if num_gpus > 1: + raise ValueError("`OneDeviceStrategy` can not be used for more than " + "one device.") + return tf.distribute.OneDeviceStrategy("device:GPU:0") + + if distribution_strategy == "mirrored": + if num_gpus == 0: + devices = ["device:CPU:0"] + else: + devices = ["device:GPU:%d" % i for i in range(num_gpus)] + return tf.distribute.MirroredStrategy( + devices=devices, + cross_device_ops=_mirrored_cross_device_ops(all_reduce_alg, num_packs)) + + if distribution_strategy == "parameter_server": + return tf.distribute.experimental.ParameterServerStrategy() + + raise ValueError( + "Unrecognized Distribution Strategy: %r" % distribution_strategy) + + +def configure_cluster(worker_hosts=None, task_index=-1): + """Set multi-worker cluster spec in TF_CONFIG environment variable. + + Args: + worker_hosts: comma-separated list of worker ip:port pairs. + + Returns: + Number of workers in the cluster. + """ + tf_config = json.loads(os.environ.get("TF_CONFIG", "{}")) + if tf_config: + num_workers = (len(tf_config["cluster"].get("chief", [])) + + len(tf_config["cluster"].get("worker", []))) + elif worker_hosts: + workers = worker_hosts.split(",") + num_workers = len(workers) + if num_workers > 1 and task_index < 0: + raise ValueError("Must specify task_index when number of workers > 1") + task_index = 0 if num_workers == 1 else task_index + os.environ["TF_CONFIG"] = json.dumps({ + "cluster": { + "worker": workers + }, + "task": {"type": "worker", "index": task_index} + }) + else: + num_workers = 1 + return num_workers + + +def get_strategy_scope(strategy): + if strategy: + strategy_scope = strategy.scope() + else: + strategy_scope = DummyContextManager() + + return strategy_scope + + +class DummyContextManager(object): + + def __enter__(self): + pass + + def __exit__(self, *args): + pass diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/distribution_utils_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/distribution_utils_test.py new file mode 100644 index 0000000..4fd7bff --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/distribution_utils_test.py @@ -0,0 +1,49 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" Tests for distribution util functions.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow.compat.v2 as tf + +from official.utils.misc import distribution_utils + + +class GetDistributionStrategyTest(tf.test.TestCase): + """Tests for get_distribution_strategy.""" + def test_one_device_strategy_cpu(self): + ds = distribution_utils.get_distribution_strategy(num_gpus=0) + self.assertEquals(ds.num_replicas_in_sync, 1) + self.assertEquals(len(ds.extended.worker_devices), 1) + self.assertIn('CPU', ds.extended.worker_devices[0]) + + def test_one_device_strategy_gpu(self): + ds = distribution_utils.get_distribution_strategy(num_gpus=1) + self.assertEquals(ds.num_replicas_in_sync, 1) + self.assertEquals(len(ds.extended.worker_devices), 1) + self.assertIn('GPU', ds.extended.worker_devices[0]) + + def test_mirrored_strategy(self): + ds = distribution_utils.get_distribution_strategy(num_gpus=5) + self.assertEquals(ds.num_replicas_in_sync, 5) + self.assertEquals(len(ds.extended.worker_devices), 5) + for device in ds.extended.worker_devices: + self.assertIn('GPU', device) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/keras_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/keras_utils.py new file mode 100644 index 0000000..cf18b9e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/keras_utils.py @@ -0,0 +1,262 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Helper functions for the Keras implementations of models.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import multiprocessing +import os +import time + +from absl import logging +import tensorflow.compat.v2 as tf +from tensorflow.python import tf2 +from tensorflow.python.profiler import profiler_v2 as profiler + + +class BatchTimestamp(object): + """A structure to store batch time stamp.""" + + def __init__(self, batch_index, timestamp): + self.batch_index = batch_index + self.timestamp = timestamp + + def __repr__(self): + return "'BatchTimestamp'".format( + self.batch_index, self.timestamp) + + +class TimeHistory(tf.keras.callbacks.Callback): + """Callback for Keras models.""" + + def __init__(self, batch_size, log_steps, logdir=None): + """Callback for logging performance. + + Args: + batch_size: Total batch size. + log_steps: Interval of steps between logging of batch level stats. + logdir: Optional directory to write TensorBoard summaries. + """ + # TODO(wcromar): remove this parameter and rely on `logs` parameter of + # on_train_batch_end() + self.batch_size = batch_size + super(TimeHistory, self).__init__() + self.log_steps = log_steps + self.last_log_step = 0 + self.steps_before_epoch = 0 + self.steps_in_epoch = 0 + self.start_time = None + + if logdir: + self.summary_writer = tf.summary.create_file_writer(logdir) + else: + self.summary_writer = None + + # Logs start of step 1 then end of each step based on log_steps interval. + self.timestamp_log = [] + + # Records the time each epoch takes to run from start to finish of epoch. + self.epoch_runtime_log = [] + + @property + def global_steps(self): + """The current 1-indexed global step.""" + return self.steps_before_epoch + self.steps_in_epoch + + @property + def average_steps_per_second(self): + """The average training steps per second across all epochs.""" + return self.global_steps / sum(self.epoch_runtime_log) + + @property + def average_examples_per_second(self): + """The average number of training examples per second across all epochs.""" + return self.average_steps_per_second * self.batch_size + + def on_train_end(self, logs=None): + self.train_finish_time = time.time() + + if self.summary_writer: + self.summary_writer.flush() + + def on_epoch_begin(self, epoch, logs=None): + self.epoch_start = time.time() + + def on_batch_begin(self, batch, logs=None): + if not self.start_time: + self.start_time = time.time() + + # Record the timestamp of the first global step + if not self.timestamp_log: + self.timestamp_log.append(BatchTimestamp(self.global_steps, + self.start_time)) + + def on_batch_end(self, batch, logs=None): + """Records elapse time of the batch and calculates examples per second.""" + self.steps_in_epoch = batch + 1 + steps_since_last_log = self.global_steps - self.last_log_step + if steps_since_last_log >= self.log_steps: + now = time.time() + elapsed_time = now - self.start_time + steps_per_second = steps_since_last_log / elapsed_time + examples_per_second = steps_per_second * self.batch_size + + self.timestamp_log.append(BatchTimestamp(self.global_steps, now)) + logging.info( + 'TimeHistory: %.2f seconds, %.2f examples/second between steps %d ' + 'and %d', elapsed_time, examples_per_second, self.last_log_step, + self.global_steps) + + if self.summary_writer: + with self.summary_writer.as_default(): + tf.summary.scalar('global_step/sec', steps_per_second, + self.global_steps) + tf.summary.scalar('examples/sec', examples_per_second, + self.global_steps) + + self.last_log_step = self.global_steps + self.start_time = None + + def on_epoch_end(self, epoch, logs=None): + epoch_run_time = time.time() - self.epoch_start + self.epoch_runtime_log.append(epoch_run_time) + + self.steps_before_epoch += self.steps_in_epoch + self.steps_in_epoch = 0 + + +def get_profiler_callback(model_dir, profile_steps, enable_tensorboard, + steps_per_epoch): + """Validate profile_steps flag value and return profiler callback.""" + profile_steps_error_message = ( + 'profile_steps must be a comma separated pair of positive integers, ' + 'specifying the first and last steps to be profiled.' + ) + try: + profile_steps = [int(i) for i in profile_steps.split(',')] + except ValueError: + raise ValueError(profile_steps_error_message) + if len(profile_steps) != 2: + raise ValueError(profile_steps_error_message) + start_step, stop_step = profile_steps + if start_step < 0 or start_step > stop_step: + raise ValueError(profile_steps_error_message) + if enable_tensorboard: + logging.warning( + 'Both TensorBoard and profiler callbacks are used. Note that the ' + 'TensorBoard callback profiles the 2nd step (unless otherwise ' + 'specified). Please make sure the steps profiled by the two callbacks ' + 'do not overlap.') + return ProfilerCallback(model_dir, start_step, stop_step, steps_per_epoch) + + +class ProfilerCallback(tf.keras.callbacks.Callback): + """Save profiles in specified step range to log directory.""" + + def __init__(self, log_dir, start_step, stop_step, steps_per_epoch): + super(ProfilerCallback, self).__init__() + self.log_dir = log_dir + self.start_step = start_step + self.stop_step = stop_step + self.start_epoch = start_step // steps_per_epoch + self.stop_epoch = stop_step // steps_per_epoch + self.start_step_in_epoch = start_step % steps_per_epoch + self.stop_step_in_epoch = stop_step % steps_per_epoch + self.should_start = False + self.should_stop = False + + def on_epoch_begin(self, epoch, logs=None): + if epoch == self.start_epoch: + self.should_start = True + if epoch == self.stop_epoch: + self.should_stop = True + + def on_batch_begin(self, batch, logs=None): + if batch == self.start_step_in_epoch and self.should_start: + self.should_start = False + profiler.start(self.log_dir) + logging.info('Profiler started at Step %s', self.start_step) + + def on_batch_end(self, batch, logs=None): + if batch == self.stop_step_in_epoch and self.should_stop: + self.should_stop = False + profiler.stop() + logging.info('Profiler saved profiles for steps between %s and %s to %s', + self.start_step, self.stop_step, self.log_dir) + + +def set_session_config(enable_eager=False, + enable_xla=False): + """Sets the session config.""" + if is_v2_0(): + set_config_v2(enable_xla=enable_xla) + else: + config = get_config_proto_v1(enable_xla=enable_xla) + if enable_eager: + tf.compat.v1.enable_eager_execution(config=config) + else: + sess = tf.compat.v1.Session(config=config) + tf.compat.v1.keras.backend.set_session(sess) + + +def get_config_proto_v1(enable_xla=False): + """Return config proto according to flag settings, or None to use default.""" + config = None + if enable_xla: + config = tf.compat.v1.ConfigProto() + config.graph_options.optimizer_options.global_jit_level = ( + tf.OptimizerOptions.ON_2) + return config + + +def set_config_v2(enable_xla=False): + """Config eager context according to flag values using TF 2.0 API.""" + if enable_xla: + tf.config.optimizer.set_jit(True) + + +def is_v2_0(): + """Returns true if using tf 2.0.""" + return tf2.enabled() + + +def set_gpu_thread_mode_and_count(gpu_thread_mode, + datasets_num_private_threads, + num_gpus, per_gpu_thread_count): + """Set GPU thread mode and count, and adjust dataset threads count.""" + cpu_count = multiprocessing.cpu_count() + logging.info('Logical CPU cores: %s', cpu_count) + + # Allocate private thread pool for each GPU to schedule and launch kernels + per_gpu_thread_count = per_gpu_thread_count or 2 + os.environ['TF_GPU_THREAD_MODE'] = gpu_thread_mode + os.environ['TF_GPU_THREAD_COUNT'] = str(per_gpu_thread_count) + logging.info('TF_GPU_THREAD_COUNT: %s', + os.environ['TF_GPU_THREAD_COUNT']) + logging.info('TF_GPU_THREAD_MODE: %s', + os.environ['TF_GPU_THREAD_MODE']) + + # Limit data preprocessing threadpool to CPU cores minus number of total GPU + # private threads and memory copy threads. + total_gpu_thread_count = per_gpu_thread_count * num_gpus + num_runtime_threads = num_gpus + if not datasets_num_private_threads: + datasets_num_private_threads = min( + cpu_count - total_gpu_thread_count - num_runtime_threads, + num_gpus * 8) + logging.info('Set datasets_num_private_threads to %s', + datasets_num_private_threads) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/model_helpers.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/model_helpers.py new file mode 100644 index 0000000..9a44e50 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/model_helpers.py @@ -0,0 +1,95 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Miscellaneous functions that can be called by models.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numbers + +from absl import logging +import tensorflow as tf + +from tensorflow.python.util import nest +# pylint:disable=logging-format-interpolation + + +def past_stop_threshold(stop_threshold, eval_metric): + """Return a boolean representing whether a model should be stopped. + + Args: + stop_threshold: float, the threshold above which a model should stop + training. + eval_metric: float, the current value of the relevant metric to check. + + Returns: + True if training should stop, False otherwise. + + Raises: + ValueError: if either stop_threshold or eval_metric is not a number + """ + if stop_threshold is None: + return False + + if not isinstance(stop_threshold, numbers.Number): + raise ValueError("Threshold for checking stop conditions must be a number.") + if not isinstance(eval_metric, numbers.Number): + raise ValueError("Eval metric being checked against stop conditions " + "must be a number.") + + if eval_metric >= stop_threshold: + logging.info("Stop threshold of {} was passed with metric value {}.".format( + stop_threshold, eval_metric)) + return True + + return False + + +def generate_synthetic_data( + input_shape, input_value=0, input_dtype=None, label_shape=None, + label_value=0, label_dtype=None): + """Create a repeating dataset with constant values. + + Args: + input_shape: a tf.TensorShape object or nested tf.TensorShapes. The shape of + the input data. + input_value: Value of each input element. + input_dtype: Input dtype. If None, will be inferred by the input value. + label_shape: a tf.TensorShape object or nested tf.TensorShapes. The shape of + the label data. + label_value: Value of each input element. + label_dtype: Input dtype. If None, will be inferred by the target value. + + Returns: + Dataset of tensors or tuples of tensors (if label_shape is set). + """ + # TODO(kathywu): Replace with SyntheticDataset once it is in contrib. + element = input_element = nest.map_structure( + lambda s: tf.constant(input_value, input_dtype, s), input_shape) + + if label_shape: + label_element = nest.map_structure( + lambda s: tf.constant(label_value, label_dtype, s), label_shape) + element = (input_element, label_element) + + return tf.data.Dataset.from_tensors(element).repeat() + + +def apply_clean(flags_obj): + if flags_obj.clean and tf.io.gfile.exists(flags_obj.model_dir): + logging.info("--clean flag set. Removing existing model dir:" + " {}".format(flags_obj.model_dir)) + tf.io.gfile.rmtree(flags_obj.model_dir) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/model_helpers_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/model_helpers_test.py new file mode 100644 index 0000000..f34a594 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/model_helpers_test.py @@ -0,0 +1,127 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Model Helper functions.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.utils.misc import keras_utils +from official.utils.misc import model_helpers + + +class PastStopThresholdTest(tf.test.TestCase): + """Tests for past_stop_threshold.""" + + def setUp(self): + super(PastStopThresholdTest, self).setUp() + if keras_utils.is_v2_0: + tf.compat.v1.disable_eager_execution() + + def test_past_stop_threshold(self): + """Tests for normal operating conditions.""" + self.assertTrue(model_helpers.past_stop_threshold(0.54, 1)) + self.assertTrue(model_helpers.past_stop_threshold(54, 100)) + self.assertFalse(model_helpers.past_stop_threshold(0.54, 0.1)) + self.assertFalse(model_helpers.past_stop_threshold(-0.54, -1.5)) + self.assertTrue(model_helpers.past_stop_threshold(-0.54, 0)) + self.assertTrue(model_helpers.past_stop_threshold(0, 0)) + self.assertTrue(model_helpers.past_stop_threshold(0.54, 0.54)) + + def test_past_stop_threshold_none_false(self): + """Tests that check None returns false.""" + self.assertFalse(model_helpers.past_stop_threshold(None, -1.5)) + self.assertFalse(model_helpers.past_stop_threshold(None, None)) + self.assertFalse(model_helpers.past_stop_threshold(None, 1.5)) + # Zero should be okay, though. + self.assertTrue(model_helpers.past_stop_threshold(0, 1.5)) + + def test_past_stop_threshold_not_number(self): + """Tests for error conditions.""" + with self.assertRaises(ValueError): + model_helpers.past_stop_threshold("str", 1) + + with self.assertRaises(ValueError): + model_helpers.past_stop_threshold("str", tf.constant(5)) + + with self.assertRaises(ValueError): + model_helpers.past_stop_threshold("str", "another") + + with self.assertRaises(ValueError): + model_helpers.past_stop_threshold(0, None) + + with self.assertRaises(ValueError): + model_helpers.past_stop_threshold(0.7, "str") + + with self.assertRaises(ValueError): + model_helpers.past_stop_threshold(tf.constant(4), None) + + +class SyntheticDataTest(tf.test.TestCase): + """Tests for generate_synthetic_data.""" + + def test_generate_synethetic_data(self): + input_element, label_element = tf.compat.v1.data.make_one_shot_iterator( + model_helpers.generate_synthetic_data(input_shape=tf.TensorShape([5]), + input_value=123, + input_dtype=tf.float32, + label_shape=tf.TensorShape([]), + label_value=456, + label_dtype=tf.int32)).get_next() + + with self.session() as sess: + for n in range(5): + inp, lab = sess.run((input_element, label_element)) + self.assertAllClose(inp, [123., 123., 123., 123., 123.]) + self.assertEquals(lab, 456) + + def test_generate_only_input_data(self): + d = model_helpers.generate_synthetic_data( + input_shape=tf.TensorShape([4]), + input_value=43.5, + input_dtype=tf.float32) + + element = tf.compat.v1.data.make_one_shot_iterator(d).get_next() + self.assertFalse(isinstance(element, tuple)) + + with self.session() as sess: + inp = sess.run(element) + self.assertAllClose(inp, [43.5, 43.5, 43.5, 43.5]) + + def test_generate_nested_data(self): + d = model_helpers.generate_synthetic_data( + input_shape={'a': tf.TensorShape([2]), + 'b': {'c': tf.TensorShape([3]), 'd': tf.TensorShape([])}}, + input_value=1.1) + + element = tf.compat.v1.data.make_one_shot_iterator(d).get_next() + self.assertIn('a', element) + self.assertIn('b', element) + self.assertEquals(len(element['b']), 2) + self.assertIn('c', element['b']) + self.assertIn('d', element['b']) + self.assertNotIn('c', element) + + with self.session() as sess: + inp = sess.run(element) + self.assertAllClose(inp['a'], [1.1, 1.1]) + self.assertAllClose(inp['b']['c'], [1.1, 1.1, 1.1]) + self.assertAllClose(inp['b']['d'], 1.1) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/tpu_lib.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/tpu_lib.py new file mode 100644 index 0000000..4d4cddb --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/misc/tpu_lib.py @@ -0,0 +1,34 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Initializes TPU system for TF 2.0.""" + +import tensorflow as tf + + +def tpu_initialize(tpu_address): + """Initializes TPU for TF 2.0 training. + + Args: + tpu_address: string, bns address of master TPU worker. + + Returns: + A TPUClusterResolver. + """ + cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver( + tpu=tpu_address) + if tpu_address not in ('', 'local'): + tf.config.experimental_connect_to_cluster(cluster_resolver) + tf.tpu.experimental.initialize_tpu_system(cluster_resolver) + return cluster_resolver diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/integration.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/integration.py new file mode 100644 index 0000000..b4809a4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/integration.py @@ -0,0 +1,71 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Helper code to run complete models from within python. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import shutil +import sys +import tempfile + +from absl import flags +from absl.testing import flagsaver + +from official.utils.flags import core as flags_core + + +@flagsaver.flagsaver +def run_synthetic(main, tmp_root, extra_flags=None, synth=True, train_epochs=1, + epochs_between_evals=1): + """Performs a minimal run of a model. + + This function is intended to test for syntax errors throughout a model. A + very limited run is performed using synthetic data. + + Args: + main: The primary function used to exercise a code path. Generally this + function is ".main(argv)". + tmp_root: Root path for the temp directory created by the test class. + extra_flags: Additional flags passed by the caller of this function. + synth: Use synthetic data. + train_epochs: Value of the --train_epochs flag. + epochs_between_evals: Value of the --epochs_between_evals flag. + """ + + extra_flags = [] if extra_flags is None else extra_flags + + model_dir = tempfile.mkdtemp(dir=tmp_root) + + args = [sys.argv[0], "--model_dir", model_dir] + extra_flags + + if synth: + args.append("--use_synthetic_data") + + if train_epochs is not None: + args.extend(["--train_epochs", str(train_epochs)]) + + if epochs_between_evals is not None: + args.extend(["--epochs_between_evals", str(epochs_between_evals)]) + + try: + flags_core.parse_flags(argv=args) + main(flags.FLAGS) + finally: + if os.path.exists(model_dir): + shutil.rmtree(model_dir) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/mock_lib.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/mock_lib.py new file mode 100644 index 0000000..ee4de3c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/mock_lib.py @@ -0,0 +1,36 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Mock objects and related functions for testing.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +class MockBenchmarkLogger(object): + """This is a mock logger that can be used in dependent tests.""" + + def __init__(self): + self.logged_metric = [] + + def log_metric(self, name, value, unit=None, global_step=None, + extras=None): + self.logged_metric.append({ + "name": name, + "value": float(value), + "unit": unit, + "global_step": global_step, + "extras": extras}) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/pylint.rcfile b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/pylint.rcfile new file mode 100644 index 0000000..b872802 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/pylint.rcfile @@ -0,0 +1,168 @@ +[MESSAGES CONTROL] +disable=R,W,bad-option-value,trailing-newlines,no-name-in-module + +[REPORTS] +# Tells whether to display a full report or only the messages +reports=no + +# Activate the evaluation score. +score=no + +[BASIC] + +# Regular expression matching correct argument names +argument-rgx=^[a-z][a-z0-9_]*$ + +# Regular expression matching correct attribute names +attr-rgx=^_{0,2}[a-z][a-z0-9_]*$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ + +# Regular expression matching correct class names +class-rgx=^_?[A-Z][a-zA-Z0-9]*$ + +# Regular expression matching correct constant names +const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=10 + +# Regular expression matching correct function names +function-rgx=^(?:(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$ + +# Good variable names which should always be accepted, separated by a comma +good-names=main,_ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=^[a-z][a-z0-9_]*$ + +# Regular expression matching correct method names +method-rgx=^(?:(?P__[a-z0-9_]+__|next)|(?P_{0,2}[A-Z][a-zA-Z0-9]*)|(?P_{0,2}[a-z][a-z0-9_]*)|(setUp|tearDown))$ + +# Regular expression matching correct module names +module-rgx=^(_?[a-z][a-z0-9_]*)|__init__|PRESUBMIT|PRESUBMIT_unittest$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=(__.*__|main|.*ArgParser) + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=^[a-z][a-z0-9_]*$ + +[TYPECHECK] + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules=absl, absl.*, official, official.*, tensorflow, tensorflow.*, LazyLoader, google, google.cloud.* + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + +# This is deprecated, because it is not used anymore. +#ignore-iface-methods= + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls,class_ + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of arguments for function / method +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of statements in function / method body +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=StandardError,Exception,BaseException + + +[FORMAT] + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=80 + +# Maximum number of lines in a module +max-module-lines=99999 + +# List of optional constructs for which whitespace checking is disabled +no-space-check= + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=yes + +# Allow URLs and comment type annotations to exceed the max line length as neither can be easily +# split across lines. +ignore-long-lines=^\s*(?:(# )??$|# type:) + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_) + +# Tells whether we should check for unused import in __init__ files. +init-import=no diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/scripts/builds_common.sh b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/scripts/builds_common.sh new file mode 100644 index 0000000..3cf08bb --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/scripts/builds_common.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# +# Common Bash functions used by build scripts + +COLOR_NC='\033[0m' +COLOR_BOLD='\033[1m' +COLOR_LIGHT_GRAY='\033[0;37m' +COLOR_GREEN='\033[0;32m' +COLOR_RED='\033[0;31m' + +die() { + # Print a message and exit with code 1. + # + # Usage: die + # e.g., die "Something bad happened." + + echo $@ + exit 1 +} + +num_cpus() { + # Get the number of CPUs + N_CPUS=$(grep -c ^processor /proc/cpuinfo) + if [[ -z ${N_CPUS} ]]; then + die "ERROR: Unable to determine the number of CPUs" + fi + + echo ${N_CPUS} +} + +# List files changed (i.e., added, or revised) from +# the common ancestor of HEAD and the latest master branch. +# Usage: get_changed_files_from_master_branch +get_changed_files_from_master_branch() { + ANCESTOR=$(git merge-base HEAD master origin/master) + git diff ${ANCESTOR} --diff-filter=d --name-only "$@" +} + +# List python files changed that still exist, +# i.e., not removed. +# Usage: get_py_files_to_check [--incremental] +get_py_files_to_check() { + if [[ "$1" == "--incremental" ]]; then + get_changed_files_from_master_branch -- '*.py' + elif [[ -z "$1" ]]; then + find official/ -name '*.py' + else + die "Found unsupported args: $@ for get_py_files_to_check." + fi +} diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/scripts/ci_sanity.sh b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/scripts/ci_sanity.sh new file mode 100644 index 0000000..97d6bc2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/scripts/ci_sanity.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# Sanity check script that runs tests and lint under local environment. +# Make sure that tensorflow and pylint is installed. +# usage: models >: ./official/utils/testing/scripts/ci_sanity.sh do_pylint --incremental +set +x + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/builds_common.sh" +cd "$SCRIPT_DIR/../../../.." +MODEL_ROOT="$(pwd)" + +export PYTHONPATH="$PYTHONPATH:${MODEL_ROOT}" + +# Run pylint +do_pylint() { + # Usage: do_pylint [--incremental] + # + # Options: + # --incremental Performs check on only the python files changed in the + # last non-merge git commit. + + # Use this list to whitelist pylint errors + ERROR_WHITELIST="" + + echo "ERROR_WHITELIST=\"${ERROR_WHITELIST}\"" + + PYLINT_BIN="python3 -m pylint" + + PYTHON_SRC_FILES=$(get_py_files_to_check $1) + if [[ -z ${PYTHON_SRC_FILES} ]]; then + echo "do_pylint found no Python files to check. Returning." + return 0 + fi + + PYLINTRC_FILE="official/utils/testing/pylint.rcfile" + + if [[ ! -f "${PYLINTRC_FILE}" ]]; then + die "ERROR: Cannot find pylint rc file at ${PYLINTRC_FILE}" + fi + + NUM_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w) + NUM_CPUS=$(num_cpus) + + echo "Running pylint on ${NUM_SRC_FILES} files with ${NUM_CPUS} "\ + "parallel jobs..." + echo "" + + PYLINT_START_TIME=$(date +'%s') + OUTPUT_FILE="$(mktemp)_pylint_output.log" + ERRORS_FILE="$(mktemp)_pylint_errors.log" + NONWL_ERRORS_FILE="$(mktemp)_pylint_nonwl_errors.log" + + rm -rf ${OUTPUT_FILE} + rm -rf ${ERRORS_FILE} + rm -rf ${NONWL_ERRORS_FILE} + touch ${NONWL_ERRORS_FILE} + + ${PYLINT_BIN} --rcfile="${PYLINTRC_FILE}" --output-format=parseable \ + --jobs=${NUM_CPUS} ${PYTHON_SRC_FILES} > ${OUTPUT_FILE} 2>&1 + PYLINT_END_TIME=$(date +'%s') + + echo "" + echo "pylint took $((PYLINT_END_TIME - PYLINT_START_TIME)) s" + echo "" + + # Report only what we care about + # Ref https://pylint.readthedocs.io/en/latest/technical_reference/features.html + # E: all errors + # W0311 bad-indentation + # W0312 mixed-indentation + # C0330 bad-continuation + # C0301 line-too-long + # C0326 bad-whitespace + # W0611 unused-import + # W0622 redefined-builtin + grep -E '(\[E|\[W0311|\[W0312|\[C0330|\[C0301|\[C0326|\[W0611|\[W0622)' ${OUTPUT_FILE} > ${ERRORS_FILE} + + N_ERRORS=0 + while read -r LINE; do + IS_WHITELISTED=0 + for WL_REGEX in ${ERROR_WHITELIST}; do + if echo ${LINE} | grep -q "${WL_REGEX}"; then + echo "Found a whitelisted error:" + echo " ${LINE}" + IS_WHITELISTED=1 + fi + done + + if [[ ${IS_WHITELISTED} == "0" ]]; then + echo "${LINE}" >> ${NONWL_ERRORS_FILE} + echo "" >> ${NONWL_ERRORS_FILE} + ((N_ERRORS++)) + fi + done <${ERRORS_FILE} + + echo "Raw lint output file: ${OUTPUT_FILE}" + + echo "" + if [[ ${N_ERRORS} != 0 ]]; then + echo "FAIL: Found ${N_ERRORS} non-whitelited pylint errors:" + cat "${NONWL_ERRORS_FILE}" + return 1 + else + echo "PASS: No non-whitelisted pylint errors were found." + return 0 + fi +} + +test_result=0 + +TESTS="$@" + +for t in "${TESTS}"; do + ${t} || test_result=$? +done + +exit "${test_result}" diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/scripts/presubmit.sh b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/scripts/presubmit.sh new file mode 100644 index 0000000..954d96d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/utils/testing/scripts/presubmit.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# Presubmit script that runs tests and lint under local environment. +# Make sure that tensorflow and pylint is installed. +# usage: models >: ./official/utils/testing/scripts/presubmit.sh +# usage: models >: ./official/utils/testing/scripts/presubmit.sh lint py2_test py3_test +set +x + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR/../../../.." +MODEL_ROOT="$(pwd)" + +export PYTHONPATH="$PYTHONPATH:${MODEL_ROOT}" + +py_test() { + local PY_BINARY="$1" + local exit_code=0 + + echo "===========Running Python test============" + + for test_file in `find official/ -name '*test.py' -print` + do + echo "####=======Testing ${test_file}=======####" + ${PY_BINARY} "${test_file}" + _exit_code=$? + if [[ $_exit_code != 0 ]]; then + exit_code=$_exit_code + echo "FAIL: ${test_file}" + fi + done + + return "${exit_code}" +} + +py2_test() { + local PY_BINARY=$(which python2) + py_test "$PY_BINARY" + return $? +} + +py3_test() { + local PY_BINARY=$(which python3) + py_test "$PY_BINARY" + return $? +} + +test_result=0 + +if [ "$#" -eq 0 ]; then + TESTS="lint py2_test py3_test" +else + TESTS="$@" +fi + +for t in "${TESTS}"; do + ${t} || test_result=$? +done + +exit "${test_result}" diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/README.md new file mode 100644 index 0000000..a28caba --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/README.md @@ -0,0 +1,244 @@ +# Object Detection Models on TensorFlow 2 + +**Note**: This repository is still under construction. +More features and instructions will be added soon. + +## Prerequsite +To get started, download the code from TensorFlow models GitHub repository or +use the pre-installed Google Cloud VM. + +```bash +git clone https://github.com/tensorflow/models.git +``` + +Next, make sure to use TensorFlow 2.1+ on Google Cloud. Also here are +a few package you need to install to get started: + +```bash +sudo apt-get install -y python-tk && \ +pip3 install -r ~/models/official/requirements.txt +``` + +## Train RetinaNet on TPU + +### Train a vanilla ResNet-50 based RetinaNet. + +```bash +TPU_NAME="" +MODEL_DIR="" +RESNET_CHECKPOINT="" +TRAIN_FILE_PATTERN="" +EVAL_FILE_PATTERN="" +VAL_JSON_FILE="" +python3 ~/models/official/vision/detection/main.py \ + --strategy_type=tpu \ + --tpu="${TPU_NAME?}" \ + --model_dir="${MODEL_DIR?}" \ + --mode=train \ + --params_override="{ type: retinanet, train: { checkpoint: { path: ${RESNET_CHECKPOINT?}, prefix: resnet50/ }, train_file_pattern: ${TRAIN_FILE_PATTERN?} }, eval: { val_json_file: ${VAL_JSON_FILE?}, eval_file_pattern: ${EVAL_FILE_PATTERN?} } }" +``` + +### Train a custom RetinaNet using the config file. + +First, create a YAML config file, e.g. *my_retinanet.yaml*. This file specifies +the parameters to be overridden, which should at least include the following +fields. + +```YAML +# my_retinanet.yaml +type: 'retinanet' +train: + train_file_pattern: +eval: + eval_file_pattern: + val_json_file: +``` + +Once the YAML config file is created, you can launch the training using the +following command. + +```bash +TPU_NAME="" +MODEL_DIR="" +python3 ~/models/official/vision/detection/main.py \ + --strategy_type=tpu \ + --tpu="${TPU_NAME?}" \ + --model_dir="${MODEL_DIR?}" \ + --mode=train \ + --config_file="my_retinanet.yaml" +``` + +## Train RetinaNet on GPU + +Training on GPU is similar to that on TPU. The major change is the strategy +type (use "[mirrored](https://www.tensorflow.org/api_docs/python/tf/distribute/MirroredStrategy)" for multiple GPU and +"[one_device](https://www.tensorflow.org/api_docs/python/tf/distribute/OneDeviceStrategy)" for single GPU). + +Multi-GPUs example (assuming there are 8GPU connected to the host): + +```bash +MODEL_DIR="" +python3 ~/models/official/vision/detection/main.py \ + --strategy_type=mirrored \ + --num_gpus=8 \ + --model_dir="${MODEL_DIR?}" \ + --mode=train \ + --config_file="my_retinanet.yaml" +``` + +```bash +MODEL_DIR="" +python3 ~/models/official/vision/detection/main.py \ + --strategy_type=one_device \ + --num_gpus=1 \ + --model_dir="${MODEL_DIR?}" \ + --mode=train \ + --config_file="my_retinanet.yaml" +``` + +An example with inline configuration (YAML or JSON format): + +``` +python3 ~/models/official/vision/detection/main.py \ + --model_dir= \ + --strategy_type=one_device \ + --num_gpus=1 \ + --mode=train \ + --params_override="eval: + eval_file_pattern: + batch_size: 8 + val_json_file: +predict: + predict_batch_size: 8 +architecture: + use_bfloat16: False +retinanet_parser: + use_bfloat16: Flase +train: + total_steps: 1 + batch_size: 8 + train_file_pattern: +use_tpu: False +" +``` + +--- + +## Train Mask R-CNN on TPU + +### Train a vanilla ResNet-50 based Mask R-CNN. + +```bash +TPU_NAME="" +MODEL_DIR="" +RESNET_CHECKPOINT="" +TRAIN_FILE_PATTERN="" +EVAL_FILE_PATTERN="" +VAL_JSON_FILE="" +python3 ~/models/official/vision/detection/main.py \ + --strategy_type=tpu \ + --tpu=${TPU_NAME} \ + --model_dir=${MODEL_DIR} \ + --mode=train \ + --model=mask_rcnn \ + --params_override="{train: { checkpoint: { path: ${RESNET_CHECKPOINT}, prefix: resnet50/ }, train_file_pattern: ${TRAIN_FILE_PATTERN} }, eval: { val_json_file: ${VAL_JSON_FILE}, eval_file_pattern: ${EVAL_FILE_PATTERN} } }" +``` + +### Train a custom Mask R-CNN using the config file. + +First, create a YAML config file, e.g. *my_maskrcnn.yaml*. +This file specifies the parameters to be overridden, +which should at least include the following fields. + +```YAML +# my_maskrcnn.yaml +train: + train_file_pattern: +eval: + eval_file_pattern: + val_json_file: +``` + +Once the YAML config file is created, you can launch the training using the +following command. + +```bash +TPU_NAME="" +MODEL_DIR="" +python3 ~/models/official/vision/detection/main.py \ + --strategy_type=tpu \ + --tpu=${TPU_NAME} \ + --model_dir=${MODEL_DIR} \ + --mode=train \ + --model=mask_rcnn \ + --config_file="my_maskrcnn.yaml" +``` + +## Train Mask R-CNN on GPU + +Training on GPU is similar to that on TPU. The major change is the strategy type +(use +"[mirrored](https://www.tensorflow.org/api_docs/python/tf/distribute/MirroredStrategy)" +for multiple GPU and +"[one_device](https://www.tensorflow.org/api_docs/python/tf/distribute/OneDeviceStrategy)" +for single GPU). + +Multi-GPUs example (assuming there are 8GPU connected to the host): + +```bash +MODEL_DIR="" +python3 ~/models/official/vision/detection/main.py \ + --strategy_type=mirrored \ + --num_gpus=8 \ + --model_dir=${MODEL_DIR} \ + --mode=train \ + --model=mask_rcnn \ + --config_file="my_maskrcnn.yaml" +``` + +```bash +MODEL_DIR="" +python3 ~/models/official/vision/detection/main.py \ + --strategy_type=one_device \ + --num_gpus=1 \ + --model_dir=${MODEL_DIR} \ + --mode=train \ + --model=mask_rcnn \ + --config_file="my_maskrcnn.yaml" +``` + +An example with inline configuration (YAML or JSON format): + +``` +python3 ~/models/official/vision/detection/main.py \ + --model_dir= \ + --strategy_type=one_device \ + --num_gpus=1 \ + --mode=train \ + --model=mask_rcnn \ + --params_override="eval: + eval_file_pattern: + batch_size: 8 + val_json_file: +predict: + predict_batch_size: 8 +architecture: + use_bfloat16: False +maskrcnn_parser: + use_bfloat16: Flase +train: + total_steps: 1000 + batch_size: 8 + train_file_pattern: +use_tpu: False +" +``` + +Note: The JSON groundtruth file is useful for [COCO dataset](http://cocodataset.org/#home) and can be +downloaded from the [COCO website](http://cocodataset.org/#download). For custom dataset, it is unncessary because the groundtruth can be included in the TFRecord files. + +## References + +1. [Focal Loss for Dense Object Detection](https://arxiv.org/abs/1708.02002). + Tsung-Yi Lin, Priya Goyal, Ross Girshick, Kaiming He, and Piotr Dollár. IEEE + International Conference on Computer Vision (ICCV), 2017. diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/__init__.py new file mode 100644 index 0000000..931c2ef --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/base_config.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/base_config.py new file mode 100644 index 0000000..b23a7d5 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/base_config.py @@ -0,0 +1,120 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Base config template.""" + +# pylint: disable=line-too-long + +# For ResNet, this freezes the variables of the first conv1 and conv2_x +# layers [1], which leads to higher training speed and slightly better testing +# accuracy. The intuition is that the low-level architecture (e.g., ResNet-50) +# is able to capture low-level features such as edges; therefore, it does not +# need to be fine-tuned for the detection task. +# Note that we need to trailing `/` to avoid the incorrect match. +# [1]: https://github.com/facebookresearch/Detectron/blob/master/detectron/core/config.py#L198 +RESNET_FROZEN_VAR_PREFIX = r'(resnet\d+)\/(conv2d(|_([1-9]|10))|batch_normalization(|_([1-9]|10)))\/' + +REGULARIZATION_VAR_REGEX = r'.*(kernel|weight):0$' + +BASE_CFG = { + 'model_dir': '', + 'use_tpu': True, + 'strategy_type': 'tpu', + 'isolate_session_state': False, + 'train': { + 'iterations_per_loop': 100, + 'batch_size': 64, + 'total_steps': 22500, + 'num_cores_per_replica': None, + 'input_partition_dims': None, + 'optimizer': { + 'type': 'momentum', + 'momentum': 0.9, + }, + 'learning_rate': { + 'type': 'step', + 'warmup_learning_rate': 0.0067, + 'warmup_steps': 500, + 'init_learning_rate': 0.08, + 'learning_rate_levels': [0.008, 0.0008], + 'learning_rate_steps': [15000, 20000], + 'total_steps': 22500, + }, + 'checkpoint': { + 'path': '', + 'prefix': '', + }, + 'frozen_variable_prefix': RESNET_FROZEN_VAR_PREFIX, + 'train_file_pattern': '', + 'train_dataset_type': 'tfrecord', + 'transpose_input': False, + 'regularization_variable_regex': REGULARIZATION_VAR_REGEX, + 'l2_weight_decay': 0.0001, + 'gradient_clip_norm': 0.0, + }, + 'eval': { + 'batch_size': 8, + 'eval_samples': 5000, + 'min_eval_interval': 180, + 'eval_timeout': None, + 'num_steps_per_eval': 1000, + 'type': 'box', + 'use_json_file': True, + 'val_json_file': '', + 'eval_file_pattern': '', + 'eval_dataset_type': 'tfrecord', + }, + 'predict': { + 'batch_size': 8, + }, + 'anchor': { + 'min_level': 3, + 'max_level': 7, + 'num_scales': 3, + 'aspect_ratios': [1.0, 2.0, 0.5], + 'anchor_size': 4.0, + }, + 'resnet': { + 'resnet_depth': 50, + 'batch_norm': { + 'batch_norm_momentum': 0.997, + 'batch_norm_epsilon': 1e-4, + 'batch_norm_trainable': True, + 'use_sync_bn': False, + }, + }, + 'fpn': { + 'min_level': 3, + 'max_level': 7, + 'fpn_feat_dims': 256, + 'use_separable_conv': False, + 'use_batch_norm': True, + 'batch_norm': { + 'batch_norm_momentum': 0.997, + 'batch_norm_epsilon': 1e-4, + 'batch_norm_trainable': True, + 'use_sync_bn': False, + }, + }, + 'postprocess': { + 'use_batched_nms': False, + 'max_total_size': 100, + 'nms_iou_threshold': 0.5, + 'score_threshold': 0.05, + 'pre_nms_num_boxes': 5000, + }, + 'enable_summary': False, +} + +# pylint: enable=line-too-long diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/factory.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/factory.py new file mode 100644 index 0000000..186d89b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/factory.py @@ -0,0 +1,33 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Factory to provide model configs.""" + +from official.vision.detection.configs import maskrcnn_config +from official.vision.detection.configs import retinanet_config +from official.modeling.hyperparams import params_dict + + +def config_generator(model): + """Model function generator.""" + if model == 'retinanet': + default_config = retinanet_config.RETINANET_CFG + restrictions = retinanet_config.RETINANET_RESTRICTIONS + elif model == 'mask_rcnn': + default_config = maskrcnn_config.MASKRCNN_CFG + restrictions = maskrcnn_config.MASKRCNN_RESTRICTIONS + else: + raise ValueError('Model %s is not supported.' % model) + + return params_dict.ParamsDict(default_config, restrictions) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/maskrcnn_config.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/maskrcnn_config.py new file mode 100644 index 0000000..2f4f3f1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/maskrcnn_config.py @@ -0,0 +1,169 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Config template to train Mask R-CNN.""" + +from official.modeling.hyperparams import params_dict +from official.vision.detection.configs import base_config + + +# pylint: disable=line-too-long +MASKRCNN_CFG = params_dict.ParamsDict(base_config.BASE_CFG) +MASKRCNN_CFG.override({ + 'type': 'mask_rcnn', + 'eval': { + 'type': 'box_and_mask', + 'num_images_to_visualize': 0, + }, + 'architecture': { + 'parser': 'maskrcnn_parser', + 'backbone': 'resnet', + 'multilevel_features': 'fpn', + 'use_bfloat16': True, + 'include_mask': True, + }, + 'maskrcnn_parser': { + 'use_bfloat16': True, + 'output_size': [1024, 1024], + 'num_channels': 3, + 'rpn_match_threshold': 0.7, + 'rpn_unmatched_threshold': 0.3, + 'rpn_batch_size_per_im': 256, + 'rpn_fg_fraction': 0.5, + 'aug_rand_hflip': True, + 'aug_scale_min': 1.0, + 'aug_scale_max': 1.0, + 'skip_crowd_during_training': True, + 'max_num_instances': 100, + 'include_mask': True, + 'mask_crop_size': 112, + }, + 'anchor': { + 'min_level': 2, + 'max_level': 6, + 'num_scales': 1, + 'anchor_size': 8, + }, + 'fpn': { + 'min_level': 2, + 'max_level': 6, + }, + 'nasfpn': { + 'min_level': 2, + 'max_level': 6, + }, + # tunable_nasfpn:strip_begin + 'tunable_nasfpn_v1': { + 'min_level': 2, + 'max_level': 6, + }, + # tunable_nasfpn:strip_end + 'rpn_head': { + 'min_level': 2, + 'max_level': 6, + 'anchors_per_location': 3, + 'num_convs': 2, + 'num_filters': 256, + 'use_separable_conv': False, + 'use_batch_norm': False, + 'batch_norm': { + 'batch_norm_momentum': 0.997, + 'batch_norm_epsilon': 1e-4, + 'batch_norm_trainable': True, + 'use_sync_bn': False, + }, + }, + 'frcnn_head': { + # Note that `num_classes` is the total number of classes including + # one background classes whose index is 0. + 'num_classes': 91, + 'num_convs': 0, + 'num_filters': 256, + 'use_separable_conv': False, + 'num_fcs': 2, + 'fc_dims': 1024, + 'use_batch_norm': False, + 'batch_norm': { + 'batch_norm_momentum': 0.997, + 'batch_norm_epsilon': 1e-4, + 'batch_norm_trainable': True, + 'use_sync_bn': False, + }, + }, + 'mrcnn_head': { + 'num_classes': 91, + 'mask_target_size': 28, + 'num_convs': 4, + 'num_filters': 256, + 'use_separable_conv': False, + 'use_batch_norm': False, + 'batch_norm': { + 'batch_norm_momentum': 0.997, + 'batch_norm_epsilon': 1e-4, + 'batch_norm_trainable': True, + 'use_sync_bn': False, + }, + }, + 'rpn_score_loss': { + 'rpn_batch_size_per_im': 256, + }, + 'rpn_box_loss': { + 'huber_loss_delta': 1.0 / 9.0, + }, + 'frcnn_box_loss': { + 'huber_loss_delta': 1.0, + }, + 'roi_proposal': { + 'rpn_pre_nms_top_k': 2000, + 'rpn_post_nms_top_k': 1000, + 'rpn_nms_threshold': 0.7, + 'rpn_score_threshold': 0.0, + 'rpn_min_size_threshold': 0.0, + 'test_rpn_pre_nms_top_k': 1000, + 'test_rpn_post_nms_top_k': 1000, + 'test_rpn_nms_threshold': 0.7, + 'test_rpn_score_threshold': 0.0, + 'test_rpn_min_size_threshold': 0.0, + 'use_batched_nms': False, + }, + 'roi_sampling': { + 'num_samples_per_image': 512, + 'fg_fraction': 0.25, + 'fg_iou_thresh': 0.5, + 'bg_iou_thresh_hi': 0.5, + 'bg_iou_thresh_lo': 0.0, + 'mix_gt_boxes': True, + }, + 'mask_sampling': { + 'num_mask_samples_per_image': 128, # Typically = `num_samples_per_image` * `fg_fraction`. + 'mask_target_size': 28, + }, + 'postprocess': { + 'use_batched_nms': False, + 'max_total_size': 100, + 'nms_iou_threshold': 0.5, + 'score_threshold': 0.05, + 'pre_nms_num_boxes': 1000, + }, +}, is_strict=False) + + +MASKRCNN_RESTRICTIONS = [ + 'architecture.use_bfloat16 == maskrcnn_parser.use_bfloat16', + 'architecture.include_mask == maskrcnn_parser.include_mask', + 'anchor.min_level == rpn_head.min_level', + 'anchor.max_level == rpn_head.max_level', + 'mrcnn_head.mask_target_size == mask_sampling.mask_target_size', +] +# pylint: enable=line-too-long diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/retinanet_config.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/retinanet_config.py new file mode 100644 index 0000000..70ee4bb --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/configs/retinanet_config.py @@ -0,0 +1,171 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Config template to train Retinanet.""" + +# pylint: disable=line-too-long + +# For ResNet-50, this freezes the variables of the first conv1 and conv2_x +# layers [1], which leads to higher training speed and slightly better testing +# accuracy. The intuition is that the low-level architecture (e.g., ResNet-50) +# is able to capture low-level features such as edges; therefore, it does not +# need to be fine-tuned for the detection task. +# Note that we need to trailing `/` to avoid the incorrect match. +# [1]: https://github.com/facebookresearch/Detectron/blob/master/detectron/core/config.py#L198 +RESNET_FROZEN_VAR_PREFIX = r'(resnet\d+)\/(conv2d(|_([1-9]|10))|batch_normalization(|_([1-9]|10)))\/' +REGULARIZATION_VAR_REGEX = r'.*(kernel|weight):0$' + +# pylint: disable=line-too-long +RETINANET_CFG = { + 'type': 'retinanet', + 'model_dir': '', + 'use_tpu': True, + 'strategy_type': 'tpu', + 'train': { + 'batch_size': 64, + 'iterations_per_loop': 500, + 'total_steps': 22500, + 'optimizer': { + 'type': 'momentum', + 'momentum': 0.9, + 'nesterov': True, # `False` is better for TPU v3-128. + }, + 'learning_rate': { + 'type': 'step', + 'warmup_learning_rate': 0.0067, + 'warmup_steps': 500, + 'init_learning_rate': 0.08, + 'learning_rate_levels': [0.008, 0.0008], + 'learning_rate_steps': [15000, 20000], + }, + 'checkpoint': { + 'path': '', + 'prefix': '', + }, + 'frozen_variable_prefix': RESNET_FROZEN_VAR_PREFIX, + 'train_file_pattern': '', + # TODO(b/142174042): Support transpose_input option. + 'transpose_input': False, + 'regularization_variable_regex': REGULARIZATION_VAR_REGEX, + 'l2_weight_decay': 0.0001, + 'input_sharding': False, + }, + 'eval': { + 'batch_size': 8, + 'min_eval_interval': 180, + 'eval_timeout': None, + 'eval_samples': 5000, + 'type': 'box', + 'val_json_file': '', + 'eval_file_pattern': '', + 'input_sharding': True, + # When visualizing images, set evaluation batch size to 40 to avoid + # potential OOM. + 'num_images_to_visualize': 0, + }, + 'predict': { + 'predict_batch_size': 8, + }, + 'architecture': { + 'parser': 'retinanet_parser', + 'backbone': 'resnet', + 'multilevel_features': 'fpn', + 'use_bfloat16': False, + }, + 'anchor': { + 'min_level': 3, + 'max_level': 7, + 'num_scales': 3, + 'aspect_ratios': [1.0, 2.0, 0.5], + 'anchor_size': 4.0, + }, + 'retinanet_parser': { + 'use_bfloat16': False, + 'output_size': [640, 640], + 'num_channels': 3, + 'match_threshold': 0.5, + 'unmatched_threshold': 0.5, + 'aug_rand_hflip': True, + 'aug_scale_min': 1.0, + 'aug_scale_max': 1.0, + 'use_autoaugment': False, + 'autoaugment_policy_name': 'v0', + 'skip_crowd_during_training': True, + 'max_num_instances': 100, + }, + 'resnet': { + 'resnet_depth': 50, + 'batch_norm': { + 'batch_norm_momentum': 0.997, + 'batch_norm_epsilon': 1e-4, + 'batch_norm_trainable': True, + }, + }, + 'fpn': { + 'min_level': 3, + 'max_level': 7, + 'fpn_feat_dims': 256, + 'use_separable_conv': False, + 'use_batch_norm': True, + 'batch_norm': { + 'batch_norm_momentum': 0.997, + 'batch_norm_epsilon': 1e-4, + 'batch_norm_trainable': True, + }, + }, + 'retinanet_head': { + 'min_level': 3, + 'max_level': 7, + # Note that `num_classes` is the total number of classes including + # one background classes whose index is 0. + 'num_classes': 91, + 'anchors_per_location': 9, + 'retinanet_head_num_convs': 4, + 'retinanet_head_num_filters': 256, + 'use_separable_conv': False, + 'batch_norm': { + 'batch_norm_momentum': 0.997, + 'batch_norm_epsilon': 1e-4, + 'batch_norm_trainable': True, + }, + }, + 'retinanet_loss': { + 'num_classes': 91, + 'focal_loss_alpha': 0.25, + 'focal_loss_gamma': 1.5, + 'huber_loss_delta': 0.1, + 'box_loss_weight': 50, + }, + 'postprocess': { + 'use_batched_nms': False, + 'min_level': 3, + 'max_level': 7, + 'max_total_size': 100, + 'nms_iou_threshold': 0.5, + 'score_threshold': 0.05, + 'pre_nms_num_boxes': 5000, + }, + 'enable_summary': False, +} + +RETINANET_RESTRICTIONS = [ + 'architecture.use_bfloat16 == retinanet_parser.use_bfloat16', + 'anchor.min_level == retinanet_head.min_level', + 'anchor.max_level == retinanet_head.max_level', + 'anchor.min_level == postprocess.min_level', + 'anchor.max_level == postprocess.max_level', + 'retinanet_head.num_classes == retinanet_loss.num_classes', +] + +# pylint: enable=line-too-long diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/__init__.py new file mode 100644 index 0000000..931c2ef --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/anchor.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/anchor.py new file mode 100644 index 0000000..bb7fca8 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/anchor.py @@ -0,0 +1,292 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Anchor box and labeler definition.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import tensorflow.compat.v2 as tf +from official.vision.detection.utils.object_detection import argmax_matcher +from official.vision.detection.utils.object_detection import balanced_positive_negative_sampler +from official.vision.detection.utils.object_detection import box_list +from official.vision.detection.utils.object_detection import faster_rcnn_box_coder +from official.vision.detection.utils.object_detection import region_similarity_calculator +from official.vision.detection.utils.object_detection import target_assigner + + +class Anchor(object): + """Anchor class for anchor-based object detectors.""" + + def __init__(self, + min_level, + max_level, + num_scales, + aspect_ratios, + anchor_size, + image_size): + """Constructs multiscale anchors. + + Args: + min_level: integer number of minimum level of the output feature pyramid. + max_level: integer number of maximum level of the output feature pyramid. + num_scales: integer number representing intermediate scales added + on each level. For instances, num_scales=2 adds one additional + intermediate anchor scales [2^0, 2^0.5] on each level. + aspect_ratios: list of float numbers representing the aspect raito anchors + added on each level. The number indicates the ratio of width to height. + For instances, aspect_ratios=[1.0, 2.0, 0.5] adds three anchors on each + scale level. + anchor_size: float number representing the scale of size of the base + anchor to the feature stride 2^level. + image_size: a list of integer numbers or Tensors representing + [height, width] of the input image size.The image_size should be divided + by the largest feature stride 2^max_level. + """ + self.min_level = min_level + self.max_level = max_level + self.num_scales = num_scales + self.aspect_ratios = aspect_ratios + self.anchor_size = anchor_size + self.image_size = image_size + self.boxes = self._generate_boxes() + + def _generate_boxes(self): + """Generates multiscale anchor boxes. + + Returns: + a Tensor of shape [N, 4], represneting anchor boxes of all levels + concatenated together. + """ + boxes_all = [] + for level in range(self.min_level, self.max_level + 1): + boxes_l = [] + for scale in range(self.num_scales): + for aspect_ratio in self.aspect_ratios: + stride = 2 ** level + intermidate_scale = 2 ** (scale / float(self.num_scales)) + base_anchor_size = self.anchor_size * stride * intermidate_scale + aspect_x = aspect_ratio ** 0.5 + aspect_y = aspect_ratio ** -0.5 + half_anchor_size_x = base_anchor_size * aspect_x / 2.0 + half_anchor_size_y = base_anchor_size * aspect_y / 2.0 + x = tf.range(stride / 2, self.image_size[1], stride) + y = tf.range(stride / 2, self.image_size[0], stride) + xv, yv = tf.meshgrid(x, y) + xv = tf.cast(tf.reshape(xv, [-1]), dtype=tf.float32) + yv = tf.cast(tf.reshape(yv, [-1]), dtype=tf.float32) + # Tensor shape Nx4. + boxes = tf.stack([yv - half_anchor_size_y, xv - half_anchor_size_x, + yv + half_anchor_size_y, xv + half_anchor_size_x], + axis=1) + boxes_l.append(boxes) + # Concat anchors on the same level to tensor shape NxAx4. + boxes_l = tf.stack(boxes_l, axis=1) + boxes_l = tf.reshape(boxes_l, [-1, 4]) + boxes_all.append(boxes_l) + return tf.concat(boxes_all, axis=0) + + def unpack_labels(self, labels): + """Unpacks an array of labels into multiscales labels.""" + unpacked_labels = collections.OrderedDict() + count = 0 + for level in range(self.min_level, self.max_level + 1): + feat_size_y = tf.cast(self.image_size[0] / 2 ** level, tf.int32) + feat_size_x = tf.cast(self.image_size[1] / 2 ** level, tf.int32) + steps = feat_size_y * feat_size_x * self.anchors_per_location + unpacked_labels[level] = tf.reshape( + labels[count:count + steps], [feat_size_y, feat_size_x, -1]) + count += steps + return unpacked_labels + + @property + def anchors_per_location(self): + return self.num_scales * len(self.aspect_ratios) + + @property + def multilevel_boxes(self): + return self.unpack_labels(self.boxes) + + +class AnchorLabeler(object): + """Labeler for dense object detector.""" + + def __init__(self, + anchor, + match_threshold=0.5, + unmatched_threshold=0.5): + """Constructs anchor labeler to assign labels to anchors. + + Args: + anchor: an instance of class Anchors. + match_threshold: a float number between 0 and 1 representing the + lower-bound threshold to assign positive labels for anchors. An anchor + with a score over the threshold is labeled positive. + unmatched_threshold: a float number between 0 and 1 representing the + upper-bound threshold to assign negative labels for anchors. An anchor + with a score below the threshold is labeled negative. + """ + similarity_calc = region_similarity_calculator.IouSimilarity() + matcher = argmax_matcher.ArgMaxMatcher( + match_threshold, + unmatched_threshold=unmatched_threshold, + negatives_lower_than_unmatched=True, + force_match_for_each_row=True) + box_coder = faster_rcnn_box_coder.FasterRcnnBoxCoder() + + self._target_assigner = target_assigner.TargetAssigner( + similarity_calc, matcher, box_coder) + self._anchor = anchor + self._match_threshold = match_threshold + self._unmatched_threshold = unmatched_threshold + + def label_anchors(self, gt_boxes, gt_labels): + """Labels anchors with ground truth inputs. + + Args: + gt_boxes: A float tensor with shape [N, 4] representing groundtruth boxes. + For each row, it stores [y0, x0, y1, x1] for four corners of a box. + gt_labels: A integer tensor with shape [N, 1] representing groundtruth + classes. + Returns: + cls_targets_dict: ordered dictionary with keys + [min_level, min_level+1, ..., max_level]. The values are tensor with + shape [height_l, width_l, num_anchors_per_location]. The height_l and + width_l represent the dimension of class logits at l-th level. + box_targets_dict: ordered dictionary with keys + [min_level, min_level+1, ..., max_level]. The values are tensor with + shape [height_l, width_l, num_anchors_per_location * 4]. The height_l + and width_l represent the dimension of bounding box regression output at + l-th level. + num_positives: scalar tensor storing number of positives in an image. + """ + gt_box_list = box_list.BoxList(gt_boxes) + anchor_box_list = box_list.BoxList(self._anchor.boxes) + + # The cls_weights, box_weights are not used. + cls_targets, _, box_targets, _, matches = self._target_assigner.assign( + anchor_box_list, gt_box_list, gt_labels) + + # Labels definition in matches.match_results: + # (1) match_results[i]>=0, meaning that column i is matched with row + # match_results[i]. + # (2) match_results[i]=-1, meaning that column i is not matched. + # (3) match_results[i]=-2, meaning that column i is ignored. + match_results = tf.expand_dims(matches.match_results, axis=1) + cls_targets = tf.cast(cls_targets, tf.int32) + cls_targets = tf.where( + tf.equal(match_results, -1), -tf.ones_like(cls_targets), cls_targets) + cls_targets = tf.where( + tf.equal(match_results, -2), -2 * tf.ones_like(cls_targets), + cls_targets) + + # Unpacks labels into multi-level representations. + cls_targets_dict = self._anchor.unpack_labels(cls_targets) + box_targets_dict = self._anchor.unpack_labels(box_targets) + num_positives = tf.reduce_sum( + input_tensor=tf.cast(tf.greater(matches.match_results, -1), tf.float32)) + + return cls_targets_dict, box_targets_dict, num_positives + + +class RpnAnchorLabeler(AnchorLabeler): + """Labeler for Region Proposal Network.""" + + def __init__(self, anchor, match_threshold=0.7, + unmatched_threshold=0.3, rpn_batch_size_per_im=256, + rpn_fg_fraction=0.5): + AnchorLabeler.__init__(self, anchor, match_threshold=0.7, + unmatched_threshold=0.3) + self._rpn_batch_size_per_im = rpn_batch_size_per_im + self._rpn_fg_fraction = rpn_fg_fraction + + def _get_rpn_samples(self, match_results): + """Computes anchor labels. + + This function performs subsampling for foreground (fg) and background (bg) + anchors. + Args: + match_results: A integer tensor with shape [N] representing the + matching results of anchors. (1) match_results[i]>=0, + meaning that column i is matched with row match_results[i]. + (2) match_results[i]=-1, meaning that column i is not matched. + (3) match_results[i]=-2, meaning that column i is ignored. + Returns: + score_targets: a integer tensor with the a shape of [N]. + (1) score_targets[i]=1, the anchor is a positive sample. + (2) score_targets[i]=0, negative. (3) score_targets[i]=-1, the anchor is + don't care (ignore). + """ + sampler = ( + balanced_positive_negative_sampler.BalancedPositiveNegativeSampler( + positive_fraction=self._rpn_fg_fraction, is_static=False)) + # indicator includes both positive and negative labels. + # labels includes only positives labels. + # positives = indicator & labels. + # negatives = indicator & !labels. + # ignore = !indicator. + indicator = tf.greater(match_results, -2) + labels = tf.greater(match_results, -1) + + samples = sampler.subsample( + indicator, self._rpn_batch_size_per_im, labels) + positive_labels = tf.where( + tf.logical_and(samples, labels), + tf.constant(2, dtype=tf.int32, shape=match_results.shape), + tf.constant(0, dtype=tf.int32, shape=match_results.shape)) + negative_labels = tf.where( + tf.logical_and(samples, tf.logical_not(labels)), + tf.constant(1, dtype=tf.int32, shape=match_results.shape), + tf.constant(0, dtype=tf.int32, shape=match_results.shape)) + ignore_labels = tf.fill(match_results.shape, -1) + + return (ignore_labels + positive_labels + negative_labels, + positive_labels, negative_labels) + + def label_anchors(self, gt_boxes, gt_labels): + """Labels anchors with ground truth inputs. + + Args: + gt_boxes: A float tensor with shape [N, 4] representing groundtruth boxes. + For each row, it stores [y0, x0, y1, x1] for four corners of a box. + gt_labels: A integer tensor with shape [N, 1] representing groundtruth + classes. + Returns: + score_targets_dict: ordered dictionary with keys + [min_level, min_level+1, ..., max_level]. The values are tensor with + shape [height_l, width_l, num_anchors]. The height_l and width_l + represent the dimension of class logits at l-th level. + box_targets_dict: ordered dictionary with keys + [min_level, min_level+1, ..., max_level]. The values are tensor with + shape [height_l, width_l, num_anchors * 4]. The height_l and + width_l represent the dimension of bounding box regression output at + l-th level. + """ + gt_box_list = box_list.BoxList(gt_boxes) + anchor_box_list = box_list.BoxList(self._anchor.boxes) + + # cls_targets, cls_weights, box_weights are not used. + _, _, box_targets, _, matches = self._target_assigner.assign( + anchor_box_list, gt_box_list, gt_labels) + + # score_targets contains the subsampled positive and negative anchors. + score_targets, _, _ = self._get_rpn_samples(matches.match_results) + + # Unpacks labels. + score_targets_dict = self._anchor.unpack_labels(score_targets) + box_targets_dict = self._anchor.unpack_labels(box_targets) + + return score_targets_dict, box_targets_dict diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/factory.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/factory.py new file mode 100644 index 0000000..186bd85 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/factory.py @@ -0,0 +1,102 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model architecture factory.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from official.vision.detection.dataloader import maskrcnn_parser +from official.vision.detection.dataloader import retinanet_parser +from official.vision.detection.dataloader import shapemask_parser + +def parser_generator(params, mode): + """Generator function for various dataset parser.""" + if params.architecture.parser == 'retinanet_parser': + anchor_params = params.anchor + parser_params = params.retinanet_parser + parser_fn = retinanet_parser.Parser( + output_size=parser_params.output_size, + min_level=anchor_params.min_level, + max_level=anchor_params.max_level, + num_scales=anchor_params.num_scales, + aspect_ratios=anchor_params.aspect_ratios, + anchor_size=anchor_params.anchor_size, + match_threshold=parser_params.match_threshold, + unmatched_threshold=parser_params.unmatched_threshold, + aug_rand_hflip=parser_params.aug_rand_hflip, + aug_scale_min=parser_params.aug_scale_min, + aug_scale_max=parser_params.aug_scale_max, + use_autoaugment=parser_params.use_autoaugment, + autoaugment_policy_name=parser_params.autoaugment_policy_name, + skip_crowd_during_training=parser_params.skip_crowd_during_training, + max_num_instances=parser_params.max_num_instances, + use_bfloat16=parser_params.use_bfloat16, + mode=mode) + elif params.architecture.parser == 'maskrcnn_parser': + anchor_params = params.anchor + parser_params = params.maskrcnn_parser + parser_fn = maskrcnn_parser.Parser( + output_size=parser_params.output_size, + min_level=anchor_params.min_level, + max_level=anchor_params.max_level, + num_scales=anchor_params.num_scales, + aspect_ratios=anchor_params.aspect_ratios, + anchor_size=anchor_params.anchor_size, + rpn_match_threshold=parser_params.rpn_match_threshold, + rpn_unmatched_threshold=parser_params.rpn_unmatched_threshold, + rpn_batch_size_per_im=parser_params.rpn_batch_size_per_im, + rpn_fg_fraction=parser_params.rpn_fg_fraction, + aug_rand_hflip=parser_params.aug_rand_hflip, + aug_scale_min=parser_params.aug_scale_min, + aug_scale_max=parser_params.aug_scale_max, + skip_crowd_during_training=parser_params.skip_crowd_during_training, + max_num_instances=parser_params.max_num_instances, + include_mask=parser_params.include_mask, + mask_crop_size=parser_params.mask_crop_size, + use_bfloat16=parser_params.use_bfloat16, + mode=mode) + elif params.architecture.parser == 'shapemask_parser': + anchor_params = params.anchor + parser_params = params.shapemask_parser + parser_fn = shapemask_parser.Parser( + output_size=parser_params.output_size, + min_level=anchor_params.min_level, + max_level=anchor_params.max_level, + num_scales=anchor_params.num_scales, + aspect_ratios=anchor_params.aspect_ratios, + anchor_size=anchor_params.anchor_size, + use_category=parser_params.use_category, + outer_box_scale=parser_params.outer_box_scale, + box_jitter_scale=parser_params.box_jitter_scale, + num_sampled_masks=parser_params.num_sampled_masks, + mask_crop_size=parser_params.mask_crop_size, + mask_min_level=parser_params.mask_min_level, + mask_max_level=parser_params.mask_max_level, + upsample_factor=parser_params.upsample_factor, + match_threshold=parser_params.match_threshold, + unmatched_threshold=parser_params.unmatched_threshold, + aug_rand_hflip=parser_params.aug_rand_hflip, + aug_scale_min=parser_params.aug_scale_min, + aug_scale_max=parser_params.aug_scale_max, + skip_crowd_during_training=parser_params.skip_crowd_during_training, + max_num_instances=parser_params.max_num_instances, + use_bfloat16=parser_params.use_bfloat16, + mask_train_class=parser_params.mask_train_class, + mode=mode) + else: + raise ValueError('Parser %s is not supported.' % params.architecture.parser) + + return parser_fn diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/input_reader.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/input_reader.py new file mode 100644 index 0000000..dd24b7e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/input_reader.py @@ -0,0 +1,109 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Data loader and input processing.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow.compat.v2 as tf + +from typing import Text, Optional +from official.modeling.hyperparams import params_dict +from official.vision.detection.dataloader import factory +from official.vision.detection.dataloader import mode_keys as ModeKeys + + +class InputFn(object): + """Input function that creates dataset from files.""" + + def __init__(self, + file_pattern: Text, + params: params_dict.ParamsDict, + mode: Text, + batch_size: int, + num_examples: Optional[int] = -1): + """Initialize. + + Args: + file_pattern: the file pattern for the data example (TFRecords). + params: the parameter object for constructing example parser and model. + mode: ModeKeys.TRAIN or ModeKeys.Eval + batch_size: the data batch size. + num_examples: If positive, only takes this number of examples and raise + tf.errors.OutOfRangeError after that. If non-positive, it will be + ignored. + """ + assert file_pattern is not None + assert mode is not None + assert batch_size is not None + self._file_pattern = file_pattern + self._mode = mode + self._is_training = (mode == ModeKeys.TRAIN) + self._batch_size = batch_size + self._num_examples = num_examples + self._parser_fn = factory.parser_generator(params, mode) + self._dataset_fn = tf.data.TFRecordDataset + + self._input_sharding = (not self._is_training) + try: + if self._is_training: + self._input_sharding = params.train.input_sharding + else: + self._input_sharding = params.eval.input_sharding + except AttributeError: + pass + + def __call__(self, ctx=None, batch_size: int = None): + """Provides tf.data.Dataset object. + + Args: + ctx: context object. + batch_size: expected batch size input data. + + Returns: + tf.data.Dataset object. + """ + if not batch_size: + batch_size = self._batch_size + assert batch_size is not None + dataset = tf.data.Dataset.list_files( + self._file_pattern, shuffle=self._is_training) + + if self._input_sharding and ctx and ctx.num_input_pipelines > 1: + dataset = dataset.shard(ctx.num_input_pipelines, ctx.input_pipeline_id) + dataset = dataset.cache() + + if self._is_training: + dataset = dataset.repeat() + + dataset = dataset.interleave( + map_func=lambda file_name: self._dataset_fn(file_name), cycle_length=32, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + + if self._is_training: + # Large shuffle size is critical for 2vm input pipeline. Can use small + # value (e.g. 64) for 1vm. + dataset = dataset.shuffle(1000) + if self._num_examples > 0: + dataset = dataset.take(self._num_examples) + + # Parses the fetched records to input tensors for model function. + dataset = dataset.map( + self._parser_fn, num_parallel_calls=tf.data.experimental.AUTOTUNE) + dataset = dataset.batch(batch_size, drop_remainder=True) + dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE) + return dataset diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/maskrcnn_parser.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/maskrcnn_parser.py new file mode 100644 index 0000000..0f44619 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/maskrcnn_parser.py @@ -0,0 +1,385 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Data parser and processing for Mask R-CNN.""" + +import tensorflow.compat.v2 as tf + +from official.vision.detection.dataloader import anchor +from official.vision.detection.dataloader import mode_keys as ModeKeys +from official.vision.detection.dataloader import tf_example_decoder +from official.vision.detection.utils import box_utils +from official.vision.detection.utils import dataloader_utils +from official.vision.detection.utils import input_utils + + +class Parser(object): + """Parser to parse an image and its annotations into a dictionary of tensors.""" + + def __init__(self, + output_size, + min_level, + max_level, + num_scales, + aspect_ratios, + anchor_size, + rpn_match_threshold=0.7, + rpn_unmatched_threshold=0.3, + rpn_batch_size_per_im=256, + rpn_fg_fraction=0.5, + aug_rand_hflip=False, + aug_scale_min=1.0, + aug_scale_max=1.0, + skip_crowd_during_training=True, + max_num_instances=100, + include_mask=False, + mask_crop_size=112, + use_bfloat16=True, + mode=None): + """Initializes parameters for parsing annotations in the dataset. + + Args: + output_size: `Tensor` or `list` for [height, width] of output image. The + output_size should be divided by the largest feature stride 2^max_level. + min_level: `int` number of minimum level of the output feature pyramid. + max_level: `int` number of maximum level of the output feature pyramid. + num_scales: `int` number representing intermediate scales added + on each level. For instances, num_scales=2 adds one additional + intermediate anchor scales [2^0, 2^0.5] on each level. + aspect_ratios: `list` of float numbers representing the aspect raito + anchors added on each level. The number indicates the ratio of width to + height. For instances, aspect_ratios=[1.0, 2.0, 0.5] adds three anchors + on each scale level. + anchor_size: `float` number representing the scale of size of the base + anchor to the feature stride 2^level. + rpn_match_threshold: + rpn_unmatched_threshold: + rpn_batch_size_per_im: + rpn_fg_fraction: + aug_rand_hflip: `bool`, if True, augment training with random + horizontal flip. + aug_scale_min: `float`, the minimum scale applied to `output_size` for + data augmentation during training. + aug_scale_max: `float`, the maximum scale applied to `output_size` for + data augmentation during training. + skip_crowd_during_training: `bool`, if True, skip annotations labeled with + `is_crowd` equals to 1. + max_num_instances: `int` number of maximum number of instances in an + image. The groundtruth data will be padded to `max_num_instances`. + include_mask: a bool to indicate whether parse mask groundtruth. + mask_crop_size: the size which groundtruth mask is cropped to. + use_bfloat16: `bool`, if True, cast output image to tf.bfloat16. + mode: a ModeKeys. Specifies if this is training, evaluation, prediction + or prediction with groundtruths in the outputs. + """ + self._mode = mode + self._max_num_instances = max_num_instances + self._skip_crowd_during_training = skip_crowd_during_training + self._is_training = (mode == ModeKeys.TRAIN) + + self._example_decoder = tf_example_decoder.TfExampleDecoder( + include_mask=include_mask) + + # Anchor. + self._output_size = output_size + self._min_level = min_level + self._max_level = max_level + self._num_scales = num_scales + self._aspect_ratios = aspect_ratios + self._anchor_size = anchor_size + + # Target assigning. + self._rpn_match_threshold = rpn_match_threshold + self._rpn_unmatched_threshold = rpn_unmatched_threshold + self._rpn_batch_size_per_im = rpn_batch_size_per_im + self._rpn_fg_fraction = rpn_fg_fraction + + # Data augmentation. + self._aug_rand_hflip = aug_rand_hflip + self._aug_scale_min = aug_scale_min + self._aug_scale_max = aug_scale_max + + # Mask. + self._include_mask = include_mask + self._mask_crop_size = mask_crop_size + + # Device. + self._use_bfloat16 = use_bfloat16 + + # Data is parsed depending on the model Modekey. + if mode == ModeKeys.TRAIN: + self._parse_fn = self._parse_train_data + elif mode == ModeKeys.EVAL: + self._parse_fn = self._parse_eval_data + elif mode == ModeKeys.PREDICT or mode == ModeKeys.PREDICT_WITH_GT: + self._parse_fn = self._parse_predict_data + else: + raise ValueError('mode is not defined.') + + def __call__(self, value): + """Parses data to an image and associated training labels. + + Args: + value: a string tensor holding a serialized tf.Example proto. + + Returns: + image, labels: if mode == ModeKeys.TRAIN. see _parse_train_data. + {'images': image, 'labels': labels}: if mode == ModeKeys.PREDICT + or ModeKeys.PREDICT_WITH_GT. + """ + with tf.name_scope('parser'): + data = self._example_decoder.decode(value) + return self._parse_fn(data) + + def _parse_train_data(self, data): + """Parses data for training. + + Args: + data: the decoded tensor dictionary from TfExampleDecoder. + + Returns: + image: image tensor that is preproessed to have normalized value and + dimension [output_size[0], output_size[1], 3] + labels: a dictionary of tensors used for training. The following describes + {key: value} pairs in the dictionary. + image_info: a 2D `Tensor` that encodes the information of the image and + the applied preprocessing. It is in the format of + [[original_height, original_width], [scaled_height, scaled_width], + anchor_boxes: ordered dictionary with keys + [min_level, min_level+1, ..., max_level]. The values are tensor with + shape [height_l, width_l, 4] representing anchor boxes at each level. + rpn_score_targets: ordered dictionary with keys + [min_level, min_level+1, ..., max_level]. The values are tensor with + shape [height_l, width_l, anchors_per_location]. The height_l and + width_l represent the dimension of class logits at l-th level. + rpn_box_targets: ordered dictionary with keys + [min_level, min_level+1, ..., max_level]. The values are tensor with + shape [height_l, width_l, anchors_per_location * 4]. The height_l and + width_l represent the dimension of bounding box regression output at + l-th level. + gt_boxes: Groundtruth bounding box annotations. The box is represented + in [y1, x1, y2, x2] format. The coordinates are w.r.t the scaled + image that is fed to the network. The tennsor is padded with -1 to + the fixed dimension [self._max_num_instances, 4]. + gt_classes: Groundtruth classes annotations. The tennsor is padded + with -1 to the fixed dimension [self._max_num_instances]. + gt_masks: groundtrugh masks cropped by the bounding box and + resized to a fixed size determined by mask_crop_size. + """ + classes = data['groundtruth_classes'] + boxes = data['groundtruth_boxes'] + if self._include_mask: + masks = data['groundtruth_instance_masks'] + + is_crowds = data['groundtruth_is_crowd'] + # Skips annotations with `is_crowd` = True. + if self._skip_crowd_during_training and self._is_training: + num_groundtrtuhs = tf.shape(classes)[0] + with tf.control_dependencies([num_groundtrtuhs, is_crowds]): + indices = tf.cond( + tf.greater(tf.size(is_crowds), 0), + lambda: tf.where(tf.logical_not(is_crowds))[:, 0], + lambda: tf.cast(tf.range(num_groundtrtuhs), tf.int64)) + classes = tf.gather(classes, indices) + boxes = tf.gather(boxes, indices) + if self._include_mask: + masks = tf.gather(masks, indices) + + # Gets original image and its size. + image = data['image'] + image_shape = tf.shape(image)[0:2] + + # Normalizes image with mean and std pixel values. + image = input_utils.normalize_image(image) + + # Flips image randomly during training. + if self._aug_rand_hflip: + if self._include_mask: + image, boxes, masks = input_utils.random_horizontal_flip( + image, boxes, masks) + else: + image, boxes = input_utils.random_horizontal_flip( + image, boxes) + + # Converts boxes from normalized coordinates to pixel coordinates. + # Now the coordinates of boxes are w.r.t. the original image. + boxes = box_utils.denormalize_boxes(boxes, image_shape) + + # Resizes and crops image. + image, image_info = input_utils.resize_and_crop_image( + image, + self._output_size, + padded_size=input_utils.compute_padded_size( + self._output_size, 2 ** self._max_level), + aug_scale_min=self._aug_scale_min, + aug_scale_max=self._aug_scale_max) + image_height, image_width, _ = image.get_shape().as_list() + + # Resizes and crops boxes. + # Now the coordinates of boxes are w.r.t the scaled image. + image_scale = image_info[2, :] + offset = image_info[3, :] + boxes = input_utils.resize_and_crop_boxes( + boxes, image_scale, image_info[1, :], offset) + + # Filters out ground truth boxes that are all zeros. + indices = box_utils.get_non_empty_box_indices(boxes) + boxes = tf.gather(boxes, indices) + classes = tf.gather(classes, indices) + if self._include_mask: + masks = tf.gather(masks, indices) + # Transfer boxes to the original image space and do normalization. + cropped_boxes = boxes + tf.tile(tf.expand_dims(offset, axis=0), [1, 2]) + cropped_boxes /= tf.tile(tf.expand_dims(image_scale, axis=0), [1, 2]) + cropped_boxes = box_utils.normalize_boxes(cropped_boxes, image_shape) + num_masks = tf.shape(masks)[0] + masks = tf.image.crop_and_resize( + tf.expand_dims(masks, axis=-1), + cropped_boxes, + box_indices=tf.range(num_masks, dtype=tf.int32), + crop_size=[self._mask_crop_size, self._mask_crop_size], + method='bilinear') + masks = tf.squeeze(masks, axis=-1) + + # Assigns anchor targets. + # Note that after the target assignment, box targets are absolute pixel + # offsets w.r.t. the scaled image. + input_anchor = anchor.Anchor( + self._min_level, + self._max_level, + self._num_scales, + self._aspect_ratios, + self._anchor_size, + (image_height, image_width)) + anchor_labeler = anchor.RpnAnchorLabeler( + input_anchor, + self._rpn_match_threshold, + self._rpn_unmatched_threshold, + self._rpn_batch_size_per_im, + self._rpn_fg_fraction) + rpn_score_targets, rpn_box_targets = anchor_labeler.label_anchors( + boxes, tf.cast(tf.expand_dims(classes, axis=-1), dtype=tf.float32)) + + # If bfloat16 is used, casts input image to tf.bfloat16. + if self._use_bfloat16: + image = tf.cast(image, dtype=tf.bfloat16) + + inputs = { + 'image': image, + 'image_info': image_info, + } + # Packs labels for model_fn outputs. + labels = { + 'anchor_boxes': input_anchor.multilevel_boxes, + 'image_info': image_info, + 'rpn_score_targets': rpn_score_targets, + 'rpn_box_targets': rpn_box_targets, + } + inputs['gt_boxes'] = input_utils.pad_to_fixed_size(boxes, + self._max_num_instances, + -1) + inputs['gt_classes'] = input_utils.pad_to_fixed_size( + classes, self._max_num_instances, -1) + if self._include_mask: + inputs['gt_masks'] = input_utils.pad_to_fixed_size( + masks, self._max_num_instances, -1) + + return inputs, labels + + def _parse_eval_data(self, data): + """Parses data for evaluation.""" + raise NotImplementedError('Not implemented!') + + def _parse_predict_data(self, data): + """Parses data for prediction. + + Args: + data: the decoded tensor dictionary from TfExampleDecoder. + + Returns: + A dictionary of {'images': image, 'labels': labels} where + image: image tensor that is preproessed to have normalized value and + dimension [output_size[0], output_size[1], 3] + labels: a dictionary of tensors used for training. The following + describes {key: value} pairs in the dictionary. + source_ids: Source image id. Default value -1 if the source id is + empty in the groundtruth annotation. + image_info: a 2D `Tensor` that encodes the information of the image + and the applied preprocessing. It is in the format of + [[original_height, original_width], [scaled_height, scaled_width], + anchor_boxes: ordered dictionary with keys + [min_level, min_level+1, ..., max_level]. The values are tensor with + shape [height_l, width_l, 4] representing anchor boxes at each + level. + """ + # Gets original image and its size. + image = data['image'] + image_shape = tf.shape(image)[0:2] + + # Normalizes image with mean and std pixel values. + image = input_utils.normalize_image(image) + + # Resizes and crops image. + image, image_info = input_utils.resize_and_crop_image( + image, + self._output_size, + padded_size=input_utils.compute_padded_size( + self._output_size, 2 ** self._max_level), + aug_scale_min=1.0, + aug_scale_max=1.0) + image_height, image_width, _ = image.get_shape().as_list() + + # If bfloat16 is used, casts input image to tf.bfloat16. + if self._use_bfloat16: + image = tf.cast(image, dtype=tf.bfloat16) + + # Compute Anchor boxes. + input_anchor = anchor.Anchor( + self._min_level, + self._max_level, + self._num_scales, + self._aspect_ratios, + self._anchor_size, + (image_height, image_width)) + + labels = { + 'image_info': image_info, + } + + if self._mode == ModeKeys.PREDICT_WITH_GT: + # Converts boxes from normalized coordinates to pixel coordinates. + boxes = box_utils.denormalize_boxes( + data['groundtruth_boxes'], image_shape) + groundtruths = { + 'source_id': data['source_id'], + 'height': data['height'], + 'width': data['width'], + 'num_detections': tf.shape(data['groundtruth_classes']), + 'boxes': boxes, + 'classes': data['groundtruth_classes'], + 'areas': data['groundtruth_area'], + 'is_crowds': tf.cast(data['groundtruth_is_crowd'], tf.int32), + } + groundtruths['source_id'] = dataloader_utils.process_source_id( + groundtruths['source_id']) + groundtruths = dataloader_utils.pad_groundtruths_to_fixed_size( + groundtruths, self._max_num_instances) + # TODO(yeqing): Remove the `groundtrtuh` layer key (no longer needed). + labels['groundtruths'] = groundtruths + inputs = { + 'image': image, + 'image_info': image_info, + } + + return inputs, labels diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/mode_keys.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/mode_keys.py new file mode 100644 index 0000000..020382b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/mode_keys.py @@ -0,0 +1,33 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Standard names for input dataloader modes. + +The following standard keys are defined: + +* `TRAIN`: training mode. +* `EVAL`: evaluation mode. +* `PREDICT`: prediction mode. +* `PREDICT_WITH_GT`: prediction mode with groundtruths in returned variables. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +TRAIN = 'train' +EVAL = 'eval' +PREDICT = 'predict' +PREDICT_WITH_GT = 'predict_with_gt' diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/retinanet_parser.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/retinanet_parser.py new file mode 100644 index 0000000..8f37bc9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/retinanet_parser.py @@ -0,0 +1,429 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Data parser and processing. + +Parse image and ground truths in a dataset to training targets and package them +into (image, labels) tuple for RetinaNet. + +T.-Y. Lin, P. Goyal, R. Girshick, K. He, and P. Dollar +Focal Loss for Dense Object Detection. arXiv:1708.02002 +""" + +import tensorflow.compat.v2 as tf + +from official.vision.detection.dataloader import anchor +from official.vision.detection.dataloader import mode_keys as ModeKeys +from official.vision.detection.dataloader import tf_example_decoder +from official.vision.detection.utils import autoaugment_utils +from official.vision.detection.utils import box_utils +from official.vision.detection.utils import input_utils + + +def process_source_id(source_id): + """Processes source_id to the right format.""" + if source_id.dtype == tf.string: + source_id = tf.cast(tf.strings.to_number(source_id), tf.int32) + with tf.control_dependencies([source_id]): + source_id = tf.cond( + pred=tf.equal(tf.size(input=source_id), 0), + true_fn=lambda: tf.cast(tf.constant(-1), tf.int32), + false_fn=lambda: tf.identity(source_id)) + return source_id + + +def pad_groundtruths_to_fixed_size(gt, n): + """Pads the first dimension of groundtruths labels to the fixed size.""" + gt['boxes'] = input_utils.pad_to_fixed_size(gt['boxes'], n, -1) + gt['is_crowds'] = input_utils.pad_to_fixed_size(gt['is_crowds'], n, 0) + gt['areas'] = input_utils.pad_to_fixed_size(gt['areas'], n, -1) + gt['classes'] = input_utils.pad_to_fixed_size(gt['classes'], n, -1) + return gt + + +class Parser(object): + """Parser to parse an image and its annotations into a dictionary of tensors.""" + + def __init__(self, + output_size, + min_level, + max_level, + num_scales, + aspect_ratios, + anchor_size, + match_threshold=0.5, + unmatched_threshold=0.5, + aug_rand_hflip=False, + aug_scale_min=1.0, + aug_scale_max=1.0, + use_autoaugment=False, + autoaugment_policy_name='v0', + skip_crowd_during_training=True, + max_num_instances=100, + use_bfloat16=True, + mode=None): + """Initializes parameters for parsing annotations in the dataset. + + Args: + output_size: `Tensor` or `list` for [height, width] of output image. The + output_size should be divided by the largest feature stride 2^max_level. + min_level: `int` number of minimum level of the output feature pyramid. + max_level: `int` number of maximum level of the output feature pyramid. + num_scales: `int` number representing intermediate scales added + on each level. For instances, num_scales=2 adds one additional + intermediate anchor scales [2^0, 2^0.5] on each level. + aspect_ratios: `list` of float numbers representing the aspect raito + anchors added on each level. The number indicates the ratio of width to + height. For instances, aspect_ratios=[1.0, 2.0, 0.5] adds three anchors + on each scale level. + anchor_size: `float` number representing the scale of size of the base + anchor to the feature stride 2^level. + match_threshold: `float` number between 0 and 1 representing the + lower-bound threshold to assign positive labels for anchors. An anchor + with a score over the threshold is labeled positive. + unmatched_threshold: `float` number between 0 and 1 representing the + upper-bound threshold to assign negative labels for anchors. An anchor + with a score below the threshold is labeled negative. + aug_rand_hflip: `bool`, if True, augment training with random + horizontal flip. + aug_scale_min: `float`, the minimum scale applied to `output_size` for + data augmentation during training. + aug_scale_max: `float`, the maximum scale applied to `output_size` for + data augmentation during training. + use_autoaugment: `bool`, if True, use the AutoAugment augmentation policy + during training. + autoaugment_policy_name: `string` that specifies the name of the + AutoAugment policy that will be used during training. + skip_crowd_during_training: `bool`, if True, skip annotations labeled with + `is_crowd` equals to 1. + max_num_instances: `int` number of maximum number of instances in an + image. The groundtruth data will be padded to `max_num_instances`. + use_bfloat16: `bool`, if True, cast output image to tf.bfloat16. + mode: a ModeKeys. Specifies if this is training, evaluation, prediction + or prediction with groundtruths in the outputs. + """ + self._mode = mode + self._max_num_instances = max_num_instances + self._skip_crowd_during_training = skip_crowd_during_training + self._is_training = (mode == ModeKeys.TRAIN) + + self._example_decoder = tf_example_decoder.TfExampleDecoder( + include_mask=False) + + # Anchor. + self._output_size = output_size + self._min_level = min_level + self._max_level = max_level + self._num_scales = num_scales + self._aspect_ratios = aspect_ratios + self._anchor_size = anchor_size + self._match_threshold = match_threshold + self._unmatched_threshold = unmatched_threshold + + # Data augmentation. + self._aug_rand_hflip = aug_rand_hflip + self._aug_scale_min = aug_scale_min + self._aug_scale_max = aug_scale_max + + # Data Augmentation with AutoAugment. + self._use_autoaugment = use_autoaugment + self._autoaugment_policy_name = autoaugment_policy_name + + # Device. + self._use_bfloat16 = use_bfloat16 + + # Data is parsed depending on the model Modekey. + if mode == ModeKeys.TRAIN: + self._parse_fn = self._parse_train_data + elif mode == ModeKeys.EVAL: + self._parse_fn = self._parse_eval_data + elif mode == ModeKeys.PREDICT or mode == ModeKeys.PREDICT_WITH_GT: + self._parse_fn = self._parse_predict_data + else: + raise ValueError('mode is not defined.') + + def __call__(self, value): + """Parses data to an image and associated training labels. + + Args: + value: a string tensor holding a serialized tf.Example proto. + + Returns: + image: image tensor that is preproessed to have normalized value and + dimension [output_size[0], output_size[1], 3] + labels: + cls_targets: ordered dictionary with keys + [min_level, min_level+1, ..., max_level]. The values are tensor with + shape [height_l, width_l, anchors_per_location]. The height_l and + width_l represent the dimension of class logits at l-th level. + box_targets: ordered dictionary with keys + [min_level, min_level+1, ..., max_level]. The values are tensor with + shape [height_l, width_l, anchors_per_location * 4]. The height_l and + width_l represent the dimension of bounding box regression output at + l-th level. + num_positives: number of positive anchors in the image. + anchor_boxes: ordered dictionary with keys + [min_level, min_level+1, ..., max_level]. The values are tensor with + shape [height_l, width_l, 4] representing anchor boxes at each level. + image_info: a 2D `Tensor` that encodes the information of the image and + the applied preprocessing. It is in the format of + [[original_height, original_width], [scaled_height, scaled_width], + [y_scale, x_scale], [y_offset, x_offset]]. + groundtruths: + source_id: source image id. Default value -1 if the source id is empty + in the groundtruth annotation. + boxes: groundtruth bounding box annotations. The box is represented in + [y1, x1, y2, x2] format. The tennsor is padded with -1 to the fixed + dimension [self._max_num_instances, 4]. + classes: groundtruth classes annotations. The tennsor is padded with + -1 to the fixed dimension [self._max_num_instances]. + areas: groundtruth areas annotations. The tennsor is padded with -1 + to the fixed dimension [self._max_num_instances]. + is_crowds: groundtruth annotations to indicate if an annotation + represents a group of instances by value {0, 1}. The tennsor is + padded with 0 to the fixed dimension [self._max_num_instances]. + """ + with tf.name_scope('parser'): + data = self._example_decoder.decode(value) + return self._parse_fn(data) + + def _parse_train_data(self, data): + """Parses data for training and evaluation.""" + classes = data['groundtruth_classes'] + boxes = data['groundtruth_boxes'] + is_crowds = data['groundtruth_is_crowd'] + # Skips annotations with `is_crowd` = True. + if self._skip_crowd_during_training and self._is_training: + num_groundtrtuhs = tf.shape(input=classes)[0] + with tf.control_dependencies([num_groundtrtuhs, is_crowds]): + indices = tf.cond( + pred=tf.greater(tf.size(input=is_crowds), 0), + true_fn=lambda: tf.where(tf.logical_not(is_crowds))[:, 0], + false_fn=lambda: tf.cast(tf.range(num_groundtrtuhs), tf.int64)) + classes = tf.gather(classes, indices) + boxes = tf.gather(boxes, indices) + + # Gets original image and its size. + image = data['image'] + + # NOTE: The autoaugment method works best when used alongside the standard + # horizontal flipping of images along with size jittering and normalization. + if self._use_autoaugment: + image, boxes = autoaugment_utils.distort_image_with_autoaugment( + image, boxes, self._autoaugment_policy_name) + + image_shape = tf.shape(input=image)[0:2] + + # Normalizes image with mean and std pixel values. + image = input_utils.normalize_image(image) + + # Flips image randomly during training. + if self._aug_rand_hflip: + image, boxes = input_utils.random_horizontal_flip(image, boxes) + + # Converts boxes from normalized coordinates to pixel coordinates. + boxes = box_utils.denormalize_boxes(boxes, image_shape) + + # Resizes and crops image. + image, image_info = input_utils.resize_and_crop_image( + image, + self._output_size, + padded_size=input_utils.compute_padded_size( + self._output_size, 2 ** self._max_level), + aug_scale_min=self._aug_scale_min, + aug_scale_max=self._aug_scale_max) + image_height, image_width, _ = image.get_shape().as_list() + + # Resizes and crops boxes. + image_scale = image_info[2, :] + offset = image_info[3, :] + boxes = input_utils.resize_and_crop_boxes( + boxes, image_scale, image_info[1, :], offset) + # Filters out ground truth boxes that are all zeros. + indices = box_utils.get_non_empty_box_indices(boxes) + boxes = tf.gather(boxes, indices) + classes = tf.gather(classes, indices) + + # Assigns anchors. + input_anchor = anchor.Anchor( + self._min_level, self._max_level, self._num_scales, + self._aspect_ratios, self._anchor_size, (image_height, image_width)) + anchor_labeler = anchor.AnchorLabeler( + input_anchor, self._match_threshold, self._unmatched_threshold) + (cls_targets, box_targets, num_positives) = anchor_labeler.label_anchors( + boxes, + tf.cast(tf.expand_dims(classes, axis=1), tf.float32)) + + # If bfloat16 is used, casts input image to tf.bfloat16. + if self._use_bfloat16: + image = tf.cast(image, dtype=tf.bfloat16) + + # Packs labels for model_fn outputs. + labels = { + 'cls_targets': cls_targets, + 'box_targets': box_targets, + 'anchor_boxes': input_anchor.multilevel_boxes, + 'num_positives': num_positives, + 'image_info': image_info, + } + return image, labels + + def _parse_eval_data(self, data): + """Parses data for training and evaluation.""" + groundtruths = {} + classes = data['groundtruth_classes'] + boxes = data['groundtruth_boxes'] + + # Gets original image and its size. + image = data['image'] + image_shape = tf.shape(input=image)[0:2] + + # Normalizes image with mean and std pixel values. + image = input_utils.normalize_image(image) + + # Converts boxes from normalized coordinates to pixel coordinates. + boxes = box_utils.denormalize_boxes(boxes, image_shape) + + # Resizes and crops image. + image, image_info = input_utils.resize_and_crop_image( + image, + self._output_size, + padded_size=input_utils.compute_padded_size( + self._output_size, 2 ** self._max_level), + aug_scale_min=1.0, + aug_scale_max=1.0) + image_height, image_width, _ = image.get_shape().as_list() + + # Resizes and crops boxes. + image_scale = image_info[2, :] + offset = image_info[3, :] + boxes = input_utils.resize_and_crop_boxes( + boxes, image_scale, image_info[1, :], offset) + # Filters out ground truth boxes that are all zeros. + indices = box_utils.get_non_empty_box_indices(boxes) + boxes = tf.gather(boxes, indices) + classes = tf.gather(classes, indices) + + # Assigns anchors. + input_anchor = anchor.Anchor( + self._min_level, self._max_level, self._num_scales, + self._aspect_ratios, self._anchor_size, (image_height, image_width)) + anchor_labeler = anchor.AnchorLabeler( + input_anchor, self._match_threshold, self._unmatched_threshold) + (cls_targets, box_targets, num_positives) = anchor_labeler.label_anchors( + boxes, + tf.cast(tf.expand_dims(classes, axis=1), tf.float32)) + + # If bfloat16 is used, casts input image to tf.bfloat16. + if self._use_bfloat16: + image = tf.cast(image, dtype=tf.bfloat16) + + # Sets up groundtruth data for evaluation. + groundtruths = { + 'source_id': data['source_id'], + 'num_groundtrtuhs': tf.shape(data['groundtruth_classes']), + 'image_info': image_info, + 'boxes': box_utils.denormalize_boxes( + data['groundtruth_boxes'], image_shape), + 'classes': data['groundtruth_classes'], + 'areas': data['groundtruth_area'], + 'is_crowds': tf.cast(data['groundtruth_is_crowd'], tf.int32), + } + groundtruths['source_id'] = process_source_id(groundtruths['source_id']) + groundtruths = pad_groundtruths_to_fixed_size( + groundtruths, self._max_num_instances) + + # Packs labels for model_fn outputs. + labels = { + 'cls_targets': cls_targets, + 'box_targets': box_targets, + 'anchor_boxes': input_anchor.multilevel_boxes, + 'num_positives': num_positives, + 'image_info': image_info, + 'groundtruths': groundtruths, + } + return image, labels + + def _parse_predict_data(self, data): + """Parses data for prediction.""" + # Gets original image and its size. + image = data['image'] + image_shape = tf.shape(input=image)[0:2] + + # Normalizes image with mean and std pixel values. + image = input_utils.normalize_image(image) + + # Resizes and crops image. + image, image_info = input_utils.resize_and_crop_image( + image, + self._output_size, + padded_size=input_utils.compute_padded_size( + self._output_size, 2 ** self._max_level), + aug_scale_min=1.0, + aug_scale_max=1.0) + image_height, image_width, _ = image.get_shape().as_list() + + # If bfloat16 is used, casts input image to tf.bfloat16. + if self._use_bfloat16: + image = tf.cast(image, dtype=tf.bfloat16) + + # Compute Anchor boxes. + input_anchor = anchor.Anchor( + self._min_level, self._max_level, self._num_scales, + self._aspect_ratios, self._anchor_size, (image_height, image_width)) + + labels = { + 'anchor_boxes': input_anchor.multilevel_boxes, + 'image_info': image_info, + } + # If mode is PREDICT_WITH_GT, returns groundtruths and training targets + # in labels. + if self._mode == ModeKeys.PREDICT_WITH_GT: + # Converts boxes from normalized coordinates to pixel coordinates. + boxes = box_utils.denormalize_boxes( + data['groundtruth_boxes'], image_shape) + groundtruths = { + 'source_id': data['source_id'], + 'num_detections': tf.shape(data['groundtruth_classes']), + 'boxes': boxes, + 'classes': data['groundtruth_classes'], + 'areas': data['groundtruth_area'], + 'is_crowds': tf.cast(data['groundtruth_is_crowd'], tf.int32), + } + groundtruths['source_id'] = process_source_id(groundtruths['source_id']) + groundtruths = pad_groundtruths_to_fixed_size( + groundtruths, self._max_num_instances) + labels['groundtruths'] = groundtruths + + # Computes training objective for evaluation loss. + classes = data['groundtruth_classes'] + + image_scale = image_info[2, :] + offset = image_info[3, :] + boxes = input_utils.resize_and_crop_boxes( + boxes, image_scale, image_info[1, :], offset) + # Filters out ground truth boxes that are all zeros. + indices = box_utils.get_non_empty_box_indices(boxes) + boxes = tf.gather(boxes, indices) + + # Assigns anchors. + anchor_labeler = anchor.AnchorLabeler( + input_anchor, self._match_threshold, self._unmatched_threshold) + (cls_targets, box_targets, num_positives) = anchor_labeler.label_anchors( + boxes, + tf.cast(tf.expand_dims(classes, axis=1), tf.float32)) + labels['cls_targets'] = cls_targets + labels['box_targets'] = box_targets + labels['num_positives'] = num_positives + return image, labels diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/shapemask_parser.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/shapemask_parser.py new file mode 100644 index 0000000..f9c1c24 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/shapemask_parser.py @@ -0,0 +1,478 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Data parser and processing. + +Parse image and ground truths in a dataset to training targets and package them +into (image, labels) tuple for ShapeMask. + +Weicheng Kuo, Anelia Angelova, Jitendra Malik, Tsung-Yi Lin +ShapeMask: Learning to Segment Novel Objects by Refining Shape Priors. +arXiv:1904.03239. +""" + +import tensorflow.compat.v2 as tf + +from official.vision.detection.dataloader import anchor +from official.vision.detection.dataloader import mode_keys as ModeKeys +from official.vision.detection.dataloader import tf_example_decoder +from official.vision.detection.utils import box_utils +from official.vision.detection.utils import class_utils +from official.vision.detection.utils import dataloader_utils +from official.vision.detection.utils import input_utils + + +class Parser(object): + """Parser to parse an image and its annotations into a dictionary of tensors.""" + + def __init__(self, + output_size, + min_level, + max_level, + num_scales, + aspect_ratios, + anchor_size, + use_category=True, + outer_box_scale=1.0, + box_jitter_scale=0.025, + num_sampled_masks=8, + mask_crop_size=32, + mask_min_level=3, + mask_max_level=5, + upsample_factor=4, + match_threshold=0.5, + unmatched_threshold=0.5, + aug_rand_hflip=False, + aug_scale_min=1.0, + aug_scale_max=1.0, + skip_crowd_during_training=True, + max_num_instances=100, + use_bfloat16=True, + mask_train_class='all', + mode=None): + """Initializes parameters for parsing annotations in the dataset. + + Args: + output_size: `Tensor` or `list` for [height, width] of output image. The + output_size should be divided by the largest feature stride 2^max_level. + min_level: `int` number of minimum level of the output feature pyramid. + max_level: `int` number of maximum level of the output feature pyramid. + num_scales: `int` number representing intermediate scales added + on each level. For instances, num_scales=2 adds one additional + intermediate anchor scales [2^0, 2^0.5] on each level. + aspect_ratios: `list` of float numbers representing the aspect raito + anchors added on each level. The number indicates the ratio of width to + height. For instances, aspect_ratios=[1.0, 2.0, 0.5] adds three anchors + on each scale level. + anchor_size: `float` number representing the scale of size of the base + anchor to the feature stride 2^level. + use_category: if `False`, treat all object in all classes in one + foreground category. + outer_box_scale: `float` number in a range of [1.0, inf) representing + the scale from object box to outer box. The mask branch predicts + instance mask enclosed in outer box. + box_jitter_scale: `float` number representing the noise magnitude to + jitter the training groundtruth boxes for mask branch. + num_sampled_masks: `int` number of sampled masks for training. + mask_crop_size: `list` for [height, width] of output training masks. + mask_min_level: `int` number indicating the minimum feature level to + obtain instance features. + mask_max_level: `int` number indicating the maximum feature level to + obtain instance features. + upsample_factor: `int` factor of upsampling the fine mask predictions. + match_threshold: `float` number between 0 and 1 representing the + lower-bound threshold to assign positive labels for anchors. An anchor + with a score over the threshold is labeled positive. + unmatched_threshold: `float` number between 0 and 1 representing the + upper-bound threshold to assign negative labels for anchors. An anchor + with a score below the threshold is labeled negative. + aug_rand_hflip: `bool`, if True, augment training with random + horizontal flip. + aug_scale_min: `float`, the minimum scale applied to `output_size` for + data augmentation during training. + aug_scale_max: `float`, the maximum scale applied to `output_size` for + data augmentation during training. + skip_crowd_during_training: `bool`, if True, skip annotations labeled with + `is_crowd` equals to 1. + max_num_instances: `int` number of maximum number of instances in an + image. The groundtruth data will be padded to `max_num_instances`. + use_bfloat16: `bool`, if True, cast output image to tf.bfloat16. + mask_train_class: a string of experiment mode: `all`, `voc` or `nonvoc`. + mode: a ModeKeys. Specifies if this is training, evaluation, prediction + or prediction with groundtruths in the outputs. + """ + self._mode = mode + self._mask_train_class = mask_train_class + self._max_num_instances = max_num_instances + self._skip_crowd_during_training = skip_crowd_during_training + self._is_training = (mode == ModeKeys.TRAIN) + + self._example_decoder = tf_example_decoder.TfExampleDecoder( + include_mask=True) + + # Anchor. + self._output_size = output_size + self._min_level = min_level + self._max_level = max_level + self._num_scales = num_scales + self._aspect_ratios = aspect_ratios + self._anchor_size = anchor_size + self._match_threshold = match_threshold + self._unmatched_threshold = unmatched_threshold + + # Data augmentation. + self._aug_rand_hflip = aug_rand_hflip + self._aug_scale_min = aug_scale_min + self._aug_scale_max = aug_scale_max + + # Device. + self._use_bfloat16 = use_bfloat16 + + # ShapeMask specific. + # Control of which category to use. + self._use_category = use_category + self._num_sampled_masks = num_sampled_masks + self._mask_crop_size = mask_crop_size + self._mask_min_level = mask_min_level + self._mask_max_level = mask_max_level + self._outer_box_scale = outer_box_scale + self._box_jitter_scale = box_jitter_scale + self._up_sample_factor = upsample_factor + + # Data is parsed depending on the model Modekey. + if mode == ModeKeys.TRAIN: + self._parse_fn = self._parse_train_data + elif mode == ModeKeys.EVAL: + self._parse_fn = self._parse_eval_data + elif mode == ModeKeys.PREDICT or mode == ModeKeys.PREDICT_WITH_GT: + self._parse_fn = self._parse_predict_data + else: + raise ValueError('mode is not defined.') + + def __call__(self, value): + """Parses data to an image and associated training labels. + + Args: + value: a string tensor holding a serialized tf.Example proto. + + Returns: + image: image tensor that is preproessed to have normalized value and + dimension [output_size[0], output_size[1], 3] + labels: + cls_targets: ordered dictionary with keys + [min_level, min_level+1, ..., max_level]. The values are tensor with + shape [height_l, width_l, anchors_per_location]. The height_l and + width_l represent the dimension of class logits at l-th level. + box_targets: ordered dictionary with keys + [min_level, min_level+1, ..., max_level]. The values are tensor with + shape [height_l, width_l, anchors_per_location * 4]. The height_l and + width_l represent the dimension of bounding box regression output at + l-th level. + num_positives: number of positive anchors in the image. + anchor_boxes: ordered dictionary with keys + [min_level, min_level+1, ..., max_level]. The values are tensor with + shape [height_l, width_l, 4] representing anchor boxes at each level. + image_scale: 2D float `Tensor` representing scale factors that apply + to [height, width] of input image. + mask_boxes: sampled boxes that tightly enclose the training masks. The + box is represented in [y1, x1, y2, x2] format. The tensor is sampled + to the fixed dimension [self._num_sampled_masks, 4]. + mask_outer_boxes: loose box that enclose sampled tight box. The + box is represented in [y1, x1, y2, x2] format. The tensor is sampled + to the fixed dimension [self._num_sampled_masks, 4]. + mask_targets: training binary mask targets. The tensor has shape + [self._num_sampled_masks, self._mask_crop_size, self._mask_crop_size]. + mask_classes: the class ids of sampled training masks. The tensor has + shape [self._num_sampled_masks]. + mask_is_valid: the binary tensor to indicate if the sampled masks are + valide. The sampled masks are invalid when no mask annotations are + included in the image. The tensor has shape [1]. + groundtruths: + source_id: source image id. Default value -1 if the source id is empty + in the groundtruth annotation. + boxes: groundtruth bounding box annotations. The box is represented in + [y1, x1, y2, x2] format. The tensor is padded with -1 to the fixed + dimension [self._max_num_instances, 4]. + classes: groundtruth classes annotations. The tensor is padded with + -1 to the fixed dimension [self._max_num_instances]. + areas: groundtruth areas annotations. The tensor is padded with -1 + to the fixed dimension [self._max_num_instances]. + is_crowds: groundtruth annotations to indicate if an annotation + represents a group of instances by value {0, 1}. The tensor is + padded with 0 to the fixed dimension [self._max_num_instances]. + """ + with tf.name_scope('parser'): + data = self._example_decoder.decode(value) + return self._parse_fn(data) + + def _parse_train_data(self, data): + """Parse data for ShapeMask training.""" + classes = data['groundtruth_classes'] + boxes = data['groundtruth_boxes'] + masks = data['groundtruth_instance_masks'] + is_crowds = data['groundtruth_is_crowd'] + # Skips annotations with `is_crowd` = True. + if self._skip_crowd_during_training and self._is_training: + num_groundtrtuhs = tf.shape(classes)[0] + with tf.control_dependencies([num_groundtrtuhs, is_crowds]): + indices = tf.cond( + tf.greater(tf.size(is_crowds), 0), + lambda: tf.where(tf.logical_not(is_crowds))[:, 0], + lambda: tf.cast(tf.range(num_groundtrtuhs), tf.int64)) + classes = tf.gather(classes, indices) + boxes = tf.gather(boxes, indices) + masks = tf.gather(masks, indices) + + # Gets original image and its size. + image = data['image'] + image_shape = tf.shape(image)[0:2] + + # If not using category, makes all categories with id = 0. + if not self._use_category: + classes = tf.cast(tf.greater(classes, 0), dtype=tf.float32) + + # Normalizes image with mean and std pixel values. + image = input_utils.normalize_image(image) + + # Flips image randomly during training. + if self._aug_rand_hflip: + image, boxes, masks = input_utils.random_horizontal_flip( + image, boxes, masks) + + # Converts boxes from normalized coordinates to pixel coordinates. + boxes = box_utils.denormalize_boxes(boxes, image_shape) + + # Resizes and crops image. + image, image_info = input_utils.resize_and_crop_image( + image, + self._output_size, + self._output_size, + aug_scale_min=self._aug_scale_min, + aug_scale_max=self._aug_scale_max) + image_scale = image_info[2, :] + offset = image_info[3, :] + + # Resizes and crops boxes and masks. + boxes = input_utils.resize_and_crop_boxes( + boxes, image_scale, image_info[1, :], offset) + + # Filters out ground truth boxes that are all zeros. + indices = box_utils.get_non_empty_box_indices(boxes) + boxes = tf.gather(boxes, indices) + classes = tf.gather(classes, indices) + masks = tf.gather(masks, indices) + + # Assigns anchors. + input_anchor = anchor.Anchor( + self._min_level, self._max_level, self._num_scales, + self._aspect_ratios, self._anchor_size, self._output_size) + anchor_labeler = anchor.AnchorLabeler( + input_anchor, self._match_threshold, self._unmatched_threshold) + (cls_targets, + box_targets, + num_positives) = anchor_labeler.label_anchors( + boxes, + tf.cast(tf.expand_dims(classes, axis=1), tf.float32)) + + # Sample groundtruth masks/boxes/classes for mask branch. + num_masks = tf.shape(masks)[0] + mask_shape = tf.shape(masks)[1:3] + + # Pad sampled boxes/masks/classes to a constant batch size. + padded_boxes = input_utils.pad_to_fixed_size(boxes, self._num_sampled_masks) + padded_classes = input_utils.pad_to_fixed_size( + classes, self._num_sampled_masks) + padded_masks = input_utils.pad_to_fixed_size(masks, self._num_sampled_masks) + + # Randomly sample groundtruth masks for mask branch training. For the image + # without groundtruth masks, it will sample the dummy padded tensors. + rand_indices = tf.random.shuffle( + tf.range(tf.maximum(num_masks, self._num_sampled_masks))) + rand_indices = tf.math.mod(rand_indices, tf.maximum(num_masks, 1)) + rand_indices = rand_indices[0:self._num_sampled_masks] + rand_indices = tf.reshape(rand_indices, [self._num_sampled_masks]) + + sampled_boxes = tf.gather(padded_boxes, rand_indices) + sampled_classes = tf.gather(padded_classes, rand_indices) + sampled_masks = tf.gather(padded_masks, rand_indices) + # Jitter the sampled boxes to mimic the noisy detections. + sampled_boxes = box_utils.jitter_boxes( + sampled_boxes, noise_scale=self._box_jitter_scale) + sampled_boxes = box_utils.clip_boxes(sampled_boxes, self._output_size) + # Compute mask targets in feature crop. A feature crop fully contains a + # sampled box. + mask_outer_boxes = box_utils.compute_outer_boxes( + sampled_boxes, tf.shape(image)[0:2], scale=self._outer_box_scale) + mask_outer_boxes = box_utils.clip_boxes(mask_outer_boxes, self._output_size) + # Compensate the offset of mask_outer_boxes to map it back to original image + # scale. + mask_outer_boxes_ori = mask_outer_boxes + mask_outer_boxes_ori += tf.tile(tf.expand_dims(offset, axis=0), [1, 2]) + mask_outer_boxes_ori /= tf.tile(tf.expand_dims(image_scale, axis=0), [1, 2]) + norm_mask_outer_boxes_ori = box_utils.normalize_boxes( + mask_outer_boxes_ori, mask_shape) + + # Set sampled_masks shape to [batch_size, height, width, 1]. + sampled_masks = tf.cast(tf.expand_dims(sampled_masks, axis=-1), tf.float32) + mask_targets = tf.image.crop_and_resize( + sampled_masks, + norm_mask_outer_boxes_ori, + box_indices=tf.range(self._num_sampled_masks), + crop_size=[self._mask_crop_size, self._mask_crop_size], + method='bilinear', + extrapolation_value=0, + name='train_mask_targets') + mask_targets = tf.where(tf.greater_equal(mask_targets, 0.5), + tf.ones_like(mask_targets), + tf.zeros_like(mask_targets)) + mask_targets = tf.squeeze(mask_targets, axis=-1) + if self._up_sample_factor > 1: + fine_mask_targets = tf.image.crop_and_resize( + sampled_masks, + norm_mask_outer_boxes_ori, + box_indices=tf.range(self._num_sampled_masks), + crop_size=[ + self._mask_crop_size * self._up_sample_factor, + self._mask_crop_size * self._up_sample_factor + ], + method='bilinear', + extrapolation_value=0, + name='train_mask_targets') + fine_mask_targets = tf.where( + tf.greater_equal(fine_mask_targets, 0.5), + tf.ones_like(fine_mask_targets), tf.zeros_like(fine_mask_targets)) + fine_mask_targets = tf.squeeze(fine_mask_targets, axis=-1) + else: + fine_mask_targets = mask_targets + + # If bfloat16 is used, casts input image to tf.bfloat16. + if self._use_bfloat16: + image = tf.cast(image, dtype=tf.bfloat16) + + valid_image = tf.cast(tf.not_equal(num_masks, 0), tf.int32) + if self._mask_train_class == 'all': + mask_is_valid = valid_image * tf.ones_like(sampled_classes, tf.int32) + else: + # Get the intersection of sampled classes with training splits. + mask_valid_classes = tf.cast( + tf.expand_dims( + class_utils.coco_split_class_ids(self._mask_train_class), 1), + sampled_classes.dtype) + match = tf.reduce_any( + tf.equal(tf.expand_dims(sampled_classes, 0), mask_valid_classes), 0) + mask_is_valid = valid_image * tf.cast(match, tf.int32) + + # Packs labels for model_fn outputs. + labels = { + 'cls_targets': cls_targets, + 'box_targets': box_targets, + 'anchor_boxes': input_anchor.multilevel_boxes, + 'num_positives': num_positives, + 'image_info': image_info, + # For ShapeMask. + 'mask_boxes': sampled_boxes, + 'mask_outer_boxes': mask_outer_boxes, + 'mask_targets': mask_targets, + 'fine_mask_targets': fine_mask_targets, + 'mask_classes': sampled_classes, + 'mask_is_valid': mask_is_valid, + } + return image, labels + + def _parse_predict_data(self, data): + """Parse data for ShapeMask training.""" + classes = data['groundtruth_classes'] + boxes = data['groundtruth_boxes'] + masks = data['groundtruth_instance_masks'] + + # Gets original image and its size. + image = data['image'] + image_shape = tf.shape(image)[0:2] + + # If not using category, makes all categories with id = 0. + if not self._use_category: + classes = tf.cast(tf.greater(classes, 0), dtype=tf.float32) + + # Normalizes image with mean and std pixel values. + image = input_utils.normalize_image(image) + + # Converts boxes from normalized coordinates to pixel coordinates. + boxes = box_utils.denormalize_boxes(boxes, image_shape) + + # Resizes and crops image. + image, image_info = input_utils.resize_and_crop_image( + image, + self._output_size, + self._output_size, + aug_scale_min=1.0, + aug_scale_max=1.0) + image_scale = image_info[2, :] + offset = image_info[3, :] + + # Resizes and crops boxes and masks. + boxes = input_utils.resize_and_crop_boxes( + boxes, image_scale, image_info[1, :], offset) + masks = input_utils.resize_and_crop_masks( + tf.expand_dims(masks, axis=-1), image_scale, self._output_size, offset) + + # Filters out ground truth boxes that are all zeros. + indices = box_utils.get_non_empty_box_indices(boxes) + boxes = tf.gather(boxes, indices) + classes = tf.gather(classes, indices) + + # Assigns anchors. + input_anchor = anchor.Anchor( + self._min_level, self._max_level, self._num_scales, + self._aspect_ratios, self._anchor_size, self._output_size) + anchor_labeler = anchor.AnchorLabeler( + input_anchor, self._match_threshold, self._unmatched_threshold) + + # If bfloat16 is used, casts input image to tf.bfloat16. + if self._use_bfloat16: + image = tf.cast(image, dtype=tf.bfloat16) + + labels = { + 'anchor_boxes': input_anchor.multilevel_boxes, + 'image_info': image_info, + } + if self._mode == ModeKeys.PREDICT_WITH_GT: + # Converts boxes from normalized coordinates to pixel coordinates. + groundtruths = { + 'source_id': data['source_id'], + 'num_detections': tf.shape(data['groundtruth_classes']), + 'boxes': box_utils.denormalize_boxes( + data['groundtruth_boxes'], image_shape), + 'classes': data['groundtruth_classes'], + # 'masks': tf.squeeze(masks, axis=-1), + 'areas': data['groundtruth_area'], + 'is_crowds': tf.cast(data['groundtruth_is_crowd'], tf.int32), + } + groundtruths['source_id'] = dataloader_utils.process_source_id( + groundtruths['source_id']) + groundtruths = dataloader_utils.pad_groundtruths_to_fixed_size( + groundtruths, self._max_num_instances) + # Computes training labels. + (cls_targets, + box_targets, + num_positives) = anchor_labeler.label_anchors( + boxes, + tf.cast(tf.expand_dims(classes, axis=1), tf.float32)) + # Packs labels for model_fn outputs. + labels.update({ + 'cls_targets': cls_targets, + 'box_targets': box_targets, + 'num_positives': num_positives, + 'groundtruths': groundtruths, + }) + return image, labels diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/tf_example_decoder.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/tf_example_decoder.py new file mode 100644 index 0000000..a4b8124 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/dataloader/tf_example_decoder.py @@ -0,0 +1,156 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Tensorflow Example proto decoder for object detection. + +A decoder to decode string tensors containing serialized tensorflow.Example +protos for object detection. +""" +import tensorflow.compat.v2 as tf + + +class TfExampleDecoder(object): + """Tensorflow Example proto decoder.""" + + def __init__(self, include_mask=False): + self._include_mask = include_mask + self._keys_to_features = { + 'image/encoded': + tf.io.FixedLenFeature((), tf.string), + 'image/source_id': + tf.io.FixedLenFeature((), tf.string), + 'image/height': + tf.io.FixedLenFeature((), tf.int64), + 'image/width': + tf.io.FixedLenFeature((), tf.int64), + 'image/object/bbox/xmin': + tf.io.VarLenFeature(tf.float32), + 'image/object/bbox/xmax': + tf.io.VarLenFeature(tf.float32), + 'image/object/bbox/ymin': + tf.io.VarLenFeature(tf.float32), + 'image/object/bbox/ymax': + tf.io.VarLenFeature(tf.float32), + 'image/object/class/label': + tf.io.VarLenFeature(tf.int64), + 'image/object/area': + tf.io.VarLenFeature(tf.float32), + 'image/object/is_crowd': + tf.io.VarLenFeature(tf.int64), + } + if include_mask: + self._keys_to_features.update({ + 'image/object/mask': + tf.io.VarLenFeature(tf.string), + }) + + def _decode_image(self, parsed_tensors): + """Decodes the image and set its static shape.""" + image = tf.io.decode_image(parsed_tensors['image/encoded'], channels=3) + image.set_shape([None, None, 3]) + return image + + def _decode_boxes(self, parsed_tensors): + """Concat box coordinates in the format of [ymin, xmin, ymax, xmax].""" + xmin = parsed_tensors['image/object/bbox/xmin'] + xmax = parsed_tensors['image/object/bbox/xmax'] + ymin = parsed_tensors['image/object/bbox/ymin'] + ymax = parsed_tensors['image/object/bbox/ymax'] + return tf.stack([ymin, xmin, ymax, xmax], axis=-1) + + def _decode_masks(self, parsed_tensors): + """Decode a set of PNG masks to the tf.float32 tensors.""" + def _decode_png_mask(png_bytes): + mask = tf.squeeze( + tf.io.decode_png(png_bytes, channels=1, dtype=tf.uint8), axis=-1) + mask = tf.cast(mask, dtype=tf.float32) + mask.set_shape([None, None]) + return mask + + height = parsed_tensors['image/height'] + width = parsed_tensors['image/width'] + masks = parsed_tensors['image/object/mask'] + return tf.cond( + pred=tf.greater(tf.size(input=masks), 0), + true_fn=lambda: tf.map_fn(_decode_png_mask, masks, dtype=tf.float32), + false_fn=lambda: tf.zeros([0, height, width], dtype=tf.float32)) + + def _decode_areas(self, parsed_tensors): + xmin = parsed_tensors['image/object/bbox/xmin'] + xmax = parsed_tensors['image/object/bbox/xmax'] + ymin = parsed_tensors['image/object/bbox/ymin'] + ymax = parsed_tensors['image/object/bbox/ymax'] + return tf.cond( + tf.greater(tf.shape(parsed_tensors['image/object/area'])[0], 0), + lambda: parsed_tensors['image/object/area'], + lambda: (xmax - xmin) * (ymax - ymin)) + + def decode(self, serialized_example): + """Decode the serialized example. + + Args: + serialized_example: a single serialized tf.Example string. + + Returns: + decoded_tensors: a dictionary of tensors with the following fields: + - image: a uint8 tensor of shape [None, None, 3]. + - source_id: a string scalar tensor. + - height: an integer scalar tensor. + - width: an integer scalar tensor. + - groundtruth_classes: a int64 tensor of shape [None]. + - groundtruth_is_crowd: a bool tensor of shape [None]. + - groundtruth_area: a float32 tensor of shape [None]. + - groundtruth_boxes: a float32 tensor of shape [None, 4]. + - groundtruth_instance_masks: a float32 tensor of shape + [None, None, None]. + - groundtruth_instance_masks_png: a string tensor of shape [None]. + """ + parsed_tensors = tf.io.parse_single_example( + serialized=serialized_example, features=self._keys_to_features) + for k in parsed_tensors: + if isinstance(parsed_tensors[k], tf.SparseTensor): + if parsed_tensors[k].dtype == tf.string: + parsed_tensors[k] = tf.sparse.to_dense( + parsed_tensors[k], default_value='') + else: + parsed_tensors[k] = tf.sparse.to_dense( + parsed_tensors[k], default_value=0) + + image = self._decode_image(parsed_tensors) + boxes = self._decode_boxes(parsed_tensors) + areas = self._decode_areas(parsed_tensors) + is_crowds = tf.cond( + tf.greater(tf.shape(parsed_tensors['image/object/is_crowd'])[0], 0), + lambda: tf.cast(parsed_tensors['image/object/is_crowd'], dtype=tf.bool), + lambda: tf.zeros_like(parsed_tensors['image/object/class/label'], dtype=tf.bool)) # pylint: disable=line-too-long + if self._include_mask: + masks = self._decode_masks(parsed_tensors) + + decoded_tensors = { + 'image': image, + 'source_id': parsed_tensors['image/source_id'], + 'height': parsed_tensors['image/height'], + 'width': parsed_tensors['image/width'], + 'groundtruth_classes': parsed_tensors['image/object/class/label'], + 'groundtruth_is_crowd': is_crowds, + 'groundtruth_area': areas, + 'groundtruth_boxes': boxes, + } + if self._include_mask: + decoded_tensors.update({ + 'groundtruth_instance_masks': masks, + 'groundtruth_instance_masks_png': parsed_tensors['image/object/mask'], + }) + return decoded_tensors diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/evaluation/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/evaluation/__init__.py new file mode 100644 index 0000000..931c2ef --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/evaluation/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/evaluation/coco_evaluator.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/evaluation/coco_evaluator.py new file mode 100644 index 0000000..6413faa --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/evaluation/coco_evaluator.py @@ -0,0 +1,343 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The COCO-style evaluator. + +The following snippet demonstrates the use of interfaces: + + evaluator = COCOEvaluator(...) + for _ in range(num_evals): + for _ in range(num_batches_per_eval): + predictions, groundtruth = predictor.predict(...) # pop a batch. + evaluator.update(predictions, groundtruths) # aggregate internal stats. + evaluator.evaluate() # finish one full eval. + +See also: https://github.com/cocodataset/cocoapi/ +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import atexit +import tempfile +import numpy as np +from absl import logging +from pycocotools import cocoeval +import six +import tensorflow.compat.v2 as tf + +from official.vision.detection.evaluation import coco_utils +from official.vision.detection.utils import class_utils + + +class MetricWrapper(object): + # This is only a wrapper for COCO metric and works on for numpy array. So it + # doesn't inherit from tf.keras.layers.Layer or tf.keras.metrics.Metric. + + def __init__(self, evaluator): + self._evaluator = evaluator + + def update_state(self, y_true, y_pred): + labels = tf.nest.map_structure(lambda x: x.numpy(), y_true) + outputs = tf.nest.map_structure(lambda x: x.numpy(), y_pred) + groundtruths = {} + predictions = {} + for key, val in outputs.items(): + if isinstance(val, tuple): + val = np.concatenate(val) + predictions[key] = val + for key, val in labels.items(): + if isinstance(val, tuple): + val = np.concatenate(val) + groundtruths[key] = val + self._evaluator.update(predictions, groundtruths) + + def result(self): + return self._evaluator.evaluate() + + def reset_states(self): + return self._evaluator.reset() + + +class COCOEvaluator(object): + """COCO evaluation metric class.""" + + def __init__(self, annotation_file, include_mask, need_rescale_bboxes=True): + """Constructs COCO evaluation class. + + The class provides the interface to metrics_fn in TPUEstimator. The + _update_op() takes detections from each image and push them to + self.detections. The _evaluate() loads a JSON file in COCO annotation format + as the groundtruths and runs COCO evaluation. + + Args: + annotation_file: a JSON file that stores annotations of the eval dataset. + If `annotation_file` is None, groundtruth annotations will be loaded + from the dataloader. + include_mask: a boolean to indicate whether or not to include the mask + eval. + need_rescale_bboxes: If true bboxes in `predictions` will be rescaled back + to absolute values (`image_info` is needed in this case). + """ + if annotation_file: + if annotation_file.startswith('gs://'): + _, local_val_json = tempfile.mkstemp(suffix='.json') + tf.io.gfile.remove(local_val_json) + + tf.io.gfile.copy(annotation_file, local_val_json) + atexit.register(tf.io.gfile.remove, local_val_json) + else: + local_val_json = annotation_file + self._coco_gt = coco_utils.COCOWrapper( + eval_type=('mask' if include_mask else 'box'), + annotation_file=local_val_json) + self._annotation_file = annotation_file + self._include_mask = include_mask + self._metric_names = [ + 'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'ARmax1', 'ARmax10', + 'ARmax100', 'ARs', 'ARm', 'ARl' + ] + self._required_prediction_fields = [ + 'source_id', 'num_detections', 'detection_classes', 'detection_scores', + 'detection_boxes' + ] + self._need_rescale_bboxes = need_rescale_bboxes + if self._need_rescale_bboxes: + self._required_prediction_fields.append('image_info') + self._required_groundtruth_fields = [ + 'source_id', 'height', 'width', 'classes', 'boxes' + ] + if self._include_mask: + mask_metric_names = ['mask_' + x for x in self._metric_names] + self._metric_names.extend(mask_metric_names) + self._required_prediction_fields.extend(['detection_masks']) + self._required_groundtruth_fields.extend(['masks']) + + self.reset() + + def reset(self): + """Resets internal states for a fresh run.""" + self._predictions = {} + if not self._annotation_file: + self._groundtruths = {} + + def evaluate(self): + """Evaluates with detections from all images with COCO API. + + Returns: + coco_metric: float numpy array with shape [24] representing the + coco-style evaluation metrics (box and mask). + """ + if not self._annotation_file: + logging.info('Thre is no annotation_file in COCOEvaluator.') + gt_dataset = coco_utils.convert_groundtruths_to_coco_dataset( + self._groundtruths) + coco_gt = coco_utils.COCOWrapper( + eval_type=('mask' if self._include_mask else 'box'), + gt_dataset=gt_dataset) + else: + logging.info('Using annotation file: %s', self._annotation_file) + coco_gt = self._coco_gt + coco_predictions = coco_utils.convert_predictions_to_coco_annotations( + self._predictions) + coco_dt = coco_gt.loadRes(predictions=coco_predictions) + image_ids = [ann['image_id'] for ann in coco_predictions] + + coco_eval = cocoeval.COCOeval(coco_gt, coco_dt, iouType='bbox') + coco_eval.params.imgIds = image_ids + coco_eval.evaluate() + coco_eval.accumulate() + coco_eval.summarize() + coco_metrics = coco_eval.stats + + if self._include_mask: + mcoco_eval = cocoeval.COCOeval(coco_gt, coco_dt, iouType='segm') + mcoco_eval.params.imgIds = image_ids + mcoco_eval.evaluate() + mcoco_eval.accumulate() + mcoco_eval.summarize() + mask_coco_metrics = mcoco_eval.stats + + if self._include_mask: + metrics = np.hstack((coco_metrics, mask_coco_metrics)) + else: + metrics = coco_metrics + + # Cleans up the internal variables in order for a fresh eval next time. + self.reset() + + metrics_dict = {} + for i, name in enumerate(self._metric_names): + metrics_dict[name] = metrics[i].astype(np.float32) + return metrics_dict + + def _process_predictions(self, predictions): + image_scale = np.tile(predictions['image_info'][:, 2:3, :], (1, 1, 2)) + predictions['detection_boxes'] = ( + predictions['detection_boxes'].astype(np.float32)) + predictions['detection_boxes'] /= image_scale + if 'detection_outer_boxes' in predictions: + predictions['detection_outer_boxes'] = ( + predictions['detection_outer_boxes'].astype(np.float32)) + predictions['detection_outer_boxes'] /= image_scale + + def update(self, predictions, groundtruths=None): + """Update and aggregate detection results and groundtruth data. + + Args: + predictions: a dictionary of numpy arrays including the fields below. + See different parsers under `../dataloader` for more details. + Required fields: + - source_id: a numpy array of int or string of shape [batch_size]. + - image_info [if `need_rescale_bboxes` is True]: a numpy array of + float of shape [batch_size, 4, 2]. + - num_detections: a numpy array of + int of shape [batch_size]. + - detection_boxes: a numpy array of float of shape [batch_size, K, 4]. + - detection_classes: a numpy array of int of shape [batch_size, K]. + - detection_scores: a numpy array of float of shape [batch_size, K]. + Optional fields: + - detection_masks: a numpy array of float of shape + [batch_size, K, mask_height, mask_width]. + groundtruths: a dictionary of numpy arrays including the fields below. + See also different parsers under `../dataloader` for more details. + Required fields: + - source_id: a numpy array of int or string of shape [batch_size]. + - height: a numpy array of int of shape [batch_size]. + - width: a numpy array of int of shape [batch_size]. + - num_detections: a numpy array of int of shape [batch_size]. + - boxes: a numpy array of float of shape [batch_size, K, 4]. + - classes: a numpy array of int of shape [batch_size, K]. + Optional fields: + - is_crowds: a numpy array of int of shape [batch_size, K]. If the + field is absent, it is assumed that this instance is not crowd. + - areas: a numy array of float of shape [batch_size, K]. If the + field is absent, the area is calculated using either boxes or + masks depending on which one is available. + - masks: a numpy array of float of shape + [batch_size, K, mask_height, mask_width], + + Raises: + ValueError: if the required prediction or groundtruth fields are not + present in the incoming `predictions` or `groundtruths`. + """ + for k in self._required_prediction_fields: + if k not in predictions: + raise ValueError( + 'Missing the required key `{}` in predictions!'.format(k)) + if self._need_rescale_bboxes: + self._process_predictions(predictions) + for k, v in six.iteritems(predictions): + if k not in self._predictions: + self._predictions[k] = [v] + else: + self._predictions[k].append(v) + + if not self._annotation_file: + assert groundtruths + for k in self._required_groundtruth_fields: + if k not in groundtruths: + raise ValueError( + 'Missing the required key `{}` in groundtruths!'.format(k)) + for k, v in six.iteritems(groundtruths): + if k not in self._groundtruths: + self._groundtruths[k] = [v] + else: + self._groundtruths[k].append(v) + + +class ShapeMaskCOCOEvaluator(COCOEvaluator): + """COCO evaluation metric class for ShapeMask.""" + + def __init__(self, mask_eval_class, **kwargs): + """Constructs COCO evaluation class. + + The class provides the interface to metrics_fn in TPUEstimator. The + _update_op() takes detections from each image and push them to + self.detections. The _evaluate() loads a JSON file in COCO annotation format + as the groundtruths and runs COCO evaluation. + + Args: + mask_eval_class: the set of classes for mask evaluation. + **kwargs: other keyword arguments passed to the parent class initializer. + """ + super(ShapeMaskCOCOEvaluator, self).__init__(**kwargs) + self._mask_eval_class = mask_eval_class + self._eval_categories = class_utils.coco_split_class_ids(mask_eval_class) + if mask_eval_class != 'all': + self._metric_names = [ + x.replace('mask', 'novel_mask') for x in self._metric_names + ] + + def evaluate(self): + """Evaluates with detections from all images with COCO API. + + Returns: + coco_metric: float numpy array with shape [24] representing the + coco-style evaluation metrics (box and mask). + """ + if not self._annotation_file: + gt_dataset = coco_utils.convert_groundtruths_to_coco_dataset( + self._groundtruths) + coco_gt = coco_utils.COCOWrapper( + eval_type=('mask' if self._include_mask else 'box'), + gt_dataset=gt_dataset) + else: + coco_gt = self._coco_gt + coco_predictions = coco_utils.convert_predictions_to_coco_annotations( + self._predictions) + coco_dt = coco_gt.loadRes(predictions=coco_predictions) + image_ids = [ann['image_id'] for ann in coco_predictions] + + coco_eval = cocoeval.COCOeval(coco_gt, coco_dt, iouType='bbox') + coco_eval.params.imgIds = image_ids + coco_eval.evaluate() + coco_eval.accumulate() + coco_eval.summarize() + coco_metrics = coco_eval.stats + + if self._include_mask: + mcoco_eval = cocoeval.COCOeval(coco_gt, coco_dt, iouType='segm') + mcoco_eval.params.imgIds = image_ids + mcoco_eval.evaluate() + mcoco_eval.accumulate() + mcoco_eval.summarize() + if self._mask_eval_class == 'all': + metrics = np.hstack((coco_metrics, mcoco_eval.stats)) + else: + mask_coco_metrics = mcoco_eval.category_stats + val_catg_idx = np.isin(mcoco_eval.params.catIds, + self._eval_categories) + # Gather the valid evaluation of the eval categories. + if np.any(val_catg_idx): + mean_val_metrics = [] + for mid in range(len(self._metric_names) // 2): + mean_val_metrics.append( + np.nanmean(mask_coco_metrics[mid][val_catg_idx])) + + mean_val_metrics = np.array(mean_val_metrics) + else: + mean_val_metrics = np.zeros(len(self._metric_names) // 2) + metrics = np.hstack((coco_metrics, mean_val_metrics)) + else: + metrics = coco_metrics + + # Cleans up the internal variables in order for a fresh eval next time. + self.reset() + + metrics_dict = {} + for i, name in enumerate(self._metric_names): + metrics_dict[name] = metrics[i].astype(np.float32) + return metrics_dict diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/evaluation/coco_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/evaluation/coco_utils.py new file mode 100644 index 0000000..7c63449 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/evaluation/coco_utils.py @@ -0,0 +1,374 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Util functions related to pycocotools and COCO eval.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import copy +import json + +from absl import logging +import numpy as np +from PIL import Image +from pycocotools import coco +from pycocotools import mask as mask_api +import six +import tensorflow.compat.v2 as tf + +from official.vision.detection.dataloader import tf_example_decoder +from official.vision.detection.utils import box_utils +from official.vision.detection.utils import mask_utils + + +class COCOWrapper(coco.COCO): + """COCO wrapper class. + + This class wraps COCO API object, which provides the following additional + functionalities: + 1. Support string type image id. + 2. Support loading the groundtruth dataset using the external annotation + dictionary. + 3. Support loading the prediction results using the external annotation + dictionary. + """ + + def __init__(self, eval_type='box', annotation_file=None, gt_dataset=None): + """Instantiates a COCO-style API object. + + Args: + eval_type: either 'box' or 'mask'. + annotation_file: a JSON file that stores annotations of the eval dataset. + This is required if `gt_dataset` is not provided. + gt_dataset: the groundtruth eval datatset in COCO API format. + """ + if ((annotation_file and gt_dataset) or + ((not annotation_file) and (not gt_dataset))): + raise ValueError('One and only one of `annotation_file` and `gt_dataset` ' + 'needs to be specified.') + + if eval_type not in ['box', 'mask']: + raise ValueError('The `eval_type` can only be either `box` or `mask`.') + + coco.COCO.__init__(self, annotation_file=annotation_file) + self._eval_type = eval_type + if gt_dataset: + self.dataset = gt_dataset + self.createIndex() + + def loadRes(self, predictions): + """Loads result file and return a result api object. + + Args: + predictions: a list of dictionary each representing an annotation in COCO + format. The required fields are `image_id`, `category_id`, `score`, + `bbox`, `segmentation`. + + Returns: + res: result COCO api object. + + Raises: + ValueError: if the set of image id from predctions is not the subset of + the set of image id of the groundtruth dataset. + """ + res = coco.COCO() + res.dataset['images'] = copy.deepcopy(self.dataset['images']) + res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) + + image_ids = [ann['image_id'] for ann in predictions] + if set(image_ids) != (set(image_ids) & set(self.getImgIds())): + raise ValueError('Results do not correspond to the current dataset!') + for ann in predictions: + x1, x2, y1, y2 = [ann['bbox'][0], ann['bbox'][0] + ann['bbox'][2], + ann['bbox'][1], ann['bbox'][1] + ann['bbox'][3]] + if self._eval_type == 'box': + ann['area'] = ann['bbox'][2] * ann['bbox'][3] + ann['segmentation'] = [ + [x1, y1, x1, y2, x2, y2, x2, y1]] + elif self._eval_type == 'mask': + ann['area'] = mask_api.area(ann['segmentation']) + + res.dataset['annotations'] = copy.deepcopy(predictions) + res.createIndex() + return res + + +def convert_predictions_to_coco_annotations(predictions): + """Converts a batch of predictions to annotations in COCO format. + + Args: + predictions: a dictionary of lists of numpy arrays including the following + fields. K below denotes the maximum number of instances per image. + Required fields: + - source_id: a list of numpy arrays of int or string of shape + [batch_size]. + - num_detections: a list of numpy arrays of int of shape [batch_size]. + - detection_boxes: a list of numpy arrays of float of shape + [batch_size, K, 4], where coordinates are in the original image + space (not the scaled image space). + - detection_classes: a list of numpy arrays of int of shape + [batch_size, K]. + - detection_scores: a list of numpy arrays of float of shape + [batch_size, K]. + Optional fields: + - detection_masks: a list of numpy arrays of float of shape + [batch_size, K, mask_height, mask_width]. + + Returns: + coco_predictions: prediction in COCO annotation format. + """ + coco_predictions = [] + num_batches = len(predictions['source_id']) + batch_size = predictions['source_id'][0].shape[0] + max_num_detections = predictions['detection_classes'][0].shape[1] + use_outer_box = 'detection_outer_boxes' in predictions + for i in range(num_batches): + predictions['detection_boxes'][i] = box_utils.yxyx_to_xywh( + predictions['detection_boxes'][i]) + if use_outer_box: + predictions['detection_outer_boxes'][i] = box_utils.yxyx_to_xywh( + predictions['detection_outer_boxes'][i]) + mask_boxes = predictions['detection_outer_boxes'] + else: + mask_boxes = predictions['detection_boxes'] + + for j in range(batch_size): + if 'detection_masks' in predictions: + image_masks = mask_utils.paste_instance_masks( + predictions['detection_masks'][i][j], + mask_boxes[i][j], + int(predictions['image_info'][i][j, 0, 0]), + int(predictions['image_info'][i][j, 0, 1])) + binary_masks = (image_masks > 0.0).astype(np.uint8) + encoded_masks = [ + mask_api.encode(np.asfortranarray(binary_mask)) + for binary_mask in list(binary_masks)] + for k in range(max_num_detections): + ann = {} + ann['image_id'] = predictions['source_id'][i][j] + ann['category_id'] = predictions['detection_classes'][i][j, k] + ann['bbox'] = predictions['detection_boxes'][i][j, k] + ann['score'] = predictions['detection_scores'][i][j, k] + if 'detection_masks' in predictions: + ann['segmentation'] = encoded_masks[k] + coco_predictions.append(ann) + + for i, ann in enumerate(coco_predictions): + ann['id'] = i + 1 + + return coco_predictions + + +def convert_groundtruths_to_coco_dataset(groundtruths, label_map=None): + """Converts groundtruths to the dataset in COCO format. + + Args: + groundtruths: a dictionary of numpy arrays including the fields below. + Note that each element in the list represent the number for a single + example without batch dimension. K below denotes the actual number of + instances for each image. + Required fields: + - source_id: a list of numpy arrays of int or string of shape + [batch_size]. + - height: a list of numpy arrays of int of shape [batch_size]. + - width: a list of numpy arrays of int of shape [batch_size]. + - num_detections: a list of numpy arrays of int of shape [batch_size]. + - boxes: a list of numpy arrays of float of shape [batch_size, K, 4], + where coordinates are in the original image space (not the + normalized coordinates). + - classes: a list of numpy arrays of int of shape [batch_size, K]. + Optional fields: + - is_crowds: a list of numpy arrays of int of shape [batch_size, K]. If + th field is absent, it is assumed that this instance is not crowd. + - areas: a list of numy arrays of float of shape [batch_size, K]. If the + field is absent, the area is calculated using either boxes or + masks depending on which one is available. + - masks: a list of numpy arrays of string of shape [batch_size, K], + label_map: (optional) a dictionary that defines items from the category id + to the category name. If `None`, collect the category mappping from the + `groundtruths`. + + Returns: + coco_groundtruths: the groundtruth dataset in COCO format. + """ + source_ids = np.concatenate(groundtruths['source_id'], axis=0) + heights = np.concatenate(groundtruths['height'], axis=0) + widths = np.concatenate(groundtruths['width'], axis=0) + gt_images = [{'id': int(i), 'height': int(h), 'width': int(w)} for i, h, w + in zip(source_ids, heights, widths)] + + gt_annotations = [] + num_batches = len(groundtruths['source_id']) + batch_size = groundtruths['source_id'][0].shape[0] + for i in range(num_batches): + for j in range(batch_size): + num_instances = groundtruths['num_detections'][i][j] + for k in range(num_instances): + ann = {} + ann['image_id'] = int(groundtruths['source_id'][i][j]) + if 'is_crowds' in groundtruths: + ann['iscrowd'] = int(groundtruths['is_crowds'][i][j, k]) + else: + ann['iscrowd'] = 0 + ann['category_id'] = int(groundtruths['classes'][i][j, k]) + boxes = groundtruths['boxes'][i] + ann['bbox'] = [ + float(boxes[j, k, 1]), + float(boxes[j, k, 0]), + float(boxes[j, k, 3] - boxes[j, k, 1]), + float(boxes[j, k, 2] - boxes[j, k, 0])] + if 'areas' in groundtruths: + ann['area'] = float(groundtruths['areas'][i][j, k]) + else: + ann['area'] = float( + (boxes[j, k, 3] - boxes[j, k, 1]) * + (boxes[j, k, 2] - boxes[j, k, 0])) + if 'masks' in groundtruths: + mask = Image.open(six.StringIO(groundtruths['masks'][i][j, k])) + width, height = mask.size + np_mask = ( + np.array(mask.getdata()).reshape(height, width).astype(np.uint8)) + np_mask[np_mask > 0] = 255 + encoded_mask = mask_api.encode(np.asfortranarray(np_mask)) + ann['segmentation'] = encoded_mask + if 'areas' not in groundtruths: + ann['area'] = mask_api.area(encoded_mask) + gt_annotations.append(ann) + + for i, ann in enumerate(gt_annotations): + ann['id'] = i + 1 + + if label_map: + gt_categories = [{'id': i, 'name': label_map[i]} for i in label_map] + else: + category_ids = [gt['category_id'] for gt in gt_annotations] + gt_categories = [{'id': i} for i in set(category_ids)] + + gt_dataset = { + 'images': gt_images, + 'categories': gt_categories, + 'annotations': copy.deepcopy(gt_annotations), + } + return gt_dataset + + +class COCOGroundtruthGenerator(object): + """Generates the groundtruth annotations from a single example.""" + + def __init__(self, file_pattern, num_examples, include_mask): + self._file_pattern = file_pattern + self._num_examples = num_examples + self._include_mask = include_mask + self._dataset_fn = tf.data.TFRecordDataset + + def _parse_single_example(self, example): + """Parses a single serialized tf.Example proto. + + Args: + example: a serialized tf.Example proto string. + + Returns: + A dictionary of groundtruth with the following fields: + source_id: a scalar tensor of int64 representing the image source_id. + height: a scalar tensor of int64 representing the image height. + width: a scalar tensor of int64 representing the image width. + boxes: a float tensor of shape [K, 4], representing the groundtruth + boxes in absolute coordinates with respect to the original image size. + classes: a int64 tensor of shape [K], representing the class labels of + each instances. + is_crowds: a bool tensor of shape [K], indicating whether the instance + is crowd. + areas: a float tensor of shape [K], indicating the area of each + instance. + masks: a string tensor of shape [K], containing the bytes of the png + mask of each instance. + """ + decoder = tf_example_decoder.TfExampleDecoder( + include_mask=self._include_mask) + decoded_tensors = decoder.decode(example) + + image = decoded_tensors['image'] + image_size = tf.shape(image)[0:2] + boxes = box_utils.denormalize_boxes( + decoded_tensors['groundtruth_boxes'], image_size) + groundtruths = { + 'source_id': tf.string_to_number( + decoded_tensors['source_id'], out_type=tf.int64), + 'height': decoded_tensors['height'], + 'width': decoded_tensors['width'], + 'num_detections': tf.shape(decoded_tensors['groundtruth_classes'])[0], + 'boxes': boxes, + 'classes': decoded_tensors['groundtruth_classes'], + 'is_crowds': decoded_tensors['groundtruth_is_crowd'], + 'areas': decoded_tensors['groundtruth_area'], + } + if self._include_mask: + groundtruths.update({ + 'masks': decoded_tensors['groundtruth_instance_masks_png'], + }) + return groundtruths + + def _build_pipeline(self): + """Builds data pipeline to generate groundtruth annotations.""" + dataset = tf.data.Dataset.list_files(self._file_pattern, shuffle=False) + dataset = dataset.apply( + tf.data.experimental.parallel_interleave( + lambda filename: self._dataset_fn(filename).prefetch(1), + cycle_length=32, + sloppy=False)) + dataset = dataset.map(self._parse_single_example, num_parallel_calls=64) + dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE) + dataset = dataset.batch(1, drop_remainder=False) + return dataset + + def __call__(self): + with tf.Graph().as_default(): + dataset = self._build_pipeline() + groundtruth = dataset.make_one_shot_iterator().get_next() + + with tf.Session() as sess: + for _ in range(self._num_examples): + groundtruth_result = sess.run(groundtruth) + yield groundtruth_result + + +def scan_and_generator_annotation_file(file_pattern, + num_samples, + include_mask, + annotation_file): + """Scans and generate the COCO-style annotation JSON file given a dataset.""" + groundtruth_generator = COCOGroundtruthGenerator( + file_pattern, num_samples, include_mask) + generate_annotation_file(groundtruth_generator, annotation_file) + + +def generate_annotation_file(groundtruth_generator, + annotation_file): + """Generates COCO-style annotation JSON file given a groundtruth generator.""" + groundtruths = {} + logging.info('Loading groundtruth annotations from dataset to memory...') + for groundtruth in groundtruth_generator(): + for k, v in six.iteritems(groundtruth): + if k not in groundtruths: + groundtruths[k] = [v] + else: + groundtruths[k].append(v) + gt_dataset = convert_groundtruths_to_coco_dataset(groundtruths) + + logging.info('Saving groundtruth annotations to the JSON file...') + with tf.io.gfile.GFile(annotation_file, 'w') as f: + f.write(json.dumps(gt_dataset)) + logging.info('Done saving the JSON file...') diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/evaluation/factory.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/evaluation/factory.py new file mode 100644 index 0000000..5894715 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/evaluation/factory.py @@ -0,0 +1,35 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Evaluator factory.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from official.vision.detection.evaluation import coco_evaluator + + +def evaluator_generator(params): + """Generator function for various evaluators.""" + if params.type == 'box': + evaluator = coco_evaluator.COCOEvaluator( + annotation_file=params.val_json_file, include_mask=False) + elif params.type == 'box_and_mask': + evaluator = coco_evaluator.COCOEvaluator( + annotation_file=params.val_json_file, include_mask=True) + else: + raise ValueError('Evaluator %s is not supported.' % params.type) + + return coco_evaluator.MetricWrapper(evaluator) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/executor/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/executor/__init__.py new file mode 100644 index 0000000..931c2ef --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/executor/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/executor/detection_executor.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/executor/detection_executor.py new file mode 100644 index 0000000..799a239 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/executor/detection_executor.py @@ -0,0 +1,160 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""An executor class for running model on TensorFlow 2.0.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +from absl import logging + +import tensorflow.compat.v2 as tf +from official.modeling.training import distributed_executor as executor +from official.vision.detection.utils.object_detection import visualization_utils + + +class DetectionDistributedExecutor(executor.DistributedExecutor): + """Detection specific customer training loop executor. + + Subclasses the DistributedExecutor and adds support for numpy based metrics. + """ + + def __init__(self, + predict_post_process_fn=None, + trainable_variables_filter=None, + **kwargs): + super(DetectionDistributedExecutor, self).__init__(**kwargs) + if predict_post_process_fn: + assert callable(predict_post_process_fn) + if trainable_variables_filter: + assert callable(trainable_variables_filter) + self._predict_post_process_fn = predict_post_process_fn + self._trainable_variables_filter = trainable_variables_filter + self.eval_steps = tf.Variable( + 0, + trainable=False, + dtype=tf.int32, + synchronization=tf.VariableSynchronization.ON_READ, + aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA, + shape=[]) + + def _create_replicated_step(self, + strategy, + model, + loss_fn, + optimizer, + metric=None): + trainable_variables = model.trainable_variables + if self._trainable_variables_filter: + trainable_variables = self._trainable_variables_filter( + trainable_variables) + logging.info('Filter trainable variables from %d to %d', + len(model.trainable_variables), len(trainable_variables)) + _update_state = lambda labels, outputs: None + if isinstance(metric, tf.keras.metrics.Metric): + _update_state = lambda labels, outputs: metric.update_state( + labels, outputs) + else: + logging.error('Detection: train metric is not an instance of ' + 'tf.keras.metrics.Metric.') + + def _replicated_step(inputs): + """Replicated training step.""" + inputs, labels = inputs + + with tf.GradientTape() as tape: + outputs = model(inputs, training=True) + all_losses = loss_fn(labels, outputs) + losses = {} + for k, v in all_losses.items(): + losses[k] = tf.reduce_mean(v) + per_replica_loss = losses['total_loss'] / strategy.num_replicas_in_sync + _update_state(labels, outputs) + + grads = tape.gradient(per_replica_loss, trainable_variables) + optimizer.apply_gradients(zip(grads, trainable_variables)) + return losses + + return _replicated_step + + def _create_test_step(self, strategy, model, metric): + """Creates a distributed test step.""" + + @tf.function + def test_step(iterator, eval_steps): + """Calculates evaluation metrics on distributed devices.""" + + def _test_step_fn(inputs, eval_steps): + """Replicated accuracy calculation.""" + inputs, labels = inputs + model_outputs = model(inputs, training=False) + if self._predict_post_process_fn: + labels, prediction_outputs = self._predict_post_process_fn( + labels, model_outputs) + num_remaining_visualizations = ( + self._params.eval.num_images_to_visualize - eval_steps) + # If there are remaining number of visualizations that needs to be + # done, add next batch outputs for visualization. + # + # TODO(hongjunchoi): Once dynamic slicing is supported on TPU, only + # write correct slice of outputs to summary file. + if num_remaining_visualizations > 0: + visualization_utils.visualize_images_with_bounding_boxes( + inputs, prediction_outputs['detection_boxes'], + self.global_train_step, self.eval_summary_writer) + + return labels, prediction_outputs + + labels, outputs = strategy.run( + _test_step_fn, args=( + next(iterator), + eval_steps, + )) + outputs = tf.nest.map_structure(strategy.experimental_local_results, + outputs) + labels = tf.nest.map_structure(strategy.experimental_local_results, + labels) + + eval_steps.assign_add(self._params.eval.batch_size) + return labels, outputs + + return test_step + + def _run_evaluation(self, test_step, current_training_step, metric, + test_iterator): + """Runs validation steps and aggregate metrics.""" + self.eval_steps.assign(0) + if not test_iterator or not metric: + logging.warning( + 'Both test_iterator (%s) and metrics (%s) must not be None.', + test_iterator, metric) + return None + logging.info('Running evaluation after step: %s.', current_training_step) + while True: + try: + labels, outputs = test_step(test_iterator, self.eval_steps) + if metric: + metric.update_state(labels, outputs) + except (StopIteration, tf.errors.OutOfRangeError): + break + + metric_result = metric.result() + if isinstance(metric, tf.keras.metrics.Metric): + metric_result = tf.nest.map_structure(lambda x: x.numpy().astype(float), + metric_result) + logging.info('Step: [%d] Validation metric = %s', current_training_step, + metric_result) + return metric_result diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/main.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/main.py new file mode 100644 index 0000000..8113471 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/main.py @@ -0,0 +1,255 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Main function to train various object detection models.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +from absl import app +from absl import flags +from absl import logging +import functools +import os +import pprint +import tensorflow.compat.v2 as tf + +from official.modeling.hyperparams import params_dict +from official.modeling.training import distributed_executor as executor +from official.utils import hyperparams_flags +from official.vision.detection.configs import factory as config_factory +from official.vision.detection.dataloader import input_reader +from official.vision.detection.dataloader import mode_keys as ModeKeys +from official.vision.detection.executor.detection_executor import DetectionDistributedExecutor +from official.vision.detection.modeling import factory as model_factory +from official.utils.flags import core as flags_core +from official.utils.misc import distribution_utils +from official.utils.misc import keras_utils + +hyperparams_flags.initialize_common_flags() +flags_core.define_log_steps() + +flags.DEFINE_bool( + 'enable_xla', + default=False, + help='Enable XLA for GPU') + +flags.DEFINE_string( + 'mode', + default='train', + help='Mode to run: `train`, `eval` or `train_and_eval`.') + +flags.DEFINE_string( + 'model', default='retinanet', + help='Model to run: `retinanet` or `mask_rcnn`.') + +flags.DEFINE_string('training_file_pattern', None, + 'Location of the train data.') + +flags.DEFINE_string('eval_file_pattern', None, 'Location of ther eval data') + +flags.DEFINE_string( + 'checkpoint_path', None, + 'The checkpoint path to eval. Only used in eval_once mode.') + +FLAGS = flags.FLAGS + + +def run_executor(params, + train_input_fn=None, + eval_input_fn=None, + callbacks=None, + strategy=None): + """Runs Retinanet model on distribution strategy defined by the user.""" + + if params.architecture.use_bfloat16: + policy = tf.compat.v2.keras.mixed_precision.experimental.Policy( + 'mixed_bfloat16') + tf.compat.v2.keras.mixed_precision.experimental.set_policy(policy) + + model_builder = model_factory.model_generator(params) + + if strategy is None: + strategy_config = params.strategy_config + distribution_utils.configure_cluster(strategy_config.worker_hosts, + strategy_config.task_index) + strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=params.strategy_type, + num_gpus=strategy_config.num_gpus, + all_reduce_alg=strategy_config.all_reduce_alg, + num_packs=strategy_config.num_packs, + tpu_address=strategy_config.tpu) + + num_workers = int(strategy.num_replicas_in_sync + 7) // 8 + is_multi_host = (int(num_workers) >= 2) + + if FLAGS.mode == 'train': + + def _model_fn(params): + return model_builder.build_model(params, mode=ModeKeys.TRAIN) + + logging.info( + 'Train num_replicas_in_sync %d num_workers %d is_multi_host %s', + strategy.num_replicas_in_sync, num_workers, is_multi_host) + + dist_executor = DetectionDistributedExecutor( + strategy=strategy, + params=params, + model_fn=_model_fn, + loss_fn=model_builder.build_loss_fn, + is_multi_host=is_multi_host, + predict_post_process_fn=model_builder.post_processing, + trainable_variables_filter=model_builder + .make_filter_trainable_variables_fn()) + + if is_multi_host: + train_input_fn = functools.partial( + train_input_fn, + batch_size=params.train.batch_size // strategy.num_replicas_in_sync) + + return dist_executor.train( + train_input_fn=train_input_fn, + model_dir=params.model_dir, + iterations_per_loop=params.train.iterations_per_loop, + total_steps=params.train.total_steps, + init_checkpoint=model_builder.make_restore_checkpoint_fn(), + custom_callbacks=callbacks, + save_config=True) + + elif FLAGS.mode == 'eval' or FLAGS.mode == 'eval_once': + + def _model_fn(params): + return model_builder.build_model(params, mode=ModeKeys.PREDICT_WITH_GT) + + logging.info('Eval num_replicas_in_sync %d num_workers %d is_multi_host %s', + strategy.num_replicas_in_sync, num_workers, is_multi_host) + + if is_multi_host: + eval_input_fn = functools.partial( + eval_input_fn, + batch_size=params.eval.batch_size // strategy.num_replicas_in_sync) + + dist_executor = DetectionDistributedExecutor( + strategy=strategy, + params=params, + model_fn=_model_fn, + loss_fn=model_builder.build_loss_fn, + is_multi_host=is_multi_host, + predict_post_process_fn=model_builder.post_processing, + trainable_variables_filter=model_builder + .make_filter_trainable_variables_fn()) + + if FLAGS.mode == 'eval': + results = dist_executor.evaluate_from_model_dir( + model_dir=params.model_dir, + eval_input_fn=eval_input_fn, + eval_metric_fn=model_builder.eval_metrics, + eval_timeout=params.eval.eval_timeout, + min_eval_interval=params.eval.min_eval_interval, + total_steps=params.train.total_steps) + else: + # Run evaluation once for a single checkpoint. + if not FLAGS.checkpoint_path: + raise ValueError('FLAGS.checkpoint_path cannot be empty.') + checkpoint_path = FLAGS.checkpoint_path + if tf.io.gfile.isdir(checkpoint_path): + checkpoint_path = tf.train.latest_checkpoint(checkpoint_path) + summary_writer = executor.SummaryWriter(params.model_dir, 'eval') + results, _ = dist_executor.evaluate_checkpoint( + checkpoint_path=checkpoint_path, + eval_input_fn=eval_input_fn, + eval_metric_fn=model_builder.eval_metrics, + summary_writer=summary_writer) + for k, v in results.items(): + logging.info('Final eval metric %s: %f', k, v) + return results + else: + raise ValueError('Mode not found: %s.' % FLAGS.mode) + + +def run(callbacks=None): + keras_utils.set_session_config(enable_xla=FLAGS.enable_xla) + + params = config_factory.config_generator(FLAGS.model) + + params = params_dict.override_params_dict( + params, FLAGS.config_file, is_strict=True) + + params = params_dict.override_params_dict( + params, FLAGS.params_override, is_strict=True) + params.override( + { + 'strategy_type': FLAGS.strategy_type, + 'model_dir': FLAGS.model_dir, + 'strategy_config': executor.strategy_flags_dict(), + }, + is_strict=False) + params.validate() + params.lock() + pp = pprint.PrettyPrinter() + params_str = pp.pformat(params.as_dict()) + logging.info('Model Parameters: {}'.format(params_str)) + + train_input_fn = None + eval_input_fn = None + training_file_pattern = FLAGS.training_file_pattern or params.train.train_file_pattern + eval_file_pattern = FLAGS.eval_file_pattern or params.eval.eval_file_pattern + if not training_file_pattern and not eval_file_pattern: + raise ValueError('Must provide at least one of training_file_pattern and ' + 'eval_file_pattern.') + + if training_file_pattern: + # Use global batch size for single host. + train_input_fn = input_reader.InputFn( + file_pattern=training_file_pattern, + params=params, + mode=input_reader.ModeKeys.TRAIN, + batch_size=params.train.batch_size) + + if eval_file_pattern: + eval_input_fn = input_reader.InputFn( + file_pattern=eval_file_pattern, + params=params, + mode=input_reader.ModeKeys.PREDICT_WITH_GT, + batch_size=params.eval.batch_size, + num_examples=params.eval.eval_samples) + + if callbacks is None: + callbacks = [] + + if FLAGS.log_steps: + callbacks.append( + keras_utils.TimeHistory( + batch_size=params.train.batch_size, + log_steps=FLAGS.log_steps, + )) + + return run_executor( + params, + train_input_fn=train_input_fn, + eval_input_fn=eval_input_fn, + callbacks=callbacks) + + +def main(argv): + del argv # Unused. + + run() + + +if __name__ == '__main__': + tf.config.set_soft_device_placement(True) + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/__init__.py new file mode 100644 index 0000000..931c2ef --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/__init__.py new file mode 100644 index 0000000..931c2ef --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/factory.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/factory.py new file mode 100644 index 0000000..71925af --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/factory.py @@ -0,0 +1,136 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model architecture factory.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from official.vision.detection.modeling.architecture import fpn +from official.vision.detection.modeling.architecture import heads +from official.vision.detection.modeling.architecture import identity +from official.vision.detection.modeling.architecture import nn_ops +from official.vision.detection.modeling.architecture import resnet + + +def batch_norm_relu_generator(params): + + def _batch_norm_op(**kwargs): + return nn_ops.BatchNormRelu( + momentum=params.batch_norm_momentum, + epsilon=params.batch_norm_epsilon, + trainable=params.batch_norm_trainable, + **kwargs) + + return _batch_norm_op + + +def backbone_generator(params): + """Generator function for various backbone models.""" + if params.architecture.backbone == 'resnet': + resnet_params = params.resnet + backbone_fn = resnet.Resnet( + resnet_depth=resnet_params.resnet_depth, + batch_norm_relu=batch_norm_relu_generator(resnet_params.batch_norm)) + else: + raise ValueError('Backbone model %s is not supported.' % + params.architecture.backbone) + + return backbone_fn + + +def multilevel_features_generator(params): + """Generator function for various FPN models.""" + if params.architecture.multilevel_features == 'fpn': + fpn_params = params.fpn + fpn_fn = fpn.Fpn( + min_level=fpn_params.min_level, + max_level=fpn_params.max_level, + fpn_feat_dims=fpn_params.fpn_feat_dims, + use_separable_conv=fpn_params.use_separable_conv, + use_batch_norm=fpn_params.use_batch_norm, + batch_norm_relu=batch_norm_relu_generator(fpn_params.batch_norm)) + elif params.architecture.multilevel_features == 'identity': + fpn_fn = identity.Identity() + else: + raise ValueError('The multi-level feature model %s is not supported.' + % params.architecture.multilevel_features) + return fpn_fn + + +def retinanet_head_generator(params): + """Generator function for RetinaNet head architecture.""" + return heads.RetinanetHead( + params.min_level, + params.max_level, + params.num_classes, + params.anchors_per_location, + params.retinanet_head_num_convs, + params.retinanet_head_num_filters, + params.use_separable_conv, + batch_norm_relu=batch_norm_relu_generator(params.batch_norm)) + + +def rpn_head_generator(params): + """Generator function for RPN head architecture.""" + return heads.RpnHead(params.min_level, + params.max_level, + params.anchors_per_location, + params.num_convs, + params.num_filters, + params.use_separable_conv, + params.use_batch_norm, + batch_norm_relu=batch_norm_relu_generator( + params.batch_norm)) + + +def fast_rcnn_head_generator(params): + """Generator function for Fast R-CNN head architecture.""" + return heads.FastrcnnHead(params.num_classes, + params.num_convs, + params.num_filters, + params.use_separable_conv, + params.num_fcs, + params.fc_dims, + params.use_batch_norm, + batch_norm_relu=batch_norm_relu_generator( + params.batch_norm)) + + +def mask_rcnn_head_generator(params): + """Generator function for Mask R-CNN head architecture.""" + return heads.MaskrcnnHead(params.num_classes, + params.mask_target_size, + params.num_convs, + params.num_filters, + params.use_separable_conv, + params.use_batch_norm, + batch_norm_relu=batch_norm_relu_generator( + params.batch_norm)) + + +def shapeprior_head_generator(params): + """Generator function for Shapemask head architecture.""" + raise NotImplementedError('Unimplemented') + + +def coarsemask_head_generator(params): + """Generator function for Shapemask head architecture.""" + raise NotImplementedError('Unimplemented') + + +def finemask_head_generator(params): + """Generator function for Shapemask head architecture.""" + raise NotImplementedError('Unimplemented') diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/fpn.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/fpn.py new file mode 100644 index 0000000..d74f541 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/fpn.py @@ -0,0 +1,143 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Feature Pyramid Networks. + +Feature Pyramid Networks were proposed in: +[1] Tsung-Yi Lin, Piotr Dollar, Ross Girshick, Kaiming He, Bharath Hariharan, + , and Serge Belongie + Feature Pyramid Networks for Object Detection. CVPR 2017. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools + +import tensorflow.compat.v2 as tf + +from tensorflow.python.keras import backend +from official.vision.detection.modeling.architecture import nn_ops +from official.vision.detection.ops import spatial_transform_ops + + +class Fpn(object): + """Feature pyramid networks.""" + + def __init__(self, + min_level=3, + max_level=7, + fpn_feat_dims=256, + use_separable_conv=False, + use_batch_norm=True, + batch_norm_relu=nn_ops.BatchNormRelu): + """FPN initialization function. + + Args: + min_level: `int` minimum level in FPN output feature maps. + max_level: `int` maximum level in FPN output feature maps. + fpn_feat_dims: `int` number of filters in FPN layers. + use_separable_conv: `bool`, if True use separable convolution for + convolution in FPN layers. + use_batch_norm: 'bool', indicating whether batchnorm layers are added. + batch_norm_relu: an operation that includes a batch normalization layer + followed by a relu layer(optional). + """ + self._min_level = min_level + self._max_level = max_level + self._fpn_feat_dims = fpn_feat_dims + if use_separable_conv: + self._conv2d_op = functools.partial( + tf.keras.layers.SeparableConv2D, depth_multiplier=1) + else: + self._conv2d_op = tf.keras.layers.Conv2D + self._use_batch_norm = use_batch_norm + self._batch_norm_relu = batch_norm_relu + + self._batch_norm_relus = {} + self._lateral_conv2d_op = {} + self._post_hoc_conv2d_op = {} + self._coarse_conv2d_op = {} + for level in range(self._min_level, self._max_level + 1): + if self._use_batch_norm: + self._batch_norm_relus[level] = batch_norm_relu( + relu=False, name='p%d-bn' % level) + self._lateral_conv2d_op[level] = self._conv2d_op( + filters=self._fpn_feat_dims, + kernel_size=(1, 1), + padding='same', + name='l%d' % level) + self._post_hoc_conv2d_op[level] = self._conv2d_op( + filters=self._fpn_feat_dims, + strides=(1, 1), + kernel_size=(3, 3), + padding='same', + name='post_hoc_d%d' % level) + self._coarse_conv2d_op[level] = self._conv2d_op( + filters=self._fpn_feat_dims, + strides=(2, 2), + kernel_size=(3, 3), + padding='same', + name='p%d' % level) + + def __call__(self, multilevel_features, is_training=None): + """Returns the FPN features for a given multilevel features. + + Args: + multilevel_features: a `dict` containing `int` keys for continuous feature + levels, e.g., [2, 3, 4, 5]. The values are corresponding features with + shape [batch_size, height_l, width_l, num_filters]. + is_training: `bool` if True, the model is in training mode. + + Returns: + a `dict` containing `int` keys for continuous feature levels + [min_level, min_level + 1, ..., max_level]. The values are corresponding + FPN features with shape [batch_size, height_l, width_l, fpn_feat_dims]. + """ + input_levels = list(multilevel_features.keys()) + if min(input_levels) > self._min_level: + raise ValueError( + 'The minimum backbone level %d should be '%(min(input_levels)) + + 'less or equal to FPN minimum level %d.:'%(self._min_level)) + backbone_max_level = min(max(input_levels), self._max_level) + with backend.get_graph().as_default(), tf.name_scope('fpn'): + # Adds lateral connections. + feats_lateral = {} + for level in range(self._min_level, backbone_max_level + 1): + feats_lateral[level] = self._lateral_conv2d_op[level]( + multilevel_features[level]) + + # Adds top-down path. + feats = {backbone_max_level: feats_lateral[backbone_max_level]} + for level in range(backbone_max_level - 1, self._min_level - 1, -1): + feats[level] = spatial_transform_ops.nearest_upsampling( + feats[level + 1], 2) + feats_lateral[level] + + # Adds post-hoc 3x3 convolution kernel. + for level in range(self._min_level, backbone_max_level + 1): + feats[level] = self._post_hoc_conv2d_op[level](feats[level]) + + # Adds coarser FPN levels introduced for RetinaNet. + for level in range(backbone_max_level + 1, self._max_level + 1): + feats_in = feats[level - 1] + if level > backbone_max_level + 1: + feats_in = tf.nn.relu(feats_in) + feats[level] = self._coarse_conv2d_op[level](feats_in) + if self._use_batch_norm: + # Adds batch_norm layer. + for level in range(self._min_level, self._max_level + 1): + feats[level] = self._batch_norm_relus[level]( + feats[level], is_training=is_training) + return feats diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/heads.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/heads.py new file mode 100644 index 0000000..9b6fc67 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/heads.py @@ -0,0 +1,1108 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Classes to build various prediction heads in all supported models.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools +import pickle + +from absl import logging +import numpy as np +import tensorflow.compat.v2 as tf +from tensorflow.python.keras import backend +from official.vision.detection.modeling.architecture import nn_ops +from official.vision.detection.ops import spatial_transform_ops + + +class RpnHead(tf.keras.layers.Layer): + """Region Proposal Network head.""" + + def __init__(self, + min_level, + max_level, + anchors_per_location, + num_convs=2, + num_filters=256, + use_separable_conv=False, + use_batch_norm=True, + batch_norm_relu=nn_ops.BatchNormRelu): + """Initialize params to build Region Proposal Network head. + + Args: + min_level: `int` number of minimum feature level. + max_level: `int` number of maximum feature level. + anchors_per_location: `int` number of number of anchors per pixel + location. + num_convs: `int` number that represents the number of the intermediate + conv layers before the prediction. + num_filters: `int` number that represents the number of filters of the + intermediate conv layers. + use_separable_conv: `bool`, indicating whether the separable conv layers + is used. + use_batch_norm: 'bool', indicating whether batchnorm layers are added. + batch_norm_relu: an operation that includes a batch normalization layer + followed by a relu layer(optional). + """ + self._min_level = min_level + self._max_level = max_level + self._anchors_per_location = anchors_per_location + self._use_batch_norm = use_batch_norm + + if use_separable_conv: + self._conv2d_op = functools.partial( + tf.keras.layers.SeparableConv2D, + depth_multiplier=1, + bias_initializer=tf.zeros_initializer()) + else: + self._conv2d_op = functools.partial( + tf.keras.layers.Conv2D, + kernel_initializer=tf.keras.initializers.RandomNormal(stddev=0.01), + bias_initializer=tf.zeros_initializer()) + + self._rpn_conv = self._conv2d_op( + num_filters, + kernel_size=(3, 3), + strides=(1, 1), + activation=(None if self._use_batch_norm else tf.nn.relu), + padding='same', + name='rpn') + self._rpn_class_conv = self._conv2d_op( + anchors_per_location, + kernel_size=(1, 1), + strides=(1, 1), + padding='valid', + name='rpn-class') + self._rpn_box_conv = self._conv2d_op( + 4 * anchors_per_location, + kernel_size=(1, 1), + strides=(1, 1), + padding='valid', + name='rpn-box') + + self._batch_norm_relus = {} + if self._use_batch_norm: + for level in range(self._min_level, self._max_level + 1): + self._batch_norm_relus[level] = batch_norm_relu(name='rpn-l%d-bn' % + level) + + def _shared_rpn_heads(self, features, anchors_per_location, level, + is_training): + """Shared RPN heads.""" + features = self._rpn_conv(features) + if self._use_batch_norm: + # The batch normalization layers are not shared between levels. + features = self._batch_norm_relus[level]( + features, is_training=is_training) + # Proposal classification scores + scores = self._rpn_class_conv(features) + # Proposal bbox regression deltas + bboxes = self._rpn_box_conv(features) + + return scores, bboxes + + def __call__(self, features, is_training=None): + + scores_outputs = {} + box_outputs = {} + + with backend.get_graph().as_default(), tf.name_scope('rpn_head'): + for level in range(self._min_level, self._max_level + 1): + scores_output, box_output = self._shared_rpn_heads( + features[level], self._anchors_per_location, level, is_training) + scores_outputs[level] = scores_output + box_outputs[level] = box_output + return scores_outputs, box_outputs + + +class FastrcnnHead(tf.keras.layers.Layer): + """Fast R-CNN box head.""" + + def __init__(self, + num_classes, + num_convs=0, + num_filters=256, + use_separable_conv=False, + num_fcs=2, + fc_dims=1024, + use_batch_norm=True, + batch_norm_relu=nn_ops.BatchNormRelu): + """Initialize params to build Fast R-CNN box head. + + Args: + num_classes: a integer for the number of classes. + num_convs: `int` number that represents the number of the intermediate + conv layers before the FC layers. + num_filters: `int` number that represents the number of filters of the + intermediate conv layers. + use_separable_conv: `bool`, indicating whether the separable conv layers + is used. + num_fcs: `int` number that represents the number of FC layers before the + predictions. + fc_dims: `int` number that represents the number of dimension of the FC + layers. + use_batch_norm: 'bool', indicating whether batchnorm layers are added. + batch_norm_relu: an operation that includes a batch normalization layer + followed by a relu layer(optional). + """ + self._num_classes = num_classes + + self._num_convs = num_convs + self._num_filters = num_filters + if use_separable_conv: + self._conv2d_op = functools.partial( + tf.keras.layers.SeparableConv2D, + depth_multiplier=1, + bias_initializer=tf.zeros_initializer()) + else: + self._conv2d_op = functools.partial( + tf.keras.layers.Conv2D, + kernel_initializer=tf.keras.initializers.VarianceScaling( + scale=2, mode='fan_out', distribution='untruncated_normal'), + bias_initializer=tf.zeros_initializer()) + + self._num_fcs = num_fcs + self._fc_dims = fc_dims + + self._use_batch_norm = use_batch_norm + self._batch_norm_relu = batch_norm_relu + + self._conv_ops = [] + self._conv_bn_ops = [] + for i in range(self._num_convs): + self._conv_ops.append( + self._conv2d_op( + self._num_filters, + kernel_size=(3, 3), + strides=(1, 1), + padding='same', + dilation_rate=(1, 1), + activation=(None if self._use_batch_norm else tf.nn.relu), + name='conv_{}'.format(i))) + if self._use_batch_norm: + self._conv_bn_ops.append(self._batch_norm_relu()) + + self._fc_ops = [] + self._fc_bn_ops = [] + for i in range(self._num_fcs): + self._fc_ops.append( + tf.keras.layers.Dense( + units=self._fc_dims, + activation=(None if self._use_batch_norm else tf.nn.relu), + name='fc{}'.format(i))) + if self._use_batch_norm: + self._fc_bn_ops.append(self._batch_norm_relu(fused=False)) + + self._class_predict = tf.keras.layers.Dense( + self._num_classes, + kernel_initializer=tf.keras.initializers.RandomNormal(stddev=0.01), + bias_initializer=tf.zeros_initializer(), + name='class-predict') + self._box_predict = tf.keras.layers.Dense( + self._num_classes * 4, + kernel_initializer=tf.keras.initializers.RandomNormal(stddev=0.001), + bias_initializer=tf.zeros_initializer(), + name='box-predict') + + def __call__(self, roi_features, is_training=None): + """Box and class branches for the Mask-RCNN model. + + Args: + roi_features: A ROI feature tensor of shape + [batch_size, num_rois, height_l, width_l, num_filters]. + is_training: `boolean`, if True if model is in training mode. + + Returns: + class_outputs: a tensor with a shape of + [batch_size, num_rois, num_classes], representing the class predictions. + box_outputs: a tensor with a shape of + [batch_size, num_rois, num_classes * 4], representing the box + predictions. + """ + + with backend.get_graph().as_default(), tf.name_scope('fast_rcnn_head'): + # reshape inputs beofre FC. + _, num_rois, height, width, filters = roi_features.get_shape().as_list() + + net = tf.reshape(roi_features, [-1, height, width, filters]) + for i in range(self._num_convs): + net = self._conv_ops[i](net) + if self._use_batch_norm: + net = self._conv_bn_ops[i](net, is_training=is_training) + + filters = self._num_filters if self._num_convs > 0 else filters + net = tf.reshape(net, [-1, num_rois, height * width * filters]) + + for i in range(self._num_fcs): + net = self._fc_ops[i](net) + if self._use_batch_norm: + net = self._fc_bn_ops[i](net, is_training=is_training) + + class_outputs = self._class_predict(net) + box_outputs = self._box_predict(net) + return class_outputs, box_outputs + + +class MaskrcnnHead(tf.keras.layers.Layer): + """Mask R-CNN head.""" + + def __init__(self, + num_classes, + mask_target_size, + num_convs=4, + num_filters=256, + use_separable_conv=False, + use_batch_norm=True, + batch_norm_relu=nn_ops.BatchNormRelu): + """Initialize params to build Fast R-CNN head. + + Args: + num_classes: a integer for the number of classes. + mask_target_size: a integer that is the resolution of masks. + num_convs: `int` number that represents the number of the intermediate + conv layers before the prediction. + num_filters: `int` number that represents the number of filters of the + intermediate conv layers. + use_separable_conv: `bool`, indicating whether the separable conv layers + is used. + use_batch_norm: 'bool', indicating whether batchnorm layers are added. + batch_norm_relu: an operation that includes a batch normalization layer + followed by a relu layer(optional). + """ + self._num_classes = num_classes + self._mask_target_size = mask_target_size + + self._num_convs = num_convs + self._num_filters = num_filters + if use_separable_conv: + self._conv2d_op = functools.partial( + tf.keras.layers.SeparableConv2D, + depth_multiplier=1, + bias_initializer=tf.zeros_initializer()) + else: + self._conv2d_op = functools.partial( + tf.keras.layers.Conv2D, + kernel_initializer=tf.keras.initializers.VarianceScaling( + scale=2, mode='fan_out', distribution='untruncated_normal'), + bias_initializer=tf.zeros_initializer()) + + self._use_batch_norm = use_batch_norm + self._batch_norm_relu = batch_norm_relu + self._conv2d_ops = [] + for i in range(self._num_convs): + self._conv2d_ops.append( + self._conv2d_op( + self._num_filters, + kernel_size=(3, 3), + strides=(1, 1), + padding='same', + dilation_rate=(1, 1), + activation=(None if self._use_batch_norm else tf.nn.relu), + name='mask-conv-l%d' % i)) + self._mask_conv_transpose = tf.keras.layers.Conv2DTranspose( + self._num_filters, + kernel_size=(2, 2), + strides=(2, 2), + padding='valid', + activation=(None if self._use_batch_norm else tf.nn.relu), + kernel_initializer=tf.keras.initializers.VarianceScaling( + scale=2, mode='fan_out', distribution='untruncated_normal'), + bias_initializer=tf.zeros_initializer(), + name='conv5-mask') + + def __call__(self, roi_features, class_indices, is_training=None): + """Mask branch for the Mask-RCNN model. + + Args: + roi_features: A ROI feature tensor of shape + [batch_size, num_rois, height_l, width_l, num_filters]. + class_indices: a Tensor of shape [batch_size, num_rois], indicating + which class the ROI is. + is_training: `boolean`, if True if model is in training mode. + + Returns: + mask_outputs: a tensor with a shape of + [batch_size, num_masks, mask_height, mask_width, num_classes], + representing the mask predictions. + fg_gather_indices: a tensor with a shape of [batch_size, num_masks, 2], + representing the fg mask targets. + Raises: + ValueError: If boxes is not a rank-3 tensor or the last dimension of + boxes is not 4. + """ + + with backend.get_graph().as_default(): + with tf.name_scope('mask_head'): + _, num_rois, height, width, filters = roi_features.get_shape().as_list() + net = tf.reshape(roi_features, [-1, height, width, filters]) + + for i in range(self._num_convs): + net = self._conv2d_ops[i](net) + if self._use_batch_norm: + net = self._batch_norm_relu()(net, is_training=is_training) + + net = self._mask_conv_transpose(net) + if self._use_batch_norm: + net = self._batch_norm_relu()(net, is_training=is_training) + + mask_outputs = self._conv2d_op( + self._num_classes, + kernel_size=(1, 1), + strides=(1, 1), + padding='valid', + name='mask_fcn_logits')( + net) + mask_outputs = tf.reshape(mask_outputs, [ + -1, num_rois, self._mask_target_size, self._mask_target_size, + self._num_classes + ]) + + with tf.name_scope('masks_post_processing'): + # TODO(pengchong): Figure out the way not to use the static inferred + # batch size. + batch_size, num_masks = class_indices.get_shape().as_list() + mask_outputs = tf.transpose(a=mask_outputs, perm=[0, 1, 4, 2, 3]) + # Contructs indices for gather. + batch_indices = tf.tile( + tf.expand_dims(tf.range(batch_size), axis=1), [1, num_masks]) + mask_indices = tf.tile( + tf.expand_dims(tf.range(num_masks), axis=0), [batch_size, 1]) + gather_indices = tf.stack( + [batch_indices, mask_indices, class_indices], axis=2) + mask_outputs = tf.gather_nd(mask_outputs, gather_indices) + return mask_outputs + + +class RetinanetHead(object): + """RetinaNet head.""" + + def __init__(self, + min_level, + max_level, + num_classes, + anchors_per_location, + num_convs=4, + num_filters=256, + use_separable_conv=False, + batch_norm_relu=nn_ops.BatchNormRelu): + """Initialize params to build RetinaNet head. + + Args: + min_level: `int` number of minimum feature level. + max_level: `int` number of maximum feature level. + num_classes: `int` number of classification categories. + anchors_per_location: `int` number of anchors per pixel location. + num_convs: `int` number of stacked convolution before the last prediction + layer. + num_filters: `int` number of filters used in the head architecture. + use_separable_conv: `bool` to indicate whether to use separable + convoluation. + batch_norm_relu: an operation that includes a batch normalization layer + followed by a relu layer(optional). + """ + self._min_level = min_level + self._max_level = max_level + + self._num_classes = num_classes + self._anchors_per_location = anchors_per_location + + self._num_convs = num_convs + self._num_filters = num_filters + self._use_separable_conv = use_separable_conv + + with tf.name_scope('class_net') as scope_name: + self._class_name_scope = tf.name_scope(scope_name) + with tf.name_scope('box_net') as scope_name: + self._box_name_scope = tf.name_scope(scope_name) + self._build_class_net_layers(batch_norm_relu) + self._build_box_net_layers(batch_norm_relu) + + def _class_net_batch_norm_name(self, i, level): + return 'class-%d-%d' % (i, level) + + def _box_net_batch_norm_name(self, i, level): + return 'box-%d-%d' % (i, level) + + def _build_class_net_layers(self, batch_norm_relu): + """Build re-usable layers for class prediction network.""" + if self._use_separable_conv: + self._class_predict = tf.keras.layers.SeparableConv2D( + self._num_classes * self._anchors_per_location, + kernel_size=(3, 3), + bias_initializer=tf.constant_initializer(-np.log((1 - 0.01) / 0.01)), + padding='same', + name='class-predict') + else: + self._class_predict = tf.keras.layers.Conv2D( + self._num_classes * self._anchors_per_location, + kernel_size=(3, 3), + bias_initializer=tf.constant_initializer(-np.log((1 - 0.01) / 0.01)), + kernel_initializer=tf.keras.initializers.RandomNormal(stddev=1e-5), + padding='same', + name='class-predict') + self._class_conv = [] + self._class_batch_norm_relu = {} + for i in range(self._num_convs): + if self._use_separable_conv: + self._class_conv.append( + tf.keras.layers.SeparableConv2D( + self._num_filters, + kernel_size=(3, 3), + bias_initializer=tf.zeros_initializer(), + activation=None, + padding='same', + name='class-' + str(i))) + else: + self._class_conv.append( + tf.keras.layers.Conv2D( + self._num_filters, + kernel_size=(3, 3), + bias_initializer=tf.zeros_initializer(), + kernel_initializer=tf.keras.initializers.RandomNormal( + stddev=0.01), + activation=None, + padding='same', + name='class-' + str(i))) + for level in range(self._min_level, self._max_level + 1): + name = self._class_net_batch_norm_name(i, level) + self._class_batch_norm_relu[name] = batch_norm_relu(name=name) + + def _build_box_net_layers(self, batch_norm_relu): + """Build re-usable layers for box prediction network.""" + if self._use_separable_conv: + self._box_predict = tf.keras.layers.SeparableConv2D( + 4 * self._anchors_per_location, + kernel_size=(3, 3), + bias_initializer=tf.zeros_initializer(), + padding='same', + name='box-predict') + else: + self._box_predict = tf.keras.layers.Conv2D( + 4 * self._anchors_per_location, + kernel_size=(3, 3), + bias_initializer=tf.zeros_initializer(), + kernel_initializer=tf.keras.initializers.RandomNormal(stddev=1e-5), + padding='same', + name='box-predict') + self._box_conv = [] + self._box_batch_norm_relu = {} + for i in range(self._num_convs): + if self._use_separable_conv: + self._box_conv.append( + tf.keras.layers.SeparableConv2D( + self._num_filters, + kernel_size=(3, 3), + activation=None, + bias_initializer=tf.zeros_initializer(), + padding='same', + name='box-' + str(i))) + else: + self._box_conv.append( + tf.keras.layers.Conv2D( + self._num_filters, + kernel_size=(3, 3), + activation=None, + bias_initializer=tf.zeros_initializer(), + kernel_initializer=tf.keras.initializers.RandomNormal( + stddev=0.01), + padding='same', + name='box-' + str(i))) + for level in range(self._min_level, self._max_level + 1): + name = self._box_net_batch_norm_name(i, level) + self._box_batch_norm_relu[name] = batch_norm_relu(name=name) + + def __call__(self, fpn_features, is_training=None): + """Returns outputs of RetinaNet head.""" + class_outputs = {} + box_outputs = {} + with backend.get_graph().as_default(), tf.name_scope('retinanet'): + for level in range(self._min_level, self._max_level + 1): + features = fpn_features[level] + + class_outputs[level] = self.class_net( + features, level, is_training=is_training) + box_outputs[level] = self.box_net( + features, level, is_training=is_training) + return class_outputs, box_outputs + + def class_net(self, features, level, is_training): + """Class prediction network for RetinaNet.""" + with self._class_name_scope: + for i in range(self._num_convs): + features = self._class_conv[i](features) + # The convolution layers in the class net are shared among all levels, but + # each level has its batch normlization to capture the statistical + # difference among different levels. + name = self._class_net_batch_norm_name(i, level) + features = self._class_batch_norm_relu[name]( + features, is_training=is_training) + + classes = self._class_predict(features) + return classes + + def box_net(self, features, level, is_training=None): + """Box regression network for RetinaNet.""" + with self._box_name_scope: + for i in range(self._num_convs): + features = self._box_conv[i](features) + # The convolution layers in the box net are shared among all levels, but + # each level has its batch normlization to capture the statistical + # difference among different levels. + name = self._box_net_batch_norm_name(i, level) + features = self._box_batch_norm_relu[name]( + features, is_training=is_training) + + boxes = self._box_predict(features) + return boxes + + +# TODO(yeqing): Refactor this class when it is ready for var_scope reuse. +class ShapemaskPriorHead(object): + """ShapeMask Prior head.""" + + def __init__(self, + num_classes, + num_downsample_channels, + mask_crop_size, + use_category_for_mask, + num_of_instances, + min_mask_level, + max_mask_level, + num_clusters, + temperature, + shape_prior_path=None): + """Initialize params to build RetinaNet head. + + Args: + num_classes: Number of output classes. + num_downsample_channels: number of channels in mask branch. + mask_crop_size: feature crop size. + use_category_for_mask: use class information in mask branch. + num_of_instances: number of instances to sample in training time. + min_mask_level: minimum FPN level to crop mask feature from. + max_mask_level: maximum FPN level to crop mask feature from. + num_clusters: number of clusters to use in K-Means. + temperature: the temperature for shape prior learning. + shape_prior_path: the path to load shape priors. + """ + self._mask_num_classes = num_classes + self._num_downsample_channels = num_downsample_channels + self._mask_crop_size = mask_crop_size + self._use_category_for_mask = use_category_for_mask + self._num_of_instances = num_of_instances + self._min_mask_level = min_mask_level + self._max_mask_level = max_mask_level + self._num_clusters = num_clusters + self._temperature = temperature + self._shape_prior_path = shape_prior_path + + def __call__(self, + fpn_features, + boxes, + outer_boxes, + classes, + is_training=None): + """Generate the detection priors from the box detections and FPN features. + + This corresponds to the Fig. 4 of the ShapeMask paper at + https://arxiv.org/pdf/1904.03239.pdf + + Args: + fpn_features: a dictionary of FPN features. + boxes: a float tensor of shape [batch_size, num_instances, 4] + representing the tight gt boxes from dataloader/detection. + outer_boxes: a float tensor of shape [batch_size, num_instances, 4] + representing the loose gt boxes from dataloader/detection. + classes: a int Tensor of shape [batch_size, num_instances] + of instance classes. + is_training: training mode or not. + + Returns: + crop_features: a float Tensor of shape [batch_size * num_instances, + mask_crop_size, mask_crop_size, num_downsample_channels]. This is the + instance feature crop. + detection_priors: A float Tensor of shape [batch_size * num_instances, + mask_size, mask_size, 1]. + """ + with backend.get_graph().as_default(): + # loads class specific or agnostic shape priors + if self._shape_prior_path: + if self._use_category_for_mask: + fid = tf.io.gfile.GFile(self._shape_prior_path, 'rb') + # The encoding='bytes' options is for incompatibility between python2 + # and python3 pickle. + class_tups = pickle.load(fid, encoding='bytes') + max_class_id = class_tups[-1][0] + 1 + class_masks = np.zeros((max_class_id, self._num_clusters, + self._mask_crop_size, self._mask_crop_size), + dtype=np.float32) + for cls_id, _, cls_mask in class_tups: + assert cls_mask.shape == (self._num_clusters, + self._mask_crop_size**2) + class_masks[cls_id] = cls_mask.reshape(self._num_clusters, + self._mask_crop_size, + self._mask_crop_size) + + self.class_priors = tf.convert_to_tensor( + value=class_masks, dtype=tf.float32) + else: + npy_path = tf.io.gfile.GFile(self._shape_prior_path) + class_np_masks = np.load(npy_path) + assert class_np_masks.shape == ( + self._num_clusters, self._mask_crop_size, + self._mask_crop_size), 'Invalid priors!!!' + self.class_priors = tf.convert_to_tensor( + value=class_np_masks, dtype=tf.float32) + else: + self.class_priors = tf.zeros( + [self._num_clusters, self._mask_crop_size, self._mask_crop_size], + tf.float32) + + batch_size = boxes.get_shape()[0] + min_level_shape = fpn_features[self._min_mask_level].get_shape().as_list() + self._max_feature_size = min_level_shape[1] + detection_prior_levels = self._compute_box_levels(boxes) + level_outer_boxes = outer_boxes / tf.pow( + 2., tf.expand_dims(detection_prior_levels, -1)) + detection_prior_levels = tf.cast(detection_prior_levels, tf.int32) + uniform_priors = spatial_transform_ops.crop_mask_in_target_box( + tf.ones([ + batch_size, self._num_of_instances, self._mask_crop_size, + self._mask_crop_size + ], tf.float32), boxes, outer_boxes, self._mask_crop_size) + + # Prepare crop features. + multi_level_features = self._get_multilevel_features(fpn_features) + crop_features = spatial_transform_ops.single_level_feature_crop( + multi_level_features, level_outer_boxes, detection_prior_levels, + self._min_mask_level, self._mask_crop_size) + + # Predict and fuse shape priors. + shape_weights = self._classify_and_fuse_detection_priors( + uniform_priors, classes, crop_features) + fused_shape_priors = self._fuse_priors(shape_weights, classes) + fused_shape_priors = tf.reshape(fused_shape_priors, [ + batch_size, self._num_of_instances, self._mask_crop_size, + self._mask_crop_size + ]) + predicted_detection_priors = spatial_transform_ops.crop_mask_in_target_box( + fused_shape_priors, boxes, outer_boxes, self._mask_crop_size) + predicted_detection_priors = tf.reshape( + predicted_detection_priors, + [-1, self._mask_crop_size, self._mask_crop_size, 1]) + + return crop_features, predicted_detection_priors + + def _get_multilevel_features(self, fpn_features): + """Get multilevel features from FPN feature dictionary into one tensor. + + Args: + fpn_features: a dictionary of FPN features. + + Returns: + features: a float tensor of shape [batch_size, num_levels, + max_feature_size, max_feature_size, num_downsample_channels]. + """ + # TODO(yeqing): Recover reuse=tf.AUTO_REUSE logic. + with tf.name_scope('masknet'): + mask_feats = {} + # Reduce the feature dimension at each FPN level by convolution. + for feat_level in range(self._min_mask_level, self._max_mask_level + 1): + mask_feats[feat_level] = tf.keras.layers.Conv2D( + self._num_downsample_channels, + kernel_size=(1, 1), + bias_initializer=tf.zeros_initializer(), + kernel_initializer=tf.keras.initializers.RandomNormal(stddev=0.01), + padding='same', + name='mask-downsample')( + fpn_features[feat_level]) + + # Concat features through padding to the max size. + features = [mask_feats[self._min_mask_level]] + for feat_level in range(self._min_mask_level + 1, + self._max_mask_level + 1): + features.append(tf.image.pad_to_bounding_box( + mask_feats[feat_level], 0, 0, + self._max_feature_size, self._max_feature_size)) + + features = tf.stack(features, axis=1) + + return features + + def _compute_box_levels(self, boxes): + """Compute the box FPN levels. + + Args: + boxes: a float tensor of shape [batch_size, num_instances, 4]. + + Returns: + levels: a int tensor of shape [batch_size, num_instances]. + """ + object_sizes = tf.stack([ + boxes[:, :, 2] - boxes[:, :, 0], + boxes[:, :, 3] - boxes[:, :, 1], + ], axis=2) + object_sizes = tf.reduce_max(input_tensor=object_sizes, axis=2) + ratios = object_sizes / self._mask_crop_size + levels = tf.math.ceil(tf.math.log(ratios) / tf.math.log(2.)) + levels = tf.maximum(tf.minimum(levels, self._max_mask_level), + self._min_mask_level) + return levels + + def _classify_and_fuse_detection_priors(self, uniform_priors, + detection_prior_classes, + crop_features): + """Classify the uniform prior by predicting the shape modes. + + Classify the object crop features into K modes of the clusters for each + category. + + Args: + uniform_priors: A float Tensor of shape [batch_size, num_instances, + mask_size, mask_size] representing the uniform detection priors. + detection_prior_classes: A int Tensor of shape [batch_size, num_instances] + of detection class ids. + crop_features: A float Tensor of shape [batch_size * num_instances, + mask_size, mask_size, num_channels]. + + Returns: + shape_weights: A float Tensor of shape + [batch_size * num_instances, num_clusters] representing the classifier + output probability over all possible shapes. + """ + location_detection_priors = tf.reshape( + uniform_priors, [-1, self._mask_crop_size, self._mask_crop_size, 1]) + # Generate image embedding to shape. + fused_shape_features = crop_features * location_detection_priors + + shape_embedding = tf.reduce_mean( + input_tensor=fused_shape_features, axis=(1, 2)) + if not self._use_category_for_mask: + # TODO(weicheng) use custom op for performance + shape_logits = tf.keras.layers.Dense( + self._num_clusters, + kernel_initializer=tf.keras.initializers.RandomNormal(stddev=0.01))( + shape_embedding) + shape_logits = tf.reshape(shape_logits, + [-1, self._num_clusters]) / self._temperature + shape_weights = tf.nn.softmax(shape_logits, name='shape_prior_weights') + else: + shape_logits = tf.keras.layers.Dense( + self._mask_num_classes * self._num_clusters, + kernel_initializer=tf.keras.initializers.RandomNormal(stddev=0.01))( + shape_embedding) + shape_logits = tf.reshape( + shape_logits, [-1, self._mask_num_classes, self._num_clusters]) + training_classes = tf.reshape(detection_prior_classes, [-1]) + class_idx = tf.stack( + [tf.range(tf.size(input=training_classes)), training_classes - 1], + axis=1) + shape_logits = tf.gather_nd(shape_logits, class_idx) / self._temperature + shape_weights = tf.nn.softmax(shape_logits, name='shape_prior_weights') + + return shape_weights + + def _fuse_priors(self, shape_weights, detection_prior_classes): + """Fuse shape priors by the predicted shape probability. + + Args: + shape_weights: A float Tensor of shape [batch_size * num_instances, + num_clusters] of predicted shape probability distribution. + detection_prior_classes: A int Tensor of shape [batch_size, num_instances] + of detection class ids. + + Returns: + detection_priors: A float Tensor of shape [batch_size * num_instances, + mask_size, mask_size, 1]. + """ + if self._use_category_for_mask: + object_class_priors = tf.gather( + self.class_priors, detection_prior_classes) + else: + num_batch_instances = shape_weights.get_shape()[0] + object_class_priors = tf.tile( + tf.expand_dims(self.class_priors, 0), + [num_batch_instances, 1, 1, 1]) + + vector_class_priors = tf.reshape( + object_class_priors, + [-1, self._num_clusters, + self._mask_crop_size * self._mask_crop_size]) + detection_priors = tf.matmul( + tf.expand_dims(shape_weights, 1), vector_class_priors)[:, 0, :] + detection_priors = tf.reshape( + detection_priors, [-1, self._mask_crop_size, self._mask_crop_size, 1]) + return detection_priors + + +class ShapemaskCoarsemaskHead(object): + """ShapemaskCoarsemaskHead head.""" + + def __init__(self, + num_classes, + num_downsample_channels, + mask_crop_size, + use_category_for_mask, + num_convs): + """Initialize params to build ShapeMask coarse and fine prediction head. + + Args: + num_classes: `int` number of mask classification categories. + num_downsample_channels: `int` number of filters at mask head. + mask_crop_size: feature crop size. + use_category_for_mask: use class information in mask branch. + num_convs: `int` number of stacked convolution before the last prediction + layer. + """ + self._mask_num_classes = num_classes + self._num_downsample_channels = num_downsample_channels + self._mask_crop_size = mask_crop_size + self._use_category_for_mask = use_category_for_mask + self._num_convs = num_convs + if not use_category_for_mask: + assert num_classes == 1 + + def __call__(self, + crop_features, + detection_priors, + inst_classes, + is_training=None): + """Generate instance masks from FPN features and detection priors. + + This corresponds to the Fig. 5-6 of the ShapeMask paper at + https://arxiv.org/pdf/1904.03239.pdf + + Args: + crop_features: a float Tensor of shape [batch_size * num_instances, + mask_crop_size, mask_crop_size, num_downsample_channels]. This is the + instance feature crop. + detection_priors: a float Tensor of shape [batch_size * num_instances, + mask_crop_size, mask_crop_size, 1]. This is the detection prior for + the instance. + inst_classes: a int Tensor of shape [batch_size, num_instances] + of instance classes. + is_training: a bool indicating whether in training mode. + + Returns: + mask_outputs: instance mask prediction as a float Tensor of shape + [batch_size * num_instances, mask_size, mask_size, num_classes]. + """ + # Embed the anchor map into some feature space for anchor conditioning. + detection_prior_features = tf.keras.layers.Conv2D( + self._num_downsample_channels, + kernel_size=(1, 1), + bias_initializer=tf.zeros_initializer(), + kernel_initializer=tf.keras.initializers.RandomNormal( + mean=0., stddev=0.01), + padding='same', + name='anchor-conv')( + detection_priors) + + prior_conditioned_features = crop_features + detection_prior_features + coarse_output_features = self.coarsemask_decoder_net( + prior_conditioned_features, is_training) + + coarse_mask_classes = tf.keras.layers.Conv2D( + self._mask_num_classes, + kernel_size=(1, 1), + # Focal loss bias initialization to have foreground 0.01 probability. + bias_initializer=tf.constant_initializer(-np.log((1 - 0.01) / 0.01)), + kernel_initializer=tf.keras.initializers.RandomNormal( + mean=0, stddev=0.01), + padding='same', + name='class-predict')( + coarse_output_features) + + if self._use_category_for_mask: + inst_classes = tf.cast(tf.reshape(inst_classes, [-1]), tf.int32) + coarse_mask_classes_t = tf.transpose( + a=coarse_mask_classes, perm=(0, 3, 1, 2)) + # pylint: disable=g-long-lambda + coarse_mask_logits = tf.cond( + pred=tf.size(input=inst_classes) > 0, + true_fn=lambda: tf.gather_nd( + coarse_mask_classes_t, + tf.stack( + [tf.range(tf.size(input=inst_classes)), inst_classes - 1], + axis=1)), + false_fn=lambda: coarse_mask_classes_t[:, 0, :, :]) + # pylint: enable=g-long-lambda + coarse_mask_logits = tf.expand_dims(coarse_mask_logits, -1) + else: + coarse_mask_logits = coarse_mask_classes + + coarse_class_probs = tf.nn.sigmoid(coarse_mask_logits) + class_probs = tf.cast(coarse_class_probs, prior_conditioned_features.dtype) + + return coarse_mask_classes, class_probs, prior_conditioned_features + + def coarsemask_decoder_net(self, + images, + is_training=None, + batch_norm_relu=nn_ops.BatchNormRelu): + """Coarse mask decoder network architecture. + + Args: + images: A tensor of size [batch, height_in, width_in, channels_in]. + is_training: Whether batch_norm layers are in training mode. + batch_norm_relu: an operation that includes a batch normalization layer + followed by a relu layer(optional). + Returns: + images: A feature tensor of size [batch, output_size, output_size, + num_channels] + """ + for i in range(self._num_convs): + images = tf.keras.layers.Conv2D( + self._num_downsample_channels, + kernel_size=(3, 3), + bias_initializer=tf.zeros_initializer(), + kernel_initializer=tf.keras.initializers.RandomNormal(stddev=0.01), + activation=None, + padding='same', + name='coarse-class-%d' % i)( + images) + images = batch_norm_relu(name='coarse-class-%d-bn' % i)( + images, is_training=is_training) + + return images + + +class ShapemaskFinemaskHead(object): + """ShapemaskFinemaskHead head.""" + + def __init__(self, + num_classes, + num_downsample_channels, + mask_crop_size, + num_convs, + coarse_mask_thr, + gt_upsample_scale, + batch_norm_relu=nn_ops.BatchNormRelu): + """Initialize params to build ShapeMask coarse and fine prediction head. + + Args: + num_classes: `int` number of mask classification categories. + num_downsample_channels: `int` number of filters at mask head. + mask_crop_size: feature crop size. + num_convs: `int` number of stacked convolution before the last prediction + layer. + coarse_mask_thr: the threshold for suppressing noisy coarse prediction. + gt_upsample_scale: scale for upsampling groundtruths. + batch_norm_relu: an operation that includes a batch normalization layer + followed by a relu layer(optional). + """ + self._mask_num_classes = num_classes + self._num_downsample_channels = num_downsample_channels + self._mask_crop_size = mask_crop_size + self._num_convs = num_convs + self._coarse_mask_thr = coarse_mask_thr + self._gt_upsample_scale = gt_upsample_scale + + self._class_predict_conv = tf.keras.layers.Conv2D( + self._mask_num_classes, + kernel_size=(1, 1), + # Focal loss bias initialization to have foreground 0.01 probability. + bias_initializer=tf.constant_initializer(-np.log((1 - 0.01) / 0.01)), + kernel_initializer=tf.keras.initializers.RandomNormal( + mean=0, stddev=0.01), + padding='same', + name='affinity-class-predict') + self._upsample_conv = tf.keras.layers.Conv2DTranspose( + self._num_downsample_channels // 2, + (self._gt_upsample_scale, self._gt_upsample_scale), + (self._gt_upsample_scale, self._gt_upsample_scale)) + self._fine_class_conv = [] + self._fine_class_bn = [] + for i in range(self._num_convs): + self._fine_class_conv.append( + tf.keras.layers.Conv2D( + self._num_downsample_channels, + kernel_size=(3, 3), + bias_initializer=tf.zeros_initializer(), + kernel_initializer=tf.keras.initializers.RandomNormal( + stddev=0.01), + activation=None, + padding='same', + name='fine-class-%d' % i)) + self._fine_class_bn.append(batch_norm_relu(name='fine-class-%d-bn' % i)) + + def __call__(self, prior_conditioned_features, class_probs, is_training=None): + """Generate instance masks from FPN features and detection priors. + + This corresponds to the Fig. 5-6 of the ShapeMask paper at + https://arxiv.org/pdf/1904.03239.pdf + + Args: + prior_conditioned_features: a float Tensor of shape [batch_size * + num_instances, mask_crop_size, mask_crop_size, num_downsample_channels]. + This is the instance feature crop. + class_probs: a float Tensor of shape [batch_size * num_instances, + mask_crop_size, mask_crop_size, 1]. This is the class probability of + instance segmentation. + is_training: a bool indicating whether in training mode. + + Returns: + mask_outputs: instance mask prediction as a float Tensor of shape + [batch_size * num_instances, mask_size, mask_size, num_classes]. + """ + with backend.get_graph().as_default(), tf.name_scope('affinity-masknet'): + # Extract the foreground mean features + point_samp_prob_thr = 1. / (1. + tf.exp(-self._coarse_mask_thr)) + point_samp_prob_thr = tf.cast(point_samp_prob_thr, class_probs.dtype) + class_probs = tf.where( + tf.greater(class_probs, point_samp_prob_thr), class_probs, + tf.zeros_like(class_probs)) + weighted_features = class_probs * prior_conditioned_features + sum_class_vector = tf.reduce_sum( + input_tensor=class_probs, axis=(1, 2)) + tf.constant( + 1e-20, class_probs.dtype) + instance_embedding = tf.reduce_sum( + input_tensor=weighted_features, axis=(1, 2)) / sum_class_vector + + # Take the difference between crop features and mean instance features. + instance_features = prior_conditioned_features - tf.reshape( + instance_embedding, (-1, 1, 1, self._num_downsample_channels)) + + # Decoder to generate upsampled segmentation mask. + affinity_output_features = self.finemask_decoder_net( + instance_features, is_training) + + # Predict per-class instance masks. + affinity_mask_classes = self._class_predict_conv(affinity_output_features) + + return affinity_mask_classes + + def finemask_decoder_net(self, images, is_training=None): + """Fine mask decoder network architecture. + + Args: + images: A tensor of size [batch, height_in, width_in, channels_in]. + is_training: Whether batch_norm layers are in training mode. + + Returns: + images: A feature tensor of size [batch, output_size, output_size, + num_channels], where output size is self._gt_upsample_scale times + that of input. + """ + for i in range(self._num_convs): + images = self._fine_class_conv[i](images) + images = self._fine_class_bn[i](images, is_training=is_training) + + if self._gt_upsample_scale > 1: + images = self._upsample_conv(images) + + return images diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/identity.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/identity.py new file mode 100644 index 0000000..acc90c4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/identity.py @@ -0,0 +1,28 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Identity Fn that forwards the input features.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +class Identity(object): + """Identity function that forwards the input features.""" + + def __call__(self, features, is_training=False): + """Only forwards the input features.""" + return features + diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/nn_ops.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/nn_ops.py new file mode 100644 index 0000000..556d620 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/nn_ops.py @@ -0,0 +1,86 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Neural network operations commonly shared by the architectures.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import logging +import tensorflow.compat.v2 as tf +from tensorflow.python.keras import backend + + +class BatchNormRelu(tf.keras.layers.Layer): + """Combined Batch Normalization and ReLU layers.""" + + def __init__(self, + momentum=0.997, + epsilon=1e-4, + trainable=True, + relu=True, + init_zero=False, + fused=True, + name=None): + """A class to construct layers for a batch normalization followed by a ReLU. + + Args: + momentum: momentum for the moving average. + epsilon: small float added to variance to avoid dividing by zero. + trainable: `boolean`, if True also add variables to the graph collection + GraphKeys.TRAINABLE_VARIABLES. If False, freeze batch normalization + layer. + relu: `bool` if False, omits the ReLU operation. + init_zero: `bool` if True, initializes scale parameter of batch + normalization with 0. If False, initialize it with 1. + fused: `bool` fused option in batch normalziation. + name: `str` name for the operation. + """ + super(BatchNormRelu, self).__init__(trainable=trainable) + self._use_relu = relu + if init_zero: + gamma_initializer = tf.keras.initializers.Zeros() + else: + gamma_initializer = tf.keras.initializers.Ones() + self._batch_norm_op = tf.keras.layers.BatchNormalization( + momentum=momentum, + epsilon=epsilon, + center=True, + scale=True, + trainable=trainable, + fused=fused, + gamma_initializer=gamma_initializer, + name=name) + + def __call__(self, inputs, is_training=None): + """Builds layers for a batch normalization followed by a ReLU. + + Args: + inputs: `Tensor` of shape `[batch, channels, ...]`. + is_training: `boolean`, if True if model is in training mode. + + Returns: + A normalized `Tensor` with the same `data_format`. + """ + # We will need to keep training=None by default, so that it can be inherit + # from keras.Model.training + if is_training and self.trainable: + is_training = True + inputs = self._batch_norm_op(inputs, training=is_training) + + if self._use_relu: + inputs = tf.nn.relu(inputs) + return inputs + diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/resnet.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/resnet.py new file mode 100644 index 0000000..ccd4c5c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/architecture/resnet.py @@ -0,0 +1,304 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains definitions for the post-activation form of Residual Networks. + +Residual networks (ResNets) were proposed in: +[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Deep Residual Learning for Image Recognition. arXiv:1512.03385 +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import logging +import tensorflow.compat.v2 as tf +from tensorflow.python.keras import backend +from official.vision.detection.modeling.architecture import nn_ops + +# TODO(b/140112644): Refactor the code with Keras style, i.e. build and call. +class Resnet(object): + """Class to build ResNet family model.""" + + def __init__(self, + resnet_depth, + batch_norm_relu=nn_ops.BatchNormRelu, + data_format='channels_last'): + """ResNet initialization function. + + Args: + resnet_depth: `int` depth of ResNet backbone model. + batch_norm_relu: an operation that includes a batch normalization layer + followed by a relu layer(optional). + data_format: `str` either "channels_first" for `[batch, channels, height, + width]` or "channels_last for `[batch, height, width, channels]`. + """ + self._resnet_depth = resnet_depth + + self._batch_norm_relu = batch_norm_relu + + self._data_format = data_format + + model_params = { + 10: {'block': self.residual_block, 'layers': [1, 1, 1, 1]}, + 18: {'block': self.residual_block, 'layers': [2, 2, 2, 2]}, + 34: {'block': self.residual_block, 'layers': [3, 4, 6, 3]}, + 50: {'block': self.bottleneck_block, 'layers': [3, 4, 6, 3]}, + 101: {'block': self.bottleneck_block, 'layers': [3, 4, 23, 3]}, + 152: {'block': self.bottleneck_block, 'layers': [3, 8, 36, 3]}, + 200: {'block': self.bottleneck_block, 'layers': [3, 24, 36, 3]} + } + + if resnet_depth not in model_params: + valid_resnet_depths = ', '.join( + [str(depth) for depth in sorted(model_params.keys())]) + raise ValueError( + 'The resnet_depth should be in [%s]. Not a valid resnet_depth:'%( + valid_resnet_depths), self._resnet_depth) + params = model_params[resnet_depth] + self._resnet_fn = self.resnet_v1_generator( + params['block'], params['layers']) + + def __call__(self, inputs, is_training=None): + """Returns the ResNet model for a given size and number of output classes. + + Args: + inputs: a `Tesnor` with shape [batch_size, height, width, 3] representing + a batch of images. + is_training: `bool` if True, the model is in training mode. + + Returns: + a `dict` containing `int` keys for continuous feature levels [2, 3, 4, 5]. + The values are corresponding feature hierarchy in ResNet with shape + [batch_size, height_l, width_l, num_filters]. + """ + with backend.get_graph().as_default(): + with tf.name_scope('resnet%s' % self._resnet_depth): + return self._resnet_fn(inputs, is_training) + + def fixed_padding(self, inputs, kernel_size): + """Pads the input along the spatial dimensions independently of input size. + + Args: + inputs: `Tensor` of size `[batch, channels, height, width]` or + `[batch, height, width, channels]` depending on `data_format`. + kernel_size: `int` kernel size to be used for `conv2d` or max_pool2d` + operations. Should be a positive integer. + + Returns: + A padded `Tensor` of the same `data_format` with size either intact + (if `kernel_size == 1`) or padded (if `kernel_size > 1`). + """ + pad_total = kernel_size - 1 + pad_beg = pad_total // 2 + pad_end = pad_total - pad_beg + if self._data_format == 'channels_first': + padded_inputs = tf.pad( + tensor=inputs, + paddings=[[0, 0], [0, 0], [pad_beg, pad_end], [pad_beg, pad_end]]) + else: + padded_inputs = tf.pad( + tensor=inputs, + paddings=[[0, 0], [pad_beg, pad_end], [pad_beg, pad_end], [0, 0]]) + + return padded_inputs + + def conv2d_fixed_padding(self, inputs, filters, kernel_size, strides): + """Strided 2-D convolution with explicit padding. + + The padding is consistent and is based only on `kernel_size`, not on the + dimensions of `inputs` (as opposed to using `tf.layers.conv2d` alone). + + Args: + inputs: `Tensor` of size `[batch, channels, height_in, width_in]`. + filters: `int` number of filters in the convolution. + kernel_size: `int` size of the kernel to be used in the convolution. + strides: `int` strides of the convolution. + + Returns: + A `Tensor` of shape `[batch, filters, height_out, width_out]`. + """ + if strides > 1: + inputs = self.fixed_padding(inputs, kernel_size) + + return tf.keras.layers.Conv2D( + filters=filters, + kernel_size=kernel_size, + strides=strides, + padding=('SAME' if strides == 1 else 'VALID'), + use_bias=False, + kernel_initializer=tf.initializers.VarianceScaling(), + data_format=self._data_format)( + inputs=inputs) + + def residual_block(self, + inputs, + filters, + strides, + use_projection=False, + is_training=None): + """Standard building block for residual networks with BN after convolutions. + + Args: + inputs: `Tensor` of size `[batch, channels, height, width]`. + filters: `int` number of filters for the first two convolutions. Note that + the third and final convolution will use 4 times as many filters. + strides: `int` block stride. If greater than 1, this block will ultimately + downsample the input. + use_projection: `bool` for whether this block should use a projection + shortcut (versus the default identity shortcut). This is usually + `True` for the first block of a block group, which may change the + number of filters and the resolution. + is_training: `bool` if True, the model is in training mode. + Returns: + The output `Tensor` of the block. + """ + shortcut = inputs + if use_projection: + # Projection shortcut in first layer to match filters and strides + shortcut = self.conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=1, strides=strides) + shortcut = self._batch_norm_relu(relu=False)( + shortcut, is_training=is_training) + + inputs = self.conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=3, strides=strides) + inputs = self._batch_norm_relu()(inputs, is_training=is_training) + + inputs = self.conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=3, strides=1) + inputs = self._batch_norm_relu()( + inputs, relu=False, init_zero=True, is_training=is_training) + + return tf.nn.relu(inputs + shortcut) + + def bottleneck_block(self, + inputs, + filters, + strides, + use_projection=False, + is_training=None): + """Bottleneck block variant for residual networks with BN after convolutions. + + Args: + inputs: `Tensor` of size `[batch, channels, height, width]`. + filters: `int` number of filters for the first two convolutions. Note that + the third and final convolution will use 4 times as many filters. + strides: `int` block stride. If greater than 1, this block will ultimately + downsample the input. + use_projection: `bool` for whether this block should use a projection + shortcut (versus the default identity shortcut). This is usually + `True` for the first block of a block group, which may change the + number of filters and the resolution. + is_training: `bool` if True, the model is in training mode. + + Returns: + The output `Tensor` of the block. + """ + shortcut = inputs + if use_projection: + # Projection shortcut only in first block within a group. Bottleneck + # blocks end with 4 times the number of filters. + filters_out = 4 * filters + shortcut = self.conv2d_fixed_padding( + inputs=inputs, filters=filters_out, kernel_size=1, strides=strides) + shortcut = self._batch_norm_relu(relu=False)( + shortcut, is_training=is_training) + + inputs = self.conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=1, strides=1) + inputs = self._batch_norm_relu()(inputs, is_training=is_training) + + inputs = self.conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=3, strides=strides) + inputs = self._batch_norm_relu()(inputs, is_training=is_training) + + inputs = self.conv2d_fixed_padding( + inputs=inputs, filters=4 * filters, kernel_size=1, strides=1) + inputs = self._batch_norm_relu( + relu=False, init_zero=True)( + inputs, is_training=is_training) + + return tf.nn.relu(inputs + shortcut) + + def block_group(self, inputs, filters, block_fn, blocks, strides, name, + is_training): + """Creates one group of blocks for the ResNet model. + + Args: + inputs: `Tensor` of size `[batch, channels, height, width]`. + filters: `int` number of filters for the first convolution of the layer. + block_fn: `function` for the block to use within the model + blocks: `int` number of blocks contained in the layer. + strides: `int` stride to use for the first convolution of the layer. If + greater than 1, this layer will downsample the input. + name: `str`name for the Tensor output of the block layer. + is_training: `bool` if True, the model is in training mode. + + Returns: + The output `Tensor` of the block layer. + """ + # Only the first block per block_group uses projection shortcut and strides. + inputs = block_fn(inputs, filters, strides, use_projection=True, + is_training=is_training) + + for _ in range(1, blocks): + inputs = block_fn(inputs, filters, 1, is_training=is_training) + + return tf.identity(inputs, name) + + def resnet_v1_generator(self, block_fn, layers): + """Generator for ResNet v1 models. + + Args: + block_fn: `function` for the block to use within the model. Either + `residual_block` or `bottleneck_block`. + layers: list of 4 `int`s denoting the number of blocks to include in each + of the 4 block groups. Each group consists of blocks that take inputs of + the same resolution. + + Returns: + Model `function` that takes in `inputs` and `is_training` and returns the + output `Tensor` of the ResNet model. + """ + + def model(inputs, is_training=None): + """Creation of the model graph.""" + inputs = self.conv2d_fixed_padding( + inputs=inputs, filters=64, kernel_size=7, strides=2) + inputs = tf.identity(inputs, 'initial_conv') + inputs = self._batch_norm_relu()(inputs, is_training=is_training) + + inputs = tf.keras.layers.MaxPool2D( + pool_size=3, strides=2, padding='SAME', + data_format=self._data_format)( + inputs) + inputs = tf.identity(inputs, 'initial_max_pool') + + c2 = self.block_group( + inputs=inputs, filters=64, block_fn=block_fn, blocks=layers[0], + strides=1, name='block_group1', is_training=is_training) + c3 = self.block_group( + inputs=c2, filters=128, block_fn=block_fn, blocks=layers[1], + strides=2, name='block_group2', is_training=is_training) + c4 = self.block_group( + inputs=c3, filters=256, block_fn=block_fn, blocks=layers[2], + strides=2, name='block_group3', is_training=is_training) + c5 = self.block_group( + inputs=c4, filters=512, block_fn=block_fn, blocks=layers[3], + strides=2, name='block_group4', is_training=is_training) + return {2: c2, 3: c3, 4: c4, 5: c5} + + return model diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/base_model.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/base_model.py new file mode 100644 index 0000000..65778a4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/base_model.py @@ -0,0 +1,167 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Base Model definition.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import abc +import functools +import re +import tensorflow.compat.v2 as tf +from official.vision.detection.modeling import checkpoint_utils +from official.vision.detection.modeling import learning_rates + + +class OptimizerFactory(object): + """Class to generate optimizer function.""" + + def __init__(self, params): + """Creates optimized based on the specified flags.""" + if params.type == 'momentum': + nesterov = False + try: + nesterov = params.nesterov + except AttributeError: + pass + self._optimizer = functools.partial( + tf.keras.optimizers.SGD, + momentum=params.momentum, + nesterov=nesterov) + elif params.type == 'adam': + self._optimizer = tf.keras.optimizers.Adam + elif params.type == 'adadelta': + self._optimizer = tf.keras.optimizers.Adadelta + elif params.type == 'adagrad': + self._optimizer = tf.keras.optimizers.Adagrad + elif params.type == 'rmsprop': + self._optimizer = functools.partial( + tf.keras.optimizers.RMSprop, momentum=params.momentum) + else: + raise ValueError('Unsupported optimizer type %s.' % self._optimizer) + + def __call__(self, learning_rate): + return self._optimizer(learning_rate=learning_rate) + + +def _make_filter_trainable_variables_fn(frozen_variable_prefix): + """Creates a function for filtering trainable varialbes.""" + + def _filter_trainable_variables(variables): + """Filters trainable varialbes. + + Args: + variables: a list of tf.Variable to be filtered. + + Returns: + filtered_variables: a list of tf.Variable filtered out the frozen ones. + """ + # frozen_variable_prefix: a regex string specifing the prefix pattern of + # the frozen variables' names. + filtered_variables = [ + v for v in variables + if not re.match(frozen_variable_prefix, v.name) + ] + return filtered_variables + + return _filter_trainable_variables + + +class Model(object): + """Base class for model function.""" + + __metaclass__ = abc.ABCMeta + + def __init__(self, params): + self._use_bfloat16 = params.architecture.use_bfloat16 + + if params.architecture.use_bfloat16: + policy = tf.compat.v2.keras.mixed_precision.experimental.Policy( + 'mixed_bfloat16') + tf.compat.v2.keras.mixed_precision.experimental.set_policy(policy) + + # Optimization. + self._optimizer_fn = OptimizerFactory(params.train.optimizer) + self._learning_rate = learning_rates.learning_rate_generator( + params.train.learning_rate) + + self._frozen_variable_prefix = params.train.frozen_variable_prefix + self._regularization_var_regex = params.train.regularization_variable_regex + self._l2_weight_decay = params.train.l2_weight_decay + + # Checkpoint restoration. + self._checkpoint = params.train.checkpoint.as_dict() + + # Summary. + self._enable_summary = params.enable_summary + self._model_dir = params.model_dir + + @abc.abstractmethod + def build_outputs(self, inputs, mode): + """Build the graph of the forward path.""" + pass + + @abc.abstractmethod + def build_model(self, params, mode): + """Build the model object.""" + pass + + @abc.abstractmethod + def build_loss_fn(self): + """Build the model object.""" + pass + + def post_processing(self, labels, outputs): + """Post-processing function.""" + return labels, outputs + + def model_outputs(self, inputs, mode): + """Build the model outputs.""" + return self.build_outputs(inputs, mode) + + def build_optimizer(self): + """Returns train_op to optimize total loss.""" + # Sets up the optimizer. + return self._optimizer_fn(self._learning_rate) + + def make_filter_trainable_variables_fn(self): + """Creates a function for filtering trainable varialbes.""" + return _make_filter_trainable_variables_fn(self._frozen_variable_prefix) + + def weight_decay_loss(self, trainable_variables): + reg_variables = [ + v for v in trainable_variables + if self._regularization_var_regex is None + or re.match(self._regularization_var_regex, v.name) + ] + + return self._l2_weight_decay * tf.add_n( + [tf.nn.l2_loss(v) for v in reg_variables]) + + def make_restore_checkpoint_fn(self): + """Returns scaffold function to restore parameters from v1 checkpoint.""" + if 'skip_checkpoint_variables' in self._checkpoint: + skip_regex = self._checkpoint['skip_checkpoint_variables'] + else: + skip_regex = None + return checkpoint_utils.make_restore_checkpoint_fn( + self._checkpoint['path'], + prefix=self._checkpoint['prefix'], + skip_regex=skip_regex) + + def eval_metrics(self): + """Returns tuple of metric function and its inputs for evaluation.""" + raise NotImplementedError('Unimplemented eval_metrics') diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/checkpoint_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/checkpoint_utils.py new file mode 100644 index 0000000..a4346e6 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/checkpoint_utils.py @@ -0,0 +1,131 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Util functions for loading checkpoints. Especially for loading Tensorflow 1.x +checkpoint to Tensorflow 2.x (keras) model. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +import re +from absl import logging + +import tensorflow.compat.v2 as tf + + +def _build_assignment_map(keras_model, + prefix='', + skip_variables_regex=None, + var_to_shape_map=None): + """Compute an assignment mapping for loading older checkpoints into a Keras + model. Variable names are remapped from the original TPUEstimator model to + the new Keras name. + + Args: + keras_model: tf.keras.Model object to provide variables to assign. + prefix: prefix in the variable name to be remove for alignment with names in + the checkpoint. + skip_variables_regex: regular expression to math the names of variables that + do not need to be assign. + var_to_shape_map: variable name to shape mapping from the checkpoint. + + Returns: + The variable assignment map. + """ + assignment_map = {} + + + checkpoint_names = None + if var_to_shape_map: + checkpoint_names = list(filter( + lambda x: not x.endswith('Momentum') and not x.endswith( + 'global_step'), var_to_shape_map.keys())) + + for var in keras_model.variables: + var_name = var.name + + if skip_variables_regex and re.match(skip_variables_regex, var_name): + continue + # Trim the index of the variable. + if ':' in var_name: + var_name = var_name[:var_name.rindex(':')] + if var_name.startswith(prefix): + var_name = var_name[len(prefix):] + + if not var_to_shape_map: + assignment_map[var_name] = var + continue + + # Match name with variables in the checkpoint. + match_names = list(filter(lambda x: x.endswith(var_name), checkpoint_names)) + try: + if match_names: + assert len(match_names) == 1, 'more then on matches for {}: {}'.format( + var_name, match_names) + checkpoint_names.remove(match_names[0]) + assignment_map[match_names[0]] = var + else: + logging.info('Error not found var name: %s', var_name) + except Exception as e: + logging.info('Error removing the match_name: %s', match_names) + logging.info('Exception: %s', e) + raise + logging.info('Found variable in checkpoint: %d', len(assignment_map)) + return assignment_map + + +def _get_checkpoint_map(checkpoint_path): + reader = tf.train.load_checkpoint(checkpoint_path) + return reader.get_variable_to_shape_map() + + +def make_restore_checkpoint_fn(checkpoint_path, prefix='', skip_regex=None): + """Returns scaffold function to restore parameters from v1 checkpoint. + Args: + checkpoint_path: path of the checkpoint folder or file. + Example 1: '/path/to/model_dir/' + Example 2: '/path/to/model.ckpt-22500' + prefix: prefix in the variable name to be remove for alignment with names in + the checkpoint. + skip_regex: regular expression to math the names of variables that + do not need to be assign. + + Returns: + Callable[tf.kears.Model] -> void. Fn to load v1 checkpoint to keras model. + """ + + def _restore_checkpoint_fn(keras_model): + """Loads pretrained model through scaffold function.""" + if not checkpoint_path: + logging.info('checkpoint_path is empty') + return + var_prefix = prefix + if prefix and not prefix.endswith('/'): + var_prefix += '/' + var_to_shape_map = _get_checkpoint_map(checkpoint_path) + assert var_to_shape_map, 'var_to_shape_map should not be empty' + vars_to_load = _build_assignment_map( + keras_model, + prefix=var_prefix, + skip_variables_regex=skip_regex, + var_to_shape_map=var_to_shape_map) + if not vars_to_load: + raise ValueError('Variables to load is empty.') + tf.compat.v1.train.init_from_checkpoint(checkpoint_path, + vars_to_load) + + return _restore_checkpoint_fn diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/factory.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/factory.py new file mode 100644 index 0000000..bc7c33f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/factory.py @@ -0,0 +1,31 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Factory to build detection model.""" + + +from official.vision.detection.modeling import maskrcnn_model +from official.vision.detection.modeling import retinanet_model + + +def model_generator(params): + """Model function generator.""" + if params.type == 'retinanet': + model_fn = retinanet_model.RetinanetModel(params) + elif params.type == 'mask_rcnn': + model_fn = maskrcnn_model.MaskrcnnModel(params) + else: + raise ValueError('Model %s is not supported.'% params.type) + + return model_fn diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/learning_rates.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/learning_rates.py new file mode 100644 index 0000000..99d5d7c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/learning_rates.py @@ -0,0 +1,96 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Learning rate schedule.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools + +import numpy as np +import tensorflow.compat.v2 as tf +from official.modeling.hyperparams import params_dict + + +class StepLearningRateWithLinearWarmup(tf.keras.optimizers.schedules.LearningRateSchedule): + """Class to generate learning rate tensor.""" + + def __init__(self, params): + """Creates the step learning rate tensor with linear warmup.""" + super(StepLearningRateWithLinearWarmup, self).__init__() + assert isinstance(params, (dict, params_dict.ParamsDict)) + if isinstance(params, dict): + params = params_dict.ParamsDict(params) + self._params = params + + def __call__(self, global_step): + warmup_lr = self._params.warmup_learning_rate + warmup_steps = self._params.warmup_steps + init_lr = self._params.init_learning_rate + lr_levels = self._params.learning_rate_levels + lr_steps = self._params.learning_rate_steps + linear_warmup = ( + warmup_lr + tf.cast(global_step, dtype=tf.float32) / warmup_steps * + (init_lr - warmup_lr)) + learning_rate = tf.where(global_step < warmup_steps, linear_warmup, init_lr) + + for next_learning_rate, start_step in zip(lr_levels, lr_steps): + learning_rate = tf.where(global_step >= start_step, next_learning_rate, + learning_rate) + return learning_rate + + def get_config(self): + return {'_params': self._params.as_dict()} + + +class CosineLearningRateWithLinearWarmup(tf.keras.optimizers.schedules.LearningRateSchedule): + """Class to generate learning rate tensor.""" + + def __init__(self, params): + """Creates the consine learning rate tensor with linear warmup.""" + super(CosineLearningRateWithLinearWarmup, self).__init__() + assert isinstance(params, (dict, params_dict.ParamsDict)) + if isinstance(params, dict): + params = params_dict.ParamsDict(params) + self._params = params + + def __call__(self, global_step): + global_step = tf.cast(global_step, dtype=tf.float32) + warmup_lr = self._params.warmup_learning_rate + warmup_steps = self._params.warmup_steps + init_lr = self._params.init_learning_rate + total_steps = self._params.total_steps + linear_warmup = ( + warmup_lr + global_step / warmup_steps * (init_lr - warmup_lr)) + cosine_learning_rate = ( + init_lr * (tf.cos(np.pi * (global_step - warmup_steps) / + (total_steps - warmup_steps)) + 1.0) / 2.0) + learning_rate = tf.where(global_step < warmup_steps, linear_warmup, + cosine_learning_rate) + return learning_rate + + def get_config(self): + return {'_params': self._params.as_dict()} + + +def learning_rate_generator(params): + """The learning rate function generator.""" + if params.type == 'step': + return StepLearningRateWithLinearWarmup(params) + elif params.type == 'cosine': + return CosineLearningRateWithLinearWarmup(params) + else: + raise ValueError('Unsupported learning rate type: {}.'.format(params.type)) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/losses.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/losses.py new file mode 100644 index 0000000..7573349 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/losses.py @@ -0,0 +1,490 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Losses used for Mask-RCNN.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import logging +import tensorflow.compat.v2 as tf + + +def focal_loss(logits, targets, alpha, gamma, normalizer): + """Compute the focal loss between `logits` and the golden `target` values. + + Focal loss = -(1-pt)^gamma * log(pt) + where pt is the probability of being classified to the true class. + + Args: + logits: A float32 tensor of size + [batch, height_in, width_in, num_predictions]. + targets: A float32 tensor of size + [batch, height_in, width_in, num_predictions]. + alpha: A float32 scalar multiplying alpha to the loss from positive examples + and (1-alpha) to the loss from negative examples. + gamma: A float32 scalar modulating loss from hard and easy examples. + normalizer: A float32 scalar normalizes the total loss from all examples. + + Returns: + loss: A float32 Tensor of size [batch, height_in, width_in, num_predictions] + representing normalized loss on the prediction map. + """ + with tf.name_scope('focal_loss'): + positive_label_mask = tf.math.equal(targets, 1.0) + cross_entropy = ( + tf.nn.sigmoid_cross_entropy_with_logits(labels=targets, logits=logits)) + # Below are comments/derivations for computing modulator. + # For brevity, let x = logits, z = targets, r = gamma, and p_t = sigmod(x) + # for positive samples and 1 - sigmoid(x) for negative examples. + # + # The modulator, defined as (1 - P_t)^r, is a critical part in focal loss + # computation. For r > 0, it puts more weights on hard examples, and less + # weights on easier ones. However if it is directly computed as (1 - P_t)^r, + # its back-propagation is not stable when r < 1. The implementation here + # resolves the issue. + # + # For positive samples (labels being 1), + # (1 - p_t)^r + # = (1 - sigmoid(x))^r + # = (1 - (1 / (1 + exp(-x))))^r + # = (exp(-x) / (1 + exp(-x)))^r + # = exp(log((exp(-x) / (1 + exp(-x)))^r)) + # = exp(r * log(exp(-x)) - r * log(1 + exp(-x))) + # = exp(- r * x - r * log(1 + exp(-x))) + # + # For negative samples (labels being 0), + # (1 - p_t)^r + # = (sigmoid(x))^r + # = (1 / (1 + exp(-x)))^r + # = exp(log((1 / (1 + exp(-x)))^r)) + # = exp(-r * log(1 + exp(-x))) + # + # Therefore one unified form for positive (z = 1) and negative (z = 0) + # samples is: + # (1 - p_t)^r = exp(-r * z * x - r * log(1 + exp(-x))). + neg_logits = -1.0 * logits + modulator = tf.math.exp(gamma * targets * neg_logits - + gamma * tf.math.log1p(tf.math.exp(neg_logits))) + loss = modulator * cross_entropy + weighted_loss = tf.where(positive_label_mask, alpha * loss, + (1.0 - alpha) * loss) + weighted_loss /= normalizer + return weighted_loss + + +class RpnScoreLoss(object): + """Region Proposal Network score loss function.""" + + def __init__(self, params): + self._rpn_batch_size_per_im = params.rpn_batch_size_per_im + self._binary_crossentropy = tf.keras.losses.BinaryCrossentropy( + reduction=tf.keras.losses.Reduction.SUM, from_logits=True) + + def __call__(self, score_outputs, labels): + """Computes total RPN detection loss. + + Computes total RPN detection loss including box and score from all levels. + + Args: + score_outputs: an OrderDict with keys representing levels and values + representing scores in [batch_size, height, width, num_anchors]. + labels: the dictionary that returned from dataloader that includes + groundturth targets. + + Returns: + rpn_score_loss: a scalar tensor representing total score loss. + """ + with tf.name_scope('rpn_loss'): + levels = sorted(score_outputs.keys()) + + score_losses = [] + for level in levels: + score_losses.append( + self._rpn_score_loss( + score_outputs[level], + labels[level], + normalizer=tf.cast( + tf.shape(score_outputs[level])[0] * + self._rpn_batch_size_per_im, dtype=tf.float32))) + + # Sums per level losses to total loss. + return tf.math.add_n(score_losses) + + def _rpn_score_loss(self, score_outputs, score_targets, normalizer=1.0): + """Computes score loss.""" + # score_targets has three values: + # (1) score_targets[i]=1, the anchor is a positive sample. + # (2) score_targets[i]=0, negative. + # (3) score_targets[i]=-1, the anchor is don't care (ignore). + with tf.name_scope('rpn_score_loss'): + mask = tf.math.logical_or(tf.math.equal(score_targets, 1), + tf.math.equal(score_targets, 0)) + + score_targets = tf.math.maximum(score_targets, + tf.zeros_like(score_targets)) + + score_targets = tf.expand_dims(score_targets, axis=-1) + score_outputs = tf.expand_dims(score_outputs, axis=-1) + score_loss = self._binary_crossentropy( + score_targets, score_outputs, sample_weight=mask) + + score_loss /= normalizer + return score_loss + + +class RpnBoxLoss(object): + """Region Proposal Network box regression loss function.""" + + def __init__(self, params): + logging.info('RpnBoxLoss huber_loss_delta %s', params.huber_loss_delta) + # The delta is typically around the mean value of regression target. + # for instances, the regression targets of 512x512 input with 6 anchors on + # P2-P6 pyramid is about [0.1, 0.1, 0.2, 0.2]. + self._huber_loss = tf.keras.losses.Huber( + delta=params.huber_loss_delta, reduction=tf.keras.losses.Reduction.SUM) + + def __call__(self, box_outputs, labels): + """Computes total RPN detection loss. + + Computes total RPN detection loss including box and score from all levels. + + Args: + box_outputs: an OrderDict with keys representing levels and values + representing box regression targets in + [batch_size, height, width, num_anchors * 4]. + labels: the dictionary that returned from dataloader that includes + groundturth targets. + + Returns: + rpn_box_loss: a scalar tensor representing total box regression loss. + """ + with tf.name_scope('rpn_loss'): + levels = sorted(box_outputs.keys()) + + box_losses = [] + for level in levels: + box_losses.append(self._rpn_box_loss(box_outputs[level], labels[level])) + + # Sum per level losses to total loss. + return tf.add_n(box_losses) + + def _rpn_box_loss(self, box_outputs, box_targets, normalizer=1.0): + """Computes box regression loss.""" + with tf.name_scope('rpn_box_loss'): + mask = tf.cast(tf.not_equal(box_targets, 0.0), dtype=tf.float32) + box_targets = tf.expand_dims(box_targets, axis=-1) + box_outputs = tf.expand_dims(box_outputs, axis=-1) + box_loss = self._huber_loss(box_targets, box_outputs, sample_weight=mask) + # The loss is normalized by the sum of non-zero weights and additional + # normalizer provided by the function caller. Using + 0.01 here to avoid + # division by zero. + box_loss /= normalizer * (tf.reduce_sum(mask) + 0.01) + return box_loss + + +class FastrcnnClassLoss(object): + """Fast R-CNN classification loss function.""" + + def __init__(self): + self._categorical_crossentropy = tf.keras.losses.CategoricalCrossentropy( + reduction=tf.keras.losses.Reduction.SUM, from_logits=True) + + def __call__(self, class_outputs, class_targets): + """Computes the class loss (Fast-RCNN branch) of Mask-RCNN. + + This function implements the classification loss of the Fast-RCNN. + + The classification loss is softmax on all RoIs. + Reference: https://github.com/facebookresearch/Detectron/blob/master/detectron/modeling/fast_rcnn_heads.py # pylint: disable=line-too-long + + Args: + class_outputs: a float tensor representing the class prediction for each box + with a shape of [batch_size, num_boxes, num_classes]. + class_targets: a float tensor representing the class label for each box + with a shape of [batch_size, num_boxes]. + + Returns: + a scalar tensor representing total class loss. + """ + with tf.name_scope('fast_rcnn_loss'): + batch_size, num_boxes, num_classes = class_outputs.get_shape().as_list() + class_targets = tf.cast(class_targets, dtype=tf.int32) + class_targets_one_hot = tf.one_hot(class_targets, num_classes) + return self._fast_rcnn_class_loss(class_outputs, class_targets_one_hot, + normalizer=batch_size * num_boxes / 2.0) + + def _fast_rcnn_class_loss(self, class_outputs, class_targets_one_hot, + normalizer): + """Computes classification loss.""" + with tf.name_scope('fast_rcnn_class_loss'): + class_loss = self._categorical_crossentropy(class_targets_one_hot, + class_outputs) + + class_loss /= normalizer + return class_loss + + +class FastrcnnBoxLoss(object): + """Fast R-CNN box regression loss function.""" + + def __init__(self, params): + logging.info('FastrcnnBoxLoss huber_loss_delta %s', params.huber_loss_delta) + # The delta is typically around the mean value of regression target. + # for instances, the regression targets of 512x512 input with 6 anchors on + # P2-P6 pyramid is about [0.1, 0.1, 0.2, 0.2]. + self._huber_loss = tf.keras.losses.Huber( + delta=params.huber_loss_delta, reduction=tf.keras.losses.Reduction.SUM) + + def __call__(self, box_outputs, class_targets, box_targets): + """Computes the box loss (Fast-RCNN branch) of Mask-RCNN. + + This function implements the box regression loss of the Fast-RCNN. As the + `box_outputs` produces `num_classes` boxes for each RoI, the reference model + expands `box_targets` to match the shape of `box_outputs` and selects only + the target that the RoI has a maximum overlap. (Reference: https://github.com/facebookresearch/Detectron/blob/master/detectron/roi_data/fast_rcnn.py) # pylint: disable=line-too-long + Instead, this function selects the `box_outputs` by the `class_targets` so + that it doesn't expand `box_targets`. + + The box loss is smooth L1-loss on only positive samples of RoIs. + Reference: https://github.com/facebookresearch/Detectron/blob/master/detectron/modeling/fast_rcnn_heads.py # pylint: disable=line-too-long + + Args: + box_outputs: a float tensor representing the box prediction for each box + with a shape of [batch_size, num_boxes, num_classes * 4]. + class_targets: a float tensor representing the class label for each box + with a shape of [batch_size, num_boxes]. + box_targets: a float tensor representing the box label for each box + with a shape of [batch_size, num_boxes, 4]. + + Returns: + box_loss: a scalar tensor representing total box regression loss. + """ + with tf.name_scope('fast_rcnn_loss'): + class_targets = tf.cast(class_targets, dtype=tf.int32) + + # Selects the box from `box_outputs` based on `class_targets`, with which + # the box has the maximum overlap. + (batch_size, num_rois, + num_class_specific_boxes) = box_outputs.get_shape().as_list() + num_classes = num_class_specific_boxes // 4 + box_outputs = tf.reshape(box_outputs, + [batch_size, num_rois, num_classes, 4]) + + box_indices = tf.reshape( + class_targets + tf.tile( + tf.expand_dims( + tf.range(batch_size) * num_rois * num_classes, 1), + [1, num_rois]) + tf.tile( + tf.expand_dims(tf.range(num_rois) * num_classes, 0), + [batch_size, 1]), [-1]) + + box_outputs = tf.matmul( + tf.one_hot( + box_indices, + batch_size * num_rois * num_classes, + dtype=box_outputs.dtype), tf.reshape(box_outputs, [-1, 4])) + box_outputs = tf.reshape(box_outputs, [batch_size, -1, 4]) + + return self._fast_rcnn_box_loss(box_outputs, box_targets, class_targets) + + def _fast_rcnn_box_loss(self, box_outputs, box_targets, class_targets, + normalizer=1.0): + """Computes box regression loss.""" + with tf.name_scope('fast_rcnn_box_loss'): + mask = tf.tile(tf.expand_dims(tf.greater(class_targets, 0), axis=2), + [1, 1, 4]) + mask = tf.cast(mask, dtype=tf.float32) + box_targets = tf.expand_dims(box_targets, axis=-1) + box_outputs = tf.expand_dims(box_outputs, axis=-1) + box_loss = self._huber_loss(box_targets, box_outputs, sample_weight=mask) + # The loss is normalized by the number of ones in mask, + # additianal normalizer provided by the user and using 0.01 here to avoid + # division by 0. + box_loss /= normalizer * (tf.reduce_sum(mask) + 0.01) + return box_loss + + +class MaskrcnnLoss(object): + """Mask R-CNN instance segmentation mask loss function.""" + + def __init__(self): + self._binary_crossentropy = tf.keras.losses.BinaryCrossentropy( + reduction=tf.keras.losses.Reduction.SUM, from_logits=True) + + def __call__(self, mask_outputs, mask_targets, select_class_targets): + """Computes the mask loss of Mask-RCNN. + + This function implements the mask loss of Mask-RCNN. As the `mask_outputs` + produces `num_classes` masks for each RoI, the reference model expands + `mask_targets` to match the shape of `mask_outputs` and selects only the + target that the RoI has a maximum overlap. (Reference: https://github.com/facebookresearch/Detectron/blob/master/detectron/roi_data/mask_rcnn.py) # pylint: disable=line-too-long + Instead, this implementation selects the `mask_outputs` by the `class_targets` + so that it doesn't expand `mask_targets`. Note that the selection logic is + done in the post-processing of mask_rcnn_fn in mask_rcnn_architecture.py. + + Args: + mask_outputs: a float tensor representing the prediction for each mask, + with a shape of + [batch_size, num_masks, mask_height, mask_width]. + mask_targets: a float tensor representing the binary mask of ground truth + labels for each mask with a shape of + [batch_size, num_masks, mask_height, mask_width]. + select_class_targets: a tensor with a shape of [batch_size, num_masks], + representing the foreground mask targets. + + Returns: + mask_loss: a float tensor representing total mask loss. + """ + with tf.name_scope('mask_rcnn_loss'): + (batch_size, num_masks, mask_height, + mask_width) = mask_outputs.get_shape().as_list() + + weights = tf.tile( + tf.reshape(tf.greater(select_class_targets, 0), + [batch_size, num_masks, 1, 1]), + [1, 1, mask_height, mask_width]) + weights = tf.cast(weights, dtype=tf.float32) + + mask_targets = tf.expand_dims(mask_targets, axis=-1) + mask_outputs = tf.expand_dims(mask_outputs, axis=-1) + mask_loss = self._binary_crossentropy(mask_targets, mask_outputs, + sample_weight=weights) + + # The loss is normalized by the number of 1's in weights and + # + 0.01 is used to avoid division by zero. + return mask_loss / (tf.reduce_sum(weights) + 0.01) + + +class RetinanetClassLoss(object): + """RetinaNet class loss.""" + + def __init__(self, params): + self._num_classes = params.num_classes + self._focal_loss_alpha = params.focal_loss_alpha + self._focal_loss_gamma = params.focal_loss_gamma + + def __call__(self, cls_outputs, labels, num_positives): + """Computes total detection loss. + + Computes total detection loss including box and class loss from all levels. + + Args: + cls_outputs: an OrderDict with keys representing levels and values + representing logits in [batch_size, height, width, + num_anchors * num_classes]. + labels: the dictionary that returned from dataloader that includes + class groundturth targets. + num_positives: number of positive examples in the minibatch. + + Returns: + an integar tensor representing total class loss. + """ + # Sums all positives in a batch for normalization and avoids zero + # num_positives_sum, which would lead to inf loss during training + num_positives_sum = tf.reduce_sum(input_tensor=num_positives) + 1.0 + + cls_losses = [] + for level in cls_outputs.keys(): + cls_losses.append(self.class_loss( + cls_outputs[level], labels[level], num_positives_sum)) + # Sums per level losses to total loss. + return tf.add_n(cls_losses) + + def class_loss(self, cls_outputs, cls_targets, num_positives, + ignore_label=-2): + """Computes RetinaNet classification loss.""" + # Onehot encoding for classification labels. + cls_targets_one_hot = tf.one_hot(cls_targets, self._num_classes) + bs, height, width, _, _ = cls_targets_one_hot.get_shape().as_list() + cls_targets_one_hot = tf.reshape(cls_targets_one_hot, + [bs, height, width, -1]) + loss = focal_loss(cls_outputs, cls_targets_one_hot, + self._focal_loss_alpha, self._focal_loss_gamma, + num_positives) + + ignore_loss = tf.where( + tf.equal(cls_targets, ignore_label), + tf.zeros_like(cls_targets, dtype=tf.float32), + tf.ones_like(cls_targets, dtype=tf.float32), + ) + ignore_loss = tf.expand_dims(ignore_loss, -1) + ignore_loss = tf.tile(ignore_loss, [1, 1, 1, 1, self._num_classes]) + ignore_loss = tf.reshape(ignore_loss, tf.shape(input=loss)) + return tf.reduce_sum(input_tensor=ignore_loss * loss) + + +class RetinanetBoxLoss(object): + """RetinaNet box loss.""" + + def __init__(self, params): + self._huber_loss = tf.keras.losses.Huber( + delta=params.huber_loss_delta, reduction=tf.keras.losses.Reduction.SUM) + + def __call__(self, box_outputs, labels, num_positives): + """Computes box detection loss. + + Computes total detection loss including box and class loss from all levels. + + Args: + box_outputs: an OrderDict with keys representing levels and values + representing box regression targets in [batch_size, height, width, + num_anchors * 4]. + labels: the dictionary that returned from dataloader that includes + box groundturth targets. + num_positives: number of positive examples in the minibatch. + + Returns: + an integar tensor representing total box regression loss. + """ + # Sums all positives in a batch for normalization and avoids zero + # num_positives_sum, which would lead to inf loss during training + num_positives_sum = tf.reduce_sum(input_tensor=num_positives) + 1.0 + + box_losses = [] + for level in box_outputs.keys(): + # Onehot encoding for classification labels. + box_targets_l = labels[level] + box_losses.append( + self.box_loss(box_outputs[level], box_targets_l, num_positives_sum)) + # Sums per level losses to total loss. + return tf.add_n(box_losses) + + def box_loss(self, box_outputs, box_targets, num_positives): + """Computes RetinaNet box regression loss.""" + # The delta is typically around the mean value of regression target. + # for instances, the regression targets of 512x512 input with 6 anchors on + # P3-P7 pyramid is about [0.1, 0.1, 0.2, 0.2]. + normalizer = num_positives * 4.0 + mask = tf.cast(tf.not_equal(box_targets, 0.0), dtype=tf.float32) + box_targets = tf.expand_dims(box_targets, axis=-1) + box_outputs = tf.expand_dims(box_outputs, axis=-1) + box_loss = self._huber_loss(box_targets, box_outputs, sample_weight=mask) + box_loss /= normalizer + return box_loss + + +class ShapemaskMseLoss(object): + """ShapeMask mask Mean Squared Error loss function wrapper.""" + + def __init__(self): + raise NotImplementedError('Not Implemented.') + + +class ShapemaskLoss(object): + """ShapeMask mask loss function wrapper.""" + + def __init__(self): + raise NotImplementedError('Not Implemented.') diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/maskrcnn_model.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/maskrcnn_model.py new file mode 100644 index 0000000..e196a31 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/maskrcnn_model.py @@ -0,0 +1,342 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model defination for the Mask R-CNN Model.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow.compat.v2 as tf + +from tensorflow.python.keras import backend +from official.vision.detection.dataloader import anchor +from official.vision.detection.dataloader import mode_keys +from official.vision.detection.evaluation import factory as eval_factory +from official.vision.detection.modeling import base_model +from official.vision.detection.modeling import losses +from official.vision.detection.modeling.architecture import factory +from official.vision.detection.ops import postprocess_ops +from official.vision.detection.ops import roi_ops +from official.vision.detection.ops import sampling_ops +from official.vision.detection.ops import spatial_transform_ops +from official.vision.detection.utils import box_utils + + +class MaskrcnnModel(base_model.Model): + """Mask R-CNN model function.""" + + def __init__(self, params): + super(MaskrcnnModel, self).__init__(params) + + # For eval metrics. + self._params = params + self._keras_model = None + + self._include_mask = params.architecture.include_mask + + # Architecture generators. + self._backbone_fn = factory.backbone_generator(params) + self._fpn_fn = factory.multilevel_features_generator(params) + self._rpn_head_fn = factory.rpn_head_generator(params.rpn_head) + self._generate_rois_fn = roi_ops.ROIGenerator(params.roi_proposal) + self._sample_rois_fn = sampling_ops.ROISampler(params.roi_sampling) + self._sample_masks_fn = sampling_ops.MaskSampler(params.mask_sampling) + + self._frcnn_head_fn = factory.fast_rcnn_head_generator(params.frcnn_head) + if self._include_mask: + self._mrcnn_head_fn = factory.mask_rcnn_head_generator(params.mrcnn_head) + + # Loss function. + self._rpn_score_loss_fn = losses.RpnScoreLoss(params.rpn_score_loss) + self._rpn_box_loss_fn = losses.RpnBoxLoss(params.rpn_box_loss) + self._frcnn_class_loss_fn = losses.FastrcnnClassLoss() + self._frcnn_box_loss_fn = losses.FastrcnnBoxLoss(params.frcnn_box_loss) + if self._include_mask: + self._mask_loss_fn = losses.MaskrcnnLoss() + + self._generate_detections_fn = postprocess_ops.GenericDetectionGenerator( + params.postprocess) + + self._transpose_input = params.train.transpose_input + assert not self._transpose_input, 'Transpose input is not supportted.' + + def build_outputs(self, inputs, mode): + is_training = mode == mode_keys.TRAIN + model_outputs = {} + + image = inputs['image'] + _, image_height, image_width, _ = image.get_shape().as_list() + backbone_features = self._backbone_fn(image, is_training) + fpn_features = self._fpn_fn(backbone_features, is_training) + + rpn_score_outputs, rpn_box_outputs = self._rpn_head_fn( + fpn_features, is_training) + model_outputs.update({ + 'rpn_score_outputs': + tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), + rpn_score_outputs), + 'rpn_box_outputs': + tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), + rpn_box_outputs), + }) + input_anchor = anchor.Anchor(self._params.anchor.min_level, + self._params.anchor.max_level, + self._params.anchor.num_scales, + self._params.anchor.aspect_ratios, + self._params.anchor.anchor_size, + (image_height, image_width)) + rpn_rois, _ = self._generate_rois_fn(rpn_box_outputs, rpn_score_outputs, + input_anchor.multilevel_boxes, + inputs['image_info'][:, 1, :], + is_training) + if is_training: + rpn_rois = tf.stop_gradient(rpn_rois) + + # Sample proposals. + rpn_rois, matched_gt_boxes, matched_gt_classes, matched_gt_indices = ( + self._sample_rois_fn(rpn_rois, inputs['gt_boxes'], + inputs['gt_classes'])) + + # Create bounding box training targets. + box_targets = box_utils.encode_boxes( + matched_gt_boxes, rpn_rois, weights=[10.0, 10.0, 5.0, 5.0]) + # If the target is background, the box target is set to all 0s. + box_targets = tf.where( + tf.tile( + tf.expand_dims(tf.equal(matched_gt_classes, 0), axis=-1), + [1, 1, 4]), + tf.zeros_like(box_targets), + box_targets) + model_outputs.update({ + 'class_targets': matched_gt_classes, + 'box_targets': box_targets, + }) + + roi_features = spatial_transform_ops.multilevel_crop_and_resize( + fpn_features, rpn_rois, output_size=7) + + class_outputs, box_outputs = self._frcnn_head_fn(roi_features, is_training) + + model_outputs.update({ + 'class_outputs': + tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), + class_outputs), + 'box_outputs': + tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), + box_outputs), + }) + + # Add this output to train to make the checkpoint loadable in predict mode. + # If we skip it in train mode, the heads will be out-of-order and checkpoint + # loading will fail. + boxes, scores, classes, valid_detections = self._generate_detections_fn( + box_outputs, class_outputs, rpn_rois, inputs['image_info'][:, 1:2, :]) + model_outputs.update({ + 'num_detections': valid_detections, + 'detection_boxes': boxes, + 'detection_classes': classes, + 'detection_scores': scores, + }) + + if not self._include_mask: + return model_outputs + + if is_training: + rpn_rois, classes, mask_targets = self._sample_masks_fn( + rpn_rois, matched_gt_boxes, matched_gt_classes, matched_gt_indices, + inputs['gt_masks']) + mask_targets = tf.stop_gradient(mask_targets) + + classes = tf.cast(classes, dtype=tf.int32) + + model_outputs.update({ + 'mask_targets': mask_targets, + 'sampled_class_targets': classes, + }) + else: + rpn_rois = boxes + classes = tf.cast(classes, dtype=tf.int32) + + mask_roi_features = spatial_transform_ops.multilevel_crop_and_resize( + fpn_features, rpn_rois, output_size=14) + + mask_outputs = self._mrcnn_head_fn(mask_roi_features, classes, is_training) + + if is_training: + model_outputs.update({ + 'mask_outputs': + tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), + mask_outputs), + }) + else: + model_outputs.update({ + 'detection_masks': tf.nn.sigmoid(mask_outputs) + }) + + return model_outputs + + def build_loss_fn(self): + if self._keras_model is None: + raise ValueError('build_loss_fn() must be called after build_model().') + + filter_fn = self.make_filter_trainable_variables_fn() + trainable_variables = filter_fn(self._keras_model.trainable_variables) + + def _total_loss_fn(labels, outputs): + rpn_score_loss = self._rpn_score_loss_fn(outputs['rpn_score_outputs'], + labels['rpn_score_targets']) + rpn_box_loss = self._rpn_box_loss_fn(outputs['rpn_box_outputs'], + labels['rpn_box_targets']) + + frcnn_class_loss = self._frcnn_class_loss_fn(outputs['class_outputs'], + outputs['class_targets']) + frcnn_box_loss = self._frcnn_box_loss_fn(outputs['box_outputs'], + outputs['class_targets'], + outputs['box_targets']) + + if self._include_mask: + mask_loss = self._mask_loss_fn(outputs['mask_outputs'], + outputs['mask_targets'], + outputs['sampled_class_targets']) + else: + mask_loss = 0.0 + + model_loss = ( + rpn_score_loss + rpn_box_loss + frcnn_class_loss + frcnn_box_loss + + mask_loss) + + l2_regularization_loss = self.weight_decay_loss(trainable_variables) + total_loss = model_loss + l2_regularization_loss + return { + 'total_loss': total_loss, + 'loss': total_loss, + 'fast_rcnn_class_loss': frcnn_class_loss, + 'fast_rcnn_box_loss': frcnn_box_loss, + 'mask_loss': mask_loss, + 'model_loss': model_loss, + 'l2_regularization_loss': l2_regularization_loss, + 'rpn_score_loss': rpn_score_loss, + 'rpn_box_loss': rpn_box_loss, + } + + return _total_loss_fn + + def build_input_layers(self, params, mode): + is_training = mode == mode_keys.TRAIN + input_shape = ( + params.maskrcnn_parser.output_size + + [params.maskrcnn_parser.num_channels]) + if is_training: + batch_size = params.train.batch_size + input_layer = { + 'image': + tf.keras.layers.Input( + shape=input_shape, + batch_size=batch_size, + name='image', + dtype=tf.bfloat16 if self._use_bfloat16 else tf.float32), + 'image_info': + tf.keras.layers.Input( + shape=[4, 2], + batch_size=batch_size, + name='image_info', + ), + 'gt_boxes': + tf.keras.layers.Input( + shape=[params.maskrcnn_parser.max_num_instances, 4], + batch_size=batch_size, + name='gt_boxes'), + 'gt_classes': + tf.keras.layers.Input( + shape=[params.maskrcnn_parser.max_num_instances], + batch_size=batch_size, + name='gt_classes', + dtype=tf.int64), + } + if self._include_mask: + input_layer['gt_masks'] = tf.keras.layers.Input( + shape=[ + params.maskrcnn_parser.max_num_instances, + params.maskrcnn_parser.mask_crop_size, + params.maskrcnn_parser.mask_crop_size + ], + batch_size=batch_size, + name='gt_masks') + else: + batch_size = params.eval.batch_size + input_layer = { + 'image': + tf.keras.layers.Input( + shape=input_shape, + batch_size=batch_size, + name='image', + dtype=tf.bfloat16 if self._use_bfloat16 else tf.float32), + 'image_info': + tf.keras.layers.Input( + shape=[4, 2], + batch_size=batch_size, + name='image_info', + ), + } + return input_layer + + def build_model(self, params, mode): + if self._keras_model is None: + input_layers = self.build_input_layers(self._params, mode) + with backend.get_graph().as_default(): + outputs = self.model_outputs(input_layers, mode) + + model = tf.keras.models.Model( + inputs=input_layers, outputs=outputs, name='maskrcnn') + assert model is not None, 'Fail to build tf.keras.Model.' + model.optimizer = self.build_optimizer() + self._keras_model = model + + return self._keras_model + + def post_processing(self, labels, outputs): + required_output_fields = ['class_outputs', 'box_outputs'] + for field in required_output_fields: + if field not in outputs: + raise ValueError('"%s" is missing in outputs, requried %s found %s' + %(field, required_output_fields, outputs.keys())) + predictions = { + 'image_info': labels['image_info'], + 'num_detections': outputs['num_detections'], + 'detection_boxes': outputs['detection_boxes'], + 'detection_classes': outputs['detection_classes'], + 'detection_scores': outputs['detection_scores'], + } + if self._include_mask: + predictions.update({ + 'detection_masks': outputs['detection_masks'], + }) + + if 'groundtruths' in labels: + predictions['source_id'] = labels['groundtruths']['source_id'] + predictions['gt_source_id'] = labels['groundtruths']['source_id'] + predictions['gt_height'] = labels['groundtruths']['height'] + predictions['gt_width'] = labels['groundtruths']['width'] + predictions['gt_image_info'] = labels['image_info'] + predictions['gt_num_detections'] = ( + labels['groundtruths']['num_detections']) + predictions['gt_boxes'] = labels['groundtruths']['boxes'] + predictions['gt_classes'] = labels['groundtruths']['classes'] + predictions['gt_areas'] = labels['groundtruths']['areas'] + predictions['gt_is_crowds'] = labels['groundtruths']['is_crowds'] + return labels, predictions + + def eval_metrics(self): + return eval_factory.evaluator_generator(self._params.eval) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/retinanet_model.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/retinanet_model.py new file mode 100644 index 0000000..cf58f23 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/modeling/retinanet_model.py @@ -0,0 +1,170 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model defination for the RetinaNet Model.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import numpy as np +from absl import logging +import tensorflow.compat.v2 as tf + +from tensorflow.python.keras import backend +from official.vision.detection.dataloader import mode_keys +from official.vision.detection.evaluation import factory as eval_factory +from official.vision.detection.modeling import base_model +from official.vision.detection.modeling import losses +from official.vision.detection.modeling.architecture import factory +from official.vision.detection.ops import postprocess_ops + + +class RetinanetModel(base_model.Model): + """RetinaNet model function.""" + + def __init__(self, params): + super(RetinanetModel, self).__init__(params) + + # For eval metrics. + self._params = params + + # Architecture generators. + self._backbone_fn = factory.backbone_generator(params) + self._fpn_fn = factory.multilevel_features_generator(params) + self._head_fn = factory.retinanet_head_generator(params.retinanet_head) + + # Loss function. + self._cls_loss_fn = losses.RetinanetClassLoss(params.retinanet_loss) + self._box_loss_fn = losses.RetinanetBoxLoss(params.retinanet_loss) + self._box_loss_weight = params.retinanet_loss.box_loss_weight + self._keras_model = None + + # Predict function. + self._generate_detections_fn = postprocess_ops.MultilevelDetectionGenerator( + params.postprocess) + + self._transpose_input = params.train.transpose_input + assert not self._transpose_input, 'Transpose input is not supportted.' + # Input layer. + input_shape = ( + params.retinanet_parser.output_size + + [params.retinanet_parser.num_channels]) + self._input_layer = tf.keras.layers.Input( + shape=input_shape, name='', + dtype=tf.bfloat16 if self._use_bfloat16 else tf.float32) + + def build_outputs(self, inputs, mode): + # If the input image is transposed (from NHWC to HWCN), we need to revert it + # back to the original shape before it's used in the computation. + if self._transpose_input: + inputs = tf.transpose(inputs, [3, 0, 1, 2]) + + backbone_features = self._backbone_fn( + inputs, is_training=(mode == mode_keys.TRAIN)) + fpn_features = self._fpn_fn( + backbone_features, is_training=(mode == mode_keys.TRAIN)) + cls_outputs, box_outputs = self._head_fn( + fpn_features, is_training=(mode == mode_keys.TRAIN)) + + if self._use_bfloat16: + levels = cls_outputs.keys() + for level in levels: + cls_outputs[level] = tf.cast(cls_outputs[level], tf.float32) + box_outputs[level] = tf.cast(box_outputs[level], tf.float32) + + model_outputs = { + 'cls_outputs': cls_outputs, + 'box_outputs': box_outputs, + } + return model_outputs + + def build_loss_fn(self): + if self._keras_model is None: + raise ValueError('build_loss_fn() must be called after build_model().') + + filter_fn = self.make_filter_trainable_variables_fn() + trainable_variables = filter_fn(self._keras_model.trainable_variables) + + def _total_loss_fn(labels, outputs): + cls_loss = self._cls_loss_fn(outputs['cls_outputs'], + labels['cls_targets'], + labels['num_positives']) + box_loss = self._box_loss_fn(outputs['box_outputs'], + labels['box_targets'], + labels['num_positives']) + model_loss = cls_loss + self._box_loss_weight * box_loss + l2_regularization_loss = self.weight_decay_loss(trainable_variables) + total_loss = model_loss + l2_regularization_loss + return { + 'total_loss': total_loss, + 'cls_loss': cls_loss, + 'box_loss': box_loss, + 'model_loss': model_loss, + 'l2_regularization_loss': l2_regularization_loss, + } + + return _total_loss_fn + + def build_model(self, params, mode=None): + if self._keras_model is None: + with backend.get_graph().as_default(): + outputs = self.model_outputs(self._input_layer, mode) + + model = tf.keras.models.Model( + inputs=self._input_layer, outputs=outputs, name='retinanet') + assert model is not None, 'Fail to build tf.keras.Model.' + model.optimizer = self.build_optimizer() + self._keras_model = model + + return self._keras_model + + def post_processing(self, labels, outputs): + # TODO(yeqing): Moves the output related part into build_outputs. + required_output_fields = ['cls_outputs', 'box_outputs'] + for field in required_output_fields: + if field not in outputs: + raise ValueError('"%s" is missing in outputs, requried %s found %s', + field, required_output_fields, outputs.keys()) + required_label_fields = ['image_info', 'groundtruths'] + for field in required_label_fields: + if field not in labels: + raise ValueError('"%s" is missing in outputs, requried %s found %s', + field, required_label_fields, labels.keys()) + boxes, scores, classes, valid_detections = self._generate_detections_fn( + outputs['box_outputs'], outputs['cls_outputs'], + labels['anchor_boxes'], labels['image_info'][:, 1:2, :]) + # Discards the old output tensors to save memory. The `cls_outputs` and + # `box_outputs` are pretty big and could potentiall lead to memory issue. + outputs = { + 'source_id': labels['groundtruths']['source_id'], + 'image_info': labels['image_info'], + 'num_detections': valid_detections, + 'detection_boxes': boxes, + 'detection_classes': classes, + 'detection_scores': scores, + } + + if 'groundtruths' in labels: + labels['source_id'] = labels['groundtruths']['source_id'] + labels['boxes'] = labels['groundtruths']['boxes'] + labels['classes'] = labels['groundtruths']['classes'] + labels['areas'] = labels['groundtruths']['areas'] + labels['is_crowds'] = labels['groundtruths']['is_crowds'] + + return labels, outputs + + def eval_metrics(self): + return eval_factory.evaluator_generator(self._params.eval) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/__init__.py new file mode 100644 index 0000000..931c2ef --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/nms.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/nms.py new file mode 100644 index 0000000..c6efead --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/nms.py @@ -0,0 +1,205 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tensorflow implementation of non max suppression.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow.compat.v2 as tf + +from official.vision.detection.utils import box_utils + + +NMS_TILE_SIZE = 512 + + +def _self_suppression(iou, _, iou_sum): + batch_size = tf.shape(iou)[0] + can_suppress_others = tf.cast( + tf.reshape(tf.reduce_max(iou, 1) <= 0.5, [batch_size, -1, 1]), iou.dtype) + iou_suppressed = tf.reshape( + tf.cast(tf.reduce_max(can_suppress_others * iou, 1) <= 0.5, iou.dtype), + [batch_size, -1, 1]) * iou + iou_sum_new = tf.reduce_sum(iou_suppressed, [1, 2]) + return [ + iou_suppressed, + tf.reduce_any(iou_sum - iou_sum_new > 0.5), iou_sum_new + ] + + +def _cross_suppression(boxes, box_slice, iou_threshold, inner_idx): + batch_size = tf.shape(boxes)[0] + new_slice = tf.slice(boxes, [0, inner_idx * NMS_TILE_SIZE, 0], + [batch_size, NMS_TILE_SIZE, 4]) + iou = box_utils.bbox_overlap(new_slice, box_slice) + ret_slice = tf.expand_dims( + tf.cast(tf.reduce_all(iou < iou_threshold, [1]), box_slice.dtype), + 2) * box_slice + return boxes, ret_slice, iou_threshold, inner_idx + 1 + + +def _suppression_loop_body(boxes, iou_threshold, output_size, idx): + """Process boxes in the range [idx*NMS_TILE_SIZE, (idx+1)*NMS_TILE_SIZE). + + Args: + boxes: a tensor with a shape of [batch_size, anchors, 4]. + iou_threshold: a float representing the threshold for deciding whether boxes + overlap too much with respect to IOU. + output_size: an int32 tensor of size [batch_size]. Representing the number + of selected boxes for each batch. + idx: an integer scalar representing induction variable. + + Returns: + boxes: updated boxes. + iou_threshold: pass down iou_threshold to the next iteration. + output_size: the updated output_size. + idx: the updated induction variable. + """ + num_tiles = tf.shape(boxes)[1] // NMS_TILE_SIZE + batch_size = tf.shape(boxes)[0] + + # Iterates over tiles that can possibly suppress the current tile. + box_slice = tf.slice(boxes, [0, idx * NMS_TILE_SIZE, 0], + [batch_size, NMS_TILE_SIZE, 4]) + _, box_slice, _, _ = tf.while_loop( + lambda _boxes, _box_slice, _threshold, inner_idx: inner_idx < idx, + _cross_suppression, [boxes, box_slice, iou_threshold, + tf.constant(0)]) + + # Iterates over the current tile to compute self-suppression. + iou = box_utils.bbox_overlap(box_slice, box_slice) + mask = tf.expand_dims( + tf.reshape(tf.range(NMS_TILE_SIZE), [1, -1]) > tf.reshape( + tf.range(NMS_TILE_SIZE), [-1, 1]), 0) + iou *= tf.cast(tf.logical_and(mask, iou >= iou_threshold), iou.dtype) + suppressed_iou, _, _ = tf.while_loop( + lambda _iou, loop_condition, _iou_sum: loop_condition, _self_suppression, + [iou, tf.constant(True), + tf.reduce_sum(iou, [1, 2])]) + suppressed_box = tf.reduce_sum(suppressed_iou, 1) > 0 + box_slice *= tf.expand_dims(1.0 - tf.cast(suppressed_box, box_slice.dtype), 2) + + # Uses box_slice to update the input boxes. + mask = tf.reshape( + tf.cast(tf.equal(tf.range(num_tiles), idx), boxes.dtype), [1, -1, 1, 1]) + boxes = tf.tile(tf.expand_dims( + box_slice, [1]), [1, num_tiles, 1, 1]) * mask + tf.reshape( + boxes, [batch_size, num_tiles, NMS_TILE_SIZE, 4]) * (1 - mask) + boxes = tf.reshape(boxes, [batch_size, -1, 4]) + + # Updates output_size. + output_size += tf.reduce_sum( + tf.cast(tf.reduce_any(box_slice > 0, [2]), tf.int32), [1]) + return boxes, iou_threshold, output_size, idx + 1 + + +def sorted_non_max_suppression_padded(scores, + boxes, + max_output_size, + iou_threshold): + """A wrapper that handles non-maximum suppression. + + Assumption: + * The boxes are sorted by scores unless the box is a dot (all coordinates + are zero). + * Boxes with higher scores can be used to suppress boxes with lower scores. + + The overal design of the algorithm is to handle boxes tile-by-tile: + + boxes = boxes.pad_to_multiply_of(tile_size) + num_tiles = len(boxes) // tile_size + output_boxes = [] + for i in range(num_tiles): + box_tile = boxes[i*tile_size : (i+1)*tile_size] + for j in range(i - 1): + suppressing_tile = boxes[j*tile_size : (j+1)*tile_size] + iou = bbox_overlap(box_tile, suppressing_tile) + # if the box is suppressed in iou, clear it to a dot + box_tile *= _update_boxes(iou) + # Iteratively handle the diagnal tile. + iou = _box_overlap(box_tile, box_tile) + iou_changed = True + while iou_changed: + # boxes that are not suppressed by anything else + suppressing_boxes = _get_suppressing_boxes(iou) + # boxes that are suppressed by suppressing_boxes + suppressed_boxes = _get_suppressed_boxes(iou, suppressing_boxes) + # clear iou to 0 for boxes that are suppressed, as they cannot be used + # to suppress other boxes any more + new_iou = _clear_iou(iou, suppressed_boxes) + iou_changed = (new_iou != iou) + iou = new_iou + # remaining boxes that can still suppress others, are selected boxes. + output_boxes.append(_get_suppressing_boxes(iou)) + if len(output_boxes) >= max_output_size: + break + + Args: + scores: a tensor with a shape of [batch_size, anchors]. + boxes: a tensor with a shape of [batch_size, anchors, 4]. + max_output_size: a scalar integer `Tensor` representing the maximum number + of boxes to be selected by non max suppression. + iou_threshold: a float representing the threshold for deciding whether boxes + overlap too much with respect to IOU. + + Returns: + nms_scores: a tensor with a shape of [batch_size, anchors]. It has same + dtype as input scores. + nms_proposals: a tensor with a shape of [batch_size, anchors, 4]. It has + same dtype as input boxes. + """ + batch_size = tf.shape(boxes)[0] + num_boxes = tf.shape(boxes)[1] + pad = tf.cast( + tf.math.ceil(tf.cast(num_boxes, tf.float32) / NMS_TILE_SIZE), + tf.int32) * NMS_TILE_SIZE - num_boxes + boxes = tf.pad(tf.cast(boxes, tf.float32), [[0, 0], [0, pad], [0, 0]]) + scores = tf.pad( + tf.cast(scores, tf.float32), [[0, 0], [0, pad]], constant_values=-1) + num_boxes += pad + + def _loop_cond(unused_boxes, unused_threshold, output_size, idx): + return tf.logical_and( + tf.reduce_min(output_size) < max_output_size, + idx < num_boxes // NMS_TILE_SIZE) + + selected_boxes, _, output_size, _ = tf.while_loop( + _loop_cond, _suppression_loop_body, [ + boxes, iou_threshold, + tf.zeros([batch_size], tf.int32), + tf.constant(0) + ]) + idx = num_boxes - tf.cast( + tf.nn.top_k( + tf.cast(tf.reduce_any(selected_boxes > 0, [2]), tf.int32) * + tf.expand_dims(tf.range(num_boxes, 0, -1), 0), max_output_size)[0], + tf.int32) + idx = tf.minimum(idx, num_boxes - 1) + idx = tf.reshape( + idx + tf.reshape(tf.range(batch_size) * num_boxes, [-1, 1]), [-1]) + boxes = tf.reshape( + tf.gather(tf.reshape(boxes, [-1, 4]), idx), + [batch_size, max_output_size, 4]) + boxes = boxes * tf.cast( + tf.reshape(tf.range(max_output_size), [1, -1, 1]) < tf.reshape( + output_size, [-1, 1, 1]), boxes.dtype) + scores = tf.reshape( + tf.gather(tf.reshape(scores, [-1, 1]), idx), + [batch_size, max_output_size]) + scores = scores * tf.cast( + tf.reshape(tf.range(max_output_size), [1, -1]) < tf.reshape( + output_size, [-1, 1]), scores.dtype) + return scores, boxes diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/postprocess_ops.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/postprocess_ops.py new file mode 100644 index 0000000..209555e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/postprocess_ops.py @@ -0,0 +1,412 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Post-processing model outputs to generate detection.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools +import tensorflow.compat.v2 as tf + +from official.vision.detection.ops import nms +from official.vision.detection.utils import box_utils + + +def generate_detections_factory(params): + """Factory to select function to generate detection.""" + if params.use_batched_nms: + func = functools.partial( + _generate_detections_batched, + max_total_size=params.max_total_size, + nms_iou_threshold=params.nms_iou_threshold, + score_threshold=params.score_threshold) + else: + func = functools.partial( + _generate_detections, + max_total_size=params.max_total_size, + nms_iou_threshold=params.nms_iou_threshold, + score_threshold=params.score_threshold, + pre_nms_num_boxes=params.pre_nms_num_boxes) + return func + + +def _select_top_k_scores(scores_in, pre_nms_num_detections): + """Select top_k scores and indices for each class. + + Args: + scores_in: a Tensor with shape [batch_size, N, num_classes], which stacks + class logit outputs on all feature levels. The N is the number of total + anchors on all levels. The num_classes is the number of classes predicted + by the model. + pre_nms_num_detections: Number of candidates before NMS. + + Returns: + scores and indices: Tensors with shape [batch_size, pre_nms_num_detections, + num_classes]. + """ + batch_size, num_anchors, num_class = scores_in.get_shape().as_list() + scores_trans = tf.transpose(scores_in, perm=[0, 2, 1]) + scores_trans = tf.reshape(scores_trans, [-1, num_anchors]) + + top_k_scores, top_k_indices = tf.nn.top_k( + scores_trans, k=pre_nms_num_detections, sorted=True) + + top_k_scores = tf.reshape(top_k_scores, + [batch_size, num_class, pre_nms_num_detections]) + top_k_indices = tf.reshape(top_k_indices, + [batch_size, num_class, pre_nms_num_detections]) + + return tf.transpose(top_k_scores, + [0, 2, 1]), tf.transpose(top_k_indices, [0, 2, 1]) + + +def _generate_detections(boxes, + scores, + max_total_size=100, + nms_iou_threshold=0.3, + score_threshold=0.05, + pre_nms_num_boxes=5000): + """Generate the final detections given the model outputs. + + This uses classes unrolling with while loop based NMS, could be parralled + at batch dimension. + + Args: + boxes: a tensor with shape [batch_size, N, num_classes, 4] or [batch_size, + N, 1, 4], which box predictions on all feature levels. The N is the number + of total anchors on all levels. + scores: a tensor with shape [batch_size, N, num_classes], which stacks class + probability on all feature levels. The N is the number of total anchors on + all levels. The num_classes is the number of classes predicted by the + model. Note that the class_outputs here is the raw score. + max_total_size: a scalar representing maximum number of boxes retained over + all classes. + nms_iou_threshold: a float representing the threshold for deciding whether + boxes overlap too much with respect to IOU. + score_threshold: a float representing the threshold for deciding when to + remove boxes based on score. + pre_nms_num_boxes: an int number of top candidate detections per class + before NMS. + + Returns: + nms_boxes: `float` Tensor of shape [batch_size, max_total_size, 4] + representing top detected boxes in [y1, x1, y2, x2]. + nms_scores: `float` Tensor of shape [batch_size, max_total_size] + representing sorted confidence scores for detected boxes. The values are + between [0, 1]. + nms_classes: `int` Tensor of shape [batch_size, max_total_size] representing + classes for detected boxes. + valid_detections: `int` Tensor of shape [batch_size] only the top + `valid_detections` boxes are valid detections. + """ + with tf.name_scope('generate_detections'): + nmsed_boxes = [] + nmsed_classes = [] + nmsed_scores = [] + valid_detections = [] + batch_size, _, num_classes_for_box, _ = boxes.get_shape().as_list() + _, total_anchors, num_classes = scores.get_shape().as_list() + # Selects top pre_nms_num scores and indices before NMS. + scores, indices = _select_top_k_scores( + scores, min(total_anchors, pre_nms_num_boxes)) + for i in range(num_classes): + boxes_i = boxes[:, :, min(num_classes_for_box - 1, i), :] + scores_i = scores[:, :, i] + # Obtains pre_nms_num_boxes before running NMS. + boxes_i = tf.gather(boxes_i, indices[:, :, i], batch_dims=1, axis=1) + + # Filter out scores. + boxes_i, scores_i = box_utils.filter_boxes_by_scores( + boxes_i, scores_i, min_score_threshold=score_threshold) + + (nmsed_scores_i, nmsed_boxes_i) = nms.sorted_non_max_suppression_padded( + tf.cast(scores_i, tf.float32), + tf.cast(boxes_i, tf.float32), + max_total_size, + iou_threshold=nms_iou_threshold) + nmsed_classes_i = tf.fill([batch_size, max_total_size], i) + nmsed_boxes.append(nmsed_boxes_i) + nmsed_scores.append(nmsed_scores_i) + nmsed_classes.append(nmsed_classes_i) + nmsed_boxes = tf.concat(nmsed_boxes, axis=1) + nmsed_scores = tf.concat(nmsed_scores, axis=1) + nmsed_classes = tf.concat(nmsed_classes, axis=1) + nmsed_scores, indices = tf.nn.top_k( + nmsed_scores, k=max_total_size, sorted=True) + nmsed_boxes = tf.gather(nmsed_boxes, indices, batch_dims=1, axis=1) + nmsed_classes = tf.gather(nmsed_classes, indices, batch_dims=1) + valid_detections = tf.reduce_sum( + input_tensor=tf.cast(tf.greater(nmsed_scores, -1), tf.int32), axis=1) + return nmsed_boxes, nmsed_scores, nmsed_classes, valid_detections + + +def _generate_detections_per_image(boxes, + scores, + max_total_size=100, + nms_iou_threshold=0.3, + score_threshold=0.05, + pre_nms_num_boxes=5000): + """Generate the final detections per image given the model outputs. + + Args: + boxes: a tensor with shape [N, num_classes, 4] or [N, 1, 4], which box + predictions on all feature levels. The N is the number of total anchors on + all levels. + scores: a tensor with shape [N, num_classes], which stacks class probability + on all feature levels. The N is the number of total anchors on all levels. + The num_classes is the number of classes predicted by the model. Note that + the class_outputs here is the raw score. + max_total_size: a scalar representing maximum number of boxes retained over + all classes. + nms_iou_threshold: a float representing the threshold for deciding whether + boxes overlap too much with respect to IOU. + score_threshold: a float representing the threshold for deciding when to + remove boxes based on score. + pre_nms_num_boxes: an int number of top candidate detections per class + before NMS. + + Returns: + nms_boxes: `float` Tensor of shape [max_total_size, 4] representing top + detected boxes in [y1, x1, y2, x2]. + nms_scores: `float` Tensor of shape [max_total_size] representing sorted + confidence scores for detected boxes. The values are between [0, 1]. + nms_classes: `int` Tensor of shape [max_total_size] representing classes for + detected boxes. + valid_detections: `int` Tensor of shape [1] only the top `valid_detections` + boxes are valid detections. + """ + nmsed_boxes = [] + nmsed_scores = [] + nmsed_classes = [] + num_classes_for_box = boxes.get_shape().as_list()[1] + num_classes = scores.get_shape().as_list()[1] + for i in range(num_classes): + boxes_i = boxes[:, min(num_classes_for_box - 1, i)] + scores_i = scores[:, i] + + # Obtains pre_nms_num_boxes before running NMS. + scores_i, indices = tf.nn.top_k( + scores_i, k=tf.minimum(tf.shape(input=scores_i)[-1], pre_nms_num_boxes)) + boxes_i = tf.gather(boxes_i, indices) + + (nmsed_indices_i, + nmsed_num_valid_i) = tf.image.non_max_suppression_padded( + tf.cast(boxes_i, tf.float32), + tf.cast(scores_i, tf.float32), + max_total_size, + iou_threshold=nms_iou_threshold, + score_threshold=score_threshold, + pad_to_max_output_size=True, + name='nms_detections_' + str(i)) + nmsed_boxes_i = tf.gather(boxes_i, nmsed_indices_i) + nmsed_scores_i = tf.gather(scores_i, nmsed_indices_i) + # Sets scores of invalid boxes to -1. + nmsed_scores_i = tf.where( + tf.less(tf.range(max_total_size), [nmsed_num_valid_i]), nmsed_scores_i, + -tf.ones_like(nmsed_scores_i)) + nmsed_classes_i = tf.fill([max_total_size], i) + nmsed_boxes.append(nmsed_boxes_i) + nmsed_scores.append(nmsed_scores_i) + nmsed_classes.append(nmsed_classes_i) + + # Concats results from all classes and sort them. + nmsed_boxes = tf.concat(nmsed_boxes, axis=0) + nmsed_scores = tf.concat(nmsed_scores, axis=0) + nmsed_classes = tf.concat(nmsed_classes, axis=0) + nmsed_scores, indices = tf.nn.top_k( + nmsed_scores, k=max_total_size, sorted=True) + nmsed_boxes = tf.gather(nmsed_boxes, indices) + nmsed_classes = tf.gather(nmsed_classes, indices) + valid_detections = tf.reduce_sum( + input_tensor=tf.cast(tf.greater(nmsed_scores, -1), tf.int32)) + return nmsed_boxes, nmsed_scores, nmsed_classes, valid_detections + + +def _generate_detections_batched(boxes, + scores, + max_total_size, + nms_iou_threshold, + score_threshold): + """Generates detected boxes with scores and classes for one-stage detector. + + The function takes output of multi-level ConvNets and anchor boxes and + generates detected boxes. Note that this used batched nms, which is not + supported on TPU currently. + + Args: + boxes: a tensor with shape [batch_size, N, num_classes, 4] or + [batch_size, N, 1, 4], which box predictions on all feature levels. The N + is the number of total anchors on all levels. + scores: a tensor with shape [batch_size, N, num_classes], which + stacks class probability on all feature levels. The N is the number of + total anchors on all levels. The num_classes is the number of classes + predicted by the model. Note that the class_outputs here is the raw score. + max_total_size: a scalar representing maximum number of boxes retained over + all classes. + nms_iou_threshold: a float representing the threshold for deciding whether + boxes overlap too much with respect to IOU. + score_threshold: a float representing the threshold for deciding when to + remove boxes based on score. + Returns: + nms_boxes: `float` Tensor of shape [batch_size, max_total_size, 4] + representing top detected boxes in [y1, x1, y2, x2]. + nms_scores: `float` Tensor of shape [batch_size, max_total_size] + representing sorted confidence scores for detected boxes. The values are + between [0, 1]. + nms_classes: `int` Tensor of shape [batch_size, max_total_size] representing + classes for detected boxes. + valid_detections: `int` Tensor of shape [batch_size] only the top + `valid_detections` boxes are valid detections. + """ + with tf.name_scope('generate_detections'): + # TODO(tsungyi): Removes normalization/denomalization once the + # tf.image.combined_non_max_suppression is coordinate system agnostic. + # Normalizes maximum box cooridinates to 1. + normalizer = tf.reduce_max(boxes) + boxes /= normalizer + (nmsed_boxes, nmsed_scores, nmsed_classes, + valid_detections) = tf.image.combined_non_max_suppression( + boxes, + scores, + max_output_size_per_class=max_total_size, + max_total_size=max_total_size, + iou_threshold=nms_iou_threshold, + score_threshold=score_threshold, + pad_per_class=False,) + # De-normalizes box cooridinates. + nmsed_boxes *= normalizer + return nmsed_boxes, nmsed_scores, nmsed_classes, valid_detections + + +class MultilevelDetectionGenerator(object): + """Generates detected boxes with scores and classes for one-stage detector.""" + + def __init__(self, params): + self._generate_detections = generate_detections_factory(params) + self._min_level = params.min_level + self._max_level = params.max_level + + def __call__(self, box_outputs, class_outputs, anchor_boxes, image_shape): + # Collects outputs from all levels into a list. + boxes = [] + scores = [] + for i in range(self._min_level, self._max_level + 1): + box_outputs_i_shape = tf.shape(box_outputs[i]) + batch_size = box_outputs_i_shape[0] + num_anchors_per_locations = box_outputs_i_shape[-1] // 4 + num_classes = tf.shape(class_outputs[i])[-1] // num_anchors_per_locations + + # Applies score transformation and remove the implicit background class. + scores_i = tf.sigmoid( + tf.reshape(class_outputs[i], [batch_size, -1, num_classes])) + scores_i = tf.slice(scores_i, [0, 0, 1], [-1, -1, -1]) + + # Box decoding. + # The anchor boxes are shared for all data in a batch. + # One stage detector only supports class agnostic box regression. + anchor_boxes_i = tf.reshape(anchor_boxes[i], [batch_size, -1, 4]) + box_outputs_i = tf.reshape(box_outputs[i], [batch_size, -1, 4]) + boxes_i = box_utils.decode_boxes(box_outputs_i, anchor_boxes_i) + + # Box clipping. + boxes_i = box_utils.clip_boxes(boxes_i, image_shape) + + boxes.append(boxes_i) + scores.append(scores_i) + boxes = tf.concat(boxes, axis=1) + scores = tf.concat(scores, axis=1) + + nmsed_boxes, nmsed_scores, nmsed_classes, valid_detections = ( + self._generate_detections(tf.expand_dims(boxes, axis=2), scores)) + + # Adds 1 to offset the background class which has index 0. + nmsed_classes += 1 + return nmsed_boxes, nmsed_scores, nmsed_classes, valid_detections + + +class GenericDetectionGenerator(object): + """Generates the final detected boxes with scores and classes.""" + + def __init__(self, params): + self._generate_detections = generate_detections_factory(params) + + def __call__(self, box_outputs, class_outputs, anchor_boxes, image_shape): + """Generate final detections. + + Args: + box_outputs: a tensor of shape of [batch_size, K, num_classes * 4] + representing the class-specific box coordinates relative to anchors. + class_outputs: a tensor of shape of [batch_size, K, num_classes] + representing the class logits before applying score activiation. + anchor_boxes: a tensor of shape of [batch_size, K, 4] representing the + corresponding anchor boxes w.r.t `box_outputs`. + image_shape: a tensor of shape of [batch_size, 2] storing the image height + and width w.r.t. the scaled image, i.e. the same image space as + `box_outputs` and `anchor_boxes`. + + Returns: + nms_boxes: `float` Tensor of shape [batch_size, max_total_size, 4] + representing top detected boxes in [y1, x1, y2, x2]. + nms_scores: `float` Tensor of shape [batch_size, max_total_size] + representing sorted confidence scores for detected boxes. The values are + between [0, 1]. + nms_classes: `int` Tensor of shape [batch_size, max_total_size] + representing classes for detected boxes. + valid_detections: `int` Tensor of shape [batch_size] only the top + `valid_detections` boxes are valid detections. + """ + class_outputs = tf.nn.softmax(class_outputs, axis=-1) + + # Removes the background class. + class_outputs_shape = tf.shape(class_outputs) + batch_size = class_outputs_shape[0] + num_locations = class_outputs_shape[1] + num_classes = class_outputs_shape[-1] + num_detections = num_locations * (num_classes - 1) + + class_outputs = tf.slice(class_outputs, [0, 0, 1], [-1, -1, -1]) + box_outputs = tf.reshape( + box_outputs, + tf.stack([batch_size, num_locations, num_classes, 4], axis=-1)) + box_outputs = tf.slice( + box_outputs, [0, 0, 1, 0], [-1, -1, -1, -1]) + anchor_boxes = tf.tile( + tf.expand_dims(anchor_boxes, axis=2), [1, 1, num_classes - 1, 1]) + box_outputs = tf.reshape( + box_outputs, + tf.stack([batch_size, num_detections, 4], axis=-1)) + anchor_boxes = tf.reshape( + anchor_boxes, + tf.stack([batch_size, num_detections, 4], axis=-1)) + + # Box decoding. + decoded_boxes = box_utils.decode_boxes( + box_outputs, anchor_boxes, weights=[10.0, 10.0, 5.0, 5.0]) + + # Box clipping + decoded_boxes = box_utils.clip_boxes(decoded_boxes, image_shape) + + decoded_boxes = tf.reshape( + decoded_boxes, + tf.stack([batch_size, num_locations, num_classes - 1, 4], axis=-1)) + + nmsed_boxes, nmsed_scores, nmsed_classes, valid_detections = ( + self._generate_detections(decoded_boxes, class_outputs)) + + # Adds 1 to offset the background class which has index 0. + nmsed_classes += 1 + + return nmsed_boxes, nmsed_scores, nmsed_classes, valid_detections diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/roi_ops.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/roi_ops.py new file mode 100644 index 0000000..2eab277 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/roi_ops.py @@ -0,0 +1,237 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""ROI-related ops.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow.compat.v2 as tf + +from official.vision.detection.ops import nms +from official.vision.detection.utils import box_utils + + +def multilevel_propose_rois(rpn_boxes, + rpn_scores, + anchor_boxes, + image_shape, + rpn_pre_nms_top_k=2000, + rpn_post_nms_top_k=1000, + rpn_nms_threshold=0.7, + rpn_score_threshold=0.0, + rpn_min_size_threshold=0.0, + decode_boxes=True, + clip_boxes=True, + use_batched_nms=False, + apply_sigmoid_to_score=True): + """Proposes RoIs given a group of candidates from different FPN levels. + + The following describes the steps: + 1. For each individual level: + a. Apply sigmoid transform if specified. + b. Decode boxes if specified. + c. Clip boxes if specified. + d. Filter small boxes and those fall outside image if specified. + e. Apply pre-NMS filtering including pre-NMS top k and score thresholding. + f. Apply NMS. + 2. Aggregate post-NMS boxes from each level. + 3. Apply an overall top k to generate the final selected RoIs. + + Args: + rpn_boxes: a dict with keys representing FPN levels and values representing + box tenors of shape [batch_size, feature_h, feature_w, num_anchors * 4]. + rpn_scores: a dict with keys representing FPN levels and values representing + logit tensors of shape [batch_size, feature_h, feature_w, num_anchors]. + anchor_boxes: a dict with keys representing FPN levels and values + representing anchor box tensors of shape + [batch_size, feature_h, feature_w, num_anchors * 4]. + image_shape: a tensor of shape [batch_size, 2] where the last dimension are + [height, width] of the scaled image. + rpn_pre_nms_top_k: an integer of top scoring RPN proposals *per level* to + keep before applying NMS. Default: 2000. + rpn_post_nms_top_k: an integer of top scoring RPN proposals *in total* to + keep after applying NMS. Default: 1000. + rpn_nms_threshold: a float between 0 and 1 representing the IoU threshold + used for NMS. If 0.0, no NMS is applied. Default: 0.7. + rpn_score_threshold: a float between 0 and 1 representing the minimal box + score to keep before applying NMS. This is often used as a pre-filtering + step for better performance. If 0, no filtering is applied. Default: 0. + rpn_min_size_threshold: a float representing the minimal box size in each + side (w.r.t. the scaled image) to keep before applying NMS. This is often + used as a pre-filtering step for better performance. If 0, no filtering is + applied. Default: 0. + decode_boxes: a boolean indicating whether `rpn_boxes` needs to be decoded + using `anchor_boxes`. If False, use `rpn_boxes` directly and ignore + `anchor_boxes`. Default: True. + clip_boxes: a boolean indicating whether boxes are first clipped to the + scaled image size before appliying NMS. If False, no clipping is applied + and `image_shape` is ignored. Default: True. + use_batched_nms: a boolean indicating whether NMS is applied in batch using + `tf.image.combined_non_max_suppression`. Currently only available in + CPU/GPU. Default: False. + apply_sigmoid_to_score: a boolean indicating whether apply sigmoid to + `rpn_scores` before applying NMS. Default: True. + + Returns: + selected_rois: a tensor of shape [batch_size, rpn_post_nms_top_k, 4], + representing the box coordinates of the selected proposals w.r.t. the + scaled image. + selected_roi_scores: a tensor of shape [batch_size, rpn_post_nms_top_k, 1], + representing the scores of the selected proposals. + """ + with tf.name_scope('multilevel_propose_rois'): + rois = [] + roi_scores = [] + image_shape = tf.expand_dims(image_shape, axis=1) + for level in sorted(rpn_scores.keys()): + with tf.name_scope('level_%d' % level): + _, feature_h, feature_w, num_anchors_per_location = ( + rpn_scores[level].get_shape().as_list()) + + num_boxes = feature_h * feature_w * num_anchors_per_location + this_level_scores = tf.reshape(rpn_scores[level], [-1, num_boxes]) + this_level_boxes = tf.reshape(rpn_boxes[level], [-1, num_boxes, 4]) + this_level_anchors = tf.cast( + tf.reshape(anchor_boxes[level], [-1, num_boxes, 4]), + dtype=this_level_scores.dtype) + + if apply_sigmoid_to_score: + this_level_scores = tf.sigmoid(this_level_scores) + + if decode_boxes: + this_level_boxes = box_utils.decode_boxes( + this_level_boxes, this_level_anchors) + if clip_boxes: + this_level_boxes = box_utils.clip_boxes( + this_level_boxes, image_shape) + + if rpn_min_size_threshold > 0.0: + this_level_boxes, this_level_scores = box_utils.filter_boxes( + this_level_boxes, + this_level_scores, + image_shape, + rpn_min_size_threshold) + + this_level_pre_nms_top_k = min(num_boxes, rpn_pre_nms_top_k) + this_level_post_nms_top_k = min(num_boxes, rpn_post_nms_top_k) + if rpn_nms_threshold > 0.0: + if use_batched_nms: + this_level_rois, this_level_roi_scores, _, _ = ( + tf.image.combined_non_max_suppression( + tf.expand_dims(this_level_boxes, axis=2), + tf.expand_dims(this_level_scores, axis=-1), + max_output_size_per_class=this_level_pre_nms_top_k, + max_total_size=this_level_post_nms_top_k, + iou_threshold=rpn_nms_threshold, + score_threshold=rpn_score_threshold, + pad_per_class=False, + clip_boxes=False)) + else: + if rpn_score_threshold > 0.0: + this_level_boxes, this_level_scores = ( + box_utils.filter_boxes_by_scores( + this_level_boxes, this_level_scores, rpn_score_threshold)) + this_level_boxes, this_level_scores = box_utils.top_k_boxes( + this_level_boxes, this_level_scores, k=this_level_pre_nms_top_k) + this_level_roi_scores, this_level_rois = ( + nms.sorted_non_max_suppression_padded( + this_level_scores, + this_level_boxes, + max_output_size=this_level_post_nms_top_k, + iou_threshold=rpn_nms_threshold)) + else: + this_level_rois, this_level_roi_scores = box_utils.top_k_boxes( + this_level_rois, + this_level_scores, + k=this_level_post_nms_top_k) + + rois.append(this_level_rois) + roi_scores.append(this_level_roi_scores) + + all_rois = tf.concat(rois, axis=1) + all_roi_scores = tf.concat(roi_scores, axis=1) + + with tf.name_scope('top_k_rois'): + _, num_valid_rois = all_roi_scores.get_shape().as_list() + overall_top_k = min(num_valid_rois, rpn_post_nms_top_k) + + selected_rois, selected_roi_scores = box_utils.top_k_boxes( + all_rois, all_roi_scores, k=overall_top_k) + + return selected_rois, selected_roi_scores + + +class ROIGenerator(object): + """Proposes RoIs for the second stage processing.""" + + def __init__(self, params): + self._rpn_pre_nms_top_k = params.rpn_pre_nms_top_k + self._rpn_post_nms_top_k = params.rpn_post_nms_top_k + self._rpn_nms_threshold = params.rpn_nms_threshold + self._rpn_score_threshold = params.rpn_score_threshold + self._rpn_min_size_threshold = params.rpn_min_size_threshold + self._test_rpn_pre_nms_top_k = params.test_rpn_pre_nms_top_k + self._test_rpn_post_nms_top_k = params.test_rpn_post_nms_top_k + self._test_rpn_nms_threshold = params.test_rpn_nms_threshold + self._test_rpn_score_threshold = params.test_rpn_score_threshold + self._test_rpn_min_size_threshold = params.test_rpn_min_size_threshold + self._use_batched_nms = params.use_batched_nms + + def __call__(self, boxes, scores, anchor_boxes, image_shape, is_training): + """Generates RoI proposals. + + Args: + boxes: a dict with keys representing FPN levels and values representing + box tenors of shape [batch_size, feature_h, feature_w, num_anchors * 4]. + scores: a dict with keys representing FPN levels and values representing + logit tensors of shape [batch_size, feature_h, feature_w, num_anchors]. + anchor_boxes: a dict with keys representing FPN levels and values + representing anchor box tensors of shape + [batch_size, feature_h, feature_w, num_anchors * 4]. + image_shape: a tensor of shape [batch_size, 2] where the last dimension + are [height, width] of the scaled image. + is_training: a bool indicating whether it is in training or inference + mode. + + Returns: + proposed_rois: a tensor of shape [batch_size, rpn_post_nms_top_k, 4], + representing the box coordinates of the proposed RoIs w.r.t. the + scaled image. + proposed_roi_scores: a tensor of shape + [batch_size, rpn_post_nms_top_k, 1], representing the scores of the + proposed RoIs. + + """ + proposed_rois, proposed_roi_scores = multilevel_propose_rois( + boxes, + scores, + anchor_boxes, + image_shape, + rpn_pre_nms_top_k=(self._rpn_pre_nms_top_k if is_training + else self._test_rpn_pre_nms_top_k), + rpn_post_nms_top_k=(self._rpn_post_nms_top_k if is_training + else self._test_rpn_post_nms_top_k), + rpn_nms_threshold=(self._rpn_nms_threshold if is_training + else self._test_rpn_nms_threshold), + rpn_score_threshold=(self._rpn_score_threshold if is_training + else self._test_rpn_score_threshold), + rpn_min_size_threshold=(self._rpn_min_size_threshold if is_training + else self._test_rpn_min_size_threshold), + decode_boxes=True, + clip_boxes=True, + use_batched_nms=self._use_batched_nms, + apply_sigmoid_to_score=True) + return proposed_rois, proposed_roi_scores diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/sampling_ops.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/sampling_ops.py new file mode 100644 index 0000000..1777b9d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/sampling_ops.py @@ -0,0 +1,399 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Sampling related ops.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow.compat.v2 as tf + +from official.vision.detection.ops import spatial_transform_ops +from official.vision.detection.utils import box_utils +from official.vision.detection.utils.object_detection import balanced_positive_negative_sampler + + +def box_matching(boxes, gt_boxes, gt_classes): + """Match boxes to groundtruth boxes. + + Given the proposal boxes and the groundtruth boxes and classes, perform the + groundtruth matching by taking the argmax of the IoU between boxes and + groundtruth boxes. + + Args: + boxes: a tensor of shape of [batch_size, N, 4] representing the box + coordiantes to be matched to groundtruth boxes. + gt_boxes: a tensor of shape of [batch_size, MAX_INSTANCES, 4] representing + the groundtruth box coordinates. It is padded with -1s to indicate the + invalid boxes. + gt_classes: [batch_size, MAX_INSTANCES] representing the groundtruth box + classes. It is padded with -1s to indicate the invalid classes. + + Returns: + matched_gt_boxes: a tensor of shape of [batch_size, N, 4], representing + the matched groundtruth box coordinates for each input box. If the box + does not overlap with any groundtruth boxes, the matched boxes of it + will be set to all 0s. + matched_gt_classes: a tensor of shape of [batch_size, N], representing + the matched groundtruth classes for each input box. If the box does not + overlap with any groundtruth boxes, the matched box classes of it will + be set to 0, which corresponds to the background class. + matched_gt_indices: a tensor of shape of [batch_size, N], representing + the indices of the matched groundtruth boxes in the original gt_boxes + tensor. If the box does not overlap with any groundtruth boxes, the + index of the matched groundtruth will be set to -1. + matched_iou: a tensor of shape of [batch_size, N], representing the IoU + between the box and its matched groundtruth box. The matched IoU is the + maximum IoU of the box and all the groundtruth boxes. + iou: a tensor of shape of [batch_size, N, K], representing the IoU matrix + between boxes and the groundtruth boxes. The IoU between a box and the + invalid groundtruth boxes whose coordinates are [-1, -1, -1, -1] is -1. + """ + # Compute IoU between boxes and gt_boxes. + # iou <- [batch_size, N, K] + iou = box_utils.bbox_overlap(boxes, gt_boxes) + + # max_iou <- [batch_size, N] + # 0.0 -> no match to gt, or -1.0 match to no gt + matched_iou = tf.reduce_max(iou, axis=-1) + + # background_box_mask <- bool, [batch_size, N] + background_box_mask = tf.less_equal(matched_iou, 0.0) + + argmax_iou_indices = tf.argmax(iou, axis=-1, output_type=tf.int32) + + argmax_iou_indices_shape = tf.shape(argmax_iou_indices) + batch_indices = ( + tf.expand_dims(tf.range(argmax_iou_indices_shape[0]), axis=-1) * + tf.ones([1, argmax_iou_indices_shape[-1]], dtype=tf.int32)) + gather_nd_indices = tf.stack([batch_indices, argmax_iou_indices], axis=-1) + + matched_gt_boxes = tf.gather_nd(gt_boxes, gather_nd_indices) + matched_gt_boxes = tf.where( + tf.tile(tf.expand_dims(background_box_mask, axis=-1), [1, 1, 4]), + tf.zeros_like(matched_gt_boxes, dtype=matched_gt_boxes.dtype), + matched_gt_boxes) + + matched_gt_classes = tf.gather_nd(gt_classes, gather_nd_indices) + matched_gt_classes = tf.where( + background_box_mask, + tf.zeros_like(matched_gt_classes), + matched_gt_classes) + + matched_gt_indices = tf.where( + background_box_mask, + -tf.ones_like(argmax_iou_indices), + argmax_iou_indices) + + return (matched_gt_boxes, matched_gt_classes, matched_gt_indices, + matched_iou, iou) + + +def assign_and_sample_proposals(proposed_boxes, + gt_boxes, + gt_classes, + num_samples_per_image=512, + mix_gt_boxes=True, + fg_fraction=0.25, + fg_iou_thresh=0.5, + bg_iou_thresh_hi=0.5, + bg_iou_thresh_lo=0.0): + """Assigns the proposals with groundtruth classes and performs subsmpling. + + Given `proposed_boxes`, `gt_boxes`, and `gt_classes`, the function uses the + following algorithm to generate the final `num_samples_per_image` RoIs. + 1. Calculates the IoU between each proposal box and each gt_boxes. + 2. Assigns each proposed box with a groundtruth class and box by choosing + the largest IoU overlap. + 3. Samples `num_samples_per_image` boxes from all proposed boxes, and + returns box_targets, class_targets, and RoIs. + + Args: + proposed_boxes: a tensor of shape of [batch_size, N, 4]. N is the number + of proposals before groundtruth assignment. The last dimension is the + box coordinates w.r.t. the scaled images in [ymin, xmin, ymax, xmax] + format. + gt_boxes: a tensor of shape of [batch_size, MAX_NUM_INSTANCES, 4]. + The coordinates of gt_boxes are in the pixel coordinates of the scaled + image. This tensor might have padding of values -1 indicating the invalid + box coordinates. + gt_classes: a tensor with a shape of [batch_size, MAX_NUM_INSTANCES]. This + tensor might have paddings with values of -1 indicating the invalid + classes. + num_samples_per_image: a integer represents RoI minibatch size per image. + mix_gt_boxes: a bool indicating whether to mix the groundtruth boxes before + sampling proposals. + fg_fraction: a float represents the target fraction of RoI minibatch that + is labeled foreground (i.e., class > 0). + fg_iou_thresh: a float represents the IoU overlap threshold for an RoI to be + considered foreground (if >= fg_iou_thresh). + bg_iou_thresh_hi: a float represents the IoU overlap threshold for an RoI to + be considered background (class = 0 if overlap in [LO, HI)). + bg_iou_thresh_lo: a float represents the IoU overlap threshold for an RoI to + be considered background (class = 0 if overlap in [LO, HI)). + + Returns: + sampled_rois: a tensor of shape of [batch_size, K, 4], representing the + coordinates of the sampled RoIs, where K is the number of the sampled + RoIs, i.e. K = num_samples_per_image. + sampled_gt_boxes: a tensor of shape of [batch_size, K, 4], storing the + box coordinates of the matched groundtruth boxes of the samples RoIs. + sampled_gt_classes: a tensor of shape of [batch_size, K], storing the + classes of the matched groundtruth boxes of the sampled RoIs. + sampled_gt_indices: a tensor of shape of [batch_size, K], storing the + indices of the sampled groudntruth boxes in the original `gt_boxes` + tensor, i.e. gt_boxes[sampled_gt_indices[:, i]] = sampled_gt_boxes[:, i]. + """ + + with tf.name_scope('sample_proposals'): + if mix_gt_boxes: + boxes = tf.concat([proposed_boxes, gt_boxes], axis=1) + else: + boxes = proposed_boxes + + (matched_gt_boxes, matched_gt_classes, matched_gt_indices, + matched_iou, _) = box_matching(boxes, gt_boxes, gt_classes) + + positive_match = tf.greater(matched_iou, fg_iou_thresh) + negative_match = tf.logical_and( + tf.greater_equal(matched_iou, bg_iou_thresh_lo), + tf.less(matched_iou, bg_iou_thresh_hi)) + ignored_match = tf.less(matched_iou, 0.0) + + # re-assign negatively matched boxes to the background class. + matched_gt_classes = tf.where( + negative_match, tf.zeros_like(matched_gt_classes), matched_gt_classes) + matched_gt_indices = tf.where( + negative_match, tf.zeros_like(matched_gt_indices), matched_gt_indices) + + sample_candidates = tf.logical_and( + tf.logical_or(positive_match, negative_match), + tf.logical_not(ignored_match)) + + sampler = ( + balanced_positive_negative_sampler.BalancedPositiveNegativeSampler( + positive_fraction=fg_fraction, is_static=True)) + + batch_size, _ = sample_candidates.get_shape().as_list() + sampled_indicators = [] + for i in range(batch_size): + sampled_indicator = sampler.subsample( + sample_candidates[i], num_samples_per_image, positive_match[i]) + sampled_indicators.append(sampled_indicator) + sampled_indicators = tf.stack(sampled_indicators) + _, sampled_indices = tf.nn.top_k( + tf.cast(sampled_indicators, dtype=tf.int32), + k=num_samples_per_image, + sorted=True) + + sampled_indices_shape = tf.shape(sampled_indices) + batch_indices = ( + tf.expand_dims(tf.range(sampled_indices_shape[0]), axis=-1) * + tf.ones([1, sampled_indices_shape[-1]], dtype=tf.int32)) + gather_nd_indices = tf.stack([batch_indices, sampled_indices], axis=-1) + + sampled_rois = tf.gather_nd(boxes, gather_nd_indices) + sampled_gt_boxes = tf.gather_nd(matched_gt_boxes, gather_nd_indices) + sampled_gt_classes = tf.gather_nd( + matched_gt_classes, gather_nd_indices) + sampled_gt_indices = tf.gather_nd( + matched_gt_indices, gather_nd_indices) + + return (sampled_rois, sampled_gt_boxes, sampled_gt_classes, + sampled_gt_indices) + + +def sample_and_crop_foreground_masks(candidate_rois, + candidate_gt_boxes, + candidate_gt_classes, + candidate_gt_indices, + gt_masks, + num_mask_samples_per_image=128, + mask_target_size=28): + """Samples and creates cropped foreground masks for training. + + Args: + candidate_rois: a tensor of shape of [batch_size, N, 4], where N is the + number of candidate RoIs to be considered for mask sampling. It includes + both positive and negative RoIs. The `num_mask_samples_per_image` positive + RoIs will be sampled to create mask training targets. + candidate_gt_boxes: a tensor of shape of [batch_size, N, 4], storing the + corresponding groundtruth boxes to the `candidate_rois`. + candidate_gt_classes: a tensor of shape of [batch_size, N], storing the + corresponding groundtruth classes to the `candidate_rois`. 0 in the tensor + corresponds to the background class, i.e. negative RoIs. + candidate_gt_indices: a tensor of shape [batch_size, N], storing the + corresponding groundtruth instance indices to the `candidate_gt_boxes`, + i.e. gt_boxes[candidate_gt_indices[:, i]] = candidate_gt_boxes[:, i] and + gt_boxes which is of shape [batch_size, MAX_INSTANCES, 4], M >= N, is the + superset of candidate_gt_boxes. + gt_masks: a tensor of [batch_size, MAX_INSTANCES, mask_height, mask_width] + containing all the groundtruth masks which sample masks are drawn from. + num_mask_samples_per_image: an integer which specifies the number of masks + to sample. + mask_target_size: an integer which specifies the final cropped mask size + after sampling. The output masks are resized w.r.t the sampled RoIs. + + Returns: + foreground_rois: a tensor of shape of [batch_size, K, 4] storing the RoI + that corresponds to the sampled foreground masks, where + K = num_mask_samples_per_image. + foreground_classes: a tensor of shape of [batch_size, K] storing the classes + corresponding to the sampled foreground masks. + cropoped_foreground_masks: a tensor of shape of + [batch_size, K, mask_target_size, mask_target_size] storing the cropped + foreground masks used for training. + """ + with tf.name_scope('sample_and_crop_foreground_masks'): + _, fg_instance_indices = tf.nn.top_k( + tf.cast(tf.greater(candidate_gt_classes, 0), dtype=tf.int32), + k=num_mask_samples_per_image) + + fg_instance_indices_shape = tf.shape(fg_instance_indices) + batch_indices = ( + tf.expand_dims(tf.range(fg_instance_indices_shape[0]), axis=-1) * + tf.ones([1, fg_instance_indices_shape[-1]], dtype=tf.int32)) + + gather_nd_instance_indices = tf.stack( + [batch_indices, fg_instance_indices], axis=-1) + foreground_rois = tf.gather_nd( + candidate_rois, gather_nd_instance_indices) + foreground_boxes = tf.gather_nd( + candidate_gt_boxes, gather_nd_instance_indices) + foreground_classes = tf.gather_nd( + candidate_gt_classes, gather_nd_instance_indices) + foreground_gt_indices = tf.gather_nd( + candidate_gt_indices, gather_nd_instance_indices) + + foreground_gt_indices_shape = tf.shape(foreground_gt_indices) + batch_indices = ( + tf.expand_dims(tf.range(foreground_gt_indices_shape[0]), axis=-1) * + tf.ones([1, foreground_gt_indices_shape[-1]], dtype=tf.int32)) + gather_nd_gt_indices = tf.stack( + [batch_indices, foreground_gt_indices], axis=-1) + foreground_masks = tf.gather_nd(gt_masks, gather_nd_gt_indices) + + cropped_foreground_masks = spatial_transform_ops.crop_mask_in_target_box( + foreground_masks, foreground_boxes, foreground_rois, mask_target_size, + sample_offset=0.5) + + return foreground_rois, foreground_classes, cropped_foreground_masks + + +class ROISampler(object): + """Samples RoIs and creates training targets.""" + + def __init__(self, params): + self._num_samples_per_image = params.num_samples_per_image + self._fg_fraction = params.fg_fraction + self._fg_iou_thresh = params.fg_iou_thresh + self._bg_iou_thresh_hi = params.bg_iou_thresh_hi + self._bg_iou_thresh_lo = params.bg_iou_thresh_lo + self._mix_gt_boxes = params.mix_gt_boxes + + def __call__(self, rois, gt_boxes, gt_classes): + """Sample and assign RoIs for training. + + Args: + rois: a tensor of shape of [batch_size, N, 4]. N is the number + of proposals before groundtruth assignment. The last dimension is the + box coordinates w.r.t. the scaled images in [ymin, xmin, ymax, xmax] + format. + gt_boxes: a tensor of shape of [batch_size, MAX_NUM_INSTANCES, 4]. + The coordinates of gt_boxes are in the pixel coordinates of the scaled + image. This tensor might have padding of values -1 indicating the + invalid box coordinates. + gt_classes: a tensor with a shape of [batch_size, MAX_NUM_INSTANCES]. This + tensor might have paddings with values of -1 indicating the invalid + classes. + + Returns: + sampled_rois: a tensor of shape of [batch_size, K, 4], representing the + coordinates of the sampled RoIs, where K is the number of the sampled + RoIs, i.e. K = num_samples_per_image. + sampled_gt_boxes: a tensor of shape of [batch_size, K, 4], storing the + box coordinates of the matched groundtruth boxes of the samples RoIs. + sampled_gt_classes: a tensor of shape of [batch_size, K], storing the + classes of the matched groundtruth boxes of the sampled RoIs. + """ + sampled_rois, sampled_gt_boxes, sampled_gt_classes, sampled_gt_indices = ( + assign_and_sample_proposals( + rois, + gt_boxes, + gt_classes, + num_samples_per_image=self._num_samples_per_image, + mix_gt_boxes=self._mix_gt_boxes, + fg_fraction=self._fg_fraction, + fg_iou_thresh=self._fg_iou_thresh, + bg_iou_thresh_hi=self._bg_iou_thresh_hi, + bg_iou_thresh_lo=self._bg_iou_thresh_lo)) + return (sampled_rois, sampled_gt_boxes, sampled_gt_classes, + sampled_gt_indices) + + +class MaskSampler(object): + """Samples and creates mask training targets.""" + + def __init__(self, params): + self._num_mask_samples_per_image = params.num_mask_samples_per_image + self._mask_target_size = params.mask_target_size + + def __call__(self, + candidate_rois, + candidate_gt_boxes, + candidate_gt_classes, + candidate_gt_indices, + gt_masks): + """Sample and create mask targets for training. + + Args: + candidate_rois: a tensor of shape of [batch_size, N, 4], where N is the + number of candidate RoIs to be considered for mask sampling. It includes + both positive and negative RoIs. The `num_mask_samples_per_image` + positive RoIs will be sampled to create mask training targets. + candidate_gt_boxes: a tensor of shape of [batch_size, N, 4], storing the + corresponding groundtruth boxes to the `candidate_rois`. + candidate_gt_classes: a tensor of shape of [batch_size, N], storing the + corresponding groundtruth classes to the `candidate_rois`. 0 in the + tensor corresponds to the background class, i.e. negative RoIs. + candidate_gt_indices: a tensor of shape [batch_size, N], storing the + corresponding groundtruth instance indices to the `candidate_gt_boxes`, + i.e. gt_boxes[candidate_gt_indices[:, i]] = candidate_gt_boxes[:, i], + where gt_boxes which is of shape [batch_size, MAX_INSTANCES, 4], M >= N, + is the superset of candidate_gt_boxes. + gt_masks: a tensor of [batch_size, MAX_INSTANCES, mask_height, mask_width] + containing all the groundtruth masks which sample masks are drawn from. + after sampling. The output masks are resized w.r.t the sampled RoIs. + + Returns: + foreground_rois: a tensor of shape of [batch_size, K, 4] storing the RoI + that corresponds to the sampled foreground masks, where + K = num_mask_samples_per_image. + foreground_classes: a tensor of shape of [batch_size, K] storing the + classes corresponding to the sampled foreground masks. + cropoped_foreground_masks: a tensor of shape of + [batch_size, K, mask_target_size, mask_target_size] storing the + cropped foreground masks used for training. + """ + foreground_rois, foreground_classes, cropped_foreground_masks = ( + sample_and_crop_foreground_masks( + candidate_rois, + candidate_gt_boxes, + candidate_gt_classes, + candidate_gt_indices, + gt_masks, + self._num_mask_samples_per_image, + self._mask_target_size)) + return foreground_rois, foreground_classes, cropped_foreground_masks diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/spatial_transform_ops.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/spatial_transform_ops.py new file mode 100644 index 0000000..60ef494 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/ops/spatial_transform_ops.py @@ -0,0 +1,608 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functions to performa spatial transformation for Tensor.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow.compat.v2 as tf + + +_EPSILON = 1e-8 + + +def nearest_upsampling(data, scale): + """Nearest neighbor upsampling implementation. + + Args: + data: A tensor with a shape of [batch, height_in, width_in, channels]. + scale: An integer multiple to scale resolution of input data. + Returns: + data_up: A tensor with a shape of + [batch, height_in*scale, width_in*scale, channels]. Same dtype as input + data. + """ + with tf.name_scope('nearest_upsampling'): + bs, _, _, c = data.get_shape().as_list() + shape = tf.shape(input=data) + h = shape[1] + w = shape[2] + bs = -1 if bs is None else bs + # Uses reshape to quickly upsample the input. The nearest pixel is selected + # implicitly via broadcasting. + data = tf.reshape(data, [bs, h, 1, w, 1, c]) * tf.ones( + [1, 1, scale, 1, scale, 1], dtype=data.dtype) + return tf.reshape(data, [bs, h * scale, w * scale, c]) + + +def feature_bilinear_interpolation(features, kernel_y, kernel_x): + """Feature bilinear interpolation. + + The RoIAlign feature f can be computed by bilinear interpolation + of four neighboring feature points f0, f1, f2, and f3. + + f(y, x) = [hy, ly] * [[f00, f01], * [hx, lx]^T + [f10, f11]] + f(y, x) = (hy*hx)f00 + (hy*lx)f01 + (ly*hx)f10 + (lx*ly)f11 + f(y, x) = w00*f00 + w01*f01 + w10*f10 + w11*f11 + kernel_y = [hy, ly] + kernel_x = [hx, lx] + + Args: + features: The features are in shape of [batch_size, num_boxes, output_size * + 2, output_size * 2, num_filters]. + kernel_y: Tensor of size [batch_size, boxes, output_size, 2, 1]. + kernel_x: Tensor of size [batch_size, boxes, output_size, 2, 1]. + + Returns: + A 5-D tensor representing feature crop of shape + [batch_size, num_boxes, output_size, output_size, num_filters]. + + """ + (batch_size, num_boxes, output_size, _, + num_filters) = features.get_shape().as_list() + output_size = output_size // 2 + kernel_y = tf.reshape(kernel_y, [batch_size, num_boxes, output_size * 2, 1]) + kernel_x = tf.reshape(kernel_x, [batch_size, num_boxes, 1, output_size * 2]) + # Use implicit broadcast to generate the interpolation kernel. The + # multiplier `4` is for avg pooling. + interpolation_kernel = kernel_y * kernel_x * 4 + + # Interpolate the gathered features with computed interpolation kernels. + features *= tf.cast( + tf.expand_dims(interpolation_kernel, axis=-1), dtype=features.dtype) + features = tf.reshape( + features, + [batch_size * num_boxes, output_size * 2, output_size * 2, num_filters]) + features = tf.nn.avg_pool(features, [1, 2, 2, 1], [1, 2, 2, 1], 'VALID') + features = tf.reshape( + features, [batch_size, num_boxes, output_size, output_size, num_filters]) + return features + + +def compute_grid_positions(boxes, boundaries, output_size, sample_offset): + """Compute the grid position w.r.t. + + the corresponding feature map. + + Args: + boxes: a 3-D tensor of shape [batch_size, num_boxes, 4] encoding the + information of each box w.r.t. the corresponding feature map. + boxes[:, :, 0:2] are the grid position in (y, x) (float) of the top-left + corner of each box. boxes[:, :, 2:4] are the box sizes in (h, w) (float) + in terms of the number of pixels of the corresponding feature map size. + boundaries: a 3-D tensor of shape [batch_size, num_boxes, 2] representing + the boundary (in (y, x)) of the corresponding feature map for each box. + Any resampled grid points that go beyond the bounary will be clipped. + output_size: a scalar indicating the output crop size. + sample_offset: a float number in [0, 1] indicates the subpixel sample offset + from grid point. + + Returns: + kernel_y: Tensor of size [batch_size, boxes, output_size, 2, 1]. + kernel_x: Tensor of size [batch_size, boxes, output_size, 2, 1]. + box_grid_y0y1: Tensor of size [batch_size, boxes, output_size, 2] + box_grid_x0x1: Tensor of size [batch_size, boxes, output_size, 2] + """ + batch_size, num_boxes, _ = boxes.get_shape().as_list() + box_grid_x = [] + box_grid_y = [] + for i in range(output_size): + box_grid_x.append(boxes[:, :, 1] + + (i + sample_offset) * boxes[:, :, 3] / output_size) + box_grid_y.append(boxes[:, :, 0] + + (i + sample_offset) * boxes[:, :, 2] / output_size) + box_grid_x = tf.stack(box_grid_x, axis=2) + box_grid_y = tf.stack(box_grid_y, axis=2) + + box_grid_y0 = tf.floor(box_grid_y) + box_grid_x0 = tf.floor(box_grid_x) + box_grid_x0 = tf.maximum(0., box_grid_x0) + box_grid_y0 = tf.maximum(0., box_grid_y0) + + box_grid_x0 = tf.minimum(box_grid_x0, tf.expand_dims(boundaries[:, :, 1], -1)) + box_grid_x1 = tf.minimum(box_grid_x0 + 1, + tf.expand_dims(boundaries[:, :, 1], -1)) + box_grid_y0 = tf.minimum(box_grid_y0, tf.expand_dims(boundaries[:, :, 0], -1)) + box_grid_y1 = tf.minimum(box_grid_y0 + 1, + tf.expand_dims(boundaries[:, :, 0], -1)) + + box_gridx0x1 = tf.stack([box_grid_x0, box_grid_x1], axis=-1) + box_gridy0y1 = tf.stack([box_grid_y0, box_grid_y1], axis=-1) + + # The RoIAlign feature f can be computed by bilinear interpolation of four + # neighboring feature points f0, f1, f2, and f3. + # f(y, x) = [hy, ly] * [[f00, f01], * [hx, lx]^T + # [f10, f11]] + # f(y, x) = (hy*hx)f00 + (hy*lx)f01 + (ly*hx)f10 + (lx*ly)f11 + # f(y, x) = w00*f00 + w01*f01 + w10*f10 + w11*f11 + ly = box_grid_y - box_grid_y0 + lx = box_grid_x - box_grid_x0 + hy = 1.0 - ly + hx = 1.0 - lx + kernel_y = tf.reshape( + tf.stack([hy, ly], axis=3), [batch_size, num_boxes, output_size, 2, 1]) + kernel_x = tf.reshape( + tf.stack([hx, lx], axis=3), [batch_size, num_boxes, output_size, 2, 1]) + return kernel_y, kernel_x, box_gridy0y1, box_gridx0x1 + + +def get_grid_one_hot(box_gridy0y1, box_gridx0x1, feature_height, feature_width): + """Get grid_one_hot from indices and feature_size.""" + (batch_size, num_boxes, output_size, _) = box_gridx0x1.get_shape().as_list() + y_indices = tf.cast( + tf.reshape(box_gridy0y1, [batch_size, num_boxes, output_size, 2]), + dtype=tf.int32) + x_indices = tf.cast( + tf.reshape(box_gridx0x1, [batch_size, num_boxes, output_size, 2]), + dtype=tf.int32) + + # shape is [batch_size, num_boxes, output_size, 2, height] + grid_y_one_hot = tf.one_hot(tf.cast(y_indices, tf.int32), feature_height) + # shape is [batch_size, num_boxes, output_size, 2, width] + grid_x_one_hot = tf.one_hot(tf.cast(x_indices, tf.int32), feature_width) + + return grid_y_one_hot, grid_x_one_hot + + +def selective_crop_and_resize(features, + boxes, + box_levels, + boundaries, + output_size=7, + sample_offset=0.5, + use_einsum_gather=False): + """Crop and resize boxes on a set of feature maps. + + Given multiple features maps indexed by different levels, and a set of boxes + where each box is mapped to a certain level, it selectively crops and resizes + boxes from the corresponding feature maps to generate the box features. + + We follow the ROIAlign technique (see https://arxiv.org/pdf/1703.06870.pdf, + figure 3 for reference). Specifically, for each feature map, we select an + (output_size, output_size) set of pixels corresponding to the box location, + and then use bilinear interpolation to select the feature value for each + pixel. + + For performance, we perform the gather and interpolation on all layers as a + single operation. In this op the multi-level features are first stacked and + gathered into [2*output_size, 2*output_size] feature points. Then bilinear + interpolation is performed on the gathered feature points to generate + [output_size, output_size] RoIAlign feature map. + + Here is the step-by-step algorithm: + 1. The multi-level features are gathered into a + [batch_size, num_boxes, output_size*2, output_size*2, num_filters] + Tensor. The Tensor contains four neighboring feature points for each + vertice in the output grid. + 2. Compute the interpolation kernel of shape + [batch_size, num_boxes, output_size*2, output_size*2]. The last 2 axis + can be seen as stacking 2x2 interpolation kernels for all vertices in the + output grid. + 3. Element-wise multiply the gathered features and interpolation kernel. + Then apply 2x2 average pooling to reduce spatial dimension to + output_size. + + Args: + features: a 5-D tensor of shape [batch_size, num_levels, max_height, + max_width, num_filters] where cropping and resizing are based. + boxes: a 3-D tensor of shape [batch_size, num_boxes, 4] encoding the + information of each box w.r.t. the corresponding feature map. + boxes[:, :, 0:2] are the grid position in (y, x) (float) of the top-left + corner of each box. boxes[:, :, 2:4] are the box sizes in (h, w) (float) + in terms of the number of pixels of the corresponding feature map size. + box_levels: a 3-D tensor of shape [batch_size, num_boxes, 1] representing + the 0-based corresponding feature level index of each box. + boundaries: a 3-D tensor of shape [batch_size, num_boxes, 2] representing + the boundary (in (y, x)) of the corresponding feature map for each box. + Any resampled grid points that go beyond the bounary will be clipped. + output_size: a scalar indicating the output crop size. + sample_offset: a float number in [0, 1] indicates the subpixel sample offset + from grid point. + use_einsum_gather: use einsum to replace gather or not. Replacing einsum + with gather can improve performance when feature size is not large, einsum + is friendly with model partition as well. Gather's performance is better + when feature size is very large and there are multiple box levels. + + Returns: + features_per_box: a 5-D tensor of shape + [batch_size, num_boxes, output_size, output_size, num_filters] + representing the cropped features. + """ + (batch_size, num_levels, max_feature_height, max_feature_width, + num_filters) = features.get_shape().as_list() + _, num_boxes, _ = boxes.get_shape().as_list() + + kernel_y, kernel_x, box_gridy0y1, box_gridx0x1 = compute_grid_positions( + boxes, boundaries, output_size, sample_offset) + x_indices = tf.cast( + tf.reshape(box_gridx0x1, [batch_size, num_boxes, output_size * 2]), + dtype=tf.int32) + y_indices = tf.cast( + tf.reshape(box_gridy0y1, [batch_size, num_boxes, output_size * 2]), + dtype=tf.int32) + + if use_einsum_gather: + # Blinear interpolation is done during the last two gathers: + # f(y, x) = [hy, ly] * [[f00, f01], * [hx, lx]^T + # [f10, f11]] + # [[f00, f01], + # [f10, f11]] = tf.einsum(tf.einsum(features, y_one_hot), x_one_hot) + # where [hy, ly] and [hx, lx] are the bilinear interpolation kernel. + + # shape is [batch_size, boxes, output_size, 2, 1] + grid_y_one_hot, grid_x_one_hot = get_grid_one_hot(box_gridy0y1, + box_gridx0x1, + max_feature_height, + max_feature_width) + + # shape is [batch_size, num_boxes, output_size, height] + grid_y_weight = tf.reduce_sum( + tf.multiply(grid_y_one_hot, kernel_y), axis=-2) + # shape is [batch_size, num_boxes, output_size, width] + grid_x_weight = tf.reduce_sum( + tf.multiply(grid_x_one_hot, kernel_x), axis=-2) + + # Gather for y_axis. + # shape is [batch_size, num_boxes, output_size, width, features] + features_per_box = tf.einsum('bmhwf,bmoh->bmowf', features, + tf.cast(grid_y_weight, features.dtype)) + # Gather for x_axis. + # shape is [batch_size, num_boxes, output_size, output_size, features] + features_per_box = tf.einsum('bmhwf,bmow->bmhof', features_per_box, + tf.cast(grid_x_weight, features.dtype)) + else: + height_dim_offset = max_feature_width + level_dim_offset = max_feature_height * height_dim_offset + batch_dim_offset = num_levels * level_dim_offset + + batch_size_offset = tf.tile( + tf.reshape( + tf.range(batch_size) * batch_dim_offset, [batch_size, 1, 1, 1]), + [1, num_boxes, output_size * 2, output_size * 2]) + box_levels_offset = tf.tile( + tf.reshape(box_levels * level_dim_offset, + [batch_size, num_boxes, 1, 1]), + [1, 1, output_size * 2, output_size * 2]) + y_indices_offset = tf.tile( + tf.reshape(y_indices * height_dim_offset, + [batch_size, num_boxes, output_size * 2, 1]), + [1, 1, 1, output_size * 2]) + x_indices_offset = tf.tile( + tf.reshape(x_indices, [batch_size, num_boxes, 1, output_size * 2]), + [1, 1, output_size * 2, 1]) + + indices = tf.reshape( + batch_size_offset + box_levels_offset + y_indices_offset + + x_indices_offset, [-1]) + + features = tf.reshape(features, [-1, num_filters]) + # TODO(wangtao): replace tf.gather with tf.gather_nd and try to get similar + # performance. + features_per_box = tf.reshape( + tf.gather(features, indices), + [batch_size, num_boxes, output_size * 2, output_size * 2, num_filters]) + features_per_box = feature_bilinear_interpolation(features_per_box, + kernel_y, kernel_x) + + return features_per_box + + +def multilevel_crop_and_resize(features, boxes, output_size=7): + """Crop and resize on multilevel feature pyramid. + + Generate the (output_size, output_size) set of pixels for each input box + by first locating the box into the correct feature level, and then cropping + and resizing it using the correspoding feature map of that level. + + Args: + features: A dictionary with key as pyramid level and value as features. The + features are in shape of [batch_size, height_l, width_l, num_filters]. + boxes: A 3-D Tensor of shape [batch_size, num_boxes, 4]. Each row represents + a box with [y1, x1, y2, x2] in un-normalized coordinates. + output_size: A scalar to indicate the output crop size. + + Returns: + A 5-D tensor representing feature crop of shape + [batch_size, num_boxes, output_size, output_size, num_filters]. + """ + + with tf.name_scope('multilevel_crop_and_resize'): + levels = list(features.keys()) + min_level = min(levels) + max_level = max(levels) + batch_size, max_feature_height, max_feature_width, num_filters = ( + features[min_level].get_shape().as_list()) + _, num_boxes, _ = boxes.get_shape().as_list() + + # Stack feature pyramid into a features_all of shape + # [batch_size, levels, height, width, num_filters]. + features_all = [] + feature_heights = [] + feature_widths = [] + for level in range(min_level, max_level + 1): + shape = features[level].get_shape().as_list() + feature_heights.append(shape[1]) + feature_widths.append(shape[2]) + # Concat tensor of [batch_size, height_l * width_l, num_filters] for each + # levels. + features_all.append( + tf.reshape(features[level], [batch_size, -1, num_filters])) + features_r2 = tf.reshape(tf.concat(features_all, 1), [-1, num_filters]) + + # Calculate height_l * width_l for each level. + level_dim_sizes = [ + feature_widths[i] * feature_heights[i] + for i in range(len(feature_widths)) + ] + # level_dim_offsets is accumulated sum of level_dim_size. + level_dim_offsets = [0] + for i in range(len(feature_widths) - 1): + level_dim_offsets.append(level_dim_offsets[i] + level_dim_sizes[i]) + batch_dim_size = level_dim_offsets[-1] + level_dim_sizes[-1] + level_dim_offsets = tf.constant(level_dim_offsets, tf.int32) + height_dim_sizes = tf.constant(feature_widths, tf.int32) + + # Assigns boxes to the right level. + box_width = boxes[:, :, 3] - boxes[:, :, 1] + box_height = boxes[:, :, 2] - boxes[:, :, 0] + areas_sqrt = tf.sqrt(box_height * box_width) + levels = tf.cast( + tf.math.floordiv( + tf.math.log(tf.divide(areas_sqrt, 224.0)), tf.math.log(2.0)) + + 4.0, + dtype=tf.int32) + # Maps levels between [min_level, max_level]. + levels = tf.minimum(max_level, tf.maximum(levels, min_level)) + + # Projects box location and sizes to corresponding feature levels. + scale_to_level = tf.cast( + tf.pow(tf.constant(2.0), tf.cast(levels, tf.float32)), + dtype=boxes.dtype) + boxes /= tf.expand_dims(scale_to_level, axis=2) + box_width /= scale_to_level + box_height /= scale_to_level + boxes = tf.concat([boxes[:, :, 0:2], + tf.expand_dims(box_height, -1), + tf.expand_dims(box_width, -1)], axis=-1) + + # Maps levels to [0, max_level-min_level]. + levels -= min_level + level_strides = tf.pow([[2.0]], tf.cast(levels, tf.float32)) + boundary = tf.cast( + tf.concat([ + tf.expand_dims( + [[tf.cast(max_feature_height, tf.float32)]] / level_strides - 1, + axis=-1), + tf.expand_dims( + [[tf.cast(max_feature_width, tf.float32)]] / level_strides - 1, + axis=-1), + ], + axis=-1), boxes.dtype) + + # Compute grid positions. + kernel_y, kernel_x, box_gridy0y1, box_gridx0x1 = compute_grid_positions( + boxes, boundary, output_size, sample_offset=0.5) + + x_indices = tf.cast( + tf.reshape(box_gridx0x1, [batch_size, num_boxes, output_size * 2]), + dtype=tf.int32) + y_indices = tf.cast( + tf.reshape(box_gridy0y1, [batch_size, num_boxes, output_size * 2]), + dtype=tf.int32) + + batch_size_offset = tf.tile( + tf.reshape( + tf.range(batch_size) * batch_dim_size, [batch_size, 1, 1, 1]), + [1, num_boxes, output_size * 2, output_size * 2]) + # Get level offset for each box. Each box belongs to one level. + levels_offset = tf.tile( + tf.reshape( + tf.gather(level_dim_offsets, levels), + [batch_size, num_boxes, 1, 1]), + [1, 1, output_size * 2, output_size * 2]) + y_indices_offset = tf.tile( + tf.reshape( + y_indices * tf.expand_dims(tf.gather(height_dim_sizes, levels), -1), + [batch_size, num_boxes, output_size * 2, 1]), + [1, 1, 1, output_size * 2]) + x_indices_offset = tf.tile( + tf.reshape(x_indices, [batch_size, num_boxes, 1, output_size * 2]), + [1, 1, output_size * 2, 1]) + indices = tf.reshape( + batch_size_offset + levels_offset + y_indices_offset + x_indices_offset, + [-1]) + + # TODO(wangtao): replace tf.gather with tf.gather_nd and try to get similar + # performance. + features_per_box = tf.reshape( + tf.gather(features_r2, indices), + [batch_size, num_boxes, output_size * 2, output_size * 2, num_filters]) + + # Bilinear interpolation. + features_per_box = feature_bilinear_interpolation(features_per_box, + kernel_y, kernel_x) + return features_per_box + + +def single_level_feature_crop(features, level_boxes, detection_prior_levels, + min_mask_level, mask_crop_size): + """Crop the FPN features at the appropriate levels for each detection. + + + Args: + features: a float tensor of shape [batch_size, num_levels, + max_feature_size, max_feature_size, num_downsample_channels]. + level_boxes: a float Tensor of the level boxes to crop from. + [batch_size, num_instances, 4]. + detection_prior_levels: an int Tensor of instance assigned level of shape + [batch_size, num_instances]. + min_mask_level: minimum FPN level to crop mask feature from. + mask_crop_size: an int of mask crop size. + + Returns: + crop_features: a float Tensor of shape [batch_size * num_instances, + mask_crop_size, mask_crop_size, num_downsample_channels]. This is the + instance feature crop. + """ + (batch_size, num_levels, max_feature_size, + _, num_downsample_channels) = features.get_shape().as_list() + _, num_of_instances, _ = level_boxes.get_shape().as_list() + level_boxes = tf.cast(level_boxes, tf.int32) + assert num_of_instances == detection_prior_levels.get_shape().as_list()[1] + + x_start_indices = level_boxes[:, :, 1] + y_start_indices = level_boxes[:, :, 0] + # generate the full indices (not just the starting index) + x_idx_list = [] + y_idx_list = [] + for i in range(mask_crop_size): + x_idx_list.append(x_start_indices + i) + y_idx_list.append(y_start_indices + i) + + x_indices = tf.stack(x_idx_list, axis=2) + y_indices = tf.stack(y_idx_list, axis=2) + levels = detection_prior_levels - min_mask_level + height_dim_size = max_feature_size + level_dim_size = max_feature_size * height_dim_size + batch_dim_size = num_levels * level_dim_size + # TODO(weicheng) change this to gather_nd for better readability. + indices = tf.reshape( + tf.tile( + tf.reshape( + tf.range(batch_size) * batch_dim_size, + [batch_size, 1, 1, 1]), + [1, num_of_instances, + mask_crop_size, mask_crop_size]) + + tf.tile( + tf.reshape(levels * level_dim_size, + [batch_size, num_of_instances, 1, 1]), + [1, 1, mask_crop_size, mask_crop_size]) + + tf.tile( + tf.reshape(y_indices * height_dim_size, + [batch_size, num_of_instances, + mask_crop_size, 1]), + [1, 1, 1, mask_crop_size]) + + tf.tile( + tf.reshape(x_indices, + [batch_size, num_of_instances, + 1, mask_crop_size]), + [1, 1, mask_crop_size, 1]), [-1]) + + features_r2 = tf.reshape(features, + [-1, num_downsample_channels]) + crop_features = tf.reshape( + tf.gather(features_r2, indices), + [batch_size * num_of_instances, + mask_crop_size, mask_crop_size, + num_downsample_channels]) + + return crop_features + + +def crop_mask_in_target_box(masks, + boxes, + target_boxes, + output_size, + sample_offset=0, + use_einsum=True): + """Crop masks in target boxes. + + Args: + masks: A tensor with a shape of [batch_size, num_masks, height, width]. + boxes: a float tensor representing box cooridnates that tightly enclose + masks with a shape of [batch_size, num_masks, 4] in un-normalized + coordinates. A box is represented by [ymin, xmin, ymax, xmax]. + target_boxes: a float tensor representing target box cooridnates for + masks with a shape of [batch_size, num_masks, 4] in un-normalized + coordinates. A box is represented by [ymin, xmin, ymax, xmax]. + output_size: A scalar to indicate the output crop size. It currently only + supports to output a square shape outputs. + sample_offset: a float number in [0, 1] indicates the subpixel sample offset + from grid point. + use_einsum: Use einsum to replace gather in selective_crop_and_resize. + + Returns: + A 4-D tensor representing feature crop of shape + [batch_size, num_boxes, output_size, output_size]. + """ + with tf.name_scope('crop_mask_in_target_box'): + batch_size, num_masks, height, width = masks.get_shape().as_list() + masks = tf.reshape(masks, [batch_size*num_masks, height, width, 1]) + # Pad zeros on the boundary of masks. + masks = tf.image.pad_to_bounding_box(masks, 2, 2, height + 4, width + 4) + masks = tf.reshape(masks, [batch_size, num_masks, height+4, width+4, 1]) + + # Projects target box locations and sizes to corresponding cropped + # mask coordinates. + gt_y_min, gt_x_min, gt_y_max, gt_x_max = tf.split( + value=boxes, num_or_size_splits=4, axis=2) + bb_y_min, bb_x_min, bb_y_max, bb_x_max = tf.split( + value=target_boxes, num_or_size_splits=4, axis=2) + y_transform = (bb_y_min - gt_y_min) * height / ( + gt_y_max - gt_y_min + _EPSILON) + 2 + x_transform = (bb_x_min - gt_x_min) * height / ( + gt_x_max - gt_x_min + _EPSILON) + 2 + h_transform = (bb_y_max - bb_y_min) * width / ( + gt_y_max - gt_y_min + _EPSILON) + w_transform = (bb_x_max - bb_x_min) * width / ( + gt_x_max - gt_x_min + _EPSILON) + + boundaries = tf.concat([ + tf.cast( + tf.ones_like(y_transform) * ((height + 4) - 1), dtype=tf.float32), + tf.cast( + tf.ones_like(x_transform) * ((width + 4) - 1), dtype=tf.float32) + ], + axis=-1) + + # Reshape tensors to have the right shape for selective_crop_and_resize. + trasnformed_boxes = tf.concat( + [y_transform, x_transform, h_transform, w_transform], -1) + levels = tf.tile(tf.reshape(tf.range(num_masks), [1, num_masks]), + [batch_size, 1]) + + cropped_masks = selective_crop_and_resize( + masks, + trasnformed_boxes, + levels, + boundaries, + output_size, + sample_offset=sample_offset, + use_einsum_gather=use_einsum) + cropped_masks = tf.squeeze(cropped_masks, axis=-1) + + return cropped_masks diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/__init__.py new file mode 100644 index 0000000..931c2ef --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/autoaugment_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/autoaugment_utils.py new file mode 100644 index 0000000..d16c9fa --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/autoaugment_utils.py @@ -0,0 +1,25 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""AutoAugment util file.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow.compat.v2 as tf + + +def distort_image_with_autoaugment(image, bboxes, augmentation_name): + raise NotImplementedError("Not TF 2.0 ready.") diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/box_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/box_utils.py new file mode 100644 index 0000000..cd31b39 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/box_utils.py @@ -0,0 +1,551 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utility functions for bounding box processing.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import numpy as np +import tensorflow.compat.v2 as tf + +EPSILON = 1e-8 +BBOX_XFORM_CLIP = np.log(1000. / 16.) + + +def visualize_images_with_bounding_boxes(images, box_outputs, step, + summary_writer): + """Records subset of evaluation images with bounding boxes.""" + image_shape = tf.shape(images[0]) + image_height = tf.cast(image_shape[0], tf.float32) + image_width = tf.cast(image_shape[1], tf.float32) + normalized_boxes = normalize_boxes(box_outputs, [image_height, image_width]) + + bounding_box_color = tf.constant([[1.0, 1.0, 0.0, 1.0]]) + image_summary = tf.image.draw_bounding_boxes(images, normalized_boxes, + bounding_box_color) + with summary_writer.as_default(): + tf.summary.image('bounding_box_summary', image_summary, step=step) + summary_writer.flush() + + +def yxyx_to_xywh(boxes): + """Converts boxes from ymin, xmin, ymax, xmax to xmin, ymin, width, height. + + Args: + boxes: a numpy array whose last dimension is 4 representing the coordinates + of boxes in ymin, xmin, ymax, xmax order. + + Returns: + boxes: a numpy array whose shape is the same as `boxes` in new format. + + Raises: + ValueError: If the last dimension of boxes is not 4. + """ + if boxes.shape[-1] != 4: + raise ValueError('boxes.shape[-1] is {:d}, but must be 4.'.format( + boxes.shape[-1])) + + boxes_ymin = boxes[..., 0] + boxes_xmin = boxes[..., 1] + boxes_width = boxes[..., 3] - boxes[..., 1] + boxes_height = boxes[..., 2] - boxes[..., 0] + new_boxes = np.stack([boxes_xmin, boxes_ymin, boxes_width, boxes_height], + axis=-1) + + return new_boxes + + +def jitter_boxes(boxes, noise_scale=0.025): + """Jitter the box coordinates by some noise distribution. + + Args: + boxes: a tensor whose last dimension is 4 representing the coordinates of + boxes in ymin, xmin, ymax, xmax order. + noise_scale: a python float which specifies the magnitude of noise. The rule + of thumb is to set this between (0, 0.1]. The default value is found to + mimic the noisy detections best empirically. + + Returns: + jittered_boxes: a tensor whose shape is the same as `boxes` representing + the jittered boxes. + + Raises: + ValueError: If the last dimension of boxes is not 4. + """ + if boxes.shape[-1] != 4: + raise ValueError('boxes.shape[-1] is {:d}, but must be 4.'.format( + boxes.shape[-1])) + + with tf.name_scope('jitter_boxes'): + bbox_jitters = tf.random.normal(boxes.get_shape(), stddev=noise_scale) + ymin = boxes[..., 0:1] + xmin = boxes[..., 1:2] + ymax = boxes[..., 2:3] + xmax = boxes[..., 3:4] + width = xmax - xmin + height = ymax - ymin + new_center_x = (xmin + xmax) / 2.0 + bbox_jitters[..., 0:1] * width + new_center_y = (ymin + ymax) / 2.0 + bbox_jitters[..., 1:2] * height + new_width = width * tf.math.exp(bbox_jitters[..., 2:3]) + new_height = height * tf.math.exp(bbox_jitters[..., 3:4]) + jittered_boxes = tf.concat([ + new_center_y - new_height * 0.5, new_center_x - new_width * 0.5, + new_center_y + new_height * 0.5, new_center_x + new_width * 0.5 + ], + axis=-1) + + return jittered_boxes + + +def normalize_boxes(boxes, image_shape): + """Converts boxes to the normalized coordinates. + + Args: + boxes: a tensor whose last dimension is 4 representing the coordinates + of boxes in ymin, xmin, ymax, xmax order. + image_shape: a list of two integers, a two-element vector or a tensor such + that all but the last dimensions are `broadcastable` to `boxes`. The last + dimension is 2, which represents [height, width]. + + Returns: + normalized_boxes: a tensor whose shape is the same as `boxes` representing + the normalized boxes. + + Raises: + ValueError: If the last dimension of boxes is not 4. + """ + if boxes.shape[-1] != 4: + raise ValueError('boxes.shape[-1] is {:d}, but must be 4.'.format( + boxes.shape[-1])) + + with tf.name_scope('normalize_boxes'): + if isinstance(image_shape, list) or isinstance(image_shape, tuple): + height, width = image_shape + else: + image_shape = tf.cast(image_shape, dtype=boxes.dtype) + height = image_shape[..., 0:1] + width = image_shape[..., 1:2] + + ymin = boxes[..., 0:1] / height + xmin = boxes[..., 1:2] / width + ymax = boxes[..., 2:3] / height + xmax = boxes[..., 3:4] / width + + normalized_boxes = tf.concat([ymin, xmin, ymax, xmax], axis=-1) + return normalized_boxes + + +def denormalize_boxes(boxes, image_shape): + """Converts boxes normalized by [height, width] to pixel coordinates. + + Args: + boxes: a tensor whose last dimension is 4 representing the coordinates + of boxes in ymin, xmin, ymax, xmax order. + image_shape: a list of two integers, a two-element vector or a tensor such + that all but the last dimensions are `broadcastable` to `boxes`. The last + dimension is 2, which represents [height, width]. + + Returns: + denormalized_boxes: a tensor whose shape is the same as `boxes` representing + the denormalized boxes. + + Raises: + ValueError: If the last dimension of boxes is not 4. + """ + with tf.name_scope('denormalize_boxes'): + if isinstance(image_shape, list) or isinstance(image_shape, tuple): + height, width = image_shape + else: + image_shape = tf.cast(image_shape, dtype=boxes.dtype) + height, width = tf.split(image_shape, 2, axis=-1) + + ymin, xmin, ymax, xmax = tf.split(boxes, 4, axis=-1) + ymin = ymin * height + xmin = xmin * width + ymax = ymax * height + xmax = xmax * width + + denormalized_boxes = tf.concat([ymin, xmin, ymax, xmax], axis=-1) + return denormalized_boxes + + +def clip_boxes(boxes, image_shape): + """Clips boxes to image boundaries. + + Args: + boxes: a tensor whose last dimension is 4 representing the coordinates + of boxes in ymin, xmin, ymax, xmax order. + image_shape: a list of two integers, a two-element vector or a tensor such + that all but the last dimensions are `broadcastable` to `boxes`. The last + dimension is 2, which represents [height, width]. + + Returns: + clipped_boxes: a tensor whose shape is the same as `boxes` representing the + clipped boxes. + + Raises: + ValueError: If the last dimension of boxes is not 4. + """ + if boxes.shape[-1] != 4: + raise ValueError('boxes.shape[-1] is {:d}, but must be 4.'.format( + boxes.shape[-1])) + + with tf.name_scope('clip_boxes'): + if isinstance(image_shape, list) or isinstance(image_shape, tuple): + height, width = image_shape + max_length = [height - 1.0, width - 1.0, height - 1.0, width - 1.0] + else: + image_shape = tf.cast(image_shape, dtype=boxes.dtype) + height, width = tf.unstack(image_shape, axis=-1) + max_length = tf.stack( + [height - 1.0, width - 1.0, height - 1.0, width - 1.0], axis=-1) + + clipped_boxes = tf.math.maximum(tf.math.minimum(boxes, max_length), 0.0) + return clipped_boxes + + +def compute_outer_boxes(boxes, image_shape, scale=1.0): + """Compute outer box encloses an object with a margin. + + Args: + boxes: a tensor whose last dimension is 4 representing the coordinates of + boxes in ymin, xmin, ymax, xmax order. + image_shape: a list of two integers, a two-element vector or a tensor such + that all but the last dimensions are `broadcastable` to `boxes`. The last + dimension is 2, which represents [height, width]. + scale: a float number specifying the scale of output outer boxes to input + `boxes`. + + Returns: + outer_boxes: a tensor whose shape is the same as `boxes` representing the + outer boxes. + """ + if scale < 1.0: + raise ValueError( + 'scale is {}, but outer box scale must be greater than 1.0.'.format( + scale)) + centers_y = (boxes[..., 0] + boxes[..., 2]) / 2.0 + centers_x = (boxes[..., 1] + boxes[..., 3]) / 2.0 + box_height = (boxes[..., 2] - boxes[..., 0]) * scale + box_width = (boxes[..., 3] - boxes[..., 1]) * scale + outer_boxes = tf.stack([ + centers_y - box_height / 2.0, centers_x - box_width / 2.0, + centers_y + box_height / 2.0, centers_x + box_width / 2.0 + ], + axis=1) + outer_boxes = clip_boxes(outer_boxes, image_shape) + return outer_boxes + + +def encode_boxes(boxes, anchors, weights=None): + """Encode boxes to targets. + + Args: + boxes: a tensor whose last dimension is 4 representing the coordinates + of boxes in ymin, xmin, ymax, xmax order. + anchors: a tensor whose shape is the same as, or `broadcastable` to `boxes`, + representing the coordinates of anchors in ymin, xmin, ymax, xmax order. + weights: None or a list of four float numbers used to scale coordinates. + + Returns: + encoded_boxes: a tensor whose shape is the same as `boxes` representing the + encoded box targets. + + Raises: + ValueError: If the last dimension of boxes is not 4. + """ + if boxes.shape[-1] != 4: + raise ValueError('boxes.shape[-1] is {:d}, but must be 4.'.format( + boxes.shape[-1])) + + with tf.name_scope('encode_boxes'): + boxes = tf.cast(boxes, dtype=anchors.dtype) + ymin = boxes[..., 0:1] + xmin = boxes[..., 1:2] + ymax = boxes[..., 2:3] + xmax = boxes[..., 3:4] + box_h = ymax - ymin + 1.0 + box_w = xmax - xmin + 1.0 + box_yc = ymin + 0.5 * box_h + box_xc = xmin + 0.5 * box_w + + anchor_ymin = anchors[..., 0:1] + anchor_xmin = anchors[..., 1:2] + anchor_ymax = anchors[..., 2:3] + anchor_xmax = anchors[..., 3:4] + anchor_h = anchor_ymax - anchor_ymin + 1.0 + anchor_w = anchor_xmax - anchor_xmin + 1.0 + anchor_yc = anchor_ymin + 0.5 * anchor_h + anchor_xc = anchor_xmin + 0.5 * anchor_w + + encoded_dy = (box_yc - anchor_yc) / anchor_h + encoded_dx = (box_xc - anchor_xc) / anchor_w + encoded_dh = tf.math.log(box_h / anchor_h) + encoded_dw = tf.math.log(box_w / anchor_w) + if weights: + encoded_dy *= weights[0] + encoded_dx *= weights[1] + encoded_dh *= weights[2] + encoded_dw *= weights[3] + + encoded_boxes = tf.concat( + [encoded_dy, encoded_dx, encoded_dh, encoded_dw], + axis=-1) + return encoded_boxes + + +def decode_boxes(encoded_boxes, anchors, weights=None): + """Decode boxes. + + Args: + encoded_boxes: a tensor whose last dimension is 4 representing the + coordinates of encoded boxes in ymin, xmin, ymax, xmax order. + anchors: a tensor whose shape is the same as, or `broadcastable` to `boxes`, + representing the coordinates of anchors in ymin, xmin, ymax, xmax order. + weights: None or a list of four float numbers used to scale coordinates. + + Returns: + encoded_boxes: a tensor whose shape is the same as `boxes` representing the + decoded box targets. + """ + if encoded_boxes.shape[-1] != 4: + raise ValueError('encoded_boxes.shape[-1] is {:d}, but must be 4.'.format( + encoded_boxes.shape[-1])) + + with tf.name_scope('decode_boxes'): + encoded_boxes = tf.cast(encoded_boxes, dtype=anchors.dtype) + dy = encoded_boxes[..., 0:1] + dx = encoded_boxes[..., 1:2] + dh = encoded_boxes[..., 2:3] + dw = encoded_boxes[..., 3:4] + if weights: + dy /= weights[0] + dx /= weights[1] + dh /= weights[2] + dw /= weights[3] + dh = tf.math.minimum(dh, BBOX_XFORM_CLIP) + dw = tf.math.minimum(dw, BBOX_XFORM_CLIP) + + anchor_ymin = anchors[..., 0:1] + anchor_xmin = anchors[..., 1:2] + anchor_ymax = anchors[..., 2:3] + anchor_xmax = anchors[..., 3:4] + anchor_h = anchor_ymax - anchor_ymin + 1.0 + anchor_w = anchor_xmax - anchor_xmin + 1.0 + anchor_yc = anchor_ymin + 0.5 * anchor_h + anchor_xc = anchor_xmin + 0.5 * anchor_w + + decoded_boxes_yc = dy * anchor_h + anchor_yc + decoded_boxes_xc = dx * anchor_w + anchor_xc + decoded_boxes_h = tf.math.exp(dh) * anchor_h + decoded_boxes_w = tf.math.exp(dw) * anchor_w + + decoded_boxes_ymin = decoded_boxes_yc - 0.5 * decoded_boxes_h + decoded_boxes_xmin = decoded_boxes_xc - 0.5 * decoded_boxes_w + decoded_boxes_ymax = decoded_boxes_ymin + decoded_boxes_h - 1.0 + decoded_boxes_xmax = decoded_boxes_xmin + decoded_boxes_w - 1.0 + + decoded_boxes = tf.concat( + [decoded_boxes_ymin, decoded_boxes_xmin, + decoded_boxes_ymax, decoded_boxes_xmax], + axis=-1) + return decoded_boxes + + +def filter_boxes(boxes, scores, image_shape, min_size_threshold): + """Filter and remove boxes that are too small or fall outside the image. + + Args: + boxes: a tensor whose last dimension is 4 representing the coordinates of + boxes in ymin, xmin, ymax, xmax order. + scores: a tensor whose shape is the same as tf.shape(boxes)[:-1] + representing the original scores of the boxes. + image_shape: a tensor whose shape is the same as, or `broadcastable` to + `boxes` except the last dimension, which is 2, representing [height, + width] of the scaled image. + min_size_threshold: a float representing the minimal box size in each side + (w.r.t. the scaled image). Boxes whose sides are smaller than it will be + filtered out. + + Returns: + filtered_boxes: a tensor whose shape is the same as `boxes` but with + the position of the filtered boxes are filled with 0. + filtered_scores: a tensor whose shape is the same as 'scores' but with + the positinon of the filtered boxes filled with 0. + """ + if boxes.shape[-1] != 4: + raise ValueError('boxes.shape[1] is {:d}, but must be 4.'.format( + boxes.shape[-1])) + + with tf.name_scope('filter_boxes'): + if isinstance(image_shape, list) or isinstance(image_shape, tuple): + height, width = image_shape + else: + image_shape = tf.cast(image_shape, dtype=boxes.dtype) + height = image_shape[..., 0] + width = image_shape[..., 1] + + ymin = boxes[..., 0] + xmin = boxes[..., 1] + ymax = boxes[..., 2] + xmax = boxes[..., 3] + + h = ymax - ymin + 1.0 + w = xmax - xmin + 1.0 + yc = ymin + 0.5 * h + xc = xmin + 0.5 * w + + min_size = tf.cast( + tf.math.maximum(min_size_threshold, 1.0), dtype=boxes.dtype) + + filtered_size_mask = tf.math.logical_and( + tf.math.greater(h, min_size), tf.math.greater(w, min_size)) + filtered_center_mask = tf.logical_and( + tf.math.logical_and(tf.math.greater(yc, 0.0), tf.math.less(yc, height)), + tf.math.logical_and(tf.math.greater(xc, 0.0), tf.math.less(xc, width))) + filtered_mask = tf.math.logical_and(filtered_size_mask, + filtered_center_mask) + + filtered_scores = tf.where(filtered_mask, scores, tf.zeros_like(scores)) + filtered_boxes = tf.cast( + tf.expand_dims(filtered_mask, axis=-1), dtype=boxes.dtype) * boxes + + return filtered_boxes, filtered_scores + + +def filter_boxes_by_scores(boxes, scores, min_score_threshold): + """Filter and remove boxes whose scores are smaller than the threshold. + + Args: + boxes: a tensor whose last dimension is 4 representing the coordinates of + boxes in ymin, xmin, ymax, xmax order. + scores: a tensor whose shape is the same as tf.shape(boxes)[:-1] + representing the original scores of the boxes. + min_score_threshold: a float representing the minimal box score threshold. + Boxes whose score are smaller than it will be filtered out. + + Returns: + filtered_boxes: a tensor whose shape is the same as `boxes` but with + the position of the filtered boxes are filled with -1. + filtered_scores: a tensor whose shape is the same as 'scores' but with + the + """ + if boxes.shape[-1] != 4: + raise ValueError('boxes.shape[1] is {:d}, but must be 4.'.format( + boxes.shape[-1])) + + with tf.name_scope('filter_boxes_by_scores'): + filtered_mask = tf.math.greater(scores, min_score_threshold) + filtered_scores = tf.where(filtered_mask, scores, -tf.ones_like(scores)) + filtered_boxes = tf.cast( + tf.expand_dims(filtered_mask, axis=-1), dtype=boxes.dtype) * boxes + + return filtered_boxes, filtered_scores + + +def top_k_boxes(boxes, scores, k): + """Sort and select top k boxes according to the scores. + + Args: + boxes: a tensor of shape [batch_size, N, 4] representing the coordiante of + the boxes. N is the number of boxes per image. + scores: a tensor of shsape [batch_size, N] representing the socre of the + boxes. + k: an integer or a tensor indicating the top k number. + + Returns: + selected_boxes: a tensor of shape [batch_size, k, 4] representing the + selected top k box coordinates. + selected_scores: a tensor of shape [batch_size, k] representing the selected + top k box scores. + """ + with tf.name_scope('top_k_boxes'): + selected_scores, top_k_indices = tf.nn.top_k(scores, k=k, sorted=True) + + batch_size, _ = scores.get_shape().as_list() + if batch_size == 1: + selected_boxes = tf.squeeze( + tf.gather(boxes, top_k_indices, axis=1), axis=1) + else: + top_k_indices_shape = tf.shape(top_k_indices) + batch_indices = ( + tf.expand_dims(tf.range(top_k_indices_shape[0]), axis=-1) * + tf.ones([1, top_k_indices_shape[-1]], dtype=tf.int32)) + gather_nd_indices = tf.stack([batch_indices, top_k_indices], axis=-1) + selected_boxes = tf.gather_nd(boxes, gather_nd_indices) + + return selected_boxes, selected_scores + + +def bbox_overlap(boxes, gt_boxes): + """Calculates the overlap between proposal and ground truth boxes. + + Some `gt_boxes` may have been padded. The returned `iou` tensor for these + boxes will be -1. + + Args: + boxes: a tensor with a shape of [batch_size, N, 4]. N is the number of + proposals before groundtruth assignment (e.g., rpn_post_nms_topn). The + last dimension is the pixel coordinates in [ymin, xmin, ymax, xmax] form. + gt_boxes: a tensor with a shape of [batch_size, MAX_NUM_INSTANCES, 4]. This + tensor might have paddings with a negative value. + + Returns: + iou: a tensor with as a shape of [batch_size, N, MAX_NUM_INSTANCES]. + """ + with tf.name_scope('bbox_overlap'): + bb_y_min, bb_x_min, bb_y_max, bb_x_max = tf.split( + value=boxes, num_or_size_splits=4, axis=2) + gt_y_min, gt_x_min, gt_y_max, gt_x_max = tf.split( + value=gt_boxes, num_or_size_splits=4, axis=2) + + # Calculates the intersection area. + i_xmin = tf.math.maximum(bb_x_min, tf.transpose(gt_x_min, [0, 2, 1])) + i_xmax = tf.math.minimum(bb_x_max, tf.transpose(gt_x_max, [0, 2, 1])) + i_ymin = tf.math.maximum(bb_y_min, tf.transpose(gt_y_min, [0, 2, 1])) + i_ymax = tf.math.minimum(bb_y_max, tf.transpose(gt_y_max, [0, 2, 1])) + i_area = tf.math.maximum((i_xmax - i_xmin), 0) * tf.math.maximum( + (i_ymax - i_ymin), 0) + + # Calculates the union area. + bb_area = (bb_y_max - bb_y_min) * (bb_x_max - bb_x_min) + gt_area = (gt_y_max - gt_y_min) * (gt_x_max - gt_x_min) + # Adds a small epsilon to avoid divide-by-zero. + u_area = bb_area + tf.transpose(gt_area, [0, 2, 1]) - i_area + 1e-8 + + # Calculates IoU. + iou = i_area / u_area + + # Fills -1 for IoU entries between the padded ground truth boxes. + gt_invalid_mask = tf.less( + tf.reduce_max(gt_boxes, axis=-1, keepdims=True), 0.0) + padding_mask = tf.logical_or( + tf.zeros_like(bb_x_min, dtype=tf.bool), + tf.transpose(gt_invalid_mask, [0, 2, 1])) + iou = tf.where(padding_mask, -tf.ones_like(iou), iou) + + return iou + + +def get_non_empty_box_indices(boxes): + """Get indices for non-empty boxes.""" + # Selects indices if box height or width is 0. + height = boxes[:, 2] - boxes[:, 0] + width = boxes[:, 3] - boxes[:, 1] + indices = tf.where(tf.logical_and(tf.greater(height, 0), + tf.greater(width, 0))) + return indices[:, 0] diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/class_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/class_utils.py new file mode 100644 index 0000000..cce9cf9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/class_utils.py @@ -0,0 +1,44 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utility functions for handling dataset object categories.""" + + +def coco_split_class_ids(split_name): + """Return the COCO class split ids based on split name and training mode. + + Args: + split_name: The name of dataset split. + + Returns: + class_ids: a python list of integer. + """ + if split_name == 'all': + return [] + + elif split_name == 'voc': + return [ + 1, 2, 3, 4, 5, 6, 7, 9, 16, 17, 18, 19, 20, 21, 44, 62, 63, 64, 67, 72 + ] + + elif split_name == 'nonvoc': + return [ + 8, 10, 11, 13, 14, 15, 22, 23, 24, 25, 27, 28, 31, 32, 33, 34, 35, 36, + 37, 38, 39, 40, 41, 42, 43, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + 57, 58, 59, 60, 61, 65, 70, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 84, + 85, 86, 87, 88, 89, 90 + ] + + else: + raise ValueError('Invalid split name {}!!!'.format(split_name)) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/dataloader_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/dataloader_utils.py new file mode 100644 index 0000000..59b45d6 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/dataloader_utils.py @@ -0,0 +1,40 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utility functions for dataloader.""" + +import tensorflow.compat.v2 as tf + +from official.vision.detection.utils import input_utils + + +def process_source_id(source_id): + """Processes source_id to the right format.""" + if source_id.dtype == tf.string: + source_id = tf.cast(tf.strings.to_number(source_id), tf.int64) + with tf.control_dependencies([source_id]): + source_id = tf.cond( + pred=tf.equal(tf.size(input=source_id), 0), + true_fn=lambda: tf.cast(tf.constant(-1), tf.int64), + false_fn=lambda: tf.identity(source_id)) + return source_id + + +def pad_groundtruths_to_fixed_size(gt, n): + """Pads the first dimension of groundtruths labels to the fixed size.""" + gt['boxes'] = input_utils.pad_to_fixed_size(gt['boxes'], n, -1) + gt['is_crowds'] = input_utils.pad_to_fixed_size(gt['is_crowds'], n, 0) + gt['areas'] = input_utils.pad_to_fixed_size(gt['areas'], n, -1) + gt['classes'] = input_utils.pad_to_fixed_size(gt['classes'], n, -1) + return gt diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/input_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/input_utils.py new file mode 100644 index 0000000..3233f62 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/input_utils.py @@ -0,0 +1,366 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utility functions for input processing.""" + +import math +import tensorflow.compat.v2 as tf + +from official.vision.detection.utils import box_utils +from official.vision.detection.utils.object_detection import preprocessor + + +def pad_to_fixed_size(input_tensor, size, constant_values=0): + """Pads data to a fixed length at the first dimension. + + Args: + input_tensor: `Tensor` with any dimension. + size: `int` number for the first dimension of output Tensor. + constant_values: `int` value assigned to the paddings. + + Returns: + `Tensor` with the first dimension padded to `size`. + """ + input_shape = input_tensor.get_shape().as_list() + padding_shape = [] + + # Computes the padding length on the first dimension. + padding_length = size - tf.shape(input=input_tensor)[0] + assert_length = tf.Assert( + tf.greater_equal(padding_length, 0), [padding_length]) + with tf.control_dependencies([assert_length]): + padding_shape.append(padding_length) + + # Copies shapes of the rest of input shape dimensions. + for i in range(1, len(input_shape)): + padding_shape.append(tf.shape(input=input_tensor)[i]) + + # Pads input tensor to the fixed first dimension. + paddings = tf.cast(constant_values * tf.ones(padding_shape), + input_tensor.dtype) + padded_tensor = tf.concat([input_tensor, paddings], axis=0) + output_shape = input_shape + output_shape[0] = size + padded_tensor.set_shape(output_shape) + return padded_tensor + + +def normalize_image(image, + offset=(0.485, 0.456, 0.406), + scale=(0.229, 0.224, 0.225)): + """Normalizes the image to zero mean and unit variance.""" + image = tf.image.convert_image_dtype(image, dtype=tf.float32) + offset = tf.constant(offset) + offset = tf.expand_dims(offset, axis=0) + offset = tf.expand_dims(offset, axis=0) + image -= offset + + scale = tf.constant(scale) + scale = tf.expand_dims(scale, axis=0) + scale = tf.expand_dims(scale, axis=0) + image /= scale + return image + + +def compute_padded_size(desired_size, stride): + """Compute the padded size given the desired size and the stride. + + The padded size will be the smallest rectangle, such that each dimension is + the smallest multiple of the stride which is larger than the desired + dimension. For example, if desired_size = (100, 200) and stride = 32, + the output padded_size = (128, 224). + + Args: + desired_size: a `Tensor` or `int` list/tuple of two elements representing + [height, width] of the target output image size. + stride: an integer, the stride of the backbone network. + + Returns: + padded_size: a `Tensor` or `int` list/tuple of two elements representing + [height, width] of the padded output image size. + """ + if isinstance(desired_size, list) or isinstance(desired_size, tuple): + padded_size = [int(math.ceil(d * 1.0 / stride) * stride) + for d in desired_size] + else: + padded_size = tf.cast( + tf.math.ceil( + tf.cast(desired_size, dtype=tf.float32) / stride) * stride, + tf.int32) + return padded_size + + +def resize_and_crop_image(image, + desired_size, + padded_size, + aug_scale_min=1.0, + aug_scale_max=1.0, + seed=1, + method=tf.image.ResizeMethod.BILINEAR): + """Resizes the input image to output size. + + Resize and pad images given the desired output size of the image and + stride size. + + Here are the preprocessing steps. + 1. For a given image, keep its aspect ratio and rescale the image to make it + the largest rectangle to be bounded by the rectangle specified by the + `desired_size`. + 2. Pad the rescaled image to the padded_size. + + Args: + image: a `Tensor` of shape [height, width, 3] representing an image. + desired_size: a `Tensor` or `int` list/tuple of two elements representing + [height, width] of the desired actual output image size. + padded_size: a `Tensor` or `int` list/tuple of two elements representing + [height, width] of the padded output image size. Padding will be applied + after scaling the image to the desired_size. + aug_scale_min: a `float` with range between [0, 1.0] representing minimum + random scale applied to desired_size for training scale jittering. + aug_scale_max: a `float` with range between [1.0, inf] representing maximum + random scale applied to desired_size for training scale jittering. + seed: seed for random scale jittering. + method: function to resize input image to scaled image. + + Returns: + output_image: `Tensor` of shape [height, width, 3] where [height, width] + equals to `output_size`. + image_info: a 2D `Tensor` that encodes the information of the image and the + applied preprocessing. It is in the format of + [[original_height, original_width], [desired_height, desired_width], + [y_scale, x_scale], [y_offset, x_offset]], where [desired_height, + desireed_width] is the actual scaled image size, and [y_scale, x_scale] is + the scaling factory, which is the ratio of + scaled dimension / original dimension. + """ + with tf.name_scope('resize_and_crop_image'): + image_size = tf.cast(tf.shape(input=image)[0:2], tf.float32) + + random_jittering = (aug_scale_min != 1.0 or aug_scale_max != 1.0) + + if random_jittering: + random_scale = tf.random.uniform([], + aug_scale_min, + aug_scale_max, + seed=seed) + scaled_size = tf.round(random_scale * desired_size) + else: + scaled_size = desired_size + + scale = tf.minimum( + scaled_size[0] / image_size[0], scaled_size[1] / image_size[1]) + scaled_size = tf.round(image_size * scale) + + # Computes 2D image_scale. + image_scale = scaled_size / image_size + + # Selects non-zero random offset (x, y) if scaled image is larger than + # desired_size. + if random_jittering: + max_offset = scaled_size - desired_size + max_offset = tf.where(tf.less(max_offset, 0), + tf.zeros_like(max_offset), + max_offset) + offset = max_offset * tf.random.uniform([ + 2, + ], 0, 1, seed=seed) + offset = tf.cast(offset, tf.int32) + else: + offset = tf.zeros((2,), tf.int32) + + scaled_image = tf.image.resize( + image, tf.cast(scaled_size, tf.int32), method=method) + + if random_jittering: + scaled_image = scaled_image[offset[0]:offset[0] + desired_size[0], + offset[1]:offset[1] + desired_size[1], :] + + output_image = tf.image.pad_to_bounding_box(scaled_image, 0, 0, + padded_size[0], padded_size[1]) + + image_info = tf.stack([ + image_size, + tf.cast(desired_size, dtype=tf.float32), + image_scale, + tf.cast(offset, tf.float32)]) + return output_image, image_info + + +def resize_and_crop_image_v2(image, + short_side, + long_side, + padded_size, + aug_scale_min=1.0, + aug_scale_max=1.0, + seed=1, + method=tf.image.ResizeMethod.BILINEAR): + """Resizes the input image to output size (Faster R-CNN style). + + Resize and pad images given the specified short / long side length and the + stride size. + + Here are the preprocessing steps. + 1. For a given image, keep its aspect ratio and first try to rescale the short + side of the original image to `short_side`. + 2. If the scaled image after 1 has a long side that exceeds `long_side`, keep + the aspect ratio and rescal the long side of the image to `long_side`. + 2. Pad the rescaled image to the padded_size. + + Args: + image: a `Tensor` of shape [height, width, 3] representing an image. + short_side: a scalar `Tensor` or `int` representing the desired short side + to be rescaled to. + long_side: a scalar `Tensor` or `int` representing the desired long side to + be rescaled to. + padded_size: a `Tensor` or `int` list/tuple of two elements representing + [height, width] of the padded output image size. Padding will be applied + after scaling the image to the desired_size. + aug_scale_min: a `float` with range between [0, 1.0] representing minimum + random scale applied to desired_size for training scale jittering. + aug_scale_max: a `float` with range between [1.0, inf] representing maximum + random scale applied to desired_size for training scale jittering. + seed: seed for random scale jittering. + method: function to resize input image to scaled image. + + Returns: + output_image: `Tensor` of shape [height, width, 3] where [height, width] + equals to `output_size`. + image_info: a 2D `Tensor` that encodes the information of the image and the + applied preprocessing. It is in the format of + [[original_height, original_width], [desired_height, desired_width], + [y_scale, x_scale], [y_offset, x_offset]], where [desired_height, + desired_width] is the actual scaled image size, and [y_scale, x_scale] is + the scaling factor, which is the ratio of + scaled dimension / original dimension. + """ + with tf.name_scope('resize_and_crop_image_v2'): + image_size = tf.cast(tf.shape(image)[0:2], tf.float32) + + scale_using_short_side = ( + short_side / tf.math.minimum(image_size[0], image_size[1])) + scale_using_long_side = ( + long_side / tf.math.maximum(image_size[0], image_size[1])) + + scaled_size = tf.math.round(image_size * scale_using_short_side) + scaled_size = tf.where( + tf.math.greater( + tf.math.maximum(scaled_size[0], scaled_size[1]), long_side), + tf.math.round(image_size * scale_using_long_side), scaled_size) + desired_size = scaled_size + + random_jittering = (aug_scale_min != 1.0 or aug_scale_max != 1.0) + + if random_jittering: + random_scale = tf.random.uniform([], + aug_scale_min, + aug_scale_max, + seed=seed) + scaled_size = tf.math.round(random_scale * scaled_size) + + # Computes 2D image_scale. + image_scale = scaled_size / image_size + + # Selects non-zero random offset (x, y) if scaled image is larger than + # desired_size. + if random_jittering: + max_offset = scaled_size - desired_size + max_offset = tf.where( + tf.math.less(max_offset, 0), tf.zeros_like(max_offset), max_offset) + offset = max_offset * tf.random.uniform([ + 2, + ], 0, 1, seed=seed) + offset = tf.cast(offset, tf.int32) + else: + offset = tf.zeros((2,), tf.int32) + + scaled_image = tf.image.resize( + image, tf.cast(scaled_size, tf.int32), method=method) + + if random_jittering: + scaled_image = scaled_image[ + offset[0]:offset[0] + desired_size[0], + offset[1]:offset[1] + desired_size[1], :] + + output_image = tf.image.pad_to_bounding_box( + scaled_image, 0, 0, padded_size[0], padded_size[1]) + + image_info = tf.stack([ + image_size, + tf.cast(desired_size, dtype=tf.float32), + image_scale, + tf.cast(offset, tf.float32)]) + return output_image, image_info + + +def resize_and_crop_boxes(boxes, + image_scale, + output_size, + offset): + """Resizes boxes to output size with scale and offset. + + Args: + boxes: `Tensor` of shape [N, 4] representing ground truth boxes. + image_scale: 2D float `Tensor` representing scale factors that apply to + [height, width] of input image. + output_size: 2D `Tensor` or `int` representing [height, width] of target + output image size. + offset: 2D `Tensor` representing top-left corner [y0, x0] to crop scaled + boxes. + + Returns: + boxes: `Tensor` of shape [N, 4] representing the scaled boxes. + """ + # Adjusts box coordinates based on image_scale and offset. + boxes *= tf.tile(tf.expand_dims(image_scale, axis=0), [1, 2]) + boxes -= tf.tile(tf.expand_dims(offset, axis=0), [1, 2]) + # Clips the boxes. + boxes = box_utils.clip_boxes(boxes, output_size) + return boxes + + +def resize_and_crop_masks(masks, + image_scale, + output_size, + offset): + """Resizes boxes to output size with scale and offset. + + Args: + masks: `Tensor` of shape [N, H, W, 1] representing ground truth masks. + image_scale: 2D float `Tensor` representing scale factors that apply to + [height, width] of input image. + output_size: 2D `Tensor` or `int` representing [height, width] of target + output image size. + offset: 2D `Tensor` representing top-left corner [y0, x0] to crop scaled + boxes. + + Returns: + masks: `Tensor` of shape [N, H, W, 1] representing the scaled masks. + """ + mask_size = tf.shape(input=masks)[1:3] + scaled_size = tf.cast(image_scale * tf.cast(mask_size, image_scale.dtype), + tf.int32) + scaled_masks = tf.image.resize( + masks, scaled_size, method=tf.image.ResizeMethod.NEAREST_NEIGHBOR) + offset = tf.cast(offset, tf.int32) + scaled_masks = scaled_masks[:, offset[0]:offset[0] + output_size[0], + offset[1]:offset[1] + output_size[1], :] + + output_masks = tf.image.pad_to_bounding_box(scaled_masks, 0, 0, + output_size[0], output_size[1]) + return output_masks + + +def random_horizontal_flip(image, boxes=None, masks=None): + """Randomly flips input image and bounding boxes.""" + return preprocessor.random_horizontal_flip(image, boxes, masks) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/mask_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/mask_utils.py new file mode 100644 index 0000000..637d048 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/mask_utils.py @@ -0,0 +1,192 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utility functions for segmentations.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math +import numpy as np +import cv2 + + +def paste_instance_masks(masks, + detected_boxes, + image_height, + image_width): + """Paste instance masks to generate the image segmentation results. + + Args: + masks: a numpy array of shape [N, mask_height, mask_width] representing the + instance masks w.r.t. the `detected_boxes`. + detected_boxes: a numpy array of shape [N, 4] representing the reference + bounding boxes. + image_height: an integer representing the height of the image. + image_width: an integer representing the width of the image. + + Returns: + segms: a numpy array of shape [N, image_height, image_width] representing + the instance masks *pasted* on the image canvas. + """ + + def expand_boxes(boxes, scale): + """Expands an array of boxes by a given scale.""" + # Reference: https://github.com/facebookresearch/Detectron/blob/master/detectron/utils/boxes.py#L227 # pylint: disable=line-too-long + # The `boxes` in the reference implementation is in [x1, y1, x2, y2] form, + # whereas `boxes` here is in [x1, y1, w, h] form + w_half = boxes[:, 2] * .5 + h_half = boxes[:, 3] * .5 + x_c = boxes[:, 0] + w_half + y_c = boxes[:, 1] + h_half + + w_half *= scale + h_half *= scale + + boxes_exp = np.zeros(boxes.shape) + boxes_exp[:, 0] = x_c - w_half + boxes_exp[:, 2] = x_c + w_half + boxes_exp[:, 1] = y_c - h_half + boxes_exp[:, 3] = y_c + h_half + + return boxes_exp + + # Reference: https://github.com/facebookresearch/Detectron/blob/master/detectron/core/test.py#L812 # pylint: disable=line-too-long + # To work around an issue with cv2.resize (it seems to automatically pad + # with repeated border values), we manually zero-pad the masks by 1 pixel + # prior to resizing back to the original image resolution. This prevents + # "top hat" artifacts. We therefore need to expand the reference boxes by an + # appropriate factor. + _, mask_height, mask_width = masks.shape + scale = max((mask_width + 2.0) / mask_width, + (mask_height + 2.0) / mask_height) + + ref_boxes = expand_boxes(detected_boxes, scale) + ref_boxes = ref_boxes.astype(np.int32) + padded_mask = np.zeros((mask_height + 2, mask_width + 2), dtype=np.float32) + segms = [] + for mask_ind, mask in enumerate(masks): + im_mask = np.zeros((image_height, image_width), dtype=np.uint8) + # Process mask inside bounding boxes. + padded_mask[1:-1, 1:-1] = mask[:, :] + + ref_box = ref_boxes[mask_ind, :] + w = ref_box[2] - ref_box[0] + 1 + h = ref_box[3] - ref_box[1] + 1 + w = np.maximum(w, 1) + h = np.maximum(h, 1) + + mask = cv2.resize(padded_mask, (w, h)) + mask = np.array(mask > 0.5, dtype=np.uint8) + + x_0 = min(max(ref_box[0], 0), image_width) + x_1 = min(max(ref_box[2] + 1, 0), image_width) + y_0 = min(max(ref_box[1], 0), image_height) + y_1 = min(max(ref_box[3] + 1, 0), image_height) + + im_mask[y_0:y_1, x_0:x_1] = mask[ + (y_0 - ref_box[1]):(y_1 - ref_box[1]), + (x_0 - ref_box[0]):(x_1 - ref_box[0]) + ] + segms.append(im_mask) + + segms = np.array(segms) + assert masks.shape[0] == segms.shape[0] + return segms + + +def paste_instance_masks_v2(masks, + detected_boxes, + image_height, + image_width): + """Paste instance masks to generate the image segmentation (v2). + + Args: + masks: a numpy array of shape [N, mask_height, mask_width] representing the + instance masks w.r.t. the `detected_boxes`. + detected_boxes: a numpy array of shape [N, 4] representing the reference + bounding boxes. + image_height: an integer representing the height of the image. + image_width: an integer representing the width of the image. + + Returns: + segms: a numpy array of shape [N, image_height, image_width] representing + the instance masks *pasted* on the image canvas. + """ + _, mask_height, mask_width = masks.shape + + segms = [] + for i, mask in enumerate(masks): + box = detected_boxes[i, :] + xmin = box[0] + ymin = box[1] + xmax = xmin + box[2] + ymax = ymin + box[3] + + # Sample points of the cropped mask w.r.t. the image grid. + # Note that these coordinates may fall beyond the image. + # Pixel clipping will happen after warping. + xmin_int = int(math.floor(xmin)) + xmax_int = int(math.ceil(xmax)) + ymin_int = int(math.floor(ymin)) + ymax_int = int(math.ceil(ymax)) + + alpha = box[2] / (1.0 * mask_width) + beta = box[3] / (1.0 * mask_height) + # pylint: disable=invalid-name + # Transformation from mask pixel indices to image coordinate. + M_mask_to_image = np.array( + [[alpha, 0, xmin], + [0, beta, ymin], + [0, 0, 1]], + dtype=np.float32) + # Transformation from image to cropped mask coordinate. + M_image_to_crop = np.array( + [[1, 0, -xmin_int], + [0, 1, -ymin_int], + [0, 0, 1]], + dtype=np.float32) + M = np.dot(M_image_to_crop, M_mask_to_image) + # Compensate the half pixel offset that OpenCV has in the + # warpPerspective implementation: the top-left pixel is sampled + # at (0,0), but we want it to be at (0.5, 0.5). + M = np.dot( + np.dot( + np.array([[1, 0, -0.5], + [0, 1, -0.5], + [0, 0, 1]], np.float32), + M), + np.array([[1, 0, 0.5], + [0, 1, 0.5], + [0, 0, 1]], np.float32)) + # pylint: enable=invalid-name + cropped_mask = cv2.warpPerspective( + mask.astype(np.float32), M, + (xmax_int - xmin_int, ymax_int - ymin_int)) + cropped_mask = np.array(cropped_mask > 0.5, dtype=np.uint8) + + img_mask = np.zeros((image_height, image_width)) + x0 = max(min(xmin_int, image_width), 0) + x1 = max(min(xmax_int, image_width), 0) + y0 = max(min(ymin_int, image_height), 0) + y1 = max(min(ymax_int, image_height), 0) + img_mask[y0:y1, x0:x1] = cropped_mask[ + (y0 - ymin_int):(y1 - ymin_int), + (x0 - xmin_int):(x1 - xmin_int)] + + segms.append(img_mask) + + segms = np.array(segms) + return segms + diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/__init__.py new file mode 100644 index 0000000..85c94f4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/argmax_matcher.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/argmax_matcher.py new file mode 100644 index 0000000..b433eb4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/argmax_matcher.py @@ -0,0 +1,201 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Argmax matcher implementation. + +This class takes a similarity matrix and matches columns to rows based on the +maximum value per column. One can specify matched_thresholds and +to prevent columns from matching to rows (generally resulting in a negative +training example) and unmatched_theshold to ignore the match (generally +resulting in neither a positive or negative training example). + +This matcher is used in Fast(er)-RCNN. + +Note: matchers are used in TargetAssigners. There is a create_target_assigner +factory function for popular implementations. +""" +import tensorflow.compat.v2 as tf + +from official.vision.detection.utils.object_detection import matcher +from official.vision.detection.utils.object_detection import shape_utils + + +class ArgMaxMatcher(matcher.Matcher): + """Matcher based on highest value. + + This class computes matches from a similarity matrix. Each column is matched + to a single row. + + To support object detection target assignment this class enables setting both + matched_threshold (upper threshold) and unmatched_threshold (lower thresholds) + defining three categories of similarity which define whether examples are + positive, negative, or ignored: + (1) similarity >= matched_threshold: Highest similarity. Matched/Positive! + (2) matched_threshold > similarity >= unmatched_threshold: Medium similarity. + Depending on negatives_lower_than_unmatched, this is either + Unmatched/Negative OR Ignore. + (3) unmatched_threshold > similarity: Lowest similarity. Depending on flag + negatives_lower_than_unmatched, either Unmatched/Negative OR Ignore. + For ignored matches this class sets the values in the Match object to -2. + """ + + def __init__(self, + matched_threshold, + unmatched_threshold=None, + negatives_lower_than_unmatched=True, + force_match_for_each_row=False): + """Construct ArgMaxMatcher. + + Args: + matched_threshold: Threshold for positive matches. Positive if + sim >= matched_threshold, where sim is the maximum value of the + similarity matrix for a given column. Set to None for no threshold. + unmatched_threshold: Threshold for negative matches. Negative if + sim < unmatched_threshold. Defaults to matched_threshold + when set to None. + negatives_lower_than_unmatched: Boolean which defaults to True. If True + then negative matches are the ones below the unmatched_threshold, + whereas ignored matches are in between the matched and umatched + threshold. If False, then negative matches are in between the matched + and unmatched threshold, and everything lower than unmatched is ignored. + force_match_for_each_row: If True, ensures that each row is matched to + at least one column (which is not guaranteed otherwise if the + matched_threshold is high). Defaults to False. See + argmax_matcher_test.testMatcherForceMatch() for an example. + + Raises: + ValueError: if unmatched_threshold is set but matched_threshold is not set + or if unmatched_threshold > matched_threshold. + """ + if (matched_threshold is None) and (unmatched_threshold is not None): + raise ValueError('Need to also define matched_threshold when' + 'unmatched_threshold is defined') + self._matched_threshold = matched_threshold + if unmatched_threshold is None: + self._unmatched_threshold = matched_threshold + else: + if unmatched_threshold > matched_threshold: + raise ValueError('unmatched_threshold needs to be smaller or equal' + 'to matched_threshold') + self._unmatched_threshold = unmatched_threshold + if not negatives_lower_than_unmatched: + if self._unmatched_threshold == self._matched_threshold: + raise ValueError('When negatives are in between matched and ' + 'unmatched thresholds, these cannot be of equal ' + 'value. matched: %s, unmatched: %s', + self._matched_threshold, self._unmatched_threshold) + self._force_match_for_each_row = force_match_for_each_row + self._negatives_lower_than_unmatched = negatives_lower_than_unmatched + + def _match(self, similarity_matrix): + """Tries to match each column of the similarity matrix to a row. + + Args: + similarity_matrix: tensor of shape [N, M] representing any similarity + metric. + + Returns: + Match object with corresponding matches for each of M columns. + """ + + def _match_when_rows_are_empty(): + """Performs matching when the rows of similarity matrix are empty. + + When the rows are empty, all detections are false positives. So we return + a tensor of -1's to indicate that the columns do not match to any rows. + + Returns: + matches: int32 tensor indicating the row each column matches to. + """ + similarity_matrix_shape = shape_utils.combined_static_and_dynamic_shape( + similarity_matrix) + return -1 * tf.ones([similarity_matrix_shape[1]], dtype=tf.int32) + + def _match_when_rows_are_non_empty(): + """Performs matching when the rows of similarity matrix are non empty. + + Returns: + matches: int32 tensor indicating the row each column matches to. + """ + # Matches for each column + matches = tf.argmax(input=similarity_matrix, axis=0, output_type=tf.int32) + + # Deal with matched and unmatched threshold + if self._matched_threshold is not None: + # Get logical indices of ignored and unmatched columns as tf.int64 + matched_vals = tf.reduce_max(input_tensor=similarity_matrix, axis=0) + below_unmatched_threshold = tf.greater(self._unmatched_threshold, + matched_vals) + between_thresholds = tf.logical_and( + tf.greater_equal(matched_vals, self._unmatched_threshold), + tf.greater(self._matched_threshold, matched_vals)) + + if self._negatives_lower_than_unmatched: + matches = self._set_values_using_indicator(matches, + below_unmatched_threshold, + -1) + matches = self._set_values_using_indicator(matches, + between_thresholds, + -2) + else: + matches = self._set_values_using_indicator(matches, + below_unmatched_threshold, + -2) + matches = self._set_values_using_indicator(matches, + between_thresholds, + -1) + + if self._force_match_for_each_row: + similarity_matrix_shape = shape_utils.combined_static_and_dynamic_shape( + similarity_matrix) + force_match_column_ids = tf.argmax( + input=similarity_matrix, axis=1, output_type=tf.int32) + force_match_column_indicators = tf.one_hot( + force_match_column_ids, depth=similarity_matrix_shape[1]) + force_match_row_ids = tf.argmax( + input=force_match_column_indicators, axis=0, output_type=tf.int32) + force_match_column_mask = tf.cast( + tf.reduce_max(input_tensor=force_match_column_indicators, axis=0), + tf.bool) + final_matches = tf.where(force_match_column_mask, force_match_row_ids, + matches) + return final_matches + else: + return matches + + if similarity_matrix.shape.is_fully_defined(): + if similarity_matrix.shape.dims[0].value == 0: + return _match_when_rows_are_empty() + else: + return _match_when_rows_are_non_empty() + else: + return tf.cond( + pred=tf.greater(tf.shape(input=similarity_matrix)[0], 0), + true_fn=_match_when_rows_are_non_empty, + false_fn=_match_when_rows_are_empty) + + def _set_values_using_indicator(self, x, indicator, val): + """Set the indicated fields of x to val. + + Args: + x: tensor. + indicator: boolean with same shape as x. + val: scalar with value to set. + + Returns: + modified tensor. + """ + indicator = tf.cast(indicator, x.dtype) + return tf.add(tf.multiply(x, 1 - indicator), val * indicator) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/balanced_positive_negative_sampler.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/balanced_positive_negative_sampler.py new file mode 100644 index 0000000..fec123e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/balanced_positive_negative_sampler.py @@ -0,0 +1,274 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Class to subsample minibatches by balancing positives and negatives. + +Subsamples minibatches based on a pre-specified positive fraction in range +[0,1]. The class presumes there are many more negatives than positive examples: +if the desired batch_size cannot be achieved with the pre-specified positive +fraction, it fills the rest with negative examples. If this is not sufficient +for obtaining the desired batch_size, it returns fewer examples. + +The main function to call is Subsample(self, indicator, labels). For convenience +one can also call SubsampleWeights(self, weights, labels) which is defined in +the minibatch_sampler base class. + +When is_static is True, it implements a method that guarantees static shapes. +It also ensures the length of output of the subsample is always batch_size, even +when number of examples set to True in indicator is less than batch_size. + +This is originally implemented in TensorFlow Object Detection API. +""" + +import tensorflow.compat.v2 as tf + +from official.vision.detection.utils.object_detection import minibatch_sampler +from official.vision.detection.utils.object_detection import ops + + +class BalancedPositiveNegativeSampler(minibatch_sampler.MinibatchSampler): + """Subsamples minibatches to a desired balance of positives and negatives.""" + + def __init__(self, positive_fraction=0.5, is_static=False): + """Constructs a minibatch sampler. + + Args: + positive_fraction: desired fraction of positive examples (scalar in [0,1]) + in the batch. + is_static: If True, uses an implementation with static shape guarantees. + + Raises: + ValueError: if positive_fraction < 0, or positive_fraction > 1 + """ + if positive_fraction < 0 or positive_fraction > 1: + raise ValueError('positive_fraction should be in range [0,1]. ' + 'Received: %s.' % positive_fraction) + self._positive_fraction = positive_fraction + self._is_static = is_static + + def _get_num_pos_neg_samples(self, sorted_indices_tensor, sample_size): + """Counts the number of positives and negatives numbers to be sampled. + + Args: + sorted_indices_tensor: A sorted int32 tensor of shape [N] which contains + the signed indices of the examples where the sign is based on the label + value. The examples that cannot be sampled are set to 0. It samples + atmost sample_size*positive_fraction positive examples and remaining + from negative examples. + sample_size: Size of subsamples. + + Returns: + A tuple containing the number of positive and negative labels in the + subsample. + """ + input_length = tf.shape(input=sorted_indices_tensor)[0] + valid_positive_index = tf.greater(sorted_indices_tensor, + tf.zeros(input_length, tf.int32)) + num_sampled_pos = tf.reduce_sum( + input_tensor=tf.cast(valid_positive_index, tf.int32)) + max_num_positive_samples = tf.constant( + int(sample_size * self._positive_fraction), tf.int32) + num_positive_samples = tf.minimum(max_num_positive_samples, num_sampled_pos) + num_negative_samples = tf.constant(sample_size, + tf.int32) - num_positive_samples + + return num_positive_samples, num_negative_samples + + def _get_values_from_start_and_end(self, input_tensor, num_start_samples, + num_end_samples, total_num_samples): + """slices num_start_samples and last num_end_samples from input_tensor. + + Args: + input_tensor: An int32 tensor of shape [N] to be sliced. + num_start_samples: Number of examples to be sliced from the beginning + of the input tensor. + num_end_samples: Number of examples to be sliced from the end of the + input tensor. + total_num_samples: Sum of is num_start_samples and num_end_samples. This + should be a scalar. + + Returns: + A tensor containing the first num_start_samples and last num_end_samples + from input_tensor. + + """ + input_length = tf.shape(input=input_tensor)[0] + start_positions = tf.less(tf.range(input_length), num_start_samples) + end_positions = tf.greater_equal( + tf.range(input_length), input_length - num_end_samples) + selected_positions = tf.logical_or(start_positions, end_positions) + selected_positions = tf.cast(selected_positions, tf.float32) + indexed_positions = tf.multiply(tf.cumsum(selected_positions), + selected_positions) + one_hot_selector = tf.one_hot(tf.cast(indexed_positions, tf.int32) - 1, + total_num_samples, + dtype=tf.float32) + return tf.cast(tf.tensordot(tf.cast(input_tensor, tf.float32), + one_hot_selector, axes=[0, 0]), tf.int32) + + def _static_subsample(self, indicator, batch_size, labels): + """Returns subsampled minibatch. + + Args: + indicator: boolean tensor of shape [N] whose True entries can be sampled. + N should be a complie time constant. + batch_size: desired batch size. This scalar cannot be None. + labels: boolean tensor of shape [N] denoting positive(=True) and negative + (=False) examples. N should be a complie time constant. + + Returns: + sampled_idx_indicator: boolean tensor of shape [N], True for entries which + are sampled. It ensures the length of output of the subsample is always + batch_size, even when number of examples set to True in indicator is + less than batch_size. + + Raises: + ValueError: if labels and indicator are not 1D boolean tensors. + """ + # Check if indicator and labels have a static size. + if not indicator.shape.is_fully_defined(): + raise ValueError('indicator must be static in shape when is_static is' + 'True') + if not labels.shape.is_fully_defined(): + raise ValueError('labels must be static in shape when is_static is' + 'True') + if not isinstance(batch_size, int): + raise ValueError('batch_size has to be an integer when is_static is' + 'True.') + + input_length = tf.shape(input=indicator)[0] + + # Set the number of examples set True in indicator to be at least + # batch_size. + num_true_sampled = tf.reduce_sum( + input_tensor=tf.cast(indicator, tf.float32)) + additional_false_sample = tf.less_equal( + tf.cumsum(tf.cast(tf.logical_not(indicator), tf.float32)), + batch_size - num_true_sampled) + indicator = tf.logical_or(indicator, additional_false_sample) + + # Shuffle indicator and label. Need to store the permutation to restore the + # order post sampling. + permutation = tf.random.shuffle(tf.range(input_length)) + indicator = ops.matmul_gather_on_zeroth_axis( + tf.cast(indicator, tf.float32), permutation) + labels = ops.matmul_gather_on_zeroth_axis( + tf.cast(labels, tf.float32), permutation) + + # index (starting from 1) when indicator is True, 0 when False + indicator_idx = tf.where( + tf.cast(indicator, tf.bool), tf.range(1, input_length + 1), + tf.zeros(input_length, tf.int32)) + + # Replace -1 for negative, +1 for positive labels + signed_label = tf.where( + tf.cast(labels, tf.bool), tf.ones(input_length, tf.int32), + tf.scalar_mul(-1, tf.ones(input_length, tf.int32))) + # negative of index for negative label, positive index for positive label, + # 0 when indicator is False. + signed_indicator_idx = tf.multiply(indicator_idx, signed_label) + sorted_signed_indicator_idx = tf.nn.top_k( + signed_indicator_idx, input_length, sorted=True).values + + [num_positive_samples, + num_negative_samples] = self._get_num_pos_neg_samples( + sorted_signed_indicator_idx, batch_size) + + sampled_idx = self._get_values_from_start_and_end( + sorted_signed_indicator_idx, num_positive_samples, + num_negative_samples, batch_size) + + # Shift the indices to start from 0 and remove any samples that are set as + # False. + sampled_idx = tf.abs(sampled_idx) - tf.ones(batch_size, tf.int32) + sampled_idx = tf.multiply( + tf.cast(tf.greater_equal(sampled_idx, tf.constant(0)), tf.int32), + sampled_idx) + + sampled_idx_indicator = tf.cast( + tf.reduce_sum( + input_tensor=tf.one_hot(sampled_idx, depth=input_length), axis=0), + tf.bool) + + # project back the order based on stored permutations + reprojections = tf.one_hot(permutation, depth=input_length, + dtype=tf.float32) + return tf.cast(tf.tensordot( + tf.cast(sampled_idx_indicator, tf.float32), + reprojections, axes=[0, 0]), tf.bool) + + def subsample(self, indicator, batch_size, labels, scope=None): + """Returns subsampled minibatch. + + Args: + indicator: boolean tensor of shape [N] whose True entries can be sampled. + batch_size: desired batch size. If None, keeps all positive samples and + randomly selects negative samples so that the positive sample fraction + matches self._positive_fraction. It cannot be None is is_static is True. + labels: boolean tensor of shape [N] denoting positive(=True) and negative + (=False) examples. + scope: name scope. + + Returns: + sampled_idx_indicator: boolean tensor of shape [N], True for entries which + are sampled. + + Raises: + ValueError: if labels and indicator are not 1D boolean tensors. + """ + if len(indicator.get_shape().as_list()) != 1: + raise ValueError('indicator must be 1 dimensional, got a tensor of ' + 'shape %s' % indicator.get_shape()) + if len(labels.get_shape().as_list()) != 1: + raise ValueError('labels must be 1 dimensional, got a tensor of ' + 'shape %s' % labels.get_shape()) + if labels.dtype != tf.bool: + raise ValueError('labels should be of type bool. Received: %s' % + labels.dtype) + if indicator.dtype != tf.bool: + raise ValueError('indicator should be of type bool. Received: %s' % + indicator.dtype) + scope = scope or 'BalancedPositiveNegativeSampler' + with tf.name_scope(scope): + if self._is_static: + return self._static_subsample(indicator, batch_size, labels) + + else: + # Only sample from indicated samples + negative_idx = tf.logical_not(labels) + positive_idx = tf.logical_and(labels, indicator) + negative_idx = tf.logical_and(negative_idx, indicator) + + # Sample positive and negative samples separately + if batch_size is None: + max_num_pos = tf.reduce_sum( + input_tensor=tf.cast(positive_idx, dtype=tf.int32)) + else: + max_num_pos = int(self._positive_fraction * batch_size) + sampled_pos_idx = self.subsample_indicator(positive_idx, max_num_pos) + num_sampled_pos = tf.reduce_sum( + input_tensor=tf.cast(sampled_pos_idx, tf.int32)) + if batch_size is None: + negative_positive_ratio = ( + 1 - self._positive_fraction) / self._positive_fraction + max_num_neg = tf.cast( + negative_positive_ratio * + tf.cast(num_sampled_pos, dtype=tf.float32), + dtype=tf.int32) + else: + max_num_neg = batch_size - num_sampled_pos + sampled_neg_idx = self.subsample_indicator(negative_idx, max_num_neg) + + return tf.logical_or(sampled_pos_idx, sampled_neg_idx) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/box_coder.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/box_coder.py new file mode 100644 index 0000000..1b89bb8 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/box_coder.py @@ -0,0 +1,151 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Base box coder. + +Box coders convert between coordinate frames, namely image-centric +(with (0,0) on the top left of image) and anchor-centric (with (0,0) being +defined by a specific anchor). + +Users of a BoxCoder can call two methods: + encode: which encodes a box with respect to a given anchor + (or rather, a tensor of boxes wrt a corresponding tensor of anchors) and + decode: which inverts this encoding with a decode operation. +In both cases, the arguments are assumed to be in 1-1 correspondence already; +it is not the job of a BoxCoder to perform matching. +""" +from abc import ABCMeta +from abc import abstractmethod +from abc import abstractproperty + +import tensorflow.compat.v2 as tf + + +# Box coder types. +FASTER_RCNN = 'faster_rcnn' +KEYPOINT = 'keypoint' +MEAN_STDDEV = 'mean_stddev' +SQUARE = 'square' + + +class BoxCoder(object): + """Abstract base class for box coder.""" + __metaclass__ = ABCMeta + + @abstractproperty + def code_size(self): + """Return the size of each code. + + This number is a constant and should agree with the output of the `encode` + op (e.g. if rel_codes is the output of self.encode(...), then it should have + shape [N, code_size()]). This abstractproperty should be overridden by + implementations. + + Returns: + an integer constant + """ + pass + + def encode(self, boxes, anchors): + """Encode a box list relative to an anchor collection. + + Args: + boxes: BoxList holding N boxes to be encoded + anchors: BoxList of N anchors + + Returns: + a tensor representing N relative-encoded boxes + """ + with tf.name_scope('Encode'): + return self._encode(boxes, anchors) + + def decode(self, rel_codes, anchors): + """Decode boxes that are encoded relative to an anchor collection. + + Args: + rel_codes: a tensor representing N relative-encoded boxes + anchors: BoxList of anchors + + Returns: + boxlist: BoxList holding N boxes encoded in the ordinary way (i.e., + with corners y_min, x_min, y_max, x_max) + """ + with tf.name_scope('Decode'): + return self._decode(rel_codes, anchors) + + @abstractmethod + def _encode(self, boxes, anchors): + """Method to be overriden by implementations. + + Args: + boxes: BoxList holding N boxes to be encoded + anchors: BoxList of N anchors + + Returns: + a tensor representing N relative-encoded boxes + """ + pass + + @abstractmethod + def _decode(self, rel_codes, anchors): + """Method to be overriden by implementations. + + Args: + rel_codes: a tensor representing N relative-encoded boxes + anchors: BoxList of anchors + + Returns: + boxlist: BoxList holding N boxes encoded in the ordinary way (i.e., + with corners y_min, x_min, y_max, x_max) + """ + pass + + +def batch_decode(encoded_boxes, box_coder, anchors): + """Decode a batch of encoded boxes. + + This op takes a batch of encoded bounding boxes and transforms + them to a batch of bounding boxes specified by their corners in + the order of [y_min, x_min, y_max, x_max]. + + Args: + encoded_boxes: a float32 tensor of shape [batch_size, num_anchors, + code_size] representing the location of the objects. + box_coder: a BoxCoder object. + anchors: a BoxList of anchors used to encode `encoded_boxes`. + + Returns: + decoded_boxes: a float32 tensor of shape [batch_size, num_anchors, + coder_size] representing the corners of the objects in the order + of [y_min, x_min, y_max, x_max]. + + Raises: + ValueError: if batch sizes of the inputs are inconsistent, or if + the number of anchors inferred from encoded_boxes and anchors are + inconsistent. + """ + encoded_boxes.get_shape().assert_has_rank(3) + if encoded_boxes.get_shape()[1].value != anchors.num_boxes_static(): + raise ValueError('The number of anchors inferred from encoded_boxes' + ' and anchors are inconsistent: shape[1] of encoded_boxes' + ' %s should be equal to the number of anchors: %s.' % + (encoded_boxes.get_shape()[1].value, + anchors.num_boxes_static())) + + decoded_boxes = tf.stack([ + box_coder.decode(boxes, anchors).get() + for boxes in tf.unstack(encoded_boxes) + ]) + return decoded_boxes diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/box_list.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/box_list.py new file mode 100644 index 0000000..7c4da99 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/box_list.py @@ -0,0 +1,211 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Bounding Box List definition. + +BoxList represents a list of bounding boxes as tensorflow +tensors, where each bounding box is represented as a row of 4 numbers, +[y_min, x_min, y_max, x_max]. It is assumed that all bounding boxes +within a given list correspond to a single image. See also +box_list_ops.py for common box related operations (such as area, iou, etc). + +Optionally, users can add additional related fields (such as weights). +We assume the following things to be true about fields: +* they correspond to boxes in the box_list along the 0th dimension +* they have inferrable rank at graph construction time +* all dimensions except for possibly the 0th can be inferred + (i.e., not None) at graph construction time. + +Some other notes: + * Following tensorflow conventions, we use height, width ordering, + and correspondingly, y,x (or ymin, xmin, ymax, xmax) ordering + * Tensors are always provided as (flat) [N, 4] tensors. +""" + +import tensorflow.compat.v2 as tf + + +class BoxList(object): + """Box collection.""" + + def __init__(self, boxes): + """Constructs box collection. + + Args: + boxes: a tensor of shape [N, 4] representing box corners + + Raises: + ValueError: if invalid dimensions for bbox data or if bbox data is not in + float32 format. + """ + if len(boxes.get_shape()) != 2 or boxes.get_shape()[-1] != 4: + raise ValueError('Invalid dimensions for box data.') + if boxes.dtype != tf.float32: + raise ValueError('Invalid tensor type: should be tf.float32') + self.data = {'boxes': boxes} + + def num_boxes(self): + """Returns number of boxes held in collection. + + Returns: + a tensor representing the number of boxes held in the collection. + """ + return tf.shape(input=self.data['boxes'])[0] + + def num_boxes_static(self): + """Returns number of boxes held in collection. + + This number is inferred at graph construction time rather than run-time. + + Returns: + Number of boxes held in collection (integer) or None if this is not + inferrable at graph construction time. + """ + return self.data['boxes'].get_shape().dims[0].value + + def get_all_fields(self): + """Returns all fields.""" + return self.data.keys() + + def get_extra_fields(self): + """Returns all non-box fields (i.e., everything not named 'boxes').""" + return [k for k in self.data.keys() if k != 'boxes'] + + def add_field(self, field, field_data): + """Add field to box list. + + This method can be used to add related box data such as + weights/labels, etc. + + Args: + field: a string key to access the data via `get` + field_data: a tensor containing the data to store in the BoxList + """ + self.data[field] = field_data + + def has_field(self, field): + return field in self.data + + def get(self): + """Convenience function for accessing box coordinates. + + Returns: + a tensor with shape [N, 4] representing box coordinates. + """ + return self.get_field('boxes') + + def set(self, boxes): + """Convenience function for setting box coordinates. + + Args: + boxes: a tensor of shape [N, 4] representing box corners + + Raises: + ValueError: if invalid dimensions for bbox data + """ + if len(boxes.get_shape()) != 2 or boxes.get_shape()[-1] != 4: + raise ValueError('Invalid dimensions for box data.') + self.data['boxes'] = boxes + + def get_field(self, field): + """Accesses a box collection and associated fields. + + This function returns specified field with object; if no field is specified, + it returns the box coordinates. + + Args: + field: this optional string parameter can be used to specify + a related field to be accessed. + + Returns: + a tensor representing the box collection or an associated field. + + Raises: + ValueError: if invalid field + """ + if not self.has_field(field): + raise ValueError('field ' + str(field) + ' does not exist') + return self.data[field] + + def set_field(self, field, value): + """Sets the value of a field. + + Updates the field of a box_list with a given value. + + Args: + field: (string) name of the field to set value. + value: the value to assign to the field. + + Raises: + ValueError: if the box_list does not have specified field. + """ + if not self.has_field(field): + raise ValueError('field %s does not exist' % field) + self.data[field] = value + + def get_center_coordinates_and_sizes(self, scope=None): + """Computes the center coordinates, height and width of the boxes. + + Args: + scope: name scope of the function. + + Returns: + a list of 4 1-D tensors [ycenter, xcenter, height, width]. + """ + if not scope: + scope = 'get_center_coordinates_and_sizes' + with tf.name_scope(scope): + box_corners = self.get() + ymin, xmin, ymax, xmax = tf.unstack(tf.transpose(a=box_corners)) + width = xmax - xmin + height = ymax - ymin + ycenter = ymin + height / 2. + xcenter = xmin + width / 2. + return [ycenter, xcenter, height, width] + + def transpose_coordinates(self, scope=None): + """Transpose the coordinate representation in a boxlist. + + Args: + scope: name scope of the function. + """ + if not scope: + scope = 'transpose_coordinates' + with tf.name_scope(scope): + y_min, x_min, y_max, x_max = tf.split( + value=self.get(), num_or_size_splits=4, axis=1) + self.set(tf.concat([x_min, y_min, x_max, y_max], 1)) + + def as_tensor_dict(self, fields=None): + """Retrieves specified fields as a dictionary of tensors. + + Args: + fields: (optional) list of fields to return in the dictionary. + If None (default), all fields are returned. + + Returns: + tensor_dict: A dictionary of tensors specified by fields. + + Raises: + ValueError: if specified field is not contained in boxlist. + """ + tensor_dict = {} + if fields is None: + fields = self.get_all_fields() + for field in fields: + if not self.has_field(field): + raise ValueError('boxlist must contain all specified fields') + tensor_dict[field] = self.get_field(field) + return tensor_dict diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/box_list_ops.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/box_list_ops.py new file mode 100644 index 0000000..2299f2b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/box_list_ops.py @@ -0,0 +1,1079 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Bounding Box List operations. + +Example box operations that are supported: + * areas: compute bounding box areas + * iou: pairwise intersection-over-union scores + * sq_dist: pairwise distances between bounding boxes + +Whenever box_list_ops functions output a BoxList, the fields of the incoming +BoxList are retained unless documented otherwise. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from six.moves import range +import tensorflow.compat.v2 as tf + +from official.vision.detection.utils.object_detection import box_list +from official.vision.detection.utils.object_detection import ops + + +class SortOrder(object): + """Enum class for sort order. + + Attributes: + ascend: ascend order. + descend: descend order. + """ + ascend = 1 + descend = 2 + + +def area(boxlist, scope=None): + """Computes area of boxes. + + Args: + boxlist: BoxList holding N boxes + scope: name scope. + + Returns: + a tensor with shape [N] representing box areas. + """ + with tf.name_scope(scope, 'Area'): + y_min, x_min, y_max, x_max = tf.split( + value=boxlist.get(), num_or_size_splits=4, axis=1) + return tf.squeeze((y_max - y_min) * (x_max - x_min), [1]) + + +def height_width(boxlist, scope=None): + """Computes height and width of boxes in boxlist. + + Args: + boxlist: BoxList holding N boxes + scope: name scope. + + Returns: + Height: A tensor with shape [N] representing box heights. + Width: A tensor with shape [N] representing box widths. + """ + with tf.name_scope(scope, 'HeightWidth'): + y_min, x_min, y_max, x_max = tf.split( + value=boxlist.get(), num_or_size_splits=4, axis=1) + return tf.squeeze(y_max - y_min, [1]), tf.squeeze(x_max - x_min, [1]) + + +def scale(boxlist, y_scale, x_scale, scope=None): + """scale box coordinates in x and y dimensions. + + Args: + boxlist: BoxList holding N boxes + y_scale: (float) scalar tensor + x_scale: (float) scalar tensor + scope: name scope. + + Returns: + boxlist: BoxList holding N boxes + """ + with tf.name_scope(scope, 'Scale'): + y_scale = tf.cast(y_scale, tf.float32) + x_scale = tf.cast(x_scale, tf.float32) + y_min, x_min, y_max, x_max = tf.split( + value=boxlist.get(), num_or_size_splits=4, axis=1) + y_min = y_scale * y_min + y_max = y_scale * y_max + x_min = x_scale * x_min + x_max = x_scale * x_max + scaled_boxlist = box_list.BoxList( + tf.concat([y_min, x_min, y_max, x_max], 1)) + return _copy_extra_fields(scaled_boxlist, boxlist) + + +def clip_to_window(boxlist, window, filter_nonoverlapping=True, scope=None): + """Clip bounding boxes to a window. + + This op clips any input bounding boxes (represented by bounding box + corners) to a window, optionally filtering out boxes that do not + overlap at all with the window. + + Args: + boxlist: BoxList holding M_in boxes + window: a tensor of shape [4] representing the [y_min, x_min, y_max, x_max] + window to which the op should clip boxes. + filter_nonoverlapping: whether to filter out boxes that do not overlap at + all with the window. + scope: name scope. + + Returns: + a BoxList holding M_out boxes where M_out <= M_in + """ + with tf.name_scope(scope, 'ClipToWindow'): + y_min, x_min, y_max, x_max = tf.split( + value=boxlist.get(), num_or_size_splits=4, axis=1) + win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window) + y_min_clipped = tf.maximum(tf.minimum(y_min, win_y_max), win_y_min) + y_max_clipped = tf.maximum(tf.minimum(y_max, win_y_max), win_y_min) + x_min_clipped = tf.maximum(tf.minimum(x_min, win_x_max), win_x_min) + x_max_clipped = tf.maximum(tf.minimum(x_max, win_x_max), win_x_min) + clipped = box_list.BoxList( + tf.concat([y_min_clipped, x_min_clipped, y_max_clipped, x_max_clipped], + 1)) + clipped = _copy_extra_fields(clipped, boxlist) + if filter_nonoverlapping: + areas = area(clipped) + nonzero_area_indices = tf.cast( + tf.reshape(tf.where(tf.greater(areas, 0.0)), [-1]), tf.int32) + clipped = gather(clipped, nonzero_area_indices) + return clipped + + +def prune_outside_window(boxlist, window, scope=None): + """Prunes bounding boxes that fall outside a given window. + + This function prunes bounding boxes that even partially fall outside the given + window. See also clip_to_window which only prunes bounding boxes that fall + completely outside the window, and clips any bounding boxes that partially + overflow. + + Args: + boxlist: a BoxList holding M_in boxes. + window: a float tensor of shape [4] representing [ymin, xmin, ymax, xmax] + of the window + scope: name scope. + + Returns: + pruned_corners: a tensor with shape [M_out, 4] where M_out <= M_in + valid_indices: a tensor with shape [M_out] indexing the valid bounding boxes + in the input tensor. + """ + with tf.name_scope(scope, 'PruneOutsideWindow'): + y_min, x_min, y_max, x_max = tf.split( + value=boxlist.get(), num_or_size_splits=4, axis=1) + win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window) + coordinate_violations = tf.concat([ + tf.less(y_min, win_y_min), tf.less(x_min, win_x_min), + tf.greater(y_max, win_y_max), tf.greater(x_max, win_x_max) + ], 1) + valid_indices = tf.reshape( + tf.where(tf.logical_not(tf.reduce_any(coordinate_violations, 1))), [-1]) + return gather(boxlist, valid_indices), valid_indices + + +def prune_completely_outside_window(boxlist, window, scope=None): + """Prunes bounding boxes that fall completely outside of the given window. + + The function clip_to_window prunes bounding boxes that fall + completely outside the window, but also clips any bounding boxes that + partially overflow. This function does not clip partially overflowing boxes. + + Args: + boxlist: a BoxList holding M_in boxes. + window: a float tensor of shape [4] representing [ymin, xmin, ymax, xmax] + of the window + scope: name scope. + + Returns: + pruned_boxlist: a new BoxList with all bounding boxes partially or fully in + the window. + valid_indices: a tensor with shape [M_out] indexing the valid bounding boxes + in the input tensor. + """ + with tf.name_scope(scope, 'PruneCompleteleyOutsideWindow'): + y_min, x_min, y_max, x_max = tf.split( + value=boxlist.get(), num_or_size_splits=4, axis=1) + win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window) + coordinate_violations = tf.concat([ + tf.greater_equal(y_min, win_y_max), tf.greater_equal(x_min, win_x_max), + tf.less_equal(y_max, win_y_min), tf.less_equal(x_max, win_x_min) + ], 1) + valid_indices = tf.reshape( + tf.where(tf.logical_not(tf.reduce_any(coordinate_violations, 1))), [-1]) + return gather(boxlist, valid_indices), valid_indices + + +def intersection(boxlist1, boxlist2, scope=None): + """Compute pairwise intersection areas between boxes. + + Args: + boxlist1: BoxList holding N boxes + boxlist2: BoxList holding M boxes + scope: name scope. + + Returns: + a tensor with shape [N, M] representing pairwise intersections + """ + with tf.name_scope(scope, 'Intersection'): + y_min1, x_min1, y_max1, x_max1 = tf.split( + value=boxlist1.get(), num_or_size_splits=4, axis=1) + y_min2, x_min2, y_max2, x_max2 = tf.split( + value=boxlist2.get(), num_or_size_splits=4, axis=1) + all_pairs_min_ymax = tf.minimum(y_max1, tf.transpose(y_max2)) + all_pairs_max_ymin = tf.maximum(y_min1, tf.transpose(y_min2)) + intersect_heights = tf.maximum(0.0, all_pairs_min_ymax - all_pairs_max_ymin) + all_pairs_min_xmax = tf.minimum(x_max1, tf.transpose(x_max2)) + all_pairs_max_xmin = tf.maximum(x_min1, tf.transpose(x_min2)) + intersect_widths = tf.maximum(0.0, all_pairs_min_xmax - all_pairs_max_xmin) + return intersect_heights * intersect_widths + + +def matched_intersection(boxlist1, boxlist2, scope=None): + """Compute intersection areas between corresponding boxes in two boxlists. + + Args: + boxlist1: BoxList holding N boxes + boxlist2: BoxList holding N boxes + scope: name scope. + + Returns: + a tensor with shape [N] representing pairwise intersections + """ + with tf.name_scope(scope, 'MatchedIntersection'): + y_min1, x_min1, y_max1, x_max1 = tf.split( + value=boxlist1.get(), num_or_size_splits=4, axis=1) + y_min2, x_min2, y_max2, x_max2 = tf.split( + value=boxlist2.get(), num_or_size_splits=4, axis=1) + min_ymax = tf.minimum(y_max1, y_max2) + max_ymin = tf.maximum(y_min1, y_min2) + intersect_heights = tf.maximum(0.0, min_ymax - max_ymin) + min_xmax = tf.minimum(x_max1, x_max2) + max_xmin = tf.maximum(x_min1, x_min2) + intersect_widths = tf.maximum(0.0, min_xmax - max_xmin) + return tf.reshape(intersect_heights * intersect_widths, [-1]) + + +def iou(boxlist1, boxlist2, scope=None): + """Computes pairwise intersection-over-union between box collections. + + Args: + boxlist1: BoxList holding N boxes + boxlist2: BoxList holding M boxes + scope: name scope. + + Returns: + a tensor with shape [N, M] representing pairwise iou scores. + """ + with tf.name_scope(scope, 'IOU'): + intersections = intersection(boxlist1, boxlist2) + areas1 = area(boxlist1) + areas2 = area(boxlist2) + unions = ( + tf.expand_dims(areas1, 1) + tf.expand_dims(areas2, 0) - intersections) + return tf.where( + tf.equal(intersections, 0.0), + tf.zeros_like(intersections), tf.truediv(intersections, unions)) + + +def matched_iou(boxlist1, boxlist2, scope=None): + """Compute intersection-over-union between corresponding boxes in boxlists. + + Args: + boxlist1: BoxList holding N boxes + boxlist2: BoxList holding N boxes + scope: name scope. + + Returns: + a tensor with shape [N] representing pairwise iou scores. + """ + with tf.name_scope(scope, 'MatchedIOU'): + intersections = matched_intersection(boxlist1, boxlist2) + areas1 = area(boxlist1) + areas2 = area(boxlist2) + unions = areas1 + areas2 - intersections + return tf.where( + tf.equal(intersections, 0.0), + tf.zeros_like(intersections), tf.truediv(intersections, unions)) + + +def ioa(boxlist1, boxlist2, scope=None): + """Computes pairwise intersection-over-area between box collections. + + intersection-over-area (IOA) between two boxes box1 and box2 is defined as + their intersection area over box2's area. Note that ioa is not symmetric, + that is, ioa(box1, box2) != ioa(box2, box1). + + Args: + boxlist1: BoxList holding N boxes + boxlist2: BoxList holding M boxes + scope: name scope. + + Returns: + a tensor with shape [N, M] representing pairwise ioa scores. + """ + with tf.name_scope(scope, 'IOA'): + intersections = intersection(boxlist1, boxlist2) + areas = tf.expand_dims(area(boxlist2), 0) + return tf.truediv(intersections, areas) + + +def prune_non_overlapping_boxes( + boxlist1, boxlist2, min_overlap=0.0, scope=None): + """Prunes the boxes in boxlist1 that overlap less than thresh with boxlist2. + + For each box in boxlist1, we want its IOA to be more than minoverlap with + at least one of the boxes in boxlist2. If it does not, we remove it. + + Args: + boxlist1: BoxList holding N boxes. + boxlist2: BoxList holding M boxes. + min_overlap: Minimum required overlap between boxes, to count them as + overlapping. + scope: name scope. + + Returns: + new_boxlist1: A pruned boxlist with size [N', 4]. + keep_inds: A tensor with shape [N'] indexing kept bounding boxes in the + first input BoxList `boxlist1`. + """ + with tf.name_scope(scope, 'PruneNonOverlappingBoxes'): + ioa_ = ioa(boxlist2, boxlist1) # [M, N] tensor + ioa_ = tf.reduce_max(ioa_, reduction_indices=[0]) # [N] tensor + keep_bool = tf.greater_equal(ioa_, tf.constant(min_overlap)) + keep_inds = tf.squeeze(tf.where(keep_bool), axis=[1]) + new_boxlist1 = gather(boxlist1, keep_inds) + return new_boxlist1, keep_inds + + +def prune_small_boxes(boxlist, min_side, scope=None): + """Prunes small boxes in the boxlist which have a side smaller than min_side. + + Args: + boxlist: BoxList holding N boxes. + min_side: Minimum width AND height of box to survive pruning. + scope: name scope. + + Returns: + A pruned boxlist. + """ + with tf.name_scope(scope, 'PruneSmallBoxes'): + height, width = height_width(boxlist) + is_valid = tf.logical_and(tf.greater_equal(width, min_side), + tf.greater_equal(height, min_side)) + return gather(boxlist, tf.reshape(tf.where(is_valid), [-1])) + + +def change_coordinate_frame(boxlist, window, scope=None): + """Change coordinate frame of the boxlist to be relative to window's frame. + + Given a window of the form [ymin, xmin, ymax, xmax], + changes bounding box coordinates from boxlist to be relative to this window + (e.g., the min corner maps to (0,0) and the max corner maps to (1,1)). + + An example use case is data augmentation: where we are given groundtruth + boxes (boxlist) and would like to randomly crop the image to some + window (window). In this case we need to change the coordinate frame of + each groundtruth box to be relative to this new window. + + Args: + boxlist: A BoxList object holding N boxes. + window: A rank 1 tensor [4]. + scope: name scope. + + Returns: + Returns a BoxList object with N boxes. + """ + with tf.name_scope(scope, 'ChangeCoordinateFrame'): + win_height = window[2] - window[0] + win_width = window[3] - window[1] + boxlist_new = scale(box_list.BoxList( + boxlist.get() - [window[0], window[1], window[0], window[1]]), + 1.0 / win_height, 1.0 / win_width) + boxlist_new = _copy_extra_fields(boxlist_new, boxlist) + return boxlist_new + + +def sq_dist(boxlist1, boxlist2, scope=None): + """Computes the pairwise squared distances between box corners. + + This op treats each box as if it were a point in a 4d Euclidean space and + computes pairwise squared distances. + + Mathematically, we are given two matrices of box coordinates X and Y, + where X(i,:) is the i'th row of X, containing the 4 numbers defining the + corners of the i'th box in boxlist1. Similarly Y(j,:) corresponds to + boxlist2. We compute + Z(i,j) = ||X(i,:) - Y(j,:)||^2 + = ||X(i,:)||^2 + ||Y(j,:)||^2 - 2 X(i,:)' * Y(j,:), + + Args: + boxlist1: BoxList holding N boxes + boxlist2: BoxList holding M boxes + scope: name scope. + + Returns: + a tensor with shape [N, M] representing pairwise distances + """ + with tf.name_scope(scope, 'SqDist'): + sqnorm1 = tf.reduce_sum(tf.square(boxlist1.get()), 1, keep_dims=True) + sqnorm2 = tf.reduce_sum(tf.square(boxlist2.get()), 1, keep_dims=True) + innerprod = tf.matmul(boxlist1.get(), boxlist2.get(), + transpose_a=False, transpose_b=True) + return sqnorm1 + tf.transpose(sqnorm2) - 2.0 * innerprod + + +def boolean_mask(boxlist, indicator, fields=None, scope=None, + use_static_shapes=False, indicator_sum=None): + """Select boxes from BoxList according to indicator and return new BoxList. + + `boolean_mask` returns the subset of boxes that are marked as "True" by the + indicator tensor. By default, `boolean_mask` returns boxes corresponding to + the input index list, as well as all additional fields stored in the boxlist + (indexing into the first dimension). However one can optionally only draw + from a subset of fields. + + Args: + boxlist: BoxList holding N boxes + indicator: a rank-1 boolean tensor + fields: (optional) list of fields to also gather from. If None (default), + all fields are gathered from. Pass an empty fields list to only gather + the box coordinates. + scope: name scope. + use_static_shapes: Whether to use an implementation with static shape + gurantees. + indicator_sum: An integer containing the sum of `indicator` vector. Only + required if `use_static_shape` is True. + + Returns: + subboxlist: a BoxList corresponding to the subset of the input BoxList + specified by indicator + Raises: + ValueError: if `indicator` is not a rank-1 boolean tensor. + """ + with tf.name_scope(scope, 'BooleanMask'): + if indicator.shape.ndims != 1: + raise ValueError('indicator should have rank 1') + if indicator.dtype != tf.bool: + raise ValueError('indicator should be a boolean tensor') + if use_static_shapes: + if not (indicator_sum and isinstance(indicator_sum, int)): + raise ValueError('`indicator_sum` must be a of type int') + selected_positions = tf.cast(indicator, dtype=tf.float32) + indexed_positions = tf.cast( + tf.multiply( + tf.cumsum(selected_positions), selected_positions), + dtype=tf.int32) + one_hot_selector = tf.one_hot( + indexed_positions - 1, indicator_sum, dtype=tf.float32) + sampled_indices = tf.cast( + tf.tensordot( + tf.cast(tf.range(tf.shape(indicator)[0]), dtype=tf.float32), + one_hot_selector, + axes=[0, 0]), + dtype=tf.int32) + return gather(boxlist, sampled_indices, use_static_shapes=True) + else: + subboxlist = box_list.BoxList(tf.boolean_mask(boxlist.get(), indicator)) + if fields is None: + fields = boxlist.get_extra_fields() + for field in fields: + if not boxlist.has_field(field): + raise ValueError('boxlist must contain all specified fields') + subfieldlist = tf.boolean_mask(boxlist.get_field(field), indicator) + subboxlist.add_field(field, subfieldlist) + return subboxlist + + +def gather(boxlist, indices, fields=None, scope=None, use_static_shapes=False): + """Gather boxes from BoxList according to indices and return new BoxList. + + By default, `gather` returns boxes corresponding to the input index list, as + well as all additional fields stored in the boxlist (indexing into the + first dimension). However one can optionally only gather from a + subset of fields. + + Args: + boxlist: BoxList holding N boxes + indices: a rank-1 tensor of type int32 / int64 + fields: (optional) list of fields to also gather from. If None (default), + all fields are gathered from. Pass an empty fields list to only gather + the box coordinates. + scope: name scope. + use_static_shapes: Whether to use an implementation with static shape + gurantees. + + Returns: + subboxlist: a BoxList corresponding to the subset of the input BoxList + specified by indices + Raises: + ValueError: if specified field is not contained in boxlist or if the + indices are not of type int32 + """ + with tf.name_scope(scope, 'Gather'): + if len(indices.shape.as_list()) != 1: + raise ValueError('indices should have rank 1') + if indices.dtype != tf.int32 and indices.dtype != tf.int64: + raise ValueError('indices should be an int32 / int64 tensor') + gather_op = tf.gather + if use_static_shapes: + gather_op = ops.matmul_gather_on_zeroth_axis + subboxlist = box_list.BoxList(gather_op(boxlist.get(), indices)) + if fields is None: + fields = boxlist.get_extra_fields() + fields += ['boxes'] + for field in fields: + if not boxlist.has_field(field): + raise ValueError('boxlist must contain all specified fields') + subfieldlist = gather_op(boxlist.get_field(field), indices) + subboxlist.add_field(field, subfieldlist) + return subboxlist + + +def concatenate(boxlists, fields=None, scope=None): + """Concatenate list of BoxLists. + + This op concatenates a list of input BoxLists into a larger BoxList. It also + handles concatenation of BoxList fields as long as the field tensor shapes + are equal except for the first dimension. + + Args: + boxlists: list of BoxList objects + fields: optional list of fields to also concatenate. By default, all + fields from the first BoxList in the list are included in the + concatenation. + scope: name scope. + + Returns: + a BoxList with number of boxes equal to + sum([boxlist.num_boxes() for boxlist in BoxList]) + Raises: + ValueError: if boxlists is invalid (i.e., is not a list, is empty, or + contains non BoxList objects), or if requested fields are not contained in + all boxlists + """ + with tf.name_scope(scope, 'Concatenate'): + if not isinstance(boxlists, list): + raise ValueError('boxlists should be a list') + if not boxlists: + raise ValueError('boxlists should have nonzero length') + for boxlist in boxlists: + if not isinstance(boxlist, box_list.BoxList): + raise ValueError('all elements of boxlists should be BoxList objects') + concatenated = box_list.BoxList( + tf.concat([boxlist.get() for boxlist in boxlists], 0)) + if fields is None: + fields = boxlists[0].get_extra_fields() + for field in fields: + first_field_shape = boxlists[0].get_field(field).get_shape().as_list() + first_field_shape[0] = -1 + if None in first_field_shape: + raise ValueError('field %s must have fully defined shape except for the' + ' 0th dimension.' % field) + for boxlist in boxlists: + if not boxlist.has_field(field): + raise ValueError('boxlist must contain all requested fields') + field_shape = boxlist.get_field(field).get_shape().as_list() + field_shape[0] = -1 + if field_shape != first_field_shape: + raise ValueError('field %s must have same shape for all boxlists ' + 'except for the 0th dimension.' % field) + concatenated_field = tf.concat( + [boxlist.get_field(field) for boxlist in boxlists], 0) + concatenated.add_field(field, concatenated_field) + return concatenated + + +def sort_by_field(boxlist, field, order=SortOrder.descend, scope=None): + """Sort boxes and associated fields according to a scalar field. + + A common use case is reordering the boxes according to descending scores. + + Args: + boxlist: BoxList holding N boxes. + field: A BoxList field for sorting and reordering the BoxList. + order: (Optional) descend or ascend. Default is descend. + scope: name scope. + + Returns: + sorted_boxlist: A sorted BoxList with the field in the specified order. + + Raises: + ValueError: if specified field does not exist + ValueError: if the order is not either descend or ascend + """ + with tf.name_scope(scope, 'SortByField'): + if order != SortOrder.descend and order != SortOrder.ascend: + raise ValueError('Invalid sort order') + + field_to_sort = boxlist.get_field(field) + if len(field_to_sort.shape.as_list()) != 1: + raise ValueError('Field should have rank 1') + + num_boxes = boxlist.num_boxes() + num_entries = tf.size(field_to_sort) + length_assert = tf.Assert( + tf.equal(num_boxes, num_entries), + ['Incorrect field size: actual vs expected.', num_entries, num_boxes]) + + with tf.control_dependencies([length_assert]): + _, sorted_indices = tf.nn.top_k(field_to_sort, num_boxes, sorted=True) + + if order == SortOrder.ascend: + sorted_indices = tf.reverse_v2(sorted_indices, [0]) + + return gather(boxlist, sorted_indices) + + +def visualize_boxes_in_image(image, boxlist, normalized=False, scope=None): + """Overlay bounding box list on image. + + Currently this visualization plots a 1 pixel thick red bounding box on top + of the image. Note that tf.image.draw_bounding_boxes essentially is + 1 indexed. + + Args: + image: an image tensor with shape [height, width, 3] + boxlist: a BoxList + normalized: (boolean) specify whether corners are to be interpreted + as absolute coordinates in image space or normalized with respect to the + image size. + scope: name scope. + + Returns: + image_and_boxes: an image tensor with shape [height, width, 3] + """ + with tf.name_scope(scope, 'VisualizeBoxesInImage'): + if not normalized: + height, width, _ = tf.unstack(tf.shape(image)) + boxlist = scale(boxlist, + 1.0 / tf.cast(height, tf.float32), + 1.0 / tf.cast(width, tf.float32)) + corners = tf.expand_dims(boxlist.get(), 0) + image = tf.expand_dims(image, 0) + return tf.squeeze(tf.image.draw_bounding_boxes(image, corners), [0]) + + +def filter_field_value_equals(boxlist, field, value, scope=None): + """Filter to keep only boxes with field entries equal to the given value. + + Args: + boxlist: BoxList holding N boxes. + field: field name for filtering. + value: scalar value. + scope: name scope. + + Returns: + a BoxList holding M boxes where M <= N + + Raises: + ValueError: if boxlist not a BoxList object or if it does not have + the specified field. + """ + with tf.name_scope(scope, 'FilterFieldValueEquals'): + if not isinstance(boxlist, box_list.BoxList): + raise ValueError('boxlist must be a BoxList') + if not boxlist.has_field(field): + raise ValueError('boxlist must contain the specified field') + filter_field = boxlist.get_field(field) + gather_index = tf.reshape(tf.where(tf.equal(filter_field, value)), [-1]) + return gather(boxlist, gather_index) + + +def filter_greater_than(boxlist, thresh, scope=None): + """Filter to keep only boxes with score exceeding a given threshold. + + This op keeps the collection of boxes whose corresponding scores are + greater than the input threshold. + + TODO(jonathanhuang): Change function name to filter_scores_greater_than + + Args: + boxlist: BoxList holding N boxes. Must contain a 'scores' field + representing detection scores. + thresh: scalar threshold + scope: name scope. + + Returns: + a BoxList holding M boxes where M <= N + + Raises: + ValueError: if boxlist not a BoxList object or if it does not + have a scores field + """ + with tf.name_scope(scope, 'FilterGreaterThan'): + if not isinstance(boxlist, box_list.BoxList): + raise ValueError('boxlist must be a BoxList') + if not boxlist.has_field('scores'): + raise ValueError('input boxlist must have \'scores\' field') + scores = boxlist.get_field('scores') + if len(scores.shape.as_list()) > 2: + raise ValueError('Scores should have rank 1 or 2') + if len(scores.shape.as_list()) == 2 and scores.shape.as_list()[1] != 1: + raise ValueError('Scores should have rank 1 or have shape ' + 'consistent with [None, 1]') + high_score_indices = tf.cast(tf.reshape( + tf.where(tf.greater(scores, thresh)), + [-1]), tf.int32) + return gather(boxlist, high_score_indices) + + +def non_max_suppression(boxlist, thresh, max_output_size, scope=None): + """Non maximum suppression. + + This op greedily selects a subset of detection bounding boxes, pruning + away boxes that have high IOU (intersection over union) overlap (> thresh) + with already selected boxes. Note that this only works for a single class --- + to apply NMS to multi-class predictions, use MultiClassNonMaxSuppression. + + Args: + boxlist: BoxList holding N boxes. Must contain a 'scores' field + representing detection scores. + thresh: scalar threshold + max_output_size: maximum number of retained boxes + scope: name scope. + + Returns: + a BoxList holding M boxes where M <= max_output_size + Raises: + ValueError: if thresh is not in [0, 1] + """ + with tf.name_scope(scope, 'NonMaxSuppression'): + if not 0 <= thresh <= 1.0: + raise ValueError('thresh must be between 0 and 1') + if not isinstance(boxlist, box_list.BoxList): + raise ValueError('boxlist must be a BoxList') + if not boxlist.has_field('scores'): + raise ValueError('input boxlist must have \'scores\' field') + selected_indices = tf.image.non_max_suppression( + boxlist.get(), boxlist.get_field('scores'), + max_output_size, iou_threshold=thresh) + return gather(boxlist, selected_indices) + + +def _copy_extra_fields(boxlist_to_copy_to, boxlist_to_copy_from): + """Copies the extra fields of boxlist_to_copy_from to boxlist_to_copy_to. + + Args: + boxlist_to_copy_to: BoxList to which extra fields are copied. + boxlist_to_copy_from: BoxList from which fields are copied. + + Returns: + boxlist_to_copy_to with extra fields. + """ + for field in boxlist_to_copy_from.get_extra_fields(): + boxlist_to_copy_to.add_field(field, boxlist_to_copy_from.get_field(field)) + return boxlist_to_copy_to + + +def to_normalized_coordinates(boxlist, height, width, + check_range=True, scope=None): + """Converts absolute box coordinates to normalized coordinates in [0, 1]. + + Usually one uses the dynamic shape of the image or conv-layer tensor: + boxlist = box_list_ops.to_normalized_coordinates(boxlist, + tf.shape(images)[1], + tf.shape(images)[2]), + + This function raises an assertion failed error at graph execution time when + the maximum coordinate is smaller than 1.01 (which means that coordinates are + already normalized). The value 1.01 is to deal with small rounding errors. + + Args: + boxlist: BoxList with coordinates in terms of pixel-locations. + height: Maximum value for height of absolute box coordinates. + width: Maximum value for width of absolute box coordinates. + check_range: If True, checks if the coordinates are normalized or not. + scope: name scope. + + Returns: + boxlist with normalized coordinates in [0, 1]. + """ + with tf.name_scope(scope, 'ToNormalizedCoordinates'): + height = tf.cast(height, tf.float32) + width = tf.cast(width, tf.float32) + + if check_range: + max_val = tf.reduce_max(boxlist.get()) + max_assert = tf.Assert(tf.greater(max_val, 1.01), + ['max value is lower than 1.01: ', max_val]) + with tf.control_dependencies([max_assert]): + width = tf.identity(width) + + return scale(boxlist, 1 / height, 1 / width) + + +def to_absolute_coordinates(boxlist, + height, + width, + check_range=True, + maximum_normalized_coordinate=1.1, + scope=None): + """Converts normalized box coordinates to absolute pixel coordinates. + + This function raises an assertion failed error when the maximum box coordinate + value is larger than maximum_normalized_coordinate (in which case coordinates + are already absolute). + + Args: + boxlist: BoxList with coordinates in range [0, 1]. + height: Maximum value for height of absolute box coordinates. + width: Maximum value for width of absolute box coordinates. + check_range: If True, checks if the coordinates are normalized or not. + maximum_normalized_coordinate: Maximum coordinate value to be considered + as normalized, default to 1.1. + scope: name scope. + + Returns: + boxlist with absolute coordinates in terms of the image size. + + """ + with tf.name_scope(scope, 'ToAbsoluteCoordinates'): + height = tf.cast(height, tf.float32) + width = tf.cast(width, tf.float32) + + # Ensure range of input boxes is correct. + if check_range: + box_maximum = tf.reduce_max(boxlist.get()) + max_assert = tf.Assert( + tf.greater_equal(maximum_normalized_coordinate, box_maximum), + ['maximum box coordinate value is larger ' + 'than %f: ' % maximum_normalized_coordinate, box_maximum]) + with tf.control_dependencies([max_assert]): + width = tf.identity(width) + + return scale(boxlist, height, width) + + +def refine_boxes_multi_class(pool_boxes, + num_classes, + nms_iou_thresh, + nms_max_detections, + voting_iou_thresh=0.5): + """Refines a pool of boxes using non max suppression and box voting. + + Box refinement is done independently for each class. + + Args: + pool_boxes: (BoxList) A collection of boxes to be refined. pool_boxes must + have a rank 1 'scores' field and a rank 1 'classes' field. + num_classes: (int scalar) Number of classes. + nms_iou_thresh: (float scalar) iou threshold for non max suppression (NMS). + nms_max_detections: (int scalar) maximum output size for NMS. + voting_iou_thresh: (float scalar) iou threshold for box voting. + + Returns: + BoxList of refined boxes. + + Raises: + ValueError: if + a) nms_iou_thresh or voting_iou_thresh is not in [0, 1]. + b) pool_boxes is not a BoxList. + c) pool_boxes does not have a scores and classes field. + """ + if not 0.0 <= nms_iou_thresh <= 1.0: + raise ValueError('nms_iou_thresh must be between 0 and 1') + if not 0.0 <= voting_iou_thresh <= 1.0: + raise ValueError('voting_iou_thresh must be between 0 and 1') + if not isinstance(pool_boxes, box_list.BoxList): + raise ValueError('pool_boxes must be a BoxList') + if not pool_boxes.has_field('scores'): + raise ValueError('pool_boxes must have a \'scores\' field') + if not pool_boxes.has_field('classes'): + raise ValueError('pool_boxes must have a \'classes\' field') + + refined_boxes = [] + for i in range(num_classes): + boxes_class = filter_field_value_equals(pool_boxes, 'classes', i) + refined_boxes_class = refine_boxes(boxes_class, nms_iou_thresh, + nms_max_detections, voting_iou_thresh) + refined_boxes.append(refined_boxes_class) + return sort_by_field(concatenate(refined_boxes), 'scores') + + +def refine_boxes(pool_boxes, + nms_iou_thresh, + nms_max_detections, + voting_iou_thresh=0.5): + """Refines a pool of boxes using non max suppression and box voting. + + Args: + pool_boxes: (BoxList) A collection of boxes to be refined. pool_boxes must + have a rank 1 'scores' field. + nms_iou_thresh: (float scalar) iou threshold for non max suppression (NMS). + nms_max_detections: (int scalar) maximum output size for NMS. + voting_iou_thresh: (float scalar) iou threshold for box voting. + + Returns: + BoxList of refined boxes. + + Raises: + ValueError: if + a) nms_iou_thresh or voting_iou_thresh is not in [0, 1]. + b) pool_boxes is not a BoxList. + c) pool_boxes does not have a scores field. + """ + if not 0.0 <= nms_iou_thresh <= 1.0: + raise ValueError('nms_iou_thresh must be between 0 and 1') + if not 0.0 <= voting_iou_thresh <= 1.0: + raise ValueError('voting_iou_thresh must be between 0 and 1') + if not isinstance(pool_boxes, box_list.BoxList): + raise ValueError('pool_boxes must be a BoxList') + if not pool_boxes.has_field('scores'): + raise ValueError('pool_boxes must have a \'scores\' field') + + nms_boxes = non_max_suppression( + pool_boxes, nms_iou_thresh, nms_max_detections) + return box_voting(nms_boxes, pool_boxes, voting_iou_thresh) + + +def box_voting(selected_boxes, pool_boxes, iou_thresh=0.5): + """Performs box voting as described in S. Gidaris and N. Komodakis, ICCV 2015. + + Performs box voting as described in 'Object detection via a multi-region & + semantic segmentation-aware CNN model', Gidaris and Komodakis, ICCV 2015. For + each box 'B' in selected_boxes, we find the set 'S' of boxes in pool_boxes + with iou overlap >= iou_thresh. The location of B is set to the weighted + average location of boxes in S (scores are used for weighting). And the score + of B is set to the average score of boxes in S. + + Args: + selected_boxes: BoxList containing a subset of boxes in pool_boxes. These + boxes are usually selected from pool_boxes using non max suppression. + pool_boxes: BoxList containing a set of (possibly redundant) boxes. + iou_thresh: (float scalar) iou threshold for matching boxes in + selected_boxes and pool_boxes. + + Returns: + BoxList containing averaged locations and scores for each box in + selected_boxes. + + Raises: + ValueError: if + a) selected_boxes or pool_boxes is not a BoxList. + b) if iou_thresh is not in [0, 1]. + c) pool_boxes does not have a scores field. + """ + if not 0.0 <= iou_thresh <= 1.0: + raise ValueError('iou_thresh must be between 0 and 1') + if not isinstance(selected_boxes, box_list.BoxList): + raise ValueError('selected_boxes must be a BoxList') + if not isinstance(pool_boxes, box_list.BoxList): + raise ValueError('pool_boxes must be a BoxList') + if not pool_boxes.has_field('scores'): + raise ValueError('pool_boxes must have a \'scores\' field') + + iou_ = iou(selected_boxes, pool_boxes) + match_indicator = tf.cast(tf.greater(iou_, iou_thresh), dtype=tf.float32) + num_matches = tf.reduce_sum(match_indicator, 1) + # TODO(kbanoop): Handle the case where some boxes in selected_boxes do not + # match to any boxes in pool_boxes. For such boxes without any matches, we + # should return the original boxes without voting. + match_assert = tf.Assert( + tf.reduce_all(tf.greater(num_matches, 0)), + ['Each box in selected_boxes must match with at least one box ' + 'in pool_boxes.']) + + scores = tf.expand_dims(pool_boxes.get_field('scores'), 1) + scores_assert = tf.Assert( + tf.reduce_all(tf.greater_equal(scores, 0)), + ['Scores must be non negative.']) + + with tf.control_dependencies([scores_assert, match_assert]): + sum_scores = tf.matmul(match_indicator, scores) + averaged_scores = tf.reshape(sum_scores, [-1]) / num_matches + + box_locations = tf.matmul(match_indicator, + pool_boxes.get() * scores) / sum_scores + averaged_boxes = box_list.BoxList(box_locations) + _copy_extra_fields(averaged_boxes, selected_boxes) + averaged_boxes.add_field('scores', averaged_scores) + return averaged_boxes + + +def get_minimal_coverage_box(boxlist, + default_box=None, + scope=None): + """Creates a single bounding box which covers all boxes in the boxlist. + + Args: + boxlist: A Boxlist. + default_box: A [1, 4] float32 tensor. If no boxes are present in `boxlist`, + this default box will be returned. If None, will use a default box of + [[0., 0., 1., 1.]]. + scope: Name scope. + + Returns: + A [1, 4] float32 tensor with a bounding box that tightly covers all the + boxes in the box list. If the boxlist does not contain any boxes, the + default box is returned. + """ + with tf.name_scope(scope, 'CreateCoverageBox'): + num_boxes = boxlist.num_boxes() + + def coverage_box(bboxes): + y_min, x_min, y_max, x_max = tf.split( + value=bboxes, num_or_size_splits=4, axis=1) + y_min_coverage = tf.reduce_min(y_min, axis=0) + x_min_coverage = tf.reduce_min(x_min, axis=0) + y_max_coverage = tf.reduce_max(y_max, axis=0) + x_max_coverage = tf.reduce_max(x_max, axis=0) + return tf.stack( + [y_min_coverage, x_min_coverage, y_max_coverage, x_max_coverage], + axis=1) + + default_box = default_box or tf.constant([[0., 0., 1., 1.]]) + return tf.cond( + tf.greater_equal(num_boxes, 1), + true_fn=lambda: coverage_box(boxlist.get()), + false_fn=lambda: default_box) + + +def sample_boxes_by_jittering(boxlist, + num_boxes_to_sample, + stddev=0.1, + scope=None): + """Samples num_boxes_to_sample boxes by jittering around boxlist boxes. + + It is possible that this function might generate boxes with size 0. The larger + the stddev, this is more probable. For a small stddev of 0.1 this probability + is very small. + + Args: + boxlist: A boxlist containing N boxes in normalized coordinates. + num_boxes_to_sample: A positive integer containing the number of boxes to + sample. + stddev: Standard deviation. This is used to draw random offsets for the + box corners from a normal distribution. The offset is multiplied by the + box size so will be larger in terms of pixels for larger boxes. + scope: Name scope. + + Returns: + sampled_boxlist: A boxlist containing num_boxes_to_sample boxes in + normalized coordinates. + """ + with tf.name_scope(scope, 'SampleBoxesByJittering'): + num_boxes = boxlist.num_boxes() + box_indices = tf.random_uniform( + [num_boxes_to_sample], + minval=0, + maxval=num_boxes, + dtype=tf.int32) + sampled_boxes = tf.gather(boxlist.get(), box_indices) + sampled_boxes_height = sampled_boxes[:, 2] - sampled_boxes[:, 0] + sampled_boxes_width = sampled_boxes[:, 3] - sampled_boxes[:, 1] + rand_miny_gaussian = tf.random_normal([num_boxes_to_sample], stddev=stddev) + rand_minx_gaussian = tf.random_normal([num_boxes_to_sample], stddev=stddev) + rand_maxy_gaussian = tf.random_normal([num_boxes_to_sample], stddev=stddev) + rand_maxx_gaussian = tf.random_normal([num_boxes_to_sample], stddev=stddev) + miny = rand_miny_gaussian * sampled_boxes_height + sampled_boxes[:, 0] + minx = rand_minx_gaussian * sampled_boxes_width + sampled_boxes[:, 1] + maxy = rand_maxy_gaussian * sampled_boxes_height + sampled_boxes[:, 2] + maxx = rand_maxx_gaussian * sampled_boxes_width + sampled_boxes[:, 3] + maxy = tf.maximum(miny, maxy) + maxx = tf.maximum(minx, maxx) + sampled_boxes = tf.stack([miny, minx, maxy, maxx], axis=1) + sampled_boxes = tf.maximum(tf.minimum(sampled_boxes, 1.0), 0.0) + return box_list.BoxList(sampled_boxes) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/faster_rcnn_box_coder.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/faster_rcnn_box_coder.py new file mode 100644 index 0000000..2472f57 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/faster_rcnn_box_coder.py @@ -0,0 +1,118 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Faster RCNN box coder. + +Faster RCNN box coder follows the coding schema described below: + ty = (y - ya) / ha + tx = (x - xa) / wa + th = log(h / ha) + tw = log(w / wa) + where x, y, w, h denote the box's center coordinates, width and height + respectively. Similarly, xa, ya, wa, ha denote the anchor's center + coordinates, width and height. tx, ty, tw and th denote the anchor-encoded + center, width and height respectively. + + See http://arxiv.org/abs/1506.01497 for details. +""" + +import tensorflow.compat.v2 as tf + +from official.vision.detection.utils.object_detection import box_coder +from official.vision.detection.utils.object_detection import box_list + +EPSILON = 1e-8 + + +class FasterRcnnBoxCoder(box_coder.BoxCoder): + """Faster RCNN box coder.""" + + def __init__(self, scale_factors=None): + """Constructor for FasterRcnnBoxCoder. + + Args: + scale_factors: List of 4 positive scalars to scale ty, tx, th and tw. + If set to None, does not perform scaling. For Faster RCNN, + the open-source implementation recommends using [10.0, 10.0, 5.0, 5.0]. + """ + if scale_factors: + assert len(scale_factors) == 4 + for scalar in scale_factors: + assert scalar > 0 + self._scale_factors = scale_factors + + @property + def code_size(self): + return 4 + + def _encode(self, boxes, anchors): + """Encode a box collection with respect to anchor collection. + + Args: + boxes: BoxList holding N boxes to be encoded. + anchors: BoxList of anchors. + + Returns: + a tensor representing N anchor-encoded boxes of the format + [ty, tx, th, tw]. + """ + # Convert anchors to the center coordinate representation. + ycenter_a, xcenter_a, ha, wa = anchors.get_center_coordinates_and_sizes() + ycenter, xcenter, h, w = boxes.get_center_coordinates_and_sizes() + # Avoid NaN in division and log below. + ha += EPSILON + wa += EPSILON + h += EPSILON + w += EPSILON + + tx = (xcenter - xcenter_a) / wa + ty = (ycenter - ycenter_a) / ha + tw = tf.math.log(w / wa) + th = tf.math.log(h / ha) + # Scales location targets as used in paper for joint training. + if self._scale_factors: + ty *= self._scale_factors[0] + tx *= self._scale_factors[1] + th *= self._scale_factors[2] + tw *= self._scale_factors[3] + return tf.transpose(a=tf.stack([ty, tx, th, tw])) + + def _decode(self, rel_codes, anchors): + """Decode relative codes to boxes. + + Args: + rel_codes: a tensor representing N anchor-encoded boxes. + anchors: BoxList of anchors. + + Returns: + boxes: BoxList holding N bounding boxes. + """ + ycenter_a, xcenter_a, ha, wa = anchors.get_center_coordinates_and_sizes() + + ty, tx, th, tw = tf.unstack(tf.transpose(a=rel_codes)) + if self._scale_factors: + ty /= self._scale_factors[0] + tx /= self._scale_factors[1] + th /= self._scale_factors[2] + tw /= self._scale_factors[3] + w = tf.exp(tw) * wa + h = tf.exp(th) * ha + ycenter = ty * ha + ycenter_a + xcenter = tx * wa + xcenter_a + ymin = ycenter - h / 2. + xmin = xcenter - w / 2. + ymax = ycenter + h / 2. + xmax = xcenter + w / 2. + return box_list.BoxList(tf.transpose(a=tf.stack([ymin, xmin, ymax, xmax]))) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/matcher.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/matcher.py new file mode 100644 index 0000000..9effb71 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/matcher.py @@ -0,0 +1,243 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Matcher interface and Match class. + +This module defines the Matcher interface and the Match object. The job of the +matcher is to match row and column indices based on the similarity matrix and +other optional parameters. Each column is matched to at most one row. There +are three possibilities for the matching: + +1) match: A column matches a row. +2) no_match: A column does not match any row. +3) ignore: A column that is neither 'match' nor no_match. + +The ignore case is regularly encountered in object detection: when an anchor has +a relatively small overlap with a ground-truth box, one neither wants to +consider this box a positive example (match) nor a negative example (no match). + +The Match class is used to store the match results and it provides simple apis +to query the results. +""" +from abc import ABCMeta +from abc import abstractmethod + +import tensorflow.compat.v2 as tf + + +class Match(object): + """Class to store results from the matcher. + + This class is used to store the results from the matcher. It provides + convenient methods to query the matching results. + """ + + def __init__(self, match_results): + """Constructs a Match object. + + Args: + match_results: Integer tensor of shape [N] with (1) match_results[i]>=0, + meaning that column i is matched with row match_results[i]. + (2) match_results[i]=-1, meaning that column i is not matched. + (3) match_results[i]=-2, meaning that column i is ignored. + + Raises: + ValueError: if match_results does not have rank 1 or is not an + integer int32 scalar tensor + """ + if match_results.shape.ndims != 1: + raise ValueError('match_results should have rank 1') + if match_results.dtype != tf.int32: + raise ValueError('match_results should be an int32 or int64 scalar ' + 'tensor') + self._match_results = match_results + + @property + def match_results(self): + """The accessor for match results. + + Returns: + the tensor which encodes the match results. + """ + return self._match_results + + def matched_column_indices(self): + """Returns column indices that match to some row. + + The indices returned by this op are always sorted in increasing order. + + Returns: + column_indices: int32 tensor of shape [K] with column indices. + """ + return self._reshape_and_cast(tf.where(tf.greater(self._match_results, -1))) + + def matched_column_indicator(self): + """Returns column indices that are matched. + + Returns: + column_indices: int32 tensor of shape [K] with column indices. + """ + return tf.greater_equal(self._match_results, 0) + + def num_matched_columns(self): + """Returns number (int32 scalar tensor) of matched columns.""" + return tf.size(input=self.matched_column_indices()) + + def unmatched_column_indices(self): + """Returns column indices that do not match any row. + + The indices returned by this op are always sorted in increasing order. + + Returns: + column_indices: int32 tensor of shape [K] with column indices. + """ + return self._reshape_and_cast(tf.where(tf.equal(self._match_results, -1))) + + def unmatched_column_indicator(self): + """Returns column indices that are unmatched. + + Returns: + column_indices: int32 tensor of shape [K] with column indices. + """ + return tf.equal(self._match_results, -1) + + def num_unmatched_columns(self): + """Returns number (int32 scalar tensor) of unmatched columns.""" + return tf.size(input=self.unmatched_column_indices()) + + def ignored_column_indices(self): + """Returns column indices that are ignored (neither Matched nor Unmatched). + + The indices returned by this op are always sorted in increasing order. + + Returns: + column_indices: int32 tensor of shape [K] with column indices. + """ + return self._reshape_and_cast(tf.where(self.ignored_column_indicator())) + + def ignored_column_indicator(self): + """Returns boolean column indicator where True means the colum is ignored. + + Returns: + column_indicator: boolean vector which is True for all ignored column + indices. + """ + return tf.equal(self._match_results, -2) + + def num_ignored_columns(self): + """Returns number (int32 scalar tensor) of matched columns.""" + return tf.size(input=self.ignored_column_indices()) + + def unmatched_or_ignored_column_indices(self): + """Returns column indices that are unmatched or ignored. + + The indices returned by this op are always sorted in increasing order. + + Returns: + column_indices: int32 tensor of shape [K] with column indices. + """ + return self._reshape_and_cast(tf.where(tf.greater(0, self._match_results))) + + def matched_row_indices(self): + """Returns row indices that match some column. + + The indices returned by this op are ordered so as to be in correspondence + with the output of matched_column_indicator(). For example if + self.matched_column_indicator() is [0,2], and self.matched_row_indices() is + [7, 3], then we know that column 0 was matched to row 7 and column 2 was + matched to row 3. + + Returns: + row_indices: int32 tensor of shape [K] with row indices. + """ + return self._reshape_and_cast( + tf.gather(self._match_results, self.matched_column_indices())) + + def _reshape_and_cast(self, t): + return tf.cast(tf.reshape(t, [-1]), tf.int32) + + def gather_based_on_match(self, input_tensor, unmatched_value, + ignored_value): + """Gathers elements from `input_tensor` based on match results. + + For columns that are matched to a row, gathered_tensor[col] is set to + input_tensor[match_results[col]]. For columns that are unmatched, + gathered_tensor[col] is set to unmatched_value. Finally, for columns that + are ignored gathered_tensor[col] is set to ignored_value. + + Note that the input_tensor.shape[1:] must match with unmatched_value.shape + and ignored_value.shape + + Args: + input_tensor: Tensor to gather values from. + unmatched_value: Constant tensor value for unmatched columns. + ignored_value: Constant tensor value for ignored columns. + + Returns: + gathered_tensor: A tensor containing values gathered from input_tensor. + The shape of the gathered tensor is [match_results.shape[0]] + + input_tensor.shape[1:]. + """ + input_tensor = tf.concat([tf.stack([ignored_value, unmatched_value]), + input_tensor], axis=0) + gather_indices = tf.maximum(self.match_results + 2, 0) + gathered_tensor = tf.gather(input_tensor, gather_indices) + return gathered_tensor + + +class Matcher(object): + """Abstract base class for matcher. + """ + __metaclass__ = ABCMeta + + def match(self, similarity_matrix, scope=None, **params): + """Computes matches among row and column indices and returns the result. + + Computes matches among the row and column indices based on the similarity + matrix and optional arguments. + + Args: + similarity_matrix: Float tensor of shape [N, M] with pairwise similarity + where higher value means more similar. + scope: Op scope name. Defaults to 'Match' if None. + **params: Additional keyword arguments for specific implementations of + the Matcher. + + Returns: + A Match object with the results of matching. + """ + if not scope: + scope = 'Match' + with tf.name_scope(scope) as scope: + return Match(self._match(similarity_matrix, **params)) + + @abstractmethod + def _match(self, similarity_matrix, **params): + """Method to be overridden by implementations. + + Args: + similarity_matrix: Float tensor of shape [N, M] with pairwise similarity + where higher value means more similar. + **params: Additional keyword arguments for specific implementations of + the Matcher. + + Returns: + match_results: Integer tensor of shape [M]: match_results[i]>=0 means + that column i is matched to row match_results[i], match_results[i]=-1 + means that the column is not matched. match_results[i]=-2 means that + the column is ignored (usually this happens when there is a very weak + match which one neither wants as positive nor negative example). + """ + pass diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/minibatch_sampler.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/minibatch_sampler.py new file mode 100644 index 0000000..1e92801 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/minibatch_sampler.py @@ -0,0 +1,93 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Base minibatch sampler module. + +The job of the minibatch_sampler is to subsample a minibatch based on some +criterion. + +The main function call is: + subsample(indicator, batch_size, **params). +Indicator is a 1d boolean tensor where True denotes which examples can be +sampled. It returns a boolean indicator where True denotes an example has been +sampled.. + +Subclasses should implement the Subsample function and can make use of the +@staticmethod SubsampleIndicator. + +This is originally implemented in TensorFlow Object Detection API. +""" + +from abc import ABCMeta +from abc import abstractmethod + +import tensorflow.compat.v2 as tf + +from official.vision.detection.utils.object_detection import ops + + +class MinibatchSampler(object): + """Abstract base class for subsampling minibatches.""" + __metaclass__ = ABCMeta + + def __init__(self): + """Constructs a minibatch sampler.""" + pass + + @abstractmethod + def subsample(self, indicator, batch_size, **params): + """Returns subsample of entries in indicator. + + Args: + indicator: boolean tensor of shape [N] whose True entries can be sampled. + batch_size: desired batch size. + **params: additional keyword arguments for specific implementations of + the MinibatchSampler. + + Returns: + sample_indicator: boolean tensor of shape [N] whose True entries have been + sampled. If sum(indicator) >= batch_size, sum(is_sampled) = batch_size + """ + pass + + @staticmethod + def subsample_indicator(indicator, num_samples): + """Subsample indicator vector. + + Given a boolean indicator vector with M elements set to `True`, the function + assigns all but `num_samples` of these previously `True` elements to + `False`. If `num_samples` is greater than M, the original indicator vector + is returned. + + Args: + indicator: a 1-dimensional boolean tensor indicating which elements + are allowed to be sampled and which are not. + num_samples: int32 scalar tensor + + Returns: + a boolean tensor with the same shape as input (indicator) tensor + """ + indices = tf.where(indicator) + indices = tf.random.shuffle(indices) + indices = tf.reshape(indices, [-1]) + + num_samples = tf.minimum(tf.size(input=indices), num_samples) + selected_indices = tf.slice(indices, [0], tf.reshape(num_samples, [1])) + + selected_indicator = ops.indices_to_dense_vector( + selected_indices, + tf.shape(input=indicator)[0]) + + return tf.equal(selected_indicator, 1) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/ops.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/ops.py new file mode 100644 index 0000000..683d856 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/ops.py @@ -0,0 +1,82 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""A module for helper tensorflow ops. + +This is originally implemented in TensorFlow Object Detection API. +""" + +import tensorflow.compat.v2 as tf + +from official.vision.detection.utils.object_detection import shape_utils + + +def indices_to_dense_vector(indices, + size, + indices_value=1., + default_value=0, + dtype=tf.float32): + """Creates dense vector with indices set to specific value and rest to zeros. + + This function exists because it is unclear if it is safe to use + tf.sparse_to_dense(indices, [size], 1, validate_indices=False) + with indices which are not ordered. + This function accepts a dynamic size (e.g. tf.shape(tensor)[0]) + + Args: + indices: 1d Tensor with integer indices which are to be set to + indices_values. + size: scalar with size (integer) of output Tensor. + indices_value: values of elements specified by indices in the output vector + default_value: values of other elements in the output vector. + dtype: data type. + + Returns: + dense 1D Tensor of shape [size] with indices set to indices_values and the + rest set to default_value. + """ + size = tf.cast(size, dtype=tf.int32) + zeros = tf.ones([size], dtype=dtype) * default_value + values = tf.ones_like(indices, dtype=dtype) * indices_value + + return tf.dynamic_stitch( + [tf.range(size), tf.cast(indices, dtype=tf.int32)], [zeros, values]) + + +def matmul_gather_on_zeroth_axis(params, indices, scope=None): + """Matrix multiplication based implementation of tf.gather on zeroth axis. + + TODO(rathodv, jonathanhuang): enable sparse matmul option. + + Args: + params: A float32 Tensor. The tensor from which to gather values. + Must be at least rank 1. + indices: A Tensor. Must be one of the following types: int32, int64. + Must be in range [0, params.shape[0]) + scope: A name for the operation (optional). + + Returns: + A Tensor. Has the same type as params. Values from params gathered + from indices given by indices, with shape indices.shape + params.shape[1:]. + """ + scope = scope or 'MatMulGather' + with tf.name_scope(scope): + params_shape = shape_utils.combined_static_and_dynamic_shape(params) + indices_shape = shape_utils.combined_static_and_dynamic_shape(indices) + params2d = tf.reshape(params, [params_shape[0], -1]) + indicator_matrix = tf.one_hot(indices, params_shape[0]) + gathered_result_flattened = tf.matmul(indicator_matrix, params2d) + return tf.reshape(gathered_result_flattened, + tf.stack(indices_shape + params_shape[1:])) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/preprocessor.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/preprocessor.py new file mode 100644 index 0000000..a37f391 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/preprocessor.py @@ -0,0 +1,525 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Preprocess images and bounding boxes for detection. + +We perform two sets of operations in preprocessing stage: +(a) operations that are applied to both training and testing data, +(b) operations that are applied only to training data for the purpose of + data augmentation. + +A preprocessing function receives a set of inputs, +e.g. an image and bounding boxes, +performs an operation on them, and returns them. +Some examples are: randomly cropping the image, randomly mirroring the image, + randomly changing the brightness, contrast, hue and + randomly jittering the bounding boxes. + +The image is a rank 4 tensor: [1, height, width, channels] with +dtype=tf.float32. The groundtruth_boxes is a rank 2 tensor: [N, 4] where +in each row there is a box with [ymin xmin ymax xmax]. +Boxes are in normalized coordinates meaning +their coordinate values range in [0, 1] + +Important Note: In tensor_dict, images is a rank 4 tensor, but preprocessing +functions receive a rank 3 tensor for processing the image. Thus, inside the +preprocess function we squeeze the image to become a rank 3 tensor and then +we pass it to the functions. At the end of the preprocess we expand the image +back to rank 4. +""" + +import tensorflow.compat.v2 as tf + +import numpy as np + +from official.vision.detection.utils.object_detection import box_list + + +def _flip_boxes_left_right(boxes): + """Left-right flip the boxes. + + Args: + boxes: rank 2 float32 tensor containing the bounding boxes -> [N, 4]. + Boxes are in normalized form meaning their coordinates vary + between [0, 1]. + Each row is in the form of [ymin, xmin, ymax, xmax]. + + Returns: + Flipped boxes. + """ + ymin, xmin, ymax, xmax = tf.split(value=boxes, num_or_size_splits=4, axis=1) + flipped_xmin = tf.subtract(1.0, xmax) + flipped_xmax = tf.subtract(1.0, xmin) + flipped_boxes = tf.concat([ymin, flipped_xmin, ymax, flipped_xmax], 1) + return flipped_boxes + + +def _flip_masks_left_right(masks): + """Left-right flip masks. + + Args: + masks: rank 3 float32 tensor with shape + [num_instances, height, width] representing instance masks. + + Returns: + flipped masks: rank 3 float32 tensor with shape + [num_instances, height, width] representing instance masks. + """ + return masks[:, :, ::-1] + + +def keypoint_flip_horizontal(keypoints, flip_point, flip_permutation, + scope=None): + """Flips the keypoints horizontally around the flip_point. + + This operation flips the x coordinate for each keypoint around the flip_point + and also permutes the keypoints in a manner specified by flip_permutation. + + Args: + keypoints: a tensor of shape [num_instances, num_keypoints, 2] + flip_point: (float) scalar tensor representing the x coordinate to flip the + keypoints around. + flip_permutation: rank 1 int32 tensor containing the keypoint flip + permutation. This specifies the mapping from original keypoint indices + to the flipped keypoint indices. This is used primarily for keypoints + that are not reflection invariant. E.g. Suppose there are 3 keypoints + representing ['head', 'right_eye', 'left_eye'], then a logical choice for + flip_permutation might be [0, 2, 1] since we want to swap the 'left_eye' + and 'right_eye' after a horizontal flip. + scope: name scope. + + Returns: + new_keypoints: a tensor of shape [num_instances, num_keypoints, 2] + """ + if not scope: + scope = 'FlipHorizontal' + with tf.name_scope(scope): + keypoints = tf.transpose(a=keypoints, perm=[1, 0, 2]) + keypoints = tf.gather(keypoints, flip_permutation) + v, u = tf.split(value=keypoints, num_or_size_splits=2, axis=2) + u = flip_point * 2.0 - u + new_keypoints = tf.concat([v, u], 2) + new_keypoints = tf.transpose(a=new_keypoints, perm=[1, 0, 2]) + return new_keypoints + + +def keypoint_change_coordinate_frame(keypoints, window, scope=None): + """Changes coordinate frame of the keypoints to be relative to window's frame. + + Given a window of the form [y_min, x_min, y_max, x_max], changes keypoint + coordinates from keypoints of shape [num_instances, num_keypoints, 2] + to be relative to this window. + + An example use case is data augmentation: where we are given groundtruth + keypoints and would like to randomly crop the image to some window. In this + case we need to change the coordinate frame of each groundtruth keypoint to be + relative to this new window. + + Args: + keypoints: a tensor of shape [num_instances, num_keypoints, 2] + window: a tensor of shape [4] representing the [y_min, x_min, y_max, x_max] + window we should change the coordinate frame to. + scope: name scope. + + Returns: + new_keypoints: a tensor of shape [num_instances, num_keypoints, 2] + """ + if not scope: + scope = 'ChangeCoordinateFrame' + with tf.name_scope(scope): + win_height = window[2] - window[0] + win_width = window[3] - window[1] + new_keypoints = box_list_ops.scale(keypoints - [window[0], window[1]], + 1.0 / win_height, 1.0 / win_width) + return new_keypoints + + +def keypoint_prune_outside_window(keypoints, window, scope=None): + """Prunes keypoints that fall outside a given window. + + This function replaces keypoints that fall outside the given window with nan. + See also clip_to_window which clips any keypoints that fall outside the given + window. + + Args: + keypoints: a tensor of shape [num_instances, num_keypoints, 2] + window: a tensor of shape [4] representing the [y_min, x_min, y_max, x_max] + window outside of which the op should prune the keypoints. + scope: name scope. + + Returns: + new_keypoints: a tensor of shape [num_instances, num_keypoints, 2] + """ + if not scope: + scope = 'PruneOutsideWindow' + with tf.name_scope(scope): + y, x = tf.split(value=keypoints, num_or_size_splits=2, axis=2) + win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window) + + valid_indices = tf.logical_and( + tf.logical_and(y >= win_y_min, y <= win_y_max), + tf.logical_and(x >= win_x_min, x <= win_x_max)) + + new_y = tf.where(valid_indices, y, np.nan * tf.ones_like(y)) + new_x = tf.where(valid_indices, x, np.nan * tf.ones_like(x)) + new_keypoints = tf.concat([new_y, new_x], 2) + + return new_keypoints + + +def random_horizontal_flip(image, + boxes=None, + masks=None, + keypoints=None, + keypoint_flip_permutation=None, + seed=None): + """Randomly flips the image and detections horizontally. + + The probability of flipping the image is 50%. + + Args: + image: rank 3 float32 tensor with shape [height, width, channels]. + boxes: (optional) rank 2 float32 tensor with shape [N, 4] + containing the bounding boxes. + Boxes are in normalized form meaning their coordinates vary + between [0, 1]. + Each row is in the form of [ymin, xmin, ymax, xmax]. + masks: (optional) rank 3 float32 tensor with shape + [num_instances, height, width] containing instance masks. The masks + are of the same height, width as the input `image`. + keypoints: (optional) rank 3 float32 tensor with shape + [num_instances, num_keypoints, 2]. The keypoints are in y-x + normalized coordinates. + keypoint_flip_permutation: rank 1 int32 tensor containing the keypoint flip + permutation. + seed: random seed + + Returns: + image: image which is the same shape as input image. + + If boxes, masks, keypoints, and keypoint_flip_permutation are not None, + the function also returns the following tensors. + + boxes: rank 2 float32 tensor containing the bounding boxes -> [N, 4]. + Boxes are in normalized form meaning their coordinates vary + between [0, 1]. + masks: rank 3 float32 tensor with shape [num_instances, height, width] + containing instance masks. + keypoints: rank 3 float32 tensor with shape + [num_instances, num_keypoints, 2] + + Raises: + ValueError: if keypoints are provided but keypoint_flip_permutation is not. + """ + + def _flip_image(image): + # flip image + image_flipped = tf.image.flip_left_right(image) + return image_flipped + + if keypoints is not None and keypoint_flip_permutation is None: + raise ValueError( + 'keypoints are provided but keypoints_flip_permutation is not provided') + + with tf.name_scope('RandomHorizontalFlip'): + result = [] + # random variable defining whether to do flip or not + do_a_flip_random = tf.greater(tf.random.uniform([], seed=seed), 0.5) + + # flip image + image = tf.cond( + pred=do_a_flip_random, + true_fn=lambda: _flip_image(image), + false_fn=lambda: image) + result.append(image) + + # flip boxes + if boxes is not None: + boxes = tf.cond( + pred=do_a_flip_random, + true_fn=lambda: _flip_boxes_left_right(boxes), + false_fn=lambda: boxes) + result.append(boxes) + + # flip masks + if masks is not None: + masks = tf.cond( + pred=do_a_flip_random, + true_fn=lambda: _flip_masks_left_right(masks), + false_fn=lambda: masks) + result.append(masks) + + # flip keypoints + if keypoints is not None and keypoint_flip_permutation is not None: + permutation = keypoint_flip_permutation + keypoints = tf.cond( + pred=do_a_flip_random, + true_fn=lambda: keypoint_flip_horizontal(keypoints, 0.5, permutation), + false_fn=lambda: keypoints) + result.append(keypoints) + + return tuple(result) + + +def _compute_new_static_size(image, min_dimension, max_dimension): + """Compute new static shape for resize_to_range method.""" + image_shape = image.get_shape().as_list() + orig_height = image_shape[0] + orig_width = image_shape[1] + num_channels = image_shape[2] + orig_min_dim = min(orig_height, orig_width) + # Calculates the larger of the possible sizes + large_scale_factor = min_dimension / float(orig_min_dim) + # Scaling orig_(height|width) by large_scale_factor will make the smaller + # dimension equal to min_dimension, save for floating point rounding errors. + # For reasonably-sized images, taking the nearest integer will reliably + # eliminate this error. + large_height = int(round(orig_height * large_scale_factor)) + large_width = int(round(orig_width * large_scale_factor)) + large_size = [large_height, large_width] + if max_dimension: + # Calculates the smaller of the possible sizes, use that if the larger + # is too big. + orig_max_dim = max(orig_height, orig_width) + small_scale_factor = max_dimension / float(orig_max_dim) + # Scaling orig_(height|width) by small_scale_factor will make the larger + # dimension equal to max_dimension, save for floating point rounding + # errors. For reasonably-sized images, taking the nearest integer will + # reliably eliminate this error. + small_height = int(round(orig_height * small_scale_factor)) + small_width = int(round(orig_width * small_scale_factor)) + small_size = [small_height, small_width] + new_size = large_size + if max(large_size) > max_dimension: + new_size = small_size + else: + new_size = large_size + return tf.constant(new_size + [num_channels]) + + +def _compute_new_dynamic_size(image, min_dimension, max_dimension): + """Compute new dynamic shape for resize_to_range method.""" + image_shape = tf.shape(input=image) + orig_height = tf.cast(image_shape[0], dtype=tf.float32) + orig_width = tf.cast(image_shape[1], dtype=tf.float32) + num_channels = image_shape[2] + orig_min_dim = tf.minimum(orig_height, orig_width) + # Calculates the larger of the possible sizes + min_dimension = tf.constant(min_dimension, dtype=tf.float32) + large_scale_factor = min_dimension / orig_min_dim + # Scaling orig_(height|width) by large_scale_factor will make the smaller + # dimension equal to min_dimension, save for floating point rounding errors. + # For reasonably-sized images, taking the nearest integer will reliably + # eliminate this error. + large_height = tf.cast( + tf.round(orig_height * large_scale_factor), dtype=tf.int32) + large_width = tf.cast( + tf.round(orig_width * large_scale_factor), dtype=tf.int32) + large_size = tf.stack([large_height, large_width]) + if max_dimension: + # Calculates the smaller of the possible sizes, use that if the larger + # is too big. + orig_max_dim = tf.maximum(orig_height, orig_width) + max_dimension = tf.constant(max_dimension, dtype=tf.float32) + small_scale_factor = max_dimension / orig_max_dim + # Scaling orig_(height|width) by small_scale_factor will make the larger + # dimension equal to max_dimension, save for floating point rounding + # errors. For reasonably-sized images, taking the nearest integer will + # reliably eliminate this error. + small_height = tf.cast( + tf.round(orig_height * small_scale_factor), dtype=tf.int32) + small_width = tf.cast( + tf.round(orig_width * small_scale_factor), dtype=tf.int32) + small_size = tf.stack([small_height, small_width]) + new_size = tf.cond( + pred=tf.cast(tf.reduce_max(input_tensor=large_size), dtype=tf.float32) > + max_dimension, + true_fn=lambda: small_size, + false_fn=lambda: large_size) + else: + new_size = large_size + return tf.stack(tf.unstack(new_size) + [num_channels]) + + +def resize_to_range(image, + masks=None, + min_dimension=None, + max_dimension=None, + method=tf.image.ResizeMethod.BILINEAR, + align_corners=False, + pad_to_max_dimension=False): + """Resizes an image so its dimensions are within the provided value. + + The output size can be described by two cases: + 1. If the image can be rescaled so its minimum dimension is equal to the + provided value without the other dimension exceeding max_dimension, + then do so. + 2. Otherwise, resize so the largest dimension is equal to max_dimension. + + Args: + image: A 3D tensor of shape [height, width, channels] + masks: (optional) rank 3 float32 tensor with shape + [num_instances, height, width] containing instance masks. + min_dimension: (optional) (scalar) desired size of the smaller image + dimension. + max_dimension: (optional) (scalar) maximum allowed size + of the larger image dimension. + method: (optional) interpolation method used in resizing. Defaults to + BILINEAR. + align_corners: bool. If true, exactly align all 4 corners of the input + and output. Defaults to False. + pad_to_max_dimension: Whether to resize the image and pad it with zeros + so the resulting image is of the spatial size + [max_dimension, max_dimension]. If masks are included they are padded + similarly. + + Returns: + Note that the position of the resized_image_shape changes based on whether + masks are present. + resized_image: A 3D tensor of shape [new_height, new_width, channels], + where the image has been resized (with bilinear interpolation) so that + min(new_height, new_width) == min_dimension or + max(new_height, new_width) == max_dimension. + resized_masks: If masks is not None, also outputs masks. A 3D tensor of + shape [num_instances, new_height, new_width]. + resized_image_shape: A 1D tensor of shape [3] containing shape of the + resized image. + + Raises: + ValueError: if the image is not a 3D tensor. + """ + if len(image.get_shape()) != 3: + raise ValueError('Image should be 3D tensor') + + with tf.name_scope('ResizeToRange'): + if image.get_shape().is_fully_defined(): + new_size = _compute_new_static_size(image, min_dimension, max_dimension) + else: + new_size = _compute_new_dynamic_size(image, min_dimension, max_dimension) + new_image = tf.image.resize(image, new_size[:-1], method=method) + + if pad_to_max_dimension: + new_image = tf.image.pad_to_bounding_box( + new_image, 0, 0, max_dimension, max_dimension) + + result = [new_image] + if masks is not None: + new_masks = tf.expand_dims(masks, 3) + new_masks = tf.image.resize( + new_masks, + new_size[:-1], + method=tf.image.ResizeMethod.NEAREST_NEIGHBOR) + new_masks = tf.squeeze(new_masks, 3) + if pad_to_max_dimension: + new_masks = tf.image.pad_to_bounding_box( + new_masks, 0, 0, max_dimension, max_dimension) + result.append(new_masks) + + result.append(new_size) + return result + + +def _copy_extra_fields(boxlist_to_copy_to, boxlist_to_copy_from): + """Copies the extra fields of boxlist_to_copy_from to boxlist_to_copy_to. + + Args: + boxlist_to_copy_to: BoxList to which extra fields are copied. + boxlist_to_copy_from: BoxList from which fields are copied. + + Returns: + boxlist_to_copy_to with extra fields. + """ + for field in boxlist_to_copy_from.get_extra_fields(): + boxlist_to_copy_to.add_field(field, boxlist_to_copy_from.get_field(field)) + return boxlist_to_copy_to + + +def box_list_scale(boxlist, y_scale, x_scale, scope=None): + """scale box coordinates in x and y dimensions. + + Args: + boxlist: BoxList holding N boxes + y_scale: (float) scalar tensor + x_scale: (float) scalar tensor + scope: name scope. + + Returns: + boxlist: BoxList holding N boxes + """ + if not scope: + scope = 'Scale' + with tf.name_scope(scope): + y_scale = tf.cast(y_scale, tf.float32) + x_scale = tf.cast(x_scale, tf.float32) + y_min, x_min, y_max, x_max = tf.split( + value=boxlist.get(), num_or_size_splits=4, axis=1) + y_min = y_scale * y_min + y_max = y_scale * y_max + x_min = x_scale * x_min + x_max = x_scale * x_max + scaled_boxlist = box_list.BoxList( + tf.concat([y_min, x_min, y_max, x_max], 1)) + return _copy_extra_fields(scaled_boxlist, boxlist) + + +def keypoint_scale(keypoints, y_scale, x_scale, scope=None): + """Scales keypoint coordinates in x and y dimensions. + + Args: + keypoints: a tensor of shape [num_instances, num_keypoints, 2] + y_scale: (float) scalar tensor + x_scale: (float) scalar tensor + scope: name scope. + + Returns: + new_keypoints: a tensor of shape [num_instances, num_keypoints, 2] + """ + if not scope: + scope = 'Scale' + with tf.name_scope(scope): + y_scale = tf.cast(y_scale, tf.float32) + x_scale = tf.cast(x_scale, tf.float32) + new_keypoints = keypoints * [[[y_scale, x_scale]]] + return new_keypoints + + +def scale_boxes_to_pixel_coordinates(image, boxes, keypoints=None): + """Scales boxes from normalized to pixel coordinates. + + Args: + image: A 3D float32 tensor of shape [height, width, channels]. + boxes: A 2D float32 tensor of shape [num_boxes, 4] containing the bounding + boxes in normalized coordinates. Each row is of the form + [ymin, xmin, ymax, xmax]. + keypoints: (optional) rank 3 float32 tensor with shape + [num_instances, num_keypoints, 2]. The keypoints are in y-x normalized + coordinates. + + Returns: + image: unchanged input image. + scaled_boxes: a 2D float32 tensor of shape [num_boxes, 4] containing the + bounding boxes in pixel coordinates. + scaled_keypoints: a 3D float32 tensor with shape + [num_instances, num_keypoints, 2] containing the keypoints in pixel + coordinates. + """ + boxlist = box_list.BoxList(boxes) + image_height = tf.shape(input=image)[0] + image_width = tf.shape(input=image)[1] + scaled_boxes = box_list_scale(boxlist, image_height, image_width).get() + result = [image, scaled_boxes] + if keypoints is not None: + scaled_keypoints = keypoint_scale(keypoints, image_height, image_width) + result.append(scaled_keypoints) + return tuple(result) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/region_similarity_calculator.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/region_similarity_calculator.py new file mode 100644 index 0000000..f5f1d69 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/region_similarity_calculator.py @@ -0,0 +1,143 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Region Similarity Calculators for BoxLists. + +Region Similarity Calculators compare a pairwise measure of similarity +between the boxes in two BoxLists. +""" +from abc import ABCMeta +from abc import abstractmethod + +import tensorflow.compat.v2 as tf + + +def area(boxlist, scope=None): + """Computes area of boxes. + + Args: + boxlist: BoxList holding N boxes + scope: name scope. + + Returns: + a tensor with shape [N] representing box areas. + """ + if not scope: + scope = 'Area' + with tf.name_scope(scope): + y_min, x_min, y_max, x_max = tf.split( + value=boxlist.get(), num_or_size_splits=4, axis=1) + return tf.squeeze((y_max - y_min) * (x_max - x_min), [1]) + + +def intersection(boxlist1, boxlist2, scope=None): + """Compute pairwise intersection areas between boxes. + + Args: + boxlist1: BoxList holding N boxes + boxlist2: BoxList holding M boxes + scope: name scope. + + Returns: + a tensor with shape [N, M] representing pairwise intersections + """ + if not scope: + scope = 'Intersection' + with tf.name_scope(scope): + y_min1, x_min1, y_max1, x_max1 = tf.split( + value=boxlist1.get(), num_or_size_splits=4, axis=1) + y_min2, x_min2, y_max2, x_max2 = tf.split( + value=boxlist2.get(), num_or_size_splits=4, axis=1) + all_pairs_min_ymax = tf.minimum(y_max1, tf.transpose(a=y_max2)) + all_pairs_max_ymin = tf.maximum(y_min1, tf.transpose(a=y_min2)) + intersect_heights = tf.maximum(0.0, all_pairs_min_ymax - all_pairs_max_ymin) + all_pairs_min_xmax = tf.minimum(x_max1, tf.transpose(a=x_max2)) + all_pairs_max_xmin = tf.maximum(x_min1, tf.transpose(a=x_min2)) + intersect_widths = tf.maximum(0.0, all_pairs_min_xmax - all_pairs_max_xmin) + return intersect_heights * intersect_widths + + +def iou(boxlist1, boxlist2, scope=None): + """Computes pairwise intersection-over-union between box collections. + + Args: + boxlist1: BoxList holding N boxes + boxlist2: BoxList holding M boxes + scope: name scope. + + Returns: + a tensor with shape [N, M] representing pairwise iou scores. + """ + if not scope: + scope = 'IOU' + with tf.name_scope(scope): + intersections = intersection(boxlist1, boxlist2) + areas1 = area(boxlist1) + areas2 = area(boxlist2) + unions = ( + tf.expand_dims(areas1, 1) + tf.expand_dims(areas2, 0) - intersections) + return tf.where( + tf.equal(intersections, 0.0), tf.zeros_like(intersections), + tf.truediv(intersections, unions)) + + +class RegionSimilarityCalculator(object): + """Abstract base class for region similarity calculator.""" + __metaclass__ = ABCMeta + + def compare(self, boxlist1, boxlist2, scope=None): + """Computes matrix of pairwise similarity between BoxLists. + + This op (to be overriden) computes a measure of pairwise similarity between + the boxes in the given BoxLists. Higher values indicate more similarity. + + Note that this method simply measures similarity and does not explicitly + perform a matching. + + Args: + boxlist1: BoxList holding N boxes. + boxlist2: BoxList holding M boxes. + scope: Op scope name. Defaults to 'Compare' if None. + + Returns: + a (float32) tensor of shape [N, M] with pairwise similarity score. + """ + if not scope: + scope = 'Compare' + with tf.name_scope(scope) as scope: + return self._compare(boxlist1, boxlist2) + + @abstractmethod + def _compare(self, boxlist1, boxlist2): + pass + + +class IouSimilarity(RegionSimilarityCalculator): + """Class to compute similarity based on Intersection over Union (IOU) metric. + + This class computes pairwise similarity between two BoxLists based on IOU. + """ + + def _compare(self, boxlist1, boxlist2): + """Compute pairwise IOU similarity between the two BoxLists. + + Args: + boxlist1: BoxList holding N boxes. + boxlist2: BoxList holding M boxes. + + Returns: + A tensor with shape [N, M] representing pairwise iou scores. + """ + return iou(boxlist1, boxlist2) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/shape_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/shape_utils.py new file mode 100644 index 0000000..bff3bf2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/shape_utils.py @@ -0,0 +1,112 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Utils used to manipulate tensor shapes.""" + +import tensorflow.compat.v2 as tf + + +def assert_shape_equal(shape_a, shape_b): + """Asserts that shape_a and shape_b are equal. + + If the shapes are static, raises a ValueError when the shapes + mismatch. + + If the shapes are dynamic, raises a tf InvalidArgumentError when the shapes + mismatch. + + Args: + shape_a: a list containing shape of the first tensor. + shape_b: a list containing shape of the second tensor. + + Returns: + Either a tf.no_op() when shapes are all static and a tf.assert_equal() op + when the shapes are dynamic. + + Raises: + ValueError: When shapes are both static and unequal. + """ + if (all(isinstance(dim, int) for dim in shape_a) and + all(isinstance(dim, int) for dim in shape_b)): + if shape_a != shape_b: + raise ValueError('Unequal shapes {}, {}'.format(shape_a, shape_b)) + else: return tf.no_op() + else: + return tf.assert_equal(shape_a, shape_b) + + +def combined_static_and_dynamic_shape(tensor): + """Returns a list containing static and dynamic values for the dimensions. + + Returns a list of static and dynamic values for shape dimensions. This is + useful to preserve static shapes when available in reshape operation. + + Args: + tensor: A tensor of any type. + + Returns: + A list of size tensor.shape.ndims containing integers or a scalar tensor. + """ + static_tensor_shape = tensor.shape.as_list() + dynamic_tensor_shape = tf.shape(input=tensor) + combined_shape = [] + for index, dim in enumerate(static_tensor_shape): + if dim is not None: + combined_shape.append(dim) + else: + combined_shape.append(dynamic_tensor_shape[index]) + return combined_shape + + +def pad_or_clip_nd(tensor, output_shape): + """Pad or Clip given tensor to the output shape. + + Args: + tensor: Input tensor to pad or clip. + output_shape: A list of integers / scalar tensors (or None for dynamic dim) + representing the size to pad or clip each dimension of the input tensor. + + Returns: + Input tensor padded and clipped to the output shape. + """ + tensor_shape = tf.shape(input=tensor) + clip_size = [ + tf.where(tensor_shape[i] - shape > 0, shape, -1) + if shape is not None else -1 for i, shape in enumerate(output_shape) + ] + clipped_tensor = tf.slice( + tensor, + begin=tf.zeros(len(clip_size), dtype=tf.int32), + size=clip_size) + + # Pad tensor if the shape of clipped tensor is smaller than the expected + # shape. + clipped_tensor_shape = tf.shape(input=clipped_tensor) + trailing_paddings = [ + shape - clipped_tensor_shape[i] if shape is not None else 0 + for i, shape in enumerate(output_shape) + ] + paddings = tf.stack( + [ + tf.zeros(len(trailing_paddings), dtype=tf.int32), + trailing_paddings + ], + axis=1) + padded_tensor = tf.pad(tensor=clipped_tensor, paddings=paddings) + output_static_shape = [ + dim if not isinstance(dim, tf.Tensor) else None for dim in output_shape + ] + padded_tensor.set_shape(output_static_shape) + return padded_tensor diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/target_assigner.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/target_assigner.py new file mode 100644 index 0000000..b9e20a7 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/target_assigner.py @@ -0,0 +1,314 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Base target assigner module. + +The job of a TargetAssigner is, for a given set of anchors (bounding boxes) and +groundtruth detections (bounding boxes), to assign classification and regression +targets to each anchor as well as weights to each anchor (specifying, e.g., +which anchors should not contribute to training loss). + +It assigns classification/regression targets by performing the following steps: +1) Computing pairwise similarity between anchors and groundtruth boxes using a + provided RegionSimilarity Calculator +2) Computing a matching based on the similarity matrix using a provided Matcher +3) Assigning regression targets based on the matching and a provided BoxCoder +4) Assigning classification targets based on the matching and groundtruth labels + +Note that TargetAssigners only operate on detections from a single +image at a time, so any logic for applying a TargetAssigner to multiple +images must be handled externally. +""" +import tensorflow.compat.v2 as tf + +from official.vision.detection.utils.object_detection import box_list +from official.vision.detection.utils.object_detection import shape_utils + + +KEYPOINTS_FIELD_NAME = 'keypoints' + + +class TargetAssigner(object): + """Target assigner to compute classification and regression targets.""" + + def __init__(self, similarity_calc, matcher, box_coder, + negative_class_weight=1.0, unmatched_cls_target=None): + """Construct Object Detection Target Assigner. + + Args: + similarity_calc: a RegionSimilarityCalculator + matcher: Matcher used to match groundtruth to anchors. + box_coder: BoxCoder used to encode matching groundtruth boxes with + respect to anchors. + negative_class_weight: classification weight to be associated to negative + anchors (default: 1.0). The weight must be in [0., 1.]. + unmatched_cls_target: a float32 tensor with shape [d_1, d_2, ..., d_k] + which is consistent with the classification target for each + anchor (and can be empty for scalar targets). This shape must thus be + compatible with the groundtruth labels that are passed to the "assign" + function (which have shape [num_gt_boxes, d_1, d_2, ..., d_k]). + If set to None, unmatched_cls_target is set to be [0] for each anchor. + + Raises: + ValueError: if similarity_calc is not a RegionSimilarityCalculator or + if matcher is not a Matcher or if box_coder is not a BoxCoder + """ + self._similarity_calc = similarity_calc + self._matcher = matcher + self._box_coder = box_coder + self._negative_class_weight = negative_class_weight + if unmatched_cls_target is None: + self._unmatched_cls_target = tf.constant([0], tf.float32) + else: + self._unmatched_cls_target = unmatched_cls_target + + @property + def box_coder(self): + return self._box_coder + + def assign(self, anchors, groundtruth_boxes, groundtruth_labels=None, + groundtruth_weights=None, **params): + """Assign classification and regression targets to each anchor. + + For a given set of anchors and groundtruth detections, match anchors + to groundtruth_boxes and assign classification and regression targets to + each anchor as well as weights based on the resulting match (specifying, + e.g., which anchors should not contribute to training loss). + + Anchors that are not matched to anything are given a classification target + of self._unmatched_cls_target which can be specified via the constructor. + + Args: + anchors: a BoxList representing N anchors + groundtruth_boxes: a BoxList representing M groundtruth boxes + groundtruth_labels: a tensor of shape [M, d_1, ... d_k] + with labels for each of the ground_truth boxes. The subshape + [d_1, ... d_k] can be empty (corresponding to scalar inputs). When set + to None, groundtruth_labels assumes a binary problem where all + ground_truth boxes get a positive label (of 1). + groundtruth_weights: a float tensor of shape [M] indicating the weight to + assign to all anchors match to a particular groundtruth box. The weights + must be in [0., 1.]. If None, all weights are set to 1. + **params: Additional keyword arguments for specific implementations of + the Matcher. + + Returns: + cls_targets: a float32 tensor with shape [num_anchors, d_1, d_2 ... d_k], + where the subshape [d_1, ..., d_k] is compatible with groundtruth_labels + which has shape [num_gt_boxes, d_1, d_2, ... d_k]. + cls_weights: a float32 tensor with shape [num_anchors] + reg_targets: a float32 tensor with shape [num_anchors, box_code_dimension] + reg_weights: a float32 tensor with shape [num_anchors] + match: a matcher.Match object encoding the match between anchors and + groundtruth boxes, with rows corresponding to groundtruth boxes + and columns corresponding to anchors. + + Raises: + ValueError: if anchors or groundtruth_boxes are not of type + box_list.BoxList + """ + if not isinstance(anchors, box_list.BoxList): + raise ValueError('anchors must be an BoxList') + if not isinstance(groundtruth_boxes, box_list.BoxList): + raise ValueError('groundtruth_boxes must be an BoxList') + + if groundtruth_labels is None: + groundtruth_labels = tf.ones(tf.expand_dims(groundtruth_boxes.num_boxes(), + 0)) + groundtruth_labels = tf.expand_dims(groundtruth_labels, -1) + unmatched_shape_assert = shape_utils.assert_shape_equal( + shape_utils.combined_static_and_dynamic_shape(groundtruth_labels)[1:], + shape_utils.combined_static_and_dynamic_shape( + self._unmatched_cls_target)) + labels_and_box_shapes_assert = shape_utils.assert_shape_equal( + shape_utils.combined_static_and_dynamic_shape( + groundtruth_labels)[:1], + shape_utils.combined_static_and_dynamic_shape( + groundtruth_boxes.get())[:1]) + + if groundtruth_weights is None: + num_gt_boxes = groundtruth_boxes.num_boxes_static() + if not num_gt_boxes: + num_gt_boxes = groundtruth_boxes.num_boxes() + groundtruth_weights = tf.ones([num_gt_boxes], dtype=tf.float32) + with tf.control_dependencies( + [unmatched_shape_assert, labels_and_box_shapes_assert]): + match_quality_matrix = self._similarity_calc.compare(groundtruth_boxes, + anchors) + match = self._matcher.match(match_quality_matrix, **params) + reg_targets = self._create_regression_targets(anchors, + groundtruth_boxes, + match) + cls_targets = self._create_classification_targets(groundtruth_labels, + match) + reg_weights = self._create_regression_weights(match, groundtruth_weights) + cls_weights = self._create_classification_weights(match, + groundtruth_weights) + + num_anchors = anchors.num_boxes_static() + if num_anchors is not None: + reg_targets = self._reset_target_shape(reg_targets, num_anchors) + cls_targets = self._reset_target_shape(cls_targets, num_anchors) + reg_weights = self._reset_target_shape(reg_weights, num_anchors) + cls_weights = self._reset_target_shape(cls_weights, num_anchors) + + return cls_targets, cls_weights, reg_targets, reg_weights, match + + def _reset_target_shape(self, target, num_anchors): + """Sets the static shape of the target. + + Args: + target: the target tensor. Its first dimension will be overwritten. + num_anchors: the number of anchors, which is used to override the target's + first dimension. + + Returns: + A tensor with the shape info filled in. + """ + target_shape = target.get_shape().as_list() + target_shape[0] = num_anchors + target.set_shape(target_shape) + return target + + def _create_regression_targets(self, anchors, groundtruth_boxes, match): + """Returns a regression target for each anchor. + + Args: + anchors: a BoxList representing N anchors + groundtruth_boxes: a BoxList representing M groundtruth_boxes + match: a matcher.Match object + + Returns: + reg_targets: a float32 tensor with shape [N, box_code_dimension] + """ + matched_gt_boxes = match.gather_based_on_match( + groundtruth_boxes.get(), + unmatched_value=tf.zeros(4), + ignored_value=tf.zeros(4)) + matched_gt_boxlist = box_list.BoxList(matched_gt_boxes) + if groundtruth_boxes.has_field(KEYPOINTS_FIELD_NAME): + groundtruth_keypoints = groundtruth_boxes.get_field(KEYPOINTS_FIELD_NAME) + matched_keypoints = match.gather_based_on_match( + groundtruth_keypoints, + unmatched_value=tf.zeros(groundtruth_keypoints.get_shape()[1:]), + ignored_value=tf.zeros(groundtruth_keypoints.get_shape()[1:])) + matched_gt_boxlist.add_field(KEYPOINTS_FIELD_NAME, matched_keypoints) + matched_reg_targets = self._box_coder.encode(matched_gt_boxlist, anchors) + match_results_shape = shape_utils.combined_static_and_dynamic_shape( + match.match_results) + + # Zero out the unmatched and ignored regression targets. + unmatched_ignored_reg_targets = tf.tile( + self._default_regression_target(), [match_results_shape[0], 1]) + matched_anchors_mask = match.matched_column_indicator() + # To broadcast matched_anchors_mask to the same shape as + # matched_reg_targets. + matched_anchors_mask = tf.tile( + tf.expand_dims(matched_anchors_mask, 1), + [1, tf.shape(matched_reg_targets)[1]]) + reg_targets = tf.where(matched_anchors_mask, matched_reg_targets, + unmatched_ignored_reg_targets) + return reg_targets + + def _default_regression_target(self): + """Returns the default target for anchors to regress to. + + Default regression targets are set to zero (though in + this implementation what these targets are set to should + not matter as the regression weight of any box set to + regress to the default target is zero). + + Returns: + default_target: a float32 tensor with shape [1, box_code_dimension] + """ + return tf.constant([self._box_coder.code_size*[0]], tf.float32) + + def _create_classification_targets(self, groundtruth_labels, match): + """Create classification targets for each anchor. + + Assign a classification target of for each anchor to the matching + groundtruth label that is provided by match. Anchors that are not matched + to anything are given the target self._unmatched_cls_target + + Args: + groundtruth_labels: a tensor of shape [num_gt_boxes, d_1, ... d_k] + with labels for each of the ground_truth boxes. The subshape + [d_1, ... d_k] can be empty (corresponding to scalar labels). + match: a matcher.Match object that provides a matching between anchors + and groundtruth boxes. + + Returns: + a float32 tensor with shape [num_anchors, d_1, d_2 ... d_k], where the + subshape [d_1, ..., d_k] is compatible with groundtruth_labels which has + shape [num_gt_boxes, d_1, d_2, ... d_k]. + """ + return match.gather_based_on_match( + groundtruth_labels, + unmatched_value=self._unmatched_cls_target, + ignored_value=self._unmatched_cls_target) + + def _create_regression_weights(self, match, groundtruth_weights): + """Set regression weight for each anchor. + + Only positive anchors are set to contribute to the regression loss, so this + method returns a weight of 1 for every positive anchor and 0 for every + negative anchor. + + Args: + match: a matcher.Match object that provides a matching between anchors + and groundtruth boxes. + groundtruth_weights: a float tensor of shape [M] indicating the weight to + assign to all anchors match to a particular groundtruth box. + + Returns: + a float32 tensor with shape [num_anchors] representing regression weights. + """ + return match.gather_based_on_match( + groundtruth_weights, ignored_value=0., unmatched_value=0.) + + def _create_classification_weights(self, + match, + groundtruth_weights): + """Create classification weights for each anchor. + + Positive (matched) anchors are associated with a weight of + positive_class_weight and negative (unmatched) anchors are associated with + a weight of negative_class_weight. When anchors are ignored, weights are set + to zero. By default, both positive/negative weights are set to 1.0, + but they can be adjusted to handle class imbalance (which is almost always + the case in object detection). + + Args: + match: a matcher.Match object that provides a matching between anchors + and groundtruth boxes. + groundtruth_weights: a float tensor of shape [M] indicating the weight to + assign to all anchors match to a particular groundtruth box. + + Returns: + a float32 tensor with shape [num_anchors] representing classification + weights. + """ + return match.gather_based_on_match( + groundtruth_weights, + ignored_value=0., + unmatched_value=self._negative_class_weight) + + def get_box_coder(self): + """Get BoxCoder of this TargetAssigner. + + Returns: + BoxCoder object. + """ + return self._box_coder diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/visualization_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/visualization_utils.py new file mode 100644 index 0000000..6f7b8ad --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/detection/utils/object_detection/visualization_utils.py @@ -0,0 +1,733 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""A set of functions that are used for visualization. + +These functions often receive an image, perform some visualization on the image. +The functions do not return a value, instead they modify the image itself. + +""" +import collections +import functools +from absl import logging +# Set headless-friendly backend. +import matplotlib; matplotlib.use('Agg') # pylint: disable=multiple-statements +import matplotlib.pyplot as plt # pylint: disable=g-import-not-at-top +import numpy as np +import PIL.Image as Image +import PIL.ImageColor as ImageColor +import PIL.ImageDraw as ImageDraw +import PIL.ImageFont as ImageFont +import six +import tensorflow.compat.v2 as tf + +from official.vision.detection.utils import box_utils +from official.vision.detection.utils.object_detection import shape_utils + + +_TITLE_LEFT_MARGIN = 10 +_TITLE_TOP_MARGIN = 10 +STANDARD_COLORS = [ + 'AliceBlue', 'Chartreuse', 'Aqua', 'Aquamarine', 'Azure', 'Beige', 'Bisque', + 'BlanchedAlmond', 'BlueViolet', 'BurlyWood', 'CadetBlue', 'AntiqueWhite', + 'Chocolate', 'Coral', 'CornflowerBlue', 'Cornsilk', 'Crimson', 'Cyan', + 'DarkCyan', 'DarkGoldenRod', 'DarkGrey', 'DarkKhaki', 'DarkOrange', + 'DarkOrchid', 'DarkSalmon', 'DarkSeaGreen', 'DarkTurquoise', 'DarkViolet', + 'DeepPink', 'DeepSkyBlue', 'DodgerBlue', 'FireBrick', 'FloralWhite', + 'ForestGreen', 'Fuchsia', 'Gainsboro', 'GhostWhite', 'Gold', 'GoldenRod', + 'Salmon', 'Tan', 'HoneyDew', 'HotPink', 'IndianRed', 'Ivory', 'Khaki', + 'Lavender', 'LavenderBlush', 'LawnGreen', 'LemonChiffon', 'LightBlue', + 'LightCoral', 'LightCyan', 'LightGoldenRodYellow', 'LightGray', 'LightGrey', + 'LightGreen', 'LightPink', 'LightSalmon', 'LightSeaGreen', 'LightSkyBlue', + 'LightSlateGray', 'LightSlateGrey', 'LightSteelBlue', 'LightYellow', 'Lime', + 'LimeGreen', 'Linen', 'Magenta', 'MediumAquaMarine', 'MediumOrchid', + 'MediumPurple', 'MediumSeaGreen', 'MediumSlateBlue', 'MediumSpringGreen', + 'MediumTurquoise', 'MediumVioletRed', 'MintCream', 'MistyRose', 'Moccasin', + 'NavajoWhite', 'OldLace', 'Olive', 'OliveDrab', 'Orange', 'OrangeRed', + 'Orchid', 'PaleGoldenRod', 'PaleGreen', 'PaleTurquoise', 'PaleVioletRed', + 'PapayaWhip', 'PeachPuff', 'Peru', 'Pink', 'Plum', 'PowderBlue', 'Purple', + 'Red', 'RosyBrown', 'RoyalBlue', 'SaddleBrown', 'Green', 'SandyBrown', + 'SeaGreen', 'SeaShell', 'Sienna', 'Silver', 'SkyBlue', 'SlateBlue', + 'SlateGray', 'SlateGrey', 'Snow', 'SpringGreen', 'SteelBlue', 'GreenYellow', + 'Teal', 'Thistle', 'Tomato', 'Turquoise', 'Violet', 'Wheat', 'White', + 'WhiteSmoke', 'Yellow', 'YellowGreen' +] + + +def save_image_array_as_png(image, output_path): + """Saves an image (represented as a numpy array) to PNG. + + Args: + image: a numpy array with shape [height, width, 3]. + output_path: path to which image should be written. + """ + image_pil = Image.fromarray(np.uint8(image)).convert('RGB') + with tf.io.gfile.GFile(output_path, 'w') as fid: + image_pil.save(fid, 'PNG') + + +def encode_image_array_as_png_str(image): + """Encodes a numpy array into a PNG string. + + Args: + image: a numpy array with shape [height, width, 3]. + + Returns: + PNG encoded image string. + """ + image_pil = Image.fromarray(np.uint8(image)) + output = six.BytesIO() + image_pil.save(output, format='PNG') + png_string = output.getvalue() + output.close() + return png_string + + +def visualize_images_with_bounding_boxes(images, box_outputs, step, + summary_writer): + """Records subset of evaluation images with bounding boxes.""" + if not isinstance(images, list): + logging.warning('visualize_images_with_bounding_boxes expects list of ' + 'images but received type: %s and value: %s', + type(images), images) + return + + image_shape = tf.shape(images[0]) + image_height = tf.cast(image_shape[0], tf.float32) + image_width = tf.cast(image_shape[1], tf.float32) + normalized_boxes = box_utils.normalize_boxes(box_outputs, + [image_height, image_width]) + + bounding_box_color = tf.constant([[1.0, 1.0, 0.0, 1.0]]) + image_summary = tf.image.draw_bounding_boxes( + tf.cast(images, tf.float32), normalized_boxes, bounding_box_color) + with summary_writer.as_default(): + tf.summary.image('bounding_box_summary', image_summary, step=step) + summary_writer.flush() + + +def draw_bounding_box_on_image_array(image, + ymin, + xmin, + ymax, + xmax, + color='red', + thickness=4, + display_str_list=(), + use_normalized_coordinates=True): + """Adds a bounding box to an image (numpy array). + + Bounding box coordinates can be specified in either absolute (pixel) or + normalized coordinates by setting the use_normalized_coordinates argument. + + Args: + image: a numpy array with shape [height, width, 3]. + ymin: ymin of bounding box. + xmin: xmin of bounding box. + ymax: ymax of bounding box. + xmax: xmax of bounding box. + color: color to draw bounding box. Default is red. + thickness: line thickness. Default value is 4. + display_str_list: list of strings to display in box + (each to be shown on its own line). + use_normalized_coordinates: If True (default), treat coordinates + ymin, xmin, ymax, xmax as relative to the image. Otherwise treat + coordinates as absolute. + """ + image_pil = Image.fromarray(np.uint8(image)).convert('RGB') + draw_bounding_box_on_image(image_pil, ymin, xmin, ymax, xmax, color, + thickness, display_str_list, + use_normalized_coordinates) + np.copyto(image, np.array(image_pil)) + + +def draw_bounding_box_on_image(image, + ymin, + xmin, + ymax, + xmax, + color='red', + thickness=4, + display_str_list=(), + use_normalized_coordinates=True): + """Adds a bounding box to an image. + + Bounding box coordinates can be specified in either absolute (pixel) or + normalized coordinates by setting the use_normalized_coordinates argument. + + Each string in display_str_list is displayed on a separate line above the + bounding box in black text on a rectangle filled with the input 'color'. + If the top of the bounding box extends to the edge of the image, the strings + are displayed below the bounding box. + + Args: + image: a PIL.Image object. + ymin: ymin of bounding box. + xmin: xmin of bounding box. + ymax: ymax of bounding box. + xmax: xmax of bounding box. + color: color to draw bounding box. Default is red. + thickness: line thickness. Default value is 4. + display_str_list: list of strings to display in box + (each to be shown on its own line). + use_normalized_coordinates: If True (default), treat coordinates + ymin, xmin, ymax, xmax as relative to the image. Otherwise treat + coordinates as absolute. + """ + draw = ImageDraw.Draw(image) + im_width, im_height = image.size + if use_normalized_coordinates: + (left, right, top, bottom) = (xmin * im_width, xmax * im_width, + ymin * im_height, ymax * im_height) + else: + (left, right, top, bottom) = (xmin, xmax, ymin, ymax) + draw.line([(left, top), (left, bottom), (right, bottom), + (right, top), (left, top)], width=thickness, fill=color) + try: + font = ImageFont.truetype('arial.ttf', 24) + except IOError: + font = ImageFont.load_default() + + # If the total height of the display strings added to the top of the bounding + # box exceeds the top of the image, stack the strings below the bounding box + # instead of above. + display_str_heights = [font.getsize(ds)[1] for ds in display_str_list] + # Each display_str has a top and bottom margin of 0.05x. + total_display_str_height = (1 + 2 * 0.05) * sum(display_str_heights) + + if top > total_display_str_height: + text_bottom = top + else: + text_bottom = bottom + total_display_str_height + # Reverse list and print from bottom to top. + for display_str in display_str_list[::-1]: + text_width, text_height = font.getsize(display_str) + margin = np.ceil(0.05 * text_height) + draw.rectangle( + [(left, text_bottom - text_height - 2 * margin), (left + text_width, + text_bottom)], + fill=color) + draw.text( + (left + margin, text_bottom - text_height - margin), + display_str, + fill='black', + font=font) + text_bottom -= text_height - 2 * margin + + +def draw_bounding_boxes_on_image_array(image, + boxes, + color='red', + thickness=4, + display_str_list_list=()): + """Draws bounding boxes on image (numpy array). + + Args: + image: a numpy array object. + boxes: a 2 dimensional numpy array of [N, 4]: (ymin, xmin, ymax, xmax). + The coordinates are in normalized format between [0, 1]. + color: color to draw bounding box. Default is red. + thickness: line thickness. Default value is 4. + display_str_list_list: list of list of strings. + a list of strings for each bounding box. + The reason to pass a list of strings for a + bounding box is that it might contain + multiple labels. + + Raises: + ValueError: if boxes is not a [N, 4] array + """ + image_pil = Image.fromarray(image) + draw_bounding_boxes_on_image(image_pil, boxes, color, thickness, + display_str_list_list) + np.copyto(image, np.array(image_pil)) + + +def draw_bounding_boxes_on_image(image, + boxes, + color='red', + thickness=4, + display_str_list_list=()): + """Draws bounding boxes on image. + + Args: + image: a PIL.Image object. + boxes: a 2 dimensional numpy array of [N, 4]: (ymin, xmin, ymax, xmax). + The coordinates are in normalized format between [0, 1]. + color: color to draw bounding box. Default is red. + thickness: line thickness. Default value is 4. + display_str_list_list: list of list of strings. + a list of strings for each bounding box. + The reason to pass a list of strings for a + bounding box is that it might contain + multiple labels. + + Raises: + ValueError: if boxes is not a [N, 4] array + """ + boxes_shape = boxes.shape + if not boxes_shape: + return + if len(boxes_shape) != 2 or boxes_shape[1] != 4: + raise ValueError('Input must be of size [N, 4]') + for i in range(boxes_shape[0]): + display_str_list = () + if display_str_list_list: + display_str_list = display_str_list_list[i] + draw_bounding_box_on_image(image, boxes[i, 0], boxes[i, 1], boxes[i, 2], + boxes[i, 3], color, thickness, display_str_list) + + +def _visualize_boxes(image, boxes, classes, scores, category_index, **kwargs): + return visualize_boxes_and_labels_on_image_array( + image, boxes, classes, scores, category_index=category_index, **kwargs) + + +def _visualize_boxes_and_masks(image, boxes, classes, scores, masks, + category_index, **kwargs): + return visualize_boxes_and_labels_on_image_array( + image, + boxes, + classes, + scores, + category_index=category_index, + instance_masks=masks, + **kwargs) + + +def _visualize_boxes_and_keypoints(image, boxes, classes, scores, keypoints, + category_index, **kwargs): + return visualize_boxes_and_labels_on_image_array( + image, + boxes, + classes, + scores, + category_index=category_index, + keypoints=keypoints, + **kwargs) + + +def _visualize_boxes_and_masks_and_keypoints( + image, boxes, classes, scores, masks, keypoints, category_index, **kwargs): + return visualize_boxes_and_labels_on_image_array( + image, + boxes, + classes, + scores, + category_index=category_index, + instance_masks=masks, + keypoints=keypoints, + **kwargs) + + +def _resize_original_image(image, image_shape): + image = tf.expand_dims(image, 0) + image = tf.image.resize( + image, image_shape, method=tf.image.ResizeMethod.NEAREST_NEIGHBOR) + return tf.cast(tf.squeeze(image, 0), tf.uint8) + + +def draw_bounding_boxes_on_image_tensors(images, + boxes, + classes, + scores, + category_index, + original_image_spatial_shape=None, + true_image_shape=None, + instance_masks=None, + keypoints=None, + max_boxes_to_draw=20, + min_score_thresh=0.2, + use_normalized_coordinates=True): + """Draws bounding boxes, masks, and keypoints on batch of image tensors. + + Args: + images: A 4D uint8 image tensor of shape [N, H, W, C]. If C > 3, additional + channels will be ignored. If C = 1, then we convert the images to RGB + images. + boxes: [N, max_detections, 4] float32 tensor of detection boxes. + classes: [N, max_detections] int tensor of detection classes. Note that + classes are 1-indexed. + scores: [N, max_detections] float32 tensor of detection scores. + category_index: a dict that maps integer ids to category dicts. e.g. + {1: {1: 'dog'}, 2: {2: 'cat'}, ...} + original_image_spatial_shape: [N, 2] tensor containing the spatial size of + the original image. + true_image_shape: [N, 3] tensor containing the spatial size of unpadded + original_image. + instance_masks: A 4D uint8 tensor of shape [N, max_detection, H, W] with + instance masks. + keypoints: A 4D float32 tensor of shape [N, max_detection, num_keypoints, 2] + with keypoints. + max_boxes_to_draw: Maximum number of boxes to draw on an image. Default 20. + min_score_thresh: Minimum score threshold for visualization. Default 0.2. + use_normalized_coordinates: Whether to assume boxes and kepoints are in + normalized coordinates (as opposed to absolute coordiantes). + Default is True. + + Returns: + 4D image tensor of type uint8, with boxes drawn on top. + """ + # Additional channels are being ignored. + if images.shape[3] > 3: + images = images[:, :, :, 0:3] + elif images.shape[3] == 1: + images = tf.image.grayscale_to_rgb(images) + visualization_keyword_args = { + 'use_normalized_coordinates': use_normalized_coordinates, + 'max_boxes_to_draw': max_boxes_to_draw, + 'min_score_thresh': min_score_thresh, + 'agnostic_mode': False, + 'line_thickness': 4 + } + if true_image_shape is None: + true_shapes = tf.constant(-1, shape=[images.shape.as_list()[0], 3]) + else: + true_shapes = true_image_shape + if original_image_spatial_shape is None: + original_shapes = tf.constant(-1, shape=[images.shape.as_list()[0], 2]) + else: + original_shapes = original_image_spatial_shape + + if instance_masks is not None and keypoints is None: + visualize_boxes_fn = functools.partial( + _visualize_boxes_and_masks, + category_index=category_index, + **visualization_keyword_args) + elems = [ + true_shapes, original_shapes, images, boxes, classes, scores, + instance_masks + ] + elif instance_masks is None and keypoints is not None: + visualize_boxes_fn = functools.partial( + _visualize_boxes_and_keypoints, + category_index=category_index, + **visualization_keyword_args) + elems = [ + true_shapes, original_shapes, images, boxes, classes, scores, keypoints + ] + elif instance_masks is not None and keypoints is not None: + visualize_boxes_fn = functools.partial( + _visualize_boxes_and_masks_and_keypoints, + category_index=category_index, + **visualization_keyword_args) + elems = [ + true_shapes, original_shapes, images, boxes, classes, scores, + instance_masks, keypoints + ] + else: + visualize_boxes_fn = functools.partial( + _visualize_boxes, + category_index=category_index, + **visualization_keyword_args) + elems = [ + true_shapes, original_shapes, images, boxes, classes, scores + ] + + def draw_boxes(image_and_detections): + """Draws boxes on image.""" + true_shape = image_and_detections[0] + original_shape = image_and_detections[1] + if true_image_shape is not None: + image = shape_utils.pad_or_clip_nd( + image_and_detections[2], [true_shape[0], true_shape[1], 3]) + if original_image_spatial_shape is not None: + image_and_detections[2] = _resize_original_image(image, original_shape) + + image_with_boxes = tf.compat.v1.py_func(visualize_boxes_fn, + image_and_detections[2:], tf.uint8) + return image_with_boxes + + images = tf.map_fn(draw_boxes, elems, dtype=tf.uint8, back_prop=False) + return images + + +def draw_keypoints_on_image_array(image, + keypoints, + color='red', + radius=2, + use_normalized_coordinates=True): + """Draws keypoints on an image (numpy array). + + Args: + image: a numpy array with shape [height, width, 3]. + keypoints: a numpy array with shape [num_keypoints, 2]. + color: color to draw the keypoints with. Default is red. + radius: keypoint radius. Default value is 2. + use_normalized_coordinates: if True (default), treat keypoint values as + relative to the image. Otherwise treat them as absolute. + """ + image_pil = Image.fromarray(np.uint8(image)).convert('RGB') + draw_keypoints_on_image(image_pil, keypoints, color, radius, + use_normalized_coordinates) + np.copyto(image, np.array(image_pil)) + + +def draw_keypoints_on_image(image, + keypoints, + color='red', + radius=2, + use_normalized_coordinates=True): + """Draws keypoints on an image. + + Args: + image: a PIL.Image object. + keypoints: a numpy array with shape [num_keypoints, 2]. + color: color to draw the keypoints with. Default is red. + radius: keypoint radius. Default value is 2. + use_normalized_coordinates: if True (default), treat keypoint values as + relative to the image. Otherwise treat them as absolute. + """ + draw = ImageDraw.Draw(image) + im_width, im_height = image.size + keypoints_x = [k[1] for k in keypoints] + keypoints_y = [k[0] for k in keypoints] + if use_normalized_coordinates: + keypoints_x = tuple([im_width * x for x in keypoints_x]) + keypoints_y = tuple([im_height * y for y in keypoints_y]) + for keypoint_x, keypoint_y in zip(keypoints_x, keypoints_y): + draw.ellipse([(keypoint_x - radius, keypoint_y - radius), + (keypoint_x + radius, keypoint_y + radius)], + outline=color, fill=color) + + +def draw_mask_on_image_array(image, mask, color='red', alpha=0.4): + """Draws mask on an image. + + Args: + image: uint8 numpy array with shape (img_height, img_height, 3) + mask: a uint8 numpy array of shape (img_height, img_height) with + values between either 0 or 1. + color: color to draw the keypoints with. Default is red. + alpha: transparency value between 0 and 1. (default: 0.4) + + Raises: + ValueError: On incorrect data type for image or masks. + """ + if image.dtype != np.uint8: + raise ValueError('`image` not of type np.uint8') + if mask.dtype != np.uint8: + raise ValueError('`mask` not of type np.uint8') + if np.any(np.logical_and(mask != 1, mask != 0)): + raise ValueError('`mask` elements should be in [0, 1]') + if image.shape[:2] != mask.shape: + raise ValueError('The image has spatial dimensions %s but the mask has ' + 'dimensions %s' % (image.shape[:2], mask.shape)) + rgb = ImageColor.getrgb(color) + pil_image = Image.fromarray(image) + + solid_color = np.expand_dims( + np.ones_like(mask), axis=2) * np.reshape(list(rgb), [1, 1, 3]) + pil_solid_color = Image.fromarray(np.uint8(solid_color)).convert('RGBA') + pil_mask = Image.fromarray(np.uint8(255.0*alpha*mask)).convert('L') + pil_image = Image.composite(pil_solid_color, pil_image, pil_mask) + np.copyto(image, np.array(pil_image.convert('RGB'))) + + +def visualize_boxes_and_labels_on_image_array( + image, + boxes, + classes, + scores, + category_index, + instance_masks=None, + instance_boundaries=None, + keypoints=None, + use_normalized_coordinates=False, + max_boxes_to_draw=20, + min_score_thresh=.5, + agnostic_mode=False, + line_thickness=4, + groundtruth_box_visualization_color='black', + skip_scores=False, + skip_labels=False): + """Overlay labeled boxes on an image with formatted scores and label names. + + This function groups boxes that correspond to the same location + and creates a display string for each detection and overlays these + on the image. Note that this function modifies the image in place, and returns + that same image. + + Args: + image: uint8 numpy array with shape (img_height, img_width, 3) + boxes: a numpy array of shape [N, 4] + classes: a numpy array of shape [N]. Note that class indices are 1-based, + and match the keys in the label map. + scores: a numpy array of shape [N] or None. If scores=None, then + this function assumes that the boxes to be plotted are groundtruth + boxes and plot all boxes as black with no classes or scores. + category_index: a dict containing category dictionaries (each holding + category index `id` and category name `name`) keyed by category indices. + instance_masks: a numpy array of shape [N, image_height, image_width] with + values ranging between 0 and 1, can be None. + instance_boundaries: a numpy array of shape [N, image_height, image_width] + with values ranging between 0 and 1, can be None. + keypoints: a numpy array of shape [N, num_keypoints, 2], can + be None + use_normalized_coordinates: whether boxes is to be interpreted as + normalized coordinates or not. + max_boxes_to_draw: maximum number of boxes to visualize. If None, draw + all boxes. + min_score_thresh: minimum score threshold for a box to be visualized + agnostic_mode: boolean (default: False) controlling whether to evaluate in + class-agnostic mode or not. This mode will display scores but ignore + classes. + line_thickness: integer (default: 4) controlling line width of the boxes. + groundtruth_box_visualization_color: box color for visualizing groundtruth + boxes + skip_scores: whether to skip score when drawing a single detection + skip_labels: whether to skip label when drawing a single detection + + Returns: + uint8 numpy array with shape (img_height, img_width, 3) with overlaid boxes. + """ + # Create a display string (and color) for every box location, group any boxes + # that correspond to the same location. + box_to_display_str_map = collections.defaultdict(list) + box_to_color_map = collections.defaultdict(str) + box_to_instance_masks_map = {} + box_to_instance_boundaries_map = {} + box_to_keypoints_map = collections.defaultdict(list) + if not max_boxes_to_draw: + max_boxes_to_draw = boxes.shape[0] + for i in range(min(max_boxes_to_draw, boxes.shape[0])): + if scores is None or scores[i] > min_score_thresh: + box = tuple(boxes[i].tolist()) + if instance_masks is not None: + box_to_instance_masks_map[box] = instance_masks[i] + if instance_boundaries is not None: + box_to_instance_boundaries_map[box] = instance_boundaries[i] + if keypoints is not None: + box_to_keypoints_map[box].extend(keypoints[i]) + if scores is None: + box_to_color_map[box] = groundtruth_box_visualization_color + else: + display_str = '' + if not skip_labels: + if not agnostic_mode: + if classes[i] in category_index.keys(): + class_name = category_index[classes[i]]['name'] + else: + class_name = 'N/A' + display_str = str(class_name) + if not skip_scores: + if not display_str: + display_str = '{}%'.format(int(100*scores[i])) + else: + display_str = '{}: {}%'.format(display_str, int(100*scores[i])) + box_to_display_str_map[box].append(display_str) + if agnostic_mode: + box_to_color_map[box] = 'DarkOrange' + else: + box_to_color_map[box] = STANDARD_COLORS[ + classes[i] % len(STANDARD_COLORS)] + + # Draw all boxes onto image. + for box, color in box_to_color_map.items(): + ymin, xmin, ymax, xmax = box + if instance_masks is not None: + draw_mask_on_image_array( + image, + box_to_instance_masks_map[box], + color=color + ) + if instance_boundaries is not None: + draw_mask_on_image_array( + image, + box_to_instance_boundaries_map[box], + color='red', + alpha=1.0 + ) + draw_bounding_box_on_image_array( + image, + ymin, + xmin, + ymax, + xmax, + color=color, + thickness=line_thickness, + display_str_list=box_to_display_str_map[box], + use_normalized_coordinates=use_normalized_coordinates) + if keypoints is not None: + draw_keypoints_on_image_array( + image, + box_to_keypoints_map[box], + color=color, + radius=line_thickness / 2, + use_normalized_coordinates=use_normalized_coordinates) + + return image + + +def add_cdf_image_summary(values, name): + """Adds a tf.summary.image for a CDF plot of the values. + + Normalizes `values` such that they sum to 1, plots the cumulative distribution + function and creates a tf image summary. + + Args: + values: a 1-D float32 tensor containing the values. + name: name for the image summary. + """ + def cdf_plot(values): + """Numpy function to plot CDF.""" + normalized_values = values / np.sum(values) + sorted_values = np.sort(normalized_values) + cumulative_values = np.cumsum(sorted_values) + fraction_of_examples = (np.arange(cumulative_values.size, dtype=np.float32) + / cumulative_values.size) + fig = plt.figure(frameon=False) + ax = fig.add_subplot('111') + ax.plot(fraction_of_examples, cumulative_values) + ax.set_ylabel('cumulative normalized values') + ax.set_xlabel('fraction of examples') + fig.canvas.draw() + width, height = fig.get_size_inches() * fig.get_dpi() + image = np.fromstring(fig.canvas.tostring_rgb(), dtype='uint8').reshape( + 1, int(height), int(width), 3) + return image + + cdf_plot = tf.compat.v1.py_func(cdf_plot, [values], tf.uint8) + tf.compat.v1.summary.image(name, cdf_plot) + + +def add_hist_image_summary(values, bins, name): + """Adds a tf.summary.image for a histogram plot of the values. + + Plots the histogram of values and creates a tf image summary. + + Args: + values: a 1-D float32 tensor containing the values. + bins: bin edges which will be directly passed to np.histogram. + name: name for the image summary. + """ + + def hist_plot(values, bins): + """Numpy function to plot hist.""" + fig = plt.figure(frameon=False) + ax = fig.add_subplot('111') + y, x = np.histogram(values, bins=bins) + ax.plot(x[:-1], y) + ax.set_ylabel('count') + ax.set_xlabel('value') + fig.canvas.draw() + width, height = fig.get_size_inches() * fig.get_dpi() + image = np.fromstring( + fig.canvas.tostring_rgb(), dtype='uint8').reshape( + 1, int(height), int(width), 3) + return image + + hist_plot = tf.compat.v1.py_func(hist_plot, [values, bins], tf.uint8) + tf.compat.v1.summary.image(name, hist_plot) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/README.md new file mode 100644 index 0000000..c16fdc0 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/README.md @@ -0,0 +1,164 @@ +# Image Classification + +This folder contains TF 2.0 model examples for image classification: + +* [MNIST](#mnist) +* [Classifier Trainer](#classifier-trainer), a framework that uses the Keras +compile/fit methods for image classification models, including: + * ResNet + * EfficientNet[^1] + +[^1]: Currently a work in progress. We cannot match "AutoAugment (AA)" in [the original version](https://github.com/tensorflow/tpu/tree/master/models/official/efficientnet). +For more information about other types of models, please refer to this +[README file](../../README.md). + +## Before you begin +Please make sure that you have the latest version of TensorFlow +installed and +[add the models folder to your Python path](/official/#running-the-models). + +### ImageNet preparation + +#### Using TFDS +`classifier_trainer.py` supports ImageNet with +[TensorFlow Datasets (TFDS)](https://www.tensorflow.org/datasets/overview). + +Please see the following [example snippet](https://github.com/tensorflow/datasets/blob/master/tensorflow_datasets/scripts/download_and_prepare.py) +for more information on how to use TFDS to download and prepare datasets, and +specifically the [TFDS ImageNet readme](https://github.com/tensorflow/datasets/blob/master/docs/catalog/imagenet2012.md) +for manual download instructions. + +#### Legacy TFRecords +Download the ImageNet dataset and convert it to TFRecord format. +The following [script](https://github.com/tensorflow/tpu/blob/master/tools/datasets/imagenet_to_gcs.py) +and [README](https://github.com/tensorflow/tpu/tree/master/tools/datasets#imagenet_to_gcspy) +provide a few options. + +Note that the legacy ResNet runners, e.g. [resnet/resnet_ctl_imagenet_main.py](resnet/resnet_ctl_imagenet_main.py) +require TFRecords whereas `classifier_trainer.py` can use both by setting the +builder to 'records' or 'tfds' in the configurations. + +### Running on Cloud TPUs + +Note: These models will **not** work with TPUs on Colab. + +You can train image classification models on Cloud TPUs using +[tf.distribute.experimental.TPUStrategy](https://www.tensorflow.org/api_docs/python/tf/distribute/experimental/TPUStrategy?version=nightly). +If you are not familiar with Cloud TPUs, it is strongly recommended that you go +through the +[quickstart](https://cloud.google.com/tpu/docs/quickstart) to learn how to +create a TPU and GCE VM. + +### Running on multiple GPU hosts + +You can also train these models on multiple hosts, each with GPUs, using +[tf.distribute.experimental.MultiWorkerMirroredStrategy](https://www.tensorflow.org/api_docs/python/tf/distribute/experimental/MultiWorkerMirroredStrategy). + +The easiest way to run multi-host benchmarks is to set the +[`TF_CONFIG`](https://www.tensorflow.org/guide/distributed_training#TF_CONFIG) +appropriately at each host. e.g., to run using `MultiWorkerMirroredStrategy` on +2 hosts, the `cluster` in `TF_CONFIG` should have 2 `host:port` entries, and +host `i` should have the `task` in `TF_CONFIG` set to `{"type": "worker", +"index": i}`. `MultiWorkerMirroredStrategy` will automatically use all the +available GPUs at each host. + +## MNIST + +To download the data and run the MNIST sample model locally for the first time, +run one of the following command: + +```bash +python3 mnist_main.py \ + --model_dir=$MODEL_DIR \ + --data_dir=$DATA_DIR \ + --train_epochs=10 \ + --distribution_strategy=one_device \ + --num_gpus=$NUM_GPUS \ + --download +``` + +To train the model on a Cloud TPU, run the following command: + +```bash +python3 mnist_main.py \ + --tpu=$TPU_NAME \ + --model_dir=$MODEL_DIR \ + --data_dir=$DATA_DIR \ + --train_epochs=10 \ + --distribution_strategy=tpu \ + --download +``` + +Note: the `--download` flag is only required the first time you run the model. + + +## Classifier Trainer +The classifier trainer is a unified framework for running image classification +models using Keras's compile/fit methods. Experiments should be provided in the +form of YAML files, some examples are included within the configs/examples +folder. Please see [configs/examples](./configs/examples) for more example +configurations. + +The provided configuration files use a per replica batch size and is scaled +by the number of devices. For instance, if `batch size` = 64, then for 1 GPU +the global batch size would be 64 * 1 = 64. For 8 GPUs, the global batch size +would be 64 * 8 = 512. Similarly, for a v3-8 TPU, the global batch size would +be 64 * 8 = 512, and for a v3-32, the global batch size is 64 * 32 = 2048. + +### ResNet50 + +#### On GPU: +```bash +python3 classifier_trainer.py \ + --mode=train_and_eval \ + --model_type=resnet \ + --dataset=imagenet \ + --model_dir=$MODEL_DIR \ + --data_dir=$DATA_DIR \ + --config_file=configs/examples/resnet/imagenet/gpu.yaml \ + --params_override='runtime.num_gpus=$NUM_GPUS' +``` + +#### On TPU: +```bash +python3 classifier_trainer.py \ + --mode=train_and_eval \ + --model_type=resnet \ + --dataset=imagenet \ + --tpu=$TPU_NAME \ + --model_dir=$MODEL_DIR \ + --data_dir=$DATA_DIR \ + --config_file=configs/examples/resnet/imagenet/tpu.yaml +``` + +### EfficientNet +**Note: EfficientNet development is a work in progress.** +#### On GPU: +```bash +python3 classifier_trainer.py \ + --mode=train_and_eval \ + --model_type=efficientnet \ + --dataset=imagenet \ + --model_dir=$MODEL_DIR \ + --data_dir=$DATA_DIR \ + --config_file=configs/examples/efficientnet/imagenet/efficientnet-b0-gpu.yaml \ + --params_override='runtime.num_gpus=$NUM_GPUS' +``` + + +#### On TPU: +```bash +python3 classifier_trainer.py \ + --mode=train_and_eval \ + --model_type=efficientnet \ + --dataset=imagenet \ + --tpu=$TPU_NAME \ + --model_dir=$MODEL_DIR \ + --data_dir=$DATA_DIR \ + --config_file=configs/examples/efficientnet/imagenet/efficientnet-b0-tpu.yaml +``` + +Note that the number of GPU devices can be overridden in the command line using +`--params_overrides`. The TPU does not need this override as the device is fixed +by providing the TPU address or name with the `--tpu` flag. + diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/augment.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/augment.py new file mode 100644 index 0000000..3771821 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/augment.py @@ -0,0 +1,999 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""AutoAugment and RandAugment policies for enhanced image preprocessing. + +AutoAugment Reference: https://arxiv.org/abs/1805.09501 +RandAugment Reference: https://arxiv.org/abs/1909.13719 +""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import math +import tensorflow as tf +from typing import Any, Dict, List, Optional, Text, Tuple + +from tensorflow.python.keras.layers.preprocessing import image_preprocessing as image_ops + +# This signifies the max integer that the controller RNN could predict for the +# augmentation scheme. +_MAX_LEVEL = 10. + + +def to_4d(image: tf.Tensor) -> tf.Tensor: + """Converts an input Tensor to 4 dimensions. + + 4D image => [N, H, W, C] or [N, C, H, W] + 3D image => [1, H, W, C] or [1, C, H, W] + 2D image => [1, H, W, 1] + + Args: + image: The 2/3/4D input tensor. + + Returns: + A 4D image tensor. + + Raises: + `TypeError` if `image` is not a 2/3/4D tensor. + + """ + shape = tf.shape(image) + original_rank = tf.rank(image) + left_pad = tf.cast(tf.less_equal(original_rank, 3), dtype=tf.int32) + right_pad = tf.cast(tf.equal(original_rank, 2), dtype=tf.int32) + new_shape = tf.concat( + [ + tf.ones(shape=left_pad, dtype=tf.int32), + shape, + tf.ones(shape=right_pad, dtype=tf.int32), + ], + axis=0, + ) + return tf.reshape(image, new_shape) + + +def from_4d(image: tf.Tensor, ndims: tf.Tensor) -> tf.Tensor: + """Converts a 4D image back to `ndims` rank.""" + shape = tf.shape(image) + begin = tf.cast(tf.less_equal(ndims, 3), dtype=tf.int32) + end = 4 - tf.cast(tf.equal(ndims, 2), dtype=tf.int32) + new_shape = shape[begin:end] + return tf.reshape(image, new_shape) + + +def _convert_translation_to_transform(translations: tf.Tensor) -> tf.Tensor: + """Converts translations to a projective transform. + + The translation matrix looks like this: + [[1 0 -dx] + [0 1 -dy] + [0 0 1]] + + Args: + translations: The 2-element list representing [dx, dy], or a matrix of + 2-element lists representing [dx dy] to translate for each image. The + shape must be static. + + Returns: + The transformation matrix of shape (num_images, 8). + + Raises: + `TypeError` if + - the shape of `translations` is not known or + - the shape of `translations` is not rank 1 or 2. + + """ + translations = tf.convert_to_tensor(translations, dtype=tf.float32) + if translations.get_shape().ndims is None: + raise TypeError('translations rank must be statically known') + elif len(translations.get_shape()) == 1: + translations = translations[None] + elif len(translations.get_shape()) != 2: + raise TypeError('translations should have rank 1 or 2.') + num_translations = tf.shape(translations)[0] + + return tf.concat( + values=[ + tf.ones((num_translations, 1), tf.dtypes.float32), + tf.zeros((num_translations, 1), tf.dtypes.float32), + -translations[:, 0, None], + tf.zeros((num_translations, 1), tf.dtypes.float32), + tf.ones((num_translations, 1), tf.dtypes.float32), + -translations[:, 1, None], + tf.zeros((num_translations, 2), tf.dtypes.float32), + ], + axis=1, + ) + + +def _convert_angles_to_transform( + angles: tf.Tensor, + image_width: tf.Tensor, + image_height: tf.Tensor) -> tf.Tensor: + """Converts an angle or angles to a projective transform. + + Args: + angles: A scalar to rotate all images, or a vector to rotate a batch of + images. This must be a scalar. + image_width: The width of the image(s) to be transformed. + image_height: The height of the image(s) to be transformed. + + Returns: + A tensor of shape (num_images, 8). + + Raises: + `TypeError` if `angles` is not rank 0 or 1. + + """ + angles = tf.convert_to_tensor(angles, dtype=tf.float32) + if len(angles.get_shape()) == 0: # pylint:disable=g-explicit-length-test + angles = angles[None] + elif len(angles.get_shape()) != 1: + raise TypeError('Angles should have a rank 0 or 1.') + x_offset = ((image_width - 1) - + (tf.math.cos(angles) * (image_width - 1) - tf.math.sin(angles) * + (image_height - 1))) / 2.0 + y_offset = ((image_height - 1) - + (tf.math.sin(angles) * (image_width - 1) + tf.math.cos(angles) * + (image_height - 1))) / 2.0 + num_angles = tf.shape(angles)[0] + return tf.concat( + values=[ + tf.math.cos(angles)[:, None], + -tf.math.sin(angles)[:, None], + x_offset[:, None], + tf.math.sin(angles)[:, None], + tf.math.cos(angles)[:, None], + y_offset[:, None], + tf.zeros((num_angles, 2), tf.dtypes.float32), + ], + axis=1, + ) + + +def transform(image: tf.Tensor, transforms) -> tf.Tensor: + """Prepares input data for `image_ops.transform`.""" + original_ndims = tf.rank(image) + transforms = tf.convert_to_tensor(transforms, dtype=tf.float32) + if len(tf.shape(transforms)) == 1: + transforms = transforms[None] + image = to_4d(image) + image = image_ops.transform( + images=image, + transforms=transforms, + interpolation='nearest') + return from_4d(image, original_ndims) + + +def translate(image: tf.Tensor, translations) -> tf.Tensor: + """Translates image(s) by provided vectors. + + Args: + image: An image Tensor of type uint8. + translations: A vector or matrix representing [dx dy]. + + Returns: + The translated version of the image. + + """ + transforms = _convert_translation_to_transform(translations) + return transform(image, transforms=transforms) + + +def rotate(image: tf.Tensor, degrees: float) -> tf.Tensor: + """Rotates the image by degrees either clockwise or counterclockwise. + + Args: + image: An image Tensor of type uint8. + degrees: Float, a scalar angle in degrees to rotate all images by. If + degrees is positive the image will be rotated clockwise otherwise it will + be rotated counterclockwise. + + Returns: + The rotated version of image. + + """ + # Convert from degrees to radians. + degrees_to_radians = math.pi / 180.0 + radians = tf.cast(degrees * degrees_to_radians, tf.float32) + + original_ndims = tf.rank(image) + image = to_4d(image) + + image_height = tf.cast(tf.shape(image)[1], tf.float32) + image_width = tf.cast(tf.shape(image)[2], tf.float32) + transforms = _convert_angles_to_transform(angles=radians, + image_width=image_width, + image_height=image_height) + # In practice, we should randomize the rotation degrees by flipping + # it negatively half the time, but that's done on 'degrees' outside + # of the function. + image = transform(image, transforms=transforms) + return from_4d(image, original_ndims) + + +def blend(image1: tf.Tensor, image2: tf.Tensor, factor: float) -> tf.Tensor: + """Blend image1 and image2 using 'factor'. + + Factor can be above 0.0. A value of 0.0 means only image1 is used. + A value of 1.0 means only image2 is used. A value between 0.0 and + 1.0 means we linearly interpolate the pixel values between the two + images. A value greater than 1.0 "extrapolates" the difference + between the two pixel values, and we clip the results to values + between 0 and 255. + + Args: + image1: An image Tensor of type uint8. + image2: An image Tensor of type uint8. + factor: A floating point value above 0.0. + + Returns: + A blended image Tensor of type uint8. + """ + if factor == 0.0: + return tf.convert_to_tensor(image1) + if factor == 1.0: + return tf.convert_to_tensor(image2) + + image1 = tf.cast(image1, tf.float32) + image2 = tf.cast(image2, tf.float32) + + difference = image2 - image1 + scaled = factor * difference + + # Do addition in float. + temp = tf.cast(image1, tf.float32) + scaled + + # Interpolate + if factor > 0.0 and factor < 1.0: + # Interpolation means we always stay within 0 and 255. + return tf.cast(temp, tf.uint8) + + # Extrapolate: + # + # We need to clip and then cast. + return tf.cast(tf.clip_by_value(temp, 0.0, 255.0), tf.uint8) + + +def cutout(image: tf.Tensor, pad_size: int, replace: int = 0) -> tf.Tensor: + """Apply cutout (https://arxiv.org/abs/1708.04552) to image. + + This operation applies a (2*pad_size x 2*pad_size) mask of zeros to + a random location within `img`. The pixel values filled in will be of the + value `replace`. The located where the mask will be applied is randomly + chosen uniformly over the whole image. + + Args: + image: An image Tensor of type uint8. + pad_size: Specifies how big the zero mask that will be generated is that + is applied to the image. The mask will be of size + (2*pad_size x 2*pad_size). + replace: What pixel value to fill in the image in the area that has + the cutout mask applied to it. + + Returns: + An image Tensor that is of type uint8. + """ + image_height = tf.shape(image)[0] + image_width = tf.shape(image)[1] + + # Sample the center location in the image where the zero mask will be applied. + cutout_center_height = tf.random.uniform( + shape=[], minval=0, maxval=image_height, + dtype=tf.int32) + + cutout_center_width = tf.random.uniform( + shape=[], minval=0, maxval=image_width, + dtype=tf.int32) + + lower_pad = tf.maximum(0, cutout_center_height - pad_size) + upper_pad = tf.maximum(0, image_height - cutout_center_height - pad_size) + left_pad = tf.maximum(0, cutout_center_width - pad_size) + right_pad = tf.maximum(0, image_width - cutout_center_width - pad_size) + + cutout_shape = [image_height - (lower_pad + upper_pad), + image_width - (left_pad + right_pad)] + padding_dims = [[lower_pad, upper_pad], [left_pad, right_pad]] + mask = tf.pad( + tf.zeros(cutout_shape, dtype=image.dtype), + padding_dims, constant_values=1) + mask = tf.expand_dims(mask, -1) + mask = tf.tile(mask, [1, 1, 3]) + image = tf.where( + tf.equal(mask, 0), + tf.ones_like(image, dtype=image.dtype) * replace, + image) + return image + + +def solarize(image: tf.Tensor, threshold: int = 128) -> tf.Tensor: + # For each pixel in the image, select the pixel + # if the value is less than the threshold. + # Otherwise, subtract 255 from the pixel. + return tf.where(image < threshold, image, 255 - image) + + +def solarize_add(image: tf.Tensor, + addition: int = 0, + threshold: int = 128) -> tf.Tensor: + # For each pixel in the image less than threshold + # we add 'addition' amount to it and then clip the + # pixel value to be between 0 and 255. The value + # of 'addition' is between -128 and 128. + added_image = tf.cast(image, tf.int64) + addition + added_image = tf.cast(tf.clip_by_value(added_image, 0, 255), tf.uint8) + return tf.where(image < threshold, added_image, image) + + +def color(image: tf.Tensor, factor: float) -> tf.Tensor: + """Equivalent of PIL Color.""" + degenerate = tf.image.grayscale_to_rgb(tf.image.rgb_to_grayscale(image)) + return blend(degenerate, image, factor) + + +def contrast(image: tf.Tensor, factor: float) -> tf.Tensor: + """Equivalent of PIL Contrast.""" + degenerate = tf.image.rgb_to_grayscale(image) + # Cast before calling tf.histogram. + degenerate = tf.cast(degenerate, tf.int32) + + # Compute the grayscale histogram, then compute the mean pixel value, + # and create a constant image size of that value. Use that as the + # blending degenerate target of the original image. + hist = tf.histogram_fixed_width(degenerate, [0, 255], nbins=256) + mean = tf.reduce_sum(tf.cast(hist, tf.float32)) / 256.0 + degenerate = tf.ones_like(degenerate, dtype=tf.float32) * mean + degenerate = tf.clip_by_value(degenerate, 0.0, 255.0) + degenerate = tf.image.grayscale_to_rgb(tf.cast(degenerate, tf.uint8)) + return blend(degenerate, image, factor) + + +def brightness(image: tf.Tensor, factor: float) -> tf.Tensor: + """Equivalent of PIL Brightness.""" + degenerate = tf.zeros_like(image) + return blend(degenerate, image, factor) + + +def posterize(image: tf.Tensor, bits: int) -> tf.Tensor: + """Equivalent of PIL Posterize.""" + shift = 8 - bits + return tf.bitwise.left_shift(tf.bitwise.right_shift(image, shift), shift) + + +def wrapped_rotate(image: tf.Tensor, degrees: float, replace: int) -> tf.Tensor: + """Applies rotation with wrap/unwrap.""" + image = rotate(wrap(image), degrees=degrees) + return unwrap(image, replace) + + +def translate_x(image: tf.Tensor, pixels: int, replace: int) -> tf.Tensor: + """Equivalent of PIL Translate in X dimension.""" + image = translate(wrap(image), [-pixels, 0]) + return unwrap(image, replace) + + +def translate_y(image: tf.Tensor, pixels: int, replace: int) -> tf.Tensor: + """Equivalent of PIL Translate in Y dimension.""" + image = translate(wrap(image), [0, -pixels]) + return unwrap(image, replace) + + +def shear_x(image: tf.Tensor, level: float, replace: int) -> tf.Tensor: + """Equivalent of PIL Shearing in X dimension.""" + # Shear parallel to x axis is a projective transform + # with a matrix form of: + # [1 level + # 0 1]. + image = transform(image=wrap(image), + transforms=[1., level, 0., 0., 1., 0., 0., 0.]) + return unwrap(image, replace) + + +def shear_y(image: tf.Tensor, level: float, replace: int) -> tf.Tensor: + """Equivalent of PIL Shearing in Y dimension.""" + # Shear parallel to y axis is a projective transform + # with a matrix form of: + # [1 0 + # level 1]. + image = transform(image=wrap(image), + transforms=[1., 0., 0., level, 1., 0., 0., 0.]) + return unwrap(image, replace) + + +def autocontrast(image: tf.Tensor) -> tf.Tensor: + """Implements Autocontrast function from PIL using TF ops. + + Args: + image: A 3D uint8 tensor. + + Returns: + The image after it has had autocontrast applied to it and will be of type + uint8. + """ + + def scale_channel(image: tf.Tensor) -> tf.Tensor: + """Scale the 2D image using the autocontrast rule.""" + # A possibly cheaper version can be done using cumsum/unique_with_counts + # over the histogram values, rather than iterating over the entire image. + # to compute mins and maxes. + lo = tf.cast(tf.reduce_min(image), tf.float32) + hi = tf.cast(tf.reduce_max(image), tf.float32) + + # Scale the image, making the lowest value 0 and the highest value 255. + def scale_values(im): + scale = 255.0 / (hi - lo) + offset = -lo * scale + im = tf.cast(im, tf.float32) * scale + offset + im = tf.clip_by_value(im, 0.0, 255.0) + return tf.cast(im, tf.uint8) + + result = tf.cond(hi > lo, lambda: scale_values(image), lambda: image) + return result + + # Assumes RGB for now. Scales each channel independently + # and then stacks the result. + s1 = scale_channel(image[:, :, 0]) + s2 = scale_channel(image[:, :, 1]) + s3 = scale_channel(image[:, :, 2]) + image = tf.stack([s1, s2, s3], 2) + return image + + +def sharpness(image: tf.Tensor, factor: float) -> tf.Tensor: + """Implements Sharpness function from PIL using TF ops.""" + orig_image = image + image = tf.cast(image, tf.float32) + # Make image 4D for conv operation. + image = tf.expand_dims(image, 0) + # SMOOTH PIL Kernel. + kernel = tf.constant( + [[1, 1, 1], [1, 5, 1], [1, 1, 1]], dtype=tf.float32, + shape=[3, 3, 1, 1]) / 13. + # Tile across channel dimension. + kernel = tf.tile(kernel, [1, 1, 3, 1]) + strides = [1, 1, 1, 1] + degenerate = tf.nn.depthwise_conv2d( + image, kernel, strides, padding='VALID', dilations=[1, 1]) + degenerate = tf.clip_by_value(degenerate, 0.0, 255.0) + degenerate = tf.squeeze(tf.cast(degenerate, tf.uint8), [0]) + + # For the borders of the resulting image, fill in the values of the + # original image. + mask = tf.ones_like(degenerate) + padded_mask = tf.pad(mask, [[1, 1], [1, 1], [0, 0]]) + padded_degenerate = tf.pad(degenerate, [[1, 1], [1, 1], [0, 0]]) + result = tf.where(tf.equal(padded_mask, 1), padded_degenerate, orig_image) + + # Blend the final result. + return blend(result, orig_image, factor) + + +def equalize(image: tf.Tensor) -> tf.Tensor: + """Implements Equalize function from PIL using TF ops.""" + def scale_channel(im, c): + """Scale the data in the channel to implement equalize.""" + im = tf.cast(im[:, :, c], tf.int32) + # Compute the histogram of the image channel. + histo = tf.histogram_fixed_width(im, [0, 255], nbins=256) + + # For the purposes of computing the step, filter out the nonzeros. + nonzero = tf.where(tf.not_equal(histo, 0)) + nonzero_histo = tf.reshape(tf.gather(histo, nonzero), [-1]) + step = (tf.reduce_sum(nonzero_histo) - nonzero_histo[-1]) // 255 + + def build_lut(histo, step): + # Compute the cumulative sum, shifting by step // 2 + # and then normalization by step. + lut = (tf.cumsum(histo) + (step // 2)) // step + # Shift lut, prepending with 0. + lut = tf.concat([[0], lut[:-1]], 0) + # Clip the counts to be in range. This is done + # in the C code for image.point. + return tf.clip_by_value(lut, 0, 255) + + # If step is zero, return the original image. Otherwise, build + # lut from the full histogram and step and then index from it. + result = tf.cond(tf.equal(step, 0), + lambda: im, + lambda: tf.gather(build_lut(histo, step), im)) + + return tf.cast(result, tf.uint8) + + # Assumes RGB for now. Scales each channel independently + # and then stacks the result. + s1 = scale_channel(image, 0) + s2 = scale_channel(image, 1) + s3 = scale_channel(image, 2) + image = tf.stack([s1, s2, s3], 2) + return image + + +def invert(image: tf.Tensor) -> tf.Tensor: + """Inverts the image pixels.""" + image = tf.convert_to_tensor(image) + return 255 - image + + +def wrap(image: tf.Tensor) -> tf.Tensor: + """Returns 'image' with an extra channel set to all 1s.""" + shape = tf.shape(image) + extended_channel = tf.ones([shape[0], shape[1], 1], image.dtype) + extended = tf.concat([image, extended_channel], axis=2) + return extended + + +def unwrap(image: tf.Tensor, replace: int) -> tf.Tensor: + """Unwraps an image produced by wrap. + + Where there is a 0 in the last channel for every spatial position, + the rest of the three channels in that spatial dimension are grayed + (set to 128). Operations like translate and shear on a wrapped + Tensor will leave 0s in empty locations. Some transformations look + at the intensity of values to do preprocessing, and we want these + empty pixels to assume the 'average' value, rather than pure black. + + + Args: + image: A 3D Image Tensor with 4 channels. + replace: A one or three value 1D tensor to fill empty pixels. + + Returns: + image: A 3D image Tensor with 3 channels. + """ + image_shape = tf.shape(image) + # Flatten the spatial dimensions. + flattened_image = tf.reshape(image, [-1, image_shape[2]]) + + # Find all pixels where the last channel is zero. + alpha_channel = tf.expand_dims(flattened_image[:, 3], axis=-1) + + replace = tf.concat([replace, tf.ones([1], image.dtype)], 0) + + # Where they are zero, fill them in with 'replace'. + flattened_image = tf.where( + tf.equal(alpha_channel, 0), + tf.ones_like(flattened_image, dtype=image.dtype) * replace, + flattened_image) + + image = tf.reshape(flattened_image, image_shape) + image = tf.slice(image, [0, 0, 0], [image_shape[0], image_shape[1], 3]) + return image + + +def _randomly_negate_tensor(tensor): + """With 50% prob turn the tensor negative.""" + should_flip = tf.cast(tf.floor(tf.random.uniform([]) + 0.5), tf.bool) + final_tensor = tf.cond(should_flip, lambda: tensor, lambda: -tensor) + return final_tensor + + +def _rotate_level_to_arg(level: float): + level = (level/_MAX_LEVEL) * 30. + level = _randomly_negate_tensor(level) + return (level,) + + +def _shrink_level_to_arg(level: float): + """Converts level to ratio by which we shrink the image content.""" + if level == 0: + return (1.0,) # if level is zero, do not shrink the image + # Maximum shrinking ratio is 2.9. + level = 2. / (_MAX_LEVEL / level) + 0.9 + return (level,) + + +def _enhance_level_to_arg(level: float): + return ((level/_MAX_LEVEL) * 1.8 + 0.1,) + + +def _shear_level_to_arg(level: float): + level = (level/_MAX_LEVEL) * 0.3 + # Flip level to negative with 50% chance. + level = _randomly_negate_tensor(level) + return (level,) + + +def _translate_level_to_arg(level: float, translate_const: float): + level = (level/_MAX_LEVEL) * float(translate_const) + # Flip level to negative with 50% chance. + level = _randomly_negate_tensor(level) + return (level,) + + +def _mult_to_arg(level: float, multiplier: float = 1.): + return (int((level / _MAX_LEVEL) * multiplier),) + + +def _apply_func_with_prob(func: Any, + image: tf.Tensor, + args: Any, + prob: float): + """Apply `func` to image w/ `args` as input with probability `prob`.""" + assert isinstance(args, tuple) + + # Apply the function with probability `prob`. + should_apply_op = tf.cast( + tf.floor(tf.random.uniform([], dtype=tf.float32) + prob), tf.bool) + augmented_image = tf.cond( + should_apply_op, + lambda: func(image, *args), + lambda: image) + return augmented_image + + +def select_and_apply_random_policy(policies: Any, image: tf.Tensor): + """Select a random policy from `policies` and apply it to `image`.""" + policy_to_select = tf.random.uniform([], maxval=len(policies), dtype=tf.int32) + # Note that using tf.case instead of tf.conds would result in significantly + # larger graphs and would even break export for some larger policies. + for (i, policy) in enumerate(policies): + image = tf.cond( + tf.equal(i, policy_to_select), + lambda selected_policy=policy: selected_policy(image), + lambda: image) + return image + + +NAME_TO_FUNC = { + 'AutoContrast': autocontrast, + 'Equalize': equalize, + 'Invert': invert, + 'Rotate': wrapped_rotate, + 'Posterize': posterize, + 'Solarize': solarize, + 'SolarizeAdd': solarize_add, + 'Color': color, + 'Contrast': contrast, + 'Brightness': brightness, + 'Sharpness': sharpness, + 'ShearX': shear_x, + 'ShearY': shear_y, + 'TranslateX': translate_x, + 'TranslateY': translate_y, + 'Cutout': cutout, +} + +# Functions that have a 'replace' parameter +REPLACE_FUNCS = frozenset({ + 'Rotate', + 'TranslateX', + 'ShearX', + 'ShearY', + 'TranslateY', + 'Cutout', +}) + + +def level_to_arg(cutout_const: float, translate_const: float): + """Creates a dict mapping image operation names to their arguments.""" + + no_arg = lambda level: () + posterize_arg = lambda level: _mult_to_arg(level, 4) + solarize_arg = lambda level: _mult_to_arg(level, 256) + solarize_add_arg = lambda level: _mult_to_arg(level, 110) + cutout_arg = lambda level: _mult_to_arg(level, cutout_const) + translate_arg = lambda level: _translate_level_to_arg(level, translate_const) + + args = { + 'AutoContrast': no_arg, + 'Equalize': no_arg, + 'Invert': no_arg, + 'Rotate': _rotate_level_to_arg, + 'Posterize': posterize_arg, + 'Solarize': solarize_arg, + 'SolarizeAdd': solarize_add_arg, + 'Color': _enhance_level_to_arg, + 'Contrast': _enhance_level_to_arg, + 'Brightness': _enhance_level_to_arg, + 'Sharpness': _enhance_level_to_arg, + 'ShearX': _shear_level_to_arg, + 'ShearY': _shear_level_to_arg, + 'Cutout': cutout_arg, + 'TranslateX': translate_arg, + 'TranslateY': translate_arg, + } + return args + + +def _parse_policy_info(name: Text, + prob: float, + level: float, + replace_value: List[int], + cutout_const: float, + translate_const: float) -> Tuple[Any, float, Any]: + """Return the function that corresponds to `name` and update `level` param.""" + func = NAME_TO_FUNC[name] + args = level_to_arg(cutout_const, translate_const)[name](level) + + if name in REPLACE_FUNCS: + # Add in replace arg if it is required for the function that is called. + args = tuple(list(args) + [replace_value]) + + return func, prob, args + + +class ImageAugment(object): + """Image augmentation class for applying image distortions.""" + + def distort(self, image: tf.Tensor) -> tf.Tensor: + """Given an image tensor, returns a distorted image with the same shape. + + Args: + image: `Tensor` of shape [height, width, 3] representing an image. + + Returns: + The augmented version of `image`. + """ + raise NotImplementedError() + + +class AutoAugment(ImageAugment): + """Applies the AutoAugment policy to images. + + AutoAugment is from the paper: https://arxiv.org/abs/1805.09501. + """ + + def __init__(self, + augmentation_name: Text = 'v0', + policies: Optional[Dict[Text, Any]] = None, + cutout_const: float = 100, + translate_const: float = 250): + """Applies the AutoAugment policy to images. + + Args: + augmentation_name: The name of the AutoAugment policy to use. The + available options are `v0` and `test`. `v0` is the policy used for all + of the results in the paper and was found to achieve the best results on + the COCO dataset. `v1`, `v2` and `v3` are additional good policies found + on the COCO dataset that have slight variation in what operations were + used during the search procedure along with how many operations are + applied in parallel to a single image (2 vs 3). + policies: list of lists of tuples in the form `(func, prob, level)`, + `func` is a string name of the augmentation function, `prob` is the + probability of applying the `func` operation, `level` is the input + argument for `func`. + cutout_const: multiplier for applying cutout. + translate_const: multiplier for applying translation. + """ + super(AutoAugment, self).__init__() + + if policies is None: + self.available_policies = { + 'v0': self.policy_v0(), + 'test': self.policy_test(), + 'simple': self.policy_simple(), + } + + if augmentation_name not in self.available_policies: + raise ValueError( + 'Invalid augmentation_name: {}'.format(augmentation_name)) + + self.augmentation_name = augmentation_name + self.policies = self.available_policies[augmentation_name] + self.cutout_const = float(cutout_const) + self.translate_const = float(translate_const) + + def distort(self, image: tf.Tensor) -> tf.Tensor: + """Applies the AutoAugment policy to `image`. + + AutoAugment is from the paper: https://arxiv.org/abs/1805.09501. + + Args: + image: `Tensor` of shape [height, width, 3] representing an image. + + Returns: + A version of image that now has data augmentation applied to it based on + the `policies` pass into the function. + """ + input_image_type = image.dtype + + if input_image_type != tf.uint8: + image = tf.clip_by_value(image, 0.0, 255.0) + image = tf.cast(image, dtype=tf.uint8) + + replace_value = [128] * 3 + + # func is the string name of the augmentation function, prob is the + # probability of applying the operation and level is the parameter + # associated with the tf op. + + # tf_policies are functions that take in an image and return an augmented + # image. + tf_policies = [] + for policy in self.policies: + tf_policy = [] + # Link string name to the correct python function and make sure the + # correct argument is passed into that function. + for policy_info in policy: + policy_info = list(policy_info) + [ + replace_value, self.cutout_const, self.translate_const + ] + tf_policy.append(_parse_policy_info(*policy_info)) + # Now build the tf policy that will apply the augmentation procedue + # on image. + def make_final_policy(tf_policy_): + + def final_policy(image_): + for func, prob, args in tf_policy_: + image_ = _apply_func_with_prob(func, image_, args, prob) + return image_ + + return final_policy + + tf_policies.append(make_final_policy(tf_policy)) + + image = select_and_apply_random_policy(tf_policies, image) + image = tf.cast(image, dtype=input_image_type) + return image + + @staticmethod + def policy_v0(): + """Autoaugment policy that was used in AutoAugment Paper. + + Each tuple is an augmentation operation of the form + (operation, probability, magnitude). Each element in policy is a + sub-policy that will be applied sequentially on the image. + + Returns: + the policy. + """ + + # TODO(dankondratyuk): tensorflow_addons defines custom ops, which + # for some reason are not included when building/linking + # This results in the error, "Op type not registered + # 'Addons>ImageProjectiveTransformV2' in binary" when running on borg TPUs + policy = [ + [('Equalize', 0.8, 1), ('ShearY', 0.8, 4)], + [('Color', 0.4, 9), ('Equalize', 0.6, 3)], + [('Color', 0.4, 1), ('Rotate', 0.6, 8)], + [('Solarize', 0.8, 3), ('Equalize', 0.4, 7)], + [('Solarize', 0.4, 2), ('Solarize', 0.6, 2)], + [('Color', 0.2, 0), ('Equalize', 0.8, 8)], + [('Equalize', 0.4, 8), ('SolarizeAdd', 0.8, 3)], + [('ShearX', 0.2, 9), ('Rotate', 0.6, 8)], + [('Color', 0.6, 1), ('Equalize', 1.0, 2)], + [('Invert', 0.4, 9), ('Rotate', 0.6, 0)], + [('Equalize', 1.0, 9), ('ShearY', 0.6, 3)], + [('Color', 0.4, 7), ('Equalize', 0.6, 0)], + [('Posterize', 0.4, 6), ('AutoContrast', 0.4, 7)], + [('Solarize', 0.6, 8), ('Color', 0.6, 9)], + [('Solarize', 0.2, 4), ('Rotate', 0.8, 9)], + [('Rotate', 1.0, 7), ('TranslateY', 0.8, 9)], + [('ShearX', 0.0, 0), ('Solarize', 0.8, 4)], + [('ShearY', 0.8, 0), ('Color', 0.6, 4)], + [('Color', 1.0, 0), ('Rotate', 0.6, 2)], + [('Equalize', 0.8, 4), ('Equalize', 0.0, 8)], + [('Equalize', 1.0, 4), ('AutoContrast', 0.6, 2)], + [('ShearY', 0.4, 7), ('SolarizeAdd', 0.6, 7)], + [('Posterize', 0.8, 2), ('Solarize', 0.6, 10)], + [('Solarize', 0.6, 8), ('Equalize', 0.6, 1)], + [('Color', 0.8, 6), ('Rotate', 0.4, 5)], + ] + return policy + + @staticmethod + def policy_simple(): + """Same as `policy_v0`, except with custom ops removed.""" + + policy = [ + [('Color', 0.4, 9), ('Equalize', 0.6, 3)], + [('Solarize', 0.8, 3), ('Equalize', 0.4, 7)], + [('Solarize', 0.4, 2), ('Solarize', 0.6, 2)], + [('Color', 0.2, 0), ('Equalize', 0.8, 8)], + [('Equalize', 0.4, 8), ('SolarizeAdd', 0.8, 3)], + [('Color', 0.6, 1), ('Equalize', 1.0, 2)], + [('Color', 0.4, 7), ('Equalize', 0.6, 0)], + [('Posterize', 0.4, 6), ('AutoContrast', 0.4, 7)], + [('Solarize', 0.6, 8), ('Color', 0.6, 9)], + [('Equalize', 0.8, 4), ('Equalize', 0.0, 8)], + [('Equalize', 1.0, 4), ('AutoContrast', 0.6, 2)], + [('Posterize', 0.8, 2), ('Solarize', 0.6, 10)], + [('Solarize', 0.6, 8), ('Equalize', 0.6, 1)], + ] + return policy + + @staticmethod + def policy_test(): + """Autoaugment test policy for debugging.""" + policy = [ + [('TranslateX', 1.0, 4), ('Equalize', 1.0, 10)], + ] + return policy + + +class RandAugment(ImageAugment): + """Applies the RandAugment policy to images. + + RandAugment is from the paper https://arxiv.org/abs/1909.13719, + """ + + def __init__(self, + num_layers: int = 2, + magnitude: float = 10., + cutout_const: float = 40., + translate_const: float = 100.): + """Applies the RandAugment policy to images. + + Args: + num_layers: Integer, the number of augmentation transformations to apply + sequentially to an image. Represented as (N) in the paper. Usually best + values will be in the range [1, 3]. + magnitude: Integer, shared magnitude across all augmentation operations. + Represented as (M) in the paper. Usually best values are in the range + [5, 10]. + cutout_const: multiplier for applying cutout. + translate_const: multiplier for applying translation. + """ + super(RandAugment, self).__init__() + + self.num_layers = num_layers + self.magnitude = float(magnitude) + self.cutout_const = float(cutout_const) + self.translate_const = float(translate_const) + self.available_ops = [ + 'AutoContrast', 'Equalize', 'Invert', 'Rotate', 'Posterize', 'Solarize', + 'Color', 'Contrast', 'Brightness', 'Sharpness', 'ShearX', 'ShearY', + 'TranslateX', 'TranslateY', 'Cutout', 'SolarizeAdd' + ] + + def distort(self, image: tf.Tensor) -> tf.Tensor: + """Applies the RandAugment policy to `image`. + + Args: + image: `Tensor` of shape [height, width, 3] representing an image. + + Returns: + The augmented version of `image`. + """ + input_image_type = image.dtype + + if input_image_type != tf.uint8: + image = tf.clip_by_value(image, 0.0, 255.0) + image = tf.cast(image, dtype=tf.uint8) + + replace_value = [128] * 3 + min_prob, max_prob = 0.2, 0.8 + + for _ in range(self.num_layers): + op_to_select = tf.random.uniform( + [], maxval=len(self.available_ops) + 1, dtype=tf.int32) + + branch_fns = [] + for (i, op_name) in enumerate(self.available_ops): + prob = tf.random.uniform([], + minval=min_prob, + maxval=max_prob, + dtype=tf.float32) + func, _, args = _parse_policy_info(op_name, + prob, + self.magnitude, + replace_value, + self.cutout_const, + self.translate_const) + branch_fns.append(( + i, + # pylint:disable=g-long-lambda + lambda selected_func=func, selected_args=args: selected_func( + image, *selected_args))) + # pylint:enable=g-long-lambda + + image = tf.switch_case(branch_index=op_to_select, + branch_fns=branch_fns, + default=lambda: tf.identity(image)) + + image = tf.cast(image, dtype=input_image_type) + return image diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/augment_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/augment_test.py new file mode 100644 index 0000000..76bdb2b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/augment_test.py @@ -0,0 +1,143 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for autoaugment.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +from absl.testing import parameterized + +import tensorflow as tf + +from official.vision.image_classification import augment + + +def get_dtype_test_cases(): + return [ + ('uint8', tf.uint8), + ('int32', tf.int32), + ('float16', tf.float16), + ('float32', tf.float32), + ] + + +@parameterized.named_parameters(get_dtype_test_cases()) +class TransformsTest(parameterized.TestCase, tf.test.TestCase): + """Basic tests for fundamental transformations.""" + + def test_to_from_4d(self, dtype): + for shape in [(10, 10), (10, 10, 10), (10, 10, 10, 10)]: + original_ndims = len(shape) + image = tf.zeros(shape, dtype=dtype) + image_4d = augment.to_4d(image) + self.assertEqual(4, tf.rank(image_4d)) + self.assertAllEqual(image, augment.from_4d(image_4d, original_ndims)) + + def test_transform(self, dtype): + image = tf.constant([[1, 2], [3, 4]], dtype=dtype) + self.assertAllEqual(augment.transform(image, transforms=[1]*8), + [[4, 4], [4, 4]]) + + def test_translate(self, dtype): + image = tf.constant( + [[1, 0, 1, 0], + [0, 1, 0, 1], + [1, 0, 1, 0], + [0, 1, 0, 1]], + dtype=dtype) + translations = [-1, -1] + translated = augment.translate(image=image, + translations=translations) + expected = [ + [1, 0, 1, 1], + [0, 1, 0, 0], + [1, 0, 1, 1], + [1, 0, 1, 1]] + self.assertAllEqual(translated, expected) + + def test_translate_shapes(self, dtype): + translation = [0, 0] + for shape in [(3, 3), (5, 5), (224, 224, 3)]: + image = tf.zeros(shape, dtype=dtype) + self.assertAllEqual(image, augment.translate(image, translation)) + + def test_translate_invalid_translation(self, dtype): + image = tf.zeros((1, 1), dtype=dtype) + invalid_translation = [[[1, 1]]] + with self.assertRaisesRegex(TypeError, 'rank 1 or 2'): + _ = augment.translate(image, invalid_translation) + + def test_rotate(self, dtype): + image = tf.reshape(tf.cast(tf.range(9), dtype), (3, 3)) + rotation = 90. + transformed = augment.rotate(image=image, degrees=rotation) + expected = [[2, 5, 8], + [1, 4, 7], + [0, 3, 6]] + self.assertAllEqual(transformed, expected) + + def test_rotate_shapes(self, dtype): + degrees = 0. + for shape in [(3, 3), (5, 5), (224, 224, 3)]: + image = tf.zeros(shape, dtype=dtype) + self.assertAllEqual(image, augment.rotate(image, degrees)) + + +class AutoaugmentTest(tf.test.TestCase): + + def test_autoaugment(self): + """Smoke test to be sure there are no syntax errors.""" + image = tf.zeros((224, 224, 3), dtype=tf.uint8) + + augmenter = augment.AutoAugment() + aug_image = augmenter.distort(image) + + self.assertEqual((224, 224, 3), aug_image.shape) + + def test_randaug(self): + """Smoke test to be sure there are no syntax errors.""" + image = tf.zeros((224, 224, 3), dtype=tf.uint8) + + augmenter = augment.RandAugment() + aug_image = augmenter.distort(image) + + self.assertEqual((224, 224, 3), aug_image.shape) + + def test_all_policy_ops(self): + """Smoke test to be sure all augmentation functions can execute.""" + + prob = 1 + magnitude = 10 + replace_value = [128] * 3 + cutout_const = 100 + translate_const = 250 + + image = tf.ones((224, 224, 3), dtype=tf.uint8) + + for op_name in augment.NAME_TO_FUNC: + func, _, args = augment._parse_policy_info(op_name, + prob, + magnitude, + replace_value, + cutout_const, + translate_const) + image = func(image, *args) + + self.assertEqual((224, 224, 3), image.shape) + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/callbacks.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/callbacks.py new file mode 100644 index 0000000..985d0c6 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/callbacks.py @@ -0,0 +1,258 @@ +# Lint as: python3 +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Common modules for callbacks.""" +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import os +from typing import Any, List, MutableMapping, Text +from absl import logging +import tensorflow as tf + +from official.utils.misc import keras_utils +from official.vision.image_classification import optimizer_factory + + +def get_callbacks(model_checkpoint: bool = True, + include_tensorboard: bool = True, + time_history: bool = True, + track_lr: bool = True, + write_model_weights: bool = True, + apply_moving_average: bool = False, + initial_step: int = 0, + batch_size: int = 0, + log_steps: int = 0, + model_dir: str = None) -> List[tf.keras.callbacks.Callback]: + """Get all callbacks.""" + model_dir = model_dir or '' + callbacks = [] + if model_checkpoint: + ckpt_full_path = os.path.join(model_dir, 'model.ckpt-{epoch:04d}') + callbacks.append(tf.keras.callbacks.ModelCheckpoint( + ckpt_full_path, save_weights_only=True, verbose=1)) + if include_tensorboard: + callbacks.append( + CustomTensorBoard( + log_dir=model_dir, + track_lr=track_lr, + initial_step=initial_step, + write_images=write_model_weights)) + if time_history: + callbacks.append( + keras_utils.TimeHistory( + batch_size, + log_steps, + logdir=model_dir if include_tensorboard else None)) + if apply_moving_average: + # Save moving average model to a different file so that + # we can resume training from a checkpoint + ckpt_full_path = os.path.join( + model_dir, 'average', 'model.ckpt-{epoch:04d}') + callbacks.append(AverageModelCheckpoint( + update_weights=False, + filepath=ckpt_full_path, + save_weights_only=True, + verbose=1)) + callbacks.append(MovingAverageCallback()) + return callbacks + + +def get_scalar_from_tensor(t: tf.Tensor) -> int: + """Utility function to convert a Tensor to a scalar.""" + t = tf.keras.backend.get_value(t) + if callable(t): + return t() + else: + return t + + +class CustomTensorBoard(tf.keras.callbacks.TensorBoard): + """A customized TensorBoard callback that tracks additional datapoints. + + Metrics tracked: + - Global learning rate + + Attributes: + log_dir: the path of the directory where to save the log files to be parsed + by TensorBoard. + track_lr: `bool`, whether or not to track the global learning rate. + initial_step: the initial step, used for preemption recovery. + **kwargs: Additional arguments for backwards compatibility. Possible key is + `period`. + """ + + # TODO(b/146499062): track params, flops, log lr, l2 loss, + # classification loss + + def __init__(self, + log_dir: str, + track_lr: bool = False, + initial_step: int = 0, + **kwargs): + super(CustomTensorBoard, self).__init__(log_dir=log_dir, **kwargs) + self.step = initial_step + self._track_lr = track_lr + + def on_batch_begin(self, + epoch: int, + logs: MutableMapping[str, Any] = None) -> None: + self.step += 1 + if logs is None: + logs = {} + logs.update(self._calculate_metrics()) + super(CustomTensorBoard, self).on_batch_begin(epoch, logs) + + def on_epoch_begin(self, + epoch: int, + logs: MutableMapping[str, Any] = None) -> None: + if logs is None: + logs = {} + metrics = self._calculate_metrics() + logs.update(metrics) + for k, v in metrics.items(): + logging.info('Current %s: %f', k, v) + super(CustomTensorBoard, self).on_epoch_begin(epoch, logs) + + def on_epoch_end(self, + epoch: int, + logs: MutableMapping[str, Any] = None) -> None: + if logs is None: + logs = {} + metrics = self._calculate_metrics() + logs.update(metrics) + super(CustomTensorBoard, self).on_epoch_end(epoch, logs) + + def _calculate_metrics(self) -> MutableMapping[str, Any]: + logs = {} + # TODO(b/149030439): disable LR reporting. + # if self._track_lr: + # logs['learning_rate'] = self._calculate_lr() + return logs + + def _calculate_lr(self) -> int: + """Calculates the learning rate given the current step.""" + return get_scalar_from_tensor( + self._get_base_optimizer()._decayed_lr(var_dtype=tf.float32)) # pylint:disable=protected-access + + def _get_base_optimizer(self) -> tf.keras.optimizers.Optimizer: + """Get the base optimizer used by the current model.""" + + optimizer = self.model.optimizer + + # The optimizer might be wrapped by another class, so unwrap it + while hasattr(optimizer, '_optimizer'): + optimizer = optimizer._optimizer # pylint:disable=protected-access + + return optimizer + + +class MovingAverageCallback(tf.keras.callbacks.Callback): + """A Callback to be used with a `MovingAverage` optimizer. + + Applies moving average weights to the model during validation time to test + and predict on the averaged weights rather than the current model weights. + Once training is complete, the model weights will be overwritten with the + averaged weights (by default). + + Attributes: + overwrite_weights_on_train_end: Whether to overwrite the current model + weights with the averaged weights from the moving average optimizer. + **kwargs: Any additional callback arguments. + """ + + def __init__(self, + overwrite_weights_on_train_end: bool = False, + **kwargs): + super(MovingAverageCallback, self).__init__(**kwargs) + self.overwrite_weights_on_train_end = overwrite_weights_on_train_end + + def set_model(self, model: tf.keras.Model): + super(MovingAverageCallback, self).set_model(model) + assert isinstance(self.model.optimizer, + optimizer_factory.MovingAverage) + self.model.optimizer.shadow_copy(self.model) + + def on_test_begin(self, logs: MutableMapping[Text, Any] = None): + self.model.optimizer.swap_weights() + + def on_test_end(self, logs: MutableMapping[Text, Any] = None): + self.model.optimizer.swap_weights() + + def on_train_end(self, logs: MutableMapping[Text, Any] = None): + if self.overwrite_weights_on_train_end: + self.model.optimizer.assign_average_vars(self.model.variables) + + +class AverageModelCheckpoint(tf.keras.callbacks.ModelCheckpoint): + """Saves and, optionally, assigns the averaged weights. + + Taken from tfa.callbacks.AverageModelCheckpoint. + + Attributes: + update_weights: If True, assign the moving average weights + to the model, and save them. If False, keep the old + non-averaged weights, but the saved model uses the + average weights. + See `tf.keras.callbacks.ModelCheckpoint` for the other args. + """ + + def __init__( + self, + update_weights: bool, + filepath: str, + monitor: str = 'val_loss', + verbose: int = 0, + save_best_only: bool = False, + save_weights_only: bool = False, + mode: str = 'auto', + save_freq: str = 'epoch', + **kwargs): + self.update_weights = update_weights + super().__init__( + filepath, + monitor, + verbose, + save_best_only, + save_weights_only, + mode, + save_freq, + **kwargs) + + def set_model(self, model): + if not isinstance(model.optimizer, optimizer_factory.MovingAverage): + raise TypeError( + 'AverageModelCheckpoint is only used when training' + 'with MovingAverage') + return super().set_model(model) + + def _save_model(self, epoch, logs): + assert isinstance(self.model.optimizer, optimizer_factory.MovingAverage) + + if self.update_weights: + self.model.optimizer.assign_average_vars(self.model.variables) + return super()._save_model(epoch, logs) + else: + # Note: `model.get_weights()` gives us the weights (non-ref) + # whereas `model.variables` returns references to the variables. + non_avg_weights = self.model.get_weights() + self.model.optimizer.assign_average_vars(self.model.variables) + # result is currently None, since `super._save_model` doesn't + # return anything, but this may change in the future. + result = super()._save_model(epoch, logs) + self.model.set_weights(non_avg_weights) + return result diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/classifier_trainer.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/classifier_trainer.py new file mode 100644 index 0000000..c3805a7 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/classifier_trainer.py @@ -0,0 +1,458 @@ +# Lint as: python3 +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Runs an Image Classification model.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +import pprint +from typing import Any, Tuple, Text, Optional, Mapping + +from absl import app +from absl import flags +from absl import logging +import tensorflow as tf + +from official.modeling import performance +from official.modeling.hyperparams import params_dict +from official.utils import hyperparams_flags +from official.utils.logs import logger +from official.utils.misc import distribution_utils +from official.utils.misc import keras_utils +from official.vision.image_classification import callbacks as custom_callbacks +from official.vision.image_classification import dataset_factory +from official.vision.image_classification import optimizer_factory +from official.vision.image_classification.configs import base_configs +from official.vision.image_classification.configs import configs +from official.vision.image_classification.efficientnet import efficientnet_model +from official.vision.image_classification.resnet import common +from official.vision.image_classification.resnet import resnet_model + + +def get_models() -> Mapping[str, tf.keras.Model]: + """Returns the mapping from model type name to Keras model.""" + return { + 'efficientnet': efficientnet_model.EfficientNet.from_name, + 'resnet': resnet_model.resnet50, + } + + +def get_dtype_map() -> Mapping[str, tf.dtypes.DType]: + """Returns the mapping from dtype string representations to TF dtypes.""" + return { + 'float32': tf.float32, + 'bfloat16': tf.bfloat16, + 'float16': tf.float16, + 'fp32': tf.float32, + 'bf16': tf.bfloat16, + } + + +def _get_metrics(one_hot: bool) -> Mapping[Text, Any]: + """Get a dict of available metrics to track.""" + if one_hot: + return { + # (name, metric_fn) + 'acc': tf.keras.metrics.CategoricalAccuracy(name='accuracy'), + 'accuracy': tf.keras.metrics.CategoricalAccuracy(name='accuracy'), + 'top_1': tf.keras.metrics.CategoricalAccuracy(name='accuracy'), + 'top_5': tf.keras.metrics.TopKCategoricalAccuracy( + k=5, + name='top_5_accuracy'), + } + else: + return { + # (name, metric_fn) + 'acc': tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy'), + 'accuracy': tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy'), + 'top_1': tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy'), + 'top_5': tf.keras.metrics.SparseTopKCategoricalAccuracy( + k=5, + name='top_5_accuracy'), + } + + +def get_image_size_from_model( + params: base_configs.ExperimentConfig) -> Optional[int]: + """If the given model has a preferred image size, return it.""" + if params.model_name == 'efficientnet': + efficientnet_name = params.model.model_params.model_name + if efficientnet_name in efficientnet_model.MODEL_CONFIGS: + return efficientnet_model.MODEL_CONFIGS[efficientnet_name].resolution + return None + + +def _get_dataset_builders(params: base_configs.ExperimentConfig, + strategy: tf.distribute.Strategy, + one_hot: bool + ) -> Tuple[Any, Any, Any]: + """Create and return train, validation, and test dataset builders.""" + if one_hot: + logging.warning('label_smoothing > 0, so datasets will be one hot encoded.') + else: + logging.warning('label_smoothing not applied, so datasets will not be one ' + 'hot encoded.') + + num_devices = strategy.num_replicas_in_sync if strategy else 1 + + image_size = get_image_size_from_model(params) + + dataset_configs = [ + params.train_dataset, params.validation_dataset + ] + builders = [] + + for config in dataset_configs: + if config is not None and config.has_data: + builder = dataset_factory.DatasetBuilder( + config, + image_size=image_size or config.image_size, + num_devices=num_devices, + one_hot=one_hot) + else: + builder = None + builders.append(builder) + + return builders + + +def get_loss_scale(params: base_configs.ExperimentConfig, + fp16_default: float = 128.) -> float: + """Returns the loss scale for initializations.""" + loss_scale = params.runtime.loss_scale + if loss_scale == 'dynamic': + return loss_scale + elif loss_scale is not None: + return float(loss_scale) + elif (params.train_dataset.dtype == 'float32' or + params.train_dataset.dtype == 'bfloat16'): + return 1. + else: + assert params.train_dataset.dtype == 'float16' + return fp16_default + + +def _get_params_from_flags(flags_obj: flags.FlagValues): + """Get ParamsDict from flags.""" + model = flags_obj.model_type.lower() + dataset = flags_obj.dataset.lower() + params = configs.get_config(model=model, dataset=dataset) + + flags_overrides = { + 'model_dir': flags_obj.model_dir, + 'mode': flags_obj.mode, + 'model': { + 'name': model, + }, + 'runtime': { + 'run_eagerly': flags_obj.run_eagerly, + 'tpu': flags_obj.tpu, + }, + 'train_dataset': { + 'data_dir': flags_obj.data_dir, + }, + 'validation_dataset': { + 'data_dir': flags_obj.data_dir, + }, + 'train': { + 'time_history': { + 'log_steps': flags_obj.log_steps, + }, + }, + } + + overriding_configs = (flags_obj.config_file, + flags_obj.params_override, + flags_overrides) + + pp = pprint.PrettyPrinter() + + logging.info('Base params: %s', pp.pformat(params.as_dict())) + + for param in overriding_configs: + logging.info('Overriding params: %s', param) + # Set is_strict to false because we can have dynamic dict parameters. + params = params_dict.override_params_dict(params, param, is_strict=False) + + params.validate() + params.lock() + + logging.info('Final model parameters: %s', pp.pformat(params.as_dict())) + return params + + +def resume_from_checkpoint(model: tf.keras.Model, + model_dir: str, + train_steps: int) -> int: + """Resumes from the latest checkpoint, if possible. + + Loads the model weights and optimizer settings from a checkpoint. + This function should be used in case of preemption recovery. + + Args: + model: The model whose weights should be restored. + model_dir: The directory where model weights were saved. + train_steps: The number of steps to train. + + Returns: + The epoch of the latest checkpoint, or 0 if not restoring. + + """ + logging.info('Load from checkpoint is enabled.') + latest_checkpoint = tf.train.latest_checkpoint(model_dir) + logging.info('latest_checkpoint: %s', latest_checkpoint) + if not latest_checkpoint: + logging.info('No checkpoint detected.') + return 0 + + logging.info('Checkpoint file %s found and restoring from ' + 'checkpoint', latest_checkpoint) + model.load_weights(latest_checkpoint) + initial_epoch = model.optimizer.iterations // train_steps + logging.info('Completed loading from checkpoint.') + logging.info('Resuming from epoch %d', initial_epoch) + return int(initial_epoch) + + +def initialize(params: base_configs.ExperimentConfig, + dataset_builder: dataset_factory.DatasetBuilder): + """Initializes backend related initializations.""" + keras_utils.set_session_config( + enable_xla=params.runtime.enable_xla) + if params.runtime.gpu_thread_mode: + keras_utils.set_gpu_thread_mode_and_count( + per_gpu_thread_count=params.runtime.per_gpu_thread_count, + gpu_thread_mode=params.runtime.gpu_thread_mode, + num_gpus=params.runtime.num_gpus, + datasets_num_private_threads=params.runtime.dataset_num_private_threads) + + performance.set_mixed_precision_policy(dataset_builder.dtype, + get_loss_scale(params)) + if tf.config.list_physical_devices('GPU'): + data_format = 'channels_first' + else: + data_format = 'channels_last' + tf.keras.backend.set_image_data_format(data_format) + distribution_utils.configure_cluster( + params.runtime.worker_hosts, + params.runtime.task_index) + if params.runtime.run_eagerly: + # Enable eager execution to allow step-by-step debugging + tf.config.experimental_run_functions_eagerly(True) + + +def define_classifier_flags(): + """Defines common flags for image classification.""" + hyperparams_flags.initialize_common_flags() + flags.DEFINE_string( + 'data_dir', + default=None, + help='The location of the input data.') + flags.DEFINE_string( + 'mode', + default=None, + help='Mode to run: `train`, `eval`, `train_and_eval` or `export`.') + flags.DEFINE_bool( + 'run_eagerly', + default=None, + help='Use eager execution and disable autograph for debugging.') + flags.DEFINE_string( + 'model_type', + default=None, + help='The type of the model, e.g. EfficientNet, etc.') + flags.DEFINE_string( + 'dataset', + default=None, + help='The name of the dataset, e.g. ImageNet, etc.') + flags.DEFINE_integer( + 'log_steps', + default=100, + help='The interval of steps between logging of batch level stats.') + + +def serialize_config(params: base_configs.ExperimentConfig, + model_dir: str): + """Serializes and saves the experiment config.""" + params_save_path = os.path.join(model_dir, 'params.yaml') + logging.info('Saving experiment configuration to %s', params_save_path) + tf.io.gfile.makedirs(model_dir) + params_dict.save_params_dict_to_yaml(params, params_save_path) + + +def train_and_eval( + params: base_configs.ExperimentConfig, + strategy_override: tf.distribute.Strategy) -> Mapping[str, Any]: + """Runs the train and eval path using compile/fit.""" + logging.info('Running train and eval.') + + # Note: for TPUs, strategy and scope should be created before the dataset + strategy = strategy_override or distribution_utils.get_distribution_strategy( + distribution_strategy=params.runtime.distribution_strategy, + all_reduce_alg=params.runtime.all_reduce_alg, + num_gpus=params.runtime.num_gpus, + tpu_address=params.runtime.tpu) + + strategy_scope = distribution_utils.get_strategy_scope(strategy) + + logging.info('Detected %d devices.', + strategy.num_replicas_in_sync if strategy else 1) + + label_smoothing = params.model.loss.label_smoothing + one_hot = label_smoothing and label_smoothing > 0 + + builders = _get_dataset_builders(params, strategy, one_hot) + datasets = [builder.build() if builder else None for builder in builders] + + # Unpack datasets and builders based on train/val/test splits + train_builder, validation_builder = builders # pylint: disable=unbalanced-tuple-unpacking + train_dataset, validation_dataset = datasets + + train_epochs = params.train.epochs + train_steps = params.train.steps or train_builder.num_steps + validation_steps = params.evaluation.steps or validation_builder.num_steps + + initialize(params, train_builder) + + logging.info('Global batch size: %d', train_builder.global_batch_size) + + with strategy_scope: + model_params = params.model.model_params.as_dict() + model = get_models()[params.model.name](**model_params) + learning_rate = optimizer_factory.build_learning_rate( + params=params.model.learning_rate, + batch_size=train_builder.global_batch_size, + train_steps=train_steps) + optimizer = optimizer_factory.build_optimizer( + optimizer_name=params.model.optimizer.name, + base_learning_rate=learning_rate, + params=params.model.optimizer.as_dict()) + + metrics_map = _get_metrics(one_hot) + metrics = [metrics_map[metric] for metric in params.train.metrics] + + if one_hot: + loss_obj = tf.keras.losses.CategoricalCrossentropy( + label_smoothing=params.model.loss.label_smoothing) + else: + loss_obj = tf.keras.losses.SparseCategoricalCrossentropy() + model.compile(optimizer=optimizer, + loss=loss_obj, + metrics=metrics) + + initial_epoch = 0 + if params.train.resume_checkpoint: + initial_epoch = resume_from_checkpoint(model=model, + model_dir=params.model_dir, + train_steps=train_steps) + + callbacks = custom_callbacks.get_callbacks( + model_checkpoint=params.train.callbacks.enable_checkpoint_and_export, + include_tensorboard=params.train.callbacks.enable_tensorboard, + time_history=params.train.callbacks.enable_time_history, + track_lr=params.train.tensorboard.track_lr, + write_model_weights=params.train.tensorboard.write_model_weights, + initial_step=initial_epoch * train_steps, + batch_size=train_builder.global_batch_size, + log_steps=params.train.time_history.log_steps, + model_dir=params.model_dir) + + serialize_config(params=params, model_dir=params.model_dir) + + if params.evaluation.skip_eval: + validation_kwargs = {} + else: + validation_kwargs = { + 'validation_data': validation_dataset, + 'validation_steps': validation_steps, + 'validation_freq': params.evaluation.epochs_between_evals, + } + + history = model.fit( + train_dataset, + epochs=train_epochs, + steps_per_epoch=train_steps, + initial_epoch=initial_epoch, + callbacks=callbacks, + **validation_kwargs, + experimental_steps_per_execution=params.train.steps_per_loop, + verbose=2) + + validation_output = None + if not params.evaluation.skip_eval: + validation_output = model.evaluate( + validation_dataset, steps=validation_steps, verbose=2) + + # TODO(dankondratyuk): eval and save final test accuracy + stats = common.build_stats(history, + validation_output, + callbacks) + return stats + + +def export(params: base_configs.ExperimentConfig): + """Runs the model export functionality.""" + logging.info('Exporting model.') + model_params = params.model.model_params.as_dict() + model = get_models()[params.model.name](**model_params) + checkpoint = params.export.checkpoint + if checkpoint is None: + logging.info('No export checkpoint was provided. Using the latest ' + 'checkpoint from model_dir.') + checkpoint = tf.train.latest_checkpoint(params.model_dir) + + model.load_weights(checkpoint) + model.save(params.export.destination) + + +def run(flags_obj: flags.FlagValues, + strategy_override: tf.distribute.Strategy = None) -> Mapping[str, Any]: + """Runs Image Classification model using native Keras APIs. + + Args: + flags_obj: An object containing parsed flag values. + strategy_override: A `tf.distribute.Strategy` object to use for model. + + Returns: + Dictionary of training/eval stats + """ + params = _get_params_from_flags(flags_obj) + if params.mode == 'train_and_eval': + return train_and_eval(params, strategy_override) + elif params.mode == 'export_only': + export(params) + else: + raise ValueError('{} is not a valid mode.'.format(params.mode)) + + +def main(_): + with logger.benchmark_context(flags.FLAGS): + stats = run(flags.FLAGS) + if stats: + logging.info('Run stats:\n%s', stats) + + +if __name__ == '__main__': + logging.set_verbosity(logging.INFO) + define_classifier_flags() + flags.mark_flag_as_required('data_dir') + flags.mark_flag_as_required('mode') + flags.mark_flag_as_required('model_type') + flags.mark_flag_as_required('dataset') + + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/classifier_trainer_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/classifier_trainer_test.py new file mode 100644 index 0000000..0c0f464 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/classifier_trainer_test.py @@ -0,0 +1,386 @@ +# Lint as: python3 +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Unit tests for the classifier trainer models.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import copy +import functools +import json + +import os +import sys + +from typing import Any, Callable, Iterable, Mapping, MutableMapping, Optional, Tuple + +from absl import flags +from absl.testing import parameterized +import tensorflow as tf + +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +from official.utils.flags import core as flags_core +from official.vision.image_classification import classifier_trainer +from official.vision.image_classification import dataset_factory +from official.vision.image_classification import test_utils +from official.vision.image_classification.configs import base_configs + +classifier_trainer.define_classifier_flags() + + +def distribution_strategy_combinations() -> Iterable[Tuple[Any, ...]]: + """Returns the combinations of end-to-end tests to run.""" + return combinations.combine( + distribution=[ + strategy_combinations.default_strategy, + strategy_combinations.tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + ], + model=[ + 'efficientnet', + 'resnet', + ], + mode='eager', + dataset=[ + 'imagenet', + ], + ) + + +def get_params_override(params_override: Mapping[str, Any]) -> str: + """Converts params_override dict to string command.""" + return '--params_override=' + json.dumps(params_override) + + +def basic_params_override(dtype: str = 'float32') -> MutableMapping[str, Any]: + """Returns a basic parameter configuration for testing.""" + return { + 'train_dataset': { + 'builder': 'synthetic', + 'use_per_replica_batch_size': True, + 'batch_size': 1, + 'image_size': 224, + 'dtype': dtype, + }, + 'validation_dataset': { + 'builder': 'synthetic', + 'batch_size': 1, + 'use_per_replica_batch_size': True, + 'image_size': 224, + 'dtype': dtype, + }, + 'train': { + 'steps': 1, + 'epochs': 1, + 'callbacks': { + 'enable_checkpoint_and_export': True, + 'enable_tensorboard': False, + }, + }, + 'evaluation': { + 'steps': 1, + }, + } + + +def get_trivial_model(num_classes: int) -> tf.keras.Model: + """Creates and compiles trivial model for ImageNet dataset.""" + model = test_utils.trivial_model(num_classes=num_classes) + lr = 0.01 + optimizer = tf.keras.optimizers.SGD(learning_rate=lr) + loss_obj = tf.keras.losses.SparseCategoricalCrossentropy() + model.compile(optimizer=optimizer, + loss=loss_obj, + run_eagerly=True) + return model + + +def get_trivial_data() -> tf.data.Dataset: + """Gets trivial data in the ImageNet size.""" + def generate_data(_) -> tf.data.Dataset: + image = tf.zeros(shape=(224, 224, 3), dtype=tf.float32) + label = tf.zeros([1], dtype=tf.int32) + return image, label + + dataset = tf.data.Dataset.range(1) + dataset = dataset.repeat() + dataset = dataset.map(generate_data, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + dataset = dataset.prefetch(buffer_size=1).batch(1) + return dataset + + +def run_end_to_end(main: Callable[[Any], None], + extra_flags: Optional[Iterable[str]] = None, + model_dir: Optional[str] = None): + """Runs the classifier trainer end-to-end.""" + extra_flags = [] if extra_flags is None else extra_flags + args = [sys.argv[0], '--model_dir', model_dir] + extra_flags + flags_core.parse_flags(argv=args) + main(flags.FLAGS) + + +class ClassifierTest(tf.test.TestCase, parameterized.TestCase): + """Unit tests for Keras models.""" + _tempdir = None + + @classmethod + def setUpClass(cls): # pylint: disable=invalid-name + super(ClassifierTest, cls).setUpClass() + + def tearDown(self): + super(ClassifierTest, self).tearDown() + tf.io.gfile.rmtree(self.get_temp_dir()) + + @combinations.generate(distribution_strategy_combinations()) + def test_end_to_end_train_and_eval(self, distribution, model, dataset): + """Test train_and_eval and export for Keras classifier models.""" + # Some parameters are not defined as flags (e.g. cannot run + # classifier_train.py --batch_size=...) by design, so use + # "--params_override=..." instead + model_dir = self.get_temp_dir() + base_flags = [ + '--data_dir=not_used', + '--model_type=' + model, + '--dataset=' + dataset, + ] + train_and_eval_flags = base_flags + [ + get_params_override(basic_params_override()), + '--mode=train_and_eval', + ] + + run = functools.partial(classifier_trainer.run, + strategy_override=distribution) + run_end_to_end(main=run, + extra_flags=train_and_eval_flags, + model_dir=model_dir) + + @combinations.generate( + combinations.combine( + distribution=[ + strategy_combinations.one_device_strategy_gpu, + ], + model=[ + 'efficientnet', + 'resnet', + ], + mode='eager', + dataset='imagenet', + dtype='float16', + )) + def test_gpu_train(self, distribution, model, dataset, dtype): + """Test train_and_eval and export for Keras classifier models.""" + # Some parameters are not defined as flags (e.g. cannot run + # classifier_train.py --batch_size=...) by design, so use + # "--params_override=..." instead + model_dir = self.get_temp_dir() + base_flags = [ + '--data_dir=not_used', + '--model_type=' + model, + '--dataset=' + dataset, + ] + train_and_eval_flags = base_flags + [ + get_params_override(basic_params_override(dtype)), + '--mode=train_and_eval', + ] + + export_params = basic_params_override() + export_path = os.path.join(model_dir, 'export') + export_params['export'] = {} + export_params['export']['destination'] = export_path + export_flags = base_flags + [ + '--mode=export_only', + get_params_override(export_params) + ] + + run = functools.partial(classifier_trainer.run, + strategy_override=distribution) + run_end_to_end(main=run, + extra_flags=train_and_eval_flags, + model_dir=model_dir) + run_end_to_end(main=run, + extra_flags=export_flags, + model_dir=model_dir) + self.assertTrue(os.path.exists(export_path)) + + @combinations.generate( + combinations.combine( + distribution=[ + strategy_combinations.tpu_strategy, + ], + model=[ + 'efficientnet', + 'resnet', + ], + mode='eager', + dataset='imagenet', + dtype='bfloat16', + )) + def test_tpu_train(self, distribution, model, dataset, dtype): + """Test train_and_eval and export for Keras classifier models.""" + # Some parameters are not defined as flags (e.g. cannot run + # classifier_train.py --batch_size=...) by design, so use + # "--params_override=..." instead + model_dir = self.get_temp_dir() + base_flags = [ + '--data_dir=not_used', + '--model_type=' + model, + '--dataset=' + dataset, + ] + train_and_eval_flags = base_flags + [ + get_params_override(basic_params_override(dtype)), + '--mode=train_and_eval', + ] + + run = functools.partial(classifier_trainer.run, + strategy_override=distribution) + run_end_to_end(main=run, + extra_flags=train_and_eval_flags, + model_dir=model_dir) + + @combinations.generate(distribution_strategy_combinations()) + def test_end_to_end_invalid_mode(self, distribution, model, dataset): + """Test the Keras EfficientNet model with `strategy`.""" + model_dir = self.get_temp_dir() + extra_flags = [ + '--data_dir=not_used', + '--mode=invalid_mode', + '--model_type=' + model, + '--dataset=' + dataset, + get_params_override(basic_params_override()), + ] + + run = functools.partial(classifier_trainer.run, + strategy_override=distribution) + with self.assertRaises(ValueError): + run_end_to_end(main=run, extra_flags=extra_flags, model_dir=model_dir) + + +class UtilTests(parameterized.TestCase, tf.test.TestCase): + """Tests for individual utility functions within classifier_trainer.py.""" + + @parameterized.named_parameters( + ('efficientnet-b0', 'efficientnet', 'efficientnet-b0', 224), + ('efficientnet-b1', 'efficientnet', 'efficientnet-b1', 240), + ('efficientnet-b2', 'efficientnet', 'efficientnet-b2', 260), + ('efficientnet-b3', 'efficientnet', 'efficientnet-b3', 300), + ('efficientnet-b4', 'efficientnet', 'efficientnet-b4', 380), + ('efficientnet-b5', 'efficientnet', 'efficientnet-b5', 456), + ('efficientnet-b6', 'efficientnet', 'efficientnet-b6', 528), + ('efficientnet-b7', 'efficientnet', 'efficientnet-b7', 600), + ('resnet', 'resnet', '', None), + ) + def test_get_model_size(self, model, model_name, expected): + config = base_configs.ExperimentConfig( + model_name=model, + model=base_configs.ModelConfig( + model_params={ + 'model_name': model_name, + }, + ) + ) + size = classifier_trainer.get_image_size_from_model(config) + self.assertEqual(size, expected) + + @parameterized.named_parameters( + ('dynamic', 'dynamic', None, 'dynamic'), + ('scalar', 128., None, 128.), + ('float32', None, 'float32', 1), + ('float16', None, 'float16', 128), + ) + def test_get_loss_scale(self, loss_scale, dtype, expected): + config = base_configs.ExperimentConfig( + runtime=base_configs.RuntimeConfig( + loss_scale=loss_scale), + train_dataset=dataset_factory.DatasetConfig(dtype=dtype)) + ls = classifier_trainer.get_loss_scale(config, fp16_default=128) + self.assertEqual(ls, expected) + + @parameterized.named_parameters( + ('float16', 'float16'), + ('bfloat16', 'bfloat16') + ) + def test_initialize(self, dtype): + config = base_configs.ExperimentConfig( + runtime=base_configs.RuntimeConfig( + run_eagerly=False, + enable_xla=False, + per_gpu_thread_count=1, + gpu_thread_mode='gpu_private', + num_gpus=1, + dataset_num_private_threads=1, + ), + train_dataset=dataset_factory.DatasetConfig(dtype=dtype), + model=base_configs.ModelConfig(), + ) + + class EmptyClass: + pass + fake_ds_builder = EmptyClass() + fake_ds_builder.dtype = dtype + fake_ds_builder.config = EmptyClass() + classifier_trainer.initialize(config, fake_ds_builder) + + def test_resume_from_checkpoint(self): + """Tests functionality for resuming from checkpoint.""" + # Set the keras policy + policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16') + tf.keras.mixed_precision.experimental.set_policy(policy) + + # Get the model, datasets, and compile it. + model = get_trivial_model(10) + + # Create the checkpoint + model_dir = self.get_temp_dir() + train_epochs = 1 + train_steps = 10 + ds = get_trivial_data() + callbacks = [ + tf.keras.callbacks.ModelCheckpoint( + os.path.join(model_dir, 'model.ckpt-{epoch:04d}'), + save_weights_only=True) + ] + model.fit( + ds, + callbacks=callbacks, + epochs=train_epochs, + steps_per_epoch=train_steps) + + # Test load from checkpoint + clean_model = get_trivial_model(10) + weights_before_load = copy.deepcopy(clean_model.get_weights()) + initial_epoch = classifier_trainer.resume_from_checkpoint( + model=clean_model, + model_dir=model_dir, + train_steps=train_steps) + self.assertEqual(initial_epoch, 1) + self.assertNotAllClose(weights_before_load, clean_model.get_weights()) + + tf.io.gfile.rmtree(model_dir) + + def test_serialize_config(self): + """Tests functionality for serializing data.""" + config = base_configs.ExperimentConfig() + model_dir = self.get_temp_dir() + classifier_trainer.serialize_config(params=config, model_dir=model_dir) + saved_params_path = os.path.join(model_dir, 'params.yaml') + self.assertTrue(os.path.exists(saved_params_path)) + tf.io.gfile.rmtree(model_dir) + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/__init__.py new file mode 100644 index 0000000..931c2ef --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/base_configs.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/base_configs.py new file mode 100644 index 0000000..8065f03 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/base_configs.py @@ -0,0 +1,239 @@ +# Lint as: python3 +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Definitions for high level configuration groups..""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +from typing import Any, List, Mapping, Optional + +import dataclasses + +from official.modeling.hyperparams import base_config + + +CallbacksConfig = base_config.CallbacksConfig +TensorboardConfig = base_config.TensorboardConfig +RuntimeConfig = base_config.RuntimeConfig + + +@dataclasses.dataclass +class ExportConfig(base_config.Config): + """Configuration for exports. + + Attributes: + checkpoint: the path to the checkpoint to export. + destination: the path to where the checkpoint should be exported. + + """ + checkpoint: str = None + destination: str = None + + +@dataclasses.dataclass +class MetricsConfig(base_config.Config): + """Configuration for Metrics. + + Attributes: + accuracy: Whether or not to track accuracy as a Callback. Defaults to None. + top_5: Whether or not to track top_5_accuracy as a Callback. Defaults to + None. + + """ + accuracy: bool = None + top_5: bool = None + + +@dataclasses.dataclass +class TimeHistoryConfig(base_config.Config): + """Configuration for the TimeHistory callback. + + Attributes: + log_steps: Interval of steps between logging of batch level stats. + + """ + log_steps: int = None + + +@dataclasses.dataclass +class TrainConfig(base_config.Config): + """Configuration for training. + + Attributes: + resume_checkpoint: Whether or not to enable load checkpoint loading. + Defaults to None. + epochs: The number of training epochs to run. Defaults to None. + steps: The number of steps to run per epoch. If None, then this will be + inferred based on the number of images and batch size. Defaults to None. + callbacks: An instance of CallbacksConfig. + metrics: An instance of MetricsConfig. + tensorboard: An instance of TensorboardConfig. + steps_per_loop: The number of batches to run during each `tf.function` + call during training, which can increase training speed. + + """ + resume_checkpoint: bool = None + epochs: int = None + steps: int = None + callbacks: CallbacksConfig = CallbacksConfig() + metrics: MetricsConfig = None + tensorboard: TensorboardConfig = TensorboardConfig() + time_history: TimeHistoryConfig = TimeHistoryConfig() + steps_per_loop: int = None + + +@dataclasses.dataclass +class EvalConfig(base_config.Config): + """Configuration for evaluation. + + Attributes: + epochs_between_evals: The number of train epochs to run between evaluations. + Defaults to None. + steps: The number of eval steps to run during evaluation. If None, this will + be inferred based on the number of images and batch size. Defaults to + None. + skip_eval: Whether or not to skip evaluation. + + """ + epochs_between_evals: int = None + steps: int = None + skip_eval: bool = False + + +@dataclasses.dataclass +class LossConfig(base_config.Config): + """Configuration for Loss. + + Attributes: + name: The name of the loss. Defaults to None. + label_smoothing: Whether or not to apply label smoothing to the loss. This + only applies to 'categorical_cross_entropy'. + + """ + name: str = None + label_smoothing: float = None + + +@dataclasses.dataclass +class OptimizerConfig(base_config.Config): + """Configuration for Optimizers. + + Attributes: + name: The name of the optimizer. Defaults to None. + decay: Decay or rho, discounting factor for gradient. Defaults to None. + epsilon: Small value used to avoid 0 denominator. Defaults to None. + momentum: Plain momentum constant. Defaults to None. + nesterov: Whether or not to apply Nesterov momentum. Defaults to None. + moving_average_decay: The amount of decay to apply. If 0 or None, then + exponential moving average is not used. Defaults to None. + lookahead: Whether or not to apply the lookahead optimizer. Defaults to + None. + beta_1: The exponential decay rate for the 1st moment estimates. Used in + the Adam optimizers. Defaults to None. + beta_2: The exponential decay rate for the 2nd moment estimates. Used in + the Adam optimizers. Defaults to None. + epsilon: Small value used to avoid 0 denominator. Defaults to 1e-7. + + """ + name: str = None + decay: float = None + epsilon: float = None + momentum: float = None + nesterov: bool = None + moving_average_decay: Optional[float] = None + lookahead: Optional[bool] = None + beta_1: float = None + beta_2: float = None + epsilon: float = None + + +@dataclasses.dataclass +class LearningRateConfig(base_config.Config): + """Configuration for learning rates. + + Attributes: + name: The name of the learning rate. Defaults to None. + initial_lr: The initial learning rate. Defaults to None. + decay_epochs: The number of decay epochs. Defaults to None. + decay_rate: The rate of decay. Defaults to None. + warmup_epochs: The number of warmup epochs. Defaults to None. + batch_lr_multiplier: The multiplier to apply to the base learning rate, + if necessary. Defaults to None. + examples_per_epoch: the number of examples in a single epoch. + Defaults to None. + boundaries: boundaries used in piecewise constant decay with warmup. + multipliers: multipliers used in piecewise constant decay with warmup. + scale_by_batch_size: Scale the learning rate by a fraction of the batch + size. Set to 0 for no scaling (default). + staircase: Apply exponential decay at discrete values instead of continuous. + + """ + name: str = None + initial_lr: float = None + decay_epochs: float = None + decay_rate: float = None + warmup_epochs: int = None + examples_per_epoch: int = None + boundaries: List[int] = None + multipliers: List[float] = None + scale_by_batch_size: float = 0. + staircase: bool = None + + +@dataclasses.dataclass +class ModelConfig(base_config.Config): + """Configuration for Models. + + Attributes: + name: The name of the model. Defaults to None. + model_params: The parameters used to create the model. Defaults to None. + num_classes: The number of classes in the model. Defaults to None. + loss: A `LossConfig` instance. Defaults to None. + optimizer: An `OptimizerConfig` instance. Defaults to None. + + """ + name: str = None + model_params: Mapping[str, Any] = None + num_classes: int = None + loss: LossConfig = None + optimizer: OptimizerConfig = None + + +@dataclasses.dataclass +class ExperimentConfig(base_config.Config): + """Base configuration for an image classification experiment. + + Attributes: + model_dir: The directory to use when running an experiment. + mode: e.g. 'train_and_eval', 'export' + runtime: A `RuntimeConfig` instance. + train: A `TrainConfig` instance. + evaluation: An `EvalConfig` instance. + model: A `ModelConfig` instance. + export: An `ExportConfig` instance. + + """ + model_dir: str = None + model_name: str = None + mode: str = None + runtime: RuntimeConfig = None + train_dataset: Any = None + validation_dataset: Any = None + train: TrainConfig = None + evaluation: EvalConfig = None + model: ModelConfig = None + export: ExportConfig = None diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/configs.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/configs.py new file mode 100644 index 0000000..f87d992 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/configs.py @@ -0,0 +1,118 @@ +# Lint as: python3 +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Configuration utils for image classification experiments.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import dataclasses + +from official.vision.image_classification import dataset_factory +from official.vision.image_classification.configs import base_configs +from official.vision.image_classification.efficientnet import efficientnet_config +from official.vision.image_classification.resnet import resnet_config + + +@dataclasses.dataclass +class EfficientNetImageNetConfig(base_configs.ExperimentConfig): + """Base configuration to train efficientnet-b0 on ImageNet. + + Attributes: + export: An `ExportConfig` instance + runtime: A `RuntimeConfig` instance. + dataset: A `DatasetConfig` instance. + train: A `TrainConfig` instance. + evaluation: An `EvalConfig` instance. + model: A `ModelConfig` instance. + + """ + export: base_configs.ExportConfig = base_configs.ExportConfig() + runtime: base_configs.RuntimeConfig = base_configs.RuntimeConfig() + train_dataset: dataset_factory.DatasetConfig = \ + dataset_factory.ImageNetConfig(split='train') + validation_dataset: dataset_factory.DatasetConfig = \ + dataset_factory.ImageNetConfig(split='validation') + train: base_configs.TrainConfig = base_configs.TrainConfig( + resume_checkpoint=True, + epochs=500, + steps=None, + callbacks=base_configs.CallbacksConfig(enable_checkpoint_and_export=True, + enable_tensorboard=True), + metrics=['accuracy', 'top_5'], + time_history=base_configs.TimeHistoryConfig(log_steps=100), + tensorboard=base_configs.TensorboardConfig(track_lr=True, + write_model_weights=False), + steps_per_loop=1) + evaluation: base_configs.EvalConfig = base_configs.EvalConfig( + epochs_between_evals=1, + steps=None) + model: base_configs.ModelConfig = \ + efficientnet_config.EfficientNetModelConfig() + + +@dataclasses.dataclass +class ResNetImagenetConfig(base_configs.ExperimentConfig): + """Base configuration to train resnet-50 on ImageNet.""" + export: base_configs.ExportConfig = base_configs.ExportConfig() + runtime: base_configs.RuntimeConfig = base_configs.RuntimeConfig() + train_dataset: dataset_factory.DatasetConfig = \ + dataset_factory.ImageNetConfig(split='train', + one_hot=False, + mean_subtract=True, + standardize=True) + validation_dataset: dataset_factory.DatasetConfig = \ + dataset_factory.ImageNetConfig(split='validation', + one_hot=False, + mean_subtract=True, + standardize=True) + train: base_configs.TrainConfig = base_configs.TrainConfig( + resume_checkpoint=True, + epochs=90, + steps=None, + callbacks=base_configs.CallbacksConfig(enable_checkpoint_and_export=True, + enable_tensorboard=True), + metrics=['accuracy', 'top_5'], + time_history=base_configs.TimeHistoryConfig(log_steps=100), + tensorboard=base_configs.TensorboardConfig(track_lr=True, + write_model_weights=False), + steps_per_loop=1) + evaluation: base_configs.EvalConfig = base_configs.EvalConfig( + epochs_between_evals=1, + steps=None) + model: base_configs.ModelConfig = resnet_config.ResNetModelConfig() + + +def get_config(model: str, dataset: str) -> base_configs.ExperimentConfig: + """Given model and dataset names, return the ExperimentConfig.""" + dataset_model_config_map = { + 'imagenet': { + 'efficientnet': EfficientNetImageNetConfig(), + 'resnet': ResNetImagenetConfig(), + } + } + try: + return dataset_model_config_map[dataset][model] + except KeyError: + if dataset not in dataset_model_config_map: + raise KeyError('Invalid dataset received. Received: {}. Supported ' + 'datasets include: {}'.format( + dataset, + ', '.join(dataset_model_config_map.keys()))) + raise KeyError('Invalid model received. Received: {}. Supported models for' + '{} include: {}'.format( + model, + dataset, + ', '.join(dataset_model_config_map[dataset].keys()))) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-gpu.yaml b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-gpu.yaml new file mode 100644 index 0000000..ebe6488 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-gpu.yaml @@ -0,0 +1,53 @@ +# Training configuration for EfficientNet-b0 trained on ImageNet on GPUs. +# Takes ~32 minutes per epoch for 8 V100s. +# Reaches ~76.1% within 350 epochs. +# Note: This configuration uses a scaled per-replica batch size based on the number of devices. +runtime: + model_dir: null + mode: 'train_and_eval' + distribution_strategy: 'mirrored' + num_gpus: 1 +train_dataset: + name: 'imagenet2012' + data_dir: null + builder: 'records' + split: 'train' + num_classes: 1000 + num_examples: 1281167 + batch_size: 32 + use_per_replica_batch_size: True + dtype: 'float32' + augmenter: + name: 'autoaugment' +validation_dataset: + name: 'imagenet2012' + data_dir: null + builder: 'records' + split: 'validation' + num_classes: 1000 + num_examples: 50000 + batch_size: 32 + use_per_replica_batch_size: True + dtype: 'float32' +model: + model_params: + model_name: 'efficientnet-b0' + overrides: + num_classes: 1000 + batch_norm: 'default' + dtype: 'float32' + optimizer: + name: 'rmsprop' + momentum: 0.9 + decay: 0.9 + moving_average_decay: 0.0 + lookahead: false + learning_rate: + name: 'exponential' + loss: + label_smoothing: 0.1 +train: + resume_checkpoint: True + epochs: 500 +evaluation: + epochs_between_evals: 1 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-tpu.yaml b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-tpu.yaml new file mode 100644 index 0000000..0f17558 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-tpu.yaml @@ -0,0 +1,52 @@ +# Training configuration for EfficientNet-b0 trained on ImageNet on TPUs. +# Takes ~2 minutes, 50 seconds per epoch for v3-32. +# Reaches ~76.1% within 350 epochs. +# Note: This configuration uses a scaled per-replica batch size based on the number of devices. +runtime: + model_dir: null + mode: 'train_and_eval' + distribution_strategy: 'tpu' +train_dataset: + name: 'imagenet2012' + data_dir: null + builder: 'records' + split: 'train' + num_classes: 1000 + num_examples: 1281167 + batch_size: 128 + use_per_replica_batch_size: True + dtype: 'bfloat16' + augmenter: + name: 'autoaugment' +validation_dataset: + name: 'imagenet2012' + data_dir: null + builder: 'records' + split: 'validation' + num_classes: 1000 + num_examples: 50000 + batch_size: 128 + use_per_replica_batch_size: True + dtype: 'bfloat16' +model: + model_params: + model_name: 'efficientnet-b0' + overrides: + num_classes: 1000 + batch_norm: 'tpu' + dtype: 'bfloat16' + optimizer: + name: 'rmsprop' + momentum: 0.9 + decay: 0.9 + moving_average_decay: 0.0 + lookahead: false + learning_rate: + name: 'exponential' + loss: + label_smoothing: 0.1 +train: + resume_checkpoint: True + epochs: 500 +evaluation: + epochs_between_evals: 1 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-gpu.yaml b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-gpu.yaml new file mode 100644 index 0000000..1692bca --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-gpu.yaml @@ -0,0 +1,46 @@ +# Note: This configuration uses a scaled per-replica batch size based on the number of devices. +runtime: + model_dir: null + mode: 'train_and_eval' + distribution_strategy: 'mirrored' + num_gpus: 1 +train_dataset: + name: 'imagenet2012' + data_dir: null + builder: 'records' + split: 'train' + num_classes: 1000 + num_examples: 1281167 + batch_size: 32 + dtype: 'float32' +validation_dataset: + name: 'imagenet2012' + data_dir: null + builder: 'records' + split: 'validation' + num_classes: 1000 + num_examples: 50000 + batch_size: 32 + dtype: 'float32' +model: + model_params: + model_name: 'efficientnet-b1' + overrides: + num_classes: 1000 + batch_norm: 'default' + dtype: 'float32' + optimizer: + name: 'rmsprop' + momentum: 0.9 + decay: 0.9 + moving_average_decay: 0.0 + lookahead: false + learning_rate: + name: 'exponential' + loss: + label_smoothing: 0.1 +train: + resume_checkpoint: True + epochs: 500 +evaluation: + epochs_between_evals: 1 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-tpu.yaml b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-tpu.yaml new file mode 100644 index 0000000..4ae6827 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-tpu.yaml @@ -0,0 +1,51 @@ +# Training configuration for EfficientNet-b1 trained on ImageNet on TPUs. +# Takes ~3 minutes, 15 seconds per epoch for v3-32. +# Note: This configuration uses a scaled per-replica batch size based on the number of devices. +runtime: + model_dir: null + mode: 'train_and_eval' + distribution_strategy: 'tpu' +train_dataset: + name: 'imagenet2012' + data_dir: null + builder: 'records' + split: 'train' + num_classes: 1000 + num_examples: 1281167 + batch_size: 128 + use_per_replica_batch_size: True + dtype: 'bfloat16' + augmenter: + name: 'autoaugment' +validation_dataset: + name: 'imagenet2012' + data_dir: null + builder: 'records' + split: 'validation' + num_classes: 1000 + num_examples: 50000 + batch_size: 128 + use_per_replica_batch_size: True + dtype: 'bfloat16' +model: + model_params: + model_name: 'efficientnet-b1' + overrides: + num_classes: 1000 + batch_norm: 'tpu' + dtype: 'bfloat16' + optimizer: + name: 'rmsprop' + momentum: 0.9 + decay: 0.9 + moving_average_decay: 0.0 + lookahead: false + learning_rate: + name: 'exponential' + loss: + label_smoothing: 0.1 +train: + resume_checkpoint: True + epochs: 500 +evaluation: + epochs_between_evals: 1 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/resnet/imagenet/gpu.yaml b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/resnet/imagenet/gpu.yaml new file mode 100644 index 0000000..e2c340e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/resnet/imagenet/gpu.yaml @@ -0,0 +1,52 @@ +# Training configuration for ResNet trained on ImageNet on GPUs. +# Reaches > 76.1% within 90 epochs. +# Note: This configuration uses a scaled per-replica batch size based on the number of devices. +runtime: + model_dir: null + mode: 'train_and_eval' + distribution_strategy: 'mirrored' + num_gpus: 1 +train_dataset: + name: 'imagenet2012' + data_dir: null + builder: 'tfds' + split: 'train' + image_size: 224 + num_classes: 1000 + num_examples: 1281167 + batch_size: 128 + use_per_replica_batch_size: True + dtype: 'float32' + mean_subtract: True + standardize: True +validation_dataset: + name: 'imagenet2012' + data_dir: null + builder: 'tfds' + split: 'validation' + image_size: 224 + num_classes: 1000 + num_examples: 50000 + batch_size: 128 + use_per_replica_batch_size: True + dtype: 'float32' + mean_subtract: True + standardize: True +model: + model_name: 'resnet' + model_params: + rescale_inputs: False + optimizer: + name: 'momentum' + momentum: 0.9 + decay: 0.9 + epsilon: 0.001 + learning_rate: + name: 'piecewise_constant_with_warmup' + loss: + label_smoothing: 0.1 +train: + resume_checkpoint: True + epochs: 90 +evaluation: + epochs_between_evals: 1 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/resnet/imagenet/tpu.yaml b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/resnet/imagenet/tpu.yaml new file mode 100644 index 0000000..b6fb342 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/configs/examples/resnet/imagenet/tpu.yaml @@ -0,0 +1,58 @@ +# Training configuration for ResNet trained on ImageNet on TPUs. +# Takes ~4 minutes, 30 seconds seconds per epoch for a v3-32. +# Reaches > 76.1% within 90 epochs. +# Note: This configuration uses a scaled per-replica batch size based on the number of devices. +runtime: + model_dir: null + mode: 'train_and_eval' + distribution_strategy: 'tpu' +train_dataset: + name: 'imagenet2012' + data_dir: null + builder: 'tfds' + split: 'train' + one_hot: False + image_size: 224 + num_classes: 1000 + num_examples: 1281167 + batch_size: 128 + use_per_replica_batch_size: True + mean_subtract: True + standardize: True + dtype: 'bfloat16' +validation_dataset: + name: 'imagenet2012' + data_dir: null + builder: 'tfds' + split: 'validation' + one_hot: False + image_size: 224 + num_classes: 1000 + num_examples: 50000 + batch_size: 128 + use_per_replica_batch_size: True + mean_subtract: True + standardize: True + dtype: 'bfloat16' +model: + model_name: 'resnet' + model_params: + rescale_inputs: False + optimizer: + name: 'momentum' + momentum: 0.9 + decay: 0.9 + epsilon: 0.001 + moving_average_decay: 0. + lookahead: false + learning_rate: + name: 'piecewise_constant_with_warmup' + loss: + label_smoothing: 0.1 +train: + callbacks: + enable_checkpoint_and_export: True + resume_checkpoint: True + epochs: 90 +evaluation: + epochs_between_evals: 1 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/dataset_factory.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/dataset_factory.py new file mode 100644 index 0000000..5626980 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/dataset_factory.py @@ -0,0 +1,497 @@ +# Lint as: python3 +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Dataset utilities for vision tasks using TFDS and tf.data.Dataset.""" +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import os +from typing import Any, List, Optional, Tuple, Mapping, Union +from absl import logging +from dataclasses import dataclass +import tensorflow as tf +import tensorflow_datasets as tfds + +from official.modeling.hyperparams import base_config +from official.vision.image_classification import augment +from official.vision.image_classification import preprocessing + + +AUGMENTERS = { + 'autoaugment': augment.AutoAugment, + 'randaugment': augment.RandAugment, +} + + +@dataclass +class AugmentConfig(base_config.Config): + """Configuration for image augmenters. + + Attributes: + name: The name of the image augmentation to use. Possible options are + None (default), 'autoaugment', or 'randaugment'. + params: Any paramaters used to initialize the augmenter. + """ + name: Optional[str] = None + params: Optional[Mapping[str, Any]] = None + + def build(self) -> augment.ImageAugment: + """Build the augmenter using this config.""" + params = self.params or {} + augmenter = AUGMENTERS.get(self.name, None) + return augmenter(**params) if augmenter is not None else None + + +@dataclass +class DatasetConfig(base_config.Config): + """The base configuration for building datasets. + + Attributes: + name: The name of the Dataset. Usually should correspond to a TFDS dataset. + data_dir: The path where the dataset files are stored, if available. + filenames: Optional list of strings representing the TFRecord names. + builder: The builder type used to load the dataset. Value should be one of + 'tfds' (load using TFDS), 'records' (load from TFRecords), or 'synthetic' + (generate dummy synthetic data without reading from files). + split: The split of the dataset. Usually 'train', 'validation', or 'test'. + image_size: The size of the image in the dataset. This assumes that + `width` == `height`. Set to 'infer' to infer the image size from TFDS + info. This requires `name` to be a registered dataset in TFDS. + num_classes: The number of classes given by the dataset. Set to 'infer' + to infer the image size from TFDS info. This requires `name` to be a + registered dataset in TFDS. + num_channels: The number of channels given by the dataset. Set to 'infer' + to infer the image size from TFDS info. This requires `name` to be a + registered dataset in TFDS. + num_examples: The number of examples given by the dataset. Set to 'infer' + to infer the image size from TFDS info. This requires `name` to be a + registered dataset in TFDS. + batch_size: The base batch size for the dataset. + use_per_replica_batch_size: Whether to scale the batch size based on + available resources. If set to `True`, the dataset builder will return + batch_size multiplied by `num_devices`, the number of device replicas + (e.g., the number of GPUs or TPU cores). + num_devices: The number of replica devices to use. This should be set by + `strategy.num_replicas_in_sync` when using a distribution strategy. + dtype: The desired dtype of the dataset. This will be set during + preprocessing. + one_hot: Whether to apply one hot encoding. Set to `True` to be able to use + label smoothing. + augmenter: The augmenter config to use. No augmentation is used by default. + download: Whether to download data using TFDS. + shuffle_buffer_size: The buffer size used for shuffling training data. + file_shuffle_buffer_size: The buffer size used for shuffling raw training + files. + skip_decoding: Whether to skip image decoding when loading from TFDS. + deterministic_train: Whether the examples in the training set should output + in a deterministic order. + use_slack: whether to introduce slack in the last prefetch. This may reduce + CPU contention at the start of a training step. + cache: whether to cache to dataset examples. Can be used to avoid re-reading + from disk on the second epoch. Requires significant memory overhead. + mean_subtract: whether or not to apply mean subtraction to the dataset. + standardize: whether or not to apply standardization to the dataset. + """ + name: Optional[str] = None + data_dir: Optional[str] = None + filenames: Optional[List[str]] = None + builder: str = 'tfds' + split: str = 'train' + image_size: Union[int, str] = 'infer' + num_classes: Union[int, str] = 'infer' + num_channels: Union[int, str] = 'infer' + num_examples: Union[int, str] = 'infer' + batch_size: int = 128 + use_per_replica_batch_size: bool = False + num_devices: int = 1 + dtype: str = 'float32' + one_hot: bool = True + augmenter: AugmentConfig = AugmentConfig() + download: bool = False + shuffle_buffer_size: int = 10000 + file_shuffle_buffer_size: int = 1024 + skip_decoding: bool = True + deterministic_train: bool = False + use_slack: bool = True + cache: bool = False + mean_subtract: bool = False + standardize: bool = False + + @property + def has_data(self): + """Whether this dataset is has any data associated with it.""" + return self.name or self.data_dir or self.filenames + + +@dataclass +class ImageNetConfig(DatasetConfig): + """The base ImageNet dataset config.""" + name: str = 'imagenet2012' + # Note: for large datasets like ImageNet, using records is faster than tfds + builder: str = 'records' + image_size: int = 224 + batch_size: int = 128 + + +@dataclass +class Cifar10Config(DatasetConfig): + """The base CIFAR-10 dataset config.""" + name: str = 'cifar10' + image_size: int = 224 + batch_size: int = 128 + download: bool = True + cache: bool = True + + +class DatasetBuilder: + """An object for building datasets. + + Allows building various pipelines fetching examples, preprocessing, etc. + Maintains additional state information calculated from the dataset, i.e., + training set split, batch size, and number of steps (batches). + """ + + def __init__(self, config: DatasetConfig, **overrides: Any): + """Initialize the builder from the config.""" + self.config = config.replace(**overrides) + self.builder_info = None + + if self.config.augmenter is not None: + logging.info('Using augmentation: %s', self.config.augmenter.name) + self.augmenter = self.config.augmenter.build() + else: + self.augmenter = None + + @property + def is_training(self) -> bool: + """Whether this is the training set.""" + return self.config.split == 'train' + + @property + def batch_size(self) -> int: + """The batch size, multiplied by the number of replicas (if configured).""" + if self.config.use_per_replica_batch_size: + return self.global_batch_size + else: + return self.config.batch_size + + @property + def global_batch_size(self): + """The global batch size across all replicas.""" + return self.config.batch_size * self.config.num_devices + + @property + def num_steps(self) -> int: + """The number of steps (batches) to exhaust this dataset.""" + # Always divide by the global batch size to get the correct # of steps + return self.num_examples // self.global_batch_size + + @property + def dtype(self) -> tf.dtypes.DType: + """Converts the config's dtype string to a tf dtype. + + Returns: + A mapping from string representation of a dtype to the `tf.dtypes.DType`. + + Raises: + ValueError if the config's dtype is not supported. + + """ + dtype_map = { + 'float32': tf.float32, + 'bfloat16': tf.bfloat16, + 'float16': tf.float16, + 'fp32': tf.float32, + 'bf16': tf.bfloat16, + } + try: + return dtype_map[self.config.dtype] + except: + raise ValueError('Invalid DType provided. Supported types: {}'.format( + dtype_map.keys())) + + @property + def image_size(self) -> int: + """The size of each image (can be inferred from the dataset).""" + + if self.config.image_size == 'infer': + return self.info.features['image'].shape[0] + else: + return int(self.config.image_size) + + @property + def num_channels(self) -> int: + """The number of image channels (can be inferred from the dataset).""" + if self.config.num_channels == 'infer': + return self.info.features['image'].shape[-1] + else: + return int(self.config.num_channels) + + @property + def num_examples(self) -> int: + """The number of examples (can be inferred from the dataset).""" + if self.config.num_examples == 'infer': + return self.info.splits[self.config.split].num_examples + else: + return int(self.config.num_examples) + + @property + def num_classes(self) -> int: + """The number of classes (can be inferred from the dataset).""" + if self.config.num_classes == 'infer': + return self.info.features['label'].num_classes + else: + return int(self.config.num_classes) + + @property + def info(self) -> tfds.core.DatasetInfo: + """The TFDS dataset info, if available.""" + if self.builder_info is None: + self.builder_info = tfds.builder(self.config.name).info + return self.builder_info + + def build(self, input_context: tf.distribute.InputContext = None + ) -> tf.data.Dataset: + """Construct a dataset end-to-end and return it. + + Args: + input_context: An optional context provided by `tf.distribute` for + cross-replica training. This isn't necessary if using Keras + compile/fit. + + Returns: + A TensorFlow dataset outputting batched images and labels. + """ + + builders = { + 'tfds': self.load_tfds, + 'records': self.load_records, + 'synthetic': self.load_synthetic, + } + + builder = builders.get(self.config.builder, None) + + if builder is None: + raise ValueError('Unknown builder type {}'.format(self.config.builder)) + + dataset = builder() + dataset = self.pipeline(dataset, input_context) + + return dataset + + def load_tfds(self) -> tf.data.Dataset: + """Return a dataset loading files from TFDS.""" + + logging.info('Using TFDS to load data.') + + builder = tfds.builder(self.config.name, + data_dir=self.config.data_dir) + + if self.config.download: + builder.download_and_prepare() + + decoders = {} + + if self.config.skip_decoding: + decoders['image'] = tfds.decode.SkipDecoding() + + read_config = tfds.ReadConfig( + interleave_parallel_reads=64, + interleave_block_length=1) + + dataset = builder.as_dataset( + split=self.config.split, + as_supervised=True, + shuffle_files=True, + decoders=decoders, + read_config=read_config) + + return dataset + + def load_records(self) -> tf.data.Dataset: + """Return a dataset loading files with TFRecords.""" + logging.info('Using TFRecords to load data.') + + if self.config.filenames is None: + if self.config.data_dir is None: + raise ValueError('Dataset must specify a path for the data files.') + + file_pattern = os.path.join(self.config.data_dir, + '{}*'.format(self.config.split)) + dataset = tf.data.Dataset.list_files(file_pattern, shuffle=True) + else: + dataset = tf.data.Dataset.from_tensor_slices(self.config.filenames) + if self.is_training: + # Shuffle the input files. + dataset.shuffle(buffer_size=self.config.file_shuffle_buffer_size) + + return dataset + + def load_synthetic(self) -> tf.data.Dataset: + """Return a dataset generating dummy synthetic data.""" + logging.info('Generating a synthetic dataset.') + + def generate_data(_): + image = tf.zeros([self.image_size, self.image_size, self.num_channels], + dtype=self.dtype) + label = tf.zeros([1], dtype=tf.int32) + return image, label + + dataset = tf.data.Dataset.range(1) + dataset = dataset.repeat() + dataset = dataset.map(generate_data, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + return dataset + + def pipeline(self, + dataset: tf.data.Dataset, + input_context: tf.distribute.InputContext = None + ) -> tf.data.Dataset: + """Build a pipeline fetching, shuffling, and preprocessing the dataset. + + Args: + dataset: A `tf.data.Dataset` that loads raw files. + input_context: An optional context provided by `tf.distribute` for + cross-replica training. This isn't necessary if using Keras + compile/fit. + + Returns: + A TensorFlow dataset outputting batched images and labels. + """ + if input_context and input_context.num_input_pipelines > 1: + dataset = dataset.shard(input_context.num_input_pipelines, + input_context.input_pipeline_id) + + if self.is_training and not self.config.cache: + dataset = dataset.repeat() + + if self.config.builder == 'records': + # Read the data from disk in parallel + buffer_size = 8 * 1024 * 1024 # Use 8 MiB per file + dataset = dataset.interleave( + lambda name: tf.data.TFRecordDataset(name, buffer_size=buffer_size), + cycle_length=16, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + + dataset = dataset.prefetch(self.global_batch_size) + + if self.config.cache: + dataset = dataset.cache() + + if self.is_training: + dataset = dataset.shuffle(self.config.shuffle_buffer_size) + dataset = dataset.repeat() + + # Parse, pre-process, and batch the data in parallel + if self.config.builder == 'records': + preprocess = self.parse_record + else: + preprocess = self.preprocess + dataset = dataset.map(preprocess, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + + dataset = dataset.batch(self.batch_size, drop_remainder=self.is_training) + + # Note: we could do image normalization here, but we defer it to the model + # which can perform it much faster on a GPU/TPU + # TODO(dankondratyuk): if we fix prefetching, we can do it here + + if self.is_training and self.config.deterministic_train is not None: + options = tf.data.Options() + options.experimental_deterministic = self.config.deterministic_train + options.experimental_slack = self.config.use_slack + options.experimental_optimization.parallel_batch = True + options.experimental_optimization.map_fusion = True + options.experimental_optimization.map_vectorization.enabled = True + options.experimental_optimization.map_parallelization = True + dataset = dataset.with_options(options) + + # Prefetch overlaps in-feed with training + # Note: autotune here is not recommended, as this can lead to memory leaks. + # Instead, use a constant prefetch size like the the number of devices. + dataset = dataset.prefetch(self.config.num_devices) + + return dataset + + def parse_record(self, record: tf.Tensor) -> Tuple[tf.Tensor, tf.Tensor]: + """Parse an ImageNet record from a serialized string Tensor.""" + keys_to_features = { + 'image/encoded': + tf.io.FixedLenFeature((), tf.string, ''), + 'image/format': + tf.io.FixedLenFeature((), tf.string, 'jpeg'), + 'image/class/label': + tf.io.FixedLenFeature([], tf.int64, -1), + 'image/class/text': + tf.io.FixedLenFeature([], tf.string, ''), + 'image/object/bbox/xmin': + tf.io.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymin': + tf.io.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/xmax': + tf.io.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymax': + tf.io.VarLenFeature(dtype=tf.float32), + 'image/object/class/label': + tf.io.VarLenFeature(dtype=tf.int64), + } + + parsed = tf.io.parse_single_example(record, keys_to_features) + + label = tf.reshape(parsed['image/class/label'], shape=[1]) + label = tf.cast(label, dtype=tf.int32) + + # Subtract one so that labels are in [0, 1000) + label -= 1 + + image_bytes = tf.reshape(parsed['image/encoded'], shape=[]) + image, label = self.preprocess(image_bytes, label) + + return image, label + + def preprocess(self, image: tf.Tensor, label: tf.Tensor + ) -> Tuple[tf.Tensor, tf.Tensor]: + """Apply image preprocessing and augmentation to the image and label.""" + if self.is_training: + image = preprocessing.preprocess_for_train( + image, + image_size=self.image_size, + mean_subtract=self.config.mean_subtract, + standardize=self.config.standardize, + dtype=self.dtype, + augmenter=self.augmenter) + else: + image = preprocessing.preprocess_for_eval( + image, + image_size=self.image_size, + num_channels=self.num_channels, + mean_subtract=self.config.mean_subtract, + standardize=self.config.standardize, + dtype=self.dtype) + + label = tf.cast(label, tf.int32) + if self.config.one_hot: + label = tf.one_hot(label, self.num_classes) + label = tf.reshape(label, [self.num_classes]) + + return image, label + + @classmethod + def from_params(cls, *args, **kwargs): + """Construct a dataset builder from a default config and any overrides.""" + config = DatasetConfig.from_args(*args, **kwargs) + return cls(config) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/efficientnet/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/efficientnet/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/efficientnet/common_modules.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/efficientnet/common_modules.py new file mode 100644 index 0000000..9c9c209 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/efficientnet/common_modules.py @@ -0,0 +1,117 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Common modeling utilities.""" +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import numpy as np +import tensorflow as tf +import tensorflow.compat.v1 as tf1 +from typing import Text, Optional + +from tensorflow.python.tpu import tpu_function + + +@tf.keras.utils.register_keras_serializable(package='Vision') +class TpuBatchNormalization(tf.keras.layers.BatchNormalization): + """Cross replica batch normalization.""" + + def __init__(self, fused: Optional[bool] = False, **kwargs): + if fused in (True, None): + raise ValueError('TpuBatchNormalization does not support fused=True.') + super(TpuBatchNormalization, self).__init__(fused=fused, **kwargs) + + def _cross_replica_average(self, t: tf.Tensor, num_shards_per_group: int): + """Calculates the average value of input tensor across TPU replicas.""" + num_shards = tpu_function.get_tpu_context().number_of_shards + group_assignment = None + if num_shards_per_group > 1: + if num_shards % num_shards_per_group != 0: + raise ValueError( + 'num_shards: %d mod shards_per_group: %d, should be 0' % + (num_shards, num_shards_per_group)) + num_groups = num_shards // num_shards_per_group + group_assignment = [[ + x for x in range(num_shards) if x // num_shards_per_group == y + ] for y in range(num_groups)] + return tf1.tpu.cross_replica_sum(t, group_assignment) / tf.cast( + num_shards_per_group, t.dtype) + + def _moments(self, inputs: tf.Tensor, reduction_axes: int, keep_dims: int): + """Compute the mean and variance: it overrides the original _moments.""" + shard_mean, shard_variance = super(TpuBatchNormalization, self)._moments( + inputs, reduction_axes, keep_dims=keep_dims) + + num_shards = tpu_function.get_tpu_context().number_of_shards or 1 + if num_shards <= 8: # Skip cross_replica for 2x2 or smaller slices. + num_shards_per_group = 1 + else: + num_shards_per_group = max(8, num_shards // 8) + if num_shards_per_group > 1: + # Compute variance using: Var[X]= E[X^2] - E[X]^2. + shard_square_of_mean = tf.math.square(shard_mean) + shard_mean_of_square = shard_variance + shard_square_of_mean + group_mean = self._cross_replica_average(shard_mean, num_shards_per_group) + group_mean_of_square = self._cross_replica_average( + shard_mean_of_square, num_shards_per_group) + group_variance = group_mean_of_square - tf.math.square(group_mean) + return (group_mean, group_variance) + else: + return (shard_mean, shard_variance) + + +def get_batch_norm(batch_norm_type: Text) -> tf.keras.layers.BatchNormalization: + """A helper to create a batch normalization getter. + + Args: + batch_norm_type: The type of batch normalization layer implementation. `tpu` + will use `TpuBatchNormalization`. + + Returns: + An instance of `tf.keras.layers.BatchNormalization`. + """ + if batch_norm_type == 'tpu': + return TpuBatchNormalization + + return tf.keras.layers.BatchNormalization + + +def count_params(model, trainable_only=True): + """Returns the count of all model parameters, or just trainable ones.""" + if not trainable_only: + return model.count_params() + else: + return int(np.sum([tf.keras.backend.count_params(p) + for p in model.trainable_weights])) + + +def load_weights(model: tf.keras.Model, + model_weights_path: Text, + weights_format: Text = 'saved_model'): + """Load model weights from the given file path. + + Args: + model: the model to load weights into + model_weights_path: the path of the model weights + weights_format: the model weights format. One of 'saved_model', 'h5', + or 'checkpoint'. + """ + if weights_format == 'saved_model': + loaded_model = tf.keras.models.load_model(model_weights_path) + model.set_weights(loaded_model.get_weights()) + else: + model.load_weights(model_weights_path) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/efficientnet/efficientnet_config.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/efficientnet/efficientnet_config.py new file mode 100644 index 0000000..e2ec0cb --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/efficientnet/efficientnet_config.py @@ -0,0 +1,76 @@ +# Lint as: python3 +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Configuration definitions for EfficientNet losses, learning rates, and optimizers.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from typing import Any, Mapping + +import dataclasses + +from official.vision.image_classification.configs import base_configs + + +@dataclasses.dataclass +class EfficientNetModelConfig(base_configs.ModelConfig): + """Configuration for the EfficientNet model. + + This configuration will default to settings used for training efficientnet-b0 + on a v3-8 TPU on ImageNet. + + Attributes: + name: The name of the model. Defaults to 'EfficientNet'. + num_classes: The number of classes in the model. + model_params: A dictionary that represents the parameters of the + EfficientNet model. These will be passed in to the "from_name" function. + loss: The configuration for loss. Defaults to a categorical cross entropy + implementation. + optimizer: The configuration for optimizations. Defaults to an RMSProp + configuration. + learning_rate: The configuration for learning rate. Defaults to an + exponential configuration. + + """ + name: str = 'EfficientNet' + num_classes: int = 1000 + model_params: Mapping[str, Any] = dataclasses.field(default_factory=lambda: { + 'model_name': 'efficientnet-b0', + 'model_weights_path': '', + 'weights_format': 'saved_model', + 'overrides': { + 'batch_norm': 'default', + 'rescale_input': True, + 'num_classes': 1000, + } + }) + loss: base_configs.LossConfig = base_configs.LossConfig( + name='categorical_crossentropy', + label_smoothing=0.1) + optimizer: base_configs.OptimizerConfig = base_configs.OptimizerConfig( + name='rmsprop', + decay=0.9, + epsilon=0.001, + momentum=0.9, + moving_average_decay=None) + learning_rate: base_configs.LearningRateConfig = base_configs.LearningRateConfig( # pylint: disable=line-too-long + name='exponential', + initial_lr=0.008, + decay_epochs=2.4, + decay_rate=0.97, + warmup_epochs=5, + scale_by_batch_size=1. / 128., + staircase=True) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/efficientnet/efficientnet_model.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/efficientnet/efficientnet_model.py new file mode 100644 index 0000000..955b5ff --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/efficientnet/efficientnet_model.py @@ -0,0 +1,504 @@ +# Lint as: python3 +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains definitions for EfficientNet model. + +[1] Mingxing Tan, Quoc V. Le + EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks. + ICML'19, https://arxiv.org/abs/1905.11946 +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math +import os +from typing import Any, Dict, Optional, Text, Tuple + +from absl import logging +from dataclasses import dataclass +import tensorflow as tf + +from official.modeling import tf_utils +from official.modeling.hyperparams import base_config +from official.vision.image_classification import preprocessing +from official.vision.image_classification.efficientnet import common_modules + + +@dataclass +class BlockConfig(base_config.Config): + """Config for a single MB Conv Block.""" + input_filters: int = 0 + output_filters: int = 0 + kernel_size: int = 3 + num_repeat: int = 1 + expand_ratio: int = 1 + strides: Tuple[int, int] = (1, 1) + se_ratio: Optional[float] = None + id_skip: bool = True + fused_conv: bool = False + conv_type: str = 'depthwise' + + +@dataclass +class ModelConfig(base_config.Config): + """Default Config for Efficientnet-B0.""" + width_coefficient: float = 1.0 + depth_coefficient: float = 1.0 + resolution: int = 224 + dropout_rate: float = 0.2 + blocks: Tuple[BlockConfig, ...] = ( + # (input_filters, output_filters, kernel_size, num_repeat, + # expand_ratio, strides, se_ratio) + # pylint: disable=bad-whitespace + BlockConfig.from_args(32, 16, 3, 1, 1, (1, 1), 0.25), + BlockConfig.from_args(16, 24, 3, 2, 6, (2, 2), 0.25), + BlockConfig.from_args(24, 40, 5, 2, 6, (2, 2), 0.25), + BlockConfig.from_args(40, 80, 3, 3, 6, (2, 2), 0.25), + BlockConfig.from_args(80, 112, 5, 3, 6, (1, 1), 0.25), + BlockConfig.from_args(112, 192, 5, 4, 6, (2, 2), 0.25), + BlockConfig.from_args(192, 320, 3, 1, 6, (1, 1), 0.25), + # pylint: enable=bad-whitespace + ) + stem_base_filters: int = 32 + top_base_filters: int = 1280 + activation: str = 'simple_swish' + batch_norm: str = 'default' + bn_momentum: float = 0.99 + bn_epsilon: float = 1e-3 + # While the original implementation used a weight decay of 1e-5, + # tf.nn.l2_loss divides it by 2, so we halve this to compensate in Keras + weight_decay: float = 5e-6 + drop_connect_rate: float = 0.2 + depth_divisor: int = 8 + min_depth: Optional[int] = None + use_se: bool = True + input_channels: int = 3 + num_classes: int = 1000 + model_name: str = 'efficientnet' + rescale_input: bool = True + data_format: str = 'channels_last' + dtype: str = 'float32' + + +MODEL_CONFIGS = { + # (width, depth, resolution, dropout) + 'efficientnet-b0': ModelConfig.from_args(1.0, 1.0, 224, 0.2), + 'efficientnet-b1': ModelConfig.from_args(1.0, 1.1, 240, 0.2), + 'efficientnet-b2': ModelConfig.from_args(1.1, 1.2, 260, 0.3), + 'efficientnet-b3': ModelConfig.from_args(1.2, 1.4, 300, 0.3), + 'efficientnet-b4': ModelConfig.from_args(1.4, 1.8, 380, 0.4), + 'efficientnet-b5': ModelConfig.from_args(1.6, 2.2, 456, 0.4), + 'efficientnet-b6': ModelConfig.from_args(1.8, 2.6, 528, 0.5), + 'efficientnet-b7': ModelConfig.from_args(2.0, 3.1, 600, 0.5), + 'efficientnet-b8': ModelConfig.from_args(2.2, 3.6, 672, 0.5), + 'efficientnet-l2': ModelConfig.from_args(4.3, 5.3, 800, 0.5), +} + +CONV_KERNEL_INITIALIZER = { + 'class_name': 'VarianceScaling', + 'config': { + 'scale': 2.0, + 'mode': 'fan_out', + # Note: this is a truncated normal distribution + 'distribution': 'normal' + } +} + +DENSE_KERNEL_INITIALIZER = { + 'class_name': 'VarianceScaling', + 'config': { + 'scale': 1 / 3.0, + 'mode': 'fan_out', + 'distribution': 'uniform' + } +} + + +def round_filters(filters: int, + config: ModelConfig) -> int: + """Round number of filters based on width coefficient.""" + width_coefficient = config.width_coefficient + min_depth = config.min_depth + divisor = config.depth_divisor + orig_filters = filters + + if not width_coefficient: + return filters + + filters *= width_coefficient + min_depth = min_depth or divisor + new_filters = max(min_depth, int(filters + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_filters < 0.9 * filters: + new_filters += divisor + logging.info('round_filter input=%s output=%s', orig_filters, new_filters) + return int(new_filters) + + +def round_repeats(repeats: int, depth_coefficient: float) -> int: + """Round number of repeats based on depth coefficient.""" + return int(math.ceil(depth_coefficient * repeats)) + + +def conv2d_block(inputs: tf.Tensor, + conv_filters: Optional[int], + config: ModelConfig, + kernel_size: Any = (1, 1), + strides: Any = (1, 1), + use_batch_norm: bool = True, + use_bias: bool = False, + activation: Any = None, + depthwise: bool = False, + name: Text = None): + """A conv2d followed by batch norm and an activation.""" + batch_norm = common_modules.get_batch_norm(config.batch_norm) + bn_momentum = config.bn_momentum + bn_epsilon = config.bn_epsilon + data_format = tf.keras.backend.image_data_format() + weight_decay = config.weight_decay + + name = name or '' + + # Collect args based on what kind of conv2d block is desired + init_kwargs = { + 'kernel_size': kernel_size, + 'strides': strides, + 'use_bias': use_bias, + 'padding': 'same', + 'name': name + '_conv2d', + 'kernel_regularizer': tf.keras.regularizers.l2(weight_decay), + 'bias_regularizer': tf.keras.regularizers.l2(weight_decay), + } + + if depthwise: + conv2d = tf.keras.layers.DepthwiseConv2D + init_kwargs.update({'depthwise_initializer': CONV_KERNEL_INITIALIZER}) + else: + conv2d = tf.keras.layers.Conv2D + init_kwargs.update({'filters': conv_filters, + 'kernel_initializer': CONV_KERNEL_INITIALIZER}) + + x = conv2d(**init_kwargs)(inputs) + + if use_batch_norm: + bn_axis = 1 if data_format == 'channels_first' else -1 + x = batch_norm(axis=bn_axis, + momentum=bn_momentum, + epsilon=bn_epsilon, + name=name + '_bn')(x) + + if activation is not None: + x = tf.keras.layers.Activation(activation, + name=name + '_activation')(x) + return x + + +def mb_conv_block(inputs: tf.Tensor, + block: BlockConfig, + config: ModelConfig, + prefix: Text = None): + """Mobile Inverted Residual Bottleneck. + + Args: + inputs: the Keras input to the block + block: BlockConfig, arguments to create a Block + config: ModelConfig, a set of model parameters + prefix: prefix for naming all layers + + Returns: + the output of the block + """ + use_se = config.use_se + activation = tf_utils.get_activation(config.activation) + drop_connect_rate = config.drop_connect_rate + data_format = tf.keras.backend.image_data_format() + use_depthwise = block.conv_type != 'no_depthwise' + prefix = prefix or '' + + filters = block.input_filters * block.expand_ratio + + x = inputs + + if block.fused_conv: + # If we use fused mbconv, skip expansion and use regular conv. + x = conv2d_block(x, + filters, + config, + kernel_size=block.kernel_size, + strides=block.strides, + activation=activation, + name=prefix + 'fused') + else: + if block.expand_ratio != 1: + # Expansion phase + kernel_size = (1, 1) if use_depthwise else (3, 3) + x = conv2d_block(x, + filters, + config, + kernel_size=kernel_size, + activation=activation, + name=prefix + 'expand') + + # Depthwise Convolution + if use_depthwise: + x = conv2d_block(x, + conv_filters=None, + config=config, + kernel_size=block.kernel_size, + strides=block.strides, + activation=activation, + depthwise=True, + name=prefix + 'depthwise') + + # Squeeze and Excitation phase + if use_se: + assert block.se_ratio is not None + assert 0 < block.se_ratio <= 1 + num_reduced_filters = max(1, int( + block.input_filters * block.se_ratio + )) + + if data_format == 'channels_first': + se_shape = (filters, 1, 1) + else: + se_shape = (1, 1, filters) + + se = tf.keras.layers.GlobalAveragePooling2D(name=prefix + 'se_squeeze')(x) + se = tf.keras.layers.Reshape(se_shape, name=prefix + 'se_reshape')(se) + + se = conv2d_block(se, + num_reduced_filters, + config, + use_bias=True, + use_batch_norm=False, + activation=activation, + name=prefix + 'se_reduce') + se = conv2d_block(se, + filters, + config, + use_bias=True, + use_batch_norm=False, + activation='sigmoid', + name=prefix + 'se_expand') + x = tf.keras.layers.multiply([x, se], name=prefix + 'se_excite') + + # Output phase + x = conv2d_block(x, + block.output_filters, + config, + activation=None, + name=prefix + 'project') + + # Add identity so that quantization-aware training can insert quantization + # ops correctly. + x = tf.keras.layers.Activation(tf_utils.get_activation('identity'), + name=prefix + 'id')(x) + + if (block.id_skip + and all(s == 1 for s in block.strides) + and block.input_filters == block.output_filters): + if drop_connect_rate and drop_connect_rate > 0: + # Apply dropconnect + # The only difference between dropout and dropconnect in TF is scaling by + # drop_connect_rate during training. See: + # https://github.com/keras-team/keras/pull/9898#issuecomment-380577612 + x = tf.keras.layers.Dropout(drop_connect_rate, + noise_shape=(None, 1, 1, 1), + name=prefix + 'drop')(x) + + x = tf.keras.layers.add([x, inputs], name=prefix + 'add') + + return x + + +def efficientnet(image_input: tf.keras.layers.Input, + config: ModelConfig): + """Creates an EfficientNet graph given the model parameters. + + This function is wrapped by the `EfficientNet` class to make a tf.keras.Model. + + Args: + image_input: the input batch of images + config: the model config + + Returns: + the output of efficientnet + """ + depth_coefficient = config.depth_coefficient + blocks = config.blocks + stem_base_filters = config.stem_base_filters + top_base_filters = config.top_base_filters + activation = tf_utils.get_activation(config.activation) + dropout_rate = config.dropout_rate + drop_connect_rate = config.drop_connect_rate + num_classes = config.num_classes + input_channels = config.input_channels + rescale_input = config.rescale_input + data_format = tf.keras.backend.image_data_format() + dtype = config.dtype + weight_decay = config.weight_decay + + x = image_input + if data_format == 'channels_first': + # Happens on GPU/TPU if available. + x = tf.keras.layers.Permute((3, 1, 2))(x) + if rescale_input: + x = preprocessing.normalize_images(x, + num_channels=input_channels, + dtype=dtype, + data_format=data_format) + + # Build stem + x = conv2d_block(x, + round_filters(stem_base_filters, config), + config, + kernel_size=[3, 3], + strides=[2, 2], + activation=activation, + name='stem') + + # Build blocks + num_blocks_total = sum(block.num_repeat for block in blocks) + block_num = 0 + + for stack_idx, block in enumerate(blocks): + assert block.num_repeat > 0 + # Update block input and output filters based on depth multiplier + block = block.replace( + input_filters=round_filters(block.input_filters, config), + output_filters=round_filters(block.output_filters, config), + num_repeat=round_repeats(block.num_repeat, depth_coefficient)) + + # The first block needs to take care of stride and filter size increase + drop_rate = drop_connect_rate * float(block_num) / num_blocks_total + config = config.replace(drop_connect_rate=drop_rate) + block_prefix = 'stack_{}/block_0/'.format(stack_idx) + x = mb_conv_block(x, block, config, block_prefix) + block_num += 1 + if block.num_repeat > 1: + block = block.replace( + input_filters=block.output_filters, + strides=[1, 1] + ) + + for block_idx in range(block.num_repeat - 1): + drop_rate = drop_connect_rate * float(block_num) / num_blocks_total + config = config.replace(drop_connect_rate=drop_rate) + block_prefix = 'stack_{}/block_{}/'.format(stack_idx, block_idx + 1) + x = mb_conv_block(x, block, config, prefix=block_prefix) + block_num += 1 + + # Build top + x = conv2d_block(x, + round_filters(top_base_filters, config), + config, + activation=activation, + name='top') + + # Build classifier + x = tf.keras.layers.GlobalAveragePooling2D(name='top_pool')(x) + if dropout_rate and dropout_rate > 0: + x = tf.keras.layers.Dropout(dropout_rate, name='top_dropout')(x) + x = tf.keras.layers.Dense( + num_classes, + kernel_initializer=DENSE_KERNEL_INITIALIZER, + kernel_regularizer=tf.keras.regularizers.l2(weight_decay), + bias_regularizer=tf.keras.regularizers.l2(weight_decay), + name='logits')(x) + x = tf.keras.layers.Activation('softmax', name='probs')(x) + + return x + + +@tf.keras.utils.register_keras_serializable(package='Vision') +class EfficientNet(tf.keras.Model): + """Wrapper class for an EfficientNet Keras model. + + Contains helper methods to build, manage, and save metadata about the model. + """ + + def __init__(self, + config: ModelConfig = None, + overrides: Dict[Text, Any] = None): + """Create an EfficientNet model. + + Args: + config: (optional) the main model parameters to create the model + overrides: (optional) a dict containing keys that can override + config + """ + overrides = overrides or {} + config = config or ModelConfig() + + self.config = config.replace(**overrides) + + input_channels = self.config.input_channels + model_name = self.config.model_name + input_shape = (None, None, input_channels) # Should handle any size image + image_input = tf.keras.layers.Input(shape=input_shape) + + output = efficientnet(image_input, self.config) + + # Cast to float32 in case we have a different model dtype + output = tf.cast(output, tf.float32) + + logging.info('Building model %s with params %s', + model_name, + self.config) + + super(EfficientNet, self).__init__( + inputs=image_input, outputs=output, name=model_name) + + @classmethod + def from_name(cls, + model_name: Text, + model_weights_path: Text = None, + weights_format: Text = 'saved_model', + overrides: Dict[Text, Any] = None): + """Construct an EfficientNet model from a predefined model name. + + E.g., `EfficientNet.from_name('efficientnet-b0')`. + + Args: + model_name: the predefined model name + model_weights_path: the path to the weights (h5 file or saved model dir) + weights_format: the model weights format. One of 'saved_model', 'h5', + or 'checkpoint'. + overrides: (optional) a dict containing keys that can override config + + Returns: + A constructed EfficientNet instance. + """ + model_configs = dict(MODEL_CONFIGS) + overrides = dict(overrides) if overrides else {} + + # One can define their own custom models if necessary + model_configs.update(overrides.pop('model_config', {})) + + if model_name not in model_configs: + raise ValueError('Unknown model name {}'.format(model_name)) + + config = model_configs[model_name] + + model = cls(config=config, overrides=overrides) + + if model_weights_path: + common_modules.load_weights(model, + model_weights_path, + weights_format=weights_format) + + return model diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/learning_rate.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/learning_rate.py new file mode 100644 index 0000000..262b4aa --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/learning_rate.py @@ -0,0 +1,120 @@ +# Lint as: python3 +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Learning rate utilities for vision tasks.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from typing import Any, List, Mapping + +import tensorflow as tf + +BASE_LEARNING_RATE = 0.1 + + +class WarmupDecaySchedule(tf.keras.optimizers.schedules.LearningRateSchedule): + """A wrapper for LearningRateSchedule that includes warmup steps.""" + + def __init__( + self, + lr_schedule: tf.keras.optimizers.schedules.LearningRateSchedule, + warmup_steps: int): + """Add warmup decay to a learning rate schedule. + + Args: + lr_schedule: base learning rate scheduler + warmup_steps: number of warmup steps + + """ + super(WarmupDecaySchedule, self).__init__() + self._lr_schedule = lr_schedule + self._warmup_steps = warmup_steps + + def __call__(self, step: int): + lr = self._lr_schedule(step) + if self._warmup_steps: + initial_learning_rate = tf.convert_to_tensor( + self._lr_schedule.initial_learning_rate, name="initial_learning_rate") + dtype = initial_learning_rate.dtype + global_step_recomp = tf.cast(step, dtype) + warmup_steps = tf.cast(self._warmup_steps, dtype) + warmup_lr = initial_learning_rate * global_step_recomp / warmup_steps + lr = tf.cond(global_step_recomp < warmup_steps, + lambda: warmup_lr, + lambda: lr) + return lr + + def get_config(self) -> Mapping[str, Any]: + config = self._lr_schedule.get_config() + config.update({ + "warmup_steps": self._warmup_steps, + }) + return config + + +# TODO(b/149030439) - refactor this with +# tf.keras.optimizers.schedules.PiecewiseConstantDecay + WarmupDecaySchedule. +class PiecewiseConstantDecayWithWarmup( + tf.keras.optimizers.schedules.LearningRateSchedule): + """Piecewise constant decay with warmup schedule.""" + + def __init__(self, + batch_size: int, + epoch_size: int, + warmup_epochs: int, + boundaries: List[int], + multipliers: List[float]): + """Piecewise constant decay with warmup. + + Args: + batch_size: The training batch size used in the experiment. + epoch_size: The size of an epoch, or the number of examples in an epoch. + warmup_epochs: The number of warmup epochs to apply. + boundaries: The list of floats with strictly increasing entries. + multipliers: The list of multipliers/learning rates to use for the + piecewise portion. The length must be 1 less than that of boundaries. + + """ + super(PiecewiseConstantDecayWithWarmup, self).__init__() + if len(boundaries) != len(multipliers) - 1: + raise ValueError("The length of boundaries must be 1 less than the " + "length of multipliers") + + base_lr_batch_size = 256 + steps_per_epoch = epoch_size // batch_size + + self._rescaled_lr = BASE_LEARNING_RATE * batch_size / base_lr_batch_size + self._step_boundaries = [float(steps_per_epoch) * x for x in boundaries] + self._lr_values = [self._rescaled_lr * m for m in multipliers] + self._warmup_steps = warmup_epochs * steps_per_epoch + + def __call__(self, step: int): + """Compute learning rate at given step.""" + def warmup_lr(): + return self._rescaled_lr * ( + step / tf.cast(self._warmup_steps, tf.float32)) + def piecewise_lr(): + return tf.compat.v1.train.piecewise_constant( + tf.cast(step, tf.float32), self._step_boundaries, self._lr_values) + return tf.cond(step < self._warmup_steps, warmup_lr, piecewise_lr) + + def get_config(self) -> Mapping[str, Any]: + return { + "rescaled_lr": self._rescaled_lr, + "step_boundaries": self._step_boundaries, + "lr_values": self._lr_values, + "warmup_steps": self._warmup_steps, + } diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/learning_rate_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/learning_rate_test.py new file mode 100644 index 0000000..8241151 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/learning_rate_test.py @@ -0,0 +1,89 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for learning_rate.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from official.vision.image_classification import learning_rate + + +class LearningRateTests(tf.test.TestCase): + + def test_warmup_decay(self): + """Basic computational test for warmup decay.""" + initial_lr = 0.01 + decay_steps = 100 + decay_rate = 0.01 + warmup_steps = 10 + + base_lr = tf.keras.optimizers.schedules.ExponentialDecay( + initial_learning_rate=initial_lr, + decay_steps=decay_steps, + decay_rate=decay_rate) + lr = learning_rate.WarmupDecaySchedule( + lr_schedule=base_lr, + warmup_steps=warmup_steps) + + for step in range(warmup_steps - 1): + config = lr.get_config() + self.assertEqual(config['warmup_steps'], warmup_steps) + self.assertAllClose(self.evaluate(lr(step)), + step / warmup_steps * initial_lr) + + def test_piecewise_constant_decay_with_warmup(self): + """Basic computational test for piecewise constant decay with warmup.""" + boundaries = [1, 2, 3] + warmup_epochs = boundaries[0] + learning_rate_multipliers = [1.0, 0.1, 0.001] + expected_keys = [ + 'rescaled_lr', 'step_boundaries', 'lr_values', 'warmup_steps', + ] + + expected_lrs = [0.0, 0.1, 0.1] + + lr = learning_rate.PiecewiseConstantDecayWithWarmup( + batch_size=256, + epoch_size=256, + warmup_epochs=warmup_epochs, + boundaries=boundaries[1:], + multipliers=learning_rate_multipliers) + + step = 0 + + config = lr.get_config() + self.assertAllInSet(list(config.keys()), expected_keys) + + for boundary, expected_lr in zip(boundaries, expected_lrs): + for _ in range(step, boundary): + self.assertAllClose(self.evaluate(lr(step)), expected_lr) + step += 1 + + def test_piecewise_constant_decay_invalid_boundaries(self): + with self.assertRaisesRegex(ValueError, + 'The length of boundaries must be 1 less '): + learning_rate.PiecewiseConstantDecayWithWarmup( + batch_size=256, + epoch_size=256, + warmup_epochs=1, + boundaries=[1, 2], + multipliers=[1, 2]) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/mnist_main.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/mnist_main.py new file mode 100644 index 0000000..1470c02 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/mnist_main.py @@ -0,0 +1,171 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Runs a simple model on the MNIST dataset.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from absl import app +from absl import flags +from absl import logging +import tensorflow as tf +import tensorflow_datasets as tfds + +from official.utils.flags import core as flags_core +from official.utils.misc import distribution_utils +from official.utils.misc import model_helpers +from official.vision.image_classification.resnet import common + +FLAGS = flags.FLAGS + + +def build_model(): + """Constructs the ML model used to predict handwritten digits.""" + + image = tf.keras.layers.Input(shape=(28, 28, 1)) + + y = tf.keras.layers.Conv2D(filters=32, + kernel_size=5, + padding='same', + activation='relu')(image) + y = tf.keras.layers.MaxPooling2D(pool_size=(2, 2), + strides=(2, 2), + padding='same')(y) + y = tf.keras.layers.Conv2D(filters=32, + kernel_size=5, + padding='same', + activation='relu')(y) + y = tf.keras.layers.MaxPooling2D(pool_size=(2, 2), + strides=(2, 2), + padding='same')(y) + y = tf.keras.layers.Flatten()(y) + y = tf.keras.layers.Dense(1024, activation='relu')(y) + y = tf.keras.layers.Dropout(0.4)(y) + + probs = tf.keras.layers.Dense(10, activation='softmax')(y) + + model = tf.keras.models.Model(image, probs, name='mnist') + + return model + + +@tfds.decode.make_decoder(output_dtype=tf.float32) +def decode_image(example, feature): + """Convert image to float32 and normalize from [0, 255] to [0.0, 1.0].""" + return tf.cast(feature.decode_example(example), dtype=tf.float32) / 255 + + +def run(flags_obj, datasets_override=None, strategy_override=None): + """Run MNIST model training and eval loop using native Keras APIs. + + Args: + flags_obj: An object containing parsed flag values. + datasets_override: A pair of `tf.data.Dataset` objects to train the model, + representing the train and test sets. + strategy_override: A `tf.distribute.Strategy` object to use for model. + + Returns: + Dictionary of training and eval stats. + """ + strategy = strategy_override or distribution_utils.get_distribution_strategy( + distribution_strategy=flags_obj.distribution_strategy, + num_gpus=flags_obj.num_gpus, + tpu_address=flags_obj.tpu) + + strategy_scope = distribution_utils.get_strategy_scope(strategy) + + mnist = tfds.builder('mnist', data_dir=flags_obj.data_dir) + if flags_obj.download: + mnist.download_and_prepare() + + mnist_train, mnist_test = datasets_override or mnist.as_dataset( + split=['train', 'test'], + decoders={'image': decode_image()}, # pylint: disable=no-value-for-parameter + as_supervised=True) + train_input_dataset = mnist_train.cache().repeat().shuffle( + buffer_size=50000).batch(flags_obj.batch_size) + eval_input_dataset = mnist_test.cache().repeat().batch(flags_obj.batch_size) + + with strategy_scope: + lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay( + 0.05, decay_steps=100000, decay_rate=0.96) + optimizer = tf.keras.optimizers.SGD(learning_rate=lr_schedule) + + model = build_model() + model.compile( + optimizer=optimizer, + loss='sparse_categorical_crossentropy', + metrics=['sparse_categorical_accuracy']) + + num_train_examples = mnist.info.splits['train'].num_examples + train_steps = num_train_examples // flags_obj.batch_size + train_epochs = flags_obj.train_epochs + + ckpt_full_path = os.path.join(flags_obj.model_dir, 'model.ckpt-{epoch:04d}') + callbacks = [ + tf.keras.callbacks.ModelCheckpoint( + ckpt_full_path, save_weights_only=True), + tf.keras.callbacks.TensorBoard(log_dir=flags_obj.model_dir), + ] + + num_eval_examples = mnist.info.splits['test'].num_examples + num_eval_steps = num_eval_examples // flags_obj.batch_size + + history = model.fit( + train_input_dataset, + epochs=train_epochs, + steps_per_epoch=train_steps, + callbacks=callbacks, + validation_steps=num_eval_steps, + validation_data=eval_input_dataset, + validation_freq=flags_obj.epochs_between_evals) + + export_path = os.path.join(flags_obj.model_dir, 'saved_model') + model.save(export_path, include_optimizer=False) + + eval_output = model.evaluate( + eval_input_dataset, steps=num_eval_steps, verbose=2) + + stats = common.build_stats(history, eval_output, callbacks) + return stats + + +def define_mnist_flags(): + """Define command line flags for MNIST model.""" + flags_core.define_base( + clean=True, + num_gpu=True, + train_epochs=True, + epochs_between_evals=True, + distribution_strategy=True) + flags_core.define_device() + flags_core.define_distribution() + flags.DEFINE_bool('download', False, + 'Whether to download data to `--data_dir`.') + FLAGS.set_default('batch_size', 1024) + + +def main(_): + model_helpers.apply_clean(FLAGS) + stats = run(flags.FLAGS) + logging.info('Run stats:\n%s', stats) + + +if __name__ == '__main__': + logging.set_verbosity(logging.INFO) + define_mnist_flags() + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/mnist_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/mnist_test.py new file mode 100644 index 0000000..a9c233d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/mnist_test.py @@ -0,0 +1,90 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Test the Keras MNIST model on GPU.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools + +from absl.testing import parameterized +import tensorflow as tf + +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +from official.utils.misc import keras_utils +from official.utils.testing import integration +from official.vision.image_classification import mnist_main + + +def eager_strategy_combinations(): + return combinations.combine( + distribution=[ + strategy_combinations.default_strategy, + strategy_combinations.tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + ], + mode="eager", + ) + + +class KerasMnistTest(tf.test.TestCase, parameterized.TestCase): + """Unit tests for sample Keras MNIST model.""" + _tempdir = None + + @classmethod + def setUpClass(cls): # pylint: disable=invalid-name + super(KerasMnistTest, cls).setUpClass() + mnist_main.define_mnist_flags() + + def tearDown(self): + super(KerasMnistTest, self).tearDown() + tf.io.gfile.rmtree(self.get_temp_dir()) + + @combinations.generate(eager_strategy_combinations()) + def test_end_to_end(self, distribution): + """Test Keras MNIST model with `strategy`.""" + config = keras_utils.get_config_proto_v1() + tf.compat.v1.enable_eager_execution(config=config) + + extra_flags = [ + "-train_epochs", "1", + # Let TFDS find the metadata folder automatically + "--data_dir=" + ] + + dummy_data = ( + tf.ones(shape=(10, 28, 28, 1), dtype=tf.int32), + tf.range(10), + ) + datasets = ( + tf.data.Dataset.from_tensor_slices(dummy_data), + tf.data.Dataset.from_tensor_slices(dummy_data), + ) + + run = functools.partial(mnist_main.run, + datasets_override=datasets, + strategy_override=distribution) + + integration.run_synthetic( + main=run, + synth=False, + tmp_root=self.get_temp_dir(), + extra_flags=extra_flags) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/optimizer_factory.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/optimizer_factory.py new file mode 100644 index 0000000..4320583 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/optimizer_factory.py @@ -0,0 +1,383 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Optimizer factory for vision tasks.""" +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +from absl import logging +import tensorflow as tf +import tensorflow_addons as tfa + +from typing import Any, Dict, Text, List +from official.vision.image_classification import learning_rate +from official.vision.image_classification.configs import base_configs + +# pylint: disable=protected-access + + +class MovingAverage(tf.keras.optimizers.Optimizer): + """Optimizer that computes a moving average of the variables. + + Empirically it has been found that using the moving average of the trained + parameters of a deep network is better than using its trained parameters + directly. This optimizer allows you to compute this moving average and swap + the variables at save time so that any code outside of the training loop + will use by default the average values instead of the original ones. + + Example of usage for training: + ```python + opt = tf.keras.optimizers.SGD(learning_rate) + opt = MovingAverage(opt) + + opt.shadow_copy(model) + ``` + + At test time, swap the shadow variables to evaluate on the averaged weights: + ```python + opt.swap_weights() + # Test eval the model here + opt.swap_weights() + ``` + """ + + def __init__(self, + optimizer: tf.keras.optimizers.Optimizer, + average_decay: float = 0.99, + start_step: int = 0, + dynamic_decay: bool = True, + name: Text = 'moving_average', + **kwargs): + """Construct a new MovingAverage optimizer. + + Args: + optimizer: `tf.keras.optimizers.Optimizer` that will be + used to compute and apply gradients. + average_decay: float. Decay to use to maintain the moving averages + of trained variables. + start_step: int. What step to start the moving average. + dynamic_decay: bool. Whether to change the decay based on the number + of optimizer updates. Decay will start at 0.1 and gradually increase + up to `average_decay` after each optimizer update. This behavior is + similar to `tf.train.ExponentialMovingAverage` in TF 1.x. + name: Optional name for the operations created when applying + gradients. Defaults to "moving_average". + **kwargs: keyword arguments. Allowed to be {`clipnorm`, + `clipvalue`, `lr`, `decay`}. + """ + super(MovingAverage, self).__init__(name, **kwargs) + self._optimizer = optimizer + self._average_decay = average_decay + self._start_step = tf.constant(start_step, tf.float32) + self._dynamic_decay = dynamic_decay + + def shadow_copy(self, model: tf.keras.Model): + """Creates shadow variables for the given model weights.""" + for var in model.weights: + self.add_slot(var, 'average', initializer='zeros') + self._average_weights = [ + self.get_slot(var, 'average') for var in model.weights + ] + self._model_weights = model.weights + + @property + def has_shadow_copy(self): + """Whether this optimizer has created shadow variables.""" + return self._model_weights is not None + + def _create_slots(self, var_list): + self._optimizer._create_slots(var_list=var_list) # pylint: disable=protected-access + + def apply_gradients(self, grads_and_vars, name: Text = None): + result = self._optimizer.apply_gradients(grads_and_vars, name) + self.update_average(self._optimizer.iterations) + return result + + @tf.function + def update_average(self, step: tf.Tensor): + step = tf.cast(step, tf.float32) + if step < self._start_step: + decay = tf.constant(0., tf.float32) + elif self._dynamic_decay: + decay = step - self._start_step + decay = tf.minimum(self._average_decay, (1. + decay) / (10. + decay)) + else: + decay = self._average_decay + + def _apply_moving(v_moving, v_normal): + diff = v_moving - v_normal + v_moving.assign_sub(tf.cast(1. - decay, v_moving.dtype) * diff) + return v_moving + + def _update(strategy, v_moving_and_v_normal): + for v_moving, v_normal in v_moving_and_v_normal: + strategy.extended.update(v_moving, _apply_moving, args=(v_normal,)) + + ctx = tf.distribute.get_replica_context() + return ctx.merge_call(_update, args=(zip(self._average_weights, + self._model_weights),)) + + def swap_weights(self): + """Swap the average and moving weights. + + This is a convenience method to allow one to evaluate the averaged weights + at test time. Loads the weights stored in `self._average` into the model, + keeping a copy of the original model weights. Swapping twice will return + the original weights. + """ + if tf.distribute.in_cross_replica_context(): + strategy = tf.distribute.get_strategy() + strategy.run(self._swap_weights, args=()) + else: + raise ValueError('Swapping weights must occur under a ' + 'tf.distribute.Strategy') + + @tf.function + def _swap_weights(self): + def fn_0(a, b): + a.assign_add(b) + return a + def fn_1(b, a): + b.assign(a - b) + return b + def fn_2(a, b): + a.assign_sub(b) + return a + + def swap(strategy, a_and_b): + """Swap `a` and `b` and mirror to all devices.""" + for a, b in a_and_b: + strategy.extended.update(a, fn_0, args=(b,)) # a = a + b + strategy.extended.update(b, fn_1, args=(a,)) # b = a - b + strategy.extended.update(a, fn_2, args=(b,)) # a = a - b + + ctx = tf.distribute.get_replica_context() + return ctx.merge_call( + swap, args=(zip(self._average_weights, self._model_weights),)) + + def assign_average_vars(self, var_list: List[tf.Variable]): + """Assign variables in var_list with their respective averages. + + Args: + var_list: List of model variables to be assigned to their average. + Returns: + assign_op: The op corresponding to the assignment operation of + variables to their average. + """ + assign_op = tf.group([ + var.assign(self.get_slot(var, 'average')) for var in var_list + if var.trainable + ]) + return assign_op + + def _create_hypers(self): + self._optimizer._create_hypers() # pylint: disable=protected-access + + def _prepare(self, var_list): + return self._optimizer._prepare(var_list=var_list) # pylint: disable=protected-access + + @property + def iterations(self): + return self._optimizer.iterations + + @iterations.setter + def iterations(self, variable): + self._optimizer.iterations = variable + + @property + def weights(self): + # return self._weights + self._optimizer.weights + return self._optimizer.weights + + @property + def lr(self): + return self._optimizer._get_hyper('learning_rate') + + @lr.setter + def lr(self, lr): + self._optimizer._set_hyper('learning_rate', lr) + + @property + def learning_rate(self): + return self._optimizer._get_hyper('learning_rate') + + @learning_rate.setter + def learning_rate(self, learning_rate): # pylint: disable=redefined-outer-name + self._optimizer._set_hyper('learning_rate', learning_rate) + + def _resource_apply_dense(self, grad, var): + return self._optimizer._resource_apply_dense(grad, var) + + def _resource_apply_sparse(self, grad, var, indices): + return self._optimizer._resource_apply_sparse(grad, var, indices) + + def _resource_apply_sparse_duplicate_indices(self, grad, var, indices): + return self._optimizer._resource_apply_sparse_duplicate_indices( + grad, var, indices) + + def get_config(self): + config = { + 'optimizer': tf.keras.optimizers.serialize(self._optimizer), + 'average_decay': self._average_decay, + 'start_step': self._start_step, + 'dynamic_decay': self._dynamic_decay, + } + base_config = super(MovingAverage, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + @classmethod + def from_config(cls, config, custom_objects=None): + optimizer = tf.keras.optimizers.deserialize( + config.pop('optimizer'), + custom_objects=custom_objects, + ) + return cls(optimizer, **config) + + +def build_optimizer( + optimizer_name: Text, + base_learning_rate: tf.keras.optimizers.schedules.LearningRateSchedule, + params: Dict[Text, Any]): + """Build the optimizer based on name. + + Args: + optimizer_name: String representation of the optimizer name. Examples: + sgd, momentum, rmsprop. + base_learning_rate: `tf.keras.optimizers.schedules.LearningRateSchedule` + base learning rate. + params: String -> Any dictionary representing the optimizer params. + This should contain optimizer specific parameters such as + `base_learning_rate`, `decay`, etc. + + Returns: + A tf.keras.Optimizer. + + Raises: + ValueError if the provided optimizer_name is not supported. + + """ + optimizer_name = optimizer_name.lower() + logging.info('Building %s optimizer with params %s', optimizer_name, params) + + if optimizer_name == 'sgd': + logging.info('Using SGD optimizer') + nesterov = params.get('nesterov', False) + optimizer = tf.keras.optimizers.SGD(learning_rate=base_learning_rate, + nesterov=nesterov) + elif optimizer_name == 'momentum': + logging.info('Using momentum optimizer') + nesterov = params.get('nesterov', False) + optimizer = tf.keras.optimizers.SGD(learning_rate=base_learning_rate, + momentum=params['momentum'], + nesterov=nesterov) + elif optimizer_name == 'rmsprop': + logging.info('Using RMSProp') + rho = params.get('decay', None) or params.get('rho', 0.9) + momentum = params.get('momentum', 0.9) + epsilon = params.get('epsilon', 1e-07) + optimizer = tf.keras.optimizers.RMSprop(learning_rate=base_learning_rate, + rho=rho, + momentum=momentum, + epsilon=epsilon) + elif optimizer_name == 'adam': + logging.info('Using Adam') + beta_1 = params.get('beta_1', 0.9) + beta_2 = params.get('beta_2', 0.999) + epsilon = params.get('epsilon', 1e-07) + optimizer = tf.keras.optimizers.Adam(learning_rate=base_learning_rate, + beta_1=beta_1, + beta_2=beta_2, + epsilon=epsilon) + elif optimizer_name == 'adamw': + logging.info('Using AdamW') + weight_decay = params.get('weight_decay', 0.01) + beta_1 = params.get('beta_1', 0.9) + beta_2 = params.get('beta_2', 0.999) + epsilon = params.get('epsilon', 1e-07) + optimizer = tfa.optimizers.AdamW(weight_decay=weight_decay, + learning_rate=base_learning_rate, + beta_1=beta_1, + beta_2=beta_2, + epsilon=epsilon) + else: + raise ValueError('Unknown optimizer %s' % optimizer_name) + + if params.get('lookahead', None): + logging.info('Using lookahead optimizer.') + optimizer = tfa.optimizers.Lookahead(optimizer) + + # Moving average should be applied last, as it's applied at test time + moving_average_decay = params.get('moving_average_decay', 0.) + if moving_average_decay is not None and moving_average_decay > 0.: + logging.info('Including moving average decay.') + optimizer = MovingAverage( + optimizer, + average_decay=moving_average_decay) + return optimizer + + +def build_learning_rate(params: base_configs.LearningRateConfig, + batch_size: int = None, + train_steps: int = None): + """Build the learning rate given the provided configuration.""" + decay_type = params.name + base_lr = params.initial_lr + decay_rate = params.decay_rate + if params.decay_epochs is not None: + decay_steps = params.decay_epochs * train_steps + else: + decay_steps = 0 + if params.warmup_epochs is not None: + warmup_steps = params.warmup_epochs * train_steps + else: + warmup_steps = 0 + + lr_multiplier = params.scale_by_batch_size + + if lr_multiplier and lr_multiplier > 0: + # Scale the learning rate based on the batch size and a multiplier + base_lr *= lr_multiplier * batch_size + logging.info('Scaling the learning rate based on the batch size ' + 'multiplier. New base_lr: %f', base_lr) + + if decay_type == 'exponential': + logging.info('Using exponential learning rate with: ' + 'initial_learning_rate: %f, decay_steps: %d, ' + 'decay_rate: %f', base_lr, decay_steps, decay_rate) + lr = tf.keras.optimizers.schedules.ExponentialDecay( + initial_learning_rate=base_lr, + decay_steps=decay_steps, + decay_rate=decay_rate, + staircase=params.staircase) + elif decay_type == 'piecewise_constant_with_warmup': + logging.info('Using Piecewise constant decay with warmup. ' + 'Parameters: batch_size: %d, epoch_size: %d, ' + 'warmup_epochs: %d, boundaries: %s, multipliers: %s', + batch_size, params.examples_per_epoch, + params.warmup_epochs, params.boundaries, + params.multipliers) + lr = learning_rate.PiecewiseConstantDecayWithWarmup( + batch_size=batch_size, + epoch_size=params.examples_per_epoch, + warmup_epochs=params.warmup_epochs, + boundaries=params.boundaries, + multipliers=params.multipliers) + if warmup_steps > 0: + if decay_type != 'piecewise_constant_with_warmup': + logging.info('Applying %d warmup steps to the learning rate', + warmup_steps) + lr = learning_rate.WarmupDecaySchedule(lr, warmup_steps) + return lr diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/optimizer_factory_test.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/optimizer_factory_test.py new file mode 100644 index 0000000..da16155 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/optimizer_factory_test.py @@ -0,0 +1,114 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for optimizer_factory.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow as tf + +from absl.testing import parameterized +from official.vision.image_classification import optimizer_factory +from official.vision.image_classification.configs import base_configs + + +class OptimizerFactoryTest(tf.test.TestCase, parameterized.TestCase): + + @parameterized.named_parameters( + ('sgd', 'sgd', 0., False), + ('momentum', 'momentum', 0., False), + ('rmsprop', 'rmsprop', 0., False), + ('adam', 'adam', 0., False), + ('adamw', 'adamw', 0., False), + ('momentum_lookahead', 'momentum', 0., True), + ('sgd_ema', 'sgd', 0.999, False), + ('momentum_ema', 'momentum', 0.999, False), + ('rmsprop_ema', 'rmsprop', 0.999, False)) + def test_optimizer(self, optimizer_name, moving_average_decay, lookahead): + """Smoke test to be sure no syntax errors.""" + params = { + 'learning_rate': 0.001, + 'rho': 0.09, + 'momentum': 0., + 'epsilon': 1e-07, + 'moving_average_decay': moving_average_decay, + 'lookahead': lookahead, + } + optimizer = optimizer_factory.build_optimizer( + optimizer_name=optimizer_name, + base_learning_rate=params['learning_rate'], + params=params) + self.assertTrue(issubclass(type(optimizer), tf.keras.optimizers.Optimizer)) + + def test_unknown_optimizer(self): + with self.assertRaises(ValueError): + optimizer_factory.build_optimizer( + optimizer_name='this_optimizer_does_not_exist', + base_learning_rate=None, + params=None) + + def test_learning_rate_without_decay_or_warmups(self): + params = base_configs.LearningRateConfig( + name='exponential', + initial_lr=0.01, + decay_rate=0.01, + decay_epochs=None, + warmup_epochs=None, + scale_by_batch_size=0.01, + examples_per_epoch=1, + boundaries=[0], + multipliers=[0, 1]) + batch_size = 1 + train_steps = 1 + + lr = optimizer_factory.build_learning_rate( + params=params, + batch_size=batch_size, + train_steps=train_steps) + self.assertTrue( + issubclass( + type(lr), tf.keras.optimizers.schedules.LearningRateSchedule)) + + @parameterized.named_parameters( + ('exponential', 'exponential'), + ('piecewise_constant_with_warmup', 'piecewise_constant_with_warmup')) + def test_learning_rate_with_decay_and_warmup(self, lr_decay_type): + """Basic smoke test for syntax.""" + params = base_configs.LearningRateConfig( + name=lr_decay_type, + initial_lr=0.01, + decay_rate=0.01, + decay_epochs=1, + warmup_epochs=1, + scale_by_batch_size=0.01, + examples_per_epoch=1, + boundaries=[0], + multipliers=[0, 1]) + batch_size = 1 + train_steps = 1 + + lr = optimizer_factory.build_learning_rate( + params=params, + batch_size=batch_size, + train_steps=train_steps) + self.assertTrue( + issubclass( + type(lr), tf.keras.optimizers.schedules.LearningRateSchedule)) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/preprocessing.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/preprocessing.py new file mode 100644 index 0000000..3f20191 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/preprocessing.py @@ -0,0 +1,391 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Preprocessing functions for images.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import tensorflow as tf +from typing import List, Optional, Text, Tuple + +from official.vision.image_classification import augment + + +# Calculated from the ImageNet training set +MEAN_RGB = (0.485 * 255, 0.456 * 255, 0.406 * 255) +STDDEV_RGB = (0.229 * 255, 0.224 * 255, 0.225 * 255) + +IMAGE_SIZE = 224 +CROP_PADDING = 32 + + +def mean_image_subtraction( + image_bytes: tf.Tensor, + means: Tuple[float, ...], + num_channels: int = 3, + dtype: tf.dtypes.DType = tf.float32, +) -> tf.Tensor: + """Subtracts the given means from each image channel. + + For example: + means = [123.68, 116.779, 103.939] + image_bytes = mean_image_subtraction(image_bytes, means) + + Note that the rank of `image` must be known. + + Args: + image_bytes: a tensor of size [height, width, C]. + means: a C-vector of values to subtract from each channel. + num_channels: number of color channels in the image that will be distorted. + dtype: the dtype to convert the images to. Set to `None` to skip conversion. + + Returns: + the centered image. + + Raises: + ValueError: If the rank of `image` is unknown, if `image` has a rank other + than three or if the number of channels in `image` doesn't match the + number of values in `means`. + """ + if image_bytes.get_shape().ndims != 3: + raise ValueError('Input must be of size [height, width, C>0]') + + if len(means) != num_channels: + raise ValueError('len(means) must match the number of channels') + + # We have a 1-D tensor of means; convert to 3-D. + # Note(b/130245863): we explicitly call `broadcast` instead of simply + # expanding dimensions for better performance. + means = tf.broadcast_to(means, tf.shape(image_bytes)) + if dtype is not None: + means = tf.cast(means, dtype=dtype) + + return image_bytes - means + + +def standardize_image( + image_bytes: tf.Tensor, + stddev: Tuple[float, ...], + num_channels: int = 3, + dtype: tf.dtypes.DType = tf.float32, +) -> tf.Tensor: + """Divides the given stddev from each image channel. + + For example: + stddev = [123.68, 116.779, 103.939] + image_bytes = standardize_image(image_bytes, stddev) + + Note that the rank of `image` must be known. + + Args: + image_bytes: a tensor of size [height, width, C]. + stddev: a C-vector of values to divide from each channel. + num_channels: number of color channels in the image that will be distorted. + dtype: the dtype to convert the images to. Set to `None` to skip conversion. + + Returns: + the centered image. + + Raises: + ValueError: If the rank of `image` is unknown, if `image` has a rank other + than three or if the number of channels in `image` doesn't match the + number of values in `stddev`. + """ + if image_bytes.get_shape().ndims != 3: + raise ValueError('Input must be of size [height, width, C>0]') + + if len(stddev) != num_channels: + raise ValueError('len(stddev) must match the number of channels') + + # We have a 1-D tensor of stddev; convert to 3-D. + # Note(b/130245863): we explicitly call `broadcast` instead of simply + # expanding dimensions for better performance. + stddev = tf.broadcast_to(stddev, tf.shape(image_bytes)) + if dtype is not None: + stddev = tf.cast(stddev, dtype=dtype) + + return image_bytes / stddev + + +def normalize_images(features: tf.Tensor, + mean_rgb: Tuple[float, ...] = MEAN_RGB, + stddev_rgb: Tuple[float, ...] = STDDEV_RGB, + num_channels: int = 3, + dtype: tf.dtypes.DType = tf.float32, + data_format: Text = 'channels_last') -> tf.Tensor: + """Normalizes the input image channels with the given mean and stddev. + + Args: + features: `Tensor` representing decoded images in float format. + mean_rgb: the mean of the channels to subtract. + stddev_rgb: the stddev of the channels to divide. + num_channels: the number of channels in the input image tensor. + dtype: the dtype to convert the images to. Set to `None` to skip conversion. + data_format: the format of the input image tensor + ['channels_first', 'channels_last']. + + Returns: + A normalized image `Tensor`. + """ + # TODO(allencwang) - figure out how to use mean_image_subtraction and + # standardize_image on batches of images and replace the following. + if data_format == 'channels_first': + stats_shape = [num_channels, 1, 1] + else: + stats_shape = [1, 1, num_channels] + + if dtype is not None: + features = tf.image.convert_image_dtype(features, dtype=dtype) + + if mean_rgb is not None: + mean_rgb = tf.constant(mean_rgb, + shape=stats_shape, + dtype=features.dtype) + mean_rgb = tf.broadcast_to(mean_rgb, tf.shape(features)) + features = features - mean_rgb + + if stddev_rgb is not None: + stddev_rgb = tf.constant(stddev_rgb, + shape=stats_shape, + dtype=features.dtype) + stddev_rgb = tf.broadcast_to(stddev_rgb, tf.shape(features)) + features = features / stddev_rgb + + return features + + +def decode_and_center_crop(image_bytes: tf.Tensor, + image_size: int = IMAGE_SIZE, + crop_padding: int = CROP_PADDING) -> tf.Tensor: + """Crops to center of image with padding then scales image_size. + + Args: + image_bytes: `Tensor` representing an image binary of arbitrary size. + image_size: image height/width dimension. + crop_padding: the padding size to use when centering the crop. + + Returns: + A decoded and cropped image `Tensor`. + """ + decoded = image_bytes.dtype != tf.string + shape = (tf.shape(image_bytes) if decoded + else tf.image.extract_jpeg_shape(image_bytes)) + image_height = shape[0] + image_width = shape[1] + + padded_center_crop_size = tf.cast( + ((image_size / (image_size + crop_padding)) * + tf.cast(tf.minimum(image_height, image_width), tf.float32)), + tf.int32) + + offset_height = ((image_height - padded_center_crop_size) + 1) // 2 + offset_width = ((image_width - padded_center_crop_size) + 1) // 2 + crop_window = tf.stack([offset_height, offset_width, + padded_center_crop_size, padded_center_crop_size]) + if decoded: + image = tf.image.crop_to_bounding_box( + image_bytes, + offset_height=offset_height, + offset_width=offset_width, + target_height=padded_center_crop_size, + target_width=padded_center_crop_size) + else: + image = tf.image.decode_and_crop_jpeg(image_bytes, crop_window, channels=3) + + image = resize_image(image_bytes=image, + height=image_size, + width=image_size) + + return image + + +def decode_crop_and_flip(image_bytes: tf.Tensor) -> tf.Tensor: + """Crops an image to a random part of the image, then randomly flips. + + Args: + image_bytes: `Tensor` representing an image binary of arbitrary size. + + Returns: + A decoded and cropped image `Tensor`. + + """ + decoded = image_bytes.dtype != tf.string + bbox = tf.constant([0.0, 0.0, 1.0, 1.0], dtype=tf.float32, shape=[1, 1, 4]) + shape = (tf.shape(image_bytes) if decoded + else tf.image.extract_jpeg_shape(image_bytes)) + sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box( + shape, + bounding_boxes=bbox, + min_object_covered=0.1, + aspect_ratio_range=[0.75, 1.33], + area_range=[0.05, 1.0], + max_attempts=100, + use_image_if_no_bounding_boxes=True) + bbox_begin, bbox_size, _ = sample_distorted_bounding_box + + # Reassemble the bounding box in the format the crop op requires. + offset_height, offset_width, _ = tf.unstack(bbox_begin) + target_height, target_width, _ = tf.unstack(bbox_size) + crop_window = tf.stack([offset_height, offset_width, + target_height, target_width]) + if decoded: + cropped = tf.image.crop_to_bounding_box( + image_bytes, + offset_height=offset_height, + offset_width=offset_width, + target_height=target_height, + target_width=target_width) + else: + cropped = tf.image.decode_and_crop_jpeg(image_bytes, + crop_window, + channels=3) + + # Flip to add a little more random distortion in. + cropped = tf.image.random_flip_left_right(cropped) + return cropped + + +def resize_image(image_bytes: tf.Tensor, + height: int = IMAGE_SIZE, + width: int = IMAGE_SIZE) -> tf.Tensor: + """Resizes an image to a given height and width. + + Args: + image_bytes: `Tensor` representing an image binary of arbitrary size. + height: image height dimension. + width: image width dimension. + + Returns: + A tensor containing the resized image. + + """ + return tf.compat.v1.image.resize( + image_bytes, [height, width], method=tf.image.ResizeMethod.BILINEAR, + align_corners=False) + + +def preprocess_for_eval( + image_bytes: tf.Tensor, + image_size: int = IMAGE_SIZE, + num_channels: int = 3, + mean_subtract: bool = False, + standardize: bool = False, + dtype: tf.dtypes.DType = tf.float32 +) -> tf.Tensor: + """Preprocesses the given image for evaluation. + + Args: + image_bytes: `Tensor` representing an image binary of arbitrary size. + image_size: image height/width dimension. + num_channels: number of image input channels. + mean_subtract: whether or not to apply mean subtraction. + standardize: whether or not to apply standardization. + dtype: the dtype to convert the images to. Set to `None` to skip conversion. + + Returns: + A preprocessed and normalized image `Tensor`. + """ + images = decode_and_center_crop(image_bytes, image_size) + images = tf.reshape(images, [image_size, image_size, num_channels]) + + if mean_subtract: + images = mean_image_subtraction(image_bytes=images, means=MEAN_RGB) + if standardize: + images = standardize_image(image_bytes=images, stddev=STDDEV_RGB) + if dtype is not None: + images = tf.image.convert_image_dtype(images, dtype=dtype) + + return images + + +def load_eval_image(filename: Text, image_size: int = IMAGE_SIZE) -> tf.Tensor: + """Reads an image from the filesystem and applies image preprocessing. + + Args: + filename: a filename path of an image. + image_size: image height/width dimension. + + Returns: + A preprocessed and normalized image `Tensor`. + """ + image_bytes = tf.io.read_file(filename) + image = preprocess_for_eval(image_bytes, image_size) + + return image + + +def build_eval_dataset(filenames: List[Text], + labels: List[int] = None, + image_size: int = IMAGE_SIZE, + batch_size: int = 1) -> tf.Tensor: + """Builds a tf.data.Dataset from a list of filenames and labels. + + Args: + filenames: a list of filename paths of images. + labels: a list of labels corresponding to each image. + image_size: image height/width dimension. + batch_size: the batch size used by the dataset + + Returns: + A preprocessed and normalized image `Tensor`. + """ + if labels is None: + labels = [0] * len(filenames) + + filenames = tf.constant(filenames) + labels = tf.constant(labels) + dataset = tf.data.Dataset.from_tensor_slices((filenames, labels)) + + dataset = dataset.map( + lambda filename, label: (load_eval_image(filename, image_size), label)) + dataset = dataset.batch(batch_size) + + return dataset + + +def preprocess_for_train(image_bytes: tf.Tensor, + image_size: int = IMAGE_SIZE, + augmenter: Optional[augment.ImageAugment] = None, + mean_subtract: bool = False, + standardize: bool = False, + dtype: tf.dtypes.DType = tf.float32) -> tf.Tensor: + """Preprocesses the given image for training. + + Args: + image_bytes: `Tensor` representing an image binary of + arbitrary size of dtype tf.uint8. + image_size: image height/width dimension. + augmenter: the image augmenter to apply. + mean_subtract: whether or not to apply mean subtraction. + standardize: whether or not to apply standardization. + dtype: the dtype to convert the images to. Set to `None` to skip conversion. + + Returns: + A preprocessed and normalized image `Tensor`. + """ + images = decode_crop_and_flip(image_bytes=image_bytes) + images = resize_image(images, height=image_size, width=image_size) + if mean_subtract: + images = mean_image_subtraction(image_bytes=images, means=MEAN_RGB) + if standardize: + images = standardize_image(image_bytes=images, stddev=STDDEV_RGB) + if augmenter is not None: + images = augmenter.distort(images) + if dtype is not None: + images = tf.image.convert_image_dtype(images, dtype) + + return images diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/README.md b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/README.md new file mode 100644 index 0000000..d5923a8 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/README.md @@ -0,0 +1,129 @@ +This folder contains a compile/fit and +[custom training loop (CTL)](#resnet-custom-training-loop) implementation for +ResNet50. + +## Before you begin +Please refer to the [README](../README.md) in the parent directory for +information on setup and preparing the data. + +## ResNet (custom training loop) + +Similar to the [estimator implementation](../../../r1/resnet), the Keras +implementation has code for the ImageNet dataset. The ImageNet +version uses a ResNet50 model implemented in +[`resnet_model.py`](./resnet_model.py). + + +### Pretrained Models + +* [ResNet50 Checkpoints](https://storage.googleapis.com/cloud-tpu-checkpoints/resnet/resnet50.tar.gz) + +* ResNet50 TFHub: [feature vector](https://tfhub.dev/tensorflow/resnet_50/feature_vector/1) +and [classification](https://tfhub.dev/tensorflow/resnet_50/classification/1) + +```bash +python3 resnet_imagenet_main.py +``` + +Again, if you did not download the data to the default directory, specify the +location with the `--data_dir` flag: + +```bash +python3 resnet_imagenet_main.py --data_dir=/path/to/imagenet +``` + +There are more flag options you can specify. Here are some examples: + +- `--use_synthetic_data`: when set to true, synthetic data, rather than real +data, are used; +- `--batch_size`: the batch size used for the model; +- `--model_dir`: the directory to save the model checkpoint; +- `--train_epochs`: number of epoches to run for training the model; +- `--train_steps`: number of steps to run for training the model. We now only +support a number that is smaller than the number of batches in an epoch. +- `--skip_eval`: when set to true, evaluation as well as validation during +training is skipped + +For example, this is a typical command line to run with ImageNet data with +batch size 128 per GPU: + +```bash +python3 -m resnet_imagenet_main.py \ + --model_dir=/tmp/model_dir/something \ + --num_gpus=2 \ + --batch_size=128 \ + --train_epochs=90 \ + --train_steps=10 \ + --use_synthetic_data=false +``` + +See [`common.py`](common.py) for full list of options. + +### Using multiple GPUs + +You can train these models on multiple GPUs using `tf.distribute.Strategy` API. +You can read more about them in this +[guide](https://www.tensorflow.org/guide/distribute_strategy). + +In this example, we have made it easier to use is with just a command line flag +`--num_gpus`. By default this flag is 1 if TensorFlow is compiled with CUDA, +and 0 otherwise. + +- --num_gpus=0: Uses tf.distribute.OneDeviceStrategy with CPU as the device. +- --num_gpus=1: Uses tf.distribute.OneDeviceStrategy with GPU as the device. +- --num_gpus=2+: Uses tf.distribute.MirroredStrategy to run synchronous +distributed training across the GPUs. + +If you wish to run without `tf.distribute.Strategy`, you can do so by setting +`--distribution_strategy=off`. + +### Running on multiple GPU hosts + +You can also train these models on multiple hosts, each with GPUs, using +`tf.distribute.Strategy`. + +The easiest way to run multi-host benchmarks is to set the +[`TF_CONFIG`](https://www.tensorflow.org/guide/distributed_training#TF_CONFIG) +appropriately at each host. e.g., to run using `MultiWorkerMirroredStrategy` on +2 hosts, the `cluster` in `TF_CONFIG` should have 2 `host:port` entries, and +host `i` should have the `task` in `TF_CONFIG` set to `{"type": "worker", +"index": i}`. `MultiWorkerMirroredStrategy` will automatically use all the +available GPUs at each host. + +### Running on Cloud TPUs + +Note: This model will **not** work with TPUs on Colab. + +You can train the ResNet CTL model on Cloud TPUs using +`tf.distribute.TPUStrategy`. If you are not familiar with Cloud TPUs, it is +strongly recommended that you go through the +[quickstart](https://cloud.google.com/tpu/docs/quickstart) to learn how to +create a TPU and GCE VM. + +To run ResNet model on a TPU, you must set `--distribution_strategy=tpu` and +`--tpu=$TPU_NAME`, where `$TPU_NAME` the name of your TPU in the Cloud Console. +From a GCE VM, you can run the following command to train ResNet for one epoch +on a v2-8 or v3-8 TPU by setting `TRAIN_EPOCHS` to 1: + +```bash +python3 resnet_ctl_imagenet_main.py \ + --tpu=$TPU_NAME \ + --model_dir=$MODEL_DIR \ + --data_dir=$DATA_DIR \ + --batch_size=1024 \ + --steps_per_loop=500 \ + --train_epochs=$TRAIN_EPOCHS \ + --use_synthetic_data=false \ + --dtype=fp32 \ + --enable_eager=true \ + --enable_tensorboard=true \ + --distribution_strategy=tpu \ + --log_steps=50 \ + --single_l2_loss_op=true \ + --use_tf_function=true +``` + +To train the ResNet to convergence, run it for 90 epochs by setting +`TRAIN_EPOCHS` to 90. + +Note: `$MODEL_DIR` and `$DATA_DIR` must be GCS paths. diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/cifar_preprocessing.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/cifar_preprocessing.py new file mode 100644 index 0000000..18d7fe6 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/cifar_preprocessing.py @@ -0,0 +1,159 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Provides utilities to Cifar-10 dataset.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +from absl import logging +import tensorflow as tf + +from official.vision.image_classification.resnet import imagenet_preprocessing + +HEIGHT = 32 +WIDTH = 32 +NUM_CHANNELS = 3 +_DEFAULT_IMAGE_BYTES = HEIGHT * WIDTH * NUM_CHANNELS +# The record is the image plus a one-byte label +_RECORD_BYTES = _DEFAULT_IMAGE_BYTES + 1 + +# TODO(tobyboyd): Change to best practice 45K(train)/5K(val)/10K(test) splits. +NUM_IMAGES = { + 'train': 50000, + 'validation': 10000, +} +_NUM_DATA_FILES = 5 +NUM_CLASSES = 10 + + +def parse_record(raw_record, is_training, dtype): + """Parses a record containing a training example of an image. + + The input record is parsed into a label and image, and the image is passed + through preprocessing steps (cropping, flipping, and so on). + + This method converts the label to one hot to fit the loss function. + + Args: + raw_record: scalar Tensor tf.string containing a serialized + Example protocol buffer. + is_training: A boolean denoting whether the input is for training. + dtype: Data type to use for input images. + + Returns: + Tuple with processed image tensor and one-hot-encoded label tensor. + """ + # Convert bytes to a vector of uint8 that is record_bytes long. + record_vector = tf.io.decode_raw(raw_record, tf.uint8) + + # The first byte represents the label, which we convert from uint8 to int32 + # and then to one-hot. + label = tf.cast(record_vector[0], tf.int32) + + # The remaining bytes after the label represent the image, which we reshape + # from [depth * height * width] to [depth, height, width]. + depth_major = tf.reshape(record_vector[1:_RECORD_BYTES], + [NUM_CHANNELS, HEIGHT, WIDTH]) + + # Convert from [depth, height, width] to [height, width, depth], and cast as + # float32. + image = tf.cast(tf.transpose(a=depth_major, perm=[1, 2, 0]), tf.float32) + + image = preprocess_image(image, is_training) + image = tf.cast(image, dtype) + + return image, label + + +def preprocess_image(image, is_training): + """Preprocess a single image of layout [height, width, depth].""" + if is_training: + # Resize the image to add four extra pixels on each side. + image = tf.image.resize_with_crop_or_pad( + image, HEIGHT + 8, WIDTH + 8) + + # Randomly crop a [HEIGHT, WIDTH] section of the image. + image = tf.image.random_crop(image, [HEIGHT, WIDTH, NUM_CHANNELS]) + + # Randomly flip the image horizontally. + image = tf.image.random_flip_left_right(image) + + # Subtract off the mean and divide by the variance of the pixels. + image = tf.image.per_image_standardization(image) + return image + + +def get_filenames(is_training, data_dir): + """Returns a list of filenames.""" + assert tf.io.gfile.exists(data_dir), ( + 'Run cifar10_download_and_extract.py first to download and extract the ' + 'CIFAR-10 data.') + + if is_training: + return [ + os.path.join(data_dir, 'data_batch_%d.bin' % i) + for i in range(1, _NUM_DATA_FILES + 1) + ] + else: + return [os.path.join(data_dir, 'test_batch.bin')] + + +def input_fn(is_training, + data_dir, + batch_size, + dtype=tf.float32, + datasets_num_private_threads=None, + parse_record_fn=parse_record, + input_context=None, + drop_remainder=False): + """Input function which provides batches for train or eval. + + Args: + is_training: A boolean denoting whether the input is for training. + data_dir: The directory containing the input data. + batch_size: The number of samples per batch. + dtype: Data type to use for images/features + datasets_num_private_threads: Number of private threads for tf.data. + parse_record_fn: Function to use for parsing the records. + input_context: A `tf.distribute.InputContext` object passed in by + `tf.distribute.Strategy`. + drop_remainder: A boolean indicates whether to drop the remainder of the + batches. If True, the batch dimension will be static. + + Returns: + A dataset that can be used for iteration. + """ + filenames = get_filenames(is_training, data_dir) + dataset = tf.data.FixedLengthRecordDataset(filenames, _RECORD_BYTES) + + if input_context: + logging.info( + 'Sharding the dataset: input_pipeline_id=%d num_input_pipelines=%d', + input_context.input_pipeline_id, input_context.num_input_pipelines) + dataset = dataset.shard(input_context.num_input_pipelines, + input_context.input_pipeline_id) + + return imagenet_preprocessing.process_record_dataset( + dataset=dataset, + is_training=is_training, + batch_size=batch_size, + shuffle_buffer=NUM_IMAGES['train'], + parse_record_fn=parse_record_fn, + dtype=dtype, + datasets_num_private_threads=datasets_num_private_threads, + drop_remainder=drop_remainder + ) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/common.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/common.py new file mode 100644 index 0000000..5aae778 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/common.py @@ -0,0 +1,398 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Common util functions and classes used by both keras cifar and imagenet.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from absl import flags +import tensorflow as tf + +from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_v2 +import tensorflow_model_optimization as tfmot +from official.utils.flags import core as flags_core +from official.utils.misc import keras_utils + +FLAGS = flags.FLAGS +BASE_LEARNING_RATE = 0.1 # This matches Jing's version. +TRAIN_TOP_1 = 'training_accuracy_top_1' +LR_SCHEDULE = [ # (multiplier, epoch to start) tuples + (1.0, 5), (0.1, 30), (0.01, 60), (0.001, 80) +] + + +class PiecewiseConstantDecayWithWarmup( + tf.keras.optimizers.schedules.LearningRateSchedule): + """Piecewise constant decay with warmup schedule.""" + + def __init__(self, batch_size, epoch_size, warmup_epochs, boundaries, + multipliers, compute_lr_on_cpu=True, name=None): + super(PiecewiseConstantDecayWithWarmup, self).__init__() + if len(boundaries) != len(multipliers) - 1: + raise ValueError('The length of boundaries must be 1 less than the ' + 'length of multipliers') + + base_lr_batch_size = 256 + steps_per_epoch = epoch_size // batch_size + + self.rescaled_lr = BASE_LEARNING_RATE * batch_size / base_lr_batch_size + self.step_boundaries = [float(steps_per_epoch) * x for x in boundaries] + self.lr_values = [self.rescaled_lr * m for m in multipliers] + self.warmup_steps = warmup_epochs * steps_per_epoch + self.compute_lr_on_cpu = compute_lr_on_cpu + self.name = name + + self.learning_rate_ops_cache = {} + + def __call__(self, step): + if tf.executing_eagerly(): + return self._get_learning_rate(step) + + # In an eager function or graph, the current implementation of optimizer + # repeatedly call and thus create ops for the learning rate schedule. To + # avoid this, we cache the ops if not executing eagerly. + graph = tf.compat.v1.get_default_graph() + if graph not in self.learning_rate_ops_cache: + if self.compute_lr_on_cpu: + with tf.device('/device:CPU:0'): + self.learning_rate_ops_cache[graph] = self._get_learning_rate(step) + else: + self.learning_rate_ops_cache[graph] = self._get_learning_rate(step) + return self.learning_rate_ops_cache[graph] + + def _get_learning_rate(self, step): + """Compute learning rate at given step.""" + with tf.compat.v1.name_scope(self.name, 'PiecewiseConstantDecayWithWarmup', + [self.rescaled_lr, self.step_boundaries, + self.lr_values, self.warmup_steps, + self.compute_lr_on_cpu]): + def warmup_lr(step): + return self.rescaled_lr * ( + tf.cast(step, tf.float32) / tf.cast(self.warmup_steps, tf.float32)) + def piecewise_lr(step): + return tf.compat.v1.train.piecewise_constant( + step, self.step_boundaries, self.lr_values) + return tf.cond(step < self.warmup_steps, + lambda: warmup_lr(step), + lambda: piecewise_lr(step)) + + def get_config(self): + return { + 'rescaled_lr': self.rescaled_lr, + 'step_boundaries': self.step_boundaries, + 'lr_values': self.lr_values, + 'warmup_steps': self.warmup_steps, + 'compute_lr_on_cpu': self.compute_lr_on_cpu, + 'name': self.name + } + + +def get_optimizer(learning_rate=0.1): + """Returns optimizer to use.""" + # The learning_rate is overwritten at the beginning of each step by callback. + return gradient_descent_v2.SGD(learning_rate=learning_rate, momentum=0.9) + + +def get_callbacks( + steps_per_epoch, + pruning_method=None, + enable_checkpoint_and_export=False, + model_dir=None): + """Returns common callbacks.""" + time_callback = keras_utils.TimeHistory( + FLAGS.batch_size, + FLAGS.log_steps, + logdir=FLAGS.model_dir if FLAGS.enable_tensorboard else None) + callbacks = [time_callback] + + if FLAGS.enable_tensorboard: + tensorboard_callback = tf.keras.callbacks.TensorBoard( + log_dir=FLAGS.model_dir) + callbacks.append(tensorboard_callback) + + if FLAGS.profile_steps: + profiler_callback = keras_utils.get_profiler_callback( + FLAGS.model_dir, + FLAGS.profile_steps, + FLAGS.enable_tensorboard, + steps_per_epoch) + callbacks.append(profiler_callback) + + is_pruning_enabled = pruning_method is not None + if is_pruning_enabled: + callbacks.append(tfmot.sparsity.keras.UpdatePruningStep()) + if model_dir is not None: + callbacks.append(tfmot.sparsity.keras.PruningSummaries( + log_dir=model_dir, profile_batch=0)) + + if enable_checkpoint_and_export: + if model_dir is not None: + ckpt_full_path = os.path.join(model_dir, 'model.ckpt-{epoch:04d}') + callbacks.append( + tf.keras.callbacks.ModelCheckpoint(ckpt_full_path, + save_weights_only=True)) + return callbacks + + +def build_stats(history, eval_output, callbacks): + """Normalizes and returns dictionary of stats. + + Args: + history: Results of the training step. Supports both categorical_accuracy + and sparse_categorical_accuracy. + eval_output: Output of the eval step. Assumes first value is eval_loss and + second value is accuracy_top_1. + callbacks: a list of callbacks which might include a time history callback + used during keras.fit. + + Returns: + Dictionary of normalized results. + """ + stats = {} + if eval_output: + stats['accuracy_top_1'] = float(eval_output[1]) + stats['eval_loss'] = float(eval_output[0]) + if history and history.history: + train_hist = history.history + # Gets final loss from training. + stats['loss'] = float(train_hist['loss'][-1]) + # Gets top_1 training accuracy. + if 'categorical_accuracy' in train_hist: + stats[TRAIN_TOP_1] = float(train_hist['categorical_accuracy'][-1]) + elif 'sparse_categorical_accuracy' in train_hist: + stats[TRAIN_TOP_1] = float(train_hist['sparse_categorical_accuracy'][-1]) + elif 'accuracy' in train_hist: + stats[TRAIN_TOP_1] = float(train_hist['accuracy'][-1]) + + if not callbacks: + return stats + + # Look for the time history callback which was used during keras.fit + for callback in callbacks: + if isinstance(callback, keras_utils.TimeHistory): + timestamp_log = callback.timestamp_log + stats['step_timestamp_log'] = timestamp_log + stats['train_finish_time'] = callback.train_finish_time + if callback.epoch_runtime_log: + stats['avg_exp_per_second'] = callback.average_examples_per_second + + return stats + + +def define_keras_flags( + dynamic_loss_scale=True, + model=False, + optimizer=False, + pretrained_filepath=False): + """Define flags for Keras models.""" + flags_core.define_base(clean=True, num_gpu=True, run_eagerly=True, + train_epochs=True, epochs_between_evals=True, + distribution_strategy=True) + flags_core.define_performance(num_parallel_calls=False, + synthetic_data=True, + dtype=True, + all_reduce_alg=True, + num_packs=True, + tf_gpu_thread_mode=True, + datasets_num_private_threads=True, + dynamic_loss_scale=dynamic_loss_scale, + loss_scale=True, + fp16_implementation=True, + tf_data_experimental_slack=True, + enable_xla=True, + training_dataset_cache=True) + flags_core.define_image() + flags_core.define_benchmark() + flags_core.define_distribution() + flags.adopt_module_key_flags(flags_core) + + flags.DEFINE_boolean(name='enable_eager', default=False, help='Enable eager?') + flags.DEFINE_boolean(name='skip_eval', default=False, help='Skip evaluation?') + # TODO(b/135607288): Remove this flag once we understand the root cause of + # slowdown when setting the learning phase in Keras backend. + flags.DEFINE_boolean( + name='set_learning_phase_to_train', default=True, + help='If skip eval, also set Keras learning phase to 1 (training).') + flags.DEFINE_boolean( + name='explicit_gpu_placement', default=False, + help='If not using distribution strategy, explicitly set device scope ' + 'for the Keras training loop.') + flags.DEFINE_boolean(name='use_trivial_model', default=False, + help='Whether to use a trivial Keras model.') + flags.DEFINE_boolean(name='report_accuracy_metrics', default=True, + help='Report metrics during training and evaluation.') + flags.DEFINE_boolean(name='use_tensor_lr', default=True, + help='Use learning rate tensor instead of a callback.') + flags.DEFINE_boolean( + name='enable_tensorboard', default=False, + help='Whether to enable Tensorboard callback.') + flags.DEFINE_integer( + name='train_steps', default=None, + help='The number of steps to run for training. If it is larger than ' + '# batches per epoch, then use # batches per epoch. This flag will be ' + 'ignored if train_epochs is set to be larger than 1. ') + flags.DEFINE_string( + name='profile_steps', default=None, + help='Save profiling data to model dir at given range of global steps. The ' + 'value must be a comma separated pair of positive integers, specifying ' + 'the first and last step to profile. For example, "--profile_steps=2,4" ' + 'triggers the profiler to process 3 steps, starting from the 2nd step. ' + 'Note that profiler has a non-trivial performance overhead, and the ' + 'output file can be gigantic if profiling many steps.') + flags.DEFINE_boolean( + name='batchnorm_spatial_persistent', default=True, + help='Enable the spacial persistent mode for CuDNN batch norm kernel.') + flags.DEFINE_boolean( + name='enable_get_next_as_optional', default=False, + help='Enable get_next_as_optional behavior in DistributedIterator.') + flags.DEFINE_boolean( + name='enable_checkpoint_and_export', default=False, + help='Whether to enable a checkpoint callback and export the savedmodel.') + flags.DEFINE_string( + name='tpu', default='', help='TPU address to connect to.') + flags.DEFINE_integer( + name='steps_per_loop', + default=500, + help='Number of steps per training loop. Only training step happens ' + 'inside the loop. Callbacks will not be called inside. Will be capped at ' + 'steps per epoch.') + flags.DEFINE_boolean( + name='use_tf_while_loop', + default=True, + help='Whether to build a tf.while_loop inside the training loop on the ' + 'host. Setting it to True is critical to have peak performance on ' + 'TPU.') + + if model: + flags.DEFINE_string('model', 'resnet50_v1.5', + 'Name of model preset. (mobilenet, resnet50_v1.5)') + if optimizer: + flags.DEFINE_string('optimizer', 'resnet50_default', + 'Name of optimizer preset. ' + '(mobilenet_default, resnet50_default)') + # TODO(kimjaehong): Replace as general hyper-params not only for mobilenet. + flags.DEFINE_float('initial_learning_rate_per_sample', 0.00007, + 'Initial value of learning rate per sample for ' + 'mobilenet_default.') + flags.DEFINE_float('lr_decay_factor', 0.94, + 'Learning rate decay factor for mobilenet_default.') + flags.DEFINE_float('num_epochs_per_decay', 2.5, + 'Number of epochs per decay for mobilenet_default.') + if pretrained_filepath: + flags.DEFINE_string('pretrained_filepath', '', + 'Pretrained file path.') + + +def get_synth_data(height, width, num_channels, num_classes, dtype): + """Creates a set of synthetic random data. + + Args: + height: Integer height that will be used to create a fake image tensor. + width: Integer width that will be used to create a fake image tensor. + num_channels: Integer depth that will be used to create a fake image tensor. + num_classes: Number of classes that should be represented in the fake labels + tensor + dtype: Data type for features/images. + + Returns: + A tuple of tensors representing the inputs and labels. + + """ + # Synthetic input should be within [0, 255]. + inputs = tf.random.truncated_normal([height, width, num_channels], + dtype=dtype, + mean=127, + stddev=60, + name='synthetic_inputs') + labels = tf.random.uniform([1], + minval=0, + maxval=num_classes - 1, + dtype=tf.int32, + name='synthetic_labels') + return inputs, labels + + +def define_pruning_flags(): + """Define flags for pruning methods.""" + flags.DEFINE_string('pruning_method', None, + 'Pruning method.' + 'None (no pruning) or polynomial_decay.') + flags.DEFINE_float('pruning_initial_sparsity', 0.0, + 'Initial sparsity for pruning.') + flags.DEFINE_float('pruning_final_sparsity', 0.5, + 'Final sparsity for pruning.') + flags.DEFINE_integer('pruning_begin_step', 0, + 'Begin step for pruning.') + flags.DEFINE_integer('pruning_end_step', 100000, + 'End step for pruning.') + flags.DEFINE_integer('pruning_frequency', 100, + 'Frequency for pruning.') + + +def get_synth_input_fn(height, width, num_channels, num_classes, + dtype=tf.float32, drop_remainder=True): + """Returns an input function that returns a dataset with random data. + + This input_fn returns a data set that iterates over a set of random data and + bypasses all preprocessing, e.g. jpeg decode and copy. The host to device + copy is still included. This used to find the upper throughput bound when + tuning the full input pipeline. + + Args: + height: Integer height that will be used to create a fake image tensor. + width: Integer width that will be used to create a fake image tensor. + num_channels: Integer depth that will be used to create a fake image tensor. + num_classes: Number of classes that should be represented in the fake labels + tensor + dtype: Data type for features/images. + drop_remainder: A boolean indicates whether to drop the remainder of the + batches. If True, the batch dimension will be static. + + Returns: + An input_fn that can be used in place of a real one to return a dataset + that can be used for iteration. + """ + # pylint: disable=unused-argument + def input_fn(is_training, data_dir, batch_size, *args, **kwargs): + """Returns dataset filled with random data.""" + inputs, labels = get_synth_data(height=height, + width=width, + num_channels=num_channels, + num_classes=num_classes, + dtype=dtype) + # Cast to float32 for Keras model. + labels = tf.cast(labels, dtype=tf.float32) + data = tf.data.Dataset.from_tensors((inputs, labels)).repeat() + + # `drop_remainder` will make dataset produce outputs with known shapes. + data = data.batch(batch_size, drop_remainder=drop_remainder) + data = data.prefetch(buffer_size=tf.data.experimental.AUTOTUNE) + return data + + return input_fn + + +def set_cudnn_batchnorm_mode(): + """Set CuDNN batchnorm mode for better performance. + + Note: Spatial Persistent mode may lead to accuracy losses for certain + models. + """ + if FLAGS.batchnorm_spatial_persistent: + os.environ['TF_USE_CUDNN_BATCHNORM_SPATIAL_PERSISTENT'] = '1' + else: + os.environ.pop('TF_USE_CUDNN_BATCHNORM_SPATIAL_PERSISTENT', None) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/imagenet_preprocessing.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/imagenet_preprocessing.py new file mode 100644 index 0000000..f1490c2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/imagenet_preprocessing.py @@ -0,0 +1,561 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Provides utilities to preprocess images. + +Training images are sampled using the provided bounding boxes, and subsequently +cropped to the sampled bounding box. Images are additionally flipped randomly, +then resized to the target output size (without aspect-ratio preservation). + +Images used during evaluation are resized (with aspect-ratio preservation) and +centrally cropped. + +All images undergo mean color subtraction. + +Note that these steps are colloquially referred to as "ResNet preprocessing," +and they differ from "VGG preprocessing," which does not use bounding boxes +and instead does an aspect-preserving resize followed by random crop during +training. (These both differ from "Inception preprocessing," which introduces +color distortion steps.) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +from absl import logging +import tensorflow as tf + +DEFAULT_IMAGE_SIZE = 224 +NUM_CHANNELS = 3 +NUM_CLASSES = 1001 + +NUM_IMAGES = { + 'train': 1281167, + 'validation': 50000, +} + +_NUM_TRAIN_FILES = 1024 +_SHUFFLE_BUFFER = 10000 + +_R_MEAN = 123.68 +_G_MEAN = 116.78 +_B_MEAN = 103.94 +CHANNEL_MEANS = [_R_MEAN, _G_MEAN, _B_MEAN] + +# The lower bound for the smallest side of the image for aspect-preserving +# resizing. For example, if an image is 500 x 1000, it will be resized to +# _RESIZE_MIN x (_RESIZE_MIN * 2). +_RESIZE_MIN = 256 + + +def process_record_dataset(dataset, + is_training, + batch_size, + shuffle_buffer, + parse_record_fn, + dtype=tf.float32, + datasets_num_private_threads=None, + drop_remainder=False, + tf_data_experimental_slack=False): + """Given a Dataset with raw records, return an iterator over the records. + + Args: + dataset: A Dataset representing raw records + is_training: A boolean denoting whether the input is for training. + batch_size: The number of samples per batch. + shuffle_buffer: The buffer size to use when shuffling records. A larger + value results in better randomness, but smaller values reduce startup + time and use less memory. + parse_record_fn: A function that takes a raw record and returns the + corresponding (image, label) pair. + dtype: Data type to use for images/features. + datasets_num_private_threads: Number of threads for a private + threadpool created for all datasets computation. + drop_remainder: A boolean indicates whether to drop the remainder of the + batches. If True, the batch dimension will be static. + tf_data_experimental_slack: Whether to enable tf.data's + `experimental_slack` option. + + Returns: + Dataset of (image, label) pairs ready for iteration. + """ + # Defines a specific size thread pool for tf.data operations. + if datasets_num_private_threads: + options = tf.data.Options() + options.experimental_threading.private_threadpool_size = ( + datasets_num_private_threads) + dataset = dataset.with_options(options) + logging.info( + 'datasets_num_private_threads: %s', datasets_num_private_threads) + + if is_training: + # Shuffles records before repeating to respect epoch boundaries. + dataset = dataset.shuffle(buffer_size=shuffle_buffer) + # Repeats the dataset for the number of epochs to train. + dataset = dataset.repeat() + + # Parses the raw records into images and labels. + dataset = dataset.map( + lambda value: parse_record_fn(value, is_training, dtype), + num_parallel_calls=tf.data.experimental.AUTOTUNE) + dataset = dataset.batch(batch_size, drop_remainder=drop_remainder) + + # Operations between the final prefetch and the get_next call to the iterator + # will happen synchronously during run time. We prefetch here again to + # background all of the above processing work and keep it out of the + # critical training path. Setting buffer_size to tf.data.experimental.AUTOTUNE + # allows DistributionStrategies to adjust how many batches to fetch based + # on how many devices are present. + dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE) + + options = tf.data.Options() + options.experimental_slack = tf_data_experimental_slack + dataset = dataset.with_options(options) + + return dataset + + +def get_filenames(is_training, data_dir): + """Return filenames for dataset.""" + if is_training: + return [ + os.path.join(data_dir, 'train-%05d-of-01024' % i) + for i in range(_NUM_TRAIN_FILES)] + else: + return [ + os.path.join(data_dir, 'validation-%05d-of-00128' % i) + for i in range(128)] + + +def parse_example_proto(example_serialized): + """Parses an Example proto containing a training example of an image. + + The output of the build_image_data.py image preprocessing script is a dataset + containing serialized Example protocol buffers. Each Example proto contains + the following fields (values are included as examples): + + image/height: 462 + image/width: 581 + image/colorspace: 'RGB' + image/channels: 3 + image/class/label: 615 + image/class/synset: 'n03623198' + image/class/text: 'knee pad' + image/object/bbox/xmin: 0.1 + image/object/bbox/xmax: 0.9 + image/object/bbox/ymin: 0.2 + image/object/bbox/ymax: 0.6 + image/object/bbox/label: 615 + image/format: 'JPEG' + image/filename: 'ILSVRC2012_val_00041207.JPEG' + image/encoded: + + Args: + example_serialized: scalar Tensor tf.string containing a serialized + Example protocol buffer. + + Returns: + image_buffer: Tensor tf.string containing the contents of a JPEG file. + label: Tensor tf.int32 containing the label. + bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords] + where each coordinate is [0, 1) and the coordinates are arranged as + [ymin, xmin, ymax, xmax]. + """ + # Dense features in Example proto. + feature_map = { + 'image/encoded': tf.io.FixedLenFeature([], dtype=tf.string, + default_value=''), + 'image/class/label': tf.io.FixedLenFeature([], dtype=tf.int64, + default_value=-1), + 'image/class/text': tf.io.FixedLenFeature([], dtype=tf.string, + default_value=''), + } + sparse_float32 = tf.io.VarLenFeature(dtype=tf.float32) + # Sparse features in Example proto. + feature_map.update( + {k: sparse_float32 for k in [ + 'image/object/bbox/xmin', 'image/object/bbox/ymin', + 'image/object/bbox/xmax', 'image/object/bbox/ymax']}) + + features = tf.io.parse_single_example(serialized=example_serialized, + features=feature_map) + label = tf.cast(features['image/class/label'], dtype=tf.int32) + + xmin = tf.expand_dims(features['image/object/bbox/xmin'].values, 0) + ymin = tf.expand_dims(features['image/object/bbox/ymin'].values, 0) + xmax = tf.expand_dims(features['image/object/bbox/xmax'].values, 0) + ymax = tf.expand_dims(features['image/object/bbox/ymax'].values, 0) + + # Note that we impose an ordering of (y, x) just to make life difficult. + bbox = tf.concat([ymin, xmin, ymax, xmax], 0) + + # Force the variable number of bounding boxes into the shape + # [1, num_boxes, coords]. + bbox = tf.expand_dims(bbox, 0) + bbox = tf.transpose(a=bbox, perm=[0, 2, 1]) + + return features['image/encoded'], label, bbox + + +def parse_record(raw_record, is_training, dtype): + """Parses a record containing a training example of an image. + + The input record is parsed into a label and image, and the image is passed + through preprocessing steps (cropping, flipping, and so on). + + Args: + raw_record: scalar Tensor tf.string containing a serialized + Example protocol buffer. + is_training: A boolean denoting whether the input is for training. + dtype: data type to use for images/features. + + Returns: + Tuple with processed image tensor in a channel-last format and + one-hot-encoded label tensor. + """ + image_buffer, label, bbox = parse_example_proto(raw_record) + + image = preprocess_image( + image_buffer=image_buffer, + bbox=bbox, + output_height=DEFAULT_IMAGE_SIZE, + output_width=DEFAULT_IMAGE_SIZE, + num_channels=NUM_CHANNELS, + is_training=is_training) + image = tf.cast(image, dtype) + + # Subtract one so that labels are in [0, 1000), and cast to float32 for + # Keras model. + label = tf.cast(tf.cast(tf.reshape(label, shape=[1]), dtype=tf.int32) - 1, + dtype=tf.float32) + return image, label + + +def get_parse_record_fn(use_keras_image_data_format=False): + """Get a function for parsing the records, accounting for image format. + + This is useful by handling different types of Keras models. For instance, + the current resnet_model.resnet50 input format is always channel-last, + whereas the keras_applications mobilenet input format depends on + tf.keras.backend.image_data_format(). We should set + use_keras_image_data_format=False for the former and True for the latter. + + Args: + use_keras_image_data_format: A boolean denoting whether data format is keras + backend image data format. If False, the image format is channel-last. If + True, the image format matches tf.keras.backend.image_data_format(). + + Returns: + Function to use for parsing the records. + """ + def parse_record_fn(raw_record, is_training, dtype): + image, label = parse_record(raw_record, is_training, dtype) + if use_keras_image_data_format: + if tf.keras.backend.image_data_format() == 'channels_first': + image = tf.transpose(image, perm=[2, 0, 1]) + return image, label + return parse_record_fn + + +def input_fn(is_training, + data_dir, + batch_size, + dtype=tf.float32, + datasets_num_private_threads=None, + parse_record_fn=parse_record, + input_context=None, + drop_remainder=False, + tf_data_experimental_slack=False, + training_dataset_cache=False, + filenames=None): + """Input function which provides batches for train or eval. + + Args: + is_training: A boolean denoting whether the input is for training. + data_dir: The directory containing the input data. + batch_size: The number of samples per batch. + dtype: Data type to use for images/features + datasets_num_private_threads: Number of private threads for tf.data. + parse_record_fn: Function to use for parsing the records. + input_context: A `tf.distribute.InputContext` object passed in by + `tf.distribute.Strategy`. + drop_remainder: A boolean indicates whether to drop the remainder of the + batches. If True, the batch dimension will be static. + tf_data_experimental_slack: Whether to enable tf.data's + `experimental_slack` option. + training_dataset_cache: Whether to cache the training dataset on workers. + Typically used to improve training performance when training data is in + remote storage and can fit into worker memory. + filenames: Optional field for providing the file names of the TFRecords. + + Returns: + A dataset that can be used for iteration. + """ + if filenames is None: + filenames = get_filenames(is_training, data_dir) + dataset = tf.data.Dataset.from_tensor_slices(filenames) + + if input_context: + logging.info( + 'Sharding the dataset: input_pipeline_id=%d num_input_pipelines=%d', + input_context.input_pipeline_id, input_context.num_input_pipelines) + dataset = dataset.shard(input_context.num_input_pipelines, + input_context.input_pipeline_id) + + if is_training: + # Shuffle the input files + dataset = dataset.shuffle(buffer_size=_NUM_TRAIN_FILES) + + # Convert to individual records. + # cycle_length = 10 means that up to 10 files will be read and deserialized in + # parallel. You may want to increase this number if you have a large number of + # CPU cores. + dataset = dataset.interleave( + tf.data.TFRecordDataset, + cycle_length=10, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + + if is_training and training_dataset_cache: + # Improve training performance when training data is in remote storage and + # can fit into worker memory. + dataset = dataset.cache() + + return process_record_dataset( + dataset=dataset, + is_training=is_training, + batch_size=batch_size, + shuffle_buffer=_SHUFFLE_BUFFER, + parse_record_fn=parse_record_fn, + dtype=dtype, + datasets_num_private_threads=datasets_num_private_threads, + drop_remainder=drop_remainder, + tf_data_experimental_slack=tf_data_experimental_slack, + ) + + +def _decode_crop_and_flip(image_buffer, bbox, num_channels): + """Crops the given image to a random part of the image, and randomly flips. + + We use the fused decode_and_crop op, which performs better than the two ops + used separately in series, but note that this requires that the image be + passed in as an un-decoded string Tensor. + + Args: + image_buffer: scalar string Tensor representing the raw JPEG image buffer. + bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords] + where each coordinate is [0, 1) and the coordinates are arranged as + [ymin, xmin, ymax, xmax]. + num_channels: Integer depth of the image buffer for decoding. + + Returns: + 3-D tensor with cropped image. + + """ + # A large fraction of image datasets contain a human-annotated bounding box + # delineating the region of the image containing the object of interest. We + # choose to create a new bounding box for the object which is a randomly + # distorted version of the human-annotated bounding box that obeys an + # allowed range of aspect ratios, sizes and overlap with the human-annotated + # bounding box. If no box is supplied, then we assume the bounding box is + # the entire image. + sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box( + tf.image.extract_jpeg_shape(image_buffer), + bounding_boxes=bbox, + min_object_covered=0.1, + aspect_ratio_range=[0.75, 1.33], + area_range=[0.05, 1.0], + max_attempts=100, + use_image_if_no_bounding_boxes=True) + bbox_begin, bbox_size, _ = sample_distorted_bounding_box + + # Reassemble the bounding box in the format the crop op requires. + offset_y, offset_x, _ = tf.unstack(bbox_begin) + target_height, target_width, _ = tf.unstack(bbox_size) + crop_window = tf.stack([offset_y, offset_x, target_height, target_width]) + + # Use the fused decode and crop op here, which is faster than each in series. + cropped = tf.image.decode_and_crop_jpeg( + image_buffer, crop_window, channels=num_channels) + + # Flip to add a little more random distortion in. + cropped = tf.image.random_flip_left_right(cropped) + return cropped + + +def _central_crop(image, crop_height, crop_width): + """Performs central crops of the given image list. + + Args: + image: a 3-D image tensor + crop_height: the height of the image following the crop. + crop_width: the width of the image following the crop. + + Returns: + 3-D tensor with cropped image. + """ + shape = tf.shape(input=image) + height, width = shape[0], shape[1] + + amount_to_be_cropped_h = (height - crop_height) + crop_top = amount_to_be_cropped_h // 2 + amount_to_be_cropped_w = (width - crop_width) + crop_left = amount_to_be_cropped_w // 2 + return tf.slice( + image, [crop_top, crop_left, 0], [crop_height, crop_width, -1]) + + +def _mean_image_subtraction(image, means, num_channels): + """Subtracts the given means from each image channel. + + For example: + means = [123.68, 116.779, 103.939] + image = _mean_image_subtraction(image, means) + + Note that the rank of `image` must be known. + + Args: + image: a tensor of size [height, width, C]. + means: a C-vector of values to subtract from each channel. + num_channels: number of color channels in the image that will be distorted. + + Returns: + the centered image. + + Raises: + ValueError: If the rank of `image` is unknown, if `image` has a rank other + than three or if the number of channels in `image` doesn't match the + number of values in `means`. + """ + if image.get_shape().ndims != 3: + raise ValueError('Input must be of size [height, width, C>0]') + + if len(means) != num_channels: + raise ValueError('len(means) must match the number of channels') + + # We have a 1-D tensor of means; convert to 3-D. + # Note(b/130245863): we explicitly call `broadcast` instead of simply + # expanding dimensions for better performance. + means = tf.broadcast_to(means, tf.shape(image)) + + return image - means + + +def _smallest_size_at_least(height, width, resize_min): + """Computes new shape with the smallest side equal to `smallest_side`. + + Computes new shape with the smallest side equal to `smallest_side` while + preserving the original aspect ratio. + + Args: + height: an int32 scalar tensor indicating the current height. + width: an int32 scalar tensor indicating the current width. + resize_min: A python integer or scalar `Tensor` indicating the size of + the smallest side after resize. + + Returns: + new_height: an int32 scalar tensor indicating the new height. + new_width: an int32 scalar tensor indicating the new width. + """ + resize_min = tf.cast(resize_min, tf.float32) + + # Convert to floats to make subsequent calculations go smoothly. + height, width = tf.cast(height, tf.float32), tf.cast(width, tf.float32) + + smaller_dim = tf.minimum(height, width) + scale_ratio = resize_min / smaller_dim + + # Convert back to ints to make heights and widths that TF ops will accept. + new_height = tf.cast(height * scale_ratio, tf.int32) + new_width = tf.cast(width * scale_ratio, tf.int32) + + return new_height, new_width + + +def _aspect_preserving_resize(image, resize_min): + """Resize images preserving the original aspect ratio. + + Args: + image: A 3-D image `Tensor`. + resize_min: A python integer or scalar `Tensor` indicating the size of + the smallest side after resize. + + Returns: + resized_image: A 3-D tensor containing the resized image. + """ + shape = tf.shape(input=image) + height, width = shape[0], shape[1] + + new_height, new_width = _smallest_size_at_least(height, width, resize_min) + + return _resize_image(image, new_height, new_width) + + +def _resize_image(image, height, width): + """Simple wrapper around tf.resize_images. + + This is primarily to make sure we use the same `ResizeMethod` and other + details each time. + + Args: + image: A 3-D image `Tensor`. + height: The target height for the resized image. + width: The target width for the resized image. + + Returns: + resized_image: A 3-D tensor containing the resized image. The first two + dimensions have the shape [height, width]. + """ + return tf.compat.v1.image.resize( + image, [height, width], method=tf.image.ResizeMethod.BILINEAR, + align_corners=False) + + +def preprocess_image(image_buffer, bbox, output_height, output_width, + num_channels, is_training=False): + """Preprocesses the given image. + + Preprocessing includes decoding, cropping, and resizing for both training + and eval images. Training preprocessing, however, introduces some random + distortion of the image to improve accuracy. + + Args: + image_buffer: scalar string Tensor representing the raw JPEG image buffer. + bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords] + where each coordinate is [0, 1) and the coordinates are arranged as + [ymin, xmin, ymax, xmax]. + output_height: The height of the image after preprocessing. + output_width: The width of the image after preprocessing. + num_channels: Integer depth of the image buffer for decoding. + is_training: `True` if we're preprocessing the image for training and + `False` otherwise. + + Returns: + A preprocessed image. + """ + if is_training: + # For training, we want to randomize some of the distortions. + image = _decode_crop_and_flip(image_buffer, bbox, num_channels) + image = _resize_image(image, output_height, output_width) + else: + # For validation, we want to decode, resize, then just crop the middle. + image = tf.image.decode_jpeg(image_buffer, channels=num_channels) + image = _aspect_preserving_resize(image, _RESIZE_MIN) + image = _central_crop(image, output_height, output_width) + + image.set_shape([output_height, output_width, num_channels]) + + return _mean_image_subtraction(image, CHANNEL_MEANS, num_channels) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_config.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_config.py new file mode 100644 index 0000000..d94e65a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_config.py @@ -0,0 +1,61 @@ +# Lint as: python3 +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Configuration definitions for ResNet losses, learning rates, and optimizers.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from typing import Any, Mapping + +import dataclasses + +from official.vision.image_classification.configs import base_configs + + +_RESNET_LR_SCHEDULE = [ # (multiplier, epoch to start) tuples + (1.0, 5), (0.1, 30), (0.01, 60), (0.001, 80) +] +_RESNET_LR_BOUNDARIES = list(p[1] for p in _RESNET_LR_SCHEDULE[1:]) +_RESNET_LR_MULTIPLIERS = list(p[0] for p in _RESNET_LR_SCHEDULE) +_RESNET_LR_WARMUP_EPOCHS = _RESNET_LR_SCHEDULE[0][1] + + +@dataclasses.dataclass +class ResNetModelConfig(base_configs.ModelConfig): + """Configuration for the ResNet model.""" + name: str = 'ResNet' + num_classes: int = 1000 + model_params: Mapping[str, Any] = dataclasses.field(default_factory=lambda: { + 'num_classes': 1000, + 'batch_size': None, + 'use_l2_regularizer': True, + 'rescale_inputs': False, + }) + loss: base_configs.LossConfig = base_configs.LossConfig( + name='sparse_categorical_crossentropy') + optimizer: base_configs.OptimizerConfig = base_configs.OptimizerConfig( + name='momentum', + decay=0.9, + epsilon=0.001, + momentum=0.9, + moving_average_decay=None) + learning_rate: base_configs.LearningRateConfig = ( + base_configs.LearningRateConfig( + name='piecewise_constant_with_warmup', + examples_per_epoch=1281167, + warmup_epochs=_RESNET_LR_WARMUP_EPOCHS, + boundaries=_RESNET_LR_BOUNDARIES, + multipliers=_RESNET_LR_MULTIPLIERS)) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py new file mode 100644 index 0000000..b337252 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py @@ -0,0 +1,192 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Runs a ResNet model on the ImageNet dataset using custom training loops.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import app +from absl import flags +from absl import logging +import tensorflow as tf + +from official.modeling import performance +from official.staging.training import controller +from official.utils.flags import core as flags_core +from official.utils.logs import logger +from official.utils.misc import distribution_utils +from official.utils.misc import keras_utils +from official.utils.misc import model_helpers +from official.vision.image_classification.resnet import common +from official.vision.image_classification.resnet import imagenet_preprocessing +from official.vision.image_classification.resnet import resnet_runnable + +flags.DEFINE_boolean(name='use_tf_function', default=True, + help='Wrap the train and test step inside a ' + 'tf.function.') +flags.DEFINE_boolean(name='single_l2_loss_op', default=False, + help='Calculate L2_loss on concatenated weights, ' + 'instead of using Keras per-layer L2 loss.') + + +def build_stats(runnable, time_callback): + """Normalizes and returns dictionary of stats. + + Args: + runnable: The module containing all the training and evaluation metrics. + time_callback: Time tracking callback instance. + + Returns: + Dictionary of normalized results. + """ + stats = {} + + if not runnable.flags_obj.skip_eval: + stats['eval_loss'] = runnable.test_loss.result().numpy() + stats['eval_acc'] = runnable.test_accuracy.result().numpy() + + stats['train_loss'] = runnable.train_loss.result().numpy() + stats['train_acc'] = runnable.train_accuracy.result().numpy() + + if time_callback: + timestamp_log = time_callback.timestamp_log + stats['step_timestamp_log'] = timestamp_log + stats['train_finish_time'] = time_callback.train_finish_time + if time_callback.epoch_runtime_log: + stats['avg_exp_per_second'] = time_callback.average_examples_per_second + + return stats + + +def get_num_train_iterations(flags_obj): + """Returns the number of training steps, train and test epochs.""" + train_steps = ( + imagenet_preprocessing.NUM_IMAGES['train'] // flags_obj.batch_size) + train_epochs = flags_obj.train_epochs + + if flags_obj.train_steps: + train_steps = min(flags_obj.train_steps, train_steps) + train_epochs = 1 + + eval_steps = ( + imagenet_preprocessing.NUM_IMAGES['validation'] // flags_obj.batch_size) + + return train_steps, train_epochs, eval_steps + + +def _steps_to_run(steps_in_current_epoch, steps_per_epoch, steps_per_loop): + """Calculates steps to run on device.""" + if steps_per_loop <= 0: + raise ValueError('steps_per_loop should be positive integer.') + if steps_per_loop == 1: + return steps_per_loop + return min(steps_per_loop, steps_per_epoch - steps_in_current_epoch) + + +def run(flags_obj): + """Run ResNet ImageNet training and eval loop using custom training loops. + + Args: + flags_obj: An object containing parsed flag values. + + Raises: + ValueError: If fp16 is passed as it is not currently supported. + + Returns: + Dictionary of training and eval stats. + """ + keras_utils.set_session_config( + enable_eager=flags_obj.enable_eager, + enable_xla=flags_obj.enable_xla) + performance.set_mixed_precision_policy(flags_core.get_tf_dtype(flags_obj)) + + # This only affects GPU. + common.set_cudnn_batchnorm_mode() + + # TODO(anj-s): Set data_format without using Keras. + data_format = flags_obj.data_format + if data_format is None: + data_format = ('channels_first' if tf.config.list_physical_devices('GPU') + else 'channels_last') + tf.keras.backend.set_image_data_format(data_format) + + strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=flags_obj.distribution_strategy, + num_gpus=flags_obj.num_gpus, + all_reduce_alg=flags_obj.all_reduce_alg, + num_packs=flags_obj.num_packs, + tpu_address=flags_obj.tpu) + + per_epoch_steps, train_epochs, eval_steps = get_num_train_iterations( + flags_obj) + steps_per_loop = min(flags_obj.steps_per_loop, per_epoch_steps) + + logging.info( + 'Training %d epochs, each epoch has %d steps, ' + 'total steps: %d; Eval %d steps', train_epochs, per_epoch_steps, + train_epochs * per_epoch_steps, eval_steps) + + time_callback = keras_utils.TimeHistory( + flags_obj.batch_size, + flags_obj.log_steps, + logdir=flags_obj.model_dir if flags_obj.enable_tensorboard else None) + with distribution_utils.get_strategy_scope(strategy): + runnable = resnet_runnable.ResnetRunnable(flags_obj, time_callback, + per_epoch_steps) + + eval_interval = flags_obj.epochs_between_evals * per_epoch_steps + checkpoint_interval = ( + per_epoch_steps if flags_obj.enable_checkpoint_and_export else None) + summary_interval = per_epoch_steps if flags_obj.enable_tensorboard else None + + checkpoint_manager = tf.train.CheckpointManager( + runnable.checkpoint, + directory=flags_obj.model_dir, + max_to_keep=10, + step_counter=runnable.global_step, + checkpoint_interval=checkpoint_interval) + + resnet_controller = controller.Controller( + strategy, + runnable.train, + runnable.evaluate, + global_step=runnable.global_step, + steps_per_loop=steps_per_loop, + train_steps=per_epoch_steps * train_epochs, + checkpoint_manager=checkpoint_manager, + summary_interval=summary_interval, + eval_steps=eval_steps, + eval_interval=eval_interval) + + time_callback.on_train_begin() + resnet_controller.train(evaluate=not flags_obj.skip_eval) + time_callback.on_train_end() + + stats = build_stats(runnable, time_callback) + return stats + + +def main(_): + model_helpers.apply_clean(flags.FLAGS) + with logger.benchmark_context(flags.FLAGS): + stats = run(flags.FLAGS) + logging.info('Run stats:\n%s', stats) + + +if __name__ == '__main__': + logging.set_verbosity(logging.INFO) + common.define_keras_flags() + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_imagenet_main.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_imagenet_main.py new file mode 100644 index 0000000..dd4f40c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_imagenet_main.py @@ -0,0 +1,305 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Runs a ResNet model on the ImageNet dataset.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from absl import app +from absl import flags +from absl import logging +import tensorflow as tf + +import tensorflow_model_optimization as tfmot +from official.modeling import performance +from official.utils.flags import core as flags_core +from official.utils.logs import logger +from official.utils.misc import distribution_utils +from official.utils.misc import keras_utils +from official.utils.misc import model_helpers +from official.vision.image_classification import test_utils +from official.vision.image_classification.resnet import common +from official.vision.image_classification.resnet import imagenet_preprocessing +from official.vision.image_classification.resnet import resnet_model + + +def run(flags_obj): + """Run ResNet ImageNet training and eval loop using native Keras APIs. + + Args: + flags_obj: An object containing parsed flag values. + + Raises: + ValueError: If fp16 is passed as it is not currently supported. + NotImplementedError: If some features are not currently supported. + + Returns: + Dictionary of training and eval stats. + """ + keras_utils.set_session_config( + enable_eager=flags_obj.enable_eager, + enable_xla=flags_obj.enable_xla) + + # Execute flag override logic for better model performance + if flags_obj.tf_gpu_thread_mode: + keras_utils.set_gpu_thread_mode_and_count( + per_gpu_thread_count=flags_obj.per_gpu_thread_count, + gpu_thread_mode=flags_obj.tf_gpu_thread_mode, + num_gpus=flags_obj.num_gpus, + datasets_num_private_threads=flags_obj.datasets_num_private_threads) + common.set_cudnn_batchnorm_mode() + + dtype = flags_core.get_tf_dtype(flags_obj) + performance.set_mixed_precision_policy( + flags_core.get_tf_dtype(flags_obj), + flags_core.get_loss_scale(flags_obj, default_for_fp16=128)) + + data_format = flags_obj.data_format + if data_format is None: + data_format = ('channels_first' if tf.config.list_physical_devices('GPU') + else 'channels_last') + tf.keras.backend.set_image_data_format(data_format) + + # Configures cluster spec for distribution strategy. + _ = distribution_utils.configure_cluster(flags_obj.worker_hosts, + flags_obj.task_index) + + strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=flags_obj.distribution_strategy, + num_gpus=flags_obj.num_gpus, + all_reduce_alg=flags_obj.all_reduce_alg, + num_packs=flags_obj.num_packs, + tpu_address=flags_obj.tpu) + + if strategy: + # flags_obj.enable_get_next_as_optional controls whether enabling + # get_next_as_optional behavior in DistributedIterator. If true, last + # partial batch can be supported. + strategy.extended.experimental_enable_get_next_as_optional = ( + flags_obj.enable_get_next_as_optional + ) + + strategy_scope = distribution_utils.get_strategy_scope(strategy) + + # pylint: disable=protected-access + if flags_obj.use_synthetic_data: + input_fn = common.get_synth_input_fn( + height=imagenet_preprocessing.DEFAULT_IMAGE_SIZE, + width=imagenet_preprocessing.DEFAULT_IMAGE_SIZE, + num_channels=imagenet_preprocessing.NUM_CHANNELS, + num_classes=imagenet_preprocessing.NUM_CLASSES, + dtype=dtype, + drop_remainder=True) + else: + input_fn = imagenet_preprocessing.input_fn + + # When `enable_xla` is True, we always drop the remainder of the batches + # in the dataset, as XLA-GPU doesn't support dynamic shapes. + drop_remainder = flags_obj.enable_xla + + # Current resnet_model.resnet50 input format is always channel-last. + # We use keras_application mobilenet model which input format is depends on + # the keras beckend image data format. + # This use_keras_image_data_format flags indicates whether image preprocessor + # output format should be same as the keras backend image data format or just + # channel-last format. + use_keras_image_data_format = (flags_obj.model == 'mobilenet') + train_input_dataset = input_fn( + is_training=True, + data_dir=flags_obj.data_dir, + batch_size=flags_obj.batch_size, + parse_record_fn=imagenet_preprocessing.get_parse_record_fn( + use_keras_image_data_format=use_keras_image_data_format), + datasets_num_private_threads=flags_obj.datasets_num_private_threads, + dtype=dtype, + drop_remainder=drop_remainder, + tf_data_experimental_slack=flags_obj.tf_data_experimental_slack, + training_dataset_cache=flags_obj.training_dataset_cache, + ) + + eval_input_dataset = None + if not flags_obj.skip_eval: + eval_input_dataset = input_fn( + is_training=False, + data_dir=flags_obj.data_dir, + batch_size=flags_obj.batch_size, + parse_record_fn=imagenet_preprocessing.get_parse_record_fn( + use_keras_image_data_format=use_keras_image_data_format), + dtype=dtype, + drop_remainder=drop_remainder) + + lr_schedule = common.PiecewiseConstantDecayWithWarmup( + batch_size=flags_obj.batch_size, + epoch_size=imagenet_preprocessing.NUM_IMAGES['train'], + warmup_epochs=common.LR_SCHEDULE[0][1], + boundaries=list(p[1] for p in common.LR_SCHEDULE[1:]), + multipliers=list(p[0] for p in common.LR_SCHEDULE), + compute_lr_on_cpu=True) + steps_per_epoch = ( + imagenet_preprocessing.NUM_IMAGES['train'] // flags_obj.batch_size) + + with strategy_scope: + if flags_obj.optimizer == 'resnet50_default': + optimizer = common.get_optimizer(lr_schedule) + elif flags_obj.optimizer == 'mobilenet_default': + initial_learning_rate = \ + flags_obj.initial_learning_rate_per_sample * flags_obj.batch_size + optimizer = tf.keras.optimizers.SGD( + learning_rate=tf.keras.optimizers.schedules.ExponentialDecay( + initial_learning_rate, + decay_steps=steps_per_epoch * flags_obj.num_epochs_per_decay, + decay_rate=flags_obj.lr_decay_factor, + staircase=True), + momentum=0.9) + if flags_obj.fp16_implementation == 'graph_rewrite': + # Note: when flags_obj.fp16_implementation == "graph_rewrite", dtype as + # determined by flags_core.get_tf_dtype(flags_obj) would be 'float32' + # which will ensure tf.compat.v2.keras.mixed_precision and + # tf.train.experimental.enable_mixed_precision_graph_rewrite do not double + # up. + optimizer = tf.train.experimental.enable_mixed_precision_graph_rewrite( + optimizer) + + # TODO(hongkuny): Remove trivial model usage and move it to benchmark. + if flags_obj.use_trivial_model: + model = test_utils.trivial_model(imagenet_preprocessing.NUM_CLASSES) + elif flags_obj.model == 'resnet50_v1.5': + model = resnet_model.resnet50( + num_classes=imagenet_preprocessing.NUM_CLASSES) + elif flags_obj.model == 'mobilenet': + # TODO(kimjaehong): Remove layers attribute when minimum TF version + # support 2.0 layers by default. + model = tf.keras.applications.mobilenet.MobileNet( + weights=None, + classes=imagenet_preprocessing.NUM_CLASSES, + layers=tf.keras.layers) + if flags_obj.pretrained_filepath: + model.load_weights(flags_obj.pretrained_filepath) + + if flags_obj.pruning_method == 'polynomial_decay': + if dtype != tf.float32: + raise NotImplementedError( + 'Pruning is currently only supported on dtype=tf.float32.') + pruning_params = { + 'pruning_schedule': + tfmot.sparsity.keras.PolynomialDecay( + initial_sparsity=flags_obj.pruning_initial_sparsity, + final_sparsity=flags_obj.pruning_final_sparsity, + begin_step=flags_obj.pruning_begin_step, + end_step=flags_obj.pruning_end_step, + frequency=flags_obj.pruning_frequency), + } + model = tfmot.sparsity.keras.prune_low_magnitude(model, **pruning_params) + elif flags_obj.pruning_method: + raise NotImplementedError( + 'Only polynomial_decay is currently supported.') + + model.compile( + loss='sparse_categorical_crossentropy', + optimizer=optimizer, + metrics=(['sparse_categorical_accuracy'] + if flags_obj.report_accuracy_metrics else None), + run_eagerly=flags_obj.run_eagerly) + + train_epochs = flags_obj.train_epochs + + callbacks = common.get_callbacks( + steps_per_epoch=steps_per_epoch, + pruning_method=flags_obj.pruning_method, + enable_checkpoint_and_export=flags_obj.enable_checkpoint_and_export, + model_dir=flags_obj.model_dir) + + # if mutliple epochs, ignore the train_steps flag. + if train_epochs <= 1 and flags_obj.train_steps: + steps_per_epoch = min(flags_obj.train_steps, steps_per_epoch) + train_epochs = 1 + + num_eval_steps = ( + imagenet_preprocessing.NUM_IMAGES['validation'] // flags_obj.batch_size) + + validation_data = eval_input_dataset + if flags_obj.skip_eval: + # Only build the training graph. This reduces memory usage introduced by + # control flow ops in layers that have different implementations for + # training and inference (e.g., batch norm). + if flags_obj.set_learning_phase_to_train: + # TODO(haoyuzhang): Understand slowdown of setting learning phase when + # not using distribution strategy. + tf.keras.backend.set_learning_phase(1) + num_eval_steps = None + validation_data = None + + if not strategy and flags_obj.explicit_gpu_placement: + # TODO(b/135607227): Add device scope automatically in Keras training loop + # when not using distribition strategy. + no_dist_strat_device = tf.device('/device:GPU:0') + no_dist_strat_device.__enter__() + + history = model.fit(train_input_dataset, + epochs=train_epochs, + steps_per_epoch=steps_per_epoch, + callbacks=callbacks, + validation_steps=num_eval_steps, + validation_data=validation_data, + validation_freq=flags_obj.epochs_between_evals, + verbose=2) + + eval_output = None + if not flags_obj.skip_eval: + eval_output = model.evaluate(eval_input_dataset, + steps=num_eval_steps, + verbose=2) + + if flags_obj.pruning_method: + model = tfmot.sparsity.keras.strip_pruning(model) + if flags_obj.enable_checkpoint_and_export: + if dtype == tf.bfloat16: + logging.warning('Keras model.save does not support bfloat16 dtype.') + else: + # Keras model.save assumes a float32 input designature. + export_path = os.path.join(flags_obj.model_dir, 'saved_model') + model.save(export_path, include_optimizer=False) + + if not strategy and flags_obj.explicit_gpu_placement: + no_dist_strat_device.__exit__() + + stats = common.build_stats(history, eval_output, callbacks) + return stats + + +def define_imagenet_keras_flags(): + common.define_keras_flags( + model=True, + optimizer=True, + pretrained_filepath=True) + common.define_pruning_flags() + flags_core.set_defaults() + flags.adopt_module_key_flags(common) + + +def main(_): + model_helpers.apply_clean(flags.FLAGS) + with logger.benchmark_context(flags.FLAGS): + stats = run(flags.FLAGS) + logging.info('Run stats:\n%s', stats) + + +if __name__ == '__main__': + logging.set_verbosity(logging.INFO) + define_imagenet_keras_flags() + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_model.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_model.py new file mode 100644 index 0000000..10f1233 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_model.py @@ -0,0 +1,329 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""ResNet50 model for Keras. + +Adapted from tf.keras.applications.resnet50.ResNet50(). +This is ResNet model version 1.5. + +Related papers/blogs: +- https://arxiv.org/abs/1512.03385 +- https://arxiv.org/pdf/1603.05027v2.pdf +- http://torch.ch/blog/2016/02/04/resnets.html + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from tensorflow.python.keras import backend +from tensorflow.python.keras import initializers +from tensorflow.python.keras import models +from tensorflow.python.keras import regularizers +from official.vision.image_classification.resnet import imagenet_preprocessing + +layers = tf.keras.layers + + +def _gen_l2_regularizer(use_l2_regularizer=True, l2_weight_decay=1e-4): + return regularizers.l2(l2_weight_decay) if use_l2_regularizer else None + + +def identity_block(input_tensor, + kernel_size, + filters, + stage, + block, + use_l2_regularizer=True, + batch_norm_decay=0.9, + batch_norm_epsilon=1e-5): + """The identity block is the block that has no conv layer at shortcut. + + Args: + input_tensor: input tensor + kernel_size: default 3, the kernel size of middle conv layer at main path + filters: list of integers, the filters of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + block: 'a','b'..., current block label, used for generating layer names + use_l2_regularizer: whether to use L2 regularizer on Conv layer. + batch_norm_decay: Moment of batch norm layers. + batch_norm_epsilon: Epsilon of batch borm layers. + + Returns: + Output tensor for the block. + """ + filters1, filters2, filters3 = filters + if backend.image_data_format() == 'channels_last': + bn_axis = 3 + else: + bn_axis = 1 + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + + x = layers.Conv2D( + filters1, (1, 1), + use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=_gen_l2_regularizer(use_l2_regularizer), + name=conv_name_base + '2a')( + input_tensor) + x = layers.BatchNormalization( + axis=bn_axis, + momentum=batch_norm_decay, + epsilon=batch_norm_epsilon, + name=bn_name_base + '2a')( + x) + x = layers.Activation('relu')(x) + + x = layers.Conv2D( + filters2, + kernel_size, + padding='same', + use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=_gen_l2_regularizer(use_l2_regularizer), + name=conv_name_base + '2b')( + x) + x = layers.BatchNormalization( + axis=bn_axis, + momentum=batch_norm_decay, + epsilon=batch_norm_epsilon, + name=bn_name_base + '2b')( + x) + x = layers.Activation('relu')(x) + + x = layers.Conv2D( + filters3, (1, 1), + use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=_gen_l2_regularizer(use_l2_regularizer), + name=conv_name_base + '2c')( + x) + x = layers.BatchNormalization( + axis=bn_axis, + momentum=batch_norm_decay, + epsilon=batch_norm_epsilon, + name=bn_name_base + '2c')( + x) + + x = layers.add([x, input_tensor]) + x = layers.Activation('relu')(x) + return x + + +def conv_block(input_tensor, + kernel_size, + filters, + stage, + block, + strides=(2, 2), + use_l2_regularizer=True, + batch_norm_decay=0.9, + batch_norm_epsilon=1e-5): + """A block that has a conv layer at shortcut. + + Note that from stage 3, + the second conv layer at main path is with strides=(2, 2) + And the shortcut should have strides=(2, 2) as well + + Args: + input_tensor: input tensor + kernel_size: default 3, the kernel size of middle conv layer at main path + filters: list of integers, the filters of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + block: 'a','b'..., current block label, used for generating layer names + strides: Strides for the second conv layer in the block. + use_l2_regularizer: whether to use L2 regularizer on Conv layer. + batch_norm_decay: Moment of batch norm layers. + batch_norm_epsilon: Epsilon of batch borm layers. + + Returns: + Output tensor for the block. + """ + filters1, filters2, filters3 = filters + if backend.image_data_format() == 'channels_last': + bn_axis = 3 + else: + bn_axis = 1 + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + + x = layers.Conv2D( + filters1, (1, 1), + use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=_gen_l2_regularizer(use_l2_regularizer), + name=conv_name_base + '2a')( + input_tensor) + x = layers.BatchNormalization( + axis=bn_axis, + momentum=batch_norm_decay, + epsilon=batch_norm_epsilon, + name=bn_name_base + '2a')( + x) + x = layers.Activation('relu')(x) + + x = layers.Conv2D( + filters2, + kernel_size, + strides=strides, + padding='same', + use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=_gen_l2_regularizer(use_l2_regularizer), + name=conv_name_base + '2b')( + x) + x = layers.BatchNormalization( + axis=bn_axis, + momentum=batch_norm_decay, + epsilon=batch_norm_epsilon, + name=bn_name_base + '2b')( + x) + x = layers.Activation('relu')(x) + + x = layers.Conv2D( + filters3, (1, 1), + use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=_gen_l2_regularizer(use_l2_regularizer), + name=conv_name_base + '2c')( + x) + x = layers.BatchNormalization( + axis=bn_axis, + momentum=batch_norm_decay, + epsilon=batch_norm_epsilon, + name=bn_name_base + '2c')( + x) + + shortcut = layers.Conv2D( + filters3, (1, 1), + strides=strides, + use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=_gen_l2_regularizer(use_l2_regularizer), + name=conv_name_base + '1')( + input_tensor) + shortcut = layers.BatchNormalization( + axis=bn_axis, + momentum=batch_norm_decay, + epsilon=batch_norm_epsilon, + name=bn_name_base + '1')( + shortcut) + + x = layers.add([x, shortcut]) + x = layers.Activation('relu')(x) + return x + + +def resnet50(num_classes, + batch_size=None, + use_l2_regularizer=True, + rescale_inputs=False, + batch_norm_decay=0.9, + batch_norm_epsilon=1e-5): + """Instantiates the ResNet50 architecture. + + Args: + num_classes: `int` number of classes for image classification. + batch_size: Size of the batches for each step. + use_l2_regularizer: whether to use L2 regularizer on Conv/Dense layer. + rescale_inputs: whether to rescale inputs from 0 to 1. + batch_norm_decay: Moment of batch norm layers. + batch_norm_epsilon: Epsilon of batch borm layers. + + Returns: + A Keras model instance. + """ + input_shape = (224, 224, 3) + img_input = layers.Input(shape=input_shape, batch_size=batch_size) + if rescale_inputs: + # Hub image modules expect inputs in the range [0, 1]. This rescales these + # inputs to the range expected by the trained model. + x = layers.Lambda( + lambda x: x * 255.0 - backend.constant( + imagenet_preprocessing.CHANNEL_MEANS, + shape=[1, 1, 3], + dtype=x.dtype), + name='rescale')( + img_input) + else: + x = img_input + + if backend.image_data_format() == 'channels_first': + x = layers.Permute((3, 1, 2))(x) + bn_axis = 1 + else: # channels_last + bn_axis = 3 + + block_config = dict( + use_l2_regularizer=use_l2_regularizer, + batch_norm_decay=batch_norm_decay, + batch_norm_epsilon=batch_norm_epsilon) + x = layers.ZeroPadding2D(padding=(3, 3), name='conv1_pad')(x) + x = layers.Conv2D( + 64, (7, 7), + strides=(2, 2), + padding='valid', + use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=_gen_l2_regularizer(use_l2_regularizer), + name='conv1')( + x) + x = layers.BatchNormalization( + axis=bn_axis, + momentum=batch_norm_decay, + epsilon=batch_norm_epsilon, + name='bn_conv1')( + x) + x = layers.Activation('relu')(x) + x = layers.MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) + + x = conv_block( + x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1), **block_config) + x = identity_block(x, 3, [64, 64, 256], stage=2, block='b', **block_config) + x = identity_block(x, 3, [64, 64, 256], stage=2, block='c', **block_config) + + x = conv_block(x, 3, [128, 128, 512], stage=3, block='a', **block_config) + x = identity_block(x, 3, [128, 128, 512], stage=3, block='b', **block_config) + x = identity_block(x, 3, [128, 128, 512], stage=3, block='c', **block_config) + x = identity_block(x, 3, [128, 128, 512], stage=3, block='d', **block_config) + + x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a', **block_config) + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b', **block_config) + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c', **block_config) + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d', **block_config) + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e', **block_config) + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f', **block_config) + + x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a', **block_config) + x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b', **block_config) + x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c', **block_config) + + x = layers.GlobalAveragePooling2D()(x) + x = layers.Dense( + num_classes, + kernel_initializer=initializers.RandomNormal(stddev=0.01), + kernel_regularizer=_gen_l2_regularizer(use_l2_regularizer), + bias_regularizer=_gen_l2_regularizer(use_l2_regularizer), + name='fc1000')( + x) + + # A softmax that is followed by the model loss must be done cannot be done + # in float16 due to numeric issues. So we pass dtype=float32. + x = layers.Activation('softmax', dtype='float32')(x) + + # Create model. + return models.Model(img_input, x, name='resnet50') diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_runnable.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_runnable.py new file mode 100644 index 0000000..fd9816f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/resnet_runnable.py @@ -0,0 +1,222 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Runs a ResNet model on the ImageNet dataset using custom training loops.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from official.modeling import performance +from official.staging.training import grad_utils +from official.staging.training import standard_runnable +from official.staging.training import utils +from official.utils.flags import core as flags_core +from official.vision.image_classification.resnet import common +from official.vision.image_classification.resnet import imagenet_preprocessing +from official.vision.image_classification.resnet import resnet_model + + +class ResnetRunnable(standard_runnable.StandardTrainable, + standard_runnable.StandardEvaluable): + """Implements the training and evaluation APIs for Resnet model.""" + + def __init__(self, flags_obj, time_callback, epoch_steps): + standard_runnable.StandardTrainable.__init__(self, + flags_obj.use_tf_while_loop, + flags_obj.use_tf_function) + standard_runnable.StandardEvaluable.__init__(self, + flags_obj.use_tf_function) + + self.strategy = tf.distribute.get_strategy() + self.flags_obj = flags_obj + self.dtype = flags_core.get_tf_dtype(flags_obj) + self.time_callback = time_callback + + # Input pipeline related + batch_size = flags_obj.batch_size + if batch_size % self.strategy.num_replicas_in_sync != 0: + raise ValueError( + 'Batch size must be divisible by number of replicas : {}'.format( + self.strategy.num_replicas_in_sync)) + + # As auto rebatching is not supported in + # `experimental_distribute_datasets_from_function()` API, which is + # required when cloning dataset to multiple workers in eager mode, + # we use per-replica batch size. + self.batch_size = int(batch_size / self.strategy.num_replicas_in_sync) + + if self.flags_obj.use_synthetic_data: + self.input_fn = common.get_synth_input_fn( + height=imagenet_preprocessing.DEFAULT_IMAGE_SIZE, + width=imagenet_preprocessing.DEFAULT_IMAGE_SIZE, + num_channels=imagenet_preprocessing.NUM_CHANNELS, + num_classes=imagenet_preprocessing.NUM_CLASSES, + dtype=self.dtype, + drop_remainder=True) + else: + self.input_fn = imagenet_preprocessing.input_fn + + self.model = resnet_model.resnet50( + num_classes=imagenet_preprocessing.NUM_CLASSES, + batch_size=flags_obj.batch_size, + use_l2_regularizer=not flags_obj.single_l2_loss_op) + + lr_schedule = common.PiecewiseConstantDecayWithWarmup( + batch_size=flags_obj.batch_size, + epoch_size=imagenet_preprocessing.NUM_IMAGES['train'], + warmup_epochs=common.LR_SCHEDULE[0][1], + boundaries=list(p[1] for p in common.LR_SCHEDULE[1:]), + multipliers=list(p[0] for p in common.LR_SCHEDULE), + compute_lr_on_cpu=True) + self.optimizer = common.get_optimizer(lr_schedule) + # Make sure iterations variable is created inside scope. + self.global_step = self.optimizer.iterations + + use_graph_rewrite = flags_obj.fp16_implementation == 'graph_rewrite' + if use_graph_rewrite and not flags_obj.use_tf_function: + raise ValueError('--fp16_implementation=graph_rewrite requires ' + '--use_tf_function to be true') + self.optimizer = performance.configure_optimizer( + self.optimizer, + use_float16=self.dtype == tf.float16, + use_graph_rewrite=use_graph_rewrite, + loss_scale=flags_core.get_loss_scale(flags_obj, default_for_fp16=128)) + + self.train_loss = tf.keras.metrics.Mean('train_loss', dtype=tf.float32) + self.train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy( + 'train_accuracy', dtype=tf.float32) + self.test_loss = tf.keras.metrics.Mean('test_loss', dtype=tf.float32) + self.test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy( + 'test_accuracy', dtype=tf.float32) + + self.checkpoint = tf.train.Checkpoint( + model=self.model, optimizer=self.optimizer) + + # Handling epochs. + self.epoch_steps = epoch_steps + self.epoch_helper = utils.EpochHelper(epoch_steps, self.global_step) + + def build_train_dataset(self): + """See base class.""" + return utils.make_distributed_dataset( + self.strategy, + self.input_fn, + is_training=True, + data_dir=self.flags_obj.data_dir, + batch_size=self.batch_size, + parse_record_fn=imagenet_preprocessing.parse_record, + datasets_num_private_threads=self.flags_obj + .datasets_num_private_threads, + dtype=self.dtype, + drop_remainder=True) + + def build_eval_dataset(self): + """See base class.""" + return utils.make_distributed_dataset( + self.strategy, + self.input_fn, + is_training=False, + data_dir=self.flags_obj.data_dir, + batch_size=self.batch_size, + parse_record_fn=imagenet_preprocessing.parse_record, + dtype=self.dtype) + + def train_loop_begin(self): + """See base class.""" + # Reset all metrics + self.train_loss.reset_states() + self.train_accuracy.reset_states() + + self._epoch_begin() + self.time_callback.on_batch_begin(self.epoch_helper.batch_index) + + def train_step(self, iterator): + """See base class.""" + + def step_fn(inputs): + """Function to run on the device.""" + images, labels = inputs + with tf.GradientTape() as tape: + logits = self.model(images, training=True) + + prediction_loss = tf.keras.losses.sparse_categorical_crossentropy( + labels, logits) + loss = tf.reduce_sum(prediction_loss) * (1.0 / + self.flags_obj.batch_size) + num_replicas = self.strategy.num_replicas_in_sync + l2_weight_decay = 1e-4 + if self.flags_obj.single_l2_loss_op: + l2_loss = l2_weight_decay * 2 * tf.add_n([ + tf.nn.l2_loss(v) + for v in self.model.trainable_variables + if 'bn' not in v.name + ]) + + loss += (l2_loss / num_replicas) + else: + loss += (tf.reduce_sum(self.model.losses) / num_replicas) + + grad_utils.minimize_using_explicit_allreduce( + tape, self.optimizer, loss, self.model.trainable_variables) + self.train_loss.update_state(loss) + self.train_accuracy.update_state(labels, logits) + + self.strategy.run(step_fn, args=(next(iterator),)) + + def train_loop_end(self): + """See base class.""" + metrics = { + 'train_loss': self.train_loss.result(), + 'train_accuracy': self.train_accuracy.result(), + } + self.time_callback.on_batch_end(self.epoch_helper.batch_index - 1) + self._epoch_end() + return metrics + + def eval_begin(self): + """See base class.""" + self.test_loss.reset_states() + self.test_accuracy.reset_states() + + def eval_step(self, iterator): + """See base class.""" + + def step_fn(inputs): + """Function to run on the device.""" + images, labels = inputs + logits = self.model(images, training=False) + loss = tf.keras.losses.sparse_categorical_crossentropy(labels, logits) + loss = tf.reduce_sum(loss) * (1.0 / self.flags_obj.batch_size) + self.test_loss.update_state(loss) + self.test_accuracy.update_state(labels, logits) + + self.strategy.run(step_fn, args=(next(iterator),)) + + def eval_end(self): + """See base class.""" + return { + 'test_loss': self.test_loss.result(), + 'test_accuracy': self.test_accuracy.result() + } + + def _epoch_begin(self): + if self.epoch_helper.epoch_begin(): + self.time_callback.on_epoch_begin(self.epoch_helper.current_epoch) + + def _epoch_end(self): + if self.epoch_helper.epoch_end(): + self.time_callback.on_epoch_end(self.epoch_helper.current_epoch) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/tfhub_export.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/tfhub_export.py new file mode 100644 index 0000000..db00173 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/resnet/tfhub_export.py @@ -0,0 +1,67 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A script to export TF-Hub SavedModel.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import os + +from absl import app +from absl import flags + +import tensorflow as tf + +from official.vision.image_classification.resnet import imagenet_preprocessing +from official.vision.image_classification.resnet import resnet_model + +FLAGS = flags.FLAGS + +flags.DEFINE_string("model_path", None, + "File path to TF model checkpoint or H5 file.") +flags.DEFINE_string("export_path", None, + "TF-Hub SavedModel destination path to export.") + + +def export_tfhub(model_path, hub_destination): + """Restores a tf.keras.Model and saves for TF-Hub.""" + model = resnet_model.resnet50( + num_classes=imagenet_preprocessing.NUM_CLASSES, rescale_inputs=True) + model.load_weights(model_path) + model.save( + os.path.join(hub_destination, "classification"), include_optimizer=False) + + # Extracts a sub-model to use pooling feature vector as model output. + image_input = model.get_layer(index=0).get_output_at(0) + feature_vector_output = model.get_layer(name="reduce_mean").get_output_at(0) + hub_model = tf.keras.Model(image_input, feature_vector_output) + + # Exports a SavedModel. + hub_model.save( + os.path.join(hub_destination, "feature-vector"), include_optimizer=False) + + +def main(argv): + if len(argv) > 1: + raise app.UsageError("Too many command-line arguments.") + + tf.enable_v2_behavior() + export_tfhub(FLAGS.model_path, FLAGS.export_path) + + +if __name__ == "__main__": + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/test_utils.py b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/test_utils.py new file mode 100644 index 0000000..a6dc91d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/code/official/vision/image_classification/test_utils.py @@ -0,0 +1,38 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Test utilities for image classification tasks.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.keras import backend +from tensorflow.python.keras import layers +from tensorflow.python.keras import models + + +def trivial_model(num_classes): + """Trivial model for ImageNet dataset.""" + + input_shape = (224, 224, 3) + img_input = layers.Input(shape=input_shape) + + x = layers.Lambda(lambda x: backend.reshape(x, [-1, 224 * 224 * 3]), + name='reshape')(img_input) + x = layers.Dense(1, name='fc1')(x) + x = layers.Dense(num_classes, name='fc1000')(x) + x = layers.Activation('softmax', dtype='float32')(x) + + return models.Model(img_input, x, name='trivial') diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/config/npu_set_env.sh b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/config/npu_set_env.sh new file mode 100644 index 0000000..4619007 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/config/npu_set_env.sh @@ -0,0 +1,41 @@ +# main env +if [ -d /usr/local/Ascend/nnae/latest ];then + + export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/Ascend/driver/tools/hccn_tool/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages + export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp +else + export LD_LIBRARY_PATH=/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/ascend-toolkit/latest//fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/tfplugin/python/site-packages:$projectDir + export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + +fi +export SOC_VERSION=Ascend910 +export HCCL_CONNECT_TIMEOUT=600 + +# user env +export JOB_ID={JOB_ID} +export RANK_TABLE_FILE={RANK_TABLE_FILE} +#export RANK_SIZE={RANK_SIZE} +#export RANK_INDEX={RANK_INDEX} +#export RANK_ID={RANK_ID} + +# profiling env + +export PROFILING_MODE=false +export PROFILING_OPTIONS=training_trace +export FP_POINT=resnet34/conv2d/Conv2Dresnet34/batch_normalization/FusedBatchNormV3_Reduce +export BP_POINT=Momentum/update_resnet34/conv2d/kernel/ApplyMomentum +export AICPU_PROFILING_MODE=false + +# debug env +#export DUMP_GE_GRAPH=2 +#export DUMP_OP=1 +#export DUMP_OP_LESS=1 +#export PRINT_MODEL=1 +#export TE_PARALLEL_COMPILER=0 + +# system env +ulimit -c unlimited diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/scripts/run.sh b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/scripts/run.sh new file mode 100644 index 0000000..f3d48eb --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/scripts/run.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +rank_size=$1 +yamlPath=$2 +toolsPath=$3 +if [ -f /.dockerenv ];then + CLUSTER=$4 + MPIRUN_ALL_IP="$5" + export CLUSTER=${CLUSTER} +fi +currentDir=$(cd "$(dirname "$0")/.."; pwd) +# 配置环境变量并调用 train 方法 +currtime=`date +%Y%m%d%H%M%S` +mkdir -p ${currentDir%train*}/train/result/tf_resnet101/training_job_${currtime}/ +train_job_dir=${currentDir%train*}/train/result/tf_resnet101/training_job_${currtime}/ +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir} &" + +# user env +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 +export SLOG_PRINT_TO_STDOUT=0 +export RANK_TABLE_FILE=${currentDir}/config/${rank_size}p.json + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "tensorflow_config") + + +# device 列表, 若无指定 device 根据 rank_size 顺序选择 +eval device_group=\$device_group_${rank_size}p +if [ x"${device_group}" == x"" ] || [ ${rank_size} -ge 8 ];then + device_group="$(seq 0 "$(expr $rank_size - 1)")" +fi + +# get last device id in device_group, hw log in performance from the dir named first_device_id +device_group_str=`echo ${device_group} | sed 's/ //g'` +first_device_id=`echo ${device_group_str: 0:1}` + + +rank_id=0 + +if [ x"${CLUSTER}" == x"True" ];then + # ln hw log + ln -snf ${train_job_dir}/0/hw_resnet101.log ${train_job_dir} + this_ip=$(hostname -I |awk '{print $1}') + for ip in $MPIRUN_ALL_IP;do + if [ x"$ip" != x"$this_ip" ];then + scp $yamlPath root@$ip:$yamlPath + fi + done + export PATH=$PATH:/usr/local/mpirun4.0/bin + mpirun -H ${mpirun_ip} \ + --bind-to none -map-by slot\ + --allow-run-as-root \ + --mca btl_tcp_if_exclude lo,docker0,endvnic,virbr0,vethf40501b,docker_gwbridge,br-f42ac38052b4\ + --prefix /usr/local/mpirun4.0/ \ + ${currentDir}/scripts/train.sh 0 $rank_size $yamlPath $currtime ${toolsPath} ${CLUSTER} +else + # ln hw log + ln -snf ${train_job_dir}/${first_device_id}/hw_resnet101.log ${train_job_dir} + for device_id in $device_group;do + #echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] start: train ${device_id} & " >> ${currentDir}/result/main.log + ${currentDir}/scripts/train.sh $device_id $rank_size $yamlPath $currtime ${toolsPath} $rank_id & + let rank_id++ + done +fi +wait + + +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir} &" \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/scripts/train.sh b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/scripts/train.sh new file mode 100644 index 0000000..e02a29c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet101/tensorflow/scripts/train.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash + +device_id=$1 +rank_size=$2 +yamlPath=$3 +currtime=$4 +toolsPath=$5 + +currentDir=$(cd "$(dirname "$0")/.."; pwd) + +export REMARK_LOG_FILE=hw_resnet101.log + +mkdir -p ${currentDir%train*}/train/result/tf_resnet101/training_job_${currtime}/ +export train_job_dir=${currentDir%train*}/train/result/tf_resnet101/training_job_${currtime}/ + + +source ${currentDir}/config/npu_set_env.sh + +benchmark_log_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils +#atlasboost_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +code_dir_path=${currentDir}/code +export PYTHONPATH=$PYTHONPATH:${benchmark_log_path}:${code_dir_path} + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "tensorflow_config") + +# user env +export YAML_PATH=$3 +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999002 +export RANK_TABLE_FILE=${currentDir}/config/${rank_size}p.json +export RANK_SIZE=${rank_size} +export RANK_INDEX=0 +export SLOG_PRINT_TO_STDOUT=0 +export DEVICE_ID=$1 +DEVICE_INDEX=$(( DEVICE_ID + RANK_INDEX * 8 )) +export DEVICE_INDEX=${DEVICE_INDEX} + +cd ${train_job_dir} +curd_dir=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +export PYTHONPATH=$PYTHONPATH:${curd_dir} + +if [ x"$6" != x"True" ];then + rank_id=$6 + export RANK_ID=$6 +else + device_id_mo=$(python3.7 -c "import src.tensorflow.mpi_ops as atlasboost;atlasboost.init(); \ + device_id = atlasboost.local_rank();cluster_device_id = str(device_id); \ + atlasboost.set_device_id(device_id);print(atlasboost.rank())") + device_id_mo=`echo $device_id_mo` + rank_id=${device_id_mo##* } + export RANK_ID=${rank_id} + device=${device_id_mo##*deviceid = } + device_id=${device%% phyid=*} + export DEVICE_ID=${device_id} + hccljson=${train_job_dir}/*.json + cp ${hccljson} ${currentDir}/config/${rank_size}p.json +fi + +export MODEL_CKPT_PATH=${train_job_dir}/${device_id}/ckpt${device_id} + +#mkdir exec path +mkdir -p ${train_job_dir}/${device_id} +cd ${train_job_dir}/${device_id} + +startTime=`date +%Y%m%d-%H:%M:%S` +startTime_s=`date +%s` + +# 根据单卡/多卡区分调用参数 + +if [ x"$6" == x"True" ];then + export CLUSTER=True + echo "run cluster" + python3.7 ${currentDir}/code/official/r1/resnet/imagenet_main.py \ + --device_count=${rank_size} \ + --resnet_size=101 \ + --batch_size=${batch_size} \ + --num_gpus=1 \ + --max_train_steps=${max_train_steps} \ + --cosine_lr=True \ + --dtype=fp16 \ + --label_smoothing=0.1 \ + --loss_scale=512 \ + --train_epochs=${epoches} \ + --epochs_between_evals=${epochs_between_evals} \ + --hooks=ExamplesPerSecondHook,loggingtensorhook,loggingmetrichook \ + --data_dir=${data_url} \ + --model_dir=${MODEL_CKPT_PATH} > ${train_job_dir}/train_${device_id}.log 2>&1 +else + python3.7 ${currentDir}/code/official/r1/resnet/imagenet_main.py \ + --device_count=${rank_size} \ + --resnet_size=101 \ + --batch_size=${batch_size} \ + --num_gpus=1 \ + --max_train_steps=${max_train_steps} \ + --cosine_lr=True \ + --dtype=fp16 \ + --label_smoothing=0.1 \ + --loss_scale=512 \ + --train_epochs=${epoches} \ + --epochs_between_evals=${epochs_between_evals} \ + --hooks=ExamplesPerSecondHook,loggingtensorhook,loggingmetrichook \ + --data_dir=${data_url} \ + --model_dir=${MODEL_CKPT_PATH} > ${train_job_dir}/train_${device_id}.log 2>&1 +fi + +if [ $? -eq 0 ] ;then + echo ":::ABK 1.0.0 resnet101 train success" + echo ":::ABK 1.0.0 resnet101 train success" >> ${train_job_dir}/train_${device_id}.log + echo ":::ABK 1.0.0 resnet101 train success" >> ${train_job_dir}/${device_id}/hw_resnet101.log +else + echo ":::ABK 1.0.0 resnet101 train failed" + echo ":::ABK 1.0.0 resnet101 train failed" >> ${train_job_dir}/train_${device_id}.log + echo ":::ABK 1.0.0 resnet101 train failed" >> ${train_job_dir}/${device_id}/hw_resnet101.log +fi + +endTime=`date +%Y%m%d-%H:%M:%S` +endTime_s=`date +%s` +sumTime=$[ $endTime_s - $startTime_s ] +hour=$(( $sumTime/3600 )) +min=$(( ($sumTime-${hour}*3600)/60 )) +sec=$(( $sumTime-${hour}*3600-${min}*60 )) +echo ":::ABK 1.0.0 resnet101 train total time ${hour}:${min}:${sec}" +echo ":::ABK 1.0.0 resnet101 train total time ${hour}:${min}:${sec}" >> ${train_job_dir}/${device_id}/hw_resnet101.log diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/README.md b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/README.md new file mode 100644 index 0000000..b821fd2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/README.md @@ -0,0 +1,480 @@ +# Contents + +- [ResNet Description](#resnet-description) +- [Model Architecture](#model-architecture) +- [Dataset](#dataset) +- [Features](#features) + - [Mixed Precision](#mixed-precision) +- [Environment Requirements](#environment-requirements) +- [Quick Start](#quick-start) +- [Script Description](#script-description) + - [Script and Sample Code](#script-and-sample-code) + - [Script Parameters](#script-parameters) + - [Training Process](#training-process) + - [Evaluation Process](#evaluation-process) +- [Model Description](#model-description) + - [Performance](#performance) + - [Evaluation Performance](#evaluation-performance) +- [Description of Random Situation](#description-of-random-situation) +- [ModelZoo Homepage](#modelzoo-homepage) + + +# [ResNet Description](#contents) +## Description +ResNet (residual neural network) was proposed by Kaiming He and other four Chinese of Microsoft Research Institute. Through the use of ResNet unit, it successfully trained 152 layers of neural network, and won the championship in ilsvrc2015. The error rate on top 5 was 3.57%, and the parameter quantity was lower than vggnet, so the effect was very outstanding. Traditional convolution network or full connection network will have more or less information loss. At the same time, it will lead to the disappearance or explosion of gradient, which leads to the failure of deep network training. ResNet solves this problem to a certain extent. By passing the input information to the output, the integrity of the information is protected. The whole network only needs to learn the part of the difference between input and output, which simplifies the learning objectives and difficulties.The structure of ResNet can accelerate the training of neural network very quickly, and the accuracy of the model is also greatly improved. At the same time, ResNet is very popular, even can be directly used in the concept net network. + +These are examples of training ResNet50/ResNet101/SE-ResNet50 with CIFAR-10/ImageNet2012 dataset in MindSpore.ResNet50 and ResNet101 can reference [paper 1](https://arxiv.org/pdf/1512.03385.pdf) below, and SE-ResNet50 is a variant of ResNet50 which reference [paper 2](https://arxiv.org/abs/1709.01507) and [paper 3](https://arxiv.org/abs/1812.01187) below, Training SE-ResNet50 for just 24 epochs using 8 Ascend 910, we can reach top-1 accuracy of 75.9%.(Training ResNet101 with dataset CIFAR-10 and SE-ResNet50 with CIFAR-10 is not supported yet.) + +## Paper +1.[paper](https://arxiv.org/pdf/1512.03385.pdf):Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. "Deep Residual Learning for Image Recognition" + +2.[paper](https://arxiv.org/abs/1709.01507):Jie Hu, Li Shen, Samuel Albanie, Gang Sun, Enhua Wu. "Squeeze-and-Excitation Networks" + +3.[paper](https://arxiv.org/abs/1812.01187):Tong He, Zhi Zhang, Hang Zhang, Zhongyue Zhang, Junyuan Xie, Mu Li. "Bag of Tricks for Image Classification with Convolutional Neural Networks" + +# [Model Architecture](#contents) + +The overall network architecture of ResNet is show below: +[Link](https://arxiv.org/pdf/1512.03385.pdf) + +# [Dataset](#contents) + +Dataset used: [CIFAR-10]() +- Dataset size：60,000 32*32 colorful images in 10 classes + - Train：50,000 images + - Test： 10,000 images +- Data format：binary files + - Note：Data will be processed in dataset.py +- Download the dataset, the directory structure is as follows: + +``` +├─cifar-10-batches-bin +│ +└─cifar-10-verify-bin +``` + +Dataset used: [ImageNet2012](http://www.image-net.org/) + +- Dataset size 224*224 colorful images in 1000 classes + - Train：1,281,167 images + - Test： 50,000 images +- Data format：jpeg + - Note：Data will be processed in dataset.py +- Download the dataset, the directory structure is as follows: + + ``` +└─dataset + ├─ilsvrc # train dataset + └─validation_preprocess # evaluate dataset +``` + +# [Features](#contents) + +## Mixed Precision + +The [mixed precision](https://www.mindspore.cn/tutorial/training/en/master/advanced_use/enable_mixed_precision.html) training method accelerates the deep learning neural network training process by using both the single-precision and half-precision data types, and maintains the network precision achieved by the single-precision training at the same time. Mixed precision training can accelerate the computation process, reduce memory usage, and enable a larger model or batch size to be trained on specific hardware. +For FP16 operators, if the input data type is FP32, the backend of MindSpore will automatically handle it with reduced precision. Users could check the reduced-precision operators by enabling INFO log and then searching ‘reduce precision’. + +# [Environment Requirements](#contents) + +- Hardware（Ascend/GPU） + - Prepare hardware environment with Ascend or GPU processor. If you want to try Ascend , please send the [application form](https://obs-9be7.obs.cn-east-2.myhuaweicloud.com/file/other/Ascend%20Model%20Zoo%E4%BD%93%E9%AA%8C%E8%B5%84%E6%BA%90%E7%94%B3%E8%AF%B7%E8%A1%A8.docx) to ascend@huawei.com. Once approved, you can get the resources. +- Framework + - [MindSpore](https://www.mindspore.cn/install/en) +- For more information, please check the resources below： + - [MindSpore Tutorials](https://www.mindspore.cn/tutorial/training/en/master/index.html) + - [MindSpore Python API](https://www.mindspore.cn/doc/api_python/en/master/index.html) + + + +# [Quick Start](#contents) + +After installing MindSpore via the official website, you can start training and evaluation as follows: + +- Runing on Ascend +``` +# distributed training +Usage: sh run_distribute_train.sh [resnet50|resnet101|se-resnet50] [cifar10|imagenet2012] [RANK_TABLE_FILE] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional) + +# standalone training +Usage: sh run_standalone_train.sh [resnet50|resnet101|se-resnet50] [cifar10|imagenet2012] [DATASET_PATH] +[PRETRAINED_CKPT_PATH](optional) + +# run evaluation example +Usage: sh run_eval.sh [resnet50|resnet101|se-resnet50] [cifar10|imagenet2012] [DATASET_PATH] [CHECKPOINT_PATH] +``` + +- Runing on GPU +``` +# distributed training example +sh run_distribute_train_gpu.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional) + +# standalone training example +sh run_standalone_train_gpu.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional) + +# infer example +sh run_eval_gpu.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [CHECKPOINT_PATH] +``` + +# [Script Description](#contents) + +## [Script and Sample Code](#contents) + +```shell +. +└──resnet + ├── README.md + ├── script + ├── run_distribute_train.sh # launch ascend distributed training(8 pcs) + ├── run_parameter_server_train.sh # launch ascend parameter server training(8 pcs) + ├── run_eval.sh # launch ascend evaluation + ├── run_standalone_train.sh # launch ascend standalone training(1 pcs) + ├── run_distribute_train_gpu.sh # launch gpu distributed training(8 pcs) + ├── run_parameter_server_train_gpu.sh # launch gpu parameter server training(8 pcs) + ├── run_eval_gpu.sh # launch gpu evaluation + └── run_standalone_train_gpu.sh # launch gpu standalone training(1 pcs) + ├── src + ├── config.py # parameter configuration + ├── dataset.py # data preprocessing + ├── crossentropy.py # loss definition for ImageNet2012 dataset + ├── lr_generator.py # generate learning rate for each step + └── resnet.py # resnet backbone, including resnet50 and resnet101 and se-resnet50 + ├── eval.py # eval net + └── train.py # train net +``` + +## [Script Parameters](#contents) + +Parameters for both training and evaluation can be set in config.py. + +- Config for ResNet50, CIFAR-10 dataset + +``` +"class_num": 10, # dataset class num +"batch_size": 32, # batch size of input tensor +"loss_scale": 1024, # loss scale +"momentum": 0.9, # momentum +"weight_decay": 1e-4, # weight decay +"epoch_size": 90, # only valid for taining, which is always 1 for inference +"pretrain_epoch_size": 0, # epoch size that model has been trained before loading pretrained checkpoint, actual training epoch size is equal to epoch_size minus pretrain_epoch_size +"save_checkpoint": True, # whether save checkpoint or not +"save_checkpoint_epochs": 5, # the epoch interval between two checkpoints. By default, the last checkpoint will be saved after the last step +"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint +"save_checkpoint_path": "./", # path to save checkpoint +"warmup_epochs": 5, # number of warmup epoch +"lr_decay_mode": "poly" # decay mode can be selected in steps, ploy and default +"lr_init": 0.01, # initial learning rate +"lr_end": 0.00001, # final learning rate +"lr_max": 0.1, # maximum learning rate +``` + +- Config for ResNet50, ImageNet2012 dataset + +``` +"class_num": 1001, # dataset class number +"batch_size": 32, # batch size of input tensor +"loss_scale": 1024, # loss scale +"momentum": 0.9, # momentum optimizer +"weight_decay": 1e-4, # weight decay +"epoch_size": 90, # only valid for taining, which is always 1 for inference +"pretrain_epoch_size": 0, # epoch size that model has been trained before loading pretrained checkpoint, actual training epoch size is equal to epoch_size minus pretrain_epoch_size +"save_checkpoint": True, # whether save checkpoint or not +"save_checkpoint_epochs": 5, # the epoch interval between two checkpoints. By default, the last checkpoint will be saved after the last epoch +"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint +"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path +"warmup_epochs": 0, # number of warmup epoch +"lr_decay_mode": "Linear", # decay mode for generating learning rate +"label_smooth": True, # label smooth +"label_smooth_factor": 0.1, # label smooth factor +"lr_init": 0, # initial learning rate +"lr_max": 0.1, # maximum learning rate +"lr_end": 0.0, # minimum learning rate +``` + +- Config for ResNet101, ImageNet2012 dataset + +``` +"class_num": 1001, # dataset class number +"batch_size": 32, # batch size of input tensor +"loss_scale": 1024, # loss scale +"momentum": 0.9, # momentum optimizer +"weight_decay": 1e-4, # weight decay +"epoch_size": 120, # epoch size for training +"pretrain_epoch_size": 0, # epoch size that model has been trained before loading pretrained checkpoint, actual training epoch size is equal to epoch_size minus pretrain_epoch_size +"save_checkpoint": True, # whether save checkpoint or not +"save_checkpoint_epochs": 5, # the epoch interval between two checkpoints. By default, the last checkpoint will be saved after the last epoch +"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint +"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path +"warmup_epochs": 0, # number of warmup epoch +"lr_decay_mode": "cosine" # decay mode for generating learning rate +"label_smooth": 1, # label_smooth +"label_smooth_factor": 0.1, # label_smooth_factor +"lr": 0.1 # base learning rate +``` + +- Config for SE-ResNet50, ImageNet2012 dataset + +``` +"class_num": 1001, # dataset class number +"batch_size": 32, # batch size of input tensor +"loss_scale": 1024, # loss scale +"momentum": 0.9, # momentum optimizer +"weight_decay": 1e-4, # weight decay +"epoch_size": 28 , # epoch size for creating learning rate +"train_epoch_size": 24 # actual train epoch size +"pretrain_epoch_size": 0, # epoch size that model has been trained before loading pretrained checkpoint, actual training epoch size is equal to epoch_size minus pretrain_epoch_size +"save_checkpoint": True, # whether save checkpoint or not +"save_checkpoint_epochs": 4, # the epoch interval between two checkpoints. By default, the last checkpoint will be saved after the last epoch +"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint +"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path +"warmup_epochs": 3, # number of warmup epoch +"lr_decay_mode": "cosine" # decay mode for generating learning rate +"label_smooth": True, # label_smooth +"label_smooth_factor": 0.1, # label_smooth_factor +"lr_init": 0.0, # initial learning rate +"lr_max": 0.3, # maximum learning rate +"lr_end": 0.0001, # end learning rate +``` + +## [Training Process](#contents) + +### Usage +#### Running on Ascend +``` +# distributed training +Usage: sh run_distribute_train.sh [resnet50|resnet101|se-resnet50] [cifar10|imagenet2012] [RANK_TABLE_FILE] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional) + +# standalone training +Usage: sh run_standalone_train.sh [resnet50|resnet101|se-resnet50] [cifar10|imagenet2012] [DATASET_PATH] +[PRETRAINED_CKPT_PATH](optional) + +# run evaluation example +Usage: sh run_eval.sh [resnet50|resnet101|se-resnet50] [cifar10|imagenet2012] [DATASET_PATH] [CHECKPOINT_PATH] + +``` +For distributed training, a hccl configuration file with JSON format needs to be created in advance. + +Please follow the instructions in the link [hccn_tools](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/utils/hccl_tools). + +Training result will be stored in the example path, whose folder name begins with "train" or "train_parallel". Under this, you can find checkpoint file together with result like the followings in log. + +#### Running on GPU + +``` +# distributed training example +sh run_distribute_train_gpu.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional) + +# standalone training example +sh run_standalone_train_gpu.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional) + +# infer example +sh run_eval_gpu.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [CHECKPOINT_PATH] +``` + +#### Running parameter server mode training + +- Parameter server training Ascend example + +``` +sh run_parameter_server_train.sh [resnet50|resnet101] [cifar10|imagenet2012] [RANK_TABLE_FILE] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional) +``` + +- Parameter server training GPU example +``` +sh run_parameter_server_train_gpu.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional) +``` + +### Result + +- Training ResNet50 with CIFAR-10 dataset + +``` +# distribute training result(8 pcs) +epoch: 1 step: 195, loss is 1.9601055 +epoch: 2 step: 195, loss is 1.8555021 +epoch: 3 step: 195, loss is 1.6707983 +epoch: 4 step: 195, loss is 1.8162166 +epoch: 5 step: 195, loss is 1.393667 +... +``` + +- Training ResNet50 with ImageNet2012 dataset + +``` +# distribute training result(8 pcs) +epoch: 1 step: 5004, loss is 4.8995576 +epoch: 2 step: 5004, loss is 3.9235563 +epoch: 3 step: 5004, loss is 3.833077 +epoch: 4 step: 5004, loss is 3.2795618 +epoch: 5 step: 5004, loss is 3.1978393 +... +``` + +- Training ResNet101 with ImageNet2012 dataset + +``` +# distribute training result(8p) +epoch: 1 step: 5004, loss is 4.805483 +epoch: 2 step: 5004, loss is 3.2121816 +epoch: 3 step: 5004, loss is 3.429647 +epoch: 4 step: 5004, loss is 3.3667371 +epoch: 5 step: 5004, loss is 3.1718972 +... +epoch: 67 step: 5004, loss is 2.2768745 +epoch: 68 step: 5004, loss is 1.7223864 +epoch: 69 step: 5004, loss is 2.0665488 +epoch: 70 step: 5004, loss is 1.8717369 +... +``` +- Training SE-ResNet50 with ImageNet2012 dataset + +``` +# distribute training result(8 pcs) +epoch: 1 step: 5004, loss is 5.1779146 +epoch: 2 step: 5004, loss is 4.139395 +epoch: 3 step: 5004, loss is 3.9240637 +epoch: 4 step: 5004, loss is 3.5011306 +epoch: 5 step: 5004, loss is 3.3501816 +... +``` + +## [Evaluation Process](#contents) + +### Usage + +#### Running on Ascend +``` +# evaluation +Usage: sh run_eval.sh [resnet50|resnet101|se-resnet50] [cifar10|imagenet2012] [DATASET_PATH] [CHECKPOINT_PATH] +``` + +``` +# evaluation example +sh run_eval.sh resnet50 cifar10 ~/cifar10-10-verify-bin ~/resnet50_cifar10/train_parallel0/resnet-90_195.ckpt +``` + +> checkpoint can be produced in training process. + +#### Running on GPU +``` +sh run_eval_gpu.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [CHECKPOINT_PATH] +``` + +### Result + +Evaluation result will be stored in the example path, whose folder name is "eval". Under this, you can find result like the followings in log. + +- Evaluating ResNet50 with CIFAR-10 dataset + +``` +result: {'acc': 0.91446314102564111} ckpt=~/resnet50_cifar10/train_parallel0/resnet-90_195.ckpt +``` + +- Evaluating ResNet50 with ImageNet2012 dataset + +``` +result: {'acc': 0.7671054737516005} ckpt=train_parallel0/resnet-90_5004.ckpt +``` + +- Evaluating ResNet101 with ImageNet2012 dataset + +``` +result: {'top_5_accuracy': 0.9429417413572343, 'top_1_accuracy': 0.7853513124199744} ckpt=train_parallel0/resnet-120_5004.ckpt +``` + +- Evaluating SE-ResNet50 with ImageNet2012 dataset + +``` +result: {'top_5_accuracy': 0.9342589628681178, 'top_1_accuracy': 0.768065781049936} ckpt=train_parallel0/resnet-24_5004.ckpt + +``` + +# [Model Description](#contents) +## [Performance](#contents) + +### Evaluation Performance + +#### ResNet50 on CIFAR-10 +| Parameters | Ascend 910 | GPU | +| -------------------------- | -------------------------------------- |---------------------------------- | +| Model Version | ResNet50-v1.5 |ResNet50-v1.5| +| Resource | Ascend 910，CPU 2.60GHz 56cores，Memory 314G | GPU(Tesla V100 SXM2)，CPU 2.1GHz 24cores，Memory 128G +| uploaded Date | 04/01/2020 (month/day/year) | 08/01/2020 (month/day/year) +| MindSpore Version | 0.1.0-alpha |0.6.0-alpha | +| Dataset | CIFAR-10 | CIFAR-10 +| Training Parameters | epoch=90, steps per epoch=195, batch_size = 32 |epoch=90, steps per epoch=195, batch_size = 32 | +| Optimizer | Momentum |Momentum| +| Loss Function | Softmax Cross Entropy |Softmax Cross Entropy | +| outputs | probability | probability | +| Loss | 0.000356 | 0.000716 | +| Speed | 18.4ms/step（8pcs） |69ms/step（8pcs）| +| Total time | 6 mins | 20.2 mins| +| Parameters (M) | 25.5 | 25.5 | +| Checkpoint for Fine tuning | 179.7M (.ckpt file) |179.7M (.ckpt file)| +| Scripts | [Link](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/cv/resnet) | [Link](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/cv/resnet) | + +#### ResNet50 on ImageNet2012 +| Parameters | Ascend 910 | GPU | +| -------------------------- | -------------------------------------- |---------------------------------- | +| Model Version | ResNet50-v1.5 |ResNet50-v1.5| +| Resource | Ascend 910，CPU 2.60GHz 56cores，Memory 314G | GPU(Tesla V100 SXM2)，CPU 2.1GHz 24cores，Memory 128G +| uploaded Date | 04/01/2020 (month/day/year) ； | 08/01/2020 (month/day/year) +| MindSpore Version | 0.1.0-alpha |0.6.0-alpha | +| Dataset | ImageNet2012 | ImageNet2012| +| Training Parameters | epoch=90, steps per epoch=5004, batch_size = 32 |epoch=90, steps per epoch=5004, batch_size = 32 | +| Optimizer | Momentum |Momentum| +| Loss Function | Softmax Cross Entropy |Softmax Cross Entropy | +| outputs | probability | probability | +| Loss | 1.8464266 | 1.9023 | +| Speed | 18.4ms/step（8pcs） |67.1ms/step（8pcs）| +| Total time | 139 mins | 500 mins| +| Parameters (M) | 25.5 | 25.5 | +| Checkpoint for Fine tuning | 197M (.ckpt file) |197M (.ckpt file) | +| Scripts | [Link](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/cv/resnet) | [Link](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/cv/resnet) | + +#### ResNet101 on ImageNet2012 +| Parameters | Ascend 910 | GPU | +| -------------------------- | -------------------------------------- |---------------------------------- | +| Model Version | ResNet101 |ResNet101| +| Resource | Ascend 910，CPU 2.60GHz 56cores，Memory 314G | GPU(Tesla V100 SXM2)，CPU 2.1GHz 24cores，Memory 128G +| uploaded Date | 04/01/2020 (month/day/year) | 08/01/2020 (month/day/year) +| MindSpore Version | 0.1.0-alpha |0.6.0-alpha | +| Dataset | ImageNet2012 | ImageNet2012| +| Training Parameters | epoch=120, steps per epoch=5004, batch_size = 32 |epoch=120, steps per epoch=5004, batch_size = 32 | +| Optimizer | Momentum |Momentum| +| Loss Function | Softmax Cross Entropy |Softmax Cross Entropy | +| outputs | probability | probability | +| Loss | 1.6453942 | 1.7023412 | +| Speed | 30.3ms/step（8pcs） |108.6ms/step（8pcs）| +| Total time | 301 mins | 1100 mins| +| Parameters (M) | 44.6 | 44.6 | +| Checkpoint for Fine tuning | 343M (.ckpt file) |343M (.ckpt file) | +| Scripts | [Link](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/cv/resnet) | [Link](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/cv/resnet) | + +#### SE-ResNet50 on ImageNet2012 + +| Parameters | Ascend 910 +| -------------------------- | ------------------------------------------------------------------------ | +| Model Version | SE-ResNet50 | +| Resource | Ascend 910，CPU 2.60GHz 56cores，Memory 314G | +| uploaded Date | 08/16/2020 (month/day/year) ； | +| MindSpore Version | 0.7.0-alpha | +| Dataset | ImageNet2012 | +| Training Parameters | epoch=24, steps per epoch=5004, batch_size = 32 | +| Optimizer | Momentum | +| Loss Function | Softmax Cross Entropy | +| outputs | probability | +| Loss | 1.754404 | +| Speed | 24.6ms/step（8pcs） | +| Total time | 49.3 mins | +| Parameters (M) | 25.5 | +| Checkpoint for Fine tuning | 215.9M (.ckpt file) | +| Scripts | [Link](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/cv/resnet) | + +# [Description of Random Situation](#contents) + +In dataset.py, we set the seed inside “create_dataset" function. We also use random seed in train.py. + + +# [ModelZoo Homepage](#contents) + Please check the official [homepage](https://gitee.com/mindspore/mindspore/tree/master/model_zoo). \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/eval.py b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/eval.py new file mode 100644 index 0000000..3928ea3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/eval.py @@ -0,0 +1,89 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""train resnet.""" +import os +import argparse +from mindspore import context +from mindspore.common import set_seed +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.train.model import Model +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from src.CrossEntropySmooth import CrossEntropySmooth + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--net', type=str, default=None, help='Resnet Model, either resnet50 or resnet101') +parser.add_argument('--dataset', type=str, default=None, help='Dataset, either cifar10 or imagenet2012') + +parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') +args_opt = parser.parse_args() + +set_seed(1) + +if args_opt.net == "resnet50": + from src.resnet import resnet50 as resnet + if args_opt.dataset == "cifar10": + from src.config import config1 as config + from src.dataset import create_dataset1 as create_dataset + else: + from src.config import config2 as config + from src.dataset import create_dataset2 as create_dataset +elif args_opt.net == "resnet101": + from src.resnet import resnet101 as resnet + from src.config import config3 as config + from src.dataset import create_dataset3 as create_dataset +else: + from src.resnet import se_resnet50 as resnet + from src.config import config4 as config + from src.dataset import create_dataset4 as create_dataset + +if __name__ == '__main__': + target = args_opt.device_target + + # init context + context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False) + if target != "GPU": + device_id = int(os.getenv('DEVICE_ID')) + context.set_context(device_id=device_id) + + # create dataset + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size, + target=target) + step_size = dataset.get_dataset_size() + + # define net + net = resnet(class_num=config.class_num) + + # load checkpoint + param_dict = load_checkpoint(args_opt.checkpoint_path) + load_param_into_net(net, param_dict) + net.set_train(False) + + # define loss, model + if args_opt.dataset == "imagenet2012": + if not config.use_label_smooth: + config.label_smooth_factor = 0.0 + loss = CrossEntropySmooth(sparse=True, reduction='mean', + smooth_factor=config.label_smooth_factor, num_classes=config.class_num) + else: + loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') + + # define model + model = Model(net, loss_fn=loss, metrics={'top_1_accuracy', 'top_5_accuracy'}) + + # eval model + res = model.eval(dataset) + print("result:", res, "ckpt=", args_opt.checkpoint_path) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/mindspore_hub_conf.py b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/mindspore_hub_conf.py new file mode 100644 index 0000000..e9ffe87 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/mindspore_hub_conf.py @@ -0,0 +1,25 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""hub config.""" +from src.resnet import resnet50, resnet101, se_resnet50 + +def create_network(name, *args, **kwargs): + if name == 'resnet50': + return resnet50(*args, **kwargs) + if name == 'resnet101': + return resnet101(*args, **kwargs) + if name == 'se_resnet50': + return se_resnet50(*args, **kwargs) + raise NotImplementedError(f"{name} is not implemented in the repo") diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/CrossEntropySmooth.py b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/CrossEntropySmooth.py new file mode 100644 index 0000000..bf38c6e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/CrossEntropySmooth.py @@ -0,0 +1,38 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""define loss function for network""" +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common import dtype as mstype +from mindspore.nn.loss.loss import _Loss +from mindspore.ops import functional as F +from mindspore.ops import operations as P + + +class CrossEntropySmooth(_Loss): + """CrossEntropy""" + def __init__(self, sparse=True, reduction='mean', smooth_factor=0., num_classes=1000): + super(CrossEntropySmooth, self).__init__() + self.onehot = P.OneHot() + self.sparse = sparse + self.on_value = Tensor(1.0 - smooth_factor, mstype.float32) + self.off_value = Tensor(1.0 * smooth_factor / (num_classes - 1), mstype.float32) + self.ce = nn.SoftmaxCrossEntropyWithLogits(reduction=reduction) + + def construct(self, logit, label): + if self.sparse: + label = self.onehot(label, F.shape(logit)[1], self.on_value, self.off_value) + loss = self.ce(logit, label) + return loss diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/config.py b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/config.py new file mode 100644 index 0000000..83e2fc4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/config.py @@ -0,0 +1,106 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +network config setting, will be used in train.py and eval.py +""" +from easydict import EasyDict as ed + + +# config for resnet50, imagenet2012 +config2 = ed({ + 'class_num': 1001, + 'batch_size': 256, + 'loss_scale': 1024, + 'momentum': 0.9, + 'weight_decay': 1e-4, + 'epoch_size': 5, + 'pretrain_epoch_size': 0, + 'save_checkpoint': True, + 'save_checkpoint_epochs': 5, + 'keep_checkpoint_max': 10, + 'save_checkpoint_path': './', + 'warmup_epochs': 0, + 'lr_decay_mode': 'linear', + 'use_label_smooth': True, + 'label_smooth_factor': 0.1, + 'lr_init': 0, + 'lr_max': 0.8, + 'lr_end': 0.0 +}) + + +# config for resent50, cifar10 +config1 = ed({ + 'class_num': 10, + 'batch_size': 32, + 'loss_scale': 1024, + 'momentum': 0.9, + 'weight_decay': 1e-4, + 'epoch_size': 90, + 'pretrain_epoch_size': 0, + 'save_checkpoint': True, + 'save_checkpoint_epochs': 5, + 'keep_checkpoint_max': 10, + 'save_checkpoint_path': './', + 'warmup_epochs': 5, + 'lr_decay_mode': 'poly', + 'lr_init': 0.01, + 'lr_end': 0.00001, + 'lr_max': 0.1 +}) + + +# config for resent101, imagenet2012 +config3 = ed({ + 'class_num': 1001, + 'batch_size': 32, + 'loss_scale': 1024, + 'momentum': 0.9, + 'weight_decay': 1e-4, + 'epoch_size': 120, + 'pretrain_epoch_size': 0, + 'save_checkpoint': True, + 'save_checkpoint_epochs': 5, + 'keep_checkpoint_max': 10, + 'save_checkpoint_path': './', + 'warmup_epochs': 0, + 'lr_decay_mode': 'cosine', + 'use_label_smooth': True, + 'label_smooth_factor': 0.1, + 'lr': 0.1 +}) + +# config for se-resnet50, imagenet2012 +config4 = ed({ + 'class_num': 1001, + 'batch_size': 32, + 'loss_scale': 1024, + 'momentum': 0.9, + 'weight_decay': 1e-4, + 'epoch_size': 28, + 'train_epoch_size': 24, + 'pretrain_epoch_size': 0, + 'save_checkpoint': True, + 'save_checkpoint_epochs': 4, + 'keep_checkpoint_max': 10, + 'save_checkpoint_path': './', + 'warmup_epochs': 3, + 'lr_decay_mode': 'cosine', + 'use_label_smooth': True, + 'label_smooth_factor': 0.1, + 'lr_init': 0.0, + 'lr_max': 0.3, + 'lr_end': 0.0001 +}) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/dataset.py b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/dataset.py new file mode 100644 index 0000000..cfc503a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/dataset.py @@ -0,0 +1,263 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +create train or eval dataset. +""" +import os +import mindspore.common.dtype as mstype +import mindspore.dataset.engine as de +import mindspore.dataset.vision.c_transforms as C +import mindspore.dataset.transforms.c_transforms as C2 +from mindspore.communication.management import init, get_rank, get_group_size + + +def create_dataset1(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): + """ + create a train or evaluate cifar10 dataset for resnet50 + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + target(str): the device target. Default: Ascend + + Returns: + dataset + """ + if target == "Ascend": + device_num, rank_id = _get_rank_info() + else: + init() + rank_id = get_rank() + device_num = get_group_size() + + if device_num == 1: + ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True) + else: + ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) + + # define map operations + trans = [] + if do_train: + trans += [ + C.RandomCrop((32, 32), (4, 4, 4, 4)), + C.RandomHorizontalFlip(prob=0.5) + ] + + trans += [ + C.Resize((224, 224)), + C.Rescale(1.0 / 255.0, 0.0), + C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]), + C.HWC2CHW() + ] + + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) + ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=8) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds + + +def create_dataset2(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): + """ + create a train or eval imagenet2012 dataset for resnet50 + + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + target(str): the device target. Default: Ascend + + Returns: + dataset + """ + if target == "Ascend": + device_num, rank_id = _get_rank_info() + else: + init() + rank_id = get_rank() + device_num = get_group_size() + + if device_num == 1: + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) + else: + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) + + image_size = 224 + mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] + std = [0.229 * 255, 0.224 * 255, 0.225 * 255] + + # define map operations + if do_train: + trans = [ + C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), + C.RandomHorizontalFlip(prob=0.5), + C.Normalize(mean=mean, std=std), + C.HWC2CHW() + ] + else: + trans = [ + C.Decode(), + C.Resize(256), + C.CenterCrop(image_size), + C.Normalize(mean=mean, std=std), + C.HWC2CHW() + ] + + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=8) + ds = ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds + + +def create_dataset3(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): + """ + create a train or eval imagenet2012 dataset for resnet101 + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + + Returns: + dataset + """ + device_num, rank_id = _get_rank_info() + + if device_num == 1: + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) + else: + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) + image_size = 224 + mean = [0.475 * 255, 0.451 * 255, 0.392 * 255] + std = [0.275 * 255, 0.267 * 255, 0.278 * 255] + + # define map operations + if do_train: + trans = [ + C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), + C.RandomHorizontalFlip(rank_id / (rank_id + 1)), + C.Normalize(mean=mean, std=std), + C.HWC2CHW() + ] + else: + trans = [ + C.Decode(), + C.Resize(256), + C.CenterCrop(image_size), + C.Normalize(mean=mean, std=std), + C.HWC2CHW() + ] + + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=8) + ds = ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds + + +def create_dataset4(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): + """ + create a train or eval imagenet2012 dataset for se-resnet50 + + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + target(str): the device target. Default: Ascend + + Returns: + dataset + """ + if target == "Ascend": + device_num, rank_id = _get_rank_info() + if device_num == 1: + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=12, shuffle=True) + else: + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=12, shuffle=True, + num_shards=device_num, shard_id=rank_id) + image_size = 224 + mean = [123.68, 116.78, 103.94] + std = [1.0, 1.0, 1.0] + + # define map operations + if do_train: + trans = [ + C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), + C.RandomHorizontalFlip(prob=0.5), + C.Normalize(mean=mean, std=std), + C.HWC2CHW() + ] + else: + trans = [ + C.Decode(), + C.Resize(292), + C.CenterCrop(256), + C.Normalize(mean=mean, std=std), + C.HWC2CHW() + ] + + type_cast_op = C2.TypeCast(mstype.int32) + ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=12) + ds = ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=12) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds + + +def _get_rank_info(): + """ + get rank size and rank id + """ + rank_size = int(os.environ.get("RANK_SIZE", 1)) + + if rank_size > 1: + rank_size = get_group_size() + rank_id = get_rank() + else: + rank_size = 1 + rank_id = 0 + + return rank_size, rank_id diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/lr_generator.py b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/lr_generator.py new file mode 100644 index 0000000..ed9e81a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/lr_generator.py @@ -0,0 +1,205 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""learning rate generator""" +import math +import numpy as np + + +def _generate_steps_lr(lr_init, lr_max, total_steps, warmup_steps): + """ + Applies three steps decay to generate learning rate array. + + Args: + lr_init(float): init learning rate. + lr_max(float): max learning rate. + total_steps(int): all steps in training. + warmup_steps(int): all steps in warmup epochs. + + Returns: + np.array, learning rate array. + """ + decay_epoch_index = [0.3 * total_steps, 0.6 * total_steps, 0.8 * total_steps] + lr_each_step = [] + for i in range(total_steps): + if i < warmup_steps: + lr = lr_init + (lr_max - lr_init) * i / warmup_steps + else: + if i < decay_epoch_index[0]: + lr = lr_max + elif i < decay_epoch_index[1]: + lr = lr_max * 0.1 + elif i < decay_epoch_index[2]: + lr = lr_max * 0.01 + else: + lr = lr_max * 0.001 + lr_each_step.append(lr) + return lr_each_step + + +def _generate_poly_lr(lr_init, lr_end, lr_max, total_steps, warmup_steps): + """ + Applies polynomial decay to generate learning rate array. + + Args: + lr_init(float): init learning rate. + lr_end(float): end learning rate + lr_max(float): max learning rate. + total_steps(int): all steps in training. + warmup_steps(int): all steps in warmup epochs. + + Returns: + np.array, learning rate array. + """ + lr_each_step = [] + if warmup_steps != 0: + inc_each_step = (float(lr_max) - float(lr_init)) / float(warmup_steps) + else: + inc_each_step = 0 + for i in range(total_steps): + if i < warmup_steps: + lr = float(lr_init) + inc_each_step * float(i) + else: + base = (1.0 - (float(i) - float(warmup_steps)) / (float(total_steps) - float(warmup_steps))) + lr = float(lr_max) * base * base + if lr < 0.0: + lr = 0.0 + lr_each_step.append(lr) + return lr_each_step + + +def _generate_cosine_lr(lr_init, lr_end, lr_max, total_steps, warmup_steps): + """ + Applies cosine decay to generate learning rate array. + + Args: + lr_init(float): init learning rate. + lr_end(float): end learning rate + lr_max(float): max learning rate. + total_steps(int): all steps in training. + warmup_steps(int): all steps in warmup epochs. + + Returns: + np.array, learning rate array. + """ + decay_steps = total_steps - warmup_steps + lr_each_step = [] + for i in range(total_steps): + if i < warmup_steps: + lr_inc = (float(lr_max) - float(lr_init)) / float(warmup_steps) + lr = float(lr_init) + lr_inc * (i + 1) + else: + cosine_decay = 0.5 * (1 + math.cos(math.pi * (i-warmup_steps) / decay_steps)) + lr = (lr_max-lr_end)*cosine_decay + lr_end + lr_each_step.append(lr) + return lr_each_step + + +def _generate_liner_lr(lr_init, lr_end, lr_max, total_steps, warmup_steps): + """ + Applies liner decay to generate learning rate array. + + Args: + lr_init(float): init learning rate. + lr_end(float): end learning rate + lr_max(float): max learning rate. + total_steps(int): all steps in training. + warmup_steps(int): all steps in warmup epochs. + + Returns: + np.array, learning rate array. + """ + lr_each_step = [] + for i in range(total_steps): + if i < warmup_steps: + lr = lr_init + (lr_max - lr_init) * i / warmup_steps + else: + lr = lr_max - (lr_max - lr_end) * (i - warmup_steps) / (total_steps - warmup_steps) + lr_each_step.append(lr) + return lr_each_step + + + +def get_lr(lr_init, lr_end, lr_max, warmup_epochs, total_epochs, steps_per_epoch, lr_decay_mode): + """ + generate learning rate array + + Args: + lr_init(float): init learning rate + lr_end(float): end learning rate + lr_max(float): max learning rate + warmup_epochs(int): number of warmup epochs + total_epochs(int): total epoch of training + steps_per_epoch(int): steps of one epoch + lr_decay_mode(string): learning rate decay mode, including steps, poly, cosine or liner(default) + + Returns: + np.array, learning rate array + """ + lr_each_step = [] + total_steps = steps_per_epoch * total_epochs + warmup_steps = steps_per_epoch * warmup_epochs + + if lr_decay_mode == 'steps': + lr_each_step = _generate_steps_lr(lr_init, lr_max, total_steps, warmup_steps) + elif lr_decay_mode == 'poly': + lr_each_step = _generate_poly_lr(lr_init, lr_end, lr_max, total_steps, warmup_steps) + elif lr_decay_mode == 'cosine': + lr_each_step = _generate_cosine_lr(lr_init, lr_end, lr_max, total_steps, warmup_steps) + else: + lr_each_step = _generate_liner_lr(lr_init, lr_end, lr_max, total_steps, warmup_steps) + + lr_each_step = np.array(lr_each_step).astype(np.float32) + return lr_each_step + + +def linear_warmup_lr(current_step, warmup_steps, base_lr, init_lr): + lr_inc = (float(base_lr) - float(init_lr)) / float(warmup_steps) + lr = float(init_lr) + lr_inc * current_step + return lr + + +def warmup_cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch=120, global_step=0): + """ + generate learning rate array with cosine + + Args: + lr(float): base learning rate + steps_per_epoch(int): steps size of one epoch + warmup_epochs(int): number of warmup epochs + max_epoch(int): total epochs of training + global_step(int): the current start index of lr array + Returns: + np.array, learning rate array + """ + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch) + warmup_steps = int(warmup_epochs * steps_per_epoch) + decay_steps = total_steps - warmup_steps + + lr_each_step = [] + for i in range(total_steps): + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + linear_decay = (total_steps - i) / decay_steps + cosine_decay = 0.5 * (1 + math.cos(math.pi * 2 * 0.47 * i / decay_steps)) + decayed = linear_decay * cosine_decay + 0.00001 + lr = base_lr * decayed + lr_each_step.append(lr) + + lr_each_step = np.array(lr_each_step).astype(np.float32) + learning_rate = lr_each_step[global_step:] + return learning_rate diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/resnet.py b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/resnet.py new file mode 100644 index 0000000..73ef4b5 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/src/resnet.py @@ -0,0 +1,393 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""ResNet.""" +import numpy as np +import mindspore.nn as nn +import mindspore.common.dtype as mstype +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.common.tensor import Tensor +from scipy.stats import truncnorm + +def _conv_variance_scaling_initializer(in_channel, out_channel, kernel_size): + fan_in = in_channel * kernel_size * kernel_size + scale = 1.0 + scale /= max(1., fan_in) + stddev = (scale ** 0.5) / .87962566103423978 + mu, sigma = 0, stddev + weight = truncnorm(-2, 2, loc=mu, scale=sigma).rvs(out_channel * in_channel * kernel_size * kernel_size) + weight = np.reshape(weight, (out_channel, in_channel, kernel_size, kernel_size)) + return Tensor(weight, dtype=mstype.float32) + +def _weight_variable(shape, factor=0.01): + init_value = np.random.randn(*shape).astype(np.float32) * factor + return Tensor(init_value) + + +def _conv3x3(in_channel, out_channel, stride=1, use_se=False): + if use_se: + weight = _conv_variance_scaling_initializer(in_channel, out_channel, kernel_size=3) + else: + weight_shape = (out_channel, in_channel, 3, 3) + weight = _weight_variable(weight_shape) + return nn.Conv2d(in_channel, out_channel, + kernel_size=3, stride=stride, padding=0, pad_mode='same', weight_init=weight) + + +def _conv1x1(in_channel, out_channel, stride=1, use_se=False): + if use_se: + weight = _conv_variance_scaling_initializer(in_channel, out_channel, kernel_size=1) + else: + weight_shape = (out_channel, in_channel, 1, 1) + weight = _weight_variable(weight_shape) + return nn.Conv2d(in_channel, out_channel, + kernel_size=1, stride=stride, padding=0, pad_mode='same', weight_init=weight) + + +def _conv7x7(in_channel, out_channel, stride=1, use_se=False): + if use_se: + weight = _conv_variance_scaling_initializer(in_channel, out_channel, kernel_size=7) + else: + weight_shape = (out_channel, in_channel, 7, 7) + weight = _weight_variable(weight_shape) + return nn.Conv2d(in_channel, out_channel, + kernel_size=7, stride=stride, padding=0, pad_mode='same', weight_init=weight) + + +def _bn(channel): + return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, + gamma_init=1, beta_init=0, moving_mean_init=0, moving_var_init=1) + + +def _bn_last(channel): + return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, + gamma_init=0, beta_init=0, moving_mean_init=0, moving_var_init=1) + + +def _fc(in_channel, out_channel, use_se=False): + if use_se: + weight = np.random.normal(loc=0, scale=0.01, size=out_channel*in_channel) + weight = Tensor(np.reshape(weight, (out_channel, in_channel)), dtype=mstype.float32) + else: + weight_shape = (out_channel, in_channel) + weight = _weight_variable(weight_shape) + return nn.Dense(in_channel, out_channel, has_bias=True, weight_init=weight, bias_init=0) + + +class ResidualBlock(nn.Cell): + """ + ResNet V1 residual block definition. + + Args: + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the first convolutional layer. Default: 1. + use_se (bool): enable SE-ResNet50 net. Default: False. + se_block(bool): use se block in SE-ResNet50 net. Default: False. + + Returns: + Tensor, output tensor. + + Examples: + >>> ResidualBlock(3, 256, stride=2) + """ + expansion = 4 + + def __init__(self, + in_channel, + out_channel, + stride=1, + use_se=False, se_block=False): + super(ResidualBlock, self).__init__() + self.stride = stride + self.use_se = use_se + self.se_block = se_block + channel = out_channel // self.expansion + self.conv1 = _conv1x1(in_channel, channel, stride=1, use_se=self.use_se) + self.bn1 = _bn(channel) + if self.use_se and self.stride != 1: + self.e2 = nn.SequentialCell([_conv3x3(channel, channel, stride=1, use_se=True), _bn(channel), + nn.ReLU(), nn.MaxPool2d(kernel_size=2, stride=2, pad_mode='same')]) + else: + self.conv2 = _conv3x3(channel, channel, stride=stride, use_se=self.use_se) + self.bn2 = _bn(channel) + + self.conv3 = _conv1x1(channel, out_channel, stride=1, use_se=self.use_se) + self.bn3 = _bn_last(out_channel) + if self.se_block: + self.se_global_pool = P.ReduceMean(keep_dims=False) + self.se_dense_0 = _fc(out_channel, int(out_channel/4), use_se=self.use_se) + self.se_dense_1 = _fc(int(out_channel/4), out_channel, use_se=self.use_se) + self.se_sigmoid = nn.Sigmoid() + self.se_mul = P.Mul() + self.relu = nn.ReLU() + + self.down_sample = False + + if stride != 1 or in_channel != out_channel: + self.down_sample = True + self.down_sample_layer = None + + if self.down_sample: + if self.use_se: + if stride == 1: + self.down_sample_layer = nn.SequentialCell([_conv1x1(in_channel, out_channel, + stride, use_se=self.use_se), _bn(out_channel)]) + else: + self.down_sample_layer = nn.SequentialCell([nn.MaxPool2d(kernel_size=2, stride=2, pad_mode='same'), + _conv1x1(in_channel, out_channel, 1, + use_se=self.use_se), _bn(out_channel)]) + else: + self.down_sample_layer = nn.SequentialCell([_conv1x1(in_channel, out_channel, stride, + use_se=self.use_se), _bn(out_channel)]) + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + if self.use_se and self.stride != 1: + out = self.e2(out) + else: + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + out = self.conv3(out) + out = self.bn3(out) + if self.se_block: + out_se = out + out = self.se_global_pool(out, (2, 3)) + out = self.se_dense_0(out) + out = self.relu(out) + out = self.se_dense_1(out) + out = self.se_sigmoid(out) + out = F.reshape(out, F.shape(out) + (1, 1)) + out = self.se_mul(out, out_se) + + if self.down_sample: + identity = self.down_sample_layer(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResNet(nn.Cell): + """ + ResNet architecture. + + Args: + block (Cell): Block for network. + layer_nums (list): Numbers of block in different layers. + in_channels (list): Input channel in each layer. + out_channels (list): Output channel in each layer. + strides (list): Stride size in each layer. + num_classes (int): The number of classes that the training images are belonging to. + use_se (bool): enable SE-ResNet50 net. Default: False. + se_block(bool): use se block in SE-ResNet50 net in layer 3 and layer 4. Default: False. + Returns: + Tensor, output tensor. + + Examples: + >>> ResNet(ResidualBlock, + >>> [3, 4, 6, 3], + >>> [64, 256, 512, 1024], + >>> [256, 512, 1024, 2048], + >>> [1, 2, 2, 2], + >>> 10) + """ + + def __init__(self, + block, + layer_nums, + in_channels, + out_channels, + strides, + num_classes, + use_se=False): + super(ResNet, self).__init__() + + if not len(layer_nums) == len(in_channels) == len(out_channels) == 4: + raise ValueError("the length of layer_num, in_channels, out_channels list must be 4!") + self.use_se = use_se + self.se_block = False + if self.use_se: + self.se_block = True + + if self.use_se: + self.conv1_0 = _conv3x3(3, 32, stride=2, use_se=self.use_se) + self.bn1_0 = _bn(32) + self.conv1_1 = _conv3x3(32, 32, stride=1, use_se=self.use_se) + self.bn1_1 = _bn(32) + self.conv1_2 = _conv3x3(32, 64, stride=1, use_se=self.use_se) + else: + self.conv1 = _conv7x7(3, 64, stride=2) + self.bn1 = _bn(64) + self.relu = P.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode="same") + self.layer1 = self._make_layer(block, + layer_nums[0], + in_channel=in_channels[0], + out_channel=out_channels[0], + stride=strides[0], + use_se=self.use_se) + self.layer2 = self._make_layer(block, + layer_nums[1], + in_channel=in_channels[1], + out_channel=out_channels[1], + stride=strides[1], + use_se=self.use_se) + self.layer3 = self._make_layer(block, + layer_nums[2], + in_channel=in_channels[2], + out_channel=out_channels[2], + stride=strides[2], + use_se=self.use_se, + se_block=self.se_block) + self.layer4 = self._make_layer(block, + layer_nums[3], + in_channel=in_channels[3], + out_channel=out_channels[3], + stride=strides[3], + use_se=self.use_se, + se_block=self.se_block) + + self.mean = P.ReduceMean(keep_dims=True) + self.flatten = nn.Flatten() + self.end_point = _fc(out_channels[3], num_classes, use_se=self.use_se) + + def _make_layer(self, block, layer_num, in_channel, out_channel, stride, use_se=False, se_block=False): + """ + Make stage network of ResNet. + + Args: + block (Cell): Resnet block. + layer_num (int): Layer number. + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the first convolutional layer. + se_block(bool): use se block in SE-ResNet50 net. Default: False. + Returns: + SequentialCell, the output layer. + + Examples: + >>> _make_layer(ResidualBlock, 3, 128, 256, 2) + """ + layers = [] + + resnet_block = block(in_channel, out_channel, stride=stride, use_se=use_se) + layers.append(resnet_block) + if se_block: + for _ in range(1, layer_num - 1): + resnet_block = block(out_channel, out_channel, stride=1, use_se=use_se) + layers.append(resnet_block) + resnet_block = block(out_channel, out_channel, stride=1, use_se=use_se, se_block=se_block) + layers.append(resnet_block) + else: + for _ in range(1, layer_num): + resnet_block = block(out_channel, out_channel, stride=1, use_se=use_se) + layers.append(resnet_block) + return nn.SequentialCell(layers) + + def construct(self, x): + if self.use_se: + x = self.conv1_0(x) + x = self.bn1_0(x) + x = self.relu(x) + x = self.conv1_1(x) + x = self.bn1_1(x) + x = self.relu(x) + x = self.conv1_2(x) + else: + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + c1 = self.maxpool(x) + + c2 = self.layer1(c1) + c3 = self.layer2(c2) + c4 = self.layer3(c3) + c5 = self.layer4(c4) + + out = self.mean(c5, (2, 3)) + out = self.flatten(out) + out = self.end_point(out) + + return out + + +def resnet50(class_num=10): + """ + Get ResNet50 neural network. + + Args: + class_num (int): Class number. + + Returns: + Cell, cell instance of ResNet50 neural network. + + Examples: + >>> net = resnet50(10) + """ + return ResNet(ResidualBlock, + [3, 4, 6, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [1, 2, 2, 2], + class_num) + +def se_resnet50(class_num=1001): + """ + Get SE-ResNet50 neural network. + + Args: + class_num (int): Class number. + + Returns: + Cell, cell instance of SE-ResNet50 neural network. + + Examples: + >>> net = se-resnet50(1001) + """ + return ResNet(ResidualBlock, + [3, 4, 6, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [1, 2, 2, 2], + class_num, + use_se=True) + +def resnet101(class_num=1001): + """ + Get ResNet101 neural network. + + Args: + class_num (int): Class number. + + Returns: + Cell, cell instance of ResNet101 neural network. + + Examples: + >>> net = resnet101(1001) + """ + return ResNet(ResidualBlock, + [3, 4, 23, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [1, 2, 2, 2], + class_num) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/train.py b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/train.py new file mode 100644 index 0000000..3c29241 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/code/train.py @@ -0,0 +1,191 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""train resnet.""" +import os +import argparse +import ast +from mindspore import context +from mindspore import Tensor +from mindspore.nn.optim.momentum import Momentum +from mindspore.train.model import Model +from mindspore.context import ParallelMode +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.train.loss_scale_manager import FixedLossScaleManager +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from mindspore.communication.management import init, get_rank, get_group_size +from mindspore.common import set_seed +import mindspore.nn as nn +import mindspore.common.initializer as weight_init +from src.lr_generator import get_lr, warmup_cosine_annealing_lr +from src.CrossEntropySmooth import CrossEntropySmooth + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--net', type=str, default=None, help='Resnet Model, either resnet50 or resnet101') +parser.add_argument('--dataset', type=str, default=None, help='Dataset, either cifar10 or imagenet2012') +parser.add_argument('--run_distribute', type=ast.literal_eval, default=False, help='Run distribute') +parser.add_argument('--device_num', type=int, default=1, help='Device num.') + +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') +parser.add_argument('--pre_trained', type=str, default=None, help='Pretrained checkpoint path') +parser.add_argument('--parameter_server', type=ast.literal_eval, default=False, help='Run parameter server train') +args_opt = parser.parse_args() + +set_seed(1) + +if args_opt.net == "resnet50": + from src.resnet import resnet50 as resnet + if args_opt.dataset == "cifar10": + from src.config import config1 as config + from src.dataset import create_dataset1 as create_dataset + else: + from src.config import config2 as config + from src.dataset import create_dataset2 as create_dataset +elif args_opt.net == "resnet101": + from src.resnet import resnet101 as resnet + from src.config import config3 as config + from src.dataset import create_dataset3 as create_dataset +else: + from src.resnet import se_resnet50 as resnet + from src.config import config4 as config + from src.dataset import create_dataset4 as create_dataset + + +if __name__ == '__main__': + target = args_opt.device_target + ckpt_save_dir = config.save_checkpoint_path + + # init context + context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False) + if args_opt.parameter_server: + context.set_ps_context(enable_ps=True) + if args_opt.run_distribute: + if target == "Ascend": + device_id = int(os.getenv('DEVICE_ID')) + context.set_context(device_id=device_id, enable_auto_mixed_precision=True) + context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, + gradients_mean=True) + if args_opt.net == "resnet50" or args_opt.net == "se-resnet50": + context.set_auto_parallel_context(all_reduce_fusion_config=[85, 160]) + else: + context.set_auto_parallel_context(all_reduce_fusion_config=[180, 313]) + init() + # GPU target + else: + init() + context.set_auto_parallel_context(device_num=get_group_size(), parallel_mode=ParallelMode.DATA_PARALLEL, + gradients_mean=True) + if args_opt.net == "resnet50": + context.set_auto_parallel_context(all_reduce_fusion_config=[85, 160]) + ckpt_save_dir = config.save_checkpoint_path + "ckpt_" + str(get_rank()) + "/" + + # create dataset + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, repeat_num=1, + batch_size=config.batch_size, target=target) + step_size = dataset.get_dataset_size() + + # define net + net = resnet(class_num=config.class_num) + if args_opt.parameter_server: + net.set_param_ps() + + # init weight + if args_opt.pre_trained: + param_dict = load_checkpoint(args_opt.pre_trained) + load_param_into_net(net, param_dict) + else: + for _, cell in net.cells_and_names(): + if isinstance(cell, nn.Conv2d): + cell.weight.set_data(weight_init.initializer(weight_init.XavierUniform(), + cell.weight.shape, + cell.weight.dtype)) + if isinstance(cell, nn.Dense): + cell.weight.set_data(weight_init.initializer(weight_init.TruncatedNormal(), + cell.weight.shape, + cell.weight.dtype)) + + # init lr + if args_opt.net == "resnet50" or args_opt.net == "se-resnet50": + lr = get_lr(lr_init=config.lr_init, lr_end=config.lr_end, lr_max=config.lr_max, + warmup_epochs=config.warmup_epochs, total_epochs=config.epoch_size, steps_per_epoch=step_size, + lr_decay_mode=config.lr_decay_mode) + else: + lr = warmup_cosine_annealing_lr(config.lr, step_size, config.warmup_epochs, config.epoch_size, + config.pretrain_epoch_size * step_size) + lr = Tensor(lr) + + # define opt + decayed_params = [] + no_decayed_params = [] + for param in net.trainable_params(): + if 'beta' not in param.name and 'gamma' not in param.name and 'bias' not in param.name: + decayed_params.append(param) + else: + no_decayed_params.append(param) + + group_params = [{'params': decayed_params, 'weight_decay': config.weight_decay}, + {'params': no_decayed_params}, + {'order_params': net.trainable_params()}] + opt = Momentum(group_params, lr, config.momentum, loss_scale=config.loss_scale) + # define loss, model + if target == "Ascend": + if args_opt.dataset == "imagenet2012": + if not config.use_label_smooth: + config.label_smooth_factor = 0.0 + loss = CrossEntropySmooth(sparse=True, reduction="mean", + smooth_factor=config.label_smooth_factor, num_classes=config.class_num) + else: + loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') + loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) + model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}, + amp_level="O2", keep_batchnorm_fp32=False) + else: + # GPU target + if args_opt.dataset == "imagenet2012": + if not config.use_label_smooth: + config.label_smooth_factor = 0.0 + loss = CrossEntropySmooth(sparse=True, reduction="mean", + smooth_factor=config.label_smooth_factor, num_classes=config.class_num) + else: + loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") + + if (args_opt.net == "resnet101" or args_opt.net == "resnet50") and not args_opt.parameter_server: + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, config.weight_decay, + config.loss_scale) + loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) + # Mixed precision + model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}, + amp_level="O2", keep_batchnorm_fp32=True) + else: + ## fp32 training + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, config.weight_decay) + model = Model(net, loss_fn=loss, optimizer=opt, metrics={'acc'}) + + # define callbacks + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossMonitor() + cb = [time_cb, loss_cb] + if config.save_checkpoint: + config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs * step_size, + keep_checkpoint_max=config.keep_checkpoint_max) + ckpt_cb = ModelCheckpoint(prefix="resnet", directory=ckpt_save_dir, config=config_ck) + cb += [ckpt_cb] + + # train model + if args_opt.net == "se-resnet50": + config.epoch_size = config.train_epoch_size + model.train(config.epoch_size - config.pretrain_epoch_size, dataset, callbacks=cb, + dataset_sink_mode=(not args_opt.parameter_server)) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/config/npu_set_env.sh b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/config/npu_set_env.sh new file mode 100644 index 0000000..55d68ec --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/config/npu_set_env.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +rm -rf /var/log/npu/slog/host-0/* +if [ -d /usr/local/Ascend/nnae/latest ];then + + export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/Ascend/driver/tools/hccn_tool/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages + export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp + export TBE_IMPL_PATH=/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core +else + export LD_LIBRARY_PATH=/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/ascend-toolkit/latest//fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/tfplugin/python/site-packages:$projectDir + export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + export TBE_IMPL_PATH=TBE_IMPL_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core +fi + diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/scripts/eval.sh b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/scripts/eval.sh new file mode 100644 index 0000000..c18780c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/scripts/eval.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +yamlPath=$1 +toolsPath=$2 +currtime=$3 +currentDir=$(cd "$(dirname "$0")/.."; pwd) +train_job_dir=${currentDir%train*}/train/result/ms_resnet50/training_job_${currtime} +mkdir -p ${currentDir%train*}/train/result/ms_resnet50/training_job_${currtime}/ + +source ${currentDir}/config/npu_set_env.sh + +export REMARK_LOG_FILE=hw_resnet50.log +benchmark_log_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils +export PYTHONPATH=$PYTHONPATH:${benchmark_log_path} + +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "mindspore_config") + +device_id=${eval_device_id} +export DEVICE_NUM=1 +export DEVICE_ID=${device_id} +export RANK_SIZE=${DEVICE_NUM} +export RANK_ID=${device_id} + + + +python3.7 ${currentDir}/code/eval.py \ + --net=resnet50 \ + --dataset=imagenet2012 \ + --dataset_path=${data_url} \ + --checkpoint_path=${checkpoint_path} > ${train_job_dir}/eval.out 2>&1 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/scripts/run.sh b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/scripts/run.sh new file mode 100644 index 0000000..e3a661d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/scripts/run.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +rank_size=$1 +yamlPath=$2 +toolsPath=$3 +if [ -f /.dockerenv ];then + CLUSTER=$4 +MPIRUN_ALL_IP="$5" + export CLUSTER=${CLUSTER} +fi +currentDir=$(cd "$(dirname "$0")/.."; pwd) +# 配置环境变量并调用 train 方法 +currtime=`date +%Y%m%d%H%M%S` +mkdir -p ${currentDir%train*}/train/result/ms_resnet50/training_job_${currtime}/ +train_job_dir=${currentDir%train*}/train/result/ms_resnet50/training_job_${currtime}/ +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir} &" +# user env +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 +export SLOG_PRINT_TO_STDOUT=0 +export RANK_TABLE_FILE=${currentDir}/config/${rank_size}p.json + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "mindspore_config") + + + +data_url_new=`echo ${data_url//\//\\\\/}` + +jsonFilePath=${currentDir}/code/src/config.py +echo "start to modify inner config file" +#echo "jsonfilepath is "${jsonFilePath} + +sed -i "0,/epoch_size.*$/s//epoch_size\': ${epoches},/" ${jsonFilePath} +sed -i "0,/batch_size.*$/s//batch_size\': ${batch_size},/" ${jsonFilePath} +sed -i "0,/save_checkpoint_epochs.*$/s//save_checkpoint_epochs\': ${save_checkpoint_epochs},/" ${jsonFilePath} +sed -i "0,/loss_scale.*$/s//loss_scale\': ${loss_scale},/" ${jsonFilePath} +sed -i 's/\r//g' ${jsonFilePath} +if [ $? -eq 0 ] ; +then + echo "modify inner config file success" +else + echo "modify inner config file fail" + exit +fi + + + +# device 列表, 若无指定 device 根据 rank_size 顺序选择 +eval device_group=\$device_group_${rank_size}p +if [ x"${device_group}" == x"" ] || [ ${rank_size} -ge 8 ];then + device_group="$(seq 0 "$(expr $rank_size - 1)")" +fi + +# get last device id in device_group, hw log in performance from the dir named last_device_id +device_group_str=`echo ${device_group} | sed 's/ //g'` +first_device_id=`echo ${device_group_str: 0:1}` + +rank_id=0 + +if [ x"${CLUSTER}" == x"True" ];then + # ln hw log + ln -snf ${train_job_dir}/0/hw_resnet50.log ${train_job_dir} + this_ip=$(hostname -I |awk '{print $1}') + for ip in $MPIRUN_ALL_IP;do + if [ x"$ip" != x"$this_ip" ];then + scp $yamlPath root@$ip:$yamlPath + scp $jsonFilePath root@$ip:$jsonFilePath + fi + done + export PATH=$PATH:/usr/local/mpirun4.0/bin + mpirun -H ${mpirun_ip} \ + --bind-to none -map-by slot\ + --allow-run-as-root \ + --mca btl_tcp_if_exclude lo,docker0,endvnic,virbr0,vethf40501b,docker_gwbridge,br-f42ac38052b4\ + --prefix /usr/local/mpirun4.0/ \ + ${currentDir}/scripts/train.sh 0 $rank_size $yamlPath $currtime ${toolsPath} ${CLUSTER} +elif [ x"$mode" == x"train" ];then + # ln hw log + ln -snf ${train_job_dir}/${first_device_id}/hw_resnet50.log ${train_job_dir} + for device_id in $device_group;do + ${currentDir}/scripts/train.sh $device_id $rank_size $yamlPath $currtime ${toolsPath} $rank_id & + let rank_id++ + done +else + echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${ckpt_path%results*}/results &" + ln -snf ${train_job_dir}/${first_device_id}/hw_resnet50.log ${train_job_dir} + bash ${currentDir}/scripts/eval.sh ${yamlPath} ${toolsPath} $currtime + +fi +wait + + +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir} &" + diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/scripts/train.sh b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/scripts/train.sh new file mode 100644 index 0000000..d01bc85 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/mindspore/scripts/train.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash + +device_id=$1 +rank_size=$2 +yamlPath=$3 +currtime=$4 +toolsPath=$5 + +currentDir=$(cd "$(dirname "$0")/.."; pwd) + +export REMARK_LOG_FILE=hw_resnet50.log + +# ${mainDir%train*}/train/result/tensorflow/Bert/TrainingJob-${currtime} # +mkdir -p ${currentDir%train*}/train/result/ms_resnet50/training_job_${currtime}/ +export train_job_dir=${currentDir%train*}/train/result/ms_resnet50/training_job_${currtime}/ + +source ${currentDir}/config/npu_set_env.sh + +benchmark_log_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils +#atlasboost_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +code_dir_path=${currentDir}/code +export PYTHONPATH=$PYTHONPATH:${benchmark_log_path}:${code_dir_path} + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "mindspore_config") + +# user env +export YAML_PATH=$3 +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 +export RANK_TABLE_FILE=${currentDir}/config/${rank_size}p.json +export RANK_SIZE=${rank_size} +export RANK_INDEX=0 +export SLOG_PRINT_TO_STDOUT=0 +export DEVICE_ID=$1 +DEVICE_INDEX=$(( DEVICE_ID + RANK_INDEX * 8 )) +export DEVICE_INDEX=${DEVICE_INDEX} +export MODEL_CKPT_PATH=${train_job_dir}/${device_id}/ckpt${device_id} +export SERVER_ID=0 + +cd ${train_job_dir} +curd_dir=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +export PYTHONPATH=$PYTHONPATH:${curd_dir} + +if [ x"$6" != x"True" ];then + rank_id=$6 + export RANK_ID=$6 +else + device_id_mo=$(python3.7 -c "import src.tensorflow.mpi_ops as atlasboost;atlasboost.init(); \ + device_id = atlasboost.local_rank();cluster_device_id = str(device_id); \ + atlasboost.set_device_id(device_id);print(atlasboost.rank())") + device_id_mo=`echo $device_id_mo` + rank_id=${device_id_mo##* } + export RANK_ID=${rank_id} + device=${device_id_mo##*deviceid = } + device_id=${device%% phyid=*} + export DEVICE_ID=${device_id} + hccljson=${train_job_dir}/*.json + cp ${hccljson} ${currentDir}/config/${rank_size}p.json +fi + +#mkdir exec path +mkdir -p ${train_job_dir}/${device_id} +cd ${train_job_dir}/${device_id} + +startTime=`date +%Y%m%d-%H:%M:%S` +startTime_s=`date +%s` + + + +# 根据单卡/多卡区分调用参数 + + +if [ x"$6" == x"True" ];then + export CLUSTER=True + echo "run cluster" + --net=resnet50 \ + --dataset=imagenet2012 \ + --run_distribute=${run_distribute} \ + --pre_trained=${pre_trained} \ + --dataset_path=${data_url} > ${train_job_dir}/train_${device_id}.log 2>&1 +elif [ ${rank_size} -le 8 ];then + # 多卡单机 + if [ ${rank_size} -eq 1 ]; then + run_distribute=False + device_num=1 + else + run_distribute=True + device_num=${rank_size} + fi + export DEVICE_NUM=${device_num} + if [ x"${pre_trained}" == x"None" ];then + python3.7 ${currentDir}/code/train.py \ + --net=resnet50 \ + --dataset=imagenet2012 \ + --run_distribute=${run_distribute} \ + --device_num=${device_num} \ + --dataset_path=${data_url} > ${train_job_dir}/train_${device_id}.log 2>&1 + + else + python3.7 ${currentDir}/code/train.py \ + --net=resnet50 \ + --dataset=${data_url} \ + --run_distribute=${run_distribute} \ + --device_num=${device_num} \ + --pre_trained=${pre_trained} \ + --dataset_path=${data_url} > ${train_job_dir}/train_${device_id}.log 2>&1 + fi +fi + + +if [ $? -eq 0 ] ; +then + echo ":::ABK 1.0.0 resnet50 train success" + echo ":::ABK 1.0.0 resnet50 train success" >> ${train_job_dir}/train_${device_id}.log + echo ":::ABK 1.0.0 resnet50 train success" >> ${train_job_dir}/${device_id}/hw_resnet50.log +else + echo ":::ABK 1.0.0 resnet50 train failed" + echo ":::ABK 1.0.0 resnet50 train failed" >> ${train_job_dir}/train_${device_id}.log + echo ":::ABK 1.0.0 resnet50 train failed" >> ${train_job_dir}/${device_id}/hw_resnet50.log +fi + +endTime=`date +%Y%m%d-%H:%M:%S` +endTime_s=`date +%s` + +sumTime=$[ $endTime_s - $startTime_s ] + +hour=$(( $sumTime/3600 )) +min=$(( ($sumTime-${hour}*3600)/60 )) +sec=$(( $sumTime-${hour}*3600-${min}*60 )) +echo ":::ABK 1.0.0 resnet50 train total time：${hour}:${min}:${sec}" + +echo ":::ABK 1.0.0 resnet50 train total time：${hour}:${min}:${sec}" >> ${train_job_dir}/${device_id}/hw_resnet50.log + diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/README.md b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/README.md new file mode 100644 index 0000000..89e6fc1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/README.md @@ -0,0 +1,26 @@ +# ResNet50_pytorch训练说明 + +### 1. 模型训练参数配置 + +在train/yaml/ResNet50.yaml中修改相应配置， 配置项含义: + +``` + pytorch_config: + data_url: 数据集路径 + batch_size: 跑1p时batch_size为512；跑8p时batch_size为4096 + epoches: 跑多少个epoch + mode: train_and_evaluate、evaluate两种模式 + ckpt_path: /home/train/result/pt_resnet50/training_job_20200916042624/7/checkpoint_npu7model_best.pth.tar + docker_image: docker 镜像名称:版本号 + lr: 默认参数1p 0.2，8p 2.048 +``` + +------ + + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/__init__.py new file mode 100644 index 0000000..9fbfb7e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#from . import logger +#from . import dataloaders +#from . import training +#from . import utils +#from . import mixup +#from . import resnet +#from . import smoothing diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/dataloaders.py b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/dataloaders.py new file mode 100644 index 0000000..43bc5b5 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/dataloaders.py @@ -0,0 +1,369 @@ +# Copyright (c) 2018-2019, NVIDIA CORPORATION +# Copyright (c) 2017- Facebook, Inc +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import os +import torch +import numpy as np +import torchvision.datasets as datasets +import torchvision.transforms as transforms +from PIL import Image + +DATA_BACKEND_CHOICES = ['pytorch', 'syntetic'] +# try: +# from nvidia.dali.plugin.pytorch import DALIClassificationIterator +# from nvidia.dali.pipeline import Pipeline +# import nvidia.dali.ops as ops +# import nvidia.dali.types as types +# DATA_BACKEND_CHOICES.append('dali-gpu') +# DATA_BACKEND_CHOICES.append('dali-cpu') +# except ImportError: +# print("Please install DALI from https://www.github.com/NVIDIA/DALI to run this example.") + + +def load_jpeg_from_file(path, cuda=True, fp16=False): + img_transforms = transforms.Compose( + [transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor()] + ) + + img = img_transforms(Image.open(path)) + with torch.no_grad(): + # mean and std are not multiplied by 255 as they are in training script + # torch dataloader reads data into bytes whereas loading directly + # through PIL creates a tensor with floats in [0,1] range + mean = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1) + std = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1) + + if cuda: + mean = mean.cuda() + std = std.cuda() + img = img.cuda() + if fp16: + mean = mean.half() + std = std.half() + img = img.half() + else: + img = img.float() + + input = img.unsqueeze(0).sub_(mean).div_(std) + + return input + + +# class HybridTrainPipe(Pipeline): +# def __init__(self, batch_size, num_threads, device_id, data_dir, crop, dali_cpu=False): +# super(HybridTrainPipe, self).__init__(batch_size, num_threads, device_id, seed = 12 + device_id) +# if torch.distributed.is_initialized(): +# rank = torch.distributed.get_rank() +# world_size = torch.distributed.get_world_size() +# else: +# rank = 0 +# world_size = 1 + +# self.input = ops.FileReader( +# file_root = data_dir, +# shard_id = rank, +# num_shards = world_size, +# random_shuffle = True) + +# if dali_cpu: +# dali_device = "cpu" +# self.decode = ops.ImageDecoder(device=dali_device, output_type=types.RGB) +# else: +# dali_device = "gpu" +# # This padding sets the size of the internal nvJPEG buffers to be able to handle all images from full-sized ImageNet +# # without additional reallocations +# self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB, device_memory_padding=211025920, host_memory_padding=140544512) + +# self.res = ops.RandomResizedCrop( +# device=dali_device, +# size=[crop, crop], +# interp_type=types.INTERP_LINEAR, +# random_aspect_ratio=[0.75, 4./3.], +# random_area=[0.08, 1.0], +# num_attempts=100) + +# self.cmnp = ops.CropMirrorNormalize(device = "gpu", +# output_dtype = types.FLOAT, +# output_layout = types.NCHW, +# crop = (crop, crop), +# image_type = types.RGB, +# mean = [0.485 * 255,0.456 * 255,0.406 * 255], +# std = [0.229 * 255,0.224 * 255,0.225 * 255]) +# self.coin = ops.CoinFlip(probability = 0.5) + +# def define_graph(self): +# rng = self.coin() +# self.jpegs, self.labels = self.input(name = "Reader") +# images = self.decode(self.jpegs) +# images = self.res(images) +# output = self.cmnp(images.gpu(), mirror = rng) +# return [output, self.labels] + + +# class HybridValPipe(Pipeline): +# def __init__(self, batch_size, num_threads, device_id, data_dir, crop, size): +# super(HybridValPipe, self).__init__(batch_size, num_threads, device_id, seed = 12 + device_id) +# if torch.distributed.is_initialized(): +# rank = torch.distributed.get_rank() +# world_size = torch.distributed.get_world_size() +# else: +# rank = 0 +# world_size = 1 + +# self.input = ops.FileReader( +# file_root = data_dir, +# shard_id = rank, +# num_shards = world_size, +# random_shuffle = False) + +# self.decode = ops.ImageDecoder(device = "mixed", output_type = types.RGB) +# self.res = ops.Resize(device = "gpu", resize_shorter = size) +# self.cmnp = ops.CropMirrorNormalize(device = "gpu", +# output_dtype = types.FLOAT, +# output_layout = types.NCHW, +# crop = (crop, crop), +# image_type = types.RGB, +# mean = [0.485 * 255,0.456 * 255,0.406 * 255], +# std = [0.229 * 255,0.224 * 255,0.225 * 255]) + +# def define_graph(self): +# self.jpegs, self.labels = self.input(name = "Reader") +# images = self.decode(self.jpegs) +# images = self.res(images) +# output = self.cmnp(images) +# return [output, self.labels] + + +class DALIWrapper(object): + def gen_wrapper(dalipipeline, num_classes, one_hot): + for data in dalipipeline: + input = data[0]["data"] + target = torch.reshape(data[0]["label"], [-1]).cuda().long() + if one_hot: + target = expand(num_classes, torch.float, target) + yield input, target + dalipipeline.reset() + + def __init__(self, dalipipeline, num_classes, one_hot): + self.dalipipeline = dalipipeline + self.num_classes = num_classes + self.one_hot = one_hot + + def __iter__(self): + return DALIWrapper.gen_wrapper(self.dalipipeline, self.num_classes, self.one_hot) + +def get_dali_train_loader(dali_cpu=False): + # def gdtl(data_path, batch_size, num_classes, one_hot, workers=5, _worker_init_fn=None, fp16=False): + # if torch.distributed.is_initialized(): + # rank = torch.distributed.get_rank() + # world_size = torch.distributed.get_world_size() + # else: + # rank = 0 + # world_size = 1 + + # traindir = os.path.join(data_path, 'train') + + # pipe = HybridTrainPipe(batch_size=batch_size, num_threads=workers, + # device_id = rank % torch.cuda.device_count(), + # data_dir = traindir, crop = 224, dali_cpu=dali_cpu) + + # pipe.build() + # train_loader = DALIClassificationIterator(pipe, size = int(pipe.epoch_size("Reader") / world_size)) + + # return DALIWrapper(train_loader, num_classes, one_hot), int(pipe.epoch_size("Reader") / (world_size * batch_size)) + + # return gdtl + def gdtl(data_path, batch_size, num_classes, one_hot, workers=5, _worker_init_fn=None, fp16=False): + return False + return gdvl + +def get_dali_val_loader(): + # def gdvl(data_path, batch_size, num_classes, one_hot, workers=5, _worker_init_fn=None, fp16=False): + # if torch.distributed.is_initialized(): + # rank = torch.distributed.get_rank() + # world_size = torch.distributed.get_world_size() + # else: + # rank = 0 + # world_size = 1 + + # valdir = os.path.join(data_path, 'val') + + # pipe = HybridValPipe(batch_size=batch_size, num_threads=workers, + # device_id = rank % torch.cuda.device_count(), + # data_dir = valdir, + # crop = 224, size = 256) + + # pipe.build() + # val_loader = DALIClassificationIterator(pipe, size = int(pipe.epoch_size("Reader") / world_size)) + + # return DALIWrapper(val_loader, num_classes, one_hot), int(pipe.epoch_size("Reader") / (world_size * batch_size)) + # return gdvl + def gdvl(data_path, batch_size, num_classes, one_hot, workers=5, _worker_init_fn=None, fp16=False): + return False + return gdvl + +def fast_collate(batch): + imgs = [img[0] for img in batch] + targets = torch.tensor([target[1] for target in batch], dtype=torch.int64) + w = imgs[0].size[0] + h = imgs[0].size[1] + tensor = torch.zeros( (len(imgs), 3, h, w), dtype=torch.uint8 ) + for i, img in enumerate(imgs): + nump_array = np.asarray(img, dtype=np.uint8) + tens = torch.from_numpy(nump_array) + if(nump_array.ndim < 3): + nump_array = np.expand_dims(nump_array, axis=-1) + nump_array = np.rollaxis(nump_array, 2) + + tensor[i] += torch.from_numpy(nump_array) + + return tensor, targets + + +def expand(num_classes, dtype, tensor): + e = torch.zeros(tensor.size(0), num_classes, dtype=dtype, device=torch.device('cuda')) + e = e.scatter(1, tensor.unsqueeze(1), 1.0) + return e + +class PrefetchedWrapper(object): + def prefetched_loader(loader, num_classes, fp16, one_hot): + mean = torch.tensor([0.485 * 255, 0.456 * 255, 0.406 * 255]).cuda().view(1,3,1,1) + std = torch.tensor([0.229 * 255, 0.224 * 255, 0.225 * 255]).cuda().view(1,3,1,1) + if fp16: + mean = mean.half() + std = std.half() + + stream = torch.cuda.Stream() + first = True + + for next_input, next_target in loader: + with torch.cuda.stream(stream): + next_input = next_input.cuda(non_blocking=True) + next_target = next_target.cuda(non_blocking=True) + if fp16: + next_input = next_input.half() + if one_hot: + next_target = expand(num_classes, torch.half, next_target) + else: + next_input = next_input.float() + if one_hot: + next_target = expand(num_classes, torch.float, next_target) + + next_input = next_input.sub_(mean).div_(std) + + if not first: + yield input, target + else: + first = False + + torch.cuda.current_stream().wait_stream(stream) + input = next_input + target = next_target + + yield input, target + + def __init__(self, dataloader, num_classes, fp16, one_hot): + self.dataloader = dataloader + self.fp16 = fp16 + self.epoch = 0 + self.one_hot = one_hot + self.num_classes = num_classes + + def __iter__(self): + if (self.dataloader.sampler is not None and + isinstance(self.dataloader.sampler, + torch.utils.data.distributed.DistributedSampler)): + + self.dataloader.sampler.set_epoch(self.epoch) + self.epoch += 1 + return PrefetchedWrapper.prefetched_loader(self.dataloader, self.num_classes, self.fp16, self.one_hot) + +def get_pytorch_train_loader(data_path, batch_size, num_classes, one_hot, workers=5, _worker_init_fn=None, fp16=False): + traindir = os.path.join(data_path, 'train') + train_dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + ])) + + if torch.distributed.is_initialized(): + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=batch_size, shuffle=(train_sampler is None), + num_workers=workers, worker_init_fn=_worker_init_fn, pin_memory=True, sampler=train_sampler, collate_fn=fast_collate, drop_last=True) + + return PrefetchedWrapper(train_loader, num_classes, fp16, one_hot), len(train_loader) + +def get_pytorch_val_loader(data_path, batch_size, num_classes, one_hot, workers=5, _worker_init_fn=None, fp16=False): + valdir = os.path.join(data_path, 'val') + val_dataset = datasets.ImageFolder( + valdir, transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + ])) + + if torch.distributed.is_initialized(): + val_sampler = torch.utils.data.distributed.DistributedSampler(val_dataset) + else: + val_sampler = None + + val_loader = torch.utils.data.DataLoader( + val_dataset, + sampler=val_sampler, + batch_size=batch_size, shuffle=False, + num_workers=workers, worker_init_fn=_worker_init_fn, pin_memory=True, + collate_fn=fast_collate) + + return PrefetchedWrapper(val_loader, num_classes, fp16, one_hot), len(val_loader) + +class SynteticDataLoader(object): + def __init__(self, fp16, batch_size, num_classes, num_channels, height, width, one_hot): + input_data = torch.empty(batch_size, num_channels, height, width).cuda().normal_(0, 1.0) + if one_hot: + input_target = torch.empty(batch_size, num_classes).cuda() + input_target[:, 0] = 1.0 + else: + input_target = torch.randint(0, num_classes, (batch_size,)) + input_target=input_target.cuda() + if fp16: + input_data = input_data.half() + + self.input_data = input_data + self.input_target = input_target + + def __iter__(self): + while True: + yield self.input_data, self.input_target + +def get_syntetic_loader(data_path, batch_size, num_classes, one_hot, workers=None, _worker_init_fn=None, fp16=False): + return SynteticDataLoader(fp16, batch_size, 1000, 3, 224, 224, one_hot), -1 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/logger.py b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/logger.py new file mode 100644 index 0000000..e92f2bc --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/logger.py @@ -0,0 +1,310 @@ +# Copyright (c) 2018-2019, NVIDIA CORPORATION +# Copyright (c) 2017- Facebook, Inc +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from collections import OrderedDict +import dllogger +import numpy as np + + +def format_step(step): + if isinstance(step, str): + return step + s = "" + if len(step) > 0: + s += "Epoch: {} ".format(step[0]) + if len(step) > 1: + s += "Iteration: {} ".format(step[1]) + if len(step) > 2: + s += "Validation Iteration: {} ".format(step[2]) + if len(step) == 0: + s = "Summary:" + return s + + +PERF_METER = lambda: Meter(AverageMeter(), AverageMeter(), AverageMeter()) +LOSS_METER = lambda: Meter(AverageMeter(), AverageMeter(), MinMeter()) +ACC_METER = lambda: Meter(AverageMeter(), AverageMeter(), MaxMeter()) +LR_METER = lambda: Meter(LastMeter(), LastMeter(), LastMeter()) + +LAT_100 = lambda: Meter(QuantileMeter(1), QuantileMeter(1), QuantileMeter(1)) +LAT_99 = lambda: Meter(QuantileMeter(0.99), QuantileMeter(0.99), QuantileMeter(0.99)) +LAT_95 = lambda: Meter(QuantileMeter(0.95), QuantileMeter(0.95), QuantileMeter(0.95)) + +class Meter(object): + def __init__(self, iteration_aggregator, epoch_aggregator, run_aggregator): + self.run_aggregator = run_aggregator + self.epoch_aggregator = epoch_aggregator + self.iteration_aggregator = iteration_aggregator + + def record(self, val, n=1): + self.iteration_aggregator.record(val, n=n) + + def get_iteration(self): + v, n = self.iteration_aggregator.get_val() + return v + + def reset_iteration(self): + v, n = self.iteration_aggregator.get_data() + self.iteration_aggregator.reset() + if v is not None: + self.epoch_aggregator.record(v, n=n) + + def get_epoch(self): + v, n = self.epoch_aggregator.get_val() + return v + + def reset_epoch(self): + v, n = self.epoch_aggregator.get_data() + self.epoch_aggregator.reset() + if v is not None: + self.run_aggregator.record(v, n=n) + + def get_run(self): + v, n = self.run_aggregator.get_val() + return v + + def reset_run(self): + self.run_aggregator.reset() + + +class QuantileMeter(object): + def __init__(self, q): + self.q = q + self.reset() + + def reset(self): + self.vals = [] + self.n = 0 + + def record(self, val, n=1): + if isinstance(val, list): + self.vals += val + self.n += len(val) + else: + self.vals += [val] * n + self.n += n + + def get_val(self): + if not self.vals: + return None, self.n + return np.quantile(self.vals, self.q, interpolation='nearest'), self.n + + def get_data(self): + return self.vals, self.n + + +class MaxMeter(object): + def __init__(self): + self.reset() + + def reset(self): + self.max = None + self.n = 0 + + def record(self, val, n=1): + if self.max is None: + self.max = val + else: + self.max = max(self.max, val) + self.n = n + + def get_val(self): + return self.max, self.n + + def get_data(self): + return self.max, self.n + + +class MinMeter(object): + def __init__(self): + self.reset() + + def reset(self): + self.min = None + self.n = 0 + + def record(self, val, n=1): + if self.min is None: + self.min = val + else: + self.min = max(self.min, val) + self.n = n + + def get_val(self): + return self.min, self.n + + def get_data(self): + return self.min, self.n + + +class LastMeter(object): + def __init__(self): + self.reset() + + def reset(self): + self.last = None + self.n = 0 + + def record(self, val, n=1): + self.last = val + self.n = n + + def get_val(self): + return self.last, self.n + + def get_data(self): + return self.last, self.n + + +class AverageMeter(object): + def __init__(self): + self.reset() + + def reset(self): + self.n = 0 + self.val = 0 + + def record(self, val, n=1): + self.n += n + self.val += val * n + + def get_val(self): + if self.n == 0: + return None, 0 + return self.val / self.n, self.n + + def get_data(self): + if self.n == 0: + return None, 0 + return self.val / self.n, self.n + + +class Logger(object): + def __init__(self, print_interval, backends, verbose=False): + self.epoch = -1 + self.iteration = -1 + self.val_iteration = -1 + self.metrics = OrderedDict() + self.backends = backends + self.print_interval = print_interval + self.verbose = verbose + dllogger.init(backends) + + def log_parameter(self, data, verbosity=0): + dllogger.log(step="PARAMETER", data=data, verbosity=verbosity) + + def register_metric(self, metric_name, meter, verbosity=0, metadata={}): + if self.verbose: + print("Registering metric: {}".format(metric_name)) + self.metrics[metric_name] = {'meter': meter, 'level': verbosity} + dllogger.metadata(metric_name, metadata) + + def log_metric(self, metric_name, val, n=1): + self.metrics[metric_name]['meter'].record(val, n=n) + + def start_iteration(self, val=False): + if val: + self.val_iteration += 1 + else: + self.iteration += 1 + + def end_iteration(self, val=False): + it = self.val_iteration if val else self.iteration + if (it % self.print_interval == 0): + metrics = { + n: m + for n, m in self.metrics.items() if n.startswith('val') == val + } + step = (self.epoch, + self.iteration) if not val else (self.epoch, + self.iteration, + self.val_iteration) + + verbositys = {m['level'] for _, m in metrics.items()} + for ll in verbositys: + llm = {n: m for n, m in metrics.items() if m['level'] == ll} + + dllogger.log(step=step, + data={ + n: m['meter'].get_iteration() + for n, m in llm.items() + }, + verbosity=ll) + + for n, m in metrics.items(): + m['meter'].reset_iteration() + + dllogger.flush() + + def start_epoch(self): + self.epoch += 1 + self.iteration = 0 + self.val_iteration = 0 + + for n, m in self.metrics.items(): + m['meter'].reset_epoch() + + def end_epoch(self): + for n, m in self.metrics.items(): + m['meter'].reset_iteration() + + verbositys = {m['level'] for _, m in self.metrics.items()} + for ll in verbositys: + llm = {n: m for n, m in self.metrics.items() if m['level'] == ll} + dllogger.log(step=(self.epoch, ), + data={n: m['meter'].get_epoch() + for n, m in llm.items()}) + + def end(self): + for n, m in self.metrics.items(): + m['meter'].reset_epoch() + + verbositys = {m['level'] for _, m in self.metrics.items()} + for ll in verbositys: + llm = {n: m for n, m in self.metrics.items() if m['level'] == ll} + dllogger.log(step=tuple(), + data={n: m['meter'].get_run() + for n, m in llm.items()}) + + for n, m in self.metrics.items(): + m['meter'].reset_epoch() + + dllogger.flush() + + def iteration_generator_wrapper(self, gen, val=False): + for g in gen: + self.start_iteration(val=val) + yield g + self.end_iteration(val=val) + + def epoch_generator_wrapper(self, gen): + for g in gen: + self.start_epoch() + yield g + self.end_epoch() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/mixup.py b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/mixup.py new file mode 100644 index 0000000..e946dc9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/mixup.py @@ -0,0 +1,67 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +import torch.nn as nn +import numpy as np + + +def mixup(alpha, num_classes, data, target): + with torch.no_grad(): + bs = data.size(0) + c = np.random.beta(alpha, alpha) + + perm = torch.randperm(bs).cuda() + + md = c * data + (1-c) * data[perm, :] + mt = c * target + (1-c) * target[perm, :] + return md, mt + + +class MixUpWrapper(object): + def __init__(self, alpha, num_classes, dataloader): + self.alpha = alpha + self.dataloader = dataloader + self.num_classes = num_classes + + def mixup_loader(self, loader): + for input, target in loader: + i, t = mixup(self.alpha, self.num_classes, input, target) + yield i, t + + def __iter__(self): + return self.mixup_loader(self.dataloader) + + +class NLLMultiLabelSmooth(nn.Module): + def __init__(self, smoothing = 0.0): + super(NLLMultiLabelSmooth, self).__init__() + self.confidence = 1.0 - smoothing + self.smoothing = smoothing + + def forward(self, x, target): + if self.training: + x = x.float() + target = target.float() + logprobs = torch.nn.functional.log_softmax(x, dim = -1) + + nll_loss = -logprobs * target + nll_loss = nll_loss.sum(-1) + + smooth_loss = -logprobs.mean(dim=-1) + + loss = self.confidence * nll_loss + self.smoothing * smooth_loss + + return loss.mean() + else: + return torch.nn.functional.cross_entropy(x, target) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/resnet.py b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/resnet.py new file mode 100644 index 0000000..b8c40d9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/resnet.py @@ -0,0 +1,370 @@ +# Copyright (c) 2018-2019, NVIDIA CORPORATION +# Copyright (c) 2017- Facebook, Inc +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import math +import torch +import torch.nn as nn +import numpy as np + +__all__ = ['ResNet', 'build_resnet', 'resnet_versions', 'resnet_configs'] + +# ResNetBuilder {{{ + +class ResNetBuilder(object): + def __init__(self, version, config): + self.conv3x3_cardinality = 1 if 'cardinality' not in version.keys() else version['cardinality'] + self.config = config + + def conv(self, kernel_size, in_planes, out_planes, groups=1, stride=1): + conv = nn.Conv2d( + in_planes, out_planes, + kernel_size=kernel_size, groups=groups, + stride=stride, padding=int((kernel_size - 1)/2), + bias=False) + + if self.config['nonlinearity'] == 'relu': + # torch.nn.init.kaiming_normal_(tensor, a=0, mode='fan_in', nonlinearity='leaky_relu') + # Copy + # 用论文 “Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification” - He, K. et al. (2015) 中提及的正态分布初始化输入 Tensor。初始化后的张量中的值采样 ) 且 + # %20%5Ctimes%20%5Ctext%7Bfan%5C_in%7D%7D%7D%0D%0A%0D%0A) + # 也被称作 He initialization。 + # 参数： + # tensor – n 维 torch.Tensor + # a – 该层后面一层的整流函数中负的斜率 (默认为 0，此时为 Relu) + # mode – 'fan_in' (default) 或者 'fan_out'。使用fan_in保持weights的方差在前向传播中不变；使用fan_out保持weights的方差在反向传播中不变。 + # nonlinearity – 非线性函数 (nn.functional 中的名字)，推荐只使用 'relu' 或 'leaky_relu' (default)。 + # 例子 + # >>> w = torch.empty(3, 5) + # >>> nn.init.kaiming_normal_(w, mode='fan_out', nonlinearity='relu') + nn.init.kaiming_normal_(conv.weight, + mode=self.config['conv_init'], + nonlinearity=self.config['nonlinearity']) + + + + + return conv + + def conv3x3(self, in_planes, out_planes, stride=1): + """3x3 convolution with padding""" + c = self.conv(3, in_planes, out_planes, groups=self.conv3x3_cardinality, stride=stride) + return c + + def conv1x1(self, in_planes, out_planes, stride=1): + """1x1 convolution with padding""" + c = self.conv(1, in_planes, out_planes, stride=stride) + return c + + def conv7x7(self, in_planes, out_planes, stride=1): + """7x7 convolution with padding""" + c = self.conv(7, in_planes, out_planes, stride=stride) + return c + + def conv5x5(self, in_planes, out_planes, stride=1): + """5x5 convolution with padding""" + c = self.conv(5, in_planes, out_planes, stride=stride) + return c + + def batchnorm(self, planes, last_bn=False): + bn = nn.BatchNorm2d(planes) + gamma_init_val = 0 if last_bn and self.config['last_bn_0_init'] else 1 + nn.init.constant_(bn.weight, gamma_init_val) + nn.init.constant_(bn.bias, 0) + + return bn + + def activation(self): + return self.config['activation']() + +# ResNetBuilder }}} + +# BasicBlock {{{ +class BasicBlock(nn.Module): + def __init__(self, builder, inplanes, planes, expansion, stride=1, downsample=None): + super(BasicBlock, self).__init__() + self.conv1 = builder.conv3x3(inplanes, planes, stride) + self.bn1 = builder.batchnorm(planes) + self.relu = builder.activation() + self.conv2 = builder.conv3x3(planes, planes*expansion) + self.bn2 = builder.batchnorm(planes*expansion, last_bn=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + if self.bn1 is not None: + out = self.bn1(out) + + out = self.relu(out) + + out = self.conv2(out) + + if self.bn2 is not None: + out = self.bn2(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out +# BasicBlock }}} + +# SqueezeAndExcitation {{{ +class SqueezeAndExcitation(nn.Module): + def __init__(self, planes, squeeze): + super(SqueezeAndExcitation, self).__init__() + self.squeeze = nn.Linear(planes, squeeze) + self.expand = nn.Linear(squeeze, planes) + self.relu = nn.ReLU(inplace=True) + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + out = torch.mean(x.view(x.size(0), x.size(1), -1), 2) + out = self.squeeze(out) + out = self.relu(out) + out = self.expand(out) + out = self.sigmoid(out) + out = out.unsqueeze(2).unsqueeze(3) + + return out + +# }}} + +# Bottleneck {{{ +class Bottleneck(nn.Module): + def __init__(self, builder, inplanes, planes, expansion, stride=1, se=False, se_squeeze=16, downsample=None): + super(Bottleneck, self).__init__() + self.conv1 = builder.conv1x1(inplanes, planes) + self.bn1 = builder.batchnorm(planes) + self.conv2 = builder.conv3x3(planes, planes, stride=stride) + self.bn2 = builder.batchnorm(planes) + self.conv3 = builder.conv1x1(planes, planes * expansion) + self.bn3 = builder.batchnorm(planes * expansion, last_bn=True) + self.relu = builder.activation() + self.downsample = downsample + self.stride = stride + self.squeeze = SqueezeAndExcitation(planes*expansion, se_squeeze) if se else None + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + if self.squeeze is None: + out += residual + else: + out = torch.addcmul(residual, 1.0, out, self.squeeze(out)) + + out = self.relu(out) + + return out + +def SEBottleneck(builder, inplanes, planes, expansion, stride=1, downsample=None): + return Bottleneck(builder, inplanes, planes, expansion, stride=stride, se=True, se_squeeze=16, downsample=downsample) +# Bottleneck }}} + +# ResNet {{{ +class ResNet(nn.Module): + def __init__(self, builder, block, expansion, layers, widths, num_classes=1000): + self.inplanes = 64 + super(ResNet, self).__init__() + self.conv1 = builder.conv7x7(3, 64, stride=2) + self.bn1 = builder.batchnorm(64) + self.relu = builder.activation() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(builder, block, expansion, widths[0], layers[0]) + self.layer2 = self._make_layer(builder, block, expansion, widths[1], layers[1], stride=2) + self.layer3 = self._make_layer(builder, block, expansion, widths[2], layers[2], stride=2) + self.layer4 = self._make_layer(builder, block, expansion, widths[3], layers[3], stride=2) + self.avgpool = nn.AdaptiveAvgPool2d(1) + self.fc = nn.Linear(widths[3] * expansion, num_classes) + + def _make_layer(self, builder, block, expansion, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * expansion: + dconv = builder.conv1x1(self.inplanes, planes * expansion, + stride=stride) + dbn = builder.batchnorm(planes * expansion) + if dbn is not None: + downsample = nn.Sequential(dconv, dbn) + else: + downsample = dconv + + layers = [] + layers.append(block(builder, self.inplanes, planes, expansion, stride=stride, downsample=downsample)) + self.inplanes = planes * expansion + for i in range(1, blocks): + layers.append(block(builder, self.inplanes, planes, expansion)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv1(x) + if self.bn1 is not None: + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + x = x.view(x.size(0), -1) + x = self.fc(x) + + return x +# ResNet }}} + +resnet_configs = { + 'classic' : { + 'conv' : nn.Conv2d, + 'conv_init' : 'fan_out', + 'nonlinearity' : 'relu', + 'last_bn_0_init' : False, + 'activation' : lambda: nn.ReLU(inplace=True), + }, + 'fanin' : { + 'conv' : nn.Conv2d, + 'conv_init' : 'fan_in', + 'nonlinearity' : 'relu', + 'last_bn_0_init' : False, + 'activation' : lambda: nn.ReLU(inplace=True), + }, + 'grp-fanin' : { + 'conv' : nn.Conv2d, + 'conv_init' : 'fan_in', + 'nonlinearity' : 'relu', + 'last_bn_0_init' : False, + 'activation' : lambda: nn.ReLU(inplace=True), + }, + 'grp-fanout' : { + 'conv' : nn.Conv2d, + 'conv_init' : 'fan_out', + 'nonlinearity' : 'relu', + 'last_bn_0_init' : False, + 'activation' : lambda: nn.ReLU(inplace=True), + }, + } + +resnet_versions = { + 'resnet18' : { + 'net' : ResNet, + 'block' : BasicBlock, + 'layers' : [2, 2, 2, 2], + 'widths' : [64, 128, 256, 512], + 'expansion' : 1, + 'num_classes' : 1000, + }, + 'resnet34' : { + 'net' : ResNet, + 'block' : BasicBlock, + 'layers' : [3, 4, 6, 3], + 'widths' : [64, 128, 256, 512], + 'expansion' : 1, + 'num_classes' : 1000, + }, + 'resnet50' : { + 'net' : ResNet, + 'block' : Bottleneck, + 'layers' : [3, 4, 6, 3], + 'widths' : [64, 128, 256, 512], + 'expansion' : 4, + 'num_classes' : 1000, + }, + 'resnet101' : { + 'net' : ResNet, + 'block' : Bottleneck, + 'layers' : [3, 4, 23, 3], + 'widths' : [64, 128, 256, 512], + 'expansion' : 4, + 'num_classes' : 1000, + }, + 'resnet152' : { + 'net' : ResNet, + 'block' : Bottleneck, + 'layers' : [3, 8, 36, 3], + 'widths' : [64, 128, 256, 512], + 'expansion' : 4, + 'num_classes' : 1000, + }, + 'resnext101-32x4d' : { + 'net' : ResNet, + 'block' : Bottleneck, + 'cardinality' : 32, + 'layers' : [3, 4, 23, 3], + 'widths' : [128, 256, 512, 1024], + 'expansion' : 2, + 'num_classes' : 1000, + }, + 'se-resnext101-32x4d' : { + 'net' : ResNet, + 'block' : SEBottleneck, + 'cardinality' : 32, + 'layers' : [3, 4, 23, 3], + 'widths' : [128, 256, 512, 1024], + 'expansion' : 2, + 'num_classes' : 1000, + }, + } + + +def build_resnet(version, config, verbose=True): + version = resnet_versions[version] + config = resnet_configs[config] + + builder = ResNetBuilder(version, config) + if verbose: + print("Version: {}".format(version)) + print("Config: {}".format(config)) + model = version['net'](builder, + version['block'], + version['expansion'], + version['layers'], + version['widths'], + version['num_classes']) + + return model diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/smoothing.py b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/smoothing.py new file mode 100644 index 0000000..0d34a34 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/smoothing.py @@ -0,0 +1,91 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +import torch.nn as nn + + +class CrossEntropy(nn.CrossEntropyLoss): + def __init__(self, smooth_factor=0., num_classes=1000): + super(CrossEntropy, self).__init__() + self.on_value = 1.0 - smooth_factor + self.off_value = 1.0 * smooth_factor / (num_classes - 1) + + def forward(self, input, target): + one_hot_label = torch.npu_one_hot(target, -1, input.size(1), self.on_value, self.off_value) + one_hot_label = one_hot_label.to(torch.float16) + loss = torch.npu_softmax_cross_entropy_with_logits(input.to(torch.float16), one_hot_label) + + loss = torch.mean(loss, [0], keepdim=False, dtype=torch.float32) + return loss + +class LabelSmoothingNpu(nn.Module): + """ + NLL loss with label smoothing. + """ + def __init__(self, smoothing=0.0): + """ + Constructor for the LabelSmoothing module. + + :param smoothing: label smoothing factor + """ + super(LabelSmoothingNpu, self).__init__() + self.confidence = 1.0 - smoothing + self.smoothing = smoothing + + self.epsilon = 0.1 + self.num_classes = 1000 + + def forward(self, x, target): + CALCULATE_DEVICE = x.device + logprobs = torch.nn.functional.log_softmax(x, dim=-1).to("cpu") + + targets = torch.zeros_like(logprobs).scatter_(1, target.unsqueeze(1), 1) + targets = (1 - self.epsilon) * targets + self.epsilon / self.num_classes + loss = (-targets * logprobs).mean(0).sum() + + # nll_loss = -logprobs.gather(dim=-1, index=target.unsqueeze(1)) + # nll_loss = nll_loss.squeeze(1) + # smooth_loss = -logprobs.mean(dim=-1) + # loss = self.confidence * nll_loss + self.smoothing * smooth_loss + return loss.to(CALCULATE_DEVICE) + + +class LabelSmoothingGpu(nn.Module): + """ + NLL loss with label smoothing. + """ + def __init__(self, smoothing=0.0): + """ + Constructor for the LabelSmoothing module. + + :param smoothing: label smoothing factor + """ + super(LabelSmoothingGpu, self).__init__() + self.confidence = 1.0 - smoothing + self.smoothing = smoothing + # print("----------------------LabelSooothing.__init__") + # def __call__(self,x,target): + # print("----------------------LabelSooothing.__call__") + # return self.forward(self,x,target) + + def forward(self, x, target): + logprobs = torch.nn.functional.log_softmax(x, dim=-1) + + nll_loss = -logprobs.gather(dim=-1, index=target.unsqueeze(1)) + nll_loss = nll_loss.squeeze(1) + smooth_loss = -logprobs.mean(dim=-1) + loss = self.confidence * nll_loss + self.smoothing * smooth_loss + #print("================",type(x),x.size()) + #print("------------------",type(target),target.size(),target) + return loss.mean() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/smoothing_tocpu.py b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/smoothing_tocpu.py new file mode 100644 index 0000000..cbd8772 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/smoothing_tocpu.py @@ -0,0 +1,51 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +import torch.nn as nn + +class LabelSmoothing(nn.Module): + """ + NLL loss with label smoothing. + """ + def __init__(self, smoothing=0.0): + """ + Constructor for the LabelSmoothing module. + + :param smoothing: label smoothing factor + """ + super(LabelSmoothing, self).__init__() + self.confidence = 1.0 - smoothing + self.smoothing = smoothing + # print("----------------------LabelSooothing.__init__") + # def __call__(self,x,target): + # print("----------------------LabelSooothing.__call__") + # return self.forward(self,x,target) + + def forward(self, x, target): + device_x = x.device + device_target = target.device + x = x.to("cpu") + target = target.to("cpu") + logprobs = torch.nn.functional.log_softmax(x, dim=-1) + + nll_loss = -logprobs.gather(dim=-1, index=target.unsqueeze(1)) + nll_loss = nll_loss.squeeze(1) + smooth_loss = -logprobs.mean(dim=-1) + loss = self.confidence * nll_loss + self.smoothing * smooth_loss + #print("================",type(x),x.size()) + #print("------------------",type(target),target.size(),target) + + x = x.to(device_x) + target = target.to(device_target) + return loss.mean() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/training.py b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/training.py new file mode 100644 index 0000000..905d973 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/training.py @@ -0,0 +1,534 @@ +# Copyright (c) 2018-2019, NVIDIA CORPORATION +# Copyright (c) 2017- Facebook, Inc +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import os +import time +import numpy as np +import torch +import torch.nn as nn +from torch.autograd import Variable +from . import logger as log +from . import resnet as nvmodels +from . import utils +import dllogger +try: + #from apex.parallel import DistributedDataParallel as DDP #可以采用pytorch torch.distributed + from apex.fp16_utils import * + from apex import amp +except ImportError: + raise ImportError( + "Please install apex from https://www.github.com/nvidia/apex to run this example." + ) + +ACC_METADATA = {'unit': '%','format': ':.2f'} +IPS_METADATA = {'unit': 'img/s', 'format': ':.2f'} +TIME_METADATA = {'unit': 's', 'format': ':.5f'} +LOSS_METADATA = {'format': ':.5f'} + + +class ModelAndLoss(nn.Module): + def __init__(self, + arch, + loss, + pretrained_weights=None, + cuda=True, + fp16=False): + super(ModelAndLoss, self).__init__() + self.arch = arch + + print("=> creating model '{}'".format(arch)) + model = nvmodels.build_resnet(arch[0], arch[1]) + if pretrained_weights is not None: + print("=> using pre-trained model from a file '{}'".format(arch)) + model.load_state_dict(pretrained_weights) + + if cuda: + model = model.cuda() + if fp16: + model = network_to_half(model) + + # define loss function (criterion) and optimizer + criterion = loss() + + if cuda: + criterion = criterion.cuda() + + self.model = model + self.loss = criterion + + def forward(self, data, target): + output = self.model(data) + loss = self.loss(output, target) + + return loss, output + + def distributed(self): + #self.model = DDP(self.model) + return + + def load_model_state(self, state): + if not state is None: + self.model.load_state_dict(state) + + +def get_optimizer(parameters, + fp16, + lr, + momentum, + weight_decay, + nesterov=False, + state=None, + static_loss_scale=1., + dynamic_loss_scale=False, + bn_weight_decay=False): + + if bn_weight_decay: + print(" ! Weight decay applied to BN parameters ") + optimizer = torch.optim.SGD([v for n, v in parameters], + lr, + momentum=momentum, + weight_decay=weight_decay, + nesterov=nesterov) + else: + print(" ! Weight decay NOT applied to BN parameters ") + bn_params = [v for n, v in parameters if 'bn' in n] + rest_params = [v for n, v in parameters if not 'bn' in n] + print(len(bn_params)) + print(len(rest_params)) + optimizer = torch.optim.SGD([{ + 'params': bn_params, + 'weight_decay': 0 + }, { + 'params': rest_params, + 'weight_decay': weight_decay + }], + lr, + momentum=momentum, + weight_decay=weight_decay, + nesterov=nesterov) + if fp16: + optimizer = FP16_Optimizer(optimizer, + static_loss_scale=static_loss_scale, + dynamic_loss_scale=dynamic_loss_scale, + verbose=False) + + if not state is None: + optimizer.load_state_dict(state) + + return optimizer + + +def lr_policy(lr_fn, logger=None): + if logger is not None: + logger.register_metric('lr', + log.LR_METER(), + verbosity=dllogger.Verbosity.VERBOSE) + + def _alr(optimizer, iteration, epoch): + lr = lr_fn(iteration, epoch) + + if logger is not None: + logger.log_metric('lr', lr) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + + return _alr + + +def lr_step_policy(base_lr, steps, decay_factor, warmup_length, logger=None): + def _lr_fn(iteration, epoch): + if epoch < warmup_length: + lr = base_lr * (epoch + 1) / warmup_length + else: + lr = base_lr + for s in steps: + if epoch >= s: + lr *= decay_factor + return lr + + return lr_policy(_lr_fn, logger=logger) + + +def lr_linear_policy(base_lr, warmup_length, epochs, logger=None): + def _lr_fn(iteration, epoch): + if epoch < warmup_length: + lr = base_lr * (epoch + 1) / warmup_length + else: + e = epoch - warmup_length + es = epochs - warmup_length + lr = base_lr * (1 - (e / es)) + return lr + + return lr_policy(_lr_fn, logger=logger) + + +def lr_cosine_policy(base_lr, warmup_length, epochs, logger=None): + def _lr_fn(iteration, epoch): + if epoch < warmup_length: + lr = base_lr * (epoch + 1) / warmup_length + else: + e = epoch - warmup_length + es = epochs - warmup_length + lr = 0.5 * (1 + np.cos(np.pi * e / es)) * base_lr + return lr + + return lr_policy(_lr_fn, logger=logger) + + +def lr_exponential_policy(base_lr, + warmup_length, + epochs, + final_multiplier=0.001, + logger=None): + es = epochs - warmup_length + epoch_decay = np.power(2, np.log2(final_multiplier) / es) + + def _lr_fn(iteration, epoch): + if epoch < warmup_length: + lr = base_lr * (epoch + 1) / warmup_length + else: + e = epoch - warmup_length + lr = base_lr * (epoch_decay**e) + return lr + + return lr_policy(_lr_fn, logger=logger) + + +def get_train_step(model_and_loss, + optimizer, + fp16, + use_amp=False, + batch_size_multiplier=1): + def _step(input, target, optimizer_step=True): + input_var = Variable(input) + target_var = Variable(target) + loss, output = model_and_loss(input_var, target_var) + if torch.distributed.is_initialized(): + print('utils.reduce_tensor(loss.data)') + reduced_loss = utils.reduce_tensor(loss.data) + else: + reduced_loss = loss.data + + if fp16: + optimizer.backward(loss) + elif use_amp: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + + if optimizer_step: + opt = optimizer.optimizer if isinstance( + optimizer, FP16_Optimizer) else optimizer + for param_group in opt.param_groups: + for param in param_group['params']: + param.grad /= batch_size_multiplier + + optimizer.step() + optimizer.zero_grad() + + torch.cuda.synchronize() + + return reduced_loss + + return _step + + +def train(train_loader, + model_and_loss, + optimizer, + lr_scheduler, + fp16, + logger, + epoch, + use_amp=False, + prof=-1, + batch_size_multiplier=1, + register_metrics=True): + + if register_metrics and logger is not None: + logger.register_metric('train.loss', + log.LOSS_METER(), + verbosity=dllogger.Verbosity.DEFAULT, + metadata=LOSS_METADATA) + logger.register_metric('train.compute_ips', + log.PERF_METER(), + verbosity=dllogger.Verbosity.VERBOSE, + metadata=IPS_METADATA) + logger.register_metric('train.total_ips', + log.PERF_METER(), + verbosity=dllogger.Verbosity.DEFAULT, + metadata=IPS_METADATA) + logger.register_metric('train.data_time', + log.PERF_METER(), + verbosity=dllogger.Verbosity.VERBOSE, + metadata=TIME_METADATA) + logger.register_metric('train.compute_time', + log.PERF_METER(), + verbosity=dllogger.Verbosity.VERBOSE, + metadata=TIME_METADATA) + + step = get_train_step(model_and_loss, + optimizer, + fp16, + use_amp=use_amp, + batch_size_multiplier=batch_size_multiplier) + + model_and_loss.train() + end = time.time() + + optimizer.zero_grad() + + data_iter = enumerate(train_loader) + if logger is not None: + data_iter = logger.iteration_generator_wrapper(data_iter) + if prof > 0: + data_iter = utils.first_n(prof, data_iter) + + for i, (input, target) in data_iter: + bs = input.size(0) + lr_scheduler(optimizer, i, epoch) + data_time = time.time() - end + + optimizer_step = ((i + 1) % batch_size_multiplier) == 0 + loss = step(input, target, optimizer_step=optimizer_step) + + it_time = time.time() - end + + if logger is not None: + logger.log_metric('train.loss', to_python_float(loss), bs) + logger.log_metric('train.compute_ips', + calc_ips(bs, it_time - data_time)) + logger.log_metric('train.total_ips', calc_ips(bs, it_time)) + logger.log_metric('train.data_time', data_time) + logger.log_metric('train.compute_time', it_time - data_time) + + end = time.time() + + +def get_val_step(model_and_loss): + def _step(input, target): + input_var = Variable(input) + target_var = Variable(target) + + with torch.no_grad(): + loss, output = model_and_loss(input_var, target_var) + + prec1, prec5 = utils.accuracy(output.data, target, topk=(1, 5)) + + if torch.distributed.is_initialized(): + reduced_loss = utils.reduce_tensor(loss.data) + prec1 = utils.reduce_tensor(prec1) + prec5 = utils.reduce_tensor(prec5) + else: + reduced_loss = loss.data + + torch.cuda.synchronize() + + return reduced_loss, prec1, prec5 + + return _step + + +def validate(val_loader, + model_and_loss, + fp16, + logger, + epoch, + prof=-1, + register_metrics=True): + if register_metrics and logger is not None: + logger.register_metric('val.top1', + log.ACC_METER(), + verbosity=dllogger.Verbosity.DEFAULT, + metadata=ACC_METADATA) + logger.register_metric('val.top5', + log.ACC_METER(), + verbosity=dllogger.Verbosity.DEFAULT, + metadata=ACC_METADATA) + logger.register_metric('val.loss', + log.LOSS_METER(), + verbosity=dllogger.Verbosity.DEFAULT, + metadata=LOSS_METADATA) + logger.register_metric('val.compute_ips', + log.PERF_METER(), + verbosity=dllogger.Verbosity.VERBOSE, + metadata=IPS_METADATA) + logger.register_metric('val.total_ips', + log.PERF_METER(), + verbosity=dllogger.Verbosity.DEFAULT, + metadata=IPS_METADATA) + logger.register_metric('val.data_time', + log.PERF_METER(), + verbosity=dllogger.Verbosity.VERBOSE, + metadata=TIME_METADATA) + logger.register_metric('val.compute_latency', + log.PERF_METER(), + verbosity=dllogger.Verbosity.VERBOSE, + metadata=TIME_METADATA) + logger.register_metric('val.compute_latency_at100', + log.LAT_100(), + verbosity=dllogger.Verbosity.VERBOSE, + metadata=TIME_METADATA) + logger.register_metric('val.compute_latency_at99', + log.LAT_99(), + verbosity=dllogger.Verbosity.VERBOSE, + metadata=TIME_METADATA) + logger.register_metric('val.compute_latency_at95', + log.LAT_95(), + verbosity=dllogger.Verbosity.VERBOSE, + metadata=TIME_METADATA) + + + step = get_val_step(model_and_loss) + + top1 = log.AverageMeter() + # switch to evaluate mode + model_and_loss.eval() + + end = time.time() + + data_iter = enumerate(val_loader) + if not logger is None: + data_iter = logger.iteration_generator_wrapper(data_iter, val=True) + if prof > 0: + data_iter = utils.first_n(prof, data_iter) + + for i, (input, target) in data_iter: + bs = input.size(0) + data_time = time.time() - end + + loss, prec1, prec5 = step(input, target) + + it_time = time.time() - end + + top1.record(to_python_float(prec1), bs) + if logger is not None: + logger.log_metric('val.top1', to_python_float(prec1), bs) + logger.log_metric('val.top5', to_python_float(prec5), bs) + logger.log_metric('val.loss', to_python_float(loss), bs) + logger.log_metric('val.compute_ips', + calc_ips(bs, it_time - data_time)) + logger.log_metric('val.total_ips', calc_ips(bs, it_time)) + logger.log_metric('val.data_time', data_time) + logger.log_metric('val.compute_latency', it_time - data_time) + logger.log_metric('val.compute_latency_at95', it_time - data_time) + logger.log_metric('val.compute_latency_at99', it_time - data_time) + logger.log_metric('val.compute_latency_at100', it_time - data_time) + + end = time.time() + + return top1.get_val() + + +# Train loop {{{ +def calc_ips(batch_size, time): + world_size = torch.distributed.get_world_size( + ) if torch.distributed.is_initialized() else 1 + tbs = world_size * batch_size + return tbs / time + + +def train_loop(model_and_loss, + optimizer, + lr_scheduler, + train_loader, + val_loader, + epochs, + fp16, + logger, + should_backup_checkpoint, + use_amp=False, + batch_size_multiplier=1, + best_prec1=0, + start_epoch=0, + prof=-1, + skip_training=False, + skip_validation=False, + save_checkpoints=True, + checkpoint_dir='./'): + + prec1 = -1 + + epoch_iter = range(start_epoch, epochs) + for epoch in epoch_iter: + if logger is not None: + logger.start_epoch() + if not skip_training: + train(train_loader, + model_and_loss, + optimizer, + lr_scheduler, + fp16, + logger, + epoch, + use_amp=use_amp, + prof=prof, + register_metrics=epoch == start_epoch, + batch_size_multiplier=batch_size_multiplier) + + if not skip_validation: + prec1, nimg = validate(val_loader, + model_and_loss, + fp16, + logger, + epoch, + prof=prof, + register_metrics=epoch == start_epoch) + if logger is not None: + logger.end_epoch() + + if save_checkpoints and (not torch.distributed.is_initialized() + or torch.distributed.get_rank() == 0): + if not skip_validation: + is_best = logger.metrics['val.top1']['meter'].get_epoch() > best_prec1 + best_prec1 = max(logger.metrics['val.top1']['meter'].get_epoch(), + best_prec1) + else: + is_best = False + best_prec1 = 0 + + if should_backup_checkpoint(epoch): + backup_filename = 'checkpoint-{}.pth.tar'.format(epoch + 1) + else: + backup_filename = None + utils.save_checkpoint( + { + 'epoch': epoch + 1, + 'arch': model_and_loss.arch, + 'state_dict': model_and_loss.model.state_dict(), + 'best_prec1': best_prec1, + 'optimizer': optimizer.state_dict(), + }, + is_best, + checkpoint_dir=checkpoint_dir, + backup_filename=backup_filename) + + +# }}} diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/utils.py b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/utils.py new file mode 100644 index 0000000..a6825a6 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/image_classification/utils.py @@ -0,0 +1,106 @@ +# Copyright (c) 2018-2019, NVIDIA CORPORATION +# Copyright (c) 2017- Facebook, Inc +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import os +import numpy as np +import torch +import shutil +import torch.distributed as dist + + +def should_backup_checkpoint(args): + def _sbc(epoch): + return args.gather_checkpoints and (epoch < 10 or epoch % 10 == 0) + + return _sbc + + +def save_checkpoint(state, + is_best, + filename='checkpoint.pth.tar', + checkpoint_dir='./', + backup_filename=None): + if (not torch.distributed.is_initialized() + ) or torch.distributed.get_rank() == 0: + filename = os.path.join(checkpoint_dir, filename) + print("SAVING {}".format(filename)) + torch.save(state, filename) + if is_best: + shutil.copyfile(filename, + os.path.join(checkpoint_dir, 'model_best.pth.tar')) + if backup_filename is not None: + shutil.copyfile(filename, + os.path.join(checkpoint_dir, backup_filename)) + + +def timed_generator(gen): + start = time.time() + for g in gen: + end = time.time() + t = end - start + yield g, t + start = time.time() + + +def timed_function(f): + def _timed_function(*args, **kwargs): + start = time.time() + ret = f(*args, **kwargs) + return ret, time.time() - start + + return _timed_function + + +def accuracy(output, target, topk=(1, )): + """Computes the precision@k for the specified values of k""" + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def reduce_tensor(tensor): + rt = tensor.clone() + dist.all_reduce(rt, op=dist.ReduceOp.SUM) + rt /= torch.distributed.get_world_size( + ) if torch.distributed.is_initialized() else 1 + return rt + + +def first_n(n, generator): + for i, d in zip(range(n), generator): + yield d diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/main-apex-d76-npu.py b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/main-apex-d76-npu.py new file mode 100644 index 0000000..427011a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/DistributedResnet50/main-apex-d76-npu.py @@ -0,0 +1,1121 @@ +# coding： utf8 + +import argparse +import os +import random +import shutil +import time +import warnings + +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.distributed as dist +import torch.optim +import torch.multiprocessing as mp +import torch.utils.data +import torch.utils.data.distributed +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torchvision.models as models + +from apex import amp +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter + +######################################################################### +#NV 代码移植 +######################################################################### +import image_classification.resnet as nvmodels +from image_classification.smoothing import LabelSmoothingGpu +from image_classification.smoothing import CrossEntropy +from image_classification.mixup import NLLMultiLabelSmooth, MixUpWrapper + +# import image_classification.resnet as models +import image_classification.logger as log + +# from image_classification.smoothing import LabelSmoothing +# from image_classification.mixup import NLLMultiLabelSmooth, MixUpWrapper +# from image_classification.dataloaders import * +from image_classification.training import * +# from image_classification.utils import * +######################################################################### + +''' +####################################### +多P命令参考： +--------------------------------------- +GPU 多P命令参考 +--------------------------------------- +python ./main-apex.py -p 1 -a resnet50 --dist-url 'tcp://127.0.0.1:50000' --dist-backend 'hccl' --multiprocessing-distributed --world-size 1 --batch_size=256 --rank 0 --seed 49 --device 'npu' --gpu=$i useruserdataimagenet .logtest.txt +python ori-resnet50-main-apex.py --addr='10.136.181.51' -j64 -p 1 -a resnet50 --dist-url 'tcp127.0.0.150000' --dist-backend 'hccl' --multiprocessing-distributed --world-size 1 --batch-size=2048 --rank 0 --gpu=$i --seed 49 --device 'npu' /opt/npu/imagenet > ${logfile} & +python ./main-apex-d76.py -p 1 -a resnet50 --dist-url 'tcp://127.0.0.1:50000' --dist-backend 'nccl' --multiprocessing-distributed --world-size 1 -b 2048 --epochs 90 --rank 0 /user/user/data/imagenet > log-nv-parameter-bs2048-epoch90-ampo2-benchmark0-20200722-0932.txt +--------------------------------------- +NPU多P命令参考 +--------------------------------------- +python ./main-apex-d76-npu.py --addr='10.136.181.51' -j256 --lr 2.048 --warmup 8 --label-smoothing 0.1 --mom 0.875 --wd 3.0517578125e-05 --static-loss-scale 128 -p 1 -a resnet50 --dist-url 'tcp://127.0.0.1:50000' --dist-backend 'hccl' --multiprocessing-distributed --world-size 1 -b 4096 --epochs 90 --rank 0 --benchmark 0 --device 'npu' /data/imagenet > 8p-bs4096.txt + +####################################### +单P命令参考： +--------------------------------------- +GPU 单P命令参考 +--------------------------------------- + +--------------------------------------- +NPU 单P命令参考 +--------------------------------------- +python ../../../test_py/precision/d76/main-apex-d76-npu.py -p 1 -a resnet50 --benchmark 1 -b 512 --seed 49 --device 'npu' --checkpoint-freq 10 --checkpoint-nameprefix 'checkpoint-npu-1p-benchmark1-bs512' --gpu 5 /data/imagenet > ./log-npu-nv-pytorch-1p-parameter-benchmark1-bs512-epoch90-ampo2-20200723-1912.txt +python ../../../test_py/precision/d76/main-apex-d76-npu.py -p 1 -a resnet50 --benchmark 0 -b 512 --seed 49 --device 'npu' --checkpoint-freq 10 --checkpoint-nameprefix 'checkpoint-npu-1p-benchmark0-bs512' --gpu 4 /data/imagenet > ./log-npu-nv-pytorch-1p-parameter-benchmark0-bs512-epoch90-ampo2-20200723-1912.txt +####################################### +''' + + +BATCH_SIZE = 512 +OPTIMIZER_BATCH_SIZE=2048 +LOG_STEP = 10 + +model_names = sorted(name for name in models.__dict__ + if name.islower() and not name.startswith("__") + and callable(models.__dict__[name])) + +parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') +parser.add_argument('--data', metavar='DIR', + help='path to dataset') +parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet50', + choices=model_names, + help='model architecture: ' + + ' | '.join(model_names) + + ' (default: resnet18)') +parser.add_argument('-j', '--workers', default=32, type=int, metavar='N', + help='number of data loading workers (default: 4)') +parser.add_argument('--epochs', default=90, type=int, metavar='N', + help='number of total epochs to run') +parser.add_argument('--start-epoch', default=0, type=int, metavar='N', + help='manual epoch number (useful on restarts)') +parser.add_argument('-b', '--batch-size', default=BATCH_SIZE, type=int, + metavar='N', + help='mini-batch size (default: 256), this is the total ' + 'batch size of all GPUs on the current node when ' + 'using Data Parallel or Distributed Data Parallel') + + +############################################################################### +#参数调节 +############################################################################### +#============================================================ +# DeepLearningExamples-master\PyTorch\Classification\ConvNets\resnet50v1.5\training\AMP\DGX1_RN50_AMP_90E.sh +#命令参数 +#============================================================ +# python ./multiproc.py --nproc_per_node 8 +# ./main.py /imagenet +# --data-backend dali-cpu +# --raport-file raport.json +# -j5 +# -p 100 +# （✔）--lr 2.048 +# --optimizer-batch-size 2048 +# （✔）--warmup 8 +# --arch resnet50 +# （卍）-c fanin ---------------------------('--model-config') +# （✔）--label-smoothing 0.1 +# （✔）--lr-schedule cosine +# （✔）--mom 0.875 ------------------------(标准动量优化算法) +# （✔）--wd 3.0517578125e-05 +# （D）--bn_weight_decay,default:false +# （D）--nesterov,default:false +# --workspace ${1:-./} +# -b 2048,8P +# （✔）--amp +# （卍）--static-loss-scale 128 +# --epochs 90 +#============================================================ + +######################################## +#'--lr', '--learning-rate', +######################################## +#---------------------------------------- +# NV set +#---------------------------------------- +#待补充 +#---------------------------------------- +# Pytorch set +#---------------------------------------- +# def adjust_learning_rate(optimizer, epoch, args): +# """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" +# lr = args.lr * (0.1 ** (epoch // 30)) +# for param_group in optimizer.param_groups: +# param_group['lr'] = lr +#---------------------------------------- +parser.add_argument('--lr', '--learning-rate', + #default=0.1, #pytorch + default=2.048, #nvidia + type=float, + metavar='LR', help='initial learning rate', dest='lr') + +parser.add_argument('--momentum', + #default=0.9, #pytorch + default=0.875, #nvidia + type=float, metavar='M', + help='momentum') #标准动量优化算法 +parser.add_argument('--wd', '--weight-decay', + #default=1e-4, #pytorch + default=3.0517578125e-05, #nvidia + type=float, + metavar='W', help='weight decay (default: 1e-4)', + dest='weight_decay') + +# parser.add_argument('--momentum', +# default=0.9, +# type=float, +# metavar='M', +# help='momentum') +# parser.add_argument('--weight-decay', +# '--wd', +# default=1e-4, +# type=float, +# metavar='W', +# help='weight decay (default: 1e-4)') +############################################################### +#nv cmd parameter +############################################################### + +model_configs = nvmodels.resnet_configs.keys() +parser.add_argument('--model-config', + '-c', + metavar='CONF', + default='classic', + choices=model_configs, + help='model configs: ' + ' | '.join(model_configs) + + '(default: classic)') + + + # parser.add_argument('--lr', + # '--learning-rate', + # default=0.1, + # type=float, + # metavar='LR', + # help='initial learning rate') + +#----------------------------------------------------- +# https://pytorch.org/docs/master/optim.html#how-to-adjust-learning-rate +# torch.optim.lr_scheduler.LambdaLr +# torch.optim.lr_scheduler.StepLR +# torch.optim.lr_scheduler.MultiStepLR +# torch.optim.lr_scheduler.ExponentialLR +# torch.optim.lr_sheduler.CosineAnneaingLR +# torch.optim.lr_scheduler.ReduceLROnPlateau +#----------------------------------------------------- +parser.add_argument('--lr-schedule', + default='cosine', + type=str, + metavar='SCHEDULE', + choices=['step', 'linear', 'cosine'], + help='Type of LR schedule: {}, {}, {}'.format( + 'step', 'linear', 'cosine')) +parser.add_argument('--warmup', + default=8, + type=int, + metavar='E', + help='number of warmup epochs') + +parser.add_argument('--label-smoothing', + default=0.0, + type=float, + metavar='S', + help='label smoothing') +#------------------------------------ +#数据增广方式 +#------------------------------------ +parser.add_argument('--mixup', + default=0.0, + type=float, + metavar='ALPHA', + help='mixup alpha') + + +parser.add_argument( + '--bn-weight-decay', + action='store_true', + help= + 'use weight_decay on batch normalization learnable parameters, (default: false)' +) + +parser.add_argument( + '--static-loss-scale', + type=float, + default=128, + help= + 'Static loss scale, positive power of 2 values can improve fp16 convergence.' +) +parser.add_argument( + '--dynamic-loss-scale', + action='store_true', + help='Use dynamic loss scaling. If supplied, this argument supersedes ' + + '--static-loss-scale.') + +parser.add_argument('--nesterov', + action='store_true', + help='use nesterov momentum, (default: false)') +parser.add_argument('--amp', + action='store_true', + help='Run model AMP (automatic mixed precision) mode.') +parser.add_argument('--fp16', + action='store_true', + help='Run model fp16 mode.') +parser.add_argument( + '--workspace', + type=str, + default='./', + metavar='DIR', + help='path to directory where checkpoints will be stored') +parser.add_argument('--raport-file', + default='experiment_raport.json', + type=str, + help='file in which to store JSON experiment raport') +############################################################### + +parser.add_argument('-p', '--print-freq', default=10, type=int, + metavar='N', help='print frequency (default: 10)') +parser.add_argument('--resume', default='', type=str, metavar='PATH', + help='path to latest checkpoint (default: none)') +parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', + help='evaluate model on validation set') +parser.add_argument('--pretrained', dest='pretrained', action='store_true', + help='use pre-trained model') +parser.add_argument('--world-size', default=-1, type=int, + help='number of nodes for distributed training') +parser.add_argument('--rank', default=-1, type=int, + help='node rank for distributed training') +parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, + help='url used to set up distributed training') +parser.add_argument('--dist-backend', default='nccl', type=str, + help='distributed backend') +parser.add_argument('--seed', default=None, type=int, + help='seed for initializing training. ') +parser.add_argument('--gpu', default=None, type=int, + help='GPU id to use.') +parser.add_argument('--multiprocessing-distributed', action='store_true', + help='Use multi-processing distributed training to launch ' + 'N processes per node, which has N GPUs. This is the ' + 'fastest way to use PyTorch for either single node or ' + 'multi node data parallel training') + +parser.add_argument('-bm', '--benchmark', default=0, type=int, + metavar='N', help='set benchmark status (default: 1,run benchmark)') +parser.add_argument('--device', default='npu', type=str, + help='npu or gpu') +parser.add_argument('--device_num', default=-1, type=int, + help='device num') +parser.add_argument('--addr', default='10.136.181.115', type=str, + help='master addr') +parser.add_argument('--checkpoint-nameprefix', default='checkpoint', type=str, + help='checkpoint-nameprefix') +parser.add_argument('--checkpoint-freq', default=10, type=int, + metavar='N', help='checkpoint frequency (default: 0)' + '0: save only one file whitch per epoch;' + 'n: save diff file per n epoch' + '-1:no checkpoint,not support') +best_acc1 = 0 + +#-c,--model-config fanin +###################################################################### +#PyTorch\Classification\ConvNets\main-org.py:352 +# model_and_loss = ModelAndLoss((args.arch, args.model_config), +# loss, +# pretrained_weights=pretrained_weights, +# cuda=True, +# fp16=args.fp16) +###################################################################### + # + # + # # # + # # # + # +###################################################################### +#image_classification\training.py -----> ModelAndLoss.__init__ +#model = models.build_resnet(arch[0], arch[1]) +###################################################################### + # + # + # # # + # # # + # +###################################################################### +# image_classification\resnet.py:build_resnet +# def build_resnet(version, config, verbose=True): +# version = resnet_versions[version] +# config = resnet_configs[config] + +# builder = ResNetBuilder(version, config) +# if verbose: +# print("Version: {}".format(version)) +# print("Config: {}".format(config)) +# model = version['net'](builder, +# version['block'], +# version['expansion'], +# version['layers'], +# version['widths'], +# version['num_classes']) +# return model +###################################################################### +# fanin +# 'resnet50' : { +# 'net' : ResNet, +# 'block' : Bottleneck, +# 'layers' : [3, 4, 6, 3], +# 'widths' : [64, 128, 256, 512], +# 'expansion' : 4, +# 'num_classes' : 1000, +# }, +def nvidia_model_config(args): + model = ResNet(builder, # builder, + Bottleneck, # version['block'], + 4, # version['expansion'], + [3, 4, 6, 3], # version['layers'], + [64, 128, 256, 512], # version['widths'], + 1000) # version['num_classes']) + return model + +def nvidia_logger_init(args): + #if not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0: + if False: + logger = log.Logger(args.print_freq, [ + dllogger.StdOutBackend(dllogger.Verbosity.DEFAULT, + step_format=log.format_step), + dllogger.JSONStreamBackend( + dllogger.Verbosity.VERBOSE, + os.path.join(args.workspace, args.raport_file)) + ]) + + else: + logger = log.Logger(args.print_freq, []) + logger.log_parameter(args.__dict__, verbosity=dllogger.Verbosity.DEFAULT) + args.logger = logger +#--label-smoothing 0.1 命令参数中用到，需调测 +def nvidia_mixup_and_label_smoothing_getlossfunction(args): + #PyTorch\Classification\ConvNets\main-org.py:346 获取loss接口 + loss = nn.CrossEntropyLoss + if args.mixup > 0.0: #mixup命令参数中未用到，暂不调测 + loss = lambda: NLLMultiLabelSmooth(args.label_smoothing) + elif args.label_smoothing > 0.0: #--label-smoothing 0.1 命令参数中用到，需调测 + if args.device == 'npu': + loss = lambda: CrossEntropy(args.label_smoothing) + if args.device == 'gpu': + loss = lambda: LabelSmoothingGpu(args.label_smoothing) + #criterion = loss() + + if args.gpu is not None: + if args.device == 'npu': + loc = 'npu:{}'.format(args.gpu) + criterion = loss().to(loc) + else: + criterion = loss().cuda(args.gpu) + print("nvidia_mixup_and_label_smoothing_getlossfunction return") + return criterion + # model_and_loss = ModelAndLoss((args.arch, args.model_config), + # loss, + # pretrained_weights=pretrained_weights, + # cuda=True, + # fp16=args.fp16) + # #============================================================ + + +#mixup命令参数中未用到，暂不调测 +def nvidia_mixup_get_train_loader_iter(args): + #PyTorch\Classification\ConvNets\main-org.py:378 获取训练loader + #mixup命令参数中未用到，暂不调测 + # train_loader, train_loader_len = get_train_loader(args.data, + # args.batch_size, + # 1000, + # args.mixup > 0.0, + # workers=args.workers, + # fp16=args.fp16) + if args.mixup != 0.0: #mixup命令参数中未用到，暂不调测 + train_loader = MixUpWrapper(args.mixup, 1000, train_loader) + + #PyTorch\Classification\ConvNets\main-org.py:403 + # optimizer = get_optimizer(list(model_and_loss.model.named_parameters()), + # args.fp16, + # args.lr, + # args.momentum, + # args.weight_decay, + # nesterov=args.nesterov, + # bn_weight_decay=args.bn_weight_decay, + # state=optimizer_state, + # static_loss_scale=args.static_loss_scale, + # dynamic_loss_scale=args.dynamic_loss_scale) + +#input: +# --warmup 8 +# --lr-schedule cosine +#output: +# +# +def nvidia_lr_policy(args): + #PyTorch\Classification\ConvNets\main-org.py:414 + logger=args.logger + if args.lr_schedule == 'step': + lr_policy = lr_step_policy(args.lr, [30, 60, 80], + 0.1, + args.warmup, + logger=logger) + elif args.lr_schedule == 'cosine': + lr_policy = lr_cosine_policy(args.lr, + args.warmup, + args.epochs, + logger=logger) + elif args.lr_schedule == 'linear': + lr_policy = lr_linear_policy(args.lr, + args.warmup, + args.epochs, + logger=logger) + return lr_policy + # criterion = lr_policy + # return criterion + +def seed_everything(seed): + random.seed(seed) + os.environ['PYTHONHASHSEED'] = str(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + cudnn.deterministic = True + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + +def main(): + args = parser.parse_args() + print("===============main()=================") + print(args) + print("===============main()=================") + #print("+++++++++++++++++++++++++++ before set KERNEL_NAME_ID：",os.environ['KERNEL_NAME_ID']) + + os.environ['KERNEL_NAME_ID'] = str(0) + + print("+++++++++++++++++++++++++++KERNEL_NAME_ID:",os.environ['KERNEL_NAME_ID']) + + if args.seed is not None: + random.seed(args.seed) + torch.manual_seed(args.seed) + cudnn.deterministic = True + warnings.warn('You have chosen to seed training. ' + 'This will turn on the CUDNN deterministic setting, ' + 'which can slow down your training considerably! ' + 'You may see unexpected behavior when restarting ' + 'from checkpoints.') + + os.environ['MASTER_ADDR'] = args.addr # '10.136.181.51' + os.environ['MASTER_PORT'] = '29501' + + if args.gpu is not None: + warnings.warn('You have chosen a specific GPU. This will completely ' + 'disable data parallelism.') + + if args.dist_url == "env://" and args.world_size == -1: + args.world_size = int(os.environ["WORLD_SIZE"]) + + args.distributed = args.world_size > 1 or args.multiprocessing_distributed + + if args.device_num != -1: + ngpus_per_node = args.device_num + elif args.device == 'npu': + ngpus_per_node = torch.npu.device_count() + else: + ngpus_per_node = torch.cuda.device_count() + if args.multiprocessing_distributed: + # Since we have ngpus_per_node processes per node, the total world_size + # needs to be adjusted accordingly + args.world_size = ngpus_per_node * args.world_size + # Use torch.multiprocessing.spawn to launch distributed processes: the + # main_worker process function + # The child process uses the environment variables of the parent process, + # we have to set KERNEL_NAME_ID for every proc + if args.device == 'npu': + main_worker(args.gpu, ngpus_per_node, args) + #mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) + else: + mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) + else: + # Simply call main_worker function + main_worker(args.gpu, ngpus_per_node, args) + + +def main_worker(gpu, ngpus_per_node, args): + global best_acc1 + args.gpu = gpu + + print("[gpu id:",args.gpu,"]","+++++++++++++++++++++++++++ before set KERNEL_NAME_ID:",os.environ['KERNEL_NAME_ID']) + os.environ['KERNEL_NAME_ID'] = str(gpu) + print("[gpu id:",args.gpu,"]","+++++++++++++++++++++++++++KERNEL_NAME_ID:",os.environ['KERNEL_NAME_ID']) + + if args.gpu is not None: + print("[gpu id:",args.gpu,"]","Use GPU: {} for training".format(args.gpu)) + + if args.distributed: + if args.dist_url == "env://" and args.rank == -1: + args.rank = int(os.environ["RANK"]) + if args.multiprocessing_distributed: + # For multiprocessing distributed training, rank needs to be the + # global rank among all the processes + args.rank = args.rank# * ngpus_per_node + gpu + + if args.device == 'npu': + dist.init_process_group(backend=args.dist_backend, #init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + else: + dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + # create model + if args.pretrained: + print("[gpu id:",args.gpu,"]","=> using pre-trained model '{}'".format(args.arch)) + model = models.__dict__[args.arch](pretrained=True) + else: + print("[gpu id:",args.gpu,"]","=> creating model '{}'".format(args.arch)) + model = models.__dict__[args.arch]() + + print("[gpu id:",args.gpu,"]","===============main_worker()=================") + print("[gpu id:",args.gpu,"]",args) + print("[gpu id:",args.gpu,"]","===============main_worker()=================") + nvidia_logger_init(args) + #lr_scheduler = 0 + if args.distributed: + # For multiprocessing distributed, DistributedDataParallel constructor + # should always set the single device scope, otherwise, + # DistributedDataParallel will use all available devices. + if args.gpu is not None: + if args.device == 'npu': + loc = 'npu:{}'.format(args.gpu) + torch.npu.set_device(loc) + model = model.to(loc) + else: + torch.cuda.set_device(args.gpu) + model.cuda(args.gpu) + + + # When using a single GPU per process and per + # DistributedDataParallel, we need to divide the batch size + # ourselves based on the total number of GPUs we have + args.batch_size = int(args.batch_size / ngpus_per_node) + args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) + + + #model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + else: + if args.device == 'npu': + loc = 'npu:{}'.format(args.gpu) + model = model.to(loc) + else: + model.cuda() + # DistributedDataParallel will divide and allocate batch_size to all + # available GPUs if device_ids are not set + print("[gpu id:",args.gpu,"]","============================test args.gpu is not None else==========================") + #model = torch.nn.parallel.DistributedDataParallel(model) + elif args.gpu is not None: + print("[gpu id:",args.gpu,"]","============================test elif args.gpu is not None:==========================") + if args.device == 'npu': + loc = 'npu:{}'.format(args.gpu) + torch.npu.set_device(args.gpu) + model = model.to(loc) + else: + torch.cuda.set_device(args.gpu) + model = model.cuda(args.gpu) + + else: + # DataParallel will divide and allocate batch_size to all available GPUs + print("[gpu id:",args.gpu,"]","============================test 1==========================") + if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): + print("[gpu id:",args.gpu,"]","============================test 2==========================") + #model.features = torch.nn.DataParallel(model.features) + #model.cuda() + else: + print("[gpu id:",args.gpu,"]","============================test 3==========================") + if args.device == 'npu': + loc = 'npu:{}'.format(args.gpu) + #model = torch.nn.DataParallel(model).to(loc) + else: + #model = torch.nn.DataParallel(model).cuda() + print("before : model = torch.nn.DataParallel(model).cuda()") + + # optimizer = torch.optim.SGD(model.parameters(), args.lr, + # momentum=args.momentum, + # weight_decay=args.weight_decay) + model_state = None + optimizer_state = None + optimizer = get_optimizer(list(model.named_parameters()), + args.fp16, + args.lr, + args.momentum, + args.weight_decay, + nesterov=args.nesterov, + bn_weight_decay=args.bn_weight_decay, + state=optimizer_state, + static_loss_scale=args.static_loss_scale, + dynamic_loss_scale=args.dynamic_loss_scale) + lr_scheduler = nvidia_lr_policy(args) + ############################### + #混合精度 + #model, optimizer = amp.initialize(model, optimizer, opt_level="O1",loss_scale = 2.0,verbosity=1) + model, optimizer = amp.initialize(model, optimizer, opt_level="O2",loss_scale = 1024,verbosity=1) + #model, optimizer = amp.initialize(model, optimizer, opt_level="O1",verbosity=1) + ############################### + + if args.distributed: + # For multiprocessing distributed, DistributedDataParallel constructor + # should always set the single device scope, otherwise, + # DistributedDataParallel will use all available devices. + if args.gpu is not None: + # if args.device == 'npu': + # loc = 'npu:{}'.format(args.gpu) + # torch.npu.set_device(loc) + # model = model.to(loc) + # else: + # torch.cuda.set_device(args.gpu) + # model.cuda(args.gpu) + + + # When using a single GPU per process and per + # DistributedDataParallel, we need to divide the batch size + # ourselves based on the total number of GPUs we have + # args.batch_size = int(args.batch_size / ngpus_per_node) + # args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) + + #model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], broadcast_buffers=False) + + else: + # if args.device == 'npu': + # loc = 'npu:{}'.format(args.gpu) + # model = model.to(loc) + # else: + # model.cuda() + # DistributedDataParallel will divide and allocate batch_size to all + # available GPUs if device_ids are not set + print("[gpu id:",args.gpu,"]","============================test args.gpu is not None else==========================") + model = torch.nn.parallel.DistributedDataParallel(model) + elif args.gpu is not None: + print("[gpu id:",args.gpu,"]","============================test elif args.gpu is not None:==========================") + # if args.device == 'npu': + # loc = 'npu:{}'.format(args.gpu) + # torch.npu.set_device(args.gpu) + # model = model.to(loc) + # else: + # torch.cuda.set_device(args.gpu) + # model = model.cuda(args.gpu) + + else: + # DataParallel will divide and allocate batch_size to all available GPUs + print("[gpu id:",args.gpu,"]","============================test 1==========================") + if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): + print("[gpu id:",args.gpu,"]","============================test 2==========================") + model.features = torch.nn.DataParallel(model.features) + model.cuda() + else: + print("[gpu id:",args.gpu,"]","============================test 3==========================") + if args.device == 'npu': + loc = 'npu:{}'.format(args.gpu) + model = torch.nn.DataParallel(model).to(loc) + else: + model = torch.nn.DataParallel(model).cuda() + + # define loss function (criterion) and optimizer + if args.device == 'npu': + loc = 'npu:{}'.format(args.gpu) + criterion = nn.CrossEntropyLoss().to(loc) + else: + criterion = nn.CrossEntropyLoss().cuda(args.gpu) + criterion = nvidia_mixup_and_label_smoothing_getlossfunction(args)#需增加设备类型参数(npu/gpu) + print("criterion = nvidia_mixup_and_label_smoothing_getlossfunction(args)") + # optionally resume from a checkpoint + if args.resume: + if os.path.isfile(args.resume): + print("[gpu id:",args.gpu,"]","=> loading checkpoint '{}'".format(args.resume)) + if args.gpu is None: + checkpoint = torch.load(args.resume) + else: + # Map model to be loaded to specified single gpu. + if args.device == 'npu': + loc = 'npu:{}'.format(args.gpu) + else: + loc = 'cuda:{}'.format(args.gpu) + checkpoint = torch.load(args.resume, map_location=loc) + args.start_epoch = checkpoint['epoch'] + best_acc1 = checkpoint['best_acc1'] + if args.gpu is not None: + # best_acc1 may be from a checkpoint from a different GPU + best_acc1 = best_acc1.to(args.gpu) + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + print("[gpu id:",args.gpu,"]","=> loaded checkpoint '{}' (epoch {})" + .format(args.resume, checkpoint['epoch'])) + else: + print("[gpu id:",args.gpu,"]","=> no checkpoint found at '{}'".format(args.resume)) + + cudnn.benchmark = True + + # Data loading code + traindir = os.path.join(args.data, 'train') + valdir = os.path.join(args.data, 'val') + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + + train_dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ])) + + if args.distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), + num_workers=args.workers, pin_memory=False, sampler=train_sampler, drop_last=True) + + val_loader = torch.utils.data.DataLoader( + datasets.ImageFolder(valdir, transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + normalize, + ])), + batch_size=args.batch_size, shuffle=True, + num_workers=args.workers, pin_memory=False, drop_last=True) + + if args.evaluate: + validate(val_loader, model, criterion, args) + return + + #if not args.multiprocessing_distributed or (args.multiprocessing_distributed + # and args.rank % ngpus_per_node == 0): + # pstxtpath = 'echo "=================================" >> "./ps/ps-before-epoch-0-`date \"+%Y%m%d-%H%M%S\"`.txt"' + # os.system(pstxtpath) + # pstxtpath = 'ps -aux |grep python >> "./ps/ps-before-epoch-0-`date \"+%Y%m%d-%H%M%S\"`.txt"' + # os.system(pstxtpath) + # print(pstxtpath) + # pstxtpath = 'echo "=================================" >> "./ps/ps-before-epoch-0-`date \"+%Y%m%d-%H%M%S\"`.txt"' + # os.system(pstxtpath) + for epoch in range(args.start_epoch, args.epochs): + if args.distributed: + train_sampler.set_epoch(epoch) + #adjust_learning_rate(optimizer, epoch, args) + + # train for one epoch + #train(train_loader, model, criterion, optimizer, epoch, args,ngpus_per_node, + acc1 = train(train_loader, model, criterion, optimizer, epoch, args,ngpus_per_node, + lr_scheduler) + #if not args.multiprocessing_distributed or (args.multiprocessing_distributed + # and args.rank % ngpus_per_node == 0): + #os.system(' ps -aux |grep python > "ps-`date \"+%Y%m%d-%H%M%S\"`.txt"') + # pstxtpath = ' ps -aux |grep python >> "./ps/ps-epoch-'+str(epoch)+'-`date \"+%Y%m%d-%H%M%S\"`.txt"' + # os.system(pstxtpath) + # pstxtpath = ' echo "==============================" >> "./ps/ps-epoch-'+str(epoch)+'-`date \"+%Y%m%d-%H%M%S\"`.txt"' + # os.system(pstxtpath) + #break + + # evaluate on validation set + acc1 = validate(val_loader, model, criterion, args,ngpus_per_node) + + # remember best acc@1 and save checkpoint + is_best = acc1 > best_acc1 + best_acc1 = max(acc1, best_acc1) + + if args.device == 'gpu': + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer' : optimizer.state_dict(), + }, is_best) + elif args.device == 'npu': + #保存恢复点 + if (not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0 and epoch == args.epochs - 1) ): + #单P情况，每个epoch均保存checkpoint文件 + #多P情况，仅最后一个epoch，保存rank 0的checkpoint文件 + + filename = args.checkpoint_nameprefix + ".pth.tar" + + #print("=================begin save checkpoint======================") + #modeltmp = model + #modeltmp = modeltmp.to("cpu") + modeltmp = model.cpu() + #保留最后一个epoch的checkpoint，防止异常退出 + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': modeltmp.state_dict(), + #'state_dict': model, + 'best_acc1': best_acc1.to("cpu"), + #'optimizer' : optimizer.state_dict(), + }, is_best.to("cpu"),filename=filename) + + if (epoch == (args.epochs - 1)) or ((args.checkpoint_freq > 0) and (((epoch+1) % args.checkpoint_freq) == 0)): + #保留每个freq的checkpoint，共epochs/freq个checkpoint文件 + #最后一个epoch保存独立的checkpoint文件 + #每隔freq个epoch保存一个checkpoint文件 + filename=args.checkpoint_nameprefix + "-epoch"+str(epoch) + ".pth.tar" + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': modeltmp.state_dict(), + #'state_dict': model, + 'best_acc1': best_acc1.to("cpu"), + #'optimizer' : optimizer.state_dict(), + }, is_best.to("cpu"),filename=filename) + + + loc = 'npu:{}'.format(args.gpu) + modeltmp.to(loc) + #print("=================end save checkpoint======================") + + +def train(train_loader, model, criterion, optimizer, epoch, args,ngpus_per_node, + lr_scheduler): + batch_time = AverageMeter('Time', ':6.3f') + data_time = AverageMeter('Data', ':6.3f') + losses = AverageMeter('Loss', ':6.4f') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(train_loader), + [batch_time, data_time, losses, top1, top5], + prefix="Epoch: [{}]".format(epoch)) + + # switch to train mode + model.train() + + end = time.time() + + if args.benchmark == 1 : + optimizer.zero_grad() + for i, (images, target) in enumerate(train_loader): + # measure data loading time + data_time.update(time.time() - end) + + lr_scheduler(optimizer, i, epoch) + + if args.device == 'npu': + loc = 'npu:{}'.format(args.gpu) + target = target.to(torch.int32).to(loc, non_blocking=False) + images = images.to(loc, non_blocking=False) + else: + images = images.cuda(args.gpu, non_blocking=False) + target = target.cuda(args.gpu, non_blocking=False) + + # compute output + output = model(images) + + #if not args.multiprocessing_distributed or (args.multiprocessing_distributed + # and args.rank % ngpus_per_node == 0): + # print("before:loss = criterion(output, target)") + + loss = criterion(output, target) + + #if not args.multiprocessing_distributed or (args.multiprocessing_distributed + # and args.rank % ngpus_per_node == 0): + # print("behind:loss = criterion(output, target)") + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # compute gradient and do SGD step + if args.benchmark == 0 : + optimizer.zero_grad() + + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + + if args.benchmark == 0 : + optimizer.step() + elif args.benchmark == 1 : + BATCH_SIZE_multiplier = int(OPTIMIZER_BATCH_SIZE / args.batch_size) + BM_optimizer_step = ((i + 1) % BATCH_SIZE_multiplier) == 0 + if BM_optimizer_step: + #print("==================exec step & zero_grad===================") + for param_group in optimizer.param_groups: + for param in param_group['params']: + param.grad /= BATCH_SIZE_multiplier + optimizer.step() + optimizer.zero_grad() + + # torch.npu.synchronize() + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + progress.display(i) + + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + print("[npu id:",args.gpu,"]", "batch_size:", ngpus_per_node*args.batch_size, 'Time: {:.3f}'.format(batch_time.avg), '* FPS@all {:.3f}'.format( + ngpus_per_node*args.batch_size/batch_time.avg)) + hwlog.remark_print(key=hwlog.FPS, value=' * FPS@all {:.3f}'.format(ngpus_per_node * args.batch_size / batch_time.avg)) + return top1.avg + + +def validate(val_loader, model, criterion, args,ngpus_per_node): + batch_time = AverageMeter('Time', ':6.3f') + losses = AverageMeter('Loss', ':6.4f') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(val_loader), + [batch_time, losses, top1, top5], + prefix='Test: ') + + # switch to evaluate mode + model.eval() + + with torch.no_grad(): + end = time.time() + for i, (images, target) in enumerate(val_loader): + if args.gpu is not None: + if args.device == 'npu': + loc = 'npu:{}'.format(args.gpu) + images = images.to(loc) + else: + images = images.cuda(args.gpu, non_blocking=True) + if args.device == 'npu': + loc = 'npu:{}'.format(args.gpu) + target = target.to(torch.int32).to(loc, non_blocking=True) + else: + target = target.cuda(args.gpu, non_blocking=True) + + # compute output + output = model(images) + loss = criterion(output, target) + target = target.to(torch.int32).to(loc, non_blocking=True) + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + progress.display(i) + + # TODO: this should also be done with the ProgressMeter + if i % args.print_freq == 0: + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + print("[gpu id:",args.gpu,"]",'[AVG-ACC] * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}' + .format(top1=top1, top5=top5)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value="{top1.avg:.3f}".format(top1=top1)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value="{top5.avg:.3f}".format(top5=top5)) + + return top1.avg + + +def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): + torch.save(state, filename) + if is_best: + shutil.copyfile(filename, 'model_best.pth.tar') + + +class AverageMeter(object): + """Computes and stores the average and current value""" + def __init__(self, name, fmt=':f'): + self.name = name + self.fmt = fmt + self.reset() + self.start_count_index = 10 + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.count += n + if self.count > (self.start_count_index * n): + self.sum += val * n + self.avg = self.sum / (self.count - self.start_count_index * n) + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + +class ProgressMeter(object): + def __init__(self, num_batches, meters, prefix=""): + self.batch_fmtstr = self._get_batch_fmtstr(num_batches) + self.meters = meters + self.prefix = prefix + + def display(self, batch): + entries = [self.prefix + self.batch_fmtstr.format(batch)] + entries += [str(meter) for meter in self.meters] + print("[gpu id:",os.environ['KERNEL_NAME_ID'],"]",'\t'.join(entries)) + + def _get_batch_fmtstr(self, num_batches): + num_digits = len(str(num_batches // 1)) + fmt = '{:' + str(num_digits) + 'd}' + return '[' + fmt + '/' + fmt.format(num_batches) + ']' + + +def adjust_learning_rate(optimizer, epoch, args): + """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" + lr = args.lr * (0.1 ** (epoch // 30)) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + + +def accuracy(output, target, topk=(1,)): + """Computes the accuracy over the k top predictions for the specified values of k""" + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +if __name__ == '__main__': + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("pytorch") + config_info = get_model_parameter("pytorch_config") + initinal_data = {"base_lr": 0.1, "dataset": "imagenet", "optimizer": "SGD", "loss_scale": 1024} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + main() + diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/pytorch-resnet50-apex.py b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/pytorch-resnet50-apex.py new file mode 100644 index 0000000..c0d376c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/code/pytorch-resnet50-apex.py @@ -0,0 +1,609 @@ +import argparse +import os +import random +import shutil +import time +import warnings +import math +import numpy as np + +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.distributed as dist +import torch.optim +import torch.multiprocessing as mp +import torch.utils.data +import torch.utils.data.distributed +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torchvision.models as models +import torch.npu + +from apex import amp +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter +''' +python3.7 pytorch-resnet50-apex.py --data /opt/npu/dataset/imagenet --npu 7 -j64 -b512 --lr 0.2 --warmup 5 --epochs 90 --label-smoothing 0.1 --optimizer-batch-size 1024 > batch1024-lr0.2-wd.txt & +''' +BATCH_SIZE = 512 +EPOCHS_SIZE = 100 +TRAIN_STEP = 8000 +LOG_STEP = 1 + +CALCULATE_DEVICE = "npu:7" +PRINT_DEVICE = "cpu" +SOURCE_DIR = "/data/imagenet" + +model_names = sorted(name for name in models.__dict__ + if name.islower() and not name.startswith("__") + and callable(models.__dict__[name])) + +parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') +parser.add_argument('--data', metavar='DIR', default=SOURCE_DIR, + help='path to dataset') +parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet50', + choices=model_names, + help='model architecture: ' + + ' | '.join(model_names) + + ' (default: resnet18)') +parser.add_argument('-j', '--workers', default=32, type=int, metavar='N', + help='number of data loading workers (default: 8)') +parser.add_argument('--epochs', default=EPOCHS_SIZE, type=int, metavar='N', + help='number of total epochs to run') +parser.add_argument('--start-epoch', default=0, type=int, metavar='N', + help='manual epoch number (useful on restarts)') +parser.add_argument('-b', '--batch-size', default=BATCH_SIZE, type=int, + metavar='N', + help='mini-batch size (default: 256), this is the total ' + 'batch size of all GPUs on the current node when ' + 'using Data Parallel or Distributed Data Parallel') +parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, + metavar='LR', help='initial learning rate', dest='lr') +parser.add_argument('--momentum', default=0.9, type=float, metavar='M', + help='momentum') +parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, + metavar='W', help='weight decay (default: 1e-4)', + dest='weight_decay') +parser.add_argument('-p', '--print-freq', default=10, type=int, + metavar='N', help='print frequency (default: 10)') +parser.add_argument('--resume', default='', type=str, metavar='PATH', + help='path to latest checkpoint (default: none)') +parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', + help='evaluate model on validation set') +parser.add_argument('--pretrained', dest='pretrained', action='store_true', + help='use pre-trained model') +parser.add_argument('--world-size', default=-1, type=int, + help='number of nodes for distributed training') +parser.add_argument('--rank', default=-1, type=int, + help='node rank for distributed training') +parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, + help='url used to set up distributed training') +parser.add_argument('--dist-backend', default='nccl', type=str, + help='distributed backend') +parser.add_argument('--seed', default=None, type=int, + help='seed for initializing training. ') +parser.add_argument('--gpu', default=None, type=int, + help='GPU id to use.') +parser.add_argument('--npu', default=None, type=int, + help='NPU id to use.') +parser.add_argument('--multiprocessing-distributed', action='store_true', + help='Use multi-processing distributed training to launch ' + 'N processes per node, which has N GPUs. This is the ' + 'fastest way to use PyTorch for either single node or ' + 'multi node data parallel training') +parser.add_argument('--warmup', + default=0, + type=int, + metavar='E', + help='number of warmup epochs') +parser.add_argument('--label-smoothing', + default=0.0, + type=float, + metavar='S', + help='label smoothing') +parser.add_argument('--optimizer-batch-size', + default=-1, + type=int, + metavar='N', + help= + 'size of a total batch size, for simulating bigger batches using gradient accumulation') + +parser.add_argument( + '--static-loss-scale', + type=float, + default=1, + help= + 'Static loss scale, positive power of 2 values can improve fp16 convergence.') + +best_acc1 = 0 + + +def main(): + args = parser.parse_args() + if args.npu is None: + args.npu = 0 + global CALCULATE_DEVICE + CALCULATE_DEVICE = "npu:{}".format(args.npu) + torch.npu.set_device(CALCULATE_DEVICE) + print("use ", CALCULATE_DEVICE) + + if args.seed is not None: + random.seed(args.seed) + torch.manual_seed(args.seed) + cudnn.deterministic = True + warnings.warn('You have chosen to seed training. ' + 'This will turn on the CUDNN deterministic setting, ' + 'which can slow down your training considerably! ' + 'You may see unexpected behavior when restarting ' + 'from checkpoints.') + + if args.gpu is not None: + warnings.warn('You have chosen a specific GPU. This will completely ' + 'disable data parallelism.') + + if args.dist_url == "env://" and args.world_size == -1: + args.world_size = int(os.environ["WORLD_SIZE"]) + + args.distributed = args.world_size > 1 or args.multiprocessing_distributed + + ngpus_per_node = torch.cuda.device_count() + if args.multiprocessing_distributed: + # Since we have ngpus_per_node processes per node, the total world_size + # needs to be adjusted accordingly + args.world_size = ngpus_per_node * args.world_size + # Use torch.multiprocessing.spawn to launch distributed processes: the + # main_worker process function + mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) + else: + # Simply call main_worker function + main_worker(args.gpu, ngpus_per_node, args) + + +def main_worker(gpu, ngpus_per_node, args): + global best_acc1 + args.gpu = gpu + + if args.gpu is not None: + print("Use GPU: {} for training".format(args.gpu)) + + if args.distributed: + if args.dist_url == "env://" and args.rank == -1: + args.rank = int(os.environ["RANK"]) + if args.multiprocessing_distributed: + # For multiprocessing distributed training, rank needs to be the + # global rank among all the processes + args.rank = args.rank * ngpus_per_node + gpu + dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + # create model + if args.pretrained: + print("=> using pre-trained model '{}'".format(args.arch)) + model = models.__dict__[args.arch](pretrained=True) + else: + print("=> creating model '{}'".format(args.arch)) + model = models.__dict__[args.arch](zero_init_residual=True) + for layer in model.modules(): + if isinstance(layer, nn.Linear): + torch.nn.init.kaiming_normal_(layer.weight, a=math.sqrt(5), ) + if args.distributed: + # For multiprocessing distributed, DistributedDataParallel constructor + # should always set the single device scope, otherwise, + # DistributedDataParallel will use all available devices. + if args.gpu is not None: + torch.cuda.set_device(args.gpu) + model.cuda(args.gpu) + # When using a single GPU per process and per + # DistributedDataParallel, we need to divide the batch size + # ourselves based on the total number of GPUs we have + args.batch_size = int(args.batch_size / ngpus_per_node) + args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + else: + model.cuda() + # DistributedDataParallel will divide and allocate batch_size to all + # available GPUs if device_ids are not set + model = torch.nn.parallel.DistributedDataParallel(model) + elif args.gpu is not None: + torch.cuda.set_device(args.gpu) + model = model.cuda(args.gpu) + else: + # DataParallel will divide and allocate batch_size to all available GPUs + if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): + model.features = torch.nn.DataParallel(model.features) + model.cuda() + else: + #model = torch.nn.DataParallel(model).cuda() + model = model.to(CALCULATE_DEVICE) + + lr_policy = lr_cosine_policy(args.lr, + args.warmup, + args.epochs) + + + # define loss function (criterion) and optimizer + #criterion = nn.CrossEntropyLoss().cuda(args.gpu) + loss = nn.CrossEntropyLoss + if args.label_smoothing > 0.0: + loss = lambda: LabelSmoothing(args.label_smoothing) + criterion = loss().to(CALCULATE_DEVICE) + optimizer = torch.optim.SGD([ + {'params': [param for name, param in model.named_parameters() if name[-4:] == 'bias'], 'weight_decay': 0.0}, + {'params': [param for name, param in model.named_parameters() if name[-4:] != 'bias'], 'weight_decay': args.weight_decay}], + args.lr, + momentum=args.momentum) + + model, optimizer = amp.initialize(model, optimizer, opt_level="O2", loss_scale=1024, verbosity=1) + + # optionally resume from a checkpoint + if args.resume: + if os.path.isfile(args.resume): + print("=> loading checkpoint '{}'".format(args.resume)) + if args.npu is not None: + checkpoint = torch.load(args.resume) + elif args.gpu is None: + checkpoint = torch.load(args.resume) + else: + # Map model to be loaded to specified single gpu. + loc = 'cuda:{}'.format(args.gpu) + checkpoint = torch.load(args.resume, map_location=loc) + args.start_epoch = checkpoint['epoch'] + best_acc1 = checkpoint['best_acc1'] + if args.npu is not None: + best_acc1 = best_acc1.to("npu:{}".format(args.npu)) + elif args.gpu is not None: + # best_acc1 may be from a checkpoint from a different GPU + best_acc1 = best_acc1.to(args.gpu) + model.load_state_dict(checkpoint['state_dict']) + #optimizer.load_state_dict(checkpoint['optimizer']) + print("=> loaded checkpoint '{}' (epoch {})" + .format(args.resume, checkpoint['epoch'])) + else: + print("=> no checkpoint found at '{}'".format(args.resume)) + + cudnn.benchmark = True + + # Data loading code + traindir = os.path.join(args.data, 'train') + valdir = os.path.join(args.data, 'val') + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + + train_dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ])) + + if args.distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), + num_workers=args.workers, pin_memory=True, sampler=train_sampler) + + val_loader = torch.utils.data.DataLoader( + datasets.ImageFolder(valdir, transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + normalize, + ])), + batch_size=args.batch_size, shuffle=True, + num_workers=args.workers, pin_memory=True) + + if args.evaluate: + validate(val_loader, model, criterion, args) + return + + for epoch in range(args.start_epoch, args.epochs): + if args.distributed: + train_sampler.set_epoch(epoch) + #adjust_learning_rate(optimizer, epoch, args) + lr_policy(optimizer, 0, epoch) + # train for one epoch + train(train_loader, model, criterion, optimizer, epoch, args) + + # evaluate on validation set + acc1 = validate(val_loader, model, criterion, args) + + # remember best acc@1 and save checkpoint + is_best = acc1 > best_acc1 + best_acc1 = max(acc1, best_acc1) + file_name = "checkpoint_npu{}".format(args.npu) + modeltmp = model.cpu() + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': modeltmp.state_dict(), + # 'state_dict': model, + 'best_acc1': best_acc1.to("cpu"), + # 'optimizer' : optimizer.state_dict(), + }, is_best.to("cpu"), file_name) + modeltmp.to(CALCULATE_DEVICE) + +def train(train_loader, model, criterion, optimizer, epoch, args): + if args.optimizer_batch_size < 0: + batch_size_multiplier = 1 + else: + tbs = 1 * args.batch_size + if args.optimizer_batch_size % tbs != 0: + print( + "Warning: simulated batch size {} is not divisible by actual batch size {}" + .format(args.optimizer_batch_size, tbs)) + batch_size_multiplier = int(args.optimizer_batch_size / tbs) + print("BSM: {}".format(batch_size_multiplier)) + + batch_time = AverageMeter('Time', ':6.3f') + data_time = AverageMeter('Data', ':6.3f') + losses = AverageMeter('Loss', ':.4e') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(train_loader), + [batch_time, data_time, losses, top1, top5], + prefix="Epoch: [{}]".format(epoch)) + + # switch to train mode + model.train() + optimizer.zero_grad() + end = time.time() + for i, (images, target) in enumerate(train_loader): + #with torch.autograd.profiler.profile() as prof: + # measure data loading time + data_time.update(time.time() - end) + + if args.gpu is not None: + images = images.cuda(args.gpu, non_blocking=True) + #target = target.cuda(args.gpu, non_blocking=True) + #if 'npu' in CALCULATE_DEVICE: + # target = target.to(torch.int32) + images = images.to(CALCULATE_DEVICE, non_blocking=True) + if args.label_smoothing == 0.0: + target = target.to(torch.int32).to(CALCULATE_DEVICE, non_blocking=True) + # compute output + output = model(images) + loss = criterion(output, target) + + if args.label_smoothing > 0.0: + target = target.to(torch.int32).to(CALCULATE_DEVICE, non_blocking=True) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # compute gradient and do SGD step + + #loss.backward() + ############################### + with amp.scale_loss(loss, optimizer) as scaled_loss: + #print("middle") + scaled_loss.backward() + optimizer_step = ((i + 1) % batch_size_multiplier) == 0 + if optimizer_step: + if batch_size_multiplier != 1: + for param_group in optimizer.param_groups: + for param in param_group['params']: + param.grad /= batch_size_multiplier + optimizer.step() + optimizer.zero_grad() + + if i % LOG_STEP == 0: + progress.display(i) + #print(prof.key_averages().table(sort_by="self_cpu_time_total")) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + if i == TRAIN_STEP: + break + + +def validate(val_loader, model, criterion, args): + batch_time = AverageMeter('Time', ':6.3f') + losses = AverageMeter('Loss', ':.4e') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(val_loader), + [batch_time, losses, top1, top5], + prefix='Test: ') + + # switch to evaluate mode + model.eval() + + with torch.no_grad(): + end = time.time() + for i, (images, target) in enumerate(val_loader): + #with torch.autograd.profiler.profile() as prof: + if args.gpu is not None: + images = images.cuda(args.gpu, non_blocking=True) + #target = target.cuda(args.gpu, non_blocking=True) + #if 'npu' in CALCULATE_DEVICE: + # target = target.to(torch.int32) + images = images.to(CALCULATE_DEVICE, non_blocking=True) + if args.label_smoothing == 0.0: + target = target.to(torch.int32).to(CALCULATE_DEVICE, non_blocking=True) + + # compute output + output = model(images) + loss = criterion(output, target) + + if args.label_smoothing > 0.0: + target = target.to(torch.int32).to(CALCULATE_DEVICE, non_blocking=True) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % LOG_STEP == 0: + progress.display(i) + #print(prof.key_averages().table(sort_by="self_cpu_time_total")) + + # TODO: this should also be done with the ProgressMeter + print(' * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}' + .format(top1=top1, top5=top5)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value="{top1.avg:.3f}".format(top1=top1)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value="{top5.avg:.3f}".format(top5=top5)) + return top1.avg + + +def save_checkpoint(state, is_best, filename='checkpoint'): + filename2 = filename + ".pth.tar" + torch.save(state, filename2) + if is_best: + shutil.copyfile(filename2, filename+'model_best.pth.tar') + + +class AverageMeter(object): + """Computes and stores the average and current value""" + def __init__(self, name, fmt=':f'): + self.name = name + self.fmt = fmt + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + +class ProgressMeter(object): + def __init__(self, num_batches, meters, prefix=""): + self.batch_fmtstr = self._get_batch_fmtstr(num_batches) + self.meters = meters + self.prefix = prefix + + def display(self, batch): + entries = [self.prefix + self.batch_fmtstr.format(batch)] + entries += [str(meter) for meter in self.meters] + print('\t'.join(entries)) + current_run_time=str(entries).split("Time")[1].split("Data")[0].strip().split(" ")[0] + args = parser.parse_args() + batch_size = args.batch_size + if "Epoch" in self.prefix: + if float(current_run_time) > 0: + FPS = int(batch_size)/float(current_run_time) + hwlog.remark_print(key=hwlog.FPS, value=float(FPS)) + + def _get_batch_fmtstr(self, num_batches): + num_digits = len(str(num_batches // 1)) + fmt = '{:' + str(num_digits) + 'd}' + return '[' + fmt + '/' + fmt.format(num_batches) + ']' + + +def adjust_learning_rate(optimizer, epoch, args): + """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" + lr = args.lr * (0.1 ** (epoch // 30)) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + + +def accuracy(output, target, topk=(1,)): + """Computes the accuracy over the k top predictions for the specified values of k""" + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + +class LabelSmoothing(nn.Module): + """ + NLL loss with label smoothing. + """ + def __init__(self, smoothing=0.0): + """ + Constructor for the LabelSmoothing module. + + :param smoothing: label smoothing factor + """ + super(LabelSmoothing, self).__init__() + self.confidence = 1.0 - smoothing + self.smoothing = smoothing + + def forward(self, x, target): + logprobs = torch.nn.functional.log_softmax(x, dim=-1).to("cpu") + nll_loss = -logprobs.gather(dim=-1, index=target.unsqueeze(1)) + nll_loss = nll_loss.squeeze(1) + smooth_loss = -logprobs.mean(dim=-1) + loss = self.confidence * nll_loss + self.smoothing * smooth_loss + return loss.mean().to(CALCULATE_DEVICE) + +def lr_policy(lr_fn, logger=None): + if logger is not None: + logger.register_metric('lr', + log.LR_METER(), + verbosity=dllogger.Verbosity.VERBOSE) + + def _alr(optimizer, iteration, epoch): + lr = lr_fn(iteration, epoch) + + if logger is not None: + logger.log_metric('lr', lr) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + + return _alr + +def lr_cosine_policy(base_lr, warmup_length, epochs, logger=None): + def _lr_fn(iteration, epoch): + if epoch < warmup_length: + lr = base_lr * (epoch + 1) / warmup_length + else: + e = epoch - warmup_length + es = epochs - warmup_length + lr = 0.5 * (1 + np.cos(np.pi * e / es)) * base_lr + return lr + + return lr_policy(_lr_fn, logger=logger) + +if __name__ == '__main__': + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("pytorch") + config_info = get_model_parameter("pytorch_config") + initinal_data = {"base_lr": 0.1, "dataset": "imagenet", "optimizer": "SGD", "loss_scale": 1024} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/config/npu_set_env.sh b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/config/npu_set_env.sh new file mode 100644 index 0000000..7618849 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/config/npu_set_env.sh @@ -0,0 +1,31 @@ +############## toolkit situation ################ +#export LD_LIBRARY_PATH=/usr/local/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH +#export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/ascend-toolkit/latest/toolkit/tools/ide_daemon/bin/ +#export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ +#export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so +#export PYTHONPATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH + + +############## nnae situation ################ + + +if [ -d /usr/local/Ascend/nnae/latest ];then + export LD_LIBRARY_PATH=/usr/local/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/aarch64_64-linux-gnu:$LD_LIBRARY_PATH + export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/nnae/latest/toolkit/tools/ide_daemon/bin/ + export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp/ + export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so + export PYTHONPATH=/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH +else + export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/aarch64-linux-gnu:$LD_LIBRARY_PATH + export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/ascend-toolkit/latest/toolkit/tools/ide_daemon/bin/ + export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so + export PYTHONPATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH +fi + +# ln -s /usr/local/Ascend/ascend-toolkit/latest/toolkit/bin/adc /usr/local/bin/ + +export SLOG_PRINT_TO_STDOUT=0 +#su HwHiAiUser -c "adc --host 0.0.0.0:22118 --log \"SetLogLevel(0)[error]\" --device 0" + +export TASK_QUEUE_ENABLE=1 \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/scripts/run.sh b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/scripts/run.sh new file mode 100644 index 0000000..dc72992 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/scripts/run.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +rank_size=$1 +yamlPath=$2 +toolsPath=$3 + +# ${rank_size} ${yamlPath} ${currentDir} + + +if [ -f /.dockerenv ];then + CLUSTER=$4 +MPIRUN_ALL_IP="$5" + export CLUSTER=${CLUSTER} +fi +currentDir=$(cd "$(dirname "$0")/.."; pwd) +# 配置环境变量并调用 train 方法 +currtime=`date +%Y%m%d%H%M%S` +currtime=`date +%Y%m%d%H%M%S` +mkdir -p ${currentDir%train*}/train/result/pt_resnet50/training_job_${currtime}/ +train_job_dir=${currentDir%train*}/train/result/pt_resnet50/training_job_${currtime}/ +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir}" +# user env +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 +export SLOG_PRINT_TO_STDOUT=0 +export RANK_TABLE_FILE=${currentDir}/config/${rank_size}p.json + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "pytorch_config") + + + +# device 列表, 若无指定 device 根据 rank_size 顺序选择 +eval device_group=\$device_group_${rank_size}p +if [ x"${device_group}" == x"" ] || [ ${rank_size} -ge 8 ];then + device_group="$(seq 0 "$(expr $rank_size - 1)")" +fi + +# get last device id in device_group, hw log in performance from the dir named last_device_id +device_group_str=`echo ${device_group} | sed 's/ //g'` +first_device_id=`echo ${device_group_str: 0:1}` + +if [ x"${rank_size}" == x"8" ]; then + export WHICH_OP=GEOP + export NEW_GE_FE_ID=1 + export GE_AICPU_FLAG=1 +fi +rank_id=0 + +if [ x"${CLUSTER}" == x"True" ];then + # ln hw log + ln -snf ${train_job_dir}/0/hw_resnet50.log ${train_job_dir} + this_ip=$(hostname -I |awk '{print $1}') + for ip in $MPIRUN_ALL_IP;do + if [ x"$ip" != x"$this_ip" ];then + scp $yamlPath root@$ip:$yamlPath + scp $jsonFilePath root@$ip:$jsonFilePath + fi + done + export PATH=$PATH:/usr/local/mpirun4.0/bin + mpirun -H ${mpirun_ip} \ + --bind-to none -map-by slot\ + --allow-run-as-root \ + --mca btl_tcp_if_exclude lo,docker0,endvnic,virbr0,vethf40501b,docker_gwbridge,br-f42ac38052b4\ + --prefix /usr/local/mpirun4.0/ \ + ${currentDir}/scripts/train.sh 0 $rank_size $yamlPath $currtime ${toolsPath} ${CLUSTER} +else + # ln hw log + ln -snf ${train_job_dir}/${first_device_id}/hw_resnet50.log ${train_job_dir} + for device_id in $device_group;do + #echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] start: train ${device_id} & " >> ${currentDir}/result/main.log + ${currentDir}/scripts/train.sh $device_id $rank_size $yamlPath $currtime ${toolsPath} $rank_id & + let rank_id++ + done +fi +wait + +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir} " + diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/scripts/train.sh b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/scripts/train.sh new file mode 100644 index 0000000..acf621d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/pytorch/scripts/train.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +device_id=$1 +rank_size=$2 +yamlPath=$3 +currtime=$4 +toolsPath=$5 + +currentDir=$(cd "$(dirname "$0")/.."; pwd) + +export REMARK_LOG_FILE=hw_resnet50.log + +mkdir -p ${currentDir%train*}/train/result/pt_resnet50/training_job_${currtime}/ +export train_job_dir=${currentDir%train*}/train/result/pt_resnet50/training_job_${currtime}/ + + +source ${currentDir}/config/npu_set_env.sh + + +benchmark_log_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils +#atlasboost_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +code_dir_path=${currentDir}/code +export PYTHONPATH=$PYTHONPATH:${benchmark_log_path}:${code_dir_path} + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "pytorch_config") + +# user env +export YAML_PATH=$3 +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 +#export HCCL_RANK_TABLE_PATH=${currentDir}/config/${rank_size}p.json +export RANK_SIZE=${rank_size} +export RANK_INDEX=0 +export SLOG_PRINT_TO_STDOUT=0 +export DEVICE_ID=$1 +DEVICE_INDEX=$(( DEVICE_ID + RANK_INDEX * 8 )) +export DEVICE_INDEX=${DEVICE_INDEX} +export MODEL_CKPT_PATH=${train_job_dir}/${device_id}/ckpt${device_id} + +cd ${train_job_dir} +curd_dir=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +export PYTHONPATH=$PYTHONPATH:${curd_dir} + + +if [ x"$6" != x"True" ];then + rank_id=$6 + export RANK_ID=$6 +else + device_id_mo=$(python3.7 -c "import src.tensorflow.mpi_ops as atlasboost;atlasboost.init(); \ + device_id = atlasboost.local_rank();cluster_device_id = str(device_id); \ + atlasboost.set_device_id(device_id);print(atlasboost.rank())") + device_id_mo=`echo $device_id_mo` + rank_id=${device_id_mo##* } + export RANK_ID=${rank_id} + device=${device_id_mo##*deviceid = } + device_id=${device%% phyid=*} + export DEVICE_ID=${device_id} + hccljson=${train_job_dir}/*.json + cp ${hccljson} ${currentDir}/config/${rank_size}p.json +fi + +#mkdir exec path +mkdir -p ${train_job_dir}/${device_id} +cd ${train_job_dir}/${device_id} + +startTime=`date +%Y%m%d-%H:%M:%S` +startTime_s=`date +%s` + + +if [ x"${mode}" == x"evaluate" ];then + eval_data_url="--data=${data_url} --evaluate --resume=${ckpt_path}" +else + eval_data_url="--data=${data_url}" +fi +if [ x"${rank_size}" == x"1" ];then + python3.7 ${currentDir}/code/pytorch-resnet50-apex.py \ + ${eval_data_url} \ + --workers=64 \ + --epochs=${epoches} \ + --batch-size=${batch_size} \ + --learning-rate=${lr} \ + --warmup=5 \ + --label-smoothing=0.1 \ + --optimizer-batch-size=1024 \ + --npu=${device_id} > ${train_job_dir}/train_1p.log 2>&1 + +else + export KERNEL_NAME_ID=${device_id} + rank_id=$6 + python3.7 ${currentDir}/code/DistributedResnet50/main-apex-d76-npu.py \ + --data=${data_url} \ + --addr=$(hostname -I |awk '{print $1}') \ + --seed=49 \ + --workers=184 \ + --learning-rate=${lr} \ + --warmup=8 \ + --label-smoothing=0.1 \ + --mom=0.875 \ + --weight-decay=3.0517578125e-05 \ + --static-loss-scale=128 \ + --print-freq=1 \ + --dist-url='tcp://127.0.0.1:50000' \ + --dist-backend='hccl' \ + --multiprocessing-distributed \ + --world-size=1 \ + --rank=${rank_id} \ + --gpu=${device_id} \ + --benchmark=0 \ + --device='npu' \ + --epochs=${epoches} \ + --device_num=${rank_size} \ + --batch-size=${batch_size} > ${train_job_dir}/train_${rank_size}p.log 2>&1 +fi + +if [ $? -eq 0 ] ; +then + echo ":::ABK 1.0.0 resnet50 train success" + echo ":::ABK 1.0.0 resnet50 train success" >> ${train_job_dir}/train_${rank_size}p.log + echo ":::ABK 1.0.0 resnet50 train success" >> ${train_job_dir}/${device_id}/hw_resnet50.log +else + echo ":::ABK 1.0.0 resnet50 train failed" + echo ":::ABK 1.0.0 resnet50 train failed" >>${train_job_dir}/train_${rank_size}p.log + echo ":::ABK 1.0.0 resnet50 train failed" >> ${train_job_dir}/${device_id}/hw_resnet50.log +fi + +endTime=`date +%Y%m%d-%H:%M:%S` +endTime_s=`date +%s` + +sumTime=$[ $endTime_s - $startTime_s ] + +hour=$(( $sumTime/3600 )) +min=$(( ($sumTime-${hour}*3600)/60 )) +sec=$(( $sumTime-${hour}*3600-${min}*60 )) +echo ":::ABK 1.0.0 resnet50 train total time：${hour}:${min}:${sec}" + +echo ":::ABK 1.0.0 resnet50 train total time：${hour}:${min}:${sec}" >> ${train_job_dir}/${device_id}/hw_resnet50.log + diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/README.md b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/README.md new file mode 100644 index 0000000..c495b88 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/README.md @@ -0,0 +1,38 @@ +# ResNet50_tensorflow训练说明 + +### 1. 模型训练参数配置 + +在train/yaml/ResNet50.yaml中修改相应配置， 配置项含义: + +``` + tensorflow_config: + data_url: 数据集路径 + batch_size: 32 + # 1p/8p, epoches设为90 + epoches: 1 + max_train_steps: 1000 + epochs_between_evals: 1 + iterations_per_loop: 100 + save_checkpoints_steps: 115200 + + # 仅多机执行需要配置: ip1:卡数量1,ip2:卡数量2 + mpirun_ip: 90.90.176.152:8,90.90.176.154:8 + + # docker 镜像名称:版本号 + docker_image: c73:b02 + + # 指定 device id, 多个 id 使用空格分隔, 数量需与 rank_size 相同 + device_group_1p: 0 + device_group_2p: 0 1 + device_group_4p: 0 1 2 3 +``` + +------ + + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/Dockerfile b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/Dockerfile new file mode 100644 index 0000000..db01c07 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/Dockerfile @@ -0,0 +1,36 @@ +FROM nvidia/cuda:9.0-cudnn7-runtime-ubuntu16.04 + + +WORKDIR /research + +RUN apt-get update + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + build-essential \ + git \ + python \ + python-pip + + +ENV HOME /research +ENV PYENV_ROOT $HOME/.pyenv +ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH + + +RUN apt-get install -y python-setuptools + +RUN apt-get install -y python-pip python3-pip virtualenv htop +RUN pip3 install --upgrade numpy scipy sklearn tf-nightly-gpu + + +# Mount data into the docker +ADD . /research/resnet + + +WORKDIR /research/resnet +RUN pip3 install -r official/requirements.txt + + +ENTRYPOINT ["/bin/bash"] + diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/cloud_docker_init.sh b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/cloud_docker_init.sh new file mode 100644 index 0000000..3981b51 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/cloud_docker_init.sh @@ -0,0 +1,47 @@ +#!/bin/sh +currentDir=$(cd "$(dirname "$0")"; pwd) +cd ${currentDir} + +DEVICE_LIST=$@ + +export exec_type={MODE} + +prog_exit() +{ + if [ x"${exec_type}" = xdocker ]; + then + # stop slogd progress + bash /usr/local/Ascend/driver/tools/docker_stop_post_sys.sh + fi +} + +# register prog_exit +trap "prog_exit" SIGTERM + +if [ x"${exec_type}" = xdocker ]; +then + #set env + . ${currentDir}/npu_set_env.sh + + # start slogd progress + mkdir -p /var/log/npu/slog/slogd + /usr/local/Ascend/driver/tools/docker/slogd & + + # start main.sh + ${currentDir}/main.sh ${DEVICE_LIST} & + + # wait slogd stop + flag=1 + while [ $flag -ne 0 ]; + do + sleep 5; + flag=`ps -ef | grep train.sh | grep -v grep | wc -l` + ps -ef >> ${currentDir}/ps.log + echo "" >> ${currentDir}/ps.log + done +else + # start main.sh + su - HwHiAiUser -c ". ${currentDir}/npu_set_env.sh;${currentDir}/main.sh ${DEVICE_LIST}" & + wait +fi + diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/hccl_sample.json b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/hccl_sample.json new file mode 100644 index 0000000..4f0474d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/hccl_sample.json @@ -0,0 +1,13 @@ +{ +"group_count": "1", +"group_list": [ +{ + "group_name": "worker", + "device_count": "{device_count}", + "instance_count": "{instance_count}", + "instance_list": [{instance_list}] +} +], +"status": "completed" +} + diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/main_sample.sh b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/main_sample.sh new file mode 100644 index 0000000..a43856f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/main_sample.sh @@ -0,0 +1,18 @@ +#!/bin/sh +currentDir=$(cd "$(dirname "$0")"; pwd) +cd ${currentDir} + +device_group=$@ +device_num=$# + +touch ${currentDir}/main.log + +for device_phy_id in ${device_group} +do + echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] start: train.sh ${device_phy_id} & " >> ${currentDir}/main.log + ${currentDir}/train.sh ${device_phy_id} & +done + +wait + +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] all train.sh exit " >> ${currentDir}/main.log diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/npu_set_env.sh b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/npu_set_env.sh new file mode 100644 index 0000000..1935bb0 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/npu_set_env.sh @@ -0,0 +1,28 @@ +# main env +export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib/:/usr/local/Ascend/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/ +export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/opp/op_impl/built-in/ai_core/tbe:/code +export PATH=$PATH:/usr/local/Ascend/fwkacllib/ccec_compiler/bin +export ASCEND_OPP_PATH=/usr/local/Ascend/opp +export DDK_VERSION_FLAG=1.60.T17.B830 +export HCCL_CONNECT_TIMEOUT=600 + +# user env +export JOB_ID={JOB_ID} +export RANK_TABLE_FILE={RANK_TABLE_FILE} +export RANK_SIZE={RANK_SIZE} +export RANK_INDEX={RANK_INDEX} +export RANK_ID={RANK_ID} + +# profiling env +export PROFILING_MODE={PROFILING_MODE} +export AICPU_PROFILING_MODE={AICPU_PROFILING_MODE} +export PROFILING_OPTIONS={PROFILING_OPTIONS} +export FP_POINT={FP_POINT} +export BP_POINT={BP_POINT} + +# debug env +#export DUMP_GE_GRAPH=2 +#export DUMP_OP=1 +#export DUMP_OP_LESS=1 +#export PRINT_MODEL=1 +#export TE_PARALLEL_COMPILER=0 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/train_sample.sh b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/train_sample.sh new file mode 100644 index 0000000..b4ee768 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/bin/train_sample.sh @@ -0,0 +1,33 @@ +#!/bin/sh +currentDir=$(cd "$(dirname "$0")"; pwd) +cd ${currentDir} + +PWD=${currentDir} + +device_id=$1 +if [ x"${device_id}" = x ] ; +then + echo "turing train fail" >> ${currentDir}/train_${device_id}.log + exit +else + export DEVICE_ID=${device_id} +fi + +DEVICE_INDEX=$(( DEVICE_ID + RANK_INDEX * 8 )) +export DEVICE_INDEX=${DEVICE_INDEX} + +env > ${currentDir}/env_${device_id}.log + +#mkdir exec path +mkdir -p ${currentDir}/${device_id} +rm -rf ${currentDir}/${device_id}/* +cd ${currentDir}/${device_id} + +#start exec +python3.7 {RUN_ALGORITHM_CMD} {CHECKPOINT_DIR} > ${currentDir}/train_${device_id}.log 2>&1 +if [ $? -eq 0 ] ; +then + echo "turing train success" >> ${currentDir}/train_${device_id}.log +else + echo "turing train fail" >> ${currentDir}/train_${device_id}.log +fi diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/LICENSE b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/LICENSE new file mode 100644 index 0000000..d3da228 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/LICENSE @@ -0,0 +1,203 @@ +Copyright 2015 The TensorFlow Authors. All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015, The TensorFlow Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/README-TPU.md b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/README-TPU.md new file mode 100644 index 0000000..5d888b9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/README-TPU.md @@ -0,0 +1,20 @@ +# Offically Supported TensorFlow 2.1 Models on Cloud TPU + +## Natural Language Processing + +* [bert](nlp/bert): A powerful pre-trained language representation model: + BERT, which stands for Bidirectional Encoder Representations from + Transformers. + [BERT FineTuning with Cloud TPU](https://cloud.google.com/tpu/docs/tutorials/bert-2.x) provides step by step instructions on Cloud TPU training. You can look [Bert MNLI Tensorboard.dev metrics](https://tensorboard.dev/experiment/mIah5lppTASvrHqWrdr6NA) for MNLI fine tuning task. +* [transformer](nlp/transformer): A transformer model to translate the WMT + English to German dataset. + [Training transformer on Cloud TPU](https://cloud.google.com/tpu/docs/tutorials/transformer-2.x) for step by step instructions on Cloud TPU training. + +## Computer Vision + +* [mnist](vision/image_classification): A basic model to classify digits + from the MNIST dataset. See [Running MNIST on Cloud TPU](https://cloud.google.com/tpu/docs/tutorials/mnist-2.x) tutorial and [Tensorboard.dev metrics](https://tensorboard.dev/experiment/mIah5lppTASvrHqWrdr6NA). +* [resnet](vision/image_classification): A deep residual network that can + be used to classify ImageNet's dataset of 1000 classes. + See [Training ResNet on Cloud TPU](https://cloud.google.com/tpu/docs/tutorials/resnet-2.x) tutorial and [Tensorboard.dev metrics](https://tensorboard.dev/experiment/CxlDK8YMRrSpYEGtBRpOhg). +* [retinanet](vision/detection): A fast and powerful object detector. See [Tensorboard.dev training metrics](https://tensorboard.dev/experiment/b8NRnWU3TqG6Rw0UxueU6Q). diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/README.md b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/README.md new file mode 100644 index 0000000..359b9d9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/README.md @@ -0,0 +1,149 @@ +# TensorFlow Official Models + +The TensorFlow official models are a collection of models that use +TensorFlow's high-level APIs. They are intended to be well-maintained, tested, +and kept up to date with the latest TensorFlow API. They should also be +reasonably optimized for fast performance while still being easy to read. + +These models are used as end-to-end tests, ensuring that the models run with the +same or improved speed and performance with each new TensorFlow build. + +## Tensorflow releases + +The master branch of the models are **in development** with TensorFlow 2.x, and +they target the +[nightly binaries](https://github.com/tensorflow/tensorflow#installation) built +from the +[master branch of TensorFlow](https://github.com/tensorflow/tensorflow/tree/master). +You may start from installing with pip: + +```shell +pip3 install tf-nightly +``` + +**Stable versions** of the official models targeting releases of TensorFlow are +available as tagged branches or +[downloadable releases](https://github.com/tensorflow/models/releases). Model +repository version numbers match the target TensorFlow release, such that +[release v2.1.0](https://github.com/tensorflow/models/releases/tag/v2.1.0) are +compatible with +[TensorFlow v2.1.0](https://github.com/tensorflow/tensorflow/releases/tag/v2.1.0). + +If you are on a version of TensorFlow earlier than 1.4, please +[update your installation](https://www.tensorflow.org/install/). + +## Requirements + +Please follow the below steps before running models in this repo: + +1. TensorFlow + [nightly binaries](https://github.com/tensorflow/tensorflow#installation) + +2. If users would like to clone this repo but do not care about change history, +please consider: + + ```shell + export repo_version="master" + git clone -b ${repo_version} https://github.com/tensorflow/models.git --depth=1 + ``` + +3. Add the top-level ***/models*** folder to the Python path with the command: + + ```shell + export PYTHONPATH=$PYTHONPATH:/path/to/models + ``` + + Using Colab: + + ```python + import os + os.environ['PYTHONPATH'] += ":/path/to/models" + ``` + +4. Install dependencies: + + ```shell + pip3 install --user -r official/requirements.txt + ``` + + +To make Official Models easier to use, we are planning to create a pip +installable Official Models package. This is being tracked in +[#917](https://github.com/tensorflow/models/issues/917). + +## Available models + +**NOTE: For Officially Supported TPU models please check [README-TPU](README-TPU.md).** + +**NOTE:** Please make sure to follow the steps in the +[Requirements](#requirements) section. + +### Natural Language Processing + +* [bert](nlp/bert): A powerful pre-trained language representation model: + BERT, which stands for Bidirectional Encoder Representations from + Transformers. +* [transformer](nlp/transformer): A transformer model to translate the WMT English + to German dataset. +* [xlnet](nlp/xlnet): XLNet: Generalized Autoregressive Pretraining for + Language Understanding. + +### Computer Vision + +* [mnist](vision/image_classification): A basic model to classify digits from + the MNIST dataset. +* [resnet](vision/image_classification): A deep residual network that can be + used to classify both CIFAR-10 and ImageNet's dataset of 1000 classes. +* [retinanet](vision/detection): A fast and powerful object detector. + +### Others + +* [ncf](recommendation): Neural Collaborative Filtering model for + recommendation tasks. + +Models that will not update to TensorFlow 2.x stay inside R1 directory: + +* [boosted_trees](r1/boosted_trees): A Gradient Boosted Trees model to + classify higgs boson process from HIGGS Data Set. +* [wide_deep](r1/wide_deep): A model that combines a wide model and deep + network to classify census income data. + +## More models to come! + +We are in the progress to revamp official model garden with TensorFlow 2.0 and +Keras. In the near future, we will bring: + +* State-of-the-art language understanding models: XLNet, GPT2, and more + members in Transformer family. +* Start-of-the-art image classification models: EfficientNet, MnasNet and + variants. +* A set of excellent objection detection models. + +If you would like to make any fixes or improvements to the models, please +[submit a pull request](https://github.com/tensorflow/models/compare). + +## New Models + +The team is actively working to add new models to the repository. Every model +should follow the following guidelines, to uphold the our objectives of +readable, usable, and maintainable code. + +**General guidelines** + +* Code should be well documented and tested. +* Runnable from a blank environment with relative ease. +* Trainable on: single GPU/CPU (baseline), multiple GPUs, TPU +* Compatible with Python 3 (using [six](https://pythonhosted.org/six/) when + being compatible with Python 2 is necessary) +* Conform to [Google Python Style Guide](https://github.com/google/styleguide/blob/gh-pages/pyguide.md) + +**Implementation guidelines** + +These guidelines exist so the model implementations are consistent for better +readability and maintainability. + +* Use [common utility functions](utils) +* Export SavedModel at the end of training. +* Consistent flags and flag-parsing library + ([read more here](utils/flags/guidelines.md)) +* Produce benchmarks and logs ([read more here](utils/logs/guidelines.md)) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/datastore/schema/benchmark_metric.json b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/datastore/schema/benchmark_metric.json new file mode 100644 index 0000000..cc571d4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/datastore/schema/benchmark_metric.json @@ -0,0 +1,56 @@ +[ + { + "description": "The ID of the benchmark run, where this metric should tie to.", + "mode": "REQUIRED", + "name": "run_id", + "type": "STRING" + }, + { + "description": "The name of the metric, which should be descriptive. E.g. training_loss, accuracy.", + "mode": "REQUIRED", + "name": "name", + "type": "STRING" + }, + { + "description": "The unit of the metric. E.g. MB per sec.", + "mode": "NULLABLE", + "name": "unit", + "type": "STRING" + }, + { + "description": "The value of the metric.", + "mode": "NULLABLE", + "name": "value", + "type": "FLOAT" + }, + { + "description": "The timestamp when the metric is recorded.", + "mode": "REQUIRED", + "name": "timestamp", + "type": "TIMESTAMP" + }, + { + "description": "The global step when this metric is recorded.", + "mode": "NULLABLE", + "name": "global_step", + "type": "INTEGER" + }, + { + "description": "Free format metadata for the extra information about the metric.", + "mode": "REPEATED", + "name": "extras", + "type": "RECORD", + "fields": [ + { + "mode": "NULLABLE", + "name": "name", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "STRING" + } + ] + } +] diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/datastore/schema/benchmark_run.json b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/datastore/schema/benchmark_run.json new file mode 100644 index 0000000..58e5ddc --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/datastore/schema/benchmark_run.json @@ -0,0 +1,368 @@ +[ + { + "description": "The UUID of the run for the benchmark.", + "mode": "REQUIRED", + "name": "model_id", + "type": "STRING" + }, + { + "description": "The name of the model, E.g ResNet50, LeNet-5 etc.", + "mode": "REQUIRED", + "name": "model_name", + "type": "STRING" + }, + { + "description": "The date when the test of the model is started", + "mode": "REQUIRED", + "name": "run_date", + "type": "TIMESTAMP" + }, + { + "description": "The unique name for a test by the combination of key parameters, eg batch size, num of GPU, etc. It is hardware independent.", + "mode": "NULLABLE", + "name": "test_id", + "type": "STRING" + }, + { + "description": "The tensorflow version information.", + "fields": [ + { + "description": "Version of the tensorflow. E.g. 1.7.0-rc0", + "mode": "REQUIRED", + "name": "version", + "type": "STRING" + }, + { + "description": "Git Hash of the tensorflow", + "mode": "NULLABLE", + "name": "git_hash", + "type": "STRING" + }, + { + "description": "The channel of the tensorflow binary, eg, nightly, RC, final, custom.", + "mode": "NULLABLE", + "name": "channel", + "type": "STRING" + }, + { + "description": "Identify anything special about the build, eg CUDA 10, NCCL, MKL, etc.", + "mode": "NULLABLE", + "name": "build_type", + "type": "STRING" + } + ], + "mode": "REQUIRED", + "name": "tensorflow_version", + "type": "RECORD" + }, + { + "description": "The arbitrary attribute of the model.", + "fields": [ + { + "description": "The name of the attribute.", + "mode": "REQUIRED", + "name": "name", + "type": "STRING" + }, + { + "description": "The value of the attribute.", + "mode": "NULLABLE", + "name": "value", + "type": "STRING" + } + ], + "mode": "REPEATED", + "name": "attribute", + "type": "RECORD" + }, + { + "description": "Environment variables when the benchmark run is executed.", + "fields": [ + { + "description": "The name of the variable.", + "mode": "REQUIRED", + "name": "name", + "type": "STRING" + }, + { + "description": "The value of the variable.", + "mode": "NULLABLE", + "name": "value", + "type": "STRING" + } + ], + "mode": "REPEATED", + "name": "environment_variable", + "type": "RECORD" + }, + { + "description": "TF Environment variables when the benchmark run is executed.", + "fields": [ + { + "description": "The name of the variable.", + "mode": "REQUIRED", + "name": "name", + "type": "STRING" + }, + { + "description": "The value of the variable.", + "mode": "NULLABLE", + "name": "value", + "type": "STRING" + } + ], + "mode": "REPEATED", + "name": "tensorflow_environment_variables", + "type": "RECORD" + }, + { + "description": "The list of parameters run with the model. It could contain hyperparameters or others.", + "fields": [ + { + "description": "The name of the parameter.", + "mode": "REQUIRED", + "name": "name", + "type": "STRING" + }, + { + "description": "The string value of the parameter.", + "mode": "NULLABLE", + "name": "string_value", + "type": "STRING" + }, + { + "description": "The bool value of the parameter.", + "mode": "NULLABLE", + "name": "bool_value", + "type": "STRING" + }, + { + "description": "The int/long value of the parameter.", + "mode": "NULLABLE", + "name": "long_value", + "type": "INTEGER" + }, + { + "description": "The double/float value of parameter.", + "mode": "NULLABLE", + "name": "float_value", + "type": "FLOAT" + } + ], + "mode": "REPEATED", + "name": "run_parameters", + "type": "RECORD" + }, + { + "description": "The dataset that run with the benchmark.", + "mode": "NULLABLE", + "name": "dataset", + "type": "RECORD", + "fields": [ + { + "description": "The name of the dataset that the model is trained/validated with. E.g ImageNet, mnist.", + "mode": "REQUIRED", + "name": "name", + "type": "STRING" + }, + { + "description": "The arbitrary attribute of the dataset.", + "fields": [ + { + "description": "The name of the attribute.", + "mode": "REQUIRED", + "name": "name", + "type": "STRING" + }, + { + "description": "The value of the attribute.", + "mode": "NULLABLE", + "name": "value", + "type": "STRING" + } + ], + "mode": "REPEATED", + "name": "attribute", + "type": "RECORD" + } + ] + }, + { + "description": "Used to differentiate from AWS, GCE or DGX-1 at a high level", + "mode": "NULLABLE", + "name": "test_environment", + "type": "STRING" + }, + { + "description": "The machine configuration of the benchmark run.", + "mode": "NULLABLE", + "name": "machine_config", + "type": "RECORD", + "fields": [ + { + "description": "The platform information of the benchmark run.", + "mode": "NULLABLE", + "name": "platform_info", + "type": "RECORD", + "fields": [ + { + "description": "Eg: 64bit.", + "mode": "NULLABLE", + "name": "bits", + "type": "STRING" + }, + { + "description": "Eg: ELF.", + "mode": "NULLABLE", + "name": "linkage", + "type": "STRING" + }, + { + "description": "Eg: i386.", + "mode": "NULLABLE", + "name": "machine", + "type": "STRING" + }, + { + "description": "Eg: 3.13.0-76-generic.", + "mode": "NULLABLE", + "name": "release", + "type": "STRING" + }, + { + "description": "Eg: Linux.", + "mode": "NULLABLE", + "name": "system", + "type": "STRING" + }, + { + "description": "Eg: #120-Ubuntu SMP Mon Jan 18 15:59:10 UTC 2016.", + "mode": "NULLABLE", + "name": "version", + "type": "STRING" + } + ] + }, + { + "description": "The CPU information of the benchmark run.", + "mode": "NULLABLE", + "name": "cpu_info", + "type": "RECORD", + "fields": [ + { + "mode": "NULLABLE", + "name": "num_cores", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "num_cores_allowed", + "type": "INTEGER" + }, + { + "description" : "How fast are those CPUs.", + "mode": "NULLABLE", + "name": "mhz_per_cpu", + "type": "FLOAT" + }, + { + "description" : "Additional CPU info, Eg: Intel Ivybridge with HyperThreading (24 cores).", + "mode": "NULLABLE", + "name": "cpu_info", + "type": "STRING" + }, + { + "description" : "What kind of cpu scaling is enabled on the host. Eg performance, ondemand, conservative, mixed.", + "mode": "NULLABLE", + "name": "cpu_governor", + "type": "STRING" + }, + { + "description": "Cache size of the CPUs.", + "mode": "NULLABLE", + "name": "cache_size", + "type": "RECORD", + "fields": [ + { + "mode": "NULLABLE", + "name": "level", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "size", + "type": "INTEGER" + } + ] + } + ] + }, + { + "mode": "NULLABLE", + "name": "gpu_info", + "type": "RECORD", + "fields": [ + { + "mode": "NULLABLE", + "name": "count", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "model", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "cuda_version", + "type": "STRING" + } + ] + }, + { + "description": "The cloud instance inforation if the benchmark run is executed on cloud", + "mode": "NULLABLE", + "name": "cloud_info", + "type": "RECORD", + "fields": [ + { + "description": "The instance type, E.g. n1-standard-4.", + "mode": "NULLABLE", + "name": "instance_type", + "type": "STRING" + }, + { + "description": "The arbitrary attribute of the cloud info.", + "fields": [ + { + "description": "The name of the attribute.", + "mode": "REQUIRED", + "name": "name", + "type": "STRING" + }, + { + "description": "The value of the attribute.", + "mode": "NULLABLE", + "name": "value", + "type": "STRING" + } + ], + "mode": "REPEATED", + "name": "attribute", + "type": "RECORD" + } + ] + }, + { + "mode": "NULLABLE", + "name": "memory_total", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "memory_available", + "type": "STRING" + } + ] + } +] diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/datastore/schema/benchmark_run_status.json b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/datastore/schema/benchmark_run_status.json new file mode 100644 index 0000000..f7ac59e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/datastore/schema/benchmark_run_status.json @@ -0,0 +1,14 @@ +[ + { + "description": "The UUID of the run for the benchmark.", + "mode": "REQUIRED", + "name": "run_id", + "type": "STRING" + }, + { + "description": "The status of the run for the benchmark. Eg, running, failed, success", + "mode": "REQUIRED", + "name": "status", + "type": "STRING" + } +] \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/models/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/models/resnet_cifar_main.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/models/resnet_cifar_main.py new file mode 100644 index 0000000..e2d69b5 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/models/resnet_cifar_main.py @@ -0,0 +1,285 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Runs a ResNet model on the Cifar-10 dataset.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import app +from absl import flags +import numpy as np +import tensorflow as tf +from official.benchmark.models import resnet_cifar_model +from official.utils.flags import core as flags_core +from official.utils.logs import logger +from official.utils.misc import distribution_utils +from official.utils.misc import keras_utils +from official.vision.image_classification.resnet import cifar_preprocessing +from official.vision.image_classification.resnet import common + + +LR_SCHEDULE = [ # (multiplier, epoch to start) tuples + (0.1, 91), (0.01, 136), (0.001, 182) +] + + +def learning_rate_schedule(current_epoch, + current_batch, + batches_per_epoch, + batch_size): + """Handles linear scaling rule and LR decay. + + Scale learning rate at epoch boundaries provided in LR_SCHEDULE by the + provided scaling factor. + + Args: + current_epoch: integer, current epoch indexed from 0. + current_batch: integer, current batch in the current epoch, indexed from 0. + batches_per_epoch: integer, number of steps in an epoch. + batch_size: integer, total batch sized. + + Returns: + Adjusted learning rate. + """ + del current_batch, batches_per_epoch # not used + initial_learning_rate = common.BASE_LEARNING_RATE * batch_size / 128 + learning_rate = initial_learning_rate + for mult, start_epoch in LR_SCHEDULE: + if current_epoch >= start_epoch: + learning_rate = initial_learning_rate * mult + else: + break + return learning_rate + + +class LearningRateBatchScheduler(tf.keras.callbacks.Callback): + """Callback to update learning rate on every batch (not epoch boundaries). + + N.B. Only support Keras optimizers, not TF optimizers. + + Attributes: + schedule: a function that takes an epoch index and a batch index as input + (both integer, indexed from 0) and returns a new learning rate as + output (float). + """ + + def __init__(self, schedule, batch_size, steps_per_epoch): + super(LearningRateBatchScheduler, self).__init__() + self.schedule = schedule + self.steps_per_epoch = steps_per_epoch + self.batch_size = batch_size + self.epochs = -1 + self.prev_lr = -1 + + def on_epoch_begin(self, epoch, logs=None): + if not hasattr(self.model.optimizer, 'learning_rate'): + raise ValueError('Optimizer must have a "learning_rate" attribute.') + self.epochs += 1 + + def on_batch_begin(self, batch, logs=None): + """Executes before step begins.""" + lr = self.schedule(self.epochs, + batch, + self.steps_per_epoch, + self.batch_size) + if not isinstance(lr, (float, np.float32, np.float64)): + raise ValueError('The output of the "schedule" function should be float.') + if lr != self.prev_lr: + self.model.optimizer.learning_rate = lr # lr should be a float here + self.prev_lr = lr + tf.compat.v1.logging.debug( + 'Epoch %05d Batch %05d: LearningRateBatchScheduler ' + 'change learning rate to %s.', self.epochs, batch, lr) + + +def run(flags_obj): + """Run ResNet Cifar-10 training and eval loop using native Keras APIs. + + Args: + flags_obj: An object containing parsed flag values. + + Raises: + ValueError: If fp16 is passed as it is not currently supported. + + Returns: + Dictionary of training and eval stats. + """ + keras_utils.set_session_config( + enable_eager=flags_obj.enable_eager, + enable_xla=flags_obj.enable_xla) + + # Execute flag override logic for better model performance + if flags_obj.tf_gpu_thread_mode: + keras_utils.set_gpu_thread_mode_and_count( + per_gpu_thread_count=flags_obj.per_gpu_thread_count, + gpu_thread_mode=flags_obj.tf_gpu_thread_mode, + num_gpus=flags_obj.num_gpus, + datasets_num_private_threads=flags_obj.datasets_num_private_threads) + common.set_cudnn_batchnorm_mode() + + dtype = flags_core.get_tf_dtype(flags_obj) + if dtype == 'fp16': + raise ValueError('dtype fp16 is not supported in Keras. Use the default ' + 'value(fp32).') + + data_format = flags_obj.data_format + if data_format is None: + data_format = ('channels_first' + if tf.test.is_built_with_cuda() else 'channels_last') + tf.keras.backend.set_image_data_format(data_format) + + strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=flags_obj.distribution_strategy, + num_gpus=flags_obj.num_gpus, + all_reduce_alg=flags_obj.all_reduce_alg, + num_packs=flags_obj.num_packs) + + if strategy: + # flags_obj.enable_get_next_as_optional controls whether enabling + # get_next_as_optional behavior in DistributedIterator. If true, last + # partial batch can be supported. + strategy.extended.experimental_enable_get_next_as_optional = ( + flags_obj.enable_get_next_as_optional + ) + + strategy_scope = distribution_utils.get_strategy_scope(strategy) + + if flags_obj.use_synthetic_data: + distribution_utils.set_up_synthetic_data() + input_fn = common.get_synth_input_fn( + height=cifar_preprocessing.HEIGHT, + width=cifar_preprocessing.WIDTH, + num_channels=cifar_preprocessing.NUM_CHANNELS, + num_classes=cifar_preprocessing.NUM_CLASSES, + dtype=flags_core.get_tf_dtype(flags_obj), + drop_remainder=True) + else: + distribution_utils.undo_set_up_synthetic_data() + input_fn = cifar_preprocessing.input_fn + + train_input_dataset = input_fn( + is_training=True, + data_dir=flags_obj.data_dir, + batch_size=flags_obj.batch_size, + parse_record_fn=cifar_preprocessing.parse_record, + datasets_num_private_threads=flags_obj.datasets_num_private_threads, + dtype=dtype, + # Setting drop_remainder to avoid the partial batch logic in normalization + # layer, which triggers tf.where and leads to extra memory copy of input + # sizes between host and GPU. + drop_remainder=(not flags_obj.enable_get_next_as_optional)) + + eval_input_dataset = None + if not flags_obj.skip_eval: + eval_input_dataset = input_fn( + is_training=False, + data_dir=flags_obj.data_dir, + batch_size=flags_obj.batch_size, + parse_record_fn=cifar_preprocessing.parse_record) + + steps_per_epoch = ( + cifar_preprocessing.NUM_IMAGES['train'] // flags_obj.batch_size) + lr_schedule = 0.1 + if flags_obj.use_tensor_lr: + initial_learning_rate = common.BASE_LEARNING_RATE * flags_obj.batch_size / 128 + lr_schedule = tf.keras.optimizers.schedules.PiecewiseConstantDecay( + boundaries=list(p[1] * steps_per_epoch for p in LR_SCHEDULE), + values=[initial_learning_rate] + + list(p[0] * initial_learning_rate for p in LR_SCHEDULE)) + + with strategy_scope: + optimizer = common.get_optimizer(lr_schedule) + model = resnet_cifar_model.resnet56(classes=cifar_preprocessing.NUM_CLASSES) + model.compile( + loss='sparse_categorical_crossentropy', + optimizer=optimizer, + metrics=(['sparse_categorical_accuracy'] + if flags_obj.report_accuracy_metrics else None), + run_eagerly=flags_obj.run_eagerly) + + train_epochs = flags_obj.train_epochs + + callbacks = common.get_callbacks(steps_per_epoch) + + if not flags_obj.use_tensor_lr: + lr_callback = LearningRateBatchScheduler( + schedule=learning_rate_schedule, + batch_size=flags_obj.batch_size, + steps_per_epoch=steps_per_epoch) + callbacks.append(lr_callback) + + # if mutliple epochs, ignore the train_steps flag. + if train_epochs <= 1 and flags_obj.train_steps: + steps_per_epoch = min(flags_obj.train_steps, steps_per_epoch) + train_epochs = 1 + + num_eval_steps = (cifar_preprocessing.NUM_IMAGES['validation'] // + flags_obj.batch_size) + + validation_data = eval_input_dataset + if flags_obj.skip_eval: + if flags_obj.set_learning_phase_to_train: + # TODO(haoyuzhang): Understand slowdown of setting learning phase when + # not using distribution strategy. + tf.keras.backend.set_learning_phase(1) + num_eval_steps = None + validation_data = None + + if not strategy and flags_obj.explicit_gpu_placement: + # TODO(b/135607227): Add device scope automatically in Keras training loop + # when not using distribition strategy. + no_dist_strat_device = tf.device('/device:GPU:0') + no_dist_strat_device.__enter__() + + history = model.fit(train_input_dataset, + epochs=train_epochs, + steps_per_epoch=steps_per_epoch, + callbacks=callbacks, + validation_steps=num_eval_steps, + validation_data=validation_data, + validation_freq=flags_obj.epochs_between_evals, + verbose=2) + eval_output = None + if not flags_obj.skip_eval: + eval_output = model.evaluate(eval_input_dataset, + steps=num_eval_steps, + verbose=2) + + if not strategy and flags_obj.explicit_gpu_placement: + no_dist_strat_device.__exit__() + + stats = common.build_stats(history, eval_output, callbacks) + return stats + + +def define_cifar_flags(): + common.define_keras_flags(dynamic_loss_scale=False) + + flags_core.set_defaults(data_dir='/tmp/cifar10_data/cifar-10-batches-bin', + model_dir='/tmp/cifar10_model', + epochs_between_evals=10, + batch_size=128) + + +def main(_): + with logger.benchmark_context(flags.FLAGS): + return run(flags.FLAGS) + + +if __name__ == '__main__': + tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO) + define_cifar_flags() + app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/models/resnet_cifar_model.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/models/resnet_cifar_model.py new file mode 100644 index 0000000..1b50738 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/models/resnet_cifar_model.py @@ -0,0 +1,262 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""ResNet56 model for Keras adapted from tf.keras.applications.ResNet50. + +# Reference: +- [Deep Residual Learning for Image Recognition]( + https://arxiv.org/abs/1512.03385) +Adapted from code contributed by BigMoyan. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools +import tensorflow as tf +from tensorflow.python.keras import backend +from tensorflow.python.keras import initializers +from tensorflow.python.keras import layers +from tensorflow.python.keras import regularizers + + +BATCH_NORM_DECAY = 0.997 +BATCH_NORM_EPSILON = 1e-5 +L2_WEIGHT_DECAY = 2e-4 + + +def identity_building_block(input_tensor, + kernel_size, + filters, + stage, + block, + training=None): + """The identity block is the block that has no conv layer at shortcut. + + Arguments: + input_tensor: input tensor + kernel_size: default 3, the kernel size of + middle conv layer at main path + filters: list of integers, the filters of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + block: current block label, used for generating layer names + training: Only used if training keras model with Estimator. In other + scenarios it is handled automatically. + + Returns: + Output tensor for the block. + """ + filters1, filters2 = filters + if backend.image_data_format() == 'channels_last': + bn_axis = 3 + else: + bn_axis = 1 + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + + x = layers.Conv2D(filters1, kernel_size, + padding='same', use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), + name=conv_name_base + '2a')(input_tensor) + x = layers.BatchNormalization( + axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, + name=bn_name_base + '2a')(x, training=training) + x = layers.Activation('relu')(x) + + x = layers.Conv2D(filters2, kernel_size, + padding='same', use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), + name=conv_name_base + '2b')(x) + x = layers.BatchNormalization( + axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, + name=bn_name_base + '2b')(x, training=training) + + x = layers.add([x, input_tensor]) + x = layers.Activation('relu')(x) + return x + + +def conv_building_block(input_tensor, + kernel_size, + filters, + stage, + block, + strides=(2, 2), + training=None): + """A block that has a conv layer at shortcut. + + Arguments: + input_tensor: input tensor + kernel_size: default 3, the kernel size of + middle conv layer at main path + filters: list of integers, the filters of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + block: current block label, used for generating layer names + strides: Strides for the first conv layer in the block. + training: Only used if training keras model with Estimator. In other + scenarios it is handled automatically. + + Returns: + Output tensor for the block. + + Note that from stage 3, + the first conv layer at main path is with strides=(2, 2) + And the shortcut should have strides=(2, 2) as well + """ + filters1, filters2 = filters + if tf.keras.backend.image_data_format() == 'channels_last': + bn_axis = 3 + else: + bn_axis = 1 + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + + x = layers.Conv2D(filters1, kernel_size, strides=strides, + padding='same', use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), + name=conv_name_base + '2a')(input_tensor) + x = layers.BatchNormalization( + axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, + name=bn_name_base + '2a')(x, training=training) + x = layers.Activation('relu')(x) + + x = layers.Conv2D(filters2, kernel_size, padding='same', use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), + name=conv_name_base + '2b')(x) + x = layers.BatchNormalization( + axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, + name=bn_name_base + '2b')(x, training=training) + + shortcut = layers.Conv2D(filters2, (1, 1), strides=strides, use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), + name=conv_name_base + '1')(input_tensor) + shortcut = layers.BatchNormalization( + axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, + name=bn_name_base + '1')(shortcut, training=training) + + x = layers.add([x, shortcut]) + x = layers.Activation('relu')(x) + return x + + +def resnet_block(input_tensor, + size, + kernel_size, + filters, + stage, + conv_strides=(2, 2), + training=None): + """A block which applies conv followed by multiple identity blocks. + + Arguments: + input_tensor: input tensor + size: integer, number of constituent conv/identity building blocks. + A conv block is applied once, followed by (size - 1) identity blocks. + kernel_size: default 3, the kernel size of + middle conv layer at main path + filters: list of integers, the filters of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + conv_strides: Strides for the first conv layer in the block. + training: Only used if training keras model with Estimator. In other + scenarios it is handled automatically. + + Returns: + Output tensor after applying conv and identity blocks. + """ + + x = conv_building_block(input_tensor, kernel_size, filters, stage=stage, + strides=conv_strides, block='block_0', + training=training) + for i in range(size - 1): + x = identity_building_block(x, kernel_size, filters, stage=stage, + block='block_%d' % (i + 1), training=training) + return x + + +def resnet(num_blocks, classes=10, training=None): + """Instantiates the ResNet architecture. + + Arguments: + num_blocks: integer, the number of conv/identity blocks in each block. + The ResNet contains 3 blocks with each block containing one conv block + followed by (layers_per_block - 1) number of idenity blocks. Each + conv/idenity block has 2 convolutional layers. With the input + convolutional layer and the pooling layer towards the end, this brings + the total size of the network to (6*num_blocks + 2) + classes: optional number of classes to classify images into + training: Only used if training keras model with Estimator. In other + scenarios it is handled automatically. + + Returns: + A Keras model instance. + """ + + input_shape = (32, 32, 3) + img_input = layers.Input(shape=input_shape) + + if backend.image_data_format() == 'channels_first': + x = layers.Lambda(lambda x: backend.permute_dimensions(x, (0, 3, 1, 2)), + name='transpose')(img_input) + bn_axis = 1 + else: # channel_last + x = img_input + bn_axis = 3 + + x = layers.ZeroPadding2D(padding=(1, 1), name='conv1_pad')(x) + x = layers.Conv2D(16, (3, 3), + strides=(1, 1), + padding='valid', use_bias=False, + kernel_initializer='he_normal', + kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), + name='conv1')(x) + x = layers.BatchNormalization(axis=bn_axis, + momentum=BATCH_NORM_DECAY, + epsilon=BATCH_NORM_EPSILON, + name='bn_conv1',)(x, training=training) + x = layers.Activation('relu')(x) + + x = resnet_block(x, size=num_blocks, kernel_size=3, filters=[16, 16], + stage=2, conv_strides=(1, 1), training=training) + + x = resnet_block(x, size=num_blocks, kernel_size=3, filters=[32, 32], + stage=3, conv_strides=(2, 2), training=training) + + x = resnet_block(x, size=num_blocks, kernel_size=3, filters=[64, 64], + stage=4, conv_strides=(2, 2), training=training) + + rm_axes = [1, 2] if backend.image_data_format() == 'channels_last' else [2, 3] + x = layers.Lambda(lambda x: backend.mean(x, rm_axes), name='reduce_mean')(x) + x = layers.Dense(classes, + activation='softmax', + kernel_initializer=initializers.RandomNormal(stddev=0.01), + kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), + bias_regularizer=regularizers.l2(L2_WEIGHT_DECAY), + name='fc10')(x) + + inputs = img_input + # Create model. + model = tf.keras.models.Model(inputs, x, name='resnet56') + + return model + + +resnet20 = functools.partial(resnet, num_blocks=3) +resnet32 = functools.partial(resnet, num_blocks=5) +resnet56 = functools.partial(resnet, num_blocks=9) +resnet10 = functools.partial(resnet, num_blocks=110) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/models/resnet_cifar_test.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/models/resnet_cifar_test.py new file mode 100644 index 0000000..6dbb2fa --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/models/resnet_cifar_test.py @@ -0,0 +1,187 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Test the keras ResNet model with Cifar data.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tempfile + +import tensorflow as tf + +from tensorflow.python.eager import context +from tensorflow.python.platform import googletest +from official.benchmark.models import resnet_cifar_main +from official.utils.misc import keras_utils +from official.utils.testing import integration +from official.vision.image_classification.resnet import cifar_preprocessing + + +class KerasCifarTest(googletest.TestCase): + """Unit tests for Keras ResNet with Cifar.""" + + _extra_flags = [ + "-batch_size", "4", + "-train_steps", "1", + "-use_synthetic_data", "true" + ] + _tempdir = None + + def get_temp_dir(self): + if not self._tempdir: + self._tempdir = tempfile.mkdtemp(dir=googletest.GetTempDir()) + return self._tempdir + + @classmethod + def setUpClass(cls): # pylint: disable=invalid-name + super(KerasCifarTest, cls).setUpClass() + resnet_cifar_main.define_cifar_flags() + + def setUp(self): + super(KerasCifarTest, self).setUp() + cifar_preprocessing.NUM_IMAGES["validation"] = 4 + + def tearDown(self): + super(KerasCifarTest, self).tearDown() + tf.io.gfile.rmtree(self.get_temp_dir()) + + def test_end_to_end_no_dist_strat(self): + """Test Keras model with 1 GPU, no distribution strategy.""" + config = keras_utils.get_config_proto_v1() + tf.compat.v1.enable_eager_execution(config=config) + + extra_flags = [ + "-distribution_strategy", "off", + "-model_dir", "keras_cifar_no_dist_strat", + "-data_format", "channels_last", + ] + extra_flags = extra_flags + self._extra_flags + + integration.run_synthetic( + main=resnet_cifar_main.run, + tmp_root=self.get_temp_dir(), + extra_flags=extra_flags + ) + + def test_end_to_end_graph_no_dist_strat(self): + """Test Keras model in legacy graph mode with 1 GPU, no dist strat.""" + extra_flags = [ + "-enable_eager", "false", + "-distribution_strategy", "off", + "-model_dir", "keras_cifar_graph_no_dist_strat", + "-data_format", "channels_last", + ] + extra_flags = extra_flags + self._extra_flags + + integration.run_synthetic( + main=resnet_cifar_main.run, + tmp_root=self.get_temp_dir(), + extra_flags=extra_flags + ) + + def test_end_to_end_1_gpu(self): + """Test Keras model with 1 GPU.""" + config = keras_utils.get_config_proto_v1() + tf.compat.v1.enable_eager_execution(config=config) + + if context.num_gpus() < 1: + self.skipTest( + "{} GPUs are not available for this test. {} GPUs are available". + format(1, context.num_gpus())) + + extra_flags = [ + "-num_gpus", "1", + "-distribution_strategy", "mirrored", + "-model_dir", "keras_cifar_1_gpu", + "-data_format", "channels_last", + ] + extra_flags = extra_flags + self._extra_flags + + integration.run_synthetic( + main=resnet_cifar_main.run, + tmp_root=self.get_temp_dir(), + extra_flags=extra_flags + ) + + def test_end_to_end_graph_1_gpu(self): + """Test Keras model in legacy graph mode with 1 GPU.""" + if context.num_gpus() < 1: + self.skipTest( + "{} GPUs are not available for this test. {} GPUs are available". + format(1, context.num_gpus())) + + extra_flags = [ + "-num_gpus", "1", + "-noenable_eager", + "-distribution_strategy", "mirrored", + "-model_dir", "keras_cifar_graph_1_gpu", + "-data_format", "channels_last", + ] + extra_flags = extra_flags + self._extra_flags + + integration.run_synthetic( + main=resnet_cifar_main.run, + tmp_root=self.get_temp_dir(), + extra_flags=extra_flags + ) + + def test_end_to_end_2_gpu(self): + """Test Keras model with 2 GPUs.""" + config = keras_utils.get_config_proto_v1() + tf.compat.v1.enable_eager_execution(config=config) + + if context.num_gpus() < 2: + self.skipTest( + "{} GPUs are not available for this test. {} GPUs are available". + format(2, context.num_gpus())) + + extra_flags = [ + "-num_gpus", "2", + "-distribution_strategy", "mirrored", + "-model_dir", "keras_cifar_2_gpu", + ] + extra_flags = extra_flags + self._extra_flags + + integration.run_synthetic( + main=resnet_cifar_main.run, + tmp_root=self.get_temp_dir(), + extra_flags=extra_flags + ) + + def test_end_to_end_graph_2_gpu(self): + """Test Keras model in legacy graph mode with 2 GPUs.""" + if context.num_gpus() < 2: + self.skipTest( + "{} GPUs are not available for this test. {} GPUs are available". + format(2, context.num_gpus())) + + extra_flags = [ + "-num_gpus", "2", + "-enable_eager", "false", + "-distribution_strategy", "mirrored", + "-model_dir", "keras_cifar_graph_2_gpu", + ] + extra_flags = extra_flags + self._extra_flags + + integration.run_synthetic( + main=resnet_cifar_main.run, + tmp_root=self.get_temp_dir(), + extra_flags=extra_flags + ) + + +if __name__ == "__main__": + googletest.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/resnet_tf_r1_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/resnet_tf_r1_benchmark.py new file mode 100644 index 0000000..124b755 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/benchmark/resnet_tf_r1_benchmark.py @@ -0,0 +1,259 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Executes CTL benchmarks and accuracy tests.""" +from __future__ import print_function + +import os +import sys +import time +# import pydevd_pycharm +# pydevd_pycharm.settrace('90.253.17.223', port=8008, stdoutToServer=True, stderrToServer=True, suspend=False) +# pylint: disable=g-bad-import-order +from absl import flags +import tensorflow as tf + +#sys.path.append(r"/home/wx933135/0708/ResNet50/tensorflow/code") + +sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)),'../../../../../../utils/atlasboost')) + +from official.r1.resnet import imagenet_main +from official.utils.testing.perfzero_benchmark import PerfZeroBenchmark +from official.utils.testing import benchmark_wrappers +from official.utils.flags import core as flags_core +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter + + +MIN_TOP_1_ACCURACY = 0.76 +MAX_TOP_1_ACCURACY = 0.77 + +flags.DEFINE_integer('iterations_per_loop', 1000,'iterations per loop') +flags.DEFINE_integer('save_checkpoints_steps', 115200,'save checkpoints steps') +FLAGS = flags.FLAGS +sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)),'../../../config')) + + +class CtlBenchmark(PerfZeroBenchmark): + """Base benchmark class with methods to simplify testing.""" + + def __init__(self, output_dir=None, default_flags=None, flag_methods=None): + self.output_dir = output_dir + self.default_flags = default_flags or {} + self.flag_methods = flag_methods or {} + super(CtlBenchmark, self).__init__( + output_dir=self.output_dir, + default_flags=self.default_flags, + flag_methods=self.flag_methods) + + def _report_benchmark(self, + stats, + wall_time_sec, + top_1_max=None, + top_1_min=None, + total_batch_size=None, + log_steps=None, + warmup=1): + """Report benchmark results by writing to local protobuf file. + + Args: + stats: dict returned from keras models with known entries. + wall_time_sec: the during of the benchmark execution in seconds + top_1_max: highest passing level for top_1 accuracy. + top_1_min: lowest passing level for top_1 accuracy. + total_batch_size: Global batch-size. + log_steps: How often the log was created for stats['step_timestamp_log']. + warmup: number of entries in stats['step_timestamp_log'] to ignore. + """ + + metrics = [] + if 'eval_acc' in stats: + metrics.append({ + 'name': 'accuracy_top_1', + 'value': stats['eval_acc'], + 'min_value': top_1_min, + 'max_value': top_1_max + }) + metrics.append({'name': 'eval_loss', 'value': stats['eval_loss']}) + + metrics.append({ + 'name': 'top_1_train_accuracy', + 'value': stats['train_acc'] + }) + metrics.append({'name': 'train_loss', 'value': stats['train_loss']}) + + if (warmup and 'step_timestamp_log' in stats and + len(stats['step_timestamp_log']) > warmup + 1): + # first entry in the time_log is start of step 0. The rest of the + # entries are the end of each step recorded + time_log = stats['step_timestamp_log'] + steps_elapsed = time_log[-1].batch_index - time_log[warmup].batch_index + time_elapsed = time_log[-1].timestamp - time_log[warmup].timestamp + examples_per_sec = total_batch_size * (steps_elapsed / time_elapsed) + metrics.append({'name': 'exp_per_second', 'value': examples_per_sec}) + + if 'avg_exp_per_second' in stats: + metrics.append({ + 'name': 'avg_exp_per_second', + 'value': stats['avg_exp_per_second'] + }) + print("start flags_core.get_nondefault_flags_as_str") + flags_str = flags_core.get_nondefault_flags_as_str() + self.report_benchmark( + iters=-1, + wall_time=wall_time_sec, + metrics=metrics, + extras={'flags': flags_str}) + + +class Resnet50CtlAccuracy(CtlBenchmark): + """Benchmark accuracy tests for ResNet50 in CTL.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + """A benchmark class. + + Args: + output_dir: directory where to output e.g. log files + root_data_dir: directory under which to look for dataset + **kwargs: arbitrary named arguments. This is needed to make the + constructor forward compatible in case PerfZero provides more named + arguments before updating the constructor. + """ + + # flag_methods = [common.define_keras_flags] + + self.data_dir = os.path.join(root_data_dir, 'imagenet') + super(Resnet50CtlAccuracy, self).__init__( + output_dir=output_dir, flag_methods=flags) + + + # @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self): + start_time_sec = time.time() + stats = imagenet_main.main(flags.FLAGS) + wall_time_sec = time.time() - start_time_sec + + super(Resnet50CtlAccuracy, self)._report_benchmark( + stats, + wall_time_sec, + top_1_min=MIN_TOP_1_ACCURACY, + top_1_max=MAX_TOP_1_ACCURACY, + total_batch_size=FLAGS.batch_size, + log_steps=100) + + def _get_model_dir(self, folder_name): + return os.path.join(self.output_dir, folder_name) + + +class Resnet50CtlBenchmarkBase(CtlBenchmark): + """Resnet50 benchmarks.""" + + def __init__(self, output_dir=None, default_flags=None): + + super(Resnet50CtlBenchmarkBase, self).__init__( + output_dir=output_dir, + flag_methods=flags, + default_flags=default_flags) + + # @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self): + start_time_sec = time.time() + stats = imagenet_main.benchmark_main() + wall_time_sec = time.time() - start_time_sec + + # Number of logged step time entries that are excluded in performance + # report. We keep results from last 100 batches in this case. + warmup = (FLAGS.train_steps - 100) // FLAGS.log_steps + + super(Resnet50CtlBenchmarkBase, self)._report_benchmark( + stats, + wall_time_sec, + total_batch_size=FLAGS.batch_size, + log_steps=FLAGS.log_steps, + warmup=warmup) + + + def benchmark_1_npu_fp16(self, config_dict, cluster_device_id): + """Test v1 model with 1 NPU with tf mixed precision.""" + print("start benchmark_1_npu_fp16") + FLAGS.resnet_size = 50 + FLAGS.resnet_version = 1 + # FLAGS.max_train_steps = 1000 # this is not global step , only the step per epoch. default is according to train images + FLAGS.max_train_steps = config_dict.get('max_train_steps') + FLAGS.hooks = ['examplespersecondhook'] + #FLAGS.data_dir = '/home/w00563133/data/resnet/imagenet_TF' + FLAGS.data_dir = config_dict.get('data_dir') + FLAGS.model_dir = os.getenv('MODEL_CKPT_PATH') + FLAGS.train_epochs = config_dict.get('train_epochs') + FLAGS.batch_size = config_dict.get('batch_size') + # FLAGS.epochs_between_evals = 1 + FLAGS.epochs_between_evals = config_dict.get('epochs_between_evals') + FLAGS.iterations_per_loop = config_dict.get('iterations_per_loop') + FLAGS.save_checkpoints_steps = config_dict.get('save_checkpoints_steps') + FLAGS.stop_threshold = MIN_TOP_1_ACCURACY + self._run_and_report_benchmark() + + +class Resnet50CtlBenchmarkReal(Resnet50CtlBenchmarkBase): + """Resnet50 real data benchmark tests.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + def_flags = {} + # def_flags['skip_eval'] = True + # def_flags['data_dir'] = os.path.join(root_data_dir, 'imagenet') + # def_flags['train_steps'] = 110 + # def_flags['steps_per_loop'] = 20 + # def_flags['log_steps'] = 10 + + super(Resnet50CtlBenchmarkReal, self).__init__( + output_dir=output_dir, default_flags=def_flags) + + +if __name__ == '__main__': + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("tensorflow") + config_info = get_model_parameter("tensorflow_config") + initinal_data = {"base_lr": 0.128, "dataset": "imagenet1024", "optimizer": "SGD", "loss_scale": 512} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + hwlog.remark_print(key=hwlog.INPUT_BATCH_SIZE, value=initinal_data.get("batchsize")) + cluster_device_id = None + rank_count = sys.argv[1] + if rank_count == "1": + from resnet_config_1p_npu import resnet50_config + elif rank_count == "2": + from resnet_config_2p_npu import resnet50_config + elif rank_count == "4": + from resnet_config_4p_npu import resnet50_config + elif rank_count == "16": + from resnet_config_16p_npu import resnet50_config + elif rank_count == "32": + from resnet_config_32p_npu import resnet50_config + else: + from resnet_config_8p_npu import resnet50_config + config_dict = resnet50_config() + print("config dict info is {}".format(config_dict)) + imagenet_main.benchmark_pre() + test=Resnet50CtlBenchmarkReal("./result","./result") + test.benchmark_1_npu_fp16(config_dict, cluster_device_id) + diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/__init__.py new file mode 100644 index 0000000..2b558fe --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Activations package definition.""" +from official.modeling.activations.gelu import gelu +from official.modeling.activations.swish import hard_swish +from official.modeling.activations.swish import identity +from official.modeling.activations.swish import simple_swish diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/gelu.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/gelu.py new file mode 100644 index 0000000..c045bff --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/gelu.py @@ -0,0 +1,40 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Gaussian error linear unit.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math + +import tensorflow as tf + + +@tf.keras.utils.register_keras_serializable(package='Text') +def gelu(x): + """Gaussian Error Linear Unit. + + This is a smoother version of the RELU. + Original paper: https://arxiv.org/abs/1606.08415 + Args: + x: float Tensor to perform activation. + + Returns: + `x` with the GELU activation applied. + """ + cdf = 0.5 * (1.0 + tf.tanh( + (math.sqrt(2 / math.pi) * (x + 0.044715 * tf.pow(x, 3))))) + return x * cdf diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/gelu_test.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/gelu_test.py new file mode 100644 index 0000000..dc3b95c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/gelu_test.py @@ -0,0 +1,38 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for the Gaussian error linear unit.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.modeling import activations + + +@keras_parameterized.run_all_keras_modes +class GeluTest(keras_parameterized.TestCase): + + def test_gelu(self): + expected_data = [[0.14967535, 0., -0.10032465], + [-0.15880796, -0.04540223, 2.9963627]] + gelu_data = activations.gelu([[.25, 0, -.25], [-1, -2, 3]]) + self.assertAllClose(expected_data, gelu_data) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/swish.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/swish.py new file mode 100644 index 0000000..1d79961 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/swish.py @@ -0,0 +1,75 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Customized Swish activation.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + + +@tf.keras.utils.register_keras_serializable(package='Text') +def simple_swish(features): + """Computes the Swish activation function. + + The tf.nn.swish operation uses a custom gradient to reduce memory usage. + Since saving custom gradients in SavedModel is currently not supported, and + one would not be able to use an exported TF-Hub module for fine-tuning, we + provide this wrapper that can allow to select whether to use the native + TensorFlow swish operation, or whether to use a customized operation that + has uses default TensorFlow gradient computation. + + Args: + features: A `Tensor` representing preactivation values. + + Returns: + The activation value. + """ + features = tf.convert_to_tensor(features) + return features * tf.nn.sigmoid(features) + + +@tf.keras.utils.register_keras_serializable(package='Text') +def hard_swish(features): + """Computes a hard version of the swish function. + + This operation can be used to reduce computational cost and improve + quantization for edge devices. + + Args: + features: A `Tensor` representing preactivation values. + + Returns: + The activation value. + """ + features = tf.convert_to_tensor(features) + return features * tf.nn.relu6(features + tf.constant(3.)) * (1. / 6.) + + +@tf.keras.utils.register_keras_serializable(package='Text') +def identity(features): + """Computes the identity function. + + Useful for helping in quantization. + + Args: + features: A `Tensor` representing preactivation values. + + Returns: + The activation value. + """ + features = tf.convert_to_tensor(features) + return tf.identity(features) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/swish_test.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/swish_test.py new file mode 100644 index 0000000..22042e9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/activations/swish_test.py @@ -0,0 +1,49 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for the customized Swish activation.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.modeling import activations + + +@keras_parameterized.run_all_keras_modes +class CustomizedSwishTest(keras_parameterized.TestCase): + + def _hard_swish_np(self, x): + x = np.float32(x) + return x * np.clip(x + 3, 0, 6) / 6 + + def test_simple_swish(self): + features = [[.25, 0, -.25], [-1, -2, 3]] + customized_swish_data = activations.simple_swish(features) + swish_data = tf.nn.swish(features) + self.assertAllClose(customized_swish_data, swish_data) + + def test_hard_swish(self): + features = [[.25, 0, -.25], [-1, -2, 3]] + customized_swish_data = activations.hard_swish(features) + swish_data = self._hard_swish_np(features) + self.assertAllClose(customized_swish_data, swish_data) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/base_config.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/base_config.py new file mode 100644 index 0000000..582900c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/base_config.py @@ -0,0 +1,318 @@ +# Lint as: python3 +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Base configurations to standardize experiments.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import copy +import functools +from typing import Any, List, Mapping, Optional, Type + +import dataclasses +import tensorflow as tf +import yaml + +from official.modeling.hyperparams import params_dict + + +@dataclasses.dataclass +class Config(params_dict.ParamsDict): + """The base configuration class that supports YAML/JSON based overrides. + + * It recursively enforces a whitelist of basic types and container types, so + it avoids surprises with copy and reuse caused by unanticipated types. + * It converts dict to Config even within sequences, + e.g. for config = Config({'key': [([{'a': 42}],)]), + type(config.key[0][0][0]) is Config rather than dict. + """ + + # It's safe to add bytes and other immutable types here. + IMMUTABLE_TYPES = (str, int, float, bool, type(None)) + # It's safe to add set, frozenset and other collections here. + SEQUENCE_TYPES = (list, tuple) + + default_params: dataclasses.InitVar[Optional[Mapping[str, Any]]] = None + restrictions: dataclasses.InitVar[Optional[List[str]]] = None + + @classmethod + def _isvalidsequence(cls, v): + """Check if the input values are valid sequences. + + Args: + v: Input sequence. + + Returns: + True if the sequence is valid. Valid sequence includes the sequence + type in cls.SEQUENCE_TYPES and element type is in cls.IMMUTABLE_TYPES or + is dict or ParamsDict. + """ + if not isinstance(v, cls.SEQUENCE_TYPES): + return False + return (all(isinstance(e, cls.IMMUTABLE_TYPES) for e in v) or + all(isinstance(e, dict) for e in v) or + all(isinstance(e, params_dict.ParamsDict) for e in v)) + + @classmethod + def _import_config(cls, v, subconfig_type): + """Returns v with dicts converted to Configs, recursively.""" + if not issubclass(subconfig_type, params_dict.ParamsDict): + raise TypeError( + 'Subconfig_type should be subclass of ParamsDict, found {!r}'.format( + subconfig_type)) + if isinstance(v, cls.IMMUTABLE_TYPES): + return v + elif isinstance(v, cls.SEQUENCE_TYPES): + # Only support one layer of sequence. + if not cls._isvalidsequence(v): + raise TypeError( + 'Invalid sequence: only supports single level {!r} of {!r} or ' + 'dict or ParamsDict found: {!r}'.format(cls.SEQUENCE_TYPES, + cls.IMMUTABLE_TYPES, v)) + import_fn = functools.partial( + cls._import_config, subconfig_type=subconfig_type) + return type(v)(map(import_fn, v)) + elif isinstance(v, params_dict.ParamsDict): + # Deepcopy here is a temporary solution for preserving type in nested + # Config object. + return copy.deepcopy(v) + elif isinstance(v, dict): + return subconfig_type(v) + else: + raise TypeError('Unknown type: {!r}'.format(type(v))) + + @classmethod + def _export_config(cls, v): + """Returns v with Configs converted to dicts, recursively.""" + if isinstance(v, cls.IMMUTABLE_TYPES): + return v + elif isinstance(v, cls.SEQUENCE_TYPES): + return type(v)(map(cls._export_config, v)) + elif isinstance(v, params_dict.ParamsDict): + return v.as_dict() + elif isinstance(v, dict): + raise TypeError('dict value not supported in converting.') + else: + raise TypeError('Unknown type: {!r}'.format(type(v))) + + @classmethod + def _get_subconfig_type(cls, k) -> Type[params_dict.ParamsDict]: + """Get element type by the field name. + + Args: + k: the key/name of the field. + + Returns: + Config as default. If a type annotation is found for `k`, + 1) returns the type of the annotation if it is subtype of ParamsDict; + 2) returns the element type if the annotation of `k` is List[SubType] + or Tuple[SubType]. + """ + subconfig_type = Config + if k in cls.__annotations__: + # Directly Config subtype. + type_annotation = cls.__annotations__[k] + if (isinstance(type_annotation, type) and + issubclass(type_annotation, Config)): + subconfig_type = cls.__annotations__[k] + else: + # Check if the field is a sequence of subtypes. + field_type = getattr(type_annotation, '__origin__', type(None)) + if (isinstance(field_type, type) and + issubclass(field_type, cls.SEQUENCE_TYPES)): + element_type = getattr(type_annotation, '__args__', [type(None)])[0] + subconfig_type = ( + element_type if issubclass(element_type, params_dict.ParamsDict) + else subconfig_type) + return subconfig_type + + def __post_init__(self, default_params, restrictions, *args, **kwargs): + super().__init__(default_params=default_params, + restrictions=restrictions, + *args, + **kwargs) + + def _set(self, k, v): + """Overrides same method in ParamsDict. + + Also called by ParamsDict methods. + + Args: + k: key to set. + v: value. + + Raises: + RuntimeError + """ + subconfig_type = self._get_subconfig_type(k) + if isinstance(v, dict): + if k not in self.__dict__ or not self.__dict__[k]: + # If the key not exist or the value is None, a new Config-family object + # sould be created for the key. + self.__dict__[k] = subconfig_type(v) + else: + self.__dict__[k].override(v) + else: + self.__dict__[k] = self._import_config(v, subconfig_type) + + def __setattr__(self, k, v): + if k not in self.RESERVED_ATTR: + if getattr(self, '_locked', False): + raise ValueError('The Config has been locked. ' 'No change is allowed.') + self._set(k, v) + + def _override(self, override_dict, is_strict=True): + """Overrides same method in ParamsDict. + + Also called by ParamsDict methods. + + Args: + override_dict: dictionary to write to . + is_strict: If True, not allows to add new keys. + + Raises: + KeyError: overriding reserved keys or keys not exist (is_strict=True). + """ + for k, v in sorted(override_dict.items()): + if k in self.RESERVED_ATTR: + raise KeyError('The key {!r} is internally reserved. ' + 'Can not be overridden.'.format(k)) + if k not in self.__dict__: + if is_strict: + raise KeyError('The key {!r} does not exist in {!r}. ' + 'To extend the existing keys, use ' + '`override` with `is_strict` = False.'.format( + k, type(self))) + else: + self._set(k, v) + else: + if isinstance(v, dict) and self.__dict__[k]: + self.__dict__[k]._override(v, is_strict) # pylint: disable=protected-access + elif isinstance(v, params_dict.ParamsDict) and self.__dict__[k]: + self.__dict__[k]._override(v.as_dict(), is_strict) # pylint: disable=protected-access + else: + self._set(k, v) + + def as_dict(self): + """Returns a dict representation of params_dict.ParamsDict. + + For the nested params_dict.ParamsDict, a nested dict will be returned. + """ + return { + k: self._export_config(v) + for k, v in self.__dict__.items() + if k not in self.RESERVED_ATTR + } + + def replace(self, **kwargs): + """Like `override`, but returns a copy with the current config unchanged.""" + params = self.__class__(self) + params.override(kwargs, is_strict=True) + return params + + @classmethod + def from_yaml(cls, file_path: str): + # Note: This only works if the Config has all default values. + with tf.io.gfile.GFile(file_path, 'r') as f: + loaded = yaml.load(f) + config = cls() + config.override(loaded) + return config + + @classmethod + def from_json(cls, file_path: str): + """Wrapper for `from_yaml`.""" + return cls.from_yaml(file_path) + + @classmethod + def from_args(cls, *args, **kwargs): + """Builds a config from the given list of arguments.""" + attributes = list(cls.__annotations__.keys()) + default_params = {a: p for a, p in zip(attributes, args)} + default_params.update(kwargs) + return cls(default_params) + + +@dataclasses.dataclass +class RuntimeConfig(Config): + """High-level configurations for Runtime. + + These include parameters that are not directly related to the experiment, + e.g. directories, accelerator type, etc. + + Attributes: + distribution_strategy: e.g. 'mirrored', 'tpu', etc. + enable_eager: Whether or not to enable eager mode. + enable_xla: Whether or not to enable XLA. + per_gpu_thread_count: thread count per GPU. + gpu_threads_enabled: Whether or not GPU threads are enabled. + gpu_thread_mode: Whether and how the GPU device uses its own threadpool. + dataset_num_private_threads: Number of threads for a private threadpool + created for all datasets computation. + tpu: The address of the TPU to use, if any. + num_gpus: The number of GPUs to use, if any. + worker_hosts: comma-separated list of worker ip:port pairs for running + multi-worker models with DistributionStrategy. + task_index: If multi-worker training, the task index of this worker. + all_reduce_alg: Defines the algorithm for performing all-reduce. + num_packs: Sets `num_packs` in the cross device ops used in + MirroredStrategy. For details, see tf.distribute.NcclAllReduce. + """ + distribution_strategy: str = 'mirrored' + enable_eager: bool = False + enable_xla: bool = False + gpu_threads_enabled: bool = False + gpu_thread_mode: Optional[str] = None + dataset_num_private_threads: Optional[int] = None + per_gpu_thread_count: int = 0 + tpu: Optional[str] = None + num_gpus: int = 0 + worker_hosts: Optional[str] = None + task_index: int = -1 + all_reduce_alg: Optional[str] = None + num_packs: int = 1 + + +@dataclasses.dataclass +class TensorboardConfig(Config): + """Configuration for Tensorboard. + + Attributes: + track_lr: Whether or not to track the learning rate in Tensorboard. Defaults + to True. + write_model_weights: Whether or not to write the model weights as + images in Tensorboard. Defaults to False. + + """ + track_lr: bool = True + write_model_weights: bool = False + + +@dataclasses.dataclass +class CallbacksConfig(Config): + """Configuration for Callbacks. + + Attributes: + enable_checkpoint_and_export: Whether or not to enable checkpoints as a + Callback. Defaults to True. + enable_tensorboard: Whether or not to enable Tensorboard as a Callback. + Defaults to True. + + """ + enable_checkpoint_and_export: bool = True + enable_tensorboard: bool = True diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/base_config_test.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/base_config_test.py new file mode 100644 index 0000000..501f958 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/base_config_test.py @@ -0,0 +1,299 @@ +# Lint as: python3 +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import pprint +from typing import List, Tuple + +from absl.testing import parameterized +import dataclasses +import tensorflow as tf +from official.modeling.hyperparams import base_config + + +@dataclasses.dataclass +class DumpConfig1(base_config.Config): + a: int = 1 + b: str = 'text' + + +@dataclasses.dataclass +class DumpConfig2(base_config.Config): + c: int = 2 + d: str = 'text' + e: DumpConfig1 = DumpConfig1() + + +@dataclasses.dataclass +class DumpConfig3(DumpConfig2): + f: int = 2 + g: str = 'text' + h: List[DumpConfig1] = dataclasses.field( + default_factory=lambda: [DumpConfig1(), DumpConfig1()]) + g: Tuple[DumpConfig1, ...] = (DumpConfig1(),) + + +class BaseConfigTest(parameterized.TestCase, tf.test.TestCase): + + def assertHasSameTypes(self, c, d, msg=''): + """Checks if a Config has the same structure as a given dict. + + Args: + c: the Config object to be check. + d: the reference dict object. + msg: The error message to show when type mismatched. + """ + # Make sure d is not a Config. Assume d is either + # dictionary or primitive type and c is the Config or primitive types. + self.assertNotIsInstance(d, base_config.Config) + if isinstance(d, base_config.Config.IMMUTABLE_TYPES): + self.assertEqual(pprint.pformat(c), pprint.pformat(d), msg=msg) + elif isinstance(d, base_config.Config.SEQUENCE_TYPES): + self.assertEqual(type(c), type(d), msg=msg) + for i, v in enumerate(d): + self.assertHasSameTypes(c[i], v, msg='{}[{!r}]'.format(msg, i)) + elif isinstance(d, dict): + self.assertIsInstance(c, base_config.Config, msg=msg) + for k, v in sorted(d.items()): + self.assertHasSameTypes(getattr(c, k), v, msg='{}[{!r}]'.format(msg, k)) + else: + raise TypeError('Unknown type: %r' % type(d)) + + def assertImportExport(self, v): + config = base_config.Config({'key': v}) + back = config.as_dict()['key'] + self.assertEqual(pprint.pformat(back), pprint.pformat(v)) + self.assertHasSameTypes(config.key, v, msg='=%s v' % pprint.pformat(v)) + + def test_invalid_keys(self): + params = base_config.Config() + with self.assertRaises(AttributeError): + _ = params.a + + def test_nested_config_types(self): + config = DumpConfig3() + self.assertIsInstance(config.e, DumpConfig1) + self.assertIsInstance(config.h[0], DumpConfig1) + self.assertIsInstance(config.h[1], DumpConfig1) + self.assertIsInstance(config.g[0], DumpConfig1) + + config.override({'e': {'a': 2, 'b': 'new text'}}) + self.assertIsInstance(config.e, DumpConfig1) + self.assertEqual(config.e.a, 2) + self.assertEqual(config.e.b, 'new text') + + config.override({'h': [{'a': 3, 'b': 'new text 2'}]}) + self.assertIsInstance(config.h[0], DumpConfig1) + self.assertLen(config.h, 1) + self.assertEqual(config.h[0].a, 3) + self.assertEqual(config.h[0].b, 'new text 2') + + config.override({'g': [{'a': 4, 'b': 'new text 3'}]}) + self.assertIsInstance(config.g[0], DumpConfig1) + self.assertLen(config.g, 1) + self.assertEqual(config.g[0].a, 4) + self.assertEqual(config.g[0].b, 'new text 3') + + @parameterized.parameters( + ('_locked', "The key '_locked' is internally reserved."), + ('_restrictions', "The key '_restrictions' is internally reserved."), + ('aa', "The key 'aa' does not exist."), + ) + def test_key_error(self, key, msg): + params = base_config.Config() + with self.assertRaisesRegex(KeyError, msg): + params.override({key: True}) + + @parameterized.parameters( + ('str data',), + (123,), + (1.23,), + (None,), + (['str', 1, 2.3, None],), + (('str', 1, 2.3, None),), + ) + def test_import_export_immutable_types(self, v): + self.assertImportExport(v) + out = base_config.Config({'key': v}) + self.assertEqual(pprint.pformat(v), pprint.pformat(out.key)) + + def test_override_is_strict_true(self): + params = base_config.Config({ + 'a': 'aa', + 'b': 2, + 'c': { + 'c1': 'cc', + 'c2': 20 + } + }) + params.override({'a': 2, 'c': {'c1': 'ccc'}}, is_strict=True) + self.assertEqual(params.a, 2) + self.assertEqual(params.c.c1, 'ccc') + with self.assertRaises(KeyError): + params.override({'d': 'ddd'}, is_strict=True) + with self.assertRaises(KeyError): + params.override({'c': {'c3': 30}}, is_strict=True) + + config = base_config.Config({'key': [{'a': 42}]}) + config.override({'key': [{'b': 43}]}) + self.assertEqual(config.key[0].b, 43) + with self.assertRaisesRegex(AttributeError, 'The key `a` does not exist'): + _ = config.key[0].a + + @parameterized.parameters( + (lambda x: x, 'Unknown type'), + (object(), 'Unknown type'), + (set(), 'Unknown type'), + (frozenset(), 'Unknown type'), + ) + def test_import_unsupport_types(self, v, msg): + with self.assertRaisesRegex(TypeError, msg): + _ = base_config.Config({'key': v}) + + @parameterized.parameters( + ({ + 'a': [{ + 'b': 2, + }, { + 'c': 3, + }] + },), + ({ + 'c': [{ + 'f': 1.1, + }, { + 'h': [1, 2], + }] + },), + (({ + 'a': 'aa', + 'b': 2, + 'c': { + 'c1': 10, + 'c2': 20, + } + },),), + ) + def test_import_export_nested_structure(self, d): + self.assertImportExport(d) + + @parameterized.parameters( + ([{ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + }],), + (({ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + },),), + ) + def test_import_export_nested_sequences(self, v): + self.assertImportExport(v) + + @parameterized.parameters( + ([([{}],)],), + ([['str', 1, 2.3, None]],), + ((('str', 1, 2.3, None),),), + ([ + ('str', 1, 2.3, None), + ],), + ([ + ('str', 1, 2.3, None), + ],), + ([[{ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + }]],), + ([[[{ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + }]]],), + ((({ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + },),),), + (((({ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + },),),),), + ([({ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + },)],), + (([{ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + }],),), + ) + def test_import_export_unsupport_sequence(self, v): + with self.assertRaisesRegex(TypeError, + 'Invalid sequence: only supports single level'): + _ = base_config.Config({'key': v}) + + def test_construct_subtype(self): + pass + + def test_import_config(self): + params = base_config.Config({'a': [{'b': 2}, {'c': {'d': 3}}]}) + self.assertLen(params.a, 2) + self.assertEqual(params.a[0].b, 2) + self.assertEqual(type(params.a[0]), base_config.Config) + self.assertEqual(pprint.pformat(params.a[0].b), '2') + self.assertEqual(type(params.a[1]), base_config.Config) + self.assertEqual(type(params.a[1].c), base_config.Config) + self.assertEqual(pprint.pformat(params.a[1].c.d), '3') + + def test_override(self): + params = base_config.Config({'a': [{'b': 2}, {'c': {'d': 3}}]}) + params.override({'a': [{'b': 4}, {'c': {'d': 5}}]}, is_strict=False) + self.assertEqual(type(params.a), list) + self.assertEqual(type(params.a[0]), base_config.Config) + self.assertEqual(pprint.pformat(params.a[0].b), '4') + self.assertEqual(type(params.a[1]), base_config.Config) + self.assertEqual(type(params.a[1].c), base_config.Config) + self.assertEqual(pprint.pformat(params.a[1].c.d), '5') + + @parameterized.parameters( + ([{}],), + (({},),), + ) + def test_config_vs_params_dict(self, v): + d = {'key': v} + self.assertEqual(type(base_config.Config(d).key[0]), base_config.Config) + self.assertEqual(type(base_config.params_dict.ParamsDict(d).key[0]), dict) + + def test_ppformat(self): + self.assertEqual( + pprint.pformat([ + 's', 1, 1.0, True, None, {}, [], (), { + (2,): (3, [4], { + 6: 7, + }), + 8: 9, + } + ]), + "['s', 1, 1.0, True, None, {}, [], (), {8: 9, (2,): (3, [4], {6: 7})}]") + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/params_dict.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/params_dict.py new file mode 100644 index 0000000..9806a2b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/params_dict.py @@ -0,0 +1,410 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A parameter dictionary class which supports the nest structure.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import copy +import re + +import six +import tensorflow as tf +import yaml + +# regex pattern that matches on key-value pairs in a comma-separated +# key-value pair string. It splits each k-v pair on the = sign, and +# matches on values that are within single quotes, double quotes, single +# values (e.g. floats, ints, etc.), and a lists within brackets. +_PARAM_RE = re.compile(r""" + (?P[a-zA-Z][\w\.]*) # variable name: "var" or "x" + \s*=\s* + ((?P\'(.*?)\' # single quote + | + \"(.*?)\" # double quote + | + [^,\[]* # single value + | + \[[^\]]*\])) # list of values + ($|,\s*)""", re.VERBOSE) + + +class ParamsDict(object): + """A hyperparameter container class.""" + + RESERVED_ATTR = ['_locked', '_restrictions'] + + def __init__(self, default_params=None, restrictions=None): + """Instantiate a ParamsDict. + + Instantiate a ParamsDict given a set of default parameters and a list of + restrictions. Upon initialization, it validates itself by checking all the + defined restrictions, and raise error if it finds inconsistency. + + Args: + default_params: a Python dict or another ParamsDict object including the + default parameters to initialize. + restrictions: a list of strings, which define a list of restrictions to + ensure the consistency of different parameters internally. Each + restriction string is defined as a binary relation with a set of + operators, including {'==', '!=', '<', '<=', '>', '>='}. + """ + self._locked = False + self._restrictions = [] + if restrictions: + self._restrictions = restrictions + if default_params is None: + default_params = {} + self.override(default_params, is_strict=False) + self.validate() + + def _set(self, k, v): + if isinstance(v, dict): + self.__dict__[k] = ParamsDict(v) + else: + self.__dict__[k] = copy.deepcopy(v) + + def __setattr__(self, k, v): + """Sets the value of the existing key. + + Note that this does not allow directly defining a new key. Use the + `override` method with `is_strict=False` instead. + + Args: + k: the key string. + v: the value to be used to set the key `k`. + + Raises: + KeyError: if k is not defined in the ParamsDict. + """ + if k not in ParamsDict.RESERVED_ATTR: + if k not in self.__dict__.keys(): + raise KeyError('The key `%{}` does not exist. ' + 'To extend the existing keys, use ' + '`override` with `is_strict` = True.'.format(k)) + if self._locked: + raise ValueError('The ParamsDict has been locked. ' + 'No change is allowed.') + self._set(k, v) + + def __getattr__(self, k): + """Gets the value of the existing key. + + Args: + k: the key string. + + Returns: + the value of the key. + + Raises: + AttributeError: if k is not defined in the ParamsDict. + """ + if k not in self.__dict__.keys(): + raise AttributeError('The key `{}` does not exist. '.format(k)) + return self.__dict__[k] + + def __contains__(self, key): + """Implements the membership test operator.""" + return key in self.__dict__ + + def get(self, key, value=None): + """Accesses through built-in dictionary get method.""" + return self.__dict__.get(key, value) + + def override(self, override_params, is_strict=True): + """Override the ParamsDict with a set of given params. + + Args: + override_params: a dict or a ParamsDict specifying the parameters to + be overridden. + is_strict: a boolean specifying whether override is strict or not. If + True, keys in `override_params` must be present in the ParamsDict. + If False, keys in `override_params` can be different from what is + currently defined in the ParamsDict. In this case, the ParamsDict will + be extended to include the new keys. + """ + if self._locked: + raise ValueError('The ParamsDict has been locked. No change is allowed.') + if isinstance(override_params, ParamsDict): + override_params = override_params.as_dict() + self._override(override_params, is_strict) # pylint: disable=protected-access + + def _override(self, override_dict, is_strict=True): + """The implementation of `override`.""" + for k, v in six.iteritems(override_dict): + if k in ParamsDict.RESERVED_ATTR: + raise KeyError('The key `%{}` is internally reserved. ' + 'Can not be overridden.') + if k not in self.__dict__.keys(): + if is_strict: + raise KeyError('The key `{}` does not exist. ' + 'To extend the existing keys, use ' + '`override` with `is_strict` = False.'.format(k)) + else: + self._set(k, v) + else: + if isinstance(v, dict): + self.__dict__[k]._override(v, is_strict) # pylint: disable=protected-access + elif isinstance(v, ParamsDict): + self.__dict__[k]._override(v.as_dict(), is_strict) # pylint: disable=protected-access + else: + self.__dict__[k] = copy.deepcopy(v) + + def lock(self): + """Makes the ParamsDict immutable.""" + self._locked = True + + def as_dict(self): + """Returns a dict representation of ParamsDict. + + For the nested ParamsDict, a nested dict will be returned. + """ + params_dict = {} + for k, v in six.iteritems(self.__dict__): + if k not in ParamsDict.RESERVED_ATTR: + if isinstance(v, ParamsDict): + params_dict[k] = v.as_dict() + else: + params_dict[k] = copy.deepcopy(v) + return params_dict + + def validate(self): + """Validate the parameters consistency based on the restrictions. + + This method validates the internal consistency using the pre-defined list of + restrictions. A restriction is defined as a string which specfiies a binary + operation. The supported binary operations are {'==', '!=', '<', '<=', '>', + '>='}. Note that the meaning of these operators are consistent with the + underlying Python immplementation. Users should make sure the define + restrictions on their type make sense. + + For example, for a ParamsDict like the following + ``` + a: + a1: 1 + a2: 2 + b: + bb: + bb1: 10 + bb2: 20 + ccc: + a1: 1 + a3: 3 + ``` + one can define two restrictions like this + ['a.a1 == b.ccc.a1', 'a.a2 <= b.bb.bb2'] + + What it enforces are: + - a.a1 = 1 == b.ccc.a1 = 2 + - a.a2 = 2 <= b.bb.bb2 = 20 + + Raises: + KeyError: if any of the following happens + (1) any of parameters in any of restrictions is not defined in + ParamsDict, + (2) any inconsistency violating the restriction is found. + ValueError: if the restriction defined in the string is not supported. + """ + def _get_kv(dotted_string, params_dict): + tokenized_params = dotted_string.split('.') + v = params_dict + for t in tokenized_params: + v = v[t] + return tokenized_params[-1], v + + def _get_kvs(tokens, params_dict): + if len(tokens) != 2: + raise ValueError('Only support binary relation in restriction.') + stripped_tokens = [t.strip() for t in tokens] + left_k, left_v = _get_kv(stripped_tokens[0], params_dict) + right_k, right_v = _get_kv(stripped_tokens[1], params_dict) + return left_k, left_v, right_k, right_v + + params_dict = self.as_dict() + for restriction in self._restrictions: + if '==' in restriction: + tokens = restriction.split('==') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v != right_v: + raise KeyError('Found inconsistncy between key `{}` and key `{}`.' + .format(tokens[0], tokens[1])) + elif '!=' in restriction: + tokens = restriction.split('!=') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v == right_v: + raise KeyError('Found inconsistncy between key `{}` and key `{}`.' + .format(tokens[0], tokens[1])) + elif '<' in restriction: + tokens = restriction.split('<') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v >= right_v: + raise KeyError('Found inconsistncy between key `{}` and key `{}`.' + .format(tokens[0], tokens[1])) + elif '<=' in restriction: + tokens = restriction.split('<=') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v > right_v: + raise KeyError('Found inconsistncy between key `{}` and key `{}`.' + .format(tokens[0], tokens[1])) + elif '>' in restriction: + tokens = restriction.split('>') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v <= right_v: + raise KeyError('Found inconsistncy between key `{}` and key `{}`.' + .format(tokens[0], tokens[1])) + elif '>=' in restriction: + tokens = restriction.split('>=') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v < right_v: + raise KeyError('Found inconsistncy between key `{}` and key `{}`.' + .format(tokens[0], tokens[1])) + else: + raise ValueError('Unsupported relation in restriction.') + + +def read_yaml_to_params_dict(file_path): + """Reads a YAML file to a ParamsDict.""" + with tf.io.gfile.GFile(file_path, 'r') as f: + params_dict = yaml.load(f) + return ParamsDict(params_dict) + + +def save_params_dict_to_yaml(params, file_path): + """Saves the input ParamsDict to a YAML file.""" + with tf.io.gfile.GFile(file_path, 'w') as f: + + def _my_list_rep(dumper, data): + # u'tag:yaml.org,2002:seq' is the YAML internal tag for sequence. + return dumper.represent_sequence( + u'tag:yaml.org,2002:seq', data, flow_style=True) + yaml.add_representer(list, _my_list_rep) + yaml.dump(params.as_dict(), f, default_flow_style=False) + + +def nested_csv_str_to_json_str(csv_str): + """Converts a nested (using '.') comma-separated k=v string to a JSON string. + + Converts a comma-separated string of key/value pairs that supports + nesting of keys to a JSON string. Nesting is implemented using + '.' between levels for a given key. + + Spacing between commas and = is supported (e.g. there is no difference between + "a=1,b=2", "a = 1, b = 2", or "a=1, b=2") but there should be no spaces before + keys or after values (e.g. " a=1,b=2" and "a=1,b=2 " are not supported). + + Note that this will only support values supported by CSV, meaning + values such as nested lists (e.g. "a=[[1,2,3],[4,5,6]]") are not + supported. Strings are supported as well, e.g. "a='hello'". + + An example conversion would be: + + "a=1, b=2, c.a=2, c.b=3, d.a.a=5" + + to + + "{ a: 1, b : 2, c: {a : 2, b : 3}, d: {a: {a : 5}}}" + + Args: + csv_str: the comma separated string. + + Returns: + the converted JSON string. + + Raises: + ValueError: If csv_str is not in a comma separated string or + if the string is formatted incorrectly. + """ + if not csv_str: + return '' + + formatted_entries = [] + nested_map = collections.defaultdict(list) + pos = 0 + while pos < len(csv_str): + m = _PARAM_RE.match(csv_str, pos) + if not m: + raise ValueError('Malformed hyperparameter value while parsing ' + 'CSV string: %s' % csv_str[pos:]) + pos = m.end() + # Parse the values. + m_dict = m.groupdict() + name = m_dict['name'] + v = m_dict['val'] + + # If a GCS path (e.g. gs://...) is provided, wrap this in quotes + # as yaml.load would otherwise throw an exception + if re.match(r'(?=[^\"\'])(?=[gs://])', v): + v = '\'{}\''.format(v) + + name_nested = name.split('.') + if len(name_nested) > 1: + grouping = name_nested[0] + value = '.'.join(name_nested[1:]) + '=' + v + nested_map[grouping].append(value) + else: + formatted_entries.append('%s : %s' % (name, v)) + + for grouping, value in nested_map.items(): + value = ','.join(value) + value = nested_csv_str_to_json_str(value) + formatted_entries.append('%s : %s' % (grouping, value)) + return '{' + ', '.join(formatted_entries) + '}' + + +def override_params_dict(params, dict_or_string_or_yaml_file, is_strict): + """Override a given ParamsDict using a dict, JSON/YAML/CSV string or YAML file. + + The logic of the function is outlined below: + 1. Test that the input is a dict. If not, proceed to 2. + 2. Tests that the input is a string. If not, raise unknown ValueError + 2.1. Test if the string is in a CSV format. If so, parse. + If not, proceed to 2.2. + 2.2. Try loading the string as a YAML/JSON. If successful, parse to + dict and use it to override. If not, proceed to 2.3. + 2.3. Try using the string as a file path and load the YAML file. + + Args: + params: a ParamsDict object to be overridden. + dict_or_string_or_yaml_file: a Python dict, JSON/YAML/CSV string or + path to a YAML file specifying the parameters to be overridden. + is_strict: a boolean specifying whether override is strict or not. + + Returns: + params: the overridden ParamsDict object. + + Raises: + ValueError: if failed to override the parameters. + """ + if not dict_or_string_or_yaml_file: + return params + if isinstance(dict_or_string_or_yaml_file, dict): + params.override(dict_or_string_or_yaml_file, is_strict) + elif isinstance(dict_or_string_or_yaml_file, six.string_types): + try: + dict_or_string_or_yaml_file = ( + nested_csv_str_to_json_str(dict_or_string_or_yaml_file)) + except ValueError: + pass + params_dict = yaml.load(dict_or_string_or_yaml_file) + if isinstance(params_dict, dict): + params.override(params_dict, is_strict) + else: + with tf.io.gfile.GFile(dict_or_string_or_yaml_file) as f: + params.override(yaml.load(f), is_strict) + else: + raise ValueError('Unknown input type to parse.') + return params diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/params_dict_test.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/params_dict_test.py new file mode 100644 index 0000000..3d53ea1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/hyperparams/params_dict_test.py @@ -0,0 +1,322 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Tests for official.modeling.hyperparams.params_dict.py.""" + +import os + +import tensorflow as tf +import yaml + +from official.modeling.hyperparams import params_dict + + +class ParamsDictTest(tf.test.TestCase): + + def test_init_from_an_empty_dict(self): + params = params_dict.ParamsDict() + with self.assertRaises(AttributeError): + _ = params.a + + with self.assertRaises(KeyError): + params.a = 'aa' + + def test_init_from_a_dict(self): + params = params_dict.ParamsDict({'a': 'aa', 'b': 2}) + self.assertEqual(params.a, 'aa') + self.assertEqual(params.b, 2) + + def test_init_from_a_param_dict(self): + params_init = params_dict.ParamsDict({'a': 'aa', 'b': 2}) + params = params_dict.ParamsDict(params_init) + self.assertEqual(params.a, 'aa') + self.assertEqual(params.b, 2) + + def test_lock(self): + params = params_dict.ParamsDict({'a': 1, 'b': 2}) + params.lock() + with self.assertRaises(ValueError): + params.a = 10 + with self.assertRaises(ValueError): + params.override({'b': 20}) + + def test_setattr(self): + params = params_dict.ParamsDict() + params.override( + {'a': 'aa', 'b': 2, 'c': None}, is_strict=False) + params.c = 'ccc' + self.assertEqual(params.a, 'aa') + self.assertEqual(params.b, 2) + self.assertEqual(params.c, 'ccc') + + def test_getattr(self): + params = params_dict.ParamsDict() + params.override( + {'a': 'aa', 'b': 2, 'c': None}, is_strict=False) + self.assertEqual(params.a, 'aa') + self.assertEqual(params.b, 2) + self.assertEqual(params.c, None) + + def test_contains(self): + params = params_dict.ParamsDict() + params.override( + {'a': 'aa'}, is_strict=False) + self.assertIn('a', params) + self.assertNotIn('b', params) + + def test_get(self): + params = params_dict.ParamsDict() + params.override( + {'a': 'aa'}, is_strict=False) + self.assertEqual(params.get('a'), 'aa') + self.assertEqual(params.get('b', 2), 2) + self.assertEqual(params.get('b'), None) + + def test_override_is_strict_true(self): + params = params_dict.ParamsDict( + {'a': 'aa', 'b': 2, 'c': {'c1': 'cc', 'c2': 20}}) + params.override({'a': 2, 'c': {'c1': 'ccc'}}, is_strict=True) + self.assertEqual(params.a, 2) + self.assertEqual(params.c.c1, 'ccc') + with self.assertRaises(KeyError): + params.override({'d': 'ddd'}, is_strict=True) + with self.assertRaises(KeyError): + params.override({'c': {'c3': 30}}, is_strict=True) + + def test_override_is_strict_false(self): + params = params_dict.ParamsDict( + {'a': 'aa', 'b': 2, 'c': {'c1': 10, 'c2': 20}}) + params.override({'a': 2, 'c': {'c3': 3000}}, is_strict=False) + self.assertEqual(params.a, 2) + self.assertEqual(params.c.c3, 3000) + params.override({'d': 'ddd'}, is_strict=False) + self.assertEqual(params.d, 'ddd') + params.override({'c': {'c4': 4444}}, is_strict=False) + self.assertEqual(params.c.c4, 4444) + + def test_as_dict(self): + params = params_dict.ParamsDict( + {'a': 'aa', 'b': 2, 'c': {'c1': 10, 'c2': 20}}) + params_d = params.as_dict() + self.assertEqual(params_d['a'], 'aa') + self.assertEqual(params_d['b'], 2) + self.assertEqual(params_d['c']['c1'], 10) + self.assertEqual(params_d['c']['c2'], 20) + + def test_validate(self): + # Raise error due to the unknown parameter. + with self.assertRaises(KeyError): + params = params_dict.ParamsDict( + {'a': 1, 'b': {'a': 11}}, ['a == c']) + + # OK to check equality of two nested dicts. + params = params_dict.ParamsDict( + {'a': 1, 'b': {'a': 10}, 'c': {'a': 10}}, ['b == c']) + + # Raise error due to inconsistency + with self.assertRaises(KeyError): + params = params_dict.ParamsDict( + {'a': 1, 'c': {'a': 10}}, ['a == c.a']) + + # Valid rule. + params = params_dict.ParamsDict( + {'a': 1, 'c': {'a': 1}}, ['a == c.a']) + + # Overridding violates the existing rule, raise error upon validate. + params.override({'a': 11}) + with self.assertRaises(KeyError): + params.validate() + + +class ParamsDictIOTest(tf.test.TestCase): + + def write_temp_file(self, filename, text): + temp_file = os.path.join(self.get_temp_dir(), filename) + with tf.io.gfile.GFile(temp_file, 'w') as writer: + writer.write(text) + return temp_file + + def test_save_params_dict_to_yaml(self): + params = params_dict.ParamsDict( + {'a': 'aa', 'b': 2, 'c': {'c1': 10, 'c2': 20}}) + output_yaml_file = os.path.join(self.get_temp_dir(), 'params.yaml') + params_dict.save_params_dict_to_yaml(params, output_yaml_file) + + with tf.io.gfile.GFile(output_yaml_file, 'r') as f: + params_d = yaml.load(f) + self.assertEqual(params.a, params_d['a']) + self.assertEqual(params.b, params_d['b']) + self.assertEqual(params.c.c1, params_d['c']['c1']) + self.assertEqual(params.c.c2, params_d['c']['c2']) + + def test_read_yaml_to_params_dict(self): + input_yaml_file = self.write_temp_file( + 'params.yaml', r""" + a: 'aa' + b: 2 + c: + c1: 10 + c2: 20 + """) + params = params_dict.read_yaml_to_params_dict(input_yaml_file) + + self.assertEqual(params.a, 'aa') + self.assertEqual(params.b, 2) + self.assertEqual(params.c.c1, 10) + self.assertEqual(params.c.c2, 20) + + def test_override_params_dict_using_dict(self): + params = params_dict.ParamsDict({ + 'a': 1, 'b': 2.5, 'c': [3, 4], 'd': 'hello', 'e': False}) + override_dict = {'b': 5.2, 'c': [30, 40]} + params = params_dict.override_params_dict( + params, override_dict, is_strict=True) + self.assertEqual(1, params.a) + self.assertEqual(5.2, params.b) + self.assertEqual([30, 40], params.c) + self.assertEqual('hello', params.d) + self.assertEqual(False, params.e) + + def test_override_params_dict_using_yaml_string(self): + params = params_dict.ParamsDict({ + 'a': 1, 'b': 2.5, 'c': [3, 4], 'd': 'hello', 'e': False}) + override_yaml_string = "'b': 5.2\n'c': [30, 40]" + params = params_dict.override_params_dict( + params, override_yaml_string, is_strict=True) + self.assertEqual(1, params.a) + self.assertEqual(5.2, params.b) + self.assertEqual([30, 40], params.c) + self.assertEqual('hello', params.d) + self.assertEqual(False, params.e) + + def test_override_params_dict_using_json_string(self): + params = params_dict.ParamsDict({ + 'a': 1, 'b': {'b1': 2, 'b2': [2, 3],}, + 'd': {'d1': {'d2': 'hello'}}, 'e': False}) + override_json_string = "{ b: { b2: [3, 4] }, d: { d1: { d2: 'hi' } } }" + params = params_dict.override_params_dict( + params, override_json_string, is_strict=True) + self.assertEqual(1, params.a) + self.assertEqual(2, params.b.b1) + self.assertEqual([3, 4], params.b.b2) + self.assertEqual('hi', params.d.d1.d2) + self.assertEqual(False, params.e) + + def test_override_params_dict_using_csv_string(self): + params = params_dict.ParamsDict({ + 'a': 1, 'b': {'b1': 2, 'b2': [2, 3],}, + 'd': {'d1': {'d2': 'hello'}}, 'e': False}) + override_csv_string = "b.b2=[3,4], d.d1.d2='hi, world', e=gs://test" + params = params_dict.override_params_dict( + params, override_csv_string, is_strict=True) + self.assertEqual(1, params.a) + self.assertEqual(2, params.b.b1) + self.assertEqual([3, 4], params.b.b2) + self.assertEqual('hi, world', params.d.d1.d2) + self.assertEqual('gs://test', params.e) + + def test_override_params_dict_using_yaml_file(self): + params = params_dict.ParamsDict({ + 'a': 1, 'b': 2.5, 'c': [3, 4], 'd': 'hello', 'e': False}) + override_yaml_file = self.write_temp_file( + 'params.yaml', r""" + b: 5.2 + c: [30, 40] + """) + params = params_dict.override_params_dict( + params, override_yaml_file, is_strict=True) + self.assertEqual(1, params.a) + self.assertEqual(5.2, params.b) + self.assertEqual([30, 40], params.c) + self.assertEqual('hello', params.d) + self.assertEqual(False, params.e) + + +class IOTest(tf.test.TestCase): + + def test_basic_csv_str_to_json_str(self): + csv_str = 'a=1,b=2,c=3' + json_str = '{a : 1, b : 2, c : 3}' + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + self.assertEqual(converted_csv_str, json_str) + + def test_basic_csv_str_load(self): + csv_str = 'a=1,b=2,c=3' + expected_output = {'a': 1, 'b': 2, 'c': 3} + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + converted_dict = yaml.load(converted_csv_str) + self.assertDictEqual(converted_dict, expected_output) + + def test_basic_nested_csv_str_to_json_str(self): + csv_str = 'a=1,b.b1=2' + json_str = '{a : 1, b : {b1 : 2}}' + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + self.assertEqual(converted_csv_str, json_str) + + def test_basic_nested_csv_str_load(self): + csv_str = 'a=1,b.b1=2,c.c1=3' + expected_output = {'a': 1, 'b': {'b1': 2}, 'c': {'c1': 3}} + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + converted_dict = yaml.load(converted_csv_str) + self.assertDictEqual(converted_dict, expected_output) + + def test_complex_nested_csv_str_to_json_str(self): + csv_str = 'a.aa.aaa.aaaaa.a=1' + json_str = '{a : {aa : {aaa : {aaaaa : {a : 1}}}}}' + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + self.assertEqual(converted_csv_str, json_str) + + def test_complex_nested_csv_str_load(self): + csv_str = 'a.aa.aaa.aaaaa.a=1,a.a=2' + expected_output = {'a': {'aa': {'aaa': {'aaaaa': {'a': 1}}}, 'a': 2}} + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + converted_dict = yaml.load(converted_csv_str) + self.assertDictEqual(converted_dict, expected_output) + + def test_csv_str_load_supported_datatypes(self): + csv_str = 'a=1,b=2.,c=[1,2,3],d=\'hello, there\',e=\"Hi.\"' + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + converted_dict = yaml.load(converted_csv_str) + self.assertEqual(converted_dict['a'], 1) + self.assertEqual(converted_dict['b'], 2.) + self.assertEqual(converted_dict['c'], [1, 2, 3]) + self.assertEqual(converted_dict['d'], 'hello, there') + self.assertEqual(converted_dict['e'], 'Hi.') + + def test_csv_str_load_unsupported_datatypes(self): + csv_str = 'a=[[1,2,3],[4,5,6]]' + self.assertRaises(ValueError, + params_dict.nested_csv_str_to_json_str, + csv_str) + + def test_csv_str_to_json_str_spacing(self): + csv_str1 = 'a=1,b=2,c=3' + csv_str2 = 'a = 1, b = 2, c = 3' + json_str = '{a : 1, b : 2, c : 3}' + converted_csv_str1 = params_dict.nested_csv_str_to_json_str(csv_str1) + converted_csv_str2 = params_dict.nested_csv_str_to_json_str(csv_str2) + self.assertEqual(converted_csv_str1, converted_csv_str2) + self.assertEqual(converted_csv_str1, json_str) + self.assertEqual(converted_csv_str2, json_str) + + def test_gcs_added_quotes(self): + csv_str = 'a=gs://abc, b=gs://def' + expected_output = '{a : \'gs://abc\', b : \'gs://def\'}' + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + self.assertEqual(converted_csv_str, expected_output) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/model_training_utils.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/model_training_utils.py new file mode 100644 index 0000000..2f66d1c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/model_training_utils.py @@ -0,0 +1,491 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A light weight utilities to train NLP models.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import os +import tempfile + +from absl import logging +import tensorflow as tf +from official.staging.training import grad_utils +from official.utils.misc import distribution_utils + +_SUMMARY_TXT = 'training_summary.txt' +_MIN_SUMMARY_STEPS = 10 + + +def _should_export_checkpoint(strategy): + return (not strategy) or strategy.extended.should_checkpoint + + +def _should_export_summary(strategy): + return (not strategy) or strategy.extended.should_save_summary + + +def _save_checkpoint(strategy, checkpoint, model_dir, checkpoint_prefix): + """Saves model to with provided checkpoint prefix.""" + + if _should_export_checkpoint(strategy): + checkpoint_path = os.path.join(model_dir, checkpoint_prefix) + saved_path = checkpoint.save(checkpoint_path) + logging.info('Saving model as TF checkpoint: %s', saved_path) + else: + # In multi worker training we need every worker to save checkpoint, because + # variables can trigger synchronization on read and synchronization needs + # all workers to participate. To avoid workers overriding each other we save + # to a temporary directory on non-chief workers. + tmp_dir = tempfile.mkdtemp() + checkpoint.save(os.path.join(tmp_dir, 'ckpt')) + tf.io.gfile.rmtree(tmp_dir) + return + + +def _get_input_iterator(input_fn, strategy): + """Returns distributed dataset iterator.""" + # When training with TPU pods, datasets needs to be cloned across + # workers. Since Dataset instance cannot be cloned in eager mode, we instead + # pass callable that returns a dataset. + if not callable(input_fn): + raise ValueError('`input_fn` should be a closure that returns a dataset.') + iterator = iter( + strategy.experimental_distribute_datasets_from_function(input_fn)) + return iterator + + +def _float_metric_value(metric): + """Gets the value of a float-value keras metric.""" + return metric.result().numpy().astype(float) + + +def steps_to_run(current_step, steps_per_epoch, steps_per_loop): + """Calculates steps to run on device.""" + if steps_per_loop <= 0: + raise ValueError('steps_per_loop should be positive integer.') + if steps_per_loop == 1: + return steps_per_loop + remainder_in_epoch = current_step % steps_per_epoch + if remainder_in_epoch != 0: + return min(steps_per_epoch - remainder_in_epoch, steps_per_loop) + else: + return steps_per_loop + + +def write_txt_summary(training_summary, summary_dir): + """Writes a summary text file to record stats.""" + summary_path = os.path.join(summary_dir, _SUMMARY_TXT) + with tf.io.gfile.GFile(summary_path, 'wb') as f: + logging.info('Training Summary: \n%s', str(training_summary)) + f.write(json.dumps(training_summary, indent=4)) + + +def run_customized_training_loop( + # pylint: disable=invalid-name + _sentinel=None, + # pylint: enable=invalid-name + strategy=None, + model_fn=None, + loss_fn=None, + scale_loss=True, + model_dir=None, + train_input_fn=None, + steps_per_epoch=None, + steps_per_loop=1, + epochs=1, + eval_input_fn=None, + eval_steps=None, + metric_fn=None, + init_checkpoint=None, + custom_callbacks=None, + run_eagerly=False, + sub_model_export_name=None, + explicit_allreduce=False, + pre_allreduce_callbacks=None, + post_allreduce_callbacks=None): + """Run BERT pretrain model training using low-level API. + + Arguments: + _sentinel: Used to prevent positional parameters. Internal, do not use. + strategy: Distribution strategy on which to run low level training loop. + model_fn: Function that returns a tuple (model, sub_model). Caller of this + function should add optimizer to the `model` via calling + `model.compile()` API or manually setting `model.optimizer` attribute. + Second element of the returned tuple(sub_model) is an optional sub model + to be used for initial checkpoint -- if provided. + loss_fn: Function with signature func(labels, logits) and returns a loss + tensor. + scale_loss: Whether to divide the raw loss by number of replicas before + gradients calculation. + model_dir: Model directory used during training for restoring/saving model + weights. + train_input_fn: Function that returns a tf.data.Dataset used for training. + steps_per_epoch: Number of steps to run per epoch. At the end of each + epoch, model checkpoint will be saved and evaluation will be conducted + if evaluation dataset is provided. + steps_per_loop: Number of steps per graph-mode loop. In order to reduce + communication in eager context, training logs are printed every + steps_per_loop. + epochs: Number of epochs to train. + eval_input_fn: Function that returns evaluation dataset. If none, + evaluation is skipped. + eval_steps: Number of steps to run evaluation. Required if `eval_input_fn` + is not none. + metric_fn: A metrics function that returns a Keras Metric object to record + evaluation result using evaluation dataset or with training dataset + after every epoch. + init_checkpoint: Optional checkpoint to load to `sub_model` returned by + `model_fn`. + custom_callbacks: A list of Keras Callbacks objects to run during + training. More specifically, `on_batch_begin()`, `on_batch_end()`, + methods are invoked during training. + run_eagerly: Whether to run model training in pure eager execution. This + should be disable for TPUStrategy. + sub_model_export_name: If not None, will export `sub_model` returned by + `model_fn` into checkpoint files. The name of intermediate checkpoint + file is {sub_model_export_name}_step_{step}.ckpt and the last + checkpint's name is {sub_model_export_name}.ckpt; + if None, `sub_model` will not be exported as checkpoint. + explicit_allreduce: Whether to explicitly perform gradient allreduce, + instead of relying on implicit allreduce in optimizer.apply_gradients(). + default is False. For now, if training using FP16 mixed precision, + explicit allreduce will aggregate gradients in FP16 format. For TPU and + GPU training using FP32, explicit allreduce will aggregate gradients in + FP32 format. + pre_allreduce_callbacks: A list of callback functions that takes gradients + and model variables pairs as input, manipulate them, and returns a new + gradients and model variables paris. The callback functions will be + invoked in the list order and before gradients are allreduced. + With mixed precision training, the pre_allreduce_allbacks will be + applied on scaled_gradients. Default is no callbacks. + Only used when explicit_allreduce=True. + post_allreduce_callbacks: A list of callback functions that takes + gradients and model variables pairs as input, manipulate them, and + returns a new gradients and model variables paris. The callback + functions will be invoked in the list order and right before gradients + are applied to variables for updates. Default is no callbacks. Only used + when explicit_allreduce=True. + + Returns: + Trained model. + + Raises: + ValueError: (1) When model returned by `model_fn` does not have optimizer + attribute or when required parameters are set to none. (2) eval args are + not specified correctly. (3) metric_fn must be a callable if specified. + (4) sub_model_checkpoint_name is specified, but `sub_model` returned + by `model_fn` is None. + """ + + if _sentinel is not None: + raise ValueError('only call `run_customized_training_loop()` ' + 'with named arguments.') + + required_arguments = [ + strategy, model_fn, loss_fn, model_dir, steps_per_epoch, train_input_fn + ] + if [arg for arg in required_arguments if arg is None]: + raise ValueError('`strategy`, `model_fn`, `loss_fn`, `model_dir`, ' + '`steps_per_loop` and `steps_per_epoch` are required ' + 'parameters.') + if steps_per_loop > steps_per_epoch: + logging.error( + 'steps_per_loop: %d is specified to be greater than ' + ' steps_per_epoch: %d, we will use steps_per_epoch as' + ' steps_per_loop.', steps_per_loop, steps_per_epoch) + steps_per_loop = steps_per_epoch + assert tf.executing_eagerly() + + if run_eagerly: + if isinstance(strategy, tf.distribute.experimental.TPUStrategy): + raise ValueError( + 'TPUStrategy should not run eagerly as it heavily relies on graph' + ' optimization for the distributed system.') + + if eval_input_fn and (eval_steps is None or metric_fn is None): + raise ValueError( + '`eval_step` and `metric_fn` are required when `eval_input_fn ` ' + 'is not none.') + if metric_fn and not callable(metric_fn): + raise ValueError( + 'if `metric_fn` is specified, metric_fn must be a callable.') + + total_training_steps = steps_per_epoch * epochs + train_iterator = _get_input_iterator(train_input_fn, strategy) + + with distribution_utils.get_strategy_scope(strategy): + # To correctly place the model weights on accelerators, + # model and optimizer should be created in scope. + model, sub_model = model_fn() + if not hasattr(model, 'optimizer'): + raise ValueError('User should set optimizer attribute to model ' + 'inside `model_fn`.') + if sub_model_export_name and sub_model is None: + raise ValueError('sub_model_export_name is specified as %s, but ' + 'sub_model is None.' % sub_model_export_name) + + optimizer = model.optimizer + + if init_checkpoint: + logging.info( + 'Checkpoint file %s found and restoring from ' + 'initial checkpoint for core model.', init_checkpoint) + checkpoint = tf.train.Checkpoint(model=sub_model) + checkpoint.restore(init_checkpoint).assert_existing_objects_matched() + logging.info('Loading from checkpoint file completed') + + train_loss_metric = tf.keras.metrics.Mean( + 'training_loss', dtype=tf.float32) + eval_metrics = [metric_fn()] if metric_fn else [] + # If evaluation is required, make a copy of metric as it will be used by + # both train and evaluation. + train_metrics = [ + metric.__class__.from_config(metric.get_config()) + for metric in eval_metrics + ] + + # Create summary writers + if _should_export_summary(strategy): + summary_dir = os.path.join(model_dir, 'summaries') + else: + # In multi worker training we need every worker to write summary, because + # variables can trigger synchronization on read and synchronization needs + # all workers to participate. + summary_dir = tempfile.mkdtemp() + eval_summary_writer = tf.summary.create_file_writer( + os.path.join(summary_dir, 'eval')) + if steps_per_loop >= _MIN_SUMMARY_STEPS: + # Only writes summary when the stats are collected sufficiently over + # enough steps. + train_summary_writer = tf.summary.create_file_writer( + os.path.join(summary_dir, 'train')) + else: + train_summary_writer = None + + # Collects training variables. + training_vars = model.trainable_variables + + def _replicated_step(inputs): + """Replicated training step.""" + + inputs, labels = inputs + with tf.GradientTape() as tape: + model_outputs = model(inputs, training=True) + loss = loss_fn(labels, model_outputs) + # Raw loss is used for reporting in metrics/logs. + raw_loss = loss + if scale_loss: + # Scales down the loss for gradients to be invariant from replicas. + loss = loss / strategy.num_replicas_in_sync + + if explicit_allreduce: + grad_utils.minimize_using_explicit_allreduce(tape, optimizer, loss, + training_vars, + pre_allreduce_callbacks, + post_allreduce_callbacks) + else: + if isinstance(optimizer, + tf.keras.mixed_precision.experimental.LossScaleOptimizer): + with tape: + scaled_loss = optimizer.get_scaled_loss(loss) + scaled_grads = tape.gradient(scaled_loss, training_vars) + grads = optimizer.get_unscaled_gradients(scaled_grads) + else: + grads = tape.gradient(loss, training_vars) + optimizer.apply_gradients(zip(grads, training_vars)) + # For reporting, the metric takes the mean of losses. + train_loss_metric.update_state(raw_loss) + for metric in train_metrics: + metric.update_state(labels, model_outputs) + + @tf.function + def train_steps(iterator, steps): + """Performs distributed training steps in a loop. + + Args: + iterator: the distributed iterator of training datasets. + steps: an tf.int32 integer tensor to specify number of steps to run + inside host training loop. + + Raises: + ValueError: Any of the arguments or tensor shapes are invalid. + """ + if not isinstance(steps, tf.Tensor): + raise ValueError('steps should be an Tensor. Python object may cause ' + 'retracing.') + + for _ in tf.range(steps): + strategy.run(_replicated_step, args=(next(iterator),)) + + def train_single_step(iterator): + """Performs a distributed training step. + + Args: + iterator: the distributed iterator of training datasets. + + Raises: + ValueError: Any of the arguments or tensor shapes are invalid. + """ + strategy.run(_replicated_step, args=(next(iterator),)) + + def test_step(iterator): + """Calculates evaluation metrics on distributed devices.""" + + def _test_step_fn(inputs): + """Replicated accuracy calculation.""" + + inputs, labels = inputs + model_outputs = model(inputs, training=False) + for metric in eval_metrics: + metric.update_state(labels, model_outputs) + + strategy.run(_test_step_fn, args=(next(iterator),)) + + if not run_eagerly: + train_single_step = tf.function(train_single_step) + test_step = tf.function(test_step) + + def _run_evaluation(current_training_step, test_iterator): + """Runs validation steps and aggregate metrics.""" + for _ in range(eval_steps): + test_step(test_iterator) + + with eval_summary_writer.as_default(): + for metric in eval_metrics + model.metrics: + metric_value = _float_metric_value(metric) + logging.info('Step: [%d] Validation %s = %f', current_training_step, + metric.name, metric_value) + tf.summary.scalar( + metric.name, metric_value, step=current_training_step) + eval_summary_writer.flush() + + def _run_callbacks_on_batch_begin(batch): + """Runs custom callbacks at the start of every step.""" + if not custom_callbacks: + return + for callback in custom_callbacks: + callback.on_batch_begin(batch) + + def _run_callbacks_on_batch_end(batch, logs): + """Runs custom callbacks at the end of every step.""" + if not custom_callbacks: + return + for callback in custom_callbacks: + callback.on_batch_end(batch, logs) + + # Training loop starts here. + checkpoint = tf.train.Checkpoint(model=model, optimizer=optimizer) + sub_model_checkpoint = tf.train.Checkpoint( + model=sub_model) if sub_model_export_name else None + + latest_checkpoint_file = tf.train.latest_checkpoint(model_dir) + if latest_checkpoint_file: + logging.info( + 'Checkpoint file %s found and restoring from ' + 'checkpoint', latest_checkpoint_file) + checkpoint.restore(latest_checkpoint_file) + logging.info('Loading from checkpoint file completed') + + current_step = optimizer.iterations.numpy() + checkpoint_name = 'ctl_step_{step}.ckpt' + + while current_step < total_training_steps: + # Training loss/metric are taking average over steps inside micro + # training loop. We reset the their values before each round. + train_loss_metric.reset_states() + for metric in train_metrics + model.metrics: + metric.reset_states() + + _run_callbacks_on_batch_begin(current_step) + # Runs several steps in the host while loop. + steps = steps_to_run(current_step, steps_per_epoch, steps_per_loop) + + if tf.test.is_built_with_cuda(): + # TODO(zongweiz): merge with train_steps once tf.while_loop + # GPU performance bugs are fixed. + for _ in range(steps): + train_single_step(train_iterator) + else: + # Converts steps to a Tensor to avoid tf.function retracing. + train_steps(train_iterator, + tf.convert_to_tensor(steps, dtype=tf.int32)) + train_loss = _float_metric_value(train_loss_metric) + current_step += steps + _run_callbacks_on_batch_end(current_step - 1, {'loss': train_loss}) + + # Updates training logging. + training_status = 'Train Step: %d/%d / loss = %s' % ( + current_step, total_training_steps, train_loss) + + if train_summary_writer: + with train_summary_writer.as_default(): + tf.summary.scalar( + train_loss_metric.name, train_loss, step=current_step) + for metric in train_metrics + model.metrics: + metric_value = _float_metric_value(metric) + training_status += ' %s = %f' % (metric.name, metric_value) + tf.summary.scalar(metric.name, metric_value, step=current_step) + train_summary_writer.flush() + logging.info(training_status) + + # Saves model checkpoints and run validation steps at every epoch end. + if current_step % steps_per_epoch == 0: + # To avoid repeated model saving, we do not save after the last + # step of training. + if current_step < total_training_steps: + _save_checkpoint(strategy, checkpoint, model_dir, + checkpoint_name.format(step=current_step)) + if sub_model_export_name: + _save_checkpoint( + strategy, sub_model_checkpoint, model_dir, + '%s_step_%d.ckpt' % (sub_model_export_name, current_step)) + if eval_input_fn: + logging.info('Running evaluation after step: %s.', current_step) + _run_evaluation(current_step, + _get_input_iterator(eval_input_fn, strategy)) + # Re-initialize evaluation metric. + for metric in eval_metrics + model.metrics: + metric.reset_states() + + _save_checkpoint(strategy, checkpoint, model_dir, + checkpoint_name.format(step=current_step)) + if sub_model_export_name: + _save_checkpoint(strategy, sub_model_checkpoint, model_dir, + '%s.ckpt' % sub_model_export_name) + + if eval_input_fn: + logging.info('Running final evaluation after training is complete.') + _run_evaluation(current_step, + _get_input_iterator(eval_input_fn, strategy)) + + training_summary = { + 'total_training_steps': total_training_steps, + 'train_loss': _float_metric_value(train_loss_metric), + } + if eval_metrics: + # TODO(hongkuny): Cleans up summary reporting in text. + training_summary['last_train_metrics'] = _float_metric_value( + train_metrics[0]) + training_summary['eval_metrics'] = _float_metric_value(eval_metrics[0]) + + write_txt_summary(training_summary, summary_dir) + + if not _should_export_summary(strategy): + tf.io.gfile.rmtree(summary_dir) + + return model diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/model_training_utils_test.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/model_training_utils_test.py new file mode 100644 index 0000000..647d6b9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/model_training_utils_test.py @@ -0,0 +1,235 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for official.modeling.training.model_training_utils.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from absl.testing import parameterized +from absl.testing.absltest import mock +import numpy as np +import tensorflow as tf + +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +from official.modeling import model_training_utils + + +def eager_strategy_combinations(): + return combinations.combine( + distribution=[ + strategy_combinations.default_strategy, + strategy_combinations.tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + strategy_combinations.mirrored_strategy_with_gpu_and_cpu, + strategy_combinations.mirrored_strategy_with_two_gpus, + ], + mode='eager', + ) + + +def eager_gpu_strategy_combinations(): + return combinations.combine( + distribution=[ + strategy_combinations.default_strategy, + strategy_combinations.one_device_strategy_gpu, + strategy_combinations.mirrored_strategy_with_gpu_and_cpu, + strategy_combinations.mirrored_strategy_with_two_gpus, + ], + mode='eager', + ) + + +def create_fake_data_input_fn(batch_size, features_shape, num_classes): + """Creates a dummy input function with the given feature and label shapes. + + Args: + batch_size: integer. + features_shape: list[int]. Feature shape for an individual example. + num_classes: integer. Number of labels. + + Returns: + An input function that is usable in the executor. + """ + + def _dataset_fn(input_context=None): + """An input function for generating fake data.""" + local_batch_size = input_context.get_per_replica_batch_size(batch_size) + features = np.random.rand(64, *features_shape) + labels = np.random.randint(2, size=[64, num_classes]) + # Convert the inputs to a Dataset. + dataset = tf.data.Dataset.from_tensor_slices((features, labels)) + dataset = dataset.shard(input_context.num_input_pipelines, + input_context.input_pipeline_id) + + def _assign_dtype(features, labels): + features = tf.cast(features, tf.float32) + labels = tf.cast(labels, tf.float32) + return features, labels + + # Shuffle, repeat, and batch the examples. + dataset = dataset.map(_assign_dtype) + dataset = dataset.shuffle(64).repeat() + dataset = dataset.batch(local_batch_size, drop_remainder=True) + dataset = dataset.prefetch(buffer_size=64) + return dataset + + return _dataset_fn + + +def create_model_fn(input_shape, num_classes, use_float16=False): + + def _model_fn(): + """A one-layer softmax model suitable for testing.""" + input_layer = tf.keras.layers.Input(shape=input_shape) + x = tf.keras.layers.Dense(num_classes, activation='relu')(input_layer) + output_layer = tf.keras.layers.Dense(num_classes, activation='softmax')(x) + sub_model = tf.keras.models.Model(input_layer, x, name='sub_model') + model = tf.keras.models.Model(input_layer, output_layer, name='model') + model.add_metric( + tf.reduce_mean(input_layer), name='mean_input', aggregation='mean') + model.optimizer = tf.keras.optimizers.SGD(learning_rate=0.1, momentum=0.9) + if use_float16: + model.optimizer = ( + tf.keras.mixed_precision.experimental.LossScaleOptimizer( + model.optimizer, loss_scale='dynamic')) + return model, sub_model + + return _model_fn + + +def metric_fn(): + """Gets a tf.keras metric object.""" + return tf.keras.metrics.CategoricalAccuracy(name='accuracy', dtype=tf.float32) + + +def summaries_with_matching_keyword(keyword, summary_dir): + """Yields summary protos matching given keyword from event file.""" + event_paths = tf.io.gfile.glob(os.path.join(summary_dir, 'events*')) + for event in tf.compat.v1.train.summary_iterator(event_paths[-1]): + if event.summary is not None: + for value in event.summary.value: + if keyword in value.tag: + tf.compat.v1.logging.error(event) + yield event.summary + + +def check_eventfile_for_keyword(keyword, summary_dir): + """Checks event files for the keyword.""" + return any(summaries_with_matching_keyword(keyword, summary_dir)) + + +class ModelTrainingUtilsTest(tf.test.TestCase, parameterized.TestCase): + + def setUp(self): + super(ModelTrainingUtilsTest, self).setUp() + self._model_fn = create_model_fn(input_shape=[128], num_classes=3) + + def run_training(self, strategy, model_dir, steps_per_loop, run_eagerly): + input_fn = create_fake_data_input_fn( + batch_size=8, features_shape=[128], num_classes=3) + model_training_utils.run_customized_training_loop( + strategy=strategy, + model_fn=self._model_fn, + loss_fn=tf.keras.losses.categorical_crossentropy, + model_dir=model_dir, + steps_per_epoch=20, + steps_per_loop=steps_per_loop, + epochs=2, + train_input_fn=input_fn, + eval_input_fn=input_fn, + eval_steps=10, + init_checkpoint=None, + metric_fn=metric_fn, + custom_callbacks=None, + run_eagerly=run_eagerly) + + @combinations.generate(eager_strategy_combinations()) + def test_train_eager_single_step(self, distribution): + model_dir = self.get_temp_dir() + if isinstance(distribution, tf.distribute.experimental.TPUStrategy): + with self.assertRaises(ValueError): + self.run_training( + distribution, model_dir, steps_per_loop=1, run_eagerly=True) + else: + self.run_training( + distribution, model_dir, steps_per_loop=1, run_eagerly=True) + + @combinations.generate(eager_gpu_strategy_combinations()) + def test_train_eager_mixed_precision(self, distribution): + model_dir = self.get_temp_dir() + policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16') + tf.keras.mixed_precision.experimental.set_policy(policy) + self._model_fn = create_model_fn( + input_shape=[128], num_classes=3, use_float16=True) + self.run_training( + distribution, model_dir, steps_per_loop=1, run_eagerly=True) + + @combinations.generate(eager_strategy_combinations()) + def test_train_check_artifacts(self, distribution): + model_dir = self.get_temp_dir() + self.run_training( + distribution, model_dir, steps_per_loop=10, run_eagerly=False) + + # Two checkpoints should be saved after two epochs. + self.assertNotEmpty(tf.io.gfile.glob(os.path.join(model_dir, 'ctl_step_*'))) + self.assertNotEmpty( + tf.io.gfile.glob( + os.path.join(model_dir, 'summaries/training_summary*'))) + + # Loss and accuracy values should be written into summaries. + self.assertTrue( + check_eventfile_for_keyword('loss', + os.path.join(model_dir, 'summaries/train'))) + self.assertTrue( + check_eventfile_for_keyword('accuracy', + os.path.join(model_dir, 'summaries/train'))) + self.assertTrue( + check_eventfile_for_keyword('mean_input', + os.path.join(model_dir, 'summaries/train'))) + self.assertTrue( + check_eventfile_for_keyword('accuracy', + os.path.join(model_dir, 'summaries/eval'))) + self.assertTrue( + check_eventfile_for_keyword('mean_input', + os.path.join(model_dir, 'summaries/eval'))) + + @combinations.generate( + combinations.combine( + distribution=[ + strategy_combinations.one_device_strategy_gpu, + ], + mode='eager', + )) + def test_train_check_artifacts_non_chief(self, distribution): + # We shouldn't export artifacts on non-chief workers. Since there's no easy + # way to test with real MultiWorkerMirroredStrategy, we patch the strategy + # to make it as if it's MultiWorkerMirroredStrategy on non-chief workers. + extended = distribution.extended + with mock.patch.object(extended.__class__, 'should_checkpoint', + new_callable=mock.PropertyMock, return_value=False), \ + mock.patch.object(extended.__class__, 'should_save_summary', + new_callable=mock.PropertyMock, return_value=False): + model_dir = self.get_temp_dir() + self.run_training( + distribution, model_dir, steps_per_loop=10, run_eagerly=False) + self.assertEmpty(tf.io.gfile.listdir(model_dir)) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/performance.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/performance.py new file mode 100644 index 0000000..4b264f5 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/performance.py @@ -0,0 +1,56 @@ +# Lint as: python3 +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functions and classes related to training performance.""" + +import tensorflow as tf + + +def configure_optimizer(optimizer, + use_float16=False, + use_graph_rewrite=False, + loss_scale="dynamic"): + """Configures optimizer object with performance options.""" + if use_float16: + # Wraps optimizer with a LossScaleOptimizer. This is done automatically + # in compile() with the "mixed_float16" policy, but since we do not call + # compile(), we must wrap the optimizer manually. + optimizer = ( + tf.keras.mixed_precision.experimental.LossScaleOptimizer( + optimizer, loss_scale=loss_scale)) + if use_graph_rewrite: + # Note: the model dtype must be 'float32', which will ensure + # tf.ckeras.mixed_precision and + # tf.train.experimental.enable_mixed_precision_graph_rewrite do not double + # up. + optimizer = tf.train.experimental.enable_mixed_precision_graph_rewrite( + optimizer) + return optimizer + + +def set_mixed_precision_policy(dtype, loss_scale=None): + """Sets mix precision policy.""" + if dtype == tf.float16: + policy = tf.keras.mixed_precision.experimental.Policy( + 'mixed_float16', loss_scale=loss_scale) + tf.keras.mixed_precision.experimental.set_policy(policy) + elif dtype == tf.bfloat16: + policy = tf.keras.mixed_precision.experimental.Policy( + 'mixed_bfloat16') + tf.keras.mixed_precision.experimental.set_policy(policy) + elif dtype == tf.float32: + tf.keras.mixed_precision.experimental.set_policy('float32') + else: + raise ValueError("Unexpected dtype: %s" % dtype) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/tf_utils.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/tf_utils.py new file mode 100644 index 0000000..34f8f66 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/tf_utils.py @@ -0,0 +1,175 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Common TF utilities.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import six +import tensorflow as tf + +from tensorflow.python.util import deprecation +from official.modeling import activations + + +@deprecation.deprecated( + None, + "tf.keras.layers.Layer supports multiple positional args and kwargs as " + "input tensors. pack/unpack inputs to override __call__ is no longer " + "needed." +) +def pack_inputs(inputs): + """Pack a list of `inputs` tensors to a tuple. + + Args: + inputs: a list of tensors. + + Returns: + a tuple of tensors. if any input is None, replace it with a special constant + tensor. + """ + inputs = tf.nest.flatten(inputs) + outputs = [] + for x in inputs: + if x is None: + outputs.append(tf.constant(0, shape=[], dtype=tf.int32)) + else: + outputs.append(x) + return tuple(outputs) + + +@deprecation.deprecated( + None, + "tf.keras.layers.Layer supports multiple positional args and kwargs as " + "input tensors. pack/unpack inputs to override __call__ is no longer " + "needed." +) +def unpack_inputs(inputs): + """unpack a tuple of `inputs` tensors to a tuple. + + Args: + inputs: a list of tensors. + + Returns: + a tuple of tensors. if any input is a special constant tensor, replace it + with None. + """ + inputs = tf.nest.flatten(inputs) + outputs = [] + for x in inputs: + if is_special_none_tensor(x): + outputs.append(None) + else: + outputs.append(x) + x = tuple(outputs) + + # To trick the very pointless 'unbalanced-tuple-unpacking' pylint check + # from triggering. + if len(x) == 1: + return x[0] + return tuple(outputs) + + +def is_special_none_tensor(tensor): + """Checks if a tensor is a special None Tensor.""" + return tensor.shape.ndims == 0 and tensor.dtype == tf.int32 + + +# TODO(hongkuny): consider moving custom string-map lookup to keras api. +def get_activation(identifier): + """Maps a identifier to a Python function, e.g., "relu" => `tf.nn.relu`. + + It checks string first and if it is one of customized activation not in TF, + the corresponding activation will be returned. For non-customized activation + names and callable identifiers, always fallback to tf.keras.activations.get. + + Args: + identifier: String name of the activation function or callable. + + Returns: + A Python function corresponding to the activation function. + """ + if isinstance(identifier, six.string_types): + name_to_fn = { + "gelu": activations.gelu, + "simple_swish": activations.simple_swish, + "hard_swish": activations.hard_swish, + "identity": activations.identity, + } + identifier = str(identifier).lower() + if identifier in name_to_fn: + return tf.keras.activations.get(name_to_fn[identifier]) + return tf.keras.activations.get(identifier) + + +def get_shape_list(tensor, expected_rank=None, name=None): + """Returns a list of the shape of tensor, preferring static dimensions. + + Args: + tensor: A tf.Tensor object to find the shape of. + expected_rank: (optional) int. The expected rank of `tensor`. If this is + specified and the `tensor` has a different rank, and exception will be + thrown. + name: Optional name of the tensor for the error message. + + Returns: + A list of dimensions of the shape of tensor. All static dimensions will + be returned as python integers, and dynamic dimensions will be returned + as tf.Tensor scalars. + """ + if expected_rank is not None: + assert_rank(tensor, expected_rank, name) + + shape = tensor.shape.as_list() + + non_static_indexes = [] + for (index, dim) in enumerate(shape): + if dim is None: + non_static_indexes.append(index) + + if not non_static_indexes: + return shape + + dyn_shape = tf.shape(tensor) + for index in non_static_indexes: + shape[index] = dyn_shape[index] + return shape + + +def assert_rank(tensor, expected_rank, name=None): + """Raises an exception if the tensor rank is not of the expected rank. + + Args: + tensor: A tf.Tensor to check the rank of. + expected_rank: Python integer or list of integers, expected rank. + name: Optional name of the tensor for the error message. + + Raises: + ValueError: If the expected shape doesn't match the actual shape. + """ + expected_rank_dict = {} + if isinstance(expected_rank, six.integer_types): + expected_rank_dict[expected_rank] = True + else: + for x in expected_rank: + expected_rank_dict[x] = True + + actual_rank = tensor.shape.ndims + if actual_rank not in expected_rank_dict: + raise ValueError( + "For the tensor `%s`, the actual tensor rank `%d` (shape = %s) is not " + "equal to the expected tensor rank `%s`" % + (name, actual_rank, str(tensor.shape), str(expected_rank))) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/training/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/training/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/training/distributed_executor.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/training/distributed_executor.py new file mode 100644 index 0000000..1906294 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/modeling/training/distributed_executor.py @@ -0,0 +1,735 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Custom training loop for running TensorFlow 2.0 models.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +import json +import os + +from absl import flags +from absl import logging + +import numpy as np +import tensorflow as tf + +# pylint: disable=unused-import,g-import-not-at-top,redefined-outer-name,reimported +from typing import Optional, Dict, List, Text, Callable, Union, Iterator, Any +from official.modeling.hyperparams import params_dict +from official.utils.misc import distribution_utils +from official.utils import hyperparams_flags + +FLAGS = flags.FLAGS + +strategy_flags_dict = hyperparams_flags.strategy_flags_dict +hparam_flags_dict = hyperparams_flags.hparam_flags_dict + + +def _save_checkpoint(checkpoint, model_dir, checkpoint_prefix): + """Saves model to model_dir with provided checkpoint prefix.""" + + checkpoint_path = os.path.join(model_dir, checkpoint_prefix) + saved_path = checkpoint.save(checkpoint_path) + logging.info('Saving model as TF checkpoint: %s', saved_path) + + +def _steps_to_run(current_step, total_steps, steps_per_loop): + """Calculates steps to run on device.""" + if steps_per_loop <= 0: + raise ValueError('steps_per_loop should be positive integer.') + return min(total_steps - current_step, steps_per_loop) + + +def _no_metric(): + return None + + +class SummaryWriter(object): + """Simple SummaryWriter for writing dictionary of metrics. + + Attributes: + writer: The tf.SummaryWriter. + """ + + def __init__(self, model_dir: Text, name: Text): + """Inits SummaryWriter with paths. + + Arguments: + model_dir: the model folder path. + name: the summary subfolder name. + """ + self.writer = tf.summary.create_file_writer(os.path.join(model_dir, name)) + + def __call__(self, metrics: Union[Dict[Text, float], float], step: int): + """Write metrics to summary with the given writer. + + Args: + metrics: a dictionary of metrics values. Prefer dictionary. + step: integer. The training step. + """ + if not isinstance(metrics, dict): + # Support scalar metric without name. + logging.warning('Warning: summary writer prefer metrics as dictionary.') + metrics = {'metric': metrics} + + with self.writer.as_default(): + for k, v in metrics.items(): + tf.summary.scalar(k, v, step=step) + self.writer.flush() + + +class DistributedExecutor(object): + """Interface to train and eval models with tf.distribute.Strategy. + + Arguments: + strategy: an instance of tf.distribute.Strategy. + params: Model configuration needed to run distribution strategy. + model_fn: Keras model function. Signature: + (params: ParamsDict) -> tf.keras.models.Model. + loss_fn: loss function. Signature: + (y_true: Tensor, y_pred: Tensor) -> Tensor + metric_fn: metric function. Signature: () -> tf.keras.metrics.Metric. + is_multi_host: Set to True when using multi hosts for training, like multi + worker GPU or TPU pod (slice). Otherwise, False. + """ + + def __init__(self, + strategy, + params, + model_fn, + loss_fn, + is_multi_host=False): + + self._params = params + self._model_fn = model_fn + self._loss_fn = loss_fn + self._strategy = strategy + self._checkpoint_name = 'ctl_step_{step}.ckpt' + self._is_multi_host = is_multi_host + self.train_summary_writer = None + self.eval_summary_writer = None + self.global_train_step = None + + @property + def checkpoint_name(self): + """Returns default checkpoint name.""" + return self._checkpoint_name + + @checkpoint_name.setter + def checkpoint_name(self, name): + """Sets default summary writer for the current thread.""" + self._checkpoint_name = name + + def loss_fn(self): + return self._loss_fn() + + def model_fn(self, params): + return self._model_fn(params) + + def _save_config(self, model_dir): + """Save parameters to config files if model_dir is defined.""" + + logging.info('Save config to model_dir %s.', model_dir) + if model_dir: + if not tf.io.gfile.exists(model_dir): + tf.io.gfile.makedirs(model_dir) + self._params.lock() + params_dict.save_params_dict_to_yaml(self._params, + model_dir + '/params.yaml') + else: + logging.warning('model_dir is empty, so skip the save config.') + + def _get_input_iterator( + self, input_fn: Callable[..., tf.data.Dataset], + strategy: tf.distribute.Strategy) -> Optional[Iterator[Any]]: + """Returns distributed dataset iterator. + + Args: + input_fn: (params: dict) -> tf.data.Dataset. + strategy: an instance of tf.distribute.Strategy. + + Returns: + An iterator that yields input tensors. + """ + + if input_fn is None: + return None + # When training with multiple TPU workers, datasets needs to be cloned + # across workers. Since Dataset instance cannot be cloned in eager mode, + # we instead pass callable that returns a dataset. + if self._is_multi_host: + return iter( + strategy.experimental_distribute_datasets_from_function(input_fn)) + else: + input_data = input_fn() + return iter(strategy.experimental_distribute_dataset(input_data)) + + def _create_replicated_step(self, + strategy, + model, + loss_fn, + optimizer, + metric=None): + + def _replicated_step(inputs): + """Replicated training step.""" + inputs, labels = inputs + + with tf.GradientTape() as tape: + outputs = model(inputs, training=True) + prediction_loss = loss_fn(labels, outputs) + loss = tf.reduce_mean(prediction_loss) + loss = loss / strategy.num_replicas_in_sync + if isinstance(metric, tf.keras.metrics.Metric): + metric.update_state(labels, outputs) + else: + logging.error('train metric is not an instance of ' + 'tf.keras.metrics.Metric.') + + grads = tape.gradient(loss, model.trainable_variables) + optimizer.apply_gradients(zip(grads, model.trainable_variables)) + return loss + + return _replicated_step + + def _create_train_step(self, + strategy, + model, + loss_fn, + optimizer, + metric=None): + """Creates a distributed training step. + + Args: + strategy: an instance of tf.distribute.Strategy. + model: (Tensor, bool) -> Tensor. model function. + loss_fn: (y_true: Tensor, y_pred: Tensor) -> Tensor. + optimizer: tf.keras.optimizers.Optimizer. + iterator: an iterator that yields input tensors. + metric: tf.keras.metrics.Metric subclass. + + Returns: + The training step callable. + """ + _replicated_step = self._create_replicated_step(strategy, model, loss_fn, + optimizer, metric) + + @tf.function + def train_step(iterator, num_steps): + """Performs a distributed training step. + + Args: + iterator: an iterator that yields input tensors. + + Returns: + The loss tensor. + """ + if not isinstance(num_steps, tf.Tensor): + raise ValueError('steps should be an Tensor. Python object may cause ' + 'retracing.') + + per_replica_losses = strategy.run( + _replicated_step, args=(next(iterator),)) + for _ in tf.range(num_steps - 1): + per_replica_losses = strategy.run( + _replicated_step, args=(next(iterator),)) + + # For reporting, we returns the mean of losses. + losses = tf.nest.map_structure( + lambda x: strategy.reduce(tf.distribute.ReduceOp.MEAN, x, axis=None), + per_replica_losses) + return losses + + return train_step + + def _create_test_step(self, strategy, model, metric): + """Creates a distributed test step.""" + + @tf.function + def test_step(iterator): + """Calculates evaluation metrics on distributed devices.""" + if not metric: + logging.info('Skip test_step because metric is None (%s)', metric) + return None, None + if not isinstance(metric, tf.keras.metrics.Metric): + raise ValueError( + 'Metric must be an instance of tf.keras.metrics.Metric ' + 'for running in test_step. Actual {}'.format(metric)) + + def _test_step_fn(inputs): + """Replicated accuracy calculation.""" + inputs, labels = inputs + model_outputs = model(inputs, training=False) + metric.update_state(labels, model_outputs) + return labels, model_outputs + + return strategy.run(_test_step_fn, args=(next(iterator),)) + + return test_step + + def train(self, + train_input_fn: Callable[[params_dict.ParamsDict], tf.data.Dataset], + eval_input_fn: Callable[[params_dict.ParamsDict], + tf.data.Dataset] = None, + model_dir: Text = None, + total_steps: int = 1, + iterations_per_loop: int = 1, + train_metric_fn: Callable[[], Any] = None, + eval_metric_fn: Callable[[], Any] = None, + summary_writer_fn: Callable[[Text, Text], + SummaryWriter] = SummaryWriter, + init_checkpoint: Callable[[tf.keras.Model], Any] = None, + custom_callbacks: List[tf.keras.callbacks.Callback] = None, + save_config: bool = True): + """Runs distributed training. + + Args: + train_input_fn: (params: dict) -> tf.data.Dataset training data input + function. + eval_input_fn: (Optional) same type as train_input_fn. If not None, will + trigger evaluting metric on eval data. If None, will not run eval step. + model_dir: the folder path for model checkpoints. + total_steps: total training steps. + iterations_per_loop: train steps per loop. After each loop, this job will + update metrics like loss and save checkpoint. + train_metric_fn: metric_fn for evaluation in train_step. + eval_metric_fn: metric_fn for evaluation in test_step. + summary_writer_fn: function to create summary writer. + init_checkpoint: function to load checkpoint. + custom_callbacks: A list of Keras Callbacks objects to run during + training. More specifically, `on_batch_begin()`, `on_batch_end()`, + methods are invoked during training. + save_config: bool. Whether to save params to model_dir. + + Returns: + The training loss and eval metrics. + """ + assert train_input_fn is not None + if train_metric_fn and not callable(train_metric_fn): + raise ValueError('if `train_metric_fn` is specified, ' + 'train_metric_fn must be a callable.') + if eval_metric_fn and not callable(eval_metric_fn): + raise ValueError('if `eval_metric_fn` is specified, ' + 'eval_metric_fn must be a callable.') + train_metric_fn = train_metric_fn or _no_metric + eval_metric_fn = eval_metric_fn or _no_metric + + if custom_callbacks and iterations_per_loop != 1: + logging.error( + 'It is sematically wrong to run callbacks when ' + 'iterations_per_loop is not one (%s)', iterations_per_loop) + + def _run_callbacks_on_batch_begin(batch): + """Runs custom callbacks at the start of every step.""" + if not custom_callbacks: + return + for callback in custom_callbacks: + if callback: + callback.on_batch_begin(batch) + + def _run_callbacks_on_batch_end(batch): + """Runs custom callbacks at the end of every step.""" + if not custom_callbacks: + return + for callback in custom_callbacks: + if callback: + callback.on_batch_end(batch) + + if save_config: + self._save_config(model_dir) + + if FLAGS.save_checkpoint_freq: + save_freq = FLAGS.save_checkpoint_freq + else: + save_freq = iterations_per_loop + + params = self._params + strategy = self._strategy + # To reduce unnecessary send/receive input pipeline operation, we place + # input pipeline ops in worker task. + train_iterator = self._get_input_iterator(train_input_fn, strategy) + train_loss = None + eval_metric_result = None + with strategy.scope(): + # To correctly place the model weights on accelerators, + # model and optimizer should be created in scope. + model = self.model_fn(params.as_dict()) + if not hasattr(model, 'optimizer'): + raise ValueError('User should set optimizer attribute to model ' + 'inside `model_fn`.') + optimizer = model.optimizer + + # Training loop starts here. + checkpoint = tf.train.Checkpoint(model=model, optimizer=optimizer) + latest_checkpoint_file = tf.train.latest_checkpoint(model_dir) + initial_step = 0 + if latest_checkpoint_file: + logging.info( + 'Checkpoint file %s found and restoring from ' + 'checkpoint', latest_checkpoint_file) + checkpoint.restore(latest_checkpoint_file) + initial_step = optimizer.iterations.numpy() + logging.info('Loading from checkpoint file completed. Init step %d', + initial_step) + elif init_checkpoint: + logging.info('Restoring from init checkpoint function') + init_checkpoint(model) + logging.info('Loading from init checkpoint file completed') + + current_step = optimizer.iterations.numpy() + checkpoint_name = self.checkpoint_name + + eval_metric = eval_metric_fn() + train_metric = train_metric_fn() + train_summary_writer = summary_writer_fn(model_dir, 'eval_train') + self.train_summary_writer = train_summary_writer.writer + + test_summary_writer = summary_writer_fn(model_dir, 'eval_test') + self.eval_summary_writer = test_summary_writer.writer + + # Continue training loop. + train_step = self._create_train_step( + strategy=strategy, + model=model, + loss_fn=self.loss_fn(), + optimizer=optimizer, + metric=train_metric) + test_step = None + if eval_input_fn and eval_metric: + self.global_train_step = model.optimizer.iterations + test_step = self._create_test_step(strategy, model, metric=eval_metric) + + logging.info('Training started') + last_save_checkpoint_step = current_step + while current_step < total_steps: + + num_steps = _steps_to_run(current_step, total_steps, iterations_per_loop) + _run_callbacks_on_batch_begin(current_step) + train_loss = train_step(train_iterator, + tf.convert_to_tensor(num_steps, dtype=tf.int32)) + _run_callbacks_on_batch_end(current_step) + current_step += num_steps + + train_loss = tf.nest.map_structure(lambda x: x.numpy().astype(float), + train_loss) + if not isinstance(train_loss, dict): + train_loss = {'total_loss': train_loss} + if np.isnan(train_loss['total_loss']): + raise ValueError('total loss is NaN.') + + if train_metric: + train_metric_result = train_metric.result() + if isinstance(train_metric, tf.keras.metrics.Metric): + train_metric_result = tf.nest.map_structure( + lambda x: x.numpy().astype(float), train_metric_result) + if not isinstance(train_metric_result, dict): + train_metric_result = {'metric': train_metric_result} + train_metric_result.update(train_loss) + else: + train_metric_result = train_loss + if callable(optimizer.lr): + train_metric_result.update( + {'learning_rate': optimizer.lr(current_step).numpy()}) + else: + train_metric_result.update({'learning_rate': optimizer.lr.numpy()}) + logging.info('Train Step: %d/%d / loss = %s / training metric = %s', + current_step, total_steps, train_loss, + train_metric_result) + + train_summary_writer( + metrics=train_metric_result, step=optimizer.iterations) + + # Saves model checkpoints and run validation steps at every + # iterations_per_loop steps. + # To avoid repeated model saving, we do not save after the last + # step of training. + if save_freq > 0 and current_step < total_steps and ( + current_step - last_save_checkpoint_step) >= save_freq: + _save_checkpoint(checkpoint, model_dir, + checkpoint_name.format(step=current_step)) + last_save_checkpoint_step = current_step + + if test_step: + eval_iterator = self._get_input_iterator(eval_input_fn, strategy) + eval_metric_result = self._run_evaluation(test_step, current_step, + eval_metric, eval_iterator) + logging.info('Step: %s evalation metric = %s.', current_step, + eval_metric_result) + test_summary_writer( + metrics=eval_metric_result, step=optimizer.iterations) + + # Re-initialize evaluation metric, except the last step. + if eval_metric and current_step < total_steps: + eval_metric.reset_states() + if train_metric and current_step < total_steps: + train_metric.reset_states() + + # Reaches the end of training and saves the last checkpoint. + if last_save_checkpoint_step < total_steps: + _save_checkpoint(checkpoint, model_dir, + checkpoint_name.format(step=current_step)) + + if test_step: + logging.info('Running final evaluation after training is complete.') + eval_iterator = self._get_input_iterator(eval_input_fn, strategy) + eval_metric_result = self._run_evaluation(test_step, current_step, + eval_metric, eval_iterator) + logging.info('Final evaluation metric = %s.', eval_metric_result) + test_summary_writer( + metrics=eval_metric_result, step=optimizer.iterations) + + return train_loss, eval_metric_result + + def _run_evaluation(self, test_step, current_training_step, metric, + test_iterator): + """Runs validation steps and aggregate metrics.""" + if not test_iterator or not metric: + logging.warning( + 'Both test_iterator (%s) and metrics (%s) must not be None.', + test_iterator, metric) + return None + logging.info('Running evaluation after step: %s.', current_training_step) + while True: + try: + test_step(test_iterator) + except (StopIteration, tf.errors.OutOfRangeError): + break + + metric_result = metric.result() + if isinstance(metric, tf.keras.metrics.Metric): + metric_result = metric_result.numpy().astype(float) + logging.info('Step: [%d] Validation metric = %f', current_training_step, + metric_result) + return metric_result + + def evaluate_from_model_dir( + self, + model_dir: Text, + eval_input_fn: Callable[[params_dict.ParamsDict], tf.data.Dataset], + eval_metric_fn: Callable[[], Any], + total_steps: int = -1, + eval_timeout: int = None, + min_eval_interval: int = 180, + summary_writer_fn: Callable[[Text, Text], SummaryWriter] = SummaryWriter): + """Runs distributed evaluation on model folder. + + Args: + eval_input_fn: (Optional) same type as train_input_fn. If not None, will + trigger evaluting metric on eval data. If None, will not run eval step. + eval_metric_fn: metric_fn for evaluation in test_step. + model_dir: the folder for storing model checkpoints. + total_steps: total training steps. If the current step reaches the + total_steps, the evaluation loop will stop. + eval_timeout: The maximum number of seconds to wait between checkpoints. + If left as None, then the process will wait indefinitely. Used by + tf.train.checkpoints_iterator. + min_eval_interval: The minimum number of seconds between yielding + checkpoints. Used by tf.train.checkpoints_iterator. + summary_writer_fn: function to create summary writer. + + Returns: + Eval metrics dictionary of the last checkpoint. + """ + + if not model_dir: + raise ValueError('model_dir must be set.') + + def terminate_eval(): + tf.logging.info('Terminating eval after %d seconds of no checkpoints' % + eval_timeout) + return True + + summary_writer = summary_writer_fn(model_dir, 'eval') + self.eval_summary_writer = summary_writer.writer + + # Read checkpoints from the given model directory + # until `eval_timeout` seconds elapses. + for checkpoint_path in tf.train.checkpoints_iterator( + model_dir, + min_interval_secs=min_eval_interval, + timeout=eval_timeout, + timeout_fn=terminate_eval): + eval_metric_result, current_step = self.evaluate_checkpoint( + checkpoint_path=checkpoint_path, + eval_input_fn=eval_input_fn, + eval_metric_fn=eval_metric_fn, + summary_writer=summary_writer) + if total_steps > 0 and current_step >= total_steps: + logging.info('Evaluation finished after training step %d', current_step) + break + return eval_metric_result + + def evaluate_checkpoint(self, + checkpoint_path: Text, + eval_input_fn: Callable[[params_dict.ParamsDict], + tf.data.Dataset], + eval_metric_fn: Callable[[], Any], + summary_writer: SummaryWriter = None): + """Runs distributed evaluation on the one checkpoint. + + Args: + eval_input_fn: (Optional) same type as train_input_fn. If not None, will + trigger evaluting metric on eval data. If None, will not run eval step. + eval_metric_fn: metric_fn for evaluation in test_step. + checkpoint_path: the checkpoint to evaluate. + summary_writer_fn: function to create summary writer. + + Returns: + Eval metrics dictionary of the last checkpoint. + """ + if not callable(eval_metric_fn): + raise ValueError('if `eval_metric_fn` is specified, ' + 'eval_metric_fn must be a callable.') + + params = self._params + strategy = self._strategy + # To reduce unnecessary send/receive input pipeline operation, we place + # input pipeline ops in worker task. + with strategy.scope(): + + # To correctly place the model weights on accelerators, + # model and optimizer should be created in scope. + model = self.model_fn(params.as_dict()) + checkpoint = tf.train.Checkpoint(model=model) + + eval_metric = eval_metric_fn() + assert eval_metric, 'eval_metric does not exist' + test_step = self._create_test_step(strategy, model, metric=eval_metric) + + logging.info('Starting to evaluate.') + if not checkpoint_path: + raise ValueError('checkpoint path is empty') + reader = tf.compat.v1.train.NewCheckpointReader(checkpoint_path) + current_step = reader.get_tensor( + 'optimizer/iter/.ATTRIBUTES/VARIABLE_VALUE') + logging.info( + 'Checkpoint file %s found and restoring from ' + 'checkpoint', checkpoint_path) + checkpoint.restore(checkpoint_path) + + self.global_train_step = model.optimizer.iterations + eval_iterator = self._get_input_iterator(eval_input_fn, strategy) + eval_metric_result = self._run_evaluation(test_step, current_step, + eval_metric, eval_iterator) + logging.info('Step: %s evalation metric = %s.', current_step, + eval_metric_result) + summary_writer(metrics=eval_metric_result, step=current_step) + eval_metric.reset_states() + + return eval_metric_result, current_step + + def predict(self): + return NotImplementedError('Unimplmented function.') + + +class ExecutorBuilder(object): + """Builder of DistributedExecutor. + + Example 1: Builds an executor with supported Strategy. + builder = ExecutorBuilder( + strategy_type='tpu', + strategy_config={'tpu': '/bns/xxx'}) + dist_executor = builder.build_executor( + params=params, + model_fn=my_model_fn, + loss_fn=my_loss_fn, + metric_fn=my_metric_fn) + + Example 2: Builds an executor with customized Strategy. + builder = ExecutorBuilder() + builder.strategy = + dist_executor = builder.build_executor( + params=params, + model_fn=my_model_fn, + loss_fn=my_loss_fn, + metric_fn=my_metric_fn) + + Example 3: Builds a customized executor with customized Strategy. + class MyDistributedExecutor(DistributedExecutor): + # implementation ... + + builder = ExecutorBuilder() + builder.strategy = + dist_executor = builder.build_executor( + class_ctor=MyDistributedExecutor, + params=params, + model_fn=my_model_fn, + loss_fn=my_loss_fn, + metric_fn=my_metric_fn) + + Args: + strategy_type: string. One of 'tpu', 'mirrored', 'multi_worker_mirrored'. If + None. User is responsible to set the strategy before calling + build_executor(...). + strategy_config: necessary config for constructing the proper Strategy. + Check strategy_flags_dict() for examples of the structure. + """ + + def __init__(self, strategy_type=None, strategy_config=None): + _ = distribution_utils.configure_cluster( + strategy_config.worker_hosts, strategy_config.task_index) + self._strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=strategy_type, + num_gpus=strategy_config.num_gpus, + all_reduce_alg=strategy_config.all_reduce_alg, + num_packs=strategy_config.num_packs, + tpu_address=strategy_config.tpu) + + @property + def strategy(self): + """Returns default checkpoint name.""" + return self._strategy + + @strategy.setter + def strategy(self, new_strategy): + """Sets default summary writer for the current thread.""" + self._strategy = new_strategy + + + def build_executor(self, + class_ctor=DistributedExecutor, + params=None, + model_fn=None, + loss_fn=None, + **kwargs): + """Creates an executor according to strategy type. + + See doc string of the DistributedExecutor.__init__ for more information of + the + input arguments. + + Args: + class_ctor: A constructor of executor (default: DistributedExecutor). + params: ParamsDict, all the model parameters and runtime parameters. + model_fn: Keras model function. + loss_fn: loss function. + **kwargs: other arguments to the executor constructor. + + Returns: + An instance of DistributedExecutor or its subclass. + """ + if self._strategy is None: + raise ValueError('`strategy` should not be None. You need to specify ' + '`strategy_type` in the builder contructor or directly ' + 'set the `strategy` property of the builder.') + return class_ctor( + strategy=self._strategy, + params=params, + model_fn=model_fn, + loss_fn=loss_fn, + **kwargs) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/pip_package/setup.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/pip_package/setup.py new file mode 100644 index 0000000..bfd95a1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/pip_package/setup.py @@ -0,0 +1,88 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Sets up TensorFlow Official Models.""" +import datetime +import os +import sys + +from setuptools import find_packages +from setuptools import setup + +version = '2.2.0' + +project_name = 'tf-models-official' + +long_description = """The TensorFlow official models are a collection of +models that use TensorFlow's high-level APIs. +They are intended to be well-maintained, tested, and kept up to date with the +latest TensorFlow API. They should also be reasonably optimized for fast +performance while still being easy to read.""" + +if '--project_name' in sys.argv: + project_name_idx = sys.argv.index('--project_name') + project_name = sys.argv[project_name_idx + 1] + sys.argv.remove('--project_name') + sys.argv.pop(project_name_idx) + + +def _get_requirements(): + """Parses requirements.txt file.""" + install_requires_tmp = [] + dependency_links_tmp = [] + with open( + os.path.join(os.path.dirname(__file__), '../requirements.txt'), 'r') as f: + for line in f: + package_name = line.strip() + if package_name.startswith('-e '): + dependency_links_tmp.append(package_name[3:].strip()) + else: + install_requires_tmp.append(package_name) + return install_requires_tmp, dependency_links_tmp + +install_requires, dependency_links = _get_requirements() + +if project_name == 'tf-models-nightly': + version += '.dev' + datetime.datetime.now().strftime('%Y%m%d') + install_requires.append('tf-nightly') +else: + install_requires.append('tensorflow>=2.1.0') + +print('install_requires: ', install_requires) +print('dependency_links: ', dependency_links) + +setup( + name=project_name, + version=version, + description='TensorFlow Official Models', + long_description=long_description, + author='Google Inc.', + author_email='no-reply@google.com', + url='https://github.com/tensorflow/models', + license='Apache 2.0', + packages=find_packages(exclude=[ + 'research*', + 'tutorials*', + 'samples*', + 'official.r1*', + 'official.pip_package*', + 'official.benchmark*', + ]), + exclude_package_data={ + '': ['*_test.py',], + }, + install_requires=install_requires, + dependency_links=dependency_links, + python_requires='>=3.6', +) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/README.md b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/README.md new file mode 100644 index 0000000..203b295 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/README.md @@ -0,0 +1,23 @@ +# Legacy Models Collection + +The R1 folder contains legacy model implmentation and models that will not +update to TensorFlow 2.x. They do not have solid performance tracking. + +**Note: models will be removed from the master branch by 2020/06.** + +After removal, you can still access to these legacy models in the previous +released tags, e.g. [v2.1.0](https://github.com/tensorflow/models/releases/tag/v2.1.0). + + +## Legacy model implmentation + +Transformer and MNIST implementation uses pure TF 1.x TF-Estimator. +Users should follow the corresponding TF 2.x implmentation inside the +official model garden. + +## Models that will not update to TensorFlow 2.x + +* [boosted_trees](boosted_trees): A Gradient Boosted Trees model to + classify higgs boson process from HIGGS Data Set. +* [wide_deep](wide_deep): A model that combines a wide model and deep + network to classify census income data. diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/boosted_trees/README.md b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/boosted_trees/README.md new file mode 100644 index 0000000..d8c8fcc --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/boosted_trees/README.md @@ -0,0 +1,112 @@ +# Classifying Higgs boson processes in the HIGGS Data Set +## Overview +The [HIGGS Data Set](https://archive.ics.uci.edu/ml/datasets/HIGGS) contains 11 million samples with 28 features, and is for the classification problem to distinguish between a signal process which produces Higgs bosons and a background process which does not. + +We use Gradient Boosted Trees algorithm to distinguish the two classes. + +--- + +The code sample uses the high level `tf.estimator.Estimator` and `tf.data.Dataset`. These APIs are great for fast iteration and quickly adapting models to your own datasets without major code overhauls. It allows you to move from single-worker training to distributed training, and makes it easy to export model binaries for prediction. Here, for further simplicity and faster execution, we use a utility function `tf.contrib.estimator.boosted_trees_classifier_train_in_memory`. This utility function is especially effective when the input is provided as in-memory data sets like numpy arrays. + +An input function for the `Estimator` typically uses `tf.data.Dataset` API, which can handle various data control like streaming, batching, transform and shuffling. However `boosted_trees_classifier_train_in_memory()` utility function requires that the entire data is provided as a single batch (i.e. without using `batch()` API). Thus in this practice, simply `Dataset.from_tensors()` is used to convert numpy arrays into structured tensors, and `Dataset.zip()` is used to put features and label together. +For further references of `Dataset`, [Read more here](https://www.tensorflow.org/guide/datasets). + +## Running the code +First make sure you've [added the models folder to your Python path](/official/#running-the-models); otherwise you may encounter an error like `ImportError: No module named official.boosted_trees`. + +### Setup +The [HIGGS Data Set](https://archive.ics.uci.edu/ml/datasets/HIGGS) that this sample uses for training is hosted by the [UC Irvine Machine Learning Repository](https://archive.ics.uci.edu/ml/datasets/). We have provided a script that downloads and cleans the necessary files. + +``` +python data_download.py +``` + +This will download a file and store the processed file under the directory designated by `--data_dir` (defaults to `/tmp/higgs_data/`). To change the target directory, set the `--data_dir` flag. The directory could be network storages that Tensorflow supports (like Google Cloud Storage, `gs:////`). +The file downloaded to the local temporary folder is about 2.8 GB, and the processed file is about 0.8 GB, so there should be enough storage to handle them. + + +### Training + +This example uses about 3 GB of RAM during training. +You can run the code locally as follows: + +``` +python train_higgs.py +``` + +The model is by default saved to `/tmp/higgs_model`, which can be changed using the `--model_dir` flag. +Note that the model_dir is cleaned up before every time training starts. + +Model parameters can be adjusted by flags, like `--n_trees`, `--max_depth`, `--learning_rate` and so on. Check out the code for details. + +The final accuracy will be around 74% and loss will be around 0.516 over the eval set, when trained with the default parameters. + +By default, the first 1 million examples among 11 millions are used for training, and the last 1 million examples are used for evaluation. +The training/evaluation data can be selected as index ranges by flags `--train_start`, `--train_count`, `--eval_start`, `--eval_count`, etc. + +### TensorBoard + +Run TensorBoard to inspect the details about the graph and training progression. + +``` +tensorboard --logdir=/tmp/higgs_model # set logdir as --model_dir set during training. +``` + +## Inference with SavedModel +You can export the model into Tensorflow [SavedModel](https://www.tensorflow.org/guide/saved_model) format by using the argument `--export_dir`: + +``` +python train_higgs.py --export_dir /tmp/higgs_boosted_trees_saved_model +``` + +After the model finishes training, use [`saved_model_cli`](https://www.tensorflow.org/guide/saved_model#cli_to_inspect_and_execute_savedmodel) to inspect and execute the SavedModel. + +Try the following commands to inspect the SavedModel: + +**Replace `${TIMESTAMP}` with the folder produced (e.g. 1524249124)** +``` +# List possible tag_sets. Only one metagraph is saved, so there will be one option. +saved_model_cli show --dir /tmp/higgs_boosted_trees_saved_model/${TIMESTAMP}/ + +# Show SignatureDefs for tag_set=serve. SignatureDefs define the outputs to show. +saved_model_cli show --dir /tmp/higgs_boosted_trees_saved_model/${TIMESTAMP}/ \ + --tag_set serve --all +``` + +### Inference +Let's use the model to predict the income group of two examples. +Note that this model exports SavedModel with the custom parsing module that accepts csv lines as features. (Each line is an example with 28 columns; be careful to not add a label column, unlike in the training data.) + +``` +saved_model_cli run --dir /tmp/boosted_trees_higgs_saved_model/${TIMESTAMP}/ \ + --tag_set serve --signature_def="predict" \ + --input_exprs='inputs=["0.869293,-0.635082,0.225690,0.327470,-0.689993,0.754202,-0.248573,-1.092064,0.0,1.374992,-0.653674,0.930349,1.107436,1.138904,-1.578198,-1.046985,0.0,0.657930,-0.010455,-0.045767,3.101961,1.353760,0.979563,0.978076,0.920005,0.721657,0.988751,0.876678", "1.595839,-0.607811,0.007075,1.818450,-0.111906,0.847550,-0.566437,1.581239,2.173076,0.755421,0.643110,1.426367,0.0,0.921661,-1.190432,-1.615589,0.0,0.651114,-0.654227,-1.274345,3.101961,0.823761,0.938191,0.971758,0.789176,0.430553,0.961357,0.957818"]' +``` + +This will print out the predicted classes and class probabilities. Something like: + +``` +Result for output key class_ids: +[[1] + [0]] +Result for output key classes: +[['1'] + ['0']] +Result for output key logistic: +[[0.6440273 ] + [0.10902369]] +Result for output key logits: +[[ 0.59288704] + [-2.1007526 ]] +Result for output key probabilities: +[[0.3559727 0.6440273] + [0.8909763 0.1090237]] +``` + +Please note that "predict" signature_def gives out different (more detailed) results than "classification" or "serving_default". + +## Additional Links + +If you are interested in distributed training, take a look at [Distributed TensorFlow](https://www.tensorflow.org/deploy/distributed). + +You can also [train models on Cloud ML Engine](https://cloud.google.com/ml-engine/docs/getting-started-training-prediction), which provides [hyperparameter tuning](https://cloud.google.com/ml-engine/docs/getting-started-training-prediction#hyperparameter_tuning) to maximize your model's results and enables [deploying your model for prediction](https://cloud.google.com/ml-engine/docs/getting-started-training-prediction#deploy_a_model_to_support_prediction). diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/boosted_trees/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/boosted_trees/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/boosted_trees/data_download.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/boosted_trees/data_download.py new file mode 100644 index 0000000..1b6fc05 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/boosted_trees/data_download.py @@ -0,0 +1,97 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Downloads the UCI HIGGS Dataset and prepares train data. + +The details on the dataset are in https://archive.ics.uci.edu/ml/datasets/HIGGS + +It takes a while as it needs to download 2.8 GB over the network, process, then +store it into the specified location as a compressed numpy file. + +Usage: +$ python data_download.py --data_dir=/tmp/higgs_data +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gzip +import os +import tempfile + +# pylint: disable=g-bad-import-order +import numpy as np +import pandas as pd +from six.moves import urllib +from absl import app as absl_app +from absl import flags +import tensorflow as tf + +from official.utils.flags import core as flags_core + +URL_ROOT = "https://archive.ics.uci.edu/ml/machine-learning-databases/00280" +INPUT_FILE = "HIGGS.csv.gz" +NPZ_FILE = "HIGGS.csv.gz.npz" # numpy compressed file to contain "data" array. + + +def _download_higgs_data_and_save_npz(data_dir): + """Download higgs data and store as a numpy compressed file.""" + input_url = URL_ROOT + "/" + INPUT_FILE + np_filename = os.path.join(data_dir, NPZ_FILE) + if tf.gfile.Exists(np_filename): + raise ValueError("data_dir already has the processed data file: {}".format( + np_filename)) + if not tf.gfile.Exists(data_dir): + tf.gfile.MkDir(data_dir) + # 2.8 GB to download. + try: + tf.logging.info("Data downloading...") + temp_filename, _ = urllib.request.urlretrieve(input_url) + # Reading and parsing 11 million csv lines takes 2~3 minutes. + tf.logging.info("Data processing... taking multiple minutes...") + with gzip.open(temp_filename, "rb") as csv_file: + data = pd.read_csv( + csv_file, + dtype=np.float32, + names=["c%02d" % i for i in range(29)] # label + 28 features. + ).as_matrix() + finally: + tf.gfile.Remove(temp_filename) + + # Writing to temporary location then copy to the data_dir (0.8 GB). + f = tempfile.NamedTemporaryFile() + np.savez_compressed(f, data=data) + tf.gfile.Copy(f.name, np_filename) + tf.logging.info("Data saved to: {}".format(np_filename)) + + +def main(unused_argv): + if not tf.gfile.Exists(FLAGS.data_dir): + tf.gfile.MkDir(FLAGS.data_dir) + _download_higgs_data_and_save_npz(FLAGS.data_dir) + + +def define_data_download_flags(): + """Add flags specifying data download arguments.""" + flags.DEFINE_string( + name="data_dir", default="/tmp/higgs_data", + help=flags_core.help_wrap( + "Directory to download higgs dataset and store training/eval data.")) + + +if __name__ == "__main__": + tf.logging.set_verbosity(tf.logging.INFO) + define_data_download_flags() + FLAGS = flags.FLAGS + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/boosted_trees/train_higgs.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/boosted_trees/train_higgs.py new file mode 100644 index 0000000..d496a39 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/boosted_trees/train_higgs.py @@ -0,0 +1,297 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""A script that builds boosted trees over higgs data. + +If you haven't, please run data_download.py beforehand to prepare the data. + +For some more details on this example, please refer to README.md as well. + +Note that the model_dir is cleaned up before starting the training. + +Usage: +$ python train_higgs.py --n_trees=100 --max_depth=6 --learning_rate=0.1 \ + --model_dir=/tmp/higgs_model + +Note that BoostedTreesClassifier is available since Tensorflow 1.8.0. +So you need to install recent enough version of Tensorflow to use this example. + +The training data is by default the first million examples out of 11M examples, +and eval data is by default the last million examples. +They are controlled by --train_start, --train_count, --eval_start, --eval_count. +e.g. to train over the first 10 million examples instead of 1 million: +$ python train_higgs.py --n_trees=100 --max_depth=6 --learning_rate=0.1 \ + --model_dir=/tmp/higgs_model --train_count=10000000 + +Training history and metrics can be inspected using tensorboard. +Set --logdir as the --model_dir set by flag when training +(or the default /tmp/higgs_model). +$ tensorboard --logdir=/tmp/higgs_model +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +# pylint: disable=g-bad-import-order +import numpy as np +from absl import app as absl_app +from absl import flags +import tensorflow as tf +# pylint: enable=g-bad-import-order + +from official.utils.flags import core as flags_core +from official.utils.flags._conventions import help_wrap +from official.utils.logs import logger + +NPZ_FILE = "HIGGS.csv.gz.npz" # numpy compressed file containing "data" array + + +def read_higgs_data(data_dir, train_start, train_count, eval_start, eval_count): + """Reads higgs data from csv and returns train and eval data. + + Args: + data_dir: A string, the directory of higgs dataset. + train_start: An integer, the start index of train examples within the data. + train_count: An integer, the number of train examples within the data. + eval_start: An integer, the start index of eval examples within the data. + eval_count: An integer, the number of eval examples within the data. + + Returns: + Numpy array of train data and eval data. + """ + npz_filename = os.path.join(data_dir, NPZ_FILE) + try: + # gfile allows numpy to read data from network data sources as well. + with tf.gfile.Open(npz_filename, "rb") as npz_file: + with np.load(npz_file) as npz: + data = npz["data"] + except tf.errors.NotFoundError as e: + raise RuntimeError( + "Error loading data; use data_download.py to prepare the data.\n{}: {}" + .format(type(e).__name__, e)) + return (data[train_start:train_start+train_count], + data[eval_start:eval_start+eval_count]) + + +# This showcases how to make input_fn when the input data is available in the +# form of numpy arrays. +def make_inputs_from_np_arrays(features_np, label_np): + """Makes and returns input_fn and feature_columns from numpy arrays. + + The generated input_fn will return tf.data.Dataset of feature dictionary and a + label, and feature_columns will consist of the list of + tf.feature_column.BucketizedColumn. + + Note, for in-memory training, tf.data.Dataset should contain the whole data + as a single tensor. Don't use batch. + + Args: + features_np: A numpy ndarray (shape=[batch_size, num_features]) for + float32 features. + label_np: A numpy ndarray (shape=[batch_size, 1]) for labels. + + Returns: + input_fn: A function returning a Dataset of feature dict and label. + feature_names: A list of feature names. + feature_column: A list of tf.feature_column.BucketizedColumn. + """ + num_features = features_np.shape[1] + features_np_list = np.split(features_np, num_features, axis=1) + # 1-based feature names. + feature_names = ["feature_%02d" % (i + 1) for i in range(num_features)] + + # Create source feature_columns and bucketized_columns. + def get_bucket_boundaries(feature): + """Returns bucket boundaries for feature by percentiles.""" + return np.unique(np.percentile(feature, range(0, 100))).tolist() + source_columns = [ + tf.feature_column.numeric_column( + feature_name, dtype=tf.float32, + # Although higgs data have no missing values, in general, default + # could be set as 0 or some reasonable value for missing values. + default_value=0.0) + for feature_name in feature_names + ] + bucketized_columns = [ + tf.feature_column.bucketized_column( + source_columns[i], + boundaries=get_bucket_boundaries(features_np_list[i])) + for i in range(num_features) + ] + + # Make an input_fn that extracts source features. + def input_fn(): + """Returns features as a dictionary of numpy arrays, and a label.""" + features = { + feature_name: tf.constant(features_np_list[i]) + for i, feature_name in enumerate(feature_names) + } + return tf.data.Dataset.zip((tf.data.Dataset.from_tensors(features), + tf.data.Dataset.from_tensors(label_np),)) + + return input_fn, feature_names, bucketized_columns + + +def make_eval_inputs_from_np_arrays(features_np, label_np): + """Makes eval input as streaming batches.""" + num_features = features_np.shape[1] + features_np_list = np.split(features_np, num_features, axis=1) + # 1-based feature names. + feature_names = ["feature_%02d" % (i + 1) for i in range(num_features)] + + def input_fn(): + features = { + feature_name: tf.constant(features_np_list[i]) + for i, feature_name in enumerate(feature_names) + } + return tf.data.Dataset.zip(( + tf.data.Dataset.from_tensor_slices(features), + tf.data.Dataset.from_tensor_slices(label_np),)).batch(1000) + + return input_fn + + +def _make_csv_serving_input_receiver_fn(column_names, column_defaults): + """Returns serving_input_receiver_fn for csv. + + The input arguments are relevant to `tf.decode_csv()`. + + Args: + column_names: a list of column names in the order within input csv. + column_defaults: a list of default values with the same size of + column_names. Each entity must be either a list of one scalar, or an + empty list to denote the corresponding column is required. + e.g. [[""], [2.5], []] indicates the third column is required while + the first column must be string and the second must be float/double. + + Returns: + a serving_input_receiver_fn that handles csv for serving. + """ + def serving_input_receiver_fn(): + csv = tf.placeholder(dtype=tf.string, shape=[None], name="csv") + features = dict(zip(column_names, tf.decode_csv(csv, column_defaults))) + receiver_tensors = {"inputs": csv} + return tf.estimator.export.ServingInputReceiver(features, receiver_tensors) + + return serving_input_receiver_fn + + +def train_boosted_trees(flags_obj): + """Train boosted_trees estimator on HIGGS data. + + Args: + flags_obj: An object containing parsed flag values. + """ + # Clean up the model directory if present. + if tf.gfile.Exists(flags_obj.model_dir): + tf.gfile.DeleteRecursively(flags_obj.model_dir) + tf.logging.info("## Data loading...") + train_data, eval_data = read_higgs_data( + flags_obj.data_dir, flags_obj.train_start, flags_obj.train_count, + flags_obj.eval_start, flags_obj.eval_count) + tf.logging.info("## Data loaded; train: {}{}, eval: {}{}".format( + train_data.dtype, train_data.shape, eval_data.dtype, eval_data.shape)) + # Data consists of one label column followed by 28 feature columns. + train_input_fn, feature_names, feature_columns = make_inputs_from_np_arrays( + features_np=train_data[:, 1:], label_np=train_data[:, 0:1]) + eval_input_fn = make_eval_inputs_from_np_arrays( + features_np=eval_data[:, 1:], label_np=eval_data[:, 0:1]) + tf.logging.info("## Features prepared. Training starts...") + + # Create benchmark logger to log info about the training and metric values + run_params = { + "train_start": flags_obj.train_start, + "train_count": flags_obj.train_count, + "eval_start": flags_obj.eval_start, + "eval_count": flags_obj.eval_count, + "n_trees": flags_obj.n_trees, + "max_depth": flags_obj.max_depth, + } + benchmark_logger = logger.config_benchmark_logger(flags_obj) + benchmark_logger.log_run_info( + model_name="boosted_trees", + dataset_name="higgs", + run_params=run_params, + test_id=flags_obj.benchmark_test_id) + + # Though BoostedTreesClassifier is under tf.estimator, faster in-memory + # training is yet provided as a contrib library. + from tensorflow.contrib import estimator as contrib_estimator # pylint: disable=g-import-not-at-top + classifier = contrib_estimator.boosted_trees_classifier_train_in_memory( + train_input_fn, + feature_columns, + model_dir=flags_obj.model_dir or None, + n_trees=flags_obj.n_trees, + max_depth=flags_obj.max_depth, + learning_rate=flags_obj.learning_rate) + + # Evaluation. + eval_results = classifier.evaluate(eval_input_fn) + # Benchmark the evaluation results + benchmark_logger.log_evaluation_result(eval_results) + + # Exporting the savedmodel with csv parsing. + if flags_obj.export_dir is not None: + classifier.export_savedmodel( + flags_obj.export_dir, + _make_csv_serving_input_receiver_fn( + column_names=feature_names, + # columns are all floats. + column_defaults=[[0.0]] * len(feature_names)), + strip_default_attrs=True) + + +def main(_): + train_boosted_trees(flags.FLAGS) + + +def define_train_higgs_flags(): + """Add tree related flags as well as training/eval configuration.""" + flags_core.define_base(clean=False, stop_threshold=False, batch_size=False, + num_gpu=False, export_dir=True) + flags_core.define_benchmark() + flags.adopt_module_key_flags(flags_core) + + flags.DEFINE_integer( + name="train_start", default=0, + help=help_wrap("Start index of train examples within the data.")) + flags.DEFINE_integer( + name="train_count", default=1000000, + help=help_wrap("Number of train examples within the data.")) + flags.DEFINE_integer( + name="eval_start", default=10000000, + help=help_wrap("Start index of eval examples within the data.")) + flags.DEFINE_integer( + name="eval_count", default=1000000, + help=help_wrap("Number of eval examples within the data.")) + + flags.DEFINE_integer( + "n_trees", default=100, help=help_wrap("Number of trees to build.")) + flags.DEFINE_integer( + "max_depth", default=6, help=help_wrap("Maximum depths of each tree.")) + flags.DEFINE_float( + "learning_rate", default=0.1, + help=help_wrap("The learning rate.")) + + flags_core.set_defaults(data_dir="/tmp/higgs_data", + model_dir="/tmp/higgs_model") + + +if __name__ == "__main__": + # Training progress and eval results are shown as logging.INFO; so enables it. + tf.logging.set_verbosity(tf.logging.INFO) + define_train_higgs_flags() + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/README.md b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/README.md new file mode 100644 index 0000000..890f42f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/README.md @@ -0,0 +1,152 @@ +# ResNet in TensorFlow + +Deep residual networks, or ResNets for short, provided the breakthrough idea of +identity mappings in order to enable training of very deep convolutional neural +networks. This folder contains an implementation of ResNet for the ImageNet +dataset written in TensorFlow. + +See the following papers for more background: + +[1] [Deep Residual Learning for Image Recognition](https://arxiv.org/pdf/1512.03385.pdf) by Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun, Dec 2015. + +[2] [Identity Mappings in Deep Residual Networks](https://arxiv.org/pdf/1603.05027.pdf) by Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun, Jul 2016. + +In code, v1 refers to the ResNet defined in [1] but where a stride 2 is used on +the 3x3 conv rather than the first 1x1 in the bottleneck. This change results +in higher and more stable accuracy with less epochs than the original v1 and has +shown to scale to higher batch sizes with minimal degradation in accuracy. +There is no originating paper. The first mention we are aware of was in the +torch version of [ResNetv1](https://github.com/facebook/fb.resnet.torch). Most +popular v1 implementations are this implementation which we call ResNetv1.5. + +In testing we found v1.5 requires ~12% more compute to train and has 6% reduced +throughput for inference compared to ResNetv1. CIFAR-10 ResNet does not use the +bottleneck and is thus the same for v1 as v1.5. + +v2 refers to [2]. The principle difference between the two versions is that v1 +applies batch normalization and activation after convolution, while v2 applies +batch normalization, then activation, and finally convolution. A schematic +comparison is presented in Figure 1 (left) of [2]. + +Please proceed according to which dataset you would like to train/evaluate on: + + +## CIFAR-10 + +### Setup + +You need to have the latest version of TensorFlow installed. +First, make sure [the models folder is in your Python path](/official/#running-the-models); otherwise you may encounter `ImportError: No module named official.resnet`. + +Then, download and extract the CIFAR-10 data from Alex's website, specifying the location with the `--data_dir` flag. Run the following: + +```bash +python cifar10_download_and_extract.py --data_dir +``` + +Then, to train the model: + +```bash +python cifar10_main.py --data_dir /cifar-10-batches-bin --model_dir +``` + +Use `--data_dir` to specify the location of the CIFAR-10 data used in the previous step. There are more flag options as described in `cifar10_main.py`. + +To export a `SavedModel` from the trained checkpoint: + +```bash +python cifar10_main.py --data_dir /cifar-10-batches-bin --model_dir --eval_only --export_dir +``` + +Note: The `` must be present. You might want to run `mkdir ` beforehand. + +The `SavedModel` can then be [loaded](https://www.tensorflow.org/guide/saved_model#loading_a_savedmodel_in_python) in order to use the ResNet for prediction. + + +## ImageNet + +### Setup +To begin, you will need to download the ImageNet dataset and convert it to +TFRecord format. The following [script](https://github.com/tensorflow/tpu/blob/master/tools/datasets/imagenet_to_gcs.py) +and [README](https://github.com/tensorflow/tpu/tree/master/tools/datasets#imagenet_to_gcspy) +provide a few options. + +Once your dataset is ready, you can begin training the model as follows: + +```bash +python imagenet_main.py --data_dir=/path/to/imagenet +``` + +The model will begin training and will automatically evaluate itself on the +validation data roughly once per epoch. + +Note that there are a number of other options you can specify, including +`--model_dir` to choose where to store the model and `--resnet_size` to choose +the model size (options include ResNet-18 through ResNet-200). See +[`resnet_run_loop.py`](resnet_run_loop.py) for the full list of options. + + +## Compute Devices +Training is accomplished using the DistributionStrategies API. (https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/distribute/README.md) + +The appropriate distribution strategy is chosen based on the `--num_gpus` flag. +By default this flag is one if TensorFlow is compiled with CUDA, and zero +otherwise. + +num_gpus: ++ 0: Use OneDeviceStrategy and train on CPU. ++ 1: Use OneDeviceStrategy and train on GPU. ++ 2+: Use MirroredStrategy (data parallelism) to distribute a batch between devices. + +### Pre-trained model +You can download pre-trained versions of ResNet-50. Reported accuracies are top-1 single-crop accuracy for the ImageNet validation set. +Models are reported as both checkpoints produced by Estimator during training, and as SavedModels which are more portable. Checkpoints are fragile, +and these are not guaranteed to work with future versions of the code. Both ResNet v1 +and ResNet v2 have been trained in both fp16 and fp32 precision. (Here v1 refers to "v1.5". See the note above.) Furthermore, SavedModels +are generated to accept either tensor or JPG inputs, and with channels_first (NCHW) and channels_last (NHWC) convolutions. NCHW is generally +better for GPUs, while NHWC is generally better for CPUs. See the TensorFlow [performance guide](https://www.tensorflow.org/performance/performance_guide#data_formats) +for more details. + +ResNet-50 v2 (fp32, Accuracy 76.47%): +* [Checkpoint](http://download.tensorflow.org/models/official/20181001_resnet/checkpoints/resnet_imagenet_v2_fp32_20181001.tar.gz) +* SavedModel [(NCHW)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp32_savedmodel_NCHW.tar.gz), +[(NCHW, JPG)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp32_savedmodel_NCHW_jpg.tar.gz), +[(NHWC)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp32_savedmodel_NHWC.tar.gz), +[(NHWC, JPG)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp32_savedmodel_NHWC_jpg.tar.gz) + +ResNet-50 v2 (fp16, Accuracy 76.56%): +* [Checkpoint](http://download.tensorflow.org/models/official/20181001_resnet/checkpoints/resnet_imagenet_v2_fp16_20180928.tar.gz) +* SavedModel [(NCHW)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp16_savedmodel_NCHW.tar.gz), +[(NCHW, JPG)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp16_savedmodel_NCHW_jpg.tar.gz), +[(NHWC)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp16_savedmodel_NHWC.tar.gz), +[(NHWC, JPG)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp16_savedmodel_NHWC_jpg.tar.gz) + +ResNet-50 v1 (fp32, Accuracy 76.53%): +* [Checkpoint](http://download.tensorflow.org/models/official/20181001_resnet/checkpoints/resnet_imagenet_v1_fp32_20181001.tar.gz) +* SavedModel [(NCHW)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v1_fp32_savedmodel_NCHW.tar.gz), +[(NCHW, JPG)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v1_fp32_savedmodel_NCHW_jpg.tar.gz), +[(NHWC)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v1_fp32_savedmodel_NHWC.tar.gz), +[(NHWC, JPG)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v1_fp32_savedmodel_NHWC_jpg.tar.gz) + +ResNet-50 v1 (fp16, Accuracy 76.18%): +* [Checkpoint](http://download.tensorflow.org/models/official/20181001_resnet/checkpoints/resnet_imagenet_v1_fp16_20181001.tar.gz) +* SavedModel [(NCHW)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v1_fp16_savedmodel_NCHW.tar.gz), +[(NCHW, JPG)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v1_fp16_savedmodel_NCHW_jpg.tar.gz), +[(NHWC)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v1_fp16_savedmodel_NHWC.tar.gz), +[(NHWC, JPG)](http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v1_fp16_savedmodel_NHWC_jpg.tar.gz) + +### Transfer Learning +You can use a pretrained model to initialize a training process. In addition you are able to freeze all but the final fully connected layers to fine tune your model. Transfer Learning is useful when training on your own small datasets. For a brief look at transfer learning in the context of convolutional neural networks, we recommend reading these [short notes](http://cs231n.github.io/transfer-learning/). + + +To fine tune a pretrained resnet you must make three changes to your training procedure: + +1) Build the exact same model as previously except we change the number of labels in the final classification layer. + +2) Restore all weights from the pre-trained resnet except for the final classification layer; this will get randomly initialized instead. + +3) Freeze earlier layers of the network + +We can perform these three operations by specifying two flags: ```--pretrained_model_checkpoint_path``` and ```--fine_tune```. The first flag is a string that points to the path of a pre-trained resnet model. If this flag is specified, it will load all but the final classification layer. A key thing to note: if both ```--pretrained_model_checkpoint_path``` and a non empty ```model_dir``` directory are passed, the tensorflow estimator will load only the ```model_dir```. For more on this please see [WarmStartSettings](https://www.tensorflow.org/versions/master/api_docs/python/tf/estimator/WarmStartSettings) and [Estimators](https://www.tensorflow.org/guide/estimators). + +The second flag ```--fine_tune``` is a boolean that indicates whether earlier layers of the network should be frozen. You may set this flag to false if you wish to continue training a pre-trained model from a checkpoint. If you set this flag to true, you can train a new classification layer from scratch. diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/estimator_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/estimator_benchmark.py new file mode 100644 index 0000000..18056fa --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/estimator_benchmark.py @@ -0,0 +1,499 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Executes Estimator benchmarks and accuracy tests.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import time + +from absl import flags +from absl.testing import flagsaver +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.r1.resnet import cifar10_main as cifar_main +from official.r1.resnet import imagenet_main +from official.utils.flags import core as flags_core +from official.utils.logs import hooks + +IMAGENET_DATA_DIR_NAME = 'imagenet' +CIFAR_DATA_DIR_NAME = 'cifar-10-batches-bin' +FLAGS = flags.FLAGS + + +class EstimatorBenchmark(tf.test.Benchmark): + """Base class to hold methods common to test classes in the module. + + Code under test for Estimator models (ResNet50 and 56) report mostly the + same data and require the same FLAG setup. + """ + + local_flags = None + + def __init__(self, output_dir=None, default_flags=None, flag_methods=None): + if not output_dir: + output_dir = '/tmp' + self.output_dir = output_dir + self.default_flags = default_flags or {} + self.flag_methods = flag_methods or {} + + def _get_model_dir(self, folder_name): + """Returns directory to store info, e.g. saved model and event log.""" + return os.path.join(self.output_dir, folder_name) + + def _setup(self): + """Sets up and resets flags before each test.""" + tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO) + if EstimatorBenchmark.local_flags is None: + for flag_method in self.flag_methods: + flag_method() + # Loads flags to get defaults to then override. List cannot be empty. + flags.FLAGS(['foo']) + # Overrides flag values with defaults for the class of tests. + for k, v in self.default_flags.items(): + setattr(FLAGS, k, v) + saved_flag_values = flagsaver.save_flag_values() + EstimatorBenchmark.local_flags = saved_flag_values + else: + flagsaver.restore_flag_values(EstimatorBenchmark.local_flags) + + def _report_benchmark(self, + stats, + wall_time_sec, + top_1_max=None, + top_1_min=None): + """Report benchmark results by writing to local protobuf file. + + Args: + stats: dict returned from estimator models with known entries. + wall_time_sec: the during of the benchmark execution in seconds + top_1_max: highest passing level for top_1 accuracy. + top_1_min: lowest passing level for top_1 accuracy. + """ + + examples_per_sec_hook = None + for hook in stats['train_hooks']: + if isinstance(hook, hooks.ExamplesPerSecondHook): + examples_per_sec_hook = hook + break + + eval_results = stats['eval_results'] + metrics = [] + if 'accuracy' in eval_results: + metrics.append({'name': 'accuracy_top_1', + 'value': float(eval_results['accuracy']), + 'min_value': top_1_min, + 'max_value': top_1_max}) + if 'accuracy_top_5' in eval_results: + metrics.append({'name': 'accuracy_top_5', + 'value': float(eval_results['accuracy_top_5'])}) + + if examples_per_sec_hook: + exp_per_second_list = examples_per_sec_hook.current_examples_per_sec_list + # ExamplesPerSecondHook skips the first 10 steps. + exp_per_sec = sum(exp_per_second_list) / (len(exp_per_second_list)) + metrics.append({'name': 'exp_per_second', + 'value': exp_per_sec}) + flags_str = flags_core.get_nondefault_flags_as_str() + self.report_benchmark( + iters=eval_results.get('global_step', None), + wall_time=wall_time_sec, + metrics=metrics, + extras={'flags': flags_str}) + + +class Resnet50EstimatorAccuracy(EstimatorBenchmark): + """Benchmark accuracy tests for ResNet50 w/ Estimator.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + """Benchmark accuracy tests for ResNet50 w/ Estimator. + + Args: + output_dir: directory where to output e.g. log files + root_data_dir: directory under which to look for dataset + **kwargs: arbitrary named arguments. This is needed to make the + constructor forward compatible in case PerfZero provides more + named arguments before updating the constructor. + """ + flag_methods = [imagenet_main.define_imagenet_flags] + + self.data_dir = os.path.join(root_data_dir, IMAGENET_DATA_DIR_NAME) + super(Resnet50EstimatorAccuracy, self).__init__( + output_dir=output_dir, flag_methods=flag_methods) + + def benchmark_graph_8_gpu(self): + """Test 8 GPUs graph mode.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 128 * 8 + FLAGS.train_epochs = 90 + FLAGS.epochs_between_evals = 10 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_8_gpu') + FLAGS.dtype = 'fp32' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_8_gpu(self): + """Test FP16 8 GPUs graph mode.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 256 * 8 + FLAGS.train_epochs = 90 + FLAGS.epochs_between_evals = 10 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_fp16_8_gpu') + FLAGS.dtype = 'fp16' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_graph_rewrite_8_gpu(self): + """Test FP16 graph rewrite 8 GPUs graph mode.""" + self._setup() + FLAGS.num_gpus = 8 + FLAGS.data_dir = self.data_dir + FLAGS.batch_size = 256 * 8 + FLAGS.train_epochs = 90 + FLAGS.epochs_between_evals = 10 + FLAGS.model_dir = self._get_model_dir( + 'benchmark_graph_fp16_graph_rewrite_8_gpu') + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def _run_and_report_benchmark(self): + start_time_sec = time.time() + stats = imagenet_main.run_imagenet(flags.FLAGS) + wall_time_sec = time.time() - start_time_sec + self._report_benchmark(stats, + wall_time_sec, + top_1_min=0.762, + top_1_max=0.766) + + +class Resnet50EstimatorBenchmarkBase(EstimatorBenchmark): + """Base class for benchmarks for ResNet50 using Estimator.""" + local_flags = None + + def __init__(self, output_dir=None, default_flags=None): + flag_methods = [imagenet_main.define_imagenet_flags] + + super(Resnet50EstimatorBenchmarkBase, self).__init__( + output_dir=output_dir, + default_flags=default_flags, + flag_methods=flag_methods) + + def _run_and_report_benchmark(self): + start_time_sec = time.time() + stats = imagenet_main.run_imagenet(FLAGS) + wall_time_sec = time.time() - start_time_sec + print(stats) + # Remove values to skip triggering accuracy check. + stats['eval_results'].pop('accuracy', None) + stats['eval_results'].pop('accuracy_top_5', None) + + self._report_benchmark(stats, wall_time_sec) + + +class Resnet50EstimatorBenchmark(Resnet50EstimatorBenchmarkBase): + """Benchmarks for ResNet50 using Estimator with 1 worker.""" + + def __init__(self, output_dir=None, default_flags=None): + super(Resnet50EstimatorBenchmark, self).__init__( + output_dir=output_dir, + default_flags=default_flags) + + def benchmark_graph_fp16_1_gpu(self): + """Benchmarks graph fp16 1 gpu.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_fp16_1_gpu') + FLAGS.batch_size = 128 + FLAGS.dtype = 'fp16' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_1_gpu_tweaked(self): + """Benchmarks graph fp16 1 gpu tweaked.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.intra_op_parallelism_threads = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_fp16_1_gpu_tweaked') + FLAGS.batch_size = 256 + FLAGS.dtype = 'fp16' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_graph_rewrite_1_gpu_tweaked(self): + """Benchmarks graph fp16 graph rewrite 1 gpu tweaked.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.intra_op_parallelism_threads = 1 + FLAGS.model_dir = self._get_model_dir( + 'benchmark_graph_fp16_graph_rewrite_1_gpu_tweaked') + FLAGS.batch_size = 256 + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_1_gpu(self): + """Benchmarks graph 1 gpu.""" + self._setup() + + FLAGS.num_gpus = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_1_gpu') + FLAGS.batch_size = 128 + FLAGS.dtype = 'fp32' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_8_gpu(self): + """Benchmarks graph 8 gpus.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_8_gpu') + FLAGS.batch_size = 128*8 + FLAGS.dtype = 'fp32' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_8_gpu(self): + """Benchmarks graph fp16 8 gpus.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_fp16_8_gpu') + FLAGS.batch_size = 256*8 + FLAGS.dtype = 'fp16' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_8_gpu_tweaked(self): + """Benchmarks graph fp16 8 gpus tweaked.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.intra_op_parallelism_threads = 1 + FLAGS.model_dir = self._get_model_dir('benchmark_graph_fp16_8_gpu_tweaked') + FLAGS.batch_size = 256*8 + FLAGS.dtype = 'fp16' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_graph_rewrite_8_gpu_tweaked(self): + """Benchmarks graph fp16 graph rewrite 8 gpus tweaked.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.intra_op_parallelism_threads = 1 + FLAGS.model_dir = self._get_model_dir( + 'benchmark_graph_fp16_graph_rewrite_8_gpu_tweaked') + FLAGS.batch_size = 256*8 + FLAGS.dtype = 'fp16' + FLAGS.fp16_implementation = 'graph_rewrite' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + +class Resnet50EstimatorBenchmarkSynth(Resnet50EstimatorBenchmark): + """Resnet50 synthetic benchmark tests.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + def_flags = {} + def_flags['use_synthetic_data'] = True + def_flags['max_train_steps'] = 110 + def_flags['train_epochs'] = 1 + + super(Resnet50EstimatorBenchmarkSynth, self).__init__( + output_dir=output_dir, default_flags=def_flags) + + +class Resnet50EstimatorBenchmarkReal(Resnet50EstimatorBenchmark): + """Resnet50 real data benchmark tests.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + def_flags = {} + def_flags['data_dir'] = os.path.join(root_data_dir, IMAGENET_DATA_DIR_NAME) + def_flags['max_train_steps'] = 110 + def_flags['train_epochs'] = 1 + + super(Resnet50EstimatorBenchmarkReal, self).__init__( + output_dir=output_dir, default_flags=def_flags) + + +class Resnet50MultiWorkerEstimatorBenchmark(Resnet50EstimatorBenchmarkBase): + """Benchmarks for ResNet50 using Estimator with multiple workers.""" + + def __init__(self, output_dir=None, default_flags=None): + super(Resnet50MultiWorkerEstimatorBenchmark, self).__init__( + output_dir=output_dir, + default_flags=default_flags) + + def benchmark_graph_fp16_8_gpu_ring_tweaked(self): + """Benchmarks graph fp16 8 gpus with ring collective tweaked.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.distribution_strategy = 'multi_worker_mirrored' + FLAGS.all_reduce_alg = 'ring' + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.intra_op_parallelism_threads = 1 + FLAGS.datasets_num_private_threads = 32 + FLAGS.model_dir = self._get_model_dir( + folder_name='benchmark_graph_fp16_8_gpu_ring_tweaked') + FLAGS.batch_size = 256*8 + FLAGS.dtype = 'fp16' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_8_gpu_nccl_tweaked(self): + """Benchmarks graph fp16 8 gpus with nccl collective tweaked.""" + self._setup() + + FLAGS.num_gpus = 8 + FLAGS.distribution_strategy = 'multi_worker_mirrored' + FLAGS.all_reduce_alg = 'nccl' + FLAGS.tf_gpu_thread_mode = 'gpu_private' + FLAGS.intra_op_parallelism_threads = 1 + FLAGS.datasets_num_private_threads = 32 + FLAGS.model_dir = self._get_model_dir( + folder_name='benchmark_graph_fp16_8_gpu_nccl_tweaked') + FLAGS.batch_size = 256*8 + FLAGS.dtype = 'fp16' + FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + +class Resnet50MultiWorkerEstimatorBenchmarkSynth( + Resnet50MultiWorkerEstimatorBenchmark): + """ResNet50, multi-worker, Estimator, synthetic data.""" + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + def_flags = {} + def_flags['use_synthetic_data'] = True + def_flags['max_train_steps'] = 110 + def_flags['train_epochs'] = 1 + + super(Resnet50MultiWorkerEstimatorBenchmarkSynth, self).__init__( + output_dir=output_dir, default_flags=def_flags) + + +class Resnet56EstimatorAccuracy(EstimatorBenchmark): + """Accuracy tests for Estimator ResNet56.""" + + local_flags = None + + def __init__(self, output_dir=None, root_data_dir=None, **kwargs): + """A benchmark class. + + Args: + output_dir: directory where to output e.g. log files + root_data_dir: directory under which to look for dataset + **kwargs: arbitrary named arguments. This is needed to make the + constructor forward compatible in case PerfZero provides more + named arguments before updating the constructor. + """ + flag_methods = [cifar_main.define_cifar_flags] + + self.data_dir = os.path.join(root_data_dir, CIFAR_DATA_DIR_NAME) + super(Resnet56EstimatorAccuracy, self).__init__( + output_dir=output_dir, flag_methods=flag_methods) + + def benchmark_graph_1_gpu(self): + """Test layers model with Estimator and distribution strategies.""" + self._setup() + flags.FLAGS.num_gpus = 1 + flags.FLAGS.data_dir = self.data_dir + flags.FLAGS.batch_size = 128 + flags.FLAGS.train_epochs = 182 + flags.FLAGS.model_dir = self._get_model_dir('benchmark_graph_1_gpu') + flags.FLAGS.resnet_size = 56 + flags.FLAGS.dtype = 'fp32' + flags.FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_1_gpu(self): + """Test layers FP16 model with Estimator and distribution strategies.""" + self._setup() + flags.FLAGS.num_gpus = 1 + flags.FLAGS.data_dir = self.data_dir + flags.FLAGS.batch_size = 128 + flags.FLAGS.train_epochs = 182 + flags.FLAGS.model_dir = self._get_model_dir('benchmark_graph_fp16_1_gpu') + flags.FLAGS.resnet_size = 56 + flags.FLAGS.dtype = 'fp16' + flags.FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_2_gpu(self): + """Test layers model with Estimator and dist_strat. 2 GPUs.""" + self._setup() + flags.FLAGS.num_gpus = 2 + flags.FLAGS.data_dir = self.data_dir + flags.FLAGS.batch_size = 128 + flags.FLAGS.train_epochs = 182 + flags.FLAGS.model_dir = self._get_model_dir('benchmark_graph_2_gpu') + flags.FLAGS.resnet_size = 56 + flags.FLAGS.dtype = 'fp32' + flags.FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def benchmark_graph_fp16_2_gpu(self): + """Test layers FP16 model with Estimator and dist_strat. 2 GPUs.""" + self._setup() + flags.FLAGS.num_gpus = 2 + flags.FLAGS.data_dir = self.data_dir + flags.FLAGS.batch_size = 128 + flags.FLAGS.train_epochs = 182 + flags.FLAGS.model_dir = self._get_model_dir('benchmark_graph_fp16_2_gpu') + flags.FLAGS.resnet_size = 56 + flags.FLAGS.dtype = 'fp16' + flags.FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def unit_test(self): + """A lightweight test that can finish quickly.""" + self._setup() + flags.FLAGS.num_gpus = 1 + flags.FLAGS.data_dir = self.data_dir + flags.FLAGS.batch_size = 128 + flags.FLAGS.train_epochs = 1 + flags.FLAGS.model_dir = self._get_model_dir('unit_test') + flags.FLAGS.resnet_size = 8 + flags.FLAGS.dtype = 'fp32' + flags.FLAGS.hooks = ['ExamplesPerSecondHook'] + self._run_and_report_benchmark() + + def _run_and_report_benchmark(self): + """Executes benchmark and reports result.""" + start_time_sec = time.time() + stats = cifar_main.run_cifar(flags.FLAGS) + wall_time_sec = time.time() - start_time_sec + + self._report_benchmark(stats, + wall_time_sec, + top_1_min=0.926, + top_1_max=0.938) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/imagenet_main.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/imagenet_main.py new file mode 100644 index 0000000..1c4fef1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/imagenet_main.py @@ -0,0 +1,433 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Runs a ResNet model on the ImageNet dataset.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys +import datetime + +import time + +sys.path.append(os.path.dirname(os.path.realpath(__file__)) + '/../../../') + +# import pydevd_pycharm +# pydevd_pycharm.settrace('10.174.181.209', port=8008, stdoutToServer=True, stderrToServer=True) + + +from absl import app as absl_app +from absl import flags +from six.moves import range +import tensorflow as tf + +from official.r1.resnet import imagenet_preprocessing +from official.r1.resnet import resnet_model +from official.r1.resnet import resnet_run_loop +from official.utils.flags import core as flags_core +from official.utils.logs import logger +import logging +############## npu modify begin ############# +from npu_bridge.estimator import npu_ops +from hccl.manage.api import get_local_rank_id +from hccl.manage.api import get_rank_size +from hccl.manage.api import get_rank_id +from tensorflow.core.protobuf import rewriter_config_pb2 +tf.compat.v1.logging.set_verbosity(tf.logging.INFO) +############## npu modify end ############### + +DEFAULT_IMAGE_SIZE = 224 +NUM_CHANNELS = 3 +NUM_CLASSES = 1001 + +NUM_IMAGES = { + 'train': 1281167, + 'validation': 50000, +} + +_NUM_TRAIN_FILES = 1024 +_SHUFFLE_BUFFER = 10000 + +DATASET_NAME = 'ImageNet' +#log_file1 = 'result/logger_resnet50.log' +#log_file1 = os.path.join(os.path.abspath(os.path.dirname(__file__)),'../../../../result/logger_resnet50.log') + + +############################################################################### +# Data processing +############################################################################### +def get_filenames(is_training, data_dir): + """Return filenames for dataset.""" + if is_training: + return [ + os.path.join(data_dir, 'train-%05d-of-01024' % i) + for i in range(_NUM_TRAIN_FILES)] + else: + return [ + os.path.join(data_dir, 'validation-%05d-of-00128' % i) + for i in range(128)] + + +def _parse_example_proto(example_serialized): + """Parses an Example proto containing a training example of an image. + + The output of the build_image_data.py image preprocessing script is a dataset + containing serialized Example protocol buffers. Each Example proto contains + the following fields (values are included as examples): + + image/height: 462 + image/width: 581 + image/colorspace: 'RGB' + image/channels: 3 + image/class/label: 615 + image/class/synset: 'n03623198' + image/class/text: 'knee pad' + image/object/bbox/xmin: 0.1 + image/object/bbox/xmax: 0.9 + image/object/bbox/ymin: 0.2 + image/object/bbox/ymax: 0.6 + image/object/bbox/label: 615 + image/format: 'JPEG' + image/filename: 'ILSVRC2012_val_00041207.JPEG' + image/encoded: + + Args: + example_serialized: scalar Tensor tf.string containing a serialized + Example protocol buffer. + + Returns: + image_buffer: Tensor tf.string containing the contents of a JPEG file. + label: Tensor tf.int32 containing the label. + bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords] + where each coordinate is [0, 1) and the coordinates are arranged as + [ymin, xmin, ymax, xmax]. + """ + # Dense features in Example proto. + feature_map = { + 'image/encoded': tf.io.FixedLenFeature([], dtype=tf.string, + default_value=''), + 'image/class/label': tf.io.FixedLenFeature([], dtype=tf.int64, + default_value=-1), + 'image/class/text': tf.io.FixedLenFeature([], dtype=tf.string, + default_value=''), + } + sparse_float32 = tf.io.VarLenFeature(dtype=tf.float32) + # Sparse features in Example proto. + feature_map.update( + {k: sparse_float32 for k in ['image/object/bbox/xmin', + 'image/object/bbox/ymin', + 'image/object/bbox/xmax', + 'image/object/bbox/ymax']}) + + features = tf.io.parse_single_example(serialized=example_serialized, + features=feature_map) + label = tf.cast(features['image/class/label'], dtype=tf.int32) + + xmin = tf.expand_dims(features['image/object/bbox/xmin'].values, 0) + ymin = tf.expand_dims(features['image/object/bbox/ymin'].values, 0) + xmax = tf.expand_dims(features['image/object/bbox/xmax'].values, 0) + ymax = tf.expand_dims(features['image/object/bbox/ymax'].values, 0) + + # Note that we impose an ordering of (y, x) just to make life difficult. + bbox = tf.concat([ymin, xmin, ymax, xmax], 0) + + # Force the variable number of bounding boxes into the shape + # [1, num_boxes, coords]. + bbox = tf.expand_dims(bbox, 0) + bbox = tf.transpose(a=bbox, perm=[0, 2, 1]) + + return features['image/encoded'], label, bbox + + +def parse_record(raw_record, is_training, dtype): + """Parses a record containing a training example of an image. + + The input record is parsed into a label and image, and the image is passed + through preprocessing steps (cropping, flipping, and so on). + + Args: + raw_record: scalar Tensor tf.string containing a serialized + Example protocol buffer. + is_training: A boolean denoting whether the input is for training. + dtype: data type to use for images/features. + + Returns: + Tuple with processed image tensor and one-hot-encoded label tensor. + """ + image_buffer, label, bbox = _parse_example_proto(raw_record) + #work_num, root_dir, datatime, resnet_logger = hwlog.env(log_file1) + # add 预处理 + #resnet_logger.info("namespace:%s,time_ts:%s, event_type:pre_process_event" % (work_num, date_time)) + #resnet_logger.info("namespace:%s,time_ts:%s,event_type:init_start" % (work_num, date_time)) + image = imagenet_preprocessing.preprocess_image( + image_buffer=image_buffer, + bbox=bbox, + output_height=DEFAULT_IMAGE_SIZE, + output_width=DEFAULT_IMAGE_SIZE, + num_channels=NUM_CHANNELS, + is_training=is_training) + # resnet_logger.info("namespace:%s,time_ts:%d,event_type:init_end, root_dir:%s" % (work_num, datatime, root_dir)) + image = tf.cast(image, dtype) + + return image, label + + +def input_fn(is_training, + data_dir, + batch_size, + num_epochs=1, + dtype=tf.float32, + datasets_num_private_threads=None, + parse_record_fn=parse_record, + input_context=None, + drop_remainder=False, + tf_data_experimental_slack=False): + """Input function which provides batches for train or eval. + + Args: + is_training: A boolean denoting whether the input is for training. + data_dir: The directory containing the input data. + batch_size: The number of samples per batch. + num_epochs: The number of epochs to repeat the dataset. + dtype: Data type to use for images/features + datasets_num_private_threads: Number of private threads for tf.data. + parse_record_fn: Function to use for parsing the records. + input_context: A `tf.distribute.InputContext` object passed in by + `tf.distribute.Strategy`. + drop_remainder: A boolean indicates whether to drop the remainder of the + batches. If True, the batch dimension will be static. + tf_data_experimental_slack: Whether to enable tf.data's + `experimental_slack` option. + + Returns: + A dataset that can be used for iteration. + """ + filenames = get_filenames(is_training, data_dir) + dataset = tf.data.Dataset.from_tensor_slices(filenames) + + if input_context: + ############## npu modify begin ############# + dataset = dataset.shard(get_rank_size(), + get_rank_id()) + ############## npu modify end ############### + + if is_training: + # Shuffle the input files + dataset = dataset.shuffle(buffer_size=_NUM_TRAIN_FILES) + + # Convert to individual records. + # cycle_length = 10 means that up to 10 files will be read and deserialized in + # parallel. You may want to increase this number if you have a large number of + # CPU cores. + dataset = dataset.interleave( + tf.data.TFRecordDataset, + cycle_length=10, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + + return resnet_run_loop.process_record_dataset( + dataset=dataset, + is_training=is_training, + batch_size=batch_size, + shuffle_buffer=_SHUFFLE_BUFFER, + parse_record_fn=parse_record_fn, + num_epochs=num_epochs, + dtype=dtype, + datasets_num_private_threads=datasets_num_private_threads, + drop_remainder=drop_remainder, + tf_data_experimental_slack=tf_data_experimental_slack, + ) + + +def get_synth_input_fn(dtype): + return resnet_run_loop.get_synth_input_fn( + DEFAULT_IMAGE_SIZE, DEFAULT_IMAGE_SIZE, NUM_CHANNELS, NUM_CLASSES, + dtype=dtype) + + +############################################################################### +# Running the model +############################################################################### +class ImagenetModel(resnet_model.Model): + """Model class with appropriate defaults for Imagenet data.""" + + def __init__(self, resnet_size, data_format=None, num_classes=NUM_CLASSES, + resnet_version=resnet_model.DEFAULT_VERSION, + dtype=resnet_model.DEFAULT_DTYPE): + """These are the parameters that work for Imagenet data. + + Args: + resnet_size: The number of convolutional layers needed in the model. + data_format: Either 'channels_first' or 'channels_last', specifying which + data format to use when setting up the model. + num_classes: The number of output classes needed from the model. This + enables users to extend the same model to their own datasets. + resnet_version: Integer representing which version of the ResNet network + to use. See README for details. Valid values: [1, 2] + dtype: The TensorFlow dtype to use for calculations. + """ + + # For bigger models, we want to use "bottleneck" layers + if resnet_size < 50: + bottleneck = False + else: + bottleneck = True + + super(ImagenetModel, self).__init__( + resnet_size=resnet_size, + bottleneck=bottleneck, + num_classes=num_classes, + num_filters=64, + kernel_size=7, + conv_stride=2, + first_pool_size=3, + first_pool_stride=2, + block_sizes=_get_block_sizes(resnet_size), + block_strides=[1, 2, 2, 2], + resnet_version=resnet_version, + data_format=data_format, + dtype=dtype + ) + + +def _get_block_sizes(resnet_size): + """Retrieve the size of each block_layer in the ResNet model. + + The number of block layers used for the Resnet model varies according + to the size of the model. This helper grabs the layer set we want, throwing + an error if a non-standard size has been selected. + + Args: + resnet_size: The number of convolutional layers needed in the model. + + Returns: + A list of block sizes to use in building the model. + + Raises: + KeyError: if invalid resnet_size is received. + """ + choices = { + 18: [2, 2, 2, 2], + 34: [3, 4, 6, 3], + 50: [3, 4, 6, 3], + 101: [3, 4, 23, 3], + 152: [3, 8, 36, 3], + 200: [3, 24, 36, 3] + } + + try: + return choices[resnet_size] + except KeyError: + err = ('Could not find layers for selected Resnet size.\n' + 'Size received: {}; sizes allowed: {}.'.format( + resnet_size, list(choices.keys()))) + raise ValueError(err) + + +def imagenet_model_fn(features, labels, mode, params): + """Our model_fn for ResNet to be used with our Estimator.""" + + # Warmup and higher lr may not be valid for fine tuning with small batches + # and smaller numbers of training images. + if params['fine_tune']: + warmup = False + base_lr = .1 + else: + warmup = True + base_lr = .128 + + learning_rate_fn = resnet_run_loop.learning_rate_with_decay( + batch_size=params['num_gpus']*params['batch_size'], + batch_denom=256, num_images=NUM_IMAGES['train'], + boundary_epochs=[30, 60, 80, 90], decay_rates=[1, 0.1, 0.01, 0.001, 1e-4], + warmup=warmup, base_lr=base_lr) + + return resnet_run_loop.resnet_model_fn( + features=features, + labels=labels, + mode=mode, + model_class=ImagenetModel, + resnet_size=params['resnet_size'], + weight_decay=flags.FLAGS.weight_decay, + learning_rate_fn=learning_rate_fn, + momentum=0.9, + data_format=params['data_format'], + resnet_version=params['resnet_version'], + loss_scale=params['loss_scale'], + loss_filter_fn=None, + dtype=params['dtype'], + fine_tune=params['fine_tune'], + label_smoothing=flags.FLAGS.label_smoothing + ) + + +def define_imagenet_flags(): + resnet_run_loop.define_resnet_flags( + resnet_size_choices=['18', '34', '50', '101', '152', '200'], + dynamic_loss_scale=True, + fp16_implementation=True) + flags.adopt_module_key_flags(resnet_run_loop) + flags_core.set_defaults(train_epochs=90) + + #Loss scale is defautt used because Davinci core supports mixed precision naturally + flags_core.set_defaults(loss_scale='512') + +def run_imagenet(flags_obj): + """Run ResNet ImageNet training and eval loop. + + Args: + flags_obj: An object containing parsed flag values. + + Returns: + Dict of results of the run. Contains the keys `eval_results` and + `train_hooks`. `eval_results` contains accuracy (top_1) and + accuracy_top_5. `train_hooks` is a list the instances of hooks used during + training. + """ + input_function = (flags_obj.use_synthetic_data and + get_synth_input_fn(flags_core.get_tf_dtype(flags_obj)) or + input_fn) + + result = resnet_run_loop.resnet_main( + flags_obj, imagenet_model_fn, input_function, DATASET_NAME,NUM_IMAGES, + shape=[DEFAULT_IMAGE_SIZE, DEFAULT_IMAGE_SIZE, NUM_CHANNELS],) + + return result +def main(flags_obj): + ############## npu modify begin ############# + # Init NPU ,then can call HCCL Interface + init_sess,npu_init=resnet_run_loop.init_npu() + + init_sess.run(npu_init) + # i=1 + # while(1): + # i+=1 + ############## npu modify end ############### + + with logger.benchmark_context(flags.FLAGS): + run_imagenet(flags.FLAGS) + +def benchmark_main(): + absl_app.run(main) + +def benchmark_pre(): + tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO) + define_imagenet_flags() + +if __name__ == '__main__': + tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO) + define_imagenet_flags() + absl_app.run(main) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/imagenet_preprocessing.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/imagenet_preprocessing.py new file mode 100644 index 0000000..891b58a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/imagenet_preprocessing.py @@ -0,0 +1,262 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Provides utilities to preprocess images. + +Training images are sampled using the provided bounding boxes, and subsequently +cropped to the sampled bounding box. Images are additionally flipped randomly, +then resized to the target output size (without aspect-ratio preservation). + +Images used during evaluation are resized (with aspect-ratio preservation) and +centrally cropped. + +All images undergo mean color subtraction. + +Note that these steps are colloquially referred to as "ResNet preprocessing," +and they differ from "VGG preprocessing," which does not use bounding boxes +and instead does an aspect-preserving resize followed by random crop during +training. (These both differ from "Inception preprocessing," which introduces +color distortion steps.) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +_R_MEAN = 123.68 +_G_MEAN = 116.78 +_B_MEAN = 103.94 +_CHANNEL_MEANS = [_R_MEAN, _G_MEAN, _B_MEAN] + +# The lower bound for the smallest side of the image for aspect-preserving +# resizing. For example, if an image is 500 x 1000, it will be resized to +# _RESIZE_MIN x (_RESIZE_MIN * 2). +_RESIZE_MIN = 256 + + +def _decode_crop_and_flip(image_buffer, bbox, num_channels): + """Crops the given image to a random part of the image, and randomly flips. + + We use the fused decode_and_crop op, which performs better than the two ops + used separately in series, but note that this requires that the image be + passed in as an un-decoded string Tensor. + + Args: + image_buffer: scalar string Tensor representing the raw JPEG image buffer. + bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords] + where each coordinate is [0, 1) and the coordinates are arranged as + [ymin, xmin, ymax, xmax]. + num_channels: Integer depth of the image buffer for decoding. + + Returns: + 3-D tensor with cropped image. + + """ + # A large fraction of image datasets contain a human-annotated bounding box + # delineating the region of the image containing the object of interest. We + # choose to create a new bounding box for the object which is a randomly + # distorted version of the human-annotated bounding box that obeys an + # allowed range of aspect ratios, sizes and overlap with the human-annotated + # bounding box. If no box is supplied, then we assume the bounding box is + # the entire image. + sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box( + tf.image.extract_jpeg_shape(image_buffer), + bounding_boxes=bbox, + min_object_covered=0.1, + aspect_ratio_range=[0.75, 1.33], + area_range=[0.05, 1.0], + max_attempts=100, + use_image_if_no_bounding_boxes=True) + bbox_begin, bbox_size, _ = sample_distorted_bounding_box + + # Reassemble the bounding box in the format the crop op requires. + offset_y, offset_x, _ = tf.unstack(bbox_begin) + target_height, target_width, _ = tf.unstack(bbox_size) + crop_window = tf.stack([offset_y, offset_x, target_height, target_width]) + + # Use the fused decode and crop op here, which is faster than each in series. + cropped = tf.image.decode_and_crop_jpeg( + image_buffer, crop_window, channels=num_channels) + + # Flip to add a little more random distortion in. + cropped = tf.image.random_flip_left_right(cropped) + return cropped + + +def _central_crop(image, crop_height, crop_width): + """Performs central crops of the given image list. + + Args: + image: a 3-D image tensor + crop_height: the height of the image following the crop. + crop_width: the width of the image following the crop. + + Returns: + 3-D tensor with cropped image. + """ + shape = tf.shape(input=image) + height, width = shape[0], shape[1] + + amount_to_be_cropped_h = (height - crop_height) + crop_top = amount_to_be_cropped_h // 2 + amount_to_be_cropped_w = (width - crop_width) + crop_left = amount_to_be_cropped_w // 2 + return tf.slice( + image, [crop_top, crop_left, 0], [crop_height, crop_width, -1]) + + +def _mean_image_subtraction(image, means, num_channels): + """Subtracts the given means from each image channel. + + For example: + means = [123.68, 116.779, 103.939] + image = _mean_image_subtraction(image, means) + + Note that the rank of `image` must be known. + + Args: + image: a tensor of size [height, width, C]. + means: a C-vector of values to subtract from each channel. + num_channels: number of color channels in the image that will be distorted. + + Returns: + the centered image. + + Raises: + ValueError: If the rank of `image` is unknown, if `image` has a rank other + than three or if the number of channels in `image` doesn't match the + number of values in `means`. + """ + if image.get_shape().ndims != 3: + raise ValueError('Input must be of size [height, width, C>0]') + + if len(means) != num_channels: + raise ValueError('len(means) must match the number of channels') + + # We have a 1-D tensor of means; convert to 3-D. + # Note(b/130245863): we explicitly call `broadcast` instead of simply + # expanding dimensions for better performance. + means = tf.broadcast_to(means, tf.shape(image)) + + return image - means + + +def _smallest_size_at_least(height, width, resize_min): + """Computes new shape with the smallest side equal to `smallest_side`. + + Computes new shape with the smallest side equal to `smallest_side` while + preserving the original aspect ratio. + + Args: + height: an int32 scalar tensor indicating the current height. + width: an int32 scalar tensor indicating the current width. + resize_min: A python integer or scalar `Tensor` indicating the size of + the smallest side after resize. + + Returns: + new_height: an int32 scalar tensor indicating the new height. + new_width: an int32 scalar tensor indicating the new width. + """ + resize_min = tf.cast(resize_min, tf.float32) + + # Convert to floats to make subsequent calculations go smoothly. + height, width = tf.cast(height, tf.float32), tf.cast(width, tf.float32) + + smaller_dim = tf.minimum(height, width) + scale_ratio = resize_min / smaller_dim + + # Convert back to ints to make heights and widths that TF ops will accept. + new_height = tf.cast(height * scale_ratio, tf.int32) + new_width = tf.cast(width * scale_ratio, tf.int32) + + return new_height, new_width + + +def _aspect_preserving_resize(image, resize_min): + """Resize images preserving the original aspect ratio. + + Args: + image: A 3-D image `Tensor`. + resize_min: A python integer or scalar `Tensor` indicating the size of + the smallest side after resize. + + Returns: + resized_image: A 3-D tensor containing the resized image. + """ + shape = tf.shape(input=image) + height, width = shape[0], shape[1] + + new_height, new_width = _smallest_size_at_least(height, width, resize_min) + + return _resize_image(image, new_height, new_width) + + +def _resize_image(image, height, width): + """Simple wrapper around tf.resize_images. + + This is primarily to make sure we use the same `ResizeMethod` and other + details each time. + + Args: + image: A 3-D image `Tensor`. + height: The target height for the resized image. + width: The target width for the resized image. + + Returns: + resized_image: A 3-D tensor containing the resized image. The first two + dimensions have the shape [height, width]. + """ + return tf.compat.v1.image.resize( + image, [height, width], method=tf.image.ResizeMethod.BILINEAR, + align_corners=False) + + +def preprocess_image(image_buffer, bbox, output_height, output_width, + num_channels, is_training=False): + """Preprocesses the given image. + + Preprocessing includes decoding, cropping, and resizing for both training + and eval images. Training preprocessing, however, introduces some random + distortion of the image to improve accuracy. + + Args: + image_buffer: scalar string Tensor representing the raw JPEG image buffer. + bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords] + where each coordinate is [0, 1) and the coordinates are arranged as + [ymin, xmin, ymax, xmax]. + output_height: The height of the image after preprocessing. + output_width: The width of the image after preprocessing. + num_channels: Integer depth of the image buffer for decoding. + is_training: `True` if we're preprocessing the image for training and + `False` otherwise. + + Returns: + A preprocessed image. + """ + if is_training: + # For training, we want to randomize some of the distortions. + image = _decode_crop_and_flip(image_buffer, bbox, num_channels) + image = _resize_image(image, output_height, output_width) + else: + # For validation, we want to decode, resize, then just crop the middle. + image = tf.image.decode_jpeg(image_buffer, channels=num_channels) + image = _aspect_preserving_resize(image, _RESIZE_MIN) + image = _central_crop(image, output_height, output_width) + + image.set_shape([output_height, output_width, num_channels]) + + return _mean_image_subtraction(image, _CHANNEL_MEANS, num_channels) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/imagenet_test.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/imagenet_test.py new file mode 100644 index 0000000..5a7ddcd --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/imagenet_test.py @@ -0,0 +1,326 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest + +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.r1.resnet import imagenet_main +from official.utils.misc import keras_utils +from official.utils.testing import integration + +tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR) + +_BATCH_SIZE = 32 +_LABEL_CLASSES = 1001 + + +class BaseTest(tf.test.TestCase): + + _num_validation_images = None + + @classmethod + def setUpClass(cls): # pylint: disable=invalid-name + super(BaseTest, cls).setUpClass() + imagenet_main.define_imagenet_flags() + + def setUp(self): + super(BaseTest, self).setUp() + if keras_utils.is_v2_0: + tf.compat.v1.disable_eager_execution() + self._num_validation_images = imagenet_main.NUM_IMAGES['validation'] + imagenet_main.NUM_IMAGES['validation'] = 4 + + def tearDown(self): + super(BaseTest, self).tearDown() + tf.io.gfile.rmtree(self.get_temp_dir()) + imagenet_main.NUM_IMAGES['validation'] = self._num_validation_images + + def _tensor_shapes_helper(self, resnet_size, resnet_version, dtype, with_gpu): + """Checks the tensor shapes after each phase of the ResNet model.""" + def reshape(shape): + """Returns the expected dimensions depending on if a GPU is being used.""" + + # If a GPU is used for the test, the shape is returned (already in NCHW + # form). When GPU is not used, the shape is converted to NHWC. + if with_gpu: + return shape + return shape[0], shape[2], shape[3], shape[1] + + graph = tf.Graph() + + with graph.as_default(), self.test_session( + graph=graph, use_gpu=with_gpu, force_gpu=with_gpu): + model = imagenet_main.ImagenetModel( + resnet_size=resnet_size, + data_format='channels_first' if with_gpu else 'channels_last', + resnet_version=resnet_version, + dtype=dtype + ) + inputs = tf.random.uniform([1, 224, 224, 3]) + output = model(inputs, training=True) + + initial_conv = graph.get_tensor_by_name('resnet_model/initial_conv:0') + max_pool = graph.get_tensor_by_name('resnet_model/initial_max_pool:0') + block_layer1 = graph.get_tensor_by_name('resnet_model/block_layer1:0') + block_layer2 = graph.get_tensor_by_name('resnet_model/block_layer2:0') + block_layer3 = graph.get_tensor_by_name('resnet_model/block_layer3:0') + block_layer4 = graph.get_tensor_by_name('resnet_model/block_layer4:0') + reduce_mean = graph.get_tensor_by_name('resnet_model/final_reduce_mean:0') + dense = graph.get_tensor_by_name('resnet_model/final_dense:0') + + self.assertAllEqual(initial_conv.shape, reshape((1, 64, 112, 112))) + self.assertAllEqual(max_pool.shape, reshape((1, 64, 56, 56))) + + # The number of channels after each block depends on whether we're + # using the building_block or the bottleneck_block. + if resnet_size < 50: + self.assertAllEqual(block_layer1.shape, reshape((1, 64, 56, 56))) + self.assertAllEqual(block_layer2.shape, reshape((1, 128, 28, 28))) + self.assertAllEqual(block_layer3.shape, reshape((1, 256, 14, 14))) + self.assertAllEqual(block_layer4.shape, reshape((1, 512, 7, 7))) + self.assertAllEqual(reduce_mean.shape, reshape((1, 512, 1, 1))) + else: + self.assertAllEqual(block_layer1.shape, reshape((1, 256, 56, 56))) + self.assertAllEqual(block_layer2.shape, reshape((1, 512, 28, 28))) + self.assertAllEqual(block_layer3.shape, reshape((1, 1024, 14, 14))) + self.assertAllEqual(block_layer4.shape, reshape((1, 2048, 7, 7))) + self.assertAllEqual(reduce_mean.shape, reshape((1, 2048, 1, 1))) + + self.assertAllEqual(dense.shape, (1, _LABEL_CLASSES)) + self.assertAllEqual(output.shape, (1, _LABEL_CLASSES)) + + def tensor_shapes_helper(self, resnet_size, resnet_version, with_gpu=False): + self._tensor_shapes_helper(resnet_size=resnet_size, + resnet_version=resnet_version, + dtype=tf.float32, with_gpu=with_gpu) + self._tensor_shapes_helper(resnet_size=resnet_size, + resnet_version=resnet_version, + dtype=tf.float16, with_gpu=with_gpu) + + def test_tensor_shapes_resnet_18_v1(self): + self.tensor_shapes_helper(18, resnet_version=1) + + def test_tensor_shapes_resnet_18_v2(self): + self.tensor_shapes_helper(18, resnet_version=2) + + def test_tensor_shapes_resnet_34_v1(self): + self.tensor_shapes_helper(34, resnet_version=1) + + def test_tensor_shapes_resnet_34_v2(self): + self.tensor_shapes_helper(34, resnet_version=2) + + def test_tensor_shapes_resnet_50_v1(self): + self.tensor_shapes_helper(50, resnet_version=1) + + def test_tensor_shapes_resnet_50_v2(self): + self.tensor_shapes_helper(50, resnet_version=2) + + def test_tensor_shapes_resnet_101_v1(self): + self.tensor_shapes_helper(101, resnet_version=1) + + def test_tensor_shapes_resnet_101_v2(self): + self.tensor_shapes_helper(101, resnet_version=2) + + def test_tensor_shapes_resnet_152_v1(self): + self.tensor_shapes_helper(152, resnet_version=1) + + def test_tensor_shapes_resnet_152_v2(self): + self.tensor_shapes_helper(152, resnet_version=2) + + def test_tensor_shapes_resnet_200_v1(self): + self.tensor_shapes_helper(200, resnet_version=1) + + def test_tensor_shapes_resnet_200_v2(self): + self.tensor_shapes_helper(200, resnet_version=2) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_18_with_gpu_v1(self): + self.tensor_shapes_helper(18, resnet_version=1, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_18_with_gpu_v2(self): + self.tensor_shapes_helper(18, resnet_version=2, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_34_with_gpu_v1(self): + self.tensor_shapes_helper(34, resnet_version=1, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_34_with_gpu_v2(self): + self.tensor_shapes_helper(34, resnet_version=2, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_50_with_gpu_v1(self): + self.tensor_shapes_helper(50, resnet_version=1, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_50_with_gpu_v2(self): + self.tensor_shapes_helper(50, resnet_version=2, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_101_with_gpu_v1(self): + self.tensor_shapes_helper(101, resnet_version=1, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_101_with_gpu_v2(self): + self.tensor_shapes_helper(101, resnet_version=2, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_152_with_gpu_v1(self): + self.tensor_shapes_helper(152, resnet_version=1, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_152_with_gpu_v2(self): + self.tensor_shapes_helper(152, resnet_version=2, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_200_with_gpu_v1(self): + self.tensor_shapes_helper(200, resnet_version=1, with_gpu=True) + + @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU') + def test_tensor_shapes_resnet_200_with_gpu_v2(self): + self.tensor_shapes_helper(200, resnet_version=2, with_gpu=True) + + def resnet_model_fn_helper(self, mode, resnet_version, dtype): + """Tests that the EstimatorSpec is given the appropriate arguments.""" + tf.compat.v1.train.create_global_step() + + input_fn = imagenet_main.get_synth_input_fn(dtype) + dataset = input_fn(True, '', _BATCH_SIZE) + iterator = tf.compat.v1.data.make_initializable_iterator(dataset) + features, labels = iterator.get_next() + spec = imagenet_main.imagenet_model_fn( + features, labels, mode, { + 'dtype': dtype, + 'resnet_size': 50, + 'data_format': 'channels_last', + 'batch_size': _BATCH_SIZE, + 'resnet_version': resnet_version, + 'loss_scale': 128 if dtype == tf.float16 else 1, + 'fine_tune': False, + }) + + predictions = spec.predictions + self.assertAllEqual(predictions['probabilities'].shape, + (_BATCH_SIZE, _LABEL_CLASSES)) + self.assertEqual(predictions['probabilities'].dtype, tf.float32) + self.assertAllEqual(predictions['classes'].shape, (_BATCH_SIZE,)) + self.assertEqual(predictions['classes'].dtype, tf.int64) + + if mode != tf.estimator.ModeKeys.PREDICT: + loss = spec.loss + self.assertAllEqual(loss.shape, ()) + self.assertEqual(loss.dtype, tf.float32) + + if mode == tf.estimator.ModeKeys.EVAL: + eval_metric_ops = spec.eval_metric_ops + self.assertAllEqual(eval_metric_ops['accuracy'][0].shape, ()) + self.assertAllEqual(eval_metric_ops['accuracy'][1].shape, ()) + self.assertEqual(eval_metric_ops['accuracy'][0].dtype, tf.float32) + self.assertEqual(eval_metric_ops['accuracy'][1].dtype, tf.float32) + + def test_resnet_model_fn_train_mode_v1(self): + self.resnet_model_fn_helper(tf.estimator.ModeKeys.TRAIN, resnet_version=1, + dtype=tf.float32) + + def test_resnet_model_fn_train_mode_v2(self): + self.resnet_model_fn_helper(tf.estimator.ModeKeys.TRAIN, resnet_version=2, + dtype=tf.float32) + + def test_resnet_model_fn_eval_mode_v1(self): + self.resnet_model_fn_helper(tf.estimator.ModeKeys.EVAL, resnet_version=1, + dtype=tf.float32) + + def test_resnet_model_fn_eval_mode_v2(self): + self.resnet_model_fn_helper(tf.estimator.ModeKeys.EVAL, resnet_version=2, + dtype=tf.float32) + + def test_resnet_model_fn_predict_mode_v1(self): + self.resnet_model_fn_helper(tf.estimator.ModeKeys.PREDICT, resnet_version=1, + dtype=tf.float32) + + def test_resnet_model_fn_predict_mode_v2(self): + self.resnet_model_fn_helper(tf.estimator.ModeKeys.PREDICT, resnet_version=2, + dtype=tf.float32) + + def _test_imagenetmodel_shape(self, resnet_version): + batch_size = 135 + num_classes = 246 + + model = imagenet_main.ImagenetModel( + 50, data_format='channels_last', num_classes=num_classes, + resnet_version=resnet_version) + + fake_input = tf.random.uniform([batch_size, 224, 224, 3]) + output = model(fake_input, training=True) + + self.assertAllEqual(output.shape, (batch_size, num_classes)) + + def test_imagenetmodel_shape_v1(self): + self._test_imagenetmodel_shape(resnet_version=1) + + def test_imagenetmodel_shape_v2(self): + self._test_imagenetmodel_shape(resnet_version=2) + + def test_imagenet_end_to_end_synthetic_v1(self): + integration.run_synthetic( + main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(), + extra_flags=['-resnet_version', '1', '-batch_size', '4', + '--max_train_steps', '1'] + ) + + def test_imagenet_end_to_end_synthetic_v2(self): + integration.run_synthetic( + main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(), + extra_flags=['-resnet_version', '2', '-batch_size', '4', + '--max_train_steps', '1'] + ) + + def test_imagenet_end_to_end_synthetic_v1_tiny(self): + integration.run_synthetic( + main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(), + extra_flags=['-resnet_version', '1', '-batch_size', '4', + '-resnet_size', '18', '--max_train_steps', '1'] + ) + + def test_imagenet_end_to_end_synthetic_v2_tiny(self): + integration.run_synthetic( + main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(), + extra_flags=['-resnet_version', '2', '-batch_size', '4', + '-resnet_size', '18', '--max_train_steps', '1'] + ) + + def test_imagenet_end_to_end_synthetic_v1_huge(self): + integration.run_synthetic( + main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(), + extra_flags=['-resnet_version', '1', '-batch_size', '4', + '-resnet_size', '200', '--max_train_steps', '1'] + ) + + def test_imagenet_end_to_end_synthetic_v2_huge(self): + integration.run_synthetic( + main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(), + extra_flags=['-resnet_version', '2', '-batch_size', '4', + '-resnet_size', '200', '--max_train_steps', '1'] + ) + + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/resnet_model.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/resnet_model.py new file mode 100644 index 0000000..611c880 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/resnet_model.py @@ -0,0 +1,552 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains definitions for Residual Networks. + +Residual networks ('v1' ResNets) were originally proposed in: +[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Deep Residual Learning for Image Recognition. arXiv:1512.03385 + +The full preactivation 'v2' ResNet variant was introduced by: +[2] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Identity Mappings in Deep Residual Networks. arXiv: 1603.05027 + +The key difference of the full preactivation 'v2' variant compared to the +'v1' variant in [1] is the use of batch normalization before every weight layer +rather than after. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +_BATCH_NORM_DECAY = 0.997 +_BATCH_NORM_EPSILON = 1e-5 +DEFAULT_VERSION = 2 +DEFAULT_DTYPE = tf.float32 +CASTABLE_TYPES = (tf.float16,) +ALLOWED_TYPES = (DEFAULT_DTYPE,) + CASTABLE_TYPES + + +################################################################################ +# Convenience functions for building the ResNet model. +################################################################################ +def batch_norm(inputs, training, data_format): + """Performs a batch normalization using a standard set of parameters.""" + # We set fused=True for a significant performance boost. See + # https://www.tensorflow.org/performance/performance_guide#common_fused_ops + return tf.compat.v1.layers.batch_normalization( + inputs=inputs, axis=1 if data_format == 'channels_first' else 3, + momentum=_BATCH_NORM_DECAY, epsilon=_BATCH_NORM_EPSILON, center=True, + scale=True, training=training, fused=True) + + +def fixed_padding(inputs, kernel_size, data_format): + """Pads the input along the spatial dimensions independently of input size. + + Args: + inputs: A tensor of size [batch, channels, height_in, width_in] or + [batch, height_in, width_in, channels] depending on data_format. + kernel_size: The kernel to be used in the conv2d or max_pool2d operation. + Should be a positive integer. + data_format: The input format ('channels_last' or 'channels_first'). + + Returns: + A tensor with the same format as the input with the data either intact + (if kernel_size == 1) or padded (if kernel_size > 1). + """ + pad_total = kernel_size - 1 + pad_beg = pad_total // 2 + pad_end = pad_total - pad_beg + + if data_format == 'channels_first': + padded_inputs = tf.pad(tensor=inputs, + paddings=[[0, 0], [0, 0], [pad_beg, pad_end], + [pad_beg, pad_end]]) + else: + padded_inputs = tf.pad(tensor=inputs, + paddings=[[0, 0], [pad_beg, pad_end], + [pad_beg, pad_end], [0, 0]]) + return padded_inputs + + +def conv2d_fixed_padding(inputs, filters, kernel_size, strides, data_format): + """Strided 2-D convolution with explicit padding.""" + # The padding is consistent and is based only on `kernel_size`, not on the + # dimensions of `inputs` (as opposed to using `tf.layers.conv2d` alone). + if strides > 1: + inputs = fixed_padding(inputs, kernel_size, data_format) + + return tf.compat.v1.layers.conv2d( + inputs=inputs, filters=filters, kernel_size=kernel_size, strides=strides, + padding=('SAME' if strides == 1 else 'VALID'), use_bias=False, + kernel_initializer=tf.compat.v1.variance_scaling_initializer(), + data_format=data_format) + + +################################################################################ +# ResNet block definitions. +################################################################################ +def _building_block_v1(inputs, filters, training, projection_shortcut, strides, + data_format): + """A single block for ResNet v1, without a bottleneck. + + Convolution then batch normalization then ReLU as described by: + Deep Residual Learning for Image Recognition + https://arxiv.org/pdf/1512.03385.pdf + by Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun, Dec 2015. + + Args: + inputs: A tensor of size [batch, channels, height_in, width_in] or + [batch, height_in, width_in, channels] depending on data_format. + filters: The number of filters for the convolutions. + training: A Boolean for whether the model is in training or inference + mode. Needed for batch normalization. + projection_shortcut: The function to use for projection shortcuts + (typically a 1x1 convolution when downsampling the input). + strides: The block's stride. If greater than 1, this block will ultimately + downsample the input. + data_format: The input format ('channels_last' or 'channels_first'). + + Returns: + The output tensor of the block; shape should match inputs. + """ + shortcut = inputs + + if projection_shortcut is not None: + shortcut = projection_shortcut(inputs) + shortcut = batch_norm(inputs=shortcut, training=training, + data_format=data_format) + + inputs = conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=3, strides=strides, + data_format=data_format) + inputs = batch_norm(inputs, training, data_format) + inputs = tf.nn.relu(inputs) + + inputs = conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=3, strides=1, + data_format=data_format) + inputs = batch_norm(inputs, training, data_format) + inputs += shortcut + inputs = tf.nn.relu(inputs) + + return inputs + + +def _building_block_v2(inputs, filters, training, projection_shortcut, strides, + data_format): + """A single block for ResNet v2, without a bottleneck. + + Batch normalization then ReLu then convolution as described by: + Identity Mappings in Deep Residual Networks + https://arxiv.org/pdf/1603.05027.pdf + by Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun, Jul 2016. + + Args: + inputs: A tensor of size [batch, channels, height_in, width_in] or + [batch, height_in, width_in, channels] depending on data_format. + filters: The number of filters for the convolutions. + training: A Boolean for whether the model is in training or inference + mode. Needed for batch normalization. + projection_shortcut: The function to use for projection shortcuts + (typically a 1x1 convolution when downsampling the input). + strides: The block's stride. If greater than 1, this block will ultimately + downsample the input. + data_format: The input format ('channels_last' or 'channels_first'). + + Returns: + The output tensor of the block; shape should match inputs. + """ + shortcut = inputs + inputs = batch_norm(inputs, training, data_format) + inputs = tf.nn.relu(inputs) + + # The projection shortcut should come after the first batch norm and ReLU + # since it performs a 1x1 convolution. + if projection_shortcut is not None: + shortcut = projection_shortcut(inputs) + + inputs = conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=3, strides=strides, + data_format=data_format) + + inputs = batch_norm(inputs, training, data_format) + inputs = tf.nn.relu(inputs) + inputs = conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=3, strides=1, + data_format=data_format) + + return inputs + shortcut + + +def _bottleneck_block_v1(inputs, filters, training, projection_shortcut, + strides, data_format): + """A single block for ResNet v1, with a bottleneck. + + Similar to _building_block_v1(), except using the "bottleneck" blocks + described in: + Convolution then batch normalization then ReLU as described by: + Deep Residual Learning for Image Recognition + https://arxiv.org/pdf/1512.03385.pdf + by Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun, Dec 2015. + + Args: + inputs: A tensor of size [batch, channels, height_in, width_in] or + [batch, height_in, width_in, channels] depending on data_format. + filters: The number of filters for the convolutions. + training: A Boolean for whether the model is in training or inference + mode. Needed for batch normalization. + projection_shortcut: The function to use for projection shortcuts + (typically a 1x1 convolution when downsampling the input). + strides: The block's stride. If greater than 1, this block will ultimately + downsample the input. + data_format: The input format ('channels_last' or 'channels_first'). + + Returns: + The output tensor of the block; shape should match inputs. + """ + shortcut = inputs + + if projection_shortcut is not None: + shortcut = projection_shortcut(inputs) + shortcut = batch_norm(inputs=shortcut, training=training, + data_format=data_format) + + inputs = conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=1, strides=1, + data_format=data_format) + inputs = batch_norm(inputs, training, data_format) + inputs = tf.nn.relu(inputs) + + inputs = conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=3, strides=strides, + data_format=data_format) + inputs = batch_norm(inputs, training, data_format) + inputs = tf.nn.relu(inputs) + + inputs = conv2d_fixed_padding( + inputs=inputs, filters=4 * filters, kernel_size=1, strides=1, + data_format=data_format) + inputs = batch_norm(inputs, training, data_format) + inputs += shortcut + inputs = tf.nn.relu(inputs) + + return inputs + + +def _bottleneck_block_v2(inputs, filters, training, projection_shortcut, + strides, data_format): + """A single block for ResNet v2, with a bottleneck. + + Similar to _building_block_v2(), except using the "bottleneck" blocks + described in: + Convolution then batch normalization then ReLU as described by: + Deep Residual Learning for Image Recognition + https://arxiv.org/pdf/1512.03385.pdf + by Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun, Dec 2015. + + Adapted to the ordering conventions of: + Batch normalization then ReLu then convolution as described by: + Identity Mappings in Deep Residual Networks + https://arxiv.org/pdf/1603.05027.pdf + by Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun, Jul 2016. + + Args: + inputs: A tensor of size [batch, channels, height_in, width_in] or + [batch, height_in, width_in, channels] depending on data_format. + filters: The number of filters for the convolutions. + training: A Boolean for whether the model is in training or inference + mode. Needed for batch normalization. + projection_shortcut: The function to use for projection shortcuts + (typically a 1x1 convolution when downsampling the input). + strides: The block's stride. If greater than 1, this block will ultimately + downsample the input. + data_format: The input format ('channels_last' or 'channels_first'). + + Returns: + The output tensor of the block; shape should match inputs. + """ + shortcut = inputs + inputs = batch_norm(inputs, training, data_format) + inputs = tf.nn.relu(inputs) + + # The projection shortcut should come after the first batch norm and ReLU + # since it performs a 1x1 convolution. + if projection_shortcut is not None: + shortcut = projection_shortcut(inputs) + + inputs = conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=1, strides=1, + data_format=data_format) + + inputs = batch_norm(inputs, training, data_format) + inputs = tf.nn.relu(inputs) + inputs = conv2d_fixed_padding( + inputs=inputs, filters=filters, kernel_size=3, strides=strides, + data_format=data_format) + + inputs = batch_norm(inputs, training, data_format) + inputs = tf.nn.relu(inputs) + inputs = conv2d_fixed_padding( + inputs=inputs, filters=4 * filters, kernel_size=1, strides=1, + data_format=data_format) + + return inputs + shortcut + + +def block_layer(inputs, filters, bottleneck, block_fn, blocks, strides, + training, name, data_format): + """Creates one layer of blocks for the ResNet model. + + Args: + inputs: A tensor of size [batch, channels, height_in, width_in] or + [batch, height_in, width_in, channels] depending on data_format. + filters: The number of filters for the first convolution of the layer. + bottleneck: Is the block created a bottleneck block. + block_fn: The block to use within the model, either `building_block` or + `bottleneck_block`. + blocks: The number of blocks contained in the layer. + strides: The stride to use for the first convolution of the layer. If + greater than 1, this layer will ultimately downsample the input. + training: Either True or False, whether we are currently training the + model. Needed for batch norm. + name: A string name for the tensor output of the block layer. + data_format: The input format ('channels_last' or 'channels_first'). + + Returns: + The output tensor of the block layer. + """ + + # Bottleneck blocks end with 4x the number of filters as they start with + filters_out = filters * 4 if bottleneck else filters + + def projection_shortcut(inputs): + return conv2d_fixed_padding( + inputs=inputs, filters=filters_out, kernel_size=1, strides=strides, + data_format=data_format) + + # Only the first block per block_layer uses projection_shortcut and strides + inputs = block_fn(inputs, filters, training, projection_shortcut, strides, + data_format) + + for _ in range(1, blocks): + inputs = block_fn(inputs, filters, training, None, 1, data_format) + + return tf.identity(inputs, name) + + +class Model(object): + """Base class for building the Resnet Model.""" + + def __init__(self, resnet_size, bottleneck, num_classes, num_filters, + kernel_size, + conv_stride, first_pool_size, first_pool_stride, + block_sizes, block_strides, + resnet_version=DEFAULT_VERSION, data_format=None, + dtype=DEFAULT_DTYPE): + """Creates a model for classifying an image. + + Args: + resnet_size: A single integer for the size of the ResNet model. + bottleneck: Use regular blocks or bottleneck blocks. + num_classes: The number of classes used as labels. + num_filters: The number of filters to use for the first block layer + of the model. This number is then doubled for each subsequent block + layer. + kernel_size: The kernel size to use for convolution. + conv_stride: stride size for the initial convolutional layer + first_pool_size: Pool size to be used for the first pooling layer. + If none, the first pooling layer is skipped. + first_pool_stride: stride size for the first pooling layer. Not used + if first_pool_size is None. + block_sizes: A list containing n values, where n is the number of sets of + block layers desired. Each value should be the number of blocks in the + i-th set. + block_strides: List of integers representing the desired stride size for + each of the sets of block layers. Should be same length as block_sizes. + resnet_version: Integer representing which version of the ResNet network + to use. See README for details. Valid values: [1, 2] + data_format: Input format ('channels_last', 'channels_first', or None). + If set to None, the format is dependent on whether a GPU is available. + dtype: The TensorFlow dtype to use for calculations. If not specified + tf.float32 is used. + + Raises: + ValueError: if invalid version is selected. + """ + self.resnet_size = resnet_size + + if not data_format: + data_format = ( + 'channels_first' if tf.test.is_built_with_cuda() else 'channels_last') + + self.resnet_version = resnet_version + if resnet_version not in (1, 2): + raise ValueError( + 'Resnet version should be 1 or 2. See README for citations.') + + self.bottleneck = bottleneck + if bottleneck: + if resnet_version == 1: + self.block_fn = _bottleneck_block_v1 + else: + self.block_fn = _bottleneck_block_v2 + else: + if resnet_version == 1: + self.block_fn = _building_block_v1 + else: + self.block_fn = _building_block_v2 + + if dtype not in ALLOWED_TYPES: + raise ValueError('dtype must be one of: {}'.format(ALLOWED_TYPES)) + + self.data_format = data_format + self.num_classes = num_classes + self.num_filters = num_filters + self.kernel_size = kernel_size + self.conv_stride = conv_stride + self.first_pool_size = first_pool_size + self.first_pool_stride = first_pool_stride + self.block_sizes = block_sizes + self.block_strides = block_strides + self.dtype = dtype + self.pre_activation = resnet_version == 2 + + def _custom_dtype_getter(self, getter, name, shape=None, dtype=DEFAULT_DTYPE, + *args, **kwargs): + """Creates variables in fp32, then casts to fp16 if necessary. + + This function is a custom getter. A custom getter is a function with the + same signature as tf.get_variable, except it has an additional getter + parameter. Custom getters can be passed as the `custom_getter` parameter of + tf.variable_scope. Then, tf.get_variable will call the custom getter, + instead of directly getting a variable itself. This can be used to change + the types of variables that are retrieved with tf.get_variable. + The `getter` parameter is the underlying variable getter, that would have + been called if no custom getter was used. Custom getters typically get a + variable with `getter`, then modify it in some way. + + This custom getter will create an fp32 variable. If a low precision + (e.g. float16) variable was requested it will then cast the variable to the + requested dtype. The reason we do not directly create variables in low + precision dtypes is that applying small gradients to such variables may + cause the variable not to change. + + Args: + getter: The underlying variable getter, that has the same signature as + tf.get_variable and returns a variable. + name: The name of the variable to get. + shape: The shape of the variable to get. + dtype: The dtype of the variable to get. Note that if this is a low + precision dtype, the variable will be created as a tf.float32 variable, + then cast to the appropriate dtype + *args: Additional arguments to pass unmodified to getter. + **kwargs: Additional keyword arguments to pass unmodified to getter. + + Returns: + A variable which is cast to fp16 if necessary. + """ + + if dtype in CASTABLE_TYPES: + var = getter(name, shape, tf.float32, *args, **kwargs) + return tf.cast(var, dtype=dtype, name=name + '_cast') + else: + return getter(name, shape, dtype, *args, **kwargs) + + def _model_variable_scope(self): + """Returns a variable scope that the model should be created under. + + If self.dtype is a castable type, model variable will be created in fp32 + then cast to self.dtype before being used. + + Returns: + A variable scope for the model. + """ + + return tf.compat.v1.variable_scope('resnet_model', + custom_getter=self._custom_dtype_getter) + + def __call__(self, inputs, training): + """Add operations to classify a batch of input images. + + Args: + inputs: A Tensor representing a batch of input images. + training: A boolean. Set to True to add operations required only when + training the classifier. + + Returns: + A logits Tensor with shape [, self.num_classes]. + """ + + with self._model_variable_scope(): + if self.data_format == 'channels_first': + # Convert the inputs from channels_last (NHWC) to channels_first (NCHW). + # This provides a large performance boost on GPU. See + # https://www.tensorflow.org/performance/performance_guide#data_formats + inputs = tf.transpose(a=inputs, perm=[0, 3, 1, 2]) + + inputs = conv2d_fixed_padding( + inputs=inputs, filters=self.num_filters, kernel_size=self.kernel_size, + strides=self.conv_stride, data_format=self.data_format) + inputs = tf.identity(inputs, 'initial_conv') + + # We do not include batch normalization or activation functions in V2 + # for the initial conv1 because the first ResNet unit will perform these + # for both the shortcut and non-shortcut paths as part of the first + # block's projection. Cf. Appendix of [2]. + if self.resnet_version == 1: + inputs = batch_norm(inputs, training, self.data_format) + inputs = tf.nn.relu(inputs) + + if self.first_pool_size: + ############## npu modify begin ############# + #max_pooling2d is replaced by max_pool_with_argmax for better performance + inputs,argmax = tf.compat.v1.nn.max_pool_with_argmax( + input=inputs, ksize=(1,self.first_pool_size,self.first_pool_size,1), + strides=(1,self.first_pool_stride,self.first_pool_stride,1), padding='SAME', + data_format='NCHW' if self.data_format == 'channels_first' else 'NHWC') + ############## npu modify end ############### + + inputs = tf.identity(inputs, 'initial_max_pool') + + for i, num_blocks in enumerate(self.block_sizes): + num_filters = self.num_filters * (2**i) + inputs = block_layer( + inputs=inputs, filters=num_filters, bottleneck=self.bottleneck, + block_fn=self.block_fn, blocks=num_blocks, + strides=self.block_strides[i], training=training, + name='block_layer{}'.format(i + 1), data_format=self.data_format) + + # Only apply the BN and ReLU for model that does pre_activation in each + # building/bottleneck block, eg resnet V2. + if self.pre_activation: + inputs = batch_norm(inputs, training, self.data_format) + inputs = tf.nn.relu(inputs) + + # The current top layer has shape + # `batch_size x pool_size x pool_size x final_size`. + # ResNet does an Average Pooling layer over pool_size, + # but that is the same as doing a reduce_mean. We do a reduce_mean + # here because it performs better than AveragePooling2D. + axes = [2, 3] if self.data_format == 'channels_first' else [1, 2] + inputs = tf.reduce_mean(input_tensor=inputs, axis=axes, keepdims=True) + inputs = tf.identity(inputs, 'final_reduce_mean') + + inputs = tf.squeeze(inputs, axes) + inputs = tf.compat.v1.layers.dense(inputs=inputs, units=self.num_classes) + inputs = tf.identity(inputs, 'final_dense') + return inputs diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/resnet_run_loop.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/resnet_run_loop.py new file mode 100644 index 0000000..9af2a34 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/resnet_run_loop.py @@ -0,0 +1,979 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains utility and supporting functions for ResNet. + + This module contains ResNet code which does not directly build layers. This +includes dataset management, hyperparameter and optimizer code, and argument +parsing. Code for defining the ResNet layers can be found in resnet_model.py. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools +import math +import multiprocessing +import os +import datetime + +import time +from absl import flags +import tensorflow as tf + +import logging +import sys + + +############## npu modify begin ############# +from npu_bridge.estimator.npu.npu_config import NPURunConfig +from npu_bridge.estimator.npu.npu_estimator import NPUEstimator +from npu_bridge.estimator.npu.npu_optimizer import NPUDistributedOptimizer +from npu_bridge.estimator import npu_ops +from npu_bridge.hccl import hccl_ops +from hccl.manage.api import get_local_rank_id +from hccl.manage.api import get_rank_size +from hccl.manage.api import get_rank_id +from tensorflow.core.protobuf import rewriter_config_pb2 +############## npu modify end ############### + +from official.r1.resnet import imagenet_preprocessing +from official.r1.resnet import resnet_model +from official.r1.utils import export +from official.utils.flags import core as flags_core +from official.utils.logs import hooks_helper +from official.utils.logs import logger +from official.utils.misc import distribution_utils +from official.utils.misc import model_helpers +from benchmark_log import hwlog + + +################################################################################ +# Functions for input processing. +################################################################################ +def process_record_dataset(dataset, + is_training, + batch_size, + shuffle_buffer, + parse_record_fn, + num_epochs=1, + dtype=tf.float32, + datasets_num_private_threads=None, + drop_remainder=False, + tf_data_experimental_slack=False): + """Given a Dataset with raw records, return an iterator over the records. + + Args: + dataset: A Dataset representing raw records + is_training: A boolean denoting whether the input is for training. + batch_size: The number of samples per batch. + shuffle_buffer: The buffer size to use when shuffling records. A larger + value results in better randomness, but smaller values reduce startup + time and use less memory. + parse_record_fn: A function that takes a raw record and returns the + corresponding (image, label) pair. + num_epochs: The number of epochs to repeat the dataset. + dtype: Data type to use for images/features. + datasets_num_private_threads: Number of threads for a private + threadpool created for all datasets computation. + drop_remainder: A boolean indicates whether to drop the remainder of the + batches. If True, the batch dimension will be static. + tf_data_experimental_slack: Whether to enable tf.data's + `experimental_slack` option. + + Returns: + Dataset of (image, label) pairs ready for iteration. + """ + # Defines a specific size thread pool for tf.data operations. + if datasets_num_private_threads: + options = tf.data.Options() + options.experimental_threading.private_threadpool_size = ( + datasets_num_private_threads) + dataset = dataset.with_options(options) + tf.compat.v1.logging.info('datasets_num_private_threads: %s', + datasets_num_private_threads) + + # Disable intra-op parallelism to optimize for throughput instead of latency. + options = tf.data.Options() + options.experimental_threading.max_intra_op_parallelism = 1 + dataset = dataset.with_options(options) + + # Prefetches a batch at a time to smooth out the time taken to load input + # files for shuffling and processing. + dataset = dataset.prefetch(buffer_size=batch_size) + if is_training: + # Shuffles records before repeating to respect epoch boundaries. + dataset = dataset.shuffle(buffer_size=shuffle_buffer) + + # Repeats the dataset for the number of epochs to train. + #dataset = dataset.repeat(num_epochs) + dataset = dataset.repeat() + # Parses the raw records into images and labels. + dataset = dataset.map( + lambda value: parse_record_fn(value, is_training, dtype), + num_parallel_calls=tf.data.experimental.AUTOTUNE) + dataset = dataset.batch(batch_size, drop_remainder=drop_remainder) + + # Operations between the final prefetch and the get_next call to the iterator + # will happen synchronously during run time. We prefetch here again to + # background all of the above processing work and keep it out of the + # critical training path. Setting buffer_size to tf.data.experimental.AUTOTUNE + # allows DistributionStrategies to adjust how many batches to fetch based + # on how many devices are present. + dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE) + + if tf_data_experimental_slack: + options = tf.data.Options() + options.experimental_slack = True + dataset = dataset.with_options(options) + + return dataset + + +def get_synth_input_fn(height, width, num_channels, num_classes, + dtype=tf.float32): + """Returns an input function that returns a dataset with random data. + + This input_fn returns a data set that iterates over a set of random data and + bypasses all preprocessing, e.g. jpeg decode and copy. The host to device + copy is still included. This used to find the upper throughput bound when + tunning the full input pipeline. + + Args: + height: Integer height that will be used to create a fake image tensor. + width: Integer width that will be used to create a fake image tensor. + num_channels: Integer depth that will be used to create a fake image tensor. + num_classes: Number of classes that should be represented in the fake labels + tensor + dtype: Data type for features/images. + + Returns: + An input_fn that can be used in place of a real one to return a dataset + that can be used for iteration. + """ + # pylint: disable=unused-argument + def input_fn(is_training, data_dir, batch_size, *args, **kwargs): + """Returns dataset filled with random data.""" + # Synthetic input should be within [0, 255]. + inputs = tf.random.truncated_normal( + [batch_size] + [height, width, num_channels], + dtype=dtype, + mean=127, + stddev=60, + name='synthetic_inputs') + + labels = tf.random.uniform( + [batch_size], + minval=0, + maxval=num_classes - 1, + dtype=tf.int32, + name='synthetic_labels') + data = tf.data.Dataset.from_tensors((inputs, labels)).repeat() + data = data.prefetch(buffer_size=tf.data.experimental.AUTOTUNE) + return data + + return input_fn + + +def image_bytes_serving_input_fn(image_shape, dtype=tf.float32): + """Serving input fn for raw jpeg images.""" + + def _preprocess_image(image_bytes): + """Preprocess a single raw image.""" + # Bounding box around the whole image. + bbox = tf.constant([0.0, 0.0, 1.0, 1.0], dtype=dtype, shape=[1, 1, 4]) + height, width, num_channels = image_shape + image = imagenet_preprocessing.preprocess_image( + image_bytes, bbox, height, width, num_channels, is_training=False) + return image + + image_bytes_list = tf.compat.v1.placeholder( + shape=[None], dtype=tf.string, name='input_tensor') + images = tf.map_fn( + _preprocess_image, image_bytes_list, back_prop=False, dtype=dtype) + return tf.estimator.export.TensorServingInputReceiver( + images, {'image_bytes': image_bytes_list}) + + +def override_flags_and_set_envars_for_gpu_thread_pool(flags_obj): + """Override flags and set env_vars for performance. + + These settings exist to test the difference between using stock settings + and manual tuning. It also shows some of the ENV_VARS that can be tweaked to + squeeze a few extra examples per second. These settings are defaulted to the + current platform of interest, which changes over time. + + On systems with small numbers of cpu cores, e.g. under 8 logical cores, + setting up a gpu thread pool with `tf_gpu_thread_mode=gpu_private` may perform + poorly. + + Args: + flags_obj: Current flags, which will be adjusted possibly overriding + what has been set by the user on the command-line. + """ + cpu_count = multiprocessing.cpu_count() + tf.compat.v1.logging.info('Logical CPU cores: %s', cpu_count) + + # Sets up thread pool for each GPU for op scheduling. + per_gpu_thread_count = 1 + total_gpu_thread_count = per_gpu_thread_count * flags_obj.num_gpus + os.environ['TF_GPU_THREAD_MODE'] = flags_obj.tf_gpu_thread_mode + os.environ['TF_GPU_THREAD_COUNT'] = str(per_gpu_thread_count) + tf.compat.v1.logging.info('TF_GPU_THREAD_COUNT: %s', + os.environ['TF_GPU_THREAD_COUNT']) + tf.compat.v1.logging.info('TF_GPU_THREAD_MODE: %s', + os.environ['TF_GPU_THREAD_MODE']) + + # Reduces general thread pool by number of threads used for GPU pool. + main_thread_count = cpu_count - total_gpu_thread_count + flags_obj.inter_op_parallelism_threads = main_thread_count + + # Sets thread count for tf.data. Logical cores minus threads assign to the + # private GPU pool along with 2 thread per GPU for event monitoring and + # sending / receiving tensors. + num_monitoring_threads = 2 * flags_obj.num_gpus + flags_obj.datasets_num_private_threads = (cpu_count - total_gpu_thread_count + - num_monitoring_threads) + + +################################################################################ +# Functions for running training/eval/validation loops for the model. +################################################################################ +def learning_rate_with_decay( + batch_size, batch_denom, num_images, boundary_epochs, decay_rates, + base_lr=0.1, warmup=False): + """Get a learning rate that decays step-wise as training progresses. + + Args: + batch_size: the number of examples processed in each training batch. + batch_denom: this value will be used to scale the base learning rate. + `0.1 * batch size` is divided by this number, such that when + batch_denom == batch_size, the initial learning rate will be 0.1. + num_images: total number of images that will be used for training. + boundary_epochs: list of ints representing the epochs at which we + decay the learning rate. + decay_rates: list of floats representing the decay rates to be used + for scaling the learning rate. It should have one more element + than `boundary_epochs`, and all elements should have the same type. + base_lr: Initial learning rate scaled based on batch_denom. + warmup: Run a 5 epoch warmup to the initial lr. + Returns: + Returns a function that takes a single argument - the number of batches + trained so far (global_step)- and returns the learning rate to be used + for training the next batch. + """ + initial_learning_rate = base_lr * batch_size / batch_denom + batches_per_epoch = num_images / batch_size + + # Reduce the learning rate at certain epochs. + # CIFAR-10: divide by 10 at epoch 100, 150, and 200 + # ImageNet: divide by 10 at epoch 30, 60, 80, and 90 + boundaries = [int(batches_per_epoch * epoch) for epoch in boundary_epochs] + vals = [initial_learning_rate * decay for decay in decay_rates] + + def learning_rate_fn(global_step): + """Builds scaled learning rate function with 5 epoch warm up.""" + + ############## npu modify begin ############# + #Using int32 for better computing performance + global_step=tf.cast(global_step,tf.int32) + ############## npu modify end ############### + + lr = tf.compat.v1.train.piecewise_constant(global_step, boundaries, vals) + if warmup: + warmup_steps = int(batches_per_epoch * 5) + warmup_lr = ( + initial_learning_rate * tf.cast(global_step, tf.float32) / tf.cast( + warmup_steps, tf.float32)) + return tf.cond(pred=global_step < warmup_steps, + true_fn=lambda: warmup_lr, + false_fn=lambda: lr) + return lr + + def poly_rate_fn(global_step): + """Handles linear scaling rule, gradual warmup, and LR decay. + + The learning rate starts at 0, then it increases linearly per step. After + FLAGS.poly_warmup_epochs, we reach the base learning rate (scaled to account + for batch size). The learning rate is then decayed using a polynomial rate + decay schedule with power 2.0. + + Args: + global_step: the current global_step + + Returns: + returns the current learning rate + """ + + # Learning rate schedule for LARS polynomial schedule + if flags.FLAGS.batch_size < 8192: + plr = 5.0 + w_epochs = 5 + elif flags.FLAGS.batch_size < 16384: + plr = 10.0 + w_epochs = 5 + elif flags.FLAGS.batch_size < 32768: + plr = 25.0 + w_epochs = 5 + else: + plr = 32.0 + w_epochs = 14 + + w_steps = int(w_epochs * batches_per_epoch) + wrate = (plr * tf.cast(global_step, tf.float32) / tf.cast( + w_steps, tf.float32)) + + # TODO(pkanwar): use a flag to help calc num_epochs. + num_epochs = 90 + train_steps = batches_per_epoch * num_epochs + + min_step = tf.constant(1, dtype=tf.int64) + decay_steps = tf.maximum(min_step, tf.subtract(global_step, w_steps)) + poly_rate = tf.train.polynomial_decay( + plr, + decay_steps, + train_steps - w_steps + 1, + power=2.0) + return tf.where(global_step <= w_steps, wrate, poly_rate) + + # For LARS we have a new learning rate schedule + if flags.FLAGS.enable_lars: + return poly_rate_fn + + return learning_rate_fn + + +def resnet_model_fn(features, labels, mode, model_class, + resnet_size, weight_decay, learning_rate_fn, momentum, + data_format, resnet_version, loss_scale, + loss_filter_fn=None, dtype=resnet_model.DEFAULT_DTYPE, + fine_tune=False, label_smoothing=0.0): + """Shared functionality for different resnet model_fns. + + Initializes the ResnetModel representing the model layers + and uses that model to build the necessary EstimatorSpecs for + the `mode` in question. For training, this means building losses, + the optimizer, and the train op that get passed into the EstimatorSpec. + For evaluation and prediction, the EstimatorSpec is returned without + a train op, but with the necessary parameters for the given mode. + + Args: + features: tensor representing input images + labels: tensor representing class labels for all input images + mode: current estimator mode; should be one of + `tf.estimator.ModeKeys.TRAIN`, `EVALUATE`, `PREDICT` + model_class: a class representing a TensorFlow model that has a __call__ + function. We assume here that this is a subclass of ResnetModel. + resnet_size: A single integer for the size of the ResNet model. + weight_decay: weight decay loss rate used to regularize learned variables. + learning_rate_fn: function that returns the current learning rate given + the current global_step + momentum: momentum term used for optimization + data_format: Input format ('channels_last', 'channels_first', or None). + If set to None, the format is dependent on whether a GPU is available. + resnet_version: Integer representing which version of the ResNet network to + use. See README for details. Valid values: [1, 2] + loss_scale: The factor to scale the loss for numerical stability. A detailed + summary is present in the arg parser help text. + loss_filter_fn: function that takes a string variable name and returns + True if the var should be included in loss calculation, and False + otherwise. If None, batch_normalization variables will be excluded + from the loss. + dtype: the TensorFlow dtype to use for calculations. + fine_tune: If True only train the dense layers(final layers). + label_smoothing: If greater than 0 then smooth the labels. + + Returns: + EstimatorSpec parameterized according to the input params and the + current mode. + """ + + # Generate a summary node for the images + tf.compat.v1.summary.image('images', features, max_outputs=6) + + ############## npu modify begin ############# + # Checks that features/images have same data type being used for calculations. + if features.dtype != dtype: + features=tf.cast(features,dtype) + ############## npu modify end ############### + + model = model_class(resnet_size, data_format, resnet_version=resnet_version, + dtype=dtype) + + logits = model(features, mode == tf.estimator.ModeKeys.TRAIN) + + # This acts as a no-op if the logits are already in fp32 (provided logits are + # not a SparseTensor). If dtype is is low precision, logits must be cast to + # fp32 for numerical stability. + logits = tf.cast(logits, tf.float32) + + predictions = { + 'classes': tf.argmax(input=logits, axis=1), + 'probabilities': tf.nn.softmax(logits, name='softmax_tensor') + } + + if mode == tf.estimator.ModeKeys.PREDICT: + # Return the predictions and the specification for serving a SavedModel + return tf.estimator.EstimatorSpec( + mode=mode, + predictions=predictions, + export_outputs={ + 'predict': tf.estimator.export.PredictOutput(predictions) + }) + + # Calculate loss, which includes softmax cross entropy and L2 regularization. + if label_smoothing != 0.0: + one_hot_labels = tf.one_hot(labels, 1001) + cross_entropy = tf.losses.softmax_cross_entropy( + logits=logits, onehot_labels=one_hot_labels, + label_smoothing=label_smoothing) + else: + cross_entropy = tf.compat.v1.losses.sparse_softmax_cross_entropy( + logits=logits, labels=labels) + + # Create a tensor named cross_entropy for logging purposes. + tf.identity(cross_entropy, name='cross_entropy') + tf.compat.v1.summary.scalar('cross_entropy', cross_entropy) + + # If no loss_filter_fn is passed, assume we want the default behavior, + # which is that batch_normalization variables are excluded from loss. + def exclude_batch_norm(name): + return 'batch_normalization' not in name + loss_filter_fn = loss_filter_fn or exclude_batch_norm + + # Add weight decay to the loss. + l2_loss = weight_decay * tf.add_n( + # loss is computed using fp32 for numerical stability. + [ + tf.nn.l2_loss(tf.cast(v, tf.float32)) + for v in tf.compat.v1.trainable_variables() + if loss_filter_fn(v.name) + ]) + tf.compat.v1.summary.scalar('l2_loss', l2_loss) + loss = cross_entropy + l2_loss + + if mode == tf.estimator.ModeKeys.TRAIN: + global_step = tf.compat.v1.train.get_or_create_global_step() + + learning_rate = learning_rate_fn(global_step) + + # Create a tensor named learning_rate for logging purposes + tf.identity(learning_rate, name='learning_rate') + tf.compat.v1.summary.scalar('learning_rate', learning_rate) + + if flags.FLAGS.enable_lars: + from tensorflow.contrib import opt as contrib_opt # pylint: disable=g-import-not-at-top + optimizer = contrib_opt.LARSOptimizer( + learning_rate, + momentum=momentum, + weight_decay=weight_decay, + skip_list=['batch_normalization', 'bias']) + else: + optimizer = tf.compat.v1.train.MomentumOptimizer( + learning_rate=learning_rate, + momentum=momentum + ) + + ############## npu modify begin ############# + optimizer = NPUDistributedOptimizer(optimizer) + ############## npu modify end ############### + + fp16_implementation = getattr(flags.FLAGS, 'fp16_implementation', None) + if fp16_implementation == 'graph_rewrite': + optimizer = ( + tf.compat.v1.train.experimental.enable_mixed_precision_graph_rewrite( + optimizer, loss_scale=loss_scale)) + + def _dense_grad_filter(gvs): + """Only apply gradient updates to the final layer. + + This function is used for fine tuning. + + Args: + gvs: list of tuples with gradients and variable info + Returns: + filtered gradients so that only the dense layer remains + """ + return [(g, v) for g, v in gvs if 'dense' in v.name] + #loss_scale = 512 + if loss_scale != 1 and fp16_implementation != 'graph_rewrite': + # When computing fp16 gradients, often intermediate tensor values are + # so small, they underflow to 0. To avoid this, we multiply the loss by + # loss_scale to make these tensor values loss_scale times bigger. + scaled_grad_vars = optimizer.compute_gradients(loss * loss_scale) + print(">>>>>>>>>>>>>>>>>>>") + print(loss_scale) + print("<<<<<<<<<<<<<<<<<<") + if fine_tune: + scaled_grad_vars = _dense_grad_filter(scaled_grad_vars) + + # Once the gradient computation is complete we can scale the gradients + # back to the correct scale before passing them to the optimizer. + unscaled_grad_vars = [(grad / loss_scale, var) + for grad, var in scaled_grad_vars] + minimize_op = optimizer.apply_gradients(unscaled_grad_vars, global_step) + else: + grad_vars = optimizer.compute_gradients(loss) + if fine_tune: + grad_vars = _dense_grad_filter(grad_vars) + minimize_op = optimizer.apply_gradients(grad_vars, global_step) + update_ops = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.UPDATE_OPS) + train_op = tf.group(minimize_op, update_ops) + else: + train_op = None + + ############## npu modify begin ############# + #Using float32 for better performance + accuracy = tf.compat.v1.metrics.accuracy(tf.cast(labels,tf.float32), predictions['classes']) + ############## npu modify end ############### + + accuracy_top_5 = tf.compat.v1.metrics.mean( + tf.nn.in_top_k(predictions=logits, targets=labels, k=5, name='top_5_op')) + + ############## npu modify begin ############# + #Using for 8P + rank_size = int(os.getenv('RANK_SIZE')) + newaccuracy = (hccl_ops.allreduce(accuracy[0], "sum") / rank_size, accuracy[1]) + newaccuracy_top_5 = (hccl_ops.allreduce(accuracy_top_5[0], "sum") / rank_size, accuracy_top_5[1]) + ############## npu modify begin ############# + + metrics = {'accuracy': newaccuracy, + 'accuracy_top_5': newaccuracy_top_5} + + # Create a tensor named train_accuracy for logging purposes + tf.identity(accuracy[1], name='train_accuracy') + tf.identity(accuracy_top_5[1], name='train_accuracy_top_5') + tf.compat.v1.summary.scalar('train_accuracy', accuracy[1]) + tf.compat.v1.summary.scalar('train_accuracy_top_5', accuracy_top_5[1]) + + return tf.estimator.EstimatorSpec( + mode=mode, + predictions=predictions, + loss=loss, + train_op=train_op, + eval_metric_ops=metrics) + +############## npu modify begin ############# +def init_npu(): + """Initialize npu manually. + Returns: + `init_sess` npu init session config. + `npu_init` npu init ops. + """ + npu_init = npu_ops.initialize_system() + config = tf.ConfigProto() + + #npu mix precision attribute set to true when using mix precision + config.graph_options.rewrite_options.remapping = rewriter_config_pb2.RewriterConfig.OFF + custom_op = config.graph_options.rewrite_options.custom_optimizers.add() + custom_op.name = "NpuOptimizer" + #custom_op.parameter_map["precision_mode"].b = True + custom_op.parameter_map["precision_mode"].s = tf.compat.as_bytes("allow_mix_precision") + custom_op.parameter_map["use_off_line"].b = True + + init_sess = tf.Session(config=config) + print("this is init sess config ------------- ",config) + print("this is npu_init ------------- ", npu_init) + # i=1 + # while(1): + # i+=1 + return init_sess,npu_init +############## npu modify end ############### + +def resnet_main( + flags_obj, model_function, input_function, dataset_name, num_images, shape=None): + """Shared main loop for ResNet Models. + + Args: + flags_obj: An object containing parsed flags. See define_resnet_flags() + for details. + model_function: the function that instantiates the Model and builds the + ops for train/eval. This will be passed directly into the estimator. + input_function: the function that processes the dataset and returns a + dataset that the estimator can train on. This will be wrapped with + all the relevant flags for running and passed to estimator. + dataset_name: the name of the dataset for training and evaluation. This is + used for logging purpose. + shape: list of ints representing the shape of the images used for training. + This is only used if flags_obj.export_dir is passed. + + Returns: + Dict of results of the run. Contains the keys `eval_results` and + `train_hooks`. `eval_results` contains accuracy (top_1) and accuracy_top_5. + `train_hooks` is a list the instances of hooks used during training. + """ + # Set other logger configurations + # work_num="work " + str(os.environ.get("DEVICE_INDEX")) + # hwlog.config( + # default_namespace=work_num, + # default_stack_offset=1, + # default_clear_line=False, + # root_dir=os.path.normpath( + # os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", ".."))) + + # global logger1 + # logger1 = get_logger('rizhi', log_file1) + + # print("work_num is ", work_num) + # exit() + model_helpers.apply_clean(flags.FLAGS) + + # Ensures flag override logic is only executed if explicitly triggered. + if flags_obj.tf_gpu_thread_mode: + override_flags_and_set_envars_for_gpu_thread_pool(flags_obj) + + # Configures cluster spec for distribution strategy. + num_workers = distribution_utils.configure_cluster(flags_obj.worker_hosts, + flags_obj.task_index) + + # Creates session config. allow_soft_placement = True, is required for + # multi-GPU and is not harmful for other modes. + session_config = tf.compat.v1.ConfigProto( + inter_op_parallelism_threads=flags_obj.inter_op_parallelism_threads, + intra_op_parallelism_threads=flags_obj.intra_op_parallelism_threads, + allow_soft_placement=True) + + distribution_strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=flags_obj.distribution_strategy, + num_gpus=flags_core.get_num_gpus(flags_obj), + all_reduce_alg=flags_obj.all_reduce_alg, + num_packs=flags_obj.num_packs) + + ############## npu modify begin ############# + # Creates a `NPURunConfig` that checkpoints every 115200 steps + run_config = NPURunConfig( + model_dir=flags_obj.model_dir, + session_config=session_config, + keep_checkpoint_max=5, + save_summary_steps=0, + #save_checkpoints_steps=115200, + save_checkpoints_steps=flags_obj.save_checkpoints_steps, + enable_data_pre_proc=True, + #iterations_per_loop=100, + iterations_per_loop=flags_obj.iterations_per_loop, + #enable_auto_mix_precision=True, + precision_mode='allow_mix_precision', + hcom_parallel=True + ) + ############## npu modify end ############### + + # Initializes model with all but the dense layer from pretrained ResNet. + if flags_obj.pretrained_model_checkpoint_path is not None: + warm_start_settings = tf.estimator.WarmStartSettings( + flags_obj.pretrained_model_checkpoint_path, + vars_to_warm_start='^(?!.*dense)') + else: + warm_start_settings = None + + ############## npu modify begin ############# + # Creates a `NPUEstimator` instead of using tf.estimator.Estimator + classifier = NPUEstimator( + model_fn=model_function, model_dir=flags_obj.model_dir, config=run_config, + params={ + 'resnet_size': int(flags_obj.resnet_size), + 'data_format': flags_obj.data_format, + 'batch_size': flags_obj.batch_size, + 'resnet_version': int(flags_obj.resnet_version), + 'loss_scale': flags_core.get_loss_scale(flags_obj, + default_for_fp16=128), + 'dtype': flags_core.get_tf_dtype(flags_obj), + 'fine_tune': flags_obj.fine_tune, + 'num_workers': num_workers, + 'num_gpus' : flags_core.get_num_gpus(flags_obj), + }) + ############## npu modify end ############### + + run_params = { + 'batch_size': flags_obj.batch_size, + 'dtype': flags_core.get_tf_dtype(flags_obj), + 'resnet_size': flags_obj.resnet_size, + 'resnet_version': flags_obj.resnet_version, + 'synthetic_data': flags_obj.use_synthetic_data, + 'train_epochs': flags_obj.train_epochs, + 'num_workers': num_workers, + } + if flags_obj.use_synthetic_data: + dataset_name = dataset_name + '-synthetic' + + benchmark_logger = logger.get_benchmark_logger() + benchmark_logger.log_run_info('resnet', dataset_name, run_params, + test_id=flags_obj.benchmark_test_id) + + train_hooks = hooks_helper.get_train_hooks( + flags_obj.hooks, + model_dir=flags_obj.model_dir, + batch_size=flags_obj.batch_size) + + def input_fn_train(num_epochs, input_context=None): + ############## npu modify begin ############# + # Using dtype=tf.float16 for higher data transmission performance + # drop_remainder currently only support true + # batch_size means single card batch instead of global batch size + return input_function( + is_training=True, + data_dir=flags_obj.data_dir, + batch_size=flags_obj.batch_size, + num_epochs=num_epochs, + dtype=tf.float16, + input_context=input_context, + drop_remainder=True) + + def input_fn_eval(): + # batch_size means single card batch instead of global batch size + # Using dtype=tf.float16 for higher data transmission performance + # drop_remainder currently only support true + return input_function( + is_training=False, + data_dir=flags_obj.data_dir, + batch_size=flags_obj.batch_size, + num_epochs=1, + dtype=tf.float16, + input_context=True, + drop_remainder=True) + ############## npu modify end ############### + + train_epochs = (0 if flags_obj.eval_only or not flags_obj.train_epochs else + flags_obj.train_epochs) + + use_train_and_evaluate = flags_obj.use_train_and_evaluate or num_workers > 1 + + ############## npu_kai modify end ############### + # init_sess, npu_init = init_npu() + # npu_shutdown = npu_ops.shutdown_system() + ############## npu_kai modify end ############### + + if use_train_and_evaluate: + train_spec = tf.estimator.TrainSpec( + input_fn=lambda input_context=None: input_fn_train( + train_epochs, input_context=input_context), + hooks=train_hooks, + max_steps=flags_obj.max_train_steps) + eval_spec = tf.estimator.EvalSpec(input_fn=input_fn_eval) + tf.compat.v1.logging.info('Starting to train and evaluate.') + tf.estimator.train_and_evaluate(classifier, train_spec, eval_spec) + # tf.estimator.train_and_evalute doesn't return anything in multi-worker + # case. + eval_results = {} + else: + if train_epochs == 0: + # If --eval_only is set, perform a single loop with zero train epochs. + schedule, n_loops = [0], 1 + else: + # Compute the number of times to loop while training. All but the last + # pass will train for `epochs_between_evals` epochs, while the last will + # train for the number needed to reach `training_epochs`. For instance if + # train_epochs = 25 and epochs_between_evals = 10 + # schedule will be set to [10, 10, 5]. That is to say, the loop will: + # Train for 10 epochs and then evaluate. + # Train for another 10 epochs and then evaluate. + # Train for a final 5 epochs (to reach 25 epochs) and then evaluate. + n_loops = math.ceil(train_epochs / flags_obj.epochs_between_evals) + schedule = [flags_obj.epochs_between_evals for _ in range(int(n_loops))] + schedule[-1] = train_epochs - sum(schedule[:-1]) # over counting. + + current_max_steps = 0 + ############## npu modify begin ############# + #if flags_obj.max_train_steps is None: + # flags_obj.max_train_steps = (num_images['train']/flags_obj.batch_size)/flags_core.get_num_gpus(flags_obj) + # max_eval_steps = num_images['validation']/flags_obj.batch_size + # else: + # max_eval_steps = flags_obj.max_train_steps + # for cycle_index, num_train_epochs in enumerate(schedule): + # print(cycle_index) + # print(num_train_epochs) + ############## npu modify end ############# + for cycle_index, num_train_epochs in enumerate(schedule): + tf.compat.v1.logging.info('Starting cycle: %d/%d', cycle_index, + int(n_loops)) + ############## npu modify begin ############# + if flags_obj.max_train_steps is None: + current_max_steps += ( + num_images['train'] / flags_obj.batch_size) * num_train_epochs / flags_core.get_num_gpus( + flags_obj) + else: + current_max_steps += flags_obj.max_train_steps + ############## npu modify end ############# + + # add zwx5326390训练开始 + # hwlogger.event(key=hwlog.constants.GLOBAL_BATCH_SIZE, value=flags_obj.batch_size) + #work_num, root_dir, datatime, resnet_logger = hwlog.env(log_file1) + #date_time = hwlog.get_time() + #resnet_logger.info("namespace: %s,time_ts: %s, global_batch_size: %d, num_train_epochs: %d" %(\ + #work_num, date_time, flags_obj.batch_size, num_train_epochs)) + #remark_logger.info("ABK time_ts: %s, current_epoch: %d, batch_size: %d, file: %s, lineno: %s" % (date_time, + # num_train_epochs, flags_obj.batch_size,file_name, sys._getframe().f_lineno)) + hwlog.remark_print(key=hwlog.CURRENT_EPOCH, value=num_train_epochs) + + if num_train_epochs: + # Since we are calling classifier.train immediately in each loop, the + # value of num_train_epochs in the lambda function will not be changed + # before it is used. So it is safe to ignore the pylint error here + # pylint: disable=cell-var-from-loop + # hwlogger.start(key=hwlog.constants.EPOCH_START) + + from hccl.split.api import set_split_strategy_by_idx + set_split_strategy_by_idx([86,160]) + classifier.train( + input_fn=lambda input_context=True: input_fn_train( + num_train_epochs, input_context=input_context), + hooks=train_hooks, + max_steps=current_max_steps) + + # hwlogger.end(key=hwlog.constants.EPOCH_STOP) + ############## npu modify begin ############# + # npu resorce will be destoryed When the training is over + # Reinitialize is needed if using hccl interface before next process + init_sess,npu_init=init_npu() + npu_shutdown = npu_ops.shutdown_system() + init_sess.run(npu_shutdown) + init_sess.run(npu_init) + ############## npu modify end ############### + + # flags_obj.max_train_steps is generally associated with testing and + # profiling. As a result it is frequently called with synthetic data, + # which will iterate forever. Passing steps=flags_obj.max_train_steps + # allows the eval (which is generally unimportant in those circumstances) + # to terminate. Note that eval will run for max_train_steps each loop, + # regardless of the global_step count. + tf.compat.v1.logging.info('Starting to evaluate.') + eval_results = classifier.evaluate(input_fn=input_fn_eval, + steps=num_images['validation']/flags_obj.batch_size) + benchmark_logger.log_evaluation_result(eval_results) + + #date_time = hwlog.get_time() + #remark_logger.info("ABK time_ts: %s, accuracy: %f, accuracy_top_5: %f, file: %s, lineno: %s" % (date_time, + # float(eval_results.get("accuracy")),float(eval_results.get("accuracy_top_5")), file_name,sys._getframe().f_lineno)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value=float(eval_results.get("accuracy"))) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value=float(eval_results.get("accuracy_top_5"))) + if model_helpers.past_stop_threshold( + flags_obj.stop_threshold, eval_results['accuracy']): + break + + ############## npu modify begin ############# + # npu resorce will be destoryed when evaluate finish + # Reinitialize is needed before using hccl interface + if cycle_index < n_loops-1: + init_sess,npu_init=init_npu() + npu_shutdown = npu_ops.shutdown_system() + init_sess.run(npu_shutdown) + #from hccl.split.api import set_split_strategy_by_idx + # set_split_strategy_by_idx([86,160]) + init_sess.run(npu_init) + ############## npu modify end ############### + + if flags_obj.export_dir is not None: + # Exports a saved model for the given classifier. + export_dtype = flags_core.get_tf_dtype(flags_obj) + if flags_obj.image_bytes_as_serving_input: + input_receiver_fn = functools.partial( + image_bytes_serving_input_fn, shape, dtype=export_dtype) + else: + input_receiver_fn = export.build_tensor_serving_input_receiver_fn( + shape, batch_size=flags_obj.batch_size, dtype=export_dtype) + classifier.export_savedmodel(flags_obj.export_dir, input_receiver_fn, + strip_default_attrs=True) + + ############## npu modify begin ############# + npu_shutdown = npu_ops.shutdown_system() + init_sess.run(npu_shutdown) + ############## npu modify end ############### + + stats = {} + stats['eval_results'] = eval_results + stats['train_hooks'] = train_hooks + + return stats + + +def define_resnet_flags(resnet_size_choices=None, dynamic_loss_scale=False, + fp16_implementation=False): + """Add flags and validators for ResNet.""" + flags_core.define_base(clean=True, train_epochs=True, + epochs_between_evals=True, stop_threshold=True, + num_gpu=True, hooks=True, export_dir=True, + distribution_strategy=True) + flags_core.define_performance(num_parallel_calls=False, + inter_op=True, + intra_op=True, + synthetic_data=True, + dtype=True, + all_reduce_alg=True, + num_packs=True, + tf_gpu_thread_mode=True, + datasets_num_private_threads=True, + dynamic_loss_scale=dynamic_loss_scale, + fp16_implementation=fp16_implementation, + loss_scale=True, + tf_data_experimental_slack=True, + max_train_steps=True) + flags_core.define_image() + flags_core.define_benchmark() + flags_core.define_distribution() + flags.adopt_module_key_flags(flags_core) + + flags.DEFINE_enum( + name='resnet_version', short_name='rv', default='1', + enum_values=['1', '2'], + help=flags_core.help_wrap( + 'Version of ResNet. (1 or 2) See README.md for details.')) + flags.DEFINE_bool( + name='fine_tune', short_name='ft', default=False, + help=flags_core.help_wrap( + 'If True do not train any parameters except for the final layer.')) + flags.DEFINE_string( + name='pretrained_model_checkpoint_path', short_name='pmcp', default=None, + help=flags_core.help_wrap( + 'If not None initialize all the network except the final layer with ' + 'these values')) + flags.DEFINE_boolean( + name='eval_only', default=False, + help=flags_core.help_wrap('Skip training and only perform evaluation on ' + 'the latest checkpoint.')) + flags.DEFINE_boolean( + name='image_bytes_as_serving_input', default=False, + help=flags_core.help_wrap( + 'If True exports savedmodel with serving signature that accepts ' + 'JPEG image bytes instead of a fixed size [HxWxC] tensor that ' + 'represents the image. The former is easier to use for serving at ' + 'the expense of image resize/cropping being done as part of model ' + 'inference. Note, this flag only applies to ImageNet and cannot ' + 'be used for CIFAR.')) + flags.DEFINE_boolean( + name='use_train_and_evaluate', default=False, + help=flags_core.help_wrap( + 'If True, uses `tf.estimator.train_and_evaluate` for the training ' + 'and evaluation loop, instead of separate calls to `classifier.train ' + 'and `classifier.evaluate`, which is the default behavior.')) + flags.DEFINE_bool( + name='enable_lars', default=False, + help=flags_core.help_wrap( + 'Enable LARS optimizer for large batch training.')) + flags.DEFINE_float( + name='label_smoothing', default=0.0, + help=flags_core.help_wrap( + 'Label smoothing parameter used in the softmax_cross_entropy')) + flags.DEFINE_float( + name='weight_decay', default=1e-4, + help=flags_core.help_wrap( + 'Weight decay coefficiant for l2 regularization.')) + + choice_kwargs = dict( + name='resnet_size', short_name='rs', default='50', + help=flags_core.help_wrap('The size of the ResNet model to use.')) + + if resnet_size_choices is None: + flags.DEFINE_string(**choice_kwargs) + else: + flags.DEFINE_enum(enum_values=resnet_size_choices, **choice_kwargs) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/resnet_run_loop_orgin.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/resnet_run_loop_orgin.py new file mode 100644 index 0000000..9cc46a4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/resnet_run_loop_orgin.py @@ -0,0 +1,901 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains utility and supporting functions for ResNet. + + This module contains ResNet code which does not directly build layers. This +includes dataset management, hyperparameter and optimizer code, and argument +parsing. Code for defining the ResNet layers can be found in resnet_model.py. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools +import math +import multiprocessing +import os + +from absl import flags +import tensorflow as tf + +############## npu modify begin ############# +from npu_bridge.estimator.npu.npu_config import NPURunConfig +from npu_bridge.estimator.npu.npu_estimator import NPUEstimator +from npu_bridge.estimator.npu.npu_optimizer import NPUDistributedOptimizer +from npu_bridge.estimator import npu_ops +from hccl.manage.api import get_local_rank_id +from hccl.manage.api import get_rank_size +from hccl.manage.api import get_rank_id +from tensorflow.core.protobuf import rewriter_config_pb2 +############## npu modify end ############### + +from official.r1.resnet import imagenet_preprocessing +from official.r1.resnet import resnet_model +from official.r1.utils import export +from official.utils.flags import core as flags_core +from official.utils.logs import hooks_helper +from official.utils.logs import logger +from official.utils.misc import distribution_utils +from official.utils.misc import model_helpers + + +################################################################################ +# Functions for input processing. +################################################################################ +def process_record_dataset(dataset, + is_training, + batch_size, + shuffle_buffer, + parse_record_fn, + num_epochs=1, + dtype=tf.float32, + datasets_num_private_threads=None, + drop_remainder=False, + tf_data_experimental_slack=False): + """Given a Dataset with raw records, return an iterator over the records. + + Args: + dataset: A Dataset representing raw records + is_training: A boolean denoting whether the input is for training. + batch_size: The number of samples per batch. + shuffle_buffer: The buffer size to use when shuffling records. A larger + value results in better randomness, but smaller values reduce startup + time and use less memory. + parse_record_fn: A function that takes a raw record and returns the + corresponding (image, label) pair. + num_epochs: The number of epochs to repeat the dataset. + dtype: Data type to use for images/features. + datasets_num_private_threads: Number of threads for a private + threadpool created for all datasets computation. + drop_remainder: A boolean indicates whether to drop the remainder of the + batches. If True, the batch dimension will be static. + tf_data_experimental_slack: Whether to enable tf.data's + `experimental_slack` option. + + Returns: + Dataset of (image, label) pairs ready for iteration. + """ + # Defines a specific size thread pool for tf.data operations. + if datasets_num_private_threads: + options = tf.data.Options() + options.experimental_threading.private_threadpool_size = ( + datasets_num_private_threads) + dataset = dataset.with_options(options) + tf.compat.v1.logging.info('datasets_num_private_threads: %s', + datasets_num_private_threads) + + # Disable intra-op parallelism to optimize for throughput instead of latency. + options = tf.data.Options() + options.experimental_threading.max_intra_op_parallelism = 1 + dataset = dataset.with_options(options) + + # Prefetches a batch at a time to smooth out the time taken to load input + # files for shuffling and processing. + dataset = dataset.prefetch(buffer_size=batch_size) + if is_training: + # Shuffles records before repeating to respect epoch boundaries. + dataset = dataset.shuffle(buffer_size=shuffle_buffer) + + # Repeats the dataset for the number of epochs to train. + #dataset = dataset.repeat(num_epochs) + dataset = dataset.repeat() + # Parses the raw records into images and labels. + dataset = dataset.map( + lambda value: parse_record_fn(value, is_training, dtype), + num_parallel_calls=tf.data.experimental.AUTOTUNE) + dataset = dataset.batch(batch_size, drop_remainder=drop_remainder) + + # Operations between the final prefetch and the get_next call to the iterator + # will happen synchronously during run time. We prefetch here again to + # background all of the above processing work and keep it out of the + # critical training path. Setting buffer_size to tf.data.experimental.AUTOTUNE + # allows DistributionStrategies to adjust how many batches to fetch based + # on how many devices are present. + dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE) + + if tf_data_experimental_slack: + options = tf.data.Options() + options.experimental_slack = True + dataset = dataset.with_options(options) + + return dataset + + +def get_synth_input_fn(height, width, num_channels, num_classes, + dtype=tf.float32): + """Returns an input function that returns a dataset with random data. + + This input_fn returns a data set that iterates over a set of random data and + bypasses all preprocessing, e.g. jpeg decode and copy. The host to device + copy is still included. This used to find the upper throughput bound when + tunning the full input pipeline. + + Args: + height: Integer height that will be used to create a fake image tensor. + width: Integer width that will be used to create a fake image tensor. + num_channels: Integer depth that will be used to create a fake image tensor. + num_classes: Number of classes that should be represented in the fake labels + tensor + dtype: Data type for features/images. + + Returns: + An input_fn that can be used in place of a real one to return a dataset + that can be used for iteration. + """ + # pylint: disable=unused-argument + def input_fn(is_training, data_dir, batch_size, *args, **kwargs): + """Returns dataset filled with random data.""" + # Synthetic input should be within [0, 255]. + inputs = tf.random.truncated_normal( + [batch_size] + [height, width, num_channels], + dtype=dtype, + mean=127, + stddev=60, + name='synthetic_inputs') + + labels = tf.random.uniform( + [batch_size], + minval=0, + maxval=num_classes - 1, + dtype=tf.int32, + name='synthetic_labels') + data = tf.data.Dataset.from_tensors((inputs, labels)).repeat() + data = data.prefetch(buffer_size=tf.data.experimental.AUTOTUNE) + return data + + return input_fn + + +def image_bytes_serving_input_fn(image_shape, dtype=tf.float32): + """Serving input fn for raw jpeg images.""" + + def _preprocess_image(image_bytes): + """Preprocess a single raw image.""" + # Bounding box around the whole image. + bbox = tf.constant([0.0, 0.0, 1.0, 1.0], dtype=dtype, shape=[1, 1, 4]) + height, width, num_channels = image_shape + image = imagenet_preprocessing.preprocess_image( + image_bytes, bbox, height, width, num_channels, is_training=False) + return image + + image_bytes_list = tf.compat.v1.placeholder( + shape=[None], dtype=tf.string, name='input_tensor') + images = tf.map_fn( + _preprocess_image, image_bytes_list, back_prop=False, dtype=dtype) + return tf.estimator.export.TensorServingInputReceiver( + images, {'image_bytes': image_bytes_list}) + + +def override_flags_and_set_envars_for_gpu_thread_pool(flags_obj): + """Override flags and set env_vars for performance. + + These settings exist to test the difference between using stock settings + and manual tuning. It also shows some of the ENV_VARS that can be tweaked to + squeeze a few extra examples per second. These settings are defaulted to the + current platform of interest, which changes over time. + + On systems with small numbers of cpu cores, e.g. under 8 logical cores, + setting up a gpu thread pool with `tf_gpu_thread_mode=gpu_private` may perform + poorly. + + Args: + flags_obj: Current flags, which will be adjusted possibly overriding + what has been set by the user on the command-line. + """ + cpu_count = multiprocessing.cpu_count() + tf.compat.v1.logging.info('Logical CPU cores: %s', cpu_count) + + # Sets up thread pool for each GPU for op scheduling. + per_gpu_thread_count = 1 + total_gpu_thread_count = per_gpu_thread_count * flags_obj.num_gpus + os.environ['TF_GPU_THREAD_MODE'] = flags_obj.tf_gpu_thread_mode + os.environ['TF_GPU_THREAD_COUNT'] = str(per_gpu_thread_count) + tf.compat.v1.logging.info('TF_GPU_THREAD_COUNT: %s', + os.environ['TF_GPU_THREAD_COUNT']) + tf.compat.v1.logging.info('TF_GPU_THREAD_MODE: %s', + os.environ['TF_GPU_THREAD_MODE']) + + # Reduces general thread pool by number of threads used for GPU pool. + main_thread_count = cpu_count - total_gpu_thread_count + flags_obj.inter_op_parallelism_threads = main_thread_count + + # Sets thread count for tf.data. Logical cores minus threads assign to the + # private GPU pool along with 2 thread per GPU for event monitoring and + # sending / receiving tensors. + num_monitoring_threads = 2 * flags_obj.num_gpus + flags_obj.datasets_num_private_threads = (cpu_count - total_gpu_thread_count + - num_monitoring_threads) + + +################################################################################ +# Functions for running training/eval/validation loops for the model. +################################################################################ +def learning_rate_with_decay( + batch_size, batch_denom, num_images, boundary_epochs, decay_rates, + base_lr=0.1, warmup=False): + """Get a learning rate that decays step-wise as training progresses. + + Args: + batch_size: the number of examples processed in each training batch. + batch_denom: this value will be used to scale the base learning rate. + `0.1 * batch size` is divided by this number, such that when + batch_denom == batch_size, the initial learning rate will be 0.1. + num_images: total number of images that will be used for training. + boundary_epochs: list of ints representing the epochs at which we + decay the learning rate. + decay_rates: list of floats representing the decay rates to be used + for scaling the learning rate. It should have one more element + than `boundary_epochs`, and all elements should have the same type. + base_lr: Initial learning rate scaled based on batch_denom. + warmup: Run a 5 epoch warmup to the initial lr. + Returns: + Returns a function that takes a single argument - the number of batches + trained so far (global_step)- and returns the learning rate to be used + for training the next batch. + """ + initial_learning_rate = base_lr * batch_size / batch_denom + batches_per_epoch = num_images / batch_size + + # Reduce the learning rate at certain epochs. + # CIFAR-10: divide by 10 at epoch 100, 150, and 200 + # ImageNet: divide by 10 at epoch 30, 60, 80, and 90 + boundaries = [int(batches_per_epoch * epoch) for epoch in boundary_epochs] + vals = [initial_learning_rate * decay for decay in decay_rates] + + def learning_rate_fn(global_step): + """Builds scaled learning rate function with 5 epoch warm up.""" + + ############## npu modify begin ############# + #Using int32 for better computing performance + global_step=tf.cast(global_step,tf.int32) + ############## npu modify end ############### + + lr = tf.compat.v1.train.piecewise_constant(global_step, boundaries, vals) + if warmup: + warmup_steps = int(batches_per_epoch * 5) + warmup_lr = ( + initial_learning_rate * tf.cast(global_step, tf.float32) / tf.cast( + warmup_steps, tf.float32)) + return tf.cond(pred=global_step < warmup_steps, + true_fn=lambda: warmup_lr, + false_fn=lambda: lr) + return lr + + def poly_rate_fn(global_step): + """Handles linear scaling rule, gradual warmup, and LR decay. + + The learning rate starts at 0, then it increases linearly per step. After + FLAGS.poly_warmup_epochs, we reach the base learning rate (scaled to account + for batch size). The learning rate is then decayed using a polynomial rate + decay schedule with power 2.0. + + Args: + global_step: the current global_step + + Returns: + returns the current learning rate + """ + + # Learning rate schedule for LARS polynomial schedule + if flags.FLAGS.batch_size < 8192: + plr = 5.0 + w_epochs = 5 + elif flags.FLAGS.batch_size < 16384: + plr = 10.0 + w_epochs = 5 + elif flags.FLAGS.batch_size < 32768: + plr = 25.0 + w_epochs = 5 + else: + plr = 32.0 + w_epochs = 14 + + w_steps = int(w_epochs * batches_per_epoch) + wrate = (plr * tf.cast(global_step, tf.float32) / tf.cast( + w_steps, tf.float32)) + + # TODO(pkanwar): use a flag to help calc num_epochs. + num_epochs = 90 + train_steps = batches_per_epoch * num_epochs + + min_step = tf.constant(1, dtype=tf.int64) + decay_steps = tf.maximum(min_step, tf.subtract(global_step, w_steps)) + poly_rate = tf.train.polynomial_decay( + plr, + decay_steps, + train_steps - w_steps + 1, + power=2.0) + return tf.where(global_step <= w_steps, wrate, poly_rate) + + # For LARS we have a new learning rate schedule + if flags.FLAGS.enable_lars: + return poly_rate_fn + + return learning_rate_fn + + +def resnet_model_fn(features, labels, mode, model_class, + resnet_size, weight_decay, learning_rate_fn, momentum, + data_format, resnet_version, loss_scale, + loss_filter_fn=None, dtype=resnet_model.DEFAULT_DTYPE, + fine_tune=False, label_smoothing=0.0): + """Shared functionality for different resnet model_fns. + + Initializes the ResnetModel representing the model layers + and uses that model to build the necessary EstimatorSpecs for + the `mode` in question. For training, this means building losses, + the optimizer, and the train op that get passed into the EstimatorSpec. + For evaluation and prediction, the EstimatorSpec is returned without + a train op, but with the necessary parameters for the given mode. + + Args: + features: tensor representing input images + labels: tensor representing class labels for all input images + mode: current estimator mode; should be one of + `tf.estimator.ModeKeys.TRAIN`, `EVALUATE`, `PREDICT` + model_class: a class representing a TensorFlow model that has a __call__ + function. We assume here that this is a subclass of ResnetModel. + resnet_size: A single integer for the size of the ResNet model. + weight_decay: weight decay loss rate used to regularize learned variables. + learning_rate_fn: function that returns the current learning rate given + the current global_step + momentum: momentum term used for optimization + data_format: Input format ('channels_last', 'channels_first', or None). + If set to None, the format is dependent on whether a GPU is available. + resnet_version: Integer representing which version of the ResNet network to + use. See README for details. Valid values: [1, 2] + loss_scale: The factor to scale the loss for numerical stability. A detailed + summary is present in the arg parser help text. + loss_filter_fn: function that takes a string variable name and returns + True if the var should be included in loss calculation, and False + otherwise. If None, batch_normalization variables will be excluded + from the loss. + dtype: the TensorFlow dtype to use for calculations. + fine_tune: If True only train the dense layers(final layers). + label_smoothing: If greater than 0 then smooth the labels. + + Returns: + EstimatorSpec parameterized according to the input params and the + current mode. + """ + + # Generate a summary node for the images + tf.compat.v1.summary.image('images', features, max_outputs=6) + + ############## npu modify begin ############# + # Checks that features/images have same data type being used for calculations. + if features.dtype != dtype: + features=tf.cast(features,dtype) + ############## npu modify end ############### + + model = model_class(resnet_size, data_format, resnet_version=resnet_version, + dtype=dtype) + + logits = model(features, mode == tf.estimator.ModeKeys.TRAIN) + + # This acts as a no-op if the logits are already in fp32 (provided logits are + # not a SparseTensor). If dtype is is low precision, logits must be cast to + # fp32 for numerical stability. + logits = tf.cast(logits, tf.float32) + + predictions = { + 'classes': tf.argmax(input=logits, axis=1), + 'probabilities': tf.nn.softmax(logits, name='softmax_tensor') + } + + if mode == tf.estimator.ModeKeys.PREDICT: + # Return the predictions and the specification for serving a SavedModel + return tf.estimator.EstimatorSpec( + mode=mode, + predictions=predictions, + export_outputs={ + 'predict': tf.estimator.export.PredictOutput(predictions) + }) + + # Calculate loss, which includes softmax cross entropy and L2 regularization. + if label_smoothing != 0.0: + one_hot_labels = tf.one_hot(labels, 1001) + cross_entropy = tf.losses.softmax_cross_entropy( + logits=logits, onehot_labels=one_hot_labels, + label_smoothing=label_smoothing) + else: + cross_entropy = tf.compat.v1.losses.sparse_softmax_cross_entropy( + logits=logits, labels=labels) + + # Create a tensor named cross_entropy for logging purposes. + tf.identity(cross_entropy, name='cross_entropy') + tf.compat.v1.summary.scalar('cross_entropy', cross_entropy) + + # If no loss_filter_fn is passed, assume we want the default behavior, + # which is that batch_normalization variables are excluded from loss. + def exclude_batch_norm(name): + return 'batch_normalization' not in name + loss_filter_fn = loss_filter_fn or exclude_batch_norm + + # Add weight decay to the loss. + l2_loss = weight_decay * tf.add_n( + # loss is computed using fp32 for numerical stability. + [ + tf.nn.l2_loss(tf.cast(v, tf.float32)) + for v in tf.compat.v1.trainable_variables() + if loss_filter_fn(v.name) + ]) + tf.compat.v1.summary.scalar('l2_loss', l2_loss) + loss = cross_entropy + l2_loss + + if mode == tf.estimator.ModeKeys.TRAIN: + global_step = tf.compat.v1.train.get_or_create_global_step() + + learning_rate = learning_rate_fn(global_step) + + # Create a tensor named learning_rate for logging purposes + tf.identity(learning_rate, name='learning_rate') + tf.compat.v1.summary.scalar('learning_rate', learning_rate) + + if flags.FLAGS.enable_lars: + from tensorflow.contrib import opt as contrib_opt # pylint: disable=g-import-not-at-top + optimizer = contrib_opt.LARSOptimizer( + learning_rate, + momentum=momentum, + weight_decay=weight_decay, + skip_list=['batch_normalization', 'bias']) + else: + optimizer = tf.compat.v1.train.MomentumOptimizer( + learning_rate=learning_rate, + momentum=momentum + ) + + ############## npu modify begin ############# + optimizer = NPUDistributedOptimizer(optimizer) + ############## npu modify end ############### + + fp16_implementation = getattr(flags.FLAGS, 'fp16_implementation', None) + if fp16_implementation == 'graph_rewrite': + optimizer = ( + tf.compat.v1.train.experimental.enable_mixed_precision_graph_rewrite( + optimizer, loss_scale=loss_scale)) + + def _dense_grad_filter(gvs): + """Only apply gradient updates to the final layer. + + This function is used for fine tuning. + + Args: + gvs: list of tuples with gradients and variable info + Returns: + filtered gradients so that only the dense layer remains + """ + return [(g, v) for g, v in gvs if 'dense' in v.name] + + # if loss_scale != 1 and fp16_implementation != 'graph_rewrite': + # When computing fp16 gradients, often intermediate tensor values are + # so small, they underflow to 0. To avoid this, we multiply the loss by + # loss_scale to make these tensor values loss_scale times bigger. + loss_scale = 512 + scaled_grad_vars = optimizer.compute_gradients(loss * loss_scale) + + if fine_tune: + scaled_grad_vars = _dense_grad_filter(scaled_grad_vars) + + # Once the gradient computation is complete we can scale the gradients + # back to the correct scale before passing them to the optimizer. + unscaled_grad_vars = [(grad / loss_scale, var) + for grad, var in scaled_grad_vars] + minimize_op = optimizer.apply_gradients(unscaled_grad_vars, global_step) + #else: + # grad_vars = optimizer.compute_gradients(loss) + # if fine_tune: + # grad_vars = _dense_grad_filter(grad_vars) + # minimize_op = optimizer.apply_gradients(grad_vars, global_step) + + update_ops = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.UPDATE_OPS) + train_op = tf.group(minimize_op, update_ops) + else: + train_op = None + + ############## npu modify begin ############# + #Using float32 for better performance + accuracy = tf.compat.v1.metrics.accuracy(tf.cast(labels,tf.float32), predictions['classes']) + ############## npu modify end ############### + + accuracy_top_5 = tf.compat.v1.metrics.mean( + tf.nn.in_top_k(predictions=logits, targets=labels, k=5, name='top_5_op')) + metrics = {'accuracy': accuracy, + 'accuracy_top_5': accuracy_top_5} + + # Create a tensor named train_accuracy for logging purposes + tf.identity(accuracy[1], name='train_accuracy') + tf.identity(accuracy_top_5[1], name='train_accuracy_top_5') + tf.compat.v1.summary.scalar('train_accuracy', accuracy[1]) + tf.compat.v1.summary.scalar('train_accuracy_top_5', accuracy_top_5[1]) + + return tf.estimator.EstimatorSpec( + mode=mode, + predictions=predictions, + loss=loss, + train_op=train_op, + eval_metric_ops=metrics) + +############## npu modify begin ############# +def init_npu(): + """Initialize npu manually. + Returns: + `init_sess` npu init session config. + `npu_init` npu init ops. + """ + npu_init = npu_ops.initialize_system() + config = tf.ConfigProto() + + #npu mix precision attribute set to true when using mix precision + config.graph_options.rewrite_options.remapping = rewriter_config_pb2.RewriterConfig.OFF + custom_op = config.graph_options.rewrite_options.custom_optimizers.add() + custom_op.name = "NpuOptimizer" + #custom_op.parameter_map["precision_mode"].b = True + custom_op.parameter_map["precision_mode"].s = tf.compat.as_bytes("allow_mix_precision") + custom_op.parameter_map["use_off_line"].b = True + + init_sess = tf.Session(config=config) + return init_sess,npu_init +############## npu modify end ############### + +def resnet_main( + flags_obj, model_function, input_function, dataset_name, num_images, shape=None): + """Shared main loop for ResNet Models. + + Args: + flags_obj: An object containing parsed flags. See define_resnet_flags() + for details. + model_function: the function that instantiates the Model and builds the + ops for train/eval. This will be passed directly into the estimator. + input_function: the function that processes the dataset and returns a + dataset that the estimator can train on. This will be wrapped with + all the relevant flags for running and passed to estimator. + dataset_name: the name of the dataset for training and evaluation. This is + used for logging purpose. + shape: list of ints representing the shape of the images used for training. + This is only used if flags_obj.export_dir is passed. + + Returns: + Dict of results of the run. Contains the keys `eval_results` and + `train_hooks`. `eval_results` contains accuracy (top_1) and accuracy_top_5. + `train_hooks` is a list the instances of hooks used during training. + """ + + model_helpers.apply_clean(flags.FLAGS) + + # Ensures flag override logic is only executed if explicitly triggered. + if flags_obj.tf_gpu_thread_mode: + override_flags_and_set_envars_for_gpu_thread_pool(flags_obj) + + # Configures cluster spec for distribution strategy. + num_workers = distribution_utils.configure_cluster(flags_obj.worker_hosts, + flags_obj.task_index) + + # Creates session config. allow_soft_placement = True, is required for + # multi-GPU and is not harmful for other modes. + session_config = tf.compat.v1.ConfigProto( + inter_op_parallelism_threads=flags_obj.inter_op_parallelism_threads, + intra_op_parallelism_threads=flags_obj.intra_op_parallelism_threads, + allow_soft_placement=True) + + distribution_strategy = distribution_utils.get_distribution_strategy( + distribution_strategy=flags_obj.distribution_strategy, + num_gpus=flags_core.get_num_gpus(flags_obj), + all_reduce_alg=flags_obj.all_reduce_alg, + num_packs=flags_obj.num_packs) + + ############## npu modify begin ############# + # Creates a `NPURunConfig` that checkpoints every 115200 steps + run_config = NPURunConfig( + model_dir=flags_obj.model_dir, + session_config=session_config, + keep_checkpoint_max=5, + save_checkpoints_steps=115200, + enable_data_pre_proc=True, + iterations_per_loop=100, + #enable_auto_mix_precision=True, + precision_mode='allow_mix_precision', + hcom_parallel=True + ) + ############## npu modify end ############### + + # Initializes model with all but the dense layer from pretrained ResNet. + if flags_obj.pretrained_model_checkpoint_path is not None: + warm_start_settings = tf.estimator.WarmStartSettings( + flags_obj.pretrained_model_checkpoint_path, + vars_to_warm_start='^(?!.*dense)') + else: + warm_start_settings = None + + ############## npu modify begin ############# + # Creates a `NPUEstimator` instead of using tf.estimator.Estimator + classifier = NPUEstimator( + model_fn=model_function, model_dir=flags_obj.model_dir, config=run_config, + params={ + 'resnet_size': int(flags_obj.resnet_size), + 'data_format': flags_obj.data_format, + 'batch_size': flags_obj.batch_size, + 'resnet_version': int(flags_obj.resnet_version), + 'loss_scale': flags_core.get_loss_scale(flags_obj, + default_for_fp16=128), + 'dtype': flags_core.get_tf_dtype(flags_obj), + 'fine_tune': flags_obj.fine_tune, + 'num_workers': num_workers, + 'num_gpus' : flags_core.get_num_gpus(flags_obj), + }) + ############## npu modify end ############### + + run_params = { + 'batch_size': flags_obj.batch_size, + 'dtype': flags_core.get_tf_dtype(flags_obj), + 'resnet_size': flags_obj.resnet_size, + 'resnet_version': flags_obj.resnet_version, + 'synthetic_data': flags_obj.use_synthetic_data, + 'train_epochs': flags_obj.train_epochs, + 'num_workers': num_workers, + } + if flags_obj.use_synthetic_data: + dataset_name = dataset_name + '-synthetic' + + benchmark_logger = logger.get_benchmark_logger() + benchmark_logger.log_run_info('resnet', dataset_name, run_params, + test_id=flags_obj.benchmark_test_id) + + train_hooks = hooks_helper.get_train_hooks( + flags_obj.hooks, + model_dir=flags_obj.model_dir, + batch_size=flags_obj.batch_size) + + def input_fn_train(num_epochs, input_context=None): + ############## npu modify begin ############# + # Using dtype=tf.float16 for higher data transmission performance + # drop_remainder currently only support true + # batch_size means single card batch instead of global batch size + return input_function( + is_training=True, + data_dir=flags_obj.data_dir, + batch_size=flags_obj.batch_size, + num_epochs=num_epochs, + dtype=tf.float16, + input_context=input_context, + drop_remainder=True) + + def input_fn_eval(): + # batch_size means single card batch instead of global batch size + # Using dtype=tf.float16 for higher data transmission performance + # drop_remainder currently only support true + return input_function( + is_training=False, + data_dir=flags_obj.data_dir, + batch_size=flags_obj.batch_size, + num_epochs=1, + dtype=tf.float16, + drop_remainder=True) + ############## npu modify end ############### + + train_epochs = (0 if flags_obj.eval_only or not flags_obj.train_epochs else + flags_obj.train_epochs) + + use_train_and_evaluate = flags_obj.use_train_and_evaluate or num_workers > 1 + if use_train_and_evaluate: + train_spec = tf.estimator.TrainSpec( + input_fn=lambda input_context=None: input_fn_train( + train_epochs, input_context=input_context), + hooks=train_hooks, + max_steps=flags_obj.max_train_steps) + eval_spec = tf.estimator.EvalSpec(input_fn=input_fn_eval) + tf.compat.v1.logging.info('Starting to train and evaluate.') + tf.estimator.train_and_evaluate(classifier, train_spec, eval_spec) + # tf.estimator.train_and_evalute doesn't return anything in multi-worker + # case. + eval_results = {} + else: + if train_epochs == 0: + # If --eval_only is set, perform a single loop with zero train epochs. + schedule, n_loops = [0], 1 + else: + # Compute the number of times to loop while training. All but the last + # pass will train for `epochs_between_evals` epochs, while the last will + # train for the number needed to reach `training_epochs`. For instance if + # train_epochs = 25 and epochs_between_evals = 10 + # schedule will be set to [10, 10, 5]. That is to say, the loop will: + # Train for 10 epochs and then evaluate. + # Train for another 10 epochs and then evaluate. + # Train for a final 5 epochs (to reach 25 epochs) and then evaluate. + n_loops = math.ceil(train_epochs / flags_obj.epochs_between_evals) + schedule = [flags_obj.epochs_between_evals for _ in range(int(n_loops))] + schedule[-1] = train_epochs - sum(schedule[:-1]) # over counting. + + ############## npu modify begin ############# + if flags_obj.max_train_steps is None: + flags_obj.max_train_steps = (num_images['train']/flags_obj.batch_size)/flags_core.get_num_gpus(flags_obj) + max_eval_steps = num_images['validation']/flags_obj.batch_size + else: + max_eval_steps = flags_obj.max_train_steps + ############## npu modify end ############# + for cycle_index, num_train_epochs in enumerate(schedule): + tf.compat.v1.logging.info('Starting cycle: %d/%d', cycle_index, + int(n_loops)) + + if num_train_epochs: + # Since we are calling classifier.train immediately in each loop, the + # value of num_train_epochs in the lambda function will not be changed + # before it is used. So it is safe to ignore the pylint error here + # pylint: disable=cell-var-from-loop + classifier.train( + input_fn=lambda input_context=True: input_fn_train( + num_train_epochs, input_context=input_context), + hooks=train_hooks, + max_steps=flags_obj.max_train_steps*(cycle_index+1)) + + ############## npu modify begin ############# + # npu resorce will be destoryed When the training is over + # Reinitialize is needed if using hccl interface before next process + init_sess,npu_init=init_npu() + npu_shutdown = npu_ops.shutdown_system() + init_sess.run(npu_shutdown) + init_sess.run(npu_init) + ############## npu modify end ############### + + # flags_obj.max_train_steps is generally associated with testing and + # profiling. As a result it is frequently called with synthetic data, + # which will iterate forever. Passing steps=flags_obj.max_train_steps + # allows the eval (which is generally unimportant in those circumstances) + # to terminate. Note that eval will run for max_train_steps each loop, + # regardless of the global_step count. + tf.compat.v1.logging.info('Starting to evaluate.') + eval_results = classifier.evaluate(input_fn=input_fn_eval, + steps=max_eval_steps) + benchmark_logger.log_evaluation_result(eval_results) + + + if model_helpers.past_stop_threshold( + flags_obj.stop_threshold, eval_results['accuracy']): + break + + ############## npu modify begin ############# + # npu resorce will be destoryed when evaluate finish + # Reinitialize is needed before using hccl interface + init_sess,npu_init=init_npu() + npu_shutdown = npu_ops.shutdown_system() + init_sess.run(npu_shutdown) + init_sess.run(npu_init) + ############## npu modify end ############### + + if flags_obj.export_dir is not None: + # Exports a saved model for the given classifier. + export_dtype = flags_core.get_tf_dtype(flags_obj) + if flags_obj.image_bytes_as_serving_input: + input_receiver_fn = functools.partial( + image_bytes_serving_input_fn, shape, dtype=export_dtype) + else: + input_receiver_fn = export.build_tensor_serving_input_receiver_fn( + shape, batch_size=flags_obj.batch_size, dtype=export_dtype) + classifier.export_savedmodel(flags_obj.export_dir, input_receiver_fn, + strip_default_attrs=True) + + ############## npu modify begin ############# + npu_shutdown = npu_ops.shutdown_system() + init_sess.run(npu_shutdown) + ############## npu modify end ############### + + stats = {} + stats['eval_results'] = eval_results + stats['train_hooks'] = train_hooks + + return stats + + +def define_resnet_flags(resnet_size_choices=None, dynamic_loss_scale=False, + fp16_implementation=False): + """Add flags and validators for ResNet.""" + flags_core.define_base(clean=True, train_epochs=True, + epochs_between_evals=True, stop_threshold=True, + num_gpu=True, hooks=True, export_dir=True, + distribution_strategy=True) + flags_core.define_performance(num_parallel_calls=False, + inter_op=True, + intra_op=True, + synthetic_data=True, + dtype=True, + all_reduce_alg=True, + num_packs=True, + tf_gpu_thread_mode=True, + datasets_num_private_threads=True, + dynamic_loss_scale=dynamic_loss_scale, + fp16_implementation=fp16_implementation, + loss_scale=True, + tf_data_experimental_slack=True, + max_train_steps=True) + flags_core.define_image() + flags_core.define_benchmark() + flags_core.define_distribution() + flags.adopt_module_key_flags(flags_core) + + flags.DEFINE_enum( + name='resnet_version', short_name='rv', default='1', + enum_values=['1', '2'], + help=flags_core.help_wrap( + 'Version of ResNet. (1 or 2) See README.md for details.')) + flags.DEFINE_bool( + name='fine_tune', short_name='ft', default=False, + help=flags_core.help_wrap( + 'If True do not train any parameters except for the final layer.')) + flags.DEFINE_string( + name='pretrained_model_checkpoint_path', short_name='pmcp', default=None, + help=flags_core.help_wrap( + 'If not None initialize all the network except the final layer with ' + 'these values')) + flags.DEFINE_boolean( + name='eval_only', default=False, + help=flags_core.help_wrap('Skip training and only perform evaluation on ' + 'the latest checkpoint.')) + flags.DEFINE_boolean( + name='image_bytes_as_serving_input', default=False, + help=flags_core.help_wrap( + 'If True exports savedmodel with serving signature that accepts ' + 'JPEG image bytes instead of a fixed size [HxWxC] tensor that ' + 'represents the image. The former is easier to use for serving at ' + 'the expense of image resize/cropping being done as part of model ' + 'inference. Note, this flag only applies to ImageNet and cannot ' + 'be used for CIFAR.')) + flags.DEFINE_boolean( + name='use_train_and_evaluate', default=False, + help=flags_core.help_wrap( + 'If True, uses `tf.estimator.train_and_evaluate` for the training ' + 'and evaluation loop, instead of separate calls to `classifier.train ' + 'and `classifier.evaluate`, which is the default behavior.')) + flags.DEFINE_bool( + name='enable_lars', default=False, + help=flags_core.help_wrap( + 'Enable LARS optimizer for large batch training.')) + flags.DEFINE_float( + name='label_smoothing', default=0.0, + help=flags_core.help_wrap( + 'Label smoothing parameter used in the softmax_cross_entropy')) + flags.DEFINE_float( + name='weight_decay', default=1e-4, + help=flags_core.help_wrap( + 'Weight decay coefficiant for l2 regularization.')) + + choice_kwargs = dict( + name='resnet_size', short_name='rs', default='50', + help=flags_core.help_wrap('The size of the ResNet model to use.')) + + if resnet_size_choices is None: + flags.DEFINE_string(**choice_kwargs) + else: + flags.DEFINE_enum(enum_values=resnet_size_choices, **choice_kwargs) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/slog b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/slog new file mode 100644 index 0000000..d5694e9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/resnet/slog @@ -0,0 +1,581 @@ +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:20.962.831 [tdt/device/../common/src/log.cpp:158][TSDaemon] begin to send heartbeat to appmon,[tdt/device/src/tsd/tsdaemon.cpp:1580:SendHeartBeatToAppMon]8462 Msg: running ok +[EVENT] TDT(8380,tsdaemon):2020-05-12-11:05:20.963.000 [tdt/device/../common/src/log.cpp:149][TsdEVENT] send heartbeat to appmon success,[tdt/device/src/tsd/tsdaemon.cpp:1587:SendHeartBeatToAppMon]8462 +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.243.682 [tdt/device/../common/src/log.cpp:158][HdcSever] drv accept an session,[tdt/device/../common/src/hdc_server.cpp:330:AcceptHdcSession]8454 Msg: running ok +[INFO] HDC(8380,tsdaemon):2020-05-12-11:05:22.243.730 [hardware/dev_plat/../dev_plat/devhdc/hdc_core.c:1609][drvHdcSetSessionReference:1609] >>> session 55, pid 8380 +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.243.752 [tdt/device/../common/src/log.cpp:158][HdcSever] drvHdcSetSessionReference success,[tdt/device/../common/src/hdc_server.cpp:342:AcceptHdcSession]8454 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.243.772 [tdt/device/../common/src/log.cpp:158][HdcSever] drv accept an session and drvHdcSetSessionReference success, sessionId=1,[tdt/device/../common/src/hdc_server.cpp:351:AcceptHdcSession]8454 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.243.788 [tdt/device/../common/src/log.cpp:158][HdcSever] accept an session sessionId=1, open recv thread,[tdt/device/../common/src/hdc_server.cpp:279:Accept]8454 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.243.823 [tdt/device/../common/src/log.cpp:158]HdcServer::AcceptConnection Start,[tdt/device/../common/src/hdc_server.cpp:310:AcceptHdcSession]8454 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.243.854 [tdt/device/../common/src/log.cpp:158]HdcServer::RecvData thread = 281470605762992,[tdt/device/../common/src/hdc_server.cpp:154:RecvData]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.243.945 [tdt/device/../common/src/log.cpp:158]tsdaemon get process sign successfully, procpid:40927 signSize:48,[tdt/device/src/tsd/tsdaemon.cpp:901:FmkToTsdMsg]30221 Msg: running ok +[EVENT] TDT(8380,tsdaemon):2020-05-12-11:05:22.243.963 [tdt/device/../common/src/log.cpp:149][TsdEVENT]FmkToTsdMsg dev[0] msg[6] sessionId[1] realDev[0] fmkSignPid[40927] profilingMode[0] rankSize[1],[tdt/device/src/tsd/tsdaemon.cpp:905:FmkToTsdMsg]30221 +[EVENT] TDT(8380,tsdaemon):2020-05-12-11:05:22.243.982 [tdt/device/../common/src/log.cpp:149][TsdEVENT]From FMK Start >>>>>>>>>> TSD dev[0] sessionId[1] realDev[0] fmkPid[40927] rankSize[1],[tdt/device/src/tsd/tsdaemon.cpp:853:FmkToTsdMsgProc]30221 +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.000 [tdt/device/../common/src/log.cpp:158][TSDaemon] isAllLastRcvThreadClean_ value:0,[tdt/device/src/tsd/tsdaemon.cpp:819:CleanAllLastRcvThreads]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.013 [tdt/device/../common/src/log.cpp:158][TSDPPCSER] JoinAllPPCRcvThreads() enter [threadSize=1]!,[tdt/device/src/tsd/ppc_server.cpp:96:JoinAllPPCRcvThreads]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.039 [tdt/device/../common/src/log.cpp:158][TSDPPCSER] JoinAllPPCRcvThreads() [ppc tid=281470588977584] [threadSize=1] [freeThreadSize=1].,[tdt/device/src/tsd/ppc_server.cpp:105:JoinAllPPCRcvThreads]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.056 [tdt/device/../common/src/log.cpp:158][TSDPPCSER] JoinAllPPCRcvThreads() [free tid=281470588977584].,[tdt/device/src/tsd/ppc_server.cpp:111:JoinAllPPCRcvThreads]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.071 [tdt/device/../common/src/log.cpp:158][TSDPPCSER] JoinAllPPCRcvThreads() Find free tid and joinable.,[tdt/device/src/tsd/ppc_server.cpp:114:JoinAllPPCRcvThreads]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.086 [tdt/device/../common/src/log.cpp:158][TSDPPCSER] JoinAllPPCRcvThreads() exit [threadSize=0] [freeThreadSize=0].,[tdt/device/src/tsd/ppc_server.cpp:129:JoinAllPPCRcvThreads]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.101 [tdt/device/../common/src/log.cpp:158][TSDaemon] CleanTsdRcvHdcThreads() enter [threadSize=2]!,[tdt/device/src/tsd/tsdaemon.cpp:333:CleanTsdRcvHdcThreads]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.120 [tdt/device/../common/src/log.cpp:158][TSDaemon] CleanTsdRcvHdcThreads() [tid=281470597370288] [threadSize=2]!,[tdt/device/src/tsd/tsdaemon.cpp:341:CleanTsdRcvHdcThreads]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.139 [tdt/device/../common/src/log.cpp:158][TSDaemon] CleanTsdRcvHdcThreads() [tid=281470572192176] [threadSize=2]!,[tdt/device/src/tsd/tsdaemon.cpp:341:CleanTsdRcvHdcThreads]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.152 [tdt/device/../common/src/log.cpp:158][TSDaemon] CleanTsdRcvHdcThreads() exit [threadSize=0]!,[tdt/device/src/tsd/tsdaemon.cpp:346:CleanTsdRcvHdcThreads]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.167 [tdt/device/../common/src/log.cpp:158][TSDaemon] CleanTsdRcvPPCThreads() enter [threadSize=2]!,[tdt/device/src/tsd/tsdaemon.cpp:356:CleanTsdRcvPPCThreads]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.181 [tdt/device/../common/src/log.cpp:158][TSDaemon] CleanTsdRcvPPCThreads() [tid=281470580584880] [threadSize=2]!,[tdt/device/src/tsd/tsdaemon.cpp:364:CleanTsdRcvPPCThreads]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.197 [tdt/device/../common/src/log.cpp:158][TSDaemon] CleanTsdRcvPPCThreads() [tid=281470563799472] [threadSize=2]!,[tdt/device/src/tsd/tsdaemon.cpp:364:CleanTsdRcvPPCThreads]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.241 [tdt/device/../common/src/log.cpp:158][TSDaemon] CleanTsdRcvPPCThreads() exit [threadSize=0]!,[tdt/device/src/tsd/tsdaemon.cpp:369:CleanTsdRcvPPCThreads]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.259 [tdt/device/../common/src/log.cpp:158][TSDaemon] StartSubProcess deviceId: 0, fmkPid: 40927, sessionId: 1, state: 0,[tdt/device/src/tsd/tsdaemon.cpp:630:StartSubProcess]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.273 [tdt/device/../common/src/log.cpp:158][TSDaemon] StartSubProcess rankSize: 1,,[tdt/device/src/tsd/tsdaemon.cpp:635:StartSubProcess]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.285 [tdt/device/../common/src/log.cpp:158][TSDaemon] Process HCCP is abandoned to start, the rank size is 1,[tdt/device/src/tsd/tsdaemon.cpp:651:StartSubProcess]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.337 [tdt/device/../common/src/log.cpp:158][TSDaemon] start delete file, direct is /home/HwHiAiUser/hdcd/device0/,[tdt/device/src/tsd/tsdaemon.cpp:1878:DeleteFileByPath]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.367 [tdt/device/../common/src/log.cpp:158][TSDaemon] start scan file, ent name is .,[tdt/device/src/tsd/tsdaemon.cpp:1887:DeleteFileByPath]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.382 [tdt/device/../common/src/log.cpp:158][TSDaemon] start scan file, ent name is ..,[tdt/device/src/tsd/tsdaemon.cpp:1887:DeleteFileByPath]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.398 [tdt/device/../common/src/log.cpp:158][TSDaemon] start scan file, ent name is etc,[tdt/device/src/tsd/tsdaemon.cpp:1887:DeleteFileByPath]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.412 [tdt/device/../common/src/log.cpp:158][TSDaemon] start scan file, ent name is upgrade,[tdt/device/src/tsd/tsdaemon.cpp:1887:DeleteFileByPath]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.457 [tdt/device/../common/src/log.cpp:158][TSDaemon] ExecuteStart() [tid=281470563799472]!,[tdt/device/src/tsd/tsdaemon.cpp:477:ExecuteStart]30222 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.244.477 [tdt/device/../common/src/log.cpp:158][TSDaemon] check pathName[/var/aicpu_scheduler], pathLen[20] procName[aicpu_scheduler], len[15], MAX_LEN[256] ,[tdt/device/src/tsd/tsdaemon.cpp:1514:CheckProcessInputParam]30222 Msg: running ok +[EVENT] TDT(8380,tsdaemon):2020-05-12-11:05:22.245.278 [tdt/device/../common/src/log.cpp:149][TsdEVENT]#### Start TSD->SubProcess[PROC] Start Msg Device[0] Proc[aicpu_scheduler] fmkPid[40927] fatherPid[8380] subPid[30223] #### profilingMode is[0],[tdt/device/src/tsd/tsdaemon.cpp:498:ExecuteStart]30222 +[OPLOG] TDT(8380,tsdaemon):2020-05-12-11:05:22.245.328 [tdt/device/../common/src/log.cpp:151][tdt/device/src/tsd/tsdaemon.cpp:499:ExecuteStart]30222 alloc resource {devOS:[30223]} for {dev:0} +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.245.390 [tdt/device/../common/src/log.cpp:158][TSDaemon] SetTsdToFmkMsg:deviceId[0], sessionId[1], subProcPid[30223],[tdt/device/src/tsd/tsdaemon.cpp:774:SetTsdToFmkMsg]30222 Msg: running ok +[INFO] HDC(30223,aicpu_scheduler):2020-05-12-11:05:22.279.681 [hardware/dev_plat/../dev_plat/devhdc/hdc_cfg_parse.c:190][CfgFileOpen:190] >>> /etc/hdcBasic.cfg not exist +[INFO] HDC(30223,aicpu_scheduler):2020-05-12-11:05:22.279.712 [hardware/dev_plat/../dev_plat/devhdc/hdc_core.c:554][hdcInit:554] >>> HDC pcie init,use default segment(524288) +[INFO] HDC(30223,aicpu_scheduler):2020-05-12-11:05:22.279.765 [hardware/dev_plat/../dev_plat/devhdc/hdc_core.c:539][hdcPcieInit:539] >>> after init hdc segment 524224, socket segment 0 +[INFO] HDC(30223,aicpu_scheduler):2020-05-12-11:05:22.279.780 [hardware/dev_plat/../dev_plat/devhdc/hdc_core.c:574][hdcInit:574] >>> HDC init success. +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.281.768 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.646658] [hdcdrv] [hdcdrv_config 2288] pid 30223 use segment 524224. +[WARNING] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.284.329 [tdt/device/../common/src/log.cpp:143]Register data type failed: hiaiSerializeFunc is existed,[tdt/common/common_inc/data_type_reg.h:192:Register]30223 Msg: func has already existed +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.284.723 [aicpu/aicpu_device/aicpu_schedule/compute_process/main.cc:185][AICPUFW] [main 185] Compute process(cloud) compile time is 02:13:48 Apr 26 2020 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.284.751 [aicpu/aicpu_device/aicpu_schedule/compute_process/main.cc:168][AICPUFW] [ParseArgs 168] Parse args success. deviceId=0, pid=40927, pidSign=e9b203cc443d80564c8c88a0d111cb95145fae36b00d1ec1, profilingMode=0. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.284.960 [hardware/dev_plat/../dev_core/devdrv/devdrv_container.c:193][devdrv] [devdrv_do_container 193] para.num(4). +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.284.983 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:83][AICPUFW] [Start 83] Aicpu_scheduler will start, hostpid=40927, deviceId=0, hostDeviceId = 0. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.284.999 [hardware/dev_plat/aicpufw/aicpufw_api.c:125][AICPUFW] [drvDevBindPid 125] drvDevBindPid enter: chip_id = 0, hostpid=40927, mode=0. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.285.015 [hardware/dev_plat/aicpufw/aicpufw_dev.c:348][AICPUFW] [aicpufw_dev_bind_pid 348] chip0 aicpu bind pid (40927). +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.285.805 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.651711] [devdrv] [devdrv_manager_container_init_devlist_ns 1139] num(0), dev(0) +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.285.825 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.651716] [devdrv] [devdrv_manager_container_init_devlist_ns 1139] num(1), dev(1) +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.285.836 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.651719] [devdrv] [devdrv_manager_container_init_devlist_ns 1139] num(2), dev(2) +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.285.846 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.651722] [devdrv] [devdrv_manager_container_init_devlist_ns 1139] num(3), dev(3) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.370.966 [hardware/dev_plat/aicpufw/aicpufw_dev.c:76][AICPUFW] [aicpufw_dev_open 76] chip_id:0 is opened success, fd=18. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.371.155 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:283][AICPUFW] [InitFd 283] InitFd begin, deviceId=0. +[INFO] DP(30223,aicpu_scheduler):2020-05-12-11:05:22.371.186 [datapreprocess/src/task_queue.cc:58][DP_PREPROCESS] [I] [datapreprocess/src/task_queue.cc:58] Begin create task queue eventfd. +[INFO] DP(30223,aicpu_scheduler):2020-05-12-11:05:22.371.209 [datapreprocess/src/task_queue.cc:70][DP_PREPROCESS] [I] [datapreprocess/src/task_queue.cc:70] End create task queue eventfd 19. +[INFO] DP(30223,aicpu_scheduler):2020-05-12-11:05:22.371.224 [datapreprocess/src/task_queue.cc:58][DP_PREPROCESS] [I] [datapreprocess/src/task_queue.cc:58] Begin create task queue eventfd. +[INFO] DP(30223,aicpu_scheduler):2020-05-12-11:05:22.371.241 [datapreprocess/src/task_queue.cc:70][DP_PREPROCESS] [I] [datapreprocess/src/task_queue.cc:70] End create task queue eventfd 20. +[TRACE] DP(30223,aicpu_scheduler):2020-05-12-11:05:22.371.256 [status:START] [datapreprocess/src/task_queue.cc:262]DP_PREPROCESS module has been initialized +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.371.275 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:315][AICPUFW] [InitFd 315] InitFd end, deviceId=0. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.371.288 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:100][AICPUFW] [Start 100] Begin start aicpu work task, deviceId=0, hostpid=40927. +[INFO] DEVMM(30223,aicpu_scheduler):2020-05-12-11:05:22.371.300 [hardware/dev_plat/../dev_plat/devmm/agentmm/agentmm_svm.c:137][drvMemInitSvmDevice 137] init svm start pid=40927. +[INFO] DEVMM(30223,aicpu_scheduler):2020-05-12-11:05:22.372.570 [hardware/dev_plat/../dev_plat/devmm/agentmm/agentmm_svm.c:120][devmm_init_svm_device 120] init svm (hpid:40927) succ. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.372.593 [hardware/dev_plat/aicpufw/aicpufw_api.c:89][AICPUFW] [drvCreateAicpuWorkTasks 89] drvCreateAicpuWorkTasks enter: chip_id = 0, pid=40927, mode=0. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.372.610 [hardware/dev_plat/aicpufw/aicpufw_api.c:103][AICPUFW] [drvCreateAicpuWorkTasks 103] chip[0] start load kernel serve, pid=40927. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.372.739 [hardware/dev_plat/aicpufw/aicpufw_dev.c:226][AICPUFW] [aicpufw_dev_register_pid 226] chip0 register pid (40927). +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.372.839 [hardware/dev_plat/../dev_core/devdrv/devdrv_manager.c:838][devdrv] [drvGetCpuInfo 838] Dev[1] cpu info:1 14 1 4 0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.372.875 [hardware/dev_plat/aicpufw/aicpufw_dev.c:209][AICPUFW] [aicpufw_dev_get_info 209] _ts_irq(65531) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.372.891 [hardware/dev_plat/aicpufw/aicpufw_dev.c:210][AICPUFW] [aicpufw_dev_get_info 210] _cpu_irq(65515) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.372.902 [hardware/dev_plat/aicpufw/aicpufw_dev.c:209][AICPUFW] [aicpufw_dev_get_info 209] _ts_irq(65530) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.372.914 [hardware/dev_plat/aicpufw/aicpufw_dev.c:210][AICPUFW] [aicpufw_dev_get_info 210] _cpu_irq(65514) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.372.923 [hardware/dev_plat/aicpufw/aicpufw_dev.c:209][AICPUFW] [aicpufw_dev_get_info 209] _ts_irq(65529) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.372.935 [hardware/dev_plat/aicpufw/aicpufw_dev.c:210][AICPUFW] [aicpufw_dev_get_info 210] _cpu_irq(65513) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.372.943 [hardware/dev_plat/aicpufw/aicpufw_dev.c:209][AICPUFW] [aicpufw_dev_get_info 209] _ts_irq(65528) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.372.955 [hardware/dev_plat/aicpufw/aicpufw_dev.c:210][AICPUFW] [aicpufw_dev_get_info 210] _cpu_irq(65512) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.372.964 [hardware/dev_plat/aicpufw/aicpufw_dev.c:209][AICPUFW] [aicpufw_dev_get_info 209] _ts_irq(65527) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.372.976 [hardware/dev_plat/aicpufw/aicpufw_dev.c:210][AICPUFW] [aicpufw_dev_get_info 210] _cpu_irq(65511) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.372.985 [hardware/dev_plat/aicpufw/aicpufw_dev.c:209][AICPUFW] [aicpufw_dev_get_info 209] _ts_irq(65526) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.372.998 [hardware/dev_plat/aicpufw/aicpufw_dev.c:210][AICPUFW] [aicpufw_dev_get_info 210] _cpu_irq(65510) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.007 [hardware/dev_plat/aicpufw/aicpufw_dev.c:209][AICPUFW] [aicpufw_dev_get_info 209] _ts_irq(65525) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.019 [hardware/dev_plat/aicpufw/aicpufw_dev.c:210][AICPUFW] [aicpufw_dev_get_info 210] _cpu_irq(65509) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.029 [hardware/dev_plat/aicpufw/aicpufw_dev.c:209][AICPUFW] [aicpufw_dev_get_info 209] _ts_irq(65524) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.041 [hardware/dev_plat/aicpufw/aicpufw_dev.c:210][AICPUFW] [aicpufw_dev_get_info 210] _cpu_irq(65508) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.050 [hardware/dev_plat/aicpufw/aicpufw_dev.c:209][AICPUFW] [aicpufw_dev_get_info 209] _ts_irq(65523) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.061 [hardware/dev_plat/aicpufw/aicpufw_dev.c:210][AICPUFW] [aicpufw_dev_get_info 210] _cpu_irq(65507) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.070 [hardware/dev_plat/aicpufw/aicpufw_dev.c:209][AICPUFW] [aicpufw_dev_get_info 209] _ts_irq(65522) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.081 [hardware/dev_plat/aicpufw/aicpufw_dev.c:210][AICPUFW] [aicpufw_dev_get_info 210] _cpu_irq(65506) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.090 [hardware/dev_plat/aicpufw/aicpufw_dev.c:209][AICPUFW] [aicpufw_dev_get_info 209] _ts_irq(65521) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.101 [hardware/dev_plat/aicpufw/aicpufw_dev.c:210][AICPUFW] [aicpufw_dev_get_info 210] _cpu_irq(65505) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.110 [hardware/dev_plat/aicpufw/aicpufw_dev.c:209][AICPUFW] [aicpufw_dev_get_info 209] _ts_irq(65520) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.122 [hardware/dev_plat/aicpufw/aicpufw_dev.c:210][AICPUFW] [aicpufw_dev_get_info 210] _cpu_irq(65504) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.130 [hardware/dev_plat/aicpufw/aicpufw_dev.c:209][AICPUFW] [aicpufw_dev_get_info 209] _ts_irq(65519) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.142 [hardware/dev_plat/aicpufw/aicpufw_dev.c:210][AICPUFW] [aicpufw_dev_get_info 210] _cpu_irq(65503) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.150 [hardware/dev_plat/aicpufw/aicpufw_dev.c:209][AICPUFW] [aicpufw_dev_get_info 209] _ts_irq(65518) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.162 [hardware/dev_plat/aicpufw/aicpufw_dev.c:210][AICPUFW] [aicpufw_dev_get_info 210] _cpu_irq(65502) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.170 [hardware/dev_plat/aicpufw/aicpufw_dev.c:209][AICPUFW] [aicpufw_dev_get_info 209] _ts_irq(0) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.182 [hardware/dev_plat/aicpufw/aicpufw_dev.c:210][AICPUFW] [aicpufw_dev_get_info 210] _cpu_irq(0) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.191 [hardware/dev_plat/aicpufw/aicpufw_dev.c:209][AICPUFW] [aicpufw_dev_get_info 209] _ts_irq(0) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.202 [hardware/dev_plat/aicpufw/aicpufw_dev.c:210][AICPUFW] [aicpufw_dev_get_info 210] _cpu_irq(0) +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.215 [hardware/dev_plat/aicpufw/aicpufw_dev.c:142][AICPUFW] [aicpufw_dev_mmap 142] mmap opened fd=18. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.227 [hardware/dev_plat/aicpufw/aicpufw_dev.c:152][AICPUFW] [aicpufw_dev_mmap 152] start mmap, fd=18, offset: 4096, size: 258048. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.255 [hardware/dev_plat/aicpufw/aicpufw_dev.c:155][AICPUFW] [aicpufw_dev_mmap 155] finish mmap, fd=18, offset: 4096, size: 258048, addr: 0x0xfffefe877000. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.270 [hardware/dev_plat/aicpufw/aicpufw_thread.c:121][AICPUFW] [aicpufw_thread_data_init 121] chip[0] ts[0] finish mmap sram_offset[4096] sram_size[258048], ret[0] +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.287 [hardware/dev_plat/aicpufw/aicpufw_dev.c:142][AICPUFW] [aicpufw_dev_mmap 142] mmap opened fd=18. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.297 [hardware/dev_plat/aicpufw/aicpufw_dev.c:152][AICPUFW] [aicpufw_dev_mmap 152] start mmap, fd=18, offset: 262144, size: 1048576. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.323 [hardware/dev_plat/aicpufw/aicpufw_dev.c:155][AICPUFW] [aicpufw_dev_mmap 155] finish mmap, fd=18, offset: 262144, size: 1048576, addr: 0x0xfffefe777000. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.338 [hardware/dev_plat/aicpufw/aicpufw_thread.c:142][AICPUFW] [aicpufw_thread_data_init 142] chip[0] chip_info.chip_id is 0x6528. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.354 [hardware/dev_plat/aicpufw/aicpufw_dev.c:142][AICPUFW] [aicpufw_dev_mmap 142] mmap opened fd=18. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.368 [hardware/dev_plat/aicpufw/aicpufw_dev.c:152][AICPUFW] [aicpufw_dev_mmap 152] start mmap, fd=18, offset: 1310720, size: 4096. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.385 [hardware/dev_plat/aicpufw/aicpufw_dev.c:155][AICPUFW] [aicpufw_dev_mmap 155] finish mmap, fd=18, offset: 1310720, size: 4096, addr: 0x0xfffeffffa000. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.400 [hardware/dev_plat/aicpufw/aicpufw_thread.c:165][AICPUFW] [aicpufw_thread_data_init 165] finish mmap chip[0] ts[0] sram[0x0xfffefe877000] +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.420 [hardware/dev_plat/aicpufw/aicpufw_thread.c:710][AICPUFW] [aicpufw_thread_create 710] chip0 aicpu num: 14, first_aicpu: 2. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.441 [hardware/dev_plat/aicpufw/aicpufw_thread.c:720][AICPUFW] [aicpufw_thread_create 720] pthread_create for aicpu_index=0 begin +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.454 [hardware/dev_plat/aicpufw/aicpufw_thread.c:656][AICPUFW] [aicpufw_thread_create_one 656] thread_create_one chip_id:0, aicpu_index:0, ts_ind:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.852 [hardware/dev_plat/aicpufw/aicpufw_thread.c:682][AICPUFW] [aicpufw_thread_create_one 682] thread_create_one aicpu_index:0, ret:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.872 [hardware/dev_plat/aicpufw/aicpufw_thread.c:689][AICPUFW] [aicpufw_thread_create_one 689] thread[0] id 281470656012688 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.373.874 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.737826] [aicpufw-drv] [aicpufw_init_dfx 467] there are 2 processes open,current tgid: 30223 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.881 [hardware/dev_plat/aicpufw/aicpufw_thread.c:722][AICPUFW] [aicpufw_thread_create 722] pthread_create for aicpu_index=0 end ret=0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.892 [hardware/dev_plat/aicpufw/aicpufw_thread.c:720][AICPUFW] [aicpufw_thread_create 720] pthread_create for aicpu_index=1 begin +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.373.896 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.739661] [aicpufw-drv] [aicpufw_drv_add_match_info_check 1240] register pid(40927) ts_index(0) monitor_is_running(1). +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.900 [hardware/dev_plat/aicpufw/aicpufw_thread.c:656][AICPUFW] [aicpufw_thread_create_one 656] thread_create_one chip_id:0, aicpu_index:1, ts_ind:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.373.910 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.739665] [aicpufw-drv] [aicpufw_drv_add_match_info 1282] register pid(40927) ts_index(0) monitor_is_running(1). +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.373.921 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.739680] [aicpufw-drv] [aicpufw_drv_get_moniter_info 1583] aicpufw event happened. dev_id:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.373.930 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.739733] [devdrv] [devdrv_manager_get_cpu_info 1927] aicpu_num=14, ccpu_num=1 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.373.941 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.740140] [aicpufw-drv] [aicpufw_drv_mmap 1123] mmap_sram,ts_index=0, offset=4096, ts_size=4096. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.940 [hardware/dev_plat/aicpufw/aicpufw_thread.c:682][AICPUFW] [aicpufw_thread_create_one 682] thread_create_one aicpu_index:1, ret:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.373.950 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.740143] [aicpufw-drv] [aicpufw_drv_mmap_sram 974] sram status mem: virt_addr = 0xfffefe877000, tgid = 30223, size = 258048,offset = 4096, numa node = 0, ts = 0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.952 [hardware/dev_plat/aicpufw/aicpufw_thread.c:689][AICPUFW] [aicpufw_thread_create_one 689] thread[1] id 281470647619984 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.962 [hardware/dev_plat/aicpufw/aicpufw_thread.c:722][AICPUFW] [aicpufw_thread_create 722] pthread_create for aicpu_index=1 end ret=0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.373.964 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.740150] [aicpufw-drv] [aicpufw_drv_mmap_sram 995] finish sram status mem: virt_addr = 0xfffefe877000, tgid = 30223, size = 258048,offset = 4096, numa node = 0, ts = 0, ret = 0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.973 [hardware/dev_plat/aicpufw/aicpufw_thread.c:720][AICPUFW] [aicpufw_thread_create 720] pthread_create for aicpu_index=2 begin +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.373.976 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.740210] [aicpufw-drv] [aicpufw_drv_mmap 1127] mmap_gicd,ts_index=0, offset=262144, ts_size=4096, sram_size=258048. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.373.983 [hardware/dev_plat/aicpufw/aicpufw_thread.c:656][AICPUFW] [aicpufw_thread_create_one 656] thread_create_one chip_id:0, aicpu_index:2, ts_ind:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.373.988 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.740212] [aicpufw-drv] [aicpufw_drv_mmap_gicd 926] gicd status mem: virt_addr = 0xfffefe777000, tgid = 30223, size = 1048576,offset = 262144, numa node = 0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.373.999 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.740278] [aicpufw-drv] [aicpufw_drv_mmap 1132] mmap_gicr,ts_index=0, offset=1310720, ts_size=4096, sram_size=258048, gicd_size=1048576. +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.374.008 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.740280] [aicpufw-drv] [aicpufw_drv_mmap_gicr 892] ts gicr mem: virt_addr = 0xfffeffffa000, tgid = 30223, size = 4096,offset = 1310720, numa node = 0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.011 [hardware/dev_plat/aicpufw/aicpufw_thread.c:682][AICPUFW] [aicpufw_thread_create_one 682] thread_create_one aicpu_index:2, ret:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.023 [hardware/dev_plat/aicpufw/aicpufw_thread.c:689][AICPUFW] [aicpufw_thread_create_one 689] thread[2] id 281470639227280 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.033 [hardware/dev_plat/aicpufw/aicpufw_thread.c:722][AICPUFW] [aicpufw_thread_create 722] pthread_create for aicpu_index=2 end ret=0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.042 [hardware/dev_plat/aicpufw/aicpufw_thread.c:720][AICPUFW] [aicpufw_thread_create 720] pthread_create for aicpu_index=3 begin +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.051 [hardware/dev_plat/aicpufw/aicpufw_thread.c:656][AICPUFW] [aicpufw_thread_create_one 656] thread_create_one chip_id:0, aicpu_index:3, ts_ind:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.079 [hardware/dev_plat/aicpufw/aicpufw_thread.c:682][AICPUFW] [aicpufw_thread_create_one 682] thread_create_one aicpu_index:3, ret:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.090 [hardware/dev_plat/aicpufw/aicpufw_thread.c:689][AICPUFW] [aicpufw_thread_create_one 689] thread[3] id 281470630834576 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.099 [hardware/dev_plat/aicpufw/aicpufw_thread.c:722][AICPUFW] [aicpufw_thread_create 722] pthread_create for aicpu_index=3 end ret=0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.109 [hardware/dev_plat/aicpufw/aicpufw_thread.c:720][AICPUFW] [aicpufw_thread_create 720] pthread_create for aicpu_index=4 begin +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.118 [hardware/dev_plat/aicpufw/aicpufw_thread.c:656][AICPUFW] [aicpufw_thread_create_one 656] thread_create_one chip_id:0, aicpu_index:4, ts_ind:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.154 [hardware/dev_plat/aicpufw/aicpufw_thread.c:682][AICPUFW] [aicpufw_thread_create_one 682] thread_create_one aicpu_index:4, ret:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.168 [hardware/dev_plat/aicpufw/aicpufw_thread.c:689][AICPUFW] [aicpufw_thread_create_one 689] thread[4] id 281470622441872 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.178 [hardware/dev_plat/aicpufw/aicpufw_thread.c:722][AICPUFW] [aicpufw_thread_create 722] pthread_create for aicpu_index=4 end ret=0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.190 [hardware/dev_plat/aicpufw/aicpufw_thread.c:720][AICPUFW] [aicpufw_thread_create 720] pthread_create for aicpu_index=5 begin +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.199 [hardware/dev_plat/aicpufw/aicpufw_thread.c:656][AICPUFW] [aicpufw_thread_create_one 656] thread_create_one chip_id:0, aicpu_index:5, ts_ind:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.230 [hardware/dev_plat/aicpufw/aicpufw_thread.c:682][AICPUFW] [aicpufw_thread_create_one 682] thread_create_one aicpu_index:5, ret:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.243 [hardware/dev_plat/aicpufw/aicpufw_thread.c:689][AICPUFW] [aicpufw_thread_create_one 689] thread[5] id 281470614049168 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.253 [hardware/dev_plat/aicpufw/aicpufw_thread.c:722][AICPUFW] [aicpufw_thread_create 722] pthread_create for aicpu_index=5 end ret=0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.265 [hardware/dev_plat/aicpufw/aicpufw_thread.c:720][AICPUFW] [aicpufw_thread_create 720] pthread_create for aicpu_index=6 begin +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.275 [hardware/dev_plat/aicpufw/aicpufw_thread.c:656][AICPUFW] [aicpufw_thread_create_one 656] thread_create_one chip_id:0, aicpu_index:6, ts_ind:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.307 [hardware/dev_plat/aicpufw/aicpufw_thread.c:682][AICPUFW] [aicpufw_thread_create_one 682] thread_create_one aicpu_index:6, ret:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.321 [hardware/dev_plat/aicpufw/aicpufw_thread.c:689][AICPUFW] [aicpufw_thread_create_one 689] thread[6] id 281470605656464 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.330 [hardware/dev_plat/aicpufw/aicpufw_thread.c:722][AICPUFW] [aicpufw_thread_create 722] pthread_create for aicpu_index=6 end ret=0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.342 [hardware/dev_plat/aicpufw/aicpufw_thread.c:720][AICPUFW] [aicpufw_thread_create 720] pthread_create for aicpu_index=7 begin +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.351 [hardware/dev_plat/aicpufw/aicpufw_thread.c:656][AICPUFW] [aicpufw_thread_create_one 656] thread_create_one chip_id:0, aicpu_index:7, ts_ind:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.381 [hardware/dev_plat/aicpufw/aicpufw_thread.c:682][AICPUFW] [aicpufw_thread_create_one 682] thread_create_one aicpu_index:7, ret:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.395 [hardware/dev_plat/aicpufw/aicpufw_thread.c:689][AICPUFW] [aicpufw_thread_create_one 689] thread[7] id 281470597263760 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.404 [hardware/dev_plat/aicpufw/aicpufw_thread.c:722][AICPUFW] [aicpufw_thread_create 722] pthread_create for aicpu_index=7 end ret=0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.415 [hardware/dev_plat/aicpufw/aicpufw_thread.c:720][AICPUFW] [aicpufw_thread_create 720] pthread_create for aicpu_index=8 begin +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.425 [hardware/dev_plat/aicpufw/aicpufw_thread.c:656][AICPUFW] [aicpufw_thread_create_one 656] thread_create_one chip_id:0, aicpu_index:8, ts_ind:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.455 [hardware/dev_plat/aicpufw/aicpufw_thread.c:682][AICPUFW] [aicpufw_thread_create_one 682] thread_create_one aicpu_index:8, ret:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.469 [hardware/dev_plat/aicpufw/aicpufw_thread.c:689][AICPUFW] [aicpufw_thread_create_one 689] thread[8] id 281470588871056 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.478 [hardware/dev_plat/aicpufw/aicpufw_thread.c:722][AICPUFW] [aicpufw_thread_create 722] pthread_create for aicpu_index=8 end ret=0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.490 [hardware/dev_plat/aicpufw/aicpufw_thread.c:720][AICPUFW] [aicpufw_thread_create 720] pthread_create for aicpu_index=9 begin +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.499 [hardware/dev_plat/aicpufw/aicpufw_thread.c:656][AICPUFW] [aicpufw_thread_create_one 656] thread_create_one chip_id:0, aicpu_index:9, ts_ind:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.528 [hardware/dev_plat/aicpufw/aicpufw_thread.c:682][AICPUFW] [aicpufw_thread_create_one 682] thread_create_one aicpu_index:9, ret:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.541 [hardware/dev_plat/aicpufw/aicpufw_thread.c:689][AICPUFW] [aicpufw_thread_create_one 689] thread[9] id 281470580478352 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.550 [hardware/dev_plat/aicpufw/aicpufw_thread.c:722][AICPUFW] [aicpufw_thread_create 722] pthread_create for aicpu_index=9 end ret=0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.561 [hardware/dev_plat/aicpufw/aicpufw_thread.c:720][AICPUFW] [aicpufw_thread_create 720] pthread_create for aicpu_index=10 begin +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.571 [hardware/dev_plat/aicpufw/aicpufw_thread.c:656][AICPUFW] [aicpufw_thread_create_one 656] thread_create_one chip_id:0, aicpu_index:10, ts_ind:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.599 [hardware/dev_plat/aicpufw/aicpufw_thread.c:682][AICPUFW] [aicpufw_thread_create_one 682] thread_create_one aicpu_index:10, ret:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.612 [hardware/dev_plat/aicpufw/aicpufw_thread.c:689][AICPUFW] [aicpufw_thread_create_one 689] thread[10] id 281470572085648 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.621 [hardware/dev_plat/aicpufw/aicpufw_thread.c:722][AICPUFW] [aicpufw_thread_create 722] pthread_create for aicpu_index=10 end ret=0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.633 [hardware/dev_plat/aicpufw/aicpufw_thread.c:720][AICPUFW] [aicpufw_thread_create 720] pthread_create for aicpu_index=11 begin +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.642 [hardware/dev_plat/aicpufw/aicpufw_thread.c:656][AICPUFW] [aicpufw_thread_create_one 656] thread_create_one chip_id:0, aicpu_index:11, ts_ind:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.676 [hardware/dev_plat/aicpufw/aicpufw_thread.c:682][AICPUFW] [aicpufw_thread_create_one 682] thread_create_one aicpu_index:11, ret:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.690 [hardware/dev_plat/aicpufw/aicpufw_thread.c:689][AICPUFW] [aicpufw_thread_create_one 689] thread[11] id 281470563692944 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.699 [hardware/dev_plat/aicpufw/aicpufw_thread.c:722][AICPUFW] [aicpufw_thread_create 722] pthread_create for aicpu_index=11 end ret=0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.711 [hardware/dev_plat/aicpufw/aicpufw_thread.c:720][AICPUFW] [aicpufw_thread_create 720] pthread_create for aicpu_index=12 begin +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.720 [hardware/dev_plat/aicpufw/aicpufw_thread.c:656][AICPUFW] [aicpufw_thread_create_one 656] thread_create_one chip_id:0, aicpu_index:12, ts_ind:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.748 [hardware/dev_plat/aicpufw/aicpufw_thread.c:682][AICPUFW] [aicpufw_thread_create_one 682] thread_create_one aicpu_index:12, ret:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.762 [hardware/dev_plat/aicpufw/aicpufw_thread.c:689][AICPUFW] [aicpufw_thread_create_one 689] thread[12] id 281470555300240 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.771 [hardware/dev_plat/aicpufw/aicpufw_thread.c:722][AICPUFW] [aicpufw_thread_create 722] pthread_create for aicpu_index=12 end ret=0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.783 [hardware/dev_plat/aicpufw/aicpufw_thread.c:720][AICPUFW] [aicpufw_thread_create 720] pthread_create for aicpu_index=13 begin +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.792 [hardware/dev_plat/aicpufw/aicpufw_thread.c:656][AICPUFW] [aicpufw_thread_create_one 656] thread_create_one chip_id:0, aicpu_index:13, ts_ind:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.820 [hardware/dev_plat/aicpufw/aicpufw_thread.c:682][AICPUFW] [aicpufw_thread_create_one 682] thread_create_one aicpu_index:13, ret:0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.833 [hardware/dev_plat/aicpufw/aicpufw_thread.c:689][AICPUFW] [aicpufw_thread_create_one 689] thread[13] id 281470546907536 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.842 [hardware/dev_plat/aicpufw/aicpufw_thread.c:722][AICPUFW] [aicpufw_thread_create 722] pthread_create for aicpu_index=13 end ret=0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.854 [hardware/dev_plat/aicpufw/aicpufw_thread.c:731][AICPUFW] [aicpufw_thread_create 731] sem_post start chip_id=0, ts_ind=0, aicpu_index0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.864 [hardware/dev_plat/aicpufw/aicpufw_thread.c:733][AICPUFW] [aicpufw_thread_create 733] sem_post end chip_id=0, ts_ind=0, aicpu_index0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.876 [hardware/dev_plat/aicpufw/aicpufw_thread.c:731][AICPUFW] [aicpufw_thread_create 731] sem_post start chip_id=0, ts_ind=0, aicpu_index1 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.885 [hardware/dev_plat/aicpufw/aicpufw_thread.c:733][AICPUFW] [aicpufw_thread_create 733] sem_post end chip_id=0, ts_ind=0, aicpu_index1 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.897 [hardware/dev_plat/aicpufw/aicpufw_thread.c:731][AICPUFW] [aicpufw_thread_create 731] sem_post start chip_id=0, ts_ind=0, aicpu_index2 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.907 [hardware/dev_plat/aicpufw/aicpufw_thread.c:733][AICPUFW] [aicpufw_thread_create 733] sem_post end chip_id=0, ts_ind=0, aicpu_index2 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.918 [hardware/dev_plat/aicpufw/aicpufw_thread.c:731][AICPUFW] [aicpufw_thread_create 731] sem_post start chip_id=0, ts_ind=0, aicpu_index3 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.928 [hardware/dev_plat/aicpufw/aicpufw_thread.c:733][AICPUFW] [aicpufw_thread_create 733] sem_post end chip_id=0, ts_ind=0, aicpu_index3 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.940 [hardware/dev_plat/aicpufw/aicpufw_thread.c:731][AICPUFW] [aicpufw_thread_create 731] sem_post start chip_id=0, ts_ind=0, aicpu_index4 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.949 [hardware/dev_plat/aicpufw/aicpufw_thread.c:733][AICPUFW] [aicpufw_thread_create 733] sem_post end chip_id=0, ts_ind=0, aicpu_index4 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.961 [hardware/dev_plat/aicpufw/aicpufw_thread.c:731][AICPUFW] [aicpufw_thread_create 731] sem_post start chip_id=0, ts_ind=0, aicpu_index5 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.970 [hardware/dev_plat/aicpufw/aicpufw_thread.c:733][AICPUFW] [aicpufw_thread_create 733] sem_post end chip_id=0, ts_ind=0, aicpu_index5 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.982 [hardware/dev_plat/aicpufw/aicpufw_thread.c:731][AICPUFW] [aicpufw_thread_create 731] sem_post start chip_id=0, ts_ind=0, aicpu_index6 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.374.992 [hardware/dev_plat/aicpufw/aicpufw_thread.c:733][AICPUFW] [aicpufw_thread_create 733] sem_post end chip_id=0, ts_ind=0, aicpu_index6 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.003 [hardware/dev_plat/aicpufw/aicpufw_thread.c:731][AICPUFW] [aicpufw_thread_create 731] sem_post start chip_id=0, ts_ind=0, aicpu_index7 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.012 [hardware/dev_plat/aicpufw/aicpufw_thread.c:733][AICPUFW] [aicpufw_thread_create 733] sem_post end chip_id=0, ts_ind=0, aicpu_index7 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.024 [hardware/dev_plat/aicpufw/aicpufw_thread.c:731][AICPUFW] [aicpufw_thread_create 731] sem_post start chip_id=0, ts_ind=0, aicpu_index8 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.033 [hardware/dev_plat/aicpufw/aicpufw_thread.c:733][AICPUFW] [aicpufw_thread_create 733] sem_post end chip_id=0, ts_ind=0, aicpu_index8 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.045 [hardware/dev_plat/aicpufw/aicpufw_thread.c:731][AICPUFW] [aicpufw_thread_create 731] sem_post start chip_id=0, ts_ind=0, aicpu_index9 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.054 [hardware/dev_plat/aicpufw/aicpufw_thread.c:733][AICPUFW] [aicpufw_thread_create 733] sem_post end chip_id=0, ts_ind=0, aicpu_index9 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.065 [hardware/dev_plat/aicpufw/aicpufw_thread.c:731][AICPUFW] [aicpufw_thread_create 731] sem_post start chip_id=0, ts_ind=0, aicpu_index10 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.075 [hardware/dev_plat/aicpufw/aicpufw_thread.c:733][AICPUFW] [aicpufw_thread_create 733] sem_post end chip_id=0, ts_ind=0, aicpu_index10 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.086 [hardware/dev_plat/aicpufw/aicpufw_thread.c:731][AICPUFW] [aicpufw_thread_create 731] sem_post start chip_id=0, ts_ind=0, aicpu_index11 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.095 [hardware/dev_plat/aicpufw/aicpufw_thread.c:733][AICPUFW] [aicpufw_thread_create 733] sem_post end chip_id=0, ts_ind=0, aicpu_index11 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.107 [hardware/dev_plat/aicpufw/aicpufw_thread.c:731][AICPUFW] [aicpufw_thread_create 731] sem_post start chip_id=0, ts_ind=0, aicpu_index12 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.117 [hardware/dev_plat/aicpufw/aicpufw_thread.c:733][AICPUFW] [aicpufw_thread_create 733] sem_post end chip_id=0, ts_ind=0, aicpu_index12 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.129 [hardware/dev_plat/aicpufw/aicpufw_thread.c:731][AICPUFW] [aicpufw_thread_create 731] sem_post start chip_id=0, ts_ind=0, aicpu_index13 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.138 [hardware/dev_plat/aicpufw/aicpufw_thread.c:733][AICPUFW] [aicpufw_thread_create 733] sem_post end chip_id=0, ts_ind=0, aicpu_index13 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.150 [hardware/dev_plat/aicpufw/aicpufw_thread.c:737][AICPUFW] [aicpufw_thread_create 737] sem_wait start chip_id=0, ts_ind=0, aicpu_index0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.217 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:181][AICPUFW] [EventThreadTask 181] Aicpu device[0]:thread[ 5] begin, tid: 30230. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.329 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:181][AICPUFW] [EventThreadTask 181] Aicpu device[0]:thread[ 6] begin, tid: 30231. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.366 [hardware/dev_plat/aicpufw/aicpufw_thread.c:475][AICPUFW] [aicpufw_thread_set_affinity 475] thread_set_affinity, bind_cpu_index 7, chip 0, ts 0, aicpu_index 5, thread id 30230 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.411 [hardware/dev_plat/aicpufw/aicpufw_thread.c:536][AICPUFW] [aicpufw_thread_callback 536] sem_wait start, chip_id: 0, ts_ind: 0, aicpu index: 5 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.443 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:181][AICPUFW] [EventThreadTask 181] Aicpu device[0]:thread[ 3] begin, tid: 30228. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.503 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:181][AICPUFW] [EventThreadTask 181] Aicpu device[0]:thread[ 8] begin, tid: 30233. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.550 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:181][AICPUFW] [EventThreadTask 181] Aicpu device[0]:thread[ 9] begin, tid: 30234. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.602 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:181][AICPUFW] [EventThreadTask 181] Aicpu device[0]:thread[ 4] begin, tid: 30229. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.634 [hardware/dev_plat/aicpufw/aicpufw_thread.c:475][AICPUFW] [aicpufw_thread_set_affinity 475] thread_set_affinity, bind_cpu_index 8, chip 0, ts 0, aicpu_index 6, thread id 30231 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.673 [hardware/dev_plat/aicpufw/aicpufw_thread.c:536][AICPUFW] [aicpufw_thread_callback 536] sem_wait start, chip_id: 0, ts_ind: 0, aicpu index: 6 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.690 [hardware/dev_plat/aicpufw/aicpufw_thread.c:538][AICPUFW] [aicpufw_thread_callback 538] sem_wait end, chip_id: 0, ts_ind: 0, aicpu index: 6 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.706 [hardware/dev_plat/aicpufw/aicpufw_thread.c:546][AICPUFW] [aicpufw_thread_callback 546] sem_post start, chip_id: 0, ts_ind: 0, aicpu index: 6 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.721 [hardware/dev_plat/aicpufw/aicpufw_thread.c:548][AICPUFW] [aicpufw_thread_callback 548] sem_post end, chip_id: 0, ts_ind: 0, aicpu index: 6 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.741 [hardware/dev_plat/aicpufw/aicpufw_thread.c:538][AICPUFW] [aicpufw_thread_callback 538] sem_wait end, chip_id: 0, ts_ind: 0, aicpu index: 5 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.762 [hardware/dev_plat/aicpufw/aicpufw_thread.c:546][AICPUFW] [aicpufw_thread_callback 546] sem_post start, chip_id: 0, ts_ind: 0, aicpu index: 5 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.776 [hardware/dev_plat/aicpufw/aicpufw_thread.c:548][AICPUFW] [aicpufw_thread_callback 548] sem_post end, chip_id: 0, ts_ind: 0, aicpu index: 5 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.799 [hardware/dev_plat/aicpufw/aicpufw_thread.c:475][AICPUFW] [aicpufw_thread_set_affinity 475] thread_set_affinity, bind_cpu_index 11, chip 0, ts 0, aicpu_index 9, thread id 30234 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.824 [hardware/dev_plat/aicpufw/aicpufw_thread.c:536][AICPUFW] [aicpufw_thread_callback 536] sem_wait start, chip_id: 0, ts_ind: 0, aicpu index: 9 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.841 [hardware/dev_plat/aicpufw/aicpufw_thread.c:538][AICPUFW] [aicpufw_thread_callback 538] sem_wait end, chip_id: 0, ts_ind: 0, aicpu index: 9 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.855 [hardware/dev_plat/aicpufw/aicpufw_thread.c:546][AICPUFW] [aicpufw_thread_callback 546] sem_post start, chip_id: 0, ts_ind: 0, aicpu index: 9 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.870 [hardware/dev_plat/aicpufw/aicpufw_thread.c:548][AICPUFW] [aicpufw_thread_callback 548] sem_post end, chip_id: 0, ts_ind: 0, aicpu index: 9 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.900 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:181][AICPUFW] [EventThreadTask 181] Aicpu device[0]:thread[ 0] begin, tid: 30225. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.375.963 [hardware/dev_plat/../dev_core/devdrv/devdrv_aicpu.c:449][devdrv] [devdrv_load_kernel_serve_thread 449] thread for load kernel start. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.027 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:181][AICPUFW] [EventThreadTask 181] Aicpu device[0]:thread[11] begin, tid: 30236. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.081 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:181][AICPUFW] [EventThreadTask 181] Aicpu device[0]:thread[12] begin, tid: 30237. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.127 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:181][AICPUFW] [EventThreadTask 181] Aicpu device[0]:thread[ 7] begin, tid: 30232. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.179 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:181][AICPUFW] [EventThreadTask 181] Aicpu device[0]:thread[13] begin, tid: 30238. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.204 [hardware/dev_plat/aicpufw/aicpufw_thread.c:475][AICPUFW] [aicpufw_thread_set_affinity 475] thread_set_affinity, bind_cpu_index 5, chip 0, ts 0, aicpu_index 3, thread id 30228 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.231 [hardware/dev_plat/aicpufw/aicpufw_thread.c:536][AICPUFW] [aicpufw_thread_callback 536] sem_wait start, chip_id: 0, ts_ind: 0, aicpu index: 3 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.247 [hardware/dev_plat/aicpufw/aicpufw_thread.c:538][AICPUFW] [aicpufw_thread_callback 538] sem_wait end, chip_id: 0, ts_ind: 0, aicpu index: 3 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.259 [hardware/dev_plat/aicpufw/aicpufw_thread.c:546][AICPUFW] [aicpufw_thread_callback 546] sem_post start, chip_id: 0, ts_ind: 0, aicpu index: 3 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.273 [hardware/dev_plat/aicpufw/aicpufw_thread.c:548][AICPUFW] [aicpufw_thread_callback 548] sem_post end, chip_id: 0, ts_ind: 0, aicpu index: 3 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.291 [hardware/dev_plat/aicpufw/aicpufw_thread.c:475][AICPUFW] [aicpufw_thread_set_affinity 475] thread_set_affinity, bind_cpu_index 6, chip 0, ts 0, aicpu_index 4, thread id 30229 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.316 [hardware/dev_plat/aicpufw/aicpufw_thread.c:536][AICPUFW] [aicpufw_thread_callback 536] sem_wait start, chip_id: 0, ts_ind: 0, aicpu index: 4 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.332 [hardware/dev_plat/aicpufw/aicpufw_thread.c:538][AICPUFW] [aicpufw_thread_callback 538] sem_wait end, chip_id: 0, ts_ind: 0, aicpu index: 4 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.346 [hardware/dev_plat/aicpufw/aicpufw_thread.c:546][AICPUFW] [aicpufw_thread_callback 546] sem_post start, chip_id: 0, ts_ind: 0, aicpu index: 4 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.362 [hardware/dev_plat/aicpufw/aicpufw_thread.c:548][AICPUFW] [aicpufw_thread_callback 548] sem_post end, chip_id: 0, ts_ind: 0, aicpu index: 4 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.391 [hardware/dev_plat/aicpufw/aicpufw_thread.c:475][AICPUFW] [aicpufw_thread_set_affinity 475] thread_set_affinity, bind_cpu_index 15, chip 0, ts 0, aicpu_index 13, thread id 30238 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.431 [hardware/dev_plat/aicpufw/aicpufw_thread.c:536][AICPUFW] [aicpufw_thread_callback 536] sem_wait start, chip_id: 0, ts_ind: 0, aicpu index: 13 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.447 [hardware/dev_plat/aicpufw/aicpufw_thread.c:538][AICPUFW] [aicpufw_thread_callback 538] sem_wait end, chip_id: 0, ts_ind: 0, aicpu index: 13 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.462 [hardware/dev_plat/aicpufw/aicpufw_thread.c:546][AICPUFW] [aicpufw_thread_callback 546] sem_post start, chip_id: 0, ts_ind: 0, aicpu index: 13 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.477 [hardware/dev_plat/aicpufw/aicpufw_thread.c:548][AICPUFW] [aicpufw_thread_callback 548] sem_post end, chip_id: 0, ts_ind: 0, aicpu index: 13 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.500 [hardware/dev_plat/aicpufw/aicpufw_thread.c:475][AICPUFW] [aicpufw_thread_set_affinity 475] thread_set_affinity, bind_cpu_index 2, chip 0, ts 0, aicpu_index 0, thread id 30225 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.528 [hardware/dev_plat/aicpufw/aicpufw_thread.c:536][AICPUFW] [aicpufw_thread_callback 536] sem_wait start, chip_id: 0, ts_ind: 0, aicpu index: 0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.545 [hardware/dev_plat/aicpufw/aicpufw_thread.c:538][AICPUFW] [aicpufw_thread_callback 538] sem_wait end, chip_id: 0, ts_ind: 0, aicpu index: 0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.559 [hardware/dev_plat/aicpufw/aicpufw_thread.c:546][AICPUFW] [aicpufw_thread_callback 546] sem_post start, chip_id: 0, ts_ind: 0, aicpu index: 0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.578 [hardware/dev_plat/aicpufw/aicpufw_thread.c:548][AICPUFW] [aicpufw_thread_callback 548] sem_post end, chip_id: 0, ts_ind: 0, aicpu index: 0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.604 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:181][AICPUFW] [EventThreadTask 181] Aicpu device[0]:thread[10] begin, tid: 30235. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.657 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:181][AICPUFW] [EventThreadTask 181] Aicpu device[0]:thread[ 2] begin, tid: 30227. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.706 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:181][AICPUFW] [EventThreadTask 181] Aicpu device[0]:thread[ 1] begin, tid: 30226. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.731 [hardware/dev_plat/aicpufw/aicpufw_thread.c:475][AICPUFW] [aicpufw_thread_set_affinity 475] thread_set_affinity, bind_cpu_index 13, chip 0, ts 0, aicpu_index 11, thread id 30236 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.760 [hardware/dev_plat/aicpufw/aicpufw_thread.c:475][AICPUFW] [aicpufw_thread_set_affinity 475] thread_set_affinity, bind_cpu_index 3, chip 0, ts 0, aicpu_index 1, thread id 30226 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.785 [hardware/dev_plat/aicpufw/aicpufw_thread.c:536][AICPUFW] [aicpufw_thread_callback 536] sem_wait start, chip_id: 0, ts_ind: 0, aicpu index: 1 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.801 [hardware/dev_plat/aicpufw/aicpufw_thread.c:538][AICPUFW] [aicpufw_thread_callback 538] sem_wait end, chip_id: 0, ts_ind: 0, aicpu index: 1 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.815 [hardware/dev_plat/aicpufw/aicpufw_thread.c:546][AICPUFW] [aicpufw_thread_callback 546] sem_post start, chip_id: 0, ts_ind: 0, aicpu index: 1 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.830 [hardware/dev_plat/aicpufw/aicpufw_thread.c:548][AICPUFW] [aicpufw_thread_callback 548] sem_post end, chip_id: 0, ts_ind: 0, aicpu index: 1 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.851 [hardware/dev_plat/aicpufw/aicpufw_thread.c:475][AICPUFW] [aicpufw_thread_set_affinity 475] thread_set_affinity, bind_cpu_index 4, chip 0, ts 0, aicpu_index 2, thread id 30227 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.878 [hardware/dev_plat/aicpufw/aicpufw_thread.c:536][AICPUFW] [aicpufw_thread_callback 536] sem_wait start, chip_id: 0, ts_ind: 0, aicpu index: 2 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.894 [hardware/dev_plat/aicpufw/aicpufw_thread.c:538][AICPUFW] [aicpufw_thread_callback 538] sem_wait end, chip_id: 0, ts_ind: 0, aicpu index: 2 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.908 [hardware/dev_plat/aicpufw/aicpufw_thread.c:546][AICPUFW] [aicpufw_thread_callback 546] sem_post start, chip_id: 0, ts_ind: 0, aicpu index: 2 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.925 [hardware/dev_plat/aicpufw/aicpufw_thread.c:548][AICPUFW] [aicpufw_thread_callback 548] sem_post end, chip_id: 0, ts_ind: 0, aicpu index: 2 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.945 [hardware/dev_plat/aicpufw/aicpufw_thread.c:739][AICPUFW] [aicpufw_thread_create 739] sem_wait end chip_id=0, ts_ind=0, aicpu_index0 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.963 [hardware/dev_plat/aicpufw/aicpufw_thread.c:737][AICPUFW] [aicpufw_thread_create 737] sem_wait start chip_id=0, ts_ind=0, aicpu_index1 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.978 [hardware/dev_plat/aicpufw/aicpufw_thread.c:739][AICPUFW] [aicpufw_thread_create 739] sem_wait end chip_id=0, ts_ind=0, aicpu_index1 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.376.993 [hardware/dev_plat/aicpufw/aicpufw_thread.c:737][AICPUFW] [aicpufw_thread_create 737] sem_wait start chip_id=0, ts_ind=0, aicpu_index2 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.008 [hardware/dev_plat/aicpufw/aicpufw_thread.c:739][AICPUFW] [aicpufw_thread_create 739] sem_wait end chip_id=0, ts_ind=0, aicpu_index2 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.023 [hardware/dev_plat/aicpufw/aicpufw_thread.c:737][AICPUFW] [aicpufw_thread_create 737] sem_wait start chip_id=0, ts_ind=0, aicpu_index3 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.039 [hardware/dev_plat/aicpufw/aicpufw_thread.c:739][AICPUFW] [aicpufw_thread_create 739] sem_wait end chip_id=0, ts_ind=0, aicpu_index3 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.054 [hardware/dev_plat/aicpufw/aicpufw_thread.c:737][AICPUFW] [aicpufw_thread_create 737] sem_wait start chip_id=0, ts_ind=0, aicpu_index4 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.070 [hardware/dev_plat/aicpufw/aicpufw_thread.c:739][AICPUFW] [aicpufw_thread_create 739] sem_wait end chip_id=0, ts_ind=0, aicpu_index4 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.085 [hardware/dev_plat/aicpufw/aicpufw_thread.c:737][AICPUFW] [aicpufw_thread_create 737] sem_wait start chip_id=0, ts_ind=0, aicpu_index5 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.099 [hardware/dev_plat/aicpufw/aicpufw_thread.c:739][AICPUFW] [aicpufw_thread_create 739] sem_wait end chip_id=0, ts_ind=0, aicpu_index5 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.114 [hardware/dev_plat/aicpufw/aicpufw_thread.c:737][AICPUFW] [aicpufw_thread_create 737] sem_wait start chip_id=0, ts_ind=0, aicpu_index6 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.130 [hardware/dev_plat/aicpufw/aicpufw_thread.c:739][AICPUFW] [aicpufw_thread_create 739] sem_wait end chip_id=0, ts_ind=0, aicpu_index6 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.145 [hardware/dev_plat/aicpufw/aicpufw_thread.c:737][AICPUFW] [aicpufw_thread_create 737] sem_wait start chip_id=0, ts_ind=0, aicpu_index7 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.168 [hardware/dev_plat/aicpufw/aicpufw_thread.c:475][AICPUFW] [aicpufw_thread_set_affinity 475] thread_set_affinity, bind_cpu_index 9, chip 0, ts 0, aicpu_index 7, thread id 30232 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.195 [hardware/dev_plat/aicpufw/aicpufw_thread.c:536][AICPUFW] [aicpufw_thread_callback 536] sem_wait start, chip_id: 0, ts_ind: 0, aicpu index: 7 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.211 [hardware/dev_plat/aicpufw/aicpufw_thread.c:538][AICPUFW] [aicpufw_thread_callback 538] sem_wait end, chip_id: 0, ts_ind: 0, aicpu index: 7 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.225 [hardware/dev_plat/aicpufw/aicpufw_thread.c:546][AICPUFW] [aicpufw_thread_callback 546] sem_post start, chip_id: 0, ts_ind: 0, aicpu index: 7 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.242 [hardware/dev_plat/aicpufw/aicpufw_thread.c:548][AICPUFW] [aicpufw_thread_callback 548] sem_post end, chip_id: 0, ts_ind: 0, aicpu index: 7 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.261 [hardware/dev_plat/aicpufw/aicpufw_thread.c:475][AICPUFW] [aicpufw_thread_set_affinity 475] thread_set_affinity, bind_cpu_index 10, chip 0, ts 0, aicpu_index 8, thread id 30233 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.284 [hardware/dev_plat/aicpufw/aicpufw_thread.c:536][AICPUFW] [aicpufw_thread_callback 536] sem_wait start, chip_id: 0, ts_ind: 0, aicpu index: 8 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.300 [hardware/dev_plat/aicpufw/aicpufw_thread.c:538][AICPUFW] [aicpufw_thread_callback 538] sem_wait end, chip_id: 0, ts_ind: 0, aicpu index: 8 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.315 [hardware/dev_plat/aicpufw/aicpufw_thread.c:546][AICPUFW] [aicpufw_thread_callback 546] sem_post start, chip_id: 0, ts_ind: 0, aicpu index: 8 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.330 [hardware/dev_plat/aicpufw/aicpufw_thread.c:548][AICPUFW] [aicpufw_thread_callback 548] sem_post end, chip_id: 0, ts_ind: 0, aicpu index: 8 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.351 [hardware/dev_plat/aicpufw/aicpufw_thread.c:475][AICPUFW] [aicpufw_thread_set_affinity 475] thread_set_affinity, bind_cpu_index 12, chip 0, ts 0, aicpu_index 10, thread id 30235 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.377 [hardware/dev_plat/aicpufw/aicpufw_thread.c:536][AICPUFW] [aicpufw_thread_callback 536] sem_wait start, chip_id: 0, ts_ind: 0, aicpu index: 10 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.393 [hardware/dev_plat/aicpufw/aicpufw_thread.c:538][AICPUFW] [aicpufw_thread_callback 538] sem_wait end, chip_id: 0, ts_ind: 0, aicpu index: 10 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.407 [hardware/dev_plat/aicpufw/aicpufw_thread.c:546][AICPUFW] [aicpufw_thread_callback 546] sem_post start, chip_id: 0, ts_ind: 0, aicpu index: 10 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.422 [hardware/dev_plat/aicpufw/aicpufw_thread.c:548][AICPUFW] [aicpufw_thread_callback 548] sem_post end, chip_id: 0, ts_ind: 0, aicpu index: 10 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.441 [hardware/dev_plat/aicpufw/aicpufw_thread.c:475][AICPUFW] [aicpufw_thread_set_affinity 475] thread_set_affinity, bind_cpu_index 14, chip 0, ts 0, aicpu_index 12, thread id 30237 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.465 [hardware/dev_plat/aicpufw/aicpufw_thread.c:536][AICPUFW] [aicpufw_thread_callback 536] sem_wait start, chip_id: 0, ts_ind: 0, aicpu index: 12 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.481 [hardware/dev_plat/aicpufw/aicpufw_thread.c:538][AICPUFW] [aicpufw_thread_callback 538] sem_wait end, chip_id: 0, ts_ind: 0, aicpu index: 12 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.496 [hardware/dev_plat/aicpufw/aicpufw_thread.c:546][AICPUFW] [aicpufw_thread_callback 546] sem_post start, chip_id: 0, ts_ind: 0, aicpu index: 12 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.510 [hardware/dev_plat/aicpufw/aicpufw_thread.c:548][AICPUFW] [aicpufw_thread_callback 548] sem_post end, chip_id: 0, ts_ind: 0, aicpu index: 12 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.527 [hardware/dev_plat/aicpufw/aicpufw_thread.c:739][AICPUFW] [aicpufw_thread_create 739] sem_wait end chip_id=0, ts_ind=0, aicpu_index7 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.543 [hardware/dev_plat/aicpufw/aicpufw_thread.c:737][AICPUFW] [aicpufw_thread_create 737] sem_wait start chip_id=0, ts_ind=0, aicpu_index8 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.558 [hardware/dev_plat/aicpufw/aicpufw_thread.c:739][AICPUFW] [aicpufw_thread_create 739] sem_wait end chip_id=0, ts_ind=0, aicpu_index8 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.572 [hardware/dev_plat/aicpufw/aicpufw_thread.c:737][AICPUFW] [aicpufw_thread_create 737] sem_wait start chip_id=0, ts_ind=0, aicpu_index9 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.587 [hardware/dev_plat/aicpufw/aicpufw_thread.c:739][AICPUFW] [aicpufw_thread_create 739] sem_wait end chip_id=0, ts_ind=0, aicpu_index9 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.602 [hardware/dev_plat/aicpufw/aicpufw_thread.c:737][AICPUFW] [aicpufw_thread_create 737] sem_wait start chip_id=0, ts_ind=0, aicpu_index10 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.618 [hardware/dev_plat/aicpufw/aicpufw_thread.c:739][AICPUFW] [aicpufw_thread_create 739] sem_wait end chip_id=0, ts_ind=0, aicpu_index10 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.633 [hardware/dev_plat/aicpufw/aicpufw_thread.c:737][AICPUFW] [aicpufw_thread_create 737] sem_wait start chip_id=0, ts_ind=0, aicpu_index11 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.652 [hardware/dev_plat/aicpufw/aicpufw_thread.c:536][AICPUFW] [aicpufw_thread_callback 536] sem_wait start, chip_id: 0, ts_ind: 0, aicpu index: 11 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.668 [hardware/dev_plat/aicpufw/aicpufw_thread.c:538][AICPUFW] [aicpufw_thread_callback 538] sem_wait end, chip_id: 0, ts_ind: 0, aicpu index: 11 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.692 [hardware/dev_plat/aicpufw/aicpufw_thread.c:546][AICPUFW] [aicpufw_thread_callback 546] sem_post start, chip_id: 0, ts_ind: 0, aicpu index: 11 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.703 [hardware/dev_plat/aicpufw/aicpufw_thread.c:548][AICPUFW] [aicpufw_thread_callback 548] sem_post end, chip_id: 0, ts_ind: 0, aicpu index: 11 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.713 [hardware/dev_plat/aicpufw/aicpufw_thread.c:739][AICPUFW] [aicpufw_thread_create 739] sem_wait end chip_id=0, ts_ind=0, aicpu_index11 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.725 [hardware/dev_plat/aicpufw/aicpufw_thread.c:737][AICPUFW] [aicpufw_thread_create 737] sem_wait start chip_id=0, ts_ind=0, aicpu_index12 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.734 [hardware/dev_plat/aicpufw/aicpufw_thread.c:739][AICPUFW] [aicpufw_thread_create 739] sem_wait end chip_id=0, ts_ind=0, aicpu_index12 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.742 [hardware/dev_plat/aicpufw/aicpufw_thread.c:737][AICPUFW] [aicpufw_thread_create 737] sem_wait start chip_id=0, ts_ind=0, aicpu_index13 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.749 [hardware/dev_plat/aicpufw/aicpufw_thread.c:739][AICPUFW] [aicpufw_thread_create 739] sem_wait end chip_id=0, ts_ind=0, aicpu_index13 +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.761 [hardware/dev_plat/aicpufw/aicpufw_timer.c:133][AICPUFW] [aicpufw_timer_init 133] aicpufw_timer_init end, AICPU_TASK_TIMEOUT=30. +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.377.775 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.743120] [devdrv] [devdrv_manager_get_kernel_lib_process 198] begin to get kernel lib, device pid: 30223. +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.377.790 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.743123] [devdrv] [devdrv_manager_get_kernel_lib_process 211] host_pid: 40927. +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.377.799 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.743131] [devdrv] [devdrv_manager_get_kernel_lib_process 247] begin to wait, host pid: 40927, device pid: 30223. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.833 [hardware/dev_plat/aicpufw/aicpufw_supervisor_thread.c:173][AICPUFW] [aicpufw_sup_thread_proc 173] supervisor thread begin, current tid:30239 thread +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.854 [hardware/dev_plat/aicpufw/aicpufw_thread.c:797][AICPUFW] [aicpufw_thread_init 797] chip_id 0 thread init end. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.866 [hardware/dev_plat/aicpufw/aicpufw_api.c:77][AICPUFW] [aicpufw_create_work_tasks 77] drvCreateAicpuWorkTasks exit: chip_id = 0, pid=40927, mode=0. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.377.892 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:233][AICPUFW] [StartTdtServer 233] Start tdt server, deviceId=0. +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.377.932 [tdt/device/../common/src/log.cpp:158]BindCpu init and bindCoreList size is 1,[tdt/device/src/hdc/bind_cpu.cpp:38:Init]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.034 [tdt/device/../common/src/log.cpp:158]BindCpu cpu core num = 64,[tdt/device/src/hdc/bind_cpu.cpp:40:Init]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.051 [tdt/device/../common/src/log.cpp:158]BindCoreList member is 1,[tdt/device/src/hdc/bind_cpu.cpp:42:Init]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.075 [tdt/device/../common/src/log.cpp:158]Begin to Init tdtserver [devicID_=0] and [newInputDeviceId=0],[tdt/device/src/hdc/tdt_server_impl.cpp:664:Init]30223 Msg: running ok +[WARNING] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.110 [tdt/device/../common/src/log.cpp:143]{"Start":"TDT_RECV"},[tdt/device/../common/src/statistic.cpp:113:PeriodStatisticManager]30223 Msg: warnging +[WARNING] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.128 [tdt/device/../common/src/log.cpp:143]{"Start":"DP_ENQUEUE"},[tdt/device/../common/src/statistic.cpp:114:PeriodStatisticManager]30223 Msg: warnging +[WARNING] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.147 [tdt/device/../common/src/log.cpp:143]{"Start":"RECV_ENLARGE"},[tdt/device/../common/src/statistic.cpp:115:PeriodStatisticManager]30223 Msg: warnging +[WARNING] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.167 [tdt/device/../common/src/log.cpp:143]{"Start":"RECV_REDUCE"},[tdt/device/../common/src/statistic.cpp:116:PeriodStatisticManager]30223 Msg: warnging +[WARNING] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.185 [tdt/device/../common/src/log.cpp:143]{"Start":"RELEASE_DATA"},[tdt/device/../common/src/statistic.cpp:117:PeriodStatisticManager]30223 Msg: warnging +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.285 [tdt/device/../common/src/log.cpp:158]HdcServer::Init Start,[tdt/device/../common/src/hdc_server.cpp:104:Init]30223 Msg: running ok +[INFO] HDC(30223,aicpu_scheduler):2020-05-12-11:05:22.378.335 [hardware/dev_plat/../dev_plat/devhdc/hdc_server.c:287][drvHdcServerCreate:287] >>> create server (listen device: 0) success +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.356 [tdt/device/../common/src/log.cpp:158]HdcCommon::InitMsgSize Start,[tdt/device/../common/src/hdc_common.cpp:28:InitMsgSize]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.374 [tdt/device/../common/src/log.cpp:158]msgMaxSize_ = 524224, msgShortHeadDataMaxSize_ = 524212 msgLongHeadDataMaxSize_ = 524200,[tdt/device/../common/src/hdc_common.cpp:42:InitMsgSize]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.410 [tdt/device/../common/src/log.cpp:158]hdcserver in tdtserver is already initialed,[tdt/device/src/hdc/tdt_server_impl.cpp:652:InitDirectly]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.428 [tdt/device/../common/src/log.cpp:158]Begin to init device's hdc channel of tdt.,[tdt/device/src/hdc/tdt_server.cpp:32:TDTServerInit]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.448 [tdt/device/../common/src/log.cpp:158]begin tdt device init, [deviceId=0],[tdt/device/src/hdc/tdt_device_impl.cpp:65:Init]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.466 [tdt/device/../common/src/log.cpp:158]TuningDataTransfer is initialized with deviceID:0,[tdt/device/src/hdc/tuning_data_transfer.cpp:71:Init]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.640 [tdt/device/../common/src/log.cpp:158]HdcCommon::InitMsgSize Start,[tdt/device/../common/src/hdc_common.cpp:28:InitMsgSize]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.657 [tdt/device/../common/src/log.cpp:158]msgMaxSize_ = 524224, msgShortHeadDataMaxSize_ = 524212 msgLongHeadDataMaxSize_ = 524200,[tdt/device/../common/src/hdc_common.cpp:42:InitMsgSize]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.673 [tdt/device/../common/src/log.cpp:158]capacity.maxSegment: 524224,[tdt/device/../common/src/hdc_client.cpp:152:Init]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.690 [tdt/device/../common/src/log.cpp:158]HdcClient::CreateHdcSession Start,[tdt/device/../common/src/hdc_client.cpp:284:CreateHdcSession]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.809 [tdt/device/../common/src/log.cpp:158]HdcServer::Accept thread = 281470521594256,[tdt/device/../common/src/hdc_server.cpp:259:Accept]30242 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.878 [tdt/device/../common/src/log.cpp:158][HdcClient] deviceId: 0 connect session,[tdt/device/../common/src/hdc_client.cpp:306:CreateHdcSession]30223 Msg: running ok +[INFO] HDC(30223,aicpu_scheduler):2020-05-12-11:05:22.378.908 [hardware/dev_plat/../dev_plat/devhdc/hdc_core.c:1609][drvHdcSetSessionReference:1609] >>> session 56, pid 30223 +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.926 [tdt/device/../common/src/log.cpp:158][HdcClient] drvHdcSetSessionReference success,[tdt/device/../common/src/hdc_client.cpp:317:CreateHdcSession]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.378.945 [tdt/device/../common/src/log.cpp:158][HdcClient] deviceId: 0 connect session and drvHdcSetSessionReference success, sessionId=1,[tdt/device/../common/src/hdc_client.cpp:327:CreateHdcSession]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.006 [tdt/device/../common/src/log.cpp:158][HdcClient] SendPidMsg sessionId=1, tdt_main_pid=30223,[tdt/device/../common/src/hdc_client.cpp:346:sendPidMsg]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.073 [tdt/device/../common/src/log.cpp:158]TuningDataTransfer tdt client wait for send data begin,[tdt/device/src/hdc/tuning_data_transfer.cpp:173:CreateSingleSession]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.101 [tdt/device/../common/src/log.cpp:158]BindCPUCore Start,[tdt/device/src/hdc/bind_cpu.cpp:58:BindCPUCore]30242 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.213 [tdt/device/../common/src/log.cpp:158]BindCpu thread=281470521594256 CPU_ISSET successfully on processor[1],[tdt/device/src/hdc/bind_cpu.cpp:103:BindCPUCore]30242 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.239 [tdt/device/../common/src/log.cpp:158]Thread[281470521594256] bindCpu setaffinity successfully,[tdt/device/src/hdc/bind_cpu.cpp:109:BindCPUCore]30242 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.282 [tdt/device/../common/src/log.cpp:158]Free Tdt thread ID = 281470529986960,[tdt/device/src/hdc/tdt_server_impl.cpp:555:FreeTdtMemoryThread]30241 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.304 [tdt/device/../common/src/log.cpp:158]BindCPUCore Start,[tdt/device/src/hdc/bind_cpu.cpp:58:BindCPUCore]30241 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.359 [tdt/device/../common/src/log.cpp:158]TimerFunc thread = 281470538379664,[tdt/device/../common/src/statistic.cpp:140:TimerFunc]30240 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.377 [tdt/device/../common/src/log.cpp:158]HdcServer::AcceptConnection Start,[tdt/device/../common/src/hdc_server.cpp:310:AcceptHdcSession]30242 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.403 [tdt/device/../common/src/log.cpp:158]BindCpu thread=281470529986960 CPU_ISSET successfully on processor[1],[tdt/device/src/hdc/bind_cpu.cpp:103:BindCPUCore]30241 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.421 [tdt/device/../common/src/log.cpp:158]BindCPUCore Start,[tdt/device/src/hdc/bind_cpu.cpp:58:BindCPUCore]30240 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.476 [tdt/device/../common/src/log.cpp:158]BindCPUCore Start,[tdt/device/src/hdc/bind_cpu.cpp:58:BindCPUCore]30243 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.496 [tdt/device/../common/src/log.cpp:158]Thread[281470529986960] bindCpu setaffinity successfully,[tdt/device/src/hdc/bind_cpu.cpp:109:BindCPUCore]30241 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.516 [tdt/device/../common/src/log.cpp:158][deviceId=0] Free Tdt thread is success to bind cpu core,[tdt/device/src/hdc/tdt_server_impl.cpp:559:FreeTdtMemoryThread]30241 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.534 [tdt/device/../common/src/log.cpp:158]BindCpu thread=281470538379664 CPU_ISSET successfully on processor[1],[tdt/device/src/hdc/bind_cpu.cpp:103:BindCPUCore]30240 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.550 [tdt/device/../common/src/log.cpp:158]Thread[281470538379664] bindCpu setaffinity successfully,[tdt/device/src/hdc/bind_cpu.cpp:109:BindCPUCore]30240 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.576 [tdt/device/../common/src/log.cpp:158]BindCpu thread=281470513033616 CPU_ISSET successfully on processor[1],[tdt/device/src/hdc/bind_cpu.cpp:103:BindCPUCore]30243 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.591 [tdt/device/../common/src/log.cpp:158]Thread[281470513033616] bindCpu setaffinity successfully,[tdt/device/src/hdc/bind_cpu.cpp:109:BindCPUCore]30243 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.637 [tdt/device/../common/src/log.cpp:158]TuningDataTransfer find channel OK, sessionID: 1,[tdt/device/src/hdc/tuning_data_transfer.cpp:126:SetSendFlagBySessionID]30243 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.664 [tdt/device/../common/src/log.cpp:158]TuningDataTransfer tdt client wait for send data end,[tdt/device/src/hdc/tuning_data_transfer.cpp:178:CreateSingleSession]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.688 [tdt/device/../common/src/log.cpp:158]Begin to start send thread.,[tdt/device/src/hdc/tdt_device_impl.cpp:179:StartSendThread]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.732 [tdt/device/../common/src/log.cpp:158]Success start send thread.,[tdt/device/src/hdc/tdt_device_impl.cpp:181:StartSendThread]30223 Msg: running ok +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.379.746 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:265][AICPUFW] [StartTdtServer 265] TDT server init success. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.379.758 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:135][AICPUFW] [Start 135] Aicpu_scheduler has started, deviceId=0, hostpid=40927. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:22.379.771 [aicpu/aicpu_device/aicpu_schedule/compute_process/main.cc:206][AICPUFW] [main 206] Compute process start success. +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.793 [tdt/device/../common/src/log.cpp:158][TSDPPCCLI] TsdWaitForShutdownProc() deviceId: 0 waitType:1(0:HCCP,1:COMPUTE) is running,[tdt/device/src/tsd/ppc_client.cpp:220:TsdWaitForShutdownProc]30223 Msg: running ok +[EVENT] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.810 [tdt/device/../common/src/log.cpp:149][PPCCLIEVENT] TsdWaitForShutdown!,[tdt/device/src/tsd/ppc_client.cpp:225:TsdWaitForShutdownProc]30223 +[EVENT] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.826 [tdt/device/../common/src/log.cpp:149][PPCCLIEVENT] TsdWaitForShutdown Start Rsp device[0] procType:1(0:HCCP,1:COMPUTE), subProcPid[30223] ,[tdt/device/src/tsd/ppc_client.cpp:244:TsdWaitForShutdownProc]30223 +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.846 [tdt/device/../common/src/log.cpp:158][TSDPPCCLI] GetPpcSession() will Create session use serverId: 19280,[tdt/device/src/tsd/ppc_client.cpp:110:GetPpcSession]30223 Msg: running ok +[INFO] HDC(30223,aicpu_scheduler):2020-05-12-11:05:22.379.899 [hardware/dev_plat/../dev_plat/devhdc/hdc_ppc.c:129][drvPpcSessionConnect:129] >>> Ppc connect session 26, pid 30223 +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.917 [tdt/device/../common/src/log.cpp:158][TSDPPCCLI] [serverId=19280] GetPpcSession() receive thread has been started,[tdt/device/src/tsd/ppc_client.cpp:118:GetPpcSession]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.379.935 [tdt/device/../common/src/log.cpp:158]PpcInterface::SendMsg,size=10, subpid=30223,[tdt/device/src/tsd/ppc_interface.cpp:39:SendMsg]30223 Msg: running ok +[INFO] HDC(8380,tsdaemon):2020-05-12-11:05:22.379.941 [hardware/dev_plat/../dev_plat/devhdc/hdc_ppc.c:268][drvPpcSessionAccept:268] >>> Ppc Accept Session 16, Server fd 12, pid 8380 +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.380.006 [tdt/device/../common/src/log.cpp:158]tdt device begin to send to host.,[tdt/device/src/hdc/tdt_device_impl.cpp:275:TdtDeviceSendImpl]30244 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:22.380.025 [tdt/device/../common/src/log.cpp:158]begin to check queueDataSize_.,[tdt/device/src/hdc/tdt_device_impl.cpp:286:TdtDeviceSendImpl]30244 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.380.091 [tdt/device/../common/src/log.cpp:158]PpcInterface::RecvMsg, size=10, subpid=30223,[tdt/device/src/tsd/ppc_interface.cpp:122:RecvMsg]30245 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.380.114 [tdt/device/../common/src/log.cpp:158][PpcServer] SetPpcSession, deviceId:0, subProcPid:30223,[tdt/device/src/tsd/ppc_server.cpp:338:SetPpcSession]30245 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.380.139 [tdt/device/../common/src/log.cpp:158][TSDaemon]PPCSerToTsdMsg deviceId[0], subProcPid[30223]msgType[0](0:START RSP,2:SHUTDOWN,1:SHUTDOWN RSP,3:SOCKET CLOSE), procType[1](0:HCCP,1:COMPUTE) state[6],[tdt/device/src/tsd/tsdaemon.cpp:1441:PPCSerToTsdMsg]30245 Msg: running ok +[EVENT] TDT(8380,tsdaemon):2020-05-12-11:05:22.380.160 [tdt/device/../common/src/log.cpp:149][TsdEVENT] #### PPCSer->TSD RspMsg device[0] Start Rsp procType[1] ####,[tdt/device/src/tsd/tsdaemon.cpp:1406:PPCSerToTsdProc]30245 +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.380.205 [tdt/device/../common/src/log.cpp:158][TSDPPCSER] ##### [threadName:ppc_srv_recv_0] RecvData [ret=16846848] Save [notifyDeviceId:0] [notifyProcType:1]####,[tdt/device/src/tsd/ppc_server.cpp:242:RecvData]30245 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.380.256 [tdt/device/../common/src/log.cpp:158][TSDaemon] StartRspProc() [dev=0][subProcPid=30223][procType=1(0:HCCP,1:COMPUTE)][tid=281470572192176]!,[tdt/device/src/tsd/tsdaemon.cpp:1190:StartRspProc]30246 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.380.276 [tdt/device/../common/src/log.cpp:158][TSDaemon] StartRspProc curState is: 6,[tdt/device/src/tsd/tsdaemon.cpp:1193:StartRspProc]30246 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.380.295 [tdt/device/../common/src/log.cpp:158][TSDaemon] GetTsdToFmkMsg deviceId[0] subProcPid[30223],[tdt/device/src/tsd/tsdaemon.cpp:959:GetTsdToFmkMsg]30246 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.380.314 [tdt/device/../common/src/log.cpp:158][TSDaemon] tsdToFmkSessionIdMap size = 1,[tdt/device/src/tsd/tsdaemon.cpp:964:GetTsdToFmkMsg]30246 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.380.331 [tdt/device/../common/src/log.cpp:158][TSDaemon] tsdToFmkSessionIdMap [deviceId] size = 1,[tdt/device/src/tsd/tsdaemon.cpp:967:GetTsdToFmkMsg]30246 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.380.349 [tdt/device/../common/src/log.cpp:158][TSDaemon] SendRspMsgToFmk deviceId: 0, subProcPid: 30223, sessionID: 1,[tdt/device/src/tsd/tsdaemon.cpp:1016:SendRspMsgToFmk]30246 Msg: running ok +[EVENT] TDT(8380,tsdaemon):2020-05-12-11:05:22.380.364 [tdt/device/../common/src/log.cpp:149][TsdEVENT] #### Start Rsp TSD->FMK device[0] sessionID[1] realDeviceId[0]####,[tdt/device/src/tsd/tsdaemon.cpp:1018:SendRspMsgToFmk]30246 +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:22.380.402 [tdt/device/../common/src/log.cpp:158][TSDaemon] StartRspProc subRunState is: 3,[tdt/device/src/tsd/tsdaemon.cpp:1173:TsdWaitRspProcForStar]30246 Msg: running ok +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:22.381.752 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48430.745228] [hdcdrv] [hdcdrv_server_create 2330] dev_id 0 service_type service_TDT server create +[EVENT] TDT(30223,aicpu_scheduler):2020-05-12-11:05:23.379.610 [tdt/device/../common/src/log.cpp:149],[tdt/common/common_inc/queue_manager.h:683:ShowSizeForEveryChannel]30240 +[EVENT] TDT(30223,aicpu_scheduler):2020-05-12-11:05:23.379.656 [tdt/device/../common/src/log.cpp:149]"DeviceSendPool: " "DeviceRecvPool: " "HostRecvPool: " "DeviceCtrlPool: {SendPool: 0, FreePool: 0}, {RecvPool: 0, FreePool: 0}",[tdt/device/../common/src/memory_pool.cpp:707:GetDevicePoolStatus]30240 +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:23.379.673 [tdt/device/../common/src/log.cpp:158]DeviceRecNormalData: Device receive normal message number:0,[tdt/device/../common/src/memory_pool.cpp:709:GetDevicePoolStatus]30240 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.063.322 [tdt/device/../common/src/log.cpp:158]tsdaemon get process sign successfully, procpid:0 signSize:0,[tdt/device/src/tsd/tsdaemon.cpp:901:FmkToTsdMsg]30221 Msg: running ok +[EVENT] TDT(8380,tsdaemon):2020-05-12-11:05:24.063.360 [tdt/device/../common/src/log.cpp:149][TsdEVENT]FmkToTsdMsg dev[0] msg[7] sessionId[1] realDev[0] fmkSignPid[0] profilingMode[0] rankSize[1],[tdt/device/src/tsd/tsdaemon.cpp:905:FmkToTsdMsg]30221 +[EVENT] TDT(8380,tsdaemon):2020-05-12-11:05:24.063.375 [tdt/device/../common/src/log.cpp:149][TsdEVENT]From FMK Close <<<<<<<<<<< TSDdev[0] sessionId[1] realDev[0] fmkPid[0],[tdt/device/src/tsd/tsdaemon.cpp:861:FmkToTsdMsgProc]30221 +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.063.397 [tdt/device/../common/src/log.cpp:158][TSDaemon] Begin closeSubProcess deviceId:0, sessionId:1, cpProcPid:30223, hccpProcPid:0, subProcState:3,[tdt/device/src/tsd/tsdaemon.cpp:729:CloseSubProcess]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.063.414 [tdt/device/../common/src/log.cpp:158][TSDaemon] SetTsdToFmkMsg:deviceId[0], sessionId[1], subProcPid[30223],[tdt/device/src/tsd/tsdaemon.cpp:774:SetTsdToFmkMsg]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.063.429 [tdt/device/../common/src/log.cpp:158][TSDaemon] Process HCCP is abandoned to close, the rank size is 1,[tdt/device/src/tsd/tsdaemon.cpp:747:CloseSubProcess]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.063.473 [tdt/device/../common/src/log.cpp:158][TSDaemon] start delete file, direct is /home/HwHiAiUser/hdcd/device0/,[tdt/device/src/tsd/tsdaemon.cpp:1878:DeleteFileByPath]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.063.509 [tdt/device/../common/src/log.cpp:158][TSDaemon] start scan file, ent name is .,[tdt/device/src/tsd/tsdaemon.cpp:1887:DeleteFileByPath]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.063.524 [tdt/device/../common/src/log.cpp:158][TSDaemon] start scan file, ent name is ..,[tdt/device/src/tsd/tsdaemon.cpp:1887:DeleteFileByPath]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.063.541 [tdt/device/../common/src/log.cpp:158][TSDaemon] start scan file, ent name is etc,[tdt/device/src/tsd/tsdaemon.cpp:1887:DeleteFileByPath]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.063.560 [tdt/device/../common/src/log.cpp:158][TSDaemon] start scan file, ent name is upgrade,[tdt/device/src/tsd/tsdaemon.cpp:1887:DeleteFileByPath]30221 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.063.624 [tdt/device/../common/src/log.cpp:158][PpcServer] GetPpcSession, deviceId:0, subProcPid:30223,[tdt/device/src/tsd/ppc_server.cpp:324:GetPpcSession]30255 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.063.648 [tdt/device/../common/src/log.cpp:158]PpcInterface::SendMsg,size=12, subpid=30223,[tdt/device/src/tsd/ppc_interface.cpp:39:SendMsg]30255 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.063.686 [tdt/device/../common/src/log.cpp:158][TSDaemon] Begin ExecuteClose deviceId:0, subProcPid:30223,[tdt/device/src/tsd/tsdaemon.cpp:703:ExecuteClose]30255 Msg: running ok +[OPLOG] TDT(8380,tsdaemon):2020-05-12-11:05:24.063.704 [tdt/device/../common/src/log.cpp:151][tdt/device/src/tsd/tsdaemon.cpp:705:ExecuteClose]30255 free resource {devOS:[30223]} for {dev:0} +[EVENT] TDT(8380,tsdaemon):2020-05-12-11:05:24.063.731 [tdt/device/../common/src/log.cpp:149][TsdEVENT] #### Send TSD->SubProcess[PPCSer] Close Msg Device[0] proType[1] [tid=281470597370288]####,[tdt/device/src/tsd/tsdaemon.cpp:709:ExecuteClose]30255 +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.063.728 [tdt/device/../common/src/log.cpp:158]PpcInterface::RecvMsg, size=12, subpid=30223,[tdt/device/src/tsd/ppc_interface.cpp:122:RecvMsg]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.063.755 [tdt/device/../common/src/log.cpp:158]PpcClient::RecvMsgProc, size=12, subpid=30223,[tdt/device/src/tsd/ppc_client.cpp:150:RecvMsgProc]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.063.773 [tdt/device/../common/src/log.cpp:158]PpcClient::RecvMsgProc, subpid1=30223, subpid2=30223,[tdt/device/src/tsd/ppc_client.cpp:160:RecvMsgProc]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.063.789 [tdt/device/../common/src/log.cpp:158]PpcInterface::SendMsg,size=12, subpid=30223,[tdt/device/src/tsd/ppc_interface.cpp:39:SendMsg]30223 Msg: running ok +[EVENT] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.063.816 [tdt/device/../common/src/log.cpp:149][PPCCLIEVENT] #### PPCClient->TSD Close Rsp send OK device[0] procType:1 ####,[tdt/device/src/tsd/ppc_client.cpp:166:RecvMsgProc]30223 +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.063.832 [tdt/device/../common/src/log.cpp:158][PPCClient]Process Exit DevId:0 procType:1(0:HCCP,1:COMPUTE),[tdt/device/src/tsd/ppc_client.cpp:200:RecvData]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.063.848 [tdt/device/../common/src/log.cpp:158]TsdWaitForShutdown exit,[tdt/device/src/tsd/ppc_client.cpp:274:TsdWaitForShutdown]30223 Msg: running ok +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:24.063.861 [aicpu/aicpu_device/aicpu_schedule/compute_process/main.cc:214][AICPUFW] [main 214] Tsd wait for shut down success. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:24.063.872 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:272][AICPUFW] [StopTdtServer 272] Stop tdt server, deviceId=0. +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.063.987 [tdt/device/../common/src/log.cpp:158]PpcInterface::RecvMsg, size=12, subpid=30223,[tdt/device/src/tsd/ppc_interface.cpp:122:RecvMsg]30245 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.064.025 [tdt/device/../common/src/log.cpp:158][PpcServer] SetPpcSession, deviceId:0, subProcPid:30223,[tdt/device/src/tsd/ppc_server.cpp:338:SetPpcSession]30245 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.064.057 [tdt/device/../common/src/log.cpp:158][TSDaemon]PPCSerToTsdMsg deviceId[0], subProcPid[30223]msgType[1](0:START RSP,2:SHUTDOWN,1:SHUTDOWN RSP,3:SOCKET CLOSE), procType[1](0:HCCP,1:COMPUTE) state[6],[tdt/device/src/tsd/tsdaemon.cpp:1441:PPCSerToTsdMsg]30245 Msg: running ok +[EVENT] TDT(8380,tsdaemon):2020-05-12-11:05:24.064.078 [tdt/device/../common/src/log.cpp:149][TsdEVENT] #### PPCSer->TSD RspMsg device[0] Close Rsp procType[1] ####,[tdt/device/src/tsd/tsdaemon.cpp:1415:PPCSerToTsdProc]30245 +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.064.195 [tdt/device/../common/src/log.cpp:158][TSDPPCSER] ##### [threadName:ppc_srv_recv_0] RecvData [ret=16846848] Save [notifyDeviceId:0] [notifyProcType:1]####,[tdt/device/src/tsd/ppc_server.cpp:242:RecvData]30245 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.064.227 [tdt/device/../common/src/log.cpp:158]tdtserver is destroying, [devicID_=0],[tdt/device/src/hdc/tdt_server_impl.cpp:709:Destroy]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.064.248 [tdt/device/../common/src/log.cpp:158]Stop QueueManager success,[tdt/device/src/hdc/tdt_server_impl.cpp:716:Destroy]30223 Msg: running ok +[INFO] DP(30223,aicpu_scheduler):2020-05-12-11:05:24.064.263 [datapreprocess/src/dp_interface.cc:288][DP_PREPROCESS] [I] [datapreprocess/src/dp_interface.cc:288] Release blocked TDT threads. +[INFO] DP(30223,aicpu_scheduler):2020-05-12-11:05:24.064.280 [datapreprocess/src/dp_interface.cc:406][DP_PREPROCESS] [I] [datapreprocess/src/dp_interface.cc:406] Begin write queue blocking eventfd of source(). +[INFO] DP(30223,aicpu_scheduler):2020-05-12-11:05:24.064.294 [datapreprocess/src/dp_interface.cc:409][DP_PREPROCESS] [I] [datapreprocess/src/dp_interface.cc:409] Got empty source name, start write all blocking eventfd. +[INFO] DP(30223,aicpu_scheduler):2020-05-12-11:05:24.064.305 [datapreprocess/src/dp_interface.cc:291][DP_PREPROCESS] [I] [datapreprocess/src/dp_interface.cc:291] All TDT threads mark info memory have released. +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.064.320 [tdt/device/../common/src/log.cpp:158]DataPreprocess exited,[tdt/device/src/hdc/tdt_server_impl.cpp:720:Destroy]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.064.338 [tdt/device/../common/src/log.cpp:158]Enqueue Thread exited,[tdt/device/src/hdc/tdt_server_impl.cpp:722:Destroy]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.064.368 [tdt/device/../common/src/log.cpp:158][deviceId=0] Free Tdt thread is exist,total free number = 0,enqueue number is = 0,[tdt/device/src/hdc/tdt_server_impl.cpp:570:FreeTdtMemoryThread]30241 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.064.412 [tdt/device/../common/src/log.cpp:158]Free Thread exited,[tdt/device/src/hdc/tdt_server_impl.cpp:725:Destroy]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.064.428 [tdt/device/../common/src/log.cpp:158]enter HdcServer::Destroy() function,[tdt/device/../common/src/hdc_server.cpp:582:Destroy]30223 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.064.462 [tdt/device/../common/src/log.cpp:158][TSDaemon] CloseRspProc() [dev=0][subPid=30223][procType=1(0:HCCP,1:COMPUTE)][tid=281470588977584]!,[tdt/device/src/tsd/tsdaemon.cpp:1267:CloseRspProc]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.064.485 [tdt/device/../common/src/log.cpp:158][TSDaemon] CloseRspProc curState is: 6,[tdt/device/src/tsd/tsdaemon.cpp:1271:CloseRspProc]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.064.508 [tdt/device/../common/src/log.cpp:158][TSDaemon] GetTsdToFmkMsg deviceId[0] subProcPid[30223],[tdt/device/src/tsd/tsdaemon.cpp:959:GetTsdToFmkMsg]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.064.527 [tdt/device/../common/src/log.cpp:158][TSDaemon] tsdToFmkSessionIdMap size = 1,[tdt/device/src/tsd/tsdaemon.cpp:964:GetTsdToFmkMsg]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.064.544 [tdt/device/../common/src/log.cpp:158][TSDaemon] tsdToFmkSessionIdMap [deviceId] size = 1,[tdt/device/src/tsd/tsdaemon.cpp:967:GetTsdToFmkMsg]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.064.568 [tdt/device/../common/src/log.cpp:158][TSDaemon] subProcessPid is 30223 on device[0], procType[1](0:HCCP PROC,1:COMPUTE PROC),[tdt/device/src/tsd/tsdaemon.cpp:1085:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.064.588 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] HDC(30223,aicpu_scheduler):2020-05-12-11:05:24.074.535 [hardware/dev_plat/../dev_plat/devhdc/hdc_server.c:367][drvHdcServerDestroy:367] >>> destroy server success, deviceId 0, serviceType 10 +[INFO] HDC(30223,aicpu_scheduler):2020-05-12-11:05:24.074.556 [hardware/dev_plat/../dev_plat/devhdc/hdc_server.c:236][drvHdcPcieSessionAccept:236] >>> device:0 server 10 is destroyed, ret:18 +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.074.575 [tdt/device/../common/src/log.cpp:158]drv accept exception, because acceptSwitch has been set false, ret=18,[tdt/device/../common/src/hdc_server.cpp:324:AcceptHdcSession]30242 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.074.592 [tdt/device/../common/src/log.cpp:158]acceptSwitch has been set false,[tdt/device/../common/src/hdc_server.cpp:269:Accept]30242 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.074.634 [tdt/device/../common/src/log.cpp:158][HdcServer] SessionIdPidMsg enter into ClearSessionIdPid,[tdt/device/../common/src/hdc_server.cpp:495:ClearSessionIdPid]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.074.658 [tdt/device/../common/src/log.cpp:158]Begin StopMemoryPool,[tdt/device/../common/src/memory_pool.cpp:3389:StopMemoryPool]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.074.685 [tdt/device/../common/src/log.cpp:158]DestroyMemoryPool, memoryEnd = 2,[tdt/device/../common/src/memory_pool.cpp:1323:DestroyMemoryPool]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.074.703 [tdt/device/../common/src/log.cpp:158]FreeMemoryByMap, memoryEnd = 2, memoryType = 1, devId_ = 0,[tdt/device/../common/src/memory_pool.cpp:1131:FreeMemoryByMap]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.074.721 [tdt/device/../common/src/log.cpp:158]FreeMemoryByMap, memoryEnd = 2, memoryType = 1, devId_ = 0,[tdt/device/../common/src/memory_pool.cpp:1131:FreeMemoryByMap]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.074.743 [tdt/device/../common/src/log.cpp:158]hdcServer_ destroyed,[tdt/device/src/hdc/tdt_server_impl.cpp:735:Destroy]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.074.759 [tdt/device/../common/src/log.cpp:158]Begin to destroy device's hdc channel of tdt.,[tdt/device/src/hdc/tdt_server.cpp:48:TDTServerStop]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.074.774 [tdt/device/../common/src/log.cpp:158]begin to destroy.,[tdt/device/src/hdc/tdt_device_impl.cpp:367:Destroy]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.074.793 [tdt/device/../common/src/log.cpp:158]Stop QueueManager success,[tdt/device/src/hdc/tdt_device_impl.cpp:381:Destroy]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.074.808 [tdt/device/../common/src/log.cpp:158]Begin to stop send thread.,[tdt/device/src/hdc/tdt_device_impl.cpp:192:StopSendThread]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.074.866 [tdt/device/../common/src/log.cpp:158]Success stop send thread.,[tdt/device/src/hdc/tdt_device_impl.cpp:196:StopSendThread]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.074.885 [tdt/device/../common/src/log.cpp:158]TuningDataTransfer destory hdc client,[tdt/device/src/hdc/tuning_data_transfer.cpp:456:Destroy]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.074.900 [tdt/device/../common/src/log.cpp:158]enter HdcClient::Destroy() function,[tdt/device/../common/src/hdc_client.cpp:468:Destroy]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.074.917 [tdt/device/../common/src/log.cpp:158]begin drvHdcSessionClose,[tdt/device/../common/src/hdc_client.cpp:415:ClearAllSession]30223 Msg: running ok +[INFO] HDC(30223,aicpu_scheduler):2020-05-12-11:05:24.074.930 [hardware/dev_plat/../dev_plat/devhdc/hdc_client.c:413][drvHdcClientSessionClose:413] >>> destroy client session(sock: 56) +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.074.960 [tdt/device/../common/src/log.cpp:158]end drvHdcSessionClose,[tdt/device/../common/src/hdc_client.cpp:420:ClearAllSession]30223 Msg: running ok +[INFO] HDC(30223,aicpu_scheduler):2020-05-12-11:05:24.074.976 [hardware/dev_plat/../dev_plat/devhdc/hdc_core.c:1300][drvHdcRecvMsgLen:1300] >>> the session 56 local or remote was closed +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.074.996 [tdt/device/../common/src/log.cpp:158]begin HdcClient::JoinAllRecvThread,[tdt/device/../common/src/hdc_client.cpp:383:JoinAllRecvThread]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.016 [tdt/device/../common/src/log.cpp:158]drvHdcRecv() return 25,[tdt/device/../common/src/hdc_common.cpp:494:RecvHdcDefaultMsg]30243 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.075.018 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.038 [tdt/device/../common/src/log.cpp:158]Receive() return 17903651, which means : hdc service or client socket closed,[tdt/device/../common/src/hdc_common.cpp:438:RecvMsg]30243 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.057 [tdt/device/../common/src/log.cpp:158][deviceId=0][sessionId=1] recv runswitch has been set false,[tdt/device/../common/src/hdc_client.cpp:175:RecvData]30243 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.072 [tdt/device/../common/src/log.cpp:158][deviceId=0] the recv data pthread exit,[tdt/device/../common/src/hdc_client.cpp:190:RecvData]30243 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.138 [tdt/device/../common/src/log.cpp:158]end HdcClient::JoinAllRecvThread,[tdt/device/../common/src/hdc_client.cpp:392:JoinAllRecvThread]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.156 [tdt/device/../common/src/log.cpp:158]begin drvHdcClientDestroy,[tdt/device/../common/src/hdc_client.cpp:450:DestroyClient]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.236 [tdt/device/../common/src/log.cpp:158]end drvHdcClientDestroy,[tdt/device/../common/src/hdc_client.cpp:455:DestroyClient]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.252 [tdt/device/../common/src/log.cpp:158]begin HdcClient::ClearClientPtr,[tdt/device/../common/src/hdc_client.cpp:432:ClearClientPtr]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.269 [tdt/device/../common/src/log.cpp:158]end HdcClient::ClearClientPtr,[tdt/device/../common/src/hdc_client.cpp:440:ClearClientPtr]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.284 [tdt/device/../common/src/log.cpp:158]begin HdcClient::ClearAll,[tdt/device/../common/src/hdc_client.cpp:401:ClearAll]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.299 [tdt/device/../common/src/log.cpp:158]end HdcClient::ClearAll,[tdt/device/../common/src/hdc_client.cpp:404:ClearAll]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.314 [tdt/device/../common/src/log.cpp:158]end HdcClient::Destroy() function,[tdt/device/../common/src/hdc_client.cpp:476:Destroy]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.330 [tdt/device/../common/src/log.cpp:158]success destroy.,[tdt/device/src/hdc/tdt_device_impl.cpp:388:Destroy]30223 Msg: running ok +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:24.075.343 [aicpu/aicpu_device/aicpu_schedule/compute_process/compute_process.cc:277][AICPUFW] [StopTdtServer 277] TDT server stop success. +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:24.075.355 [hardware/dev_plat/aicpufw/aicpufw_dev.c:94][AICPUFW] [aicpufw_dev_close 94] chip_id:0 will be closed, fd=18. +[TRACE] DP(30223,aicpu_scheduler):2020-05-12-11:05:24.075.373 [status:STOP] [datapreprocess/src/task_queue.cc:292]DP_PREPROCESS module has been closed +[INFO] DRV(30223,aicpu_scheduler):2020-05-12-11:05:24.075.389 [aicpu/aicpu_device/aicpu_schedule/compute_process/main.cc:219][AICPUFW] [main 219] Compute process stopped. +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.408 [tdt/device/../common/src/log.cpp:158]PpcClient::~PpcClient() destructor function called,[tdt/device/src/tsd/ppc_client.cpp:23:~PpcClient]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.422 [tdt/device/../common/src/log.cpp:158][TSDPPCCLI] Destroy() enter,[tdt/device/src/tsd/ppc_client.cpp:44:Destroy]30223 Msg: running ok +[INFO] HDC(30223,aicpu_scheduler):2020-05-12-11:05:24.075.437 [hardware/dev_plat/../dev_plat/devhdc/hdc_ppc.c:151][drvPpcSessionDestroy:151] >>> Ppc Destroy session 26, pid 30223 +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.473 [tdt/device/../common/src/log.cpp:158][TSDPPCCLI] Destroy() call drvPpcSessionDestroy func return [drvRet:0],[tdt/device/src/tsd/ppc_client.cpp:54:Destroy]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.490 [tdt/device/../common/src/log.cpp:158][TSDPPCCLI] Destroy() exit,[tdt/device/src/tsd/ppc_client.cpp:73:Destroy]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.508 [tdt/device/../common/src/log.cpp:158]TdtDeviceImpl::~TdtDeviceImpl() destructor function called.,[tdt/device/src/hdc/tdt_device_impl.cpp:54:~TdtDeviceImpl]30223 Msg: running ok +[INFO] TDT(30223,aicpu_scheduler):2020-05-12-11:05:24.075.532 [tdt/device/../common/src/log.cpp:158]TdtServerImpl::~TdtServerImpl() destructor function called,[tdt/device/src/hdc/tdt_server_impl.cpp:71:~TdtServerImpl]30223 Msg: running ok +[INFO] HDC(30223,aicpu_scheduler):2020-05-12-11:05:24.076.267 [hardware/dev_plat/../dev_plat/devhdc/hdc_core.c:2002][drv_hdc_exit:2002] >>> HDC uninit success +[INFO] HDC(8380,tsdaemon):2020-05-12-11:05:24.077.353 [hardware/dev_plat/../dev_plat/devhdc/hdc_core.c:626][hdcSocketRecvPeek:626] >>> client connection closed: Success(errno: 0)(sock: 16) +[WARNING] TDT(8380,tsdaemon):2020-05-12-11:05:24.077.393 [tdt/device/../common/src/log.cpp:143][TSDPPCIF] drvPpcRecv fail 25,[tdt/device/src/tsd/ppc_interface.cpp:92:RecvMsg]30245 Msg: warnging +[EVENT] TDT(8380,tsdaemon):2020-05-12-11:05:24.077.412 [tdt/device/../common/src/log.cpp:149][TSDPPCSER] 0 SOCKET_CLOSED notify dev[0] procType[1] ret=[17379389]?[17379389]TSD to clean,[tdt/device/src/tsd/ppc_server.cpp:199:RecvData]30245 +[EVENT] TDT(8380,tsdaemon):2020-05-12-11:05:24.077.429 [tdt/device/../common/src/log.cpp:149][TSDPPCSER] 1 SOCKET_CLOSED notify dev[0] procType[1] ret[17379389] TSD to clean,[tdt/device/src/tsd/ppc_server.cpp:204:RecvData]30245 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.077.804 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.441417] [hdcdrv] [hdcdrv_server_destroy 2415] dev_id 0 service_type service_TDT server destroy +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.077.829 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.441427] [hdcdrv] [hdcdrv_accept 2470] dev_id 0 service_type service_TDT accept wait dev 0 quit, dev status 1, listen status 0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.077.841 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.441858] [hdcdrv] [hdcdrv_recv_peek 2899] dev 0 session 56 local or remote close, local_close_state closed_by_user, remote_close_state closed_by_user,local_session_fd 56, remote_session_fd 227. +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.077.850 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.444353] [devdrv] [devdrv_manager_get_kernel_lib_process 251] wait_event_interruptible return: -512. +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.077.860 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.444495] [devmm] [devmm_notifier_release_private 1225] device wait ts exit hostpid(40927) exit(0). +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.085.102 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.095.180 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.105.257 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.115.334 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.125.408 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.133.771 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.496591] [devmm] [devmm_notifier_release_private 1231] ts exited,device hostpid(40927) begin recover resource. +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.137.750 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.147.847 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.157.927 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.168.004 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.178.083 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.188.161 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.198.238 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.208.316 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.218.391 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.228.469 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.238.544 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.248.619 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.258.697 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.268.776 [tdt/device/../common/src/log.cpp:158]The current kill result is [0],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] DRV(8314,aicpufw_monitor):2020-05-12-11:05:24.275.875 [hardware/dev_plat/aicpufw/aicpufw_thread.c:931][AICPUFW] [aicpufw_monitor_recycle_so 931] recycle so pid_dir=/home/HwHiAiUser/tmp/30223/. +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.277.856 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.642610] [aicpufw-drv] [aicpufw_drv_release_debug 650] 0 :rev_int_cnt 0 rev_int_ok 0, rev_int_invalid 0,send_to_int_cnt 0 wait_satisfied 0 wait_event_time 0 rev_int_pending 0. dev_id:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.277.878 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.642614] [aicpufw-drv] [aicpufw_drv_release_debug 650] 1 :rev_int_cnt 0 rev_int_ok 0, rev_int_invalid 0,send_to_int_cnt 0 wait_satisfied 0 wait_event_time 0 rev_int_pending 0. dev_id:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.277.889 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.642616] [aicpufw-drv] [aicpufw_drv_release_debug 650] 2 :rev_int_cnt 0 rev_int_ok 0, rev_int_invalid 0,send_to_int_cnt 0 wait_satisfied 0 wait_event_time 0 rev_int_pending 0. dev_id:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.277.899 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.642618] [aicpufw-drv] [aicpufw_drv_release_debug 650] 3 :rev_int_cnt 0 rev_int_ok 0, rev_int_invalid 0,send_to_int_cnt 0 wait_satisfied 0 wait_event_time 0 rev_int_pending 0. dev_id:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.277.915 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.642620] [aicpufw-drv] [aicpufw_drv_release_debug 650] 4 :rev_int_cnt 0 rev_int_ok 0, rev_int_invalid 0,send_to_int_cnt 0 wait_satisfied 0 wait_event_time 0 rev_int_pending 0. dev_id:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.277.924 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.642622] [aicpufw-drv] [aicpufw_drv_release_debug 650] 5 :rev_int_cnt 0 rev_int_ok 0, rev_int_invalid 0,send_to_int_cnt 0 wait_satisfied 0 wait_event_time 0 rev_int_pending 0. dev_id:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.277.933 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.642624] [aicpufw-drv] [aicpufw_drv_release_debug 650] 6 :rev_int_cnt 0 rev_int_ok 0, rev_int_invalid 0,send_to_int_cnt 0 wait_satisfied 0 wait_event_time 0 rev_int_pending 0. dev_id:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.277.942 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.642627] [aicpufw-drv] [aicpufw_drv_release_debug 650] 7 :rev_int_cnt 0 rev_int_ok 0, rev_int_invalid 0,send_to_int_cnt 0 wait_satisfied 0 wait_event_time 0 rev_int_pending 0. dev_id:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.277.950 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.642628] [aicpufw-drv] [aicpufw_drv_release_debug 650] 8 :rev_int_cnt 0 rev_int_ok 0, rev_int_invalid 0,send_to_int_cnt 0 wait_satisfied 0 wait_event_time 0 rev_int_pending 0. dev_id:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.277.959 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.642630] [aicpufw-drv] [aicpufw_drv_release_debug 650] 9 :rev_int_cnt 0 rev_int_ok 0, rev_int_invalid 0,send_to_int_cnt 0 wait_satisfied 0 wait_event_time 0 rev_int_pending 0. dev_id:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.277.967 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.642632] [aicpufw-drv] [aicpufw_drv_release_debug 650] 10 :rev_int_cnt 0 rev_int_ok 0, rev_int_invalid 0,send_to_int_cnt 0 wait_satisfied 0 wait_event_time 0 rev_int_pending 0. dev_id:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.277.975 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.642634] [aicpufw-drv] [aicpufw_drv_release_debug 650] 11 :rev_int_cnt 0 rev_int_ok 0, rev_int_invalid 0,send_to_int_cnt 0 wait_satisfied 0 wait_event_time 0 rev_int_pending 0. dev_id:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.277.984 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.642636] [aicpufw-drv] [aicpufw_drv_release_debug 650] 12 :rev_int_cnt 0 rev_int_ok 0, rev_int_invalid 0,send_to_int_cnt 0 wait_satisfied 0 wait_event_time 0 rev_int_pending 0. dev_id:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.277.994 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.642638] [aicpufw-drv] [aicpufw_drv_release_debug 650] 13 :rev_int_cnt 0 rev_int_ok 0, rev_int_invalid 0,send_to_int_cnt 0 wait_satisfied 0 wait_event_time 0 rev_int_pending 0. dev_id:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.278.003 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.642641] [aicpufw-drv] [aicpufw_drv_delete_context 568] delete match-pid(40927). dev_id:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.278.011 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.642642] [aicpufw-drv] [aicpufw_drv_release 690] processes(1),process pid(30223) released.current tgid: 30223 numa node:0. dev_id:0 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.278.019 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.642657] [aicpufw-drv] [aicpufw_drv_get_moniter_info 1583] aicpufw event happened. dev_id:0 +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.278.860 [tdt/device/../common/src/log.cpp:158]The current kill result is [-1],[tdt/device/src/tsd/tsdaemon.cpp:1090:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.278.880 [tdt/device/../common/src/log.cpp:158][TSDaemon] Computer Process stop success.,[tdt/device/src/tsd/tsdaemon.cpp:1096:CheckSubProcessExitByType]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.278.892 [tdt/device/../common/src/log.cpp:158][TSDaemon] SubProcesses exit success on device[0], tryTimes is 21,[tdt/device/src/tsd/tsdaemon.cpp:1098:CheckSubProcessExitByType]30256 Msg: running ok +[EVENT] TDT(8380,tsdaemon):2020-05-12-11:05:24.278.908 [tdt/device/../common/src/log.cpp:149][TsdEVENT] #### Close Rsp TSD->FMK device[0] sessionID[1] realDeviceId[0]####,[tdt/device/src/tsd/tsdaemon.cpp:1023:SendRspMsgToFmk]30256 +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.278.950 [tdt/device/../common/src/log.cpp:158][TSDaemon] CloseRspProc subRunState is: 0,[tdt/device/src/tsd/tsdaemon.cpp:1248:TsdWaitRspProcForClose]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.278.966 [tdt/device/../common/src/log.cpp:158][TSDaemon] EraseTsdToFmkMsg:deviceId[0], subProcPid[30223],[tdt/device/src/tsd/tsdaemon.cpp:789:EraseTsdToFmkMsg]30256 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.279.027 [tdt/device/../common/src/log.cpp:158][TSDaemon]###### PPCSerToTsdAbnormalMsg deviceId[0] msgType[3](0:START RSP,1:SHUTDOWN,2:SHUTDOWN RSP,3:SOCKET CLOSE), procType[1](0:HCCP,1:COMPUTE), state[0],[tdt/device/src/tsd/tsdaemon.cpp:1471:PPCSerToTsdAbnormalMsg]30245 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.279.057 [tdt/device/../common/src/log.cpp:158][TSDaemon] Begin to ClearPPCThreadCleanFlag,[tdt/device/src/tsd/tsdaemon.cpp:1473:PPCSerToTsdAbnormalMsg]30245 Msg: running ok +[EVENT] TDT(8380,tsdaemon):2020-05-12-11:05:24.279.081 [tdt/device/../common/src/log.cpp:149][TSDPPCSER] 2 SOCKET_CLOSED notify dev[0] procType[1] ret[17379389] TSD to clean,[tdt/device/src/tsd/ppc_server.cpp:211:RecvData]30245 +[INFO] HDC(8380,tsdaemon):2020-05-12-11:05:24.279.117 [hardware/dev_plat/../dev_plat/devhdc/hdc_ppc.c:304][drvPpcSessionClose:304] >>> Ppc Close session fd 16, pid 8380 session 0xfffee0000d30 +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.279.139 [tdt/device/../common/src/log.cpp:158][TSDPPCSER] CloseSomeSession enter [sessionSize:1],[tdt/device/src/tsd/ppc_server.cpp:80:RemoveFromPpcSessionList]30245 Msg: running ok +[INFO] TDT(8380,tsdaemon):2020-05-12-11:05:24.279.157 [tdt/device/../common/src/log.cpp:158][TSDPPCSER] CloseSomeSession exit [sessionSize:0],[tdt/device/src/tsd/ppc_server.cpp:85:RemoveFromPpcSessionList]30245 Msg: running ok +[EVENT] TDT(8380,tsdaemon):2020-05-12-11:05:24.279.190 [tdt/device/../common/src/log.cpp:149][TSDPPCSER] [threadName:ppc_srv_recv_0] RecvData [ret=17379389] [tid=281470580584880]thread exit,[tdt/device/src/tsd/ppc_server.cpp:218:RecvData]30245 +[INFO] KERNEL(8189,sklogd):2020-05-12-11:05:24.609.779 [toolchain/log/slog/sklog/device/../src/klogd.c:249][48432.973818] [devmm] [devmm_chan_close_device_h2d 1124] device process exited, hostpid=40927, devpid=30223, devid=0. diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/data/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/data/file_io.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/data/file_io.py new file mode 100644 index 0000000..0f7f3b9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/data/file_io.py @@ -0,0 +1,207 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Convenience functions for managing dataset file buffers.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import atexit +import multiprocessing +import multiprocessing.dummy +import os +import tempfile +import uuid + +import numpy as np +import six + +import tensorflow as tf + + +class _GarbageCollector(object): + """Deletes temporary buffer files at exit. + + Certain tasks (such as NCF Recommendation) require writing buffers to + temporary files. (Which may be local or distributed.) It is not generally safe + to delete these files during operation, but they should be cleaned up. This + class keeps track of temporary files created, and deletes them at exit. + """ + def __init__(self): + self.temp_buffers = [] + + def register(self, filepath): + self.temp_buffers.append(filepath) + + def purge(self): + try: + for i in self.temp_buffers: + if tf.io.gfile.exists(i): + tf.io.gfile.remove(i) + tf.compat.v1.logging.info("Buffer file {} removed".format(i)) + except Exception as e: + tf.compat.v1.logging.error("Failed to cleanup buffer files: {}".format(e)) + + +_GARBAGE_COLLECTOR = _GarbageCollector() +atexit.register(_GARBAGE_COLLECTOR.purge) + +_ROWS_PER_CORE = 50000 + + +def write_to_temp_buffer(dataframe, buffer_folder, columns): + if buffer_folder is None: + _, buffer_path = tempfile.mkstemp() + else: + tf.io.gfile.makedirs(buffer_folder) + buffer_path = os.path.join(buffer_folder, str(uuid.uuid4())) + _GARBAGE_COLLECTOR.register(buffer_path) + + return write_to_buffer(dataframe, buffer_path, columns) + + +def iter_shard_dataframe(df, rows_per_core=1000): + """Two way shard of a dataframe. + + This function evenly shards a dataframe so that it can be mapped efficiently. + It yields a list of dataframes with length equal to the number of CPU cores, + with each dataframe having rows_per_core rows. (Except for the last batch + which may have fewer rows in the dataframes.) Passing vectorized inputs to + a pool is more effecient than iterating through a dataframe in serial and + passing a list of inputs to the pool. + + Args: + df: Pandas dataframe to be sharded. + rows_per_core: Number of rows in each shard. + + Returns: + A list of dataframe shards. + """ + n = len(df) + num_cores = min([multiprocessing.cpu_count(), n]) + + num_blocks = int(np.ceil(n / num_cores / rows_per_core)) + max_batch_size = num_cores * rows_per_core + for i in range(num_blocks): + min_index = i * max_batch_size + max_index = min([(i + 1) * max_batch_size, n]) + df_shard = df[min_index:max_index] + n_shard = len(df_shard) + boundaries = np.linspace(0, n_shard, num_cores + 1, dtype=np.int64) + yield [df_shard[boundaries[j]:boundaries[j+1]] for j in range(num_cores)] + + +def _shard_dict_to_examples(shard_dict): + """Converts a dict of arrays into a list of example bytes.""" + n = [i for i in shard_dict.values()][0].shape[0] + feature_list = [{} for _ in range(n)] + for column, values in shard_dict.items(): + if len(values.shape) == 1: + values = np.reshape(values, values.shape + (1,)) + + if values.dtype.kind == "i": + feature_map = lambda x: tf.train.Feature( + int64_list=tf.train.Int64List(value=x)) + elif values.dtype.kind == "f": + feature_map = lambda x: tf.train.Feature( + float_list=tf.train.FloatList(value=x)) + else: + raise ValueError("Invalid dtype") + for i in range(n): + feature_list[i][column] = feature_map(values[i]) + examples = [ + tf.train.Example(features=tf.train.Features(feature=example_features)) + for example_features in feature_list + ] + + return [e.SerializeToString() for e in examples] + + +def _serialize_shards(df_shards, columns, pool, writer): + """Map sharded dataframes to bytes, and write them to a buffer. + + Args: + df_shards: A list of pandas dataframes. (Should be of similar size) + columns: The dataframe columns to be serialized. + pool: A pool to serialize in parallel. + writer: A TFRecordWriter to write the serialized shards. + """ + # Pandas does not store columns of arrays as nd arrays. stack remedies this. + map_inputs = [{c: np.stack(shard[c].values, axis=0) for c in columns} + for shard in df_shards] + + # Failure within pools is very irksome. Thus, it is better to thoroughly check + # inputs in the main process. + for inp in map_inputs: + # Check that all fields have the same number of rows. + assert len(set([v.shape[0] for v in inp.values()])) == 1 + for val in inp.values(): + assert hasattr(val, "dtype") + assert hasattr(val.dtype, "kind") + assert val.dtype.kind in ("i", "f") + assert len(val.shape) in (1, 2) + shard_bytes = pool.map(_shard_dict_to_examples, map_inputs) + for s in shard_bytes: + for example in s: + writer.write(example) + + +def write_to_buffer(dataframe, buffer_path, columns, expected_size=None): + """Write a dataframe to a binary file for a dataset to consume. + + Args: + dataframe: The pandas dataframe to be serialized. + buffer_path: The path where the serialized results will be written. + columns: The dataframe columns to be serialized. + expected_size: The size in bytes of the serialized results. This is used to + lazily construct the buffer. + + Returns: + The path of the buffer. + """ + if (tf.io.gfile.exists(buffer_path) and + tf.io.gfile.stat(buffer_path).length > 0): + actual_size = tf.io.gfile.stat(buffer_path).length + if expected_size == actual_size: + return buffer_path + tf.compat.v1.logging.warning( + "Existing buffer {} has size {}. Expected size {}. Deleting and " + "rebuilding buffer.".format(buffer_path, actual_size, expected_size)) + tf.io.gfile.remove(buffer_path) + + if dataframe is None: + raise ValueError( + "dataframe was None but a valid existing buffer was not found.") + + tf.io.gfile.makedirs(os.path.split(buffer_path)[0]) + + tf.compat.v1.logging.info("Constructing TFRecordDataset buffer: {}" + .format(buffer_path)) + + count = 0 + pool = multiprocessing.dummy.Pool(multiprocessing.cpu_count()) + try: + with tf.io.TFRecordWriter(buffer_path) as writer: + for df_shards in iter_shard_dataframe(df=dataframe, + rows_per_core=_ROWS_PER_CORE): + _serialize_shards(df_shards, columns, pool, writer) + count += sum([len(s) for s in df_shards]) + tf.compat.v1.logging.info("{}/{} examples written." + .format(str(count).ljust(8), len(dataframe))) + finally: + pool.terminate() + + tf.compat.v1.logging.info("Buffer write complete.") + return buffer_path diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/data/file_io_test.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/data/file_io_test.py new file mode 100644 index 0000000..ba90d94 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/data/file_io_test.py @@ -0,0 +1,199 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for binary data file utilities.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import contextlib +import multiprocessing + +# pylint: disable=wrong-import-order +import numpy as np +import pandas as pd +import tensorflow as tf +# pylint: enable=wrong-import-order + +from official.r1.utils.data import file_io +from official.utils.misc import keras_utils + + +_RAW_ROW = "raw_row" +_DUMMY_COL = "column_0" +_DUMMY_VEC_COL = "column_1" +_DUMMY_VEC_LEN = 4 + +_ROWS_PER_CORE = 4 +_TEST_CASES = [ + # One batch of one + dict(row_count=1, cpu_count=1, expected=[ + [[0]] + ]), + + dict(row_count=10, cpu_count=1, expected=[ + [[0, 1, 2, 3]], [[4, 5, 6, 7]], [[8, 9]] + ]), + + dict(row_count=21, cpu_count=1, expected=[ + [[0, 1, 2, 3]], [[4, 5, 6, 7]], [[8, 9, 10, 11]], + [[12, 13, 14, 15]], [[16, 17, 18, 19]], [[20]] + ]), + + dict(row_count=1, cpu_count=4, expected=[ + [[0]] + ]), + + dict(row_count=10, cpu_count=4, expected=[ + [[0, 1], [2, 3, 4], [5, 6], [7, 8, 9]] + ]), + + dict(row_count=21, cpu_count=4, expected=[ + [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11], [12, 13, 14, 15]], + [[16], [17], [18], [19, 20]] + ]), + + dict(row_count=10, cpu_count=8, expected=[ + [[0], [1], [2], [3, 4], [5], [6], [7], [8, 9]] + ]), + + dict(row_count=40, cpu_count=8, expected=[ + [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11], [12, 13, 14, 15], + [16, 17, 18, 19], [20, 21, 22, 23], [24, 25, 26, 27], + [28, 29, 30, 31]], + [[32], [33], [34], [35], [36], [37], [38], [39]] + ]), +] + +_FEATURE_MAP = { + _RAW_ROW: tf.io.FixedLenFeature([1], dtype=tf.int64), + _DUMMY_COL: tf.io.FixedLenFeature([1], dtype=tf.int64), + _DUMMY_VEC_COL: tf.io.FixedLenFeature([_DUMMY_VEC_LEN], dtype=tf.float32) +} + + +@contextlib.contextmanager +def fixed_core_count(cpu_count): + """Override CPU count. + + file_io.py uses the cpu_count function to scale to the size of the instance. + However, this is not desirable for testing because it can make the test flaky. + Instead, this context manager fixes the count for more robust testing. + + Args: + cpu_count: How many cores multiprocessing claims to have. + + Yields: + Nothing. (for context manager only) + """ + old_count_fn = multiprocessing.cpu_count + multiprocessing.cpu_count = lambda: cpu_count + yield + multiprocessing.cpu_count = old_count_fn + + +class BaseTest(tf.test.TestCase): + + def setUp(self): + super(BaseTest, self).setUp() + if keras_utils.is_v2_0: + tf.compat.v1.disable_eager_execution() + + def _test_sharding(self, row_count, cpu_count, expected): + df = pd.DataFrame({_DUMMY_COL: list(range(row_count))}) + with fixed_core_count(cpu_count): + shards = list(file_io.iter_shard_dataframe(df, _ROWS_PER_CORE)) + result = [[j[_DUMMY_COL].tolist() for j in i] for i in shards] + self.assertAllEqual(expected, result) + + def test_tiny_rows_low_core(self): + self._test_sharding(**_TEST_CASES[0]) + + def test_small_rows_low_core(self): + self._test_sharding(**_TEST_CASES[1]) + + def test_large_rows_low_core(self): + self._test_sharding(**_TEST_CASES[2]) + + def test_tiny_rows_medium_core(self): + self._test_sharding(**_TEST_CASES[3]) + + def test_small_rows_medium_core(self): + self._test_sharding(**_TEST_CASES[4]) + + def test_large_rows_medium_core(self): + self._test_sharding(**_TEST_CASES[5]) + + def test_small_rows_large_core(self): + self._test_sharding(**_TEST_CASES[6]) + + def test_large_rows_large_core(self): + self._test_sharding(**_TEST_CASES[7]) + + def _serialize_deserialize(self, num_cores=1, num_rows=20): + np.random.seed(1) + df = pd.DataFrame({ + # Serialization order is only deterministic for num_cores=1. raw_row is + # used in validation after the deserialization. + _RAW_ROW: np.array(range(num_rows), dtype=np.int64), + _DUMMY_COL: np.random.randint(0, 35, size=(num_rows,)), + _DUMMY_VEC_COL: [ + np.array([np.random.random() for _ in range(_DUMMY_VEC_LEN)]) + for i in range(num_rows) # pylint: disable=unused-variable + ] + }) + + with fixed_core_count(num_cores): + buffer_path = file_io.write_to_temp_buffer( + df, self.get_temp_dir(), [_RAW_ROW, _DUMMY_COL, _DUMMY_VEC_COL]) + + with self.session(graph=tf.Graph()) as sess: + dataset = tf.data.TFRecordDataset(buffer_path) + dataset = dataset.batch(1).map( + lambda x: tf.io.parse_example(serialized=x, features=_FEATURE_MAP)) + + data_iter = tf.compat.v1.data.make_one_shot_iterator(dataset) + seen_rows = set() + for i in range(num_rows+5): + row = data_iter.get_next() + try: + row_id, val_0, val_1 = sess.run( + [row[_RAW_ROW], row[_DUMMY_COL], row[_DUMMY_VEC_COL]]) + row_id, val_0, val_1 = row_id[0][0], val_0[0][0], val_1[0] + assert row_id not in seen_rows + seen_rows.add(row_id) + + self.assertEqual(val_0, df[_DUMMY_COL][row_id]) + self.assertAllClose(val_1, df[_DUMMY_VEC_COL][row_id]) + + self.assertLess(i, num_rows, msg="Too many rows.") + except tf.errors.OutOfRangeError: + self.assertGreaterEqual(i, num_rows, msg="Too few rows.") + + file_io._GARBAGE_COLLECTOR.purge() + assert not tf.io.gfile.exists(buffer_path) + + def test_serialize_deserialize_0(self): + self._serialize_deserialize(num_cores=1) + + def test_serialize_deserialize_1(self): + self._serialize_deserialize(num_cores=2) + + def test_serialize_deserialize_2(self): + self._serialize_deserialize(num_cores=8) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/export.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/export.py new file mode 100644 index 0000000..8061c28 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/export.py @@ -0,0 +1,49 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Convenience functions for exporting models as SavedModels or other types.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + + +def build_tensor_serving_input_receiver_fn(shape, dtype=tf.float32, + batch_size=1): + """Returns a input_receiver_fn that can be used during serving. + + This expects examples to come through as float tensors, and simply + wraps them as TensorServingInputReceivers. + + Arguably, this should live in tf.estimator.export. Testing here first. + + Args: + shape: list representing target size of a single example. + dtype: the expected datatype for the input example + batch_size: number of input tensors that will be passed for prediction + + Returns: + A function that itself returns a TensorServingInputReceiver. + """ + def serving_input_receiver_fn(): + # Prep a placeholder where the input example will be fed in + features = tf.compat.v1.placeholder( + dtype=dtype, shape=[batch_size] + shape, name='input_tensor') + + return tf.estimator.export.TensorServingInputReceiver( + features=features, receiver_tensors=features) + + return serving_input_receiver_fn diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/export_test.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/export_test.py new file mode 100644 index 0000000..3785edd --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/export_test.py @@ -0,0 +1,63 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for exporting utils.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.r1.utils import export + + +class ExportUtilsTest(tf.test.TestCase): + """Tests for the ExportUtils.""" + + def test_build_tensor_serving_input_receiver_fn(self): + receiver_fn = export.build_tensor_serving_input_receiver_fn(shape=[4, 5]) + with tf.Graph().as_default(): + receiver = receiver_fn() + self.assertIsInstance( + receiver, tf.estimator.export.TensorServingInputReceiver) + + self.assertIsInstance(receiver.features, tf.Tensor) + self.assertEqual(receiver.features.shape, tf.TensorShape([1, 4, 5])) + self.assertEqual(receiver.features.dtype, tf.float32) + self.assertIsInstance(receiver.receiver_tensors, dict) + # Note that Python 3 can no longer index .values() directly; cast to list. + self.assertEqual(list(receiver.receiver_tensors.values())[0].shape, + tf.TensorShape([1, 4, 5])) + + def test_build_tensor_serving_input_receiver_fn_batch_dtype(self): + receiver_fn = export.build_tensor_serving_input_receiver_fn( + shape=[4, 5], dtype=tf.int8, batch_size=10) + + with tf.Graph().as_default(): + receiver = receiver_fn() + self.assertIsInstance( + receiver, tf.estimator.export.TensorServingInputReceiver) + + self.assertIsInstance(receiver.features, tf.Tensor) + self.assertEqual(receiver.features.shape, tf.TensorShape([10, 4, 5])) + self.assertEqual(receiver.features.dtype, tf.int8) + self.assertIsInstance(receiver.receiver_tensors, dict) + # Note that Python 3 can no longer index .values() directly; cast to list. + self.assertEqual(list(receiver.receiver_tensors.values())[0].shape, + tf.TensorShape([10, 4, 5])) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/tpu.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/tpu.py new file mode 100644 index 0000000..737a794 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/tpu.py @@ -0,0 +1,116 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functions specific to running TensorFlow on TPUs.""" + +import tensorflow as tf + + +# "local" is a magic word in the TPU cluster resolver; it informs the resolver +# to use the local CPU as the compute device. This is useful for testing and +# debugging; the code flow is ostensibly identical, but without the need to +# actually have a TPU on the other end. +LOCAL = "local" + + +def construct_scalar_host_call(metric_dict, model_dir, prefix=""): + """Construct a host call to log scalars when training on TPU. + + Args: + metric_dict: A dict of the tensors to be logged. + model_dir: The location to write the summary. + prefix: The prefix (if any) to prepend to the metric names. + + Returns: + A tuple of (function, args_to_be_passed_to_said_function) + """ + # type: (dict, str) -> (function, list) + metric_names = list(metric_dict.keys()) + + def host_call_fn(global_step, *args): + """Training host call. Creates scalar summaries for training metrics. + + This function is executed on the CPU and should not directly reference + any Tensors in the rest of the `model_fn`. To pass Tensors from the + model to the `metric_fn`, provide as part of the `host_call`. See + https://www.tensorflow.org/api_docs/python/tf/contrib/tpu/TPUEstimatorSpec + for more information. + + Arguments should match the list of `Tensor` objects passed as the second + element in the tuple passed to `host_call`. + + Args: + global_step: `Tensor with shape `[batch]` for the global_step + *args: Remaining tensors to log. + + Returns: + List of summary ops to run on the CPU host. + """ + step = global_step[0] + with tf.compat.v1.summary.create_file_writer( + logdir=model_dir, filename_suffix=".host_call").as_default(): + with tf.compat.v1.summary.always_record_summaries(): + for i, name in enumerate(metric_names): + tf.compat.v1.summary.scalar(prefix + name, args[i][0], step=step) + + return tf.compat.v1.summary.all_summary_ops() + + # To log the current learning rate, and gradient norm for Tensorboard, the + # summary op needs to be run on the host CPU via host_call. host_call + # expects [batch_size, ...] Tensors, thus reshape to introduce a batch + # dimension. These Tensors are implicitly concatenated to + # [params['batch_size']]. + global_step_tensor = tf.reshape( + tf.compat.v1.train.get_or_create_global_step(), [1]) + other_tensors = [tf.reshape(metric_dict[key], [1]) for key in metric_names] + + return host_call_fn, [global_step_tensor] + other_tensors + + +def embedding_matmul(embedding_table, values, mask, name="embedding_matmul"): + """Performs embedding lookup via a matmul. + + The matrix to be multiplied by the embedding table Tensor is constructed + via an implementation of scatter based on broadcasting embedding indices + and performing an equality comparison against a broadcasted + range(num_embedding_table_rows). All masked positions will produce an + embedding vector of zeros. + + Args: + embedding_table: Tensor of embedding table. + Rank 2 (table_size x embedding dim) + values: Tensor of embedding indices. Rank 2 (batch x n_indices) + mask: Tensor of mask / weights. Rank 2 (batch x n_indices) + name: Optional name scope for created ops + + Returns: + Rank 3 tensor of embedding vectors. + """ + + with tf.name_scope(name): + n_embeddings = embedding_table.get_shape().as_list()[0] + batch_size, padded_size = values.shape.as_list() + + emb_idcs = tf.tile( + tf.reshape(values, (batch_size, padded_size, 1)), (1, 1, n_embeddings)) + emb_weights = tf.tile( + tf.reshape(mask, (batch_size, padded_size, 1)), (1, 1, n_embeddings)) + col_idcs = tf.tile( + tf.reshape(tf.range(n_embeddings), (1, 1, n_embeddings)), + (batch_size, padded_size, 1)) + one_hot = tf.where( + tf.equal(emb_idcs, col_idcs), emb_weights, + tf.zeros((batch_size, padded_size, n_embeddings))) + + return tf.tensordot(one_hot, embedding_table, 1) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/tpu_test.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/tpu_test.py new file mode 100644 index 0000000..ba5b868 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/r1/utils/tpu_test.py @@ -0,0 +1,108 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Test TPU optimized matmul embedding.""" + +import numpy as np +import tensorflow as tf + +from official.r1.utils import tpu as tpu_utils + + +TEST_CASES = [ + dict(embedding_dim=256, vocab_size=1000, sequence_length=64, + batch_size=32, seed=54131), + dict(embedding_dim=8, vocab_size=15, sequence_length=12, + batch_size=256, seed=536413), + dict(embedding_dim=2048, vocab_size=512, sequence_length=50, + batch_size=8, seed=35124) +] + + +class TPUBaseTester(tf.test.TestCase): + def construct_embedding_and_values(self, embedding_dim, vocab_size, + sequence_length, batch_size, seed): + np.random.seed(seed) + + embeddings = np.random.random(size=(vocab_size, embedding_dim)) + embedding_table = tf.convert_to_tensor(value=embeddings, dtype=tf.float32) + + tokens = np.random.randint(low=1, high=vocab_size-1, + size=(batch_size, sequence_length)) + for i in range(batch_size): + tokens[i, np.random.randint(low=0, high=sequence_length-1):] = 0 + values = tf.convert_to_tensor(value=tokens, dtype=tf.int32) + mask = tf.cast(tf.not_equal(values, 0), dtype=tf.float32) + return embedding_table, values, mask + + def _test_embedding(self, embedding_dim, vocab_size, + sequence_length, batch_size, seed): + """Test that matmul embedding matches embedding lookup (gather).""" + + with self.test_session(): + embedding_table, values, mask = self.construct_embedding_and_values( + embedding_dim=embedding_dim, + vocab_size=vocab_size, + sequence_length=sequence_length, + batch_size=batch_size, + seed=seed + ) + + embedding = (tf.nn.embedding_lookup(params=embedding_table, ids=values) * + tf.expand_dims(mask, -1)) + + matmul_embedding = tpu_utils.embedding_matmul( + embedding_table=embedding_table, values=values, mask=mask) + + self.assertAllClose(embedding, matmul_embedding) + + def _test_masking(self, embedding_dim, vocab_size, + sequence_length, batch_size, seed): + """Test that matmul embedding properly zeros masked positions.""" + with self.test_session(): + embedding_table, values, mask = self.construct_embedding_and_values( + embedding_dim=embedding_dim, + vocab_size=vocab_size, + sequence_length=sequence_length, + batch_size=batch_size, + seed=seed + ) + + matmul_embedding = tpu_utils.embedding_matmul( + embedding_table=embedding_table, values=values, mask=mask) + + self.assertAllClose(matmul_embedding, + matmul_embedding * tf.expand_dims(mask, -1)) + + def test_embedding_0(self): + self._test_embedding(**TEST_CASES[0]) + + def test_embedding_1(self): + self._test_embedding(**TEST_CASES[1]) + + def test_embedding_2(self): + self._test_embedding(**TEST_CASES[2]) + + def test_masking_0(self): + self._test_masking(**TEST_CASES[0]) + + def test_masking_1(self): + self._test_masking(**TEST_CASES[1]) + + def test_masking_2(self): + self._test_masking(**TEST_CASES[2]) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/requirements.txt b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/requirements.txt new file mode 100644 index 0000000..3e30f54 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/requirements.txt @@ -0,0 +1,24 @@ +six +google-api-python-client>=1.6.7 +google-cloud-bigquery>=0.31.0 +kaggle>=1.3.9 +mlperf_compliance==0.0.10 +numpy>=1.15.4 +oauth2client>=4.1.2 +pandas>=0.22.0 +psutil>=5.4.3 +py-cpuinfo>=3.3.0 +scipy>=0.19.1 +tensorflow-hub>=0.6.0 +tensorflow-model-optimization>=0.2.1 +tensorflow_datasets +dataclasses +gin-config +typing +sentencepiece +Cython +matplotlib +opencv-python-headless +pyyaml +Pillow +-e git+https://github.com/cocodataset/cocoapi#egg=pycocotools&subdirectory=PythonAPI diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/README.md b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/README.md new file mode 100644 index 0000000..18160f7 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/README.md @@ -0,0 +1,97 @@ +# Adding Abseil (absl) flags quickstart +## Defining a flag +absl flag definitions are similar to argparse, although they are defined on a global namespace. + +For instance defining a string flag looks like: +```$xslt +from absl import flags +flags.DEFINE_string( + name="my_flag", + default="a_sensible_default", + help="Here is what this flag does." +) +``` + +All three arguments are required, but default may be `None`. A common optional argument is +short_name for defining abreviations. Certain `DEFINE_*` methods will have other required arguments. +For instance `DEFINE_enum` requires the `enum_values` argument to be specified. + +## Key Flags +absl has the concept of a key flag. Any flag defined in `__main__` is considered a key flag by +default. Key flags are displayed in `--help`, others only appear in `--helpfull`. In order to +handle key flags that are defined outside the module in question, absl provides the +`flags.adopt_module_key_flags()` method. This adds the key flags of a different module to one's own +key flags. For example: +```$xslt +File: flag_source.py +--------------------------------------- + +from absl import flags +flags.DEFINE_string(name="my_flag", default="abc", help="a flag.") +``` + +```$xslt +File: my_module.py +--------------------------------------- + +from absl import app as absl_app +from absl import flags + +import flag_source + +flags.adopt_module_key_flags(flag_source) + +def main(_): + pass + +absl_app.run(main, [__file__, "-h"] +``` + +when `my_module.py` is run it will show the help text for `my_flag`. Because not all flags defined +in a file are equally important, `official/utils/flags/core.py` (generally imported as flags_core) +provides an abstraction for handling key flag declaration in an easy way through the +`register_key_flags_in_core()` function, which allows a module to make a single +`adopt_key_flags(flags_core)` call when using the util flag declaration functions. + +## Validators +Often the constraints on a flag are complicated. absl provides the validator decorator to allow +one to mark a function as a flag validation function. Suppose we want users to provide a flag +which is a palindrome. + +```$xslt +from absl import flags + +flags.DEFINE_string(name="pal_flag", short_name="pf", default="", help="Give me a palindrome") + +@flags.validator("pal_flag") +def _check_pal(provided_pal_flag): + return provided_pal_flag == provided_pal_flag[::-1] + +``` + +Validators take the form that returning True (truthy) passes, and all others +(False, None, exception) fail. + +## Testing +To test using absl, simply declare flags in the setupClass method of TensorFlow's TestCase. + +```$xslt +from absl import flags +import tensorflow as tf + +def define_flags(): + flags.DEFINE_string(name="test_flag", default="abc", help="an example flag") + + +class BaseTester(unittest.TestCase): + + @classmethod + def setUpClass(cls): + super(BaseTester, cls).setUpClass() + define_flags() + + def test_trivial(self): + flags_core.parse_flags([__file__, "test_flag", "def"]) + self.AssertEqual(flags.FLAGS.test_flag, "def") + +``` diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_base.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_base.py new file mode 100644 index 0000000..fe1eeed --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_base.py @@ -0,0 +1,163 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Flags which will be nearly universal across models.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import flags +import tensorflow as tf + +from official.utils.flags._conventions import help_wrap +from official.utils.logs import hooks_helper + +############## npu modify begin ############# +from hccl.manage.api import get_rank_size +from hccl.manage.api import get_rank_id +############## npu modify end ############### + +def define_base(data_dir=True, model_dir=True, clean=False, train_epochs=False, + epochs_between_evals=False, stop_threshold=False, + batch_size=True, num_gpu=False, hooks=False, export_dir=False, + distribution_strategy=False, run_eagerly=False): + """Register base flags. + + Args: + data_dir: Create a flag for specifying the input data directory. + model_dir: Create a flag for specifying the model file directory. + clean: Create a flag for removing the model_dir. + train_epochs: Create a flag to specify the number of training epochs. + epochs_between_evals: Create a flag to specify the frequency of testing. + stop_threshold: Create a flag to specify a threshold accuracy or other + eval metric which should trigger the end of training. + batch_size: Create a flag to specify the batch size. + num_gpu: Create a flag to specify the number of GPUs used. + hooks: Create a flag to specify hooks for logging. + export_dir: Create a flag to specify where a SavedModel should be exported. + distribution_strategy: Create a flag to specify which Distribution Strategy + to use. + run_eagerly: Create a flag to specify to run eagerly op by op. + Returns: + A list of flags for core.py to marks as key flags. + """ + key_flags = [] + + if data_dir: + flags.DEFINE_string( + name="data_dir", short_name="dd", default="/tmp", + help=help_wrap("The location of the input data.")) + key_flags.append("data_dir") + + if model_dir: + flags.DEFINE_string( + name="model_dir", short_name="md", default="/tmp", + help=help_wrap("The location of the model checkpoint files.")) + key_flags.append("model_dir") + + if clean: + flags.DEFINE_boolean( + name="clean", default=False, + help=help_wrap("If set, model_dir will be removed if it exists.")) + key_flags.append("clean") + + if train_epochs: + flags.DEFINE_integer( + name="train_epochs", short_name="te", default=1, + help=help_wrap("The number of epochs used to train.")) + key_flags.append("train_epochs") + + if epochs_between_evals: + flags.DEFINE_integer( + name="epochs_between_evals", short_name="ebe", default=1, + help=help_wrap("The number of training epochs to run between " + "evaluations.")) + key_flags.append("epochs_between_evals") + + if stop_threshold: + flags.DEFINE_float( + name="stop_threshold", short_name="st", + default=None, + help=help_wrap("If passed, training will stop at the earlier of " + "train_epochs and when the evaluation metric is " + "greater than or equal to stop_threshold.")) + + if batch_size: + flags.DEFINE_integer( + name="batch_size", short_name="bs", default=32, + help=help_wrap("Batch size for training and evaluation. When using " + "multiple gpus, this is the global batch size for " + "all devices. For example, if the batch size is 32 " + "and there are 4 GPUs, each GPU will get 8 examples on " + "each step.")) + key_flags.append("batch_size") + + if num_gpu: + flags.DEFINE_integer( + name="num_gpus", short_name="ng", + default=1, + help=help_wrap( + "How many GPUs to use at each worker with the " + "DistributionStrategies API. The default is 1.")) + + if run_eagerly: + flags.DEFINE_boolean( + name="run_eagerly", default=False, + help="Run the model op by op without building a model function.") + + if hooks: + # Construct a pretty summary of hooks. + hook_list_str = ( + u"\ufeff Hook:\n" + u"\n".join([u"\ufeff {}".format(key) for key + in hooks_helper.HOOKS])) + flags.DEFINE_list( + name="hooks", short_name="hk", default="LoggingTensorHook", + help=help_wrap( + u"A list of (case insensitive) strings to specify the names of " + u"training hooks.\n{}\n\ufeff Example: `--hooks ProfilerHook," + u"ExamplesPerSecondHook`\n See official.utils.logs.hooks_helper " + u"for details.".format(hook_list_str)) + ) + key_flags.append("hooks") + + if export_dir: + flags.DEFINE_string( + name="export_dir", short_name="ed", default=None, + help=help_wrap("If set, a SavedModel serialization of the model will " + "be exported to this directory at the end of training. " + "See the README for more details and relevant links.") + ) + key_flags.append("export_dir") + + if distribution_strategy: + flags.DEFINE_string( + name="distribution_strategy", short_name="ds", default="mirrored", + help=help_wrap("The Distribution Strategy to use for training. " + "Accepted values are 'off', 'one_device', " + "'mirrored', 'parameter_server', 'collective', " + "case insensitive. 'off' means not to use " + "Distribution Strategy; 'default' means to choose " + "from `MirroredStrategy` or `OneDeviceStrategy` " + "according to the number of GPUs.") + ) + + + return key_flags + + +def get_num_gpus(flags_obj): + """get the num npus using hccl api""" + ############## npu modify begin ############# + return get_rank_size() + ############## npu modify end ############### \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_benchmark.py new file mode 100644 index 0000000..eddae80 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_benchmark.py @@ -0,0 +1,109 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Flags for benchmarking models.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import flags + +from official.utils.flags._conventions import help_wrap + + +def define_log_steps(): + flags.DEFINE_integer( + name="log_steps", default=100, + help="Frequency with which to log timing information with TimeHistory.") + + return [] + + +def define_benchmark(benchmark_log_dir=True, bigquery_uploader=True): + """Register benchmarking flags. + + Args: + benchmark_log_dir: Create a flag to specify location for benchmark logging. + bigquery_uploader: Create flags for uploading results to BigQuery. + + Returns: + A list of flags for core.py to marks as key flags. + """ + + key_flags = [] + + flags.DEFINE_enum( + name="benchmark_logger_type", default="BaseBenchmarkLogger", + enum_values=["BaseBenchmarkLogger", "BenchmarkFileLogger", + "BenchmarkBigQueryLogger"], + help=help_wrap("The type of benchmark logger to use. Defaults to using " + "BaseBenchmarkLogger which logs to STDOUT. Different " + "loggers will require other flags to be able to work.")) + flags.DEFINE_string( + name="benchmark_test_id", short_name="bti", default=None, + help=help_wrap("The unique test ID of the benchmark run. It could be the " + "combination of key parameters. It is hardware " + "independent and could be used compare the performance " + "between different test runs. This flag is designed for " + "human consumption, and does not have any impact within " + "the system.")) + + define_log_steps() + + if benchmark_log_dir: + flags.DEFINE_string( + name="benchmark_log_dir", short_name="bld", default=None, + help=help_wrap("The location of the benchmark logging.") + ) + + if bigquery_uploader: + flags.DEFINE_string( + name="gcp_project", short_name="gp", default=None, + help=help_wrap( + "The GCP project name where the benchmark will be uploaded.")) + + flags.DEFINE_string( + name="bigquery_data_set", short_name="bds", default="test_benchmark", + help=help_wrap( + "The Bigquery dataset name where the benchmark will be uploaded.")) + + flags.DEFINE_string( + name="bigquery_run_table", short_name="brt", default="benchmark_run", + help=help_wrap("The Bigquery table name where the benchmark run " + "information will be uploaded.")) + + flags.DEFINE_string( + name="bigquery_run_status_table", short_name="brst", + default="benchmark_run_status", + help=help_wrap("The Bigquery table name where the benchmark run " + "status information will be uploaded.")) + + flags.DEFINE_string( + name="bigquery_metric_table", short_name="bmt", + default="benchmark_metric", + help=help_wrap("The Bigquery table name where the benchmark metric " + "information will be uploaded.")) + + @flags.multi_flags_validator( + ["benchmark_logger_type", "benchmark_log_dir"], + message="--benchmark_logger_type=BenchmarkFileLogger will require " + "--benchmark_log_dir being set") + def _check_benchmark_log_dir(flags_dict): + benchmark_logger_type = flags_dict["benchmark_logger_type"] + if benchmark_logger_type == "BenchmarkFileLogger": + return flags_dict["benchmark_log_dir"] + return True + + return key_flags diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_conventions.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_conventions.py new file mode 100644 index 0000000..81ad21b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_conventions.py @@ -0,0 +1,54 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Central location for shared argparse convention definitions.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys +import codecs +import functools + +from absl import app as absl_app +from absl import flags + + +# This codifies help string conventions and makes it easy to update them if +# necessary. Currently the only major effect is that help bodies start on the +# line after flags are listed. All flag definitions should wrap the text bodies +# with help wrap when calling DEFINE_*. +_help_wrap = functools.partial(flags.text_wrap, length=80, indent="", + firstline_indent="\n") + + +# Pretty formatting causes issues when utf-8 is not installed on a system. +def _stdout_utf8(): + try: + codecs.lookup("utf-8") + except LookupError: + return False + return sys.stdout.encoding == "UTF-8" + + +if _stdout_utf8(): + help_wrap = _help_wrap +else: + def help_wrap(text, *args, **kwargs): + return _help_wrap(text, *args, **kwargs).replace(u"\ufeff", u"") + + +# Replace None with h to also allow -h +absl_app.HelpshortFlag.SHORT_NAME = "h" diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_device.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_device.py new file mode 100644 index 0000000..edaf2f9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_device.py @@ -0,0 +1,85 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Flags for managing compute devices. Currently only contains TPU flags.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import flags +import tensorflow as tf + +from official.utils.flags._conventions import help_wrap + + +def require_cloud_storage(flag_names): + """Register a validator to check directory flags. + Args: + flag_names: An iterable of strings containing the names of flags to be + checked. + """ + msg = "TPU requires GCS path for {}".format(", ".join(flag_names)) + @flags.multi_flags_validator(["tpu"] + flag_names, message=msg) + def _path_check(flag_values): # pylint: disable=missing-docstring + if flag_values["tpu"] is None: + return True + + valid_flags = True + for key in flag_names: + if not flag_values[key].startswith("gs://"): + tf.compat.v1.logging.error("{} must be a GCS path.".format(key)) + valid_flags = False + + return valid_flags + + +def define_device(tpu=True): + """Register device specific flags. + Args: + tpu: Create flags to specify TPU operation. + Returns: + A list of flags for core.py to marks as key flags. + """ + + key_flags = [] + + if tpu: + flags.DEFINE_string( + name="tpu", default=None, + help=help_wrap( + "The Cloud TPU to use for training. This should be either the name " + "used when creating the Cloud TPU, or a " + "grpc://ip.address.of.tpu:8470 url. Passing `local` will use the" + "CPU of the local instance instead. (Good for debugging.)")) + key_flags.append("tpu") + + flags.DEFINE_string( + name="tpu_zone", default=None, + help=help_wrap( + "[Optional] GCE zone where the Cloud TPU is located in. If not " + "specified, we will attempt to automatically detect the GCE " + "project from metadata.")) + + flags.DEFINE_string( + name="tpu_gcp_project", default=None, + help=help_wrap( + "[Optional] Project name for the Cloud TPU-enabled project. If not " + "specified, we will attempt to automatically detect the GCE " + "project from metadata.")) + + flags.DEFINE_integer(name="num_tpu_shards", default=8, + help=help_wrap("Number of shards (TPU chips).")) + + return key_flags diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_distribution.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_distribution.py new file mode 100644 index 0000000..ca331bf --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_distribution.py @@ -0,0 +1,54 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Flags related to distributed execution.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import flags +import tensorflow as tf + +from official.utils.flags._conventions import help_wrap + + +def define_distribution(worker_hosts=True, task_index=True): + """Register distributed execution flags. + + Args: + worker_hosts: Create a flag for specifying comma-separated list of workers. + task_index: Create a flag for specifying index of task. + + Returns: + A list of flags for core.py to marks as key flags. + """ + key_flags = [] + + if worker_hosts: + flags.DEFINE_string( + name='worker_hosts', default=None, + help=help_wrap( + 'Comma-separated list of worker ip:port pairs for running ' + 'multi-worker models with DistributionStrategy. The user would ' + 'start the program on each host with identical value for this ' + 'flag.')) + + if task_index: + flags.DEFINE_integer( + name='task_index', default=-1, + help=help_wrap('If multi-worker training, the task_index of this ' + 'worker.')) + + return key_flags diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_misc.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_misc.py new file mode 100644 index 0000000..c6fa24b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_misc.py @@ -0,0 +1,50 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Misc flags.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import flags + +from official.utils.flags._conventions import help_wrap + + +def define_image(data_format=True): + """Register image specific flags. + + Args: + data_format: Create a flag to specify image axis convention. + + Returns: + A list of flags for core.py to marks as key flags. + """ + + key_flags = [] + + if data_format: + flags.DEFINE_enum( + name="data_format", short_name="df", default=None, + enum_values=["channels_first", "channels_last"], + help=help_wrap( + "A flag to override the data format used in the model. " + "channels_first provides a performance boost on GPU but is not " + "always compatible with CPU. If left unspecified, the data format " + "will be chosen automatically based on whether TensorFlow was " + "built for CPU or GPU.")) + key_flags.append("data_format") + + return key_flags diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_performance.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_performance.py new file mode 100644 index 0000000..cc5840f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/_performance.py @@ -0,0 +1,289 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Register flags for optimizing performance.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import multiprocessing + +from absl import flags # pylint: disable=g-bad-import-order +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.utils.flags._conventions import help_wrap + + +# Map string to TensorFlow dtype +DTYPE_MAP = { + "fp16": tf.float16, + "bf16": tf.bfloat16, + "fp32": tf.float32, +} + + +def get_tf_dtype(flags_obj): + if getattr(flags_obj, "fp16_implementation", None) == "graph_rewrite": + # If the graph_rewrite is used, we build the graph with fp32, and let the + # graph rewrite change ops to fp16. + return tf.float32 + return DTYPE_MAP[flags_obj.dtype] + + +def get_loss_scale(flags_obj, default_for_fp16): + dtype = get_tf_dtype(flags_obj) + if flags_obj.loss_scale == "dynamic": + return flags_obj.loss_scale + elif flags_obj.loss_scale is not None: + return float(flags_obj.loss_scale) + elif dtype == tf.float32 or dtype == tf.bfloat16: + return 1 # No loss scaling is needed for fp32 + else: + assert dtype == tf.float16 + return default_for_fp16 + + +def define_performance(num_parallel_calls=False, inter_op=False, intra_op=False, + synthetic_data=False, max_train_steps=False, dtype=False, + all_reduce_alg=False, num_packs=False, + tf_gpu_thread_mode=False, + datasets_num_private_threads=False, + datasets_num_parallel_batches=False, + dynamic_loss_scale=False, fp16_implementation=False, + loss_scale=False, + tf_data_experimental_slack=False, enable_xla=False, + training_dataset_cache=False): + """Register flags for specifying performance tuning arguments. + + Args: + num_parallel_calls: Create a flag to specify parallelism of data loading. + inter_op: Create a flag to allow specification of inter op threads. + intra_op: Create a flag to allow specification of intra op threads. + synthetic_data: Create a flag to allow the use of synthetic data. + max_train_steps: Create a flags to allow specification of maximum number + of training steps + dtype: Create flags for specifying dtype. + all_reduce_alg: If set forces a specific algorithm for multi-gpu. + num_packs: If set provides number of packs for MirroredStrategy's cross + device ops. + tf_gpu_thread_mode: gpu_private triggers us of private thread pool. + datasets_num_private_threads: Number of private threads for datasets. + datasets_num_parallel_batches: Determines how many batches to process in + parallel when using map and batch from tf.data. + dynamic_loss_scale: Allow the "loss_scale" flag to take on the value + "dynamic". Only valid if `dtype` is True. + fp16_implementation: Create fp16_implementation flag. + loss_scale: Controls the loss scaling, normally for mixed-precision + training. Can only be turned on if dtype is also True. + tf_data_experimental_slack: Determines whether to enable tf.data's + `experimental_slack` option. + enable_xla: Determines if XLA (auto clustering) is turned on. + training_dataset_cache: Whether to cache the training dataset on workers. + Typically used to improve training performance when training data is in + remote storage and can fit into worker memory. + + Returns: + A list of flags for core.py to marks as key flags. + """ + + key_flags = [] + if num_parallel_calls: + flags.DEFINE_integer( + name="num_parallel_calls", short_name="npc", + default=multiprocessing.cpu_count(), + help=help_wrap("The number of records that are processed in parallel " + "during input processing. This can be optimized per " + "data set but for generally homogeneous data sets, " + "should be approximately the number of available CPU " + "cores. (default behavior)")) + + if inter_op: + flags.DEFINE_integer( + name="inter_op_parallelism_threads", short_name="inter", default=0, + help=help_wrap("Number of inter_op_parallelism_threads to use for CPU. " + "See TensorFlow config.proto for details.") + ) + + if intra_op: + flags.DEFINE_integer( + name="intra_op_parallelism_threads", short_name="intra", default=0, + help=help_wrap("Number of intra_op_parallelism_threads to use for CPU. " + "See TensorFlow config.proto for details.")) + + if synthetic_data: + flags.DEFINE_bool( + name="use_synthetic_data", short_name="synth", default=False, + help=help_wrap( + "If set, use fake data (zeroes) instead of a real dataset. " + "This mode is useful for performance debugging, as it removes " + "input processing steps, but will not learn anything.")) + + if max_train_steps: + flags.DEFINE_integer( + name="max_train_steps", short_name="mts", default=None, help=help_wrap( + "The model will stop training if the global_step reaches this " + "value. If not set, training will run until the specified number " + "of epochs have run as usual. It is generally recommended to set " + "--train_epochs=1 when using this flag." + )) + + if dtype: + flags.DEFINE_enum( + name="dtype", short_name="dt", default="fp32", + enum_values=DTYPE_MAP.keys(), + help=help_wrap("The TensorFlow datatype used for calculations. " + "Variables may be cast to a higher precision on a " + "case-by-case basis for numerical stability.")) + + loss_scale_help_text = ( + "The amount to scale the loss by when the model is run. {}. Before " + "gradients are computed, the loss is multiplied by the loss scale, " + "making all gradients loss_scale times larger. To adjust for this, " + "gradients are divided by the loss scale before being applied to " + "variables. This is mathematically equivalent to training without " + "a loss scale, but the loss scale helps avoid some intermediate " + "gradients from underflowing to zero. If not provided the default " + "for fp16 is 128 and 1 for all other dtypes.{}" + ) + if dynamic_loss_scale: + loss_scale_help_text = loss_scale_help_text.format( + "This can be an int/float or the string 'dynamic'", + " The string 'dynamic' can be used to dynamically determine the " + "optimal loss scale during training, but currently this " + "significantly slows down performance") + loss_scale_validation_msg = ("loss_scale should be a positive int/float " + "or the string 'dynamic'.") + else: + loss_scale_help_text = loss_scale_help_text.format( + "This must be an int/float", "") + loss_scale_validation_msg = "loss_scale should be a positive int/float." + if loss_scale: + flags.DEFINE_string( + name="loss_scale", short_name="ls", default=None, + help=help_wrap(loss_scale_help_text)) + + @flags.validator(flag_name="loss_scale", + message=loss_scale_validation_msg) + def _check_loss_scale(loss_scale): # pylint: disable=unused-variable + """Validator to check the loss scale flag is valid.""" + if loss_scale is None: + return True # null case is handled in get_loss_scale() + + if loss_scale == "dynamic" and dynamic_loss_scale: + return True + + try: + loss_scale = float(loss_scale) + except ValueError: + return False + + return loss_scale > 0 + + if fp16_implementation: + flags.DEFINE_enum( + name="fp16_implementation", default="keras", + enum_values=("keras', 'graph_rewrite"), + help=help_wrap( + "When --dtype=fp16, how fp16 should be implemented. This has no " + "impact on correctness. 'keras' uses the " + "tf.keras.mixed_precision API. 'graph_rewrite' uses the " + "tf.train.experimental.enable_mixed_precision_graph_rewrite " + "API.")) + + @flags.multi_flags_validator(["fp16_implementation", "dtype", + "loss_scale"]) + def _check_fp16_implementation(flags_dict): + """Validator to check fp16_implementation flag is valid.""" + if (flags_dict["fp16_implementation"] == "graph_rewrite" and + flags_dict["dtype"] != "fp16"): + raise flags.ValidationError("--fp16_implementation should not be " + "specified unless --dtype=fp16") + return True + + if all_reduce_alg: + flags.DEFINE_string( + name="all_reduce_alg", short_name="ara", default=None, + help=help_wrap("Defines the algorithm to use for performing all-reduce." + "When specified with MirroredStrategy for single " + "worker, this controls " + "tf.contrib.distribute.AllReduceCrossTowerOps. When " + "specified with MultiWorkerMirroredStrategy, this " + "controls " + "tf.distribute.experimental.CollectiveCommunication; " + "valid options are `ring` and `nccl`.")) + + if num_packs: + flags.DEFINE_integer( + name="num_packs", default=1, + help=help_wrap("Sets `num_packs` in the cross device ops used in " + "MirroredStrategy. For details, see " + "tf.distribute.NcclAllReduce.")) + + if tf_gpu_thread_mode: + flags.DEFINE_string( + name="tf_gpu_thread_mode", short_name="gt_mode", default=None, + help=help_wrap( + "Whether and how the GPU device uses its own threadpool.") + ) + + flags.DEFINE_integer( + name="per_gpu_thread_count", short_name="pgtc", default=0, + help=help_wrap( + "The number of threads to use for GPU. Only valid when " + "tf_gpu_thread_mode is not global.") + ) + + if datasets_num_private_threads: + flags.DEFINE_integer( + name="datasets_num_private_threads", + default=None, + help=help_wrap( + "Number of threads for a private threadpool created for all" + "datasets computation..") + ) + + if datasets_num_parallel_batches: + flags.DEFINE_integer( + name="datasets_num_parallel_batches", + default=None, + help=help_wrap( + "Determines how many batches to process in parallel when using " + "map and batch from tf.data.") + ) + + if training_dataset_cache: + flags.DEFINE_boolean( + name="training_dataset_cache", + default=False, + help=help_wrap( + "Determines whether to cache the training dataset on workers. " + "Typically used to improve training performance when training " + "data is in remote storage and can fit into worker memory.") + ) + + if tf_data_experimental_slack: + flags.DEFINE_boolean( + name="tf_data_experimental_slack", + default=False, + help=help_wrap( + "Whether to enable tf.data's `experimental_slack` option.") + ) + + if enable_xla: + flags.DEFINE_boolean( + name="enable_xla", default=False, + help="Whether to enable XLA auto jit compilation") + + return key_flags diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/core.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/core.py new file mode 100644 index 0000000..fa36944 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/core.py @@ -0,0 +1,133 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Public interface for flag definition. + +See _example.py for detailed instructions on defining flags. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys +from six.moves import shlex_quote + +from absl import app as absl_app +from absl import flags + +from official.utils.flags import _base +from official.utils.flags import _benchmark +from official.utils.flags import _conventions +from official.utils.flags import _device +from official.utils.flags import _distribution +from official.utils.flags import _misc +from official.utils.flags import _performance + + +def set_defaults(**kwargs): + for key, value in kwargs.items(): + flags.FLAGS.set_default(name=key, value=value) + + +def parse_flags(argv=None): + """Reset flags and reparse. Currently only used in testing.""" + flags.FLAGS.unparse_flags() + absl_app.parse_flags_with_usage(argv or sys.argv) + + +def register_key_flags_in_core(f): + """Defines a function in core.py, and registers its key flags. + + absl uses the location of a flags.declare_key_flag() to determine the context + in which a flag is key. By making all declares in core, this allows model + main functions to call flags.adopt_module_key_flags() on core and correctly + chain key flags. + + Args: + f: The function to be wrapped + + Returns: + The "core-defined" version of the input function. + """ + + def core_fn(*args, **kwargs): + key_flags = f(*args, **kwargs) + [flags.declare_key_flag(fl) for fl in key_flags] # pylint: disable=expression-not-assigned + return core_fn + + +define_base = register_key_flags_in_core(_base.define_base) +# We have define_base_eager for compatibility, since it used to be a separate +# function from define_base. +define_base_eager = define_base +define_log_steps = register_key_flags_in_core(_benchmark.define_log_steps) +define_benchmark = register_key_flags_in_core(_benchmark.define_benchmark) +define_device = register_key_flags_in_core(_device.define_device) +define_image = register_key_flags_in_core(_misc.define_image) +define_performance = register_key_flags_in_core(_performance.define_performance) +define_distribution = register_key_flags_in_core( + _distribution.define_distribution) + + +help_wrap = _conventions.help_wrap + + +get_num_gpus = _base.get_num_gpus +get_tf_dtype = _performance.get_tf_dtype +get_loss_scale = _performance.get_loss_scale +DTYPE_MAP = _performance.DTYPE_MAP +require_cloud_storage = _device.require_cloud_storage + +def _get_nondefault_flags_as_dict(): + """Returns the nondefault flags as a dict from flag name to value.""" + nondefault_flags = {} + for flag_name in flags.FLAGS: + flag_value = getattr(flags.FLAGS, flag_name) + if (flag_name != flags.FLAGS[flag_name].short_name and + flag_value != flags.FLAGS[flag_name].default): + nondefault_flags[flag_name] = flag_value + return nondefault_flags + + +def get_nondefault_flags_as_str(): + """Returns flags as a string that can be passed as command line arguments. + + E.g., returns: "--batch_size=256 --use_synthetic_data" for the following code + block: + + ``` + flags.FLAGS.batch_size = 256 + flags.FLAGS.use_synthetic_data = True + print(get_nondefault_flags_as_str()) + ``` + + Only flags with nondefault values are returned, as passing default flags as + command line arguments has no effect. + + Returns: + A string with the flags, that can be passed as command line arguments to a + program to use the flags. + """ + nondefault_flags = _get_nondefault_flags_as_dict() + flag_strings = [] + for name, value in sorted(nondefault_flags.items()): + if isinstance(value, bool): + flag_str = '--{}'.format(name) if value else '--no{}'.format(name) + elif isinstance(value, list): + flag_str = '--{}={}'.format(name, ','.join(value)) + else: + flag_str = '--{}={}'.format(name, value) + flag_strings.append(flag_str) + return ' '.join(shlex_quote(flag_str) for flag_str in flag_strings) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/flags_test.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/flags_test.py new file mode 100644 index 0000000..e11a164 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/flags_test.py @@ -0,0 +1,162 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import unittest + +from absl import flags +import tensorflow as tf + +from official.utils.flags import core as flags_core # pylint: disable=g-bad-import-order + + +def define_flags(): + flags_core.define_base(clean=True, num_gpu=False, stop_threshold=True, + hooks=True, train_epochs=True, + epochs_between_evals=True) + flags_core.define_performance( + num_parallel_calls=True, inter_op=True, intra_op=True, + dynamic_loss_scale=True, loss_scale=True, synthetic_data=True, + dtype=True) + flags_core.define_image() + flags_core.define_benchmark() + + +class BaseTester(unittest.TestCase): + + @classmethod + def setUpClass(cls): + super(BaseTester, cls).setUpClass() + define_flags() + + def test_default_setting(self): + """Test to ensure fields exist and defaults can be set. + """ + + defaults = dict( + data_dir="dfgasf", + model_dir="dfsdkjgbs", + train_epochs=534, + epochs_between_evals=15, + batch_size=256, + hooks=["LoggingTensorHook"], + num_parallel_calls=18, + inter_op_parallelism_threads=5, + intra_op_parallelism_threads=10, + data_format="channels_first" + ) + + flags_core.set_defaults(**defaults) + flags_core.parse_flags() + + for key, value in defaults.items(): + assert flags.FLAGS.get_flag_value(name=key, default=None) == value + + def test_benchmark_setting(self): + defaults = dict( + hooks=["LoggingMetricHook"], + benchmark_log_dir="/tmp/12345", + gcp_project="project_abc", + ) + + flags_core.set_defaults(**defaults) + flags_core.parse_flags() + + for key, value in defaults.items(): + assert flags.FLAGS.get_flag_value(name=key, default=None) == value + + def test_booleans(self): + """Test to ensure boolean flags trigger as expected. + """ + + flags_core.parse_flags([__file__, "--use_synthetic_data"]) + + assert flags.FLAGS.use_synthetic_data + + def test_parse_dtype_info(self): + flags_core.parse_flags([__file__, "--dtype", "fp16"]) + self.assertEqual(flags_core.get_tf_dtype(flags.FLAGS), tf.float16) + self.assertEqual(flags_core.get_loss_scale(flags.FLAGS, + default_for_fp16=2), 2) + + flags_core.parse_flags( + [__file__, "--dtype", "fp16", "--loss_scale", "5"]) + self.assertEqual(flags_core.get_loss_scale(flags.FLAGS, + default_for_fp16=2), 5) + + flags_core.parse_flags( + [__file__, "--dtype", "fp16", "--loss_scale", "dynamic"]) + self.assertEqual(flags_core.get_loss_scale(flags.FLAGS, + default_for_fp16=2), "dynamic") + + flags_core.parse_flags([__file__, "--dtype", "fp32"]) + self.assertEqual(flags_core.get_tf_dtype(flags.FLAGS), tf.float32) + self.assertEqual(flags_core.get_loss_scale(flags.FLAGS, + default_for_fp16=2), 1) + + flags_core.parse_flags([__file__, "--dtype", "fp32", "--loss_scale", "5"]) + self.assertEqual(flags_core.get_loss_scale(flags.FLAGS, + default_for_fp16=2), 5) + + + with self.assertRaises(SystemExit): + flags_core.parse_flags([__file__, "--dtype", "int8"]) + + with self.assertRaises(SystemExit): + flags_core.parse_flags([__file__, "--dtype", "fp16", + "--loss_scale", "abc"]) + + def test_get_nondefault_flags_as_str(self): + defaults = dict( + clean=True, + data_dir="abc", + hooks=["LoggingTensorHook"], + stop_threshold=1.5, + use_synthetic_data=False + ) + flags_core.set_defaults(**defaults) + flags_core.parse_flags() + + expected_flags = "" + self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags) + + flags.FLAGS.clean = False + expected_flags += "--noclean" + self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags) + + flags.FLAGS.data_dir = "xyz" + expected_flags += " --data_dir=xyz" + self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags) + + flags.FLAGS.hooks = ["aaa", "bbb", "ccc"] + expected_flags += " --hooks=aaa,bbb,ccc" + self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags) + + flags.FLAGS.stop_threshold = 3. + expected_flags += " --stop_threshold=3.0" + self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags) + + flags.FLAGS.use_synthetic_data = True + expected_flags += " --use_synthetic_data" + self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags) + + # Assert that explicit setting a flag to its default value does not cause it + # to appear in the string + flags.FLAGS.use_synthetic_data = False + expected_flags = expected_flags[:-len(" --use_synthetic_data")] + self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags) + + +if __name__ == "__main__": + unittest.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/guidelines.md b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/guidelines.md new file mode 100644 index 0000000..db963aa --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/flags/guidelines.md @@ -0,0 +1,65 @@ +# Using flags in official models + +1. **All common flags must be incorporated in the models.** + + Common flags (i.e. batch_size, model_dir, etc.) are provided by various flag definition functions, + and channeled through `official.utils.flags.core`. For instance to define common supervised + learning parameters one could use the following code: + + ```$xslt + from absl import app as absl_app + from absl import flags + + from official.utils.flags import core as flags_core + + + def define_flags(): + flags_core.define_base() + flags.adopt_key_flags(flags_core) + + + def main(_): + flags_obj = flags.FLAGS + print(flags_obj) + + + if __name__ == "__main__" + absl_app.run(main) + ``` +2. **Validate flag values.** + + See the [Validators](#validators) section for implementation details. + + Validators in the official model repo should not access the file system, such as verifying + that files exist, due to the strict ordering requirements. + +3. **Flag values should not be mutated.** + + Instead of mutating flag values, use getter functions to return the desired values. An example + getter function is `get_tf_dtype` function below: + + ``` + # Map string to TensorFlow dtype + DTYPE_MAP = { + "fp16": tf.float16, + "fp32": tf.float32, + } + + def get_tf_dtype(flags_obj): + if getattr(flags_obj, "fp16_implementation", None) == "graph_rewrite": + # If the graph_rewrite is used, we build the graph with fp32, and let the + # graph rewrite change ops to fp16. + return tf.float32 + return DTYPE_MAP[flags_obj.dtype] + + + def main(_): + flags_obj = flags.FLAGS() + + # Do not mutate flags_obj + # if flags_obj.fp16_implementation == "graph_rewrite": + # flags_obj.dtype = "float32" # Don't do this + + print(get_tf_dtype(flags_obj)) + ... + ``` \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/hyperparams_flags.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/hyperparams_flags.py new file mode 100644 index 0000000..961d615 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/hyperparams_flags.py @@ -0,0 +1,119 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Common flags for importing hyperparameters.""" + +from __future__ import absolute_import +from __future__ import division +# from __future__ import google_type_annotations +from __future__ import print_function + +from absl import flags +from official.utils.flags import core as flags_core + +FLAGS = flags.FLAGS + + +def define_common_hparams_flags(): + """Define the common flags across models.""" + + flags.DEFINE_string( + 'model_dir', + default=None, + help=('The directory where the model and training/evaluation summaries' + 'are stored.')) + + flags.DEFINE_integer( + 'train_batch_size', default=None, help='Batch size for training.') + + flags.DEFINE_integer( + 'eval_batch_size', default=None, help='Batch size for evaluation.') + + flags.DEFINE_string( + 'precision', + default=None, + help=('Precision to use; one of: {bfloat16, float32}')) + + flags.DEFINE_string( + 'config_file', + default=None, + help=('A YAML file which specifies overrides. Note that this file can be ' + 'used as an override template to override the default parameters ' + 'specified in Python. If the same parameter is specified in both ' + '`--config_file` and `--params_override`, the one in ' + '`--params_override` will be used finally.')) + + flags.DEFINE_string( + 'params_override', + default=None, + help=('a YAML/JSON string or a YAML file which specifies additional ' + 'overrides over the default parameters and those specified in ' + '`--config_file`. Note that this is supposed to be used only to ' + 'override the model parameters, but not the parameters like TPU ' + 'specific flags. One canonical use case of `--config_file` and ' + '`--params_override` is users first define a template config file ' + 'using `--config_file`, then use `--params_override` to adjust the ' + 'minimal set of tuning parameters, for example setting up different' + ' `train_batch_size`. ' + 'The final override order of parameters: default_model_params --> ' + 'params from config_file --> params in params_override.' + 'See also the help message of `--config_file`.')) + flags.DEFINE_integer('save_checkpoint_freq', None, + 'Number of steps to save checkpoint.') + + +def initialize_common_flags(): + """Define the common flags across models.""" + define_common_hparams_flags() + + flags_core.define_device(tpu=True) + flags_core.define_base( + num_gpu=True, model_dir=False, data_dir=False, batch_size=False) + flags_core.define_distribution(worker_hosts=True, task_index=True) + flags_core.define_performance(all_reduce_alg=True, num_packs=True) + + # Reset the default value of num_gpus to zero. + FLAGS.num_gpus = 0 + + flags.DEFINE_string( + 'strategy_type', 'mirrored', 'Type of distribute strategy.' + 'One of mirrored, tpu and multiworker.') + + +def strategy_flags_dict(): + """Returns TPU and/or GPU related flags in a dictionary.""" + return { + # TPUStrategy related flags. + 'tpu': FLAGS.tpu, + # MultiWorkerMirroredStrategy related flags. + 'all_reduce_alg': FLAGS.all_reduce_alg, + 'worker_hosts': FLAGS.worker_hosts, + 'task_index': FLAGS.task_index, + # MirroredStrategy and OneDeviceStrategy + 'num_gpus': FLAGS.num_gpus, + 'num_packs': FLAGS.num_packs, + } + + +def hparam_flags_dict(): + """Returns model params related flags in a dictionary.""" + return { + 'data_dir': FLAGS.data_dir, + 'model_dir': FLAGS.model_dir, + 'train_batch_size': FLAGS.train_batch_size, + 'eval_batch_size': FLAGS.eval_batch_size, + 'precision': FLAGS.precision, + 'config_file': FLAGS.config_file, + 'params_override': FLAGS.params_override, + } diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/cloud_lib.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/cloud_lib.py new file mode 100644 index 0000000..a2d9bd3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/cloud_lib.py @@ -0,0 +1,34 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Utilities that interact with cloud service. +""" + +import requests + +GCP_METADATA_URL = "http://metadata/computeMetadata/v1/instance/hostname" +GCP_METADATA_HEADER = {"Metadata-Flavor": "Google"} + + +def on_gcp(): + """Detect whether the current running environment is on GCP.""" + try: + # Timeout in 5 seconds, in case the test environment has connectivity issue. + # There is not default timeout, which means it might block forever. + response = requests.get( + GCP_METADATA_URL, headers=GCP_METADATA_HEADER, timeout=5) + return response.status_code == 200 + except requests.exceptions.RequestException: + return False diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/cloud_lib_test.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/cloud_lib_test.py new file mode 100644 index 0000000..901576d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/cloud_lib_test.py @@ -0,0 +1,48 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Tests for cloud_lib.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest + +import mock +import requests + +from official.utils.logs import cloud_lib + + +class CloudLibTest(unittest.TestCase): + + @mock.patch("requests.get") + def test_on_gcp(self, mock_requests_get): + mock_response = mock.MagicMock() + mock_requests_get.return_value = mock_response + mock_response.status_code = 200 + + self.assertEqual(cloud_lib.on_gcp(), True) + + @mock.patch("requests.get") + def test_not_on_gcp(self, mock_requests_get): + mock_requests_get.side_effect = requests.exceptions.ConnectionError() + + self.assertEqual(cloud_lib.on_gcp(), False) + + +if __name__ == "__main__": + unittest.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/guidelines.md b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/guidelines.md new file mode 100644 index 0000000..408c3cd --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/guidelines.md @@ -0,0 +1,58 @@ +# Logging in official models + +This library adds logging functions that print or save tensor values. Official models should define all common hooks +(using hooks helper) and a benchmark logger. + +1. **Training Hooks** + + Hooks are a TensorFlow concept that define specific actions at certain points of the execution. We use them to obtain and log + tensor values during training. + + hooks_helper.py provides an easy way to create common hooks. The following hooks are currently defined: + * LoggingTensorHook: Logs tensor values + * ProfilerHook: Writes a timeline json that can be loaded into chrome://tracing. + * ExamplesPerSecondHook: Logs the number of examples processed per second. + * LoggingMetricHook: Similar to LoggingTensorHook, except that the tensors are logged in a format defined by our data + anaylsis pipeline. + + +2. **Benchmarks** + + The benchmark logger provides useful functions for logging environment information, and evaluation results. + The module also contains a context which is used to update the status of the run. + +Example usage: + +``` +from absl import app as absl_app + +from official.utils.logs import hooks_helper +from official.utils.logs import logger + +def model_main(flags_obj): + estimator = ... + + benchmark_logger = logger.get_benchmark_logger() + benchmark_logger.log_run_info(...) + + train_hooks = hooks_helper.get_train_hooks(...) + + for epoch in range(10): + estimator.train(..., hooks=train_hooks) + eval_results = estimator.evaluate(...) + + # Log a dictionary of metrics + benchmark_logger.log_evaluation_result(eval_results) + + # Log an individual metric + benchmark_logger.log_metric(...) + + +def main(_): + with logger.benchmark_context(flags.FLAGS): + model_main(flags.FLAGS) + +if __name__ == "__main__": + # define flags + absl_app.run(main) +``` diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/hooks.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/hooks.py new file mode 100644 index 0000000..652ebf2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/hooks.py @@ -0,0 +1,146 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Hook that counts examples per second every N steps or seconds.""" + + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from hccl.manage.api import get_rank_size +import tensorflow as tf # pylint: disable=g-bad-import-order +from official.utils.logs import logger +from benchmark_log import hwlog +import time +import sys + + +class ExamplesPerSecondHook(tf.estimator.SessionRunHook): + """Hook to print out examples per second. + + Total time is tracked and then divided by the total number of steps + to get the average step time and then batch_size is used to determine + the running average of examples per second. The examples per second for the + most recent interval is also logged. + """ + + def __init__(self, + batch_size, + every_n_steps=None, + every_n_secs=None, + warm_steps=0, + metric_logger=None): + """Initializer for ExamplesPerSecondHook. + + Args: + batch_size: Total batch size across all workers used to calculate + examples/second from global time. + every_n_steps: Log stats every n steps. + every_n_secs: Log stats every n seconds. Exactly one of the + `every_n_steps` or `every_n_secs` should be set. + warm_steps: The number of steps to be skipped before logging and running + average calculation. warm_steps steps refers to global steps across all + workers, not on each worker + metric_logger: instance of `BenchmarkLogger`, the benchmark logger that + hook should use to write the log. If None, BaseBenchmarkLogger will + be used. + + Raises: + ValueError: if neither `every_n_steps` or `every_n_secs` is set, or + both are set. + """ + if (every_n_steps is None) == (every_n_secs is None): + raise ValueError("exactly one of every_n_steps" + " and every_n_secs should be provided.") + + self._logger = metric_logger or logger.BaseBenchmarkLogger() + + self._timer = tf.estimator.SecondOrStepTimer( + every_steps=every_n_steps, every_secs=every_n_secs) + + self._step_train_time = 0 + self._total_steps = 0 + self._batch_size = batch_size + self._warm_steps = warm_steps + # List of examples per second logged every_n_steps. + self.current_examples_per_sec_list = [] + + def begin(self): + """Called once before using the session to check global step.""" + tf.compat.v1.logging.warning("##########ExamplesPerSecondHook begin") + self._global_step_tensor = tf.compat.v1.train.get_global_step() + if self._global_step_tensor is None: + raise RuntimeError( + "Global step should be created to use StepCounterHook.") + + def before_run(self, run_context): # pylint: disable=unused-argument + """Called before each call to run(). + + Args: + run_context: A SessionRunContext object. + + Returns: + A SessionRunArgs object or None if never triggered. + """ + self.t0 = time.time() + tf.compat.v1.logging.warning("##########ExamplesPerSecondHook before") + return tf.estimator.SessionRunArgs(self._global_step_tensor) + + def after_run(self, run_context, run_values): # pylint: disable=unused-argument + """Called after each call to run(). + + Args: + run_context: A SessionRunContext object. + run_values: A SessionRunValues object. + """ + tf.compat.v1.logging.warning("##########ExamplesPerSecondHook after_run") + global_step = run_values.results + + #if self._timer.should_trigger_for_step( + #global_step) and global_step > self._warm_steps: + elapsed_time, elapsed_steps = self._timer.update_last_triggered_step( + global_step) + batch_time = time.time() - self.t0 + ips = self._batch_size/batch_time + if elapsed_time is not None: + self._step_train_time += elapsed_time + self._total_steps += elapsed_steps + + # average examples per second is based on the total (accumulative) + # training steps and training time so far + average_examples_per_sec = self._batch_size * ( + self._total_steps / self._step_train_time) + # current examples per second is based on the elapsed training steps + # and training time per batch + current_examples_per_sec = self._batch_size * get_rank_size() * ( + elapsed_steps / elapsed_time) + # Logs entries to be read from hook during or after run. + self.current_examples_per_sec_list.append(current_examples_per_sec) + self._logger.log_metric( + "average_examples_per_sec", average_examples_per_sec, + global_step=global_step) + + self._logger.log_metric( + "current_examples_per_sec", current_examples_per_sec, + global_step=global_step) + tf.compat.v1.logging.warning( + "steps: %s,elapsed_steps:%d,batch:%d,FPS:%f,ips:%f,batch_time:%f", int(self._total_steps), + int(elapsed_steps),int(self._batch_size),float(current_examples_per_sec),float(ips), + float(batch_time)) + # get FPS info, add by wx933135 + #date_time = hwlog.get_time() + #remark_logger.info("ABK time_ts: %s, fps: %f, steps: %s, file: %s, lineno: %s" % (date_time, + # float(current_examples_per_sec), int(self._total_steps), file_name, sys._getframe().f_lineno)) + hwlog.remark_print(key=hwlog.FPS, value=float(current_examples_per_sec)) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/hooks_helper.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/hooks_helper.py new file mode 100644 index 0000000..50a380d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/hooks_helper.py @@ -0,0 +1,172 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Hooks helper to return a list of TensorFlow hooks for training by name. + +More hooks can be added to this set. To add a new hook, 1) add the new hook to +the registry in HOOKS, 2) add a corresponding function that parses out necessary +parameters. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.utils.logs import hooks +from official.utils.logs import logger +from official.utils.logs import metric_hook + +_TENSORS_TO_LOG = dict((x, x) for x in ['learning_rate', + 'cross_entropy', + 'train_accuracy']) + + +def get_train_hooks(name_list, use_tpu=False, **kwargs): + """Factory for getting a list of TensorFlow hooks for training by name. + + Args: + name_list: a list of strings to name desired hook classes. Allowed: + LoggingTensorHook, ProfilerHook, ExamplesPerSecondHook, which are defined + as keys in HOOKS + use_tpu: Boolean of whether computation occurs on a TPU. This will disable + hooks altogether. + **kwargs: a dictionary of arguments to the hooks. + + Returns: + list of instantiated hooks, ready to be used in a classifier.train call. + + Raises: + ValueError: if an unrecognized name is passed. + """ + + if not name_list: + return [] + + if use_tpu: + tf.compat.v1.logging.warning('hooks_helper received name_list `{}`, but a ' + 'TPU is specified. No hooks will be used.' + .format(name_list)) + return [] + + train_hooks = [] + for name in name_list: + hook_name = HOOKS.get(name.strip().lower()) + if hook_name is None: + raise ValueError('Unrecognized training hook requested: {}'.format(name)) + else: + train_hooks.append(hook_name(**kwargs)) + + return train_hooks + + +def get_logging_tensor_hook(every_n_iter=100, tensors_to_log=None, **kwargs): # pylint: disable=unused-argument + """Function to get LoggingTensorHook. + + Args: + every_n_iter: `int`, print the values of `tensors` once every N local + steps taken on the current worker. + tensors_to_log: List of tensor names or dictionary mapping labels to tensor + names. If not set, log _TENSORS_TO_LOG by default. + **kwargs: a dictionary of arguments to LoggingTensorHook. + + Returns: + Returns a LoggingTensorHook with a standard set of tensors that will be + printed to stdout. + """ + if tensors_to_log is None: + tensors_to_log = _TENSORS_TO_LOG + + return tf.estimator.LoggingTensorHook( + tensors=tensors_to_log, + every_n_iter=every_n_iter) + + +def get_profiler_hook(model_dir, save_steps=1000, **kwargs): # pylint: disable=unused-argument + """Function to get ProfilerHook. + + Args: + model_dir: The directory to save the profile traces to. + save_steps: `int`, print profile traces every N steps. + **kwargs: a dictionary of arguments to ProfilerHook. + + Returns: + Returns a ProfilerHook that writes out timelines that can be loaded into + profiling tools like chrome://tracing. + """ + return tf.estimator.ProfilerHook(save_steps=save_steps, output_dir=model_dir) + + +def get_examples_per_second_hook(every_n_steps=100, + batch_size=128, + warm_steps=5, + **kwargs): # pylint: disable=unused-argument + """Function to get ExamplesPerSecondHook. + + Args: + every_n_steps: `int`, print current and average examples per second every + N steps. + batch_size: `int`, total batch size used to calculate examples/second from + global time. + warm_steps: skip this number of steps before logging and running average. + **kwargs: a dictionary of arguments to ExamplesPerSecondHook. + + Returns: + Returns a ProfilerHook that writes out timelines that can be loaded into + profiling tools like chrome://tracing. + """ + return hooks.ExamplesPerSecondHook( + batch_size=batch_size, every_n_steps=every_n_steps, + warm_steps=warm_steps, metric_logger=logger.get_benchmark_logger()) + + +def get_logging_metric_hook(tensors_to_log=None, + every_n_secs=600, + **kwargs): # pylint: disable=unused-argument + """Function to get LoggingMetricHook. + + Args: + tensors_to_log: List of tensor names or dictionary mapping labels to tensor + names. If not set, log _TENSORS_TO_LOG by default. + every_n_secs: `int`, the frequency for logging the metric. Default to every + 10 mins. + **kwargs: a dictionary of arguments. + + Returns: + Returns a LoggingMetricHook that saves tensor values in a JSON format. + """ + if tensors_to_log is None: + tensors_to_log = _TENSORS_TO_LOG + return metric_hook.LoggingMetricHook( + tensors=tensors_to_log, + metric_logger=logger.get_benchmark_logger(), + every_n_secs=every_n_secs) + + +def get_step_counter_hook(**kwargs): + """Function to get StepCounterHook.""" + del kwargs + return tf.estimator.StepCounterHook() + + +# A dictionary to map one hook name and its corresponding function +HOOKS = { + 'loggingtensorhook': get_logging_tensor_hook, + 'profilerhook': get_profiler_hook, + 'examplespersecondhook': get_examples_per_second_hook, + 'loggingmetrichook': get_logging_metric_hook, + 'stepcounterhook': get_step_counter_hook +} diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/hooks_helper_test.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/hooks_helper_test.py new file mode 100644 index 0000000..693311b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/hooks_helper_test.py @@ -0,0 +1,73 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Tests for hooks_helper.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest + +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.utils.logs import hooks_helper +from official.utils.misc import keras_utils + + +class BaseTest(unittest.TestCase): + + def setUp(self): + super(BaseTest, self).setUp() + if keras_utils.is_v2_0: + tf.compat.v1.disable_eager_execution() + + def test_raise_in_non_list_names(self): + with self.assertRaises(ValueError): + hooks_helper.get_train_hooks( + 'LoggingTensorHook, ProfilerHook', model_dir="", batch_size=256) + + def test_raise_in_invalid_names(self): + invalid_names = ['StepCounterHook', 'StopAtStepHook'] + with self.assertRaises(ValueError): + hooks_helper.get_train_hooks(invalid_names, model_dir="", batch_size=256) + + def validate_train_hook_name(self, + test_hook_name, + expected_hook_name, + **kwargs): + returned_hook = hooks_helper.get_train_hooks( + [test_hook_name], model_dir="", **kwargs) + self.assertEqual(len(returned_hook), 1) + self.assertIsInstance(returned_hook[0], tf.estimator.SessionRunHook) + self.assertEqual(returned_hook[0].__class__.__name__.lower(), + expected_hook_name) + + def test_get_train_hooks_logging_tensor_hook(self): + self.validate_train_hook_name('LoggingTensorHook', 'loggingtensorhook') + + def test_get_train_hooks_profiler_hook(self): + self.validate_train_hook_name('ProfilerHook', 'profilerhook') + + def test_get_train_hooks_examples_per_second_hook(self): + self.validate_train_hook_name('ExamplesPerSecondHook', + 'examplespersecondhook') + + def test_get_logging_metric_hook(self): + test_hook_name = 'LoggingMetricHook' + self.validate_train_hook_name(test_hook_name, 'loggingmetrichook') + +if __name__ == '__main__': + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/hooks_test.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/hooks_test.py new file mode 100644 index 0000000..7069779 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/hooks_test.py @@ -0,0 +1,158 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Tests for hooks.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import time + +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.utils.logs import hooks +from official.utils.testing import mock_lib + +tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.DEBUG) + + +class ExamplesPerSecondHookTest(tf.test.TestCase): + """Tests for the ExamplesPerSecondHook. + + In the test, we explicitly run global_step tensor after train_op in order to + keep the global_step value and the train_op (which increase the glboal_step + by 1) consistent. This is to correct the discrepancies in reported global_step + value when running on GPUs. + """ + + def setUp(self): + """Mock out logging calls to verify if correct info is being monitored.""" + self._logger = mock_lib.MockBenchmarkLogger() + + self.graph = tf.Graph() + with self.graph.as_default(): + tf.compat.v1.train.create_global_step() + self.train_op = tf.compat.v1.assign_add( + tf.compat.v1.train.get_global_step(), 1) + self.global_step = tf.compat.v1.train.get_global_step() + + def test_raise_in_both_secs_and_steps(self): + with self.assertRaises(ValueError): + hooks.ExamplesPerSecondHook( + batch_size=256, + every_n_steps=10, + every_n_secs=20, + metric_logger=self._logger) + + def test_raise_in_none_secs_and_steps(self): + with self.assertRaises(ValueError): + hooks.ExamplesPerSecondHook( + batch_size=256, + every_n_steps=None, + every_n_secs=None, + metric_logger=self._logger) + + def _validate_log_every_n_steps(self, every_n_steps, warm_steps): + hook = hooks.ExamplesPerSecondHook( + batch_size=256, + every_n_steps=every_n_steps, + warm_steps=warm_steps, + metric_logger=self._logger) + + with tf.compat.v1.train.MonitoredSession( + tf.compat.v1.train.ChiefSessionCreator(), [hook]) as mon_sess: + for _ in range(every_n_steps): + # Explicitly run global_step after train_op to get the accurate + # global_step value + mon_sess.run(self.train_op) + mon_sess.run(self.global_step) + # Nothing should be in the list yet + self.assertFalse(self._logger.logged_metric) + + mon_sess.run(self.train_op) + global_step_val = mon_sess.run(self.global_step) + + if global_step_val > warm_steps: + self._assert_metrics() + else: + # Nothing should be in the list yet + self.assertFalse(self._logger.logged_metric) + + # Add additional run to verify proper reset when called multiple times. + prev_log_len = len(self._logger.logged_metric) + mon_sess.run(self.train_op) + global_step_val = mon_sess.run(self.global_step) + + if every_n_steps == 1 and global_step_val > warm_steps: + # Each time, we log two additional metrics. Did exactly 2 get added? + self.assertEqual(len(self._logger.logged_metric), prev_log_len + 2) + else: + # No change in the size of the metric list. + self.assertEqual(len(self._logger.logged_metric), prev_log_len) + + def test_examples_per_sec_every_1_steps(self): + with self.graph.as_default(): + self._validate_log_every_n_steps(1, 0) + + def test_examples_per_sec_every_5_steps(self): + with self.graph.as_default(): + self._validate_log_every_n_steps(5, 0) + + def test_examples_per_sec_every_1_steps_with_warm_steps(self): + with self.graph.as_default(): + self._validate_log_every_n_steps(1, 10) + + def test_examples_per_sec_every_5_steps_with_warm_steps(self): + with self.graph.as_default(): + self._validate_log_every_n_steps(5, 10) + + def _validate_log_every_n_secs(self, every_n_secs): + hook = hooks.ExamplesPerSecondHook( + batch_size=256, + every_n_steps=None, + every_n_secs=every_n_secs, + metric_logger=self._logger) + + with tf.compat.v1.train.MonitoredSession( + tf.compat.v1.train.ChiefSessionCreator(), [hook]) as mon_sess: + # Explicitly run global_step after train_op to get the accurate + # global_step value + mon_sess.run(self.train_op) + mon_sess.run(self.global_step) + # Nothing should be in the list yet + self.assertFalse(self._logger.logged_metric) + time.sleep(every_n_secs) + + mon_sess.run(self.train_op) + mon_sess.run(self.global_step) + self._assert_metrics() + + def test_examples_per_sec_every_1_secs(self): + with self.graph.as_default(): + self._validate_log_every_n_secs(1) + + def test_examples_per_sec_every_5_secs(self): + with self.graph.as_default(): + self._validate_log_every_n_secs(5) + + def _assert_metrics(self): + metrics = self._logger.logged_metric + self.assertEqual(metrics[-2]["name"], "average_examples_per_sec") + self.assertEqual(metrics[-1]["name"], "current_examples_per_sec") + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/logger.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/logger.py new file mode 100644 index 0000000..398aa8a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/logger.py @@ -0,0 +1,423 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Logging utilities for benchmark. + +For collecting local environment metrics like CPU and memory, certain python +packages need be installed. See README for details. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import contextlib +import datetime +import json +import multiprocessing +import numbers +import os +import threading +import uuid + +from six.moves import _thread as thread +from absl import flags +import tensorflow as tf +from tensorflow.python.client import device_lib + +from official.utils.logs import cloud_lib + +METRIC_LOG_FILE_NAME = "metric.log" +BENCHMARK_RUN_LOG_FILE_NAME = "benchmark_run.log" +_DATE_TIME_FORMAT_PATTERN = "%Y-%m-%dT%H:%M:%S.%fZ" +GCP_TEST_ENV = "GCP" +RUN_STATUS_SUCCESS = "success" +RUN_STATUS_FAILURE = "failure" +RUN_STATUS_RUNNING = "running" + + +FLAGS = flags.FLAGS + +# Don't use it directly. Use get_benchmark_logger to access a logger. +_benchmark_logger = None +_logger_lock = threading.Lock() + + +def config_benchmark_logger(flag_obj=None): + """Config the global benchmark logger.""" + _logger_lock.acquire() + try: + global _benchmark_logger + if not flag_obj: + flag_obj = FLAGS + + if (not hasattr(flag_obj, "benchmark_logger_type") or + flag_obj.benchmark_logger_type == "BaseBenchmarkLogger"): + _benchmark_logger = BaseBenchmarkLogger() + elif flag_obj.benchmark_logger_type == "BenchmarkFileLogger": + _benchmark_logger = BenchmarkFileLogger(flag_obj.benchmark_log_dir) + elif flag_obj.benchmark_logger_type == "BenchmarkBigQueryLogger": + from official.benchmark import benchmark_uploader as bu # pylint: disable=g-import-not-at-top + bq_uploader = bu.BigQueryUploader(gcp_project=flag_obj.gcp_project) + _benchmark_logger = BenchmarkBigQueryLogger( + bigquery_uploader=bq_uploader, + bigquery_data_set=flag_obj.bigquery_data_set, + bigquery_run_table=flag_obj.bigquery_run_table, + bigquery_run_status_table=flag_obj.bigquery_run_status_table, + bigquery_metric_table=flag_obj.bigquery_metric_table, + run_id=str(uuid.uuid4())) + else: + raise ValueError("Unrecognized benchmark_logger_type: %s" + % flag_obj.benchmark_logger_type) + + finally: + _logger_lock.release() + return _benchmark_logger + + +def get_benchmark_logger(): + if not _benchmark_logger: + config_benchmark_logger() + return _benchmark_logger + + +@contextlib.contextmanager +def benchmark_context(flag_obj): + """Context of benchmark, which will update status of the run accordingly.""" + benchmark_logger = config_benchmark_logger(flag_obj) + try: + yield + benchmark_logger.on_finish(RUN_STATUS_SUCCESS) + except Exception: # pylint: disable=broad-except + # Catch all the exception, update the run status to be failure, and re-raise + benchmark_logger.on_finish(RUN_STATUS_FAILURE) + raise + + +class BaseBenchmarkLogger(object): + """Class to log the benchmark information to STDOUT.""" + + def log_evaluation_result(self, eval_results): + """Log the evaluation result. + + The evaluate result is a dictionary that contains metrics defined in + model_fn. It also contains a entry for global_step which contains the value + of the global step when evaluation was performed. + + Args: + eval_results: dict, the result of evaluate. + """ + if not isinstance(eval_results, dict): + tf.compat.v1.logging.warning( + "eval_results should be dictionary for logging. Got %s", + type(eval_results)) + return + global_step = eval_results[tf.compat.v1.GraphKeys.GLOBAL_STEP] + for key in sorted(eval_results): + if key != tf.compat.v1.GraphKeys.GLOBAL_STEP: + self.log_metric(key, eval_results[key], global_step=global_step) + + def log_metric(self, name, value, unit=None, global_step=None, extras=None): + """Log the benchmark metric information to local file. + + Currently the logging is done in a synchronized way. This should be updated + to log asynchronously. + + Args: + name: string, the name of the metric to log. + value: number, the value of the metric. The value will not be logged if it + is not a number type. + unit: string, the unit of the metric, E.g "image per second". + global_step: int, the global_step when the metric is logged. + extras: map of string:string, the extra information about the metric. + """ + metric = _process_metric_to_json(name, value, unit, global_step, extras) + if metric: + tf.compat.v1.logging.info("Benchmark metric: %s", metric) + + def log_run_info(self, model_name, dataset_name, run_params, test_id=None): + tf.compat.v1.logging.info( + "Benchmark run: %s", _gather_run_info(model_name, dataset_name, + run_params, test_id)) + + def on_finish(self, status): + pass + + +class BenchmarkFileLogger(BaseBenchmarkLogger): + """Class to log the benchmark information to local disk.""" + + def __init__(self, logging_dir): + super(BenchmarkFileLogger, self).__init__() + self._logging_dir = logging_dir + if not tf.io.gfile.isdir(self._logging_dir): + tf.io.gfile.makedirs(self._logging_dir) + self._metric_file_handler = tf.io.gfile.GFile( + os.path.join(self._logging_dir, METRIC_LOG_FILE_NAME), "a") + + def log_metric(self, name, value, unit=None, global_step=None, extras=None): + """Log the benchmark metric information to local file. + + Currently the logging is done in a synchronized way. This should be updated + to log asynchronously. + + Args: + name: string, the name of the metric to log. + value: number, the value of the metric. The value will not be logged if it + is not a number type. + unit: string, the unit of the metric, E.g "image per second". + global_step: int, the global_step when the metric is logged. + extras: map of string:string, the extra information about the metric. + """ + metric = _process_metric_to_json(name, value, unit, global_step, extras) + if metric: + try: + json.dump(metric, self._metric_file_handler) + self._metric_file_handler.write("\n") + self._metric_file_handler.flush() + except (TypeError, ValueError) as e: + tf.compat.v1.logging.warning( + "Failed to dump metric to log file: name %s, value %s, error %s", + name, value, e) + + def log_run_info(self, model_name, dataset_name, run_params, test_id=None): + """Collect most of the TF runtime information for the local env. + + The schema of the run info follows official/benchmark/datastore/schema. + + Args: + model_name: string, the name of the model. + dataset_name: string, the name of dataset for training and evaluation. + run_params: dict, the dictionary of parameters for the run, it could + include hyperparameters or other params that are important for the run. + test_id: string, the unique name of the test run by the combination of key + parameters, eg batch size, num of GPU. It is hardware independent. + """ + run_info = _gather_run_info(model_name, dataset_name, run_params, test_id) + + with tf.io.gfile.GFile(os.path.join( + self._logging_dir, BENCHMARK_RUN_LOG_FILE_NAME), "w") as f: + try: + json.dump(run_info, f) + f.write("\n") + except (TypeError, ValueError) as e: + tf.compat.v1.logging.warning( + "Failed to dump benchmark run info to log file: %s", e) + + def on_finish(self, status): + self._metric_file_handler.flush() + self._metric_file_handler.close() + + +class BenchmarkBigQueryLogger(BaseBenchmarkLogger): + """Class to log the benchmark information to BigQuery data store.""" + + def __init__(self, + bigquery_uploader, + bigquery_data_set, + bigquery_run_table, + bigquery_run_status_table, + bigquery_metric_table, + run_id): + super(BenchmarkBigQueryLogger, self).__init__() + self._bigquery_uploader = bigquery_uploader + self._bigquery_data_set = bigquery_data_set + self._bigquery_run_table = bigquery_run_table + self._bigquery_run_status_table = bigquery_run_status_table + self._bigquery_metric_table = bigquery_metric_table + self._run_id = run_id + + def log_metric(self, name, value, unit=None, global_step=None, extras=None): + """Log the benchmark metric information to bigquery. + + Args: + name: string, the name of the metric to log. + value: number, the value of the metric. The value will not be logged if it + is not a number type. + unit: string, the unit of the metric, E.g "image per second". + global_step: int, the global_step when the metric is logged. + extras: map of string:string, the extra information about the metric. + """ + metric = _process_metric_to_json(name, value, unit, global_step, extras) + if metric: + # Starting new thread for bigquery upload in case it might take long time + # and impact the benchmark and performance measurement. Starting a new + # thread might have potential performance impact for model that run on + # CPU. + thread.start_new_thread( + self._bigquery_uploader.upload_benchmark_metric_json, + (self._bigquery_data_set, + self._bigquery_metric_table, + self._run_id, + [metric])) + + def log_run_info(self, model_name, dataset_name, run_params, test_id=None): + """Collect most of the TF runtime information for the local env. + + The schema of the run info follows official/benchmark/datastore/schema. + + Args: + model_name: string, the name of the model. + dataset_name: string, the name of dataset for training and evaluation. + run_params: dict, the dictionary of parameters for the run, it could + include hyperparameters or other params that are important for the run. + test_id: string, the unique name of the test run by the combination of key + parameters, eg batch size, num of GPU. It is hardware independent. + """ + run_info = _gather_run_info(model_name, dataset_name, run_params, test_id) + # Starting new thread for bigquery upload in case it might take long time + # and impact the benchmark and performance measurement. Starting a new + # thread might have potential performance impact for model that run on CPU. + thread.start_new_thread( + self._bigquery_uploader.upload_benchmark_run_json, + (self._bigquery_data_set, + self._bigquery_run_table, + self._run_id, + run_info)) + thread.start_new_thread( + self._bigquery_uploader.insert_run_status, + (self._bigquery_data_set, + self._bigquery_run_status_table, + self._run_id, + RUN_STATUS_RUNNING)) + + def on_finish(self, status): + self._bigquery_uploader.update_run_status( + self._bigquery_data_set, + self._bigquery_run_status_table, + self._run_id, + status) + + +def _gather_run_info(model_name, dataset_name, run_params, test_id): + """Collect the benchmark run information for the local environment.""" + run_info = { + "model_name": model_name, + "dataset": {"name": dataset_name}, + "machine_config": {}, + "test_id": test_id, + "run_date": datetime.datetime.utcnow().strftime( + _DATE_TIME_FORMAT_PATTERN)} + _collect_tensorflow_info(run_info) + _collect_tensorflow_environment_variables(run_info) + _collect_run_params(run_info, run_params) + _collect_cpu_info(run_info) + _collect_memory_info(run_info) + _collect_test_environment(run_info) + return run_info + + +def _process_metric_to_json( + name, value, unit=None, global_step=None, extras=None): + """Validate the metric data and generate JSON for insert.""" + if not isinstance(value, numbers.Number): + tf.compat.v1.logging.warning( + "Metric value to log should be a number. Got %s", type(value)) + return None + + extras = _convert_to_json_dict(extras) + return { + "name": name, + "value": float(value), + "unit": unit, + "global_step": global_step, + "timestamp": datetime.datetime.utcnow().strftime( + _DATE_TIME_FORMAT_PATTERN), + "extras": extras} + + +def _collect_tensorflow_info(run_info): + run_info["tensorflow_version"] = { + "version": tf.version.VERSION, "git_hash": tf.version.GIT_VERSION} + + +def _collect_run_params(run_info, run_params): + """Log the parameter information for the benchmark run.""" + def process_param(name, value): + type_check = { + str: {"name": name, "string_value": value}, + int: {"name": name, "long_value": value}, + bool: {"name": name, "bool_value": str(value)}, + float: {"name": name, "float_value": value}, + } + return type_check.get(type(value), + {"name": name, "string_value": str(value)}) + if run_params: + run_info["run_parameters"] = [ + process_param(k, v) for k, v in sorted(run_params.items())] + + +def _collect_tensorflow_environment_variables(run_info): + run_info["tensorflow_environment_variables"] = [ + {"name": k, "value": v} + for k, v in sorted(os.environ.items()) if k.startswith("TF_")] + + +# The following code is mirrored from tensorflow/tools/test/system_info_lib +# which is not exposed for import. +def _collect_cpu_info(run_info): + """Collect the CPU information for the local environment.""" + cpu_info = {} + + cpu_info["num_cores"] = multiprocessing.cpu_count() + + try: + # Note: cpuinfo is not installed in the TensorFlow OSS tree. + # It is installable via pip. + import cpuinfo # pylint: disable=g-import-not-at-top + + info = cpuinfo.get_cpu_info() + cpu_info["cpu_info"] = info["brand"] + cpu_info["mhz_per_cpu"] = info["hz_advertised_raw"][0] / 1.0e6 + + run_info["machine_config"]["cpu_info"] = cpu_info + except ImportError: + tf.compat.v1.logging.warn( + "'cpuinfo' not imported. CPU info will not be logged.") + + +def _collect_memory_info(run_info): + try: + # Note: psutil is not installed in the TensorFlow OSS tree. + # It is installable via pip. + import psutil # pylint: disable=g-import-not-at-top + vmem = psutil.virtual_memory() + run_info["machine_config"]["memory_total"] = vmem.total + run_info["machine_config"]["memory_available"] = vmem.available + except ImportError: + tf.compat.v1.logging.warn( + "'psutil' not imported. Memory info will not be logged.") + + +def _collect_test_environment(run_info): + """Detect the local environment, eg GCE, AWS or DGX, etc.""" + if cloud_lib.on_gcp(): + run_info["test_environment"] = GCP_TEST_ENV + # TODO(scottzhu): Add more testing env detection for other platform + + +def _parse_gpu_model(physical_device_desc): + # Assume all the GPU connected are same model + for kv in physical_device_desc.split(","): + k, _, v = kv.partition(":") + if k.strip() == "name": + return v.strip() + return None + + +def _convert_to_json_dict(input_dict): + if input_dict: + return [{"name": k, "value": v} for k, v in sorted(input_dict.items())] + else: + return [] diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/logger_test.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/logger_test.py new file mode 100644 index 0000000..520db5f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/logger_test.py @@ -0,0 +1,365 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Tests for benchmark logger.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import os +import tempfile +import time +import unittest + +import mock +from absl.testing import flagsaver +import tensorflow as tf # pylint: disable=g-bad-import-order + +try: + from google.cloud import bigquery +except ImportError: + bigquery = None + +from official.utils.misc import keras_utils +from official.utils.flags import core as flags_core +from official.utils.logs import logger + + +class BenchmarkLoggerTest(tf.test.TestCase): + + @classmethod + def setUpClass(cls): # pylint: disable=invalid-name + super(BenchmarkLoggerTest, cls).setUpClass() + flags_core.define_benchmark() + + def test_get_default_benchmark_logger(self): + with flagsaver.flagsaver(benchmark_logger_type="foo"): + self.assertIsInstance(logger.get_benchmark_logger(), + logger.BaseBenchmarkLogger) + + def test_config_base_benchmark_logger(self): + with flagsaver.flagsaver(benchmark_logger_type="BaseBenchmarkLogger"): + logger.config_benchmark_logger() + self.assertIsInstance(logger.get_benchmark_logger(), + logger.BaseBenchmarkLogger) + + def test_config_benchmark_file_logger(self): + # Set the benchmark_log_dir first since the benchmark_logger_type will need + # the value to be set when it does the validation. + with flagsaver.flagsaver(benchmark_log_dir="/tmp"): + with flagsaver.flagsaver(benchmark_logger_type="BenchmarkFileLogger"): + logger.config_benchmark_logger() + self.assertIsInstance(logger.get_benchmark_logger(), + logger.BenchmarkFileLogger) + + @unittest.skipIf(bigquery is None, "Bigquery dependency is not installed.") + @mock.patch.object(bigquery, "Client") + def test_config_benchmark_bigquery_logger(self, mock_bigquery_client): + with flagsaver.flagsaver(benchmark_logger_type="BenchmarkBigQueryLogger"): + logger.config_benchmark_logger() + self.assertIsInstance(logger.get_benchmark_logger(), + logger.BenchmarkBigQueryLogger) + + @mock.patch("official.utils.logs.logger.config_benchmark_logger") + def test_benchmark_context(self, mock_config_benchmark_logger): + mock_logger = mock.MagicMock() + mock_config_benchmark_logger.return_value = mock_logger + with logger.benchmark_context(None): + tf.compat.v1.logging.info("start benchmarking") + mock_logger.on_finish.assert_called_once_with(logger.RUN_STATUS_SUCCESS) + + @mock.patch("official.utils.logs.logger.config_benchmark_logger") + def test_benchmark_context_failure(self, mock_config_benchmark_logger): + mock_logger = mock.MagicMock() + mock_config_benchmark_logger.return_value = mock_logger + with self.assertRaises(RuntimeError): + with logger.benchmark_context(None): + raise RuntimeError("training error") + mock_logger.on_finish.assert_called_once_with(logger.RUN_STATUS_FAILURE) + + +class BaseBenchmarkLoggerTest(tf.test.TestCase): + + def setUp(self): + super(BaseBenchmarkLoggerTest, self).setUp() + self._actual_log = tf.compat.v1.logging.info + self.logged_message = None + + def mock_log(*args, **kwargs): + self.logged_message = args + self._actual_log(*args, **kwargs) + + tf.compat.v1.logging.info = mock_log + + def tearDown(self): + super(BaseBenchmarkLoggerTest, self).tearDown() + tf.compat.v1.logging.info = self._actual_log + + def test_log_metric(self): + log = logger.BaseBenchmarkLogger() + log.log_metric("accuracy", 0.999, global_step=1e4, extras={"name": "value"}) + + expected_log_prefix = "Benchmark metric:" + self.assertRegexpMatches(str(self.logged_message), expected_log_prefix) + + +class BenchmarkFileLoggerTest(tf.test.TestCase): + + def setUp(self): + super(BenchmarkFileLoggerTest, self).setUp() + # Avoid pulling extra env vars from test environment which affects the test + # result, eg. Kokoro test has a TF_PKG env which affect the test case + # test_collect_tensorflow_environment_variables() + self.original_environ = dict(os.environ) + os.environ.clear() + + def tearDown(self): + super(BenchmarkFileLoggerTest, self).tearDown() + tf.io.gfile.rmtree(self.get_temp_dir()) + os.environ.clear() + os.environ.update(self.original_environ) + + def test_create_logging_dir(self): + non_exist_temp_dir = os.path.join(self.get_temp_dir(), "unknown_dir") + self.assertFalse(tf.io.gfile.isdir(non_exist_temp_dir)) + + logger.BenchmarkFileLogger(non_exist_temp_dir) + self.assertTrue(tf.io.gfile.isdir(non_exist_temp_dir)) + + def test_log_metric(self): + log_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + log = logger.BenchmarkFileLogger(log_dir) + log.log_metric("accuracy", 0.999, global_step=1e4, extras={"name": "value"}) + + metric_log = os.path.join(log_dir, "metric.log") + self.assertTrue(tf.io.gfile.exists(metric_log)) + with tf.io.gfile.GFile(metric_log) as f: + metric = json.loads(f.readline()) + self.assertEqual(metric["name"], "accuracy") + self.assertEqual(metric["value"], 0.999) + self.assertEqual(metric["unit"], None) + self.assertEqual(metric["global_step"], 1e4) + self.assertEqual(metric["extras"], [{"name": "name", "value": "value"}]) + + def test_log_multiple_metrics(self): + log_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + log = logger.BenchmarkFileLogger(log_dir) + log.log_metric("accuracy", 0.999, global_step=1e4, extras={"name": "value"}) + log.log_metric("loss", 0.02, global_step=1e4) + + metric_log = os.path.join(log_dir, "metric.log") + self.assertTrue(tf.io.gfile.exists(metric_log)) + with tf.io.gfile.GFile(metric_log) as f: + accuracy = json.loads(f.readline()) + self.assertEqual(accuracy["name"], "accuracy") + self.assertEqual(accuracy["value"], 0.999) + self.assertEqual(accuracy["unit"], None) + self.assertEqual(accuracy["global_step"], 1e4) + self.assertEqual(accuracy["extras"], [{"name": "name", "value": "value"}]) + + loss = json.loads(f.readline()) + self.assertEqual(loss["name"], "loss") + self.assertEqual(loss["value"], 0.02) + self.assertEqual(loss["unit"], None) + self.assertEqual(loss["global_step"], 1e4) + self.assertEqual(loss["extras"], []) + + def test_log_non_number_value(self): + log_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + log = logger.BenchmarkFileLogger(log_dir) + const = tf.constant(1) + log.log_metric("accuracy", const) + + metric_log = os.path.join(log_dir, "metric.log") + self.assertFalse(tf.io.gfile.exists(metric_log)) + + def test_log_evaluation_result(self): + eval_result = {"loss": 0.46237424, + "global_step": 207082, + "accuracy": 0.9285} + log_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + log = logger.BenchmarkFileLogger(log_dir) + log.log_evaluation_result(eval_result) + + metric_log = os.path.join(log_dir, "metric.log") + self.assertTrue(tf.io.gfile.exists(metric_log)) + with tf.io.gfile.GFile(metric_log) as f: + accuracy = json.loads(f.readline()) + self.assertEqual(accuracy["name"], "accuracy") + self.assertEqual(accuracy["value"], 0.9285) + self.assertEqual(accuracy["unit"], None) + self.assertEqual(accuracy["global_step"], 207082) + + loss = json.loads(f.readline()) + self.assertEqual(loss["name"], "loss") + self.assertEqual(loss["value"], 0.46237424) + self.assertEqual(loss["unit"], None) + self.assertEqual(loss["global_step"], 207082) + + def test_log_evaluation_result_with_invalid_type(self): + eval_result = "{'loss': 0.46237424, 'global_step': 207082}" + log_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + log = logger.BenchmarkFileLogger(log_dir) + log.log_evaluation_result(eval_result) + + metric_log = os.path.join(log_dir, "metric.log") + self.assertFalse(tf.io.gfile.exists(metric_log)) + + @mock.patch("official.utils.logs.logger._gather_run_info") + def test_log_run_info(self, mock_gather_run_info): + log_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + log = logger.BenchmarkFileLogger(log_dir) + run_info = {"model_name": "model_name", + "dataset": "dataset_name", + "run_info": "run_value"} + mock_gather_run_info.return_value = run_info + log.log_run_info("model_name", "dataset_name", {}) + + run_log = os.path.join(log_dir, "benchmark_run.log") + self.assertTrue(tf.io.gfile.exists(run_log)) + with tf.io.gfile.GFile(run_log) as f: + run_info = json.loads(f.readline()) + self.assertEqual(run_info["model_name"], "model_name") + self.assertEqual(run_info["dataset"], "dataset_name") + self.assertEqual(run_info["run_info"], "run_value") + + def test_collect_tensorflow_info(self): + run_info = {} + logger._collect_tensorflow_info(run_info) + self.assertNotEqual(run_info["tensorflow_version"], {}) + self.assertEqual(run_info["tensorflow_version"]["version"], + tf.version.VERSION) + self.assertEqual(run_info["tensorflow_version"]["git_hash"], + tf.version.GIT_VERSION) + + def test_collect_run_params(self): + run_info = {} + run_parameters = { + "batch_size": 32, + "synthetic_data": True, + "train_epochs": 100.00, + "dtype": "fp16", + "resnet_size": 50, + "random_tensor": tf.constant(2.0) + } + logger._collect_run_params(run_info, run_parameters) + self.assertEqual(len(run_info["run_parameters"]), 6) + self.assertEqual(run_info["run_parameters"][0], + {"name": "batch_size", "long_value": 32}) + self.assertEqual(run_info["run_parameters"][1], + {"name": "dtype", "string_value": "fp16"}) + v1_tensor = {"name": "random_tensor", "string_value": + "Tensor(\"Const:0\", shape=(), dtype=float32)"} + v2_tensor = {"name": "random_tensor", "string_value": + "tf.Tensor(2.0, shape=(), dtype=float32)"} + self.assertIn(run_info["run_parameters"][2], [v1_tensor, v2_tensor]) + + + self.assertEqual(run_info["run_parameters"][3], + {"name": "resnet_size", "long_value": 50}) + self.assertEqual(run_info["run_parameters"][4], + {"name": "synthetic_data", "bool_value": "True"}) + self.assertEqual(run_info["run_parameters"][5], + {"name": "train_epochs", "float_value": 100.00}) + + def test_collect_tensorflow_environment_variables(self): + os.environ["TF_ENABLE_WINOGRAD_NONFUSED"] = "1" + os.environ["TF_OTHER"] = "2" + os.environ["OTHER"] = "3" + + run_info = {} + logger._collect_tensorflow_environment_variables(run_info) + self.assertIsNotNone(run_info["tensorflow_environment_variables"]) + expected_tf_envs = [ + {"name": "TF_ENABLE_WINOGRAD_NONFUSED", "value": "1"}, + {"name": "TF_OTHER", "value": "2"}, + ] + self.assertEqual(run_info["tensorflow_environment_variables"], + expected_tf_envs) + + def test_collect_memory_info(self): + run_info = {"machine_config": {}} + logger._collect_memory_info(run_info) + self.assertIsNotNone(run_info["machine_config"]["memory_total"]) + self.assertIsNotNone(run_info["machine_config"]["memory_available"]) + + +@unittest.skipIf(bigquery is None, "Bigquery dependency is not installed.") +class BenchmarkBigQueryLoggerTest(tf.test.TestCase): + + def setUp(self): + super(BenchmarkBigQueryLoggerTest, self).setUp() + # Avoid pulling extra env vars from test environment which affects the test + # result, eg. Kokoro test has a TF_PKG env which affect the test case + # test_collect_tensorflow_environment_variables() + self.original_environ = dict(os.environ) + os.environ.clear() + + self.mock_bq_uploader = mock.MagicMock() + self.logger = logger.BenchmarkBigQueryLogger( + self.mock_bq_uploader, "dataset", "run_table", "run_status_table", + "metric_table", "run_id") + + def tearDown(self): + super(BenchmarkBigQueryLoggerTest, self).tearDown() + tf.io.gfile.rmtree(self.get_temp_dir()) + os.environ.clear() + os.environ.update(self.original_environ) + + def test_log_metric(self): + self.logger.log_metric( + "accuracy", 0.999, global_step=1e4, extras={"name": "value"}) + expected_metric_json = [{ + "name": "accuracy", + "value": 0.999, + "unit": None, + "global_step": 1e4, + "timestamp": mock.ANY, + "extras": [{"name": "name", "value": "value"}] + }] + # log_metric will call upload_benchmark_metric_json in a separate thread. + # Give it some grace period for the new thread before assert. + time.sleep(1) + self.mock_bq_uploader.upload_benchmark_metric_json.assert_called_once_with( + "dataset", "metric_table", "run_id", expected_metric_json) + + @mock.patch("official.utils.logs.logger._gather_run_info") + def test_log_run_info(self, mock_gather_run_info): + run_info = {"model_name": "model_name", + "dataset": "dataset_name", + "run_info": "run_value"} + mock_gather_run_info.return_value = run_info + self.logger.log_run_info("model_name", "dataset_name", {}) + # log_metric will call upload_benchmark_metric_json in a separate thread. + # Give it some grace period for the new thread before assert. + time.sleep(1) + self.mock_bq_uploader.upload_benchmark_run_json.assert_called_once_with( + "dataset", "run_table", "run_id", run_info) + self.mock_bq_uploader.insert_run_status.assert_called_once_with( + "dataset", "run_status_table", "run_id", "running") + + def test_on_finish(self): + self.logger.on_finish(logger.RUN_STATUS_SUCCESS) + # log_metric will call upload_benchmark_metric_json in a separate thread. + # Give it some grace period for the new thread before assert. + time.sleep(1) + self.mock_bq_uploader.update_run_status.assert_called_once_with( + "dataset", "run_status_table", "run_id", logger.RUN_STATUS_SUCCESS) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/metric_hook.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/metric_hook.py new file mode 100644 index 0000000..f408e3e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/metric_hook.py @@ -0,0 +1,97 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Session hook for logging benchmark metric.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf # pylint: disable=g-bad-import-order + + +class LoggingMetricHook(tf.estimator.LoggingTensorHook): + """Hook to log benchmark metric information. + + This hook is very similar as tf.train.LoggingTensorHook, which logs given + tensors every N local steps, every N seconds, or at the end. The metric + information will be logged to given log_dir or via metric_logger in JSON + format, which can be consumed by data analysis pipeline later. + + Note that if `at_end` is True, `tensors` should not include any tensor + whose evaluation produces a side effect such as consuming additional inputs. + """ + + def __init__(self, tensors, metric_logger=None, + every_n_iter=None, every_n_secs=None, at_end=False): + """Initializer for LoggingMetricHook. + + Args: + tensors: `dict` that maps string-valued tags to tensors/tensor names, + or `iterable` of tensors/tensor names. + metric_logger: instance of `BenchmarkLogger`, the benchmark logger that + hook should use to write the log. + every_n_iter: `int`, print the values of `tensors` once every N local + steps taken on the current worker. + every_n_secs: `int` or `float`, print the values of `tensors` once every N + seconds. Exactly one of `every_n_iter` and `every_n_secs` should be + provided. + at_end: `bool` specifying whether to print the values of `tensors` at the + end of the run. + + Raises: + ValueError: + 1. `every_n_iter` is non-positive, or + 2. Exactly one of every_n_iter and every_n_secs should be provided. + 3. Exactly one of log_dir and metric_logger should be provided. + """ + super(LoggingMetricHook, self).__init__( + tensors=tensors, + every_n_iter=every_n_iter, + every_n_secs=every_n_secs, + at_end=at_end) + + if metric_logger is None: + raise ValueError("metric_logger should be provided.") + self._logger = metric_logger + + def begin(self): + super(LoggingMetricHook, self).begin() + self._global_step_tensor = tf.compat.v1.train.get_global_step() + if self._global_step_tensor is None: + raise RuntimeError( + "Global step should be created to use LoggingMetricHook.") + if self._global_step_tensor.name not in self._current_tensors: + self._current_tensors[self._global_step_tensor.name] = ( + self._global_step_tensor) + + def after_run(self, unused_run_context, run_values): + # should_trigger is a internal state that populated at before_run, and it is + # using self_timer to determine whether it should trigger. + if self._should_trigger: + self._log_metric(run_values.results) + + self._iter_count += 1 + + def end(self, session): + if self._log_at_end: + values = session.run(self._current_tensors) + self._log_metric(values) + + def _log_metric(self, tensor_values): + self._timer.update_last_triggered_step(self._iter_count) + global_step = tensor_values[self._global_step_tensor.name] + # self._tag_order is populated during the init of LoggingTensorHook + for tag in self._tag_order: + self._logger.log_metric(tag, tensor_values[tag], global_step=global_step) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/metric_hook_test.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/metric_hook_test.py new file mode 100644 index 0000000..870ed6e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/metric_hook_test.py @@ -0,0 +1,217 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for metric_hook.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tempfile +import time + +import tensorflow as tf # pylint: disable=g-bad-import-order +from tensorflow.python.training import monitored_session # pylint: disable=g-bad-import-order + +from official.utils.logs import metric_hook +from official.utils.testing import mock_lib + + +class LoggingMetricHookTest(tf.test.TestCase): + """Tests for LoggingMetricHook.""" + + def setUp(self): + super(LoggingMetricHookTest, self).setUp() + + self._log_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + self._logger = mock_lib.MockBenchmarkLogger() + + def tearDown(self): + super(LoggingMetricHookTest, self).tearDown() + tf.io.gfile.rmtree(self.get_temp_dir()) + + def test_illegal_args(self): + with self.assertRaisesRegexp(ValueError, "nvalid every_n_iter"): + metric_hook.LoggingMetricHook(tensors=["t"], every_n_iter=0) + with self.assertRaisesRegexp(ValueError, "nvalid every_n_iter"): + metric_hook.LoggingMetricHook(tensors=["t"], every_n_iter=-10) + with self.assertRaisesRegexp(ValueError, "xactly one of"): + metric_hook.LoggingMetricHook( + tensors=["t"], every_n_iter=5, every_n_secs=5) + with self.assertRaisesRegexp(ValueError, "xactly one of"): + metric_hook.LoggingMetricHook(tensors=["t"]) + with self.assertRaisesRegexp(ValueError, "metric_logger"): + metric_hook.LoggingMetricHook(tensors=["t"], every_n_iter=5) + + def test_print_at_end_only(self): + with tf.Graph().as_default(), tf.compat.v1.Session() as sess: + tf.compat.v1.train.get_or_create_global_step() + t = tf.constant(42.0, name="foo") + train_op = tf.constant(3) + hook = metric_hook.LoggingMetricHook( + tensors=[t.name], at_end=True, metric_logger=self._logger) + hook.begin() + mon_sess = monitored_session._HookedSession(sess, [hook]) # pylint: disable=protected-access + sess.run(tf.compat.v1.global_variables_initializer()) + + for _ in range(3): + mon_sess.run(train_op) + self.assertEqual(self._logger.logged_metric, []) + + hook.end(sess) + self.assertEqual(len(self._logger.logged_metric), 1) + metric = self._logger.logged_metric[0] + self.assertRegexpMatches(metric["name"], "foo") + self.assertEqual(metric["value"], 42.0) + self.assertEqual(metric["unit"], None) + self.assertEqual(metric["global_step"], 0) + + def test_global_step_not_found(self): + with tf.Graph().as_default(): + t = tf.constant(42.0, name="foo") + hook = metric_hook.LoggingMetricHook( + tensors=[t.name], at_end=True, metric_logger=self._logger) + + with self.assertRaisesRegexp( + RuntimeError, "should be created to use LoggingMetricHook."): + hook.begin() + + def test_log_tensors(self): + with tf.Graph().as_default(), tf.compat.v1.Session() as sess: + tf.compat.v1.train.get_or_create_global_step() + t1 = tf.constant(42.0, name="foo") + t2 = tf.constant(43.0, name="bar") + train_op = tf.constant(3) + hook = metric_hook.LoggingMetricHook( + tensors=[t1, t2], at_end=True, metric_logger=self._logger) + hook.begin() + mon_sess = monitored_session._HookedSession(sess, [hook]) # pylint: disable=protected-access + sess.run(tf.compat.v1.global_variables_initializer()) + + for _ in range(3): + mon_sess.run(train_op) + self.assertEqual(self._logger.logged_metric, []) + + hook.end(sess) + self.assertEqual(len(self._logger.logged_metric), 2) + metric1 = self._logger.logged_metric[0] + self.assertRegexpMatches(str(metric1["name"]), "foo") + self.assertEqual(metric1["value"], 42.0) + self.assertEqual(metric1["unit"], None) + self.assertEqual(metric1["global_step"], 0) + + metric2 = self._logger.logged_metric[1] + self.assertRegexpMatches(str(metric2["name"]), "bar") + self.assertEqual(metric2["value"], 43.0) + self.assertEqual(metric2["unit"], None) + self.assertEqual(metric2["global_step"], 0) + + def _validate_print_every_n_steps(self, sess, at_end): + t = tf.constant(42.0, name="foo") + + train_op = tf.constant(3) + hook = metric_hook.LoggingMetricHook( + tensors=[t.name], every_n_iter=10, at_end=at_end, + metric_logger=self._logger) + hook.begin() + mon_sess = monitored_session._HookedSession(sess, [hook]) # pylint: disable=protected-access + sess.run(tf.compat.v1.global_variables_initializer()) + mon_sess.run(train_op) + self.assertRegexpMatches(str(self._logger.logged_metric), t.name) + for _ in range(3): + self._logger.logged_metric = [] + for _ in range(9): + mon_sess.run(train_op) + # assertNotRegexpMatches is not supported by python 3.1 and later + self.assertEqual(str(self._logger.logged_metric).find(t.name), -1) + mon_sess.run(train_op) + self.assertRegexpMatches(str(self._logger.logged_metric), t.name) + + # Add additional run to verify proper reset when called multiple times. + self._logger.logged_metric = [] + mon_sess.run(train_op) + # assertNotRegexpMatches is not supported by python 3.1 and later + self.assertEqual(str(self._logger.logged_metric).find(t.name), -1) + + self._logger.logged_metric = [] + hook.end(sess) + if at_end: + self.assertRegexpMatches(str(self._logger.logged_metric), t.name) + else: + # assertNotRegexpMatches is not supported by python 3.1 and later + self.assertEqual(str(self._logger.logged_metric).find(t.name), -1) + + def test_print_every_n_steps(self): + with tf.Graph().as_default(), tf.compat.v1.Session() as sess: + tf.compat.v1.train.get_or_create_global_step() + self._validate_print_every_n_steps(sess, at_end=False) + # Verify proper reset. + self._validate_print_every_n_steps(sess, at_end=False) + + def test_print_every_n_steps_and_end(self): + with tf.Graph().as_default(), tf.compat.v1.Session() as sess: + tf.compat.v1.train.get_or_create_global_step() + self._validate_print_every_n_steps(sess, at_end=True) + # Verify proper reset. + self._validate_print_every_n_steps(sess, at_end=True) + + def _validate_print_every_n_secs(self, sess, at_end): + t = tf.constant(42.0, name="foo") + train_op = tf.constant(3) + + hook = metric_hook.LoggingMetricHook( + tensors=[t.name], every_n_secs=1.0, at_end=at_end, + metric_logger=self._logger) + hook.begin() + mon_sess = monitored_session._HookedSession(sess, [hook]) # pylint: disable=protected-access + sess.run(tf.compat.v1.global_variables_initializer()) + + mon_sess.run(train_op) + self.assertRegexpMatches(str(self._logger.logged_metric), t.name) + + # assertNotRegexpMatches is not supported by python 3.1 and later + self._logger.logged_metric = [] + mon_sess.run(train_op) + self.assertEqual(str(self._logger.logged_metric).find(t.name), -1) + time.sleep(1.0) + + self._logger.logged_metric = [] + mon_sess.run(train_op) + self.assertRegexpMatches(str(self._logger.logged_metric), t.name) + + self._logger.logged_metric = [] + hook.end(sess) + if at_end: + self.assertRegexpMatches(str(self._logger.logged_metric), t.name) + else: + # assertNotRegexpMatches is not supported by python 3.1 and later + self.assertEqual(str(self._logger.logged_metric).find(t.name), -1) + + def test_print_every_n_secs(self): + with tf.Graph().as_default(), tf.compat.v1.Session() as sess: + tf.compat.v1.train.get_or_create_global_step() + self._validate_print_every_n_secs(sess, at_end=False) + # Verify proper reset. + self._validate_print_every_n_secs(sess, at_end=False) + + def test_print_every_n_secs_and_end(self): + with tf.Graph().as_default(), tf.compat.v1.Session() as sess: + tf.compat.v1.train.get_or_create_global_step() + self._validate_print_every_n_secs(sess, at_end=True) + # Verify proper reset. + self._validate_print_every_n_secs(sess, at_end=True) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/mlperf_helper.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/mlperf_helper.py new file mode 100644 index 0000000..c9c0434 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/logs/mlperf_helper.py @@ -0,0 +1,192 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Wrapper for the mlperf logging utils. + +MLPerf compliance logging is only desired under a limited set of circumstances. +This module is intended to keep users from needing to consider logging (or +install the module) unless they are performing mlperf runs. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from collections import namedtuple +import json +import os +import re +import subprocess +import sys +import typing + +import tensorflow as tf + +_MIN_VERSION = (0, 0, 10) +_STACK_OFFSET = 2 + +SUDO = "sudo" if os.geteuid() else "" + +# This indirection is used in docker. +DROP_CACHE_LOC = os.getenv("DROP_CACHE_LOC", "/proc/sys/vm/drop_caches") + +_NCF_PREFIX = "NCF_RAW_" + +# TODO(robieta): move line parsing to mlperf util +_PREFIX = r"(?:{})?:::MLPv([0-9]+).([0-9]+).([0-9]+)".format(_NCF_PREFIX) +_BENCHMARK = r"([a-zA-Z0-9_]+)" +_TIMESTAMP = r"([0-9]+\.[0-9]+)" +_CALLSITE = r"\((.+):([0-9]+)\)" +_TAG = r"([a-zA-Z0-9_]+)" +_VALUE = r"(.*)" + +ParsedLine = namedtuple("ParsedLine", ["version", "benchmark", "timestamp", + "callsite", "tag", "value"]) + +LINE_PATTERN = re.compile( + "^{prefix} {benchmark} {timestamp} {callsite} {tag}(: |$){value}?$".format( + prefix=_PREFIX, benchmark=_BENCHMARK, timestamp=_TIMESTAMP, + callsite=_CALLSITE, tag=_TAG, value=_VALUE)) + + +def parse_line(line): # type: (str) -> typing.Optional[ParsedLine] + match = LINE_PATTERN.match(line.strip()) + if not match: + return + + major, minor, micro, benchmark, timestamp = match.groups()[:5] + call_file, call_line, tag, _, value = match.groups()[5:] + + return ParsedLine(version=(int(major), int(minor), int(micro)), + benchmark=benchmark, timestamp=timestamp, + callsite=(call_file, call_line), tag=tag, value=value) + + +def unparse_line(parsed_line): # type: (ParsedLine) -> str + version_str = "{}.{}.{}".format(*parsed_line.version) + callsite_str = "({}:{})".format(*parsed_line.callsite) + value_str = ": {}".format(parsed_line.value) if parsed_line.value else "" + return ":::MLPv{} {} {} {} {} {}".format( + version_str, parsed_line.benchmark, parsed_line.timestamp, callsite_str, + parsed_line.tag, value_str) + + +def get_mlperf_log(): + """Shielded import of mlperf_log module.""" + try: + import mlperf_compliance + + def test_mlperf_log_pip_version(): + """Check that mlperf_compliance is up to date.""" + import pkg_resources + version = pkg_resources.get_distribution("mlperf_compliance") + version = tuple(int(i) for i in version.version.split(".")) + if version < _MIN_VERSION: + tf.compat.v1.logging.warning( + "mlperf_compliance is version {}, must be >= {}".format( + ".".join([str(i) for i in version]), + ".".join([str(i) for i in _MIN_VERSION]))) + raise ImportError + return mlperf_compliance.mlperf_log + + mlperf_log = test_mlperf_log_pip_version() + + except ImportError: + mlperf_log = None + + return mlperf_log + + +class Logger(object): + """MLPerf logger indirection class. + + This logger only logs for MLPerf runs, and prevents various errors associated + with not having the mlperf_compliance package installed. + """ + class Tags(object): + def __init__(self, mlperf_log): + self._enabled = False + self._mlperf_log = mlperf_log + + def __getattr__(self, item): + if self._mlperf_log is None or not self._enabled: + return + return getattr(self._mlperf_log, item) + + def __init__(self): + self._enabled = False + self._mlperf_log = get_mlperf_log() + self.tags = self.Tags(self._mlperf_log) + + def __call__(self, enable=False): + if enable and self._mlperf_log is None: + raise ImportError("MLPerf logging was requested, but mlperf_compliance " + "module could not be loaded.") + + self._enabled = enable + self.tags._enabled = enable + return self + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + self._enabled = False + self.tags._enabled = False + + @property + def log_file(self): + if self._mlperf_log is None: + return + return self._mlperf_log.LOG_FILE + + @property + def enabled(self): + return self._enabled + + def ncf_print(self, key, value=None, stack_offset=_STACK_OFFSET, + deferred=False, extra_print=False, prefix=_NCF_PREFIX): + if self._mlperf_log is None or not self.enabled: + return + self._mlperf_log.ncf_print(key=key, value=value, stack_offset=stack_offset, + deferred=deferred, extra_print=extra_print, + prefix=prefix) + + def set_ncf_root(self, path): + if self._mlperf_log is None: + return + self._mlperf_log.ROOT_DIR_NCF = path + + +LOGGER = Logger() +ncf_print, set_ncf_root = LOGGER.ncf_print, LOGGER.set_ncf_root +TAGS = LOGGER.tags + + +def clear_system_caches(): + if not LOGGER.enabled: + return + ret_code = subprocess.call( + ["sync && echo 3 | {} tee {}".format(SUDO, DROP_CACHE_LOC)], + shell=True) + + if ret_code: + raise ValueError("Failed to clear caches") + + +if __name__ == "__main__": + tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO) + with LOGGER(True): + ncf_print(key=TAGS.RUN_START) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/callstack_sampler.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/callstack_sampler.py new file mode 100644 index 0000000..984f133 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/callstack_sampler.py @@ -0,0 +1,62 @@ +"""A simple Python callstack sampler.""" + +import contextlib +import datetime +import signal +import traceback + + +class CallstackSampler(object): + """A simple signal-based Python callstack sampler. + """ + + def __init__(self, interval=None): + self.stacks = [] + self.interval = 0.001 if interval is None else interval + + def _sample(self, signum, frame): + """Samples the current stack.""" + del signum + stack = traceback.extract_stack(frame) + formatted_stack = [] + formatted_stack.append(datetime.datetime.utcnow()) + for filename, lineno, function_name, text in stack: + formatted_frame = '{}:{}({})({})'.format(filename, lineno, function_name, + text) + formatted_stack.append(formatted_frame) + self.stacks.append(formatted_stack) + signal.setitimer(signal.ITIMER_VIRTUAL, self.interval, 0) + + @contextlib.contextmanager + def profile(self): + signal.signal(signal.SIGVTALRM, self._sample) + signal.setitimer(signal.ITIMER_VIRTUAL, self.interval, 0) + try: + yield + finally: + signal.setitimer(signal.ITIMER_VIRTUAL, 0) + + def save(self, fname): + with open(fname, 'w') as f: + for s in self.stacks: + for l in s: + f.write('%s\n' % l) + f.write('\n') + + +@contextlib.contextmanager +def callstack_sampling(filename, interval=None): + """Periodically samples the Python callstack. + + Args: + filename: the filename + interval: the sampling interval, in seconds. Defaults to 0.001. + + Yields: + nothing + """ + sampler = CallstackSampler(interval=interval) + with sampler.profile(): + yield + sampler.save(filename) + diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/distribution_utils.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/distribution_utils.py new file mode 100644 index 0000000..fd5c742 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/distribution_utils.py @@ -0,0 +1,338 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Helper functions for running models in a distributed setting.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import os +import random +import string +import tensorflow.compat.v2 as tf + +from official.utils.misc import tpu_lib + + +def _collective_communication(all_reduce_alg): + """Return a CollectiveCommunication based on all_reduce_alg. + + Args: + all_reduce_alg: a string specifying which collective communication to pick, + or None. + + Returns: + tf.distribute.experimental.CollectiveCommunication object + + Raises: + ValueError: if `all_reduce_alg` not in [None, 'ring', 'nccl'] + """ + collective_communication_options = { + None: tf.distribute.experimental.CollectiveCommunication.AUTO, + "ring": tf.distribute.experimental.CollectiveCommunication.RING, + "nccl": tf.distribute.experimental.CollectiveCommunication.NCCL + } + if all_reduce_alg not in collective_communication_options: + raise ValueError( + "When used with `multi_worker_mirrored`, valid values for " + "all_reduce_alg are ['ring', 'nccl']. Supplied value: {}".format( + all_reduce_alg)) + return collective_communication_options[all_reduce_alg] + + +def _mirrored_cross_device_ops(all_reduce_alg, num_packs): + """Return a CrossDeviceOps based on all_reduce_alg and num_packs. + + Args: + all_reduce_alg: a string specifying which cross device op to pick, or None. + num_packs: an integer specifying number of packs for the cross device op. + + Returns: + tf.distribute.CrossDeviceOps object or None. + + Raises: + ValueError: if `all_reduce_alg` not in [None, 'nccl', 'hierarchical_copy']. + """ + if all_reduce_alg is None: + return None + mirrored_all_reduce_options = { + "nccl": tf.distribute.NcclAllReduce, + "hierarchical_copy": tf.distribute.HierarchicalCopyAllReduce + } + if all_reduce_alg not in mirrored_all_reduce_options: + raise ValueError( + "When used with `mirrored`, valid values for all_reduce_alg are " + "['nccl', 'hierarchical_copy']. Supplied value: {}".format( + all_reduce_alg)) + cross_device_ops_class = mirrored_all_reduce_options[all_reduce_alg] + return cross_device_ops_class(num_packs=num_packs) + + +def get_distribution_strategy(distribution_strategy="mirrored", + num_gpus=0, + all_reduce_alg=None, + num_packs=1, + tpu_address=None): + """Return a DistributionStrategy for running the model. + + Args: + distribution_strategy: a string specifying which distribution strategy to + use. Accepted values are 'off', 'one_device', 'mirrored', + 'parameter_server', 'multi_worker_mirrored', and 'tpu' -- case insensitive. + 'off' means not to use Distribution Strategy; 'tpu' means to use + TPUStrategy using `tpu_address`. + num_gpus: Number of GPUs to run this model. + all_reduce_alg: Optional. Specifies which algorithm to use when performing + all-reduce. For `MirroredStrategy`, valid values are "nccl" and + "hierarchical_copy". For `MultiWorkerMirroredStrategy`, valid values are + "ring" and "nccl". If None, DistributionStrategy will choose based on + device topology. + num_packs: Optional. Sets the `num_packs` in `tf.distribute.NcclAllReduce` + or `tf.distribute.HierarchicalCopyAllReduce` for `MirroredStrategy`. + tpu_address: Optional. String that represents TPU to connect to. Must not + be None if `distribution_strategy` is set to `tpu`. + Returns: + tf.distribute.DistibutionStrategy object. + Raises: + ValueError: if `distribution_strategy` is 'off' or 'one_device' and + `num_gpus` is larger than 1; or `num_gpus` is negative or if + `distribution_strategy` is `tpu` but `tpu_address` is not specified. + """ + if num_gpus < 0: + raise ValueError("`num_gpus` can not be negative.") + + distribution_strategy = distribution_strategy.lower() + if distribution_strategy == "off": + if num_gpus > 1: + raise ValueError( + "When {} GPUs are specified, distribution_strategy " + "flag cannot be set to 'off'.".format(num_gpus)) + return None + + if distribution_strategy == "tpu": + # When tpu_address is an empty string, we communicate with local TPUs. + cluster_resolver = tpu_lib.tpu_initialize(tpu_address) + return tf.distribute.experimental.TPUStrategy(cluster_resolver) + + if distribution_strategy == "multi_worker_mirrored": + return tf.distribute.experimental.MultiWorkerMirroredStrategy( + communication=_collective_communication(all_reduce_alg)) + + if distribution_strategy == "one_device": + if num_gpus == 0: + return tf.distribute.OneDeviceStrategy("device:CPU:0") + if num_gpus > 1: + raise ValueError("`OneDeviceStrategy` can not be used for more than " + "one device.") + return tf.distribute.OneDeviceStrategy("device:GPU:0") + + if distribution_strategy == "mirrored": + if num_gpus == 0: + devices = ["device:CPU:0"] + else: + devices = ["device:GPU:%d" % i for i in range(num_gpus)] + return tf.distribute.MirroredStrategy( + devices=devices, + cross_device_ops=_mirrored_cross_device_ops(all_reduce_alg, num_packs)) + + if distribution_strategy == "parameter_server": + return tf.distribute.experimental.ParameterServerStrategy() + + raise ValueError( + "Unrecognized Distribution Strategy: %r" % distribution_strategy) + + +def per_replica_batch_size(batch_size, num_gpus): + """For multi-gpu, batch-size must be a multiple of the number of GPUs. + + + Note that distribution strategy handles this automatically when used with + Keras. For using with Estimator, we need to get per GPU batch. + + Args: + batch_size: Global batch size to be divided among devices. This should be + equal to num_gpus times the single-GPU batch_size for multi-gpu training. + num_gpus: How many GPUs are used with DistributionStrategies. + + Returns: + Batch size per device. + + Raises: + ValueError: if batch_size is not divisible by number of devices + """ + if num_gpus <= 1: + return batch_size + + remainder = batch_size % num_gpus + if remainder: + err = ('When running with multiple GPUs, batch size ' + 'must be a multiple of the number of available GPUs. Found {} ' + 'GPUs with a batch size of {}; try --batch_size={} instead.' + ).format(num_gpus, batch_size, batch_size - remainder) + raise ValueError(err) + return int(batch_size / num_gpus) + + +# The `SyntheticDataset` is a temporary solution for generating synthetic data +# directly on devices. It is only useful for Keras with Distribution +# Strategies. We will have better support in `tf.data` or Distribution Strategy +# later. +class SyntheticDataset(object): + """A dataset that generates synthetic data on each device.""" + + def __init__(self, dataset, split_by=1): + # dataset.take(1) doesn't have GPU kernel. + with tf.device('device:CPU:0'): + tensor = tf.data.experimental.get_single_element(dataset.take(1)) + flat_tensor = tf.nest.flatten(tensor) + variable_data = [] + initializers = [] + for t in flat_tensor: + rebatched_t = tf.split(t, num_or_size_splits=split_by, axis=0)[0] + assert rebatched_t.shape.is_fully_defined(), rebatched_t.shape + v = tf.compat.v1.get_local_variable(self._random_name(), + initializer=rebatched_t) + variable_data.append(v) + initializers.append(v.initializer) + input_data = tf.nest.pack_sequence_as(tensor, variable_data) + self._iterator = SyntheticIterator(input_data, initializers) + + def _random_name(self, size=10, chars=string.ascii_uppercase + string.digits): + return ''.join(random.choice(chars) for _ in range(size)) + + def __iter__(self): + return self._iterator + + def make_one_shot_iterator(self): + return self._iterator + + def make_initializable_iterator(self): + return self._iterator + + +class SyntheticIterator(object): + """A dataset that generates synthetic data on each device.""" + + def __init__(self, input_data, initializers): + self._input_data = input_data + self._initializers = initializers + + def get_next(self): + return self._input_data + + def next(self): + return self.__next__() + + def __next__(self): + try: + return self.get_next() + except tf.errors.OutOfRangeError: + raise StopIteration + + def initialize(self): + if tf.executing_eagerly(): + return tf.no_op() + else: + return self._initializers + + +def _monkey_patch_dataset_method(strategy): + """Monkey-patch `strategy`'s `make_dataset_iterator` method.""" + def make_dataset(self, dataset): + tf.compat.v1.logging.info('Using pure synthetic data.') + with self.scope(): + if self.extended._global_batch_size: # pylint: disable=protected-access + return SyntheticDataset(dataset, self.num_replicas_in_sync) + else: + return SyntheticDataset(dataset) + + def make_iterator(self, dataset): + dist_dataset = make_dataset(self, dataset) + return iter(dist_dataset) + + strategy.orig_make_dataset_iterator = strategy.make_dataset_iterator + strategy.make_dataset_iterator = make_iterator + strategy.orig_distribute_dataset = strategy.experimental_distribute_dataset + strategy.experimental_distribute_dataset = make_dataset + + +def _undo_monkey_patch_dataset_method(strategy): + if hasattr(strategy, 'orig_make_dataset_iterator'): + strategy.make_dataset_iterator = strategy.orig_make_dataset_iterator + if hasattr(strategy, 'orig_distribute_dataset'): + strategy.make_dataset_iterator = strategy.orig_distribute_dataset + + +def set_up_synthetic_data(): + _monkey_patch_dataset_method(tf.distribute.OneDeviceStrategy) + _monkey_patch_dataset_method(tf.distribute.MirroredStrategy) + _monkey_patch_dataset_method( + tf.distribute.experimental.MultiWorkerMirroredStrategy) + + +def undo_set_up_synthetic_data(): + _undo_monkey_patch_dataset_method(tf.distribute.OneDeviceStrategy) + _undo_monkey_patch_dataset_method(tf.distribute.MirroredStrategy) + _undo_monkey_patch_dataset_method( + tf.distribute.experimental.MultiWorkerMirroredStrategy) + + +def configure_cluster(worker_hosts=None, task_index=-1): + """Set multi-worker cluster spec in TF_CONFIG environment variable. + + Args: + worker_hosts: comma-separated list of worker ip:port pairs. + + Returns: + Number of workers in the cluster. + """ + tf_config = json.loads(os.environ.get('TF_CONFIG', '{}')) + if tf_config: + num_workers = (len(tf_config['cluster'].get('chief', [])) + + len(tf_config['cluster'].get('worker', []))) + elif worker_hosts: + workers = worker_hosts.split(',') + num_workers = len(workers) + if num_workers > 1 and task_index < 0: + raise ValueError('Must specify task_index when number of workers > 1') + task_index = 0 if num_workers == 1 else task_index + os.environ['TF_CONFIG'] = json.dumps({ + 'cluster': { + 'worker': workers + }, + 'task': {'type': 'worker', 'index': task_index} + }) + else: + num_workers = 1 + return num_workers + + +def get_strategy_scope(strategy): + if strategy: + strategy_scope = strategy.scope() + else: + strategy_scope = DummyContextManager() + + return strategy_scope + + +class DummyContextManager(object): + + def __enter__(self): + pass + + def __exit__(self, *args): + pass diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/distribution_utils_test.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/distribution_utils_test.py new file mode 100644 index 0000000..856c3b3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/distribution_utils_test.py @@ -0,0 +1,65 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" Tests for distribution util functions.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow.compat.v2 as tf + +from official.utils.misc import distribution_utils + + +class GetDistributionStrategyTest(tf.test.TestCase): + """Tests for get_distribution_strategy.""" + def test_one_device_strategy_cpu(self): + ds = distribution_utils.get_distribution_strategy(num_gpus=0) + self.assertEquals(ds.num_replicas_in_sync, 1) + self.assertEquals(len(ds.extended.worker_devices), 1) + self.assertIn('CPU', ds.extended.worker_devices[0]) + + def test_one_device_strategy_gpu(self): + ds = distribution_utils.get_distribution_strategy(num_gpus=1) + self.assertEquals(ds.num_replicas_in_sync, 1) + self.assertEquals(len(ds.extended.worker_devices), 1) + self.assertIn('GPU', ds.extended.worker_devices[0]) + + def test_mirrored_strategy(self): + ds = distribution_utils.get_distribution_strategy(num_gpus=5) + self.assertEquals(ds.num_replicas_in_sync, 5) + self.assertEquals(len(ds.extended.worker_devices), 5) + for device in ds.extended.worker_devices: + self.assertIn('GPU', device) + + +class PerReplicaBatchSizeTest(tf.test.TestCase): + """Tests for per_replica_batch_size.""" + + def test_batch_size(self): + self.assertEquals( + distribution_utils.per_replica_batch_size(147, num_gpus=0), 147) + self.assertEquals( + distribution_utils.per_replica_batch_size(147, num_gpus=1), 147) + self.assertEquals( + distribution_utils.per_replica_batch_size(147, num_gpus=7), 21) + + def test_batch_size_with_remainder(self): + with self.assertRaises(ValueError): + distribution_utils.per_replica_batch_size(147, num_gpus=5) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/keras_utils.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/keras_utils.py new file mode 100644 index 0000000..fd014f1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/keras_utils.py @@ -0,0 +1,262 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Helper functions for the Keras implementations of models.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import multiprocessing +import os +import time + +from absl import logging +import tensorflow.compat.v2 as tf +from tensorflow.python import tf2 +from tensorflow.python.profiler import profiler as profiler + + +class BatchTimestamp(object): + """A structure to store batch time stamp.""" + + def __init__(self, batch_index, timestamp): + self.batch_index = batch_index + self.timestamp = timestamp + + def __repr__(self): + return "'BatchTimestamp'".format( + self.batch_index, self.timestamp) + + +class TimeHistory(tf.keras.callbacks.Callback): + """Callback for Keras models.""" + + def __init__(self, batch_size, log_steps, logdir=None): + """Callback for logging performance. + + Args: + batch_size: Total batch size. + log_steps: Interval of steps between logging of batch level stats. + logdir: Optional directory to write TensorBoard summaries. + """ + # TODO(wcromar): remove this parameter and rely on `logs` parameter of + # on_train_batch_end() + self.batch_size = batch_size + super(TimeHistory, self).__init__() + self.log_steps = log_steps + self.last_log_step = 0 + self.steps_before_epoch = 0 + self.steps_in_epoch = 0 + self.start_time = None + + if logdir: + self.summary_writer = tf.summary.create_file_writer(logdir) + else: + self.summary_writer = None + + # Logs start of step 1 then end of each step based on log_steps interval. + self.timestamp_log = [] + + # Records the time each epoch takes to run from start to finish of epoch. + self.epoch_runtime_log = [] + + @property + def global_steps(self): + """The current 1-indexed global step.""" + return self.steps_before_epoch + self.steps_in_epoch + + @property + def average_steps_per_second(self): + """The average training steps per second across all epochs.""" + return self.global_steps / sum(self.epoch_runtime_log) + + @property + def average_examples_per_second(self): + """The average number of training examples per second across all epochs.""" + return self.average_steps_per_second * self.batch_size + + def on_train_end(self, logs=None): + self.train_finish_time = time.time() + + if self.summary_writer: + self.summary_writer.flush() + + def on_epoch_begin(self, epoch, logs=None): + self.epoch_start = time.time() + + def on_batch_begin(self, batch, logs=None): + if not self.start_time: + self.start_time = time.time() + + # Record the timestamp of the first global step + if not self.timestamp_log: + self.timestamp_log.append(BatchTimestamp(self.global_steps, + self.start_time)) + + def on_batch_end(self, batch, logs=None): + """Records elapse time of the batch and calculates examples per second.""" + self.steps_in_epoch = batch + 1 + steps_since_last_log = self.global_steps - self.last_log_step + if steps_since_last_log >= self.log_steps: + now = time.time() + elapsed_time = now - self.start_time + steps_per_second = steps_since_last_log / elapsed_time + examples_per_second = steps_per_second * self.batch_size + + self.timestamp_log.append(BatchTimestamp(self.global_steps, now)) + logging.info( + 'TimeHistory: %.2f seconds, %.2f examples/second between steps %d ' + 'and %d', elapsed_time, examples_per_second, self.last_log_step, + self.global_steps) + + if self.summary_writer: + with self.summary_writer.as_default(): + tf.summary.scalar('global_step/sec', steps_per_second, + self.global_steps) + tf.summary.scalar('examples/sec', examples_per_second, + self.global_steps) + + self.last_log_step = self.global_steps + self.start_time = None + + def on_epoch_end(self, epoch, logs=None): + epoch_run_time = time.time() - self.epoch_start + self.epoch_runtime_log.append(epoch_run_time) + + self.steps_before_epoch += self.steps_in_epoch + self.steps_in_epoch = 0 + + +def get_profiler_callback(model_dir, profile_steps, enable_tensorboard, + steps_per_epoch): + """Validate profile_steps flag value and return profiler callback.""" + profile_steps_error_message = ( + 'profile_steps must be a comma separated pair of positive integers, ' + 'specifying the first and last steps to be profiled.' + ) + try: + profile_steps = [int(i) for i in profile_steps.split(',')] + except ValueError: + raise ValueError(profile_steps_error_message) + if len(profile_steps) != 2: + raise ValueError(profile_steps_error_message) + start_step, stop_step = profile_steps + if start_step < 0 or start_step > stop_step: + raise ValueError(profile_steps_error_message) + if enable_tensorboard: + logging.warning( + 'Both TensorBoard and profiler callbacks are used. Note that the ' + 'TensorBoard callback profiles the 2nd step (unless otherwise ' + 'specified). Please make sure the steps profiled by the two callbacks ' + 'do not overlap.') + return ProfilerCallback(model_dir, start_step, stop_step, steps_per_epoch) + + +class ProfilerCallback(tf.keras.callbacks.Callback): + """Save profiles in specified step range to log directory.""" + + def __init__(self, log_dir, start_step, stop_step, steps_per_epoch): + super(ProfilerCallback, self).__init__() + self.log_dir = log_dir + self.start_step = start_step + self.stop_step = stop_step + self.start_epoch = start_step // steps_per_epoch + self.stop_epoch = stop_step // steps_per_epoch + self.start_step_in_epoch = start_step % steps_per_epoch + self.stop_step_in_epoch = stop_step % steps_per_epoch + self.should_start = False + self.should_stop = False + + def on_epoch_begin(self, epoch, logs=None): + if epoch == self.start_epoch: + self.should_start = True + if epoch == self.stop_epoch: + self.should_stop = True + + def on_batch_begin(self, batch, logs=None): + if batch == self.start_step_in_epoch and self.should_start: + self.should_start = False + profiler.start(self.log_dir) + logging.info('Profiler started at Step %s', self.start_step) + + def on_batch_end(self, batch, logs=None): + if batch == self.stop_step_in_epoch and self.should_stop: + self.should_stop = False + profiler.stop() + logging.info('Profiler saved profiles for steps between %s and %s to %s', + self.start_step, self.stop_step, self.log_dir) + + +def set_session_config(enable_eager=False, + enable_xla=False): + """Sets the session config.""" + if is_v2_0(): + set_config_v2(enable_xla=enable_xla) + else: + config = get_config_proto_v1(enable_xla=enable_xla) + if enable_eager: + tf.compat.v1.enable_eager_execution(config=config) + else: + sess = tf.compat.v1.Session(config=config) + tf.compat.v1.keras.backend.set_session(sess) + + +def get_config_proto_v1(enable_xla=False): + """Return config proto according to flag settings, or None to use default.""" + config = None + if enable_xla: + config = tf.compat.v1.ConfigProto() + config.graph_options.optimizer_options.global_jit_level = ( + tf.OptimizerOptions.ON_2) + return config + + +def set_config_v2(enable_xla=False): + """Config eager context according to flag values using TF 2.0 API.""" + if enable_xla: + tf.config.optimizer.set_jit(True) + + +def is_v2_0(): + """Returns true if using tf 2.0.""" + return tf2.enabled() + + +def set_gpu_thread_mode_and_count(gpu_thread_mode, + datasets_num_private_threads, + num_gpus, per_gpu_thread_count): + """Set GPU thread mode and count, and adjust dataset threads count.""" + cpu_count = multiprocessing.cpu_count() + logging.info('Logical CPU cores: %s', cpu_count) + + # Allocate private thread pool for each GPU to schedule and launch kernels + per_gpu_thread_count = per_gpu_thread_count or 2 + os.environ['TF_GPU_THREAD_MODE'] = gpu_thread_mode + os.environ['TF_GPU_THREAD_COUNT'] = str(per_gpu_thread_count) + logging.info('TF_GPU_THREAD_COUNT: %s', + os.environ['TF_GPU_THREAD_COUNT']) + logging.info('TF_GPU_THREAD_MODE: %s', + os.environ['TF_GPU_THREAD_MODE']) + + # Limit data preprocessing threadpool to CPU cores minus number of total GPU + # private threads and memory copy threads. + total_gpu_thread_count = per_gpu_thread_count * num_gpus + num_runtime_threads = num_gpus + if not datasets_num_private_threads: + datasets_num_private_threads = min( + cpu_count - total_gpu_thread_count - num_runtime_threads, + num_gpus * 8) + logging.info('Set datasets_num_private_threads to %s', + datasets_num_private_threads) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/model_helpers.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/model_helpers.py new file mode 100644 index 0000000..c112bac --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/model_helpers.py @@ -0,0 +1,93 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Miscellaneous functions that can be called by models.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numbers + +import tensorflow as tf +from tensorflow.python.util import nest + + +def past_stop_threshold(stop_threshold, eval_metric): + """Return a boolean representing whether a model should be stopped. + + Args: + stop_threshold: float, the threshold above which a model should stop + training. + eval_metric: float, the current value of the relevant metric to check. + + Returns: + True if training should stop, False otherwise. + + Raises: + ValueError: if either stop_threshold or eval_metric is not a number + """ + if stop_threshold is None: + return False + + if not isinstance(stop_threshold, numbers.Number): + raise ValueError("Threshold for checking stop conditions must be a number.") + if not isinstance(eval_metric, numbers.Number): + raise ValueError("Eval metric being checked against stop conditions " + "must be a number.") + + if eval_metric >= stop_threshold: + tf.compat.v1.logging.info( + "Stop threshold of {} was passed with metric value {}.".format( + stop_threshold, eval_metric)) + return True + + return False + + +def generate_synthetic_data( + input_shape, input_value=0, input_dtype=None, label_shape=None, + label_value=0, label_dtype=None): + """Create a repeating dataset with constant values. + + Args: + input_shape: a tf.TensorShape object or nested tf.TensorShapes. The shape of + the input data. + input_value: Value of each input element. + input_dtype: Input dtype. If None, will be inferred by the input value. + label_shape: a tf.TensorShape object or nested tf.TensorShapes. The shape of + the label data. + label_value: Value of each input element. + label_dtype: Input dtype. If None, will be inferred by the target value. + + Returns: + Dataset of tensors or tuples of tensors (if label_shape is set). + """ + # TODO(kathywu): Replace with SyntheticDataset once it is in contrib. + element = input_element = nest.map_structure( + lambda s: tf.constant(input_value, input_dtype, s), input_shape) + + if label_shape: + label_element = nest.map_structure( + lambda s: tf.constant(label_value, label_dtype, s), label_shape) + element = (input_element, label_element) + + return tf.data.Dataset.from_tensors(element).repeat() + + +def apply_clean(flags_obj): + if flags_obj.clean and tf.io.gfile.exists(flags_obj.model_dir): + tf.compat.v1.logging.info("--clean flag set. Removing existing model dir:" + " {}".format(flags_obj.model_dir)) + tf.io.gfile.rmtree(flags_obj.model_dir) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/model_helpers_test.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/model_helpers_test.py new file mode 100644 index 0000000..f34a594 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/model_helpers_test.py @@ -0,0 +1,127 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Model Helper functions.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf # pylint: disable=g-bad-import-order + +from official.utils.misc import keras_utils +from official.utils.misc import model_helpers + + +class PastStopThresholdTest(tf.test.TestCase): + """Tests for past_stop_threshold.""" + + def setUp(self): + super(PastStopThresholdTest, self).setUp() + if keras_utils.is_v2_0: + tf.compat.v1.disable_eager_execution() + + def test_past_stop_threshold(self): + """Tests for normal operating conditions.""" + self.assertTrue(model_helpers.past_stop_threshold(0.54, 1)) + self.assertTrue(model_helpers.past_stop_threshold(54, 100)) + self.assertFalse(model_helpers.past_stop_threshold(0.54, 0.1)) + self.assertFalse(model_helpers.past_stop_threshold(-0.54, -1.5)) + self.assertTrue(model_helpers.past_stop_threshold(-0.54, 0)) + self.assertTrue(model_helpers.past_stop_threshold(0, 0)) + self.assertTrue(model_helpers.past_stop_threshold(0.54, 0.54)) + + def test_past_stop_threshold_none_false(self): + """Tests that check None returns false.""" + self.assertFalse(model_helpers.past_stop_threshold(None, -1.5)) + self.assertFalse(model_helpers.past_stop_threshold(None, None)) + self.assertFalse(model_helpers.past_stop_threshold(None, 1.5)) + # Zero should be okay, though. + self.assertTrue(model_helpers.past_stop_threshold(0, 1.5)) + + def test_past_stop_threshold_not_number(self): + """Tests for error conditions.""" + with self.assertRaises(ValueError): + model_helpers.past_stop_threshold("str", 1) + + with self.assertRaises(ValueError): + model_helpers.past_stop_threshold("str", tf.constant(5)) + + with self.assertRaises(ValueError): + model_helpers.past_stop_threshold("str", "another") + + with self.assertRaises(ValueError): + model_helpers.past_stop_threshold(0, None) + + with self.assertRaises(ValueError): + model_helpers.past_stop_threshold(0.7, "str") + + with self.assertRaises(ValueError): + model_helpers.past_stop_threshold(tf.constant(4), None) + + +class SyntheticDataTest(tf.test.TestCase): + """Tests for generate_synthetic_data.""" + + def test_generate_synethetic_data(self): + input_element, label_element = tf.compat.v1.data.make_one_shot_iterator( + model_helpers.generate_synthetic_data(input_shape=tf.TensorShape([5]), + input_value=123, + input_dtype=tf.float32, + label_shape=tf.TensorShape([]), + label_value=456, + label_dtype=tf.int32)).get_next() + + with self.session() as sess: + for n in range(5): + inp, lab = sess.run((input_element, label_element)) + self.assertAllClose(inp, [123., 123., 123., 123., 123.]) + self.assertEquals(lab, 456) + + def test_generate_only_input_data(self): + d = model_helpers.generate_synthetic_data( + input_shape=tf.TensorShape([4]), + input_value=43.5, + input_dtype=tf.float32) + + element = tf.compat.v1.data.make_one_shot_iterator(d).get_next() + self.assertFalse(isinstance(element, tuple)) + + with self.session() as sess: + inp = sess.run(element) + self.assertAllClose(inp, [43.5, 43.5, 43.5, 43.5]) + + def test_generate_nested_data(self): + d = model_helpers.generate_synthetic_data( + input_shape={'a': tf.TensorShape([2]), + 'b': {'c': tf.TensorShape([3]), 'd': tf.TensorShape([])}}, + input_value=1.1) + + element = tf.compat.v1.data.make_one_shot_iterator(d).get_next() + self.assertIn('a', element) + self.assertIn('b', element) + self.assertEquals(len(element['b']), 2) + self.assertIn('c', element['b']) + self.assertIn('d', element['b']) + self.assertNotIn('c', element) + + with self.session() as sess: + inp = sess.run(element) + self.assertAllClose(inp['a'], [1.1, 1.1]) + self.assertAllClose(inp['b']['c'], [1.1, 1.1, 1.1]) + self.assertAllClose(inp['b']['d'], 1.1) + + +if __name__ == "__main__": + tf.test.main() diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/tpu_lib.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/tpu_lib.py new file mode 100644 index 0000000..4d4cddb --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/misc/tpu_lib.py @@ -0,0 +1,34 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Initializes TPU system for TF 2.0.""" + +import tensorflow as tf + + +def tpu_initialize(tpu_address): + """Initializes TPU for TF 2.0 training. + + Args: + tpu_address: string, bns address of master TPU worker. + + Returns: + A TPUClusterResolver. + """ + cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver( + tpu=tpu_address) + if tpu_address not in ('', 'local'): + tf.config.experimental_connect_to_cluster(cluster_resolver) + tf.tpu.experimental.initialize_tpu_system(cluster_resolver) + return cluster_resolver diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/__init__.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/benchmark_wrappers.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/benchmark_wrappers.py new file mode 100644 index 0000000..a1c5327 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/benchmark_wrappers.py @@ -0,0 +1,83 @@ +# Lint as: python3 +"""Utils to annotate and trace benchmarks.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import flags +from absl import logging +from absl.testing import flagsaver + +FLAGS = flags.FLAGS + +flags.DEFINE_multi_string( + 'benchmark_method_flags', None, + 'Optional list of runtime flags of the form key=value. Specify ' + 'multiple times to specify different flags. These will override the FLAGS ' + 'object directly after hardcoded settings in individual benchmark methods ' + 'before they call _run_and_report benchmark. Example if we set ' + '--benchmark_method_flags=train_steps=10 and a benchmark method hardcodes ' + 'FLAGS.train_steps=10000 and later calls _run_and_report_benchmark, ' + 'it\'ll only run for 10 steps. This is useful for ' + 'debugging/profiling workflows.') + + +def enable_runtime_flags(decorated_func): + """Sets attributes from --benchmark_method_flags for method execution. + + @enable_runtime_flags decorator temporarily adds flags passed in via + --benchmark_method_flags and runs the decorated function in that context. + + A user can set --benchmark_method_flags=train_steps=5 to run the benchmark + method in the snippet below with FLAGS.train_steps=5 for debugging (without + modifying the benchmark code). + + class ModelBenchmark(): + + @benchmark_wrappers.enable_runtime_flags + def _run_and_report_benchmark(self): + # run benchmark ... + # report benchmark results ... + + def benchmark_method(self): + FLAGS.train_steps = 1000 + ... + self._run_and_report_benchmark() + + Args: + decorated_func: The method that runs the benchmark after previous setup + execution that set some flags. + + Returns: + new_func: The same method which executes in a temporary context where flag + overrides from --benchmark_method_flags are active. + """ + + def runner(*args, **kwargs): + """Creates a temporary context to activate --benchmark_method_flags.""" + if FLAGS.benchmark_method_flags: + saved_flag_values = flagsaver.save_flag_values() + for key_value in FLAGS.benchmark_method_flags: + key, value = key_value.split('=', 1) + try: + numeric_float = float(value) + numeric_int = int(numeric_float) + if abs(numeric_int) == abs(numeric_float): + flag_value = numeric_int + else: + flag_value = numeric_float + except ValueError: + flag_value = value + logging.info('Setting --%s=%s', key, flag_value) + setattr(FLAGS, key, flag_value) + else: + saved_flag_values = None + try: + result = decorated_func(*args, **kwargs) + return result + finally: + if saved_flag_values: + flagsaver.restore_flag_values(saved_flag_values) + + return runner diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/integration.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/integration.py new file mode 100644 index 0000000..b4809a4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/integration.py @@ -0,0 +1,71 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Helper code to run complete models from within python. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import shutil +import sys +import tempfile + +from absl import flags +from absl.testing import flagsaver + +from official.utils.flags import core as flags_core + + +@flagsaver.flagsaver +def run_synthetic(main, tmp_root, extra_flags=None, synth=True, train_epochs=1, + epochs_between_evals=1): + """Performs a minimal run of a model. + + This function is intended to test for syntax errors throughout a model. A + very limited run is performed using synthetic data. + + Args: + main: The primary function used to exercise a code path. Generally this + function is ".main(argv)". + tmp_root: Root path for the temp directory created by the test class. + extra_flags: Additional flags passed by the caller of this function. + synth: Use synthetic data. + train_epochs: Value of the --train_epochs flag. + epochs_between_evals: Value of the --epochs_between_evals flag. + """ + + extra_flags = [] if extra_flags is None else extra_flags + + model_dir = tempfile.mkdtemp(dir=tmp_root) + + args = [sys.argv[0], "--model_dir", model_dir] + extra_flags + + if synth: + args.append("--use_synthetic_data") + + if train_epochs is not None: + args.extend(["--train_epochs", str(train_epochs)]) + + if epochs_between_evals is not None: + args.extend(["--epochs_between_evals", str(epochs_between_evals)]) + + try: + flags_core.parse_flags(argv=args) + main(flags.FLAGS) + finally: + if os.path.exists(model_dir): + shutil.rmtree(model_dir) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/mock_lib.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/mock_lib.py new file mode 100644 index 0000000..ee4de3c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/mock_lib.py @@ -0,0 +1,36 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Mock objects and related functions for testing.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +class MockBenchmarkLogger(object): + """This is a mock logger that can be used in dependent tests.""" + + def __init__(self): + self.logged_metric = [] + + def log_metric(self, name, value, unit=None, global_step=None, + extras=None): + self.logged_metric.append({ + "name": name, + "value": float(value), + "unit": unit, + "global_step": global_step, + "extras": extras}) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/perfzero_benchmark.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/perfzero_benchmark.py new file mode 100644 index 0000000..af4a912 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/perfzero_benchmark.py @@ -0,0 +1,90 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utils for creating PerfZero benchmarks.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from absl import flags +from absl.testing import flagsaver +import tensorflow as tf # pylint: disable=g-bad-import-order + +FLAGS = flags.FLAGS + + +class PerfZeroBenchmark(tf.test.Benchmark): + """Common methods used in PerfZero Benchmarks. + + Handles the resetting of flags between tests, loading of default_flags, + overriding of defaults. PerfZero (OSS) runs each test in a separate + process reducing some need to reset the flags. + """ + local_flags = None + + def __init__(self, + output_dir=None, + default_flags=None, + flag_methods=None, + tpu=None): + """Initialize class. + + Args: + output_dir: Base directory to store all output for the test. + default_flags: Set of flags to pass to model. + flag_methods: Set of flag methods to run during setup. + tpu: (optional) TPU name to use in a TPU benchmark. + """ + if os.getenv('BENCHMARK_OUTPUT_DIR'): + self.output_dir = os.getenv('BENCHMARK_OUTPUT_DIR') + elif output_dir: + self.output_dir = output_dir + else: + self.output_dir = '/tmp' + self.default_flags = default_flags or {} + self.flag_methods = flag_methods or {} + + if os.getenv('BENCHMARK_TPU'): + resolved_tpu = os.getenv('BENCHMARK_TPU') + elif tpu: + resolved_tpu = tpu + else: + resolved_tpu = None + + if resolved_tpu: + # TPU models are expected to accept a --tpu=name flag. PerfZero creates + # the TPU at runtime and passes the TPU's name to this flag. + self.default_flags['tpu'] = resolved_tpu + + def _get_model_dir(self, folder_name): + """Returns directory to store info, e.g. saved model and event log.""" + return os.path.join(self.output_dir, folder_name) + + def _setup(self): + """Sets up and resets flags before each test.""" + tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO) + if PerfZeroBenchmark.local_flags is None: + for flag_method in self.flag_methods: + flag_method() + # Loads flags to get defaults to then override. List cannot be empty. + flags.FLAGS(['foo']) + # Overrides flag values with defaults for the class of tests. + for k, v in self.default_flags.items(): + setattr(FLAGS, k, v) + saved_flag_values = flagsaver.save_flag_values() + PerfZeroBenchmark.local_flags = saved_flag_values + else: + flagsaver.restore_flag_values(PerfZeroBenchmark.local_flags) diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/pylint.rcfile b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/pylint.rcfile new file mode 100644 index 0000000..b872802 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/pylint.rcfile @@ -0,0 +1,168 @@ +[MESSAGES CONTROL] +disable=R,W,bad-option-value,trailing-newlines,no-name-in-module + +[REPORTS] +# Tells whether to display a full report or only the messages +reports=no + +# Activate the evaluation score. +score=no + +[BASIC] + +# Regular expression matching correct argument names +argument-rgx=^[a-z][a-z0-9_]*$ + +# Regular expression matching correct attribute names +attr-rgx=^_{0,2}[a-z][a-z0-9_]*$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ + +# Regular expression matching correct class names +class-rgx=^_?[A-Z][a-zA-Z0-9]*$ + +# Regular expression matching correct constant names +const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=10 + +# Regular expression matching correct function names +function-rgx=^(?:(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$ + +# Good variable names which should always be accepted, separated by a comma +good-names=main,_ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=^[a-z][a-z0-9_]*$ + +# Regular expression matching correct method names +method-rgx=^(?:(?P__[a-z0-9_]+__|next)|(?P_{0,2}[A-Z][a-zA-Z0-9]*)|(?P_{0,2}[a-z][a-z0-9_]*)|(setUp|tearDown))$ + +# Regular expression matching correct module names +module-rgx=^(_?[a-z][a-z0-9_]*)|__init__|PRESUBMIT|PRESUBMIT_unittest$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=(__.*__|main|.*ArgParser) + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=^[a-z][a-z0-9_]*$ + +[TYPECHECK] + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules=absl, absl.*, official, official.*, tensorflow, tensorflow.*, LazyLoader, google, google.cloud.* + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + +# This is deprecated, because it is not used anymore. +#ignore-iface-methods= + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls,class_ + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of arguments for function / method +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of statements in function / method body +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=StandardError,Exception,BaseException + + +[FORMAT] + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=80 + +# Maximum number of lines in a module +max-module-lines=99999 + +# List of optional constructs for which whitespace checking is disabled +no-space-check= + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=yes + +# Allow URLs and comment type annotations to exceed the max line length as neither can be easily +# split across lines. +ignore-long-lines=^\s*(?:(# )??$|# type:) + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_) + +# Tells whether we should check for unused import in __init__ files. +init-import=no diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/scripts/builds_common.sh b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/scripts/builds_common.sh new file mode 100644 index 0000000..3cf08bb --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/scripts/builds_common.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# +# Common Bash functions used by build scripts + +COLOR_NC='\033[0m' +COLOR_BOLD='\033[1m' +COLOR_LIGHT_GRAY='\033[0;37m' +COLOR_GREEN='\033[0;32m' +COLOR_RED='\033[0;31m' + +die() { + # Print a message and exit with code 1. + # + # Usage: die + # e.g., die "Something bad happened." + + echo $@ + exit 1 +} + +num_cpus() { + # Get the number of CPUs + N_CPUS=$(grep -c ^processor /proc/cpuinfo) + if [[ -z ${N_CPUS} ]]; then + die "ERROR: Unable to determine the number of CPUs" + fi + + echo ${N_CPUS} +} + +# List files changed (i.e., added, or revised) from +# the common ancestor of HEAD and the latest master branch. +# Usage: get_changed_files_from_master_branch +get_changed_files_from_master_branch() { + ANCESTOR=$(git merge-base HEAD master origin/master) + git diff ${ANCESTOR} --diff-filter=d --name-only "$@" +} + +# List python files changed that still exist, +# i.e., not removed. +# Usage: get_py_files_to_check [--incremental] +get_py_files_to_check() { + if [[ "$1" == "--incremental" ]]; then + get_changed_files_from_master_branch -- '*.py' + elif [[ -z "$1" ]]; then + find official/ -name '*.py' + else + die "Found unsupported args: $@ for get_py_files_to_check." + fi +} diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/scripts/ci_sanity.sh b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/scripts/ci_sanity.sh new file mode 100644 index 0000000..97d6bc2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/scripts/ci_sanity.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# Sanity check script that runs tests and lint under local environment. +# Make sure that tensorflow and pylint is installed. +# usage: models >: ./official/utils/testing/scripts/ci_sanity.sh do_pylint --incremental +set +x + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/builds_common.sh" +cd "$SCRIPT_DIR/../../../.." +MODEL_ROOT="$(pwd)" + +export PYTHONPATH="$PYTHONPATH:${MODEL_ROOT}" + +# Run pylint +do_pylint() { + # Usage: do_pylint [--incremental] + # + # Options: + # --incremental Performs check on only the python files changed in the + # last non-merge git commit. + + # Use this list to whitelist pylint errors + ERROR_WHITELIST="" + + echo "ERROR_WHITELIST=\"${ERROR_WHITELIST}\"" + + PYLINT_BIN="python3 -m pylint" + + PYTHON_SRC_FILES=$(get_py_files_to_check $1) + if [[ -z ${PYTHON_SRC_FILES} ]]; then + echo "do_pylint found no Python files to check. Returning." + return 0 + fi + + PYLINTRC_FILE="official/utils/testing/pylint.rcfile" + + if [[ ! -f "${PYLINTRC_FILE}" ]]; then + die "ERROR: Cannot find pylint rc file at ${PYLINTRC_FILE}" + fi + + NUM_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w) + NUM_CPUS=$(num_cpus) + + echo "Running pylint on ${NUM_SRC_FILES} files with ${NUM_CPUS} "\ + "parallel jobs..." + echo "" + + PYLINT_START_TIME=$(date +'%s') + OUTPUT_FILE="$(mktemp)_pylint_output.log" + ERRORS_FILE="$(mktemp)_pylint_errors.log" + NONWL_ERRORS_FILE="$(mktemp)_pylint_nonwl_errors.log" + + rm -rf ${OUTPUT_FILE} + rm -rf ${ERRORS_FILE} + rm -rf ${NONWL_ERRORS_FILE} + touch ${NONWL_ERRORS_FILE} + + ${PYLINT_BIN} --rcfile="${PYLINTRC_FILE}" --output-format=parseable \ + --jobs=${NUM_CPUS} ${PYTHON_SRC_FILES} > ${OUTPUT_FILE} 2>&1 + PYLINT_END_TIME=$(date +'%s') + + echo "" + echo "pylint took $((PYLINT_END_TIME - PYLINT_START_TIME)) s" + echo "" + + # Report only what we care about + # Ref https://pylint.readthedocs.io/en/latest/technical_reference/features.html + # E: all errors + # W0311 bad-indentation + # W0312 mixed-indentation + # C0330 bad-continuation + # C0301 line-too-long + # C0326 bad-whitespace + # W0611 unused-import + # W0622 redefined-builtin + grep -E '(\[E|\[W0311|\[W0312|\[C0330|\[C0301|\[C0326|\[W0611|\[W0622)' ${OUTPUT_FILE} > ${ERRORS_FILE} + + N_ERRORS=0 + while read -r LINE; do + IS_WHITELISTED=0 + for WL_REGEX in ${ERROR_WHITELIST}; do + if echo ${LINE} | grep -q "${WL_REGEX}"; then + echo "Found a whitelisted error:" + echo " ${LINE}" + IS_WHITELISTED=1 + fi + done + + if [[ ${IS_WHITELISTED} == "0" ]]; then + echo "${LINE}" >> ${NONWL_ERRORS_FILE} + echo "" >> ${NONWL_ERRORS_FILE} + ((N_ERRORS++)) + fi + done <${ERRORS_FILE} + + echo "Raw lint output file: ${OUTPUT_FILE}" + + echo "" + if [[ ${N_ERRORS} != 0 ]]; then + echo "FAIL: Found ${N_ERRORS} non-whitelited pylint errors:" + cat "${NONWL_ERRORS_FILE}" + return 1 + else + echo "PASS: No non-whitelisted pylint errors were found." + return 0 + fi +} + +test_result=0 + +TESTS="$@" + +for t in "${TESTS}"; do + ${t} || test_result=$? +done + +exit "${test_result}" diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/scripts/presubmit.sh b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/scripts/presubmit.sh new file mode 100644 index 0000000..954d96d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/code/official/utils/testing/scripts/presubmit.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# Presubmit script that runs tests and lint under local environment. +# Make sure that tensorflow and pylint is installed. +# usage: models >: ./official/utils/testing/scripts/presubmit.sh +# usage: models >: ./official/utils/testing/scripts/presubmit.sh lint py2_test py3_test +set +x + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR/../../../.." +MODEL_ROOT="$(pwd)" + +export PYTHONPATH="$PYTHONPATH:${MODEL_ROOT}" + +py_test() { + local PY_BINARY="$1" + local exit_code=0 + + echo "===========Running Python test============" + + for test_file in `find official/ -name '*test.py' -print` + do + echo "####=======Testing ${test_file}=======####" + ${PY_BINARY} "${test_file}" + _exit_code=$? + if [[ $_exit_code != 0 ]]; then + exit_code=$_exit_code + echo "FAIL: ${test_file}" + fi + done + + return "${exit_code}" +} + +py2_test() { + local PY_BINARY=$(which python2) + py_test "$PY_BINARY" + return $? +} + +py3_test() { + local PY_BINARY=$(which python3) + py_test "$PY_BINARY" + return $? +} + +test_result=0 + +if [ "$#" -eq 0 ]; then + TESTS="lint py2_test py3_test" +else + TESTS="$@" +fi + +for t in "${TESTS}"; do + ${t} || test_result=$? +done + +exit "${test_result}" diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/npu_set_env.sh b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/npu_set_env.sh new file mode 100644 index 0000000..e015ca1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/npu_set_env.sh @@ -0,0 +1,42 @@ +# main env + +if [ -d /usr/local/Ascend/nnae/latest ];then + + export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/Ascend/driver/tools/hccn_tool/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages + export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp +else + export LD_LIBRARY_PATH=/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/ascend-toolkit/latest//fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/tfplugin/python/site-packages:$projectDir + export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + +fi + +export SOC_VERSION=Ascend910 +export HCCL_CONNECT_TIMEOUT=600 + +# user env +export JOB_ID={JOB_ID} +export RANK_TABLE_FILE={RANK_TABLE_FILE} +#export RANK_SIZE={RANK_SIZE} +#export RANK_INDEX={RANK_INDEX} +#export RANK_ID={RANK_ID} + +# profiling env +export PROFILING_MODE=false +export PROFILING_OPTIONS=training_trace +export FP_POINT=resnet_model/conv2d/Conv2Dresnet_model/batch_normalization/FusedBatchNormV3_Reduce +export BP_POINT=gradients/AddN_70 +export AICPU_PROFILING_MODE=false + +# debug env +#export DUMP_GE_GRAPH=2 +#export DUMP_OP=1 +#export DUMP_OP_LESS=1 +#export PRINT_MODEL=1 +#export TE_PARALLEL_COMPILER=0 + +# system env +ulimit -c unlimited diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_16p_npu.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_16p_npu.py new file mode 100644 index 0000000..c5565ed --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_16p_npu.py @@ -0,0 +1,18 @@ + + +config = { + 'batch_size': 32, + 'train_epochs': 2, + 'data_dir': '/home/imagenet_TF/', + 'epochs_between_evals': 1, + 'dynamic_loss_scale': True, + 'rank_size': 16, + 'max_train_steps': 1000, + 'iterations_per_loop': 100, + 'save_checkpoints_steps': 115200, +} + + +def resnet50_config(): + config['global_batch_size'] = config['batch_size'] * config['rank_size'] + return config diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_1p_npu.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_1p_npu.py new file mode 100644 index 0000000..268720c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_1p_npu.py @@ -0,0 +1,18 @@ + + +config = { + 'batch_size': 32, + 'train_epochs': 1, + 'data_dir': '/home/data/imagenet_TF/', + 'epochs_between_evals': 1, + 'dynamic_loss_scale': True, + 'rank_size': 1, + 'max_train_steps': 1000, + 'iterations_per_loop': 1000, + 'save_checkpoints_steps': 115200, +} + + +def resnet50_config(): + config['global_batch_size'] = config['batch_size'] * config['rank_size'] + return config diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_2p_npu.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_2p_npu.py new file mode 100644 index 0000000..a056853 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_2p_npu.py @@ -0,0 +1,18 @@ + + +config = { + 'batch_size': 32, + 'train_epochs': 1, + 'data_dir': '/home/imagenet_TF/', + 'epochs_between_evals': 1, + 'dynamic_loss_scale': True, + 'rank_size': 2, + 'max_train_steps': 1000, + 'iterations_per_loop': 1000, + 'save_checkpoints_steps': 115200, +} + + +def resnet50_config(): + config['global_batch_size'] = config['batch_size'] * config['rank_size'] + return config diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_4p_npu.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_4p_npu.py new file mode 100644 index 0000000..2216a39 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_4p_npu.py @@ -0,0 +1,18 @@ + + +config = { + 'batch_size': 32, + 'train_epochs': 1, + 'data_dir': '/home/imagenet_TF/', + 'epochs_between_evals': 1, + 'dynamic_loss_scale': True, + 'rank_size': 4, + 'max_train_steps': 1000, + 'iterations_per_loop': 1000, + 'save_checkpoints_steps': 115200, +} + + +def resnet50_config(): + config['global_batch_size'] = config['batch_size'] * config['rank_size'] + return config diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_8p_npu.py b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_8p_npu.py new file mode 100644 index 0000000..d6ffa1b --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/config/resnet_config_8p_npu.py @@ -0,0 +1,18 @@ + + +config = { + 'batch_size': 32, + 'train_epochs': 2, + 'data_dir': '/home/imagenet_TF/', + 'epochs_between_evals': 1, + 'dynamic_loss_scale': True, + 'rank_size': 8, + 'max_train_steps': 1000, + 'iterations_per_loop': 100, + 'save_checkpoints_steps': 115200, +} + + +def resnet50_config(): + config['global_batch_size'] = config['batch_size'] * config['rank_size'] + return config diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/scripts/run.sh b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/scripts/run.sh new file mode 100644 index 0000000..882c73c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/scripts/run.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +rank_size=$1 +yamlPath=$2 +toolsPath=$3 +if [ -f /.dockerenv ];then + CLUSTER=$4 +MPIRUN_ALL_IP="$5" + export CLUSTER=${CLUSTER} +fi +currentDir=$(cd "$(dirname "$0")/.."; pwd) +# 配置环境变量并调用 train 方法 +currtime=`date +%Y%m%d%H%M%S` +mkdir -p ${currentDir%train*}/train/result/tf_resnet50/training_job_${currtime}/ +train_job_dir=${currentDir%train*}/train/result/tf_resnet50/training_job_${currtime}/ +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir} &" +# user env +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 +export SLOG_PRINT_TO_STDOUT=0 +export RANK_TABLE_FILE=${currentDir}/config/${rank_size}p.json + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "tensorflow_config") + + +data_url_new=`echo ${data_url//\//\\\\/}` + +jsonFilePath=${currentDir}/config/resnet_config_${rank_size}p_npu.py +echo "start to modify inner config file" +#echo "jsonfilepath is "${jsonFilePath} + +sed -i "s/train_epochs.*$/train_epochs\': ${epoches},/g" ${jsonFilePath} +sed -i "0,/batch_size.*$/s//batch_size\': ${batch_size},/" ${jsonFilePath} +sed -i "s/max_train_steps.*$/max_train_steps\': ${max_train_steps},/g" ${jsonFilePath} +sed -i "s/data_dir.*$/data_dir\': \'${data_url_new}\',/g" ${jsonFilePath} +#sed -i "s/rank_size.*$/rank_size\': ${rank_size},/g" ${jsonFilePath} +sed -i "s/epochs_between_evals.*$/epochs_between_evals\': ${epochs_between_evals},/g" ${jsonFilePath} +sed -i "s/iterations_per_loop.*$/iterations_per_loop\': ${iterations_per_loop},/g" ${jsonFilePath} +sed -i "s/save_checkpoints_steps.*$/save_checkpoints_steps\': ${save_checkpoints_steps},/g" ${jsonFilePath} +sed -i 's/\r//g' ${jsonFilePath} +if [ $? -eq 0 ] ; +then + echo "modify inner config file success" +else + echo "modify inner config file fail" + exit +fi + +# device 列表, 若无指定 device 根据 rank_size 顺序选择 +eval device_group=\$device_group_${rank_size}p +if [ x"${device_group}" == x"" ] || [ ${rank_size} -ge 8 ];then + device_group="$(seq 0 "$(expr $rank_size - 1)")" +fi + +# get last device id in device_group, hw log in performance from the dir named last_device_id +device_group_str=`echo ${device_group} | sed 's/ //g'` +first_device_id=`echo ${device_group_str: 0:1}` + +rank_id=0 + +if [ x"${CLUSTER}" == x"True" ];then + # ln hw log + ln -snf ${train_job_dir}/0/hw_resnet50.log ${train_job_dir} + this_ip=$(hostname -I |awk '{print $1}') + for ip in $MPIRUN_ALL_IP;do + if [ x"$ip" != x"$this_ip" ];then + scp $yamlPath root@$ip:$yamlPath + scp $jsonFilePath root@$ip:$jsonFilePath + fi + done + export PATH=$PATH:/usr/local/mpirun4.0/bin + mpirun -H ${mpirun_ip} \ + --bind-to none -map-by slot\ + --allow-run-as-root \ + --mca btl_tcp_if_exclude lo,docker0,endvnic,virbr0,vethf40501b,docker_gwbridge,br-f42ac38052b4\ + --prefix /usr/local/mpirun4.0/ \ + ${currentDir}/scripts/train.sh 0 $rank_size $yamlPath $currtime ${toolsPath} ${CLUSTER} +else + # ln hw log + ln -snf ${train_job_dir}/${first_device_id}/hw_resnet50.log ${train_job_dir} + for device_id in $device_group;do + ${currentDir}/scripts/train.sh $device_id $rank_size $yamlPath $currtime ${toolsPath} $rank_id & + let rank_id++ + done +fi +wait + +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir} &" + diff --git a/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/scripts/train.sh b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/scripts/train.sh new file mode 100644 index 0000000..b108d01 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNet50/tensorflow/scripts/train.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +device_id=$1 +rank_size=$2 +yamlPath=$3 +currtime=$4 +toolsPath=$5 + +currentDir=$(cd "$(dirname "$0")/.."; pwd) + +export REMARK_LOG_FILE=hw_resnet50.log + +# ${mainDir%train*}/train/result/tensorflow/Bert/TrainingJob-${currtime} # +mkdir -p ${currentDir%train*}/train/result/tf_resnet50/training_job_${currtime}/ +export train_job_dir=${currentDir%train*}/train/result/tf_resnet50/training_job_${currtime}/ + + +source ${currentDir}/config/npu_set_env.sh + +benchmark_log_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils +#atlasboost_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +code_dir_path=${currentDir}/code +export PYTHONPATH=$PYTHONPATH:${benchmark_log_path}:${code_dir_path} + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "tensorflow_config") + +# user env +export YAML_PATH=$3 +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 +export RANK_TABLE_FILE=${currentDir}/config/${rank_size}p.json +export RANK_SIZE=${rank_size} +export RANK_INDEX=0 +export SLOG_PRINT_TO_STDOUT=0 +export DEVICE_ID=$1 +DEVICE_INDEX=$(( DEVICE_ID + RANK_INDEX * 8 )) +export DEVICE_INDEX=${DEVICE_INDEX} + +cd ${train_job_dir} +curd_dir=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +export PYTHONPATH=$PYTHONPATH:${curd_dir} + +if [ x"$6" != x"True" ];then + rank_id=$6 + export RANK_ID=$6 +else + device_id_mo=$(python3.7 -c "import src.tensorflow.mpi_ops as atlasboost;atlasboost.init(); \ + device_id = atlasboost.local_rank();cluster_device_id = str(device_id); \ + atlasboost.set_device_id(device_id);print(atlasboost.rank())") + device_id_mo=`echo $device_id_mo` + rank_id=${device_id_mo##* } + export RANK_ID=${rank_id} + device=${device_id_mo##*deviceid = } + device_id=${device%% phyid=*} + export DEVICE_ID=${device_id} + hccljson=${train_job_dir}/*.json + cp ${hccljson} ${currentDir}/config/${rank_size}p.json +fi + +export MODEL_CKPT_PATH=${train_job_dir}/${device_id}/ckpt${device_id} + +#mkdir exec path +mkdir -p ${train_job_dir}/${device_id} +cd ${train_job_dir}/${device_id} + +startTime=`date +%Y%m%d-%H:%M:%S` +startTime_s=`date +%s` + +# 根据单卡/多卡区分调用参数 + +if [ x"$6" == x"True" ];then + export CLUSTER=True + echo "run cluster" + python3.7 ${currentDir}/code/official/benchmark/resnet_tf_r1_benchmark.py ${rank_size} > ${train_job_dir}/train_${device_id}.log 2>&1 +elif [ ${rank_size} -le 4 ];then + # 单卡 + python3.7 ${currentDir}/code/official/benchmark/resnet_tf_r1_benchmark.py ${rank_size} > ${train_job_dir}/train_${device_id}.log 2>&1 +elif [ ${rank_size} -le 8 ];then + # 多卡单机 + python3.7 ${currentDir}/code/official/benchmark/resnet_tf_r1_benchmark.py ${rank_size} > ${train_job_dir}/train_${device_id}.log 2>&1 +fi + +if [ $? -eq 0 ] ; +then + echo ":::ABK 1.0.0 resnet50 train success" + echo ":::ABK 1.0.0 resnet50 train success" >> ${train_job_dir}/train_${device_id}.log + echo ":::ABK 1.0.0 resnet50 train success" >> ${train_job_dir}/${device_id}/hw_resnet50.log +else + echo ":::ABK 1.0.0 resnet50 train failed" + echo ":::ABK 1.0.0 resnet50 train failed" >> ${train_job_dir}/train_${device_id}.log + echo ":::ABK 1.0.0 resnet50 train failed" >> ${train_job_dir}/${device_id}/hw_resnet50.log +fi + +endTime=`date +%Y%m%d-%H:%M:%S` +endTime_s=`date +%s` + +sumTime=$[ $endTime_s - $startTime_s ] + +hour=$(( $sumTime/3600 )) +min=$(( ($sumTime-${hour}*3600)/60 )) +sec=$(( $sumTime-${hour}*3600-${min}*60 )) +echo ":::ABK 1.0.0 resnet50 train total time：${hour}:${min}:${sec}" + +echo ":::ABK 1.0.0 resnet50 train total time：${hour}:${min}:${sec}" >> ${train_job_dir}/${device_id}/hw_resnet50.log + diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/__init__.py b/train/atlas_benchmark-master/image_classification/ResNext50/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/README.md b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/README.md new file mode 100644 index 0000000..142b99e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/README.md @@ -0,0 +1,46 @@ +# ResNext50_tensorflow训练说明 + +### 1. 模型训练参数配置 + +在train/yaml/ResNext50.yaml中修改相应配置， 配置项含义: + +``` +tensorflow_config: + # 基本参数 + max_steps: 1000 + data_url: /home/imagenet_TF/ + epoches: 1 + epochs_between_evals: 1 + batch_size: 32 + + # 仅多机执行需要配置: ip1:卡数量1,ip2:卡数量2 + mpirun_ip: 90.90.176.152:8,90.90.176.154:8 + + # docker 镜像名称:版本号 + docker_image: mpirun3:latest + + + # 1. 指定 device id, 多个 id 使用空格分隔, 数量需与 rank_size 相同 + # 2. 仅在小于 8p 时生效 + # 3. 若不使用该配置, 请使用在行首添加'#'注释的方法将其关闭 + # device_group: 0 1 2 3 + device_group_1p: 0 + device_group_2p: 0 1 + device_group_4p: 0 1 2 3 + + profiling_mode: false + profiling_options: training_trace + fp_point: fp32_vars/conv2d/Conv2Dfp32_vars/BatchNorm/FusedBatchNormV3_Reduce + bp_point: loss_scale/gradients/AddN_70 + aicpu_profiling_mode: false +``` + +------ + + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/configs/__init__.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/configs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/configs/pid b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/configs/pid new file mode 100644 index 0000000..0d49bb1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/configs/pid @@ -0,0 +1 @@ +13650 \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/configs/res50_32bs_1p.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/configs/res50_32bs_1p.py new file mode 100644 index 0000000..1938397 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/configs/res50_32bs_1p.py @@ -0,0 +1,115 @@ +import tensorflow as tf + +import os +log_dir = '../result/'+os.path.basename(__file__).split('.')[0] + +#256 +config = { + # ============ for testing ===================== + 'accelerator': '1980', # 'gpu', '1980' + 'shuffle_enable': 'yes', + 'shuffle_buffer_size': 10000, + 'rank_size': 1, + 'shard': False, + + # ======= basic config ======= # + 'mode':'train', # "train","evaluate","train_and_evaluate" + 'epochs_between_evals': 4, #used if mode is "train_and_evaluate" + 'stop_threshold': 80.0, #used if mode is "train_and_evaluate" + #'data_dir':'/opt/npu/resnet_data_new', + 'data_url': '/home/imagenet_TF', #data + 'data_type': 'TFRECORD', + 'model_name': 'resnet50', + 'num_classes': 1001, + 'num_epochs': 1, + 'height':224, + 'width':224, + 'dtype': tf.float32, + 'data_format': 'channels_last', + 'use_nesterov': True, + 'eval_interval': 1, + 'num_evaluating_samples': 50000, + 'loss_scale': 1024, #could be float or string. If float, static loss scaling is applied. + #If string, the corresponding automatic loss scaling algorithm is used. + #Must be one of 'Backoff' of 'LogMax' (case insensitive). + 'use_lars': False, + 'label_smoothing':0.1, #If greater than 0 then smooth the labels. + 'weight_decay': 0.0001, + 'batch_size':32, #minibatch size per node, total batchsize = batch_size*hvd.size()*itersize + + 'momentum': [0.9], + + #======= data processing config ======= + 'min_object_covered': 0.1, #used for random crop + 'aspect_ratio_range':[3. / 4., 4. / 3.], + 'area_range':[0.16, 1.0], + 'max_attempts': 100, + + #======= data augment config ======= + 'increased_aug': False, + 'brightness':0.3, + 'saturation': 0.6, + 'contrast': 0.6, + 'hue': 0.13, + 'num_preproc_threads': 22, + + #======= initialization config ======= + 'conv_init': tf.variance_scaling_initializer(), + 'bn_init_mode': 'adv_bn_init', # "conv_bn_init" or "adv_bn_init",initializer the gamma in bn in different modes + # "adv_bn_init" means initialize gamma to 0 in each residual block's last bn, and initialize other gamma to 1 + # "conv_bn_init" means initialize all the gamma to a constant, defined by "bn_gamma_initial_value" + 'bn_gamma_initial_value': 1.0, + + #======== model architecture ========== + #'resnet_version': 'v1.5', + 'resnet_version': 'resnext', + 'arch_type': 'original', # ------ input ------- + # C1,C2,C3: input block, stride in different layer + # ------ shortcut ------ + # D1: average_pooling + conv1*1 in shortcut in downsample block + # D2: conv3*3,stride=2 in shortcut in downsample block + # D3: conv1*1 +average_pooling in shortcut in downsample block + # ------ mainstream ---- + # E1: average_pooling + conv3*3 in mainstream in downsample block + # E2: conv3*3 + average_pooling in mainstream in downsample block + + #======= logger config ======= + 'display_every': 1, + 'log_name': 'resnet50.log', + 'log_dir': log_dir, + #'ckpt_dir': '/data/resnext50/opp2/ckpt0', + 'ckpt_dir': os.path.join(os.path.abspath(os.path.dirname(__file__)),'../../../result/ckpt0'), + + #======= Learning Rate Config ======= + 'lr_warmup_mode': 'linear', # "linear" or "cosine" + 'warmup_lr': 0.0, + 'warmup_epochs': 10, + 'learning_rate_maximum': 0.1, + + 'lr_decay_mode': 'steps', # "steps", "poly", "poly_cycle", "cosine", "linear_cosine", "linear_twice", "constant" for 1980 only + 'learning_rate_end': 0.00001, + + 'decay_steps': '10,20,30', #for "steps" + 'lr_decay_steps': '6.4,0.64,0.064', + + 'ploy_power': 2.0, #for "poly" and "poly_cycle" + + 'cdr_first_decay_ratio': 0.33, #for "cosine_decay_restarts" + 'cdr_t_mul':2.0, + 'cdr_m_mul':0.1, + + 'lc_periods':0.47, #for "linear_consine" + 'lc_beta':0.00001, + + 'lr_mid': 0.5, #for "linear_twice" + 'epoch_mid': 80, + + 'bn_lr_scale':1.0, + + } + +def res50_config(): + config['global_batch_size'] = config['batch_size'] * config['rank_size'] + config['do_checkpoint'] = True + + return config diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/configs/res50_32bs_8p.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/configs/res50_32bs_8p.py new file mode 100644 index 0000000..cb5d667 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/configs/res50_32bs_8p.py @@ -0,0 +1,115 @@ +import tensorflow as tf + +import os +log_dir = '../result/'+os.path.basename(__file__).split('.')[0] + +#256 +config = { + # ============ for testing ===================== + 'accelerator': '1980', # 'gpu', '1980' + 'shuffle_enable': 'yes', + 'shuffle_buffer_size': 10000, + 'rank_size': 8, + 'shard': True, + + # ======= basic config ======= # + 'mode':'train', # "train","evaluate","train_and_evaluate" + 'epochs_between_evals': 4, #used if mode is "train_and_evaluate" + 'stop_threshold': 80.0, #used if mode is "train_and_evaluate" + 'data_dir':'/opt/npu/resnet_data_new', + 'data_url': '/home/imagenet_TF', + 'data_type': 'TFRECORD', + 'model_name': 'resnet50', + 'num_classes': 1001, + 'num_epochs': 120, #None + 'height':224, + 'width':224, + 'dtype': tf.float32, + 'data_format': 'channels_last', + 'use_nesterov': True, + 'eval_interval': 1, + 'loss_scale': 1024, #could be float or string. If float, static loss scaling is applied. + #If string, the corresponding automatic loss scaling algorithm is used. + #Must be one of 'Backoff' of 'LogMax' (case insensitive). + 'use_lars': False, + 'label_smoothing':0.1, #If greater than 0 then smooth the labels. + 'weight_decay': 0.0001, + 'batch_size':32, #minibatch size per node, total batchsize = batch_size*hvd.size()*itersize + + 'momentum': [0.9], + + #======= data processing config ======= + 'min_object_covered': 0.1, #used for random crop + 'aspect_ratio_range':[3. / 4., 4. / 3.], + 'area_range':[0.16, 1.0], + 'max_attempts': 100, + + #======= data augment config ======= + 'increased_aug': False, + 'brightness':0.3, + 'saturation': 0.6, + 'contrast': 0.6, + 'hue': 0.13, + 'num_preproc_threads': 22, + + #======= initialization config ======= + 'conv_init': tf.variance_scaling_initializer(), + 'bn_init_mode': 'adv_bn_init', # "conv_bn_init" or "adv_bn_init",initializer the gamma in bn in different modes + # "adv_bn_init" means initialize gamma to 0 in each residual block's last bn, and initialize other gamma to 1 + # "conv_bn_init" means initialize all the gamma to a constant, defined by "bn_gamma_initial_value" + 'bn_gamma_initial_value': 1.0, + + #======== model architecture ========== + #'resnet_version': 'v1.5', + 'resnet_version': 'resnext', + 'arch_type': 'original', # ------ input ------- + # C1,C2,C3: input block, stride in different layer + # ------ shortcut ------ + # D1: average_pooling + conv1*1 in shortcut in downsample block + # D2: conv3*3,stride=2 in shortcut in downsample block + # D3: conv1*1 +average_pooling in shortcut in downsample block + # ------ mainstream ---- + # E1: average_pooling + conv3*3 in mainstream in downsample block + # E2: conv3*3 + average_pooling in mainstream in downsample block + + #======= logger config ======= + 'display_every': 1, + 'log_name': 'resnet50.log', + #'ckpt_dir': '/data/resnext50_10w/ckpt0', #log_dir + #'ckpt_dir': '/d_solution/ckpt0', + 'ckpt_dir': os.path.join(os.path.abspath(os.path.dirname(__file__)),'../../../result/ckpt0'), + 'log_dir': log_dir, + + #======= Learning Rate Config ======= + 'lr_warmup_mode': 'linear', # "linear" or "cosine" + 'warmup_lr': 0.0, + 'warmup_epochs': 5, + 'learning_rate_maximum': 0.1, + + 'lr_decay_mode': 'cosine', # "steps", "poly", "poly_cycle", "cosine", "linear_cosine", "linear_twice", "constant" for 1980 only + 'learning_rate_end': 0.000001, + + 'decay_steps': '10,20,30', #for "steps" + 'lr_decay_steps': '6.4,0.64,0.064', + + 'ploy_power': 2.0, #for "poly" and "poly_cycle" + + 'cdr_first_decay_ratio': 0.33, #for "cosine_decay_restarts" + 'cdr_t_mul':2.0, + 'cdr_m_mul':0.1, + + 'lc_periods':0.47, #for "linear_consine" + 'lc_beta':0.00001, + + 'lr_mid': 0.5, #for "linear_twice" + 'epoch_mid': 80, + + 'bn_lr_scale':1.0, + + } + +def res50_config(): + config['global_batch_size'] = config['batch_size'] * config['rank_size'] + config['do_checkpoint'] = True + + return config diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/data_loader/__init__.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/data_loader/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/data_loader/resnet50/data_loader.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/data_loader/resnet50/data_loader.py new file mode 100644 index 0000000..4cec4ba --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/data_loader/resnet50/data_loader.py @@ -0,0 +1,236 @@ +import numpy as np +from . import preprocessing +import tensorflow as tf +from tensorflow.python.util import nest +import os,sys +import numpy as np +sys.path.append("..") +from trainers.train_helper import stage + +class DataLoader: + + def __init__(self, config): + self.config = config + + # dataset info + num_training_samples = 1281167 + self.config['num_evaluating_samples'] = 50000 + #num_evaluating_samples = get_num_records(self.eval_filenames) + self.config['num_training_samples'] = num_training_samples + print( 'total num_training_sampels: %d' % num_training_samples ) + + self.training_samples_per_rank = num_training_samples + + + def get_train_input_fn_synthetic(self): + batch_size = self.config['batch_size'] + input_shape = [self.config['height'], self.config['width'], 3] + input_element = nest.map_structure(lambda s: tf.constant(0.5, tf.float32, s), tf.TensorShape(input_shape)) + label_element = nest.map_structure(lambda s: tf.constant(1, tf.int32, s), tf.TensorShape([1])) + element = (input_element, label_element) + ds = tf.data.Dataset.from_tensors(element).repeat() + ds = ds.batch(batch_size) + return ds + + def get_train_input_fn(self): + # filenames = self.train_filenames + filenames = None + take_count = self.training_samples_per_rank + batch_size = self.config['batch_size'] + height = self.config['height'] + width = self.config['width'] + brightness = self.config['brightness'] + contrast = self.config['contrast'] + saturation = self.config['saturation'] + hue = self.config['hue'] + num_threads = self.config['num_preproc_threads'] + increased_aug = self.config['increased_aug'] + shard = self.config['shard'] + + return make_dataset(self.config, filenames, take_count, batch_size, height, width, + brightness, contrast, saturation, hue, + training=True, num_threads=num_threads, nsummary=10, shard=shard, synthetic=False, + increased_aug=increased_aug ) + + def get_eval_input_fn(self): + # filenames = self.eval_filenames + filenames = None + # take_count = get_num_records(self.eval_filenames) + take_count = 50000 + batch_size = self.config['batch_size'] + height = self.config['height'] + width = self.config['width'] + brightness = self.config['brightness'] + contrast = self.config['contrast'] + saturation = self.config['saturation'] + hue = self.config['hue'] + num_threads = self.config['num_preproc_threads'] + shard = self.config['shard'] + + return make_dataset(self.config, filenames, take_count, batch_size, height, width, + brightness, contrast, saturation, hue, + training=False, num_threads=num_threads, nsummary=10, shard=shard, synthetic=False, + increased_aug=False) + + def get_input_pipeline_op(self, inputs, labels, mode): + with tf.device('/cpu:0'): + preload_op, (inputs, labels) = stage([inputs, labels]) + + with tf.device('/gpu:0'): + gpucopy_op, (inputs, labels) = stage([inputs, labels]) + return preload_op, gpucopy_op, inputs, labels + + def normalize_and_format(self, inputs, data_format): + + dataset_mean = np.array([121, 115, 100], dtype=np.float32) + dataset_std = np.array([70, 68, 71], dtype=np.float32) + inputs = tf.subtract(inputs, dataset_mean) + inputs = tf.multiply(inputs, 1. / dataset_std) + if data_format == 'channels_first': + inputs = tf.transpose(inputs, [0, 3, 1, 2]) + return inputs + + + + +#-------------------------------- Funcs ----------------------------------- +def get_num_records(filenames): + def count_records(tf_record_filename): + count = 0 + for _ in tf.python_io.tf_record_iterator(tf_record_filename): + count += 1 + return count + + nfile = len(filenames) + return (count_records(filenames[0]) * (nfile - 1) + + count_records(filenames[-1])) + +def _parse_example_proto(example_serialized): + feature_map = { + 'image/encoded': tf.FixedLenFeature([], dtype=tf.string, + default_value=''), + 'image/class/label': tf.FixedLenFeature([], dtype=tf.int64, default_value=-1), + 'image/class/text': tf.FixedLenFeature([], dtype=tf.string, + default_value=''), + } + sparse_float32 = tf.VarLenFeature(dtype=tf.float32) + # Sparse features in Example proto. + feature_map.update( + {k: sparse_float32 for k in ['image/object/bbox/xmin', + 'image/object/bbox/ymin', + 'image/object/bbox/xmax', + 'image/object/bbox/ymax']}) + + features = tf.parse_single_example(example_serialized, feature_map) + label = tf.cast(features['image/class/label'], dtype=tf.int32) + + xmin = tf.expand_dims(features['image/object/bbox/xmin'].values, 0) + ymin = tf.expand_dims(features['image/object/bbox/ymin'].values, 0) + xmax = tf.expand_dims(features['image/object/bbox/xmax'].values, 0) + ymax = tf.expand_dims(features['image/object/bbox/ymax'].values, 0) + + # Note that we impose an ordering of (y, x) just to make life difficult. + bbox = tf.concat([ymin, xmin, ymax, xmax], 0) + + # Force the variable number of bounding boxes into the shape + # [1, num_boxes, coords]. + bbox = tf.expand_dims(bbox, 0) + bbox = tf.transpose(bbox, [0, 2, 1]) + + return features['image/encoded'], label, bbox + +def parse_record(raw_record,is_training): + image_buffer, label, bbox = _parse_example_proto(raw_record) + # for 1980 only + config={'min_object_covered': 0.1, 'aspect_ratio_range': [3. / 4., 4. / 3.], 'area_range': [0.08, 1.0], 'max_attempts': 100} + image = preprocessing.parse_and_preprocess_image_record( + config, image_buffer, height=224, width=224, + brightness=0.4, contrast=0.4, saturation=0.4, hue=0.13, + distort=is_training, nsummary=10, increased_aug=True, random_search_aug=False) + return image, label + +def read_rawdata(file_path_tensor): + def _read_file(file_path): + image = tf.gfile.GFile(file_path, 'rb').read() + return image + return tf.py_func(_read_file, inp=[file_path_tensor], Tout=tf.string) + +def parse_function(filename, label): + image = read_rawdata(filename) + image_decoded = tf.image.decode_jpeg(image, channels=3) + image_resized = tf.image.resize_images(image_decoded, [224, 224]) + # 7.3rawĬϸʽΪint64Ŀǰresnet50ֻ֧int32³ǰӰ죬³ûӸתӣӰܿǡ + label = tf.cast(label, dtype=tf.int32) + return image_resized, label + + +def make_dataset(config, filenames, take_count, batch_size, height, width, + brightness, contrast, saturation, hue, + training=False, num_threads=10, nsummary=10, shard=False, synthetic=False, + increased_aug=False, random_search_aug=False): + if synthetic and training: + input_shape = [height, width, 3] + input_element = nest.map_structure(lambda s: tf.constant(0.5, tf.float32, s), tf.TensorShape(input_shape)) + label_element = nest.map_structure(lambda s: tf.constant(1, tf.int32, s), tf.TensorShape([1])) + element = (input_element, label_element) + ds = tf.data.Dataset.from_tensors(element).repeat() + ds = ds.batch(batch_size) + return ds + else: + shuffle_buffer_size = 10000 + num_readers = 10 + rank_size = int(os.getenv('RANK_SIZE')) + rank_id = int(os.getenv('DEVICE_INDEX')) + + if config['data_type'] == 'RAW DATA': + images = [] + labels = [] + with tf.gfile.GFile(config['label_index_url'], 'r') as f: + for line in f.readlines(): + tmp_list = line.strip().split(" ") + image_file = os.path.join(config['data_url'], tmp_list[0]) + #image_raw = tf.gfile.GFile(image_file, 'rb').read() + #images.append(image_raw) + images.append(image_file) + labels.append(int(tmp_list[-1])) + + #images = tf.convert_to_tensor(images, dtype=tf.string) + #labels = tf.convert_to_tensor(labels, dtype=tf.int32) + ds = tf.data.Dataset.from_tensor_slices((images, labels)) + else: + if training: + filename_pattern = os.path.join(config['data_url'], '%s-*') + filenames = sorted(tf.gfile.Glob(filename_pattern % 'train')) + else: + filename_pattern = os.path.join(config['data_url'], '%s-*') + filenames = sorted(tf.gfile.Glob(filename_pattern % 'validation')) + + ds = tf.data.Dataset.from_tensor_slices(filenames) + + if shard: + # split the dataset into parts for each GPU + ds = ds.shard(rank_size, rank_id) + + if not training: + ds = ds.take(take_count) # make sure all ranks have the same amount + + if training: + ds = ds.shuffle(1000, seed=7 * (1 + rank_id)) + + if config['data_type'] == 'TFRECORD': + ds = ds.interleave(tf.data.TFRecordDataset, cycle_length=num_readers, block_length=1) + counter = tf.data.Dataset.range(sys.maxsize) + ds = tf.data.Dataset.zip((ds, counter)) + + if training: + ds = ds.apply(tf.data.experimental.shuffle_and_repeat(shuffle_buffer_size, seed=5*(1+rank_id))) + + if config['data_type'] == 'RAW DATA': + ds = ds.map(lambda image, label: parse_function(image, label), num_parallel_calls=14) + else: + ds = ds.map(lambda image, label: parse_record(image, training), num_parallel_calls=14) + #ds = ds.prefetch(10) + ds = ds.batch(batch_size, drop_remainder=True) + return ds + + diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/data_loader/resnet50/preprocessing.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/data_loader/resnet50/preprocessing.py new file mode 100644 index 0000000..bde13df --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/data_loader/resnet50/preprocessing.py @@ -0,0 +1,152 @@ +import tensorflow as tf +#import horovod.tensorflow as hvd +from tensorflow.contrib.image.python.ops import distort_image_ops +import math +#from .data_aug_search import random_aug_search + + + +def deserialize_image_record(record): + feature_map = { + 'image/encoded': tf.FixedLenFeature([], tf.string, ''), + 'image/class/label': tf.FixedLenFeature([1], tf.int64, -1), + 'image/class/text': tf.FixedLenFeature([], tf.string, ''), + 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32) + } + with tf.name_scope('deserialize_image_record'): + obj = tf.parse_single_example(record, feature_map) + imgdata = obj['image/encoded'] + label = tf.cast(obj['image/class/label'], tf.int32) + bbox = tf.stack([obj['image/object/bbox/%s' % x].values + for x in ['ymin', 'xmin', 'ymax', 'xmax']]) + bbox = tf.transpose(tf.expand_dims(bbox, 0), [0, 2, 1]) + text = obj['image/class/text'] + return imgdata, label, bbox, text + +def decode_jpeg(imgdata, channels=3): + return tf.image.decode_jpeg(imgdata, channels=channels, + fancy_upscaling=False, + dct_method='INTEGER_FAST') + + +def crop_and_resize_image(config, image, height, width, + distort=False, nsummary=10): + with tf.name_scope('crop_and_resize'): + # Evaluation is done on a center-crop of this ratio + eval_crop_ratio = 0.8 + if distort: + initial_shape = [int(round(height / eval_crop_ratio)), + int(round(width / eval_crop_ratio)), + 3] + jpeg_shape = tf.image.extract_jpeg_shape( image ) + + bbox_begin, bbox_size, bbox = \ + tf.image.sample_distorted_bounding_box( + initial_shape, + bounding_boxes=tf.constant([0.0, 0.0, 1.0, 1.0], dtype=tf.float32, shape=[1, 1, 4]), + # tf.zeros(shape=[1,0,4]), # No bounding boxes + min_object_covered=config['min_object_covered'], + aspect_ratio_range=config['aspect_ratio_range'], + area_range=config['area_range'], + max_attempts=config['max_attempts'], + # seed=11 , # Need to set for deterministic results + use_image_if_no_bounding_boxes=True) + bbox = bbox[0, 0] # Remove batch, box_idx dims + + # offset_y, offset_x, _ = tf.unstack(bbox_begin) + # target_height, target_width, _ = tf.unstack( bbox_size ) + # + + + + + # offset_y = tf.minimum( offset_y, jpeg_shape[0] - 1 ) + # offset_x = tf.minimum( offset_x, jpeg_shape[1] - 1 ) + + # target_height, target_width, _ = tf.unstack( bbox_size ) + # new_height = tf.maximum( tf.minimum( offset_y + target_height, jpeg_shape[0] ) - offset_y, 0 ) + # new_width = tf.maximum( tf.minimum( offset_x + target_width, jpeg_shape[1] ) - offset_x, 0 ) + + y_min = tf.cast( bbox[0] * (tf.cast( jpeg_shape[0], tf.float32) ), tf.int32) + x_min = tf.cast( bbox[1] * (tf.cast(jpeg_shape[1], tf.float32) ), tf.int32) + y_max = tf.cast( bbox[2] * (tf.cast(jpeg_shape[0], tf.float32) ), tf.int32) + x_max = tf.cast( bbox[3] * (tf.cast(jpeg_shape[1], tf.float32) ), tf.int32) + + crop_height = y_max - y_min + crop_width = x_max - x_min + # crop_window = tf.stack( [offset_y, offset_x, new_height, new_width] ) + crop_window = tf.stack( [y_min, x_min, crop_height, crop_width] ) + image = tf.image.decode_and_crop_jpeg( image, crop_window, channels=3 ) + image = tf.image.resize_images( image, [height, width] ) + + + # def func_decode_and_crop(image): + # image = tf.image.decode_and_crop_jpeg( image, crop_window, channels=3 ) + # image = tf.image.resize_images( image, [height, width] ) + # return image + + # def func_crop_and_resize(image): + # image = decode_jpeg(image, channels=3) + # image = tf.image.crop_and_resize( + # image[None, :, :, :], bbox[None, :], [0], [height, width])[0] + # return image + + + # condtion_1 = tf.logical_and( tf.less(target_height, jpeg_shape[0]), tf.less( target_width, jpeg_shape[1] ) ) + # condtion_2 = tf.logical_and( tf.less(target_height + offset_y, jpeg_shape[0]), tf.less( target_width + offset_x, jpeg_shape[1] ) ) + + # image = tf.cond( tf.logical_and( condtion_1, condtion_2 ), lambda:func_decode_and_crop(image), lambda:func_crop_and_resize(image) ) + + + else: + # Central crop + + image = decode_jpeg(image, channels=3) + ratio_y = ratio_x = eval_crop_ratio + bbox = tf.constant([0.5 * (1 - ratio_y), 0.5 * (1 - ratio_x), + 0.5 * (1 + ratio_y), 0.5 * (1 + ratio_x)]) + image = tf.image.crop_and_resize( + image[None, :, :, :], bbox[None, :], [0], [height, width])[0] + + return image + + +def parse_and_preprocess_image_record(config, record, height, width, + brightness, contrast, saturation, hue, + distort, nsummary=10, increased_aug=False, random_search_aug=False): + #imgdata, label, bbox, text = deserialize_image_record(record) + #label -= 1 # Change to 0-based (don't use background class) + with tf.name_scope('preprocess_train'): + image = crop_and_resize_image(config, record, height, width, distort) + if distort: + image = tf.image.random_flip_left_right(image) + if increased_aug: + image = tf.image.random_brightness(image, max_delta=brightness) + #image = distort_image_ops.random_hsv_in_yiq(image, + # lower_saturation=saturation, + # upper_saturation=2.0 - saturation, + # max_delta_hue=hue * math.pi) + image = tf.image.random_contrast(image, lower=contrast, upper=2.0 - contrast) + image = tf.image.random_saturation(image, lower=saturation, upper=2.0-saturation) + # tf.summary.image('distorted_color_image', tf.expand_dims(image, 0)) + + image = tf.clip_by_value(image, 0., 255.) + #image = tf.cast(image, tf.uint8) + # if random_search_aug: + # image = random_aug_search(image) + image = normalize(image) + image = tf.cast(image, tf.float16) + return image +def normalize(inputs): + imagenet_mean = [121.0, 115.0, 100.0] #np.array([121, 115, 100], dtype=np.float32) + imagenet_std = [70.0, 68.0, 71.0] #np.array([70, 68, 71], dtype=np.float32) + imagenet_mean = tf.expand_dims(tf.expand_dims(imagenet_mean, 0), 0) + imagenet_std = tf.expand_dims(tf.expand_dims(imagenet_std, 0), 0) + inputs = inputs - imagenet_mean #tf.subtract(inputs, imagenet_mean) + inputs = inputs * (1.0 / imagenet_std) + #inputs = tf.multiply(inputs, 1. / imagenet_std) + + return inputs diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/hyper_param/__init__.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/hyper_param/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/hyper_param/hyper_param.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/hyper_param/hyper_param.py new file mode 100644 index 0000000..eed8743 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/hyper_param/hyper_param.py @@ -0,0 +1,50 @@ +import tensorflow as tf +from .lr_schedule import warmup_decay, get_lr, get_1980_lr + + +class HyperParams: + def __init__(self, config): + self.config=config + nsteps_per_epoch = self.config['num_training_samples'] // self.config['global_batch_size'] + self.config['nsteps_per_epoch'] = nsteps_per_epoch + # nstep = self.config['num_training_samples'] * self.config['num_epochs'] // self.config['global_batch_size'] + if self.config['num_epochs']: + nstep = nsteps_per_epoch * self.config['num_epochs'] #------calculate nsteps in a different way------ + else: + nstep = self.config['max_train_steps'] + self.config['nstep'] = nstep + + self.config['total_steps_include_iterations'] = int( self.config['nstep'] + self.config['iterations_per_loop']) + self.config['save_summary_steps'] = nsteps_per_epoch + self.config['save_checkpoints_steps'] = nsteps_per_epoch + + + def get_hyper_params(self): + hyper_params = {} + hyper_params['learning_rate'] = self.get_learning_rate() + + return hyper_params + + + def get_learning_rate(self): + global_step = tf.train.get_global_step() + nsteps_per_epoch = self.config['nsteps_per_epoch'] + + warmup_lr = self.config['warmup_lr'] + lr = self.config['learning_rate_maximum'] + lr_end = self.config['learning_rate_end'] + lr_decay_mode = self.config['lr_decay_mode'] + + + + with tf.device('/cpu:0'): # Allow fallback to CPU if no GPU support for these ops + + if lr_decay_mode == 'constant' or self.config['num_epochs'] == None: + learning_rate = tf.constant(lr, tf.float32) + else: + learning_rate = get_1980_lr(self.config, global_step, warmup_lr, lr_end, lr, self.config['warmup_epochs'], nsteps_per_epoch, self.config['nstep'], lr_decay_mode ) + + learning_rate = tf.identity(learning_rate, 'learning_rate') + return learning_rate + + diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/hyper_param/lr_schedule.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/hyper_param/lr_schedule.py new file mode 100644 index 0000000..4484a9a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/hyper_param/lr_schedule.py @@ -0,0 +1,172 @@ +import tensorflow as tf +import numpy as np + +def get_lr(lr, lr_end, lr_decay_mode, warmup_it, decay_steps, global_step, steps, lr_steps, ploy_power, + cdr_first_decay_ratio, cdr_t_mul, cdr_m_mul, cdr_alpha, cd_alpha, lc_periods, lc_alpha, lc_beta, lr_mid, it_mid): + if lr_decay_mode == 'steps': + learning_rate = tf.train.piecewise_constant(global_step, + steps, lr_steps) + elif lr_decay_mode == 'poly' or lr_decay_mode == 'poly_cycle': + cycle = lr_decay_mode == 'poly_cycle' + learning_rate = tf.train.polynomial_decay(lr, + global_step - warmup_it, + decay_steps=decay_steps - warmup_it, + end_learning_rate=lr_end, + power=ploy_power, + cycle=cycle) + elif lr_decay_mode == 'cosine_decay_restarts': + learning_rate = tf.train.cosine_decay_restarts(lr, + global_step - warmup_it, + (decay_steps - warmup_it) * cdr_first_decay_ratio, + t_mul=cdr_t_mul, + m_mul=cdr_m_mul, + alpha=cdr_alpha) + elif lr_decay_mode == 'cosine': + learning_rate = tf.train.cosine_decay(lr, + global_step - warmup_it, + decay_steps=decay_steps - warmup_it, + alpha=cd_alpha) + elif lr_decay_mode == 'linear_cosine': + learning_rate = tf.train.linear_cosine_decay(lr, + global_step - warmup_it, + decay_steps=decay_steps - warmup_it, + num_periods=lc_periods,#0.47, + alpha=lc_alpha,#0.0, + beta=lc_beta)#0.00001) + elif lr_decay_mode == 'linear_twice': + learning_rate = decay_linear_twice(lr, lr_mid, lr_end, warmup_it, it_mid, decay_steps, global_step ) + + else: + raise ValueError('Invalid type of lr_decay_mode') + return learning_rate + + +def cos_warmup_1980( global_step, warmup_steps, max_lr ): + PI = 3.14159265359 + ang = PI + PI * ( float(global_step+1) / float(warmup_steps) ) + offset = max_lr * 0.5*( 1.0 + np.cos( ang ) ) + return offset + +def cos_decay_1980( global_step, warmup_steps, total_steps, max_lr,end_lr ): + PI = 3.14159265359 + ang = PI * ( float(global_step - warmup_steps+1) / float(total_steps - warmup_steps) ) + #offset = max_lr * 0.5*( 1.0 + np.cos( ang ) ) + + #zp-cosine + cosine_decay_tmp=0.5*( 1.0 + np.cos( ang ) ) + decayed_tmp = (1 - end_lr) * cosine_decay_tmp + end_lr + offset = max_lr * decayed_tmp + return offset + +def get_1980_lr(config, global_step, lr_init, lr_end, lr_max, warmup_epochs, steps_per_epoch, nsteps, lr_decay_mode): + lr_each_step = [] + + if lr_decay_mode == 'steps': + decay_epoch_index = [30 * steps_per_epoch,60 * steps_per_epoch,80 * steps_per_epoch] + total_steps = int(nsteps) + for i in range(total_steps): + if i < decay_epoch_index[0]: + lr = lr_max + elif i < decay_epoch_index[1]: + lr = lr_max * 0.1 + elif i < decay_epoch_index[2]: + lr = lr_max * 0.01 + else: + lr = lr_max * 0.001 + lr_each_step.append(lr) + elif lr_decay_mode == 'poly': + total_steps = int(nsteps) + warmup_steps = steps_per_epoch * warmup_epochs + inc_each_step = ( float(lr_max) - float(lr_init) ) / float(warmup_steps) + for i in range( config['total_steps_include_iterations'] ): + if i < warmup_steps: + lr = float(lr_init) + inc_each_step * float(i) + elif i <= total_steps: + base = ( 1.0 - (float(i)-float(warmup_steps))/(float(total_steps)-float(warmup_steps)) ) + lr = float(lr_max) * base + else: + lr = 0.0 + lr_each_step.append(lr) + + elif lr_decay_mode == 'cosine': + total_steps = int(nsteps) + + warmup_steps = steps_per_epoch * warmup_epochs + for i in range( config['total_steps_include_iterations'] ): + if i < warmup_steps: + lr = cos_warmup_1980( i, warmup_steps, lr_max ) + elif i <= total_steps: + lr = cos_decay_1980( i, warmup_steps, total_steps, lr_max ,lr_end) + else: + lr = lr_end * 0.01 + lr_each_step.append(lr) + elif lr_decay_mode == 'linear_cosine': + total_steps = int(nsteps) + warmup_steps = steps_per_epoch * warmup_epochs + inc_each_step = ( float(lr_max) - float(lr_init) ) / float(warmup_steps) + for i in range( config['total_steps_include_iterations'] ): + if i < warmup_steps: + lr = float(lr_init) + inc_each_step * float(i) + elif i <= total_steps: + lr = cos_decay_1980( i, warmup_steps, total_steps, lr_max ) + else: + lr = 0.0 + lr_each_step.append(lr) + else: + total_steps = int(nsteps) + warmup_steps = steps_per_epoch * warmup_epochs + for i in range(total_steps): + if i < warmup_steps: + lr = lr_init + (lr_max - lr_init) * i / warmup_steps + else: + lr = lr_max - ( lr_max - lr_end ) * (i - warmup_steps) / (total_steps - warmup_steps) + lr_each_step.append( lr ) + + # current_step = tf.to_int32( tf.cast(global_step,tf.float32) / float(steps_per_epoch) ) + current_step = global_step + lr_each_step = tf.convert_to_tensor( lr_each_step ) + print (lr_each_step) + learning_rate = tf.gather( lr_each_step, current_step ) + + return learning_rate + +def warmup_decay(lr_warmup_mode, warmup_lr, global_step, warmup_steps, warmup_end_lr): + if lr_warmup_mode == 'linear': + learning_rate = linear_warmup(warmup_lr, global_step, warmup_steps, warmup_end_lr) + elif lr_warmup_mode == 'cosine': + learning_rate = cos_warmup(warmup_lr, global_step, warmup_steps, warmup_end_lr) + else: + raise ValueError('Invalid type of lr_warmup_mode') + return learning_rate + + +def linear_warmup(warmup_lr, global_step, warmup_steps, warmup_end_lr): + from tensorflow.python.ops import math_ops + p = tf.cast(global_step, tf.float32) / tf.cast(warmup_steps, tf.float32) + diff = math_ops.subtract(warmup_end_lr, warmup_lr) + res = math_ops.add(warmup_lr, math_ops.multiply(diff, p)) + return res + +def cos_warmup( warmup_lr, global_step, warmup_steps, warmup_end_lr ): + PI = 3.14159265359 + diff = tf.subtract( warmup_end_lr, warmup_lr ) + ang = PI + PI * ( tf.cast( global_step, tf.float32 ) / tf.cast( warmup_steps,tf.float32 )) + offset = diff * 0.5 * ( 1.0 + tf.math.cos( ang ) ) + res = tf.add( warmup_lr, offset ) + return res + + +def decay_linear( lr_start, lr_end, it_start, it_end, global_step ): + down_steps = it_end - it_start + down_range = lr_start - lr_end + down_per_step = float( down_range ) / float( down_steps ) + res = tf.subtract( tf.cast(lr_start, tf.float32), tf.multiply( tf.cast(down_per_step, tf.float32), tf.subtract(tf.cast(global_step, tf.float32), tf.cast(it_start, tf.float32) )) ) + return res + +def decay_linear_twice(lr_start, lr_mid, lr_end, it_start, it_mid, it_end, global_step ): + learning_rate = tf.cond( global_step < it_start, lambda: tf.cast(lr_start, tf.float32), lambda: decay_linear(lr_start, lr_mid, it_start, it_mid, global_step)) + learning_rate = tf.cond( global_step > it_mid, lambda: decay_linear(lr_mid, lr_end, it_mid, it_end, global_step) , lambda: learning_rate ) + return learning_rate + + + diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/layers/__init__.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/layers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/layers/layers.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/layers/layers.py new file mode 100644 index 0000000..0312452 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/layers/layers.py @@ -0,0 +1,23 @@ +import tensorflow as tf +#from tensorflow.contrib.hccl.python.ops import hccl_ops +from npu_bridge.hccl import hccl_ops + +class Layers: + + def get_accuracy(self, labels, predicted_classes, logits, config): + accuracy = tf.metrics.accuracy( + labels=labels, predictions=predicted_classes) + top5acc = tf.metrics.mean( + tf.cast(tf.nn.in_top_k(logits, labels, 5), tf.float32)) + if config['rank_size'] == 1: + newaccuracy = (accuracy[0], accuracy[1]) + newtop5acc = (top5acc[0], top5acc[1]) + else: + newaccuracy = (hccl_ops.allreduce(accuracy[0],"sum")/config['rank_size'], accuracy[1]) + newtop5acc = (hccl_ops.allreduce(top5acc[0],"sum")/config['rank_size'], top5acc[1]) + metrics = {'val-top1acc': newaccuracy, 'val-top5acc': newtop5acc} + return metrics + + + + diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/losses/__init__.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/losses/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/losses/res50_loss.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/losses/res50_loss.py new file mode 100644 index 0000000..ad383f9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/losses/res50_loss.py @@ -0,0 +1,36 @@ +import tensorflow as tf + +class Loss: + def __init__(self,config): + self.config = config + + def get_loss(self, logits, labels): + labels_one_hot = tf.one_hot(labels, self.config['num_classes']) + loss = tf.losses.softmax_cross_entropy( + logits=logits, onehot_labels=labels_one_hot,label_smoothing=self.config['label_smoothing']) + loss = tf.identity(loss, name='loss') + return loss + + def get_total_loss(self, loss): + reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) + total_loss = tf.add_n([loss] + reg_losses, name='total_loss') + return total_loss + + + def optimize_loss(self, total_loss, opt): + gate_gradients = (tf.train.Optimizer.GATE_NONE) + # grads_and_vars = opt.compute_gradients(total_loss, colocate_gradients_with_ops=True, gate_gradients=gate_gradients) + grads_and_vars = opt.compute_gradients(total_loss, gate_gradients=gate_gradients) + + # train_op = opt.apply_gradients( grads_and_vars, global_step=None ) + train_op = opt.apply_gradients( grads_and_vars) + + return train_op + + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/mains/clean.sh b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/mains/clean.sh new file mode 100644 index 0000000..14ec7df --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/mains/clean.sh @@ -0,0 +1,7 @@ + ps -ef | grep TdtMain | awk '{print $2}' | xargs kill -9 +rm -rf *.pbtxt +rm -rf /var/log/npu/slog/*.log +rm ckpt* -rf +find ./ -name "*.pyc" | xargs rm -rf +find ./ -name __pycache__ | xargs rm -rf +rm /var/log/npu/dataset/* -rf diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/mains/res50.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/mains/res50.py new file mode 100644 index 0000000..724c89a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/mains/res50.py @@ -0,0 +1,141 @@ +import tensorflow as tf +import sys +import ast +#sys.path.append("..") +#sys.path.append("../models") +#sys.path.append("./resnet50_train/") +#sys.path.append("./resnet50_train/models") +import os +sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)),'../../../../../../utils')) +sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)),'../../../../../../utils/atlasboost')) +base_path=os.path.split(os.path.realpath(__file__))[0] +print ("#########base_path:", base_path) +path_1 = base_path + "/.." +print (path_1) +path_2 = base_path + "/../models" +print (path_2) +path_3 = base_path + "/../../" +print (path_3) + + +sys.path.append(base_path + "/..") +sys.path.append(base_path + "/../models") +sys.path.append(base_path + "/../../") +sys.path.append(base_path + "/../../models") + +from utils import create_session as cs +from utils import logger as lg +from data_loader.resnet50 import data_loader as dl +from models.resnet50 import res50_model as ml +from optimizers import optimizer as op +from losses import res50_loss as ls +from trainers import gpu_base_trainer as tr +# from configs import res50_config as cfg +from hyper_param import hyper_param as hp +from layers import layers as ly +from datetime import datetime +# from utils import hwlog +import argparse + +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter +# import hwlog +# remark_logger = hwlog.get_logger(__file__, "hw_Resnext50.log") +# initinal_data={"base_lr": 0.128, "dataset": "imagenet1024", "optimizer": "SGD", "loss_scale": 512, "batchsize": 32} +# hwlog.add_additional_info(remark_logger, "Resnext50", "tensorflow", initinal_data) # logger_obj, model_name, framework, initinal_data + + +def main(): + #-------------------choose the config file in .sh file----------- + cmdline = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + cmdline.add_argument('--config_file', default="", + help="""config file used.""") + cmdline.add_argument('--iterations_per_loop', default=1, + help="""config file used.""") + cmdline.add_argument('--max_train_steps', default=200, + help="""config file used.""") + cmdline.add_argument('--debug', default=True, type=ast.literal_eval, + help="""config file used.""") + cmdline.add_argument('--eval', default=False, type=ast.literal_eval, + help="""config file used.""") + cmdline.add_argument('--model_dir', default="./model_dir", + help="""config file used.""") + FLAGS, unknown_args = cmdline.parse_known_args() + if len(unknown_args) > 0: + for bad_arg in unknown_args: + print("ERROR: Unknown command line arg: %s" % bad_arg) + raise ValueError("Invalid command line arg(s)") + + cfg_file = FLAGS.config_file + configs = 'configs' + cfg = getattr(__import__(configs, fromlist=[cfg_file]), cfg_file) + #------------------------------------------------------------------ + + config = cfg.res50_config() + config['iterations_per_loop'] = int(FLAGS.iterations_per_loop) + config['max_train_steps'] = int(FLAGS.max_train_steps) + config['debug'] = FLAGS.debug + config['eval'] = FLAGS.eval + config['model_dir'] = FLAGS.model_dir + print("iterations_per_loop:%d" %(config['iterations_per_loop'])) + print("max_train_steps :%d" %(config['max_train_steps'])) + print("debug :%s" %(config['debug'])) + print("eval :%s" %(config['eval'])) + print("model_dir :%s" %(config['model_dir'])) + Session = cs.CreateSession(config) + data = dl.DataLoader(config) + hyper_param = hp.HyperParams(config) + layers = ly.Layers() + optimizer = op.Optimizer(config) + loss = ls.Loss(config) + logger = lg.LogSessionRunHook(config) # add tensorboard summary + + model = ml.Model(config, data, hyper_param,layers, optimizer, loss, logger) # get the model + trainer = tr.GPUBaseTrain(Session, config, data, model, logger) # use Estimator to build training process + + # work_num = "device " + str(os.environ.get("DEVICE_INDEX")) + # date_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') + # try: + if config['mode'] =='train': + trainer.train() + if config['eval'] : + trainer.evaluate() + elif config['mode'] =='evaluate': + trainer.evaluate() + elif config['mode'] =='train_and_evaluate': + trainer.train_and_evaluate() + else: + raise ValueError('Invalid type of mode') + # hwlog.vlogger.info("namespace:%s,time_ts:%s,event_type:benchmark_stop" % (work_num, date_time)) + # hwlog.vlogger.info("atlas benchmark train success") + # remark_logger.info("ABK train success") + # except: + # # hwlog.vlogger.info("namespace:%s,time_ts:%s,event_type:benchmark_stop" % (work_num, date_time)) + # # hwlog.vlogger.info("atlas benchmark train failed") + # remark_logger.info("ABK train failed") + + # add by zwx5326390 + + + +if __name__ == '__main__': + # add zwx5326390 日志打点 + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("tensorflow") + config_info = get_model_parameter("tensorflow_config") + initinal_data = {"base_lr": 0.01, "dataset": "imagenet1024", "optimizer": "SGD", "loss_scale": 512, + "batchsize": 32} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + hwlog.remark_print(key=hwlog.INPUT_BATCH_SIZE, value=initinal_data.get("batchsize")) + main() diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/mains/train_res50.sh b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/mains/train_res50.sh new file mode 100644 index 0000000..bbd2da5 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/mains/train_res50.sh @@ -0,0 +1,21 @@ +#!/bin/bash +#export CUDA_VISIBLE_DEVICES=0 +dir=`pwd` + +#cp -rf ./config /tmp/ +export JOB_ID=10086 +#export PROFILING_DIR=/var/log/npu/profiling/container/0 +export DEVICE_ID=0 +#export PROFILING_MODE=true +export PRINT_MODEL=1 +#export ENABLE_DATA_PRE_PROC=1 +export RANK_ID=0 +export RANK_SIZE=1 +export RANK_TABLE_FILE=/home/lxh/config/new_rank_table_1p.json +export FUSION_TENSOR_SIZE=1000000000 +export PYTHONPATH=${dir} +export LD_LIBRARY_PATH=/usr/local/HiAI/runtime/lib64/ +/usr/local/HiAI/runtime/bin/TdtMain --configfile=/home/lxh/test/config/job_tdt_2p_$DEVICE_ID.json & +sleep 5 + +python3.6 res50.py --config_file res50_baseline diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/mains/train_res50_gpu.sh b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/mains/train_res50_gpu.sh new file mode 100644 index 0000000..c89b78e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/mains/train_res50_gpu.sh @@ -0,0 +1,4 @@ +#!/bin/bash +export CUDA_VISIBLE_DEVICES=7 + +python3.5 res50.py --config_file res50_baseline_gpu diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/models/__init__.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/models/resnet50/res50_helper.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/models/resnet50/res50_helper.py new file mode 100644 index 0000000..0e16df6 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/models/resnet50/res50_helper.py @@ -0,0 +1,24 @@ +import tensorflow as tf + +def _fp32_trainvar_getter(getter, name, shape=None, dtype=None, + trainable=True, regularizer=None, + *args, **kwargs): + storage_dtype = dtype + variable = getter(name, shape, dtype=storage_dtype, + trainable=trainable, + regularizer=regularizer if trainable and 'BatchNorm' not in name and 'batchnorm' not in name and 'batch_norm' not in name and 'Batch_Norm' not in name else None, + *args, **kwargs) + + return variable + + +def fp32_trainable_vars(name='fp32_vars', *args, **kwargs): + """A varible scope with custom variable getter to convert fp16 trainable + variables with fp32 storage followed by fp16 cast. + """ + return tf.variable_scope( + name, custom_getter=_fp32_trainvar_getter, *args, **kwargs) + +def custom_getter_with_fp16_and_weight_decay(dtype, weight_decay): + return fp32_trainable_vars(dtype=dtype, regularizer=tf.contrib.layers.l2_regularizer(weight_decay)) + diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/models/resnet50/res50_model.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/models/resnet50/res50_model.py new file mode 100644 index 0000000..02e3958 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/models/resnet50/res50_model.py @@ -0,0 +1,222 @@ + +import tensorflow as tf +from . import resnet, res50_helper +from trainers.train_helper import stage +#from tensorflow.contrib.offline_train.python.npu.npu_optimizer import NPUDistributedOptimizer +from npu_bridge.estimator.npu.npu_optimizer import NPUDistributedOptimizer +#from tensorflow.contrib.offline_train.python import npu_ops +from npu_bridge.estimator import npu_ops +_NUM_EXAMPLES_NAME="num_examples" + + +class Model(object): + def __init__(self, config, data, hyper_param, layers, optimizer, loss, logger): + self.config = config + self.data = data + self.hyper_param = hyper_param + self.layers = layers + self.optimizer = optimizer + self.loss = loss + self.logger = logger + + def get_estimator_model_func(self, features, labels, mode, params=None): + labels = tf.reshape(labels, (-1,)) # Squash unnecessary unary dim #----------------not use when use onehot label + + model_func = self.get_model_func() + inputs = features # TODO: Should be using feature columns? + is_training = (mode == tf.estimator.ModeKeys.TRAIN) + + with tf.device('/gpu:0'): + if self.config['accelerator'] == 'gpu': + inputs = tf.cast(inputs, self.config['dtype']) + + inputs = tf.cast(inputs, self.config['dtype']) + with res50_helper.custom_getter_with_fp16_and_weight_decay(dtype=self.config['dtype'], weight_decay=self.config['weight_decay']): # no BN decay + + top_layer = model_func( + inputs, data_format=self.config['data_format'], training=is_training, + conv_initializer=self.config['conv_init'], + bn_init_mode=self.config['bn_init_mode'], bn_gamma_initial_value=self.config['bn_gamma_initial_value']) + + + logits = top_layer + predicted_classes = tf.argmax(logits, axis=1, output_type=tf.int32) + logits = tf.cast(logits, tf.float32) + + #loss = self.loss.get_loss(logits, labels) + #loss = tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels) + + labels_one_hot = tf.one_hot(labels, depth=1001) + loss = tf.losses.softmax_cross_entropy( + logits=logits, onehot_labels=labels_one_hot, label_smoothing=self.config['label_smoothing']) + + + base_loss = tf.identity(loss, name='loss') # For access by logger (TODO: Better way to access it?) + # base_loss = tf.add_n([loss]) + + def exclude_batch_norm(name): + #return 'batch_normalization' not in name + return 'BatchNorm' not in name + loss_filter_fn = exclude_batch_norm + + # Add weight decay to the loss. + l2_loss = self.config['weight_decay'] * tf.add_n( + # loss is computed using fp32 for numerical stability. + [tf.nn.l2_loss(tf.cast(v, tf.float32)) for v in tf.trainable_variables() + if loss_filter_fn(v.name)]) + #tf.summary.scalar('l2_loss', l2_loss) + # total_loss = base_loss + l2_loss + if self.config['use_lars']: + total_loss = base_loss + else: + total_loss = base_loss + l2_loss + + total_loss = tf.identity(total_loss, name = 'total_loss') + + + if mode == tf.estimator.ModeKeys.EVAL: + with tf.device(None): + metrics = self.layers.get_accuracy( labels, predicted_classes, logits, self.config) + + return tf.estimator.EstimatorSpec( + mode, loss=loss, eval_metric_ops=metrics) + + assert (mode == tf.estimator.ModeKeys.TRAIN) + + #reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) + #total_loss = tf.add_n([tf.saturate_cast(loss, self.config['dtype']) ] + reg_losses, name='total_loss') + #total_loss = tf.add_n([loss], name='total_loss') + + batch_size = tf.shape(inputs)[0] + + global_step = tf.train.get_global_step() + with tf.device('/cpu:0'): + learning_rate = self.hyper_param.get_learning_rate() + + #-----------------------batchsize scaling---------------------------------- + momentum = self.config['momentum'][0] + #------------------------------end------------------------------------------ + + opt = tf.train.MomentumOptimizer( + learning_rate, momentum, use_nesterov=self.config['use_nesterov']) + opt=NPUDistributedOptimizer(opt) + if self.config['accelerator'] == 'gpu': + opt = self.optimizer.get_lbs_optimizer(opt) + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) or [] + with tf.control_dependencies(update_ops): + if self.config['accelerator'] == 'gpu': + gate_gradients = (tf.train.Optimizer.GATE_NONE) + grads_and_vars = opt.compute_gradients(total_loss, gate_gradients=gate_gradients) + train_op = opt.apply_gradients( grads_and_vars,global_step = global_step) + else: + with tf.name_scope('loss_scale'): + loss_scale = float( self.config['loss_scale'] ) + scaled_grads_and_vars = opt.compute_gradients( total_loss * loss_scale ) + unscaled_grads_and_vars = [ (g/loss_scale, v) for g,v in scaled_grads_and_vars ] + + + #-----------------------------------------Lars------------------------------------------ + with tf.name_scope('LARS'): + fp32_grads_and_vars = [ (tf.cast(g, tf.float32), v) for g,v in unscaled_grads_and_vars ] + grad_var_list = [] + + if self.config['use_lars']: + if self.config['accelerator'] == 'gpu': + for g, var in fp32_grads_and_vars: + + if 'BatchNorm' not in var.name and 'bias' not in var.name: + grad_norm = tf.norm(g,ord='euclidean') + weight_norm = tf.norm(var,ord='euclidean') + grad_norm_wd = tf.add( grad_norm, tf.multiply( self.config['weight_decay'] , weight_norm ) ) + rescale_factor = tf.div( tf.multiply(0.001, weight_norm), tf.add(grad_norm_wd, tf.constant(1e-5, tf.float32)) ) + decayed_g = tf.add( g, tf.multiply(self.config['weight_decay'], var ) ) + + with tf.name_scope('lars_grad'): + g = tf.multiply(rescale_factor, decayed_g) + + g_and_v = ( g, var ) + grad_var_list.append( g_and_v ) + + elif self.config['accelerator'] == '1980': + print('lars9999999999999999999999') + g_list_bn_bias = [] + var_list_bn_bias = [] + g_list_else = [] + var_list_else = [] + for g, var in fp32_grads_and_vars: + if 'BatchNorm' not in var.name and 'bias' not in var.name: + g_list_else.append(g) + var_list_else.append(var) + else: + g_list_bn_bias.append(g) + var_list_bn_bias.append(var) + + + g_list_else_lars = npu_ops.LARS(inputs_w=var_list_else, + inputs_g=g_list_else, + weight_decay=self.config['weight_decay'], + hyperpara=0.001, + epsilon=1e-5) + + g_list_lars = g_list_bn_bias + g_list_else_lars + var_list = var_list_bn_bias + var_list_else + + for (g, var) in zip(g_list_lars,var_list): + g_and_v = ( g, var ) + grad_var_list.append( g_and_v ) + + + else: + print('do not use lars111111111111111111') + for g, var in fp32_grads_and_vars: + #if 'BatchNorm' not in var.name and 'bias' not in var.name: + # decayed_g = tf.add( g, tf.multiply( self.config['weight_decay'], var ) ) + # g = decayed_g + g_and_v = ( g, var ) + grad_var_list.append( g_and_v ) + #-----------------------------------------end Lars------------------------------------------ + + + + + train_op = opt.apply_gradients( grad_var_list, global_step = global_step ) + + train_op = tf.group(train_op) + + #with tf.device('/cpu:0'): + #tf.summary.scalar('total_loss', total_loss) + #tf.summary.scalar('base_loss', base_loss) + #tf.summary.scalar('learning_rate', learning_rate) + #tf.contrib.summary.flush() +# if self.config['do_checkpoint']: +# summary_hook = tf.train.SummarySaverHook( save_steps=20, +# output_dir=self.config['log_dir']+'/train_summary', +# summary_op = tf.summary.merge_all() ) + + #return tf.estimator.EstimatorSpec(mode, loss=total_loss, train_op=train_op, training_hooks=[summary_hook] )\ + # if self.config['do_checkpoint'] else tf.estimator.EstimatorSpec(mode, loss=total_loss, train_op=train_op ) + return tf.estimator.EstimatorSpec(mode, loss=total_loss, train_op=train_op ) + + # return tf.estimator.EstimatorSpec(mode, loss=total_loss, train_op=train_op) + + + + def get_model_func(self): + model_name = self.config['model_name'] + if model_name.startswith('resnet'): + nlayer = int(model_name[len('resnet'):]) + return lambda images, *args, **kwargs: \ + resnet.inference_resnet_v1(self.config,images, nlayer, *args, **kwargs) + else: + raise ValueError("Invalid model type: %s" % model_name) + + + + + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/models/resnet50/resnet.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/models/resnet50/resnet.py new file mode 100644 index 0000000..71a10e2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/models/resnet50/resnet.py @@ -0,0 +1,545 @@ +import tensorflow as tf + +_BATCH_NORM_EPSILON = 1e-4 +_BATCH_NORM_DECAY = 0.9 + +_Cardi = 32 + +class LayerBuilder(object): + def __init__(self, activation=None, data_format='channels_last', + training=False, use_batch_norm=False, batch_norm_config=None, + conv_initializer=None, bn_init_mode='adv_bn_init', bn_gamma_initial_value=1.0 ): + self.activation = activation + self.data_format = data_format + self.training = training + self.use_batch_norm = use_batch_norm + self.batch_norm_config = batch_norm_config + self.conv_initializer = conv_initializer + self.bn_init_mode = bn_init_mode + self.bn_gamma_initial_value = bn_gamma_initial_value + if self.batch_norm_config is None: + self.batch_norm_config = { + 'decay': _BATCH_NORM_DECAY, + 'epsilon': _BATCH_NORM_EPSILON, + 'scale': True, + 'zero_debias_moving_mean': False, + } + + def _conv2d(self, inputs, activation, *args, **kwargs): + x = tf.layers.conv2d( + inputs, data_format=self.data_format, + # use_bias=not self.use_batch_norm, + use_bias=False, + kernel_initializer=self.conv_initializer, + activation=None if self.use_batch_norm else activation, + *args, **kwargs) + if self.use_batch_norm: + param_initializers = { + 'moving_mean': tf.zeros_initializer(), + 'moving_variance': tf.ones_initializer(), + 'beta': tf.zeros_initializer(), + } + if self.bn_init_mode == 'adv_bn_init': + param_initializers['gamma'] = tf.ones_initializer() + elif self.bn_init_mode == 'conv_bn_init': + param_initializers['gamma'] = tf.constant_initializer(self.bn_gamma_initial_value) + else: + raise ValueError("--bn_init_mode must be 'conv_bn_init' or 'adv_bn_init' ") + + x = self.batch_norm(x) + x = activation(x) if activation is not None else x + return x + + def conv2d_linear_last_bn(self, inputs, *args, **kwargs): + x = tf.layers.conv2d( + inputs, data_format=self.data_format, + use_bias=False, + kernel_initializer=self.conv_initializer, + activation=None, *args, **kwargs) + param_initializers = { + 'moving_mean': tf.zeros_initializer(), + 'moving_variance': tf.ones_initializer(), + 'beta': tf.zeros_initializer(), + } + if self.bn_init_mode == 'adv_bn_init': + param_initializers['gamma'] = tf.zeros_initializer() + elif self.bn_init_mode == 'conv_bn_init': + param_initializers['gamma'] = tf.constant_initializer(self.bn_gamma_initial_value) + else: + raise ValueError("--bn_init_mode must be 'conv_bn_init' or 'adv_bn_init' ") + + x = self.batch_norm(x, param_initializers=param_initializers) + return x + + def conv2d_no_act_no_bn(self, inputs, *args, **kwargs): + x = tf.layers.conv2d( + inputs, data_format=self.data_format, + use_bias=False, + kernel_initializer=self.conv_initializer, + activation=None, *args, **kwargs) + return x + + def conv2d_linear(self, inputs, *args, **kwargs): + return self._conv2d(inputs, None, *args, **kwargs) + + def conv2d(self, inputs, *args, **kwargs): + return self._conv2d(inputs, self.activation, *args, **kwargs) + + def pad2d(self, inputs, begin, end=None): + if end is None: + end = begin + try: + _ = begin[1] + except TypeError: + begin = [begin, begin] + try: + _ = end[1] + except TypeError: + end = [end, end] + if self.data_format == 'channels_last': + padding = [[0, 0], [begin[0], end[0]], [begin[1], end[1]], [0, 0]] + else: + padding = [[0, 0], [0, 0], [begin[0], end[0]], [begin[1], end[1]]] + return tf.pad(inputs, padding) + + def max_pooling2d(self, inputs, *args, **kwargs): + return tf.layers.max_pooling2d( + inputs, data_format=self.data_format, *args, **kwargs) + + def average_pooling2d_stride_1(self, inputs, *args, **kwargs): + # inputs = tf.nn.avg_pool(inputs, ksize=[1,1,1,1],strides=[1,1,1,1], padding="VALID", data_format="NHWC" ) + return inputs + + def average_pooling2d(self, inputs, *args, **kwargs): + inputs = tf.nn.avg_pool(inputs, ksize=[1,2,2,1],strides=[1,2,2,1], padding="VALID", data_format="NHWC" ) + return inputs + + # return tf.layers.average_pooling2d( + # inputs, data_format=self.data_format, *args, **kwargs) + + def dense_linear(self, inputs, units, **kwargs): + return tf.layers.dense(inputs, units, activation=None) + + def dense(self, inputs, units, **kwargs): + return tf.layers.dense(inputs, units, activation=self.activation) + + def activate(self, inputs, activation=None): + activation = activation or self.activation + return activation(inputs) if activation is not None else inputs + + def batch_norm(self, inputs, **kwargs): + all_kwargs = dict(self.batch_norm_config) + all_kwargs.update(kwargs) + data_format = 'NHWC' if self.data_format == 'channels_last' else 'NCHW' + bn_inputs = inputs + outputs = tf.contrib.layers.batch_norm( + inputs, is_training=self.training, data_format=data_format, + fused=True, **all_kwargs) + + return outputs + + def spatial_average2d(self, inputs): + shape = inputs.get_shape().as_list() + if self.data_format == 'channels_last': + n, h, w, c = shape + else: + n, c, h, w = shape + n = -1 if n is None else n + x = tf.layers.average_pooling2d(inputs, (h, w), (1, 1), + data_format=self.data_format) + return tf.reshape(x, [n, c]) + + def flatten2d(self, inputs): + x = inputs + if self.data_format != 'channel_last': + # Note: This ensures the output order matches that of NHWC networks + x = tf.transpose(x, [0, 2, 3, 1]) + input_shape = x.get_shape().as_list() + num_inputs = 1 + for dim in input_shape[1:]: + num_inputs *= dim + return tf.reshape(x, [-1, num_inputs], name='flatten') + + def residual2d(self, inputs, network, units=None, scale=1.0, activate=False): + outputs = network(inputs) + c_axis = -1 if self.data_format == 'channels_last' else 1 + h_axis = 1 if self.data_format == 'channels_last' else 2 + w_axis = h_axis + 1 + ishape, oshape = [y.get_shape().as_list() for y in [inputs, outputs]] + ichans, ochans = ishape[c_axis], oshape[c_axis] + strides = ((ishape[h_axis] - 1) // oshape[h_axis] + 1, + (ishape[w_axis] - 1) // oshape[w_axis] + 1) + with tf.name_scope('residual'): + if (ochans != ichans or strides[0] != 1 or strides[1] != 1): + inputs = self.conv2d_linear(inputs, units, 1, strides, 'SAME') + x = inputs + scale * outputs + if activate: + x = self.activate(x) + return x + + +def resnet_bottleneck_v1(builder, inputs, depth, depth_bottleneck, stride, filters, arch_type, + basic=False): + num_inputs = inputs.get_shape().as_list()[3] + x = inputs + #with tf.name_scope('resnet_model'): + if depth == num_inputs: + if stride == 1:#v1.5 + shortcut = x + else:#v1 + shortcut = builder.max_pooling2d(x, 1, stride) + else: # the downsample(first) block in each layer + if 'D1' in arch_type: + if stride == 1: + shortcut = builder.average_pooling2d_stride_1(x, stride, stride) #--------------------Resnet-D------------ + else: + shortcut = builder.average_pooling2d(x, stride, stride) #--------------------Resnet-D------------ + shortcut = builder.conv2d_linear(shortcut, depth, 1, 1, 'SAME') + elif 'D2' in arch_type: + shortcut = builder.conv2d_linear(x, depth, 3, stride, 'SAME') + elif 'D3' in arch_type: + shortcut = builder.conv2d_linear(x, depth, 1, 1, 'SAME') + shortcut = builder.average_pooling2d(shortcut, stride, stride) #--------------------Resnet-D------------ + else: + shortcut = builder.conv2d_linear(x, depth, 1, stride, 'SAME') + conv_input = x + + if basic: + x = builder.pad2d(x, 1) + x = builder.conv2d(x, depth_bottleneck, 3, stride, 'VALID') + x = builder.conv2d_linear(x, depth, 3, 1, 'SAME') + else: + conv_input = x + x = builder.conv2d(x, depth_bottleneck, 1, 1, 'SAME') + conv_input = x + if stride == 1: + x = builder.conv2d(x, depth_bottleneck, 3, stride, 'SAME') + else: + if 'E1' in arch_type: + x = builder.average_pooling2d( x, stride, stride ) + x = builder.conv2d(x, depth_bottleneck, 3, 1, 'SAME') + elif 'E2' in arch_type: + x = builder.conv2d(x, depth_bottleneck, 3, 1, 'SAME') + if stride == 1: + x = builder.average_pooling2d_stride_1( x, stride, stride ) + else: + x = builder.average_pooling2d( x, stride, stride ) + else: # E0 + x = builder.conv2d(x, depth_bottleneck, 3, stride, 'SAME') + + # x = builder.conv2d_linear(x, depth, 1, 1, 'SAME') + conv_input = x + x = builder.conv2d_linear_last_bn(x, depth, 1, 1, 'SAME') + + x = tf.nn.relu(x + shortcut) + return x + + + + +def resnext_bottleneck(builder, inputs, depth, depth_bottleneck, stride, filters, arch_type, + basic=False): + num_inputs = inputs.get_shape().as_list()[3] + x = inputs + with tf.name_scope('resnet_v1'): + if depth == num_inputs: + if stride == 1:#v1.5 + shortcut = x + else:#v1 + shortcut = builder.max_pooling2d(x, 1, stride) + else: # the downsample(first) block in each layer + shortcut = builder.conv2d_linear(x, depth, 1, stride, 'SAME') + if basic: + x = builder.pad2d(x, 1) + x = builder.conv2d(x, depth_bottleneck, 3, stride, 'VALID') + x = builder.conv2d_linear(x, depth, 3, 1, 'SAME') + else: + + #----- split layer ------ + x = builder.conv2d( x, depth_bottleneck, 1, 1, 'SAME' ) + + group_inputs = tf.split( x, _Cardi, axis=3 ) + + layers_split=[] + tmp = x + for i in range(_Cardi): + with tf.name_scope('cardi_'+str(i)): + split = builder.conv2d_no_act_no_bn( group_inputs[i], depth_bottleneck/_Cardi, 3, stride, 'SAME' ) + layers_split.append(split) + + x = tf.concat(layers_split, axis=3) + x = builder.batch_norm(x) + x = tf.nn.relu(x) + + x = builder.conv2d_linear_last_bn(x, depth, 1, 1, 'SAME') + x = tf.nn.relu(x + shortcut) + return x + + + + + + +def resnet_bottleneck_v2(builder, inputs, depth, depth_bottleneck, stride, filters, arch_type, + basic=False): + num_inputs = inputs.get_shape().as_list()[1] + x = inputs + with tf.name_scope('resnet_v1'): + # ------- shortcut --------------- + if depth == num_inputs: + if stride == 1:#v1.5 + shortcut = x + x = builder.batch_norm(x) + x = tf.nn.relu(x) + else:#v1 + shortcut = builder.max_pooling2d(x, 1, stride) + else: # the downsample(first) block in each layer + x = builder.batch_norm(x) + x = tf.nn.relu(x) + + if 'D1' in arch_type: + shortcut = builder.average_pooling2d(x, stride, stride) #--------------------Resnet-D------------ + shortcut = builder.conv2d_linear(shortcut, depth, 1, 1, 'SAME') + elif 'D2' in arch_type: + shortcut = builder.conv2d_linear(x, depth, 3, stride, 'SAME') + elif 'D3' in arch_type: + shortcut = builder.conv2d_linear(x, depth, 1, 1, 'SAME') + shortcut = builder.average_pooling2d(shortcut, stride, stride) #--------------------Resnet-D------------ + else: + shortcut = builder.conv2d_linear(x, depth, 1, stride, 'SAME') + + # -------- mainstream ---------------- + if basic: + x = builder.pad2d(x, 1) + x = builder.conv2d(x, depth_bottleneck, 3, stride, 'VALID') + x = builder.conv2d_linear(x, depth, 3, 1, 'SAME') + else: + x = builder.conv2d(x, depth_bottleneck, 1, 1, 'SAME') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + + if stride == 1: + x = builder.conv2d(x, depth_bottleneck, 3, stride, 'SAME') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + else: + if 'E1' in arch_type: + x = builder.average_pooling2d( x, stride, stride ) + x = builder.conv2d(x, depth_bottleneck, 3, 1, 'SAME') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + elif 'E2' in arch_type: + x = builder.conv2d(x, depth_bottleneck, 3, 1, 'SAME') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + x = builder.average_pooling2d( x, stride, stride ) + else: # E0 + x = builder.conv2d(x, depth_bottleneck, 3, stride, 'SAME') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + + x = builder.conv2d_linear(x, depth, 1, 1, 'SAME') + + + x = x + shortcut + return x + +def inference_resnext_impl(builder, inputs, layer_counts, arch_type='C1+D', resnet_version='v1.5', basic=False): + x = inputs + #x = builder.batch_norm(x) + x = builder.pad2d(x, 3) + x = builder.conv2d(x, 64, 7, 2, 'VALID') + #x = builder.conv2d(x, 64, 7, 2, 'SAME') + + + num_filters=64 + x = builder.max_pooling2d(x, 3, 2, 'SAME') + #x, argmax = tf.nn.max_pool_with_argmax(input=x, ksize=(1,3,3,1), strides=(1,2,2,1), padding='SAME') + + for i in range(layer_counts[0]): + x = resnext_bottleneck(builder, x, 256, 128, 1, num_filters, arch_type, basic) + for i in range(layer_counts[1]): + num_filters=num_filters*2 + x = resnext_bottleneck(builder, x, 512, 256, 2 if i == 0 else 1, num_filters, arch_type, basic) + for i in range(layer_counts[2]): + num_filters=num_filters*2 + x = resnext_bottleneck(builder, x, 1024, 512, 2 if i == 0 else 1, num_filters, arch_type, basic) + for i in range(layer_counts[3]): + num_filters=num_filters*2 + x = resnext_bottleneck(builder, x, 2048, 1024, 2 if i == 0 else 1, num_filters, arch_type, basic) + print ('====================Final x:', x) + + + axes = [1,2] + x = tf.reduce_mean( x, axes, keepdims=True ) + x = tf.identity(x, 'final_reduce_mean') + x = tf.reshape( x, [-1, 2048] ) + x = tf.layers.dense(inputs=x, units=1001,kernel_initializer= tf.variance_scaling_initializer() ) + x = tf.identity( x, 'final_dense' ) + return x + + +def inference_resnet_v1_impl(builder, inputs, layer_counts, arch_type='C1+D', resnet_version='v1.5', basic=False): + x = inputs + #x = builder.pad2d(x, 1) + + if 'C1' in arch_type: # --- Resnet C ----- + x = builder.conv2d(x, 32, 3, 2, 'SAME') + x = builder.conv2d(x, 32, 3, 1, 'SAME') + x = builder.conv2d(x, 64, 3, 1, 'SAME') + elif 'C2' in arch_type: + x = builder.conv2d(x, 32, 3, 1, 'SAME') + x = builder.conv2d(x, 32, 3, 2, 'VALID') + x = builder.conv2d(x, 64, 3, 1, 'VALID') + elif 'C3' in arch_type: + x = builder.conv2d(x, 32, 3, 1, 'VALID') + x = builder.conv2d(x, 32, 3, 1, 'VALID') + x = builder.conv2d(x, 64, 3, 2, 'VALID') + else: + x = builder.conv2d(x, 64, 7, 2, 'SAME') + + num_filters=64 + + pooled_inputs = x + #x = builder.max_pooling2d(x, 3, 2, 'SAME') + x, argmax = tf.nn.max_pool_with_argmax(input=x, ksize=(1,3,3,1), strides=(1,2,2,1), padding='SAME') + + for i in range(layer_counts[0]): + x = resnet_bottleneck_v1(builder, x, 256, 64, 1, num_filters, arch_type, basic) + for i in range(layer_counts[1]): + num_filters=num_filters*2 + x = resnet_bottleneck_v1(builder, x, 512, 128, 2 if i == 0 else 1, num_filters, arch_type, basic) + for i in range(layer_counts[2]): + num_filters=num_filters*2 + x = resnet_bottleneck_v1(builder, x, 1024, 256, 2 if i == 0 else 1, num_filters, arch_type, basic) + for i in range(layer_counts[3]): + num_filters=num_filters*2 + x = resnet_bottleneck_v1(builder, x, 2048, 512, 2 if i == 0 else 1, num_filters, arch_type, basic) + + axes = [1,2] + x = tf.reduce_mean( x, axes, keepdims=True ) + x = tf.identity(x, 'final_reduce_mean') + x = tf.reshape( x, [-1, 2048] ) + x = tf.layers.dense(inputs=x, units=1001,kernel_initializer=tf.random_normal_initializer(stddev=0.01)) + x = tf.identity( x, 'final_dense' ) + return x + +def inference_resnet_v2_impl(builder, inputs, layer_counts, arch_type='C1+D', basic=False): + x = inputs + x = builder.pad2d(x, 3) + + if 'C1' in arch_type: # --- Resnet C ----- + x = builder.conv2d(x, 32, 3, 2, 'VALID') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + x = builder.conv2d(x, 32, 3, 1, 'VALID') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + x = builder.conv2d(x, 64, 3, 1, 'SAME') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + elif 'C2' in arch_type: + x = builder.conv2d(x, 32, 3, 1, 'SAME') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + x = builder.conv2d(x, 32, 3, 2, 'VALID') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + x = builder.conv2d(x, 64, 3, 1, 'VALID') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + elif 'C3' in arch_type: + x = builder.conv2d(x, 32, 3, 1, 'VALID') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + x = builder.conv2d(x, 32, 3, 1, 'VALID') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + x = builder.conv2d(x, 64, 3, 2, 'VALID') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + else: + x = builder.conv2d(x, 64, 7, 2, 'VALID') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + + num_filters=64 + + pooled_inputs = x + x = builder.max_pooling2d(x, 3, 2, 'SAME') + + for i in range(layer_counts[0]): + x = resnet_bottleneck_v2(builder, x, 256, 64, 1, num_filters, arch_type, basic) + for i in range(layer_counts[1]): + num_filters=num_filters*2 + x = resnet_bottleneck_v2(builder, x, 512, 128, 2 if i == 0 else 1, num_filters, arch_type, basic) + for i in range(layer_counts[2]): + num_filters=num_filters*2 + x = resnet_bottleneck_v2(builder, x, 1024, 256, 2 if i == 0 else 1, num_filters, arch_type, basic) + for i in range(layer_counts[3]): + num_filters=num_filters*2 + x = resnet_bottleneck_v2(builder, x, 2048, 512, 2 if i == 0 else 1, num_filters, arch_type, basic) + return builder.spatial_average2d(x) + +def inference_resnet_v1(config, inputs, nlayer, data_format='channels_last', + training=False, conv_initializer=None, bn_init_mode='adv_bn_init', bn_gamma_initial_value=1.0 ): + """Deep Residual Networks family of models + https://arxiv.org/abs/1512.03385 + """ + if config['resnet_version'] == 'v1.5': + builder = LayerBuilder(tf.nn.relu, data_format, training, use_batch_norm=True, + conv_initializer=conv_initializer, bn_init_mode=bn_init_mode, bn_gamma_initial_value=bn_gamma_initial_value) + if nlayer == 18: + return inference_resnet_v1_impl(builder, inputs, [2, 2, 2, 2], config['arch_type'], config['resnet_version'], basic=True) + elif nlayer == 34: + return inference_resnet_v1_impl(builder, inputs, [3, 4, 6, 3], config['arch_type'], config['resnet_version'], basic=True) + elif nlayer == 50: + return inference_resnet_v1_impl(builder, inputs, [3, 4, 6, 3], config['arch_type'], config['resnet_version']) + elif nlayer == 101: + return inference_resnet_v1_impl(builder, inputs, [3, 4, 23, 3], config['arch_type'], config['resnet_version']) + elif nlayer == 152: + return inference_resnet_v1_impl(builder, inputs, [3, 8, 36, 3], config['arch_type'], config['resnet_version']) + else: + raise ValueError("Invalid nlayer (%i); must be one of: 18,34,50,101,152" % + nlayer) + + elif config['resnet_version'] == 'v2': + builder = LayerBuilder( None, data_format, training, use_batch_norm=False, + conv_initializer=conv_initializer, bn_init_mode=bn_init_mode, bn_gamma_initial_value=bn_gamma_initial_value) + if nlayer == 18: + return inference_resnet_v2_impl(builder, inputs, [2, 2, 2, 2], config['arch_type'], basic=True) + elif nlayer == 34: + return inference_resnet_v2_impl(builder, inputs, [3, 4, 6, 3], config['arch_type'], basic=True) + elif nlayer == 50: + return inference_resnet_v2_impl(builder, inputs, [3, 4, 6, 3], config['arch_type']) + elif nlayer == 101: + return inference_resnet_v2_impl(builder, inputs, [3, 4, 23, 3], config['arch_type']) + elif nlayer == 152: + return inference_resnet_v2_impl(builder, inputs, [3, 8, 36, 3], config['arch_type']) + else: + raise ValueError("Invalid nlayer (%i); must be one of: 18,34,50,101,152" % + nlayer) + + elif config['resnet_version'] == 'resnext': + builder = LayerBuilder( tf.nn.relu, data_format, training, use_batch_norm=True, + conv_initializer=conv_initializer, bn_init_mode=bn_init_mode, bn_gamma_initial_value=bn_gamma_initial_value) + if nlayer == 18: + return inference_resnext_impl(builder, inputs, [2, 2, 2, 2], config['arch_type'], basic=True) + elif nlayer == 34: + return inference_resnext_impl(builder, inputs, [3, 4, 6, 3], config['arch_type'], basic=True) + elif nlayer == 50: + return inference_resnext_impl(builder, inputs, [3, 4, 6, 3], config['arch_type']) + elif nlayer == 101: + return inference_resnext_impl(builder, inputs, [3, 4, 23, 3], config['arch_type']) + elif nlayer == 152: + return inference_resnext_impl(builder, inputs, [3, 8, 36, 3], config['arch_type']) + else: + raise ValueError("Invalid nlayer (%i); must be one of: 18,34,50,101,152" % + nlayer) + + + else: + raise ValueError("Invalid resnet version") + + + diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/optimizers/__init__.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/optimizers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/optimizers/optimizer.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/optimizers/optimizer.py new file mode 100644 index 0000000..ce42d6a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/optimizers/optimizer.py @@ -0,0 +1,228 @@ +import six +import tensorflow as tf + +class Optimizer: + def __init__(self, config): + self.config = config + + def get_lbs_optimizer(self, opt): #TODO input is ( self, hyper_param ) + + # opt = LargeBatchSizeOptimizer(opt, weight_decay=self.config['weight_decay'], + # accum_dtype = self.config['dtype'], + # use_lars = self.config['use_lars'], + # bn_lr_scale = self.config.get('bn_lr_scale', 1.0) + # ) + opt = MixedPrecisionOptimizer(opt, self.config) + + return opt + +class MixedPrecisionOptimizer(tf.train.Optimizer): + """An optimizer that updates trainable variables in fp32.""" + + def __init__(self, optimizer, config): + super(MixedPrecisionOptimizer, self).__init__( + optimizer._use_locking, + optimizer._name + '-MP', + ) + self._optimizer = optimizer + self._config = config + loss_scale=self._config['loss_scale'] + self._loss_scale = float(loss_scale) + self._fp32_to_fp16 = {} + + var_list = ( + tf.trainable_variables() + + tf.get_collection(tf.GraphKeys.TRAINABLE_RESOURCE_VARIABLES)) + with tf.device('/gpu:0'): + self.var_fp32_copy = [ tf.Variable( tf.cast(v.initialized_value(), tf.float32), + dtype=tf.float32, trainable=False, + collections=[tf.GraphKeys.GLOBAL_VARIABLES, "FP32_MASTER_COPIES"] ) for v in var_list ] + + def compute_gradients(self, loss, var_list=None, + gate_gradients=tf.train.Optimizer.GATE_OP, + aggregation_method=None, + colocate_gradients_with_ops=False, + grad_loss=None): + if var_list is None: + var_list = ( + tf.trainable_variables() + + tf.get_collection(tf.GraphKeys.TRAINABLE_RESOURCE_VARIABLES)) + + if self._loss_scale != 1.0: + loss = tf.scalar_mul(self._loss_scale, loss) + + grads_and_vars_fp16 = self._optimizer.compute_gradients( + loss, var_list=var_list, + gate_gradients=gate_gradients, + aggregation_method=aggregation_method, + colocate_gradients_with_ops=colocate_gradients_with_ops, + grad_loss=grad_loss, + ) + # creating FP-32 variables and filling the fp32 dict + grads_and_vars_fp32 = [] + + with tf.variable_scope('FP32-master-copy'): + for i, (grad, var) in enumerate(grads_and_vars_fp16): + if grad is not None: + if var.dtype.base_dtype == tf.float16: + fp32_var = self.var_fp32_copy[i] + self._fp32_to_fp16[fp32_var.name] = var + fp32_grad = tf.cast(grad, tf.float32) + grads_and_vars_fp32.append((fp32_grad, fp32_var)) + else: + grads_and_vars_fp32.append((grad, var)) + else: + grads_and_vars_fp32.append((None, var)) + + grads_and_vars_fp32_rescaled = [ (g/self._loss_scale, v) for g,v in grads_and_vars_fp32 ] + + + return grads_and_vars_fp32_rescaled + + def apply_gradients(self, grads_and_vars, *args, **kwargs): + update_op = self._optimizer.apply_gradients(grads_and_vars, *args, **kwargs) + apply_ops = [] + with tf.control_dependencies([update_op]): + for grad, var in grads_and_vars: + if var.name in self._fp32_to_fp16: + dst_var = self._fp32_to_fp16[var.name] + apply_ops.append( + tf.assign(dst_var, tf.saturate_cast(var, tf.float16))) + if apply_ops: + return tf.group(apply_ops) + return update_op + + +class LargeBatchSizeOptimizer(tf.train.Optimizer): + """ LARC implementation + ------------------- + Parameters: + - optimizer: initial optimizer that you wanna apply + example: tf.train.MomentumOptimizer + - learning_rate: initial learning_rate from initial optimizer + - clip: if True apply LARC otherwise LARS + - epsilon: default value is weights or grads are 0. + - name + - use_locking + """ + + def __init__(self, optimizer, weight_decay, clip=True, epsilon=1., accum_dtype=tf.float16, use_lars=True, bn_lr_scale=1.0, + name="LarcOptimizer", use_locking=False): + super(LargeBatchSizeOptimizer, self).__init__( + name=name, use_locking=use_locking) + self._optimizer = optimizer + # self._learning_rate = learning_rate + self._weight_decay = weight_decay + self._clip = clip + self._epsilon = float(epsilon) + self._accum_dtype=accum_dtype + self._use_lars=use_lars + self._bn_lr_scale=bn_lr_scale + + var_list = ( + tf.trainable_variables() + + tf.get_collection(tf.GraphKeys.TRAINABLE_RESOURCE_VARIABLES)) + with tf.device('/gpu:0'): + self._grads_accum = [ tf.Variable( tf.cast(tf.zeros_like(v.initialized_value()), self._accum_dtype), dtype=self._accum_dtype, trainable=False, collections=[tf.GraphKeys.LOCAL_VARIABLES] ) for v in var_list ] + + + def compute_gradients(self, *args, **kwargs): + return self._optimizer.compute_gradients(*args, **kwargs) + + + def apply_gradients(self, gradvars, loss_scale, *args, **kwargs): + + global_step = tf.train.get_global_step() + + grads_and_vars_clean = [] + for grad, var in gradvars: + if grad is not None: + grads_and_vars_clean.append( (grad, var) ) + + processed_grads_and_vars = self.post_process_grads(grads_and_vars_clean, loss_scale) # post_process_grads includes Lars + + def apply(): + red_grad_updates = self._optimizer.apply_gradients( processed_grads_and_vars, global_step=tf.train.get_global_step() ) + return tf.group(red_grad_updates) + + update_weight_op_1 = apply() + return update_weight_op_1 + + apply_gradients_op = update_weight_op_1 + + with tf.device('/cpu:0'): + #tf.summary.scalar('loss_scale', loss_scale) + for grad, var in gradvars: + g = grad / loss_scale + v_norm_2 = tf.norm(var, ord='euclidean') + g_norm_2 = tf.norm(g, ord='euclidean') + v_g_norm2_ratio = v_norm_2 / ( + g_norm_2 + self._weight_decay * v_norm_2) + if grad is not None: + if 'BatchNorm' in var.name: + with tf.name_scope('bn_norm2/'): + tf.summary.scalar(var.name + '/norm2', + v_norm_2) + with tf.name_scope('grad_bn_norm2/'): + tf.summary.scalar(var.name + '/grad_norm2', + g_norm_2) + with tf.name_scope('bn_ratio_var_grad/'): + tf.summary.scalar(var.name + '/ratio_var_grad', + v_g_norm2_ratio) + else: + with tf.name_scope('conv_norm2/'): + tf.summary.scalar(var.name + '/norm2', + v_norm_2) + with tf.name_scope('grad_conv_norm2/'): + tf.summary.scalar(var.name + '/grad_norm2', + g_norm_2) + with tf.name_scope('conv_ratio_var_grad/'): + tf.summary.scalar(var.name + '/ratio_var_grad', + v_g_norm2_ratio) + + return apply_gradients_op + + def post_process_grads(self, grads_and_vars, loss_scale): + + g_and_v_scaled = [] + for g, v in grads_and_vars: + g = g / loss_scale + g_and_v_scaled.append((g,v)) + + # Lars + if self._use_lars: + grad_var_list = [] + #-----------------------------------------------LARS and weight decay----------------------------------- + for g, var in g_and_v_scaled: + if 'BatchNorm' not in var.name and 'bias' not in var.name: + grad_norm = tf.norm(g,ord='euclidean') + weight_norm = tf.norm(var,ord='euclidean') + + grad_norm_wd = tf.add( grad_norm, tf.multiply( self._weight_decay, weight_norm ) ) + rescale_factor = tf.div( tf.multiply(0.001, weight_norm), tf.add(grad_norm_wd, tf.constant(1e-5, tf.float32)) ) + + coeffi = tf.clip_by_value( rescale_factor, 0.001, 50.0 ) + decayed_g = tf.add( g, tf.multiply( self._weight_decay, var ) ) + + g = tf.multiply(coeffi, decayed_g) + else: + g = self._bn_lr_scale * g + + g_and_v = ( g, var ) + grad_var_list.append( g_and_v ) + #-------------------------------------------LARS end--------------------------------- + return grad_var_list + else: + grad_var_list_without_lars = [] + #----------------------------------------weight decay----------------------------------- + for g, var in g_and_v_scaled: + if 'BatchNorm' not in var.name and 'bias' not in var.name: + decayed_g = tf.add( g, tf.multiply( self._weight_decay, var ) ) + g = decayed_g + else: + g = self._bn_lr_scale * g + + g_and_v = ( g, var ) + grad_var_list_without_lars.append( g_and_v ) + + return grad_var_list_without_lars diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/trainers/2q b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/trainers/2q new file mode 100644 index 0000000..ef8c703 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/trainers/2q @@ -0,0 +1,172 @@ +import tensorflow as tf +import math +import time +from . import train_helper +from .train_helper import stage +from utils.logger import rank0log + +from tensorflow.contrib.offline_train.python.npu.npu_config import NPURunConfig +from tensorflow.contrib.offline_train.python.npu.npu_estimator import NPUEstimator +from tensorflow.contrib.offline_train.python.npu.npu_optimizer import NPUDistributedOptimizer + +class GPUBaseTrain(object): + def __init__(self, session, config, data, model, logger): + self.sess = session + self.config = config + self.data = data + self.model = model + self.logger = logger + self.print_logger = self.logger.logger + self.all_preds = [] + self.all_targets = [] + if self.config['accelerator'] == 'gpu': + self.classifier, self.training_hook = self.get_classifier() + else: + from tensorflow.contrib.offline_train.python.npu.npu_config import NPURunConfig + from tensorflow.contrib.offline_train.python.npu.npu_estimator import NPUEstimator + from tensorflow.contrib.offline_train.python.npu.npu_optimizer import NPUDistributedOptimizer + self.classifier, self.training_hook = self.get_npu_classifier() + + + + def get_classifier(self): + classifier = tf.estimator.Estimator( + model_fn=self.model.get_estimator_model_func, + model_dir=self.config['log_dir'], + config = tf.estimator.RunConfig( + session_config=self.sess.get_config(), + save_summary_steps=self.config['save_summary_steps'] if self.config['do_checkpoint'] else None, + save_checkpoints_steps=self.config['save_checkpoints_steps'] if self.config['do_checkpoint'] else None, + keep_checkpoint_max=None + ) + ) + + training_hooks = [train_helper.PrefillStagingAreasHook()] + training_hooks.append(self.logger) + + return classifier, training_hooks + + def get_npu_classifier(self): + session_config = tf.ConfigProto( + inter_op_parallelism_threads=10, + intra_op_parallelism_threads=10, + allow_soft_placement=True) + + if self.config['debug'] : + run_config = NPURunConfig(enable_auto_mix_precision=True, enable_data_pre_proc=True, save_checkpoints_steps=112590, session_config=session_config, model_dir = self.config['model_dir'], iterations_per_loop=self.config['iterations_per_loop'], keep_checkpoint_max=5) + else : + run_config = NPURunConfig(enable_auto_mix_precision=True, save_summary_steps=0, log_step_count_steps=None, enable_data_pre_proc=True,save_checkpoints_secs=1e9, session_config=session_config, model_dir = self.config['model_dir'], iterations_per_loop=self.config['iterations_per_loop']) +# run_config = NPURunConfig(enable_data_pre_proc=True,save_checkpoints_secs=1e9, session_config=session_config, model_dir = self.config['model_dir']) + + # classifier = tf.estimator.Estimator( + # model_fn=self.model.get_estimator_model_func, + # model_dir=self.config['log_dir'], + # config = tf.estimator.RunConfig( + # session_config=self.sess.get_config(), + # save_summary_steps=self.config['save_summary_steps'] if self.config['do_checkpoint'] else None, + # save_checkpoints_steps=self.config['save_checkpoints_steps'] if self.config['do_checkpoint'] else None, + # keep_checkpoint_max=None + # ) + # ) + + classifier =NPUEstimator( + model_fn= self.model.get_estimator_model_func, + config= run_config +# job_start_file='/tmp/config/deviceid_devindex_jobstart' + ) + + training_hooks = [] + if self.config['debug']: + training_hooks = [train_helper.PrefillStagingAreasHook()] + training_hooks.append(self.logger) + + return classifier, training_hooks + + def train(self): + print ('training steps: %d' % self.config['nstep']) + self.classifier.train( input_fn=lambda:self.data.get_train_input_fn(), + # max_steps = self.config['max_train_steps'], + max_steps = self.config['nstep'], + #steps = 100, + hooks = self.training_hook + ) + + + def evaluate(self): + rank0log(self.print_logger, "Evaluating") + rank0log(self.print_logger, "Validation dataset size: {}".format(self.config['num_evaluating_samples'] )) + time.sleep(5) # a little extra margin... + try: + ckpts = train_helper.sort_and_load_ckpts(self.config['log_dir']) + for i, c in enumerate(ckpts): + if i < len(ckpts) - 1: + if i % self.config['eval_interval'] != 0: + continue + eval_result = self.classifier.evaluate( + input_fn=lambda: self.data.get_eval_input_fn(), + checkpoint_path=c['path']) + c['epoch'] = math.ceil(c['step'] / (self.config['num_training_samples']/ (self.config['batch_size']))) + c['top1'] = eval_result['val-top1acc'] + c['top5'] = eval_result['val-top5acc'] + c['loss'] = eval_result['loss'] + + rank0log(self.print_logger, ' step epoch top1 top5 loss checkpoint_time(UTC)') + for i, c in enumerate(ckpts): + if 'top1' not in c: + continue + rank0log(self.print_logger,'{:5d} {:5.1f} {:5.3f} {:6.2f} {:6.2f} {time}' + .format(c['step'], + c['epoch'], + c['top1'] * 100, + c['top5'] * 100, + c['loss'], + time=time.strftime('%Y-%m-%d %H:%M:%S', + time.localtime(c['mtime'])))) + rank0log(self.print_logger, "Finished evaluation") + except KeyboardInterrupt: + self.print_logger.error("Keyboard interrupt") + + def train_and_evaluate(self): + success = False + epochs_between_evals = self.config.get('epochs_between_evals', 4) + + + for i in range(self.config['num_epochs'] // epochs_between_evals): + + rank0log(self.print_logger, "Starting a training cycle") + + self.classifier.train(input_fn=lambda:self.data.get_train_input_fn(), + steps = self.config['nsteps_per_epoch']*epochs_between_evals, + hooks = self.training_hook ) + + rank0log(self.print_logger, "Starting to evaluate") + rank0log(self.print_logger, "Validation dataset size: {}".format(self.config['num_evaluating_samples'] )) + time.sleep(5) # a little extra margin... + + ckpts = train_helper.sort_and_load_ckpts(self.config['log_dir']) + c = ckpts[-1] + eval_result = self.classifier.evaluate( + input_fn=lambda: self.data.get_eval_input_fn(), + checkpoint_path=c['path']) + + c['epoch'] = math.ceil(c['step'] / (self.config['num_training_samples']/ (self.config['batch_size'] * hvd.size()))) + c['top1'] = eval_result['val-top1acc'] + c['top5'] = eval_result['val-top5acc'] + c['loss'] = eval_result['loss'] + + rank0log(self.print_logger, ' step epoch top1 top5 loss checkpoint_time(UTC)') + + rank0log(self.print_logger,'{:5d} {:5.1f} {:5.3f} {:6.2f} {:6.2f} {time}' + .format(c['step'], + c['epoch'], + c['top1'] * 100, + c['top5'] * 100, + c['loss'], + time=time.strftime('%Y-%m-%d %H:%M:%S', + time.localtime(c['mtime'])))) + if eval_result['val-top1acc']*100 > self.config.get('stop_threshold', 74.9): + success = True + break + + + diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/trainers/__init__.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/trainers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/trainers/gpu_base_trainer.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/trainers/gpu_base_trainer.py new file mode 100644 index 0000000..02b3e15 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/trainers/gpu_base_trainer.py @@ -0,0 +1,253 @@ +import os + +import sys +import tensorflow as tf +import math +import time +from . import train_helper +from .train_helper import stage +from utils.logger import rank0log +# add by zwx5326390 +from datetime import datetime +# import hwlog +from benchmark_log import hwlog + +#from tensorflow.contrib.offline_train.python.npu.npu_config import NPURunConfig +#from tensorflow.contrib.offline_train.python.npu.npu_estimator import NPUEstimator +#from tensorflow.contrib.offline_train.python.npu.npu_optimizer import NPUDistributedOptimizer +from npu_bridge.estimator.npu.npu_config import NPURunConfig +from npu_bridge.estimator.npu.npu_estimator import NPUEstimator +from npu_bridge.estimator.npu.npu_optimizer import NPUDistributedOptimizer + +# remark_logger = hwlog.get_logger(__file__, "hw_Resnext50.log") +# file_name = hwlog.get_file_name(__file__) + +class GPUBaseTrain(object): + def __init__(self, session, config, data, model, logger): + self.sess = session + self.config = config + self.data = data + self.model = model + self.logger = logger + self.print_logger = self.logger.logger + self.all_preds = [] + self.all_targets = [] + + # add by zwx5326390 + # work_num = "device " + str(os.environ.get("DEVICE_INDEX")) + # date_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') + # hwlog.vlogger.info("namespace:%s,time_ts:%s,event_type:benchmark_start" % (work_num, date_time)) + + if self.config['accelerator'] == 'gpu': + self.classifier, self.training_hook = self.get_classifier() + else: + #from tensorflow.contrib.offline_train.python.npu.npu_config import NPURunConfig + #from tensorflow.contrib.offline_train.python.npu.npu_estimator import NPUEstimator + #from tensorflow.contrib.offline_train.python.npu.npu_optimizer import NPUDistributedOptimizer + from npu_bridge.estimator.npu.npu_config import NPURunConfig + from npu_bridge.estimator.npu.npu_estimator import NPUEstimator + from npu_bridge.estimator.npu.npu_optimizer import NPUDistributedOptimizer + self.classifier, self.training_hook = self.get_npu_classifier() + + + + def get_classifier(self): + classifier = tf.estimator.Estimator( + model_fn=self.model.get_estimator_model_func, + model_dir=self.config['log_dir'], + config = tf.estimator.RunConfig( + session_config=self.sess.get_config(), + save_summary_steps=self.config['save_summary_steps'] if self.config['do_checkpoint'] else None, + save_checkpoints_steps=self.config['save_checkpoints_steps'] if self.config['do_checkpoint'] else None, + keep_checkpoint_max=None + ) + ) + + training_hooks = [train_helper.PrefillStagingAreasHook()] + training_hooks.append(self.logger) + + return classifier, training_hooks + + def get_npu_classifier(self): + session_config = tf.ConfigProto( + inter_op_parallelism_threads=10, + intra_op_parallelism_threads=10, + allow_soft_placement=True) + print (" config.debug:") + print ( self.config['debug']) + print (self.config['log_dir']) + if self.config['debug'] : + run_config = NPURunConfig(hcom_parallel=True, precision_mode='allow_mix_precision', enable_data_pre_proc=True, save_checkpoints_steps=112590, session_config=session_config, model_dir = self.config['model_dir'], iterations_per_loop=self.config['iterations_per_loop'], keep_checkpoint_max=5) + else : + run_config = NPURunConfig(hcom_parallel=True, precision_mode='allow_mix_precision', save_summary_steps=0, log_step_count_steps=None, enable_data_pre_proc=True,save_checkpoints_secs=1e9, session_config=session_config, model_dir = self.config['model_dir'], iterations_per_loop=self.config['iterations_per_loop']) +# run_config = NPURunConfig(enable_data_pre_proc=True,save_checkpoints_secs=1e9, session_config=session_config, model_dir = self.config['model_dir']) + + # classifier = tf.estimator.Estimator( + # model_fn=self.model.get_estimator_model_func, + # model_dir=self.config['log_dir'], + # config = tf.estimator.RunConfig( + # session_config=self.sess.get_config(), + # save_summary_steps=self.config['save_summary_steps'] if self.config['do_checkpoint'] else None, + # save_checkpoints_steps=self.config['save_checkpoints_steps'] if self.config['do_checkpoint'] else None, + # keep_checkpoint_max=None + # ) + # ) + + classifier =NPUEstimator( + model_fn= self.model.get_estimator_model_func, + config= run_config +# job_start_file='/tmp/config/deviceid_devindex_jobstart' + ) + + training_hooks = [] + if self.config['debug']: + training_hooks = [train_helper.PrefillStagingAreasHook()] + training_hooks.append(self.logger) + + return classifier, training_hooks + + def train(self): + # add by zwx5326390 + # work_num = "device " + str(os.environ.get("DEVICE_INDEX")) + # date_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') + # hwlog.vlogger.info("nemespace:%s,time_ts:%s,event_type:epoch_start, num_train_epochs: %d" % ( \ + # work_num, date_time, self.config['num_epochs'])) + # date_time = hwlog.get_time() + # remark_logger.info("ABK time_ts: %s, current_epoch: %d, batch_size: %d, file: %s, lineno: %s" % \ + # (date_time, self.config['num_epochs'], self.config['batch_size'], file_name, + # sys._getframe().f_lineno)) + hwlog.remark_print(key=hwlog.CURRENT_EPOCH, value=self.config['num_epochs']) + + print ('training steps: %d' % self.config['nstep']) + self.classifier.train( input_fn=lambda:self.data.get_train_input_fn(), + # max_steps = self.config['max_train_steps'], + max_steps = self.config['nstep'], + #steps = 100, + hooks = self.training_hook + ) + # hwlog.vlogger.info("namespace:%s,time_ts:%s,event_type:epoch_stop, num_train_epochs: %d" % ( \ + # work_num, date_time, self.config['num_epochs'])) + + def evaluate(self): + rank0log(self.print_logger, "Evaluating") + rank0log(self.print_logger, "Validation dataset size: {}".format(self.config['num_evaluating_samples'] )) + time.sleep(5) # a little extra margin... + try: + ckpts = train_helper.sort_and_load_ckpts(self.config['ckpt_dir']) + for i, c in enumerate(ckpts): + if i < len(ckpts) - 1: + if i % self.config['eval_interval'] != 0: + continue + eval_result = self.classifier.evaluate( + input_fn=lambda: self.data.get_eval_input_fn(), + checkpoint_path=c['path']) + + # add by zwx5326390 + # work_num = "device " + str(os.environ.get("DEVICE_INDEX")) + # date_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') + # hwlog.vlogger.info("namespace:%s, time_ts:%s, val-top1acc:%d, val-top5acc: %d" % ( + # work_num, date_time, eval_result.get("val-top1acc"), eval_result.get("val-top5acc") + # )) + # date_time = hwlog.get_time() + # remark_logger.info("ABK time_ts: %s, accuracy: %f, accuracy_top_5: %f, file: %s, lineno: %s" % \ + # (date_time, float(eval_result.get("val-top1acc")), + # float(eval_result.get("val-top5acc")), \ + # file_name, sys._getframe().f_lineno)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value=float(eval_result.get("val-top1acc"))) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value=float(eval_result.get("val-top5acc"))) + + #c['epoch'] = math.ceil(c['step'] / (self.config['num_training_samples']/ (self.config['batch_size']))) + c['epoch'] = math.ceil(c['step'] / (self.config['num_training_samples']/ (self.config['batch_size'] * self.config['rank_size']))) + c['top1'] = eval_result['val-top1acc'] + c['top5'] = eval_result['val-top5acc'] + c['loss'] = eval_result['loss'] + + rank0log(self.print_logger, ' step epoch top1 top5 loss checkpoint_time(UTC)') + for i, c in enumerate(ckpts): + if 'top1' not in c: + continue + rank0log(self.print_logger,'{:5d} {:5.1f} {:5.3f} {:6.2f} {:6.2f} {time}' + .format(c['step'], + c['epoch'], + c['top1'] * 100, + c['top5'] * 100, + c['loss'], + time=time.strftime('%Y-%m-%d %H:%M:%S', + time.localtime(c['mtime'])))) + rank0log(self.print_logger, "Finished evaluation") + except KeyboardInterrupt: + self.print_logger.error("Keyboard interrupt") + + def train_and_evaluate(self): + success = False + epochs_between_evals = self.config.get('epochs_between_evals', 4) + + + for i in range(self.config['num_epochs'] // epochs_between_evals): + + rank0log(self.print_logger, "Starting a training cycle") + + # add by zwx5326390 + # work_num = "device " + str(os.environ.get("DEVICE_INDEX")) + # date_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') + # hwlog.vlogger.info("nemespace:%s,time_ts:%s,event_type:epoch_start, num_train_epochs: %d" % (\ + # work_num, date_time, self.config['num_epochs'])) + # date_time = hwlog.get_time() + # remark_logger.info("ABK time_ts: %s, current_epoch: %d, batch_size: %d, file: %s, lineno: %s" % \ + # (date_time, self.config['num_epochs'], self.config['batch_size'], file_name, + # sys._getframe().f_lineno)) + hwlog.remark_print(key=hwlog.CURRENT_EPOCH, value=self.config['num_epochs']) + + self.classifier.train(input_fn=lambda:self.data.get_train_input_fn(), + steps = self.config['nsteps_per_epoch']*epochs_between_evals, + hooks = self.training_hook ) + + # hwlog.vlogger.info("namespace:%s,time_ts:%s,event_type:epoch_stop, num_train_epochs: %d" % ( \ + # work_num, date_time, self.config['num_epochs'])) + + rank0log(self.print_logger, "Starting to evaluate") + rank0log(self.print_logger, "Validation dataset size: {}".format(self.config['num_evaluating_samples'] )) + time.sleep(5) # a little extra margin... + + ckpts = train_helper.sort_and_load_ckpts(self.config['log_dir']) + c = ckpts[-1] + eval_result = self.classifier.evaluate( + input_fn=lambda: self.data.get_eval_input_fn(), + checkpoint_path=c['path']) + + # add by zwx5326390 + # work_num = "device " + str(os.environ.get("DEVICE_INDEX")) + # date_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') + # hwlog.vlogger.info("namespace:%s, time_ts:%s, val-top1acc:%d, val-top5acc: %d" % ( + # work_num, date_time, eval_result.get("val-top1acc"), eval_result.get("val-top5acc") + # )) + # date_time = hwlog.get_time() + # remark_logger.info("ABK time_ts: %s, accuracy: %f, accuracy_top_5: %f, file: %s, lineno: %s" % \ + # (date_time, float(eval_result.get("val-top1acc")), float(eval_result.get("val-top5acc")), \ + # file_name, sys._getframe().f_lineno)) + + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value=float(eval_result.get("val-top1acc"))) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value=float(eval_result.get("val-top5acc"))) + + + c['epoch'] = math.ceil(c['step'] / (self.config['num_training_samples']/ (self.config['batch_size'] * self.config['rank_size']))) + c['top1'] = eval_result['val-top1acc'] + c['top5'] = eval_result['val-top5acc'] + c['loss'] = eval_result['loss'] + + rank0log(self.print_logger, ' step epoch top1 top5 loss checkpoint_time(UTC)') + + rank0log(self.print_logger,'{:5d} {:5.1f} {:5.3f} {:6.2f} {:6.2f} {time}' + .format(c['step'], + c['epoch'], + c['top1'] * 100, + c['top5'] * 100, + c['loss'], + time=time.strftime('%Y-%m-%d %H:%M:%S', + time.localtime(c['mtime'])))) + if eval_result['val-top1acc']*100 > self.config.get('stop_threshold', 74.9): + success = True + break + + + diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/trainers/train_helper.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/trainers/train_helper.py new file mode 100644 index 0000000..8d1fa8f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/trainers/train_helper.py @@ -0,0 +1,39 @@ +import tensorflow as tf +from tensorflow.python.ops import data_flow_ops +import re +import os +from operator import itemgetter + +class PrefillStagingAreasHook(tf.train.SessionRunHook): + def after_create_session(self, session, coord): + enqueue_ops = tf.get_collection('STAGING_AREA_PUTS') + for i in range(len(enqueue_ops)): + session.run(enqueue_ops[:i + 1]) + +def stage(tensors): + """Stages the given tensors in a StagingArea for asynchronous put/get. + """ + stage_area = data_flow_ops.StagingArea( + dtypes=[tensor.dtype for tensor in tensors], + shapes=[tensor.get_shape() for tensor in tensors]) + put_op = stage_area.put(tensors) + get_tensors = stage_area.get() + tf.add_to_collection('STAGING_AREA_PUTS', put_op) + return put_op, get_tensors + + +def sort_and_load_ckpts(log_dir): + ckpts = [] + for f in os.listdir(log_dir): + m = re.match(r'model.ckpt-([0-9]+).index', f) + if m is None: + continue + fullpath = os.path.join(log_dir, f) + ckpts.append({'step': int(m.group(1)), + 'path': os.path.splitext(fullpath)[0], + 'mtime': os.stat(fullpath).st_mtime, + }) + ckpts.sort(key=itemgetter('step')) + return ckpts + + diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/utils/__init__.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/utils/create_session.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/utils/create_session.py new file mode 100644 index 0000000..86853d1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/utils/create_session.py @@ -0,0 +1,48 @@ +import tensorflow as tf +import os,sys + +class CreateSession(): + def __init__(self, config): + self.config = config + + if self.config['accelerator'] == '1980': + from tensorflow.python.client import device_lib + #from tensorflow.contrib.offline_train.python import npu_ops + from npu_bridge.estimator import npu_ops + #self.estimator_config = tf.ConfigProto(allow_soft_placement=True, min_group_size=20, use_off_line=True) + self.estimator_config = tf.ConfigProto(allow_soft_placement=True) + custom_op = self.estimator_config.graph_options.rewrite_options.custom_optimizers.add() + custom_op.name = "NpuOptimizer" + custom_op.parameter_map["use_off_line"].b = True + custom_op.parameter_map["min_group_size"].b = 20 + else: + self.estimator_config = tf.ConfigProto(allow_soft_placement=False) + + self.estimator_config.gpu_options.allow_growth = True + + if self.config['accelerator'] == '1980': + local_device_protos = device_lib.list_local_devices(self.estimator_config) + + self.set_env() + + + def set_env(self): + # TODO, get env from config file + gpu_thread_count = 2 + os.environ['TF_GPU_THREAD_MODE'] = 'gpu_private' + os.environ['TF_GPU_THREAD_COUNT'] = str(gpu_thread_count) + os.environ['TF_USE_CUDNN_BATCHNORM_SPATIAL_PERSISTENT'] = '1' + os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = '1' + + # barrier = self.hvd.allreduce(tf.constant(0, dtype=tf.float32)) + # tf.Session(config=self.estimator_config).run(barrier) + + + def get_config(self): + self.estimator_config.gpu_options.visible_device_list = str(0) +# self.estimator_config.gpu_options.force_gpu_compatible = True # Force pinned memory + self.estimator_config.intra_op_parallelism_threads = 1 # Avoid pool of Eigen threads + self.estimator_config.inter_op_parallelism_threads = 5 + return self.estimator_config + + diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/utils/logger.py b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/utils/logger.py new file mode 100644 index 0000000..07ede21 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/code/resnext50_train/utils/logger.py @@ -0,0 +1,103 @@ +from __future__ import print_function +import tensorflow as tf +import logging +import numpy as np +import time +import sys,os +from datetime import datetime +# import hwlog +# remark_logger = hwlog.get_logger(__file__, "hw_Resnext50.log") +# file_name = hwlog.get_file_name(__file__) +from benchmark_log import hwlog + +class LogSessionRunHook(tf.train.SessionRunHook): + def __init__(self, config, warmup_steps=5): + # def __init__(self, global_batch_size, num_records, display_every=10, logger=None): + self.global_batch_size = config['global_batch_size'] + self.iterations_per_loop = config['iterations_per_loop'] + self.warmup_steps = warmup_steps + self.iter_times = [] + self.num_records = config['num_training_samples'] + self.display_every = config['display_every'] + self.logger = get_logger(config['log_name'], config['log_dir']) + rank0log(self.logger, 'PY' + str(sys.version) + 'TF' + str(tf.__version__)) + + + + def after_create_session(self, session, coord): + rank0log(self.logger, 'Step Epoch Speed Loss FinLoss LR') + self.elapsed_secs = 0. + self.count = 0 + + def before_run(self, run_context): + self.t0 = time.time() + return tf.train.SessionRunArgs( + fetches=[tf.train.get_global_step(), 'loss:0', 'total_loss:0', 'learning_rate:0']) +# 'loss:0', 'loss:0', 'learning_rate:0']) + + def after_run(self, run_context, run_values): + batch_time = time.time() - self.t0 + self.iter_times.append(batch_time) + self.elapsed_secs += batch_time + self.count += 1 + global_step, loss, total_loss, lr = run_values.results + if global_step == 1 or global_step % self.display_every == 0: + dt = self.elapsed_secs / self.count + img_per_sec = self.global_batch_size * self.iterations_per_loop / dt + epoch = global_step * self.global_batch_size / self.num_records + self.logger.info('step:%6i epoch:%5.1f FPS:%7.1f loss:%6.3f total_loss:%6.3f lr:%7.7f' % + (global_step, epoch, img_per_sec, loss, total_loss, lr)) + self.elapsed_secs = 0. + self.count = 0 + # add by zwx5326390 + # work_num = "device " + str(os.environ.get("DEVICE_INDEX")) + # date_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') + # hwlog.vlogger.info("namespace:%s, time_ts:%s, FPS:%f, steps: %s" % (work_num, date_time, + # img_per_sec, + # global_step)) + # date_time = hwlog.get_time() + # remark_logger.info("ABK time_ts: %s, fps: %f, steps: %s, file: %s, lineno: %s" % \ + # (date_time, img_per_sec, global_step, file_name, \ + # sys._getframe().f_lineno)) + hwlog.remark_print(key=hwlog.FPS, value='%7.1f'%img_per_sec) + + + def get_average_speed(self): + avg_time = np.mean(self.iter_times[self.warmup_steps:]) + speed = self.global_batch_size / avg_time + return speed + + + +def rank0log(logger, *args, **kwargs): + if logger: + logger.info(''.join([str(x) for x in list(args)])) + else: + print(*args, **kwargs) + + +def get_logger(log_name, log_dir): + logger = logging.getLogger(log_name) + logger.setLevel(logging.INFO) # INFO, ERROR + # file handler which logs debug messages + if not os.path.isdir(log_dir): + try: + os.makedirs(log_dir) + except FileExistsError: + # if log_dir is common for multiple ranks like on nfs + pass + # console handler + ch = logging.StreamHandler() + ch.setLevel(logging.INFO) + # add formatter to the handlers + # formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + formatter = logging.Formatter('%(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) + fh = logging.FileHandler(os.path.join(log_dir, log_name)) + fh.setLevel(logging.DEBUG) + fh.setFormatter(formatter) + # add handlers to logger + logger.addHandler(fh) + return logger + diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/config/hccl_sample.json b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/config/hccl_sample.json new file mode 100644 index 0000000..cec6b14 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/config/hccl_sample.json @@ -0,0 +1,6 @@ +{ + "server_count": "1", + "server_list": [{"device":[{devices}],"server_id":"127.0.0.1"}], + "status": "completed", + "version": "1.0" +} diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/config/main_sample.sh b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/config/main_sample.sh new file mode 100644 index 0000000..a43856f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/config/main_sample.sh @@ -0,0 +1,18 @@ +#!/bin/sh +currentDir=$(cd "$(dirname "$0")"; pwd) +cd ${currentDir} + +device_group=$@ +device_num=$# + +touch ${currentDir}/main.log + +for device_phy_id in ${device_group} +do + echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] start: train.sh ${device_phy_id} & " >> ${currentDir}/main.log + ${currentDir}/train.sh ${device_phy_id} & +done + +wait + +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] all train.sh exit " >> ${currentDir}/main.log diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/config/npu_set_env.sh b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/config/npu_set_env.sh new file mode 100644 index 0000000..65ad3e2 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/config/npu_set_env.sh @@ -0,0 +1,41 @@ +# main env +if [ -d /usr/local/Ascend/nnae/latest ];then + + export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/Ascend/driver/tools/hccn_tool/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages + export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp +else + export LD_LIBRARY_PATH=/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/ascend-toolkit/latest//fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/tfplugin/python/site-packages:$projectDir + export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + +fi +export SOC_VERSION=Ascend910 +export HCCL_CONNECT_TIMEOUT=600 + +# user env +export JOB_ID={JOB_ID} +export RANK_TABLE_FILE={RANK_TABLE_FILE} +#export RANK_SIZE={RANK_SIZE} +#export RANK_INDEX={RANK_INDEX} +#export RANK_ID={RANK_ID} + +# profiling env +export PROFILING_MODE={PROFILING_MODE} +export AICPU_PROFILING_MODE={AICPU_PROFILING_MODE} +export PROFILING_OPTIONS={PROFILING_OPTIONS} +export FP_POINT={FP_POINT} +export BP_POINT={BP_POINT} + + +# debug env +#export DUMP_GE_GRAPH=2 +#export DUMP_OP=1 +#export DUMP_OP_LESS=1 +#export PRINT_MODEL=1 +#export TE_PARALLEL_COMPILER=0 + +# system env +ulimit -c unlimited diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/config/train_sample.sh b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/config/train_sample.sh new file mode 100644 index 0000000..b4ee768 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/config/train_sample.sh @@ -0,0 +1,33 @@ +#!/bin/sh +currentDir=$(cd "$(dirname "$0")"; pwd) +cd ${currentDir} + +PWD=${currentDir} + +device_id=$1 +if [ x"${device_id}" = x ] ; +then + echo "turing train fail" >> ${currentDir}/train_${device_id}.log + exit +else + export DEVICE_ID=${device_id} +fi + +DEVICE_INDEX=$(( DEVICE_ID + RANK_INDEX * 8 )) +export DEVICE_INDEX=${DEVICE_INDEX} + +env > ${currentDir}/env_${device_id}.log + +#mkdir exec path +mkdir -p ${currentDir}/${device_id} +rm -rf ${currentDir}/${device_id}/* +cd ${currentDir}/${device_id} + +#start exec +python3.7 {RUN_ALGORITHM_CMD} {CHECKPOINT_DIR} > ${currentDir}/train_${device_id}.log 2>&1 +if [ $? -eq 0 ] ; +then + echo "turing train success" >> ${currentDir}/train_${device_id}.log +else + echo "turing train fail" >> ${currentDir}/train_${device_id}.log +fi diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/scripts/run.sh b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/scripts/run.sh new file mode 100644 index 0000000..62a1b1f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/scripts/run.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +rank_size=$1 +yamlPath=$2 +toolsPath=$3 +currentDir=$(cd "$(dirname "$0")/.."; pwd) +if [ -f /.dockerenv ];then + CLUSTER=$4 + MPIRUN_ALL_IP="$5" + export CLUSTER=${CLUSTER} +fi +#export RANK_ID=npu${rank_size}p + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "tensorflow_config") + +data_url_new=`echo ${data_url//\//\\\\/}` +echo ${data_url} +echo ${max_steps} +echo ${epoches} +if [ x"${CLUSTER}" == x"True" ];then + jsonFilePath=${currentDir}/code/resnext50_train/configs/res50_32bs_8p.py +elif [ ${rank_size} -lt 8 ];then + jsonFilePath=${currentDir}/code/resnext50_train/configs/res50_32bs_1p.py + if [ ${rank_size} -eq 1 ];then + sed -i "0,/rank_size.*$/s//rank_size\': ${rank_size},/" ${jsonFilePath} + + elif [ ${rank_size} -eq 2 ];then + sed -i "0,/rank_size.*$/s//rank_size\': ${rank_size},/" ${jsonFilePath} + else + sed -i "0,/rank_size.*$/s//rank_size\': ${rank_size},/" ${jsonFilePath} + fi + +else + jsonFilePath=${currentDir}/code/resnext50_train/configs/res50_32bs_8p.py + if [ ${rank_size} -eq 8 ];then + sed -i "0,/rank_size.*$/s//rank_size\': ${rank_size},/" ${jsonFilePath} + else + rank_size=16 + sed -i "0,/rank_size.*$/s//rank_size\': ${rank_size},/" ${jsonFilePath} + fi +fi + +#echo "jsonfilepath is "${jsonFilePath} +sed -i "s/data_url.*$/data_url\': \'${data_url_new}\',/g" ${jsonFilePath} +#sed -i "s/max_train_steps.*$/max_train_steps\': ${max_steps},/g" ${jsonFilePath} +sed -i "s/num_epochs.*$/num_epochs\': ${epoches},/g" ${jsonFilePath} +sed -i "0,/batch_size.*$/s//batch_size\': ${batch_size},/" ${jsonFilePath} +sed -i "s/epochs_between_evals.*$/epochs_between_evals\': ${epochs_between_evals},/g" ${jsonFilePath} +sed -i 's/\r//g' ${jsonFilePath} + + +currtime=`date +%Y%m%d%H%M%S` +mkdir -p ${currentDir%train*}/train/result/tf_resnext50/training_job_${currtime}/ +train_job_dir=${currentDir%train*}/train/result/tf_resnext50/training_job_${currtime}/ +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir} &" +# device 列表, 若无指定 device 或大于等于 8p 时根据 rank_size 顺序选择 +eval device_group=\$device_group_${rank_size}p +if [ x"${device_group}" == x"" ] || [ ${rank_size} -ge 8 ];then + device_group="$(seq 0 "$(expr $rank_size - 1)")" +fi + +# get last device id in device_group, hw log in performance from the dir named last_device_id +device_group_str=`echo ${device_group} | sed 's/ //g'` +first_device_id=`echo ${device_group_str: 0:1}` +echo ${device_group_str} +echo ${first_device_id} + +rank_id=0 + +if [ x"${CLUSTER}" == x"True" ];then + # ln hw log + ln -snf ${currentDir%train*}/train/result/tf_resnext50/training_job_${currtime}/0/hw_resnext50.log ${currentDir%train*}/train/result/tf_resnext50/training_job_${currtime}/ + this_ip=$(hostname -I |awk '{print $1}') + for ip in $MPIRUN_ALL_IP;do + if [ x"$this_ip" != x"$ip" ];then + scp $yamlPath root@$ip:$yamlPath + scp $jsonFilePath root@$ip:$jsonFilePath + fi + done + export PATH=$PATH:/usr/local/mpirun4.0/bin + mpirun -H ${mpirun_ip} \ + --bind-to none -map-by slot\ + --allow-run-as-root \ + --mca btl_tcp_if_exclude lo,docker0,endvnic,virbr0,vethf40501b,docker_gwbridge,br-f42ac38052b4\ + --prefix /usr/local/mpirun4.0/ \ + ${currentDir}/scripts/train.sh 0 $rank_size $yamlPath $currtime ${toolsPath} ${CLUSTER} +else + # ln hw log + ln -snf ${currentDir%train*}/train/result/tf_resnext50/training_job_${currtime}/${first_device_id}/hw_resnext50.log ${currentDir%train*}/train/result/tf_resnext50/training_job_${currtime}/ + for device_id in $device_group;do + #echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] start: train ${device_id} & " >> ${currentDir}/result/main.log + ${currentDir}/scripts/train.sh $device_id $rank_size $yamlPath $currtime ${toolsPath} $rank_id& + let rank_id++ + done +fi +wait + + diff --git a/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/scripts/train.sh b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/scripts/train.sh new file mode 100644 index 0000000..b897c6f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ResNext50/tensorflow/scripts/train.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash + +device_id=$1 +rank_size=$2 +yamlPath=$3 +currtime=$4 +toolsPath=$5 +currentDir=$(cd "$(dirname "$0")/.."; pwd) +mkdir -p ${currentDir%train*}/train/result/tf_resnext50/training_job_${currtime}/ +export train_job_dir=${currentDir%train*}/train/result/tf_resnext50/training_job_${currtime}/ + +source ${currentDir}/config/npu_set_env.sh + +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "tensorflow_config") + +# 声明变量 +export REMARK_LOG_FILE=hw_resnext50.log # 打点日志文件名称， 必须hw_后跟模型名称小写 +# 添加日志打点模块路径 +benchmark_log_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils +export PYTHONPATH=$PYTHONPATH:${benchmark_log_path} + + + +# user env +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 +export RANK_TABLE_FILE=${currentDir}/config/${rank_size}p.json +export RANK_SIZE=${rank_size} +export RANK_INDEX=0 +export SLOG_PRINT_TO_STDOUT=0 +export DEVICE_ID=$1 +DEVICE_INDEX=$(( DEVICE_ID + RANK_INDEX * 8 )) +export DEVICE_INDEX=${DEVICE_INDEX} +export YAML_PATH=$3 +export MODEL_CKPT_PATH=${currentDir}/result/ckpt${device_id} + +if [ ${profiling_mode} == True ]; +then + export PROFILING_MODE=true +else + export PROFILING_MODE=false +fi + +if [ ${aicpu_profiling_mode} == True ]; +then + export AICPU_PROFILING_MODE=true +else + export AICPU_PROFILING_MODE=false +fi + +export PROFILING_OPTIONS=${profiling_options} +export FP_POINT=${fp_point} +export BP_POINT=${bp_point} + +cd ${train_job_dir} +curd_dir=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +export PYTHONPATH=$PYTHONPATH:${curd_dir} + +if [ x"$6" != x"True" ];then + rank_id=$6 + export RANK_ID=$6 +else + device_id_mo=$(python3.7 -c "import src.tensorflow.mpi_ops as atlasboost;atlasboost.init(); \ + device_id = atlasboost.local_rank();cluster_device_id = str(device_id); \ + atlasboost.set_device_id(device_id);print(atlasboost.rank())") + device_id_mo=`echo $device_id_mo` + rank_id=${device_id_mo##* } + export RANK_ID=${rank_id} + device=${device_id_mo##*deviceid = } + device_id=${device%% phyid=*} + export DEVICE_ID=${device_id} + hccljson=${train_job_dir}/*.json + cp ${hccljson} ${currentDir}/config/${rank_size}p.json +fi + +#mkdir exec path +mkdir -p ${train_job_dir}/${device_id} +cd ${train_job_dir}/${device_id} + +startTime=`date +%Y%m%d-%H:%M:%S` +startTime_s=`date +%s` +#cd ${currentDir}/code +# 根据单卡/多卡区分调用参数 +if [ x"$6" == x"True" ];then + export CLUSTER=True + # 多卡多机 + rm -rf ${currentDir}/result/*.log + rm -rf ${currentDir}/code/core.* + python3.7 ${currentDir}/code/resnext50_train/mains/res50.py --config_file=res50_32bs_8p --max_train_steps=${max_steps} --iterations_per_loop=1000 --debug=True --eval=True --model_dir=${currentDir}/result/ckpt${device_id} > ${train_job_dir}/train_${device_id}.log 2>&1 +elif [ ${rank_size} -le 4 ];then + # 单卡 + python3.7 ${currentDir}/code/resnext50_train/mains/res50.py --config_file=res50_32bs_1p --max_train_steps=${max_steps} --iterations_per_loop=1000 --debug=True --eval=False --model_dir=${currentDir}/result/ckpt${device_id} > ${train_job_dir}/train_${device_id}.log 2>&1 +elif [ ${rank_size} -le 8 ];then + # 多卡单机 + python3.7 ${currentDir}/code/resnext50_train/mains/res50.py --config_file=res50_32bs_8p --max_train_steps=${max_steps} --iterations_per_loop=1000 --debug=True --eval=True --model_dir=${currentDir}/result/ckpt${device_id} > ${train_job_dir}/train_${device_id}.log 2>&1 +fi + +if [ $? -eq 0 ] ;then + echo ":::ABK 1.0.0 resnext50 train success" + echo ":::ABK 1.0.0 resnext50 train success" >> ${train_job_dir}/train_${device_id}.log + echo ":::ABK 1.0.0 resnext50 train success" >> ${train_job_dir}/${device_id}/hw_resnext50.log +else + echo ":::ABK 1.0.0 resnext50 train failed" + echo ":::ABK 1.0.0 resnext50 train failed" >> ${train_job_dir}/train_${device_id}.log + echo ":::ABK 1.0.0 resnext50 train failed" >> ${train_job_dir}/${device_id}/hw_resnext50.log +fi + +endTime=`date +%Y%m%d-%H:%M:%S` +endTime_s=`date +%s` +sumTime=$[ $endTime_s - $startTime_s ] +hour=$(( $sumTime/3600 )) +min=$(( ($sumTime-${hour}*3600)/60 )) +sec=$(( $sumTime-${hour}*3600-${min}*60 )) +echo ${hour}:${min}:${sec} +echo ":::ABK 1.0.0 resnext50 train total time ${hour}:${min}:${sec}" >> ${train_job_dir}/${device_id}/hw_resnext50.log + + diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/__init__.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/README.md b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/README.md new file mode 100644 index 0000000..11778d1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/README.md @@ -0,0 +1,40 @@ +# Resnet50_HC_tensorflow训练说明 + +### 1. 模型训练参数配置 + +在train/yaml/Resnet50_HC.yaml中修改相应配置， 配置项含义: + +``` +tensorflow_config: + # 基本参数 + max_steps: 1000 + data_url: /home/imagenet_TF + epoches: 1 + epochs_between_evals: 1 + mode: train + # 仅多机执行需要配置: ip1:卡数量1,ip2:卡数量2 + mpirun_ip: 90.90.176.154:8,90.90.176.54:8 + + # docker 镜像名称:版本号 + docker_image: mpirun3:latest + + + # 1. 指定 device id, 多个 id 使用空格分隔, 数量需与 rank_size 相同 + # 2. 仅在小于 8p 时生效 + # 3. 若不使用该配置, 请使用在行首添加'#'注释的方法将其关闭 + # device_group: 0 1 2 3 + device_group_1p: 0 + device_group_2p: 0 1 + device_group_4p: 0 1 2 3 + +``` + +------ + + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/__init__.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/pid b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/pid new file mode 100644 index 0000000..0d49bb1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/pid @@ -0,0 +1 @@ +13650 \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_16p.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_16p.py new file mode 100644 index 0000000..9e3a47a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_16p.py @@ -0,0 +1,111 @@ +import tensorflow as tf + +import os +log_dir = 'ckpt/' + +#256 +config = { + # ============ for testing ===================== + 'accelerator': '1980', # 'gpu', '1980' + 'shuffle_enable': 'yes', + 'shuffle_buffer_size': 10000, + 'rank_size': 32, + 'shard': True, + + # ======= basic config ======= # + 'mode':'train', # "train","evaluate","train_and_evaluate" + 'epochs_between_evals': 4, #used if mode is "train_and_evaluate" + 'stop_threshold': 80.0, #used if mode is "train_and_evaluate" + #'data_dir':'/opt/npu/resnet_data_new', + 'data_url': '{DATA_URL}', + 'data_type': 'TFRECORD', + 'model_name': 'resnet50', + 'num_classes': 1001, + 'num_epochs': None, + 'height':224, + 'width':224, + 'dtype': tf.float32, + 'data_format': 'channels_last', + 'use_nesterov': True, + 'eval_interval': 1, + 'loss_scale': 1024, #could be float or string. If float, static loss scaling is applied. + #If string, the corresponding automatic loss scaling algorithm is used. + #Must be one of 'Backoff' of 'LogMax' (case insensitive). + 'use_lars': False, + 'label_smoothing':0.1, #If greater than 0 then smooth the labels. + 'weight_decay': 0.0001, + 'batch_size':256, #minibatch size per node, total batchsize = batch_size*hvd.size()*itersize + + 'momentum': [0.9], + + #======= data processing config ======= + 'min_object_covered': 0.1, #used for random crop + 'aspect_ratio_range':[3. / 4., 4. / 3.], + 'area_range':[0.16, 1.0], + 'max_attempts': 100, + + #======= data augment config ======= + 'increased_aug': False, + 'brightness':0.3, + 'saturation': 0.6, + 'contrast': 0.6, + 'hue': 0.13, + 'num_preproc_threads': 22, + + #======= initialization config ======= + 'conv_init': tf.variance_scaling_initializer(), + 'bn_init_mode': 'adv_bn_init', # "conv_bn_init" or "adv_bn_init",initializer the gamma in bn in different modes + # "adv_bn_init" means initialize gamma to 0 in each residual block's last bn, and initialize other gamma to 1 + # "conv_bn_init" means initialize all the gamma to a constant, defined by "bn_gamma_initial_value" + 'bn_gamma_initial_value': 1.0, + + #======== model architecture ========== + 'resnet_version': 'v1.5', + 'arch_type': 'original', # ------ input ------- + # C1,C2,C3: input block, stride in different layer + # ------ shortcut ------ + # D1: average_pooling + conv1*1 in shortcut in downsample block + # D2: conv3*3,stride=2 in shortcut in downsample block + # D3: conv1*1 +average_pooling in shortcut in downsample block + # ------ mainstream ---- + # E1: average_pooling + conv3*3 in mainstream in downsample block + # E2: conv3*3 + average_pooling in mainstream in downsample block + + #======= logger config ======= + 'display_every': 1, + 'log_name': 'resnet50.log', + 'log_dir': log_dir, + + #======= Learning Rate Config ======= + 'lr_warmup_mode': 'linear', # "linear" or "cosine" + 'warmup_lr': 0.0, + 'warmup_epochs': 10, + 'learning_rate_maximum': 3.2, + + 'lr_decay_mode': 'cosine', # "steps", "poly", "poly_cycle", "cosine", "linear_cosine", "linear_twice", "constant" for 1980 only + 'learning_rate_end': 0.00001, + + 'decay_steps': '10,20,30', #for "steps" + 'lr_decay_steps': '6.4,0.64,0.064', + + 'ploy_power': 2.0, #for "poly" and "poly_cycle" + + 'cdr_first_decay_ratio': 0.33, #for "cosine_decay_restarts" + 'cdr_t_mul':2.0, + 'cdr_m_mul':0.1, + + 'lc_periods':0.47, #for "linear_consine" + 'lc_beta':0.00001, + + 'lr_mid': 0.5, #for "linear_twice" + 'epoch_mid': 80, + + 'bn_lr_scale':1.0, + + } + +def res50_config(): + config['global_batch_size'] = config['batch_size'] * config['rank_size'] + config['do_checkpoint'] = True + + return config diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_1p.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_1p.py new file mode 100644 index 0000000..e2d07ef --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_1p.py @@ -0,0 +1,109 @@ +import tensorflow as tf + +import os +log_dir = 'ckpt/' + +#256 +config = { + # ============ for testing ===================== + 'accelerator': '1980', # 'gpu', '1980' + 'shuffle_enable': 'yes', + 'shuffle_buffer_size': 10000, + 'rank_size': 1, + 'shard': False, + + # ======= basic config ======= # + 'mode': 'train', + 'epochs_between_evals': 5, + 'stop_threshold': 80.0, #used if mode': 'train', + #'data_dir':'/opt/npu/resnet_data_new', + 'data_url': '/home/mencai/training_shop-master/02-e2e/e2e_function/e2e_func_node/data/resnet50/imagenet_TF', + 'data_type': 'TFRECORD', + 'model_name': 'resnet50', + 'num_classes': 1001, + 'num_epochs': 100, + 'height':224, + 'width':224, + 'dtype': tf.float32, + 'data_format': 'channels_last', + 'use_nesterov': True, + 'eval_interval': 1, + 'loss_scale': 1024, #could be float or string. If float, static loss scaling is applied. + #If string, the corresponding automatic loss scaling algorithm is used. + #Must be one of 'Backoff' of 'LogMax' (case insensitive). + 'use_lars': False, + 'label_smoothing':0.1, #If greater than 0 then smooth the labels. + 'weight_decay': 0.0001, + 'batch_size':256, #minibatch size per node, total batchsize = batch_size*hvd.size()*itersize + + 'momentum': [0.9], + + #======= data processing config ======= + 'min_object_covered': 0.1, #used for random crop + 'aspect_ratio_range':[3. / 4., 4. / 3.], + 'area_range':[0.16, 1.0], + 'max_attempts': 100, + + #======= data augment config ======= + 'increased_aug': False, + 'brightness':0.3, + 'saturation': 0.6, + 'contrast': 0.6, + 'hue': 0.13, + 'num_preproc_threads': 22, + + #======= initialization config ======= + 'conv_init': tf.variance_scaling_initializer(), + 'bn_init_mode': 'adv_bn_init', # "adv_bn_init" means initialize gamma to 0 in each residual block's last bn, and initialize other gamma to 1 + # "conv_bn_init" means initialize all the gamma to a constant, defined by "bn_gamma_initial_value" + 'bn_gamma_initial_value': 1.0, + + #======== mode': 'train', + 'resnet_version': 'v1.5', + 'arch_type': 'original', # ------ input ------- + # C1,C2,C3: input block, stride in different layer + # ------ shortcut ------ + # D1: average_pooling + conv1*1 in shortcut in downsample block + # D2: conv3*3,stride=2 in shortcut in downsample block + # D3: conv1*1 +average_pooling in shortcut in downsample block + # ------ mainstream ---- + # E1: average_pooling + conv3*3 in mainstream in downsample block + # E2: conv3*3 + average_pooling in mainstream in downsample block + + #======= logger config ======= + 'display_every': 1, + 'log_name': 'resnet50.log', + 'log_dir': log_dir, + #======= Learning Rate Config ======= + 'lr_warmup_mode': 'train', + 'warmup_lr': 0.0, + 'warmup_epochs': 10, + 'learning_rate_maximum': 0.1, + + 'lr_decay_mode': 'train', + 'learning_rate_end': 0.00001, + + 'decay_steps': '10,20,30', #for "steps" + 'lr_decay_steps': '6.4,0.64,0.064', + + 'ploy_power': 2.0, #for "poly" and "poly_cycle" + + 'cdr_first_decay_ratio': 0.33, #for "cosine_decay_restarts" + 'cdr_t_mul':2.0, + 'cdr_m_mul':0.1, + + 'lc_periods':0.47, #for "linear_consine" + 'lc_beta':0.00001, + + 'lr_mid': 0.5, #for "linear_twice" + 'epoch_mid': 80, + + 'bn_lr_scale':1.0, + + } + +def res50_config(): + config['global_batch_size'] = config['batch_size'] * config['rank_size'] + config['do_checkpoint'] = True + + return config diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_2p.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_2p.py new file mode 100644 index 0000000..7338e70 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_2p.py @@ -0,0 +1,109 @@ +import tensorflow as tf + +import os +log_dir = 'ckpt/' + +#256 +config = { + # ============ for testing ===================== + 'accelerator': '1980', # 'gpu', '1980' + 'shuffle_enable': 'yes', + 'shuffle_buffer_size': 10000, + 'rank_size': 2, + 'shard': True, + + # ======= basic config ======= # + 'mode': 'train', + 'epochs_between_evals': 5, + 'stop_threshold': 80.0, #used if mode': 'train', + #'data_dir':'/opt/npu/resnet_data_new', + 'data_url': '/home/mencai/training_shop-master/02-e2e/e2e_function/e2e_func_node/data/resnet50/imagenet_TF', + 'data_type': 'TFRECORD', + 'model_name': 'resnet50', + 'num_classes': 1001, + 'num_epochs': 100, + 'height':224, + 'width':224, + 'dtype': tf.float32, + 'data_format': 'channels_last', + 'use_nesterov': True, + 'eval_interval': 1, + 'loss_scale': 1024, #could be float or string. If float, static loss scaling is applied. + #If string, the corresponding automatic loss scaling algorithm is used. + #Must be one of 'Backoff' of 'LogMax' (case insensitive). + 'use_lars': False, + 'label_smoothing':0.1, #If greater than 0 then smooth the labels. + 'weight_decay': 0.0001, + 'batch_size':256, #minibatch size per node, total batchsize = batch_size*hvd.size()*itersize + + 'momentum': [0.9], + + #======= data processing config ======= + 'min_object_covered': 0.1, #used for random crop + 'aspect_ratio_range':[3. / 4., 4. / 3.], + 'area_range':[0.16, 1.0], + 'max_attempts': 100, + + #======= data augment config ======= + 'increased_aug': False, + 'brightness':0.3, + 'saturation': 0.6, + 'contrast': 0.6, + 'hue': 0.13, + 'num_preproc_threads': 22, + + #======= initialization config ======= + 'conv_init': tf.variance_scaling_initializer(), + 'bn_init_mode': 'adv_bn_init', # "adv_bn_init" means initialize gamma to 0 in each residual block's last bn, and initialize other gamma to 1 + # "conv_bn_init" means initialize all the gamma to a constant, defined by "bn_gamma_initial_value" + 'bn_gamma_initial_value': 1.0, + + #======== mode': 'train', + 'resnet_version': 'v1.5', + 'arch_type': 'original', # ------ input ------- + # C1,C2,C3: input block, stride in different layer + # ------ shortcut ------ + # D1: average_pooling + conv1*1 in shortcut in downsample block + # D2: conv3*3,stride=2 in shortcut in downsample block + # D3: conv1*1 +average_pooling in shortcut in downsample block + # ------ mainstream ---- + # E1: average_pooling + conv3*3 in mainstream in downsample block + # E2: conv3*3 + average_pooling in mainstream in downsample block + + #======= logger config ======= + 'display_every': 1, + 'log_name': 'resnet50.log', + 'log_dir': log_dir, + #======= Learning Rate Config ======= + 'lr_warmup_mode': 'train', + 'warmup_lr': 0.0, + 'warmup_epochs': 10, + 'learning_rate_maximum': 0.1, + + 'lr_decay_mode': 'train', + 'learning_rate_end': 0.00001, + + 'decay_steps': '10,20,30', #for "steps" + 'lr_decay_steps': '6.4,0.64,0.064', + + 'ploy_power': 2.0, #for "poly" and "poly_cycle" + + 'cdr_first_decay_ratio': 0.33, #for "cosine_decay_restarts" + 'cdr_t_mul':2.0, + 'cdr_m_mul':0.1, + + 'lc_periods':0.47, #for "linear_consine" + 'lc_beta':0.00001, + + 'lr_mid': 0.5, #for "linear_twice" + 'epoch_mid': 80, + + 'bn_lr_scale':1.0, + + } + +def res50_config(): + config['global_batch_size'] = config['batch_size'] * config['rank_size'] + config['do_checkpoint'] = True + + return config diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_4p.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_4p.py new file mode 100644 index 0000000..94e91ba --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_4p.py @@ -0,0 +1,109 @@ +import tensorflow as tf + +import os +log_dir = 'ckpt/' + +#256 +config = { + # ============ for testing ===================== + 'accelerator': '1980', # 'gpu', '1980' + 'shuffle_enable': 'yes', + 'shuffle_buffer_size': 10000, + 'rank_size': 4, + 'shard': True, + + # ======= basic config ======= # + 'mode': 'train', + 'epochs_between_evals': 5, + 'stop_threshold': 80.0, #used if mode': 'train', + #'data_dir':'/opt/npu/resnet_data_new', + 'data_url': '/home/mencai/training_shop-master/02-e2e/e2e_function/e2e_func_node/data/resnet50/imagenet_TF', + 'data_type': 'TFRECORD', + 'model_name': 'resnet50', + 'num_classes': 1001, + 'num_epochs': 100, + 'height':224, + 'width':224, + 'dtype': tf.float32, + 'data_format': 'channels_last', + 'use_nesterov': True, + 'eval_interval': 1, + 'loss_scale': 1024, #could be float or string. If float, static loss scaling is applied. + #If string, the corresponding automatic loss scaling algorithm is used. + #Must be one of 'Backoff' of 'LogMax' (case insensitive). + 'use_lars': False, + 'label_smoothing':0.1, #If greater than 0 then smooth the labels. + 'weight_decay': 0.0001, + 'batch_size':256, #minibatch size per node, total batchsize = batch_size*hvd.size()*itersize + + 'momentum': [0.9], + + #======= data processing config ======= + 'min_object_covered': 0.1, #used for random crop + 'aspect_ratio_range':[3. / 4., 4. / 3.], + 'area_range':[0.16, 1.0], + 'max_attempts': 100, + + #======= data augment config ======= + 'increased_aug': False, + 'brightness':0.3, + 'saturation': 0.6, + 'contrast': 0.6, + 'hue': 0.13, + 'num_preproc_threads': 22, + + #======= initialization config ======= + 'conv_init': tf.variance_scaling_initializer(), + 'bn_init_mode': 'adv_bn_init', # "adv_bn_init" means initialize gamma to 0 in each residual block's last bn, and initialize other gamma to 1 + # "conv_bn_init" means initialize all the gamma to a constant, defined by "bn_gamma_initial_value" + 'bn_gamma_initial_value': 1.0, + + #======== mode': 'train', + 'resnet_version': 'v1.5', + 'arch_type': 'original', # ------ input ------- + # C1,C2,C3: input block, stride in different layer + # ------ shortcut ------ + # D1: average_pooling + conv1*1 in shortcut in downsample block + # D2: conv3*3,stride=2 in shortcut in downsample block + # D3: conv1*1 +average_pooling in shortcut in downsample block + # ------ mainstream ---- + # E1: average_pooling + conv3*3 in mainstream in downsample block + # E2: conv3*3 + average_pooling in mainstream in downsample block + + #======= logger config ======= + 'display_every': 1, + 'log_name': 'resnet50.log', + 'log_dir': log_dir, + #======= Learning Rate Config ======= + 'lr_warmup_mode': 'train', + 'warmup_lr': 0.0, + 'warmup_epochs': 10, + 'learning_rate_maximum': 0.1, + + 'lr_decay_mode': 'train', + 'learning_rate_end': 0.00001, + + 'decay_steps': '10,20,30', #for "steps" + 'lr_decay_steps': '6.4,0.64,0.064', + + 'ploy_power': 2.0, #for "poly" and "poly_cycle" + + 'cdr_first_decay_ratio': 0.33, #for "cosine_decay_restarts" + 'cdr_t_mul':2.0, + 'cdr_m_mul':0.1, + + 'lc_periods':0.47, #for "linear_consine" + 'lc_beta':0.00001, + + 'lr_mid': 0.5, #for "linear_twice" + 'epoch_mid': 80, + + 'bn_lr_scale':1.0, + + } + +def res50_config(): + config['global_batch_size'] = config['batch_size'] * config['rank_size'] + config['do_checkpoint'] = True + + return config diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_8p.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_8p.py new file mode 100644 index 0000000..9163dc0 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/configs/res50_256bs_8p.py @@ -0,0 +1,111 @@ +import tensorflow as tf + +import os +log_dir = 'ckpt/' + +#256 +config = { + # ============ for testing ===================== + 'accelerator': '1980', # 'gpu', '1980' + 'shuffle_enable': 'yes', + 'shuffle_buffer_size': 10000, + 'rank_size': 8, + 'shard': True, + + # ======= basic config ======= # + 'mode':'train', # "train","evaluate","train_and_evaluate" + 'epochs_between_evals': 4, #used if mode is "train_and_evaluate" + 'stop_threshold': 80.0, #used if mode is "train_and_evaluate" + #'data_dir':'/opt/npu/resnet_data_new', + 'data_url': '{DATA_URL}', + 'data_type': 'TFRECORD', + 'model_name': 'resnet50', + 'num_classes': 1001, + 'num_epochs': None, + 'height':224, + 'width':224, + 'dtype': tf.float32, + 'data_format': 'channels_last', + 'use_nesterov': True, + 'eval_interval': 1, + 'loss_scale': 1024, #could be float or string. If float, static loss scaling is applied. + #If string, the corresponding automatic loss scaling algorithm is used. + #Must be one of 'Backoff' of 'LogMax' (case insensitive). + 'use_lars': False, + 'label_smoothing':0.1, #If greater than 0 then smooth the labels. + 'weight_decay': 0.0001, + 'batch_size':256, #minibatch size per node, total batchsize = batch_size*hvd.size()*itersize + + 'momentum': [0.9], + + #======= data processing config ======= + 'min_object_covered': 0.1, #used for random crop + 'aspect_ratio_range':[3. / 4., 4. / 3.], + 'area_range':[0.16, 1.0], + 'max_attempts': 100, + + #======= data augment config ======= + 'increased_aug': False, + 'brightness':0.3, + 'saturation': 0.6, + 'contrast': 0.6, + 'hue': 0.13, + 'num_preproc_threads': 22, + + #======= initialization config ======= + 'conv_init': tf.variance_scaling_initializer(), + 'bn_init_mode': 'adv_bn_init', # "conv_bn_init" or "adv_bn_init",initializer the gamma in bn in different modes + # "adv_bn_init" means initialize gamma to 0 in each residual block's last bn, and initialize other gamma to 1 + # "conv_bn_init" means initialize all the gamma to a constant, defined by "bn_gamma_initial_value" + 'bn_gamma_initial_value': 1.0, + + #======== model architecture ========== + 'resnet_version': 'v1.5', + 'arch_type': 'original', # ------ input ------- + # C1,C2,C3: input block, stride in different layer + # ------ shortcut ------ + # D1: average_pooling + conv1*1 in shortcut in downsample block + # D2: conv3*3,stride=2 in shortcut in downsample block + # D3: conv1*1 +average_pooling in shortcut in downsample block + # ------ mainstream ---- + # E1: average_pooling + conv3*3 in mainstream in downsample block + # E2: conv3*3 + average_pooling in mainstream in downsample block + + #======= logger config ======= + 'display_every': 1, + 'log_name': 'resnet50.log', + 'log_dir': log_dir, + + #======= Learning Rate Config ======= + 'lr_warmup_mode': 'linear', # "linear" or "cosine" + 'warmup_lr': 0.0, + 'warmup_epochs': 10, + 'learning_rate_maximum': 0.8, + + 'lr_decay_mode': 'cosine', # "steps", "poly", "poly_cycle", "cosine", "linear_cosine", "linear_twice", "constant" for 1980 only + 'learning_rate_end': 0.00001, + + 'decay_steps': '10,20,30', #for "steps" + 'lr_decay_steps': '6.4,0.64,0.064', + + 'ploy_power': 2.0, #for "poly" and "poly_cycle" + + 'cdr_first_decay_ratio': 0.33, #for "cosine_decay_restarts" + 'cdr_t_mul':2.0, + 'cdr_m_mul':0.1, + + 'lc_periods':0.47, #for "linear_consine" + 'lc_beta':0.00001, + + 'lr_mid': 0.5, #for "linear_twice" + 'epoch_mid': 80, + + 'bn_lr_scale':1.0, + + } + +def res50_config(): + config['global_batch_size'] = config['batch_size'] * config['rank_size'] + config['do_checkpoint'] = True + + return config diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/data_loader/__init__.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/data_loader/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/data_loader/resnet50/data_loader.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/data_loader/resnet50/data_loader.py new file mode 100644 index 0000000..5d5285e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/data_loader/resnet50/data_loader.py @@ -0,0 +1,237 @@ +import numpy as np +from . import preprocessing +import tensorflow as tf +from tensorflow.python.util import nest +import os,sys +import numpy as np +sys.path.append("..") +from trainers.train_helper import stage + +class DataLoader: + + def __init__(self, config): + self.config = config + + num_training_samples = 1281167 + #num_evaluating_samples = get_num_records(self.eval_filenames) + self.config['num_training_samples'] = num_training_samples + self.config['num_evaluating_samples'] = 50000 + print( 'total num_training_sampels: %d' % num_training_samples ) + + self.training_samples_per_rank = num_training_samples + + + def get_train_input_fn_synthetic(self): + batch_size = self.config['batch_size'] + input_shape = [self.config['height'], self.config['width'], 3] + input_element = nest.map_structure(lambda s: tf.constant(0.5, tf.float32, s), tf.TensorShape(input_shape)) + label_element = nest.map_structure(lambda s: tf.constant(1, tf.int32, s), tf.TensorShape([1])) + element = (input_element, label_element) + ds = tf.data.Dataset.from_tensors(element).repeat() + ds = ds.batch(batch_size) + return ds + + def get_train_input_fn(self): + # filenames = self.train_filenames + filenames = None + take_count = self.training_samples_per_rank + batch_size = self.config['batch_size'] + height = self.config['height'] + width = self.config['width'] + brightness = self.config['brightness'] + contrast = self.config['contrast'] + saturation = self.config['saturation'] + hue = self.config['hue'] + num_threads = self.config['num_preproc_threads'] + increased_aug = self.config['increased_aug'] + shard = self.config['shard'] + + return make_dataset(self.config, filenames, take_count, batch_size, height, width, + brightness, contrast, saturation, hue, + training=True, num_threads=num_threads, nsummary=10, shard=shard, synthetic=False, + increased_aug=increased_aug ) + + def get_eval_input_fn(self): + # filenames = self.eval_filenames + filenames = None + # take_count = get_num_records(self.eval_filenames) + take_count = 50000 + batch_size = self.config['batch_size'] + height = self.config['height'] + width = self.config['width'] + brightness = self.config['brightness'] + contrast = self.config['contrast'] + saturation = self.config['saturation'] + hue = self.config['hue'] + num_threads = self.config['num_preproc_threads'] + shard = self.config['shard'] + + return make_dataset(self.config, filenames, take_count, batch_size, height, width, + brightness, contrast, saturation, hue, + training=False, num_threads=num_threads, nsummary=10, shard=shard, synthetic=False, + increased_aug=False) + + def get_input_pipeline_op(self, inputs, labels, mode): + with tf.device('/cpu:0'): + preload_op, (inputs, labels) = stage([inputs, labels]) + + with tf.device('/gpu:0'): + gpucopy_op, (inputs, labels) = stage([inputs, labels]) + return preload_op, gpucopy_op, inputs, labels + + def normalize_and_format(self, inputs, data_format): + + dataset_mean = np.array([121, 115, 100], dtype=np.float32) + dataset_std = np.array([70, 68, 71], dtype=np.float32) + inputs = tf.subtract(inputs, dataset_mean) + inputs = tf.multiply(inputs, 1. / dataset_std) + if data_format == 'channels_first': + inputs = tf.transpose(inputs, [0, 3, 1, 2]) + return inputs + + + + +#-------------------------------- Funcs ----------------------------------- +def get_num_records(filenames): + def count_records(tf_record_filename): + count = 0 + for _ in tf.python_io.tf_record_iterator(tf_record_filename): + count += 1 + return count + + nfile = len(filenames) + return (count_records(filenames[0]) * (nfile - 1) + + count_records(filenames[-1])) + +def _parse_example_proto(example_serialized): + feature_map = { + 'image/encoded': tf.FixedLenFeature([], dtype=tf.string, + default_value=''), + 'image/class/label': tf.FixedLenFeature([], dtype=tf.int64, default_value=-1), + 'image/class/text': tf.FixedLenFeature([], dtype=tf.string, + default_value=''), + } + sparse_float32 = tf.VarLenFeature(dtype=tf.float32) + # Sparse features in Example proto. + feature_map.update( + {k: sparse_float32 for k in ['image/object/bbox/xmin', + 'image/object/bbox/ymin', + 'image/object/bbox/xmax', + 'image/object/bbox/ymax']}) + + features = tf.parse_single_example(example_serialized, feature_map) + label = tf.cast(features['image/class/label'], dtype=tf.int32) + + xmin = tf.expand_dims(features['image/object/bbox/xmin'].values, 0) + ymin = tf.expand_dims(features['image/object/bbox/ymin'].values, 0) + xmax = tf.expand_dims(features['image/object/bbox/xmax'].values, 0) + ymax = tf.expand_dims(features['image/object/bbox/ymax'].values, 0) + + # Note that we impose an ordering of (y, x) just to make life difficult. + bbox = tf.concat([ymin, xmin, ymax, xmax], 0) + + # Force the variable number of bounding boxes into the shape + # [1, num_boxes, coords]. + bbox = tf.expand_dims(bbox, 0) + bbox = tf.transpose(bbox, [0, 2, 1]) + + return features['image/encoded'], label, bbox + +def parse_record(raw_record): + image_buffer, label, bbox = _parse_example_proto(raw_record) + # for 1980 only + config={'min_object_covered': 0.1, 'aspect_ratio_range': [3. / 4., 4. / 3.], 'area_range': [0.16, 1.0], 'max_attempts': 100} + image = preprocessing.parse_and_preprocess_image_record( + config, image_buffer, height=224, width=224, + brightness=0.3, contrast=0.6, saturation=0.6, hue=0.13, + distort=True, nsummary=10, increased_aug=False, random_search_aug=False) + return image, label + +def read_rawdata(file_path_tensor): + def _read_file(file_path): + image = tf.gfile.GFile(file_path, 'rb').read() + return image + return tf.py_func(_read_file, inp=[file_path_tensor], Tout=tf.string) + +def parse_function(filename, label): + image = read_rawdata(filename) + image_decoded = tf.image.decode_jpeg(image, channels=3) + image_resized = tf.image.resize_images(image_decoded, [224, 224]) + # 7.3，raw默认格式为int64，目前resnet50只支持int32，下沉前不影响，下沉后，没有增加该转换算子，影响性能考虑。 + label = tf.cast(label, dtype=tf.int32) + return image_resized, label + +def parse_record1(image, label): + image = preprocessing.split_device(image) + return image, label + +def make_dataset(config, filenames, take_count, batch_size, height, width, + brightness, contrast, saturation, hue, + training=False, num_threads=10, nsummary=10, shard=False, synthetic=False, + increased_aug=False, random_search_aug=False): + if synthetic and training: + input_shape = [height, width, 3] + input_element = nest.map_structure(lambda s: tf.constant(0.5, tf.float32, s), tf.TensorShape(input_shape)) + label_element = nest.map_structure(lambda s: tf.constant(1, tf.int32, s), tf.TensorShape([1])) + element = (input_element, label_element) + ds = tf.data.Dataset.from_tensors(element).repeat() + ds = ds.batch(batch_size) + return ds + else: + shuffle_buffer_size = 10000 + num_readers = 10 + rank_size = int(os.getenv('RANK_SIZE')) + rank_id = int(os.getenv('DEVICE_INDEX')) + + if config['data_type'] == 'RAW DATA': + images = [] + labels = [] + with tf.gfile.GFile(config['label_index_url'], 'r') as f: + for line in f.readlines(): + tmp_list = line.strip().split(" ") + image_file = os.path.join(config['data_url'], tmp_list[0]) + #image_raw = tf.gfile.GFile(image_file, 'rb').read() + #images.append(image_raw) + images.append(image_file) + labels.append(int(tmp_list[-1])) + + #images = tf.convert_to_tensor(images, dtype=tf.string) + #labels = tf.convert_to_tensor(labels, dtype=tf.int32) + ds = tf.data.Dataset.from_tensor_slices((images, labels)) + else: + filename_pattern = os.path.join(config['data_url'], '%s-*') + filenames = sorted(tf.gfile.Glob(filename_pattern % 'train')) + ds = tf.data.Dataset.from_tensor_slices(filenames) + + if shard: + # split the dataset into parts for each GPU + ds = ds.shard(rank_size, rank_id) + + if not training: + ds = ds.take(take_count) # make sure all ranks have the same amount + + if training: + ds = ds.shuffle(1000, seed=7 * (1 + rank_id)) + + if config['data_type'] == 'TFRECORD': + ds = ds.interleave(tf.data.TFRecordDataset, cycle_length=num_readers, block_length=1) + counter = tf.data.Dataset.range(sys.maxsize) + ds = tf.data.Dataset.zip((ds, counter)) + + if training: + ds = ds.apply(tf.data.experimental.shuffle_and_repeat(shuffle_buffer_size, seed=5*(1+rank_id))) + + if config['data_type'] == 'RAW DATA': + ds = ds.map(lambda image, label: parse_function(image, label), num_parallel_calls=14) + else: + #ds = ds.map(lambda image, label: parse_record(image), num_parallel_calls=192) + #ds = ds.prefetch(buffer_size=tf.contrib.data.AUTOTUNE) + #ds = ds.map(lambda image, label: parse_record1(image, label), num_parallel_calls=14) + ds = ds.map(lambda image, label: parse_record(image), num_parallel_calls=192) + #ds = ds.prefetch(10) + ds = ds.batch(batch_size, drop_remainder=True) + ds = ds.prefetch(buffer_size=tf.contrib.data.AUTOTUNE) + return ds + + diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/data_loader/resnet50/preprocessing.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/data_loader/resnet50/preprocessing.py new file mode 100644 index 0000000..9486a81 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/data_loader/resnet50/preprocessing.py @@ -0,0 +1,160 @@ +import tensorflow as tf +#import horovod.tensorflow as hvd +from tensorflow.contrib.image.python.ops import distort_image_ops +import math +#from .data_aug_search import random_aug_search + + + +def deserialize_image_record(record): + feature_map = { + 'image/encoded': tf.FixedLenFeature([], tf.string, ''), + 'image/class/label': tf.FixedLenFeature([1], tf.int64, -1), + 'image/class/text': tf.FixedLenFeature([], tf.string, ''), + 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32) + } + with tf.name_scope('deserialize_image_record'): + obj = tf.parse_single_example(record, feature_map) + imgdata = obj['image/encoded'] + label = tf.cast(obj['image/class/label'], tf.int32) + bbox = tf.stack([obj['image/object/bbox/%s' % x].values + for x in ['ymin', 'xmin', 'ymax', 'xmax']]) + bbox = tf.transpose(tf.expand_dims(bbox, 0), [0, 2, 1]) + text = obj['image/class/text'] + return imgdata, label, bbox, text + +def decode_jpeg(imgdata, channels=3): + return tf.image.decode_jpeg(imgdata, channels=channels, + fancy_upscaling=False, + dct_method='INTEGER_FAST') + + +def crop_and_resize_image(config, image, height, width, + distort=False, nsummary=10): + with tf.name_scope('crop_and_resize'): + # Evaluation is done on a center-crop of this ratio + eval_crop_ratio = 0.8 + if distort: + # crop_window = tf.stack( [0, 0, 7, 7] ) + # image = tf.image.decode_and_crop_jpeg( image, crop_window, channels=3 ) + # image = tf.image.resize_images( image, [height, width] ) + initial_shape = [int(round(height / eval_crop_ratio)), + int(round(width / eval_crop_ratio)), + 3] + jpeg_shape = tf.image.extract_jpeg_shape( image ) + + bbox_begin, bbox_size, bbox = \ + tf.image.sample_distorted_bounding_box( + initial_shape, + bounding_boxes=tf.constant([0.0, 0.0, 1.0, 1.0], dtype=tf.float32, shape=[1, 1, 4]), + # tf.zeros(shape=[1,0,4]), # No bounding boxes + min_object_covered=config['min_object_covered'], + aspect_ratio_range=config['aspect_ratio_range'], + area_range=config['area_range'], + max_attempts=config['max_attempts'], + # seed=11 , # Need to set for deterministic results + use_image_if_no_bounding_boxes=True) + bbox = bbox[0, 0] # Remove batch, box_idx dims + + # offset_y, offset_x, _ = tf.unstack(bbox_begin) + # target_height, target_width, _ = tf.unstack( bbox_size ) + # + + + + + # offset_y = tf.minimum( offset_y, jpeg_shape[0] - 1 ) + # offset_x = tf.minimum( offset_x, jpeg_shape[1] - 1 ) + + # target_height, target_width, _ = tf.unstack( bbox_size ) + # new_height = tf.maximum( tf.minimum( offset_y + target_height, jpeg_shape[0] ) - offset_y, 0 ) + # new_width = tf.maximum( tf.minimum( offset_x + target_width, jpeg_shape[1] ) - offset_x, 0 ) + + y_min = tf.cast( bbox[0] * (tf.cast( jpeg_shape[0], tf.float32) ), tf.int32) + x_min = tf.cast( bbox[1] * (tf.cast(jpeg_shape[1], tf.float32) ), tf.int32) + y_max = tf.cast( bbox[2] * (tf.cast(jpeg_shape[0], tf.float32) ), tf.int32) + x_max = tf.cast( bbox[3] * (tf.cast(jpeg_shape[1], tf.float32) ), tf.int32) + + crop_height = y_max - y_min + crop_width = x_max - x_min + # crop_window = tf.stack( [offset_y, offset_x, new_height, new_width] ) + crop_window = tf.stack( [y_min, x_min, crop_height, crop_width] ) + image = tf.image.decode_and_crop_jpeg( image, crop_window, channels=3 ) + image = tf.image.resize_images( image, [height, width] ) + + + # def func_decode_and_crop(image): + # image = tf.image.decode_and_crop_jpeg( image, crop_window, channels=3 ) + # image = tf.image.resize_images( image, [height, width] ) + # return image + + # def func_crop_and_resize(image): + # image = decode_jpeg(image, channels=3) + # image = tf.image.crop_and_resize( + # image[None, :, :, :], bbox[None, :], [0], [height, width])[0] + # return image + + + # condtion_1 = tf.logical_and( tf.less(target_height, jpeg_shape[0]), tf.less( target_width, jpeg_shape[1] ) ) + # condtion_2 = tf.logical_and( tf.less(target_height + offset_y, jpeg_shape[0]), tf.less( target_width + offset_x, jpeg_shape[1] ) ) + + # image = tf.cond( tf.logical_and( condtion_1, condtion_2 ), lambda:func_decode_and_crop(image), lambda:func_crop_and_resize(image) ) + + + else: + # Central crop + + image = decode_jpeg(image, channels=3) + ratio_y = ratio_x = eval_crop_ratio + bbox = tf.constant([0.5 * (1 - ratio_y), 0.5 * (1 - ratio_x), + 0.5 * (1 + ratio_y), 0.5 * (1 + ratio_x)]) + image = tf.image.crop_and_resize( + image[None, :, :, :], bbox[None, :], [0], [height, width])[0] + + return image + + +def parse_and_preprocess_image_record(config, record, height, width, + brightness, contrast, saturation, hue, + distort, nsummary=10, increased_aug=False, random_search_aug=False): + #imgdata, label, bbox, text = deserialize_image_record(record) + #label -= 1 # Change to 0-based (don't use background class) + with tf.name_scope('preprocess_train'): + image = crop_and_resize_image(config, record, height, width, distort) + if distort: + image = tf.image.random_flip_left_right(image) + if increased_aug: + image = tf.image.random_brightness(image, max_delta=brightness) + image = distort_image_ops.random_hsv_in_yiq(image, + lower_saturation=saturation, + upper_saturation=2.0 - saturation, + max_delta_hue=hue * math.pi) + image = tf.image.random_contrast(image, lower=contrast, upper=2.0 - contrast) + tf.summary.image('distorted_color_image', tf.expand_dims(image, 0)) + +# image = tf.clip_by_value(image, 0., 255.) + #image = tf.cast(image, tf.uint8) + # if random_search_aug: + # image = random_aug_search(image) +# image = normalize(image) +# image = tf.cast(image, tf.float16) + return image +def normalize(inputs): + dataset_mean = [121.0, 115.0, 100.0] #np.array([121, 115, 100], dtype=np.float32) + dataset_std = [70.0, 68.0, 71.0] #np.array([70, 68, 71], dtype=np.float32) + dataset_mean = tf.expand_dims(tf.expand_dims(dataset_mean, 0), 0) + dataset_std = tf.expand_dims(tf.expand_dims(dataset_std, 0), 0) + inputs = inputs - dataset_mean #tf.subtract(inputs, dataset_mean) + inputs = inputs * (1.0 / dataset_std) + #inputs = tf.multiply(inputs, 1. / dataset_std) + + return inputs + +def split_device(image): + image = tf.clip_by_value(image, 0., 255.) + image = normalize(image) + image = tf.cast(image, tf.float16) + return image diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/hyper_param/__init__.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/hyper_param/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/hyper_param/hyper_param.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/hyper_param/hyper_param.py new file mode 100644 index 0000000..eed8743 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/hyper_param/hyper_param.py @@ -0,0 +1,50 @@ +import tensorflow as tf +from .lr_schedule import warmup_decay, get_lr, get_1980_lr + + +class HyperParams: + def __init__(self, config): + self.config=config + nsteps_per_epoch = self.config['num_training_samples'] // self.config['global_batch_size'] + self.config['nsteps_per_epoch'] = nsteps_per_epoch + # nstep = self.config['num_training_samples'] * self.config['num_epochs'] // self.config['global_batch_size'] + if self.config['num_epochs']: + nstep = nsteps_per_epoch * self.config['num_epochs'] #------calculate nsteps in a different way------ + else: + nstep = self.config['max_train_steps'] + self.config['nstep'] = nstep + + self.config['total_steps_include_iterations'] = int( self.config['nstep'] + self.config['iterations_per_loop']) + self.config['save_summary_steps'] = nsteps_per_epoch + self.config['save_checkpoints_steps'] = nsteps_per_epoch + + + def get_hyper_params(self): + hyper_params = {} + hyper_params['learning_rate'] = self.get_learning_rate() + + return hyper_params + + + def get_learning_rate(self): + global_step = tf.train.get_global_step() + nsteps_per_epoch = self.config['nsteps_per_epoch'] + + warmup_lr = self.config['warmup_lr'] + lr = self.config['learning_rate_maximum'] + lr_end = self.config['learning_rate_end'] + lr_decay_mode = self.config['lr_decay_mode'] + + + + with tf.device('/cpu:0'): # Allow fallback to CPU if no GPU support for these ops + + if lr_decay_mode == 'constant' or self.config['num_epochs'] == None: + learning_rate = tf.constant(lr, tf.float32) + else: + learning_rate = get_1980_lr(self.config, global_step, warmup_lr, lr_end, lr, self.config['warmup_epochs'], nsteps_per_epoch, self.config['nstep'], lr_decay_mode ) + + learning_rate = tf.identity(learning_rate, 'learning_rate') + return learning_rate + + diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/hyper_param/lr_schedule.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/hyper_param/lr_schedule.py new file mode 100644 index 0000000..37103e9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/hyper_param/lr_schedule.py @@ -0,0 +1,168 @@ +import tensorflow as tf +import numpy as np + +def get_lr(lr, lr_end, lr_decay_mode, warmup_it, decay_steps, global_step, steps, lr_steps, ploy_power, + cdr_first_decay_ratio, cdr_t_mul, cdr_m_mul, cdr_alpha, cd_alpha, lc_periods, lc_alpha, lc_beta, lr_mid, it_mid): + if lr_decay_mode == 'steps': + learning_rate = tf.train.piecewise_constant(global_step, + steps, lr_steps) + elif lr_decay_mode == 'poly' or lr_decay_mode == 'poly_cycle': + cycle = lr_decay_mode == 'poly_cycle' + learning_rate = tf.train.polynomial_decay(lr, + global_step - warmup_it, + decay_steps=decay_steps - warmup_it, + end_learning_rate=lr_end, + power=ploy_power, + cycle=cycle) + elif lr_decay_mode == 'cosine_decay_restarts': + learning_rate = tf.train.cosine_decay_restarts(lr, + global_step - warmup_it, + (decay_steps - warmup_it) * cdr_first_decay_ratio, + t_mul=cdr_t_mul, + m_mul=cdr_m_mul, + alpha=cdr_alpha) + elif lr_decay_mode == 'cosine': + learning_rate = tf.train.cosine_decay(lr, + global_step - warmup_it, + decay_steps=decay_steps - warmup_it, + alpha=cd_alpha) + elif lr_decay_mode == 'linear_cosine': + learning_rate = tf.train.linear_cosine_decay(lr, + global_step - warmup_it, + decay_steps=decay_steps - warmup_it, + num_periods=lc_periods,#0.47, + alpha=lc_alpha,#0.0, + beta=lc_beta)#0.00001) + elif lr_decay_mode == 'linear_twice': + learning_rate = decay_linear_twice(lr, lr_mid, lr_end, warmup_it, it_mid, decay_steps, global_step ) + + else: + raise ValueError('Invalid type of lr_decay_mode') + return learning_rate + + +def cos_warmup_1980( global_step, warmup_steps, max_lr ): + PI = 3.14159265359 + ang = PI + PI * ( float(global_step+1) / float(warmup_steps) ) + offset = max_lr * 0.5*( 1.0 + np.cos( ang ) ) + return offset + +def cos_decay_1980( global_step, warmup_steps, total_steps, max_lr ): + PI = 3.14159265359 + ang = PI * ( float(global_step - warmup_steps+1) / float(total_steps - warmup_steps) ) + offset = max_lr * 0.5*( 1.0 + np.cos( ang ) ) + return offset + + +def get_1980_lr(config, global_step, lr_init, lr_end, lr_max, warmup_epochs, steps_per_epoch, nsteps, lr_decay_mode): + lr_each_step = [] + + if lr_decay_mode == 'steps': + decay_epoch_index = [30 * steps_per_epoch,60 * steps_per_epoch,80 * steps_per_epoch] + total_steps = int(nsteps) + for i in range(total_steps): + if i < decay_epoch_index[0]: + lr = lr_max + elif i < decay_epoch_index[1]: + lr = lr_max * 0.1 + elif i < decay_epoch_index[2]: + lr = lr_max * 0.01 + else: + lr = lr_max * 0.001 + lr_each_step.append(lr) + elif lr_decay_mode == 'poly': + total_steps = int(nsteps) + warmup_steps = steps_per_epoch * warmup_epochs + inc_each_step = ( float(lr_max) - float(lr_init) ) / float(warmup_steps) + for i in range( config['total_steps_include_iterations'] ): + if i <= warmup_steps: + lr = float(lr_init) + inc_each_step * float(i) + elif i < total_steps: + base = ( 1.0 - (float(i)-float(warmup_steps))/(float(total_steps)-float(warmup_steps)) ) + lr = float(lr_max) * base + else: + lr = 0.0 + lr_each_step.append(lr) + + elif lr_decay_mode == 'cosine': + total_steps = int(nsteps) + + warmup_steps = steps_per_epoch * warmup_epochs + for i in range( config['total_steps_include_iterations'] ): + if i <= warmup_steps: + lr = cos_warmup_1980( i, warmup_steps, lr_max ) + elif i < total_steps: + lr = cos_decay_1980( i, warmup_steps, total_steps, lr_max ) + else: + lr = 0.0 + lr_each_step.append(lr) + elif lr_decay_mode == 'linear_cosine': + total_steps = int(nsteps) + warmup_steps = steps_per_epoch * warmup_epochs + inc_each_step = ( float(lr_max) - float(lr_init) ) / float(warmup_steps) + for i in range( config['total_steps_include_iterations'] ): + if i <= warmup_steps: + lr = float(lr_init) + inc_each_step * float(i) + elif i < total_steps: + lr = cos_decay_1980( i, warmup_steps, total_steps, lr_max ) + else: + lr = 0.0 + lr_each_step.append(lr) + else: + total_steps = int(nsteps) + warmup_steps = steps_per_epoch * warmup_epochs + for i in range(total_steps): + if i <= warmup_steps: + lr = lr_init + (lr_max - lr_init) * i / warmup_steps + else: + lr = lr_max - ( lr_max - lr_end ) * (i - warmup_steps) / (total_steps - warmup_steps) + lr_each_step.append( lr ) + + # current_step = tf.to_int32( tf.cast(global_step,tf.float32) / float(steps_per_epoch) ) + current_step = global_step + lr_each_step = tf.convert_to_tensor( lr_each_step ) + print (lr_each_step) + learning_rate = tf.gather( lr_each_step, current_step ) + + return learning_rate + +def warmup_decay(lr_warmup_mode, warmup_lr, global_step, warmup_steps, warmup_end_lr): + if lr_warmup_mode == 'linear': + learning_rate = linear_warmup(warmup_lr, global_step, warmup_steps, warmup_end_lr) + elif lr_warmup_mode == 'cosine': + learning_rate = cos_warmup(warmup_lr, global_step, warmup_steps, warmup_end_lr) + else: + raise ValueError('Invalid type of lr_warmup_mode') + return learning_rate + + +def linear_warmup(warmup_lr, global_step, warmup_steps, warmup_end_lr): + from tensorflow.python.ops import math_ops + p = tf.cast(global_step, tf.float32) / tf.cast(warmup_steps, tf.float32) + diff = math_ops.subtract(warmup_end_lr, warmup_lr) + res = math_ops.add(warmup_lr, math_ops.multiply(diff, p)) + return res + +def cos_warmup( warmup_lr, global_step, warmup_steps, warmup_end_lr ): + PI = 3.14159265359 + diff = tf.subtract( warmup_end_lr, warmup_lr ) + ang = PI + PI * ( tf.cast( global_step, tf.float32 ) / tf.cast( warmup_steps,tf.float32 )) + offset = diff * 0.5 * ( 1.0 + tf.math.cos( ang ) ) + res = tf.add( warmup_lr, offset ) + return res + + +def decay_linear( lr_start, lr_end, it_start, it_end, global_step ): + down_steps = it_end - it_start + down_range = lr_start - lr_end + down_per_step = float( down_range ) / float( down_steps ) + res = tf.subtract( tf.cast(lr_start, tf.float32), tf.multiply( tf.cast(down_per_step, tf.float32), tf.subtract(tf.cast(global_step, tf.float32), tf.cast(it_start, tf.float32) )) ) + return res + +def decay_linear_twice(lr_start, lr_mid, lr_end, it_start, it_mid, it_end, global_step ): + learning_rate = tf.cond( global_step < it_start, lambda: tf.cast(lr_start, tf.float32), lambda: decay_linear(lr_start, lr_mid, it_start, it_mid, global_step)) + learning_rate = tf.cond( global_step > it_mid, lambda: decay_linear(lr_mid, lr_end, it_mid, it_end, global_step) , lambda: learning_rate ) + return learning_rate + + + diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/hyper_param/lr_schedule_0907_back.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/hyper_param/lr_schedule_0907_back.py new file mode 100644 index 0000000..126efc8 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/hyper_param/lr_schedule_0907_back.py @@ -0,0 +1,169 @@ +import tensorflow as tf +import numpy as np + +def get_lr(lr, lr_end, lr_decay_mode, warmup_it, decay_steps, global_step, steps, lr_steps, ploy_power, + cdr_first_decay_ratio, cdr_t_mul, cdr_m_mul, cdr_alpha, cd_alpha, lc_periods, lc_alpha, lc_beta, lr_mid, it_mid): + if lr_decay_mode == 'steps': + learning_rate = tf.train.piecewise_constant(global_step, + steps, lr_steps) + elif lr_decay_mode == 'poly' or lr_decay_mode == 'poly_cycle': + cycle = lr_decay_mode == 'poly_cycle' + learning_rate = tf.train.polynomial_decay(lr, + global_step - warmup_it, + decay_steps=decay_steps - warmup_it, + end_learning_rate=lr_end, + power=ploy_power, + cycle=cycle) + elif lr_decay_mode == 'cosine_decay_restarts': + learning_rate = tf.train.cosine_decay_restarts(lr, + global_step - warmup_it, + (decay_steps - warmup_it) * cdr_first_decay_ratio, + t_mul=cdr_t_mul, + m_mul=cdr_m_mul, + alpha=cdr_alpha) + elif lr_decay_mode == 'cosine': + learning_rate = tf.train.cosine_decay(lr, + global_step - warmup_it, + decay_steps=decay_steps - warmup_it, + alpha=cd_alpha) + elif lr_decay_mode == 'linear_cosine': + learning_rate = tf.train.linear_cosine_decay(lr, + global_step - warmup_it, + decay_steps=decay_steps - warmup_it, + num_periods=lc_periods,#0.47, + alpha=lc_alpha,#0.0, + beta=lc_beta)#0.00001) + elif lr_decay_mode == 'linear_twice': + learning_rate = decay_linear_twice(lr, lr_mid, lr_end, warmup_it, it_mid, decay_steps, global_step ) + + else: + raise ValueError('Invalid type of lr_decay_mode') + return learning_rate + + +def cos_warmup_1980( global_step, warmup_steps, max_lr ): + PI = 3.14159265359 + ang = PI + PI * ( float(global_step) / float(warmup_steps) ) + offset = max_lr * 0.5*( 1.0 + np.cos( ang ) ) + return offset + +def cos_decay_1980( global_step, warmup_steps, total_steps, max_lr ): + PI = 3.14159265359 + ang = PI * ( float(global_step - warmup_steps) / float(total_steps - warmup_steps) ) + offset = max_lr * 0.5*( 1.0 + np.cos( ang ) ) + return offset + + +def get_1980_lr(config, global_step, lr_init, lr_end, lr_max, warmup_epochs, total_epochs, steps_per_epoch, lr_decay_mode): + lr_each_step = [] + + if lr_decay_mode == 'steps': + decay_epoch_index = [30 * steps_per_epoch,60 * steps_per_epoch,80 * steps_per_epoch] + total_steps = int(steps_per_epoch * total_epochs) + # total_steps = total_epochs + for i in range(total_steps): + if i < decay_epoch_index[0]: + lr = lr_max + elif i < decay_epoch_index[1]: + lr = lr_max * 0.1 + elif i < decay_epoch_index[2]: + lr = lr_max * 0.01 + else: + lr = lr_max * 0.001 + lr_each_step.append(lr) + elif lr_decay_mode == 'poly': + total_steps = int(steps_per_epoch * total_epochs) + warmup_steps = steps_per_epoch * warmup_epochs + inc_each_step = ( float(lr_max) - float(lr_init) ) / float(warmup_steps) + for i in range( config['total_steps_include_iterations'] ): + if i < warmup_steps: + lr = float(lr_init) + inc_each_step * float(i) + elif i <= total_steps: + base = ( 1.0 - (float(i)-float(warmup_steps))/(float(total_steps)-float(warmup_steps)) ) + lr = float(lr_max) * base + else: + lr = 0.0 + lr_each_step.append(lr) + + elif lr_decay_mode == 'cosine': + total_steps = int(steps_per_epoch * total_epochs) + + warmup_steps = steps_per_epoch * warmup_epochs + for i in range( config['total_steps_include_iterations'] ): + if i < warmup_steps: + lr = cos_warmup_1980( i, warmup_steps, lr_max ) + elif i <= total_steps: + lr = cos_decay_1980( i, warmup_steps, total_steps, lr_max ) + else: + lr = 0.0 + lr_each_step.append(lr) + # elif lr_decay_mode == 'linear_cosine': + # total_steps = int(steps_per_epoch * total_epochs) + # warmup_steps = steps_per_epoch * warmup_epochs + # inc_each_step = ( float(lr_max) - float(lr_init) ) / float(warmup_steps) + # for i in range( config['total_steps_include_iterations'] ): + # if i < warmup_steps: + # lr = float(lr_init) + inc_each_step * float(i) + # elif i <= total_steps: + # lr = cos_decay_1980( i, warmup_steps, total_steps, lr_max ) + # else: + # lr = 0.0 + # lr_each_step.append(lr) + else: + total_steps = steps_per_epoch * total_epochs + warmup_steps = steps_per_epoch * warmup_epochs + for i in range(total_steps): + if i < warmup_steps: + lr = lr_init + (lr_max - lr_init) * i / warmup_steps + else: + lr = lr_max - ( lr_max - lr_end ) * (i - warmup_steps) / (total_steps - warmup_steps) + lr_each_step.append( lr ) + + # current_step = tf.to_int32( tf.cast(global_step,tf.float32) / float(steps_per_epoch) ) + current_step = global_step + lr_each_step = tf.convert_to_tensor( lr_each_step ) + print (lr_each_step) + learning_rate = tf.gather( lr_each_step, current_step ) + + return learning_rate + +def warmup_decay(lr_warmup_mode, warmup_lr, global_step, warmup_steps, warmup_end_lr): + if lr_warmup_mode == 'linear': + learning_rate = linear_warmup(warmup_lr, global_step, warmup_steps, warmup_end_lr) + elif lr_warmup_mode == 'cosine': + learning_rate = cos_warmup(warmup_lr, global_step, warmup_steps, warmup_end_lr) + else: + raise ValueError('Invalid type of lr_warmup_mode') + return learning_rate + + +def linear_warmup(warmup_lr, global_step, warmup_steps, warmup_end_lr): + from tensorflow.python.ops import math_ops + p = tf.cast(global_step, tf.float32) / tf.cast(warmup_steps, tf.float32) + diff = math_ops.subtract(warmup_end_lr, warmup_lr) + res = math_ops.add(warmup_lr, math_ops.multiply(diff, p)) + return res + +def cos_warmup( warmup_lr, global_step, warmup_steps, warmup_end_lr ): + PI = 3.14159265359 + diff = tf.subtract( warmup_end_lr, warmup_lr ) + ang = PI + PI * ( tf.cast( global_step, tf.float32 ) / tf.cast( warmup_steps,tf.float32 )) + offset = diff * 0.5 * ( 1.0 + tf.math.cos( ang ) ) + res = tf.add( warmup_lr, offset ) + return res + + +def decay_linear( lr_start, lr_end, it_start, it_end, global_step ): + down_steps = it_end - it_start + down_range = lr_start - lr_end + down_per_step = float( down_range ) / float( down_steps ) + res = tf.subtract( tf.cast(lr_start, tf.float32), tf.multiply( tf.cast(down_per_step, tf.float32), tf.subtract(tf.cast(global_step, tf.float32), tf.cast(it_start, tf.float32) )) ) + return res + +def decay_linear_twice(lr_start, lr_mid, lr_end, it_start, it_mid, it_end, global_step ): + learning_rate = tf.cond( global_step < it_start, lambda: tf.cast(lr_start, tf.float32), lambda: decay_linear(lr_start, lr_mid, it_start, it_mid, global_step)) + learning_rate = tf.cond( global_step > it_mid, lambda: decay_linear(lr_mid, lr_end, it_mid, it_end, global_step) , lambda: learning_rate ) + return learning_rate + + + diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/layers/__init__.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/layers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/layers/layers.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/layers/layers.py new file mode 100644 index 0000000..0312452 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/layers/layers.py @@ -0,0 +1,23 @@ +import tensorflow as tf +#from tensorflow.contrib.hccl.python.ops import hccl_ops +from npu_bridge.hccl import hccl_ops + +class Layers: + + def get_accuracy(self, labels, predicted_classes, logits, config): + accuracy = tf.metrics.accuracy( + labels=labels, predictions=predicted_classes) + top5acc = tf.metrics.mean( + tf.cast(tf.nn.in_top_k(logits, labels, 5), tf.float32)) + if config['rank_size'] == 1: + newaccuracy = (accuracy[0], accuracy[1]) + newtop5acc = (top5acc[0], top5acc[1]) + else: + newaccuracy = (hccl_ops.allreduce(accuracy[0],"sum")/config['rank_size'], accuracy[1]) + newtop5acc = (hccl_ops.allreduce(top5acc[0],"sum")/config['rank_size'], top5acc[1]) + metrics = {'val-top1acc': newaccuracy, 'val-top5acc': newtop5acc} + return metrics + + + + diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/losses/__init__.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/losses/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/losses/res50_loss.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/losses/res50_loss.py new file mode 100644 index 0000000..ad383f9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/losses/res50_loss.py @@ -0,0 +1,36 @@ +import tensorflow as tf + +class Loss: + def __init__(self,config): + self.config = config + + def get_loss(self, logits, labels): + labels_one_hot = tf.one_hot(labels, self.config['num_classes']) + loss = tf.losses.softmax_cross_entropy( + logits=logits, onehot_labels=labels_one_hot,label_smoothing=self.config['label_smoothing']) + loss = tf.identity(loss, name='loss') + return loss + + def get_total_loss(self, loss): + reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) + total_loss = tf.add_n([loss] + reg_losses, name='total_loss') + return total_loss + + + def optimize_loss(self, total_loss, opt): + gate_gradients = (tf.train.Optimizer.GATE_NONE) + # grads_and_vars = opt.compute_gradients(total_loss, colocate_gradients_with_ops=True, gate_gradients=gate_gradients) + grads_and_vars = opt.compute_gradients(total_loss, gate_gradients=gate_gradients) + + # train_op = opt.apply_gradients( grads_and_vars, global_step=None ) + train_op = opt.apply_gradients( grads_and_vars) + + return train_op + + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/mains/clean.sh b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/mains/clean.sh new file mode 100644 index 0000000..14ec7df --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/mains/clean.sh @@ -0,0 +1,7 @@ + ps -ef | grep TdtMain | awk '{print $2}' | xargs kill -9 +rm -rf *.pbtxt +rm -rf /var/log/npu/slog/*.log +rm ckpt* -rf +find ./ -name "*.pyc" | xargs rm -rf +find ./ -name __pycache__ | xargs rm -rf +rm /var/log/npu/dataset/* -rf diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/mains/res50.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/mains/res50.py new file mode 100644 index 0000000..0b30c0c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/mains/res50.py @@ -0,0 +1,120 @@ +import tensorflow as tf +import sys +import ast +#sys.path.append("..") +#sys.path.append("../models") +#sys.path.append("./resnet50_train/") +#sys.path.append("./resnet50_train/models") +import os +sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)),'../../../../../../utils')) +sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)),'../../../../../../utils/atlasboost')) +base_path=os.path.split(os.path.realpath(__file__))[0] +print ("#########base_path:", base_path) +path_1 = base_path + "/.." +print (path_1) +path_2 = base_path + "/../models" +print (path_2) +path_3 = base_path + "/../../" +print (path_3) + + +sys.path.append(base_path + "/..") +sys.path.append(base_path + "/../models") +sys.path.append(base_path + "/../../") +sys.path.append(base_path + "/../../models") + +from utils import create_session as cs +from utils import logger as lg +from data_loader.resnet50 import data_loader as dl +from models.resnet50 import res50_model as ml +from optimizers import optimizer as op +from losses import res50_loss as ls +from trainers import gpu_base_trainer as tr +# from configs import res50_config as cfg +from hyper_param import hyper_param as hp +from layers import layers as ly + +import argparse +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter + +def main(): + #-------------------choose the config file in .sh file----------- + cmdline = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + cmdline.add_argument('--config_file', default="", + help="""config file used.""") + cmdline.add_argument('--iterations_per_loop', default=1, + help="""config file used.""") + cmdline.add_argument('--max_train_steps', default=200, + help="""config file used.""") + cmdline.add_argument('--debug', default=True, type=ast.literal_eval, + help="""config file used.""") + cmdline.add_argument('--eval', default=False, type=ast.literal_eval, + help="""config file used.""") + cmdline.add_argument('--model_dir', default="./model_dir", + help="""config file used.""") + FLAGS, unknown_args = cmdline.parse_known_args() + if len(unknown_args) > 0: + for bad_arg in unknown_args: + print("ERROR: Unknown command line arg: %s" % bad_arg) + raise ValueError("Invalid command line arg(s)") + + cfg_file = FLAGS.config_file + configs = 'configs' + cfg = getattr(__import__(configs, fromlist=[cfg_file]), cfg_file) + #------------------------------------------------------------------ + + config = cfg.res50_config() + config['iterations_per_loop'] = int(FLAGS.iterations_per_loop) + config['max_train_steps'] = int(FLAGS.max_train_steps) + config['debug'] = FLAGS.debug + config['eval'] = FLAGS.eval + config['model_dir'] = FLAGS.model_dir + print("iterations_per_loop:%d" %(config['iterations_per_loop'])) + print("max_train_steps :%d" %(config['max_train_steps'])) + print("debug :%s" %(config['debug'])) + print("eval :%s" %(config['eval'])) + print("model_dir :%s" %(config['model_dir'])) + Session = cs.CreateSession(config) + data = dl.DataLoader(config) + hyper_param = hp.HyperParams(config) + layers = ly.Layers() + optimizer = op.Optimizer(config) + loss = ls.Loss(config) + logger = lg.LogSessionRunHook(config) # add tensorboard summary + + model = ml.Model(config, data, hyper_param,layers, optimizer, loss, logger) # get the model + trainer = tr.GPUBaseTrain(Session, config, data, model, logger) # use Estimator to build training process + + if config['mode'] =='train': + trainer.train() + if config['eval'] : + trainer.evaluate() + elif config['mode'] =='evaluate': + trainer.evaluate() + elif config['mode'] =='train_and_evaluate': + trainer.train_and_evaluate() + else: + raise ValueError('Invalid type of mode') + +if __name__ == '__main__': + # add zwx5326390 日志打点 + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("tensorflow") + config_info = get_model_parameter("tensorflow_config") + initinal_data = {"base_lr": 0.1, "dataset": "imagenet1024", "optimizer": "SGD", "loss_scale": 512, + "batchsize": 256} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + hwlog.remark_print(key=hwlog.INPUT_BATCH_SIZE, value=initinal_data.get("batchsize")) + main() diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/mains/train_res50.sh b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/mains/train_res50.sh new file mode 100644 index 0000000..bbd2da5 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/mains/train_res50.sh @@ -0,0 +1,21 @@ +#!/bin/bash +#export CUDA_VISIBLE_DEVICES=0 +dir=`pwd` + +#cp -rf ./config /tmp/ +export JOB_ID=10086 +#export PROFILING_DIR=/var/log/npu/profiling/container/0 +export DEVICE_ID=0 +#export PROFILING_MODE=true +export PRINT_MODEL=1 +#export ENABLE_DATA_PRE_PROC=1 +export RANK_ID=0 +export RANK_SIZE=1 +export RANK_TABLE_FILE=/home/lxh/config/new_rank_table_1p.json +export FUSION_TENSOR_SIZE=1000000000 +export PYTHONPATH=${dir} +export LD_LIBRARY_PATH=/usr/local/HiAI/runtime/lib64/ +/usr/local/HiAI/runtime/bin/TdtMain --configfile=/home/lxh/test/config/job_tdt_2p_$DEVICE_ID.json & +sleep 5 + +python3.6 res50.py --config_file res50_baseline diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/mains/train_res50_gpu.sh b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/mains/train_res50_gpu.sh new file mode 100644 index 0000000..c89b78e --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/mains/train_res50_gpu.sh @@ -0,0 +1,4 @@ +#!/bin/bash +export CUDA_VISIBLE_DEVICES=7 + +python3.5 res50.py --config_file res50_baseline_gpu diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/__init__.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/resnet50/res50_helper.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/resnet50/res50_helper.py new file mode 100644 index 0000000..0e16df6 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/resnet50/res50_helper.py @@ -0,0 +1,24 @@ +import tensorflow as tf + +def _fp32_trainvar_getter(getter, name, shape=None, dtype=None, + trainable=True, regularizer=None, + *args, **kwargs): + storage_dtype = dtype + variable = getter(name, shape, dtype=storage_dtype, + trainable=trainable, + regularizer=regularizer if trainable and 'BatchNorm' not in name and 'batchnorm' not in name and 'batch_norm' not in name and 'Batch_Norm' not in name else None, + *args, **kwargs) + + return variable + + +def fp32_trainable_vars(name='fp32_vars', *args, **kwargs): + """A varible scope with custom variable getter to convert fp16 trainable + variables with fp32 storage followed by fp16 cast. + """ + return tf.variable_scope( + name, custom_getter=_fp32_trainvar_getter, *args, **kwargs) + +def custom_getter_with_fp16_and_weight_decay(dtype, weight_decay): + return fp32_trainable_vars(dtype=dtype, regularizer=tf.contrib.layers.l2_regularizer(weight_decay)) + diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/resnet50/res50_model.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/resnet50/res50_model.py new file mode 100644 index 0000000..02e3958 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/resnet50/res50_model.py @@ -0,0 +1,222 @@ + +import tensorflow as tf +from . import resnet, res50_helper +from trainers.train_helper import stage +#from tensorflow.contrib.offline_train.python.npu.npu_optimizer import NPUDistributedOptimizer +from npu_bridge.estimator.npu.npu_optimizer import NPUDistributedOptimizer +#from tensorflow.contrib.offline_train.python import npu_ops +from npu_bridge.estimator import npu_ops +_NUM_EXAMPLES_NAME="num_examples" + + +class Model(object): + def __init__(self, config, data, hyper_param, layers, optimizer, loss, logger): + self.config = config + self.data = data + self.hyper_param = hyper_param + self.layers = layers + self.optimizer = optimizer + self.loss = loss + self.logger = logger + + def get_estimator_model_func(self, features, labels, mode, params=None): + labels = tf.reshape(labels, (-1,)) # Squash unnecessary unary dim #----------------not use when use onehot label + + model_func = self.get_model_func() + inputs = features # TODO: Should be using feature columns? + is_training = (mode == tf.estimator.ModeKeys.TRAIN) + + with tf.device('/gpu:0'): + if self.config['accelerator'] == 'gpu': + inputs = tf.cast(inputs, self.config['dtype']) + + inputs = tf.cast(inputs, self.config['dtype']) + with res50_helper.custom_getter_with_fp16_and_weight_decay(dtype=self.config['dtype'], weight_decay=self.config['weight_decay']): # no BN decay + + top_layer = model_func( + inputs, data_format=self.config['data_format'], training=is_training, + conv_initializer=self.config['conv_init'], + bn_init_mode=self.config['bn_init_mode'], bn_gamma_initial_value=self.config['bn_gamma_initial_value']) + + + logits = top_layer + predicted_classes = tf.argmax(logits, axis=1, output_type=tf.int32) + logits = tf.cast(logits, tf.float32) + + #loss = self.loss.get_loss(logits, labels) + #loss = tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels) + + labels_one_hot = tf.one_hot(labels, depth=1001) + loss = tf.losses.softmax_cross_entropy( + logits=logits, onehot_labels=labels_one_hot, label_smoothing=self.config['label_smoothing']) + + + base_loss = tf.identity(loss, name='loss') # For access by logger (TODO: Better way to access it?) + # base_loss = tf.add_n([loss]) + + def exclude_batch_norm(name): + #return 'batch_normalization' not in name + return 'BatchNorm' not in name + loss_filter_fn = exclude_batch_norm + + # Add weight decay to the loss. + l2_loss = self.config['weight_decay'] * tf.add_n( + # loss is computed using fp32 for numerical stability. + [tf.nn.l2_loss(tf.cast(v, tf.float32)) for v in tf.trainable_variables() + if loss_filter_fn(v.name)]) + #tf.summary.scalar('l2_loss', l2_loss) + # total_loss = base_loss + l2_loss + if self.config['use_lars']: + total_loss = base_loss + else: + total_loss = base_loss + l2_loss + + total_loss = tf.identity(total_loss, name = 'total_loss') + + + if mode == tf.estimator.ModeKeys.EVAL: + with tf.device(None): + metrics = self.layers.get_accuracy( labels, predicted_classes, logits, self.config) + + return tf.estimator.EstimatorSpec( + mode, loss=loss, eval_metric_ops=metrics) + + assert (mode == tf.estimator.ModeKeys.TRAIN) + + #reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) + #total_loss = tf.add_n([tf.saturate_cast(loss, self.config['dtype']) ] + reg_losses, name='total_loss') + #total_loss = tf.add_n([loss], name='total_loss') + + batch_size = tf.shape(inputs)[0] + + global_step = tf.train.get_global_step() + with tf.device('/cpu:0'): + learning_rate = self.hyper_param.get_learning_rate() + + #-----------------------batchsize scaling---------------------------------- + momentum = self.config['momentum'][0] + #------------------------------end------------------------------------------ + + opt = tf.train.MomentumOptimizer( + learning_rate, momentum, use_nesterov=self.config['use_nesterov']) + opt=NPUDistributedOptimizer(opt) + if self.config['accelerator'] == 'gpu': + opt = self.optimizer.get_lbs_optimizer(opt) + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) or [] + with tf.control_dependencies(update_ops): + if self.config['accelerator'] == 'gpu': + gate_gradients = (tf.train.Optimizer.GATE_NONE) + grads_and_vars = opt.compute_gradients(total_loss, gate_gradients=gate_gradients) + train_op = opt.apply_gradients( grads_and_vars,global_step = global_step) + else: + with tf.name_scope('loss_scale'): + loss_scale = float( self.config['loss_scale'] ) + scaled_grads_and_vars = opt.compute_gradients( total_loss * loss_scale ) + unscaled_grads_and_vars = [ (g/loss_scale, v) for g,v in scaled_grads_and_vars ] + + + #-----------------------------------------Lars------------------------------------------ + with tf.name_scope('LARS'): + fp32_grads_and_vars = [ (tf.cast(g, tf.float32), v) for g,v in unscaled_grads_and_vars ] + grad_var_list = [] + + if self.config['use_lars']: + if self.config['accelerator'] == 'gpu': + for g, var in fp32_grads_and_vars: + + if 'BatchNorm' not in var.name and 'bias' not in var.name: + grad_norm = tf.norm(g,ord='euclidean') + weight_norm = tf.norm(var,ord='euclidean') + grad_norm_wd = tf.add( grad_norm, tf.multiply( self.config['weight_decay'] , weight_norm ) ) + rescale_factor = tf.div( tf.multiply(0.001, weight_norm), tf.add(grad_norm_wd, tf.constant(1e-5, tf.float32)) ) + decayed_g = tf.add( g, tf.multiply(self.config['weight_decay'], var ) ) + + with tf.name_scope('lars_grad'): + g = tf.multiply(rescale_factor, decayed_g) + + g_and_v = ( g, var ) + grad_var_list.append( g_and_v ) + + elif self.config['accelerator'] == '1980': + print('lars9999999999999999999999') + g_list_bn_bias = [] + var_list_bn_bias = [] + g_list_else = [] + var_list_else = [] + for g, var in fp32_grads_and_vars: + if 'BatchNorm' not in var.name and 'bias' not in var.name: + g_list_else.append(g) + var_list_else.append(var) + else: + g_list_bn_bias.append(g) + var_list_bn_bias.append(var) + + + g_list_else_lars = npu_ops.LARS(inputs_w=var_list_else, + inputs_g=g_list_else, + weight_decay=self.config['weight_decay'], + hyperpara=0.001, + epsilon=1e-5) + + g_list_lars = g_list_bn_bias + g_list_else_lars + var_list = var_list_bn_bias + var_list_else + + for (g, var) in zip(g_list_lars,var_list): + g_and_v = ( g, var ) + grad_var_list.append( g_and_v ) + + + else: + print('do not use lars111111111111111111') + for g, var in fp32_grads_and_vars: + #if 'BatchNorm' not in var.name and 'bias' not in var.name: + # decayed_g = tf.add( g, tf.multiply( self.config['weight_decay'], var ) ) + # g = decayed_g + g_and_v = ( g, var ) + grad_var_list.append( g_and_v ) + #-----------------------------------------end Lars------------------------------------------ + + + + + train_op = opt.apply_gradients( grad_var_list, global_step = global_step ) + + train_op = tf.group(train_op) + + #with tf.device('/cpu:0'): + #tf.summary.scalar('total_loss', total_loss) + #tf.summary.scalar('base_loss', base_loss) + #tf.summary.scalar('learning_rate', learning_rate) + #tf.contrib.summary.flush() +# if self.config['do_checkpoint']: +# summary_hook = tf.train.SummarySaverHook( save_steps=20, +# output_dir=self.config['log_dir']+'/train_summary', +# summary_op = tf.summary.merge_all() ) + + #return tf.estimator.EstimatorSpec(mode, loss=total_loss, train_op=train_op, training_hooks=[summary_hook] )\ + # if self.config['do_checkpoint'] else tf.estimator.EstimatorSpec(mode, loss=total_loss, train_op=train_op ) + return tf.estimator.EstimatorSpec(mode, loss=total_loss, train_op=train_op ) + + # return tf.estimator.EstimatorSpec(mode, loss=total_loss, train_op=train_op) + + + + def get_model_func(self): + model_name = self.config['model_name'] + if model_name.startswith('resnet'): + nlayer = int(model_name[len('resnet'):]) + return lambda images, *args, **kwargs: \ + resnet.inference_resnet_v1(self.config,images, nlayer, *args, **kwargs) + else: + raise ValueError("Invalid model type: %s" % model_name) + + + + + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/resnet50/res50_model.py_bak b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/resnet50/res50_model.py_bak new file mode 100644 index 0000000..f3465ee --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/resnet50/res50_model.py_bak @@ -0,0 +1,143 @@ +import tensorflow as tf +from . import resnet, res50_helper +from trainers.train_helper import stage +from tensorflow.contrib.offline_train.python.npu.npu_optimizer import NPUDistributedOptimizer +_NUM_EXAMPLES_NAME="num_examples" + + +class Model(object): + def __init__(self, config, data, hyper_param, layers, optimizer, loss, logger): + self.config = config + self.data = data + self.hyper_param = hyper_param + self.layers = layers + self.optimizer = optimizer + self.loss = loss + self.logger = logger + + def get_estimator_model_func(self, features, labels, mode, params=None): + labels = tf.reshape(labels, (-1,)) # Squash unnecessary unary dim + + model_func = self.get_model_func() + inputs = features # TODO: Should be using feature columns? + is_training = (mode == tf.estimator.ModeKeys.TRAIN) + + #if mode == tf.estimator.ModeKeys.TRAIN: + # preload_op, gpucopy_op, inputs, labels = self.data.get_input_pipeline_op( inputs, labels, mode ) + + with tf.device('/gpu:0'): + if self.config['accelerator'] == 'gpu': + inputs = tf.cast(inputs, self.config['dtype']) + + with res50_helper.custom_getter_with_fp16_and_weight_decay(dtype=self.config['dtype'], weight_decay=self.config['weight_decay']): # no BN decay + + top_layer = model_func( + inputs, data_format=self.config['data_format'], training=is_training, + conv_initializer=self.config['conv_init'], + bn_init_mode=self.config['bn_init_mode'], bn_gamma_initial_value=self.config['bn_gamma_initial_value']) + + + logits = top_layer + predicted_classes = tf.argmax(logits, axis=1, output_type=tf.int32) + logits = tf.cast(logits, tf.float32) + + #loss = self.loss.get_loss(logits, labels) + loss = tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels) + base_loss = tf.identity(loss, name='loss') # For access by logger (TODO: Better way to access it?) + # base_loss = tf.add_n([loss]) + + # def exclude_batch_norm(name): + # #return 'batch_normalization' not in name + # return 'BatchNorm' not in name + # loss_filter_fn = exclude_batch_norm + + # Add weight decay to the loss. + # l2_loss = self.config['weight_decay'] * tf.add_n( + # # loss is computed using fp32 for numerical stability. + # [tf.nn.l2_loss(tf.cast(v, tf.float32)) for v in tf.trainable_variables() + # if loss_filter_fn(v.name)]) + # #tf.summary.scalar('l2_loss', l2_loss) + # total_loss = base_loss + l2_loss + total_loss = base_loss + total_loss = tf.identity(total_loss, name = 'total_loss') + + + if mode == tf.estimator.ModeKeys.EVAL: + with tf.device(None): + metrics = self.layers.get_accuracy( labels, predicted_classes, logits ) + + return tf.estimator.EstimatorSpec( + mode, loss=loss, eval_metric_ops=metrics) + + assert (mode == tf.estimator.ModeKeys.TRAIN) + + #reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) + #total_loss = tf.add_n([tf.saturate_cast(loss, self.config['dtype']) ] + reg_losses, name='total_loss') + #total_loss = tf.add_n([loss], name='total_loss') + + batch_size = tf.shape(inputs)[0] + + global_step = tf.train.get_global_step() + with tf.device('/cpu:0'): + learning_rate = self.hyper_param.get_learning_rate() + + #-----------------------batchsize scaling---------------------------------- + momentum = self.config['momentum'][0] + #------------------------------end------------------------------------------ + + opt = tf.train.MomentumOptimizer( + learning_rate, momentum, use_nesterov=False) + optimizer=NPUDistributedOptimizer(opt) + if self.config['accelerator'] == 'gpu': + opt = self.optimizer.get_lbs_optimizer(opt) + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) or [] + with tf.control_dependencies(update_ops): + if self.config['accelerator'] == 'gpu': + gate_gradients = (tf.train.Optimizer.GATE_NONE) + grads_and_vars = opt.compute_gradients(total_loss, gate_gradients=gate_gradients) + train_op = opt.apply_gradients( grads_and_vars,global_step = global_step) + else: + with tf.name_scope('loss_scale'): + loss_scale = float( self.config['loss_scale'] ) + scaled_grads_and_vars = opt.compute_gradients( total_loss * loss_scale ) + unscaled_grads_and_vars = [ (g/loss_scale, v) for g,v in scaled_grads_and_vars ] + train_op = opt.apply_gradients( unscaled_grads_and_vars, global_step = global_step ) + + train_op = tf.group(train_op) + + with tf.device('/cpu:0'): + #tf.summary.scalar('total_loss', total_loss) + tf.summary.scalar('base_loss', base_loss) + tf.summary.scalar('learning_rate', learning_rate) + tf.contrib.summary.flush() + if self.config['do_checkpoint']: + summary_hook = tf.train.SummarySaverHook( save_steps=20, + output_dir=self.config['log_dir']+'/train_summary', + summary_op = tf.summary.merge_all() ) + + return tf.estimator.EstimatorSpec(mode, loss=total_loss, train_op=train_op, training_hooks=[summary_hook] )\ + if self.config['do_checkpoint'] else tf.estimator.EstimatorSpec(mode, loss=total_loss, train_op=train_op ) + + # return tf.estimator.EstimatorSpec(mode, loss=total_loss, train_op=train_op) + + + + def get_model_func(self): + model_name = self.config['model_name'] + if model_name.startswith('resnet'): + nlayer = int(model_name[len('resnet'):]) + return lambda images, *args, **kwargs: \ + resnet.inference_resnet_v1(self.config,images, nlayer, *args, **kwargs) + else: + raise ValueError("Invalid model type: %s" % model_name) + + + + + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/resnet50/resnet.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/resnet50/resnet.py new file mode 100644 index 0000000..6e81f17 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/models/resnet50/resnet.py @@ -0,0 +1,436 @@ +import tensorflow as tf + +_BATCH_NORM_EPSILON = 1e-4 +_BATCH_NORM_DECAY = 0.9 + + +class LayerBuilder(object): + def __init__(self, activation=None, data_format='channels_last', + training=False, use_batch_norm=False, batch_norm_config=None, + conv_initializer=None, bn_init_mode='adv_bn_init', bn_gamma_initial_value=1.0 ): + self.activation = activation + self.data_format = data_format + self.training = training + self.use_batch_norm = use_batch_norm + self.batch_norm_config = batch_norm_config + self.conv_initializer = conv_initializer + self.bn_init_mode = bn_init_mode + self.bn_gamma_initial_value = bn_gamma_initial_value + if self.batch_norm_config is None: + self.batch_norm_config = { + 'decay': _BATCH_NORM_DECAY, + 'epsilon': _BATCH_NORM_EPSILON, + 'scale': True, + 'zero_debias_moving_mean': False, + } + + def _conv2d(self, inputs, activation, *args, **kwargs): + x = tf.layers.conv2d( + inputs, data_format=self.data_format, + # use_bias=not self.use_batch_norm, + use_bias=False, + kernel_initializer=self.conv_initializer, + activation=None if self.use_batch_norm else activation, + *args, **kwargs) + if self.use_batch_norm: + param_initializers = { + 'moving_mean': tf.zeros_initializer(), + 'moving_variance': tf.ones_initializer(), + 'beta': tf.zeros_initializer(), + } + if self.bn_init_mode == 'adv_bn_init': + param_initializers['gamma'] = tf.ones_initializer() + elif self.bn_init_mode == 'conv_bn_init': + param_initializers['gamma'] = tf.constant_initializer(self.bn_gamma_initial_value) + else: + raise ValueError("--bn_init_mode must be 'conv_bn_init' or 'adv_bn_init' ") + + x = self.batch_norm(x) + x = activation(x) if activation is not None else x + return x + + def conv2d_linear_last_bn(self, inputs, *args, **kwargs): + x = tf.layers.conv2d( + inputs, data_format=self.data_format, + use_bias=False, + kernel_initializer=self.conv_initializer, + activation=None, *args, **kwargs) + param_initializers = { + 'moving_mean': tf.zeros_initializer(), + 'moving_variance': tf.ones_initializer(), + 'beta': tf.zeros_initializer(), + } + if self.bn_init_mode == 'adv_bn_init': + param_initializers['gamma'] = tf.zeros_initializer() + elif self.bn_init_mode == 'conv_bn_init': + param_initializers['gamma'] = tf.constant_initializer(self.bn_gamma_initial_value) + else: + raise ValueError("--bn_init_mode must be 'conv_bn_init' or 'adv_bn_init' ") + + x = self.batch_norm(x, param_initializers=param_initializers) + return x + + def conv2d_linear(self, inputs, *args, **kwargs): + return self._conv2d(inputs, None, *args, **kwargs) + + def conv2d(self, inputs, *args, **kwargs): + return self._conv2d(inputs, self.activation, *args, **kwargs) + + def pad2d(self, inputs, begin, end=None): + if end is None: + end = begin + try: + _ = begin[1] + except TypeError: + begin = [begin, begin] + try: + _ = end[1] + except TypeError: + end = [end, end] + if self.data_format == 'channels_last': + padding = [[0, 0], [begin[0], end[0]], [begin[1], end[1]], [0, 0]] + else: + padding = [[0, 0], [0, 0], [begin[0], end[0]], [begin[1], end[1]]] + return tf.pad(inputs, padding) + + def max_pooling2d(self, inputs, *args, **kwargs): + return tf.layers.max_pooling2d( + inputs, data_format=self.data_format, *args, **kwargs) + + def average_pooling2d_stride_1(self, inputs, *args, **kwargs): + # inputs = tf.nn.avg_pool(inputs, ksize=[1,1,1,1],strides=[1,1,1,1], padding="VALID", data_format="NHWC" ) + return inputs + + def average_pooling2d(self, inputs, *args, **kwargs): + inputs = tf.nn.avg_pool(inputs, ksize=[1,2,2,1],strides=[1,2,2,1], padding="VALID", data_format="NHWC" ) + return inputs + + # return tf.layers.average_pooling2d( + # inputs, data_format=self.data_format, *args, **kwargs) + + def dense_linear(self, inputs, units, **kwargs): + return tf.layers.dense(inputs, units, activation=None) + + def dense(self, inputs, units, **kwargs): + return tf.layers.dense(inputs, units, activation=self.activation) + + def activate(self, inputs, activation=None): + activation = activation or self.activation + return activation(inputs) if activation is not None else inputs + + def batch_norm(self, inputs, **kwargs): + all_kwargs = dict(self.batch_norm_config) + all_kwargs.update(kwargs) + data_format = 'NHWC' if self.data_format == 'channels_last' else 'NCHW' + bn_inputs = inputs + outputs = tf.contrib.layers.batch_norm( + inputs, is_training=self.training, data_format=data_format, + fused=True, **all_kwargs) + + return outputs + + def spatial_average2d(self, inputs): + shape = inputs.get_shape().as_list() + if self.data_format == 'channels_last': + n, h, w, c = shape + else: + n, c, h, w = shape + n = -1 if n is None else n + x = tf.layers.average_pooling2d(inputs, (h, w), (1, 1), + data_format=self.data_format) + return tf.reshape(x, [n, c]) + + def flatten2d(self, inputs): + x = inputs + if self.data_format != 'channel_last': + # Note: This ensures the output order matches that of NHWC networks + x = tf.transpose(x, [0, 2, 3, 1]) + input_shape = x.get_shape().as_list() + num_inputs = 1 + for dim in input_shape[1:]: + num_inputs *= dim + return tf.reshape(x, [-1, num_inputs], name='flatten') + + def residual2d(self, inputs, network, units=None, scale=1.0, activate=False): + outputs = network(inputs) + c_axis = -1 if self.data_format == 'channels_last' else 1 + h_axis = 1 if self.data_format == 'channels_last' else 2 + w_axis = h_axis + 1 + ishape, oshape = [y.get_shape().as_list() for y in [inputs, outputs]] + ichans, ochans = ishape[c_axis], oshape[c_axis] + strides = ((ishape[h_axis] - 1) // oshape[h_axis] + 1, + (ishape[w_axis] - 1) // oshape[w_axis] + 1) + with tf.name_scope('residual'): + if (ochans != ichans or strides[0] != 1 or strides[1] != 1): + inputs = self.conv2d_linear(inputs, units, 1, strides, 'SAME') + x = inputs + scale * outputs + if activate: + x = self.activate(x) + return x + + +def resnet_bottleneck_v1(builder, inputs, depth, depth_bottleneck, stride, filters, arch_type, + basic=False): + num_inputs = inputs.get_shape().as_list()[3] + x = inputs + #with tf.name_scope('resnet_model'): + if depth == num_inputs: + if stride == 1:#v1.5 + shortcut = x + else:#v1 + shortcut = builder.max_pooling2d(x, 1, stride) + else: # the downsample(first) block in each layer + if 'D1' in arch_type: + if stride == 1: + shortcut = builder.average_pooling2d_stride_1(x, stride, stride) #--------------------Resnet-D------------ + else: + shortcut = builder.average_pooling2d(x, stride, stride) #--------------------Resnet-D------------ + shortcut = builder.conv2d_linear(shortcut, depth, 1, 1, 'SAME') + elif 'D2' in arch_type: + shortcut = builder.conv2d_linear(x, depth, 3, stride, 'SAME') + elif 'D3' in arch_type: + shortcut = builder.conv2d_linear(x, depth, 1, 1, 'SAME') + shortcut = builder.average_pooling2d(shortcut, stride, stride) #--------------------Resnet-D------------ + else: + shortcut = builder.conv2d_linear(x, depth, 1, stride, 'SAME') + conv_input = x + + if basic: + x = builder.pad2d(x, 1) + x = builder.conv2d(x, depth_bottleneck, 3, stride, 'VALID') + x = builder.conv2d_linear(x, depth, 3, 1, 'SAME') + else: + conv_input = x + x = builder.conv2d(x, depth_bottleneck, 1, 1, 'SAME') + conv_input = x + if stride == 1: + x = builder.conv2d(x, depth_bottleneck, 3, stride, 'SAME') + else: + if 'E1' in arch_type: + x = builder.average_pooling2d( x, stride, stride ) + x = builder.conv2d(x, depth_bottleneck, 3, 1, 'SAME') + elif 'E2' in arch_type: + x = builder.conv2d(x, depth_bottleneck, 3, 1, 'SAME') + if stride == 1: + x = builder.average_pooling2d_stride_1( x, stride, stride ) + else: + x = builder.average_pooling2d( x, stride, stride ) + else: # E0 + x = builder.conv2d(x, depth_bottleneck, 3, stride, 'SAME') + + # x = builder.conv2d_linear(x, depth, 1, 1, 'SAME') + conv_input = x + x = builder.conv2d_linear_last_bn(x, depth, 1, 1, 'SAME') + + x = tf.nn.relu(x + shortcut) + return x + +def resnet_bottleneck_v2(builder, inputs, depth, depth_bottleneck, stride, filters, arch_type, + basic=False): + num_inputs = inputs.get_shape().as_list()[1] + x = inputs + with tf.name_scope('resnet_v1'): + # ------- shortcut --------------- + if depth == num_inputs: + if stride == 1:#v1.5 + shortcut = x + x = builder.batch_norm(x) + x = tf.nn.relu(x) + else:#v1 + shortcut = builder.max_pooling2d(x, 1, stride) + else: # the downsample(first) block in each layer + x = builder.batch_norm(x) + x = tf.nn.relu(x) + + if 'D1' in arch_type: + shortcut = builder.average_pooling2d(x, stride, stride) #--------------------Resnet-D------------ + shortcut = builder.conv2d_linear(shortcut, depth, 1, 1, 'SAME') + elif 'D2' in arch_type: + shortcut = builder.conv2d_linear(x, depth, 3, stride, 'SAME') + elif 'D3' in arch_type: + shortcut = builder.conv2d_linear(x, depth, 1, 1, 'SAME') + shortcut = builder.average_pooling2d(shortcut, stride, stride) #--------------------Resnet-D------------ + else: + shortcut = builder.conv2d_linear(x, depth, 1, stride, 'SAME') + + # -------- mainstream ---------------- + if basic: + x = builder.pad2d(x, 1) + x = builder.conv2d(x, depth_bottleneck, 3, stride, 'VALID') + x = builder.conv2d_linear(x, depth, 3, 1, 'SAME') + else: + x = builder.conv2d(x, depth_bottleneck, 1, 1, 'SAME') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + + if stride == 1: + x = builder.conv2d(x, depth_bottleneck, 3, stride, 'SAME') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + else: + if 'E1' in arch_type: + x = builder.average_pooling2d( x, stride, stride ) + x = builder.conv2d(x, depth_bottleneck, 3, 1, 'SAME') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + elif 'E2' in arch_type: + x = builder.conv2d(x, depth_bottleneck, 3, 1, 'SAME') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + x = builder.average_pooling2d( x, stride, stride ) + else: # E0 + x = builder.conv2d(x, depth_bottleneck, 3, stride, 'SAME') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + + x = builder.conv2d_linear(x, depth, 1, 1, 'SAME') + + + x = x + shortcut + return x + +def inference_resnet_v1_impl(builder, inputs, layer_counts, arch_type='C1+D', resnet_version='v1.5', basic=False): + x = inputs + #x = builder.pad2d(x, 1) + + if 'C1' in arch_type: # --- Resnet C ----- + x = builder.conv2d(x, 32, 3, 2, 'SAME') + x = builder.conv2d(x, 32, 3, 1, 'SAME') + x = builder.conv2d(x, 64, 3, 1, 'SAME') + elif 'C2' in arch_type: + x = builder.conv2d(x, 32, 3, 1, 'SAME') + x = builder.conv2d(x, 32, 3, 2, 'VALID') + x = builder.conv2d(x, 64, 3, 1, 'VALID') + elif 'C3' in arch_type: + x = builder.conv2d(x, 32, 3, 1, 'VALID') + x = builder.conv2d(x, 32, 3, 1, 'VALID') + x = builder.conv2d(x, 64, 3, 2, 'VALID') + else: + x = builder.conv2d(x, 64, 7, 2, 'SAME') + + num_filters=64 + + pooled_inputs = x + #x = builder.max_pooling2d(x, 3, 2, 'SAME') + x, argmax = tf.nn.max_pool_with_argmax(input=x, ksize=(1,3,3,1), strides=(1,2,2,1), padding='SAME') + + for i in range(layer_counts[0]): + x = resnet_bottleneck_v1(builder, x, 256, 64, 1, num_filters, arch_type, basic) + for i in range(layer_counts[1]): + num_filters=num_filters*2 + x = resnet_bottleneck_v1(builder, x, 512, 128, 2 if i == 0 else 1, num_filters, arch_type, basic) + for i in range(layer_counts[2]): + num_filters=num_filters*2 + x = resnet_bottleneck_v1(builder, x, 1024, 256, 2 if i == 0 else 1, num_filters, arch_type, basic) + for i in range(layer_counts[3]): + num_filters=num_filters*2 + x = resnet_bottleneck_v1(builder, x, 2048, 512, 2 if i == 0 else 1, num_filters, arch_type, basic) + + axes = [1,2] + x = tf.reduce_mean( x, axes, keepdims=True ) + x = tf.identity(x, 'final_reduce_mean') + x = tf.reshape( x, [-1, 2048] ) + x = tf.layers.dense(inputs=x, units=1001,kernel_initializer=tf.random_normal_initializer(stddev=0.01)) + x = tf.identity( x, 'final_dense' ) + return x + +def inference_resnet_v2_impl(builder, inputs, layer_counts, arch_type='C1+D', basic=False): + x = inputs + x = builder.pad2d(x, 3) + + if 'C1' in arch_type: # --- Resnet C ----- + x = builder.conv2d(x, 32, 3, 2, 'VALID') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + x = builder.conv2d(x, 32, 3, 1, 'VALID') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + x = builder.conv2d(x, 64, 3, 1, 'SAME') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + elif 'C2' in arch_type: + x = builder.conv2d(x, 32, 3, 1, 'SAME') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + x = builder.conv2d(x, 32, 3, 2, 'VALID') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + x = builder.conv2d(x, 64, 3, 1, 'VALID') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + elif 'C3' in arch_type: + x = builder.conv2d(x, 32, 3, 1, 'VALID') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + x = builder.conv2d(x, 32, 3, 1, 'VALID') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + x = builder.conv2d(x, 64, 3, 2, 'VALID') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + else: + x = builder.conv2d(x, 64, 7, 2, 'VALID') + x = builder.batch_norm(x) + x = tf.nn.relu(x) + + num_filters=64 + + pooled_inputs = x + x = builder.max_pooling2d(x, 3, 2, 'SAME') + + for i in range(layer_counts[0]): + x = resnet_bottleneck_v2(builder, x, 256, 64, 1, num_filters, arch_type, basic) + for i in range(layer_counts[1]): + num_filters=num_filters*2 + x = resnet_bottleneck_v2(builder, x, 512, 128, 2 if i == 0 else 1, num_filters, arch_type, basic) + for i in range(layer_counts[2]): + num_filters=num_filters*2 + x = resnet_bottleneck_v2(builder, x, 1024, 256, 2 if i == 0 else 1, num_filters, arch_type, basic) + for i in range(layer_counts[3]): + num_filters=num_filters*2 + x = resnet_bottleneck_v2(builder, x, 2048, 512, 2 if i == 0 else 1, num_filters, arch_type, basic) + return builder.spatial_average2d(x) + +def inference_resnet_v1(config, inputs, nlayer, data_format='channels_last', + training=False, conv_initializer=None, bn_init_mode='adv_bn_init', bn_gamma_initial_value=1.0 ): + """Deep Residual Networks family of models + https://arxiv.org/abs/1512.03385 + """ + if config['resnet_version'] == 'v1.5': + builder = LayerBuilder(tf.nn.relu, data_format, training, use_batch_norm=True, + conv_initializer=conv_initializer, bn_init_mode=bn_init_mode, bn_gamma_initial_value=bn_gamma_initial_value) + if nlayer == 18: + return inference_resnet_v1_impl(builder, inputs, [2, 2, 2, 2], config['arch_type'], config['resnet_version'], basic=True) + elif nlayer == 34: + return inference_resnet_v1_impl(builder, inputs, [3, 4, 6, 3], config['arch_type'], config['resnet_version'], basic=True) + elif nlayer == 50: + return inference_resnet_v1_impl(builder, inputs, [3, 4, 6, 3], config['arch_type'], config['resnet_version']) + elif nlayer == 101: + return inference_resnet_v1_impl(builder, inputs, [3, 4, 23, 3], config['arch_type'], config['resnet_version']) + elif nlayer == 152: + return inference_resnet_v1_impl(builder, inputs, [3, 8, 36, 3], config['arch_type'], config['resnet_version']) + else: + raise ValueError("Invalid nlayer (%i); must be one of: 18,34,50,101,152" % + nlayer) + + elif config['resnet_version'] == 'v2': + builder = LayerBuilder( None, data_format, training, use_batch_norm=False, + conv_initializer=conv_initializer, bn_init_mode=bn_init_mode, bn_gamma_initial_value=bn_gamma_initial_value) + if nlayer == 18: + return inference_resnet_v2_impl(builder, inputs, [2, 2, 2, 2], config['arch_type'], basic=True) + elif nlayer == 34: + return inference_resnet_v2_impl(builder, inputs, [3, 4, 6, 3], config['arch_type'], basic=True) + elif nlayer == 50: + return inference_resnet_v2_impl(builder, inputs, [3, 4, 6, 3], config['arch_type']) + elif nlayer == 101: + return inference_resnet_v2_impl(builder, inputs, [3, 4, 23, 3], config['arch_type']) + elif nlayer == 152: + return inference_resnet_v2_impl(builder, inputs, [3, 8, 36, 3], config['arch_type']) + else: + raise ValueError("Invalid nlayer (%i); must be one of: 18,34,50,101,152" % + nlayer) + else: + raise ValueError("Invalid resnet version") + + + diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/optimizers/__init__.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/optimizers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/optimizers/optimizer.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/optimizers/optimizer.py new file mode 100644 index 0000000..ce42d6a --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/optimizers/optimizer.py @@ -0,0 +1,228 @@ +import six +import tensorflow as tf + +class Optimizer: + def __init__(self, config): + self.config = config + + def get_lbs_optimizer(self, opt): #TODO input is ( self, hyper_param ) + + # opt = LargeBatchSizeOptimizer(opt, weight_decay=self.config['weight_decay'], + # accum_dtype = self.config['dtype'], + # use_lars = self.config['use_lars'], + # bn_lr_scale = self.config.get('bn_lr_scale', 1.0) + # ) + opt = MixedPrecisionOptimizer(opt, self.config) + + return opt + +class MixedPrecisionOptimizer(tf.train.Optimizer): + """An optimizer that updates trainable variables in fp32.""" + + def __init__(self, optimizer, config): + super(MixedPrecisionOptimizer, self).__init__( + optimizer._use_locking, + optimizer._name + '-MP', + ) + self._optimizer = optimizer + self._config = config + loss_scale=self._config['loss_scale'] + self._loss_scale = float(loss_scale) + self._fp32_to_fp16 = {} + + var_list = ( + tf.trainable_variables() + + tf.get_collection(tf.GraphKeys.TRAINABLE_RESOURCE_VARIABLES)) + with tf.device('/gpu:0'): + self.var_fp32_copy = [ tf.Variable( tf.cast(v.initialized_value(), tf.float32), + dtype=tf.float32, trainable=False, + collections=[tf.GraphKeys.GLOBAL_VARIABLES, "FP32_MASTER_COPIES"] ) for v in var_list ] + + def compute_gradients(self, loss, var_list=None, + gate_gradients=tf.train.Optimizer.GATE_OP, + aggregation_method=None, + colocate_gradients_with_ops=False, + grad_loss=None): + if var_list is None: + var_list = ( + tf.trainable_variables() + + tf.get_collection(tf.GraphKeys.TRAINABLE_RESOURCE_VARIABLES)) + + if self._loss_scale != 1.0: + loss = tf.scalar_mul(self._loss_scale, loss) + + grads_and_vars_fp16 = self._optimizer.compute_gradients( + loss, var_list=var_list, + gate_gradients=gate_gradients, + aggregation_method=aggregation_method, + colocate_gradients_with_ops=colocate_gradients_with_ops, + grad_loss=grad_loss, + ) + # creating FP-32 variables and filling the fp32 dict + grads_and_vars_fp32 = [] + + with tf.variable_scope('FP32-master-copy'): + for i, (grad, var) in enumerate(grads_and_vars_fp16): + if grad is not None: + if var.dtype.base_dtype == tf.float16: + fp32_var = self.var_fp32_copy[i] + self._fp32_to_fp16[fp32_var.name] = var + fp32_grad = tf.cast(grad, tf.float32) + grads_and_vars_fp32.append((fp32_grad, fp32_var)) + else: + grads_and_vars_fp32.append((grad, var)) + else: + grads_and_vars_fp32.append((None, var)) + + grads_and_vars_fp32_rescaled = [ (g/self._loss_scale, v) for g,v in grads_and_vars_fp32 ] + + + return grads_and_vars_fp32_rescaled + + def apply_gradients(self, grads_and_vars, *args, **kwargs): + update_op = self._optimizer.apply_gradients(grads_and_vars, *args, **kwargs) + apply_ops = [] + with tf.control_dependencies([update_op]): + for grad, var in grads_and_vars: + if var.name in self._fp32_to_fp16: + dst_var = self._fp32_to_fp16[var.name] + apply_ops.append( + tf.assign(dst_var, tf.saturate_cast(var, tf.float16))) + if apply_ops: + return tf.group(apply_ops) + return update_op + + +class LargeBatchSizeOptimizer(tf.train.Optimizer): + """ LARC implementation + ------------------- + Parameters: + - optimizer: initial optimizer that you wanna apply + example: tf.train.MomentumOptimizer + - learning_rate: initial learning_rate from initial optimizer + - clip: if True apply LARC otherwise LARS + - epsilon: default value is weights or grads are 0. + - name + - use_locking + """ + + def __init__(self, optimizer, weight_decay, clip=True, epsilon=1., accum_dtype=tf.float16, use_lars=True, bn_lr_scale=1.0, + name="LarcOptimizer", use_locking=False): + super(LargeBatchSizeOptimizer, self).__init__( + name=name, use_locking=use_locking) + self._optimizer = optimizer + # self._learning_rate = learning_rate + self._weight_decay = weight_decay + self._clip = clip + self._epsilon = float(epsilon) + self._accum_dtype=accum_dtype + self._use_lars=use_lars + self._bn_lr_scale=bn_lr_scale + + var_list = ( + tf.trainable_variables() + + tf.get_collection(tf.GraphKeys.TRAINABLE_RESOURCE_VARIABLES)) + with tf.device('/gpu:0'): + self._grads_accum = [ tf.Variable( tf.cast(tf.zeros_like(v.initialized_value()), self._accum_dtype), dtype=self._accum_dtype, trainable=False, collections=[tf.GraphKeys.LOCAL_VARIABLES] ) for v in var_list ] + + + def compute_gradients(self, *args, **kwargs): + return self._optimizer.compute_gradients(*args, **kwargs) + + + def apply_gradients(self, gradvars, loss_scale, *args, **kwargs): + + global_step = tf.train.get_global_step() + + grads_and_vars_clean = [] + for grad, var in gradvars: + if grad is not None: + grads_and_vars_clean.append( (grad, var) ) + + processed_grads_and_vars = self.post_process_grads(grads_and_vars_clean, loss_scale) # post_process_grads includes Lars + + def apply(): + red_grad_updates = self._optimizer.apply_gradients( processed_grads_and_vars, global_step=tf.train.get_global_step() ) + return tf.group(red_grad_updates) + + update_weight_op_1 = apply() + return update_weight_op_1 + + apply_gradients_op = update_weight_op_1 + + with tf.device('/cpu:0'): + #tf.summary.scalar('loss_scale', loss_scale) + for grad, var in gradvars: + g = grad / loss_scale + v_norm_2 = tf.norm(var, ord='euclidean') + g_norm_2 = tf.norm(g, ord='euclidean') + v_g_norm2_ratio = v_norm_2 / ( + g_norm_2 + self._weight_decay * v_norm_2) + if grad is not None: + if 'BatchNorm' in var.name: + with tf.name_scope('bn_norm2/'): + tf.summary.scalar(var.name + '/norm2', + v_norm_2) + with tf.name_scope('grad_bn_norm2/'): + tf.summary.scalar(var.name + '/grad_norm2', + g_norm_2) + with tf.name_scope('bn_ratio_var_grad/'): + tf.summary.scalar(var.name + '/ratio_var_grad', + v_g_norm2_ratio) + else: + with tf.name_scope('conv_norm2/'): + tf.summary.scalar(var.name + '/norm2', + v_norm_2) + with tf.name_scope('grad_conv_norm2/'): + tf.summary.scalar(var.name + '/grad_norm2', + g_norm_2) + with tf.name_scope('conv_ratio_var_grad/'): + tf.summary.scalar(var.name + '/ratio_var_grad', + v_g_norm2_ratio) + + return apply_gradients_op + + def post_process_grads(self, grads_and_vars, loss_scale): + + g_and_v_scaled = [] + for g, v in grads_and_vars: + g = g / loss_scale + g_and_v_scaled.append((g,v)) + + # Lars + if self._use_lars: + grad_var_list = [] + #-----------------------------------------------LARS and weight decay----------------------------------- + for g, var in g_and_v_scaled: + if 'BatchNorm' not in var.name and 'bias' not in var.name: + grad_norm = tf.norm(g,ord='euclidean') + weight_norm = tf.norm(var,ord='euclidean') + + grad_norm_wd = tf.add( grad_norm, tf.multiply( self._weight_decay, weight_norm ) ) + rescale_factor = tf.div( tf.multiply(0.001, weight_norm), tf.add(grad_norm_wd, tf.constant(1e-5, tf.float32)) ) + + coeffi = tf.clip_by_value( rescale_factor, 0.001, 50.0 ) + decayed_g = tf.add( g, tf.multiply( self._weight_decay, var ) ) + + g = tf.multiply(coeffi, decayed_g) + else: + g = self._bn_lr_scale * g + + g_and_v = ( g, var ) + grad_var_list.append( g_and_v ) + #-------------------------------------------LARS end--------------------------------- + return grad_var_list + else: + grad_var_list_without_lars = [] + #----------------------------------------weight decay----------------------------------- + for g, var in g_and_v_scaled: + if 'BatchNorm' not in var.name and 'bias' not in var.name: + decayed_g = tf.add( g, tf.multiply( self._weight_decay, var ) ) + g = decayed_g + else: + g = self._bn_lr_scale * g + + g_and_v = ( g, var ) + grad_var_list_without_lars.append( g_and_v ) + + return grad_var_list_without_lars diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/results/res50_baseline/resnet50.log b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/results/res50_baseline/resnet50.log new file mode 100644 index 0000000..d94c637 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/results/res50_baseline/resnet50.log @@ -0,0 +1,3 @@ +PY3.6.1 (default, Jun 6 2019, 16:37:03) +[GCC 4.8.5 20150623 (EulerOS 4.8.5-28)]TF1.12.0 +Step Epoch Speed Loss FinLoss LR diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/trainers/__init__.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/trainers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/trainers/gpu_base_trainer.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/trainers/gpu_base_trainer.py new file mode 100644 index 0000000..099d4b0 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/trainers/gpu_base_trainer.py @@ -0,0 +1,189 @@ +import tensorflow as tf +import math +import time +from . import train_helper +from .train_helper import stage +from utils.logger import rank0log +from benchmark_log import hwlog +#from tensorflow.contrib.offline_train.python.npu.npu_config import NPURunConfig +from npu_bridge.estimator.npu.npu_config import NPURunConfig +#from tensorflow.contrib.offline_train.python.npu.npu_estimator import NPUEstimator +from npu_bridge.estimator.npu.npu_estimator import NPUEstimator +#from tensorflow.contrib.offline_train.python.npu.npu_optimizer import NPUDistributedOptimizer +from npu_bridge.estimator.npu.npu_optimizer import NPUDistributedOptimizer + +class GPUBaseTrain(object): + def __init__(self, session, config, data, model, logger): + self.sess = session + self.config = config + self.data = data + self.model = model + self.logger = logger + self.print_logger = self.logger.logger + self.all_preds = [] + self.all_targets = [] + if self.config['accelerator'] == 'gpu': + self.classifier, self.training_hook = self.get_classifier() + else: + # from tensorflow.contrib.offline_train.python.npu.npu_config import NPURunConfig + from npu_bridge.estimator.npu.npu_config import NPURunConfig + # from tensorflow.contrib.offline_train.python.npu.npu_estimator import NPUEstimator + from npu_bridge.estimator.npu.npu_estimator import NPUEstimator +# from tensorflow.contrib.offline_train.python.npu.npu_optimizer import NPUDistributedOptimizer + from npu_bridge.estimator.npu.npu_optimizer import NPUDistributedOptimizer + self.classifier, self.training_hook = self.get_npu_classifier() + + + + def get_classifier(self): + classifier = tf.estimator.Estimator( + model_fn=self.model.get_estimator_model_func, + model_dir=self.config['log_dir'], + config = tf.estimator.RunConfig( + session_config=self.sess.get_config(), + save_summary_steps=self.config['save_summary_steps'] if self.config['do_checkpoint'] else None, + save_checkpoints_steps=self.config['save_checkpoints_steps'] if self.config['do_checkpoint'] else None, + keep_checkpoint_max=None + ) + ) + + training_hooks = [train_helper.PrefillStagingAreasHook()] + training_hooks.append(self.logger) + + return classifier, training_hooks + + def get_npu_classifier(self): + session_config = tf.ConfigProto( + inter_op_parallelism_threads=10, + intra_op_parallelism_threads=10, + allow_soft_placement=True,) + + + if self.config['debug'] : + run_config = NPURunConfig(hcom_parallel=True, precision_mode="allow_mix_precision", enable_data_pre_proc=True, save_checkpoints_steps=112590, session_config=session_config, model_dir = self.config['model_dir'], iterations_per_loop=self.config['iterations_per_loop'], keep_checkpoint_max=5) + else : + run_config = NPURunConfig(hcom_parallel=True, precision_mode="allow_mix_precision", save_summary_steps=0, log_step_count_steps=None, enable_data_pre_proc=True,save_checkpoints_secs=1e9, session_config=session_config, model_dir = self.config['model_dir'], iterations_per_loop=self.config['iterations_per_loop']) +# run_config = NPURunConfig(enable_data_pre_proc=True,save_checkpoints_secs=1e9, session_config=session_config, model_dir = self.config['model_dir']) + + # classifier = tf.estimator.Estimator( + # model_fn=self.model.get_estimator_model_func, + # model_dir=self.config['log_dir'], + # config = tf.estimator.RunConfig( + # session_config=self.sess.get_config(), + # save_summary_steps=self.config['save_summary_steps'] if self.config['do_checkpoint'] else None, + # save_checkpoints_steps=self.config['save_checkpoints_steps'] if self.config['do_checkpoint'] else None, + # keep_checkpoint_max=None + # ) + # ) + + classifier =NPUEstimator( + model_fn= self.model.get_estimator_model_func, + config= run_config + ) + + training_hooks = [] + if self.config['debug']: + training_hooks = [train_helper.PrefillStagingAreasHook()] + training_hooks.append(self.logger) + + return classifier, training_hooks + + def train(self): + hwlog.remark_print(key=hwlog.CURRENT_EPOCH, value=self.config['num_epochs']) + print ('training steps: %d' % self.config['nstep']) + self.classifier.train( input_fn=lambda:self.data.get_train_input_fn(), + max_steps = self.config['nstep'], + hooks = self.training_hook + ) + + def evaluate(self): + rank0log(self.print_logger, "Evaluating") + rank0log(self.print_logger, "Validation dataset size: {}".format(self.config['num_evaluating_samples'] )) + time.sleep(5) # a little extra margin... + try: + ckpts = train_helper.sort_and_load_ckpts(self.config['log_dir']) + print("=========ckpt==========") + print(ckpts) + print("=========ckpt==========") + for i, c in enumerate(ckpts): + if i < len(ckpts) - 1: + if i % self.config['eval_interval'] != 0: + continue + eval_result = self.classifier.evaluate( + input_fn=lambda: self.data.get_eval_input_fn(), + checkpoint_path=c['path']) + + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value=float(eval_result.get("val-top1acc"))) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value=float(eval_result.get("val-top5acc"))) + c['epoch'] = math.ceil(c['step'] / (self.config['num_training_samples']/ (self.config['global_batch_size']))) + c['top1'] = eval_result['val-top1acc'] + c['top5'] = eval_result['val-top5acc'] + c['loss'] = eval_result['loss'] + + rank0log(self.print_logger, ' step epoch top1 top5 loss checkpoint_time(UTC)') + for i, c in enumerate(ckpts): + if 'top1' not in c: + continue + rank0log(self.print_logger,'{:5d} {:5.1f} {:5.3f} {:6.2f} {:6.2f} {time}' + .format(c['step'], + c['epoch'], + c['top1'] * 100, + c['top5'] * 100, + c['loss'], + time=time.strftime('%Y-%m-%d %H:%M:%S', + time.localtime(c['mtime'])))) + rank0log(self.print_logger, "Finished evaluation") + except KeyboardInterrupt: + self.print_logger.error("Keyboard interrupt") + + def train_and_evaluate(self): + success = False + epochs_between_evals = self.config.get('epochs_between_evals', 4) + + + for i in range(self.config['num_epochs'] // epochs_between_evals): + + rank0log(self.print_logger, "Starting a training cycle") + # add by zwx5326390 + itepoches = 0 + itepoches += self.config['num_epochs'] + hwlog.remark_print(key=hwlog.CURRENT_EPOCH, value=itepoches) + + self.classifier.train(input_fn=lambda:self.data.get_train_input_fn(), + steps = self.config['nsteps_per_epoch']*epochs_between_evals, + hooks = self.training_hook ) + + rank0log(self.print_logger, "Starting to evaluate") + rank0log(self.print_logger, "Validation dataset size: {}".format(self.config['num_evaluating_samples'] )) + time.sleep(5) # a little extra margin... + + ckpts = train_helper.sort_and_load_ckpts(self.config['log_dir']) + c = ckpts[-1] + eval_result = self.classifier.evaluate( + input_fn=lambda: self.data.get_eval_input_fn(), + checkpoint_path=c['path']) + + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value=float(eval_result.get("val-top1acc"))) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value=float(eval_result.get("val-top5acc"))) + + c['epoch'] = math.ceil(c['step'] / (self.config['num_training_samples']/ (self.config['global_batch_size']))) + c['top1'] = eval_result['val-top1acc'] + c['top5'] = eval_result['val-top5acc'] + c['loss'] = eval_result['loss'] + + rank0log(self.print_logger, ' step epoch top1 top5 loss checkpoint_time(UTC)') + + rank0log(self.print_logger,'{:5d} {:5.1f} {:5.3f} {:6.2f} {:6.2f} {time}' + .format(c['step'], + c['epoch'], + c['top1'] * 100, + c['top5'] * 100, + c['loss'], + time=time.strftime('%Y-%m-%d %H:%M:%S', + time.localtime(c['mtime'])))) + if eval_result['val-top1acc']*100 > self.config.get('stop_threshold', 74.9): + success = True + break + + + diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/trainers/train_helper.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/trainers/train_helper.py new file mode 100644 index 0000000..8d1fa8f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/trainers/train_helper.py @@ -0,0 +1,39 @@ +import tensorflow as tf +from tensorflow.python.ops import data_flow_ops +import re +import os +from operator import itemgetter + +class PrefillStagingAreasHook(tf.train.SessionRunHook): + def after_create_session(self, session, coord): + enqueue_ops = tf.get_collection('STAGING_AREA_PUTS') + for i in range(len(enqueue_ops)): + session.run(enqueue_ops[:i + 1]) + +def stage(tensors): + """Stages the given tensors in a StagingArea for asynchronous put/get. + """ + stage_area = data_flow_ops.StagingArea( + dtypes=[tensor.dtype for tensor in tensors], + shapes=[tensor.get_shape() for tensor in tensors]) + put_op = stage_area.put(tensors) + get_tensors = stage_area.get() + tf.add_to_collection('STAGING_AREA_PUTS', put_op) + return put_op, get_tensors + + +def sort_and_load_ckpts(log_dir): + ckpts = [] + for f in os.listdir(log_dir): + m = re.match(r'model.ckpt-([0-9]+).index', f) + if m is None: + continue + fullpath = os.path.join(log_dir, f) + ckpts.append({'step': int(m.group(1)), + 'path': os.path.splitext(fullpath)[0], + 'mtime': os.stat(fullpath).st_mtime, + }) + ckpts.sort(key=itemgetter('step')) + return ckpts + + diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/utils/__init__.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/utils/create_session.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/utils/create_session.py new file mode 100644 index 0000000..7f2ff17 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/utils/create_session.py @@ -0,0 +1,43 @@ +import tensorflow as tf +import os,sys + +class CreateSession(): + def __init__(self, config): + self.config = config + + if self.config['accelerator'] == '1980': + from tensorflow.python.client import device_lib + from npu_bridge.estimator import npu_ops + # self.estimator_config = tf.ConfigProto(allow_soft_placement=True, min_group_size=20, use_off_line=True) + self.estimator_config = tf.ConfigProto(allow_soft_placement=True) + else: + self.estimator_config = tf.ConfigProto(allow_soft_placement=False) + + self.estimator_config.gpu_options.allow_growth = True + + if self.config['accelerator'] == '1980': + local_device_protos = device_lib.list_local_devices(self.estimator_config) + + self.set_env() + + + def set_env(self): + # TODO, get env from config file + gpu_thread_count = 2 + os.environ['TF_GPU_THREAD_MODE'] = 'gpu_private' + os.environ['TF_GPU_THREAD_COUNT'] = str(gpu_thread_count) + os.environ['TF_USE_CUDNN_BATCHNORM_SPATIAL_PERSISTENT'] = '1' + os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = '1' + + # barrier = self.hvd.allreduce(tf.constant(0, dtype=tf.float32)) + # tf.Session(config=self.estimator_config).run(barrier) + + + def get_config(self): + self.estimator_config.gpu_options.visible_device_list = str(0) +# self.estimator_config.gpu_options.force_gpu_compatible = True # Force pinned memory + self.estimator_config.intra_op_parallelism_threads = 1 # Avoid pool of Eigen threads + self.estimator_config.inter_op_parallelism_threads = 5 + return self.estimator_config + + diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/utils/logger.py b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/utils/logger.py new file mode 100644 index 0000000..349c71d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/code/resnet50_train/utils/logger.py @@ -0,0 +1,87 @@ +from __future__ import print_function +import tensorflow as tf +import logging +import numpy as np +import time +import sys,os +from benchmark_log import hwlog +class LogSessionRunHook(tf.train.SessionRunHook): + def __init__(self, config, warmup_steps=5): + # def __init__(self, global_batch_size, num_records, display_every=10, logger=None): + self.global_batch_size = config['global_batch_size'] + self.iterations_per_loop = config['iterations_per_loop'] + self.warmup_steps = warmup_steps + self.iter_times = [] + self.num_records = config['num_training_samples'] + self.display_every = config['display_every'] + self.logger = get_logger(config['log_name'], config['log_dir']) + rank0log(self.logger, 'PY' + str(sys.version) + 'TF' + str(tf.__version__)) + + + + def after_create_session(self, session, coord): + rank0log(self.logger, 'Step Epoch Speed Loss FinLoss LR') + self.elapsed_secs = 0. + self.count = 0 + + def before_run(self, run_context): + self.t0 = time.time() + return tf.train.SessionRunArgs( + fetches=[tf.train.get_global_step(), 'loss:0', 'total_loss:0', 'learning_rate:0']) +# 'loss:0', 'loss:0', 'learning_rate:0']) + + def after_run(self, run_context, run_values): + batch_time = time.time() - self.t0 + self.iter_times.append(batch_time) + self.elapsed_secs += batch_time + self.count += 1 + global_step, loss, total_loss, lr = run_values.results + if global_step == 1 or global_step % self.display_every == 0: + dt = self.elapsed_secs / self.count + img_per_sec = self.global_batch_size * self.iterations_per_loop / dt + epoch = global_step * self.global_batch_size / self.num_records + self.logger.info('step:%6i epoch:%5.1f FPS:%7.1f loss:%6.3f total_loss:%6.3f lr:%7.5f' % + (global_step, epoch, img_per_sec, loss, total_loss, lr)) + self.elapsed_secs = 0. + self.count = 0 + hwlog.remark_print(key=hwlog.FPS, value='%7.1f' % img_per_sec) + + def get_average_speed(self): + avg_time = np.mean(self.iter_times[self.warmup_steps:]) + speed = self.global_batch_size / avg_time + return speed + + + +def rank0log(logger, *args, **kwargs): + if logger: + logger.info(''.join([str(x) for x in list(args)])) + else: + print(*args, **kwargs) + + +def get_logger(log_name, log_dir): + logger = logging.getLogger(log_name) + logger.setLevel(logging.INFO) # INFO, ERROR + # file handler which logs debug messages + if not os.path.isdir(log_dir): + try: + os.makedirs(log_dir) + except FileExistsError: + # if log_dir is common for multiple ranks like on nfs + pass + # console handler + ch = logging.StreamHandler() + ch.setLevel(logging.INFO) + # add formatter to the handlers + # formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + formatter = logging.Formatter('%(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) + fh = logging.FileHandler(os.path.join(log_dir, log_name)) + fh.setLevel(logging.DEBUG) + fh.setFormatter(formatter) + # add handlers to logger + logger.addHandler(fh) + return logger + diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/config/npu_set_env.sh b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/config/npu_set_env.sh new file mode 100644 index 0000000..bf0d716 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/config/npu_set_env.sh @@ -0,0 +1,41 @@ +# main env +if [ -d /usr/local/Ascend/nnae/latest ];then + + export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/Ascend/driver/tools/hccn_tool/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages + export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp +else + export LD_LIBRARY_PATH=/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/ascend-toolkit/latest//fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/tfplugin/python/site-packages:$projectDir + export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + +fi + +export SOC_VERSION=Ascend910 +export HCCL_CONNECT_TIMEOUT=600 + +# user env +export JOB_ID={JOB_ID} +export RANK_TABLE_FILE={RANK_TABLE_FILE} +#export RANK_SIZE={RANK_SIZE} +#export RANK_INDEX={RANK_INDEX} +#export RANK_ID={RANK_ID} + +# profiling env +export PROFILING_MODE={PROFILING_MODE} +export AICPU_PROFILING_MODE={AICPU_PROFILING_MODE} +export PROFILING_OPTIONS={PROFILING_OPTIONS} +export FP_POINT={FP_POINT} +export BP_POINT={BP_POINT} + +# debug env +#export DUMP_GE_GRAPH=2 +#export DUMP_OP=1 +#export DUMP_OP_LESS=1 +#export PRINT_MODEL=1 +#export TE_PARALLEL_COMPILER=0 + +# system env +ulimit -c unlimited diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/scripts/run.sh b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/scripts/run.sh new file mode 100644 index 0000000..9226a23 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/scripts/run.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +rank_size=$1 +yamlPath=$2 +toolsPath=$3 +currentDir=$(cd "$(dirname "$0")/.."; pwd) +if [ -f /.dockerenv ];then + CLUSTER=$4 + MPIRUN_ALL_IP="$5" + export CLUSTER=${CLUSTER} +fi +#export RANK_ID=npu${rank_size}p + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "tensorflow_config") + +data_url_new=`echo ${data_url//\//\\\\/}` +echo ${data_url} +echo ${max_steps} +echo ${epoches} +if [ x"${CLUSTER}" == x"True" ];then + jsonFilePath=${currentDir}/code/resnet50_train/configs/res50_256bs_16p.py +elif [ ${rank_size} -le 1 ];then + jsonFilePath=${currentDir}/code/resnet50_train/configs/res50_256bs_1p.py +elif [ ${rank_size} -le 2 ];then + jsonFilePath=${currentDir}/code/resnet50_train/configs/res50_256bs_2p.py +elif [ ${rank_size} -le 4 ];then + jsonFilePath=${currentDir}/code/resnet50_train/configs/res50_256bs_4p.py +else + jsonFilePath=${currentDir}/code/resnet50_train/configs/res50_256bs_8p.py +fi + +#echo "jsonfilepath is "${jsonFilePath} +sed -i "s/data_url.*$/data_url\': \'${data_url_new}\',/g" ${jsonFilePath} +#sed -i "s/max_train_steps.*$/max_train_steps\': ${max_steps},/g" ${jsonFilePath} +sed -i "s/num_epochs.*$/num_epochs\': ${epoches},/g" ${jsonFilePath} +sed -i "s/epochs_between_evals.*$/epochs_between_evals\': ${epochs_between_evals},/g" ${jsonFilePath} +sed -i "0,/mode.*$/s//mode\': \'${mode}\',/g" ${jsonFilePath} +sed -i 's/\r//g' ${jsonFilePath} +sed -i "0,/batch_size.*$/s//batch_size\': ${batch_size},/" ${jsonFilePath} + +currtime=`date +%Y%m%d%H%M%S` +mkdir -p ${currentDir%train*}/train/result/tf_resnet50_hc/training_job_${currtime}/ +train_job_dir=${currentDir%train*}/train/result/tf_resnet50_hc/training_job_${currtime}/ +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir} &" +# device 列表, 若无指定 device 或大于等于 8p 时根据 rank_size 顺序选择 +eval device_group=\$device_group_${rank_size}p +if [ x"${device_group}" == x"" ] || [ ${rank_size} -ge 8 ];then + device_group="$(seq 0 "$(expr $rank_size - 1)")" +fi + +# get last device id in device_group, hw log in performance from the dir named last_device_id +device_group_str=`echo ${device_group} | sed 's/ //g'` +first_device_id=`echo ${device_group_str: 0:1}` + +rank_id=0 + +if [ x"${CLUSTER}" == x"True" ];then + # ln hw log + ln -snf ${currentDir%train*}/train/result/tf_resnet50_hc/training_job_${currtime}/0/hw_resnet50_hc.log ${currentDir%train*}/train/result/tf_resnet50_hc/training_job_${currtime}/ + this_ip=$(hostname -I |awk '{print $1}') + for ip in $MPIRUN_ALL_IP;do + if [ x"$this_ip" != x"$ip" ];then + scp $yamlPath root@$ip:$yamlPath + scp $jsonFilePath root@$ip:$jsonFilePath + fi + done + export PATH=$PATH:/usr/local/mpirun4.0/bin + mpirun -H ${mpirun_ip} \ + --bind-to none -map-by slot\ + --allow-run-as-root \ + --mca btl_tcp_if_exclude lo,docker0,endvnic,virbr0,vethf40501b,docker_gwbridge,br-f42ac38052b4\ + --prefix /usr/local/mpirun4.0/ \ + ${currentDir}/scripts/train.sh 0 $rank_size $yamlPath $currtime ${toolsPath} ${CLUSTER} +else + # ln hw log + ln -snf ${currentDir%train*}/train/result/tf_resnet50_hc/training_job_${currtime}/0/hw_resnet50_hc.log ${currentDir%train*}/train/result/tf_resnet50_hc/training_job_${currtime}/ + for device_id in $device_group;do + #echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] start: train ${device_id} & " >> ${currentDir}/result/main.log + ${currentDir}/scripts/train.sh $device_id $rank_size $yamlPath $currtime ${toolsPath} $rank_id& + let rank_id++ + done +fi +wait + + diff --git a/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/scripts/train.sh b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/scripts/train.sh new file mode 100644 index 0000000..3974d99 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/Resnet50_HC/tensorflow/scripts/train.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +device_id=$1 +rank_size=$2 +yamlPath=$3 +currtime=$4 +toolsPath=$5 +currentDir=$(cd "$(dirname "$0")/.."; pwd) +mkdir -p ${currentDir%train*}/train/result/tf_resnet50_hc/training_job_${currtime}/ +export train_job_dir=${currentDir%train*}/train/result/tf_resnet50_hc/training_job_${currtime}/ + + +source ${currentDir}/config/npu_set_env.sh + +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "tensorflow_config") + +# 声明变量 +export REMARK_LOG_FILE=hw_resnet50_hc.log # 打点日志文件名称， 必须hw_后跟模型名称小写 +# 添加日志打点模块路径 +benchmark_log_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils +export PYTHONPATH=$PYTHONPATH:${benchmark_log_path} + + + +# user env +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 +export RANK_TABLE_FILE=${currentDir}/config/${rank_size}p.json +export RANK_SIZE=${rank_size} +export RANK_INDEX=0 +export SLOG_PRINT_TO_STDOUT=0 +export DEVICE_ID=$1 +DEVICE_INDEX=$(( DEVICE_ID + RANK_INDEX * 8 )) +export DEVICE_INDEX=${DEVICE_INDEX} +export YAML_PATH=$3 +export MODEL_CKPT_PATH=${currentDir}/result/ckpt${device_id} + +cd ${train_job_dir} +curd_dir=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +export PYTHONPATH=$PYTHONPATH:${curd_dir} + +if [ x"$6" != x"True" ];then + rank_id=$6 + export RANK_ID=$6 +else + device_id_mo=$(python3.7 -c "import src.tensorflow.mpi_ops as atlasboost;atlasboost.init(); \ + device_id = atlasboost.local_rank();cluster_device_id = str(device_id); \ + atlasboost.set_device_id(device_id);print(atlasboost.rank())") + device_id_mo=`echo $device_id_mo` + rank_id=${device_id_mo##* } + export RANK_ID=${rank_id} + device=${device_id_mo##*deviceid = } + device_id=${device%% phyid=*} + export DEVICE_ID=${device_id} + hccljson=${train_job_dir}/*.json + cp ${hccljson} ${currentDir}/config/${rank_size}p.json +fi + +#mkdir exec path +mkdir -p ${train_job_dir}/${device_id} +cd ${train_job_dir}/${device_id} + +startTime=`date +%Y%m%d-%H:%M:%S` +startTime_s=`date +%s` +#cd ${currentDir}/code +# 根据单卡/多卡区分调用参数 +if [ x"$6" == x"True" ];then + export CLUSTER=True + # 多卡多机 + rm -rf ${currentDir}/result/*.log + rm -rf ${currentDir}/code/core.* + python3.7 ${currentDir}/code/resnet50_train/mains/res50.py --config_file=res50_256bs_16p --max_train_steps=${max_steps} --iterations_per_loop=1000 --debug=True --eval=True --model_dir=./ckpt > ${train_job_dir}/train_${device_id}.log 2>&1 + +elif [ ${rank_size} -le 1 ];then + # 单卡 + + python3.7 ${currentDir}/code/resnet50_train/mains/res50.py --config_file=res50_256bs_1p --max_train_steps=${max_steps} --iterations_per_loop=1000 --debug=True --eval=False --model_dir=./ckpt > ${train_job_dir}/train_${device_id}.log 2>&1 +elif [ ${rank_size} -le 2 ];then + # 单卡 + python3.7 ${currentDir}/code/resnet50_train/mains/res50.py --config_file=res50_256bs_2p --max_train_steps=${max_steps} --iterations_per_loop=1000 --debug=True --eval=False --model_dir=./ckpt > ${train_job_dir}/train_${device_id}.log 2>&1 +elif [ ${rank_size} -le 4 ];then + # 单卡 + python3.7 ${currentDir}/code/resnet50_train/mains/res50.py --config_file=res50_256bs_4p --max_train_steps=${max_steps} --iterations_per_loop=1000 --debug=True --eval=False --model_dir=./ckpt > ${train_job_dir}/train_${device_id}.log 2>&1 +elif [ ${rank_size} -le 8 ];then + # 多卡单机 + python3.7 ${currentDir}/code/resnet50_train/mains/res50.py --config_file=res50_256bs_8p --max_train_steps=${max_steps} --iterations_per_loop=1000 --debug=True --eval=True --model_dir=./ckpt > ${train_job_dir}/train_${device_id}.log 2>&1 +fi +#cp ./hw_resnet50_hc.log ${currentDir}/../../../../performance/ + +if [ $? -eq 0 ] ;then + echo ":::ABK 1.0.0 resnet50_hc train success" + echo ":::ABK 1.0.0 resnet50_hc train success" >> ${train_job_dir}/${device_id}/train_${device_id}.log + echo ":::ABK 1.0.0 resnet50_hc train success" >> ${train_job_dir}/${device_id}/hw_resnet50_hc.log +else + echo ":::ABK 1.0.0 resnet50_hc train failed" + echo ":::ABK 1.0.0 resnet50_hc train failed" >> ${train_job_dir}/${device_id}/train_${device_id}.log + echo ":::ABK 1.0.0 resnet50_hc train failed" >> ${train_job_dir}/${device_id}/hw_resnet50_hc.log +fi + +endTime=`date +%Y%m%d-%H:%M:%S` +endTime_s=`date +%s` +sumTime=$[ $endTime_s - $startTime_s ] +hour=$(( $sumTime/3600 )) +min=$(( ($sumTime-${hour}*3600)/60 )) +sec=$(( $sumTime-${hour}*3600-${min}*60 )) +echo ${hour}:${min}:${sec} +echo ":::ABK 1.0.0 resnet50_hc train total time ${hour}:${min}:${sec}" >> ${train_job_dir}/${device_id}/hw_resnet50_hc.log + + diff --git a/train/atlas_benchmark-master/image_classification/ShuffleNet/__init__.py b/train/atlas_benchmark-master/image_classification/ShuffleNet/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/README.md b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/README.md new file mode 100644 index 0000000..7cfc001 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/README.md @@ -0,0 +1,35 @@ +# ShuffleNet_pytorch训练说明 + +### 1. 模型训练参数配置 + +在train/yaml/ShuffleNet.yaml中修改相应配置， 配置项含义: + +``` +pytorch_config: + # 基本参数 + data_url: /home/imagenet/ + #跑1p时batch_size为1024, 2p为2048，4p为4096，8p时为8196 + batch_size: 1024 + epoches: 240 + epochs_between_evals: 5 + # 默认参数1p 0.5 2p 1 4p 2 8p 4 + lr: 0.5 + + # docker 镜像名称:版本号 + docker_image: c73:b02 + + # 指定 device id, 多个 id 使用空格分隔, 数量需与 rank_size 相同，目前，只有shufflenet 的1p支持指定device id,多p不支持指定device id。 + device_group_1p: 0 + device_group_2p: 0 1 + device_group_4p: 0 1 2 3 +``` + +------ + + + + + + + + diff --git a/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/8p_main.py b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/8p_main.py new file mode 100644 index 0000000..75c3fde --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/8p_main.py @@ -0,0 +1,541 @@ +# -*- coding: utf-8 -*- + +import argparse +import os +import random +import shutil +import time +import warnings + +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.distributed as dist +import torch.optim +import torch.multiprocessing as mp +import torch.utils.data +import torch.utils.data.distributed +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import models as models +import numpy as np + +from apex import amp +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter + +BATCH_SIZE = 512 +OPTIMIZER_BATCH_SIZE = 2048 +model_names = sorted(name for name in models.__dict__ + if name.islower() and not name.startswith("__") + and callable(models.__dict__[name])) + +parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') +parser.add_argument('--data', metavar='DIR', default='/opt/npu/dataset/imagenet', + help='path to dataset') +parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet50', + help='model architecture: ' + + ' | '.join(model_names) + + ' (default: resnet18)') +parser.add_argument('-j', '--workers', default=32, type=int, metavar='N', + help='number of data loading workers (default: 4)') +parser.add_argument('--epochs', default=90, type=int, metavar='N', + help='number of total epochs to run') +parser.add_argument('--start-epoch', default=0, type=int, metavar='N', + help='manual epoch number (useful on restarts)') +parser.add_argument('-b', '--batch-size', default=BATCH_SIZE, type=int, + metavar='N', + help='mini-batch size (default: 256), this is the total ' + 'batch size of all GPUs on the current node when ' + 'using Data Parallel or Distributed Data Parallel') +parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, + metavar='LR', help='initial learning rate', dest='lr') +parser.add_argument('--momentum', default=0.9, type=float, metavar='M', + help='momentum') +parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, + metavar='W', help='weight decay (default: 1e-4)', + dest='weight_decay') +parser.add_argument('--workspace', type=str, default='./', metavar='DIR', + help='path to directory where checkpoints will be stored') +parser.add_argument('-p', '--print-freq', default=10, type=int, + metavar='N', help='print frequency (default: 10)') +parser.add_argument('-ef', '--eval-freq', default=5, type=int, + metavar='N', help='evaluate frequency (default: 5)') +parser.add_argument('--resume', default='', type=str, metavar='PATH', + help='path to latest checkpoint (default: none)') +parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', + help='evaluate model on validation set') +parser.add_argument('--pretrained', dest='pretrained', action='store_true', + help='use pre-trained model') +parser.add_argument('--world-size', default=-1, type=int, + help='number of nodes for distributed training') +parser.add_argument('--rank', default=-1, type=int, + help='node rank for distributed training') +parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, + help='url used to set up distributed training') +parser.add_argument('--dist-backend', default='nccl', type=str, + help='distributed backend') +parser.add_argument('--seed', default=None, type=int, + help='seed for initializing training. ') +parser.add_argument('--gpu', default=None, type=int, + help='GPU id to use.') +parser.add_argument('--multiprocessing-distributed', action='store_true', + help='Use multi-processing distributed training to launch ' + 'N processes per node, which has N GPUs. This is the ' + 'fastest way to use PyTorch for either single node or ' + 'multi node data parallel training') +parser.add_argument('-bm', '--benchmark', default=0, type=int, + metavar='N', help='set benchmark status (default: 1,run benchmark)') +parser.add_argument('--device', default='npu', type=str, + help='npu or gpu') +parser.add_argument('--addr', default='10.136.181.115', type=str, + help='master addr') +parser.add_argument('--checkpoint-nameprefix', default='checkpoint', type=str, + help='checkpoint-nameprefix') +parser.add_argument('--checkpoint-freq', default=0, type=int, + metavar='N', help='checkpoint frequency (default: 0)' + '0: save only one file whitch per epoch;' + 'n: save diff file per n epoch' + '-1:no checkpoint,not support') +parser.add_argument('--device_num', default=-1, type=int, + help='device_num') +parser.add_argument('--warm_up_epochs', default=0, type=int, + help='warm up') + +# apex +parser.add_argument('--amp', default=False, action='store_true', + help='use amp to train the model') +parser.add_argument('--loss-scale', default=64., type=float, + help='loss scale using in amp, default -1 means dynamic') +parser.add_argument('--opt-level', default='O2', type=str, + help='loss scale using in amp, default -1 means dynamic') + +warnings.filterwarnings('ignore') +best_acc1 = 0 + + +def main(): + args = parser.parse_args() + print("===============main()=================") + print(args) + print("===============main()=================") + + os.environ['KERNEL_NAME_ID'] = str(0) + print("+++++++++++++++++++++++++++KERNEL_NAME_ID:", os.environ['KERNEL_NAME_ID']) + + if args.seed is not None: + random.seed(args.seed) + torch.manual_seed(args.seed) + cudnn.deterministic = True + warnings.warn('You have chosen to seed training. ' + 'This will turn on the CUDNN deterministic setting, ' + 'which can slow down your training considerably! ' + 'You may see unexpected behavior when restarting ' + 'from checkpoints.') + + os.environ['MASTER_ADDR'] = args.addr # '10.136.181.51' + os.environ['MASTER_PORT'] = '29688' + + if args.gpu is not None: + warnings.warn('You have chosen a specific GPU. This will completely ' + 'disable data parallelism.') + + if args.dist_url == "env://" and args.world_size == -1: + args.world_size = int(os.environ["WORLD_SIZE"]) + + args.distributed = args.world_size > 1 or args.multiprocessing_distributed + + if args.device_num != -1: + ngpus_per_node = args.device_num + elif args.device == 'npu': + ngpus_per_node = torch.npu.device_count() + else: + ngpus_per_node = torch.cuda.device_count() + if args.multiprocessing_distributed: + # Since we have ngpus_per_node processes per node, the total world_size + # needs to be adjusted accordingly + args.world_size = ngpus_per_node * args.world_size + # Use torch.multiprocessing.spawn to launch distributed processes: the + # main_worker process function + # The child process uses the environment variables of the parent process, + # we have to set KERNEL_NAME_ID for every proc + mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) + + else: + # Simply call main_worker function + main_worker(args.gpu, ngpus_per_node, args) + + +def main_worker(gpu, ngpus_per_node, args): + global best_acc1 + args.gpu = gpu + print("[npu id:", args.gpu, "]", "+++++++++++++++++++++++++++ before set KERNEL_NAME_ID:", + os.environ['KERNEL_NAME_ID']) + os.environ['KERNEL_NAME_ID'] = str(gpu) + print("[npu id:", args.gpu, "]", "+++++++++++++++++++++++++++KERNEL_NAME_ID:", os.environ['KERNEL_NAME_ID']) + + if args.gpu is not None: + print("[npu id:", args.gpu, "]", "Use GPU: {} for training".format(args.gpu)) + + if args.distributed: + if args.dist_url == "env://" and args.rank == -1: + args.rank = int(os.environ["RANK"]) + if args.multiprocessing_distributed: + # For multiprocessing distributed training, rank needs to be the + # global rank among all the processes + args.rank = args.rank * ngpus_per_node + gpu + + if args.device == 'npu': + dist.init_process_group(backend=args.dist_backend, # init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + else: + dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + + loc = 'npu:{}'.format(args.gpu) + torch.npu.set_device(loc) + + args.batch_size = int(args.batch_size / ngpus_per_node) + args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) + + print("[npu id:", args.gpu, "]", "===============main_worker()=================") + print("[npu id:", args.gpu, "]", args) + print("[npu id:", args.gpu, "]", "===============main_worker()=================") + + # Data loading code + traindir = os.path.join(args.data, 'train') + valdir = os.path.join(args.data, 'val') + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + + train_dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ])) + + if args.distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), + num_workers=args.workers, pin_memory=True, sampler=train_sampler, drop_last=True) + + val_loader = torch.utils.data.DataLoader( + datasets.ImageFolder(valdir, transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + normalize, + ])), + batch_size=args.batch_size, shuffle=True, + num_workers=args.workers, pin_memory=True, drop_last=True) + + # create model + print("[npu id:", args.gpu, "]", "=> creating model '{}'".format(args.arch)) + model = models.__dict__[args.arch]() + # model = densenet121() + model = model.to(loc) + + # define loss function (criterion) and optimizer + criterion = nn.CrossEntropyLoss().to(loc) + optimizer = torch.optim.SGD(model.parameters(), args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay) + + if args.amp: + model, optimizer = amp.initialize(model, optimizer, opt_level=args.opt_level, loss_scale=args.loss_scale) + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], broadcast_buffers=False) + + # optionally resume from a checkpoint + if args.resume: + if os.path.isfile(args.resume): + print("=> loading checkpoint '{}'".format(args.resume)) + checkpoint = torch.load(args.resume, map_location=loc) + args.start_epoch = checkpoint['epoch'] + best_acc1 = checkpoint['best_acc1'] + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + if args.amp: + amp.load_state_dict(checkpoint['amp']) + print("=> loaded checkpoint '{}' (epoch {})" + .format(args.resume, checkpoint['epoch'])) + else: + print("=> no checkpoint found at '{}'".format(args.resume)) + + cudnn.benchmark = True + + if args.evaluate: + validate(val_loader, model, criterion, args) + return + + for epoch in range(args.start_epoch, args.epochs): + if args.distributed: + train_sampler.set_epoch(epoch) + adjust_learning_rate(optimizer, epoch, args) + + # train for one epoch + train(train_loader, model, criterion, optimizer, epoch, args, ngpus_per_node) + + if (epoch + 1) % (args.eval_freq) == 0 or epoch == args.epochs - 1: + # evaluate on validation set + acc1 = validate(val_loader, model, criterion, args, ngpus_per_node) + + # remember best acc@1 and save checkpoint + is_best = acc1 > best_acc1 + best_acc1 = max(acc1, best_acc1) + + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0 or epoch == args.epochs - 1): + if args.amp: + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer': optimizer.state_dict(), + 'amp': amp.state_dict(), + }, is_best) + else: + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer': optimizer.state_dict(), + }, is_best) + + +def train(train_loader, model, criterion, optimizer, epoch, args, ngpus_per_node): + batch_time = AverageMeter('Time', ':6.3f') + data_time = AverageMeter('Data', ':6.3f') + losses = AverageMeter('Loss', ':.4e', start_count_index=0) + top1 = AverageMeter('Acc@1', ':6.2f', start_count_index=0) + top5 = AverageMeter('Acc@5', ':6.2f', start_count_index=0) + progress = ProgressMeter( + len(train_loader), + [batch_time, data_time, losses, top1, top5], + prefix="Epoch: [{}]".format(epoch)) + + # switch to train mode + model.train() + end = time.time() + if args.benchmark == 1: + optimizer.zero_grad() + for i, (images, target) in enumerate(train_loader): + # measure data loading time + data_time.update(time.time() - end) + + loc = 'npu:{}'.format(args.gpu) + target = target.to(torch.int32) + images, target = images.to(loc, non_blocking=False), target.to(loc, non_blocking=False) + + # compute output + output = model(images) + + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # compute gradient and do SGD step + if args.benchmark == 0: + optimizer.zero_grad() + + if args.amp: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + + if args.benchmark == 0: + optimizer.step() + elif args.benchmark == 1: + BATCH_SIZE_multiplier = int(OPTIMIZER_BATCH_SIZE / args.batch_size) + BM_optimizer_step = ((i + 1) % BATCH_SIZE_multiplier) == 0 + if BM_optimizer_step: + for param_group in optimizer.param_groups: + for param in param_group['params']: + param.grad /= BATCH_SIZE_multiplier + optimizer.step() + optimizer.zero_grad() + + if i % args.print_freq == 0: + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + progress.display(i) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + print("[npu id:", args.gpu, "]", '* FPS@all {:.3f}'.format(ngpus_per_node * args.batch_size / batch_time.avg)) + hwlog.remark_print(key=hwlog.FPS, value=' * FPS@all {:.3f}'.format(ngpus_per_node * args.batch_size / batch_time.avg)) + + +def validate(val_loader, model, criterion, args, ngpus_per_node): + batch_time = AverageMeter('Time', ':6.3f', start_count_index=0) + losses = AverageMeter('Loss', ':.4e', start_count_index=0) + top1 = AverageMeter('Acc@1', ':6.2f', start_count_index=0) + top5 = AverageMeter('Acc@5', ':6.2f', start_count_index=0) + progress = ProgressMeter( + len(val_loader), + [batch_time, losses, top1, top5], + prefix='Test: ') + + # switch to evaluate mode + model.eval() + + with torch.no_grad(): + end = time.time() + for i, (images, target) in enumerate(val_loader): + + loc = 'npu:{}'.format(args.gpu) + target = target.to(torch.int32) + images, target = images.to(loc, non_blocking=False), target.to(loc, non_blocking=False) + + # compute output + output = model(images) + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + progress.display(i) + + # TODO: this should also be done with the ProgressMeter + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + print("[npu id:", args.gpu, "]", '[AVG-ACC] * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}' + .format(top1=top1, top5=top5)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value="{top1.avg:.3f}".format(top1=top1)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value="{top5.avg:.3f}".format(top5=top5)) + + return top1.avg + + +def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): + torch.save(state, filename) + if is_best: + shutil.copyfile(filename, 'model_best_acc%.4f_epoch%d.pth.tar' % (state['best_acc1'], state['epoch'])) + + +class AverageMeter(object): + """Computes and stores the average and current value""" + + def __init__(self, name, fmt=':f', start_count_index=10): + self.name = name + self.fmt = fmt + self.reset() + self.start_count_index = start_count_index + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.count += n + if self.count > (self.start_count_index * n): + self.sum += val * n + self.avg = self.sum / (self.count - self.start_count_index * n) + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + +class ProgressMeter(object): + def __init__(self, num_batches, meters, prefix=""): + self.batch_fmtstr = self._get_batch_fmtstr(num_batches) + self.meters = meters + self.prefix = prefix + + def display(self, batch): + entries = [self.prefix + self.batch_fmtstr.format(batch)] + entries += [str(meter) for meter in self.meters] + print("[npu id:", os.environ['KERNEL_NAME_ID'], "]", '\t'.join(entries)) + + def _get_batch_fmtstr(self, num_batches): + num_digits = len(str(num_batches // 1)) + fmt = '{:' + str(num_digits) + 'd}' + return '[' + fmt + '/' + fmt.format(num_batches) + ']' + + +def adjust_learning_rate(optimizer, epoch, args): + """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" + # lr = args.lr * (0.1 ** (epoch // (args.epochs//3 - 3))) + + if args.warm_up_epochs > 0 and epoch < args.warm_up_epochs: + lr = args.lr * ((epoch+1) / (args.warm_up_epochs+1)) + else: + alpha = 0 + cosine_decay = 0.5 * ( + 1 + np.cos(np.pi * (epoch - args.warm_up_epochs) / (args.epochs - args.warm_up_epochs))) + decayed = (1 - alpha) * cosine_decay + alpha + lr = args.lr * decayed + + print("=> Epoch[%d] Setting lr: %.4f" % (epoch, lr)) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + + +def accuracy(output, target, topk=(1,)): + """Computes the accuracy over the k top predictions for the specified values of k""" + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +if __name__ == '__main__': + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("pytorch") + config_info = get_model_parameter("pytorch_config") + initinal_data = {"base_lr": 4, "dataset": "imagenet", "optimizer": "SGD", "loss_scale": 64} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + main() + diff --git a/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/8p_main_med.py b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/8p_main_med.py new file mode 100644 index 0000000..674f36c --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/8p_main_med.py @@ -0,0 +1,605 @@ +# -*- coding: utf-8 -*- + +import argparse +import os +import random +import shutil +import time +import warnings + +import numpy as np + +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.distributed as dist +import torch.optim +import torch.multiprocessing as mp +import torch.utils.data +import torch.utils.data.distributed +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import models as models +import numpy as np + +from apex import amp +from multi_epochs_dataloader import MultiEpochsDataLoader +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter + +BATCH_SIZE = 512 +OPTIMIZER_BATCH_SIZE = 2048 +model_names = sorted(name for name in models.__dict__ + if name.islower() and not name.startswith("__") + and callable(models.__dict__[name])) + +parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') +parser.add_argument('--data', metavar='DIR', default='/opt/npu/dataset/imagenet', + help='path to dataset') +parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet50', + help='model architecture: ' + + ' | '.join(model_names) + + ' (default: resnet18)') +parser.add_argument('-j', '--workers', default=32, type=int, metavar='N', + help='number of data loading workers (default: 4)') +parser.add_argument('--epochs', default=90, type=int, metavar='N', + help='number of total epochs to run') +parser.add_argument('--start-epoch', default=0, type=int, metavar='N', + help='manual epoch number (useful on restarts)') +parser.add_argument('-b', '--batch-size', default=BATCH_SIZE, type=int, + metavar='N', + help='mini-batch size (default: 256), this is the total ' + 'batch size of all GPUs on the current node when ' + 'using Data Parallel or Distributed Data Parallel') +parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, + metavar='LR', help='initial learning rate', dest='lr') +parser.add_argument('--momentum', default=0.9, type=float, metavar='M', + help='momentum') +parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, + metavar='W', help='weight decay (default: 1e-4)', + dest='weight_decay') +parser.add_argument('--workspace', type=str, default='./', metavar='DIR', + help='path to directory where checkpoints will be stored') +parser.add_argument('-p', '--print-freq', default=10, type=int, + metavar='N', help='print frequency (default: 10)') +parser.add_argument('-ef', '--eval-freq', default=5, type=int, + metavar='N', help='evaluate frequency (default: 5)') +parser.add_argument('--resume', default='', type=str, metavar='PATH', + help='path to latest checkpoint (default: none)') +parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', + help='evaluate model on validation set') +parser.add_argument('--pretrained', dest='pretrained', action='store_true', + help='use pre-trained model') +parser.add_argument('--world-size', default=-1, type=int, + help='number of nodes for distributed training') +parser.add_argument('--rank', default=-1, type=int, + help='node rank for distributed training') +parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, + help='url used to set up distributed training') +parser.add_argument('--dist-backend', default='nccl', type=str, + help='distributed backend') +parser.add_argument('--seed', default=None, type=int, + help='seed for initializing training. ') +parser.add_argument('--gpu', default=None, type=int, + help='GPU id to use.') +parser.add_argument('--multiprocessing-distributed', action='store_true', + help='Use multi-processing distributed training to launch ' + 'N processes per node, which has N GPUs. This is the ' + 'fastest way to use PyTorch for either single node or ' + 'multi node data parallel training') +parser.add_argument('-bm', '--benchmark', default=0, type=int, + metavar='N', help='set benchmark status (default: 1,run benchmark)') +parser.add_argument('--device', default='npu', type=str, help='npu or gpu') +parser.add_argument('--addr', default='10.136.181.115', type=str, help='master addr') +parser.add_argument('--checkpoint-nameprefix', default='checkpoint', type=str, help='checkpoint-nameprefix') +parser.add_argument('--checkpoint-freq', default=0, type=int, + metavar='N', help='checkpoint frequency (default: 0)' + '0: save only one file whitch per epoch;' + 'n: save diff file per n epoch' + '-1:no checkpoint,not support') +parser.add_argument('--device_num', default=-1, type=int, + help='device_num') +parser.add_argument('--device-list', default='', type=str, help='device id list') +parser.add_argument('--warm_up_epochs', default=0, type=int, + help='warm up') + +# apex +parser.add_argument('--amp', default=False, action='store_true', + help='use amp to train the model') +parser.add_argument('--loss-scale', default=64., type=float, + help='loss scale using in amp, default -1 means dynamic') +parser.add_argument('--opt-level', default='O2', type=str, + help='loss scale using in amp, default -1 means dynamic') + +warnings.filterwarnings('ignore') +best_acc1 = 0 + + +def main(): + args = parser.parse_args() + print("===============main()=================") + print(args) + print("===============main()=================") + + os.environ['KERNEL_NAME_ID'] = str(0) + print("+++++++++++++++++++++++++++KERNEL_NAME_ID:", os.environ['KERNEL_NAME_ID']) + + if args.seed is not None: + random.seed(args.seed) + torch.manual_seed(args.seed) + cudnn.deterministic = True + warnings.warn('You have chosen to seed training. ' + 'This will turn on the CUDNN deterministic setting, ' + 'which can slow down your training considerably! ' + 'You may see unexpected behavior when restarting ' + 'from checkpoints.') + + os.environ['MASTER_ADDR'] = args.addr # '10.136.181.51' + os.environ['MASTER_PORT'] = '29688' + + if args.gpu is not None: + warnings.warn('You have chosen a specific GPU. This will completely ' + 'disable data parallelism.') + + if args.dist_url == "env://" and args.world_size == -1: + args.world_size = int(os.environ["WORLD_SIZE"]) + + args.distributed = args.world_size > 1 or args.multiprocessing_distributed + + if args.device_list != '': + ngpus_per_node = len(args.device_list.split(',')) + elif args.device_num != -1: + ngpus_per_node = args.device_num + elif args.device == 'npu': + ngpus_per_node = torch.npu.device_count() + else: + ngpus_per_node = torch.cuda.device_count() + if args.multiprocessing_distributed: + # Since we have ngpus_per_node processes per node, the total world_size + # needs to be adjusted accordingly + args.world_size = ngpus_per_node * args.world_size + # Use torch.multiprocessing.spawn to launch distributed processes: the + # main_worker process function + # The child process uses the environment variables of the parent process, + # we have to set KERNEL_NAME_ID for every proc + if args.device == 'npu': + # main_worker(args.gpu, ngpus_per_node, args) + mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) + else: + mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) + else: + # Simply call main_worker function + main_worker(args.gpu, ngpus_per_node, args) + + +def main_worker(gpu, ngpus_per_node, args): + global best_acc1 + + if args.device_list != '': + args.gpu = int(args.device_list.split(',')[gpu]) + else: + args.gpu = gpu + + print("[npu id:", args.gpu, "]", "++++++++++++++++ before set KERNEL_NAME_ID:", os.environ['KERNEL_NAME_ID']) + os.environ['KERNEL_NAME_ID'] = str(args.gpu) + print("[npu id:", args.gpu, "]", "++++++++++++++++ KERNEL_NAME_ID:", os.environ['KERNEL_NAME_ID']) + + if args.gpu is not None: + print("[npu id:", args.gpu, "]", "Use GPU: {} for training".format(args.gpu)) + + if args.distributed: + if args.dist_url == "env://" and args.rank == -1: + args.rank = int(os.environ["RANK"]) + if args.multiprocessing_distributed: + # For multiprocessing distributed training, rank needs to be the + # global rank among all the processes + args.rank = args.rank * ngpus_per_node + gpu + + if args.device == 'npu': + dist.init_process_group(backend=args.dist_backend, # init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + else: + dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + + loc = 'npu:{}'.format(args.gpu) + torch.npu.set_device(loc) + + args.batch_size = int(args.batch_size / ngpus_per_node) + args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) + + print("[npu id:", args.gpu, "]", "===============main_worker()=================") + print("[npu id:", args.gpu, "]", args) + print("[npu id:", args.gpu, "]", "===============main_worker()=================") + + train_loader, train_loader_len, train_sampler = get_pytorch_train_loader(args.data, + args.batch_size, + workers=args.workers, + distributed=args.distributed) + + val_loader = get_pytorch_val_loader(args.data, args.batch_size, args.workers, distributed=False) + + # create model + print("[npu id:", args.gpu, "]", "=> creating model '{}'".format(args.arch)) + model = models.__dict__[args.arch]() + model = model.to(loc) + + # define loss function (criterion) and optimizer + criterion = nn.CrossEntropyLoss().to(loc) + optimizer = torch.optim.SGD(model.parameters(), args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay) + + if args.amp: + model, optimizer = amp.initialize(model, optimizer, opt_level=args.opt_level, loss_scale=args.loss_scale) + + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], broadcast_buffers=False) + + # optionally resume from a checkpoint + if args.resume: + if os.path.isfile(args.resume): + print("=> loading checkpoint '{}'".format(args.resume)) + checkpoint = torch.load(args.resume, map_location=loc) + args.start_epoch = checkpoint['epoch'] + best_acc1 = checkpoint['best_acc1'] + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + if args.amp: + amp.load_state_dict(checkpoint['amp']) + print("=> loaded checkpoint '{}' (epoch {})".format(args.resume, checkpoint['epoch'])) + else: + print("=> no checkpoint found at '{}'".format(args.resume)) + + cudnn.benchmark = True + + if args.evaluate: + validate(val_loader, model, criterion, args, ngpus_per_node) + return + + for epoch in range(args.start_epoch, args.epochs): + if args.distributed: + train_sampler.set_epoch(epoch) + adjust_learning_rate(optimizer, epoch, args) + + # train for one epoch + train(train_loader, train_loader_len, model, criterion, optimizer, epoch, args, ngpus_per_node) + + if (epoch + 1) % args.eval_freq == 0 or epoch == args.epochs - 1 or epoch > int(args.epochs * 0.9): + # evaluate on validation set + acc1 = validate(val_loader, model, criterion, args, ngpus_per_node) + + # remember best acc@1 and save checkpoint + is_best = acc1 > best_acc1 + best_acc1 = max(acc1, best_acc1) + + if not args.multiprocessing_distributed or \ + (args.multiprocessing_distributed and args.rank % ngpus_per_node == 0 or epoch == args.epochs - 1): + if args.amp: + save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer': optimizer.state_dict(), + 'amp': amp.state_dict(), + }, is_best) + else: + save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer': optimizer.state_dict(), + }, is_best) + + +def train(train_loader, train_loader_len, model, criterion, optimizer, epoch, args, ngpus_per_node): + batch_time = AverageMeter('Time', ':6.3f') + data_time = AverageMeter('Data', ':6.3f') + losses = AverageMeter('Loss', ':.4e', start_count_index=0) + top1 = AverageMeter('Acc@1', ':6.2f', start_count_index=0) + top5 = AverageMeter('Acc@5', ':6.2f', start_count_index=0) + progress = ProgressMeter( + train_loader_len, + [batch_time, data_time, losses, top1, top5], + prefix="Epoch: [{}]".format(epoch)) + + loc = 'npu:{}'.format(args.gpu) + + mean = torch.tensor([0.485 * 255, 0.456 * 255, 0.406 * 255]).view(1, 3, 1, 1) + std = torch.tensor([0.229 * 255, 0.224 * 255, 0.225 * 255]).view(1, 3, 1, 1) + mean = mean.to(loc, non_blocking=True) + std = std.to(loc, non_blocking=True) + + # switch to train mode + model.train() + end = time.time() + if args.benchmark == 1: + optimizer.zero_grad() + + steps_per_epoch = train_loader_len + print('==========step per epoch======================', steps_per_epoch) + for i, (images, target) in enumerate(train_loader): + # measure data loading time + data_time.update(time.time() - end) + + target = target.to(torch.int32) + images = images.to(loc, non_blocking=True).to(torch.float).sub(mean).div(std) + target = target.to(loc, non_blocking=True) + + # compute output + output = model(images) + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # compute gradient and do SGD step + if args.benchmark == 0: + optimizer.zero_grad() + + if args.amp: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + + stream = torch.npu.current_stream() + stream.synchronize() + + if args.benchmark == 0: + optimizer.step() + elif args.benchmark == 1: + BATCH_SIZE_multiplier = int(OPTIMIZER_BATCH_SIZE / args.batch_size) + BM_optimizer_step = ((i + 1) % BATCH_SIZE_multiplier) == 0 + if BM_optimizer_step: + for param_group in optimizer.param_groups: + for param in param_group['params']: + param.grad /= BATCH_SIZE_multiplier + optimizer.step() + optimizer.zero_grad() + stream = torch.npu.current_stream() + stream.synchronize() + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + progress.display(i) + + + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + print("[npu id:", args.gpu, "]", '* FPS@all {:.3f}, TIME@all {:.3f}'.format(ngpus_per_node * args.batch_size / batch_time.avg, batch_time.avg)) + hwlog.remark_print(key=hwlog.FPS, value=' * FPS@all {:.3f}'.format(ngpus_per_node * args.batch_size / batch_time.avg)) + + +def validate(val_loader, model, criterion, args, ngpus_per_node): + batch_time = AverageMeter('Time', ':6.3f') + losses = AverageMeter('Loss', ':.4e', start_count_index=0) + top1 = AverageMeter('Acc@1', ':6.2f', start_count_index=0) + top5 = AverageMeter('Acc@5', ':6.2f', start_count_index=0) + progress = ProgressMeter( + len(val_loader), + [batch_time, losses, top1, top5], + prefix='Test: ') + + # switch to evaluate mode + model.eval() + + with torch.no_grad(): + loc = 'npu:{}'.format(args.gpu) + mean = torch.tensor([0.485 * 255, 0.456 * 255, 0.406 * 255]).view(1, 3, 1, 1) + std = torch.tensor([0.229 * 255, 0.224 * 255, 0.225 * 255]).view(1, 3, 1, 1) + mean = mean.to(loc, non_blocking=True) + std = std.to(loc, non_blocking=True) + + end = time.time() + for i, (images, target) in enumerate(val_loader): + + target = target.to(torch.int32) + images = images.to(loc, non_blocking=True).to(torch.float).sub(mean).div(std) + target = target.to(loc, non_blocking=True) + + # compute output + output = model(images) + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + if not args.multiprocessing_distributed or \ + (args.multiprocessing_distributed and args.rank % ngpus_per_node == 0): + progress.display(i) + + # TODO: this should also be done with the ProgressMeter + if not args.multiprocessing_distributed or \ + (args.multiprocessing_distributed and args.rank % ngpus_per_node == 0): + print("[npu id:", args.gpu, "]", '[AVG-ACC] * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}' + .format(top1=top1, top5=top5)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value="{top1.avg:.3f}".format(top1=top1)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value="{top5.avg:.3f}".format(top5=top5)) + return top1.avg + + +def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): + torch.save(state, filename) + if is_best: + shutil.copyfile(filename, 'model_best_acc%.4f_epoch%d.pth.tar' % (state['best_acc1'], state['epoch'])) + + +class AverageMeter(object): + """Computes and stores the average and current value""" + + def __init__(self, name, fmt=':f', start_count_index=10): + self.name = name + self.fmt = fmt + self.reset() + self.start_count_index = start_count_index + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + if self.count == 0: + self.N = n + + self.val = val + self.count += n + if self.count > (self.start_count_index * self.N): + self.sum += val * n + self.avg = self.sum / (self.count - self.start_count_index * self.N) + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + +class ProgressMeter(object): + def __init__(self, num_batches, meters, prefix=""): + self.batch_fmtstr = self._get_batch_fmtstr(num_batches) + self.meters = meters + self.prefix = prefix + + def display(self, batch): + entries = [self.prefix + self.batch_fmtstr.format(batch)] + entries += [str(meter) for meter in self.meters] + print("[npu id:", os.environ['KERNEL_NAME_ID'], "]", '\t'.join(entries)) + + def _get_batch_fmtstr(self, num_batches): + num_digits = len(str(num_batches // 1)) + fmt = '{:' + str(num_digits) + 'd}' + return '[' + fmt + '/' + fmt.format(num_batches) + ']' + + +def adjust_learning_rate(optimizer, epoch, args): + """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" + # lr = args.lr * (0.1 ** (epoch // (args.epochs//3 - 3))) + + if args.warm_up_epochs > 0 and epoch < args.warm_up_epochs: + lr = args.lr * ((epoch + 1) / (args.warm_up_epochs + 1)) + else: + alpha = 0 + cosine_decay = 0.5 * ( + 1 + np.cos(np.pi * (epoch - args.warm_up_epochs) / (args.epochs - args.warm_up_epochs))) + decayed = (1 - alpha) * cosine_decay + alpha + lr = args.lr * decayed + + print("=> Epoch[%d] Setting lr: %.4f" % (epoch, lr)) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + + +def accuracy(output, target, topk=(1,)): + """Computes the accuracy over the k top predictions for the specified values of k""" + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def fast_collate(batch): + imgs = [img[0] for img in batch] + targets = torch.tensor([target[1] for target in batch], dtype=torch.int64) + w = imgs[0].size[0] + h = imgs[0].size[1] + tensor = torch.zeros((len(imgs), 3, h, w), dtype=torch.uint8) + for i, img in enumerate(imgs): + nump_array = np.asarray(img, dtype=np.uint8) + if nump_array.ndim < 3: + nump_array = np.expand_dims(nump_array, axis=-1) + nump_array = np.rollaxis(nump_array, 2) + + tensor[i] += torch.from_numpy(nump_array) + + return tensor, targets + + +def get_pytorch_train_loader(data_path, batch_size, workers=5, _worker_init_fn=None, distributed=False): + traindir = os.path.join(data_path, 'train') + train_dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2), + transforms.RandomHorizontalFlip(), + ])) + + if distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + + dataloader_fn = MultiEpochsDataLoader # torch.utils.data.DataLoader + train_loader = dataloader_fn( + train_dataset, batch_size=batch_size, shuffle=(train_sampler is None), + num_workers=workers, worker_init_fn=_worker_init_fn, pin_memory=True, sampler=train_sampler, + collate_fn=fast_collate, drop_last=True) + return train_loader, len(train_loader), train_sampler + + +def get_pytorch_val_loader(data_path, batch_size, workers=5, _worker_init_fn=None, distributed=False): + valdir = os.path.join(data_path, 'val') + val_dataset = datasets.ImageFolder( + valdir, transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + ])) + + if distributed: + val_sampler = torch.utils.data.distributed.DistributedSampler(val_dataset) + else: + val_sampler = None + + dataloader_fn = MultiEpochsDataLoader # torch.utils.data.DataLoader + val_loader = dataloader_fn( + val_dataset, + sampler=val_sampler, + batch_size=batch_size, shuffle=(val_sampler is None), + num_workers=workers, worker_init_fn=_worker_init_fn, pin_memory=True, collate_fn=fast_collate) + + return val_loader + + +if __name__ == '__main__': + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("pytorch") + config_info = get_model_parameter("pytorch_config") + initinal_data = { "dataset": "imagenet", "optimizer": "SGD", "loss_scale": 64} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + main() diff --git a/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/README.md b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/README.md new file mode 100644 index 0000000..f658759 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/README.md @@ -0,0 +1,60 @@ +# ImageNet training in PyTorch + +This implements training of ShuffleNetV2 on the ImageNet dataset, mainly modified from [pytorch/examples](https://github.com/pytorch/examples/tree/master/imagenet). + +## ShuffleNet Detail +As of the current date, Ascend-Pytorch is still inefficient for contiguous operations. +Therefore, ShufflenetV2 is re-implemented using semantics such as custom OP. For details, see models/shufflenetv2_wock_op_woct.py . + + +## Requirements + +- Install PyTorch ([pytorch.org](http://pytorch.org)) +- `pip install -r requirements.txt` +- Download the ImageNet dataset from http://www.image-net.org/ + - Then, and move validation images to labeled subfolders, using [the following shell script](https://raw.githubusercontent.com/soumith/imagenetloader.torch/master/valprep.sh) + +## Training 1p + +To train a model, run `main.py` with the desired model architecture and the path to the ImageNet dataset: + +```bash +# FP32 training +bash 1p.sh + +# O2 training +bash 1p_amp.sh + +# FP32 training for 20 epoch experiment +bash 1p_20e.sh + +# FP32 training for 20 epoch experiment +bash 1p_20e_amp.sh + +``` + +## Training multi-cards +```bash +# O2 training 2p +# Only Support device-list setting in [[0,1], [2,3], [4,5], [6,7]] +bash 2p_amp_med.sh + +# O2 training 4p +# Only Support device-list setting in [[0,1,2,3], [4,5,6,7]] +bash 4p_amp_med.sh + +# O2 training 8p +bash 8p_amp_med.sh + +``` + +## ShufflenetV2 training result + +| Acc@1 | FPS | Npu_nums| Epochs | Type | +| :------: | :------: | :------ | :------: | :------: | +| 61.5 | 1200 | 1 | 20 | O2 | +| 68.5 | 2200 | 1 | 240 | O2 | +| 66.3 | 14000 | 1 | 240 | O2 | + + + diff --git a/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/main.py b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/main.py new file mode 100644 index 0000000..3a3cd73 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/main.py @@ -0,0 +1,649 @@ +import argparse +import os +import random +import shutil +import time +import warnings + +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.distributed as dist +import torch.optim +import torch.multiprocessing as mp +import torch.utils.data +import torch.utils.data.distributed +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import models as models + +# Apex +import numpy as np +from apex import amp + +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter + +# from megvii repo +class CrossEntropyLabelSmooth(nn.Module): + def __init__(self, num_classes, epsilon): + super(CrossEntropyLabelSmooth, self).__init__() + self.num_classes = num_classes + self.epsilon = epsilon + # self.logsoftmax = nn.LogSoftmax(dim=1) + + def forward(self, inputs, targets): + # log_probs = self.logsoftmax(inputs) + # targets = torch.zeros_like(log_probs).scatter_(1, targets.unsqueeze(1), 1) + # targets = (1 - self.epsilon) * targets + self.epsilon / self.num_classes + # loss = (-targets * log_probs).mean(0).sum() + # return loss + + logprobs = torch.nn.functional.log_softmax(inputs, dim=-1).to("cpu") + targets = torch.zeros_like(logprobs).scatter_(1, targets.unsqueeze(1), 1) + targets = (1 - self.epsilon) * targets + self.epsilon / self.num_classes + loss = (-targets * logprobs).mean(0).sum() + return loss + + + +model_names = sorted(name for name in models.__dict__ + if name.islower() and not name.startswith("__") + and callable(models.__dict__[name])) + +parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') +parser.add_argument('data', metavar='DIR', + help='path to dataset') +parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet18', + choices=model_names, + help='model architecture: ' + + ' | '.join(model_names) + + ' (default: resnet18)') +parser.add_argument('-j', '--workers', default=8, type=int, metavar='N', + help='number of data loading workers (default: 4)') +parser.add_argument('--epochs', default=90, type=int, metavar='N', + help='number of total epochs to run') +parser.add_argument('--start-epoch', default=0, type=int, metavar='N', + help='manual epoch number (useful on restarts)') +parser.add_argument('-b', '--batch-size', default=256, type=int, + metavar='N', + help='mini-batch size (default: 256), this is the total ' + 'batch size of all GPUs on the current node when ' + 'using Data Parallel or Distributed Data Parallel') +parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, + metavar='LR', help='initial learning rate', dest='lr') +parser.add_argument('--momentum', default=0.9, type=float, metavar='M', + help='momentum') +parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, + metavar='W', help='weight decay (default: 1e-4)', + dest='weight_decay') +parser.add_argument('-p', '--print-freq', default=10, type=int, + metavar='N', help='print frequency (default: 10)') +parser.add_argument('--resume', default='', type=str, metavar='PATH', + help='path to latest checkpoint (default: none)') +parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', + help='evaluate model on validation set') +parser.add_argument('--pretrained', dest='pretrained', action='store_true', + help='use pre-trained model') +parser.add_argument('--world-size', default=-1, type=int, + help='number of nodes for distributed training') +parser.add_argument('--rank', default=-1, type=int, + help='node rank for distributed training') +parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, + help='url used to set up distributed training') +parser.add_argument('--dist-backend', default='nccl', type=str, + help='distributed backend') +parser.add_argument('--seed', default=None, type=int, + help='seed for initializing training. ') +parser.add_argument('--gpu', default=None, type=int, + help='GPU id to use.') +parser.add_argument('--multiprocessing-distributed', action='store_true', + help='Use multi-processing distributed training to launch ' + 'N processes per node, which has N GPUs. This is the ' + 'fastest way to use PyTorch for either single node or ' + 'multi node data parallel training') + +# npu +parser.add_argument('--npu', default=None, type=int, + help='NPU id to use.') + +# add +parser.add_argument('--eval_between_epochs', default=1, type=int, + help='setting bigger interval to speed up training.') +parser.add_argument('--label_smooth', default=0, type=float, + help='label smoothing using in CE') +parser.add_argument('--lr_scheduler_type', default='step_epoch', type=str, + help='lr_scheduler type, such as linear,cosine') +parser.add_argument('--warm_up_epochs', default=0, type=int, + help='warm up') +parser.add_argument('--total_steps', default=-1, type=float, + help='warm up') +parser.add_argument('--save_path', default='./training/save', type=str, + help='save model base path') +parser.add_argument('--tb_path', default='./training/events', type=str, + help='save tensorboard events path') + +# apex +parser.add_argument('--amp', default=False, action='store_true', + help='use amp to train the model') +parser.add_argument('--loss_scale', default='dynamic', type=str, + help='loss scale using in amp, default -1 means dynamic') +parser.add_argument('--opt_level', default='O1', type=str, + help='opt_level using in amp, default O1.') + +best_acc1 = 0 + + +def main(): + args = parser.parse_args() + print(args) + + if args.seed is not None: + random.seed(args.seed) + torch.manual_seed(args.seed) + cudnn.deterministic = True + warnings.warn('You have chosen to seed training. ' + 'This will turn on the CUDNN deterministic setting, ' + 'which can slow down your training considerably! ' + 'You may see unexpected behavior when restarting ' + 'from checkpoints.') + + if args.gpu is not None: + warnings.warn('You have chosen a specific GPU. This will completely ' + 'disable data parallelism.') + + if args.dist_url == "env://" and args.world_size == -1: + args.world_size = int(os.environ["WORLD_SIZE"]) + + args.distributed = args.world_size > 1 or args.multiprocessing_distributed + + ngpus_per_node = torch.cuda.device_count() + if args.multiprocessing_distributed: + # Since we have ngpus_per_node processes per node, the total world_size + # needs to be adjusted accordingly + args.world_size = ngpus_per_node * args.world_size + # Use torch.multiprocessing.spawn to launch distributed processes: the + # main_worker process function + mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) + else: + # Simply call main_worker function + main_worker(args.gpu, ngpus_per_node, args) + + +def main_worker(gpu, ngpus_per_node, args): + global best_acc1 + args.gpu = gpu + + if args.gpu is not None: + print("Use GPU: {} for training".format(args.gpu)) + + if args.distributed: + if args.dist_url == "env://" and args.rank == -1: + args.rank = int(os.environ["RANK"]) + if args.multiprocessing_distributed: + # For multiprocessing distributed training, rank needs to be the + # global rank among all the processes + args.rank = args.rank * ngpus_per_node + gpu + dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + # create model + if args.pretrained: + print("=> using pre-trained model '{}'".format(args.arch)) + model = models.__dict__[args.arch](pretrained=True) + else: + print("=> creating model '{}'".format(args.arch)) + model = models.__dict__[args.arch]() + + + optimizer = torch.optim.SGD(model.parameters(), args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay) + + if args.gpu is not None: + torch.cuda.set_device(args.gpu) + model = model.cuda(args.gpu) + elif args.npu is not None: + torch.npu.set_device("npu:%d"%args.npu) + model = model.to("npu:%d"%args.npu) + # else: + # # DataParallel will divide and allocate batch_size to all available GPUs + # if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): + # model.features = torch.nn.DataParallel(model.features) + # model.cuda() + # else: + # model = torch.nn.DataParallel(model).cuda() + + # apex + if args.amp: + # Initialization + model, optimizer = amp.initialize(model, optimizer, opt_level=args.opt_level, loss_scale=args.loss_scale) + print("=> Using amp mode.") + + # if args.distributed: + # # For multiprocessing distributed, DistributedDataParallel constructor + # # should always set the single device scope, otherwise, + # # DistributedDataParallel will use all available devices. + # if args.gpu is not None: + # # torch.cuda.set_device(args.gpu) + # # model.cuda(args.gpu) + # # When using a single GPU per process and per + # # DistributedDataParallel, we need to divide the batch size + # # ourselves based on the total number of GPUs we have + # args.batch_size = int(args.batch_size / ngpus_per_node) + # args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) + # model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + # else: + # # model.cuda() + # # DistributedDataParallel will divide and allocate batch_size to all + # # available GPUs if device_ids are not set + # model = torch.nn.parallel.DistributedDataParallel(model) + + + # define loss function (criterion) and optimizer + if args.label_smooth > 0: + # criterion = CrossEntropyLabelSmooth(1000, 0.1).cuda(args.gpu) + criterion = CrossEntropyLabelSmooth(1000, 0.1).to("npu:%d"%args.npu) + else: + # criterion = nn.CrossEntropyLoss().cuda(args.gpu) + criterion = nn.CrossEntropyLoss().to("npu:%d"%args.npu) + + + + + # optionally resume from a checkpoint + if args.resume: + if os.path.isfile(args.resume): + print("=> loading checkpoint '{}'".format(args.resume)) + if args.gpu is None: + checkpoint = torch.load(args.resume) + else: + # Map model to be loaded to specified single gpu. + loc = 'cuda:{}'.format(args.gpu) + checkpoint = torch.load(args.resume, map_location=loc) + args.start_epoch = checkpoint['epoch'] + best_acc1 = checkpoint['best_acc1'] + if args.gpu is not None: + # best_acc1 may be from a checkpoint from a different GPU + best_acc1 = best_acc1.to(args.gpu) + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + print("=> loaded checkpoint '{}' (epoch {})" + .format(args.resume, checkpoint['epoch'])) + else: + print("=> no checkpoint found at '{}'".format(args.resume)) + + # cudnn.benchmark = True + + # Data loading code + traindir = os.path.join(args.data, 'train') + valdir = os.path.join(args.data, 'val') + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + + train_dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ])) + + if args.distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), + num_workers=args.workers, pin_memory=True, sampler=train_sampler, + drop_last=True, + ) + + val_loader = torch.utils.data.DataLoader( + datasets.ImageFolder(valdir, transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + normalize, + ])), + batch_size=args.batch_size, shuffle=False, + num_workers=args.workers, pin_memory=True, + drop_last=True, + ) + + if args.evaluate: + validate(val_loader, model, criterion, args) + return + + global_step = args.start_epoch * len(train_loader) + if args.total_steps < 0: + args.total_steps = len(train_loader) * args.epochs + if args.warm_up_epochs > 0: + args.warm_up_steps = len(train_loader) * args.warm_up_epochs + else: + args.warm_up_steps = 0 + for epoch in range(args.start_epoch, args.epochs): + if args.distributed: + train_sampler.set_epoch(epoch) + + if 'epoch' in args.lr_scheduler_type: + if args.lr_scheduler_type in ['', 'step_epoch']: + adjust_learning_rate(optimizer, epoch, args) + else: + adjust_learning_rate_epoch(optimizer, args, epoch) + + + # train for one epoch + global_step = train(train_loader, model, criterion, optimizer, epoch, args, global_step=global_step) + + + if (epoch + 1) % args.eval_between_epochs == 0 or epoch > int(args.epochs * 0.9): + # evaluate on validation set + acc1 = validate(val_loader, model, criterion, args) + + # remember best acc@1 and save checkpoint + is_best = acc1 > best_acc1 + best_acc1 = max(acc1, best_acc1) + + model = model.to("cpu") + if args.multiprocessing_distributed: + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + # 'optimizer' : optimizer.state_dict(), + }, is_best.to("cpu"), save_path=os.path.join(args.save_path, str(args.gpu))) + else: + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + # 'optimizer': optimizer.state_dict(), + }, is_best.to("cpu"), save_path=args.save_path) + else: + model = model.to("cpu") + if args.multiprocessing_distributed: + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + # 'optimizer': optimizer.state_dict(), + }, False, save_path=os.path.join(args.save_path, str(args.gpu))) + else: + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + # 'optimizer': optimizer.state_dict(), + }, False, save_path=args.save_path) + + model = model.to("npu") + +def train(train_loader, model, criterion, optimizer, epoch, args, global_step): + batch_time = AverageMeter('Time', ':6.3f', start_count_index=10) + data_time = AverageMeter('Data', ':6.3f', start_count_index=10) + losses = AverageMeter('Loss', ':.4e') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(train_loader), + [batch_time, data_time, losses, top1, top5], + prefix="Epoch: [{}]".format(epoch)) + + # switch to train mode + model.train() + print('==> enter train mode.') + + end = time.time() + for i, (images, target) in enumerate(train_loader): + if 'epoch' not in args.lr_scheduler_type: + lr_step = adjust_learning_rate_step(optimizer, args, global_step) + + # measure data loading time + data_time.update(time.time() - end) + + if args.gpu is not None: + images = images.cuda(args.gpu, non_blocking=True) + target = target.cuda(args.gpu, non_blocking=True) + if args.npu is not None: + images = images.to("npu:%d" % args.npu, non_blocking=True) + if not args.label_smooth > 0: + target = target.to(torch.int32) + target = target.to("npu:%d" % args.npu, non_blocking=True) + + # compute output + output = model(images) + if args.label_smooth > 0: + loss = criterion(output, target).to("npu:%d" % args.npu, non_blocking=True) + else: + loss = criterion(output, target) + + # measure accuracy and record loss + if args.label_smooth > 0: + target = target.to(torch.int32) + target = target.to("npu:%d" % args.npu, non_blocking=True) + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # compute gradient and do SGD step + optimizer.zero_grad() + if args.amp: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + + optimizer.step() + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + progress.display(i) + + global_step += 1 + + # if i > 50: + # break + print("[npu id:", args.gpu, "]", '* FPS@all {:.3f}'.format(args.batch_size / batch_time.avg)) + hwlog.remark_print(key=hwlog.FPS, value=' * FPS@all {:.3f}'.format(args.batch_size / batch_time.avg)) + + + return global_step + +def validate(val_loader, model, criterion, args): + batch_time = AverageMeter('Time', ':6.3f') + losses = AverageMeter('Loss', ':.4e') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(val_loader), + [batch_time, losses, top1, top5], + prefix='Test: ') + + # switch to evaluate mode + model.eval() + print('==> enter eval mode.') + + with torch.no_grad(): + end = time.time() + for i, (images, target) in enumerate(val_loader): + if args.gpu is not None: + images = images.cuda(args.gpu, non_blocking=True) + target = target.cuda(args.gpu, non_blocking=True) + if args.npu is not None: + target = target.to(torch.int32) + images = images.to("npu:%d" % args.npu, non_blocking=True) + target = target.to("npu:%d" % args.npu, non_blocking=True) + + # compute output + output = model(images) + if args.label_smooth > 0: + loss = criterion(output, target).to("npu:%d" % args.npu, non_blocking=True) + else: + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + progress.display(i) + + # TODO: this should also be done with the ProgressMeter + print(' * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}' + .format(top1=top1, top5=top5)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value="{top1.avg:.3f}".format(top1=top1)) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value="{top5.avg:.3f}".format(top5=top5)) + return top1.avg + + +def save_checkpoint(state, is_best, filename='checkpoint.pth.tar', save_path='./'): + if not os.path.exists(save_path): + os.makedirs(save_path) + torch.save(state, os.path.join(save_path, filename)) + if is_best: + shutil.copyfile(os.path.join(save_path, filename), os.path.join(save_path, 'model_best_acc%.4f_epoch%d.pth.tar'%(state['best_acc1'], state['epoch']))) + + +class AverageMeter(object): + """Computes and stores the average and current value""" + + def __init__(self, name, fmt=':f', start_count_index=0): + self.name = name + self.fmt = fmt + self.reset() + self.start_count_index = start_count_index + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.count += n + if self.count > (self.start_count_index * n): + self.sum += val * n + self.avg = self.sum / (self.count - self.start_count_index * n) + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + +class ProgressMeter(object): + def __init__(self, num_batches, meters, prefix=""): + self.batch_fmtstr = self._get_batch_fmtstr(num_batches) + self.meters = meters + self.prefix = prefix + + def display(self, batch): + entries = [self.prefix + self.batch_fmtstr.format(batch)] + entries += [str(meter) for meter in self.meters] + print('\t'.join(entries)) + + def _get_batch_fmtstr(self, num_batches): + num_digits = len(str(num_batches // 1)) + fmt = '{:' + str(num_digits) + 'd}' + return '[' + fmt + '/' + fmt.format(num_batches) + ']' + + +def adjust_learning_rate(optimizer, epoch, args): + """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" + lr = args.lr * (0.1 ** (epoch // 30)) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + return lr + +def adjust_learning_rate_step(optimizer, args, global_step): + """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" + if args.warm_up_steps > 0 and global_step < args.warm_up_steps: + lr = args.lr * (global_step / args.warm_up_steps) + else: + if args.lr_scheduler_type == 'linear': + lr = args.lr * (1 - (global_step - args.warm_up_steps) / (args.total_steps - - args.warm_up_steps)) + elif args.lr_scheduler_type == 'cosine': + alpha = 0 + cosine_decay = 0.5 * (1 + np.cos(np.pi * (global_step - args.warm_up_steps) / (args.total_steps - args.warm_up_steps))) + decayed = (1 - alpha) * cosine_decay + alpha + lr = args.lr * decayed + + lr = max(lr, 0) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + return lr + + +def adjust_learning_rate_epoch(optimizer, args, global_epoch): + """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" + if args.warm_up_epochs > 0 and global_epoch < args.warm_up_epochs: + lr = args.lr * ((global_epoch+1) / (args.warm_up_epochs+1)) + else: + if args.lr_scheduler_type == 'linear_epoch': + lr = args.lr * (1 - (global_epoch - args.warm_up_epochs) / (args.epochs - - args.warm_up_epochs)) + elif args.lr_scheduler_type == 'cosine_epoch': + alpha = 0 + cosine_decay = 0.5 * (1 + np.cos(np.pi * (global_epoch - args.warm_up_epochs) / (args.epochs - args.warm_up_epochs))) + decayed = (1 - alpha) * cosine_decay + alpha + lr = args.lr * decayed + + lr = max(lr, 0) + + print("=> Epoch[%d] Setting lr: %.4f"%(global_epoch, lr)) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + return lr + + +def accuracy(output, target, topk=(1,)): + """Computes the accuracy over the k top predictions for the specified values of k""" + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +if __name__ == '__main__': + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("pytorch") + config_info = get_model_parameter("pytorch_config") + initinal_data = {"base_lr": 0.256, "dataset": "imagenet", "optimizer": "SGD", "loss_scale": 64} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + main() diff --git a/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/__init__.py b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/__init__.py new file mode 100644 index 0000000..a3d351d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/__init__.py @@ -0,0 +1 @@ +from .shufflenetv2_wock_op_woct_8p import * diff --git a/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/_utils.py b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/_utils.py new file mode 100644 index 0000000..291041d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/_utils.py @@ -0,0 +1,67 @@ +from collections import OrderedDict + +import torch +from torch import nn +from torch.jit.annotations import Dict + + +class IntermediateLayerGetter(nn.ModuleDict): + """ + Module wrapper that returns intermediate layers from a model + + It has a strong assumption that the modules have been registered + into the model in the same order as they are used. + This means that one should **not** reuse the same nn.Module + twice in the forward if you want this to work. + + Additionally, it is only able to query submodules that are directly + assigned to the model. So if `model` is passed, `model.feature1` can + be returned, but not `model.feature1.layer2`. + + Arguments: + model (nn.Module): model on which we will extract the features + return_layers (Dict[name, new_name]): a dict containing the names + of the modules for which the activations will be returned as + the key of the dict, and the value of the dict is the name + of the returned activation (which the user can specify). + + Examples:: + + >>> m = torchvision.models.resnet18(pretrained=True) + >>> # extract layer1 and layer3, giving as names `feat1` and feat2` + >>> new_m = torchvision.models._utils.IntermediateLayerGetter(m, + >>> {'layer1': 'feat1', 'layer3': 'feat2'}) + >>> out = new_m(torch.rand(1, 3, 224, 224)) + >>> print([(k, v.shape) for k, v in out.items()]) + >>> [('feat1', torch.Size([1, 64, 56, 56])), + >>> ('feat2', torch.Size([1, 256, 14, 14]))] + """ + _version = 2 + __annotations__ = { + "return_layers": Dict[str, str], + } + + def __init__(self, model, return_layers): + if not set(return_layers).issubset([name for name, _ in model.named_children()]): + raise ValueError("return_layers are not present in model") + orig_return_layers = return_layers + return_layers = {str(k): str(v) for k, v in return_layers.items()} + layers = OrderedDict() + for name, module in model.named_children(): + layers[name] = module + if name in return_layers: + del return_layers[name] + if not return_layers: + break + + super(IntermediateLayerGetter, self).__init__(layers) + self.return_layers = orig_return_layers + + def forward(self, x): + out = OrderedDict() + for name, module in self.items(): + x = module(x) + if name in self.return_layers: + out_name = self.return_layers[name] + out[out_name] = x + return out diff --git a/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/shufflenetv2.py b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/shufflenetv2.py new file mode 100644 index 0000000..14f9521 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/shufflenetv2.py @@ -0,0 +1,208 @@ +import torch +import torch.nn as nn +from .utils import load_state_dict_from_url + + +__all__ = [ + 'ShuffleNetV2', 'shufflenet_v2_x0_5', 'shufflenet_v2_x1_0', + 'shufflenet_v2_x1_5', 'shufflenet_v2_x2_0' +] + +model_urls = { + 'shufflenetv2_x0.5': 'https://download.pytorch.org/models/shufflenetv2_x0.5-f707e7126e.pth', + 'shufflenetv2_x1.0': 'https://download.pytorch.org/models/shufflenetv2_x1-5666bf0f80.pth', + 'shufflenetv2_x1.5': None, + 'shufflenetv2_x2.0': None, +} + + +def channel_shuffle(x, groups): + # type: (torch.Tensor, int) -> torch.Tensor + batchsize, num_channels, height, width = x.data.size() + channels_per_group = num_channels // groups + + # reshape + x = x.view(batchsize, groups, + channels_per_group, height, width) + + x = torch.transpose(x, 1, 2).contiguous() + + # flatten + x = x.view(batchsize, -1, height, width) + + return x + + +class InvertedResidual(nn.Module): + def __init__(self, inp, oup, stride): + super(InvertedResidual, self).__init__() + + if not (1 <= stride <= 3): + raise ValueError('illegal stride value') + self.stride = stride + + branch_features = oup // 2 + assert (self.stride != 1) or (inp == branch_features << 1) + + if self.stride > 1: + self.branch1 = nn.Sequential( + self.depthwise_conv(inp, inp, kernel_size=3, stride=self.stride, padding=1), + nn.BatchNorm2d(inp), + nn.Conv2d(inp, branch_features, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(branch_features), + nn.ReLU(inplace=True), + ) + else: + self.branch1 = nn.Sequential() + + self.branch2 = nn.Sequential( + nn.Conv2d(inp if (self.stride > 1) else branch_features, + branch_features, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(branch_features), + nn.ReLU(inplace=True), + self.depthwise_conv(branch_features, branch_features, kernel_size=3, stride=self.stride, padding=1), + nn.BatchNorm2d(branch_features), + nn.Conv2d(branch_features, branch_features, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(branch_features), + nn.ReLU(inplace=True), + ) + + @staticmethod + def depthwise_conv(i, o, kernel_size, stride=1, padding=0, bias=False): + return nn.Conv2d(i, o, kernel_size, stride, padding, bias=bias, groups=i) + + def forward(self, x): + if self.stride == 1: + x1, x2 = x.chunk(2, dim=1) + out = torch.cat((x1, self.branch2(x2)), dim=1) + else: + out = torch.cat((self.branch1(x), self.branch2(x)), dim=1) + + out = channel_shuffle(out, 2) + + return out + + +class ShuffleNetV2(nn.Module): + def __init__(self, stages_repeats, stages_out_channels, num_classes=1000, inverted_residual=InvertedResidual): + super(ShuffleNetV2, self).__init__() + + if len(stages_repeats) != 3: + raise ValueError('expected stages_repeats as list of 3 positive ints') + if len(stages_out_channels) != 5: + raise ValueError('expected stages_out_channels as list of 5 positive ints') + self._stage_out_channels = stages_out_channels + + input_channels = 3 + output_channels = self._stage_out_channels[0] + self.conv1 = nn.Sequential( + nn.Conv2d(input_channels, output_channels, 3, 2, 1, bias=False), + nn.BatchNorm2d(output_channels), + nn.ReLU(inplace=True), + ) + input_channels = output_channels + + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + stage_names = ['stage{}'.format(i) for i in [2, 3, 4]] + for name, repeats, output_channels in zip( + stage_names, stages_repeats, self._stage_out_channels[1:]): + seq = [inverted_residual(input_channels, output_channels, 2)] + for i in range(repeats - 1): + seq.append(inverted_residual(output_channels, output_channels, 1)) + setattr(self, name, nn.Sequential(*seq)) + input_channels = output_channels + + output_channels = self._stage_out_channels[-1] + self.conv5 = nn.Sequential( + nn.Conv2d(input_channels, output_channels, 1, 1, 0, bias=False), + nn.BatchNorm2d(output_channels), + nn.ReLU(inplace=True), + ) + + self.fc = nn.Linear(output_channels, num_classes) + + def _forward_impl(self, x): + # See note [TorchScript super()] + x = self.conv1(x) + x = self.maxpool(x) + x = self.stage2(x) + x = self.stage3(x) + x = self.stage4(x) + x = self.conv5(x) + x = x.mean([2, 3]) # globalpool + x = self.fc(x) + return x + + def forward(self, x): + return self._forward_impl(x) + + +def _shufflenetv2(arch, pretrained, progress, *args, **kwargs): + model = ShuffleNetV2(*args, **kwargs) + + if pretrained: + model_url = model_urls[arch] + if model_url is None: + raise NotImplementedError('pretrained {} is not supported as of now'.format(arch)) + else: + state_dict = load_state_dict_from_url(model_url, progress=progress) + model.load_state_dict(state_dict) + + return model + + +def shufflenet_v2_x0_5(pretrained=False, progress=True, **kwargs): + """ + Constructs a ShuffleNetV2 with 0.5x output channels, as described in + `"ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design" + `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _shufflenetv2('shufflenetv2_x0.5', pretrained, progress, + [4, 8, 4], [24, 48, 96, 192, 1024], **kwargs) + + +def shufflenet_v2_x1_0(pretrained=False, progress=True, **kwargs): + """ + Constructs a ShuffleNetV2 with 1.0x output channels, as described in + `"ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design" + `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _shufflenetv2('shufflenetv2_x1.0', pretrained, progress, + [4, 8, 4], [24, 116, 232, 464, 1024], **kwargs) + + +def shufflenet_v2_x1_5(pretrained=False, progress=True, **kwargs): + """ + Constructs a ShuffleNetV2 with 1.5x output channels, as described in + `"ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design" + `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _shufflenetv2('shufflenetv2_x1.5', pretrained, progress, + [4, 8, 4], [24, 176, 352, 704, 1024], **kwargs) + + +def shufflenet_v2_x2_0(pretrained=False, progress=True, **kwargs): + """ + Constructs a ShuffleNetV2 with 2.0x output channels, as described in + `"ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design" + `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _shufflenetv2('shufflenetv2_x2.0', pretrained, progress, + [4, 8, 4], [24, 244, 488, 976, 2048], **kwargs) diff --git a/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/shufflenetv2_group_shuffle.py b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/shufflenetv2_group_shuffle.py new file mode 100644 index 0000000..19a0fb4 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/shufflenetv2_group_shuffle.py @@ -0,0 +1,256 @@ +import torch +import torch.nn as nn + +try: + from .utils import load_state_dict_from_url +except: + pass + +import numpy as np + +__all__ = [ + 'ShuffleNetV2', 'shufflenet_v2_x0_5', 'shufflenet_v2_x1_0', + 'shufflenet_v2_x1_5', 'shufflenet_v2_x2_0' +] + +model_urls = { + 'shufflenetv2_x0.5': 'https://download.pytorch.org/models/shufflenetv2_x0.5-f707e7126e.pth', + 'shufflenetv2_x1.0': 'https://download.pytorch.org/models/shufflenetv2_x1-5666bf0f80.pth', + 'shufflenetv2_x1.5': None, + 'shufflenetv2_x2.0': None, +} + + +class Channel_Shuffle(nn.Module): + def __init__(self, inp, groups=4, split_shuffle=True): + super(Channel_Shuffle, self).__init__() + + self.split_shuffle = split_shuffle + self.groups = groups + + def forward(self, x1, x2): + x1_list = x1.chunk(self.groups // 2, 1) + x2_list = x2.chunk(self.groups // 2, 1) + + if self.split_shuffle: + split_point = len(x1_list) // 2 + out1 = [] + out2 = [] + for idx in range(split_point): + out1.append(x1_list[idx]) + out1.append(x2_list[idx]) + for idx in range(split_point, len(x1_list), 1): + out2.append(x1_list[idx]) + out2.append(x2_list[idx]) + return torch.cat(out1, 1), torch.cat(out2, 1) + else: + out = [] + for idx in range(len(x1_list)): + out.append(x1_list[idx]) + out.append(x2_list[idx]) + return torch.cat(out, 1) + + + +class InvertedResidual(nn.Module): + def __init__(self, inp, oup, stride, split_shuffle=True): + super(InvertedResidual, self).__init__() + + if not (1 <= stride <= 3): + raise ValueError('illegal stride value') + self.stride = stride + + branch_features = oup // 2 + assert (self.stride != 1) or (inp == branch_features << 1) + + if self.stride > 1: + self.branch1 = nn.Sequential( + self.depthwise_conv(inp, inp, kernel_size=3, stride=self.stride, padding=1), + nn.BatchNorm2d(inp), + nn.Conv2d(inp, branch_features, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(branch_features), + nn.ReLU(inplace=True), + ) + else: + self.branch1 = nn.Sequential() + + self.branch2 = nn.Sequential( + nn.Conv2d(inp if (self.stride > 1) else branch_features, + branch_features, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(branch_features), + nn.ReLU(inplace=True), + self.depthwise_conv(branch_features, branch_features, kernel_size=3, stride=self.stride, padding=1), + nn.BatchNorm2d(branch_features), + nn.Conv2d(branch_features, branch_features, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(branch_features), + nn.ReLU(inplace=True), + ) + + if self.stride > 1: + self.channel_shuffle = Channel_Shuffle(inp=branch_features + branch_features, + split_shuffle=split_shuffle) + else: + self.channel_shuffle = Channel_Shuffle(inp=inp, split_shuffle=split_shuffle) + + @staticmethod + def depthwise_conv(i, o, kernel_size, stride=1, padding=0, bias=False): + return nn.Conv2d(i, o, kernel_size, stride, padding, bias=bias, groups=i) + + def forward(self, x): + if self.stride == 1: + x1, x2 = x + x2 = self.branch2(x2) + else: + x1 = self.branch1(x) + x2 = self.branch2(x) + + # out = channel_shuffle(out, 2) + out = self.channel_shuffle(x1, x2) + + return out + + +class ShuffleNetV2(nn.Module): + def __init__(self, stages_repeats, stages_out_channels, num_classes=1000, inverted_residual=InvertedResidual): + super(ShuffleNetV2, self).__init__() + + if len(stages_repeats) != 3: + raise ValueError('expected stages_repeats as list of 3 positive ints') + if len(stages_out_channels) != 5: + raise ValueError('expected stages_out_channels as list of 5 positive ints') + self._stage_out_channels = stages_out_channels + + input_channels = 3 + output_channels = self._stage_out_channels[0] + self.conv1 = nn.Sequential( + nn.Conv2d(input_channels, output_channels, 3, 2, 1, bias=False), + nn.BatchNorm2d(output_channels), + nn.ReLU(inplace=True), + ) + input_channels = output_channels + + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + stage_names = ['stage{}'.format(i) for i in [2, 3, 4]] + for name, repeats, output_channels in zip( + stage_names, stages_repeats, self._stage_out_channels[1:]): + seq = [inverted_residual(input_channels, output_channels, 2)] + for i in range(repeats - 1): + if i == repeats - 2: + seq.append(inverted_residual(output_channels, output_channels, 1, split_shuffle=False)) + else: + seq.append(inverted_residual(output_channels, output_channels, 1)) + setattr(self, name, nn.Sequential(*seq)) + input_channels = output_channels + + output_channels = self._stage_out_channels[-1] + self.conv5 = nn.Sequential( + nn.Conv2d(input_channels, output_channels, 1, 1, 0, bias=False), + nn.BatchNorm2d(output_channels), + nn.ReLU(inplace=True), + ) + + self.fc = nn.Linear(output_channels, num_classes) + + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + + def _forward_impl(self, x): + + # See note [TorchScript super()] + x = self.conv1(x) + x = self.maxpool(x) + x = self.stage2(x) + x = self.stage3(x) + x = self.stage4(x) + x = self.conv5(x) + # x = x.mean([2, 3]) # globalpool + x = self.avgpool(x) + x = torch.flatten(x, 1) + + x = self.fc(x) + return x + + def forward(self, x): + return self._forward_impl(x) + + +def _shufflenetv2(arch, pretrained, progress, *args, **kwargs): + model = ShuffleNetV2(*args, **kwargs) + + if pretrained: + model_url = model_urls[arch] + if model_url is None: + raise NotImplementedError('pretrained {} is not supported as of now'.format(arch)) + else: + state_dict = load_state_dict_from_url(model_url, progress=progress) + model.load_state_dict(state_dict) + + return model + + +def shufflenet_v2_x0_5(pretrained=False, progress=True, **kwargs): + """ + Constructs a ShuffleNetV2 with 0.5x output channels, as described in + `"ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design" + `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _shufflenetv2('shufflenetv2_x0.5', pretrained, progress, + [4, 8, 4], [24, 48, 96, 192, 1024], **kwargs) + + +def shufflenet_v2_x1_0(pretrained=False, progress=True, **kwargs): + """ + Constructs a ShuffleNetV2 with 1.0x output channels, as described in + `"ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design" + `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + # return _shufflenetv2('shufflenetv2_x1.0', pretrained, progress, + # [4, 8, 4], [24, 116, 232, 464, 1024], **kwargs) + return _shufflenetv2('shufflenetv2_x1.0', pretrained, progress, + [4, 8, 4], [16, 128, 256, 512, 1024], **kwargs) + + +def shufflenet_v2_x1_5(pretrained=False, progress=True, **kwargs): + """ + Constructs a ShuffleNetV2 with 1.5x output channels, as described in + `"ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design" + `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _shufflenetv2('shufflenetv2_x1.5', pretrained, progress, + [4, 8, 4], [24, 176, 352, 704, 1024], **kwargs) + + +def shufflenet_v2_x2_0(pretrained=False, progress=True, **kwargs): + """ + Constructs a ShuffleNetV2 with 2.0x output channels, as described in + `"ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design" + `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _shufflenetv2('shufflenetv2_x2.0', pretrained, progress, + [4, 8, 4], [24, 244, 488, 976, 2048], **kwargs) + + +if __name__ == '__main__': + model = shufflenet_v2_x1_0() + print(model) + x = torch.randn(1, 3, 224, 224) + y = model(x) + loss = y.sum() + loss.backward() + diff --git a/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/shufflenetv2_wock_op_woct.py b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/shufflenetv2_wock_op_woct.py new file mode 100644 index 0000000..2b3fca6 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/shufflenetv2_wock_op_woct.py @@ -0,0 +1,330 @@ +import torch +import torch.nn as nn + +try: + from .utils import load_state_dict_from_url +except: + pass + +import numpy as np + +__all__ = [ + 'ShuffleNetV2', 'shufflenet_v2_x0_5', 'shufflenet_v2_x1_0', + 'shufflenet_v2_x1_5', 'shufflenet_v2_x2_0' +] + +model_urls = { + 'shufflenetv2_x0.5': 'https://download.pytorch.org/models/shufflenetv2_x0.5-f707e7126e.pth', + 'shufflenetv2_x1.0': 'https://download.pytorch.org/models/shufflenetv2_x1-5666bf0f80.pth', + 'shufflenetv2_x1.5': None, + 'shufflenetv2_x2.0': None, +} + + +class IndexSelectFullImplementation(torch.autograd.Function): + @staticmethod + def forward(ctx, x1, x2, fp_index, bp_index1, bp_index2): + stream = torch.npu.current_stream() + stream.synchronize() + + ctx.bp_index1 = bp_index1 + ctx.bp_index2 = bp_index2 + x = torch.cat([x1, x2], dim=1) + result = x.index_select(1, fp_index) + + + return result + + @staticmethod + def backward(ctx, grad_output): + stream = torch.npu.current_stream() + stream.synchronize() + + # convert to NCHW to avoid extra 5HD --> 4D + grad_output.data = grad_output.data.npu_format_cast(0) + out1 = grad_output.index_select(1, ctx.bp_index1) + out2 = grad_output.index_select(1, ctx.bp_index2) + return out1, out2, None, None, None, None + + +class IndexSelectHalfImplementation(torch.autograd.Function): + @staticmethod + def forward(ctx, x1, x2, fp_index1, fp_index2, bp_index1, bp_index2): + ctx.bp_index1 = bp_index1 + ctx.bp_index2 = bp_index2 + x = torch.cat([x1, x2], dim=1) + return x.index_select(1, fp_index1), x.index_select(1, fp_index2) + + @staticmethod + def backward(ctx, grad_output1, grad_output2): + grad_output = torch.cat([grad_output1, grad_output2], 1) + out1 = grad_output.index_select(1, ctx.bp_index1) + out2 = grad_output.index_select(1, ctx.bp_index2) + return out1, out2, None, None, None, None + + +class Channel_Shuffle(nn.Module): + def __init__(self, inp, groups=2, split_shuffle=True): + super(Channel_Shuffle, self).__init__() + + self.split_shuffle = split_shuffle + self.group_len = inp // groups + self.out = np.array(list(range(inp))).reshape(groups, self.group_len).transpose(1, 0).flatten().tolist() + if self.split_shuffle: + self.register_buffer('fp_index1', torch.tensor(self.out[:self.group_len])) + self.register_buffer('fp_index2', torch.tensor(self.out[self.group_len:])) + else: + self.register_buffer('fp_index', torch.tensor(self.out)) + # self.register_buffer('bp_index', torch.tensor(list(range(0, inp, 2))+list(range(1,inp,2)))) + self.register_buffer('bp_index1', torch.tensor(list(range(0, inp, 2)))) + self.register_buffer('bp_index2', torch.tensor(list(range(1, inp, 2)))) + + def forward(self, x1, x2): + if self.split_shuffle: + return IndexSelectHalfImplementation.apply(x1, x2, self.fp_index1, self.fp_index2, self.bp_index1, + self.bp_index2) + else: + return IndexSelectFullImplementation.apply(x1, x2, self.fp_index, self.bp_index1, self.bp_index2) + + +class InvertedResidual(nn.Module): + def __init__(self, inp, oup, stride, split_shuffle=True): + super(InvertedResidual, self).__init__() + + if not (1 <= stride <= 3): + raise ValueError('illegal stride value') + self.stride = stride + + branch_features = oup // 2 + assert (self.stride != 1) or (inp == branch_features << 1) + + if self.stride > 1: + self.branch1 = nn.Sequential( + self.depthwise_conv(inp, inp, kernel_size=3, stride=self.stride, padding=1), + nn.BatchNorm2d(inp), + nn.Conv2d(inp, branch_features, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(branch_features), + nn.ReLU(inplace=True), + ) + else: + self.branch1 = nn.Sequential() + + self.branch2 = nn.Sequential( + nn.Conv2d(inp if (self.stride > 1) else branch_features, + branch_features, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(branch_features), + nn.ReLU(inplace=True), + self.depthwise_conv(branch_features, branch_features, kernel_size=3, stride=self.stride, padding=1), + nn.BatchNorm2d(branch_features), + nn.Conv2d(branch_features, branch_features, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(branch_features), + nn.ReLU(inplace=True), + ) + + if self.stride > 1: + self.channel_shuffle = Channel_Shuffle(inp=branch_features + branch_features, groups=2, + split_shuffle=split_shuffle) + else: + self.channel_shuffle = Channel_Shuffle(inp=inp, groups=2, split_shuffle=split_shuffle) + + @staticmethod + def depthwise_conv(i, o, kernel_size, stride=1, padding=0, bias=False): + return nn.Conv2d(i, o, kernel_size, stride, padding, bias=bias, groups=i) + + def forward(self, x): + if self.stride == 1: + x1, x2 = x + x2 = self.branch2(x2) + else: + x1 = self.branch1(x) + x2 = self.branch2(x) + + # out = channel_shuffle(out, 2) + out = self.channel_shuffle(x1, x2) + + return out + + +class ShuffleNetV2(nn.Module): + def __init__(self, stages_repeats, stages_out_channels, num_classes=1000, inverted_residual=InvertedResidual): + super(ShuffleNetV2, self).__init__() + + if len(stages_repeats) != 3: + raise ValueError('expected stages_repeats as list of 3 positive ints') + if len(stages_out_channels) != 5: + raise ValueError('expected stages_out_channels as list of 5 positive ints') + self._stage_out_channels = stages_out_channels + + input_channels = 3 + output_channels = self._stage_out_channels[0] + self.conv1 = nn.Sequential( + nn.Conv2d(input_channels, output_channels, 3, 2, 1, bias=False), + nn.BatchNorm2d(output_channels), + nn.ReLU(inplace=True), + ) + input_channels = output_channels + + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + stage_names = ['stage{}'.format(i) for i in [2, 3, 4]] + for name, repeats, output_channels in zip( + stage_names, stages_repeats, self._stage_out_channels[1:]): + seq = [inverted_residual(input_channels, output_channels, 2)] + for i in range(repeats - 1): + if i == repeats - 2: + seq.append(inverted_residual(output_channels, output_channels, 1, split_shuffle=False)) + else: + seq.append(inverted_residual(output_channels, output_channels, 1)) + setattr(self, name, nn.Sequential(*seq)) + input_channels = output_channels + + output_channels = self._stage_out_channels[-1] + self.conv5 = nn.Sequential( + nn.Conv2d(input_channels, output_channels, 1, 1, 0, bias=False), + nn.BatchNorm2d(output_channels), + nn.ReLU(inplace=True), + ) + + self.fc = nn.Linear(output_channels, num_classes) + + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + + def _forward_impl(self, x): + + # See note [TorchScript super()] + x = self.conv1(x) + x = self.maxpool(x) + x = self.stage2(x) + x = self.stage3(x) + x = self.stage4(x) + x = self.conv5(x) + # x = x.mean([2, 3]) # globalpool + x = self.avgpool(x) + x = torch.flatten(x, 1) + + x = self.fc(x) + return x + + def forward(self, x): + return self._forward_impl(x) + + +def _shufflenetv2(arch, pretrained, progress, *args, **kwargs): + model = ShuffleNetV2(*args, **kwargs) + + if pretrained: + model_url = model_urls[arch] + if model_url is None: + raise NotImplementedError('pretrained {} is not supported as of now'.format(arch)) + else: + state_dict = load_state_dict_from_url(model_url, progress=progress) + model.load_state_dict(state_dict) + + return model + + +def shufflenet_v2_x0_5(pretrained=False, progress=True, **kwargs): + """ + Constructs a ShuffleNetV2 with 0.5x output channels, as described in + `"ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design" + `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _shufflenetv2('shufflenetv2_x0.5', pretrained, progress, + [4, 8, 4], [24, 48, 96, 192, 1024], **kwargs) + + +def shufflenet_v2_x1_0(pretrained=False, progress=True, **kwargs): + """ + Constructs a ShuffleNetV2 with 1.0x output channels, as described in + `"ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design" + `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _shufflenetv2('shufflenetv2_x1.0', pretrained, progress, + [4, 8, 4], [24, 116, 232, 464, 1024], **kwargs) + # return _shufflenetv2('shufflenetv2_x1.0', pretrained, progress, + # [4, 8, 4], [16, 128, 256, 464, 1024], **kwargs) + + +def shufflenet_v2_x1_5(pretrained=False, progress=True, **kwargs): + """ + Constructs a ShuffleNetV2 with 1.5x output channels, as described in + `"ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design" + `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _shufflenetv2('shufflenetv2_x1.5', pretrained, progress, + [4, 8, 4], [24, 176, 352, 704, 1024], **kwargs) + + +def shufflenet_v2_x2_0(pretrained=False, progress=True, **kwargs): + """ + Constructs a ShuffleNetV2 with 2.0x output channels, as described in + `"ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design" + `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _shufflenetv2('shufflenetv2_x2.0', pretrained, progress, + [4, 8, 4], [24, 244, 488, 976, 2048], **kwargs) + + +if __name__ == '__main__': + import pickle + + + def init(): + # init input + x = np.random.randn(32, 3, 224, 224).astype(np.float32) + with open('input_tensor.pkl', 'wb')as f: + pickle.dump(x, f) + model = shufflenet_v2_x1_0() + with open('init_weight.pth', 'wb')as f: + torch.save(model.state_dict(), f) + + + with open('input_tensor.pkl', 'rb')as f: + input_tensor = torch.from_numpy(pickle.load(f)) + input_tensor.requires_grad = True + + model = shufflenet_v2_x1_0() + with open('init_weight.pth', 'rb')as f: + model.load_state_dict(torch.load(f)) + + inter_feature = {} + inter_gradient = {} + def make_hook(name, flag): + if flag == 'forward': + def hook(m, input, output): + inter_feature[name] = input + + return hook + elif flag == 'backward': + def hook(m, input, output): + inter_gradient[name] = output + + return hook + else: + assert False + for name, m in model.named_modules(): + m.register_forward_hook(make_hook(name, 'forward')) + m.register_backward_hook(make_hook(name, 'backward')) + + out = model(input_tensor) + loss = out.sum() + loss.backward() + + print(inter_feature) + print(inter_gradient) diff --git a/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/shufflenetv2_wock_op_woct_8p.py b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/shufflenetv2_wock_op_woct_8p.py new file mode 100644 index 0000000..041fc01 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/shufflenetv2_wock_op_woct_8p.py @@ -0,0 +1,328 @@ +import torch +import torch.nn as nn + +try: + from .utils import load_state_dict_from_url +except: + pass + +import numpy as np + +__all__ = [ + 'ShuffleNetV2', 'shufflenet_v2_x0_5', 'shufflenet_v2_x1_0', + 'shufflenet_v2_x1_5', 'shufflenet_v2_x2_0' +] + +model_urls = { + 'shufflenetv2_x0.5': 'https://download.pytorch.org/models/shufflenetv2_x0.5-f707e7126e.pth', + 'shufflenetv2_x1.0': 'https://download.pytorch.org/models/shufflenetv2_x1-5666bf0f80.pth', + 'shufflenetv2_x1.5': None, + 'shufflenetv2_x2.0': None, +} + + +class IndexSelectFullImplementation(torch.autograd.Function): + @staticmethod + def forward(ctx, x1, x2, fp_index, bp_index1, bp_index2): + stream = torch.npu.current_stream() + stream.synchronize() + + ctx.bp_index1 = bp_index1 + ctx.bp_index2 = bp_index2 + x = torch.cat([x1, x2], dim=1) + result = x.index_select(1, fp_index) + + return result + + @staticmethod + def backward(ctx, grad_output): + stream = torch.npu.current_stream() + stream.synchronize() + + # convert to NCHW to avoid extra 5HD --> 4D + grad_output.data = grad_output.data.npu_format_cast(0) + out1 = grad_output.index_select(1, ctx.bp_index1) + out2 = grad_output.index_select(1, ctx.bp_index2) + return out1, out2, None, None, None, None + + +class IndexSelectHalfImplementation(torch.autograd.Function): + @staticmethod + def forward(ctx, x1, x2, fp_index1, fp_index2, bp_index1, bp_index2): + ctx.bp_index1 = bp_index1 + ctx.bp_index2 = bp_index2 + x = torch.cat([x1, x2], dim=1) + return x.index_select(1, fp_index1), x.index_select(1, fp_index2) + + @staticmethod + def backward(ctx, grad_output1, grad_output2): + grad_output = torch.cat([grad_output1, grad_output2], 1) + out1 = grad_output.index_select(1, ctx.bp_index1) + out2 = grad_output.index_select(1, ctx.bp_index2) + return out1, out2, None, None, None, None + + +class Channel_Shuffle(nn.Module): + def __init__(self, inp, groups=2, split_shuffle=True): + super(Channel_Shuffle, self).__init__() + + self.split_shuffle = split_shuffle + self.group_len = inp // groups + self.out = np.array(list(range(inp))).reshape(groups, self.group_len).transpose(1, 0).flatten().tolist() + if self.split_shuffle: + self.register_buffer('fp_index1', torch.tensor(self.out[:self.group_len], dtype=torch.int32)) + self.register_buffer('fp_index2', torch.tensor(self.out[self.group_len:], dtype=torch.int32)) + else: + self.register_buffer('fp_index', torch.tensor(self.out, dtype=torch.int32)) + self.register_buffer('bp_index1', torch.tensor(list(range(0, inp, 2)), dtype=torch.int32)) + self.register_buffer('bp_index2', torch.tensor(list(range(1, inp, 2)), dtype=torch.int32)) + + def forward(self, x1, x2): + if self.split_shuffle: + return IndexSelectHalfImplementation.apply(x1, x2, self.fp_index1, self.fp_index2, self.bp_index1, + self.bp_index2) + else: + return IndexSelectFullImplementation.apply(x1, x2, self.fp_index, self.bp_index1, self.bp_index2) + + +class InvertedResidual(nn.Module): + def __init__(self, inp, oup, stride, split_shuffle=True): + super(InvertedResidual, self).__init__() + + if not (1 <= stride <= 3): + raise ValueError('illegal stride value') + self.stride = stride + + branch_features = oup // 2 + assert (self.stride != 1) or (inp == branch_features << 1) + + if self.stride > 1: + self.branch1 = nn.Sequential( + self.depthwise_conv(inp, inp, kernel_size=3, stride=self.stride, padding=1), + nn.BatchNorm2d(inp), + nn.Conv2d(inp, branch_features, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(branch_features), + nn.ReLU(inplace=True), + ) + else: + self.branch1 = nn.Sequential() + + self.branch2 = nn.Sequential( + nn.Conv2d(inp if (self.stride > 1) else branch_features, + branch_features, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(branch_features), + nn.ReLU(inplace=True), + self.depthwise_conv(branch_features, branch_features, kernel_size=3, stride=self.stride, padding=1), + nn.BatchNorm2d(branch_features), + nn.Conv2d(branch_features, branch_features, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(branch_features), + nn.ReLU(inplace=True), + ) + + if self.stride > 1: + self.channel_shuffle = Channel_Shuffle(inp=branch_features + branch_features, groups=2, + split_shuffle=split_shuffle) + else: + self.channel_shuffle = Channel_Shuffle(inp=inp, groups=2, split_shuffle=split_shuffle) + + @staticmethod + def depthwise_conv(i, o, kernel_size, stride=1, padding=0, bias=False): + return nn.Conv2d(i, o, kernel_size, stride, padding, bias=bias, groups=i) + + def forward(self, x): + if self.stride == 1: + x1, x2 = x + x2 = self.branch2(x2) + else: + x1 = self.branch1(x) + x2 = self.branch2(x) + + # out = channel_shuffle(out, 2) + out = self.channel_shuffle(x1, x2) + + return out + + +class ShuffleNetV2(nn.Module): + def __init__(self, stages_repeats, stages_out_channels, num_classes=1000, inverted_residual=InvertedResidual): + super(ShuffleNetV2, self).__init__() + + if len(stages_repeats) != 3: + raise ValueError('expected stages_repeats as list of 3 positive ints') + if len(stages_out_channels) != 5: + raise ValueError('expected stages_out_channels as list of 5 positive ints') + self._stage_out_channels = stages_out_channels + + input_channels = 3 + output_channels = self._stage_out_channels[0] + self.conv1 = nn.Sequential( + nn.Conv2d(input_channels, output_channels, 3, 2, 1, bias=False), + nn.BatchNorm2d(output_channels), + nn.ReLU(inplace=True), + ) + input_channels = output_channels + + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + stage_names = ['stage{}'.format(i) for i in [2, 3, 4]] + for name, repeats, output_channels in zip( + stage_names, stages_repeats, self._stage_out_channels[1:]): + seq = [inverted_residual(input_channels, output_channels, 2)] + for i in range(repeats - 1): + if i == repeats - 2: + seq.append(inverted_residual(output_channels, output_channels, 1, split_shuffle=False)) + else: + seq.append(inverted_residual(output_channels, output_channels, 1)) + setattr(self, name, nn.Sequential(*seq)) + input_channels = output_channels + + output_channels = self._stage_out_channels[-1] + self.conv5 = nn.Sequential( + nn.Conv2d(input_channels, output_channels, 1, 1, 0, bias=False), + nn.BatchNorm2d(output_channels), + nn.ReLU(inplace=True), + ) + + self.fc = nn.Linear(output_channels, num_classes) + + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + + def _forward_impl(self, x): + + # See note [TorchScript super()] + x = self.conv1(x) + x = self.maxpool(x) + x = self.stage2(x) + x = self.stage3(x) + x = self.stage4(x) + x = self.conv5(x) + # x = x.mean([2, 3]) # globalpool + x = self.avgpool(x) + x = torch.flatten(x, 1) + + x = self.fc(x) + return x + + def forward(self, x): + return self._forward_impl(x) + + +def _shufflenetv2(arch, pretrained, progress, *args, **kwargs): + model = ShuffleNetV2(*args, **kwargs) + + if pretrained: + model_url = model_urls[arch] + if model_url is None: + raise NotImplementedError('pretrained {} is not supported as of now'.format(arch)) + else: + state_dict = load_state_dict_from_url(model_url, progress=progress) + model.load_state_dict(state_dict) + + return model + + +def shufflenet_v2_x0_5(pretrained=False, progress=True, **kwargs): + """ + Constructs a ShuffleNetV2 with 0.5x output channels, as described in + `"ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design" + `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _shufflenetv2('shufflenetv2_x0.5', pretrained, progress, + [4, 8, 4], [24, 48, 96, 192, 1024], **kwargs) + + +def shufflenet_v2_x1_0(pretrained=False, progress=True, **kwargs): + """ + Constructs a ShuffleNetV2 with 1.0x output channels, as described in + `"ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design" + `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _shufflenetv2('shufflenetv2_x1.0', pretrained, progress, + [4, 8, 4], [24, 116, 232, 464, 1024], **kwargs) + # return _shufflenetv2('shufflenetv2_x1.0', pretrained, progress, + # [4, 8, 4], [16, 128, 256, 464, 1024], **kwargs) + + +def shufflenet_v2_x1_5(pretrained=False, progress=True, **kwargs): + """ + Constructs a ShuffleNetV2 with 1.5x output channels, as described in + `"ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design" + `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _shufflenetv2('shufflenetv2_x1.5', pretrained, progress, + [4, 8, 4], [24, 176, 352, 704, 1024], **kwargs) + + +def shufflenet_v2_x2_0(pretrained=False, progress=True, **kwargs): + """ + Constructs a ShuffleNetV2 with 2.0x output channels, as described in + `"ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design" + `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _shufflenetv2('shufflenetv2_x2.0', pretrained, progress, + [4, 8, 4], [24, 244, 488, 976, 2048], **kwargs) + + +if __name__ == '__main__': + import pickle + + + def init(): + # init input + x = np.random.randn(32, 3, 224, 224).astype(np.float32) + with open('input_tensor.pkl', 'wb')as f: + pickle.dump(x, f) + model = shufflenet_v2_x1_0() + with open('init_weight.pth', 'wb')as f: + torch.save(model.state_dict(), f) + + + with open('input_tensor.pkl', 'rb')as f: + input_tensor = torch.from_numpy(pickle.load(f)) + input_tensor.requires_grad = True + + model = shufflenet_v2_x1_0() + with open('init_weight.pth', 'rb')as f: + model.load_state_dict(torch.load(f)) + + inter_feature = {} + inter_gradient = {} + def make_hook(name, flag): + if flag == 'forward': + def hook(m, input, output): + inter_feature[name] = input + + return hook + elif flag == 'backward': + def hook(m, input, output): + inter_gradient[name] = output + + return hook + else: + assert False + for name, m in model.named_modules(): + m.register_forward_hook(make_hook(name, 'forward')) + m.register_backward_hook(make_hook(name, 'backward')) + + out = model(input_tensor) + loss = out.sum() + loss.backward() + + print(inter_feature) + print(inter_gradient) diff --git a/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/utils.py b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/utils.py new file mode 100644 index 0000000..638ef07 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/models/utils.py @@ -0,0 +1,4 @@ +try: + from torch.hub import load_state_dict_from_url +except ImportError: + from torch.utils.model_zoo import load_url as load_state_dict_from_url diff --git a/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/multi_epochs_dataloader.py b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/multi_epochs_dataloader.py new file mode 100644 index 0000000..28b6679 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/code/multi_epochs_dataloader.py @@ -0,0 +1,31 @@ +import torch + + +class MultiEpochsDataLoader(torch.utils.data.DataLoader): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._DataLoader__initialized = False + self.batch_sampler = _RepeatSampler(self.batch_sampler) + self._DataLoader__initialized = True + self.iterator = super().__iter__() + + def __len__(self): + return len(self.batch_sampler.sampler) + + def __iter__(self): + for _ in range(len(self)): + yield next(self.iterator) + + +class _RepeatSampler(object): + """ Sampler that repeats forever. + Args: + sampler (Sampler) + """ + + def __init__(self, sampler): + self.sampler = sampler + + def __iter__(self): + while True: + yield from iter(self.sampler) diff --git a/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/config/npu_set_env.sh b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/config/npu_set_env.sh new file mode 100644 index 0000000..7618849 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/config/npu_set_env.sh @@ -0,0 +1,31 @@ +############## toolkit situation ################ +#export LD_LIBRARY_PATH=/usr/local/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH +#export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/ascend-toolkit/latest/toolkit/tools/ide_daemon/bin/ +#export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ +#export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so +#export PYTHONPATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH + + +############## nnae situation ################ + + +if [ -d /usr/local/Ascend/nnae/latest ];then + export LD_LIBRARY_PATH=/usr/local/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/aarch64_64-linux-gnu:$LD_LIBRARY_PATH + export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/nnae/latest/toolkit/tools/ide_daemon/bin/ + export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp/ + export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/nnae/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so + export PYTHONPATH=/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH +else + export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/lib/aarch64-linux-gnu:$LD_LIBRARY_PATH + export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin/:/usr/local/Ascend/ascend-toolkit/latest/toolkit/tools/ide_daemon/bin/ + export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so + export PYTHONPATH=/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH +fi + +# ln -s /usr/local/Ascend/ascend-toolkit/latest/toolkit/bin/adc /usr/local/bin/ + +export SLOG_PRINT_TO_STDOUT=0 +#su HwHiAiUser -c "adc --host 0.0.0.0:22118 --log \"SetLogLevel(0)[error]\" --device 0" + +export TASK_QUEUE_ENABLE=1 \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/scripts/run.sh b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/scripts/run.sh new file mode 100644 index 0000000..9f367a1 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/scripts/run.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +rank_size=$1 +yamlPath=$2 +toolsPath=$3 +if [ -f /.dockerenv ];then + CLUSTER=$4 + MPIRUN_ALL_IP="$5" + export CLUSTER=${CLUSTER} +fi +currentDir=$(cd "$(dirname "$0")/.."; pwd) +# 配置环境变量并调用 train 方法 +currtime=`date +%Y%m%d%H%M%S` +mkdir -p ${currentDir%train*}/train/result/pt_shufflenet/training_job_${currtime}/ +train_job_dir=${currentDir%train*}/train/result/pt_shufflenet/training_job_${currtime}/ + +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir}" +# user env +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 +export SLOG_PRINT_TO_STDOUT=0 +export RANK_TABLE_FILE=${currentDir}/config/${rank_size}p.json + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "pytorch_config") + + + +# device 列表, 若无指定 device 根据 rank_size 顺序选择 +eval device_group=\$device_group_${rank_size}p +if [ x"${device_group}" == x"" ] || [ ${rank_size} -ge 8 ];then + device_group="$(seq 0 "$(expr $rank_size - 1)")" +fi + +# get last device id in device_group, hw log in performance from the dir named first_device_id +device_group_str=`echo ${device_group} | sed 's/ //g'` +first_device_id=`echo ${device_group_str: 0:1}` + +rank_id=0 + +if [ x"${CLUSTER}" == x"True" ];then + # ln hw log + ln -snf ${train_job_dir}/0/hw_shufflenet.log ${train_job_dir} + this_ip=$(hostname -I |awk '{print $1}') + for ip in $MPIRUN_ALL_IP;do + if [ x"$ip" != x"$this_ip" ];then + scp $yamlPath root@$ip:$yamlPath + scp $jsonFilePath root@$ip:$jsonFilePath + fi + done + export PATH=$PATH:/usr/local/mpirun4.0/bin + mpirun -H ${mpirun_ip} \ + --bind-to none -map-by slot\ + --allow-run-as-root \ + --mca btl_tcp_if_exclude lo,docker0,endvnic,virbr0,vethf40501b,docker_gwbridge,br-f42ac38052b4\ + --prefix /usr/local/mpirun4.0/ \ + ${currentDir}/scripts/train.sh 0 $rank_size $yamlPath $currtime ${toolsPath} ${CLUSTER} +else + # ln hw log + ln -snf ${train_job_dir}/${first_device_id}/hw_shufflenet.log ${train_job_dir} + #for device_id in $device_group;do + #echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] start: train ${device_id} & " >> ${currentDir}/result/main.log + #${currentDir}/scripts/train.sh $device_id $rank_size $yamlPath $currtime ${toolsPath} $rank_id & + #let rank_id++ + device_id=${first_device_id} + ${currentDir}/scripts/train.sh $device_id $rank_size $yamlPath $currtime ${toolsPath} $rank_id & + #done +fi +wait + +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir} " + diff --git a/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/scripts/train.sh b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/scripts/train.sh new file mode 100644 index 0000000..c3cb488 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/ShuffleNet/pytorch/scripts/train.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash + +device_id=$1 +rank_size=$2 +yamlPath=$3 +currtime=$4 +toolsPath=$5 + +currentDir=$(cd "$(dirname "$0")/.."; pwd) + +export REMARK_LOG_FILE=hw_shufflenet.log + +mkdir -p ${currentDir%train*}/train/result/pt_shufflenet/training_job_${currtime}/ +export train_job_dir=${currentDir%train*}/train/result/pt_shufflenet/training_job_${currtime}/ + + +source ${currentDir}/config/npu_set_env.sh + + +benchmark_log_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils +#atlasboost_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +code_dir_path=${currentDir}/code +export PYTHONPATH=$PYTHONPATH:${benchmark_log_path}:${code_dir_path} + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "pytorch_config") + +# user env +export YAML_PATH=$3 +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 +export RANK_TABLE_FILE=${currentDir}/config/${rank_size}p.json +export RANK_SIZE=${rank_size} +export RANK_INDEX=0 +export SLOG_PRINT_TO_STDOUT=0 +export DEVICE_ID=$1 +DEVICE_INDEX=$(( DEVICE_ID + RANK_INDEX * 8 )) +export DEVICE_INDEX=${DEVICE_INDEX} +export MODEL_CKPT_PATH=${train_job_dir}/${device_id}/ckpt${device_id} + +cd ${train_job_dir} +curd_dir=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +export PYTHONPATH=$PYTHONPATH:${curd_dir} + +if [ x"$6" != x"True" ];then + rank_id=$6 + export RANK_ID=$6 +else + device_id_mo=$(python3.7 -c "import src.tensorflow.mpi_ops as atlasboost;atlasboost.init(); \ + device_id = atlasboost.local_rank();cluster_device_id = str(device_id); \ + atlasboost.set_device_id(device_id);print(atlasboost.rank())") + device_id_mo=`echo $device_id_mo` + rank_id=${device_id_mo##* } + export RANK_ID=${rank_id} + device=${device_id_mo##*deviceid = } + device_id=${device%% phyid=*} + export DEVICE_ID=${device_id} + hccljson=${train_job_dir}/*.json + cp ${hccljson} ${currentDir}/config/${rank_size}p.json +fi + +#mkdir exec path +mkdir -p ${train_job_dir}/${device_id} +cd ${train_job_dir}/${device_id} + +startTime=`date +%Y%m%d-%H:%M:%S` +startTime_s=`date +%s` + +#echo "rank size is ${rank_size}" + +if [ x"${rank_size}" == x"1" ];then + python3.7 ${currentDir}/code/8p_main_med.py \ + --addr=$(hostname -I |awk '{print $1}') \ + --seed=49 \ + --workers=128 \ + --learning-rate=${lr} \ + --print-freq=1 \ + --eval-freq=${epochs_between_evals} \ + --arch=shufflenet_v2_x1_0 \ + --dist-url='tcp://127.0.0.1:50000' \ + --dist-backend='hccl' \ + --multiprocessing-distributed \ + --world-size=1 \ + --batch-size=${batch_size} \ + --epochs=${epoches} \ + --warm_up_epochs=${warm_up_epochs} \ + --rank=0 \ + --amp \ + --momentum=0 \ + --wd=3.0517578125e-05 \ + --device-list=${device_id} \ + --benchmark 0 \ + --data=${data_url} > ${train_job_dir}/train_1p.log 2>&1 +else + device_number=${rank_size} + #echo "device_group_multi ${device_group_multi}" + python3.7 ${currentDir}/code/8p_main_med.py \ + --addr=$(hostname -I |awk '{print $1}') \ + --seed=49 \ + --workers=184 \ + --learning-rate=${lr} \ + --print-freq=1 \ + --eval-freq=${epochs_between_evals} \ + --arch=shufflenet_v2_x1_0 \ + --dist-url='tcp://127.0.0.1:50000' \ + --dist-backend='hccl' \ + --multiprocessing-distributed \ + --world-size=1 \ + --batch-size=${batch_size} \ + --epochs=${epoches} \ + --warm_up_epochs=${warm_up_epochs} \ + --device_num=${device_number} \ + --rank=0 \ + --amp \ + --momentum=0 \ + --device-list=${device_group_multi} \ + --benchmark 0 \ + --data=${data_url} > ${train_job_dir}/train_${rank_size}p.log 2>&1 +fi + +if [ $? -eq 0 ] ; +then + echo ":::ABK 1.0.0 ShuffleNet train success" + echo ":::ABK 1.0.0 ShuffleNet train success" >> ${train_job_dir}/train_${rank_size}p.log + echo ":::ABK 1.0.0 ShuffleNet train success" >> ${train_job_dir}/${device_id}/hw_shufflenet.log +else + echo ":::ABK 1.0.0 ShuffleNet train failed" + echo ":::ABK 1.0.0 ShuffleNet train failed" >> ${train_job_dir}/train_${rank_size}p.log + echo ":::ABK 1.0.0 ShuffleNet train failed" >> ${train_job_dir}/${device_id}/hw_shufflenet.log +fi + +endTime=`date +%Y%m%d-%H:%M:%S` +endTime_s=`date +%s` + +sumTime=$[ $endTime_s - $startTime_s ] + +hour=$(( $sumTime/3600 )) +min=$(( ($sumTime-${hour}*3600)/60 )) +sec=$(( $sumTime-${hour}*3600-${min}*60 )) +echo ":::ABK 1.0.0 ShuffleNet train total time：${hour}:${min}:${sec}" + +echo ":::ABK 1.0.0 ShuffleNet train total time：${hour}:${min}:${sec}" >> ${train_job_dir}/${device_id}/hw_shufflenet.log + diff --git a/train/atlas_benchmark-master/image_classification/vgg16/__init__.py b/train/atlas_benchmark-master/image_classification/vgg16/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/16p.json b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/16p.json new file mode 100644 index 0000000..42ea84d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/16p.json @@ -0,0 +1,182 @@ +{ + "board_id" : "0x002F", + "chip_info" : "910", + "deploy_mode" : "lab", + "group_count" : "1", + "group_list" : [ + { + "device_num" : "16", + "server_num" : "2", + "group_name" : "", + "instance_count" : "16", + "instance_list" : [ + { + "devices" : [ + { + "device_id" : "0", + "device_ip" : "192.168.104.101" + } + ], + "rank_id" : "0", + "server_id" : "90.90.176.104" + }, + { + "devices" : [ + { + "device_id" : "1", + "device_ip" : "192.168.105.101" + } + ], + "rank_id" : "1", + "server_id" : "90.90.176.104" + }, + { + "devices" : [ + { + "device_id" : "2", + "device_ip" : "192.168.106.101" + } + ], + "rank_id" : "2", + "server_id" : "90.90.176.104" + }, + { + "devices" : [ + { + "device_id" : "3", + "device_ip" : "192.168.107.101" + } + ], + "rank_id" : "3", + "server_id" : "90.90.176.104" + }, + { + "devices" : [ + { + "device_id" : "4", + "device_ip" : "192.168.108.100" + } + ], + "rank_id" : "4", + "server_id" : "90.90.176.104" + }, + { + "devices" : [ + { + "device_id" : "5", + "device_ip" : "192.168.109.100" + } + ], + "rank_id" : "5", + "server_id" : "90.90.176.104" + }, + { + "devices" : [ + { + "device_id" : "6", + "device_ip" : "192.168.110.100" + } + ], + "rank_id" : "6", + "server_id" : "90.90.176.104" + }, + { + "devices" : [ + { + "device_id" : "7", + "device_ip" : "192.168.111.100" + } + ], + "rank_id" : "7", + "server_id" : "90.90.176.104" + }, + { + "devices" : [ + { + "device_id" : "0", + "device_ip" : "192.168.100.101" + } + ], + "rank_id" : "8", + "server_id" : "90.90.176.102" + }, + { + "devices" : [ + { + "device_id" : "1", + "device_ip" : "192.168.101.101" + } + ], + "rank_id" : "9", + "server_id" : "90.90.176.102" + }, + { + "devices" : [ + { + "device_id" : "2", + "device_ip" : "192.168.102.101" + } + ], + "rank_id" : "10", + "server_id" : "90.90.176.102" + }, + { + "devices" : [ + { + "device_id" : "3", + "device_ip" : "192.168.103.101" + } + ], + "rank_id" : "11", + "server_id" : "90.90.176.102" + }, + { + "devices" : [ + { + "device_id" : "4", + "device_ip" : "192.168.100.100" + } + ], + "rank_id" : "12", + "server_id" : "90.90.176.102" + }, + { + "devices" : [ + { + "device_id" : "5", + "device_ip" : "192.168.101.100" + } + ], + "rank_id" : "13", + "server_id" : "90.90.176.102" + }, + { + "devices" : [ + { + "device_id" : "6", + "device_ip" : "192.168.102.100" + } + ], + "rank_id" : "14", + "server_id" : "90.90.176.102" + }, + { + "devices" : [ + { + "device_id" : "7", + "device_ip" : "192.168.103.100" + } + ], + "rank_id" : "15", + "server_id" : "90.90.176.102" + } + ] + } + ], + "para_plane_nic_location" : "device", + "para_plane_nic_name" : [ + "eth0" + ], + "para_plane_nic_num" : "1", + "status" : "completed" +} \ No newline at end of file diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/1p.json b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/1p.json new file mode 100644 index 0000000..233cccf --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/1p.json @@ -0,0 +1,12 @@ +{ +"group_count": "1", +"group_list": [ +{ + "group_name": "worker", + "device_count": "1", + "instance_count": "1", + "instance_list": [{"devices":[{"device_id":"2","device_ip":"192.168.101.102"}],"pod_name":"npu1p","server_id":"127.0.0.1"}] +} +], +"status": "completed" +} diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/8p.json b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/8p.json new file mode 100644 index 0000000..84139c7 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/8p.json @@ -0,0 +1,12 @@ +{ +"group_count": "1", +"group_list": [ +{ + "group_name": "worker", + "device_count": "8", + "instance_count": "1", + "instance_list": [{"devices":[{"device_id":"0","device_ip":"192.168.190.102"},{"device_id":"1","device_ip":"192.168.191.102"},{"device_id":"2","device_ip":"192.168.192.102"},{"device_id":"3","device_ip":"192.168.193.102"},{"device_id":"4","device_ip":"192.168.190.103"},{"device_id":"5","device_ip":"192.168.191.103"},{"device_id":"6","device_ip":"192.168.192.103"},{"device_id":"7","device_ip":"192.168.193.103"}],"pod_name":"npu8p","server_id":"127.0.0.1"}] +} +], +"status": "completed" +} diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/cloud_docker_init.sh b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/cloud_docker_init.sh new file mode 100644 index 0000000..a5cddc9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/cloud_docker_init.sh @@ -0,0 +1,48 @@ +#!/bin/sh +currentDir=$(cd "$(dirname "$0")"; pwd) +cd ${currentDir} + +DEVICE_LIST=$@ + +export exec_type={MODE} + +prog_exit() +{ + if [ x"${exec_type}" = xdocker ]; + then + # stop slogd progress + bash /usr/local/Ascend/driver/tools/docker_stop_post_sys.sh + fi +} + +# register prog_exit +trap "prog_exit" SIGTERM + +if [ x"${exec_type}" = xdocker ]; +then + #set env + . ${currentDir}/npu_set_env.sh + + # start slogd progress + mkdir -p /var/log/npu/slog/slogd + /usr/local/Ascend/driver/tools/docker/slogd & + + # start main.sh + ${currentDir}/main.sh ${DEVICE_LIST} & + + # wait slogd stop + flag=1 + while [ $flag -ne 0 ]; + do + sleep 5; + flag=`ps -ef | grep train.sh | grep -v grep | wc -l` + ps -ef >> ${currentDir}/ps.log + echo "" >> ${currentDir}/ps.log + done +else + RANK_ID=`cat ${currentDir}/npu_set_env.sh | grep "RANK_ID=" | awk -F"=" '{print $2}'` + # start main.sh + su - HwHiAiUser -c ". ${currentDir}/npu_set_env.sh;export PROFILING_DIR=/var/log/npu/profiling/container/${RANK_ID};${currentDir}/main.sh ${DEVICE_LIST}" & + wait +fi + diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/hccl_sample.json b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/hccl_sample.json new file mode 100644 index 0000000..cec6b14 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/hccl_sample.json @@ -0,0 +1,6 @@ +{ + "server_count": "1", + "server_list": [{"device":[{devices}],"server_id":"127.0.0.1"}], + "status": "completed", + "version": "1.0" +} diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/main_sample.sh b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/main_sample.sh new file mode 100644 index 0000000..a43856f --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/main_sample.sh @@ -0,0 +1,18 @@ +#!/bin/sh +currentDir=$(cd "$(dirname "$0")"; pwd) +cd ${currentDir} + +device_group=$@ +device_num=$# + +touch ${currentDir}/main.log + +for device_phy_id in ${device_group} +do + echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] start: train.sh ${device_phy_id} & " >> ${currentDir}/main.log + ${currentDir}/train.sh ${device_phy_id} & +done + +wait + +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] all train.sh exit " >> ${currentDir}/main.log diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/npu_set_env.sh b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/npu_set_env.sh new file mode 100644 index 0000000..c791ac3 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/npu_set_env.sh @@ -0,0 +1,40 @@ +# main env +if [ -d /usr/local/Ascend/nnae/latest ];then + + export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/Ascend/driver/tools/hccn_tool/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages + export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp +else + export LD_LIBRARY_PATH=/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/ascend-toolkit/latest//fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/tfplugin/python/site-packages:$projectDir + export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + +fi +export SOC_VERSION=Ascend910 +export HCCL_CONNECT_TIMEOUT=600 + +# user env +export JOB_ID={JOB_ID} +export RANK_TABLE_FILE={RANK_TABLE_FILE} +#export RANK_SIZE={RANK_SIZE} +#export RANK_INDEX={RANK_INDEX} +#export RANK_ID={RANK_ID} + +# profiling env +export PROFILING_MODE={PROFILING_MODE} +export AICPU_PROFILING_MODE={AICPU_PROFILING_MODE} +export PROFILING_OPTIONS={PROFILING_OPTIONS} +export FP_POINT={FP_POINT} +export BP_POINT={BP_POINT} + +# debug env +#export DUMP_GE_GRAPH=2 +#export DUMP_OP=1 +#export DUMP_OP_LESS=1 +#export PRINT_MODEL=1 +#export TE_PARALLEL_COMPILER=0 + +# system env +ulimit -c unlimited diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/train_sample.sh b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/train_sample.sh new file mode 100644 index 0000000..b4ee768 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/config/train_sample.sh @@ -0,0 +1,33 @@ +#!/bin/sh +currentDir=$(cd "$(dirname "$0")"; pwd) +cd ${currentDir} + +PWD=${currentDir} + +device_id=$1 +if [ x"${device_id}" = x ] ; +then + echo "turing train fail" >> ${currentDir}/train_${device_id}.log + exit +else + export DEVICE_ID=${device_id} +fi + +DEVICE_INDEX=$(( DEVICE_ID + RANK_INDEX * 8 )) +export DEVICE_INDEX=${DEVICE_INDEX} + +env > ${currentDir}/env_${device_id}.log + +#mkdir exec path +mkdir -p ${currentDir}/${device_id} +rm -rf ${currentDir}/${device_id}/* +cd ${currentDir}/${device_id} + +#start exec +python3.7 {RUN_ALGORITHM_CMD} {CHECKPOINT_DIR} > ${currentDir}/train_${device_id}.log 2>&1 +if [ $? -eq 0 ] ; +then + echo "turing train success" >> ${currentDir}/train_${device_id}.log +else + echo "turing train fail" >> ${currentDir}/train_${device_id}.log +fi diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/scripts/run.sh b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/scripts/run.sh new file mode 100644 index 0000000..1e18916 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/scripts/run.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +rank_size=$1 +yamlPath=$2 +toolsPath=$3 +if [ -f /.dockerenv ];then + CLUSTER=$4 + MPIRUN_ALL_IP="$5" + export CLUSTER=${CLUSTER} +fi +currentDir=$(cd "$(dirname "$0")/.."; pwd) + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "tensorflow_config") + +currtime=`date +%Y%m%d%H%M%S` +mkdir -p ${currentDir%train*}/train/result/tf_vgg16/training_job_${currtime}/ +train_job_dir=${currentDir%train*}/train/result/tf_vgg16/training_job_${currtime}/ + +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] ${train_job_dir} &" +# device 列表, 若无指定 device 或大于等于 8p 时根据 rank_size 顺序选择 +eval device_group=\$device_group_${rank_size}p +if [ x"${device_group}" == x"" ] || [ ${rank_size} -ge 8 ];then + device_group="$(seq 0 "$(expr $rank_size - 1)")" +fi + +# get last device id in device_group, hw log in performance from the dir named last_device_id +device_group_str=`echo ${device_group} | sed 's/ //g'` +first_device_id=`echo ${device_group_str: 0:1}` + +rank_id=0 + +if [ x"${CLUSTER}" == x"True" ];then + # ln hw log + ln -snf ${currentDir%train*}/train/result/tf_vgg16/training_job_${currtime}/0/hw_vgg16.log ${currentDir%train*}/train/result/tf_vgg16/training_job_${currtime}/ + this_ip=$(hostname -I |awk '{print $1}') + for ip in $MPIRUN_ALL_IP;do + if [ x"$this_ip" != x"$ip" ];then + scp $yamlPath root@$ip:$yamlPath + fi + done + export PATH=$PATH:/usr/local/mpirun4.0/bin + mpirun -H ${mpirun_ip} \ + --bind-to none -map-by slot\ + --allow-run-as-root \ + --mca btl_tcp_if_exclude lo,docker0,endvnic,virbr0,vethf40501b,docker_gwbridge,br-f42ac38052b4\ + --prefix /usr/local/mpirun4.0/ \ + ${currentDir}/scripts/train.sh 0 $rank_size $yamlPath $currtime ${toolsPath} ${CLUSTER} +else + # ln hw log + ln -snf ${currentDir%train*}/train/result/tf_vgg16/training_job_${currtime}/${first_device_id}/hw_vgg16.log ${currentDir%train*}/train/result/tf_vgg16/training_job_${currtime}/ + for device_id in $device_group;do + #echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] start: train ${device_id} & " >> ${currentDir}/result/main.log + ${currentDir}/scripts/train.sh $device_id $rank_size $yamlPath $currtime ${toolsPath} $rank_id& + let rank_id++ + done +fi +wait + + diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/scripts/train.sh b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/scripts/train.sh new file mode 100644 index 0000000..5576f9d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/scripts/train.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash + +device_id=$1 +rank_size=$2 +yamlPath=$3 +currentDir=$(cd "$(dirname "$0")/.."; pwd) +currtime=$4 +toolsPath=$5 +mkdir -p ${currentDir%train*}/train/result/tf_vgg16/training_job_${currtime}/ +export train_job_dir=${currentDir%train*}/train/result/tf_vgg16/training_job_${currtime}/ + +source ${currentDir}/config/npu_set_env.sh + +# 声明变量 +export REMARK_LOG_FILE=hw_vgg16.log # 打点日志文件名称， 必须hw_后跟模型名称小写 +# 添加日志打点模块路径 +benchmark_log_path=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils +export PYTHONPATH=$PYTHONPATH:${benchmark_log_path} + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "tensorflow_config") + +# user env +export HCCL_CONNECT_TIMEOUT=600 +export JOB_ID=9999001 +export RANK_TABLE_FILE=${currentDir}/config/${rank_size}p.json +export RANK_SIZE=${rank_size} +export RANK_INDEX=0 +export SLOG_PRINT_TO_STDOUT=0 +export DEVICE_ID=$1 +DEVICE_INDEX=$(( DEVICE_ID + RANK_INDEX * 8 )) +export DEVICE_INDEX=${DEVICE_INDEX} +export YAML_PATH=$3 + +cd ${train_job_dir} +curd_dir=${currentDir%atlas_benchmark-master*}/atlas_benchmark-master/utils/atlasboost +export PYTHONPATH=$PYTHONPATH:${curd_dir} + +if [ x"$6" != x"True" ];then + rank_id=$6 + export RANK_ID=$6 +else + device_id_mo=$(python3.7 -c "import src.tensorflow.mpi_ops as atlasboost;atlasboost.init(); \ + device_id = atlasboost.local_rank();cluster_device_id = str(device_id); \ + atlasboost.set_device_id(device_id);print(atlasboost.rank())") + device_id_mo=`echo $device_id_mo` + rank_id=${device_id_mo##* } + export RANK_ID=${rank_id} + device=${device_id_mo##*deviceid = } + device_id=${device%% phyid=*} + export DEVICE_ID=${device_id} + hccljson=${train_job_dir}/*.json + cp ${hccljson} ${currentDir}/config/${rank_size}p.json +fi + +#mkdir exec path +mkdir -p ${train_job_dir}/${device_id} +cd ${train_job_dir}/${device_id} + +startTime=`date +%Y%m%d-%H:%M:%S` +startTime_s=`date +%s` + +# 根据单卡/多卡区分调用参数 + +if [ x"$6" == x"True" ];then + export CLUSTER=True + rm -rf ${currentDir}/result/*.log + if [ ${rank_size} -le 4 ];then + # python3.7 ${currentDir}/vgg16/train.py --config_file vgg16_config_1p > ${train_job_dir}/train_${device_id}.log 2>&1 + + python3.7 ${currentDir}/vgg16/train.py --rank_size=${rank_size}\ + --mode=${mode_1p} \ + --max_train_steps=${max_train_steps} \ + --iterations_per_loop=${iterations_per_loop_1p} \ + --data_dir=${data_url} \ + --display_every=${display_every} \ + --log_dir=${log_dir} \ + --log_name=${log_name_1p}> ${train_job_dir}/train_${device_id}.log 2>&1 + else + # python3.7 ${currentDir}/vgg16/train.py --config_file vgg16_config_${rank_size}p > ${train_job_dir}/train_${device_id}.log 2>&1 + + python3.7 ${currentDir}/vgg16/train.py --rank_size=${rank_size} \ + --mode=${mode_8p} \ + --max_epochs=${max_epoches} \ + --iterations_per_loop=${iterations_per_loop_8p}\ + --epochs_between_evals=${epochs_between_evals} \ + --data_dir=${data_url} \ + --lr=${lr} \ + --log_dir=${log_dir} \ + --log_name=${log_name_8p} > ${train_job_dir}/train_${device_id}.log 2>&1 + fi + +elif [ ${rank_size} -le 4 ];then + # 单卡 + # python3.7 ${currentDir}/vgg16/train.py --config_file vgg16_config_1p > ${train_job_dir}/train_${device_id}.log 2>&1 + + python3.7 ${currentDir}/vgg16/train.py --rank_size=${rank_size} \ + --mode=${mode_1p} \ + --max_train_steps=${max_train_steps} \ + --iterations_per_loop=${iterations_per_loop_1p} \ + --data_dir=${data_url} \ + --display_every=${display_every} \ + --log_dir=${log_dir} \ + --log_name=${log_name_1p}> ${train_job_dir}/train_${device_id}.log 2>&1 + +elif [ ${rank_size} -le 8 ];then + # 多卡单机 + # python3.7 ${currentDir}/vgg16/train.py --config_file vgg16_config_8p > ${train_job_dir}/train_${device_id}.log 2>&1 + + python3.7 ${currentDir}/vgg16/train.py --rank_size=${rank_size} \ + --mode=${mode_8p} \ + --max_epochs=${max_epoches} \ + --iterations_per_loop=${iterations_per_loop_8p}\ + --epochs_between_evals=${epochs_between_evals} \ + --data_dir=${data_url} \ + --lr=${lr} \ + --log_dir=${log_dir} \ + --log_name=${log_name_8p} > ${train_job_dir}/train_${device_id}.log 2>&1 +fi + +#cp ./hw_vgg16.log ${currentDir}/../../../../performance/ + +if [ $? -eq 0 ] ;then + echo ":::ABK 1.0.0 vgg16 train success" + echo ":::ABK 1.0.0 vgg16 train success" >> ${train_job_dir}/train_${device_id}.log 2 + echo ":::ABK 1.0.0 vgg16 train success" >> ${train_job_dir}/${device_id}/hw_vgg16.log +else + echo ":::ABK 1.0.0 vgg16 train failed" + echo ":::ABK 1.0.0 vgg16 train failed" >> ${train_job_dir}/train_${device_id}.log 2 + echo ":::ABK 1.0.0 vgg16 train failed" >> ${train_job_dir}/${device_id}/hw_vgg16.log +fi + +endTime=`date +%Y%m%d-%H:%M:%S` +endTime_s=`date +%s` +sumTime=$[ $endTime_s - $startTime_s ] +hour=$(( $sumTime/3600 )) +min=$(( ($sumTime-${hour}*3600)/60 )) +sec=$(( $sumTime-${hour}*3600-${min}*60 )) +echo ${hour}:${min}:${sec} +echo ":::ABK 1.0.0 vgg16 train total time ${hour}:${min}:${sec}" >> ${train_job_dir}/${device_id}/hw_vgg16.log diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/create_session.py b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/create_session.py new file mode 100644 index 0000000..781356d --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/create_session.py @@ -0,0 +1,22 @@ +import tensorflow as tf +import os,sys + + +class CreateSession(): + def __init__(self): + self.estimator_config = tf.ConfigProto( + inter_op_parallelism_threads=10, + intra_op_parallelism_threads=10, + allow_soft_placement=True) + + self.estimator_config.gpu_options.allow_growth = True + + self.set_env() + + def set_env(self): + gpu_thread_count = 2 + os.environ['TF_GPU_THREAD_MODE'] = 'gpu_private' + os.environ['TF_GPU_THREAD_COUNT'] = str(gpu_thread_count) + os.environ['TF_USE_CUDNN_BATCHNORM_SPATIAL_PERSISTENT'] = '1' + os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = '1' + diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/data_loader.py b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/data_loader.py new file mode 100644 index 0000000..4c4a803 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/data_loader.py @@ -0,0 +1,133 @@ +import numpy as np +import preprocessing +import tensorflow as tf +from tensorflow.python.util import nest +import os,sys +import numpy as np + + +class DataLoader: + + def __init__(self, args): + self.args = args + + filename_pattern = os.path.join(args.data_dir, '%s-*') + filenames_train = sorted(tf.gfile.Glob(filename_pattern % 'train')) + self.num_training_samples = get_num_records(filenames_train) + self.args.num_training_samples = self.num_training_samples + + filename_pattern = os.path.join(args.data_dir, '%s-*') + filenames_val = sorted(tf.gfile.Glob(filename_pattern % 'validation')) + self.num_evaluating_samples = get_num_records(filenames_val) + self.args.num_evaluating_samples = self.num_evaluating_samples + + print( 'total num_training_sampels: %d' % self.num_training_samples ) + print( 'total num_evaluating_sampels: %d' % self.num_evaluating_samples ) + + self.training_samples_per_rank = self.num_training_samples + + def get_train_input_fn(self): + take_count = self.training_samples_per_rank + + return make_dataset(self.args, take_count, self.args.batch_size, training=True) + + def get_eval_input_fn(self): + take_count = self.num_evaluating_samples + + return make_dataset(self.args, take_count, self.args.batch_size, training=False) + + +def get_num_records(filenames): + def count_records(tf_record_filename): + count = 0 + for _ in tf.python_io.tf_record_iterator(tf_record_filename): + count += 1 + return count + + nfile = len(filenames) + return (count_records(filenames[0]) * (nfile - 1) + + count_records(filenames[-1])) + + +def _parse_example_proto(example_serialized): + feature_map = { + 'image/encoded': tf.FixedLenFeature([], dtype=tf.string, + default_value=''), + 'image/class/label': tf.FixedLenFeature([], dtype=tf.int64, default_value=-1), + 'image/class/text': tf.FixedLenFeature([], dtype=tf.string, + default_value=''), + } + sparse_float32 = tf.VarLenFeature(dtype=tf.float32) + # Sparse features in Example proto. + feature_map.update( + {k: sparse_float32 for k in ['image/object/bbox/xmin', + 'image/object/bbox/ymin', + 'image/object/bbox/xmax', + 'image/object/bbox/ymax']}) + + features = tf.parse_single_example(example_serialized, feature_map) + label = tf.cast(features['image/class/label'], dtype=tf.int32) + + xmin = tf.expand_dims(features['image/object/bbox/xmin'].values, 0) + ymin = tf.expand_dims(features['image/object/bbox/ymin'].values, 0) + xmax = tf.expand_dims(features['image/object/bbox/xmax'].values, 0) + ymax = tf.expand_dims(features['image/object/bbox/ymax'].values, 0) + + # Note that we impose an ordering of (y, x) just to make life difficult. + bbox = tf.concat([ymin, xmin, ymax, xmax], 0) + + # Force the variable number of bounding boxes into the shape + # [1, num_boxes, coords]. + bbox = tf.expand_dims(bbox, 0) + bbox = tf.transpose(bbox, [0, 2, 1]) + + return features['image/encoded'], label, bbox + + +# since the preprocessing is done here, we add args file +def parse_record(raw_record, is_training): + image_buffer, label, bbox = _parse_example_proto(raw_record) + + image = preprocessing.parse_and_preprocess_image_record(image_buffer, bbox, training=is_training) + + # label-1 for VGG16 + return image, label-1 + + +def make_dataset(args, take_count, batch_size, + training=False, shard=False): + + shuffle_buffer_size = 10000 + num_readers = 10 + + rank_size = int(os.getenv('RANK_SIZE')) + rank_id = int(os.getenv('DEVICE_INDEX')) + + if training: + filename_pattern = os.path.join(args.data_dir, '%s-*') + filenames = sorted(tf.gfile.Glob(filename_pattern % 'train')) + else: + filename_pattern = os.path.join(args.data_dir, '%s-*') + filenames = sorted(tf.gfile.Glob(filename_pattern % 'validation')) + + ds = tf.data.Dataset.from_tensor_slices(filenames) + + if not training: + ds = ds.take(take_count) + + if training: + ds = ds.shuffle(1000, seed=7*(1+rank_id)) + + ds = ds.interleave(tf.data.TFRecordDataset, cycle_length=num_readers, block_length=1) + counter = tf.data.Dataset.range(sys.maxsize) + ds = tf.data.Dataset.zip((ds, counter)) + + if training: + ds = ds.apply(tf.data.experimental.shuffle_and_repeat(shuffle_buffer_size, seed=5*(1+rank_id))) + + ds = ds.map(lambda image, counter: parse_record(image, training), num_parallel_calls=14) + + ds = ds.batch(batch_size, drop_remainder=True) + return ds + + diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/hyper_param.py b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/hyper_param.py new file mode 100644 index 0000000..52fd6f9 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/hyper_param.py @@ -0,0 +1,45 @@ +import tensorflow as tf +import math +import numpy as np + + +def warmup_cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch, T_max, eta_min=0): + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch) + warmup_steps = int(warmup_epochs * steps_per_epoch) + + lr_each_step = [] + for i in range(total_steps): + last_epoch = i // steps_per_epoch + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + lr = eta_min + (base_lr - eta_min) * (1. + math.cos(math.pi*last_epoch / T_max)) / 2 + lr_each_step.append(lr) + + return np.array(lr_each_step).astype(np.float32) + + +class HyperParams: + def __init__(self, args): + self.args=args + nsteps_per_epoch = self.args.num_training_samples // self.args.global_batch_size + self.args.nsteps_per_epoch = nsteps_per_epoch + if self.args.max_epochs: + nstep = nsteps_per_epoch * self.args.max_epochs + else: + nstep = self.args.max_train_steps + self.args.nstep = nstep + + self.cos_lr = warmup_cosine_annealing_lr(self.args.lr, nsteps_per_epoch, 0, self.args.T_max, self.args.T_max, 0.0) + + def get_learning_rate(self): + global_step = tf.train.get_global_step() + + learning_rate = tf.gather(tf.convert_to_tensor(self.cos_lr), global_step) + + learning_rate = tf.identity(learning_rate, 'learning_rate') + + return learning_rate + diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/layers.py b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/layers.py new file mode 100644 index 0000000..c2938bc --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/layers.py @@ -0,0 +1,22 @@ +import tensorflow as tf + + +class Layers: + def get_accuracy(self, labels, predicted_classes, logits, args): + accuracy = tf.metrics.accuracy( + labels=labels, predictions=predicted_classes) + top5acc = tf.metrics.mean( + tf.cast(tf.nn.in_top_k(logits, labels, 5), tf.float32)) + if args.rank_size == 1: + newaccuracy = (accuracy[0], accuracy[1]) + newtop5acc = (top5acc[0], top5acc[1]) + else: + from npu_bridge.hccl import hccl_ops + newaccuracy = (hccl_ops.allreduce(accuracy[0],"sum")/args.rank_size, accuracy[1]) + newtop5acc = (hccl_ops.allreduce(top5acc[0],"sum")/args.rank_size, top5acc[1]) + metrics = {'val-top1acc': newaccuracy, 'val-top5acc': newtop5acc} + return metrics + + + + diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/logger.py b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/logger.py new file mode 100644 index 0000000..13a77f0 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/logger.py @@ -0,0 +1,88 @@ +from __future__ import print_function +import tensorflow as tf +import logging +import numpy as np +import time +import sys,os + +from benchmark_log import hwlog + +class LogSessionRunHook(tf.train.SessionRunHook): + def __init__(self, args, warmup_steps=5): + self.global_batch_size = args.global_batch_size + if args.iterations_per_loop is not None: + self.iterations_per_loop = args.iterations_per_loop + else: + self.iterations_per_loop = args.nsteps_per_epoch + self.warmup_steps = warmup_steps + self.iter_times = [] + self.num_records = args.num_training_samples + self.display_every = args.display_every + self.logger = get_logger(args.log_name, args.log_dir) + rank0log(self.logger, 'PY' + str(sys.version) + 'TF' + str(tf.__version__)) + + def after_create_session(self, session, coord): + rank0log(self.logger, 'Step Epoch Speed Loss FinLoss LR') + self.elapsed_secs = 0. + self.count = 0 + + def before_run(self, run_context): + self.t0 = time.time() + return tf.train.SessionRunArgs( + fetches=[tf.train.get_global_step(), 'loss:0', 'total_loss:0', 'learning_rate:0']) + + def after_run(self, run_context, run_values): + batch_time = time.time() - self.t0 + self.iter_times.append(batch_time) + self.elapsed_secs += batch_time + self.count += 1 + global_step, loss, total_loss, lr = run_values.results + if global_step == 1 or global_step % self.display_every == 0: + dt = self.elapsed_secs / self.count + img_per_sec = self.global_batch_size * self.iterations_per_loop / dt + epoch = global_step * self.global_batch_size / self.num_records + self.logger.info('step:%6i epoch:%5.1f FPS:%7.1f loss:%6.3f total_loss:%6.3f lr:%7.5f' % + (global_step, epoch, img_per_sec, loss, total_loss, lr)) + self.elapsed_secs = 0. + self.count = 0 + + hwlog.remark_print(key=hwlog.FPS, value='%7.1f'%img_per_sec) + + def get_average_speed(self): + avg_time = np.mean(self.iter_times[self.warmup_steps:]) + speed = self.global_batch_size / avg_time + return speed + + + +def rank0log(logger, *args, **kwargs): + if logger: + logger.info(''.join([str(x) for x in list(args)])) + else: + print(*args, **kwargs) + + +def get_logger(log_name, log_dir): + logger = logging.getLogger(log_name) + logger.setLevel(logging.INFO) # INFO, ERROR + # file handler which logs debug messages + if not os.path.isdir(log_dir): + try: + os.makedirs(log_dir) + except FileExistsError: + # if log_dir is common for multiple ranks like on nfs + pass + # console handler + ch = logging.StreamHandler() + ch.setLevel(logging.INFO) + # add formatter to the handlers + formatter = logging.Formatter('%(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) + fh = logging.FileHandler(os.path.join(log_dir, log_name)) + fh.setLevel(logging.DEBUG) + fh.setFormatter(formatter) + # add handlers to logger + logger.addHandler(fh) + return logger + diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/model.py b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/model.py new file mode 100644 index 0000000..eb3b7bb --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/model.py @@ -0,0 +1,71 @@ +import tensorflow as tf +import vgg + + +class Model(object): + def __init__(self, args, data, hyper_param, layers, logger): + self.args = args + self.data = data + self.hyper_param = hyper_param + self.layers = layers + self.logger = logger + + def get_estimator_model_func(self, features, labels, mode, params=None): + labels = tf.reshape(labels, (-1,)) + + inputs = features + is_training = (mode == tf.estimator.ModeKeys.TRAIN) + + inputs = tf.cast(inputs, self.args.dtype) + + top_layer = vgg.vgg_impl(inputs, is_training) + + logits = top_layer + predicted_classes = tf.argmax(logits, axis=1, output_type=tf.int32) + logits = tf.cast(logits, tf.float32) + + labels_one_hot = tf.one_hot(labels, depth=1000) + loss = tf.losses.softmax_cross_entropy( + logits=logits, onehot_labels=labels_one_hot, label_smoothing=self.args.label_smoothing) + + base_loss = tf.identity(loss, name='loss') + + l2_loss = tf.add_n([tf.nn.l2_loss(tf.cast(v, tf.float32)) for v in tf.trainable_variables()]) + l2_loss = tf.multiply(l2_loss, self.args.weight_decay) + total_loss = base_loss + l2_loss + + total_loss = tf.identity(total_loss, name = 'total_loss') + + if mode == tf.estimator.ModeKeys.EVAL: + with tf.device(None): + metrics = self.layers.get_accuracy( labels, predicted_classes, logits, self.args) + + return tf.estimator.EstimatorSpec( + mode, loss=loss, eval_metric_ops=metrics) + + assert (mode == tf.estimator.ModeKeys.TRAIN) + + batch_size = tf.shape(inputs)[0] + + global_step = tf.train.get_global_step() + learning_rate = self.hyper_param.get_learning_rate() + + momentum = self.args.momentum + + opt = tf.train.MomentumOptimizer( + learning_rate, momentum, use_nesterov=self.args.use_nesterov) + + from npu_bridge.estimator.npu.npu_optimizer import NPUDistributedOptimizer + opt = NPUDistributedOptimizer(opt) + + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) or [] + + with tf.control_dependencies(update_ops): + gate_gradients = tf.train.Optimizer.GATE_NONE + grads_and_vars = opt.compute_gradients(total_loss, gate_gradients=gate_gradients) + train_op = opt.apply_gradients(grads_and_vars, global_step=global_step) + + train_op = tf.group(train_op) + + return tf.estimator.EstimatorSpec(mode, loss=total_loss, train_op=train_op) + diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/preprocessing.py b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/preprocessing.py new file mode 100644 index 0000000..3637496 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/preprocessing.py @@ -0,0 +1,73 @@ +import tensorflow as tf +from tensorflow.contrib.image.python.ops import distort_image_ops +import math +import random + + +def decode_jpeg(imgdata, channels=3): + return tf.image.decode_jpeg(imgdata, channels=channels, + fancy_upscaling=False, + dct_method='INTEGER_FAST') + + +def random_horizontal_flip(image, prob): + if prob > random.random(): + image = tf.image.flip_left_right(image) + return image + + +def decode_crop_and_resize(record, bbox, size, scale, ratio): + with tf.name_scope('decode_crop_and_resize'): + height = 224 + width = 224 + crop_ratio = 0.8 + initial_shape = [int(round(height / crop_ratio)), + int(round(width / crop_ratio)), 3] + jpeg_shape = tf.image.extract_jpeg_shape( record ) + + bbox_begin, bbox_size, bbox = \ + tf.image.sample_distorted_bounding_box( + tf.image.extract_jpeg_shape(record), + bounding_boxes=bbox, + min_object_covered=0.1, + aspect_ratio_range=ratio, + area_range=scale, + max_attempts=10, + use_image_if_no_bounding_boxes=True) + + # Reassemble the bounding box in the format the crop op requires. + offset_y, offset_x, _ = tf.unstack(bbox_begin) + target_height, target_width, _ = tf.unstack(bbox_size) + crop_window = tf.stack([offset_y, offset_x, target_height, target_width]) + + image = tf.image.decode_and_crop_jpeg( record, crop_window, channels=3 ) + image = tf.image.resize_images( image, [height, width] ) + + return image + + +def parse_and_preprocess_image_record(record, bbox, training): + with tf.name_scope('preprocess'): + if training: + image = decode_crop_and_resize(record, bbox, 224, (0.08, 1.0), (0.75, 1.333)) + image = random_horizontal_flip(image, 0.5) + image = normalize(image) + else: + image = decode_jpeg(record, channels=3) + image = tf.image.resize_images(image, [256, 256]) + image = tf.image.central_crop(image, 224.0/256) + image = normalize(image) + + return image + + +def normalize(inputs): + imagenet_mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] + imagenet_std = [0.229 * 255, 0.224 * 255, 0.225 * 255] + imagenet_mean = tf.expand_dims(tf.expand_dims(imagenet_mean, 0), 0) + imagenet_std = tf.expand_dims(tf.expand_dims(imagenet_std, 0), 0) + inputs = inputs - imagenet_mean + inputs = inputs * (1.0 / imagenet_std) + + return inputs + diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/train.py b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/train.py new file mode 100644 index 0000000..901eeae --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/train.py @@ -0,0 +1,143 @@ +import tensorflow as tf +import numpy as np +import os +import sys +import ast + +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '../'))) +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '../config'))) +sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)),'../../../../utils')) +sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)),'../../../../utils/atlasboost')) + + +import data_loader as dl +import model as ml +import hyper_param as hp +import layers as ly +import logger as lg +import trainer as tr +import create_session as cs + +print(os.getcwd()) + +import argparse + +#import hwlog +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter + + +def parse_args(): + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + + parser.add_argument('--rank_size', default=1,type=int, + help="""number of NPUs to use.""") + + # mode and parameters related + parser.add_argument('--mode', default='train_and_evaluate', + help="""mode to run the program e.g. train, evaluate, and + train_and_evaluate""") + parser.add_argument('--max_train_steps', default=100,type=int, + help="""max steps to train""") + parser.add_argument('--iterations_per_loop', default=10, type=int, + help="""the number of steps in devices for each iteration""") + parser.add_argument('--max_epochs', default=None, type=int, + help="""total epochs for training""") + parser.add_argument('--epochs_between_evals', default=5, type=int, + help="""the interval between train and evaluation , only meaningful + when the mode is train_and_evaluate""") + + # dataset + parser.add_argument('--data_dir', default='path/data', + help="""directory of dataset.""") + + # path for evaluation + parser.add_argument('--eval_dir', default='path/eval', + help="""directory to evaluate.""") + + parser.add_argument('--dtype', default=tf.float32, + help="""data type of inputs.""") + parser.add_argument('--use_nesterov', default=True, type=ast.literal_eval, + help="""whether to use Nesterov in optimizer""") + parser.add_argument('--label_smoothing', default=0.1, type=float, + help="""label smoothing factor""") + parser.add_argument('--weight_decay', default=0.0001, + help="""weight decay for regularization""") + parser.add_argument('--batch_size', default=32, type=int, + help="""batch size for one NPU""") + + # learning rate and momentum + parser.add_argument('--lr', default=0.01, type=float, + help="""initial learning rate""") + parser.add_argument('--T_max', default=150, type=int, + help="""T_max for cosing_annealing learning rate""") + parser.add_argument('--momentum', default=0.9, type=float, + help="""momentum used in optimizer.""") + + # display frequency + parser.add_argument('--display_every', default=1, type=int, + help="""the frequency to display info""") + + # log file + parser.add_argument('--log_name', default='vgg16.log', + help="""name of log file""") + parser.add_argument('--log_dir', default='./model_1p', + help="""log directory""") + + args, unknown_args = parser.parse_known_args() + if len(unknown_args) > 0: + for bad_arg in unknown_args: + print("ERROR: Unknown command line arg: %s" % bad_arg) + raise ValueError("Invalid command line arg(s)") + + return args + + +def main(): + + args = parse_args() + args.global_batch_size = args.batch_size * args.rank_size + + session = cs.CreateSession() + data = dl.DataLoader(args) + hyper_param = hp.HyperParams(args) + layers = ly.Layers() + logger = lg.LogSessionRunHook(args) + model = ml.Model(args, data, hyper_param, layers, logger) + + trainer = tr.Trainer(session, args, data, model, logger) + + if args.mode == 'train': + trainer.train() + elif args.mode == 'evaluate': + trainer.evaluate() + elif args.mode == 'train_and_evaluate': + trainer.train_and_evaluate() + else: + raise ValueError("Invalid mode.") + + +if __name__ == '__main__': + + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("tensorflow") + config_info = get_model_parameter("tensorflow_config") + initinal_data = {"base_lr": 0.01, "dataset": "imagenet1024", "optimizer": "SGD", "loss_scale": 512, + "batchsize": 32} + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + hwlog.remark_print(key=hwlog.INPUT_BATCH_SIZE, value=initinal_data.get("batchsize")) + + tf.logging.set_verbosity(tf.logging.INFO) + main() + diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/train_helper.py b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/train_helper.py new file mode 100644 index 0000000..9d3d3de --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/train_helper.py @@ -0,0 +1,21 @@ +import tensorflow as tf +from tensorflow.python.ops import data_flow_ops +import re +import os +from operator import itemgetter + + +def sort_and_load_ckpts(log_dir): + ckpts = [] + for f in os.listdir(log_dir): + m = re.match(r'model.ckpt-([0-9]+).index', f) + if m is None: + continue + fullpath = os.path.join(log_dir, f) + ckpts.append({'step': int(m.group(1)), + 'path': os.path.splitext(fullpath)[0], + 'mtime': os.stat(fullpath).st_mtime, + }) + ckpts.sort(key=itemgetter('step')) + return ckpts + diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/trainer.py b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/trainer.py new file mode 100644 index 0000000..7bcbcad --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/trainer.py @@ -0,0 +1,142 @@ +import tensorflow as tf +import math +import time +import os +import train_helper +from logger import rank0log + +import sys +sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)),'../../../../utils')) +from benchmark_log import hwlog + + +class Trainer(object): + def __init__(self, session, args, data, model, logger): + self.sess = session + self.args = args + self.data = data + self.model = model + self.logger = logger + self.print_logger = self.logger.logger + self.all_preds = [] + self.all_targets = [] + + self.classifier, self.training_hook = self.get_npu_classifier() + + def get_npu_classifier(self): + from npu_bridge.estimator.npu.npu_config import NPURunConfig + from npu_bridge.estimator.npu.npu_estimator import NPUEstimator + + run_config = NPURunConfig( + hcom_parallel=True, + precision_mode="allow_mix_precision", + enable_data_pre_proc=True, + save_checkpoints_steps=self.args.nsteps_per_epoch, + session_config=self.sess.estimator_config, + model_dir=self.args.log_dir, + iterations_per_loop=self.args.iterations_per_loop, + keep_checkpoint_max=5) + + classifier =NPUEstimator( + model_fn= self.model.get_estimator_model_func, + config= run_config + ) + + training_hooks = [] + training_hooks.append(self.logger) + + return classifier, training_hooks + + def train(self): + + hwlog.remark_print(key=hwlog.CURRENT_EPOCH, value=self.args.max_epochs) + + print ('training steps: %d' % self.args.nstep) + self.classifier.train( input_fn=lambda:self.data.get_train_input_fn(), + max_steps = self.args.nstep, + hooks = self.training_hook + ) + + def evaluate(self): + rank0log(self.print_logger, "Evaluating") + rank0log(self.print_logger, "Validation dataset size: {}".format(self.args.num_evaluating_samples)) + time.sleep(5) # a little extra margin... + try: + ckpts = train_helper.sort_and_load_ckpts(self.args.eval_dir) + print("=========ckpt==========") + print(ckpts) + print("=========ckpt==========") + for i, c in enumerate(ckpts): + eval_result = self.classifier.evaluate( + input_fn=lambda: self.data.get_eval_input_fn(), + checkpoint_path=c['path']) + + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value=float(eval_result.get("val-top1acc"))) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value=float(eval_result.get("val-top5acc"))) + + + c['epoch'] = math.ceil(c['step'] / (self.args.num_training_samples/ (self.args.batch_size))) + c['top1'] = eval_result['val-top1acc'] + c['top5'] = eval_result['val-top5acc'] + c['loss'] = eval_result['loss'] + + rank0log(self.print_logger, ' step epoch top1 top5 loss checkpoint_time(UTC)') + for i, c in enumerate(ckpts): + if 'top1' not in c: + continue + rank0log(self.print_logger,'{:5d} {:5.1f} {:5.3f} {:6.2f} {:6.2f} {time}' + .format(c['step'], + c['epoch'], + c['top1'] * 100, + c['top5'] * 100, + c['loss'], + time=time.strftime('%Y-%m-%d %H:%M:%S', + time.localtime(c['mtime'])))) + rank0log(self.print_logger, "Finished evaluation") + except KeyboardInterrupt: + self.print_logger.error("Keyboard interrupt") + + def train_and_evaluate(self): + epochs_between_evals = self.args.epochs_between_evals + + for i in range(self.args.max_epochs // epochs_between_evals): + + rank0log(self.print_logger, "Starting a training cycle") + + hwlog.remark_print(key=hwlog.CURRENT_EPOCH, value=self.args.max_epochs) + + + self.classifier.train(input_fn=lambda:self.data.get_train_input_fn(), + steps = self.args.nsteps_per_epoch*epochs_between_evals, + hooks = self.training_hook ) + + rank0log(self.print_logger, "Starting to evaluate") + rank0log(self.print_logger, "Validation dataset size: {}".format(self.args.num_evaluating_samples)) + time.sleep(5) # a little extra margin... + + ckpts = train_helper.sort_and_load_ckpts(self.args.log_dir) + c = ckpts[-1] + eval_result = self.classifier.evaluate( + input_fn=lambda: self.data.get_eval_input_fn(), + checkpoint_path=c['path']) + + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP1, value=float(eval_result.get("val-top1acc"))) + hwlog.remark_print(key=hwlog.EVAL_ACCURACY_TOP5, value=float(eval_result.get("val-top5acc"))) + + + c['epoch'] = math.ceil(c['step'] / (self.args.num_training_samples / (self.args.batch_size * self.args.rank_size))) + c['top1'] = eval_result['val-top1acc'] + c['top5'] = eval_result['val-top5acc'] + c['loss'] = eval_result['loss'] + + rank0log(self.print_logger, ' step epoch top1 top5 loss checkpoint_time(UTC)') + + rank0log(self.print_logger,'{:5d} {:5.1f} {:5.3f} {:6.2f} {:6.2f} {time}' + .format(c['step'], + c['epoch'], + c['top1'] * 100, + c['top5'] * 100, + c['loss'], + time=time.strftime('%Y-%m-%d %H:%M:%S', + time.localtime(c['mtime'])))) + diff --git a/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/vgg.py b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/vgg.py new file mode 100644 index 0000000..7f38b85 --- /dev/null +++ b/train/atlas_benchmark-master/image_classification/vgg16/tensorflow/vgg16/vgg.py @@ -0,0 +1,63 @@ +import tensorflow as tf + +from npu_bridge.estimator import npu_ops + +# vgg with initialization method in gluoncv +def vgg_impl(inputs, is_training=True): + x = inputs + + # conv1 + x = tf.layers.conv2d(x, 64, kernel_size=(3, 3), activation=tf.nn.relu, padding='SAME', use_bias=True, kernel_initializer=tf.initializers.variance_scaling(scale=2.0, mode='fan_out')) + x = tf.layers.conv2d(x, 64, kernel_size=(3, 3), activation=tf.nn.relu, padding='SAME', use_bias=True, kernel_initializer=tf.initializers.variance_scaling(scale=2.0, mode='fan_out')) + + # mp1 + x = tf.layers.max_pooling2d(x, (2, 2), (2, 2), padding='SAME') + + # covn2 + x = tf.layers.conv2d(x, 128, kernel_size=(3, 3), activation=tf.nn.relu, padding='SAME', use_bias=True, kernel_initializer=tf.initializers.variance_scaling(scale=2.0, mode='fan_out')) + x = tf.layers.conv2d(x, 128, kernel_size=(3, 3), activation=tf.nn.relu, padding='SAME', use_bias=True, kernel_initializer=tf.initializers.variance_scaling(scale=2.0, mode='fan_out')) + + # mp2 + x = tf.layers.max_pooling2d(x, (2, 2), (2, 2), padding='SAME') + + # conv3 + x = tf.layers.conv2d(x, 256, kernel_size=(3, 3), activation=tf.nn.relu, padding='SAME', use_bias=True, kernel_initializer=tf.initializers.variance_scaling(scale=2.0, mode='fan_out')) + x = tf.layers.conv2d(x, 256, kernel_size=(3, 3), activation=tf.nn.relu, padding='SAME', use_bias=True, kernel_initializer=tf.initializers.variance_scaling(scale=2.0, mode='fan_out')) + x = tf.layers.conv2d(x, 256, kernel_size=(3, 3), activation=tf.nn.relu, padding='SAME', use_bias=True, kernel_initializer=tf.initializers.variance_scaling(scale=2.0, mode='fan_out')) + + # mp3 + x = tf.layers.max_pooling2d(x, (2, 2), (2, 2), padding='SAME') + + # conv4 + x = tf.layers.conv2d(x, 512, kernel_size=(3, 3), activation=tf.nn.relu, padding='SAME', use_bias=True, kernel_initializer=tf.initializers.variance_scaling(scale=2.0, mode='fan_out')) + x = tf.layers.conv2d(x, 512, kernel_size=(3, 3), activation=tf.nn.relu, padding='SAME', use_bias=True, kernel_initializer=tf.initializers.variance_scaling(scale=2.0, mode='fan_out')) + x = tf.layers.conv2d(x, 512, kernel_size=(3, 3), activation=tf.nn.relu, padding='SAME', use_bias=True, kernel_initializer=tf.initializers.variance_scaling(scale=2.0, mode='fan_out')) + + # mp4 + x = tf.layers.max_pooling2d(x, (2, 2), (2, 2), padding='SAME') + + # conv5 + x = tf.layers.conv2d(x, 512, kernel_size=(3, 3), activation=tf.nn.relu, padding='SAME', use_bias=True, kernel_initializer=tf.initializers.variance_scaling(scale=2.0, mode='fan_out')) + x = tf.layers.conv2d(x, 512, kernel_size=(3, 3), activation=tf.nn.relu, padding='SAME', use_bias=True, kernel_initializer=tf.initializers.variance_scaling(scale=2.0, mode='fan_out')) + x = tf.layers.conv2d(x, 512, kernel_size=(3, 3), activation=tf.nn.relu, padding='SAME', use_bias=True, kernel_initializer=tf.initializers.variance_scaling(scale=2.0, mode='fan_out')) + + # mp5 + x = tf.layers.max_pooling2d(x, (2, 2), (2, 2), padding='SAME') + + x = tf.reshape(x, [-1, 7 * 7 * 512]) + + # fc6 + x = tf.layers.dense(x, 4096, activation=tf.nn.relu, use_bias=True, kernel_initializer=tf.keras.initializers.RandomNormal(stddev=0.01)) + # drop6 + if is_training: + x = npu_ops.dropout(x, 0.5) + # fc7 + x = tf.layers.dense(x, 4096, activation=tf.nn.relu, use_bias=True, kernel_initializer=tf.keras.initializers.RandomNormal(stddev=0.01)) + # drop7 + if is_training: + x = npu_ops.dropout(x, 0.5) + # fc8 + x = tf.layers.dense(x, 1000, activation=None, use_bias=True, kernel_initializer=tf.keras.initializers.RandomNormal(stddev=0.01)) + + return x + diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/README.md b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/README.md new file mode 100644 index 0000000..427e3da --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/README.md @@ -0,0 +1,56 @@ +# Bert-Base_tensorflow训练说明 + +### 1. 模型训练参数配置 + +在train/yaml/Bert-Base.yaml中修改相应配置， 配置项含义: + +``` + tensorflow_config: + #layer层数有6和12两种，中文数据集用 bert_base_layer6_cn.json/bert_base_layer12_cn.json 英文用bert_base_layer6_cn.json/bert_base_layer12_en.json + bert_config_file: bert_base_layer6_cn.json + #数据集句子长度是256时 设置为 256,40，句子长度是128时设置为128,20 + max_seq_length: 128 + max_predictions_per_seq: 20 + + # 最佳性能train_batch_size为160 + train_batch_size: 160 + learning_rate: 1e-4 + num_warmup_steps: 100 + num_train_steps: 1000 + optimizer_type: adam + manual_fp16: True + use_fp16_cls: True + input_files_dir: 数据集路径 + eval_files_dir: 数据集路径 + npu_bert_debug: False + npu_bert_use_tdt: True + distributed: True + do_train: True + do_eval: False + num_accumulation_steps: 1 + iterations_per_loop: 100 + npu_bert_loss_scale: 0 + save_checkpoints_steps: 1000 + npu_bert_clip_by_global_norm: False + + # docker 镜像名称:版本号 + docker_image: c73:b021 + + # 仅多机执行需要配置: ip1:卡数量1,ip2:卡数量2 + mpirun_ip: 90.90.140.199:8,90.90.140.229:8 + + # 指定 device id, 多个 id 使用空格分隔, 数量需与 rank_size 相同 + device_group_1p: 6 + device_group_2p: 0 1 + device_group_4p: 0 1 2 3 +``` + +------ + + + + + + + + diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/__init__.py b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/__init__.py new file mode 100644 index 0000000..effb57b --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/__init__.py @@ -0,0 +1,15 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/bert_base_config.json b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/bert_base_config.json new file mode 100644 index 0000000..012ec3b --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/bert_base_config.json @@ -0,0 +1,13 @@ +{ + "attention_probs_dropout_prob": 0.1, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 768, + "initializer_range": 0.02, + "intermediate_size": 3072, + "max_position_embeddings": 512, + "num_attention_heads": 12, + "num_hidden_layers": 12, + "type_vocab_size": 2, + "vocab_size": 30522 +} diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/bert_base_vocab.txt b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/bert_base_vocab.txt new file mode 100644 index 0000000..ca4f978 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/bert_base_vocab.txt @@ -0,0 +1,21128 @@ +[PAD] +[unused1] +[unused2] +[unused3] +[unused4] +[unused5] +[unused6] +[unused7] +[unused8] +[unused9] +[unused10] +[unused11] +[unused12] +[unused13] +[unused14] +[unused15] +[unused16] +[unused17] +[unused18] +[unused19] +[unused20] +[unused21] +[unused22] +[unused23] +[unused24] +[unused25] +[unused26] +[unused27] +[unused28] +[unused29] +[unused30] +[unused31] +[unused32] +[unused33] +[unused34] +[unused35] +[unused36] +[unused37] +[unused38] +[unused39] +[unused40] +[unused41] +[unused42] +[unused43] +[unused44] +[unused45] +[unused46] +[unused47] +[unused48] +[unused49] +[unused50] +[unused51] +[unused52] +[unused53] +[unused54] +[unused55] +[unused56] +[unused57] +[unused58] +[unused59] +[unused60] +[unused61] +[unused62] +[unused63] +[unused64] +[unused65] +[unused66] +[unused67] +[unused68] +[unused69] +[unused70] +[unused71] +[unused72] +[unused73] +[unused74] +[unused75] +[unused76] +[unused77] +[unused78] +[unused79] +[unused80] +[unused81] +[unused82] +[unused83] +[unused84] +[unused85] +[unused86] +[unused87] +[unused88] +[unused89] +[unused90] +[unused91] +[unused92] +[unused93] +[unused94] +[unused95] +[unused96] +[unused97] +[unused98] +[unused99] +[UNK] +[CLS] +[SEP] +[MASK] + + +! +" +# +$ +% +& +' +( +) +* ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; +< += +> +? +@ +[ +\ +] +^ +_ +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +{ +| +} +~ +£ +¤ +¥ +§ +© +« +® +° +± +² +³ +µ +· +¹ +º +» +¼ +× +ß +æ +÷ +ø +đ +ŋ +ɔ +ə +ɡ +ʰ +ˇ +ˈ +ˊ +ˋ +ˍ +ː +˙ +˚ +ˢ +α +β +γ +δ +ε +η +θ +ι +κ +λ +μ +ν +ο +π +ρ +ς +σ +τ +υ +φ +χ +ψ +ω +а +б +в +г +д +е +ж +з +и +к +л +м +н +о +п +р +с +т +у +ф +х +ц +ч +ш +ы +ь +я +і +ا +ب +ة +ت +د +ر +س +ع +ل +م +ن +ه +و +ي +۩ +ก +ง +น +ม +ย +ร +อ +า +เ +๑ +་ +ღ +ᄀ +ᄁ +ᄂ +ᄃ +ᄅ +ᄆ +ᄇ +ᄈ +ᄉ +ᄋ +ᄌ +ᄎ +ᄏ +ᄐ +ᄑ +ᄒ +ᅡ +ᅢ +ᅣ +ᅥ +ᅦ +ᅧ +ᅨ +ᅩ +ᅪ +ᅬ +ᅭ +ᅮ +ᅯ +ᅲ +ᅳ +ᅴ +ᅵ +ᆨ +ᆫ +ᆯ +ᆷ +ᆸ +ᆺ +ᆻ +ᆼ +ᗜ +ᵃ +ᵉ +ᵍ +ᵏ +ᵐ +ᵒ +ᵘ +‖ +„ +† +• +‥ +‧ +
 +‰ +′ +″ +‹ +› +※ +‿ +⁄ +ⁱ +⁺ +ⁿ +₁ +₂ +₃ +₄ +€ +℃ +№ +™ +ⅰ +ⅱ +ⅲ +ⅳ +ⅴ +← +↑ +→ +↓ +↔ +↗ +↘ +⇒ +∀ +− +∕ +∙ +√ +∞ +∟ +∠ +∣ +∥ +∩ +∮ +∶ +∼ +∽ +≈ +≒ +≡ +≤ +≥ +≦ +≧ +≪ +≫ +⊙ +⋅ +⋈ +⋯ +⌒ +① +② +③ +④ +⑤ +⑥ +⑦ +⑧ +⑨ +⑩ +⑴ +⑵ +⑶ +⑷ +⑸ +⒈ +⒉ +⒊ +⒋ +ⓒ +ⓔ +ⓘ +─ +━ +│ +┃ +┅ +┆ +┊ +┌ +└ +├ +┣ +═ +║ +╚ +╞ +╠ +╭ +╮ +╯ +╰ +╱ +╳ +▂ +▃ +▅ +▇ +█ +▉ +▋ +▌ +▍ +▎ +■ +□ +▪ +▫ +▬ +▲ +△ +▶ +► +▼ +▽ +◆ +◇ +○ +◎ +● +◕ +◠ +◢ +◤ +☀ +★ +☆ +☕ +☞ +☺ +☼ +♀ +♂ +♠ +♡ +♣ +♥ +♦ +♪ +♫ +♬ +✈ +✔ +✕ +✖ +✦ +✨ +✪ +✰ +✿ +❀ +❤ +➜ +➤ +⦿ +、 +。 +〃 +々 +〇 +〈 +〉 +《 +》 +「 +」 +『 +』 +【 +】 +〓 +〔 +〕 +〖 +〗 +〜 +〝 +〞 +ぁ +あ +ぃ +い +う +ぇ +え +お +か +き +く +け +こ +さ +し +す +せ +そ +た +ち +っ +つ +て +と +な +に +ぬ +ね +の +は +ひ +ふ +へ +ほ +ま +み +む +め +も +ゃ +や +ゅ +ゆ +ょ +よ +ら +り +る +れ +ろ +わ +を +ん +゜ +ゝ +ァ +ア +ィ +イ +ゥ +ウ +ェ +エ +ォ +オ +カ +キ +ク +ケ +コ +サ +シ +ス +セ +ソ +タ +チ +ッ +ツ +テ +ト +ナ +ニ +ヌ +ネ +ノ +ハ +ヒ +フ +ヘ +ホ +マ +ミ +ム +メ +モ +ャ +ヤ +ュ +ユ +ョ +ヨ +ラ +リ +ル +レ +ロ +ワ +ヲ +ン +ヶ +・ +ー +ヽ +ㄅ +ㄆ +ㄇ +ㄉ +ㄋ +ㄌ +ㄍ +ㄎ +ㄏ +ㄒ +ㄚ +ㄛ +ㄞ +ㄟ +ㄢ +ㄤ +ㄥ +ㄧ +ㄨ +ㆍ +㈦ +㊣ +㎡ +㗎 +一 +丁 +七 +万 +丈 +三 +上 +下 +不 +与 +丐 +丑 +专 +且 +丕 +世 +丘 +丙 +业 +丛 +东 +丝 +丞 +丟 +両 +丢 +两 +严 +並 +丧 +丨 +个 +丫 +中 +丰 +串 +临 +丶 +丸 +丹 +为 +主 +丼 +丽 +举 +丿 +乂 +乃 +久 +么 +义 +之 +乌 +乍 +乎 +乏 +乐 +乒 +乓 +乔 +乖 +乗 +乘 +乙 +乜 +九 +乞 +也 +习 +乡 +书 +乩 +买 +乱 +乳 +乾 +亀 +亂 +了 +予 +争 +事 +二 +于 +亏 +云 +互 +五 +井 +亘 +亙 +亚 +些 +亜 +亞 +亟 +亡 +亢 +交 +亥 +亦 +产 +亨 +亩 +享 +京 +亭 +亮 +亲 +亳 +亵 +人 +亿 +什 +仁 +仃 +仄 +仅 +仆 +仇 +今 +介 +仍 +从 +仏 +仑 +仓 +仔 +仕 +他 +仗 +付 +仙 +仝 +仞 +仟 +代 +令 +以 +仨 +仪 +们 +仮 +仰 +仲 +件 +价 +任 +份 +仿 +企 +伉 +伊 +伍 +伎 +伏 +伐 +休 +伕 +众 +优 +伙 +会 +伝 +伞 +伟 +传 +伢 +伤 +伦 +伪 +伫 +伯 +估 +伴 +伶 +伸 +伺 +似 +伽 +佃 +但 +佇 +佈 +位 +低 +住 +佐 +佑 +体 +佔 +何 +佗 +佘 +余 +佚 +佛 +作 +佝 +佞 +佟 +你 +佢 +佣 +佤 +佥 +佩 +佬 +佯 +佰 +佳 +併 +佶 +佻 +佼 +使 +侃 +侄 +來 +侈 +例 +侍 +侏 +侑 +侖 +侗 +供 +依 +侠 +価 +侣 +侥 +侦 +侧 +侨 +侬 +侮 +侯 +侵 +侶 +侷 +便 +係 +促 +俄 +俊 +俎 +俏 +俐 +俑 +俗 +俘 +俚 +保 +俞 +俟 +俠 +信 +俨 +俩 +俪 +俬 +俭 +修 +俯 +俱 +俳 +俸 +俺 +俾 +倆 +倉 +個 +倌 +倍 +倏 +們 +倒 +倔 +倖 +倘 +候 +倚 +倜 +借 +倡 +値 +倦 +倩 +倪 +倫 +倬 +倭 +倶 +债 +值 +倾 +偃 +假 +偈 +偉 +偌 +偎 +偏 +偕 +做 +停 +健 +側 +偵 +偶 +偷 +偻 +偽 +偿 +傀 +傅 +傍 +傑 +傘 +備 +傚 +傢 +傣 +傥 +储 +傩 +催 +傭 +傲 +傳 +債 +傷 +傻 +傾 +僅 +働 +像 +僑 +僕 +僖 +僚 +僥 +僧 +僭 +僮 +僱 +僵 +價 +僻 +儀 +儂 +億 +儆 +儉 +儋 +儒 +儕 +儘 +償 +儡 +優 +儲 +儷 +儼 +儿 +兀 +允 +元 +兄 +充 +兆 +兇 +先 +光 +克 +兌 +免 +児 +兑 +兒 +兔 +兖 +党 +兜 +兢 +入 +內 +全 +兩 +八 +公 +六 +兮 +兰 +共 +兲 +关 +兴 +兵 +其 +具 +典 +兹 +养 +兼 +兽 +冀 +内 +円 +冇 +冈 +冉 +冊 +册 +再 +冏 +冒 +冕 +冗 +写 +军 +农 +冠 +冢 +冤 +冥 +冨 +冪 +冬 +冯 +冰 +冲 +决 +况 +冶 +冷 +冻 +冼 +冽 +冾 +净 +凄 +准 +凇 +凈 +凉 +凋 +凌 +凍 +减 +凑 +凛 +凜 +凝 +几 +凡 +凤 +処 +凪 +凭 +凯 +凰 +凱 +凳 +凶 +凸 +凹 +出 +击 +函 +凿 +刀 +刁 +刃 +分 +切 +刈 +刊 +刍 +刎 +刑 +划 +列 +刘 +则 +刚 +创 +初 +删 +判 +別 +刨 +利 +刪 +别 +刮 +到 +制 +刷 +券 +刹 +刺 +刻 +刽 +剁 +剂 +剃 +則 +剉 +削 +剋 +剌 +前 +剎 +剐 +剑 +剔 +剖 +剛 +剜 +剝 +剣 +剤 +剥 +剧 +剩 +剪 +副 +割 +創 +剷 +剽 +剿 +劃 +劇 +劈 +劉 +劊 +劍 +劏 +劑 +力 +劝 +办 +功 +加 +务 +劣 +动 +助 +努 +劫 +劭 +励 +劲 +劳 +労 +劵 +効 +劾 +势 +勁 +勃 +勇 +勉 +勋 +勐 +勒 +動 +勖 +勘 +務 +勛 +勝 +勞 +募 +勢 +勤 +勧 +勳 +勵 +勸 +勺 +勻 +勾 +勿 +匀 +包 +匆 +匈 +匍 +匐 +匕 +化 +北 +匙 +匝 +匠 +匡 +匣 +匪 +匮 +匯 +匱 +匹 +区 +医 +匾 +匿 +區 +十 +千 +卅 +升 +午 +卉 +半 +卍 +华 +协 +卑 +卒 +卓 +協 +单 +卖 +南 +単 +博 +卜 +卞 +卟 +占 +卡 +卢 +卤 +卦 +卧 +卫 +卮 +卯 +印 +危 +即 +却 +卵 +卷 +卸 +卻 +卿 +厂 +厄 +厅 +历 +厉 +压 +厌 +厕 +厘 +厚 +厝 +原 +厢 +厥 +厦 +厨 +厩 +厭 +厮 +厲 +厳 +去 +县 +叁 +参 +參 +又 +叉 +及 +友 +双 +反 +収 +发 +叔 +取 +受 +变 +叙 +叛 +叟 +叠 +叡 +叢 +口 +古 +句 +另 +叨 +叩 +只 +叫 +召 +叭 +叮 +可 +台 +叱 +史 +右 +叵 +叶 +号 +司 +叹 +叻 +叼 +叽 +吁 +吃 +各 +吆 +合 +吉 +吊 +吋 +同 +名 +后 +吏 +吐 +向 +吒 +吓 +吕 +吖 +吗 +君 +吝 +吞 +吟 +吠 +吡 +否 +吧 +吨 +吩 +含 +听 +吭 +吮 +启 +吱 +吳 +吴 +吵 +吶 +吸 +吹 +吻 +吼 +吽 +吾 +呀 +呂 +呃 +呆 +呈 +告 +呋 +呎 +呐 +呓 +呕 +呗 +员 +呛 +呜 +呢 +呤 +呦 +周 +呱 +呲 +味 +呵 +呷 +呸 +呻 +呼 +命 +咀 +咁 +咂 +咄 +咆 +咋 +和 +咎 +咏 +咐 +咒 +咔 +咕 +咖 +咗 +咘 +咙 +咚 +咛 +咣 +咤 +咦 +咧 +咨 +咩 +咪 +咫 +咬 +咭 +咯 +咱 +咲 +咳 +咸 +咻 +咽 +咿 +哀 +品 +哂 +哄 +哆 +哇 +哈 +哉 +哋 +哌 +响 +哎 +哏 +哐 +哑 +哒 +哔 +哗 +哟 +員 +哥 +哦 +哧 +哨 +哩 +哪 +哭 +哮 +哲 +哺 +哼 +哽 +唁 +唄 +唆 +唇 +唉 +唏 +唐 +唑 +唔 +唠 +唤 +唧 +唬 +售 +唯 +唰 +唱 +唳 +唷 +唸 +唾 +啃 +啄 +商 +啉 +啊 +問 +啓 +啕 +啖 +啜 +啞 +啟 +啡 +啤 +啥 +啦 +啧 +啪 +啫 +啬 +啮 +啰 +啱 +啲 +啵 +啶 +啷 +啸 +啻 +啼 +啾 +喀 +喂 +喃 +善 +喆 +喇 +喉 +喊 +喋 +喎 +喏 +喔 +喘 +喙 +喚 +喜 +喝 +喟 +喧 +喪 +喫 +喬 +單 +喰 +喱 +喲 +喳 +喵 +営 +喷 +喹 +喺 +喻 +喽 +嗅 +嗆 +嗇 +嗎 +嗑 +嗒 +嗓 +嗔 +嗖 +嗚 +嗜 +嗝 +嗟 +嗡 +嗣 +嗤 +嗦 +嗨 +嗪 +嗬 +嗯 +嗰 +嗲 +嗳 +嗶 +嗷 +嗽 +嘀 +嘅 +嘆 +嘈 +嘉 +嘌 +嘍 +嘎 +嘔 +嘖 +嘗 +嘘 +嘚 +嘛 +嘜 +嘞 +嘟 +嘢 +嘣 +嘤 +嘧 +嘩 +嘭 +嘮 +嘯 +嘰 +嘱 +嘲 +嘴 +嘶 +嘸 +嘹 +嘻 +嘿 +噁 +噌 +噎 +噓 +噔 +噗 +噙 +噜 +噠 +噢 +噤 +器 +噩 +噪 +噬 +噱 +噴 +噶 +噸 +噹 +噻 +噼 +嚀 +嚇 +嚎 +嚏 +嚐 +嚓 +嚕 +嚟 +嚣 +嚥 +嚨 +嚮 +嚴 +嚷 +嚼 +囂 +囉 +囊 +囍 +囑 +囔 +囗 +囚 +四 +囝 +回 +囟 +因 +囡 +团 +団 +囤 +囧 +囪 +囫 +园 +困 +囱 +囲 +図 +围 +囹 +固 +国 +图 +囿 +圃 +圄 +圆 +圈 +國 +圍 +圏 +園 +圓 +圖 +團 +圜 +土 +圣 +圧 +在 +圩 +圭 +地 +圳 +场 +圻 +圾 +址 +坂 +均 +坊 +坍 +坎 +坏 +坐 +坑 +块 +坚 +坛 +坝 +坞 +坟 +坠 +坡 +坤 +坦 +坨 +坪 +坯 +坳 +坵 +坷 +垂 +垃 +垄 +型 +垒 +垚 +垛 +垠 +垢 +垣 +垦 +垩 +垫 +垭 +垮 +垵 +埂 +埃 +埋 +城 +埔 +埕 +埗 +域 +埠 +埤 +埵 +執 +埸 +培 +基 +埼 +堀 +堂 +堃 +堅 +堆 +堇 +堑 +堕 +堙 +堡 +堤 +堪 +堯 +堰 +報 +場 +堵 +堺 +堿 +塊 +塌 +塑 +塔 +塗 +塘 +塚 +塞 +塢 +塩 +填 +塬 +塭 +塵 +塾 +墀 +境 +墅 +墉 +墊 +墒 +墓 +増 +墘 +墙 +墜 +增 +墟 +墨 +墩 +墮 +墳 +墻 +墾 +壁 +壅 +壆 +壇 +壊 +壑 +壓 +壕 +壘 +壞 +壟 +壢 +壤 +壩 +士 +壬 +壮 +壯 +声 +売 +壳 +壶 +壹 +壺 +壽 +处 +备 +変 +复 +夏 +夔 +夕 +外 +夙 +多 +夜 +够 +夠 +夢 +夥 +大 +天 +太 +夫 +夭 +央 +夯 +失 +头 +夷 +夸 +夹 +夺 +夾 +奂 +奄 +奇 +奈 +奉 +奋 +奎 +奏 +奐 +契 +奔 +奕 +奖 +套 +奘 +奚 +奠 +奢 +奥 +奧 +奪 +奬 +奮 +女 +奴 +奶 +奸 +她 +好 +如 +妃 +妄 +妆 +妇 +妈 +妊 +妍 +妒 +妓 +妖 +妘 +妙 +妝 +妞 +妣 +妤 +妥 +妨 +妩 +妪 +妮 +妲 +妳 +妹 +妻 +妾 +姆 +姉 +姊 +始 +姍 +姐 +姑 +姒 +姓 +委 +姗 +姚 +姜 +姝 +姣 +姥 +姦 +姨 +姪 +姫 +姬 +姹 +姻 +姿 +威 +娃 +娄 +娅 +娆 +娇 +娉 +娑 +娓 +娘 +娛 +娜 +娟 +娠 +娣 +娥 +娩 +娱 +娲 +娴 +娶 +娼 +婀 +婁 +婆 +婉 +婊 +婕 +婚 +婢 +婦 +婧 +婪 +婭 +婴 +婵 +婶 +婷 +婺 +婿 +媒 +媚 +媛 +媞 +媧 +媲 +媳 +媽 +媾 +嫁 +嫂 +嫉 +嫌 +嫑 +嫔 +嫖 +嫘 +嫚 +嫡 +嫣 +嫦 +嫩 +嫲 +嫵 +嫻 +嬅 +嬉 +嬌 +嬗 +嬛 +嬢 +嬤 +嬪 +嬰 +嬴 +嬷 +嬸 +嬿 +孀 +孃 +子 +孑 +孔 +孕 +孖 +字 +存 +孙 +孚 +孛 +孜 +孝 +孟 +孢 +季 +孤 +学 +孩 +孪 +孫 +孬 +孰 +孱 +孳 +孵 +學 +孺 +孽 +孿 +宁 +它 +宅 +宇 +守 +安 +宋 +完 +宏 +宓 +宕 +宗 +官 +宙 +定 +宛 +宜 +宝 +实 +実 +宠 +审 +客 +宣 +室 +宥 +宦 +宪 +宫 +宮 +宰 +害 +宴 +宵 +家 +宸 +容 +宽 +宾 +宿 +寂 +寄 +寅 +密 +寇 +富 +寐 +寒 +寓 +寛 +寝 +寞 +察 +寡 +寢 +寥 +實 +寧 +寨 +審 +寫 +寬 +寮 +寰 +寵 +寶 +寸 +对 +寺 +寻 +导 +対 +寿 +封 +専 +射 +将 +將 +專 +尉 +尊 +尋 +對 +導 +小 +少 +尔 +尕 +尖 +尘 +尚 +尝 +尤 +尧 +尬 +就 +尴 +尷 +尸 +尹 +尺 +尻 +尼 +尽 +尾 +尿 +局 +屁 +层 +屄 +居 +屆 +屈 +屉 +届 +屋 +屌 +屍 +屎 +屏 +屐 +屑 +展 +屜 +属 +屠 +屡 +屢 +層 +履 +屬 +屯 +山 +屹 +屿 +岀 +岁 +岂 +岌 +岐 +岑 +岔 +岖 +岗 +岘 +岙 +岚 +岛 +岡 +岩 +岫 +岬 +岭 +岱 +岳 +岷 +岸 +峇 +峋 +峒 +峙 +峡 +峤 +峥 +峦 +峨 +峪 +峭 +峯 +峰 +峴 +島 +峻 +峽 +崁 +崂 +崆 +崇 +崎 +崑 +崔 +崖 +崗 +崙 +崛 +崧 +崩 +崭 +崴 +崽 +嵇 +嵊 +嵋 +嵌 +嵐 +嵘 +嵩 +嵬 +嵯 +嶂 +嶄 +嶇 +嶋 +嶙 +嶺 +嶼 +嶽 +巅 +巍 +巒 +巔 +巖 +川 +州 +巡 +巢 +工 +左 +巧 +巨 +巩 +巫 +差 +己 +已 +巳 +巴 +巷 +巻 +巽 +巾 +巿 +币 +市 +布 +帅 +帆 +师 +希 +帐 +帑 +帕 +帖 +帘 +帚 +帛 +帜 +帝 +帥 +带 +帧 +師 +席 +帮 +帯 +帰 +帳 +帶 +帷 +常 +帼 +帽 +幀 +幂 +幄 +幅 +幌 +幔 +幕 +幟 +幡 +幢 +幣 +幫 +干 +平 +年 +并 +幸 +幹 +幺 +幻 +幼 +幽 +幾 +广 +庁 +広 +庄 +庆 +庇 +床 +序 +庐 +库 +应 +底 +庖 +店 +庙 +庚 +府 +庞 +废 +庠 +度 +座 +庫 +庭 +庵 +庶 +康 +庸 +庹 +庾 +廁 +廂 +廃 +廈 +廉 +廊 +廓 +廖 +廚 +廝 +廟 +廠 +廢 +廣 +廬 +廳 +延 +廷 +建 +廿 +开 +弁 +异 +弃 +弄 +弈 +弊 +弋 +式 +弑 +弒 +弓 +弔 +引 +弗 +弘 +弛 +弟 +张 +弥 +弦 +弧 +弩 +弭 +弯 +弱 +張 +強 +弹 +强 +弼 +弾 +彅 +彆 +彈 +彌 +彎 +归 +当 +录 +彗 +彙 +彝 +形 +彤 +彥 +彦 +彧 +彩 +彪 +彫 +彬 +彭 +彰 +影 +彷 +役 +彻 +彼 +彿 +往 +征 +径 +待 +徇 +很 +徉 +徊 +律 +後 +徐 +徑 +徒 +従 +徕 +得 +徘 +徙 +徜 +從 +徠 +御 +徨 +復 +循 +徬 +微 +徳 +徴 +徵 +德 +徹 +徼 +徽 +心 +必 +忆 +忌 +忍 +忏 +忐 +忑 +忒 +忖 +志 +忘 +忙 +応 +忠 +忡 +忤 +忧 +忪 +快 +忱 +念 +忻 +忽 +忿 +怀 +态 +怂 +怅 +怆 +怎 +怏 +怒 +怔 +怕 +怖 +怙 +怜 +思 +怠 +怡 +急 +怦 +性 +怨 +怪 +怯 +怵 +总 +怼 +恁 +恃 +恆 +恋 +恍 +恐 +恒 +恕 +恙 +恚 +恢 +恣 +恤 +恥 +恨 +恩 +恪 +恫 +恬 +恭 +息 +恰 +恳 +恵 +恶 +恸 +恺 +恻 +恼 +恿 +悄 +悅 +悉 +悌 +悍 +悔 +悖 +悚 +悟 +悠 +患 +悦 +您 +悩 +悪 +悬 +悯 +悱 +悲 +悴 +悵 +悶 +悸 +悻 +悼 +悽 +情 +惆 +惇 +惊 +惋 +惑 +惕 +惘 +惚 +惜 +惟 +惠 +惡 +惦 +惧 +惨 +惩 +惫 +惬 +惭 +惮 +惯 +惰 +惱 +想 +惴 +惶 +惹 +惺 +愁 +愆 +愈 +愉 +愍 +意 +愕 +愚 +愛 +愜 +感 +愣 +愤 +愧 +愫 +愷 +愿 +慄 +慈 +態 +慌 +慎 +慑 +慕 +慘 +慚 +慟 +慢 +慣 +慧 +慨 +慫 +慮 +慰 +慳 +慵 +慶 +慷 +慾 +憂 +憊 +憋 +憎 +憐 +憑 +憔 +憚 +憤 +憧 +憨 +憩 +憫 +憬 +憲 +憶 +憾 +懂 +懇 +懈 +應 +懊 +懋 +懑 +懒 +懦 +懲 +懵 +懶 +懷 +懸 +懺 +懼 +懾 +懿 +戀 +戈 +戊 +戌 +戍 +戎 +戏 +成 +我 +戒 +戕 +或 +战 +戚 +戛 +戟 +戡 +戦 +截 +戬 +戮 +戰 +戲 +戳 +戴 +戶 +户 +戸 +戻 +戾 +房 +所 +扁 +扇 +扈 +扉 +手 +才 +扎 +扑 +扒 +打 +扔 +払 +托 +扛 +扣 +扦 +执 +扩 +扪 +扫 +扬 +扭 +扮 +扯 +扰 +扱 +扳 +扶 +批 +扼 +找 +承 +技 +抄 +抉 +把 +抑 +抒 +抓 +投 +抖 +抗 +折 +抚 +抛 +抜 +択 +抟 +抠 +抡 +抢 +护 +报 +抨 +披 +抬 +抱 +抵 +抹 +押 +抽 +抿 +拂 +拄 +担 +拆 +拇 +拈 +拉 +拋 +拌 +拍 +拎 +拐 +拒 +拓 +拔 +拖 +拗 +拘 +拙 +拚 +招 +拜 +拟 +拡 +拢 +拣 +拥 +拦 +拧 +拨 +择 +括 +拭 +拮 +拯 +拱 +拳 +拴 +拷 +拼 +拽 +拾 +拿 +持 +挂 +指 +挈 +按 +挎 +挑 +挖 +挙 +挚 +挛 +挝 +挞 +挟 +挠 +挡 +挣 +挤 +挥 +挨 +挪 +挫 +振 +挲 +挹 +挺 +挽 +挾 +捂 +捅 +捆 +捉 +捋 +捌 +捍 +捎 +捏 +捐 +捕 +捞 +损 +捡 +换 +捣 +捧 +捨 +捩 +据 +捱 +捲 +捶 +捷 +捺 +捻 +掀 +掂 +掃 +掇 +授 +掉 +掌 +掏 +掐 +排 +掖 +掘 +掙 +掛 +掠 +採 +探 +掣 +接 +控 +推 +掩 +措 +掬 +掰 +掲 +掳 +掴 +掷 +掸 +掺 +揀 +揃 +揄 +揆 +揉 +揍 +描 +提 +插 +揖 +揚 +換 +握 +揣 +揩 +揪 +揭 +揮 +援 +揶 +揸 +揹 +揽 +搀 +搁 +搂 +搅 +損 +搏 +搐 +搓 +搔 +搖 +搗 +搜 +搞 +搡 +搪 +搬 +搭 +搵 +搶 +携 +搽 +摀 +摁 +摄 +摆 +摇 +摈 +摊 +摒 +摔 +摘 +摞 +摟 +摧 +摩 +摯 +摳 +摸 +摹 +摺 +摻 +撂 +撃 +撅 +撇 +撈 +撐 +撑 +撒 +撓 +撕 +撚 +撞 +撤 +撥 +撩 +撫 +撬 +播 +撮 +撰 +撲 +撵 +撷 +撸 +撻 +撼 +撿 +擀 +擁 +擂 +擄 +擅 +擇 +擊 +擋 +操 +擎 +擒 +擔 +擘 +據 +擞 +擠 +擡 +擢 +擦 +擬 +擰 +擱 +擲 +擴 +擷 +擺 +擼 +擾 +攀 +攏 +攒 +攔 +攘 +攙 +攜 +攝 +攞 +攢 +攣 +攤 +攥 +攪 +攫 +攬 +支 +收 +攸 +改 +攻 +放 +政 +故 +效 +敌 +敍 +敎 +敏 +救 +敕 +敖 +敗 +敘 +教 +敛 +敝 +敞 +敢 +散 +敦 +敬 +数 +敲 +整 +敵 +敷 +數 +斂 +斃 +文 +斋 +斌 +斎 +斐 +斑 +斓 +斗 +料 +斛 +斜 +斟 +斡 +斤 +斥 +斧 +斩 +斫 +斬 +断 +斯 +新 +斷 +方 +於 +施 +旁 +旃 +旅 +旋 +旌 +旎 +族 +旖 +旗 +无 +既 +日 +旦 +旧 +旨 +早 +旬 +旭 +旮 +旱 +时 +旷 +旺 +旻 +昀 +昂 +昆 +昇 +昉 +昊 +昌 +明 +昏 +易 +昔 +昕 +昙 +星 +映 +春 +昧 +昨 +昭 +是 +昱 +昴 +昵 +昶 +昼 +显 +晁 +時 +晃 +晉 +晋 +晌 +晏 +晒 +晓 +晔 +晕 +晖 +晗 +晚 +晝 +晞 +晟 +晤 +晦 +晨 +晩 +普 +景 +晰 +晴 +晶 +晷 +智 +晾 +暂 +暄 +暇 +暈 +暉 +暌 +暐 +暑 +暖 +暗 +暝 +暢 +暧 +暨 +暫 +暮 +暱 +暴 +暸 +暹 +曄 +曆 +曇 +曉 +曖 +曙 +曜 +曝 +曠 +曦 +曬 +曰 +曲 +曳 +更 +書 +曹 +曼 +曾 +替 +最 +會 +月 +有 +朋 +服 +朐 +朔 +朕 +朗 +望 +朝 +期 +朦 +朧 +木 +未 +末 +本 +札 +朮 +术 +朱 +朴 +朵 +机 +朽 +杀 +杂 +权 +杆 +杈 +杉 +李 +杏 +材 +村 +杓 +杖 +杜 +杞 +束 +杠 +条 +来 +杨 +杭 +杯 +杰 +東 +杳 +杵 +杷 +杼 +松 +板 +极 +构 +枇 +枉 +枋 +析 +枕 +林 +枚 +果 +枝 +枢 +枣 +枪 +枫 +枭 +枯 +枰 +枱 +枳 +架 +枷 +枸 +柄 +柏 +某 +柑 +柒 +染 +柔 +柘 +柚 +柜 +柞 +柠 +柢 +查 +柩 +柬 +柯 +柱 +柳 +柴 +柵 +査 +柿 +栀 +栃 +栄 +栅 +标 +栈 +栉 +栋 +栎 +栏 +树 +栓 +栖 +栗 +校 +栩 +株 +样 +核 +根 +格 +栽 +栾 +桀 +桁 +桂 +桃 +桅 +框 +案 +桉 +桌 +桎 +桐 +桑 +桓 +桔 +桜 +桠 +桡 +桢 +档 +桥 +桦 +桧 +桨 +桩 +桶 +桿 +梁 +梅 +梆 +梏 +梓 +梗 +條 +梟 +梢 +梦 +梧 +梨 +梭 +梯 +械 +梳 +梵 +梶 +检 +棂 +棄 +棉 +棋 +棍 +棒 +棕 +棗 +棘 +棚 +棟 +棠 +棣 +棧 +森 +棱 +棲 +棵 +棹 +棺 +椁 +椅 +椋 +植 +椎 +椒 +検 +椪 +椭 +椰 +椹 +椽 +椿 +楂 +楊 +楓 +楔 +楚 +楝 +楞 +楠 +楣 +楨 +楫 +業 +楮 +極 +楷 +楸 +楹 +楼 +楽 +概 +榄 +榆 +榈 +榉 +榔 +榕 +榖 +榛 +榜 +榨 +榫 +榭 +榮 +榱 +榴 +榷 +榻 +槁 +槃 +構 +槌 +槍 +槎 +槐 +槓 +様 +槛 +槟 +槤 +槭 +槲 +槳 +槻 +槽 +槿 +樁 +樂 +樊 +樑 +樓 +標 +樞 +樟 +模 +樣 +権 +横 +樫 +樯 +樱 +樵 +樸 +樹 +樺 +樽 +樾 +橄 +橇 +橋 +橐 +橘 +橙 +機 +橡 +橢 +橫 +橱 +橹 +橼 +檀 +檄 +檎 +檐 +檔 +檗 +檜 +檢 +檬 +檯 +檳 +檸 +檻 +櫃 +櫚 +櫛 +櫥 +櫸 +櫻 +欄 +權 +欒 +欖 +欠 +次 +欢 +欣 +欧 +欲 +欸 +欺 +欽 +款 +歆 +歇 +歉 +歌 +歎 +歐 +歓 +歙 +歛 +歡 +止 +正 +此 +步 +武 +歧 +歩 +歪 +歯 +歲 +歳 +歴 +歷 +歸 +歹 +死 +歼 +殁 +殃 +殆 +殇 +殉 +殊 +残 +殒 +殓 +殖 +殘 +殞 +殡 +殤 +殭 +殯 +殲 +殴 +段 +殷 +殺 +殼 +殿 +毀 +毁 +毂 +毅 +毆 +毋 +母 +毎 +每 +毒 +毓 +比 +毕 +毗 +毘 +毙 +毛 +毡 +毫 +毯 +毽 +氈 +氏 +氐 +民 +氓 +气 +氖 +気 +氙 +氛 +氟 +氡 +氢 +氣 +氤 +氦 +氧 +氨 +氪 +氫 +氮 +氯 +氰 +氲 +水 +氷 +永 +氹 +氾 +汀 +汁 +求 +汆 +汇 +汉 +汎 +汐 +汕 +汗 +汙 +汛 +汝 +汞 +江 +池 +污 +汤 +汨 +汩 +汪 +汰 +汲 +汴 +汶 +汹 +決 +汽 +汾 +沁 +沂 +沃 +沅 +沈 +沉 +沌 +沏 +沐 +沒 +沓 +沖 +沙 +沛 +沟 +没 +沢 +沣 +沥 +沦 +沧 +沪 +沫 +沭 +沮 +沱 +河 +沸 +油 +治 +沼 +沽 +沾 +沿 +況 +泄 +泉 +泊 +泌 +泓 +法 +泗 +泛 +泞 +泠 +泡 +波 +泣 +泥 +注 +泪 +泫 +泮 +泯 +泰 +泱 +泳 +泵 +泷 +泸 +泻 +泼 +泽 +泾 +洁 +洄 +洋 +洒 +洗 +洙 +洛 +洞 +津 +洩 +洪 +洮 +洱 +洲 +洵 +洶 +洸 +洹 +活 +洼 +洽 +派 +流 +浃 +浄 +浅 +浆 +浇 +浊 +测 +济 +浏 +浑 +浒 +浓 +浔 +浙 +浚 +浜 +浣 +浦 +浩 +浪 +浬 +浮 +浯 +浴 +海 +浸 +涂 +涅 +涇 +消 +涉 +涌 +涎 +涓 +涔 +涕 +涙 +涛 +涝 +涞 +涟 +涠 +涡 +涣 +涤 +润 +涧 +涨 +涩 +涪 +涮 +涯 +液 +涵 +涸 +涼 +涿 +淀 +淄 +淅 +淆 +淇 +淋 +淌 +淑 +淒 +淖 +淘 +淙 +淚 +淞 +淡 +淤 +淦 +淨 +淩 +淪 +淫 +淬 +淮 +深 +淳 +淵 +混 +淹 +淺 +添 +淼 +清 +済 +渉 +渊 +渋 +渍 +渎 +渐 +渔 +渗 +渙 +渚 +減 +渝 +渠 +渡 +渣 +渤 +渥 +渦 +温 +測 +渭 +港 +渲 +渴 +游 +渺 +渾 +湃 +湄 +湊 +湍 +湖 +湘 +湛 +湟 +湧 +湫 +湮 +湯 +湳 +湾 +湿 +満 +溃 +溅 +溉 +溏 +源 +準 +溜 +溝 +溟 +溢 +溥 +溧 +溪 +溫 +溯 +溱 +溴 +溶 +溺 +溼 +滁 +滂 +滄 +滅 +滇 +滋 +滌 +滑 +滓 +滔 +滕 +滙 +滚 +滝 +滞 +滟 +满 +滢 +滤 +滥 +滦 +滨 +滩 +滬 +滯 +滲 +滴 +滷 +滸 +滾 +滿 +漁 +漂 +漆 +漉 +漏 +漓 +演 +漕 +漠 +漢 +漣 +漩 +漪 +漫 +漬 +漯 +漱 +漲 +漳 +漸 +漾 +漿 +潆 +潇 +潋 +潍 +潑 +潔 +潘 +潛 +潜 +潞 +潟 +潢 +潤 +潦 +潧 +潭 +潮 +潰 +潴 +潸 +潺 +潼 +澀 +澄 +澆 +澈 +澍 +澎 +澗 +澜 +澡 +澤 +澧 +澱 +澳 +澹 +激 +濁 +濂 +濃 +濑 +濒 +濕 +濘 +濛 +濟 +濠 +濡 +濤 +濫 +濬 +濮 +濯 +濱 +濺 +濾 +瀅 +瀆 +瀉 +瀋 +瀏 +瀑 +瀕 +瀘 +瀚 +瀛 +瀝 +瀞 +瀟 +瀧 +瀨 +瀬 +瀰 +瀾 +灌 +灏 +灑 +灘 +灝 +灞 +灣 +火 +灬 +灭 +灯 +灰 +灵 +灶 +灸 +灼 +災 +灾 +灿 +炀 +炁 +炅 +炉 +炊 +炎 +炒 +炔 +炕 +炖 +炙 +炜 +炫 +炬 +炭 +炮 +炯 +炳 +炷 +炸 +点 +為 +炼 +炽 +烁 +烂 +烃 +烈 +烊 +烏 +烘 +烙 +烛 +烟 +烤 +烦 +烧 +烨 +烩 +烫 +烬 +热 +烯 +烷 +烹 +烽 +焉 +焊 +焕 +焖 +焗 +焘 +焙 +焚 +焜 +無 +焦 +焯 +焰 +焱 +然 +焼 +煅 +煉 +煊 +煌 +煎 +煒 +煖 +煙 +煜 +煞 +煤 +煥 +煦 +照 +煨 +煩 +煮 +煲 +煸 +煽 +熄 +熊 +熏 +熒 +熔 +熙 +熟 +熠 +熨 +熬 +熱 +熵 +熹 +熾 +燁 +燃 +燄 +燈 +燉 +燊 +燎 +燒 +燔 +燕 +燙 +燜 +營 +燥 +燦 +燧 +燭 +燮 +燴 +燻 +燼 +燿 +爆 +爍 +爐 +爛 +爪 +爬 +爭 +爰 +爱 +爲 +爵 +父 +爷 +爸 +爹 +爺 +爻 +爽 +爾 +牆 +片 +版 +牌 +牍 +牒 +牙 +牛 +牝 +牟 +牠 +牡 +牢 +牦 +牧 +物 +牯 +牲 +牴 +牵 +特 +牺 +牽 +犀 +犁 +犄 +犊 +犍 +犒 +犢 +犧 +犬 +犯 +状 +犷 +犸 +犹 +狀 +狂 +狄 +狈 +狎 +狐 +狒 +狗 +狙 +狞 +狠 +狡 +狩 +独 +狭 +狮 +狰 +狱 +狸 +狹 +狼 +狽 +猎 +猕 +猖 +猗 +猙 +猛 +猜 +猝 +猥 +猩 +猪 +猫 +猬 +献 +猴 +猶 +猷 +猾 +猿 +獄 +獅 +獎 +獐 +獒 +獗 +獠 +獣 +獨 +獭 +獰 +獲 +獵 +獷 +獸 +獺 +獻 +獼 +獾 +玄 +率 +玉 +王 +玑 +玖 +玛 +玟 +玠 +玥 +玩 +玫 +玮 +环 +现 +玲 +玳 +玷 +玺 +玻 +珀 +珂 +珅 +珈 +珉 +珊 +珍 +珏 +珐 +珑 +珙 +珞 +珠 +珣 +珥 +珩 +珪 +班 +珮 +珲 +珺 +現 +球 +琅 +理 +琇 +琉 +琊 +琍 +琏 +琐 +琛 +琢 +琥 +琦 +琨 +琪 +琬 +琮 +琰 +琲 +琳 +琴 +琵 +琶 +琺 +琼 +瑀 +瑁 +瑄 +瑋 +瑕 +瑗 +瑙 +瑚 +瑛 +瑜 +瑞 +瑟 +瑠 +瑣 +瑤 +瑩 +瑪 +瑯 +瑰 +瑶 +瑾 +璀 +璁 +璃 +璇 +璉 +璋 +璎 +璐 +璜 +璞 +璟 +璧 +璨 +環 +璽 +璿 +瓊 +瓏 +瓒 +瓜 +瓢 +瓣 +瓤 +瓦 +瓮 +瓯 +瓴 +瓶 +瓷 +甄 +甌 +甕 +甘 +甙 +甚 +甜 +生 +產 +産 +甥 +甦 +用 +甩 +甫 +甬 +甭 +甯 +田 +由 +甲 +申 +电 +男 +甸 +町 +画 +甾 +畀 +畅 +界 +畏 +畑 +畔 +留 +畜 +畝 +畢 +略 +畦 +番 +畫 +異 +畲 +畳 +畴 +當 +畸 +畹 +畿 +疆 +疇 +疊 +疏 +疑 +疔 +疖 +疗 +疙 +疚 +疝 +疟 +疡 +疣 +疤 +疥 +疫 +疮 +疯 +疱 +疲 +疳 +疵 +疸 +疹 +疼 +疽 +疾 +痂 +病 +症 +痈 +痉 +痊 +痍 +痒 +痔 +痕 +痘 +痙 +痛 +痞 +痠 +痢 +痣 +痤 +痧 +痨 +痪 +痫 +痰 +痱 +痴 +痹 +痺 +痼 +痿 +瘀 +瘁 +瘋 +瘍 +瘓 +瘘 +瘙 +瘟 +瘠 +瘡 +瘢 +瘤 +瘦 +瘧 +瘩 +瘪 +瘫 +瘴 +瘸 +瘾 +療 +癇 +癌 +癒 +癖 +癜 +癞 +癡 +癢 +癣 +癥 +癫 +癬 +癮 +癱 +癲 +癸 +発 +登 +發 +白 +百 +皂 +的 +皆 +皇 +皈 +皋 +皎 +皑 +皓 +皖 +皙 +皚 +皮 +皰 +皱 +皴 +皺 +皿 +盂 +盃 +盅 +盆 +盈 +益 +盎 +盏 +盐 +监 +盒 +盔 +盖 +盗 +盘 +盛 +盜 +盞 +盟 +盡 +監 +盤 +盥 +盧 +盪 +目 +盯 +盱 +盲 +直 +相 +盹 +盼 +盾 +省 +眈 +眉 +看 +県 +眙 +眞 +真 +眠 +眦 +眨 +眩 +眯 +眶 +眷 +眸 +眺 +眼 +眾 +着 +睁 +睇 +睏 +睐 +睑 +睛 +睜 +睞 +睡 +睢 +督 +睥 +睦 +睨 +睪 +睫 +睬 +睹 +睽 +睾 +睿 +瞄 +瞅 +瞇 +瞋 +瞌 +瞎 +瞑 +瞒 +瞓 +瞞 +瞟 +瞠 +瞥 +瞧 +瞩 +瞪 +瞬 +瞭 +瞰 +瞳 +瞻 +瞼 +瞿 +矇 +矍 +矗 +矚 +矛 +矜 +矢 +矣 +知 +矩 +矫 +短 +矮 +矯 +石 +矶 +矽 +矾 +矿 +码 +砂 +砌 +砍 +砒 +研 +砖 +砗 +砚 +砝 +砣 +砥 +砧 +砭 +砰 +砲 +破 +砷 +砸 +砺 +砼 +砾 +础 +硅 +硐 +硒 +硕 +硝 +硫 +硬 +确 +硯 +硼 +碁 +碇 +碉 +碌 +碍 +碎 +碑 +碓 +碗 +碘 +碚 +碛 +碟 +碣 +碧 +碩 +碰 +碱 +碳 +碴 +確 +碼 +碾 +磁 +磅 +磊 +磋 +磐 +磕 +磚 +磡 +磨 +磬 +磯 +磲 +磷 +磺 +礁 +礎 +礙 +礡 +礦 +礪 +礫 +礴 +示 +礼 +社 +祀 +祁 +祂 +祇 +祈 +祉 +祎 +祐 +祕 +祖 +祗 +祚 +祛 +祜 +祝 +神 +祟 +祠 +祢 +祥 +票 +祭 +祯 +祷 +祸 +祺 +祿 +禀 +禁 +禄 +禅 +禍 +禎 +福 +禛 +禦 +禧 +禪 +禮 +禱 +禹 +禺 +离 +禽 +禾 +禿 +秀 +私 +秃 +秆 +秉 +秋 +种 +科 +秒 +秘 +租 +秣 +秤 +秦 +秧 +秩 +秭 +积 +称 +秸 +移 +秽 +稀 +稅 +程 +稍 +税 +稔 +稗 +稚 +稜 +稞 +稟 +稠 +稣 +種 +稱 +稲 +稳 +稷 +稹 +稻 +稼 +稽 +稿 +穀 +穂 +穆 +穌 +積 +穎 +穗 +穢 +穩 +穫 +穴 +究 +穷 +穹 +空 +穿 +突 +窃 +窄 +窈 +窍 +窑 +窒 +窓 +窕 +窖 +窗 +窘 +窜 +窝 +窟 +窠 +窥 +窦 +窨 +窩 +窪 +窮 +窯 +窺 +窿 +竄 +竅 +竇 +竊 +立 +竖 +站 +竜 +竞 +竟 +章 +竣 +童 +竭 +端 +競 +竹 +竺 +竽 +竿 +笃 +笆 +笈 +笋 +笏 +笑 +笔 +笙 +笛 +笞 +笠 +符 +笨 +第 +笹 +笺 +笼 +筆 +等 +筊 +筋 +筍 +筏 +筐 +筑 +筒 +答 +策 +筛 +筝 +筠 +筱 +筲 +筵 +筷 +筹 +签 +简 +箇 +箋 +箍 +箏 +箐 +箔 +箕 +算 +箝 +管 +箩 +箫 +箭 +箱 +箴 +箸 +節 +篁 +範 +篆 +篇 +築 +篑 +篓 +篙 +篝 +篠 +篡 +篤 +篩 +篪 +篮 +篱 +篷 +簇 +簌 +簍 +簡 +簦 +簧 +簪 +簫 +簷 +簸 +簽 +簾 +簿 +籁 +籃 +籌 +籍 +籐 +籟 +籠 +籤 +籬 +籮 +籲 +米 +类 +籼 +籽 +粄 +粉 +粑 +粒 +粕 +粗 +粘 +粟 +粤 +粥 +粧 +粪 +粮 +粱 +粲 +粳 +粵 +粹 +粼 +粽 +精 +粿 +糅 +糊 +糍 +糕 +糖 +糗 +糙 +糜 +糞 +糟 +糠 +糧 +糬 +糯 +糰 +糸 +系 +糾 +紀 +紂 +約 +紅 +紉 +紊 +紋 +納 +紐 +紓 +純 +紗 +紘 +紙 +級 +紛 +紜 +素 +紡 +索 +紧 +紫 +紮 +累 +細 +紳 +紹 +紺 +終 +絃 +組 +絆 +経 +結 +絕 +絞 +絡 +絢 +給 +絨 +絮 +統 +絲 +絳 +絵 +絶 +絹 +綁 +綏 +綑 +經 +継 +続 +綜 +綠 +綢 +綦 +綫 +綬 +維 +綱 +網 +綴 +綵 +綸 +綺 +綻 +綽 +綾 +綿 +緊 +緋 +総 +緑 +緒 +緘 +線 +緝 +緞 +締 +緣 +編 +緩 +緬 +緯 +練 +緹 +緻 +縁 +縄 +縈 +縛 +縝 +縣 +縫 +縮 +縱 +縴 +縷 +總 +績 +繁 +繃 +繆 +繇 +繋 +織 +繕 +繚 +繞 +繡 +繩 +繪 +繫 +繭 +繳 +繹 +繼 +繽 +纂 +續 +纍 +纏 +纓 +纔 +纖 +纜 +纠 +红 +纣 +纤 +约 +级 +纨 +纪 +纫 +纬 +纭 +纯 +纰 +纱 +纲 +纳 +纵 +纶 +纷 +纸 +纹 +纺 +纽 +纾 +线 +绀 +练 +组 +绅 +细 +织 +终 +绊 +绍 +绎 +经 +绑 +绒 +结 +绔 +绕 +绘 +给 +绚 +绛 +络 +绝 +绞 +统 +绡 +绢 +绣 +绥 +绦 +继 +绩 +绪 +绫 +续 +绮 +绯 +绰 +绳 +维 +绵 +绶 +绷 +绸 +绻 +综 +绽 +绾 +绿 +缀 +缄 +缅 +缆 +缇 +缈 +缉 +缎 +缓 +缔 +缕 +编 +缘 +缙 +缚 +缜 +缝 +缠 +缢 +缤 +缥 +缨 +缩 +缪 +缭 +缮 +缰 +缱 +缴 +缸 +缺 +缽 +罂 +罄 +罌 +罐 +网 +罔 +罕 +罗 +罚 +罡 +罢 +罩 +罪 +置 +罰 +署 +罵 +罷 +罹 +羁 +羅 +羈 +羊 +羌 +美 +羔 +羚 +羞 +羟 +羡 +羣 +群 +羥 +羧 +羨 +義 +羯 +羲 +羸 +羹 +羽 +羿 +翁 +翅 +翊 +翌 +翎 +習 +翔 +翘 +翟 +翠 +翡 +翦 +翩 +翰 +翱 +翳 +翹 +翻 +翼 +耀 +老 +考 +耄 +者 +耆 +耋 +而 +耍 +耐 +耒 +耕 +耗 +耘 +耙 +耦 +耨 +耳 +耶 +耷 +耸 +耻 +耽 +耿 +聂 +聆 +聊 +聋 +职 +聒 +联 +聖 +聘 +聚 +聞 +聪 +聯 +聰 +聲 +聳 +聴 +聶 +職 +聽 +聾 +聿 +肃 +肄 +肅 +肆 +肇 +肉 +肋 +肌 +肏 +肓 +肖 +肘 +肚 +肛 +肝 +肠 +股 +肢 +肤 +肥 +肩 +肪 +肮 +肯 +肱 +育 +肴 +肺 +肽 +肾 +肿 +胀 +胁 +胃 +胄 +胆 +背 +胍 +胎 +胖 +胚 +胛 +胜 +胝 +胞 +胡 +胤 +胥 +胧 +胫 +胭 +胯 +胰 +胱 +胳 +胴 +胶 +胸 +胺 +能 +脂 +脅 +脆 +脇 +脈 +脉 +脊 +脍 +脏 +脐 +脑 +脓 +脖 +脘 +脚 +脛 +脣 +脩 +脫 +脯 +脱 +脲 +脳 +脸 +脹 +脾 +腆 +腈 +腊 +腋 +腌 +腎 +腐 +腑 +腓 +腔 +腕 +腥 +腦 +腩 +腫 +腭 +腮 +腰 +腱 +腳 +腴 +腸 +腹 +腺 +腻 +腼 +腾 +腿 +膀 +膈 +膊 +膏 +膑 +膘 +膚 +膛 +膜 +膝 +膠 +膦 +膨 +膩 +膳 +膺 +膻 +膽 +膾 +膿 +臀 +臂 +臃 +臆 +臉 +臊 +臍 +臓 +臘 +臟 +臣 +臥 +臧 +臨 +自 +臬 +臭 +至 +致 +臺 +臻 +臼 +臾 +舀 +舂 +舅 +舆 +與 +興 +舉 +舊 +舌 +舍 +舎 +舐 +舒 +舔 +舖 +舗 +舛 +舜 +舞 +舟 +航 +舫 +般 +舰 +舱 +舵 +舶 +舷 +舸 +船 +舺 +舾 +艇 +艋 +艘 +艙 +艦 +艮 +良 +艰 +艱 +色 +艳 +艷 +艹 +艺 +艾 +节 +芃 +芈 +芊 +芋 +芍 +芎 +芒 +芙 +芜 +芝 +芡 +芥 +芦 +芩 +芪 +芫 +芬 +芭 +芮 +芯 +花 +芳 +芷 +芸 +芹 +芻 +芽 +芾 +苁 +苄 +苇 +苋 +苍 +苏 +苑 +苒 +苓 +苔 +苕 +苗 +苛 +苜 +苞 +苟 +苡 +苣 +若 +苦 +苫 +苯 +英 +苷 +苹 +苻 +茁 +茂 +范 +茄 +茅 +茉 +茎 +茏 +茗 +茜 +茧 +茨 +茫 +茬 +茭 +茯 +茱 +茲 +茴 +茵 +茶 +茸 +茹 +茼 +荀 +荃 +荆 +草 +荊 +荏 +荐 +荒 +荔 +荖 +荘 +荚 +荞 +荟 +荠 +荡 +荣 +荤 +荥 +荧 +荨 +荪 +荫 +药 +荳 +荷 +荸 +荻 +荼 +荽 +莅 +莆 +莉 +莊 +莎 +莒 +莓 +莖 +莘 +莞 +莠 +莢 +莧 +莪 +莫 +莱 +莲 +莴 +获 +莹 +莺 +莽 +莿 +菀 +菁 +菅 +菇 +菈 +菊 +菌 +菏 +菓 +菖 +菘 +菜 +菟 +菠 +菡 +菩 +華 +菱 +菲 +菸 +菽 +萁 +萃 +萄 +萊 +萋 +萌 +萍 +萎 +萘 +萝 +萤 +营 +萦 +萧 +萨 +萩 +萬 +萱 +萵 +萸 +萼 +落 +葆 +葉 +著 +葚 +葛 +葡 +董 +葦 +葩 +葫 +葬 +葭 +葯 +葱 +葳 +葵 +葷 +葺 +蒂 +蒋 +蒐 +蒔 +蒙 +蒜 +蒞 +蒟 +蒡 +蒨 +蒲 +蒸 +蒹 +蒻 +蒼 +蒿 +蓁 +蓄 +蓆 +蓉 +蓋 +蓑 +蓓 +蓖 +蓝 +蓟 +蓦 +蓬 +蓮 +蓼 +蓿 +蔑 +蔓 +蔔 +蔗 +蔘 +蔚 +蔡 +蔣 +蔥 +蔫 +蔬 +蔭 +蔵 +蔷 +蔺 +蔻 +蔼 +蔽 +蕁 +蕃 +蕈 +蕉 +蕊 +蕎 +蕙 +蕤 +蕨 +蕩 +蕪 +蕭 +蕲 +蕴 +蕻 +蕾 +薄 +薅 +薇 +薈 +薊 +薏 +薑 +薔 +薙 +薛 +薦 +薨 +薩 +薪 +薬 +薯 +薰 +薹 +藉 +藍 +藏 +藐 +藓 +藕 +藜 +藝 +藤 +藥 +藩 +藹 +藻 +藿 +蘆 +蘇 +蘊 +蘋 +蘑 +蘚 +蘭 +蘸 +蘼 +蘿 +虎 +虏 +虐 +虑 +虔 +處 +虚 +虛 +虜 +虞 +號 +虢 +虧 +虫 +虬 +虱 +虹 +虻 +虽 +虾 +蚀 +蚁 +蚂 +蚊 +蚌 +蚓 +蚕 +蚜 +蚝 +蚣 +蚤 +蚩 +蚪 +蚯 +蚱 +蚵 +蛀 +蛆 +蛇 +蛊 +蛋 +蛎 +蛐 +蛔 +蛙 +蛛 +蛟 +蛤 +蛭 +蛮 +蛰 +蛳 +蛹 +蛻 +蛾 +蜀 +蜂 +蜃 +蜆 +蜇 +蜈 +蜊 +蜍 +蜒 +蜓 +蜕 +蜗 +蜘 +蜚 +蜜 +蜡 +蜢 +蜥 +蜱 +蜴 +蜷 +蜻 +蜿 +蝇 +蝈 +蝉 +蝌 +蝎 +蝕 +蝗 +蝙 +蝟 +蝠 +蝦 +蝨 +蝴 +蝶 +蝸 +蝼 +螂 +螃 +融 +螞 +螢 +螨 +螯 +螳 +螺 +蟀 +蟄 +蟆 +蟋 +蟎 +蟑 +蟒 +蟠 +蟬 +蟲 +蟹 +蟻 +蟾 +蠅 +蠍 +蠔 +蠕 +蠛 +蠟 +蠡 +蠢 +蠣 +蠱 +蠶 +蠹 +蠻 +血 +衄 +衅 +衆 +行 +衍 +術 +衔 +街 +衙 +衛 +衝 +衞 +衡 +衢 +衣 +补 +表 +衩 +衫 +衬 +衮 +衰 +衲 +衷 +衹 +衾 +衿 +袁 +袂 +袄 +袅 +袈 +袋 +袍 +袒 +袖 +袜 +袞 +袤 +袪 +被 +袭 +袱 +裁 +裂 +装 +裆 +裊 +裏 +裔 +裕 +裘 +裙 +補 +裝 +裟 +裡 +裤 +裨 +裱 +裳 +裴 +裸 +裹 +製 +裾 +褂 +複 +褐 +褒 +褓 +褔 +褚 +褥 +褪 +褫 +褲 +褶 +褻 +襁 +襄 +襟 +襠 +襪 +襬 +襯 +襲 +西 +要 +覃 +覆 +覇 +見 +規 +覓 +視 +覚 +覦 +覧 +親 +覬 +観 +覷 +覺 +覽 +觀 +见 +观 +规 +觅 +视 +览 +觉 +觊 +觎 +觐 +觑 +角 +觞 +解 +觥 +触 +觸 +言 +訂 +計 +訊 +討 +訓 +訕 +訖 +託 +記 +訛 +訝 +訟 +訣 +訥 +訪 +設 +許 +訳 +訴 +訶 +診 +註 +証 +詆 +詐 +詔 +評 +詛 +詞 +詠 +詡 +詢 +詣 +試 +詩 +詫 +詬 +詭 +詮 +詰 +話 +該 +詳 +詹 +詼 +誅 +誇 +誉 +誌 +認 +誓 +誕 +誘 +語 +誠 +誡 +誣 +誤 +誥 +誦 +誨 +說 +説 +読 +誰 +課 +誹 +誼 +調 +諄 +談 +請 +諏 +諒 +論 +諗 +諜 +諡 +諦 +諧 +諫 +諭 +諮 +諱 +諳 +諷 +諸 +諺 +諾 +謀 +謁 +謂 +謄 +謊 +謎 +謐 +謔 +謗 +謙 +講 +謝 +謠 +謨 +謬 +謹 +謾 +譁 +證 +譎 +譏 +識 +譙 +譚 +譜 +警 +譬 +譯 +議 +譲 +譴 +護 +譽 +讀 +變 +讓 +讚 +讞 +计 +订 +认 +讥 +讧 +讨 +让 +讪 +讫 +训 +议 +讯 +记 +讲 +讳 +讴 +讶 +讷 +许 +讹 +论 +讼 +讽 +设 +访 +诀 +证 +诃 +评 +诅 +识 +诈 +诉 +诊 +诋 +词 +诏 +译 +试 +诗 +诘 +诙 +诚 +诛 +话 +诞 +诟 +诠 +诡 +询 +诣 +诤 +该 +详 +诧 +诩 +诫 +诬 +语 +误 +诰 +诱 +诲 +说 +诵 +诶 +请 +诸 +诺 +读 +诽 +课 +诿 +谀 +谁 +调 +谄 +谅 +谆 +谈 +谊 +谋 +谌 +谍 +谎 +谏 +谐 +谑 +谒 +谓 +谔 +谕 +谗 +谘 +谙 +谚 +谛 +谜 +谟 +谢 +谣 +谤 +谥 +谦 +谧 +谨 +谩 +谪 +谬 +谭 +谯 +谱 +谲 +谴 +谶 +谷 +豁 +豆 +豇 +豈 +豉 +豊 +豌 +豎 +豐 +豔 +豚 +象 +豢 +豪 +豫 +豬 +豹 +豺 +貂 +貅 +貌 +貓 +貔 +貘 +貝 +貞 +負 +財 +貢 +貧 +貨 +販 +貪 +貫 +責 +貯 +貰 +貳 +貴 +貶 +買 +貸 +費 +貼 +貽 +貿 +賀 +賁 +賂 +賃 +賄 +資 +賈 +賊 +賑 +賓 +賜 +賞 +賠 +賡 +賢 +賣 +賤 +賦 +質 +賬 +賭 +賴 +賺 +購 +賽 +贅 +贈 +贊 +贍 +贏 +贓 +贖 +贛 +贝 +贞 +负 +贡 +财 +责 +贤 +败 +账 +货 +质 +贩 +贪 +贫 +贬 +购 +贮 +贯 +贰 +贱 +贲 +贴 +贵 +贷 +贸 +费 +贺 +贻 +贼 +贾 +贿 +赁 +赂 +赃 +资 +赅 +赈 +赊 +赋 +赌 +赎 +赏 +赐 +赓 +赔 +赖 +赘 +赚 +赛 +赝 +赞 +赠 +赡 +赢 +赣 +赤 +赦 +赧 +赫 +赭 +走 +赳 +赴 +赵 +赶 +起 +趁 +超 +越 +趋 +趕 +趙 +趟 +趣 +趨 +足 +趴 +趵 +趸 +趺 +趾 +跃 +跄 +跆 +跋 +跌 +跎 +跑 +跖 +跚 +跛 +距 +跟 +跡 +跤 +跨 +跩 +跪 +路 +跳 +践 +跷 +跹 +跺 +跻 +踉 +踊 +踌 +踏 +踐 +踝 +踞 +踟 +踢 +踩 +踪 +踮 +踱 +踴 +踵 +踹 +蹂 +蹄 +蹇 +蹈 +蹉 +蹊 +蹋 +蹑 +蹒 +蹙 +蹟 +蹣 +蹤 +蹦 +蹩 +蹬 +蹭 +蹲 +蹴 +蹶 +蹺 +蹼 +蹿 +躁 +躇 +躉 +躊 +躋 +躍 +躏 +躪 +身 +躬 +躯 +躲 +躺 +軀 +車 +軋 +軌 +軍 +軒 +軟 +転 +軸 +軼 +軽 +軾 +較 +載 +輒 +輓 +輔 +輕 +輛 +輝 +輟 +輩 +輪 +輯 +輸 +輻 +輾 +輿 +轄 +轅 +轆 +轉 +轍 +轎 +轟 +车 +轧 +轨 +轩 +转 +轭 +轮 +软 +轰 +轲 +轴 +轶 +轻 +轼 +载 +轿 +较 +辄 +辅 +辆 +辇 +辈 +辉 +辊 +辍 +辐 +辑 +输 +辕 +辖 +辗 +辘 +辙 +辛 +辜 +辞 +辟 +辣 +辦 +辨 +辩 +辫 +辭 +辮 +辯 +辰 +辱 +農 +边 +辺 +辻 +込 +辽 +达 +迁 +迂 +迄 +迅 +过 +迈 +迎 +运 +近 +返 +还 +这 +进 +远 +违 +连 +迟 +迢 +迤 +迥 +迦 +迩 +迪 +迫 +迭 +述 +迴 +迷 +迸 +迹 +迺 +追 +退 +送 +适 +逃 +逅 +逆 +选 +逊 +逍 +透 +逐 +递 +途 +逕 +逗 +這 +通 +逛 +逝 +逞 +速 +造 +逢 +連 +逮 +週 +進 +逵 +逶 +逸 +逻 +逼 +逾 +遁 +遂 +遅 +遇 +遊 +運 +遍 +過 +遏 +遐 +遑 +遒 +道 +達 +違 +遗 +遙 +遛 +遜 +遞 +遠 +遢 +遣 +遥 +遨 +適 +遭 +遮 +遲 +遴 +遵 +遶 +遷 +選 +遺 +遼 +遽 +避 +邀 +邁 +邂 +邃 +還 +邇 +邈 +邊 +邋 +邏 +邑 +邓 +邕 +邛 +邝 +邢 +那 +邦 +邨 +邪 +邬 +邮 +邯 +邰 +邱 +邳 +邵 +邸 +邹 +邺 +邻 +郁 +郅 +郊 +郎 +郑 +郜 +郝 +郡 +郢 +郤 +郦 +郧 +部 +郫 +郭 +郴 +郵 +郷 +郸 +都 +鄂 +鄉 +鄒 +鄔 +鄙 +鄞 +鄢 +鄧 +鄭 +鄰 +鄱 +鄲 +鄺 +酉 +酊 +酋 +酌 +配 +酐 +酒 +酗 +酚 +酝 +酢 +酣 +酥 +酩 +酪 +酬 +酮 +酯 +酰 +酱 +酵 +酶 +酷 +酸 +酿 +醃 +醇 +醉 +醋 +醍 +醐 +醒 +醚 +醛 +醜 +醞 +醣 +醪 +醫 +醬 +醮 +醯 +醴 +醺 +釀 +釁 +采 +釉 +释 +釋 +里 +重 +野 +量 +釐 +金 +釗 +釘 +釜 +針 +釣 +釦 +釧 +釵 +鈀 +鈉 +鈍 +鈎 +鈔 +鈕 +鈞 +鈣 +鈦 +鈪 +鈴 +鈺 +鈾 +鉀 +鉄 +鉅 +鉉 +鉑 +鉗 +鉚 +鉛 +鉤 +鉴 +鉻 +銀 +銃 +銅 +銑 +銓 +銖 +銘 +銜 +銬 +銭 +銮 +銳 +銷 +銹 +鋁 +鋅 +鋒 +鋤 +鋪 +鋰 +鋸 +鋼 +錄 +錐 +錘 +錚 +錠 +錢 +錦 +錨 +錫 +錮 +錯 +録 +錳 +錶 +鍊 +鍋 +鍍 +鍛 +鍥 +鍰 +鍵 +鍺 +鍾 +鎂 +鎊 +鎌 +鎏 +鎔 +鎖 +鎗 +鎚 +鎧 +鎬 +鎮 +鎳 +鏈 +鏖 +鏗 +鏘 +鏞 +鏟 +鏡 +鏢 +鏤 +鏽 +鐘 +鐮 +鐲 +鐳 +鐵 +鐸 +鐺 +鑄 +鑊 +鑑 +鑒 +鑣 +鑫 +鑰 +鑲 +鑼 +鑽 +鑾 +鑿 +针 +钉 +钊 +钎 +钏 +钒 +钓 +钗 +钙 +钛 +钜 +钝 +钞 +钟 +钠 +钡 +钢 +钣 +钤 +钥 +钦 +钧 +钨 +钩 +钮 +钯 +钰 +钱 +钳 +钴 +钵 +钺 +钻 +钼 +钾 +钿 +铀 +铁 +铂 +铃 +铄 +铅 +铆 +铉 +铎 +铐 +铛 +铜 +铝 +铠 +铡 +铢 +铣 +铤 +铨 +铩 +铬 +铭 +铮 +铰 +铲 +铵 +银 +铸 +铺 +链 +铿 +销 +锁 +锂 +锄 +锅 +锆 +锈 +锉 +锋 +锌 +锏 +锐 +锑 +错 +锚 +锟 +锡 +锢 +锣 +锤 +锥 +锦 +锭 +键 +锯 +锰 +锲 +锵 +锹 +锺 +锻 +镀 +镁 +镂 +镇 +镉 +镌 +镍 +镐 +镑 +镕 +镖 +镗 +镛 +镜 +镣 +镭 +镯 +镰 +镳 +镶 +長 +长 +門 +閃 +閉 +開 +閎 +閏 +閑 +閒 +間 +閔 +閘 +閡 +関 +閣 +閥 +閨 +閩 +閱 +閲 +閹 +閻 +閾 +闆 +闇 +闊 +闌 +闍 +闔 +闕 +闖 +闘 +關 +闡 +闢 +门 +闪 +闫 +闭 +问 +闯 +闰 +闲 +间 +闵 +闷 +闸 +闹 +闺 +闻 +闽 +闾 +阀 +阁 +阂 +阅 +阆 +阇 +阈 +阉 +阎 +阐 +阑 +阔 +阕 +阖 +阙 +阚 +阜 +队 +阡 +阪 +阮 +阱 +防 +阳 +阴 +阵 +阶 +阻 +阿 +陀 +陂 +附 +际 +陆 +陇 +陈 +陋 +陌 +降 +限 +陕 +陛 +陝 +陞 +陟 +陡 +院 +陣 +除 +陨 +险 +陪 +陰 +陲 +陳 +陵 +陶 +陷 +陸 +険 +陽 +隅 +隆 +隈 +隊 +隋 +隍 +階 +随 +隐 +隔 +隕 +隘 +隙 +際 +障 +隠 +隣 +隧 +隨 +險 +隱 +隴 +隶 +隸 +隻 +隼 +隽 +难 +雀 +雁 +雄 +雅 +集 +雇 +雉 +雋 +雌 +雍 +雎 +雏 +雑 +雒 +雕 +雖 +雙 +雛 +雜 +雞 +離 +難 +雨 +雪 +雯 +雰 +雲 +雳 +零 +雷 +雹 +電 +雾 +需 +霁 +霄 +霆 +震 +霈 +霉 +霊 +霍 +霎 +霏 +霑 +霓 +霖 +霜 +霞 +霧 +霭 +霰 +露 +霸 +霹 +霽 +霾 +靂 +靄 +靈 +青 +靓 +靖 +静 +靚 +靛 +靜 +非 +靠 +靡 +面 +靥 +靦 +革 +靳 +靴 +靶 +靼 +鞅 +鞋 +鞍 +鞏 +鞑 +鞘 +鞠 +鞣 +鞦 +鞭 +韆 +韋 +韌 +韓 +韜 +韦 +韧 +韩 +韬 +韭 +音 +韵 +韶 +韻 +響 +頁 +頂 +頃 +項 +順 +須 +頌 +預 +頑 +頒 +頓 +頗 +領 +頜 +頡 +頤 +頫 +頭 +頰 +頷 +頸 +頹 +頻 +頼 +顆 +題 +額 +顎 +顏 +顔 +願 +顛 +類 +顧 +顫 +顯 +顱 +顴 +页 +顶 +顷 +项 +顺 +须 +顼 +顽 +顾 +顿 +颁 +颂 +预 +颅 +领 +颇 +颈 +颉 +颊 +颌 +颍 +颐 +频 +颓 +颔 +颖 +颗 +题 +颚 +颛 +颜 +额 +颞 +颠 +颡 +颢 +颤 +颦 +颧 +風 +颯 +颱 +颳 +颶 +颼 +飄 +飆 +风 +飒 +飓 +飕 +飘 +飙 +飚 +飛 +飞 +食 +飢 +飨 +飩 +飪 +飯 +飲 +飼 +飽 +飾 +餃 +餅 +餉 +養 +餌 +餐 +餒 +餓 +餘 +餚 +餛 +餞 +餡 +館 +餮 +餵 +餾 +饅 +饈 +饋 +饌 +饍 +饑 +饒 +饕 +饗 +饞 +饥 +饨 +饪 +饬 +饭 +饮 +饯 +饰 +饱 +饲 +饴 +饵 +饶 +饷 +饺 +饼 +饽 +饿 +馀 +馁 +馄 +馅 +馆 +馈 +馋 +馍 +馏 +馒 +馔 +首 +馗 +香 +馥 +馨 +馬 +馭 +馮 +馳 +馴 +駁 +駄 +駅 +駆 +駐 +駒 +駕 +駛 +駝 +駭 +駱 +駿 +騁 +騎 +騏 +験 +騙 +騨 +騰 +騷 +驀 +驅 +驊 +驍 +驒 +驕 +驗 +驚 +驛 +驟 +驢 +驥 +马 +驭 +驮 +驯 +驰 +驱 +驳 +驴 +驶 +驷 +驸 +驹 +驻 +驼 +驾 +驿 +骁 +骂 +骄 +骅 +骆 +骇 +骈 +骊 +骋 +验 +骏 +骐 +骑 +骗 +骚 +骛 +骜 +骞 +骠 +骡 +骤 +骥 +骧 +骨 +骯 +骰 +骶 +骷 +骸 +骼 +髂 +髅 +髋 +髏 +髒 +髓 +體 +髖 +高 +髦 +髪 +髮 +髯 +髻 +鬃 +鬆 +鬍 +鬓 +鬚 +鬟 +鬢 +鬣 +鬥 +鬧 +鬱 +鬼 +魁 +魂 +魄 +魅 +魇 +魍 +魏 +魔 +魘 +魚 +魯 +魷 +鮑 +鮨 +鮪 +鮭 +鮮 +鯉 +鯊 +鯖 +鯛 +鯨 +鯰 +鯽 +鰍 +鰓 +鰭 +鰲 +鰻 +鰾 +鱈 +鱉 +鱔 +鱗 +鱷 +鱸 +鱼 +鱿 +鲁 +鲈 +鲍 +鲑 +鲛 +鲜 +鲟 +鲢 +鲤 +鲨 +鲫 +鲱 +鲲 +鲶 +鲷 +鲸 +鳃 +鳄 +鳅 +鳌 +鳍 +鳕 +鳖 +鳗 +鳝 +鳞 +鳥 +鳩 +鳳 +鳴 +鳶 +鴉 +鴕 +鴛 +鴦 +鴨 +鴻 +鴿 +鵑 +鵜 +鵝 +鵡 +鵬 +鵰 +鵲 +鶘 +鶩 +鶯 +鶴 +鷗 +鷲 +鷹 +鷺 +鸚 +鸞 +鸟 +鸠 +鸡 +鸢 +鸣 +鸥 +鸦 +鸨 +鸪 +鸭 +鸯 +鸳 +鸵 +鸽 +鸾 +鸿 +鹂 +鹃 +鹄 +鹅 +鹈 +鹉 +鹊 +鹌 +鹏 +鹑 +鹕 +鹘 +鹜 +鹞 +鹤 +鹦 +鹧 +鹫 +鹭 +鹰 +鹳 +鹵 +鹹 +鹼 +鹽 +鹿 +麂 +麋 +麒 +麓 +麗 +麝 +麟 +麥 +麦 +麩 +麴 +麵 +麸 +麺 +麻 +麼 +麽 +麾 +黃 +黄 +黍 +黎 +黏 +黑 +黒 +黔 +默 +黛 +黜 +黝 +點 +黠 +黨 +黯 +黴 +鼋 +鼎 +鼐 +鼓 +鼠 +鼬 +鼹 +鼻 +鼾 +齁 +齊 +齋 +齐 +齒 +齡 +齢 +齣 +齦 +齿 +龄 +龅 +龈 +龊 +龋 +龌 +龍 +龐 +龔 +龕 +龙 +龚 +龛 +龜 +龟 +︰ +︱ +︶ +︿ +﹁ +﹂ +﹍ +﹏ +﹐ +﹑ +﹒ +﹔ +﹕ +﹖ +﹗ +﹙ +﹚ +﹝ +﹞ +﹡ +﹣ +！ +＂ +＃ +＄ +％ +＆ +＇ +（ +） +＊ +＋ +， +－ +． +／ +０ +１ +２ +３ +４ +５ +６ +７ +８ +９ +： +； +＜ +＝ +＞ +？ +＠ +［ +＼ +］ +＾ +＿ +｀ +ａ +ｂ +ｃ +ｄ +ｅ +ｆ +ｇ +ｈ +ｉ +ｊ +ｋ +ｌ +ｍ +ｎ +ｏ +ｐ +ｑ +ｒ +ｓ +ｔ +ｕ +ｖ +ｗ +ｘ +ｙ +ｚ +｛ +｜ +｝ +～ +｡ +｢ +｣ +､ +･ +ｯ +ｰ +ｲ +ｸ +ｼ +ｽ +ﾄ +ﾉ +ﾌ +ﾗ +ﾙ +ﾝ +ﾞ +ﾟ +￣ +￥ +👍 +🔥 +😂 +😎 +... +yam +10 +2017 +12 +11 +2016 +20 +30 +15 +06 +lofter +##s +2015 +by +16 +14 +18 +13 +24 +17 +2014 +21 +##0 +22 +19 +25 +23 +com +100 +00 +05 +2013 +##a +03 +09 +08 +28 +##2 +50 +01 +04 +##1 +27 +02 +2012 +##3 +26 +##e +07 +##8 +##5 +##6 +##4 +##9 +##7 +29 +2011 +40 +##t +2010 +##o +##d +##i +2009 +##n +app +www +the +##m +31 +##c +##l +##y +##r +##g +2008 +60 +http +200 +qq +##p +80 +##f +google +pixnet +90 +cookies +tripadvisor +500 +##er +##k +35 +##h +facebook +2007 +2000 +70 +##b +of +##x +##u +45 +300 +iphone +32 +1000 +2006 +48 +ip +36 +in +38 +3d +##w +##ing +55 +ctrip +##on +##v +33 +##の +to +34 +400 +id +2005 +it +37 +windows +llc +top +99 +42 +39 +000 +led +at +##an +41 +51 +52 +46 +49 +43 +53 +44 +##z +android +58 +and +59 +2004 +56 +vr +##か +5000 +2003 +47 +blogthis +twitter +54 +##le +150 +ok +2018 +57 +75 +cn +no +ios +##in +##mm +##00 +800 +on +te +3000 +65 +2001 +360 +95 +ig +lv +120 +##ng +##を +##us +##に +pc +てす +── +600 +##te +85 +2002 +88 +##ed +html +ncc +wifi +email +64 +blog +is +##10 +##て +mail +online +##al +dvd +##ic +studio +##は +##℃ +##ia +##と +line +vip +72 +##q +98 +##ce +##en +for +##is +##ra +##es +##j +usb +net +cp +1999 +asia +4g +##cm +diy +new +3c +##お +ta +66 +language +vs +apple +tw +86 +web +##ne +ipad +62 +you +##re +101 +68 +##tion +ps +de +bt +pony +atm +##2017 +1998 +67 +##ch +ceo +##or +go +##na +av +pro +cafe +96 +pinterest +97 +63 +pixstyleme3c +##ta +more +said +##2016 +1997 +mp3 +700 +##ll +nba +jun +##20 +92 +tv +1995 +pm +61 +76 +nbsp +250 +##ie +linux +##ma +cd +110 +hd +##17 +78 +##ion +77 +6000 +am +##th +##st +94 +##se +##et +69 +180 +gdp +my +105 +81 +abc +89 +flash +79 +one +93 +1990 +1996 +##ck +gps +##も +##ly +web885 +106 +2020 +91 +##ge +4000 +1500 +xd +boss +isbn +1994 +org +##ry +me +love +##11 +0fork +73 +##12 +3g +##ter +##ar +71 +82 +##la +hotel +130 +1970 +pk +83 +87 +140 +ie +##os +##30 +##el +74 +##50 +seo +cpu +##ml +p2p +84 +may +##る +sun +tue +internet +cc +posted +youtube +##at +##ン +##man +ii +##ル +##15 +abs +nt +pdf +yahoo +ago +1980 +##it +news +mac +104 +##てす +##me +##り +java +1992 +spa +##de +##nt +hk +all +plus +la +1993 +##mb +##16 +##ve +west +##da +160 +air +##い +##ps +から +##to +1989 +logo +htc +php +https +fi +momo +##son +sat +##ke +##80 +ebd +suv +wi +day +apk +##88 +##um +mv +galaxy +wiki +or +brake +##ス +1200 +する +this +1991 +mon +##こ +❤2017 +po +##ない +javascript +life +home +june +##ss +system +900 +##ー +##０ +pp +1988 +world +fb +4k +br +##as +ic +ai +leonardo +safari +##60 +live +free +xx +wed +win7 +kiehl +##co +lg +o2o +##go +us +235 +1949 +mm +しい +vfm +kanye +##90 +##2015 +##id +jr +##ey +123 +rss +##sa +##ro +##am +##no +thu +fri +350 +##sh +##ki +103 +comments +name +##のて +##pe +##ine +max +1987 +8000 +uber +##mi +##ton +wordpress +office +1986 +1985 +##ment +107 +bd +win10 +##ld +##li +gmail +bb +dior +##rs +##ri +##rd +##ます +up +cad +##® +dr +して +read +##21 +をお +##io +##99 +url +1984 +pvc +paypal +show +policy +##40 +##ty +##18 +with +##★ +##01 +txt +102 +##ba +dna +from +post +mini +ar +taiwan +john +##ga +privacy +agoda +##13 +##ny +word +##24 +##22 +##by +##ur +##hz +1982 +##ang +265 +cookie +netscape +108 +##ka +##～ +##ad +house +share +note +ibm +code +hello +nike +sim +survey +##016 +1979 +1950 +wikia +##32 +##017 +5g +cbc +##tor +##kg +1983 +##rt +##14 +campaign +store +2500 +os +##ct +##ts +##° +170 +api +##ns +365 +excel +##な +##ao +##ら +##し +～～ +##nd +university +163 +には +518 +##70 +##ya +##il +##25 +pierre +ipo +0020 +897 +##23 +hotels +##ian +のお +125 +years +6606 +##ers +##26 +high +##day +time +##ay +bug +##line +##く +##す +##be +xp +talk2yam +yamservice +10000 +coco +##dy +sony +##ies +1978 +microsoft +david +people +##ha +1960 +instagram +intel +その +##ot +iso +1981 +##va +115 +##mo +##land +xxx +man +co +ltxsw +##ation +baby +220 +##pa +##ol +1945 +7000 +tag +450 +##ue +msn +##31 +oppo +##ト +##ca +control +##om +st +chrome +##ure +##ん +be +##き +lol +##19 +した +##bo +240 +lady +##100 +##way +##から +4600 +##ko +##do +##un +4s +corporation +168 +##ni +herme +##28 +ｃｐ +978 +##up +##06 +ui +##ds +ppt +admin +three +します +bbc +re +128 +##48 +ca +##015 +##35 +hp +##ee +tpp +##た +##ive +×× +root +##cc +##ました +##ble +##ity +adobe +park +114 +et +oled +city +##ex +##ler +##ap +china +##book +20000 +view +##ice +global +##km +your +hong +##mg +out +##ms +ng +ebay +##29 +menu +ubuntu +##cy +rom +##view +open +ktv +do +server +##lo +if +english +##ね +##５ +##oo +1600 +##02 +step1 +kong +club +135 +july +inc +1976 +mr +hi +##net +touch +##ls +##ii +michael +lcd +##05 +##33 +phone +james +step2 +1300 +ios9 +##box +dc +##２ +##ley +samsung +111 +280 +pokemon +css +##ent +##les +いいえ +##１ +s8 +atom +play +bmw +##said +sa +etf +ctrl +♥yoyo♥ +##55 +2025 +##2014 +##66 +adidas +amazon +1958 +##ber +##ner +visa +##77 +##der +1800 +connectivity +##hi +firefox +109 +118 +hr +so +style +mark +pop +ol +skip +1975 +as +##27 +##ir +##61 +190 +mba +##う +##ai +le +##ver +1900 +cafe2017 +lte +super +113 +129 +##ron +amd +like +##☆ +are +##ster +we +##sk +paul +data +international +##ft +longchamp +ssd +good +##ート +##ti +reply +##my +↓↓↓ +apr +star +##ker +source +136 +js +112 +get +force +photo +##one +126 +##2013 +##ow +link +bbs +1972 +goods +##lin +python +119 +##ip +game +##ics +##ません +blue +##● +520 +##45 +page +itunes +##03 +1955 +260 +1968 +gt +gif +618 +##ff +##47 +group +くたさい +about +bar +ganji +##nce +music +lee +not +1977 +1971 +1973 +##per +an +faq +comment +##って +days +##ock +116 +##bs +1974 +1969 +v1 +player +1956 +xbox +sql +fm +f1 +139 +##ah +210 +##lv +##mp +##000 +melody +1957 +##３ +550 +17life +199 +1966 +xml +market +##au +##71 +999 +##04 +what +gl +##95 +##age +tips +##68 +book +##ting +mysql +can +1959 +230 +##ung +wonderland +watch +10℃ +##ction +9000 +mar +mobile +1946 +1962 +article +##db +part +▲top +party +って +1967 +1964 +1948 +##07 +##ore +##op +この +dj +##78 +##38 +010 +main +225 +1965 +##ong +art +320 +ad +134 +020 +##73 +117 +pm2 +japan +228 +##08 +ts +1963 +##ica +der +sm +##36 +2019 +##wa +ct +##７ +##や +##64 +1937 +homemesh +search +##85 +##れは +##tv +##di +macbook +##９ +##くたさい +service +##♥ +type +った +750 +##ier +##si +##75 +##います +##ok +best +##ット +goris +lock +##った +cf +3m +big +##ut +ftp +carol +##vi +１０ +1961 +happy +sd +##ac +122 +anti +pe +cnn +iii +1920 +138 +##ラ +1940 +esp +jan +tags +##98 +##51 +august +vol +##86 +154 +##™ +##fs +##れ +##sion +design +ac +##ム +press +jordan +ppp +that +key +check +##６ +##tt +##㎡ +1080p +##lt +power +##42 +1952 +##bc +vivi +##ック +he +133 +121 +jpg +##rry +201 +175 +3500 +1947 +nb +##ted +##rn +しています +1954 +usd +##t00 +master +##ンク +001 +model +##58 +al +##09 +1953 +##34 +ram +goo +ても +##ui +127 +1930 +red +##ary +rpg +item +##pm +##41 +270 +##za +project +##2012 +hot +td +blogabstract +##ger +##62 +650 +##44 +gr2 +##します +##ｍ +black +electronic +nfc +year +asus +また +html5 +cindy +##hd +m3 +132 +esc +##od +booking +##53 +fed +tvb +##81 +##ina +mit +165 +##いる +chan +192 +distribution +next +になる +peter +bios +steam +cm +1941 +にも +pk10 +##ix +##65 +##91 +dec +nasa +##ana +icecat +00z +b1 +will +##46 +li +se +##ji +##み +##ard +oct +##ain +jp +##ze +##bi +cio +##56 +smart +h5 +##39 +##port +curve +vpn +##nm +##dia +utc +##あり +12345678910 +##52 +rmvb +chanel +a4 +miss +##and +##im +media +who +##63 +she +girl +5s +124 +vera +##して +class +vivo +king +##フ +##ei +national +ab +1951 +5cm +888 +145 +ipod +ap +1100 +5mm +211 +ms +2756 +##69 +mp4 +msci +##po +##89 +131 +mg +index +380 +##bit +##out +##zz +##97 +##67 +158 +apec +##８ +photoshop +opec +￥799 +ては +##96 +##tes +##ast +2g +○○ +##ール +￥2899 +##ling +##よ +##ory +1938 +##ical +kitty +content +##43 +step3 +##cn +win8 +155 +vc +1400 +iphone7 +robert +##した +tcl +137 +beauty +##87 +en +dollars +##ys +##oc +step +pay +yy +a1 +##2011 +##lly +##ks +##♪ +1939 +188 +download +1944 +sep +exe +ph +います +school +gb +center +pr +street +##board +uv +##37 +##lan +winrar +##que +##ua +##com +1942 +1936 +480 +gpu +##４ +ettoday +fu +tom +##54 +##ren +##via +149 +##72 +b2b +144 +##79 +##tch +rose +arm +mb +##49 +##ial +##nn +nvidia +step4 +mvp +00㎡ +york +156 +##イ +how +cpi +591 +2765 +gov +kg +joe +##xx +mandy +pa +##ser +copyright +fashion +1935 +don +##け +ecu +##ist +##art +erp +wap +have +##lm +talk +##ek +##ning +##if +ch +##ite +video +1943 +cs +san +iot +look +##84 +##2010 +##ku +october +##ux +trump +##hs +##ide +box +141 +first +##ins +april +##ight +##83 +185 +angel +protected +aa +151 +162 +x1 +m2 +##fe +##× +##ho +size +143 +min +ofo +fun +gomaji +ex +hdmi +food +dns +march +chris +kevin +##のか +##lla +##pp +##ec +ag +ems +6s +720p +##rm +##ham +off +##92 +asp +team +fandom +ed +299 +▌♥ +##ell +info +されています +##82 +sina +4066 +161 +##able +##ctor +330 +399 +315 +dll +rights +ltd +idc +jul +3kg +1927 +142 +ma +surface +##76 +##ク +～～～ +304 +mall +eps +146 +green +##59 +map +space +donald +v2 +sodu +##light +1931 +148 +1700 +まて +310 +reserved +htm +##han +##57 +2d +178 +mod +##ise +##tions +152 +ti +##shi +doc +1933 +icp +055 +wang +##ram +shopping +aug +##pi +##well +now +wam +b2 +からお +##hu +236 +1928 +##gb +266 +f2 +##93 +153 +mix +##ef +##uan +bwl +##plus +##res +core +##ess +tea +5℃ +hktvmall +nhk +##ate +list +##ese +301 +feb +4m +inn +ての +nov +159 +12345 +daniel +##ci +pass +##bet +##nk +coffee +202 +ssl +airbnb +##ute +fbi +woshipm +skype +ea +cg +sp +##fc +##www +yes +edge +alt +007 +##94 +fpga +##ght +##gs +iso9001 +さい +##ile +##wood +##uo +image +lin +icon +american +##em +1932 +set +says +##king +##tive +blogger +##74 +なと +256 +147 +##ox +##zy +##red +##ium +##lf +nokia +claire +##リ +##ding +november +lohas +##500 +##tic +##マ +##cs +##ある +##che +##ire +##gy +##ult +db +january +win +##カ +166 +road +ptt +##ま +##つ +198 +##fa +##mer +anna +pchome +はい +udn +ef +420 +##time +##tte +2030 +##ア +g20 +white +かかります +1929 +308 +garden +eleven +di +##おります +chen +309b +777 +172 +young +cosplay +ちてない +4500 +bat +##123 +##tra +##ては +kindle +npc +steve +etc +##ern +##｜ +call +xperia +ces +travel +sk +s7 +##ous +1934 +##int +みいたたけます +183 +edu +file +cho +qr +##car +##our +186 +##ant +##ｄ +eric +1914 +rends +##jo +##する +mastercard +##2000 +kb +##min +290 +##ino +vista +##ris +##ud +jack +2400 +##set +169 +pos +1912 +##her +##ou +taipei +しく +205 +beta +##ませんか +232 +##fi +express +255 +body +##ill +aphojoy +user +december +meiki +##ick +tweet +richard +##av +##ᆫ +iphone6 +##dd +ちてすか +views +##mark +321 +pd +##００ +times +##▲ +level +##ash +10g +point +5l +##ome +208 +koreanmall +##ak +george +q2 +206 +wma +tcp +##200 +スタッフ +full +mlb +##lle +##watch +tm +run +179 +911 +smith +business +##und +1919 +color +##tal +222 +171 +##less +moon +4399 +##rl +update +pcb +shop +499 +157 +little +なし +end +##mhz +van +dsp +easy +660 +##house +##key +history +##ｏ +oh +##001 +##hy +##web +oem +let +was +##2009 +##gg +review +##wan +182 +##°c +203 +uc +title +##val +united +233 +2021 +##ons +doi +trivago +overdope +sbs +##ance +##ち +grand +special +573032185 +imf +216 +wx17house +##so +##ーム +audi +##he +london +william +##rp +##ake +science +beach +cfa +amp +ps4 +880 +##800 +##link +##hp +crm +ferragamo +bell +make +##eng +195 +under +zh +photos +2300 +##style +##ント +via +176 +da +##gi +company +i7 +##ray +thomas +370 +ufo +i5 +##max +plc +ben +back +research +8g +173 +mike +##pc +##ッフ +september +189 +##ace +vps +february +167 +pantos +wp +lisa +1921 +★★ +jquery +night +long +offer +##berg +##news +1911 +##いて +ray +fks +wto +せます +over +164 +340 +##all +##rus +1924 +##888 +##works +blogtitle +loftpermalink +##→ +187 +martin +test +ling +km +##め +15000 +fda +v3 +##ja +##ロ +ｗedding +かある +outlet +family +##ea +をこ +##top +story +##ness +salvatore +##lu +204 +swift +215 +room +している +oracle +##ul +1925 +sam +b2c +week +pi +rock +##のは +##ａ +##けと +##ean +##300 +##gle +cctv +after +chinese +##back +powered +x2 +##tan +1918 +##nes +##イン +canon +only +181 +##zi +##las +say +##oe +184 +##sd +221 +##bot +##world +##zo +sky +made +top100 +just +1926 +pmi +802 +234 +gap +##vr +177 +les +174 +▲topoct +ball +vogue +vi +ing +ofweek +cos +##list +##ort +▲topmay +##なら +##lon +として +last +##tc +##of +##bus +##gen +real +eva +##コ +a3 +nas +##lie +##ria +##coin +##bt +▲topapr +his +212 +cat +nata +vive +health +⋯⋯ +drive +sir +▲topmar +du +cup +##カー +##ook +##よう +##sy +alex +msg +tour +しました +3ce +##word +193 +ebooks +r8 +block +318 +##より +2200 +nice +pvp +207 +months +1905 +rewards +##ther +1917 +0800 +##xi +##チ +##sc +micro +850 +gg +blogfp +op +1922 +daily +m1 +264 +true +##bb +ml +##tar +##のお +##ky +anthony +196 +253 +##yo +state +218 +##ara +##aa +##rc +##tz +##ston +より +gear +##eo +##ade +ge +see +1923 +##win +##ura +ss +heart +##den +##ita +down +##sm +el +png +2100 +610 +rakuten +whatsapp +bay +dream +add +##use +680 +311 +pad +gucci +mpv +##ode +##fo +island +▲topjun +##▼ +223 +jason +214 +chicago +##❤ +しの +##hone +io +##れる +##ことか +sogo +be2 +##ology +990 +cloud +vcd +##con +2～3 +##ford +##joy +##kb +##こさいます +##rade +but +##ach +docker +##ful +rfid +ul +##ase +hit +ford +##star +580 +##○ +１１ +a2 +sdk +reading +edited +##are +cmos +##mc +238 +siri +light +##ella +##ため +bloomberg +##read +pizza +##ison +jimmy +##vm +college +node +journal +ba +18k +##play +245 +##cer +２０ +magic +##yu +191 +jump +288 +tt +##ings +asr +##lia +3200 +step5 +network +##cd +mc +いします +1234 +pixstyleme +273 +##600 +2800 +money +★★★★★ +1280 +１２ +430 +bl +みの +act +##tus +tokyo +##rial +##life +emba +##ae +saas +tcs +##rk +##wang +summer +##sp +ko +##ving +390 +premium +##その +netflix +##ヒ +uk +mt +##lton +right +frank +two +209 +える +##ple +##cal +021 +##んな +##sen +##ville +hold +nexus +dd +##ius +てお +##mah +##なく +tila +zero +820 +ce +##tin +resort +##ws +charles +old +p10 +5d +report +##360 +##ru +##には +bus +vans +lt +##est +pv +##レ +links +rebecca +##ツ +##dm +azure +##365 +きな +limited +bit +4gb +##mon +1910 +moto +##eam +213 +1913 +var +eos +なとの +226 +blogspot +された +699 +e3 +dos +dm +fc +##ments +##ik +##kw +boy +##bin +##ata +960 +er +##せ +219 +##vin +##tu +##ula +194 +##∥ +station +##ろ +##ature +835 +files +zara +hdr +top10 +nature +950 +magazine +s6 +marriott +##シ +avira +case +##っと +tab +##ran +tony +##home +oculus +im +##ral +jean +saint +cry +307 +rosie +##force +##ini +ice +##bert +のある +##nder +##mber +pet +2600 +##◆ +plurk +▲topdec +##sis +00kg +▲topnov +720 +##ence +tim +##ω +##nc +##ても +##name +log +ips +great +ikea +malaysia +unix +##イト +3600 +##ncy +##nie +12000 +akb48 +##ye +##oid +404 +##chi +##いた +oa +xuehai +##1000 +##orm +##rf +275 +さん +##ware +##リー +980 +ho +##pro +text +##era +560 +bob +227 +##ub +##2008 +8891 +scp +avi +##zen +2022 +mi +wu +museum +qvod +apache +lake +jcb +▲topaug +★★★ +ni +##hr +hill +302 +ne +weibo +490 +ruby +##ーシ +##ヶ +##row +4d +▲topjul +iv +##ish +github +306 +mate +312 +##スト +##lot +##ane +andrew +のハイト +##tina +t1 +rf +ed2k +##vel +##900 +way +final +りの +ns +5a +705 +197 +##メ +sweet +bytes +##ene +▲topjan +231 +##cker +##2007 +##px +100g +topapp +229 +helpapp +rs +low +14k +g4g +care +630 +ldquo +あり +##fork +leave +rm +edition +##gan +##zon +##qq +▲topsep +##google +##ism +gold +224 +explorer +##zer +toyota +category +select +visual +##labels +restaurant +##md +posts +s1 +##ico +もっと +angelababy +123456 +217 +sports +s3 +mbc +1915 +してくたさい +shell +x86 +candy +##new +kbs +face +xl +470 +##here +4a +swissinfo +v8 +▲topfeb +dram +##ual +##vice +3a +##wer +sport +q1 +ios10 +public +int +card +##ｃ +ep +au +rt +##れた +1080 +bill +##mll +kim +３０ +460 +wan +##uk +##ミ +x3 +298 +0t +scott +##ming +239 +e5 +##3d +h7n9 +worldcat +brown +##あります +##vo +##led +##580 +##ax +249 +410 +##ert +paris +##～6 +polo +925 +##lr +599 +##ナ +capital +##hing +bank +cv +1g +##chat +##ｓ +##たい +adc +##ule +2m +##ｅ +digital +hotmail +268 +##pad +870 +bbq +quot +##ring +before +wali +##まて +mcu +2k +2b +という +costco +316 +north +333 +switch +##city +##ｐ +philips +##mann +management +panasonic +##cl +##vd +##ping +##rge +alice +##lk +##ましょう +css3 +##ney +vision +alpha +##ular +##400 +##tter +lz +にお +##ありません +mode +gre +1916 +pci +##tm +237 +1～2 +##yan +##そ +について +##let +##キ +work +war +coach +ah +mary +##ᅵ +huang +##pt +a8 +pt +follow +##berry +1895 +##ew +a5 +ghost +##ション +##wn +##og +south +##code +girls +##rid +action +villa +git +r11 +table +games +##cket +error +##anonymoussaid +##ag +here +##ame +##gc +qa +##■ +##lis +gmp +##gin +vmalife +##cher +yu +wedding +##tis +demo +dragon +530 +soho +social +bye +##rant +river +orz +acer +325 +##↑ +##ース +##ats +261 +del +##ven +440 +ups +##ように +##ター +305 +value +macd +yougou +##dn +661 +##ano +ll +##urt +##rent +continue +script +##wen +##ect +paper +263 +319 +shift +##chel +##フト +##cat +258 +x5 +fox +243 +##さん +car +aaa +##blog +loading +##yn +##tp +kuso +799 +si +sns +イカせるテンマ +ヒンクテンマ3 +rmb +vdc +forest +central +prime +help +ultra +##rmb +##ような +241 +square +688 +##しい +のないフロクに +##field +##reen +##ors +##ju +c1 +start +510 +##air +##map +cdn +##wo +cba +stephen +m8 +100km +##get +opera +##base +##ood +vsa +com™ +##aw +##ail +251 +なのて +count +t2 +##ᅡ +##een +2700 +hop +##gp +vsc +tree +##eg +##ose +816 +285 +##ories +##shop +alphago +v4 +1909 +simon +##ᆼ +fluke62max +zip +スホンサー +##sta +louis +cr +bas +##～10 +bc +##yer +hadoop +##ube +##wi +1906 +0755 +hola +##low +place +centre +5v +d3 +##fer +252 +##750 +##media +281 +540 +0l +exchange +262 +series +##ハー +##san +eb +##bank +##ｋ +q3 +##nge +##mail +take +##lp +259 +1888 +client +east +cache +event +vincent +##ールを +きを +##nse +sui +855 +adchoice +##и +##stry +##なたの +246 +##zone +ga +apps +sea +##ab +248 +cisco +##タ +##rner +kymco +##care +dha +##pu +##yi +minkoff +royal +p1 +への +annie +269 +collection +kpi +playstation +257 +になります +866 +bh +##bar +queen +505 +radio +1904 +andy +armani +##xy +manager +iherb +##ery +##share +spring +raid +johnson +1908 +##ob +volvo +hall +##ball +v6 +our +taylor +##hk +bi +242 +##cp +kate +bo +water +technology +##rie +サイトは +277 +##ona +##sl +hpv +303 +gtx +hip +rdquo +jayz +stone +##lex +##rum +namespace +##やり +620 +##ale +##atic +des +##erson +##ql +##ves +##type +enter +##この +##てきます +d2 +##168 +##mix +##bian +との +a9 +jj +ky +##lc +access +movie +##hc +リストに +tower +##ration +##mit +ます +##nch +ua +tel +prefix +##o2 +1907 +##point +1901 +ott +～10 +##http +##ury +baidu +##ink +member +##logy +bigbang +nownews +##js +##shot +##tb +##こと +247 +eba +##tics +##lus +ける +v5 +spark +##ama +there +##ions +god +##lls +##down +hiv +##ress +burberry +day2 +##kv +◆◆ +jeff +related +film +edit +joseph +283 +##ark +cx +32gb +order +g9 +30000 +##ans +##tty +s5 +##bee +かあります +thread +xr +buy +sh +005 +land +spotify +mx +##ari +276 +##verse +×email +sf +why +##ことて +244 +7headlines +nego +sunny +dom +exo +401 +666 +positioning +fit +rgb +##tton +278 +kiss +alexa +adam +lp +みリストを +##ｇ +mp +##ties +##llow +amy +##du +np +002 +institute +271 +##rth +##lar +2345 +590 +##des +sidebar +１５ +imax +site +##cky +##kit +##ime +##009 +season +323 +##fun +##ンター +##ひ +gogoro +a7 +pu +lily +fire +twd600 +##ッセーシを +いて +##vis +30ml +##cture +##をお +information +##オ +close +friday +##くれる +yi +nick +てすか +##tta +##tel +6500 +##lock +cbd +economy +254 +かお +267 +tinker +double +375 +8gb +voice +##app +oops +channel +today +985 +##right +raw +xyz +##＋ +jim +edm +##cent +7500 +supreme +814 +ds +##its +##asia +dropbox +##てすか +##tti +books +272 +100ml +##tle +##ller +##ken +##more +##boy +sex +309 +##dom +t3 +##ider +##なります +##unch +1903 +810 +feel +5500 +##かった +##put +により +s2 +mo +##gh +men +ka +amoled +div +##tr +##n1 +port +howard +##tags +ken +dnf +##nus +adsense +##а +ide +##へ +buff +thunder +##town +##ique +has +##body +auto +pin +##erry +tee +てした +295 +number +##the +##013 +object +psp +cool +udnbkk +16gb +##mic +miui +##tro +most +r2 +##alk +##nity +1880 +±0 +##いました +428 +s4 +law +version +##oa +n1 +sgs +docomo +##tf +##ack +henry +fc2 +##ded +##sco +##014 +##rite +286 +0mm +linkedin +##ada +##now +wii +##ndy +ucbug +##◎ +sputniknews +legalminer +##ika +##xp +2gb +##bu +q10 +oo +b6 +come +##rman +cheese +ming +maker +##gm +nikon +##fig +ppi +kelly +##ります +jchere +てきます +ted +md +003 +fgo +tech +##tto +dan +soc +##gl +##len +hair +earth +640 +521 +img +##pper +##a1 +##てきる +##ロク +acca +##ition +##ference +suite +##ig +outlook +##mond +##cation +398 +##pr +279 +101vip +358 +##999 +282 +64gb +3800 +345 +airport +##over +284 +##おり +jones +##ith +lab +##su +##いるのて +co2 +town +piece +##llo +no1 +vmware +24h +##qi +focus +reader +##admin +##ora +tb +false +##log +1898 +know +lan +838 +##ces +f4 +##ume +motel +stop +##oper +na +flickr +netcomponents +##af +##─ +pose +williams +local +##ound +##cg +##site +##iko +いお +274 +5m +gsm +con +##ath +1902 +friends +##hip +cell +317 +##rey +780 +cream +##cks +012 +##dp +facebooktwitterpinterestgoogle +sso +324 +shtml +song +swiss +##mw +##キンク +lumia +xdd +string +tiffany +522 +marc +られた +insee +russell +sc +dell +##ations +ｏｋ +camera +289 +##vs +##flow +##late +classic +287 +##nter +stay +g1 +mtv +512 +##ever +##lab +##nger +qe +sata +ryan +d1 +50ml +cms +##cing +su +292 +3300 +editor +296 +##nap +security +sunday +association +##ens +##700 +##bra +acg +##かり +sofascore +とは +mkv +##ign +jonathan +gary +build +labels +##oto +tesla +moba +qi +gohappy +general +ajax +1024 +##かる +サイト +society +##test +##urs +wps +fedora +##ich +mozilla +328 +##480 +##dr +usa +urn +##lina +##ｒ +grace +##die +##try +##ader +1250 +##なり +elle +570 +##chen +##ᆯ +price +##ten +uhz +##ough +eq +##hen +states +push +session +balance +wow +506 +##cus +##py +when +##ward +##ep +34e +wong +library +prada +##サイト +##cle +running +##ree +313 +ck +date +q4 +##ctive +##ool +##＞ +mk +##ira +##163 +388 +die +secret +rq +dota +buffet +は１ヶ +e6 +##ez +pan +368 +ha +##card +##cha +2a +##さ +alan +day3 +eye +f3 +##end +france +keep +adi +rna +tvbs +##ala +solo +nova +##え +##tail +##ょう +support +##ries +##なる +##ved +base +copy +iis +fps +##ways +hero +hgih +profile +fish +mu +ssh +entertainment +chang +##wd +click +cake +##ond +pre +##tom +kic +pixel +##ov +##fl +product +6a +##pd +dear +##gate +es +yumi +audio +##² +##sky +echo +bin +where +##ture +329 +##ape +find +sap +isis +##なと +nand +##101 +##load +##ream +band +a6 +525 +never +##post +festival +50cm +##we +555 +guide +314 +zenfone +##ike +335 +gd +forum +jessica +strong +alexander +##ould +software +allen +##ious +program +360° +else +lohasthree +##gar +することかてきます +please +##れます +rc +##ggle +##ric +bim +50000 +##own +eclipse +355 +brian +3ds +##side +061 +361 +##other +##ける +##tech +##ator +485 +engine +##ged +##ｔ +plaza +##fit +cia +ngo +westbrook +shi +tbs +50mm +##みませんか +sci +291 +reuters +##ily +contextlink +##hn +af +##cil +bridge +very +##cel +1890 +cambridge +##ize +15g +##aid +##data +790 +frm +##head +award +butler +##sun +meta +##mar +america +ps3 +puma +pmid +##すか +lc +670 +kitchen +##lic +オーフン5 +きなしソフトサーヒス +そして +day1 +future +★★★★ +##text +##page +##rris +pm1 +##ket +fans +##っています +1001 +christian +bot +kids +trackback +##hai +c3 +display +##hl +n2 +1896 +idea +さんも +##sent +airmail +##ug +##men +pwm +けます +028 +##lution +369 +852 +awards +schemas +354 +asics +wikipedia +font +##tional +##vy +c2 +293 +##れている +##dget +##ein +っている +contact +pepper +スキル +339 +##～5 +294 +##uel +##ument +730 +##hang +みてす +q5 +##sue +rain +##ndi +wei +swatch +##cept +わせ +331 +popular +##ste +##tag +p2 +501 +trc +1899 +##west +##live +justin +honda +ping +messenger +##rap +v9 +543 +##とは +unity +appqq +はすへて +025 +leo +##tone +##テ +##ass +uniqlo +##010 +502 +her +jane +memory +moneydj +##tical +human +12306 +していると +##m2 +coc +miacare +##mn +tmt +##core +vim +kk +##may +fan +target +use +too +338 +435 +2050 +867 +737 +fast +##2c +services +##ope +omega +energy +##わ +pinkoi +1a +##なから +##rain +jackson +##ement +##シャンルの +374 +366 +そんな +p9 +rd +##ᆨ +1111 +##tier +##vic +zone +##│ +385 +690 +dl +isofix +cpa +m4 +322 +kimi +めて +davis +##lay +lulu +##uck +050 +weeks +qs +##hop +920 +##ｎ +ae +##ear +～5 +eia +405 +##fly +korea +jpeg +boost +##ship +small +##リア +1860 +eur +297 +425 +valley +##iel +simple +##ude +rn +k2 +##ena +されます +non +patrick +しているから +##ナー +feed +5757 +30g +process +well +qqmei +##thing +they +aws +lu +pink +##ters +##kin +または +board +##vertisement +wine +##ien +unicode +##dge +r1 +359 +##tant +いを +##twitter +##3c +cool1 +される +##れて +##ｌ +isp +##012 +standard +45㎡2 +402 +##150 +matt +##fu +326 +##iner +googlemsn +pixnetfacebookyahoo +##ラン +x7 +886 +##uce +メーカー +sao +##ev +##きました +##file +9678 +403 +xddd +shirt +6l +##rio +##hat +3mm +givenchy +ya +bang +##lio +monday +crystal +ロクイン +##abc +336 +head +890 +ubuntuforumwikilinuxpastechat +##vc +##～20 +##rity +cnc +7866 +ipv6 +null +1897 +##ost +yang +imsean +tiger +##fet +##ンス +352 +##＝ +dji +327 +ji +maria +##come +##んて +foundation +3100 +##beth +##なった +1m +601 +active +##aft +##don +3p +sr +349 +emma +##khz +living +415 +353 +1889 +341 +709 +457 +sas +x6 +##face +pptv +x4 +##mate +han +sophie +##jing +337 +fifa +##mand +other +sale +inwedding +##gn +てきちゃいます +##mmy +##pmlast +bad +nana +nbc +してみてくたさいね +なとはお +##wu +##かあります +##あ +note7 +single +##340 +せからこ +してくたさい♪この +しにはとんとんワークケートを +するとあなたにもっとマッチした +ならワークケートへ +もみつかっちゃうかも +ワークケートの +##bel +window +##dio +##ht +union +age +382 +１４ +##ivity +##ｙ +コメント +domain +neo +##isa +##lter +5k +f5 +steven +##cts +powerpoint +tft +self +g2 +ft +##テル +zol +##act +mwc +381 +343 +もう +nbapop +408 +てある +eds +ace +##room +previous +author +tomtom +il +##ets +hu +financial +☆☆☆ +っています +bp +5t +chi +1gb +##hg +fairmont +cross +008 +gay +h2 +function +##けて +356 +also +1b +625 +##ータ +##raph +1894 +3～5 +##ils +i3 +334 +avenue +##host +による +##bon +##tsu +message +navigation +50g +fintech +h6 +##ことを +8cm +##ject +##vas +##firm +credit +##wf +xxxx +form +##nor +##space +huawei +plan +json +sbl +##dc +machine +921 +392 +wish +##120 +##sol +windows7 +edward +##ために +development +washington +##nsis +lo +818 +##sio +##ym +##bor +planet +##～8 +##wt +ieee +gpa +##めて +camp +ann +gm +##tw +##oka +connect +##rss +##work +##atus +wall +chicken +soul +2mm +##times +fa +##ather +##cord +009 +##eep +hitachi +gui +harry +##pan +e1 +disney +##press +##ーション +wind +386 +frigidaire +##tl +liu +hsu +332 +basic +von +ev +いた +てきる +スホンサーサイト +learning +##ull +expedia +archives +change +##wei +santa +cut +ins +6gb +turbo +brand +cf1 +508 +004 +return +747 +##rip +h1 +##nis +##をこ +128gb +##にお +3t +application +しており +emc +rx +##oon +384 +quick +412 +15058 +wilson +wing +chapter +##bug +beyond +##cms +##dar +##oh +zoom +e2 +trip +sb +##nba +rcep +342 +aspx +ci +080 +gc +gnu +める +##count +advanced +dance +dv +##url +##ging +367 +8591 +am09 +shadow +battle +346 +##ｉ +##cia +##という +emily +##のてす +##tation +host +ff +techorz +sars +##mini +##mporary +##ering +nc +4200 +798 +##next +cma +##mbps +##gas +##ift +##dot +##ィ +455 +##～17 +amana +##りの +426 +##ros +ir +00㎡1 +##eet +##ible +##↓ +710 +ˋ▽ˊ +##aka +dcs +iq +##ｖ +l1 +##lor +maggie +##011 +##iu +588 +##～1 +830 +##gt +1tb +articles +create +##burg +##iki +database +fantasy +##rex +##cam +dlc +dean +##you +hard +path +gaming +victoria +maps +cb +##lee +##itor +overchicstoretvhome +systems +##xt +416 +p3 +sarah +760 +##nan +407 +486 +x9 +install +second +626 +##ann +##ph +##rcle +##nic +860 +##nar +ec +##とう +768 +metro +chocolate +##rian +～4 +##table +##しています +skin +##sn +395 +mountain +##0mm +inparadise +6m +7x24 +ib +4800 +##jia +eeworld +creative +g5 +g3 +357 +parker +ecfa +village +からの +18000 +sylvia +サーヒス +hbl +##ques +##onsored +##x2 +##きます +##v4 +##tein +ie6 +383 +##stack +389 +ver +##ads +##baby +sound +bbe +##110 +##lone +##uid +ads +022 +gundam +351 +thinkpad +006 +scrum +match +##ave +mems +##470 +##oy +##なりました +##talk +glass +lamigo +span +##eme +job +##a5 +jay +wade +kde +498 +##lace +ocean +tvg +##covery +##r3 +##ners +##rea +junior +think +##aine +cover +##ision +##sia +↓↓ +##bow +msi +413 +458 +406 +##love +711 +801 +soft +z2 +##pl +456 +1840 +mobil +mind +##uy +427 +nginx +##oi +めた +##rr +6221 +##mple +##sson +##ーシてす +371 +##nts +91tv +comhd +crv3000 +##uard +1868 +397 +deep +lost +field +gallery +##bia +rate +spf +redis +traction +930 +icloud +011 +なら +fe +jose +372 +##tory +into +sohu +fx +899 +379 +kicstart2 +##hia +すく +##～3 +##sit +ra +２４ +##walk +##xure +500g +##pact +pacific +xa +natural +carlo +##250 +##walker +1850 +##can +cto +gigi +516 +##サー +pen +##hoo +ob +matlab +##ｂ +##yy +13913459 +##iti +mango +##bbs +sense +c5 +oxford +##ニア +walker +jennifer +##ola +course +##bre +701 +##pus +##rder +lucky +075 +##ぁ +ivy +なお +##nia +sotheby +side +##ugh +joy +##orage +##ush +##bat +##dt +364 +r9 +##2d +##gio +511 +country +wear +##lax +##～7 +##moon +393 +seven +study +411 +348 +lonzo +8k +##ェ +evolution +##イフ +##kk +gs +kd +##レス +arduino +344 +b12 +##lux +arpg +##rdon +cook +##x5 +dark +five +##als +##ida +とても +sign +362 +##ちの +something +20mm +##nda +387 +##posted +fresh +tf +1870 +422 +cam +##mine +##skip +##form +##ssion +education +394 +##tee +dyson +stage +##jie +want +##night +epson +pack +あります +##ppy +テリヘル +##█ +wd +##eh +##rence +left +##lvin +golden +mhz +discovery +##trix +##n2 +loft +##uch +##dra +##sse +speed +～1 +1mdb +sorry +welcome +##urn +wave +gaga +##lmer +teddy +##160 +トラックハック +せよ +611 +##f2016 +378 +rp +##sha +rar +##あなたに +##きた +840 +holiday +##ュー +373 +074 +##vg +##nos +##rail +gartner +gi +6p +##dium +kit +488 +b3 +eco +##ろう +20g +sean +##stone +autocad +nu +##np +f16 +write +029 +m5 +##ias +images +atp +##dk +fsm +504 +1350 +ve +52kb +##xxx +##のに +##cake +414 +unit +lim +ru +1v +##ification +published +angela +16g +analytics +ak +##ｑ +##nel +gmt +##icon +again +##₂ +##bby +ios11 +445 +かこさいます +waze +いてす +##ハ +9985 +##ust +##ティー +framework +##007 +iptv +delete +52sykb +cl +wwdc +027 +30cm +##fw +##ての +1389 +##xon +brandt +##ses +##dragon +tc +vetements +anne +monte +modern +official +##へて +##ere +##nne +##oud +もちろん +５０ +etnews +##a2 +##graphy +421 +863 +##ちゃん +444 +##rtex +##てお +l2 +##gma +mount +ccd +たと +archive +morning +tan +ddos +e7 +##ホ +day4 +##ウ +gis +453 +its +495 +factory +bruce +pg +##ito +ってくたさい +guest +cdma +##lling +536 +n3 +しかし +3～4 +mega +eyes +ro +１３ +women +dac +church +##jun +singapore +##facebook +6991 +starbucks +##tos +##stin +##shine +zen +##mu +tina +20℃ +1893 +##たけて +503 +465 +request +##gence +qt +##っ +1886 +347 +363 +q7 +##zzi +diary +##tore +409 +##ead +468 +cst +##osa +canada +agent +va +##jiang +##ちは +##ーク +##lam +sg +##nix +##sday +##よって +g6 +##master +bing +##zl +charlie +１６ +8mm +nb40 +##ーン +thai +##ルフ +ln284ct +##itz +##2f +bonnie +##food +##lent +originals +##stro +##lts +418 +∟∣ +##bscribe +children +ntd +yesstyle +##かも +hmv +##tment +d5 +2cm +arts +sms +##pn +##я +##いい +topios9 +539 +lifestyle +virtual +##ague +xz +##deo +muji +024 +unt +##nnis +##ᅩ +faq1 +1884 +396 +##ette +fly +64㎡ +はしめまして +441 +curry +##pop +のこ +release +##← +##◆◆ +##cast +073 +ありな +500ml +##ews +5c +##stle +ios7 +##ima +787 +dog +lenovo +##r4 +roger +013 +cbs +vornado +100m +417 +##desk +##クok +##ald +1867 +9595 +2900 +##van +oil +##ｘ +some +break +common +##jy +##lines +g7 +twice +419 +ella +nano +belle +にこ +##mes +##self +##note +jb +##ことかてきます +benz +##との +##ova +451 +save +##wing +##ますのて +kai +りは +##hua +##rect +rainer +##unge +448 +##0m +adsl +##かな +guestname +##uma +##kins +##zu +tokichoi +##price +county +##med +##mus +rmk +391 +address +vm +えて +openload +##group +##hin +##iginal +amg +urban +##oz +jobs +emi +##public +beautiful +##sch +album +##dden +##bell +jerry +works +hostel +miller +##drive +##rmin +##１０ +376 +boot +828 +##370 +##fx +##cm～ +1885 +##nome +##ctionary +##oman +##lish +##cr +##hm +433 +##how +432 +francis +xi +c919 +b5 +evernote +##uc +vga +##3000 +coupe +##urg +##cca +##uality +019 +6g +れる +multi +##また +##ett +em +hey +##ani +##tax +##rma +inside +than +740 +leonnhurt +##jin +ict +れた +bird +notes +200mm +くの +##dical +##lli +result +442 +iu +ee +438 +smap +gopro +##last +yin +pure +998 +32g +けた +5kg +##dan +##rame +mama +##oot +bean +marketing +##hur +2l +bella +sync +xuite +##ground +515 +discuz +##getrelax +##ince +##bay +##5s +cj +##イス +gmat +apt +##pass +jing +##rix +c4 +rich +##とても +niusnews +##ello +bag +770 +##eting +##mobile +１８ +culture +015 +##のてすか +377 +1020 +area +##ience +616 +details +gp +universal +silver +dit +はお +private +ddd +u11 +kanshu +##ified +fung +##nny +dx +##520 +tai +475 +023 +##fr +##lean +3s +##pin +429 +##rin +25000 +ly +rick +##bility +usb3 +banner +##baru +##gion +metal +dt +vdf +1871 +karl +qualcomm +bear +1010 +oldid +ian +jo +##tors +population +##ernel +1882 +mmorpg +##mv +##bike +603 +##© +ww +friend +##ager +exhibition +##del +##pods +fpx +structure +##free +##tings +kl +##rley +##copyright +##mma +california +3400 +orange +yoga +4l +canmake +honey +##anda +##コメント +595 +nikkie +##ルハイト +dhl +publishing +##mall +##gnet +20cm +513 +##クセス +##┅ +e88 +970 +##dog +fishbase +##! +##" +### +##$ +##% +##& +##' +##( +##) +##* +##+ +##, +##- +##. +##/ +##: +##; +##< +##= +##> +##? +##@ +##[ +##\ +##] +##^ +##_ +##{ +##| +##} +##~ +##£ +##¤ +##¥ +##§ +##« +##± +##³ +##µ +##· +##¹ +##º +##» +##¼ +##ß +##æ +##÷ +##ø +##đ +##ŋ +##ɔ +##ə +##ɡ +##ʰ +##ˇ +##ˈ +##ˊ +##ˋ +##ˍ +##ː +##˙ +##˚ +##ˢ +##α +##β +##γ +##δ +##ε +##η +##θ +##ι +##κ +##λ +##μ +##ν +##ο +##π +##ρ +##ς +##σ +##τ +##υ +##φ +##χ +##ψ +##б +##в +##г +##д +##е +##ж +##з +##к +##л +##м +##н +##о +##п +##р +##с +##т +##у +##ф +##х +##ц +##ч +##ш +##ы +##ь +##і +##ا +##ب +##ة +##ت +##د +##ر +##س +##ع +##ل +##م +##ن +##ه +##و +##ي +##۩ +##ก +##ง +##น +##ม +##ย +##ร +##อ +##า +##เ +##๑ +##་ +##ღ +##ᄀ +##ᄁ +##ᄂ +##ᄃ +##ᄅ +##ᄆ +##ᄇ +##ᄈ +##ᄉ +##ᄋ +##ᄌ +##ᄎ +##ᄏ +##ᄐ +##ᄑ +##ᄒ +##ᅢ +##ᅣ +##ᅥ +##ᅦ +##ᅧ +##ᅨ +##ᅪ +##ᅬ +##ᅭ +##ᅮ +##ᅯ +##ᅲ +##ᅳ +##ᅴ +##ᆷ +##ᆸ +##ᆺ +##ᆻ +##ᗜ +##ᵃ +##ᵉ +##ᵍ +##ᵏ +##ᵐ +##ᵒ +##ᵘ +##‖ +##„ +##† +##• +##‥ +##‧ +##
 +##‰ +##′ +##″ +##‹ +##› +##※ +##‿ +##⁄ +##ⁱ +##⁺ +##ⁿ +##₁ +##₃ +##₄ +##€ +##№ +##ⅰ +##ⅱ +##ⅲ +##ⅳ +##ⅴ +##↔ +##↗ +##↘ +##⇒ +##∀ +##− +##∕ +##∙ +##√ +##∞ +##∟ +##∠ +##∣ +##∩ +##∮ +##∶ +##∼ +##∽ +##≈ +##≒ +##≡ +##≤ +##≥ +##≦ +##≧ +##≪ +##≫ +##⊙ +##⋅ +##⋈ +##⋯ +##⌒ +##① +##② +##③ +##④ +##⑤ +##⑥ +##⑦ +##⑧ +##⑨ +##⑩ +##⑴ +##⑵ +##⑶ +##⑷ +##⑸ +##⒈ +##⒉ +##⒊ +##⒋ +##ⓒ +##ⓔ +##ⓘ +##━ +##┃ +##┆ +##┊ +##┌ +##└ +##├ +##┣ +##═ +##║ +##╚ +##╞ +##╠ +##╭ +##╮ +##╯ +##╰ +##╱ +##╳ +##▂ +##▃ +##▅ +##▇ +##▉ +##▋ +##▌ +##▍ +##▎ +##□ +##▪ +##▫ +##▬ +##△ +##▶ +##► +##▽ +##◇ +##◕ +##◠ +##◢ +##◤ +##☀ +##☕ +##☞ +##☺ +##☼ +##♀ +##♂ +##♠ +##♡ +##♣ +##♦ +##♫ +##♬ +##✈ +##✔ +##✕ +##✖ +##✦ +##✨ +##✪ +##✰ +##✿ +##❀ +##➜ +##➤ +##⦿ +##、 +##。 +##〃 +##々 +##〇 +##〈 +##〉 +##《 +##》 +##「 +##」 +##『 +##』 +##【 +##】 +##〓 +##〔 +##〕 +##〖 +##〗 +##〜 +##〝 +##〞 +##ぃ +##ぇ +##ぬ +##ふ +##ほ +##む +##ゃ +##ゅ +##ゆ +##ょ +##゜ +##ゝ +##ァ +##ゥ +##エ +##ォ +##ケ +##サ +##セ +##ソ +##ッ +##ニ +##ヌ +##ネ +##ノ +##ヘ +##モ +##ャ +##ヤ +##ュ +##ユ +##ョ +##ヨ +##ワ +##ヲ +##・ +##ヽ +##ㄅ +##ㄆ +##ㄇ +##ㄉ +##ㄋ +##ㄌ +##ㄍ +##ㄎ +##ㄏ +##ㄒ +##ㄚ +##ㄛ +##ㄞ +##ㄟ +##ㄢ +##ㄤ +##ㄥ +##ㄧ +##ㄨ +##ㆍ +##㈦ +##㊣ +##㗎 +##一 +##丁 +##七 +##万 +##丈 +##三 +##上 +##下 +##不 +##与 +##丐 +##丑 +##专 +##且 +##丕 +##世 +##丘 +##丙 +##业 +##丛 +##东 +##丝 +##丞 +##丟 +##両 +##丢 +##两 +##严 +##並 +##丧 +##丨 +##个 +##丫 +##中 +##丰 +##串 +##临 +##丶 +##丸 +##丹 +##为 +##主 +##丼 +##丽 +##举 +##丿 +##乂 +##乃 +##久 +##么 +##义 +##之 +##乌 +##乍 +##乎 +##乏 +##乐 +##乒 +##乓 +##乔 +##乖 +##乗 +##乘 +##乙 +##乜 +##九 +##乞 +##也 +##习 +##乡 +##书 +##乩 +##买 +##乱 +##乳 +##乾 +##亀 +##亂 +##了 +##予 +##争 +##事 +##二 +##于 +##亏 +##云 +##互 +##五 +##井 +##亘 +##亙 +##亚 +##些 +##亜 +##亞 +##亟 +##亡 +##亢 +##交 +##亥 +##亦 +##产 +##亨 +##亩 +##享 +##京 +##亭 +##亮 +##亲 +##亳 +##亵 +##人 +##亿 +##什 +##仁 +##仃 +##仄 +##仅 +##仆 +##仇 +##今 +##介 +##仍 +##从 +##仏 +##仑 +##仓 +##仔 +##仕 +##他 +##仗 +##付 +##仙 +##仝 +##仞 +##仟 +##代 +##令 +##以 +##仨 +##仪 +##们 +##仮 +##仰 +##仲 +##件 +##价 +##任 +##份 +##仿 +##企 +##伉 +##伊 +##伍 +##伎 +##伏 +##伐 +##休 +##伕 +##众 +##优 +##伙 +##会 +##伝 +##伞 +##伟 +##传 +##伢 +##伤 +##伦 +##伪 +##伫 +##伯 +##估 +##伴 +##伶 +##伸 +##伺 +##似 +##伽 +##佃 +##但 +##佇 +##佈 +##位 +##低 +##住 +##佐 +##佑 +##体 +##佔 +##何 +##佗 +##佘 +##余 +##佚 +##佛 +##作 +##佝 +##佞 +##佟 +##你 +##佢 +##佣 +##佤 +##佥 +##佩 +##佬 +##佯 +##佰 +##佳 +##併 +##佶 +##佻 +##佼 +##使 +##侃 +##侄 +##來 +##侈 +##例 +##侍 +##侏 +##侑 +##侖 +##侗 +##供 +##依 +##侠 +##価 +##侣 +##侥 +##侦 +##侧 +##侨 +##侬 +##侮 +##侯 +##侵 +##侶 +##侷 +##便 +##係 +##促 +##俄 +##俊 +##俎 +##俏 +##俐 +##俑 +##俗 +##俘 +##俚 +##保 +##俞 +##俟 +##俠 +##信 +##俨 +##俩 +##俪 +##俬 +##俭 +##修 +##俯 +##俱 +##俳 +##俸 +##俺 +##俾 +##倆 +##倉 +##個 +##倌 +##倍 +##倏 +##們 +##倒 +##倔 +##倖 +##倘 +##候 +##倚 +##倜 +##借 +##倡 +##値 +##倦 +##倩 +##倪 +##倫 +##倬 +##倭 +##倶 +##债 +##值 +##倾 +##偃 +##假 +##偈 +##偉 +##偌 +##偎 +##偏 +##偕 +##做 +##停 +##健 +##側 +##偵 +##偶 +##偷 +##偻 +##偽 +##偿 +##傀 +##傅 +##傍 +##傑 +##傘 +##備 +##傚 +##傢 +##傣 +##傥 +##储 +##傩 +##催 +##傭 +##傲 +##傳 +##債 +##傷 +##傻 +##傾 +##僅 +##働 +##像 +##僑 +##僕 +##僖 +##僚 +##僥 +##僧 +##僭 +##僮 +##僱 +##僵 +##價 +##僻 +##儀 +##儂 +##億 +##儆 +##儉 +##儋 +##儒 +##儕 +##儘 +##償 +##儡 +##優 +##儲 +##儷 +##儼 +##儿 +##兀 +##允 +##元 +##兄 +##充 +##兆 +##兇 +##先 +##光 +##克 +##兌 +##免 +##児 +##兑 +##兒 +##兔 +##兖 +##党 +##兜 +##兢 +##入 +##內 +##全 +##兩 +##八 +##公 +##六 +##兮 +##兰 +##共 +##兲 +##关 +##兴 +##兵 +##其 +##具 +##典 +##兹 +##养 +##兼 +##兽 +##冀 +##内 +##円 +##冇 +##冈 +##冉 +##冊 +##册 +##再 +##冏 +##冒 +##冕 +##冗 +##写 +##军 +##农 +##冠 +##冢 +##冤 +##冥 +##冨 +##冪 +##冬 +##冯 +##冰 +##冲 +##决 +##况 +##冶 +##冷 +##冻 +##冼 +##冽 +##冾 +##净 +##凄 +##准 +##凇 +##凈 +##凉 +##凋 +##凌 +##凍 +##减 +##凑 +##凛 +##凜 +##凝 +##几 +##凡 +##凤 +##処 +##凪 +##凭 +##凯 +##凰 +##凱 +##凳 +##凶 +##凸 +##凹 +##出 +##击 +##函 +##凿 +##刀 +##刁 +##刃 +##分 +##切 +##刈 +##刊 +##刍 +##刎 +##刑 +##划 +##列 +##刘 +##则 +##刚 +##创 +##初 +##删 +##判 +##別 +##刨 +##利 +##刪 +##别 +##刮 +##到 +##制 +##刷 +##券 +##刹 +##刺 +##刻 +##刽 +##剁 +##剂 +##剃 +##則 +##剉 +##削 +##剋 +##剌 +##前 +##剎 +##剐 +##剑 +##剔 +##剖 +##剛 +##剜 +##剝 +##剣 +##剤 +##剥 +##剧 +##剩 +##剪 +##副 +##割 +##創 +##剷 +##剽 +##剿 +##劃 +##劇 +##劈 +##劉 +##劊 +##劍 +##劏 +##劑 +##力 +##劝 +##办 +##功 +##加 +##务 +##劣 +##动 +##助 +##努 +##劫 +##劭 +##励 +##劲 +##劳 +##労 +##劵 +##効 +##劾 +##势 +##勁 +##勃 +##勇 +##勉 +##勋 +##勐 +##勒 +##動 +##勖 +##勘 +##務 +##勛 +##勝 +##勞 +##募 +##勢 +##勤 +##勧 +##勳 +##勵 +##勸 +##勺 +##勻 +##勾 +##勿 +##匀 +##包 +##匆 +##匈 +##匍 +##匐 +##匕 +##化 +##北 +##匙 +##匝 +##匠 +##匡 +##匣 +##匪 +##匮 +##匯 +##匱 +##匹 +##区 +##医 +##匾 +##匿 +##區 +##十 +##千 +##卅 +##升 +##午 +##卉 +##半 +##卍 +##华 +##协 +##卑 +##卒 +##卓 +##協 +##单 +##卖 +##南 +##単 +##博 +##卜 +##卞 +##卟 +##占 +##卡 +##卢 +##卤 +##卦 +##卧 +##卫 +##卮 +##卯 +##印 +##危 +##即 +##却 +##卵 +##卷 +##卸 +##卻 +##卿 +##厂 +##厄 +##厅 +##历 +##厉 +##压 +##厌 +##厕 +##厘 +##厚 +##厝 +##原 +##厢 +##厥 +##厦 +##厨 +##厩 +##厭 +##厮 +##厲 +##厳 +##去 +##县 +##叁 +##参 +##參 +##又 +##叉 +##及 +##友 +##双 +##反 +##収 +##发 +##叔 +##取 +##受 +##变 +##叙 +##叛 +##叟 +##叠 +##叡 +##叢 +##口 +##古 +##句 +##另 +##叨 +##叩 +##只 +##叫 +##召 +##叭 +##叮 +##可 +##台 +##叱 +##史 +##右 +##叵 +##叶 +##号 +##司 +##叹 +##叻 +##叼 +##叽 +##吁 +##吃 +##各 +##吆 +##合 +##吉 +##吊 +##吋 +##同 +##名 +##后 +##吏 +##吐 +##向 +##吒 +##吓 +##吕 +##吖 +##吗 +##君 +##吝 +##吞 +##吟 +##吠 +##吡 +##否 +##吧 +##吨 +##吩 +##含 +##听 +##吭 +##吮 +##启 +##吱 +##吳 +##吴 +##吵 +##吶 +##吸 +##吹 +##吻 +##吼 +##吽 +##吾 +##呀 +##呂 +##呃 +##呆 +##呈 +##告 +##呋 +##呎 +##呐 +##呓 +##呕 +##呗 +##员 +##呛 +##呜 +##呢 +##呤 +##呦 +##周 +##呱 +##呲 +##味 +##呵 +##呷 +##呸 +##呻 +##呼 +##命 +##咀 +##咁 +##咂 +##咄 +##咆 +##咋 +##和 +##咎 +##咏 +##咐 +##咒 +##咔 +##咕 +##咖 +##咗 +##咘 +##咙 +##咚 +##咛 +##咣 +##咤 +##咦 +##咧 +##咨 +##咩 +##咪 +##咫 +##咬 +##咭 +##咯 +##咱 +##咲 +##咳 +##咸 +##咻 +##咽 +##咿 +##哀 +##品 +##哂 +##哄 +##哆 +##哇 +##哈 +##哉 +##哋 +##哌 +##响 +##哎 +##哏 +##哐 +##哑 +##哒 +##哔 +##哗 +##哟 +##員 +##哥 +##哦 +##哧 +##哨 +##哩 +##哪 +##哭 +##哮 +##哲 +##哺 +##哼 +##哽 +##唁 +##唄 +##唆 +##唇 +##唉 +##唏 +##唐 +##唑 +##唔 +##唠 +##唤 +##唧 +##唬 +##售 +##唯 +##唰 +##唱 +##唳 +##唷 +##唸 +##唾 +##啃 +##啄 +##商 +##啉 +##啊 +##問 +##啓 +##啕 +##啖 +##啜 +##啞 +##啟 +##啡 +##啤 +##啥 +##啦 +##啧 +##啪 +##啫 +##啬 +##啮 +##啰 +##啱 +##啲 +##啵 +##啶 +##啷 +##啸 +##啻 +##啼 +##啾 +##喀 +##喂 +##喃 +##善 +##喆 +##喇 +##喉 +##喊 +##喋 +##喎 +##喏 +##喔 +##喘 +##喙 +##喚 +##喜 +##喝 +##喟 +##喧 +##喪 +##喫 +##喬 +##單 +##喰 +##喱 +##喲 +##喳 +##喵 +##営 +##喷 +##喹 +##喺 +##喻 +##喽 +##嗅 +##嗆 +##嗇 +##嗎 +##嗑 +##嗒 +##嗓 +##嗔 +##嗖 +##嗚 +##嗜 +##嗝 +##嗟 +##嗡 +##嗣 +##嗤 +##嗦 +##嗨 +##嗪 +##嗬 +##嗯 +##嗰 +##嗲 +##嗳 +##嗶 +##嗷 +##嗽 +##嘀 +##嘅 +##嘆 +##嘈 +##嘉 +##嘌 +##嘍 +##嘎 +##嘔 +##嘖 +##嘗 +##嘘 +##嘚 +##嘛 +##嘜 +##嘞 +##嘟 +##嘢 +##嘣 +##嘤 +##嘧 +##嘩 +##嘭 +##嘮 +##嘯 +##嘰 +##嘱 +##嘲 +##嘴 +##嘶 +##嘸 +##嘹 +##嘻 +##嘿 +##噁 +##噌 +##噎 +##噓 +##噔 +##噗 +##噙 +##噜 +##噠 +##噢 +##噤 +##器 +##噩 +##噪 +##噬 +##噱 +##噴 +##噶 +##噸 +##噹 +##噻 +##噼 +##嚀 +##嚇 +##嚎 +##嚏 +##嚐 +##嚓 +##嚕 +##嚟 +##嚣 +##嚥 +##嚨 +##嚮 +##嚴 +##嚷 +##嚼 +##囂 +##囉 +##囊 +##囍 +##囑 +##囔 +##囗 +##囚 +##四 +##囝 +##回 +##囟 +##因 +##囡 +##团 +##団 +##囤 +##囧 +##囪 +##囫 +##园 +##困 +##囱 +##囲 +##図 +##围 +##囹 +##固 +##国 +##图 +##囿 +##圃 +##圄 +##圆 +##圈 +##國 +##圍 +##圏 +##園 +##圓 +##圖 +##團 +##圜 +##土 +##圣 +##圧 +##在 +##圩 +##圭 +##地 +##圳 +##场 +##圻 +##圾 +##址 +##坂 +##均 +##坊 +##坍 +##坎 +##坏 +##坐 +##坑 +##块 +##坚 +##坛 +##坝 +##坞 +##坟 +##坠 +##坡 +##坤 +##坦 +##坨 +##坪 +##坯 +##坳 +##坵 +##坷 +##垂 +##垃 +##垄 +##型 +##垒 +##垚 +##垛 +##垠 +##垢 +##垣 +##垦 +##垩 +##垫 +##垭 +##垮 +##垵 +##埂 +##埃 +##埋 +##城 +##埔 +##埕 +##埗 +##域 +##埠 +##埤 +##埵 +##執 +##埸 +##培 +##基 +##埼 +##堀 +##堂 +##堃 +##堅 +##堆 +##堇 +##堑 +##堕 +##堙 +##堡 +##堤 +##堪 +##堯 +##堰 +##報 +##場 +##堵 +##堺 +##堿 +##塊 +##塌 +##塑 +##塔 +##塗 +##塘 +##塚 +##塞 +##塢 +##塩 +##填 +##塬 +##塭 +##塵 +##塾 +##墀 +##境 +##墅 +##墉 +##墊 +##墒 +##墓 +##増 +##墘 +##墙 +##墜 +##增 +##墟 +##墨 +##墩 +##墮 +##墳 +##墻 +##墾 +##壁 +##壅 +##壆 +##壇 +##壊 +##壑 +##壓 +##壕 +##壘 +##壞 +##壟 +##壢 +##壤 +##壩 +##士 +##壬 +##壮 +##壯 +##声 +##売 +##壳 +##壶 +##壹 +##壺 +##壽 +##处 +##备 +##変 +##复 +##夏 +##夔 +##夕 +##外 +##夙 +##多 +##夜 +##够 +##夠 +##夢 +##夥 +##大 +##天 +##太 +##夫 +##夭 +##央 +##夯 +##失 +##头 +##夷 +##夸 +##夹 +##夺 +##夾 +##奂 +##奄 +##奇 +##奈 +##奉 +##奋 +##奎 +##奏 +##奐 +##契 +##奔 +##奕 +##奖 +##套 +##奘 +##奚 +##奠 +##奢 +##奥 +##奧 +##奪 +##奬 +##奮 +##女 +##奴 +##奶 +##奸 +##她 +##好 +##如 +##妃 +##妄 +##妆 +##妇 +##妈 +##妊 +##妍 +##妒 +##妓 +##妖 +##妘 +##妙 +##妝 +##妞 +##妣 +##妤 +##妥 +##妨 +##妩 +##妪 +##妮 +##妲 +##妳 +##妹 +##妻 +##妾 +##姆 +##姉 +##姊 +##始 +##姍 +##姐 +##姑 +##姒 +##姓 +##委 +##姗 +##姚 +##姜 +##姝 +##姣 +##姥 +##姦 +##姨 +##姪 +##姫 +##姬 +##姹 +##姻 +##姿 +##威 +##娃 +##娄 +##娅 +##娆 +##娇 +##娉 +##娑 +##娓 +##娘 +##娛 +##娜 +##娟 +##娠 +##娣 +##娥 +##娩 +##娱 +##娲 +##娴 +##娶 +##娼 +##婀 +##婁 +##婆 +##婉 +##婊 +##婕 +##婚 +##婢 +##婦 +##婧 +##婪 +##婭 +##婴 +##婵 +##婶 +##婷 +##婺 +##婿 +##媒 +##媚 +##媛 +##媞 +##媧 +##媲 +##媳 +##媽 +##媾 +##嫁 +##嫂 +##嫉 +##嫌 +##嫑 +##嫔 +##嫖 +##嫘 +##嫚 +##嫡 +##嫣 +##嫦 +##嫩 +##嫲 +##嫵 +##嫻 +##嬅 +##嬉 +##嬌 +##嬗 +##嬛 +##嬢 +##嬤 +##嬪 +##嬰 +##嬴 +##嬷 +##嬸 +##嬿 +##孀 +##孃 +##子 +##孑 +##孔 +##孕 +##孖 +##字 +##存 +##孙 +##孚 +##孛 +##孜 +##孝 +##孟 +##孢 +##季 +##孤 +##学 +##孩 +##孪 +##孫 +##孬 +##孰 +##孱 +##孳 +##孵 +##學 +##孺 +##孽 +##孿 +##宁 +##它 +##宅 +##宇 +##守 +##安 +##宋 +##完 +##宏 +##宓 +##宕 +##宗 +##官 +##宙 +##定 +##宛 +##宜 +##宝 +##实 +##実 +##宠 +##审 +##客 +##宣 +##室 +##宥 +##宦 +##宪 +##宫 +##宮 +##宰 +##害 +##宴 +##宵 +##家 +##宸 +##容 +##宽 +##宾 +##宿 +##寂 +##寄 +##寅 +##密 +##寇 +##富 +##寐 +##寒 +##寓 +##寛 +##寝 +##寞 +##察 +##寡 +##寢 +##寥 +##實 +##寧 +##寨 +##審 +##寫 +##寬 +##寮 +##寰 +##寵 +##寶 +##寸 +##对 +##寺 +##寻 +##导 +##対 +##寿 +##封 +##専 +##射 +##将 +##將 +##專 +##尉 +##尊 +##尋 +##對 +##導 +##小 +##少 +##尔 +##尕 +##尖 +##尘 +##尚 +##尝 +##尤 +##尧 +##尬 +##就 +##尴 +##尷 +##尸 +##尹 +##尺 +##尻 +##尼 +##尽 +##尾 +##尿 +##局 +##屁 +##层 +##屄 +##居 +##屆 +##屈 +##屉 +##届 +##屋 +##屌 +##屍 +##屎 +##屏 +##屐 +##屑 +##展 +##屜 +##属 +##屠 +##屡 +##屢 +##層 +##履 +##屬 +##屯 +##山 +##屹 +##屿 +##岀 +##岁 +##岂 +##岌 +##岐 +##岑 +##岔 +##岖 +##岗 +##岘 +##岙 +##岚 +##岛 +##岡 +##岩 +##岫 +##岬 +##岭 +##岱 +##岳 +##岷 +##岸 +##峇 +##峋 +##峒 +##峙 +##峡 +##峤 +##峥 +##峦 +##峨 +##峪 +##峭 +##峯 +##峰 +##峴 +##島 +##峻 +##峽 +##崁 +##崂 +##崆 +##崇 +##崎 +##崑 +##崔 +##崖 +##崗 +##崙 +##崛 +##崧 +##崩 +##崭 +##崴 +##崽 +##嵇 +##嵊 +##嵋 +##嵌 +##嵐 +##嵘 +##嵩 +##嵬 +##嵯 +##嶂 +##嶄 +##嶇 +##嶋 +##嶙 +##嶺 +##嶼 +##嶽 +##巅 +##巍 +##巒 +##巔 +##巖 +##川 +##州 +##巡 +##巢 +##工 +##左 +##巧 +##巨 +##巩 +##巫 +##差 +##己 +##已 +##巳 +##巴 +##巷 +##巻 +##巽 +##巾 +##巿 +##币 +##市 +##布 +##帅 +##帆 +##师 +##希 +##帐 +##帑 +##帕 +##帖 +##帘 +##帚 +##帛 +##帜 +##帝 +##帥 +##带 +##帧 +##師 +##席 +##帮 +##帯 +##帰 +##帳 +##帶 +##帷 +##常 +##帼 +##帽 +##幀 +##幂 +##幄 +##幅 +##幌 +##幔 +##幕 +##幟 +##幡 +##幢 +##幣 +##幫 +##干 +##平 +##年 +##并 +##幸 +##幹 +##幺 +##幻 +##幼 +##幽 +##幾 +##广 +##庁 +##広 +##庄 +##庆 +##庇 +##床 +##序 +##庐 +##库 +##应 +##底 +##庖 +##店 +##庙 +##庚 +##府 +##庞 +##废 +##庠 +##度 +##座 +##庫 +##庭 +##庵 +##庶 +##康 +##庸 +##庹 +##庾 +##廁 +##廂 +##廃 +##廈 +##廉 +##廊 +##廓 +##廖 +##廚 +##廝 +##廟 +##廠 +##廢 +##廣 +##廬 +##廳 +##延 +##廷 +##建 +##廿 +##开 +##弁 +##异 +##弃 +##弄 +##弈 +##弊 +##弋 +##式 +##弑 +##弒 +##弓 +##弔 +##引 +##弗 +##弘 +##弛 +##弟 +##张 +##弥 +##弦 +##弧 +##弩 +##弭 +##弯 +##弱 +##張 +##強 +##弹 +##强 +##弼 +##弾 +##彅 +##彆 +##彈 +##彌 +##彎 +##归 +##当 +##录 +##彗 +##彙 +##彝 +##形 +##彤 +##彥 +##彦 +##彧 +##彩 +##彪 +##彫 +##彬 +##彭 +##彰 +##影 +##彷 +##役 +##彻 +##彼 +##彿 +##往 +##征 +##径 +##待 +##徇 +##很 +##徉 +##徊 +##律 +##後 +##徐 +##徑 +##徒 +##従 +##徕 +##得 +##徘 +##徙 +##徜 +##從 +##徠 +##御 +##徨 +##復 +##循 +##徬 +##微 +##徳 +##徴 +##徵 +##德 +##徹 +##徼 +##徽 +##心 +##必 +##忆 +##忌 +##忍 +##忏 +##忐 +##忑 +##忒 +##忖 +##志 +##忘 +##忙 +##応 +##忠 +##忡 +##忤 +##忧 +##忪 +##快 +##忱 +##念 +##忻 +##忽 +##忿 +##怀 +##态 +##怂 +##怅 +##怆 +##怎 +##怏 +##怒 +##怔 +##怕 +##怖 +##怙 +##怜 +##思 +##怠 +##怡 +##急 +##怦 +##性 +##怨 +##怪 +##怯 +##怵 +##总 +##怼 +##恁 +##恃 +##恆 +##恋 +##恍 +##恐 +##恒 +##恕 +##恙 +##恚 +##恢 +##恣 +##恤 +##恥 +##恨 +##恩 +##恪 +##恫 +##恬 +##恭 +##息 +##恰 +##恳 +##恵 +##恶 +##恸 +##恺 +##恻 +##恼 +##恿 +##悄 +##悅 +##悉 +##悌 +##悍 +##悔 +##悖 +##悚 +##悟 +##悠 +##患 +##悦 +##您 +##悩 +##悪 +##悬 +##悯 +##悱 +##悲 +##悴 +##悵 +##悶 +##悸 +##悻 +##悼 +##悽 +##情 +##惆 +##惇 +##惊 +##惋 +##惑 +##惕 +##惘 +##惚 +##惜 +##惟 +##惠 +##惡 +##惦 +##惧 +##惨 +##惩 +##惫 +##惬 +##惭 +##惮 +##惯 +##惰 +##惱 +##想 +##惴 +##惶 +##惹 +##惺 +##愁 +##愆 +##愈 +##愉 +##愍 +##意 +##愕 +##愚 +##愛 +##愜 +##感 +##愣 +##愤 +##愧 +##愫 +##愷 +##愿 +##慄 +##慈 +##態 +##慌 +##慎 +##慑 +##慕 +##慘 +##慚 +##慟 +##慢 +##慣 +##慧 +##慨 +##慫 +##慮 +##慰 +##慳 +##慵 +##慶 +##慷 +##慾 +##憂 +##憊 +##憋 +##憎 +##憐 +##憑 +##憔 +##憚 +##憤 +##憧 +##憨 +##憩 +##憫 +##憬 +##憲 +##憶 +##憾 +##懂 +##懇 +##懈 +##應 +##懊 +##懋 +##懑 +##懒 +##懦 +##懲 +##懵 +##懶 +##懷 +##懸 +##懺 +##懼 +##懾 +##懿 +##戀 +##戈 +##戊 +##戌 +##戍 +##戎 +##戏 +##成 +##我 +##戒 +##戕 +##或 +##战 +##戚 +##戛 +##戟 +##戡 +##戦 +##截 +##戬 +##戮 +##戰 +##戲 +##戳 +##戴 +##戶 +##户 +##戸 +##戻 +##戾 +##房 +##所 +##扁 +##扇 +##扈 +##扉 +##手 +##才 +##扎 +##扑 +##扒 +##打 +##扔 +##払 +##托 +##扛 +##扣 +##扦 +##执 +##扩 +##扪 +##扫 +##扬 +##扭 +##扮 +##扯 +##扰 +##扱 +##扳 +##扶 +##批 +##扼 +##找 +##承 +##技 +##抄 +##抉 +##把 +##抑 +##抒 +##抓 +##投 +##抖 +##抗 +##折 +##抚 +##抛 +##抜 +##択 +##抟 +##抠 +##抡 +##抢 +##护 +##报 +##抨 +##披 +##抬 +##抱 +##抵 +##抹 +##押 +##抽 +##抿 +##拂 +##拄 +##担 +##拆 +##拇 +##拈 +##拉 +##拋 +##拌 +##拍 +##拎 +##拐 +##拒 +##拓 +##拔 +##拖 +##拗 +##拘 +##拙 +##拚 +##招 +##拜 +##拟 +##拡 +##拢 +##拣 +##拥 +##拦 +##拧 +##拨 +##择 +##括 +##拭 +##拮 +##拯 +##拱 +##拳 +##拴 +##拷 +##拼 +##拽 +##拾 +##拿 +##持 +##挂 +##指 +##挈 +##按 +##挎 +##挑 +##挖 +##挙 +##挚 +##挛 +##挝 +##挞 +##挟 +##挠 +##挡 +##挣 +##挤 +##挥 +##挨 +##挪 +##挫 +##振 +##挲 +##挹 +##挺 +##挽 +##挾 +##捂 +##捅 +##捆 +##捉 +##捋 +##捌 +##捍 +##捎 +##捏 +##捐 +##捕 +##捞 +##损 +##捡 +##换 +##捣 +##捧 +##捨 +##捩 +##据 +##捱 +##捲 +##捶 +##捷 +##捺 +##捻 +##掀 +##掂 +##掃 +##掇 +##授 +##掉 +##掌 +##掏 +##掐 +##排 +##掖 +##掘 +##掙 +##掛 +##掠 +##採 +##探 +##掣 +##接 +##控 +##推 +##掩 +##措 +##掬 +##掰 +##掲 +##掳 +##掴 +##掷 +##掸 +##掺 +##揀 +##揃 +##揄 +##揆 +##揉 +##揍 +##描 +##提 +##插 +##揖 +##揚 +##換 +##握 +##揣 +##揩 +##揪 +##揭 +##揮 +##援 +##揶 +##揸 +##揹 +##揽 +##搀 +##搁 +##搂 +##搅 +##損 +##搏 +##搐 +##搓 +##搔 +##搖 +##搗 +##搜 +##搞 +##搡 +##搪 +##搬 +##搭 +##搵 +##搶 +##携 +##搽 +##摀 +##摁 +##摄 +##摆 +##摇 +##摈 +##摊 +##摒 +##摔 +##摘 +##摞 +##摟 +##摧 +##摩 +##摯 +##摳 +##摸 +##摹 +##摺 +##摻 +##撂 +##撃 +##撅 +##撇 +##撈 +##撐 +##撑 +##撒 +##撓 +##撕 +##撚 +##撞 +##撤 +##撥 +##撩 +##撫 +##撬 +##播 +##撮 +##撰 +##撲 +##撵 +##撷 +##撸 +##撻 +##撼 +##撿 +##擀 +##擁 +##擂 +##擄 +##擅 +##擇 +##擊 +##擋 +##操 +##擎 +##擒 +##擔 +##擘 +##據 +##擞 +##擠 +##擡 +##擢 +##擦 +##擬 +##擰 +##擱 +##擲 +##擴 +##擷 +##擺 +##擼 +##擾 +##攀 +##攏 +##攒 +##攔 +##攘 +##攙 +##攜 +##攝 +##攞 +##攢 +##攣 +##攤 +##攥 +##攪 +##攫 +##攬 +##支 +##收 +##攸 +##改 +##攻 +##放 +##政 +##故 +##效 +##敌 +##敍 +##敎 +##敏 +##救 +##敕 +##敖 +##敗 +##敘 +##教 +##敛 +##敝 +##敞 +##敢 +##散 +##敦 +##敬 +##数 +##敲 +##整 +##敵 +##敷 +##數 +##斂 +##斃 +##文 +##斋 +##斌 +##斎 +##斐 +##斑 +##斓 +##斗 +##料 +##斛 +##斜 +##斟 +##斡 +##斤 +##斥 +##斧 +##斩 +##斫 +##斬 +##断 +##斯 +##新 +##斷 +##方 +##於 +##施 +##旁 +##旃 +##旅 +##旋 +##旌 +##旎 +##族 +##旖 +##旗 +##无 +##既 +##日 +##旦 +##旧 +##旨 +##早 +##旬 +##旭 +##旮 +##旱 +##时 +##旷 +##旺 +##旻 +##昀 +##昂 +##昆 +##昇 +##昉 +##昊 +##昌 +##明 +##昏 +##易 +##昔 +##昕 +##昙 +##星 +##映 +##春 +##昧 +##昨 +##昭 +##是 +##昱 +##昴 +##昵 +##昶 +##昼 +##显 +##晁 +##時 +##晃 +##晉 +##晋 +##晌 +##晏 +##晒 +##晓 +##晔 +##晕 +##晖 +##晗 +##晚 +##晝 +##晞 +##晟 +##晤 +##晦 +##晨 +##晩 +##普 +##景 +##晰 +##晴 +##晶 +##晷 +##智 +##晾 +##暂 +##暄 +##暇 +##暈 +##暉 +##暌 +##暐 +##暑 +##暖 +##暗 +##暝 +##暢 +##暧 +##暨 +##暫 +##暮 +##暱 +##暴 +##暸 +##暹 +##曄 +##曆 +##曇 +##曉 +##曖 +##曙 +##曜 +##曝 +##曠 +##曦 +##曬 +##曰 +##曲 +##曳 +##更 +##書 +##曹 +##曼 +##曾 +##替 +##最 +##會 +##月 +##有 +##朋 +##服 +##朐 +##朔 +##朕 +##朗 +##望 +##朝 +##期 +##朦 +##朧 +##木 +##未 +##末 +##本 +##札 +##朮 +##术 +##朱 +##朴 +##朵 +##机 +##朽 +##杀 +##杂 +##权 +##杆 +##杈 +##杉 +##李 +##杏 +##材 +##村 +##杓 +##杖 +##杜 +##杞 +##束 +##杠 +##条 +##来 +##杨 +##杭 +##杯 +##杰 +##東 +##杳 +##杵 +##杷 +##杼 +##松 +##板 +##极 +##构 +##枇 +##枉 +##枋 +##析 +##枕 +##林 +##枚 +##果 +##枝 +##枢 +##枣 +##枪 +##枫 +##枭 +##枯 +##枰 +##枱 +##枳 +##架 +##枷 +##枸 +##柄 +##柏 +##某 +##柑 +##柒 +##染 +##柔 +##柘 +##柚 +##柜 +##柞 +##柠 +##柢 +##查 +##柩 +##柬 +##柯 +##柱 +##柳 +##柴 +##柵 +##査 +##柿 +##栀 +##栃 +##栄 +##栅 +##标 +##栈 +##栉 +##栋 +##栎 +##栏 +##树 +##栓 +##栖 +##栗 +##校 +##栩 +##株 +##样 +##核 +##根 +##格 +##栽 +##栾 +##桀 +##桁 +##桂 +##桃 +##桅 +##框 +##案 +##桉 +##桌 +##桎 +##桐 +##桑 +##桓 +##桔 +##桜 +##桠 +##桡 +##桢 +##档 +##桥 +##桦 +##桧 +##桨 +##桩 +##桶 +##桿 +##梁 +##梅 +##梆 +##梏 +##梓 +##梗 +##條 +##梟 +##梢 +##梦 +##梧 +##梨 +##梭 +##梯 +##械 +##梳 +##梵 +##梶 +##检 +##棂 +##棄 +##棉 +##棋 +##棍 +##棒 +##棕 +##棗 +##棘 +##棚 +##棟 +##棠 +##棣 +##棧 +##森 +##棱 +##棲 +##棵 +##棹 +##棺 +##椁 +##椅 +##椋 +##植 +##椎 +##椒 +##検 +##椪 +##椭 +##椰 +##椹 +##椽 +##椿 +##楂 +##楊 +##楓 +##楔 +##楚 +##楝 +##楞 +##楠 +##楣 +##楨 +##楫 +##業 +##楮 +##極 +##楷 +##楸 +##楹 +##楼 +##楽 +##概 +##榄 +##榆 +##榈 +##榉 +##榔 +##榕 +##榖 +##榛 +##榜 +##榨 +##榫 +##榭 +##榮 +##榱 +##榴 +##榷 +##榻 +##槁 +##槃 +##構 +##槌 +##槍 +##槎 +##槐 +##槓 +##様 +##槛 +##槟 +##槤 +##槭 +##槲 +##槳 +##槻 +##槽 +##槿 +##樁 +##樂 +##樊 +##樑 +##樓 +##標 +##樞 +##樟 +##模 +##樣 +##権 +##横 +##樫 +##樯 +##樱 +##樵 +##樸 +##樹 +##樺 +##樽 +##樾 +##橄 +##橇 +##橋 +##橐 +##橘 +##橙 +##機 +##橡 +##橢 +##橫 +##橱 +##橹 +##橼 +##檀 +##檄 +##檎 +##檐 +##檔 +##檗 +##檜 +##檢 +##檬 +##檯 +##檳 +##檸 +##檻 +##櫃 +##櫚 +##櫛 +##櫥 +##櫸 +##櫻 +##欄 +##權 +##欒 +##欖 +##欠 +##次 +##欢 +##欣 +##欧 +##欲 +##欸 +##欺 +##欽 +##款 +##歆 +##歇 +##歉 +##歌 +##歎 +##歐 +##歓 +##歙 +##歛 +##歡 +##止 +##正 +##此 +##步 +##武 +##歧 +##歩 +##歪 +##歯 +##歲 +##歳 +##歴 +##歷 +##歸 +##歹 +##死 +##歼 +##殁 +##殃 +##殆 +##殇 +##殉 +##殊 +##残 +##殒 +##殓 +##殖 +##殘 +##殞 +##殡 +##殤 +##殭 +##殯 +##殲 +##殴 +##段 +##殷 +##殺 +##殼 +##殿 +##毀 +##毁 +##毂 +##毅 +##毆 +##毋 +##母 +##毎 +##每 +##毒 +##毓 +##比 +##毕 +##毗 +##毘 +##毙 +##毛 +##毡 +##毫 +##毯 +##毽 +##氈 +##氏 +##氐 +##民 +##氓 +##气 +##氖 +##気 +##氙 +##氛 +##氟 +##氡 +##氢 +##氣 +##氤 +##氦 +##氧 +##氨 +##氪 +##氫 +##氮 +##氯 +##氰 +##氲 +##水 +##氷 +##永 +##氹 +##氾 +##汀 +##汁 +##求 +##汆 +##汇 +##汉 +##汎 +##汐 +##汕 +##汗 +##汙 +##汛 +##汝 +##汞 +##江 +##池 +##污 +##汤 +##汨 +##汩 +##汪 +##汰 +##汲 +##汴 +##汶 +##汹 +##決 +##汽 +##汾 +##沁 +##沂 +##沃 +##沅 +##沈 +##沉 +##沌 +##沏 +##沐 +##沒 +##沓 +##沖 +##沙 +##沛 +##沟 +##没 +##沢 +##沣 +##沥 +##沦 +##沧 +##沪 +##沫 +##沭 +##沮 +##沱 +##河 +##沸 +##油 +##治 +##沼 +##沽 +##沾 +##沿 +##況 +##泄 +##泉 +##泊 +##泌 +##泓 +##法 +##泗 +##泛 +##泞 +##泠 +##泡 +##波 +##泣 +##泥 +##注 +##泪 +##泫 +##泮 +##泯 +##泰 +##泱 +##泳 +##泵 +##泷 +##泸 +##泻 +##泼 +##泽 +##泾 +##洁 +##洄 +##洋 +##洒 +##洗 +##洙 +##洛 +##洞 +##津 +##洩 +##洪 +##洮 +##洱 +##洲 +##洵 +##洶 +##洸 +##洹 +##活 +##洼 +##洽 +##派 +##流 +##浃 +##浄 +##浅 +##浆 +##浇 +##浊 +##测 +##济 +##浏 +##浑 +##浒 +##浓 +##浔 +##浙 +##浚 +##浜 +##浣 +##浦 +##浩 +##浪 +##浬 +##浮 +##浯 +##浴 +##海 +##浸 +##涂 +##涅 +##涇 +##消 +##涉 +##涌 +##涎 +##涓 +##涔 +##涕 +##涙 +##涛 +##涝 +##涞 +##涟 +##涠 +##涡 +##涣 +##涤 +##润 +##涧 +##涨 +##涩 +##涪 +##涮 +##涯 +##液 +##涵 +##涸 +##涼 +##涿 +##淀 +##淄 +##淅 +##淆 +##淇 +##淋 +##淌 +##淑 +##淒 +##淖 +##淘 +##淙 +##淚 +##淞 +##淡 +##淤 +##淦 +##淨 +##淩 +##淪 +##淫 +##淬 +##淮 +##深 +##淳 +##淵 +##混 +##淹 +##淺 +##添 +##淼 +##清 +##済 +##渉 +##渊 +##渋 +##渍 +##渎 +##渐 +##渔 +##渗 +##渙 +##渚 +##減 +##渝 +##渠 +##渡 +##渣 +##渤 +##渥 +##渦 +##温 +##測 +##渭 +##港 +##渲 +##渴 +##游 +##渺 +##渾 +##湃 +##湄 +##湊 +##湍 +##湖 +##湘 +##湛 +##湟 +##湧 +##湫 +##湮 +##湯 +##湳 +##湾 +##湿 +##満 +##溃 +##溅 +##溉 +##溏 +##源 +##準 +##溜 +##溝 +##溟 +##溢 +##溥 +##溧 +##溪 +##溫 +##溯 +##溱 +##溴 +##溶 +##溺 +##溼 +##滁 +##滂 +##滄 +##滅 +##滇 +##滋 +##滌 +##滑 +##滓 +##滔 +##滕 +##滙 +##滚 +##滝 +##滞 +##滟 +##满 +##滢 +##滤 +##滥 +##滦 +##滨 +##滩 +##滬 +##滯 +##滲 +##滴 +##滷 +##滸 +##滾 +##滿 +##漁 +##漂 +##漆 +##漉 +##漏 +##漓 +##演 +##漕 +##漠 +##漢 +##漣 +##漩 +##漪 +##漫 +##漬 +##漯 +##漱 +##漲 +##漳 +##漸 +##漾 +##漿 +##潆 +##潇 +##潋 +##潍 +##潑 +##潔 +##潘 +##潛 +##潜 +##潞 +##潟 +##潢 +##潤 +##潦 +##潧 +##潭 +##潮 +##潰 +##潴 +##潸 +##潺 +##潼 +##澀 +##澄 +##澆 +##澈 +##澍 +##澎 +##澗 +##澜 +##澡 +##澤 +##澧 +##澱 +##澳 +##澹 +##激 +##濁 +##濂 +##濃 +##濑 +##濒 +##濕 +##濘 +##濛 +##濟 +##濠 +##濡 +##濤 +##濫 +##濬 +##濮 +##濯 +##濱 +##濺 +##濾 +##瀅 +##瀆 +##瀉 +##瀋 +##瀏 +##瀑 +##瀕 +##瀘 +##瀚 +##瀛 +##瀝 +##瀞 +##瀟 +##瀧 +##瀨 +##瀬 +##瀰 +##瀾 +##灌 +##灏 +##灑 +##灘 +##灝 +##灞 +##灣 +##火 +##灬 +##灭 +##灯 +##灰 +##灵 +##灶 +##灸 +##灼 +##災 +##灾 +##灿 +##炀 +##炁 +##炅 +##炉 +##炊 +##炎 +##炒 +##炔 +##炕 +##炖 +##炙 +##炜 +##炫 +##炬 +##炭 +##炮 +##炯 +##炳 +##炷 +##炸 +##点 +##為 +##炼 +##炽 +##烁 +##烂 +##烃 +##烈 +##烊 +##烏 +##烘 +##烙 +##烛 +##烟 +##烤 +##烦 +##烧 +##烨 +##烩 +##烫 +##烬 +##热 +##烯 +##烷 +##烹 +##烽 +##焉 +##焊 +##焕 +##焖 +##焗 +##焘 +##焙 +##焚 +##焜 +##無 +##焦 +##焯 +##焰 +##焱 +##然 +##焼 +##煅 +##煉 +##煊 +##煌 +##煎 +##煒 +##煖 +##煙 +##煜 +##煞 +##煤 +##煥 +##煦 +##照 +##煨 +##煩 +##煮 +##煲 +##煸 +##煽 +##熄 +##熊 +##熏 +##熒 +##熔 +##熙 +##熟 +##熠 +##熨 +##熬 +##熱 +##熵 +##熹 +##熾 +##燁 +##燃 +##燄 +##燈 +##燉 +##燊 +##燎 +##燒 +##燔 +##燕 +##燙 +##燜 +##營 +##燥 +##燦 +##燧 +##燭 +##燮 +##燴 +##燻 +##燼 +##燿 +##爆 +##爍 +##爐 +##爛 +##爪 +##爬 +##爭 +##爰 +##爱 +##爲 +##爵 +##父 +##爷 +##爸 +##爹 +##爺 +##爻 +##爽 +##爾 +##牆 +##片 +##版 +##牌 +##牍 +##牒 +##牙 +##牛 +##牝 +##牟 +##牠 +##牡 +##牢 +##牦 +##牧 +##物 +##牯 +##牲 +##牴 +##牵 +##特 +##牺 +##牽 +##犀 +##犁 +##犄 +##犊 +##犍 +##犒 +##犢 +##犧 +##犬 +##犯 +##状 +##犷 +##犸 +##犹 +##狀 +##狂 +##狄 +##狈 +##狎 +##狐 +##狒 +##狗 +##狙 +##狞 +##狠 +##狡 +##狩 +##独 +##狭 +##狮 +##狰 +##狱 +##狸 +##狹 +##狼 +##狽 +##猎 +##猕 +##猖 +##猗 +##猙 +##猛 +##猜 +##猝 +##猥 +##猩 +##猪 +##猫 +##猬 +##献 +##猴 +##猶 +##猷 +##猾 +##猿 +##獄 +##獅 +##獎 +##獐 +##獒 +##獗 +##獠 +##獣 +##獨 +##獭 +##獰 +##獲 +##獵 +##獷 +##獸 +##獺 +##獻 +##獼 +##獾 +##玄 +##率 +##玉 +##王 +##玑 +##玖 +##玛 +##玟 +##玠 +##玥 +##玩 +##玫 +##玮 +##环 +##现 +##玲 +##玳 +##玷 +##玺 +##玻 +##珀 +##珂 +##珅 +##珈 +##珉 +##珊 +##珍 +##珏 +##珐 +##珑 +##珙 +##珞 +##珠 +##珣 +##珥 +##珩 +##珪 +##班 +##珮 +##珲 +##珺 +##現 +##球 +##琅 +##理 +##琇 +##琉 +##琊 +##琍 +##琏 +##琐 +##琛 +##琢 +##琥 +##琦 +##琨 +##琪 +##琬 +##琮 +##琰 +##琲 +##琳 +##琴 +##琵 +##琶 +##琺 +##琼 +##瑀 +##瑁 +##瑄 +##瑋 +##瑕 +##瑗 +##瑙 +##瑚 +##瑛 +##瑜 +##瑞 +##瑟 +##瑠 +##瑣 +##瑤 +##瑩 +##瑪 +##瑯 +##瑰 +##瑶 +##瑾 +##璀 +##璁 +##璃 +##璇 +##璉 +##璋 +##璎 +##璐 +##璜 +##璞 +##璟 +##璧 +##璨 +##環 +##璽 +##璿 +##瓊 +##瓏 +##瓒 +##瓜 +##瓢 +##瓣 +##瓤 +##瓦 +##瓮 +##瓯 +##瓴 +##瓶 +##瓷 +##甄 +##甌 +##甕 +##甘 +##甙 +##甚 +##甜 +##生 +##產 +##産 +##甥 +##甦 +##用 +##甩 +##甫 +##甬 +##甭 +##甯 +##田 +##由 +##甲 +##申 +##电 +##男 +##甸 +##町 +##画 +##甾 +##畀 +##畅 +##界 +##畏 +##畑 +##畔 +##留 +##畜 +##畝 +##畢 +##略 +##畦 +##番 +##畫 +##異 +##畲 +##畳 +##畴 +##當 +##畸 +##畹 +##畿 +##疆 +##疇 +##疊 +##疏 +##疑 +##疔 +##疖 +##疗 +##疙 +##疚 +##疝 +##疟 +##疡 +##疣 +##疤 +##疥 +##疫 +##疮 +##疯 +##疱 +##疲 +##疳 +##疵 +##疸 +##疹 +##疼 +##疽 +##疾 +##痂 +##病 +##症 +##痈 +##痉 +##痊 +##痍 +##痒 +##痔 +##痕 +##痘 +##痙 +##痛 +##痞 +##痠 +##痢 +##痣 +##痤 +##痧 +##痨 +##痪 +##痫 +##痰 +##痱 +##痴 +##痹 +##痺 +##痼 +##痿 +##瘀 +##瘁 +##瘋 +##瘍 +##瘓 +##瘘 +##瘙 +##瘟 +##瘠 +##瘡 +##瘢 +##瘤 +##瘦 +##瘧 +##瘩 +##瘪 +##瘫 +##瘴 +##瘸 +##瘾 +##療 +##癇 +##癌 +##癒 +##癖 +##癜 +##癞 +##癡 +##癢 +##癣 +##癥 +##癫 +##癬 +##癮 +##癱 +##癲 +##癸 +##発 +##登 +##發 +##白 +##百 +##皂 +##的 +##皆 +##皇 +##皈 +##皋 +##皎 +##皑 +##皓 +##皖 +##皙 +##皚 +##皮 +##皰 +##皱 +##皴 +##皺 +##皿 +##盂 +##盃 +##盅 +##盆 +##盈 +##益 +##盎 +##盏 +##盐 +##监 +##盒 +##盔 +##盖 +##盗 +##盘 +##盛 +##盜 +##盞 +##盟 +##盡 +##監 +##盤 +##盥 +##盧 +##盪 +##目 +##盯 +##盱 +##盲 +##直 +##相 +##盹 +##盼 +##盾 +##省 +##眈 +##眉 +##看 +##県 +##眙 +##眞 +##真 +##眠 +##眦 +##眨 +##眩 +##眯 +##眶 +##眷 +##眸 +##眺 +##眼 +##眾 +##着 +##睁 +##睇 +##睏 +##睐 +##睑 +##睛 +##睜 +##睞 +##睡 +##睢 +##督 +##睥 +##睦 +##睨 +##睪 +##睫 +##睬 +##睹 +##睽 +##睾 +##睿 +##瞄 +##瞅 +##瞇 +##瞋 +##瞌 +##瞎 +##瞑 +##瞒 +##瞓 +##瞞 +##瞟 +##瞠 +##瞥 +##瞧 +##瞩 +##瞪 +##瞬 +##瞭 +##瞰 +##瞳 +##瞻 +##瞼 +##瞿 +##矇 +##矍 +##矗 +##矚 +##矛 +##矜 +##矢 +##矣 +##知 +##矩 +##矫 +##短 +##矮 +##矯 +##石 +##矶 +##矽 +##矾 +##矿 +##码 +##砂 +##砌 +##砍 +##砒 +##研 +##砖 +##砗 +##砚 +##砝 +##砣 +##砥 +##砧 +##砭 +##砰 +##砲 +##破 +##砷 +##砸 +##砺 +##砼 +##砾 +##础 +##硅 +##硐 +##硒 +##硕 +##硝 +##硫 +##硬 +##确 +##硯 +##硼 +##碁 +##碇 +##碉 +##碌 +##碍 +##碎 +##碑 +##碓 +##碗 +##碘 +##碚 +##碛 +##碟 +##碣 +##碧 +##碩 +##碰 +##碱 +##碳 +##碴 +##確 +##碼 +##碾 +##磁 +##磅 +##磊 +##磋 +##磐 +##磕 +##磚 +##磡 +##磨 +##磬 +##磯 +##磲 +##磷 +##磺 +##礁 +##礎 +##礙 +##礡 +##礦 +##礪 +##礫 +##礴 +##示 +##礼 +##社 +##祀 +##祁 +##祂 +##祇 +##祈 +##祉 +##祎 +##祐 +##祕 +##祖 +##祗 +##祚 +##祛 +##祜 +##祝 +##神 +##祟 +##祠 +##祢 +##祥 +##票 +##祭 +##祯 +##祷 +##祸 +##祺 +##祿 +##禀 +##禁 +##禄 +##禅 +##禍 +##禎 +##福 +##禛 +##禦 +##禧 +##禪 +##禮 +##禱 +##禹 +##禺 +##离 +##禽 +##禾 +##禿 +##秀 +##私 +##秃 +##秆 +##秉 +##秋 +##种 +##科 +##秒 +##秘 +##租 +##秣 +##秤 +##秦 +##秧 +##秩 +##秭 +##积 +##称 +##秸 +##移 +##秽 +##稀 +##稅 +##程 +##稍 +##税 +##稔 +##稗 +##稚 +##稜 +##稞 +##稟 +##稠 +##稣 +##種 +##稱 +##稲 +##稳 +##稷 +##稹 +##稻 +##稼 +##稽 +##稿 +##穀 +##穂 +##穆 +##穌 +##積 +##穎 +##穗 +##穢 +##穩 +##穫 +##穴 +##究 +##穷 +##穹 +##空 +##穿 +##突 +##窃 +##窄 +##窈 +##窍 +##窑 +##窒 +##窓 +##窕 +##窖 +##窗 +##窘 +##窜 +##窝 +##窟 +##窠 +##窥 +##窦 +##窨 +##窩 +##窪 +##窮 +##窯 +##窺 +##窿 +##竄 +##竅 +##竇 +##竊 +##立 +##竖 +##站 +##竜 +##竞 +##竟 +##章 +##竣 +##童 +##竭 +##端 +##競 +##竹 +##竺 +##竽 +##竿 +##笃 +##笆 +##笈 +##笋 +##笏 +##笑 +##笔 +##笙 +##笛 +##笞 +##笠 +##符 +##笨 +##第 +##笹 +##笺 +##笼 +##筆 +##等 +##筊 +##筋 +##筍 +##筏 +##筐 +##筑 +##筒 +##答 +##策 +##筛 +##筝 +##筠 +##筱 +##筲 +##筵 +##筷 +##筹 +##签 +##简 +##箇 +##箋 +##箍 +##箏 +##箐 +##箔 +##箕 +##算 +##箝 +##管 +##箩 +##箫 +##箭 +##箱 +##箴 +##箸 +##節 +##篁 +##範 +##篆 +##篇 +##築 +##篑 +##篓 +##篙 +##篝 +##篠 +##篡 +##篤 +##篩 +##篪 +##篮 +##篱 +##篷 +##簇 +##簌 +##簍 +##簡 +##簦 +##簧 +##簪 +##簫 +##簷 +##簸 +##簽 +##簾 +##簿 +##籁 +##籃 +##籌 +##籍 +##籐 +##籟 +##籠 +##籤 +##籬 +##籮 +##籲 +##米 +##类 +##籼 +##籽 +##粄 +##粉 +##粑 +##粒 +##粕 +##粗 +##粘 +##粟 +##粤 +##粥 +##粧 +##粪 +##粮 +##粱 +##粲 +##粳 +##粵 +##粹 +##粼 +##粽 +##精 +##粿 +##糅 +##糊 +##糍 +##糕 +##糖 +##糗 +##糙 +##糜 +##糞 +##糟 +##糠 +##糧 +##糬 +##糯 +##糰 +##糸 +##系 +##糾 +##紀 +##紂 +##約 +##紅 +##紉 +##紊 +##紋 +##納 +##紐 +##紓 +##純 +##紗 +##紘 +##紙 +##級 +##紛 +##紜 +##素 +##紡 +##索 +##紧 +##紫 +##紮 +##累 +##細 +##紳 +##紹 +##紺 +##終 +##絃 +##組 +##絆 +##経 +##結 +##絕 +##絞 +##絡 +##絢 +##給 +##絨 +##絮 +##統 +##絲 +##絳 +##絵 +##絶 +##絹 +##綁 +##綏 +##綑 +##經 +##継 +##続 +##綜 +##綠 +##綢 +##綦 +##綫 +##綬 +##維 +##綱 +##網 +##綴 +##綵 +##綸 +##綺 +##綻 +##綽 +##綾 +##綿 +##緊 +##緋 +##総 +##緑 +##緒 +##緘 +##線 +##緝 +##緞 +##締 +##緣 +##編 +##緩 +##緬 +##緯 +##練 +##緹 +##緻 +##縁 +##縄 +##縈 +##縛 +##縝 +##縣 +##縫 +##縮 +##縱 +##縴 +##縷 +##總 +##績 +##繁 +##繃 +##繆 +##繇 +##繋 +##織 +##繕 +##繚 +##繞 +##繡 +##繩 +##繪 +##繫 +##繭 +##繳 +##繹 +##繼 +##繽 +##纂 +##續 +##纍 +##纏 +##纓 +##纔 +##纖 +##纜 +##纠 +##红 +##纣 +##纤 +##约 +##级 +##纨 +##纪 +##纫 +##纬 +##纭 +##纯 +##纰 +##纱 +##纲 +##纳 +##纵 +##纶 +##纷 +##纸 +##纹 +##纺 +##纽 +##纾 +##线 +##绀 +##练 +##组 +##绅 +##细 +##织 +##终 +##绊 +##绍 +##绎 +##经 +##绑 +##绒 +##结 +##绔 +##绕 +##绘 +##给 +##绚 +##绛 +##络 +##绝 +##绞 +##统 +##绡 +##绢 +##绣 +##绥 +##绦 +##继 +##绩 +##绪 +##绫 +##续 +##绮 +##绯 +##绰 +##绳 +##维 +##绵 +##绶 +##绷 +##绸 +##绻 +##综 +##绽 +##绾 +##绿 +##缀 +##缄 +##缅 +##缆 +##缇 +##缈 +##缉 +##缎 +##缓 +##缔 +##缕 +##编 +##缘 +##缙 +##缚 +##缜 +##缝 +##缠 +##缢 +##缤 +##缥 +##缨 +##缩 +##缪 +##缭 +##缮 +##缰 +##缱 +##缴 +##缸 +##缺 +##缽 +##罂 +##罄 +##罌 +##罐 +##网 +##罔 +##罕 +##罗 +##罚 +##罡 +##罢 +##罩 +##罪 +##置 +##罰 +##署 +##罵 +##罷 +##罹 +##羁 +##羅 +##羈 +##羊 +##羌 +##美 +##羔 +##羚 +##羞 +##羟 +##羡 +##羣 +##群 +##羥 +##羧 +##羨 +##義 +##羯 +##羲 +##羸 +##羹 +##羽 +##羿 +##翁 +##翅 +##翊 +##翌 +##翎 +##習 +##翔 +##翘 +##翟 +##翠 +##翡 +##翦 +##翩 +##翰 +##翱 +##翳 +##翹 +##翻 +##翼 +##耀 +##老 +##考 +##耄 +##者 +##耆 +##耋 +##而 +##耍 +##耐 +##耒 +##耕 +##耗 +##耘 +##耙 +##耦 +##耨 +##耳 +##耶 +##耷 +##耸 +##耻 +##耽 +##耿 +##聂 +##聆 +##聊 +##聋 +##职 +##聒 +##联 +##聖 +##聘 +##聚 +##聞 +##聪 +##聯 +##聰 +##聲 +##聳 +##聴 +##聶 +##職 +##聽 +##聾 +##聿 +##肃 +##肄 +##肅 +##肆 +##肇 +##肉 +##肋 +##肌 +##肏 +##肓 +##肖 +##肘 +##肚 +##肛 +##肝 +##肠 +##股 +##肢 +##肤 +##肥 +##肩 +##肪 +##肮 +##肯 +##肱 +##育 +##肴 +##肺 +##肽 +##肾 +##肿 +##胀 +##胁 +##胃 +##胄 +##胆 +##背 +##胍 +##胎 +##胖 +##胚 +##胛 +##胜 +##胝 +##胞 +##胡 +##胤 +##胥 +##胧 +##胫 +##胭 +##胯 +##胰 +##胱 +##胳 +##胴 +##胶 +##胸 +##胺 +##能 +##脂 +##脅 +##脆 +##脇 +##脈 +##脉 +##脊 +##脍 +##脏 +##脐 +##脑 +##脓 +##脖 +##脘 +##脚 +##脛 +##脣 +##脩 +##脫 +##脯 +##脱 +##脲 +##脳 +##脸 +##脹 +##脾 +##腆 +##腈 +##腊 +##腋 +##腌 +##腎 +##腐 +##腑 +##腓 +##腔 +##腕 +##腥 +##腦 +##腩 +##腫 +##腭 +##腮 +##腰 +##腱 +##腳 +##腴 +##腸 +##腹 +##腺 +##腻 +##腼 +##腾 +##腿 +##膀 +##膈 +##膊 +##膏 +##膑 +##膘 +##膚 +##膛 +##膜 +##膝 +##膠 +##膦 +##膨 +##膩 +##膳 +##膺 +##膻 +##膽 +##膾 +##膿 +##臀 +##臂 +##臃 +##臆 +##臉 +##臊 +##臍 +##臓 +##臘 +##臟 +##臣 +##臥 +##臧 +##臨 +##自 +##臬 +##臭 +##至 +##致 +##臺 +##臻 +##臼 +##臾 +##舀 +##舂 +##舅 +##舆 +##與 +##興 +##舉 +##舊 +##舌 +##舍 +##舎 +##舐 +##舒 +##舔 +##舖 +##舗 +##舛 +##舜 +##舞 +##舟 +##航 +##舫 +##般 +##舰 +##舱 +##舵 +##舶 +##舷 +##舸 +##船 +##舺 +##舾 +##艇 +##艋 +##艘 +##艙 +##艦 +##艮 +##良 +##艰 +##艱 +##色 +##艳 +##艷 +##艹 +##艺 +##艾 +##节 +##芃 +##芈 +##芊 +##芋 +##芍 +##芎 +##芒 +##芙 +##芜 +##芝 +##芡 +##芥 +##芦 +##芩 +##芪 +##芫 +##芬 +##芭 +##芮 +##芯 +##花 +##芳 +##芷 +##芸 +##芹 +##芻 +##芽 +##芾 +##苁 +##苄 +##苇 +##苋 +##苍 +##苏 +##苑 +##苒 +##苓 +##苔 +##苕 +##苗 +##苛 +##苜 +##苞 +##苟 +##苡 +##苣 +##若 +##苦 +##苫 +##苯 +##英 +##苷 +##苹 +##苻 +##茁 +##茂 +##范 +##茄 +##茅 +##茉 +##茎 +##茏 +##茗 +##茜 +##茧 +##茨 +##茫 +##茬 +##茭 +##茯 +##茱 +##茲 +##茴 +##茵 +##茶 +##茸 +##茹 +##茼 +##荀 +##荃 +##荆 +##草 +##荊 +##荏 +##荐 +##荒 +##荔 +##荖 +##荘 +##荚 +##荞 +##荟 +##荠 +##荡 +##荣 +##荤 +##荥 +##荧 +##荨 +##荪 +##荫 +##药 +##荳 +##荷 +##荸 +##荻 +##荼 +##荽 +##莅 +##莆 +##莉 +##莊 +##莎 +##莒 +##莓 +##莖 +##莘 +##莞 +##莠 +##莢 +##莧 +##莪 +##莫 +##莱 +##莲 +##莴 +##获 +##莹 +##莺 +##莽 +##莿 +##菀 +##菁 +##菅 +##菇 +##菈 +##菊 +##菌 +##菏 +##菓 +##菖 +##菘 +##菜 +##菟 +##菠 +##菡 +##菩 +##華 +##菱 +##菲 +##菸 +##菽 +##萁 +##萃 +##萄 +##萊 +##萋 +##萌 +##萍 +##萎 +##萘 +##萝 +##萤 +##营 +##萦 +##萧 +##萨 +##萩 +##萬 +##萱 +##萵 +##萸 +##萼 +##落 +##葆 +##葉 +##著 +##葚 +##葛 +##葡 +##董 +##葦 +##葩 +##葫 +##葬 +##葭 +##葯 +##葱 +##葳 +##葵 +##葷 +##葺 +##蒂 +##蒋 +##蒐 +##蒔 +##蒙 +##蒜 +##蒞 +##蒟 +##蒡 +##蒨 +##蒲 +##蒸 +##蒹 +##蒻 +##蒼 +##蒿 +##蓁 +##蓄 +##蓆 +##蓉 +##蓋 +##蓑 +##蓓 +##蓖 +##蓝 +##蓟 +##蓦 +##蓬 +##蓮 +##蓼 +##蓿 +##蔑 +##蔓 +##蔔 +##蔗 +##蔘 +##蔚 +##蔡 +##蔣 +##蔥 +##蔫 +##蔬 +##蔭 +##蔵 +##蔷 +##蔺 +##蔻 +##蔼 +##蔽 +##蕁 +##蕃 +##蕈 +##蕉 +##蕊 +##蕎 +##蕙 +##蕤 +##蕨 +##蕩 +##蕪 +##蕭 +##蕲 +##蕴 +##蕻 +##蕾 +##薄 +##薅 +##薇 +##薈 +##薊 +##薏 +##薑 +##薔 +##薙 +##薛 +##薦 +##薨 +##薩 +##薪 +##薬 +##薯 +##薰 +##薹 +##藉 +##藍 +##藏 +##藐 +##藓 +##藕 +##藜 +##藝 +##藤 +##藥 +##藩 +##藹 +##藻 +##藿 +##蘆 +##蘇 +##蘊 +##蘋 +##蘑 +##蘚 +##蘭 +##蘸 +##蘼 +##蘿 +##虎 +##虏 +##虐 +##虑 +##虔 +##處 +##虚 +##虛 +##虜 +##虞 +##號 +##虢 +##虧 +##虫 +##虬 +##虱 +##虹 +##虻 +##虽 +##虾 +##蚀 +##蚁 +##蚂 +##蚊 +##蚌 +##蚓 +##蚕 +##蚜 +##蚝 +##蚣 +##蚤 +##蚩 +##蚪 +##蚯 +##蚱 +##蚵 +##蛀 +##蛆 +##蛇 +##蛊 +##蛋 +##蛎 +##蛐 +##蛔 +##蛙 +##蛛 +##蛟 +##蛤 +##蛭 +##蛮 +##蛰 +##蛳 +##蛹 +##蛻 +##蛾 +##蜀 +##蜂 +##蜃 +##蜆 +##蜇 +##蜈 +##蜊 +##蜍 +##蜒 +##蜓 +##蜕 +##蜗 +##蜘 +##蜚 +##蜜 +##蜡 +##蜢 +##蜥 +##蜱 +##蜴 +##蜷 +##蜻 +##蜿 +##蝇 +##蝈 +##蝉 +##蝌 +##蝎 +##蝕 +##蝗 +##蝙 +##蝟 +##蝠 +##蝦 +##蝨 +##蝴 +##蝶 +##蝸 +##蝼 +##螂 +##螃 +##融 +##螞 +##螢 +##螨 +##螯 +##螳 +##螺 +##蟀 +##蟄 +##蟆 +##蟋 +##蟎 +##蟑 +##蟒 +##蟠 +##蟬 +##蟲 +##蟹 +##蟻 +##蟾 +##蠅 +##蠍 +##蠔 +##蠕 +##蠛 +##蠟 +##蠡 +##蠢 +##蠣 +##蠱 +##蠶 +##蠹 +##蠻 +##血 +##衄 +##衅 +##衆 +##行 +##衍 +##術 +##衔 +##街 +##衙 +##衛 +##衝 +##衞 +##衡 +##衢 +##衣 +##补 +##表 +##衩 +##衫 +##衬 +##衮 +##衰 +##衲 +##衷 +##衹 +##衾 +##衿 +##袁 +##袂 +##袄 +##袅 +##袈 +##袋 +##袍 +##袒 +##袖 +##袜 +##袞 +##袤 +##袪 +##被 +##袭 +##袱 +##裁 +##裂 +##装 +##裆 +##裊 +##裏 +##裔 +##裕 +##裘 +##裙 +##補 +##裝 +##裟 +##裡 +##裤 +##裨 +##裱 +##裳 +##裴 +##裸 +##裹 +##製 +##裾 +##褂 +##複 +##褐 +##褒 +##褓 +##褔 +##褚 +##褥 +##褪 +##褫 +##褲 +##褶 +##褻 +##襁 +##襄 +##襟 +##襠 +##襪 +##襬 +##襯 +##襲 +##西 +##要 +##覃 +##覆 +##覇 +##見 +##規 +##覓 +##視 +##覚 +##覦 +##覧 +##親 +##覬 +##観 +##覷 +##覺 +##覽 +##觀 +##见 +##观 +##规 +##觅 +##视 +##览 +##觉 +##觊 +##觎 +##觐 +##觑 +##角 +##觞 +##解 +##觥 +##触 +##觸 +##言 +##訂 +##計 +##訊 +##討 +##訓 +##訕 +##訖 +##託 +##記 +##訛 +##訝 +##訟 +##訣 +##訥 +##訪 +##設 +##許 +##訳 +##訴 +##訶 +##診 +##註 +##証 +##詆 +##詐 +##詔 +##評 +##詛 +##詞 +##詠 +##詡 +##詢 +##詣 +##試 +##詩 +##詫 +##詬 +##詭 +##詮 +##詰 +##話 +##該 +##詳 +##詹 +##詼 +##誅 +##誇 +##誉 +##誌 +##認 +##誓 +##誕 +##誘 +##語 +##誠 +##誡 +##誣 +##誤 +##誥 +##誦 +##誨 +##說 +##説 +##読 +##誰 +##課 +##誹 +##誼 +##調 +##諄 +##談 +##請 +##諏 +##諒 +##論 +##諗 +##諜 +##諡 +##諦 +##諧 +##諫 +##諭 +##諮 +##諱 +##諳 +##諷 +##諸 +##諺 +##諾 +##謀 +##謁 +##謂 +##謄 +##謊 +##謎 +##謐 +##謔 +##謗 +##謙 +##講 +##謝 +##謠 +##謨 +##謬 +##謹 +##謾 +##譁 +##證 +##譎 +##譏 +##識 +##譙 +##譚 +##譜 +##警 +##譬 +##譯 +##議 +##譲 +##譴 +##護 +##譽 +##讀 +##變 +##讓 +##讚 +##讞 +##计 +##订 +##认 +##讥 +##讧 +##讨 +##让 +##讪 +##讫 +##训 +##议 +##讯 +##记 +##讲 +##讳 +##讴 +##讶 +##讷 +##许 +##讹 +##论 +##讼 +##讽 +##设 +##访 +##诀 +##证 +##诃 +##评 +##诅 +##识 +##诈 +##诉 +##诊 +##诋 +##词 +##诏 +##译 +##试 +##诗 +##诘 +##诙 +##诚 +##诛 +##话 +##诞 +##诟 +##诠 +##诡 +##询 +##诣 +##诤 +##该 +##详 +##诧 +##诩 +##诫 +##诬 +##语 +##误 +##诰 +##诱 +##诲 +##说 +##诵 +##诶 +##请 +##诸 +##诺 +##读 +##诽 +##课 +##诿 +##谀 +##谁 +##调 +##谄 +##谅 +##谆 +##谈 +##谊 +##谋 +##谌 +##谍 +##谎 +##谏 +##谐 +##谑 +##谒 +##谓 +##谔 +##谕 +##谗 +##谘 +##谙 +##谚 +##谛 +##谜 +##谟 +##谢 +##谣 +##谤 +##谥 +##谦 +##谧 +##谨 +##谩 +##谪 +##谬 +##谭 +##谯 +##谱 +##谲 +##谴 +##谶 +##谷 +##豁 +##豆 +##豇 +##豈 +##豉 +##豊 +##豌 +##豎 +##豐 +##豔 +##豚 +##象 +##豢 +##豪 +##豫 +##豬 +##豹 +##豺 +##貂 +##貅 +##貌 +##貓 +##貔 +##貘 +##貝 +##貞 +##負 +##財 +##貢 +##貧 +##貨 +##販 +##貪 +##貫 +##責 +##貯 +##貰 +##貳 +##貴 +##貶 +##買 +##貸 +##費 +##貼 +##貽 +##貿 +##賀 +##賁 +##賂 +##賃 +##賄 +##資 +##賈 +##賊 +##賑 +##賓 +##賜 +##賞 +##賠 +##賡 +##賢 +##賣 +##賤 +##賦 +##質 +##賬 +##賭 +##賴 +##賺 +##購 +##賽 +##贅 +##贈 +##贊 +##贍 +##贏 +##贓 +##贖 +##贛 +##贝 +##贞 +##负 +##贡 +##财 +##责 +##贤 +##败 +##账 +##货 +##质 +##贩 +##贪 +##贫 +##贬 +##购 +##贮 +##贯 +##贰 +##贱 +##贲 +##贴 +##贵 +##贷 +##贸 +##费 +##贺 +##贻 +##贼 +##贾 +##贿 +##赁 +##赂 +##赃 +##资 +##赅 +##赈 +##赊 +##赋 +##赌 +##赎 +##赏 +##赐 +##赓 +##赔 +##赖 +##赘 +##赚 +##赛 +##赝 +##赞 +##赠 +##赡 +##赢 +##赣 +##赤 +##赦 +##赧 +##赫 +##赭 +##走 +##赳 +##赴 +##赵 +##赶 +##起 +##趁 +##超 +##越 +##趋 +##趕 +##趙 +##趟 +##趣 +##趨 +##足 +##趴 +##趵 +##趸 +##趺 +##趾 +##跃 +##跄 +##跆 +##跋 +##跌 +##跎 +##跑 +##跖 +##跚 +##跛 +##距 +##跟 +##跡 +##跤 +##跨 +##跩 +##跪 +##路 +##跳 +##践 +##跷 +##跹 +##跺 +##跻 +##踉 +##踊 +##踌 +##踏 +##踐 +##踝 +##踞 +##踟 +##踢 +##踩 +##踪 +##踮 +##踱 +##踴 +##踵 +##踹 +##蹂 +##蹄 +##蹇 +##蹈 +##蹉 +##蹊 +##蹋 +##蹑 +##蹒 +##蹙 +##蹟 +##蹣 +##蹤 +##蹦 +##蹩 +##蹬 +##蹭 +##蹲 +##蹴 +##蹶 +##蹺 +##蹼 +##蹿 +##躁 +##躇 +##躉 +##躊 +##躋 +##躍 +##躏 +##躪 +##身 +##躬 +##躯 +##躲 +##躺 +##軀 +##車 +##軋 +##軌 +##軍 +##軒 +##軟 +##転 +##軸 +##軼 +##軽 +##軾 +##較 +##載 +##輒 +##輓 +##輔 +##輕 +##輛 +##輝 +##輟 +##輩 +##輪 +##輯 +##輸 +##輻 +##輾 +##輿 +##轄 +##轅 +##轆 +##轉 +##轍 +##轎 +##轟 +##车 +##轧 +##轨 +##轩 +##转 +##轭 +##轮 +##软 +##轰 +##轲 +##轴 +##轶 +##轻 +##轼 +##载 +##轿 +##较 +##辄 +##辅 +##辆 +##辇 +##辈 +##辉 +##辊 +##辍 +##辐 +##辑 +##输 +##辕 +##辖 +##辗 +##辘 +##辙 +##辛 +##辜 +##辞 +##辟 +##辣 +##辦 +##辨 +##辩 +##辫 +##辭 +##辮 +##辯 +##辰 +##辱 +##農 +##边 +##辺 +##辻 +##込 +##辽 +##达 +##迁 +##迂 +##迄 +##迅 +##过 +##迈 +##迎 +##运 +##近 +##返 +##还 +##这 +##进 +##远 +##违 +##连 +##迟 +##迢 +##迤 +##迥 +##迦 +##迩 +##迪 +##迫 +##迭 +##述 +##迴 +##迷 +##迸 +##迹 +##迺 +##追 +##退 +##送 +##适 +##逃 +##逅 +##逆 +##选 +##逊 +##逍 +##透 +##逐 +##递 +##途 +##逕 +##逗 +##這 +##通 +##逛 +##逝 +##逞 +##速 +##造 +##逢 +##連 +##逮 +##週 +##進 +##逵 +##逶 +##逸 +##逻 +##逼 +##逾 +##遁 +##遂 +##遅 +##遇 +##遊 +##運 +##遍 +##過 +##遏 +##遐 +##遑 +##遒 +##道 +##達 +##違 +##遗 +##遙 +##遛 +##遜 +##遞 +##遠 +##遢 +##遣 +##遥 +##遨 +##適 +##遭 +##遮 +##遲 +##遴 +##遵 +##遶 +##遷 +##選 +##遺 +##遼 +##遽 +##避 +##邀 +##邁 +##邂 +##邃 +##還 +##邇 +##邈 +##邊 +##邋 +##邏 +##邑 +##邓 +##邕 +##邛 +##邝 +##邢 +##那 +##邦 +##邨 +##邪 +##邬 +##邮 +##邯 +##邰 +##邱 +##邳 +##邵 +##邸 +##邹 +##邺 +##邻 +##郁 +##郅 +##郊 +##郎 +##郑 +##郜 +##郝 +##郡 +##郢 +##郤 +##郦 +##郧 +##部 +##郫 +##郭 +##郴 +##郵 +##郷 +##郸 +##都 +##鄂 +##鄉 +##鄒 +##鄔 +##鄙 +##鄞 +##鄢 +##鄧 +##鄭 +##鄰 +##鄱 +##鄲 +##鄺 +##酉 +##酊 +##酋 +##酌 +##配 +##酐 +##酒 +##酗 +##酚 +##酝 +##酢 +##酣 +##酥 +##酩 +##酪 +##酬 +##酮 +##酯 +##酰 +##酱 +##酵 +##酶 +##酷 +##酸 +##酿 +##醃 +##醇 +##醉 +##醋 +##醍 +##醐 +##醒 +##醚 +##醛 +##醜 +##醞 +##醣 +##醪 +##醫 +##醬 +##醮 +##醯 +##醴 +##醺 +##釀 +##釁 +##采 +##釉 +##释 +##釋 +##里 +##重 +##野 +##量 +##釐 +##金 +##釗 +##釘 +##釜 +##針 +##釣 +##釦 +##釧 +##釵 +##鈀 +##鈉 +##鈍 +##鈎 +##鈔 +##鈕 +##鈞 +##鈣 +##鈦 +##鈪 +##鈴 +##鈺 +##鈾 +##鉀 +##鉄 +##鉅 +##鉉 +##鉑 +##鉗 +##鉚 +##鉛 +##鉤 +##鉴 +##鉻 +##銀 +##銃 +##銅 +##銑 +##銓 +##銖 +##銘 +##銜 +##銬 +##銭 +##銮 +##銳 +##銷 +##銹 +##鋁 +##鋅 +##鋒 +##鋤 +##鋪 +##鋰 +##鋸 +##鋼 +##錄 +##錐 +##錘 +##錚 +##錠 +##錢 +##錦 +##錨 +##錫 +##錮 +##錯 +##録 +##錳 +##錶 +##鍊 +##鍋 +##鍍 +##鍛 +##鍥 +##鍰 +##鍵 +##鍺 +##鍾 +##鎂 +##鎊 +##鎌 +##鎏 +##鎔 +##鎖 +##鎗 +##鎚 +##鎧 +##鎬 +##鎮 +##鎳 +##鏈 +##鏖 +##鏗 +##鏘 +##鏞 +##鏟 +##鏡 +##鏢 +##鏤 +##鏽 +##鐘 +##鐮 +##鐲 +##鐳 +##鐵 +##鐸 +##鐺 +##鑄 +##鑊 +##鑑 +##鑒 +##鑣 +##鑫 +##鑰 +##鑲 +##鑼 +##鑽 +##鑾 +##鑿 +##针 +##钉 +##钊 +##钎 +##钏 +##钒 +##钓 +##钗 +##钙 +##钛 +##钜 +##钝 +##钞 +##钟 +##钠 +##钡 +##钢 +##钣 +##钤 +##钥 +##钦 +##钧 +##钨 +##钩 +##钮 +##钯 +##钰 +##钱 +##钳 +##钴 +##钵 +##钺 +##钻 +##钼 +##钾 +##钿 +##铀 +##铁 +##铂 +##铃 +##铄 +##铅 +##铆 +##铉 +##铎 +##铐 +##铛 +##铜 +##铝 +##铠 +##铡 +##铢 +##铣 +##铤 +##铨 +##铩 +##铬 +##铭 +##铮 +##铰 +##铲 +##铵 +##银 +##铸 +##铺 +##链 +##铿 +##销 +##锁 +##锂 +##锄 +##锅 +##锆 +##锈 +##锉 +##锋 +##锌 +##锏 +##锐 +##锑 +##错 +##锚 +##锟 +##锡 +##锢 +##锣 +##锤 +##锥 +##锦 +##锭 +##键 +##锯 +##锰 +##锲 +##锵 +##锹 +##锺 +##锻 +##镀 +##镁 +##镂 +##镇 +##镉 +##镌 +##镍 +##镐 +##镑 +##镕 +##镖 +##镗 +##镛 +##镜 +##镣 +##镭 +##镯 +##镰 +##镳 +##镶 +##長 +##长 +##門 +##閃 +##閉 +##開 +##閎 +##閏 +##閑 +##閒 +##間 +##閔 +##閘 +##閡 +##関 +##閣 +##閥 +##閨 +##閩 +##閱 +##閲 +##閹 +##閻 +##閾 +##闆 +##闇 +##闊 +##闌 +##闍 +##闔 +##闕 +##闖 +##闘 +##關 +##闡 +##闢 +##门 +##闪 +##闫 +##闭 +##问 +##闯 +##闰 +##闲 +##间 +##闵 +##闷 +##闸 +##闹 +##闺 +##闻 +##闽 +##闾 +##阀 +##阁 +##阂 +##阅 +##阆 +##阇 +##阈 +##阉 +##阎 +##阐 +##阑 +##阔 +##阕 +##阖 +##阙 +##阚 +##阜 +##队 +##阡 +##阪 +##阮 +##阱 +##防 +##阳 +##阴 +##阵 +##阶 +##阻 +##阿 +##陀 +##陂 +##附 +##际 +##陆 +##陇 +##陈 +##陋 +##陌 +##降 +##限 +##陕 +##陛 +##陝 +##陞 +##陟 +##陡 +##院 +##陣 +##除 +##陨 +##险 +##陪 +##陰 +##陲 +##陳 +##陵 +##陶 +##陷 +##陸 +##険 +##陽 +##隅 +##隆 +##隈 +##隊 +##隋 +##隍 +##階 +##随 +##隐 +##隔 +##隕 +##隘 +##隙 +##際 +##障 +##隠 +##隣 +##隧 +##隨 +##險 +##隱 +##隴 +##隶 +##隸 +##隻 +##隼 +##隽 +##难 +##雀 +##雁 +##雄 +##雅 +##集 +##雇 +##雉 +##雋 +##雌 +##雍 +##雎 +##雏 +##雑 +##雒 +##雕 +##雖 +##雙 +##雛 +##雜 +##雞 +##離 +##難 +##雨 +##雪 +##雯 +##雰 +##雲 +##雳 +##零 +##雷 +##雹 +##電 +##雾 +##需 +##霁 +##霄 +##霆 +##震 +##霈 +##霉 +##霊 +##霍 +##霎 +##霏 +##霑 +##霓 +##霖 +##霜 +##霞 +##霧 +##霭 +##霰 +##露 +##霸 +##霹 +##霽 +##霾 +##靂 +##靄 +##靈 +##青 +##靓 +##靖 +##静 +##靚 +##靛 +##靜 +##非 +##靠 +##靡 +##面 +##靥 +##靦 +##革 +##靳 +##靴 +##靶 +##靼 +##鞅 +##鞋 +##鞍 +##鞏 +##鞑 +##鞘 +##鞠 +##鞣 +##鞦 +##鞭 +##韆 +##韋 +##韌 +##韓 +##韜 +##韦 +##韧 +##韩 +##韬 +##韭 +##音 +##韵 +##韶 +##韻 +##響 +##頁 +##頂 +##頃 +##項 +##順 +##須 +##頌 +##預 +##頑 +##頒 +##頓 +##頗 +##領 +##頜 +##頡 +##頤 +##頫 +##頭 +##頰 +##頷 +##頸 +##頹 +##頻 +##頼 +##顆 +##題 +##額 +##顎 +##顏 +##顔 +##願 +##顛 +##類 +##顧 +##顫 +##顯 +##顱 +##顴 +##页 +##顶 +##顷 +##项 +##顺 +##须 +##顼 +##顽 +##顾 +##顿 +##颁 +##颂 +##预 +##颅 +##领 +##颇 +##颈 +##颉 +##颊 +##颌 +##颍 +##颐 +##频 +##颓 +##颔 +##颖 +##颗 +##题 +##颚 +##颛 +##颜 +##额 +##颞 +##颠 +##颡 +##颢 +##颤 +##颦 +##颧 +##風 +##颯 +##颱 +##颳 +##颶 +##颼 +##飄 +##飆 +##风 +##飒 +##飓 +##飕 +##飘 +##飙 +##飚 +##飛 +##飞 +##食 +##飢 +##飨 +##飩 +##飪 +##飯 +##飲 +##飼 +##飽 +##飾 +##餃 +##餅 +##餉 +##養 +##餌 +##餐 +##餒 +##餓 +##餘 +##餚 +##餛 +##餞 +##餡 +##館 +##餮 +##餵 +##餾 +##饅 +##饈 +##饋 +##饌 +##饍 +##饑 +##饒 +##饕 +##饗 +##饞 +##饥 +##饨 +##饪 +##饬 +##饭 +##饮 +##饯 +##饰 +##饱 +##饲 +##饴 +##饵 +##饶 +##饷 +##饺 +##饼 +##饽 +##饿 +##馀 +##馁 +##馄 +##馅 +##馆 +##馈 +##馋 +##馍 +##馏 +##馒 +##馔 +##首 +##馗 +##香 +##馥 +##馨 +##馬 +##馭 +##馮 +##馳 +##馴 +##駁 +##駄 +##駅 +##駆 +##駐 +##駒 +##駕 +##駛 +##駝 +##駭 +##駱 +##駿 +##騁 +##騎 +##騏 +##験 +##騙 +##騨 +##騰 +##騷 +##驀 +##驅 +##驊 +##驍 +##驒 +##驕 +##驗 +##驚 +##驛 +##驟 +##驢 +##驥 +##马 +##驭 +##驮 +##驯 +##驰 +##驱 +##驳 +##驴 +##驶 +##驷 +##驸 +##驹 +##驻 +##驼 +##驾 +##驿 +##骁 +##骂 +##骄 +##骅 +##骆 +##骇 +##骈 +##骊 +##骋 +##验 +##骏 +##骐 +##骑 +##骗 +##骚 +##骛 +##骜 +##骞 +##骠 +##骡 +##骤 +##骥 +##骧 +##骨 +##骯 +##骰 +##骶 +##骷 +##骸 +##骼 +##髂 +##髅 +##髋 +##髏 +##髒 +##髓 +##體 +##髖 +##高 +##髦 +##髪 +##髮 +##髯 +##髻 +##鬃 +##鬆 +##鬍 +##鬓 +##鬚 +##鬟 +##鬢 +##鬣 +##鬥 +##鬧 +##鬱 +##鬼 +##魁 +##魂 +##魄 +##魅 +##魇 +##魍 +##魏 +##魔 +##魘 +##魚 +##魯 +##魷 +##鮑 +##鮨 +##鮪 +##鮭 +##鮮 +##鯉 +##鯊 +##鯖 +##鯛 +##鯨 +##鯰 +##鯽 +##鰍 +##鰓 +##鰭 +##鰲 +##鰻 +##鰾 +##鱈 +##鱉 +##鱔 +##鱗 +##鱷 +##鱸 +##鱼 +##鱿 +##鲁 +##鲈 +##鲍 +##鲑 +##鲛 +##鲜 +##鲟 +##鲢 +##鲤 +##鲨 +##鲫 +##鲱 +##鲲 +##鲶 +##鲷 +##鲸 +##鳃 +##鳄 +##鳅 +##鳌 +##鳍 +##鳕 +##鳖 +##鳗 +##鳝 +##鳞 +##鳥 +##鳩 +##鳳 +##鳴 +##鳶 +##鴉 +##鴕 +##鴛 +##鴦 +##鴨 +##鴻 +##鴿 +##鵑 +##鵜 +##鵝 +##鵡 +##鵬 +##鵰 +##鵲 +##鶘 +##鶩 +##鶯 +##鶴 +##鷗 +##鷲 +##鷹 +##鷺 +##鸚 +##鸞 +##鸟 +##鸠 +##鸡 +##鸢 +##鸣 +##鸥 +##鸦 +##鸨 +##鸪 +##鸭 +##鸯 +##鸳 +##鸵 +##鸽 +##鸾 +##鸿 +##鹂 +##鹃 +##鹄 +##鹅 +##鹈 +##鹉 +##鹊 +##鹌 +##鹏 +##鹑 +##鹕 +##鹘 +##鹜 +##鹞 +##鹤 +##鹦 +##鹧 +##鹫 +##鹭 +##鹰 +##鹳 +##鹵 +##鹹 +##鹼 +##鹽 +##鹿 +##麂 +##麋 +##麒 +##麓 +##麗 +##麝 +##麟 +##麥 +##麦 +##麩 +##麴 +##麵 +##麸 +##麺 +##麻 +##麼 +##麽 +##麾 +##黃 +##黄 +##黍 +##黎 +##黏 +##黑 +##黒 +##黔 +##默 +##黛 +##黜 +##黝 +##點 +##黠 +##黨 +##黯 +##黴 +##鼋 +##鼎 +##鼐 +##鼓 +##鼠 +##鼬 +##鼹 +##鼻 +##鼾 +##齁 +##齊 +##齋 +##齐 +##齒 +##齡 +##齢 +##齣 +##齦 +##齿 +##龄 +##龅 +##龈 +##龊 +##龋 +##龌 +##龍 +##龐 +##龔 +##龕 +##龙 +##龚 +##龛 +##龜 +##龟 +##︰ +##︱ +##︶ +##︿ +##﹁ +##﹂ +##﹍ +##﹏ +##﹐ +##﹑ +##﹒ +##﹔ +##﹕ +##﹖ +##﹗ +##﹙ +##﹚ +##﹝ +##﹞ +##﹡ +##﹣ +##！ +##＂ +##＃ +##＄ +##％ +##＆ +##＇ +##（ +##） +##＊ +##， +##－ +##． +##／ +##： +##； +##＜ +##？ +##＠ +##［ +##＼ +##］ +##＾ +##＿ +##｀ +##ｆ +##ｈ +##ｊ +##ｕ +##ｗ +##ｚ +##｛ +##｝ +##｡ +##｢ +##｣ +##､ +##･ +##ｯ +##ｰ +##ｲ +##ｸ +##ｼ +##ｽ +##ﾄ +##ﾉ +##ﾌ +##ﾗ +##ﾙ +##ﾝ +##ﾞ +##ﾟ +##￣ +##￥ +##👍 +##🔥 +##😂 +##😎 diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/create_pretraining_data.py b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/create_pretraining_data.py new file mode 100644 index 0000000..8bcbe27 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/create_pretraining_data.py @@ -0,0 +1,442 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Create masked LM/next sentence masked_lm TF examples for BERT.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import random +import tokenization +import tensorflow as tf + +flags = tf.flags + +FLAGS = flags.FLAGS + +flags.DEFINE_string("input_file", None, + "Input raw text file (or comma-separated list of files).") + +flags.DEFINE_string( + "output_file", None, + "Output TF example file (or comma-separated list of files).") + +flags.DEFINE_string("vocab_file", None, + "The vocabulary file that the BERT model was trained on.") + +flags.DEFINE_bool( + "do_lower_case", True, + "Whether to lower case the input text. Should be True for uncased " + "models and False for cased models.") + +flags.DEFINE_integer("max_seq_length", 128, "Maximum sequence length.") + +flags.DEFINE_integer("max_predictions_per_seq", 20, + "Maximum number of masked LM predictions per sequence.") + +flags.DEFINE_integer("random_seed", 12345, "Random seed for data generation.") + +flags.DEFINE_integer( + "dupe_factor", 10, + "Number of times to duplicate the input data (with different masks).") + +flags.DEFINE_float("masked_lm_prob", 0.15, "Masked LM probability.") + +flags.DEFINE_float( + "short_seq_prob", 0.1, + "Probability of creating sequences which are shorter than the " + "maximum length.") + + +class TrainingInstance(object): + """A single training instance (sentence pair).""" + + def __init__(self, tokens, segment_ids, masked_lm_positions, masked_lm_labels, + is_random_next): + self.tokens = tokens + self.segment_ids = segment_ids + self.is_random_next = is_random_next + self.masked_lm_positions = masked_lm_positions + self.masked_lm_labels = masked_lm_labels + + def __str__(self): + s = "" + s += "tokens: %s\n" % (" ".join( + [tokenization.printable_text(x) for x in self.tokens])) + s += "segment_ids: %s\n" % (" ".join([str(x) for x in self.segment_ids])) + s += "is_random_next: %s\n" % self.is_random_next + s += "masked_lm_positions: %s\n" % (" ".join( + [str(x) for x in self.masked_lm_positions])) + s += "masked_lm_labels: %s\n" % (" ".join( + [tokenization.printable_text(x) for x in self.masked_lm_labels])) + s += "\n" + return s + + def __repr__(self): + return self.__str__() + + +def write_instance_to_example_files(instances, tokenizer, max_seq_length, + max_predictions_per_seq, output_files): + """Create TF example files from `TrainingInstance`s.""" + writers = [] + for output_file in output_files: + writers.append(tf.python_io.TFRecordWriter(output_file)) + + writer_index = 0 + + total_written = 0 + for (inst_index, instance) in enumerate(instances): + input_ids = tokenizer.convert_tokens_to_ids(instance.tokens) + input_mask = [1] * len(input_ids) + segment_ids = list(instance.segment_ids) + assert len(input_ids) <= max_seq_length + + while len(input_ids) < max_seq_length: + input_ids.append(0) + input_mask.append(0) + segment_ids.append(0) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + + masked_lm_positions = list(instance.masked_lm_positions) + masked_lm_ids = tokenizer.convert_tokens_to_ids(instance.masked_lm_labels) + masked_lm_weights = [1.0] * len(masked_lm_ids) + + while len(masked_lm_positions) < max_predictions_per_seq: + masked_lm_positions.append(0) + masked_lm_ids.append(0) + masked_lm_weights.append(0.0) + + next_sentence_label = 1 if instance.is_random_next else 0 + + features = collections.OrderedDict() + features["input_ids"] = create_int_feature(input_ids) + features["input_mask"] = create_int_feature(input_mask) + features["segment_ids"] = create_int_feature(segment_ids) + features["masked_lm_positions"] = create_int_feature(masked_lm_positions) + features["masked_lm_ids"] = create_int_feature(masked_lm_ids) + features["masked_lm_weights"] = create_float_feature(masked_lm_weights) + features["next_sentence_labels"] = create_int_feature([next_sentence_label]) + + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + + writers[writer_index].write(tf_example.SerializeToString()) + writer_index = (writer_index + 1) % len(writers) + + total_written += 1 + + if inst_index < 20: + tf.logging.info("*** Example ***") + tf.logging.info("tokens: %s" % " ".join( + [tokenization.printable_text(x) for x in instance.tokens])) + + for feature_name in features.keys(): + feature = features[feature_name] + values = [] + if feature.int64_list.value: + values = feature.int64_list.value + elif feature.float_list.value: + values = feature.float_list.value + tf.logging.info( + "%s: %s" % (feature_name, " ".join([str(x) for x in values]))) + + for writer in writers: + writer.close() + + tf.logging.info("Wrote %d total instances", total_written) + + +def create_int_feature(values): + feature = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) + return feature + + +def create_float_feature(values): + feature = tf.train.Feature(float_list=tf.train.FloatList(value=list(values))) + return feature + + +def create_training_instances(input_files, tokenizer, max_seq_length, + dupe_factor, short_seq_prob, masked_lm_prob, + max_predictions_per_seq, rng): + """Create `TrainingInstance`s from raw text.""" + all_documents = [[]] + + # Input file format: + # (1) One sentence per line. These should ideally be actual sentences, not + # entire paragraphs or arbitrary spans of text. (Because we use the + # sentence boundaries for the "next sentence prediction" task). + # (2) Blank lines between documents. Document boundaries are needed so + # that the "next sentence prediction" task doesn't span between documents. + for input_file in input_files: + with tf.gfile.GFile(input_file, "r") as reader: + while True: + line = tokenization.convert_to_unicode(reader.readline()) + if not line: + break + line = line.strip() + + # Empty lines are used as document delimiters + if not line: + all_documents.append([]) + tokens = tokenizer.tokenize(line) + if tokens: + all_documents[-1].append(tokens) + + # Remove empty documents + all_documents = [x for x in all_documents if x] + rng.shuffle(all_documents) + + vocab_words = list(tokenizer.vocab.keys()) + instances = [] + for _ in range(dupe_factor): + for document_index in range(len(all_documents)): + instances.extend( + create_instances_from_document( + all_documents, document_index, max_seq_length, short_seq_prob, + masked_lm_prob, max_predictions_per_seq, vocab_words, rng)) + + rng.shuffle(instances) + return instances + + +def create_instances_from_document( + all_documents, document_index, max_seq_length, short_seq_prob, + masked_lm_prob, max_predictions_per_seq, vocab_words, rng): + """Creates `TrainingInstance`s for a single document.""" + document = all_documents[document_index] + + # Account for [CLS], [SEP], [SEP] + max_num_tokens = max_seq_length - 3 + + # We *usually* want to fill up the entire sequence since we are padding + # to `max_seq_length` anyways, so short sequences are generally wasted + # computation. However, we *sometimes* + # (i.e., short_seq_prob == 0.1 == 10% of the time) want to use shorter + # sequences to minimize the mismatch between pre-training and fine-tuning. + # The `target_seq_length` is just a rough target however, whereas + # `max_seq_length` is a hard limit. + target_seq_length = max_num_tokens + if rng.random() < short_seq_prob: + target_seq_length = rng.randint(2, max_num_tokens) + + # We DON'T just concatenate all of the tokens from a document into a long + # sequence and choose an arbitrary split point because this would make the + # next sentence prediction task too easy. Instead, we split the input into + # segments "A" and "B" based on the actual "sentences" provided by the user + # input. + instances = [] + current_chunk = [] + current_length = 0 + i = 0 + while i < len(document): + segment = document[i] + current_chunk.append(segment) + current_length += len(segment) + if i == len(document) - 1 or current_length >= target_seq_length: + if current_chunk: + # `a_end` is how many segments from `current_chunk` go into the `A` + # (first) sentence. + a_end = 1 + if len(current_chunk) >= 2: + a_end = rng.randint(1, len(current_chunk) - 1) + + tokens_a = [] + for j in range(a_end): + tokens_a.extend(current_chunk[j]) + + tokens_b = [] + # Random next + is_random_next = False + if len(current_chunk) == 1 or rng.random() < 0.5: + is_random_next = True + target_b_length = target_seq_length - len(tokens_a) + + # This should rarely go for more than one iteration for large + # corpora. However, just to be careful, we try to make sure that + # the random document is not the same as the document + # we're processing. + for _ in range(10): + random_document_index = rng.randint(0, len(all_documents) - 1) + if random_document_index != document_index: + break + + random_document = all_documents[random_document_index] + random_start = rng.randint(0, len(random_document) - 1) + for j in range(random_start, len(random_document)): + tokens_b.extend(random_document[j]) + if len(tokens_b) >= target_b_length: + break + # We didn't actually use these segments so we "put them back" so + # they don't go to waste. + num_unused_segments = len(current_chunk) - a_end + i -= num_unused_segments + # Actual next + else: + is_random_next = False + for j in range(a_end, len(current_chunk)): + tokens_b.extend(current_chunk[j]) + truncate_seq_pair(tokens_a, tokens_b, max_num_tokens, rng) + + assert len(tokens_a) >= 1 + assert len(tokens_b) >= 1 + + tokens = [] + segment_ids = [] + tokens.append("[CLS]") + segment_ids.append(0) + for token in tokens_a: + tokens.append(token) + segment_ids.append(0) + + tokens.append("[SEP]") + segment_ids.append(0) + + for token in tokens_b: + tokens.append(token) + segment_ids.append(1) + tokens.append("[SEP]") + segment_ids.append(1) + + (tokens, masked_lm_positions, + masked_lm_labels) = create_masked_lm_predictions( + tokens, masked_lm_prob, max_predictions_per_seq, vocab_words, rng) + instance = TrainingInstance( + tokens=tokens, + segment_ids=segment_ids, + is_random_next=is_random_next, + masked_lm_positions=masked_lm_positions, + masked_lm_labels=masked_lm_labels) + instances.append(instance) + current_chunk = [] + current_length = 0 + i += 1 + + return instances + + +MaskedLmInstance = collections.namedtuple("MaskedLmInstance", + ["index", "label"]) + + +def create_masked_lm_predictions(tokens, masked_lm_prob, + max_predictions_per_seq, vocab_words, rng): + """Creates the predictions for the masked LM objective.""" + + cand_indexes = [] + for (i, token) in enumerate(tokens): + if token == "[CLS]" or token == "[SEP]": + continue + cand_indexes.append(i) + + rng.shuffle(cand_indexes) + + output_tokens = list(tokens) + + num_to_predict = min(max_predictions_per_seq, + max(1, int(round(len(tokens) * masked_lm_prob)))) + + masked_lms = [] + covered_indexes = set() + for index in cand_indexes: + if len(masked_lms) >= num_to_predict: + break + if index in covered_indexes: + continue + covered_indexes.add(index) + + masked_token = None + # 80% of the time, replace with [MASK] + if rng.random() < 0.8: + masked_token = "[MASK]" + else: + # 10% of the time, keep original + if rng.random() < 0.5: + masked_token = tokens[index] + # 10% of the time, replace with random word + else: + masked_token = vocab_words[rng.randint(0, len(vocab_words) - 1)] + + output_tokens[index] = masked_token + + masked_lms.append(MaskedLmInstance(index=index, label=tokens[index])) + + masked_lms = sorted(masked_lms, key=lambda x: x.index) + + masked_lm_positions = [] + masked_lm_labels = [] + for p in masked_lms: + masked_lm_positions.append(p.index) + masked_lm_labels.append(p.label) + + return (output_tokens, masked_lm_positions, masked_lm_labels) + + +def truncate_seq_pair(tokens_a, tokens_b, max_num_tokens, rng): + """Truncates a pair of sequences to a maximum sequence length.""" + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_num_tokens: + break + + trunc_tokens = tokens_a if len(tokens_a) > len(tokens_b) else tokens_b + assert len(trunc_tokens) >= 1 + + # We want to sometimes truncate from the front and sometimes from the + # back to add more randomness and avoid biases. + if rng.random() < 0.5: + del trunc_tokens[0] + else: + trunc_tokens.pop() + + +def main(_): + tf.logging.set_verbosity(tf.logging.INFO) + + tokenizer = tokenization.FullTokenizer( + vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) + + input_files = [] + for input_pattern in FLAGS.input_file.split(","): + input_files.extend(tf.gfile.Glob(input_pattern)) + + tf.logging.info("*** Reading from input files ***") + for input_file in input_files: + tf.logging.info(" %s", input_file) + + rng = random.Random(FLAGS.random_seed) + instances = create_training_instances( + input_files, tokenizer, FLAGS.max_seq_length, FLAGS.dupe_factor, + FLAGS.short_seq_prob, FLAGS.masked_lm_prob, FLAGS.max_predictions_per_seq, + rng) + + output_files = FLAGS.output_file.split(",") + tf.logging.info("*** Writing to output files ***") + for output_file in output_files: + tf.logging.info(" %s", output_file) + + write_instance_to_example_files(instances, tokenizer, FLAGS.max_seq_length, + FLAGS.max_predictions_per_seq, output_files) + + +if __name__ == "__main__": + flags.mark_flag_as_required("input_file") + flags.mark_flag_as_required("output_file") + flags.mark_flag_as_required("vocab_file") + tf.app.run() diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/extract_features.py b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/extract_features.py new file mode 100644 index 0000000..60e3830 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/extract_features.py @@ -0,0 +1,419 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Extract pre-computed feature vectors from BERT.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import codecs +import collections +import json +import re + +import modeling +import tokenization +import tensorflow as tf + +flags = tf.flags + +FLAGS = flags.FLAGS + +flags.DEFINE_string("input_file", None, "") + +flags.DEFINE_string("output_file", None, "") + +flags.DEFINE_string("layers", "-1,-2,-3,-4", "") + +flags.DEFINE_string( + "bert_config_file", None, + "The config json file corresponding to the pre-trained BERT model. " + "This specifies the model architecture.") + +flags.DEFINE_integer( + "max_seq_length", 128, + "The maximum total input sequence length after WordPiece tokenization. " + "Sequences longer than this will be truncated, and sequences shorter " + "than this will be padded.") + +flags.DEFINE_string( + "init_checkpoint", None, + "Initial checkpoint (usually from a pre-trained BERT model).") + +flags.DEFINE_string("vocab_file", None, + "The vocabulary file that the BERT model was trained on.") + +flags.DEFINE_bool( + "do_lower_case", True, + "Whether to lower case the input text. Should be True for uncased " + "models and False for cased models.") + +flags.DEFINE_integer("batch_size", 32, "Batch size for predictions.") + +flags.DEFINE_bool("use_tpu", False, "Whether to use TPU or GPU/CPU.") + +flags.DEFINE_string("master", None, + "If using a TPU, the address of the master.") + +flags.DEFINE_integer( + "num_tpu_cores", 8, + "Only used if `use_tpu` is True. Total number of TPU cores to use.") + +flags.DEFINE_bool( + "use_one_hot_embeddings", False, + "If True, tf.one_hot will be used for embedding lookups, otherwise " + "tf.nn.embedding_lookup will be used. On TPUs, this should be True " + "since it is much faster.") + + +class InputExample(object): + + def __init__(self, unique_id, text_a, text_b): + self.unique_id = unique_id + self.text_a = text_a + self.text_b = text_b + + +class InputFeatures(object): + """A single set of features of data.""" + + def __init__(self, unique_id, tokens, input_ids, input_mask, input_type_ids): + self.unique_id = unique_id + self.tokens = tokens + self.input_ids = input_ids + self.input_mask = input_mask + self.input_type_ids = input_type_ids + + +def input_fn_builder(features, seq_length): + """Creates an `input_fn` closure to be passed to TPUEstimator.""" + + all_unique_ids = [] + all_input_ids = [] + all_input_mask = [] + all_input_type_ids = [] + + for feature in features: + all_unique_ids.append(feature.unique_id) + all_input_ids.append(feature.input_ids) + all_input_mask.append(feature.input_mask) + all_input_type_ids.append(feature.input_type_ids) + + def input_fn(params): + """The actual input function.""" + batch_size = params["batch_size"] + + num_examples = len(features) + + # This is for demo purposes and does NOT scale to large data sets. We do + # not use Dataset.from_generator() because that uses tf.py_func which is + # not TPU compatible. The right way to load data is with TFRecordReader. + d = tf.data.Dataset.from_tensor_slices({ + "unique_ids": + tf.constant(all_unique_ids, shape=[num_examples], dtype=tf.int32), + "input_ids": + tf.constant( + all_input_ids, shape=[num_examples, seq_length], + dtype=tf.int32), + "input_mask": + tf.constant( + all_input_mask, + shape=[num_examples, seq_length], + dtype=tf.int32), + "input_type_ids": + tf.constant( + all_input_type_ids, + shape=[num_examples, seq_length], + dtype=tf.int32), + }) + + d = d.batch(batch_size=batch_size, drop_remainder=False) + return d + + return input_fn + + +def model_fn_builder(bert_config, init_checkpoint, layer_indexes, use_tpu, + use_one_hot_embeddings): + """Returns `model_fn` closure for TPUEstimator.""" + + def model_fn(features, labels, mode, params): # pylint: disable=unused-argument + """The `model_fn` for TPUEstimator.""" + + unique_ids = features["unique_ids"] + input_ids = features["input_ids"] + input_mask = features["input_mask"] + input_type_ids = features["input_type_ids"] + + model = modeling.BertModel( + config=bert_config, + is_training=False, + input_ids=input_ids, + input_mask=input_mask, + token_type_ids=input_type_ids, + use_one_hot_embeddings=use_one_hot_embeddings) + + if mode != tf.estimator.ModeKeys.PREDICT: + raise ValueError("Only PREDICT modes are supported: %s" % (mode)) + + tvars = tf.trainable_variables() + scaffold_fn = None + (assignment_map, + initialized_variable_names) = modeling.get_assignment_map_from_checkpoint( + tvars, init_checkpoint) + if use_tpu: + + def tpu_scaffold(): + tf.train.init_from_checkpoint(init_checkpoint, assignment_map) + return tf.train.Scaffold() + + scaffold_fn = tpu_scaffold + else: + tf.train.init_from_checkpoint(init_checkpoint, assignment_map) + + tf.logging.info("**** Trainable Variables ****") + for var in tvars: + init_string = "" + if var.name in initialized_variable_names: + init_string = ", *INIT_FROM_CKPT*" + tf.logging.info(" name = %s, shape = %s%s", var.name, var.shape, + init_string) + + all_layers = model.get_all_encoder_layers() + + predictions = { + "unique_id": unique_ids, + } + + for (i, layer_index) in enumerate(layer_indexes): + predictions["layer_output_%d" % i] = all_layers[layer_index] + + output_spec = tf.contrib.tpu.TPUEstimatorSpec( + mode=mode, predictions=predictions, scaffold_fn=scaffold_fn) + return output_spec + + return model_fn + + +def convert_examples_to_features(examples, seq_length, tokenizer): + """Loads a data file into a list of `InputBatch`s.""" + + features = [] + for (ex_index, example) in enumerate(examples): + tokens_a = tokenizer.tokenize(example.text_a) + + tokens_b = None + if example.text_b: + tokens_b = tokenizer.tokenize(example.text_b) + + if tokens_b: + # Modifies `tokens_a` and `tokens_b` in place so that the total + # length is less than the specified length. + # Account for [CLS], [SEP], [SEP] with "- 3" + _truncate_seq_pair(tokens_a, tokens_b, seq_length - 3) + else: + # Account for [CLS] and [SEP] with "- 2" + if len(tokens_a) > seq_length - 2: + tokens_a = tokens_a[0:(seq_length - 2)] + + # The convention in BERT is: + # (a) For sequence pairs: + # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] + # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 + # (b) For single sequences: + # tokens: [CLS] the dog is hairy . [SEP] + # type_ids: 0 0 0 0 0 0 0 + # + # Where "type_ids" are used to indicate whether this is the first + # sequence or the second sequence. The embedding vectors for `type=0` and + # `type=1` were learned during pre-training and are added to the wordpiece + # embedding vector (and position vector). This is not *strictly* necessary + # since the [SEP] token unambiguously separates the sequences, but it makes + # it easier for the model to learn the concept of sequences. + # + # For classification tasks, the first vector (corresponding to [CLS]) is + # used as as the "sentence vector". Note that this only makes sense because + # the entire model is fine-tuned. + tokens = [] + input_type_ids = [] + tokens.append("[CLS]") + input_type_ids.append(0) + for token in tokens_a: + tokens.append(token) + input_type_ids.append(0) + tokens.append("[SEP]") + input_type_ids.append(0) + + if tokens_b: + for token in tokens_b: + tokens.append(token) + input_type_ids.append(1) + tokens.append("[SEP]") + input_type_ids.append(1) + + input_ids = tokenizer.convert_tokens_to_ids(tokens) + + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + input_mask = [1] * len(input_ids) + + # Zero-pad up to the sequence length. + while len(input_ids) < seq_length: + input_ids.append(0) + input_mask.append(0) + input_type_ids.append(0) + + assert len(input_ids) == seq_length + assert len(input_mask) == seq_length + assert len(input_type_ids) == seq_length + + if ex_index < 5: + tf.logging.info("*** Example ***") + tf.logging.info("unique_id: %s" % (example.unique_id)) + tf.logging.info("tokens: %s" % " ".join( + [tokenization.printable_text(x) for x in tokens])) + tf.logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) + tf.logging.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) + tf.logging.info( + "input_type_ids: %s" % " ".join([str(x) for x in input_type_ids])) + + features.append( + InputFeatures( + unique_id=example.unique_id, + tokens=tokens, + input_ids=input_ids, + input_mask=input_mask, + input_type_ids=input_type_ids)) + return features + + +def _truncate_seq_pair(tokens_a, tokens_b, max_length): + """Truncates a sequence pair in place to the maximum length.""" + + # This is a simple heuristic which will always truncate the longer sequence + # one token at a time. This makes more sense than truncating an equal percent + # of tokens from each, since if one sequence is very short then each token + # that's truncated likely contains more information than a longer sequence. + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_length: + break + if len(tokens_a) > len(tokens_b): + tokens_a.pop() + else: + tokens_b.pop() + + +def read_examples(input_file): + """Read a list of `InputExample`s from an input file.""" + examples = [] + unique_id = 0 + with tf.gfile.GFile(input_file, "r") as reader: + while True: + line = tokenization.convert_to_unicode(reader.readline()) + if not line: + break + line = line.strip() + text_a = None + text_b = None + m = re.match(r"^(.*) \|\|\| (.*)$", line) + if m is None: + text_a = line + else: + text_a = m.group(1) + text_b = m.group(2) + examples.append( + InputExample(unique_id=unique_id, text_a=text_a, text_b=text_b)) + unique_id += 1 + return examples + + +def main(_): + tf.logging.set_verbosity(tf.logging.INFO) + + layer_indexes = [int(x) for x in FLAGS.layers.split(",")] + + bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file) + + tokenizer = tokenization.FullTokenizer( + vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) + + is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 + run_config = tf.contrib.tpu.RunConfig( + master=FLAGS.master, + tpu_config=tf.contrib.tpu.TPUConfig( + num_shards=FLAGS.num_tpu_cores, + per_host_input_for_training=is_per_host)) + + examples = read_examples(FLAGS.input_file) + + features = convert_examples_to_features( + examples=examples, seq_length=FLAGS.max_seq_length, tokenizer=tokenizer) + + unique_id_to_feature = {} + for feature in features: + unique_id_to_feature[feature.unique_id] = feature + + model_fn = model_fn_builder( + bert_config=bert_config, + init_checkpoint=FLAGS.init_checkpoint, + layer_indexes=layer_indexes, + use_tpu=FLAGS.use_tpu, + use_one_hot_embeddings=FLAGS.use_one_hot_embeddings) + + # If TPU is not available, this will fall back to normal Estimator on CPU + # or GPU. + estimator = tf.contrib.tpu.TPUEstimator( + use_tpu=FLAGS.use_tpu, + model_fn=model_fn, + config=run_config, + predict_batch_size=FLAGS.batch_size) + + input_fn = input_fn_builder( + features=features, seq_length=FLAGS.max_seq_length) + + with codecs.getwriter("utf-8")(tf.gfile.Open(FLAGS.output_file, + "w")) as writer: + for result in estimator.predict(input_fn, yield_single_examples=True): + unique_id = int(result["unique_id"]) + feature = unique_id_to_feature[unique_id] + output_json = collections.OrderedDict() + output_json["linex_index"] = unique_id + all_features = [] + for (i, token) in enumerate(feature.tokens): + all_layers = [] + for (j, layer_index) in enumerate(layer_indexes): + layer_output = result["layer_output_%d" % j] + layers = collections.OrderedDict() + layers["index"] = layer_index + layers["values"] = [ + round(float(x), 6) for x in layer_output[i:(i + 1)].flat + ] + all_layers.append(layers) + features = collections.OrderedDict() + features["token"] = token + features["layers"] = all_layers + all_features.append(features) + output_json["features"] = all_features + writer.write(json.dumps(output_json) + "\n") + + +if __name__ == "__main__": + flags.mark_flag_as_required("input_file") + flags.mark_flag_as_required("vocab_file") + flags.mark_flag_as_required("bert_config_file") + flags.mark_flag_as_required("init_checkpoint") + flags.mark_flag_as_required("output_file") + tf.app.run() diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/fp16_utils.py b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/fp16_utils.py new file mode 100644 index 0000000..6b8bda9 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/fp16_utils.py @@ -0,0 +1,35 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import tensorflow as tf +import numpy as np + + +def float32_variable_storage_getter(getter, name, shape=None, dtype=None, + initializer=None, regularizer=None, + trainable=True, + *args, **kwargs): + """Custom variable getter that forces trainable variables to be stored in + float32 precision and then casts them to the training precision. + """ + storage_dtype = tf.float32 if trainable else dtype + variable = getter(name, shape, dtype=storage_dtype, + initializer=initializer, regularizer=regularizer, + trainable=trainable, + *args, **kwargs) + if trainable and dtype != tf.float32: + variable = tf.cast(variable, dtype) + return variable + diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/fused_layer_norm.py b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/fused_layer_norm.py new file mode 100644 index 0000000..ff6e137 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/fused_layer_norm.py @@ -0,0 +1,141 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import copy +import json +import math +import re +import six +import tensorflow as tf + +from tensorflow.python.framework import ops +from tensorflow.contrib.layers.python.layers import utils +from tensorflow.contrib.framework.python.ops import variables +from tensorflow.python.ops import init_ops +import numpy +from tensorflow.python.ops import array_ops +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import nn + +def fused_layer_norm(inputs, + center=True, + scale=True, + activation_fn=None, + reuse=None, + variables_collections=None, + outputs_collections=None, + trainable=True, + begin_norm_axis=1, + begin_params_axis=-1, + scope=None, + use_fused_batch_norm=False): + with tf.variable_scope( + scope, 'LayerNorm', [inputs], reuse=reuse) as sc: + inputs = ops.convert_to_tensor(inputs) + inputs_shape = inputs.shape + inputs_rank = inputs_shape.ndims + if inputs_rank is None: + raise ValueError('Inputs %s has undefined rank.' % inputs.name) + dtype = inputs.dtype.base_dtype + if begin_norm_axis < 0: + begin_norm_axis = inputs_rank + begin_norm_axis + if begin_params_axis >= inputs_rank or begin_norm_axis >= inputs_rank: + raise ValueError('begin_params_axis (%d) and begin_norm_axis (%d) ' + 'must be < rank(inputs) (%d)' % + (begin_params_axis, begin_norm_axis, inputs_rank)) + params_shape = inputs_shape[begin_params_axis:] + if not params_shape.is_fully_defined(): + raise ValueError( + 'Inputs %s: shape(inputs)[%s:] is not fully defined: %s' % + (inputs.name, begin_params_axis, inputs_shape)) + # Allocate parameters for the beta and gamma of the normalization. + beta, gamma = None, None + if center: + beta_collections = utils.get_variable_collections(variables_collections, + 'beta') + beta = variables.model_variable( + 'beta', + shape=params_shape, + dtype=dtype, + initializer=init_ops.zeros_initializer(), + collections=beta_collections, + trainable=trainable) + if scale: + gamma_collections = utils.get_variable_collections( + variables_collections, 'gamma') + gamma = variables.model_variable( + 'gamma', + shape=params_shape, + dtype=dtype, + initializer=init_ops.ones_initializer(), + collections=gamma_collections, + trainable=trainable) + if use_fused_batch_norm: + # get static TensorShape if fully defined, + # otherwise retrieve shape tensor + norm_shape = inputs.shape[begin_norm_axis:] + if norm_shape.is_fully_defined(): + bn_shape = [1, -1, 1, numpy.prod(norm_shape.as_list())] + else: + norm_shape = tf.shape(inputs)[begin_norm_axis:] + bn_shape = [1, -1, 1, tf.reduce_prod(norm_shape)] + if inputs.get_shape().is_fully_defined(): + outputs_shape = inputs.get_shape() + else: + outputs_shape = tf.shape(inputs) + inputs = array_ops.reshape(inputs, bn_shape) + if inputs.get_shape().is_fully_defined(): + # static inputs TensorShape fully defined after reshape. + ones = array_ops.ones(inputs.get_shape()[1], dtype=dtypes.float32) + zeros = array_ops.zeros(inputs.get_shape()[1], dtype=dtypes.float32) + else: + # static inputs TensorShape NOT fully defined after reshape. + # must use dynamic shape, which means these input tensors + # have to be created at runtime, which causes a slowdown. + scale_shape = tf.shape(inputs)[1] + ones = array_ops.ones(scale_shape, dtype=dtypes.float32) + zeros = array_ops.zeros(scale_shape, dtype=dtypes.float32) + outputs, mean, variance = nn.fused_batch_norm( + inputs, + ones, zeros, + epsilon=1e-4, + data_format="NCHW") + outputs = array_ops.reshape(outputs, outputs_shape) + if center and scale: + outputs = outputs * gamma + beta + elif center: + outputs = outputs + beta + elif scale: + outputs = outputs * gamma + else: + # Calculate the moments on the last axis (layer activations). + norm_axes = list(range(begin_norm_axis, inputs_rank)) + mean, variance = nn.moments(inputs, norm_axes, keep_dims=True) + # Compute layer normalization using the batch_normalization function. + variance_epsilon = 1e-4 + outputs = nn.batch_normalization( + inputs, + mean, + variance, + offset=beta, + scale=gamma, + variance_epsilon=variance_epsilon) + outputs.set_shape(inputs_shape) + if activation_fn is not None: + outputs = activation_fn(outputs) + return utils.collect_named_outputs(outputs_collections, sc.name, outputs) + diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/gpu_environment.py b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/gpu_environment.py new file mode 100644 index 0000000..948c3fa --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/gpu_environment.py @@ -0,0 +1,36 @@ +# coding=utf-8 +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tensorflow as tf +import numpy as np + +def float32_variable_storage_getter(getter, name, shape=None, dtype=None, + initializer=None, regularizer=None, + trainable=True, + *args, **kwargs): + """Custom variable getter that forces trainable variables to be stored in + float32 precision and then casts them to the training precision. + """ + storage_dtype = tf.float32 if trainable else dtype + variable = getter(name, shape, dtype=storage_dtype, + initializer=initializer, regularizer=regularizer, + trainable=trainable, + *args, **kwargs) + if trainable and dtype != tf.float32: + variable = tf.cast(variable, dtype) + return variable + +def get_custom_getter(compute_type): + return float32_variable_storage_getter if compute_type == tf.float16 else None diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/modeling.py b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/modeling.py new file mode 100644 index 0000000..95a8eda --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/modeling.py @@ -0,0 +1,1031 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The main BERT model and related functions.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import copy +import json +import math +import re +import numpy as np +import six +import tensorflow as tf + +from gpu_environment import get_custom_getter + +from npu_bridge.estimator.npu_unary_ops import npu_unary_ops +from npu_bridge.estimator import npu_ops + +class BertConfig(object): + """Configuration for `BertModel`.""" + + def __init__(self, + vocab_size, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02): + """Constructs BertConfig. + + Args: + vocab_size: Vocabulary size of `inputs_ids` in `BertModel`. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. + hidden_dropout_prob: The dropout probability for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `BertModel`. + initializer_range: The stdev of the truncated_normal_initializer for + initializing all weight matrices. + """ + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + + @classmethod + def from_dict(cls, json_object): + """Constructs a `BertConfig` from a Python dictionary of parameters.""" + config = BertConfig(vocab_size=None) + for (key, value) in six.iteritems(json_object): + config.__dict__[key] = value + return config + + @classmethod + def from_json_file(cls, json_file): + """Constructs a `BertConfig` from a json file of parameters.""" + with tf.gfile.GFile(json_file, "r") as reader: + text = reader.read() + return cls.from_dict(json.loads(text)) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" + + +class BertModel(object): + """BERT model ("Bidirectional Encoder Representations from Transformers"). + + Example usage: + + ```python + # Already been converted into WordPiece token ids + input_ids = tf.constant([[31, 51, 99], [15, 5, 0]]) + input_mask = tf.constant([[1, 1, 1], [1, 1, 0]]) + token_type_ids = tf.constant([[0, 0, 1], [0, 2, 0]]) + + config = modeling.BertConfig(vocab_size=32000, hidden_size=512, + num_hidden_layers=8, num_attention_heads=6, intermediate_size=1024) + + model = modeling.BertModel(config=config, is_training=True, + input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type_ids) + + label_embeddings = tf.get_variable(...) + pooled_output = model.get_pooled_output() + logits = tf.matmul(pooled_output, label_embeddings) + ... + ``` + """ + + def __init__(self, + config, + is_training, + input_ids, + input_mask=None, + token_type_ids=None, + use_one_hot_embeddings=False, + scope=None, + compute_type=tf.float32): + """Constructor for BertModel. + + Args: + config: `BertConfig` instance. + is_training: bool. true for training model, false for eval model. Controls + whether dropout will be applied. + input_ids: int32 Tensor of shape [batch_size, seq_length]. + input_mask: (optional) int32 Tensor of shape [batch_size, seq_length]. + token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length]. + use_one_hot_embeddings: (optional) bool. Whether to use one-hot word + embeddings or tf.embedding_lookup() for the word embeddings. On the TPU, + it is much faster if this is True, on the CPU or GPU, it is faster if + this is False. + scope: (optional) variable scope. Defaults to "bert". + compute_type: (optional) either float32 or float16. Only applies to GPUs. + + Raises: + ValueError: The config is invalid or one of the input tensor shapes + is invalid. + """ + config = copy.deepcopy(config) + if not is_training: + config.hidden_dropout_prob = 0.0 + config.attention_probs_dropout_prob = 0.0 + + input_shape = get_shape_list(input_ids, expected_rank=2) + batch_size = input_shape[0] + seq_length = input_shape[1] + + if input_mask is None: + input_mask = tf.ones(shape=[batch_size, seq_length], dtype=tf.int32) + + if token_type_ids is None: + token_type_ids = tf.zeros(shape=[batch_size, seq_length], dtype=tf.int32) + + with tf.variable_scope(scope, default_name="bert", custom_getter=get_custom_getter(compute_type)): + with tf.variable_scope("embeddings"): + # For good convergence with mixed precision training, + # it is important that the embedding codes remain fp32. + # Perform embedding lookup on the word ids. + (self.embedding_output, self.embedding_table) = embedding_lookup( + input_ids=input_ids, + vocab_size=config.vocab_size, + embedding_size=config.hidden_size, + initializer_range=config.initializer_range, + word_embedding_name="word_embeddings", + use_one_hot_embeddings=use_one_hot_embeddings) + + # Add positional embeddings and token type embeddings, then layer + # normalize and perform dropout. + self.embedding_output = embedding_postprocessor( + input_tensor=self.embedding_output, + use_token_type=True, + token_type_ids=token_type_ids, + token_type_vocab_size=config.type_vocab_size, + token_type_embedding_name="token_type_embeddings", + use_position_embeddings=True, + position_embedding_name="position_embeddings", + initializer_range=config.initializer_range, + max_position_embeddings=config.max_position_embeddings, + dropout_prob=config.hidden_dropout_prob, + use_one_hot_embeddings=use_one_hot_embeddings) + + with tf.variable_scope("encoder"): + # This converts a 2D mask of shape [batch_size, seq_length] to a 3D + # mask of shape [batch_size, seq_length, seq_length] which is used + # for the attention scores. + attention_mask = create_attention_mask_from_input_mask( + input_ids, input_mask) + + # Run the stacked transformer. + # `sequence_output` shape = [batch_size, seq_length, hidden_size]. + self.all_encoder_layers = transformer_model( + input_tensor=tf.saturate_cast(self.embedding_output, compute_type), + attention_mask=attention_mask, + hidden_size=config.hidden_size, + num_hidden_layers=config.num_hidden_layers, + num_attention_heads=config.num_attention_heads, + intermediate_size=config.intermediate_size, + intermediate_act_fn=get_activation(config.hidden_act), + hidden_dropout_prob=config.hidden_dropout_prob, + attention_probs_dropout_prob=config.attention_probs_dropout_prob, + initializer_range=config.initializer_range, + do_return_all_layers=True) + self.sequence_output = tf.cast(self.all_encoder_layers[-1], tf.float32) + # The "pooler" converts the encoded sequence tensor of shape + # [batch_size, seq_length, hidden_size] to a tensor of shape + # [batch_size, hidden_size]. This is necessary for segment-level + # (or segment-pair-level) classification tasks where we need a fixed + # dimensional representation of the segment. + with tf.variable_scope("pooler"): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. We assume that this has been pre-trained + first_token_tensor = tf.squeeze(self.sequence_output[:, 0:1, :], axis=1) + if tf.flags.FLAGS.use_fp16_cls: + first_token_tensor = tf.cast(first_token_tensor, tf.float16) + self_pooled_output = tf.layers.dense( + first_token_tensor, + config.hidden_size, + activation=tf.tanh, + kernel_initializer=create_initializer(config.initializer_range)) + self.pooled_output = tf.cast(self_pooled_output, tf.float32) + + def get_pooled_output(self): + return self.pooled_output + + def get_sequence_output(self): + """Gets final hidden layer of encoder. + + Returns: + float Tensor of shape [batch_size, seq_length, hidden_size] corresponding + to the final hidden of the transformer encoder. + """ + return self.sequence_output + + def get_all_encoder_layers(self): + return self.all_encoder_layers + + def get_embedding_output(self): + """Gets output of the embedding lookup (i.e., input to the transformer). + + Returns: + float Tensor of shape [batch_size, seq_length, hidden_size] corresponding + to the output of the embedding layer, after summing the word + embeddings with the positional embeddings and the token type embeddings, + then performing layer normalization. This is the input to the transformer. + """ + return self.embedding_output + + def get_embedding_table(self): + return self.embedding_table + + +def gelu(x): + """Gaussian Error Linear Unit. + + This is a smoother version of the RELU. + Original paper: https://arxiv.org/abs/1606.08415 + Args: + x: float Tensor to perform activation. + + Returns: + `x` with the GELU activation applied. + """ + + if tf.flags.FLAGS.npu_bert_fused_gelu: + return npu_unary_ops.gelu(x) + else: + cdf = 0.5 * (1.0 + tf.tanh( + (np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))) + return x * cdf + + + +def get_activation(activation_string): + """Maps a string to a Python function, e.g., "relu" => `tf.nn.relu`. + + Args: + activation_string: String name of the activation function. + + Returns: + A Python function corresponding to the activation function. If + `activation_string` is None, empty, or "linear", this will return None. + If `activation_string` is not a string, it will return `activation_string`. + + Raises: + ValueError: The `activation_string` does not correspond to a known + activation. + """ + + # We assume that anything that"s not a string is already an activation + # function, so we just return it. + if not isinstance(activation_string, six.string_types): + return activation_string + + if not activation_string: + return None + + act = activation_string.lower() + if act == "linear": + return None + elif act == "relu": + return tf.nn.relu + elif act == "gelu": + return gelu + elif act == "tanh": + return tf.tanh + else: + raise ValueError("Unsupported activation: %s" % act) + + +def get_assignment_map_from_checkpoint(tvars, init_checkpoint): + """Compute the union of the current variables and checkpoint variables.""" + assignment_map = {} + initialized_variable_names = {} + + name_to_variable = collections.OrderedDict() + for var in tvars: + name = var.name + m = re.match("^(.*):\\d+$", name) + if m is not None: + name = m.group(1) + name_to_variable[name] = var + + init_vars = tf.train.list_variables(init_checkpoint) + + assignment_map = collections.OrderedDict() + for x in init_vars: + (name, var) = (x[0], x[1]) + if name not in name_to_variable: + continue + assignment_map[name] = name + initialized_variable_names[name] = 1 + initialized_variable_names[name + ":0"] = 1 + + return (assignment_map, initialized_variable_names) + + +def dropout(input_tensor, dropout_prob): + """Perform dropout. + + Args: + input_tensor: float Tensor. + dropout_prob: Python float. The probability of dropping out a value (NOT of + *keeping* a dimension as in `tf.nn.dropout`). + + Returns: + A version of `input_tensor` with dropout applied. + """ + + if tf.flags.FLAGS.npu_bert_debug: + return input_tensor + + if dropout_prob is None or dropout_prob == 0.0: + return input_tensor + + if tf.flags.FLAGS.npu_bert_npu_dropout: + output = npu_ops.dropout(input_tensor, 1.0 - dropout_prob) + else: + output = tf.nn.dropout(input_tensor, 1.0 - dropout_prob) + return output + + +def layer_norm(input_tensor, name=None): + """Run layer normalization on the last dimension of the tensor.""" + if input_tensor.dtype == tf.float16: + try: + from fused_layer_norm import fused_layer_norm + return fused_layer_norm( + inputs=input_tensor, begin_norm_axis=-1, begin_params_axis=-1, scope=name, + use_fused_batch_norm=True) + except ImportError: + return tf.contrib.layers.layer_norm( + inputs=input_tensor, begin_norm_axis=-1, begin_params_axis=-1, scope=name) + else: + return tf.contrib.layers.layer_norm( + inputs=input_tensor, begin_norm_axis=-1, begin_params_axis=-1, scope=name) + + +def layer_norm_and_dropout(input_tensor, dropout_prob, name=None): + """Runs layer normalization followed by dropout.""" + output_tensor = layer_norm(input_tensor, name) + output_tensor = dropout(output_tensor, dropout_prob) + return output_tensor + + +def create_initializer(initializer_range=0.02): + """Creates a `truncated_normal_initializer` with the given range.""" + return tf.truncated_normal_initializer(stddev=initializer_range) + + +def embedding_lookup(input_ids, + vocab_size, + embedding_size=128, + initializer_range=0.02, + word_embedding_name="word_embeddings", + use_one_hot_embeddings=False): + """Looks up words embeddings for id tensor. + + Args: + input_ids: int32 Tensor of shape [batch_size, seq_length] containing word + ids. + vocab_size: int. Size of the embedding vocabulary. + embedding_size: int. Width of the word embeddings. + initializer_range: float. Embedding initialization range. + word_embedding_name: string. Name of the embedding table. + use_one_hot_embeddings: bool. If True, use one-hot method for word + embeddings. If False, use `tf.gather()`. + + Returns: + float Tensor of shape [batch_size, seq_length, embedding_size]. + """ + # This function assumes that the input is of shape [batch_size, seq_length, + # num_inputs]. + # + # If the input is a 2D tensor of shape [batch_size, seq_length], we + # reshape to [batch_size, seq_length, 1]. + if input_ids.shape.ndims == 2: + input_ids = tf.expand_dims(input_ids, axis=[-1]) + + embedding_table = tf.get_variable( + name=word_embedding_name, + shape=[vocab_size, embedding_size], + initializer=create_initializer(initializer_range)) + + flat_input_ids = tf.reshape(input_ids, [-1]) + if use_one_hot_embeddings: + one_hot_input_ids = tf.one_hot(flat_input_ids, depth=vocab_size) + output = tf.matmul(one_hot_input_ids, embedding_table) + else: + if tf.flags.FLAGS.npu_gather: + output = gather_npu(embedding_table, flat_input_ids) + else: + output = tf.gather(embedding_table, flat_input_ids) + + input_shape = get_shape_list(input_ids) + + output = tf.reshape(output, + input_shape[0:-1] + [input_shape[-1] * embedding_size]) + return (output, embedding_table) + + +def embedding_postprocessor(input_tensor, + use_token_type=False, + token_type_ids=None, + token_type_vocab_size=16, + token_type_embedding_name="token_type_embeddings", + use_position_embeddings=True, + position_embedding_name="position_embeddings", + initializer_range=0.02, + max_position_embeddings=512, + dropout_prob=0.1, + use_one_hot_embeddings=False): + """Performs various post-processing on a word embedding tensor. + + Args: + input_tensor: float Tensor of shape [batch_size, seq_length, + embedding_size]. + use_token_type: bool. Whether to add embeddings for `token_type_ids`. + token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length]. + Must be specified if `use_token_type` is True. + token_type_vocab_size: int. The vocabulary size of `token_type_ids`. + token_type_embedding_name: string. The name of the embedding table variable + for token type ids. + use_position_embeddings: bool. Whether to add position embeddings for the + position of each token in the sequence. + position_embedding_name: string. The name of the embedding table variable + for positional embeddings. + initializer_range: float. Range of the weight initialization. + max_position_embeddings: int. Maximum sequence length that might ever be + used with this model. This can be longer than the sequence length of + input_tensor, but cannot be shorter. + dropout_prob: float. Dropout probability applied to the final output tensor. + use_one_hot_embeddings: (optional) bool. Whether to use one-hot word + embeddings or tf.embedding_lookup() for the word embeddings. + + Returns: + float tensor with same shape as `input_tensor`. + + Raises: + ValueError: One of the tensor shapes or input values is invalid. + """ + input_shape = get_shape_list(input_tensor, expected_rank=3) + batch_size = input_shape[0] + seq_length = input_shape[1] + width = input_shape[2] + + output = input_tensor + + if use_token_type: + if token_type_ids is None: + raise ValueError("`token_type_ids` must be specified if" + "`use_token_type` is True.") + token_type_table = tf.get_variable( + name=token_type_embedding_name, + shape=[token_type_vocab_size, width], + initializer=create_initializer(initializer_range)) + flat_token_type_ids = tf.reshape(token_type_ids, [-1]) + if use_one_hot_embeddings: + # This vocab will be small so we always do one-hot here, since it is + # always faster for a small vocabulary. + one_hot_ids = tf.one_hot(flat_token_type_ids, depth=token_type_vocab_size) + token_type_embeddings = tf.matmul(one_hot_ids, token_type_table) + else: + if tf.flags.FLAGS.npu_gather: + token_type_embeddings = gather_npu(token_type_table, flat_token_type_ids) + else: + token_type_embeddings = tf.gather(token_type_table, flat_token_type_ids) + token_type_embeddings = tf.reshape(token_type_embeddings, + [batch_size, seq_length, width]) + output += token_type_embeddings + + if use_position_embeddings: + full_position_embeddings = tf.get_variable( + name=position_embedding_name, + shape=[max_position_embeddings, width], + initializer=create_initializer(initializer_range)) + # Since the position embedding table is a learned variable, we create it + # using a (long) sequence length `max_position_embeddings`. The actual + # sequence length might be shorter than this, for faster training of + # tasks that do not have long sequences. + # + # So `full_position_embeddings` is effectively an embedding table + # for position [0, 1, 2, ..., max_position_embeddings-1], and the current + # sequence has positions [0, 1, 2, ... seq_length-1], so we can just + # perform a slice. + position_embeddings = tf.slice(full_position_embeddings, [0, 0], + [seq_length, width]) + num_dims = len(output.shape.as_list()) + + # Only the last two dimensions are relevant (`seq_length` and `width`), so + # we broadcast among the first dimensions, which is typically just + # the batch size. + position_broadcast_shape = [] + for _ in range(num_dims - 2): + position_broadcast_shape.append(1) + position_broadcast_shape.extend([seq_length, width]) + position_embeddings = tf.reshape(position_embeddings, + position_broadcast_shape) + output += position_embeddings + + output = layer_norm_and_dropout(output, dropout_prob) + return output + + +def create_attention_mask_from_input_mask(from_tensor, to_mask): + """Create 3D attention mask from a 2D tensor mask. + + Args: + from_tensor: 2D or 3D Tensor of shape [batch_size, from_seq_length, ...]. + to_mask: int32 Tensor of shape [batch_size, to_seq_length]. + + Returns: + float Tensor of shape [batch_size, from_seq_length, to_seq_length]. + """ + to_mask = tf.cast(to_mask, dtype=tf.float32) + + from_shape = get_shape_list(from_tensor, expected_rank=[2, 3]) + batch_size = from_shape[0] + + to_shape = get_shape_list(to_mask, expected_rank=2) + to_seq_length = to_shape[1] + + to_mask = tf.reshape(to_mask, [batch_size, 1, to_seq_length]) + # The mask will be automatically broadcasted to + # [batch_size, from_seq_length, to_seq_length] when it is used in the + # attention layer. + return to_mask + +def attention_layer(from_tensor, + to_tensor, + attention_mask=None, + num_attention_heads=1, + size_per_head=512, + query_act=None, + key_act=None, + value_act=None, + attention_probs_dropout_prob=0.0, + initializer_range=0.02, + do_return_2d_tensor=False, + batch_size=None, + from_seq_length=None, + to_seq_length=None): + """Performs multi-headed attention from `from_tensor` to `to_tensor`. + + This is an implementation of multi-headed attention based on "Attention + is all you Need". If `from_tensor` and `to_tensor` are the same, then + this is self-attention. Each timestep in `from_tensor` attends to the + corresponding sequence in `to_tensor`, and returns a fixed-with vector. + + This function first projects `from_tensor` into a "query" tensor and + `to_tensor` into "key" and "value" tensors. These are (effectively) a list + of tensors of length `num_attention_heads`, where each tensor is of shape + [batch_size, seq_length, size_per_head]. + + Then, the query and key tensors are dot-producted and scaled. These are + softmaxed to obtain attention probabilities. The value tensors are then + interpolated by these probabilities, then concatenated back to a single + tensor and returned. + + In practice, the multi-headed attention are done with transposes and + reshapes rather than actual separate tensors. + + Args: + from_tensor: float Tensor of shape [batch_size, from_seq_length, + from_width]. + to_tensor: float Tensor of shape [batch_size, to_seq_length, to_width]. + attention_mask: (optional) int32 Tensor of shape [batch_size, + from_seq_length, to_seq_length]. The values should be 1 or 0. The + attention scores will effectively be set to -infinity for any positions in + the mask that are 0, and will be unchanged for positions that are 1. + num_attention_heads: int. Number of attention heads. + size_per_head: int. Size of each attention head. + query_act: (optional) Activation function for the query transform. + key_act: (optional) Activation function for the key transform. + value_act: (optional) Activation function for the value transform. + attention_probs_dropout_prob: (optional) float. Dropout probability of the + attention probabilities. + initializer_range: float. Range of the weight initializer. + do_return_2d_tensor: bool. If True, the output will be of shape [batch_size + * from_seq_length, num_attention_heads * size_per_head]. If False, the + output will be of shape [batch_size, from_seq_length, num_attention_heads + * size_per_head]. + batch_size: (Optional) int. If the input is 2D, this might be the batch size + of the 3D version of the `from_tensor` and `to_tensor`. + from_seq_length: (Optional) If the input is 2D, this might be the seq length + of the 3D version of the `from_tensor`. + to_seq_length: (Optional) If the input is 2D, this might be the seq length + of the 3D version of the `to_tensor`. + + Returns: + float Tensor of shape [batch_size, from_seq_length, + num_attention_heads * size_per_head]. (If `do_return_2d_tensor` is + true, this will be of shape [batch_size * from_seq_length, + num_attention_heads * size_per_head]). + + Raises: + ValueError: Any of the arguments or tensor shapes are invalid. + """ + + def transpose_for_scores(input_tensor, batch_size, num_attention_heads, + seq_length, width): + output_tensor = tf.reshape( + input_tensor, [batch_size, seq_length, num_attention_heads, width]) + + output_tensor = tf.transpose(output_tensor, [0, 2, 1, 3]) + return output_tensor + + from_shape = get_shape_list(from_tensor, expected_rank=[2, 3]) + to_shape = get_shape_list(to_tensor, expected_rank=[2, 3]) + + if len(from_shape) != len(to_shape): + raise ValueError( + "The rank of `from_tensor` must match the rank of `to_tensor`.") + + if len(from_shape) == 3: + batch_size = from_shape[0] + from_seq_length = from_shape[1] + to_seq_length = to_shape[1] + elif len(from_shape) == 2: + if (batch_size is None or from_seq_length is None or to_seq_length is None): + raise ValueError( + "When passing in rank 2 tensors to attention_layer, the values " + "for `batch_size`, `from_seq_length`, and `to_seq_length` " + "must all be specified.") + + # Scalar dimensions referenced here: + # B = batch size (number of sequences) + # F = `from_tensor` sequence length + # T = `to_tensor` sequence length + # N = `num_attention_heads` + # H = `size_per_head` + + from_tensor_2d = reshape_to_matrix(from_tensor) + to_tensor_2d = reshape_to_matrix(to_tensor) + + # `query_layer` = [B*F, N*H] + query_layer = tf.layers.dense( + from_tensor_2d, + num_attention_heads * size_per_head, + activation=query_act, + name="query", + kernel_initializer=create_initializer(initializer_range)) + + # `key_layer` = [B*T, N*H] + key_layer = tf.layers.dense( + to_tensor_2d, + num_attention_heads * size_per_head, + activation=key_act, + name="key", + kernel_initializer=create_initializer(initializer_range)) + + # `value_layer` = [B*T, N*H] + value_layer = tf.layers.dense( + to_tensor_2d, + num_attention_heads * size_per_head, + activation=value_act, + name="value", + kernel_initializer=create_initializer(initializer_range)) + + # `query_layer` = [B, N, F, H] + query_layer = transpose_for_scores(query_layer, batch_size, + num_attention_heads, from_seq_length, + size_per_head) + + # `key_layer` = [B, N, T, H] + key_layer = transpose_for_scores(key_layer, batch_size, num_attention_heads, + to_seq_length, size_per_head) + + # Take the dot product between "query" and "key" to get the raw + # attention scores. + # `attention_scores` = [B, N, F, T] + attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) + attention_scores = tf.multiply(attention_scores, + 1.0 / math.sqrt(float(size_per_head))) + + if attention_mask is not None: + # `attention_mask` = [B, 1, F, T] + attention_mask = tf.expand_dims(attention_mask, axis=[1]) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + adder = (1.0 - tf.cast(attention_mask, attention_scores.dtype)) * -10000.0 + + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + attention_scores += adder + + # Normalize the attention scores to probabilities. + # `attention_probs` = [B, N, F, T] + attention_probs = tf.nn.softmax(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = dropout(attention_probs, attention_probs_dropout_prob) + + # `value_layer` = [B, T, N, H] + value_layer = tf.reshape( + value_layer, + [batch_size, to_seq_length, num_attention_heads, size_per_head]) + + # `value_layer` = [B, N, T, H] + value_layer = tf.transpose(value_layer, [0, 2, 1, 3]) + + # `context_layer` = [B, N, F, H] + context_layer = tf.matmul(attention_probs, value_layer) + + # `context_layer` = [B, F, N, H] + context_layer = tf.transpose(context_layer, [0, 2, 1, 3]) + + if do_return_2d_tensor: + # `context_layer` = [B*F, N*H] + context_layer = tf.reshape( + context_layer, + [batch_size * from_seq_length, num_attention_heads * size_per_head]) + else: + # `context_layer` = [B, F, N*H] + context_layer = tf.reshape( + context_layer, + [batch_size, from_seq_length, num_attention_heads * size_per_head]) + + return context_layer + + +def transformer_model(input_tensor, + attention_mask=None, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + intermediate_act_fn=gelu, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + initializer_range=0.02, + do_return_all_layers=False): + """Multi-headed, multi-layer Transformer from "Attention is All You Need". + + This is almost an exact implementation of the original Transformer encoder. + + See the original paper: + https://arxiv.org/abs/1706.03762 + + Also see: + https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/models/transformer.py + + Args: + input_tensor: float Tensor of shape [batch_size, seq_length, hidden_size]. + attention_mask: (optional) int32 Tensor of shape [batch_size, seq_length, + seq_length], with 1 for positions that can be attended to and 0 in + positions that should not be. + hidden_size: int. Hidden size of the Transformer. + num_hidden_layers: int. Number of layers (blocks) in the Transformer. + num_attention_heads: int. Number of attention heads in the Transformer. + intermediate_size: int. The size of the "intermediate" (a.k.a., feed + forward) layer. + intermediate_act_fn: function. The non-linear activation function to apply + to the output of the intermediate/feed-forward layer. + hidden_dropout_prob: float. Dropout probability for the hidden layers. + attention_probs_dropout_prob: float. Dropout probability of the attention + probabilities. + initializer_range: float. Range of the initializer (stddev of truncated + normal). + do_return_all_layers: Whether to also return all layers or just the final + layer. + + Returns: + float Tensor of shape [batch_size, seq_length, hidden_size], the final + hidden layer of the Transformer. + + Raises: + ValueError: A Tensor shape or parameter is invalid. + """ + if hidden_size % num_attention_heads != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (hidden_size, num_attention_heads)) + + attention_head_size = int(hidden_size / num_attention_heads) + input_shape = get_shape_list(input_tensor, expected_rank=3) + batch_size = input_shape[0] + seq_length = input_shape[1] + input_width = input_shape[2] + + # The Transformer performs sum residuals on all layers so the input needs + # to be the same as the hidden size. + if input_width != hidden_size: + raise ValueError("The width of the input tensor (%d) != hidden size (%d)" % + (input_width, hidden_size)) + + # We keep the representation as a 2D tensor to avoid re-shaping it back and + # forth from a 3D tensor to a 2D tensor. Re-shapes are normally free on + # the GPU/CPU but may not be free on the TPU, so we want to minimize them to + # help the optimizer. + prev_output = reshape_to_matrix(input_tensor) + + all_layer_outputs = [] + for layer_idx in range(num_hidden_layers): + with tf.variable_scope("layer_%d" % layer_idx): + layer_input = prev_output + + with tf.variable_scope("attention"): + attention_heads = [] + with tf.variable_scope("self"): + attention_head = attention_layer( + from_tensor=layer_input, + to_tensor=layer_input, + attention_mask=attention_mask, + num_attention_heads=num_attention_heads, + size_per_head=attention_head_size, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + do_return_2d_tensor=True, + batch_size=batch_size, + from_seq_length=seq_length, + to_seq_length=seq_length) + attention_heads.append(attention_head) + + attention_output = None + if len(attention_heads) == 1: + attention_output = attention_heads[0] + else: + # In the case where we have other sequences, we just concatenate + # them to the self-attention head before the projection. + attention_output = tf.concat(attention_heads, axis=-1) + + # Run a linear projection of `hidden_size` then add a residual + # with `layer_input`. + with tf.variable_scope("output"): + attention_output = tf.layers.dense( + attention_output, + hidden_size, + kernel_initializer=create_initializer(initializer_range)) + attention_output = dropout(attention_output, hidden_dropout_prob) + attention_output = layer_norm(attention_output + layer_input) + + # The activation is only applied to the "intermediate" hidden layer. + with tf.variable_scope("intermediate"): + intermediate_output = tf.layers.dense( + attention_output, + intermediate_size, + activation=intermediate_act_fn, + kernel_initializer=create_initializer(initializer_range)) + + # Down-project back to `hidden_size` then add the residual. + with tf.variable_scope("output"): + layer_output = tf.layers.dense( + intermediate_output, + hidden_size, + kernel_initializer=create_initializer(initializer_range)) + layer_output = dropout(layer_output, hidden_dropout_prob) + layer_output = layer_norm(layer_output + attention_output) + prev_output = layer_output + all_layer_outputs.append(layer_output) + + if do_return_all_layers: + final_outputs = [] + for layer_output in all_layer_outputs: + final_output = reshape_from_matrix(layer_output, input_shape) + final_outputs.append(final_output) + return final_outputs + else: + final_output = reshape_from_matrix(prev_output, input_shape) + return final_output + + +def get_shape_list(tensor, expected_rank=None, name=None): + """Returns a list of the shape of tensor, preferring static dimensions. + + Args: + tensor: A tf.Tensor object to find the shape of. + expected_rank: (optional) int. The expected rank of `tensor`. If this is + specified and the `tensor` has a different rank, and exception will be + thrown. + name: Optional name of the tensor for the error message. + + Returns: + A list of dimensions of the shape of tensor. All static dimensions will + be returned as python integers, and dynamic dimensions will be returned + as tf.Tensor scalars. + """ + if name is None: + name = tensor.name + + if expected_rank is not None: + assert_rank(tensor, expected_rank, name) + + shape = tensor.shape.as_list() + + non_static_indexes = [] + for (index, dim) in enumerate(shape): + if dim is None: + non_static_indexes.append(index) + + if not non_static_indexes: + return shape + + dyn_shape = tf.shape(tensor) + for index in non_static_indexes: + shape[index] = dyn_shape[index] + return shape + + +def reshape_to_matrix(input_tensor): + """Reshapes a >= rank 2 tensor to a rank 2 tensor (i.e., a matrix).""" + ndims = input_tensor.shape.ndims + if ndims < 2: + raise ValueError("Input tensor must have at least rank 2. Shape = %s" % + (input_tensor.shape)) + if ndims == 2: + return input_tensor + + width = input_tensor.shape[-1] + output_tensor = tf.reshape(input_tensor, [-1, width]) + return output_tensor + + +def reshape_from_matrix(output_tensor, orig_shape_list): + """Reshapes a rank 2 tensor back to its original rank >= 2 tensor.""" + if len(orig_shape_list) == 2: + return output_tensor + + output_shape = get_shape_list(output_tensor) + + orig_dims = orig_shape_list[0:-1] + width = output_shape[-1] + + return tf.reshape(output_tensor, orig_dims + [width]) + + +def assert_rank(tensor, expected_rank, name=None): + """Raises an exception if the tensor rank is not of the expected rank. + + Args: + tensor: A tf.Tensor to check the rank of. + expected_rank: Python integer or list of integers, expected rank. + name: Optional name of the tensor for the error message. + + Raises: + ValueError: If the expected shape doesn't match the actual shape. + """ + if name is None: + name = tensor.name + + expected_rank_dict = {} + if isinstance(expected_rank, six.integer_types): + expected_rank_dict[expected_rank] = True + else: + for x in expected_rank: + expected_rank_dict[x] = True + + actual_rank = tensor.shape.ndims + if actual_rank not in expected_rank_dict: + scope_name = tf.get_variable_scope().name + raise ValueError( + "For the tensor `%s` in scope `%s`, the actual rank " + "`%d` (shape = %s) is not equal to the expected rank `%s`" % + (name, scope_name, actual_rank, str(tensor.shape), str(expected_rank))) + +@tf.custom_gradient +def gather_npu(params, indices): + def grad(dy): + params_shape = tf.shape(params, out_type=tf.int64) + params_shape = tf.cast(params_shape, tf.int32) + grad_gather = tf.unsorted_segment_sum(dy, indices, params_shape[0]) + return grad_gather, None + return tf.gather(params, indices), grad + diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/optimization.py b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/optimization.py new file mode 100644 index 0000000..d4a67ce --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/optimization.py @@ -0,0 +1,439 @@ +# coding=utf-8 +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Functions and classes related to optimization (weight updates).""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import re +import tensorflow as tf +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import linalg_ops +from tensorflow.python.ops import math_ops + +from npu_bridge.estimator.npu.npu_optimizer import NPUDistributedOptimizer +from npu_bridge.estimator.npu import npu_loss_scale_optimizer as lso +from npu_bridge.estimator.npu import npu_loss_scale_manager as lsm_lib + +def create_optimizer(loss, init_lr, num_train_steps, num_warmup_steps, hvd=None, manual_fp16=False, use_fp16=False, num_accumulation_steps=1, + optimizer_type="adam", allreduce_post_accumulation=False): + """Creates an optimizer training op.""" + global_step = tf.train.get_or_create_global_step() + + # avoid step change in learning rate at end of warmup phase + if optimizer_type == "adam": + power = 1.0 + decayed_learning_rate_at_crossover_point = init_lr * ( + (1.0 - float(num_warmup_steps) / float(num_train_steps)) ** power) + else: + power = 0.5 + decayed_learning_rate_at_crossover_point = init_lr + + adjusted_init_lr = init_lr * (init_lr / decayed_learning_rate_at_crossover_point) + print('decayed_learning_rate_at_crossover_point = %e, adjusted_init_lr = %e' % (decayed_learning_rate_at_crossover_point, adjusted_init_lr)) + + learning_rate = tf.constant(value=init_lr, shape=[], dtype=tf.float32) + + # Implements linear decay of the learning rate. + learning_rate = tf.train.polynomial_decay( + learning_rate, + global_step, + num_train_steps, + end_learning_rate=0.0, + power=power, + cycle=False) + + # Implements linear warmup. I.e., if global_step < num_warmup_steps, the + # learning rate will be `global_step/num_warmup_steps * init_lr`. + if num_warmup_steps: + global_steps_int = tf.cast(global_step, tf.int32) + warmup_steps_int = tf.constant(num_warmup_steps, dtype=tf.int32) + + global_steps_float = tf.cast(global_steps_int, tf.float32) + warmup_steps_float = tf.cast(warmup_steps_int, tf.float32) + + warmup_percent_done = global_steps_float / warmup_steps_float + warmup_learning_rate = init_lr * warmup_percent_done + + is_warmup = tf.cast(global_steps_int < warmup_steps_int, tf.float32) + learning_rate = ( + (1.0 - is_warmup) * learning_rate + is_warmup * warmup_learning_rate) + + if optimizer_type == "lamb": + print("Initializing LAMB Optimizer") + optimizer = LAMBOptimizer( + learning_rate=learning_rate, + weight_decay_rate=0.01, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-6, + exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"]) + else: + print("Initializing ADAM Weight Decay Optimizer") + # It is recommended that you use this optimizer for fine tuning, since this + # is how the model was trained (note that the Adam m/v variables are NOT + # loaded from init_checkpoint.) + optimizer = AdamWeightDecayOptimizer( + learning_rate=learning_rate, + weight_decay_rate=0.01, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-4, + exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"]) + + if hvd is not None and (num_accumulation_steps == 1 or (not allreduce_post_accumulation)): + optimizer = hvd.DistributedOptimizer(optimizer, sparse_as_dense=True, compression=Compression.fp16 if use_fp16 or manual_fp16 else Compression.none) + + optimizer = NPUDistributedOptimizer(optimizer) + if tf.flags.FLAGS.npu_bert_loss_scale not in [None, -1]: + opt_tmp = optimizer + if tf.flags.FLAGS.npu_bert_loss_scale == 0: + loss_scale_manager = lsm_lib.ExponentialUpdateLossScaleManager(init_loss_scale=tf.flags.FLAGS.init_loss_scale_value, incr_every_n_steps=1000, decr_every_n_nan_or_inf=2, decr_ratio=0.5) + elif tf.flags.FLAGS.npu_bert_loss_scale >= 1: + loss_scale_manager = lsm_lib.FixedLossScaleManager(loss_scale=tf.flags.FLAGS.npu_bert_loss_scale) + else: + raise ValueError("Invalid loss scale: %d" % tf.flags.FLAGS.npu_bert_loss_scale) + optimizer = lso.NPULossScaleOptimizer(opt_tmp, loss_scale_manager, is_distributed=tf.flags.FLAGS.distributed) + + tvars = tf.trainable_variables() + grads_and_vars = optimizer.compute_gradients(loss * 1.0 / num_accumulation_steps, tvars) + + if num_accumulation_steps > 1: + local_step = tf.get_variable(name="local_step", shape=[], dtype=tf.int32, trainable=False, + initializer=tf.zeros_initializer) + batch_finite = tf.get_variable(name="batch_finite", shape=[], dtype=tf.bool, trainable=False, + initializer=tf.ones_initializer) + accum_vars = [tf.get_variable( + name=tvar.name.split(":")[0] + "/accum", + shape=tvar.shape.as_list(), + dtype=tf.float32, + trainable=False, + initializer=tf.zeros_initializer()) for tvar in tf.trainable_variables()] + + reset_step = tf.cast(tf.math.equal(local_step % num_accumulation_steps, 0), dtype=tf.bool) + local_step = tf.cond(reset_step, lambda:local_step.assign(tf.ones_like(local_step)), lambda:local_step.assign_add(1)) + + with tf.name_scope(accumulate_step): + grads_and_vars_and_accums = [(gv[0],gv[1],accum_vars[i]) for i, gv in enumerate(grads_and_vars) if gv[0] is not None] + grads, tvars, accum_vars = list(zip(*grads_and_vars_and_accums)) + + all_are_finite = tf.reduce_all([tf.reduce_all(tf.is_finite(g)) for g in grads]) if (tf.flags.FLAGS.npu_bert_loss_scale not in [None, -1]) and (manual_fp16 or use_fp16) else tf.constant(True, dtype=tf.bool) + batch_finite = tf.cond(reset_step, + lambda: batch_finite.assign(tf.math.logical_and(tf.constant(True, dtype=tf.bool), all_are_finite)), + lambda:batch_finite.assign(tf.math.logical_and(batch_finite, all_are_finite))) + + # This is how the model was pre-trained. + # ensure global norm is a finite number + # to prevent clip_by_global_norm from having a hizzy fit. + if tf.flags.FLAGS.npu_bert_clip_by_global_norm: + (clipped_grads, _) = tf.clip_by_global_norm( + grads, clip_norm=1.0, + use_norm=tf.cond( + all_are_finite, + lambda: tf.global_norm(grads), + lambda: tf.constant(1.0))) + else: + with tf.name_scope("clip_grads"): + clipped_grads = [ + (tf.clip_by_norm(grad, clip_norm=1.0)) + if grad is not None else (grad, var) for grad in grads + ] + + accum_vars = tf.cond(reset_step, + lambda: [accum_vars[i].assign(grad) for i, grad in enumerate(clipped_grads)], + lambda: [accum_vars[i].assign_add(grad) for i, grad in enumerate(clipped_grads)]) + + def update(accum_vars): + with tf.name_scope("opt_update"): + if allreduce_post_accumulation and hvd is not None: + accum_vars = [hvd.allreduce(tf.convert_to_tensor(accum_var), compression=Compression.fp16 if use_fp16 or manual_fp16 else Compression.none) if isinstance(accum_var, tf.IndexedSlices) + else hvd.allreduce(accum_var, compression=Compression.fp16 if use_fp16 or manual_fp16 else Compression.none) for accum_var in accum_vars] + return optimizer.apply_gradients(list(zip(accum_vars, tvars)), global_step=global_step) + + update_step = tf.identity(tf.cast(tf.math.equal(local_step % num_accumulation_steps, 0), dtype=tf.bool), name="update_step") + update_op = tf.cond(update_step, + lambda: update(accum_vars), lambda: tf.no_op()) + + new_global_step = tf.cond(tf.math.logical_and(update_step, tf.cast(hvd.allreduce(tf.cast(batch_finite, tf.int32)), tf.bool)), lambda: global_step+1, lambda: global_step) + new_global_step = tf.identity(new_global_step, name='step_update') + train_op = tf.group(update_op, [global_step.assign(new_global_step)]) + else: + grads_and_vars = [(g, v) for g, v in grads_and_vars if g is not None] + grads, tvars = list(zip(*grads_and_vars)) + + if tf.flags.FLAGS.npu_bert_clip_by_global_norm: + all_are_finite = tf.reduce_all( + [tf.reduce_all(tf.is_finite(g)) for g in grads]) if (tf.flags.FLAGS.npu_bert_loss_scale not in [None, -1]) and (use_fp16 or manual_fp16) else tf.constant(True, dtype=tf.bool) + + # This is how the model was pre-trained. + # ensure global norm is a finite number + # to prevent clip_by_global_norm from having a hizzy fit. + if tf.flags.FLAGS.npu_bert_clip_by_global_norm: + (clipped_grads, _) = tf.clip_by_global_norm( + grads, clip_norm=1.0, + use_norm=tf.cond( + all_are_finite, + lambda: tf.global_norm(grads), + lambda: tf.constant(1.0))) + else: + with tf.name_scope("clip_grads"): + clipped_grads = [ + (tf.clip_by_norm(grad, clip_norm=1.0)) + if grad is not None else (grad, var) for grad in grads + ] + + with tf.name_scope("apply_grads"): + train_op = optimizer.apply_gradients( + list(zip(clipped_grads, tvars)), global_step=global_step) + + #if tf.flags.FLAGS.npu_bert_clip_by_global_norm: + # new_global_step = tf.cond(all_are_finite, lambda: global_step + 1, lambda: global_step) + #else: + # new_global_step = global_step + 1 + #new_global_step = tf.identity(new_global_step, name='step_update') + #train_op = tf.group(train_op, [global_step.assign(new_global_step)]) + return train_op + + +class AdamWeightDecayOptimizer(tf.train.Optimizer): + """A basic Adam optimizer that includes "correct" L2 weight decay.""" + + def __init__(self, + learning_rate, + weight_decay_rate=0.0, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-4, + exclude_from_weight_decay=None, + name="AdamWeightDecayOptimizer"): + """Constructs a AdamWeightDecayOptimizer.""" + super(AdamWeightDecayOptimizer, self).__init__(False, name) + + self.learning_rate = tf.identity(learning_rate, name='learning_rate') + self.weight_decay_rate = weight_decay_rate + self.beta_1 = beta_1 + self.beta_2 = beta_2 + self.epsilon = epsilon + self.exclude_from_weight_decay = exclude_from_weight_decay + + def apply_gradients(self, grads_and_vars, global_step=None, name=None, + manual_fp16=False): + """See base class.""" + assignments = [] + for (grad, param) in grads_and_vars: + with tf.name_scope("apply_one_adam"): + if grad is None or param is None: + continue + + param_name = self._get_variable_name(param.name) + has_shadow = manual_fp16 and param.dtype.base_dtype != tf.float32 + if has_shadow: + # create shadow fp32 weights for fp16 variable + param_fp32 = tf.get_variable( + name=param_name + "/shadow", + dtype=tf.float32, + trainable=False, + initializer=tf.cast(param.initialized_value(),tf.float32)) + else: + param_fp32 = param + + m = tf.get_variable( + name=param_name + "/adam_m", + shape=param.shape.as_list(), + dtype=tf.float32, + trainable=False, + initializer=tf.zeros_initializer()) + v = tf.get_variable( + name=param_name + "/adam_v", + shape=param.shape.as_list(), + dtype=tf.float32, + trainable=False, + initializer=tf.zeros_initializer()) + + # Standard Adam update. + next_m = ( + tf.multiply(self.beta_1, m) + tf.multiply(1.0 - self.beta_1, grad)) + next_v = ( + tf.multiply(self.beta_2, v) + tf.multiply(1.0 - self.beta_2, + tf.square(grad))) + + update = next_m / (tf.sqrt(next_v) + self.epsilon) + + # Just adding the square of the weights to the loss function is *not* + # the correct way of using L2 regularization/weight decay with Adam, + # since that will interact with the m and v parameters in strange ways. + # + # Instead we want to decay the weights in a manner that doesn't interact + # with the m/v parameters. This is equivalent to adding the square + # of the weights to the loss with plain (non-momentum) SGD. + if self._do_use_weight_decay(param_name): + update += self.weight_decay_rate * param_fp32 + + update_with_lr = self.learning_rate * update + + next_param = param_fp32 - update_with_lr + + if has_shadow: + # cast shadow fp32 weights to fp16 and assign to trainable variable + param.assign(tf.cast(next_param, param.dtype.base_dtype)) + assignments.extend( + [param_fp32.assign(next_param), + m.assign(next_m), + v.assign(next_v)]) + new_global_step = global_step + 1 + new_global_step = tf.identity(new_global_step, name='step_update') + assignments.extend([global_step.assign(new_global_step)]) + return tf.group(*assignments, name=name) + + def _do_use_weight_decay(self, param_name): + """Whether to use L2 weight decay for `param_name`.""" + if not self.weight_decay_rate: + return False + if self.exclude_from_weight_decay: + for r in self.exclude_from_weight_decay: + if re.search(r, param_name) is not None: + return False + return True + + def _get_variable_name(self, param_name): + """Get the variable name from the tensor name.""" + m = re.match("^(.*):\\d+$", param_name) + if m is not None: + param_name = m.group(1) + return param_name + + +class LAMBOptimizer(tf.train.Optimizer): + """A LAMB optimizer that includes "correct" L2 weight decay.""" + + def __init__(self, + learning_rate, + weight_decay_rate=0.0, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-6, + exclude_from_weight_decay=None, + name="LAMBOptimizer"): + """Constructs a LAMBOptimizer.""" + super(LAMBOptimizer, self).__init__(False, name) + + self.learning_rate = tf.identity(learning_rate, name='learning_rate') + self.weight_decay_rate = weight_decay_rate + self.beta_1 = beta_1 + self.beta_2 = beta_2 + self.epsilon = epsilon + self.exclude_from_weight_decay = exclude_from_weight_decay + self.steps = 0 + + def apply_gradients(self, grads_and_vars, global_step=None, name=None, + manual_fp16=False): + """See base class.""" + assignments = [] + for (grad, param) in grads_and_vars: + with tf.name_scope("apply_one_lamb"): + if grad is None or param is None: + continue + + param_name = self._get_variable_name(param.name) + has_shadow = manual_fp16 and param.dtype.base_dtype != tf.float32 + if has_shadow: + # create shadow fp32 weights for fp16 variable + param_fp32 = tf.get_variable( + name=param_name + "/shadow", + dtype=tf.float32, + trainable=False, + initializer=tf.cast(param.initialized_value(),tf.float32)) + else: + param_fp32 = param + + m = tf.get_variable( + name=param_name + "/adam_m", + shape=param.shape.as_list(), + dtype=tf.float32, + trainable=False, + initializer=tf.zeros_initializer()) + v = tf.get_variable( + name=param_name + "/adam_v", + shape=param.shape.as_list(), + dtype=tf.float32, + trainable=False, + initializer=tf.zeros_initializer()) + + # LAMB update + next_m = ( + tf.multiply(self.beta_1, m) + tf.multiply(1.0 - self.beta_1, grad)) + next_v = ( + tf.multiply(self.beta_2, v) + tf.multiply(1.0 - self.beta_2, + tf.square(grad))) + + self.steps += 1 + beta1_correction = (1 - self.beta_1 ** self.steps) + beta2_correction = (1 - self.beta_2 ** self.steps) + + next_m_unbiased = next_m / beta1_correction + next_v_unbiased = next_v / beta2_correction + + update = next_m_unbiased / (tf.sqrt(next_v_unbiased) + self.epsilon) + + # Just adding the square of the weights to the loss function is *not* + # the correct way of using L2 regularization/weight decay with Adam, + # since that will interact with the m and v parameters in strange ways. + # + # Instead we want to decay the weights in a manner that doesn't interact + # with the m/v parameters. This is equivalent to adding the square + # of the weights to the loss with plain (non-momentum) SGD. + if self._do_use_weight_decay(param_name): + update += self.weight_decay_rate * param_fp32 + + w_norm = linalg_ops.norm(param, ord=2) + g_norm = linalg_ops.norm(update, ord=2) + ratio = array_ops.where(math_ops.greater(w_norm, 0), array_ops.where( + math_ops.greater(g_norm, 0), (w_norm / g_norm), 1.0), 1.0) + + update_with_lr = ratio * self.learning_rate * update + + next_param = param_fp32 - update_with_lr + + if has_shadow: + # cast shadow fp32 weights to fp16 and assign to trainable variable + param.assign(tf.cast(next_param, param.dtype.base_dtype)) + assignments.extend( + [param_fp32.assign(next_param), + m.assign(next_m), + v.assign(next_v)]) + new_global_step = global_step + 1 + new_global_step = tf.identity(new_global_step, name='step_update') + assignments.extend([global_step.assign(new_global_step)]) + return tf.group(*assignments, name=name) + + def _do_use_weight_decay(self, param_name): + """Whether to use L2 weight decay for `param_name`.""" + if not self.weight_decay_rate: + return False + if self.exclude_from_weight_decay: + for r in self.exclude_from_weight_decay: + if re.search(r, param_name) is not None: + return False + return True + + def _get_variable_name(self, param_name): + """Get the variable name from the tensor name.""" + m = re.match("^(.*):\\d+$", param_name) + if m is not None: + param_name = m.group(1) + return param_name diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/run_pretraining.py b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/run_pretraining.py new file mode 100644 index 0000000..3d0c770 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/run_pretraining.py @@ -0,0 +1,784 @@ +# coding=utf-8 +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run masked LM/next sentence masked_lm pre-training for BERT.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import time +import modeling +import optimization +import tensorflow as tf +import glob +from utils import LogEvalRunHook +from tensorflow.core.protobuf import rewriter_config_pb2 +from gpu_environment import get_custom_getter + +from npu_bridge.estimator.npu.npu_config import * +from npu_bridge.estimator.npu.npu_estimator import * +from npu_bridge.estimator.npu.npu_config import NPURunConfig +from npu_bridge.estimator.npu.npu_estimator import NPUEstimator + +sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '../../../../../utils/atlasboost')) +# import hwlog +from benchmark_log import hwlog +from benchmark_log.basic_utils import get_environment_info +from benchmark_log.basic_utils import get_model_parameter +os.environ['WHICH_OP'] = 'GEOP' +os.environ['NEW_GE_FE_ID'] = '1' +os.environ['GE_AICPU_FLAG'] = '1' +os.environ['GE_USE_STATIC_MEMORY'] = '1' +os.environ['OPTION_EXEC_HCCL_FLAG'] = '1' +os.environ['HCCL_CONNECT_TIMEOUT'] = '600' + +flags = tf.flags + +FLAGS = flags.FLAGS + +## Required parameters +flags.DEFINE_string( + "bert_config_file", None, + "The config json file corresponding to the pre-trained BERT model. " + "This specifies the model architecture.") + +flags.DEFINE_string( + "input_files_dir", "./data", + "Directory with input files, comma separated or single directory.") + +flags.DEFINE_string( + "eval_files_dir", None, + "Directory with eval files, comma separated or single directory. ") + +flags.DEFINE_string( + "output_dir", "./models", + "The output directory where the model checkpoints will be written.") + +## Other parameters +flags.DEFINE_string( + "init_checkpoint", None, + "Initial checkpoint (usually from a pre-trained BERT model).") + +flags.DEFINE_string( + "optimizer_type", "lamb", + "Optimizer used for training - LAMB or ADAM") + +flags.DEFINE_integer( + "max_seq_length", 128, + "The maximum total input sequence length after WordPiece tokenization. " + "Sequences longer than this will be truncated, and sequences shorter " + "than this will be padded. Must match data generation.") + +flags.DEFINE_integer( + "max_predictions_per_seq", 20, + "Maximum number of masked LM predictions per sequence. " + "Must match data generation.") + +flags.DEFINE_bool("do_train", True, "Whether to run training.") + +flags.DEFINE_bool("do_eval", False, "Whether to run eval on the dev set.") + +flags.DEFINE_integer("train_batch_size", 64, "Total batch size for training.") + +flags.DEFINE_integer("eval_batch_size", 8, "Total batch size for eval.") + +flags.DEFINE_float("learning_rate", 1e-4, "The initial learning rate for Adam.") + +flags.DEFINE_integer("num_train_steps", 1000000, "Number of training steps.") + +flags.DEFINE_integer("num_warmup_steps", 10000, "Number of warmup steps.") + +flags.DEFINE_integer("save_checkpoints_steps", 10000, + "How often to save the model checkpoint.") + +flags.DEFINE_integer("display_loss_steps", 10, + "How often to print loss") + +flags.DEFINE_integer("iterations_per_loop", 1000, + "How many steps to make in each estimator call.") + +flags.DEFINE_integer("max_eval_steps", 100, "Maximum number of eval steps.") + +flags.DEFINE_integer("num_accumulation_steps", 1, + "Number of accumulation steps before gradient update." + "Global batch size = num_accumulation_steps * train_batch_size") + +flags.DEFINE_bool("allreduce_post_accumulation", False, "Whether to all reduce after accumulation of N steps or after each step") + +flags.DEFINE_bool( + "verbose_logging", False, + "If true, all of the trainable parameters are printed") + +flags.DEFINE_bool("horovod", False, "Whether to use Horovod for multi-gpu runs") + +flags.DEFINE_bool("report_loss", True, "Whether to report total loss during training.") + +flags.DEFINE_bool("manual_fp16", True, "Whether to use fp32 or fp16 arithmetic on GPU. " + "Manual casting is done instead of using AMP") + +flags.DEFINE_bool("use_xla", False, "Whether to enable XLA JIT compilation.") + +flags.DEFINE_bool("use_fp16", False, "Whether to enable AMP ops.") + +flags.DEFINE_bool("use_fp16_cls", True, "Whether to use fp16 in cls and pooler.") + +flags.DEFINE_bool("distributed", True, "Whether to use multi-npu") + +flags.DEFINE_bool('npu_bert_fused_gelu', True, 'Whether to use npu defined gelu op') + +flags.DEFINE_bool('npu_bert_debug', False, 'If True, dropout and shuffle is disabled.') + +flags.DEFINE_bool('npu_bert_use_tdt', True, 'Whether to use tdt as dataset') + +flags.DEFINE_string("npu_bert_job_start_file", None, "CSA job start file path.") + +flags.DEFINE_integer("npu_bert_loss_scale", 0, "Whether to use loss scale, -1 is disable, 0 is dynamic loss scale, >=1 is static loss scale") + +flags.DEFINE_bool("npu_bert_clip_by_global_norm", False, "Use clip_by_global_norm if True, or use clip_by_norm for each gradient") + +flags.DEFINE_bool('npu_bert_npu_dropout', True, 'Whether to use npu defined gelu op') + +flags.DEFINE_bool('npu_gather', True, 'Whether to use gather_npu whose backward propagation avoids IndexedSlices') + +flags.DEFINE_bool('hcom_parallel', True, 'Whether to use parallel allreduce') + +flags.DEFINE_integer('init_loss_scale_value', 2**32, 'Initial loss scale value for loss scale optimizer') + +# report samples/sec, total loss and learning rate during training +class _LogSessionRunHook(tf.train.SessionRunHook): + def __init__(self, global_batch_size, num_accumulation_steps, display_every=10, hvd_rank=-1): + self.global_batch_size = global_batch_size + self.display_every = display_every + self.hvd_rank = hvd_rank + self.num_accumulation_steps = num_accumulation_steps + def after_create_session(self, session, coord): + self.elapsed_secs = 0. + self.count = 0 + self.all_count = 0 + self.avg_loss = 0.0 + + def before_run(self, run_context): + self.t0 = time.time() + if self.num_accumulation_steps <= 1: + if (tf.flags.FLAGS.npu_bert_loss_scale == 0) and (FLAGS.manual_fp16 or FLAGS.use_fp16): + return tf.train.SessionRunArgs( + fetches=['global_step:0', 'total_loss:0', + 'learning_rate:0', 'nsp_loss:0', + 'mlm_loss:0', 'loss_scale:0', 'apply_grads/All:0']) + else: + return tf.train.SessionRunArgs( + fetches=['global_step:0', 'total_loss:0', + 'learning_rate:0', 'nsp_loss:0', + 'mlm_loss:0']) + else: + if (tf.flags.FLAGS.npu_bert_loss_scale == 0) and (FLAGS.manual_fp16 or FLAGS.use_fp16): + return tf.train.SessionRunArgs( + fetches=['global_step:0', 'update_step:0', 'total_loss:0', + 'learning_rate:0', 'nsp_loss:0', + 'mlm_loss:0', 'loss_scale:0']) + else: + return tf.train.SessionRunArgs( + fetches=['global_step:0', 'update_step:0', 'total_loss:0', + 'learning_rate:0', 'nsp_loss:0', + 'mlm_loss:0']) + def after_run(self, run_context, run_values): + self.elapsed_secs += time.time() - self.t0 + if self.num_accumulation_steps <=1: + if (tf.flags.FLAGS.npu_bert_loss_scale == 0) and (FLAGS.manual_fp16 or FLAGS.use_fp16): + global_step, total_loss, lr, nsp_loss, mlm_loss, loss_scaler, custom_arg = run_values.results + else: + global_step, total_loss, lr, nsp_loss, mlm_loss = run_values. \ + results + update_step = True + else: + if (tf.flags.FLAGS.npu_bert_loss_scale == 0) and (FLAGS.manual_fp16 or FLAGS.use_fp16): + global_step, update_step, total_loss, lr, nsp_loss, mlm_loss, loss_scaler = run_values.results + else: + global_step, update_step, total_loss, lr, nsp_loss, mlm_loss = run_values.\ + results + print_step = global_step + 1 # One-based index for printing. + self.avg_loss += total_loss + self.all_count += 1 + if update_step: + self.count += 1 + dt = self.elapsed_secs / self.count + sent_per_sec = self.global_batch_size / dt * FLAGS.iterations_per_loop + avg_loss_step = self.avg_loss / self.all_count + if self.hvd_rank >= 0: + if (tf.flags.FLAGS.npu_bert_loss_scale == 0) and (FLAGS.manual_fp16 or FLAGS.use_fp16): + print('Rank = %2d :: Step = %6i Throughput = %11.1f MLM Loss = %10.4e NSP Loss = %10.4e Loss = %9.6f Average Loss = %9.6f LR = %6.4e Loss scale = %6.4e isFinite = %6i' % + (self.hvd_rank, print_step, sent_per_sec, mlm_loss, nsp_loss, total_loss, avg_loss_step, lr, loss_scaler, custom_arg), flush=True) + hwlog.remark_print(key=hwlog.CURRENT_STEP, value='%6i' % print_step) + hwlog.remark_print(key=hwlog.THROWOUT, value='%11.1f' % sent_per_sec) + else: + print('Rank = %2d :: Step = %6i Throughput = %11.1f MLM Loss = %10.4e NSP Loss = %10.4e Loss = %9.6f Average Loss = %9.6f LR = %6.4e' % + (self.hvd_rank, print_step, sent_per_sec, mlm_loss, nsp_loss, total_loss, avg_loss_step, lr), flush=True) + hwlog.remark_print(key=hwlog.CURRENT_STEP, value='%6i' % print_step) + hwlog.remark_print(key=hwlog.THROWOUT, value='%11.1f' % sent_per_sec) + else: + if (tf.flags.FLAGS.npu_bert_loss_scale == 0) and (FLAGS.manual_fp16 or FLAGS.use_fp16): + print('Step = %6i Throughput = %11.1f MLM Loss = %10.4e NSP Loss = %10.4e Loss = %9.6f Average Loss = %9.6f LR = %6.4e Loss scale = %6.4e isFinite = %6i' % + (print_step, sent_per_sec, mlm_loss, nsp_loss, total_loss, avg_loss_step, lr, loss_scaler, custom_arg), flush=True) + hwlog.remark_print(key=hwlog.CURRENT_STEP, value='%6i' % print_step) + hwlog.remark_print(key=hwlog.THROWOUT, value='%11.1f' % sent_per_sec) + else: + print('Step = %6i Throughput = %11.1f MLM Loss = %10.4e NSP Loss = %10.4e Loss = %9.6f Average Loss = %9.6f LR = %6.4e' % + (print_step, sent_per_sec, mlm_loss, nsp_loss, total_loss, avg_loss_step, lr), flush=True) + hwlog.remark_print(key=hwlog.CURRENT_STEP, value='%6i' % print_step) + hwlog.remark_print(key=hwlog.THROWOUT, value='%11.1f' % sent_per_sec) + self.elapsed_secs = 0. + self.count = 0 + self.avg_loss = 0.0 + self.all_count = 0 + +def model_fn_builder(bert_config, init_checkpoint, learning_rate, + num_train_steps, num_warmup_steps, + use_one_hot_embeddings, hvd=None): + """Returns `model_fn` closure for TPUEstimator.""" + + def model_fn(features, labels, mode, params): # pylint: disable=unused-argument + """The `model_fn` for TPUEstimator.""" + + tf.logging.info("*** Features ***") + for name in sorted(features.keys()): + tf.logging.info(" name = %s, shape = %s" % (name, features[name].shape)) + + input_ids = features["input_ids"] + input_mask = features["input_mask"] + segment_ids = features["segment_ids"] + masked_lm_positions = features["masked_lm_positions"] + masked_lm_ids = features["masked_lm_ids"] + masked_lm_weights = features["masked_lm_weights"] + next_sentence_labels = features["next_sentence_labels"] + + is_training = (mode == tf.estimator.ModeKeys.TRAIN) + + model = modeling.BertModel( + config=bert_config, + is_training=is_training, + input_ids=input_ids, + input_mask=input_mask, + token_type_ids=segment_ids, + use_one_hot_embeddings=use_one_hot_embeddings, + compute_type=tf.float16 if FLAGS.manual_fp16 else tf.float32) + + (masked_lm_loss, + masked_lm_example_loss, masked_lm_log_probs) = get_masked_lm_output( + bert_config, model.get_sequence_output(), model.get_embedding_table(), + masked_lm_positions, masked_lm_ids, + masked_lm_weights) + + (next_sentence_loss, next_sentence_example_loss, + next_sentence_log_probs) = get_next_sentence_output( + bert_config, model.get_pooled_output(), next_sentence_labels) + + masked_lm_loss = tf.identity(masked_lm_loss, name="mlm_loss") + next_sentence_loss = tf.identity(next_sentence_loss, name="nsp_loss") + total_loss = masked_lm_loss + next_sentence_loss + total_loss = tf.identity(total_loss, name='total_loss') + + tvars = tf.trainable_variables() + + initialized_variable_names = {} + if init_checkpoint and (hvd is None or hvd.rank() == 0): + print("Loading checkpoint", init_checkpoint) + (assignment_map, initialized_variable_names + ) = modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint) + + tf.train.init_from_checkpoint(init_checkpoint, assignment_map) + + if FLAGS.verbose_logging: + tf.logging.info("**** Trainable Variables ****") + for var in tvars: + init_string = "" + if var.name in initialized_variable_names: + init_string = ", *INIT_FROM_CKPT*" + tf.logging.info(" %d :: name = %s, shape = %s%s", 0 if hvd is None else hvd.rank(), var.name, var.shape, + init_string) + + output_spec = None + if mode == tf.estimator.ModeKeys.TRAIN: + train_op = optimization.create_optimizer( + total_loss, learning_rate, num_train_steps, num_warmup_steps, + hvd, FLAGS.manual_fp16, FLAGS.use_fp16, FLAGS.num_accumulation_steps, FLAGS.optimizer_type, FLAGS.allreduce_post_accumulation) + + output_spec = tf.estimator.EstimatorSpec( + mode=mode, + loss=total_loss, + train_op=train_op) + elif mode == tf.estimator.ModeKeys.EVAL: + + def metric_fn(masked_lm_example_loss, masked_lm_log_probs, masked_lm_ids, + masked_lm_weights, next_sentence_example_loss, + next_sentence_log_probs, next_sentence_labels): + """Computes the loss and accuracy of the model.""" + masked_lm_log_probs = tf.reshape(masked_lm_log_probs, + [-1, masked_lm_log_probs.shape[-1]]) + masked_lm_predictions = tf.argmax( + masked_lm_log_probs, axis=-1, output_type=tf.int32) + masked_lm_example_loss = tf.reshape(masked_lm_example_loss, [-1]) + masked_lm_ids = tf.reshape(masked_lm_ids, [-1]) + masked_lm_weights = tf.reshape(masked_lm_weights, [-1]) + masked_lm_accuracy = tf.metrics.accuracy( + labels=masked_lm_ids, + predictions=masked_lm_predictions, + weights=masked_lm_weights) + masked_lm_mean_loss = tf.metrics.mean( + values=masked_lm_example_loss, weights=masked_lm_weights) + + next_sentence_log_probs = tf.reshape( + next_sentence_log_probs, [-1, next_sentence_log_probs.shape[-1]]) + next_sentence_predictions = tf.argmax( + next_sentence_log_probs, axis=-1, output_type=tf.int32) + next_sentence_labels = tf.reshape(next_sentence_labels, [-1]) + next_sentence_accuracy = tf.metrics.accuracy( + labels=next_sentence_labels, predictions=next_sentence_predictions) + next_sentence_mean_loss = tf.metrics.mean( + values=next_sentence_example_loss) + + return { + "masked_lm_accuracy": masked_lm_accuracy, + "masked_lm_loss": masked_lm_mean_loss, + "next_sentence_accuracy": next_sentence_accuracy, + "next_sentence_loss": next_sentence_mean_loss, + } + + eval_metric_ops = metric_fn( + masked_lm_example_loss, masked_lm_log_probs, masked_lm_ids, + masked_lm_weights, next_sentence_example_loss, + next_sentence_log_probs, next_sentence_labels + ) + output_spec = tf.estimator.EstimatorSpec( + mode=mode, + loss=total_loss, + eval_metric_ops=eval_metric_ops) + else: + raise ValueError("Only TRAIN and EVAL modes are supported: %s" % (mode)) + + return output_spec + + return model_fn + + +def get_masked_lm_output(bert_config, input_tensor, output_weights, positions, + label_ids, label_weights): + """Get loss and log probs for the masked LM.""" + input_tensor = gather_indexes(input_tensor, positions) + + with tf.variable_scope("cls/predictions"): + # We apply one more non-linear transformation before the output layer. + # This matrix is not used after pre-training. + with tf.variable_scope("transform", custom_getter=get_custom_getter(compute_type=tf.float16 if FLAGS.use_fp16_cls else tf.float32)): + if FLAGS.use_fp16_cls: + input_tensor = tf.cast(input_tensor, tf.float16) + input_tensor = tf.layers.dense( + input_tensor, + units=bert_config.hidden_size, + activation=modeling.get_activation(bert_config.hidden_act), + kernel_initializer=modeling.create_initializer( + bert_config.initializer_range)) + input_tensor = tf.cast(input_tensor, tf.float32) + input_tensor = modeling.layer_norm(input_tensor) + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + output_bias = tf.get_variable( + "output_bias", + shape=[bert_config.vocab_size], + initializer=tf.zeros_initializer()) + if FLAGS.use_fp16_cls: + input_tensor = tf.cast(input_tensor, tf.float16) + logits = tf.matmul(input_tensor, tf.cast(output_weights, tf.float16), transpose_b=True) + logits = tf.cast(logits, tf.float32) + else: + logits = tf.matmul(tf.cast(input_tensor, tf.float32), output_weights, transpose_b=True) + logits = tf.nn.bias_add(logits, output_bias) + log_probs = tf.nn.log_softmax(logits, axis=-1) + + label_ids = tf.reshape(label_ids, [-1]) + label_weights = tf.reshape(label_weights, [-1]) + + one_hot_labels = tf.one_hot( + label_ids, depth=bert_config.vocab_size, dtype=tf.float32) + + # The `positions` tensor might be zero-padded (if the sequence is too + # short to have the maximum number of predictions). The `label_weights` + # tensor has a value of 1.0 for every real prediction and 0.0 for the + # padding predictions. + per_example_loss = -tf.reduce_sum(log_probs * one_hot_labels, axis=[-1]) + numerator = tf.reduce_sum(label_weights * per_example_loss) + denominator = tf.reduce_sum(label_weights) + 1e-5 + loss = numerator / denominator + + return (loss, per_example_loss, log_probs) + + +def get_next_sentence_output(bert_config, input_tensor, labels): + """Get loss and log probs for the next sentence prediction.""" + + # Simple binary classification. Note that 0 is "next sentence" and 1 is + # "random sentence". This weight matrix is not used after pre-training. + with tf.variable_scope("cls/seq_relationship"): + output_weights = tf.get_variable( + "output_weights", + shape=[2, bert_config.hidden_size], + initializer=modeling.create_initializer(bert_config.initializer_range)) + output_bias = tf.get_variable( + "output_bias", shape=[2], initializer=tf.zeros_initializer()) + + if FLAGS.use_fp16_cls: + input_tensor = tf.cast(input_tensor, tf.float16) + logits = tf.matmul(input_tensor, tf.cast(output_weights, tf.float16), transpose_b=True) + logits = tf.cast(logits, tf.float32) + else: + logits = tf.matmul(tf.cast(input_tensor, tf.float32), output_weights, transpose_b=True) + logits = tf.nn.bias_add(logits, output_bias) + log_probs = tf.nn.log_softmax(logits, axis=-1) + labels = tf.reshape(labels, [-1]) + one_hot_labels = tf.one_hot(labels, depth=2, dtype=tf.float32) + per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1) + loss = tf.reduce_mean(per_example_loss) + return (loss, per_example_loss, log_probs) + + +def gather_indexes(sequence_tensor, positions): + """Gathers the vectors at the specific positions over a minibatch.""" + sequence_shape = modeling.get_shape_list(sequence_tensor, expected_rank=3) + batch_size = sequence_shape[0] + seq_length = sequence_shape[1] + width = sequence_shape[2] + + flat_offsets = tf.reshape( + tf.range(0, batch_size, dtype=tf.int32) * seq_length, [-1, 1]) + flat_positions = tf.reshape(positions + flat_offsets, [-1]) + flat_sequence_tensor = tf.reshape(sequence_tensor, + [batch_size * seq_length, width]) + output_tensor = tf.gather(flat_sequence_tensor, flat_positions) + return output_tensor + + +def input_fn_builder(input_files, + batch_size, + max_seq_length, + max_predictions_per_seq, + is_training, + num_cpu_threads=4, + hvd=None): + """Creates an `input_fn` closure to be passed to Estimator.""" + + def input_fn(): + """The actual input function.""" + + name_to_features = { + "input_ids": + tf.FixedLenFeature([max_seq_length], tf.int64), + "input_mask": + tf.FixedLenFeature([max_seq_length], tf.int64), + "segment_ids": + tf.FixedLenFeature([max_seq_length], tf.int64), + "masked_lm_positions": + tf.FixedLenFeature([max_predictions_per_seq], tf.int64), + "masked_lm_ids": + tf.FixedLenFeature([max_predictions_per_seq], tf.int64), + "masked_lm_weights": + tf.FixedLenFeature([max_predictions_per_seq], tf.float32), + "next_sentence_labels": + tf.FixedLenFeature([1], tf.int64), + } + + # For training, we want a lot of parallel reading and shuffling. + # For eval, we want no shuffling and parallel reading doesn't matter. + if is_training: + d = tf.data.Dataset.from_tensor_slices(tf.constant(input_files)) + if FLAGS.distributed: + #rank_size = int(os.getenv('RANK_SIZE')) + #rank_id = int(os.getenv('RANK_INDEX')) + #device_id = int(os.getenv('DEVICE_ID')) + #local_rank = rank_id * 8 + device_id + #print('RANK_SIZE=', rank_size, ' RANK_ID=', local_rank) + rank_size = int(os.getenv('RANK_SIZE')) + rank_id = int(os.getenv('RANK_ID')) + print('RANK_SIZE=', rank_size, ' rank_id=', rank_id) + d = d.shard(rank_size, rank_id) + d = d.repeat() + if not FLAGS.npu_bert_debug: + d = d.shuffle(buffer_size=len(input_files)) + + # `cycle_length` is the number of parallel files that get read. + if not FLAGS.npu_bert_debug: + #cycle_length = min(num_cpu_threads, len(input_files)) + cycle_length = min(num_cpu_threads, int(len(input_files)/int(os.getenv('RANK_SIZE')))) + else: + cycle_length = 1 + + # `sloppy` mode means that the interleaving is not exact. This adds + # even more randomness to the training pipeline. + #d = d.apply( + # tf.contrib.data.parallel_interleave( + # tf.data.TFRecordDataset, + # sloppy=(not FLAGS.npu_bert_debug), + # cycle_length=cycle_length)) + d = d.interleave( + tf.data.TFRecordDataset, + cycle_length=cycle_length, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + if not FLAGS.npu_bert_debug: + d = d.shuffle(buffer_size=100) + else: + d = tf.data.TFRecordDataset(input_files) + # Since we evaluate for a fixed number of steps we don't want to encounter + # out-of-range exceptions. + d = d.repeat() + + # We must `drop_remainder` on training because the TPU requires fixed + # size dimensions. For eval, we assume we are evaluating on the CPU or GPU + # and we *don't* want to drop the remainder, otherwise we wont cover + # every sample. + d = d.apply( + tf.contrib.data.map_and_batch( + lambda record: _decode_record(record, name_to_features), + batch_size=batch_size, + num_parallel_batches=num_cpu_threads, + drop_remainder=True)) + return d + + return input_fn + + +def _decode_record(record, name_to_features): + """Decodes a record to a TensorFlow example.""" + example = tf.parse_single_example(record, name_to_features) + + # tf.Example only supports tf.int64, but the TPU only supports tf.int32. + # So cast all int64 to int32. + for name in list(example.keys()): + t = example[name] + if t.dtype == tf.int64: + t = tf.to_int32(t) + example[name] = t + + return example + + +def main(_): + for name, value in FLAGS.__flags.items(): + print("name:", name, " ", FLAGS[name].value) + + tf.logging.set_verbosity(tf.logging.INFO) + + if not FLAGS.do_train and not FLAGS.do_eval: + raise ValueError("At least one of `do_train` or `do_eval` must be True.") + + if FLAGS.use_fp16: + os.environ["TF_ENABLE_AUTO_MIXED_PRECISION_GRAPH_REWRITE"] = "1" + + if FLAGS.horovod: + import horovod.tensorflow as hvd + hvd.init() + + bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file) + + if FLAGS.npu_gather: + if FLAGS.distributed and bert_config.num_hidden_layers == 24: + #from hccl.split.api import set_split_strategy_by_idx + from hccl.split.api import set_split_strategy_by_size + #set_split_strategy_by_idx([8,72,136,200,264,328,392,397]) + set_split_strategy_by_size([10,10,10,10,15,15,15,15]) + if FLAGS.distributed and bert_config.num_hidden_layers == 12: + from hccl.split.api import set_split_strategy_by_idx + set_split_strategy_by_idx([8,56,104,152,200,205]) + if FLAGS.distributed and bert_config.num_hidden_layers == 6: + from hccl.split.api import set_split_strategy_by_idx + set_split_strategy_by_idx([8,40,72,104,109]) + + tf.gfile.MakeDirs(FLAGS.output_dir) + + input_files = [] + for input_file_dir in FLAGS.input_files_dir.split(","): + input_files.extend(tf.gfile.Glob(os.path.join(input_file_dir, "*"))) + + input_files.sort() + print("Input Files:", input_files) + + if FLAGS.horovod and len(input_files) < hvd.size(): + raise ValueError("Input Files must be sharded") + if FLAGS.use_fp16 and FLAGS.manual_fp16: + raise ValueError("AMP and Manual Mixed Precision Training are both activated! Error") + + is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 + config = tf.ConfigProto() + if FLAGS.horovod: + config.gpu_options.visible_device_list = str(hvd.local_rank()) + if hvd.rank() == 0: + tf.logging.info("***** Configuaration *****") + for key in FLAGS.__flags.keys(): + tf.logging.info(' {}: {}'.format(key, getattr(FLAGS, key))) + tf.logging.info("**************************") + +# config.gpu_options.per_process_gpu_memory_fraction = 0.7 + if FLAGS.use_xla: + config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1 + config.graph_options.rewrite_options.memory_optimization = rewriter_config_pb2.RewriterConfig.NO_MEM_OPT + + #run_config = tf.estimator.RunConfig( + run_config = NPURunConfig( + model_dir=FLAGS.output_dir, + save_summary_steps=0, + session_config=config, + save_checkpoints_steps=FLAGS.save_checkpoints_steps if not FLAGS.horovod or hvd.rank() == 0 else None, + # This variable controls how often estimator reports examples/sec. + # Default value is every 100 steps. + # When --report_loss is True, we set to very large value to prevent + # default info reporting from estimator. + # Ideally we should set it to None, but that does not work. + log_step_count_steps=1 if FLAGS.report_loss else 100, + enable_data_pre_proc=FLAGS.npu_bert_use_tdt, + iterations_per_loop=FLAGS.iterations_per_loop, + hcom_parallel=FLAGS.hcom_parallel) + + if FLAGS.distributed: + rank_size = int(os.getenv('RANK_SIZE')) + model_fn = model_fn_builder( + bert_config=bert_config, + init_checkpoint=FLAGS.init_checkpoint, + learning_rate=FLAGS.learning_rate, + num_train_steps=FLAGS.num_train_steps, + num_warmup_steps=FLAGS.num_warmup_steps, + use_one_hot_embeddings=False, + hvd=None if not FLAGS.horovod else hvd) + + training_hooks = [] + """ + if FLAGS.report_loss and (not FLAGS.horovod or hvd.rank() == 0): + global_batch_size = FLAGS.train_batch_size * FLAGS.num_accumulation_steps if not FLAGS.horovod else FLAGS.train_batch_size * FLAGS.num_accumulation_steps * hvd.size() + training_hooks.append(_LogSessionRunHook(global_batch_size, FLAGS.num_accumulation_steps, FLAGS.display_loss_steps)) + if FLAGS.horovod and hvd.size() > 1: + training_hooks.append(hvd.BroadcastGlobalVariablesHook(0)) + """ + if FLAGS.report_loss: + global_batch_size = FLAGS.train_batch_size * FLAGS.num_accumulation_steps if not FLAGS.distributed else FLAGS.train_batch_size * FLAGS.num_accumulation_steps * rank_size + training_hooks.append(_LogSessionRunHook(global_batch_size, FLAGS.num_accumulation_steps, FLAGS.display_loss_steps)) + + + #estimator = tf.estimator.Estimator( + estimator = NPUEstimator( + model_fn=model_fn, + config=run_config, + job_start_file=FLAGS.npu_bert_job_start_file) + + if FLAGS.do_train: + tf.logging.info("***** Running training *****") + tf.logging.info(" Batch size = %d", FLAGS.train_batch_size) + train_input_fn = input_fn_builder( + input_files=input_files, + batch_size=FLAGS.train_batch_size, + max_seq_length=FLAGS.max_seq_length, + max_predictions_per_seq=FLAGS.max_predictions_per_seq, + is_training=True, + hvd=None if not FLAGS.horovod else hvd) + + estimator.train(input_fn=train_input_fn, hooks=training_hooks, max_steps=FLAGS.num_train_steps) + + if FLAGS.do_eval and (not FLAGS.horovod or hvd.rank() == 0): + tf.logging.info("***** Running evaluation *****") + tf.logging.info(" Batch size = %d", FLAGS.eval_batch_size) + + eval_files = [] + for eval_file_dir in FLAGS.eval_files_dir.split(","): + eval_files.extend(tf.gfile.Glob(os.path.join(eval_file_dir, "*"))) + + eval_input_fn = input_fn_builder( + input_files=eval_files, + batch_size=FLAGS.eval_batch_size, + max_seq_length=FLAGS.max_seq_length, + max_predictions_per_seq=FLAGS.max_predictions_per_seq, + is_training=False, + hvd=None if not FLAGS.horovod else hvd) + + eval_hooks = [LogEvalRunHook(FLAGS.eval_batch_size)] + eval_start_time = time.time() + result = estimator.evaluate( + input_fn=eval_input_fn, steps=FLAGS.max_eval_steps, hooks=eval_hooks) + + eval_time_elapsed = time.time() - eval_start_time + eval_time_wo_overhead = eval_hooks[-1].total_time + + num_sentences = (eval_hooks[-1].count - eval_hooks[-1].skipped) * FLAGS.eval_batch_size + + ss_sentences_per_second = num_sentences * 1.0 / eval_time_wo_overhead + + tf.logging.info("-----------------------------") + tf.logging.info("Total Inference Time = %0.2f for Sentences = %d", eval_time_elapsed, + eval_hooks[-1].count * FLAGS.eval_batch_size) + tf.logging.info("Total Inference Time W/O Overhead = %0.2f for Sentences = %d", eval_time_wo_overhead, + (eval_hooks[-1].count - eval_hooks[-1].skipped) * FLAGS.eval_batch_size) + tf.logging.info("Summary Inference Statistics on EVAL set") + tf.logging.info("Batch size = %d", FLAGS.eval_batch_size) + tf.logging.info("Sequence Length = %d", FLAGS.max_seq_length) + tf.logging.info("Precision = %s", "fp16" if FLAGS.use_fp16 else "fp32") + tf.logging.info("Throughput Average (sentences/sec) = %0.2f", ss_sentences_per_second) + tf.logging.info("-----------------------------") + + output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt") + with tf.gfile.GFile(output_eval_file, "w") as writer: + tf.logging.info("***** Eval results *****") + for key in sorted(result.keys()): + tf.logging.info(" %s = %s", key, str(result[key])) + writer.write("%s = %s\n" % (key, str(result[key]))) + if key == 'masked_lm_accuracy': + hwlog.remark_print(key=hwlog.MASKED_LM_ACCURACY, value=str(result[key])) + elif key == 'next_sentence_accuracy ': + hwlog.remark_print(key=hwlog.NEXT_SENTENCE_ACCURACY, value=str(result[key])) + elif key == 'global_step': + hwlog.remark_print(key=hwlog.GLOBAL_STEP, value=str(result[key])) + elif key == 'loss': + hwlog.remark_print(key=hwlog.LOSS, value=str(result[key])) + elif key == 'masked_lm_loss': + hwlog.remark_print(key=hwlog.MASKED_LM_LOSS, value=str(result[key])) + elif key == 'next_sentence_loss ': + hwlog.remark_print(key=hwlog.NEXT_SENTENCE_LOSS, value=str(result[key])) + else: + pass + + +if __name__ == "__main__": + hwlog.ROOT_DIR = os.path.split(os.path.abspath(__file__))[0] + cpu_info, npu_info, framework_info, os_info, benchmark_version = get_environment_info("tensorflow") + config_info = get_model_parameter("tensorflow_config") + initinal_data = {"base_lr": 0.01, "dataset": "cn-clue/en-wiki", "optimizer": "Adam", "loss_scale": 512} + flags.mark_flag_as_required("input_files_dir") + flags.mark_flag_as_required("eval_files_dir") + flags.mark_flag_as_required("bert_config_file") + flags.mark_flag_as_required("output_dir") + flags.mark_flag_as_required("npu_bert_job_start_file") + hwlog.remark_print(key=hwlog.CPU_INFO, value=cpu_info) + hwlog.remark_print(key=hwlog.NPU_INFO, value=npu_info) + hwlog.remark_print(key=hwlog.OS_INFO, value=os_info) + hwlog.remark_print(key=hwlog.FRAMEWORK_INFO, value=framework_info) + hwlog.remark_print(key=hwlog.BENCHMARK_VERSION, value=benchmark_version) + hwlog.remark_print(key=hwlog.CONFIG_INFO, value=config_info) + hwlog.remark_print(key=hwlog.BASE_LR, value=initinal_data.get("base_lr")) + hwlog.remark_print(key=hwlog.DATASET, value=initinal_data.get("dataset")) + hwlog.remark_print(key=hwlog.OPT_NAME, value=initinal_data.get("optimizer")) + hwlog.remark_print(key=hwlog.LOSS_SCALE, value=initinal_data.get("loss_scale")) + if FLAGS.use_xla and FLAGS.manual_fp16: + print('WARNING! Combining --use_xla with --manual_fp16 may prevent convergence.') + print(' This warning message will be removed when the underlying') + print(' issues have been fixed and you are running a TF version') + print(' that has that fix.') + tf.app.run() diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/tf_metrics.py b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/tf_metrics.py new file mode 100644 index 0000000..f4c5148 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/tf_metrics.py @@ -0,0 +1,215 @@ +""" +Multiclass +from: +https://github.com/guillaumegenthial/tf_metrics/blob/master/tf_metrics/__init__.py + +""" + +__author__ = "Guillaume Genthial" + +import numpy as np +import tensorflow as tf +from tensorflow.python.ops.metrics_impl import _streaming_confusion_matrix + + +def precision(labels, predictions, num_classes, pos_indices=None, + weights=None, average='micro'): + """Multi-class precision metric for Tensorflow + Parameters + ---------- + labels : Tensor of tf.int32 or tf.int64 + The true labels + predictions : Tensor of tf.int32 or tf.int64 + The predictions, same shape as labels + num_classes : int + The number of classes + pos_indices : list of int, optional + The indices of the positive classes, default is all + weights : Tensor of tf.int32, optional + Mask, must be of compatible shape with labels + average : str, optional + 'micro': counts the total number of true positives, false + positives, and false negatives for the classes in + `pos_indices` and infer the metric from it. + 'macro': will compute the metric separately for each class in + `pos_indices` and average. Will not account for class + imbalance. + 'weighted': will compute the metric separately for each class in + `pos_indices` and perform a weighted average by the total + number of true labels for each class. + Returns + ------- + tuple of (scalar float Tensor, update_op) + """ + cm, op = _streaming_confusion_matrix( + labels, predictions, num_classes, weights) + pr, _, _ = metrics_from_confusion_matrix( + cm, pos_indices, average=average) + op, _, _ = metrics_from_confusion_matrix( + op, pos_indices, average=average) + return (pr, op) + + +def recall(labels, predictions, num_classes, pos_indices=None, weights=None, + average='micro'): + """Multi-class recall metric for Tensorflow + Parameters + ---------- + labels : Tensor of tf.int32 or tf.int64 + The true labels + predictions : Tensor of tf.int32 or tf.int64 + The predictions, same shape as labels + num_classes : int + The number of classes + pos_indices : list of int, optional + The indices of the positive classes, default is all + weights : Tensor of tf.int32, optional + Mask, must be of compatible shape with labels + average : str, optional + 'micro': counts the total number of true positives, false + positives, and false negatives for the classes in + `pos_indices` and infer the metric from it. + 'macro': will compute the metric separately for each class in + `pos_indices` and average. Will not account for class + imbalance. + 'weighted': will compute the metric separately for each class in + `pos_indices` and perform a weighted average by the total + number of true labels for each class. + Returns + ------- + tuple of (scalar float Tensor, update_op) + """ + cm, op = _streaming_confusion_matrix( + labels, predictions, num_classes, weights) + _, re, _ = metrics_from_confusion_matrix( + cm, pos_indices, average=average) + _, op, _ = metrics_from_confusion_matrix( + op, pos_indices, average=average) + return (re, op) + + +def f1(labels, predictions, num_classes, pos_indices=None, weights=None, + average='micro'): + return fbeta(labels, predictions, num_classes, pos_indices, weights, + average) + + +def fbeta(labels, predictions, num_classes, pos_indices=None, weights=None, + average='micro', beta=1): + """Multi-class fbeta metric for Tensorflow + Parameters + ---------- + labels : Tensor of tf.int32 or tf.int64 + The true labels + predictions : Tensor of tf.int32 or tf.int64 + The predictions, same shape as labels + num_classes : int + The number of classes + pos_indices : list of int, optional + The indices of the positive classes, default is all + weights : Tensor of tf.int32, optional + Mask, must be of compatible shape with labels + average : str, optional + 'micro': counts the total number of true positives, false + positives, and false negatives for the classes in + `pos_indices` and infer the metric from it. + 'macro': will compute the metric separately for each class in + `pos_indices` and average. Will not account for class + imbalance. + 'weighted': will compute the metric separately for each class in + `pos_indices` and perform a weighted average by the total + number of true labels for each class. + beta : int, optional + Weight of precision in harmonic mean + Returns + ------- + tuple of (scalar float Tensor, update_op) + """ + cm, op = _streaming_confusion_matrix( + labels, predictions, num_classes, weights) + _, _, fbeta = metrics_from_confusion_matrix( + cm, pos_indices, average=average, beta=beta) + _, _, op = metrics_from_confusion_matrix( + op, pos_indices, average=average, beta=beta) + return (fbeta, op) + + +def safe_div(numerator, denominator): + """Safe division, return 0 if denominator is 0""" + numerator, denominator = tf.to_float(numerator), tf.to_float(denominator) + zeros = tf.zeros_like(numerator, dtype=numerator.dtype) + denominator_is_zero = tf.equal(denominator, zeros) + return tf.where(denominator_is_zero, zeros, numerator / denominator) + + +def pr_re_fbeta(cm, pos_indices, beta=1): + """Uses a confusion matrix to compute precision, recall and fbeta""" + num_classes = cm.shape[0] + neg_indices = [i for i in range(num_classes) if i not in pos_indices] + cm_mask = np.ones([num_classes, num_classes]) + cm_mask[neg_indices, neg_indices] = 0 + diag_sum = tf.reduce_sum(tf.diag_part(cm * cm_mask)) + + cm_mask = np.ones([num_classes, num_classes]) + cm_mask[:, neg_indices] = 0 + tot_pred = tf.reduce_sum(cm * cm_mask) + + cm_mask = np.ones([num_classes, num_classes]) + cm_mask[neg_indices, :] = 0 + tot_gold = tf.reduce_sum(cm * cm_mask) + + pr = safe_div(diag_sum, tot_pred) + re = safe_div(diag_sum, tot_gold) + fbeta = safe_div((1. + beta**2) * pr * re, beta**2 * pr + re) + + return pr, re, fbeta + + +def metrics_from_confusion_matrix(cm, pos_indices=None, average='micro', + beta=1): + """Precision, Recall and F1 from the confusion matrix + Parameters + ---------- + cm : tf.Tensor of type tf.int32, of shape (num_classes, num_classes) + The streaming confusion matrix. + pos_indices : list of int, optional + The indices of the positive classes + beta : int, optional + Weight of precision in harmonic mean + average : str, optional + 'micro', 'macro' or 'weighted' + """ + num_classes = cm.shape[0] + if pos_indices is None: + pos_indices = [i for i in range(num_classes)] + + if average == 'micro': + return pr_re_fbeta(cm, pos_indices, beta) + elif average in {'macro', 'weighted'}: + precisions, recalls, fbetas, n_golds = [], [], [], [] + for idx in pos_indices: + pr, re, fbeta = pr_re_fbeta(cm, [idx], beta) + precisions.append(pr) + recalls.append(re) + fbetas.append(fbeta) + cm_mask = np.zeros([num_classes, num_classes]) + cm_mask[idx, :] = 1 + n_golds.append(tf.to_float(tf.reduce_sum(cm * cm_mask))) + + if average == 'macro': + pr = tf.reduce_mean(precisions) + re = tf.reduce_mean(recalls) + fbeta = tf.reduce_mean(fbetas) + return pr, re, fbeta + if average == 'weighted': + n_gold = tf.reduce_sum(n_golds) + pr_sum = sum(p * n for p, n in zip(precisions, n_golds)) + pr = safe_div(pr_sum, n_gold) + re_sum = sum(r * n for r, n in zip(recalls, n_golds)) + re = safe_div(re_sum, n_gold) + fbeta_sum = sum(f * n for f, n in zip(fbetas, n_golds)) + fbeta = safe_div(fbeta_sum, n_gold) + return pr, re, fbeta + + else: + raise NotImplementedError() diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/tokenization.py b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/tokenization.py new file mode 100644 index 0000000..6e53ce7 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/tokenization.py @@ -0,0 +1,451 @@ +# coding=utf-8 +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tokenization classes.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import unicodedata +import six +import tensorflow as tf +import re +import os + + +PRETRAINED_VOCAB_ARCHIVE_MAP = { + 'bert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt", + 'bert-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt", + 'bert-base-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-vocab.txt", + 'bert-large-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-vocab.txt", + 'bert-base-multilingual-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-uncased-vocab.txt", + 'bert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-vocab.txt", + 'bert-base-chinese': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese-vocab.txt", +} + +def validate_case_matches_checkpoint(do_lower_case, init_checkpoint): + """Checks whether the casing config is consistent with the checkpoint name.""" + + # The casing has to be passed in by the user and there is no explicit check + # as to whether it matches the checkpoint. The casing information probably + # should have been stored in the bert_config.json file, but it's not, so + # we have to heuristically detect it to validate. + + if not init_checkpoint: + return + + m = re.match("^.*?([A-Za-z0-9_-]+)/bert_model.ckpt", init_checkpoint) + if m is None: + return + + model_name = m.group(1) + + lower_models = [ + "uncased_L-24_H-1024_A-16", "uncased_L-12_H-768_A-12", + "multilingual_L-12_H-768_A-12", "chinese_L-12_H-768_A-12" + ] + + cased_models = [ + "cased_L-12_H-768_A-12", "cased_L-24_H-1024_A-16", + "multi_cased_L-12_H-768_A-12" + ] + + is_bad_config = False + if model_name in lower_models and not do_lower_case: + is_bad_config = True + actual_flag = "False" + case_name = "lowercased" + opposite_flag = "True" + + if model_name in cased_models and do_lower_case: + is_bad_config = True + actual_flag = "True" + case_name = "cased" + opposite_flag = "False" + + if is_bad_config: + raise ValueError( + "You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`. " + "However, `%s` seems to be a %s model, so you " + "should pass in `--do_lower_case=%s` so that the fine-tuning matches " + "how the model was pre-training. If this error is wrong, please " + "just comment out this check." % (actual_flag, init_checkpoint, + model_name, case_name, opposite_flag)) + + + +def convert_to_unicode(text): + """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + + +def printable_text(text): + """Returns text encoded in a way suitable for print or `tf.logging`.""" + + # These functions want `str` for both Python2 and Python3, but in one case + # it's a Unicode string and in the other it's a byte string. + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + index = 0 + with open(vocab_file, "r") as reader: + while True: + token = convert_to_unicode(reader.readline()) + if not token: + break + token = token.strip() + vocab[token] = index + index += 1 + return vocab + + +def convert_by_vocab(vocab, items): + """Converts a sequence of [tokens|ids] using the vocab.""" + output = [] + for item in items: + output.append(vocab[item]) + return output + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a peice of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class FullTokenizer(object): + """Runs end-to-end tokenziation.""" + + def __init__(self, vocab_file, do_lower_case=True): + self.vocab = load_vocab(vocab_file) + self.inv_vocab = {v: k for k, v in self.vocab.items()} + self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + + def tokenize(self, text): + split_tokens = [] + for token in self.basic_tokenizer.tokenize(text): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + + return split_tokens + + def convert_tokens_to_ids(self, tokens): + return convert_by_vocab(self.vocab, tokens) + + def convert_ids_to_tokens(self, ids): + return convert_by_vocab(self.inv_vocab, ids) + + +class BertTokenizer(object): + """Runs end-to-end tokenization: punctuation splitting + wordpiece""" + + def __init__(self, vocab_file, do_lower_case=True): + if not os.path.isfile(vocab_file): + raise ValueError( + "Can't find a vocabulary file at path '{}'. To load the vocabulary from a Google pretrained " + "model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`".format(vocab_file)) + self.vocab = load_vocab(vocab_file) + self.ids_to_tokens = collections.OrderedDict( + [(ids, tok) for tok, ids in self.vocab.items()]) + self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + + def tokenize(self, text): + split_tokens = [] + for token in self.basic_tokenizer.tokenize(text): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + return split_tokens + + def convert_tokens_to_ids(self, tokens): + """Converts a sequence of tokens into ids using the vocab.""" + ids = [] + for token in tokens: + ids.append(self.vocab[token]) + return ids + + def convert_ids_to_tokens(self, ids): + """Converts a sequence of ids in wordpiece tokens using the vocab.""" + tokens = [] + for i in ids: + tokens.append(self.ids_to_tokens[i]) + return tokens + + @classmethod + def from_pretrained(cls, pretrained_model_name, do_lower_case=True): + """ + Instantiate a PreTrainedBertModel from a pre-trained model file. + Download and cache the pre-trained model file if needed. + """ + if pretrained_model_name in PRETRAINED_VOCAB_ARCHIVE_MAP: + vocab_file = PRETRAINED_VOCAB_ARCHIVE_MAP[pretrained_model_name] + else: + vocab_file = pretrained_model_name + # redirect to the cache, if necessary + try: + resolved_vocab_file = cached_path(vocab_file) + if resolved_vocab_file == vocab_file: + + logger.info("loading vocabulary file {}".format(vocab_file)) + else: + logger.info("loading vocabulary file {} from cache at {}".format( + vocab_file, resolved_vocab_file)) + # Instantiate tokenizer. + tokenizer = cls(resolved_vocab_file, do_lower_case) + except FileNotFoundError: + logger.error( + "Model name '{}' was not found in model name list ({}). " + "We assumed '{}' was a path or url but couldn't find any file " + "associated to this path or url.".format( + pretrained_model_name, + ', '.join(PRETRAINED_VOCAB_ARCHIVE_MAP.keys()), + pretrained_model_name)) + tokenizer = None + return tokenizer + + +class BasicTokenizer(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" + + def __init__(self, do_lower_case=True): + """Constructs a BasicTokenizer. + + Args: + do_lower_case: Whether to lower case the input. + """ + self.do_lower_case = do_lower_case + + def tokenize(self, text): + """Tokenizes a piece of text.""" + text = convert_to_unicode(text) + text = self._clean_text(text) + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + text = self._tokenize_chinese_chars(text) + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case: + token = token.lower() + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token)) + + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(" ") + output.append(char) + output.append(" ") + else: + output.append(char) + return "".join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ((cp >= 0x4E00 and cp <= 0x9FFF) or # + (cp >= 0x3400 and cp <= 0x4DBF) or # + (cp >= 0x20000 and cp <= 0x2A6DF) or # + (cp >= 0x2A700 and cp <= 0x2B73F) or # + (cp >= 0x2B740 and cp <= 0x2B81F) or # + (cp >= 0x2B820 and cp <= 0x2CEAF) or + (cp >= 0xF900 and cp <= 0xFAFF) or # + (cp >= 0x2F800 and cp <= 0x2FA1F)): # + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xfffd or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenization.""" + + def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=100): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + + For example: + input = "unaffable" + output = ["un", "##aff", "##able"] + + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer. + + Returns: + A list of wordpiece tokens. + """ + + text = convert_to_unicode(text) + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = "".join(chars[start:end]) + if start > 0: + substr = "##" + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically contorl characters but we treat them + # as whitespace since they are generally considered as such. + if char == " " or char == "\t" or char == "\n" or char == "\r": + return True + cat = unicodedata.category(char) + if cat == "Zs": + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == "\t" or char == "\n" or char == "\r": + return False + cat = unicodedata.category(char) + if cat.startswith("C"): + return True + return False + + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or + (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + return True + cat = unicodedata.category(char) + if cat.startswith("P"): + return True + return False diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/utils.py b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/utils.py new file mode 100644 index 0000000..3ac12ea --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/code/pretrain/utils.py @@ -0,0 +1,62 @@ +import tensorflow as tf +import time + +# report latency and throughput during eval +class LogEvalRunHook(tf.train.SessionRunHook): + def __init__(self, global_batch_size, hvd_rank=-1): + self.global_batch_size = global_batch_size + self.hvd_rank = hvd_rank + self.total_time = 0.0 + self.count = 0 + self.skipped = 0 + self.time_list = [] + + def before_run(self, run_context): + self.t0 = time.time() + + def after_run(self, run_context, run_values): + elapsed_secs = time.time() - self.t0 + self.count += 1 + + # Removing first 2 (arbitrary) number of startup iterations from perf evaluations + if self.count <= 2: + print("Skipping time record for ", self.count, " due to overhead") + self.skipped += 1 + else: + self.time_list.append(elapsed_secs) + self.total_time += elapsed_secs + +# report throughput during training +class LogTrainRunHook(tf.train.SessionRunHook): + def __init__(self, global_batch_size, hvd_rank=-1, save_checkpoints_steps=1000): + self.global_batch_size = global_batch_size + self.hvd_rank = hvd_rank + self.save_checkpoints_steps = save_checkpoints_steps + + self.total_time = 0.0 + self.count = 0 # Holds number of iterations, including skipped iterations for fp16 loss scaling + + def after_create_session(self, session, coord): + self.init_global_step = session.run(tf.train.get_global_step()) + + def before_run(self, run_context): + self.t0 = time.time() + return tf.train.SessionRunArgs( + fetches=['step_update:0']) + + def after_run(self, run_context, run_values): + elapsed_secs = time.time() - self.t0 + self.global_step = run_values.results[0] + self.count += 1 + + # Removing first step + first two steps after every checkpoint save + if (self.global_step - self.init_global_step) % self.save_checkpoints_steps <= 1: + print("Skipping time record for ", self.global_step, " due to checkpoint-saving/warmup overhead") + else: + self.total_time += elapsed_secs + + def end(self, session): + num_global_steps = self.global_step - self.init_global_step + + self.skipped = (num_global_steps // self.save_checkpoints_steps) * 2 + \ + min(2, num_global_steps % self.save_checkpoints_steps) - 1 \ No newline at end of file diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/1p.json b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/1p.json new file mode 100644 index 0000000..5a5ef1d --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/1p.json @@ -0,0 +1,14 @@ +{ + "server_count": "1", + "server_list": [{ + "device": [ + { + "device_id": "0", + "device_ip": "192.168.10.101", + "rank_id": "0" + }], + "server_id": "127.0.0.1" + }], + "status": "completed", + "version": "1.0" +} diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/8p.json b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/8p.json new file mode 100644 index 0000000..6ba12b6 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/8p.json @@ -0,0 +1,49 @@ +{ + "server_count": "1", + "server_list": [{ + "device": [ + { + "device_id": "0", + "device_ip": "192.168.10.101", + "rank_id": "0" + }, + { + "device_id": "1", + "device_ip": "192.168.11.101", + "rank_id": "1" + }, + { + "device_id": "2", + "device_ip": "192.168.12.101", + "rank_id": "2" + }, + { + "device_id": "3", + "device_ip": "192.168.13.101", + "rank_id": "3" + }, + { + "device_id": "4", + "device_ip": "192.168.10.100", + "rank_id": "4" + }, + { + "device_id": "5", + "device_ip": "192.168.11.100", + "rank_id": "5" + }, + { + "device_id": "6", + "device_ip": "192.168.12.100", + "rank_id": "6" + }, + { + "device_id": "7", + "device_ip": "192.168.13.100", + "rank_id": "7" + }], + "server_id": "127.0.0.1" + }], + "status": "completed", + "version": "1.0" +} diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_layer12_cn.json b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_layer12_cn.json new file mode 100644 index 0000000..f3456bc --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_layer12_cn.json @@ -0,0 +1,14 @@ +{ + "attention_probs_dropout_prob": 0.1, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 768, + "initializer_range": 0.02, + "intermediate_size": 3072, + "max_position_embeddings": 512, + "num_attention_heads": 12, + "num_hidden_layers": 12, + "type_vocab_size": 2, + "vocab_size": 21136 +} + diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_layer12_en.json b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_layer12_en.json new file mode 100644 index 0000000..d832c32 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_layer12_en.json @@ -0,0 +1,14 @@ +{ + "attention_probs_dropout_prob": 0.1, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 768, + "initializer_range": 0.02, + "intermediate_size": 3072, + "max_position_embeddings": 512, + "num_attention_heads": 12, + "num_hidden_layers": 12, + "type_vocab_size": 2, + "vocab_size": 30522 +} + diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_layer6_cn.json b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_layer6_cn.json new file mode 100644 index 0000000..6ad4cbb --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_layer6_cn.json @@ -0,0 +1,14 @@ +{ + "attention_probs_dropout_prob": 0.1, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 768, + "initializer_range": 0.02, + "intermediate_size": 3072, + "max_position_embeddings": 512, + "num_attention_heads": 12, + "num_hidden_layers": 6, + "type_vocab_size": 2, + "vocab_size": 21136 +} + diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_layer6_en.json b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_layer6_en.json new file mode 100644 index 0000000..69ef27f --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_layer6_en.json @@ -0,0 +1,14 @@ +{ + "attention_probs_dropout_prob": 0.1, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 768, + "initializer_range": 0.02, + "intermediate_size": 3072, + "max_position_embeddings": 512, + "num_attention_heads": 12, + "num_hidden_layers": 6, + "type_vocab_size": 2, + "vocab_size": 30522 +} + diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_vocab.txt b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_vocab.txt new file mode 100644 index 0000000..ca4f978 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/bert_base_vocab.txt @@ -0,0 +1,21128 @@ +[PAD] +[unused1] +[unused2] +[unused3] +[unused4] +[unused5] +[unused6] +[unused7] +[unused8] +[unused9] +[unused10] +[unused11] +[unused12] +[unused13] +[unused14] +[unused15] +[unused16] +[unused17] +[unused18] +[unused19] +[unused20] +[unused21] +[unused22] +[unused23] +[unused24] +[unused25] +[unused26] +[unused27] +[unused28] +[unused29] +[unused30] +[unused31] +[unused32] +[unused33] +[unused34] +[unused35] +[unused36] +[unused37] +[unused38] +[unused39] +[unused40] +[unused41] +[unused42] +[unused43] +[unused44] +[unused45] +[unused46] +[unused47] +[unused48] +[unused49] +[unused50] +[unused51] +[unused52] +[unused53] +[unused54] +[unused55] +[unused56] +[unused57] +[unused58] +[unused59] +[unused60] +[unused61] +[unused62] +[unused63] +[unused64] +[unused65] +[unused66] +[unused67] +[unused68] +[unused69] +[unused70] +[unused71] +[unused72] +[unused73] +[unused74] +[unused75] +[unused76] +[unused77] +[unused78] +[unused79] +[unused80] +[unused81] +[unused82] +[unused83] +[unused84] +[unused85] +[unused86] +[unused87] +[unused88] +[unused89] +[unused90] +[unused91] +[unused92] +[unused93] +[unused94] +[unused95] +[unused96] +[unused97] +[unused98] +[unused99] +[UNK] +[CLS] +[SEP] +[MASK] + + +! +" +# +$ +% +& +' +( +) +* ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; +< += +> +? +@ +[ +\ +] +^ +_ +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +{ +| +} +~ +£ +¤ +¥ +§ +© +« +® +° +± +² +³ +µ +· +¹ +º +» +¼ +× +ß +æ +÷ +ø +đ +ŋ +ɔ +ə +ɡ +ʰ +ˇ +ˈ +ˊ +ˋ +ˍ +ː +˙ +˚ +ˢ +α +β +γ +δ +ε +η +θ +ι +κ +λ +μ +ν +ο +π +ρ +ς +σ +τ +υ +φ +χ +ψ +ω +а +б +в +г +д +е +ж +з +и +к +л +м +н +о +п +р +с +т +у +ф +х +ц +ч +ш +ы +ь +я +і +ا +ب +ة +ت +د +ر +س +ع +ل +م +ن +ه +و +ي +۩ +ก +ง +น +ม +ย +ร +อ +า +เ +๑ +་ +ღ +ᄀ +ᄁ +ᄂ +ᄃ +ᄅ +ᄆ +ᄇ +ᄈ +ᄉ +ᄋ +ᄌ +ᄎ +ᄏ +ᄐ +ᄑ +ᄒ +ᅡ +ᅢ +ᅣ +ᅥ +ᅦ +ᅧ +ᅨ +ᅩ +ᅪ +ᅬ +ᅭ +ᅮ +ᅯ +ᅲ +ᅳ +ᅴ +ᅵ +ᆨ +ᆫ +ᆯ +ᆷ +ᆸ +ᆺ +ᆻ +ᆼ +ᗜ +ᵃ +ᵉ +ᵍ +ᵏ +ᵐ +ᵒ +ᵘ +‖ +„ +† +• +‥ +‧ +
 +‰ +′ +″ +‹ +› +※ +‿ +⁄ +ⁱ +⁺ +ⁿ +₁ +₂ +₃ +₄ +€ +℃ +№ +™ +ⅰ +ⅱ +ⅲ +ⅳ +ⅴ +← +↑ +→ +↓ +↔ +↗ +↘ +⇒ +∀ +− +∕ +∙ +√ +∞ +∟ +∠ +∣ +∥ +∩ +∮ +∶ +∼ +∽ +≈ +≒ +≡ +≤ +≥ +≦ +≧ +≪ +≫ +⊙ +⋅ +⋈ +⋯ +⌒ +① +② +③ +④ +⑤ +⑥ +⑦ +⑧ +⑨ +⑩ +⑴ +⑵ +⑶ +⑷ +⑸ +⒈ +⒉ +⒊ +⒋ +ⓒ +ⓔ +ⓘ +─ +━ +│ +┃ +┅ +┆ +┊ +┌ +└ +├ +┣ +═ +║ +╚ +╞ +╠ +╭ +╮ +╯ +╰ +╱ +╳ +▂ +▃ +▅ +▇ +█ +▉ +▋ +▌ +▍ +▎ +■ +□ +▪ +▫ +▬ +▲ +△ +▶ +► +▼ +▽ +◆ +◇ +○ +◎ +● +◕ +◠ +◢ +◤ +☀ +★ +☆ +☕ +☞ +☺ +☼ +♀ +♂ +♠ +♡ +♣ +♥ +♦ +♪ +♫ +♬ +✈ +✔ +✕ +✖ +✦ +✨ +✪ +✰ +✿ +❀ +❤ +➜ +➤ +⦿ +、 +。 +〃 +々 +〇 +〈 +〉 +《 +》 +「 +」 +『 +』 +【 +】 +〓 +〔 +〕 +〖 +〗 +〜 +〝 +〞 +ぁ +あ +ぃ +い +う +ぇ +え +お +か +き +く +け +こ +さ +し +す +せ +そ +た +ち +っ +つ +て +と +な +に +ぬ +ね +の +は +ひ +ふ +へ +ほ +ま +み +む +め +も +ゃ +や +ゅ +ゆ +ょ +よ +ら +り +る +れ +ろ +わ +を +ん +゜ +ゝ +ァ +ア +ィ +イ +ゥ +ウ +ェ +エ +ォ +オ +カ +キ +ク +ケ +コ +サ +シ +ス +セ +ソ +タ +チ +ッ +ツ +テ +ト +ナ +ニ +ヌ +ネ +ノ +ハ +ヒ +フ +ヘ +ホ +マ +ミ +ム +メ +モ +ャ +ヤ +ュ +ユ +ョ +ヨ +ラ +リ +ル +レ +ロ +ワ +ヲ +ン +ヶ +・ +ー +ヽ +ㄅ +ㄆ +ㄇ +ㄉ +ㄋ +ㄌ +ㄍ +ㄎ +ㄏ +ㄒ +ㄚ +ㄛ +ㄞ +ㄟ +ㄢ +ㄤ +ㄥ +ㄧ +ㄨ +ㆍ +㈦ +㊣ +㎡ +㗎 +一 +丁 +七 +万 +丈 +三 +上 +下 +不 +与 +丐 +丑 +专 +且 +丕 +世 +丘 +丙 +业 +丛 +东 +丝 +丞 +丟 +両 +丢 +两 +严 +並 +丧 +丨 +个 +丫 +中 +丰 +串 +临 +丶 +丸 +丹 +为 +主 +丼 +丽 +举 +丿 +乂 +乃 +久 +么 +义 +之 +乌 +乍 +乎 +乏 +乐 +乒 +乓 +乔 +乖 +乗 +乘 +乙 +乜 +九 +乞 +也 +习 +乡 +书 +乩 +买 +乱 +乳 +乾 +亀 +亂 +了 +予 +争 +事 +二 +于 +亏 +云 +互 +五 +井 +亘 +亙 +亚 +些 +亜 +亞 +亟 +亡 +亢 +交 +亥 +亦 +产 +亨 +亩 +享 +京 +亭 +亮 +亲 +亳 +亵 +人 +亿 +什 +仁 +仃 +仄 +仅 +仆 +仇 +今 +介 +仍 +从 +仏 +仑 +仓 +仔 +仕 +他 +仗 +付 +仙 +仝 +仞 +仟 +代 +令 +以 +仨 +仪 +们 +仮 +仰 +仲 +件 +价 +任 +份 +仿 +企 +伉 +伊 +伍 +伎 +伏 +伐 +休 +伕 +众 +优 +伙 +会 +伝 +伞 +伟 +传 +伢 +伤 +伦 +伪 +伫 +伯 +估 +伴 +伶 +伸 +伺 +似 +伽 +佃 +但 +佇 +佈 +位 +低 +住 +佐 +佑 +体 +佔 +何 +佗 +佘 +余 +佚 +佛 +作 +佝 +佞 +佟 +你 +佢 +佣 +佤 +佥 +佩 +佬 +佯 +佰 +佳 +併 +佶 +佻 +佼 +使 +侃 +侄 +來 +侈 +例 +侍 +侏 +侑 +侖 +侗 +供 +依 +侠 +価 +侣 +侥 +侦 +侧 +侨 +侬 +侮 +侯 +侵 +侶 +侷 +便 +係 +促 +俄 +俊 +俎 +俏 +俐 +俑 +俗 +俘 +俚 +保 +俞 +俟 +俠 +信 +俨 +俩 +俪 +俬 +俭 +修 +俯 +俱 +俳 +俸 +俺 +俾 +倆 +倉 +個 +倌 +倍 +倏 +們 +倒 +倔 +倖 +倘 +候 +倚 +倜 +借 +倡 +値 +倦 +倩 +倪 +倫 +倬 +倭 +倶 +债 +值 +倾 +偃 +假 +偈 +偉 +偌 +偎 +偏 +偕 +做 +停 +健 +側 +偵 +偶 +偷 +偻 +偽 +偿 +傀 +傅 +傍 +傑 +傘 +備 +傚 +傢 +傣 +傥 +储 +傩 +催 +傭 +傲 +傳 +債 +傷 +傻 +傾 +僅 +働 +像 +僑 +僕 +僖 +僚 +僥 +僧 +僭 +僮 +僱 +僵 +價 +僻 +儀 +儂 +億 +儆 +儉 +儋 +儒 +儕 +儘 +償 +儡 +優 +儲 +儷 +儼 +儿 +兀 +允 +元 +兄 +充 +兆 +兇 +先 +光 +克 +兌 +免 +児 +兑 +兒 +兔 +兖 +党 +兜 +兢 +入 +內 +全 +兩 +八 +公 +六 +兮 +兰 +共 +兲 +关 +兴 +兵 +其 +具 +典 +兹 +养 +兼 +兽 +冀 +内 +円 +冇 +冈 +冉 +冊 +册 +再 +冏 +冒 +冕 +冗 +写 +军 +农 +冠 +冢 +冤 +冥 +冨 +冪 +冬 +冯 +冰 +冲 +决 +况 +冶 +冷 +冻 +冼 +冽 +冾 +净 +凄 +准 +凇 +凈 +凉 +凋 +凌 +凍 +减 +凑 +凛 +凜 +凝 +几 +凡 +凤 +処 +凪 +凭 +凯 +凰 +凱 +凳 +凶 +凸 +凹 +出 +击 +函 +凿 +刀 +刁 +刃 +分 +切 +刈 +刊 +刍 +刎 +刑 +划 +列 +刘 +则 +刚 +创 +初 +删 +判 +別 +刨 +利 +刪 +别 +刮 +到 +制 +刷 +券 +刹 +刺 +刻 +刽 +剁 +剂 +剃 +則 +剉 +削 +剋 +剌 +前 +剎 +剐 +剑 +剔 +剖 +剛 +剜 +剝 +剣 +剤 +剥 +剧 +剩 +剪 +副 +割 +創 +剷 +剽 +剿 +劃 +劇 +劈 +劉 +劊 +劍 +劏 +劑 +力 +劝 +办 +功 +加 +务 +劣 +动 +助 +努 +劫 +劭 +励 +劲 +劳 +労 +劵 +効 +劾 +势 +勁 +勃 +勇 +勉 +勋 +勐 +勒 +動 +勖 +勘 +務 +勛 +勝 +勞 +募 +勢 +勤 +勧 +勳 +勵 +勸 +勺 +勻 +勾 +勿 +匀 +包 +匆 +匈 +匍 +匐 +匕 +化 +北 +匙 +匝 +匠 +匡 +匣 +匪 +匮 +匯 +匱 +匹 +区 +医 +匾 +匿 +區 +十 +千 +卅 +升 +午 +卉 +半 +卍 +华 +协 +卑 +卒 +卓 +協 +单 +卖 +南 +単 +博 +卜 +卞 +卟 +占 +卡 +卢 +卤 +卦 +卧 +卫 +卮 +卯 +印 +危 +即 +却 +卵 +卷 +卸 +卻 +卿 +厂 +厄 +厅 +历 +厉 +压 +厌 +厕 +厘 +厚 +厝 +原 +厢 +厥 +厦 +厨 +厩 +厭 +厮 +厲 +厳 +去 +县 +叁 +参 +參 +又 +叉 +及 +友 +双 +反 +収 +发 +叔 +取 +受 +变 +叙 +叛 +叟 +叠 +叡 +叢 +口 +古 +句 +另 +叨 +叩 +只 +叫 +召 +叭 +叮 +可 +台 +叱 +史 +右 +叵 +叶 +号 +司 +叹 +叻 +叼 +叽 +吁 +吃 +各 +吆 +合 +吉 +吊 +吋 +同 +名 +后 +吏 +吐 +向 +吒 +吓 +吕 +吖 +吗 +君 +吝 +吞 +吟 +吠 +吡 +否 +吧 +吨 +吩 +含 +听 +吭 +吮 +启 +吱 +吳 +吴 +吵 +吶 +吸 +吹 +吻 +吼 +吽 +吾 +呀 +呂 +呃 +呆 +呈 +告 +呋 +呎 +呐 +呓 +呕 +呗 +员 +呛 +呜 +呢 +呤 +呦 +周 +呱 +呲 +味 +呵 +呷 +呸 +呻 +呼 +命 +咀 +咁 +咂 +咄 +咆 +咋 +和 +咎 +咏 +咐 +咒 +咔 +咕 +咖 +咗 +咘 +咙 +咚 +咛 +咣 +咤 +咦 +咧 +咨 +咩 +咪 +咫 +咬 +咭 +咯 +咱 +咲 +咳 +咸 +咻 +咽 +咿 +哀 +品 +哂 +哄 +哆 +哇 +哈 +哉 +哋 +哌 +响 +哎 +哏 +哐 +哑 +哒 +哔 +哗 +哟 +員 +哥 +哦 +哧 +哨 +哩 +哪 +哭 +哮 +哲 +哺 +哼 +哽 +唁 +唄 +唆 +唇 +唉 +唏 +唐 +唑 +唔 +唠 +唤 +唧 +唬 +售 +唯 +唰 +唱 +唳 +唷 +唸 +唾 +啃 +啄 +商 +啉 +啊 +問 +啓 +啕 +啖 +啜 +啞 +啟 +啡 +啤 +啥 +啦 +啧 +啪 +啫 +啬 +啮 +啰 +啱 +啲 +啵 +啶 +啷 +啸 +啻 +啼 +啾 +喀 +喂 +喃 +善 +喆 +喇 +喉 +喊 +喋 +喎 +喏 +喔 +喘 +喙 +喚 +喜 +喝 +喟 +喧 +喪 +喫 +喬 +單 +喰 +喱 +喲 +喳 +喵 +営 +喷 +喹 +喺 +喻 +喽 +嗅 +嗆 +嗇 +嗎 +嗑 +嗒 +嗓 +嗔 +嗖 +嗚 +嗜 +嗝 +嗟 +嗡 +嗣 +嗤 +嗦 +嗨 +嗪 +嗬 +嗯 +嗰 +嗲 +嗳 +嗶 +嗷 +嗽 +嘀 +嘅 +嘆 +嘈 +嘉 +嘌 +嘍 +嘎 +嘔 +嘖 +嘗 +嘘 +嘚 +嘛 +嘜 +嘞 +嘟 +嘢 +嘣 +嘤 +嘧 +嘩 +嘭 +嘮 +嘯 +嘰 +嘱 +嘲 +嘴 +嘶 +嘸 +嘹 +嘻 +嘿 +噁 +噌 +噎 +噓 +噔 +噗 +噙 +噜 +噠 +噢 +噤 +器 +噩 +噪 +噬 +噱 +噴 +噶 +噸 +噹 +噻 +噼 +嚀 +嚇 +嚎 +嚏 +嚐 +嚓 +嚕 +嚟 +嚣 +嚥 +嚨 +嚮 +嚴 +嚷 +嚼 +囂 +囉 +囊 +囍 +囑 +囔 +囗 +囚 +四 +囝 +回 +囟 +因 +囡 +团 +団 +囤 +囧 +囪 +囫 +园 +困 +囱 +囲 +図 +围 +囹 +固 +国 +图 +囿 +圃 +圄 +圆 +圈 +國 +圍 +圏 +園 +圓 +圖 +團 +圜 +土 +圣 +圧 +在 +圩 +圭 +地 +圳 +场 +圻 +圾 +址 +坂 +均 +坊 +坍 +坎 +坏 +坐 +坑 +块 +坚 +坛 +坝 +坞 +坟 +坠 +坡 +坤 +坦 +坨 +坪 +坯 +坳 +坵 +坷 +垂 +垃 +垄 +型 +垒 +垚 +垛 +垠 +垢 +垣 +垦 +垩 +垫 +垭 +垮 +垵 +埂 +埃 +埋 +城 +埔 +埕 +埗 +域 +埠 +埤 +埵 +執 +埸 +培 +基 +埼 +堀 +堂 +堃 +堅 +堆 +堇 +堑 +堕 +堙 +堡 +堤 +堪 +堯 +堰 +報 +場 +堵 +堺 +堿 +塊 +塌 +塑 +塔 +塗 +塘 +塚 +塞 +塢 +塩 +填 +塬 +塭 +塵 +塾 +墀 +境 +墅 +墉 +墊 +墒 +墓 +増 +墘 +墙 +墜 +增 +墟 +墨 +墩 +墮 +墳 +墻 +墾 +壁 +壅 +壆 +壇 +壊 +壑 +壓 +壕 +壘 +壞 +壟 +壢 +壤 +壩 +士 +壬 +壮 +壯 +声 +売 +壳 +壶 +壹 +壺 +壽 +处 +备 +変 +复 +夏 +夔 +夕 +外 +夙 +多 +夜 +够 +夠 +夢 +夥 +大 +天 +太 +夫 +夭 +央 +夯 +失 +头 +夷 +夸 +夹 +夺 +夾 +奂 +奄 +奇 +奈 +奉 +奋 +奎 +奏 +奐 +契 +奔 +奕 +奖 +套 +奘 +奚 +奠 +奢 +奥 +奧 +奪 +奬 +奮 +女 +奴 +奶 +奸 +她 +好 +如 +妃 +妄 +妆 +妇 +妈 +妊 +妍 +妒 +妓 +妖 +妘 +妙 +妝 +妞 +妣 +妤 +妥 +妨 +妩 +妪 +妮 +妲 +妳 +妹 +妻 +妾 +姆 +姉 +姊 +始 +姍 +姐 +姑 +姒 +姓 +委 +姗 +姚 +姜 +姝 +姣 +姥 +姦 +姨 +姪 +姫 +姬 +姹 +姻 +姿 +威 +娃 +娄 +娅 +娆 +娇 +娉 +娑 +娓 +娘 +娛 +娜 +娟 +娠 +娣 +娥 +娩 +娱 +娲 +娴 +娶 +娼 +婀 +婁 +婆 +婉 +婊 +婕 +婚 +婢 +婦 +婧 +婪 +婭 +婴 +婵 +婶 +婷 +婺 +婿 +媒 +媚 +媛 +媞 +媧 +媲 +媳 +媽 +媾 +嫁 +嫂 +嫉 +嫌 +嫑 +嫔 +嫖 +嫘 +嫚 +嫡 +嫣 +嫦 +嫩 +嫲 +嫵 +嫻 +嬅 +嬉 +嬌 +嬗 +嬛 +嬢 +嬤 +嬪 +嬰 +嬴 +嬷 +嬸 +嬿 +孀 +孃 +子 +孑 +孔 +孕 +孖 +字 +存 +孙 +孚 +孛 +孜 +孝 +孟 +孢 +季 +孤 +学 +孩 +孪 +孫 +孬 +孰 +孱 +孳 +孵 +學 +孺 +孽 +孿 +宁 +它 +宅 +宇 +守 +安 +宋 +完 +宏 +宓 +宕 +宗 +官 +宙 +定 +宛 +宜 +宝 +实 +実 +宠 +审 +客 +宣 +室 +宥 +宦 +宪 +宫 +宮 +宰 +害 +宴 +宵 +家 +宸 +容 +宽 +宾 +宿 +寂 +寄 +寅 +密 +寇 +富 +寐 +寒 +寓 +寛 +寝 +寞 +察 +寡 +寢 +寥 +實 +寧 +寨 +審 +寫 +寬 +寮 +寰 +寵 +寶 +寸 +对 +寺 +寻 +导 +対 +寿 +封 +専 +射 +将 +將 +專 +尉 +尊 +尋 +對 +導 +小 +少 +尔 +尕 +尖 +尘 +尚 +尝 +尤 +尧 +尬 +就 +尴 +尷 +尸 +尹 +尺 +尻 +尼 +尽 +尾 +尿 +局 +屁 +层 +屄 +居 +屆 +屈 +屉 +届 +屋 +屌 +屍 +屎 +屏 +屐 +屑 +展 +屜 +属 +屠 +屡 +屢 +層 +履 +屬 +屯 +山 +屹 +屿 +岀 +岁 +岂 +岌 +岐 +岑 +岔 +岖 +岗 +岘 +岙 +岚 +岛 +岡 +岩 +岫 +岬 +岭 +岱 +岳 +岷 +岸 +峇 +峋 +峒 +峙 +峡 +峤 +峥 +峦 +峨 +峪 +峭 +峯 +峰 +峴 +島 +峻 +峽 +崁 +崂 +崆 +崇 +崎 +崑 +崔 +崖 +崗 +崙 +崛 +崧 +崩 +崭 +崴 +崽 +嵇 +嵊 +嵋 +嵌 +嵐 +嵘 +嵩 +嵬 +嵯 +嶂 +嶄 +嶇 +嶋 +嶙 +嶺 +嶼 +嶽 +巅 +巍 +巒 +巔 +巖 +川 +州 +巡 +巢 +工 +左 +巧 +巨 +巩 +巫 +差 +己 +已 +巳 +巴 +巷 +巻 +巽 +巾 +巿 +币 +市 +布 +帅 +帆 +师 +希 +帐 +帑 +帕 +帖 +帘 +帚 +帛 +帜 +帝 +帥 +带 +帧 +師 +席 +帮 +帯 +帰 +帳 +帶 +帷 +常 +帼 +帽 +幀 +幂 +幄 +幅 +幌 +幔 +幕 +幟 +幡 +幢 +幣 +幫 +干 +平 +年 +并 +幸 +幹 +幺 +幻 +幼 +幽 +幾 +广 +庁 +広 +庄 +庆 +庇 +床 +序 +庐 +库 +应 +底 +庖 +店 +庙 +庚 +府 +庞 +废 +庠 +度 +座 +庫 +庭 +庵 +庶 +康 +庸 +庹 +庾 +廁 +廂 +廃 +廈 +廉 +廊 +廓 +廖 +廚 +廝 +廟 +廠 +廢 +廣 +廬 +廳 +延 +廷 +建 +廿 +开 +弁 +异 +弃 +弄 +弈 +弊 +弋 +式 +弑 +弒 +弓 +弔 +引 +弗 +弘 +弛 +弟 +张 +弥 +弦 +弧 +弩 +弭 +弯 +弱 +張 +強 +弹 +强 +弼 +弾 +彅 +彆 +彈 +彌 +彎 +归 +当 +录 +彗 +彙 +彝 +形 +彤 +彥 +彦 +彧 +彩 +彪 +彫 +彬 +彭 +彰 +影 +彷 +役 +彻 +彼 +彿 +往 +征 +径 +待 +徇 +很 +徉 +徊 +律 +後 +徐 +徑 +徒 +従 +徕 +得 +徘 +徙 +徜 +從 +徠 +御 +徨 +復 +循 +徬 +微 +徳 +徴 +徵 +德 +徹 +徼 +徽 +心 +必 +忆 +忌 +忍 +忏 +忐 +忑 +忒 +忖 +志 +忘 +忙 +応 +忠 +忡 +忤 +忧 +忪 +快 +忱 +念 +忻 +忽 +忿 +怀 +态 +怂 +怅 +怆 +怎 +怏 +怒 +怔 +怕 +怖 +怙 +怜 +思 +怠 +怡 +急 +怦 +性 +怨 +怪 +怯 +怵 +总 +怼 +恁 +恃 +恆 +恋 +恍 +恐 +恒 +恕 +恙 +恚 +恢 +恣 +恤 +恥 +恨 +恩 +恪 +恫 +恬 +恭 +息 +恰 +恳 +恵 +恶 +恸 +恺 +恻 +恼 +恿 +悄 +悅 +悉 +悌 +悍 +悔 +悖 +悚 +悟 +悠 +患 +悦 +您 +悩 +悪 +悬 +悯 +悱 +悲 +悴 +悵 +悶 +悸 +悻 +悼 +悽 +情 +惆 +惇 +惊 +惋 +惑 +惕 +惘 +惚 +惜 +惟 +惠 +惡 +惦 +惧 +惨 +惩 +惫 +惬 +惭 +惮 +惯 +惰 +惱 +想 +惴 +惶 +惹 +惺 +愁 +愆 +愈 +愉 +愍 +意 +愕 +愚 +愛 +愜 +感 +愣 +愤 +愧 +愫 +愷 +愿 +慄 +慈 +態 +慌 +慎 +慑 +慕 +慘 +慚 +慟 +慢 +慣 +慧 +慨 +慫 +慮 +慰 +慳 +慵 +慶 +慷 +慾 +憂 +憊 +憋 +憎 +憐 +憑 +憔 +憚 +憤 +憧 +憨 +憩 +憫 +憬 +憲 +憶 +憾 +懂 +懇 +懈 +應 +懊 +懋 +懑 +懒 +懦 +懲 +懵 +懶 +懷 +懸 +懺 +懼 +懾 +懿 +戀 +戈 +戊 +戌 +戍 +戎 +戏 +成 +我 +戒 +戕 +或 +战 +戚 +戛 +戟 +戡 +戦 +截 +戬 +戮 +戰 +戲 +戳 +戴 +戶 +户 +戸 +戻 +戾 +房 +所 +扁 +扇 +扈 +扉 +手 +才 +扎 +扑 +扒 +打 +扔 +払 +托 +扛 +扣 +扦 +执 +扩 +扪 +扫 +扬 +扭 +扮 +扯 +扰 +扱 +扳 +扶 +批 +扼 +找 +承 +技 +抄 +抉 +把 +抑 +抒 +抓 +投 +抖 +抗 +折 +抚 +抛 +抜 +択 +抟 +抠 +抡 +抢 +护 +报 +抨 +披 +抬 +抱 +抵 +抹 +押 +抽 +抿 +拂 +拄 +担 +拆 +拇 +拈 +拉 +拋 +拌 +拍 +拎 +拐 +拒 +拓 +拔 +拖 +拗 +拘 +拙 +拚 +招 +拜 +拟 +拡 +拢 +拣 +拥 +拦 +拧 +拨 +择 +括 +拭 +拮 +拯 +拱 +拳 +拴 +拷 +拼 +拽 +拾 +拿 +持 +挂 +指 +挈 +按 +挎 +挑 +挖 +挙 +挚 +挛 +挝 +挞 +挟 +挠 +挡 +挣 +挤 +挥 +挨 +挪 +挫 +振 +挲 +挹 +挺 +挽 +挾 +捂 +捅 +捆 +捉 +捋 +捌 +捍 +捎 +捏 +捐 +捕 +捞 +损 +捡 +换 +捣 +捧 +捨 +捩 +据 +捱 +捲 +捶 +捷 +捺 +捻 +掀 +掂 +掃 +掇 +授 +掉 +掌 +掏 +掐 +排 +掖 +掘 +掙 +掛 +掠 +採 +探 +掣 +接 +控 +推 +掩 +措 +掬 +掰 +掲 +掳 +掴 +掷 +掸 +掺 +揀 +揃 +揄 +揆 +揉 +揍 +描 +提 +插 +揖 +揚 +換 +握 +揣 +揩 +揪 +揭 +揮 +援 +揶 +揸 +揹 +揽 +搀 +搁 +搂 +搅 +損 +搏 +搐 +搓 +搔 +搖 +搗 +搜 +搞 +搡 +搪 +搬 +搭 +搵 +搶 +携 +搽 +摀 +摁 +摄 +摆 +摇 +摈 +摊 +摒 +摔 +摘 +摞 +摟 +摧 +摩 +摯 +摳 +摸 +摹 +摺 +摻 +撂 +撃 +撅 +撇 +撈 +撐 +撑 +撒 +撓 +撕 +撚 +撞 +撤 +撥 +撩 +撫 +撬 +播 +撮 +撰 +撲 +撵 +撷 +撸 +撻 +撼 +撿 +擀 +擁 +擂 +擄 +擅 +擇 +擊 +擋 +操 +擎 +擒 +擔 +擘 +據 +擞 +擠 +擡 +擢 +擦 +擬 +擰 +擱 +擲 +擴 +擷 +擺 +擼 +擾 +攀 +攏 +攒 +攔 +攘 +攙 +攜 +攝 +攞 +攢 +攣 +攤 +攥 +攪 +攫 +攬 +支 +收 +攸 +改 +攻 +放 +政 +故 +效 +敌 +敍 +敎 +敏 +救 +敕 +敖 +敗 +敘 +教 +敛 +敝 +敞 +敢 +散 +敦 +敬 +数 +敲 +整 +敵 +敷 +數 +斂 +斃 +文 +斋 +斌 +斎 +斐 +斑 +斓 +斗 +料 +斛 +斜 +斟 +斡 +斤 +斥 +斧 +斩 +斫 +斬 +断 +斯 +新 +斷 +方 +於 +施 +旁 +旃 +旅 +旋 +旌 +旎 +族 +旖 +旗 +无 +既 +日 +旦 +旧 +旨 +早 +旬 +旭 +旮 +旱 +时 +旷 +旺 +旻 +昀 +昂 +昆 +昇 +昉 +昊 +昌 +明 +昏 +易 +昔 +昕 +昙 +星 +映 +春 +昧 +昨 +昭 +是 +昱 +昴 +昵 +昶 +昼 +显 +晁 +時 +晃 +晉 +晋 +晌 +晏 +晒 +晓 +晔 +晕 +晖 +晗 +晚 +晝 +晞 +晟 +晤 +晦 +晨 +晩 +普 +景 +晰 +晴 +晶 +晷 +智 +晾 +暂 +暄 +暇 +暈 +暉 +暌 +暐 +暑 +暖 +暗 +暝 +暢 +暧 +暨 +暫 +暮 +暱 +暴 +暸 +暹 +曄 +曆 +曇 +曉 +曖 +曙 +曜 +曝 +曠 +曦 +曬 +曰 +曲 +曳 +更 +書 +曹 +曼 +曾 +替 +最 +會 +月 +有 +朋 +服 +朐 +朔 +朕 +朗 +望 +朝 +期 +朦 +朧 +木 +未 +末 +本 +札 +朮 +术 +朱 +朴 +朵 +机 +朽 +杀 +杂 +权 +杆 +杈 +杉 +李 +杏 +材 +村 +杓 +杖 +杜 +杞 +束 +杠 +条 +来 +杨 +杭 +杯 +杰 +東 +杳 +杵 +杷 +杼 +松 +板 +极 +构 +枇 +枉 +枋 +析 +枕 +林 +枚 +果 +枝 +枢 +枣 +枪 +枫 +枭 +枯 +枰 +枱 +枳 +架 +枷 +枸 +柄 +柏 +某 +柑 +柒 +染 +柔 +柘 +柚 +柜 +柞 +柠 +柢 +查 +柩 +柬 +柯 +柱 +柳 +柴 +柵 +査 +柿 +栀 +栃 +栄 +栅 +标 +栈 +栉 +栋 +栎 +栏 +树 +栓 +栖 +栗 +校 +栩 +株 +样 +核 +根 +格 +栽 +栾 +桀 +桁 +桂 +桃 +桅 +框 +案 +桉 +桌 +桎 +桐 +桑 +桓 +桔 +桜 +桠 +桡 +桢 +档 +桥 +桦 +桧 +桨 +桩 +桶 +桿 +梁 +梅 +梆 +梏 +梓 +梗 +條 +梟 +梢 +梦 +梧 +梨 +梭 +梯 +械 +梳 +梵 +梶 +检 +棂 +棄 +棉 +棋 +棍 +棒 +棕 +棗 +棘 +棚 +棟 +棠 +棣 +棧 +森 +棱 +棲 +棵 +棹 +棺 +椁 +椅 +椋 +植 +椎 +椒 +検 +椪 +椭 +椰 +椹 +椽 +椿 +楂 +楊 +楓 +楔 +楚 +楝 +楞 +楠 +楣 +楨 +楫 +業 +楮 +極 +楷 +楸 +楹 +楼 +楽 +概 +榄 +榆 +榈 +榉 +榔 +榕 +榖 +榛 +榜 +榨 +榫 +榭 +榮 +榱 +榴 +榷 +榻 +槁 +槃 +構 +槌 +槍 +槎 +槐 +槓 +様 +槛 +槟 +槤 +槭 +槲 +槳 +槻 +槽 +槿 +樁 +樂 +樊 +樑 +樓 +標 +樞 +樟 +模 +樣 +権 +横 +樫 +樯 +樱 +樵 +樸 +樹 +樺 +樽 +樾 +橄 +橇 +橋 +橐 +橘 +橙 +機 +橡 +橢 +橫 +橱 +橹 +橼 +檀 +檄 +檎 +檐 +檔 +檗 +檜 +檢 +檬 +檯 +檳 +檸 +檻 +櫃 +櫚 +櫛 +櫥 +櫸 +櫻 +欄 +權 +欒 +欖 +欠 +次 +欢 +欣 +欧 +欲 +欸 +欺 +欽 +款 +歆 +歇 +歉 +歌 +歎 +歐 +歓 +歙 +歛 +歡 +止 +正 +此 +步 +武 +歧 +歩 +歪 +歯 +歲 +歳 +歴 +歷 +歸 +歹 +死 +歼 +殁 +殃 +殆 +殇 +殉 +殊 +残 +殒 +殓 +殖 +殘 +殞 +殡 +殤 +殭 +殯 +殲 +殴 +段 +殷 +殺 +殼 +殿 +毀 +毁 +毂 +毅 +毆 +毋 +母 +毎 +每 +毒 +毓 +比 +毕 +毗 +毘 +毙 +毛 +毡 +毫 +毯 +毽 +氈 +氏 +氐 +民 +氓 +气 +氖 +気 +氙 +氛 +氟 +氡 +氢 +氣 +氤 +氦 +氧 +氨 +氪 +氫 +氮 +氯 +氰 +氲 +水 +氷 +永 +氹 +氾 +汀 +汁 +求 +汆 +汇 +汉 +汎 +汐 +汕 +汗 +汙 +汛 +汝 +汞 +江 +池 +污 +汤 +汨 +汩 +汪 +汰 +汲 +汴 +汶 +汹 +決 +汽 +汾 +沁 +沂 +沃 +沅 +沈 +沉 +沌 +沏 +沐 +沒 +沓 +沖 +沙 +沛 +沟 +没 +沢 +沣 +沥 +沦 +沧 +沪 +沫 +沭 +沮 +沱 +河 +沸 +油 +治 +沼 +沽 +沾 +沿 +況 +泄 +泉 +泊 +泌 +泓 +法 +泗 +泛 +泞 +泠 +泡 +波 +泣 +泥 +注 +泪 +泫 +泮 +泯 +泰 +泱 +泳 +泵 +泷 +泸 +泻 +泼 +泽 +泾 +洁 +洄 +洋 +洒 +洗 +洙 +洛 +洞 +津 +洩 +洪 +洮 +洱 +洲 +洵 +洶 +洸 +洹 +活 +洼 +洽 +派 +流 +浃 +浄 +浅 +浆 +浇 +浊 +测 +济 +浏 +浑 +浒 +浓 +浔 +浙 +浚 +浜 +浣 +浦 +浩 +浪 +浬 +浮 +浯 +浴 +海 +浸 +涂 +涅 +涇 +消 +涉 +涌 +涎 +涓 +涔 +涕 +涙 +涛 +涝 +涞 +涟 +涠 +涡 +涣 +涤 +润 +涧 +涨 +涩 +涪 +涮 +涯 +液 +涵 +涸 +涼 +涿 +淀 +淄 +淅 +淆 +淇 +淋 +淌 +淑 +淒 +淖 +淘 +淙 +淚 +淞 +淡 +淤 +淦 +淨 +淩 +淪 +淫 +淬 +淮 +深 +淳 +淵 +混 +淹 +淺 +添 +淼 +清 +済 +渉 +渊 +渋 +渍 +渎 +渐 +渔 +渗 +渙 +渚 +減 +渝 +渠 +渡 +渣 +渤 +渥 +渦 +温 +測 +渭 +港 +渲 +渴 +游 +渺 +渾 +湃 +湄 +湊 +湍 +湖 +湘 +湛 +湟 +湧 +湫 +湮 +湯 +湳 +湾 +湿 +満 +溃 +溅 +溉 +溏 +源 +準 +溜 +溝 +溟 +溢 +溥 +溧 +溪 +溫 +溯 +溱 +溴 +溶 +溺 +溼 +滁 +滂 +滄 +滅 +滇 +滋 +滌 +滑 +滓 +滔 +滕 +滙 +滚 +滝 +滞 +滟 +满 +滢 +滤 +滥 +滦 +滨 +滩 +滬 +滯 +滲 +滴 +滷 +滸 +滾 +滿 +漁 +漂 +漆 +漉 +漏 +漓 +演 +漕 +漠 +漢 +漣 +漩 +漪 +漫 +漬 +漯 +漱 +漲 +漳 +漸 +漾 +漿 +潆 +潇 +潋 +潍 +潑 +潔 +潘 +潛 +潜 +潞 +潟 +潢 +潤 +潦 +潧 +潭 +潮 +潰 +潴 +潸 +潺 +潼 +澀 +澄 +澆 +澈 +澍 +澎 +澗 +澜 +澡 +澤 +澧 +澱 +澳 +澹 +激 +濁 +濂 +濃 +濑 +濒 +濕 +濘 +濛 +濟 +濠 +濡 +濤 +濫 +濬 +濮 +濯 +濱 +濺 +濾 +瀅 +瀆 +瀉 +瀋 +瀏 +瀑 +瀕 +瀘 +瀚 +瀛 +瀝 +瀞 +瀟 +瀧 +瀨 +瀬 +瀰 +瀾 +灌 +灏 +灑 +灘 +灝 +灞 +灣 +火 +灬 +灭 +灯 +灰 +灵 +灶 +灸 +灼 +災 +灾 +灿 +炀 +炁 +炅 +炉 +炊 +炎 +炒 +炔 +炕 +炖 +炙 +炜 +炫 +炬 +炭 +炮 +炯 +炳 +炷 +炸 +点 +為 +炼 +炽 +烁 +烂 +烃 +烈 +烊 +烏 +烘 +烙 +烛 +烟 +烤 +烦 +烧 +烨 +烩 +烫 +烬 +热 +烯 +烷 +烹 +烽 +焉 +焊 +焕 +焖 +焗 +焘 +焙 +焚 +焜 +無 +焦 +焯 +焰 +焱 +然 +焼 +煅 +煉 +煊 +煌 +煎 +煒 +煖 +煙 +煜 +煞 +煤 +煥 +煦 +照 +煨 +煩 +煮 +煲 +煸 +煽 +熄 +熊 +熏 +熒 +熔 +熙 +熟 +熠 +熨 +熬 +熱 +熵 +熹 +熾 +燁 +燃 +燄 +燈 +燉 +燊 +燎 +燒 +燔 +燕 +燙 +燜 +營 +燥 +燦 +燧 +燭 +燮 +燴 +燻 +燼 +燿 +爆 +爍 +爐 +爛 +爪 +爬 +爭 +爰 +爱 +爲 +爵 +父 +爷 +爸 +爹 +爺 +爻 +爽 +爾 +牆 +片 +版 +牌 +牍 +牒 +牙 +牛 +牝 +牟 +牠 +牡 +牢 +牦 +牧 +物 +牯 +牲 +牴 +牵 +特 +牺 +牽 +犀 +犁 +犄 +犊 +犍 +犒 +犢 +犧 +犬 +犯 +状 +犷 +犸 +犹 +狀 +狂 +狄 +狈 +狎 +狐 +狒 +狗 +狙 +狞 +狠 +狡 +狩 +独 +狭 +狮 +狰 +狱 +狸 +狹 +狼 +狽 +猎 +猕 +猖 +猗 +猙 +猛 +猜 +猝 +猥 +猩 +猪 +猫 +猬 +献 +猴 +猶 +猷 +猾 +猿 +獄 +獅 +獎 +獐 +獒 +獗 +獠 +獣 +獨 +獭 +獰 +獲 +獵 +獷 +獸 +獺 +獻 +獼 +獾 +玄 +率 +玉 +王 +玑 +玖 +玛 +玟 +玠 +玥 +玩 +玫 +玮 +环 +现 +玲 +玳 +玷 +玺 +玻 +珀 +珂 +珅 +珈 +珉 +珊 +珍 +珏 +珐 +珑 +珙 +珞 +珠 +珣 +珥 +珩 +珪 +班 +珮 +珲 +珺 +現 +球 +琅 +理 +琇 +琉 +琊 +琍 +琏 +琐 +琛 +琢 +琥 +琦 +琨 +琪 +琬 +琮 +琰 +琲 +琳 +琴 +琵 +琶 +琺 +琼 +瑀 +瑁 +瑄 +瑋 +瑕 +瑗 +瑙 +瑚 +瑛 +瑜 +瑞 +瑟 +瑠 +瑣 +瑤 +瑩 +瑪 +瑯 +瑰 +瑶 +瑾 +璀 +璁 +璃 +璇 +璉 +璋 +璎 +璐 +璜 +璞 +璟 +璧 +璨 +環 +璽 +璿 +瓊 +瓏 +瓒 +瓜 +瓢 +瓣 +瓤 +瓦 +瓮 +瓯 +瓴 +瓶 +瓷 +甄 +甌 +甕 +甘 +甙 +甚 +甜 +生 +產 +産 +甥 +甦 +用 +甩 +甫 +甬 +甭 +甯 +田 +由 +甲 +申 +电 +男 +甸 +町 +画 +甾 +畀 +畅 +界 +畏 +畑 +畔 +留 +畜 +畝 +畢 +略 +畦 +番 +畫 +異 +畲 +畳 +畴 +當 +畸 +畹 +畿 +疆 +疇 +疊 +疏 +疑 +疔 +疖 +疗 +疙 +疚 +疝 +疟 +疡 +疣 +疤 +疥 +疫 +疮 +疯 +疱 +疲 +疳 +疵 +疸 +疹 +疼 +疽 +疾 +痂 +病 +症 +痈 +痉 +痊 +痍 +痒 +痔 +痕 +痘 +痙 +痛 +痞 +痠 +痢 +痣 +痤 +痧 +痨 +痪 +痫 +痰 +痱 +痴 +痹 +痺 +痼 +痿 +瘀 +瘁 +瘋 +瘍 +瘓 +瘘 +瘙 +瘟 +瘠 +瘡 +瘢 +瘤 +瘦 +瘧 +瘩 +瘪 +瘫 +瘴 +瘸 +瘾 +療 +癇 +癌 +癒 +癖 +癜 +癞 +癡 +癢 +癣 +癥 +癫 +癬 +癮 +癱 +癲 +癸 +発 +登 +發 +白 +百 +皂 +的 +皆 +皇 +皈 +皋 +皎 +皑 +皓 +皖 +皙 +皚 +皮 +皰 +皱 +皴 +皺 +皿 +盂 +盃 +盅 +盆 +盈 +益 +盎 +盏 +盐 +监 +盒 +盔 +盖 +盗 +盘 +盛 +盜 +盞 +盟 +盡 +監 +盤 +盥 +盧 +盪 +目 +盯 +盱 +盲 +直 +相 +盹 +盼 +盾 +省 +眈 +眉 +看 +県 +眙 +眞 +真 +眠 +眦 +眨 +眩 +眯 +眶 +眷 +眸 +眺 +眼 +眾 +着 +睁 +睇 +睏 +睐 +睑 +睛 +睜 +睞 +睡 +睢 +督 +睥 +睦 +睨 +睪 +睫 +睬 +睹 +睽 +睾 +睿 +瞄 +瞅 +瞇 +瞋 +瞌 +瞎 +瞑 +瞒 +瞓 +瞞 +瞟 +瞠 +瞥 +瞧 +瞩 +瞪 +瞬 +瞭 +瞰 +瞳 +瞻 +瞼 +瞿 +矇 +矍 +矗 +矚 +矛 +矜 +矢 +矣 +知 +矩 +矫 +短 +矮 +矯 +石 +矶 +矽 +矾 +矿 +码 +砂 +砌 +砍 +砒 +研 +砖 +砗 +砚 +砝 +砣 +砥 +砧 +砭 +砰 +砲 +破 +砷 +砸 +砺 +砼 +砾 +础 +硅 +硐 +硒 +硕 +硝 +硫 +硬 +确 +硯 +硼 +碁 +碇 +碉 +碌 +碍 +碎 +碑 +碓 +碗 +碘 +碚 +碛 +碟 +碣 +碧 +碩 +碰 +碱 +碳 +碴 +確 +碼 +碾 +磁 +磅 +磊 +磋 +磐 +磕 +磚 +磡 +磨 +磬 +磯 +磲 +磷 +磺 +礁 +礎 +礙 +礡 +礦 +礪 +礫 +礴 +示 +礼 +社 +祀 +祁 +祂 +祇 +祈 +祉 +祎 +祐 +祕 +祖 +祗 +祚 +祛 +祜 +祝 +神 +祟 +祠 +祢 +祥 +票 +祭 +祯 +祷 +祸 +祺 +祿 +禀 +禁 +禄 +禅 +禍 +禎 +福 +禛 +禦 +禧 +禪 +禮 +禱 +禹 +禺 +离 +禽 +禾 +禿 +秀 +私 +秃 +秆 +秉 +秋 +种 +科 +秒 +秘 +租 +秣 +秤 +秦 +秧 +秩 +秭 +积 +称 +秸 +移 +秽 +稀 +稅 +程 +稍 +税 +稔 +稗 +稚 +稜 +稞 +稟 +稠 +稣 +種 +稱 +稲 +稳 +稷 +稹 +稻 +稼 +稽 +稿 +穀 +穂 +穆 +穌 +積 +穎 +穗 +穢 +穩 +穫 +穴 +究 +穷 +穹 +空 +穿 +突 +窃 +窄 +窈 +窍 +窑 +窒 +窓 +窕 +窖 +窗 +窘 +窜 +窝 +窟 +窠 +窥 +窦 +窨 +窩 +窪 +窮 +窯 +窺 +窿 +竄 +竅 +竇 +竊 +立 +竖 +站 +竜 +竞 +竟 +章 +竣 +童 +竭 +端 +競 +竹 +竺 +竽 +竿 +笃 +笆 +笈 +笋 +笏 +笑 +笔 +笙 +笛 +笞 +笠 +符 +笨 +第 +笹 +笺 +笼 +筆 +等 +筊 +筋 +筍 +筏 +筐 +筑 +筒 +答 +策 +筛 +筝 +筠 +筱 +筲 +筵 +筷 +筹 +签 +简 +箇 +箋 +箍 +箏 +箐 +箔 +箕 +算 +箝 +管 +箩 +箫 +箭 +箱 +箴 +箸 +節 +篁 +範 +篆 +篇 +築 +篑 +篓 +篙 +篝 +篠 +篡 +篤 +篩 +篪 +篮 +篱 +篷 +簇 +簌 +簍 +簡 +簦 +簧 +簪 +簫 +簷 +簸 +簽 +簾 +簿 +籁 +籃 +籌 +籍 +籐 +籟 +籠 +籤 +籬 +籮 +籲 +米 +类 +籼 +籽 +粄 +粉 +粑 +粒 +粕 +粗 +粘 +粟 +粤 +粥 +粧 +粪 +粮 +粱 +粲 +粳 +粵 +粹 +粼 +粽 +精 +粿 +糅 +糊 +糍 +糕 +糖 +糗 +糙 +糜 +糞 +糟 +糠 +糧 +糬 +糯 +糰 +糸 +系 +糾 +紀 +紂 +約 +紅 +紉 +紊 +紋 +納 +紐 +紓 +純 +紗 +紘 +紙 +級 +紛 +紜 +素 +紡 +索 +紧 +紫 +紮 +累 +細 +紳 +紹 +紺 +終 +絃 +組 +絆 +経 +結 +絕 +絞 +絡 +絢 +給 +絨 +絮 +統 +絲 +絳 +絵 +絶 +絹 +綁 +綏 +綑 +經 +継 +続 +綜 +綠 +綢 +綦 +綫 +綬 +維 +綱 +網 +綴 +綵 +綸 +綺 +綻 +綽 +綾 +綿 +緊 +緋 +総 +緑 +緒 +緘 +線 +緝 +緞 +締 +緣 +編 +緩 +緬 +緯 +練 +緹 +緻 +縁 +縄 +縈 +縛 +縝 +縣 +縫 +縮 +縱 +縴 +縷 +總 +績 +繁 +繃 +繆 +繇 +繋 +織 +繕 +繚 +繞 +繡 +繩 +繪 +繫 +繭 +繳 +繹 +繼 +繽 +纂 +續 +纍 +纏 +纓 +纔 +纖 +纜 +纠 +红 +纣 +纤 +约 +级 +纨 +纪 +纫 +纬 +纭 +纯 +纰 +纱 +纲 +纳 +纵 +纶 +纷 +纸 +纹 +纺 +纽 +纾 +线 +绀 +练 +组 +绅 +细 +织 +终 +绊 +绍 +绎 +经 +绑 +绒 +结 +绔 +绕 +绘 +给 +绚 +绛 +络 +绝 +绞 +统 +绡 +绢 +绣 +绥 +绦 +继 +绩 +绪 +绫 +续 +绮 +绯 +绰 +绳 +维 +绵 +绶 +绷 +绸 +绻 +综 +绽 +绾 +绿 +缀 +缄 +缅 +缆 +缇 +缈 +缉 +缎 +缓 +缔 +缕 +编 +缘 +缙 +缚 +缜 +缝 +缠 +缢 +缤 +缥 +缨 +缩 +缪 +缭 +缮 +缰 +缱 +缴 +缸 +缺 +缽 +罂 +罄 +罌 +罐 +网 +罔 +罕 +罗 +罚 +罡 +罢 +罩 +罪 +置 +罰 +署 +罵 +罷 +罹 +羁 +羅 +羈 +羊 +羌 +美 +羔 +羚 +羞 +羟 +羡 +羣 +群 +羥 +羧 +羨 +義 +羯 +羲 +羸 +羹 +羽 +羿 +翁 +翅 +翊 +翌 +翎 +習 +翔 +翘 +翟 +翠 +翡 +翦 +翩 +翰 +翱 +翳 +翹 +翻 +翼 +耀 +老 +考 +耄 +者 +耆 +耋 +而 +耍 +耐 +耒 +耕 +耗 +耘 +耙 +耦 +耨 +耳 +耶 +耷 +耸 +耻 +耽 +耿 +聂 +聆 +聊 +聋 +职 +聒 +联 +聖 +聘 +聚 +聞 +聪 +聯 +聰 +聲 +聳 +聴 +聶 +職 +聽 +聾 +聿 +肃 +肄 +肅 +肆 +肇 +肉 +肋 +肌 +肏 +肓 +肖 +肘 +肚 +肛 +肝 +肠 +股 +肢 +肤 +肥 +肩 +肪 +肮 +肯 +肱 +育 +肴 +肺 +肽 +肾 +肿 +胀 +胁 +胃 +胄 +胆 +背 +胍 +胎 +胖 +胚 +胛 +胜 +胝 +胞 +胡 +胤 +胥 +胧 +胫 +胭 +胯 +胰 +胱 +胳 +胴 +胶 +胸 +胺 +能 +脂 +脅 +脆 +脇 +脈 +脉 +脊 +脍 +脏 +脐 +脑 +脓 +脖 +脘 +脚 +脛 +脣 +脩 +脫 +脯 +脱 +脲 +脳 +脸 +脹 +脾 +腆 +腈 +腊 +腋 +腌 +腎 +腐 +腑 +腓 +腔 +腕 +腥 +腦 +腩 +腫 +腭 +腮 +腰 +腱 +腳 +腴 +腸 +腹 +腺 +腻 +腼 +腾 +腿 +膀 +膈 +膊 +膏 +膑 +膘 +膚 +膛 +膜 +膝 +膠 +膦 +膨 +膩 +膳 +膺 +膻 +膽 +膾 +膿 +臀 +臂 +臃 +臆 +臉 +臊 +臍 +臓 +臘 +臟 +臣 +臥 +臧 +臨 +自 +臬 +臭 +至 +致 +臺 +臻 +臼 +臾 +舀 +舂 +舅 +舆 +與 +興 +舉 +舊 +舌 +舍 +舎 +舐 +舒 +舔 +舖 +舗 +舛 +舜 +舞 +舟 +航 +舫 +般 +舰 +舱 +舵 +舶 +舷 +舸 +船 +舺 +舾 +艇 +艋 +艘 +艙 +艦 +艮 +良 +艰 +艱 +色 +艳 +艷 +艹 +艺 +艾 +节 +芃 +芈 +芊 +芋 +芍 +芎 +芒 +芙 +芜 +芝 +芡 +芥 +芦 +芩 +芪 +芫 +芬 +芭 +芮 +芯 +花 +芳 +芷 +芸 +芹 +芻 +芽 +芾 +苁 +苄 +苇 +苋 +苍 +苏 +苑 +苒 +苓 +苔 +苕 +苗 +苛 +苜 +苞 +苟 +苡 +苣 +若 +苦 +苫 +苯 +英 +苷 +苹 +苻 +茁 +茂 +范 +茄 +茅 +茉 +茎 +茏 +茗 +茜 +茧 +茨 +茫 +茬 +茭 +茯 +茱 +茲 +茴 +茵 +茶 +茸 +茹 +茼 +荀 +荃 +荆 +草 +荊 +荏 +荐 +荒 +荔 +荖 +荘 +荚 +荞 +荟 +荠 +荡 +荣 +荤 +荥 +荧 +荨 +荪 +荫 +药 +荳 +荷 +荸 +荻 +荼 +荽 +莅 +莆 +莉 +莊 +莎 +莒 +莓 +莖 +莘 +莞 +莠 +莢 +莧 +莪 +莫 +莱 +莲 +莴 +获 +莹 +莺 +莽 +莿 +菀 +菁 +菅 +菇 +菈 +菊 +菌 +菏 +菓 +菖 +菘 +菜 +菟 +菠 +菡 +菩 +華 +菱 +菲 +菸 +菽 +萁 +萃 +萄 +萊 +萋 +萌 +萍 +萎 +萘 +萝 +萤 +营 +萦 +萧 +萨 +萩 +萬 +萱 +萵 +萸 +萼 +落 +葆 +葉 +著 +葚 +葛 +葡 +董 +葦 +葩 +葫 +葬 +葭 +葯 +葱 +葳 +葵 +葷 +葺 +蒂 +蒋 +蒐 +蒔 +蒙 +蒜 +蒞 +蒟 +蒡 +蒨 +蒲 +蒸 +蒹 +蒻 +蒼 +蒿 +蓁 +蓄 +蓆 +蓉 +蓋 +蓑 +蓓 +蓖 +蓝 +蓟 +蓦 +蓬 +蓮 +蓼 +蓿 +蔑 +蔓 +蔔 +蔗 +蔘 +蔚 +蔡 +蔣 +蔥 +蔫 +蔬 +蔭 +蔵 +蔷 +蔺 +蔻 +蔼 +蔽 +蕁 +蕃 +蕈 +蕉 +蕊 +蕎 +蕙 +蕤 +蕨 +蕩 +蕪 +蕭 +蕲 +蕴 +蕻 +蕾 +薄 +薅 +薇 +薈 +薊 +薏 +薑 +薔 +薙 +薛 +薦 +薨 +薩 +薪 +薬 +薯 +薰 +薹 +藉 +藍 +藏 +藐 +藓 +藕 +藜 +藝 +藤 +藥 +藩 +藹 +藻 +藿 +蘆 +蘇 +蘊 +蘋 +蘑 +蘚 +蘭 +蘸 +蘼 +蘿 +虎 +虏 +虐 +虑 +虔 +處 +虚 +虛 +虜 +虞 +號 +虢 +虧 +虫 +虬 +虱 +虹 +虻 +虽 +虾 +蚀 +蚁 +蚂 +蚊 +蚌 +蚓 +蚕 +蚜 +蚝 +蚣 +蚤 +蚩 +蚪 +蚯 +蚱 +蚵 +蛀 +蛆 +蛇 +蛊 +蛋 +蛎 +蛐 +蛔 +蛙 +蛛 +蛟 +蛤 +蛭 +蛮 +蛰 +蛳 +蛹 +蛻 +蛾 +蜀 +蜂 +蜃 +蜆 +蜇 +蜈 +蜊 +蜍 +蜒 +蜓 +蜕 +蜗 +蜘 +蜚 +蜜 +蜡 +蜢 +蜥 +蜱 +蜴 +蜷 +蜻 +蜿 +蝇 +蝈 +蝉 +蝌 +蝎 +蝕 +蝗 +蝙 +蝟 +蝠 +蝦 +蝨 +蝴 +蝶 +蝸 +蝼 +螂 +螃 +融 +螞 +螢 +螨 +螯 +螳 +螺 +蟀 +蟄 +蟆 +蟋 +蟎 +蟑 +蟒 +蟠 +蟬 +蟲 +蟹 +蟻 +蟾 +蠅 +蠍 +蠔 +蠕 +蠛 +蠟 +蠡 +蠢 +蠣 +蠱 +蠶 +蠹 +蠻 +血 +衄 +衅 +衆 +行 +衍 +術 +衔 +街 +衙 +衛 +衝 +衞 +衡 +衢 +衣 +补 +表 +衩 +衫 +衬 +衮 +衰 +衲 +衷 +衹 +衾 +衿 +袁 +袂 +袄 +袅 +袈 +袋 +袍 +袒 +袖 +袜 +袞 +袤 +袪 +被 +袭 +袱 +裁 +裂 +装 +裆 +裊 +裏 +裔 +裕 +裘 +裙 +補 +裝 +裟 +裡 +裤 +裨 +裱 +裳 +裴 +裸 +裹 +製 +裾 +褂 +複 +褐 +褒 +褓 +褔 +褚 +褥 +褪 +褫 +褲 +褶 +褻 +襁 +襄 +襟 +襠 +襪 +襬 +襯 +襲 +西 +要 +覃 +覆 +覇 +見 +規 +覓 +視 +覚 +覦 +覧 +親 +覬 +観 +覷 +覺 +覽 +觀 +见 +观 +规 +觅 +视 +览 +觉 +觊 +觎 +觐 +觑 +角 +觞 +解 +觥 +触 +觸 +言 +訂 +計 +訊 +討 +訓 +訕 +訖 +託 +記 +訛 +訝 +訟 +訣 +訥 +訪 +設 +許 +訳 +訴 +訶 +診 +註 +証 +詆 +詐 +詔 +評 +詛 +詞 +詠 +詡 +詢 +詣 +試 +詩 +詫 +詬 +詭 +詮 +詰 +話 +該 +詳 +詹 +詼 +誅 +誇 +誉 +誌 +認 +誓 +誕 +誘 +語 +誠 +誡 +誣 +誤 +誥 +誦 +誨 +說 +説 +読 +誰 +課 +誹 +誼 +調 +諄 +談 +請 +諏 +諒 +論 +諗 +諜 +諡 +諦 +諧 +諫 +諭 +諮 +諱 +諳 +諷 +諸 +諺 +諾 +謀 +謁 +謂 +謄 +謊 +謎 +謐 +謔 +謗 +謙 +講 +謝 +謠 +謨 +謬 +謹 +謾 +譁 +證 +譎 +譏 +識 +譙 +譚 +譜 +警 +譬 +譯 +議 +譲 +譴 +護 +譽 +讀 +變 +讓 +讚 +讞 +计 +订 +认 +讥 +讧 +讨 +让 +讪 +讫 +训 +议 +讯 +记 +讲 +讳 +讴 +讶 +讷 +许 +讹 +论 +讼 +讽 +设 +访 +诀 +证 +诃 +评 +诅 +识 +诈 +诉 +诊 +诋 +词 +诏 +译 +试 +诗 +诘 +诙 +诚 +诛 +话 +诞 +诟 +诠 +诡 +询 +诣 +诤 +该 +详 +诧 +诩 +诫 +诬 +语 +误 +诰 +诱 +诲 +说 +诵 +诶 +请 +诸 +诺 +读 +诽 +课 +诿 +谀 +谁 +调 +谄 +谅 +谆 +谈 +谊 +谋 +谌 +谍 +谎 +谏 +谐 +谑 +谒 +谓 +谔 +谕 +谗 +谘 +谙 +谚 +谛 +谜 +谟 +谢 +谣 +谤 +谥 +谦 +谧 +谨 +谩 +谪 +谬 +谭 +谯 +谱 +谲 +谴 +谶 +谷 +豁 +豆 +豇 +豈 +豉 +豊 +豌 +豎 +豐 +豔 +豚 +象 +豢 +豪 +豫 +豬 +豹 +豺 +貂 +貅 +貌 +貓 +貔 +貘 +貝 +貞 +負 +財 +貢 +貧 +貨 +販 +貪 +貫 +責 +貯 +貰 +貳 +貴 +貶 +買 +貸 +費 +貼 +貽 +貿 +賀 +賁 +賂 +賃 +賄 +資 +賈 +賊 +賑 +賓 +賜 +賞 +賠 +賡 +賢 +賣 +賤 +賦 +質 +賬 +賭 +賴 +賺 +購 +賽 +贅 +贈 +贊 +贍 +贏 +贓 +贖 +贛 +贝 +贞 +负 +贡 +财 +责 +贤 +败 +账 +货 +质 +贩 +贪 +贫 +贬 +购 +贮 +贯 +贰 +贱 +贲 +贴 +贵 +贷 +贸 +费 +贺 +贻 +贼 +贾 +贿 +赁 +赂 +赃 +资 +赅 +赈 +赊 +赋 +赌 +赎 +赏 +赐 +赓 +赔 +赖 +赘 +赚 +赛 +赝 +赞 +赠 +赡 +赢 +赣 +赤 +赦 +赧 +赫 +赭 +走 +赳 +赴 +赵 +赶 +起 +趁 +超 +越 +趋 +趕 +趙 +趟 +趣 +趨 +足 +趴 +趵 +趸 +趺 +趾 +跃 +跄 +跆 +跋 +跌 +跎 +跑 +跖 +跚 +跛 +距 +跟 +跡 +跤 +跨 +跩 +跪 +路 +跳 +践 +跷 +跹 +跺 +跻 +踉 +踊 +踌 +踏 +踐 +踝 +踞 +踟 +踢 +踩 +踪 +踮 +踱 +踴 +踵 +踹 +蹂 +蹄 +蹇 +蹈 +蹉 +蹊 +蹋 +蹑 +蹒 +蹙 +蹟 +蹣 +蹤 +蹦 +蹩 +蹬 +蹭 +蹲 +蹴 +蹶 +蹺 +蹼 +蹿 +躁 +躇 +躉 +躊 +躋 +躍 +躏 +躪 +身 +躬 +躯 +躲 +躺 +軀 +車 +軋 +軌 +軍 +軒 +軟 +転 +軸 +軼 +軽 +軾 +較 +載 +輒 +輓 +輔 +輕 +輛 +輝 +輟 +輩 +輪 +輯 +輸 +輻 +輾 +輿 +轄 +轅 +轆 +轉 +轍 +轎 +轟 +车 +轧 +轨 +轩 +转 +轭 +轮 +软 +轰 +轲 +轴 +轶 +轻 +轼 +载 +轿 +较 +辄 +辅 +辆 +辇 +辈 +辉 +辊 +辍 +辐 +辑 +输 +辕 +辖 +辗 +辘 +辙 +辛 +辜 +辞 +辟 +辣 +辦 +辨 +辩 +辫 +辭 +辮 +辯 +辰 +辱 +農 +边 +辺 +辻 +込 +辽 +达 +迁 +迂 +迄 +迅 +过 +迈 +迎 +运 +近 +返 +还 +这 +进 +远 +违 +连 +迟 +迢 +迤 +迥 +迦 +迩 +迪 +迫 +迭 +述 +迴 +迷 +迸 +迹 +迺 +追 +退 +送 +适 +逃 +逅 +逆 +选 +逊 +逍 +透 +逐 +递 +途 +逕 +逗 +這 +通 +逛 +逝 +逞 +速 +造 +逢 +連 +逮 +週 +進 +逵 +逶 +逸 +逻 +逼 +逾 +遁 +遂 +遅 +遇 +遊 +運 +遍 +過 +遏 +遐 +遑 +遒 +道 +達 +違 +遗 +遙 +遛 +遜 +遞 +遠 +遢 +遣 +遥 +遨 +適 +遭 +遮 +遲 +遴 +遵 +遶 +遷 +選 +遺 +遼 +遽 +避 +邀 +邁 +邂 +邃 +還 +邇 +邈 +邊 +邋 +邏 +邑 +邓 +邕 +邛 +邝 +邢 +那 +邦 +邨 +邪 +邬 +邮 +邯 +邰 +邱 +邳 +邵 +邸 +邹 +邺 +邻 +郁 +郅 +郊 +郎 +郑 +郜 +郝 +郡 +郢 +郤 +郦 +郧 +部 +郫 +郭 +郴 +郵 +郷 +郸 +都 +鄂 +鄉 +鄒 +鄔 +鄙 +鄞 +鄢 +鄧 +鄭 +鄰 +鄱 +鄲 +鄺 +酉 +酊 +酋 +酌 +配 +酐 +酒 +酗 +酚 +酝 +酢 +酣 +酥 +酩 +酪 +酬 +酮 +酯 +酰 +酱 +酵 +酶 +酷 +酸 +酿 +醃 +醇 +醉 +醋 +醍 +醐 +醒 +醚 +醛 +醜 +醞 +醣 +醪 +醫 +醬 +醮 +醯 +醴 +醺 +釀 +釁 +采 +釉 +释 +釋 +里 +重 +野 +量 +釐 +金 +釗 +釘 +釜 +針 +釣 +釦 +釧 +釵 +鈀 +鈉 +鈍 +鈎 +鈔 +鈕 +鈞 +鈣 +鈦 +鈪 +鈴 +鈺 +鈾 +鉀 +鉄 +鉅 +鉉 +鉑 +鉗 +鉚 +鉛 +鉤 +鉴 +鉻 +銀 +銃 +銅 +銑 +銓 +銖 +銘 +銜 +銬 +銭 +銮 +銳 +銷 +銹 +鋁 +鋅 +鋒 +鋤 +鋪 +鋰 +鋸 +鋼 +錄 +錐 +錘 +錚 +錠 +錢 +錦 +錨 +錫 +錮 +錯 +録 +錳 +錶 +鍊 +鍋 +鍍 +鍛 +鍥 +鍰 +鍵 +鍺 +鍾 +鎂 +鎊 +鎌 +鎏 +鎔 +鎖 +鎗 +鎚 +鎧 +鎬 +鎮 +鎳 +鏈 +鏖 +鏗 +鏘 +鏞 +鏟 +鏡 +鏢 +鏤 +鏽 +鐘 +鐮 +鐲 +鐳 +鐵 +鐸 +鐺 +鑄 +鑊 +鑑 +鑒 +鑣 +鑫 +鑰 +鑲 +鑼 +鑽 +鑾 +鑿 +针 +钉 +钊 +钎 +钏 +钒 +钓 +钗 +钙 +钛 +钜 +钝 +钞 +钟 +钠 +钡 +钢 +钣 +钤 +钥 +钦 +钧 +钨 +钩 +钮 +钯 +钰 +钱 +钳 +钴 +钵 +钺 +钻 +钼 +钾 +钿 +铀 +铁 +铂 +铃 +铄 +铅 +铆 +铉 +铎 +铐 +铛 +铜 +铝 +铠 +铡 +铢 +铣 +铤 +铨 +铩 +铬 +铭 +铮 +铰 +铲 +铵 +银 +铸 +铺 +链 +铿 +销 +锁 +锂 +锄 +锅 +锆 +锈 +锉 +锋 +锌 +锏 +锐 +锑 +错 +锚 +锟 +锡 +锢 +锣 +锤 +锥 +锦 +锭 +键 +锯 +锰 +锲 +锵 +锹 +锺 +锻 +镀 +镁 +镂 +镇 +镉 +镌 +镍 +镐 +镑 +镕 +镖 +镗 +镛 +镜 +镣 +镭 +镯 +镰 +镳 +镶 +長 +长 +門 +閃 +閉 +開 +閎 +閏 +閑 +閒 +間 +閔 +閘 +閡 +関 +閣 +閥 +閨 +閩 +閱 +閲 +閹 +閻 +閾 +闆 +闇 +闊 +闌 +闍 +闔 +闕 +闖 +闘 +關 +闡 +闢 +门 +闪 +闫 +闭 +问 +闯 +闰 +闲 +间 +闵 +闷 +闸 +闹 +闺 +闻 +闽 +闾 +阀 +阁 +阂 +阅 +阆 +阇 +阈 +阉 +阎 +阐 +阑 +阔 +阕 +阖 +阙 +阚 +阜 +队 +阡 +阪 +阮 +阱 +防 +阳 +阴 +阵 +阶 +阻 +阿 +陀 +陂 +附 +际 +陆 +陇 +陈 +陋 +陌 +降 +限 +陕 +陛 +陝 +陞 +陟 +陡 +院 +陣 +除 +陨 +险 +陪 +陰 +陲 +陳 +陵 +陶 +陷 +陸 +険 +陽 +隅 +隆 +隈 +隊 +隋 +隍 +階 +随 +隐 +隔 +隕 +隘 +隙 +際 +障 +隠 +隣 +隧 +隨 +險 +隱 +隴 +隶 +隸 +隻 +隼 +隽 +难 +雀 +雁 +雄 +雅 +集 +雇 +雉 +雋 +雌 +雍 +雎 +雏 +雑 +雒 +雕 +雖 +雙 +雛 +雜 +雞 +離 +難 +雨 +雪 +雯 +雰 +雲 +雳 +零 +雷 +雹 +電 +雾 +需 +霁 +霄 +霆 +震 +霈 +霉 +霊 +霍 +霎 +霏 +霑 +霓 +霖 +霜 +霞 +霧 +霭 +霰 +露 +霸 +霹 +霽 +霾 +靂 +靄 +靈 +青 +靓 +靖 +静 +靚 +靛 +靜 +非 +靠 +靡 +面 +靥 +靦 +革 +靳 +靴 +靶 +靼 +鞅 +鞋 +鞍 +鞏 +鞑 +鞘 +鞠 +鞣 +鞦 +鞭 +韆 +韋 +韌 +韓 +韜 +韦 +韧 +韩 +韬 +韭 +音 +韵 +韶 +韻 +響 +頁 +頂 +頃 +項 +順 +須 +頌 +預 +頑 +頒 +頓 +頗 +領 +頜 +頡 +頤 +頫 +頭 +頰 +頷 +頸 +頹 +頻 +頼 +顆 +題 +額 +顎 +顏 +顔 +願 +顛 +類 +顧 +顫 +顯 +顱 +顴 +页 +顶 +顷 +项 +顺 +须 +顼 +顽 +顾 +顿 +颁 +颂 +预 +颅 +领 +颇 +颈 +颉 +颊 +颌 +颍 +颐 +频 +颓 +颔 +颖 +颗 +题 +颚 +颛 +颜 +额 +颞 +颠 +颡 +颢 +颤 +颦 +颧 +風 +颯 +颱 +颳 +颶 +颼 +飄 +飆 +风 +飒 +飓 +飕 +飘 +飙 +飚 +飛 +飞 +食 +飢 +飨 +飩 +飪 +飯 +飲 +飼 +飽 +飾 +餃 +餅 +餉 +養 +餌 +餐 +餒 +餓 +餘 +餚 +餛 +餞 +餡 +館 +餮 +餵 +餾 +饅 +饈 +饋 +饌 +饍 +饑 +饒 +饕 +饗 +饞 +饥 +饨 +饪 +饬 +饭 +饮 +饯 +饰 +饱 +饲 +饴 +饵 +饶 +饷 +饺 +饼 +饽 +饿 +馀 +馁 +馄 +馅 +馆 +馈 +馋 +馍 +馏 +馒 +馔 +首 +馗 +香 +馥 +馨 +馬 +馭 +馮 +馳 +馴 +駁 +駄 +駅 +駆 +駐 +駒 +駕 +駛 +駝 +駭 +駱 +駿 +騁 +騎 +騏 +験 +騙 +騨 +騰 +騷 +驀 +驅 +驊 +驍 +驒 +驕 +驗 +驚 +驛 +驟 +驢 +驥 +马 +驭 +驮 +驯 +驰 +驱 +驳 +驴 +驶 +驷 +驸 +驹 +驻 +驼 +驾 +驿 +骁 +骂 +骄 +骅 +骆 +骇 +骈 +骊 +骋 +验 +骏 +骐 +骑 +骗 +骚 +骛 +骜 +骞 +骠 +骡 +骤 +骥 +骧 +骨 +骯 +骰 +骶 +骷 +骸 +骼 +髂 +髅 +髋 +髏 +髒 +髓 +體 +髖 +高 +髦 +髪 +髮 +髯 +髻 +鬃 +鬆 +鬍 +鬓 +鬚 +鬟 +鬢 +鬣 +鬥 +鬧 +鬱 +鬼 +魁 +魂 +魄 +魅 +魇 +魍 +魏 +魔 +魘 +魚 +魯 +魷 +鮑 +鮨 +鮪 +鮭 +鮮 +鯉 +鯊 +鯖 +鯛 +鯨 +鯰 +鯽 +鰍 +鰓 +鰭 +鰲 +鰻 +鰾 +鱈 +鱉 +鱔 +鱗 +鱷 +鱸 +鱼 +鱿 +鲁 +鲈 +鲍 +鲑 +鲛 +鲜 +鲟 +鲢 +鲤 +鲨 +鲫 +鲱 +鲲 +鲶 +鲷 +鲸 +鳃 +鳄 +鳅 +鳌 +鳍 +鳕 +鳖 +鳗 +鳝 +鳞 +鳥 +鳩 +鳳 +鳴 +鳶 +鴉 +鴕 +鴛 +鴦 +鴨 +鴻 +鴿 +鵑 +鵜 +鵝 +鵡 +鵬 +鵰 +鵲 +鶘 +鶩 +鶯 +鶴 +鷗 +鷲 +鷹 +鷺 +鸚 +鸞 +鸟 +鸠 +鸡 +鸢 +鸣 +鸥 +鸦 +鸨 +鸪 +鸭 +鸯 +鸳 +鸵 +鸽 +鸾 +鸿 +鹂 +鹃 +鹄 +鹅 +鹈 +鹉 +鹊 +鹌 +鹏 +鹑 +鹕 +鹘 +鹜 +鹞 +鹤 +鹦 +鹧 +鹫 +鹭 +鹰 +鹳 +鹵 +鹹 +鹼 +鹽 +鹿 +麂 +麋 +麒 +麓 +麗 +麝 +麟 +麥 +麦 +麩 +麴 +麵 +麸 +麺 +麻 +麼 +麽 +麾 +黃 +黄 +黍 +黎 +黏 +黑 +黒 +黔 +默 +黛 +黜 +黝 +點 +黠 +黨 +黯 +黴 +鼋 +鼎 +鼐 +鼓 +鼠 +鼬 +鼹 +鼻 +鼾 +齁 +齊 +齋 +齐 +齒 +齡 +齢 +齣 +齦 +齿 +龄 +龅 +龈 +龊 +龋 +龌 +龍 +龐 +龔 +龕 +龙 +龚 +龛 +龜 +龟 +︰ +︱ +︶ +︿ +﹁ +﹂ +﹍ +﹏ +﹐ +﹑ +﹒ +﹔ +﹕ +﹖ +﹗ +﹙ +﹚ +﹝ +﹞ +﹡ +﹣ +！ +＂ +＃ +＄ +％ +＆ +＇ +（ +） +＊ +＋ +， +－ +． +／ +０ +１ +２ +３ +４ +５ +６ +７ +８ +９ +： +； +＜ +＝ +＞ +？ +＠ +［ +＼ +］ +＾ +＿ +｀ +ａ +ｂ +ｃ +ｄ +ｅ +ｆ +ｇ +ｈ +ｉ +ｊ +ｋ +ｌ +ｍ +ｎ +ｏ +ｐ +ｑ +ｒ +ｓ +ｔ +ｕ +ｖ +ｗ +ｘ +ｙ +ｚ +｛ +｜ +｝ +～ +｡ +｢ +｣ +､ +･ +ｯ +ｰ +ｲ +ｸ +ｼ +ｽ +ﾄ +ﾉ +ﾌ +ﾗ +ﾙ +ﾝ +ﾞ +ﾟ +￣ +￥ +👍 +🔥 +😂 +😎 +... +yam +10 +2017 +12 +11 +2016 +20 +30 +15 +06 +lofter +##s +2015 +by +16 +14 +18 +13 +24 +17 +2014 +21 +##0 +22 +19 +25 +23 +com +100 +00 +05 +2013 +##a +03 +09 +08 +28 +##2 +50 +01 +04 +##1 +27 +02 +2012 +##3 +26 +##e +07 +##8 +##5 +##6 +##4 +##9 +##7 +29 +2011 +40 +##t +2010 +##o +##d +##i +2009 +##n +app +www +the +##m +31 +##c +##l +##y +##r +##g +2008 +60 +http +200 +qq +##p +80 +##f +google +pixnet +90 +cookies +tripadvisor +500 +##er +##k +35 +##h +facebook +2007 +2000 +70 +##b +of +##x +##u +45 +300 +iphone +32 +1000 +2006 +48 +ip +36 +in +38 +3d +##w +##ing +55 +ctrip +##on +##v +33 +##の +to +34 +400 +id +2005 +it +37 +windows +llc +top +99 +42 +39 +000 +led +at +##an +41 +51 +52 +46 +49 +43 +53 +44 +##z +android +58 +and +59 +2004 +56 +vr +##か +5000 +2003 +47 +blogthis +twitter +54 +##le +150 +ok +2018 +57 +75 +cn +no +ios +##in +##mm +##00 +800 +on +te +3000 +65 +2001 +360 +95 +ig +lv +120 +##ng +##を +##us +##に +pc +てす +── +600 +##te +85 +2002 +88 +##ed +html +ncc +wifi +email +64 +blog +is +##10 +##て +mail +online +##al +dvd +##ic +studio +##は +##℃ +##ia +##と +line +vip +72 +##q +98 +##ce +##en +for +##is +##ra +##es +##j +usb +net +cp +1999 +asia +4g +##cm +diy +new +3c +##お +ta +66 +language +vs +apple +tw +86 +web +##ne +ipad +62 +you +##re +101 +68 +##tion +ps +de +bt +pony +atm +##2017 +1998 +67 +##ch +ceo +##or +go +##na +av +pro +cafe +96 +pinterest +97 +63 +pixstyleme3c +##ta +more +said +##2016 +1997 +mp3 +700 +##ll +nba +jun +##20 +92 +tv +1995 +pm +61 +76 +nbsp +250 +##ie +linux +##ma +cd +110 +hd +##17 +78 +##ion +77 +6000 +am +##th +##st +94 +##se +##et +69 +180 +gdp +my +105 +81 +abc +89 +flash +79 +one +93 +1990 +1996 +##ck +gps +##も +##ly +web885 +106 +2020 +91 +##ge +4000 +1500 +xd +boss +isbn +1994 +org +##ry +me +love +##11 +0fork +73 +##12 +3g +##ter +##ar +71 +82 +##la +hotel +130 +1970 +pk +83 +87 +140 +ie +##os +##30 +##el +74 +##50 +seo +cpu +##ml +p2p +84 +may +##る +sun +tue +internet +cc +posted +youtube +##at +##ン +##man +ii +##ル +##15 +abs +nt +pdf +yahoo +ago +1980 +##it +news +mac +104 +##てす +##me +##り +java +1992 +spa +##de +##nt +hk +all +plus +la +1993 +##mb +##16 +##ve +west +##da +160 +air +##い +##ps +から +##to +1989 +logo +htc +php +https +fi +momo +##son +sat +##ke +##80 +ebd +suv +wi +day +apk +##88 +##um +mv +galaxy +wiki +or +brake +##ス +1200 +する +this +1991 +mon +##こ +❤2017 +po +##ない +javascript +life +home +june +##ss +system +900 +##ー +##０ +pp +1988 +world +fb +4k +br +##as +ic +ai +leonardo +safari +##60 +live +free +xx +wed +win7 +kiehl +##co +lg +o2o +##go +us +235 +1949 +mm +しい +vfm +kanye +##90 +##2015 +##id +jr +##ey +123 +rss +##sa +##ro +##am +##no +thu +fri +350 +##sh +##ki +103 +comments +name +##のて +##pe +##ine +max +1987 +8000 +uber +##mi +##ton +wordpress +office +1986 +1985 +##ment +107 +bd +win10 +##ld +##li +gmail +bb +dior +##rs +##ri +##rd +##ます +up +cad +##® +dr +して +read +##21 +をお +##io +##99 +url +1984 +pvc +paypal +show +policy +##40 +##ty +##18 +with +##★ +##01 +txt +102 +##ba +dna +from +post +mini +ar +taiwan +john +##ga +privacy +agoda +##13 +##ny +word +##24 +##22 +##by +##ur +##hz +1982 +##ang +265 +cookie +netscape +108 +##ka +##～ +##ad +house +share +note +ibm +code +hello +nike +sim +survey +##016 +1979 +1950 +wikia +##32 +##017 +5g +cbc +##tor +##kg +1983 +##rt +##14 +campaign +store +2500 +os +##ct +##ts +##° +170 +api +##ns +365 +excel +##な +##ao +##ら +##し +～～ +##nd +university +163 +には +518 +##70 +##ya +##il +##25 +pierre +ipo +0020 +897 +##23 +hotels +##ian +のお +125 +years +6606 +##ers +##26 +high +##day +time +##ay +bug +##line +##く +##す +##be +xp +talk2yam +yamservice +10000 +coco +##dy +sony +##ies +1978 +microsoft +david +people +##ha +1960 +instagram +intel +その +##ot +iso +1981 +##va +115 +##mo +##land +xxx +man +co +ltxsw +##ation +baby +220 +##pa +##ol +1945 +7000 +tag +450 +##ue +msn +##31 +oppo +##ト +##ca +control +##om +st +chrome +##ure +##ん +be +##き +lol +##19 +した +##bo +240 +lady +##100 +##way +##から +4600 +##ko +##do +##un +4s +corporation +168 +##ni +herme +##28 +ｃｐ +978 +##up +##06 +ui +##ds +ppt +admin +three +します +bbc +re +128 +##48 +ca +##015 +##35 +hp +##ee +tpp +##た +##ive +×× +root +##cc +##ました +##ble +##ity +adobe +park +114 +et +oled +city +##ex +##ler +##ap +china +##book +20000 +view +##ice +global +##km +your +hong +##mg +out +##ms +ng +ebay +##29 +menu +ubuntu +##cy +rom +##view +open +ktv +do +server +##lo +if +english +##ね +##５ +##oo +1600 +##02 +step1 +kong +club +135 +july +inc +1976 +mr +hi +##net +touch +##ls +##ii +michael +lcd +##05 +##33 +phone +james +step2 +1300 +ios9 +##box +dc +##２ +##ley +samsung +111 +280 +pokemon +css +##ent +##les +いいえ +##１ +s8 +atom +play +bmw +##said +sa +etf +ctrl +♥yoyo♥ +##55 +2025 +##2014 +##66 +adidas +amazon +1958 +##ber +##ner +visa +##77 +##der +1800 +connectivity +##hi +firefox +109 +118 +hr +so +style +mark +pop +ol +skip +1975 +as +##27 +##ir +##61 +190 +mba +##う +##ai +le +##ver +1900 +cafe2017 +lte +super +113 +129 +##ron +amd +like +##☆ +are +##ster +we +##sk +paul +data +international +##ft +longchamp +ssd +good +##ート +##ti +reply +##my +↓↓↓ +apr +star +##ker +source +136 +js +112 +get +force +photo +##one +126 +##2013 +##ow +link +bbs +1972 +goods +##lin +python +119 +##ip +game +##ics +##ません +blue +##● +520 +##45 +page +itunes +##03 +1955 +260 +1968 +gt +gif +618 +##ff +##47 +group +くたさい +about +bar +ganji +##nce +music +lee +not +1977 +1971 +1973 +##per +an +faq +comment +##って +days +##ock +116 +##bs +1974 +1969 +v1 +player +1956 +xbox +sql +fm +f1 +139 +##ah +210 +##lv +##mp +##000 +melody +1957 +##３ +550 +17life +199 +1966 +xml +market +##au +##71 +999 +##04 +what +gl +##95 +##age +tips +##68 +book +##ting +mysql +can +1959 +230 +##ung +wonderland +watch +10℃ +##ction +9000 +mar +mobile +1946 +1962 +article +##db +part +▲top +party +って +1967 +1964 +1948 +##07 +##ore +##op +この +dj +##78 +##38 +010 +main +225 +1965 +##ong +art +320 +ad +134 +020 +##73 +117 +pm2 +japan +228 +##08 +ts +1963 +##ica +der +sm +##36 +2019 +##wa +ct +##７ +##や +##64 +1937 +homemesh +search +##85 +##れは +##tv +##di +macbook +##９ +##くたさい +service +##♥ +type +った +750 +##ier +##si +##75 +##います +##ok +best +##ット +goris +lock +##った +cf +3m +big +##ut +ftp +carol +##vi +１０ +1961 +happy +sd +##ac +122 +anti +pe +cnn +iii +1920 +138 +##ラ +1940 +esp +jan +tags +##98 +##51 +august +vol +##86 +154 +##™ +##fs +##れ +##sion +design +ac +##ム +press +jordan +ppp +that +key +check +##６ +##tt +##㎡ +1080p +##lt +power +##42 +1952 +##bc +vivi +##ック +he +133 +121 +jpg +##rry +201 +175 +3500 +1947 +nb +##ted +##rn +しています +1954 +usd +##t00 +master +##ンク +001 +model +##58 +al +##09 +1953 +##34 +ram +goo +ても +##ui +127 +1930 +red +##ary +rpg +item +##pm +##41 +270 +##za +project +##2012 +hot +td +blogabstract +##ger +##62 +650 +##44 +gr2 +##します +##ｍ +black +electronic +nfc +year +asus +また +html5 +cindy +##hd +m3 +132 +esc +##od +booking +##53 +fed +tvb +##81 +##ina +mit +165 +##いる +chan +192 +distribution +next +になる +peter +bios +steam +cm +1941 +にも +pk10 +##ix +##65 +##91 +dec +nasa +##ana +icecat +00z +b1 +will +##46 +li +se +##ji +##み +##ard +oct +##ain +jp +##ze +##bi +cio +##56 +smart +h5 +##39 +##port +curve +vpn +##nm +##dia +utc +##あり +12345678910 +##52 +rmvb +chanel +a4 +miss +##and +##im +media +who +##63 +she +girl +5s +124 +vera +##して +class +vivo +king +##フ +##ei +national +ab +1951 +5cm +888 +145 +ipod +ap +1100 +5mm +211 +ms +2756 +##69 +mp4 +msci +##po +##89 +131 +mg +index +380 +##bit +##out +##zz +##97 +##67 +158 +apec +##８ +photoshop +opec +￥799 +ては +##96 +##tes +##ast +2g +○○ +##ール +￥2899 +##ling +##よ +##ory +1938 +##ical +kitty +content +##43 +step3 +##cn +win8 +155 +vc +1400 +iphone7 +robert +##した +tcl +137 +beauty +##87 +en +dollars +##ys +##oc +step +pay +yy +a1 +##2011 +##lly +##ks +##♪ +1939 +188 +download +1944 +sep +exe +ph +います +school +gb +center +pr +street +##board +uv +##37 +##lan +winrar +##que +##ua +##com +1942 +1936 +480 +gpu +##４ +ettoday +fu +tom +##54 +##ren +##via +149 +##72 +b2b +144 +##79 +##tch +rose +arm +mb +##49 +##ial +##nn +nvidia +step4 +mvp +00㎡ +york +156 +##イ +how +cpi +591 +2765 +gov +kg +joe +##xx +mandy +pa +##ser +copyright +fashion +1935 +don +##け +ecu +##ist +##art +erp +wap +have +##lm +talk +##ek +##ning +##if +ch +##ite +video +1943 +cs +san +iot +look +##84 +##2010 +##ku +october +##ux +trump +##hs +##ide +box +141 +first +##ins +april +##ight +##83 +185 +angel +protected +aa +151 +162 +x1 +m2 +##fe +##× +##ho +size +143 +min +ofo +fun +gomaji +ex +hdmi +food +dns +march +chris +kevin +##のか +##lla +##pp +##ec +ag +ems +6s +720p +##rm +##ham +off +##92 +asp +team +fandom +ed +299 +▌♥ +##ell +info +されています +##82 +sina +4066 +161 +##able +##ctor +330 +399 +315 +dll +rights +ltd +idc +jul +3kg +1927 +142 +ma +surface +##76 +##ク +～～～ +304 +mall +eps +146 +green +##59 +map +space +donald +v2 +sodu +##light +1931 +148 +1700 +まて +310 +reserved +htm +##han +##57 +2d +178 +mod +##ise +##tions +152 +ti +##shi +doc +1933 +icp +055 +wang +##ram +shopping +aug +##pi +##well +now +wam +b2 +からお +##hu +236 +1928 +##gb +266 +f2 +##93 +153 +mix +##ef +##uan +bwl +##plus +##res +core +##ess +tea +5℃ +hktvmall +nhk +##ate +list +##ese +301 +feb +4m +inn +ての +nov +159 +12345 +daniel +##ci +pass +##bet +##nk +coffee +202 +ssl +airbnb +##ute +fbi +woshipm +skype +ea +cg +sp +##fc +##www +yes +edge +alt +007 +##94 +fpga +##ght +##gs +iso9001 +さい +##ile +##wood +##uo +image +lin +icon +american +##em +1932 +set +says +##king +##tive +blogger +##74 +なと +256 +147 +##ox +##zy +##red +##ium +##lf +nokia +claire +##リ +##ding +november +lohas +##500 +##tic +##マ +##cs +##ある +##che +##ire +##gy +##ult +db +january +win +##カ +166 +road +ptt +##ま +##つ +198 +##fa +##mer +anna +pchome +はい +udn +ef +420 +##time +##tte +2030 +##ア +g20 +white +かかります +1929 +308 +garden +eleven +di +##おります +chen +309b +777 +172 +young +cosplay +ちてない +4500 +bat +##123 +##tra +##ては +kindle +npc +steve +etc +##ern +##｜ +call +xperia +ces +travel +sk +s7 +##ous +1934 +##int +みいたたけます +183 +edu +file +cho +qr +##car +##our +186 +##ant +##ｄ +eric +1914 +rends +##jo +##する +mastercard +##2000 +kb +##min +290 +##ino +vista +##ris +##ud +jack +2400 +##set +169 +pos +1912 +##her +##ou +taipei +しく +205 +beta +##ませんか +232 +##fi +express +255 +body +##ill +aphojoy +user +december +meiki +##ick +tweet +richard +##av +##ᆫ +iphone6 +##dd +ちてすか +views +##mark +321 +pd +##００ +times +##▲ +level +##ash +10g +point +5l +##ome +208 +koreanmall +##ak +george +q2 +206 +wma +tcp +##200 +スタッフ +full +mlb +##lle +##watch +tm +run +179 +911 +smith +business +##und +1919 +color +##tal +222 +171 +##less +moon +4399 +##rl +update +pcb +shop +499 +157 +little +なし +end +##mhz +van +dsp +easy +660 +##house +##key +history +##ｏ +oh +##001 +##hy +##web +oem +let +was +##2009 +##gg +review +##wan +182 +##°c +203 +uc +title +##val +united +233 +2021 +##ons +doi +trivago +overdope +sbs +##ance +##ち +grand +special +573032185 +imf +216 +wx17house +##so +##ーム +audi +##he +london +william +##rp +##ake +science +beach +cfa +amp +ps4 +880 +##800 +##link +##hp +crm +ferragamo +bell +make +##eng +195 +under +zh +photos +2300 +##style +##ント +via +176 +da +##gi +company +i7 +##ray +thomas +370 +ufo +i5 +##max +plc +ben +back +research +8g +173 +mike +##pc +##ッフ +september +189 +##ace +vps +february +167 +pantos +wp +lisa +1921 +★★ +jquery +night +long +offer +##berg +##news +1911 +##いて +ray +fks +wto +せます +over +164 +340 +##all +##rus +1924 +##888 +##works +blogtitle +loftpermalink +##→ +187 +martin +test +ling +km +##め +15000 +fda +v3 +##ja +##ロ +ｗedding +かある +outlet +family +##ea +をこ +##top +story +##ness +salvatore +##lu +204 +swift +215 +room +している +oracle +##ul +1925 +sam +b2c +week +pi +rock +##のは +##ａ +##けと +##ean +##300 +##gle +cctv +after +chinese +##back +powered +x2 +##tan +1918 +##nes +##イン +canon +only +181 +##zi +##las +say +##oe +184 +##sd +221 +##bot +##world +##zo +sky +made +top100 +just +1926 +pmi +802 +234 +gap +##vr +177 +les +174 +▲topoct +ball +vogue +vi +ing +ofweek +cos +##list +##ort +▲topmay +##なら +##lon +として +last +##tc +##of +##bus +##gen +real +eva +##コ +a3 +nas +##lie +##ria +##coin +##bt +▲topapr +his +212 +cat +nata +vive +health +⋯⋯ +drive +sir +▲topmar +du +cup +##カー +##ook +##よう +##sy +alex +msg +tour +しました +3ce +##word +193 +ebooks +r8 +block +318 +##より +2200 +nice +pvp +207 +months +1905 +rewards +##ther +1917 +0800 +##xi +##チ +##sc +micro +850 +gg +blogfp +op +1922 +daily +m1 +264 +true +##bb +ml +##tar +##のお +##ky +anthony +196 +253 +##yo +state +218 +##ara +##aa +##rc +##tz +##ston +より +gear +##eo +##ade +ge +see +1923 +##win +##ura +ss +heart +##den +##ita +down +##sm +el +png +2100 +610 +rakuten +whatsapp +bay +dream +add +##use +680 +311 +pad +gucci +mpv +##ode +##fo +island +▲topjun +##▼ +223 +jason +214 +chicago +##❤ +しの +##hone +io +##れる +##ことか +sogo +be2 +##ology +990 +cloud +vcd +##con +2～3 +##ford +##joy +##kb +##こさいます +##rade +but +##ach +docker +##ful +rfid +ul +##ase +hit +ford +##star +580 +##○ +１１ +a2 +sdk +reading +edited +##are +cmos +##mc +238 +siri +light +##ella +##ため +bloomberg +##read +pizza +##ison +jimmy +##vm +college +node +journal +ba +18k +##play +245 +##cer +２０ +magic +##yu +191 +jump +288 +tt +##ings +asr +##lia +3200 +step5 +network +##cd +mc +いします +1234 +pixstyleme +273 +##600 +2800 +money +★★★★★ +1280 +１２ +430 +bl +みの +act +##tus +tokyo +##rial +##life +emba +##ae +saas +tcs +##rk +##wang +summer +##sp +ko +##ving +390 +premium +##その +netflix +##ヒ +uk +mt +##lton +right +frank +two +209 +える +##ple +##cal +021 +##んな +##sen +##ville +hold +nexus +dd +##ius +てお +##mah +##なく +tila +zero +820 +ce +##tin +resort +##ws +charles +old +p10 +5d +report +##360 +##ru +##には +bus +vans +lt +##est +pv +##レ +links +rebecca +##ツ +##dm +azure +##365 +きな +limited +bit +4gb +##mon +1910 +moto +##eam +213 +1913 +var +eos +なとの +226 +blogspot +された +699 +e3 +dos +dm +fc +##ments +##ik +##kw +boy +##bin +##ata +960 +er +##せ +219 +##vin +##tu +##ula +194 +##∥ +station +##ろ +##ature +835 +files +zara +hdr +top10 +nature +950 +magazine +s6 +marriott +##シ +avira +case +##っと +tab +##ran +tony +##home +oculus +im +##ral +jean +saint +cry +307 +rosie +##force +##ini +ice +##bert +のある +##nder +##mber +pet +2600 +##◆ +plurk +▲topdec +##sis +00kg +▲topnov +720 +##ence +tim +##ω +##nc +##ても +##name +log +ips +great +ikea +malaysia +unix +##イト +3600 +##ncy +##nie +12000 +akb48 +##ye +##oid +404 +##chi +##いた +oa +xuehai +##1000 +##orm +##rf +275 +さん +##ware +##リー +980 +ho +##pro +text +##era +560 +bob +227 +##ub +##2008 +8891 +scp +avi +##zen +2022 +mi +wu +museum +qvod +apache +lake +jcb +▲topaug +★★★ +ni +##hr +hill +302 +ne +weibo +490 +ruby +##ーシ +##ヶ +##row +4d +▲topjul +iv +##ish +github +306 +mate +312 +##スト +##lot +##ane +andrew +のハイト +##tina +t1 +rf +ed2k +##vel +##900 +way +final +りの +ns +5a +705 +197 +##メ +sweet +bytes +##ene +▲topjan +231 +##cker +##2007 +##px +100g +topapp +229 +helpapp +rs +low +14k +g4g +care +630 +ldquo +あり +##fork +leave +rm +edition +##gan +##zon +##qq +▲topsep +##google +##ism +gold +224 +explorer +##zer +toyota +category +select +visual +##labels +restaurant +##md +posts +s1 +##ico +もっと +angelababy +123456 +217 +sports +s3 +mbc +1915 +してくたさい +shell +x86 +candy +##new +kbs +face +xl +470 +##here +4a +swissinfo +v8 +▲topfeb +dram +##ual +##vice +3a +##wer +sport +q1 +ios10 +public +int +card +##ｃ +ep +au +rt +##れた +1080 +bill +##mll +kim +３０ +460 +wan +##uk +##ミ +x3 +298 +0t +scott +##ming +239 +e5 +##3d +h7n9 +worldcat +brown +##あります +##vo +##led +##580 +##ax +249 +410 +##ert +paris +##～6 +polo +925 +##lr +599 +##ナ +capital +##hing +bank +cv +1g +##chat +##ｓ +##たい +adc +##ule +2m +##ｅ +digital +hotmail +268 +##pad +870 +bbq +quot +##ring +before +wali +##まて +mcu +2k +2b +という +costco +316 +north +333 +switch +##city +##ｐ +philips +##mann +management +panasonic +##cl +##vd +##ping +##rge +alice +##lk +##ましょう +css3 +##ney +vision +alpha +##ular +##400 +##tter +lz +にお +##ありません +mode +gre +1916 +pci +##tm +237 +1～2 +##yan +##そ +について +##let +##キ +work +war +coach +ah +mary +##ᅵ +huang +##pt +a8 +pt +follow +##berry +1895 +##ew +a5 +ghost +##ション +##wn +##og +south +##code +girls +##rid +action +villa +git +r11 +table +games +##cket +error +##anonymoussaid +##ag +here +##ame +##gc +qa +##■ +##lis +gmp +##gin +vmalife +##cher +yu +wedding +##tis +demo +dragon +530 +soho +social +bye +##rant +river +orz +acer +325 +##↑ +##ース +##ats +261 +del +##ven +440 +ups +##ように +##ター +305 +value +macd +yougou +##dn +661 +##ano +ll +##urt +##rent +continue +script +##wen +##ect +paper +263 +319 +shift +##chel +##フト +##cat +258 +x5 +fox +243 +##さん +car +aaa +##blog +loading +##yn +##tp +kuso +799 +si +sns +イカせるテンマ +ヒンクテンマ3 +rmb +vdc +forest +central +prime +help +ultra +##rmb +##ような +241 +square +688 +##しい +のないフロクに +##field +##reen +##ors +##ju +c1 +start +510 +##air +##map +cdn +##wo +cba +stephen +m8 +100km +##get +opera +##base +##ood +vsa +com™ +##aw +##ail +251 +なのて +count +t2 +##ᅡ +##een +2700 +hop +##gp +vsc +tree +##eg +##ose +816 +285 +##ories +##shop +alphago +v4 +1909 +simon +##ᆼ +fluke62max +zip +スホンサー +##sta +louis +cr +bas +##～10 +bc +##yer +hadoop +##ube +##wi +1906 +0755 +hola +##low +place +centre +5v +d3 +##fer +252 +##750 +##media +281 +540 +0l +exchange +262 +series +##ハー +##san +eb +##bank +##ｋ +q3 +##nge +##mail +take +##lp +259 +1888 +client +east +cache +event +vincent +##ールを +きを +##nse +sui +855 +adchoice +##и +##stry +##なたの +246 +##zone +ga +apps +sea +##ab +248 +cisco +##タ +##rner +kymco +##care +dha +##pu +##yi +minkoff +royal +p1 +への +annie +269 +collection +kpi +playstation +257 +になります +866 +bh +##bar +queen +505 +radio +1904 +andy +armani +##xy +manager +iherb +##ery +##share +spring +raid +johnson +1908 +##ob +volvo +hall +##ball +v6 +our +taylor +##hk +bi +242 +##cp +kate +bo +water +technology +##rie +サイトは +277 +##ona +##sl +hpv +303 +gtx +hip +rdquo +jayz +stone +##lex +##rum +namespace +##やり +620 +##ale +##atic +des +##erson +##ql +##ves +##type +enter +##この +##てきます +d2 +##168 +##mix +##bian +との +a9 +jj +ky +##lc +access +movie +##hc +リストに +tower +##ration +##mit +ます +##nch +ua +tel +prefix +##o2 +1907 +##point +1901 +ott +～10 +##http +##ury +baidu +##ink +member +##logy +bigbang +nownews +##js +##shot +##tb +##こと +247 +eba +##tics +##lus +ける +v5 +spark +##ama +there +##ions +god +##lls +##down +hiv +##ress +burberry +day2 +##kv +◆◆ +jeff +related +film +edit +joseph +283 +##ark +cx +32gb +order +g9 +30000 +##ans +##tty +s5 +##bee +かあります +thread +xr +buy +sh +005 +land +spotify +mx +##ari +276 +##verse +×email +sf +why +##ことて +244 +7headlines +nego +sunny +dom +exo +401 +666 +positioning +fit +rgb +##tton +278 +kiss +alexa +adam +lp +みリストを +##ｇ +mp +##ties +##llow +amy +##du +np +002 +institute +271 +##rth +##lar +2345 +590 +##des +sidebar +１５ +imax +site +##cky +##kit +##ime +##009 +season +323 +##fun +##ンター +##ひ +gogoro +a7 +pu +lily +fire +twd600 +##ッセーシを +いて +##vis +30ml +##cture +##をお +information +##オ +close +friday +##くれる +yi +nick +てすか +##tta +##tel +6500 +##lock +cbd +economy +254 +かお +267 +tinker +double +375 +8gb +voice +##app +oops +channel +today +985 +##right +raw +xyz +##＋ +jim +edm +##cent +7500 +supreme +814 +ds +##its +##asia +dropbox +##てすか +##tti +books +272 +100ml +##tle +##ller +##ken +##more +##boy +sex +309 +##dom +t3 +##ider +##なります +##unch +1903 +810 +feel +5500 +##かった +##put +により +s2 +mo +##gh +men +ka +amoled +div +##tr +##n1 +port +howard +##tags +ken +dnf +##nus +adsense +##а +ide +##へ +buff +thunder +##town +##ique +has +##body +auto +pin +##erry +tee +てした +295 +number +##the +##013 +object +psp +cool +udnbkk +16gb +##mic +miui +##tro +most +r2 +##alk +##nity +1880 +±0 +##いました +428 +s4 +law +version +##oa +n1 +sgs +docomo +##tf +##ack +henry +fc2 +##ded +##sco +##014 +##rite +286 +0mm +linkedin +##ada +##now +wii +##ndy +ucbug +##◎ +sputniknews +legalminer +##ika +##xp +2gb +##bu +q10 +oo +b6 +come +##rman +cheese +ming +maker +##gm +nikon +##fig +ppi +kelly +##ります +jchere +てきます +ted +md +003 +fgo +tech +##tto +dan +soc +##gl +##len +hair +earth +640 +521 +img +##pper +##a1 +##てきる +##ロク +acca +##ition +##ference +suite +##ig +outlook +##mond +##cation +398 +##pr +279 +101vip +358 +##999 +282 +64gb +3800 +345 +airport +##over +284 +##おり +jones +##ith +lab +##su +##いるのて +co2 +town +piece +##llo +no1 +vmware +24h +##qi +focus +reader +##admin +##ora +tb +false +##log +1898 +know +lan +838 +##ces +f4 +##ume +motel +stop +##oper +na +flickr +netcomponents +##af +##─ +pose +williams +local +##ound +##cg +##site +##iko +いお +274 +5m +gsm +con +##ath +1902 +friends +##hip +cell +317 +##rey +780 +cream +##cks +012 +##dp +facebooktwitterpinterestgoogle +sso +324 +shtml +song +swiss +##mw +##キンク +lumia +xdd +string +tiffany +522 +marc +られた +insee +russell +sc +dell +##ations +ｏｋ +camera +289 +##vs +##flow +##late +classic +287 +##nter +stay +g1 +mtv +512 +##ever +##lab +##nger +qe +sata +ryan +d1 +50ml +cms +##cing +su +292 +3300 +editor +296 +##nap +security +sunday +association +##ens +##700 +##bra +acg +##かり +sofascore +とは +mkv +##ign +jonathan +gary +build +labels +##oto +tesla +moba +qi +gohappy +general +ajax +1024 +##かる +サイト +society +##test +##urs +wps +fedora +##ich +mozilla +328 +##480 +##dr +usa +urn +##lina +##ｒ +grace +##die +##try +##ader +1250 +##なり +elle +570 +##chen +##ᆯ +price +##ten +uhz +##ough +eq +##hen +states +push +session +balance +wow +506 +##cus +##py +when +##ward +##ep +34e +wong +library +prada +##サイト +##cle +running +##ree +313 +ck +date +q4 +##ctive +##ool +##＞ +mk +##ira +##163 +388 +die +secret +rq +dota +buffet +は１ヶ +e6 +##ez +pan +368 +ha +##card +##cha +2a +##さ +alan +day3 +eye +f3 +##end +france +keep +adi +rna +tvbs +##ala +solo +nova +##え +##tail +##ょう +support +##ries +##なる +##ved +base +copy +iis +fps +##ways +hero +hgih +profile +fish +mu +ssh +entertainment +chang +##wd +click +cake +##ond +pre +##tom +kic +pixel +##ov +##fl +product +6a +##pd +dear +##gate +es +yumi +audio +##² +##sky +echo +bin +where +##ture +329 +##ape +find +sap +isis +##なと +nand +##101 +##load +##ream +band +a6 +525 +never +##post +festival +50cm +##we +555 +guide +314 +zenfone +##ike +335 +gd +forum +jessica +strong +alexander +##ould +software +allen +##ious +program +360° +else +lohasthree +##gar +することかてきます +please +##れます +rc +##ggle +##ric +bim +50000 +##own +eclipse +355 +brian +3ds +##side +061 +361 +##other +##ける +##tech +##ator +485 +engine +##ged +##ｔ +plaza +##fit +cia +ngo +westbrook +shi +tbs +50mm +##みませんか +sci +291 +reuters +##ily +contextlink +##hn +af +##cil +bridge +very +##cel +1890 +cambridge +##ize +15g +##aid +##data +790 +frm +##head +award +butler +##sun +meta +##mar +america +ps3 +puma +pmid +##すか +lc +670 +kitchen +##lic +オーフン5 +きなしソフトサーヒス +そして +day1 +future +★★★★ +##text +##page +##rris +pm1 +##ket +fans +##っています +1001 +christian +bot +kids +trackback +##hai +c3 +display +##hl +n2 +1896 +idea +さんも +##sent +airmail +##ug +##men +pwm +けます +028 +##lution +369 +852 +awards +schemas +354 +asics +wikipedia +font +##tional +##vy +c2 +293 +##れている +##dget +##ein +っている +contact +pepper +スキル +339 +##～5 +294 +##uel +##ument +730 +##hang +みてす +q5 +##sue +rain +##ndi +wei +swatch +##cept +わせ +331 +popular +##ste +##tag +p2 +501 +trc +1899 +##west +##live +justin +honda +ping +messenger +##rap +v9 +543 +##とは +unity +appqq +はすへて +025 +leo +##tone +##テ +##ass +uniqlo +##010 +502 +her +jane +memory +moneydj +##tical +human +12306 +していると +##m2 +coc +miacare +##mn +tmt +##core +vim +kk +##may +fan +target +use +too +338 +435 +2050 +867 +737 +fast +##2c +services +##ope +omega +energy +##わ +pinkoi +1a +##なから +##rain +jackson +##ement +##シャンルの +374 +366 +そんな +p9 +rd +##ᆨ +1111 +##tier +##vic +zone +##│ +385 +690 +dl +isofix +cpa +m4 +322 +kimi +めて +davis +##lay +lulu +##uck +050 +weeks +qs +##hop +920 +##ｎ +ae +##ear +～5 +eia +405 +##fly +korea +jpeg +boost +##ship +small +##リア +1860 +eur +297 +425 +valley +##iel +simple +##ude +rn +k2 +##ena +されます +non +patrick +しているから +##ナー +feed +5757 +30g +process +well +qqmei +##thing +they +aws +lu +pink +##ters +##kin +または +board +##vertisement +wine +##ien +unicode +##dge +r1 +359 +##tant +いを +##twitter +##3c +cool1 +される +##れて +##ｌ +isp +##012 +standard +45㎡2 +402 +##150 +matt +##fu +326 +##iner +googlemsn +pixnetfacebookyahoo +##ラン +x7 +886 +##uce +メーカー +sao +##ev +##きました +##file +9678 +403 +xddd +shirt +6l +##rio +##hat +3mm +givenchy +ya +bang +##lio +monday +crystal +ロクイン +##abc +336 +head +890 +ubuntuforumwikilinuxpastechat +##vc +##～20 +##rity +cnc +7866 +ipv6 +null +1897 +##ost +yang +imsean +tiger +##fet +##ンス +352 +##＝ +dji +327 +ji +maria +##come +##んて +foundation +3100 +##beth +##なった +1m +601 +active +##aft +##don +3p +sr +349 +emma +##khz +living +415 +353 +1889 +341 +709 +457 +sas +x6 +##face +pptv +x4 +##mate +han +sophie +##jing +337 +fifa +##mand +other +sale +inwedding +##gn +てきちゃいます +##mmy +##pmlast +bad +nana +nbc +してみてくたさいね +なとはお +##wu +##かあります +##あ +note7 +single +##340 +せからこ +してくたさい♪この +しにはとんとんワークケートを +するとあなたにもっとマッチした +ならワークケートへ +もみつかっちゃうかも +ワークケートの +##bel +window +##dio +##ht +union +age +382 +１４ +##ivity +##ｙ +コメント +domain +neo +##isa +##lter +5k +f5 +steven +##cts +powerpoint +tft +self +g2 +ft +##テル +zol +##act +mwc +381 +343 +もう +nbapop +408 +てある +eds +ace +##room +previous +author +tomtom +il +##ets +hu +financial +☆☆☆ +っています +bp +5t +chi +1gb +##hg +fairmont +cross +008 +gay +h2 +function +##けて +356 +also +1b +625 +##ータ +##raph +1894 +3～5 +##ils +i3 +334 +avenue +##host +による +##bon +##tsu +message +navigation +50g +fintech +h6 +##ことを +8cm +##ject +##vas +##firm +credit +##wf +xxxx +form +##nor +##space +huawei +plan +json +sbl +##dc +machine +921 +392 +wish +##120 +##sol +windows7 +edward +##ために +development +washington +##nsis +lo +818 +##sio +##ym +##bor +planet +##～8 +##wt +ieee +gpa +##めて +camp +ann +gm +##tw +##oka +connect +##rss +##work +##atus +wall +chicken +soul +2mm +##times +fa +##ather +##cord +009 +##eep +hitachi +gui +harry +##pan +e1 +disney +##press +##ーション +wind +386 +frigidaire +##tl +liu +hsu +332 +basic +von +ev +いた +てきる +スホンサーサイト +learning +##ull +expedia +archives +change +##wei +santa +cut +ins +6gb +turbo +brand +cf1 +508 +004 +return +747 +##rip +h1 +##nis +##をこ +128gb +##にお +3t +application +しており +emc +rx +##oon +384 +quick +412 +15058 +wilson +wing +chapter +##bug +beyond +##cms +##dar +##oh +zoom +e2 +trip +sb +##nba +rcep +342 +aspx +ci +080 +gc +gnu +める +##count +advanced +dance +dv +##url +##ging +367 +8591 +am09 +shadow +battle +346 +##ｉ +##cia +##という +emily +##のてす +##tation +host +ff +techorz +sars +##mini +##mporary +##ering +nc +4200 +798 +##next +cma +##mbps +##gas +##ift +##dot +##ィ +455 +##～17 +amana +##りの +426 +##ros +ir +00㎡1 +##eet +##ible +##↓ +710 +ˋ▽ˊ +##aka +dcs +iq +##ｖ +l1 +##lor +maggie +##011 +##iu +588 +##～1 +830 +##gt +1tb +articles +create +##burg +##iki +database +fantasy +##rex +##cam +dlc +dean +##you +hard +path +gaming +victoria +maps +cb +##lee +##itor +overchicstoretvhome +systems +##xt +416 +p3 +sarah +760 +##nan +407 +486 +x9 +install +second +626 +##ann +##ph +##rcle +##nic +860 +##nar +ec +##とう +768 +metro +chocolate +##rian +～4 +##table +##しています +skin +##sn +395 +mountain +##0mm +inparadise +6m +7x24 +ib +4800 +##jia +eeworld +creative +g5 +g3 +357 +parker +ecfa +village +からの +18000 +sylvia +サーヒス +hbl +##ques +##onsored +##x2 +##きます +##v4 +##tein +ie6 +383 +##stack +389 +ver +##ads +##baby +sound +bbe +##110 +##lone +##uid +ads +022 +gundam +351 +thinkpad +006 +scrum +match +##ave +mems +##470 +##oy +##なりました +##talk +glass +lamigo +span +##eme +job +##a5 +jay +wade +kde +498 +##lace +ocean +tvg +##covery +##r3 +##ners +##rea +junior +think +##aine +cover +##ision +##sia +↓↓ +##bow +msi +413 +458 +406 +##love +711 +801 +soft +z2 +##pl +456 +1840 +mobil +mind +##uy +427 +nginx +##oi +めた +##rr +6221 +##mple +##sson +##ーシてす +371 +##nts +91tv +comhd +crv3000 +##uard +1868 +397 +deep +lost +field +gallery +##bia +rate +spf +redis +traction +930 +icloud +011 +なら +fe +jose +372 +##tory +into +sohu +fx +899 +379 +kicstart2 +##hia +すく +##～3 +##sit +ra +２４ +##walk +##xure +500g +##pact +pacific +xa +natural +carlo +##250 +##walker +1850 +##can +cto +gigi +516 +##サー +pen +##hoo +ob +matlab +##ｂ +##yy +13913459 +##iti +mango +##bbs +sense +c5 +oxford +##ニア +walker +jennifer +##ola +course +##bre +701 +##pus +##rder +lucky +075 +##ぁ +ivy +なお +##nia +sotheby +side +##ugh +joy +##orage +##ush +##bat +##dt +364 +r9 +##2d +##gio +511 +country +wear +##lax +##～7 +##moon +393 +seven +study +411 +348 +lonzo +8k +##ェ +evolution +##イフ +##kk +gs +kd +##レス +arduino +344 +b12 +##lux +arpg +##rdon +cook +##x5 +dark +five +##als +##ida +とても +sign +362 +##ちの +something +20mm +##nda +387 +##posted +fresh +tf +1870 +422 +cam +##mine +##skip +##form +##ssion +education +394 +##tee +dyson +stage +##jie +want +##night +epson +pack +あります +##ppy +テリヘル +##█ +wd +##eh +##rence +left +##lvin +golden +mhz +discovery +##trix +##n2 +loft +##uch +##dra +##sse +speed +～1 +1mdb +sorry +welcome +##urn +wave +gaga +##lmer +teddy +##160 +トラックハック +せよ +611 +##f2016 +378 +rp +##sha +rar +##あなたに +##きた +840 +holiday +##ュー +373 +074 +##vg +##nos +##rail +gartner +gi +6p +##dium +kit +488 +b3 +eco +##ろう +20g +sean +##stone +autocad +nu +##np +f16 +write +029 +m5 +##ias +images +atp +##dk +fsm +504 +1350 +ve +52kb +##xxx +##のに +##cake +414 +unit +lim +ru +1v +##ification +published +angela +16g +analytics +ak +##ｑ +##nel +gmt +##icon +again +##₂ +##bby +ios11 +445 +かこさいます +waze +いてす +##ハ +9985 +##ust +##ティー +framework +##007 +iptv +delete +52sykb +cl +wwdc +027 +30cm +##fw +##ての +1389 +##xon +brandt +##ses +##dragon +tc +vetements +anne +monte +modern +official +##へて +##ere +##nne +##oud +もちろん +５０ +etnews +##a2 +##graphy +421 +863 +##ちゃん +444 +##rtex +##てお +l2 +##gma +mount +ccd +たと +archive +morning +tan +ddos +e7 +##ホ +day4 +##ウ +gis +453 +its +495 +factory +bruce +pg +##ito +ってくたさい +guest +cdma +##lling +536 +n3 +しかし +3～4 +mega +eyes +ro +１３ +women +dac +church +##jun +singapore +##facebook +6991 +starbucks +##tos +##stin +##shine +zen +##mu +tina +20℃ +1893 +##たけて +503 +465 +request +##gence +qt +##っ +1886 +347 +363 +q7 +##zzi +diary +##tore +409 +##ead +468 +cst +##osa +canada +agent +va +##jiang +##ちは +##ーク +##lam +sg +##nix +##sday +##よって +g6 +##master +bing +##zl +charlie +１６ +8mm +nb40 +##ーン +thai +##ルフ +ln284ct +##itz +##2f +bonnie +##food +##lent +originals +##stro +##lts +418 +∟∣ +##bscribe +children +ntd +yesstyle +##かも +hmv +##tment +d5 +2cm +arts +sms +##pn +##я +##いい +topios9 +539 +lifestyle +virtual +##ague +xz +##deo +muji +024 +unt +##nnis +##ᅩ +faq1 +1884 +396 +##ette +fly +64㎡ +はしめまして +441 +curry +##pop +のこ +release +##← +##◆◆ +##cast +073 +ありな +500ml +##ews +5c +##stle +ios7 +##ima +787 +dog +lenovo +##r4 +roger +013 +cbs +vornado +100m +417 +##desk +##クok +##ald +1867 +9595 +2900 +##van +oil +##ｘ +some +break +common +##jy +##lines +g7 +twice +419 +ella +nano +belle +にこ +##mes +##self +##note +jb +##ことかてきます +benz +##との +##ova +451 +save +##wing +##ますのて +kai +りは +##hua +##rect +rainer +##unge +448 +##0m +adsl +##かな +guestname +##uma +##kins +##zu +tokichoi +##price +county +##med +##mus +rmk +391 +address +vm +えて +openload +##group +##hin +##iginal +amg +urban +##oz +jobs +emi +##public +beautiful +##sch +album +##dden +##bell +jerry +works +hostel +miller +##drive +##rmin +##１０ +376 +boot +828 +##370 +##fx +##cm～ +1885 +##nome +##ctionary +##oman +##lish +##cr +##hm +433 +##how +432 +francis +xi +c919 +b5 +evernote +##uc +vga +##3000 +coupe +##urg +##cca +##uality +019 +6g +れる +multi +##また +##ett +em +hey +##ani +##tax +##rma +inside +than +740 +leonnhurt +##jin +ict +れた +bird +notes +200mm +くの +##dical +##lli +result +442 +iu +ee +438 +smap +gopro +##last +yin +pure +998 +32g +けた +5kg +##dan +##rame +mama +##oot +bean +marketing +##hur +2l +bella +sync +xuite +##ground +515 +discuz +##getrelax +##ince +##bay +##5s +cj +##イス +gmat +apt +##pass +jing +##rix +c4 +rich +##とても +niusnews +##ello +bag +770 +##eting +##mobile +１８ +culture +015 +##のてすか +377 +1020 +area +##ience +616 +details +gp +universal +silver +dit +はお +private +ddd +u11 +kanshu +##ified +fung +##nny +dx +##520 +tai +475 +023 +##fr +##lean +3s +##pin +429 +##rin +25000 +ly +rick +##bility +usb3 +banner +##baru +##gion +metal +dt +vdf +1871 +karl +qualcomm +bear +1010 +oldid +ian +jo +##tors +population +##ernel +1882 +mmorpg +##mv +##bike +603 +##© +ww +friend +##ager +exhibition +##del +##pods +fpx +structure +##free +##tings +kl +##rley +##copyright +##mma +california +3400 +orange +yoga +4l +canmake +honey +##anda +##コメント +595 +nikkie +##ルハイト +dhl +publishing +##mall +##gnet +20cm +513 +##クセス +##┅ +e88 +970 +##dog +fishbase +##! +##" +### +##$ +##% +##& +##' +##( +##) +##* +##+ +##, +##- +##. +##/ +##: +##; +##< +##= +##> +##? +##@ +##[ +##\ +##] +##^ +##_ +##{ +##| +##} +##~ +##£ +##¤ +##¥ +##§ +##« +##± +##³ +##µ +##· +##¹ +##º +##» +##¼ +##ß +##æ +##÷ +##ø +##đ +##ŋ +##ɔ +##ə +##ɡ +##ʰ +##ˇ +##ˈ +##ˊ +##ˋ +##ˍ +##ː +##˙ +##˚ +##ˢ +##α +##β +##γ +##δ +##ε +##η +##θ +##ι +##κ +##λ +##μ +##ν +##ο +##π +##ρ +##ς +##σ +##τ +##υ +##φ +##χ +##ψ +##б +##в +##г +##д +##е +##ж +##з +##к +##л +##м +##н +##о +##п +##р +##с +##т +##у +##ф +##х +##ц +##ч +##ш +##ы +##ь +##і +##ا +##ب +##ة +##ت +##د +##ر +##س +##ع +##ل +##م +##ن +##ه +##و +##ي +##۩ +##ก +##ง +##น +##ม +##ย +##ร +##อ +##า +##เ +##๑ +##་ +##ღ +##ᄀ +##ᄁ +##ᄂ +##ᄃ +##ᄅ +##ᄆ +##ᄇ +##ᄈ +##ᄉ +##ᄋ +##ᄌ +##ᄎ +##ᄏ +##ᄐ +##ᄑ +##ᄒ +##ᅢ +##ᅣ +##ᅥ +##ᅦ +##ᅧ +##ᅨ +##ᅪ +##ᅬ +##ᅭ +##ᅮ +##ᅯ +##ᅲ +##ᅳ +##ᅴ +##ᆷ +##ᆸ +##ᆺ +##ᆻ +##ᗜ +##ᵃ +##ᵉ +##ᵍ +##ᵏ +##ᵐ +##ᵒ +##ᵘ +##‖ +##„ +##† +##• +##‥ +##‧ +##
 +##‰ +##′ +##″ +##‹ +##› +##※ +##‿ +##⁄ +##ⁱ +##⁺ +##ⁿ +##₁ +##₃ +##₄ +##€ +##№ +##ⅰ +##ⅱ +##ⅲ +##ⅳ +##ⅴ +##↔ +##↗ +##↘ +##⇒ +##∀ +##− +##∕ +##∙ +##√ +##∞ +##∟ +##∠ +##∣ +##∩ +##∮ +##∶ +##∼ +##∽ +##≈ +##≒ +##≡ +##≤ +##≥ +##≦ +##≧ +##≪ +##≫ +##⊙ +##⋅ +##⋈ +##⋯ +##⌒ +##① +##② +##③ +##④ +##⑤ +##⑥ +##⑦ +##⑧ +##⑨ +##⑩ +##⑴ +##⑵ +##⑶ +##⑷ +##⑸ +##⒈ +##⒉ +##⒊ +##⒋ +##ⓒ +##ⓔ +##ⓘ +##━ +##┃ +##┆ +##┊ +##┌ +##└ +##├ +##┣ +##═ +##║ +##╚ +##╞ +##╠ +##╭ +##╮ +##╯ +##╰ +##╱ +##╳ +##▂ +##▃ +##▅ +##▇ +##▉ +##▋ +##▌ +##▍ +##▎ +##□ +##▪ +##▫ +##▬ +##△ +##▶ +##► +##▽ +##◇ +##◕ +##◠ +##◢ +##◤ +##☀ +##☕ +##☞ +##☺ +##☼ +##♀ +##♂ +##♠ +##♡ +##♣ +##♦ +##♫ +##♬ +##✈ +##✔ +##✕ +##✖ +##✦ +##✨ +##✪ +##✰ +##✿ +##❀ +##➜ +##➤ +##⦿ +##、 +##。 +##〃 +##々 +##〇 +##〈 +##〉 +##《 +##》 +##「 +##」 +##『 +##』 +##【 +##】 +##〓 +##〔 +##〕 +##〖 +##〗 +##〜 +##〝 +##〞 +##ぃ +##ぇ +##ぬ +##ふ +##ほ +##む +##ゃ +##ゅ +##ゆ +##ょ +##゜ +##ゝ +##ァ +##ゥ +##エ +##ォ +##ケ +##サ +##セ +##ソ +##ッ +##ニ +##ヌ +##ネ +##ノ +##ヘ +##モ +##ャ +##ヤ +##ュ +##ユ +##ョ +##ヨ +##ワ +##ヲ +##・ +##ヽ +##ㄅ +##ㄆ +##ㄇ +##ㄉ +##ㄋ +##ㄌ +##ㄍ +##ㄎ +##ㄏ +##ㄒ +##ㄚ +##ㄛ +##ㄞ +##ㄟ +##ㄢ +##ㄤ +##ㄥ +##ㄧ +##ㄨ +##ㆍ +##㈦ +##㊣ +##㗎 +##一 +##丁 +##七 +##万 +##丈 +##三 +##上 +##下 +##不 +##与 +##丐 +##丑 +##专 +##且 +##丕 +##世 +##丘 +##丙 +##业 +##丛 +##东 +##丝 +##丞 +##丟 +##両 +##丢 +##两 +##严 +##並 +##丧 +##丨 +##个 +##丫 +##中 +##丰 +##串 +##临 +##丶 +##丸 +##丹 +##为 +##主 +##丼 +##丽 +##举 +##丿 +##乂 +##乃 +##久 +##么 +##义 +##之 +##乌 +##乍 +##乎 +##乏 +##乐 +##乒 +##乓 +##乔 +##乖 +##乗 +##乘 +##乙 +##乜 +##九 +##乞 +##也 +##习 +##乡 +##书 +##乩 +##买 +##乱 +##乳 +##乾 +##亀 +##亂 +##了 +##予 +##争 +##事 +##二 +##于 +##亏 +##云 +##互 +##五 +##井 +##亘 +##亙 +##亚 +##些 +##亜 +##亞 +##亟 +##亡 +##亢 +##交 +##亥 +##亦 +##产 +##亨 +##亩 +##享 +##京 +##亭 +##亮 +##亲 +##亳 +##亵 +##人 +##亿 +##什 +##仁 +##仃 +##仄 +##仅 +##仆 +##仇 +##今 +##介 +##仍 +##从 +##仏 +##仑 +##仓 +##仔 +##仕 +##他 +##仗 +##付 +##仙 +##仝 +##仞 +##仟 +##代 +##令 +##以 +##仨 +##仪 +##们 +##仮 +##仰 +##仲 +##件 +##价 +##任 +##份 +##仿 +##企 +##伉 +##伊 +##伍 +##伎 +##伏 +##伐 +##休 +##伕 +##众 +##优 +##伙 +##会 +##伝 +##伞 +##伟 +##传 +##伢 +##伤 +##伦 +##伪 +##伫 +##伯 +##估 +##伴 +##伶 +##伸 +##伺 +##似 +##伽 +##佃 +##但 +##佇 +##佈 +##位 +##低 +##住 +##佐 +##佑 +##体 +##佔 +##何 +##佗 +##佘 +##余 +##佚 +##佛 +##作 +##佝 +##佞 +##佟 +##你 +##佢 +##佣 +##佤 +##佥 +##佩 +##佬 +##佯 +##佰 +##佳 +##併 +##佶 +##佻 +##佼 +##使 +##侃 +##侄 +##來 +##侈 +##例 +##侍 +##侏 +##侑 +##侖 +##侗 +##供 +##依 +##侠 +##価 +##侣 +##侥 +##侦 +##侧 +##侨 +##侬 +##侮 +##侯 +##侵 +##侶 +##侷 +##便 +##係 +##促 +##俄 +##俊 +##俎 +##俏 +##俐 +##俑 +##俗 +##俘 +##俚 +##保 +##俞 +##俟 +##俠 +##信 +##俨 +##俩 +##俪 +##俬 +##俭 +##修 +##俯 +##俱 +##俳 +##俸 +##俺 +##俾 +##倆 +##倉 +##個 +##倌 +##倍 +##倏 +##們 +##倒 +##倔 +##倖 +##倘 +##候 +##倚 +##倜 +##借 +##倡 +##値 +##倦 +##倩 +##倪 +##倫 +##倬 +##倭 +##倶 +##债 +##值 +##倾 +##偃 +##假 +##偈 +##偉 +##偌 +##偎 +##偏 +##偕 +##做 +##停 +##健 +##側 +##偵 +##偶 +##偷 +##偻 +##偽 +##偿 +##傀 +##傅 +##傍 +##傑 +##傘 +##備 +##傚 +##傢 +##傣 +##傥 +##储 +##傩 +##催 +##傭 +##傲 +##傳 +##債 +##傷 +##傻 +##傾 +##僅 +##働 +##像 +##僑 +##僕 +##僖 +##僚 +##僥 +##僧 +##僭 +##僮 +##僱 +##僵 +##價 +##僻 +##儀 +##儂 +##億 +##儆 +##儉 +##儋 +##儒 +##儕 +##儘 +##償 +##儡 +##優 +##儲 +##儷 +##儼 +##儿 +##兀 +##允 +##元 +##兄 +##充 +##兆 +##兇 +##先 +##光 +##克 +##兌 +##免 +##児 +##兑 +##兒 +##兔 +##兖 +##党 +##兜 +##兢 +##入 +##內 +##全 +##兩 +##八 +##公 +##六 +##兮 +##兰 +##共 +##兲 +##关 +##兴 +##兵 +##其 +##具 +##典 +##兹 +##养 +##兼 +##兽 +##冀 +##内 +##円 +##冇 +##冈 +##冉 +##冊 +##册 +##再 +##冏 +##冒 +##冕 +##冗 +##写 +##军 +##农 +##冠 +##冢 +##冤 +##冥 +##冨 +##冪 +##冬 +##冯 +##冰 +##冲 +##决 +##况 +##冶 +##冷 +##冻 +##冼 +##冽 +##冾 +##净 +##凄 +##准 +##凇 +##凈 +##凉 +##凋 +##凌 +##凍 +##减 +##凑 +##凛 +##凜 +##凝 +##几 +##凡 +##凤 +##処 +##凪 +##凭 +##凯 +##凰 +##凱 +##凳 +##凶 +##凸 +##凹 +##出 +##击 +##函 +##凿 +##刀 +##刁 +##刃 +##分 +##切 +##刈 +##刊 +##刍 +##刎 +##刑 +##划 +##列 +##刘 +##则 +##刚 +##创 +##初 +##删 +##判 +##別 +##刨 +##利 +##刪 +##别 +##刮 +##到 +##制 +##刷 +##券 +##刹 +##刺 +##刻 +##刽 +##剁 +##剂 +##剃 +##則 +##剉 +##削 +##剋 +##剌 +##前 +##剎 +##剐 +##剑 +##剔 +##剖 +##剛 +##剜 +##剝 +##剣 +##剤 +##剥 +##剧 +##剩 +##剪 +##副 +##割 +##創 +##剷 +##剽 +##剿 +##劃 +##劇 +##劈 +##劉 +##劊 +##劍 +##劏 +##劑 +##力 +##劝 +##办 +##功 +##加 +##务 +##劣 +##动 +##助 +##努 +##劫 +##劭 +##励 +##劲 +##劳 +##労 +##劵 +##効 +##劾 +##势 +##勁 +##勃 +##勇 +##勉 +##勋 +##勐 +##勒 +##動 +##勖 +##勘 +##務 +##勛 +##勝 +##勞 +##募 +##勢 +##勤 +##勧 +##勳 +##勵 +##勸 +##勺 +##勻 +##勾 +##勿 +##匀 +##包 +##匆 +##匈 +##匍 +##匐 +##匕 +##化 +##北 +##匙 +##匝 +##匠 +##匡 +##匣 +##匪 +##匮 +##匯 +##匱 +##匹 +##区 +##医 +##匾 +##匿 +##區 +##十 +##千 +##卅 +##升 +##午 +##卉 +##半 +##卍 +##华 +##协 +##卑 +##卒 +##卓 +##協 +##单 +##卖 +##南 +##単 +##博 +##卜 +##卞 +##卟 +##占 +##卡 +##卢 +##卤 +##卦 +##卧 +##卫 +##卮 +##卯 +##印 +##危 +##即 +##却 +##卵 +##卷 +##卸 +##卻 +##卿 +##厂 +##厄 +##厅 +##历 +##厉 +##压 +##厌 +##厕 +##厘 +##厚 +##厝 +##原 +##厢 +##厥 +##厦 +##厨 +##厩 +##厭 +##厮 +##厲 +##厳 +##去 +##县 +##叁 +##参 +##參 +##又 +##叉 +##及 +##友 +##双 +##反 +##収 +##发 +##叔 +##取 +##受 +##变 +##叙 +##叛 +##叟 +##叠 +##叡 +##叢 +##口 +##古 +##句 +##另 +##叨 +##叩 +##只 +##叫 +##召 +##叭 +##叮 +##可 +##台 +##叱 +##史 +##右 +##叵 +##叶 +##号 +##司 +##叹 +##叻 +##叼 +##叽 +##吁 +##吃 +##各 +##吆 +##合 +##吉 +##吊 +##吋 +##同 +##名 +##后 +##吏 +##吐 +##向 +##吒 +##吓 +##吕 +##吖 +##吗 +##君 +##吝 +##吞 +##吟 +##吠 +##吡 +##否 +##吧 +##吨 +##吩 +##含 +##听 +##吭 +##吮 +##启 +##吱 +##吳 +##吴 +##吵 +##吶 +##吸 +##吹 +##吻 +##吼 +##吽 +##吾 +##呀 +##呂 +##呃 +##呆 +##呈 +##告 +##呋 +##呎 +##呐 +##呓 +##呕 +##呗 +##员 +##呛 +##呜 +##呢 +##呤 +##呦 +##周 +##呱 +##呲 +##味 +##呵 +##呷 +##呸 +##呻 +##呼 +##命 +##咀 +##咁 +##咂 +##咄 +##咆 +##咋 +##和 +##咎 +##咏 +##咐 +##咒 +##咔 +##咕 +##咖 +##咗 +##咘 +##咙 +##咚 +##咛 +##咣 +##咤 +##咦 +##咧 +##咨 +##咩 +##咪 +##咫 +##咬 +##咭 +##咯 +##咱 +##咲 +##咳 +##咸 +##咻 +##咽 +##咿 +##哀 +##品 +##哂 +##哄 +##哆 +##哇 +##哈 +##哉 +##哋 +##哌 +##响 +##哎 +##哏 +##哐 +##哑 +##哒 +##哔 +##哗 +##哟 +##員 +##哥 +##哦 +##哧 +##哨 +##哩 +##哪 +##哭 +##哮 +##哲 +##哺 +##哼 +##哽 +##唁 +##唄 +##唆 +##唇 +##唉 +##唏 +##唐 +##唑 +##唔 +##唠 +##唤 +##唧 +##唬 +##售 +##唯 +##唰 +##唱 +##唳 +##唷 +##唸 +##唾 +##啃 +##啄 +##商 +##啉 +##啊 +##問 +##啓 +##啕 +##啖 +##啜 +##啞 +##啟 +##啡 +##啤 +##啥 +##啦 +##啧 +##啪 +##啫 +##啬 +##啮 +##啰 +##啱 +##啲 +##啵 +##啶 +##啷 +##啸 +##啻 +##啼 +##啾 +##喀 +##喂 +##喃 +##善 +##喆 +##喇 +##喉 +##喊 +##喋 +##喎 +##喏 +##喔 +##喘 +##喙 +##喚 +##喜 +##喝 +##喟 +##喧 +##喪 +##喫 +##喬 +##單 +##喰 +##喱 +##喲 +##喳 +##喵 +##営 +##喷 +##喹 +##喺 +##喻 +##喽 +##嗅 +##嗆 +##嗇 +##嗎 +##嗑 +##嗒 +##嗓 +##嗔 +##嗖 +##嗚 +##嗜 +##嗝 +##嗟 +##嗡 +##嗣 +##嗤 +##嗦 +##嗨 +##嗪 +##嗬 +##嗯 +##嗰 +##嗲 +##嗳 +##嗶 +##嗷 +##嗽 +##嘀 +##嘅 +##嘆 +##嘈 +##嘉 +##嘌 +##嘍 +##嘎 +##嘔 +##嘖 +##嘗 +##嘘 +##嘚 +##嘛 +##嘜 +##嘞 +##嘟 +##嘢 +##嘣 +##嘤 +##嘧 +##嘩 +##嘭 +##嘮 +##嘯 +##嘰 +##嘱 +##嘲 +##嘴 +##嘶 +##嘸 +##嘹 +##嘻 +##嘿 +##噁 +##噌 +##噎 +##噓 +##噔 +##噗 +##噙 +##噜 +##噠 +##噢 +##噤 +##器 +##噩 +##噪 +##噬 +##噱 +##噴 +##噶 +##噸 +##噹 +##噻 +##噼 +##嚀 +##嚇 +##嚎 +##嚏 +##嚐 +##嚓 +##嚕 +##嚟 +##嚣 +##嚥 +##嚨 +##嚮 +##嚴 +##嚷 +##嚼 +##囂 +##囉 +##囊 +##囍 +##囑 +##囔 +##囗 +##囚 +##四 +##囝 +##回 +##囟 +##因 +##囡 +##团 +##団 +##囤 +##囧 +##囪 +##囫 +##园 +##困 +##囱 +##囲 +##図 +##围 +##囹 +##固 +##国 +##图 +##囿 +##圃 +##圄 +##圆 +##圈 +##國 +##圍 +##圏 +##園 +##圓 +##圖 +##團 +##圜 +##土 +##圣 +##圧 +##在 +##圩 +##圭 +##地 +##圳 +##场 +##圻 +##圾 +##址 +##坂 +##均 +##坊 +##坍 +##坎 +##坏 +##坐 +##坑 +##块 +##坚 +##坛 +##坝 +##坞 +##坟 +##坠 +##坡 +##坤 +##坦 +##坨 +##坪 +##坯 +##坳 +##坵 +##坷 +##垂 +##垃 +##垄 +##型 +##垒 +##垚 +##垛 +##垠 +##垢 +##垣 +##垦 +##垩 +##垫 +##垭 +##垮 +##垵 +##埂 +##埃 +##埋 +##城 +##埔 +##埕 +##埗 +##域 +##埠 +##埤 +##埵 +##執 +##埸 +##培 +##基 +##埼 +##堀 +##堂 +##堃 +##堅 +##堆 +##堇 +##堑 +##堕 +##堙 +##堡 +##堤 +##堪 +##堯 +##堰 +##報 +##場 +##堵 +##堺 +##堿 +##塊 +##塌 +##塑 +##塔 +##塗 +##塘 +##塚 +##塞 +##塢 +##塩 +##填 +##塬 +##塭 +##塵 +##塾 +##墀 +##境 +##墅 +##墉 +##墊 +##墒 +##墓 +##増 +##墘 +##墙 +##墜 +##增 +##墟 +##墨 +##墩 +##墮 +##墳 +##墻 +##墾 +##壁 +##壅 +##壆 +##壇 +##壊 +##壑 +##壓 +##壕 +##壘 +##壞 +##壟 +##壢 +##壤 +##壩 +##士 +##壬 +##壮 +##壯 +##声 +##売 +##壳 +##壶 +##壹 +##壺 +##壽 +##处 +##备 +##変 +##复 +##夏 +##夔 +##夕 +##外 +##夙 +##多 +##夜 +##够 +##夠 +##夢 +##夥 +##大 +##天 +##太 +##夫 +##夭 +##央 +##夯 +##失 +##头 +##夷 +##夸 +##夹 +##夺 +##夾 +##奂 +##奄 +##奇 +##奈 +##奉 +##奋 +##奎 +##奏 +##奐 +##契 +##奔 +##奕 +##奖 +##套 +##奘 +##奚 +##奠 +##奢 +##奥 +##奧 +##奪 +##奬 +##奮 +##女 +##奴 +##奶 +##奸 +##她 +##好 +##如 +##妃 +##妄 +##妆 +##妇 +##妈 +##妊 +##妍 +##妒 +##妓 +##妖 +##妘 +##妙 +##妝 +##妞 +##妣 +##妤 +##妥 +##妨 +##妩 +##妪 +##妮 +##妲 +##妳 +##妹 +##妻 +##妾 +##姆 +##姉 +##姊 +##始 +##姍 +##姐 +##姑 +##姒 +##姓 +##委 +##姗 +##姚 +##姜 +##姝 +##姣 +##姥 +##姦 +##姨 +##姪 +##姫 +##姬 +##姹 +##姻 +##姿 +##威 +##娃 +##娄 +##娅 +##娆 +##娇 +##娉 +##娑 +##娓 +##娘 +##娛 +##娜 +##娟 +##娠 +##娣 +##娥 +##娩 +##娱 +##娲 +##娴 +##娶 +##娼 +##婀 +##婁 +##婆 +##婉 +##婊 +##婕 +##婚 +##婢 +##婦 +##婧 +##婪 +##婭 +##婴 +##婵 +##婶 +##婷 +##婺 +##婿 +##媒 +##媚 +##媛 +##媞 +##媧 +##媲 +##媳 +##媽 +##媾 +##嫁 +##嫂 +##嫉 +##嫌 +##嫑 +##嫔 +##嫖 +##嫘 +##嫚 +##嫡 +##嫣 +##嫦 +##嫩 +##嫲 +##嫵 +##嫻 +##嬅 +##嬉 +##嬌 +##嬗 +##嬛 +##嬢 +##嬤 +##嬪 +##嬰 +##嬴 +##嬷 +##嬸 +##嬿 +##孀 +##孃 +##子 +##孑 +##孔 +##孕 +##孖 +##字 +##存 +##孙 +##孚 +##孛 +##孜 +##孝 +##孟 +##孢 +##季 +##孤 +##学 +##孩 +##孪 +##孫 +##孬 +##孰 +##孱 +##孳 +##孵 +##學 +##孺 +##孽 +##孿 +##宁 +##它 +##宅 +##宇 +##守 +##安 +##宋 +##完 +##宏 +##宓 +##宕 +##宗 +##官 +##宙 +##定 +##宛 +##宜 +##宝 +##实 +##実 +##宠 +##审 +##客 +##宣 +##室 +##宥 +##宦 +##宪 +##宫 +##宮 +##宰 +##害 +##宴 +##宵 +##家 +##宸 +##容 +##宽 +##宾 +##宿 +##寂 +##寄 +##寅 +##密 +##寇 +##富 +##寐 +##寒 +##寓 +##寛 +##寝 +##寞 +##察 +##寡 +##寢 +##寥 +##實 +##寧 +##寨 +##審 +##寫 +##寬 +##寮 +##寰 +##寵 +##寶 +##寸 +##对 +##寺 +##寻 +##导 +##対 +##寿 +##封 +##専 +##射 +##将 +##將 +##專 +##尉 +##尊 +##尋 +##對 +##導 +##小 +##少 +##尔 +##尕 +##尖 +##尘 +##尚 +##尝 +##尤 +##尧 +##尬 +##就 +##尴 +##尷 +##尸 +##尹 +##尺 +##尻 +##尼 +##尽 +##尾 +##尿 +##局 +##屁 +##层 +##屄 +##居 +##屆 +##屈 +##屉 +##届 +##屋 +##屌 +##屍 +##屎 +##屏 +##屐 +##屑 +##展 +##屜 +##属 +##屠 +##屡 +##屢 +##層 +##履 +##屬 +##屯 +##山 +##屹 +##屿 +##岀 +##岁 +##岂 +##岌 +##岐 +##岑 +##岔 +##岖 +##岗 +##岘 +##岙 +##岚 +##岛 +##岡 +##岩 +##岫 +##岬 +##岭 +##岱 +##岳 +##岷 +##岸 +##峇 +##峋 +##峒 +##峙 +##峡 +##峤 +##峥 +##峦 +##峨 +##峪 +##峭 +##峯 +##峰 +##峴 +##島 +##峻 +##峽 +##崁 +##崂 +##崆 +##崇 +##崎 +##崑 +##崔 +##崖 +##崗 +##崙 +##崛 +##崧 +##崩 +##崭 +##崴 +##崽 +##嵇 +##嵊 +##嵋 +##嵌 +##嵐 +##嵘 +##嵩 +##嵬 +##嵯 +##嶂 +##嶄 +##嶇 +##嶋 +##嶙 +##嶺 +##嶼 +##嶽 +##巅 +##巍 +##巒 +##巔 +##巖 +##川 +##州 +##巡 +##巢 +##工 +##左 +##巧 +##巨 +##巩 +##巫 +##差 +##己 +##已 +##巳 +##巴 +##巷 +##巻 +##巽 +##巾 +##巿 +##币 +##市 +##布 +##帅 +##帆 +##师 +##希 +##帐 +##帑 +##帕 +##帖 +##帘 +##帚 +##帛 +##帜 +##帝 +##帥 +##带 +##帧 +##師 +##席 +##帮 +##帯 +##帰 +##帳 +##帶 +##帷 +##常 +##帼 +##帽 +##幀 +##幂 +##幄 +##幅 +##幌 +##幔 +##幕 +##幟 +##幡 +##幢 +##幣 +##幫 +##干 +##平 +##年 +##并 +##幸 +##幹 +##幺 +##幻 +##幼 +##幽 +##幾 +##广 +##庁 +##広 +##庄 +##庆 +##庇 +##床 +##序 +##庐 +##库 +##应 +##底 +##庖 +##店 +##庙 +##庚 +##府 +##庞 +##废 +##庠 +##度 +##座 +##庫 +##庭 +##庵 +##庶 +##康 +##庸 +##庹 +##庾 +##廁 +##廂 +##廃 +##廈 +##廉 +##廊 +##廓 +##廖 +##廚 +##廝 +##廟 +##廠 +##廢 +##廣 +##廬 +##廳 +##延 +##廷 +##建 +##廿 +##开 +##弁 +##异 +##弃 +##弄 +##弈 +##弊 +##弋 +##式 +##弑 +##弒 +##弓 +##弔 +##引 +##弗 +##弘 +##弛 +##弟 +##张 +##弥 +##弦 +##弧 +##弩 +##弭 +##弯 +##弱 +##張 +##強 +##弹 +##强 +##弼 +##弾 +##彅 +##彆 +##彈 +##彌 +##彎 +##归 +##当 +##录 +##彗 +##彙 +##彝 +##形 +##彤 +##彥 +##彦 +##彧 +##彩 +##彪 +##彫 +##彬 +##彭 +##彰 +##影 +##彷 +##役 +##彻 +##彼 +##彿 +##往 +##征 +##径 +##待 +##徇 +##很 +##徉 +##徊 +##律 +##後 +##徐 +##徑 +##徒 +##従 +##徕 +##得 +##徘 +##徙 +##徜 +##從 +##徠 +##御 +##徨 +##復 +##循 +##徬 +##微 +##徳 +##徴 +##徵 +##德 +##徹 +##徼 +##徽 +##心 +##必 +##忆 +##忌 +##忍 +##忏 +##忐 +##忑 +##忒 +##忖 +##志 +##忘 +##忙 +##応 +##忠 +##忡 +##忤 +##忧 +##忪 +##快 +##忱 +##念 +##忻 +##忽 +##忿 +##怀 +##态 +##怂 +##怅 +##怆 +##怎 +##怏 +##怒 +##怔 +##怕 +##怖 +##怙 +##怜 +##思 +##怠 +##怡 +##急 +##怦 +##性 +##怨 +##怪 +##怯 +##怵 +##总 +##怼 +##恁 +##恃 +##恆 +##恋 +##恍 +##恐 +##恒 +##恕 +##恙 +##恚 +##恢 +##恣 +##恤 +##恥 +##恨 +##恩 +##恪 +##恫 +##恬 +##恭 +##息 +##恰 +##恳 +##恵 +##恶 +##恸 +##恺 +##恻 +##恼 +##恿 +##悄 +##悅 +##悉 +##悌 +##悍 +##悔 +##悖 +##悚 +##悟 +##悠 +##患 +##悦 +##您 +##悩 +##悪 +##悬 +##悯 +##悱 +##悲 +##悴 +##悵 +##悶 +##悸 +##悻 +##悼 +##悽 +##情 +##惆 +##惇 +##惊 +##惋 +##惑 +##惕 +##惘 +##惚 +##惜 +##惟 +##惠 +##惡 +##惦 +##惧 +##惨 +##惩 +##惫 +##惬 +##惭 +##惮 +##惯 +##惰 +##惱 +##想 +##惴 +##惶 +##惹 +##惺 +##愁 +##愆 +##愈 +##愉 +##愍 +##意 +##愕 +##愚 +##愛 +##愜 +##感 +##愣 +##愤 +##愧 +##愫 +##愷 +##愿 +##慄 +##慈 +##態 +##慌 +##慎 +##慑 +##慕 +##慘 +##慚 +##慟 +##慢 +##慣 +##慧 +##慨 +##慫 +##慮 +##慰 +##慳 +##慵 +##慶 +##慷 +##慾 +##憂 +##憊 +##憋 +##憎 +##憐 +##憑 +##憔 +##憚 +##憤 +##憧 +##憨 +##憩 +##憫 +##憬 +##憲 +##憶 +##憾 +##懂 +##懇 +##懈 +##應 +##懊 +##懋 +##懑 +##懒 +##懦 +##懲 +##懵 +##懶 +##懷 +##懸 +##懺 +##懼 +##懾 +##懿 +##戀 +##戈 +##戊 +##戌 +##戍 +##戎 +##戏 +##成 +##我 +##戒 +##戕 +##或 +##战 +##戚 +##戛 +##戟 +##戡 +##戦 +##截 +##戬 +##戮 +##戰 +##戲 +##戳 +##戴 +##戶 +##户 +##戸 +##戻 +##戾 +##房 +##所 +##扁 +##扇 +##扈 +##扉 +##手 +##才 +##扎 +##扑 +##扒 +##打 +##扔 +##払 +##托 +##扛 +##扣 +##扦 +##执 +##扩 +##扪 +##扫 +##扬 +##扭 +##扮 +##扯 +##扰 +##扱 +##扳 +##扶 +##批 +##扼 +##找 +##承 +##技 +##抄 +##抉 +##把 +##抑 +##抒 +##抓 +##投 +##抖 +##抗 +##折 +##抚 +##抛 +##抜 +##択 +##抟 +##抠 +##抡 +##抢 +##护 +##报 +##抨 +##披 +##抬 +##抱 +##抵 +##抹 +##押 +##抽 +##抿 +##拂 +##拄 +##担 +##拆 +##拇 +##拈 +##拉 +##拋 +##拌 +##拍 +##拎 +##拐 +##拒 +##拓 +##拔 +##拖 +##拗 +##拘 +##拙 +##拚 +##招 +##拜 +##拟 +##拡 +##拢 +##拣 +##拥 +##拦 +##拧 +##拨 +##择 +##括 +##拭 +##拮 +##拯 +##拱 +##拳 +##拴 +##拷 +##拼 +##拽 +##拾 +##拿 +##持 +##挂 +##指 +##挈 +##按 +##挎 +##挑 +##挖 +##挙 +##挚 +##挛 +##挝 +##挞 +##挟 +##挠 +##挡 +##挣 +##挤 +##挥 +##挨 +##挪 +##挫 +##振 +##挲 +##挹 +##挺 +##挽 +##挾 +##捂 +##捅 +##捆 +##捉 +##捋 +##捌 +##捍 +##捎 +##捏 +##捐 +##捕 +##捞 +##损 +##捡 +##换 +##捣 +##捧 +##捨 +##捩 +##据 +##捱 +##捲 +##捶 +##捷 +##捺 +##捻 +##掀 +##掂 +##掃 +##掇 +##授 +##掉 +##掌 +##掏 +##掐 +##排 +##掖 +##掘 +##掙 +##掛 +##掠 +##採 +##探 +##掣 +##接 +##控 +##推 +##掩 +##措 +##掬 +##掰 +##掲 +##掳 +##掴 +##掷 +##掸 +##掺 +##揀 +##揃 +##揄 +##揆 +##揉 +##揍 +##描 +##提 +##插 +##揖 +##揚 +##換 +##握 +##揣 +##揩 +##揪 +##揭 +##揮 +##援 +##揶 +##揸 +##揹 +##揽 +##搀 +##搁 +##搂 +##搅 +##損 +##搏 +##搐 +##搓 +##搔 +##搖 +##搗 +##搜 +##搞 +##搡 +##搪 +##搬 +##搭 +##搵 +##搶 +##携 +##搽 +##摀 +##摁 +##摄 +##摆 +##摇 +##摈 +##摊 +##摒 +##摔 +##摘 +##摞 +##摟 +##摧 +##摩 +##摯 +##摳 +##摸 +##摹 +##摺 +##摻 +##撂 +##撃 +##撅 +##撇 +##撈 +##撐 +##撑 +##撒 +##撓 +##撕 +##撚 +##撞 +##撤 +##撥 +##撩 +##撫 +##撬 +##播 +##撮 +##撰 +##撲 +##撵 +##撷 +##撸 +##撻 +##撼 +##撿 +##擀 +##擁 +##擂 +##擄 +##擅 +##擇 +##擊 +##擋 +##操 +##擎 +##擒 +##擔 +##擘 +##據 +##擞 +##擠 +##擡 +##擢 +##擦 +##擬 +##擰 +##擱 +##擲 +##擴 +##擷 +##擺 +##擼 +##擾 +##攀 +##攏 +##攒 +##攔 +##攘 +##攙 +##攜 +##攝 +##攞 +##攢 +##攣 +##攤 +##攥 +##攪 +##攫 +##攬 +##支 +##收 +##攸 +##改 +##攻 +##放 +##政 +##故 +##效 +##敌 +##敍 +##敎 +##敏 +##救 +##敕 +##敖 +##敗 +##敘 +##教 +##敛 +##敝 +##敞 +##敢 +##散 +##敦 +##敬 +##数 +##敲 +##整 +##敵 +##敷 +##數 +##斂 +##斃 +##文 +##斋 +##斌 +##斎 +##斐 +##斑 +##斓 +##斗 +##料 +##斛 +##斜 +##斟 +##斡 +##斤 +##斥 +##斧 +##斩 +##斫 +##斬 +##断 +##斯 +##新 +##斷 +##方 +##於 +##施 +##旁 +##旃 +##旅 +##旋 +##旌 +##旎 +##族 +##旖 +##旗 +##无 +##既 +##日 +##旦 +##旧 +##旨 +##早 +##旬 +##旭 +##旮 +##旱 +##时 +##旷 +##旺 +##旻 +##昀 +##昂 +##昆 +##昇 +##昉 +##昊 +##昌 +##明 +##昏 +##易 +##昔 +##昕 +##昙 +##星 +##映 +##春 +##昧 +##昨 +##昭 +##是 +##昱 +##昴 +##昵 +##昶 +##昼 +##显 +##晁 +##時 +##晃 +##晉 +##晋 +##晌 +##晏 +##晒 +##晓 +##晔 +##晕 +##晖 +##晗 +##晚 +##晝 +##晞 +##晟 +##晤 +##晦 +##晨 +##晩 +##普 +##景 +##晰 +##晴 +##晶 +##晷 +##智 +##晾 +##暂 +##暄 +##暇 +##暈 +##暉 +##暌 +##暐 +##暑 +##暖 +##暗 +##暝 +##暢 +##暧 +##暨 +##暫 +##暮 +##暱 +##暴 +##暸 +##暹 +##曄 +##曆 +##曇 +##曉 +##曖 +##曙 +##曜 +##曝 +##曠 +##曦 +##曬 +##曰 +##曲 +##曳 +##更 +##書 +##曹 +##曼 +##曾 +##替 +##最 +##會 +##月 +##有 +##朋 +##服 +##朐 +##朔 +##朕 +##朗 +##望 +##朝 +##期 +##朦 +##朧 +##木 +##未 +##末 +##本 +##札 +##朮 +##术 +##朱 +##朴 +##朵 +##机 +##朽 +##杀 +##杂 +##权 +##杆 +##杈 +##杉 +##李 +##杏 +##材 +##村 +##杓 +##杖 +##杜 +##杞 +##束 +##杠 +##条 +##来 +##杨 +##杭 +##杯 +##杰 +##東 +##杳 +##杵 +##杷 +##杼 +##松 +##板 +##极 +##构 +##枇 +##枉 +##枋 +##析 +##枕 +##林 +##枚 +##果 +##枝 +##枢 +##枣 +##枪 +##枫 +##枭 +##枯 +##枰 +##枱 +##枳 +##架 +##枷 +##枸 +##柄 +##柏 +##某 +##柑 +##柒 +##染 +##柔 +##柘 +##柚 +##柜 +##柞 +##柠 +##柢 +##查 +##柩 +##柬 +##柯 +##柱 +##柳 +##柴 +##柵 +##査 +##柿 +##栀 +##栃 +##栄 +##栅 +##标 +##栈 +##栉 +##栋 +##栎 +##栏 +##树 +##栓 +##栖 +##栗 +##校 +##栩 +##株 +##样 +##核 +##根 +##格 +##栽 +##栾 +##桀 +##桁 +##桂 +##桃 +##桅 +##框 +##案 +##桉 +##桌 +##桎 +##桐 +##桑 +##桓 +##桔 +##桜 +##桠 +##桡 +##桢 +##档 +##桥 +##桦 +##桧 +##桨 +##桩 +##桶 +##桿 +##梁 +##梅 +##梆 +##梏 +##梓 +##梗 +##條 +##梟 +##梢 +##梦 +##梧 +##梨 +##梭 +##梯 +##械 +##梳 +##梵 +##梶 +##检 +##棂 +##棄 +##棉 +##棋 +##棍 +##棒 +##棕 +##棗 +##棘 +##棚 +##棟 +##棠 +##棣 +##棧 +##森 +##棱 +##棲 +##棵 +##棹 +##棺 +##椁 +##椅 +##椋 +##植 +##椎 +##椒 +##検 +##椪 +##椭 +##椰 +##椹 +##椽 +##椿 +##楂 +##楊 +##楓 +##楔 +##楚 +##楝 +##楞 +##楠 +##楣 +##楨 +##楫 +##業 +##楮 +##極 +##楷 +##楸 +##楹 +##楼 +##楽 +##概 +##榄 +##榆 +##榈 +##榉 +##榔 +##榕 +##榖 +##榛 +##榜 +##榨 +##榫 +##榭 +##榮 +##榱 +##榴 +##榷 +##榻 +##槁 +##槃 +##構 +##槌 +##槍 +##槎 +##槐 +##槓 +##様 +##槛 +##槟 +##槤 +##槭 +##槲 +##槳 +##槻 +##槽 +##槿 +##樁 +##樂 +##樊 +##樑 +##樓 +##標 +##樞 +##樟 +##模 +##樣 +##権 +##横 +##樫 +##樯 +##樱 +##樵 +##樸 +##樹 +##樺 +##樽 +##樾 +##橄 +##橇 +##橋 +##橐 +##橘 +##橙 +##機 +##橡 +##橢 +##橫 +##橱 +##橹 +##橼 +##檀 +##檄 +##檎 +##檐 +##檔 +##檗 +##檜 +##檢 +##檬 +##檯 +##檳 +##檸 +##檻 +##櫃 +##櫚 +##櫛 +##櫥 +##櫸 +##櫻 +##欄 +##權 +##欒 +##欖 +##欠 +##次 +##欢 +##欣 +##欧 +##欲 +##欸 +##欺 +##欽 +##款 +##歆 +##歇 +##歉 +##歌 +##歎 +##歐 +##歓 +##歙 +##歛 +##歡 +##止 +##正 +##此 +##步 +##武 +##歧 +##歩 +##歪 +##歯 +##歲 +##歳 +##歴 +##歷 +##歸 +##歹 +##死 +##歼 +##殁 +##殃 +##殆 +##殇 +##殉 +##殊 +##残 +##殒 +##殓 +##殖 +##殘 +##殞 +##殡 +##殤 +##殭 +##殯 +##殲 +##殴 +##段 +##殷 +##殺 +##殼 +##殿 +##毀 +##毁 +##毂 +##毅 +##毆 +##毋 +##母 +##毎 +##每 +##毒 +##毓 +##比 +##毕 +##毗 +##毘 +##毙 +##毛 +##毡 +##毫 +##毯 +##毽 +##氈 +##氏 +##氐 +##民 +##氓 +##气 +##氖 +##気 +##氙 +##氛 +##氟 +##氡 +##氢 +##氣 +##氤 +##氦 +##氧 +##氨 +##氪 +##氫 +##氮 +##氯 +##氰 +##氲 +##水 +##氷 +##永 +##氹 +##氾 +##汀 +##汁 +##求 +##汆 +##汇 +##汉 +##汎 +##汐 +##汕 +##汗 +##汙 +##汛 +##汝 +##汞 +##江 +##池 +##污 +##汤 +##汨 +##汩 +##汪 +##汰 +##汲 +##汴 +##汶 +##汹 +##決 +##汽 +##汾 +##沁 +##沂 +##沃 +##沅 +##沈 +##沉 +##沌 +##沏 +##沐 +##沒 +##沓 +##沖 +##沙 +##沛 +##沟 +##没 +##沢 +##沣 +##沥 +##沦 +##沧 +##沪 +##沫 +##沭 +##沮 +##沱 +##河 +##沸 +##油 +##治 +##沼 +##沽 +##沾 +##沿 +##況 +##泄 +##泉 +##泊 +##泌 +##泓 +##法 +##泗 +##泛 +##泞 +##泠 +##泡 +##波 +##泣 +##泥 +##注 +##泪 +##泫 +##泮 +##泯 +##泰 +##泱 +##泳 +##泵 +##泷 +##泸 +##泻 +##泼 +##泽 +##泾 +##洁 +##洄 +##洋 +##洒 +##洗 +##洙 +##洛 +##洞 +##津 +##洩 +##洪 +##洮 +##洱 +##洲 +##洵 +##洶 +##洸 +##洹 +##活 +##洼 +##洽 +##派 +##流 +##浃 +##浄 +##浅 +##浆 +##浇 +##浊 +##测 +##济 +##浏 +##浑 +##浒 +##浓 +##浔 +##浙 +##浚 +##浜 +##浣 +##浦 +##浩 +##浪 +##浬 +##浮 +##浯 +##浴 +##海 +##浸 +##涂 +##涅 +##涇 +##消 +##涉 +##涌 +##涎 +##涓 +##涔 +##涕 +##涙 +##涛 +##涝 +##涞 +##涟 +##涠 +##涡 +##涣 +##涤 +##润 +##涧 +##涨 +##涩 +##涪 +##涮 +##涯 +##液 +##涵 +##涸 +##涼 +##涿 +##淀 +##淄 +##淅 +##淆 +##淇 +##淋 +##淌 +##淑 +##淒 +##淖 +##淘 +##淙 +##淚 +##淞 +##淡 +##淤 +##淦 +##淨 +##淩 +##淪 +##淫 +##淬 +##淮 +##深 +##淳 +##淵 +##混 +##淹 +##淺 +##添 +##淼 +##清 +##済 +##渉 +##渊 +##渋 +##渍 +##渎 +##渐 +##渔 +##渗 +##渙 +##渚 +##減 +##渝 +##渠 +##渡 +##渣 +##渤 +##渥 +##渦 +##温 +##測 +##渭 +##港 +##渲 +##渴 +##游 +##渺 +##渾 +##湃 +##湄 +##湊 +##湍 +##湖 +##湘 +##湛 +##湟 +##湧 +##湫 +##湮 +##湯 +##湳 +##湾 +##湿 +##満 +##溃 +##溅 +##溉 +##溏 +##源 +##準 +##溜 +##溝 +##溟 +##溢 +##溥 +##溧 +##溪 +##溫 +##溯 +##溱 +##溴 +##溶 +##溺 +##溼 +##滁 +##滂 +##滄 +##滅 +##滇 +##滋 +##滌 +##滑 +##滓 +##滔 +##滕 +##滙 +##滚 +##滝 +##滞 +##滟 +##满 +##滢 +##滤 +##滥 +##滦 +##滨 +##滩 +##滬 +##滯 +##滲 +##滴 +##滷 +##滸 +##滾 +##滿 +##漁 +##漂 +##漆 +##漉 +##漏 +##漓 +##演 +##漕 +##漠 +##漢 +##漣 +##漩 +##漪 +##漫 +##漬 +##漯 +##漱 +##漲 +##漳 +##漸 +##漾 +##漿 +##潆 +##潇 +##潋 +##潍 +##潑 +##潔 +##潘 +##潛 +##潜 +##潞 +##潟 +##潢 +##潤 +##潦 +##潧 +##潭 +##潮 +##潰 +##潴 +##潸 +##潺 +##潼 +##澀 +##澄 +##澆 +##澈 +##澍 +##澎 +##澗 +##澜 +##澡 +##澤 +##澧 +##澱 +##澳 +##澹 +##激 +##濁 +##濂 +##濃 +##濑 +##濒 +##濕 +##濘 +##濛 +##濟 +##濠 +##濡 +##濤 +##濫 +##濬 +##濮 +##濯 +##濱 +##濺 +##濾 +##瀅 +##瀆 +##瀉 +##瀋 +##瀏 +##瀑 +##瀕 +##瀘 +##瀚 +##瀛 +##瀝 +##瀞 +##瀟 +##瀧 +##瀨 +##瀬 +##瀰 +##瀾 +##灌 +##灏 +##灑 +##灘 +##灝 +##灞 +##灣 +##火 +##灬 +##灭 +##灯 +##灰 +##灵 +##灶 +##灸 +##灼 +##災 +##灾 +##灿 +##炀 +##炁 +##炅 +##炉 +##炊 +##炎 +##炒 +##炔 +##炕 +##炖 +##炙 +##炜 +##炫 +##炬 +##炭 +##炮 +##炯 +##炳 +##炷 +##炸 +##点 +##為 +##炼 +##炽 +##烁 +##烂 +##烃 +##烈 +##烊 +##烏 +##烘 +##烙 +##烛 +##烟 +##烤 +##烦 +##烧 +##烨 +##烩 +##烫 +##烬 +##热 +##烯 +##烷 +##烹 +##烽 +##焉 +##焊 +##焕 +##焖 +##焗 +##焘 +##焙 +##焚 +##焜 +##無 +##焦 +##焯 +##焰 +##焱 +##然 +##焼 +##煅 +##煉 +##煊 +##煌 +##煎 +##煒 +##煖 +##煙 +##煜 +##煞 +##煤 +##煥 +##煦 +##照 +##煨 +##煩 +##煮 +##煲 +##煸 +##煽 +##熄 +##熊 +##熏 +##熒 +##熔 +##熙 +##熟 +##熠 +##熨 +##熬 +##熱 +##熵 +##熹 +##熾 +##燁 +##燃 +##燄 +##燈 +##燉 +##燊 +##燎 +##燒 +##燔 +##燕 +##燙 +##燜 +##營 +##燥 +##燦 +##燧 +##燭 +##燮 +##燴 +##燻 +##燼 +##燿 +##爆 +##爍 +##爐 +##爛 +##爪 +##爬 +##爭 +##爰 +##爱 +##爲 +##爵 +##父 +##爷 +##爸 +##爹 +##爺 +##爻 +##爽 +##爾 +##牆 +##片 +##版 +##牌 +##牍 +##牒 +##牙 +##牛 +##牝 +##牟 +##牠 +##牡 +##牢 +##牦 +##牧 +##物 +##牯 +##牲 +##牴 +##牵 +##特 +##牺 +##牽 +##犀 +##犁 +##犄 +##犊 +##犍 +##犒 +##犢 +##犧 +##犬 +##犯 +##状 +##犷 +##犸 +##犹 +##狀 +##狂 +##狄 +##狈 +##狎 +##狐 +##狒 +##狗 +##狙 +##狞 +##狠 +##狡 +##狩 +##独 +##狭 +##狮 +##狰 +##狱 +##狸 +##狹 +##狼 +##狽 +##猎 +##猕 +##猖 +##猗 +##猙 +##猛 +##猜 +##猝 +##猥 +##猩 +##猪 +##猫 +##猬 +##献 +##猴 +##猶 +##猷 +##猾 +##猿 +##獄 +##獅 +##獎 +##獐 +##獒 +##獗 +##獠 +##獣 +##獨 +##獭 +##獰 +##獲 +##獵 +##獷 +##獸 +##獺 +##獻 +##獼 +##獾 +##玄 +##率 +##玉 +##王 +##玑 +##玖 +##玛 +##玟 +##玠 +##玥 +##玩 +##玫 +##玮 +##环 +##现 +##玲 +##玳 +##玷 +##玺 +##玻 +##珀 +##珂 +##珅 +##珈 +##珉 +##珊 +##珍 +##珏 +##珐 +##珑 +##珙 +##珞 +##珠 +##珣 +##珥 +##珩 +##珪 +##班 +##珮 +##珲 +##珺 +##現 +##球 +##琅 +##理 +##琇 +##琉 +##琊 +##琍 +##琏 +##琐 +##琛 +##琢 +##琥 +##琦 +##琨 +##琪 +##琬 +##琮 +##琰 +##琲 +##琳 +##琴 +##琵 +##琶 +##琺 +##琼 +##瑀 +##瑁 +##瑄 +##瑋 +##瑕 +##瑗 +##瑙 +##瑚 +##瑛 +##瑜 +##瑞 +##瑟 +##瑠 +##瑣 +##瑤 +##瑩 +##瑪 +##瑯 +##瑰 +##瑶 +##瑾 +##璀 +##璁 +##璃 +##璇 +##璉 +##璋 +##璎 +##璐 +##璜 +##璞 +##璟 +##璧 +##璨 +##環 +##璽 +##璿 +##瓊 +##瓏 +##瓒 +##瓜 +##瓢 +##瓣 +##瓤 +##瓦 +##瓮 +##瓯 +##瓴 +##瓶 +##瓷 +##甄 +##甌 +##甕 +##甘 +##甙 +##甚 +##甜 +##生 +##產 +##産 +##甥 +##甦 +##用 +##甩 +##甫 +##甬 +##甭 +##甯 +##田 +##由 +##甲 +##申 +##电 +##男 +##甸 +##町 +##画 +##甾 +##畀 +##畅 +##界 +##畏 +##畑 +##畔 +##留 +##畜 +##畝 +##畢 +##略 +##畦 +##番 +##畫 +##異 +##畲 +##畳 +##畴 +##當 +##畸 +##畹 +##畿 +##疆 +##疇 +##疊 +##疏 +##疑 +##疔 +##疖 +##疗 +##疙 +##疚 +##疝 +##疟 +##疡 +##疣 +##疤 +##疥 +##疫 +##疮 +##疯 +##疱 +##疲 +##疳 +##疵 +##疸 +##疹 +##疼 +##疽 +##疾 +##痂 +##病 +##症 +##痈 +##痉 +##痊 +##痍 +##痒 +##痔 +##痕 +##痘 +##痙 +##痛 +##痞 +##痠 +##痢 +##痣 +##痤 +##痧 +##痨 +##痪 +##痫 +##痰 +##痱 +##痴 +##痹 +##痺 +##痼 +##痿 +##瘀 +##瘁 +##瘋 +##瘍 +##瘓 +##瘘 +##瘙 +##瘟 +##瘠 +##瘡 +##瘢 +##瘤 +##瘦 +##瘧 +##瘩 +##瘪 +##瘫 +##瘴 +##瘸 +##瘾 +##療 +##癇 +##癌 +##癒 +##癖 +##癜 +##癞 +##癡 +##癢 +##癣 +##癥 +##癫 +##癬 +##癮 +##癱 +##癲 +##癸 +##発 +##登 +##發 +##白 +##百 +##皂 +##的 +##皆 +##皇 +##皈 +##皋 +##皎 +##皑 +##皓 +##皖 +##皙 +##皚 +##皮 +##皰 +##皱 +##皴 +##皺 +##皿 +##盂 +##盃 +##盅 +##盆 +##盈 +##益 +##盎 +##盏 +##盐 +##监 +##盒 +##盔 +##盖 +##盗 +##盘 +##盛 +##盜 +##盞 +##盟 +##盡 +##監 +##盤 +##盥 +##盧 +##盪 +##目 +##盯 +##盱 +##盲 +##直 +##相 +##盹 +##盼 +##盾 +##省 +##眈 +##眉 +##看 +##県 +##眙 +##眞 +##真 +##眠 +##眦 +##眨 +##眩 +##眯 +##眶 +##眷 +##眸 +##眺 +##眼 +##眾 +##着 +##睁 +##睇 +##睏 +##睐 +##睑 +##睛 +##睜 +##睞 +##睡 +##睢 +##督 +##睥 +##睦 +##睨 +##睪 +##睫 +##睬 +##睹 +##睽 +##睾 +##睿 +##瞄 +##瞅 +##瞇 +##瞋 +##瞌 +##瞎 +##瞑 +##瞒 +##瞓 +##瞞 +##瞟 +##瞠 +##瞥 +##瞧 +##瞩 +##瞪 +##瞬 +##瞭 +##瞰 +##瞳 +##瞻 +##瞼 +##瞿 +##矇 +##矍 +##矗 +##矚 +##矛 +##矜 +##矢 +##矣 +##知 +##矩 +##矫 +##短 +##矮 +##矯 +##石 +##矶 +##矽 +##矾 +##矿 +##码 +##砂 +##砌 +##砍 +##砒 +##研 +##砖 +##砗 +##砚 +##砝 +##砣 +##砥 +##砧 +##砭 +##砰 +##砲 +##破 +##砷 +##砸 +##砺 +##砼 +##砾 +##础 +##硅 +##硐 +##硒 +##硕 +##硝 +##硫 +##硬 +##确 +##硯 +##硼 +##碁 +##碇 +##碉 +##碌 +##碍 +##碎 +##碑 +##碓 +##碗 +##碘 +##碚 +##碛 +##碟 +##碣 +##碧 +##碩 +##碰 +##碱 +##碳 +##碴 +##確 +##碼 +##碾 +##磁 +##磅 +##磊 +##磋 +##磐 +##磕 +##磚 +##磡 +##磨 +##磬 +##磯 +##磲 +##磷 +##磺 +##礁 +##礎 +##礙 +##礡 +##礦 +##礪 +##礫 +##礴 +##示 +##礼 +##社 +##祀 +##祁 +##祂 +##祇 +##祈 +##祉 +##祎 +##祐 +##祕 +##祖 +##祗 +##祚 +##祛 +##祜 +##祝 +##神 +##祟 +##祠 +##祢 +##祥 +##票 +##祭 +##祯 +##祷 +##祸 +##祺 +##祿 +##禀 +##禁 +##禄 +##禅 +##禍 +##禎 +##福 +##禛 +##禦 +##禧 +##禪 +##禮 +##禱 +##禹 +##禺 +##离 +##禽 +##禾 +##禿 +##秀 +##私 +##秃 +##秆 +##秉 +##秋 +##种 +##科 +##秒 +##秘 +##租 +##秣 +##秤 +##秦 +##秧 +##秩 +##秭 +##积 +##称 +##秸 +##移 +##秽 +##稀 +##稅 +##程 +##稍 +##税 +##稔 +##稗 +##稚 +##稜 +##稞 +##稟 +##稠 +##稣 +##種 +##稱 +##稲 +##稳 +##稷 +##稹 +##稻 +##稼 +##稽 +##稿 +##穀 +##穂 +##穆 +##穌 +##積 +##穎 +##穗 +##穢 +##穩 +##穫 +##穴 +##究 +##穷 +##穹 +##空 +##穿 +##突 +##窃 +##窄 +##窈 +##窍 +##窑 +##窒 +##窓 +##窕 +##窖 +##窗 +##窘 +##窜 +##窝 +##窟 +##窠 +##窥 +##窦 +##窨 +##窩 +##窪 +##窮 +##窯 +##窺 +##窿 +##竄 +##竅 +##竇 +##竊 +##立 +##竖 +##站 +##竜 +##竞 +##竟 +##章 +##竣 +##童 +##竭 +##端 +##競 +##竹 +##竺 +##竽 +##竿 +##笃 +##笆 +##笈 +##笋 +##笏 +##笑 +##笔 +##笙 +##笛 +##笞 +##笠 +##符 +##笨 +##第 +##笹 +##笺 +##笼 +##筆 +##等 +##筊 +##筋 +##筍 +##筏 +##筐 +##筑 +##筒 +##答 +##策 +##筛 +##筝 +##筠 +##筱 +##筲 +##筵 +##筷 +##筹 +##签 +##简 +##箇 +##箋 +##箍 +##箏 +##箐 +##箔 +##箕 +##算 +##箝 +##管 +##箩 +##箫 +##箭 +##箱 +##箴 +##箸 +##節 +##篁 +##範 +##篆 +##篇 +##築 +##篑 +##篓 +##篙 +##篝 +##篠 +##篡 +##篤 +##篩 +##篪 +##篮 +##篱 +##篷 +##簇 +##簌 +##簍 +##簡 +##簦 +##簧 +##簪 +##簫 +##簷 +##簸 +##簽 +##簾 +##簿 +##籁 +##籃 +##籌 +##籍 +##籐 +##籟 +##籠 +##籤 +##籬 +##籮 +##籲 +##米 +##类 +##籼 +##籽 +##粄 +##粉 +##粑 +##粒 +##粕 +##粗 +##粘 +##粟 +##粤 +##粥 +##粧 +##粪 +##粮 +##粱 +##粲 +##粳 +##粵 +##粹 +##粼 +##粽 +##精 +##粿 +##糅 +##糊 +##糍 +##糕 +##糖 +##糗 +##糙 +##糜 +##糞 +##糟 +##糠 +##糧 +##糬 +##糯 +##糰 +##糸 +##系 +##糾 +##紀 +##紂 +##約 +##紅 +##紉 +##紊 +##紋 +##納 +##紐 +##紓 +##純 +##紗 +##紘 +##紙 +##級 +##紛 +##紜 +##素 +##紡 +##索 +##紧 +##紫 +##紮 +##累 +##細 +##紳 +##紹 +##紺 +##終 +##絃 +##組 +##絆 +##経 +##結 +##絕 +##絞 +##絡 +##絢 +##給 +##絨 +##絮 +##統 +##絲 +##絳 +##絵 +##絶 +##絹 +##綁 +##綏 +##綑 +##經 +##継 +##続 +##綜 +##綠 +##綢 +##綦 +##綫 +##綬 +##維 +##綱 +##網 +##綴 +##綵 +##綸 +##綺 +##綻 +##綽 +##綾 +##綿 +##緊 +##緋 +##総 +##緑 +##緒 +##緘 +##線 +##緝 +##緞 +##締 +##緣 +##編 +##緩 +##緬 +##緯 +##練 +##緹 +##緻 +##縁 +##縄 +##縈 +##縛 +##縝 +##縣 +##縫 +##縮 +##縱 +##縴 +##縷 +##總 +##績 +##繁 +##繃 +##繆 +##繇 +##繋 +##織 +##繕 +##繚 +##繞 +##繡 +##繩 +##繪 +##繫 +##繭 +##繳 +##繹 +##繼 +##繽 +##纂 +##續 +##纍 +##纏 +##纓 +##纔 +##纖 +##纜 +##纠 +##红 +##纣 +##纤 +##约 +##级 +##纨 +##纪 +##纫 +##纬 +##纭 +##纯 +##纰 +##纱 +##纲 +##纳 +##纵 +##纶 +##纷 +##纸 +##纹 +##纺 +##纽 +##纾 +##线 +##绀 +##练 +##组 +##绅 +##细 +##织 +##终 +##绊 +##绍 +##绎 +##经 +##绑 +##绒 +##结 +##绔 +##绕 +##绘 +##给 +##绚 +##绛 +##络 +##绝 +##绞 +##统 +##绡 +##绢 +##绣 +##绥 +##绦 +##继 +##绩 +##绪 +##绫 +##续 +##绮 +##绯 +##绰 +##绳 +##维 +##绵 +##绶 +##绷 +##绸 +##绻 +##综 +##绽 +##绾 +##绿 +##缀 +##缄 +##缅 +##缆 +##缇 +##缈 +##缉 +##缎 +##缓 +##缔 +##缕 +##编 +##缘 +##缙 +##缚 +##缜 +##缝 +##缠 +##缢 +##缤 +##缥 +##缨 +##缩 +##缪 +##缭 +##缮 +##缰 +##缱 +##缴 +##缸 +##缺 +##缽 +##罂 +##罄 +##罌 +##罐 +##网 +##罔 +##罕 +##罗 +##罚 +##罡 +##罢 +##罩 +##罪 +##置 +##罰 +##署 +##罵 +##罷 +##罹 +##羁 +##羅 +##羈 +##羊 +##羌 +##美 +##羔 +##羚 +##羞 +##羟 +##羡 +##羣 +##群 +##羥 +##羧 +##羨 +##義 +##羯 +##羲 +##羸 +##羹 +##羽 +##羿 +##翁 +##翅 +##翊 +##翌 +##翎 +##習 +##翔 +##翘 +##翟 +##翠 +##翡 +##翦 +##翩 +##翰 +##翱 +##翳 +##翹 +##翻 +##翼 +##耀 +##老 +##考 +##耄 +##者 +##耆 +##耋 +##而 +##耍 +##耐 +##耒 +##耕 +##耗 +##耘 +##耙 +##耦 +##耨 +##耳 +##耶 +##耷 +##耸 +##耻 +##耽 +##耿 +##聂 +##聆 +##聊 +##聋 +##职 +##聒 +##联 +##聖 +##聘 +##聚 +##聞 +##聪 +##聯 +##聰 +##聲 +##聳 +##聴 +##聶 +##職 +##聽 +##聾 +##聿 +##肃 +##肄 +##肅 +##肆 +##肇 +##肉 +##肋 +##肌 +##肏 +##肓 +##肖 +##肘 +##肚 +##肛 +##肝 +##肠 +##股 +##肢 +##肤 +##肥 +##肩 +##肪 +##肮 +##肯 +##肱 +##育 +##肴 +##肺 +##肽 +##肾 +##肿 +##胀 +##胁 +##胃 +##胄 +##胆 +##背 +##胍 +##胎 +##胖 +##胚 +##胛 +##胜 +##胝 +##胞 +##胡 +##胤 +##胥 +##胧 +##胫 +##胭 +##胯 +##胰 +##胱 +##胳 +##胴 +##胶 +##胸 +##胺 +##能 +##脂 +##脅 +##脆 +##脇 +##脈 +##脉 +##脊 +##脍 +##脏 +##脐 +##脑 +##脓 +##脖 +##脘 +##脚 +##脛 +##脣 +##脩 +##脫 +##脯 +##脱 +##脲 +##脳 +##脸 +##脹 +##脾 +##腆 +##腈 +##腊 +##腋 +##腌 +##腎 +##腐 +##腑 +##腓 +##腔 +##腕 +##腥 +##腦 +##腩 +##腫 +##腭 +##腮 +##腰 +##腱 +##腳 +##腴 +##腸 +##腹 +##腺 +##腻 +##腼 +##腾 +##腿 +##膀 +##膈 +##膊 +##膏 +##膑 +##膘 +##膚 +##膛 +##膜 +##膝 +##膠 +##膦 +##膨 +##膩 +##膳 +##膺 +##膻 +##膽 +##膾 +##膿 +##臀 +##臂 +##臃 +##臆 +##臉 +##臊 +##臍 +##臓 +##臘 +##臟 +##臣 +##臥 +##臧 +##臨 +##自 +##臬 +##臭 +##至 +##致 +##臺 +##臻 +##臼 +##臾 +##舀 +##舂 +##舅 +##舆 +##與 +##興 +##舉 +##舊 +##舌 +##舍 +##舎 +##舐 +##舒 +##舔 +##舖 +##舗 +##舛 +##舜 +##舞 +##舟 +##航 +##舫 +##般 +##舰 +##舱 +##舵 +##舶 +##舷 +##舸 +##船 +##舺 +##舾 +##艇 +##艋 +##艘 +##艙 +##艦 +##艮 +##良 +##艰 +##艱 +##色 +##艳 +##艷 +##艹 +##艺 +##艾 +##节 +##芃 +##芈 +##芊 +##芋 +##芍 +##芎 +##芒 +##芙 +##芜 +##芝 +##芡 +##芥 +##芦 +##芩 +##芪 +##芫 +##芬 +##芭 +##芮 +##芯 +##花 +##芳 +##芷 +##芸 +##芹 +##芻 +##芽 +##芾 +##苁 +##苄 +##苇 +##苋 +##苍 +##苏 +##苑 +##苒 +##苓 +##苔 +##苕 +##苗 +##苛 +##苜 +##苞 +##苟 +##苡 +##苣 +##若 +##苦 +##苫 +##苯 +##英 +##苷 +##苹 +##苻 +##茁 +##茂 +##范 +##茄 +##茅 +##茉 +##茎 +##茏 +##茗 +##茜 +##茧 +##茨 +##茫 +##茬 +##茭 +##茯 +##茱 +##茲 +##茴 +##茵 +##茶 +##茸 +##茹 +##茼 +##荀 +##荃 +##荆 +##草 +##荊 +##荏 +##荐 +##荒 +##荔 +##荖 +##荘 +##荚 +##荞 +##荟 +##荠 +##荡 +##荣 +##荤 +##荥 +##荧 +##荨 +##荪 +##荫 +##药 +##荳 +##荷 +##荸 +##荻 +##荼 +##荽 +##莅 +##莆 +##莉 +##莊 +##莎 +##莒 +##莓 +##莖 +##莘 +##莞 +##莠 +##莢 +##莧 +##莪 +##莫 +##莱 +##莲 +##莴 +##获 +##莹 +##莺 +##莽 +##莿 +##菀 +##菁 +##菅 +##菇 +##菈 +##菊 +##菌 +##菏 +##菓 +##菖 +##菘 +##菜 +##菟 +##菠 +##菡 +##菩 +##華 +##菱 +##菲 +##菸 +##菽 +##萁 +##萃 +##萄 +##萊 +##萋 +##萌 +##萍 +##萎 +##萘 +##萝 +##萤 +##营 +##萦 +##萧 +##萨 +##萩 +##萬 +##萱 +##萵 +##萸 +##萼 +##落 +##葆 +##葉 +##著 +##葚 +##葛 +##葡 +##董 +##葦 +##葩 +##葫 +##葬 +##葭 +##葯 +##葱 +##葳 +##葵 +##葷 +##葺 +##蒂 +##蒋 +##蒐 +##蒔 +##蒙 +##蒜 +##蒞 +##蒟 +##蒡 +##蒨 +##蒲 +##蒸 +##蒹 +##蒻 +##蒼 +##蒿 +##蓁 +##蓄 +##蓆 +##蓉 +##蓋 +##蓑 +##蓓 +##蓖 +##蓝 +##蓟 +##蓦 +##蓬 +##蓮 +##蓼 +##蓿 +##蔑 +##蔓 +##蔔 +##蔗 +##蔘 +##蔚 +##蔡 +##蔣 +##蔥 +##蔫 +##蔬 +##蔭 +##蔵 +##蔷 +##蔺 +##蔻 +##蔼 +##蔽 +##蕁 +##蕃 +##蕈 +##蕉 +##蕊 +##蕎 +##蕙 +##蕤 +##蕨 +##蕩 +##蕪 +##蕭 +##蕲 +##蕴 +##蕻 +##蕾 +##薄 +##薅 +##薇 +##薈 +##薊 +##薏 +##薑 +##薔 +##薙 +##薛 +##薦 +##薨 +##薩 +##薪 +##薬 +##薯 +##薰 +##薹 +##藉 +##藍 +##藏 +##藐 +##藓 +##藕 +##藜 +##藝 +##藤 +##藥 +##藩 +##藹 +##藻 +##藿 +##蘆 +##蘇 +##蘊 +##蘋 +##蘑 +##蘚 +##蘭 +##蘸 +##蘼 +##蘿 +##虎 +##虏 +##虐 +##虑 +##虔 +##處 +##虚 +##虛 +##虜 +##虞 +##號 +##虢 +##虧 +##虫 +##虬 +##虱 +##虹 +##虻 +##虽 +##虾 +##蚀 +##蚁 +##蚂 +##蚊 +##蚌 +##蚓 +##蚕 +##蚜 +##蚝 +##蚣 +##蚤 +##蚩 +##蚪 +##蚯 +##蚱 +##蚵 +##蛀 +##蛆 +##蛇 +##蛊 +##蛋 +##蛎 +##蛐 +##蛔 +##蛙 +##蛛 +##蛟 +##蛤 +##蛭 +##蛮 +##蛰 +##蛳 +##蛹 +##蛻 +##蛾 +##蜀 +##蜂 +##蜃 +##蜆 +##蜇 +##蜈 +##蜊 +##蜍 +##蜒 +##蜓 +##蜕 +##蜗 +##蜘 +##蜚 +##蜜 +##蜡 +##蜢 +##蜥 +##蜱 +##蜴 +##蜷 +##蜻 +##蜿 +##蝇 +##蝈 +##蝉 +##蝌 +##蝎 +##蝕 +##蝗 +##蝙 +##蝟 +##蝠 +##蝦 +##蝨 +##蝴 +##蝶 +##蝸 +##蝼 +##螂 +##螃 +##融 +##螞 +##螢 +##螨 +##螯 +##螳 +##螺 +##蟀 +##蟄 +##蟆 +##蟋 +##蟎 +##蟑 +##蟒 +##蟠 +##蟬 +##蟲 +##蟹 +##蟻 +##蟾 +##蠅 +##蠍 +##蠔 +##蠕 +##蠛 +##蠟 +##蠡 +##蠢 +##蠣 +##蠱 +##蠶 +##蠹 +##蠻 +##血 +##衄 +##衅 +##衆 +##行 +##衍 +##術 +##衔 +##街 +##衙 +##衛 +##衝 +##衞 +##衡 +##衢 +##衣 +##补 +##表 +##衩 +##衫 +##衬 +##衮 +##衰 +##衲 +##衷 +##衹 +##衾 +##衿 +##袁 +##袂 +##袄 +##袅 +##袈 +##袋 +##袍 +##袒 +##袖 +##袜 +##袞 +##袤 +##袪 +##被 +##袭 +##袱 +##裁 +##裂 +##装 +##裆 +##裊 +##裏 +##裔 +##裕 +##裘 +##裙 +##補 +##裝 +##裟 +##裡 +##裤 +##裨 +##裱 +##裳 +##裴 +##裸 +##裹 +##製 +##裾 +##褂 +##複 +##褐 +##褒 +##褓 +##褔 +##褚 +##褥 +##褪 +##褫 +##褲 +##褶 +##褻 +##襁 +##襄 +##襟 +##襠 +##襪 +##襬 +##襯 +##襲 +##西 +##要 +##覃 +##覆 +##覇 +##見 +##規 +##覓 +##視 +##覚 +##覦 +##覧 +##親 +##覬 +##観 +##覷 +##覺 +##覽 +##觀 +##见 +##观 +##规 +##觅 +##视 +##览 +##觉 +##觊 +##觎 +##觐 +##觑 +##角 +##觞 +##解 +##觥 +##触 +##觸 +##言 +##訂 +##計 +##訊 +##討 +##訓 +##訕 +##訖 +##託 +##記 +##訛 +##訝 +##訟 +##訣 +##訥 +##訪 +##設 +##許 +##訳 +##訴 +##訶 +##診 +##註 +##証 +##詆 +##詐 +##詔 +##評 +##詛 +##詞 +##詠 +##詡 +##詢 +##詣 +##試 +##詩 +##詫 +##詬 +##詭 +##詮 +##詰 +##話 +##該 +##詳 +##詹 +##詼 +##誅 +##誇 +##誉 +##誌 +##認 +##誓 +##誕 +##誘 +##語 +##誠 +##誡 +##誣 +##誤 +##誥 +##誦 +##誨 +##說 +##説 +##読 +##誰 +##課 +##誹 +##誼 +##調 +##諄 +##談 +##請 +##諏 +##諒 +##論 +##諗 +##諜 +##諡 +##諦 +##諧 +##諫 +##諭 +##諮 +##諱 +##諳 +##諷 +##諸 +##諺 +##諾 +##謀 +##謁 +##謂 +##謄 +##謊 +##謎 +##謐 +##謔 +##謗 +##謙 +##講 +##謝 +##謠 +##謨 +##謬 +##謹 +##謾 +##譁 +##證 +##譎 +##譏 +##識 +##譙 +##譚 +##譜 +##警 +##譬 +##譯 +##議 +##譲 +##譴 +##護 +##譽 +##讀 +##變 +##讓 +##讚 +##讞 +##计 +##订 +##认 +##讥 +##讧 +##讨 +##让 +##讪 +##讫 +##训 +##议 +##讯 +##记 +##讲 +##讳 +##讴 +##讶 +##讷 +##许 +##讹 +##论 +##讼 +##讽 +##设 +##访 +##诀 +##证 +##诃 +##评 +##诅 +##识 +##诈 +##诉 +##诊 +##诋 +##词 +##诏 +##译 +##试 +##诗 +##诘 +##诙 +##诚 +##诛 +##话 +##诞 +##诟 +##诠 +##诡 +##询 +##诣 +##诤 +##该 +##详 +##诧 +##诩 +##诫 +##诬 +##语 +##误 +##诰 +##诱 +##诲 +##说 +##诵 +##诶 +##请 +##诸 +##诺 +##读 +##诽 +##课 +##诿 +##谀 +##谁 +##调 +##谄 +##谅 +##谆 +##谈 +##谊 +##谋 +##谌 +##谍 +##谎 +##谏 +##谐 +##谑 +##谒 +##谓 +##谔 +##谕 +##谗 +##谘 +##谙 +##谚 +##谛 +##谜 +##谟 +##谢 +##谣 +##谤 +##谥 +##谦 +##谧 +##谨 +##谩 +##谪 +##谬 +##谭 +##谯 +##谱 +##谲 +##谴 +##谶 +##谷 +##豁 +##豆 +##豇 +##豈 +##豉 +##豊 +##豌 +##豎 +##豐 +##豔 +##豚 +##象 +##豢 +##豪 +##豫 +##豬 +##豹 +##豺 +##貂 +##貅 +##貌 +##貓 +##貔 +##貘 +##貝 +##貞 +##負 +##財 +##貢 +##貧 +##貨 +##販 +##貪 +##貫 +##責 +##貯 +##貰 +##貳 +##貴 +##貶 +##買 +##貸 +##費 +##貼 +##貽 +##貿 +##賀 +##賁 +##賂 +##賃 +##賄 +##資 +##賈 +##賊 +##賑 +##賓 +##賜 +##賞 +##賠 +##賡 +##賢 +##賣 +##賤 +##賦 +##質 +##賬 +##賭 +##賴 +##賺 +##購 +##賽 +##贅 +##贈 +##贊 +##贍 +##贏 +##贓 +##贖 +##贛 +##贝 +##贞 +##负 +##贡 +##财 +##责 +##贤 +##败 +##账 +##货 +##质 +##贩 +##贪 +##贫 +##贬 +##购 +##贮 +##贯 +##贰 +##贱 +##贲 +##贴 +##贵 +##贷 +##贸 +##费 +##贺 +##贻 +##贼 +##贾 +##贿 +##赁 +##赂 +##赃 +##资 +##赅 +##赈 +##赊 +##赋 +##赌 +##赎 +##赏 +##赐 +##赓 +##赔 +##赖 +##赘 +##赚 +##赛 +##赝 +##赞 +##赠 +##赡 +##赢 +##赣 +##赤 +##赦 +##赧 +##赫 +##赭 +##走 +##赳 +##赴 +##赵 +##赶 +##起 +##趁 +##超 +##越 +##趋 +##趕 +##趙 +##趟 +##趣 +##趨 +##足 +##趴 +##趵 +##趸 +##趺 +##趾 +##跃 +##跄 +##跆 +##跋 +##跌 +##跎 +##跑 +##跖 +##跚 +##跛 +##距 +##跟 +##跡 +##跤 +##跨 +##跩 +##跪 +##路 +##跳 +##践 +##跷 +##跹 +##跺 +##跻 +##踉 +##踊 +##踌 +##踏 +##踐 +##踝 +##踞 +##踟 +##踢 +##踩 +##踪 +##踮 +##踱 +##踴 +##踵 +##踹 +##蹂 +##蹄 +##蹇 +##蹈 +##蹉 +##蹊 +##蹋 +##蹑 +##蹒 +##蹙 +##蹟 +##蹣 +##蹤 +##蹦 +##蹩 +##蹬 +##蹭 +##蹲 +##蹴 +##蹶 +##蹺 +##蹼 +##蹿 +##躁 +##躇 +##躉 +##躊 +##躋 +##躍 +##躏 +##躪 +##身 +##躬 +##躯 +##躲 +##躺 +##軀 +##車 +##軋 +##軌 +##軍 +##軒 +##軟 +##転 +##軸 +##軼 +##軽 +##軾 +##較 +##載 +##輒 +##輓 +##輔 +##輕 +##輛 +##輝 +##輟 +##輩 +##輪 +##輯 +##輸 +##輻 +##輾 +##輿 +##轄 +##轅 +##轆 +##轉 +##轍 +##轎 +##轟 +##车 +##轧 +##轨 +##轩 +##转 +##轭 +##轮 +##软 +##轰 +##轲 +##轴 +##轶 +##轻 +##轼 +##载 +##轿 +##较 +##辄 +##辅 +##辆 +##辇 +##辈 +##辉 +##辊 +##辍 +##辐 +##辑 +##输 +##辕 +##辖 +##辗 +##辘 +##辙 +##辛 +##辜 +##辞 +##辟 +##辣 +##辦 +##辨 +##辩 +##辫 +##辭 +##辮 +##辯 +##辰 +##辱 +##農 +##边 +##辺 +##辻 +##込 +##辽 +##达 +##迁 +##迂 +##迄 +##迅 +##过 +##迈 +##迎 +##运 +##近 +##返 +##还 +##这 +##进 +##远 +##违 +##连 +##迟 +##迢 +##迤 +##迥 +##迦 +##迩 +##迪 +##迫 +##迭 +##述 +##迴 +##迷 +##迸 +##迹 +##迺 +##追 +##退 +##送 +##适 +##逃 +##逅 +##逆 +##选 +##逊 +##逍 +##透 +##逐 +##递 +##途 +##逕 +##逗 +##這 +##通 +##逛 +##逝 +##逞 +##速 +##造 +##逢 +##連 +##逮 +##週 +##進 +##逵 +##逶 +##逸 +##逻 +##逼 +##逾 +##遁 +##遂 +##遅 +##遇 +##遊 +##運 +##遍 +##過 +##遏 +##遐 +##遑 +##遒 +##道 +##達 +##違 +##遗 +##遙 +##遛 +##遜 +##遞 +##遠 +##遢 +##遣 +##遥 +##遨 +##適 +##遭 +##遮 +##遲 +##遴 +##遵 +##遶 +##遷 +##選 +##遺 +##遼 +##遽 +##避 +##邀 +##邁 +##邂 +##邃 +##還 +##邇 +##邈 +##邊 +##邋 +##邏 +##邑 +##邓 +##邕 +##邛 +##邝 +##邢 +##那 +##邦 +##邨 +##邪 +##邬 +##邮 +##邯 +##邰 +##邱 +##邳 +##邵 +##邸 +##邹 +##邺 +##邻 +##郁 +##郅 +##郊 +##郎 +##郑 +##郜 +##郝 +##郡 +##郢 +##郤 +##郦 +##郧 +##部 +##郫 +##郭 +##郴 +##郵 +##郷 +##郸 +##都 +##鄂 +##鄉 +##鄒 +##鄔 +##鄙 +##鄞 +##鄢 +##鄧 +##鄭 +##鄰 +##鄱 +##鄲 +##鄺 +##酉 +##酊 +##酋 +##酌 +##配 +##酐 +##酒 +##酗 +##酚 +##酝 +##酢 +##酣 +##酥 +##酩 +##酪 +##酬 +##酮 +##酯 +##酰 +##酱 +##酵 +##酶 +##酷 +##酸 +##酿 +##醃 +##醇 +##醉 +##醋 +##醍 +##醐 +##醒 +##醚 +##醛 +##醜 +##醞 +##醣 +##醪 +##醫 +##醬 +##醮 +##醯 +##醴 +##醺 +##釀 +##釁 +##采 +##釉 +##释 +##釋 +##里 +##重 +##野 +##量 +##釐 +##金 +##釗 +##釘 +##釜 +##針 +##釣 +##釦 +##釧 +##釵 +##鈀 +##鈉 +##鈍 +##鈎 +##鈔 +##鈕 +##鈞 +##鈣 +##鈦 +##鈪 +##鈴 +##鈺 +##鈾 +##鉀 +##鉄 +##鉅 +##鉉 +##鉑 +##鉗 +##鉚 +##鉛 +##鉤 +##鉴 +##鉻 +##銀 +##銃 +##銅 +##銑 +##銓 +##銖 +##銘 +##銜 +##銬 +##銭 +##銮 +##銳 +##銷 +##銹 +##鋁 +##鋅 +##鋒 +##鋤 +##鋪 +##鋰 +##鋸 +##鋼 +##錄 +##錐 +##錘 +##錚 +##錠 +##錢 +##錦 +##錨 +##錫 +##錮 +##錯 +##録 +##錳 +##錶 +##鍊 +##鍋 +##鍍 +##鍛 +##鍥 +##鍰 +##鍵 +##鍺 +##鍾 +##鎂 +##鎊 +##鎌 +##鎏 +##鎔 +##鎖 +##鎗 +##鎚 +##鎧 +##鎬 +##鎮 +##鎳 +##鏈 +##鏖 +##鏗 +##鏘 +##鏞 +##鏟 +##鏡 +##鏢 +##鏤 +##鏽 +##鐘 +##鐮 +##鐲 +##鐳 +##鐵 +##鐸 +##鐺 +##鑄 +##鑊 +##鑑 +##鑒 +##鑣 +##鑫 +##鑰 +##鑲 +##鑼 +##鑽 +##鑾 +##鑿 +##针 +##钉 +##钊 +##钎 +##钏 +##钒 +##钓 +##钗 +##钙 +##钛 +##钜 +##钝 +##钞 +##钟 +##钠 +##钡 +##钢 +##钣 +##钤 +##钥 +##钦 +##钧 +##钨 +##钩 +##钮 +##钯 +##钰 +##钱 +##钳 +##钴 +##钵 +##钺 +##钻 +##钼 +##钾 +##钿 +##铀 +##铁 +##铂 +##铃 +##铄 +##铅 +##铆 +##铉 +##铎 +##铐 +##铛 +##铜 +##铝 +##铠 +##铡 +##铢 +##铣 +##铤 +##铨 +##铩 +##铬 +##铭 +##铮 +##铰 +##铲 +##铵 +##银 +##铸 +##铺 +##链 +##铿 +##销 +##锁 +##锂 +##锄 +##锅 +##锆 +##锈 +##锉 +##锋 +##锌 +##锏 +##锐 +##锑 +##错 +##锚 +##锟 +##锡 +##锢 +##锣 +##锤 +##锥 +##锦 +##锭 +##键 +##锯 +##锰 +##锲 +##锵 +##锹 +##锺 +##锻 +##镀 +##镁 +##镂 +##镇 +##镉 +##镌 +##镍 +##镐 +##镑 +##镕 +##镖 +##镗 +##镛 +##镜 +##镣 +##镭 +##镯 +##镰 +##镳 +##镶 +##長 +##长 +##門 +##閃 +##閉 +##開 +##閎 +##閏 +##閑 +##閒 +##間 +##閔 +##閘 +##閡 +##関 +##閣 +##閥 +##閨 +##閩 +##閱 +##閲 +##閹 +##閻 +##閾 +##闆 +##闇 +##闊 +##闌 +##闍 +##闔 +##闕 +##闖 +##闘 +##關 +##闡 +##闢 +##门 +##闪 +##闫 +##闭 +##问 +##闯 +##闰 +##闲 +##间 +##闵 +##闷 +##闸 +##闹 +##闺 +##闻 +##闽 +##闾 +##阀 +##阁 +##阂 +##阅 +##阆 +##阇 +##阈 +##阉 +##阎 +##阐 +##阑 +##阔 +##阕 +##阖 +##阙 +##阚 +##阜 +##队 +##阡 +##阪 +##阮 +##阱 +##防 +##阳 +##阴 +##阵 +##阶 +##阻 +##阿 +##陀 +##陂 +##附 +##际 +##陆 +##陇 +##陈 +##陋 +##陌 +##降 +##限 +##陕 +##陛 +##陝 +##陞 +##陟 +##陡 +##院 +##陣 +##除 +##陨 +##险 +##陪 +##陰 +##陲 +##陳 +##陵 +##陶 +##陷 +##陸 +##険 +##陽 +##隅 +##隆 +##隈 +##隊 +##隋 +##隍 +##階 +##随 +##隐 +##隔 +##隕 +##隘 +##隙 +##際 +##障 +##隠 +##隣 +##隧 +##隨 +##險 +##隱 +##隴 +##隶 +##隸 +##隻 +##隼 +##隽 +##难 +##雀 +##雁 +##雄 +##雅 +##集 +##雇 +##雉 +##雋 +##雌 +##雍 +##雎 +##雏 +##雑 +##雒 +##雕 +##雖 +##雙 +##雛 +##雜 +##雞 +##離 +##難 +##雨 +##雪 +##雯 +##雰 +##雲 +##雳 +##零 +##雷 +##雹 +##電 +##雾 +##需 +##霁 +##霄 +##霆 +##震 +##霈 +##霉 +##霊 +##霍 +##霎 +##霏 +##霑 +##霓 +##霖 +##霜 +##霞 +##霧 +##霭 +##霰 +##露 +##霸 +##霹 +##霽 +##霾 +##靂 +##靄 +##靈 +##青 +##靓 +##靖 +##静 +##靚 +##靛 +##靜 +##非 +##靠 +##靡 +##面 +##靥 +##靦 +##革 +##靳 +##靴 +##靶 +##靼 +##鞅 +##鞋 +##鞍 +##鞏 +##鞑 +##鞘 +##鞠 +##鞣 +##鞦 +##鞭 +##韆 +##韋 +##韌 +##韓 +##韜 +##韦 +##韧 +##韩 +##韬 +##韭 +##音 +##韵 +##韶 +##韻 +##響 +##頁 +##頂 +##頃 +##項 +##順 +##須 +##頌 +##預 +##頑 +##頒 +##頓 +##頗 +##領 +##頜 +##頡 +##頤 +##頫 +##頭 +##頰 +##頷 +##頸 +##頹 +##頻 +##頼 +##顆 +##題 +##額 +##顎 +##顏 +##顔 +##願 +##顛 +##類 +##顧 +##顫 +##顯 +##顱 +##顴 +##页 +##顶 +##顷 +##项 +##顺 +##须 +##顼 +##顽 +##顾 +##顿 +##颁 +##颂 +##预 +##颅 +##领 +##颇 +##颈 +##颉 +##颊 +##颌 +##颍 +##颐 +##频 +##颓 +##颔 +##颖 +##颗 +##题 +##颚 +##颛 +##颜 +##额 +##颞 +##颠 +##颡 +##颢 +##颤 +##颦 +##颧 +##風 +##颯 +##颱 +##颳 +##颶 +##颼 +##飄 +##飆 +##风 +##飒 +##飓 +##飕 +##飘 +##飙 +##飚 +##飛 +##飞 +##食 +##飢 +##飨 +##飩 +##飪 +##飯 +##飲 +##飼 +##飽 +##飾 +##餃 +##餅 +##餉 +##養 +##餌 +##餐 +##餒 +##餓 +##餘 +##餚 +##餛 +##餞 +##餡 +##館 +##餮 +##餵 +##餾 +##饅 +##饈 +##饋 +##饌 +##饍 +##饑 +##饒 +##饕 +##饗 +##饞 +##饥 +##饨 +##饪 +##饬 +##饭 +##饮 +##饯 +##饰 +##饱 +##饲 +##饴 +##饵 +##饶 +##饷 +##饺 +##饼 +##饽 +##饿 +##馀 +##馁 +##馄 +##馅 +##馆 +##馈 +##馋 +##馍 +##馏 +##馒 +##馔 +##首 +##馗 +##香 +##馥 +##馨 +##馬 +##馭 +##馮 +##馳 +##馴 +##駁 +##駄 +##駅 +##駆 +##駐 +##駒 +##駕 +##駛 +##駝 +##駭 +##駱 +##駿 +##騁 +##騎 +##騏 +##験 +##騙 +##騨 +##騰 +##騷 +##驀 +##驅 +##驊 +##驍 +##驒 +##驕 +##驗 +##驚 +##驛 +##驟 +##驢 +##驥 +##马 +##驭 +##驮 +##驯 +##驰 +##驱 +##驳 +##驴 +##驶 +##驷 +##驸 +##驹 +##驻 +##驼 +##驾 +##驿 +##骁 +##骂 +##骄 +##骅 +##骆 +##骇 +##骈 +##骊 +##骋 +##验 +##骏 +##骐 +##骑 +##骗 +##骚 +##骛 +##骜 +##骞 +##骠 +##骡 +##骤 +##骥 +##骧 +##骨 +##骯 +##骰 +##骶 +##骷 +##骸 +##骼 +##髂 +##髅 +##髋 +##髏 +##髒 +##髓 +##體 +##髖 +##高 +##髦 +##髪 +##髮 +##髯 +##髻 +##鬃 +##鬆 +##鬍 +##鬓 +##鬚 +##鬟 +##鬢 +##鬣 +##鬥 +##鬧 +##鬱 +##鬼 +##魁 +##魂 +##魄 +##魅 +##魇 +##魍 +##魏 +##魔 +##魘 +##魚 +##魯 +##魷 +##鮑 +##鮨 +##鮪 +##鮭 +##鮮 +##鯉 +##鯊 +##鯖 +##鯛 +##鯨 +##鯰 +##鯽 +##鰍 +##鰓 +##鰭 +##鰲 +##鰻 +##鰾 +##鱈 +##鱉 +##鱔 +##鱗 +##鱷 +##鱸 +##鱼 +##鱿 +##鲁 +##鲈 +##鲍 +##鲑 +##鲛 +##鲜 +##鲟 +##鲢 +##鲤 +##鲨 +##鲫 +##鲱 +##鲲 +##鲶 +##鲷 +##鲸 +##鳃 +##鳄 +##鳅 +##鳌 +##鳍 +##鳕 +##鳖 +##鳗 +##鳝 +##鳞 +##鳥 +##鳩 +##鳳 +##鳴 +##鳶 +##鴉 +##鴕 +##鴛 +##鴦 +##鴨 +##鴻 +##鴿 +##鵑 +##鵜 +##鵝 +##鵡 +##鵬 +##鵰 +##鵲 +##鶘 +##鶩 +##鶯 +##鶴 +##鷗 +##鷲 +##鷹 +##鷺 +##鸚 +##鸞 +##鸟 +##鸠 +##鸡 +##鸢 +##鸣 +##鸥 +##鸦 +##鸨 +##鸪 +##鸭 +##鸯 +##鸳 +##鸵 +##鸽 +##鸾 +##鸿 +##鹂 +##鹃 +##鹄 +##鹅 +##鹈 +##鹉 +##鹊 +##鹌 +##鹏 +##鹑 +##鹕 +##鹘 +##鹜 +##鹞 +##鹤 +##鹦 +##鹧 +##鹫 +##鹭 +##鹰 +##鹳 +##鹵 +##鹹 +##鹼 +##鹽 +##鹿 +##麂 +##麋 +##麒 +##麓 +##麗 +##麝 +##麟 +##麥 +##麦 +##麩 +##麴 +##麵 +##麸 +##麺 +##麻 +##麼 +##麽 +##麾 +##黃 +##黄 +##黍 +##黎 +##黏 +##黑 +##黒 +##黔 +##默 +##黛 +##黜 +##黝 +##點 +##黠 +##黨 +##黯 +##黴 +##鼋 +##鼎 +##鼐 +##鼓 +##鼠 +##鼬 +##鼹 +##鼻 +##鼾 +##齁 +##齊 +##齋 +##齐 +##齒 +##齡 +##齢 +##齣 +##齦 +##齿 +##龄 +##龅 +##龈 +##龊 +##龋 +##龌 +##龍 +##龐 +##龔 +##龕 +##龙 +##龚 +##龛 +##龜 +##龟 +##︰ +##︱ +##︶ +##︿ +##﹁ +##﹂ +##﹍ +##﹏ +##﹐ +##﹑ +##﹒ +##﹔ +##﹕ +##﹖ +##﹗ +##﹙ +##﹚ +##﹝ +##﹞ +##﹡ +##﹣ +##！ +##＂ +##＃ +##＄ +##％ +##＆ +##＇ +##（ +##） +##＊ +##， +##－ +##． +##／ +##： +##； +##＜ +##？ +##＠ +##［ +##＼ +##］ +##＾ +##＿ +##｀ +##ｆ +##ｈ +##ｊ +##ｕ +##ｗ +##ｚ +##｛ +##｝ +##｡ +##｢ +##｣ +##､ +##･ +##ｯ +##ｰ +##ｲ +##ｸ +##ｼ +##ｽ +##ﾄ +##ﾉ +##ﾌ +##ﾗ +##ﾙ +##ﾝ +##ﾞ +##ﾟ +##￣ +##￥ +##👍 +##🔥 +##😂 +##😎 diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/npu_set_env.sh b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/npu_set_env.sh new file mode 100644 index 0000000..73d48a2 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/config/npu_set_env.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +#toolkit env +#export LD_LIBRARY_PATH=/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/ +#export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/te:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/topi:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/python/site-packages/hccl:/usr/local/Ascend/ascend-toolkit/latest/tfplugin/python/site-packages:$utilDir +#export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin +#export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + +#nnae env +#export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/Ascend/driver/tools/hccn_tool/ +#export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/te:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/topi:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/hccl:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:$utilDir +#export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin +#export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp + +if [ -d /usr/local/Ascend/nnae/latest ];then + export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib/:/usr/local/Ascend/nnae/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/Ascend/driver/tools/hccn_tool/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/nnae/latest/fwkacllib/python/site-packages/:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:$utilDir + export PATH=$PATH:/usr/local/Ascend/nnae/latest/fwkacllib/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp +else + export LD_LIBRARY_PATH=/usr/local/lib/:/usr/lib/:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/ascend-toolkit/latest//fwkacllib/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/tfplugin/python/site-packages:$utilDir + export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + +fi + +export NEW_GE_FE_ID=1 +export GE_AICPU_FLAG=1 +export SOC_VERSION=Ascend910 +#export DUMP_GE_GRAPH=2 +#export DUMP_GRAPH_LEVEL=3 +#export PRINT_MODEL=1 +export SLOG_PRINT_TO_STDOUT=0 +export HCCL_CONNECT_TIMEOUT=600 + + +# system env +#ulimit -c unlimited diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/scripts/run.sh b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/scripts/run.sh new file mode 100644 index 0000000..44a6a87 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/scripts/run.sh @@ -0,0 +1,67 @@ +#!/bin/bash +rank_size=$1 +yamlPath=$2 +toolsPath=$3 +if [ -f /.dockerenv ];then + CLUSTER=$4 + MPIRUN_ALL_IP="$5" + export CLUSTER=${CLUSTER} +fi + +currentDir=$(cd "$(dirname "$0")/.."; pwd) +currtime=`date +%Y%m%d%H%M%S` +mkdir -p ${currentDir%train*}/train/result/tf_bert_base/training_job_${currtime}/ +train_job_dir=${currentDir%train*}/train/result/tf_bert_base/training_job_${currtime}/ +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] see more config info in ${currentDir}/config" +echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] train result in ${train_job_dir}" + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "tensorflow_config") + +# device 列表, 若无指定 device 根据 rank_size 顺序选择 +eval device_group=\$device_group_${rank_size}p +if [ x"${device_group}" == x"" ] || [ ${rank_size} -ge 8 ];then + device_group="$(seq 0 "$(expr $rank_size - 1)")" +fi + +# get last device id in device_group, hw log in performance from the dir named first_device_id +device_group_str=`echo ${device_group} | sed 's/ //g'` +first_device_id=`echo ${device_group_str: 0:1}` + +# user env +export JOB_ID=9999001 +export RANK_TABLE_FILE=${currentDir}/config/${rank_size}p.json +export RANK_SIZE=${rank_size} +export SLOG_PRINT_TO_STDOUT=0 +export DEVICE_ID=0 +export DEVICE_INDEX=$DEVICE_ID + +if [ x"${CLUSTER}" == x"True" ];then + # ln hw log + ln -snf ${train_job_dir}/0/hw_bert.log ${train_job_dir} + this_ip=$(hostname -I |awk '{print $1}') + for ip in $MPIRUN_ALL_IP;do + if [ x"$ip" != x"$this_ip" ];then + scp $yamlPath root@$ip:$yamlPath + fi + done + export PATH=$PATH:/usr/local/mpirun4.0/bin + mpirun -H ${mpirun_ip} \ + --bind-to none -map-by slot\ + --allow-run-as-root \ + --mca btl_tcp_if_exclude lo,docker0,endvnic,virbr0,vethf40501b,docker_gwbridge,br-f42ac38052b4\ + --prefix /usr/local/mpirun4.0/ \ + ${currentDir}/scripts/train.sh 0 $currtime $yamlPath 0 True ${toolsPath} ${rank_size} +else + # ln hw log + ln -snf ${train_job_dir}/${first_device_id}/hw_bert.log ${train_job_dir} + rank_id=0 + for device_id in ${device_group};do + #echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] start: train ${device_id} & " >> ./main.log + ${currentDir}/scripts/train.sh $device_id $currtime $yamlPath $rank_id solo ${toolsPath} ${rank_size} & + let rank_id++ + done +fi +wait + + diff --git a/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/scripts/train.sh b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/scripts/train.sh new file mode 100644 index 0000000..9b39b78 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Base/tensorflow/scripts/train.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# 0 $currtime $yamlPath 0 cluster ${toolsPath} +device_id=$1 +currtime=$2 +yamlPath=$3 +toolsPath=$6 +rank_size=$7 + + +export YAML_PATH=$3 + +mainDir=$(cd "$(dirname "$0")/.."; pwd) + +mkdir -p ${mainDir%train*}/train/result/tf_bert_base/training_job_${currtime}/ +export train_job_dir=${mainDir%train*}/train/result/tf_bert_base/training_job_${currtime}/ + + +#exec_path=${train_job_dir} + +cd ${train_job_dir} + +export utilDir=$(cd "$(dirname "$yamlPath")/../atlas_benchmark-master/utils"; pwd) +export utilDir=$(cd "$(dirname "$yamlPath")/../atlas_benchmark-master/utils/atlasboost"; pwd) +source ${mainDir}/config/npu_set_env.sh + + +# 从 yaml 获取配置 +eval $(${toolsPath}/get_params_for_yaml.sh ${yamlPath} "tensorflow_config") + +# 声明变量 +export REMARK_LOG_FILE=hw_bert.log # 打点日志文件名称， 必须hw_后跟模型名称小写 +# 添加日志打点模块路径 +benchmark_log_path=${mainDir%atlas_benchmark-master*}/atlas_benchmark-master/utils +export PYTHONPATH=$PYTHONPATH:${benchmark_log_path} + +export JOB_ID=9999001 +export RANK_TABLE_FILE=${mainDir}/config/${rank_size}p.json +export RANK_SIZE=${rank_size} + +export SLOG_PRINT_TO_STDOUT=0 +export DEVICE_ID=${device_id} +export DEVICE_INDEX=$DEVICE_ID +export RANK_INDEX=0 + + +export PROFILING_OPTIONS=${PROFILING_OPTIONS} +export FP_POINT=${FP_POINT} +export BP_POINT=${BP_POINT} + +if [ ${PROFILING_MODE} == True ]; +then + export PROFILING_MODE=true +else + export PROFILING_MODE=false +fi + +if [ ${PROFILING_MODE} == True ]; +then + export AICPU_PROFILING_MODE=true +else + export AICPU_PROFILING_MODE=false +fi + + +if [ x"${device_id}" = x ] ; +then + echo "turing train fail" >> ${exec_path}/train_${device_id}.log + exit +else + export DEVICE_ID=${device_id} +fi + + +env > ${currentDir}/env_${device_id}.log + +cd ${train_job_dir} + +if [ x"$5" != x"True" ];then + rank_id=$4 + export RANK_ID=$4 +else + device_id_mo=$(python3.7 -c "import src.tensorflow.mpi_ops as atlasboost;atlasboost.init(); \ + device_id = atlasboost.local_rank();cluster_device_id = str(device_id); \ + atlasboost.set_device_id(device_id);print(atlasboost.rank())") + device_id_mo=`echo $device_id_mo` + rank_id=${device_id_mo##* } + #echo rank_id is $rank_id + export RANK_ID=${rank_id} + device=${device_id_mo##*deviceid = } + device_id=${device%% phyid=*} + export DEVICE_ID=${device_id} + #echo device_id is $device_id + hccljson=${train_job_dir}/*.json + cp ${hccljson} ${mainDir}/config/${rank_size}p.json +fi +env > ${currentDir}/env_${device_id}.log +#mkdir exec path + + +mkdir -p ${train_job_dir}/${device_id}/ckpt${DEVICE_ID} +cd ${train_job_dir}/${device_id} + +startTime=`date +%Y%m%d-%H:%M:%S` +startTime_s=`date +%s` + + +#start exec +python3.7 ${mainDir}/code/pretrain/run_pretraining.py \ + --bert_config_file=${mainDir}/config/${bert_config_file} \ + --max_seq_length=${max_seq_length} \ + --max_predictions_per_seq=${max_predictions_per_seq} \ + --train_batch_size=${train_batch_size} \ + --learning_rate=${learning_rate} \ + --num_warmup_steps=${num_warmup_steps} \ + --num_train_steps=${num_train_steps} \ + --optimizer_type=${optimizer_type} \ + --manual_fp16=${manual_fp16} \ + --use_fp16_cls=${use_fp16_cls} \ + --input_files_dir=${input_files_dir} \ + --eval_files_dir=${eval_files_dir} \ + --npu_bert_debug=${npu_bert_debug} \ + --npu_bert_use_tdt=${npu_bert_use_tdt} \ + --do_train=${do_train} \ + --do_eval=${do_eval} \ + --num_accumulation_steps=${num_accumulation_steps} \ + --npu_bert_job_start_file=None \ + --iterations_per_loop=${iterations_per_loop} \ + --npu_bert_loss_scale=${npu_bert_loss_scale} \ + --distributed=${distributed} \ + --save_checkpoints_steps=${save_checkpoints_steps} \ + --npu_bert_clip_by_global_norm=${npu_bert_clip_by_global_norm} \ + --output_dir=${train_job_dir}/${device_id}/ckpt${DEVICE_ID} > ${train_job_dir}/train_${device_id}.log 2>&1 + + +if [ $? -eq 0 ] ;then + echo ":::ABK 1.0.0 bert train success" + echo ":::ABK 1.0.0 bert train success" >> ${train_job_dir}/train_${device_id}.log + echo ":::ABK 1.0.0 bert train success" >> ${train_job_dir}/${device_id}/hw_bert.log +else + echo ":::ABK 1.0.0 bert train failed" + echo ":::ABK 1.0.0 bert train failed" >> ${train_job_dir}/train_${device_id}.log + echo ":::ABK 1.0.0 bert train failed" >> ${train_job_dir}/${device_id}/hw_bert.log +fi + +endTime=`date +%Y%m%d-%H:%M:%S` +endTime_s=`date +%s` +sumTime=$[ $endTime_s - $startTime_s ] +hour=$(( $sumTime/3600 )) +min=$(( ($sumTime-${hour}*3600)/60 )) +sec=$(( $sumTime-${hour}*3600-${min}*60 )) +echo ":::ABK 1.0.0 bert train total time ${hour}:${min}:${sec}" +echo ":::ABK 1.0.0 bert train total time ${hour}:${min}:${sec}" >> ${train_job_dir}/${device_id}/hw_bert.log + +#if [ x"$5" == x"solo" ]; +#then +# /bin/cp -f hw_bert.log $perfDir/hw_bert.log +#fi diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/README.md b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/README.md new file mode 100644 index 0000000..ff1b06a --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/README.md @@ -0,0 +1,54 @@ +# Bert-Large_tensorflow训练说明 + +### 1. 模型训练参数配置 + +在train/yaml/Bert-Large.yaml中修改相应配置， 配置项含义: + +``` +tensorflow_config: + #中文数据用 bert_config_large_cn.json 英文用bert_config_large_en.json + bert_config_file: bert_config_large_cn.json + #数据集句子长度是256时 设置为 256,40，句子长度是128时设置为128,20 + max_seq_length: 128 + max_predictions_per_seq: 20 + + # 最佳性能train_batch_size为96，如果超显存，可调小至32 + train_batch_size: 96 + learning_rate: 3.125e-5 + num_warmup_steps: 100 + num_train_steps: 1000 + optimizer_type: adam + manual_fp16: True + use_fp16_cls: True + input_files_dir: /home/BertData/cn-wiki-128/ + eval_files_dir: /home/BertData/cn-wiki-128/ + do_train: True + do_eval: True + num_accumulation_steps: 1 + iterations_per_loop: 100 + npu_bert_loss_scale: 0 + save_checkpoints_steps: 1000 + npu_bert_clip_by_global_norm: False + + # docker 镜像名称:版本号 + docker_image: c73:b021 + + # 仅多机执行需要配置: ip1:卡数量1,ip2:卡数量2 + mpirun_ip: 90.90.140.199:8,90.90.140.229:8 + + # 指定 device id, 多个 id 使用空格分隔, 数量需与 rank_size 相同 + device_group_1p: 0 + device_group_2p: 0 1 + device_group_4p: 0 1 2 3 + +``` + +------ + + + + + + + + diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/CONTRIBUTING.md b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/CONTRIBUTING.md new file mode 100644 index 0000000..124b4b3 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/CONTRIBUTING.md @@ -0,0 +1,31 @@ +# How to Contribute + +BERT needs to maintain permanent compatibility with the pre-trained model files, +so we do not plan to make any major changes to this library (other than what was +promised in the README). However, we can accept small patches related to +re-factoring and documentation. To submit contributes, there are just a few +small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Community Guidelines + +This project follows +[Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/Dockerfile b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/Dockerfile new file mode 100644 index 0000000..f9044f5 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/Dockerfile @@ -0,0 +1,31 @@ +ARG FROM_IMAGE_NAME=nvcr.io/nvidia/tensorflow:19.08-py3 + +FROM tensorrtserver_client as trt + +FROM ${FROM_IMAGE_NAME} + +RUN apt-get update && apt-get install -y pbzip2 pv bzip2 libcurl3 + +RUN pip install toposort networkx pytest nltk tqdm html2text progressbar + +WORKDIR /workspace +RUN git clone https://github.com/openai/gradient-checkpointing.git +RUN git clone https://github.com/attardi/wikiextractor.git +RUN git clone https://github.com/soskek/bookcorpus.git +RUN git clone https://github.com/titipata/pubmed_parser + +RUN pip3 install /workspace/pubmed_parser + +#Copy the perf_client over +COPY --from=trt /workspace/install/ /workspace/install/ + +#Install the python wheel with pip +RUN pip install /workspace/install/python/tensorrtserver*.whl + +WORKDIR /workspace/bert +COPY . . + +ENV PYTHONPATH /workspace/bert +ENV BERT_PREP_WORKING_DIR /workspace/bert/data +ENV PATH //workspace/install/bin:${PATH} +ENV LD_LIBRARY_PATH /workspace/install/lib:${LD_LIBRARY_PATH} diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/LICENSE b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/NOTICE b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/NOTICE new file mode 100644 index 0000000..917c2a6 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/NOTICE @@ -0,0 +1,4 @@ +BERT TensorFlow + +This repository includes software from https://github.com/google-research/bert +licensed under the Apache License, Version 2.0 (the "License") \ No newline at end of file diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/README.md b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/README.md new file mode 100644 index 0000000..b880f57 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/README.md @@ -0,0 +1,1155 @@ +# BERT For TensorFlow + +This repository provides a script and recipe to train the BERT model for TensorFlow to achieve state-of-the-art accuracy, and is tested and maintained by NVIDIA. + +## Table Of Contents + +- [Model overview](#model-overview) + * [Model architecture](#model-architecture) + * [Default configuration](#default-configuration) + * [Feature support matrix](#feature-support-matrix) + * [Features](#features) + * [Mixed precision training](#mixed-precision-training) + * [Enabling mixed precision](#enabling-mixed-precision) + * [Glossary](#glossary) +- [Setup](#setup) + * [Requirements](#requirements) +- [Quick Start Guide](#quick-start-guide) +- [Advanced](#advanced) + * [Scripts and sample code](#scripts-and-sample-code) + * [Parameters](#parameters) + * [Command-line options](#command-line-options) + * [Getting the data](#getting-the-data) + * [Dataset guidelines](#dataset-guidelines) + * [Multi-dataset](#multi-dataset) + * [Training process](#training-process) + * [Pre-training](#pre-training) + * [Fine tuning](#fine-tuning) + * [Multi-node](#multi-node) + * [Inference process](#inference-process) + * [Deploying the BERT model using TensorRT Inference Server](#deploying-the-bert-model-using-tensorrt-inference-server) + * [BioBERT](#biobert) +- [Performance](#performance) + * [Benchmarking](#benchmarking) + * [Training performance benchmark](#training-performance-benchmark) + * [Inference performance benchmark](#inference-performance-benchmark) + * [Results](#results) + * [Training accuracy results](#training-accuracy-results) + * [Pre-training accuracy: single-node](#pre-training-accuracy-single-node) + * [Pre-training accuracy: multi-node](#pre-training-accuracy-multi-node) + * [Fine-tuning accuracy for SQuAD: NVIDIA DGX-2 (16x V100 32G)](#fine-tuning-accuracy-for-squad-nvidia-dgx-2-16x-v100-32g) + * [Training stability test](#training-stability-test) + * [Pre-training SQuAD stability test: NVIDIA DGX-2 (512x V100 32G)](#fine-tuning-squad-stability-test-nvidia-dgx-2-512x-v100-32g) + * [Fine-tuning SQuAD stability test: NVIDIA DGX-2 (16x V100 32G)](#fine-tuning-squad-stability-test-nvidia-dgx-2-16x-v100-32g) + * [Training performance results](#training-performance-results) + * [Training performance: NVIDIA DGX-1 (8x V100 16G)](#training-performance-nvidia-dgx-1-8x-v100-16g) + * [Pre-training training performance: single-node on 16G](#pre-training-training-performance-single-node-on-16g) + * [Pre-training training performance: multi-node on 16G](#pre-training-training-performance-multi-node-on-16g) + * [Fine-tuning training performance for SQuAD on 16G](#fine-tuning-training-performance-for-squad-on-16g) + * [Training performance: NVIDIA DGX-1 (8x V100 32G)](#training-performance-nvidia-dgx-1-8x-v100-32g) + * [Pre-training training performance: single-node on 32G](#pre-training-training-performance-single-node-on-32g) + * [Fine-tuning training performance for SQuAD on 32G](#fine-tuning-training-performance-for-squad-on-32g) + * [Training performance: NVIDIA DGX-2 (16x V100 32G)](#training-performance-nvidia-dgx-2-16x-v100-32g) + * [Pre-training training performance: single-node on DGX-2 32G](#pre-training-training-performance-single-node-on-dgx-2-32g) + * [Pre-training training performance: multi-node on DGX-2 32G](#pre-training-training-performance-multi-node-on-dgx-2-32g) + * [Fine-tuning training performance for SQuAD on DGX-2 32G](#fine-tuning-training-performance-for-squad-on-dgx-2-32g) + * [Inference performance results](#inference-performance-results) + * [Inference performance: NVIDIA DGX-1 (1x V100 16G)](#inference-performance-nvidia-dgx-1-1x-v100-16g) + * [Pre-training inference performance on 16G](#pre-training-inference-performance-on-16g) + * [Fine-tuning inference performance for SQuAD on 16G](#fine-tuning-inference-performance-for-squad-on-16g) + * [Inference performance: NVIDIA DGX-1 (1x V100 32G)](#inference-performance-nvidia-dgx-1-1x-v100-32g) + * [Pre-training inference performance on 32G](#pre-training-inference-performance-on-32g) + * [Fine-tuning inference performance for SQuAD on 32G](#fine-tuning-inference-performance-for-squad-on-32g) + * [Inference performance: NVIDIA DGX-2 (1x V100 32G)](#inference-performance-nvidia-dgx-2-1x-v100-32g) + * [Pre-training inference performance on DGX-2 32G](#pre-training-inference-performance-on-dgx-2-32g) + * [Fine-tuning inference performance for SQuAD on DGX-2 32G](#fine-tuning-inference-performance-for-squad-on-dgx-2-32g) + * [Inference performance: NVIDIA Tesla T4 (1x T4 16G)](#inference-performance-nvidia-tesla-t4-1x-t4-16g) + * [Fine-tuning inference performance for SQuAD on Tesla T4 16G](#fine-tuning-inference-performance-for-squad-on-tesla-t4-16g) +- [Release notes](#release-notes) + * [Changelog](#changelog) + * [Known issues](#known-issues) + + + + +## Model overview + +BERT, or Bidirectional Encoder Representations from Transformers, is a new method of pre-training language representations which obtains state-of-the-art results on a wide array of Natural Language Processing (NLP) tasks. This model is based on the [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805) paper. NVIDIA's BERT is an optimized version of [Google's official implementation](https://github.com/google-research/bert), leveraging mixed precision arithmetic and Tensor Cores on V100 GPUs for faster training times while maintaining target accuracy. + +Other publicly available implementations of BERT include: +1. [NVIDIA PyTorch](https://github.com/NVIDIA/DeepLearningExamples/tree/master/PyTorch/LanguageModeling/BERT) +2. [Hugging Face](https://github.com/huggingface/pytorch-pretrained-BERT) +3. [codertimo](https://github.com/codertimo/BERT-pytorch) +4. [gluon-nlp](https://github.com/dmlc/gluon-nlp/tree/master/scripts/bert) +5. [Google's official implementation](https://github.com/google-research/bert) + +This model is trained with mixed precision using Tensor Cores on NVIDIA Volta and Turing GPUs. Therefore, researchers can get results upto 4x faster than training without Tensor Cores, while experiencing the benefits of mixed precision training. This model is tested against each NGC monthly container release to ensure consistent accuracy and performance over time. + +### Model architecture + +BERT's model architecture is a multi-layer bidirectional Transformer encoder. Based on the model size, we have the following two default configurations of BERT: + +| **Model** | **Hidden layers** | **Hidden unit size** | **Attention heads** | **Feedforward filter size** | **Max sequence length** | **Parameters** | +|:---------:|:----------:|:----:|:---:|:--------:|:---:|:----:| +|BERTBASE |12 encoder| 768| 12|4 x 768|512|110M| +|BERTLARGE|24 encoder|1024| 16|4 x 1024|512|330M| + +BERT training consists of two steps, pre-training the language model in an unsupervised fashion on vast amounts of unannotated datasets, and then using this pre-trained model for fine-tuning for various NLP tasks, such as question and answer, sentence classification, or sentiment analysis. Fine-tuning typically adds an extra layer or two for the specific task and further trains the model using a task-specific annotated dataset, starting from the pre-trained backbone weights. The end-to-end process in depicted in the following image: + +![](data/images/bert_pipeline.png?raw=true) + +Figure 1: BERT Pipeline + +### Default configuration + +This repository contains scripts to interactively launch data download, training, benchmarking and inference routines in a Docker container for both pre-training and fine tuning for Question Answering. The major differences between the official implementation of the paper and our version of BERT are as follows: + +- Mixed precision support with TensorFlow Automatic Mixed Precision (TF-AMP), which enables mixed precision training without any changes to the code-base by performing automatic graph rewrites and loss scaling controlled by an environmental variable. +- Scripts to download dataset for: + - Pre-training - [Wikipedia](https://dumps.wikimedia.org/), [BookCorpus](http://yknzhu.wixsite.com/mbweb) + - Fine tuning - [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/) (Stanford Question Answering Dataset) + - Fine tuning - [GLUE](https://gluebenchmark.com/) (The General Language Understanding Evaluation benchmark) + - Pretrained weights from Google +- Custom fused CUDA kernels for faster computations +- Multi-GPU/Multi-node support using Horovod + +The following performance optimizations were implemented in this model: +- [XLA](https://www.tensorflow.org/xla) support (experimental). + +These techniques and optimizations improve model performance and reduce training time, allowing you to perform various NLP tasks with no additional effort. + + +### Feature support matrix + +The following features are supported by this model. + +| **Feature** | **BERT** | +|:-----------------------:|:--------------------------:| +| Horovod Multi-GPU | Yes | +| Horovod Multi-Node | Yes | +| Automatic mixed precision (AMP) | Yes | +| LAMB | Yes | + +#### Features + +Multi-GPU training with Horovod - Our model uses Horovod to implement efficient multi-GPU training with NCCL. For details, see example sources in this repository or see the [TensorFlow tutorial](https://github.com/horovod/horovod/#usage) + +[LAMB](https://arxiv.org/pdf/1904.00962.pdf) stands for Layerwise Adaptive Moments based optimizer, is a large batch optimization technique that helps accelerate training of deep neural networks using large minibatches. It allows using a global batch size of 65536 and 32768 on sequence lengths 128 and 512 respectively, compared to a batch size of 256 for Adam. The optimized implementation accumulates 1024 gradients batches in phase 1 and 4096 steps in phase 2 before updating weights once. This results in 27% training speedup on a single DGX2 node. On multi-node systems, LAMB allows scaling up to 1024 GPUs resulting in training speedups of up to 17x in comparison to [Adam](https://arxiv.org/pdf/1412.6980.pdf). Adam has limitations on the learning rate that can be used since it is applied globally on all parameters whereas LAMB follows a layerwise learning rate strategy. + +NVLAMB adds necessary tweaks to [LAMB version 1](https://arxiv.org/abs/1904.00962v1), to ensure correct convergence. A guide to implementating the LAMB optimizer can be found in our [article](https://medium.com/@NvidiaAI/a-guide-to-optimizer-implementation-for-bert-at-scale-8338cc7f45fd) on Medium.com. The algorithm is as follows: + ![NVLAMB](data/images/images_nvlamb.png) + +### Mixed precision training + +Mixed precision is the combined use of different numerical precision in a computational method. [Mixed precision](https://arxiv.org/abs/1710.03740) training offers significant computational speedup by performing operations in half-precision format, while storing minimal information in single-precision to retain as much information as possible in critical parts of the network. Since the introduction of [Tensor Cores](https://developer.nvidia.com/tensor-cores) in the Volta and Turing architecture, significant training speedups are experienced by switching to mixed precision -- up to 3x overall speedup on the most arithmetically intense model architectures. Using mixed precision training requires two steps: +1. Porting the model to use the FP16 data type where appropriate. +2. Adding loss scaling to preserve small gradient values. + +The ability to train deep learning networks with lower precision was introduced in the Pascal architecture and first supported in [CUDA 8](https://devblogs.nvidia.com/parallelforall/tag/fp16/) in the NVIDIA Deep Learning SDK. + +For information about: +- How to train using mixed precision, see the [Mixed Precision Training](https://arxiv.org/abs/1710.03740) paper and [Training With Mixed Precision](https://docs.nvidia.com/deeplearning/sdk/Mixed-Precision-training/index.html) documentation. +- Techniques used for mixed precision training, see the [Mixed Precision Training of Deep Neural Networks](https://devblogs.nvidia.com/mixed-precision-training-deep-neural-networks/) blog. +- How to access and enable AMP for TensorFlow, see [Using TF-AMP](https://docs.nvidia.com/deeplearning/dgx/tensorflow-user-guide/index.html#tfamp) from the TensorFlow User Guide. + +#### Enabling mixed precision + +Automatic Mixed Precision (AMP) for TensorFlow enables the full [mixed precision methodology](https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html#tensorflow) in your existing TensorFlow model code. AMP enables mixed precision training on Volta and Turing GPUs automatically. The TensorFlow framework code makes all necessary model changes internally. + +In TF-AMP, the computational graph is optimized to use as few casts as necessary and maximizes the use of FP16, and the loss scaling is automatically applied inside of supported optimizers. AMP can be configured to work with the existing `tf.contrib` loss scaling manager by disabling the AMP scaling with a single environment variable to perform only the automatic mixed precision optimization. It accomplishes this by automatically rewriting all computation graphs with the necessary operations to enable mixed precision training and automatic loss scaling. + + +### Glossary + +**Fine-tuning** +Training an already pretrained model further using a task specific dataset for subject-specific refinements, by adding task-specific layers on top if required. + +**Language Model** +Assigns a probability distribution over a sequence of words. Given a sequence of words, it assigns a probability to the whole sequence. + +**Pre-training** +Training a model on vast amounts of data on the same (or different) task to build general understandings. + +**Transformer** +The paper [Attention Is All You Need](https://arxiv.org/abs/1706.03762) introduces a novel architecture called Transformer that uses an attention mechanism and transforms one sequence into another. + + +## Setup + +The following section lists the requirements in order to start training the BERT model. + + +### Requirements + +This repository contains `Dockerfile` which extends the TensorFlow NGC container and encapsulates some dependencies. Aside from these dependencies, ensure you have the following components: +- [NVIDIA Docker](https://github.com/NVIDIA/nvidia-docker) +- [TensorFlow 19.08-py3+](https://ngc.nvidia.com/catalog/containers/nvidia:tensorflow) NGC container +- [NVIDIA Volta](https://www.nvidia.com/en-us/data-center/volta-gpu-architecture/) or [Turing](https://www.nvidia.com/en-us/geforce/turing/) based GPU + +For more information about how to get started with NGC containers, see the following sections from the NVIDIA GPU Cloud Documentation and the Deep Learning Documentation: +- [Getting Started Using NVIDIA GPU Cloud](https://docs.nvidia.com/ngc/ngc-getting-started-guide/index.html) +- [Accessing And Pulling From The NGC Container Registry](https://docs.nvidia.com/deeplearning/frameworks/user-guide/index.html#accessing_registry) +- [Running TensorFlow](https://docs.nvidia.com/deeplearning/frameworks/tensorflow-release-notes/running.html#running) + +For those unable to use the TensorFlow NGC container, to set up the required environment or create your own container, see the versioned [NVIDIA Container Support Matrix](https://docs.nvidia.com/deeplearning/frameworks/support-matrix/index.html). + +For multi-node, the sample provided in this repository requires [Enroot](https://github.com/NVIDIA/enroot) and [Pyxis](https://github.com/NVIDIA/pyxis) set up on a [SLURM](https://slurm.schedmd.com) cluster. + +More information on how to set up and launch can be found in the [Multi-node Documentation](https://docs.nvidia.com/ngc/multi-node-bert-user-guide). + + +## Quick Start Guide + +To pretrain or fine tune your model for Question Answering using mixed precision with Tensor Cores or using FP32, perform the following steps using the default parameters of the BERT model. + +1. Clone the repository. + +```bash +git clone https://github.com/NVIDIA/DeepLearningExamples +cd DeepLearningExamples/TensorFlow/LanguageModeling/BERT +``` + +2. Build the BERT TensorFlow NGC container. + +```bash +bash scripts/docker/build.sh +``` + +3. Download and preprocess the dataset. + +This repository provides scripts to download, verify and extract the SQuAD dataset, GLUE dataset and pretrained weights for fine tuning as well as Wikipedia and BookCorpus dataset for pre-training. + +To download, verify, and extract the required datasets, run: + +```bash +bash scripts/data_download.sh +``` + +The script launches a Docker container with the current directory mounted and downloads the datasets to a `data/` folder on the host. + +Note: The dataset is 170GB+ and takes 15+ hours to download. The BookCorpus server could sometimes get overloaded and also contain broken links resulting in HTTP 403 and 503 errors. You can either skip the missing files or retry downloading at a later time. Expired dataset links are ignored during data download. + +4. Download the pretrained models from NGC. + +We have uploaded checkpoints for both fine tuning and pre-training for various configurations on the NGC Model Registry. You can download them directly from the [NGC model catalog](https://ngc.nvidia.com/catalog/models). Download them to the `results/models/` to easily access them in your scripts. + + +5. Start an interactive session in the NGC container to run training/inference. + +After you build the container image and download the data, you can start an interactive CLI session as follows: + +```bash +bash scripts/docker/launch.sh +``` + +The `launch.sh` script assumes that the datasets are in the following locations by default after downloading the data. + +- SQuAD v1.1 - `data/download/squad/v1.1` +- SQuAD v2.0 - `data/download/squad/v2.0` +- GLUE The Corpus of Linguistic Acceptability (CoLA) - `data/download/CoLA` +- GLUE Microsoft Research Paraphrase Corpus (MRPC) - `data/download/MRPC` +- GLUE The Multi-Genre NLI Corpus (MNLI) - `data/download/MNLI` +- BERT Large - `data/download/google_pretrained_weights/uncased_L-24_H-1024_A-16` +- BERT Base - `data/download/google_pretrained_weights/uncased_L-12_H-768_A-12` +- BERT - `data/download/google_pretrained_weights/uncased_L-24_H-1024_A-16` +- Wikipedia + BookCorpus TFRecords - `data/tfrecords/books_wiki_en_corpus` + +6. Start pre-training. + +BERT is designed to pre-train deep bidirectional representations for language representations. The following scripts are to replicate pre-training on Wikipedia and BookCorpus from the [LAMB paper](https://arxiv.org/pdf/1904.00962.pdf). These scripts are general and can be used for pre-training language representations on any corpus of choice. + +From within the container, you can use the following script to run pre-training using LAMB. +```bash +bash scripts/run_pretraining_lamb.sh +``` + +For BERT Large FP16 training with XLA using a DGX-1 V100 32G, run: +```bash +bash scripts/run_pretraining_lamb.sh 64 8 8 7.5e-4 5e-4 fp16 true 8 2000 200 7820 100 128 512 large +``` + +For BERT Large FP32 training without XLA using a DGX-1 V100 32G, run: +```bash +bash scripts/run_pretraining_lamb.sh 64 8 8 7.5e-4 5e-4 fp32 false 8 2000 200 7820 100 128 512 large +``` + +Alternatively, to run pre-training with Adam as in the original [BERT paper](https://arxiv.org/pdf/1810.04805.pdf) from within the container, run: + +```bash +bash scripts/run_pretraining_adam.sh +``` + +7. Start fine tuning. + +The above pretrained BERT representations can be fine tuned with just one additional output layer for a state-of-the-art Question Answering system. From within the container, you can use the following script to run fine-training for SQuAD. + +```bash +bash scripts/run_squad.sh +``` + +For SQuAD 1.1 FP16 training with XLA using a DGX-1 V100 32G, run: +```bash +bash scripts/run_squad.sh 10 5e-6 fp16 true 8 384 128 large 1.1 data/download/google_pretrained_weights/uncased_L-24_H-1024_A-16/bert_model.ckpt 1.1 +``` + +For SQuAD 2.0 FP32 training without XLA using a DGX-1 V100 32G, run: +```bash +bash scripts/run_squad.sh 5 5e-6 fp32 false 8 384 128 large 1.1 data/download/google_pretrained_weights/uncased_L-24_H-1024_A-16/bert_model.ckpt 2.0 +``` + +Alternatively, to run fine tuning on GLUE benchmark, run: + +```bash +bash scripts/run_glue.sh +``` + +The GLUE tasks supported include CoLA, MRPC and MNLI. + +8. Start validation/evaluation. + +The `run_squad_inference.sh` script runs inference on a checkpoint fine tuned for SQuAD and evaluates the validity of predictions on the basis of exact match and F1 score. + +```bash +bash scripts/run_squad_inference.sh +``` + +For SQuAD 2.0 FP16 inference with XLA using a DGX-1 V100 32G, run: +```bash +bash scripts/run_squad_inference.sh /results/model.ckpt 8 fp16 true 384 128 large 2.0 +``` + +For SQuAD 1.1 FP32 inference without XLA using a DGX-1 V100 32G, run: +```bash +bash scripts/run_squad_inference.sh /results/model.ckpt 8 fp32 false 384 128 large 1.1 +``` + +Alternatively, to run inference on GLUE benchmark, run: +```bash +bash scripts/run_glue_inference.sh +``` + +## Advanced + +The following sections provide greater details of the dataset, running training and inference, and the training results. + +### Scripts and sample code + +In the root directory, the most important files are: +* `run_pretraining.py` - Serves as entry point for pre-training +* `run_squad.py` - Serves as entry point for SQuAD training +* `run_classifier.py` - Serves as entry point for GLUE training +* `Dockerfile` - Container with the basic set of dependencies to run BERT + +The `scripts/` folder encapsulates all the one-click scripts required for running various functionalities supported such as: +* `run_squad.sh` - Runs SQuAD training and inference using `run_squad.py` file +* `run_glue.sh` - Runs GLUE training and inference using the `run_classifier.py` file +* `run_pretraining_adam.sh` - Runs pre-training with Adam optimizer using the `run_pretraining.py` file +* `run_pretraining_lamb.sh` - Runs pre-training with LAMB optimizer using the `run_pretraining.py` file in two phases. Phase 1 does 90% of training with sequence length = 128. In phase 2, the remaining 10% of the training is done with sequence length = 512. +* `data_download.sh` - Downloads datasets using files in the `data/` folder +* `finetune_train_benchmark.sh` - Captures performance metrics of training for multiple configurations +* `finetune_inference_benchmark.sh` - Captures performance metrics of inference for multiple configurations + +Other folders included in the root directory are: +* `data/` - Necessary folders and scripts to download datasets required for fine tuning and pre-training BERT. +* `utils/` - Necessary files for preprocessing data before feeding into BERT and hooks for obtaining performance metrics from BERT. + +### Parameters + +Aside from the options to set hyperparameters, the relevant options to control the behaviour of the `run_pretraining.py` script are: + +``` + --bert_config_file: The config json file corresponding to the pre-trained BERT model. This specifies the model architecture. + --init_checkpoint: Initial checkpoint (usually from a pre-trained BERT model). + --[no]do_eval: Whether to run evaluation on the dev set.(default: 'false') + --[no]do_train: Whether to run training.(evaluation: 'false') + --eval_batch_size: Total batch size for eval.(default: '8')(an integer) + --[no]horovod: Whether to use Horovod for multi-gpu runs(default: 'false') + --[no]use_fp16: Whether to enable AMP ops.(default: 'false') + --input_files_dir: Input TF example files (can be a dir or comma separated). + --output_dir: The output directory where the model checkpoints will be written. + --optimizer_type: Optimizer used for training - LAMB or ADAM + --num_accumulation_steps: Number of accumulation steps before gradient update. Global batch size = num_accumulation_steps * train_batch_size + --allreduce_post_accumulation: Whether to all reduce after accumulation of N steps or after each step +``` + +Aside from the options to set hyperparameters, some relevant options to control the behaviour of the `run_squad.py` script are: + +``` + --bert_config_file: The config json file corresponding to the pre-trained BERT model. This specifies the model architecture. + --output_dir: The output directory where the model checkpoints will be written. + --[no]do_predict: Whether to run evaluation on the dev set. (default: 'false') + --[no]do_train: Whether to run training. (default: 'false') + --learning_rate: The initial learning rate for Adam.(default: '5e-06')(a number) + --max_answer_length: The maximum length of an answer that can be generated. This is needed because the start and end predictions are not conditioned on one another.(default: '30')(an integer) + --max_query_length: The maximum number of tokens for the question. Questions longer than this will be truncated to this length.(default: '64')(an integer) + --max_seq_length: The maximum total input sequence length after WordPiece tokenization. Sequences longer than this will be truncated, and sequences shorter than this will be padded.(default: '384')(an integer) + --predict_batch_size: Total batch size for predictions.(default: '8')(an integer) + --train_batch_size: Total batch size for training.(default: '8')(an integer) + --[no]use_fp16: Whether to enable AMP ops.(default: 'false') + --[no]use_xla: Whether to enable XLA JIT compilation.(default: 'false') + --[no]version_2_with_negative: If true, the SQuAD examples contain some that do not have an answer.(default: 'false') +``` + +Aside from the options to set hyperparameters, some relevant options to control the behaviour of the `run_classifier.py` script are: + +``` + --bert_config_file: The config json file corresponding to the pre-trained BERT model. This specifies the model architecture. + --data_dir: The input data dir. Should contain the .tsv files (or other data files) for the task. + --[no]do_eval: Whether to run eval on the dev set. + (default: 'false') + --[no]do_predict: Whether to run the model in inference mode on the test set.(default: 'false') + --[no]do_train: Whether to run training.(default: 'false') + --[no]horovod: Whether to use Horovod for multi-gpu runs(default: 'false') + --init_checkpoint: Initial checkpoint (usually from a pre-trained BERT model). + --max_seq_length: The maximum total input sequence length after WordPiece tokenization. Sequences longer than this will be truncated, and sequences shorter than this will be padded.(default: '128')(an integer) + --num_train_epochs: Total number of training epochs to perform.(default: '3.0')(a number) + --output_dir: The output directory where the model checkpoints will be written. + --task_name: The name of the task to train. + --train_batch_size: Total batch size for training.(default: '32')(an integer) + --[no]use_fp16: Whether to use fp32 or fp16 arithmetic on GPU. + (default: 'false') + --[no]use_xla: Whether to enable XLA JIT compilation. + (default: 'false') + --vocab_file: The vocabulary file that the BERT model was trained on. + --warmup_proportion: Proportion of training to perform linear learning rate warmup for. E.g., 0.1 = 10% of training.(default: '0.1')(a number) +``` + +Note: When initializing from a checkpoint using `--init_checkpoint` and a corpus of your choice, keep in mind that `bert_config_file` and `vocab_file` should remain unchanged. + +### Command-line options + +To see the full list of available options and their descriptions, use the `-h` or `--help` command-line option with the Python file, for example: + +```bash +python run_pretraining.py --help +python run_squad.py --help +python run_classifier.py --help +``` + +### Getting the data + +For pre-training BERT, we use the concatenation of Wikipedia (2500M words) as well as BookCorpus (800M words). For Wikipedia, we extract only the text passages from [here](ftp://ftpmirror.your.org/pub/wikimedia/dumps/enwiki/latest/enwiki-latest-pages-articles-multistream.xml.bz2) and ignore headers list and tables. It is structured as a document level corpus rather than a shuffled sentence level corpus because it is critical to extract long contiguous sentences. + +The next step is to run `create_pretraining_data.py` with the document level corpus as input, which generates input data and labels for the masked language modeling and next sentence prediction tasks. Pre-training can also be performed on any corpus of your choice. The collection of data generation scripts are intended to be modular to allow modifications for additional preprocessing steps or to use additional data. They can hence easily be modified for an arbitrary corpus. + +The preparation of an individual pre-training dataset is described in the `create_datasets_from_start.sh` script found in the `data/` folder. The component steps to prepare the datasets are as follows: + +1. Data download and extract - the dataset is downloaded and extracted. +2. Clean and format - document tags, etc. are removed from the dataset. The end result of this step is a `{dataset_name_one_article_per_line}.txt` file that contains the entire corpus. Each line in the text file contains an entire document from the corpus. One file per dataset is created in the `formatted_one_article_per_line` folder. +3. Sharding - the sentence segmented corpus file is split into a number of smaller text documents. The sharding is configured so that a document will not be split between two shards. Sentence segmentation is performed at this time using NLTK. +4. TFRecord file creation - each text file shard is processed by the `create_pretraining_data.py` script to produce a corresponding TFRecord file. The script generates input data and labels for masked language modeling and sentence prediction tasks for the input text shard. + + +For fine tuning BERT for the task of Question Answering, we use SQuAD and GLUE. SQuAD v1.1 has 100,000+ question-answer pairs on 500+ articles. SQuAD v2.0 combines v1.1 with an additional 50,000 new unanswerable questions and must not only answer questions but also determine when that is not possible. GLUE consists of single-sentence tasks, similarity and paraphrase tasks and inference tasks. We support one of each: CoLA, MNLI and MRPC. + +#### Dataset guidelines + +The procedure to prepare a text corpus for pre-training is described in the previous section. This section provides additional insight into how exactly raw text is processed so that it is ready for pre-training. + +First, raw text is tokenized using [WordPiece tokenization](https://arxiv.org/pdf/1609.08144.pdf). A [CLS] token is inserted at the start of every sequence, and the two sentences in the sequence are separated by a [SEP] token. + +Note: BERT pre-training looks at pairs of sentences at a time. A sentence embedding token [A] is added to the first sentence and token [B] to the next. + +BERT pre-training optimizes for two unsupervised classification tasks. The first is Masked Language Modelling (Masked LM). One training instance of Masked LM is a single modified sentence. Each token in the sentence has a 15% chance of being replaced by a [MASK] token. The chosen token is replaced with [MASK] 80% of the time, 10% with another random token and the remaining 10% with the same token. The task is then to predict the original token. + +The second task is next sentence prediction. One training instance of BERT pre-training is two sentences (a sentence pair). A sentence pair may be constructed by simply taking two adjacent sentences from a single document, or by pairing up two random sentences with equal probability. The goal of this task is to predict whether or not the second sentence followed the first in the original document. + +The `create_pretraining_data.py` script takes in raw text and creates training instances for both pre-training tasks. + +#### Multi-dataset + +We are able to combine multiple datasets into a single dataset for pre-training on a diverse text corpus. Once TFRecords have been created for each component dataset, you can create a combined dataset by adding the directory to `SOURCES` in `run_pretraining_*.sh`. This will feed all matching files to the input pipeline in `run_pretraining.py`. However, in the training process, only one TFRecord file is consumed at a time, therefore, the training instances of any given training batch will all belong to the same source dataset. + +### Training process + +The training process consists of two steps: pre-training and fine tuning. + +#### Pre-training + +Pre-training is performed using the `run_pretraining.py` script along with parameters defined in the `scripts/run_pretraining_lamb.sh`. + +The `run_pretraining_lamb.sh` script runs a job on a single node that trains the BERT-large model from scratch using the Wikipedia and BookCorpus datasets as training data. By default, the training script: +- Runs on 8 GPUs. +- Has FP16 precision enabled. +- Is XLA enabled. +- Creates a log file containing all the output. +- Saves a checkpoint every 100 iterations (keeps only the latest checkpoint) and at the end of training. All checkpoints, evaluation results and training logs are saved to the `/results` directory (in the container which can be mounted to a local directory). +- Evaluates the model at the end of each phase. + +- Phase 1 + - Runs 7038 steps with 2000 warmup steps + - Sets Maximum sequence length as 128 + - Sets Global Batch size as 64K + +- Phase 2 + - Runs 1564 steps with 200 warm-up steps + - Sets Maximum sequence length as 512 + - Sets Global Batch size as 32K + - Starts from Phase1's final checkpoint + +These parameters train Wikipedia and BookCorpus with reasonable accuracy on a DGX-1 with 32GB V100 cards. + +For example: +```bash +scripts/run_pretraining_lamb.sh +``` + +Where: +- `` is per-GPU batch size used for training in the respective phase. Batch size varies with precision, larger batch sizes run more efficiently, but require more memory. + +- `` is per-GPU batch size used for evaluation after training. + +- `` is the default rate of 1e-4 is good for global batch size 256. + +- `` is the default rate of 1e-4 is good for global batch size 256. + +- `` is the type of math in your model, can be either `fp32` or `fp16`. Specifically: + + - `fp32` is 32-bit IEEE single precision floats. + - `fp16` is Automatic rewrite of TensorFlow compute graph to take advantage of 16-bit arithmetic whenever it is safe. + +- `` is the number of GPUs to use for training. Must be equal to or smaller than the number of GPUs attached to your node. + +- `` is the number of warm-up steps at the start of training in the respective phase. + +- `` is the total number of training steps in both phases combined. + +- `` controls how often checkpoints are saved. Default is 100 steps. + +- `` is used to mimic higher batch sizes in the respective phase by accumulating gradients N times before weight update. + +- `` is used to indicate whether to pretrain BERT Large or BERT Base model + +The following sample code trains BERT-large from scratch on a single DGX-2 using FP16 arithmetic. This will take around 4.5 days. + +```bash +bert_tf/scripts/run_pretraining_lamb.sh 32 8 8 3.75e-4 2.5e-4 fp16 true 16 2000 200 7820 100 128 512 256 large +``` + +#### Fine tuning + +Fine tuning is performed using the `run_squad.py` script along with parameters defined in `scripts/run_squad.sh`. + +The `run_squad.sh` script trains a model and performs evaluation on the SQuAD dataset. By default, the training script: + +- Trains for SQuAD v1.1 dataset. +- Trains on BERT Large Model. +- Uses 8 GPUs and batch size of 10 on each GPU. +- Has FP16 precision enabled. +- Is XLA enabled. +- Runs for 2 epochs. +- Saves a checkpoint every 1000 iterations (keeps only the latest checkpoint) and at the end of training. All checkpoints, evaluation results and training logs are saved to the `/results` directory (in the container which can be mounted to a local directory). +- Evaluation is done at the end of training. To skip evaluation, modify `--do_predict` to `False`. + +This script outputs checkpoints to the `/results` directory, by default, inside the container. Mount point of `/results` can be changed in the `scripts/docker/launch.sh` file. The training log contains information about: +- Loss for the final step +- Training and evaluation performance +- F1 and exact match score on the Dev Set of SQuAD after evaluation. + +The summary after training is printed in the following format: +```bash +I0312 23:10:45.137036 140287431493376 run_squad.py:1332] 0 Total Training Time = 3007.00 Training Time W/O start up overhead = 2855.92 Sentences processed = 175176 +I0312 23:10:45.137243 140287431493376 run_squad.py:1333] 0 Training Performance = 61.3378 sentences/sec +I0312 23:14:00.550846 140287431493376 run_squad.py:1396] 0 Total Inference Time = 145.46 Inference Time W/O start up overhead = 131.86 Sentences processed = 10840 +I0312 23:14:00.550973 140287431493376 run_squad.py:1397] 0 Inference Performance = 82.2095 sentences/sec +{"exact_match": 83.69914853358561, "f1": 90.8477003317459} +``` + +Multi-GPU training is enabled with the Horovod TensorFlow module. The following example runs training on 8 GPUs: + +```bash +BERT_DIR=data/download/google_pretrained_weights/uncased_L-24_H-1024_A-16 + +mpi_command="mpirun -np 8 -H localhost:8 \ + --allow-run-as-root -bind-to none -map-by slot \ + -x NCCL_DEBUG=INFO \ + -x LD_LIBRARY_PATH \ + -x PATH -mca pml ob1 -mca btl ^openib" \ + python run_squad.py --horovod --vocab_file=$BERT_DIR/vocab.txt \ + --bert_config_file=$BERT_DIR/bert_config.json \ + --output_dir=/results +``` + +#### Multi-node + + +Multi-node runs can be launched on a pyxis/enroot Slurm cluster (see [Requirements](#requirements)) with the `run.sub` script with the following command for a 4-node DGX1 example for both phase 1 and phase 2: +``` +BATCHSIZE=16 LEARNING_RATE='1.875e-4' NUM_ACCUMULATION_STEPS=128 PHASE=1 sbatch -N4 --ntasks-per-node=8 run.sub +BATCHSIZE=2 LEARNING_RATE='1.25e-4' NUM_ACCUMULATION_STEPS=512 PHASE=1 sbatch -N4 --ntasks-per-node=8 run.sub +``` + + +Checkpoint after phase 1 will be saved in `checkpointdir` specified in `run.sub`. The checkpoint will be automatically picked up to resume training on phase 2. Note that phase 2 should be run after phase 1. + +Variables to re-run the [Training performance results](#training-performance-results) are available in the `configurations.yml` file. + +The batch variables `BATCHSIZE`, `LEARNING_RATE`, `NUM_ACCUMULATION_STEPS` refer to the Python arguments `train_batch_size`, `learning_rate`, `num_accumulation_steps` respectively. +The variable `PHASE` refers to phase specific arguments available in `run.sub`. + +Note that the `run.sub` script is a starting point that has to be adapted depending on the environment. In particular, variables such as `datadir` handle the location of the files for each phase. + +Refer to the files contents to see the full list of variables to adjust for your system. + +### Inference process + +Inference on a fine tuned Question Answering system is performed using the `run_squad.py` script along with parameters defined in `scripts/run_squad_inference.sh`. Inference is supported on a single GPU. + +The `run_squad_inference.sh` script trains a model and performs evaluation on the SQuAD dataset. By default, the inferencing script: + +- Uses SQuAD v1.1 dataset +- Has FP16 precision enabled +- Is XLA enabled +- Evaluates the latest checkpoint present in `/results` with a batch size of 8 + +This script outputs predictions file to `/results/predictions.json` and computes F1 score and exact match score using SQuAD's evaluate file. Mount point of `/results` can be changed in the `scripts/docker/launch.sh` file. + +The output log contains information about: +Inference performance +Inference Accuracy (F1 and exact match scores) on the Dev Set of SQuAD after evaluation. + +The summary after inference is printed in the following format: +```bash +I0312 23:14:00.550846 140287431493376 run_squad.py:1396] 0 Total Inference Time = 145.46 Inference Time W/O start up overhead = 131.86 Sentences processed = 10840 +I0312 23:14:00.550973 140287431493376 run_squad.py:1397] 0 Inference Performance = 82.2095 sentences/sec +{"exact_match": 83.69914853358561, "f1": 90.8477003317459} +``` + +### Deploying the BERT model using TensorRT Inference Server + +The [NVIDIA TensorRT Inference Server](https://github.com/NVIDIA/tensorrt-inference-server) provides a datacenter and cloud inferencing solution optimized for NVIDIA GPUs. The server provides an inference service via an HTTP or gRPC endpoint, allowing remote clients to request inferencing for any number of GPU or CPU models being managed by the server. More information on how to perform inference using `TensorRT Inference Server` can be found in the subfolder `./trtis/README.md`. + +### BioBERT + +Many works, including [BioBERT](https://arxiv.org/pdf/1901.08746.pdf), [SciBERT](https://arxiv.org/pdf/1903.10676.pdf), [NCBI-BERT](https://arxiv.org/pdf/1906.05474.pdf), [ClinicalBERT (MIT)](https://arxiv.org/pdf/1904.03323.pdf), [ClinicalBERT (NYU, Princeton)](https://arxiv.org/pdf/1904.05342.pdf), and others at [BioNLP’19 workshop](https://aclweb.org/aclwiki/BioNLP_Workshop), show that pre-training of BERT on large biomedical text corpus such as [PubMed](https://www.ncbi.nlm.nih.gov/pubmed/) results in better performance in biomedical text-mining tasks. + +More information on how to download a biomedical corpus and pre-train as well as finetune for biomedical tasks can be found in the subfolder `./biobert/README.md`. + +## Performance + +### Benchmarking + +The following section shows how to run benchmarks measuring the model performance in training and inference modes. + +Both of these benchmarking scripts enable you to run a number of epochs, extract performance numbers, and run the BERT model for fine tuning. + +#### Training performance benchmark + +Training benchmarking can be performed by running the script: +``` bash +scripts/finetune_train_benchmark.sh squad +``` + +This script runs 2 epochs by default on the SQuAD v1.1 dataset and extracts performance numbers for various batch sizes and sequence lengths in both FP16 and FP32. These numbers are saved at `/results/squad_train_benchmark_bert__gpu_.log`. + +#### Inference performance benchmark + +Inference benchmarking can be performed by running the script: + +``` bash +scripts/finetune_inference_benchmark.sh squad +``` + +This script runs 1024 eval iterations by default on the SQuAD v1.1 dataset and extracts performance and latency numbers for various batch sizes and sequence lengths in both FP16 with XLA and FP32 without XLA. These numbers are saved at `/results/squad_inference_benchmark_bert_.log`. + +### Results + +The following sections provide details on how we achieved our performance and accuracy in training and inference for pre-training using LAMB optimizer as well as fine tuning for Question Answering. All results are on BERT-large model unless otherwise mentioned. All fine tuning results are on SQuAD v1.1 using a sequence length of 384 unless otherwise mentioned. + +#### Training accuracy results + +##### Training accuracy + +###### Pre-training accuracy: single-node + +Our results were obtained by running the `scripts/run_pretraining_lamb.sh` training script in the TensorFlow 19.06-py3 NGC container. + +| **DGX System** | **GPUs** | **Batch size / GPU: Phase1, Phase2** | **Accumulation Steps: Phase1, Phase2** | **Time to Train - mixed precision (Hrs)** | **Final Loss - mixed precision** | +|:---:|:---:|:----:|:----:|:---:|:----:| +| DGX1 | 8 | 16, 2 | 512, 2048 | 247.51 | 1.43 | +| DGX2 | 16 | 64, 8 | 64, 256 | 108.16 | 1.58 | + +###### Pre-training accuracy: multi-node + +Our results were obtained by running the `scripts/run_pretraining_lamb.sh` training script in the TensorFlow 19.08-py3 NGC container. + +| **DGX System** | **Nodes** | **Precision** | **Batch Size/GPU: Phase1, Phase2** | **Accumulation Steps: Phase1, Phase2** | **Time to Train (Hrs)** | **Final Loss** | +|----------------|-----------|---------------|------------------------------------|----------------------------------------|----------------|-------------------------| +| DGX1 | 4 | FP16 | 16, 4 |128, 256| 62.49 | 1.72 | +| DGX1 | 16 | FP16 | 16, 4 | 32, 64 | 16.58 | 1.76 | +| DGX1 | 32 | FP16 | 16, 2 | 16, 64 | 9.85 | 1.71 | +| DGX2H | 1 | FP16 | 32, 8 |128, 256| 69.27 | 1.59 | +| DGX2H | 4 | FP16 | 32, 8 | 32, 64 | 22.17 | 1.60 | +| DGX2H | 16 | FP16 | 32, 8 | 8, 16 | 6.25 | 1.56 | +| DGX2H | 32 | FP16 | 32, 8 | 4, 8 | 3.73 | 1.58 | +| DGX2H | 64 | FP16 | 32, 8 | 2, 4 | 2.44 | 1.64 | +| DGX2H | 64 | FP32 | 32, 4 | 2, 8 | 5.76 | 1.66 | + +Note: Time to train includes upto 16 minutes of start up time for every restart. Experiments were run on clusters with a maximum wall clock time of 8 hours. + +###### Fine-tuning accuracy for SQuAD: NVIDIA DGX-2 (16x V100 32G) + +Our results were obtained by running the `scripts/run_squad.sh` training script in the TensorFlow 19.08-py3 NGC container on NVIDIA DGX-2 with 16x V100 32G GPUs. + + +| **GPUs** | **Batch size / GPU** | **Accuracy - FP32** | **Accuracy - mixed precision** | **Time to Train - FP32 (Hrs)** | **Time to Train - mixed precision (Hrs)** | +|:---:|:----:|:----:|:---:|:----:|:----:| +| 16 | 4 |90.94|90.84|0.44|0.16| + + +##### Training stability test + +###### Pre-training stability test: NVIDIA DGX-2 (512x V100 32G) + +The following tables compare `Final Loss` scores across 5 different training runs with different seeds, for both FP16. The runs showcase consistent convergence on all 5 seeds with very little deviation. + +| **FP16, 512x GPUs** | **seed 1** | **seed 2** | **seed 3** | **seed 4** | **seed 5** | **mean** | **std** | +|:-----------:|:-----:|:-----:|:-----:|:-----:|:-----:|:-----:|:-----:| +|Final Loss |1.57 |1.598 |1.614 |1.583 |1.584 |1.5898|0.017 | + +###### Fine-tuning SQuAD stability test: NVIDIA DGX-2 (16x V100 32G) + +The following tables compare `F1` scores across 5 different training runs with different seeds, for both FP16 and FP32 respectively. The runs showcase consistent convergence on all 5 seeds with very little deviation. + +| **FP16, 8x GPUs** | **seed 1** | **seed 2** | **seed 3** | **seed 4** | **seed 5** | **mean** | **std** | +|:-----------:|:-----:|:-----:|:-----:|:-----:|:-----:|:-----:|:-----:| +|F1 |90.99|90.67|91.00|90.91|90.61|90.84|0.18| +|Exact match|84.12|83.60|84.02|84.05|83.47|83.85|0.29| + +| **FP32, 8x GPUs** | **seed 1** | **seed 2** | **seed 3** | **seed 4** | **seed 5** | **mean** | **std** | +|:-----------:|:-----:|:-----:|:-----:|:-----:|:-----:|:-----:|:-----:| +|F1 |90.74|90.82|91.09|91.16|90.89|90.94|0.18 | +|Exact match|83.82|83.64|84.03|84.23|84.03|83.95|0.23 | + + +#### Training performance results + +##### Training performance: NVIDIA DGX-1 (8x V100 16G) + +###### Pre-training training performance: single-node on 16G + +Our results were obtained by running the `scripts/run_pretraining_lamb.sh` training script in the TensorFlow 19.08-py3 NGC container on NVIDIA DGX-1 with 8x V100 16G GPUs. Performance (in sentences per second) is the steady state throughput. + + +| **GPUs** | **Sequence Length**| **Batch size / GPU: mixed precision, FP32** | **Throughput - mixed precision** | **Throughput - FP32** | **Throughput speedup (FP32 to mixed precision)** | **Weak scaling - mixed precision** | **Weak scaling - FP32** | +|:-------:|:-----:|:-------:|:-------:|:-------:|:-------------:|:------:|:------:| +| 1 | 128 | 16,8 | 91.30 | 23.90 | 3.82 | 1.00 | 1.00 | +| 4 | 128 | 16,8 | 297.70 | 86.90 | 3.43 | 3.26 | 3.64 | +| 8 | 128 | 16,8 | 578.60 | 167.80 | 3.45 | 6.34 | 7.02 | +| 1 | 512 | 4,1 | 20.00 | 4.00 | 5.00 | 1.00 | 1.00 | +| 4 | 512 | 4,1 | 66.80 | 13.50 | 4.95 | 3.34 | 3.38 | +| 8 | 512 | 4,1 | 129.50 | 26.30 | 4.92 | 6.48 | 6.58 | + +Note: The respective values for FP32 runs that use a batch size of 16, 4 in sequence lengths 128 and 512 respectively are not available due to out of memory errors that arise. + +###### Pre-training training performance: multi-node on 16G + +Our results were obtained by running the `run.sub` training script in the TensorFlow 19.08-py3 NGC container using multiple NVIDIA DGX-1 with 8x V100 16G GPUs. Performance (in sentences per second) is the steady state throughput. + +| **Nodes** | **Sequence Length**| **Batch size / GPU: mixed precision, FP32** | **Throughput - mixed precision** | **Throughput - FP32** | **Throughput speedup (FP32 to mixed precision)** | **Weak scaling - mixed precision** | **Weak scaling - FP32** | +|:-------:|:-----:|:-------:|:-------:|:-------:|:-------------:|:------:|:------:| +| 1 | 128 | 16,4 | 571.877 | 109.366 | 5.229019988 | 1.00 | 1.00 | +| 4 | 128 | 16,4 | 2028.85 | 386.23 | 5.252958082 | 3.55 | 3.53 | +| 16 | 128 | 16,4 | 7299.88 | 1350.49 | 5.405356574 | 12.76 | 12.35 | +| 32 | 128 | 16,4 | 13917.37 | 2555.25 | 5.446578613 | 24.34 | 23.36 | +| 1 | 512 | 4,1 | 128.94 | 25.65 | 5.026900585 | 1.00 | 1.00 | +| 4 | 512 | 4,1 | 466 | 92.36 | 5.045474231 | 3.61 | 3.60 | +| 16 | 512 | 4,1 | 1632 | 325.22 | 5.018141566 | 12.66 | 12.68 | +| 32 | 512 | 4,1 | 3076 | 614.18 | 5.008303755 | 23.86 | 23.94 | + +Note: The respective values for FP32 runs that use a batch size of 16, 2 in sequence lengths 128 and 512 respectively are not available due to out of memory errors that arise. + +###### Fine-tuning training performance for SQuAD on 16G + +Our results were obtained by running the `scripts/run_squad.sh` training script in the TensorFlow 19.08-py3 NGC container on NVIDIA DGX-1 with 8x V100 16G GPUs. Performance (in sentences per second) is the mean throughput from 2 epochs. + +| **GPUs** | **Batch size / GPU: mixed precision, FP32** | **Throughput - mixed precision** | **Throughput - FP32** | **Throughput speedup (FP32 to mixed precision)** | **Weak scaling - FP32** | **Weak scaling - mixed precision** | +|:---:|:---:|:------:|:-----:|:----:|:----:|:----:| +| 1 | 3,2 | 17.17 | 7.35 | 2.336054422 | 1.00 | 1.00 | +| 4 | 3,2 | 50.68 | 26.38 | 1.921152388 | 3.59 | 2.95 | +| 8 | 3,2 | 89.98 | 50.17 | 1.793502093 | 6.83 | 5.24 | + +Note: The respective values for FP32 runs that use a batch size of 3 are not available due to out of memory errors that arise. Batch size of 3 is only available on using FP16. + +To achieve these same results, follow the [Quick Start Guide](#quick-start-guide) outlined above. + +##### Training performance: NVIDIA DGX-1 (8x V100 32G) + +###### Pre-training training performance: single-node on 32G + +Our results were obtained by running the `scripts/run_pretraining_lamb.sh` training script in the TensorFlow 19.08-py3 NGC container on NVIDIA DGX-1 with 8x V100 32G GPUs. Performance (in sentences per second) is the steady state throughput. + +| **GPUs** | **Sequence Length**| **Batch size / GPU: mixed precision, FP32** | **Throughput - mixed precision** | **Throughput - FP32** | **Throughput speedup (FP32 to mixed precision)** | **Weak scaling - mixed precision** | **Weak scaling - FP32** | +|:-------:|:-----:|:-------:|:-------:|:-------:|:-------------:|:------:|:------:| +| 1 | 128 | 48,32 | 140.30 | 34.30 | 4.09 | 1.00 | 1.00 | +| 4 | 128 | 48,32 | 504.40 | 131.70 | 3.83 | 3.60 | 3.84 | +| 8 | 128 | 48,32 | 986.80 | 260.10 | 3.79 | 7.03 | 7.58 | +| 1 | 512 | 8,4 | 25.60 | 6.50 | 3.94 | 1.00 | 1.00 | +| 4 | 512 | 8,4 | 89.90 | 24.70 | 3.64 | 3.51 | 3.80 | +| 8 | 512 | 8,4 | 176.70 | 48.60 | 3.64 | 6.90 | 7.48 | + +Note: The respective values for FP32 runs that use a batch size of 48, 8 in sequence lengths 128 and 512 respectively are not available due to out of memory errors that arise. + +###### Fine-tuning training performance for SQuAD on 32G + +Our results were obtained by running the `scripts/run_squad.sh` training script in the TensorFlow 19.08-py3 NGC container on NVIDIA DGX-1 with 8x V100 32G GPUs. Performance (in sentences per second) is the mean throughput from 2 epochs. + + +| **GPUs** | **Batch size / GPU: mixed precision, FP32** | **Throughput - mixed precision** | **Throughput - FP32** | **Throughput speedup (FP32 to mixed precision)** | **Weak scaling - FP32** | **Weak scaling - mixed precision** | +|---|---|-----|------|----|----|----| +| 1 | 10,4 | 33.79 | 9 | 3.754444444 | 1.00 | 1.00 | +| 4 | 10,4 | 103.38 | 32.5 | 3.180923077 | 3.61 | 3.06 | +| 8 | 10,4 | 172.46 | 63.54 | 2.714195782 | 7.06 | 5.10 | + +Note: The respective values for FP32 runs that use a batch size of 10 are not available due to out of memory errors that arise. Batch size of 10 is only available on using FP16. + +To achieve these same results, follow the [Quick Start Guide](#quick-start-guide) outlined above. + +##### Training performance: NVIDIA DGX-2 (16x V100 32G) + +###### Pre-training training performance: single-node on DGX-2 32G + +Our results were obtained by running the `scripts/run_pretraining_lamb.sh` training script in the TensorFlow 19.08-py3 NGC container on NVIDIA DGX-2 with 16x V100 32G GPUs. Performance (in sentences per second) is the steady state throughput. + +| **GPUs** | **Sequence Length**| **Batch size / GPU: mixed precision, FP32** | **Throughput - mixed precision** | **Throughput - FP32** | **Throughput speedup (FP32 to mixed precision)** | **Weak scaling - mixed precision** | **Weak scaling - FP32** | +|:-------:|:-----:|:-------:|:-------:|:-------:|:-------------:|:------:|:------:| +| 1 | 128 | 48,32 | 143.20 | 36.30 | 3.94 | 1.00 | 1.00 | +| 4 | 128 | 48,32 | 538.30 | 141.50 | 3.80 | 3.76 | 3.90 | +| 8 | 128 | 48,32 | 1057.30 | 281.30 | 3.76 | 7.38 | 7.75 | +| 16 | 128 | 48,32 | 1990.70 | 516.80 | 3.85 | 13.90 | 14.24 | +| 1 | 512 | 8,4 | 26.90 | 6.90 | 3.90 | 1.00 | 1.00 | +| 4 | 512 | 8,4 | 96.30 | 26.40 | 3.65 | 3.58 | 3.83 | +| 8 | 512 | 8,4 | 189.00 | 52.40 | 3.61 | 7.03 | 7.59 | +| 16 | 512 | 8,4 | 354.30 | 96.50 | 3.67 | 13.17 | 13.99 | + +Note: The respective values for FP32 runs that use a batch size of 48, 8 in sequence lengths 128 and 512 respectively are not available due to out of memory errors that arise. + +###### Pre-training training performance: multi-node on DGX-2H 32G + +Our results were obtained by running the `run.sub` training script in the TensorFlow 19.08-py3 NGC container using multiple NVIDIA DGX-2 with 16x V100 32G GPUs. Performance (in sentences per second) is the steady state throughput. + + +| **Nodes** | **Sequence Length**| **Batch size / GPU: mixed precision, FP32** | **Throughput - mixed precision** | **Throughput - FP32** | **Throughput speedup (FP32 to mixed precision)** | **Weak scaling - mixed precision** | **Weak scaling - FP32** | +|:-------:|:-----:|:-------:|:-------:|:-------:|:-------------:|:------:|:------:| +| 1 | 128 | 32,32 | 1758.32 | 602.22 | 2.92 | 1.00 | 1.00 | +| 4 | 128 | 32,32 | 6379.94 | 2261.10 | 2.82 | 3.63 | 3.75 | +| 16 | 128 | 32,32 | 23846.92 | 8875.42 | 2.69 | 13.56 | 14.74 | +| 32 | 128 | 32,32 | 46191.78 | 17445.53 | 2.65 | 26.27 | 28.97 | +| 64 | 128 | 32,32 | 89195.63 | 34263.71 | 2.60 | 50.73 | 56.90 | +| 1 | 512 | 8,4 | 383.35 | 109.97 | 3.49 | 1.00 | 1.00 | +| 4 | 512 | 8,4 | 1408.75 | 400.93 | 3.51 | 3.67 | 3.65 | +| 16 | 512 | 8,4 | 5344.10 | 1559.96 | 3.43 | 13.94 | 14.19 | +| 32 | 512 | 8,4 | 10323.75 | 3061.39 | 3.37 | 26.93 | 27.84 | +| 64 | 512 | 8,4 | 19766.57 | 6029.48 | 3.28 | 51.56 | 54.83 | + + +###### Fine-tuning training performance for SQuAD on DGX-2 32G + +Our results were obtained by running the `scripts/run_squad.sh` training script in the TensorFlow 19.08-py3 NGC container on NVIDIA DGX-2 with 16x V100 32G GPUs. Performance (in sentences per second) is the mean throughput from 2 epochs. + +| **GPUs** | **Batch size / GPU: mixed precision, FP32** | **Throughput - mixed precision** | **Throughput - FP32** | **Throughput speedup (FP32 to mixed precision)** | **Weak scaling - FP32** | **Weak scaling - mixed precision** | +|---|---|------|------|----|-----|-----| +| 1 | 10,4 | 36.30 | 9.59 | 3.785192909 | 1.00 | 1.00 | +| 4 | 10,4 | 115.67 | 35.46 | 3.261985336 | 3.70 | 3.19 | +| 8 | 10,4 | 197.16 | 68.00 | 2.899411765 | 7.09 | 5.43 | +| 16 | 10,4 | 304.72 | 111.62 | 2.729976707 | 11.64 | 8.39 | + + +Note: The respective values for FP32 runs that use a batch size of 10 are not available due to out of memory errors that arise. Batch size of 10 is only available on using FP16. + +To achieve these same results, follow the [Quick Start Guide](#quick-start-guide) outlined above. + +#### Inference performance results + +##### Inference performance: NVIDIA DGX-1 (1x V100 16G) + +###### Pre-training inference performance on 16G + +Our results were obtained by running the `scripts/run_pretraining_lamb.sh` script in the TensorFlow 19.06-py3 NGC container on NVIDIA DGX-1 with 1x V100 16G GPUs. + +| **Sequence Length**| **Batch size / GPU: mixed precision, FP32** | **Throughput - mixed precision** | **Throughput - FP32** | **Throughput speedup (FP32 to mixed precision)** | +|:-----:|:-------:|:-------:|:-------:|:-------------:| +|128 |8, 8 |349.51 | 104.31 | 3.35 | + +###### Fine-tuning inference performance for SQuAD on 16G + +Our results were obtained by running the `scripts/finetune_inference_benchmark.sh` script in the TensorFlow 19.08-py3 NGC container on NVIDIA DGX-1 with 1x V100 16G GPUs. Performance numbers (throughput in sentences per second and latency in milliseconds) were averaged from 1024 iterations. Latency is computed as the time taken for a batch to process as they are fed in one after another in the model ie no pipelining. + +BERT LARGE FP16 + +| Sequence Length | Batch Size | Throughput-Average(sent/sec) | Throughput speedup (FP32 to mixed precision) | Latency-Average(ms) | Latency-90%(ms) | Latency-95%(ms) | Latency-99%(ms) | +|-----------------|------------|------------------------------|----------------------------------------------|---------------------|-----------------|-----------------|-----------------| +| 128 | 1 | 95.87 | 1.433462919 | 10.43 | 10.61 | 10.71 | 11.27 | +| 128 | 2 | 168.02 | 1.871046771 | 11.9 | 12.08 | 12.18 | 12.32 | +| 128 | 4 | 263.08 | 2.617451 | 15.2 | 14.86 | 14.95 | 15.55 | +| 128 | 8 | 379.78 | 3.414366628 | 21.07 | 20.94 | 21.03 | 21.49 | +| 384 | 1 | 67.52 | 2.274932615 | 14.81 | 14.93 | 15.05 | 15.38 | +| 384 | 2 | 93.8 | 2.929419113 | 21.32 | 20.75 | 20.83 | 21.43 | +| 384 | 4 | 118.97 | 3.397201599 | 33.62 | 33.17 | 33.37 | 33.85 | +| 384 | 8 | 138.43 | 3.838879645 | 57.79 | 57 | 57.38 | 58.19 | + +BERT LARGE FP32 + +| Sequence Length | Batch Size | Throughput-Average(sent/sec) | Latency-Average(ms) | Latency-90%(ms) | Latency-95%(ms) | Latency-99%(ms) | +|-----------------|------------|------------------------------|---------------------|-----------------|-----------------|-----------------| +| 128 | 1 | 66.88 | 14.95 | 14.96 | 15.41 | 18.02 | +| 128 | 2 | 89.8 | 22.27 | 22.46 | 22.53 | 22.84 | +| 128 | 4 | 100.51 | 39.8 | 39.91 | 40.06 | 41.04 | +| 128 | 8 | 111.23 | 71.92 | 72.42 | 72.58 | 73.63 | +| 384 | 1 | 29.68 | 33.7 | 33.85 | 33.91 | 34.62 | +| 384 | 2 | 32.02 | 62.47 | 63.06 | 63.28 | 63.66 | +| 384 | 4 | 35.02 | 114.21 | 114.69 | 114.82 | 115.85 | +| 384 | 8 | 36.06 | 221.86 | 222.7 | 223.03 | 223.53 | + +BERT BASE FP16 + +| Sequence Length | Batch Size | Throughput-Average(sent/sec) | Throughput speedup (FP32 to mixed precision) | Latency-Average(ms) | Latency-90%(ms) | Latency-95%(ms) | Latency-99%(ms) | +|-----------------|------------|------------------------------|----------------------------------------------|---------------------|-----------------|-----------------|-----------------| +| 128 | 1 | 204.33 | 1.459187317 | 4.89 | 5.14 | 5.32 | 5.54 | +| 128 | 2 | 375.19 | 1.779501043 | 5.33 | 5.47 | 5.58 | 5.87 | +| 128 | 4 | 606.98 | 2.198645271 | 6.59 | 6.49 | 6.55 | 6.83 | +| 128 | 8 | 902.6 | 2.69023278 | 8.86 | 8.62 | 8.72 | 9.22 | +| 384 | 1 | 154.33 | 1.990070922 | 6.48 | 6.59 | 6.65 | 7.04 | +| 384 | 2 | 225.7 | 2.386087324 | 8.86 | 8.45 | 8.53 | 9.16 | +| 384 | 4 | 317.93 | 3.044431677 | 12.58 | 12.34 | 12.39 | 13.01 | +| 384 | 8 | 393.44 | 3.672547372 | 20.33 | 20.06 | 20.38 | 21.38 | + +BERT BASE FP32 + +| Sequence Length | Batch Size | Throughput-Average(sent/sec) | Latency-Average(ms) | Latency-90%(ms) | Latency-95%(ms) | Latency-99%(ms) | +|-----------------|------------|------------------------------|---------------------|-----------------|-----------------|-----------------| +| 128 | 1 | 140.03 | 7.14 | 7.6 | 7.78 | 7.97 | +| 128 | 2 | 210.84 | 9.49 | 9.59 | 9.65 | 10.57 | +| 128 | 4 | 276.07 | 14.49 | 14.61 | 14.71 | 15.16 | +| 128 | 8 | 335.51 | 23.84 | 23.79 | 23.89 | 24.94 | +| 384 | 1 | 77.55 | 12.89 | 13.01 | 13.05 | 14.26 | +| 384 | 2 | 94.59 | 21.14 | 21.14 | 21.23 | 21.86 | +| 384 | 4 | 104.43 | 38.3 | 38.38 | 38.45 | 39.15 | +| 384 | 8 | 107.13 | 74.68 | 75.05 | 75.19 | 76.2 | + + +To achieve these same results, follow the [Quick Start Guide](#quick-start-guide) outlined above. + +##### Inference performance: NVIDIA DGX-1 (1x V100 32G) + +###### Pre-training inference performance on 32G + +Our results were obtained by running the `scripts/run_pretraining_lamb.sh` script in the TensorFlow 19.08-py3 NGC container on NVIDIA DGX-1 with 1x V100 32G GPUs. + +| **Sequence Length**| **Batch size / GPU: mixed precision, FP32** | **Throughput - mixed precision** | **Throughput - FP32** | **Throughput speedup (FP32 to mixed precision)** | +|:-----:|:-------:|:-------:|:-------:|:-------------:| +|128 |8, 8 |345.50 | 101.84 | 3.39 | + +###### Fine-tuning inference performance for SQuAD on 32G + +Our results were obtained by running the `scripts/finetune_inference_benchmark.sh` training script in the TensorFlow 19.08-py3 NGC container on NVIDIA DGX-1 with 1x V100 32G GPUs. Performance numbers (throughput in sentences per second and latency in milliseconds) were averaged from 1024 iterations. Latency is computed as the time taken for a batch to process as they are fed in one after another in the model ie no pipelining. + +BERT LARGE FP16 + +| Sequence Length | Batch Size | Throughput-Average(sent/sec) | Throughput speedup (FP32 to mixed precision) | Latency-Average(ms) | Latency-90%(ms) | Latency-95%(ms) | Latency-99%(ms) | +|-----------------|------------|------------------------------|----------------------------------------------|---------------------|-----------------|-----------------|-----------------| +| 128 | 1 | 87.75 | 1.352913969 | 11.4 | 11.46 | 18.77 | 19.06 | +| 128 | 2 | 159.87 | 1.833161335 | 12.51 | 12.69 | 12.79 | 12.98 | +| 128 | 4 | 254.65 | 2.622014003 | 15.71 | 15.49 | 15.59 | 16.03 | +| 128 | 8 | 365.51 | 3.377783939 | 21.89 | 21.72 | 21.94 | 23.79 | +| 384 | 1 | 63.11 | 2.153924915 | 15.84 | 17.3 | 19.22 | 19.37 | +| 384 | 2 | 89.61 | 2.884132604 | 22.32 | 21.83 | 21.96 | 23.8 | +| 384 | 4 | 114.9 | 3.395390071 | 34.81 | 34.33 | 34.47 | 35.15 | +| 384 | 8 | 132.79 | 3.814708417 | 60.25 | 59.4 | 59.77 | 60.7 | + +BERT LARGE FP32 + +| Sequence Length | Batch Size | Throughput-Average(sent/sec) | Latency-Average(ms) | Latency-90%(ms) | Latency-95%(ms) | Latency-99%(ms) | +|-----------------|------------|------------------------------|---------------------|-----------------|-----------------|-----------------| +| 128 | 1 | 64.86 | 15.42 | 16.32 | 17.55 | 20.89 | +| 128 | 2 | 87.21 | 22.93 | 23.06 | 24.17 | 31.93 | +| 128 | 4 | 97.12 | 41.19 | 41.38 | 41.5 | 44.13 | +| 128 | 8 | 108.21 | 73.93 | 74.34 | 74.48 | 74.77 | +| 384 | 1 | 29.3 | 34.13 | 34.21 | 34.25 | 34.76 | +| 384 | 2 | 31.07 | 64.38 | 64.83 | 64.95 | 65.42 | +| 384 | 4 | 33.84 | 118.22 | 119.01 | 119.57 | 120.06 | +| 384 | 8 | 34.81 | 229.84 | 230.72 | 231.22 | 232.96 | + +BERT BASE FP16 + +| Sequence Length | Batch Size | Throughput-Average(sent/sec) | Throughput speedup (FP32 to mixed precision) | Latency-Average(ms) | Latency-90%(ms) | Latency-95%(ms) | Latency-99%(ms) | +|-----------------|------------|------------------------------|----------------------------------------------|---------------------|-----------------|-----------------|-----------------| +| 128 | 1 | 198.72 | 1.393352966 | 5.03 | 5.3 | 5.47 | 5.69 | +| 128 | 2 | 338.44 | 1.611158717 | 5.91 | 6.04 | 9.77 | 9.94 | +| 128 | 4 | 599.62 | 2.24804109 | 6.67 | 6.6 | 6.66 | 6.83 | +| 128 | 8 | 858.56 | 2.63370042 | 9.32 | 10.01 | 10.04 | 10.39 | +| 384 | 1 | 150.28 | 1.948146228 | 6.65 | 6.76 | 6.82 | 7.21 | +| 384 | 2 | 200.68 | 2.200197347 | 9.97 | 9.88 | 9.94 | 10.08 | +| 384 | 4 | 305.72 | 3.01707293 | 13.08 | 12.86 | 12.97 | 13.71 | +| 384 | 8 | 373.64 | 3.61249154 | 21.41 | 21.98 | 22.03 | 22.61 | + +BERT BASE FP32 + +| Sequence Length | Batch Size | Throughput-Average(sent/sec) | Latency-Average(ms) | Latency-90%(ms) | Latency-95%(ms) | Latency-99%(ms) | +|-----------------|------------|------------------------------|---------------------|-----------------|-----------------|-----------------| +| 128 | 1 | 142.62 | 7.01 | 7.07 | 7.44 | 9.23 | +| 128 | 2 | 210.06 | 9.52 | 9.63 | 9.69 | 10.22 | +| 128 | 4 | 266.73 | 15 | 15.77 | 15.91 | 16.79 | +| 128 | 8 | 325.99 | 24.54 | 24.52 | 24.6 | 25 | +| 384 | 1 | 77.14 | 12.96 | 13.01 | 13.03 | 13.67 | +| 384 | 2 | 91.21 | 21.93 | 21.93 | 21.99 | 22.31 | +| 384 | 4 | 101.33 | 39.47 | 39.69 | 39.82 | 40.88 | +| 384 | 8 | 103.43 | 77.34 | 77.76 | 77.9 | 78.45 | + + +To achieve these same results, follow the [Quick Start Guide](#quick-start-guide) outlined above. + +##### Inference performance: NVIDIA DGX-2 (1x V100 32G) + +###### Pre-training inference performance on DGX-2 32G + +Our results were obtained by running the `scripts/run_pretraining_lamb.sh` script in the TensorFlow 19.08-py3 NGC container on NVIDIA DGX-2 with 1x V100 32G GPUs. + +| **Sequence Length**| **Batch size / GPU: mixed precision, FP32** | **Throughput - mixed precision** | **Throughput - FP32** | **Throughput speedup (FP32 to mixed precision)** | +|:-----:|:-------:|:-------:|:-------:|:-------------:| +|128 |8, 8 |366.24 | 107.88 | 3.39 | + +###### Fine-tuning inference performance for SQuAD on DGX-2 32G + +Our results were obtained by running the `scripts/finetune_inference_benchmark.sh` training script in the TensorFlow 19.08-py3 NGC container on NVIDIA DGX-2 with 1x V100 32G GPUs. Performance numbers (throughput in sentences per second and latency in milliseconds) were averaged from 1024 iterations. Latency is computed as the time taken for a batch to process as they are fed in one after another in the model ie no pipelining. + +BERT LARGE FP16 + +| Sequence Length | Batch Size | Throughput-Average(sent/sec) | Throughput speedup (FP32 to mixed precision) | Latency-Average(ms) | Latency-90%(ms) | Latency-95%(ms) | Latency-99%(ms) | +|-----------------|------------|------------------------------|----------------------------------------------|---------------------|-----------------|-----------------|-----------------| +| 128 | 1 | 96.22 | 1.371045882 | 10.39 | 10.78 | 10.9 | 11.43 | +| 128 | 2 | 171.66 | 1.835935829 | 11.65 | 11.86 | 12.04 | 12.45 | +| 128 | 4 | 262.89 | 2.566032211 | 15.22 | 15.13 | 15.24 | 15.91 | +| 128 | 8 | 394.23 | 3.441253492 | 20.29 | 20.22 | 20.6 | 22.19 | +| 384 | 1 | 69.69 | 2.278195489 | 14.35 | 14.39 | 14.58 | 15.68 | +| 384 | 2 | 96.35 | 2.909118357 | 20.76 | 20.25 | 20.32 | 21.54 | +| 384 | 4 | 124.06 | 3.42612538 | 32.24 | 31.87 | 32.14 | 33.02 | +| 384 | 8 | 144.28 | 3.876410532 | 55.45 | 54.77 | 55.16 | 55.93 | + +BERT LARGE FP32 + +| Sequence Length | Batch Size | Throughput-Average(sent/sec) | Latency-Average(ms) | Latency-90%(ms) | Latency-95%(ms) | Latency-99%(ms) | +|-----------------|------------|------------------------------|---------------------|-----------------|-----------------|-----------------| +| 128 | 1 | 70.18 | 14.25 | 14.7 | 14.88 | 15.35 | +| 128 | 2 | 93.5 | 21.39 | 21.83 | 22.04 | 22.85 | +| 128 | 4 | 102.45 | 39.04 | 39.28 | 39.42 | 40.5 | +| 128 | 8 | 114.56 | 69.83 | 70.5 | 70.74 | 72.78 | +| 384 | 1 | 30.59 | 32.69 | 33.14 | 33.32 | 33.86 | +| 384 | 2 | 33.12 | 60.38 | 60.91 | 61.12 | 61.67 | +| 384 | 4 | 36.21 | 110.46 | 111.1 | 111.26 | 112.15 | +| 384 | 8 | 37.22 | 214.95 | 215.69 | 216.13 | 217.96 | + +BERT BASE FP16 + +| Sequence Length | Batch Size | Throughput-Average(sent/sec) | Throughput speedup (FP32 to mixed precision) | Latency-Average(ms) | Latency-90%(ms) | Latency-95%(ms) | Latency-99%(ms) | +|-----------------|------------|------------------------------|----------------------------------------------|---------------------|-----------------|-----------------|-----------------| +| 128 | 1 | 207.01 | 1.455050257 | 4.83 | 5.23 | 5.38 | 5.59 | +| 128 | 2 | 405.92 | 1.808429119 | 4.93 | 4.99 | 5.04 | 5.2 | +| 128 | 4 | 646.8 | 2.258695349 | 6.18 | 6.06 | 6.14 | 6.55 | +| 128 | 8 | 909.41 | 2.616781285 | 8.8 | 8.86 | 8.96 | 9.52 | +| 384 | 1 | 153.97 | 1.959653812 | 6.49 | 6.88 | 7.01 | 7.2 | +| 384 | 2 | 229.46 | 2.366298855 | 8.72 | 8.57 | 8.67 | 8.97 | +| 384 | 4 | 333.2 | 3.078913325 | 12 | 11.74 | 11.85 | 12.86 | +| 384 | 8 | 403.02 | 3.646579805 | 19.85 | 19.83 | 20 | 21.11 | + +BERT BASE FP32 + +| Sequence Length | Batch Size | Throughput-Average(sent/sec) | Latency-Average(ms) | Latency-90%(ms) | Latency-95%(ms) | Latency-99%(ms) | +|-----------------|------------|------------------------------|---------------------|-----------------|-----------------|-----------------| +| 128 | 1 | 142.27 | 7.03 | 7.39 | 7.45 | 11.7 | +| 128 | 2 | 224.46 | 8.91 | 9 | 9.08 | 9.66 | +| 128 | 4 | 286.36 | 13.97 | 14.46 | 14.52 | 14.82 | +| 128 | 8 | 347.53 | 23.02 | 23.23 | 23.4 | 24.12 | +| 384 | 1 | 78.57 | 12.73 | 13.01 | 13.1 | 14.06 | +| 384 | 2 | 96.97 | 20.62 | 21 | 21.15 | 21.82 | +| 384 | 4 | 108.22 | 36.96 | 37.05 | 37.18 | 38.12 | +| 384 | 8 | 110.52 | 72.38 | 73.06 | 73.32 | 74.64 | + + +##### Inference performance: NVIDIA Tesla T4 (1x T4 16G) + +###### Fine-tuning inference performance for SQuAD on Tesla T4 16G + +Our results were obtained by running the `scripts/finetune_inference_benchmark.sh` training script in the TensorFlow 19.08-py3 NGC container on NVIDIA Tesla T4 with 1x T4 16G GPUs. Performance numbers (throughput in sentences per second and latency in milliseconds) were averaged from 1024 iterations. Latency is computed as the time taken for a batch to process as they are fed in one after another in the model ie no pipelining. + +BERT LARGE FP16 + +| Sequence Length | Batch Size | Throughput-Average(sent/sec) | Throughput speedup (FP32 to mixed precision) | Latency-Average(ms) | Latency-90%(ms) | Latency-95%(ms) | Latency-99%(ms) | +|-----------------|------------|------------------------------|----------------------------------------------|---------------------|-----------------|-----------------|-----------------| +| 128 | 1 | 54.53 | 1.552234557 | 18.34 | 19.09 | 19.28 | 21.74 | +| 128 | 2 | 95.59 | 2.521498285 | 20.92 | 21.86 | 22.61 | 23.33 | +| 128 | 4 | 133.2 | 3.434760186 | 30.03 | 30.32 | 30.43 | 31.06 | +| 128 | 8 | 168.85 | 4.352926012 | 47.38 | 48.21 | 48.56 | 49.25 | +| 384 | 1 | 33.58 | 2.87008547 | 29.78 | 30.3 | 30.46 | 31.69 | +| 384 | 2 | 41.31 | 3.576623377 | 48.41 | 49.03 | 49.26 | 50.04 | +| 384 | 4 | 47.08 | 3.94635373 | 84.96 | 86.88 | 87.38 | 88.3 | +| 384 | 8 | 50.08 | 4.254885302 | 159.76 | 162.37 | 163.23 | 165.79 | + +BERT LARGE FP32 + +| Sequence Length | Batch Size | Throughput-Average(sent/sec) | Latency-Average(ms) | Latency-90%(ms) | Latency-95%(ms) | Latency-99%(ms) | +|-----------------|------------|------------------------------|---------------------|-----------------|-----------------|-----------------| +| 128 | 1 | 35.13 | 28.46 | 29.89 | 30.12 | 30.6 | +| 128 | 2 | 37.91 | 52.76 | 54.01 | 54.29 | 54.84 | +| 128 | 4 | 38.78 | 103.14 | 105.39 | 106.05 | 107.4 | +| 128 | 8 | 38.79 | 206.22 | 209.63 | 210.2 | 211.5 | +| 384 | 1 | 11.7 | 85.5 | 87.18 | 87.43 | 88 | +| 384 | 2 | 11.55 | 173.19 | 176.13 | 177.02 | 178.4 | +| 384 | 4 | 11.93 | 335.41 | 340.26 | 341.76 | 343.54 | +| 384 | 8 | 11.77 | 679.77 | 686.01 | 686.79 | 689.24 | + +BERT BASE FP16 + +| Sequence Length | Batch Size | Throughput-Average(sent/sec) | Throughput speedup (FP32 to mixed precision) | Latency-Average(ms) | Latency-90%(ms) | Latency-95%(ms) | Latency-99%(ms) | +|-----------------|------------|------------------------------|----------------------------------------------|---------------------|-----------------|-----------------|-----------------| +| 12.71 | 13.1 | 13.22 | 1.552234557 | 18.34 | 19.09 | 19.28 | 21.74 | +| 11.16 | 12.85 | 12.97 | 2.521498285 | 20.92 | 21.86 | 22.61 | 23.33 | +| 11.82 | 11.9 | 13.21 | 3.434760186 | 30.03 | 30.32 | 30.43 | 31.06 | +| 17.88 | 18.08 | 18.82 | 4.352926012 | 47.38 | 48.21 | 48.56 | 49.25 | +| 11.83 | 12.95 | 15.44 | 2.87008547 | 29.78 | 30.3 | 30.46 | 31.69 | +| 16.91 | 17.08 | 19.38 | 3.576623377 | 48.41 | 49.03 | 49.26 | 50.04 | +| 28.89 | 29.23 | 30.84 | 3.94635373 | 84.96 | 86.88 | 87.38 | 88.3 | +| 54.58 | 55.19 | 56.31 | 4.254885302 | 159.76 | 162.37 | 163.23 | 165.79 | + +BERT BASE FP32 + +| Sequence Length | Batch Size | Throughput-Average(sent/sec) | Latency-Average(ms) | Latency-90%(ms) | Latency-95%(ms) | Latency-99%(ms) | +|-----------------|------------|------------------------------|---------------------|-----------------|-----------------|-----------------| +| 128 | 1 | 64.15 | 15.59 | 19.77 | 21.03 | 21.82 | +| 128 | 2 | 110.69 | 18.07 | 18.92 | 20.77 | 21.6 | +| 128 | 4 | 125.8 | 31.8 | 32.82 | 33.11 | 33.93 | +| 128 | 8 | 127.55 | 62.72 | 63.9 | 64.28 | 65.25 | +| 384 | 1 | 35.46 | 28.2 | 28.83 | 28.95 | 29.43 | +| 384 | 2 | 37.15 | 53.83 | 54.75 | 55.08 | 56.01 | +| 384 | 4 | 36.86 | 108.53 | 110.57 | 111.16 | 112.48 | +| 384 | 8 | 36.1 | 221.61 | 225.94 | 226.94 | 228.58 | + + +To achieve these same results, follow the [Quick Start Guide](#quick-start-guide) outlined above. + +## Release notes + +### Changelog + +November 2019 +- Pre-training and Finetuning on BioMedical tasks and corpus + +October 2019 +- Disabling Grappler Optimizations for improved performance + +September 2019 +- Pre-training using LAMB +- Multi Node support +- Fine Tuning support for GLUE (CoLA, MNLI, MRPC) + +July 2019 +- Results obtained using 19.06 +- Inference Studies using TensorRT Inference Server + +March 2019 +- Initial release + +### Known issues + + +- There is a known performance regression with the 19.08 release on Tesla V100 boards with 16 GB memory, smaller batch sizes may be a better choice for this model on these GPUs with the 19.08 release. 32 GB GPUs are not affected. diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/__init__.py b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/__init__.py new file mode 100644 index 0000000..effb57b --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/__init__.py @@ -0,0 +1,15 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/README.md b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/README.md new file mode 100644 index 0000000..f29dd7a --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/README.md @@ -0,0 +1,567 @@ +# BioBert For TensorFlow + +This folder provides a script and recipe to train BERT for TensorFlow to achieve state-of-the-art accuracy on *biomedical text-mining* and is tested and maintained by NVIDIA. + +## Table Of Contents + +* [Model overview](#model-overview) +* [Quick Start Guide](#quick-start-guide) +* [Advanced](#advanced) + * [Scripts and sample code](#scripts-and-sample-code) + * [Parameters](#parameters) + * [Command-line options](#command-line-options) + * [Getting the data](#getting-the-data) + * [Dataset guidelines](#dataset-guidelines) + * [Multi-dataset](#multi-dataset) + * [Training process](#training-process) + * [Pre-training](#pre-training) + * [Fine tuning](#fine-tuning) + * [Multi-node](#multi-node) + * [Inference process](#inference-process) +* [Performance](#performance) + * [Benchmarking](#benchmarking) + * [Training performance benchmark](#training-performance-benchmark) + * [Inference performance benchmark](#inference-performance-benchmark) +* [Results](#results) + * [Training accuracy results](#training-accuracy-results) + * [Pre-training accuracy](#pre-training-accuracy) + * [Fine-tuning accuracy](#fine-tuning-accuracy) + * [Fine-tuning accuracy for NER Chem](#fine-tuning-accuracy-for-ner-chem) + * [Training stability test](#training-stability-test) + * [Fine-tuning stability test](#fine-tuning-stability-test) + * [Training performance results](#training-performance-results) + * [Training performance: NVIDIA DGX-1 (8x V100 16G)](#training-performance-nvidia-dgx-1-8x-v100-16g) + * [Pre-training training performance: multi-node on 16G](#pre-training-training-performance-multi-node-on-16g) + * [Fine-tuning training performance for NER on 16G](#fine-tuning-training-performance-for-ner-on-16g) + * [Training performance: NVIDIA DGX-1 (8x V100 32G)](#training-performance-nvidia-dgx-1-8x-v100-32g) + * [Fine-tuning training performance for NER on 32G](#fine-tuning-training-performance-for-ner-on-32g) + * [Training performance: NVIDIA DGX-2 (16x V100 32G)](#training-performance-nvidia-dgx-2-16x-v100-32g) + * [Pre-training training performance: multi-node on DGX-2 32G](#pre-training-training-performance-multi-node-on-dgx-2-32g) + * [Fine-tuning training performance for NER on DGX-2 32G](#fine-tuning-training-performance-for-ner-on-dgx-2-32g) +* [Release notes](#release-notes) + * [Changelog](#changelog) + * [Known issues](#known-issues) + + + +## Model overview + +In the original [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805) paper, pre-training is done on [Wikipedia](https://dumps.wikimedia.org/) and [Books Corpus](http://yknzhu.wixsite.com/mbweb), with state-of-the-art results demonstrated on [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/) (Stanford Question Answering Dataset) benchmark. + +Meanwhile, many works, including [BioBERT](https://arxiv.org/pdf/1901.08746.pdf), [SciBERT](https://arxiv.org/pdf/1903.10676.pdf), [NCBI-BERT](https://arxiv.org/pdf/1906.05474.pdf), [ClinicalBERT (MIT)](https://arxiv.org/pdf/1904.03323.pdf), [ClinicalBERT (NYU, Princeton)](https://arxiv.org/pdf/1904.05342.pdf), and others at [BioNLP’19 workshop](https://aclweb.org/aclwiki/BioNLP_Workshop), show that additional pre-training of BERT on large biomedical text corpus such as [PubMed](https://www.ncbi.nlm.nih.gov/pubmed/) results in better performance in biomedical text-mining tasks. + +This repository provides scripts and recipe to adopt the [NVIDIA BERT code-base](https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow/LanguageModeling/BERT) to achieve state-of-the-art results in the following biomedical text-mining benchmark tasks: + +- [BC5CDR-disease](https://biocreative.bioinformatics.udel.edu/tasks/biocreative-v/track-3-cdr/) A Named-Entity-Recognition task to recognize diseases mentioned in a collection of 1500 PubMed titles and abstracts ([Li et al., 2016](https://academic.oup.com/database/article/doi/10.1093/database/baw068/2630414)) + +- [BC5CDR-chemical](https://biocreative.bioinformatics.udel.edu/tasks/biocreative-v/track-3-cdr/) A Named-Entity-Recognition task to recognize chemicals mentioned in a collection of 1500 PubMed titles and abstracts ([Li et al., 2016](https://academic.oup.com/database/article/doi/10.1093/database/baw068/2630414)) + +- [ChemProt](https://biocreative.bioinformatics.udel.edu/news/corpora/) A Relation-Extraction task to determine chemical-protein interactions in a collection of 1820 PubMed abstracts ([Krallinger et al., 2017](https://biocreative.bioinformatics.udel.edu/media/store/files/2017/ProceedingsBCVI_v2.pdf?page=141)) + + +## Quick Start Guide + +To pretrain or fine tune your model for BioMedical tasks using mixed precision with Tensor Cores or using FP32, perform the following steps using the default parameters of the BERT model. + +1. Clone the repository. + +```bash +git clone https://github.com/NVIDIA/DeepLearningExamples +cd DeepLearningExamples/TensorFlow/LanguageModeling/BERT +``` + +2. Build the BERT TensorFlow NGC container. + +```bash +bash scripts/docker/build.sh +``` + +3. Download and preprocess the PubMed dataset. + +To download and preprocess pre-training data as well as the required vocab files, run the following script: + + +```bash +bash biobert/scripts/biobert_data_download.sh +``` + +Datasets for finetuning can be obtained from this [repository](https://github.com/ncbi-nlp/BLUE_Benchmark/releases/tag/0.1) + +Place them in `/workspace/bert/data/biobert/` to be automatically picked up by our scripts. + +4. Start an interactive session in the NGC container to run training/inference. + +After you build the container image and download the data, you can start an interactive CLI session as follows: + +```bash +bash scripts/docker/launch.sh +``` + +5. Download the pre-trained checkpoint, vocabulary, and configuration files. + +We have uploaded checkpoints for fine tuning and pre-training on BioMedical Corpus’s on the NGC Model Registry. You can download them directly from the [NGC model catalog](https://ngc.nvidia.com/catalog/models). + +Place our `BioBERT checkpoints` in the `results/` to easily access it in your scripts. + +6. Start pre-training. + +From within the container, you can use the following script to run the 1st phase of the pre-training using cased vocabulary: + +```bash +bash biobert/scripts/run_pretraining-pubmed_base_phase_1.sh +``` + +For the 2nd phase of the pre-training, issue: + +```bash +bash biobert/scripts/run_pretraining-pubmed_base_phase_2.sh +``` + + +Refer to (MultiNode Section)[multi-node] for details on utilizing multiple nodes for faster pretraining. + +6. Start fine tuning. + +The above pretrained BERT representations can be fine tuned with just one additional output layer for a state-of-the-art biomedical text-mining system. +From within the container, you can use the following script to run fine-training for NER. + +Note: The scripts assume you are running on 16 V100 32GB GPUs. If you are running on GPU having less than 32GB memory or fewer GPUs, batch size, learning rate and number of GPUs needs to be adjusted. + +For NER on disease entities: + +```bash +bash biobert/scripts/ner_bc5cdr-disease.sh +``` + +For NER on chemical entities: + +```bash +bash biobert/scripts/ner_bc5cdr-chem.sh +``` + +For relation extraction, issue: + +``` +bash biobert/scripts/rel_chemprot.sh +``` + +8. Start validation/evaluation. + +The `biobert/scripts/run_biobert_finetuning_inference.sh` script runs inference on a checkpoint fine tuned for a specific task and evaluates the validity of predictions on the basis of F1, precision and recall scores. + +```bash +bash biobert/scripts/run_biobert_finetuning_inference.sh +``` + +For FP16 inference for NER on BC5DR Chemical task with XLA using a DGX-2 V100 32G, run: +```bash +bash biobert/scripts/run_biobert_finetuning_inference.sh ner_bc5cdr-chem /results/model.ckpt base false fp16 true 16 +``` + +Tasks `ner_bc5cdr-chem`, `ner_bc5cdr-disease` and `rel_chemprot` are currently supported. + +## Advanced + +The following sections provide greater details of the dataset, running training and inference, and the training results. + +### Scripts and sample code + +In addition to BERT TensorFlow files, the most important files added for NER and RE fine tuning tasks are: +* `run_ner.py` - Serves as an entry point for NER training. +* `run_re.py` - Serves as an entry point for RE training. + +The `biobert/scripts/` folder encapsulates all the one-click scripts required for running various functionalities supported such as: +* `ner_bc5cdr-chem.sh` - Runs NER training and inference on the BC5CDR Chemical dataset using the `run_ner.py` file. +* `ner_bc5cdr-disease.sh` - Runs NER training and inference on the BC5CDR Disease dataset using the `run_ner.py` file. +* `rel_chemprot.sh` - Runs RE training and inference on the ChemProt dataset using the `run_re.py` file. +* `run_pretraining_pubmed_base_phase_*.sh` - Runs pre-training with LAMB optimizer using the `run_pretraining.py` file in two phases. Phase 1 does training with sequence length = 128. In phase 2, the remaining 10% of the training is done with sequence length = 512. +* `biobert_data_download.sh` - Downloads the PubMed dataset and Vocab files using files in the `data/` folder. +* `run_biobert_finetuning_inference.sh` - Runs task specific inference using a fine tuned checkpoint. + + +### Parameters + +Aside from the options to set hyperparameters, some relevant options to control the behaviour of the `run_ner.py` and `run_re.py` scripts are: + +``` + --bert_config_file: The config json file corresponding to the pre-trained BERT model. This specifies the model architecture. +--vocab_file: The vocabulary file that the BERT model was trained on. + --output_dir: The output directory where the model checkpoints will be written. + --[no]do_eval: Whether to run evaluation on the dev set. (default: 'false') + --[no]do_predict: Whether to run evaluation on the test set. (default: 'false') + --[no]do_train: Whether to run training. (default: 'false') + --learning_rate: The initial learning rate for Adam.(default: '5e-06')(a number) + --max_seq_length: The maximum total input sequence length after WordPiece tokenization. Sequences longer than this will be truncated, and sequences shorter than this will be padded.(default: '384')(an integer) + --predict_batch_size: Total batch size for predictions.(default: '8')(an integer) + --train_batch_size: Total batch size for training (default: '8')(an integer) + --[no]use_fp16: Whether to enable AMP ops.(default: 'false') + --[no]use_xla: Whether to enable XLA JIT compilation.(default: 'false') +--init_checkpoint: Initial checkpoint (usually from a pre-trained BERT model). +--num_train_epochs: Total number of training epochs to perform.(default: '3.0')(a number) + +``` + +Note: When initializing from a checkpoint using `--init_checkpoint` and a corpus of your choice, keep in mind that `bert_config_file` and `vocab_file` should remain unchanged. + +### Command-line options + +To see the full list of available options and their descriptions, use the `-h` or `--help` command-line option with the Python file, for example: + +```bash +python run_ner.py --help +python run_re.py --help +``` +### Getting the data + +For pre-training BERT, we use the PubMed Dataset. For PubMed, we extract the xml files which are structured as a document level corpus rather than a shuffled sentence level corpus because it is critical to extract long contiguous sentences. + +The next step is to run `create_pretraining_data.py` with the document level corpus as input, which generates input data and labels for the masked language modeling and next sentence prediction tasks. Pre-training can also be performed on any corpus of your choice. The collection of data generation scripts are intended to be modular to allow modifications for additional preprocessing steps or to use additional data. They can hence easily be modified for an arbitrary corpus. + +The preparation of an individual pre-training dataset is described in the `create_biobert_datasets_from_start.sh ` script found in the `data/` folder. The component steps to prepare the datasets are as follows: + +1. Data download and extract - the dataset is downloaded and extracted. +2. Clean and format - document tags, etc. are removed from the dataset. The end result of this step is a `{dataset_name_one_article_per_line}.txt` file that contains the entire corpus. Each line in the text file contains an entire document from the corpus. One file per dataset is created in the `formatted_one_article_per_line` folder. +3. Sharding - the sentence segmented corpus file is split into a number of smaller text documents. The sharding is configured so that a document will not be split between two shards. Sentence segmentation is performed at this time using NLTK. +4. TFRecord file creation - each text file shard is processed by the `create_pretraining_data.py` script to produce a corresponding TFRecord file. The script generates input data and labels for masked language modeling and sentence prediction tasks for the input text shard. + + +For fine tuning BioBERT for the task of Named Entity Recognition and Relation Extraction Tasks, we use BC5CDR and Chemprot Datasets. BC5CDR corpus consists of 1500 PubMed articles with 4409 annotated chemicals, 5818 diseases and 3116 chemical-disease interactions. +ChemProt corpus consists of text exhaustively annotated by hand with mentions of chemical compounds/drugs and genes/proteins, as well as 22 different types of compound-protein relations focussing on 5 important relation classes. It was preprocessed following [Lim and Kang](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6014134/) guidelines. + +#### Dataset guidelines + +The procedure to prepare a text corpus for pre-training is described in the previous section. This section provides additional insight into how exactly raw text is processed so that it is ready for pre-training. + +First, raw text is tokenized using [WordPiece tokenization](https://arxiv.org/pdf/1609.08144.pdf). A [CLS] token is inserted at the start of every sequence, and the two sentences in the sequence are separated by a [SEP] token. + +Note: BERT pre-training looks at pairs of sentences at a time. A sentence embedding token [A] is added to the first sentence and token [B] to the next. + +BERT pre-training optimizes for two unsupervised classification tasks. The first is Masked Language Modelling (Masked LM). One training instance of Masked LM is a single modified sentence. Each token in the sentence has a 15% chance of being replaced by a [MASK] token. The chosen token is replaced with [MASK] 80% of the time, 10% with another random token and the remaining 10% with the same token. The task is then to predict the original token. + +The second task is next sentence prediction. One training instance of BERT pre-training is two sentences (a sentence pair). A sentence pair may be constructed by simply taking two adjacent sentences from a single document, or by pairing up two random sentences with equal probability. The goal of this task is to predict whether or not the second sentence followed the first in the original document. + +The `create_pretraining_data.py` script takes in raw text and creates training instances for both pre-training tasks. + +#### Multi-dataset + +We are able to combine multiple datasets into a single dataset for pre-training on a diverse text corpus. Once TFRecords have been created for each component dataset, you can create a combined dataset by adding the directory to `*FILES_DIR` in `run_pretraining_*.sh`. This will feed all matching files to the input pipeline in `run_pretraining.py`. However, in the training process, only one TFRecord file is consumed at a time, therefore, the training instances of any given training batch will all belong to the same source dataset. + + + +### Training process + +The training process consists of two steps: pre-training and fine tuning. + +#### Pre-training + +BERT is designed to pre-train deep bidirectional representations for language representations. The following scripts are to pre-train BERT on PubMed dataset. These scripts are general and can be used for pre-training language representations on additional corpus of biomedical text. + +Pre-training is performed using the `run_pretraining.py` script along with parameters defined in the `biobert/scripts/run_pretraining_pubmed_base_phase_1.sh` and `biobert/scripts/run_pretraining_pubmed_base_phase_2.sh` scripts. + +The `biobert/scripts/run_pretraining_pubmed_base_phase*.sh` scripts run a job on a single node that trains the BERT-base model from scratch using the PubMed Corpus dataset as training data. By default, the training script: +- Runs on 16 GPUs +- Has FP16 precision enabled +- Is XLA enabled +- Creates a log file containing all the output +- Saves a checkpoint every 5000 iterations (keeps only the latest checkpoint) and at the end of training. All checkpoints, evaluation results, and training logs are saved to the `/results` directory (in the container which can be mounted to a local directory). +- Evaluates the model at the end of each phase + +- Phase 1 + - Runs 19531 steps with 1953 warmup steps + - Sets Maximum sequence length as 128 + - Sets Global Batch size as 64K + +- Phase 2 + - Runs 4340 steps with 434 warm-up steps + - Sets Maximum sequence length as 512 + - Sets Global Batch size as 32K + - Should start from Phase1's final checkpoint + +These parameters train PubMed with reasonable accuracy on a DGX-2 with 32GB V100 cards. + +For example: +```bash +biobert/scripts/run_pretraining-pubmed_base_phase_1.sh +``` + +Where: +- `` is per-GPU batch size used for training. Batch size varies with precision, larger batch sizes run more efficiently, but require more memory. + +- `` is the default rate of 3.2e-5 is good for global batch size 64k. + +- `` is set to `true` or `false` depending on whether the model should be trained on cased or uncased data. + +- `` is the type of math in your model, can be either `fp32` or `fp16`. Specifically: + + - `fp32` is 32-bit IEEE single precision floats. + - `fp16` is Automatic rewrite of TensorFlow compute graph to take advantage of 16-bit arithmetic whenever it is safe. + +- `` is the number of GPUs to use for training. Must be equal to or smaller than the number of GPUs attached to your node. + +- `` is the number of warm-up steps at the start of training. + +- `` is the total number of training steps. + +- `` controls how often checkpoints are saved. Default is 5000 steps. + +- `` is used to mimic higher batch sizes in the respective phase by accumulating gradients N times before weight update. + +- `` is used to indicate whether to pretrain BERT Large or BERT Base model. + +- `` is per-GPU batch size used for evaluation after training. + +The following sample code trains phase 1 of BERT-base from scratch on a single DGX-2 using FP16 arithmetic and uncased data. + +```bash +biobert/scripts/run_pretraining-pubmed_base_phase_1.sh 128 3.2e-5 false fp16 true 16 1953 19531 32 5000 80 +``` + +#### Fine tuning + +Fine tuning is performed using the `run_ner.py` script along with parameters defined in `biobert/scripts/ner_bc5cdr*.sh`. + +For example, `biobert/scripts/ner_bc5cdr-chem.sh` script trains a model and performs evaluation on the BC5CDR Chemical dataset. By default, the training script: + +- Trains on BERT Base Uncased Model +- Uses 16 GPUs and batch size of 8 on each GPU +- Has FP16 precision enabled +- Is XLA enabled +- Runs for 10 epochs +- Evaluation is done at the end of training. To skip evaluation, modify `--do_eval` and `--do_predict` to `False`. + +This script outputs checkpoints to the `/results` directory, by default, inside the container. Mount point of `/results` can be changed in the `scripts/docker/launch.sh` file. The training log contains information about: +- Loss for the final step +- Training and evaluation performance +- F1, Precision and Recall on the Test Set of BC5CDR Chemical after evaluation. + +The summary after training is printed in the following format: +```bash + 0: /results/biobert_finetune_ner_chem_191028154209/test_labels.txt + 0: /results/biobert_finetune_ner_chem_191028154209/test_labels_errs.txt + 0: processed 124669 tokens with 5433 phrases; found: 5484 phrases; correct: 5102. + 0: accuracy: 99.26%; precision: 93.03%; recall: 93.91%; FB1: 93.47 + 0: : precision: 93.03%; recall: 93.91%; FB1: 93.47 5484 +``` + +Multi-GPU training is enabled with the Horovod TensorFlow module. The following example runs training on 16 GPUs: + +```bash +BERT_DIR=data/download/google_pretrained_weights/uncased_L-12_H-768_A-12 +DATA_DIR=data/biobert/BC5CDR/chem + +mpi_command="mpirun -np 16 -H localhost:16 \ + --allow-run-as-root -bind-to none -map-by slot \ + -x NCCL_DEBUG=INFO \ + -x LD_LIBRARY_PATH \ + -x PATH -mca pml ob1 -mca btl ^openib" \ + python run_ner.py --horovod --use_fp16 --use_xla \ + --vocab_file=$BERT_DIR/vocab.txt \ + --bert_config_file=$BERT_DIR/bert_config.json \ + --output_dir=/results --data_dir=$DATA_DIR" +``` + +#### Multi-node + +Multi-node runs can be launched on a pyxis/enroot Slurm cluster (see [Requirements](https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow/LanguageModeling/BERT#requirements)) with the `biobert/scripts/run_biobert.sub` script with the following command for a 4-node DGX2 example for both phase 1 and phase 2: + +```bash +BATCHSIZE=128 LEARNING_RATE='8e-6' NUM_ACCUMULATION_STEPS=8 PHASE=1 sbatch -N4 --ntasks-per-node=16 biobert/scripts/run_biobert.sub +BATCHSIZE=16 LEARNING_RATE='3.2e-5' NUM_ACCUMULATION_STEPS=32 PHASE=1 sbatch -N4 --ntasks-per-node=16 biobert/scripts/run_biobert.sub +``` + +Checkpoint after phase 1 will be saved in `checkpointdir` specified in `biobert/scripts/run_biobert.sub`. The checkpoint will be automatically picked up to resume training on phase 2. Note that phase 2 should be run after phase 1. + +Variables to re-run the [Training performance results](#training-performance-results) are available in the `configurations.yml` file. + +The batch variables `BATCHSIZE`, `LEARNING_RATE`, `NUM_ACCUMULATION_STEPS` refer to the Python arguments `train_batch_size`, `learning_rate`, `num_accumulation_steps` respectively. +The variable `PHASE` refers to phase specific arguments available in `biobert/scripts/run_biobert.sub`. + +Note that the `biobert/scripts/run_biobert.sub` script is a starting point that has to be adapted depending on the environment. In particular, variables such as `datadir` handle the location of the files for each phase. + +Refer to the file contents to see the full list of variables to adjust for your system. + +### Inference process + +Inference on a fine tuned model for Bio Medical tasks is performed using the `run_ner.py` or `run_re.py` script along with parameters defined in `biobert/scripts/run_biobert_finetuning_inference.sh`. Inference is supported on a single GPU. + +The `biobert/scripts/run_biobert_finetuning_inference.sh` script performs evaluation on ChemProt or BC5CDR datasets depending on the task specified. By default, the inferencing script: + +- Uses BC5CDR Chemical dataset +- Has FP16 precision enabled +- Is XLA enabled +- Evaluates the latest checkpoint present in `/results` with a batch size of 16. + +This script computes F1, Precision and Recall scores. Mount point of `/results` can be changed in the `scripts/docker/launch.sh` file. + +## Performance + +### Benchmarking + +The following section shows how to run benchmarks measuring the model performance in training and inference modes. + +Both of these benchmarking scripts enable you to run a number of epochs, extract performance numbers, and run the BERT model for fine tuning. + +#### Training performance benchmark + +Training benchmarking can be performed by running the script: +``` bash +biobert/scripts/biobert_finetune_training_benchmark.sh +``` + +This script runs 2 epochs by default on the NER BC5CDR dataset and extracts performance numbers for various batch sizes and sequence lengths in both FP16 and FP32. These numbers are saved at `/results/tf_bert_biobert__training_benchmark____num_gpu__` + +#### Inference performance benchmark + +Training benchmarking can be performed by running the script: +``` bash +biobert/scripts/biobert_finetune_inference_benchmark.sh +``` + +This script runs inference on the test and dev sets and extracts performance and latency numbers for various batch sizes and sequence lengths in both FP16 with XLA and FP32 without XLA. These numbers are saved at `/results/tf_bert_biobert__training_benchmark____num_gpu__` + +## Results + +The following sections provide detailed results of downstream fine-tuning task on NER and RE benchmark tasks. + +### Training accuracy results + +#### Pre-training accuracy + +Our results were obtained by running the `scripts/run_pretraining_lamb.sh` training script in the TensorFlow 19.08-py3 NGC container. + +| **DGX System** | **Nodes** | **Precision** | **Batch Size/GPU: Phase1, Phase2** | **Accumulation Steps: Phase1, Phase2** | **Time to Train (Hrs)** | **Final Loss** | +|----------------|-----------|---------------|------------------------------------|----------------------------------------|----------------|-------------------------| +| DGX2H | 4 | FP16 | 128, 16 | 8, 32 | 19.14 | 0.88 | +| DGX2H | 16 | FP16 | 128, 16 | 2, 8 | 4.81 | 0.86 | +| DGX2H | 32 | FP16 | 128, 16 | 1, 4 | 2.65 | 0.87 | + +#### Fine-tuning accuracy + +| **Task** | **F1** | **Precision** | **Recall** | +|:-------:|:----:|:----:|:----:| +| NER BC5CDR-chemical | 93.47 | 93.03 | 93.91 | +| NER BC5CDR-disease | 86.22 | 85.05 | 87.43 | +| RE Chemprot | 76.27 | 77.62 | 74.98 | + +##### Fine-tuning accuracy for NER Chem + +Our results were obtained by running the `biobert/scripts/ner_bc5cdr-chem.sh` training script in the TensorFlow 19.08-py3 NGC container. + +| **DGX System** | **Batch size / GPU** | **F1 - FP32** | **F1- mixed precision** | **Time to Train - FP32 (Minutes)** | **Time to Train - mixed precision (Minutes)** | +|:---:|:----:|:----:|:---:|:----:|:----:| +| DGX-1 16G | 64 |93.33|93.40|23.95|14.13| +| DGX-1 32G | 64 |93.31|93.36|24.35|12.63| +| DGX-2 32G | 64 |93.66|93.47|12.26|8.16| + + +### Training stability test + +#### Fine-tuning stability test: + +The following tables compare F1 scores scores across 5 different training runs on the NER Chemical task with different seeds, for both FP16 and FP32. The runs showcase consistent convergence on all 5 seeds with very little deviation. + +| **16 x V100 GPUs** | **seed 1** | **seed 2** | **seed 3** | **seed 4** | **seed 5** | **mean** | **std** | +|:-----------:|:-----:|:-----:|:-----:|:-----:|:-----:|:-----:|:-----:| +| F1 Score (FP16) | 93.13 | 92.92 | 93.34 | 93.66 | 93.47 | 93.3 | 0.29 | +| F1 Score (FP32) | 93.1 | 93.28 | 93.33 | 93.45 | 93.17 | 93.27 | 0.14 | + + +### Training performance results + +#### Training performance: NVIDIA DGX-1 (8x V100 16G) + +##### Pre-training training performance: multi-node on DGX-1 16G + +Our results were obtained by running the `biobert/scripts/run_biobert.sub` training script in the TensorFlow 19.08-py3 NGC container using multiple NVIDIA DGX-1 with 8x V100 16G GPUs. Performance (in sentences per second) is the steady state throughput. + +| **Nodes** | **Sequence Length**| **Batch size / GPU: mixed precision, FP32** | **Throughput - mixed precision** | **Throughput - FP32** | **Throughput speedup (FP32 to mixed precision)** | **Weak scaling - mixed precision** | **Weak scaling - FP32** | +|:-------:|:-----:|:-------:|:-------:|:-------:|:-------------:|:------:|:------:| +| 1 | 128 | 64,32 | 2762.06 | 744.48 | 3.71 | 1.00 | 1.00 | +| 4 | 128 | 64,32 | 10283.08 | 2762.88 | 3.72 | 3.72 | 3.71 | +| 16 | 128 | 64,32 | 39051.69 | 10715.14 | 3.64 | 14.14 | 14.39 | +| 32 | 128 | 64,32 | 76077.39 | 21104.87 | 3.60 | 27.54 | 28.35 | +| 1 | 512 | 8,8 | 432.33 | 160.38 | 2.70 | 1.00 | 1.00 | +| 4 | 512 | 8,8 | 1593.00 | 604.36 | 2.64 | 3.68 | 3.77 | +| 16 | 512 | 8,8 | 5941.82 | 2356.44 | 2.52 | 13.74 | 14.69 | +| 32 | 512 | 8,8 | 11483.73 | 4631.29 | 2.48 | 26.56 | 28.88 | + +Note: The respective values for FP32 runs that use a batch size of 16, 2 in sequence lengths 128 and 512 respectively are not available due to out of memory errors that arise. + +##### Fine-tuning training performance for NER on DGX-1 16G + +Our results were obtained by running the `biobert/scripts/ner_bc5cdr-chem.sh` training script in the TensorFlow 19.08-py3 NGC container on NVIDIA DGX-1 with 8x V100 16G GPUs. Performance (in sentences per second) is the mean throughput from 2 epochs. + +| **GPUs** | **Batch size / GPU** | **Throughput - FP32** | **Throughput - mixed precision** | **Throughput speedup (FP32 to mixed precision)** | **Weak scaling - FP32** | **Weak scaling - mixed precision** | +|:---:|:---:|:------:|:-----:|:----:|:----:|:----:| +| 1 | 64 | 147.71 | 348.84 | 2.36 | 1.00 | 1.00 | +| 4 | 64 | 583.78 | 1145.46 | 1.96 | 3.95 | 3.28 | +| 8 | 64 | 981.22 | 1964.85 | 2.00 | 6.64 | 5.63 | + +To achieve these same results, follow the [Quick Start Guide](#quick-start-guide) outlined above. + +#### Training performance: NVIDIA DGX-1 (8x V100 32G) + + +##### Fine-tuning training performance for NER on DGX-1 32G + +Our results were obtained by running the `biobert/scripts/ner_bc5cdr-chem.sh` training script in the TensorFlow 19.08-py3 NGC container on NVIDIA DGX-1 with 8x V100 32G GPUs. Performance (in sentences per second) is the mean throughput from 2 epochs. + + +| **GPUs** | **Batch size / GPU** | **Throughput - FP32** | **Throughput - mixed precision** | **Throughput speedup (FP32 to mixed precision)** | **Weak scaling - FP32** | **Weak scaling - mixed precision** | +|:---:|:---:|:------:|:-----:|:----:|:----:|:----:| +| 1 | 64 | 144.1 | 417.39 | 2.89 | 1.00 | 1.00 | +| 4 | 64 | 525.15 | 1354.14 | 2.57 | 3.64 | 3.24 | +| 8 | 64 | 969.4 | 2341.39 | 2.41 | 6.73 | 5.61 | + + +To achieve these same results, follow the [Quick Start Guide](#quick-start-guide) outlined above. + +#### Training performance: NVIDIA DGX-2 (16x V100 32G) + + +##### Pre-training training performance: multi-node on DGX-2H 32G + +Our results were obtained by running the `biobert/scripts/run_biobert.sub` training script in the TensorFlow 19.08-py3 NGC container using multiple NVIDIA DGX-2H with 16x V100 32G GPUs. Performance (in sentences per second) is the steady state throughput. + + +| **Nodes** | **Sequence Length**| **Batch size / GPU: mixed precision, FP32** | **Throughput - mixed precision** | **Throughput - FP32** | **Throughput speedup (FP32 to mixed precision)** | **Weak scaling - mixed precision** | **Weak scaling - FP32** | +|:-------:|:-----:|:-------:|:-------:|:-------:|:-------------:|:------:|:------:| +| 1 | 128 | 128,128 | 7772.18 | 2165.04 | 3.59 | 1.00 | 1.00 | +| 4 | 128 | 128,128 | 29785.31 | 8516.90 | 3.50 | 3.83 | 3.93 | +| 16 | 128 | 128,128 | 115581.29 | 33699.15 | 3.43 | 14.87 | 15.57 | +| 32 | 128 | 128,128 | 226156.53 | 66996.73 | 3.38 | 29.10 | 30.94 | +| 64 | 128 | 128,128 | 444955.74 | 133424.95 | 3.33 | 57.25 | 61.63 | +| 1 | 512 | 16,16 | 1260.06 | 416.92 | 3.02 | 1.00 | 1.00 | +| 4 | 512 | 16,16 | 4781.19 | 1626.76 | 2.94 | 3.79 | 3.90 | +| 16 | 512 | 16,16 | 18405.65 | 6418.09 | 2.87 | 14.61 | 15.39 | +| 32 | 512 | 16,16 | 36071.06 | 12713.67 | 2.84 | 28.63 | 30.49 | +| 64 | 512 | 16,16 | 69950.86 | 25245.96 | 2.77 | 55.51 | 60.55 | + + +##### Fine-tuning training performance for NER on DGX-2 32G + +Our results were obtained by running the `biobert/scripts/ner_bc5cdr-chem.sh` training script in the TensorFlow 19.08-py3 NGC container on NVIDIA DGX-2 with 16x V100 32G GPUs. Performance (in sentences per second) is the mean throughput from 2 epochs. + +| **GPUs** | **Batch size / GPU** | **Throughput - FP32** | **Throughput - mixed precision** | **Throughput speedup (FP32 to mixed precision)** | **Weak scaling - FP32** | **Weak scaling - mixed precision** | +|:---:|:---:|:------:|:-----:|:----:|:----:|:----:| +| 1 | 64 | 139.59 | 475.54 | 3.4 | 1.00 | 1.00 | +| 4 | 64 | 517.08 | 1544.01 | 2.98 | 3.70 | 3.25 | +| 8 | 64 | 1009.84 | 2695.34 | 2.66 | 7.23 | 5.67 | +| 16 | 64 | 1997.73 | 4268.81 | 2.13 | 14.31 | 8.98 | + +To achieve these same results, follow the [Quick Start Guide](#quick-start-guide) outlined above. + +## Release notes + +### Changelog + +November 2019 +- Initial release + +### Known issues + + +- There are no known issues with the model. + + + diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/__init__.py b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/conlleval.py b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/conlleval.py new file mode 100644 index 0000000..d9a6471 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/conlleval.py @@ -0,0 +1,302 @@ +# Python version of the evaluation script from CoNLL'00- +# Originates from: https://github.com/spyysalo/conlleval.py + + +# Intentional differences: +# - accept any space as delimiter by default +# - optional file argument (default STDIN) +# - option to set boundary (-b argument) +# - LaTeX output (-l argument) not supported +# - raw tags (-r argument) not supported + +# add function :evaluate(predicted_label, ori_label): which will not read from file + +import sys +import re +import codecs +from collections import defaultdict, namedtuple + +ANY_SPACE = '' + + +class FormatError(Exception): + pass + +Metrics = namedtuple('Metrics', 'tp fp fn prec rec fscore') + + +class EvalCounts(object): + def __init__(self): + self.correct_chunk = 0 # number of correctly identified chunks + self.correct_tags = 0 # number of correct chunk tags + self.found_correct = 0 # number of chunks in corpus + self.found_guessed = 0 # number of identified chunks + self.token_counter = 0 # token counter (ignores sentence breaks) + + # counts by type + self.t_correct_chunk = defaultdict(int) + self.t_found_correct = defaultdict(int) + self.t_found_guessed = defaultdict(int) + + +def parse_args(argv): + import argparse + parser = argparse.ArgumentParser( + description='evaluate tagging results using CoNLL criteria', + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + arg = parser.add_argument + arg('-b', '--boundary', metavar='STR', default='-X-', + help='sentence boundary') + arg('-d', '--delimiter', metavar='CHAR', default=ANY_SPACE, + help='character delimiting items in input') + arg('-o', '--otag', metavar='CHAR', default='O', + help='alternative outside tag') + arg('file', nargs='?', default=None) + return parser.parse_args(argv) + + +def parse_tag(t): + m = re.match(r'^([^-]*)-(.*)$', t) + return m.groups() if m else (t, '') + + +def evaluate(iterable, options=None): + if options is None: + options = parse_args([]) # use defaults + + counts = EvalCounts() + num_features = None # number of features per line + in_correct = False # currently processed chunks is correct until now + last_correct = 'O' # previous chunk tag in corpus + last_correct_type = '' # type of previously identified chunk tag + last_guessed = 'O' # previously identified chunk tag + last_guessed_type = '' # type of previous chunk tag in corpus + + for i, line in enumerate(iterable): + line = line.rstrip('\r\n') + # print(line) + + if options.delimiter == ANY_SPACE: + features = line.split() + else: + features = line.split(options.delimiter) + + if num_features is None: + num_features = len(features) + elif num_features != len(features) and len(features) != 0: + raise FormatError('unexpected number of features: %d (%d) at line %d\n%s' % + (len(features), num_features, i, line)) + + if len(features) == 0 or features[0] == options.boundary: + features = [options.boundary, 'O', 'O'] + if len(features) < 3: + raise FormatError('unexpected number of features in line %s' % line) + + guessed, guessed_type = parse_tag(features.pop()) + correct, correct_type = parse_tag(features.pop()) + first_item = features.pop(0) + + if first_item == options.boundary: + guessed = 'O' + + end_correct = end_of_chunk(last_correct, correct, + last_correct_type, correct_type) + end_guessed = end_of_chunk(last_guessed, guessed, + last_guessed_type, guessed_type) + start_correct = start_of_chunk(last_correct, correct, + last_correct_type, correct_type) + start_guessed = start_of_chunk(last_guessed, guessed, + last_guessed_type, guessed_type) + + if in_correct: + if (end_correct and end_guessed and + last_guessed_type == last_correct_type): + in_correct = False + counts.correct_chunk += 1 + counts.t_correct_chunk[last_correct_type] += 1 + elif (end_correct != end_guessed or guessed_type != correct_type): + in_correct = False + + if start_correct and start_guessed and guessed_type == correct_type: + in_correct = True + + if start_correct: + counts.found_correct += 1 + counts.t_found_correct[correct_type] += 1 + if start_guessed: + counts.found_guessed += 1 + counts.t_found_guessed[guessed_type] += 1 + if first_item != options.boundary: + if correct == guessed and guessed_type == correct_type: + counts.correct_tags += 1 + counts.token_counter += 1 + + last_guessed = guessed + last_correct = correct + last_guessed_type = guessed_type + last_correct_type = correct_type + + if in_correct: + counts.correct_chunk += 1 + counts.t_correct_chunk[last_correct_type] += 1 + + return counts + + + +def uniq(iterable): + seen = set() + return [i for i in iterable if not (i in seen or seen.add(i))] + + +def calculate_metrics(correct, guessed, total): + tp, fp, fn = correct, guessed-correct, total-correct + p = 0 if tp + fp == 0 else 1.*tp / (tp + fp) + r = 0 if tp + fn == 0 else 1.*tp / (tp + fn) + f = 0 if p + r == 0 else 2 * p * r / (p + r) + return Metrics(tp, fp, fn, p, r, f) + + +def metrics(counts): + c = counts + overall = calculate_metrics( + c.correct_chunk, c.found_guessed, c.found_correct + ) + by_type = {} + for t in uniq(list(c.t_found_correct) + list(c.t_found_guessed)): + by_type[t] = calculate_metrics( + c.t_correct_chunk[t], c.t_found_guessed[t], c.t_found_correct[t] + ) + return overall, by_type + + +def report(counts, out=None): + if out is None: + out = sys.stdout + + overall, by_type = metrics(counts) + + c = counts + out.write('processed %d tokens with %d phrases; ' % + (c.token_counter, c.found_correct)) + out.write('found: %d phrases; correct: %d.\n' % + (c.found_guessed, c.correct_chunk)) + + if c.token_counter > 0: + out.write('accuracy: %6.2f%%; ' % + (100.*c.correct_tags/c.token_counter)) + out.write('precision: %6.2f%%; ' % (100.*overall.prec)) + out.write('recall: %6.2f%%; ' % (100.*overall.rec)) + out.write('FB1: %6.2f\n' % (100.*overall.fscore)) + + for i, m in sorted(by_type.items()): + out.write('%17s: ' % i) + out.write('precision: %6.2f%%; ' % (100.*m.prec)) + out.write('recall: %6.2f%%; ' % (100.*m.rec)) + out.write('FB1: %6.2f %d\n' % (100.*m.fscore, c.t_found_guessed[i])) + + +def report_notprint(counts, out=None): + if out is None: + out = sys.stdout + + overall, by_type = metrics(counts) + + c = counts + final_report = [] + line = [] + line.append('processed %d tokens with %d phrases; ' % + (c.token_counter, c.found_correct)) + line.append('found: %d phrases; correct: %d.\n' % + (c.found_guessed, c.correct_chunk)) + final_report.append("".join(line)) + + if c.token_counter > 0: + line = [] + line.append('accuracy: %6.2f%%; ' % + (100.*c.correct_tags/c.token_counter)) + line.append('precision: %6.2f%%; ' % (100.*overall.prec)) + line.append('recall: %6.2f%%; ' % (100.*overall.rec)) + line.append('FB1: %6.2f\n' % (100.*overall.fscore)) + final_report.append("".join(line)) + + for i, m in sorted(by_type.items()): + line = [] + line.append('%17s: ' % i) + line.append('precision: %6.2f%%; ' % (100.*m.prec)) + line.append('recall: %6.2f%%; ' % (100.*m.rec)) + line.append('FB1: %6.2f %d\n' % (100.*m.fscore, c.t_found_guessed[i])) + final_report.append("".join(line)) + return final_report + + +def end_of_chunk(prev_tag, tag, prev_type, type_): + # check if a chunk ended between the previous and current word + # arguments: previous and current chunk tags, previous and current types + chunk_end = False + + if prev_tag == 'E': chunk_end = True + if prev_tag == 'S': chunk_end = True + + if prev_tag == 'B' and tag == 'B': chunk_end = True + if prev_tag == 'B' and tag == 'S': chunk_end = True + if prev_tag == 'B' and tag == 'O': chunk_end = True + if prev_tag == 'I' and tag == 'B': chunk_end = True + if prev_tag == 'I' and tag == 'S': chunk_end = True + if prev_tag == 'I' and tag == 'O': chunk_end = True + + if prev_tag != 'O' and prev_tag != '.' and prev_type != type_: + chunk_end = True + + # these chunks are assumed to have length 1 + if prev_tag == ']': chunk_end = True + if prev_tag == '[': chunk_end = True + + return chunk_end + + +def start_of_chunk(prev_tag, tag, prev_type, type_): + # check if a chunk started between the previous and current word + # arguments: previous and current chunk tags, previous and current types + chunk_start = False + + if tag == 'B': chunk_start = True + if tag == 'S': chunk_start = True + + if prev_tag == 'E' and tag == 'E': chunk_start = True + if prev_tag == 'E' and tag == 'I': chunk_start = True + if prev_tag == 'S' and tag == 'E': chunk_start = True + if prev_tag == 'S' and tag == 'I': chunk_start = True + if prev_tag == 'O' and tag == 'E': chunk_start = True + if prev_tag == 'O' and tag == 'I': chunk_start = True + + if tag != 'O' and tag != '.' and prev_type != type_: + chunk_start = True + + # these chunks are assumed to have length 1 + if tag == '[': chunk_start = True + if tag == ']': chunk_start = True + + return chunk_start + + +def main(argv): + args = parse_args(argv[1:]) + + if args.file is None: + counts = evaluate(sys.stdin, args) + else: + with open(args.file) as f: + counts = evaluate(f, args) + report(counts) + + +def return_report(input_file): + with open(input_file, "r") as f: + counts = evaluate(f) + return report_notprint(counts) + +if __name__ == '__main__': + # sys.exit(main(sys.argv)) + return_report('/home/pengy6/data/sentence_similarity/data/cdr/test1/wanli_result2/label_test.txt') \ No newline at end of file diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/re_eval.py b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/re_eval.py new file mode 100644 index 0000000..2660602 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/re_eval.py @@ -0,0 +1,51 @@ +import os +import numpy as np +import pandas as pd +import sklearn.metrics +import argparse + + +parser = argparse.ArgumentParser(description='') +parser.add_argument('--output_path', type=str, help='') +parser.add_argument('--answer_path', type=str, help='') +parser.add_argument('--task', type=str, default="binary", help='default:binary, possible other options:{chemprot}') +args = parser.parse_args() + + +testdf = pd.read_csv(args.answer_path, sep="\t", index_col=0) +preddf = pd.read_csv(args.output_path, sep="\t", header=None) + + +# binary +if args.task == "binary": + pred = [preddf.iloc[i].tolist() for i in preddf.index] + pred_class = [np.argmax(v) for v in pred] + pred_prob_one = [v[1] for v in pred] + + p,r,f,s = sklearn.metrics.precision_recall_fscore_support(y_pred=pred_class, y_true=testdf["label"]) + results = dict() + results["f1 score"] = f[1] + results["recall"] = r[1] + results["precision"] = p[1] + results["specificity"] = r[0] + +# chemprot +# micro-average of 5 target classes +# see "Potent pairing: ensemble of long short-term memory networks and support vector machine for chemical-protein relation extraction (Mehryary, 2018)" for details +if args.task == "chemprot": + pred = [preddf.iloc[i].tolist() for i in preddf.index] + pred_class = [np.argmax(v) for v in pred] + str_to_int_mapper = dict() + + for i,v in enumerate(sorted(testdf["label"].unique())): + str_to_int_mapper[v] = i + test_answer = [str_to_int_mapper[v] for v in testdf["label"]] + + p,r,f,s = sklearn.metrics.precision_recall_fscore_support(y_pred=pred_class, y_true=test_answer, labels=[0,1,2,3,4], average="micro") + results = dict() + results["f1 score"] = f + results["recall"] = r + results["precision"] = p + +for k,v in results.items(): + print("{:11s} : {:.2%}".format(k,v)) diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/biobert_data_download.sh b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/biobert_data_download.sh new file mode 100644 index 0000000..531637d --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/biobert_data_download.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker run --runtime=nvidia -v $PWD:/workspace/bert \ + --rm --shm-size=1g --ulimit memlock=-1 \ + --ulimit stack=67108864 --ipc=host -t -i \ + bert bash -c "bash data/create_biobert_datasets_from_start.sh" \ No newline at end of file diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/biobert_finetune_inference_benchmark.sh b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/biobert_finetune_inference_benchmark.sh new file mode 100644 index 0000000..77a71a0 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/biobert_finetune_inference_benchmark.sh @@ -0,0 +1,187 @@ +#!/bin/bash + +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +task=${1:-"ner_bc5cdr-chem"} +bert_model=${2:-"base"} +cased=${3:-"false"} + +if [ "$cased" = "true" ] ; then + DO_LOWER_CASE=0 + CASING_DIR_PREFIX="cased" + case_flag="--do_lower_case=False" +else + DO_LOWER_CASE=1 + CASING_DIR_PREFIX="uncased" + case_flag="--do_lower_case=True" +fi + +if [ "$bert_model" = "large" ] ; then + export BERT_DIR=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_L-24_H-1024_A-16 +else + export BERT_DIR=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_L-12_H-768_A-12 +fi + +DATESTAMP=`date +'%y%m%d%H%M%S'` +printf -v TAG "tf_bert_biobert_%s_inference_benchmark_%s_%s" "$task" "$bert_model" "$CASING_DIR_PREFIX" +OUTPUT_DIR=/results/${TAG}_${DATESTAMP} +mkdir -p ${OUTPUT_DIR} + +if [ "$task" = "ner_bc5cdr-chem" ] ; then + + + DATASET_DIR=/workspace/bert/data/biobert/BC5CDR/chem + LOGFILE="${OUTPUT_DIR}/${task}_training_benchmark_bert_${bert_model}.log" + + echo "Training performance benchmarking for BERT $bert_model from $BERT_DIR" >> $LOGFILE + + echo "Precision Sequence Length Batch size Performance(sent/sec)" >> $LOGFILE + + for seq_length in 128 512; do + for batch_size in 8 32 64; do + for precision in fp16 fp32; do + res_dir=${OUTPUT_DIR}/bert_${bert_model}_sl_${seq_len}_prec_${precision}_bs_${batch_size} + mkdir -p ${res_dir} + tmp_file="${res_dir}/${task}_training_benchmark.log" + + if [ "$precision" = "fp16" ] ; then + echo "fp16 activated!" + use_fp16="--use_fp16" + use_xla_tag="--use_xla" + else + echo "fp32 activated!" + use_fp16="" + use_xla_tag="" + fi + + python /workspace/bert/run_ner.py \ + --do_prepare=true \ + --do_eval=true \ + --do_predict=true \ + --task_name="bc5cdr" \ + --vocab_file=$BERT_DIR/vocab.txt \ + --bert_config_file=$BERT_DIR/bert_config.json \ + --init_checkpoint="$BERT_DIR/bert_model.ckpt" \ + --data_dir=$DATASET_DIR \ + --output_dir=$res_dir \ + --eval_batch_size=$batch_size \ + --predict_batch_size=$batch_size \ + --max_seq_length=$seq_length \ + $use_fp16 $use_xla_tag $case_flag |& tee $tmp_file + + perf=`cat $tmp_file | grep -F 'Throughput Average (sentences/sec) =' | tail -1 | awk -F'= ' '{print $2}' | awk -F' sen' '{print $1}'` + echo "$precision $seq_len $batch_size $perf" >> $LOGFILE + + done + done + done + +elif [ "$task" = "ner_bc5cdr-disease" ] ; then + DATASET_DIR=/workspace/bert/data/biobert/BC5CDR/disease + + LOGFILE="${OUTPUT_DIR}/${task}_training_benchmark_bert_${bert_model}.log" + + echo "Training performance benchmarking for BERT $bert_model from $BERT_DIR" >> $LOGFILE + + echo "Precision Sequence Length Batch size Performance(sent/sec)" >> $LOGFILE + + for seq_length in 128 512; do + for batch_size in 8 32 64; do + for precision in fp16 fp32; do + res_dir=${OUTPUT_DIR}/bert_${bert_model}_sl_${seq_len}_prec_${precision}_bs_${batch_size} + mkdir -p ${res_dir} + tmp_file="${res_dir}/${task}_training_benchmark.log" + + if [ "$precision" = "fp16" ] ; then + echo "fp16 activated!" + use_fp16="--use_fp16" + use_xla_tag="--use_xla" + else + echo "fp32 activated!" + use_fp16="" + use_xla_tag="" + fi + python3 /workspace/bert/run_ner.py \ + --do_prepare=true \ + --do_eval=true \ + --do_predict=true \ + --task_name="bc5cdr" \ + --vocab_file=$BERT_DIR/vocab.txt \ + --bert_config_file=$BERT_DIR/bert_config.json \ + --init_checkpoint="$BERT_DIR/bert_model.ckpt" \ + --data_dir=$DATASET_DIR \ + --output_dir=$res_dir \ + --eval_batch_size=$batch_size \ + --predict_batch_size=$batch_size \ + --max_seq_length=$seq_length \ + "$use_fp16" $use_xla_tag $case_flag |& tee $tmp_file + + perf=`cat $tmp_file | grep -F 'Throughput Average (sentences/sec) =' | tail -1 | awk -F'= ' '{print $2}' | awk -F' sen' '{print $1}'` + echo "$precision $seq_len $batch_size $perf" >> $LOGFILE + + done + done + done + +elif [ "$task" = "rel_chemprot" ] ; then + DATASET_DIR=/workspace/bert/data/biobert/ChemProt + + LOGFILE="${OUTPUT_DIR}/${task}_training_benchmark_bert_${bert_model}.log" + + echo "Training performance benchmarking for BERT $bert_model from $BERT_DIR" >> $LOGFILE + + echo "Precision Sequence Length Batch size Performance(sent/sec)" >> $LOGFILE + + for seq_length in 128 512; do + for batch_size in 8 32 64; do + for precision in fp16 fp32; do + res_dir=${OUTPUT_DIR}/bert_${bert_model}_sl_${seq_len}_prec_${precision}_bs_${batch_size} + mkdir -p ${res_dir} + tmp_file="${res_dir}/${task}_training_benchmark.log" + + if [ "$precision" = "fp16" ] ; then + echo "fp16 activated!" + use_fp16="--use_fp16" + use_xla_tag="--use_xla" + else + echo "fp32 activated!" + use_fp16="" + use_xla_tag="" + fi + python3 /workspace/bert/run_re.py \ + --do_prepare=true \ + --do_eval=true \ + --do_predict=true \ + --task_name="chemprot" \ + --vocab_file=$BERT_DIR/vocab.txt \ + --bert_config_file=$BERT_DIR/bert_config.json \ + --init_checkpoint="$BERT_DIR/bert_model.ckpt" \ + --data_dir=$DATASET_DIR \ + --output_dir=$res_dir \ + --eval_batch_size=$batch_size \ + --predict_batch_size=$batch_size \ + --max_seq_length=$seq_length \ + "$use_fp16" $use_xla_tag $case_flag |& tee $tmp_file + + perf=`cat $tmp_file | grep -F 'Throughput Average (sentences/sec) =' | tail -1 | awk -F'= ' '{print $2}' | awk -F' sen' '{print $1}'` + echo "$precision $seq_len $batch_size $perf" >> $LOGFILE + + done + done + done + +else + + echo "Benchmarking for " $task "currently not supported. Sorry!" + +fi \ No newline at end of file diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/biobert_finetune_train_benchmark.sh b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/biobert_finetune_train_benchmark.sh new file mode 100644 index 0000000..bc93e3b --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/biobert_finetune_train_benchmark.sh @@ -0,0 +1,203 @@ +#!/bin/bash + +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +task=${1:-"ner_bc5cdr-chem"} +num_gpu=${2:-"2"} +bert_model=${3:-"base"} +cased=${4:-"false"} + + +epochs=2.0 + +if [ "$cased" = "true" ] ; then + DO_LOWER_CASE=0 + CASING_DIR_PREFIX="cased" + case_flag="--do_lower_case=False" +else + DO_LOWER_CASE=1 + CASING_DIR_PREFIX="uncased" + case_flag="--do_lower_case=True" +fi + +if [ "$bert_model" = "large" ] ; then + export BERT_DIR=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_L-24_H-1024_A-16 +else + export BERT_DIR=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_L-12_H-768_A-12 +fi + +if [ $num_gpu -gt 1 ] ; then + mpi_command="mpirun -np $num_gpu -H localhost:$num_gpu \ + --allow-run-as-root -bind-to none -map-by slot \ + -x NCCL_DEBUG=INFO \ + -x LD_LIBRARY_PATH \ + -x PATH -mca pml ob1 -mca btl ^openib" + use_hvd="--horovod" +else + mpi_command="" + use_hvd="" +fi + +DATESTAMP=`date +'%y%m%d%H%M%S'` +printf -v TAG "tf_bert_biobert_%s_training_benchmark_%s_%s_num_gpu_%d" "$task" "$bert_model" "$CASING_DIR_PREFIX" "$num_gpu" +OUTPUT_DIR=/results/${TAG}_${DATESTAMP} +mkdir -p ${OUTPUT_DIR} + +if [ "$task" = "ner_bc5cdr-chem" ] ; then + + DATASET_DIR=/workspace/bert/data/biobert/BC5CDR/chem + LOGFILE="${OUTPUT_DIR}/${task}_training_benchmark_bert_${bert_model}_gpu_${num_gpu}.log" + + echo "Training performance benchmarking for BERT $bert_model from $BERT_DIR" >> $LOGFILE + echo "Precision Sequence Length Batch size Performance(sent/sec)" >> $LOGFILE + + for seq_length in 128 512; do + for train_batch_size in 8 32 64; do + for precision in fp16 fp32; do + res_dir=${OUTPUT_DIR}/bert_${bert_model}_gpu_${num_gpu}_sl_${seq_length}_prec_${precision}_bs_${batch_size} + mkdir -p ${res_dir} + tmp_file="${res_dir}/${task}_training_benchmark.log" + + if [ "$precision" = "fp16" ] ; then + echo "fp16 activated!" + use_fp16="--use_fp16" + use_xla_tag="--use_xla" + else + echo "fp32 activated!" + use_fp16="" + use_xla_tag="" + fi + + $mpi_command python /workspace/bert/run_ner.py \ + --do_prepare=true \ + --do_train=true \ + --do_eval=true \ + --do_predict=true \ + --task_name=bc5cdr \ + --vocab_file=$BERT_DIR/vocab.txt \ + --bert_config_file=$BERT_DIR/bert_config.json \ + --init_checkpoint="$BERT_DIR/bert_model.ckpt" \ + --num_train_epochs=$epochs \ + --data_dir=$DATASET_DIR \ + --output_dir=$res_dir \ + --train_batch_size=$train_batch_size \ + --max_seq_length=$seq_length \ + $use_hvd $use_fp16 $use_xla_tag $case_flag |& tee $tmp_file + + perf=`cat $tmp_file | grep -F 'Throughput Average (sentences/sec) =' | head -1 | awk -F'= ' '{print $2}' | awk -F' sen' '{print $1}'` + echo "$precision $seq_length $train_batch_size $perf" >> $LOGFILE + + done + done + done + +elif [ "$task" = "ner_bc5cdr-disease" ] ; then + DATASET_DIR=/workspace/bert/data/biobert/BC5CDR/disease + LOGFILE="${OUTPUT_DIR}/${task}_training_benchmark_bert_${bert_model}_gpu_${num_gpu}.log" + + echo "Training performance benchmarking for BERT $bert_model from $BERT_DIR" >> $LOGFILE + echo "Precision Sequence Length Batch size Performance(sent/sec)" >> $LOGFILE + + for seq_length in 128 512; do + for train_batch_size in 8 32 64; do + for precision in fp16 fp32; do + res_dir=${OUTPUT_DIR}/bert_${bert_model}_gpu_${num_gpu}_sl_${seq_length}_prec_${precision}_bs_${batch_size} + mkdir -p ${res_dir} + tmp_file="${res_dir}/${task}_training_benchmark.log" + + if [ "$precision" = "fp16" ] ; then + echo "fp16 activated!" + use_fp16="--use_fp16" + use_xla_tag="--use_xla" + else + echo "fp32 activated!" + use_fp16="" + use_xla_tag="" + fi + + $mpi_command python3 /workspace/bert/run_ner.py \ + --do_prepare=true \ + --do_train=true \ + --do_eval=true \ + --do_predict=true \ + --task_name="bc5cdr" \ + --vocab_file=$BERT_DIR/vocab.txt \ + --bert_config_file=$BERT_DIR/bert_config.json \ + --init_checkpoint="$BERT_DIR/bert_model.ckpt" \ + --num_train_epochs=$epochs \ + --data_dir=$DATASET_DIR \ + --output_dir=$res_dir \ + --train_batch_size=$train_batch_size \ + --max_seq_length=$seq_length \ + "$use_hvd" "$use_fp16" $use_xla_tag $case_flag |& tee $tmp_file + + perf=`cat $tmp_file | grep -F 'Throughput Average (sentences/sec) =' | head -1 | awk -F'= ' '{print $2}' | awk -F' sen' '{print $1}'` + echo "$precision $seq_length $train_batch_size $perf" >> $LOGFILE + + done + done + done + +elif [ "$task" = "rel_chemprot" ] ; then + DATASET_DIR=/workspace/bert/data/biobert/ChemProt + LOGFILE="${OUTPUT_DIR}/${task}_training_benchmark_bert_${bert_model}_gpu_${num_gpu}.log" + + echo "Training performance benchmarking for BERT $bert_model from $BERT_DIR" >> $LOGFILE + echo "Precision Sequence Length Batch size Performance(sent/sec)" >> $LOGFILE + + for seq_length in 128 512; do + for train_batch_size in 8 32 64; do + for precision in fp16 fp32; do + res_dir=${OUTPUT_DIR}/bert_${bert_model}_gpu_${num_gpu}_sl_${seq_length}_prec_${precision}_bs_${batch_size} + mkdir -p ${res_dir} + tmp_file="${res_dir}/${task}_training_benchmark.log" + + if [ "$precision" = "fp16" ] ; then + echo "fp16 activated!" + use_fp16="--use_fp16" + use_xla_tag="--use_xla" + else + echo "fp32 activated!" + use_fp16="" + use_xla_tag="" + fi + + $mpi_command python3 /workspace/bert/run_re.py \ + --do_prepare=true \ + --do_train=true \ + --do_eval=true \ + --do_predict=true \ + --task_name="chemprot" \ + --vocab_file=$BERT_DIR/vocab.txt \ + --bert_config_file=$BERT_DIR/bert_config.json \ + --init_checkpoint="$BERT_DIR/bert_model.ckpt" \ + --num_train_epochs=$epochs \ + --data_dir=$DATASET_DIR \ + --output_dir=$res_dir \ + --train_batch_size=$train_batch_size \ + --max_seq_length=$seq_length \ + "$use_hvd" "$use_fp16" $use_xla_tag $case_flag |& tee $tmp_file + + perf=`cat $tmp_file | grep -F 'Throughput Average (sentences/sec) =' | head -1 | awk -F'= ' '{print $2}' | awk -F' sen' '{print $1}'` + echo "$precision $seq_length $train_batch_size $perf" >> $LOGFILE + + done + done + done + +else + + echo "Benchmarking for " $task "currently not supported. Sorry!" + +fi \ No newline at end of file diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/ner_bc5cdr-chem.sh b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/ner_bc5cdr-chem.sh new file mode 100644 index 0000000..321ad76 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/ner_bc5cdr-chem.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +echo "Container nvidia build = " $NVIDIA_BUILD_ID + +init_checkpoint=${1:-"/results/biobert_tf_uncased_base/model.ckpt-4340"} +train_batch_size=${2:-8} +learning_rate=${3:-3.125e-6} +cased=${4:-false} +precision=${5:-"fp16"} +use_xla=${6:-"true"} +num_gpu=${7:-"16"} +seq_length=${8:-128} +bert_model=${9:-"base"} +eval_batch_size=${10:-8} #Eval and Predict BS is assumed to be same +epochs=${11:-"10.0"} + +if [ "$cased" = "true" ] ; then + DO_LOWER_CASE=0 + CASING_DIR_PREFIX="cased" + case_flag="--do_lower_case=False" +else + DO_LOWER_CASE=1 + CASING_DIR_PREFIX="uncased" + case_flag="--do_lower_case=True" +fi + +if [ "$bert_model" = "large" ] ; then + export BERT_DIR=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_L-24_H-1024_A-16 +else + export BERT_DIR=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_L-12_H-768_A-12 +fi + + +export GBS=$(expr $train_batch_size \* $num_gpu) +printf -v TAG "tf_bert_biobert_ner_bc5cdr_chem_%s_%s_gbs%d" "$bert_model" "$precision" $GBS +DATESTAMP=`date +'%y%m%d%H%M%S'` + + +DATASET_DIR=/workspace/bert/data/biobert/BC5CDR/chem +OUTPUT_DIR=/results/${TAG}_${DATESTAMP} +mkdir -p ${OUTPUT_DIR} + +use_fp16="" +if [ "$precision" = "fp16" ] ; then + echo "fp16 activated!" + use_fp16="--use_fp16" +fi + +if [ "$use_xla" = "true" ] ; then + use_xla_tag="--use_xla" + echo "XLA activated" +else + use_xla_tag="" +fi + + +if [ $num_gpu -gt 1 ] ; then + mpi_command="mpirun -np $num_gpu -H localhost:$num_gpu \ + --allow-run-as-root -bind-to none -map-by slot \ + -x NCCL_DEBUG=INFO \ + -x LD_LIBRARY_PATH \ + -x PATH -mca pml ob1 -mca btl ^openib" + use_hvd="--horovod" +else + mpi_command="" + use_hvd="" +fi + +$mpi python /workspace/bert/run_ner.py \ + --do_prepare=true \ + --do_train=true \ + --do_eval=true \ + --do_predict=true \ + --task_name=bc5cdr \ + --vocab_file=$BERT_DIR/vocab.txt \ + --bert_config_file=$BERT_DIR/bert_config.json \ + --init_checkpoint=$init_checkpoint \ + --num_train_epochs=$epochs \ + --data_dir=$DATASET_DIR \ + --output_dir=$OUTPUT_DIR \ + --learning_rate=$learning_rate \ + --train_batch_size=$train_batch_size \ + --eval_batch_size=$eval_batch_size \ + --predict_batch_size=$eval_batch_size \ + --max_seq_length=$seq_length \ + $use_hvd $use_fp16 $use_xla_tag $case_flag diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/ner_bc5cdr-disease.sh b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/ner_bc5cdr-disease.sh new file mode 100644 index 0000000..f0992fa --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/ner_bc5cdr-disease.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +echo "Container nvidia build = " $NVIDIA_BUILD_ID + +init_checkpoint=${1:-"/results/biobert_tf_uncased_base/model.ckpt-4340"} +train_batch_size=${2:-8} +learning_rate=${3:-3.125e-6} +cased=${4:-false} +precision=${5:-"fp16"} +use_xla=${6:-"true"} +num_gpu=${7:-"16"} +seq_length=${8:-128} +bert_model=${9:-"base"} +eval_batch_size=${10:-8} #Eval and Predict BS is assumed to be same +epochs=${11:-"100.0"} + +if [ "$cased" = "true" ] ; then + DO_LOWER_CASE=0 + CASING_DIR_PREFIX="cased" + case_flag="--do_lower_case=False" +else + DO_LOWER_CASE=1 + CASING_DIR_PREFIX="uncased" + case_flag="--do_lower_case=True" +fi + +if [ "$bert_model" = "large" ] ; then + export BERT_DIR=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_L-24_H-1024_A-16 +else + export BERT_DIR=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_L-12_H-768_A-12 +fi + +export GBS=$(expr $train_batch_size \* $num_gpu) +printf -v TAG "tf_bert_biobert_ner_bc5cdr_disease_%s_%s_gbs%d" "$bert_model" "$precision" $GBS +DATESTAMP=`date +'%y%m%d%H%M%S'` + + + +DATASET_DIR=/workspace/bert/data/biobert/BC5CDR/disease +OUTPUT_DIR=/results/${TAG}_${DATESTAMP} +mkdir -p ${OUTPUT_DIR} + +use_fp16="" +if [ "$precision" = "fp16" ] ; then + echo "fp16 activated!" + use_fp16="--use_fp16" +fi + +if [ "$use_xla" = "true" ] ; then + use_xla_tag="--use_xla" + echo "XLA activated" +else + use_xla_tag="" +fi + +if [ $num_gpu -gt 1 ] ; then + mpi_command="mpirun -np $num_gpu -H localhost:$num_gpu \ + --allow-run-as-root -bind-to none -map-by slot \ + -x NCCL_DEBUG=INFO \ + -x LD_LIBRARY_PATH \ + -x PATH -mca pml ob1 -mca btl ^openib" + use_hvd="--horovod" +else + mpi_command="" + use_hvd="" +fi + +$mpi_command python3 /workspace/bert/run_ner.py \ + --do_prepare=true \ + --do_train=true \ + --do_eval=true \ + --do_predict=true \ + --task_name="bc5cdr" \ + --vocab_file=$BERT_DIR/vocab.txt \ + --bert_config_file=$BERT_DIR/bert_config.json \ + --init_checkpoint=$init_checkpoint \ + --num_train_epochs=$epochs \ + --data_dir=$DATASET_DIR \ + --output_dir=$OUTPUT_DIR \ + --learning_rate=$learning_rate \ + --train_batch_size=$train_batch_size \ + --eval_batch_size=$eval_batch_size \ + --predict_batch_size=$eval_batch_size \ + --max_seq_length=$seq_length \ + "$use_hvd" "$use_fp16" $use_xla_tag $case_flag diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/rel_chemprot.sh b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/rel_chemprot.sh new file mode 100644 index 0000000..eae0ea3 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/rel_chemprot.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +echo "Container nvidia build = " $NVIDIA_BUILD_ID + +init_checkpoint=${1:-"/results/biobert_tf_uncased_base/model.ckpt-4340"} +train_batch_size=${2:-64} +learning_rate=${3:-1.5e-6} +cased=${4:-false} +precision=${5:-"fp16"} +use_xla=${6:-"true"} +num_gpu=${7:-"16"} +seq_length=${8:-512} +bert_model=${9:-"base"} +eval_batch_size=${10:-16} #Eval and Predict BS is assumed to be same +epochs=${11:-"3.0"} + +if [ "$cased" = "true" ] ; then + DO_LOWER_CASE=0 + CASING_DIR_PREFIX="cased" + case_flag="--do_lower_case=False" +else + DO_LOWER_CASE=1 + CASING_DIR_PREFIX="uncased" + case_flag="--do_lower_case=True" +fi + +if [ "$bert_model" = "large" ] ; then + export BERT_DIR=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_L-24_H-1024_A-16 +else + export BERT_DIR=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_L-12_H-768_A-12 +fi + +export GBS=$(expr $train_batch_size \* $num_gpu) +printf -v TAG "tf_bert_biobert_rel_chemprot_%s_%s_gbs%d" "$bert_model" "$precision" $GBS +DATESTAMP=`date +'%y%m%d%H%M%S'` + + +DATASET_DIR=/workspace/bert/data/biobert/ChemProt +OUTPUT_DIR=/results/${TAG}_${DATESTAMP} +mkdir -p ${OUTPUT_DIR} + +use_fp16="" +if [ "$precision" = "fp16" ] ; then + echo "fp16 activated!" + use_fp16="--use_fp16" +fi + +if [ "$use_xla" = "true" ] ; then + use_xla_tag="--use_xla" + echo "XLA activated" +else + use_xla_tag="" +fi + +if [ $num_gpu -gt 1 ] ; then + mpi_command="mpirun -np $num_gpu -H localhost:$num_gpu \ + --allow-run-as-root -bind-to none -map-by slot \ + -x NCCL_DEBUG=INFO \ + -x LD_LIBRARY_PATH \ + -x PATH -mca pml ob1 -mca btl ^openib" + use_hvd="--horovod" +else + mpi_command="" + use_hvd="" +fi + +$mpi_command python3 /workspace/bert/run_re.py \ + --do_prepare=true \ + --do_train=true \ + --do_eval=true \ + --do_predict=true \ + --task_name="chemprot" \ + --vocab_file=$BERT_DIR/vocab.txt \ + --bert_config_file=$BERT_DIR/bert_config.json \ + --init_checkpoint=$init_checkpoint \ + --num_train_epochs=$epochs \ + --data_dir=$DATASET_DIR \ + --output_dir=$OUTPUT_DIR \ + --learning_rate=$learning_rate \ + --train_batch_size=$train_batch_size \ + --eval_batch_size=$eval_batch_size \ + --predict_batch_size=$eval_batch_size \ + --max_seq_length=$seq_length \ + "$use_hvd" "$use_fp16" $use_xla_tag $case_flag + +python3 /workspace/bert/biobert/re_eval.py --task=chemprot --output_path=$OUTPUT_DIR/test_results.tsv \ + --answer_path=$DATASET_DIR/test.tsv |& tee $OUTPUT_DIR/test_results.txt diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/run_biobert.sub b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/run_biobert.sub new file mode 100644 index 0000000..c9b87bc --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/run_biobert.sub @@ -0,0 +1,87 @@ +#!/bin/bash +#SBATCH --exclusive +#SBATCH --mem=0 +#SBATCH --overcommit + +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eux + +readonly docker_image="nvcr.io/nvidia/tensorflow:19.08-py3" +readonly datadir="/raid/data/bert" +readonly checkpointdir="$PWD/checkpoints" + +readonly mounts=".:/workspace/bert,${datadir}:/workspace/bert/data,${checkpointdir}:/results" + +DO_LOWER_CASE=${DO_LOWER_CASE:-1} +if [ "$DO_LOWER_CASE" == "1" ]; then + CASING_DIR_PREFIX="uncased" +else + CASING_DIR_PREFIX="cased" +fi + +DO_BERT_BASE=${DO_BERT_BASE:-1} +if [ "$DO_BERT_BASE" == "1" ]; then + CASING_DIR_SUFFIX="L-12_H-768_A-12" +else + CASING_DIR_SUFFIX="L-24_H-1024_A-16" +fi + +srun --ntasks="${SLURM_JOB_NUM_NODES}" --ntasks-per-node=1 mkdir -p "${checkpointdir}/biobert_phase_1" +srun --ntasks="${SLURM_JOB_NUM_NODES}" --ntasks-per-node=1 mkdir -p "${checkpointdir}/biobert_phase_2" + +PHASE1="\ + --train_batch_size=${BATCHSIZE:-128} \ + --learning_rate=${LEARNING_RATE:-3.2e-5} \ + --num_accumulation_steps=${NUM_ACCUMULATION_STEPS:-128} \ + --input_files_dir=lower_case_${DO_LOWER_CASE}_seq_len_128_max_pred_20_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5_shard_1472_test_split_10/pubmed_baseline/training \ + --eval_files_dir=lower_case_${DO_LOWER_CASE}_seq_len_128_max_pred_20_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5_shard_1472_test_split_10/pubmed_baseline/test \ + --max_seq_length=128 \ + --max_predictions_per_seq=20 \ + --num_train_steps=19531 \ + --num_warmup_steps=1953 \ + --output_dir=/results/biobert_phase_1 \ + " + +PHASE2="\ + --train_batch_size=${BATCHSIZE:-16} \ + --learning_rate=${LEARNING_RATE:-6.4e-5} \ + --num_accumulation_steps=${NUM_ACCUMULATION_STEPS:-512} \ + --input_files_dir=/workspace/bert/data/tfrecord/lower_case_${DO_LOWER_CASE}_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5_shard_1472_test_split_10/pubmed_baseline/training \ + --eval_files_dir=/workspace/bert/data/tfrecord/lower_case_${DO_LOWER_CASE}_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5_shard_1472_test_split_10/pubmed_baseline/test \ + --max_seq_length=512 \ + --max_predictions_per_seq=80 \ + --num_train_steps=4340 \ + --num_warmup_steps=434 \ + --output_dir=/results/biobert_phase_2 \ + --init_checkpoint=/results/biobert_phase_1/model.ckpt-19531 \ + " + +PHASES=( "$PHASE1" "$PHASE2" ) + +PHASE=${PHASE:-1} + +BERT_CMD="\ + python /workspace/bert/run_pretraining.py \ + ${PHASES[$((PHASE-1))]} \ + --bert_config_file=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_${CASING_DIR_SUFFIX}/bert_config.json \ + --vocab_file=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_${CASING_DIR_SUFFIX}/vocab.txt \ + --do_train=True \ + --do_eval=True \ + --save_checkpoints_steps=5000 \ + --horovod --use_fp16 --use_xla \ + --allreduce_post_accumulation=True \ + --eval_batch_size=8" + +srun --mpi=pmi2 -l --container-image="${docker_image}" --container-mounts="${mounts}" bash -c "${BERT_CMD}" \ No newline at end of file diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/run_biobert_finetuning_inference.sh b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/run_biobert_finetuning_inference.sh new file mode 100644 index 0000000..9f6f08e --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/run_biobert_finetuning_inference.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +task=${1:-"ner_bc5cdr-chem"} +init_checkpoint=${2:-"/results/biobert_tf_uncased_base/model.ckpt-4340"} +bert_model=${3:-"base"} +cased=${4:-"false"} +precision=${5:-"fp16"} +use_xla=${6:-"true"} +batch_size=${7:-"16"} + +if [ "$cased" = "true" ] ; then + DO_LOWER_CASE=0 + CASING_DIR_PREFIX="cased" + case_flag="--do_lower_case=False" +else + DO_LOWER_CASE=1 + CASING_DIR_PREFIX="uncased" + case_flag="--do_lower_case=True" +fi + +if [ "$bert_model" = "large" ] ; then + export BERT_DIR=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_L-24_H-1024_A-16 +else + export BERT_DIR=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_L-12_H-768_A-12 +fi + +use_fp16="" +if [ "$precision" = "fp16" ] ; then + echo "fp16 activated!" + use_fp16="--use_fp16" +fi + +if [ "$use_xla" = "true" ] ; then + use_xla_tag="--use_xla" + echo "XLA activated" +else + use_xla_tag="" +fi + +DATESTAMP=`date +'%y%m%d%H%M%S'` + +if [ "$task" = "ner_bc5cdr-chem" ] ; then + + printf -v TAG "tf_bert_biobert_ner_bc5cdr_chem_inference_%s_%s" "$bert_model" "$precision" + DATASET_DIR=/workspace/bert/data/biobert/BC5CDR/chem + OUTPUT_DIR=/results/${TAG}_${DATESTAMP} + + python /workspace/bert/run_ner.py \ + --do_prepare=true \ + --do_eval=true \ + --do_predict=true \ + --task_name="bc5cdr" \ + --vocab_file=$BERT_DIR/vocab.txt \ + --bert_config_file=$BERT_DIR/bert_config.json \ + --init_checkpoint=$init_checkpoint \ + --data_dir=$DATASET_DIR \ + --output_dir=$OUTPUT_DIR \ + --eval_batch_size=$batch_size \ + --predict_batch_size=$batch_size \ + --max_seq_length=128 \ + $use_fp16 $use_xla_tag $case_flag + +elif [ "$task" = "ner_bc5cdr-disease" ] ; then + printf -v TAG "tf_bert_biobert_ner_bc5cdr_disease_inference_%s_%s" "$bert_model" "$precision" + DATASET_DIR=/workspace/bert/data/biobert/BC5CDR/disease + OUTPUT_DIR=/results/${TAG}_${DATESTAMP} + + python3 /workspace/bert/run_ner.py \ + --do_prepare=true \ + --do_eval=true \ + --do_predict=true \ + --task_name="bc5cdr" \ + --vocab_file=$BERT_DIR/vocab.txt \ + --bert_config_file=$BERT_DIR/bert_config.json \ + --init_checkpoint=$init_checkpoint \ + --data_dir=$DATASET_DIR \ + --output_dir=$OUTPUT_DIR \ + --eval_batch_size=$batch_size \ + --predict_batch_size=$batch_size \ + --max_seq_length=128 \ + "$use_fp16" $use_xla_tag $case_flag + +elif [ "$task" = "rel_chemprot" ] ; then + printf -v TAG "tf_bert_biobert_rel_chemprot_inference_%s_%s_" "$bert_model" "$precision" + DATASET_DIR=/workspace/bert/data/biobert/ChemProt + OUTPUT_DIR=/results/${TAG}_${DATESTAMP} + + python3 /workspace/bert/run_re.py \ + --do_prepare=true \ + --do_eval=true \ + --do_predict=true \ + --task_name="chemprot" \ + --vocab_file=$BERT_DIR/vocab.txt \ + --bert_config_file=$BERT_DIR/bert_config.json \ + --init_checkpoint=$init_checkpoint \ + --data_dir=$DATASET_DIR \ + --output_dir=$OUTPUT_DIR \ + --eval_batch_size=$batch_size \ + --predict_batch_size=$batch_size \ + --max_seq_length=512 \ + "$use_fp16" $use_xla_tag $case_flag + + python3 /workspace/bert/biobert/re_eval.py --task=chemprot --output_path=$OUTPUT_DIR/test_results.tsv \ + --answer_path=$DATASET_DIR/test.tsv |& tee $OUTPUT_DIR/test_results.txt + +else + + echo "Benchmarking for " $task "currently not supported. Sorry!" + +fi \ No newline at end of file diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/run_pretraining_pubmed_base_phase_1.sh b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/run_pretraining_pubmed_base_phase_1.sh new file mode 100644 index 0000000..8ee9020 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/run_pretraining_pubmed_base_phase_1.sh @@ -0,0 +1,87 @@ +#! /bin/bash + +echo "Container nvidia build = " $NVIDIA_BUILD_ID + +train_batch_size=${1:-128} +learning_rate=${2:-"9.625e-5"} +cased=${3:-false} +precision=${4:-"fp16"} +use_xla=${5:-"true"} +num_gpus=${6:-16} +warmup_steps=${7:-"1953"} +train_steps=${8:-19531} +num_accumulation_steps=${9:-32} +save_checkpoint_steps=${10:-5000} +eval_batch_size=${11:-80} + +use_fp16="" +if [ "$precision" = "fp16" ] ; then + echo "fp16 activated!" + use_fp16="--use_fp16" +fi + +if [ "$use_xla" = "true" ] ; then + use_xla_tag="--use_xla" + echo "XLA activated" +else + use_xla_tag="" +fi + +if [ "$cased" = "true" ] ; then + DO_LOWER_CASE=0 + CASING_DIR_PREFIX="cased" +else + DO_LOWER_CASE=1 + CASING_DIR_PREFIX="uncased" +fi + +BERT_CONFIG=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_L-12_H-768_A-12/bert_config.json +RESULTS_DIR=/results +CHECKPOINTS_DIR=${RESULTS_DIR}/biobert_phase_1 +mkdir -p ${CHECKPOINTS_DIR} + +INIT_CHECKPOINT=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_L-12_H-768_A-12/bert_model.ckpt + +INPUT_FILES_DIR="/workspace/bert/data/tfrecord/lower_case_${DO_LOWER_CASE}_seq_len_128_max_pred_20_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5_shard_1472_test_split_10/pubmed_baseline/training" +EVAL_FILES_DIR="/workspace/bert/data/tfrecord/lower_case_${DO_LOWER_CASE}_seq_len_128_max_pred_20_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5_shard_1472_test_split_10/pubmed_baseline/test" + + +if [ $num_gpu -gt 1 ] ; then + mpi_command="mpirun -np $num_gpu -H localhost:$num_gpu \ + --allow-run-as-root -bind-to none -map-by slot \ + -x NCCL_DEBUG=INFO \ + -x LD_LIBRARY_PATH \ + -x PATH -mca pml ob1 -mca btl ^openib" + use_hvd="--horovod" +else + mpi_command="" + use_hvd="" +fi + + +export GBS=$(expr $train_batch_size \* $num_gpus \* num_accumulation_steps) +printf -v TAG "tf_bert_bio_1n_phase1_cased_%s_%s_gbs%d" "$cased" "$precision" $GBS +DATESTAMP=`date +'%y%m%d%H%M%S'` +LOGFILE=$RESULTS_DIR/$TAG.$DATESTAMP.log +printf "Logs written to %s\n" "$LOGFILE" + + +$mpi python3 /workspace/bert/run_pretraining.py \ + --input_files_dir=$INPUT_FILES_DIR \ + --eval_files_dir=$EVAL_FILES_DIR \ + --output_dir=$CHECKPOINTS_DIR \ + --bert_config_file=$BERT_CONFIG \ + --do_train=True \ + --do_eval=True \ + --train_batch_size=$train_batch_size \ + --eval_batch_size=$eval_batch_size \ + --max_seq_length=128 \ + --max_predictions_per_seq=20 \ + --num_train_steps=$train_steps \ + --num_warmup_steps=$warmup_steps \ + --save_checkpoints_steps=$save_checkpoint_steps \ + --num_accumulation_steps=$num_accumulation_steps \ + --learning_rate=$learning_rate \ + --report_loss \ + --$use_hvd $use_fp16 $use_xla_tag \ + --init_checkpoint=$INIT_CHECKPOINT |& tee $LOGFILE \ No newline at end of file diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/run_pretraining_pubmed_base_phase_2.sh b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/run_pretraining_pubmed_base_phase_2.sh new file mode 100644 index 0000000..a6e41af --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/biobert/scripts/run_pretraining_pubmed_base_phase_2.sh @@ -0,0 +1,85 @@ +#! /bin/bash + +echo "Container nvidia build = " $NVIDIA_BUILD_ID + +init_checkpoint=${1} +train_batch_size=${2:-16} +learning_rate=${3:-"2.9e-4"} +cased=${4:-false} +precision=${5:-"fp16"} +use_xla=${6:-true} +num_gpus=${7:-16} +warmup_steps=${8:-"434"} +train_steps=${9:-4340} +num_accumulation_steps=${10:-128} +save_checkpoint_steps=${11:-5000} +eval_batch_size=${12:-26} + + +use_fp16="" +if [ "$precision" = "fp16" ] ; then + echo "fp16 activated!" + use_fp16="--use_fp16" +fi + +if [ "$use_xla" = "true" ] ; then + use_xla_tag="--use_xla" + echo "XLA activated" +else + use_xla_tag="" +fi + +if [ "$cased" = "true" ] ; then + DO_LOWER_CASE=0 + CASING_DIR_PREFIX="cased" +else + DO_LOWER_CASE=1 + CASING_DIR_PREFIX="uncased" +fi + +BERT_CONFIG=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_L-12_H-768_A-12/bert_config.json +RESULTS_DIR=/results +CHECKPOINTS_DIR=${RESULTS_DIR}/biobert_phase_2 +mkdir -p ${CHECKPOINTS_DIR} + +INPUT_FILES_DIR="/workspace/bert/data/tfrecord/lower_case_${DO_LOWER_CASE}_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5_shard_1472_test_split_10/pubmed_baseline/training" +EVAL_FILES_DIR="/workspace/bert/data/tfrecord/lower_case_${DO_LOWER_CASE}_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5_shard_1472_test_split_10/pubmed_baseline/test" + +if [ $num_gpu -gt 1 ] ; then + mpi_command="mpirun -np $num_gpu -H localhost:$num_gpu \ + --allow-run-as-root -bind-to none -map-by slot \ + -x NCCL_DEBUG=INFO \ + -x LD_LIBRARY_PATH \ + -x PATH -mca pml ob1 -mca btl ^openib" + use_hvd="--horovod" +else + mpi_command="" + use_hvd="" +fi + +export GBS=$(expr $train_batch_size \* $num_gpus \* num_accumulation_steps) +printf -v TAG "tf_bert_bio_1n_phase2_cased_%s_%s_gbs%d" "$cased" "$precision" $GBS +DATESTAMP=`date +'%y%m%d%H%M%S'` +LOGFILE=$RESULTS_DIR/$TAG.$DATESTAMP.log +printf "Logs written to %s\n" "$LOGFILE" + + +$mpi python3 /workspace/bert/run_pretraining.py \ + --input_files_dir=$INPUT_FILES_DIR \ + --eval_files_dir=$EVAL_FILES_DIR \ + --output_dir=$CHECKPOINTS_DIR \ + --bert_config_file=$BERT_CONFIG \ + --do_train=True \ + --do_eval=True \ + --train_batch_size=$train_batch_size \ + --eval_batch_size=$eval_batch_size \ + --max_seq_length=512 \ + --max_predictions_per_seq=80 \ + --num_train_steps=$train_steps \ + --num_warmup_steps=$warmup_steps \ + --save_checkpoints_steps=$save_checkpoint_steps \ + --num_accumulation_steps=$num_accumulation_steps \ + --learning_rate=$learning_rate \ + --report_loss \ + --$use_hvd $use_xla_tag $use_fp16 \ + --init_checkpoint=$INIT_CHECKPOINT |& tee $LOGFILE \ No newline at end of file diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/configurations.yml b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/configurations.yml new file mode 100644 index 0000000..675c6d8 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/configurations.yml @@ -0,0 +1,206 @@ +# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#1 DGX1 phase1 +bert--DGX1: + <<: *BERT_ON_CLUSTER + <<: *DGX1 + variables: + <<: *DGX1_VARS + NNODES: "1" + BATCHSIZE: "8" + LEARNING_RATE: "7.5e-4" + NUM_ACCUMULATION_STEPS: "1024" + PHASE: "1" + +#4 DGX1 phase1 +bert--DGX1_n4: + <<: *BERT_ON_CLUSTER + <<: *DGX1 + variables: + <<: *DGX1_VARS + NNODES: "4" + BATCHSIZE: "8" + LEARNING_RATE: "1.875e-4" + NUM_ACCUMULATION_STEPS: "256" + PHASE: "1" + +#16 DGX1 phase1 +bert--DGX1_n16: + <<: *BERT_ON_CLUSTER + <<: *DGX1 + variables: + <<: *DGX1_VARS + NNODES: "16" + BATCHSIZE: "8" + LEARNING_RATE: "4.6875e-5" + NUM_ACCUMULATION_STEPS: "64" + PHASE: "1" + +#32 DGX1 phase1 +bert--DGX1_n32: + <<: *BERT_ON_CLUSTER + <<: *DGX1 + variables: + <<: *DGX1_VARS + NNODES: "32" + BATCHSIZE: "8" + LEARNING_RATE: "2.34375e-5" + NUM_ACCUMULATION_STEPS: "32" + PHASE: "1" + +#1 DGX2 phase1 +bert--DGX2: + <<: *BERT_ON_CLUSTER + <<: *DGX2 + variables: + <<: *DGX2_VARS + NNODES: "1" + BATCHSIZE: "32" + LEARNING_RATE: "3.75e-4" + NUM_ACCUMULATION_STEPS: "128" + PHASE: "1" + +#4 DGX2 phase1 +bert--DGX2_n4: + <<: *BERT_ON_CLUSTER + <<: *DGX2 + variables: + <<: *DGX2_VARS + NNODES: "4" + BATCHSIZE: "32" + LEARNING_RATE: "9.375e-5" + NUM_ACCUMULATION_STEPS: "32" + PHASE: "1" + +#16 DGX2 phase1 +bert--DGX2_n16: + <<: *BERT_ON_CLUSTER + <<: *DGX2 + variables: + <<: *DGX2_VARS + NNODES: "16" + BATCHSIZE: "256" + LEARNING_RATE: "3.75e-4" + NUM_ACCUMULATION_STEPS: "4" + PHASE: "1" + +#32 DGX2 phase1 +bert--DGX2_n32: + <<: *BERT_ON_CLUSTER + <<: *DGX2 + variables: + <<: *DGX2_VARS + NNODES: "32" + BATCHSIZE: "32" + LEARNING_RATE: "2.34375e-5" + NUM_ACCUMULATION_STEPS: "8" + PHASE: "1" + +#1 DGX1 phase2 +bert--DGX1_n1p2: + <<: *BERT_ON_CLUSTER + <<: *DGX1 + variables: + <<: *DGX1_VARS + NNODES: "1" + BATCHSIZE: "2" + LEARNING_RATE: "5e-4" + NUM_ACCUMULATION_STEPS: "4096" + PHASE: "2" + +#4 DGX1 phase2 +bert--DGX1_n4p2: + <<: *BERT_ON_CLUSTER + <<: *DGX1 + variables: + <<: *DGX1_VARS + NNODES: "4" + BATCHSIZE: "2" + LEARNING_RATE: "1.25e-4" + NUM_ACCUMULATION_STEPS: "512" + PHASE: "2" + +#16 DGX1 phase2 +bert--DGX1_n16p2: + <<: *BERT_ON_CLUSTER + <<: *DGX1 + variables: + <<: *DGX1_VARS + NNODES: "16" + BATCHSIZE: "2" + LEARNING_RATE: "1.5625e-5" + NUM_ACCUMULATION_STEPS: "128" + PHASE: "2" + +#32 DGX1 phase2 +bert--DGX1_n32p2: + <<: *BERT_ON_CLUSTER + <<: *DGX1 + variables: + <<: *DGX1_VARS + NNODES: "32" + BATCHSIZE: "2" + LEARNING_RATE: "1.5625e-5" + NUM_ACCUMULATION_STEPS: "64" + PHASE: "2" + +#1 DGX2 phase2 +bert--DGX2_n1p2: + <<: *BERT_ON_CLUSTER + <<: *DGX2 + variables: + <<: *DGX2_VARS + NNODES: "1" + BATCHSIZE: "8" + LEARNING_RATE: "2.5e-5" + NUM_ACCUMULATION_STEPS: "256" + PHASE: "2" + +#4 DGX2 phase2 +bert--DGX2_n4p2: + <<: *BERT_ON_CLUSTER + <<: *DGX2 + variables: + <<: *DGX2_VARS + NNODES: "4" + BATCHSIZE: "8" + LEARNING_RATE: "6.25e-5" + NUM_ACCUMULATION_STEPS: "64" + PHASE: "2" + +#16 DGX2 phase2 +bert--DGX2_n16p2: + <<: *BERT_ON_CLUSTER + <<: *DGX2 + variables: + <<: *DGX2_VARS + NNODES: "16" + BATCHSIZE: "8" + LEARNING_RATE: "1.5625e-5" + NUM_ACCUMULATION_STEPS: "16" + PHASE: "2" + +#32 DGX2 phase2 +bert--DGX2_n32p2: + <<: *BERT_ON_CLUSTER + <<: *DGX2 + variables: + <<: *DGX2_VARS + NNODES: "32" + BATCHSIZE: "8" + LEARNING_RATE: "7.8125e-6" + NUM_ACCUMULATION_STEPS: "8" + PHASE: "2" + diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/BooksDownloader.py b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/BooksDownloader.py new file mode 100644 index 0000000..53ee6c4 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/BooksDownloader.py @@ -0,0 +1,26 @@ +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess + +class BooksDownloader: + def __init__(self, save_path): + self.save_path = save_path + pass + + + def download(self): + bookscorpus_download_command = 'python3 /workspace/bookcorpus/download_files.py --list /workspace/bookcorpus/url_list.jsonl --out' + bookscorpus_download_command += ' ' + self.save_path + '/bookscorpus' + bookscorpus_download_command += ' --trash-bad-count' + bookscorpus_download_process = subprocess.run(bookscorpus_download_command, shell=True, check=True) \ No newline at end of file diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/BookscorpusTextFormatting.py b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/BookscorpusTextFormatting.py new file mode 100644 index 0000000..22e48d4 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/BookscorpusTextFormatting.py @@ -0,0 +1,32 @@ +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import os + +class BookscorpusTextFormatting: + def __init__(self, books_path, output_filename, recursive = False): + self.books_path = books_path + self.recursive = recursive + self.output_filename = output_filename + + + # This puts one book per line + def merge(self): + with open(self.output_filename, mode='w', newline='\n') as ofile: + for filename in glob.glob(self.books_path + '/' + '*.txt', recursive=True): + with open(filename, mode='r', encoding='utf-8-sig', newline='\n') as file: + for line in file: + if line.strip() != '': + ofile.write(line.strip() + ' ') + ofile.write("\n\n") \ No newline at end of file diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/Downloader.py b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/Downloader.py new file mode 100644 index 0000000..20b48c1 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/Downloader.py @@ -0,0 +1,120 @@ +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from GooglePretrainedWeightDownloader import GooglePretrainedWeightDownloader +from NVIDIAPretrainedWeightDownloader import NVIDIAPretrainedWeightDownloader +from WikiDownloader import WikiDownloader +from BooksDownloader import BooksDownloader +from GLUEDownloader import GLUEDownloader +from SquadDownloader import SquadDownloader +from PubMedDownloader import PubMedDownloader + +class Downloader: + def __init__(self, dataset_name, save_path): + self.dataset_name = dataset_name + self.save_path = save_path + + + def download(self): + if self.dataset_name == 'bookscorpus': + self.download_bookscorpus() + + elif self.dataset_name == 'wikicorpus_en': + self.download_wikicorpus('en') + + elif self.dataset_name == 'wikicorpus_zh': + self.download_wikicorpus('zh') + + elif self.dataset_name == 'pubmed_baseline': + self.download_pubmed('baseline') + + elif self.dataset_name == 'pubmed_daily_update': + self.download_pubmed('daily_update') + + elif self.dataset_name == 'pubmed_fulltext': + self.download_pubmed('fulltext') + + elif self.dataset_name == 'pubmed_open_access': + self.download_pubmed('open_access') + + elif self.dataset_name == 'google_pretrained_weights': + self.download_google_pretrained_weights() + + elif self.dataset_name == 'nvidia_pretrained_weights': + self.download_nvidia_pretrained_weights() + + elif self.dataset_name == 'MRPC': + self.download_glue(self.dataset_name) + + elif self.dataset_name == 'MNLI': + self.download_glue(self.dataset_name) + + elif self.dataset_name == 'CoLA': + self.download_glue(self.dataset_name) + + elif self.dataset_name == 'squad': + self.download_squad() + + elif self.dataset_name == 'all': + self.download_bookscorpus() + self.download_wikicorpus('en') + self.download_wikicorpus('zh') + self.download_pubmed('baseline') + self.download_pubmed('daily_update') + self.download_pubmed('fulltext') + self.download_pubmed('open_access') + self.download_google_pretrained_weights() + self.download_nvidia_pretrained_weights() + self.download_glue("CoLA") + self.download_glue("MNLI") + self.download_glue("MRPC") + self.download_squad() + + else: + print(self.dataset_name) + assert False, 'Unknown dataset_name provided to downloader' + + + def download_bookscorpus(self): + downloader = BooksDownloader(self.save_path) + downloader.download() + + + def download_wikicorpus(self, language): + downloader = WikiDownloader(language, self.save_path) + downloader.download() + + + def download_pubmed(self, subset): + downloader = PubMedDownloader(subset, self.save_path) + downloader.download() + + + def download_google_pretrained_weights(self): + downloader = GooglePretrainedWeightDownloader(self.save_path) + downloader.download() + + + def download_nvidia_pretrained_weights(self): + downloader = NVIDIAPretrainedWeightDownloader(self.save_path) + downloader.download() + + + def download_glue(self, glue_task_name): + downloader = GLUEDownloader(glue_task_name, self.save_path) + downloader.download() + + + def download_squad(self): + downloader = SquadDownloader(self.save_path) + downloader.download() diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/GLUEDownloader.py b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/GLUEDownloader.py new file mode 100644 index 0000000..e270b37 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/GLUEDownloader.py @@ -0,0 +1,109 @@ +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import bz2 +import os +import urllib +import sys +import zipfile +import io + +URLLIB=urllib +if sys.version_info >= (3, 0): + URLLIB=urllib.request + +class GLUEDownloader: + def __init__(self, task, save_path): + + # Documentation - Download link obtained from here: https://github.com/nyu-mll/GLUE-baselines/blob/master/download_glue_data.py + + self.TASK2PATH = {"CoLA":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FCoLA.zip?alt=media&token=46d5e637-3411-4188-bc44-5809b5bfb5f4', + "SST":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8', + "MRPC":{"mrpc_dev": 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2Fmrpc_dev_ids.tsv?alt=media&token=ec5c0836-31d5-48f4-b431-7480817f1adc', + "mrpc_train": 'https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_train.txt', + "mrpc_test": 'https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_test.txt'}, + "QQP":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FQQP.zip?alt=media&token=700c6acf-160d-4d89-81d1-de4191d02cb5', + "STS":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSTS-B.zip?alt=media&token=bddb94a7-8706-4e0d-a694-1109e12273b5', + "MNLI":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FMNLI.zip?alt=media&token=50329ea1-e339-40e2-809c-10c40afff3ce', + "SNLI":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSNLI.zip?alt=media&token=4afcfbb2-ff0c-4b2d-a09a-dbf07926f4df', + "QNLI":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FQNLI.zip?alt=media&token=c24cad61-f2df-4f04-9ab6-aa576fa829d0', + "RTE":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FRTE.zip?alt=media&token=5efa7e85-a0bb-4f19-8ea2-9e1840f077fb', + "WNLI":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FWNLI.zip?alt=media&token=068ad0a0-ded7-4bd7-99a5-5e00222e0faf', + "diagnostic":'https://storage.googleapis.com/mtl-sentence-representations.appspot.com/tsvsWithoutLabels%2FAX.tsv?GoogleAccessId=firebase-adminsdk-0khhl@mtl-sentence-representations.iam.gserviceaccount.com&Expires=2498860800&Signature=DuQ2CSPt2Yfre0C%2BiISrVYrIFaZH1Lc7hBVZDD4ZyR7fZYOMNOUGpi8QxBmTNOrNPjR3z1cggo7WXFfrgECP6FBJSsURv8Ybrue8Ypt%2FTPxbuJ0Xc2FhDi%2BarnecCBFO77RSbfuz%2Bs95hRrYhTnByqu3U%2FYZPaj3tZt5QdfpH2IUROY8LiBXoXS46LE%2FgOQc%2FKN%2BA9SoscRDYsnxHfG0IjXGwHN%2Bf88q6hOmAxeNPx6moDulUF6XMUAaXCSFU%2BnRO2RDL9CapWxj%2BDl7syNyHhB7987hZ80B%2FwFkQ3MEs8auvt5XW1%2Bd4aCU7ytgM69r8JDCwibfhZxpaa4gd50QXQ%3D%3D'} + + + self.save_path = save_path + if not os.path.exists(self.save_path): + os.makedirs(self.save_path) + + self.task = task + + def download(self): + + if self.task == 'MRPC': + self.download_mrpc() + elif self.task == 'diagnostic': + self.download_diagnostic() + else: + self.download_and_extract(self.task) + + def download_and_extract(self, task): + print("Downloading and extracting %s..." % task) + data_file = "%s.zip" % task + URLLIB.urlretrieve(self.TASK2PATH[task], data_file) + print(data_file,"\n\n\n") + with zipfile.ZipFile(data_file) as zip_ref: + zip_ref.extractall(self.save_path) + os.remove(data_file) + print("\tCompleted!") + + def download_mrpc(self): + print("Processing MRPC...") + mrpc_dir = os.path.join(self.save_path, "MRPC") + if not os.path.isdir(mrpc_dir): + os.mkdir(mrpc_dir) + + mrpc_train_file = os.path.join(mrpc_dir, "msr_paraphrase_train.txt") + mrpc_dev_file = os.path.join(mrpc_dir, "dev_ids.tsv") + mrpc_test_file = os.path.join(mrpc_dir, "msr_paraphrase_test.txt") + + URLLIB.urlretrieve(self.TASK2PATH["MRPC"]["mrpc_train"], mrpc_train_file) + URLLIB.urlretrieve(self.TASK2PATH["MRPC"]["mrpc_test"], mrpc_test_file) + URLLIB.urlretrieve(self.TASK2PATH["MRPC"]["mrpc_dev"], mrpc_dev_file) + + dev_ids = [] + with io.open(os.path.join(mrpc_dir, "dev_ids.tsv"), encoding='utf-8') as ids_fh: + for row in ids_fh: + dev_ids.append(row.strip().split('\t')) + + with io.open(mrpc_train_file, encoding='utf-8') as data_fh, \ + io.open(os.path.join(mrpc_dir, "train.tsv"), 'w', encoding='utf-8') as train_fh, \ + io.open(os.path.join(mrpc_dir, "dev.tsv"), 'w', encoding='utf-8') as dev_fh: + header = data_fh.readline() + train_fh.write(header) + dev_fh.write(header) + for row in data_fh: + label, id1, id2, s1, s2 = row.strip().split('\t') + if [id1, id2] in dev_ids: + dev_fh.write("%s\t%s\t%s\t%s\t%s\n" % (label, id1, id2, s1, s2)) + else: + train_fh.write("%s\t%s\t%s\t%s\t%s\n" % (label, id1, id2, s1, s2)) + + with io.open(mrpc_test_file, encoding='utf-8') as data_fh, \ + io.open(os.path.join(mrpc_dir, "test.tsv"), 'w', encoding='utf-8') as test_fh: + header = data_fh.readline() + test_fh.write("index\t#1 ID\t#2 ID\t#1 String\t#2 String\n") + for idx, row in enumerate(data_fh): + label, id1, id2, s1, s2 = row.strip().split('\t') + test_fh.write("%d\t%s\t%s\t%s\t%s\n" % (idx, id1, id2, s1, s2)) + print("\tCompleted!") \ No newline at end of file diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/GooglePretrainedWeightDownloader.py b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/GooglePretrainedWeightDownloader.py new file mode 100644 index 0000000..bb0684d --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/GooglePretrainedWeightDownloader.py @@ -0,0 +1,158 @@ +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import os +import urllib.request +import zipfile + +class GooglePretrainedWeightDownloader: + def __init__(self, save_path): + self.save_path = save_path + '/google_pretrained_weights' + + if not os.path.exists(self.save_path): + os.makedirs(self.save_path) + + # Download urls + self.model_urls = { + 'bert_base_uncased': ('https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip', 'uncased_L-12_H-768_A-12.zip'), + 'bert_large_uncased': ('https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-24_H-1024_A-16.zip', 'uncased_L-24_H-1024_A-16.zip'), + 'bert_base_cased': ('https://storage.googleapis.com/bert_models/2018_10_18/cased_L-12_H-768_A-12.zip', 'cased_L-12_H-768_A-12.zip'), + 'bert_large_cased': ('https://storage.googleapis.com/bert_models/2018_10_18/cased_L-24_H-1024_A-16.zip', 'cased_L-24_H-1024_A-16.zip'), + 'bert_base_multilingual_cased': ('https://storage.googleapis.com/bert_models/2018_11_23/multi_cased_L-12_H-768_A-12.zip', 'multi_cased_L-12_H-768_A-12.zip'), + 'bert_large_multilingual_uncased': ('https://storage.googleapis.com/bert_models/2018_11_03/multilingual_L-12_H-768_A-12.zip', 'multilingual_L-12_H-768_A-12.zip'), + 'bert_base_chinese': ('https://storage.googleapis.com/bert_models/2018_11_03/chinese_L-12_H-768_A-12.zip', 'chinese_L-12_H-768_A-12.zip') + } + + # SHA256sum verification for file download integrity (and checking for changes from the download source over time) + self.bert_base_uncased_sha = { + 'bert_config.json': '7b4e5f53efbd058c67cda0aacfafb340113ea1b5797d9ce6ee411704ba21fcbc', + 'bert_model.ckpt.data-00000-of-00001': '58580dc5e0bf0ae0d2efd51d0e8272b2f808857f0a43a88aaf7549da6d7a8a84', + 'bert_model.ckpt.index': '04c1323086e2f1c5b7c0759d8d3e484afbb0ab45f51793daab9f647113a0117b', + 'bert_model.ckpt.meta': 'dd5682170a10c3ea0280c2e9b9a45fee894eb62da649bbdea37b38b0ded5f60e', + 'vocab.txt': '07eced375cec144d27c900241f3e339478dec958f92fddbc551f295c992038a3', + } + + self.bert_large_uncased_sha = { + 'bert_config.json': 'bfa42236d269e2aeb3a6d30412a33d15dbe8ea597e2b01dc9518c63cc6efafcb', + 'bert_model.ckpt.data-00000-of-00001': 'bc6b3363e3be458c99ecf64b7f472d2b7c67534fd8f564c0556a678f90f4eea1', + 'bert_model.ckpt.index': '68b52f2205ffc64dc627d1120cf399c1ef1cbc35ea5021d1afc889ffe2ce2093', + 'bert_model.ckpt.meta': '6fcce8ff7628f229a885a593625e3d5ff9687542d5ef128d9beb1b0c05edc4a1', + 'vocab.txt': '07eced375cec144d27c900241f3e339478dec958f92fddbc551f295c992038a3', + } + + self.bert_base_cased_sha = { + 'bert_config.json': 'f11dfb757bea16339a33e1bf327b0aade6e57fd9c29dc6b84f7ddb20682f48bc', + 'bert_model.ckpt.data-00000-of-00001': '734d5a1b68bf98d4e9cb6b6692725d00842a1937af73902e51776905d8f760ea', + 'bert_model.ckpt.index': '517d6ef5c41fc2ca1f595276d6fccf5521810d57f5a74e32616151557790f7b1', + 'bert_model.ckpt.meta': '5f8a9771ff25dadd61582abb4e3a748215a10a6b55947cbb66d0f0ba1694be98', + 'vocab.txt': 'eeaa9875b23b04b4c54ef759d03db9d1ba1554838f8fb26c5d96fa551df93d02', + } + + self.bert_large_cased_sha = { + 'bert_config.json': '7adb2125c8225da495656c982fd1c5f64ba8f20ad020838571a3f8a954c2df57', + 'bert_model.ckpt.data-00000-of-00001': '6ff33640f40d472f7a16af0c17b1179ca9dcc0373155fb05335b6a4dd1657ef0', + 'bert_model.ckpt.index': 'ef42a53f577fbe07381f4161b13c7cab4f4fc3b167cec6a9ae382c53d18049cf', + 'bert_model.ckpt.meta': 'd2ddff3ed33b80091eac95171e94149736ea74eb645e575d942ec4a5e01a40a1', + 'vocab.txt': 'eeaa9875b23b04b4c54ef759d03db9d1ba1554838f8fb26c5d96fa551df93d02', + } + + self.bert_base_multilingual_cased_sha = { + 'bert_config.json': 'e76c3964bc14a8bb37a5530cdc802699d2f4a6fddfab0611e153aa2528f234f0', + 'bert_model.ckpt.data-00000-of-00001': '55b8a2df41f69c60c5180e50a7c31b7cdf6238909390c4ddf05fbc0d37aa1ac5', + 'bert_model.ckpt.index': '7d8509c2a62b4e300feb55f8e5f1eef41638f4998dd4d887736f42d4f6a34b37', + 'bert_model.ckpt.meta': '95e5f1997e8831f1c31e5cf530f1a2e99f121e9cd20887f2dce6fe9e3343e3fa', + 'vocab.txt': 'fe0fda7c425b48c516fc8f160d594c8022a0808447475c1a7c6d6479763f310c', + } + + self.bert_large_multilingual_uncased_sha = { + 'bert_config.json': '49063bb061390211d2fdd108cada1ed86faa5f90b80c8f6fdddf406afa4c4624', + 'bert_model.ckpt.data-00000-of-00001': '3cd83912ebeb0efe2abf35c9f1d5a515d8e80295e61c49b75c8853f756658429', + 'bert_model.ckpt.index': '87c372c1a3b1dc7effaaa9103c80a81b3cbab04c7933ced224eec3b8ad2cc8e7', + 'bert_model.ckpt.meta': '27f504f34f02acaa6b0f60d65195ec3e3f9505ac14601c6a32b421d0c8413a29', + 'vocab.txt': '87b44292b452f6c05afa49b2e488e7eedf79ea4f4c39db6f2f4b37764228ef3f', + } + + self.bert_base_chinese_sha = { + 'bert_config.json': '7aaad0335058e2640bcb2c2e9a932b1cd9da200c46ea7b8957d54431f201c015', + 'bert_model.ckpt.data-00000-of-00001': '756699356b78ad0ef1ca9ba6528297bcb3dd1aef5feadd31f4775d7c7fc989ba', + 'bert_model.ckpt.index': '46315546e05ce62327b3e2cd1bed22836adcb2ff29735ec87721396edb21b82e', + 'bert_model.ckpt.meta': 'c0f8d51e1ab986604bc2b25d6ec0af7fd21ff94cf67081996ec3f3bf5d823047', + 'vocab.txt': '45bbac6b341c319adc98a532532882e91a9cefc0329aa57bac9ae761c27b291c', + } + + # Relate SHA to urls for loop below + self.model_sha = { + 'bert_base_uncased': self.bert_base_uncased_sha, + 'bert_large_uncased': self.bert_large_uncased_sha, + 'bert_base_cased': self.bert_base_cased_sha, + 'bert_large_cased': self.bert_large_cased_sha, + 'bert_base_multilingual_cased': self.bert_base_multilingual_cased_sha, + 'bert_large_multilingual_uncased': self.bert_large_multilingual_uncased_sha, + 'bert_base_chinese': self.bert_base_chinese_sha + } + + # Helper to get sha256sum of a file + def sha256sum(self, filename): + h = hashlib.sha256() + b = bytearray(128*1024) + mv = memoryview(b) + with open(filename, 'rb', buffering=0) as f: + for n in iter(lambda : f.readinto(mv), 0): + h.update(mv[:n]) + + return h.hexdigest() + + def download(self): + # Iterate over urls: download, unzip, verify sha256sum + found_mismatch_sha = False + for model in self.model_urls: + url = self.model_urls[model][0] + file = self.save_path + '/' + self.model_urls[model][1] + + print('Downloading', url) + response = urllib.request.urlopen(url) + with open(file, 'wb') as handle: + handle.write(response.read()) + + print('Unzipping', file) + zip = zipfile.ZipFile(file, 'r') + zip.extractall(self.save_path) + zip.close() + + sha_dict = self.model_sha[model] + for extracted_file in sha_dict: + sha = sha_dict[extracted_file] + if sha != self.sha256sum(file[:-4] + '/' + extracted_file): + found_mismatch_sha = True + print('SHA256sum does not match on file:', extracted_file, 'from download url:', url) + else: + print(file[:-4] + '/' + extracted_file, '\t', 'verified') + + if not found_mismatch_sha: + print("All downloads pass sha256sum verification.") + + def serialize(self): + pass + + def deserialize(self): + pass + + def listAvailableWeights(self): + print("Available Weight Datasets") + for item in self.model_urls: + print(item) + + def listLocallyStoredWeights(self): + pass + diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/NVIDIAPretrainedWeightDownloader.py b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/NVIDIAPretrainedWeightDownloader.py new file mode 100644 index 0000000..13c9a32 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/NVIDIAPretrainedWeightDownloader.py @@ -0,0 +1,27 @@ +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +class NVIDIAPretrainedWeightDownloader: + def __init__(self, save_path): + self.save_path = save_path + '/nvidia_pretrained_weights' + + if not os.path.exists(self.save_path): + os.makedirs(self.save_path) + + pass + + + def download(self): + assert False, 'NVIDIAPretrainedWeightDownloader not implemented yet.' \ No newline at end of file diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/PubMedDownloader.py b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/PubMedDownloader.py new file mode 100644 index 0000000..a2aef07 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/PubMedDownloader.py @@ -0,0 +1,93 @@ +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import bz2 +import glob +import gzip +import os +import urllib.request +import shutil +import sys + +class PubMedDownloader: + def __init__(self, subset, save_path): + self.subset = subset + # Modifying self.save_path in two steps to handle creation of subdirectories + self.save_path = save_path + '/pubmed' + '/' + + if not os.path.exists(self.save_path): + os.makedirs(self.save_path) + + self.save_path = self.save_path + '/' + subset + + if not os.path.exists(self.save_path): + os.makedirs(self.save_path) + + self.download_urls = { + 'baseline' : 'ftp://ftp.ncbi.nlm.nih.gov/pubmed/baseline/', + 'daily_update' : 'ftp://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/', + 'fulltext' : 'ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_bulk/', + 'open_access' : 'ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_bulk/' + } + + + def download(self): + print('subset:', self.subset) + url = self.download_urls[self.subset] + self.download_files(url) + self.extract_files() + + + def download_files(self, url): + url = self.download_urls[self.subset] + output = os.popen('curl ' + url).read() + + if self.subset == 'fulltext' or self.subset == 'open_access': + line_split = 'comm_use' if self.subset == 'fulltext' else 'non_comm_use' + for line in output.splitlines(): + if line[-10:] == 'xml.tar.gz' and \ + line.split(' ')[-1].split('.')[0] == line_split: + file = os.path.join(self.save_path, line.split(' ')[-1]) + if not os.path.isfile(file): + print('Downloading', file) + response = urllib.request.urlopen(url + line.split(' ')[-1]) + with open(file, "wb") as handle: + handle.write(response.read()) + + elif self.subset == 'baseline' or self.subset == 'daily_update': + for line in output.splitlines(): + if line[-3:] == '.gz': + file = os.path.join(self.save_path, line.split(' ')[-1]) + if not os.path.isfile(file): + print('Downloading', file) + response = urllib.request.urlopen(url + line.split(' ')[-1]) + with open(file, "wb") as handle: + handle.write(response.read()) + else: + assert False, 'Invalid PubMed dataset/subset specified.' + + def extract_files(self): + files = glob.glob(self.save_path + '/*.xml.gz') + + for file in files: + print('file:', file) + input = gzip.GzipFile(file, mode='rb') + s = input.read() + input.close() + + out = open(file[:-3], mode='wb') + out.write(s) + out.close() + + + diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/PubMedTextFormatting.py b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/PubMedTextFormatting.py new file mode 100644 index 0000000..df85178 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/PubMedTextFormatting.py @@ -0,0 +1,44 @@ +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import os +import pubmed_parser as pmp + +class PubMedTextFormatting: + def __init__(self, pubmed_path, output_filename, recursive = False): + self.pubmed_path = pubmed_path + self.recursive = recursive + self.output_filename = output_filename + + + # This puts one article per line + def merge(self): + print('PubMed path:', self.pubmed_path) + + with open(self.output_filename, mode='w', newline='\n') as ofile: + for filename in glob.glob(self.pubmed_path + '/*.xml*', recursive=self.recursive): + print('file:', filename) + dicts_out = pmp.parse_medline_xml(filename) + for dict_out in dicts_out: + if not dict_out['abstract']: + continue + try: + for line in dict_out['abstract'].splitlines(): + if len(line) < 30: + continue + ofile.write(line.strip() + " ") + ofile.write("\n\n") + except: + ofile.write("\n\n") + continue diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/README.md b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/README.md new file mode 100644 index 0000000..d7544da --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/README.md @@ -0,0 +1,32 @@ +Steps to reproduce datasets from web + +1) Build the container + * docker build -t bert_tf . +2) Run the container interactively + * nvidia-docker run -it --ipc=host bert_tf + * Optional: Mount data volumes + * -v yourpath:/workspace/bert/data/wikipedia_corpus/download + * -v yourpath:/workspace/bert/data/wikipedia_corpus/extracted_articles + * -v yourpath:/workspace/bert/data/wikipedia_corpus/raw_data + * -v yourpath:/workspace/bert/data/wikipedia_corpus/intermediate_files + * -v yourpath:/workspace/bert/data/wikipedia_corpus/final_text_file_single + * -v yourpath:/workspace/bert/data/wikipedia_corpus/final_text_files_sharded + * -v yourpath:/workspace/bert/data/wikipedia_corpus/final_tfrecords_sharded + * -v yourpath:/workspace/bert/data/bookcorpus/download + * -v yourpath:/workspace/bert/data/bookcorpus/final_text_file_single + * -v yourpath:/workspace/bert/data/bookcorpus/final_text_files_sharded + * -v yourpath:/workspace/bert/data/bookcorpus/final_tfrecords_sharded + * Optional: Select visible GPUs + * -e CUDA_VISIBLE_DEVICES=0 + +** Inside of the container starting here** +3) Download pretrained weights (they contain vocab files for preprocessing) + * cd data/pretrained_models_google && python3 download_models.py +4) "One-click" SQuAD download + * cd /workspace/bert/data/squad && . squad_download.sh +5) "One-click" Wikipedia data download and prep (provides tfrecords) + * Set your configuration in data/wikipedia_corpus/config.sh + * cd /data/wikipedia_corpus && ./run_preprocessing.sh +6) "One-click" BookCorpus data download and prep (provided tfrecords) + * Set your configuration in data/wikipedia_corpus/config.sh + * cd /data/bookcorpus && ./run_preprocessing.sh diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/SquadDownloader.py b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/SquadDownloader.py new file mode 100644 index 0000000..6d64ffc --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/SquadDownloader.py @@ -0,0 +1,54 @@ +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import bz2 +import os +import urllib.request +import sys + +class SquadDownloader: + def __init__(self, save_path): + self.save_path = save_path + '/squad' + + if not os.path.exists(self.save_path): + os.makedirs(self.save_path) + + if not os.path.exists(self.save_path + '/v1.1'): + os.makedirs(self.save_path + '/v1.1') + + if not os.path.exists(self.save_path + '/v2.0'): + os.makedirs(self.save_path + '/v2.0') + + self.download_urls = { + 'https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json' : 'v1.1/train-v1.1.json', + 'https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json' : 'v1.1/dev-v1.1.json', + 'https://worksheets.codalab.org/rest/bundles/0xbcd57bee090b421c982906709c8c27e1/contents/blob/' : 'v1.1/evaluate-v1.1.py', + 'https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json' : 'v2.0/train-v2.0.json', + 'https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json' : 'v2.0/dev-v2.0.json', + 'https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/' : 'v2.0/evaluate-v2.0.py', + } + + def download(self): + for item in self.download_urls: + url = item + file = self.download_urls[item] + + print('Downloading:', url) + if os.path.isfile(self.save_path + '/' + file): + print('** Download file already exists, skipping download') + else: + response = urllib.request.urlopen(url) + with open(self.save_path + '/' + file, "wb") as handle: + handle.write(response.read()) + + diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/TextSharding.py b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/TextSharding.py new file mode 100644 index 0000000..85012a5 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/TextSharding.py @@ -0,0 +1,331 @@ +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import defaultdict +from itertools import islice + +import multiprocessing +import os +import statistics + +class Sharding: + def __init__(self, input_files, output_name_prefix, n_training_shards, n_test_shards, fraction_test_set): + assert len(input_files) > 0, 'The input file list must contain at least one file.' + assert n_training_shards > 0, 'There must be at least one output shard.' + assert n_test_shards > 0, 'There must be at least one output shard.' + + self.n_training_shards = n_training_shards + self.n_test_shards = n_test_shards + self.fraction_test_set = fraction_test_set + + self.input_files = input_files + + self.output_name_prefix = output_name_prefix + self.output_training_identifier = '_training' + self.output_test_identifier = '_test' + self.output_file_extension = '.txt' + + self.articles = {} # key: integer identifier, value: list of articles + self.sentences = {} # key: integer identifier, value: list of sentences + self.output_training_files = {} # key: filename, value: list of articles to go into file + self.output_test_files = {} # key: filename, value: list of articles to go into file + + self.init_output_files() + + + # Remember, the input files contain one article per line (the whitespace check is to skip extraneous blank lines) + def load_articles(self): + print('Start: Loading Articles') + + global_article_count = 0 + for input_file in self.input_files: + print('input file:', input_file) + with open(input_file, mode='r', newline='\n') as f: + for i, line in enumerate(f): + if line.strip(): + self.articles[global_article_count] = line.rstrip() + global_article_count += 1 + + print('End: Loading Articles: There are', len(self.articles), 'articles.') + + + def segment_articles_into_sentences(self, segmenter): + print('Start: Sentence Segmentation') + if len(self.articles) is 0: + self.load_articles() + + assert len(self.articles) is not 0, 'Please check that input files are present and contain data.' + + # TODO: WIP: multiprocessing (create independent ranges and spawn processes) + use_multiprocessing = 'serial' + + def chunks(data, size=len(self.articles)): + it = iter(data) + for i in range(0, len(data), size): + yield {k: data[k] for k in islice(it, size)} + + if use_multiprocessing == 'manager': + manager = multiprocessing.Manager() + return_dict = manager.dict() + jobs = [] + n_processes = 7 # in addition to the main process, total = n_proc+1 + + def work(articles, return_dict): + sentences = {} + for i, article in enumerate(articles): + sentences[i] = segmenter.segment_string(articles[article]) + + if i % 5000 == 0: + print('Segmenting article', i) + + return_dict.update(sentences) + + for item in chunks(self.articles, len(self.articles)): + p = multiprocessing.Process(target=work, args=(item, return_dict)) + + # Busy wait + while len(jobs) >= n_processes: + pass + + jobs.append(p) + p.start() + + for proc in jobs: + proc.join() + + elif use_multiprocessing == 'queue': + work_queue = multiprocessing.Queue() + jobs = [] + + for item in chunks(self.articles, len(self.articles)): + pass + + else: # serial option + for i, article in enumerate(self.articles): + self.sentences[i] = segmenter.segment_string(self.articles[article]) + + if i % 5000 == 0: + print('Segmenting article', i) + + print('End: Sentence Segmentation') + + + def init_output_files(self): + print('Start: Init Output Files') + assert len(self.output_training_files) is 0, 'Internal storage self.output_files already contains data. This function is intended to be used by the constructor only.' + assert len(self.output_test_files) is 0, 'Internal storage self.output_files already contains data. This function is intended to be used by the constructor only.' + + for i in range(self.n_training_shards): + name = self.output_name_prefix + self.output_training_identifier + '_' + str(i) + self.output_file_extension + self.output_training_files[name] = [] + + for i in range(self.n_test_shards): + name = self.output_name_prefix + self.output_test_identifier + '_' + str(i) + self.output_file_extension + self.output_test_files[name] = [] + + print('End: Init Output Files') + + + def get_sentences_per_shard(self, shard): + result = 0 + for article_id in shard: + result += len(self.sentences[article_id]) + + return result + + + def distribute_articles_over_shards(self): + print('Start: Distribute Articles Over Shards') + assert len(self.articles) >= self.n_training_shards + self.n_test_shards, 'There are fewer articles than shards. Please add more data or reduce the number of shards requested.' + + # Create dictionary with - key: sentence count per article, value: article id number + sentence_counts = defaultdict(lambda: []) + + max_sentences = 0 + total_sentences = 0 + + for article_id in self.sentences: + current_length = len(self.sentences[article_id]) + sentence_counts[current_length].append(article_id) + max_sentences = max(max_sentences, current_length) + total_sentences += current_length + + n_sentences_assigned_to_training = int((1 - self.fraction_test_set) * total_sentences) + nominal_sentences_per_training_shard = n_sentences_assigned_to_training // self.n_training_shards + nominal_sentences_per_test_shard = (total_sentences - n_sentences_assigned_to_training) // self.n_test_shards + + consumed_article_set = set({}) + unused_article_set = set(self.articles.keys()) + + # Make first pass and add one article worth of lines per file + for file in self.output_training_files: + current_article_id = sentence_counts[max_sentences][-1] + sentence_counts[max_sentences].pop(-1) + self.output_training_files[file].append(current_article_id) + consumed_article_set.add(current_article_id) + unused_article_set.remove(current_article_id) + + # Maintain the max sentence count + while len(sentence_counts[max_sentences]) == 0 and max_sentences > 0: + max_sentences -= 1 + + if len(self.sentences[current_article_id]) > nominal_sentences_per_training_shard: + nominal_sentences_per_training_shard = len(self.sentences[current_article_id]) + print('Warning: A single article contains more than the nominal number of sentences per training shard.') + + for file in self.output_test_files: + current_article_id = sentence_counts[max_sentences][-1] + sentence_counts[max_sentences].pop(-1) + self.output_test_files[file].append(current_article_id) + consumed_article_set.add(current_article_id) + unused_article_set.remove(current_article_id) + + # Maintain the max sentence count + while len(sentence_counts[max_sentences]) == 0 and max_sentences > 0: + max_sentences -= 1 + + if len(self.sentences[current_article_id]) > nominal_sentences_per_test_shard: + nominal_sentences_per_test_shard = len(self.sentences[current_article_id]) + print('Warning: A single article contains more than the nominal number of sentences per test shard.') + + training_counts = [] + test_counts = [] + + for shard in self.output_training_files: + training_counts.append(self.get_sentences_per_shard(self.output_training_files[shard])) + + for shard in self.output_test_files: + test_counts.append(self.get_sentences_per_shard(self.output_test_files[shard])) + + training_median = statistics.median(training_counts) + test_median = statistics.median(test_counts) + + # Make subsequent passes over files to find articles to add without going over limit + history_remaining = [] + n_history_remaining = 4 + + while len(consumed_article_set) < len(self.articles): + for fidx, file in enumerate(self.output_training_files): + nominal_next_article_size = min(nominal_sentences_per_training_shard - training_counts[fidx], max_sentences) + + # Maintain the max sentence count + while len(sentence_counts[max_sentences]) == 0 and max_sentences > 0: + max_sentences -= 1 + + while len(sentence_counts[nominal_next_article_size]) == 0 and nominal_next_article_size > 0: + nominal_next_article_size -= 1 + + if nominal_next_article_size not in sentence_counts or nominal_next_article_size is 0 or training_counts[fidx] > training_median: + continue # skip adding to this file, will come back later if no file can accept unused articles + + current_article_id = sentence_counts[nominal_next_article_size][-1] + sentence_counts[nominal_next_article_size].pop(-1) + + self.output_training_files[file].append(current_article_id) + consumed_article_set.add(current_article_id) + unused_article_set.remove(current_article_id) + + for fidx, file in enumerate(self.output_test_files): + nominal_next_article_size = min(nominal_sentences_per_test_shard - test_counts[fidx], max_sentences) + + # Maintain the max sentence count + while len(sentence_counts[max_sentences]) == 0 and max_sentences > 0: + max_sentences -= 1 + + while len(sentence_counts[nominal_next_article_size]) == 0 and nominal_next_article_size > 0: + nominal_next_article_size -= 1 + + if nominal_next_article_size not in sentence_counts or nominal_next_article_size is 0 or test_counts[fidx] > test_median: + continue # skip adding to this file, will come back later if no file can accept unused articles + + current_article_id = sentence_counts[nominal_next_article_size][-1] + sentence_counts[nominal_next_article_size].pop(-1) + + self.output_test_files[file].append(current_article_id) + consumed_article_set.add(current_article_id) + unused_article_set.remove(current_article_id) + + # If unable to place articles a few times, bump up nominal sizes by fraction until articles get placed + if len(history_remaining) == n_history_remaining: + history_remaining.pop(0) + history_remaining.append(len(unused_article_set)) + + history_same = True + for i in range(1, len(history_remaining)): + history_same = history_same and (history_remaining[i-1] == history_remaining[i]) + + if history_same: + nominal_sentences_per_training_shard += 1 + # nominal_sentences_per_test_shard += 1 + + training_counts = [] + test_counts = [] + for shard in self.output_training_files: + training_counts.append(self.get_sentences_per_shard(self.output_training_files[shard])) + + for shard in self.output_test_files: + test_counts.append(self.get_sentences_per_shard(self.output_test_files[shard])) + + training_median = statistics.median(training_counts) + test_median = statistics.median(test_counts) + + print('Distributing data over shards:', len(unused_article_set), 'articles remaining.') + + + if len(unused_article_set) != 0: + print('Warning: Some articles did not make it into output files.') + + + for shard in self.output_training_files: + print('Training shard:', self.get_sentences_per_shard(self.output_training_files[shard])) + + for shard in self.output_test_files: + print('Test shard:', self.get_sentences_per_shard(self.output_test_files[shard])) + + print('End: Distribute Articles Over Shards') + + + def write_shards_to_disk(self): + print('Start: Write Shards to Disk') + for shard in self.output_training_files: + self.write_single_shard(shard, self.output_training_files[shard], 'training') + + for shard in self.output_test_files: + self.write_single_shard(shard, self.output_test_files[shard], 'test') + + print('End: Write Shards to Disk') + + + def write_single_shard(self, shard_name, shard, split): + shard_split = os.path.split(shard_name) + shard_name = shard_split[0] + '/' + split + '/' + shard_split[1] + + with open(shard_name, mode='w', newline='\n') as f: + for article_id in shard: + for line in self.sentences[article_id]: + f.write(line + '\n') + + f.write('\n') # Line break between articles + + +import nltk + +nltk.download('punkt') + +class NLTKSegmenter: + def __init(self): + pass + + def segment_string(self, article): + return nltk.tokenize.sent_tokenize(article) + diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/WikiDownloader.py b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/WikiDownloader.py new file mode 100644 index 0000000..87f9529 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/WikiDownloader.py @@ -0,0 +1,58 @@ +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import bz2 +import os +import urllib.request +import sys +import subprocess + +class WikiDownloader: + def __init__(self, language, save_path): + self.save_path = save_path + '/wikicorpus_' + language + + if not os.path.exists(self.save_path): + os.makedirs(self.save_path) + + self.language = language + self.download_urls = { + 'en' : 'https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2', + 'zh' : 'https://dumps.wikimedia.org/zhwiki/latest/zhwiki-latest-pages-articles.xml.bz2' + } + + self.output_files = { + 'en' : 'wikicorpus_en.xml.bz2', + 'zh' : 'wikicorpus_zh.xml.bz2' + } + + + def download(self): + if self.language in self.download_urls: + url = self.download_urls[self.language] + filename = self.output_files[self.language] + + print('Downloading:', url) + if os.path.isfile(self.save_path + '/' + filename): + print('** Download file already exists, skipping download') + else: + response = urllib.request.urlopen(url) + with open(self.save_path + '/' + filename, "wb") as handle: + handle.write(response.read()) + + # Always unzipping since this is relatively fast and will overwrite + print('Unzipping:', self.output_files[self.language]) + subprocess.run('bzip2 -dk ' + self.save_path + '/' + filename, shell=True, check=True) + + else: + assert False, 'WikiDownloader not implemented for this language yet.' + diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/WikicorpusTextFormatting.py b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/WikicorpusTextFormatting.py new file mode 100644 index 0000000..9d356b1 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/WikicorpusTextFormatting.py @@ -0,0 +1,46 @@ +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import os + +class WikicorpusTextFormatting: + def __init__(self, wiki_path, output_filename, recursive = False): + self.wiki_path = wiki_path + self.recursive = recursive + self.output_filename = output_filename + + + # This puts one article per line + def merge(self): + with open(self.output_filename, mode='w', newline='\n') as ofile: + for dirname in glob.glob(self.wiki_path + '/*/', recursive=False): + for filename in glob.glob(dirname + 'wiki_*', recursive=self.recursive): + print(filename) + article_lines = [] + article_open = False + + with open(filename, mode='r', newline='\n') as file: + for line in file: + if '' in line: + article_open = False + for oline in article_lines[1:]: + if oline != '\n': + ofile.write(oline.rstrip() + " ") + ofile.write("\n\n") + article_lines = [] + else: + if article_open: + article_lines.append(line) \ No newline at end of file diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/__init__.py b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/__init__.py new file mode 100644 index 0000000..d49f0d0 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/bertPrep.py b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/bertPrep.py new file mode 100644 index 0000000..044e25a --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/bertPrep.py @@ -0,0 +1,387 @@ +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import BookscorpusTextFormatting +import Downloader +import TextSharding +import WikicorpusTextFormatting +import PubMedTextFormatting + +import argparse +import itertools +import multiprocessing +import os +import pprint +import subprocess + + +def main(args): + working_dir = os.environ['BERT_PREP_WORKING_DIR'] + + print('Working Directory:', working_dir) + print('Action:', args.action) + print('Dataset Name:', args.dataset) + + if args.input_files: + args.input_files = args.input_files.split(',') + + hdf5_tfrecord_folder_prefix = "/lower_case_" + str(args.do_lower_case) + "_seq_len_" + str(args.max_seq_length) \ + + "_max_pred_" + str(args.max_predictions_per_seq) + "_masked_lm_prob_" + str(args.masked_lm_prob) \ + + "_random_seed_" + str(args.random_seed) + "_dupe_factor_" + str(args.dupe_factor) \ + + "_shard_" + str(args.n_training_shards) + "_test_split_" + str(int(args.fraction_test_set * 100)) + directory_structure = { + 'download' : working_dir + '/download', # Downloaded and decompressed + 'extracted' : working_dir +'/extracted', # Extracted from whatever the initial format is (e.g., wikiextractor) + 'formatted' : working_dir + '/formatted_one_article_per_line', # This is the level where all sources should look the same + 'sharded' : working_dir + '/sharded', + 'tfrecord' : working_dir + '/tfrecord' + hdf5_tfrecord_folder_prefix, + 'hdf5': working_dir + '/hdf5'+ hdf5_tfrecord_folder_prefix, + } + + print('\nDirectory Structure:') + pp = pprint.PrettyPrinter(indent=2) + pp.pprint(directory_structure) + print('') + + if args.action == 'download': + if not os.path.exists(directory_structure['download']): + os.makedirs(directory_structure['download']) + + downloader = Downloader.Downloader(args.dataset, directory_structure['download']) + downloader.download() + + elif args.action == 'text_formatting': + assert args.dataset != 'google_pretrained_weights' and args.dataset != 'nvidia_pretrained_weights' \ + and args.dataset != 'squad' and args.dataset != 'MRPC' and args.dataset != 'CoLA' and \ + args.dataset != 'MNLI', 'Cannot perform text_formatting on pretrained weights' + + if not os.path.exists(directory_structure['extracted']): + os.makedirs(directory_structure['extracted']) + + if not os.path.exists(directory_structure['formatted']): + os.makedirs(directory_structure['formatted']) + + if args.dataset == 'bookscorpus': + books_path = directory_structure['download'] + '/bookscorpus' + #books_path = directory_structure['download'] + output_filename = directory_structure['formatted'] + '/bookscorpus_one_book_per_line.txt' + books_formatter = BookscorpusTextFormatting.BookscorpusTextFormatting(books_path, output_filename, recursive=True) + books_formatter.merge() + + elif args.dataset == 'wikicorpus_en': + if args.skip_wikiextractor == 0: + path_to_wikiextractor_in_container = '/workspace/wikiextractor/WikiExtractor.py' + wikiextractor_command = path_to_wikiextractor_in_container + ' ' + directory_structure['download'] + '/' + args.dataset + '/wikicorpus_en.xml ' + '-b 100M --processes ' + str(args.n_processes) + ' -o ' + directory_structure['extracted'] + '/' + args.dataset + print('WikiExtractor Command:', wikiextractor_command) + wikiextractor_process = subprocess.run(wikiextractor_command, shell=True, check=True) + + wiki_path = directory_structure['extracted'] + '/wikicorpus_en' + output_filename = directory_structure['formatted'] + '/wikicorpus_en_one_article_per_line.txt' + wiki_formatter = WikicorpusTextFormatting.WikicorpusTextFormatting(wiki_path, output_filename, recursive=True) + wiki_formatter.merge() + + elif args.dataset == 'wikicorpus_zh': + assert False, 'wikicorpus_zh not fully supported at this time. The simplified/tradition Chinese data needs to be translated and properly segmented still, and should work once this step is added.' + if args.skip_wikiextractor == 0: + path_to_wikiextractor_in_container = '/workspace/wikiextractor/WikiExtractor.py' + wikiextractor_command = path_to_wikiextractor_in_container + ' ' + directory_structure['download'] + '/' + args.dataset + '/wikicorpus_zh.xml ' + '-b 100M --processes ' + str(args.n_processes) + ' -o ' + directory_structure['extracted'] + '/' + args.dataset + print('WikiExtractor Command:', wikiextractor_command) + wikiextractor_process = subprocess.run(wikiextractor_command, shell=True, check=True) + + wiki_path = directory_structure['extracted'] + '/wikicorpus_zh' + output_filename = directory_structure['formatted'] + '/wikicorpus_zh_one_article_per_line.txt' + wiki_formatter = WikicorpusTextFormatting.WikicorpusTextFormatting(wiki_path, output_filename, recursive=True) + wiki_formatter.merge() + + elif args.dataset == 'pubmed_baseline': + pubmed_path = directory_structure['download'] + '/pubmed' + '/baseline' + output_filename = directory_structure['formatted'] + '/pubmed_baseline_one_article_per_line.txt' + pubmed_formatter = PubMedTextFormatting.PubMedTextFormatting(pubmed_path, output_filename, recursive=True) + pubmed_formatter.merge() + + elif args.action == 'sharding': + # Note: books+wiki requires user to provide list of input_files (comma-separated with no spaces) + if args.dataset == 'bookscorpus' or 'wikicorpus' in args.dataset or 'books_wiki' in args.dataset or 'pubmed' in args.dataset: + if args.input_files is None: + if args.dataset == 'bookscorpus': + args.input_files = [directory_structure['formatted'] + '/bookscorpus_one_book_per_line.txt'] + elif args.dataset == 'wikicorpus_en': + args.input_files = [directory_structure['formatted'] + '/wikicorpus_en_one_article_per_line.txt'] + elif args.dataset == 'wikicorpus_zh': + args.input_files = [directory_structure['formatted'] + '/wikicorpus_zh_one_article_per_line.txt'] + elif args.dataset == 'books_wiki_en_corpus': + args.input_files = [directory_structure['formatted'] + '/bookscorpus_one_book_per_line.txt', directory_structure['formatted'] + '/wikicorpus_en_one_article_per_line.txt'] + elif args.dataset == 'pubmed_baseline': + args.input_files = [directory_structure['formatted'] + '/pubmed_baseline_one_article_per_line.txt'] + + output_file_prefix = directory_structure['sharded'] + '/' + args.dataset + '/' + args.dataset + + if not os.path.exists(directory_structure['sharded']): + os.makedirs(directory_structure['sharded']) + + if not os.path.exists(directory_structure['sharded'] + '/' + args.dataset): + os.makedirs(directory_structure['sharded'] + '/' + args.dataset) + + if not os.path.exists(directory_structure['sharded'] + '/' + args.dataset + '/training'): + os.makedirs(directory_structure['sharded'] + '/' + args.dataset + '/training') + + if not os.path.exists(directory_structure['sharded'] + '/' + args.dataset + '/test'): + os.makedirs(directory_structure['sharded'] + '/' + args.dataset + '/test') + + # Segmentation is here because all datasets look the same in one article/book/whatever per line format, and + # it seemed unnecessarily complicated to add an additional preprocessing step to call just for this. + # Different languages (e.g., Chinese simplified/traditional) may require translation and + # other packages to be called from here -- just add a conditional branch for those extra steps + segmenter = TextSharding.NLTKSegmenter() + sharding = TextSharding.Sharding(args.input_files, output_file_prefix, args.n_training_shards, args.n_test_shards, args.fraction_test_set) + + sharding.load_articles() + sharding.segment_articles_into_sentences(segmenter) + sharding.distribute_articles_over_shards() + sharding.write_shards_to_disk() + + else: + assert False, 'Unsupported dataset for sharding' + + elif args.action == 'create_tfrecord_files': + if not os.path.exists(directory_structure['tfrecord'] + "/" + args.dataset): + os.makedirs(directory_structure['tfrecord'] + "/" + args.dataset) + + if not os.path.exists(directory_structure['tfrecord'] + "/" + args.dataset + '/training'): + os.makedirs(directory_structure['tfrecord'] + "/" + args.dataset + '/training') + + if not os.path.exists(directory_structure['tfrecord'] + "/" + args.dataset + '/test'): + os.makedirs(directory_structure['tfrecord'] + "/" + args.dataset + '/test') + + last_process = None + + def create_record_worker(filename_prefix, shard_id, output_format='tfrecord', split='training'): + bert_preprocessing_command = 'python /workspace/bert/utils/create_pretraining_data.py' + bert_preprocessing_command += ' --input_file=' + directory_structure['sharded'] + '/' + args.dataset + '/' + split + '/' + filename_prefix + '_' + str(shard_id) + '.txt' + bert_preprocessing_command += ' --output_file=' + directory_structure['tfrecord'] + '/' + args.dataset + '/' + split + '/' + filename_prefix + '_' + str(shard_id) + '.' + output_format + bert_preprocessing_command += ' --vocab_file=' + args.vocab_file + bert_preprocessing_command += ' --do_lower_case' if args.do_lower_case else '' + bert_preprocessing_command += ' --max_seq_length=' + str(args.max_seq_length) + bert_preprocessing_command += ' --max_predictions_per_seq=' + str(args.max_predictions_per_seq) + bert_preprocessing_command += ' --masked_lm_prob=' + str(args.masked_lm_prob) + bert_preprocessing_command += ' --random_seed=' + str(args.random_seed) + bert_preprocessing_command += ' --dupe_factor=' + str(args.dupe_factor) + bert_preprocessing_process = subprocess.Popen(bert_preprocessing_command, shell=True) + + last_process = bert_preprocessing_process + + # This could be better optimized (fine if all take equal time) + if shard_id % args.n_processes == 0 and shard_id > 0: + bert_preprocessing_process.wait() + + return last_process + + output_file_prefix = args.dataset + + for i in range(args.n_training_shards): + last_process = create_record_worker(output_file_prefix + '_training', i, 'tfrecord', 'training') + + last_process.wait() + + for i in range(args.n_test_shards): + last_process = create_record_worker(output_file_prefix + '_test', i, 'tfrecord', 'test') + + last_process.wait() + + + elif args.action == 'create_hdf5_files': + assert False, 'HDF5 format not fully supported in this release.' + + if not os.path.exists(directory_structure['hdf5'] + "/" + args.dataset): + os.makedirs(directory_structure['hdf5'] + "/" + args.dataset) + + last_process = None + + def create_record_worker(filename_prefix, shard_id, output_format='hdf5'): + bert_preprocessing_command = 'python /workspace/bert/utils/create_pretraining_data.py' + bert_preprocessing_command += ' --input_file=' + directory_structure['sharded'] + '/' + args.dataset + '/' + filename_prefix + '_' + str(shard_id) + '.txt' + bert_preprocessing_command += ' --output_file=' + directory_structure['hdf5'] + '/' + args.dataset + '/' + filename_prefix + '_' + str(shard_id) + '.' + output_format + bert_preprocessing_command += ' --vocab_file=' + args.vocab_file + bert_preprocessing_command += ' --do_lower_case' if args.do_lower_case else '' + bert_preprocessing_command += ' --max_seq_length=' + args.max_seq_length + bert_preprocessing_command += ' --max_predictions_per_seq=' + args.max_predictions_per_seq + bert_preprocessing_command += ' --masked_lm_prob=' + args.masked_lm_prob + bert_preprocessing_command += ' --random_seed=' + args.random_seed + bert_preprocessing_command += ' --dupe_factor=' + args.dupe_factor + bert_preprocessing_process = subprocess.Popen(bert_preprocessing_command, shell=True) + + last_process = bert_preprocessing_process + + # This could be better optimized (fine if all take equal time) + if shard_id % args.n_processes == 0 and shard_id > 0: + bert_preprocessing_process.wait() + + for i in range(args.n_training_shards): + create_record_worker(args.output_file_prefix + '_training', i) + + last_process.wait() + + for i in range(args.n_test_shards): + create_record_worker(args.output_file_prefix + '_test', i) + + last_process.wait() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description='Preprocessing Application for Everything BERT-related' + ) + + parser.add_argument( + '--action', + type=str, + help='Specify the action you want the app to take. e.g., generate vocab, segment, create tfrecords', + choices={ + 'download', # Download and verify mdf5/sha sums + 'text_formatting', # Convert into a file that contains one article/book per line + 'sharding', # Convert previous formatted text into shards containing one sentence per line + 'create_tfrecord_files', # Turn each shard into a TFrecord with masking and next sentence prediction info + 'create_hdf5_files' # Turn each shard into a HDF5 file with masking and next sentence prediction info + } + ) + + parser.add_argument( + '--dataset', + type=str, + help='Specify the dataset to perform --action on', + choices={ + 'bookscorpus', + 'wikicorpus_en', + 'wikicorpus_zh', + 'books_wiki_en_corpus', + 'pubmed_baseline', + 'pubmed_daily_update', + 'pubmed_fulltext', + 'pubmed_open_access', + 'google_pretrained_weights', + 'nvidia_pretrained_weights', + 'squad', + 'MRPC', + 'CoLA', + 'MNLI', + 'all' + } + ) + + parser.add_argument( + '--input_files', + type=str, + help='Specify the input files in a comma-separated list (no spaces)' + ) + + parser.add_argument( + '--n_training_shards', + type=int, + help='Specify the number of training shards to generate', + default=1472 + ) + + parser.add_argument( + '--n_test_shards', + type=int, + help='Specify the number of test shards to generate', + default=1472 + ) + + parser.add_argument( + '--fraction_test_set', + type=float, + help='Specify the fraction (0..1) of the data to withhold for the test data split (based on number of sequences)', + default=0.1 + ) + + parser.add_argument( + '--segmentation_method', + type=str, + help='Specify your choice of sentence segmentation', + choices={ + 'nltk' + }, + default='nltk' + ) + + parser.add_argument( + '--n_processes', + type=int, + help='Specify the max number of processes to allow at one time', + default=4 + ) + + parser.add_argument( + '--random_seed', + type=int, + help='Specify the base seed to use for any random number generation', + default=12345 + ) + + parser.add_argument( + '--dupe_factor', + type=int, + help='Specify the duplication factor', + default=5 + ) + + parser.add_argument( + '--masked_lm_prob', + type=float, + help='Specify the probability for masked lm', + default=0.15 + ) + + parser.add_argument( + '--max_seq_length', + type=int, + help='Specify the maximum sequence length', + default=512 + ) + + parser.add_argument( + '--max_predictions_per_seq', + type=int, + help='Specify the maximum number of masked words per sequence', + default=20 + ) + + parser.add_argument( + '--do_lower_case', + type=int, + help='Specify whether it is cased (0) or uncased (1) (any number greater than 0 will be treated as uncased)', + default=1 + ) + + parser.add_argument( + '--vocab_file', + type=str, + help='Specify absolute path to vocab file to use)' + ) + + parser.add_argument( + '--skip_wikiextractor', + type=int, + help='Specify whether to skip wikiextractor step 0=False, 1=True', + default=0 + ) + + parser.add_argument( + '--interactive_json_config_generator', + type=str, + help='Specify the action you want the app to take. e.g., generate vocab, segment, create tfrecords' + ) + + args = parser.parse_args() + main(args) diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/create_biobert_datasets_from_start.sh b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/create_biobert_datasets_from_start.sh new file mode 100644 index 0000000..3f1a416 --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/create_biobert_datasets_from_start.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export BERT_PREP_WORKING_DIR="${BERT_PREP_WORKING_DIR}" + +# Download +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action download --dataset pubmed_baseline + +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action download --dataset google_pretrained_weights # Includes vocab + +# Properly format the text files +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action text_formatting --dataset pubmed_baseline + + +# Shard the text files +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action sharding --dataset pubmed_baseline + +### BERT BASE + +## UNCASED + +# Create TFRecord files Phase 1 +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action create_tfrecord_files --dataset pubmed_baseline --max_seq_length 128 \ + --max_predictions_per_seq 20 --vocab_file ${BERT_PREP_WORKING_DIR}/download/google_pretrained_weights/uncased_L-12_H-768_A-12/vocab.txt + + +# Create TFRecord files Phase 2 +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action create_tfrecord_files --dataset pubmed_baseline --max_seq_length 512 \ + --max_predictions_per_seq 80 --vocab_file ${BERT_PREP_WORKING_DIR}/download/google_pretrained_weights/uncased_L-12_H-768_A-12/vocab.txt + + +## CASED + +# Create TFRecord files Phase 1 +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action create_tfrecord_files --dataset pubmed_baseline --max_seq_length 128 \ + --max_predictions_per_seq 20 --vocab_file ${BERT_PREP_WORKING_DIR}/download/google_pretrained_weights/cased_L-12_H-768_A-12/vocab.txt \ + --do_lower_case=0 + + +# Create TFRecord files Phase 2 +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action create_tfrecord_files --dataset pubmed_baseline --max_seq_length 512 \ + --max_predictions_per_seq 80 --vocab_file ${BERT_PREP_WORKING_DIR}/download/google_pretrained_weights/cased_L-12_H-768_A-12/vocab.txt \ + --do_lower_case=0 diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/create_datasets_from_start.sh b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/create_datasets_from_start.sh new file mode 100644 index 0000000..f21914e --- /dev/null +++ b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/create_datasets_from_start.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export BERT_PREP_WORKING_DIR="${BERT_PREP_WORKING_DIR}" + +# Download +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action download --dataset bookscorpus +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action download --dataset wikicorpus_en + +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action download --dataset google_pretrained_weights # Includes vocab + +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action download --dataset squad +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action download --dataset "CoLA" +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action download --dataset "MRPC" +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action download --dataset "MNLI" + + +# Properly format the text files +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action text_formatting --dataset bookscorpus +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action text_formatting --dataset wikicorpus_en + + +# Shard the text files (group wiki+books then shard) +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action sharding --dataset books_wiki_en_corpus + + +# Create TFRecord files Phase 1 +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action create_tfrecord_files --dataset books_wiki_en_corpus --max_seq_length 128 \ + --max_predictions_per_seq 20 --vocab_file ${BERT_PREP_WORKING_DIR}/download/google_pretrained_weights/uncased_L-24_H-1024_A-16/vocab.txt + + +# Create TFRecord files Phase 2 +python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action create_tfrecord_files --dataset books_wiki_en_corpus --max_seq_length 512 \ + --max_predictions_per_seq 80 --vocab_file ${BERT_PREP_WORKING_DIR}/download/google_pretrained_weights/uncased_L-24_H-1024_A-16/vocab.txt diff --git a/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/images/bert_pipeline.png b/train/atlas_benchmark-master/nlp/Bert-Large/tensorflow/code/bert-Nv/data/images/bert_pipeline.png new file mode 100644 index 0000000000000000000000000000000000000000..40193e9e13ba2daa25f70d1ee69a34f1958ca7ce GIT binary patch literal 212516 zcmeFZbyQs0_AQJB4HXCu1;IT?a3>)^@ZcUSxVr@lfx;8qNpN@fK;iB#f#B{0%{#?S zci;DW@BRCY@r}{{(cQbw*?X?J=9+8mb8qllc}YxkVstn-I83QGua)57KwvmH)KXML z;NR5YG6DYpH~dysMf~^g-|g+~mzS5>+1b?j%Y%c1g@uKQiHR=tzzVqi{r&s<`;(Is zuSU?!%uL{SvYXrM>gwv^Ni#Gww56q`qhISiJw1_KFH%xc`hKQN?N(IJIb8}f)*~^zNgM)+c@bJbbCRu}YvHp-RO{!xl8Zl#Zsxhx zBkf%*B(y3?)~rcX)`Q3`)8XLA;iO)_RB?mfOGOi#=xGQfsf!9MLwYINwrK9x#rMt_ zoZ4mNIdu2Cb$cqUV6WMAOd*9rW?zA3+v9X{Y`6R+90Hav91=MkJgVrwfBuWZza;oK z8U9s*e--iHa`10i{MQrw>y7?3hX1!K4tm6a4-9yqa4o!?~Txe%T_7Geh_rn>LFC|RjgIR$%MO^)h#s_&&o zPz@+SYZcp)Iiq#8X>|$r6Que!{DuCO3k_nw)aC(~>kVUr)!yaypWD$B`;N5V1n%~3 z*V=grT))WZN#gK5k+jw6@H~9BVC;_dNeFLAUg0;r#hP(*0jkIcfj3lzp$v8kr_mc@ z;;LdaL3uCwY4)6uYVh)hUokePg}Dw(cLgKnW&D~DdCkqIE)61QLqWJ+lKo0&f3~p@ zTKiqsgZs1Y4-dwYo6kOjjtMdyzxRwuMS9*`eoS8WVJTRWc%i=`NBmyBVONeoLI7E6 zR;LW|H`isT%PdJ4mhQvpl6BO`Vo*vkC%;|AV*E68L=oQ(jY-+-%Q~5E!I<5}3%px( zfAb`?>E7$-LTJimQv&*8vra>hN8)XciU^ZP0Jb|?Hg;^;IQ9(sn5oar+t-f^Wvl&)3fx6OyD?&&EskH0cLaGmO4f)GlqNmHnNG~uiJkpt!&W3d zv<O2*E&+{V5wjGPw+i?wYaguVM{fYIr`D2duy?^gjaV}=_M9(H1 z`Iade(!&|M#M`)#2%`Gf9B*^Dn6GSm#Wt(4@9G)d()Z~)voqg(#p{<#+}VN4VEpY9 z7L=DuK7t=SjZQT5FCRA_eBicECEMzGwfR9dy|Ld!{?(-imr8rk2^ycMF^t|6g~JyY z_n@TM+$bo4zxgU$&EdkbUPRQ|){2_g-Iux|dpN%98^1o0Y7pEZtf zoia}fLf+jwb8aIVr4EFDbzw?fvsO8x#jH$b2y?0WTs?^3`43mD5M~vorgQvEtSz`j z4yz?*1TfzRxu_22Y436@st1&}RBX@hI{4&_Du#BF*k?tO0GX1+p)AvJi`H1}G4gI~cZf^oeSGo6M`=ru| zNo*}yFV?{7m(KOn%f0hlk9j3r0y34UC6Y-)35?mUFnPl#k~E-Ys{Y%>RvIlt<% zHcLwmKZE>pfO4dhuSL<96;d(fSOsyB7InrdnFEW(kH6S@s|J>5_vPB^dX)F|py7Yg9*sOQapzr7K4dWC!otF#PQeE6ceE4(Z{+;*F!6rQ-BsiSVLHQfTTK{q zc5@_DQj%)P&D_U%#e+^Pf3S%&$BiVwcI@L`PJNz}Y_mPm-*WlsS)4NO}uDvPHuwJ`c7?WjbQu#eU@)qh*~Mm@SGQfWhNoI zoZN<3q#BiYpZ9$+wC;QLmS#h&f>`HZVP<3+SyS@8g8NJB4b9{YYuqL&jEZD@34yt8 z{^lXnZg1uG|-4coOC##HCzACK6CzC1Tg!(c%HAIBDHiv?b48SWyU&`dKUN za%{WXvU0^EKc;=GpO+ii3(jY8sVUWyvaf&aCxwP9h^a8qY}H}l*BJlW&vNvq50lY= zFr?lrEBA7H67f7El0dF@EPqL&ExC%k(cEKhRCsFJIpXi;?7IURuGg?P>!kQdiqg)H znp`cLXAVS7vDSItThEk8yCjriTI|QYI@*!P^^57C{3`jOx}+N z_QIGmUMCY_W2{!Gf=EhW@ zrt+@hzEQTn6@YzOf6?APlHZ!tl4VnAlegl#)1e%c!r=VxLh-LJ+Ka5q@|n&=OSs4V1O6u1h@sW*3rb1c_ilZ9x zqIBuHffYmO7>`X~kbd!}4XqJyKV*_a^gUAc;zJ%dB?dAxJ|IgiK1utZTPMOTi?1kE z_ut*lXig`Y=4XvJs#B--_$_U#I#l&EO~r>9CjYXgjy_t>iVa{-JgVrV_(zx?!QTl9 z{CZD%BS##WtKnRFEf6`YyvkhYW|K{b#yS-k8QCkB``fkbE<1It_F8`{vq$<17wN^5 z%4Wq%ks!81jF|8Xix46@$#{#mk;xdiV zs<_yfOK}5kptj^F+sZt>+nbNms>TNGxfA`Q|5fbrVLbDR zw$4;pKU?Z|Y(80xd`fm>fNz`anRBH|(AD*AJRUnM+59&{qT#2EbF%lv^hFw=`#Fz` zi@CzsZ!GyD2qamJR-mzj5T%OZiQo_yIyG0=^d7_) zHsd!%-xq~kI)|MuG@rUR$6nh0azU5Y|C;xDanENC%eLs6^DyW~Jsb9iXvO(bGj1{w zhh=}4c#4HM{2cnSXV~5UBtIW4C@`L3aP7CF7|AHv)J&)Mt`=MEe;O>EW2o0Y2 zf2rVigoq7?+MNtZBjibxWk;-!EamPIh>9+Uw;9pN-+P-!Gdk3MPHm3&AE`H$>TqK6 z`eaK-(mJ>M`V*2$^QOY3XM<};z3SHuO^h@QoR@9$J>E8B!Z9eBWK0;Dr29ssYoBrc zF}AM}v?!5^p9(2;4XP0{B~O?}=W%gMWqoW3;If#)Jp<2H1D8E4c8F$Eh2C1VIU0NhT?q{dlRUb@;#A{QM}aki?zxf$M4}=2uWtc7S?9I#k)oz;*0aA|sXmjC zj5GJ04fW6y7WsC(x#^`{&7`pYB|SsVGxjQGej9S>Q_!nkJl&8G=Gi98!;Rdi(7ezW zgTUCXp!=#_hD!iQwbZzf9Haa)LcjxbzO#--M$KE^+SZ)JY)cRmi&<(_$-`o=T?bif1V0^<1qKtB#W2XUX59$+S^oS-jk5>a)K<|l&Fl2guWy$ z3uJ@VUfK65g>oG=r+cP%yz>i1p;C?WKg_FCC2L=Zz~iknvQVsKC+Q~O95P3sym-lH zJCMJiX}{%ezg_qd7iDAudcA-51^YnLQ(iUQ-XAbDE6w@s`Hjm<7BD+AH%T}O6eZ4E zv+TPi>G(ZI4Z`qu%i@~-lQ%=^RyZx;PPc$7t6+F?P=D9sNZUn^-pLB0_s zXfH!E>azN-`hnwMDKgNGz)BGQM-~HTRWu1E-H&B+E@iIPRG`bb!u@cfhTzJE1dETG ztI-4wDr=n$pFEn0T@2(uZaIa#>i*idVYg_N4QM!=-)&r4M^1R+je)8e-Tf|Sg}i=l?xFR7mBw&r_Q{Wxxo1;HAT z?obJz1nebe$zW0<2ZiN}XaIPG`{W6kZka`}8vjPKZn_h+!m#}SQhWo!k2{FDId?oa?3a}lY~#h*$n~=!uQEjnCQX^tMtrtW#^hX>*Q!= zHuK(0{{?+JN2SVqei3N8nDgvu8s4K3;a4Gj77oNQ5jdy=vgD3D1;1j-2b*L=7RFy! z7kXjG3f~6|8xYGJz42E2JhZeVT8|PUmpk>#I*Sr@37~ww)7gvu|0?p#Fv>@dS>k$+ z(oN^o$1Lu<%>u4B=~Ho*Pk7vETN zl74U}>3_}60PjBCnJmX}j0zr7kIT7lSBxO)HzpUCgOghfoT5<@FTa@k_9aDB-vdUJ zpNh{U2b$H^(sJVJuUcaT|Jw21*Mc{~)O~;m0SLc;{D<0%9JhXdQ@fuWi__QXqpb?Y zcChfM9LwZ;^!r~*6!W(hPrKrZVO?HsT9CPldg<* zL!Bx(f+VT@!%B(-9N6m*dj{x{zab>x9Q(d8R-RZ?`LeJ!chj~is;R6@=hOE!SceT! zuLQM@c8edFGZIbTR%SKh3q#DAR|;Rw_UwP@7A=>%=8j@OvK^CkX$A*D|Bvwiz-{SG zoox((?a-vg!^0={IRsRv_~AVdFpI<5>w%_J4;xgkV|ki8SHq_FP;X_v%bVJUx~l0h z?{sJONhH_aUnrW4eZ?&4??@Zj=vETnyeu^BZ3}^p$hfHyik|PzpFW2+_MA3-0S(-nSX$#M07oSI-5+ zsxZ}r+2(LvT!DF^?{q?cnm+8gN!DTr%%h$_w_lnFS3(wr?DRmY=SR$Lh$z&ZT zpIZDJxk+)P%}MbjY&zOg+n~uFV@}rsB=Ib<)cv!(afM;!s@E!9;Ptkt>h5uP!G3e| zvw(m#N~9b*tfl1UwVSWk2Y31nQKkzVXzMzy`6MDYzn1$7VXDGN5bijycs!eHSy8Ka zAV(Z#?Ibnhe`mjo_CYi&+ zUs(Ch7Ml2&aGXs)^I4et+2h_9z*Xb_d=pA)R!S7Pt8=P%&cXgDW{jn#(fsjM=9dur z(q0Fl#xnAiL}{_wFUt0W9%#eph-V*mY?;r8??;9|7}O*9^`dPxjtVK{o(NI$+~^oD z6ifar(k*tu$u}j_ivFril<6t3&XE|}<=#ur{kxvtizK#L%K8vV8@YBmfPulB*SxHs z5al0-r-?*l){|!tOeFRWT+)9EUV-Oj^n6lhT1ex-<_1cezSvLJcsux8P9o9^ za0;D!j%LbOHy=2s>?H#iD)nNLn#22+vbX{J6BEOE2WNWPLH3Wn$N1X7<*R+UalFPN z5g6UiuOlon&wG*C;Mw4A@WKcL93}vrME{~-{@Hd(mig3eu+j-2@;_)zH~KuCrF#6E z90{XcTixtjEpHIQqqu6 z)tIsJJ5dL$MmOQBtMTRIR>y$Qp)uM1S|$$4h|WPv;;gUm?TRqMuMXb8?*m9@&D&}8 z$JbZmB?bknLv`ApH_FFL+EBh`bpKXEuC&el$V~9s=!1DDKJ`eUCrP59$#NEDS6_P! zFC_?rF-+z!-ZMw~?khBOP?Jzsd)1tu-*`8rC0c&@z3wrZpmZZ+z0(PAD1K-2(ai#< z9T90x;oIzdP`v7mWzy5XFDl?sc@cVEK|Uh>-C_LY#s98}f4%I#e)I32|9?S;jl<7v zgD~j-_|jue0nT~mo<9ree?-@W&X+$Wm4f|$MA%FOe!}2p_sf412eu>u5q@n2O#bxI z-(#JJ?+g8=a70OKTF@E(2TXo^SfZpu@RW-G`{Q?|$mM4W(A>@g`Ooz2cp3Z%hG)s) z{FwdTkD=u2-isLb_wylyFD#e#i=`9GFVy(T%^NwbwPj%lOU0Q zp*ixnkhDxSa+^E+{0jvh^%IhC$*Gvk5_#D%5*B1~>}{Pe^|tOuepyXBP71#jw-{>z zN047Q-H4Zc-LdLpIMnxO)8>e-}+Ng@Kh+&%6vXyo$T@j9B+(-sI&dmpD~yTx>_w|;FB z%P7sR$rPY&1?{_4%2TWaR`}x?Bfb<__9Mtd@$hj5rKyEwl4Z+4Bn}UPZhxtB*d{Cr^LeB@OfR_vwI!Zr= zXND{wVrjS{1VNLC`v72q34G&PfH)8Xhbw~rbeejJZg~$z!*8w z^$vqGD25*_Sv2H;olbtRTjYh|QOB739qS9uSlJN8C2+zd*ajIlj%coK=w zQDYEgI-JUP-R@^W3WAvLtaLpJdD#VQyQ(VWpiJGks=V|-M|k)yFuv%@7%h(ezzXfO zu<^1Af(D4EdV|5$^C6N2ZvdQar7)}@TuAoZNS}D$r-~gQqh#RfhH=7eo4*R)RoOoFTB6M0gB^>VF<^uMoh0H#bRCh~?wH2LO< zJwb-hAO-$B-b&NX{sfBW`2xI&r)-Sjqzec5>2WO_+*BG2nGoqt1E+(4tlmRlI|*>R zV0_1woi5np%T*QHcoJZYDftkx^c1$85Jm#XXgD2OGc*a+en|y}_C!+d0u2mMgus5R zEjA#Uq$N5bQ`is7i?AT0Hc`kDqyQL<)ZgMN=$%$5k+`FPhlUrqpf|4oINAKd6g}1! zg|pXCmQr11n=@#G5`W;$MT+5Fqk4t@DN~LPV9LV@|337*IBd0E?CO_CPb{}q_OP*5 zzsyPtk%4Ulf%pGkDQf{J46~aCB9^HPz?C>8PgHtf_iGZq-#1|Na(9*wo|k|<)yR33 zjsKML+weDlf{%gtoA}x_Y{T|>O)7fD6A6R6$hmKU1$Mye0zeyGS&=8wfyFR{#mG{u z1%V&g+&+i%#iR!Kg~*gjSsMmJx_zh9plV(Gfw6>G`l(Pj6gsT$4n0C}euFS3p6;YP ztqq5*>>>_=Zc$57y@cV0j>@*d_Pq@a!1({RG$}|7$aBA-N@{={iN!J6@CXb52n-kz zYZx*5YBG4wGVO zgje7f+6=V_NJ_xlmGJ@LRfdUEOj1tDKCGhUH%bBO@g=J~9rj8FUF?f?#3) z!rqOK_svhJOG3quH`*Z0%`XD*2oV4^NYO#$W$dsg%c?4mOg-rO;`>!LLxAtF2lGQn zbgpolg8*^Hh(U&-SCih zp~?W=7D936(@~5npFn;d2LVs_dh+(#7uYr;7?>zg!O#n`$oW!C3q!BPyu-lhr_`&% zGFpa|Mws|6mgq_Iumcgnp`wCA(oz7>*)9Bn;PfAf$$biw0SQ?7vO2>Cbg1E);X@km z@Kf131iunMkY>(_b3W|T!5$9X?w|mwh5q7e(<#1%4z1h)JODH`AOjFcX@Gra;>ncr zzlNo1qH+i{2vC6UHffblAv2LYrK=3~L;wNfB3Jm48h~%TBN(Oz znUa7SaGwM@|FxPKvTkMPC#nwPiN&B&$-x6p)~y~?@yI5M19)zJ4TB{jKyNEJG*+xF zLLfk-g>OT~U&0bsI#m%Y;lDp&m_Rm39pXrU_E^#jfW)vDI5A^m>1DwKvCj<%*CZ$2!GHIfGqb>qVxYo}fB~ z1DRAZNx!~L1{0IiA#t#To{-cQ=~Py@V=ijhkU?%ibo+oj@bF3wYGl+Z*!r4D!mC9= z7&`o4Kl6r>MRAf&81xxNM*x+*W=yWYu21X#xpg{w!hNe;G?=82ICTIAEwBoi`vky} z{I>lGQiC{fJyp*))p=hs?-8` z^l>TJW-L;ZdEWy%F_!9TGGMS-n1CP;<6-o1I}Z|&!~F&X7LhNUl3@?_-lPL%Zq3PH z`~cQ~}o=`5mN4Fm(o@%d2`qgKJx3ttsq zbMYAYykW*N!vS7nUGV&*ArVG_ zb74~eM_Tibr~xsZ$YA!32UU}Q1yly=sv`_P8=zRsgnMM+>OM%(qI)Q;{=qNl0#(YT z)`nqL1t^rfc-#mtzfA&1t!cRKxs`xrhRW#z;B2lhnw^{gM3YWJ0uHq*4LInOI>nmK&(=_TckY&Y3$3`?%`yQ+8{gPJ>B6&At)_{EiO= z;7GT#LjVbIx&q|9zi1I8s76ZvN-msZqYN2q2y2TR*c@q*O2`B|3`}(0Fy645ruU7@ z$Y+(A?G~Iig;twzRtj;(N!L zXQX6$O&<50!&(^qozc)O1#*vSJoJVD$oiZg=Y#jmm9Isiw=$6-@7&7Kz zI$*$%v<()1cSl9Cba$M`dQULlIazVGQh5^^`9{-*^6uWu>GsUJ+je<-C+Mvb_(G%| z9sx_k2)5-u%^2P=mncqSZ!u;3MIpmfsPUAwWLka%Zf4d7U}Sw04^DE^@l7lCi+S>& zJX@A0n17uv-14+bIvw2_d1Ca}YUk_7AMRqPJ=%}#>nLL7*tMl$ifE641Bv(F=iIj8 zfgg9Uft8ea%{yK9tOlis_6A_f6mi=u_hc8{O6kTt6i*1Og+AO(jAZfD#rb=ydBw>=mli` z2JkW4ES;C)?Qd(^Z!(X6T{%T^W8k&&s%+!l5`C6bm#Wk_s^&vllMCiuNV(3*vVLg9 zsvm_w=^HMd9y*YWFl`_|1BuIQr&+ukBOM_fS!9qZ*&%XF5PPv6FnzMG^lfM_yZe1# z(5^p?x?zyP6?cQ1+A|bk?hxN5Z7c|h>N;+XGb}|^Yu>_CGT*fhb1T-t{|Uq}D)0uA z<+nqq&9mIPV(pNAe46I5SSn_SV?nPgLG0LDrz4$&*D`^sV{h|(N7;2{HdO|7gC&;m zf}kZ6wCh{6Fb+4fsDULEn${ZU0e;p0;n#ewxq!uHjpL8eihyGl=T(5Hr}V_4hCDnIWfc)T`5&RR*m-#~x(C#;b1vC!8q0@l>`~+#C}qljC#z^9a{`L?AU%Bx$(WmZTo> z5YwK%c^>vtby8F^v0zT3yR^|A5yHl6`9^dJHwYS7c7*H~3sVPuy*hxZ8e~$p18%&`2?-okvadWsxit{Gs`S=&)Vq5{;kA5{hJ8$P9JQ` z3vUkvT+)%J?`o+43{cGH?_kZLlOkQRFINyxCm^u>Mdmlu)tX=lZ8COk@lP+gkO(8_kmE*$Auin4CZ+3Smf3kTjb>6swdfx;F z{q-$Cb#0_#PbnH;u`6~MoGf8Ay1)W$l}+cUj8=rVHV)9McunV;=W`bJZtjMm6eI4C zex&-Isxt79JdZHQX~ygqL$mt3$B3CrTZTiIUo;vfmsz}N%H=5{mRr#G5;LKl0}u`Y z5kn1Oc>8gEJ{@5frBC0HU@R<)qX0nx_x0kFqc$4bhFv0;X>waJ(5_Y`XN_1zjxkw_ z7!IuLak+_HsLqtf1TI}rR*?x`rkIG_t}#LH(>Xyomu9!ely=_yo*o?|*a#p@U`Cq=mkI~xZ0DM0KxXVonFI}JFuPGH305P8~bz}O~{X<#EUX6ko* zG$V}9k;_CGbeGrNT|ec0!Z?_uVScB3!xvEPTX>~y4?`c-;Qxt1>Vt5GbuUsZLUuvGuT>R%a%-Mn0E zdV`)3s^|G`gWqy%Tm>BQ1Y`f>vC)pP(@e2vEgV-o@lz~U;8=l00Es675|;#eCRsRS z&iMvGait_bKQ#ZyV^J2W1F-EqA*WL2vE@MA)UI2F<`-5D z+morhcQK{Gc!a;uPmh{wSY`H{&GuXAWg9N%o}K&xzVlb0aDk)#u$P(0uems4O+6uiwl7+1%F_w+lQ!gYri;D(uSp!(xwT|ZU5ZKXT4La!PoEuy91G+Jk+6T6lsk#blon~79PD7 zHUugxrR-SQ+0`fFdn=>KOhqwo)U>(M#YZy5`Qe9Rdydbvt1 zH^)!xwyV-JbI|303D$`q!)GTmRdXMma_`mV@xVLpe2?b}NbxK{z(4>G)tHTUumJdW zm~iatuK6`2Kry^~@=$N09+NjvAXsI(i_#o%V&y;5<9)Pih>nUR-*$nAG@kUGS-jRV z2@n(6AV`ZWO@E~fw1h6V$0gn|%}e%-HwR(_zxNX3&vJ(r>b%kT(h_`|jLHEo7Y>J| zK?%I>sX;t7a3KUgUfvJukGIEmCTheQDM14DKPcj$L|;*C7nXS$KJQkKe#YVK3kMo? z$la=&UhbWEH(mp$i(__#gj}}(K3&h`5Rg#7PlhIFA8iB01Aq$e&q1=*%dCcijI&xb z&DS~%#!6018)Ex}y}g+h@k@0B`ZnU&A)1spX}_iS9b)gqKP&rS4!X(%R%?1cqj5iz zj<>KV^0WNw2)7$Kjb`0*;Wo*T63TBnSQ?HG3;>2{=gwteV1TwnG1kO2Li0rZ>0cnz zHFk!!{hPWL%fO~)cbL5oaOd1K@M1ZZ&m~(6ZnJIDc*9uq@E5FY$FrV{>(^)gGM~FS zW8CZu2&X2uAqD0Lny0u8LJxsw`V)AVBGAusS~=Fptg@X!EYat*&5R@J~c_!3=UEu)XdfG%3B0K& zUw_M-^K=fRCosxG9Pp9j%{SM**J}y0(qfz>p0qY-KuEN2aGOXULPGZ^BoFOFp=Mm< z4~blZmjGf_7?Q#3Z;J#siR`Q>=2fdWYs2_tHD#UGN-2M3+}a$`a*;NJo?2M(4SeR8 zRn=4VQ5^ox>!Fayi~u~g5YMcZ=vF>})@uFKN zsY{Lq&U5H!AZ);BfG2OS{RS3$X|X@C?*g;U_g1}y1o=-dwYmIWyE+Rkx6i2=<+pdr&^DKqOy(mkQ3+;QcK>UPoq5~ z2-TkiDI0oVFCbavEc-L@+92`5W~To{!sHB}fA4d@liqD#>BWk4ZEo%kGPQ1`?~k<5)?9>xH{}7J2JdYQh33?LM^X_J(h6u?ry$!rZx%4 zI-9prdk%4+B*VJ9oC(f#E7k`w0sV=^L(|P5TB~s@tEfeir3H3*-zLTkscFbv=sq4% z*6R)+mGOC&gW#Yd%*Gy#$OIbh@7{TrE%g}`8l#t@ll87(dafsH$UGMnDrCxYS#5bocp{N>LIr{3I?6!*!)iw4nJ%{RW$si3l}7&u@2r4*5IFQz9I+V~3rx ztJ5ty(>0FRXrjqxkBVnFHta4ePNvvmp6&}DXUZOQ8|i`Lh8TBHl)5)Ze;6MaZnR4- zTksJJVeZ8pmqN=A?>TT1#s_K0Th#zhyKtSmW=#%Q8L#KC`is)v1HTKFm#oijH*IKd zczeFe)^K)nTOIW@PE37y{MoOc*VpZCzZT4^rZ4YP*xRjV1Y=K&Puc=m;&LluOL@`} zyCIi{Vah4D4O^ZCRm&psI`km!RNp#d*pyvP^lBQIVcMPJefsZHF&Pg#iJ!mX!h$HS zm9mdW$cFKqS$vp-e)dDfiLw#|dfVOU)51{b+pp(Oid`(fsASGF9CvQ!vC@h+*EJAG z{RUrG#z>#{sOVrO*oG0fO9JOg*va`;476!FUe$lj)%Lb3?I1q#ZTFbjgzUaXqGz)N z>^h)t@s+#=sQEhi`siIvwc|W}=Hmd!rZSx({!#R@y^65`^>sP2u4bodc}$l`+iipK zsJ#~lYL6#^eI1dr9K$@&wborH{YiX(VD@g<)zT}aJ&HpYZB;%Sg;+^*(KmJ(O`!S7 zj21-Yqm%H6OVxu34HzQT>-)q;n@Q&y{}xP@F*Kll)Luo-k?GU&dQ@(qg@lut+0>De z{7|-IB}M-!CKCQD;6+$PIN~9^DX8j}LO|FX3M%FL5F10p`Y#0TN4|HV5{E#|`a?&f zPFRXzHN1X8p)5sT?dja(JYNaULHMO92lbWn2 z3O=Jfm14q?(Xp)olQLP;$92uv&SNGqFz239kRM6{t)PJvGnU3+pBSg;1Ipq%X<7yQ$?d> zP+oL_0`LaiZysZ5kOP1rtZ;`B+s|OB^fxYR7O%h2`+&%}*%PXtCqU z{O(Q4ic*P9o@s_+Gsfz(ooe#jY)vm0D>tr#$m?r*&IP1y(p zd|M}kwnAi0uk}%6!-%R(D|a2ZuSrDh{c6R#?Q0ei4)>z zA-0(;fI#K#$Z$?$yK_ouNH1ygmrnI4}Ml%v$+y{Qx3c2HDu8V z_A)?s>}W!qiMR8@Y638dtvl(t4t#h+2KIl#r_W3F z`CvMIlKXS}T;g{1Nqul_^y}A+ht((nxJi<)hThCr=JW17$|H8oOMV4LM+|rMbO#1d zkVhTu3D8HPy2>NafP;<^V;eqE4?4;{9u*(|rPX>(k zgpZ;THSNO3w1eroz$HjG6;IDaLKFDRUyqqy;EY`wulqYq!9K{~D#WTl^aBHP2~h7C zD>*1W;sXtui$yAwcJ=LZZy^DTLi+{-zOW|jBvq>%T!@R|8@y^7_1GclML?2qe`nzQ zKJa2a!FRA9AG0dH!)!2nMM5kB+}Z=TRg07lt`Hx#VW4etdUz;XCERLNm^DTP(<&R7 zSVO{C3%KU;WY=<9%C6=IG(k(XyNRQ+*y>X}Vm9`E3MXZxwWX3^;&6=W=@8HHa~N43 zA<}vPU&h1P^K!H&Qi+0x!piWlQSE0^XQh?x-LA@|j3uceqzl}o-Ox;cSyO^|hrOI?Dd6YyRbREW zAfyt_NfONclB1dT@pZA5o2sw+W>*4qvRqH1&qF#1#fvr9aV#iH6)M(5h3x1d#hl5R z#m)+w)r&{Dk(bc}u!%=nld-Or@fC94>x^+ro`pF$l}t@L zc^29>P=ox_{Q@*hups-IM_By^28*tn8QzKs3}zqGkZiP)ept9l%S$l=L-!)!>KbY4 zBaA_lf0fWwB1bo#w>!4#UT2>VTfXjXEoBMNu~w7*m~-M!RcI`YUS%UB)^NHUqy+BS z#4-rZ=M!BB{aKG42)0O<<=A2mU;(1)ymDaOV`ZY^VIWg{?X za|zrf^-qi1*ph7(oq$lq8vsIOBlCEXWfMFg0rUx0x2TwYoxsHfOPakddJvTV)uP?A zi0kL6#hhxjd3Zq%KG8vF+de&+{iUWZ!q;v$dyoo3Z#?y<7d7rEXS!Xd$UQZ{<6)tzvMp zctzal@YaBxZ>ClGH|<<1<`{oS)Ol++&}{<$XAVXnXV;Hjr$=eCklBu%CFWD2G(n?x zXf6M5;#+ouyBhrBcOssBtOTp_(p!y@N(E3#k}x{+@?Lv8Ik~vFczL<%rUx)u4Asy9 z4Q3|&p6R9-TXvcmLGI7G*Lc+4PP^`5>vlrB#u4?`CGSYIuZ^)Vqhu!ly4nHulpuGJ z61XzdK9@3YcxYR-#Q)Q_`cFmhIeonh7Zk*<$~tN+zmwuPQ~5krx|(sIY4Iv}(Wc9& za-1a;TK@!dp=LLz3Yau)l+99&80yXM96##_zHg72^kp9(tEzLz9psOLn)hOZ5c*-Z z34i;<2@AN>Zde$dpFBP<$@pt*mSsZIlIwDGpwURp3I*2{fk7u0Yi~8 ztbQd{N>F@=r*Uw#bE+7}ljbH1-D<&Nv(g`g%+co8f9KP$j=LDUx71JCe)dmGjix1N+%Y2;A=8a>mFRi2`YIGH zNYe1Eyz`1~7i&xS!IKgG@nl*88*FTF2OUCuAM#w%kqmTVqU}xiCVz8I^8 zu+w%sH@n$4gKutDb*3tEoJ+3QJ6Y?~=Y8X5^p5*k99l^c zn!o9YT$V0aTHbZ7JRwrzf*sr8TLz{b@KsfBTu?c`GTP|%Su2I8%lPm}KK>_~|DEk( z2-T973zftxPrKnfweJBXKd{DE(2LkYsx#a-m}B=Z;!0a!CjG z@F7&RLIv%8S(MN~Oe8#-T(S`ipynBpB%bw*V~?7`5F=)^`_2`z&VeoY^zF#*U)k$V zmi%SR#CBnE^yRmL0sERwhVGwweO$UH{s#W)G1l34+ZrG(g zks-QC5@k}GGy!uwE;TzTGTT}KH-Z+3^Ki4R z&t$FSkMyC~6xl1EJm6pDv~j?N0@~}A^=AL*@UNG`Wmh@x?CNy1ee+~o$dhT=xqm*h zEPbstFzy000%Rw(Zo1^i=Cl#OrZV@uDy&^z+7wHmi7qipBHC!S*yDt0FD|R9OKANG zJVpEmJe`35H(9Ry(x0w#Z2SH=Ir$dNBJ?VL_COXaZua`OnEo}e140zo$mnz}o}`z<9h(V#4RxvT zQFjmZ%_guL-Uo(&QMF}<3Rh-fI^+Nt!le2IpI?3N{dDN>0cq2;Z9SioH!>iO)N>l_ zXV?H78rxn~Lc&t}$cz`Xy!y8%0KN&VA&q>ffAemq%mI5Rc^9sojc)6Edj4GfR#Sj`V`0iVz$J^2nm zTmYXC+J8YyVEs_cV;DOs26?sjT%L2{jd~zYM|zq6_&#p8hEsB`f(>#H`W_-Ad@FDJtu>nrtd|Bfi=2_aOS=}b1_Q334`ZXxBscAVE^ zcCl+p152!Zds7sS0B59aT`z9DVsoY}HUuf$$U#NopG$!ZFhEa=-@9zNUhQ9Zz z7y8rD@B6RL*=P0-Y?S}829%>)O&O^U6O(qA8y&q%(yA)=z##+{QR?_LOiDcDp?jBl z6Fxc$Cge53F*#T&M14Vj#fsX$d|L=?ih*~3``^1a4KDfA%Z8iPsFbCOkA>R9zKwSP z&8&RlbB04WjU$zAU9ZGXS3daydm*t-7W-k-qz7vb30j~fj)9NQ zuL^=!+V7W^1j7Xl>5x|sHWbA@L&ZFiO1$`!RM+vu(Y@=v$H^PrZ?C|PJ+Lp0d}6f3w#=I|XN8r_eY$HK~de^g87u66M0j(C%lQaU=F67y2(T8uB=UZKHs* zJulx^p#I1}jA=j|YuyAHC6?JU@6l;DLL{&S<6xWyi-rNGViksyd3-$0h-OK*5@ZWR zJd4|$jP+&VuFA&C9rCgel*=!kf*2#g(Oq|CQ`dJZIS#=j@mn)Fn*JFmqiwM25%)_E z)a|W{iL%^+2!^1kaEARjb;C4*PVDbxG#N%zS_1bn79t$NMm=n`+=E8ty#KKP6mNqy z-wGMw0PolLOE2a1a%rNXrrs>i`moEP%4?yKE>#S*{$dup}-Wq|&5V zsKBQ`hWas%(seCFv$-lHCNdILx3|GI4yMP-N@mogC0E`trnB|rg?s7$8NH;sX{&T& zcbGqyz9o~Q-m}XbVcDh)OC#*ds^S>$dDrB53y~hmkaDaHX>i)c`}!4S%Kb5wEk5Nl zV5c-Z%It4%{q}0yC2({jxjGT`1> z+1Mf9ScZ>2$Fx;h2C5iU%9%O?YQoQ-PS_4M%QVs%e;?n{Pny773hk)90<};{ZD??-aF>P=q3e|5^r2EzCE3oAB!D-bUhm1Z z`edTzJMzl?qdA_^t_vJu2UXOYqotEN3>aA?JlsHMA>HIl+4~U+v`!~26<$5R-_WPo z48ebqK2UU4MMvTrM0%xvTcpZ6;LCCw+=6*gJVI+r2bnv1c6@u7WR}bEBf?CS15k67 z`!rfrOs>A&z%}8Xmofa%SLJ6LBuD|Fk0#XifJyuXr6J{OEOlk_mcc#IshumX>#Uh5 z$#nNjw!K(R`@E8sHr~lAsH;lgjX_FzC&k)*lBlwK#)R(piHD#?CcEF;lPSHlK&Dp9 zIKP7OUBi1+oewVvQN9W>19KDRICBGW5JS}Y_9)u&CjE_E44`a@U@)2R+JZmsul|VaJ|6FYcJ! z&OO7{Fl^bT3iBTH5}fQH3JP2+^+O#*(V_?9y!BPz_`kVYNCz4Fp@H`za`Q7;VHRrC zNTaP44~Cy8GtUSZll}fv;+$P6SIBFZ*cd*gH+($}MYEUy0gxl%MF_9Dyz#XBounAa z5xYu~TuUCqxfOwF_u>qTxu7xt?mO35XJH$ia94N!5y4k8rc;)J^FA7ZcWWyc8g?gn zBa+B=JpOQy!R$@V=u-LX8fq{Jo|L^;PaNw6^#E89gCBjFh>-vgLr#{sWa$FhwKuCM zZF##v=eDzQoFlJio5U)Q5<_?}@>S#{PbS#*I`*CF5xDGhZC}v)f73=EZDk#P%Ba7# zv-tuI;WO#qVHTAib=6$lG>W*o{9DvVP768|Dzg^fBU#k&)MGG)Gd_&lh%#$AaP%DI z`EX#^2*_zGGE7yBrbp$8;GHp|VHTkWY_CmnwA{+hfq$TASjG~486d+#y zTkuvw`i+M?mTGVnZ-?7Y(S%p+)gThm)sa+_VWz=p=5k}KIrUAnyx2y`4>*b|E* zE&dwrjhD>+j{0dT(|=>n4hu*_vW_Z^+T>FbAKW|&X|%l;fO`0}Kbg%Wte{?AL6)@2 z_=PdV_6^r$r}paoO{q56La&G;k6?21H&XC1*KAd|{CLTq-zBf+Xbc9;Q7xm649q}4 zJb3pwj`?dGArlxSzb%9fu+I?dAWYXR`;?|`B?ih>VBD^ zLzF4!r*Ql1zq$qpw@r5qI}H^T6`7ft2P%|^+Egu94(g;0=my{O>&~Oo`NRNS3>!CQ z*9=1SKUrO=%-lfGysGOFSQGHNJ-wYUmS7JPR47SCRP}a^S61~8zep3fzdyYNPjYdn z1hd9dgXI`Z&ide1`zw|zc-OJ*?UKq}iw@rUcTqj9n^)uJw+XEiEtgk2dk5iZ`-C4} zbeM<-YYSMd?(A%pD7L(+)bkwB7i#MLjr+m78HL=#QSiMil|-(BeA%Hyk6!e&5Xj=! zCq$pBrK(4QE(%yX!2)BF1h3I=;`D-N`}s(*vH&Zb&K{WWiaz}Bnv)Ry6?gJC&bpif-5 zUZXX#bk*4HLP#MDG^UAj%<+P1A=bAsr%)1YROMR^Jvj#ts+%uC31Mp%Y0bQz{%~*R zAj7K}B}%eLWogg%m_t(JY@d~(O^fLqAi;?jp;y5hgdDyU&^tM9n}{PELu5Il7h3!O=kb07X_0??_dMA5i5pEbCf6gdUvW!Vh4u; zWj*l( zhWTPwDa3*{>4=v3ETZ(%!SSFUunnrdxgc}4ey>Z10HJIvyOuY-VBi#>wXZzM^W7WZWmo5)gX);tqU7SW)q}Q4J+wLQoWB#(q>`C`z>q57Frl+ z_=TjwUCQ`ZN;U!ieK_Ut_D+Fd_w{wWb^9{7Vr9y3qfAP`B2V!P!!~N)Qsv>>H-Nb@ zFoJ#sF((1cX!ueb)s{hyd4|5uu3v#C^^sf9PmH8QZ%=Pg*n8uO?dZS zw_J;IcPDt|z)&3~ybi9xXH`;7e>&(guvqv87C|mx4mb<2C)1`$7BxzFx<7dT-X9`h z#j= zAnh)XnX0^MIKGZ&coCfPbKL-qO;^}ZDrJp!M|4-VL%rB`W2DDx-{jMhD(+6Ln$)cf ze`sp&-y6riDl@Ho8jFb;z2J~eE1^3@It<;!pPaw!Z$GY9O!qJiY+kBL|D56+YoCr%j+P^&N|T3Slo(M9b|njjRS27i){zpR%Esq*<>#B?fNZ4s*PC{9ks^x z^}}a2n;(Z_g>H!+dGULtmCOfj3AQxJSpl>lm0hP@-Gts%HAVpiJLPVcT(6+{r#^`E zCc0sB%=e4=@u+uO55KBlQz~a=dH?aomm$#2$#pd1@Ir93t9ShwMw4DF9M0-sF|LVe zbjuPj zjx(+V49S3k1-7?y5`{m)hO@hSe_1RJPpOAxww~!ZndtHjbS`cANA5W^jEmb57S^qL zbV=1SKF%gq=tocLU1{3&ls)b{m`Gv6K_2GCDSe9kz6a+^eYL$~_39_tpWFLywyT|fF)7M&BP}@`B zPpb(a>erg7)wZ31gw6-XwqCh5^BpC``i<--W5mm6)Ny&JYRsWiBHLw|3&5A^SNN1I zblHhR*=S>M8z<9|Er!}f8e(Y13XG^~oHBKWb|Bo=0lI;r%o$wjj@lSot0`;vWM7&r zQ_JMnm{`f>87JpQYRX4)Ho8d=PYx`vG7o2D2#NU&vV8CeK1VeZ^fI85&&Ej5KNv*2 z=QZ9k446IGeb)B+JRI5kHYF3yU->kX?WFDAdct_(&2z;#Wc}3>fuFZP0i7-5ff7#% z6#jf*1#Il)PdxT^@5lwdYF1^;s3wzMMa`<M#>U62VH~)97AD_j zf^$=FAi11oFnb8&Q+tq;**NE`!LG2SF(!3|&1;G`k3RERxnWB}3is0C!)(VD|KO%* zPRJ3$eHiOklB_ymI;$p@0Jmpz2QS76w>4eHWKacbvs17t+{6sJMRXEzlAr_mmITPR z^H1@Dw4ppDbPd3?%Kip9oA;+1WVYptj$G`n-)`<+U+-@AgzL{AM%lw5nP5C#AZ!Cp zIP0+o38dRkwNt((!Q=-}==}+~rwyCn zHOZ?`jxXxx*qGbh_L^SOxcDtjM6@@-*FO|+#X(RRm}X0(e<4cx0$GVVbSii*NR;`s z^m5{6v%`4po6N$7U*2j-9PC(BTM^yO%_zLcoE0)n@mReYuo4H}(s9FrHhx zQ!Zsz9(aC0(TiRL6ssjVxr4m(&!f5}Q*!vJE5kIZ?n`LA^8ho3H95*HmoFGOnWtSb zG$J|*ofPl=Grg@Ba5@JcVn>yzla$p(j$#y~Zs`qjhSwWErJZLNLM6MK5vpZSv9pO8 z>QUn}M@$n4Kl2xrIJPsC1JqQ|Mnco4F8fm^ltC=wtU7p|1G!=z|h9RiKs7&XxOgQ`aUa(Pg}!py1q*w5=b zeg_#+TDQxgbasa@0$=DKlt!r?z_@_1Z`oBHhCFy8`k|9Ak7ExTQ(CYh_2sEt_PQI5 z4Td9^U~0!ppDSQ2J(#wY#Y}=sNj|CY&|VzEUHJ~T*Wyn&-a}LsT%@OlBBzLbHZsAo zMg`E{O?K(E8*?WkStc{g8#F)vB}IZ|LE7*mR@D2p^x|~=1B5jtE4JMWW|8E-HvAF! z#`K1P>i0UGziI-Nsw3CEHcDMxv<~vCA->ERTEqLj?Q{>YRD?$4q^3nYEOn=fBVDRU zbO)PE0h>Rs9MqM!R$8xfS4$~?bwLu_9+-#v*Ymzd29RQJxe~>{Q?K5&3Q*3D`TVeE z-tn!7(-pM0w{K3p$#-n(T)K6t`Q1=?6Z@31Z*I#YOBPGm7OzZ^eaXTb{p)m`%q)4X zQYhxV^@(TvploSBZ8L`^sH1DWB3wmDKcu}%_@JmaGz^xDehAo2cpk%mKoThn8?(Z7 z{CGzxpKaZru9e=gZXx3spJsVJf?4WRrN2m|r(8gMEj>gA20Tkt+67<6N&`7fZl)7V zj2v9&yqB-joAKhui~^qHqVICYje#u0>$s0Y8OxUY_V!BgO7TwnyfyA?CjVA5SBI0; z4z#xk9$gY&#K!K95OKyUl@E~EA$*PRM4&aUvzlkGrz8C> z#$K1{f5F{7_#QbdicAa74cO!2CoZ7SNH_d$%{o)+6dI%!5-zc2Q+hNK=b!)zquDOF z7zDR_2^N-Q$BIV4qJG(cZ^to94cX^LLW6bz*{C}y|c*V`_tFS>zfHlW)+Lx$^^ps z;OhhzN+2?Kk$7pU&||M>%zd3jaH+`W0*ket`W=klVX!13!S<{e`E3` zTDHw*cFD1>(2;^My@kS%Wn~c%2;4;mV%hhk(;ro;mOv4!m$RuPXO%$6%^N~6S-Nvt z3n5PnSt>*w;;VOCY%Np{uRDw(2AAJ|9|I_J+z?+9$ug5{R?^0(Q*G_y>>BfI^GH$m zx2wG`Wj2kgcu`~P!yJ%xFgM-iX}wI}lD`PYJT@T1qbs~bV1SbLAj`g4y*lsIL?RWu zAYb+^S+RY2kvEJJ!;l&elGUc_{JNQKmA0B1n`LMxXOgyg)T_Hd?&i)&?=L|Zi&&#I zg+0K5hBivRW|GH+;wnRVBL)HsR2V0VE5056Gzl*piac?T7r~1Q8^5)R?>XkP(~9&l zG}ILiS4oM3tC%DKe>eSZ&P8g10pjAHz+hldbX44jxDmIsA2w7FCxWx1{QX)fG z4QWb_X4Pz?cc^C*lyi9<&RWH!6J!nB;u(ow8QMV!e-tv$d~%L3I|eJJ*rS`<(R{Vj zDhpR#*|s`ugB*T9x6Hv0@47)!Pa+g1YL?JUSB1u|!F*`E3HEWbAgANG)?&UMP$7+uQ00AVk__ulNhFF(oE@ocYHa%pJaFQrOhe3i1u4sbD8s4q6h5c?(-I;%RX5O*1)|XLOpf{c8qh^PJtGTifn|^vy zP)(=$<>v?~a!!||&|5kmpmIm}X%h+PO#j*rfB;|2qblyz2w;h2T zCHWc}NOWVrvcA{maaS?xuBf&!MI5T)x?Q>$N8&$)E{J7yRt(%8`;njLL{3eU$qtfl ze7k{laK;h;<+JvPUZs8J5brRKmEs)o&lq9eyI9virMA)uO2Tay^Utq9nmI;%9)27>NW!*oTVea7C$0nzZ;FqV>~JhnyKRm2mFq^FG$i zv!mIrR|#0j5f`qe?}kUAB|_@S^`ry{&QWitef z>t2IO*5~5Vlje7WfJQT2;a1sp850uG8wwxttjIvoXkYJ1iR=H2(}6-oVLaC0mQVi~ z)r^*r12H>mYl|DK@QrL-+5q8YRqzQ`V4XMLC$yjCS>oF`>F z-f~GDdGGSz&ZS^^t_CHYwahRAZr10vciFQT2;iP~qg0r|DCo zs;!>Nmw(#3)5@6?UvN{(LXz1+D}FD|{!);*J6bxFnS`rbO`-PyIn7qhP>}yFhwSVq z^Vj9h{X{3#AGH?2!jCpOO4$gE(tcXVl;8c_-qOVYp(3yL9l9t8%VcX#yee88E`w?K zfI^Fh#kS!roMWH+IoTEjv{SE4k=JCV3a*A|Z9YwT0x60q_mEm~-&}Xrw)~9IEI~=p z%rJ2WI6#doXQmB*HJ zNjr+_ZKiYFF>5BmdG4v4y2BEe9w?Jds){b1%kr7}&raq@OT`cB;IhN(AtnKB5b=Om z<@64uW}(xkUwtdjj1?TLP&LS+tMyHrQ6>mjA5o(bJP&@XugT^u97&Or9IYD|$&qs$ z>IzMd4!qv0LF|=E?SJ(jOd9RjuY2g40Rtrt6-KvLaeQ!0ko{M#=yODPf&Ht1xSy*T z-Jl>6K4`hfJ1S!K^3l>oZ#^M!JF}CHx;cB8(sO^qj7|)~foLQ~~JjcPZgPFr$)Tvy?dOrJ6XZi%5g;VVfSd!JOA+2Ct?}48{FW z5W6~Vy7B|5EIL*;Q*&gnn=jSJR;zULF}F*N|5$*iFO`hVk?z@bAb(FCyU``D))&a) z&Eilk4GERor{Q^yYZJXTUBuxu-G-0K_7MaLH-QhiY_MskRX>B;l z?PhD!?)#$nkOY)GBkq3XrROgv10TnDRsW6gWI#HP$p$+`0&pFaQ>AWZ4&|w8NFZj! zng;foEoQURthzlPdvKiUzWxGE6fm|+yP;4h?&sp5O>2{Qu!_*f`qnwt?!1;(4Rc`;n|kO%@p@tZ0Yr zbupob>o39$#-Zl(W(+L2b;_X6T!v`Z=bJDfIT|OY%vJIA2I?=sEY>r!!Ug3)D;>|w z>VbIG1^D350cJ*@0!w8hfsx7At*N#2c=F|DS@Uuv=J{M)(51QTbG!JY{NW27H@-Tq zO0-fB!qqB<7X?m>fBnNUCATACbu&hbOZ8?aE6Wsa3iy-tX1e3aeWTtPw{IIgC!EL; zI-gdX)x6=lfX=+E!R$)1!U1Q$nD=68RX{V&hGC!n9=D!jokG zE~b-Fv3Zw{I`t~>_C&myThF+9wCy{LiaUyRCwGr5(0l7BG*+$j!7?4ajckcdM7Wof zdJVfdkgN>Ekj#Q0WP}CePwmYCO1kAw0w6eltU@?9hJ5NgS&`3_7cpmOdQVH)7srHl zK*HsuY^9$KVvM%1#*ut#dXg7CFIG z+0L8lX>?{8X`=>1p$fQ|RGDpNajNW8O4wtJCWho>6peu?c7v+vRcrElQ7l3D{1@^D z7U7+2FiI8{&5wE7Ne+XnRIr24WE099LGyIvgQ#^!g|(9v@jw>-`^vw0#Vs{s(fXFu z)Fp}D(mftH>0h*w!hpbg>*tt~$5_jO(&sj2|BR*Bb9@(iwoIfxY!Tbv?OrKqIlz!$ z>aW}7CO}~y7P7KXL2G>4tA*U{ui01ns;PMh$sAo@=xmPux(TJPE_f|=a5bX@Wd700 zG#$fOL2k=Ic66yP?m=qDm?&RQr1$OCsI!-=?p zIE3Ce$J*%|XW&j9jSOh9r|mXs%*rIN6*dm1JhF+NW$;3nwFw)1&JYj{d@WyTl|_ua z9>D}tZacXE035HPol~OAYy6!~*5}WRvX^8u%LBbS-rU_MDobfi-l2=-nbk{pT(mHx z>6~CRIB3b7{d97O*5b~0S~X~Od0V9>WTFbqhgYcf z)h6hj?y@!{j1NG(?CMkM6AUY*nXl3AbQ9-P>{tl|ye-g1OK@hzE4L!CisN#(`|@w9 zygs|6f3L<=>Q)r&ghD6q@3u>S;FxvESs<>Yq}qHO`CCCRWTM2&`NV^Yx2%5e@JPIz z%;$Vx0QBW@XhN5cW#<`;$mYouhR541ANklHM3O3xOlJIt*f!6&d$zHm#0^=9Rhr^ zJwmrfR})IGyY2fZ6mHi##LOHh+;tS2l|EEiDuGe5OhcI$xhCyI538=v>yIhppz+ zp`}sLuU#HhsvcZ#O~xJ{cSeegW@qi<)OVk&L1QU@a)*DF7|ykzk%NNJn%y-cF6wv# zC7<^#&E=sHzg!2#TU?3&jzaFWpK&Ym*I1``?iu-T9kajhb4)EKaI8+*-t6a~cDXOk zvAgsPwilfMxsL64LFGyOFhyyJ8W`Pyld~<6-CVSY>Y9SbnXvgGg`GY^%|VG#fb_*c zK}Ru%gIH31J!y1K2(h}#`kGPf?^5W{G*#17g@1L$hR`hMhsv**sB84@I?mfmKbsbs zJs)uLl0vz-F0wNs;59BZu2j_wawtX!lqBkBi^mS=SMqQBmeBI!$^NfE>67OD!gShD zU8juLj>Ip>4rrz;0^F&->uQnrZq_v+2LMsBhUv7^=>K_s%LdU&LFe3@QQ*jZ6)K8& zEv*$cKkq~S2ka!Mbvj1uTD%M^gvRC`DdKq%`?5C46Wv`5&)?BJCq#ZZ9ib1$c5fq_fi5h z_-bWFaa zjz`#y8UhxhYO~Q*DNoHk=fBN8P=)AL#P}^AI&G|WPxevja(-}?W<5hPxGP(n$!W8l zjX7IBTpJjG`_I?!N!a$NacevMdfGSw6$$yGW}h6x;1&^QRB*I}PVibY{wajp%%~mnXr4Kif4)(?8KV{^p@x^9E91AzZPhyS+TTUEk&&RzRm33BB zFGDF*|C#Oa=6oRb!`Kn67Ba6vD@MtSr%x2nBs|&HY)-|C!qsW&>#!i4W_UcalHf(A ziTBC4WubKoPFyZ%J2<5`kHdzY&l>}*ni7iB_o*b-oJ5~!_rZ>97%4JZNgG+)PCkP% zuV+U}zWR8QexDxK_(O%6nD0pXoJSe!S}{)9S!d`}zUrjI?6yxe#n3Wsw+=^(QJ*Y9 z$=+;P+C)!|hSig~k8HGDmCm2OfT_H}=jjQZ|9ismc+m@eufq=4)ot<{H#pqwCh$j{ zgJ6QBx_(39!Dy2mBCsAry+Y0=G{hjkpE%>QNNGPSjH_FhneEvHrLh2yduYZhNFIv! zhtezs$&BERmHwg^I=u-Gg>Q`O5Cz9omSV{{$f#xt(*~tgDMD3hw07sG70w5Fz|e$! zGRz+5IfOprZrb=|7r_4w@Jb2zceSdovoVZ3S;q$-WvUbK{+L^Gvpn?oYG6Y7((m5( zdK|p`OHyQd9~zm#e~4Ke$mVu&(wpr!48X7A)uqPrzJXqkjPI_$x7)TedA_xfMcFjE zWkpsyCG#iRWFiu4ve+tz8Z%w)<&P!}`K+>PMT6|~9<^;R`O)aTh#f=6Rf1&?j-rjw zX;Z+VARllj0PL{zJSYBjC=f2_1`*6Bd$+hs2;Xcy2#00_4%sslF;(8Q#)7_o?1rzm zU+q`hfaH5;UbBVfQzf!}tG;fjW}^3zcm;maEM(Hy)lotuUnlm-RJ6LeL^V)@&)=-8 zn_S1{3t3(Za}9s^Zi(Cfm0R<^aVZr1VNxw=i)n!kwl!wXZbdP3sOEj(go72Pu)Qwm zvjqw~FkiPx2$9KImPDZ_eP#)0M1Dn5oAf;yVEFut6adb|y1cr1m=8fI^LVIs3@+YH7#dJF-Eb*BSIF zvFlF17Cv;LZPY_k69R?JiFolsBh)QnS}c9qQr?3oIIh=~{DdUl3noSUo|#6m{+v_SC3pYstr)Da_vca5;put zn&GS5EKY|@YgH_y=hYtut3xi{oHfmz7M9Zn$|utXzjvH^t6$JTyaVt5S&G@B2XMtkCMi0reAo zJ-;cj$hWP+Ej?;xa@YJm=Wm4N#jogq5sp$1M%mia-4Xit?pPhgi;k7_ZHas?CABaK zAHcqh$|NImH|h`6J+8;OI{8`sQ)GzUlH()bY8dTm|0GuGdgPlpo}N!;r83GR={?#A z!ks@K+HA4TYU|4x*KD4mUJ-O7DVWe=C&OgrSPp*B$H9z3-rC*}HHX2UTVEi`^&?Sb zg48zK>N|_qYNIIle1O)VRR$)7|7dgO`QtyW!52Vlz!7I(m(bD>XynTA(~|QZ*FS7E zg@sTC0UY{LFnLz&O@JssUj2g(a^#Pme^F_1Y)xj%kSm76JOY!aKgcVa!99lWKG@Gm z3E4r$?$Bj^&h_MG$fe2C(~ObR>n*A;_FK}&VD@QEv7P0x@J8j=L5-kFVv`h`0=eA4 zPvvY+YiItQ80rQAtoJ!{imAc6Pe&5+A0@Wnf^3&VJ%6m?J%J)}h6((}TP5XuPi69GvFs3E@LyoV~uR zWHZI796!%17RP+gdUG-(`c3%t<+`Fgy|=;+s}HO#zWz#mW(jY$%eNNlQ}@1BUz$ZS z9iN%R8b}M0xE#HjEei5{F$dV62$8FH#6HP-p8qH7soDW08nb{!70#(KJIo+5iZZI- z9BvBUlTsqG8J^xzF(eHtnI_o;w4?PetpO}s7sxLLKEXOx!%EpyXdC5#cH$%@?^Z0D z`w2Bgx~dqF;#o|EwAvA#lqwS$Q0I zxR;y@-wOkZyKxD!*j@3Eo$VCfSLjAaF+_Z z`=9W)j#dblYBJ&fNwZ)s)A;_*(1cRCY(*bWO0k8Y)3%S6Y$>>rx#o+%*0q_s{(Q1f zbAVkR)R$P#xJ_8(4C7kiSKL5i7HvuREu$JWTDIDv_K0mrhv6+VWAgJrtbBB$-vc~{ z^*r_S2tz0fhTd>%(fI&!)k?z-{dWwJHd=o)yA}4ff8-xgeu-Km&9m8_Z$;EjXR1t^PGk}Y@O1gzmqG`mdr59yaJ+3h^x8A4RQtP3;Jc7 zyW$@k$^1#zwt1dRoo8wW(t>aaqXv%SAY{0t}aG0GS1{%o;O$s2tJSy-jUHgrz zjWPM(>WwB(8My>$xxyn&J*(YpgM0Sy278TWH3Qlu+`#{lrIdW&Ol^x_sZpx8{iAm)E7aZ@_O?Rggv)z27Wlgg% z*}A6&WL9uya@&8TMkxPDjR0-v#k9Uvldmv(LwW~||MVYZt@YW)D8Gy$b=K{t`&sgI znt-^*;2wGKtZdDHw$o|1eC9ZKK0Yi_#C!`b>8mNx>V$=uhG$C!?X;xmLbZ@)U;gy% z6@~M`xZZ=(B1&RHkc63uEG%*-bmqrQO5uB8aIo^j;VoU75UJzk>Id(elM|v_5-gsF zqpM(@x&9dP72C|-^PSBzg+f(VZxTGo1grAA+SQI$Sis+k58rquQR6=ZZLjK14yBR& zO?W=Lo^>y8=H|=k?Rj~4e-4o%PKtI5;{r@|FH1XjKzBDSGo!-w)~C(Pq;A>xQxfKQ`78 zIb!TgkxuNw{&fQ(ZaXVE==3VN)#%Lg={j>{0of8`wmP-Osp^N=C5Z2(^Gk}w zYcbdG6Hc6+)R>T{YfV)ArU>QDy#uP*s!l~(ehJ`LCF(uKXMo;a#lwXv2Pftm<1BkY z<^Jj?(UBmhWd#J$E6R%yED0a3%9n^oW{lO3G7=`;FZcqCTWX&vuZe2u$4XS$6|+_@ z?AFYO-wuTyC$Y0YV?~>?eqhH{>+eG~w~^&->xgC>dU>cIKNE5*;Q*gu+q|u^oxuzE z=uFa>Z)5pTcP;_H_{Le>nV}vXBL?-p9&n^_!b&BE!cFk?Sc_hydJ^#_+$ZA<5^d`N7))0goR#)P!Y^R#N&#!Ad`s zmT-AOwWV`i993uY8?%3c7i>MEC+Qh28Z8IMYg1&_JDD{qyRy$|6SNrz)#$NLw$3}x zeexb(2wV5wSVz-R+Jw3;#`l0gT1UpJ_OJb``N^-^}DoP58+rz9}-i!UQXlmKAq6Y*q8*{mj-gM(tI znjB-2SKj(YXIJcuAWYE^q_Eo(HF^IJR9>-WM1j>%=L)Mg-TdWUWw=i`Wt|*0zUZ~3#K_@dP>!a#pSes`q zf~31ugU=A82qm`bvC~!SdEabh!5A>d#H$@q6W7vV&47tcclMx ze??sjj`cK4%U$BJGY&3r-r$*?M9h*RL!@{fI*)Snn-mJvkt_r4s9cJe&Zc7-?}ES^uiflO1+qvJ%W2612>4PqpnB?nrM zDU^ZibwX7a&QaqPK&kH$ycwJKdC&-e7dBNKh+a=)3E^z1hV^qa0!%p@WKk&+ADsEh;+C{ zjJ{++vp`G(apFp zb#fF1SR>~R76P0>PveW;0xrwM6j~rvd zA)O04b-wFFF-&&ikvLrL?YkqHPY0!;=M~K_C$kn^R-@oAVsS%5&D*Qp9&lb!L4Cz0 zK1@#%;~#qO3)tosq_x$20|L#*KVfn5Xi#ayfz3v9|mw{NsM_(%f>>?$~mnjND)}ax;ya(DYv|I!@E0#QO zo!Y+FuLExSaO>z}=kVW^%S zDe*AF8#h5v0Wf`_MA~IY+8++>>;^YGx$3$fYo%B*r~jzvB-Cff6`|0C6!fCO40O(I z5yivLMmImL-R@=!i-MfmR=Se5ChAtO2;S!)xwYemJ1e?*f0ABdJMGt7{^4rG@^SBF z(XUilb6}%(fG#!8GNM-IrVndF~PWp9zYo)a8(%kj3|6DkW zGR98$0`^%o?I!`vK`7?T^9`QV8Wwj65SG|Vj!DKiKX)gk#6$yI6$0XhP}cU;esl>x z99u20-GL$>;jW-8EXVQkKD=|TXes7f1}XBpU(SareQ@sG&hk*we(&-kj`ZfS^Fpy9 zDbhED3gy#pat%QF7k(aYcUlIJ6g1k$Yjm8@vcaWsC&`2(!F2X;i9FvACIQvu=?Wxr7Y$>Pcp1imqm*`ri*2fbqM1qZ~)32MB zI9vjPA-~1HFj`){#0_j0`aev)Wmr_*`@enPDk7jDEiiy|cX!v&-Hdd1NsCAfDbg+7 z4Bermbhm&EJ#^>5v%SCn-*G&z=hd3M_S$Q$>vNqa?Ve;-$#x1^UzT^QHf&4;nag%; zG`ywLpN$=UYx7x4DRU?#^*y(a!6BAd;rnOqc~LdqP7hf_9vFS59YL10uN{*D5%>cU zJ?Q+TZg%&nQnS9eNSH+keOkRqbeRVP;l>Qi36 zM5beHn6=+m-?*_zSvzx?UwDLTGQd8=Iwbx)D5R@c6f?XL_GwjCX29j=6%|NXoHDF2 z)+CWGjLYC{cPA$Fr;P)EhTU=>fV9+)L^%7S;!ueQ5-Fl)%s9?BG)rID?(W#)WO)Oy z$_mWjS`<%jo!3)1BfF4JWD=qWgBhB8$5MG~%ZLIn=xPYblyW`Y{)8Rgz$Z;741iE{ z0AvNTcb<kfE*-NrD>WFOh1yD`G-V%O4`Vp!KjOcT(N|Uk}0ekZBT}h>bXLj=k zvg!9E86SaqHLiUYXH(P<54ekA+F*r$o#b$`(fG@aIii`dkgTi~((A4Vl()-S#0fi( zyHW7tN-YN-ID0}QrXSt&qtXTKxJhF|OBmsWVuO#UnmOI!3{NnB=0q29*UhRT?34mX zB$Vm+f&Thb$~_G9a0&&##2a(BzD%$V`rfKt8b-oyd;fn~fOTR$%$}N`2`N?#6h`}A zc^O6jy>N;x33K$=*41qzo`<}eQ_Z2|Uv`OKZR9o5BPRVGv3!+tf0uMuPQ{Zp?$}De zrqh2_x|_Ad9Dx!=3*hGIt3~tQd0VOuYCv9vvY;U&(j-9tifeAw?orJz%K!$}eZ8Ljse*<8eI*i7m|m+Q94! z>U=-nzJ76#$}YqZQyF_K=%eAv2k+k8@*5TM`hbyKyUh_P;DqJimE#PALn@_#d#yH7 z!S!KJqBp3ZvRYfk?2kSEO>W?Wu8f)ptG?U*V@y6}Tj~qT!EFn}q7ZxN^}|*@cV1h- zqSp~vlbybXS*-~M@KpY1+Ona>Y*4yeb^uE#U|Y#u*joGat%)vpy9J-`;lZol6US#N z%INm{rG<#i)|hxt)VUW!&-z&RDJ+! zyEsKajnR&7wb7VowuP4{JEIRXgBAZ{R)6kJ9}|3il*Zti${6o=jvyVQ@fYeD&5%fie%i*?HPT*FzU;zQje5T z6~F8Ky^|XkVNqc+6QnQi^)3@-G^DGU_%Z)m=f}GT?_kv?=JwV)_c>mXnG*x=+25B1 z8JT1PbFFTQviC~V9i89-b{h7}VHd?!(I8J(;iEXL zC!dGgykG5y0CDRTS@*(y_fK06(^dYB~hwk)%W7_ z30l6Fi8+Hg4=4gjq1A8~$tN&qX|ozscK0Vv^)HmC6}skq3BKxn#lL8US7>*V6llQ~ zJJpEXX29$qL+OK7^~?grEFflc9CLNF(`Q4SzWTZR-8yq))V<|i%>1>WZTyv<1M=~7 zc0g&pf|%spVRAs!F3D;{w-8_UxIEiA|3TE3elsUF{jqp?F!cqZGC*$kh54+6#0$+V z(lc7{A=9MY^b%*+OILqin)+huQoTrzuxqP1L0wuVanvPw{j)a+mtxywtDx1%@qgvXQstZ7sdVykN=U_`$qBloDq{UwK05F0<} z!HZrd!NDg(*|ucBlDlV3MEkQ^F_TBE@L!6_!)L=_>h1T)vTVy^+*;O8_;%ge7m(x9 zIp|sGDjy&GPa0nse zlL_Oq{q-c+^6y?jAyj~~^vwtz{<*=szcFXu@(vL#2E9Y)iyyf;bySx=??@IFOMVyd z75;Eyr?4%hzB|czG;}LT#rw(uCs-D)HcUNzg0u0k`s9=F@vAUCzB5yR7a|DfBC|sr zCA<+^Psq?#osqIr59YYO{lbsU{h?Tq{#SUAn|E0xEpr$`{9tz5@02B`L-MI6{l}~U zrs_p+MUHkaO%9%M{|wdwuJ@3H9N?;2cv>8)B`tjZk&>h1n@s^eUlSuboq<{2MpF6> zvbNhWJ|TdOn=stj;j4p(l`)We8}_%5syJ)^uY%~_-W9l05Qa?#KvoQ~AY%AMum{?L zu~1Dx=!}VwFWqowhoXm`nNopwK>Cx{dRu0X-|6%3VGIYn- ze>J3=YwCY4aPP_djWzG%$D9$mP=p?}dao%Dl@Ud?!86>5o~gLQ;K;k83pKkJuYFy3 zS@tWVsm%o{6l7#8W4x;tuVwnp1k=#x-+%sN@m>TS1LDu<-2kuF-GxAVuuRc{e%)M+ z8O2KK`i1e=Q@3`cz)w0uWjwO#wS@K!c#7I%pN|5;xyY>j7Kc2Ao}~e60PvPvtOZKG z)hpfy@<5J|o$3p|$+nneOM!MaJYw)dG-i$;QTo$*lvI9BZCgCMoG}ffgn4fi<$rMhF&gMF#ACqHCiEM z!8le3om+oc-#Q`g2hOJ&M9o!`veIz_1J_dq_MIdsOt{szs5e(z@Q|V4UbZUTT?ejg z3!UR z5P538-FNF<-yfjVR9m_UQbMog%~3o}`ufQh+>(!Bi|TGS9%7U!CPGXd z=D|8g0cbN@UmiVMdKx7Q+}@hhEh=0`^d!op#D(S6eX}#DFZ;UwBBa*chPh2$dEt$L zl8kHGg6zze1!AHT416c{*P`v!6#myHGzvcndfW-xcSeDdK0*K=%${UAjeY* ztr+Z9PF)ri98AX4;e{Wt_R@vXmXDfvF=5M29zEtak|d{GoKzZ8PFC5I=ak=^4eQ&u z+O0{CN%{ooP1y3#r+jsEVobMBxUY}0)5}z%P`1jm**SE5YyESu8?ircDj=Q~clzZ_c19*ZY`6hxBtR7>iY|R?fm_0Ubr4ddf zK@1GZx9+}n4f<~-T&DJI@x9Bs=e(I%E@0+LQ7ln~70_Z7l-H~jfO9lU#C9jO=eT>D z&_Plc`ck*EL0jKBPj@ed$c>%~S41hp`HJ+vaK2-9%+@`S^>zS`E2p^kVowc{MVl}6 z!}uSJ%X#iLHs&!}b`mfq3+t?eW{mm!G4k%yVf)@@+$Kw9M$8B{k z$bO0oYic)M(oY`THIPcqd@^h;h$q~GG5;${zndQPmE0@lub=L=%5JL#f?OA-Y3uo_ zJ`j;^SLuRgqI!%N8wqV?Ng_3uqu-oiXGUax?5>6KGoN%7YCPg3f|E92e>8qV?@-bv za5Ue(NdYkYsX^aElY0VY4^AH=tdiSlY29qv`#3+@5E``oun3b|_RlQmcgVcP_XYkE ze>y{G6sIjq?)Den#4~n$G!$D_6Hq^FzacmE&Vr(`#6oXTNMjE6`?1pGy;nb>%pkKT z43&`BsRmp1Y9ro=RQp2U>soQ=uTq=Mg>FVy8{BUJYKO^5!3bXy&ayt=P6lLYoza=@ z&5hg^;~6f;u$e_*pXb!iAK+Q+jp6kYvmd!CdLvs23zQ?SPvyb92XV0NR5Nw0ya+%; z`vcI>DpI`fTYq}E?z-28vWSC-^Rw@@Qf5uH7df23T9&OzR(XR(Q`7szjG+XNHE`x# zg>-18w-@4>zAJz3lXTw)Tuqe8DBA~p-4DH{q`6MbQR=k+9fvdvK7P-TJUXuW)%uQj zQx=7X0eS%oS+wCCEh&>{t_8wlpHtwA=zPl}EWOqL!mAH~D7|XMLYeO_vjoV58^;Ej z8}O}r*`bI_zxB?M`;PKOHmpcGZ*8{iJ<0^3MS)Z}3x!s*6);TZF}JHAoNy68|s=h_Dm8j8?7d9SHtAr_Wp zy>L;^zK?I=%_~>EkwSaj@Jf(7Jz>==zEfd^MVzbeRTULNhbyH*M34oGGIXb92G)JIZNTd=$zP5!0R1<5INAG)Jq1Rv(}Le{_TP{zQ4DUCcqai4M++0(894hACzxQNx!AP0HrODPA0Gm!FMyo?J(CqLn-N8NO~4`IoUu zDcTi6_9jnU+`6Hq|AQIHKRWzJ1iK!-6nnB?))k2RZ>4rq|6(uw%G!LUF0X{vL$uS( zH8qAd0eo!Ku=P^HW68sM#spZ9D0`@@%u5f6h(DsKL?XpXUS}?uU7I!3_M5o2jgQm9 zCyFOmHoO4%@p*fBf1Kk#&?2>y!LHgl3nn#U;&EfVv-iubj4|Dv?f*sSY|HFI=V;I+ z#Y#tT_-Qr&ZUlf^zmc#zRa>mA(5N0&{Yuvdfczgv4jBKEvXb2{cso{h+%68-^A)im zD?xWR_tUWQjUN&ShU=bP4c@X*|91S8>lF;RtVqsMh>;^aZOTGYQz0rNQ71euFS={( zvGvRvu8uenYk}$9ozFoP=T)M&An~zcGL)PsO~wxomsi&Vs@HLk8l;SLIvqu>>Fvd# zdxse2r3;cMjL+J?ax`=I{yd#MS3M47%kQi$_lOP`gGAtTHV3^fQ{1$r2rwXJ(f94m z?fLodi)5wu_~BdRyo3~(qaVeWdpL{B5gfmV!Vh~s!rtUYiz6gx!y;vQN zSlti%0Q|*yIQ-w%x2)3pA7^&;JjyY(jNRmk@x zU+3w*i$p6DvEkxGfIg4JL0pF*4hyuiKr08djM@lgYHG#eDMWz()|VKtO`$kZ6;MT2alVJrm-4%I-hJw9r%fMC;O7h?q%bAWAAT6H$#<*jj{qU z--iOY#*EvgL%8prY>PpA$qHR4f*D4}ez7g_12?>| z%LoHIyuZZSB7RPrT(q5#RCMdf>iiC}jqb;kVdV*DSE5lTf8;V%vYw=44E|1MYjQxO zkP+FVfV_^aI9!K`J+?Ot6e$nAk(MmqzCT_J=Jl-yRDW1K z#b8MPP4I+X5uU#@W=V#cyTISB`dMB~AU_+=5m_9ZFE}`&V7~3Fe*AKA-zX@VIO17H zQl!$@dR}jaJU#uUoY1yLRL_8s-~q(KbI_9;wG^q@rb^lAp>PqLTSZ&*t$;_xls zEQl((2ZLm@xKL9{t#OgGIU8o00V)zz4WqLjZ;5jUAZZTe|d)_$99)Uzlc;6STLo>mvq zVNJ;C(}8B_P*hNt+0Pc;Ivc;UEgL0v+hKjLTUVfstFsA|m)}rjQdILl1FrIJhG*qL zFKZcd*4pcM5m^W_{3U_;_H-@KT!9-C?yoCG9((e~tY1o~*e~dcjdT`y(@z8E)K=?1>sz$v& z+N?q*1III;vcc2@T}-R!@PUO6K}sPQjM%gJ691IXKFi=pZ!XvvK!N;bQrfMsLfiR6 zeJ=>!R;-RL>U?i z_lH#Xqe~Mk&5HcVD2E%OF(iqp(Fdv6wL8QK`+b&aSYn-o<)S-|*gn)9fVE1zsD-ON zxNQ-;M`4E-MHP`96npAbwcu}MSjYAN)3F9b@T$-pSrBVfierP_FLA8yK~lmO0qNd~T=dNG+B@O7GbVKS)Jm@grD$>FS| z`+PE7q_Y+&B8NOR?UX8xR;4ZITe0y^{l)aS?M)qXnemGRL?6%(lfkl%SH~aVXJacQ zEX-1(lnD@j$c;wz{p5LseCKz`!t|?=__tF#WMJ%z44KdLc;fLn)7SF>?Ou0 zBx{O=k^?!gl@JirwtK505#&AA;%BE?3c%BD*F{c)dH$v#8DFVsee>)4*@y&G{N+pW z`s?fn$qa1%GNkq6ZY&CZa7=;>uc|Bk2==05;R@K1C`8HF&>*xbSogS>yEdQxl(lMa zy)8LT;DS5X`<)7Ar=IJL5D(YoKwqoENIfO-hSjk0yoQ%`CcZFb?%F~VSCSqzqx3_+G6_8SW!|6r4zq}_Ds!Ci~dmfGEi-lRT=?hjU1fYd(OigIN;agB#p zE4^HHj}J=@+ZrAh)2;S2c|?=k#}z3vXpr>S4y_$^aJG~f@w#E%uu)m_z2tGS>+(y( zlS?*1rWO4rC%BIdT55+P+205rQ>CH9T~@WASkIKv@Ssu@SRu)>RIUr$KcNpOi>dS$ z@jEYTcuw-n8Xpfj=>?t)<{<5W;N@~F>$vdlovYqxF;i8K%=o`ecxu}{Z4QRU!?2CC zq{B0BZ60EEOVRJ^egr}*OuV+tXZ*Jx=d=? z7-noMu|37dc{VjKtIwpl)*>9~FA3#b4Iq7ULR-sxgWI7_Z*zr+UQ5>|EsIrk%32P; z>xX94X;nLwDKcrYcqIN~ct4Tg3fSPqYxrM>lU1pJ2s*iW&j6S#SDEpDu(3ml?`<}v z>tW!tH=QXB^G7pTECW|1!Qp0uu#mQVItclNUY^l;#p~dhz$*BJY&vZ#*!6~DG|YPj z;4&EPxOg1ym&Jmf`0#b2@Wamv%^J@n%xZ@VR?fNy+cw&zawk|J+c`Mn;bLuDhR41x z@1jt!8q$d`h`*N$G*q%D8_)2Z&CS>y> z=W3Q+m^9t5?KFtqC`KVAKXF4ZatLoYLlqvJDon#VT!XeQFGtE@x+sHR)Dc|4_DyH* z%l3Y#5dl60t8eE{qw+kq-IVhlyJCeAc+a1aQvRu&kK)f99(F2{_u7KxnK+pjeSFov z`l46*xeS%|oX|8%%k*J$umcoS()mG1PUcOj&?PjmnytfDpybGPk7t#DQeBXH$zii7 zfH28hzKbVoel4-j&rJBhayg7rmFZB&Ak}m_Hl{4hdgpm#nPI5|&8p{aF%L^*2FSk# zKC!=~{~>kSTMm?AD;3K84xh39!MI@5Td0M@J$pP?lSF%;ZdGWMhYUY0FFIC3?DJ%7 z;G8Fx=51Yx&gZvmDpZ5Ag<(C})#{ASL)NQh)`G$ZiOQ6WZGJ;RXR0mqIQHOHX%ip% z1JeqK$IhDe>S1r;oc2LSRr3bZvG2Cpn^~=o1Iqh~wtKMK>+~r71(!n|iW*2U-^^d= z8H#kT>0Q;ISI_pPe!NMS1GT2?3}FIo#Zp}}U&75W&woFxD~|rm#F-8#Vui_g)a*TY zX2<)YvR&p8mf5GLC(=fCs-!(o_L|bRaa!Lq{l;`4dM*vz3l%o(1kLYwyJidA!iapE z)m-V7VO~spWrLzAUQAjxKOKuz$C5gsD(KmG8_R}L8=XM?cQ4JTMIx#y(9~Ob`ne_z z9fZM55tUx-7wQIo%)0e@NZzv!ufga@S-$iwW{>Nsb_c(zItDJ^RD%HR-p3rKQnJ1v zI3!nvC$8eanOvrJT-xg1v0UO{EjeXN*G>>blSU6#NdnBf|)_nx7W zf+L;)ZbVG;`+tIcu-W%7fj*Db2E}&3(nWcIPs?-An<|Ai!M$oSwDEl9cjQYxHW*f8 z!lC1&=S4VxwcA}+xl@zE?NhL=lkqKEaA@u`z+~dAW-R`%A?U`zq|0`nATAr^p9`NL zzfMV9D-KNQ4#N$8=lV{@kj6kEf}>4aFYimerUqE&qQs@$9N_)O+YKqL6J#*h_Tme~ zvJ!NFEp2XLlK)(@M$^?yNyqEW4(tE2@TjM#%n?LO>u2{k9fuPNl5p~ccZLY@I!mR= zTZw9I(P;1GKV+7%ep@Jq)vgDR)d@_%)tDI!%?>Ur22jj7jU8N;Rb%&jkGf-8of z44Vd$9>o877Wv5pM(Fykvq8rdkcy#PZWS*A@(P?x^>Dsr$>zR$lMs2{$b+lz=?q0; zx))!S6#XfHl|s<5C>guz9$sB%j;Tzw0qLf02PkoAsbwP)9p>d5cYWJ!4~GZ6rrh)g zJgetB3M>dVyL^A{5X{i`AAM;X0j#gw8f56u$jQ*Kl;59UU@>VqR$CCEeI~PQJcM?- z#zkWP^D*g^%1am3OwsSzZ{y*h*5Tr4w*f0DD;eu;F%wp%)i-$)cR34();Q{=A8{_A zU>Q1!$1+ODpf={Em%7>uKL(wIcDo>e%63P1kGFUfX!`G_V5}9>)EF=0vAa~wLj#%h z;&9o67%}|qryuD!Hz_w&ar~KEq?Py}#cMe(V{m>#FP7+JjUxh(UVW+52IP&3Z5ig6qj8U2-((`C#>X^Q&?XFxk9G6g zekl=^k?)YsK7aq3u&Y5sszT(QV5qffLncntblhW&b2#+s$!p3*D@o125Elo@RV2Tg zqjjm>kq;^7zQ}>1=ZXZ~Hfh@c*tQ4;Va5IOy?XZlQQMNjv^%t`KHN;+m6gu1cj0EE zeTwMeNkXx}c=qi_S0i>fZs#1qA5z!me@g7%75;@>RhT}~4t;C2m5GwO2H{g(#deDB z#`ETh&5kNozV<_%W30wkg}9KoWy2>no%1N!R@|>y!jCtNGDu0?ZPMR8`O;j7?-_B_ zt!mnNimqgm1GLlnyn;0>LiFFdFDpNbtroDuIl2bpA}^Jcl4OTpGBH^FUlyRaHa6*k z7`Zcy_nGmIcDT5~Fp2-bxYR?sSc3wSPGy(ec50fa5@5$hUCogzXNJ+YgJoyucPBgN zBN2y`l`z8KHePXpnUsKA#_kl=Z|@U@Pq!7`-t>@EiuRg_#X6uCr|fXgPM~|CW-vfm$e7AED#>8D#Bu`{6HH4rJcai zNyi<9ZKym;I8D!SbdY}F?V8q0k+jkow*~TLZwXqqGc7$#89{i?!lR~UYUVj%?vlsu z5K&>@2l4&NwQ-rLKi$^x0+sFHl(0lgg{ixt*LAOX{Cpx?a>Gw2=tU(;T@H$sbk&qt z!e?j8I#h3U)c`Y5F4=pC0r%^^+?7?K~dNwuY$-mVTi*g7zI0}xez+T|a>b1Wf@ zHURYSJ3LzhuwiFC=RP2l_QTM87;u9$V@h^<4YNdZ0t7l*g1oOo)id9{1>}XKj}DF^ z1#HPwoMv!J;j?Gow=3c`Bp>J=wK0F(%lY9jT+#s$x&k|{T9{o?Dx^PdAqHXTI3_2! zt(HiOCU9$gYyll5m`k}f8STq%Ir)0E{?>KiOq6>$KSyW#X4Mxb%F`kfKo>?`2iebo zXocQ>`b2vIxiwW_(`@AGD$Y)&{c8Zt)F+XmceP1UA;I{wwYW#H{2rU7{pV3n%mSI$ z^jx;b%H?NePn&pvm0{IcrkVr`MRofMwA(Q>I!UI-#tJ+f8|NV$bKxgxnl{M1KIFlh zb4vV|0IM#;5Oe%_C54$vWqf|*7U~+20L|<=Oh_Q41mOpKgeE`wTKy||Tjny1)tP8* zw!GJ8mT)%ps<-@1P6F*i!a_%8=!N`POj+Z;y@2svS^(P{BVG#~%*HL+m(Gl6Od7Gg6f~D2WxmReO@ncVxJ%Z5nT(b7_puzf(w@Ki&+LBasU7$iz6xOMuH2 zQ9Y#&7}C=59S+2NitWA71Lzu=>n<~F*>GWFYtdiFfaj0FEd3V;7GDK?2Py2jSqS2377AFbj0{ z5NN(K?;1%g+;Va4Y8qyS-j-}OAQ&;{0TNVd&({C)R4jZu2oXdWT z0VsE9BTmEe@;u>*ypEwkO%HOdHY`!A0d_A)0)`@43?DZ2+ApIm?gvaRuJ4z!i-{@K z3sQF5+--xBOpXTxF)2o5*F6MHq`NzJzZlB?V-Fe2W2GEtGR|FcxC!*}9@;TCE#(<} zU}SVVsNCB)_-?}L*Hh0p#F7Xs`HRQs%cd-a*3So&K)><)AAGZljA{;`){ujsGcM72 z^TCrR!J7q5>R0udpU5zBv#&03!c`x?8aCcXJ{h! z3mP3z-_v`$cm~wuQU-jJ7@%Gf*3991x*r#57hBNMX#=?|mD`dCxU+S9?D2{ZE8fbC zPQc&8%j{p!6`IWTON{FHo^(~NM3OvvLc)UlSlqIDviXcl^~^gfOdtZb3v9Z&X+L)M zG=)Fvdpz%V7HLoyY#Ztz2m`AdTYgh3_Wii7a4cI`+(o^J;*z(k@%S}Jt5(^5B&vWO z$Mvw~n1wBb&VP(8oi8=D2UeG$8#pV3U7{XMLnUSBwybA^tiOb;%NtZh zh{DB}(H@#Mixq>iG*!>jV+4MybIIRyRjdo`>?#ZP_1=e1R9;UBmJy*nwPR*7kv`8E z7E|w{R!z=fWnfMDq2~Wj7aP(Q;$&i_JUaF+*T6$455?Q1-m|uRJGuGyX=gUZbwfR9 zTN$+uM_>E~h*UO=nf6IDVtf*ur^g5~dm?$fV4ic^WmD9h+kDzAZFY0leyPNW*>nam zNLssDh`RPUjXGS+oi;*Bp64CS`T9&O&k^)eVK<1*?AT$0IA)_Y0^EL^4U){zQ1{uy zH*=v8|zPKqP(2`AEqxg zOIv3)>B0S2dtOGN9wsanMdG%Q@LjAm+V!7*UooeMpR%RwtCw-snI!iBYS$o@!d&6I zVa=9~8^hgZ?>hn4W+u`sm^LZESd$5C*~1j#-kz8bYO6h^e|g%RtO9x+t*_{+_xP_} z%eAzn9Z)NgT#CMK!wK)A{$hoEMjAScSH9_(zAx_d-LIe}L*Q4(N4%U$&+WQ3m?d>pr-dwr z%&s%6^Dbr|@P7@TcUKH2t`#N_#ze-HcajQP*Cnct*e6^AyciR1DC)BW9D1DaTyCQ} zG+Vea%?X&;33ZNhSL+Wd%z2Oes7&Z)IH0-QBNK1`HR>8W3aJm_TWfS5 zjdz^ZP(Q*F*g>-&2oqDAgo+iecR{z_I6svwZxTn)9H&mD>cW8?(t_8psR`(j>6#mu zB=q=E%l$GhbFEckUj;3iL~dO-V5?hiiTXE89+MMtUnLB5vV#m{F6-ViCV(k3M_fcE z2{n=LW$*8Br?XsqY0W3oE5ro-GveryJ2)K#L*0uJ95F)BEReQ7bx1@uq5@%ILKB=J zcuP%}P*VybP`0&u6ISDlIx2lJtiX+}Z5jX_)Ss@DN4GYcDZWec)8t9@-IWn$EQ^6e z)+s#jGbmZJ5?VS2)*oNC^vY8Mr8@U#KCU;gkOOAp!x|86lgedFNY{$TDCD4F`kMSe za^8<_`%-gnA#*3szidsiHHj7dc&V4EZymX`X;l>MV3R=KZ5>4|e zPuJnbRU2x2$V|yC_Q27K|6!^#&(-uBk5>k_+gkJ+)-LTeg{}>Y7%8K`b^n?705TS ztkt-BtNMDlVQ1^7DL*Y+%k@1yL0msm=krj0=F~Zsi+goDnx>elXA<~Nf&}I}7h)|; z1{jlLvibS5S)AUUDI-+o8=9I^?dNYo%efcdqjUx60leL+EA!umn~zZo#(ZT*2WI>nBsO1qaN*bM~4LW zFGDG0N#7}Z2`tLx*G5}zxq^=ih;yu$d!tkdZ~{laMZ*j(Mf(D;zDX(PfuAUq{}gk8 zWs%aLs&?)6U*#Z1GIlPukEd}ls-`A=4b}l^z2uFel0igIXke_dnff>PY1jn?g91<1 z*i2wP!>L}-B4W>X*fhYX@@z7|^+pBoqAj8uI|R`ER6Teipe0~iaNCVBOJE<^*wAun z4-90!j9sQ>m%bb$w+J6gAI*5FP3!jY7{Te|nyZUBV+M|fxvSV_`hH9Ob#*FwB3O>q zs|nMc{;D~1(R(6G5)a-juEUJ{QlCD(RLxIepQxpPN@r{3JO1F5@AhXss<*YXjdLG& zNeS=rAC9a&2ts^&y}`1M)+g9=)L z)a*_>I!`3B+ppJRy~Q%;!E!tHo~S@0a}C65$e7rDK}aps>0U5Ky(D>sq>tNgg!RE!-h5 zU>9$|Pv)#>YsfdNY=iF!y^sz-Ntz-j$Ann~7#si6#zR(4i0<`mlJl0qa%?4}lU_0H zp60h}A3;d16UYdf2$6W!{K=uxps|0xet(+^{X~cPqiJe*rNy(UzOU>YOIA zWYJo~_Z*n~rejmMq4a47@0}|@-B)|E7=%viwvR)0$HymX#`&<01!b`2nV!nmZ>N!4 z)!1C~TYR>g0fpHHP6A48Nssf&fyo;!F2kndhC-?e7`gEU(j)Y5*0uP1tXO*zjBYl| z-&Bql=p?lE*fP^dJu*~FWit`&#tfe)f7KP9Wy{yA5nb(iXo{G8)C>icaz+*yNlpcD z-wlApVefoXbB~T0`k$v04mRvak@cfK+3XdY<6&cj!6Oqhy%LHk4h~Li`*zgYPR;mxzm0&kV46=J3$OO}~0K_Uvb2Z9vwP*nc_ok`rTPO=zttq3$da@J;M6t}I zqIDuB$gk)+?usC7LzGK)Y?nTZJP8%K!yf5I1y^F5;$q{Q2eY;wy)>9XMvQ+^J`&S# ziTv2{xv(&quwi*UWtXDH`894`jard=Cy_RSh8gP`6@nS_Vw8H*G_f@L5lPdkz?ti} zBVthTa*972L{oev3F?;Zg;Rf0ri&K5#4yW8aLCCZv08) zlvY-+%I`1TnmgT%Rd2&%-U990UOjGl!V42Rb8wb$v!t$W+Q5%B7N?>aqqf&+rs1^8 zrQ-Dlnnk=6i+Q`@0bi9hTGuQWYQ(Q^qA6!7N>WN<(^Jag&-&6YE1FNi#TsDTC6QMc z=2`}3u>L`D=!9Ao80AjJ0DVP_4aJWhB$xm6YpP_{*fH7+HlS+atB|2J@=kjH(ts8> z`nb;3`Z^B!hN~;!;I1BFW;Ly~gF)s(52pI|(~ZB&fQI&!lDv3S&s5D@#zx?5QD~02 zAwMW~Jj&!7TtcMiY{}7>SpFz6C{gAya%=JvrFR|uJM6szzzjDqwnNd^SNUehC+GE8 zqK{MW2<|ndiu~&oq%sD2Cbs9p(aUYo0P?QAvzC#mAH;~+K$C4`Ps`dhcyY=?!k#y` zA(3~a%GbQ&%TH(DaXnmA=vY+tH(3S&N1?=_Y9&(i7IPfp7g_!o%!vTQ!C$vDVzu1v zOMz11ds5F^@=^d@NheRu7pQ^vW4G@QX^AEQcwYZx2w~5W|7g}=0B{*+Ra}Xt8DffB zX%No`FKR$@?%k*_2&BX#dxG$p+c%3FP2ko3Bf05u+cV=nuxnYcBoEbS8cukQl(nDE zo{dN)P?R;IL2ea_;2p-?Z_&)BCB6+$76(u~f8hhYSCLXRW2TdLYNk9at8NQXv)?{{ zv)tn)qcv^Z_9`keX94Rj7xk*OnqOk1Olr#5fj1R~CF;;k4~{IcZ`S_|dEJc^Sm7cd z?K8ALch8oXR$~&~*4y=3wwU4WTMUse-2&%1SE!Aocd#mNdl|n5QVO;MqqP#h=0+m6 z`e^Gki?;{ja78XmaCpK`={nf0^ojcZ>4ji`1xM3L#F@hxFk|0WTRejU8<4t{<2rqt zDE3t8U5b03BD5_-JP4IO&=>K&(xotT z`HT@E+9HG{cFFcW#v!q>^z#cx5<5T|dvSU{*S1?;okzvxo}l#7YukfIV{4qR@@27>;@t{_fMY(cDjLP(5WS%No0~3Y zfvYJimsNaW_(v7Qcu4>9eW!;QT4|F>Vg8P@b~pkRAZ;){C@Oj{TycaCF2=lLugk!D z+vfG=`?CQewnb}V032kI(QCRq0OBqEN(0yMA$drL5ig|^e!Ss~%Ckdbo;MwiFq#>8nWKc!Qw3c92r^@b+Aq76M;LfjA4u$ZT z24e06OGkU$oXLXgJXl=>S4yn70I_J4pX)eHt>vmRZL7;efxNtR(EJTJA)b&CVeb!E z*FXH#lG4;?Tfd0(rnnZ&SsB|MymRuof^ppPkh*#Do=MyE-3x z93L-_hg3M`PjSRJS;|H9B;`Wvx&fot#}o7K%igPB{F8IuDb$@$L9Av3N6g~@Isow` zU{H6sP3@t@uLZf%7fkQ#()5>{j?bUxTyxUG~L|i8eveuc{p=% zQQ*5bL?R_0yw-by5kDD@9XROO>6QzA5q0?%!4GNZXwmBmLdiC*Q)(sxdm-nA3VnVb zMq^}KME$fBHfPss4qf(eIHGQU?{Q1*7s&Tj?BugVFIt#Dl4j=D1KR)G-Ia{wqaaS; zp;Yz1p8M1RNPUIWW(szuzXRVg$BbeQUrQvLdC>`620pbvo&P+3(AoR{TlNb0+nuFk zG0t}nywoka9)*Ie0>dl&en~OK7-1TtPmne1Hz|^}F%V>L zWa*4cu9jqXQ(z_MZAB*2V#+wd=esryX;*;V8BL6vKwI%m-$pL?i)BY{ zf3kPX2{14{Q~=UT?X#p~El1GW+mOupd1z2!GRgRko%ct`YO(G>AkhFP1z{+^S`0BZ z6wj&jb<{=mCA?<1ncV(^CIj|ok-q!>Bb>&Pj;z;8P} z@6&qZIEq{amh1dD7-9!bbIRi|8g+}dzFo^R?o984B>s7}FZtaV&hGDg2d^d!3OH7f z^0^TWSkx7)YG(xMwXn~H7UJB?2EvA=308SPx#IAwDvf&CV;xtJynI(m!4c;U6Fr^P#};e>>QJ?qmu&Iwxt4;$y@x#0U*01ZCRH|2#BGj^fw z#7oTe6y3NP(2ymvqvw-?yxM&vdtn2l{N??2R{U7V z#>d{Gwh--aLO+F+`)}^1k<$aYe{K*vY^MK?7EHGB+8v)U?+PPuCQuuDY}1u^vp1*D znc}0FhtH$`8SUj+)08;8jJhw*>$E_Fv}ZUU%+^4Q$%ajND1j~-vuFD>m~_PMC5OSj zBe~z-4e$l+`Y2S&`{#5XF_G#Tq1+LwduQr<>c|HLzu4qM=3qnz)u;>E(k|OQ7I0S$ z(p6y*#G@{!y9ww?Q|`ap0nX(8Al5(qnpn|^d{=||;>0xSky=bR4Zh$`$=+ncR6V9r zRPQx==QCI}2O>682>c4&)ciV>$kmwtQi-c2H-_^o2O^DIi+qNiHSLQsy~m{Wy-r6N ztSvsvi*`eMS+QXiAH$BC$lB8&xZ>CS@2Wiy{K1yKMS9A=s7_NVt&g97M85ieCmx4j zih_fsa5#3TN*ot0Ukkz`HV6>6dZ+Tvvc{Pd9`(xiv#8op@G$PsFR&hQxFez)fM zS(RKvneQ}kcAa7MFN*rTs}6+}=Sbs9EY+yh`J`A=VMReH5G&e6WP&=}CC}?Ik_$wA zT#)93(eYc9|7*VQxbn38u@iy1FiG*S#8((*4lpS>x2t=aoYf~A4~_ER0Tiu~VHtDk zd3B}wJCZkxi^oKLC61ryw8Ip{cp=Sbb>{2Ib`zZBp+BdhoK_^`r!U;~5mmA6Fl!Gi zP6xoE0QB}yRC~QJpv1g12t9V>Qa8Vt6Buz@>C~RsvqT{pX}QatF< zfw}L#)-ZU9=o0$bbnbD8XrXLU6@gb{>*%U0P#rT}_NMkE!66cGr}T09F3K_QOATyE z$7QdD#4M49U{y$ql3e7>mPk^rUDX8T@zI82YidAzQP(O;Knjx+j9#@xW65?bOFSbvDzz5&8S-y22*Sl>1BsjVr z)pOwL6d&a?AqO&e7Gu*I2`2V|loopf50~Ldo7%}bYS5krS!mm}L-69ag2l@YEHx*~ z`n=`-^dF zTdO3iQ83m2`qg#Sp+TM^aW$YS>kej4UE7>VmV1l+&7+x&&7YM*^eWopnvQCTu?Eb% zW0^4by$r~1@m-_H z?2m3fnu4U#n%;_qat~&xi)#vYc$9gY3J2D}#$><(nL+Wv>}H=W=6n$&KRwdl1lOmq zP9_Y99$JtZ%g|g6-5n9RH+Op0m1OSi3C#{~P*?Az{S2KWpbHTGrBtBV`laj1lgI<> zvz|W3@jR55W5DfeO#6Hp(@`OwjEvQ2J)xtM=R%oY+Y#`$?ei$zE~$QJfx9RYKfC_S z@x>eQO^m7(LTcBT&grVRw>waqXx_W7!44CSABz_FeS3%Wefr>c|L17;VSQ>oVl@1P zvKq;MOf&9zT9v;~l#V%^sjn-t_xRyDw5$0$ker3-nTjyDXC@`!F0Hn=IJ4FzWZvaD z7UfQvP)^QMV{o5ldDHqv*Qt9VK*OfRWeMFL-+uT)d;ZFjrDG`CFZcgM;e_9s9xjbn zdc1k?GyS`U3=2quSi{fDG;J?ad!8@L?XUZl{Ik(k8SFSDgJJu)AipAar>(nLJJr=_ z1e6TXn!tbGP4z6VfdqYHHRLvQ+3#x^?~31yicFnfl~W*xXKB_q+Q@D~RZ`sRi;{@g z&(^=7yyrSh)y#7;u$3O^+9;Y<2F**t`q1cKPRZju=b>9Jql{k;#NW&PoplV(7Zv}P z1psECi1FU4lffsU1~_u9Nbc94xN7@fX*db}6?%KA{mo@mz3XMa*!Wg>VeTI*QT}Wt zJAS9RzrRO?zDxxz(q?&PG3hN=NP&2}ml*%z++ig6cwktWkN=h}S}?*M@g;*ZP^~yk6aMRdCpQKkPXQ z-BlB+F^9=8ta==JTUVI2)v<(29Ag?{0=)_o3cSyZWLl#_v??(oK?&(!OUsA2tq*#- ze?-PxabiaPJZ*nnRj8+)H)i+B;{Q8)OoPLTC>M0t zQQ7KC?LQHJU&8|DBVQuGKzZoOx`q-!d^GH>aGic+<1#NrN#nB_t|-?88LsAz3=bZQY~>|K8e>qR*iAT&9(A&V1;Ue z)!2&b+f-18#*l17?WHD%SPXIH;9b(Sad4Ibzn4Smc0c!@oT3nuiZQ7kr#ykq72b*V z)zf#K`W`0qvT|}YdL)a)d0WzvPv;c-j|NG_$hEEcw#LlTw(%A=j2y(+eQmNa<2%Ia zbm;m1_K)5=)G6H#$ln%@xfMM9&bGWWoHz&Ofdvblhi;~u+L}#vRhh@vqR`RlCpb!L z&B&!WqoH$PuuL5l=XXEtR@)7cnUxrS#=GCiJC^(w#+t`vHcsX^LmJ-!QfC71p+4rp z7K4DC=zflSDv@~`HD2kTwMgwra}`u!fnLcGHS+Tnh^_yc$6kR|a9o6?VrWoDbzWeA zX$Anzae_&*mpyOfqVAW=qe>N;Q|tlQV1m%BZ)RGuc~5ZO`%gtX0B(r#E%-epL`=Pf)AOFg3c zQbq6O9}PM?N1Bbhi~4+ChXuJyA)GNXt13piv2HGmW|Of6%}>wSm|lE-nR4;00Gs25 zYW}JxKcUOBft7Amo@X}1k7wi?L!x0{GXJxTZQW9~t8lFVthN&_^>gPjyZN|OS!S>C zsw7^kA_P89zvo-m|AagT%(FCiWNsLhAf1se((*}X?Fj#ALo!Ro8+e=;#|#PvhNMe*7OGQScs_h7Z5ixF2lYQ(dFH*@Y~Rh zvY1J!bYAdh#xDIDT~q(@s6(mb27+^HcWcjc|4cifYRC>9@pL*XCUj2$J#<6_)=e~5 z6K2jBT@gGP(y?THZIj1?rQ1B^v)8im zNFVLfCAO5OL-p;(=32Hwqtz5`FHNY z2RfK{txYQ1N|kjWego$U;Sn@zOtGtJObDnoS0P%f2but3bCl$~u>nnb^6W~RM-fro zda&7A-#20R6X;@s!Hu`0qocF)X7cCZ({+@ZV-9&cZrdk(ZutcM06qnfJov@A9hTro z1gIpy2)kP>_HCO_R!F{@{a7GGuVW+Dy^b{JIq=|N&K=7ZHDCP+RjkZVUU9Wwfw~`^KAy+W3g_Et0(A^qSHHX>i_OGAWh`p<{= zgj`#v(C-?);T?)(gvSu7nu>!q)iOdVfQmUoUJFrO(7apL5>YwvYz zbUb=b-bo;c>KuSJ(g?d3%Z{m`F=DcrsK$m31>PX#&o|b@E8rS(^Qq6ERqK-xp98@* z6IZ&NR}_dA>6fgR_$Udrku@#e%aP5g*7?&1l?f2r)NOp3@xsl#F_tZl;5UGfjDK@` zR$!*Lv&MIA*Ro+%JRa^X_BY(j3ML8rYRUbkLRY_CzV8(p z&W=A8J129z<+8_Jx#wE&;n~GO7E)(R0IPDchUF;QbGfOl^kPAPDX)67_i_^h=+Mu^ zdZe>j5x5@Qz6VX0Na01NuekCzyq?2Cmox!I9iou_G=55hxBT0M4ma+$hI!6LZZt1P z87E+QJGUS9i1QEH_)<`i=<*H_>V|D!cbf)oZ%kiT!dWs0Zqdci3${ygA1+0bjh25Trdrn>G0Vs>*H&{u20 z*GfFIVCGG?Mq}@nD{P1Z_;6Hmb$_4fHe)xvGL_}HwCLU z{E--)1AwJ^5i^jZFcLVteVK~MHX_{?RRU2&lI|7OH)I#FGa{FMg47WY&R94AU&L!r0S6&r<97K5s(eJBxYsw zjV9Dsa~jYS2|Xhl*B2%LPbZ%4cUT4W*s!4A)u29$^H|M{S*kht(Cg1!P)7#SvlvxI z=$N?2Co`1}EvuTug%=H<7X2WD5B})TD-~75!_AnVE+cYo^rqE)^H&%d2zry#ZLIdA zIZhz3NVoUs8%C;Q{pLP1Qbatq)R+uz9-)U6(=h6E4T`REIlCa=`ym)BsT@FR#2`{N zu?G|ybceW9YF5K+dkga}U^Rt$MPGaF)9@2jA%jwhZ}mKiw) z6#}A{C^;0Hi}-TtblZDKi!)qvFZK+pB@KCW7=4tW9|fLt>I-CoZvf@Kx*z{hN(R{`slO9wYs8V!kRdgk z3gJ7_%sBChZhoIoLXU)Ey8k?@#1+gbU^?nC>i|thjL1-G5jILTyV~G4e((AJZqtM7z*K4Y@5ly|&iA^xeN zT;lFKo2olWH=8eSkr#jtmyOH z9;xo?M4FZJ@%4-0gY#qEEvJFXV|jS8IF3S2&jspxRSkxsAbz?1o^BR2^Hq66XE}dL zLf)GkVllr_ulf*|`p1*12L5hmW3qiN3($YCF7OTe_{0C7)iUHkEP&!60o5ZwvKmn9 zUC+{TTot+ZOTozKqmqS+Tg;*_$~8zP`xl%VYAb1g_9SR80Se{6&Q0um&O{y%lVl;| zS@h+2(xs!8uXyxnB(x}C;vLKLPA@=+oIs>aQKu{n47|7}SOV9IXR(*LkoeM)oC?O4 z@zUDg*)*GLO7kTlQ~2NxmBFUGCqcdjj(1q`9VOErHY|l6O=9mqdgnQgO$7`^3oC6c zB(~*VY4c!=_CikVLBqOqDKqa?i~)V~q2lguEYQN0mVK4#FIW!v9x%cQh8BTfSxHlxXFBTk);fa><}3i~GkAQuuBq|0ftIy}h=JC= zFt){R(|d6sQP0YtH3|L8w(jX32nihHXV ztO5OanH8S+4q*oR#$K!J@_NdnTi!DA7!)lEJ*sw=nhNW84K5avG(ys|nBEO+TS=tS zn=?7Smc5%#5~_mEyKZ^Ik|;X!<6#+3y&9|wN$rkX&ZzjE@{u>Dl8s!lK-U(AYvY5bREPI=(&VpdBGAPq8ZTzo_95mUK_{!N>KtyFQV>(&aafghI4$Ig z#B+UmRW8NA1bHT5S>?z(+Vn|2pe5dxru<4;3upEzdJLrL@l-4TBjUZh?f;lhHsl zb;@D$RGAm!SSf_-6tA4^`)&tquBl9Ahg2KvBc}36W>bANt189d zX4P*kQ?bJ+ydAr6v*w?lJnHZU?_n#wLKj1~1EB82(U*)9!8ie@pl-%UFB{;I^Mt1s#b433Dc zWz9y`dVsE z)^+42uLi}O3Efg)@PPfd6-2#t|6@NW-@JcT?~#;!n)p9bcALSKkL+=vb<}p!q2rh9 zG+>@o8tX%j*E;*-yLEt?pQlnG4p4!fo+IMu357NlqiT^9*1&F8v%b{p2rie8zRP9fVIkjnuyEEmAR|12;x- z;z8;m?Ho>D#J(S$P(Ye}ev7<60w8&xRtlpxuH6?pU4xGuWPgQWUZEVo` zPoKTkYeWQnHNskp7|&c_XIxIlaJ6FX4T=jB!NbqsQO9yL#?y95)w_2m-1Fj7UWk21<`R6YE zxib3m`O!Acj4~olBol(Qirp!14}gpWh-^3R$&`LtqxE{5!|^$j(NkJUC1}G{iQcS} z4YJ-PbVq;k8v_Ir2Q8k$KTtGa1|=!g)hiw>XxjO0FjS!9Ht3ZJ8Qi4yQy6j-5PsKa zk};rCsg(yP2>(BVZ;cb^azot9m`7&btU4u@n;-khv<}gxwmq;umOK*i&WlmI-Jaf& zu`uLs3m|^dz_f%=8m4L{Cxh<+zdjDV=-}h8!LH9TJRkfbTnw@{dQWkTD{aHeG_K20 zIpOv8Cu8q!eAQ@DH90X`%j8bIRL6N#@!QF5;iEkn4|F7RTy36%0`$ecrwmYdk=lz% z+P@c?d+n9-qY#cRr5O}04A0Po`&LQVEFAhk^UjuF9k`_>Wl%HBSPuPf) zv=v0|xrZ`)a$-|H;lT`HOt6B5820nqB>n65d_H{C3th z-GrCmCTOpLYfiHD>q5{czHec5`!8MYn^#{ycf9B`z)yPapeDd}9$qd;;*T$NwTdNBASNM==z;ch6he;TrH*Gq3s;2bc0zMU zCV8FbSl{aISmaN=OdO)lwRB$b7pL2Zy53-#M=G6Jw5q*wQQM`6w=i2kQbX` znjr(ISS6q7lWh)ao>zs1jgzGH&EKfM=~=t~^V`7nT=;bu2XV7(>r=~L48)V$N{(=y z@b?qi4&?WDx04DK<8RK-9qZ5IJR5vsDT#>n`0v>%9X0i~0oN0-N@NEE5XNV@H0Hi?x;Cc}d)f$62aY0{RBJTK^5EQ#YyHm8aR|Ktqz1nX*) z{bVm2v&pUr`_wI~9LoZI5?2FfP$g=)abqC}&Z)&9G*DG7*Ii(iCT(eHeLIi^o--^z zzv)pY-DniCYS{XwEWu|Jq<`$lPAB7NdSf-G9=tPv;dM2V&XB~gC6VoTyQ^Hfg}-L? zabC0PB*IQj0FVW6>(lXZcbZYB$d15Ad&1G=v&d%nK0IQga8^Ci!ckINmw(iF8g9h= zT2PPpQhYV-=j5C7NmAWg9VM?W=KHi}HhoJix0^5Dq+cL^KqaH2BZ=$C(9;PC$-S;^ zS$+^{9 z2EqUkxtXL(8omT|>}r%a$SeZoe^X`sC9m5-1>}BfJT7u)Zi+jCH)7~Dupf;r7Z=ho zD|#g8(=z_BEmUJBxEi6JOJr@4)00;pcOtY+( z-SL`{q3Gf}y!Bd9FBMK8uZ{=bzkEgeWh|a+$lWkoa6VVql$3WxC7XND!`9K+#iqL*neQ_#}OI}H;qd%f={nt&>l^l*N*a26} zOeS~yV4Vyjr!ps%g&pg2&RPy@t+{#LMmj_eGY19Id9Ih?3>V1Ex5`*QaXyp&?mzWD z%VYOJ=CDM&0n1M@7=7z z;+ViU^f*F+PFV>N%GpqE8ReC?NPB=%-Hf;OJJ5WoAC>(N!95=4;?lWmfz=W z#M(AyJQ>nROSie)gesLq76vaN{ZMQx)esRri#(ON5O3L2Qrl-I<+n&a1^q)o9_-6hn06yK7alkW_n#?D4)9f^hzxIhH1faHplQh zJDoMMpQ_*57RPHgB%vlUdSD(+M;;9jll$GfJu-W;ZPV~Dj^r4d+DpZ3@o<~tJu3ap z5oN0MA(u@67RIY=>m_g(2QLu3DMCX~(dzS@%|E}v(BK2&!9^i5a6n@AC_n8TmvaOy zoC-^a_0?&=$#Vb1KuStkjPPKn;?f4_sld8|Z<=Zrh`2(kD?QXvFL=)JsP$3cx?pY^ z24S@?`UQVb*HD#WJN9qQ7B`R^)2a$49_%YMqa(nT-gIP#oQG3uEwsNJj&aDq^Gnc@ z?@Pmfn(t_YOTTePQ^App>Cfgi#Ithk8S?yY+M?rSBJcTwvP*UEgK%w~nb#}K5?dQ5 zs{)#&`MnR+g@tMN6eb=nUWVlb9CpMX*lSb!1-_ilQtbv>Y$J+Z5OtjGd=8#DP zlzGW=m+Xw67YE8rs1{a$&2^i#-VJQs2k61=4V`&Q03~wR90AYS-=Sp2=EbGv3VQ3{ z!bwp{Nyv(I%h>l8EHX{f*0@M9_j#!W%rJx7q>o<=`Ni&id&O!ocW8r^?oK&*ROZ{L z?-=4}Ja3H1?o<3|;vy~L@uIAPw`HZYzRFRN%`fb;YJ2b?HsBH5O4 z2|u9#Qp^caG3L_>$gJBF0FB9-EQJ|Az1DzoLotIZ-dp)vE$?<5*otr!zqvrfzB? zap;Pg{W+IH9PdW85lcqPH1%e?N={?LVyBbzkSddOmK|{a2i?z<)jGnIe7t}10jX&3 zLh}Q2H!Y_>15U*=Q>W{$QG?6XcoWQ0Z+96h3)t&>Y!||9y{pjwxTf;VeMkzQM(?bR zRgWWz@5mm-L`8aq`H_edx#1mtr~VUwM3>fX2IzA4Hc{_P^IVbpp>{duQ3oX|<9nF) z!{A4{DC&N$YD)`zN^S!y=b0%7(p9lt{q+{_e4hk~oX2a#1ZRDFK^G@_hdqXDOF|#f zHD5)dt>UU(^eS#R`oL$``Ijs>v$szdTw|1mY|BZ2!iDN2ugoT$&$$!KFwq)}R2akg zp6w$dfcdJBTKTRdzu2P{QxsFryo;EE)ryvTT$XA4_J2;fI zTo1^2mW;%KbOU%D(GuQ3<&ZWXx+`}|?ZE$gXd#>GR!t7Jw-qW74X{NYx1Kn{^pGY! z`&mLFNPaQE%J9SUx=u%S8_widc_)zyWe=^5oHf~;x0F?1Oyy(JeibI=Oh}Eq#57h; zl)0D37v@q*lCkd-P^zpc2w<&xJ|xjeKXG7ByH3Rjdcc!(W8O*bAua$Qe_-4ILgYEB zuiRFR&&9eGPV??Ozx%nP1#tsQ?;VO+?P0tNFT`LcVr;bm{TZVM#<6zLd$sOWZj?q3 za7abm-UBb>RgO+#G5Z^X7!V-7@zwcv#RB1j(MhB4M6}}VF*3NjWb8|h;f@cRgH~TM zW(AzG!6e5dGkA>Gu3?Cdi`be~ATBmK9c3e-EGUDEuWjd3(JZSGj5aL3!2ne@Xih~Y z01CT&*ip*6r3(Gdn=D6PTFOdg0uS|b&xU1v+bAoG`-S?g2B_}mOMs@*4acYqD0ZXl zq{oXY!O?pUr!AAd`!i95>xon`1wA9vfej znwNA(sK7}GpXKiEwMfuTYE~%_7nDFwx^d753NhZmIX{r+GO+>ii2;iL111E_W}P-a zDIv+(2=z5Fan8M|m8?IuI$y_Pq38@8EW4ss8Rk9rF=i2<{&vQToqzSk*TW0?U2qQ( z&q5gPaFb1$6rzZ{8X-l4iUb?AEOrf>iH-#+q1DrD-aYqGZbG^1g0C&Yb1A*D&rLu(UN z#)*zNlK$<&)^}ApkZzV8=9Dcn)*QO=)Zpp+YRb5pLqbE4dTDwjn&s%)n&6`WfuDX_^9*1yVKCXyRPN;XY$wYJ&4A)`zxniDM7+0r7I%%cfc^ z1rxCczY|EekZx+lDraSu@6Y|7r)#aXOYy2E`xhQL9NydZWdO7U0FOXZYqNyZolwhs z|INs1e{?fM5|{X1jy=oIol2U+Sm6q&jO@Lzo*-LLeB~~}wCT&kgi?NTT9_|xYMi(h zsKFB~D4d>cZYrg9a!oU|n^(5hsM60HdNTHG(pnvNy|0;GjSfWLtA3ye_!oeEx*jti z^%^E)V|u+U*nE!Dc2?e)s3xZl{vO7AS3~MwT8xU-M-@Vo4>rloEGLc6!oC><)}QL# z3WUa-&W`6D-yL@eC zh~kkar`j39|HA^<+4kH2Y4kAWRuw)fY@&w;ZS#5H499;~cH`l5nftwE=5Tboa<+SK z&tGC-mF6K^s6?a|RjP@Bn}!dhSluCKH+Ee(^AU3X19JTqYR2)Yp<2N;^P{>9c32h8 zp6e}@%GtEL5TR95$%EYS*LnnP*_87X>9wpmnJkrv-ZOQ!`Bt)*sx0R|NqZbp!re}+ zi<0v+rWpf*?*-H;U!2R|57fz{R3v8EkNM<`hpn3Qz<6)$=pvsG4h)y_oO2{KNj!sW zj6$O9ll4VFO90o_kM2Dsj^49ce^n_l4b+c6EP$t55X?kEy86Y=ME9i~eJL_qnkApY zck*0Nwb$K`dEv;*Yo;;J`xqwb`n5@ z_ln>ZcGC7_Cvr2fsUI-}4NQFmG`EVxx z%FfE*NSb-|x`NN~J_I?zk?)nXk>CcUtKmy`UbuNp;+q4bQ1YA(%A@sf2S|km{hECr zShCFm&Nmnhqu_V%{;Z!f3`IxMnn{xuTmzVr@W`(iW`JyM*^^#}=#NChPbRRpw69F~ z*-RlQ{U+BTOQECadOIYUm!&pEyYo#vlf{=xL#mafF11B5OoGk`Vhr%HQ5q#7 zrPhxXa4(cLi0;Rm$V6yEGUAA@U7|HFy=(N`hu)d?4g7MMP;hp(x%)YJSg3*lon)ot zL{*Xg$4-NB#Ml5pxrr2e=-U{F^+hkk<{9^M6ur2jLx+_4etfaiu8}rYj>Tn=i+jgE zEv`K|zrLV7;%PU-l+&*aWXiYV&6I#yrf+zvARojd-i&qLj$M6U4mPs`Zcz4_wRyrPqrYN-Gk3P|W-#zY_Y~419h7ZR(Hy(>>t$*z|L&2A{9<)Lu+=#7XQ+VTm z*nB0o*f-E5+fB8n_T)5VJimT`op;#p|87ScNwph9{u3~uQ_L|`$-G=|k|n+jP7k*@#HL)sN!x-_(FmMkS`OwL!S z84$V4sc^G*uWEL3tFLx)a?cPthrXV(;#TvLr(a%}$6PJLnUf2C%skV+W#E)H2u$*A z0?bZV6txs>mR*{*Ey4rmr9`4c{m`^#B%xZSVfap^r)_x!teZoCk2w6UX1(NG)U#Ht zDwxyLWn(Rm2#6lt|UdOYuD#lM(=qvu8%aM(_7J`6d8yKPJ3!J2if8# z?XKs;YKF~E(_Hu}mdy(_`$P18pBY zQwg*?vKkawDGNJBU_mZCY;Vz|M?0$Y7i@ZbW(E&-Hwct&Yxc8Z9wceU7bGy6So@m; zo9C&Oi(cf1l{7EDfZSfembzSOBDPB=^UsPrh00u27@{(vKkKLJN55B z#GEtD#j%pl97wp3ho3Y)d|3~CZ_+vk7ED;QJMshc3y~ZD3N#l;oeEs7-M4$%+7{7i z{K{Sk5xF8-FKJadvn4AA)HDdfjq+e`PeqkvshCMYL#>Qn%sc}8Re-=rr?Y1}Pu_Cy z3u|6OJk(!Ndq3&Mzv2Z=s?aW^6Qk#4LB3HbZF0q6CwWdj(_Oy|pzoDiNhy$?0HBDb zYd0zB(6+t2mYmj;N+SNqWU1^>1}f!P_A(->s4vLhIDi?WRjcr0yW|r#k3xLMvMhy) z7eT&$V1jz?YbZ4oTFMzQ$QSHB-T1OlUcF~qPYx5B{f~nt1-@< zDu>KrblcAAbw6o!!q=_N^p^rncIuzoJC>`X zyq@HZa0bYcmbL=3*KOr4qH|RN7z0%jaF%nj$Fm6jdzR5fDVRjwHTx=8FHh=*nE0EWc~vuAJ?-Fz&G^ z8f|tiRm{W55y>;AhZs>SA~4!B<~VA_j1y{~KO0t`&XPV#YDj|bXur#SV@-45VB+AU&Dw8NEf|iqltiIR$FGDB82NjJ4BwBB;@ffLy4=E@EA73!_`hlj z_lP3MVDKjQ6oG~lK_w5{??7^)ZT!&n9>{Ag*}gd{S7{>@>ZA02jJT%UOH!EU1RgW3 zr7X@S$r%jjACF6MLc7+;L9D!1>9y@pP4unwd9qi3q4 zVZRmmp$K?jF*{SLK<>fLvBsF%6YFaiYH3`6{J|(a`RMb*`ii4S&A9l|rA4Ryq!Zsi zk9ROhFqF0(?vp#}NnP1q$4JX5XTJfSd*9r94SAD(+d_+By<3yp2^_Z3F(o^~crYE) z2um7~*a6`sj+?@wlEQ>a)#k^BOy&KV-V~)&r8}jScR`ZQP~Z;fA+msk{HD-^7 z-a4>p(k>--WaN%g#|cyl$cdF{Nz3}H9UJn? z+tB3o2=@DB(M23;+9xr;bGF(QqoryhxjeW$JT`4As<$}rK&*m%c0D`GgxR_Ktr?)G z2U;I76~-drt}JR;@DGXVyK;QYv>45grz5u`Mv8GJ)H3!bk@qZJ3V@t+MN5BZ$FZ7$ zWW>ktyQ+C+q2h^q@h$UQ*ZvfhvM$i0DTKc7;6f&Ntoh%%irEKsyrdD}cjAM~2~J(G z)X04*QT8tttCDFTqTllrW59<37D zMaDKv+Uew&VXHoyDMbTJQQO=v+RLYb4&W!w)T_urd-e+1iJ_c95VC*R^K6EGz){nThy>KmEW5 ztipu*O|~yLL&i8M7ssERmMv>|%Eb`9WFRL|IOovyq#*+|Zu;jxosYI`#jzw&VSf0v z$%p^Oz(RC!^Gf!j72p7=^Gu8i%tVM&I-XJ8NwFm9FTHrt2<5{5~6pPj>x zbTazG7@>I-_TD8q@2ej`_Im>`9208F_m?Np>=)|V=9gRw0cOZIXDHV*o6ySR7v(UwZxOLuYQa`1!o9SbL3g0MD{F?H{yZAYz=C|zSDih0Tn;|uy?t<6S2S>p5u;Za`kF(u(77WV?1Czni5R6?U{^v7~D}D;~&ZwG~JLQ z=imtWEhdxc48w5}bh2Z*qL=*4qZ3Ws_HOhgkZ!T1gerX^!b9vt!tkJMq>hbBEgeO1 zWvZJ`=h*;>Vkgy$$5!%}NzOH9>4eMvBiWDj%IrUwq7pQSNH%(Z{3=(5o|HWB-*jJ` zy^D5oUXnd!q;1elg!O1{y5!ObxfkG*=TY%^F{hFQnK(Se;i@PiRh;&iX*8p*tP#Od zcgZX(Ss1q^_rzgK3GWjFgQ62{E7O~UWGeXhP>hA(qN~66*#{%0xLybj1gnMTJ8FB0 zfW+z00NSLQHWuWmVBmJ{<{lg6D6;Dx0ZlbidN_)BB*C0u%9gdJ%Vk03krn=c*hKZ0 z|J7pkvSJrM`i9T!HtAcpbIf43bg&=TP#zN++#zXsNh9>}-$~<^83iPJSwk2d>YRio z|E&I*AlbaVbxxL1;~kWtK{XC`vE+*NUmzx@ZWtW5&4T1#Q47`T$EUqf#2PDP_xWs~ zHu@)}==0mV0lajd=Ic3HU5Fgf@E{=$<$)7Zs*Ii~%gO$zuqS>nF{N4)`2psDJ`1oG zHp&WI#sKcSKJtIZGYn9kfOAEX2P4PIkG^@&AdvPcu zs*;PPuuH48cg)U&@iP`K7z{IpawBt>zITk--ii^WHruO1o@2{f@9*;ummvqResW+4 zIBRC+_}lCYQ(2!-rp+v$=^ObJ*epwF)EpGK{LF1w?jzg_OWvchGRBzWCTr}=6^%7Q zn)*-Az&1r;%e~vaYC_U44r1}Ca!lw>7Y(+c?OG5upXO-`!Ct<#Ok{`XFR9lmSojCo zW9&gM0Rp@LptXnVKIso8QtPl=G(;>)O$)?GZ`kH1l#kKY&j5+ycPA;J8vwAm=DQD_ z;*6-%F_2QLcg4sp`*>Mvuos)rc{751OkQz@30xblQK6(6pj2L}>HIcl?Iv%d{PdXRKxd`=v0*Em zD*Ei(F3_+oG;J^o3+ey|JB_#ELFINFD^ZKr=iZAD#kVj2O~I5@wYGO6Jz&$VD5Q%U z8BYU9bLW;eZU?su7V^MEOGlKe$>-OlV_XPMw`dHNwzM7-MWI6fFac&BV%gwXGsl7l zq8Uj+Rh^!S}15#p8a&Mcp=`@d(ubel*;VWFb#F1WyooFC%rmNcx9ExRv&S7_F#}3pwzFZ^5 z!*AtZ?=Qs5uT_06)5i(jWhJy8-)ZYHqe}Q4$uRe$I(aIPtj4iKW^JJYb0)ti#55D( z4cWEN6Vg&7zP;}e!}Mo1s!KOjuR;TBGBjtJ>^kDB9n0E6{F$O!nY*mau((d;HgBIwcem$t4a|ri<0?Vh*CS!gG_WX z?HDxBSfWm6{XWX}td1${g7^Jey&tYlx7M(0xzl-bo7WlhSO_h<@$;Fj`5gmhdFm!d z^#u8TcftxP{GRR78ya`aR!uq<+QdZ5P@$AMp+TgV4M!xK7=GjHb0*cBs*qqn9avVL zATd5p5K)(@!vDS6V*ZFdqF#h3K1KMQ`WQZg%i$GkX)L-4j@>qDEYdWsUEUHFV2isP z%j4o;#JtSqM{=X8L_3>-;j*N(v)rsK3}d=zk@b8j#h1|QG$mo>%CpY~yvaXlx{5yt zx$j}^!|z7AOxBYkAn5vW-|?)d%8jSE!^T=U?7EDVvN8J#$DR9n5p5RjVid^~m=Dy5 zJ*BLyl@nerIt6!elSo_}BRb;CNxF=YRLkfne-jBi&dr0&!& zcAciO#~)|WOy?WZiY4b~@9^)Q*EjLztYwxjtfAvI_uu${?RIUH@R)LpSJzIICzn!M zI*@fX#OdMbf0OZK@28?a*LaX|F3LX;B?Pr5|FIT<^AQf2IoflQJV4mLBJ$XUx+I80 zHduAuotl^0k`SJ*GYoS-vx61QZ7A&y+$s0VJRBwT9Gc3~BoCkX_SlYf_k|g4sBF5w zN2gL<*Ub*$W+3zbzb8`~{TX)zW>hG(Cqwn~@f}02+@ve#Rq?817ZvwgI&bvUwo@E} zM8w~rD4aO{{ns&0wd;z3pCp!A3_oQqb-7U0qH|>0YohsfmWjZ>-(gddGXH#=$Slo8 zbx7_-U7wcr-COeNBrSY2c0ig|N}|EC#O(OGu*WQ?T0i!yT!SyHuJ$%Fgb$v|sQpSy z5OwJ~l*S$7Zn3X_*F_I5WuXDu`qWO0RQ&i_LLk%;8~;N?ERs6B-2rw`FD>F{H{)?5 z=c#d9f*SGw;Zdv+ZfsSh#FaXeT#c==tj2;iMe%+)Ng=&f|N2q!slW9;sOnHl;lp{!q?+N!}~*OdsQ-EIINJ?SsyIy2f<9 zQainp(8Xo`@%+wXs%SQp*7jBIc!yVlL+a3&t;6_69fDJld@KM=J%UDxVfL6k3jnP3 zo$2I=BM2p^(V8dB5$dg5`TT3$*u@Ys_MDvuvta9Ls)uqCGw3yG++sWK>4d~@hZ{A1 zXJn~t0#zlH{-6z@i!^+Bw&|>4= z6G^cyIX-v|%j;hm_!)x%WIBaE`G0duVBF>Xiaq8jBgoR{a7r^?!o_UTKIwY<$g<=3 zPgzBDL(pbu$KGOvHr<|+u(sgg{i94Y(=u=L`=`Yz(YDuW;7Ep52qJ1=>f=Vjh78gd8H3+U+t(VsbOE;bju)1fdj2ETw1O1c3B^67dehEDZSU zEEDx;`aFZ+ae5R?C^4x+A`UcP()@xMo-X~6s;=g8Sd3?<@xok1QlIvy*uC9e1NZ*( z8JRI=#dBukgygB)_6J+FQZ;g~9F*gIMiWgD0b<$D%$shRF(ZrbOpR?``nJ}pIZ#!F zPg#%qNo^Y7zPe+XJRbXo6FokkduUT--zDjV-^FZkcSJn9`BXg+Q9(H;h0eZ z!t0uJYMz=(gcVM_U}{Krm=b$xHba_T=+%cMkcw~s3**Y4%{8{yS%ZGp>jzlngRigz zTLnh36XsTlZrw52m31?7Nz<-2JNkT>3vp+_#_HMDOw}1Fzn(uqH3A@!CJyE&469|5 za|hW@^H)wF)2{TG;6A2o`>A1d$TRE9RrP%CA~HjOe9Q+}SSq{b#PM{$U%!~GG_9?b zPo3sB^aGMweWA`1}`EY|z8~)!nsP z4MaY5gkrs^_>A7X`;JoOS(Dev z4LF=Kg|W&Em~U5Va6kRV@wacu`tRe|zEAA%wA&UnIP*{;R=l*)$mR~a&R0zs+BJLx zYSn~Nxg%5|JlTdS^aA{}2aUXuyA^f`a!%#(LA2Bek-H~5QsB>}>m3=r!{PCRI`XRR z!5>w$7s2f2I2)BnT(p5qQqL3_@d6|{@>kx3>Ps%$|Nj+^VJF*vtg_LAM0`q;aSKPA zt<-2in*9b{Qu$Qe0EH>H^Lsp7-ejyw1pmy4jOpnn+kr5m&oJccNF|}vdJBn3?ESEQ1Ay>t<0TfHFb#eg!*S&>YOkxb+aaoE3Xr;ZFvBtqVr-C zSwr?TXGh!9t4aj2iD$-{+>w84+WC5L*CmXEF>$0N|;njo)OsTpC|+f;+k@Mk9^@SIYoBs4Ci2ekFH)@JUvj z4H!M{=bI&uu+vV70-{Z_pM&xKU-{)>B#)z4^SXeo#~MEi^w$vmxV20xQh(a%Zh;A! zU1EYO1&B}o``fQTR;{n6d@$=~%S&)K%)*oyZsIer1TJY$Rs6vn2*@^TAt6t}AM0BJ z;s4dQKlb*B0&Ql?kEhJz>*hRvzYNkOiTBrmJ~p4<_c+1>J%%20dR|hA`{zgz?OGNV^`giJV<)#t2W_e43mLI_K^?;`Z)1 zNFXGd+6HA??g{sf$u=7ec9;ObL48>DDdx)`+WGF%Mzo>N=ad>HkvHd5#}PAP@cGK0 z0E{weFLN>DyZbZbG9tBqqG;YJ&;tcbv8yyVf}0^A|1_1CbMyDs^tcGi3$6VvHNsSj z{FT6H`=jh7|NRj8+f_!@ z3e&F21()u$-@yJq2+7>*x?-PHY&c8Xt`2dD^|naRqtNtMjx3}W-9PqM10IC zJ?j&ALnqT%w9HZ&R|0`gM3Hk5%YMlw+;Vm%K&Y@^u_c+HtXuV<0;`-F0a1%j9+OeX z&g0k<06#jK59()qNE!PMIpoaNL7HA0LqR`kEsq zlqFQ~Xi+XVCQ@#v$mJ8;OQ?l=lI)~P?Y=}u5f<{zF7A;`{!eu@T%8-r;W?$}AhJ z3brCf1KNH)TJPmCegoYLFM@-8pwqOnEzyVlC{EIUlO=p3cuvBOeMD(!ANL_)b~xKYm^WzS4Z@`+s9nL)EbU#Lz%fO+Lc6 zz(J!v4xO|!mldnQMet^qT1eW7pb$*dAh8x=J8s|03$+tCv$p*mFXBMxikoEZ2HwB45^T_*;IeDC#o-Z_HUn9~JQs(+4A z^WrJH^+^1h$|%|jffs#-xG|+Ek6r3Ugr8Wk5d+=Bo-gk^IoKvxpvup3 z(dK4uXhZbLT7iJcTS-Mkb`c<+g}B-#h7R_1aa*3uc?w3p2ywS=io569F(cDFiK7<0 zsY(A@KU)@j`rMJwz8gp;)5{k3eKk|^V<{X+39&M+$+Iq*l&| zDJK|E4gD2RF()NLd<~qfaoeg)27TnJ?ku@@kT|7NnpE%tq!bxT%cAFlfsu$ygW3bR zbw8*(U(d@sHL~dxe7;IINhy}W2$b=tK|hy6qRs7;T43awRix1Zb8mmhA%C}q7+rYR z^!u2OW{-POD$oERu+^u%-+^5+7woo#i@Dv<9UQl3z{j_Z;&1-e~f}ZZ^L1~e6-DNlSRn+gXP1V{zeZ&cD(`Bj0YZa}v79X6n ze-72(P23@O;kNeBjvGueqIKC*r}wC1EyNl2xDcpF8c_{*nk?hpx(3KhV+$=#V}`{ znM)}`CgtebD-1i#NR`3(FN zZ27{PsbA#@irfz+07ia#D4}m*o3whMAX5X_15Ffla$jLzK<{wcbXn=!7bCy%;uafk z5hBkiHcuCjfhE?Tbj~2k2&}$RDl%rx;Dvd&qJnjJ+w3MW&-@~5Mu%Vdt^<3xf;f=3 zbLbZ+%*p}#>^;@Y55hUXXJeLh=~;oEPLsTN`LDMK6Yn#2K3<)Wt{#~*EfB51S>I5K z-nTvP&(UoGism|o8dz3s&M)#=D1pP;hXy~TgpF6#oJUQ>Vk*N|4ihH?t0|0Qk^lns zd=r2VH--*)SRuACnn01+(3ZR=;0;JBQAKl$y2(BWt*dW|dL}-TrIbF8I;dVqc?zdh zz^zLs`SAYu8IF)-x@u#ex;COsjuHvPtv+o;sGN2ZZle8Q)hZZvMN#)vi;mmuXVGPO)7Io( z1eCS`IKZdALl%R&)!qF!>{j%NY`K}{*|_5@7>wZ8nsg>ea7ciB;X~Cy z=-`qU*8IbOfWP(<_L#8R#f3+JQLMN+jq+)-G|l}DfJ)aK)F+5W=DR?@8HqZHy9jT1 z_|-`K`kYoQD9_9XS#Ux`Sw9*87^=u#W|jk{Hm^~CVJO&FmlRued!MHYjlJN}BnOQJ zerR+o%kHqi05gr)nVNTL{EYU_m}~WCZ^}v9uR`wt;2QPvSRSZ1aGOADc>gxV+zSQM z5(bf^TN|pI>IO3W+x92@A?v-~=K4>7`xFM9(}cX^>*fN=b#Abynq>(7PlEnK9e=hD&F>57-%9- zpbDq;R+tPB65aOm+YLhb_nFFxA6YxPc@6%?GD;&Jh~JQx&; z=`z4x9K~a{{xK>oqn&4q$^q2*UF68mV-E9YU%$UkE3+zlS7v3%c-Z)1YFMG8 zSsJ84=!Yx0+fEiM+L}?;Nb<2cMOIZ(uTEOyKF@wpz{H++>KRgyuv(9U-JXym7LWq` zrcpn4<`NpHU-Rk1F6~|O^eAd=Q_1~sOT^;4fIu!|V7LPa_B;c30td<#SJxU9BAFcP z_3lh-hmm4alZaZcCoLd90}`giMW9cSHV7yQv|eX`{)q9~!cSVSHgMEITi{O^$SxUM zgq649zjy?k%Wz6H<=;7{zO~5phYV;?++XK_jWrhZYb@t+p4kaGWzQoG9d3ed$sZy2 zF?=zVsd>k^;?q$41C_EkZBZcqsPdt0Y4HvY1U2d^7H<4gHc# z^dB?GRQr$5Te7(}at1gZ;6^md2|>*miZ>jf)+Sf~csWxcOu=%R9+Ze;TIyAoBBV_Hh(?_t&|M`SLmnYyOV%q{1N4w3- zBu&?|h0)rl4uZO|Ky<%%O}$E_D-tO8^}mxx93C)WK}oJKrXj};!UQH*>I=srr2s@AUE3Hq@%HSbgvRQM+LMn-um=?@=KqT|{ zx>2z@>429uQ1OXVejUzNvZeCYB}J{|b@j%fn3$FG-R17S_||F@cik{v)HGAAhiXgY zpcsDD6UdJQ*XMP3sblm-<;h$8cgOgFE9J^0kcVEw7D^G`Ukl|P6-bCPVkpo&`w4hi zVD!nG3@*Ng3SGY}B}Vo*4Y0o`4T!52cMASTqcvd3#UI|cCN8SmS+eAHRR<%ZSqD{T zVED@r6*7y-imQlQo|1vg3pGKPiV@KnT`-uX!Srj})+Xf585Mi?Jh#H^jpx2WLJ;*w z!Z5r9^_MDFX%BeggUmi#71*v4RW1&w6I88F5d+t)xnSSTS4fRpLg!_Mu;s5 zYk?>nX>&yzNXnlHV<_&x_mWxOy~RNrhS%PXpY7H=k$Ba(1X22nvy7fGHl@hCs9xN#=b*UJo8S3x+$+HicpU~i6&Gz)JPZ5 ze-gvq5*3~XnyMccfb8IWPyq&_(mqTRA>>5;nDZSd>A9GwB0$DEylcW?YjTbYMdi5V2{GazF%0{)%nrsZr)lMN!|l;`FZv5 zWRi|w1w26i^R*S@hAAFM#uw#t##JU`A^uAUWx62w3m(Fkb1Ivqz^+_tpz`Yo{e@sG zrJOMID?N6=j`D+MI*N4vc4ZwKW(p9MEPZCx$|^n3Rsu*u-D~=e1SGN6;AB&N+o4II zoRqj+8n0W~+2+>~Xb?kfjhx<_N@EF^b9cSeel1#p4!C>}j{Cq-UqA>%WN%l>DR9I& z5q|3H%CAfY8@?2$ZO8_DpX^0el9UN52O4^!d zaGWK`YcGd|vO1?VU1pckB->iG-a8h9nnp~xCYP!+29&&Na6%^ZYk8t7P z@K56=j8e;VHCvH`C@4LrfP6nYIlp))6-*E|rAK}NRqhvZF`Ew05e8Or{AQD1Go^qKJRz{3?T znsAMiMW?cgq@(;~IOZ?=K@Mtqs$)*z*C${R;?v{AJs<8Wby(}Ua;khxJfeAHb z<el}-RFoM>3gE(I zZ(@$(H5rTxOuzgWL%)JROb^+%6%{0^%sVly1e8xcyPyns>(k$NrQ3QO=%ci=SpzaA zdA`d7+((jwQTkdK=ryIuMv#tUo_4V8hVl5khn}4Rv_dYZEbyR`)B1i#m=<((R%NtI z6;0skJJX8CZau2cTZ;v`UGe8hy)H#-%_i2*RgGWeTKc_!zyl?RZxb3^?w3hP z?(Q^slG^ywN;02;%iWi$nj&&loUNn%A?yhQBtT=Y3S{7B?ok$@QZ98Nrafii03iCx zZXdp%RA01NEgTmb?N@ydtvA%Uj~S7vnUr(~==sEK5h-#M2U~rOzgAy5aC1Ps5RJEo_vd+uf!s8iRhihCO#CA zN==zjv5ytirF;#-U$VilNZA99#W+P9u5Vfxu)%c|twK<@5$T!X&(NiTAI&mo0!MDu zxCBqqfJT8v#b9O)#wVZMMT9*y07uckEna#ra8Gfz;nlngb`r)r8@6Cc%KJ0}^W`zg zr7x*ILQQdP>oGeWQG-`(YP6A3CH44d#FtoHaL<#d8NpVY`DaT5b9{Ucu@XX_fcZLN zMaS}hYy473L0Ir$cOYI;@2Jq-1amlf$&vl?XKp+NTiGHXLhG;TEhW+4x<8JC_8-vW z1{Xz%6Y|0pRq0ke0vg8}`Cch;KB~e=+$H8S;N$D=Hr`HTgSI`nm~#_E0T3iQ;dFY z7kX>JYJGxitopj+0(49?85j#ghGy0L~b+hLN~@!>;^_kB|A_+AS(Ecbtdu zHP3AP4hp&sMbJgiO$;;}D59n`t?nCSViKSMV8R-X?7o{YY&fc4eqz$4a!6H-en+Cs zJn>R_;;I8w_*5NM{8^LZg1n<)rTdxrn6tFb+;CTg*{{7T{W6%WT|Ya@?EJ}59g%JR z10vi>&gST;9f7;%ftnz9gI?_NnVnot@lM*61APvA{YsbX@jlGvWNhbJJ}RF=W3_8RsX8oZ9PJBE4nuHD3bZN<(EmgE_{BZ!e(1p0bYW2jZxPLA9&9amZT>QeTL84B7kIZ?3eC`xeO_*n=N6Lety?nB?{zbX5m#^ zGQ!j3u@J`+!h`#FjnHI+aA(rp>Mw49)1<$<#sd10lmfIKmF8nHJ5-d6`N0Sq3 zZ+OsW%3YH3>7LtH;z@p)7e>r~`F_QQG)C?zpA2m9j$b-r!!TE7H=mIxdGg8u2X{0iYBg1 z>cSPDrrAEc;Hu3`27B7uRFaPxYcU@TpS=iRJHuw>Y{8$aA?ep@aB%BeOwGGtZC?mk z;~%gw@ZAkw*>fP(VMhq3C`az7LG9%QGiU_WjVulvX-VMC!IEL4bJKcisesf@G?c2m z*MmsyUHGx-drq6V^fd*Z%Q0Fg!O)kt|ArBS-~p|${9By#<5=SqG*uL@vdBpX|jf!Wnvb~9YzXNxwnb# zjuBT7o|J0SDVF^Vzsvs`)~bnr)y5t|u&%$7j_k-bo_g@{`-f!b>1I`O&|1W1GSD;> zxo|rn%Im`U;>)o zEq^zCKC#6`Z#`#3|MB|zYA|h$Ij87d0$S%r-Ft6(se!PikcB+8$k-ZG1Fz_8?;pI1 zyFd25?=z5H>{ke(!WBL<;vtmKQ|T%Fc}TGTTai*k@2_8ZT`At7Pt*k}NB3F2+ynwO zl9TpP$|Y4Xo4xy4nd_D^7=>1;mEEakJRbTs$4qPA%OW%ENIf;hBjnY3Wbk=ixeY@g zD`zL{(H4aMkW_W1j!h)k?uZlc*Y5Z%Z-R&Y?FkA`gKdB~awW^VH*|U9c@Q&v zdrj$;>jU`o$XEL>za|JUH3xUVB`62Y=GCr$h{)wd8Em`1)c$$_DIfPe_qO{Jl@==? zccs?A`xgm7r;BaCwRZ+@p`6&8c1Tl6i=*=$3E-TV)I}vE9zt)yiWmg})o5aw;s&tU zFV*_J#D)(r1oM)eN(P9Ppwg&3AXHZxybchpL2Wnm-i(Sof)%x2YySo`%Rs=16gf_4x+!gQFehf|Y*$a2GK0^-2BMIE3w5cWil<5u5;vD?@Wfad1c@cW@^V#<6 zH!DKN6e(hB5%Xi;alZ)?+z?5J!3KPpkXYgI1Nj+cEh)}tfd3J$ zs}kWE`h0Fo0_(K7`OE5=?-~Nrij~P`eAW2z@~PtX@k~0xZO1`xEfW19USo7Q8;@bU z<_P@7YcwD+!N;8XVf!L#WJ@vt7}PlMTaxjw)^tELt?w_GvexilKF?8eY#k?x$WgUwb4Iv5e6l7KjHU1@Raf75hesRA}OBL(7X-)m?`=@7XX-K;BWWTk9de1?SYjR z+60Gx%u_Iq`nyMHG2+zI)!Zrswiiv%=d>=R;~cWq^!Li#TNI^zkGazGRi8g21m(BU zSs?s06zsoO?25pSF&5G>V@Tt|8+X5==N&+;ouTFZ&!cgv5>?YSZhMtTo+qouJvIeu z8(Xa$CofCTiIg~=14q|b$jgW*0(e~DAg7D2`EVWs%pt4?p|v)(T@Ry8@qPWk1chQt z=1wP}y0!`|WoyT$ilmS@2ccE}Lkx8UiPqlIv9b5;^3lop<7`90GO{MX`CzxBa$@z< z`JnJiO=kgsK?l^r=9D{g@U^H8uTi_9xoXM@5X4O>6=XFKxC@Xz82yN>916Tt>i#f72xcs;@BO_1O=?G| zg`*@iU?a-NMJOPu!r~&1i~&e+#{d(?A&wD1KqP2rv(%uI(5Vjk#q|k1;6H}Ndp>Dn zeX^?`9}`_AC51s1u6w@(3fM?^(~)&_z;R7f$XJQ&69|007?E;VIGq#lEKLu^hCZ^@ zBv8u}0P3Gd$0S#aXgQRIOeiBeK!B|aOW0-&G{m|$4&?N+nmhOdKZ4Y7 zwS0xeVcHBI`HEB#wt)V_?T>Fixv@!cIEDo*o@3bn47e?Mz?cpGkf!dhlZ3prRL}FPjH%b>n@BzW{AADrUgYNxw*x&u-*9J8BfAL)ZyWIb~ z-2c1W|GV7(ySx7#)c^19{y%|j{=2*XpWNM*LU@vi%b(qW3;m?3kG0@%QJ9)^PNOWw zzcH8E-4>;RqsrUidK%n^Ul?_|u9zmj6NMPHx;5QfOr znk?xaa96+^4J?TQ3Xr?^eR^B}4I6Z3L2Nr%EBX5`7IEO3Nde${%FO?lqcMX2wKIb4 zd9@&l!b94P?Nh!s;TgaM_fhyo2<#v? z_CaaWAHvZhL*JwQ>F0~84vrWP;VY5gv%c-Eb^2ZIt_@y0V12K@4>uLlp#?Ncd;*+8 zC|t*(pe36pr3kEm$4dAP+%@@LdUeKmT}4Gj*l?l2*mHA$4~0k{-n@UP5nc|YZQ1bne|Y!D52Qnma5uc*k&^1I4zeG&-XbkW zy^8^^eO|Ds*%J>jeT4Y{4*k&`pSPBBNT})iN@9P_Tv!x7Tc-fuD&M<;kn=-|$$tzB z`)@Ix&lfvdCs80+qOtnr;eXBw_A|REkY12uP;iq59DNMs>2FP8JQToNHAR@L@c%FZ z#*=hc04gV$$p`-HdExpE>(A3GVW@z*-}>>^%zldQPj0|I{lD)ue0ae$ljQB*0_Mk_ zbVw_IiU_E=@IhPXdGGx~o`{*-`^I-qBYrSLNRJL{&VFxf)q?lle z4NW%#rDCyre=;$>4t{>THt+&VuOlfp}b3wCa5T5i*{1uByN zAjfAoOwWv1RtNMxw<*R73iVoeGC&a!3r$^7yy*7|8>qT9(jB-2c(3i2x-~XN zusqjGzv8QGe~7bhL?A|aHaB5MEClFEj!qwXODjGIE7F3JQo?hId&!o<{XjmrNZY&T zI+I57QUw2y{Q~4;Qa7UBML1AH=q1SXn&+r725{b1-$^D@r162I-P9VGDYrlThxBqx zAE1@e@|+9`kSV{?1U#%^fdlU+Yzj}zc?Glq!Pg}JXDN&8D_9dcr5KC|IM zW@esmZaVW?U->pXSLeLj>RHc*f=myV4Lijn(GMXO^maepIdodTzc3s5sMWI;#XMMk zx1Hr|84864?N_p|haWBIrR}({4)imVP!8<6`-`Hn zH=)0Z__R3qa5t?fQKt`G8t7jCn?K(%xCc_gsZ&c-sRMapsF;tO*0LvWdRVj)$Gt{eTetDBseVq;+zxs|G8sCJ)b`*%!+b zPjQFUC4@_eE}_9nbyv!F2uMBuUT}(u!ghUz>%J(?bM$zX_8;A^pi3zuW@#aDa*lgD zkhLIZJLq+8M6edIBsoaDs=2jAy=SMZL2gkWL@DD4ZB_?s;E}I;>3Y-;9oXv_#l1&c zh?d|VH5BwfA6#gEU(4}eV#(%OD`kdA=ktZsNkO4(LMl%@@%bZ(N5S~2><`navOL&Vy_p1= zsC2fyUOv=?p9S|Tq4Q(fx*86gSH*Y<2y#Z+7~pd8tD6TI1Z?E(^AdRCnbU0`P*72n zfK-PU{1sIVGo$=P!NCTI)j*=BA%8f@EcIcGH^Qc>G8!mXX zDZ7+F9K$_31pUk~#gQwjl;BL=_tND1^FH;9_4&kmk`tvA8&d2G8d$HYndb~etGvHs z=k;{u^3WJ28;KEWn_LN}KLo7kOuBsHZ;=ZF^$q97Akn913ADyJILTrnL(k>Dr$F&} zdyZ(Ah7wUsI}_*hINq@!nnDwX+R^>%+4R&rfu7ycK&+v+;c{tD`2lMqp~sAX`g`Jg z!=X;BNy8J|H2$Jp0smvtNO@LSVokCl{2}=|8x{75*;siqVVEE1=LCikFxz-KiNy)M zob_7?cfB+OO&roWghM5gx|((!@;|H$`92D1pLH<|rKC3F`8X+1D*Ps51pa-sKX?SBNXtrYWv(r1SHFAo@caY#!xdSv9b-U5UH33hj!xic0)hO z2nX#Zx`hE5TEA8)me6?}44`hLv0gF@GPjIk|J{+nbJwjhu-9)V^8yl})!l?BPMsZq zUy0yT3;2ZI1o3Z({9Z#ozDSh7^(gvhLtE+>OuWp6y$)nJ_nquGZZdBIUhofBD#upF zAJZ=FeMz}CZ}RtYRtcv^Kq}3W)dHOoTRxvP=)}Bf`w%B!#uGhXKCLv{`eg&HM|4)y zDfeC*bX{xs`5l8BacE(6Ew)2Sr3@Z$hvl?4D!RAf)5geylqz^KsROJ+r|n#|9txy z+T>i13}$_`*Tsi(?#pekJ}H?Nma?MQ`0-{)ngMNTD+rmnKyuCq1N;Er8`ial5&T5s zPWM0J8iD`#6g(V>&RH%iYq9foI26Ax?+IAP?jWXEApZu7sy2N^v3-%dCu^eQYe{?s z-CB!(ZS)u5Vw%9uDl{cy>Do}m+DSy2x)DSsKJ-^{0HSa@kK+A%ztMav%)h$FQi+}T z%}BcXxuVOGV#0Yf_s4kwLFaB-+x!)+*sftczZJ#ooYWfCu5WTu%WgTT(}o{<^bQ;| zt_$Kz`%-0%s$QQik>G{#9zx@-a-QO!kPZ<5gp_2^82)YH%4zQo&QibYHD?YG4`Hev z7f?7XsA{ZRM>A7r7i4<^KA)3WUZHb1Rm(zZehRMS&ft=NYFw*4oQ1ZLDKk!-VDG~i z=EE&}2wXhz1m^IAbBO$B+w8^g?;;G%E?aZYT2oaun&lfmY^OAHvyBziN@kH^vF^)A zXrS)3d(=~jl{7t>kJUJ;)GaK6w}z*X7kK}$jdG=fvZ0D5Bh3ANga9 z+}93&I8S!RCkEzz#V5n~8<6?rBA&fJl@~|9IV0z_J}h#WnQgAkZH}M;WjzHOy}ZiJ zqkUK_mS;Msu1yMxq5K%@6S&yCU5~@g6za?@`zS|cP^9DAm9|l zaLmJDY)(%?*em{zr8a-yZEAp}ywQwxer4_L?b1%OD4O(i+e5;c4ZzPeS)| z;_=OZixlZAKiP2;2)r_+T@(Tji117g1**iYG==IUw_1wDLWWln9LFu1E7U9Pu5R*z zzQuf(w%jF=<_YFjy;c~E%M}#{&POms&{&py^|v|?k@~1SZ57WIm3Fq?sw$@cam^kJ z^qia1tEM%yadpO<-qxGb(>3sZv>R5t3zcmhkPQ>sg*yCD!P)!{WlBYN6m9c**TlaLjsfn#tgy{3hDIcGld3i%gCB5Lso%N>n2BOeXC{(tc+-r`J zE0xO$(2hJ zoO-T+Td}()1aR;d}wCUO=AeHJpKU4 zFIAPY;}-HSM+PiK(GUgW@3;P%icczRkw*n)Kl*!We{C%4KzImf{`JB4Ps}`o1yJIy z9D#lBZ|v}sFQ4dPaW;{#@>`#A69J%x0)3>>-`x8Py=uY&Fly@;Wi$Dxt6>V9vK+bwpU+ z`i>f`h?`1o>7T>UPmv5M0m{s$ZEN!Q)^}oHF+u65`d`}O;lHQ(I~U-$EFGc&1rAjw zobvl~f&71*l>hV7*KD(^|6#v%u%6u`E2VphhUOJbg9g@z<3dB@n$Ku$a&>dtB*{BD zB_=*m$>aV9d;B$XAAC3z9Z;*+%-GE3ZW3EWiOte56pF@t7B;7+0FsF`9vof}i0(V< zD!2ylcGg!q&@I&we&P4&T`c7KHO0fRh10Ni3G17%$|C0eT%vZr?8 zMC}<0@BcYC{$(JZV`_e}d$ip=GbiA6>bkXl>HDQ_ITMr;w7NE31$tmWd4hr`YLi$^ z-*|X&x_8!PyCil%I%D@}ey(GL4gW30Ewy}4SsuTlh&>y|&`-yWlX7NU#``i8fjM(h zbz-bOZo_$j?m)$YMu!mQnkX9D)S{19^_M5mxsC)gt&e~^W17&QcWT$Ezbz{W zO$Vd>5Dm|H*k}7oCib|n3*b%=z}(8v%1}(e&QeU$!P}kpzTB3JDpaG)8S z%NN7U_9T+8*sPV;lc58<&=&XbjQ1DCS`fi;?tik|NH7Q+EfZUO!n3Jf-7I8B;iEXi z+~p;2Bg)!yC(u2u^Lc+R1*Mzl_;~}57Tl84`2vV0VUKw?#CKgXfEO2RceQ+>UGSAe z`{KR^38fDS9v(?Bry7xFSecNAx_)7@7l*IE=pTA<2hNj>wE1aPYJ4|qK<{PQGxQJ6 zu;nO!q=kSC?g<)L(79FkxleyO*`5=rL0m#xWCLUeF6EpD;A3gFh?d(dC# zHzQdz;;OfcUd;s68FBN2n=hxzbsDoe>#Dp_I;YmQ(1M10!#IK$}yfeRjt67ma378 zg`#W_9(}Mt(P?JRmIB2&&{ltafP{K(SZe78UFPku)KT_mVOG%c9D}D_bU7ih+Quz&5sdd88oqrT}0f%`bmk6`!2!UXx{r(pT?&~|KdOTd+A`AT{EytcBs>_8>uyEmbk;sCINo@m&vmPSQX$O_V>- z9Vaeg2y5+$P@5tNcM6Rt7DZLA)DRGAC=PdPvTI{E=fReC!_#KI`cDf36XhpT3KiK{DP| z(q0%H7u@*{pUBG{@*yEP#3QJk4eN`h*@l`0pVxH#aTqG|tcIru*U@kbsw(POxG0}N zFlE{#sD?SNHqwWwL9%6qMTHRR6*StcQI%Ar50=@~_UFVaOCg)cRsS zs2IT(C!_iI%##tQ@bxf-1bQ=qRGTbAndbw}6}v@WYdbu^%t(VXpqarskw(#Hs`>$L}}L-IvZt`7*_mD>7zvD z!3;D^u@fip++0_8RYSdOGNyjE@{y0M<%N~A^QZ)Qy@be55fkia@t9Q#x`$gtiv%(-H5IH`o)#^=hP@-D}H z`2%dAKIN8y7k#1deX7pW_>0Q8Pz@+!`b0cLYl`~bJ?0}G)6tph3_LEvf&$ap*?h5m z#??<1eATJ_VMklRU5_?7K*l~i%I`sCqFXM!w6WXWsYa z^O_Hpt71IZLQP!~9J!+r&%jD$sOJMk3?A(06 z+SK-?#|7gxb>VS}#$j&Ks@m>&3e5`)Z0=ZKF5SW^cM-Ya9=pkXG9EUpbPXcbFJjw` zUaruq)Yk$n1SMYX%XdpDEQOSQ-y+n)@18qL09!^OHEB7w&17(7osx zg)R4VWYtS7cf$&9evPT?eK}K+xb?HSW3Cu=cDzsq=6@-0xcCbNH|xZm89$mmxQfON zxg=yc;Rwq$qeT_a?)BJ;=&+O3gAoVJ5sI%Q$8o%bDY{9j*_U}$V!&M^M>1PYbIy-h zdE_d%2`yZgGDCvb$-Y=Eo7Vz&E`N3Iu`n<<94e7fXj4 zT7E;8GPUFx!^;1F-hfbJ>*F-RKK#Ph@0M8;JXBVUFEWrDjVUZ@f=wU`A=h; zUIw37LkQgxN7G0MrEVoQ5f3C;@NSttV`4DvhwDY)R>0nJz=%V|9PKbIA+ zNV;IJVR?hlXhW6NSncWN76sd0ImKBw^{p=+lPYGdl;^;WKM~fU$L+plit?j`J!2uf z_HwhgErueU&y5y{p+nT<)-QfQzWyT1W`gpqPtj5WXKi$}+mXxvK92r>6=%?`E_4*v z@g#4puvr8eVwcWOB|CfI35u(qvfHw+)vPB@$pLXaD(6tCGbZsfL*m?KQ1%DS2 z^}{EKUn?01L3sxo>(BFraOBT0I>{1?H2AhN;U3IkONt}+^w#kJ86+9?N7R+e;6eXDL)`TvkNEulPiBBrpqmlkpkUQ`% zk&^M^K@hFo{k83_f-}1S_MtpOz+UD|OtXN^-U-I%sN;XLVd0U;-eToLI6!{a@zC;2Bt@u2& zQxV+3NPGL`bq=2-)KSC9Eld^eTZ}`Gr`mhf>EOb)6>yt3q(jzlmqIGFkWsI=xRHN( zuW@}K(Hk~Gy;y6v1(m)4&lwd90ei*8{`)mfpTiFu;iim0g3;}W#UK)VUh2+X_L^^w z%Psmy!$n^VKCwBoUuN&yhnU|cVx7-NBLeo-3;l^`v_F|6%;Y0y#_JKibsynhuL<~w z8uMOtv*I?FSs)|+X;?_pgn_#`7zO%H{|#i0(msW75>*C^|9G8 z_5Who52S7DH^~pKnW?G%<&71ALlZg$8ajTkx%Csv9k`^j2Z~aGx8`>T&K~gvSMohX zv)g3-sT#MX^E=z&Pip^P{HxC}?IeR}MsgYJ)5U+eo5sUG4exuh2St2Z9~9lX=VdLkyXPd=T;sB)Xn^Xgr+nTo+|fw51KD7 zC3}t|1hWJRMVsez^*!8w+{_)HA~<(mS+p)htx04J()*QdRApt_QAH&oB_^mRB?vvHUy7cYi(}s%WSic5x%FNcQ^wg~d1?FcL)X{{EqRy#;6pVBtLR+-3HHpL($&tn|i;Cc?vmWP;0PEpKla2i$O3iP1>P*6O?QNYcG)0$~He zDBOhp9@Nx-utb4^6Rcq7F5Qm4LEJw{p=KFtb@D7i>3@C$v54 zQdMRmxjIPzyky;>35FIr+QBZ~q0IW{k9r!ma2?k=A}5xh3_hPJ?;nY4)YoF56BCJSNtT4oAfb!>)ylSG>n}&L9DJ z!!1nTn^<^{d!8wSx=5^&Aq}O#jTH=z!Z4@I#j_WyObKqZ4a%F3tAvgk&oE5h^~2j$ zh_|wJC(YEi_&ioPB-T4=DchauBB|8BQ00el>hca7f-hQo+n3DDf)t+cqxUOJ7)IHA zg}RgWIc&?REmUyp`JRQlT}%$FG=$I6mErIw`a25ZMEsf>8+ZCi7u^`w!kS@B;*9!I zyz!FVt&Dt5P8a3IG}6O5wK*%>Dmh@g(7*pBvN0i+w5yg$T^BsmkuTodd#2RKr4{e8 zVwHBSrR)=q0SYzw&ak4q>yOok>1nunj!X}z1`-2u%XRhbI-NInWpkCGJkq+a zHNpDooFVEcYHt}$Fog|#w5+=H6N23t2V1v9BpvNNJo2K#Rgdx^bCY>82@e5VC_I5= zGDpSC3SD74z4^*U)o2Z!?~r;4*CFr3%;4&LZ$e+W+v9MYxCiyplp4vCe6-Acmp!z0 zX!RR7E;R5HKf2|tZ-kc6DoxBKdp4@=D~YGNl*;rG_!`Dkc4e9JD_e6Ja$zKhUnNNo z8_&_7XUjS}N4#v^YUmXhOCQBc)S!;~V!L^Pi?;f*W~-DEhZ3jA`>?w&)O(qcG1&Gf z>cH-cRT`4+R_!^3Y|O9+lmAS~ygSK4TrZ$KF%~tp$dFEHd z2KhGoPQ_RKnxHbr^<+Eo?v4t|6XgV%`VIs(Y5sL~W+Rd;=8L`@L1XeUnkyk~!k10r z6sY(5_w=y$5rp8XcC8+CcCI&Gh>`vD(F~cxB{)m(W_(9>F0l@nZ*KiZdlI@Oci(6A zPf2QAg)s4LBkb}SMfYtxd)b2r&qA$@fu?`OS4{{;PHh8i7&N?xPUe_u-n-RhsJ8K@ zJhLfj_@{n)aaCyoe?STni#`>a|TxrImU+IZCTmCXp4xvkhWd25fa7 z%y<|-o|`NveSB=@P_1>{z}}Te`$|X$zyXIf6~fIPda{*!xBO6`HprvDW-#$qcW~uGp zCRdX^Z1}dGHxF&q)p>Z{@5-=Ns`E7<;QQKOh%V)&_n7b_tx5G!+Tz!4X7fl59aL96_8PLpGC*xaewvWDwkk@%&pezK@-gYvnZPv(MH zZe~?MN--sSjxm=kD~q4#W2TV4Fs%2-yT5Y*m^N_XKT}{``W`!!aK|fjK{>vQJ`%S% zUgo~JC=3Zk5{?jp)v+Kt#0X#1CVSCN*>!NOPQBZFY`%$+?TK^0ayUCmp`BQ$F&Z8D z-4n;IP$BKywg;aJWBEQo$}`*6!@TZ=LursBA%W0DVX!=5iv~gjkD6=!6wCZq zgLcEdcLG4Z6c_wUtKMe^879aC3-NOgNAKB`KvN1N!|j+*O(27myB)9PbgRNjWs?$+ zU#-SCak&T;wmD#it_tJOo%+P@?dB<$dzTo=iuRy`r(z)^33L(ehaxr@YsW{LD?2Xp z_xM4dh~Z<&Sls85*BQQnL*9(fDVUCP9NM+2hwyl2xbDZlY9GvT9%)%BNQauT?lw=& zdlat>HN8^i8kpL4+rZ&mv83YErC&4IG!6;|?l#BUnE&D>A9fe+Tp|Qhr#;36r-K6v;He zhfqs9g386Vldt9p@W&WyG-NugDMA9jk`HKlh7k%Rq!FXXH$SH);m!4*4{@#SoA`g& zd&__*yRLr}5d}d)5D*wb5Ky`q8U>_5xjW2E%I2k-lT zp7(jr$Mf-g5@)Wt_TDRgEB4-NZOy~o(ViK6{$R~j*41%{o=KrMukyx3;G28$sjjue z=98vnvgACe-(n+zc(Nwbw6VV%)Y-F;@&oBHoa*fzXh_`(p)U2|3uo=wymGkzLG50z z{wL?&h|M?fdK`27c6>Agm&RQz-Ye;yaP(Snz0b`z3&QH8>E6y}W8o;h{+7nsM{*Q+D?%lF{KQgIiGNdxR5)t3_ROO`_u>l~D@7{UTK3kiK|aQC=$b!} z99T%r`K|k77SudZZSs1xj%)S3Wmq3DF-&xMU(!xZg>uX`sB-F-u^@V}Z0JaxxaS23 ze5$?G=}*jxGu(KS3j}1A{?ey?NU!|R?d(GSXSS75$7OBB9Ib!cVa;omD#;mGR)R;K zgkJt5WrPz!*r98&3RK=iTqJ^}_t~K#t+$qiDMg z-{Gs_&sVv7!El84s%N3DN(dF-wMt??cE++3??JD*Wa#CFj36t4Apx&z{HNbtPtA4` zT$N*QM#DS#G8$!d27x8{C-TVz`Brh*yROt&>?e_ZUx4n2^%e|d%6R?&ySf+mo zdNk1CZIJCcrRZE=iYt{h($=f&4)Ls-&T=OAdW#!Jy)TQ3--%6P^b>a)kT2fuF2y`) z8@T?sdgK%5C8APRv?ccze~mG?yI(PiwfNFpDTCzg+ExXU9we|w33*-u(%+2bUMmqdvtBC{N`Ly^?Xlf) z9b9maGS4PjI}HtjJm(60^2}N$UEA9GF09S>CklE%@t)YmL9;ZaB# zIp*X0GA(U|wW3>EgE(pGlnv`A;?2vJc8fL z7MKVQB<>Wj`(l}xz1=IzAu@wVg6bAj%e6zrY5iZq7xyv~7Tp=D~x>xa5xFCvzB@L z@q@h}`R>iPy=MOKR!(`PPH!YNmU@8D!25&sc}Cq5D^}8b+(HuOWGWlQl=!7--XGT3 z-X%2=!Na0CRyjT6eB4Lq3nDiPhvVBO^?0sRCd8o3M{L8};NVpu2Sk4sbbHU7wq-2v zVfntpxb52KwKn6l8r97`;cx4wLx;p({rb{cF@7LE@k~CoO(2;HaxjFbM_ScBlVB*Dl+OqTAG)ya3|bKTUXK-V zvPwKtyh)925Pg`c%ZNAQV)3nEmu>npWgd5wk!a53>wr2nSr+bcZ(w$}?eX@|>Q%_g zHG=HA=8pCC6CHMVi2?822Zig&K20l%uRQH(a+iJ=m+*f;IN59+y)$jgJkkr4c%$-~ zv-X~diHiT)cwob_jo)8C%gM@BEP94HWcCBgL!Z)9!Gv|2hrFZXZEf}IKnDZ;5cYgqy7!2;S5-#(vE_PBHK(vs z@7@QCdff}$L>t6t%}@&>O+<-*g2CKUie!oHP48l3JR6M&=bj(82;xGEsPGQqY6v;T zsQh=X{sHI919=10I+$Wf+P2jhet*pk6-~}C*+tBRcxgBW`kr0>uwWrzP1w2MO`24vPF7ZQ=a)*U*qMo=C{QA1tA+Xu7wIv zn1ynczRAOlbq{2@UO0EVGyv)uG>*4Deyq_(tX#ZnucB0%C^Bh-8^f{``ms%N;VbW+ zRN9X3zIfa5dr*=uu~YlvZZV~{{XWm|&`V|N#Eb`2sNsJ^x%AU~KhMJy$)h%1%aXll z2`08oFvCfD?P5AD$799Z%KLi)J+%7<8v0WrX0t^aY~~3B!xV9-N}hfw*fvABGWyf? z%;{vs@bMo+3TOFZy$QV469E3Oagut^-oE50%gP{7oompvJ26Ph!H0XzK-jdu6yOi zuRe8QRG;636~lMKd|Yd=imaC_py7y;)?Sy9|> zJBP?dm7pbA-AS7{sc_~W?{H!q6#aX8eS`fvUF8I^to6SUoC-u!hoMY#F=m;4V=6rhC1mtEE;WFQ+?XO zi4(B-q*b-LzeV?Glhk9M*Hl4dWOVAys?aGv4{xYMtW$(D?}6LCxt3wbb&P!cmy52W z?P2M-`+Q0gfvOX)VT@L5>cs2G!X(jl zCmP=1>M7G|$qBVH+5}jQLH=4oE!nvQyWdX8i@CEXa!KE6yRDJNWO^0?vfp+7(|W@! z6e4dId|(=n)0kQ0)L)BGpLs}$;%Zgkk{hRZ5XacdR^bB=VEuYcwO{BN3nwK%9rdyj zuRSXPgI3}OZo~)Z(D(95V{ZDM((XKZ9N9>KV+3`GiGfHL{%K~p7fGZ1vcGFv88ZC1 zYA;3=9Pjkkk~htq)k5?b7m}b5aQ3tJyXu){g`X4l7%F%yI)b7KMP%^r zv~g%`J#%ySITlX80guvYF>~*wx0W3BQDvS-59NN4ob6W?m(Np-&tUM$gNrpp%lv4X zYkg-0xlN=W1ah| z?bc*x*ZP!<=wqd-`>kbBEc!)L3}*y!PUM92k8e3<&0Tb`pb~5%_6_R)uoR}lRA}wv z_YC(wlG-;yBj?M~8}bj=w>LcUE{ZaET2mLlBky==Ai--bKkM3c<85!{;HG&TzgOX2 zan}U92QMpzNuu0QhhLxNfPDyyi-Sf}7g@dhi$4!DBOt;nw`<13I&fRXvj}h#S@&`s z*6vDd8X!=*K5=#9`*yv}1g?W{Uq5bW3aQYh>hon@!eq!Ls6qDLGGRv2tnQCdQ`;YC z=^+_nXS>^y*Gpnr1Rir)VJ|lygg+Hh6#A;E;2kfkOP-kai&9!3Zy&txI4yXjxh_o- z@b>@s@kr8TsGYrJLI?GO7UOd-%h4GD%dC#W<7Tc#@c_2`$8>?N$FmVj?Au*CRY#)R zq=@Q@^Wh*tnjtb{XasJCHl(|FN+t@38!^v?ysrC@Ruw1xy+bp7**!g!W1WY&`<1z} z&Je`2sGk=s;SYUfDDrG6-)T)Va@^LxLSx9pX4{pzP!<-4u+(b_c>P0<@klydm=b(zh^IhdM zR!O3ydVbp7dsN5>0ma;fg7ax#7n6}$;-OvEvhV3na<)xGQP%#nAE+O!w{IPa^{edl zSXDd>du8p{(|Q-N5Ff!C^a_)ZnL$ul8S_ za;_#i*R^f5#=u|A^*ws)Lubg%t;ij)T~jn4#BQB;L!B*7ktK?xKn+ zanxt(*QnwqRki8>hN+6IAf6EGrlWmGv6`T?6m4s2jB-tHFZS!4_QX@iFYoetclmEC zyd|-F5AJx?w{NBm@nTLKam*%)oO2gBup-`!MUD__n{kswn^Y8hk&tx0)xFLu?7?!7oF-dsE)2g8ASgM$`Ij}uIBgn!$P@`{v0iuk zAX#72n!+*G2Znnf{`aV}Dzl9P0q&7sR^q#w?SLmR-lL^ZJ=VB=e%Ps`in#w3qn1-hAp){-Av{i2q~!<}TmhcOP@p-}9qafe=Z=IbIgY zlwUU2O&5>(>B20&Pyep|DppKAh1_I%LFE?1m3c%d6SKuqvn@0-n!fn!2kY;OqcB%; z#s{4*Ly>`-EG-~P4SlyuRKTo~@14zR^OT^@Uyh2Ky~v1Pn8fyv_A1LQX-LHw^_1j9 z!N!VvE>i-$<`$EhhHyknY4_~gql6e2=f|(^e3;s%lj1^XxrmHqY`e^v%sDNc*Qw)d zV)Z#7(%2VDTI<~UjtOUG<3qe)&MuS@@82z-3?C|2)qM^wo97$xLanvQsob>f=-A78 zLu!!5M?#Qjv$Q&^XH7IwA;ajz>{>X>G9SLfo|Y_4O!rYW&76O`>-xG029Yj|BC;LP_E zM0a>n+zN?bcq*sjC8IqO9xQL^a{^WB#bOSoh0#|>#QRONRCv57g^wQD3j4WQpZNWi zo`&9UGsLLtS($xTbKmaBaS1#0GohPi-_iRawB40(gmhw@5_UZ>@#KB%cAGHt=>grex9%#TgZM80=JuSiOsA7-1{9w zN6oFvUZgitYYfA1B_m#rA_vx^t5`C7?Y=Sxe@kyTmbE zyYb8Y1Z)ITd0Fu3S-lCa#@33|WE@Z^W~!$~a)C=e8&3Rnd!E5(8eN%dO zd&nwihb0@0l#s0x&M4=E_)ln^4}B%VvSxR(vn-Uzj};Gcn&gY}j5-xUCuQM4~=klSp+JZ}wXT7^A+SNs3Q3ZWmwkzY#>ZWn57Gu_z-Kj=yIH!MJ;&!QJB&v0p-3>73kpz=iDq=P!q-{@e|9sZfE%hwuXH8U@7%|!g%KOPmFqm ztNXj--Le{vlLXBq-U^HgMU`>D;a$_#d)R6diP92Qf$3)iPR?h1Sm4rrFIWFir(TKhb8I6FYWW9qMWIoYyM+vNC>5TioCCgb%Jyo&-z+~XV>^pa$e+I5T~1pUj(lQ z9YNzmsmq_$8fbDeY;!Ss%?gelr3{Qyzgl~u#=nGN=uGNm+7~JxB=_30JQxb={}H0H zs%7J`f8vMb6?&B44_9-IN@i{ESz7(3ME?#SF2g^C2t##T5>bJ~NiCCop|s_Hewq6W z6^0D|Q)e^g&&hic*tD15#UU53r$D_e_ZL)NF@Y2^7c++A4GWH>6$4xn&ljW`Bwyu< z5-rFb36I~Q)Y~Mhr&KfKZ)ZOyRBAIUj}T;9il`q=P`_+HQr8X6%a&Xz|YJ=kLD8mbx89HdD$CD%kx+ zcow9|Dje~WYh5Lm#v?r0AYB5|OQ`b^TAA_It7kC#s_cXx%*Vg@gf%3;(GsYc_jbto zjS2rjsvRq#ijk}t!y;E){3q81l{<6t@=_`fHv?x}$i+re*R;oCnA0~G9AKnzp1bdO zlRM^at4xfVBU%;K5^oMVlG6fseygvy9lx@N>mV>;(Dp6a66WTFxBozuArw1FPB>U) zzh*!3a|p|RO+;<;Whr4+gdG?9-m}6bDG@xC0SyuO)iu|XT~0R25SIn7G2eLObEU7FF`4suffJ%Nvr?90gFG(g5kvC zL=Qc9e!XD*jnjA?8SPT258Q$pSBGENWp$wDJYz|E0U9v;tXDje%-LQ1yR))FROiVS z+wD2I<+28aK4Vet4ypY}2bZ7;I>*8^DLt$%b?=Svi?c-^B5{UKWwCwUC@|tg0tMP0 zr-Tjpd>o4Kr;b_Qy!sShq;*$8`d`)-hASwC5}c_-cwye%B45))kYOtj;#ZKrtHYDp z5AF+N^0wwFEZ*O_C4xMScJciCEUg2Dj0EpmHStnKtxkJLiv0(s26(*V$1T4<8}^sE zk}(Lua^7D-;*hlzHu7Z)0~rEzx)L`mCtPua3Eb`HX12o1=?hiN@Z&Xiuiv5T{Z+koU%VNfCqU5%bO416GH(3HZP>vWLuFnnlt&Up)V*&)uDqSjRykZ5*hO zb+}7Gxli$?X)Rqud&6HJSMl+$vUZ^_wUjcXk@D>+u zRs@K4pGIPYKIdl8N?<+%drd&%^u%16C||@VAkU`+=D*oskXu{zE?nxX9kgaB8>(Ij zGCD50N{d%qlcxe_#v(s|?t;lRaMT4J&cIO3x*|tERw~nt*otpIK-7HKBc%Q+oZ!*M zvcAWyPsp>a9c#IIS?N$EeLlT^V0m>BlOTSf=P|lumuru;*gEy^S%A%-xzWNp{OkGR z_~A%tBgplZBdbDVQDM0>osxPjK)Go#g2T!6b!ERT$wP|lLo$V1`p>d|PRdOQSX{mO z#=8a|b{pVYbuQe44{d99iSi5w#yfJ<^tpu;T>*Qb1nj|(@NEKg9?ZJY@N;cavOB6Z z4K~kUPw)D>=tJr#cShIfR)JRH9^l_n?TvTofTeZp^US9AN5_DZPNE>`TfLaufS7^J z^w<-#p4D>P#BYRe9C_oIC0=rkM<`kE$tcrfy%p}a&rvSZ@my9l#KMYLj&(OxA1XP0 z|6Q{RZ$^lpMX@A%z#xayZ{SXc$h_C2j39Z6c-wNg?i0gHg zs2Gc$0|CqzyXOH%fdtA7$=-`@s2jm_-}G7{ZdM50}(fy1P&P>gIgmOAYA#5 zb7nIMx`$DchkkYaaXdYl0p;1pl@a=xvcrgEr+aD}vECaprFSIt{0GayMjpKbEydS-TF>JHr{%$C^?XXq z3Gf}14_WRi*A7k!XRt4#F&L3`=LPFQWwHsNEaSMZ=xbqngNN&#q>s+)wFB>I+#JK* zxPQyU&^*sS@a9yor~}&QQPae6QdoT6pFTYYI5q8mAJ(;q7PTuCc!#t9`p-;Jd}Y?u zP8jL`JTeZdZoZ(pqKK9<7ts&+_Q|&XzmJCh|93?Go2dU^o`4pk!Mk7hJ@S8DYv#a6 z!Qo7?`>*r=-+zLS(KS{df(_pPoQ3G!Tcd_|AKo13{g2NdQQTTtJlIOW|G(wa{vOMY zrG3wzVETjPzZTX)^PV4~pQvK=pP%3VB8vBokl-K=b!*V4ndN^7&mbdeNV|0#T9Kgi z#_m}j;j2gg`AgB)2cXt(+wd#h{g&#e6dYrso&Pwq?uzz9#QHWBp1az7lV0nhE(|(9 z9k@;I^r-B$?Q>LC10>`=LuDnd59D9FR?MqL?pYQai0n=})xUE71OJBo{(JmqQi%yc z8l3fjmcvzV-NKxf*gGBp4_xb8kVSqm1od|qw&M2bbC>MF;r5-KtCLi_FS1y5d=)S=vJUBsh!6bVCip}+HCksNi7e((+KSNY|s*u6CZ&{VA6 zvn^Ww=TK%;1AWq6QF3>-NoGY981#>GKu9y_^?*~<{p{f9Jvnz~oQl;0cAb!WIgG9- zOLqBJm%HCR?|LYZ3%k-v-uz5n)UU;Wbqbs~ll?hiVs4JIbU;Fga|=TpZ6$Btt!mhE zKT~XTBFWqRO0vSR)lh`7$|Sd6FJuav6$QzIw_2#wIP|O*0;aIV;y-BmodlJCmh6r` znaIaw#m`V3ucE_zzwk11-k)YN^*Lw1H zZzW%LtTP#W*LqGkHID3$A2WCu^^e4>hVL!eVFbZBqLJl*n4Vn~J_F@~-e|L9oAnU* zF3S=LT8Dc62_K%5xzu2iK?5~4>%k2e`_7J5Qyq*&cPY_UWUE)r;@b?vBG;5^{;PHh z9%pTfzpNKLIz8$fmcEmq`lCdPP2WPpNBd&Pm!V^sHq7^kAB-Vpdn=JuBpS5EUhKXj z^%eA%Y!AS5+Gu|5A`HGgCRsFoh4uwBsZz~U*2H*7u+gw;he9*#|E3yIbUVioYSRaOR5Ya8S0;R}Y1t>(ZjrkHr+Ra-Re`OP+-b@Jx^t|c6 z_33yJ3)QqjkF3IqLc%6(WsyPC<>>3q8VGSHxuvRx@1sWW}Z`(ij$3BI6z{NkWh z!qJPAKmWX*^WWEV0=_T?-cCh6PYB+k*^55W4DORbs~NJtgTi4_WE)U0J@Vr3nBS6V zaXMlyVZArn@);CRB3f#Oxq^Rjiu<1({se7A><;8pwoyQnoKsR=p+9gim&*0FK%|<) zD*}=Tlj+_Cb)6Qe@)KX+#)H)xPQ1gF30W;ckE%9$@U3mp)pM^c07;v8S@1XuDdY?C zaklM5W!xLNy8S4OXXCCQDR#;i5(j8@&^l-ano4}bFJ2@y1vwHZpCYMQ=;|+I0!lym z>RvWH1%1)irlR}ILBN*|-|gIUkkhYNdxm&_?_}N{`u>6C?YE72 z8-IB_?b-;17`TNG&Lk2%?)L$*X;-4A*jxr`Q1!0;(84&01!GcZ?6LWOS+#p~s;kP`d5A+{(!3W>st34=J-mN1jUgw(cF%#c@KVt3WYvTNd~h# zE7M0;WdLN$+et|42MR~PN_B%5$V!3WL?Sw}(a?7?lr~E05A!-W=xo$BUw6kDk5Q~1 z^`vSnXw^9YU2Dera>F^$xQXq~rGIt~p50<}u$ z!L!5nsY1cT**^H(#<>3h%{4*yOu;>~K zyZ537|0=SVO+0N6H&TM2+00A~I33FltS#4ZeSiNFpsx!t)>Z zFiw!|#TqB<_IrFm?*lc#s`ov9fJ6yc&hIGziAsi_*&}6D!qLu0l=WW{HCV^4V*NuR zKBi|#rNNna%x}yGF)CD9Lk07}`uP*>gExSs$pLrrSGnL#Io}-oYYU)|5Jdf>P3^7b zM>{M)eAR{S3;?NMi^np5JZSun2x*7Jo5RxpSeHWjEirn4RQnNg1F2>o)dQBd(5*XB z3{VE=k%CxC6QJ)|)mwCB4gf>B8y}5|sBk)p>s^%qzdq~%ep$4s>>&00xYLdTwc-DY zP$evyaq=${vP%4+5L?%TBI#isvI-p+{mfV7=v} z+4bP=CEv~6OK^1S9~o*=uX*Kn_pi12iU}ldQ1&*^{6~|3Zy(g4ZHWMn76o5ai>0_A znNTF-t4;;jlYcmMr2ZI*DmP3p+yzWf_fnZg>T$osCv=RHpnpfG;;DJ<_Rk1@GGK&! z`g_3MST4`90(xGmB}bZ>f+*V3CI-a%ZUqqQ zS7Y*q>qzU7{6Ls$kn>+z8i-p-Ya{t<)t)lo`z|vBLW$M>0ff3V=!mp?3`QXS@!1^X z1PK7EOc7L_X3S9;A$25SMjJI*` zAIn@wFAtkCy8l<_AY=Y5hiZ?qefh!4jljyO^58F`k!_v@89iU5AYt|_gW6Z;KImIt z=%I0hbQT00*+X6e2c`}0>;}}IEu$hAj)GN;w}JleAe79oP33h=q|8zK? zp!;(c9m(=n0g!BQt{Z7mdnt`0D8C%ls$5PNvKL=O2Gh ziWwNjZdpR09vi1XJ@hqLKkFm_acYAZ}wpe0S@C?=p4>6Y}lr;O$;T zdfk;sBP}aRgMn5n|L@zKv2^(m|3nJX@;6S*J*2DWH4j>>nC#w7T!_mUQ`QE>u~8Mj z;aMP^*t5w9ds?t8^^c_^VWU6{bqss&farGPkRrO}E@Kt$geqnCE8SI1oPcwt&5M)$ znW{gHS{w&f&GR`K$wfrysUH7#ac%^O8ufpjoS+mQ7_@OVFkC-ekiKyV!lCS+7Zx#R4dpnnK!aFY!&1LlO;#9xd>erV`D`x*`@cuo(F-6~t2Lu%I zf*Da82&x3@HDqFHBm>j8>&CZn{U$|YJQZNIb;;T#?D=W>!}GD=Rkq|le3r%I8NYdF z=Scop_@3O`Kc8g9%Uwf%wy`Q@jO5Cw+eRdsQ^rST1(?I}Y#zMAURkKbx22u~anKvo zWzuZ+zZ!Wx_VmVtA2v4Kb(bLD0_?f_rjdR^MbSS@;}B^Wm|((JIdWgL2}!-^-mM<| zbh{n25i&cQCSJ%DmGO=SHzucZ9$g8$3^Z^+an79Q9bQ5O{?h~BHu=}8`A`U@doJ$* z8baB)3fUEg%w#?Wv{+f~EOJMxUo_w4QfVjWaWef#K%KWOQ-+|8_2?hY(LSWHkHb72kz4> zDy^uHRkna`j`jH*&^=;P#EO1C!(nrq-$1n`#?o{nQ>eytXOZJ{YMX6X7^rIP7Y_p{ z@UV)PYVH&O^e?V=5wUf7+juaDtg%m4P)4CAC&VCAdkZ*z1X@SG3~FjX?TPIO3q@N_ zG|PmTuIF>kJ+xsD3b;}8oKB{1!kt3yq*E!NW^z@-eDZ~Ffr#KPLr#7%v$3q$Axa=T znutAZOrE-$&lHRrMEk0W&ZQ7_<*?>#AI%`sJ$c-D_3&Z2C(SBj@fCLQ!xfHqXG#>M z9DvibLpOLjY|OXgj$M@WXByy204=k6ah5_W(4l54`;S%*vwgePcx7sad<84vzaA>$ z$1#rJ%-71Mg+I}Ia$J!b=zgUlJIsQ062;%6~)r(w@3?fC@~jYD?jdgs7UD#fpg zp!2!HJ4Sp3DMZP_N&VYXx$|l+6ivDHgr5gwCYfkAee`8NU~buID&>q%ZWFPCaI(d! zlr!`1jB`&{{ODQ@?VDRnE+D7AK-KSV4>zE@OxQyyzFVsSw5)#e*8I-&$zk(OSE2#o zljy)Dgf+v6Pe&QO%WtfRTYRJ(w}!p|S7H{W*1#EHYH^F2>|h`iB2o~i_@fw^Y``8@ zqP%ev{c2F#%-u_XJw282OyVG;xVkvz#dx{6u->FBD<}86AW570OVu+HXoJ^pTPW$B zKlQGOMA@t!Q{Dtz(5JHh;1{FLgwQGPUj^LVqa3T9OP<=~z*%3py}k!b^yF*uT9+Xf z9h{DaqTSKP|7os;%t(4w^YnqxUN}Pr6iKmBjkLW_5Ip zse8co1eqZ!7Wy%UR0I$hT>QOlIba0%Clo}ED&1^H7t_5T`YJyJftV&rwpF$_a1-FX zI#C|`MWkv0hlZf4|L-0@TfTa=i{2TN2QLQPj)onyy$~Uki2kw_Ba-I$2?d_5g_*Gt znlDLsudvf5D6#IcQU8a1WSN2HVG`?GI(>x}U~XKr`vA$T-Sh)+Hs?C1n_90jH{WJo zZFGIdw)H8oO$59@{`oHs`>4d*0|nJa-S}5c(5Xg=XjN7d0n$9H7{Gd`ONuYV`Gj_T&tt0W?a zvO2vEzBAq5zdtqk@u%PZ@^V2)=0N>H#ez-j5YL}qKPoOZ?V|hnZq3l9Xqyx>`(lSk zVbwNO%uo?4HY}3r3;N{yg0{u{Eo#P-FCj(#C1i`BmzJjDo*Fm4Z#*_#qrEcyTc&Td zy!A(E4sAyYPHrGl$*Wio+3ATkXtpW9p&#hZBJM~0>IvbUd9o=xVObe-zWFoV7A<2D zRNdCeEYeI07McPES91Z471>Dmi!$yND4kz|KUYSF?`!DeSDZ8&RE)!0|4X-~UWNRI ze5+nZ?on%n#v6)SDi?`YC*730aZx?INsD$DJau|>Sp{xMgjnl$r9{2dHY#qiYYeuP zQ!b(15xGx)+?X!^>2Ay+DjbdX6PYPUm}5rd__Z2lGx?^6CFXRWFbQEA+ZVk)gyZZ3 zp(s7_9GN}d$$Ee`Ktr%#qFiaPx%q2HNi()aM5#bvgx*b9q%Fxx(~O-}t-qM}mgU3D zCglPbX?uHKlsaXFE)8g+*QNI3AN)Qyroo9P#U4h3(*xV8X9dcop6W5|s0N`R(X#k+ z=F|+@uQBT54;UTm{pKe$tl#*#GOqW?0be*Ox@r~ICGqgj69xCUIGV9z4>DWDj8}if&Rt;J zY5=X#>o8f@7J47#K3krvxorM$-st+WTH;u^1akw)>eVSHh9>}o@_U5LsbL$r`FG1o z9hIK}FZ0-7L~k)V*h>!;w23RE``zwbllU={S~!a1i1wQiGL;+VF$a-LZ^e2^a6v<0 z@I?9N>ym%nXl+tk7YgeepQoby@G~{&H2o>RE(s}$E_>`F#K2cZz;0-#hyPqK!`A+% zw&%-0LnfR6t_;dykkMNeb8b~Mw$X%oGx4w~U~*+7b6t8NeXo7)dh7FV``kV4>c+qh za{r*Ul0<=RNS#~}BQa)W5#f_*ygg!hQzl2dg9g+a23X^b_bq(n-^lV z!VfWa(j4|A_7fv<-*^}O_2_b+%ikfcqCNHa7voXoaoy8^evzhYdJyIt%F+y6JrIqTQl*7|< z9v2E3S?5e_p;y%&?^qD^r2Fj=g_)#h(6PnV|%9|sjxirfu$ayFFAyBNEw^gtw|KZBP}4n+-k@ZGoRRYV_FeoQjjm$bo$Sg zwW7t~Lc@%o84d6FRS?|)iyZ7&vb!jFnxzR*2%nR|RiS1>*!Ap>7KC4D=-fU{wp1D} zrg;Y(Rw#Ctd>mRd=nP+9w^ZU_OjiOKAN2O)kVk>V(hZLoBXz7GLLm&g^Snyg&0*ol zf6;%EYAo+&O+&<`M+`eNB*m_!$Irr+q>D%sVATtuekt4}VTi z#edb8H4)ts{W8Nz6XI;sD=y1DyFz}u$1tM2Y~%$<%WlqTA;-$~2-}4N<#M(=@b<1S z=Ib?A`9bSS=GT(=5tPS1e^FGvYG6r^BWIrl#3&lLUp?GCvd66fw#KPRiIn+3cg!35BY}0iAAX@w-60E<`eHbbW^Lzf zrcwXq*hgOw8NU&*P17=TcE%*~cct)%rz`duj&7lO*N>5C3C5;B&pcRpGo{9LMAMQ< ziPSS|OL6V&Hp8lRxpLJOdHp}2i(y7$BGD3CyzG8I(gT+<2X|2@%bKm35><;pvSy$` zN{_ukMQ<8u(QLZV>8s|}Bnff(W~H&^>ub8FD)HxJsuav<{;cB9!v%576L;bCVa17& zEPdD?#TM>K#RDrWM(d&j(s~W%C$%Dz|8I=*s1}bF+ixCu%@eJVCkhFstk=}zYfvG- z8PwEm`ppp!Yf+WJp|%gJEKjV8A|$Dy8*>Fta~xCkjsU?Rzdc+(Yu|`K8%4j}4pBGg zQ-J8t+uFLUid$kP3N(wrxGqy>aOaUF3(sg$s0-CS3rEg9Hd&Cp#dPkZN z07!1kd&V4>Jb^ku+5G&Uiq^3`dcD;3K|+K$xF^R;el5xH1+|Y&umXXQg4 zWtN}mov~tM>kD6f2EpBm^b6an!d{j{@O|bG{JpJF{Oi4(mJork7c+cMXSAtdPa`7D zeD5W-4ebebv2Sc@c(LvEdFrT&#zzQi$*nj&pD;(z97_E&NXKY|AI!~F_a zu-Lv#7xPjGi(2MXah9yaSgL5-k{#GsGu>nZNgEh9OBHd*)yx!M=awdEYyMadqg(LT zc=;Lsmcj^#<BmVbpSb@;qC1Sy%(A|Jr!R(4Qz#u=T?61TC4nW#3S1$a;2SE zI@sarRdibkp`V4D6{sAJ#t(8$6lsN`C$W@?ifKp#{guFm?jwF}ny} zG^YQx%(DGDS&5NS(PJ=Vug-zps2WJnYuH5pe*dDBykDw(eLj#)U}R>v>lw^-mA5o7 zlr?B-@G9}Q0`CvSMjocPAubv#g;V(Z@<+g6sM@iD*L2r~xz=$XXg`(#_WL`Hg%+@d z-!l03Kz>A-u)1=f~wrTP@4|{p3=(Si=cFZ)Ce=7fi-v}C*Q_88FOwi5ShJ*SB=&DGoA^ktPiuGT* zs+%VxkmnK2@CK3tlE{|<15Oe6(Jj^OF>|t8O39S2VYt-*{STqT2#VuKJb^TF&nN*gg}C#@RXAC!zx=7$&xb3 ztyV+RrxOCyAVo1Yz_!591N>=5Tnl=3L%tI$yRNzH5CRT4@#*H>V*28Da|-m*zE^kX zqvhNl3Ldv}GB^yHtizvNU#P<$68I_Z2}}vc+!3pOftr{54<{SHw!n;A3z+^%?J?3xxTlmadJCkZ3Ia~nAs=ISd=9@LzB7t zAd{Kv?tuHH2;}psduJ&uxlG$qnOrwef@-VMfB|-FV0D5-zVbb_Xc^jYb2nKIWJnZr zFtEp2&Z<@AyElr9n&j(m63*e+_{Q($-MYC=8eqIZQ7nk>{k&=7N z--F3f6B681`y$dzRUc%v*Yyjq=vO}5cmuyqt=AwU>x*<=xsM`uWx%`D_UI=0-}FSg zUe}=npIysmF=q3gJ{Zu(dsUeE7e7yBA;A-F%`Hf8tp~i||1tSf-e641y(qr##B@dN z{gi5ah)*7cyWgO?o<;2vt%hj}c`d69#)Y5-vAVjDqOHT7>6M9*?(i=EjlKv-UFLmW z!CwpTS`svkap!8i)xyV&oC!MbP=}REA>*&)y$F0c%&4=<^>KLs68Dv8kO7smv6x_7 z64t`8ZJds2X!RHWf;fk3Vtf-{wjW!hzy#H!{hYjd>@>l4nPS6Ja@PIPX^=oGD6!HP zR4C-P7+WgaiYbv=zu2`o>umA))1}Il$xu^?Wkc`?F z>aU+0FNPzRZg0$I9q#O`4QdFnGziSl_oQyqMekgs=I^w2w(pu|8FuA4Afz@bp>jpr z@oN1mHEl>}R0`LupEF6=cKNhgGR*G=6EOY}KRZ5^&eiR8aHgRR_n$g=XK=!mNk2W4 z>dC1*qov0Ck&jY&wYHT|+o+ zt5&1ufjg^Z^*`*&r6nIE#NeFU$-)3-Op#kpcJB6Ld|z+-H;t6PphQchIZ(M19jDIm zgnkP^g(C*ckMI6?dY&Jx?uWd~Sl3Tye*Ea#>+rBD{b#~iNem^DU+_mv^24ll67nJ6yQ(vl%>?ycm= z-|*z(8BT{t!}$D`O<#|sEcec8${EN;SAYvB06AFFyj?f)R|g|#?}<%83o_{>#dI zsAGDxlZt)_x(?EZZv%Ve($iCxk8#RsPWG&mnS8-T$jp!hA7up_4Hcg~!$II9X--sq z9Z+7f+d=3!&H5`_&H!PY;eKU&F?5F!fhs^Bl;P}PxlgoOxWT;^mNSFw7EiI&02Wq- zY?|&K$5u)Qy0{ZSFFXbfdQ!dSWUs3mO&G^>2>6ET*8C5-$#o4D*yEUO^%g!BFaf(l zrXPuu;9@lga-&5$L8@>{Tba1S^ruS}+Hv`8{2l!%7~pIP(%t#(-?E~+@aDkeP41WY zdU=pK`40^R3@vznlEkX2o+F2Nvf&4f1kmtZQyDT%*ddP@tP4MxJ_Nog`CebfmZJ1> z#l>L7?ymiUbX*rxE*IFtGZZTDV8S67ndcU`=U}F1$k*HYqst25dj4unW$EdZy{j8s z`HOw5=91C5HVi+Tb&2gfwr6=sKgGPXpRbI(U5VfE5Y7z}snuqL%9hSMkz83=wG$y! zcC4j39&Z+2Fy47#^5q}CI83NXB%8)t6?r&NSdO0$_9$*6f}*V^#a>+k-*@`K0Ep^H_hS#}CF2o+{U96q1r>661Af#< z#)2Gk@{xALmN{v37Cy_n+D=#O?T+i;&W>4mW67z`yi6;9p9T^zS-dWGz&E0&S7-g~ zw*j#Lrf0vS2C?kVVN!ptQPP>3>z69B4v*P)eNikN*Uw$6K9$EW37S z!*YMY%)d&a`v0jUUdG&foUT1JNDcM3V~;~&bwttM$5gBZ$`0Bl_&)$+Ir{OF8OIIg zKkTY6I=(p~TX32X$!*gDYb7VhL=-1K`a+VOAda@`c1_@?dkm{Zo82W2NHN(Z=Pg`1 zc^{C0#xfHW#{LBuk2|N<1KTw~-*PIER@0o{HV0duq}HoOO4YR(Z2v%J#mUco7~+BE z-fPTII#fXd;G=NQ8#B|gX(s@P^)W-b7Brb_rPq}mz@Vg|&he z3D<1Y0tjb8a2axx?PEOtyegDj&UKG?kirF?NGow*cR2VBu4KEX0Ei@;?~Y!UsnnL^ z9H*zB<52Ci(tABX|9K<$)mHof;o)Zg$+ZIZc9JdpL^1;5CH`bOQOCS^6&cn~b-^YO5N6$7Zm>rX73+VM4dVI~gThbgcJm~)@Mj?zVk8^Su;^l|qH~fhMxz2e-*?4U{BJf^Tum*x+C*`|kaTavN!x;Is; z*;!G&Q=0$xR5y|$`7kj6$QR!dXa9(R{(sfAY**1w_$nBJH(jXks+uK(zTBSO_!_rT z8`Y$A&m#Aranrf_=0yogh-nh5GI?BQGt*U3Gfa+Q=LP8hF`E3VlT<8^8joUpQ_tuU z{`;lFUoFG56Yo&ViT>~75goXEFSMMs|J%c~9;d+MY%RgZ30PZoL|-l@)kZc8&-jkQ zM5aQzlECW&;vy9xcQL((RN$ylHVi>d4i0%!uiK*}!LUMXgYoi~mwQFIbZ;cZ5j&S>Ht#YT4IF%5B-Pb!xjF< zy|<^AxRv8C{Gv|AeU7OSu#h95B9|IB%`>~qIi_-ehZApdabU&?ug*qFBo18*uz$uXB2GzO=ffSXibjMi*YPKtFprs-0Rr@7Oaiy-C} zQQcBK+Uq}47_i;zhyhGG=e(=!0ex^UJu22(32;NfmoA%Y$Xl#(Cv^bVf-zPQusa!N zagObW5~7G)w(NK0TJhRR(^YniR;a(1+#d-r{C4l z)z#%Jdjt3EX?AN%(a+0uYAj;yvGfK2w*_2bbU>^BkN0t%bu3E!(V#p0O9)rpy-8hF z&??MYbAW_DVeEzF-T}KX2Y@80w8C2R^q$iprXsZm$Rmak7a3B2Y6rgE7f|KhcQ*NS z$VQS|Q8?vKBm8e}%-3>7h9HhScZq+M;h9RgyAw&>#mR~w)LgUfZ`EB_JzPWi35UOE zSsa2-rI`qDdvIdbz(t1hlbjj$67-Nu$GN8v`guWI3Dq(ri9DYw0_ppij9LD6MWwyj zke1tfRWQZ;?bw&f!NUV7d$7 zxDR8ug22b|)Az2V*>~?vjc*!8CnB7zR#qRLX+R0bPvZg1>)b$Gu3zfU0HW|%P0 zC6UcmK0R_u3g`?H7i#@xDz9h$S0aW^ zu=(-X*Yhwx0k?Rwr~t1#m#j(2hVtF zYSK%fNTL^wedFvIfFj#9O92`(_SIj0Qhsan21f-0)-jx;c{CQ|;+rh6DYq}8~vSZ0@E7T{X2O9 zHMs{9%9Xg$GHyDm)0Ji2Ro$87+UyL$gIVW@A}%yMko{lwWFuGse~cb71QI*5j!E)N z`9K~`sJUyJdwb|-Gu~Q?k>8w-4lvpesLdF_{6;OidOvG8l3OIQD!Yr-;a1ojk-Su1 z8;(J5*+8z&;>7t<(zkg%w6i2=I$ZTLM|q3MtGdBdIXb&`lO-1)1mHEpjo)PEy=l|Pr=I@V@czNV!al=cN2-bEgXyTRs@2GEKixurRa?Utej^Na zu#TFJR+wv8P)1r%|FM(`Sm{_SpAfeyEw|_k7EJ&BGtDgFDE36azi9W&xm|T59Oaop z$Ga5Nvae1DT9>Efoe9LgZ3qTzt0#hcUx!q0@{!cT3UVs}>EivW70Xm2+f*i-<`Vpt zrDZpr+xTtBzxk!4+R_1se>p3i_u13M<@Wk!Zm5`d+jm#)vm3DW$FJXayuxAM7sh5N z27~^`=HmfOdLAa8uI)3xOW=}WB5C=~*50_uWjK~g8Kl1sMb#M^nzY*^fUC^diWTkh zIb2-}XP#P;lQAG_^zJzPmOZLr3zsNBvz(Wn^FWxGU0&1VqDfB>2a6n9|7bYVp%DpcE(-ATTHal>yJvpoOj!K z9~;7g{z&JMw@l*!uBaQXf6?08tK3+xX ziq!gx=0R)2B{}dwVpPK!F-0gBSn{{qvSX1MQ3Cn!&Te__LR>r4%4lL1TH_i%HwG+k~n?3V>qQ< zpI)70-QL~dK}FdGhUgFkO=AZQvW=e-$tqqEuUOO*cub{KKq-Prw(xl$Dv~1}bN??4 zNxb!~l|wsuy9J4NWwO!cyFH_9@33pTZ42xOM{L;ET(!*?l&_6+t~?r^{)+=E*}^%{ zb8br$cpNblq*z&0R`%U;*W4CH)HEdIL!FS|inQ(ihsLv32ixibs4+7qhe#lk@UV*P zmDO&Yzn?XI{dEd$xt?&)CLy*71#eBOWRpThs$u&7CG%y~gJDYqQ@zjEwR#b#3^~T6 zA+(6UoD`CddJQ*V!CF3CX@5(qs+v-C-;~liI;y+!wYL=6p0@FpD!$Arjso%|c*F`_N#fa4Agn$r%CA3QK-EI9Fb9V@L7YGEnS&6Wvhs6a+(g!5m_*`dE}0MmaVxq zRpDIqjS#ujIeZLaL%svlvZLk?r#VEx5a4rjbkUy;e>WyUjH`ZC@+6KQbd)7(rvWM* zur7>VD}qv18D_h;rv{cS=N?33Epz(5Jma@qhB{x7i#h%)$KYBp~U zZBpnV`%mBGnuB#2SoRbNKhHZheqVDyP6}^M3Mqzy@Fr-d2=hmcwPG#*(kRYR!+K>B z;ZtKV}5F4KTk%fXjJ zwTbmvdszbC2!oAvKd(|pIO5^SYKHb{vR2Z2 z_l0HepHq|ph+QLFv+akIN?;NZ#fu^OKx+AWw^&~ceL4}Db$dptA-vbH-(Mu9TXs4r zQ$q(l{2el44nJZ@UiVd#?3=9Q_1=1e-CM{=LoOx)(W}q%-FhU*#SA`TL{W_2O&;$O z-37O0;;|g#H4m+%DN^CXjv&tj&3fH8`uP-^&6#xSar@&o;Eg;~OP6pF+PB`L4wtuE z)$9|iXI$x3Smd}d`{RS-h0^DPFt-YG)YaN->r`JZG>BZohLQ|BpA+?*JFv;3>}ABG zxaic(>sNvH&yM(goHeMykx)cW&Uo_*Q3E>v2(b3Mi46c+11T!|d` z{4}@QwL3YxAGKX%z(7R)v}j%IO^ld-R9PKF&TjLLBc3Udn~b3MzI?vx)KU33ktXZ&wWL*%nDZ%&(e9RBOj&(AZV9B-NX!mneZtez|+{&Y;U? zWqe%OT@bmKVuiccBM1vmJk?`Z`&_x6te$24t^QVQ%YMFij}=hY#tNiGfh&GDEOy^H zEF#?Pp`VuQ+B3#;-sepPX4lk$TNMdGq-~XHX<6uj{d zHT&(CSK^#*ymM7m+j}7 zzRXvrxV27`;`LVWfo3CqSo9Wfx~*4;g4h1I4Ilj4gMug@?E)0QFm)wRHLvtWm9dKP zbnY(3oKmxF@BhyW@c1`zt&sPP-%Wk88g&90yVaed|A)yRX*=&5x|BU0CNm*`ajfUt z86cFCL053WX?Z!Rtc_>;&ji#@64;>^!WtNz9cC%5wbh^d$E z-tk@BwY%2OL_srm<#E28Yczxe`pj`Xpn0;~C+)>q#bf4c|JDw$`UWXuO&hAY;%mkfJHK$)Z-g)?A4outnza5Cf0p|$P z*zUhd^c(4x&t;j@BSp`?sQXaLI`Z_w=GoWlS@N5Db(3w;2xo~TcyHI>v1mWYrZ{AO zr$pHtkl(g+XjopZ25kyts*Y8E$ZLTTau5sBd6?3)K>EFVe0f#^Il9C<#u06k+h@zQ4kF{L1+Q&1a2n&(Xpr4j^@C3%hQUl>G{6Eg#mG*K0}Je)-Iz5JU*xz|q|-Mz zAnftTTZcO8g*+Nn1DsT7>E4vU_OLxF<)xzs4O}AvH!mJ1bM-o=YKDEv`K#`!jrDEf zRg-JD8ftB&ke7;r^=i?_pV`i_p0>EH!sK2eA|X0iCBf2CRKawqMaOC;kOnc|4|G+R zo6q*5Hb(oV&AX)llQ!bF)zJTh7Mx|su3C&={M7vF^;5cRVe7vl9@c}{lk1sT3PowA zoc{1#?))%YLW`helf3xb+FVYw%^py^ZyT0W{{Y*U2D^?B;2|;XiU(041xzmMDwOn) zjIigXBN>v@x)r%q_Z3G}I$pfh)q8aSr8OKC^sXNf*;D;?^#Gh~w1Uhw&K^KNAASZ7 z8Wf!+uRSI%qffKBRl&9C@?35CzwIpMuDDQqAIHnHy~*lC66E0c;+B{bl99Z3rRdT_ zw@5K#xdLGqY&S}Y5&yhP!J5WbDooHGdPLhItE|r+_DjfxdWwACO67 zCmo|*!g|IIh|dW3tc;5B3y|jW+4?e?{Tp_^aNrLitcKad$8ph(3nP^SJe$9Cq9q^= zvBhN*0!x=V%^~QbAOBe4tQ>r$$Tb}zelHZKhL*CVQ}noHKQ0sHc`0KPbF4=phwb(O zP}?GK^zHv!zD9#vGtZn~NNS~mMTYfqL!&BwS`)oF#hKlE)@u3gYJ0&%es~6VYQgHq;fNha`2C-y#F>yak5})5N4XIpExeq=8yY?=#c8upPeZoOBmu|F=OH-k81mII&Eo1P|Lks z=xzAK*mfZDm4$`k;xw}{JW~);bFTHq7gLO7{q*n)u0td1W5v^xr;;#G>kdaA3pDy5 zMoZQX00qJJtNr9YMXiL2fFl>gxSOulCN>g{p@es~;>~!s25WLpl{NZ(AdV~34_%XK z&tnas$vu`zrU^z2lh{%sWIFgrKq-L>1&XV+;KHXQ8tWf?f%W^c!WD?o+CWilan#34 zNIE)u27E(;o|4n&CW{YLk{$i?uO2G}Z9lodEtY?Ey`6kp$U(cNn80aJysZ69+ohwq z=g&Qp9J)%gCOg}1A(=dblLF|*-&oHb$8`)A1eQlaIbCQT(h87R(b8qr6+0DnVSg_^ zF0*~w8JBKT?^#CtmYOztj1iKA{5D=6Zuu2HtJrN9#)$-_8#v^*SmxL)?r`AgX6-;p zL^lU!x7si)mh<_jYzJM#nmq0mm5S~|2HJlpUBewQyp;`e2m$&htdyXF^Nb@G0_R+Z zPhGLV&nrZ|8JCw`m9^Is13+HZC8DY&?QIq5lU z1=}&5)F19ADiVb=4=OkciN3HIJxZ?Q{OpN@St<7GvrTRECnxr(Kzx1mh?ll#_Qqb* zr~dZvFcjfO25l$c(<+xwUs%_{U@Y@jKq1*=e`6uXZx;c1UEnRLWN0WNxcqOE2mExY zsDSUIFwzh2>qEC;Q{{7e8}bgXZ~r+Bv$iHxAXY^i#eBKL6)Hsx-c^1@xfEfi){R=9 zVtO~NrbgQw&G-S|PYrJIIbw3p(_(TIu7gIh>cET3tRs#Jr!|(eKAKCy0&Ps6?GS$f z5^|stnd%9;VX+?BS-UNinu>EPoYhmX}yWcg-vn>PcExqSH7Okq#Y~chWy~}Jw zn0&QXPB}M*G_v_t=$=q=-N$u(7hU$Pt9h(p{QHy-w<<-nY%+yS#Yk& z13Gzqe9kAPcZly{(n*x<*qFo8JP-ozIM>%7DPg|Ri0&u1%q~@j6l~_Hr9S3f{e&J0 z+FVfYW#t+Dx?!NOj|EF0V6i>L)Y=qz{K<{rMo0g=%9(2TWVAQCK}DCRl~bG0Sfy|0 zuMxs3ETl+Qw~9Oc!mCu-o~kt={2tqo(4dEeZp^?~fuA1onzG56BGYXPdxYb4}Us}Gl#aX)gexLE5opXm5|4~+^+r9?m$V_iswr4dA-lq zkiqhbPb0slq<4zwxqL#B<}B?OC&O#4x^l%K&rs!fxOUo%lFi9uv=3^Yz`1Lr0c`yf zUkh;>KUt!o6f{lhobcnY5Ua(cq>^%GCygYth!InJ4ITC0(Mz6tC=c&4pNODoM3KL5 zM>2RIYqJ^jC|2sX@TZz28oCu9<@xISX@HgO-9t$km_sw6*kobATK%Lfq4(0t1l{vn z%2ck%$)2=H3$6hoMHprvbxV6r3Vb#}d5$C|mbe-3gQPL8o-YEyH)rmf+^aqMT?a$D zDpNVZaMP4y8i9R?V(&z7jeC~Z+MuE=HF@#rKCZ~>(>wi1XC7B{M^C|einsLswStNQ zaft1&PxcS}Tz60z66fMnhi;t{ZTLrxwX7*pm!g{vwv7h-wuufK3t zpK{*B?0)Og_&c;1s2kw)@NRDHzld+Eow$ zD%Dl45hd1*IoN&9hf>d{gy1n0}qc; zy$jn+4o(XZ8NUn2A;+O_9mFPleT~e#9M+xPh~-HHZn{Qk$;Hzn9w;C(-K(q}ZWZj@ zm~4?=P%-$JDstF`H)V(ut~=t2Tu*PhPRe$2-VGr|+wRP6Z*#tvLpBQ(vA_HrCejBU zH}lgP-4t}^Hv?_<>U;y9{EU+CrhSTyhea?g*WEy&iB9VgzD-9RV!|BNfX9|cQoNuQ z8AVI!QIB#mt6}9Nf5f8XM=#rLvzVzuPA7GU))AHW4?4IVyArn|V@qwij!$^^XFYv! z@|zQ(T?lkj(IdTlX+ZcvGyH=o$J=#p@W;?lTpa>bIX zgrE^Y*i=Qd4K1DElBN!3p14JH4wCJ>W7?`G1Fv&~QiyRI{8}QRTCA}SI||#+b7?{9 z3)De$JihlA=v|@^BEw8fWJU*rtAcLLOOvx?!F3q>!O_jDQ|O%5KBXcA&6Yy#=c5;f z)U|OcEfm)9u-Y!VD#TbqBxCK-t_^~UaKfJ*3XFB>!-Ka{=txluoIPf?E72&^kRm4pK zkFq>^3x;NyK4roFmm6TP{KYk>4&M0dg|rIIJLF(@<@PuJ&!|2yNT3uieDy(L5%yN$ zXc@ubQA5il4?q8|T(_k1U!C$ykXp^DuQ1Z>F)1jY4XL_!v{t$%LMsJzn;44LC*DJd zxi--+6oiDJkPJ^Q$@f5;Xkz|UI0BtJ`%e(LcMwGf$gw_p=?=7U)Q{-zSW7f{B7hh;zi>T(?t z?JD#JGn;egn0G3xAk`q8N>{o63&*9jTA;`!e0yM;zPsf}H*YQOJ5~}yg?NoR%0_g z@99wX>7^a_Z^rkKFYW=e?_ViHR6{>phEC?{*;5Ueh9U`Ktmj%b@p*Yzp?4CqLQ2__ zSQURDEIkv7_UrkJ1)zVg-(P{+a2N%fbx~6-tJZuAB*tRjj2E{lM3RbfoV|cG4N^Ax z;`CX}W~Ujk6dPBD^Fyf1-aZGz)kYVt@st>JAffU=vZwf&JV|(_Ax)+yS?Uio!;36j zfdNeB#zfv|t}Q(r^3nGd(F4+Y-B`=t>_e?aMK{Rn)c8E-HWcYl*WjTaSfcW_me80* zQUyY~fIH!by_SJHY&V#$b!o`UMNRaaG=w$5E}BS8TsmK!=)KBGM|0QOF63p^Aj_1z z3K9|%5^j0W+-?NAGI$Pd>oOmdIv5M-4dG@Lq>$_kcaw0)Mh;!UUhiQ4ZZ8wzz-YoX zh{5nd%~G+Zqt)tRb3lQG3&7v( zqs*-#0^Cr2L{L{&Z>|Zp1rGJQ&5DfAk-hH;*(bXh?}WHY4ec%FwVejAN5SR!f7d6g zsdpB+1^T@w`Pxu}SU;kH?*@3|t~qqT{rRSaU#wA5wZG~7pjz?*PiYgtFj)lICnm|F zfJyerDcr8^URawt?rjWl-4I8X`5tV_WceawVu@!QxH){s3dRYmw8%*qj&vnlni<}v zScs~it~Tf#P2_Z0PV{~)Refy@l&hcFsLaK*RAYs7PhJQJ{VGr>$mmqSLG#ah@4Yla zIubd{#-+1oIvT5a;wZwdL+9jcRHa!=#9vMl+_g#b$9k%Uz!1~}eX_2W`j?OCq3gm| zqSmZ&DWWL=vo!SS#byuX-A>d>w-$)_0~<{E#fj5r-M8YAE1vWWJP$o)H%x)@FI<*C z6t#x?Bsz#N!`E!cR&>g#IHzsoQ{>yEfs}x+c;x$)gcCGJ_faI>R24`}F_0ai$S7YL za~Icx+owP_T6eU82MOh{7g8T$hl_kgw9bN+(E<1|M@jCQ$o@<6FDG3FRyy)vyeGm@ z^!xBkyc;uyV6VJMrbHbZWKE&^_mG!YUGa6S=e!}kM)*Szzlaz?>lMO&@i1muC z?5TD{p5M!UU1P!U#7Kog)4yDE9HV$igoPYxW9nNLh0ro2k!N@17DS87_u4m?o&OSd z91bhb4=87Xc&?+xc@m1Fwwd1o2hpGN=8j@4 zP@>LADxZz!7Z@Ggv>J|#;`rDkJ8vjUfbeANP|`2+dg7hZ(Op{D%{Lr(k4O0;bBna= z+ipyA)Ha^Szs(+7BXh*+cxAfI;^<+yUNtT-5KJgqfDUzl$@rJI28L38LnT10KbmfN zaL&^Vjdt$T+%?=Pkrnld%YHc-UoW|fup!IkL4TQ?`uSnbc1>wGz2-KVd$Rc6vE#T+O)M;#-`3+w_;+i5s{vukWV{m$VeC(sh}4fIrAXi0cDT# z0wqGjUxX&4WGKu%Da?Y34HjOsYiFk$hpOP#Da*T!3D_v&OHeD*+2jsLTH;%fWrl@t zEW!m}3vJhp@g?#u;4D4Spu+sk+BM>+n@7@bFGv%4#6yN{x+{f6c&Pv$z%fgvaM{MD zc4TsAMM&d9l-sk1&wqYd#{utsf9z@8_N={9v@in6q-c3kjh%@V>^>kPY+H>%w!W4R zXB*@lvT2NeY-azR?nxm?hELIVf@6?39NPpOc5fS%Ho$Enz5s-AY?pyEum>6y zrWi2u#2GEf*{G+AoUEDRtmj2HoGa*+S5zR&kH*PBZ!kUFQAYDb`d!KDz5^JnTw>r; zQ`7^~fD^G>?d`t6JoT$p*{Ehhx_^ZC_qVeB8={7AkZ z@$JCBswvun^`|7Z9f)9rs4v?H|Bg#Z+BtJ&B>kca;3kvXQ}S%UfWJ!2EnFGt1jh%p zJ|T303Ae(^N6Q&&BF5y31?B1ZXOD0@IUq|=>ynsPj;ZuN1_@+;EN@f>T7!^hwGr06 z4XAR$XKOIcVOX&Y)j*bxi@vnzYx3@6xtwa%KWu<(9i5l^b>(!%<^5vO}41J1w0cW0YoV6}tYDq~lv(lct`hQm!#)A{fjIcwCwH$fN6{*Lr5 zph3AsD)1oKK#rgkiaI45F!+&h3b_8lwa!;LD3bg)?g@lZd#bO21yZs4x) z1B`V_;0ED3ssYLiOe}SG#J^4)%D}#l$v|wSo5$j`=sm{7Wz!C_8u}uUDl*k(oWdtn z73hf{?7I%tk*w;7Yae=mvcLC%Z`RME$i-K5gDL6y^8@6J{o*-z*rM11KxgMp7HS?# z=uSv0Ej6g{VSk?3Ow%+|xn(PiSowbJ`LE-&gJ2s<+#NkFjW9UULz%OezaqI7=c+%c zxZ)an4K8i0PgCT;7NiUW_TMbqP%o6OS6M3)e>@Nve80UOgq+>I!L%zPxvgaZ%^mos z#vKw5HpN$K7$vs)F*9l?LOTuD2?cizuHO#%*p9wA!ZzfK&Ke)$aP|cybct&5d{SX@ zqte;-t6T>##j!7&W}Dk_Q@voEN;8XOl0Gt=VKU>p-urL z(Z~e;SB@|eh@k0t@f}q@SqInh5rw}=-kUxJ>i=JB%Pu^)?Rg+Q?^rwTPo{ zgovasqE|vHdMFJ5YdU(2e!^igxI}&l5HkL_MQ<#oyQ`lGF~e`t>AraT7V=h*`KV74 zM;YJ8Nzkkt2v8z$%Ev|f4lLSV2w}Iq34cxHzA0BTJfmKtJKwliI{=;Sm<2~a&aI$r zMo#*mL^1^~01s`h%x+ZnvikmHaXJ93bmZ#5Eax;gla=ugk62ed7*wb9hg|L#Q}3Z{ zU^zlKm=X^zE+{59CNE8`(ZH{VHai*bKxy30Uwt+shF3pD*VjOn8~Q^#sN^vNxtOPe z1W6Z4Ur{RLXJGS+z1J<+*jVk0ZI+T9(o87IWXMjy=*n-V?~^m!HFs^&4q>CJ4eLpM zCMI-T%45`Mf|N46N4fmig)AuC?x-jmf1qbfFhoJFK~Ym)9p>G(o>XK8pT3bZiwG%g z+%Fc*YHv5Mi}V(!72Id2vQFb^#eKzedJrandi}|nruP^H+RP%7=$^8CG-st%3KUWQ znDm$!gO4;<8B>Ox)HQ0TB&UWQ(`D$=tV=rte0`ujzJRc0hH~z4?5IK}xKnxV%gd{P zPcBP=cY^H78w`i)gJLwzrpW{EpeYESoz?}*64(Hdk14prytj4P*tDjg9yhRgDbtvu zQ0oYBr5iSGTzm?WeX653_4aE>vx*@Io$(B~W41Nqnv;M$sEZt?CPoqKP@PBK+rFfV zm@T<1XwB$^l@Xz6iwuTes$eL8R36eiShH0B=)7s%)5)2@6UpWPcQAP;k&q>i2pTy2 zCUL=BgK1K}`uCfKLu%GyiJQ>Ok_v7qf<9y)e=Dg33WEBD=@u$v;m-U~3=+$k`UG#v z>w-fbYhXNb6puhKhx8nRJ)o4NLcp1cdOQwH@35w$kI^2jR z@ht}6d@b47Q_WH08<-bU$B@b;BYEjk*bbmjzt2!2*-lZuA(`c%r78Xr7p|;lCha+s z*qw{u`W2WPr*fU99{4pQoim&$2aoLRWc3AH&Xw;MMh-dqcu{iX`uvi!j9b8Ie`sLQ z(nMUfshYRm^;u*F!!1eQh6|TRPweE0Zy8_#r8^C&V2fr6IvGFnW;W2>2TQ2Jm0}JV z22eT|bs36hRFsRcsrJlaQqiY%8ER3w=%bz!wsL}b%Ku@|$2qd*w|!{8Hg+kRQLyjB zHWZS~?K>Iph9YGOx6O1};sSWDp_pU+Hx?MSe^`*S2%aeSkZo z%$QdH-4fG$ZL9yVTa!cu2StTh5|~JMWh}08Q%LqIzuKCNEh8(8UeS|r970v2@PbVXDL+W(sWAA{k|Ds7}-A=s5!H+ zPsPlwK1OorNlF?ZPVLGSp=Q@dJJ&1b{Jn?h=;+HfNxa9TK>Sf+VV45)I;e{u?

k(LqI47=D%sZ$~YaVeA(;wR&iJTtdo?+~P-;%%U}tVv6gq$L zU@E8CspsNvul-w0GACc6!7?E(3+O+$@F2a;#WYqqz)9O0`{57sRu+GE5w`}g5@g$s z5C~7TB6>6b+NskP0%C>uZt5wVOU{t8lq0-r?QdB>jz_cB4ux2Q@0UB}#eDtD(qmubv*aDfuOkl%rdebfT>?{>=6&;(WusqaJ2>e z9{rNk#&j%=%+g#zEDCMEIA$Dg>3w*gZ}KQ~?@|4sm!MNR?SdAep8V~Hh!O`wqx!-W z9g=XrFx`l|ZJ-uU#h98>5y{KPsrMpv748C}99=t@JJTaKs{$r!xqy2j^f{RI8O)h@bl``|K)K;VWjcw)#YmN3sSdE4??=o#O3q@McDT zFn`NFfr8f%d`-uBfERU1DaJnDk-x`4 zv?vDjAf>jY1A0)wXzJXgk2!)hI^$T8Lpk*RGYPSRPVwTdykLnCvfOTJ?bM+o3BLl+ z^N~Hv9GurQ2ot_UlA8*XEb;O9<^`GqSIqX$-G|s|k26pOF2DJ`{r+xu?$u;ZRRj8b zgi<2zRnyF4L`tq2))go;;xo;${pbf13{mKv%^eBCm)b8~__1-5f@~OL0cIH=1s`Wm z_~(SvMB)BPgc~yMXe;~jdOhwO`A!KKCv~zD1sWZT*tO9EHg9@#S19*2E%y zHDi!)mkfa9e0bTv*T=*kOnFx7IB&-@T)P+gJ9^Ry~}A7WkL~Fjq^SOGh62C87Hr1?^<8{wqmwf;nbLY9E*~vwrHeH^Bln-Klh`q*KkYk064jc`CZUyvN({ zs~HTQF522A@ANjkbb9tE-|SM}mU)V^j`2^1{PKdNxQQZ&>x(}$xqK*A@>-wGv^~&^T4$SA#-{pzIZ(R%#REV|u-6lM} zkoA!p#%xP5c> z>iDeVGHeTTa}NYOg0pZEc%MoRcG0Puast?e3snFKyQZ0H7wF6M{iqdYt^kRG=qhK& z6k-(m#_ls+@>>z|{h6S2$eDzU52K}8?#uOO*ZHVNK(xdeyMO~rOF-U^WDk2yGQ&iM zKMyy0z2R3WMZ}=IozqEEdJyTaw+;LMhIB5Oh6ubzlY^XTLuzeI3433H;zTt+s^2N^ ze7*`$-?pvC#y?9UU)|fB`MC4)WJwf7d%Xg<_2kq4i9*FX0Ij@8X&nfoq1OaqRHnWT zm&}WW#ev*3fu}O=krdTiQfaMD=twP2xGvvS@Lq@lJ8DH3q!1Tbu7g!zD2#>ho)Z4u zkPa@?K40C*fxPT}l=qN)X`3THJOZTV?XjPYl_~Zvh^fmX40Ia}3G+N+IKVZ|p2el7 z5d1e^>bL%CsHAhQNUJ%i^){{PTr1(7cAG7(=SCKBX*ZnuVTN1iUyX2Zf2vsk!;J>wcdqRI&EN47KuUu&y(A@7kh*g=J8gna_zV8%P}(!x)Nmy#&NOa{ zJu2eNRN=9-zr${#FiW-9&mD0^qd~9X&qprVC_)^!9Uejsq8nN4K;Y@R7!!as6lSoH zlrD)I1Y)V;@MDwL3Z47qC-F3hiv`{LkCx4X5K>Nw#Gz1oN zmKe*Q-Wcktp}Su84=^`kXU2|yu09~7&b|!PBfpiH&*|6z%FMTQ=-BRC$4>MXl+ zy8gyigs?4rRn&+c$%Tj#8$jd5#tC5dQ?YKa zSXbuws5j5w|A{`yyHoL}arMKE)>=0!ASy=Q@BbRDb8G#XT`E`AFtM9U*GS50XK$&_ z>+yY(W%S@JtDPDK0QtbE+>8_5P7egq={`DG_SQrcI5$D_(p}V-n$-JBLoY{86qu%W zzqzH&ywV4nlh7;2Z^#=k#DoE>(f7(ZPSxs$7^yY%3P0#84@i@_vx!@xSp#|DmLJyy zdFb?tk?j?KxGI_hI~&i92K}mKVXbb^f=ATm;cHDEd>AI^wwE$sWHGPdYKOFJgLSJ% zowY-i{>F}toNrv5nXHR@>A0%G7CNzw|0K|lS97|({}~N3c91Z{T>_y(4G(ofG69!xtC;X~rb@pB%14`u%Wz(YzZmYWE@&BfMhHziGbRqq271$6} z#HrTe3G(*MKK@nH{MuLksmP`3p-W>P3QbYGXUB7>_ubIai!r|jgmr)YF(BZqAdHw= zmq41sDzLIq`ozAME_ATozy(|Fr=9jj zV0aUqVsG}O78t}G{Zm3yHJW1G;5@a<+d7vh*M`dkt}ZwC{ePZTfMTr*&(094tWJ8N zNS#0?dUQa60(FH9`cG>6^M@1L%MX4luN32BEqd;VR=b~Ja4PX0QKrt>fD~Y5X+Q}G ztXj(yH+%A0gfpznc-=qmtXWHL7_qP>jIboNfNV4+;bG;AV}<`rDQ4YdVffkeYX!vj z`Bl`|^gNWS0X#lOT}eYltNcs*Fn4mvaj%ame=5}TvPF&_z&9=hM^>v{kILwf9lQz zlnw_PVm2W#7iYHyMpPZ7bQuIcuYRo^YMzeCkos|~{S2L%V@Sih@T!G~+H=bUIr1DGMtoi!mM`&m^<*0ft1;zFR75xU zgh34dEc9pISde{t=2i@Wkzp@iP6E^;k?2GP*G;c}`nmdG48?Zk+R9+k_9_?^?iv`C z4_wz}cVX39`>yV*e96WB4)ffJW9x%I4>wZ-nEiJjwtN5#@NqFjC&UVrB{;1H2Pa{; z)nRTg;89s}KV+=eAGzP)Z7M$8(`e}gEfoOqkw?JQVz;o$s+b?YRyGlaV2ARFtR^2g z;YvO|StsO|X}aR+x94rnT)Hp8T!vq2YCL}SvgEZ$-RQueIweqv8Ca=pek@q35DFcD zdaZB(nQ|ig&J&ViMcA#mDp0p3mrISkU4>w#uq(YuDst14>boQ*`bp@kop`_NN3d*f zMU#{9DWH0c85*jWh>p(2Pf<=tUY{{u%de;|Lchxa&Ly7c%r(G{HqKV!h2=T+)57pQq z(nR)ocp@mVC9I3Ow^}DUD8AjuEXn=9#}XTa*%)^r)cSHuqAgM4#m){Zi&IV8vw<}>+Sq9g4#7sqGt@dL~ zVL(AvQU;@;(is>U0F3EnI6z96D(Qt>N!8?VP(pck z8EvVXi|Yt-GKTXUeJlzicH7vz1<7~Fy*}c&ATJ>S78ceK#LVMYNTsNctM}K=PhY=9 zOD60%O^FP`o`X!5v^j#siFEP^rYoVcKZ7Y*z^#_6L5>Crp%HVZdW&f?P>B6$^2RO< z+D=qd1xHyJz_sxD)wNY)YHv{JcBlDMtHs^3MsXWyFi7*IB1*ke2b6YKpvhXZ*%27} zD0i+5Mkf5`E2U4+{5gny-6y#%0do2W-W@~FAlnh8<;{6WlW~)lEXHV^Y5nN?MM&p& zJo^M$2-_S$0aAJkYo$~1j_=wfF#1`8mH^u3=^{-EDCRF8p;3V<28WixJhY|5czqzb zFF@_2#nTyjAafP^yO`AW48mc#Q74@-`-Qaw@3O|XUB3+?4wZ&Jx+5Zqn|;CFLE*A_ z#-+%6kPUgT_?Qt?&9`_JY;peqPWI8p)T;7PKQiB-rD9Dc0c)3=kqYSEECt)}ggJ=u zJQQ(!H%aa(bSV}tDOyDj+KZ~k7h#4gBY)^5>+qv$U)}+V!Y%ch{@?0wtaFWTDpHI{;y%djp{x;p`xsRidh}7s( z<5SD*#jW+reeMH)+DaQs4i;Ch#Y#+G5p(6pjLP0u@?(EFHKhIwZ+yX01y=>jXt@qV3p-OsfK0l1BK=hhb7Gmgvdw4MIYX7IQ!%`wY+H_A9Hf#lclC%1( z-PnE!s+L<5)FravWEqT`WJRxKM=7Q_IS|+E#J`|VexLFcK{usJCd@l8QO)cV&l|Yg zu3f*yN4Eb!gapb>C{9x#;=hsLGz-4gKefY8IfYU*boNZFai9!?Cl^25*Bg9!%r3IU zDno9W8L9k7bk?n~T!t8m9cZE^3vphVN7p6b?y?Muv^?9{>2X~96Nc`*qZ`g*+{%XT z3Apq*x0n0)ijFNi`5kq7hK&`>;F|AJ9m`2|J7umR)IMYy<}#Owb;_xpOh}y5S42pw8xVJj0>RP9)TDxT0HlyVNmi-TZ zPOJ=itH&sg_?bC=tie$BWbtqJ7*YtZhb4u5EMi&R)tL0^7E6k-R?^s4EhC07_r1f- zgfHuHLG>yi!=*@aPB@pwa$l7}AUk}4!cRh@FOf&F#v zgT$$eQ!GClvr?-JFX5;(O^VCa@db0q4}o8Wtpg*!Q1lzmN9D?|V|s~SBP zh}~9&(uur?qr{#FAWo2i1`Lq?Kcy1F5x}CC~nnzn__apRKbYo+MjCv6Rr$548eZK^$ zcUV^TJVigocYMOw7P+QC=`>-K6%o3s?9lMA8TKy{iB_}s2F&$eB3uE$EHOppJK?Vp&wPakCr>jLD!l%=^`5KNJIAJyML5eEYLtY z2FEH}yG(7TP4VDg&XoJ^vonY24J!^Whg zF&`2C^%wNsi>f5*at*|gN}W5rmp+q~Qgh$ZloRUZo}FU4_Sj?n?kBmqU$Xk; zGcNSdTM^niQHmEg;yQ1bu%aPsf$>0a_0-AkS0jFyq2CYTqjwh8W2Dd@RQ(G&AACiP z^l*v>WOm~p*Eq#(rgXW#9|VJ;kW8xpZj=i- z^j3T!;#A{{ZjjGF^ofi+ay3xzA?T%{xl4Yxc0K{O6RpVCIvdF7e*@+Jke6GRh0+o^ z>uATw05kQk=Y?W;dS*9ZGamTDs68rPE0^yu$i%f*bH6KjUFLud`@*7TooLhx4(0xn zbRCyZSRWbIi4-wcSDdDenQ!#Hip(Ax0LT6ia3Z>i&6fkkT%tswcC!cWpP?+*C-IEh zodLWpaB8UA?-*~WLk^~*AUJ)ha3M<3Ho>#u;q=X_jlPmfT?nT#FD?9=_`$(IQw*v6 z^9Sx*^jYF>9VZcMensGQvi;eCiq=72(5+n@X6gcUgu@sZig05tolc&FG-v z(W& zIl-LNB=;a@g9bZPA@c5qRy8Ts?;641XwP1I=tKXUw-3P;Y?Hvwt;bSZx+6aYTdS8? z56oko*4jK5;z>5e!r&9!t;|Rk;<5h+Vt}pi9PGmC!fW|os4EcNt`HJ%0xREYBpphj zeu#OhXhtk^Jy1jeVu#PCzHGl$ApLfLh}dSsL*lQxG#y37V8hbTz;`VoO@8RMf(^O9 zFwAYS;zu9i@f}@1_SZTR*<6{i*#`*D^LZ?~KsKP)IC{3{WG-HU^1M`(P3?Zh(f=O3 z)!lSxrw=(9DU$oQ)$G%u6q&J&V-|f)_p93Q8G*ckd@ENW!S1J{j$7gP)3}Ii@ce;$ zdIv=6o{jQ^7@fA!LY!m*zVq!m|2& zyUHyAL5GG4p~JE~*FF~wPUt6QiXN#8AD^=3goST13@oU1=F#QUo3CE0-_U{Bnde<> z`}l|R)uEWtaOI`XGcFI=#j>Cf#ll4FW`fgb@P<3wKPyr;E%M`yTT?|QTV0KXCp(@} zX_3V8q$oP_Z);>8+b~G)%Vf+a*TQ&z{xm3rn(hHMF9Am%e5l0;J{ou|TVHch7w29X zv|c1l4XJ(g?BhQmd()Ewz4RH53qXQEfNK&UDLrmAg}Mn}rK_lqEt`}vD`M%TRPo=n zZHPkhjha1i7*?>>WlaoLP3{IQwB!i)q&VbNt1_M$gF{nhx7E@Yy6 zaErxnX7n0S2rSP#JU}HasM850XK{xpM8*!gsff|gBB`sUHic;)yA`Wg!{qQ%An*qF8v%Dz1% zhY90b)ai2?EK?yCfjs4M%06=zZRvtp5L63CqvDh?-5-lQ1I_cB=)??v>)NtIb5CWn zdJk=6Oi6pat_*gf6Lh6!eIgw2CmvEBIHI36j)D~n4&?PFc-Bdctrn#lCXC`J5~)r;>x>&)YB?T{)H z<^JjzuDqO07i@!ns*5L$@gPo!RCsmFn_ow}w9!uvU6rD!mz?|$P=&(oaQ6n!#xX(J zVs;2wIn3$lQS+I`EVrgAjFEx|*@BxkE2@&Z3(0Tbf_Y(vMVBu60aV8EbPs==t&Jg_ zdhO@*nJ6)exLxgXdGDwt)l5f<9&VQ#eIJokPB&$H`Kg!VIq0j{DicbPhO=W(#lRFd zh+*g9cj}=Z=u2JBzt{&D#^mu>%CDe-&psY4)Mm2k$%j0*pKQf2V)HKIl8L7LASg9{ zFA;mBh>5Xvj-#bZ|I*=AHSd329S+zu#mK7In#6zIQb9H%@0?pUYxgnjRdJ2`GLTpotqtH4^~MsiBaM)%01MIbld1kATO0UHhC;r<-8fc8kUy(Mlk zf26Zo7xL#5a*vo zC=6hnKA-Bd9`5sBxeh7Dzfm*DG*`?p`b_1aAEeC4&~nj@1{$}hxKNJA|p>0ozX{?@V1XS5YA)8dp&Zq{Aw8!x>zVgi_Gmx zw?8X{6wWfrsd==CHQvy#9Nkg=3^^D1g%B;{C);UJw29dM>++Q0K}~NMfJn)! zJ$S1QCyQ1L3z?OX6=S8`&H}Tt%~F1n32LN?YhX825t?9w1_zK!yb+H&oIjK<0J|-7GAK5FaOqp}p}C@U7XqDLuTnjReS*J83!3uxG?Juc#8C_X z5ZKl!nwi?V2RXSQjdDm(is*l~RH|#}1&&=2HDnOZ-9D`z&i#G}`6jm*B<_=y`bgfo z3K=n_jTCB%`Qd^9`RFK28&oIQ2&~>P0JiSg8;lyJH_RLq=p0E2v*Yj21Q{?U&=h*G z=oY=z;U6~}W(ODxqVtGOeX7Ja(AazVI9m*9V$wocLJt|-h^8qF^zp+z2Mm>)sar*m zGFwe?TnrHljf=zA!W^_28hNVa{y7I-kF@E)hpSMJ3n)u3Er3b$xkXq1R$W_JFVX1! z)lA6;+>$~?ci+msIBS}Me<4ksIr2c7lko9TkCaX9i(@Ho>^PofbuRavj{j9-cBqK| zq(sB#${k&iP?D$D;>i$1xik3|sr<;n0Y3lB7q4VW8pE?6Ceua5&6b>p^6y)v;0MnZ zEq7u1v)0Qa7GHn!9t^o2G>JdK_M5Vd7AqH{vEt=wkPh;PVTM+V|N0!zp=og{4Udqz zxg3DcwdjzMr>#x`CU(ruEu<^2Suj%IV0wl5)naWT`S1i|`$pg~pg}3j7yzdb7;)nNbV_}XXj`d3(L<#BP7-x`0)=s7(hk*F_LG?=Yi$-{I?e`w=@~v zeH-NPb%IDqLZ^HUCkyGgT{6lbx~YtDAx3Z>Jh23LM%ERSZZ<8_rTlW$Ue+n{a$p}G zMY?^Rutr{oiX_NNd+EzJv2?%}ga3*kve~e=?Dl_$b9{1*-bmtZ@~VBK=tDf4KN*){p;f>qiROeoV2%TgObNlzz}T4bv$L#!tNz+`-?5bo{uDGCrIEA(RwJ+%6s6 z6um6v$FAYYSu|F)?ty3m7uDFF+<9Iz}1HEhxo#bh{}K!1T_6qrhpjo+Op~au<%BDkhC!O0EfpOnfYi zGIZTGu~RArnju$#ord#A`FL~y-8^$wC}efUCNhx>E$6Zc#2W4`_qnD4gc1hCLAqdk zyp*x#HwylA{-*BUKAs-X<^Li7HtU@H<#W~DPPz+G zdSG!qjt=Z*`jgc-vpO5r{&GrZ`9er*>L+q%KFVV=LOeF9jObHZ6dhd*-rB$gMaD7Z z7t=ah-LB$h7=0lm^}bt5M&~acKp7CEUq+P1OCNt?H2KsfErI!gwN>3{*hNHxXkSR` zVQvH&bX*>OeaI4MZWo}C>vSQ5y8#8(WZnYTH+JSNthXn63RtlY7l##za-)Bln>thh z+cgCQTH2*c+uH4OrK$b#JymYw=lXVJ=ZQmhIz8+&u(ohultKJELyrd;-YJ^>DiNGp z>Dkb6LN9aBK}dnj(yrHx3ux_m5Y^%T+gLG<5V^DCy;pMQ633JdJ=ut4mM4+?{d0|# zz4+UdfoEO)9pdvRN?*==cYz~$ImA=DXmTp~GIYT3edM88MMX{m7Xf&V)F4yZ5Y|=E zr|)i;OYPEFWS=TIk{m2KbzgvF;{J3C3i*M^hDHr!;X~nckL>!PXT#n`@8d2+MniNl z4TO|wUT`+;_^uNe)(0tND~}%J4Rp(oQWZGpY&A<`n?D;L29FV1pZD;|){^Dafm6 zgIO$-y({h0L7Rs+4-d!N=YDn>~+uW z;+JWd-3C(=5i*X1%#6 z^P}(SG(a|A(blFvqXv*vB(Y*H%_~~04dRqiINV1aH=^%1B z83#|mdJv3`u9lbhTXPQcs*mN1h)lX?b7&kLU^V`WI<&Pr8`#HUB1DipnZ!c_e-Am< z4eX({j|9Wg>vDuu)HFaqw>%X+fb_Zparehmz%K1}A4FBP7_y$Gx=Z-8Jd}>*Tk28Y zZAgd|{G6-|z^rW$Xg=901}$y+pN8K5+OPj!f7e-=kC_#XJrQB@V=oXJr-9OMd?Iva z6T#YEol%hYZc9Gd*Q$JZNN3Z)YheU-m>fCOAzo?$@h?uwo7VdPV+QyY&qFpL5^cHP zesTH8&ng(MoL_p4=x;W&%NcCytJD?__(TEOsHwh~IKw`6h1_})|DrB6GW+yL90#?G z68B2k)13y6O($z%-+!Ctema}{%&dXSGR2e9n6!*c+P9vJFK}fvJ77e59?kj!5Wg)y z>rd<}g0mD%QwWLU21eV+SvvHvjv-w6Ai!S?z>0rSU;){T+qk!1r`*84I_s6;byts! zX}*%un#*qF2vf<^mA}TWt6p~~^XJ~>gVqZ+5$#Hcez-zyFtFUduI9_n4cr~B-zaoJ zWLe!jap?0olvbCHc{vE1uk`6xLdy<)ozjRq=h2K%?I?;+2V^ zhZ%W-H2*cL@*jWv`nRr0i(Et-L+b=qgd=&LNL?$s+@AoC%l^?PlluK2fm{kvH>En{TP z4L@A+T^FTu5T^#pR-GV$x0?Dv5A%QY7}b7Dm<~cALtd~8zp%^N6@&o*2DTbw_rG`i zr}DGx_Gf18wkQ&4I3t7oQsmj=qEeeNb}!LG0$X2iG55n&7n%}!*Vd&DLjcVzg|NEN z2Pknf+GiIq{>Pp?eG*g<&QXor9OzePEz(^=xo_AH*Zz-z?`jv`hjMc`1Rfo)FB%7& zu{%`7lWLzAGo1Y&S1w88A6|mCgjgf`H0c|syzl7R6G0H0wN&(dd?Qh0Gj+>Atn~0c z_5(3=kOBMTaF?(m0y=}<#ihm$JGfMtKX-z>PaRr}m6KLXvm0gdrbCrFzU(ILPJfBzgIm6QE;u{clHaWd(B^*7W zX$r#pEU6Q}b;q4!&j_@jH!rZ*YgJDY%AMg5NCFxL1SOmJ_b8jzU5IEi^8#=Y8N4L{uN!?GwOo2jpu*ss)<$o zHQrnm3V1%c$Ccxkd)O-IC~0K>mt23SoxJLf$cH{a z6Hv?B-}x%Zu8bSlVulc6VyLPwIobY9(gOQqi};$RUBsmRw?5y=r#FE_FipzT&^p`G z)xKnzQCPwvb2IX&YFGBxiH z42L3!pa_dX_MRVMgbs%Vj>L?=SD=**(Q0NKV74w(*u0g=>F-KS4oZ_kU)F;IH_utwSh;2Lkdam8=%!`-b%5D&IV^AO2PKVgU-sLwAibD zKF6e(LIkxBd)O3y)FDVz zDC{LctuxRMOu@(T15hEd1j3D%`u@Q5I=f*6yc$plqvIdWZMxx&7<*Q=yJ-oc(}wV*i$#vC;s&i*PMb%^9-iG60GIS9=uPKRl|y{p<&XCB|Frm;s&l z>=IV$P0N@4A5cz7U-|Oc{_n4xy~obC-~@sM7dFlLTf0(lJ@7D|T?ZoC;vgEGa#jhI zu(l}ok=;4r1y1#|i`+S8i~peF<^LZ4ZS0|cdz)X%)d|18vp%S}? z$umr=pI@3&zjTf#LfoQ~p4yWxo2mXn#4C>>eg-e0l{ra{{c-mhDL0>d_4{)D32{M& zs)l+an?$WAL5(G)r6uM=rTr3XyJL_3KhLU-4Ee+Yu!0~HZYk+!m&vDri4&r(5_WTLgPzzc<=?ZR zRsu)OWOdj{nCrsT2%l|#O0Se;Q5WN`o7xvMw>Q%it9sT2jj3uY(#@C_Mbe?^Yz6QB zVgTSao}Eh}K^fAbBEib23@^N#e9T|{%OspTmxS+D+<#KS>jwHrH*Y9B z^dK2tUX6Uk+Eg@2V%ymSP|q)=Un2Z9TJiag-aq`5-q(~cplzBhT?&bjsWVoj&y*I$ z_DBN=lqaP2`JtxNsDi1a*8ql6c3729W$@)z9LEc4s{UB4onYK~M_u?*r2sg~v%PskW~$1{)i1&6cDTfZ==O~9BaIZw{;}pH@YSjxswpIX z2}SzTxQEs%=T`nWCC7K69;<0is{;zUUayoNs!NSx6w8BkPbn%rj`zQvclM#6(B#h7 zofvPOt}(wE2PQ^C z?GK*UzieI!qc_)$Fkv`9FaF9nIP8(A2=7h>hJbk+J+16daV_6Ba0@D8k@gdlp00xq zl$Kz=erW=vq-)8tdB2Fks2EIL6y^}i=iJD2k1R2nd^zy8a#IQn*dV5I$VUGR>cofE zQsoX@b^pX8n;!?XPDb7kFHGh4(m z%QR5iLR6`*|M}s{6Yp+yESr%2v$y98%gWyjL#^jg_n9leENFb>$Sg5cLQ1(S8z!y5 z8P5Ba3ss#OH$_|W2J|49Zh=%)sNm*Dcu z%2ax|DCQXJY7K;6cD@tN;S%}8zERSt>WW^&{-9t4jb>92w#wJ0?CiDAl|Jg$6?OWy z@XlrI)#PWr(_t{jAry4Elb;)_{-r_I|Ec3g9Ja^vDeNt&^aF#ONC_)Zg&4 zCZ3YfZ^$sy>2V#Z6uaO56g(2wW*n`wa1cIRRD;rAt-a ze-?0SCPr6p>Wx-5kF%}rSBf=#KO3r}U`mF1#e+5WL#)AcjmtZ8q@%h|a|3vh>Phk> z>v*BWc))6*FpV@{cb!HV^uRdnWfO_X!)ae z=Bww{Hd+aM!&ZFv?b~HH@x8;T$`9p+!d*3f#(;1Nl6GrT1dYTUCI)g}GxY6-4x9-j z{;YScUIxYqPMWgkmfTAEN?154z!wTq9C@9)GSh$DP*28*BG}I6CTproZP91^ptR(ygL|l(e)+gLIDu$&Ky?QBpvpySrm!GzfwM zqhT<*b2LNx_wo1dXM6A7-Me>p@9N>5PEU5TzagYuPyx1X8$-lQ8bQvR)90I*h{tXO zSdJ-QXDMk3d^QD6s76dRlh{A~Uy7wjjQ~Q*#{)WS{XW+J{k3S-S1oD)?x1V|7XUZT zjZ+R_LXUMTt<2?rT4(dwQ3ViShd)QMid5=*7fS9NG6BdAvT{-V#`3@YeELLRpX@_f z=IZHy3Pd0g5Wpac9~6YN=ltP4b0xDHuhNEt@MucMf(jwPPxU8}FV)NSVvD$*A+6tM zD(e@0x0eG%Mo`oG_CVc$2h`b~?JpxYu204tUnEl#=K)~2cD`P_V7rO!_fcgWuoAgY zahS11DjpX?L>7dBfdmf()J31%73^U~S)LL0t3X;>$DZeo0kD?Lj}svhf45(da+(pn*OY!YvoC$Qo5X(ggm^M z0RXRNUUEIFZ2zJAf;4mLn034Y_?Z;v{JYmbIHvbJ@_*z^yYEcfA|DbfDt|Vcg|E`d z{8;+?;QWf|Sm6w7>6q(y917rAdi=Ek5^UlD=8aMe={)<)BK+9tqhpnShY@;E%U|Li z^%UU4sb5D57x)w8feX7>fBg2eKe=5{u1++ zI&|c&lYt@I{!`ozR@w#*%{%@I|8x%N8RL`%zfZjU<)A2(>c8(P&nqUUs+I3n=MLeF z1|$5kwwnK7%JR%3;AMNymj7UPB_`Wax5l_znH-2&{%MMCjGdIx^13iOP36Y=z+qMF z%(pm!P*nv=6bNim4nOxrDcan^bNTgQFQ-~|uw&Jo5!scbwAi_ZA~EehItC4KmVJe> zi4pSG$&@pEBDwjBQ9lUm7uNY~aDQEWJA`UVRMP zGDFV%+AgVqwW19w6X?GP1f~Msm+!to9M*qQ3fYSc9A`fD9iq8k7sG__|AY2Egs~WA z`QXa~3D~lkUL#rJbAc;6Ss~bIPzP`^+83{U9aYt<^+rcG)d&sGL$9TGLTLQoCH&Je z%3uQ|qLm!}puYcf(x$*RdP_!l`^D!TKc_%TU-TmbA!E% zr94JaSfGsOAAOe`6nMXt>ko0p0!@8d7`~*IjRJNQp25`ZC^wSE@R#_G3?SEOm0_jwOE8yFg_u@8x+y()p(z_J{F#KZVXd`f@rLO@9~<{P(^Zj!v+KO8Smr% zF)OS|NI&!A$ zGG|>n8>tu#SJ_`D>m;&aie!e3j{oL8IS?q9~B zZ`}dYbB>a+wU)_PV_C9hkfLBF@(r~wzUg7~jXvnbPqmo+R-SI7$r`Jw4=DlgIqBFO|>ag3;+-!xBgsuaJ)m%FSLO}ba zd0tm{G4jn+FU)Uhf}RGC$^;j*l%y2`5k%JxC{PIX=m0d8PYm4;sJf*<3DfV?1`vX3 zeBemm@8zM{_f~aGEVM7FIX3wP;CcNhVuAgX$>(1tv#T&ZSqw542g(f7JFb-=P(@7O zNR;C8yj#_qzAT6Gp&W-~c8ABcwPe~8>=a0ZxtSTIOzlRj0LQ-7YYj&7u+Sz@5X_{7 z_(tP|b0N}h>5rhpb>bxfjqDsh0B48L++uIB{Mq@K-!MvALJJ?%Y3GG|jP`E`Y}5QH2?Bu;rW-@_kRYIJA4!D(??+RntmMPl;^ zYD_9DCdr4?t)McVfmWSx;{f@U#%wEKw-e{e41(xN-z$v4(X$O{8DBncKK6#;*jxqd zqRPu;;MhAEcvnt$a3d}}bJ!D44-Qd|q)L5byOxvYjU{$`dz6bA;|y=r?T9@l$qhwf zJOM6ThgtUW*F&ut3eg{?Ryl$ls&%;=UJW!(c^MA}=IQ zi)y-SWDHmA%IRTxON&w=e)}IYVi4kM>jui>*_l{h`{5;1H}>=2x1}tn9y{eMXs?2y zv!5C>;_H!t_f>9*ZeS4Krnf&HTYv5Y(0VAoBv&-u(~eGInkyN1>`l71pHnxJr!5_i2ZxH`+n zE7B;7Cz7Ghb&|s1T^qRB9w0QxIJHUql_*N;w3G&8A@=ox!pt zux`>1elCr6uL^gmTi62tG$WjEy4vO<%dU)#VK{&#EBGB%AW41Yh2-QJ`ZV0>^tnMu zdUfWc;`^0~J^J@i=1eHjqtw2mX0OAgWIe~tEUkNXNs!~8%jyarl^pCn(rJ2qd(fBaT77asD>b{+ zfv=nx%$;}ooQhB1aYEodU`Iai=AiOBw_Qb7iPw1qS9r97DVA3jZLE$OH1k6-mtx%p z5z`;L+UKe{XEPvk{X*Op=tRxA?msjJe#{CT{+%3H{9C(BS4CoK1NCihMqXAB2(_+_ zBz`O*?4~M6M)#I!JH+t<2IBLRJsw1m>*kJ64aP9FUvtv-O^gk}Z2IELc8Mlw!w{dd zTirzmVzcG$wit+fe4{YZjH_$R2ldNhO)ZB{C^h46`F!^$-enbPDW(A8E>GXjc*4A1FOB&g(|*s`u9Ug`}{bZ zhm~Zw8cqTRtY*{CZ>F&Luk90qFJ&+U+x9>Mnz40gGy$lyq&2CJ^5B^>Qe#t+R#?Js(4*g2*OSIgO~%~ry){M(1Sl!M#iu}i`|-T6-}b;Z$>uN4boy?T9i7##juW)rtzUUx)mpH z1r7t3VmdyENtTsJfYPFneRZ^{9wU5oxcP`wIP&8+1pUEr3tC-wTay2NG6LP9(&C8= zco&jVP%Aq;=rq)IjEBW-?Xq29S?$_`)jwa$B;BjKY}X#%fF{!0Iu}z)x4A@Xx@u;4 zCde88I*W#(wYOZk;gH;-{+!~|9NN!HVo0el*H%v8ObkPg3w_;;<3C7k%Bazk zu{P2)mzXE1-p;_`26Ah6*n?kTBjCB}0qm+~%clgP-&3J*pIs5_qRynY&7+{-V>Dl41kb8`2ri-srhY zqsKpJ&l$K@zf?AbEqk87YYTT_t~431n2JpjsqlbGkrU0Xo>okgzCwZ^CeLdutGR%7 z_^oTs9MIt#%fw)f9u0OKlFavc(R}xl&vBnNzHP+==~);LFg;T3haB64duV@e;1orG z+27a^2HrA9JHPSl(zmy0R(EwB_x>4L3mHO$cAQ6D;La9t2c-P9&rDT5$YB*bq$$KV zuPmhO*H$8)j{NVR?q~9#7Z$2+0!js2?B2;N$9Ex7FUxG@*m5i9u*b zv%Kke^vfN=F|}^UEk!Y(nK-A8QUdje-9yfXnSV3MO1hizsSZLg$V24TC2uZ=GY8m1 z2NDY7n>PAHK0M+$KsI<6GU?xc%8z5^k5D)8c`_mAoA?58$;So)4Q%eSTke^kPD?=y zd0Q3?%etW%^L#FcPq}IE=m(?wbMwe|hJ(M-@{?008x?-3_?@q3L3=r#% z%yi?1L_;REwqU|uHu1w>LraH#C##5-+}#aUT#hxF7_DX29~Vs5xo>kGP+RPjJx&-4 zu_CnLzE9y?fD77Zuco?q%~NV(jI5ysHBIf-?(q5&nAE^WD^~xjp0O(6=BF0!eX#I( zOGLZzIow^57dfjR>{6qyJZ7%a?2Lpx$l2y&wEMp8LVj`#7G zBcMza?WSov<~QzNV0ANk+^LOQO$O!AZ#p!+txMpR7>@C^v5=${;2!ND`TQ76g(AWX zoJKu;r+QK5nb`$Ho8KgUfq_lte(QTNn6TDNT4#+by;d%URMo!`JA9Xf18P*_YjhC? z_ar-WbcE#J|GwgrYV4LFk9phjqlU@T1GyoII;=S9ObRz`DsQ{Ds}Y!sv1pv}vLsal zyl9Av)+BnoHGUHkRDaWmQqtEl-blzJ2#xzBxF(6bN~JPzpbEQBLvcaSw(rpZM5SI- zuST3k+tZ3G@dUO7O$@>W4+g9Bqc^>R z;OsSb)XXo%#V>hHeT5qz64y@;Gx3)$$wnz#mH@vH=o#;J;n2s`>)i24914kuuE z#@w!hh4$kTOWi^9ZLmvRlbWRs$C<9?4vP*@?9m#8jvl7>qCzyCl8 zks_^|&214QDl6U^vhkA9*S$>%xiF{3v~v$;i36L)m5v|Rg4uR?KX|y(h8lk$BlUWL z6`B|)HaB=<^t9CspAU4B8&(y!^5mve6nghLE3{`=+;g1|d>!h=U5d42StmUqteaV< zNo%HU1O@pLKw~&$8hwmOo>p$PTb89mMd~GlL<~1~xTKU*cNw1aC$sfl>gMeUa4)u2 zYqpoxB7+G)2-MotAmmp#xWX82VBJ?lD0cj0GVMcbU0hcg^M}IDq>i+y*>#ktZk<3G zmRCGJ4+J^}i)B5CuI||w%Fbq}Qq2v?D#TQkhdnD=)9J0{9XF9_`o=Nu+;b1K zuWn&&@6A_9KW$aT_nrBJE`PR-k__Iq58H>?6~FUWh}IwWRTL&17EvE#^g*3BwGeNl zhEDl|<2Kg>x*Bwb-Bh4~R(SJmv&dU_k(`o@ZBw8(G`Ltw=<(#r) zk4h?Gg38^JQA-hA?|4M~>R{(7O zE$M|SE8Awl<$k~UcT_zzFkS5F0Xha+zaIaaU7%V>F$xBFiESA}+?Bs9$bqlmGz-l7 zx0XXkNz>d?P*9M_FvKAsm*XCIl*MzX_A%>8MMZZ7?}_QyX9VNtmbvLqF=M!~AJjpe zMw_sl(WiphkLousXIzYy`;&G|*ejd3T)x4od5I~%Z6~rjG6{755N9h=BLz7HMz)gM z2)sSq7=*YeY&9oU8o59*t5?>37!#AZO0ric`_~gk<Css3uLe}(eCg1dxsd};xR zXtd6+M1#uc_VZSz|EfQ;Qmr)fMVvNG<6wnrh8FCq)-HGS?N{F9lXT<}z%co(Z~}gJ zPj5pdYk5r#+*lX}-)Y7j;)tJlcd=59rZ#r{9^d>>HG8`Mg+MkgCCEv_n}cCjBMrJ< zjrj7s5`29Mo+1I8vHVaPSzkI<_;lcyX9fCUK973C=T~`Kn(Xh0mb?${|; zHycIU)2k6M-(rsQ>HQ22Xer=0&1HryZ5>?z+tBP7d9^PUXgKh4_T=jVC@%JM?V=7@ z(#LBy!IvID#BeUkCBZyD0y1uFTKKyj-X=hJu%fhY+)jk+FGG<^F`xNY&}=YiI?_b3 zdG#HjH{kLlD1qh2&PY8?u&c?-*OUd9X1^~)1L`~8%x^lP6b_?K9F0G;gRRdJN%RnNO!RQAVA5yyKuj`NZoS>H2;%`83>*8K2!;COtSIuBCd32vxcyVF70q zu$u!OB_6@!iCyb<71oSHxr&#SVy$~8WY@8i6oq*s7uhXWF6KHUH)8Z$B>Ti+;~x7N zthM^T@`Y&!0Pks>$=%8yrj)Yno(HHe;;p2UDZZ@H@{s)>`ltrSj(l|A@<)@CbF7o9 z(N7&!D2WPtvU?s5J{M@oc+J&#Tg$mJO~udwJS%=coYugUNHsW}=zc|t=^%=q9=_B0 zy7|?+RNCw3Pr~ef*oly+HdB|NpPDw=VvaCM=~Etmt~o@rj3tOjT&x4@kKZw5T!XaD z(+`c@k4-ac!v87I&qr6%L@febXAg9xAYp>uI4NXEW73^=f4U+v)SK;bYGNRzr3y1^*+;W5?lAgOS2u^xLAGLM6d>HB{1+&2~CtW z!bD_gH-#gYp932#4EgzWR$F~bL(t3bZ@!D)?uli2;C{RZLA)qHHusHxqx_~UrvI!t zPZ#e1zAc91UkaKDp*j?6BZf9$U&!p|`Zsa%o8Tu{mz1x*aUDP0A|3j*Cl9rp7g%qX zN`T1HT*KCNDQ7PD8a5FM(E$5uUL|P{loQ<=>fSbbub@$u3nB*B4ZMN%+ODysMFUYP zQ0`!W2;Hp;RoyPAx=*W+A>Jd93ANn2|P0lB>q7ljkGC3V9x?08~tyURaPTBI!k(a ziuHS~dY~!~0Oq}@D{$rdsl9SUAiHnnI+^{UR88AK2V(2;d#y1@iXGmTRSH2Rl0jLpd~AP6231i zf==N5pRm^7OHJI7dXw^lH*4no9p!jl2OLBi}MzaZi3by^zX~F(C%VnkGU8 z!{r7`W!fRc2rr%8*reuKUt_Y%W=wcmGOO)P23z)5ZG*;lh#uoS?8K_qdKKj+*9-Q0 zG`^ZRAf4pN)^TcBQC#D0D@(Gb{t4!xQx>Ltn(0*1)hxTYFCcrScYH5;+e@9)Cni;< z6x(X$ZASWFf2N}%!dhBzjOS-P^JG+K?}8T4CV)HRxtMD;qhoPLCq+7l!}HB4E9Cpt zMM0BovN1y5Jj*4-WD@>#w;$5-M*uv`8z;EFX!~|jzVhiaAn3Nc@`YrQ7SF+dE}uFZ zyrZ>+ZF7G!!LpeAS3~AmJ=Edw47@A2(Xat~^F&!~-WI7gqoxy?HUE?Ac@1!H?H-G+ zc&)zF1kW^j)@=vgy<&^m{HoqZ-N=`~YapaoKT$Sf?xfuKvcKu4Y;pY>jJX;fH|@M~ z)#1Trslz|9xYq={`RFI>#;9DK-SYQaCvdSOVBRr^hL02dpv2aOH}KaBS$PyYEp)o> zy)zG#i2G1LLE6|dbF=Q8+8WW9Zsk;Ix5Dk1*@YG1qRU>tofZ9sS8bj%@Fp;a(;;xr zB)_=t@;#)?Ds}NLAZbRi!~c|T>zA?wV{=@;u0|LD4iVN+3x zrcuoHxrEu7!6LpiT*yjV@VEx!t)BJ|AAK-eZ_iFB>4kpGqB_`T;R9ngd>*<6T~Lds zXJd}Vby`s_!9!W0hnJSmPMoM5E!PjYZW~Fk&1JVN#?&(!CU}VpGSZsCMKfv7_o7~5{ zD8rR$UX1tebTol3<2_=20Yl5awe#(6@|$uv>) zgtRS@WjX{p`Qy5KR>}6``>J-eHZvmr-W%QR?xrO^jz4g^9h}#0HjkWH(c;vNez(iL zfOP7>0M&UrffFA~v!2d)kUXwk4~!&Y20~LBdfSI%ebS{E8UaL7nbE*tud#06T>VFm zNYP>VrY~+8oOHMn6~GO4rFA2~8%o~K?h_4bG$4R77zHEMtS4cUuZrd1cHv0TGu5B} zj&0H}DsI|13YrWa1e51){!n{W;Zbx=dv2xwZe6)u;|SWR05fv6{W$waOA`)! zzbqsZ3>(v+ySnf(=YqYo%3g>9Tu|i94?&&Jycm+GN^E{Q*aKG zyqL*uRMp->upDo>87AWeVXGMlLQt^N`Gt-5Dm zY;D-emU%{gFwA^9HeYD%K%c}#BCo9S3=~w@!*4x823>nFI5nt9B~r~eeLdK=*B*VW zd7(vNn8+a?>S%MAEcUu!J{MCQ*76@TYwU~tcsOHus)`B~hm9ffZx$`Gjfp#)sJtti(+d3M(SfdK4LdL2^Im#}-bq;&KcwUv!MUmg(zs**Ag z0&nSGTL{x4G^j#y8FZs=>H@}`5=2h-YTstskvnh&HtnrEB~*pJ zLvr1+|5)u)CaNhF(mUDI1*&`vMEzQ0O5QsG-kIHhGdtPuBM=+kQfPU~f(Z>8jX53} zwTRPGu;i-;9*2!y%>v4iJmJNzQoph+_ncr6O^-$2?;jXE3_sQofqb?+mF#Qh|kR$LwmWQ({F<$pZvZfHPfwA0ve#>Za zMLgv~czeoDY9L~6*+{ftni-NF=xawK4J5!wv?Fl@kT2nze~h=~CPc>dJ+Ox=LTXz( zK=?h&>C?G~qfKE(e9V9y@vvhiBS)^-?$u%Gx37sJ)ZfxY$K8ByDqiODG*1}ed6#=* zYIHX_vzY@<+Xr>rusM;PfQMkA@H4=3^72BQ8XWMx5ZmzIe-iKsEujD*)MtDPZpe}^OLu6VV$mvB8 zL7+3~5;k7Q%Y#7eVX%YmE5H4nrNe_Y{mD=5Jy7}QQiBwDR5U@8_0BkG2Sf2T4H=A!BP zWBCoXyV^_E*wd?`Tk{}tgW~<^zYMoGNrEwlXP48$^g_$;XLqAKJls4lGi=_v909o} znuoF&Ej8ns&ri)bu+3Et4hIhhj($M;Ic7HAkoyl6bKZ1E@LD;}+uuYzTD)Nl#vmlS zqOK4fox?M7Tn_dXl_jr*%vAX3eLDfGU%(ZrTAH;Clhq^09O8cL*-2gHN5STZ?SUhU zI0u?WZ1#q|M#XPjc2sxr4LB`)MVB<(iLM%1BD0Ln8KxIoH8-UR|K(m&2s$icvzGbC zwY$-Qe{-#`hRwR!)SW%$eE#ZVONE)bT{tIbZsx_rZcGyS&(~AvtL#oQN@Rdk>7}32 zZ3d#(H0UTP$W=hV)lK_yMX;EeM8PHva{fCIl85yLSSuA;z`)9FZF;@3rY&_Hv{&gv zp13p4mq0#rIAsmBbC=z28=PP#xvRi2=FHYf*SE`_hGo4%4y3x6I4e-rF>reJJ1BYq)CJh2P1AE<9hr6ZC+3{q+O?fYV^Miq>O(3SZ z+nLHh+PJy-&hnFRY#)&la145FG=x1e-t)Fvy;Fxozi&xa{~8CIZ&iL>tE}t2lqpJq z)`2{?cZ5y0tyDAc(ly_|3or~M;uV7h(ttqB8}Uyq6p+jOY--qy%Zr+(k-oUho4^Hx zDc(3LGU@88EVHPw0Yj}F*PG8XKW<>W4$qTb1NY6Sr`-_7F$MeE1|9P!cMHRK53(?S zSvxnJ=GP*SiYnOz_KVuAZw`)CSilWSUt)#X&2T}3!&G#t&zkhpw+EykrH*8?E$5UkT-rPtt4m{&l>Xog-%D)8%IU%GlViz zd||56ue{xvd497!L6}0{K+HTGnN>^T*HqYLo4kl+$7(EWhKM%!S=@fs`w7^f;ZA#p zX_=}xtm~;^o#G2R6}LN8Q-1H-P}2qm6F*4O{m~-o4M*>szwMXa=qHaOh`;0-81y7@ znOQIb?x&2wQ3{s0ZvmvNQjirFZFZwwht(sOq5Tcwbesqd;n%ffu*_QbF`AiL>nmUH zV%@rtH{VJLW6;)sI7$iEbwqNrFZoaDR4T+y*6Z9Yx60mD(+z)Agysqz`@fBJBvPee zo$(lS6Hh+BeDzcfZ;fS;n9ckQeh{&J4`rP4t`YpBakWQ)bxX?Uao`Ji-(9dyE zg~;sRB)J;RwV`fC>E`i+=j~f$v0P;CEoiUnnCtO#d*|AGbk1Cq-HFV9V9m*{k?x=2 zw*OGq7A`}-pHY54Vm%%C8BJ&QCs1=r{>=}hP;e#KY}Q*)#^F(p?yTr_+zIG?eSt(S&v zTzp>AlPz2F&ZuTY1}cRrh`>RADWV5Pe-dJ&%LL;3qeN=%(Q=|HQ><9~-``VpUkh|x zy|F74)2>WY81zI^E@~dQZ6$#knmYv=h#h*qu>i%)e@3>|genzg*;sM4x*p)65-xsm zj!oF3wP&_zeagVuu-DnDAugGhZVZ$r2HC;(!h;D^0HYWV29M_^Q8!MQeKA+XJ%1Jg zk&`kpJS&*>6g|9}N^+FAiWCQ4)Hr)OyXmP&G^OMIcCR}6-#}>3lExyx?7Y{G+HO^P zl; zRzEyqI4pHv*ouTak9BZqKJw+V9X-XRXEe^$@*3IFGtr`M(1tnW@BbO72Alda#Bnmb zWRi8P_~_R6k4)rNi{U|rR#Yi8t7=@#$i4;i`J7*d*)bRO`OESnNf=G~xY#29D|Inq z`6B!L;Mwr!4QZVMHWT0eq0F9No}GpIf~OFzQ(DnkUCA8i zmYu)Vq?dr@gYK4Wj-Q~H>^Jl!sew_030cf!6;G={`UJ(q*q*G29xo%R4stdc} zH(>wW#anugWjW__qMu{PJNKu`T*ztOU)NNI3oZ?RoCvRr1<(LH z_gzO%5HhMlF03A|um9d*d+eXe8Y3FxpBW3>EDW^B7(+}tM9=@o2P;dQ&3XxpDq-%$ z@;}78&XoztrvqAs^OxZ(%EGqF(R}l|vg476DLA=rOZN893!!}(aX!3(+iT`PZnGbL z`@jne+$pL;PGe{bv7fh!pZ#Pf)+T^T+n7lo=Thuq!a3d(3#|)nHj_$*#$T}FnT^Xl z?&{IZ4p-Us$u6DjHmsf3;0Gv7qKAOnMhx%t>`(=4`UvzMzt|X)T25Y7cwDtF;tl@2 z-TuXXGfIKMVhnwIFWGr?D*ts=X-lT~<_j#$`XkZ6W9SX8wewX9KcF^_8Z2@lZ~|Hb6!jZ}cg)yDG| zR9n6G(*M4aA^~j4Pj$AcFd2R*~EZ~Jh_W&`G~Up60&JB!jh z*~FYTzi4m#YPiUFq3_Srqg^W#&cy1 z$5sD|?uLw^u1>yVcm}}z_?Yfi2mXu0a{afLH70{Fqf#RO?~dNg)s@=AD}3?)z<-(Z z>->0FR;q?+JJ}u?;%wqD5?$!2(Lbnz55I}E^{_)j2&B%#WaZE{k}aaF-H_paWrJfh z_`y7WlH4D_HzDjILM*bM@}JnV@myzm)H8d-an7T=NO)$5)ia^*len{+^;p|B`T7z% zRdYbY)3__o6L@1}ans`_bg2Ogji;mam@+_!3_Z22`7!Jk2ozs(HdTwcro>t#Z8*}e zEhBE?Z}r8dG)EoxlHK(0G?<(k?e5<^ub#Iv&0hoRgfgL$sf}^zUVkg29D3ZgxX`O< z@Tf4u;U)~#-!d*KHfCK3lUd;kGWSO^s?W~t^>y9(EyZ^?6eGlUNvYbxk~DmPd|Ges zN@~NXS+2L{!lwu5Rp1FW6EW*yB5f0-8umK#P~2Dq%geGdgL1ArUADAMFYYKC#%g-< z{;@6;79=4nL_ON11a)52sl$x;5gYDvsJNofd=zi|bGy<->82Yuy9WMJJeOGOJFePt zZ04r$R-pIBSm7Gve7m{_auSYf?|+mv6ticx*-xh{h`<(VJrq4Ryjf7CWiYG)8~RR6 z>56{$b>OL4Pj4I@#|6DrHXbOPm%@>?HXJ9)^cRnbfSAnI0!|}%8KTIb?Ht>UU%ClU z7$x!vKyA8}Jayi4-o1za#v}JY5#-ueImobd;W~plMxX8{ee$plj?Mz~l!D%*d$DJx z@(4XD;O`EQW#pfUGQJ8M4`B9LJXm~*nioe5JDLo%e@6)py{wuyGIa%q<>P=Tmp%KV zVkUhTMIHDI;JXBb?GuQxmh?^QMd;SaRa(`G$#6*M^V)zrltj&J=t$48ViXA>Xo=R6 zg1GvwUE z;sXVZ>;Jmd{1b!|)SB9N(D?WTUzKrsoO7kGz^SFOtu=!f5zXfw4wYEhEQ4PH>YDbh zbvU4meuXHd!JrsinFH7*ilM<*!hcc`-`Ka>C9^&klA?I+7saF6?EC=_wXaNlp>M}a z1-PIxx>?ez2yMHJdtgp#PcFPD#SjzxG?%TP8^#j_8L(@~R|#JhBpmf+K%h?0y5Uxi z>RpVuj{VuH6|BmF7+1Eh*-9!~*V#4z;(Iw8KKH?C^&2keog_bn@T+QV8uirUa(8{1kg_Om5;lWIhsK@${e9dm4`%3o=}ZsEC1ry1ton^6~89cq^kfjCR_l z>R6QL$4pI)KB3xkvv-?J1;JJJCJNl16w^FK^FPopldS*?T(ZKPb7J61N=9lGolDK& zHL+QOCi=L)NZc=^?frALd*jP;)4CQNO9b-fRregklteCV6>=L8A7mx~dSCrAun=n{ zZ#-~tYdv#^?I{6D%W7WVTo1QhJ|u=1+?<_(2QlL9Pfhp5Y9dC+1^1IfiOOd{AftQk zFlCb?Fu-*pSBrV+eL5X)cBL{2=+M0@nvcBfybRZmt7C5jd#4^kg%~I26My4@Y_vIz z26Xdgc53uP5I0Ma(@DhHFfG3+AD73R-zqkqFcdKS@P2_~dJ~KsaJcmE5FhF70eZre z>oq3{r%QTsbOe5n?T;uNtCk|4jhzlE1=l4ZpDSMVv>4vi@~H2E432LYRB~cqh2rbL z1~ljGf?*~$p2lpeXr@1$oHLXu?N|1d(P1Ly^QpqE9`F3fiO=MBc`8&!ggcqUU+Kfl z;3?kfprDO1XXt7zRRXFEVU?ttmz4Q-Qh2yDiK$lzM?<71X zW@ga)7bAwM&BtIe)NzDDvHL1k{-U?f8G;!&C_kQ=9q%StPei14G7;BozclDk6aWSN z4L31lpj|J=cyru^?yZnm-PytizXp`fl6laYi2zhp_H`#qjVA3+Ed zi&U*A;yJg`R&rFYJ1aLZ*VNtJN}QsK1TDAG1Ak?#z0FvEz@UI83Y z#qv0s8WjR>C0B5+Y;2g80HiEv0O zDp6)h3&g8Te(jE5HvUHV7r7R(;i|L9&Mwh$VqP5fFJD^%kTh}#A^1hTYLg{iRaiv3 zbD3P#mgQkfax=w(|4SKP!1-h9rIc0{xiX!KKFBHd^OVepy*E<38YsZySI}?9I{CD9 zZtku!>Uzz&nexttBBCS@-_*Vqu;u`AFAzyPg6)s^y^*E(a_AHDKgIQEl^)=mA{ zMu%|VVDqStDuy5@-ERoqstC)GIHoEErTjC($%!h+(`4vewe`Gt5ZIKYt+cHwUt1B% zZh;%*`wikeJJXR0cxM&vJpQU zG6uaj)WjeEF$C0TZ_N!Tq3CVhqktma2>Ii|%Sx7FE)b#-K@JNn zs&;&C_XmK~`~_&Mfk;yA6<*2d~8 z*QAQkY*BTtZPk2;`GTM$PC#zFr+?(OavqZcr2h5rCxQ*0&`+E}(Fp8oE~D92r`q0m z-*SfkBm@5b*{-nHEMpRU4HVisSMpasfK57}5k&%j3WqtI-a&WqL)0Ck>RAN}dGKl5 ztfY-`gIXdX0mC=uwyDuu@#$`TuyYP)!V>RNhPNuCeQJ@`{$j3{B12{%CpjQVsU^Rm zOVZ-~Qd{*{5CSh;eRI#ttHh3IWy8t+HZft!4v>q^%1};fg+&YZ&|2#i>KrN$a=9-P z6(j8T47BLpVmtGn{i9+cA@I%}Y5{HKKT#{RwBRqho;FklMo%UfM6KgLvStBwA`nS} zN9&fCP)*ivvyS*%%DN?KE1skM)< zD-uc*!x+mdglhQmr}7HrIu-r{sqNVS5|m#g!qh|r?HVK_-lD)^Eb=VIw1YR|-b(%so6 z74`@y9dfq+US^X+eSf4E=D9x0Liiwto;pmcHmP&ycD*$F>&rKQlgb+h9GE_*==2>m z8(v$lIuG5d-sV*3Y~wP8&}#<1a=6$$eoDU9sO}5`v5f1!y%uR*Y1WMSyrTn->^BN%5C#IB57`>2jbAA{ zu5bGLzX;ZN&}b<@$|7b<#5S6Fh&l<3sG5j)_q#4KBxRfoG{@$wu7*wCw?;+IE#Os* znlZe|sYB7}arO6Qw90E;e);75^Ip(ZLZb!)1?uCG@?C*Q)np_g;sfpvx%u3)B0r)Z zk9ak^=0PA&CKFAo*h?neN>!WPGVCx=AtWq~ zO9c009$j_$TG1i*TY&(yue#u6KWFV>Pz9cC@YL3n1KRxJ!k%tng1V8c5F+U_9FA$w z(tdbwUESBqw^n6CIn5jt`Ppw4&wfr)m3Y(N=Dx{!Km(d9Z;h9GyOX{~Hj>#5+38WX za2&n44cX=2pWV!nsuEqX>B0k@mu#(Z@3-_6``cRlg>xdZtX^>q*s8BtS=>Lh=x+0+ zK7CqRBJ@NvpQ)G=gkZkFe))qqd#hrRdupL%I{Rl4i-iTo8XFd_l6n)KJ4o!sYX=li zGqVGYh$=F|l^8|5_0qnb7j%%Gi64q2L*`ogB*To<h6hXLgrbkHa$eaHdDz+? zv4ED~>oB}7#KAGRU7_(a$Bu6ebJgT~zP*uwmaS_|CTSegxW~Rpe`5WNUFVn%yz>o_ zf~CCaCfzzR^~*SKE3Eg6Y}!v8Q=dEtdJxLb6{W4P^>Kb&loW=yfhdlL`^Wp>Tvp>D z8yuRnaB+P%l0s%XK7AksCH1NdxcclUr%fL_*kBO{k+c7iPTVYH&r1@k2of*7ZLqF# z>3Nkwi(@K8yIzsyLxR^8Z$~ICS-PKpQ!H%w9Pge_?>>E#8m^owF7f;P%j@1REFkvK zM8)FMzgG-LTFl$mtqd_&oV5Y6jZ{C&{f@kWQkE75dnI1t=k*Le57Fz5Z(sZeih4Q_ z0+~9}$`l?KtxM6RYO0nr&MvxOvJ^P2^tvv%THZ<|SfdZ4|3Xia2dd$Vd4@U;O}(|+ zlv|YKjD9)Ht?Kuw+Hdga#jwM|A#i8QJyk!Eqw=ovoJ|`1U7wq6V0r&JL#M?!YtHg6hhsDL zi6!n3I;%+)O_|JPc15BR2da5!W^b$19+HsI;2}Agi=^dO6R0~$D^UxdmDsh`&5=8c zTItAc;=|=IQ}*(^8ap4H-|3s*a}EO@E71|*ua1a!YkSmJCGU={!ABZ+e=}?)Z{#qJ z4+YFVrsFhuPpCPFMcjlxWOn!f0L*aNiH)ttzYGQ*Ibi=#um;O{ofJ`9{b}(1%(T7K zuUHSh#@3ycw7Gz2<;OugEZ%|Mw61Gy@CvAss(#3SpcJ|GiI4XgYK62%T9e1Ai}+UH z?lw2L{>`it`1XdgT5LNhgb(U=J~y(s1p4o>E|nINE_?pE_aG!50M}KesOz!IY&WXR zfemvTBCF43h8>RtXimyuYPrKV>F(ke=sK5AynWGBroJQi0w=2p#KxeY)OWT5ZdSq< z6E=BGs_xnk=d&LD@Dn&K(4p6vExy|z`^71DO=*k^yaP0O98sblWqK#tV%8`03}{#r znMe{ik&225=5!Lq8(y(ng1l=yOD{9Wbq95_qJG!<5WZuqT4zXVPm5aU;e5@?3-Ecu z3wQx@wU905_{kOgHn#_ABg(@Kzg;SuUeZ5z-Fvx%<0(evGqil?WkUe<*e%llb!sPY zhyX_X_OCM~X>%A;b!~laWu<5tAGQMdmB!nf#;H1MSordUEQet>Db{4q1<3q2dnfra zbY>7M!h#BK;P8&lBs}Yf*|^UR!*|t3Ig^+9mukdzr@7$xGqD#eK%!OFUIkslqC-=z zTbzxOF{8DV2$NCd%t&&yK=m)}9Nc|WpLf`e=GB{bqbtlS6AwXA&xlfxa~IAA#=NSj zgc!YvJvdj9N{{7QCWQacW5fAwQoMR6i^rS@l#)!jvtu8(-SDkjrZ{8vCeI}&mr~Rh zLi6a`^d{ZA!0U417Eo)B2~!%&L;cDS&png|MD zIOi^73d@d8bTY?JNO5HZu`3ua&d^PNys`GFmFxQLFZBy*GsaDGIdvq8^HHr+UnQJ= zfu<1y_%Val|3}hQ2SnL4@uQ^`1e8>ekdp2cL8Xzd18KOU`;Zc83F!twkUr{;JS0TA z<7klXF3IoV{rHOdFYrvi@7s@H zoL5|(Z1?lS#^_ErBJn!)`@T&mx;Z|zC|sQ>{rmp3wcJ9Z&WCt(z{u3w8*|+Y{{LFV zMF#mRgY`xIZEW_{m7uwHV^2@3#mceOGJ_rI(Ke=4O~`MTO9C@%d`tu&B@$?Nd7(5L z?o7g8gn^l>+Dk1+w(TQeHhl?h4apgs2S{$x^%4} zU9io5a4`G`omKw^OLI*y|M-OUevBkm=AP$r4e_Dqj3}l#<|=vrsBay(e(2cQGZf-I zhOrCAzxFZg(Cg7az=W))_k^5AQQkqf!KLIma_QH#oib-SE+S8ES8AYe{bTk)s_(Yc z%_GAQOwDyfl#8AQFjBOY*7o2?d8q~Ed1R^C3ofD6wmt(h8sDjhnJ%417+L2=X@Mch zFI9&5kyay4-yMA>tjWA7{uh z428Y}o-qdWWytT+XDS&x?e2&6;g`FLW`qio@zK6=h6?C9ZmjB6L3jNP-Mkf8 zp_nZ=PrT=knvdg>qf-{u2_DA1dk_8Bx7lSDw1e?>hxo3`^63yPWo0FY{$u|{Xd{00 zb5M?Ert`YrjJ3O%-|%-*`r zNjmw2{M2_@!e#vd*ps@05RJjaXBFdk6>7v7~bSjwVZ_uA5Oqi z!8N&KMJ%-ei?sn=KEo}~tPJ@T8f}MX25@3Wq6$kq_C-u@`{Qx`mNPl$4~rHAApWXe z{z|Ug)N9hzO?JUN)de}+ysSTyQH7I}Ae(cExu=XkwEB{QW3n1bPTXRCO@JE^}+R8SduQq&Fu07zv|1vY-j?- zeZ5qOwxZbBER#JK`Mk?K1MJPOnZ1dD$%3NyXkFu<7;h`9hNOGIbwodU*+aJl&xmmv zl&4J0XLK3UUVUCQA^=rSuW}?v{ z`4ZIEYn!pVTi$pzw`{!}ivAsaJbd7IA&NF%Gy^)0#~S_;1!SW3Nw(=MjJ9kxcuuF5 zt3Cmfb8Pqtc>w|RE~2;%J3l35CC;Vl)PTn&Ax~4&0l#zg-^vP4fq_&9FO@#{E#C~2 zUzh6*Z?gGfoOhW}%UPi)yCyJnqc5voS>_Z$#&q6FW4Ov9O-YiA3lV~Ds&~iNn&p3F zYR|(5Ny?8l94}*>Gu~h<#?|i*p;DA*=NnpEFa9DjU0%Lx@KojbRNf(zZL>A{d0NK0 zMDj?~8?%Ku{P!wij`s2Tj(Y8UAFX5)O9b)CzobdW?R3cH%Ve{~Bk$eH^S@m8OlURB zCXSN9fBgP2buRftvT1FTTiAAE>a-JyO4;V&`Q5wnGg$cFUw1A^A*pp$Kmq*Xs2GR& z1G!RO_I;r~)xN(uqc8_u?PokBUon0vJ&{81Em#u<`{WXb9l*>sQ9<`X?bb!R zFO~i{m97gN*1{m(WHpq9l`=13tGKy&*!3Q`LVW*f7XG*;!7j)g3+cjC!rm3dM&7%$ zK$Wa(`r*oZ03SXQe!G~TU>6*QfB8EhW<$T&rryR-#%G_7+9J?(f3 zG}iWj6!AOB_c?Lmd-X=-Zh+`D-WMaiz^8qJ7tG1D63!vm0YK7-c6?^IbwavIYF5)8 zC!}TV@%CKX-x$?bq^6X^M%G^>?=R!;T=)pv>_95^EtH@9`+F#xIwlS}qwC5sOL`zN z_jh2vr+|^Zi+>~WfulpL9Y2@t=u|sPd~m{FcQ}BH z`r1`F1RWr_hUg~*T6MB_c$CNJIx8Fjwi{Ty!ZXqg)CT@~duTo>_ZUIdRcq<<2 zNY|sP)dH0Yi7t4n&#+V3!vck8@L??WIPZ#oihBByzseCGcOg5oFkp&kPD)@~FVnq| zo|QTkD2S)Y?&<28_lY;H@;EwUzEWuTYJ&bUelnb(P6u3l^C`Sodse#UZ5De33zRfN z5M%KuB8kpHY6MyGXJ%xG*GJlkHBUMlSQ*x6buZ@w`DlSGdht9}3i;#T)&prEC+Chd zJ~*>RaN|*eBjne(xfikC=jgnqG9I7V*T_$X#h$9^0FjxPz~g0`>3+%Oeg1|`>_g}N z$l>ccd%h-I&Q{I+XAKHsW?#7$k|Lt%K~CPeLH1*U*p#8GKV2=uxV|;J+VEiW#80Rq zqXs`7s$QPd(D)&pdJ#aKbI1zfQ$im%c^Z{nX!{B|La$chVpNm{g>?+upam1ox@@Mo z&epFydlmd-X*0}-$<=q5O+;VsaL6V95;0N<^hSBoE#DT*#=RJ5 zhQc;Cnvng-{At&D>;NVHA5APe%)MNbTfGVWmjU`+2h58lm?(Ri>_oTx2gr;`+{%;nbNMKo8-Kz6wgy zabEJMdXT(x63B7bvF-xMOkOQw`cgyqUzg{~vg^=mWGEcVS{YQQW9k}{-kjXcN%5_( z+kx;@_e)^E3m~)d#T!}U+_TKT#Vstl-+CIu=@}dEV*Nf60^6|4pwb4Jq_PvZXeevn zh!y%*(6lu^_nSV08wb`*;|D&|?s^BAw@BaLMPxv<)px89%V773w=Snkgs$ za9Nm30o!;`oh-eiuSn~g%IKM#btf~KUi6ar#PQ#Ql>8c*PxO(PowTS}AcMs?8YrC$ z7Yd_mpY(2RsRoKSlj+5p+3Xx2oZ6I}J{We`c@4PC5mV)K$jqbA5QKp=1_*tI;KG}v z?wuj2wEt=V_-OP%dBu_kMhKGU)Iv1CbXM@lm&v3~ENvJblKNL`P*v$?JOF^;N{{l+ z&O4!+zyQxkEP-uHJ=`PL_em!mbEY}WwlmJjP`?Ygto#5(s4KACWw_k)&3@<{&xkgV zc_3`m?VjzUNuh$O*eJj+K)4=Bj@0z~9tI5(66UEF!i+pN@Qu3^A<3ITa#=}6 zRS-e#QoUM=9)gm%l&p)YwnW4ON$pyG1MJC=?j8EyZK8QIC?qS|p%44xM^kHJ`W;I3|91v!Pr9J9;Ce@-c*TKRr3l zHESNqSzP|2}{f~!1bUC#au&&{hfC$?ho}kAEcwEnyl_l z@96J3Da`GPu3IDTNrc)PR!4x3155c%9@hWT_G6w5+tg4Kn+FC9Dj_GitlM>B#@1OR zwY!u0BY3V(`DBQL3Z%mh8U0J#Kot<|Gv6)u*WZ69$X+Q_1m%KZn)y|Blt~CJYf3yX z6vFD;Acj17m;Xo*wu;#zM$93cGY!(~#6&J1+SO;BFYjc(%4P7di@-{~=jFp*?BEP; zvaBhcUx|p_d8hwP=0>z3R9+r#1kx*#ZKSDiXRDz6rYUXLTc>dUTZv8A^@DNTmNdrn zec!9TJBsw4pZxbd5m>SwWX%yziapjVsks#RkC=m3*35n36pJJUDrtufP6_N zXFYfR9pEWO`a#4gp}+feHhN2@K$ja-nkc+Tq>8|(pud}+hwd<$yBf8g>C^xB**J9- z4aI%GFXUN#q;9u0L}STSVW5iFSUqx6Ga$Xxv0hlI{V$Tvq7{&PE%yNJE`9GYvgjy zv<@|^u;?a63jy(@q`+D+UMYKi{<#0$>G@s#xQO>}QBu%E*S2J+q0w!Cwzn{Ce!-Q$ z2z{5|S+>x+1V}IM_DDUum!X24VWG2(uA6Ym1DQgsBC?9n!rp=M=2|_%D^>DXNlR;U z2-O5S|FBPsZ-~Nz96uugx0m=%n|Tp~CW?M#;@$taoYV=giseH5_w_0AIOiXAh1lQ` zFa0Al$aJ@A-p+P`4pjR7M|3V;r|s}>a{%N|uVmhXUcdxPXC2{3n=bT`7^K5l@{DlC zQMmpDqojI$3>PGv55Gz4%qM*YIPc2Y)A!tF9034ujP3R!VvvKPheNHQqtz?+fcx?v z9$@oV?50tL+(5$X2-+_SRTv=XB3h5pa zf&=(zL9@8-AFEbFP!!*lrRl6hdZSe4CIT5{eb8y- z7U+63&GL#&P;?!mV`XC{C$1SSBiv`e1 zgYR#v+$F2D(@A_^{*?RK4H>>;x>;5^#x8u0GtdtKs;IW40$VC^Vud=$DOm*uYw)@3 zG`!wgfNPhFiNh`B3qR2N3`+)*a@V)id)qJ5l5F+hf?A4443?<9Pe#@dbx2iF4gfss z7D(MsdU2W3Vwke}#A6*sXk8~TMs&Yd}zJkB+pZl?D zyWg;W*@i=v!U$ho{yRnsY7v{%PL!oiIVv(rM$b;WZJb@~$MkP{mrcx;DOzODAQ?y( zubYw_mG9&GO9M3A58g@2-E5TgLZ*lt4FX`2ym02$a_=%WdimOlwLk+b)7&P7d(x2j ztq}tsFn?8}qKQ{|5UsBLTmihs?e?@Ha-_x*J9rxZamn5+Uo;yuBSmjq z{cc!wV~XnWTLiqz3PnxvX+WIGCK;{ERy^W0D+tcN-K^(>a^vG3W0uR+tNN?)#c2}| zH6uu%?Xdya^`WyEc0e%u`fJU(K652-iKD}_y$csbFl2MIzTeS7HA(6N1eCFhwhOI! zVsS3Y1Xx$~t&Hf|6KW{;3$M-b(8tqvAAVWh8%wd7q!)>5EY1$2Aw;hStA9v)MS;Fgzt8)Ba*=M z<41^vLf@NILM|RPFu!d&At*8$LdR<33nru{G>)!L__v3fF8#!BIBeQ)FWyh=w>zY(fCk& z>7{iw#|4|y!!~)!ly|)!uuC|A)y3^#%9X+me>z1?wW{>Xr-x#>`s%Ip1{O=e@#&{% z^ZWUi8> z{g&7XmGhQ2{_8P>|AeWA1^zx=V=B;v_<{T_O$wS`L3-PhySn8?!$^u=n*tTkTF*h+ zlz(SoDjF!4@39q8l}$k^%n3)u_)paI;uH+mKau79F9V`Zs1nvs<1|<(YMKD zpR@gFdK7;!J(|AdaRa3Km>G&;tq7{)=8X~so@--79#i$#ztl$z4&bC8>esQUz6$x2 z*R=eDDbxa_;;M-tCzU@3YJ>evNn9pZ;+tdHUJ4$ekk5~g}!|POrxeQK>#fYlSAqy|5xZ||% zI~(-VOC^rx$J3ofgPB|lfz_w<*8grK27tCC^;eG6x*;K~K>B9i()LxQ_wMT0`fV42 zQe@Q61}yGQOAF!&d;|%X6NSIDAwer3CPpCR9@hRbqL|ir7GT z6!3*ShMx!FQ*Y-aK+OX+TI@My_1ii1pn0a+kTg_;;plSuOQpWCXae#{c|lKIRiKUoM|V42M95C(Cm- z;e+wp3Dvzn!cP%r+4f~NFu`GnLujpI&FG}gmSFJbfPDuB)6QtZp{z@?&x1mwn4o~$)3ma)+K>v1t20N2 zyV}iSbZK;UK&2L_Or!A3*>I%^zzG7u`fb0XX}O?t1@2>z+VboG7AWw)jo-$Z|1c&< zh6c69ep7TNy!jc0&g;&b&&>mid)z)qpal_8kt7=Zwx(1C1x#%rRHhlwf}a@@^j!N- z49*Ss6-ht_xw%adRF%DL_#hrW*{_=4@6xikr?XPO!K@JA5;bl^k2@k|QIJh&&+drl zUkyq^R!~b<>DWjpmvnkv(zo{|(Xg_ZE&@%UC1=fgV_@*5=^#)ZdFd~OsKQ<`J%zJx zw*Wg=Tg&`IqgXk_{_6EWe}^IN|}np(-K*pTb7JqaQa@{?Y_Be6nxQao-cBcnJTL8jn1EJ3=j2`+T7$AiLhee;X`n8ZA zt|S{({bzzD=q;hI`&m*rhOcDI$F^_>$`}Bg3DkFOmLE1(FR;xN$hhcbC{pYV-l8T( zX%Pvay!~~xu1Wc?Fg6r*D;c|CeCFrjNsk$Bz@Z#j)+o35K25uF)it6z6cwM!h ziID)rgYi*jETJ$b$h4d**GSm$o1JhqAfKIeCXpQ!NJT?M;T_IAH%j|IS|J~A3;yun zO8~ixYU|)vZ^J4js?h~jOfo(R!$`}8I^g;HitbJ1?(bE(0VrdNi)j?>lJNzQ&=V=K z67=IVaLI0QYl0?=<|$8z6PZ%CsVNL~kR=ttIe7z3eR2ZZ7Rb zO|)5bfBEh6kI+A_;TQt#Gh6n_M2x^ji+>6QpTvmxF1#XX*`pm(90)s*M2t&TMu#GIhR%-ewh^uCF$;t{Z+$Tw(Pn$zH>?F-# zIbbl}Ev9}V3{|1K=TT`Tj?1=w@pS&O^p48py2EeOsymD^mxfXe6u|4eyb(g#ivSY4 z#fKmjOCCbs7yYJcOp~xBxZPOP#Qu_;INh7`oe37(-&%LwqLiMA zoC_IJrd*DDX+F}wC7b&QarYdXshE}dUSBWZZ|~cLnDVd?GM?>Y0uaQzoON8#nLBa` zk)&;nKd>SJUF7vU@%!CHN%5};>I%__@INEOd1VwHTm--CQajB$n~9#uIa_IQa~F%0 zvVS5fx&PorcxU2m8pXUC!(-h|b?eg~=)H4IZgN)JwLl}R*oD=?R~RK_3u6|B-+_Xg zf0)Tho!M_W?KU2jt{20&eq}V$d1UA1Oy*kA_c|giK`%|Bk}zBshAgUwfl_$pWc@_M znv**XSDPkpkx9AWs|_{y4I5JV6Klo4+xERzJ3Sy5De7|UhD{dV{?;>;TtzP{3)5>0*)!S^u3jw;gw-t*CG6DQ&n#@zOZ-CAJZe^ zB>d@>=}mt{{JzS&gA0b>a;0zJt36{lMgW1sjJ&MC6kvTiHO0pPJqa$1l@{pBuxq7s z8@i}Ai>qWtby|~Qw9K9w%v*54B`b!27V{no@hZhlw4Lyoz!OZ*ul_E386!s^&|DDH z;gqfILuDa&=~;ju0EJ@ykeI}M$^UJXtI{-JUu3Lq>u=P8+ul=tbT^`N6CaH)v!Ita zE(Bl?o~2==)W0o%425~{*Ns1cmMg<~H^~?EPo_x=N*+_<0NDZIZP{sW+TP=&SJ2P` zLxY#0DI*h)l`g;H#mnFCY{t_U^MY`8?DR>=-Mo76l^HpX7YhohcP)CAkyTdd1V~-U zB7f>1&DOh$&5pMsZ5~cK!;nsbRy$eWw>@XPlhX955Y_{r*%X*)t)FerX|PWcIZI!Z!!8T?_dcA-fu3Z<_)$h2 zuxy{1=+TD*s9VeA#6q$2LCN?+H|6vXC6(H~ZwLkqZ`kHvG<7O`LK=+mzqbHpILQ9J zfB?rNLHk1b7UI(==^q-<*WU&mt3-coke*Sn?V!Z)93)qK{CiJS(^) z5u(t`A8tup`&x()M!cxKi>Q-1xlLb!-v_Gl&>95M9B6Cdd+1p@)-cqhdbCRs@Ze_q zazD{I`9TQ*6h+FH-hBm)+4|HMNI9Uo#&OS19*D__+O$;$;Nq+D(B6pV?Xeejzs`{1 zx)8kYT0MX@G9(umNx{c)H{j7kGHYq2bfEyp=!=i>(UGVj9RU&l;LgU)gA9%5lZez;-6st{H z+xJ$c%)J>*d?@JDh0AbpY9x6(#{GJ>Gz;jV5Cpl^R6UluKhlf^(Y4w2(^M|Rh-_t;HtP4uy zKVReH9XadWIi@H2O z&en7Owj-lD zLRxXv2g;8=ixxylP3+k}MkO^7?7G}@Gh`I%5#vrUBS#a-wMF9O$_04jK#ZL(D7RnP z6HEDSbkF^FkE8FEZ?=rE)(`;Rb?8)!^ksXjFIp5hpzs-m(Q8=KwI4HC!~VX>H6* zywspM@7n8k@4uA>O5^E8R9(+og!J?h*pH4mi+vLzg_G{{_I@P?4*UI`n`ee`nO7pV zv#wkT83gQpWe%1%R>%F!C&8GHsFs+n`wDrWNYJ<+L?Bd28@5UgTEss|6Oj-Y4K`N# zL+`a~)dES~ZvZRoZN-6*n!xDwH9gZUSWPHeh)I6iVFv@8*sGgw ze_B~M21y&wJ`m=J+FEIN%6j+?&g(0=W_Y@e5#7Uf~hd}69^mV40scEX`|Eh=xk3tfK4Yrm1pDWwuP~3CR*xCaT@{(ag3E`l0By)M}v$utJ|NJTS z^rA`7?0d}8&Gm}zB&X}?JuPqG^>pza>d~ulk9NJt%!10)sQKqWyY}i14d40x)z+q| zdedRuZ@D)}VnUKO7qz@Xbh&%X*yubxUS6xX#e{ehDOCqP9y?5Bq^@EvA&CE;*skp} z*6PFjr!3#OUZAqJA_w_wp2Ea5dA-dpQ zN=QNP$+W6GT|t=#o$)S3>M4mM6-8NXdBREBkJ9R>mn^mDJ%)SL%j6P9%41%LZheIkT_}-oz&SH-=`R8o6&1K|;#2c) z^<~c{9RwW#c)UMt>XD9NXK5M?Ox`cp8wII(xF{0Yg{)BRG_FzCo2O;Z&|@cQhA!V& zbRF$8zj63m|FWI}B+T#%l2J4TaY3;Rd?FQ_n^2ML5;%|jDq#w&DgKYITR-eP_0@M9 zf%M9ITsT@*4!WHGj63hh1*D9jbJLSqi(Y}LaM}|3Yte`lefW<`QWkCo)RxA2$4_E(c>7)-?#nD)D~C7RE%~Roe#>Y= z2`J%04b>R{_gZigTGULbP%~?5L0ct8nXmX|bjp3hKJ+=1qbE}FeB^`d^+yj6l$l6@ zQ<8UUFN2Wz2XLZfvZwS*Y8ow9N9+#I1?lgDTiDzPjFP1c`GVJJL4HI!>+J|P}pRE`*ji2rRZx2e!hh14FOvxWl-?U-khB=Iaw z{n8Q5JV~3cKN9sA@DHwf#ABB3j*=C*6XgkawIr2C;}@9 zB`WEIX(EVoN{k>yXeoYOc=~MEcsF9_g(iURmQeAg=7&##;k9uQIC5_>+0yyLV;UL4 zX3Du!1Bv{(2+Oq8`2y`O!Gd7mgbLk7xnc1T*mhIExEg|Mi{~%^TJq?~Js4Lv6ES=v zc!<66b-A%VJ+Ld}>Rc9d<4i=&bX=mq;!68F6u6q-CqY{LUDF!~wP6+ExETRS>EiPh zKdV0)boZaE1!+^(jxnDZ1cM2KSUU>RYeU8@{X1ERAAaDR1gwmF03s3YIFsQG|CL$e z!FVZz#*r2T!evQ$=k>}%Z<LUVn=a=B{Ieb`uE;=VbUM$60uOEy zJp5TxrcVnn_aWv&T&QCHc6js7InZN|3ZG8O&+D*^-U_VX^ zq4t-t>02|fl%$a(VKxO{N_Pe%J3XgtFl$@WKTe3tcuQ0jfyo_{;xLDUSst{1S|;IW z$N~yJDA;6iG8#OQ*DgUj^mb_YF24)}mc#i6)R;Qlx3}q}MRCCzY*Wu;@3XyUKO;oU z7FFn+>_FszU{Fu`RABU?!6F^k%Sqo24GE^nU;LTg>2DwR8xb>W zxw~IBzN~0PL;-490##N7+cmvN&V)nEOJ&E3F z=zFrm_-m4yg9+5q7e)!?{}*WdHI{t?u{NWYMMERlf)ZHQ)Q8XgXqOf)vUjlJc2>Kk zr_IcN1TqDGx$INJPD`E-JlG;m|0j)jjcrlI=tt+hKD{vBVTZ}#oe4?sMu zxuUsO@(l@Bkzl|!QG2?-)T2PG{_!nv*zEl$Ml(GK1;z_8^TL549Q9+r69Z63yal;nk>gMF%gYi<&&KbZ^GO+#z zI0~LMspF17KH?#HcAtku!6^J89U2;;ULVYm2u!|?} z;5hT%DQOKa5ySD!&$%U?-UF4OQ%GT6;=UL56+a+M8;Gm^ehJa)c?`w@Ea1|XhA7Zx zu8>dp*5M%oRC?(Pwc+lUP70Nooj=c467vlGiwoj4Y2QHbOg(REAT`6BVZW$dcj*7R zPI+7LLJ|{r9=0mi63SiTU110n?6a{oSj6DID_vjZfpK*iT={=?+SYk&2E@bQaU723 z3gEi{Zi}@qbgw4C{rgud)CN$1?RTQYd1#@^N>rr7;F3w0=9?S0nn3c~-&Wq(0PDlk zMo5tvEPIt0MU4;|ki`Y<8+g9Zv<6j%o8hXkSRB8-9rmSIYxKCi&+?_ zgTSsMkLzU`^z*4aKTuE?lwFGT0?QCOl*?kXRtVc2j*23t{y*W&4|E0xOTp$J(uuQJ z189kj#smMADhWV$g=jbFq%Us?f(oO;LI_F>U88Es7qcaAf9m}!G6btu$d~ykPXH5H zUxCua7XoYrUmgv!@zsxu8!g$&ocNR6>YfQZFCg?Pf(tU#0AE}{{o-5+HVur}v@OhL z7R0OBJT?FN*PUnUUAcw-5BXwlRRh9fgIZ2N9Unh4>Q!=`1dLKlN7`Fa-U{21*RKv; zIfL%Fimtw<@hbfNz$Gy2mh=uNg7q*&?v6bo1Pa_@H5J~VMQ1qpKmNA!`F8j7;%OT` z-dXO4im54oQO<7wMC`K%p?g#%?c z>egF)9(J8g_i3F9Uxk?4!cj`1!--rdH*rpMDp>xbmq2pHE+JmkxrlijQc zU|$MfbP&r_%adfbBtS;(z+>n$cuOflu1d!^852%Nu{>%(TpsE4BVJ1k~y7M#+YEH%1Aji>eb2q z)eS8%HmdW+M{%eM#`Q@uDv=Dp4{!U68+@ziygr+=m9y`=VNKIeL4!5slmxBp z2sk1>vJCDor(=L5us&X<>07s)_M^=z&0pab_E3xX>5t$-EwPqnV^MTf;VVX`%f8el z04YY|CS*AKBIw2HbsF*0oUfyAJD8{Ja^UAfE*=M7tpc8UIKaI`&Hd7#YisDr;lu{p z-CPrwZMo4AMS1kZqKeFqOf8L%s26FkY1akppx?AwdliUJ#vVSfil|8yc2@6qMA^ zyf!R-etOL{c&aw$V5JUc0ti&5FMnzR`)OTCF@nY&hAc#O@NGOp-1}2lCaFnIjDqbP zj+s_5H?_M=X3#hq=LG|CgrFf@Ua}>i&SO6{Up@OK#CfttjUETj;b4{u@GFSw+9qzW zzZZvun>LAd+qpP#ye^&hXs-f3drZ;JQMlkqPd&hH+y0n1_(L54+smq$mDXGjvYfT} zY)>;~RyJj^e|qxbWox*yKzl`P`76ZKJ0KR%=F6PN&0T%)KxuuC@Yapvn{KHS(G`#L zbLqHKFUy(GpA^szXRS2&T14;uN$Fd$G$xiB+ixPryI<)?iF#jJwLAxEJ%oUbWQzSqc_2-kX`THQvHa6WhqE~Ml z?2Q*(%e-Fd z7FV61lxmCiCOAk~p@?I?p=(elOT7&?Q`$?W6fb7X z5U1*ZK_9gx_%fy-EvdqQ!=u;OtuHw~p0b(vadFd{^zXP7HVL|VdZ2Dpe`8Z3n6n9QUuowU#zMO#ctm3|IFPN`(nP}f}i$l6RfgTvGT*BpD`W-1F z#+JkxffzyVs1=F~qh&<9khGjuwIpB&lEFe*xs5|wtD)lzXvYjz-?PzYhM6nopPqvj ziyp2T%m(L8*#v1r({>SsylVNgqkU~&#d3ZjHcvz?>2N`X%k%Q^e{Iz%d?}55Gd+&T2U94ks1}S zMXa?@ao8hkVBf{^Po0rK?YFLu9J9B1|4Q0ZAB&Rg^#}RsJqF)XeN4`hNcMziKQxKn zI|-<_5xY0BQO*f}#CJ-;{M0YL6IDE|->klpbM)YSD*%!pwzVyEV3rs46R%&BnD}z= z9y;FXfDA1l5xSphL0z_|;uemrx<1u!;5c_8g(%ZItfDI4eB7zsfw&l1Od1LD&XzCp z7UfcK{k^n!!RmIKLi7(k*%=YMd%+2#3M?rM{*=QmtkS=Z9O=5CC`(r(GBI#B>Yd5w z4?vue4DGKmTQr@)glES{eapi5(-oa_T>OWht*VU~01Ly=XKlawruEY_uDis{L>rZ~ z&qTvaT<>}KL#MW+dcSbDj5^Ka*YZ2al~ME}7DH4BqxgQV#vLF&i6=u;u@Lt^2?zsO$x3JPFdHbH1XDvcsv7pflH*k~Z0wti`psiQNZGy`zHVg0dp z?2sZ`m0Tvd4a8{dTING1mFI!PS>)X^YJ&BY+Kl>KdEY^xAvk!3a9VK)g4BWwz9gl4 zVc~6P5!LLBnd@d6^z%2Q>0Ou$7eVnfCt!s#-$wKil;d+$=AgfRKov+yy78*Nmq}Hbqmykoa4^4>Ad<`v)X2l>~@J znQZNfAP8g=F{cBk<9o^3njrJ`lsR-VNOek$|M%CR%#|RXV>+-2+#aW%5_~@OM+&&X zlA?5uN}xT?@gDT;U!T=ke#yI$%y{IJkDq59Hh4mR$T>8!a0ymh~aM4^aKelb+b$i2lgnYce)twI` z>MXJO+xWBUNlX4Wsm}MVMx9)vNZ6sseW;81@`{dAqY-?{62@gTcB(+;9P4E{t7{P^ zv#Ncjp}KQ>&<5iw#yJ&ig}Uks0nUg{`yzB3W&Ko$az7_ZmTVI9*LH;-XJ}+y1ZQ|y z+xC{A3MY40hXU}nRRt{DciK96INNnUujAVLfE(`^blJ4uIe;n_VQToP;-)>m(iSQ5 z4A>b!FjCX?7f+SP#wBAsuB;OfhqLxiB!3sX*+~i?1zWePqtx@3Fg0T#QZB&Ffe&TH zwUS#Z@>lX-C%fzRoI0GCUtp^#Y)dO>*CtG4wY)6|B}jS3D zCv*p1zYEOS>om-u!Ioc&jy87ARa15(_skE#RDP|qH2DywnnIE~B-G_%u0**=jfphz zoH(vz9@DjuJ2?z_S;t3ZGTsZO5IsUdAB?|WuIwqQayl(zj`5{#$0bbng7fN4w(uB` z(@Wi;gN7KSG*KEUr$rh8S0!q$U!Mu-BnZgoJdVv@z69Exov)~!%B|!v0i6}(Id%Fv=Ax@Um$gQ2weP(3r@4mH8-nGUepj9F zb3IkTsa_k&IaICV#O_$oNM*AJJWp2T=u&o6(#uomQE@x;oj=(^=fmec@Cee=;1>Z; zkB4R8-rql>-nml60v0O%2}rt+LtA6K|0k`}1$8)G$V}phu(TN=@}XHSj|qK8|J3IU z_P&b`BW750AB%t$oe5S#ueE*g^wD6MY+`|nEh-0`^po;UAEUJyAn(mdQ(m?T2#za+ zgA?>Cj&k45lmgco{vn91tW9~)+mCIH7IPC#jV_d~XWn>ilzpwTMiB(196g}5F7#_- zc(87CSx0h_W0g8^BUYBm>#!+B=YFyH*4uCYhd3z}*af4Wk*YE(jt~A)>pfl8e zbgZSl{RmXLBHuu0XMzc_)6r~nQ%@Ys73cS4c+rCmg9;o%DGVRSV}}k3Xw|$nvJ?SJ zp7ju<4XzlXT2^XRyEyGcqj^n+b-IFLfJo0CeU%9Cb|z8lZ%hNnvjINwlx(n)u8lAp z{WS|b!C+^8#Fw?e!?`V;DX;$6s^hI8N+I3|FePjXvfk8Aw#p4bJYrz>6_~?%un_GM zxi2i9?AiO-7Nt?u+YKgr-xRS^BDx-vTxp?0h0hO{KlK6rmeh(t%V#so8-mt_ncV|j zLyf+)+%-RHRyU^@d~+vFsV9o$%179E`F{GQ-;>ubBF7;L8>V!#4A7Yg!%0~Csy59x z^qJ`5ny$(P39@alkS_TB1z2Dd5;;D9e8yr2l%xy??8ZkNl_8w8f`tR7IJvDfM22mW z^CJ(Fl1fhSu|&5c4d>%99`H!jiU1irRzO%#OZ7HFpyMJ74O#tMqF)dxH8%ga>yPQq zz?=~gfBaVQ3BJmdhW_X_Pz;j7$Ua$~6tt*+npPhh$)^F|F#C;>F$z}_wxI^|KFWGXYSk+&ph)?`vy;VNJv&M z4eby4JT<#zH_g9o_(9mfqwOrzp^wI2G}KK(+QQm`%t;YYAJF-{`m^likG#AoN*8=z z&42nv6pshFP)$Dpeq_y@;maRMh4!0u7hZO!H#e%73N*PDPh^nB_Bko@{zwgbp!4R-`5Xkd0}v<ql>RO@H?f-e1^`ilt&iSUIF7!HH#%{0UpQt zTCFhYYz;eD0-kmX+%|r`%3e!zJ~lH#5U_A)&p7@PckG4Cyx!@A8an#7$Y3B%_sJF5 zouR2eWy{Xze|(hUK;?UeC2J>sC|~4wnR&7`lvZO*`2G2F()hyU3l?$8Cun~z&gbR+ zm6XcWptj)$pKoK$X!NC<+k51#P>%qUnA?H;&t2wwW-S~uwqcYqe%`4d(Cg(BJJROm z{8k1BG12*a%L;31j*K)l7FX;pp}c`tJT><|`qlD67xT$;eo|I^a?kh|M~#oKjw5eI zvw=<-YwFOt?1>>^K3&y9h%x{0%S_xT+Bz~oawbYdo-8~ z2pb##0YXp|abzX1p7A%&u99eJ>Lyjyk@H>M=mhpFC!B8r!0hRf`3F2hZUS@Hi6uS= z0hn$DH2eNnmlCd2R!_HUugXboq2fr4slP>an@ha`$DMrPBa6#Kz9Kv+RQa%lsMbGz z8}8|0+d=e(5ax)L4Du}b7xz$obCe+Rjsh#ruwih)UM!{Z*;GFTF(6#Z_zrc6Aa)xo zFqyzW6}}YX$Dm|ZPyU^dEUt6u^?{%C>quJsflY-0Fftay&<}y|fGOFE;GsSYr5fUT zoO)0ryr=Hpk$7vx_N(o9Wi*okBZtR)t=BlDq*UfrIyZ91Z6EU$I+4k)PnuuVZ+RWV z^j?^kOpR246-BR=E3W<-8rqUVMRy$WUEgLwqHW=Cb)tQ_rKk>d zQXEmVEWq7}$9y8v1R@~m{yXrg!uR1fDb_|IcPya!FOrAtgW17U`^#0^fEVaetWme2 zjO!k=#Dl+1ORpoN6P=LyzC-48_vEJ~Mw&V=H$Gk!u5WgxV4@xWw+kx7_p>URlWI>f zZQw}C)|kHASin0gOLUu6aNeV76+@>GRpqw7R~m1evq=V41eNp95L7NLyUE{)uW-&p z1LwOz)CzuiBnN*ab9&;&ufV7uOuesOPTR{y495WHAn{Tx&{>5UH7B`5pUt4ko*QxM z&c{*bji>O_%U+Z~CZrmmasHt^4n8hybQ5sszqr6gM+v|-hJM!R5mXBZu<4_rIV+;7 zbeGIf(|PMEE%jz}v^Sf*BkWbiwFOlcz^ch8MIlhIu@;h*G}qeF`o*l~2iwtL(W?J$ z+p-40=sF&*b^Ohr+xVk)Sa8Tn3yEbwuPAdK#ms7}un&J|D*!Y#{zSbXue=_jw#BOV zATm90V+VM+z2c6%7n0>0v-APxh5+jvt(hz9(BUUB&!0_tuBy%01rI7G-YLu)ndLY) zgPJ6F-Q+64uZ>ZBuh9O0!uf-`vGkuF^X--leW>!wGGgLmsxs0WD~neH1igQQ@pHRs z)!$wgJ{j9+&em)ClCs5pb5D7#`>`tE-wOIemG#hnXrJU45`2Dte&JaCd3zyl@Z;;Q zZlmI>HAIP9>l&d}6re2r`N|r_9ce(}@+uZx#&+gl^^&Wzu8JY}V5nXfaq3X%@tu<`DaK z1hA!uXluJ9e^zI?sjz ze>RX3He8upDl(c_o&WLhm7DI-B1!z5)$n800NPR<;u&TTLuH?4hYcx1a~bB1X2NoB z3du$z#Fcjc6b51wI`$w2iZlf}>>-5MfJ|^1GR-{hn0I#OR6&^#jtJwPt*+BrSvLs{ z2BKi7vJ;)mBjgPY3uWNJu*(A+p9$d&r#H4}UE&MjUt)V;LG$ zrK~%#*d>1wFaYSNOOcYE@SP+Gu7lhM6PFM%H)rY&f1&9Yd1oySApVKY%$>H8JA-d{ zvU~tjRPJrl9Wky^R0f%+uRz1)>!L%VhO_lI-~yqP>PVhujrn1z@iW#LM z?s~MY&MW`NBIBx>XQ?Ead@&7_9 zZra$9>--_&DYBrL&#Ftofl(-?dJOlwl4iEVxW*Qi` z2AD1?^AC2iWRcal!X=L!?4&Z(Q>^BLp&7HLGWly8@rS?TG#zVfw%=jSb zs|!Mj>EFNM{U0;@#U92WVt9dq+>!3Q?|x-UWJ6rEe}$6oki`9dcNVIIJkLT# zG>|)9nx$qz3*@$XE~Yl-Elw1lYC1cO7(|%_pHL=vP|KOJ>5l0WU_MeEZt&l*{Z0i2 z^VxTB(u|^zTkTc<;lg@f;#NxK{#kLlegv2!XS%#;38Kyd;w|tXm*#MdvVLSFHzQb& ze^ZJbZq;$m@{gG$G2h;n4`=&KyR1S5Sg#i&4JmfTi}$A`GqVfi)1ClibhMK(!~ke-N+rcE(X`Vws3hr?9HoqWCV&i+O;T_8#Bhh02vAUN#V0r%NOg(eGc5OpUL>aPZW^zpO_Z8~<1Q z3{hVZ?Q1F^^_mzBt-5-^{zTGt8{1VtAP6eX`x8eknkX^%)ApykMC1>4@NoH#^-2?g35^NA zEyixqY3w3N=l*(G3C9tma4abtoW{=3t8I2Y5j8x3E%d>M7Tk`VmaDlpLQVFN zUqU-C%DHR4`FM2J>NR*u0zsmzw)c74|M)<&x=&5;iQn7=T0O;03UBd%Gy%uu zA$F@cX)kadG%%|zyB!Z*<|!jPI1cE2QznkZ4`OLJtETExyxea}RW2VAO!~A`l%V22 ze941FairZqRII?V_+6_axzigmWmxxpi(Lq`Nlfbr&$9b2S`` zsqb=}%dy4ERbWPgwYG;Fmg7p}y7kPXa`q{Z9@i)jAP44vfVyqV_lmo2)pS(UlxNRO z8AZlCh|)d0Qfql}q*`~YK!|6`6E7U~tti<)tm#PjvO8%v1}pcIab_!5%4WBUk8*+n z+ULHpf%nl4xNrEw?Lf-|3@%Zfpi&eDadc*NWc#4Hr^r@vX@G<0JDa6(x;SV~=zqA1 zijK$m4GQK=kI)Xzn|WcmP-etKxaZ#^nLG@`fxXJm9!gi-63r$p`lXwbyTkcagKmcg z9~AfbSosHdc0yL_X8HCSK%-+VMes$V@H`bE3?jY`=N?hgvTFcrX8Zg!CT{%?YcG6@ ze8R=0&QeA}FM+H!KS>0*x@Hc5F zn4}b#G{HYiWX`Nh*FK(drr(x}`?DOJhAx0WQi{W&_Z)wevUi z$v2*V9jT*w%^(7oVuwG#@$B__wIW+H&80)J~cYSwl(p){sqzG$8aLrcWCgOhab>*>A(LBW#;s7q07PWvlbig=n<_ zcMhRf=S{t&`U&<<{AY^%J8ZPH4|X{2A)JJTa%FH154^^Hq_ya4-N3 z*W`M1wUIs=cIm>I>uWWvJWe1dXJdDOl>>mv4R5C`))`6|75XeAAN?H{(f0R0;PE7{)DHHN z)4w%*UK^EsdTwkE2P zopk%Us78NP*Mo(P^VHCed|mdH&5m>2#m)MDl0a%^0pvW72oqIfOM#~S$rh@b1t3NI zFpGB@({dCECnYWEbMThnDhMyIx`~QB&}DftBZRGF>Lq+H&OQ+kC{ffrcF{2TeIhCf z>5=-5s4==xv&O8EIWaWfOBosEXWG>`VWk3qdUQ4eNz;N)#(brtHj;+w*ZIG~CKhs~ zopSb};Fa4X4e4m(o-=15KV|KW^@m1gLVeO9thx`44VN7rU{Fe2Xuh748b+!@9&o@O zv`39wrhT?s8X&c1nDQ#i(41pfJJC{sA7`214(CtpLuc;M(}Wp{)j6sNRhyI+IWdPg z_o-K=-TR$BwcHIvW$EXSuFYP}f^72vbB^|4JATcd7Q+LiG;d+IrAyzYQZ~ySdeq|0 zC*^?_jr!jA+iO^9pJMEg>$Oh(4koMhP~(kxbS~7;BE~Sd2=;1#AnnjpGNy6!5!SX* zBK;%6>z^jBQ0wsb^L7I}jZ%@l(*n(m<4KewL`mItd~~gOe~U0pyK>QCIpdFw2`av0 zH)~^Bt|7Qvk7=1o!FfkCIHS9Tu?V6_J!IAgI4^{gc-)sv{M-6UScR~QMKL}-*lYQ*6UlO`V661_-l#;nrYNvr!-Zre z+V$r4R_k{lz>v4XK>5dnO{qT_Lw#J_|J>&@0Qs2ssiH#wf1ObGqI_8eX`gwUz`m8d z3*S(w!pRe9N8OMhkVkKLWr2gLS!LcDM$aOk(#=v}Oz@-{bg)yU5cI(_)Y*!F)P18h z6Gl7bb+q-zwRq*IRg+r_Q@zkzm)_iP@p*eb?)4@N06fsJ5W4K6+w_>ok=Mu!qmn5` z4n)g1`aqS^-k9GI!9xB*m5+*XD?;ByXHQ+&Le~oVlF`8E$+5c2TBrKjH^B}& z!8R%Y&Nr(RvOFki-<)zTzpv7v(w!~u=zGw|0*Eobbd9F>=voYw7io>eZW#q8udo)+ zV1FY=btfU?V2~Ia)$3On zh@dy{SF217QxF>uzNq7-ZA?=x_rvwXS6w5Uew7m)tx?`40SCBgc1uw2uH_@Dr@%^| zF>D5T^Bbj#58q{d3)`v29e}}-hTxJzdaJrg;uW!t{6%e^8GZDH;m%6ef^{j31wB7~ z5?^Sep{dzb@)=#oL)sW{HV%uB+f`ad&h$Gw8f?F}$4#foeFN>~!b~vXDXOu)`N>9K zLjIP=;kths_l`1QEwVruT%)07jG3LKh&wj_QzvU(u_TuJNjG==@`YbIZFXoeV^Zfg z^7?`UVZN#Y6)5pK&3VfI``yLC#@A`3I!8i0m(!q(+G+=-$1RmWoQ zV;YJpXkKz7U*kb}rL|wpJ3wpq0nOI7ZK7=H{Bl`k&5thN;nn}{B!UGlrvA!k^UcQS zZznt=%w$zy`dY!YgA)6_sPZRcmJ*A#R$^kM?`5=&pX`N;Y<(RL-P93nn? zlh$g{S31^TM>GpdhUxo9Y{zf3Z`OYv60g+{6m7j%ul4oX()Mq!!^ykt(raa^@NPz z#5q`CQd_c^)Ghhm81m@kb5`fd-JKIu|DXMa7RuZHTtXt0@Y zQZpXz7#F^u6dnfAu>Vx^XYw;#&AgzTn%0%{Dl@`EH9+3_y3$^KjPy)E>m9j)+iM7} zm8FbZmwy8TG&Sa%gvi4zKpo9{j=2+QimAs%*Yl!a0z&YLzYcO1_c90s;?=Ke2QH+~ zozCK{0%Df(AM|e%dq%iALu!XAHxp%LE}%OrivK$M;sIjIaPL(mrcW(DmD?!fYB6qR zQNn6jp~PxHGYC|5_~tRnID*(qK5193cOOCx=PnOCJtgEL*9@w2@7kqaL<)W@J~1u| z&QJdGEYYzdw3_8dv=r{ehUNK$vm%l|uOr_JJ2ZjZu}|h{%HtiG(38erFNo(b9|kT+ zjUChxX_&5O*3)Q(FF4gFZYxXg>#hhR>`6US`tu`~-|xnkDh}{+RE%}!s8$%aniy3= zVV@{8gN77eV!KZR1kJ?NQBK-jl|O;ogk4Q{jmMv$_ox zzu6^ixE_wxI=&5VWe(q+5y;Oap_?cZFb36hi*b`+3p5J zYzi+n;Z4#8_3!0P94Dhz5&02jrs8Dq!&Pj0?m`SuCb|vlPq8o)CnJF^|8Jy$GAgr< z@K&7mf>yFAvs=Z+PCu%Vt+PHjRq~yN`j|kEgazOO@4;z4mgb*q-(`rExVuA>6)yOu z8ob!c=6&1Di;~^)LS5{9@w*DLR(QsMp~AVyWn?ZmF?byC!Gfr_BItf=@t66O&+PlY zDke@Ez@PdWKHef`G_nt*WV$;bPVTwzr~rJ82O`O|m` zAFBcEf205G+kfA6m?;^qk9R#q`x9*c8gDVQJ7zPwY-U`bu0Ppl6J1`}&{dtSi?cto z4IAuxXD8~_#4I-6#kk${45dwW_2jx9rrVTeqF>oyoVo$5v<~HCkEZgt95#Z@!r0ys zB!w`XRO|X%uO?zmv>x3q_VU`NuDsV;j*v3-T=c;;d4Gz2Kb$LEQ6PHcNmRXa_OCH; z=xClzPY@e7yg`lxC=?INzf|MXa-OTJDfO2nrOaPFo#iiHVZT>BUEoVjqDbM2hw$Ilq_!px zDODPD0g0werhdEP!nk;&6s-OSdX{WC8v{q%H#dZiaxE_{vQqJ}!G@hHQ_fXVp+BVI zz1}U;mj4}=HJ5d7)!WX7Rdqt0TQV+hML@vHx)j0rdrPn429}?5)9Y50UHhqSXlWOV zW)|_D`g^=zVM*){yARSUB2}2r2JH!$j) zI8x$*EgnC8Y@cTemNqf#@=iN=hmNqOS)JyJ1~IA%Cc=v!Mid3_n|F!uHSH}qM5Q>0 zFrjj`yI@nGaFtfhh=~1Exp~LpsZKy=E5kUMVX61Br!2^A(*9T zBuXWcCrs2n#>VuiU<#xKaPO78@pQe|VFDJQyb#czb(!YH;_Vk0j$ z6bvfo{5^kt;WCkpSn$sxs`9}+4oACsxA#`)+oGN7##c1TPnxq^7SXW9PIodf z{Sq-nBQW~=Kx7n1$GI_TO1f^zj0k+@>*k+rc?O~7u5Mmm_R)}&5fZn*Nt}7;wn>4V zggR{+&pu-r$PuJ%{<7s(sZyh`IME;Xw6)7|e zpaP?r;5H)VX^lGRb`)s#Uz&$h+WQ_sCHM04plwEjDY|T!^vJtmA$n%7-K9z0?89Np z$QY(`icHe*nj&1-+W@}P9S^ECd^e>%NZF0oPyUl$?deaJxIp{i&qd)D#^Z-TcB>lkz8P!bE4MKeY5$VtQfsHf8S7xcvey-4DeE+8^xQT1_my@qeYh35G-U zFGMb{`MT+KS3>Tq7|hBT;cOWi^;p=t96(ltc0SP>+?_RB;^`g)s^KxHHfA9?<(2tZ zOFa-KD6l-^R+FE-avcvN6(4yY4QbE`!ikhQ^P4| zI;cs_RD`V<2EbkhS26Q_{!c=-rR|&2c4hKa&7AHI#l$*9vHR8rgPLtzl~iSG7*l?Zz<;u3)okX5$3Vyrp<=j2 z{=JD(2K23LM((&Ut$5~|-H$q1_$k;w@EeaCPzz%dM8|;-x3)jMM0Y*7tlOJCWMw!& zXXrd)8}L}1xb_{GW5}Vuu`?MX{8n?16ZI^|K=gd;1=~Buf^`+Cvea!pqUK@;f)chq zPO-^yWm0GPCcGzr($#C(gVUmQpXBM}ZnGxS772-J@Z@;)f8m#YEeV+FJ(G|a6Gr3J zP$!zDsm7~-#4M%Tv>4MD%m2`DPM9c!v8ZQO}i}fe8K+Irsn4U#fS40i3}6gk5ogt%RO17eeR5-5PbS0ta)t9{TJjN z0%j=PM9jeD{^towTYtD#$9dYdj<2voynV? zMR|qYejuc-7Y2KdfUB`p$$LMg3#$o(dSEh9LG>Job=H~4Hk((}_7W+ro=lhLc7L%n zK1?NcH?%Osyo>qg`;lwE!OEEX#`HL$D-Vn3_t%5_h3^)($2NfzAI055FS5V- z7*nHtDRJs8A-ToGk!qh%Sfte<3De{x~sfOZHfyhc&Jif;`I@*sBr?;HOeunH>Av-!S`l^ z;PQKeA7qCgFh}BA$>c9is@keSdY`D%Cm~k^(sp$D)WZ3@7sZN<_MX|td7ITQhp(C~ z2A(yfV}y=nJu1Ki4i)Ww<#G->U?VlzEOagVWQE07(>c^hq9ml2r@)VYQY5svCCF4f zK5=s<+?ni3c2hdL6+=qBe0ovyYnzaKJz3Cq)JxUOSz`d4_sA;a?P@j7o&N77To!`J<&cwS?lt0pBYk>M?K)o zf;2LO0f}_hpZI5bQ{ekjpiCEY;vP>VqI?{y;{8uMG7zo0A0&AVrl(iS;czu=2$!(H_USt*Cav9Y}j!W+$vQ@PpexCw%?S<5Qpl;au4abx(?ppG{}) zNRN^=x7YvHxB313y#so*xHWc%As#&YibqYQqq27|E!0FaSeiEOP@rgKs)U zXpS&ZH9HVE&MjGp;Bw5o+<_!Nb>X@ZyT|!oK$Xn=tNRcwG_OVGXh~i20WmEnLl+$5 z-V3FyhQ>i2itcKwKAhHBJAk&|k-(oK=mXZ`t~ACRIl%5LX&OI`P)eOn#@`LVkwcU# z&AoXQWXb>NA55biKIU+rJsw(O-%|8_mo~C4+q1Q=T+GL-3l6iB8iciJeCts=K!rL| zTny^yVK@=y#5Y%c`fm4D;yop8nL@3S4W-@tLcT4KtgQjR*_%B$_(00Rx@u9b&#AUr z$b0U)=M}1PWX3!UK!0y7LWvdMtg=9iorl#c(t9Kf!9}*QN0_(0=-)t0ty+;27#QH7 zbGUraPkeZScxFZ+chUQ>Exu1B`0!1V_hldJ7KgG6XS6ZoC~e7J8Owa>_cJNGzC2(6 z&B29@OSB)kMl~i%=wO3An6p3ZfcM{6Oz3mH;5r+}UD`qHwA7Y~8LCqsEui7A6jtbI zz9}n*1U}F0r|>tmRu?r+v1W(%AyuQ8{%`0&=`}Az?p_>J=v?k4nA~|QuRS3 zE8$!2(x6&J1Hl`1X<>$1HYAYlHhQu9G+7*=3L6cAsV+-@lP4#4_-Mu8PZN#C%nJTFbF@Rc^f1TIY)5S_n@rprx&X1=X z7#JLc8`;EfVos(x z#c*Y|YqL0q!KKcN{N}coKnTW~5ETE*1z(BOo%K?M8j)*c+bEG|*UIH#f_rq`DZIce zPufQ|6#UJ-`G|f3a*a0Eb2F=p9E(C$<>8cjgX34Qy_3O&V2gPTjDQR~SN1Ndc+ z=FgUi_1wha$G4HM`v za(CvVZtzS$uRQ2c&H}6XFu8NxfO$SSWT}U|K1*c}rrYboAb*}^VsM~vju903v+2mDNs@Esh@R`&6iKUvr>7yEEC(0gsp6fJajqD#5%PM6cV$*c(iq{U!-$|%Q zY`pFfY?*APgej*}9m%#;0ATcM=D*c<{2w7nYHGM%^J|EPXu*MxeyNl8M)X?!xza1a zXLKdgmmx7Y(30kwHaRa|HxS7#=!F__dlu*}-f$H}<2(}D_*lfEP+>Ga_ z)E~f9vuDPW?1a%w0`#L^xP#lY zLKi3;a%WW2TZW_Y2KlmjCC58t=7*&3#xd-Vj2;?2^#7+%re)3qWq>tv-F`{_r6#^% zCKTHtV51d15@t??PMoy-qus-&f&yYEAl*sKIu$R%ECN^ySHkf;6Z&k)fAJa} zTy-|v(8vf$iDIB+h$lJ$VW{!VK<21^^E9!1t>yFHT*_`qmH+J##=WFd_qmWOG90iq zioS86iLzni4|1_94ZPb-2A9n(Dh6edAsI*<-@m>!J1?0AbbBuDPVsW+Hmt-mZR5M6 zq4AI{$Pd>|C!V6_@0Y;-rJOr6c{myMtqZ~Uh8$04fIO#p_EeuMc*ltE;={lcMm@dp z2j#lOKSi4`kVwsnV9pT_SY<7RD8O5N2i)|g(|>W=b~xTL`cpd~9j;c7J$+bP6MCyB zzB0PMBs6VQ+%=DkixQO3+sVwPV^Qkoc9=3WAbbA`nLt+Io!aMBWH9%kRm>!Rpj zV}U5n*rNAT)`_X$FROuZASircDnjwrvAwm+DvUEuXTpI$-Qd(mL`v;#b7VQ?zKQ?5 z6N|)jnQoP(mxg-2B1RztsKIyjub-Du8afvnOiaJ^Kh8pKy7n|9(a=6;RHh!(6m?y0 z^_|ZzG)k1-d9OR`c@?^?nqwInF7CZLnmZQ8%5cbO%PwwN*V!cxrrmiLJFyL)c$Y1HyggyyZj#azjfk^KUKT3&?Ow zWj?wlT8d+u+3Ho+X$Dmi__St)UoPI*UL0TRdx%kMF3RwGRI!C2j^^Dw|Jz3D3e&e} zXy^VpOD;Yl%0)9avjQwBO7P4}<4MYbQnP{i+}q4WFe;L+N9L<~RtgoWuKy8i&+yNBPxk$_(DZ1WgHkM4pkA9k2S z?c-vlyP*4C+N)~jrKh}=@sLb~!s(q%8v`J95l~gE3pvcHWvI?G8F-FB{q+t{;q*|{ z#3R26m@!U%pUDYr)sb`;iIh?dAV3Ux@qgy2G`?T3M*nB-Gm)4z(M8=4%=rq|C-8L2YbXM5_!a1qgq8BQOAa;_{ zckrAu#KVgKqn9|+)HSpp6swbLYVD^f&sxZLour=JHf>k|QY|v0IJ0{(ozl!#o={{e zwz1IKG5I1A^k23)@1*4rFhRL*mrv^$3q=Hn?*vto+dYRVS?!IpV@ITvSR}5996&|M zY<#o(i0Xi<=I;*&5e5)zp7+rfs8(w!P_kBr%h1zoFop&|9^f+8H2k*%T_3|3xTcc# z*(F5OXl6EksI=z4;gklEV1X#C)om)_Gmz-Ce;U%+7So1R-IDAxoy=zDEvvcMM&y_> z*#9%c6UuII_i#}UJAe#hj;9L-AAmsu^@7nY=rcr2xcMZP!j*_?o8U567^7wZ^k8lP~Raq%b!=a)D4z zWv^M;xMmx-;az-m;TS6Fgj)@XV92Yii4T`pSSm_ToV(_Kp`&n62^O;*;qOE*6{iqFAP#6qaTK)1ur9V+?rRuYEsa0F9Qd_#GQjkGi5 zH6jXDgoh0z_+8K0UpRWlb22^n9)^x*c>dT{sZukxLXY?f?)Ye|N4PxRrbUkdW<)aN zFJbfSNG`L0A(0k`hUK#TEkCu%!{nZ5WdDz8F*8I+4x)vnJ!GW;MhbS@A{ex+hZ0m) zT$)l@a=L%+@C7)Z)7*?88LKuFv5L#!T^4>nZ<|hJvxjn`lV^U(#oGawwuuSQ7iewR zwzbfMw9v`hv2+I-Xznn-cb+%Lw=Wz-+l?|6a1t)t2F~5ks9ECKA}vtnY4Mg)jG^tY zrLttutHehmXO@Vg-wED#O^g-6rH8SG9Mm(Xa~SyE@m&AOJ`@Qi7gb(~xFV;=+vl)# zpEu*L{lYAnzPl?&nkIdkZYpe?I-&~;d1P=H&S-?)dQmz4=x9_U|74g!_{x6KC%pT^qO4+WB^W*$&)6?C)*$+r7p zmJ4l3r4stJrQT~~eCm68`?Wxp=0O5B788B(RFd9!$d~^QALr3`GbVXeyb30Pkv8yIClKMFs+TwBJ%Wq%k zvJbT=RoqVdj&{jK5Kk6uY(ML9dg3EDYkfr$m(Ww$ak$hYgT&vh)y*Hg7k}gMceS@1 zPa}oib|~i-Fyd~568de~?7hbK#CEsu2;0i~9Jkc~x5Yv_QojODl?mJL(HqhsEu0LJ z^S#U7KHO~2N_}y}q}@munjC?t3v{^OvYmpzcdzE3h3d9vzB?rDb1iy*=yp1%yNIC0S6YZM`=d8XA4`ccN?w@Wuv` zqC$Z!YKNg!VALq9z=n57)a^7QYEdW4zrWS@B0H#A{|s`wYNKx&w=pj_X?x?>e}mt8 zDL0VOVF$pa+o90V{+6%@dnV*1icvI!K+RvTLq!%A`_g>Pd@AXhp!<-Vh%H&#M`yny zfdJC40HHCh8=p7=v*_Hn-|{nF32KQ{jycUdW#>(XFNpN>|wj%MFRTIw`u5tE+VRlQki)g^T z!xU%HQut|nSb65zXE~_b1zBs?pdg)IS)h6(3&+s**SXEF9j{?!)479@@Sv&VU*I~1 zw!cP#?s<3Om{KM7Q&mS9$-p&}r+wcE^PmWO$SL=%LP$T)GB3{4W9V`s>kyUpTR=E; zv48lp1RowGV^N=`X-K$ly;!==zGdN^sfUL=ZY*$zLaPYg^ePwRteQdRTVEET9{-}i z+o;&Fu3h#ldTt$mHe>>l=ODv#<-g}TnzYeydd_MyvQaD@6mT=)A&Al?LdcYfK7N(W zINz;8ym-!O*5^4!d}Owk`DY?^y7LjD+9QAUTad4tzI?by@e%Er_^*8H$k;F4cFysM zHwL(cyf$9Hm;a5eySPc#g;cWQ9>pl0t5N9Kg0kkG z{8g?m>BSA@&L)Ord}<*S4qU%oHi_nYVUnltP`7$p1;c|xcImCPkvF((i^nN#x|=x2 z(sm^10z}A)P}Cf4<+4A1{Qi&OdO?n#_D*^8y~TVS_F<{4%-9zxbu1)~5-~%@(^Mcb zYN0C6ZTV82>EMLgy{XQxmFt-%wmyv$3Jk~*Zo7>qXvcw$!azaAlk=4O!BAeue9mPJ zNzc2(<_Kx1RVmwua)49#m-@o=NK@w7!@8GUmsv5t5%ZVy)BYyM<9D7k0$~FY4?^i} zHu2%yLz!2LKx(fx$ESb+7nz_{c~&m;!6J!>i%+Y3dKI(66{ezW9{WsSalFp0WCB9b zv!_xDBF?NMzC1L+3eAVQEEL@wl=ZvfGWy5^$8RPZ+b@J)d&{@sf%9CpEQ~pCGc>ps zX_#2Hn@Ynz#@FNy(*HupkTN64uQ7na;|g;zL68tC51o3^Y{ zZ=pRU5>(z`dQNU?!sMa)x9hX}dwSR0zrRqXL8RmkwcEqsw1kjX`fR zdV%qSkau5(Y;XXTIo36g1*>*<#|g;_BAIgunR0zEI+4MDs9%sGaXGi~lt`o!`r+}sX)o+G zyAUd(@0{*-uP0cnG1O*LzU;+g{bm?%kIO&pVn$?YeE z)wG(o%lDj$MSN`C-g*Pfj4lI@v|qWhBEg$EGu-kUznZ* zxHFc?Cq)Gw5S2qd^=6PGi)q}nd``J6m|KQPuy7h;9SEXuti9|gxwMRQJ7(L>{!_ca zmJ1Te4op|?gX(Nb(f7Fr{Cx0-HsE^d9?05;JGWbd0AE*Dl0;rgqNPPLnbESAG8^wN0Pe#d2T2;F}17V zQ86!;KMYSK>od+dM<**rP1s(jB+xBTaPk93#tVHiirX*d+aK)>_D0h)dXZu$c4?U$@2pxw+Y0aEUfon)NM*On!kMof!`3-w+UA9Jz z*VXfoWDAtlAh-+?B>-?$?4rL1F|tB9oD!T!yX>jML_4o;=Cg3yJh#i(T`baU4 zAjv}6bAdsINL`c>$otE~8h@Ax3NG@rhE{>c&5Pz4#lR3t!+Wdo+Oq#6=_(wmYMwT} zNOw0#E8X2C-Q8XH0@BhF(nxnnOXs~bh|)-RcXu~@hu`-H?C$KI**P<_&ph)Fw!_iN zZ-rdNP2(sqVTtDZXj`fX2@H%-uD7sWtP@lw<(1?F4Ej#2#)0Y(Y{20ad1vBQ(`2Z1y ze%Frvd1F>)?@F*SD7D3-lY({>vhc9IIk8lcfD={)N`HvnK6R~CUY8q0VC#o7;JEO+ z%-SV{f!V5g;xg3W{QlE{eNzOj#L4PYWR-bMqJUs$;J9ASQJZ0_0yK?6rnSr5JvYuxFil*c zy9Z>W7B`&hvpG1JdE0*4eKTR_uMMO~lq=q!y^6-oNR9EhrD#z-NiVUAC2jv-_w@ZlzUQ%K01@m-kS$i z|D*5xg~1gQF0f3=)ui~Zg_cG38df#}-G9Fk$+u;7;c3BNa*(r}`J$q0Jq<=Ffk`lLR${p8Boh58!dwh(E(-W?Js(07YACVAv z%A~y)VE$uO!Goc!oKKUc;oPnyp9HNp*o3Lo`aRdmR{I&y!&S$ae0?rk+RnDHhtuQL zXsATr6LB`_cqLbKdge%P=I(jA7`b8os|@6LONwWZIZBgBs#%{xsXzMSX|kfh_LLn? z83-eY9&*N7FUGvCq`GoP%uEjGva#&-Uh8w6d@nyJsQw%9ynlFW;(EP6!I-RQPXROk zu%-O+6jvggeLEPakZ7|w+c}jiGGk}TMY!+R^j2>SqZ1Fh*VqlQmq^O&5RfG_dMYR+ z&h{za6~hWNME?uk70ftzf3CN*OU+(9ld(jxf2Ds4Lc*#4ucEUf18%NOhj}I*v;fL0 z>-<;YSyHpLWKUqCMtAdSii%WC zAbb*0M)^jBODW$bJ|jMut0dd_aOFoFvBpmhg_lOSNC$jT?Re0t43{Ynb-{Al4#Q?! zgY@HQi$}1ie~UKK73Jvy9pKEI{a0pVp1@Ee1D)T95I}gU%n25KJkMHJJPU*2(~=D| zw2y1wV|ka2jl^UI37Y((ukwtYyW*?XITHtJIoIdY^(eg@D$V3ku!zueh<UpZuD%nYKd(Jl4`MdG+WcVSqbY{F}^{a;xai)KNXD?8M z3&SVs9v*db@>DoDUXhDwyfQaJ>gE2R-&G^8b6wt-r2b#8~NgEP-Uk~ejKw2 z`58Qe{bi>w&v+!ib?DdqsHjcb-nz;!C#hFL12PzxtXI|TVCuqOIQjbrqH0$Ub9VyL ztIw`+))T5@Cw>?bUslFXVOz=|X1dHVl8d5`0?g?tt;n6LPG|FlHH+Vz`DSsej9$#B zD|0%R8r{_)6h=LPzm0(_U{kDWy}`AmWg25p{GkJBy4;OUW@xl>di+P38G-~Yp06^R z7F}5p>S%ivRYoKWG6Q7GO@b8n8C5ifzs(_a@pT~3e?4Mnr~dG^^?>u2NQ_cJmjr>` z^CAH(y)mHmyTRzf?+>WThOzSlkGRX39kJab>e3L%lt7`d$s+@X!lAK3>-jp~CI;W+ zy0RKBZjawD&hl<&>;?Q~$bF^Yeof7+*pV@lA23&cwYk{6%`M-9{A@)X)>k0~=oVqV zN}COO$t==IAvm`dDdn#k!G(D770EsyNqi_#GX2gz^sw=W$BR5+J~H`og)stAe#@bbsOmzQF>RJDVZg;(xyQy~f z^QXqv4*a6_w3`yvjXn~f^TXoyt6TB!B3gjO5D?w*X1oF?-z_u5_F988Si(z?pPLdl zQ87oNp~!}VkJinyMzezr^~S1w&px(uB@WaXN?d!Ga(@k$(sk=uF9K^i*s33xbny4h zzpx<&5Y>2)1+K@9_``-wWrb5JYX8@`Lu=+PH4hPHjCL~ zdopIj*(mDjk;&ox#H-g2HsX6l;$hxmNpQwrB?nxIMU)BBF2uL_MZU{b^zhVXU}iIj zGg#{wU`aw*b*E?^j@#)?Gi~p^*}}@8st>etRDXIU?58*NuX~hJYy6C_{89yKo^wM_ zuXC$r%*G{-S1maH+~i{5i3U7V+}4iBBb#KXU8uX6@z;m+ynh4$d6GuNcbPZ)4q3ZHH_~jLZ<;|{bPQKba$Zr;e zXU)bn`0o_UpK@Jt=XtOo>PW`QJ`l|UML7n4k7?C?Wh5HN0(JRLFM3Z~`=>W|_3LQl z`YwE_EZlbgKn2*{lT4#zIdUGBR7pO-Gb7#cgB(`V&aIS%2C}7*)Fy@CT-!ij-bL2yF>*I_97gywu}?!f z3i3YVY!uIt09Xe4-nRUM5ZjXkOf9#RFSwjbdo+Kb8S>m=rfv7N>Wl;Arx=Ulyu38* z7Ju>-ZYlLF>L&ktJ{*2mXPrVLrIgn;+IweGeZi>MT6D#}S#O4HAKiJY6rY0Xm8==kJ9gG;^@ zQArA7W^0xBotJMY@giJ%wa=xU&@Zm(*_hni`5prUr@2X|bK&`MXj5ODC$B2N; z*bA15Z9$1>n1%#b8DpWG58}Dp_XJ(Heku~^Xsa-Xoy+_OWxF2v4&h)YPF%yaof=G$ zPMd%Besn29lK0W5y~c{0>V&u9&!BnU=gZ^&2B2RnXIvayxHSK>fz*{en-7bnx@yL@ z=#k_G%(Hw7D5HQGrq}5ti&@H7|KI8JDU@)NUqqIGO?I~BaX!{HRV7hryeUdc;3 zz4=Wq&NlTY6%q;il+b$G; z_-4k-Y9()=SutWvZTpAi(^UAUfOtndVwtoUR6g4ocXCs~_W%&fxnV7%3JInj2Ua{{i^*ZWo{vDJbQWmk06xRrH4ZO$Gf{@rddVMSk)}m|1QYOrhes!^*$u_ zn;b$9-4SdpKB~Z2uTeF6w!lo#L4Wd(K4VpP9Ij>9&H261fO$W)ON??S9AmGT78=&q zhx&20f!k_Ezfl{v;f1eRHw&4s^~&7z57>n7qiAE36y$!Db32aD2VABYmK%rcPe>oQ zGrv&-{I(%wtd!zXsTRHp&u;=}Bb>|HE&bO8wHg)YIMz@{V1PzwoRO87di7!S-M;q4 zQiziIe9sKotM2Q(*H&1g5M2UomiEmQCktyjyLV1R>ecxh;rJQITZ({ z`2!-6l^zmgWddway78uE(7=+uBc^d4RWl7Ixl~Vo_dQ&YLu(E!$YO6zG3!Po&@^+% zO~$RG=W_y0=+`3(lMc@qSkcb8SAH@>?iX*pO0-UnDUKQWqe}bWk>#m{VAjrFZW=OI zZRj31{F|+zp8y>lJ6YtE7cf8*a2ByFYKi)HPPEt6}n*(^&hY-V>J*e_|m}ccE-gk@De15!m}|dthD=WK z+J+|9nJz<-3WC^kRey{|g@b{4HIwliw(3j0|GBoaLAKqu=sQb*2wf#XEzYn-<%VN2 zM}#Kd4|yRy)%Fcxaq26mmwv5v?*^4YJ~dkte;~%IF#{da|7yoL8ALo-oddJYRik!yg&)3Cml@sIP6hJaBRNHM`)B_P+!B zz(fx(E*P&gHLu}UoKH0`0}EDozZjcL5hrQ!OI%|>DMKn#c9l#;SEuX7E0*A|L&4OJ zzcpzuw32!j-n(+2>su@k4rB$J*By_)t!%sA0weuWI`VDaYi&DYBYKAE{yIO3TWxDJ@5MzgR2D!Z1fHzAq~@v@r9Pzgybd&YO8~oDeYe^ubke z3#AA&$`opLPFWHo{Zij5hu@8uLXzfc*sq+&iX2tk3i{#RXCka~H~%5Cxd!<4R)8)p z3bUa$K-AHJnRH4oRn#gH;bjtlhtm~#A9Q>xFN`q$lj`xBK9N0w*+@78`o{+uY9iRH zUGkp)dMmGK0D?jLekG|YZxWbcL~4hSG*h56OPDD%!J%yh)M5JAJW?xUi&{x-lc@H; z2?!|1-AoCeI)=(W`XX2J4?;%9TZYeV_P7AV2IskdgXzn?RRWk_rC&eD0Zk)p{a4OU zwpon8fbwKk>9T)~6W3L(&s%?ht-vW47?Hr;zSA-E_>_2|-AXXaU&0%=I zNEq-W#M0K=o{p%^I{CVAT#g4zwj}|SHWtzhm2cuS{`Z7*fZf7o4N%;f;y4X*yDsGq z|5HY;J+K_*okruu1LwX7@U+nheT6@+G%q#|j(Jmgg4!$G$Qfti@`q2L>CYc=%6~yw zow9@fUEEq0mw!g=g3d=S`!$G@antsR!YsUOC{xtnHmkzGG{NK0!!6FPepW=!&dBljX6Wk0Ub#=~0=(mYaZJ(b zo8__SL5|F;v9d&xBP7^xSAHo-e>k2E$_cg<_&U?ga1p}JgRj5DhwtjtHFGF%2TG-Q zsJa~3t)giK0$KikOzlXitGingmwySK*g0L5$^xh=T=^H!W zntjl45kop|x##xxDo1tS$(-P+PYn1eI7Dj3DUYNlo&;dzToH#SSY!;k_SxKtzRE}y z_{Ww($d%KoCj`*`T3(n_A4FS$;7JHU-^1 zsP2k(M2RNGR0!L8we2CbkmnA#*)HPi#~l#=_ZCkQByZ-2sBRJZ4 z#Ba(XA9d|7bBeTWAVh%(MVeBtq!8_hs2*rhDj*M;Js?jhF^rGeztQ5ssHPkA6-DpSPoCi{b5z_zu3sq~M;Tk!yQbz95K3~@CyxR=Jl&g~| zo(ew8pqef7G}31kEyXqjH89-+ckv6>lz&}NU!K1qO?uSnoVm*`vR{kYsQ8*GBX99+ zM(_Od70&y`C<=?HwOvMy2W6W-=fDT%(P|%cVRBeqZk;%Q1K_8?JnX-Av?!$q32CEi z6)EP^z8}8YZkz~+@gDL2`m7$q9nt%Mq1sB5fMmdf2(7$3)*ijv%)G1npyBQNHKy$J zLRX-n)eSF3$~A`z(A71j_lnBo-sYL*-a8_~IsM1p=v4fl0^DGE1mcthbB{5>F?7zc zEiZQ@k&G%a)$YmrkM6p!T{uP+S$vlgjjJPq^2y1^x%M@t;@@)vqHItNKE;QrY z@Y!0-PPD)s$Eo|69H%Mmj~T5NNKBqNLA~MM!}xKB)FTQB-^cUKzEik_VKaA9QezZv z7iK7Kp?j%nWk-^OsQsJORZCzdL!?TlB{>W<4;@M8%(Oqrf)yd61l22hJ=$l+nUFVM zF^K^bI|(!<-!U<+Z-(sL75ioZ8w`;|Yw&C(^wmp6o>r>noC8p)zl$i5hNPqiH^>TM z(i1zT>SJ;As?$@@_4udN2`);SaI*_FNF*hjeZ-00XMP`*yV$P%_WE$Xxoh6;0a)dO zA;@pSmm-%j8`VU?S%4n}2IfRyV4Be~)b-?68Y_nAAk5A$&%HwF=xz@ik%#v>MQCdA zmO@nEHCB!cn4q%Oh`N2YpmRK1{N#(sz{>m^azlqcsh*0cwAJ*{wb(S%nr7bXzM3&yedwONT zJ8WtxV+hVR0^CR#Ci1qsxzaYx(o_@hD`b-$6 z=MHhfs7N=0fa;(7*0(nYBN7F8J7$%#O9oN11S^$veOZ>CZu;R;_Nn1}zAq4c{+6-i z^pO%Z?CWgZfk=@=DjZdjLwm>YJyf?b-!JNGc9s6Q9|}uOD*kA5twlYrN>w=oxyAXq zccnLYrVXg}oY?_Cp5gXoC4gt%>Zo16|E=C8l>bnMwPZ!AX%L~wch#d!2 znnu;80FS|c+mwR8MY2;2e{PYPPPURQ3{@P_1YIM@tyKcSX$_M5icRrCcqCH$&CqtM z??cgNu|H;sdr}s^yq8K4gu8I-OINI9zQ*8kZWGrto0XGAX_M~$nmKT@4Y~z%_BT)W zHg@jBfF1(N>XyM}Pwq7**QxSuF}vH-R8m+@vYT}*++L!ffmxdRIbHLNN6#ES)cwB- z^EI`XaxS-LJnTXM<^}^sW+kw4nQ(5u@p=&SC0~GC8w%U*Z+*e1IP#TN;M?tZeIm`t z`9xsvyV*)-EdbpM<3zJ%$7>`>%-TZ-RJVrcnsj`AoKm1{wnI4}m?G3;LQ2WJ=!ZZ! z#j)>n$UFWG#vuYzzWeX-~1CXV=7>Rleh1E%iUbJr?XaPv8DF~v*5~UKOzw4j- zv>QPD*ST$&8Geaa2E{gh;T7)PcmR*i+cIc<@WW@h~`)t%7)(SDV*vt*!J(2S> z6lN7)RnZivO_}pa62Ti8~aE@4hN95d82}sX#-{dtlD|np`uA z&%7IJJ113R1x(U#lSQW{J@d)dQmTqbTX9orQabT^Ous9rCE)w;*HhYmfKvl=1Y<-> zQb$p^iIr&L;a8H?q-0@hi3T$RkzEQ7>o@C$!mMw+@|ur<4F@1@=#{roGZiY;`Zx`K za;~I~X%srZ&4ft}V`WKgq$}%C$(=}%D3rZ2cs=-_&-2yKJ+102f26atM!q&D%eiil zOs9OA=If(@X2R+Rf0Wy*B)&rQG5jLW;?{KPE(vUbkYAhIkT? z<7wr;b_*plT!ELoLEWJ|YC4hoV@bjxf~wy7WD6f8W_Z50r6`C&PBSNWhU%MpY*Ed$ z0I3AA2DZb{<6U9FqC47c5F+AdjxAr(_ zI78ne3>~J=2-)et1D4=n=Chl;ZtGVkVGht&#rnUXqih2}hhf-qTiTsVfnFG!UDjCO zTzV<9{F#q11x!+d?bmCK@XiP=B+$b-V-h&W#6cZPp$NXT>pa9__YFoNC2in z@xge1nvMS9a8j}B#`PlXJ^Q9ac(hu8!o&WJL%y5(s8IGpJKe|7$Z&CZ zoMpO0m}c{FRM%CXoPFM>k4Ce(ash8AFnk}jC|4_biMlyD<~nZ<6QcLz5HHbaH45e9 zg6In;XG2`HAlu!Wp9?boaj0?ClKa?pR;E`fJT5(|Nw&`>Pa!|=&lw>b}L+~=4+`o`1GE8-YI4u6(LGj4ws6yNouvY?<)9-B*n z^L2pCcz}2UWwHD*Wk`*u+IGfTX2r~J0PxB;SWG9K!La6f(?_r6XxmxVR`c0+?u|(3 zX`Y)}cp28P3_7VGcmFyvm7c8CYOnP=)SkD6(W8C0PCtK7D>vm-F2>gJN~-)~q`=PY zQu%?!WSIf~HFEC2I`~zrqI%}QaoEydME4(ZwR1oZ8?d0DYRS?H$EUGH^xpT2rzkd3 ztyQocWrnUS5fVP#JVfEHOBC8)`KHEP_ zBs!gp9tsWZu!rN7W$no}{5Xpyd}*q!mwobP(5mmkx37HA%5=`Fx4*sM-&cA|tiyy! z&er6QWmoSY@i;fz=;1rf^2f*9^O+BAaX%wXmDpf|3#N!qfAch@0Hv`XdEaTzqJsH< z13zJOTAFAp6-JLPLhAPz-5L|z==WkIU+$M5+)b*pu(Yhn#dL1`FO)9Qt|Wy6wpJ_W0w2F%4<70&mIqm0Z?T7xenw}|y8 zb%)#-4)pyVE=n?HGtFQB1vZBzQNJa8o0uqvWlp-Y@;|L`UT9@~Y*>#G-9_Ya7QcSf z#xf9i2#wx@$QWDZR8~4{=^Hfj)SI&=jpXxudQ0jP(8?Ek?geRp1IV!|sOJ_2Pg70n zt@%4m1gKkNutf#%M{!Nietqd}E0&!HVu50cVQv&Ab z5P?LqGO-d`?Q;(0oWJp*#)nl`^YZfYxjyW6?ICFuj{Nn`1*Vo86@2(vVtzwDU$gaR zQ*9U+-QP8qN|r~`3?Zd*OktLniso&(*J={rP8`oJI+km$MJk?-yXV@3O9;7*>(nlaBTe$VP*f%LG?4c(5q|Q&r_-Ua;J!0Yg8|5Eqam>&NPdFO z3Gw)>;=gMs%(rq(x@S4g`LzYsGETa}wp8}ytTiNW7^BK6eWcoE6lm_OZ zhXj>!!tF)wYr{5oB>->O=c4d?{^wOdA3d!#{8g1>3iMY5YhC&qGsB?6eORYKos%W{ zDYBhTjbaUu>Z%f3icP^=!dAh2pqqD6TgwgYZ%D}%=EnbrM%JvRwFaTqNuVCzhSzVp%aaj}w3nb6+ ziSi@)epdw_T`h^{nP?L-G&j{J$4-ysOfEk3O#f$0Klpm#vte3*1TEi>X#LM#%>+Ti zV_^4F2cf!3cV4*&VI-I@-hXH^XO9LH`Kwi<>hSi2n|_!cwqd?RlEP&+EnR~DMsp_E z*M4eC0TpQu+5m;-W~|0i=sAz%U~@Uigz%|BV*XSnMwLn=3a#k zaS3K&ZPBJE3tjP0&+Eq?&iq&uIgwV|XLpPQy7=+X5=bisvl1+tu{pn}Vu_eyLV+7AL29?=223Gd(nC zwkCSnz6CZfpEUl7J_8*>upbre&unXbXSy6)Y)~y?ODi6g}=NqN>IGUX4-5e z3-toH_%oa^P8#Br3TE4FMKq(IT^5J9#g+KQ8vrwpQT0#uPY?UMz^PQ$XGeLSj5$yd zwvhnnN}dXB>d&|}4&;U;&HdlG{1zpotZmg0V7_{DO)4rEQG}&5?bngm3chb8hGh-; zXk94$8;Ig7>&pduW73OHbw73|QuZs2vMT<2y90z4kbrFaJ9Ooa{7u z`7jDukzckX-bB`a=PWxc@n;M=H)b62^!A!YyA}UwI#9*bE&`xcJdnMxzU6&_zlS8v zx`8Mc;j(n5nzjQ{ba!;w5F!ulC;8BT$NRUP%jfR>LSd{&taf{fDhoO66Z>=i?B7`# z3~~kOQ~yzC&(0l&E10tZ-V!EA#vLQ8rI}Xq8-nNco0}%=7ka2_;*Di$KFO&C>-{Nl zdfT^O{(XO)au@?BZO$^7H8Rvr6Iy<^+~VROfM-AJa9;jEHA{>cBKtU809bD#*v`eg zL>r-1Y+kDF{TGs}2gRa(Lb4|Xb+MXz%DG6q)j*a{_8%=2{7reSC_Gv1#4i zdY(-)1KPM`?RtyalJ`HDGZ%7}G~L=nOh96JB5#jBs(Vhd*+_`Ez|(ggbUtp@jW#He zEG(ce0X^UHj9ni+%R5R)iX>*|V6nRo-@(}Me}tC@tH$CFH+GW|)>o!Eg%1XkE{LNU z8HI>ig%CPh;Ap8|^?j$XU-UF{6yyWIiNtcyEWC#sYpU4HQ6S3X?AAAlKsy{qO=(Ib zJDJ+bg&K2(9^-w0!3lC$wx}}T`CC>fT2{E1OW^;j5-+oMeO$W}|DkmDKdAG1MtN9Y z_s~D*dsvvPHQ&8Oe~f2G0ww3ubXVMKcFl?sj=b3)#iflKBxDz%y}_fRWlK@SEG2XvSg30mGw6iq%Lb9L(LXO^{)K#i}UB-(ZQMLJTrB$uy175 zmHPfdUI6e%zSm6y!;|T6SXZSq&3{R;{h$-m(>(?vM}$78j{HsTtS017PCQK=2`>W> z+szs1mmW)$s@7>vHATiKpH}rNh?Gq5s~`{s5I|NwjM(eOx%WX$pbtAW4S9Xyd|#Ur z%l7xJ8Ra&K-nu&-W~fA!Xm<|<$b7$r?H7{(M{uE6B5IHnYA;m~9N`{eDke1yExV&ehMCsM z*<=mMm9f9;+uBG)Rjp^Ckum8K;9^D<9(S}dD%p0`s|Q*Gp9qLM0W9lmn>5guPrvbI zz*hCoh$2ad*=V#kKW0lirNwr z&z=a}sTxS@Cl4Y;JcfbVY>hb z^kESDjl%x!JK?BvsLKlOZJuHIm628t4PM0DBI1C7#)CKI??(F~i3zcEiN}(;uH8wd zaP!2l5(cOb(ltx>4YB8MC9J$1>ZMTv>`X!9+L5+@!@I$br860f@I5wwkeS9L%J6>i zzg$M9u^Vx`j3^B&w(KoM`Db|OnC8h5@Tpx!qR-dzekX*H4uxR4G9g6v%|_|>QCe{@ zmu_h*gZ+9uI6Y%ox>Mkbq~8r$f$uS62&@O@*I#Uud~ zIxyZ%Mx2m$TRXXoyUNN8H*Dn|(%#k3UM;7J{YO_o<);!M5@LMa`-$Nf8Sc>xr^|51 z^0^jF1MxBpy>l}haIE`au4l9+YAf`bWU1B4AdjzJ9|RTtG0F_{y97mm7kRI{C%XH*wXO3*%sN|8#m%3zt27TvU?gu4!+;QkBHp<{;B)*_3zJ%Ydn(nY5JxZ zORImRBq$Xie_ejD88%kS5^Ye7;gIv=e=u1QUtJ=7q;l-KUVf-V?~1u=&*HbFpNt8r zRFw|@#KEWTk}Pqq)$uRJLkps%>HhCPxy!*dKf|1HFFS{c^Tw>4W=Efc=gGL|u|q)3 z-ySm4P5l;+q~<#e9T6X-^FvBCE5|)|U!)UnIx1C%F09lzaPIRJKNY+`loIT#uT+-M zs-^8h%LUs-{eW9K#z}3&_qr272s3m-Yfc zH1$~5HA9Tu``t}TQ)^dD(a;CM*e}20K)s*r+9cy=Q)T+=3ln4ku}4y(?o1?B8{;Pt zE35aln3=D@`eK1+u%ESx1AX6{r1XE}ps1D_)N|OEd?a~;d>y}Ox7>+QcZO5-_%!WV zjGT_Sq*@Bn$pHc=OZED2KcZsHwlhyDl4}k>u+nN zOLgZul;p8G5s%ln}eYoxs(z=WQuO4Yf2 zshTDkDYT8eFaN8SVQ>5Ag|O_SnJYtU=%wuR5{BGYk-W(bqsw?KxjEgcD`bc5d3#IEvTX!jyHJ&RZQZB^UEjYMP#$y|sILWX=gcs}WIMGt=$qp+kegrV?dk1EBU<37uMO02_Pb1tLeF!HzzbXZmpiT&qOWDpDBA78mP-Ki@&HFk~zfP>?~|fJxe$OJhfHNUNkgL;}-~Oe>h}S$LRWX zT0LW}ncN$4zk69z-X&N1T|~qh|%<@GQ}-`T=WGhn?5>f$1c9c|=!FkrqVtuIJ*bCg=uV($ zwSFOk$SqNwpSk<}s*IynkD`|Js~qMn$3DE_rd}c;fvM@iSo+pN7)fQ^6E~E55cE*;i&Uf{ZS#N}l8$($%|;pJai{FJ~oti&yF;^?-E4vS|Mz|kx7>pS?C z&V|x;aV|gEkavexco~bg?z}D`7L+JiJ+te){!DpNIM8}^8TaV>uqY_7o zW1nwnz=nSsODQ==OEI+(jd${sk}<^V{!(>n0x9fssr$n<(i17MxyfjpZJO+ zc!$=L34=ciF2}bc5vmJMCmX_&IDm)3U%{>lR1SAf34JLWOF$aFzgw|X)et6-BWzt_ z8;QFJw9b6)&2nOf#^fhBzWg=MGgd|P2(n*?a-HW zd@&bT2QQxa#%NKt>Uih3L`wvVmW-yk72yAW`v->SHY4Vf-ZGb5TCx4!Y$45BJP-f( zpdQ5g!L6X+hEVe74SCXhJM5>j#8DTMWV{mMCxKsssvbLWf6R!1)ZtP2>`L6b;6M!r zaZ&@S8PsXF#*Q^~(sO*2Lya-+cwC>o5?Q5|p(~bQ1aT_$g@%|=l|$=eett7wR@Qu+@z(Ren}%HOp*84I4(O!FJ&^qf%r z7F%)X6S>zoxB%~JD3;420fvoyvPV81t{%?K9lUolI_6J zII<_ziRTAHojP|vghkItZExL5OGZxb=jnzReUP7r4oYPNH8SOYwA2}tfGB(jWubMB zTf=uA%Yxt>Lst-dJ1;C_%aA;jU*G>|eq7>VB53QyNHh*Z zqy|`oJ-c6DHblz)F0=O9!n+f2JpbF%y7ZJ+=UO-%$G?p@E|#giDQ56$<-S@?$!Q4q zNHk6ejQh5V$~`6fv%smD#^%a3pUCY+&^2rq;-h|5Fzfrp@T(1mZJr9eoK8)qfA#Dd zg-}cAdpM4Km~XD~L7$Xa-3gCa zl5i*=aw$0a!|CQc0DQ7Iesl@4{b~h2-IFc6ky@P*2jLX0G;i*wG9kTZbmwj84=p@{ zHp9y2N6Az2+EPAVGD|5*umB9q>K|hD*>_coHd1@hwW`2aqdKE6=*0=R+k!7Hl#Gt! z4|T=n28L}nr$fO$uAGy)w`MY$rhV^VzQWHNOXd-|aSs%aiZ-o~D$;&x0m(2-oZ{%T zD^--tbw9S}A>f%8_x=a*Ch_e!?hpSD=4l2*V)x68S~Apn%=3LrP6y-MG`V#uF7u66 zfm#dOgs*WAv~BR!VHdJ_F_Jxzffu8P)eX3cH7S2_@7=CKG|4S*&aA^_CWla{c4%0F))Q18P=Xs)8rDz{BLC% zr(6!IAQL=Im}xlGM5W9))1o7qg2fz%1qZVv1B;#^t`QE=8a2!?X-bH}ZB-6Dtu|`s zrBfN$){f1elNT`%WgZRnUXPB62GEpKCCnM(x8ryj%ZCs;3L2NwKm>~fCF?75o5tcf zYjiB547BTFLMo4c;X88x|0hB37-}TwK-=~?Kfz>Q9KrWB5Fz8E+>4dhK3it-oQuGw ziM2xaf@iM%yJ1Pxd~uniQj{P}Fdu;0Abkf|!uZ;AQ#`#qxcTnngpsWePOdF z$jocl{2iQP&;L5<@0e@yiMvco+@&PeM>Wg1N-vCX3mR?9q&5cFWz7mLIH3`esh>ns zvoQ||z>Q@rl1>gpm_O;%@_YUn8>PT+o6{W4KzG1Ozjq~S%4wPLbaKdIN^btsS_BE+OvDT5IYyy0Tv zIwhaKO}r#qOdBi43rv!zy7Zd@$M{2filF9zdbu)+mOQOZ1U{ z=v1qMugDiaUQ=b?EbTaNzABysZF(jc2{M}9pnqORyhGkM_n9-_tYrB@?9-*`9FR8a z5c5Hpm@4k0cU(L>=R25VuHux-e}q^Qt5h(~8K)-t{!(Na5B>=ykH!W+(kp ztw8mUNG(5PIaCi82nxRO(szi&nsVis*~1?PhnBU+E85_PT78f zqb>=UkpdCa4O69OfIsLK7$5$}=ESJWEQxw?!u|OTYZPz`o?TZ1_LQ#1Fb-aut@X*C zf$dqe(7{_kPT$3rGp zzd>4Gu1phQ5}t;0w!F6MalO{?a0iz#lK|WP= zLzUX^^H`ngn$hyH-`FgaQe|;mM*1oO63R_|2f-| z?eGlCUr}JC-@(ua&eI1U6tVK7oi(8AfH%A&W3~{QY9SkcUvLUSNQ@jJqkC*nA^YOc z3ljtNpF9XHwP{;jow`-qVqq4`e~ob@y3Mr~%?yCt&PL7D=T8hX8nq%Gq;%vtZDs_T zU?Yto|Lyn~)1l9_8=Pa-j%gz*zZU)zm_ZU)F>tOhjQVPAL{u@`B=RrZnuml6`1HTM zOzk;sJ>B3c#B9+b9e63nrM)iOrQ$M8eXx6Z#&0wou4t+#D9uW*XV)~mj9m&1%Pr|R z8Jk%I7TvxBg~KR~Ex!LdCfo%!em~o)2gJ(zlDWHW+)NJ9Ps?(z4MD^ENI$}%>c}JJ zPF4o8*mOsKw+B04StWq(B;oT)*=7xP6!MC}ItL22_@%QX1s}e@^1`eMAwq~67BZ*X z+r?~$UA;7nyTuz(Nbm*8eh#Zr?LS@q_vbGB0t?8@`g;9*7ePeueNuDGxjGkiJHsA{ zAB>9lef|OJ6^lIn@?xFJ9W4nDleNExG2A$>a|UlPkA%CZWlipOTn#SC)fJWGcS`@9 z6f^XJQe9?1+${53oTYpT(o18&qiA|)@DGvV!Ew3w5e?CUQSbp~1Kw{N9w7#D zq67jGAmdKGd?9<=2W|C&%3@qW=62pB#JmE%Ui~Jk=pLVtr9UO|7UmpTbbQp;a1Lv6 zM%HqC#rC&&Kh4ymRanr`hH;b-t61m2(vjFe`R|v(yU><^C_;=a#mHmgFl8V3(_|bg zC-j~e#7zl-Led`2z21+M<Kx1zZ9nm6P5)|}JLjce@82e><1}fnwt0`SrRiq%a6wEzIZ;Xxq4j=J4QAi9rObPq=J@B0>c*5?%!QBb2!Gi@E+}+*XZE%<18XQ7^ z0KtO0yIX?0dvJFM5R&cu-k1N`bN0jT+5RvzJ=JyVK3!8?^3;8ZDqL7a4rDxn3v8H| zJW;%-UA;DuyYXke`M3;)-Zy4^vP=pcUfS^*i5lUBkpHto7fwxnM+U#p;m8up6oZ^6 zv+0T5KJ^H6%IB27>bzyEb=qiOVb zu!vJv%`#-@M0{{6QoW{%Uixhy)yjTP8T(}vHqyG#a416H-)+$I z*ORsk3ot;?gpgQ57Q;t|DFv)WPX#PG(B7?gMlq8`5RvCIuMT~ZD3}w3BOL)SKVXHK*+a9W3}AjEtarlnhL=ZU#^Q=Hrwc>X#IVOOoNiFMo|H2 z?kQ_GDDw0dq2l)f`ZC(9&SQkrg`*_@p3Fum*y9{G^84K0$7)Qo@_6&EdYKe?+DK#Z zeeUVOwVlI!N~*RGasS#bfz`-2jkMF)z8K-zMic|7WZTV{Z?rQ8+ayYT&usFoC6@Kc zzH1JV{^K(W|DA1kE4~4zY+{w3=sXA0Kl`Vy4=^J$`+GrSJSrf}sX*_9!abrB8khjo zW6WJRm%umUcewiyAR=}v{$*&u=_Z)+np1Vqp7irPvPMWHkgk_yjLqwK!YQz}Bw&qv z#^0Kxp&^_8AP0v{eWf0!L#I`0sj>1BcZm)J`cA$G4Nvw^#2-1yd%jfzm9WmVa0^e_ zq_vnMo;`F!*N(BVS<%tzh63tQZ-j?QSgi`_v(pZQ@ANU~D~s#i)pjZtGTspI<9@%; zsQi8DEeNCn>4}+5&skpnw&XVue7&`zJk>ZV)Ly_VIG&N}q?jk-46w$~=y6254Mn08 z_=*nxVRs;5&=P%?)y3*$DZx6$g9d7e)l-)u1kyAl<=-5k2iTx};Euj4)3pEU$*X(!9St2vm&l-?NjtRtmL6xdVfD`(bvvOU70(iec9My| z2UZF{q9*%vySfCIwpp(Sd8;DgiPOS>8sUpHPE2`FWDvJGB#bpOx;G9RtQCxY>#Ph+cEV?gW>C?|!XcY)3c6QFx zkN5d7P1SFuSU_ z3k!AOBtAO`s1DN3Ija(_e_gQ32D~kfH(PP0Iy$Z$^E?A?arF%vU@#N0)}yRXM6%iV zIxZN#IxG7~HIju|QRtY*iCK6Z1w@$rdJ*BT#rr1k69;H8b(O>Pgb zT3Ru#&1TbM3@$@gi}=-19g}R0Z=h)y%rar)Gy)aqueuYXz9kwmTHbg=WCWdHr!d1-bz zTNpE0kRFT)dQw_QWqd-`PSlxD+QtWZsw%78iPQL#>)+~~d2bJK-I*UwVr@bao>$2g zHs+y++7Q(p85KxH=BcH5Y1l6INhB_fVYL4g6sSNwD2AMN5g#T0q z$Z5QA?<49jYC#+$q>!(x$Q`05e~&l3vO(B!9m)cj?pOg zz*W16+c5B7TqF0#u?Fq-+z75vBSv)FTcbB#qZ|=1Xa*;GR2OQ|(CER>REcd7@&QFZ zR9rqVE2Z!H+8j9|CLPjIeBl8Bv~nh;!FNn!d6^l zhouGcm<6^Go`q>?4(CCmHnGQyT9P2f%cqF?>}%2Q)Yfj_mmiBiW$mbNJ}M2oF>#QN z#QXlimU)^s;5MVlx|hOM(G4Qu--yI^LwedKi&OV9F2dXES^}=AG+{&+m}#)s7@k-V z#mvh`zCvs|W5uD-t@f6mT9ryBd$!CDk7hsgmBcdi^9%15N_U^aeFynW5^Z`Lcl+Ey z+S8 z$0y9uR$rbnkW-B6x8dhTpEb)@(itj=1u7n=5^{Chc$RziPqZC{KP#~*N9w^4 zBhaE_+Cx#MuuXS=zL}L23tR51!oM@Km^Y3^{b`2@;X?|6oCuASL0``2rqC61dzH8} zFEsbBIjQ_zt#ROjGwJWIB*>VkP%J=24%dUx#iB?k2m z_VT;@IBEg%fE!e^@q%wy2KqbdnT$;27LnO56QJQ^r62w$N1n zV<1<>nX;FQa;ELq)R%YFQ9}BWI%J0`$-1VU4ul?!vg-FqyV&t)+Ahy>&9)QtWsc zXR_`>=6GilYDugT=_FItmsRw#o5->>qxAcW0}7OmVWq?D zT8Jt97HkMJ#qLo*r8NU;FJTN)>(V!N6pphCKLP40`c(PG@8$xw0@ztMMrtYHGG7*A zah?5j%6*mg`z4BhdZ^qTBO7)>)vz2xS5~8KAEV{N+M)fTTw)puVf~uAsZ(&*kJ!gD zHk7$(^aAt9)a$n~#>T@YS$%yf?tn+2Lp!849#Pj;bVlV19*t_)Ksp{OieisOK-JQQ z$SLY>IDTQY-ZV`>5ujl6aDWSAKPOTiJ#jx6B-dC~GQqOyWAME^&fDw5t{5W^yyCe= zbSL78Qi(g$+ttHQ!~nlZTV1<&ABlXLzku5wS~g=}DG7wHfnwdtFp2pYnc=Ku%hsAu zFHzb^TyWo=(}JluBxM=szbtf`~6L1pmT;Hkao( zoLm#{4imLd%Q&a8Ic;=)5b9%|7xxD2eM=8QDPFxj^~H(MclHv6p<}~G6NkIco`B&!iA=w>fr$jvW#%8sG$K8? z66{u+8C6}%y_>d~rIFd~g}y8IUV2s5Wpn&rc^NlME1wjP1Wkgyxr^jo395q4g|WW5 zB~y!u_&4-r2^6X_uH7cwzyC$n(Upjn#qXNE)sOMpVhh>nG$M7~G~;16y~N2s)KF^3 zyUirU!kI#MLE(pDTb1z2%{Wqi!Nn75Pn92Ay-t~IFF{og)m6k%8DeD_vo9RIEQcyv zK`8-xhcr3#Wx~6xij)eu+1gSGpe{7DPx*Tw&B*p7X?#ov)2VY-Em+V@M`!DmOZjzy zMLEYzTHxd7?5qOOjnXuyX9VzW_QT>~1n0}RwERVjjy9u;T@eUX3hwgi=ruTdsE~p*;WU;h%EBRmdw>6C3GM4Gz81$O`cN>*rXac1K#= zrfb{79({nkcB{8RDx5^Q!|}uCo(0iT38Me7kr=P**TsoC0%u|lJrYSIuBVU7IEpL| z`!r!OLdqJ;EW2mrSK^6jhV`RXP0@5F5bg=HXB(I}L%SbF{e;x_M|7zgH`-{FALB|O zm_LP$Ipgy!T$Rx1zGtkVYpgxPBD1Fy5IrI5g%Z4oZmx`P) z17RgfVvfW}AU4i;==otduST34n#X-7*W)+0I=Z6&YKa_O$VMgYU_DH^n(_={q+R7F z7Z~EEfWfN{OG3s7$Wm0V(7;jd&v(kjqXlnoNp9I9=VZ;?B(a$A#;G`;))?ZDm3>Gg zZOGs@B>CDL51e@-*oX{HTO`Hi@E>bkz7am`Y*Ao<0~7riEQ^^oB^TQNpE&a8GWL zK6c(|fu}Wav?*rg55F{F_OS%Hh-MH;&?&pD>ZEWbmr{wgW7kH36bSyi?&?z?E~Vyh zOUQ#Dn`YxZ>Vp(j^I4wWjtXQ;r#7|zi%%mTH1j3Xn-{)2PBF;MVL3+HymSs_RM7)L z2X<*E?z`*!tz3xQ?Bz1&))f!3jGB#9(eWIE09OrP#WKA3yrK9767vaWjaBcxUgx>X zX3*!FOAx=9ubHB2o5}h-mQBRLvK)PLB-=t{E@~eY605x+yker*e4x!AkROi4la7?B z@zjXk5GjVe$b!WOGfF+m!U-gQPhcX(MwNB?_`O?6xdx0I&?|NmD=H6qOM}v_9*fNt z{sph^{-|PvK6L7^hf%Fai6sEN_Fe8`&mW1pgD_0QOauA%TVqCvcbfp|y5 z`|mb;st=S=6SzNiXYN%zJidv=t7D!jk#dYS>PN4j%+5b(@WY|V_7UK{TXMy5l$X;W z=GrjMQY;qPHh@M&!Km4>OyBnWKvBK1ClNYw)StKLAzVJTMnBc?PG$GMO}TagAC zmhN<3%`zGxiqhQ@_2a;-&fJpN_)O-jJLq8*<}Y09Rd%l6z@KP*4GWS)ZIa9^N~J5` zA3sjsA!hwq)=#eo4l*G(D{P{H3qx}&A+SWa?<^PSCsb`jyEHLa!V+Msxq2gp?l4f= z6}&L)BhIYfnS<9!+O=wB+5*p&igWyh;x<-7fv?|`WpBxzQbw2sGYHQ z%_LJl1lN#T)X<;!M-1F`tQnyBQS>yir(b|YY_9E=)Go31VZIZp5wr`-LIE|UTkjL3 z#Y-pZe2q$O%rmCCu1k}j>rA}DwQw9HWWSZ`9D#dWO!4C`LGnhk7a;`q0y8lZJ%G|k z=CE;_<6T=yf>!(Z^MP6U!dzuz`^CG!EOTu*ztYCH%eQ)?=lr-T^$yxas)eg5+S0=V zn2h!tCQjEwoZMQz3E0d(7vMST4i;s4bz?KiC5eN0C*l#zedP?$62QcD?3JAi(o~%> z&PHET^Mb+=mX4zHDn>rs45nh#{qb5RxP8eL43*(^%Qz&FF*_6A@_NvMLRiy{9tveX zbY#d_{A{lwMMTfojLCp<^~);hixUJ^S?ZfomzURSemq0Tx>utfO5`Nx;tnbH8(I_n zMp`e$cIjrwA7n%yE;pU3R>irvyBu{|@0yv(W^jT~6K%*_%9?{nO4+hZxv6zMg;qa8 zjI#Bq6&?(z&i$|rsHaoC{X)k+E$!wb+BS!N(Ztk$t=Kw?Cu_38=)?NUWLS2pjNci& z!&&^{SIA(S?M{+V7;X-94Syr!ImhCoUYfwiln^Ma+nE9zf^ehNZBfk-#<@}HGCDlx zPA>>1W^x_~jRA&8`mGDpzCssMYmtfOnNaP< zxPFnw1yX!E4_NP;HDII()ew8jvth)_keLq~dq1W;^qZTNEHvzv5OC8Wn8bLDVAzVp zUs(3ha<(pHscU`-)tfDPQzOZ^2p=;+ya>E=fQ2WgWKn;SX40hd&`)d|L*ZD6+1*z) zy*VP)Xsq>WZW!cG9=Gz#sJ@CKmB9`kpEz58oQimsr-rg9sB|=q;KI`m_TH7=okD^4ken@8*xr6fjO z>}hgmTVE$m?DuOVne(bIy7ROl8ky~~8yN-Dd8yhe(7{QhK4f!4b+ml%peE8RXZ%!B z+Eg(qs$o=Fw3zFA7Q@C~3Ksv!eP+m2!`|EEzPba&%zSF4j{a6GMQAL*5WO1CzPR{T zS!1uNQ@ULF;&ElE$R;Vz&mox<18c{U-E%7v*yA&rUDt!Ko{wO}+YDmO32CcPEoI3{ z522Q_c~qc?3$V+1ZWG6QWE;0fC)4gL_63TBx)ck{Oy>IwHAN$M>+oAYDx*|V8%w8{ zwrE$0^O-iwVqvTcRu6>AGtlZBdw?(pE`-P6&a?NSYfPhUMS(zfx6&5d6nGnG_;M&G zkjYhgXkv)s`V=Wi>JIi&aE*YUa(&0r)NtRh7gNf zG0KbeJQq?D1avVTLqB90$LiDyoWvsFrrrJc`h?-V6Fx-TNUKfcLrQJvghSc^lL*=J zZpld3MxW?#iFG^)*Qt6M0ffZoqn?LIk_f))VRH21oMxMtueh%!X(sR9N61N(+oD8^ z_D{=YLN<6kZnm+8uY5q+hm8tvtdnO|mg`ZqCY>MBDT0mk^PO&Ly41|=pm?$uu4BmC z5aIZ`Zkr|VeKFr{CN$3q8FJh8>Y}#73xpj53tT%dd~viCm4Non$Cy^1Y@{NISYu+N zNbEVHOLIud*@rH1q-`C?>FGPv_9x8IO&Y8?*Q;Te6-PUk& zWZ@GPqK7*09J?w#4&KHg(%8$Bv!B)$U!WOAvXkNQ$W_hLHQ$<}z3t(W<)g=&;EluU zI!mx7rT=RzRt2?*F86>U8-m@m{2b`p68GuvEv=$yCM?1z*y-mxlKP?IC=~U`G?{~s zTp8sz@*Fu5{2Mc_HDG?Ds5h#M|3TXGD&xdH84eMDXbw{>g^+EB85TRiz z*rl}%zP`FX+Lir#N(n}ETDYzC#;+Dxg9XPJnsu2kJ+8`Wja3p$0wfm@GZSu9QZxNe zl%)4IP|j|x_oB{{o`Eptv}1qLCihhzEw9$18IiMLm zODU|O)Tu&YYO&+um_I~d5>#-MC2t~a zX9sj*KI+{5bei{!FgP5uUTZ~(l!@jWF zc{?XDAv1SKMGQ|!IPpbIZJ#PIju$kIzGXP}P6#UJSt2~HPEwf1AG#uA4h$(6B|v+| z^vzU+@5?OHgL!6Ofu}5`oC-S#F((WOT z&JMQVj6m5fr3%3YRCQv%2q9a%plkz3KYEzn5tyr@J|nb}iB>9U4EgC2^i{Y%d=ft# zkrbi)BbOgJbhW@wlqmD9Hxb^f+|OlucC^9~Nu?+S1UVtHiMYVLVgTiDve49TJ0_Yh z@@M{tB;)WstfE(XNl!VlZdDvweyAZSIl2-L3!|>!Q#STXT=JBlnRdODqk$Ek> z$EIs!B#y$f(ny_NCB$F%(~HH#Q>vXshTMI)opMyox4eTG)ZK1sxWI&($XB$&fNr_? zutTH#xGh3Z!jA|`n*lry9C4^VlYXnGl;${iw`9Cwhfn#yf@iUx*8XF8tZF!P4|8Go z43_#P4fJS5X^->W*p!GlrV+|_JlKvC*|sFRU_F?w(};tT_tluR0SWh z+f*!}qX#cne6Q6?Jjz#LBTd=p4NI=_`~^8+sf+$6bv+Dw0T|T6`b94zmB@NcD>TBd z(Jnn=l`ebHW~x{|n&tU{P75M1a113aHu~w10-2ZwbdI9VsMG(h4~Kx-@Zy{>KQW)q zZdEF@Q1m-LjifDlBmD2jOFZl42{HL$WY88o=J-q`@jj_rW#nzB1u!oe{54W))h4lf zJ0AC=AGPFXU3h@%C;1cJG8Z?VP$*|(PWc>_`#73DJ@J8zE=Kt!A5`w&2cf77k3#rL z|BT*Uf-E#)YMs+J4HLAgw%pz%USdOq|IQ!ps%x=c%`>cRF}ujaku&PYr}R!t)`CLF>;-%TX1BL{Ru0~rocN}O8Pg*TJ<(_7flHP0m>!+`xCIoIU1d7Zt#JVjMnP(r`Sv(W+O%Ly&x8mo}@+YH~%NaG4qQXi1^Ki^O z6{@GPI%!wF`_Xh3p?d(UA!K68+MLl5Frf1e{m-h9w_HNX3@JaN;!hf#C8}#Nz{u89 zke_6d; zF0#C}*KjkK$A$;xcrM`>iA+-<_tbh7MM8g@)Vn*KSE9H6C8iPaG#QvF;IX!54kkyh zvgl-HW@v*#I09EbXak|x3yfx|*{1J^+QanJQ3Xa&tx+y<`F;!dDhHDwfcSH4=1qnp z=k(SojD@;vM+m;2lmAk&BB7r;mz?3$Mu=K~si5BIX6P&`)|p8nn>smaOk|8f7GDqU zwxH_>Kpu01H^Im+mKk06!g^#snju8x)cXtVrWF$YMW~e>^W}?1flwsehU{o#^E~7{ zoAhf(`;Zh#sVC&0iS=&?i;Ih8TtGFY3g0GWxAyXWSTAsr4XqvT+!2~2pm+p{u8>;~ zXIz9mfma*76silUd@JoYKYls^H`3q&&I(RHITp_Qr(l8{-1YkEbmwi%c_$iOi`6Wc zj)?G#XzV3l0_Rr{4UI=i9{=*1(Q0+#1uA%Z;!^p>eKo2sRfLE8_kM62**>~8fN20- z3XNdayBD?iqO*)-JHS+&gTDS@@OVBfSEc;ATC8%!^`wwe$7CC?1^zwT74U-J`)dlT z^Ik)LT3|AM^blM*4Dy8Z1YUDE+Q7owgDS*i$VO!O7#qr%gJ8?MpvB0Z;@S~*TziwB zXe8{~HSW@w(4qJ6li$x|%s|L*ZdVRX!f+}-G!BE{$+l3rW7YM{MXI4L?~CcJbkE<$ z*QH6V(rMpfIt|w&%}cfk8tG)wujk06N4c{K5Ab)2R8QBEO)(w59nt1_jEWJ$FAosW>_cVwicyng0k^N@l)0wQ1g*JqCLu2P;q>_44u@ zA)rUJf|y#55=+xp>ghiHx|@Ehu87%4FF!Y~K!JbU@W{FVDfl~}9BR}Qk*eZMG{Ae2 z3%KQu{cRYn)eD#~`+2wg25Trx)`2$q&yN#_?PZB*gF`>xvp3~qQny6Bc}D8%3nN)( zgh4kH%7o$x(B6XI%7PZe?s6IBFd=*9Q0A5p&u1mt}u$HJxIJp{8AE-n?`5^3I? zr1QfmNjlG%{s8bg$`=b{aMF-ex<;A`%K9MV&Ub3f2dt6JYl_w5x2IDGR9V&w+KIU7 z7NChyU1Day+e)Wu(Max$$b(^P8Q`>uiymipFXx^0-K;6oq;I1OvFQe z8|&B#U@*Nb8R+y%q+WNt4YhHbf($#y%y%Ep`6|kX<=dlWJ>#GNrJol*zg?j*6dhyF zR7R3m1=Y=&SH4`=A2ueIj)v^)f{J%5ZhZgwdRY;QRl*Qctn42(+K5&AXNgr~fwA7{ zMtPBC;XncUkS!#6f^(^EoNwKID+y*HlLxpt`qgm95?<6|y+?N|NP;x~a#6Pi%eD-X z@+cpjoe8zs)8-s3PtQW~y1UH^CJ-gx;$~&mACvuagUe*9{m8|j^Wot15H+Du?75xo zn_w;4kFRuu@hxW0-Ql}OQ(pq4Nr{>{dn ziY6C8W#$1*zu+u8xXGl{TH1Pezuf2~e1S*GFbD{=!C{OFY$Z;$;oyF|_I`@1bn!6C z$j{$ryU=J>90DgP|31iw9~~JWs3V8TQNQ=S6WXELnD-aK!ztqT+l)))>Z3gHEddY| zK9sk7oE6lAFP5=y)t%7Z5Z|;sW|3_ztz7+R@jln5^0Q>f6cLaa0pP*|(%JsNZ7pp% z^ds(erNKI~$nPiHw{@yFEnGvOJTRzpAI5BH0uZK(iGSpIGqh{td$oI*ZnQ%sm^&cf zS)_<4X@h2P3**=2!K}=e3P{7!rHbgg?Cq8q94@o*5o{CO;=zP1^c1q{crSy3{LD*j zJ}pZk=qUPn7Et+~-R*tB_pMAr7{}6snS%T=2v@cv27);Uc(FCd8`ZkGt5ttI2pq=h z9Ex9r1ZN|JFA}vdJu=+sLGxpPdaNW;@b~k+rM1yCEi153*7s9&;#s$q73p+r)pZC( z4=)>mR{+ViRqXV3%qoj={{DJ#;Q3Z9)_pS)o{m{SRzQSXT!ft3SGS4y(SuA7BzxF> z44V_!I#UmQ6}#sW(4>}P=$UT?=zCq^7WbDhMXwtphvNiiM)$zJB8s_7El3KdxN}c%(#-Nk6=h z%-(tr^4dxXot<5H0i>YrDJc{TM$gREsz0j(R?+08d`2qj(qUA)4Gn2GAf6;B^Ofx5 z-4FovY;dnpx5uwaYon|4-3CSM%8rp~m@?-P0{XPTM)Qd7C^LM%IiQJIvugXcB7idMkNMLv2E?0r&Deoi(9} zPb6_<`LWt81hK@VAqSLP0j;*mT>1T=wS?ObXb9EvTJg>3AZPj}Rm2Ui_=paJOlFKE z0yDIrgrdc3h3Xc$NLud`KxN3><@s_|i`&q!%E2=fojwf{0&+i&%sPBSCtgkg=!ZDs zgWLUot%ld|yyyy`tVrAg60XDLQ*ri3?0^zY0mlGQ3rBJ08x;ytU~?CmylcZ2nO1`$ z{HRe~Y_|nrxy5pn!ir3FQUD&V8QR*{TZ?NtF+Ly|+`+kpQhaJ(z37!0{pgC#vVo?g z%;#&9$*%D4UiyW$90mTXxWkkSJ!ibDHK6!&6;M#K4xh9<1HqM~OH`}o`_ID&rO=LF z2pS(LF1q;Gf3V=!oP)N8TYqZKpH87V5rca4Ws(W+V(FGuGvm}Xe5y6;P76oyc>FsW z`K4vKWG2i)pg`c|QtbTGF!*jz&@+}Ew1Z_pgH6nJ8Szh9yHPkKP9bqPq7J+sceE|bDvBU23*fu}!go>Bpo>#s6`;}JD;{ow|XscPOZGq=K$ zGh39=r)XBQVO1OY^|sS$#?Epg5f09z=Y)fH@=DKGTl1qlu)|wJBITeEEZ8=hDz%|* zS(j|7>7xI;%soay*mObHY|akOR}|_!PL6c|Umvh%cGhe>K2>t)=M)fgoiVj#b^}tB zv-LDziv$e;wj;*DIH>fD!GN5tp!@9eD{+rfA63o9Ooeh$Iu|#lAPJEaijw}Y zD6m0n$@OCeW_!yKY``X8rfDriJSS|kx0z12K;Y*5p(8_|=yPaL(F6LU7XJN2HO@SM z9uw2VKB=Ycux;fgw7O~YlP;{*!6wc4h8%Q>9}o{Bmc9eF-@4{Avr20a8#N^fuG)jFAew|$RX3gL7P)C{AR|8O_0n zHYZfG&4LeP!+0cm)eEA!wpAmMs5NSt>X?2Wba5#;<)+4GIuflw!{-qS+{AWsGY z#p6kT^s1(p1~LbZo|w6MJ+}Yi6o5dvR2|s%B>y650O!){Zd;K)@W94w;0O|}(G*0u z_U^?){P9oa3Q#$Ijd=+(^jj1IG?+3$1y_f5|}VQ$?r3>kI>U zV7AV0L_o+f;82okfHW}LK>HRdT%P`y%u0?P%kkw~`r4WGfBzdqwP>n-7{vwF$W;D? zApn>t%UV_@n&~q%y?^<;+WPHhm?=+CKVG5=(q~Mh&;RoYocwq0kr=lMA6`qE(sw@e zAoxjW_sHunHHqDst=Ar~KRW^Nabz3BLyxSA2PEFLc>mDpeVH8+ZG?Q6tc1%%1J%fV zPEZBgdv{kTeGi>tZ*74G)fnmEuHu#}zOBg_F>5MX0OUbunNI9Z&lQ@Us28%rph5Pg zyymNP`kmMO?Ob06R`%0=f%WylXiCeDg%SqyuSSUqtg zWBt>hU9&&Ih92<%CL;OQBaijPmVdYc*LP@vzE4Pdst zO(_TS^BvX;KvT^-`gkX)H?jMJkl;VXvHu@OM-%s&$Uy+IrK5$LyQ`(C!@n~pGdpA+ zFvWX{e<$qhyzH+RIsf;XfB*%nf~BLiyA8#E>C})}rEKlpEnO*CrR+`JEhQ|?oh&RV zgoKd)H&gGGM%`6Ml2$DL1MN%RpcRWNbOIEax`yFzI8e_}^(WvERcJ2zDdt>&+AMfDR5!xGxs%J_Omjku(}boyR87BzR?)Rp_K13RviVU?04e|P=Se*F zE|bp$F)usc%DtLd)b6V|y-;|a*la&|llVP-`QAm~^tT2>d7sq%@acA;oxuSd?GSo<~(h7+YW!W3_b$ z_KjISLtpp|igqyazWq<`$ll`_qNkUz*F;!J##>& zMagU@rF=SMd5e!B&+jjl*ihv0;cRF-a{`5h%L5{j*ZHRy6WQqPc3A&RhAc@BOdZ91 zX+%u9Dh06hLflpQ=OY_CT>XB-OoQ#*hwIFjJ=S|;14GynK3(H?_b(j|uHSHDIoCW% zqu7I4Es-l;U{38#SIqP?DL#y;Qw9)`LbDP}VR29uhZ=>MP{=5$%i+=5)XBK1zxh3& znvZ(8P$ZCnFGH2S<{(ZA#crnB|bV^bsggA1xcb5@@zsv0yU_ zb4Nn)^6>McKiZiIV&@^CY%g1-!$~pA4%`_6`q4OiXLtT{>C%kGHn5gG7F5}74wpO$vOWbm?~%3PXT_%hx2wUHX3rBe_)6c zPYi2wnu3Q_jySu;fBhSphfrnED=gZ1QWG6V9{*I6)?V@p!IE~f;QbR=nlBIop`$IO z+Rn8Qj)B?6Q~gd|O;06D_PsPJ-B9F+gv#PP3fSC7I<>UZDVt{`uRxMmlcS)*#U2TY z_9Ao3m+%ThHdGK}l3%}0q9cZDUpv7p($oh!CCgisR_wu>T=p$p&ChgHd(@>>w6S^{ zs<0xVrc-?Z=z&aYDDiK|7jEX~rz^!Muuv2p4C2^q3uRv~TqOJwI%U_>IE#|mgHoUh zP!UIha#0aESNRi`Si7=qy%Ec0QPB%>ignPajNe(|pSnk7TV@A~5oTe475YL1zhc`` zr;mStav?d7NU(Q4N`x<0%EWtIip+S28m>(5-$O+Zo{i_X5vw54H2fik&s}HA zc}lLXn|Jdxv7W-`aLV?#aY{#9`1-j18QGk9DExJ<0^%5KKlezO%AM2~%ys(*qLj*H zMslA`oKEz&jG*OV62`l_lPQUrxWWz1Rmr=x z0yY7Lxa1jnhPW96C&suLwYATWJ-I-9fj%f!j|Az=MqTRmhwtlM{JTsaWUwb10%E=y z?23#JjXa4dO|VrsOFj+5Bbcepi}@2k1-j)g(H4`Eg~;XJ%7N-1JQ6Po=MKNu7Op3l z3D`tpS|xPy1b?=8rmCPM;6XEu&VUm{$&gZDow@$HCy6rO$8^RV}xJv!ofknBczoQp;^Vpo)T7#V@d76~yUN9l|1AUJEqp;QpX~#ug z0zpguzIkq3{=;yG>=g*X@7rzp*jZPL37G3w`;w-4R?Q3|cYl=ur=9o7}2`&fA9FSi&KmhsG#L`6a8 z3W^2BYRi_ty6=8!G#RRC!i=j-|L&lClhut@k2ZHJz-G0p%AV~^tKQGNGklPg!ex_q z?O62YO4Qy`fUw2>i|vqX(RUv$yFf6UCa&gFNH+c)=11vW8zPo{a|<{(A^;=<%N zQLzIq+9>0@+SBF|x>*!`ct{Rhawo+$v(t4u`p)>6KWY}fr%x|VyGZ#+5o~_W-aAMS zghQb!IW(G(=I1=+BrA?k4S6ewH~q#le#bW1d4Od|DcLgKoHI7$Qwsr+=d}j>xb>W8 z!#$t1U_YBL!F(pM5Hh|I>Y9)$nn)WK}V>wnPR#T`e8mDcFH8)hJliEZv+u zfZ#4}6aoUQQcjNU0s=ti>OBS5zpq)P*eJOF=Y*Yt=f4|(J{GHrtCP99rMo_>ii8xa zhNZVVtE>Z%wb=js5jR9;)loLHvo!xNFIhGU3f}*7tGbP;g{`CYKM-{pHVU?XJv=qw zl%0a@e-PDWI4Icug95H{Qn0`O-vnR^_Ww!@T;-x*f4z;Hg8lV29%S~{+juG1|3mx- z@t%U?H48R23J$ja&5n(og5xzmHVz7of2R6Z4xm6BubH!fkvabLO8>jgMZxik#Kuj* z@jq4p((_Poyuz~aQgFQD0_Em>#bsxs;C#hpXGi9I#bxK9;C#ge7;(PhvV$o&UvU9? zoUgd-+!UOzxByqqS6p^pWX@MyfGqeG7a$9M#RbTMUvUAl;8$FLEcg`{APauQ<={dF zzv2R9!LPUgS@0_^Ko9ZBkmY{G1;}#0;sRuOUU31kJg>L_S)Ny1fGp1|E*B><&wsf81}Xo3LyDFbwx(iE z-W2*k4Fet;FPMd$8~pDUPBs=E9$ugX!0$-h)yet)HIbOTrKQFHYbt41Cl6=fH~wEs z@ZUeI>Z}^BrjBmT{~82yA69X7RtZZ_TXRb_X|aDr6bD)Zprvrrr}(erhX2a_zoRwY ze`WqRTI1m4<$k@$^UrAiF8<$!YyTzu|1ew|)R#`6`u?VCvAU!_*j5w0$1Kd}|(#ta1xA5j*Pd=u!F z{g8AkDNi1esK_-_(A%~J?e9oFf7_ZD^2b_Zll@d(>&(v zn>bcdW3A>9bf3_BW)cZKQiR6dFT^d4c2mR+ULRx&?p)s6etU3O8*(2sHrvHhns-~G zwIcRvb2SZ%-IGY@COnNG2EP3)Crq)~h39_1P+OWOlX8Y+%#EX^ z^DHghP+`?pXOoRK?_H=t-yVF@vn|x5Xzf%M`1y8Frzf6+|4uA>F_!lnoA2J4QqHN8 zF88~bI)1VgG`c-EST9n0@|{qA-dH>QM$aA5vkq1BxAAA1oLk@ZmP+|-a* zbjno^I799`iPq6~E=tCB3O}8_=#7=f<*ch(mZ#i3^L;NW9}rVnUlyGG!{58FPdGm= z!o~frGk04OO^#QE%lik5V+P(yh%im#B-C$oJ1>Hdz_!c3A*4hp1=vh~Meyg4!(1kB z#2&|{`%ss84&yZm!5uT#qpUv$wu(&$ zBN#|+8$!30I)U|qo#ce7t6n2VOJHgp)B!!QI0eT$j!A(hsosI9k5iCb;uMU_C~r=D z^OnwS2=gGBPKYR!NFt2(OCCPrXB~tUcT>g&$yz?JjK*P~%4b!j3RCUb4roU=HY8Zl@-EIK8)M6Yerj0#&zf{~Q2ANATQXR|qQN6ss;nv8CS~tz2M5Bw zazl)MyyUh|F~f5q(2!w64(i;ckR?d)S89I?oRlh`c%km0M4DfDEr^csQL3=(YfBQa zj7d*%;@nyMh^B;FLJ#bQp1>~4M=^`GR{gj(paJK$hwf&dVKc0i2Cn+m9?t9`B48Cf zUf^!Ws>rOMjxswwr?H$_gg4|tb4Vs$d&yw9r)J8!6ubR7sbHo2?QfiYI|e(-XhM}Q z24AsB6k6V8TiXAb^r;j0z`BCHv~yQiVtKpC6pb2jSZh*O3?++IrtKUdNLN!LcWLfb z`mX3Br<#UIS3}%jo)V26)Zj^{1dDbOWkFH-wU3yD=oaWh-i$A;`NYcOGYpw_#hyck zJb1h~qX??VHprI9BoltA2r3mEv`1QYVXco*q=#7h5$^wD?=6Gl$hK@@F*CCzi9?JwIWa6Z`)D?*KsHIrU2O2+DBD>0sR;)2eFmey` zR5d|;<;*!h)LSmVTr;}#FVy-qz0#s#Vho2>SnA4gdfC2aKHO_mi1zwKN_eGvZ=@s> zI$2@$xr3`L;SFP7Rtg~-&Hb!X_0mWd3n68kIb|EoDF-BU)4c_E1CYW6YC!auPXHZD z=fb5yu_#F5>Bem6ZshcqoA9_65Ehf21IZ4$GeJR5wbEhhb08uOSc4ChkbAF~jl7C^ zRu=j6v*;uq+-`~K@mqOq$6p36e%p_--ql zbP~kR;uTug_%J{W*8vU6FA~$_s+EcLbokX=S!Kb>fAqv8Bu0El`E0)A_HI?`ED-W8 zay)>BLYx8I$-sET0hWisW^#5Q)Wot$p%ZyZ)lj*1<<{L;@Kd2kN68mkiyhC7)!9MW zA?*`rFo~v41;?p zdb-zPjcRymJ=-wMH1`JJc7{O^i8IVe&l<1Zr&x<0)5oP=9hJUoL!6j^9}L|e=PJHo zGM(21>R{Z=6^YUuer+ocEmdQ@vNSqPTfa3r^3zFOhm(HH5$9z(3?_Mzb7U(nC^$j2 z$iO|_g`vP4eH*a_T0&w&Hyvs>(3hSp5Zu6%R2$P3|2EX6SVdMzz>*dFU9Y`Qu+rC^ zkPIeGLRd)fvAQ7K*j(gCRXl2W2>G{HNR6Ytq-89bl+?6T8!7Q)VOl zq%N*{XKR^3ReJlcA=^jg@BO(J&tWN@05h_`G^a2d zBKnK_P5+4Egnp=0LB>m$XipE``Z;(Zc1tV~%~v_3KzdOQfsNY=TwMAH&WZELpxwRKjJS1M7c-YP zrH8zg=iVmym5RK&Lin%=*K&z$|K7N0z|eNZy?WC|jq<=3njeRwcpAt34v!AY1eCzX z(i{}o2-}Nuc^NQYnn&>{HgE4Ki}*ZUmB1&wEz9_`SFRw(wWvo41elr7VoQoe*bj5; zpCBh;zYdebFADB4skTbcX34RC+M3CY5zVBXQrI7LtUD}L9$oe?{CokkCtM^aM=G~< zfMS&z*I)(4gwD~M+fx1zGiq}z6mDKEIwM==5-e|7U*LWgE$XrVAb8p+Qw3+32@tAFHtcPQ_+$l5vO+VOGYmHWop9Qc6GJnmZS z!p9D7;XB)CZ_BW=@8b`*D66di*PpTvUUBpARC-v1m+hlE5ml&z=R947D{U01ya$07 zQf+pu(s+ov3WDg5;zr=_V+S6&3PX5mC|05$Kda{yf8PY4`lg)40nT6mFKR>-!t~_N zF^^5Fz2@V>xi=q}AghLaRxe#CYjD)n2uYEp2YaaW&;IrbPPQlb|@sjP=y3wqI5|nyDTtpVx!HR6~>mr$#^E!w<h#T=n?68ZNBCGll31236-0-o z+n|tr=%$Xq4rRXubidQ-1@D8Ms+|=ZeGYboh!nD|Ue=IQUVO_?I{(i7TCJokFgh5` zG06!n*zS7snpK)PJbngzwD%3~PGe%d04@3@eIuoHcB>`}naeRrgX_-G@5!~K&yknZ zjc(tIgun}56M5o&&6oEzEm_FlJ0+2`m(*Gt9=@&et@P7I7+t}FtM3vZn0c-edzc96bL6?3aXg_@l1LI#*0QR2< z+n?alPYes6R@T5m(a9Q$RtBH(Cm+DU(FvcP9qLb_0Q)Z(?*C2{D2U>h?BPQQJcoV_ zZfOu+Oj%trVkJPKY=`g=N1dG(tndY!u@Ykid27ec-}n{|aRqn2P4BgVgU}2UPb~Hs zRu?s}Yha?XTY2RX-;S{@uw-rFG`Opl17y)}TE>LM7(6}JDN4pR5(y)?!adYsAT8q5WvQJ+jd3fQq_n+*p(x^5#9__s3fOtCF1UQ*s7l{&R z2(DILX0D+%$c$J2z2j#8({aXa;GIMlU%dChAoYAxC}K1ZkGR8yLh6@>{?|2W;tcBQiDIcwZ{|T; zgGwJzDK7LTk4>xM(jgSM3qN*U90i$3S&buX#Xx{7(_-#uWHrU}*EYTX^K=V}`;MBJXneQ5) z4ch~Q7TNKI=fG&ZtO_wq8`|hO;Ru`mAYl+AR3szQ>3L62ElZu4lS#ZaglZSpMGqws)GJuUEw z2$+z;`p)OURV;+1CorMJ)yYE}%=^S_aD*hH+oR%(>TD7#aJ$l{mf_3q!0W}4>(mBEJ8@!;H`*^d^IYTW&%T}5MX<=I>*Nz*?@LPI@yAW z+w`4a&QH#()-!NuCQUVSd~1N@?Z_UE>wiAEj4ys|@TR7LipoAy@azttdAV=OF8SD& zym&Jdlv+Axs4AAZj6tYMONc0^N zLK{5)h8ky(Z^2V+N(LAgQ0T-M7|9mkqpU_~=v>Juw8eAaW(SupF^S7*l=@fafs-{D zS9cbnlNG_idCaIJ#8MlME8c#ii51{H=4;T3JrJqcWEr!m>fkjOV*>{PDTj{USJib0 z6BRN$u>$(SZCBrn*%q*F9N#WCkdG62`|ghU!P`5Jbh3}KhhOEv-=ui3HU&@ChhLoo z1L=5Q=`C(+b*m??pcj`uH{R`k9q1uY%<2I&YxJw6 zTc9a`Z`Y<2@lRR?w84o1lo|#qPu6mcXAq`xG85qef?WWIFmQ^Q@mSnI&e0a8?uMON z?LalEcD^G#qj=f(_(eHVeMHFm2S`NK`_4oe3Kz(JIg6@M)or?*$>r>>;7r(e{%Omt z^@D4Dg0@I+l;BnVRf5HMBf(n+l75o295yTUY1( zdCG~!{Gq*QsV?r&?7)z{|{8O_sXx8LD|H z*dGQh`68TJY?J&BGy*aOZv@k9tD3FPd0lMbObf|@IyMrEG7Gc7?sh$eORYN+4Kb7z zZ+Ob&6zPN(gkl;3!~5~D-SymIaR47IBbjj?Or@!I!}6zO?o0@iUjVGx>~aXuLvT;n z@s|AwTaiX<#}upS>MR3Abhk8EBm}V;5%-lIag0rn`rWKY;3)^-ed^7i~FHp zHDCQC$|8k1&9jU#$sP$|Eul(ALzEf7efE*`iQ^Mi+`km`0T&htUt@WKl+OFbbmG$ z-?I|>KVN14nIZn0tmOB~|43S*B^kcMg3$4#dVphAB=O$hG5^6>-6Rd>0Ksl8hHnIv zHuCxsPxWwmJ4yrGT1ObvZK$3&d~_aHnjhOM>I-y#jzkkx?KG}Q$-@EHlJY$gtS=&bv}%`#FUW|h#KMIR@aMMB z;La9=h(HvH#9mOYXM16IBRu+d|vPcqhh^ib#oiAg0$KW7cX9^3j~q`m;|iH zupe(m6I+0IjpFBtZ}{9oSs_kUnjk!fQC}!hK$RSJ@7<-d1ba&zk8p&gL#2jJCj}3_ zkYa=@ObpaQ6c>@O`;-=wUSvDNN`2j>kaNdVGEJFu&*qGoVq+TtBoT=+i44jGJV6YH z!-%hQ5qiP!InVzbvOH0=X8bQ=e{e~;6p7GvsZmb z2+I8sox-9nqiw$^xK1ZPueT5J>e_BcP%=B@b2$oR49Q_EKWq~`7BNh<9-blm^sR>u zS!R&%Jccw9%IIYyCAtS!*d3rTa z1;CKgB%L`^w1p--kD9oZwRIBSrIB4?3#F@PelB+~MaJE3=~=c5oIC;cu|t5_*+^a6 zWd}~yQqu8N2iL=(Tndy&tQ;#?n_>#S`Wg)XS?g>#ap?G*)#R*ZE}tdL`)H{pwIs;6 zUCI?XWWYs#ZY|eI*{8nn$*7@32@>wF9q@6ItvQBcAm@_j13&UcgQ;j2myRKi9lAMy zRbbs#wpY1oTsAmDRou(5f}IXa>GQc zspA{Mqhl6)1y)$lq@AgS(G4H z7!p&}JnAWP;S)8&aAq1ED5);OJ?j?CH>?=LI7dTB??ts%HG>k#!-2k;jtSph_1a#Y zc+Wv?+EO6xk1MyD(4*5wVAZG67#SXc7}FD>ByD!}4FIxVHIx#-ieTn-CJs9SOIh_P z1rD0RL9LbdPBkN}SO|6IjJP<#&{i2UCl-RHnZ%`0&}2zT5hwk2b2!uhhE)%g+MN*w z+|u>wpbXWb?YXR0^e1=GU4g$S6x7WWqE)x=dOVvM?zjcBMp;+fF5b#r9i^C@ud)lA9oPm+v#4dOW%Cs$9&^@WE{VYDvS9`3Yc3O`k$%5d+I{;sEnFgUuH@D!xK+5%P;N%bjBozbUv0ian`>LNvN$tx z)*3yuWui23ljAsL-+GzAT4wtrl;S95?w&G?$kf@)@Hjb}&X4g}&Rc7Yz!TON;?=NZ z57J+8{8(GdavtdvORQqL+bS;f$r6Q7D)Wpdnc`@(i7d@r@+fZDaeqhv!+ul1ZGVPSl4cEP(iZOO|2E?uTPs1 zX=V)p++~96xHDVI2Ao{FZxb)vB8J9Oj}hImY@-7V(;o9Af56Fx>J+_sYq6C}v&3UnaRpyWlTXfDBdmM9#MBE}fH0LJqXp9nl zMUij=yR`NkcNR$(M)h`J%bpjnpA=XFq5dW%`3bQ6*HV&yi_QLDAld(YnaCfY(;r|I zK0Vzp2G>8QBEL9+KX;Fwf$1-f6a6m+;Xm)`A5_P8RNC3V=_kwb7Z(zWR@L0-U9Lh; z#{fkuW^8V1=JZY!{981NR>0iJQO?*w(AL_{*5)TQ^L?G5t(C2VqMg3sJI?hNDGQ%g zQCi>85}*Dzb@4Av4E=BV;?I2WFa67DvZfh|D#m~}0?LK3mkcU@b%@zqq5_K_THZWT z+efjm1_eL(T&0T~G=)iUVQc;q|Mt&Q;DOo2!on~i9~(g|#QP*6=0lcy3$brwmn2PC zR@P5QgEvQsmR6TeC^9!?letaDzg%6lKASEVjoGj={60LWoi|_Hby+IeYJJ>6@7h+{ zDxVg4luBSZSy+`^dg#Kad~5|aIXYa-g<0Az&a^z~SkfKSl?0+nN*vG8-YCEt2I^t2 z9=sm2IxV9t-}+IjIc95}_aZQ}TRA*s?zxe*U9a+V{G;D4t0eAj-LmW%5>p?mwjay* zDfzGG5zxKXO3E}<-t;FL9!R6DfpGx@&z3w-PpS=zo-$)aQ(xQNr)2trkbv(r8^MuA zjA!8CPWT#8C;A_a+l~)(&eTCoNv#o^TvbFZgsm37CdhsL{)pL&Z~!2t4(&f8PRySD zCf6HeO{~|==xZ5>p{M)2EWviO-a%C|OmS<3+9IyPz>E?-4}xXXbt623^)1jQeW@1$ zV`TAE#Rzf0-VooA81H4XO1n2GO{FBCWn3Uqt?-SVz8`j~z^=rPWRlF1OU~FJjrReA zN6y2kAhWYGOB0$2fWhtSroV!#r^}hB8v7wXTpH~w)_93ObPdo7k-X69R#11}xQ1pZ z?}6ShjnqXQN3$MKitR0vZ+I$gd+hZ|+s+<2-3TK95h7wNuG=B40iigqdWdme`)8*V zBZUBCz2=VwFa{`6Th$g(Kyp96NX1Js#CNDS&cvkqB`@%U4reI}jE*%M!N76xbsNZw zgh&S~s5DpvsRs_h2RD{1?-G;9gHOlw;$gCFD#01uB)7{U$o=6OLx~H&M6C z{9c099Fi!38_~lwix?WYpqY)fbV)K$q`XT6-+&mR)TSA70~GqZ_U!$fY(j~Y1kYC_ z)yiEl&AF0?F-sK-qsj5pDhbTUF40Srbmu(xn%OE` zn0XsxQ|51p+PE^Y(rbw?6z2{QUQ+ zDg3K<9;1I(=RyF}S&H>MAn@0gF;C{}8P=##p@sJu{QNUwSdt8UOEKy3H>Um~kpB)$ z|EQ7af6>kVTaEo&qyGN`({G;pKLXQluKT~5o+4?Bt&9=WUd-J%usw3m&=^+(Odum@ zhKvqjO{9uyEr3LkIxO(T%se}=R5+dm=o?rlC`1W#u~H~#66k7IE|4%p%t7vIwVp4Z z6+&59R=1(|;glHzp6*95!pNIp4jx9w@~fTJwt<#U^bWjA#d0|Ufl#Ox*Vk7KDL1=$ zRh`w3`<5--G|rY|ZL?Gt*uNoUpvtg<_wIEx;te(;waw27i^E`9kpuVE%Ld;CRg zCu6K(Q$YW3fj zr_^qcuv1~AnoGyQ#x=;-A8X9A!F@d*n5SJ&4_a)wK^80*6w}f&7YKsI3T;NfW}N|n z@!vF*YVijFCMf#HX)?3;1_G7sbG-E?coG1_8gl&V zT@9J&e&$@_Gzpn~Shc>bR7x?3?o?$e3=X%Jd?Nb2RG~L<+H7`#leGoxPhL#|xKEdR zpds{nm;Jj5{J>O|efT08;%oGu3BE6$?~_&-&LmXypk?1CO`CJ4Od5=jZrLw`kdJa# zCWsbAtuAqZU{1$O#)_2iD9n@D0JFn-XeorFU9*)Jjz$gQuE9s8j9O)lBWQ?FK`A_T z=Nqd;26**KhU5X@$5k5aDh_DP69edX@d-Kkl3i|qa`3#`9rqEWgH5WKPzVbqVdUW; z>eV$0lK|~JNir6pnHttC&M}q^pl5Nd>v(K0wL~WlUsAarl`2W7nDU>OX>g%oR902u zB8w92N+K#UOQ(Yc+Tiovjw<=K&m1Du0J1I<`-0S`NciN?%NWNks&Shkv$sjIe}vFB zE|mp6B!eI=P5dFR?tXI_G%EKU%mHg_K#N*o_5vg5%srHaUSYmIt@ubrH0R?mrt`45 z+}Y!uAwGv221L3{zpkSUruB4QheqKo8)ZF%ni z*7y4g;L^$ux=@VABY@yBx6bBI`s1TA)`Br@ai z;lf^`XuIU2D@vBM#LH)%%g~xB0+Oh$M^D=Gem|Qw_l>)itIVW#6Hk(~p9U}j4-yLJ zA~CoZiuMFKF%0q(o+0*zIj5RFOl+qrh@YOJ<4yLO+vOj{VTeFC(N^&1WQ`TcY9 zhtb5l;<4M>bj-VtQs)SrKrgZl?al$^5hoAADsXqYpMQHCil44W=ndctl|<;@dGSB- zVkVZq0@g40`QPfp|4RYuw<_shz4jRY&wvGzT`EGq%shi&(~%930v`H1SpAM!?^9g) zsVzVzv;M}|Upl1UA&dF1W;6XS)zW{fx&MWb^;<6WkC63SGWD;wl+q7kFtQ@*aX zE_mt0oz-)VDNQBLN+S~6N+Hd{tfn8&K-0{?5QQHSrCtEFqNP|bkSrvup@0lZC*Om> zTK)&fni5F3tIo1!`{{=bTknf2LzmW7|CD)`pkU*@3|pQ{?yI)*C$Bim&iqPtyNxbi zU~o7a=<>?=DxbJc>%^;lS;3Kk{fV~rLGh$|Eq2}F=q#;eqsR6`w`}0)4(=6r8|MB@ zAA^anozV$#(73YQU$=7j2APh~#qt-JYA?)M^SNu8HN^tM4mYR^CbDQpyXPne_R#h% z+O%ITzD0qe!9FTpV07Bz;Q~x>>nLEXs)e_Hr;)j4lO5U)QH+M(bJ_L$Ua6pKw34Xw zvKr*k2*^oN_MmmPKG6A}xf#kpso-`YhS~2Bs>5Bp zz9pNil&V`fHbM5=j7kJeXl! zw6rT7_NUIa>!60m8P=e{nq%Yo%+Rx&W6V+7eTuV->sdd%g*@bxw1i|P(R+37b^z6U zq79ICL?O@9@kHrVre4!q<}=GNOvBU7&$6-{Mk-mEiP>YJ;(@^_FN`zQM9f*+vzD^C ztcW_AGXAdZ1lTg9TLnh z)653Tw3IueH^d}yyL~g#LSRTl{-|&3Y<9P&H9Sf*%1aZ?B{#H(oKsSBl5+@@bl%}L zQ2}j*8Jm^<_tXld)#8J-zb~eR#l;dwapQacM&Z5hAlhb7aRLn zSrpoeFj&V1PO&(>Io1s`O304cMU3S?U6bSUjNz}#?b3CPRTD$t?JmvhM=qKJTCB<+Ca&64%WegYstQ~?PP1SWcS1$39}>`uea>RrO~LihqSsI)Q=7Eumo>O@NN z?dfcMF!_a%^Y#=+(^8b5*tR&=afFZ#tb=n!&BTB=l`~R67onAgJQRj%oh+3PuhY%s z=mlAsofH#()Jkzq#}~h876Y5d?=Y&{k9Mu}^YNGKCs3XBM6;0u#en~Ap~gh=O4F$=CC{WxrYI2oLB#8)7tra6MY^V)y%u-=uM ze>UiTxy%1npZ#A7QNP8k|LUdxQB$Qk{;sL41pg|i-eZ(|G80Z_>fe@BzqG4Atzug6i(j?85*k8J@o51*CI%dUdd&xsVjCskPIlk$S zQPbZB*!F7VTwR;&%%0)?#7)_nIbK0XE9j>VcTz5=Dy^nn(Sb!E3yHc?+icn~na z*|8>?G9@vR^U2vpjm#@t&plQ*vcOx-a!I~pu-t-ha!?+TYmq3%fHkxAF+{tl2BAOz zW&`SB*a)x9Gk8pXAkgyl9HTQml|4gFsWH#v_#w@}3UQ=;B_=)qY!oopD`0VF#F$Z* z8b2u9uyggt%AJpT&$v=0Ul3fCA-YXRjLf(9F}IZNaJn5QO4mb;NQ7Ku+kTP~n1un! zATUgq=S?v}G9};T44y&gRqn^3j<@+DfbhokbWGlg#j7_ZLqff2;NPbPACD=_MbtdC z0MQN2v?N%YBc!%WyHgXfOwQXxI!H@MRHyVxrIFD49Jzdgnb6e?H8x%70%c0Ue2qRAkz->{bR40pTb%& zf5S1K!}SgFXD`DzpxYn%0TLQ4$PEDN+uP;1B5lUQ`~tLPyK9IF(fPto}{sbLU=t0q7Bh`43k+{@gZ?Zgd*Yo$&PpYT=w zN+T*I0!_fmydnPzEH8w)cLRThbAXTW`5?2IM0polEhKjkI#;4ym}FSZ07z-}3omrY z5I^KXuYZe}PhdiNh(w_;Ia#g0B76Ff*4b1|Z+RrpQb%0#Y^91__dQYNArMZ6S}*gE zac)eSVB-*zwC@PQ;kmD?3hps!$2y+5ff4e)1S~cMY93Vw1f6wup&H7h6lS zvIiDVx(kh48{YC$!SJ=@jyYF{e;ioqFCa=HXgh!7)_#f8e_rhW1Qq?d%>L)y+Hc+b z|H-ZW=b@rsI{BYwu`~b0rl9{Nr2miG%Jklj_@jeC&jR({jrcih>wSoXwd1>*T+ZR0 zf^6ty>+s&`_~po*t!*5CjwbjGlsV!v;=lJ)-p6$P`*B_WnOyx(DDAiI|6lf}K0%$V zK^LWeVl%lY#yuqV&=1md0t-6+M3R#F-&()&-vL|GA}Rh-R*7S z^E)f0>GAPV_bpA_ovpB@rY1M{BLM-ws_Na8&Oat5 zCc^S_wVj-p9x1fxF>haud413@3(a0#aF6TB4((&dtry4}GphfTp3L$^V3D zC(lbwP0hkGKQbcu@Hsa(_Xkne)Ra6w|F@ePXB8DRMjjp>Kd{G;4IeMBXU~PRGn)9c z)YRela1$RNA4f-Igx=HhbK0@22=93 zwzk1RK>~t;eWdRnC;uTeHPsCo8v5wa(@J$&X5*1Hnx0DSnK80m3UrS8aC#40g{fkwzkqz2_h_sw6ru%PEJ}Xs{H(Xgx#4L zWJ8_{2?vCl%*?e;EXaCdckaa?Ox)hm{9XL2R(C+8HzP@kY_%(zT6_H7uU0vTGLVo-R z3H*SE2lswJK_NwiX7osDm3V8*3=|2h~L9NgX6nVAzT6hP|R+ao_Dz)6mec@-8wGHcXfaW-yOTMppLy z)Vcj&2PAyGMNx?((2DvOE-R?v4@(3#!-rqYm#!v56Cp|8Xk(*n(J_e}cee>3V zU;y;AMfCs1E3^L`#qrZCG1AffBL}1V6D9elnG*EBmi`B*Uvrx3wmZ#;KKoT0R{#~{ zYai*?*eDN2j6MHc$@*7m|*NcwmKG;8;G?Ws>7!bxUQ zQ7F}|C)$td?ePF+WzsoW^mb419iCqtCCdGAffZomjO4*X|Dq62gxdGv>B}y!Pl4NU zWH&ys_Z!%>gGu}kCEwR0Ja0kvwZn(dHU;)VbmL(dDGkd*N&jVjwP2psBrUB1#7zmf zyjIc7vgJ8)668iSC;qp7c4r7^I z2jnKRD*P*wGI8Vjo(Wi#q6e@$rZaBLhJK z?do5Zv=y-Kd@I8pi}0U{N_;@1bRjxl$M^skzav%67#pxw!s!`iAbUzYb(?HGEDIK& zoUDW%qk(^nc<%&ejzS^fj|br2AbCa9GrD0`s$dS;L`E=gVd=WVmkyp8{!1uI*c4=O+S9)>06ZNfJ_A6}kd2 zUx<>-Fu5B960(l`20wZlFgyaJ4`nsi$%y{yv6{e|_Ct}X*-tvYRvkdAPf)@s{(Sh) zon^p%?2fOoz#`E7-P!DS!r89gPA4aieY@gA+o5*^)!asBOVI2=et7DM$2|6Ki$jc3c^r)D57Z-~UoYYMV0uNiFK+0||3 zD!p7lq|On7B;Qv(?fV>iCdXyKNQ(6KNoHRste%EW%zW0Lx;%_y&KCkn;4Q^iB!%N<0WtH+4Vu(-xWBP$L3RavxTE!vJV z^NwiUI)zh$5YuSs`JRhR)>6P{r$ROzrw!4_H0p;VJiR>0dA3dNYCuomiq#MO4&UsT z!x;??lk-sMHT30lI{SR+iCaEzFlNe%~B?-`W{EIst5-2dZ7dZeX`X-lfzi&xg#_e0p8SVNJLGiOp zkEk>Y$a}Fg2xMIDKyBtaaOjWe1@`iI0UD4rteJkFjThf@Qf;-!o_#V%j6xAaN5nBs zhaDQmzM5q{?B-^M({LcPFgB(utIuQ-?JD3np-_p*U62;H6c<7<`bH3nv;L7?5HR;G z5W!Ss+f$WCz50L!v-p_-xvYA{ncb7v8Dy9SR_t6D6VV~Y8f#Dt z%43=!aNI@fMxY)>ED0-`U&emqw)TWu1)#5D>+DFlO|@?dR|cZ5BI_6twpCJven-^V zsX21yY#WUF^|(EEEaLXXd+u1m?Mud-<@@VrM-7NNMxSoJV(A!}{(7{%S&mOWDb`s~ zslz*gLv1{?_A(}N4aU8!Iirg?ZWRPwWLnsjyx^~&s(=6bTR7thYjt~716QM}h=im& zj^HjdSa&G_qFsZ{zb9NKxSgKEA^+NeR1YIp*JLs3ElP7H*Wd)iy1Xibsd~&}n3NLI zlgcwX&&Aljj~!lgUn2WDz@G$-^jBW#2-(v;b_8GaNs?5Nq`GI0MYExbn6zav+{b4g z^9z^|tNf|Hd?g$dBOUx0REE!}kEYuVvo0Ml7eF#Scq&06Ut|X1&bXL9t~2;*p)#a4 z5a5|m$XgDwyvh`tDrkMu2A~w2p-ZCCtERR@jzO*#+6usApOhGHgOy$mt}|kY>9!jC zNRTzpwzG{zL~VQJ zKVkc*q~gph6sv(Z@n%1ZN*tl(adAIAMy#lQ!vVy#oAeB${P=8nEeMR})0#M>91ijl z*Ec3h6zIzEgr&P!`g#V{`9V%$V3%OKrS&cQsIqQUNSE~eMVMSv{Z)3cK~n-=y~d}d zi$R9xy_20q+BY4%vqvt+mMb2ld!vo~fOT5O;D=bsK50^iPJ0QIwVJ~K^Hl658xCTa zqi}dzjyWYq|8d}}wsn6-auezH!}yrm!*~%?5Ny<(u1b7YD~=lhCz#sieonrI!yVsf zTMp2#t~MME0?)!M8FpAG6>kbO6h($=nz(G{QJ zLWiiO7G3=AOgMI|eXy9goJnR)nQplhKjC7biSRIEQZpFY5K4tjW2_@_v@)wedeWVqoP_+=-<8&uKD^Cp4ZF1#8CAC$X)>w6i zBDd!uN+~R8A8P#)PSVQMGrS$=G+DXSCDtTn=vz!5e3>4~-Gnmil*sgz6)5*~)qU{c z&(x`B)9ZXPX1d7}gkPbSyl~57HTY5)(Q4vjR@EzZW9ydtlVIndgt*J9TwAAE%}&E| zx{Wq(364@Gwm8W->C#056y3Vm0K}eUTRz!me8WZddLGHSUY*=ZBXFg^F76!u`nwMvpgQSdonb}#@8}(`;z5|ED-J&>$+oF zy2y#L^Q|JsnNo04L<4NmU9zTZsnAHwuPb4YGWhaN#KKKGlf&uGG)%XuqY(CEpD-P> zdRi3o8lBgpBqb|dMwaV4_^?COg*i9+bX%}U63VqB%5&!i(U|>r5<#OYs={1jw=z2P z78uHB-lo!NWWK17`oA>AKdZHWE7E3V`ctLN!bJC%Oq>3<2Iy~@u|GHKAB5HA)-u&gMA0J>M)Mt`P?`C6r%8s*X~}Ci#yPMw6~m-6DElhrlx!%O zldxT*dxdaRSiyPQ$R{P6)oLi`)qPm8deZ$IxdPZkpu|`H?V?I%PJ!WTr&tRiliI>GdPT{MV$sviDu7_?K!&VfJUH zeMjKC)WPj~aiD*|nuY7H)CLBF7^jZz>Zj2{weIa%XS$*a ziNtWHNw&HRmEUk0=F$`IF`?Sa(j*`>j+!L5Z~}FMYo-Y>+4haoY3||O!NcE^*{EB}wPpGQ7zy+HIZ;rk(tv{0tN9}b=dwUB7<7Gk$&^q!&&$F_G z;p#cQ6Q4Q3$OsJ#Z|YQ-RSe`*CK` zQ*^PN_vp`M4QE%dmysG)2(yEIQl}b|vz*^ajh#jl)JXB;qa#P(5g!ly zP~45n7Y2}%O6|{HYLGAFh{{=m2360pBlTlbEJdP>@hYN!O;0`qu?M#R1)G1-fk*SQ zF_S6afPwsL=`ymf8k!+_&<9fg8bVa0o{7Dni?4ig(4(d%T=?Bu)l9LK?O-p!|bapNFIJ412(LbB}nUw1;U0-a-cCYXUuy3I~gc=7}4Hy>a| zKnWWO=>VmlDTLxgFA>|*g( ztYh;%ukY8`o603o`ZoJ&kmb>ShU#%aCFk4K`GN=q!Xk`!{1&~< zT6+{df&cCC{Ndx~McV`hC$&zkqF*9?CnVG4?ER~pO?w6aVSGh{K0i`wKn*S!6S~Y5; zS^P%JwEk;>MPMQ7FmJJPtf)i7;a(M5No8NtOx3jvtgz9kH)$2PS|3~%!$%0A=nv%k!)hM>+)W**? z?z@+&I$I;%zI$UyzPv*MXp=*G;3qvaw2_V*D;Oy@PTDL}nW>3uc-iylxXGT`0|{U$ zvUnx1ci}d8WF=IES>p30%7qL!pyN@lJ|=%s?_{_6(ysLg_T1gj&WimuTx} zl;6`{`Guv)C~t+eW}-D~AYk8Pny=7Cjtfl6kNBC$gC@egndV*XwTLtbeHCKBVylD& zRj36XJYktPiF_7nITp-L&pr?NR0zff=6FhFY(K!k)?T`mw!(5fM$4(^%&K-Hs50Hl z@?cxMs(N(i7t^q4f^nL#=-HWVx{|^li$zuxs=MNwd3A)>A0a2kaY2LLw1x==k($2c zJ5kl?nZ;Msm-t2bD=h=kiH_~8;*CszeCIVi9AMN9V#HS#rvje2-e6Afr7RKeN=D(? zLF8)0cLrlM^4T*LRsqAY$~Kt!Cg!T(GHiY2*lcFR?^ye43yx3$gK}R8Gu5CU!7>}? zbH2#PZ&^i>Gx{)Yl$wkf%9A&*EbhfUQN~bmQ%#A`FP1q2LgM)lr!)#|Y<+H%U}In6 zS`Vp?0A14WTmI4ssg9psySo*}@X03UOgf<1y66W$P=z$Dvb|uDx;z3R4W<-j$epX^T;Oj5#x}^6`<1I z2-^wL)&QtwcuMO{uF)&s^hn!SVdN_N+hWCOY?SgfhR>DOv2?Od<{h{qn9g9RXhxpU ztO_5u`Go1Za=mE!9&|Hi-!%00Y;j{mmk770*eC><6ZjcNd@$J~iy?{ceyoc3%Cv@C zwa;I)r!I5$dP37?6_vsZ$69J@;eytA&Uz*-TUEi=;;OAl6c+-D9tZ_vAJCj@rVu#u zWvZ;Nb>t34&jY-XD>ahp^mqnT@}Ua#`8>q<=cn@YRG5GnS+ik6xFX-lmE-nvL(pNw z5en-z;_}ZQ-1WRicr6!^=>QBV+sEAqG#Kmy!OKjbCn6U-7OOnN6IF^Uk_KU93CRL) z*7*Hwqc=@GlC69n3wNtF-^h_`%-rP%9P3vA#~shRjbqpd&rm0NEq>#wHuQ~iwG^yM zscsCVT8K@~ns^fxW})r?q0@t`V|5UXg|xhA{wU%vlspZ_6bu6HGQ&*zp*{boEaJpN z^E&klb{t#sO|M(Bl@XaFgoUTSFSkCOLt=K-U@YRiDk=+m54Lj9w!G`mq%5lF=JQ+g z<1=B`8{p#JT+Y8aigEmRcg+9equBpM-T}iO8^?bF9x(n1`TwUf!>?`T-`{&NGyJh3 z{bTd_2mk+XSO4xH_2tI;JMM~!^{cw@@9)O{ywQKMA5G-3#$kX7bO&w$md!!^r1)){ zh}>+Cv@|x{> ze9fFONa#HsJh_}59i4r&-2*(+HUQixu!>|Mb6d`92(m^ev^dJR(hontIYwr+nXVJb zJ6p-K+SlE*nkfZqT-1&anr^;iHH+;fsBo`;?IxCvJ~E3I9{{bV<;eZv&dwf6o3p(I z7zw*tx3;SIavXXotxYL&l=9B_q`l4_*D5*rDa!;`+9FwZ8saZu%U@tK>n5aA3@a=3 zV8)SGH9J6NHN+dKHquBIQ8fYRS=cxdLU#v_b&PxcuWOCzk8F&;O$TNs4*Gu_!5RMu zfcVEG`TH#RbLf9C3(mQRqo_HoT|9*|u!pK2K*_oBnW*Y~TOtnu9q2tM-3KrfF^@X$ zF4MyoLWUYju_qsLu3PT^aKQV|VX1?+6m(;+VKamxb3|AKb^suT}9G4cT7q z@(Q4s3Ci8B&u;7<>7Adz>hILVtO~O>v&U`Z{-SJ>@lq-E8KG1hqL-ek@9+1QbG^iT zqGaa<^OkFOr`|H&uaG(+ebBUK@jf9-^(|Yaomji~KW!?4 zGuEpKonOk7aao1yeOI0Z{fC@K(sa z_TB`cpOYOJ=E${^+Wg!V#@iF`Mk(Ooa$*&D*_G!kGehHH+Sr2|mEYPl2RT37s*qj- zJsLpm$gH)86A{SI1VWE9`^2bIP%+-`v{tB*Ha8Gt9iwRN4~XIvZ{w4=+?@{<4i~Z0 zYxxx!%Hi&$P21BR=jx$6Yhp~oYV2g!a*>+2l!jV`+A?#>x@R+?8w{PljaLFy2iKH$!i1c*~JZ4N&YZM`&}pVRRrtyBM?h@AKfya(3)r#yggm- zzGE;rrha+{xA2g;HO4QUGvmzn0S^btH20F;``zrt2IE6E&pK*qp!*_Ot#2B<)I0S9 zW(;uTW@`-wOKnM(sLvhqmX;EXncjR5$F3^KS9equ;(Q@R@Q^VaTdEwa_>2^Z8@pFj z7{bcfNDU=*1T2VCNv?3YWs68QEJ})ch^Y)kMs+ZJ!eTt+v2jWXcSGK$ zHmiM6iKc9gJ`m`ZbfLXL-U!(vF>|==bJ|DdSPIC*sC-V}7o-RQm;!d7Z=^J*NI+#= zkVhpCS=uoYoVcQaDR*`d$1z>5-}ijyjHWqUMiW%UXoZs%#t?|s9-!+t^RTv|(^j9M zNRk1{uTh~e5yzDBK+~g`L-v{UU#YOkiA98*=azGvdph1ot0yf7#w-xE9f%Jfgl>}c zR4%&W>6D}*IN)oVhX`((&H(zri^W=XwIo!Sv&fpwffzYVZX3~X;Ecd#>Bdh~5ykK7 z*x-!Ny!R8iz3Yx}c?N|smi{Q{fW@jIZaY3R}=RPA-8oDs}9$vuw-Gc zhh8*0c37&#>CU9{wu`F22J-0RB5SAU&>(GfZFC|hIS^oMxY`1@hxJp@VP;Y}{KU?k z{KptA4#@FX2BZ2HN~8*zxThKJ+F7Cus(D#7_Dpj_TEI}>{TAtUk%M`JqfJLwccR@F z+SLeV-&A308@t8>%vGXMT%M(_^}Tv&>~5nRVA3X-bz1unhJKDP*5rZ?rvpHfHx$;l z7vfNU&KtB!c~)e^@1r<7A%VrcoeK~2YWuP5Y-dQZmR5Tw1}}QbWO`+t)Evk7ud3;n zw$3^$!@+yB8G0nXSt!L_7^=S{;!|v7Z*%MuV=is7GXav+qdnS?$}cCrj?~r&;1M80 z^9z5IIkB8&#I*wN07OS#sR;(T7Q=2g-o69q67la{f{=BTclvE2x$iMDn1_*RcE=5+ zs1Zv>p5ik&eF@`N1cG786?M&TaR#D{h@UnUgGcJoeRao5!+_}w?RvCD^%RBDQE|!B z(FiI9a{~6m`?L@*4MW&=iXse*Qv-}RU{}10NiYy1Mc0uSPf2X9uloU&!CGplzIS{s zlr2K0f+Qe7j%BR5?KjcL%=iHECl*CJfgmTI4q^@_f0DqJKw_g9njodU6dhTOz=jhU|d)QBRyc zAiscgH7s10M>Kxm%f?LLHz?KcUjh-N>jz=l0~?yl@)hM&Owv~;A;FUR8tAjsT!1Bz z`U|&gS~#eJ6e7zTnKv6dkppdr&@+)xZb>kKwnYq&Mzs<6CvC&@;{gF%X~hF;=$8H_ zwUkeyAVFZ!YbOoZRWgjwM=1|9ijeSSD1Ha^SHwL7Xwb3V&sF1xapnDDR+3`x0-10L)n!HOf@9*seuppdg08yDqj$xblEVJVPjA z?g>OodmaIZSSl*jceOf36m!w5Mg?>$K@%9N^7M~5_e!V_)^~JXjTyk9Oa>_aV80Kr zNdl6eDC#&%iBlZH?XAQJsF$#Mm()ZVJu!GVauAq$T1Y-I6whHfG6+V;3t$K%U_N{1pX1NV(0 zgj|3xz{;3_^6OPtQFJ7ZorYXMO39~`h$ES*?;A%GRNzsf5onuVV4*RCm_fQ`%@dQv zKuk+w-?&LnGZZQ!1162xJVh9eq^T`sR$M^60wZu5@kJi7!y!n~0v{{MjrAXzbzZ5J zgH=qv*wOK4CT@dsB)#+W%_|YWNad%wvYFeWqkzxFL37|9llic#v1*BAqxpS+(cwi* zaJ}=osZF2s8T2#EBR9tDU<*MBp;JjkrkGxV5>(kl9Sk1ph8uz6qv4 zI7o(?JhQuv4(6VrosU24=4FJ9j!ff6BC#!8P9TXqc`T3cEyl5icbq7`+nVTJa2C8f zhCCU5DIj<*&M&@TfMpazh>zll0{{t|c|f|Efe%WYf;fWVj#Wv6iIIMl5PIY&B}SMM z9od{6GO==P4op8OO7V^tXf=S4dWn(W3!tVDk1K2h3chLlrw`p?kMe<8JDfByTU@Zj zDk~Vtw>_&Z$a}tIXcuKW-mY4p&^h}S<7KAw7~}+!xC6!5@Tk?y;z0m^&)%`XdPc(b z@I7ZRDQTkxF`ItDPo}8)5o=tV&G&78@j6yp7hnue8&Vs&X=9HHXsBHXDeEjPjtil?425xof<=!e_ zZ!!mZbm_FxM0l$IQm8`$BSri6p*@pyPU&eucAE{l#u z-)0rZqOB+`C~zIfmJELqEjbu0jr?j)+|QwZ*KtM{#%pQ-na#P1yBGTOjv00$%R32< z=1W;@|FxoDu@Wxs!a%hiT==&j_c z4EM7o&9YAedZ>}YS1BboxuBjHi6c(whJ|kKSYb@5Tz4B&oIHd6UaI800LwK`(j=ey z_HG~TJ;g`F7!MHvia4`Vk=@vSX154Xf{E|ZIJC*yarhpIEo7g~AfhNLt8-C{vO%4a zIc#U6;1x>phmrU^-2@M0lkLpR<%Pi|2T7!Mk662rPf?_XQzhMXQDtL*%weC6N1;w9 z5IvK2k6~aBjd({MpSD)ktaQk7wT7k7uW%PEFK)K7zW&e4VFY1qfcGFevwn2fW}Uxc>8?SYR4(jue{hI9#ju*5 zFJ?Qf-Hvw#rKam%@2*x+{%(D48Mc+u8uV^OI#uul)c)-}mIp7d8_w}Yf{kQ znf5L!0fp0FqTRqvkn7I9HH$=u?75@q8IjIMGz|WPqY25h+RttTu3`o@KBnu!0Ncv4lP5U7uMAL=ypp1h|^UOON=3XmYgLh4MC2TK8N%7iLHr1ed@xbZ1d^#q!t zo`qi-HNw>`|xsN1i^B-Tml(c)tkwl9*qj+8B(Jlg<9xpDIR*5_l zU8Uo5Dq&*Q6%}L}!zpx%~yb+bPw}F+)yoht#5_#}ZUb=EoAiG_W zm5MhrVciY0l**&>@p^tX5!=s_L4gm##_&iH_j;i#ThPv{)AKHf_KptQF;KHAc9-JO?BT2TmsMep(51ODEfn z)D~_VwSA37(%IG2HD+o}VZr8OU97CvIW_A|(Ba?_R&KwS`iKS|3@&>-x2HM=oZF^U zq@%IZUCrs*%q@44YNRJI(_LM$+w?4VuB|MY8|d_>CeRiJ&=$Bfo^CQ}s3Rhg_VrA5=UUhr$44}dpPx{?x52%=hdJL=yoW{k7H3CI|S6R?qOE2mc0S?kHMK(4>e8JTg8FzLK8@ zCdpzs4R^SwatB=o7ga51@|H2*e9@XTS`nO$zDs9$rA8%tI?Dc$ZsUPI36|A+BU->; zq8C4twRXC&zlDX=KffCIs3#(`kuEZyQnkU`8KvPfpaLP4ikZ7~H4;zLD#O*LeK+vh1-_`Q})wlR@{=CtDpn5eV4o7~ONQ5qd*}+9}wh692+n zpY}i}bhpxPUF-;RbRWfEUaBO^XGB`VQ|3||^ASrw3W=Sku*4}>q)S85#Bwf!rP85m zXEkS%4xlKv;NqE@iT30c^7lY5Ffw5;EIF4mex&5`pvas0D=-X-k08_wllXF^LScn5 z=nVw>0=1MuDTd@m*9x&7K!_G$@e-TEf^#n0_9U@r9&LF`lP*KC=e9Fc^i(zj$dGeI zmEb_JtE+ zpyCw!t~R`qejx<@vpz^q(Ta&vY(g%JuO=q4)2UA}(%NXfi;Hz|nAB*2VEM|Vkx zljV0uKq#biP6os*uU~OcN9JBp6Pw$+AfG#yWqQ@!%lRw(%lqMbX3yk^_IPu%DoaEM$P>(NmMTaR)#yx6hX9sRY%|rEejV` zeCi{$O3h$SVyD7BWcrvR69kd4*gxVAub?v1?$pT!NFTNH4B}VO2A2x5P#xk+Rqp&( zJdz@i@5=XgrEpAAQl+zUKp%nYCNw!Wr?{*rC{vAU*HzAna~tPNc}C%7_iHnGC4~yD zP0sB# z?A)WZUOMx>%{6*`4A72Oe*I)S6Xh|pmb_1;S2|V@mhFH2Lty?268?MH3M0o~AP=TL zVvGNqH2de!|6r1yYke6;nqfO{RgT*EIow8puwduP(3e&Xv4;qTD2Cw>q`*^J&Nkui zYl$`XYs9X$*QzQe&TtZpwfQ9Ndf%hvzZLYd z%~5VY!(uMvM5I8_KrL7r+QJ|3WHD|0)=3o6BTYfy-t!%Q2!$s4rS3I$mKHmS?$EFO3dlCqhBnm^n?T;4yj~4!K zZ7rn8AbIPzd^z0&OaJM#VJD390QqZgHkDWJP2S$^#JBxGL=pLzsxBio0 zhws|6G~Vy2wl^=Z?mH20AAJ~n3RLc;C`K8lCa&-BjW3wq)rgJwhl_wJZC6k0zlB?W zlZbw$c@S%_gwYv4kh!s+A3l#vX5TC@UPvVQDtY-TfoKk&ZDx}<_{{)Q9yJ$bWKr;Z zqu{BzdUou&yu)O7f{oRE%&keR+^(x@Ci^PPo;B&oCePnVee-Ysuum>{MiS>bfaO&x zBb#L43AB#TIy0@XVqwFg{cA$( z1%l!6nM8kXIwLXnE#7gvEqy|@HA^E*lVZ081qo9xT9T1>Z^ zA_&`|`mqt7BjNkxmi+^(7o6jv85r)1dwe?5Pg**%>SQ7)lGch2SolZU^L0lWk=9N< z;%+Btk!@Tl7CVIn1`y=3Oe8MV{fxNa0a_MH93;*%D8Xtt!O^4yi!XIfDE4uZF^G^4 zEo3g(pIn%A6ILjE!QO(upeZ%X9!62LM_6{r^b?6LX}zrt8n!w=9>!C4EhQ9Yj`#Lp zZ6ct_VSIIhR}06Uih2PTS*n37hmi7N+^|bpv7H*c0gTI8lC2R4X*F=xUSOyT0gge} z=x(o)thW=RA-$8y7-gH;XA1?76q)D>3^8Y`H$wwmRC1Q>u{?$q$|UNVRFf>YpI_4- z4Y~p$1MfT-^p{sVEiuc>F=HFOeiH-5P-p&%b{6u5S{IEBS`~vgFJiAmVC0+@@sFk0 zV2|Nf+S{i#ibg5lwZf%Bv)NaaHZ7@`1Wz6`2S~zgOt-YvAc_7>68$367J>ULGn{W>!&7x^q81ZYTfz7o znZw+boI(m2R})3ZSjAPpQ4U>Ol4?;}$Qa-{AY_c#vVIq|a! zXkf-A_!1lj9K_wrN1unBHGO>5CaTEBPi5j@+$H%t3UVA%uLj1q(HWB~#1xh0{2(eP z-eBBJB*Tn__1$`S?FGIbf_Pl_&HP@2_1j0G)F{>FiHh(EswP^_5=E%Xm)a46gd`-HL2O$$)QsF`=FYFVYs)!p5YrC z5R@-$pFsJ(^MIHft`e;QccIYo#BlANkdTOotQhs6yekMlLpe~O^;TFLMS*n0)^I|E zFvg39-(=bsi|mDplw&yr}z z?33@DBV(tHF{juhl$OBkhy=1^^iaTE%yyDg$N&hk#EDU?Rn1JThYEd7Iq|gAP}umB zHS&6eoO~^#gZCmT_YT;`0DLH=c4L#7xh1`YWs{qps-9(mo@H`7{oe6@^hF@BauudV z>!tn_%S}f_IU8&9aPl}7_87)$1~!q(+7|@fz;Gk7+l@^bd)06i8>X##!L2cT0BrI5 zM&baNh2VzpR~um^E(UlS5vj?>s%m1n2)v%oz3OD(pEw{=iHW^tY{|qxv}R~o$3R3) z-x!XKqwxz2zU$%20=~;RU&q46Bv2a7GIAbstx4Bn3Cprpm+0OP_CmWl^4X^i-4a>D z-~Er4=4%kinkO{tjCOVR`JWx`SEpI6R1)*-*)=)_yO6r-_Xbh!Xf-+xiRNzhG%C+{k>4qetB}mO&R2&Du7ZVcNPPS!raZT*a z8VV$~I?6YbH>9s8!=WT-P4f%rE&8Y7Z`X~T+;Q>VVS9T|V$cy$_1q5$e|{`miwwJe z=2lr%+uR_*HHK3=L}QC2^{OmU*5Z`Iv4&H}vsk06iOls5#2PVPf8(juw6{D()Z)z> z6?JSea^&ER0FUf-eKe8Z0xQ$_rpNo>%=6_4WO#6<;W&N@q5p<>K-9Cw+@oom;Aj;< z@gftkIc+o(Bb^#|5kVh}S2rKX6w>p=!iIv9UPp24s3IO-^t&hW8KGcGNu>V~6x43v zhEh8yA`H_=@g)lKM}O)NI_mb*@=3B&vG8z!5YD)fvfr9=P`X^9GGjCcAX1P*R|7VN zIS0RLx_${NZw96mBc2SJVD731TD3Iu^SoBU5#cZrJRz-u+eB^3+eM|<$QJOnJpvzh+uX;=lPLKGRBL84wF#+{PDm3b| zyB&vw8o56MMITK)RbRymveFvdVa9%qd_)^3+hxqI<+mb{-iG;#PS02yw;8NdgO7mw zV!pJ|b_F)RXKEf+bh)1jR3_!u+Go z_3kqGndr?^!uP6Wf%NT^#|eTum7xPT%z@3uF5ZhE9Fu`@lR@dnL13;S9v)n{F(u!P zZ*pdD{4|N=@t#=3o^~wW;aK0f2o@b=Xzk!=Ci>e4kow^6&Q9@z2KCgqtaNtz_MU=fUhn`PE| z4zhAhXqEeXfUT+hE+9I~${fiqNnsJ$YPH$`d=bQQOBh^!dAX2NeA#8pr>QtnO3Yrn zp;jK(t9Kx&BdNiijT=aVjtCZ9>1?bEQ(a-`R2^)YQkb!=&=QF(bZ-z%=fBI=83@GC z^wBRoJ2wTYR5dw54B+dJBAi@%rF5#V7uH#;V~T*om*fuu9I6p(ZiO7Oi|JWLQ3SS9 zOd%IlzKq3j49r3$YkOh~$e2Zl`cOtFEYlke_x#j)|W)& zWL9)ATh@76SP+Y~j&f?GgwzG54qDTjHBCfGIpNeFv14&NT>z%zNt}S@kj-%mx=u** z=KL!$%wvD@$S4(4rAI(BANZiNd`?&dS6!{&QuhWF(EXQa@^ZQ0PSdvpCWJapgFFo-Y5(Ri=T@O#e5{ zK>N<;V7A68n4A4l~0);}rgTfzFq;={3Q!dZoR{{*ekW zynU<#7`3nlaPvLQL6yIUK3&llkeZfpXqzVuU8y5D^>0>0*i~;4_c06#c6mo*(;q8p zBG&LMV#{i-ErILM*^}jgCY%)g9IzS5_=A0!#Cim&_2N>dKy^~4Ah}f&v|aUT0S5y# z>N!B-`%!UsdQi#T?LkKf5Manf`4RZU-GC>Fk7f8dj%E3OPUi;P+hsDSO5YUdn zb^@~&;0U0Si`s)qaQJ%Au;`Y|c+W`C&3XUV7u3G82|Xmu+SvGeOA6LoMZc(}D`P6o z3Ug{^Bs)~yk0*_kOr!$dM&wKffC{+4{4mcezeiF!qh=xabTA4oTYJeLb)&tiO zosKLs6C?1U;61p8i@c!~6bf_gn5>7m&at#98|Y{;wWbQ7&YX}&!ittngK9WJjbF?t zyVTdIR18I$I$Esmg?PzfD-lw)wv{gcz85%Gwl<2mbXBo`Ni8CWcCO0-?;u`Mn6TA4 z@@CFLe8rL4GgZaKoF!f04bKX4R-|N%GFyWQ)A?6MtDbDRqM1|h=J}%AR2{ELLeVF) zvgTZFicW0w&M%N}DU6(o=GYI&PYgz&?XBzUPI%K_WBJrxD*|r2zn;uGp+H@H!M37K zb{D_Gp{A(y1q$#|z}9IwOIj}24M_4-v3zrpblin*ecz9seTO_B*wssB^^R#AT_vMI zZ8LY4jaOK@t#dC=oRQqUB)YZ1OS5bI*FQ97mVZc?%#2Kbi9*bOx%mEO>ioy*`eW#S zu(~=lMs0}3BfB1TrVrt5gUek2g8)lNK!>c5fjL0j)7I$(=;2~o#=5%rYVovah7H`g z)i8pDoO#61Hs{kmd2Xei++(8mT*WB{*}Z?Jt|Di7stU3$kCDTZg~Q%HicT1^XQa+t^G|?e%HDo68ey}1@#N)`jhxu4QE#++XlGAX*pE$; zq6Paw{>}GB^(nk=`+431#7!%>t_Z6=aZ6vr4l*}sP&`+;5gW+2l~)ikg76! zS_W1KXom&%iGxnchdKfdGW~Q~0ki!WXP>C7s(&)I5OC6X*H{KRnc|M-&X}DZ*QiQt z;ovi+H)(RN>sBJpBvc9z#4v)#o?W>b`#Q| znkOR#h;+AY7vP~NehlAocXAA?dN_Xiz`KMi3`x#K0Nhi{Gg(wiIk{r#`I-!@J0*hT zS;Z}&h9b~fPh(Yk=IU)A|N0CkCZK)^bK2H2DfDWJ$3Kf&*HGU^-{7Aw6v4UMd+ze!gd@;e(ZqPd@S^(3iD*goZsbrd!+&?T`CUcW z0tZCa5N1pwBP2K+L1gYNrCGrs45TxX?jb-4=F|qh03s7WJx@S%u!jiQMnOz1hY*d6 z0MNm481Ya|(ZIKx(IWB7u{fVNDjq88eXl0&BzAD7#GaDf>1f884yw+jul`^GH!;C?e@OTt{BpTbKbvp))U0@M7F^7x%x zwQzD!bAP@}kk{d{b>2pcL5VDwr- zfrRcXQWv}Z#F&<3pLOJIdXgo?eBALUUZJ`Ig54~VPS4XXQdEYcVL0UZOW2UJN21bF z%i>b1D=-H55dxTbZW+WtP52#Gwjpq1Hq+5W^N`lJ4-MHU*aCrmrx2!C5X>FKg%}az z;y@2lVDHd5wCDi8%amubx6j-88k&wq{W)y~qaR2cSw=*FUa6lCL)W%$KtV^bQANG8 zZ=1Kb*#iYnxkyiP6bS{7T~~|QkLAyLV(Ft7+=p7$lZNQ>G*Y1n=0V?Xd})J`vluEK z#CUfxxu6IL47%jl4LM0AgIOLdKg3KhKbg%~bWd2fG$?&V1zo#8Q%!lcW)X2|pWw2y zfe@$a73hc^Oa0xxgWx;HJvXRsK@TyMZ8q|_NwJOL&$mH`#l^>M4zZ9C2PY53+Nbw| zkk`s9d}HtK?`aT?eTqv^y;Nh9KyOyG;Iz3N!oE9tFPCO3pC3-7vrq9*&8;N+F2>}% zBho6cfwzgmq5u|Jm5umRUj#K%?w_L~Ai}c*0vzcI!Xh9lT$&{E=$XEpf1dqcPVxF?^*j;UklBbq%?aW^oYMW%Zqpmi#HGa$T*c7JX zLdw>8we&{?mvGWJYkvT7Y_J#pru`6Upg3}{eUlo4lE9e# zMhr8i>)|^CE9@M8jaJMWUq!pLiDd6q5{S5%+G=x*qIt+R&sE@r-dc}-okvrI#)C#^%9m|xC-)&>69M}r{*+8mDmv()HCG6!q!(G0`1Q5ujc)i4s zV}rOM>wTylTY;&!tp0M#Fd(qu~VnvZcxghQy9EV^8i#>Rre-f@ND1&r8*vEdf_6abh zrRf;BeUhzRHa|@7C|#4Si4JqTsq4~Ebt(HnZ*Z-F$rPP(m}cvWW&dmz@>lYpSozsy zvnlS(GB(ToL1dap7{^?oThbKjkk4y0_&zVLk>9N&nVu;J)E$RKeJC8tlo^+Re-D=4 znF`)^!^_IIZv|5#QSh0K2Tf(Yg&}eTGEL%_11&v}ZYfJ~Wz$s#axlE`WUQz?1LFrxY~SsZO48#wGF{xI^N~vs zM@5@YcCA9l%orDZsTjNFvz2c9OsRL2wA@>fJ(Y3l zMByEbfyN$G7hhyvDyix)F|I|MCqdM~1E@`U8$bwrwnQ>{@7{O1%SSTPSj!Q;{u%1X7R8z4+|9eHd?)GV2^2V35|Atn`8smLQX>W&6;|kUNvML=X)6i@cy3hzT-kb zqx+93$bnC$>#43^xHOT`h(O#y%JRQoohB1WaIp;I!Iieg{qB zsBOfM*ZO`PS31&swl($*+@S3AXl1_z#An@sH_=lA?3bf_uG)-$qfNf-0UB=>sgw8l z(?M{%tCJ-S&O1LLUTEuKmQj51Q}8lxW`fqz((GFdUfUbgx8U5I==<5^(4Cl*)GN>% zuwsUj{KG!yn9EX-r2AD4{*80)yXBv~6p=0EC9le4%PAkxv#xEO86$h+7mDT)uPscZ zVeEO}6gelD=#yt^QB6Fn(p_zZO;Z z+Z5hoxM=`n0^Ogje+R6c5dj>{;4OC{V9FYeeWr&jmP#x)p6xG|luK-0Jh)m+>JSuX zuB^i2#$VXdhEh}sgJQAdA>~o=K4ccPSzp!V{jkw)_#_plIrf=$^hq+@Bi;Lc{M#{} z(fXL*l?u1mXkJ=jbMfifXD9wp-psai^)y zUPutiW-hydbL7e^u000EYSUX|!A704H##WxQ>#G>(9Ov*ez03uky}G@KmZSp&_$nR zVAENH$HFt^qW)Ul-nArjZ{0ZyGH1g%eyQn4-b`nZ7$kJxtUQe>9f5wiUyKH*MZeUN zW7nIieq}?Xt(3$&Ell_tecak1yX|ulaD^DTNTPWg%t$aY*HgBmNCV>8R86!-Xb3sM z`^@nOMNTu$3aCUG2!eF1Xd1~tXvY_-fRq=m#V`^HH3_f@kzi!i!S3`3U0~HPXV;PF zh`go|LPc^4wa^U4#ANu@60q*kRJ80!qPYAI0|ZF7e$X}>qJ z(7QfUAN1Mgw=zCfJDB$UVfm>vyGiwUtTN=eF{<_9@Dp*1W_hq5l-0quUPvsAkIL<5 ze%wBFFz6v4A)((>m9S0hHFz8PJGi48eams@a4J>&tAJ?*u(`%zEw6N=lO){h z1ZFA?_{X##=hk=T4agYO!S^;+__N&wd@eh)E6s%!L(WrGwW5sg#p8GyFCcEThZ8bC zm0uxL^A&3~ZY$S-V`|GWhk%3ae^l=iRXG|GD|kG7lEJA-n7%fTGLGaZ<2Jo@wBBs| zvV?11-c348c8(0=eaGs&+T?z*=y4B*z4M5fvnE4M-s*ia7E>}t>0GsYXq>$m(P&Qc zihIacKXjnU49nea?*=+Kdj`x#KohkaEF}vWHtE$+_*guVo4yH@BXLbISlE%FH z`Rdflh>e1%%6!Occt+c_q_eHdUQHWDQq8;P7V6p(@k+|jv&5O`^WjA26iIP@Re1AU zl1|+>CHqW*nU>E%}WsIZPdq+`jTOZ-m`xW zYz*jb8>){a2m{tfuN$_6FG%0>M?+%?vG9kN_v8ACH>NgyAqVF>qJVvGc%e#>Fc)q- zlvTfg7Mwnzm#Im@B2ZIM=*!v5Lrwx5OK8rA+Kg-Gi}ig+p&6Yg`NK}$G#i`_NqzCD zm?gnYUPV9gpBF^ZFj2DK|W+)d~hXi*YG+JaW{&5!BT4(eQ(UEB= z=#jX&(p*K-T=(bcu|cn=(Jw76gTW8GgC*@m@7{f8+lK!}?_l@`yY>GDmg%3&aQ`am z{{QBR`%@J9FD)0J`A=rlKUD}b{}oQ~RsH#wlKW4<@_%*7{f)0AVrK8~l{w1vPfG6J zvjhGejQelT`wO+ypU4%a{Z%3S>U$~I@P|QBa37MqSS2B4iJu%1ji@6dF71##y9cADfKjugU62KojXf6% z>`!C~bfLm_-SOh977is?bVsM=0*zkO1{jwThh|a^@EoFUWc*xGd!JVQZQb>1)87}w z=D^@+E?YusthRdcz&ddyBe8|9Kyc*20C41@U;Sa~NKVGnPks(}UGl{zG`ULNFL2S( zGm)o1HlPaM^z9*?&hg-66I-cRr@_eU#p~?p>FhV%SL`R^bcmU^M@Qwl^#Qf=Bl5~s zHLjJ-@b9|2TTG%JQyupZo=eIU8!7T z{c~j1%Us|jD3Uk!hj$_MxU1S8tamM}y0~tjALkzB{&iWh{1p_)OwagNZ8iJB0H? zZ5r;g0#a(U-$=n^cYR7^YNy=IbYst?_nZA9e}rCpMStIKo5J!x9J*$D{~#>R&-N~9 z-yRKH5tg83et&xih4gyy{%|MB{*=~P>e=EG(0S88I`p3D+HNP$(Zu+H#XC;5e}Sxt zvkl$b*2-Ihhr#uvLJ7jXJsFX?Ood3TwWU?HW!*l&w#_n7Z*=xWf3N5EC~SEGq#dr_2IUdVEJF|&pc14!rO5$CV?G3^M2x}NFq6F!X%RCmS z8}1KIA~lplT7~lK{1uw6%C64TI)M!4kdAU&$1BIr>jO7I#r9>8j5g7zOue@);se0j z7Wtwt$nYqWsnc{k*QHCfB9N@^0EQL(;rDnd6KpT%3v$!fCN2w3B~8`?>~tC={ZJXo zJyXMhcOtEnsej*3q_5}gN-3blcQQx8)NOnXg5)+HKS}RlkFY%Su?h}agU6c`lMV?+ z)$}{<3Ura_Z&;IvnqIOkFblx%SD+YBAp(`_ra2S}T=j0LQ}x=!a$>iNge>qJze}w& zkgTEBrAUJrH@IHP6-FQwC)zkvjH%1B z#1oHOD(;dvzbd#*vNg*=FZK-eEY-b5h_LMzJ#gI+^WO&0Z+K(Kp1C$kVNbN*kJUn4 zXQDQ>&wq$+7@y1!vv4irmXjv?VRW}gY3#0${Y7MhLMZ?o*@_=&1IYY&(h$^4X}casOHU7JJW;M2id2qX?bB%%Gh z={(CXnLJ<(+e4oL&!BKX0?6p0H@a9+mfmlS=nAiK^R3M)NG9Gxpty?2s~Pmr;uo9HS2EU)=%MzC-fqC zMx7`_WWvuE*XX-^3`41~xa}`gB-1!m39CE2BZZm=I(E#G?(of?f%1IJ+j4p7-N(Hq zGc=Y2wfipB^_4Ldx;PGI=nm!?c3i9<;q#s8nEXZ<1*kI0OXCvMZ}PfUVix*q{Rh%w zEsqXkV>99=>$It&;%kCq&WnP$ZAKsfehTuzvP1xaqGN}WWB*uT1ou4p(4JWGk5-FA zdj0gSz!gVHqq#Ri)`?WR!P!J^>oBazg z$kkaBsn_PTXsqw}Z&b3@U6_Pe*LtwTLCWNk2i0}T0%46w$eR4;C2e*wZRKe>77Ns{ zWc`{&}Q7innT;>YYq;%rU@6PP?Dj4DsK$?pOV}RMYfpFa6 zSBmvMP-|y_S4JFr0HS&%)4(`*eo?%TR!(5-cYk&u{cVQ}T1*ZYdpkJ$9c``js~7r8 zF(!Z?Xk|+iaTCExcq=Z^Yw@$k#aUa{rsn(^M04?}>)fN=R$KN;efb(HdDQ#d5 zhoJWG`KkJd0J6D4(%8YX>;ty8t6QnKBYEs#tLV>Z@qX^Ji+BBrBMQoQ3Po{)at=7v8H*Q$%esQlXFhkT%Z+K!YrgU z5mxG_;FP?RzNWcl2nm6E8-DX`>RzcIj>_7AlB78Lm3HEo2qpdEI znDhrZK?oPKZmPs=$ueWpYWt(z=CCajMtj&ca^0V$C~8~5v16|#JE<<>eZQp~&N^|> zP@)NHul!^?h^OAwZ%xL2^hdLw)`TiMiUvz$h8R84pGAI6jJ||}K&RlT%L>t!GS!0$ z+?v5!WPR~+@P#vw0wNNQ+d==joTfSgEhz5*%?TIFKDZ+TJg81)cI7iDh!hPyU;kk2 z`=^sJ(W(miSY9m<4NOsIP3_IuO*I;(#Asb}-{# z;*Gjov>M8nwdIRLw{DjV{er93k}7oubg9xcQ!*Sb{4E4}aG-}pr5!&jxXW?J7$knq zrUDppI5_T&0erT3S6>d;P;_g;^up^vVVe{TL@VQJavdt7DB@YWY+d&HFtSi@Eu#S~ zw%iVi7bs3cx!<#q+q^_Jy+;PLi**E%Wf!k|FSn;1xGUQ221S>3nIZj~m~P|8Ha=|G1bX}WZ7wd{Fp(TybV$_5TY9zig%j`dHvOyOB?uST&JeZY;%Sx zFhsVZGwDmJW+Ssg(;<4=0;$$r1RT_WNo$r|y7VQ@=x-_#VT?5MNK>y)q~<1^v4lbN zvaGB@bR7OK({Q>8gLvXjHdX1mK|`L4rZOFt8i`JaI>V5z?fAD+yeHn40}|I2NwgW- zc|TALmx4;TI-@w|qBCN5n-K4d15*d*kghsWHW1+{3oS^9dK65XBUTT<2w#ujGZ1JS zB&Lgr26PcB>9qPuOa zTps#d7@M=dUvV^7VKnDI6Vu~(_@!X#H4Y{FwR7|Uow!#^@G44%rPrFKP|)+q!bP(} zlyE@(AVO~vC%me(AoBM!-jE_JcE>>ubzRV8 zv@xjwgR8r>DNbNGH3%2o)SQRGinQy{v8dOO#y&JrkYqyGvy-uo*O)0R6Q@g+RkKnn zL!%&59H2D`pDQ^@_`a(hWIBqLl|3I+HgNBFYLvo72@z3$`=OPf-M>^ggzC(R2YOE7 zLDxN$h;cy)rn!hUEjF=+_9>)Y>F5Nrzcu{ku0M>5A@)7h0e)GBwq+v42qS=IydRjj zI1u5*3h=izZ z9kdd9@fAdS)Pc0{@(P^QBMz`z=m3*2*4f4O=H(_v#T(h;?3z|; zy6WS^`DY?4uspW{v_bqUkATFwxy-%I5`J;-Nl_;lT)ZiZOlJy=jIVWrjT<4xPkZ1P zP=h15dxZqtXfRcXw`s%nD95_KjOeiaHe7D6I5!E=@~UE|eWjxb6*DtFPAAk!SdQ*6 zpuC)vP{>nNB-XCiif`ud1_~)!uWVqGcJmdHmHx-LOlen|f_6JN zL3=bg#lN(s9Q>^y(BN}K!aA(3sf=7r!!Ixr=FR1u*|IxXgotTq6BhKrHSLTYbR%_o zr~k9|_MQDPWmI=Wk`}}U&{_MpoGPyQA8m$0#Brj(;7PistP~@yg zj?%0OnY}*Bf-4shoQM6RDMfNn5bj4u?Tl}OJD#_M<|sa70>_@~VDw#GRc)gzf`A2K z4;XjBfWp2EplN~T)GML>0c?R|5Qq>S)Ld!IW5L9%5bbl%f;~SitNMSAY za4w^dpUQ=7B13VSl2+ljchGo$spH=a<&+VSwR0~^o9$`xdlb} zh~6yht?XMJQ&4A5;MZ#t(`uGW#wRz)4U$p)Y6EWKFej7Wrr@2 zrm-I1aK;)u|A}Ua`L{BfZ)eCpFA^N+C>RY*FGN|TN2d?1;w2U8uaU~AUiaF;wVsL} zKxbW)G=1E!)CHwgordhp?ocJamqCUUO$KKn56|Iz4SP}x_8Wu|4In-J0CemlF*{bM zwberODooRT|CtaAk}YtS1P_TYX6yJ%WuE z%8^oKlNo%A^b&6PY|UNiNd`j;YUBN#pDF@0fL{mOC;iw(IVZBe!IgGIkN-O#VEwBR z-v2wc`-{x|iQokym}h$%p_^Jc z7o2$zq(iOxXqF~^Z$i$H0LSv8;q~rTery`cVAK|N^jExaRT(z6UR^Mi)(B(uWe<8Q zBWCMLzp5kuMg~uRMH{9mN6NmEBU|5&LzGx+VjOfxme~dChF^aO_vC9V8Qk8^9r>-C zNlbo2lakRRaUKj^xTG8I_kbC3q&vbd^WDY2{6Sz=XjelDUM;nxT)p_G(JAY@{;3P zYd*X~dSD8%b8tM>3B%t{hvP3F?=K<5DB({rx3AOPU-$w(KEL_DPbGHQJ?>w($m(|c zydiju7RF+A=gGeD&b&wSvneLxYR%dx^6_TpN}E*WX4;t5qz1Wc&6-X(+3?*~C8T#e z9JEeJbwxQG(>!!oG^A{(wqPx|1^FmmR;y72q@Za&A%7OH4IkC5mpjy`p13X^off!j zD<{{?*p^FIu%`^qQkcbzKHqt_amaqtfaxw+JZ63?|4pEyq^HxD%pJ9L&b-=w%;j?a z-~sLx@K-S%ywqq6ql{_`~5Nd&t8>`DYsn_8ZC+>`Cx(L7&(592lT$}6y1#7 zw$F8cy6+|H$GR${*xYjUk6B3HSDv_JU*BV@%yPOLXP~TpclH^t*cr?7&CoY^@|Qh*p> zo$n%!H;a@^Gxw@EYZP5Hhc7R%Err*R+{1wd(Wn*ci~#;zfP6Ev08P4ZF&xv%6fZ6` zOkM~>2f{ccNy}2jANbnxdTq+e!71U%KGdM#UX-xqE)i-Pb1sZ~*dPD7uIsasmTk5< zbOi!i<#q05N(Mby{9I!>;{pcEDD9-baF>=Gz70pL(O0?0)9f!FEO{fCjY4Y(n@lV= z2sYCJK93tZ%^RZMAVP+@%U8-dRZ#*_lXOVN3GDK*6~5`3Nvsn*u9^!&Jo)fzXY%28 zV`NzI{`%>a0v*PkDBZ78qUA^4rD8=Ipm_D>F6#Q&l~|gbe453%8u)K>h|R z;NHi)b$7o=bGZ&;)cU&JsXj_hu7&Ftj}Po8Np7|l9Hv+jZiXw4gur?o;!M|SFiQe?O zk_2u&30>*e1d8bp<;X)3R(Ny*-aw>#?apxArSB=6VS|!~aoNCWKvb(pK5-c(a;L#M z(wiQ}cMAJj0WCIKvJgoL(&IX_wXrnDtmOLCAJq|+@qwEV<)VhYNI*Prz5Z#`#@5Jw z&PYI~v6LnbQF*FkQq0m0X?PG0dc9@uoN`|yLZyHnjbsW9mJnol#lzwz(om%ocgeNj zAGwia6f+SfmQnqHtvsM&DC?v%rRYz6}YJZTFs$R{KN>3rle2 zJ>yD%Y;<|kq)VufY7mVojPff}y9A~yz(?eThAyQWAWL5OlN;%PC&{&GQ6?n0n-T&I7U)e}eEUbmyBfLnk@s&d>*4lya7oHbYKv z2^I>38Dl9H=d1YsaP$PQ0F%a)%XbUjR}jv3d`0Sz)q;kY_T`GjB~R!Wswh%-MDfgJ zfu-PCS#b}5eB_#AFi%F)m%=uRySOZLCS_$tqF{D8eSQw|M-DyIXREE*d8^>&^Wdn)Tg8YvO{o;rF3gXzf0linLWI0jIz1U zHK~LAv)U(!=d?3sBC@`ORTA}eF+SRT+oJXv>0OAqjXKf1$nmvlHmoKegrF!np_qjr zJxt#S2W$C~7m+3+;|HZ#%x`iH`vhQzr^FAP0?8bOjAEPWB?taOM_76%jAaco?KWHe z%$ApT%Tsp6hQgtcYEsaxG6gH#0~@m&fn>$$+TLc)N5yZL>$*vcjvchs z;EDKL&y)K_3e@08vp%4N;+1W+8iVjO$8zALrUvwl+gJp(_})kM{Mmi_PH@=M@CR z1ghf{1vgMpTMq`)Y~&$8^*4Z(bEZl6NK~&qPDw#>svYW}i=lBD^#>lfH%l8cfu`a+ zmY6(PMT`yQF|v4k0kxmf-;rjzlz&zTIyRF}4s*FCHKh+UtyPZ}a1KMGk&DW^Dg@l~ zqYU;F59;np6df;*f$S@I?^~It$l37TO)8yw#(vylM~v>;x}Wn6t_2VkMQ_8gw_-hQ zs=d<3{}w~?A;@!0B70T^343awz97HM&=xoOf&6K&h}?;g&EmRR=+ah`?Riv$)ciCS z;=mowV|N;gVZq_Y;k{|QJ*};IY|eijXD1vbt9yu|{3yiwEvkL2U$vxi&1-BdtyWTT z>F|+m>8Rf1K^6T8T?l6Pd9jY4%y4VlTFO_DKOw>*U*ZZEVhPo>EI^x-S zZQz;|2h3|cnwieqLd`Cat3|XC&yDM(k&5?4h8_>)J{9T0egDTNCDT>*IUmL55MRe+ z7L69vRx22HSxZ}vS0#OW;iSeI)1_1{XwUkz1P-;$U0lO1b}th|-Km!7abxZ%!0Q~M zExZX9*I`v+m%_yKq=(tF0{o-R)Ls4+BWu;qugvxNZy8@scCKcc+ek!uds7_6prjX# z%2!qM`CYFsLW-D^AqdSpdxwc;vM7t4V5n%q-|Yo-xRNI#3JX}X5{=@FP|zOp`0DU+ z#2U0rk?QwHu38)uoIxYisp)h~W`zeV2aZ8IrD&cq?4a zs{dXlQ3W4Z-@F&RB6fg`yH7b`IS@|Nh~*058~$AcyT}B?OUxDn!~%=(H6^SQc94M# z0lJ~6ZA1&3l({OgM&lciOZ4KFE$je(tyS-$g*pS>DOpVu`R6W1PJhDLh+t#6awV6y z=H$k&rYvOqf}thZ3K#YYm-yrLIXn37rRha3ox7jzJHIz>%03@{24_j${SQCe|4OR= zf5!!XWncY;3;qKj{BOer|Cq)6D=y&pTfV=-zv2Rpe|An_`Oikze*&=nBS+|Os*L}N z3z#|ni3^zjGzs_%7yP}@zcD$QT;P!bV}ga)@gcj~K{3jD!7^_`qsZMOvZ>AA|N6kk zraq|34P_?rzFVndF-jjWY8Q{1NPQY$Hl#2scjB*BbPrihhn6vG+zvbuvnARBotf?&x&fajnn zt5M=N^J&pL`4n;_9#P(eO-(b@Jlwi>$fw4zE?u`g&p&#+fk(uE)AI3Ix7J-GKUoJL zc|JOw$PG+oXS;_0T+hUQWMD&%L{5FSMVDUbsd!>{{kYh2H*Am5dE&P16Moe-Yvw6m zXyhqR^p--q@|HARQc}NW7rQtk`$D6;?6`X}*_v{m?@W2_%M>)dW~bFwHhwDC@;d19 ztfOG=b-&K3>jrvwRHXXvw?F$|ral%9rvJn|oPTTm_fO38AEv&44E;AQ%;g_7%|BX$ zz2N5Jb`b~~`g%Z&&~?CF5RI5OK-H#d5IUxHcNZFoC(^05y~WX*c$#>WebxS2kW?M4cyD}ZAj5qgt1ub}hE>OMHMgrJQ1+5IsEKi&U~ z!Ss!d_cx#e(E#?lxB&0*(9d`M*#7$zznD3A-JBmg3M&me00*CM6uVCpKAe{Vb;m!2 z&k1%Qu_JaZaq#1LuPk2CCnqc`dG4Ew1&C zS5B%_@paDp^98$_r`yC@RCIxZ$y@gyHTIK<%$(sjeABKZWX#fGNikgaJ(D}%o2-$5 zi7o9j7gdjG-EVZF75Zi(b=GgK%HG%@>Wd{$9ScycP$P5SZcVsv`&aD^!4c@~hGiwE zirW{zQraJjrS_K#%Qy1Rrey`YxaT+pAf{yf?`uPHSa50RxxBJtaxkS>2)(Lv2dR8u z;6*iROtWKdI$l{nn0bG{(AymQTYL@3j@fzgLiav@_g8s&7r}Fzi9RQM?WOr8ziB7z)o0g@cd&t#fkmHEMXk=YzOPcItk&e*SBBFnmo zq+rhqga?5N|0m34QAl9p4Mr7XI*Zp=d@z4|(+O~zMdvAS)v!vDz|n?wNoo=EJ*&v> zYvy5~wxsNE(ms3c#X&7`ngyc2(iPRB@1%`lpRY6lNSAh)t7)RE;sEV}hwKX#(E|&S zII>U34Zc?pmMe~!c(5ciE`u0LYoBh1$wZ!-MuA7)tIQ_rsSz7n1aPzM@}&Q)jaj15b{GaO74$#5zpiB?xI$7*U4b`nb@*ggB;i- zJ0;!q+lvm@3Np(&V4_L`j;(-?C^pW>Q0|%mg^l70Y2&BCow@jg+m?Da2h;gdrsuFB z8v{b@5n(KO{t=`P;Q6kLD`iJQyjG4mo(mnBQIMw39KkYTzK_io8~SK5ueJ@yIOgzF zg$)@#Cpa|<2I=?*Dl3$~Bat;SqFZs+6z`5zlk7DfPIH~D&S_o_#z6i+DyAP%9&|V6 zW-MU203|g#Xd-cW!z^sj*AB9VqM49X@TE{9u zM+*-*>e@frY;k{w56~{>8OoeWcLwIdomIki{~bZg<6eApEyZupw$$?8HJc;38alhS z);slkOZ5PL7RS?6XBZZDA(#K675S1_xpQ?bt7{U;kHPkGuQTrW^Qyvb}IT!;}MU z5u*w{ag`IJie{AfK+jtnaeYSMfX)9wVL9y@67z9hEAV zeI0d-i4N!?445{2HwPQLj<1K4Me7mv%rp);q*QP!`R)5BKVAt8)R#GzwB1=JLvMQr ziU@AfS9d1mmo|BL2bcVWE{`m`?upOhk5xwLJ^w4%<|H)D8u4OI&gAfe&AagPC6bMC zP${HM+Ije~S!lz$!hrAsC3IW)_^~MNnn@fDjs}d$jM9e;3d~%a)hv!85~nr zdk7VnjUEpQG2!D$1MuBP)+sdv(X@1_GSz`o|CmgCXFEV5+UPC%IFfg*|72L=<^uH| zk$K(ra?-U_L)$PPZBpx@!O@*(dJtxe^q`k2%Yww(DW@*#e8vR^tWsOv3PLA$8Irw6 z93$HMj{U>e7Z3_6|Ls`>aEOpnV!n@${d)X8#`47xO#~zL>RWu3F|Eh0Jw3$r?pji+ z-u;B@@!Y-C(-dYWaazr7%6W+^?uM46ZnVaEr42p$U10%KC|{~?^VWnXBQVtM4zbBP zss?$USm|{}U|?%b;KM7J-}ydW63hauBJtVw%_z(chqP$L+W7A>>gn?5(z_#tZ9}fa zc?}n}=Xj#L)6gzUiFW{thBWw3brqI9aP>>zHS3p01S?Q4Uz4`)-6YS5vo8@a3gS;( z%;-ujTNrk^gtpNZF#I4u@zi%Q*TKJG{Xf-OAOvw(ye2jBL8>;o5d?PWS$atIf?{ja z@b%fpp~5s|LQxAKEncK#NDv8{?uQ<7#ogvmBHI=kDn{}X${Rt^ALsdlXh^=r%h!*R zmIe$oj1d*;8D>7tx8~yldW_c5vT*p0@JOBaU7)R3nz*?@N`)XK?7W63)ANrU?d@FL zMB+zq#-zL8mq{7Zy4PfX^dHrViIOrhP5AW$75b?QZQ(ZMWf(Q(wVmCS188`usI##0 zEGiS=GTLUv%h+fFsO6#(Jc#{Rr=&1E z@TVA0Z&(U%s=%hyk|+>zlV`oIRWfeY<3U(NyLuy?FY|YwTU5L!_3vbK*iWD+VaF;{ zv{haQU_=(2cgEzEXNAP`3B_WAbNW~_jvK|Y?Vw_#5y{JeZ^6MmH5JU%<0y4Ca9Abt zHA+fpHH66ECL`$&P*y7a5+|ffM*#?}(ng)c1A;rue@xGPL*t$%?Yo&B$_QG9NFpTT z6si|NNS`uBoAY_-wG&Q|2(THUR=#qcMH{6LZ4-ZFCZiE+fR##$*H@D!N7>?e2KeNC zqQ~LwEz#MlGN^pZH|EWXAFvNet~(UN>_Fg4BiBen1_}uMw4rn**Nftyk*~+HJ-WBCGr6h=O}`~u zA4(ZKNb3ET$i*ad;mo><+Agw}NzzaOrGTgDUWOn8Pa)>afYjQ1%o4QXyfryfLtuNa z|73YM6K*u5NgQLZr^sTW@Ax&YLN3N;c9;pv&Sr?uFH|lR*5j#SV=;90>)Q9O&6}(t(?N*N+{D zoM}{xmulyC60ej`jn``8w9VuZniqPQjC<{7=07J!b8J+*+rjJQuM~jn_guMgqwW0O znc;5jYI=Pt>isOg?Y-bi@VMEcux;%Y`C8EVPI=8g))8}el|gxJV;k^0w*#U5{qHN4 zldG+HScY95B31xN%H`%xB>S`XdNTtBzssfb;x0cCDyDzG>*S`Me!its9#3=NNE;aP z3C8jkY_%R9kY*Q#6ziW1fGhBQo6!wg6m9V9hP^!(V= zAWzvd&Ch1DHk1t!j(bYYEhusj3|IeAJt_@4nXA{QnIQyJI`R?&9D&*|PLJ}(j*N!$ z;Fe6jvFhBYKDx=scuNQz^h^Kqg|TLC)* ztF>L_sVW{lqNy7PfEoLkN$jRL9846s0okXKP7snGLu76Sr;u=U;v<9JqOUCjYHM5g zCM8D%7Qvpe16>{ z1XOXoi`*szyTex}(s1{0fcwf&E^A z)I9`Ckq)nR&McKC1VYY8KkA=eliH{wHHdu32vLRnC5^F=9r!~pw9=AXB88oY1?@9uadp+{Wk*-2%4Y0t+j&zEPXO2d!kk>S^tSHbVQi0YDOH_0eV z;(vb`xZTngEEV?K>ygz8IO{;j5l46n4q4rUBp#ROOIT2yTi`Q;2bSgh zn{NWzh*0DmBG{~Z-|{Xz{*yF$ELM%21yjo-rmwCf&%wh zDrh|%dV?I_Zk-Io#&ttWZ0Z*O`0PIZ?w75&@^*{yY9g+%3LGG*sur-j>x;pV0m~;%)^mj`K!G-`6=9M6#XD;F8OMpY)TBr zIFv&+KQ61LG)~KP9DsXGu%wq-KWl$!@F~P#n9-Q_X#{MM>9t0hbSVApxR*t5?M zFkrZ(kBj-s`SA$epUDKsO{WtKcOQ@IuCI^pm(tXC8k=g@rUE%Sx;gyN*&bN6IkV`l zCw26+z0WZvj*CHAQX5cs*%(%=2$VXy@h?_{Oy@g=1xlN5liWM!Hje1tn3}AI*p|uyJ~vB0%D4?5 zA>IYfzJMhbdRPAYqk@B#=|A8XE8BmfO!j|~mHj2;_{Y$H<4(TR9*rh#LF&4vxueaU z(@G$n!OT?wZjV~@MF^H{ChJLMXLOvs=4pjYsT?am!ygSMin2IUCdZ0&^U`8YP5&ih zm47om<&8ONVpzE}zr9`j0F$x6%#f_fZ_fhLHZu5`=~MV>>MvJdLR~p?11`s%VXW@- z{CcVSv#oaA3N;DEv_WpZ<)=*d$BqwSKY_#`aFx|69OF=rl>`-NYOW+xG=r+wQMkfZ zBJU6s%lO2*f(uP+uFDZjN^u+SeqkSh*&eF}v3q{ld@0pI7`e&|Z6ow=XLg0((=EEA zyUuU(LDG>^79O8Pu}V`*f0EnWcQemV**5Qv2`Koxd;;P*GkiSrpkaAd=1Wc=b#KmN z5Gh}feA1?B(m*@p+&=^XOrysl)CDCE^9@goT>`WH=07*H6>M!$mSF-Fn4WCdXZzI^ zn;0#ciqfYpWKv*V*^X!z|g8Y#ASe9LRU4q52?+c(17Vdbb%O=kDViJfd5R zoDjNi$$TJ_HD4}i``NDW8(ut5zP{s0?(QjEZM|7?D*jp|kPgT2Kn|xLCSm(qrf1|V zpY}wM&P5RZQzW!`X|n&-BG-`qaPF_uM*;??I!{+V7DS2)`b0d?nr=vEozk79WtG8c zZv|W2F5Cdg-p8-|0j(7;5<4`lX+6VsN%gFBM60+iI*A@UrNt%R5(Bi`c?kk@fNV5y zwOl+59xEaMl@5Iod|0!g0oI3Y&fu6JCK6Ox>joYI#A!iFcqYH6HiA>mM@f4ok&_X| zAl*M`lS0jffRS10vd-b1`W>&p?{vbRTc2&6XW5cVvH~qaxqkmvL$E<)C{BpNG`2#e zi2tl>gU`FM7HD(qE*#Gi?F79RgAa3Q)sMCDDSnaFW_yA8qJC+DBeqo)dRNIy4oJ5%;4h;_<+M5|s{ z?Z4(EX;1tUmQQWx&t(aSYn?%LQn~t-;x6?|wNq96%?*2=xyNb5EaT^zTYNaot*w4h zRntR9w;dY}ir@t#&O6Wt)78#Cu3CHhAO6pC@6e86RLbW9Cz4 zg7g`?W3sQVvfN+0INL*iJ3e$cI-SoQb#X$&8gj=oS(ufSsGwFo8X|oDz-91*$tf-} zp`%B*de>VkXR;1jcxdg(`6R+5P(}Q33U*I35}XlSPxB~AmXjbs870c}VZ0CnCmJiH zu09nfGbM5EtW7k>rSoO>tLzY&)}yNz%&crk&@PM;Nlw=u-}L~e3%2w|WrK{dZ0ii1 z5o+{0*+8A|z|$n{Q5{&{m(YDgU)XALKW_h4W8&k=?##MLnvJwYqE(m-SQ2Iy zMD^;uvzgH?DAW#|klkd2rMe{L*2YW=A0Y0!LVZSUNc=M$oXW`Z1FU_Gqa#dg%!0;u z7QOMQcC?fxv(o`R_0dM&UM+Ki3E#B>J>@x8>v(bp>GbM~Rp?{!WoT^fr6U$s};SkL^vl(bWkp|f(|i^JaQq0k8ZK6*yB)eiBH<39S0 zl617}g-3p0PFmqh4k$@ud`|PH>o2Kv2G(PtrO0v#4>vLR?HD z_xtdgRXsu7dZX8eO+tom??vmHeY;_6=}=J36}Fl+=W~}39t&GF+zp0x+s;8mihgQ8 zRzfdSK;xTAbuJ6g6b{zwIKq541;B?IXTdHi-A7V|8}!WKXEuKag`#h7d70ddHn~1C zP!Ys{CKl3^38>MFzl|_rRuQ{IRIZv^ElTF9 z`(Ufu`2#vM>=nb8;Z|K3`YsM}aRHZ;`VbQnj97&&o3pSw8=I=I&O|8;La;-XNh*BG z)DiuB!;zLmJWf(xUxEXP|G`Ayk_(t|c}=^cnS`o1GtSX#E>b3`juBWxocJ**#cxB# z_)@~K1SbBx9=abQi=qMP7T=y)Z>{s|f_8BX{A#h&>B`K7s2kDvJ+ACo7IO1&i0%sO zH5N5)U6}gOrrP4=+}K(9X3^VPTF+r4OJ>wIvY=DBwcFU7iS-wylAz)umq`nqx|A(x zG;w3L*0_reTr4EQ!M;PoXuTIX6i9)Hg99v3!Rm5R-4d`5TKt5(*`^+PQN~OjV!0{p zvG|N;Y6AtOZODhpZwo-??U+Es>2J#MmyaT8EC5I#b@gM?;M=i?_|sMah6IZBr7Yg-)Q|9+POfWh{4+H0|}Da{7{T> z%P`Z&>5`&Y>(B=ft6&jPZyhwj$$@eG>pBg%+la$@6NSjC$n#B1VhKet%&`LoiN4WB z1B`8MzGf=*jb~sThFOMc4D&3U8Ys(ReFnpoQG~=7nBj2ldj03A&VjVPM)Q97r%2Sh$RmxEvM7VtUEEa zX6h+V$Wz{N_}Wksx8;ROmqtjDVUUnQB8w~Czf{s4c?+!?bKH>-4m)kRXZB`te~*X9Y~CV4GZ z<2tYCA?neUkv-4j@SJHIu@$ESob$Iz_t{H4%yS*5;$0-Rpz79-Ho{M!b3FC@-8A0< z*nS~voJrhSu!?H|Mw7y}y81$3tdBjeINI$P_p82>1*%y@xX6pjb*2!8wn?Ow7kguF z<^~jPv{X|UhQJ%nKG6DsYt4()mq)y8(Tp2u+0x(Dy%LfOwP6fbh#7ca?n@fqv6oxq zCZ%Fy((zKl>=S8-z!e0+wQ5ZuIY@F?ZbegDJtS-_$>RPdc3?|aVktPtYDC3YIis?n z1w>5G%Itn4TVZ-x+jG>qy4;c&8rsc27u%3(Bb+Q~qO&~FFf^{HD+JUHJ)PiTORBJ{ zsNg~Ityr?M=W)rAUYl}IpL6`Gqq$G$1(+(vA$75dUR}25HE3y6h!-{F3SNc1f!@mw z^@5PWl5jY)w?0(6Q>xFZb1kT^Qk@}^$=Zm!JP@ATY7{g;4Th%^bX4UU6AqbGHI!|@ z8QTBKY}~M>_A;|Yb}T^`lx+Z*lBiZss*h%IYq~DnS;5Hijv3af$nPswqctwA1}6KY z?6?R3# zYss$dUBL>0ddRFff=I*i@o5c5c-Y58`d~ged#SpoW8HE0G$-6%ydXz`ToNdy%W<=g+kb8C4y=5+$#A(D@$9mH`(e_*gyK`V zA8-$VvgdR+D3a_i*wsWE&16RXVY{cT2%gGJoeC)*-uiW5#f(7mg zjXmcHKaJzzuD9xyxVraPJdjxYB5eJo)*gb?tUrzC&}6>TdTdvUd-$ptf!gQ&=7{q` znJwVZf^dB1Nbey=JROv|cJGOXSZ}pEF3F#BK79f&@#8RSZ#UrnmgKMrH$vm<1!O(b zKIF7o;0bXW-))PHcs-}AW2%jAKGu`W@|0{Gpl}gDGT&=$k1v5Ws=)4D>gjyB#aHR3 zmwzF$S2!@UDZhtr@BDOrd0?fAysR=x4axrJYxc>?!A09J&7>OPfqfh zOl}Zp8=7u@FBIiqSAnXDt3~Y^=KKHqVCUHLcQU%L20%oP`oc zJ44-J^oPD803AgZ>F4M(Qsn6B3x{oF?b*RlEmqrzfdn%=8YW2Wa1{89q_Bi`Z>8ra z)>sW0p28K)x0}phrC$mPy%r5n%$|A@y5jMn8_0#oP$bXSb|k}<&-ZG-cc*FlAlTO+DsD zE=0r$a)XhEw^|8vgH3U+)iooWbyC)`KQQi&mS;m1T|PVjiU|EmoeD!r--_*%Fy?AB zmb0mNTz747rfiUfpO z_O*##?va3Aj@qT1i>rsEQEAM1l#TFwXS$cJguzP>NkftU=$RzfQpK1q3pS8s+5F%mr9kF_CncfxP1210&X4g>v zPtdkZjg@(Yn-To@{Tg3XM;})`*3v7jwOmK~ljic-#k=O0E-3P?!1><&d5o z5J)!&MTq zQ_YhT0>{{WkvE`+EUPMUh#iYGY4%D#;U-^R>N&=_f)&~|!Vd~;l7XFXrol5d2w!KA zqe|(Tr(?FS%sS7m29sEs=6x8T+;En8gG)?U&3d|c-8ZGc`2pvLtRdB=_F+Zreai#! z>hK%Wxs!|qTbSb$svu(5TNq@p!#yCgXa<`sJ+nv4K5RYD|=%m z9^&WreIuIoSCPf-Eu7j5tUFufYlkOEj*aW1-$L#vy<$u?&ROXgwP8RU#&6~k!^qR{ zx(_|RSBoc>l&ute>+GnXJQF&)xpJ4#Mv&O}r8HD;7-`Dvra2Md55eX+oQ+$28}Fx) z1zlC#p_dkA5&ew=d5@0Tbo!qSoI8_0L7^IK@&hRW!IbR{fK4uoaS+4Xmf5-;#^*uy z$noJ1+Vob*-@{-WNiV$KUkD!$9nk+B0I~lSP4)jl68M*u@PB{jhJT>3|BDrK{M}CY zcRlWtJQc#_ZV5f!F=*zokc%Fp z>eV+gi9Cfl3Xh3k{Xa?++BX%bRtfNaXl=KOESG>PJK!S!O!1y*y=i5XHs^{KGG zjiK6nX~+=lj8S2$Gl_#?!ufD@cKh?T`q`4;TFS7sSepEWx-8cuG>fep9oJ;~co7%{ z@f{5VdC%yPv0OQ*Nrq*eE;8i(xLXKmvnaT4^WNE!F`Tv;dE#Snj`aR)h{FEyaP=^j zAi&-`(FM1`1`Ka2U@Fzg41uAg;@G42cRK5(y257`(t?syWrC8__zwqRR1j7p1Hf(R zkqYtclPIY}%VN#g29)Kl-87JaE`!i#c`Z9qrQA$}R6jH~54HGrQ z1gR!2LHjrO7$?>tg59UR%oFP1BL{oLOt4e}b|!;fq+=(T2f2l$g0jVU^&FWE-E&-6 z2sq`_j0}3yWSQYia*XoznMR#b3(m&JVfcO)>8klgio8cTnWFR0c6%^U9ban( zt?=*H|A)JG43cbHyM?=K+qP}nwr#t**kxr^mu=hXvTd`=w%K3rea_zJ+&FLCI5)n3 z?~j}jx#C$fGBW2}kz>p;#*CPbGe%2 zBmYMB%Cy-~&N*e(CAxh)^qPD}$yYP80_P+jRSmbs@4wMtK~f1-rkj2aIAg8kJ84)WQLR|MTRKA=_DjIr367ZbV18f`{G&4_TB~c$}G-I z(123f&boi-_U91Bz+nixWB!yWm+ee6mRfPRkA}n;#|z^aSq2a=0f(cM2M$n=Mlm&6$!BRjo0uxWEDF@TSZc!m9P#RrOQgk9{c)9X1kgC z2*P~#5#(C)EYp0mmGzrc=<^!XPC!Ypjdb;O4oEhqjD?>PPdXX0!k-SGKj~A6=H<6< zAaKGLGuah++=J{X0uuw3f-;g0jgJFWOxeCwNjk*k939kGRUx&YC_iB+nbwaS}KvP{~?KzwhIq$es^P+0g`8fE0$ zN6C*!^N1BfA($*qR3o(`_Hxg_&3%_hJ{8KCL8vNA&`-CSS)n`0Hc5fS&j79u`HrtJ z9b~MMG7TIRbc7SNk+Ig~xA4=|NmYW7LSm?Gc?XX=$GqY#GrLsYVg83yX2!h^(7h5^j1L4&B;Mu?$nKX8&HhrHAXJnt~%vfsQ#x5>GR_Zl`^w!e``3iFQL* zGI%~D^PDw2MsDB<_eB3syJrS7AAx4r&`YMod{Zc`0IuzvupCLjVxrko)o1odH~7xU zLP~QYCCO{nHipe0JSo5!&)hjURfB~Tm=t#<*(RzPn8QTjps`w@t12N~x|rdZV+O%$Dw}A4*haAj*L@aKw57OPI zqfY3IrMWHfsrLq5-4-m)QOyFWN1!eFc_c$Mp0VI>fRmZLp(l5eQp40^=7nckD8V)$ zrK3qJvXb(|*%`BEi(55QQg_F$9Oc9W-yBJ;uk%^x%QeUo(Qx0M{dK% z6c@r@P)1RV9Fmw1*jl{IPR%<9j9BtGhbrf#A!Y0v6M0GDmvauDBr-+DmO8rG=kHQ` zYX~W;bIf`L4-Vs>+8GY!cxet@H8E|3!+GQ3G|Gr;OQJqCvF69ZMXXfHz$TWE1C(-M$b8iavm@XK8|n@Z)#($5MuY)+?XXRVx^Ek z2~A6|3ErJsN}M<-X3>xBHtf1~Fyjs(mRg0$`)gzKr zBLqwt%N5+*4|ImD`gR1(y+nfcuWKA{A{$z7Hfai96& z=^`z-`t@O(we<4%&BK!LG#M{t7H2zg1t$F(+Fu2$=M%RWjLWV9@+1@uRw^?@Yje+b z#-X_drm|w)VW~hL**8zxSK>%Cdj?r@c;<^a8DVTE^wM*#gc>px%-JBoBL_x;W?_lP zV0LowWQ^c5Q)8@QpCauOPGL{Px}1?rX;;du9BVqWvZgadz-kU}Rr24M)sy97kjG9y ze#Hn5z9-$bd!k=UbSXzNmuGdu)`cpK+r$A1u8$R_MS(npT}xs_mRKM2aNbMcZ^5tf zAodT9YM+W?U)+}8#hgNu^vwbo~tagQLH%Lyzld#Ru0zZy z)XCjpmo+)sMvXx3(IljF#A@@(=6g-)9pllXiD~#VJlQb;*!Ks~Q*Xpx{_jss24@|1 z7V~}=5sa@r0}T6+GIg)$Y%GW_)FG!n4#Ukz%K#(^{d1e%5ekt`*hwc%b!sPPEy+i#;0y&vO5|{w=o>8Qrt7G5#BwWP~r0(6k3`CmF%h%$m%bN;4|v*M$OT>>p*sj zZ2eVDS=CeTALv*#=zyi*rnd6F>;QvRj=MK#YPz&Sf=>50$q`JefELg~9XS5NFF5fz zy_yH@iA5ePKDo0dwXM4eS-P|XNbs11*JXZX`+I6(x{*bo#JNP_iV5=qR4S~B{UCHkdN4CeFhUO@ zv#53}d=eIt-kd-X&Tom)(VxQ!B?OW2juAB-@B`(p!dMCFc~d*8wYsaI6>*DtA~WAb zW~fOC&49L+;v6$a4q3TYNWeHnb~a_42NLn+D4_NE(aP`HLfsXTabdf=6MYH1i2+^Z|VOmb3FZ*;Yx~^`J!nj{5QmJR^Ye&7`@g{5YfkO`K z$sTqNq)dx{?x2EmQ7OL^qLroE?n8&iE(xDbE6 zZt%{_2!fsT;bhw5QUD7effLYF%Mp(@b7rDRkU)&Iv?kZDZW)#bZ`0So;c`UHbC7ey z3Le4ZsDw4Q_7*0QYDwP3eYG~07+bjB6~e;{PMnS}!%I`2fG58gsgtG3An!$=R($l> zcMK~nJ`R^&89D4YzY7O5LcyI^78thuRJiG?9>Y%o=V7@-sx#82ldZ|%so{pis+xm! z;j$wneF+e1V)lP5m#=^Xba3BRXQLOIAX@IZ8g+*N=1zb$ZMmeT4<1(OF!w#20+K>w z#s{C~bb5hPLSt594%_V;jk~L&Nakz1EO5}@Y@e0iw&$82q)|4dP3~gcrxkBc`&wz% z2h`GMX*w-qcXXpo(O#N-4)M6nD7bhYg4~S?A&b|&putTAENa;R?sW0a*E$(S{JKuM z=#A|T&nDqs85C9&;r!sb-XK1UjXd?pqRYrDWmFtW2YQx1%$JJW%T~)qm^qXfvCLHu zBO&zKW59}sIV$;SbE+<-_9PRwrynn3pN+2-4iPonfHd`8era1ks|z>#CqIQGseadls~+V7?E_8i@oaG7pg=HJ0f{kqm6VG2{V+9uNU2< zZ7=M)dvf%~>b)XU;=5*%n;IoVr;NKvYFiOBotu2LE&DnHF0lMyH79z&Uzx1HV<9Uy zplGpg6_|6hwm?<@JAQ^P*7xB(w-%vq1y8Wz7dromx0^==f6;X$^tgHFeb=Zj%9e2nE<6b*tO99QPmF`ItN)C3&9 zgRu_n8>Lmora{qqtp2N=%Dd2G6VJZ(Hnq)~gWITTpeIj5=ubvAZ4_a(j2?Z>M=b<8 zEp0rWn6?M-cwtLyO;A!lehDg=gS3k!>1EAo{3k5 zLv9unCJdkU529#E{PE&L{A~K;jY>=BD-b(n{te(6ZyD*~{rp#gj2v>7UQvb9aktef z8Wo)+gs97HF&E^E^hBxc?>4*3rjF#n1bEIVr1u6#SFd==N_A}3m)5Q$4XPW|MCK@Z z{uEy94P9XE^GlU?3Qh$G^KU}}axCCw&u8X}0s~j+i|cz3_LeU4Q?GB_yXp`e{;@VL z_gGh6znbUW3(a;PD#JXt&N&*G=q+<3m6o3$oaWUwR8;hoM5;j#CsA5p_W4*BG%lkl zGnOCqXTM@Rtmi3&@>x@ubZ^NCCA2;EkrUMvVLi|p9WZ6dO=x^MMN;dMG4T29MA0xl zO(nu7@*Ur{Y+_Drn?Q3h`##_*-PtBg7|r}!3bG0uVOS-e-f%A0py6N}>9H0E$F#%d zH%CH;T=10zC1c@olxy*Z0l3Gs>7%6T%BGk>?upsKX5!qB@vzbhWda^ij*)Q*BicW*?U}nQkYKriIC85rS0oIfhZx zq8@*^bgzC5Lzsu7P1tXbKV^i4Wf)i{qxFs$30o{lTTT}miLw@p%hKonDrMhA_bKad z_0SVEPEopJx5SDCpO|JLGiaQM z$$-)bN#GsB;bp<1M+;UIfqbFi|E2Q&sX`2O@95!!OaM;y&GXgQIF1T)r)a6P(({$n zZ!<_3Bp3XP$;ml9aDR3e?g_}XUOP1O7ZAbR@AM_k#VxyJ{zq>m_=bYq67ALFzf&@K z8X#~(?-8@(03$gFyH~p>7*1ZN%q54BDzN&TybvAWU zkXTUdo@Cy}yy-!T_b<_DmPnL$k@uq%**B=nS3bHEc{1-;ib1gQ!-Vci+&A~J4)9b2 zwLr@CWUkEo&}twnb3YK&)eOmC<BhbYN;tMNQ~@`?ON1Ra(z>3%k65dDQ>>k zvfqD}d<)nDuc(_7Kwb*na@TyhzCuxNfK#_2NU6j8edTebJbtiCM)QJQGfRqUh!^^g zDx1L+qau!*uR>?dLT8gE^Xej3ZdNj#29_V}mO)ZwLwwL{zpE@i#zU5#g4u0@*iRGf z@xkpx9KL2gFqG6>LD_Ar;0MTa4GUBXSVG#j=-jdOH1P)w2T-+tC*57voa0Bfu{I!@ zbeKsn_KC-mUpbQ^@QHa~&4ka|(ult8H0edGU!>#|P+;V%jgT1hpE`;-;!n2W^dslP z4W>%Hle8cm`)ba$7<=NyUKB;e-5Ona^wqamVY=i@EzFf}RzL_jmXwf1RADF~fD3JX zJy6z=-p$?b?glzW+aJ+;n^+xRn*4VPv}S>9py&m4;Qg)HZPzQVjVs0@*3mU)+c71( zKT5|B*e*?TG|sXFx|B|@;Ak+5yjT8h!>`h*zdMfQ^dTXo63bFH?@1U;%5`SwegiVr zrr^{5>JG%;AKg2}3VVAunW~1g|EZ`rrOuvV&_3nmRHZ#`c#1v|c5)`c2mf0auwrkp zheDIR^lYrA4~I#Ur@3*|I(qer-B%y6AekWxSE87-zI$GCDQR5hoO#POUR)fh<%1be z7@vHn$&6$&x}6w5gu|SSh9gCF-5;=Lb6hB9VGI#%kPY}4pB9cNncnY(9!%l0-rF&i zKhZ-Jm>1$Wk;mUInjMWVV$0<7pq4Ibtk+W?QT0xBF5UpVkq=Y&A!C6sQ#&ixjm4;M zrtsnJ2v-OH^ajZsbIrZt7tYN zl`I%@&*0RYL!D|%#`P}il*BK>Jne2$?{kMAL;nVLuqLow++mt ztFV|-_>>_oo?j2-Z*r_ORmvLD*FSWxjY4+Xj8i#$3OXX#jBz$9OI?}Era>scL&a|3 zC{g!qlyUA1`~=OdpAr{G%;*BF70Ud|wtf_m={2}zsULXTm3}3j*ydOcDvggzBOc~% zG5h}5E@hjxHZaDIDPV|Mu>6?U>zhl%95GSQ;;&2|*E~2dSPi+xv6u%*XoUW@rhR6^ zaORapQszCmECwkzu_Uz+&QwiR&YKRA;oeCsY8)m>)~#5hDViyA@twN>H<&KlIMZSG zsU_fQJd%o0PgF_RNa*E4Y-Wesoe(P;W(*xNIBm5(aw@g>xmFk_ON@R`y~eo@?;Mo_ z*!10z)JeUvLrq`vIniayzEsPKJtC`}iWiOQ)h-rvi{hEswBi&Rt!?>y zD!e_)F0`~gI#`qe=unDEUy9hX?u*2&7H(kIlxLuzbETWKpbjp)EwjLQkW92yllG3-U8Nl0!AB>7$26B-iM<|%zx@He%#(d6R9?>_E&*X!|aEZs5j zq4g}ifSH6MCvn4TDJ0Za$-q}-ysNdIQ3}`}{;)X6X{`+3C*u|2NXCNq>0uIwyKb{( z$MnWn_;Ya7F3Z)&%I3Hg_a))7@z>?eUNhj?k)!RTa@xj0@9rRXX4*a^YiRopl%pVY zFNE~wSqg1Fb`R!6=tzX?vg-V+Czwo%a}43^vr*(f5>Y4}&Sf$wlQjpcV*KZ&ioS>5 zjacz$>k($0{6vr8M(TgE86J9CI+@@hXae0`Vr<$OuX-OBEF0Q-1)fV39A0H%{{gCn zmm>Wke}HTpjV#B9LfIgMIWJo7UXv*zbFhhuR|~C7KE+sDTx$x19tbdPWZS))zo|zq zDAjhHgJALa!Kc00f8#9P5zJ(KTo5KX8ks+;8J&0x#HaRItnff4hK9LKcNI_sDY}ST z6w6-(EkhLqEoXc-B8E9D94;L@y^P4za^@>d^7cCT%nQ03DVx2jF~G8OW=WD6rH0b?+dUI@hfR(` z{sAh(*lj2Y+U3?+LiSI9&r*BN`pH-#d@(=2!d7}&PEFODlAKFH%a1E6#^JQSE7J$> z_>EMnv1&L$-lA(gOrMXA5e~D2BjL$K+xH3Bt)!)8olEl*Q9Yv#dE%wESKRxIV+{-H zD-+-Jm=A1V$8u%!>~eD&D7vXAO#LIL5}cUBW~(87^;&}QQYAH976}`d)rmevHQ$1J zFaFoC6u~Mf#UF4vI_-(;&21tcHL1HeROR&73}gE?f%x^RZ1gT0P1mHCnx3cbb+QTr zHLcf;Gjtr_9YzImx|8+so*(t`G;$}6nW`S!F*4#D$zkM_>gr*qpIPn3W-pprIl9to z(=>3J<4kq_tguNEmWOY3KA{Kk5>;DiM82!w2(0^VT=C?G*T-s}(h<*N={HQ8Cr&By zS{PI-J9~9=la~lO@cscM&q%y4_U%d9nesAdk>ZV3rX_w#)60!f`UKi8Hi3k|gqTc^%{#lmbE z6b0whm^xH=ZZGIfNSBbKZ)-7kMcg=Sk#|RIZmtLTO`i9A1U_B(il!c>bGS@4=D09`JabHBlsrSCFuq(j&$}oQkrSJA$&;J_zXfQY&>g9l|P#rot<7i~b zg0k0kib7>Ai33<`I_&5**N&xZYpUjIyBYR+n6RQK`FLbWicj_U!g`#bC$pEt6r)9l zX=XEs)sQwC2>S?Jl0T0#QEsw@&g=1IAMfyh$_2{GL+lEI_%w|We+M>HvqhW%)_m$N zyv<$G*1Ia_0Wn-zjQx$%B~BHaUOYX)R<1#fVU_%BJs$2RMByak`#$MXqxn*mzTH-) z!NXi>_^S1~LqdpqLP#Wo25;rOgPkjM+>cimqfeu2iJP15OaA2i|NQUH!TGNf;Qv?T z`+pIYFo^#zjjH|!s`)?QE+^w(W=>|dztImT<6i+(|C73e`S0FwOl-enr~U>$JHON2sr@UVx$qGM%JH7t6Uxg1*U7PPOMn=p+wsKP((@e06)$%DLcTE--3UERm6qWPu?p-@A&1i zY0l@{p*&UO$dz?jwE)tZvz2MW(i0l3pSRt()b6(5KEdB?U&H@%33L8! z`efn!9pd#*D<~(^U%t72lQ#UL^uPFq7jZ|c_nU8~en%Y((hSOx1`dHNWB_ev@)AOj zl#sF`PJmwP-oBL)M`aXlZQVy3A*^juM+~lM(I#(QzY#+8`18EpiUYPFjzazY@NeOEtoT-Y)*0HZJ|XP~9}q)872s|8S0zXuh}*b9ieOR{CE{!e zT=k~I;L0NxGlJ+&IrP8zJGN5y$6y2fN&pUh@`vj_2b#KOr|coviN*kzkP0Xm+Llms z6n>tl{o>eX<0852DWG4n?XXMG|E7RfCKNM7%`>AK{>vH2jZ?-E-h3}^4hwz<^v*0E z`I_S=A6?PV*WuA#qam5S12G;&iB*0D-7cto+c1TB+@U=MHhCvvn?S^BO$|oy_i2Qr zhdmsC%h$M(6Ly5~MTp6uM>r+?{hP*g6N?K)lR-M^Tt*wnLL#RWYlwZCQ&u2bgFA2d*S$*1^1$W+{M>=8-<+3y^e5 zzS$%K<}6Wx)rb%;^xUu`8Z)nVB2;T;At(4(?|gc3OwU-1=UEXi(03q`e1f`Ox`d}A z_>n#iB>{<-6lTLvqK#F-VTQ9g`umQ2%7dc7yDBO7F``4$5jjsP%(q#jXsnf@)YR-t zq9ojj%wRDP!r(>{M&dxY@+bD`A$8f?V(^|+-@s~vJPM#om7&8JU#-$NPT$5IoKl<3 z7qG9j?(yxy7=GdVeF*7x&WkRCUI1U(gMO6#d3;@>V4K(KZA3&~2u|LRYT9QxMF(T)Ktu)E=ja@BacQDo&| z_Ly%$Cvk00?dr?Nl>p}E-ZDxxUrn{f94hY|A?z94XYk12VW%I)PpT7YktNUvJX@ia zJgU!qVYm^D8*~TkYSJ6Jnoac)w*>~t9%82LioY}|La)3?#$Q=o1DAdl_ekes7)#6_ z1aTK$s3Zck!=lVNtQs+d*OP1kke0A3{7Iid<|@87E_1xs?oF!5mq!z)(S+vGxLnCw zql`ii$jZllYE3Q(0>R3xx{N9#=q!<_6?^m04jAInCVfJV(g~GNa_fkXq<|9G(I@ewvh0xO*r_s zv{(w+h!-c0vs_ntVx}s?$CTP+N>GXHhtN$tU=TE7qg4?n1D3E{hP0=o{LUCVXT)#C zoFDDM+)YDrC@vhuX7{vGF~mTPU&>5`DbjUOGk1tD_q#aHEYM`QkY<@B|JqO~ko_?n z$Bs6Rd7dIGbd4#Z^M)eN;)?lXXoKW&n^mIpp0&OiIW^;WkxU`hH}=X-c$_%u?TXuT z%URhmXMnW3G1O&?A{*t0Kz@ByLl?k z7dD**MO@j=%ysU};x6;j7^tqtgCL)5ye9^`+5T_@*9=bc-Iw+V?7H$Bu3Y5=kMtTu zgWXGnH;v;>HqN~*j$C;UnBFP#?yjBVc7nU5-Vsu3a|fazK8A8ruC(MM^kMfPR^^_W zpp|;|tw+vY8!<_oe79}+-fIV<*+qb|xN}3B5xEo;4n2AjopxPx8xX27)WvfgI1a%j zA&@d^J4Ezc&z^(wPP!7UR@|4Am8}E5&apgK*CXeAVeOQ80{w%5;&H3R5+|w*2eF>G zpnjfuVNM?OAkYO0NTrxE8j z>|ZpZks&Z+iJBbpno{0L1kKVlT%K;C@{3V(Vj1BKxZeX~NSF)V1~OMHay>0CY^rK= z%y+_PFSHdl7O7vWsR~NkL)zVJOG(Yao$FPofxc;HDeQbH9Oh0`wK@(8a^&sX z@rl1ZKLO!B_DLjTAa|r9N&RB|@Nv+hNk|z@X)?s9Rf3~Oyhjh}&|0~z(Y*MRBen20 zT9`-K2%29bffUbW!sn60Nh%iFR}~)W$Wuk1-UpvqCE`);92E~y&oO*d47I%U*|!{F zZpKGj^dX_xer3W4)9~pavoyN+doX$lBAt5qJgTgFH)lXTf~s^x*TK(h5_;N#XhxQT zenomjDywo+z(~3&|H-vA9g>7imUPZAqB%^ zH{)OU9ocnY!&EblpU~iiw9noFM9l-$Jic7IvDVu*VOt4)vc1ESM8#N`eck zB$KYL4PN(-P zBl$Ev8kZZB%;bpdz7Mg_!c6YyNT08aw;COD2c`~FXI#K|3|cB`=lhlsd%1B2HcK$- z_bP!g-XUd?Wf5N{isyh}`A`_}%d#${AB}xComidk-8Dxs>hV>zB8;|ZC~}}?*j3K42)oWv6oVwJN;MW^bWD}0i{Bedwdm$t-< zRED)?=9gTO&2rz!42N8W)mHF~2fn37TEbHzrSO1ziUXH-SXuEzRPv>;vc?l_AjF4vCDZ?$JE~!;ds+n zy201ru98Vt!FGU^zTb6>xGz!)!zomP9HG>W3^4ZBnGc>9lR7KQ##+uRYRR#lG&Ha2 zUgYmUs%95ai_SU^@*q~7n#{mV_g*iEd4$SLN!$ueM{o{03NqtXor6rBNh_A6$jax= zT8v#jAvbmJp>XY^TN=JRwk)LXyuSbv* z34@x|)>#eE991u3+5WI;_iF*8Cm0wyJS@-6+Wzs{ksUf)IT7EjiT2D@W)>5EAYxvS z^Gd(w+ddnE`_RRvh&3epV>s)B-PW@v|7MOp6(`@@qSH&;u0sdd(S^r;Cd5>?XQ2%^ z_|2U0`Mbj}phI{a?MjaxYIz{W)^qs~u(PH7q$NO0?TT#gT{)=`7-%m2eLR4z#NL}% zZLUL~xmuGx1EPy9Wz1@Xd0!~HVfl^#Go`CR